{
  "best_metric": null,
  "best_model_checkpoint": null,
  "epoch": 23.24228866922803,
  "global_step": 400000,
  "is_hyper_param_search": false,
  "is_local_process_zero": true,
  "is_world_process_zero": true,
  "log_history": [
    {
      "activations/layer0_attention_weight_max": 2.358670473098755,
      "activations/layer0_attention_weight_min": -2.5588765144348145,
      "activations/layer10_attention_weight_max": 0.24698123335838318,
      "activations/layer10_attention_weight_min": -0.2107042670249939,
      "activations/layer11_attention_weight_max": 0.18772786855697632,
      "activations/layer11_attention_weight_min": -0.21351349353790283,
      "activations/layer12_attention_weight_max": 0.19652807712554932,
      "activations/layer12_attention_weight_min": -0.18164800107479095,
      "activations/layer13_attention_weight_max": 0.17740829288959503,
      "activations/layer13_attention_weight_min": -0.16738305985927582,
      "activations/layer14_attention_weight_max": 0.17649109661579132,
      "activations/layer14_attention_weight_min": -0.15650181472301483,
      "activations/layer15_attention_weight_max": 0.14879919588565826,
      "activations/layer15_attention_weight_min": -0.1390516310930252,
      "activations/layer16_attention_weight_max": 0.14123845100402832,
      "activations/layer16_attention_weight_min": -0.14440786838531494,
      "activations/layer17_attention_weight_max": 0.14805112779140472,
      "activations/layer17_attention_weight_min": -0.14681634306907654,
      "activations/layer18_attention_weight_max": 0.12785644829273224,
      "activations/layer18_attention_weight_min": -0.12465266138315201,
      "activations/layer19_attention_weight_max": 0.1104302778840065,
      "activations/layer19_attention_weight_min": -0.11410689353942871,
      "activations/layer1_attention_weight_max": 1.2565312385559082,
      "activations/layer1_attention_weight_min": -1.2674603462219238,
      "activations/layer20_attention_weight_max": 0.11947087943553925,
      "activations/layer20_attention_weight_min": -0.11438094079494476,
      "activations/layer21_attention_weight_max": 0.10100127756595612,
      "activations/layer21_attention_weight_min": -0.11186546087265015,
      "activations/layer22_attention_weight_max": 0.10962128639221191,
      "activations/layer22_attention_weight_min": -0.10334280133247375,
      "activations/layer23_attention_weight_max": 0.09912462532520294,
      "activations/layer23_attention_weight_min": -0.09901215136051178,
      "activations/layer2_attention_weight_max": 0.8011202812194824,
      "activations/layer2_attention_weight_min": -0.8148130178451538,
      "activations/layer3_attention_weight_max": 0.597457766532898,
      "activations/layer3_attention_weight_min": -0.6321369409561157,
      "activations/layer4_attention_weight_max": 0.5122597217559814,
      "activations/layer4_attention_weight_min": -0.4911785125732422,
      "activations/layer5_attention_weight_max": 0.40989625453948975,
      "activations/layer5_attention_weight_min": -0.3806634545326233,
      "activations/layer6_attention_weight_max": 0.378837913274765,
      "activations/layer6_attention_weight_min": -0.3645941913127899,
      "activations/layer7_attention_weight_max": 0.29784512519836426,
      "activations/layer7_attention_weight_min": -0.33218201994895935,
      "activations/layer8_attention_weight_max": 0.2752392292022705,
      "activations/layer8_attention_weight_min": -0.2636415660381317,
      "activations/layer9_attention_weight_max": 0.26574862003326416,
      "activations/layer9_attention_weight_min": -0.22466714680194855,
      "epoch": 0.0,
      "learning_rate": 0,
      "loss": 11.0479,
      "step": 1
    },
    {
      "activations/layer0_attention_weight_max": 2.4784631729125977,
      "activations/layer0_attention_weight_min": -2.626638889312744,
      "activations/layer10_attention_weight_max": 0.19204039871692657,
      "activations/layer10_attention_weight_min": -0.19528621435165405,
      "activations/layer11_attention_weight_max": 0.19321861863136292,
      "activations/layer11_attention_weight_min": -0.2233302891254425,
      "activations/layer12_attention_weight_max": 0.15731504559516907,
      "activations/layer12_attention_weight_min": -0.16954723000526428,
      "activations/layer13_attention_weight_max": 0.18145857751369476,
      "activations/layer13_attention_weight_min": -0.14016646146774292,
      "activations/layer14_attention_weight_max": 0.16138094663619995,
      "activations/layer14_attention_weight_min": -0.1394677609205246,
      "activations/layer15_attention_weight_max": 0.14026974141597748,
      "activations/layer15_attention_weight_min": -0.12012164294719696,
      "activations/layer16_attention_weight_max": 0.09962975978851318,
      "activations/layer16_attention_weight_min": -0.11533737927675247,
      "activations/layer17_attention_weight_max": 0.09595400840044022,
      "activations/layer17_attention_weight_min": -0.10684633255004883,
      "activations/layer18_attention_weight_max": 0.1022835448384285,
      "activations/layer18_attention_weight_min": -0.10335562378168106,
      "activations/layer19_attention_weight_max": 0.08432042598724365,
      "activations/layer19_attention_weight_min": -0.10089582204818726,
      "activations/layer1_attention_weight_max": 1.5942950248718262,
      "activations/layer1_attention_weight_min": -1.4146223068237305,
      "activations/layer20_attention_weight_max": 0.0908288061618805,
      "activations/layer20_attention_weight_min": -0.08112384378910065,
      "activations/layer21_attention_weight_max": 0.07721935212612152,
      "activations/layer21_attention_weight_min": -0.08547916263341904,
      "activations/layer22_attention_weight_max": 0.052305202931165695,
      "activations/layer22_attention_weight_min": -0.07268647849559784,
      "activations/layer23_attention_weight_max": 0.08059562742710114,
      "activations/layer23_attention_weight_min": -0.06371283531188965,
      "activations/layer2_attention_weight_max": 0.9946755170822144,
      "activations/layer2_attention_weight_min": -1.0840190649032593,
      "activations/layer3_attention_weight_max": 0.6604662537574768,
      "activations/layer3_attention_weight_min": -0.6460756659507751,
      "activations/layer4_attention_weight_max": 0.6274613738059998,
      "activations/layer4_attention_weight_min": -0.63689786195755,
      "activations/layer5_attention_weight_max": 0.434785932302475,
      "activations/layer5_attention_weight_min": -0.4424303472042084,
      "activations/layer6_attention_weight_max": 0.4062999486923218,
      "activations/layer6_attention_weight_min": -0.53136146068573,
      "activations/layer7_attention_weight_max": 0.43087905645370483,
      "activations/layer7_attention_weight_min": -0.3100411593914032,
      "activations/layer8_attention_weight_max": 0.2568051218986511,
      "activations/layer8_attention_weight_min": -0.3213740885257721,
      "activations/layer9_attention_weight_max": 0.2475859671831131,
      "activations/layer9_attention_weight_min": -0.2645859122276306,
      "epoch": 0.0,
      "learning_rate": 6.480885262242102e-05,
      "loss": 9.3049,
      "step": 50
    },
    {
      "activations/layer0_attention_weight_max": 3.92759108543396,
      "activations/layer0_attention_weight_min": -3.5501081943511963,
      "activations/layer10_attention_weight_max": 0.7911906838417053,
      "activations/layer10_attention_weight_min": -0.7402631640434265,
      "activations/layer11_attention_weight_max": 0.5764599442481995,
      "activations/layer11_attention_weight_min": -0.7869343757629395,
      "activations/layer12_attention_weight_max": 0.739359974861145,
      "activations/layer12_attention_weight_min": -0.7386543154716492,
      "activations/layer13_attention_weight_max": 0.4789051115512848,
      "activations/layer13_attention_weight_min": -0.4728120267391205,
      "activations/layer14_attention_weight_max": 0.8282183408737183,
      "activations/layer14_attention_weight_min": -0.8925220370292664,
      "activations/layer15_attention_weight_max": 0.6684467792510986,
      "activations/layer15_attention_weight_min": -0.7914614677429199,
      "activations/layer16_attention_weight_max": 0.3807885944843292,
      "activations/layer16_attention_weight_min": -0.4368245005607605,
      "activations/layer17_attention_weight_max": 0.45317596197128296,
      "activations/layer17_attention_weight_min": -0.46363529562950134,
      "activations/layer18_attention_weight_max": 0.4421858489513397,
      "activations/layer18_attention_weight_min": -0.5154150724411011,
      "activations/layer19_attention_weight_max": 0.3590225875377655,
      "activations/layer19_attention_weight_min": -0.3894610106945038,
      "activations/layer1_attention_weight_max": 3.6293458938598633,
      "activations/layer1_attention_weight_min": -2.883967876434326,
      "activations/layer20_attention_weight_max": 0.39108872413635254,
      "activations/layer20_attention_weight_min": -0.3849237561225891,
      "activations/layer21_attention_weight_max": 0.2902712821960449,
      "activations/layer21_attention_weight_min": -0.361822247505188,
      "activations/layer22_attention_weight_max": 0.2343916893005371,
      "activations/layer22_attention_weight_min": -0.26114046573638916,
      "activations/layer23_attention_weight_max": 0.4041679799556732,
      "activations/layer23_attention_weight_min": -0.3835991322994232,
      "activations/layer2_attention_weight_max": 1.623010516166687,
      "activations/layer2_attention_weight_min": -2.015552043914795,
      "activations/layer3_attention_weight_max": 1.5219651460647583,
      "activations/layer3_attention_weight_min": -1.3555163145065308,
      "activations/layer4_attention_weight_max": 1.257572054862976,
      "activations/layer4_attention_weight_min": -1.139153003692627,
      "activations/layer5_attention_weight_max": 0.9807852506637573,
      "activations/layer5_attention_weight_min": -0.9666504859924316,
      "activations/layer6_attention_weight_max": 2.535712242126465,
      "activations/layer6_attention_weight_min": -2.0701539516448975,
      "activations/layer7_attention_weight_max": 2.0309841632843018,
      "activations/layer7_attention_weight_min": -1.4368311166763306,
      "activations/layer8_attention_weight_max": 1.3820106983184814,
      "activations/layer8_attention_weight_min": -1.2985831499099731,
      "activations/layer9_attention_weight_max": 0.7065513134002686,
      "activations/layer9_attention_weight_min": -0.6748973727226257,
      "epoch": 0.01,
      "learning_rate": 8.055800524842374e-05,
      "loss": 7.0625,
      "step": 100
    },
    {
      "activations/layer0_attention_weight_max": 4.932689189910889,
      "activations/layer0_attention_weight_min": -4.529543876647949,
      "activations/layer10_attention_weight_max": 1.6529525518417358,
      "activations/layer10_attention_weight_min": -1.1427292823791504,
      "activations/layer11_attention_weight_max": 0.9313106536865234,
      "activations/layer11_attention_weight_min": -0.9786636829376221,
      "activations/layer12_attention_weight_max": 1.614823818206787,
      "activations/layer12_attention_weight_min": -1.1710103750228882,
      "activations/layer13_attention_weight_max": 0.8707743883132935,
      "activations/layer13_attention_weight_min": -0.7555161118507385,
      "activations/layer14_attention_weight_max": 3.2361929416656494,
      "activations/layer14_attention_weight_min": -2.3720836639404297,
      "activations/layer15_attention_weight_max": 1.375784158706665,
      "activations/layer15_attention_weight_min": -1.420897364616394,
      "activations/layer16_attention_weight_max": 0.7853119969367981,
      "activations/layer16_attention_weight_min": -0.807330846786499,
      "activations/layer17_attention_weight_max": 1.43440842628479,
      "activations/layer17_attention_weight_min": -1.1936086416244507,
      "activations/layer18_attention_weight_max": 1.6731561422348022,
      "activations/layer18_attention_weight_min": -1.5563627481460571,
      "activations/layer19_attention_weight_max": 0.8667487502098083,
      "activations/layer19_attention_weight_min": -0.7975984811782837,
      "activations/layer1_attention_weight_max": 8.743864059448242,
      "activations/layer1_attention_weight_min": -5.1735076904296875,
      "activations/layer20_attention_weight_max": 1.0193678140640259,
      "activations/layer20_attention_weight_min": -0.8271754384040833,
      "activations/layer21_attention_weight_max": 1.2532367706298828,
      "activations/layer21_attention_weight_min": -1.1078239679336548,
      "activations/layer22_attention_weight_max": 0.6369765996932983,
      "activations/layer22_attention_weight_min": -0.6114957928657532,
      "activations/layer23_attention_weight_max": 1.4649327993392944,
      "activations/layer23_attention_weight_min": -1.1224498748779297,
      "activations/layer2_attention_weight_max": 1.5262031555175781,
      "activations/layer2_attention_weight_min": -1.685072660446167,
      "activations/layer3_attention_weight_max": 1.7556161880493164,
      "activations/layer3_attention_weight_min": -1.4867788553237915,
      "activations/layer4_attention_weight_max": 1.6851234436035156,
      "activations/layer4_attention_weight_min": -1.351974368095398,
      "activations/layer5_attention_weight_max": 1.6277498006820679,
      "activations/layer5_attention_weight_min": -1.2611777782440186,
      "activations/layer6_attention_weight_max": 11.62731647491455,
      "activations/layer6_attention_weight_min": -8.23647689819336,
      "activations/layer7_attention_weight_max": 7.319229602813721,
      "activations/layer7_attention_weight_min": -4.435965061187744,
      "activations/layer8_attention_weight_max": 4.602941989898682,
      "activations/layer8_attention_weight_min": -3.4044454097747803,
      "activations/layer9_attention_weight_max": 1.1935982704162598,
      "activations/layer9_attention_weight_min": -0.9657672047615051,
      "epoch": 0.01,
      "learning_rate": 8.884661472213505e-05,
      "loss": 6.5022,
      "step": 150
    },
    {
      "activations/layer0_attention_weight_max": 6.441483974456787,
      "activations/layer0_attention_weight_min": -4.980737686157227,
      "activations/layer10_attention_weight_max": 1.701425552368164,
      "activations/layer10_attention_weight_min": -1.5501997470855713,
      "activations/layer11_attention_weight_max": 1.0936222076416016,
      "activations/layer11_attention_weight_min": -1.2125530242919922,
      "activations/layer12_attention_weight_max": 1.8249903917312622,
      "activations/layer12_attention_weight_min": -1.197891116142273,
      "activations/layer13_attention_weight_max": 1.0343906879425049,
      "activations/layer13_attention_weight_min": -0.8792265057563782,
      "activations/layer14_attention_weight_max": 6.434030055999756,
      "activations/layer14_attention_weight_min": -4.030810832977295,
      "activations/layer15_attention_weight_max": 1.9589064121246338,
      "activations/layer15_attention_weight_min": -2.197458505630493,
      "activations/layer16_attention_weight_max": 1.0815229415893555,
      "activations/layer16_attention_weight_min": -1.1983228921890259,
      "activations/layer17_attention_weight_max": 2.399986982345581,
      "activations/layer17_attention_weight_min": -2.107089042663574,
      "activations/layer18_attention_weight_max": 3.9227941036224365,
      "activations/layer18_attention_weight_min": -2.856049060821533,
      "activations/layer19_attention_weight_max": 1.3460533618927002,
      "activations/layer19_attention_weight_min": -1.2114958763122559,
      "activations/layer1_attention_weight_max": 8.277085304260254,
      "activations/layer1_attention_weight_min": -4.90798807144165,
      "activations/layer20_attention_weight_max": 1.5211455821990967,
      "activations/layer20_attention_weight_min": -1.301058053970337,
      "activations/layer21_attention_weight_max": 2.2154743671417236,
      "activations/layer21_attention_weight_min": -2.006457805633545,
      "activations/layer22_attention_weight_max": 1.027833342552185,
      "activations/layer22_attention_weight_min": -1.0481325387954712,
      "activations/layer23_attention_weight_max": 3.5364110469818115,
      "activations/layer23_attention_weight_min": -2.486180067062378,
      "activations/layer2_attention_weight_max": 1.712106466293335,
      "activations/layer2_attention_weight_min": -1.5310137271881104,
      "activations/layer3_attention_weight_max": 1.6960841417312622,
      "activations/layer3_attention_weight_min": -1.265305519104004,
      "activations/layer4_attention_weight_max": 1.8189945220947266,
      "activations/layer4_attention_weight_min": -1.369127869606018,
      "activations/layer5_attention_weight_max": 1.7316234111785889,
      "activations/layer5_attention_weight_min": -1.2976946830749512,
      "activations/layer6_attention_weight_max": 15.989898681640625,
      "activations/layer6_attention_weight_min": -10.632168769836426,
      "activations/layer7_attention_weight_max": 8.839526176452637,
      "activations/layer7_attention_weight_min": -4.400343418121338,
      "activations/layer8_attention_weight_max": 7.525824546813965,
      "activations/layer8_attention_weight_min": -4.583399295806885,
      "activations/layer9_attention_weight_max": 1.327088713645935,
      "activations/layer9_attention_weight_min": -1.1284815073013306,
      "epoch": 0.01,
      "learning_rate": 9.45089594432832e-05,
      "loss": 6.2362,
      "step": 200
    },
    {
      "activations/layer0_attention_weight_max": 9.03457260131836,
      "activations/layer0_attention_weight_min": -4.551792144775391,
      "activations/layer10_attention_weight_max": 2.0027971267700195,
      "activations/layer10_attention_weight_min": -1.8956656455993652,
      "activations/layer11_attention_weight_max": 1.637374758720398,
      "activations/layer11_attention_weight_min": -1.5859336853027344,
      "activations/layer12_attention_weight_max": 1.7157340049743652,
      "activations/layer12_attention_weight_min": -1.2925399541854858,
      "activations/layer13_attention_weight_max": 0.9102450609207153,
      "activations/layer13_attention_weight_min": -1.1343189477920532,
      "activations/layer14_attention_weight_max": 6.5376105308532715,
      "activations/layer14_attention_weight_min": -4.508024215698242,
      "activations/layer15_attention_weight_max": 2.280139207839966,
      "activations/layer15_attention_weight_min": -3.013356924057007,
      "activations/layer16_attention_weight_max": 1.76285719871521,
      "activations/layer16_attention_weight_min": -1.84168541431427,
      "activations/layer17_attention_weight_max": 2.877011775970459,
      "activations/layer17_attention_weight_min": -3.193478584289551,
      "activations/layer18_attention_weight_max": 4.848188400268555,
      "activations/layer18_attention_weight_min": -3.5010170936584473,
      "activations/layer19_attention_weight_max": 1.8264459371566772,
      "activations/layer19_attention_weight_min": -1.6875966787338257,
      "activations/layer1_attention_weight_max": 9.110111236572266,
      "activations/layer1_attention_weight_min": -6.370321273803711,
      "activations/layer20_attention_weight_max": 1.9714537858963013,
      "activations/layer20_attention_weight_min": -1.7278326749801636,
      "activations/layer21_attention_weight_max": 2.6156222820281982,
      "activations/layer21_attention_weight_min": -2.529773712158203,
      "activations/layer22_attention_weight_max": 1.492140293121338,
      "activations/layer22_attention_weight_min": -1.6214191913604736,
      "activations/layer23_attention_weight_max": 5.1553754806518555,
      "activations/layer23_attention_weight_min": -3.5409505367279053,
      "activations/layer2_attention_weight_max": 1.6042113304138184,
      "activations/layer2_attention_weight_min": -1.451038122177124,
      "activations/layer3_attention_weight_max": 1.4068278074264526,
      "activations/layer3_attention_weight_min": -1.366775393486023,
      "activations/layer4_attention_weight_max": 1.5154980421066284,
      "activations/layer4_attention_weight_min": -1.4139771461486816,
      "activations/layer5_attention_weight_max": 1.6616628170013428,
      "activations/layer5_attention_weight_min": -1.7854770421981812,
      "activations/layer6_attention_weight_max": 13.70858097076416,
      "activations/layer6_attention_weight_min": -8.408220291137695,
      "activations/layer7_attention_weight_max": 6.523471832275391,
      "activations/layer7_attention_weight_min": -3.2566545009613037,
      "activations/layer8_attention_weight_max": 6.134443759918213,
      "activations/layer8_attention_weight_min": -3.4098711013793945,
      "activations/layer9_attention_weight_max": 1.3081527948379517,
      "activations/layer9_attention_weight_min": -1.153644323348999,
      "epoch": 0.01,
      "learning_rate": 9.88147896765855e-05,
      "loss": 6.093,
      "step": 250
    },
    {
      "activations/layer0_attention_weight_max": 9.719239234924316,
      "activations/layer0_attention_weight_min": -4.192633628845215,
      "activations/layer10_attention_weight_max": 4.481209754943848,
      "activations/layer10_attention_weight_min": -3.6294729709625244,
      "activations/layer11_attention_weight_max": 2.6957404613494873,
      "activations/layer11_attention_weight_min": -2.0137035846710205,
      "activations/layer12_attention_weight_max": 1.5256344079971313,
      "activations/layer12_attention_weight_min": -1.647940754890442,
      "activations/layer13_attention_weight_max": 0.8515466451644897,
      "activations/layer13_attention_weight_min": -1.1080501079559326,
      "activations/layer14_attention_weight_max": 7.107532501220703,
      "activations/layer14_attention_weight_min": -5.384039402008057,
      "activations/layer15_attention_weight_max": 3.3926424980163574,
      "activations/layer15_attention_weight_min": -3.9215126037597656,
      "activations/layer16_attention_weight_max": 2.5455164909362793,
      "activations/layer16_attention_weight_min": -2.41597843170166,
      "activations/layer17_attention_weight_max": 4.340159893035889,
      "activations/layer17_attention_weight_min": -4.107534408569336,
      "activations/layer18_attention_weight_max": 5.360610485076904,
      "activations/layer18_attention_weight_min": -3.693737745285034,
      "activations/layer19_attention_weight_max": 2.020082712173462,
      "activations/layer19_attention_weight_min": -2.0092153549194336,
      "activations/layer1_attention_weight_max": 8.713669776916504,
      "activations/layer1_attention_weight_min": -7.329385757446289,
      "activations/layer20_attention_weight_max": 2.2504007816314697,
      "activations/layer20_attention_weight_min": -1.9305328130722046,
      "activations/layer21_attention_weight_max": 2.676619529724121,
      "activations/layer21_attention_weight_min": -2.6992528438568115,
      "activations/layer22_attention_weight_max": 1.6856718063354492,
      "activations/layer22_attention_weight_min": -2.0514447689056396,
      "activations/layer23_attention_weight_max": 6.77037239074707,
      "activations/layer23_attention_weight_min": -4.08543586730957,
      "activations/layer2_attention_weight_max": 2.002096176147461,
      "activations/layer2_attention_weight_min": -1.6162590980529785,
      "activations/layer3_attention_weight_max": 1.6930047273635864,
      "activations/layer3_attention_weight_min": -1.4115556478500366,
      "activations/layer4_attention_weight_max": 1.7654510736465454,
      "activations/layer4_attention_weight_min": -1.6347904205322266,
      "activations/layer5_attention_weight_max": 2.0711636543273926,
      "activations/layer5_attention_weight_min": -2.0355114936828613,
      "activations/layer6_attention_weight_max": 13.685540199279785,
      "activations/layer6_attention_weight_min": -8.657905578613281,
      "activations/layer7_attention_weight_max": 6.107098579406738,
      "activations/layer7_attention_weight_min": -3.0800487995147705,
      "activations/layer8_attention_weight_max": 5.863523483276367,
      "activations/layer8_attention_weight_min": -4.213197708129883,
      "activations/layer9_attention_weight_max": 1.6928437948226929,
      "activations/layer9_attention_weight_min": -1.4769166707992554,
      "epoch": 0.02,
      "learning_rate": 0.00010229005231906732,
      "loss": 5.9467,
      "step": 300
    },
    {
      "activations/layer0_attention_weight_max": 10.03735637664795,
      "activations/layer0_attention_weight_min": -4.573877334594727,
      "activations/layer10_attention_weight_max": 11.002238273620605,
      "activations/layer10_attention_weight_min": -7.159815788269043,
      "activations/layer11_attention_weight_max": 2.7649810314178467,
      "activations/layer11_attention_weight_min": -2.350574254989624,
      "activations/layer12_attention_weight_max": 2.0923476219177246,
      "activations/layer12_attention_weight_min": -2.098527193069458,
      "activations/layer13_attention_weight_max": 1.370101809501648,
      "activations/layer13_attention_weight_min": -1.3858990669250488,
      "activations/layer14_attention_weight_max": 8.478209495544434,
      "activations/layer14_attention_weight_min": -6.188502788543701,
      "activations/layer15_attention_weight_max": 7.4630231857299805,
      "activations/layer15_attention_weight_min": -5.224538326263428,
      "activations/layer16_attention_weight_max": 6.3665690422058105,
      "activations/layer16_attention_weight_min": -4.096056938171387,
      "activations/layer17_attention_weight_max": 8.477813720703125,
      "activations/layer17_attention_weight_min": -5.980857849121094,
      "activations/layer18_attention_weight_max": 7.448421955108643,
      "activations/layer18_attention_weight_min": -5.37166690826416,
      "activations/layer19_attention_weight_max": 2.380138874053955,
      "activations/layer19_attention_weight_min": -2.3102667331695557,
      "activations/layer1_attention_weight_max": 9.630451202392578,
      "activations/layer1_attention_weight_min": -10.287247657775879,
      "activations/layer20_attention_weight_max": 2.7142584323883057,
      "activations/layer20_attention_weight_min": -2.2780652046203613,
      "activations/layer21_attention_weight_max": 3.9103164672851562,
      "activations/layer21_attention_weight_min": -3.230623960494995,
      "activations/layer22_attention_weight_max": 2.1636195182800293,
      "activations/layer22_attention_weight_min": -2.3650732040405273,
      "activations/layer23_attention_weight_max": 7.982816696166992,
      "activations/layer23_attention_weight_min": -4.032634258270264,
      "activations/layer2_attention_weight_max": 3.741809368133545,
      "activations/layer2_attention_weight_min": -2.776139974594116,
      "activations/layer3_attention_weight_max": 2.4247686862945557,
      "activations/layer3_attention_weight_min": -1.9459965229034424,
      "activations/layer4_attention_weight_max": 3.9913978576660156,
      "activations/layer4_attention_weight_min": -3.3536269664764404,
      "activations/layer5_attention_weight_max": 3.208408832550049,
      "activations/layer5_attention_weight_min": -2.3139703273773193,
      "activations/layer6_attention_weight_max": 12.932045936584473,
      "activations/layer6_attention_weight_min": -10.04324722290039,
      "activations/layer7_attention_weight_max": 6.183136940002441,
      "activations/layer7_attention_weight_min": -3.311145305633545,
      "activations/layer8_attention_weight_max": 6.1374430656433105,
      "activations/layer8_attention_weight_min": -4.769620895385742,
      "activations/layer9_attention_weight_max": 3.30973219871521,
      "activations/layer9_attention_weight_min": -1.9757394790649414,
      "epoch": 0.02,
      "learning_rate": 0.00010520393678084459,
      "loss": 5.8678,
      "step": 350
    },
    {
      "activations/layer0_attention_weight_max": 7.831273078918457,
      "activations/layer0_attention_weight_min": -5.422358989715576,
      "activations/layer10_attention_weight_max": 15.087559700012207,
      "activations/layer10_attention_weight_min": -11.25493335723877,
      "activations/layer11_attention_weight_max": 3.40555477142334,
      "activations/layer11_attention_weight_min": -2.999394416809082,
      "activations/layer12_attention_weight_max": 2.737172842025757,
      "activations/layer12_attention_weight_min": -2.624128818511963,
      "activations/layer13_attention_weight_max": 2.011230230331421,
      "activations/layer13_attention_weight_min": -1.6927202939987183,
      "activations/layer14_attention_weight_max": 11.304832458496094,
      "activations/layer14_attention_weight_min": -7.628824710845947,
      "activations/layer15_attention_weight_max": 9.305670738220215,
      "activations/layer15_attention_weight_min": -6.44460916519165,
      "activations/layer16_attention_weight_max": 6.747203350067139,
      "activations/layer16_attention_weight_min": -5.090256690979004,
      "activations/layer17_attention_weight_max": 8.406577110290527,
      "activations/layer17_attention_weight_min": -6.35894775390625,
      "activations/layer18_attention_weight_max": 7.7391180992126465,
      "activations/layer18_attention_weight_min": -6.284543514251709,
      "activations/layer19_attention_weight_max": 2.3539645671844482,
      "activations/layer19_attention_weight_min": -2.272437810897827,
      "activations/layer1_attention_weight_max": 12.791690826416016,
      "activations/layer1_attention_weight_min": -13.042023658752441,
      "activations/layer20_attention_weight_max": 2.5849881172180176,
      "activations/layer20_attention_weight_min": -2.2700369358062744,
      "activations/layer21_attention_weight_max": 3.9246859550476074,
      "activations/layer21_attention_weight_min": -3.2243735790252686,
      "activations/layer22_attention_weight_max": 2.2040746212005615,
      "activations/layer22_attention_weight_min": -2.589498996734619,
      "activations/layer23_attention_weight_max": 7.71193790435791,
      "activations/layer23_attention_weight_min": -3.675375461578369,
      "activations/layer2_attention_weight_max": 5.598979473114014,
      "activations/layer2_attention_weight_min": -5.262666702270508,
      "activations/layer3_attention_weight_max": 4.963005065917969,
      "activations/layer3_attention_weight_min": -4.611032962799072,
      "activations/layer4_attention_weight_max": 7.562720775604248,
      "activations/layer4_attention_weight_min": -8.285877227783203,
      "activations/layer5_attention_weight_max": 5.841756820678711,
      "activations/layer5_attention_weight_min": -5.754530906677246,
      "activations/layer6_attention_weight_max": 14.225316047668457,
      "activations/layer6_attention_weight_min": -15.457456588745117,
      "activations/layer7_attention_weight_max": 6.7567620277404785,
      "activations/layer7_attention_weight_min": -6.806295871734619,
      "activations/layer8_attention_weight_max": 8.63115406036377,
      "activations/layer8_attention_weight_min": -7.790324687957764,
      "activations/layer9_attention_weight_max": 10.458443641662598,
      "activations/layer9_attention_weight_min": -6.422760009765625,
      "epoch": 0.02,
      "learning_rate": 0.00010771283560930495,
      "loss": 5.7586,
      "step": 400
    },
    {
      "activations/layer0_attention_weight_max": 8.768975257873535,
      "activations/layer0_attention_weight_min": -7.048642635345459,
      "activations/layer10_attention_weight_max": 21.913782119750977,
      "activations/layer10_attention_weight_min": -21.95467185974121,
      "activations/layer11_attention_weight_max": 4.901906967163086,
      "activations/layer11_attention_weight_min": -5.056134223937988,
      "activations/layer12_attention_weight_max": 5.158649444580078,
      "activations/layer12_attention_weight_min": -4.8625688552856445,
      "activations/layer13_attention_weight_max": 2.80806040763855,
      "activations/layer13_attention_weight_min": -2.776700258255005,
      "activations/layer14_attention_weight_max": 9.2290620803833,
      "activations/layer14_attention_weight_min": -7.187514305114746,
      "activations/layer15_attention_weight_max": 9.705778121948242,
      "activations/layer15_attention_weight_min": -7.485869884490967,
      "activations/layer16_attention_weight_max": 8.285807609558105,
      "activations/layer16_attention_weight_min": -6.6011881828308105,
      "activations/layer17_attention_weight_max": 10.457646369934082,
      "activations/layer17_attention_weight_min": -6.713185787200928,
      "activations/layer18_attention_weight_max": 8.922666549682617,
      "activations/layer18_attention_weight_min": -6.253037452697754,
      "activations/layer19_attention_weight_max": 2.5507309436798096,
      "activations/layer19_attention_weight_min": -2.329098701477051,
      "activations/layer1_attention_weight_max": 16.6998348236084,
      "activations/layer1_attention_weight_min": -17.07223892211914,
      "activations/layer20_attention_weight_max": 2.9509618282318115,
      "activations/layer20_attention_weight_min": -2.392946481704712,
      "activations/layer21_attention_weight_max": 3.8614819049835205,
      "activations/layer21_attention_weight_min": -3.1343932151794434,
      "activations/layer22_attention_weight_max": 2.643557071685791,
      "activations/layer22_attention_weight_min": -2.599470853805542,
      "activations/layer23_attention_weight_max": 8.452896118164062,
      "activations/layer23_attention_weight_min": -3.6865506172180176,
      "activations/layer2_attention_weight_max": 11.00953483581543,
      "activations/layer2_attention_weight_min": -9.847984313964844,
      "activations/layer3_attention_weight_max": 13.099952697753906,
      "activations/layer3_attention_weight_min": -12.271729469299316,
      "activations/layer4_attention_weight_max": 23.138402938842773,
      "activations/layer4_attention_weight_min": -23.31123924255371,
      "activations/layer5_attention_weight_max": 23.205474853515625,
      "activations/layer5_attention_weight_min": -23.499168395996094,
      "activations/layer6_attention_weight_max": 25.4335994720459,
      "activations/layer6_attention_weight_min": -27.560955047607422,
      "activations/layer7_attention_weight_max": 16.711780548095703,
      "activations/layer7_attention_weight_min": -16.948545455932617,
      "activations/layer8_attention_weight_max": 21.943307876586914,
      "activations/layer8_attention_weight_min": -22.81713104248047,
      "activations/layer9_attention_weight_max": 18.335186004638672,
      "activations/layer9_attention_weight_min": -16.250425338745117,
      "epoch": 0.03,
      "learning_rate": 0.00010991570777374085,
      "loss": 5.6834,
      "step": 450
    },
    {
      "activations/layer0_attention_weight_max": 9.459610939025879,
      "activations/layer0_attention_weight_min": -7.367496967315674,
      "activations/layer10_attention_weight_max": 35.388153076171875,
      "activations/layer10_attention_weight_min": -34.58955383300781,
      "activations/layer11_attention_weight_max": 8.121853828430176,
      "activations/layer11_attention_weight_min": -6.747608184814453,
      "activations/layer12_attention_weight_max": 13.342389106750488,
      "activations/layer12_attention_weight_min": -8.874107360839844,
      "activations/layer13_attention_weight_max": 4.488685131072998,
      "activations/layer13_attention_weight_min": -3.4345364570617676,
      "activations/layer14_attention_weight_max": 10.213300704956055,
      "activations/layer14_attention_weight_min": -7.752788066864014,
      "activations/layer15_attention_weight_max": 10.125662803649902,
      "activations/layer15_attention_weight_min": -7.613223552703857,
      "activations/layer16_attention_weight_max": 9.839667320251465,
      "activations/layer16_attention_weight_min": -6.565066337585449,
      "activations/layer17_attention_weight_max": 10.402572631835938,
      "activations/layer17_attention_weight_min": -6.399139404296875,
      "activations/layer18_attention_weight_max": 8.978286743164062,
      "activations/layer18_attention_weight_min": -5.9296956062316895,
      "activations/layer19_attention_weight_max": 2.723236083984375,
      "activations/layer19_attention_weight_min": -2.9868593215942383,
      "activations/layer1_attention_weight_max": 19.53768539428711,
      "activations/layer1_attention_weight_min": -18.769392013549805,
      "activations/layer20_attention_weight_max": 3.0988988876342773,
      "activations/layer20_attention_weight_min": -2.510685443878174,
      "activations/layer21_attention_weight_max": 4.331419467926025,
      "activations/layer21_attention_weight_min": -3.658360004425049,
      "activations/layer22_attention_weight_max": 2.597534656524658,
      "activations/layer22_attention_weight_min": -2.549647569656372,
      "activations/layer23_attention_weight_max": 8.994438171386719,
      "activations/layer23_attention_weight_min": -4.166797637939453,
      "activations/layer2_attention_weight_max": 12.650923728942871,
      "activations/layer2_attention_weight_min": -10.757689476013184,
      "activations/layer3_attention_weight_max": 21.02838706970215,
      "activations/layer3_attention_weight_min": -21.638872146606445,
      "activations/layer4_attention_weight_max": 41.56938934326172,
      "activations/layer4_attention_weight_min": -45.375938415527344,
      "activations/layer5_attention_weight_max": 53.83513641357422,
      "activations/layer5_attention_weight_min": -56.28803253173828,
      "activations/layer6_attention_weight_max": 54.11566925048828,
      "activations/layer6_attention_weight_min": -57.164451599121094,
      "activations/layer7_attention_weight_max": 37.66425704956055,
      "activations/layer7_attention_weight_min": -37.96014404296875,
      "activations/layer8_attention_weight_max": 42.99534225463867,
      "activations/layer8_attention_weight_min": -48.91874694824219,
      "activations/layer9_attention_weight_max": 29.4334774017334,
      "activations/layer9_attention_weight_min": -29.757780075073242,
      "epoch": 0.03,
      "learning_rate": 0.00011187915841478604,
      "loss": 5.5751,
      "step": 500
    },
    {
      "activations/layer0_attention_weight_max": 11.116846084594727,
      "activations/layer0_attention_weight_min": -7.496310710906982,
      "activations/layer10_attention_weight_max": 43.489166259765625,
      "activations/layer10_attention_weight_min": -43.140865325927734,
      "activations/layer11_attention_weight_max": 11.908164024353027,
      "activations/layer11_attention_weight_min": -12.045219421386719,
      "activations/layer12_attention_weight_max": 16.24372100830078,
      "activations/layer12_attention_weight_min": -11.166133880615234,
      "activations/layer13_attention_weight_max": 7.145237445831299,
      "activations/layer13_attention_weight_min": -4.378142833709717,
      "activations/layer14_attention_weight_max": 11.60574722290039,
      "activations/layer14_attention_weight_min": -8.13232707977295,
      "activations/layer15_attention_weight_max": 13.52804183959961,
      "activations/layer15_attention_weight_min": -7.423277378082275,
      "activations/layer16_attention_weight_max": 13.75950813293457,
      "activations/layer16_attention_weight_min": -7.456070423126221,
      "activations/layer17_attention_weight_max": 10.056524276733398,
      "activations/layer17_attention_weight_min": -6.398423194885254,
      "activations/layer18_attention_weight_max": 9.787392616271973,
      "activations/layer18_attention_weight_min": -6.48866081237793,
      "activations/layer19_attention_weight_max": 2.8319499492645264,
      "activations/layer19_attention_weight_min": -2.644516706466675,
      "activations/layer1_attention_weight_max": 21.0802001953125,
      "activations/layer1_attention_weight_min": -16.650535583496094,
      "activations/layer20_attention_weight_max": 3.1455819606781006,
      "activations/layer20_attention_weight_min": -2.6032803058624268,
      "activations/layer21_attention_weight_max": 4.698831081390381,
      "activations/layer21_attention_weight_min": -3.8729474544525146,
      "activations/layer22_attention_weight_max": 2.531221866607666,
      "activations/layer22_attention_weight_min": -2.655003547668457,
      "activations/layer23_attention_weight_max": 9.061616897583008,
      "activations/layer23_attention_weight_min": -4.224883556365967,
      "activations/layer2_attention_weight_max": 16.25870132446289,
      "activations/layer2_attention_weight_min": -11.834967613220215,
      "activations/layer3_attention_weight_max": 28.18883514404297,
      "activations/layer3_attention_weight_min": -25.148311614990234,
      "activations/layer4_attention_weight_max": 58.085693359375,
      "activations/layer4_attention_weight_min": -59.24720001220703,
      "activations/layer5_attention_weight_max": 84.5783920288086,
      "activations/layer5_attention_weight_min": -90.60482025146484,
      "activations/layer6_attention_weight_max": 82.74507141113281,
      "activations/layer6_attention_weight_min": -90.91938018798828,
      "activations/layer7_attention_weight_max": 56.63484191894531,
      "activations/layer7_attention_weight_min": -57.50590133666992,
      "activations/layer8_attention_weight_max": 68.2411880493164,
      "activations/layer8_attention_weight_min": -73.99639892578125,
      "activations/layer9_attention_weight_max": 41.15328598022461,
      "activations/layer9_attention_weight_min": -39.7697868347168,
      "epoch": 0.03,
      "learning_rate": 0.00011365016669336267,
      "loss": 5.4399,
      "step": 550
    },
    {
      "activations/layer0_attention_weight_max": 10.479532241821289,
      "activations/layer0_attention_weight_min": -7.929759979248047,
      "activations/layer10_attention_weight_max": 42.00307846069336,
      "activations/layer10_attention_weight_min": -49.417572021484375,
      "activations/layer11_attention_weight_max": 15.003412246704102,
      "activations/layer11_attention_weight_min": -16.11397933959961,
      "activations/layer12_attention_weight_max": 14.235501289367676,
      "activations/layer12_attention_weight_min": -12.378472328186035,
      "activations/layer13_attention_weight_max": 10.305051803588867,
      "activations/layer13_attention_weight_min": -7.231587886810303,
      "activations/layer14_attention_weight_max": 12.706210136413574,
      "activations/layer14_attention_weight_min": -10.557538986206055,
      "activations/layer15_attention_weight_max": 12.3080415725708,
      "activations/layer15_attention_weight_min": -7.853063106536865,
      "activations/layer16_attention_weight_max": 18.74260139465332,
      "activations/layer16_attention_weight_min": -9.570582389831543,
      "activations/layer17_attention_weight_max": 13.745564460754395,
      "activations/layer17_attention_weight_min": -9.900359153747559,
      "activations/layer18_attention_weight_max": 10.83527946472168,
      "activations/layer18_attention_weight_min": -7.276120662689209,
      "activations/layer19_attention_weight_max": 3.0115363597869873,
      "activations/layer19_attention_weight_min": -3.058562994003296,
      "activations/layer1_attention_weight_max": 20.271631240844727,
      "activations/layer1_attention_weight_min": -17.940874099731445,
      "activations/layer20_attention_weight_max": 3.130094051361084,
      "activations/layer20_attention_weight_min": -2.8286330699920654,
      "activations/layer21_attention_weight_max": 5.057249546051025,
      "activations/layer21_attention_weight_min": -3.749682903289795,
      "activations/layer22_attention_weight_max": 2.639235258102417,
      "activations/layer22_attention_weight_min": -3.007859230041504,
      "activations/layer23_attention_weight_max": 9.149617195129395,
      "activations/layer23_attention_weight_min": -4.036314487457275,
      "activations/layer2_attention_weight_max": 15.12982177734375,
      "activations/layer2_attention_weight_min": -12.145414352416992,
      "activations/layer3_attention_weight_max": 29.929418563842773,
      "activations/layer3_attention_weight_min": -26.343881607055664,
      "activations/layer4_attention_weight_max": 71.48870086669922,
      "activations/layer4_attention_weight_min": -72.49077606201172,
      "activations/layer5_attention_weight_max": 102.68727111816406,
      "activations/layer5_attention_weight_min": -113.21868133544922,
      "activations/layer6_attention_weight_max": 102.93208312988281,
      "activations/layer6_attention_weight_min": -113.35592651367188,
      "activations/layer7_attention_weight_max": 71.6628646850586,
      "activations/layer7_attention_weight_min": -73.31793212890625,
      "activations/layer8_attention_weight_max": 78.79058837890625,
      "activations/layer8_attention_weight_min": -90.90660858154297,
      "activations/layer9_attention_weight_max": 40.8582649230957,
      "activations/layer9_attention_weight_min": -45.78266143798828,
      "epoch": 0.03,
      "learning_rate": 0.00011526311149794271,
      "loss": 5.3246,
      "step": 600
    },
    {
      "activations/layer0_attention_weight_max": 9.887779235839844,
      "activations/layer0_attention_weight_min": -7.802441596984863,
      "activations/layer10_attention_weight_max": 47.559051513671875,
      "activations/layer10_attention_weight_min": -47.20457077026367,
      "activations/layer11_attention_weight_max": 18.482341766357422,
      "activations/layer11_attention_weight_min": -18.75442123413086,
      "activations/layer12_attention_weight_max": 16.144529342651367,
      "activations/layer12_attention_weight_min": -11.212666511535645,
      "activations/layer13_attention_weight_max": 9.635083198547363,
      "activations/layer13_attention_weight_min": -6.986222267150879,
      "activations/layer14_attention_weight_max": 15.615735054016113,
      "activations/layer14_attention_weight_min": -9.284521102905273,
      "activations/layer15_attention_weight_max": 10.113509178161621,
      "activations/layer15_attention_weight_min": -7.455432415008545,
      "activations/layer16_attention_weight_max": 13.4478178024292,
      "activations/layer16_attention_weight_min": -7.408009052276611,
      "activations/layer17_attention_weight_max": 13.008235931396484,
      "activations/layer17_attention_weight_min": -8.336895942687988,
      "activations/layer18_attention_weight_max": 10.195735931396484,
      "activations/layer18_attention_weight_min": -6.503733158111572,
      "activations/layer19_attention_weight_max": 3.668996572494507,
      "activations/layer19_attention_weight_min": -3.237130880355835,
      "activations/layer1_attention_weight_max": 20.999738693237305,
      "activations/layer1_attention_weight_min": -18.083984375,
      "activations/layer20_attention_weight_max": 3.7314555644989014,
      "activations/layer20_attention_weight_min": -3.26944637298584,
      "activations/layer21_attention_weight_max": 5.5945305824279785,
      "activations/layer21_attention_weight_min": -3.6350440979003906,
      "activations/layer22_attention_weight_max": 2.9198341369628906,
      "activations/layer22_attention_weight_min": -2.9568023681640625,
      "activations/layer23_attention_weight_max": 9.169642448425293,
      "activations/layer23_attention_weight_min": -3.959893226623535,
      "activations/layer2_attention_weight_max": 17.27198028564453,
      "activations/layer2_attention_weight_min": -12.222862243652344,
      "activations/layer3_attention_weight_max": 36.65752029418945,
      "activations/layer3_attention_weight_min": -29.419294357299805,
      "activations/layer4_attention_weight_max": 81.20735168457031,
      "activations/layer4_attention_weight_min": -84.76698303222656,
      "activations/layer5_attention_weight_max": 119.26492309570312,
      "activations/layer5_attention_weight_min": -131.2852783203125,
      "activations/layer6_attention_weight_max": 119.57235717773438,
      "activations/layer6_attention_weight_min": -131.4782257080078,
      "activations/layer7_attention_weight_max": 82.13996887207031,
      "activations/layer7_attention_weight_min": -88.23854064941406,
      "activations/layer8_attention_weight_max": 90.34368133544922,
      "activations/layer8_attention_weight_min": -103.3603286743164,
      "activations/layer9_attention_weight_max": 50.304927825927734,
      "activations/layer9_attention_weight_min": -52.021156311035156,
      "epoch": 0.04,
      "learning_rate": 0.00011674390997364552,
      "loss": 5.2263,
      "step": 650
    },
    {
      "activations/layer0_attention_weight_max": 12.286136627197266,
      "activations/layer0_attention_weight_min": -8.088682174682617,
      "activations/layer10_attention_weight_max": 42.75489807128906,
      "activations/layer10_attention_weight_min": -46.70683288574219,
      "activations/layer11_attention_weight_max": 17.12057113647461,
      "activations/layer11_attention_weight_min": -15.930721282958984,
      "activations/layer12_attention_weight_max": 13.530871391296387,
      "activations/layer12_attention_weight_min": -11.069083213806152,
      "activations/layer13_attention_weight_max": 9.012950897216797,
      "activations/layer13_attention_weight_min": -6.947690963745117,
      "activations/layer14_attention_weight_max": 12.413479804992676,
      "activations/layer14_attention_weight_min": -8.945076942443848,
      "activations/layer15_attention_weight_max": 9.75110149383545,
      "activations/layer15_attention_weight_min": -6.446042060852051,
      "activations/layer16_attention_weight_max": 13.527578353881836,
      "activations/layer16_attention_weight_min": -7.739782810211182,
      "activations/layer17_attention_weight_max": 10.856817245483398,
      "activations/layer17_attention_weight_min": -8.448709487915039,
      "activations/layer18_attention_weight_max": 9.835561752319336,
      "activations/layer18_attention_weight_min": -6.996677875518799,
      "activations/layer19_attention_weight_max": 3.9113450050354004,
      "activations/layer19_attention_weight_min": -3.2746644020080566,
      "activations/layer1_attention_weight_max": 20.92273712158203,
      "activations/layer1_attention_weight_min": -17.61338233947754,
      "activations/layer20_attention_weight_max": 4.685707092285156,
      "activations/layer20_attention_weight_min": -3.4234778881073,
      "activations/layer21_attention_weight_max": 5.512387275695801,
      "activations/layer21_attention_weight_min": -3.9305007457733154,
      "activations/layer22_attention_weight_max": 2.7268316745758057,
      "activations/layer22_attention_weight_min": -3.0192084312438965,
      "activations/layer23_attention_weight_max": 9.259176254272461,
      "activations/layer23_attention_weight_min": -3.783489227294922,
      "activations/layer2_attention_weight_max": 15.319364547729492,
      "activations/layer2_attention_weight_min": -12.230119705200195,
      "activations/layer3_attention_weight_max": 33.81103515625,
      "activations/layer3_attention_weight_min": -29.933847427368164,
      "activations/layer4_attention_weight_max": 83.34495544433594,
      "activations/layer4_attention_weight_min": -87.87355041503906,
      "activations/layer5_attention_weight_max": 122.8294906616211,
      "activations/layer5_attention_weight_min": -141.0224609375,
      "activations/layer6_attention_weight_max": 124.08191680908203,
      "activations/layer6_attention_weight_min": -142.4137420654297,
      "activations/layer7_attention_weight_max": 84.19854736328125,
      "activations/layer7_attention_weight_min": -93.36534118652344,
      "activations/layer8_attention_weight_max": 92.7686538696289,
      "activations/layer8_attention_weight_min": -106.77568817138672,
      "activations/layer9_attention_weight_max": 44.788780212402344,
      "activations/layer9_attention_weight_min": -53.676300048828125,
      "epoch": 0.04,
      "learning_rate": 0.0001181125851263692,
      "loss": 5.1137,
      "step": 700
    },
    {
      "activations/layer0_attention_weight_max": 11.247169494628906,
      "activations/layer0_attention_weight_min": -8.01302433013916,
      "activations/layer10_attention_weight_max": 46.735836029052734,
      "activations/layer10_attention_weight_min": -45.27787399291992,
      "activations/layer11_attention_weight_max": 23.256925582885742,
      "activations/layer11_attention_weight_min": -17.296592712402344,
      "activations/layer12_attention_weight_max": 14.161441802978516,
      "activations/layer12_attention_weight_min": -13.480745315551758,
      "activations/layer13_attention_weight_max": 11.512337684631348,
      "activations/layer13_attention_weight_min": -7.215910911560059,
      "activations/layer14_attention_weight_max": 16.156309127807617,
      "activations/layer14_attention_weight_min": -8.616676330566406,
      "activations/layer15_attention_weight_max": 10.44541072845459,
      "activations/layer15_attention_weight_min": -6.848757266998291,
      "activations/layer16_attention_weight_max": 11.923714637756348,
      "activations/layer16_attention_weight_min": -7.474878787994385,
      "activations/layer17_attention_weight_max": 10.42469596862793,
      "activations/layer17_attention_weight_min": -8.232894897460938,
      "activations/layer18_attention_weight_max": 9.808198928833008,
      "activations/layer18_attention_weight_min": -6.8495306968688965,
      "activations/layer19_attention_weight_max": 4.6193132400512695,
      "activations/layer19_attention_weight_min": -3.645142078399658,
      "activations/layer1_attention_weight_max": 20.35073471069336,
      "activations/layer1_attention_weight_min": -17.28925132751465,
      "activations/layer20_attention_weight_max": 5.487331390380859,
      "activations/layer20_attention_weight_min": -4.072383403778076,
      "activations/layer21_attention_weight_max": 5.582104206085205,
      "activations/layer21_attention_weight_min": -3.5100278854370117,
      "activations/layer22_attention_weight_max": 2.8610949516296387,
      "activations/layer22_attention_weight_min": -2.6675708293914795,
      "activations/layer23_attention_weight_max": 9.837433815002441,
      "activations/layer23_attention_weight_min": -4.041867256164551,
      "activations/layer2_attention_weight_max": 18.259986877441406,
      "activations/layer2_attention_weight_min": -11.268362998962402,
      "activations/layer3_attention_weight_max": 39.489990234375,
      "activations/layer3_attention_weight_min": -31.82705307006836,
      "activations/layer4_attention_weight_max": 93.31866455078125,
      "activations/layer4_attention_weight_min": -89.52564239501953,
      "activations/layer5_attention_weight_max": 138.7594451904297,
      "activations/layer5_attention_weight_min": -148.9246063232422,
      "activations/layer6_attention_weight_max": 137.2274932861328,
      "activations/layer6_attention_weight_min": -150.8585968017578,
      "activations/layer7_attention_weight_max": 95.58712768554688,
      "activations/layer7_attention_weight_min": -101.80082702636719,
      "activations/layer8_attention_weight_max": 102.29683685302734,
      "activations/layer8_attention_weight_min": -109.27497100830078,
      "activations/layer9_attention_weight_max": 51.71751022338867,
      "activations/layer9_attention_weight_min": -53.81759262084961,
      "epoch": 0.04,
      "learning_rate": 0.00011938492784293858,
      "loss": 5.0507,
      "step": 750
    },
    {
      "activations/layer0_attention_weight_max": 11.489453315734863,
      "activations/layer0_attention_weight_min": -7.819090843200684,
      "activations/layer10_attention_weight_max": 44.334083557128906,
      "activations/layer10_attention_weight_min": -46.415313720703125,
      "activations/layer11_attention_weight_max": 23.456321716308594,
      "activations/layer11_attention_weight_min": -18.802101135253906,
      "activations/layer12_attention_weight_max": 14.063304901123047,
      "activations/layer12_attention_weight_min": -13.113679885864258,
      "activations/layer13_attention_weight_max": 10.522953033447266,
      "activations/layer13_attention_weight_min": -8.581656455993652,
      "activations/layer14_attention_weight_max": 14.460159301757812,
      "activations/layer14_attention_weight_min": -10.172792434692383,
      "activations/layer15_attention_weight_max": 9.98495864868164,
      "activations/layer15_attention_weight_min": -7.622045040130615,
      "activations/layer16_attention_weight_max": 12.330225944519043,
      "activations/layer16_attention_weight_min": -7.705427169799805,
      "activations/layer17_attention_weight_max": 11.694428443908691,
      "activations/layer17_attention_weight_min": -10.659455299377441,
      "activations/layer18_attention_weight_max": 9.269718170166016,
      "activations/layer18_attention_weight_min": -6.561591148376465,
      "activations/layer19_attention_weight_max": 4.946895122528076,
      "activations/layer19_attention_weight_min": -3.7025420665740967,
      "activations/layer1_attention_weight_max": 20.798320770263672,
      "activations/layer1_attention_weight_min": -17.79120445251465,
      "activations/layer20_attention_weight_max": 6.979347229003906,
      "activations/layer20_attention_weight_min": -5.450769901275635,
      "activations/layer21_attention_weight_max": 5.505956649780273,
      "activations/layer21_attention_weight_min": -3.5565266609191895,
      "activations/layer22_attention_weight_max": 2.860053062438965,
      "activations/layer22_attention_weight_min": -2.7646751403808594,
      "activations/layer23_attention_weight_max": 9.847912788391113,
      "activations/layer23_attention_weight_min": -3.851487636566162,
      "activations/layer2_attention_weight_max": 17.773815155029297,
      "activations/layer2_attention_weight_min": -13.022253036499023,
      "activations/layer3_attention_weight_max": 38.463069915771484,
      "activations/layer3_attention_weight_min": -35.05573272705078,
      "activations/layer4_attention_weight_max": 89.52536010742188,
      "activations/layer4_attention_weight_min": -96.57527923583984,
      "activations/layer5_attention_weight_max": 132.9664306640625,
      "activations/layer5_attention_weight_min": -155.52268981933594,
      "activations/layer6_attention_weight_max": 133.00784301757812,
      "activations/layer6_attention_weight_min": -155.8210906982422,
      "activations/layer7_attention_weight_max": 95.4098129272461,
      "activations/layer7_attention_weight_min": -107.94225311279297,
      "activations/layer8_attention_weight_max": 93.4787368774414,
      "activations/layer8_attention_weight_min": -111.16458892822266,
      "activations/layer9_attention_weight_max": 53.680519104003906,
      "activations/layer9_attention_weight_min": -62.442169189453125,
      "epoch": 0.05,
      "learning_rate": 0.00012057361147552512,
      "loss": 4.9805,
      "step": 800
    },
    {
      "activations/layer0_attention_weight_max": 10.768267631530762,
      "activations/layer0_attention_weight_min": -8.10888957977295,
      "activations/layer10_attention_weight_max": 49.8792839050293,
      "activations/layer10_attention_weight_min": -46.55388259887695,
      "activations/layer11_attention_weight_max": 21.17418670654297,
      "activations/layer11_attention_weight_min": -16.64020538330078,
      "activations/layer12_attention_weight_max": 13.078019142150879,
      "activations/layer12_attention_weight_min": -12.87271499633789,
      "activations/layer13_attention_weight_max": 11.027698516845703,
      "activations/layer13_attention_weight_min": -8.65995979309082,
      "activations/layer14_attention_weight_max": 15.515256881713867,
      "activations/layer14_attention_weight_min": -9.508536338806152,
      "activations/layer15_attention_weight_max": 9.873135566711426,
      "activations/layer15_attention_weight_min": -8.488434791564941,
      "activations/layer16_attention_weight_max": 12.845426559448242,
      "activations/layer16_attention_weight_min": -7.2402472496032715,
      "activations/layer17_attention_weight_max": 13.619584083557129,
      "activations/layer17_attention_weight_min": -9.200155258178711,
      "activations/layer18_attention_weight_max": 11.21716594696045,
      "activations/layer18_attention_weight_min": -6.4611663818359375,
      "activations/layer19_attention_weight_max": 5.0630202293396,
      "activations/layer19_attention_weight_min": -3.56602144241333,
      "activations/layer1_attention_weight_max": 20.77703094482422,
      "activations/layer1_attention_weight_min": -15.97847843170166,
      "activations/layer20_attention_weight_max": 9.06597900390625,
      "activations/layer20_attention_weight_min": -5.230363845825195,
      "activations/layer21_attention_weight_max": 6.235052585601807,
      "activations/layer21_attention_weight_min": -3.512917995452881,
      "activations/layer22_attention_weight_max": 2.991408109664917,
      "activations/layer22_attention_weight_min": -2.7415077686309814,
      "activations/layer23_attention_weight_max": 10.05119514465332,
      "activations/layer23_attention_weight_min": -4.188574314117432,
      "activations/layer2_attention_weight_max": 17.548006057739258,
      "activations/layer2_attention_weight_min": -13.785829544067383,
      "activations/layer3_attention_weight_max": 41.288414001464844,
      "activations/layer3_attention_weight_min": -33.11074447631836,
      "activations/layer4_attention_weight_max": 96.88099670410156,
      "activations/layer4_attention_weight_min": -95.0626449584961,
      "activations/layer5_attention_weight_max": 147.82504272460938,
      "activations/layer5_attention_weight_min": -156.51626586914062,
      "activations/layer6_attention_weight_max": 147.81846618652344,
      "activations/layer6_attention_weight_min": -160.54507446289062,
      "activations/layer7_attention_weight_max": 103.57038116455078,
      "activations/layer7_attention_weight_min": -110.60565948486328,
      "activations/layer8_attention_weight_max": 102.83710479736328,
      "activations/layer8_attention_weight_min": -111.3525619506836,
      "activations/layer9_attention_weight_max": 58.17037582397461,
      "activations/layer9_attention_weight_min": -58.63601303100586,
      "epoch": 0.05,
      "learning_rate": 0.00012168896205813997,
      "loss": 4.8919,
      "step": 850
    },
    {
      "activations/layer0_attention_weight_max": 12.064840316772461,
      "activations/layer0_attention_weight_min": -8.805413246154785,
      "activations/layer10_attention_weight_max": 51.643733978271484,
      "activations/layer10_attention_weight_min": -41.66896057128906,
      "activations/layer11_attention_weight_max": 24.855892181396484,
      "activations/layer11_attention_weight_min": -15.791610717773438,
      "activations/layer12_attention_weight_max": 13.363130569458008,
      "activations/layer12_attention_weight_min": -10.795004844665527,
      "activations/layer13_attention_weight_max": 10.390249252319336,
      "activations/layer13_attention_weight_min": -8.234644889831543,
      "activations/layer14_attention_weight_max": 12.995079040527344,
      "activations/layer14_attention_weight_min": -7.895799160003662,
      "activations/layer15_attention_weight_max": 13.539955139160156,
      "activations/layer15_attention_weight_min": -7.453927040100098,
      "activations/layer16_attention_weight_max": 11.36599063873291,
      "activations/layer16_attention_weight_min": -6.204162120819092,
      "activations/layer17_attention_weight_max": 10.902265548706055,
      "activations/layer17_attention_weight_min": -9.302597999572754,
      "activations/layer18_attention_weight_max": 11.676994323730469,
      "activations/layer18_attention_weight_min": -6.628722667694092,
      "activations/layer19_attention_weight_max": 5.002374172210693,
      "activations/layer19_attention_weight_min": -3.81565523147583,
      "activations/layer1_attention_weight_max": 19.638572692871094,
      "activations/layer1_attention_weight_min": -16.622604370117188,
      "activations/layer20_attention_weight_max": 7.860724449157715,
      "activations/layer20_attention_weight_min": -5.604986667633057,
      "activations/layer21_attention_weight_max": 5.449520111083984,
      "activations/layer21_attention_weight_min": -3.4553678035736084,
      "activations/layer22_attention_weight_max": 2.7606077194213867,
      "activations/layer22_attention_weight_min": -3.014244556427002,
      "activations/layer23_attention_weight_max": 9.458892822265625,
      "activations/layer23_attention_weight_min": -4.0927019119262695,
      "activations/layer2_attention_weight_max": 17.960952758789062,
      "activations/layer2_attention_weight_min": -12.465042114257812,
      "activations/layer3_attention_weight_max": 42.35795211791992,
      "activations/layer3_attention_weight_min": -35.2315788269043,
      "activations/layer4_attention_weight_max": 99.2356948852539,
      "activations/layer4_attention_weight_min": -94.341064453125,
      "activations/layer5_attention_weight_max": 151.14996337890625,
      "activations/layer5_attention_weight_min": -158.5092010498047,
      "activations/layer6_attention_weight_max": 150.43020629882812,
      "activations/layer6_attention_weight_min": -160.33200073242188,
      "activations/layer7_attention_weight_max": 108.63585662841797,
      "activations/layer7_attention_weight_min": -110.595458984375,
      "activations/layer8_attention_weight_max": 106.21036529541016,
      "activations/layer8_attention_weight_min": -108.4703140258789,
      "activations/layer9_attention_weight_max": 58.633331298828125,
      "activations/layer9_attention_weight_min": -54.93046188354492,
      "epoch": 0.05,
      "learning_rate": 0.0001227395044356449,
      "loss": 4.8254,
      "step": 900
    },
    {
      "activations/layer0_attention_weight_max": 11.624129295349121,
      "activations/layer0_attention_weight_min": -8.324885368347168,
      "activations/layer10_attention_weight_max": 54.24721145629883,
      "activations/layer10_attention_weight_min": -45.86784362792969,
      "activations/layer11_attention_weight_max": 23.2523193359375,
      "activations/layer11_attention_weight_min": -20.20481300354004,
      "activations/layer12_attention_weight_max": 14.275994300842285,
      "activations/layer12_attention_weight_min": -12.62340259552002,
      "activations/layer13_attention_weight_max": 10.159871101379395,
      "activations/layer13_attention_weight_min": -8.88941478729248,
      "activations/layer14_attention_weight_max": 15.822487831115723,
      "activations/layer14_attention_weight_min": -9.432673454284668,
      "activations/layer15_attention_weight_max": 13.564722061157227,
      "activations/layer15_attention_weight_min": -8.118111610412598,
      "activations/layer16_attention_weight_max": 12.696914672851562,
      "activations/layer16_attention_weight_min": -7.1076741218566895,
      "activations/layer17_attention_weight_max": 12.590985298156738,
      "activations/layer17_attention_weight_min": -9.021801948547363,
      "activations/layer18_attention_weight_max": 10.458784103393555,
      "activations/layer18_attention_weight_min": -6.413854122161865,
      "activations/layer19_attention_weight_max": 4.9758429527282715,
      "activations/layer19_attention_weight_min": -3.9736855030059814,
      "activations/layer1_attention_weight_max": 20.161165237426758,
      "activations/layer1_attention_weight_min": -16.654510498046875,
      "activations/layer20_attention_weight_max": 9.73802375793457,
      "activations/layer20_attention_weight_min": -6.586221694946289,
      "activations/layer21_attention_weight_max": 5.328779697418213,
      "activations/layer21_attention_weight_min": -3.3862361907958984,
      "activations/layer22_attention_weight_max": 2.828991413116455,
      "activations/layer22_attention_weight_min": -2.7137136459350586,
      "activations/layer23_attention_weight_max": 10.375448226928711,
      "activations/layer23_attention_weight_min": -4.243715286254883,
      "activations/layer2_attention_weight_max": 17.80264663696289,
      "activations/layer2_attention_weight_min": -13.375025749206543,
      "activations/layer3_attention_weight_max": 43.97316360473633,
      "activations/layer3_attention_weight_min": -36.58658981323242,
      "activations/layer4_attention_weight_max": 102.52203369140625,
      "activations/layer4_attention_weight_min": -98.10376739501953,
      "activations/layer5_attention_weight_max": 162.4659423828125,
      "activations/layer5_attention_weight_min": -166.26499938964844,
      "activations/layer6_attention_weight_max": 160.70530700683594,
      "activations/layer6_attention_weight_min": -164.67483520507812,
      "activations/layer7_attention_weight_max": 114.05121612548828,
      "activations/layer7_attention_weight_min": -115.01268768310547,
      "activations/layer8_attention_weight_max": 115.4356689453125,
      "activations/layer8_attention_weight_min": -114.338134765625,
      "activations/layer9_attention_weight_max": 62.61403274536133,
      "activations/layer9_attention_weight_min": -57.65483474731445,
      "epoch": 0.06,
      "learning_rate": 0.0001237323582571438,
      "loss": 4.7636,
      "step": 950
    },
    {
      "activations/layer0_attention_weight_max": 10.641590118408203,
      "activations/layer0_attention_weight_min": -8.31135082244873,
      "activations/layer10_attention_weight_max": 50.23035430908203,
      "activations/layer10_attention_weight_min": -42.31828689575195,
      "activations/layer11_attention_weight_max": 22.813756942749023,
      "activations/layer11_attention_weight_min": -16.398902893066406,
      "activations/layer12_attention_weight_max": 13.928257942199707,
      "activations/layer12_attention_weight_min": -13.178764343261719,
      "activations/layer13_attention_weight_max": 11.598997116088867,
      "activations/layer13_attention_weight_min": -9.992827415466309,
      "activations/layer14_attention_weight_max": 14.123250007629395,
      "activations/layer14_attention_weight_min": -10.664250373840332,
      "activations/layer15_attention_weight_max": 12.848624229431152,
      "activations/layer15_attention_weight_min": -8.7907075881958,
      "activations/layer16_attention_weight_max": 11.471288681030273,
      "activations/layer16_attention_weight_min": -6.926945209503174,
      "activations/layer17_attention_weight_max": 12.677962303161621,
      "activations/layer17_attention_weight_min": -8.152668952941895,
      "activations/layer18_attention_weight_max": 10.637674331665039,
      "activations/layer18_attention_weight_min": -6.037156105041504,
      "activations/layer19_attention_weight_max": 5.3200602531433105,
      "activations/layer19_attention_weight_min": -4.044188976287842,
      "activations/layer1_attention_weight_max": 21.20311164855957,
      "activations/layer1_attention_weight_min": -17.112821578979492,
      "activations/layer20_attention_weight_max": 10.919744491577148,
      "activations/layer20_attention_weight_min": -7.325877666473389,
      "activations/layer21_attention_weight_max": 5.5825514793396,
      "activations/layer21_attention_weight_min": -3.447089910507202,
      "activations/layer22_attention_weight_max": 2.7951643466949463,
      "activations/layer22_attention_weight_min": -2.945775032043457,
      "activations/layer23_attention_weight_max": 9.564732551574707,
      "activations/layer23_attention_weight_min": -4.1737470626831055,
      "activations/layer2_attention_weight_max": 17.77175521850586,
      "activations/layer2_attention_weight_min": -12.661895751953125,
      "activations/layer3_attention_weight_max": 43.98899841308594,
      "activations/layer3_attention_weight_min": -35.51685333251953,
      "activations/layer4_attention_weight_max": 95.02120971679688,
      "activations/layer4_attention_weight_min": -96.24844360351562,
      "activations/layer5_attention_weight_max": 143.40426635742188,
      "activations/layer5_attention_weight_min": -159.6646728515625,
      "activations/layer6_attention_weight_max": 141.55332946777344,
      "activations/layer6_attention_weight_min": -161.5238800048828,
      "activations/layer7_attention_weight_max": 101.5566635131836,
      "activations/layer7_attention_weight_min": -112.95075988769531,
      "activations/layer8_attention_weight_max": 97.04559326171875,
      "activations/layer8_attention_weight_min": -107.39598083496094,
      "activations/layer9_attention_weight_max": 57.744056701660156,
      "activations/layer9_attention_weight_min": -55.89616775512695,
      "epoch": 0.06,
      "learning_rate": 0.00012467353079136803,
      "loss": 4.7116,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_loss": 4.60546875,
      "eval_runtime": 8.2745,
      "eval_samples_per_second": 518.945,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_openwebtext_loss": 4.60546875,
      "eval_openwebtext_ppl": 100.02986085865793,
      "eval_openwebtext_runtime": 8.2745,
      "eval_openwebtext_samples_per_second": 518.945,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_wikitext_loss": 5.15234375,
      "eval_wikitext_ppl": 172.83610058973207,
      "eval_wikitext_runtime": 1.6766,
      "eval_wikitext_samples_per_second": 271.986,
      "step": 1000
    },
    {
      "epoch": 0.06,
      "eval_lambada_loss": 8.5,
      "eval_lambada_ppl": 4914.768840299134,
      "eval_lambada_runtime": 9.2703,
      "eval_lambada_samples_per_second": 525.225,
      "step": 1000
    },
    {
      "activations/layer0_attention_weight_max": 11.381598472595215,
      "activations/layer0_attention_weight_min": -8.757445335388184,
      "activations/layer10_attention_weight_max": 49.91594314575195,
      "activations/layer10_attention_weight_min": -51.61308288574219,
      "activations/layer11_attention_weight_max": 22.947895050048828,
      "activations/layer11_attention_weight_min": -16.10309600830078,
      "activations/layer12_attention_weight_max": 14.734343528747559,
      "activations/layer12_attention_weight_min": -14.864287376403809,
      "activations/layer13_attention_weight_max": 11.16040325164795,
      "activations/layer13_attention_weight_min": -11.15569019317627,
      "activations/layer14_attention_weight_max": 15.161927223205566,
      "activations/layer14_attention_weight_min": -10.873896598815918,
      "activations/layer15_attention_weight_max": 12.488495826721191,
      "activations/layer15_attention_weight_min": -8.97653579711914,
      "activations/layer16_attention_weight_max": 12.859784126281738,
      "activations/layer16_attention_weight_min": -7.283934116363525,
      "activations/layer17_attention_weight_max": 13.375898361206055,
      "activations/layer17_attention_weight_min": -10.1507568359375,
      "activations/layer18_attention_weight_max": 10.999382019042969,
      "activations/layer18_attention_weight_min": -6.686851501464844,
      "activations/layer19_attention_weight_max": 5.8024516105651855,
      "activations/layer19_attention_weight_min": -4.50338888168335,
      "activations/layer1_attention_weight_max": 20.418899536132812,
      "activations/layer1_attention_weight_min": -15.626005172729492,
      "activations/layer20_attention_weight_max": 12.272616386413574,
      "activations/layer20_attention_weight_min": -6.849844932556152,
      "activations/layer21_attention_weight_max": 5.555975437164307,
      "activations/layer21_attention_weight_min": -3.69227933883667,
      "activations/layer22_attention_weight_max": 3.2103397846221924,
      "activations/layer22_attention_weight_min": -2.8227052688598633,
      "activations/layer23_attention_weight_max": 10.114577293395996,
      "activations/layer23_attention_weight_min": -4.321666240692139,
      "activations/layer2_attention_weight_max": 19.19073486328125,
      "activations/layer2_attention_weight_min": -14.312461853027344,
      "activations/layer3_attention_weight_max": 43.065162658691406,
      "activations/layer3_attention_weight_min": -36.92966079711914,
      "activations/layer4_attention_weight_max": 94.57568359375,
      "activations/layer4_attention_weight_min": -96.2518539428711,
      "activations/layer5_attention_weight_max": 147.85060119628906,
      "activations/layer5_attention_weight_min": -163.55911254882812,
      "activations/layer6_attention_weight_max": 148.02264404296875,
      "activations/layer6_attention_weight_min": -167.44769287109375,
      "activations/layer7_attention_weight_max": 108.21299743652344,
      "activations/layer7_attention_weight_min": -119.0129623413086,
      "activations/layer8_attention_weight_max": 101.07582092285156,
      "activations/layer8_attention_weight_min": -112.77936553955078,
      "activations/layer9_attention_weight_max": 59.17750930786133,
      "activations/layer9_attention_weight_min": -63.8360481262207,
      "epoch": 0.06,
      "learning_rate": 0.0001255506719926155,
      "loss": 4.6626,
      "step": 1050
    },
    {
      "activations/layer0_attention_weight_max": 12.078857421875,
      "activations/layer0_attention_weight_min": -9.179473876953125,
      "activations/layer10_attention_weight_max": 54.71592330932617,
      "activations/layer10_attention_weight_min": -45.76498031616211,
      "activations/layer11_attention_weight_max": 21.126928329467773,
      "activations/layer11_attention_weight_min": -18.123310089111328,
      "activations/layer12_attention_weight_max": 17.550735473632812,
      "activations/layer12_attention_weight_min": -18.17275047302246,
      "activations/layer13_attention_weight_max": 15.833667755126953,
      "activations/layer13_attention_weight_min": -11.856423377990723,
      "activations/layer14_attention_weight_max": 14.405364990234375,
      "activations/layer14_attention_weight_min": -11.404964447021484,
      "activations/layer15_attention_weight_max": 16.542800903320312,
      "activations/layer15_attention_weight_min": -9.315685272216797,
      "activations/layer16_attention_weight_max": 14.699055671691895,
      "activations/layer16_attention_weight_min": -6.301263809204102,
      "activations/layer17_attention_weight_max": 13.627508163452148,
      "activations/layer17_attention_weight_min": -8.908482551574707,
      "activations/layer18_attention_weight_max": 14.970430374145508,
      "activations/layer18_attention_weight_min": -6.821336269378662,
      "activations/layer19_attention_weight_max": 5.507801055908203,
      "activations/layer19_attention_weight_min": -3.7978241443634033,
      "activations/layer1_attention_weight_max": 19.436359405517578,
      "activations/layer1_attention_weight_min": -14.755937576293945,
      "activations/layer20_attention_weight_max": 12.541598320007324,
      "activations/layer20_attention_weight_min": -5.980596542358398,
      "activations/layer21_attention_weight_max": 6.043969631195068,
      "activations/layer21_attention_weight_min": -3.332524061203003,
      "activations/layer22_attention_weight_max": 2.9894447326660156,
      "activations/layer22_attention_weight_min": -2.645134687423706,
      "activations/layer23_attention_weight_max": 11.225841522216797,
      "activations/layer23_attention_weight_min": -4.522704601287842,
      "activations/layer2_attention_weight_max": 20.060216903686523,
      "activations/layer2_attention_weight_min": -14.108924865722656,
      "activations/layer3_attention_weight_max": 47.240055084228516,
      "activations/layer3_attention_weight_min": -38.39889907836914,
      "activations/layer4_attention_weight_max": 102.72882080078125,
      "activations/layer4_attention_weight_min": -98.22081756591797,
      "activations/layer5_attention_weight_max": 161.98486328125,
      "activations/layer5_attention_weight_min": -169.85812377929688,
      "activations/layer6_attention_weight_max": 162.17742919921875,
      "activations/layer6_attention_weight_min": -170.1479034423828,
      "activations/layer7_attention_weight_max": 115.06095123291016,
      "activations/layer7_attention_weight_min": -119.8990478515625,
      "activations/layer8_attention_weight_max": 100.73880767822266,
      "activations/layer8_attention_weight_min": -105.70518493652344,
      "activations/layer9_attention_weight_max": 63.894771575927734,
      "activations/layer9_attention_weight_min": -60.73427963256836,
      "epoch": 0.06,
      "learning_rate": 0.00012640390827022916,
      "loss": 4.6048,
      "step": 1100
    },
    {
      "activations/layer0_attention_weight_max": 10.695474624633789,
      "activations/layer0_attention_weight_min": -9.083545684814453,
      "activations/layer10_attention_weight_max": 47.1948356628418,
      "activations/layer10_attention_weight_min": -49.436859130859375,
      "activations/layer11_attention_weight_max": 20.001270294189453,
      "activations/layer11_attention_weight_min": -15.338567733764648,
      "activations/layer12_attention_weight_max": 17.89560317993164,
      "activations/layer12_attention_weight_min": -19.360393524169922,
      "activations/layer13_attention_weight_max": 13.03063678741455,
      "activations/layer13_attention_weight_min": -11.85488224029541,
      "activations/layer14_attention_weight_max": 14.875910758972168,
      "activations/layer14_attention_weight_min": -11.523996353149414,
      "activations/layer15_attention_weight_max": 14.444130897521973,
      "activations/layer15_attention_weight_min": -9.407524108886719,
      "activations/layer16_attention_weight_max": 13.802813529968262,
      "activations/layer16_attention_weight_min": -7.831554889678955,
      "activations/layer17_attention_weight_max": 11.67714786529541,
      "activations/layer17_attention_weight_min": -7.832913875579834,
      "activations/layer18_attention_weight_max": 11.423389434814453,
      "activations/layer18_attention_weight_min": -7.681760311126709,
      "activations/layer19_attention_weight_max": 5.427567958831787,
      "activations/layer19_attention_weight_min": -4.1566691398620605,
      "activations/layer1_attention_weight_max": 19.35671615600586,
      "activations/layer1_attention_weight_min": -14.09879207611084,
      "activations/layer20_attention_weight_max": 13.600817680358887,
      "activations/layer20_attention_weight_min": -7.440022945404053,
      "activations/layer21_attention_weight_max": 5.82479190826416,
      "activations/layer21_attention_weight_min": -3.639516830444336,
      "activations/layer22_attention_weight_max": 2.9807686805725098,
      "activations/layer22_attention_weight_min": -2.933008909225464,
      "activations/layer23_attention_weight_max": 10.592529296875,
      "activations/layer23_attention_weight_min": -5.550187587738037,
      "activations/layer2_attention_weight_max": 19.154767990112305,
      "activations/layer2_attention_weight_min": -14.91831111907959,
      "activations/layer3_attention_weight_max": 44.904232025146484,
      "activations/layer3_attention_weight_min": -43.34779357910156,
      "activations/layer4_attention_weight_max": 95.51313018798828,
      "activations/layer4_attention_weight_min": -105.71595764160156,
      "activations/layer5_attention_weight_max": 146.60426330566406,
      "activations/layer5_attention_weight_min": -178.28494262695312,
      "activations/layer6_attention_weight_max": 142.12863159179688,
      "activations/layer6_attention_weight_min": -182.3129425048828,
      "activations/layer7_attention_weight_max": 102.8539047241211,
      "activations/layer7_attention_weight_min": -122.55033874511719,
      "activations/layer8_attention_weight_max": 89.03264617919922,
      "activations/layer8_attention_weight_min": -110.64627838134766,
      "activations/layer9_attention_weight_max": 55.48357009887695,
      "activations/layer9_attention_weight_min": -66.6719741821289,
      "epoch": 0.07,
      "learning_rate": 0.00012721869717123439,
      "loss": 4.5445,
      "step": 1150
    },
    {
      "activations/layer0_attention_weight_max": 12.574724197387695,
      "activations/layer0_attention_weight_min": -9.533120155334473,
      "activations/layer10_attention_weight_max": 48.1885986328125,
      "activations/layer10_attention_weight_min": -47.120574951171875,
      "activations/layer11_attention_weight_max": 21.50453758239746,
      "activations/layer11_attention_weight_min": -15.473189353942871,
      "activations/layer12_attention_weight_max": 18.304548263549805,
      "activations/layer12_attention_weight_min": -20.480985641479492,
      "activations/layer13_attention_weight_max": 11.935996055603027,
      "activations/layer13_attention_weight_min": -11.661901473999023,
      "activations/layer14_attention_weight_max": 18.26370620727539,
      "activations/layer14_attention_weight_min": -11.998866081237793,
      "activations/layer15_attention_weight_max": 18.26985740661621,
      "activations/layer15_attention_weight_min": -9.942241668701172,
      "activations/layer16_attention_weight_max": 12.4179048538208,
      "activations/layer16_attention_weight_min": -8.355260848999023,
      "activations/layer17_attention_weight_max": 12.633147239685059,
      "activations/layer17_attention_weight_min": -8.91390323638916,
      "activations/layer18_attention_weight_max": 12.84876823425293,
      "activations/layer18_attention_weight_min": -7.745028972625732,
      "activations/layer19_attention_weight_max": 6.250151634216309,
      "activations/layer19_attention_weight_min": -4.555910110473633,
      "activations/layer1_attention_weight_max": 20.98055076599121,
      "activations/layer1_attention_weight_min": -16.05783462524414,
      "activations/layer20_attention_weight_max": 12.764229774475098,
      "activations/layer20_attention_weight_min": -7.738099098205566,
      "activations/layer21_attention_weight_max": 5.5713958740234375,
      "activations/layer21_attention_weight_min": -3.703429698944092,
      "activations/layer22_attention_weight_max": 3.2836616039276123,
      "activations/layer22_attention_weight_min": -3.13188099861145,
      "activations/layer23_attention_weight_max": 11.347414016723633,
      "activations/layer23_attention_weight_min": -6.720239162445068,
      "activations/layer2_attention_weight_max": 19.338409423828125,
      "activations/layer2_attention_weight_min": -14.423972129821777,
      "activations/layer3_attention_weight_max": 47.146095275878906,
      "activations/layer3_attention_weight_min": -40.82805252075195,
      "activations/layer4_attention_weight_max": 97.3131332397461,
      "activations/layer4_attention_weight_min": -97.77656555175781,
      "activations/layer5_attention_weight_max": 153.02899169921875,
      "activations/layer5_attention_weight_min": -170.89584350585938,
      "activations/layer6_attention_weight_max": 153.15176391601562,
      "activations/layer6_attention_weight_min": -168.89291381835938,
      "activations/layer7_attention_weight_max": 107.3448257446289,
      "activations/layer7_attention_weight_min": -121.04551696777344,
      "activations/layer8_attention_weight_max": 92.07583618164062,
      "activations/layer8_attention_weight_min": -105.84859466552734,
      "activations/layer9_attention_weight_max": 60.29525375366211,
      "activations/layer9_attention_weight_min": -64.60922241210938,
      "epoch": 0.07,
      "learning_rate": 0.0001279983547364355,
      "loss": 4.4795,
      "step": 1200
    },
    {
      "activations/layer0_attention_weight_max": 10.966246604919434,
      "activations/layer0_attention_weight_min": -9.058398246765137,
      "activations/layer10_attention_weight_max": 59.954219818115234,
      "activations/layer10_attention_weight_min": -51.06071472167969,
      "activations/layer11_attention_weight_max": 22.828907012939453,
      "activations/layer11_attention_weight_min": -15.577972412109375,
      "activations/layer12_attention_weight_max": 18.245407104492188,
      "activations/layer12_attention_weight_min": -21.2323055267334,
      "activations/layer13_attention_weight_max": 14.666379928588867,
      "activations/layer13_attention_weight_min": -13.140233993530273,
      "activations/layer14_attention_weight_max": 17.29130744934082,
      "activations/layer14_attention_weight_min": -11.893112182617188,
      "activations/layer15_attention_weight_max": 14.491522789001465,
      "activations/layer15_attention_weight_min": -10.909011840820312,
      "activations/layer16_attention_weight_max": 13.876253128051758,
      "activations/layer16_attention_weight_min": -7.99700927734375,
      "activations/layer17_attention_weight_max": 13.72429370880127,
      "activations/layer17_attention_weight_min": -9.228825569152832,
      "activations/layer18_attention_weight_max": 13.442391395568848,
      "activations/layer18_attention_weight_min": -8.576897621154785,
      "activations/layer19_attention_weight_max": 6.471092224121094,
      "activations/layer19_attention_weight_min": -5.19780158996582,
      "activations/layer1_attention_weight_max": 19.360977172851562,
      "activations/layer1_attention_weight_min": -16.388595581054688,
      "activations/layer20_attention_weight_max": 12.130026817321777,
      "activations/layer20_attention_weight_min": -6.674371242523193,
      "activations/layer21_attention_weight_max": 6.411074638366699,
      "activations/layer21_attention_weight_min": -3.6642515659332275,
      "activations/layer22_attention_weight_max": 3.2522616386413574,
      "activations/layer22_attention_weight_min": -3.1917662620544434,
      "activations/layer23_attention_weight_max": 11.000761985778809,
      "activations/layer23_attention_weight_min": -6.2310709953308105,
      "activations/layer2_attention_weight_max": 20.779754638671875,
      "activations/layer2_attention_weight_min": -15.474024772644043,
      "activations/layer3_attention_weight_max": 50.474002838134766,
      "activations/layer3_attention_weight_min": -40.390724182128906,
      "activations/layer4_attention_weight_max": 104.14847564697266,
      "activations/layer4_attention_weight_min": -99.28836059570312,
      "activations/layer5_attention_weight_max": 166.32435607910156,
      "activations/layer5_attention_weight_min": -170.68875122070312,
      "activations/layer6_attention_weight_max": 165.22494506835938,
      "activations/layer6_attention_weight_min": -173.52862548828125,
      "activations/layer7_attention_weight_max": 121.40557098388672,
      "activations/layer7_attention_weight_min": -124.89720916748047,
      "activations/layer8_attention_weight_max": 104.08271789550781,
      "activations/layer8_attention_weight_min": -105.01110076904297,
      "activations/layer9_attention_weight_max": 73.39389038085938,
      "activations/layer9_attention_weight_min": -67.51972961425781,
      "epoch": 0.07,
      "learning_rate": 0.0001287457856770406,
      "loss": 4.4337,
      "step": 1250
    },
    {
      "activations/layer0_attention_weight_max": 12.184757232666016,
      "activations/layer0_attention_weight_min": -9.109187126159668,
      "activations/layer10_attention_weight_max": 57.76918411254883,
      "activations/layer10_attention_weight_min": -54.11812210083008,
      "activations/layer11_attention_weight_max": 25.094898223876953,
      "activations/layer11_attention_weight_min": -16.000083923339844,
      "activations/layer12_attention_weight_max": 20.118799209594727,
      "activations/layer12_attention_weight_min": -21.91644287109375,
      "activations/layer13_attention_weight_max": 16.95064926147461,
      "activations/layer13_attention_weight_min": -13.597829818725586,
      "activations/layer14_attention_weight_max": 17.771474838256836,
      "activations/layer14_attention_weight_min": -12.448955535888672,
      "activations/layer15_attention_weight_max": 14.036105155944824,
      "activations/layer15_attention_weight_min": -10.635589599609375,
      "activations/layer16_attention_weight_max": 12.613263130187988,
      "activations/layer16_attention_weight_min": -8.47629165649414,
      "activations/layer17_attention_weight_max": 13.235357284545898,
      "activations/layer17_attention_weight_min": -9.459239959716797,
      "activations/layer18_attention_weight_max": 11.322538375854492,
      "activations/layer18_attention_weight_min": -8.329571723937988,
      "activations/layer19_attention_weight_max": 6.417878150939941,
      "activations/layer19_attention_weight_min": -4.895137310028076,
      "activations/layer1_attention_weight_max": 17.400161743164062,
      "activations/layer1_attention_weight_min": -14.275753021240234,
      "activations/layer20_attention_weight_max": 14.307768821716309,
      "activations/layer20_attention_weight_min": -8.400795936584473,
      "activations/layer21_attention_weight_max": 6.120568752288818,
      "activations/layer21_attention_weight_min": -4.545230865478516,
      "activations/layer22_attention_weight_max": 3.176379680633545,
      "activations/layer22_attention_weight_min": -3.5598068237304688,
      "activations/layer23_attention_weight_max": 12.063577651977539,
      "activations/layer23_attention_weight_min": -7.208837032318115,
      "activations/layer2_attention_weight_max": 19.308992385864258,
      "activations/layer2_attention_weight_min": -16.194408416748047,
      "activations/layer3_attention_weight_max": 51.14238739013672,
      "activations/layer3_attention_weight_min": -43.86634063720703,
      "activations/layer4_attention_weight_max": 100.64352416992188,
      "activations/layer4_attention_weight_min": -99.77770233154297,
      "activations/layer5_attention_weight_max": 158.85487365722656,
      "activations/layer5_attention_weight_min": -171.7268524169922,
      "activations/layer6_attention_weight_max": 161.9989471435547,
      "activations/layer6_attention_weight_min": -179.59658813476562,
      "activations/layer7_attention_weight_max": 123.57054138183594,
      "activations/layer7_attention_weight_min": -126.89019012451172,
      "activations/layer8_attention_weight_max": 97.27787780761719,
      "activations/layer8_attention_weight_min": -106.1046371459961,
      "activations/layer9_attention_weight_max": 69.62931060791016,
      "activations/layer9_attention_weight_min": -72.734375,
      "epoch": 0.08,
      "learning_rate": 0.00012946354871179542,
      "loss": 4.37,
      "step": 1300
    },
    {
      "activations/layer0_attention_weight_max": 13.838836669921875,
      "activations/layer0_attention_weight_min": -8.701981544494629,
      "activations/layer10_attention_weight_max": 55.208580017089844,
      "activations/layer10_attention_weight_min": -52.13080596923828,
      "activations/layer11_attention_weight_max": 24.748859405517578,
      "activations/layer11_attention_weight_min": -16.658967971801758,
      "activations/layer12_attention_weight_max": 19.521286010742188,
      "activations/layer12_attention_weight_min": -24.08013916015625,
      "activations/layer13_attention_weight_max": 20.646495819091797,
      "activations/layer13_attention_weight_min": -13.501294136047363,
      "activations/layer14_attention_weight_max": 18.884292602539062,
      "activations/layer14_attention_weight_min": -11.875212669372559,
      "activations/layer15_attention_weight_max": 16.498376846313477,
      "activations/layer15_attention_weight_min": -10.023565292358398,
      "activations/layer16_attention_weight_max": 12.959400177001953,
      "activations/layer16_attention_weight_min": -7.1444292068481445,
      "activations/layer17_attention_weight_max": 13.819160461425781,
      "activations/layer17_attention_weight_min": -10.103763580322266,
      "activations/layer18_attention_weight_max": 12.27629280090332,
      "activations/layer18_attention_weight_min": -7.607575416564941,
      "activations/layer19_attention_weight_max": 5.769844055175781,
      "activations/layer19_attention_weight_min": -4.74537992477417,
      "activations/layer1_attention_weight_max": 18.628795623779297,
      "activations/layer1_attention_weight_min": -13.821975708007812,
      "activations/layer20_attention_weight_max": 11.474924087524414,
      "activations/layer20_attention_weight_min": -6.698044776916504,
      "activations/layer21_attention_weight_max": 6.047966957092285,
      "activations/layer21_attention_weight_min": -3.9726951122283936,
      "activations/layer22_attention_weight_max": 3.7314844131469727,
      "activations/layer22_attention_weight_min": -3.3210203647613525,
      "activations/layer23_attention_weight_max": 10.19915771484375,
      "activations/layer23_attention_weight_min": -6.258851051330566,
      "activations/layer2_attention_weight_max": 21.31755256652832,
      "activations/layer2_attention_weight_min": -14.283220291137695,
      "activations/layer3_attention_weight_max": 53.02455139160156,
      "activations/layer3_attention_weight_min": -43.73554229736328,
      "activations/layer4_attention_weight_max": 103.3116226196289,
      "activations/layer4_attention_weight_min": -101.77572631835938,
      "activations/layer5_attention_weight_max": 162.81744384765625,
      "activations/layer5_attention_weight_min": -180.9136199951172,
      "activations/layer6_attention_weight_max": 162.52622985839844,
      "activations/layer6_attention_weight_min": -178.02784729003906,
      "activations/layer7_attention_weight_max": 119.31690216064453,
      "activations/layer7_attention_weight_min": -126.997802734375,
      "activations/layer8_attention_weight_max": 96.46717834472656,
      "activations/layer8_attention_weight_min": -100.78907012939453,
      "activations/layer9_attention_weight_max": 75.22552490234375,
      "activations/layer9_attention_weight_min": -68.9813003540039,
      "epoch": 0.08,
      "learning_rate": 0.0001301539094251036,
      "loss": 4.324,
      "step": 1350
    },
    {
      "activations/layer0_attention_weight_max": 11.349065780639648,
      "activations/layer0_attention_weight_min": -9.572500228881836,
      "activations/layer10_attention_weight_max": 57.05692672729492,
      "activations/layer10_attention_weight_min": -50.1307258605957,
      "activations/layer11_attention_weight_max": 21.824005126953125,
      "activations/layer11_attention_weight_min": -15.87234878540039,
      "activations/layer12_attention_weight_max": 18.545795440673828,
      "activations/layer12_attention_weight_min": -24.798662185668945,
      "activations/layer13_attention_weight_max": 15.317700386047363,
      "activations/layer13_attention_weight_min": -14.72300910949707,
      "activations/layer14_attention_weight_max": 15.884514808654785,
      "activations/layer14_attention_weight_min": -14.017348289489746,
      "activations/layer15_attention_weight_max": 13.927318572998047,
      "activations/layer15_attention_weight_min": -10.62076473236084,
      "activations/layer16_attention_weight_max": 12.122371673583984,
      "activations/layer16_attention_weight_min": -8.381132125854492,
      "activations/layer17_attention_weight_max": 14.055788040161133,
      "activations/layer17_attention_weight_min": -10.447295188903809,
      "activations/layer18_attention_weight_max": 14.750917434692383,
      "activations/layer18_attention_weight_min": -7.237926483154297,
      "activations/layer19_attention_weight_max": 6.780613899230957,
      "activations/layer19_attention_weight_min": -5.299167633056641,
      "activations/layer1_attention_weight_max": 18.79485321044922,
      "activations/layer1_attention_weight_min": -14.080233573913574,
      "activations/layer20_attention_weight_max": 10.662460327148438,
      "activations/layer20_attention_weight_min": -6.5464253425598145,
      "activations/layer21_attention_weight_max": 6.299347877502441,
      "activations/layer21_attention_weight_min": -4.74904203414917,
      "activations/layer22_attention_weight_max": 4.003003120422363,
      "activations/layer22_attention_weight_min": -3.3014976978302,
      "activations/layer23_attention_weight_max": 11.254988670349121,
      "activations/layer23_attention_weight_min": -7.142792701721191,
      "activations/layer2_attention_weight_max": 20.223163604736328,
      "activations/layer2_attention_weight_min": -15.863086700439453,
      "activations/layer3_attention_weight_max": 53.28541564941406,
      "activations/layer3_attention_weight_min": -46.55379104614258,
      "activations/layer4_attention_weight_max": 103.17484283447266,
      "activations/layer4_attention_weight_min": -98.2845687866211,
      "activations/layer5_attention_weight_max": 169.921142578125,
      "activations/layer5_attention_weight_min": -174.83639526367188,
      "activations/layer6_attention_weight_max": 173.0667266845703,
      "activations/layer6_attention_weight_min": -175.53570556640625,
      "activations/layer7_attention_weight_max": 120.13572692871094,
      "activations/layer7_attention_weight_min": -126.21004486083984,
      "activations/layer8_attention_weight_max": 92.0516357421875,
      "activations/layer8_attention_weight_min": -102.42212677001953,
      "activations/layer9_attention_weight_max": 67.05745697021484,
      "activations/layer9_attention_weight_min": -67.62863159179688,
      "epoch": 0.08,
      "learning_rate": 0.0001308188834013138,
      "loss": 4.2716,
      "step": 1400
    },
    {
      "activations/layer0_attention_weight_max": 11.5694580078125,
      "activations/layer0_attention_weight_min": -9.786859512329102,
      "activations/layer10_attention_weight_max": 54.792388916015625,
      "activations/layer10_attention_weight_min": -50.261409759521484,
      "activations/layer11_attention_weight_max": 24.923324584960938,
      "activations/layer11_attention_weight_min": -15.613348007202148,
      "activations/layer12_attention_weight_max": 19.292980194091797,
      "activations/layer12_attention_weight_min": -25.689956665039062,
      "activations/layer13_attention_weight_max": 16.932109832763672,
      "activations/layer13_attention_weight_min": -13.76589298248291,
      "activations/layer14_attention_weight_max": 20.904644012451172,
      "activations/layer14_attention_weight_min": -11.714534759521484,
      "activations/layer15_attention_weight_max": 17.975202560424805,
      "activations/layer15_attention_weight_min": -9.715566635131836,
      "activations/layer16_attention_weight_max": 15.21570873260498,
      "activations/layer16_attention_weight_min": -8.034368515014648,
      "activations/layer17_attention_weight_max": 13.319670677185059,
      "activations/layer17_attention_weight_min": -11.0245943069458,
      "activations/layer18_attention_weight_max": 14.39918327331543,
      "activations/layer18_attention_weight_min": -9.015694618225098,
      "activations/layer19_attention_weight_max": 7.307714939117432,
      "activations/layer19_attention_weight_min": -5.014240264892578,
      "activations/layer1_attention_weight_max": 16.479679107666016,
      "activations/layer1_attention_weight_min": -12.321990966796875,
      "activations/layer20_attention_weight_max": 11.375825881958008,
      "activations/layer20_attention_weight_min": -6.6881513595581055,
      "activations/layer21_attention_weight_max": 6.510290622711182,
      "activations/layer21_attention_weight_min": -4.569206237792969,
      "activations/layer22_attention_weight_max": 3.5102527141571045,
      "activations/layer22_attention_weight_min": -3.3629064559936523,
      "activations/layer23_attention_weight_max": 10.16655158996582,
      "activations/layer23_attention_weight_min": -7.943545818328857,
      "activations/layer2_attention_weight_max": 20.297595977783203,
      "activations/layer2_attention_weight_min": -16.35942840576172,
      "activations/layer3_attention_weight_max": 51.53293228149414,
      "activations/layer3_attention_weight_min": -45.94694137573242,
      "activations/layer4_attention_weight_max": 97.35065460205078,
      "activations/layer4_attention_weight_min": -98.91429901123047,
      "activations/layer5_attention_weight_max": 158.9336700439453,
      "activations/layer5_attention_weight_min": -176.0289306640625,
      "activations/layer6_attention_weight_max": 154.2415771484375,
      "activations/layer6_attention_weight_min": -173.16526794433594,
      "activations/layer7_attention_weight_max": 116.5425033569336,
      "activations/layer7_attention_weight_min": -126.64805603027344,
      "activations/layer8_attention_weight_max": 90.27271270751953,
      "activations/layer8_attention_weight_min": -99.11727905273438,
      "activations/layer9_attention_weight_max": 66.39977264404297,
      "activations/layer9_attention_weight_min": -66.9376449584961,
      "epoch": 0.08,
      "learning_rate": 0.00013146027170578227,
      "loss": 4.2243,
      "step": 1450
    },
    {
      "activations/layer0_attention_weight_max": 11.742413520812988,
      "activations/layer0_attention_weight_min": -10.019888877868652,
      "activations/layer10_attention_weight_max": 57.644378662109375,
      "activations/layer10_attention_weight_min": -52.64725875854492,
      "activations/layer11_attention_weight_max": 23.631065368652344,
      "activations/layer11_attention_weight_min": -14.238534927368164,
      "activations/layer12_attention_weight_max": 19.957260131835938,
      "activations/layer12_attention_weight_min": -25.81734848022461,
      "activations/layer13_attention_weight_max": 17.919862747192383,
      "activations/layer13_attention_weight_min": -12.318704605102539,
      "activations/layer14_attention_weight_max": 19.28758430480957,
      "activations/layer14_attention_weight_min": -13.599281311035156,
      "activations/layer15_attention_weight_max": 17.025239944458008,
      "activations/layer15_attention_weight_min": -10.814271926879883,
      "activations/layer16_attention_weight_max": 14.950264930725098,
      "activations/layer16_attention_weight_min": -8.960222244262695,
      "activations/layer17_attention_weight_max": 16.0726261138916,
      "activations/layer17_attention_weight_min": -12.986729621887207,
      "activations/layer18_attention_weight_max": 15.841387748718262,
      "activations/layer18_attention_weight_min": -7.5200910568237305,
      "activations/layer19_attention_weight_max": 8.65396785736084,
      "activations/layer19_attention_weight_min": -5.901742935180664,
      "activations/layer1_attention_weight_max": 19.41437339782715,
      "activations/layer1_attention_weight_min": -12.70147705078125,
      "activations/layer20_attention_weight_max": 10.912050247192383,
      "activations/layer20_attention_weight_min": -7.069005489349365,
      "activations/layer21_attention_weight_max": 5.981008052825928,
      "activations/layer21_attention_weight_min": -4.583985328674316,
      "activations/layer22_attention_weight_max": 3.9463226795196533,
      "activations/layer22_attention_weight_min": -3.204946279525757,
      "activations/layer23_attention_weight_max": 11.022089958190918,
      "activations/layer23_attention_weight_min": -7.61323356628418,
      "activations/layer2_attention_weight_max": 22.105934143066406,
      "activations/layer2_attention_weight_min": -16.201374053955078,
      "activations/layer3_attention_weight_max": 58.10993957519531,
      "activations/layer3_attention_weight_min": -46.404972076416016,
      "activations/layer4_attention_weight_max": 113.86431121826172,
      "activations/layer4_attention_weight_min": -100.96025085449219,
      "activations/layer5_attention_weight_max": 182.0344696044922,
      "activations/layer5_attention_weight_min": -182.2792205810547,
      "activations/layer6_attention_weight_max": 174.4919891357422,
      "activations/layer6_attention_weight_min": -172.83306884765625,
      "activations/layer7_attention_weight_max": 122.32317352294922,
      "activations/layer7_attention_weight_min": -123.387451171875,
      "activations/layer8_attention_weight_max": 94.52806854248047,
      "activations/layer8_attention_weight_min": -95.2822494506836,
      "activations/layer9_attention_weight_max": 69.64946746826172,
      "activations/layer9_attention_weight_min": -69.66786193847656,
      "epoch": 0.09,
      "learning_rate": 0.00013207969028649846,
      "loss": 4.1928,
      "step": 1500
    },
    {
      "activations/layer0_attention_weight_max": 11.745831489562988,
      "activations/layer0_attention_weight_min": -9.247541427612305,
      "activations/layer10_attention_weight_max": 54.00115203857422,
      "activations/layer10_attention_weight_min": -48.61995315551758,
      "activations/layer11_attention_weight_max": 20.632543563842773,
      "activations/layer11_attention_weight_min": -15.286657333374023,
      "activations/layer12_attention_weight_max": 19.67116355895996,
      "activations/layer12_attention_weight_min": -23.313371658325195,
      "activations/layer13_attention_weight_max": 21.34757423400879,
      "activations/layer13_attention_weight_min": -15.453653335571289,
      "activations/layer14_attention_weight_max": 17.349985122680664,
      "activations/layer14_attention_weight_min": -12.861299514770508,
      "activations/layer15_attention_weight_max": 15.176963806152344,
      "activations/layer15_attention_weight_min": -11.941436767578125,
      "activations/layer16_attention_weight_max": 17.269433975219727,
      "activations/layer16_attention_weight_min": -10.235505104064941,
      "activations/layer17_attention_weight_max": 14.59592056274414,
      "activations/layer17_attention_weight_min": -11.229654312133789,
      "activations/layer18_attention_weight_max": 14.546314239501953,
      "activations/layer18_attention_weight_min": -9.656339645385742,
      "activations/layer19_attention_weight_max": 7.388436794281006,
      "activations/layer19_attention_weight_min": -5.647136688232422,
      "activations/layer1_attention_weight_max": 17.443403244018555,
      "activations/layer1_attention_weight_min": -13.16467571258545,
      "activations/layer20_attention_weight_max": 13.576789855957031,
      "activations/layer20_attention_weight_min": -8.536399841308594,
      "activations/layer21_attention_weight_max": 7.208333969116211,
      "activations/layer21_attention_weight_min": -5.688256740570068,
      "activations/layer22_attention_weight_max": 4.24475622177124,
      "activations/layer22_attention_weight_min": -3.783644676208496,
      "activations/layer23_attention_weight_max": 11.670286178588867,
      "activations/layer23_attention_weight_min": -7.367020606994629,
      "activations/layer2_attention_weight_max": 21.245166778564453,
      "activations/layer2_attention_weight_min": -15.786163330078125,
      "activations/layer3_attention_weight_max": 51.1893310546875,
      "activations/layer3_attention_weight_min": -46.605186462402344,
      "activations/layer4_attention_weight_max": 97.4251937866211,
      "activations/layer4_attention_weight_min": -97.91649627685547,
      "activations/layer5_attention_weight_max": 160.01296997070312,
      "activations/layer5_attention_weight_min": -172.62030029296875,
      "activations/layer6_attention_weight_max": 152.15872192382812,
      "activations/layer6_attention_weight_min": -170.5958251953125,
      "activations/layer7_attention_weight_max": 110.65218353271484,
      "activations/layer7_attention_weight_min": -122.01396942138672,
      "activations/layer8_attention_weight_max": 87.5026626586914,
      "activations/layer8_attention_weight_min": -94.15745544433594,
      "activations/layer9_attention_weight_max": 61.608741760253906,
      "activations/layer9_attention_weight_min": -67.63859558105469,
      "epoch": 0.09,
      "learning_rate": 0.00013267859450497062,
      "loss": 4.1676,
      "step": 1550
    },
    {
      "activations/layer0_attention_weight_max": 11.795195579528809,
      "activations/layer0_attention_weight_min": -9.447478294372559,
      "activations/layer10_attention_weight_max": 60.040191650390625,
      "activations/layer10_attention_weight_min": -52.30718994140625,
      "activations/layer11_attention_weight_max": 18.83375358581543,
      "activations/layer11_attention_weight_min": -17.667715072631836,
      "activations/layer12_attention_weight_max": 20.77344512939453,
      "activations/layer12_attention_weight_min": -22.999984741210938,
      "activations/layer13_attention_weight_max": 17.360139846801758,
      "activations/layer13_attention_weight_min": -16.038801193237305,
      "activations/layer14_attention_weight_max": 15.696748733520508,
      "activations/layer14_attention_weight_min": -12.987010955810547,
      "activations/layer15_attention_weight_max": 15.38347339630127,
      "activations/layer15_attention_weight_min": -12.584683418273926,
      "activations/layer16_attention_weight_max": 13.91205883026123,
      "activations/layer16_attention_weight_min": -7.8278422355651855,
      "activations/layer17_attention_weight_max": 13.78477954864502,
      "activations/layer17_attention_weight_min": -11.904071807861328,
      "activations/layer18_attention_weight_max": 15.292109489440918,
      "activations/layer18_attention_weight_min": -9.748331069946289,
      "activations/layer19_attention_weight_max": 7.995586395263672,
      "activations/layer19_attention_weight_min": -5.807618618011475,
      "activations/layer1_attention_weight_max": 19.422515869140625,
      "activations/layer1_attention_weight_min": -12.701669692993164,
      "activations/layer20_attention_weight_max": 12.65467357635498,
      "activations/layer20_attention_weight_min": -8.192094802856445,
      "activations/layer21_attention_weight_max": 7.078551292419434,
      "activations/layer21_attention_weight_min": -5.674810886383057,
      "activations/layer22_attention_weight_max": 3.6376352310180664,
      "activations/layer22_attention_weight_min": -3.8886659145355225,
      "activations/layer23_attention_weight_max": 10.354925155639648,
      "activations/layer23_attention_weight_min": -8.13918685913086,
      "activations/layer2_attention_weight_max": 21.355995178222656,
      "activations/layer2_attention_weight_min": -18.208410263061523,
      "activations/layer3_attention_weight_max": 56.572669982910156,
      "activations/layer3_attention_weight_min": -48.4176139831543,
      "activations/layer4_attention_weight_max": 104.3636703491211,
      "activations/layer4_attention_weight_min": -97.04846954345703,
      "activations/layer5_attention_weight_max": 174.39573669433594,
      "activations/layer5_attention_weight_min": -171.1091766357422,
      "activations/layer6_attention_weight_max": 170.3306427001953,
      "activations/layer6_attention_weight_min": -164.9339599609375,
      "activations/layer7_attention_weight_max": 127.18953704833984,
      "activations/layer7_attention_weight_min": -124.7647705078125,
      "activations/layer8_attention_weight_max": 95.3084945678711,
      "activations/layer8_attention_weight_min": -90.71178436279297,
      "activations/layer9_attention_weight_max": 72.21329498291016,
      "activations/layer9_attention_weight_min": -67.89950561523438,
      "epoch": 0.09,
      "learning_rate": 0.0001332582997336764,
      "loss": 4.1208,
      "step": 1600
    },
    {
      "activations/layer0_attention_weight_max": 11.543560028076172,
      "activations/layer0_attention_weight_min": -8.958653450012207,
      "activations/layer10_attention_weight_max": 68.87024688720703,
      "activations/layer10_attention_weight_min": -57.521331787109375,
      "activations/layer11_attention_weight_max": 25.100093841552734,
      "activations/layer11_attention_weight_min": -18.638538360595703,
      "activations/layer12_attention_weight_max": 20.15464973449707,
      "activations/layer12_attention_weight_min": -23.457931518554688,
      "activations/layer13_attention_weight_max": 20.650108337402344,
      "activations/layer13_attention_weight_min": -15.531478881835938,
      "activations/layer14_attention_weight_max": 20.306657791137695,
      "activations/layer14_attention_weight_min": -13.856310844421387,
      "activations/layer15_attention_weight_max": 18.514381408691406,
      "activations/layer15_attention_weight_min": -11.691062927246094,
      "activations/layer16_attention_weight_max": 14.920799255371094,
      "activations/layer16_attention_weight_min": -9.551445960998535,
      "activations/layer17_attention_weight_max": 15.406085968017578,
      "activations/layer17_attention_weight_min": -10.646567344665527,
      "activations/layer18_attention_weight_max": 15.359153747558594,
      "activations/layer18_attention_weight_min": -9.719892501831055,
      "activations/layer19_attention_weight_max": 8.566658973693848,
      "activations/layer19_attention_weight_min": -6.129786491394043,
      "activations/layer1_attention_weight_max": 19.17966079711914,
      "activations/layer1_attention_weight_min": -12.630026817321777,
      "activations/layer20_attention_weight_max": 10.896642684936523,
      "activations/layer20_attention_weight_min": -7.281403541564941,
      "activations/layer21_attention_weight_max": 6.210543155670166,
      "activations/layer21_attention_weight_min": -5.385426998138428,
      "activations/layer22_attention_weight_max": 4.290058612823486,
      "activations/layer22_attention_weight_min": -3.4914207458496094,
      "activations/layer23_attention_weight_max": 10.77054214477539,
      "activations/layer23_attention_weight_min": -8.181829452514648,
      "activations/layer2_attention_weight_max": 21.581195831298828,
      "activations/layer2_attention_weight_min": -17.979244232177734,
      "activations/layer3_attention_weight_max": 54.58561706542969,
      "activations/layer3_attention_weight_min": -47.27897644042969,
      "activations/layer4_attention_weight_max": 103.76773834228516,
      "activations/layer4_attention_weight_min": -96.3805923461914,
      "activations/layer5_attention_weight_max": 172.4586181640625,
      "activations/layer5_attention_weight_min": -170.66378784179688,
      "activations/layer6_attention_weight_max": 171.5690155029297,
      "activations/layer6_attention_weight_min": -167.0649871826172,
      "activations/layer7_attention_weight_max": 127.77606201171875,
      "activations/layer7_attention_weight_min": -125.0353775024414,
      "activations/layer8_attention_weight_max": 94.88168334960938,
      "activations/layer8_attention_weight_min": -89.63237762451172,
      "activations/layer9_attention_weight_max": 80.09044647216797,
      "activations/layer9_attention_weight_min": -71.38924407958984,
      "epoch": 0.1,
      "learning_rate": 0.00013381999875347378,
      "loss": 4.0991,
      "step": 1650
    },
    {
      "activations/layer0_attention_weight_max": 12.671285629272461,
      "activations/layer0_attention_weight_min": -9.356846809387207,
      "activations/layer10_attention_weight_max": 65.26282501220703,
      "activations/layer10_attention_weight_min": -55.12311553955078,
      "activations/layer11_attention_weight_max": 23.297019958496094,
      "activations/layer11_attention_weight_min": -17.683677673339844,
      "activations/layer12_attention_weight_max": 21.882064819335938,
      "activations/layer12_attention_weight_min": -22.484527587890625,
      "activations/layer13_attention_weight_max": 20.82293128967285,
      "activations/layer13_attention_weight_min": -15.148035049438477,
      "activations/layer14_attention_weight_max": 21.71784019470215,
      "activations/layer14_attention_weight_min": -14.125218391418457,
      "activations/layer15_attention_weight_max": 20.00650978088379,
      "activations/layer15_attention_weight_min": -11.808150291442871,
      "activations/layer16_attention_weight_max": 18.51424789428711,
      "activations/layer16_attention_weight_min": -8.76068115234375,
      "activations/layer17_attention_weight_max": 16.66361427307129,
      "activations/layer17_attention_weight_min": -10.852296829223633,
      "activations/layer18_attention_weight_max": 15.338306427001953,
      "activations/layer18_attention_weight_min": -8.843841552734375,
      "activations/layer19_attention_weight_max": 9.6813325881958,
      "activations/layer19_attention_weight_min": -6.129706382751465,
      "activations/layer1_attention_weight_max": 19.816190719604492,
      "activations/layer1_attention_weight_min": -12.434288024902344,
      "activations/layer20_attention_weight_max": 10.441514015197754,
      "activations/layer20_attention_weight_min": -7.3218255043029785,
      "activations/layer21_attention_weight_max": 8.753911018371582,
      "activations/layer21_attention_weight_min": -4.8222198486328125,
      "activations/layer22_attention_weight_max": 4.238821983337402,
      "activations/layer22_attention_weight_min": -3.594480514526367,
      "activations/layer23_attention_weight_max": 10.396974563598633,
      "activations/layer23_attention_weight_min": -8.139920234680176,
      "activations/layer2_attention_weight_max": 22.666183471679688,
      "activations/layer2_attention_weight_min": -17.3706111907959,
      "activations/layer3_attention_weight_max": 56.82188415527344,
      "activations/layer3_attention_weight_min": -47.74054718017578,
      "activations/layer4_attention_weight_max": 104.80335998535156,
      "activations/layer4_attention_weight_min": -93.00013732910156,
      "activations/layer5_attention_weight_max": 178.48043823242188,
      "activations/layer5_attention_weight_min": -171.03939819335938,
      "activations/layer6_attention_weight_max": 175.64669799804688,
      "activations/layer6_attention_weight_min": -166.9527587890625,
      "activations/layer7_attention_weight_max": 132.01766967773438,
      "activations/layer7_attention_weight_min": -124.18934631347656,
      "activations/layer8_attention_weight_max": 97.09793853759766,
      "activations/layer8_attention_weight_min": -89.98270416259766,
      "activations/layer9_attention_weight_max": 78.22765350341797,
      "activations/layer9_attention_weight_min": -71.21339416503906,
      "epoch": 0.1,
      "learning_rate": 0.00013436477652963025,
      "loss": 4.0794,
      "step": 1700
    },
    {
      "activations/layer0_attention_weight_max": 11.638100624084473,
      "activations/layer0_attention_weight_min": -9.171809196472168,
      "activations/layer10_attention_weight_max": 61.021331787109375,
      "activations/layer10_attention_weight_min": -51.569847106933594,
      "activations/layer11_attention_weight_max": 23.769420623779297,
      "activations/layer11_attention_weight_min": -16.790239334106445,
      "activations/layer12_attention_weight_max": 20.37315559387207,
      "activations/layer12_attention_weight_min": -27.206356048583984,
      "activations/layer13_attention_weight_max": 17.1856632232666,
      "activations/layer13_attention_weight_min": -15.536189079284668,
      "activations/layer14_attention_weight_max": 21.123275756835938,
      "activations/layer14_attention_weight_min": -13.389078140258789,
      "activations/layer15_attention_weight_max": 20.73871612548828,
      "activations/layer15_attention_weight_min": -11.578097343444824,
      "activations/layer16_attention_weight_max": 16.49781036376953,
      "activations/layer16_attention_weight_min": -11.278291702270508,
      "activations/layer17_attention_weight_max": 15.882973670959473,
      "activations/layer17_attention_weight_min": -13.186038970947266,
      "activations/layer18_attention_weight_max": 16.205631256103516,
      "activations/layer18_attention_weight_min": -11.120771408081055,
      "activations/layer19_attention_weight_max": 7.821939945220947,
      "activations/layer19_attention_weight_min": -6.582535743713379,
      "activations/layer1_attention_weight_max": 17.913129806518555,
      "activations/layer1_attention_weight_min": -11.962239265441895,
      "activations/layer20_attention_weight_max": 12.314682960510254,
      "activations/layer20_attention_weight_min": -7.587920665740967,
      "activations/layer21_attention_weight_max": 8.545184135437012,
      "activations/layer21_attention_weight_min": -4.716382026672363,
      "activations/layer22_attention_weight_max": 4.549147129058838,
      "activations/layer22_attention_weight_min": -4.089217185974121,
      "activations/layer23_attention_weight_max": 10.560234069824219,
      "activations/layer23_attention_weight_min": -7.966705322265625,
      "activations/layer2_attention_weight_max": 21.998062133789062,
      "activations/layer2_attention_weight_min": -20.26587677001953,
      "activations/layer3_attention_weight_max": 54.72538757324219,
      "activations/layer3_attention_weight_min": -48.09231948852539,
      "activations/layer4_attention_weight_max": 96.95903015136719,
      "activations/layer4_attention_weight_min": -96.13706970214844,
      "activations/layer5_attention_weight_max": 161.45956420898438,
      "activations/layer5_attention_weight_min": -169.59788513183594,
      "activations/layer6_attention_weight_max": 157.90740966796875,
      "activations/layer6_attention_weight_min": -171.5199432373047,
      "activations/layer7_attention_weight_max": 117.50115966796875,
      "activations/layer7_attention_weight_min": -123.79180908203125,
      "activations/layer8_attention_weight_max": 88.94962310791016,
      "activations/layer8_attention_weight_min": -88.1692886352539,
      "activations/layer9_attention_weight_max": 73.27233123779297,
      "activations/layer9_attention_weight_min": -70.59183502197266,
      "epoch": 0.1,
      "learning_rate": 0.00013489362282661488,
      "loss": 4.031,
      "step": 1750
    },
    {
      "activations/layer0_attention_weight_max": 11.697134017944336,
      "activations/layer0_attention_weight_min": -9.523213386535645,
      "activations/layer10_attention_weight_max": 68.32498931884766,
      "activations/layer10_attention_weight_min": -57.53746795654297,
      "activations/layer11_attention_weight_max": 27.293106079101562,
      "activations/layer11_attention_weight_min": -15.942954063415527,
      "activations/layer12_attention_weight_max": 19.96192741394043,
      "activations/layer12_attention_weight_min": -21.612396240234375,
      "activations/layer13_attention_weight_max": 20.26325225830078,
      "activations/layer13_attention_weight_min": -17.203214645385742,
      "activations/layer14_attention_weight_max": 21.263872146606445,
      "activations/layer14_attention_weight_min": -15.692291259765625,
      "activations/layer15_attention_weight_max": 17.02320098876953,
      "activations/layer15_attention_weight_min": -11.924254417419434,
      "activations/layer16_attention_weight_max": 16.429431915283203,
      "activations/layer16_attention_weight_min": -9.94080924987793,
      "activations/layer17_attention_weight_max": 14.224860191345215,
      "activations/layer17_attention_weight_min": -11.606047630310059,
      "activations/layer18_attention_weight_max": 13.926006317138672,
      "activations/layer18_attention_weight_min": -10.481675148010254,
      "activations/layer19_attention_weight_max": 10.269768714904785,
      "activations/layer19_attention_weight_min": -6.256299018859863,
      "activations/layer1_attention_weight_max": 18.03156089782715,
      "activations/layer1_attention_weight_min": -11.75101375579834,
      "activations/layer20_attention_weight_max": 11.215482711791992,
      "activations/layer20_attention_weight_min": -7.321226596832275,
      "activations/layer21_attention_weight_max": 6.79598331451416,
      "activations/layer21_attention_weight_min": -5.518650054931641,
      "activations/layer22_attention_weight_max": 4.364764213562012,
      "activations/layer22_attention_weight_min": -4.232901573181152,
      "activations/layer23_attention_weight_max": 11.63838005065918,
      "activations/layer23_attention_weight_min": -8.14383316040039,
      "activations/layer2_attention_weight_max": 22.88037109375,
      "activations/layer2_attention_weight_min": -16.16626739501953,
      "activations/layer3_attention_weight_max": 57.04948806762695,
      "activations/layer3_attention_weight_min": -47.71797180175781,
      "activations/layer4_attention_weight_max": 100.01689910888672,
      "activations/layer4_attention_weight_min": -93.25251007080078,
      "activations/layer5_attention_weight_max": 170.04019165039062,
      "activations/layer5_attention_weight_min": -170.53836059570312,
      "activations/layer6_attention_weight_max": 166.5521697998047,
      "activations/layer6_attention_weight_min": -166.2917938232422,
      "activations/layer7_attention_weight_max": 125.75379943847656,
      "activations/layer7_attention_weight_min": -120.60939025878906,
      "activations/layer8_attention_weight_max": 93.46721649169922,
      "activations/layer8_attention_weight_min": -89.5037612915039,
      "activations/layer9_attention_weight_max": 79.23759460449219,
      "activations/layer9_attention_weight_min": -74.8185806274414,
      "epoch": 0.1,
      "learning_rate": 0.00013540744303023553,
      "loss": 4.0087,
      "step": 1800
    },
    {
      "activations/layer0_attention_weight_max": 12.284845352172852,
      "activations/layer0_attention_weight_min": -9.058570861816406,
      "activations/layer10_attention_weight_max": 65.54681396484375,
      "activations/layer10_attention_weight_min": -59.70341491699219,
      "activations/layer11_attention_weight_max": 25.452987670898438,
      "activations/layer11_attention_weight_min": -17.203304290771484,
      "activations/layer12_attention_weight_max": 21.282941818237305,
      "activations/layer12_attention_weight_min": -23.131362915039062,
      "activations/layer13_attention_weight_max": 23.042015075683594,
      "activations/layer13_attention_weight_min": -19.516550064086914,
      "activations/layer14_attention_weight_max": 24.180105209350586,
      "activations/layer14_attention_weight_min": -14.828286170959473,
      "activations/layer15_attention_weight_max": 18.57050895690918,
      "activations/layer15_attention_weight_min": -11.789536476135254,
      "activations/layer16_attention_weight_max": 20.277334213256836,
      "activations/layer16_attention_weight_min": -8.572415351867676,
      "activations/layer17_attention_weight_max": 17.34449005126953,
      "activations/layer17_attention_weight_min": -13.817476272583008,
      "activations/layer18_attention_weight_max": 18.55667495727539,
      "activations/layer18_attention_weight_min": -9.91108512878418,
      "activations/layer19_attention_weight_max": 9.8651762008667,
      "activations/layer19_attention_weight_min": -8.483359336853027,
      "activations/layer1_attention_weight_max": 16.052318572998047,
      "activations/layer1_attention_weight_min": -11.38239860534668,
      "activations/layer20_attention_weight_max": 13.654765129089355,
      "activations/layer20_attention_weight_min": -7.077447891235352,
      "activations/layer21_attention_weight_max": 7.967654228210449,
      "activations/layer21_attention_weight_min": -5.022790908813477,
      "activations/layer22_attention_weight_max": 4.345314025878906,
      "activations/layer22_attention_weight_min": -4.065005302429199,
      "activations/layer23_attention_weight_max": 11.84277629852295,
      "activations/layer23_attention_weight_min": -8.389104843139648,
      "activations/layer2_attention_weight_max": 21.395931243896484,
      "activations/layer2_attention_weight_min": -17.92125129699707,
      "activations/layer3_attention_weight_max": 53.154685974121094,
      "activations/layer3_attention_weight_min": -47.808467864990234,
      "activations/layer4_attention_weight_max": 90.83494567871094,
      "activations/layer4_attention_weight_min": -91.6720199584961,
      "activations/layer5_attention_weight_max": 156.77713012695312,
      "activations/layer5_attention_weight_min": -168.86239624023438,
      "activations/layer6_attention_weight_max": 143.52984619140625,
      "activations/layer6_attention_weight_min": -158.5722198486328,
      "activations/layer7_attention_weight_max": 108.5128402709961,
      "activations/layer7_attention_weight_min": -121.50077819824219,
      "activations/layer8_attention_weight_max": 80.6649169921875,
      "activations/layer8_attention_weight_min": -88.31230163574219,
      "activations/layer9_attention_weight_max": 68.01600646972656,
      "activations/layer9_attention_weight_min": -69.4919204711914,
      "epoch": 0.11,
      "learning_rate": 0.00013590706747439169,
      "loss": 4.0084,
      "step": 1850
    },
    {
      "activations/layer0_attention_weight_max": 13.607232093811035,
      "activations/layer0_attention_weight_min": -9.588480949401855,
      "activations/layer10_attention_weight_max": 60.279632568359375,
      "activations/layer10_attention_weight_min": -51.16188430786133,
      "activations/layer11_attention_weight_max": 21.14326286315918,
      "activations/layer11_attention_weight_min": -16.52962875366211,
      "activations/layer12_attention_weight_max": 21.7091121673584,
      "activations/layer12_attention_weight_min": -21.47318458557129,
      "activations/layer13_attention_weight_max": 19.637893676757812,
      "activations/layer13_attention_weight_min": -18.33873748779297,
      "activations/layer14_attention_weight_max": 17.86421012878418,
      "activations/layer14_attention_weight_min": -17.64458656311035,
      "activations/layer15_attention_weight_max": 16.16900062561035,
      "activations/layer15_attention_weight_min": -13.458039283752441,
      "activations/layer16_attention_weight_max": 12.682584762573242,
      "activations/layer16_attention_weight_min": -10.144432067871094,
      "activations/layer17_attention_weight_max": 15.112832069396973,
      "activations/layer17_attention_weight_min": -14.95409870147705,
      "activations/layer18_attention_weight_max": 17.42556381225586,
      "activations/layer18_attention_weight_min": -12.198525428771973,
      "activations/layer19_attention_weight_max": 11.017783164978027,
      "activations/layer19_attention_weight_min": -9.267147064208984,
      "activations/layer1_attention_weight_max": 17.431324005126953,
      "activations/layer1_attention_weight_min": -11.753376960754395,
      "activations/layer20_attention_weight_max": 11.912291526794434,
      "activations/layer20_attention_weight_min": -8.406295776367188,
      "activations/layer21_attention_weight_max": 7.071257591247559,
      "activations/layer21_attention_weight_min": -5.48118257522583,
      "activations/layer22_attention_weight_max": 3.9852240085601807,
      "activations/layer22_attention_weight_min": -4.142963886260986,
      "activations/layer23_attention_weight_max": 12.540270805358887,
      "activations/layer23_attention_weight_min": -8.062153816223145,
      "activations/layer2_attention_weight_max": 22.752403259277344,
      "activations/layer2_attention_weight_min": -18.55895233154297,
      "activations/layer3_attention_weight_max": 56.06668472290039,
      "activations/layer3_attention_weight_min": -47.75450897216797,
      "activations/layer4_attention_weight_max": 97.47003936767578,
      "activations/layer4_attention_weight_min": -92.85001373291016,
      "activations/layer5_attention_weight_max": 168.52838134765625,
      "activations/layer5_attention_weight_min": -172.1195526123047,
      "activations/layer6_attention_weight_max": 160.01568603515625,
      "activations/layer6_attention_weight_min": -160.06011962890625,
      "activations/layer7_attention_weight_max": 120.42064666748047,
      "activations/layer7_attention_weight_min": -118.78104400634766,
      "activations/layer8_attention_weight_max": 90.95650482177734,
      "activations/layer8_attention_weight_min": -88.91925811767578,
      "activations/layer9_attention_weight_max": 71.0257797241211,
      "activations/layer9_attention_weight_min": -67.8260726928711,
      "epoch": 0.11,
      "learning_rate": 0.00013639325951374467,
      "loss": 3.989,
      "step": 1900
    },
    {
      "activations/layer0_attention_weight_max": 12.642233848571777,
      "activations/layer0_attention_weight_min": -9.81424617767334,
      "activations/layer10_attention_weight_max": 63.79451370239258,
      "activations/layer10_attention_weight_min": -56.812278747558594,
      "activations/layer11_attention_weight_max": 19.60123062133789,
      "activations/layer11_attention_weight_min": -16.319448471069336,
      "activations/layer12_attention_weight_max": 21.010461807250977,
      "activations/layer12_attention_weight_min": -22.55440902709961,
      "activations/layer13_attention_weight_max": 17.556184768676758,
      "activations/layer13_attention_weight_min": -18.40610694885254,
      "activations/layer14_attention_weight_max": 18.958829879760742,
      "activations/layer14_attention_weight_min": -14.664783477783203,
      "activations/layer15_attention_weight_max": 17.09636688232422,
      "activations/layer15_attention_weight_min": -12.406973838806152,
      "activations/layer16_attention_weight_max": 13.581459045410156,
      "activations/layer16_attention_weight_min": -10.11662483215332,
      "activations/layer17_attention_weight_max": 13.437636375427246,
      "activations/layer17_attention_weight_min": -12.746554374694824,
      "activations/layer18_attention_weight_max": 18.2977237701416,
      "activations/layer18_attention_weight_min": -10.59093189239502,
      "activations/layer19_attention_weight_max": 8.485960006713867,
      "activations/layer19_attention_weight_min": -9.997347831726074,
      "activations/layer1_attention_weight_max": 17.9666748046875,
      "activations/layer1_attention_weight_min": -11.71959114074707,
      "activations/layer20_attention_weight_max": 13.809595108032227,
      "activations/layer20_attention_weight_min": -7.99591588973999,
      "activations/layer21_attention_weight_max": 7.859332084655762,
      "activations/layer21_attention_weight_min": -5.725813388824463,
      "activations/layer22_attention_weight_max": 4.795556545257568,
      "activations/layer22_attention_weight_min": -4.333585262298584,
      "activations/layer23_attention_weight_max": 12.398405075073242,
      "activations/layer23_attention_weight_min": -9.091010093688965,
      "activations/layer2_attention_weight_max": 23.070384979248047,
      "activations/layer2_attention_weight_min": -18.398761749267578,
      "activations/layer3_attention_weight_max": 58.634674072265625,
      "activations/layer3_attention_weight_min": -47.69778060913086,
      "activations/layer4_attention_weight_max": 101.94448852539062,
      "activations/layer4_attention_weight_min": -93.79759979248047,
      "activations/layer5_attention_weight_max": 178.17572021484375,
      "activations/layer5_attention_weight_min": -168.74411010742188,
      "activations/layer6_attention_weight_max": 166.66738891601562,
      "activations/layer6_attention_weight_min": -158.7406005859375,
      "activations/layer7_attention_weight_max": 128.69802856445312,
      "activations/layer7_attention_weight_min": -114.56964874267578,
      "activations/layer8_attention_weight_max": 95.39690399169922,
      "activations/layer8_attention_weight_min": -86.37480163574219,
      "activations/layer9_attention_weight_max": 73.9715347290039,
      "activations/layer9_attention_weight_min": -68.63844299316406,
      "epoch": 0.11,
      "learning_rate": 0.00013686672253936715,
      "loss": 3.9518,
      "step": 1950
    },
    {
      "activations/layer0_attention_weight_max": 12.102486610412598,
      "activations/layer0_attention_weight_min": -9.433900833129883,
      "activations/layer10_attention_weight_max": 65.26971435546875,
      "activations/layer10_attention_weight_min": -51.84196853637695,
      "activations/layer11_attention_weight_max": 22.635948181152344,
      "activations/layer11_attention_weight_min": -15.116461753845215,
      "activations/layer12_attention_weight_max": 20.71055030822754,
      "activations/layer12_attention_weight_min": -25.18551254272461,
      "activations/layer13_attention_weight_max": 20.633031845092773,
      "activations/layer13_attention_weight_min": -16.79186248779297,
      "activations/layer14_attention_weight_max": 21.6850528717041,
      "activations/layer14_attention_weight_min": -14.990056037902832,
      "activations/layer15_attention_weight_max": 19.945449829101562,
      "activations/layer15_attention_weight_min": -13.177926063537598,
      "activations/layer16_attention_weight_max": 15.65761947631836,
      "activations/layer16_attention_weight_min": -10.005058288574219,
      "activations/layer17_attention_weight_max": 16.16702651977539,
      "activations/layer17_attention_weight_min": -14.511762619018555,
      "activations/layer18_attention_weight_max": 20.69940757751465,
      "activations/layer18_attention_weight_min": -10.524897575378418,
      "activations/layer19_attention_weight_max": 9.660137176513672,
      "activations/layer19_attention_weight_min": -10.291644096374512,
      "activations/layer1_attention_weight_max": 19.91333770751953,
      "activations/layer1_attention_weight_min": -12.534381866455078,
      "activations/layer20_attention_weight_max": 14.071388244628906,
      "activations/layer20_attention_weight_min": -9.870220184326172,
      "activations/layer21_attention_weight_max": 8.998936653137207,
      "activations/layer21_attention_weight_min": -5.403798580169678,
      "activations/layer22_attention_weight_max": 4.410192489624023,
      "activations/layer22_attention_weight_min": -4.447628974914551,
      "activations/layer23_attention_weight_max": 12.075042724609375,
      "activations/layer23_attention_weight_min": -8.831993103027344,
      "activations/layer2_attention_weight_max": 22.191884994506836,
      "activations/layer2_attention_weight_min": -17.72870635986328,
      "activations/layer3_attention_weight_max": 56.88603210449219,
      "activations/layer3_attention_weight_min": -47.38945388793945,
      "activations/layer4_attention_weight_max": 100.81546020507812,
      "activations/layer4_attention_weight_min": -92.547119140625,
      "activations/layer5_attention_weight_max": 170.99139404296875,
      "activations/layer5_attention_weight_min": -162.74740600585938,
      "activations/layer6_attention_weight_max": 160.1285858154297,
      "activations/layer6_attention_weight_min": -154.56936645507812,
      "activations/layer7_attention_weight_max": 114.81946563720703,
      "activations/layer7_attention_weight_min": -113.82530212402344,
      "activations/layer8_attention_weight_max": 83.88104248046875,
      "activations/layer8_attention_weight_min": -83.23698425292969,
      "activations/layer9_attention_weight_max": 77.09442138671875,
      "activations/layer9_attention_weight_min": -71.30587005615234,
      "epoch": 0.12,
      "learning_rate": 0.0001373281060992278,
      "loss": 3.94,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_loss": 3.859375,
      "eval_runtime": 8.1637,
      "eval_samples_per_second": 525.984,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_openwebtext_loss": 3.859375,
      "eval_openwebtext_ppl": 47.435694792893365,
      "eval_openwebtext_runtime": 8.1637,
      "eval_openwebtext_samples_per_second": 525.984,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_wikitext_loss": 4.25390625,
      "eval_wikitext_ppl": 70.37979717263202,
      "eval_wikitext_runtime": 1.4042,
      "eval_wikitext_samples_per_second": 324.743,
      "step": 2000
    },
    {
      "epoch": 0.12,
      "eval_lambada_loss": 5.18359375,
      "eval_lambada_ppl": 178.32250736130783,
      "eval_lambada_runtime": 10.0558,
      "eval_lambada_samples_per_second": 484.198,
      "step": 2000
    },
    {
      "activations/layer0_attention_weight_max": 13.291908264160156,
      "activations/layer0_attention_weight_min": -8.978801727294922,
      "activations/layer10_attention_weight_max": 73.31648254394531,
      "activations/layer10_attention_weight_min": -58.85124969482422,
      "activations/layer11_attention_weight_max": 24.879852294921875,
      "activations/layer11_attention_weight_min": -18.300678253173828,
      "activations/layer12_attention_weight_max": 22.170934677124023,
      "activations/layer12_attention_weight_min": -24.314664840698242,
      "activations/layer13_attention_weight_max": 22.313222885131836,
      "activations/layer13_attention_weight_min": -20.938989639282227,
      "activations/layer14_attention_weight_max": 22.84296226501465,
      "activations/layer14_attention_weight_min": -16.74153709411621,
      "activations/layer15_attention_weight_max": 20.261877059936523,
      "activations/layer15_attention_weight_min": -13.459449768066406,
      "activations/layer16_attention_weight_max": 20.478965759277344,
      "activations/layer16_attention_weight_min": -9.804144859313965,
      "activations/layer17_attention_weight_max": 17.428049087524414,
      "activations/layer17_attention_weight_min": -14.396364212036133,
      "activations/layer18_attention_weight_max": 17.768186569213867,
      "activations/layer18_attention_weight_min": -12.121740341186523,
      "activations/layer19_attention_weight_max": 10.860146522521973,
      "activations/layer19_attention_weight_min": -9.978174209594727,
      "activations/layer1_attention_weight_max": 19.992490768432617,
      "activations/layer1_attention_weight_min": -12.435286521911621,
      "activations/layer20_attention_weight_max": 13.43060302734375,
      "activations/layer20_attention_weight_min": -8.656103134155273,
      "activations/layer21_attention_weight_max": 8.784239768981934,
      "activations/layer21_attention_weight_min": -5.8897199630737305,
      "activations/layer22_attention_weight_max": 5.05690860748291,
      "activations/layer22_attention_weight_min": -4.326070785522461,
      "activations/layer23_attention_weight_max": 11.355189323425293,
      "activations/layer23_attention_weight_min": -8.48797607421875,
      "activations/layer2_attention_weight_max": 24.236804962158203,
      "activations/layer2_attention_weight_min": -18.391416549682617,
      "activations/layer3_attention_weight_max": 59.80683135986328,
      "activations/layer3_attention_weight_min": -49.378379821777344,
      "activations/layer4_attention_weight_max": 107.50001525878906,
      "activations/layer4_attention_weight_min": -91.15367889404297,
      "activations/layer5_attention_weight_max": 183.28038024902344,
      "activations/layer5_attention_weight_min": -167.4515380859375,
      "activations/layer6_attention_weight_max": 167.51052856445312,
      "activations/layer6_attention_weight_min": -161.0681610107422,
      "activations/layer7_attention_weight_max": 127.5479507446289,
      "activations/layer7_attention_weight_min": -119.41258239746094,
      "activations/layer8_attention_weight_max": 93.34317016601562,
      "activations/layer8_attention_weight_min": -85.65238189697266,
      "activations/layer9_attention_weight_max": 86.61910247802734,
      "activations/layer9_attention_weight_min": -69.12836456298828,
      "epoch": 0.12,
      "learning_rate": 0.0001377691219698583,
      "loss": 3.9342,
      "step": 2050
    },
    {
      "activations/layer0_attention_weight_max": 12.05197811126709,
      "activations/layer0_attention_weight_min": -9.090173721313477,
      "activations/layer10_attention_weight_max": 65.10742950439453,
      "activations/layer10_attention_weight_min": -59.616119384765625,
      "activations/layer11_attention_weight_max": 23.329435348510742,
      "activations/layer11_attention_weight_min": -17.421218872070312,
      "activations/layer12_attention_weight_max": 22.520963668823242,
      "activations/layer12_attention_weight_min": -21.461593627929688,
      "activations/layer13_attention_weight_max": 18.3591365814209,
      "activations/layer13_attention_weight_min": -17.066547393798828,
      "activations/layer14_attention_weight_max": 20.702600479125977,
      "activations/layer14_attention_weight_min": -15.016924858093262,
      "activations/layer15_attention_weight_max": 20.20027732849121,
      "activations/layer15_attention_weight_min": -16.276046752929688,
      "activations/layer16_attention_weight_max": 16.52919578552246,
      "activations/layer16_attention_weight_min": -11.508837699890137,
      "activations/layer17_attention_weight_max": 16.883800506591797,
      "activations/layer17_attention_weight_min": -14.851021766662598,
      "activations/layer18_attention_weight_max": 19.290969848632812,
      "activations/layer18_attention_weight_min": -11.228078842163086,
      "activations/layer19_attention_weight_max": 11.545976638793945,
      "activations/layer19_attention_weight_min": -9.703895568847656,
      "activations/layer1_attention_weight_max": 17.31621551513672,
      "activations/layer1_attention_weight_min": -11.815661430358887,
      "activations/layer20_attention_weight_max": 12.281943321228027,
      "activations/layer20_attention_weight_min": -8.289470672607422,
      "activations/layer21_attention_weight_max": 8.14378833770752,
      "activations/layer21_attention_weight_min": -5.104318141937256,
      "activations/layer22_attention_weight_max": 5.239075660705566,
      "activations/layer22_attention_weight_min": -4.786611080169678,
      "activations/layer23_attention_weight_max": 12.654509544372559,
      "activations/layer23_attention_weight_min": -8.387518882751465,
      "activations/layer2_attention_weight_max": 23.115009307861328,
      "activations/layer2_attention_weight_min": -18.487022399902344,
      "activations/layer3_attention_weight_max": 57.64588928222656,
      "activations/layer3_attention_weight_min": -48.37149429321289,
      "activations/layer4_attention_weight_max": 96.36970520019531,
      "activations/layer4_attention_weight_min": -92.01539611816406,
      "activations/layer5_attention_weight_max": 165.48765563964844,
      "activations/layer5_attention_weight_min": -167.97714233398438,
      "activations/layer6_attention_weight_max": 160.31170654296875,
      "activations/layer6_attention_weight_min": -162.31695556640625,
      "activations/layer7_attention_weight_max": 116.5408706665039,
      "activations/layer7_attention_weight_min": -113.90936279296875,
      "activations/layer8_attention_weight_max": 90.57137298583984,
      "activations/layer8_attention_weight_min": -87.6686782836914,
      "activations/layer9_attention_weight_max": 70.9502182006836,
      "activations/layer9_attention_weight_min": -69.34133911132812,
      "epoch": 0.12,
      "learning_rate": 0.00013820831923751152,
      "loss": 3.9035,
      "step": 2100
    },
    {
      "activations/layer0_attention_weight_max": 12.810392379760742,
      "activations/layer0_attention_weight_min": -9.285321235656738,
      "activations/layer10_attention_weight_max": 65.10906219482422,
      "activations/layer10_attention_weight_min": -54.80210876464844,
      "activations/layer11_attention_weight_max": 27.71074676513672,
      "activations/layer11_attention_weight_min": -18.029521942138672,
      "activations/layer12_attention_weight_max": 20.8460750579834,
      "activations/layer12_attention_weight_min": -21.654743194580078,
      "activations/layer13_attention_weight_max": 21.35755729675293,
      "activations/layer13_attention_weight_min": -17.64290428161621,
      "activations/layer14_attention_weight_max": 22.704288482666016,
      "activations/layer14_attention_weight_min": -15.338838577270508,
      "activations/layer15_attention_weight_max": 19.312349319458008,
      "activations/layer15_attention_weight_min": -12.268542289733887,
      "activations/layer16_attention_weight_max": 18.632537841796875,
      "activations/layer16_attention_weight_min": -10.255792617797852,
      "activations/layer17_attention_weight_max": 17.363571166992188,
      "activations/layer17_attention_weight_min": -12.688496589660645,
      "activations/layer18_attention_weight_max": 19.401437759399414,
      "activations/layer18_attention_weight_min": -10.556756019592285,
      "activations/layer19_attention_weight_max": 14.694229125976562,
      "activations/layer19_attention_weight_min": -11.121932983398438,
      "activations/layer1_attention_weight_max": 16.44789695739746,
      "activations/layer1_attention_weight_min": -11.927403450012207,
      "activations/layer20_attention_weight_max": 14.379393577575684,
      "activations/layer20_attention_weight_min": -7.98057222366333,
      "activations/layer21_attention_weight_max": 8.835837364196777,
      "activations/layer21_attention_weight_min": -5.7937469482421875,
      "activations/layer22_attention_weight_max": 5.501119136810303,
      "activations/layer22_attention_weight_min": -4.965478897094727,
      "activations/layer23_attention_weight_max": 11.257993698120117,
      "activations/layer23_attention_weight_min": -8.569896697998047,
      "activations/layer2_attention_weight_max": 25.379615783691406,
      "activations/layer2_attention_weight_min": -19.229644775390625,
      "activations/layer3_attention_weight_max": 59.314754486083984,
      "activations/layer3_attention_weight_min": -49.17512512207031,
      "activations/layer4_attention_weight_max": 100.07748413085938,
      "activations/layer4_attention_weight_min": -89.97038269042969,
      "activations/layer5_attention_weight_max": 175.76388549804688,
      "activations/layer5_attention_weight_min": -167.88702392578125,
      "activations/layer6_attention_weight_max": 164.6898193359375,
      "activations/layer6_attention_weight_min": -159.31700134277344,
      "activations/layer7_attention_weight_max": 124.4384765625,
      "activations/layer7_attention_weight_min": -113.73660278320312,
      "activations/layer8_attention_weight_max": 94.82249450683594,
      "activations/layer8_attention_weight_min": -86.35301971435547,
      "activations/layer9_attention_weight_max": 76.8976058959961,
      "activations/layer9_attention_weight_min": -70.8080062866211,
      "epoch": 0.12,
      "learning_rate": 0.00013863710306478343,
      "loss": 3.8816,
      "step": 2150
    },
    {
      "activations/layer0_attention_weight_max": 11.97166919708252,
      "activations/layer0_attention_weight_min": -9.282879829406738,
      "activations/layer10_attention_weight_max": 62.4201774597168,
      "activations/layer10_attention_weight_min": -57.776424407958984,
      "activations/layer11_attention_weight_max": 22.92061424255371,
      "activations/layer11_attention_weight_min": -18.861900329589844,
      "activations/layer12_attention_weight_max": 21.25082778930664,
      "activations/layer12_attention_weight_min": -22.637365341186523,
      "activations/layer13_attention_weight_max": 22.17965316772461,
      "activations/layer13_attention_weight_min": -16.077375411987305,
      "activations/layer14_attention_weight_max": 21.15813636779785,
      "activations/layer14_attention_weight_min": -14.4761381149292,
      "activations/layer15_attention_weight_max": 18.409343719482422,
      "activations/layer15_attention_weight_min": -13.20629596710205,
      "activations/layer16_attention_weight_max": 18.055395126342773,
      "activations/layer16_attention_weight_min": -11.335074424743652,
      "activations/layer17_attention_weight_max": 18.570287704467773,
      "activations/layer17_attention_weight_min": -13.556736946105957,
      "activations/layer18_attention_weight_max": 18.180784225463867,
      "activations/layer18_attention_weight_min": -10.898534774780273,
      "activations/layer19_attention_weight_max": 13.053290367126465,
      "activations/layer19_attention_weight_min": -10.031630516052246,
      "activations/layer1_attention_weight_max": 17.610258102416992,
      "activations/layer1_attention_weight_min": -11.709345817565918,
      "activations/layer20_attention_weight_max": 14.718749046325684,
      "activations/layer20_attention_weight_min": -9.663976669311523,
      "activations/layer21_attention_weight_max": 9.691413879394531,
      "activations/layer21_attention_weight_min": -5.840758800506592,
      "activations/layer22_attention_weight_max": 5.335594654083252,
      "activations/layer22_attention_weight_min": -5.17948579788208,
      "activations/layer23_attention_weight_max": 11.14853572845459,
      "activations/layer23_attention_weight_min": -8.706066131591797,
      "activations/layer2_attention_weight_max": 25.22795867919922,
      "activations/layer2_attention_weight_min": -18.735063552856445,
      "activations/layer3_attention_weight_max": 55.92432403564453,
      "activations/layer3_attention_weight_min": -46.191741943359375,
      "activations/layer4_attention_weight_max": 97.24510955810547,
      "activations/layer4_attention_weight_min": -87.79949188232422,
      "activations/layer5_attention_weight_max": 170.77133178710938,
      "activations/layer5_attention_weight_min": -163.8865966796875,
      "activations/layer6_attention_weight_max": 157.61631774902344,
      "activations/layer6_attention_weight_min": -154.0215606689453,
      "activations/layer7_attention_weight_max": 122.47273254394531,
      "activations/layer7_attention_weight_min": -118.24508666992188,
      "activations/layer8_attention_weight_max": 87.44951629638672,
      "activations/layer8_attention_weight_min": -87.98316955566406,
      "activations/layer9_attention_weight_max": 73.62201690673828,
      "activations/layer9_attention_weight_min": -71.0772476196289,
      "epoch": 0.13,
      "learning_rate": 0.00013905595584493412,
      "loss": 3.8868,
      "step": 2200
    },
    {
      "activations/layer0_attention_weight_max": 12.599017143249512,
      "activations/layer0_attention_weight_min": -8.668020248413086,
      "activations/layer10_attention_weight_max": 64.54971313476562,
      "activations/layer10_attention_weight_min": -52.52922058105469,
      "activations/layer11_attention_weight_max": 25.750102996826172,
      "activations/layer11_attention_weight_min": -15.838768005371094,
      "activations/layer12_attention_weight_max": 22.285400390625,
      "activations/layer12_attention_weight_min": -26.648279190063477,
      "activations/layer13_attention_weight_max": 17.9158935546875,
      "activations/layer13_attention_weight_min": -17.63064956665039,
      "activations/layer14_attention_weight_max": 16.783594131469727,
      "activations/layer14_attention_weight_min": -14.929563522338867,
      "activations/layer15_attention_weight_max": 18.861953735351562,
      "activations/layer15_attention_weight_min": -13.69039249420166,
      "activations/layer16_attention_weight_max": 12.690500259399414,
      "activations/layer16_attention_weight_min": -11.287425994873047,
      "activations/layer17_attention_weight_max": 18.614660263061523,
      "activations/layer17_attention_weight_min": -13.955582618713379,
      "activations/layer18_attention_weight_max": 19.388254165649414,
      "activations/layer18_attention_weight_min": -13.199824333190918,
      "activations/layer19_attention_weight_max": 11.199014663696289,
      "activations/layer19_attention_weight_min": -11.140559196472168,
      "activations/layer1_attention_weight_max": 16.670862197875977,
      "activations/layer1_attention_weight_min": -13.479766845703125,
      "activations/layer20_attention_weight_max": 13.966679573059082,
      "activations/layer20_attention_weight_min": -11.358918190002441,
      "activations/layer21_attention_weight_max": 11.734865188598633,
      "activations/layer21_attention_weight_min": -5.881986618041992,
      "activations/layer22_attention_weight_max": 5.246767044067383,
      "activations/layer22_attention_weight_min": -4.428495407104492,
      "activations/layer23_attention_weight_max": 11.397956848144531,
      "activations/layer23_attention_weight_min": -8.84399127960205,
      "activations/layer2_attention_weight_max": 24.32625389099121,
      "activations/layer2_attention_weight_min": -19.220684051513672,
      "activations/layer3_attention_weight_max": 57.85502243041992,
      "activations/layer3_attention_weight_min": -45.4423942565918,
      "activations/layer4_attention_weight_max": 100.9867935180664,
      "activations/layer4_attention_weight_min": -91.3573226928711,
      "activations/layer5_attention_weight_max": 179.28042602539062,
      "activations/layer5_attention_weight_min": -171.42230224609375,
      "activations/layer6_attention_weight_max": 168.9801483154297,
      "activations/layer6_attention_weight_min": -161.10580444335938,
      "activations/layer7_attention_weight_max": 128.82803344726562,
      "activations/layer7_attention_weight_min": -113.85646057128906,
      "activations/layer8_attention_weight_max": 89.1231460571289,
      "activations/layer8_attention_weight_min": -84.34843444824219,
      "activations/layer9_attention_weight_max": 82.1985092163086,
      "activations/layer9_attention_weight_min": -75.5810317993164,
      "epoch": 0.13,
      "learning_rate": 0.00013946532720898237,
      "loss": 3.8525,
      "step": 2250
    },
    {
      "activations/layer0_attention_weight_max": 12.063554763793945,
      "activations/layer0_attention_weight_min": -8.890186309814453,
      "activations/layer10_attention_weight_max": 60.4093132019043,
      "activations/layer10_attention_weight_min": -53.2578239440918,
      "activations/layer11_attention_weight_max": 22.131309509277344,
      "activations/layer11_attention_weight_min": -15.600507736206055,
      "activations/layer12_attention_weight_max": 20.92901611328125,
      "activations/layer12_attention_weight_min": -22.3308048248291,
      "activations/layer13_attention_weight_max": 19.964330673217773,
      "activations/layer13_attention_weight_min": -20.02748680114746,
      "activations/layer14_attention_weight_max": 21.25593376159668,
      "activations/layer14_attention_weight_min": -15.760008811950684,
      "activations/layer15_attention_weight_max": 15.723858833312988,
      "activations/layer15_attention_weight_min": -12.655128479003906,
      "activations/layer16_attention_weight_max": 15.977991104125977,
      "activations/layer16_attention_weight_min": -10.293360710144043,
      "activations/layer17_attention_weight_max": 15.84027099609375,
      "activations/layer17_attention_weight_min": -14.707026481628418,
      "activations/layer18_attention_weight_max": 19.684080123901367,
      "activations/layer18_attention_weight_min": -14.044548988342285,
      "activations/layer19_attention_weight_max": 14.815218925476074,
      "activations/layer19_attention_weight_min": -10.622756004333496,
      "activations/layer1_attention_weight_max": 17.064321517944336,
      "activations/layer1_attention_weight_min": -12.77941608428955,
      "activations/layer20_attention_weight_max": 14.564932823181152,
      "activations/layer20_attention_weight_min": -11.87703800201416,
      "activations/layer21_attention_weight_max": 11.006046295166016,
      "activations/layer21_attention_weight_min": -6.739113807678223,
      "activations/layer22_attention_weight_max": 6.199531555175781,
      "activations/layer22_attention_weight_min": -5.119228839874268,
      "activations/layer23_attention_weight_max": 12.121088981628418,
      "activations/layer23_attention_weight_min": -9.386524200439453,
      "activations/layer2_attention_weight_max": 22.896141052246094,
      "activations/layer2_attention_weight_min": -18.601926803588867,
      "activations/layer3_attention_weight_max": 57.881065368652344,
      "activations/layer3_attention_weight_min": -46.09794616699219,
      "activations/layer4_attention_weight_max": 97.87423706054688,
      "activations/layer4_attention_weight_min": -88.12236785888672,
      "activations/layer5_attention_weight_max": 175.14132690429688,
      "activations/layer5_attention_weight_min": -166.69680786132812,
      "activations/layer6_attention_weight_max": 160.23028564453125,
      "activations/layer6_attention_weight_min": -161.0881805419922,
      "activations/layer7_attention_weight_max": 123.67637634277344,
      "activations/layer7_attention_weight_min": -121.40654754638672,
      "activations/layer8_attention_weight_max": 90.67621612548828,
      "activations/layer8_attention_weight_min": -85.7965316772461,
      "activations/layer9_attention_weight_max": 82.05070495605469,
      "activations/layer9_attention_weight_min": -71.40740203857422,
      "epoch": 0.13,
      "learning_rate": 0.0001398656369269228,
      "loss": 3.8342,
      "step": 2300
    },
    {
      "activations/layer0_attention_weight_max": 11.758427619934082,
      "activations/layer0_attention_weight_min": -9.267449378967285,
      "activations/layer10_attention_weight_max": 56.55110549926758,
      "activations/layer10_attention_weight_min": -53.26154708862305,
      "activations/layer11_attention_weight_max": 24.500110626220703,
      "activations/layer11_attention_weight_min": -19.177988052368164,
      "activations/layer12_attention_weight_max": 21.687477111816406,
      "activations/layer12_attention_weight_min": -25.504093170166016,
      "activations/layer13_attention_weight_max": 19.892528533935547,
      "activations/layer13_attention_weight_min": -18.392080307006836,
      "activations/layer14_attention_weight_max": 22.351850509643555,
      "activations/layer14_attention_weight_min": -14.99757194519043,
      "activations/layer15_attention_weight_max": 21.119077682495117,
      "activations/layer15_attention_weight_min": -13.038750648498535,
      "activations/layer16_attention_weight_max": 17.720348358154297,
      "activations/layer16_attention_weight_min": -12.363571166992188,
      "activations/layer17_attention_weight_max": 17.4597225189209,
      "activations/layer17_attention_weight_min": -15.91393756866455,
      "activations/layer18_attention_weight_max": 18.657468795776367,
      "activations/layer18_attention_weight_min": -11.327492713928223,
      "activations/layer19_attention_weight_max": 15.873023986816406,
      "activations/layer19_attention_weight_min": -10.638632774353027,
      "activations/layer1_attention_weight_max": 17.241872787475586,
      "activations/layer1_attention_weight_min": -12.848546028137207,
      "activations/layer20_attention_weight_max": 16.633432388305664,
      "activations/layer20_attention_weight_min": -11.395513534545898,
      "activations/layer21_attention_weight_max": 9.986958503723145,
      "activations/layer21_attention_weight_min": -6.814957618713379,
      "activations/layer22_attention_weight_max": 5.350459575653076,
      "activations/layer22_attention_weight_min": -4.6802287101745605,
      "activations/layer23_attention_weight_max": 12.254462242126465,
      "activations/layer23_attention_weight_min": -8.834805488586426,
      "activations/layer2_attention_weight_max": 22.765716552734375,
      "activations/layer2_attention_weight_min": -18.375411987304688,
      "activations/layer3_attention_weight_max": 56.52077865600586,
      "activations/layer3_attention_weight_min": -48.25846481323242,
      "activations/layer4_attention_weight_max": 98.16661834716797,
      "activations/layer4_attention_weight_min": -87.92098236083984,
      "activations/layer5_attention_weight_max": 169.3616485595703,
      "activations/layer5_attention_weight_min": -165.60964965820312,
      "activations/layer6_attention_weight_max": 158.33128356933594,
      "activations/layer6_attention_weight_min": -153.5317840576172,
      "activations/layer7_attention_weight_max": 119.67782592773438,
      "activations/layer7_attention_weight_min": -109.85647583007812,
      "activations/layer8_attention_weight_max": 88.83480834960938,
      "activations/layer8_attention_weight_min": -81.62586212158203,
      "activations/layer9_attention_weight_max": 75.94486236572266,
      "activations/layer9_attention_weight_min": -73.65677642822266,
      "epoch": 0.14,
      "learning_rate": 0.0001402572774947444,
      "loss": 3.8501,
      "step": 2350
    },
    {
      "activations/layer0_attention_weight_max": 12.688002586364746,
      "activations/layer0_attention_weight_min": -9.056292533874512,
      "activations/layer10_attention_weight_max": 63.32899856567383,
      "activations/layer10_attention_weight_min": -56.53948211669922,
      "activations/layer11_attention_weight_max": 25.694015502929688,
      "activations/layer11_attention_weight_min": -19.156627655029297,
      "activations/layer12_attention_weight_max": 22.658384323120117,
      "activations/layer12_attention_weight_min": -23.58976173400879,
      "activations/layer13_attention_weight_max": 23.49591827392578,
      "activations/layer13_attention_weight_min": -18.234338760375977,
      "activations/layer14_attention_weight_max": 24.36136245727539,
      "activations/layer14_attention_weight_min": -14.981046676635742,
      "activations/layer15_attention_weight_max": 19.43714141845703,
      "activations/layer15_attention_weight_min": -14.189650535583496,
      "activations/layer16_attention_weight_max": 17.91520881652832,
      "activations/layer16_attention_weight_min": -11.45268440246582,
      "activations/layer17_attention_weight_max": 18.38742446899414,
      "activations/layer17_attention_weight_min": -16.201318740844727,
      "activations/layer18_attention_weight_max": 22.131444931030273,
      "activations/layer18_attention_weight_min": -11.761569023132324,
      "activations/layer19_attention_weight_max": 15.316550254821777,
      "activations/layer19_attention_weight_min": -12.077847480773926,
      "activations/layer1_attention_weight_max": 18.079788208007812,
      "activations/layer1_attention_weight_min": -11.985295295715332,
      "activations/layer20_attention_weight_max": 15.960737228393555,
      "activations/layer20_attention_weight_min": -10.661223411560059,
      "activations/layer21_attention_weight_max": 11.001355171203613,
      "activations/layer21_attention_weight_min": -7.206681728363037,
      "activations/layer22_attention_weight_max": 5.841657638549805,
      "activations/layer22_attention_weight_min": -4.959721565246582,
      "activations/layer23_attention_weight_max": 12.336114883422852,
      "activations/layer23_attention_weight_min": -9.231149673461914,
      "activations/layer2_attention_weight_max": 22.94019889831543,
      "activations/layer2_attention_weight_min": -19.463577270507812,
      "activations/layer3_attention_weight_max": 56.94978332519531,
      "activations/layer3_attention_weight_min": -47.768558502197266,
      "activations/layer4_attention_weight_max": 100.01773071289062,
      "activations/layer4_attention_weight_min": -86.81839752197266,
      "activations/layer5_attention_weight_max": 176.6021728515625,
      "activations/layer5_attention_weight_min": -165.9440155029297,
      "activations/layer6_attention_weight_max": 159.41171264648438,
      "activations/layer6_attention_weight_min": -148.41146850585938,
      "activations/layer7_attention_weight_max": 117.43793487548828,
      "activations/layer7_attention_weight_min": -112.64111328125,
      "activations/layer8_attention_weight_max": 86.01313018798828,
      "activations/layer8_attention_weight_min": -77.34989166259766,
      "activations/layer9_attention_weight_max": 75.50113677978516,
      "activations/layer9_attention_weight_min": -73.91680145263672,
      "epoch": 0.14,
      "learning_rate": 0.00014064061644721923,
      "loss": 3.8346,
      "step": 2400
    },
    {
      "activations/layer0_attention_weight_max": 13.104985237121582,
      "activations/layer0_attention_weight_min": -9.725150108337402,
      "activations/layer10_attention_weight_max": 71.82789611816406,
      "activations/layer10_attention_weight_min": -55.03944778442383,
      "activations/layer11_attention_weight_max": 27.481159210205078,
      "activations/layer11_attention_weight_min": -19.566654205322266,
      "activations/layer12_attention_weight_max": 24.657913208007812,
      "activations/layer12_attention_weight_min": -27.1742000579834,
      "activations/layer13_attention_weight_max": 25.21617317199707,
      "activations/layer13_attention_weight_min": -18.809471130371094,
      "activations/layer14_attention_weight_max": 25.465059280395508,
      "activations/layer14_attention_weight_min": -17.06096076965332,
      "activations/layer15_attention_weight_max": 23.02652359008789,
      "activations/layer15_attention_weight_min": -13.079548835754395,
      "activations/layer16_attention_weight_max": 20.177188873291016,
      "activations/layer16_attention_weight_min": -11.99160099029541,
      "activations/layer17_attention_weight_max": 22.5949649810791,
      "activations/layer17_attention_weight_min": -16.281517028808594,
      "activations/layer18_attention_weight_max": 20.079513549804688,
      "activations/layer18_attention_weight_min": -13.655745506286621,
      "activations/layer19_attention_weight_max": 17.77284812927246,
      "activations/layer19_attention_weight_min": -11.30802059173584,
      "activations/layer1_attention_weight_max": 18.34172821044922,
      "activations/layer1_attention_weight_min": -12.260891914367676,
      "activations/layer20_attention_weight_max": 18.20404624938965,
      "activations/layer20_attention_weight_min": -9.157175064086914,
      "activations/layer21_attention_weight_max": 10.101616859436035,
      "activations/layer21_attention_weight_min": -6.850839138031006,
      "activations/layer22_attention_weight_max": 6.13286018371582,
      "activations/layer22_attention_weight_min": -4.792407512664795,
      "activations/layer23_attention_weight_max": 14.122823715209961,
      "activations/layer23_attention_weight_min": -7.937709331512451,
      "activations/layer2_attention_weight_max": 25.096633911132812,
      "activations/layer2_attention_weight_min": -18.626361846923828,
      "activations/layer3_attention_weight_max": 59.62284851074219,
      "activations/layer3_attention_weight_min": -44.56035614013672,
      "activations/layer4_attention_weight_max": 101.99959564208984,
      "activations/layer4_attention_weight_min": -86.44535827636719,
      "activations/layer5_attention_weight_max": 185.0907440185547,
      "activations/layer5_attention_weight_min": -168.93862915039062,
      "activations/layer6_attention_weight_max": 165.53076171875,
      "activations/layer6_attention_weight_min": -154.160888671875,
      "activations/layer7_attention_weight_max": 124.64616394042969,
      "activations/layer7_attention_weight_min": -110.91960906982422,
      "activations/layer8_attention_weight_max": 92.70174407958984,
      "activations/layer8_attention_weight_min": -80.79624938964844,
      "activations/layer9_attention_weight_max": 83.13835144042969,
      "activations/layer9_attention_weight_min": -67.53290557861328,
      "epoch": 0.14,
      "learning_rate": 0.00014101599843062106,
      "loss": 3.7895,
      "step": 2450
    },
    {
      "activations/layer0_attention_weight_max": 11.456417083740234,
      "activations/layer0_attention_weight_min": -9.333545684814453,
      "activations/layer10_attention_weight_max": 65.02490234375,
      "activations/layer10_attention_weight_min": -52.88085174560547,
      "activations/layer11_attention_weight_max": 26.74449920654297,
      "activations/layer11_attention_weight_min": -20.145021438598633,
      "activations/layer12_attention_weight_max": 21.681034088134766,
      "activations/layer12_attention_weight_min": -24.871932983398438,
      "activations/layer13_attention_weight_max": 25.66880989074707,
      "activations/layer13_attention_weight_min": -20.128360748291016,
      "activations/layer14_attention_weight_max": 24.511003494262695,
      "activations/layer14_attention_weight_min": -17.136436462402344,
      "activations/layer15_attention_weight_max": 19.38917350769043,
      "activations/layer15_attention_weight_min": -13.0552339553833,
      "activations/layer16_attention_weight_max": 20.055675506591797,
      "activations/layer16_attention_weight_min": -10.658665657043457,
      "activations/layer17_attention_weight_max": 18.26388168334961,
      "activations/layer17_attention_weight_min": -14.391984939575195,
      "activations/layer18_attention_weight_max": 25.089258193969727,
      "activations/layer18_attention_weight_min": -12.964506149291992,
      "activations/layer19_attention_weight_max": 21.951513290405273,
      "activations/layer19_attention_weight_min": -11.512600898742676,
      "activations/layer1_attention_weight_max": 17.0645751953125,
      "activations/layer1_attention_weight_min": -11.989842414855957,
      "activations/layer20_attention_weight_max": 17.033742904663086,
      "activations/layer20_attention_weight_min": -10.223241806030273,
      "activations/layer21_attention_weight_max": 14.823901176452637,
      "activations/layer21_attention_weight_min": -8.477492332458496,
      "activations/layer22_attention_weight_max": 8.529914855957031,
      "activations/layer22_attention_weight_min": -5.461702346801758,
      "activations/layer23_attention_weight_max": 12.935850143432617,
      "activations/layer23_attention_weight_min": -9.40350341796875,
      "activations/layer2_attention_weight_max": 23.47553253173828,
      "activations/layer2_attention_weight_min": -18.968313217163086,
      "activations/layer3_attention_weight_max": 58.439666748046875,
      "activations/layer3_attention_weight_min": -45.729896545410156,
      "activations/layer4_attention_weight_max": 97.8577880859375,
      "activations/layer4_attention_weight_min": -87.91138458251953,
      "activations/layer5_attention_weight_max": 172.27822875976562,
      "activations/layer5_attention_weight_min": -166.66680908203125,
      "activations/layer6_attention_weight_max": 161.66815185546875,
      "activations/layer6_attention_weight_min": -152.20816040039062,
      "activations/layer7_attention_weight_max": 110.173828125,
      "activations/layer7_attention_weight_min": -106.69338989257812,
      "activations/layer8_attention_weight_max": 82.64746856689453,
      "activations/layer8_attention_weight_min": -80.67406463623047,
      "activations/layer9_attention_weight_max": 72.41463470458984,
      "activations/layer9_attention_weight_min": -69.66585540771484,
      "epoch": 0.15,
      "learning_rate": 0.00014138374706467038,
      "loss": 3.7966,
      "step": 2500
    },
    {
      "activations/layer0_attention_weight_max": 12.077981948852539,
      "activations/layer0_attention_weight_min": -8.675317764282227,
      "activations/layer10_attention_weight_max": 63.731597900390625,
      "activations/layer10_attention_weight_min": -57.41379928588867,
      "activations/layer11_attention_weight_max": 24.05840301513672,
      "activations/layer11_attention_weight_min": -18.733570098876953,
      "activations/layer12_attention_weight_max": 24.21148109436035,
      "activations/layer12_attention_weight_min": -22.698944091796875,
      "activations/layer13_attention_weight_max": 22.64223289489746,
      "activations/layer13_attention_weight_min": -19.426822662353516,
      "activations/layer14_attention_weight_max": 23.259571075439453,
      "activations/layer14_attention_weight_min": -16.252933502197266,
      "activations/layer15_attention_weight_max": 19.287641525268555,
      "activations/layer15_attention_weight_min": -13.703564643859863,
      "activations/layer16_attention_weight_max": 16.675662994384766,
      "activations/layer16_attention_weight_min": -11.260688781738281,
      "activations/layer17_attention_weight_max": 17.38075828552246,
      "activations/layer17_attention_weight_min": -16.26795196533203,
      "activations/layer18_attention_weight_max": 19.3206787109375,
      "activations/layer18_attention_weight_min": -12.42951774597168,
      "activations/layer19_attention_weight_max": 15.3689546585083,
      "activations/layer19_attention_weight_min": -10.990196228027344,
      "activations/layer1_attention_weight_max": 18.458110809326172,
      "activations/layer1_attention_weight_min": -13.290287017822266,
      "activations/layer20_attention_weight_max": 18.961166381835938,
      "activations/layer20_attention_weight_min": -10.93006706237793,
      "activations/layer21_attention_weight_max": 12.436419486999512,
      "activations/layer21_attention_weight_min": -10.595160484313965,
      "activations/layer22_attention_weight_max": 6.498691082000732,
      "activations/layer22_attention_weight_min": -5.198154926300049,
      "activations/layer23_attention_weight_max": 12.629134178161621,
      "activations/layer23_attention_weight_min": -9.142122268676758,
      "activations/layer2_attention_weight_max": 24.883880615234375,
      "activations/layer2_attention_weight_min": -21.67321014404297,
      "activations/layer3_attention_weight_max": 58.319496154785156,
      "activations/layer3_attention_weight_min": -45.99992752075195,
      "activations/layer4_attention_weight_max": 92.7694320678711,
      "activations/layer4_attention_weight_min": -87.19029235839844,
      "activations/layer5_attention_weight_max": 168.99024963378906,
      "activations/layer5_attention_weight_min": -167.11314392089844,
      "activations/layer6_attention_weight_max": 156.3563690185547,
      "activations/layer6_attention_weight_min": -153.68634033203125,
      "activations/layer7_attention_weight_max": 108.3841552734375,
      "activations/layer7_attention_weight_min": -107.06183624267578,
      "activations/layer8_attention_weight_max": 80.33732604980469,
      "activations/layer8_attention_weight_min": -80.7423095703125,
      "activations/layer9_attention_weight_max": 74.94869232177734,
      "activations/layer9_attention_weight_min": -78.25807189941406,
      "epoch": 0.15,
      "learning_rate": 0.00014174416661891364,
      "loss": 3.7986,
      "step": 2550
    },
    {
      "activations/layer0_attention_weight_max": 12.764163970947266,
      "activations/layer0_attention_weight_min": -8.558411598205566,
      "activations/layer10_attention_weight_max": 64.39186096191406,
      "activations/layer10_attention_weight_min": -51.422569274902344,
      "activations/layer11_attention_weight_max": 30.986326217651367,
      "activations/layer11_attention_weight_min": -19.7949275970459,
      "activations/layer12_attention_weight_max": 23.5411434173584,
      "activations/layer12_attention_weight_min": -24.491104125976562,
      "activations/layer13_attention_weight_max": 25.482091903686523,
      "activations/layer13_attention_weight_min": -22.567914962768555,
      "activations/layer14_attention_weight_max": 25.472156524658203,
      "activations/layer14_attention_weight_min": -16.484737396240234,
      "activations/layer15_attention_weight_max": 21.835256576538086,
      "activations/layer15_attention_weight_min": -14.793864250183105,
      "activations/layer16_attention_weight_max": 21.806726455688477,
      "activations/layer16_attention_weight_min": -13.13414478302002,
      "activations/layer17_attention_weight_max": 19.875991821289062,
      "activations/layer17_attention_weight_min": -14.66153335571289,
      "activations/layer18_attention_weight_max": 23.988426208496094,
      "activations/layer18_attention_weight_min": -13.550394058227539,
      "activations/layer19_attention_weight_max": 20.699180603027344,
      "activations/layer19_attention_weight_min": -12.412898063659668,
      "activations/layer1_attention_weight_max": 18.091148376464844,
      "activations/layer1_attention_weight_min": -12.792560577392578,
      "activations/layer20_attention_weight_max": 18.290924072265625,
      "activations/layer20_attention_weight_min": -11.580103874206543,
      "activations/layer21_attention_weight_max": 15.111234664916992,
      "activations/layer21_attention_weight_min": -9.553041458129883,
      "activations/layer22_attention_weight_max": 8.168923377990723,
      "activations/layer22_attention_weight_min": -6.111292362213135,
      "activations/layer23_attention_weight_max": 13.188039779663086,
      "activations/layer23_attention_weight_min": -9.150289535522461,
      "activations/layer2_attention_weight_max": 25.185958862304688,
      "activations/layer2_attention_weight_min": -19.576263427734375,
      "activations/layer3_attention_weight_max": 56.8466911315918,
      "activations/layer3_attention_weight_min": -46.361663818359375,
      "activations/layer4_attention_weight_max": 93.17852020263672,
      "activations/layer4_attention_weight_min": -87.1771469116211,
      "activations/layer5_attention_weight_max": 168.21713256835938,
      "activations/layer5_attention_weight_min": -167.35116577148438,
      "activations/layer6_attention_weight_max": 150.416259765625,
      "activations/layer6_attention_weight_min": -151.21580505371094,
      "activations/layer7_attention_weight_max": 107.56684112548828,
      "activations/layer7_attention_weight_min": -107.92533111572266,
      "activations/layer8_attention_weight_max": 82.87035369873047,
      "activations/layer8_attention_weight_min": -79.31694030761719,
      "activations/layer9_attention_weight_max": 70.85875701904297,
      "activations/layer9_attention_weight_min": -69.522705078125,
      "epoch": 0.15,
      "learning_rate": 0.00014209754352529724,
      "loss": 3.7728,
      "step": 2600
    },
    {
      "activations/layer0_attention_weight_max": 15.339887619018555,
      "activations/layer0_attention_weight_min": -8.957226753234863,
      "activations/layer10_attention_weight_max": 63.32475280761719,
      "activations/layer10_attention_weight_min": -54.45529556274414,
      "activations/layer11_attention_weight_max": 24.842239379882812,
      "activations/layer11_attention_weight_min": -18.412395477294922,
      "activations/layer12_attention_weight_max": 21.05619239807129,
      "activations/layer12_attention_weight_min": -24.925743103027344,
      "activations/layer13_attention_weight_max": 20.873703002929688,
      "activations/layer13_attention_weight_min": -17.636966705322266,
      "activations/layer14_attention_weight_max": 22.899658203125,
      "activations/layer14_attention_weight_min": -14.978691101074219,
      "activations/layer15_attention_weight_max": 18.67153549194336,
      "activations/layer15_attention_weight_min": -13.179362297058105,
      "activations/layer16_attention_weight_max": 19.756113052368164,
      "activations/layer16_attention_weight_min": -12.467065811157227,
      "activations/layer17_attention_weight_max": 18.016618728637695,
      "activations/layer17_attention_weight_min": -14.417509078979492,
      "activations/layer18_attention_weight_max": 18.938936233520508,
      "activations/layer18_attention_weight_min": -11.7967529296875,
      "activations/layer19_attention_weight_max": 19.955432891845703,
      "activations/layer19_attention_weight_min": -10.205793380737305,
      "activations/layer1_attention_weight_max": 17.389083862304688,
      "activations/layer1_attention_weight_min": -11.310197830200195,
      "activations/layer20_attention_weight_max": 13.707602500915527,
      "activations/layer20_attention_weight_min": -10.857429504394531,
      "activations/layer21_attention_weight_max": 14.553902626037598,
      "activations/layer21_attention_weight_min": -9.125730514526367,
      "activations/layer22_attention_weight_max": 8.150019645690918,
      "activations/layer22_attention_weight_min": -6.126676559448242,
      "activations/layer23_attention_weight_max": 12.885662078857422,
      "activations/layer23_attention_weight_min": -8.183874130249023,
      "activations/layer2_attention_weight_max": 23.36603546142578,
      "activations/layer2_attention_weight_min": -18.92416763305664,
      "activations/layer3_attention_weight_max": 56.970699310302734,
      "activations/layer3_attention_weight_min": -46.808712005615234,
      "activations/layer4_attention_weight_max": 95.82991027832031,
      "activations/layer4_attention_weight_min": -89.55435180664062,
      "activations/layer5_attention_weight_max": 174.5181121826172,
      "activations/layer5_attention_weight_min": -169.67776489257812,
      "activations/layer6_attention_weight_max": 158.22076416015625,
      "activations/layer6_attention_weight_min": -160.4736785888672,
      "activations/layer7_attention_weight_max": 113.18815612792969,
      "activations/layer7_attention_weight_min": -114.45299530029297,
      "activations/layer8_attention_weight_max": 85.45924377441406,
      "activations/layer8_attention_weight_min": -79.79306030273438,
      "activations/layer9_attention_weight_max": 74.87811279296875,
      "activations/layer9_attention_weight_min": -69.66826629638672,
      "epoch": 0.15,
      "learning_rate": 0.00014244414774577482,
      "loss": 3.7654,
      "step": 2650
    },
    {
      "activations/layer0_attention_weight_max": 12.439830780029297,
      "activations/layer0_attention_weight_min": -8.970267295837402,
      "activations/layer10_attention_weight_max": 66.35906982421875,
      "activations/layer10_attention_weight_min": -51.75226974487305,
      "activations/layer11_attention_weight_max": 35.82286071777344,
      "activations/layer11_attention_weight_min": -20.107980728149414,
      "activations/layer12_attention_weight_max": 22.69931411743164,
      "activations/layer12_attention_weight_min": -24.851537704467773,
      "activations/layer13_attention_weight_max": 23.390949249267578,
      "activations/layer13_attention_weight_min": -19.52634048461914,
      "activations/layer14_attention_weight_max": 23.88496971130371,
      "activations/layer14_attention_weight_min": -15.17501163482666,
      "activations/layer15_attention_weight_max": 19.816396713256836,
      "activations/layer15_attention_weight_min": -12.742082595825195,
      "activations/layer16_attention_weight_max": 18.552045822143555,
      "activations/layer16_attention_weight_min": -11.765106201171875,
      "activations/layer17_attention_weight_max": 22.569133758544922,
      "activations/layer17_attention_weight_min": -16.9661808013916,
      "activations/layer18_attention_weight_max": 22.480693817138672,
      "activations/layer18_attention_weight_min": -12.061013221740723,
      "activations/layer19_attention_weight_max": 18.237060546875,
      "activations/layer19_attention_weight_min": -12.301942825317383,
      "activations/layer1_attention_weight_max": 17.761028289794922,
      "activations/layer1_attention_weight_min": -12.595542907714844,
      "activations/layer20_attention_weight_max": 19.715959548950195,
      "activations/layer20_attention_weight_min": -10.565086364746094,
      "activations/layer21_attention_weight_max": 13.88475513458252,
      "activations/layer21_attention_weight_min": -10.33631420135498,
      "activations/layer22_attention_weight_max": 8.188741683959961,
      "activations/layer22_attention_weight_min": -6.142240524291992,
      "activations/layer23_attention_weight_max": 11.617506980895996,
      "activations/layer23_attention_weight_min": -7.776578426361084,
      "activations/layer2_attention_weight_max": 25.78169822692871,
      "activations/layer2_attention_weight_min": -18.877944946289062,
      "activations/layer3_attention_weight_max": 57.52930450439453,
      "activations/layer3_attention_weight_min": -46.47740173339844,
      "activations/layer4_attention_weight_max": 96.49452209472656,
      "activations/layer4_attention_weight_min": -86.52085876464844,
      "activations/layer5_attention_weight_max": 179.69029235839844,
      "activations/layer5_attention_weight_min": -166.25234985351562,
      "activations/layer6_attention_weight_max": 154.01841735839844,
      "activations/layer6_attention_weight_min": -143.989990234375,
      "activations/layer7_attention_weight_max": 112.37500762939453,
      "activations/layer7_attention_weight_min": -103.3577880859375,
      "activations/layer8_attention_weight_max": 87.5347900390625,
      "activations/layer8_attention_weight_min": -78.14877319335938,
      "activations/layer9_attention_weight_max": 77.1528091430664,
      "activations/layer9_attention_weight_min": -68.27039337158203,
      "epoch": 0.16,
      "learning_rate": 0.00014278423401130834,
      "loss": 3.7599,
      "step": 2700
    },
    {
      "activations/layer0_attention_weight_max": 13.117350578308105,
      "activations/layer0_attention_weight_min": -8.921953201293945,
      "activations/layer10_attention_weight_max": 65.83438873291016,
      "activations/layer10_attention_weight_min": -55.219451904296875,
      "activations/layer11_attention_weight_max": 26.082111358642578,
      "activations/layer11_attention_weight_min": -17.95010757446289,
      "activations/layer12_attention_weight_max": 22.8288631439209,
      "activations/layer12_attention_weight_min": -22.057966232299805,
      "activations/layer13_attention_weight_max": 22.163593292236328,
      "activations/layer13_attention_weight_min": -23.245647430419922,
      "activations/layer14_attention_weight_max": 24.16008949279785,
      "activations/layer14_attention_weight_min": -16.69082260131836,
      "activations/layer15_attention_weight_max": 18.615915298461914,
      "activations/layer15_attention_weight_min": -13.660847663879395,
      "activations/layer16_attention_weight_max": 21.37920379638672,
      "activations/layer16_attention_weight_min": -11.881958961486816,
      "activations/layer17_attention_weight_max": 17.413257598876953,
      "activations/layer17_attention_weight_min": -15.144539833068848,
      "activations/layer18_attention_weight_max": 18.768396377563477,
      "activations/layer18_attention_weight_min": -13.14130973815918,
      "activations/layer19_attention_weight_max": 18.942537307739258,
      "activations/layer19_attention_weight_min": -11.791200637817383,
      "activations/layer1_attention_weight_max": 18.064241409301758,
      "activations/layer1_attention_weight_min": -11.735589981079102,
      "activations/layer20_attention_weight_max": 17.993274688720703,
      "activations/layer20_attention_weight_min": -11.084029197692871,
      "activations/layer21_attention_weight_max": 14.777252197265625,
      "activations/layer21_attention_weight_min": -10.486763000488281,
      "activations/layer22_attention_weight_max": 8.264402389526367,
      "activations/layer22_attention_weight_min": -7.694353103637695,
      "activations/layer23_attention_weight_max": 13.972665786743164,
      "activations/layer23_attention_weight_min": -8.80261516571045,
      "activations/layer2_attention_weight_max": 25.384939193725586,
      "activations/layer2_attention_weight_min": -19.79253387451172,
      "activations/layer3_attention_weight_max": 58.75847625732422,
      "activations/layer3_attention_weight_min": -47.73459243774414,
      "activations/layer4_attention_weight_max": 97.5513687133789,
      "activations/layer4_attention_weight_min": -86.9202880859375,
      "activations/layer5_attention_weight_max": 179.6486053466797,
      "activations/layer5_attention_weight_min": -165.74478149414062,
      "activations/layer6_attention_weight_max": 153.43865966796875,
      "activations/layer6_attention_weight_min": -147.63365173339844,
      "activations/layer7_attention_weight_max": 119.23951721191406,
      "activations/layer7_attention_weight_min": -108.33696746826172,
      "activations/layer8_attention_weight_max": 87.55769348144531,
      "activations/layer8_attention_weight_min": -79.18537902832031,
      "activations/layer9_attention_weight_max": 78.99835205078125,
      "activations/layer9_attention_weight_min": -68.10237884521484,
      "epoch": 0.16,
      "learning_rate": 0.00014311804294650713,
      "loss": 3.743,
      "step": 2750
    },
    {
      "activations/layer0_attention_weight_max": 12.406450271606445,
      "activations/layer0_attention_weight_min": -8.967166900634766,
      "activations/layer10_attention_weight_max": 66.05374908447266,
      "activations/layer10_attention_weight_min": -57.719810485839844,
      "activations/layer11_attention_weight_max": 29.11117935180664,
      "activations/layer11_attention_weight_min": -20.104923248291016,
      "activations/layer12_attention_weight_max": 21.54225730895996,
      "activations/layer12_attention_weight_min": -20.693702697753906,
      "activations/layer13_attention_weight_max": 23.921405792236328,
      "activations/layer13_attention_weight_min": -21.123435974121094,
      "activations/layer14_attention_weight_max": 25.31205177307129,
      "activations/layer14_attention_weight_min": -14.892953872680664,
      "activations/layer15_attention_weight_max": 19.558500289916992,
      "activations/layer15_attention_weight_min": -12.336824417114258,
      "activations/layer16_attention_weight_max": 22.629457473754883,
      "activations/layer16_attention_weight_min": -11.152676582336426,
      "activations/layer17_attention_weight_max": 19.066604614257812,
      "activations/layer17_attention_weight_min": -14.112610816955566,
      "activations/layer18_attention_weight_max": 20.60256576538086,
      "activations/layer18_attention_weight_min": -11.767054557800293,
      "activations/layer19_attention_weight_max": 18.703947067260742,
      "activations/layer19_attention_weight_min": -10.077123641967773,
      "activations/layer1_attention_weight_max": 17.634883880615234,
      "activations/layer1_attention_weight_min": -11.189332962036133,
      "activations/layer20_attention_weight_max": 24.564592361450195,
      "activations/layer20_attention_weight_min": -10.650802612304688,
      "activations/layer21_attention_weight_max": 14.979973793029785,
      "activations/layer21_attention_weight_min": -10.3433198928833,
      "activations/layer22_attention_weight_max": 10.681516647338867,
      "activations/layer22_attention_weight_min": -7.675937175750732,
      "activations/layer23_attention_weight_max": 12.231605529785156,
      "activations/layer23_attention_weight_min": -9.543588638305664,
      "activations/layer2_attention_weight_max": 23.760635375976562,
      "activations/layer2_attention_weight_min": -18.40867042541504,
      "activations/layer3_attention_weight_max": 52.96567153930664,
      "activations/layer3_attention_weight_min": -46.70649337768555,
      "activations/layer4_attention_weight_max": 90.01690673828125,
      "activations/layer4_attention_weight_min": -85.95304870605469,
      "activations/layer5_attention_weight_max": 163.02027893066406,
      "activations/layer5_attention_weight_min": -165.80551147460938,
      "activations/layer6_attention_weight_max": 140.9651336669922,
      "activations/layer6_attention_weight_min": -144.65884399414062,
      "activations/layer7_attention_weight_max": 104.85860443115234,
      "activations/layer7_attention_weight_min": -104.98143768310547,
      "activations/layer8_attention_weight_max": 80.07757568359375,
      "activations/layer8_attention_weight_min": -78.39225006103516,
      "activations/layer9_attention_weight_max": 73.43462371826172,
      "activations/layer9_attention_weight_min": -73.0586166381836,
      "epoch": 0.16,
      "learning_rate": 0.00014344580209234327,
      "loss": 3.7269,
      "step": 2800
    },
    {
      "activations/layer0_attention_weight_max": 15.145135879516602,
      "activations/layer0_attention_weight_min": -8.528341293334961,
      "activations/layer10_attention_weight_max": 61.76569366455078,
      "activations/layer10_attention_weight_min": -54.552608489990234,
      "activations/layer11_attention_weight_max": 24.43328857421875,
      "activations/layer11_attention_weight_min": -18.039209365844727,
      "activations/layer12_attention_weight_max": 20.156213760375977,
      "activations/layer12_attention_weight_min": -21.82403564453125,
      "activations/layer13_attention_weight_max": 22.76873016357422,
      "activations/layer13_attention_weight_min": -17.444650650024414,
      "activations/layer14_attention_weight_max": 20.948963165283203,
      "activations/layer14_attention_weight_min": -14.505172729492188,
      "activations/layer15_attention_weight_max": 18.687528610229492,
      "activations/layer15_attention_weight_min": -12.275278091430664,
      "activations/layer16_attention_weight_max": 16.32059669494629,
      "activations/layer16_attention_weight_min": -11.898611068725586,
      "activations/layer17_attention_weight_max": 16.364580154418945,
      "activations/layer17_attention_weight_min": -13.862329483032227,
      "activations/layer18_attention_weight_max": 16.586658477783203,
      "activations/layer18_attention_weight_min": -12.679147720336914,
      "activations/layer19_attention_weight_max": 17.122724533081055,
      "activations/layer19_attention_weight_min": -10.871630668640137,
      "activations/layer1_attention_weight_max": 16.7828311920166,
      "activations/layer1_attention_weight_min": -12.516300201416016,
      "activations/layer20_attention_weight_max": 14.511907577514648,
      "activations/layer20_attention_weight_min": -11.546663284301758,
      "activations/layer21_attention_weight_max": 13.688246726989746,
      "activations/layer21_attention_weight_min": -11.639006614685059,
      "activations/layer22_attention_weight_max": 10.405704498291016,
      "activations/layer22_attention_weight_min": -6.983856201171875,
      "activations/layer23_attention_weight_max": 10.966230392456055,
      "activations/layer23_attention_weight_min": -8.068302154541016,
      "activations/layer2_attention_weight_max": 26.004301071166992,
      "activations/layer2_attention_weight_min": -19.749032974243164,
      "activations/layer3_attention_weight_max": 59.59838104248047,
      "activations/layer3_attention_weight_min": -45.18400192260742,
      "activations/layer4_attention_weight_max": 99.5232162475586,
      "activations/layer4_attention_weight_min": -83.33858489990234,
      "activations/layer5_attention_weight_max": 181.27365112304688,
      "activations/layer5_attention_weight_min": -156.74697875976562,
      "activations/layer6_attention_weight_max": 158.0591583251953,
      "activations/layer6_attention_weight_min": -142.5275421142578,
      "activations/layer7_attention_weight_max": 114.52262115478516,
      "activations/layer7_attention_weight_min": -104.25800323486328,
      "activations/layer8_attention_weight_max": 88.37174987792969,
      "activations/layer8_attention_weight_min": -74.83700561523438,
      "activations/layer9_attention_weight_max": 76.94585418701172,
      "activations/layer9_attention_weight_min": -69.82878112792969,
      "epoch": 0.17,
      "learning_rate": 0.0001437677268378304,
      "loss": 3.7337,
      "step": 2850
    },
    {
      "activations/layer0_attention_weight_max": 12.902713775634766,
      "activations/layer0_attention_weight_min": -8.365883827209473,
      "activations/layer10_attention_weight_max": 69.75586700439453,
      "activations/layer10_attention_weight_min": -57.277889251708984,
      "activations/layer11_attention_weight_max": 26.395767211914062,
      "activations/layer11_attention_weight_min": -17.942596435546875,
      "activations/layer12_attention_weight_max": 29.536279678344727,
      "activations/layer12_attention_weight_min": -23.56142807006836,
      "activations/layer13_attention_weight_max": 26.513614654541016,
      "activations/layer13_attention_weight_min": -19.57140350341797,
      "activations/layer14_attention_weight_max": 27.121158599853516,
      "activations/layer14_attention_weight_min": -15.880775451660156,
      "activations/layer15_attention_weight_max": 22.532466888427734,
      "activations/layer15_attention_weight_min": -14.725616455078125,
      "activations/layer16_attention_weight_max": 23.2841854095459,
      "activations/layer16_attention_weight_min": -13.666694641113281,
      "activations/layer17_attention_weight_max": 18.919946670532227,
      "activations/layer17_attention_weight_min": -14.514738082885742,
      "activations/layer18_attention_weight_max": 19.779052734375,
      "activations/layer18_attention_weight_min": -14.156242370605469,
      "activations/layer19_attention_weight_max": 15.60136604309082,
      "activations/layer19_attention_weight_min": -9.853653907775879,
      "activations/layer1_attention_weight_max": 17.29155158996582,
      "activations/layer1_attention_weight_min": -14.025238037109375,
      "activations/layer20_attention_weight_max": 24.329509735107422,
      "activations/layer20_attention_weight_min": -11.638567924499512,
      "activations/layer21_attention_weight_max": 17.585250854492188,
      "activations/layer21_attention_weight_min": -11.126222610473633,
      "activations/layer22_attention_weight_max": 11.339859962463379,
      "activations/layer22_attention_weight_min": -7.106887340545654,
      "activations/layer23_attention_weight_max": 11.938922882080078,
      "activations/layer23_attention_weight_min": -10.10919189453125,
      "activations/layer2_attention_weight_max": 24.784282684326172,
      "activations/layer2_attention_weight_min": -18.405811309814453,
      "activations/layer3_attention_weight_max": 60.691200256347656,
      "activations/layer3_attention_weight_min": -46.31121826171875,
      "activations/layer4_attention_weight_max": 103.2119369506836,
      "activations/layer4_attention_weight_min": -86.76988983154297,
      "activations/layer5_attention_weight_max": 192.30918884277344,
      "activations/layer5_attention_weight_min": -164.78897094726562,
      "activations/layer6_attention_weight_max": 168.77767944335938,
      "activations/layer6_attention_weight_min": -146.50335693359375,
      "activations/layer7_attention_weight_max": 120.32821655273438,
      "activations/layer7_attention_weight_min": -110.0719223022461,
      "activations/layer8_attention_weight_max": 90.5468521118164,
      "activations/layer8_attention_weight_min": -76.98728942871094,
      "activations/layer9_attention_weight_max": 81.56346893310547,
      "activations/layer9_attention_weight_min": -74.00287628173828,
      "epoch": 0.17,
      "learning_rate": 0.00014408402127022058,
      "loss": 3.7023,
      "step": 2900
    },
    {
      "activations/layer0_attention_weight_max": 12.517102241516113,
      "activations/layer0_attention_weight_min": -9.138641357421875,
      "activations/layer10_attention_weight_max": 73.1766128540039,
      "activations/layer10_attention_weight_min": -58.36238479614258,
      "activations/layer11_attention_weight_max": 27.220029830932617,
      "activations/layer11_attention_weight_min": -20.328149795532227,
      "activations/layer12_attention_weight_max": 22.76612091064453,
      "activations/layer12_attention_weight_min": -24.563840866088867,
      "activations/layer13_attention_weight_max": 23.011430740356445,
      "activations/layer13_attention_weight_min": -20.251575469970703,
      "activations/layer14_attention_weight_max": 23.983409881591797,
      "activations/layer14_attention_weight_min": -14.80150318145752,
      "activations/layer15_attention_weight_max": 18.36054801940918,
      "activations/layer15_attention_weight_min": -13.067950248718262,
      "activations/layer16_attention_weight_max": 19.879871368408203,
      "activations/layer16_attention_weight_min": -13.096776008605957,
      "activations/layer17_attention_weight_max": 20.052820205688477,
      "activations/layer17_attention_weight_min": -15.582236289978027,
      "activations/layer18_attention_weight_max": 28.669918060302734,
      "activations/layer18_attention_weight_min": -12.923084259033203,
      "activations/layer19_attention_weight_max": 20.486801147460938,
      "activations/layer19_attention_weight_min": -10.314311981201172,
      "activations/layer1_attention_weight_max": 18.460342407226562,
      "activations/layer1_attention_weight_min": -11.5208158493042,
      "activations/layer20_attention_weight_max": 20.756162643432617,
      "activations/layer20_attention_weight_min": -10.882437705993652,
      "activations/layer21_attention_weight_max": 17.815738677978516,
      "activations/layer21_attention_weight_min": -11.639492988586426,
      "activations/layer22_attention_weight_max": 11.552417755126953,
      "activations/layer22_attention_weight_min": -8.590027809143066,
      "activations/layer23_attention_weight_max": 13.967116355895996,
      "activations/layer23_attention_weight_min": -8.279932022094727,
      "activations/layer2_attention_weight_max": 26.125709533691406,
      "activations/layer2_attention_weight_min": -18.753555297851562,
      "activations/layer3_attention_weight_max": 58.75102615356445,
      "activations/layer3_attention_weight_min": -45.979488372802734,
      "activations/layer4_attention_weight_max": 97.12385559082031,
      "activations/layer4_attention_weight_min": -86.4360580444336,
      "activations/layer5_attention_weight_max": 177.00674438476562,
      "activations/layer5_attention_weight_min": -163.80848693847656,
      "activations/layer6_attention_weight_max": 149.4360809326172,
      "activations/layer6_attention_weight_min": -143.69993591308594,
      "activations/layer7_attention_weight_max": 120.70972442626953,
      "activations/layer7_attention_weight_min": -112.48909759521484,
      "activations/layer8_attention_weight_max": 86.4224624633789,
      "activations/layer8_attention_weight_min": -78.42134094238281,
      "activations/layer9_attention_weight_max": 80.24462127685547,
      "activations/layer9_attention_weight_min": -73.25202941894531,
      "epoch": 0.17,
      "learning_rate": 0.0001443948789521239,
      "loss": 3.7054,
      "step": 2950
    },
    {
      "activations/layer0_attention_weight_max": 12.797497749328613,
      "activations/layer0_attention_weight_min": -8.722874641418457,
      "activations/layer10_attention_weight_max": 67.5064697265625,
      "activations/layer10_attention_weight_min": -58.813323974609375,
      "activations/layer11_attention_weight_max": 29.20344352722168,
      "activations/layer11_attention_weight_min": -20.44711685180664,
      "activations/layer12_attention_weight_max": 21.931976318359375,
      "activations/layer12_attention_weight_min": -25.196231842041016,
      "activations/layer13_attention_weight_max": 23.240497589111328,
      "activations/layer13_attention_weight_min": -18.56332015991211,
      "activations/layer14_attention_weight_max": 23.39327049255371,
      "activations/layer14_attention_weight_min": -15.922884941101074,
      "activations/layer15_attention_weight_max": 21.237823486328125,
      "activations/layer15_attention_weight_min": -12.1951265335083,
      "activations/layer16_attention_weight_max": 26.426340103149414,
      "activations/layer16_attention_weight_min": -12.769564628601074,
      "activations/layer17_attention_weight_max": 20.811704635620117,
      "activations/layer17_attention_weight_min": -14.270084381103516,
      "activations/layer18_attention_weight_max": 27.922119140625,
      "activations/layer18_attention_weight_min": -12.115983963012695,
      "activations/layer19_attention_weight_max": 24.064382553100586,
      "activations/layer19_attention_weight_min": -10.625445365905762,
      "activations/layer1_attention_weight_max": 17.16322135925293,
      "activations/layer1_attention_weight_min": -12.025853157043457,
      "activations/layer20_attention_weight_max": 21.923093795776367,
      "activations/layer20_attention_weight_min": -13.98641300201416,
      "activations/layer21_attention_weight_max": 18.139019012451172,
      "activations/layer21_attention_weight_min": -10.973811149597168,
      "activations/layer22_attention_weight_max": 12.226771354675293,
      "activations/layer22_attention_weight_min": -8.377294540405273,
      "activations/layer23_attention_weight_max": 13.555704116821289,
      "activations/layer23_attention_weight_min": -8.538458824157715,
      "activations/layer2_attention_weight_max": 24.379596710205078,
      "activations/layer2_attention_weight_min": -20.02859115600586,
      "activations/layer3_attention_weight_max": 56.47306442260742,
      "activations/layer3_attention_weight_min": -46.2700080871582,
      "activations/layer4_attention_weight_max": 94.84149169921875,
      "activations/layer4_attention_weight_min": -84.66476440429688,
      "activations/layer5_attention_weight_max": 170.10824584960938,
      "activations/layer5_attention_weight_min": -160.96221923828125,
      "activations/layer6_attention_weight_max": 151.363037109375,
      "activations/layer6_attention_weight_min": -145.04025268554688,
      "activations/layer7_attention_weight_max": 109.79663848876953,
      "activations/layer7_attention_weight_min": -104.45062255859375,
      "activations/layer8_attention_weight_max": 80.00585174560547,
      "activations/layer8_attention_weight_min": -77.85957336425781,
      "activations/layer9_attention_weight_max": 72.4720458984375,
      "activations/layer9_attention_weight_min": -67.76892852783203,
      "epoch": 0.17,
      "learning_rate": 0.00014470048363296077,
      "loss": 3.6901,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_loss": 3.625,
      "eval_runtime": 8.2791,
      "eval_samples_per_second": 518.657,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_openwebtext_loss": 3.625,
      "eval_openwebtext_ppl": 37.524723159601,
      "eval_openwebtext_runtime": 8.2791,
      "eval_openwebtext_samples_per_second": 518.657,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_wikitext_loss": 4.0,
      "eval_wikitext_ppl": 54.598150033144236,
      "eval_wikitext_runtime": 1.696,
      "eval_wikitext_samples_per_second": 268.873,
      "step": 3000
    },
    {
      "epoch": 0.17,
      "eval_lambada_loss": 4.84375,
      "eval_lambada_ppl": 126.94450214932196,
      "eval_lambada_runtime": 9.2681,
      "eval_lambada_samples_per_second": 525.353,
      "step": 3000
    },
    {
      "activations/layer0_attention_weight_max": 12.435912132263184,
      "activations/layer0_attention_weight_min": -8.9002685546875,
      "activations/layer10_attention_weight_max": 74.80474853515625,
      "activations/layer10_attention_weight_min": -54.96963882446289,
      "activations/layer11_attention_weight_max": 27.73373031616211,
      "activations/layer11_attention_weight_min": -19.825576782226562,
      "activations/layer12_attention_weight_max": 20.910980224609375,
      "activations/layer12_attention_weight_min": -24.862503051757812,
      "activations/layer13_attention_weight_max": 23.272680282592773,
      "activations/layer13_attention_weight_min": -21.392818450927734,
      "activations/layer14_attention_weight_max": 24.663686752319336,
      "activations/layer14_attention_weight_min": -16.4290771484375,
      "activations/layer15_attention_weight_max": 22.057640075683594,
      "activations/layer15_attention_weight_min": -14.925469398498535,
      "activations/layer16_attention_weight_max": 22.475175857543945,
      "activations/layer16_attention_weight_min": -11.176857948303223,
      "activations/layer17_attention_weight_max": 28.835187911987305,
      "activations/layer17_attention_weight_min": -17.494333267211914,
      "activations/layer18_attention_weight_max": 21.647310256958008,
      "activations/layer18_attention_weight_min": -14.072784423828125,
      "activations/layer19_attention_weight_max": 20.048734664916992,
      "activations/layer19_attention_weight_min": -11.606810569763184,
      "activations/layer1_attention_weight_max": 17.26198387145996,
      "activations/layer1_attention_weight_min": -12.123202323913574,
      "activations/layer20_attention_weight_max": 20.1784725189209,
      "activations/layer20_attention_weight_min": -13.662580490112305,
      "activations/layer21_attention_weight_max": 18.13604736328125,
      "activations/layer21_attention_weight_min": -13.180663108825684,
      "activations/layer22_attention_weight_max": 11.974907875061035,
      "activations/layer22_attention_weight_min": -8.050271034240723,
      "activations/layer23_attention_weight_max": 15.704588890075684,
      "activations/layer23_attention_weight_min": -9.755104064941406,
      "activations/layer2_attention_weight_max": 24.984838485717773,
      "activations/layer2_attention_weight_min": -19.73166275024414,
      "activations/layer3_attention_weight_max": 57.910728454589844,
      "activations/layer3_attention_weight_min": -45.57123565673828,
      "activations/layer4_attention_weight_max": 95.68069458007812,
      "activations/layer4_attention_weight_min": -84.04263305664062,
      "activations/layer5_attention_weight_max": 176.3250732421875,
      "activations/layer5_attention_weight_min": -159.44229125976562,
      "activations/layer6_attention_weight_max": 154.01034545898438,
      "activations/layer6_attention_weight_min": -140.59304809570312,
      "activations/layer7_attention_weight_max": 117.97476959228516,
      "activations/layer7_attention_weight_min": -103.48423767089844,
      "activations/layer8_attention_weight_max": 86.86247253417969,
      "activations/layer8_attention_weight_min": -72.89154052734375,
      "activations/layer9_attention_weight_max": 83.83373260498047,
      "activations/layer9_attention_weight_min": -69.72562408447266,
      "epoch": 0.18,
      "learning_rate": 0.00014499504805694574,
      "loss": 3.6934,
      "step": 3050
    },
    {
      "activations/layer0_attention_weight_max": 12.299686431884766,
      "activations/layer0_attention_weight_min": -9.070984840393066,
      "activations/layer10_attention_weight_max": 65.4179458618164,
      "activations/layer10_attention_weight_min": -54.795379638671875,
      "activations/layer11_attention_weight_max": 24.163002014160156,
      "activations/layer11_attention_weight_min": -19.818668365478516,
      "activations/layer12_attention_weight_max": 29.77437973022461,
      "activations/layer12_attention_weight_min": -21.117328643798828,
      "activations/layer13_attention_weight_max": 23.56203269958496,
      "activations/layer13_attention_weight_min": -17.684843063354492,
      "activations/layer14_attention_weight_max": 22.982013702392578,
      "activations/layer14_attention_weight_min": -14.7377347946167,
      "activations/layer15_attention_weight_max": 22.676633834838867,
      "activations/layer15_attention_weight_min": -14.06203842163086,
      "activations/layer16_attention_weight_max": 20.624135971069336,
      "activations/layer16_attention_weight_min": -12.005414009094238,
      "activations/layer17_attention_weight_max": 18.361482620239258,
      "activations/layer17_attention_weight_min": -16.18509292602539,
      "activations/layer18_attention_weight_max": 20.609878540039062,
      "activations/layer18_attention_weight_min": -12.721035957336426,
      "activations/layer19_attention_weight_max": 17.837987899780273,
      "activations/layer19_attention_weight_min": -11.381855010986328,
      "activations/layer1_attention_weight_max": 15.834354400634766,
      "activations/layer1_attention_weight_min": -11.719090461730957,
      "activations/layer20_attention_weight_max": 21.97289276123047,
      "activations/layer20_attention_weight_min": -13.174046516418457,
      "activations/layer21_attention_weight_max": 19.463050842285156,
      "activations/layer21_attention_weight_min": -11.967880249023438,
      "activations/layer22_attention_weight_max": 11.694454193115234,
      "activations/layer22_attention_weight_min": -8.623037338256836,
      "activations/layer23_attention_weight_max": 13.792142868041992,
      "activations/layer23_attention_weight_min": -9.143279075622559,
      "activations/layer2_attention_weight_max": 24.1943416595459,
      "activations/layer2_attention_weight_min": -18.30411148071289,
      "activations/layer3_attention_weight_max": 58.256893157958984,
      "activations/layer3_attention_weight_min": -45.64580154418945,
      "activations/layer4_attention_weight_max": 97.5267333984375,
      "activations/layer4_attention_weight_min": -82.9569091796875,
      "activations/layer5_attention_weight_max": 177.14288330078125,
      "activations/layer5_attention_weight_min": -160.54776000976562,
      "activations/layer6_attention_weight_max": 155.99168395996094,
      "activations/layer6_attention_weight_min": -143.73802185058594,
      "activations/layer7_attention_weight_max": 115.75009155273438,
      "activations/layer7_attention_weight_min": -103.4950180053711,
      "activations/layer8_attention_weight_max": 86.63126373291016,
      "activations/layer8_attention_weight_min": -77.48139190673828,
      "activations/layer9_attention_weight_max": 75.79435729980469,
      "activations/layer9_attention_weight_min": -71.58867645263672,
      "epoch": 0.18,
      "learning_rate": 0.00014529075861283353,
      "loss": 3.6706,
      "step": 3100
    },
    {
      "activations/layer0_attention_weight_max": 12.964706420898438,
      "activations/layer0_attention_weight_min": -9.38455867767334,
      "activations/layer10_attention_weight_max": 70.34296417236328,
      "activations/layer10_attention_weight_min": -61.712188720703125,
      "activations/layer11_attention_weight_max": 28.301055908203125,
      "activations/layer11_attention_weight_min": -20.631546020507812,
      "activations/layer12_attention_weight_max": 21.37647819519043,
      "activations/layer12_attention_weight_min": -24.332839965820312,
      "activations/layer13_attention_weight_max": 24.51082420349121,
      "activations/layer13_attention_weight_min": -18.14968490600586,
      "activations/layer14_attention_weight_max": 27.84145164489746,
      "activations/layer14_attention_weight_min": -15.212196350097656,
      "activations/layer15_attention_weight_max": 19.2283935546875,
      "activations/layer15_attention_weight_min": -12.547612190246582,
      "activations/layer16_attention_weight_max": 23.392866134643555,
      "activations/layer16_attention_weight_min": -14.46820068359375,
      "activations/layer17_attention_weight_max": 23.14365005493164,
      "activations/layer17_attention_weight_min": -13.653419494628906,
      "activations/layer18_attention_weight_max": 25.27128791809082,
      "activations/layer18_attention_weight_min": -14.2237548828125,
      "activations/layer19_attention_weight_max": 18.981962203979492,
      "activations/layer19_attention_weight_min": -11.188525199890137,
      "activations/layer1_attention_weight_max": 18.202003479003906,
      "activations/layer1_attention_weight_min": -11.398984909057617,
      "activations/layer20_attention_weight_max": 22.25116539001465,
      "activations/layer20_attention_weight_min": -11.773025512695312,
      "activations/layer21_attention_weight_max": 18.63935089111328,
      "activations/layer21_attention_weight_min": -11.3153657913208,
      "activations/layer22_attention_weight_max": 11.76368236541748,
      "activations/layer22_attention_weight_min": -7.927661418914795,
      "activations/layer23_attention_weight_max": 12.604838371276855,
      "activations/layer23_attention_weight_min": -10.166070938110352,
      "activations/layer2_attention_weight_max": 26.766971588134766,
      "activations/layer2_attention_weight_min": -18.207496643066406,
      "activations/layer3_attention_weight_max": 58.87255859375,
      "activations/layer3_attention_weight_min": -45.839271545410156,
      "activations/layer4_attention_weight_max": 93.76603698730469,
      "activations/layer4_attention_weight_min": -83.82064819335938,
      "activations/layer5_attention_weight_max": 173.7237548828125,
      "activations/layer5_attention_weight_min": -162.70999145507812,
      "activations/layer6_attention_weight_max": 154.21707153320312,
      "activations/layer6_attention_weight_min": -146.03684997558594,
      "activations/layer7_attention_weight_max": 108.1253890991211,
      "activations/layer7_attention_weight_min": -102.99945831298828,
      "activations/layer8_attention_weight_max": 84.6000747680664,
      "activations/layer8_attention_weight_min": -77.1952896118164,
      "activations/layer9_attention_weight_max": 75.8059310913086,
      "activations/layer9_attention_weight_min": -70.6588363647461,
      "epoch": 0.18,
      "learning_rate": 0.00014558171171350442,
      "loss": 3.6641,
      "step": 3150
    },
    {
      "activations/layer0_attention_weight_max": 12.688742637634277,
      "activations/layer0_attention_weight_min": -8.766334533691406,
      "activations/layer10_attention_weight_max": 69.52981567382812,
      "activations/layer10_attention_weight_min": -54.95778274536133,
      "activations/layer11_attention_weight_max": 27.976736068725586,
      "activations/layer11_attention_weight_min": -20.126123428344727,
      "activations/layer12_attention_weight_max": 21.25655746459961,
      "activations/layer12_attention_weight_min": -24.30750274658203,
      "activations/layer13_attention_weight_max": 23.48129653930664,
      "activations/layer13_attention_weight_min": -18.168981552124023,
      "activations/layer14_attention_weight_max": 26.760066986083984,
      "activations/layer14_attention_weight_min": -14.561511993408203,
      "activations/layer15_attention_weight_max": 23.88323402404785,
      "activations/layer15_attention_weight_min": -12.725931167602539,
      "activations/layer16_attention_weight_max": 26.67867088317871,
      "activations/layer16_attention_weight_min": -15.324423789978027,
      "activations/layer17_attention_weight_max": 21.501026153564453,
      "activations/layer17_attention_weight_min": -15.486653327941895,
      "activations/layer18_attention_weight_max": 25.039154052734375,
      "activations/layer18_attention_weight_min": -13.67539119720459,
      "activations/layer19_attention_weight_max": 22.624677658081055,
      "activations/layer19_attention_weight_min": -10.858379364013672,
      "activations/layer1_attention_weight_max": 17.344392776489258,
      "activations/layer1_attention_weight_min": -12.29321575164795,
      "activations/layer20_attention_weight_max": 21.631425857543945,
      "activations/layer20_attention_weight_min": -14.01758861541748,
      "activations/layer21_attention_weight_max": 20.894250869750977,
      "activations/layer21_attention_weight_min": -12.315960884094238,
      "activations/layer22_attention_weight_max": 14.921661376953125,
      "activations/layer22_attention_weight_min": -9.192037582397461,
      "activations/layer23_attention_weight_max": 12.247426986694336,
      "activations/layer23_attention_weight_min": -8.156158447265625,
      "activations/layer2_attention_weight_max": 24.60921287536621,
      "activations/layer2_attention_weight_min": -20.36699676513672,
      "activations/layer3_attention_weight_max": 56.18008804321289,
      "activations/layer3_attention_weight_min": -46.177764892578125,
      "activations/layer4_attention_weight_max": 93.4010238647461,
      "activations/layer4_attention_weight_min": -82.45582580566406,
      "activations/layer5_attention_weight_max": 168.60043334960938,
      "activations/layer5_attention_weight_min": -159.4764404296875,
      "activations/layer6_attention_weight_max": 144.65391540527344,
      "activations/layer6_attention_weight_min": -142.91758728027344,
      "activations/layer7_attention_weight_max": 106.41243743896484,
      "activations/layer7_attention_weight_min": -102.1556625366211,
      "activations/layer8_attention_weight_max": 81.32086181640625,
      "activations/layer8_attention_weight_min": -73.12906646728516,
      "activations/layer9_attention_weight_max": 83.16876983642578,
      "activations/layer9_attention_weight_min": -72.16365814208984,
      "epoch": 0.19,
      "learning_rate": 0.0001458680580162806,
      "loss": 3.6661,
      "step": 3200
    },
    {
      "activations/layer0_attention_weight_max": 13.00948715209961,
      "activations/layer0_attention_weight_min": -8.613485336303711,
      "activations/layer10_attention_weight_max": 63.391624450683594,
      "activations/layer10_attention_weight_min": -61.71295928955078,
      "activations/layer11_attention_weight_max": 25.982946395874023,
      "activations/layer11_attention_weight_min": -22.251724243164062,
      "activations/layer12_attention_weight_max": 21.449783325195312,
      "activations/layer12_attention_weight_min": -21.685684204101562,
      "activations/layer13_attention_weight_max": 22.54620933532715,
      "activations/layer13_attention_weight_min": -21.65558624267578,
      "activations/layer14_attention_weight_max": 23.8875732421875,
      "activations/layer14_attention_weight_min": -16.667570114135742,
      "activations/layer15_attention_weight_max": 17.3114070892334,
      "activations/layer15_attention_weight_min": -13.791913032531738,
      "activations/layer16_attention_weight_max": 20.519731521606445,
      "activations/layer16_attention_weight_min": -12.639768600463867,
      "activations/layer17_attention_weight_max": 18.22989273071289,
      "activations/layer17_attention_weight_min": -16.383220672607422,
      "activations/layer18_attention_weight_max": 21.348222732543945,
      "activations/layer18_attention_weight_min": -15.422844886779785,
      "activations/layer19_attention_weight_max": 18.526975631713867,
      "activations/layer19_attention_weight_min": -11.72780704498291,
      "activations/layer1_attention_weight_max": 17.176715850830078,
      "activations/layer1_attention_weight_min": -11.89574146270752,
      "activations/layer20_attention_weight_max": 22.333629608154297,
      "activations/layer20_attention_weight_min": -12.270881652832031,
      "activations/layer21_attention_weight_max": 20.6202335357666,
      "activations/layer21_attention_weight_min": -13.31850528717041,
      "activations/layer22_attention_weight_max": 12.12446403503418,
      "activations/layer22_attention_weight_min": -9.124625205993652,
      "activations/layer23_attention_weight_max": 11.598114013671875,
      "activations/layer23_attention_weight_min": -9.13873291015625,
      "activations/layer2_attention_weight_max": 24.495033264160156,
      "activations/layer2_attention_weight_min": -18.744203567504883,
      "activations/layer3_attention_weight_max": 57.061378479003906,
      "activations/layer3_attention_weight_min": -44.6072883605957,
      "activations/layer4_attention_weight_max": 91.89102172851562,
      "activations/layer4_attention_weight_min": -82.46417999267578,
      "activations/layer5_attention_weight_max": 171.47906494140625,
      "activations/layer5_attention_weight_min": -161.46380615234375,
      "activations/layer6_attention_weight_max": 138.3293914794922,
      "activations/layer6_attention_weight_min": -135.76849365234375,
      "activations/layer7_attention_weight_max": 100.83670806884766,
      "activations/layer7_attention_weight_min": -97.75814056396484,
      "activations/layer8_attention_weight_max": 76.368408203125,
      "activations/layer8_attention_weight_min": -72.75517272949219,
      "activations/layer9_attention_weight_max": 71.22834777832031,
      "activations/layer9_attention_weight_min": -76.60967254638672,
      "epoch": 0.19,
      "learning_rate": 0.00014614994113344894,
      "loss": 3.6481,
      "step": 3250
    },
    {
      "activations/layer0_attention_weight_max": 12.433969497680664,
      "activations/layer0_attention_weight_min": -8.983194351196289,
      "activations/layer10_attention_weight_max": 59.0853271484375,
      "activations/layer10_attention_weight_min": -52.78938674926758,
      "activations/layer11_attention_weight_max": 25.212066650390625,
      "activations/layer11_attention_weight_min": -20.06647300720215,
      "activations/layer12_attention_weight_max": 25.07191276550293,
      "activations/layer12_attention_weight_min": -25.093820571899414,
      "activations/layer13_attention_weight_max": 23.436574935913086,
      "activations/layer13_attention_weight_min": -18.715450286865234,
      "activations/layer14_attention_weight_max": 22.177486419677734,
      "activations/layer14_attention_weight_min": -16.121723175048828,
      "activations/layer15_attention_weight_max": 20.3968448638916,
      "activations/layer15_attention_weight_min": -12.830049514770508,
      "activations/layer16_attention_weight_max": 22.121906280517578,
      "activations/layer16_attention_weight_min": -11.757079124450684,
      "activations/layer17_attention_weight_max": 20.123432159423828,
      "activations/layer17_attention_weight_min": -15.795377731323242,
      "activations/layer18_attention_weight_max": 19.821094512939453,
      "activations/layer18_attention_weight_min": -13.062405586242676,
      "activations/layer19_attention_weight_max": 16.79311752319336,
      "activations/layer19_attention_weight_min": -12.345189094543457,
      "activations/layer1_attention_weight_max": 18.27001190185547,
      "activations/layer1_attention_weight_min": -10.81749439239502,
      "activations/layer20_attention_weight_max": 17.123178482055664,
      "activations/layer20_attention_weight_min": -14.335740089416504,
      "activations/layer21_attention_weight_max": 19.592567443847656,
      "activations/layer21_attention_weight_min": -11.493045806884766,
      "activations/layer22_attention_weight_max": 14.33950138092041,
      "activations/layer22_attention_weight_min": -9.531825065612793,
      "activations/layer23_attention_weight_max": 12.667383193969727,
      "activations/layer23_attention_weight_min": -8.179804801940918,
      "activations/layer2_attention_weight_max": 26.24354362487793,
      "activations/layer2_attention_weight_min": -19.079612731933594,
      "activations/layer3_attention_weight_max": 57.179115295410156,
      "activations/layer3_attention_weight_min": -45.496734619140625,
      "activations/layer4_attention_weight_max": 92.69450378417969,
      "activations/layer4_attention_weight_min": -83.23363494873047,
      "activations/layer5_attention_weight_max": 171.5074920654297,
      "activations/layer5_attention_weight_min": -162.54055786132812,
      "activations/layer6_attention_weight_max": 138.3045654296875,
      "activations/layer6_attention_weight_min": -139.1259765625,
      "activations/layer7_attention_weight_max": 104.44131469726562,
      "activations/layer7_attention_weight_min": -102.42993927001953,
      "activations/layer8_attention_weight_max": 80.12047576904297,
      "activations/layer8_attention_weight_min": -77.4258804321289,
      "activations/layer9_attention_weight_max": 74.02906036376953,
      "activations/layer9_attention_weight_min": -70.67872619628906,
      "epoch": 0.19,
      "learning_rate": 0.0001464274980647812,
      "loss": 3.6426,
      "step": 3300
    },
    {
      "activations/layer0_attention_weight_max": 13.029569625854492,
      "activations/layer0_attention_weight_min": -8.341526985168457,
      "activations/layer10_attention_weight_max": 65.4102554321289,
      "activations/layer10_attention_weight_min": -56.96476745605469,
      "activations/layer11_attention_weight_max": 25.33945083618164,
      "activations/layer11_attention_weight_min": -22.22051239013672,
      "activations/layer12_attention_weight_max": 22.335935592651367,
      "activations/layer12_attention_weight_min": -23.81468391418457,
      "activations/layer13_attention_weight_max": 26.337501525878906,
      "activations/layer13_attention_weight_min": -19.773681640625,
      "activations/layer14_attention_weight_max": 22.27042579650879,
      "activations/layer14_attention_weight_min": -14.837345123291016,
      "activations/layer15_attention_weight_max": 20.953813552856445,
      "activations/layer15_attention_weight_min": -12.067950248718262,
      "activations/layer16_attention_weight_max": 23.069190979003906,
      "activations/layer16_attention_weight_min": -11.164069175720215,
      "activations/layer17_attention_weight_max": 17.608749389648438,
      "activations/layer17_attention_weight_min": -13.27884292602539,
      "activations/layer18_attention_weight_max": 26.38957405090332,
      "activations/layer18_attention_weight_min": -11.938847541809082,
      "activations/layer19_attention_weight_max": 20.147762298583984,
      "activations/layer19_attention_weight_min": -11.88595199584961,
      "activations/layer1_attention_weight_max": 18.39487648010254,
      "activations/layer1_attention_weight_min": -11.916264533996582,
      "activations/layer20_attention_weight_max": 20.44951629638672,
      "activations/layer20_attention_weight_min": -11.665282249450684,
      "activations/layer21_attention_weight_max": 20.983863830566406,
      "activations/layer21_attention_weight_min": -10.644476890563965,
      "activations/layer22_attention_weight_max": 14.774480819702148,
      "activations/layer22_attention_weight_min": -10.252100944519043,
      "activations/layer23_attention_weight_max": 12.44486141204834,
      "activations/layer23_attention_weight_min": -8.26148796081543,
      "activations/layer2_attention_weight_max": 24.443084716796875,
      "activations/layer2_attention_weight_min": -19.49900245666504,
      "activations/layer3_attention_weight_max": 56.09973907470703,
      "activations/layer3_attention_weight_min": -46.40562438964844,
      "activations/layer4_attention_weight_max": 91.03164672851562,
      "activations/layer4_attention_weight_min": -83.00868225097656,
      "activations/layer5_attention_weight_max": 163.6480255126953,
      "activations/layer5_attention_weight_min": -168.71011352539062,
      "activations/layer6_attention_weight_max": 141.90298461914062,
      "activations/layer6_attention_weight_min": -140.84608459472656,
      "activations/layer7_attention_weight_max": 105.10011291503906,
      "activations/layer7_attention_weight_min": -98.92533874511719,
      "activations/layer8_attention_weight_max": 79.78148651123047,
      "activations/layer8_attention_weight_min": -71.05484008789062,
      "activations/layer9_attention_weight_max": 72.90032196044922,
      "activations/layer9_attention_weight_min": -65.77257537841797,
      "epoch": 0.19,
      "learning_rate": 0.0001467008595973623,
      "loss": 3.6269,
      "step": 3350
    },
    {
      "activations/layer0_attention_weight_max": 11.646114349365234,
      "activations/layer0_attention_weight_min": -8.788835525512695,
      "activations/layer10_attention_weight_max": 71.11235809326172,
      "activations/layer10_attention_weight_min": -58.776145935058594,
      "activations/layer11_attention_weight_max": 28.379262924194336,
      "activations/layer11_attention_weight_min": -21.179655075073242,
      "activations/layer12_attention_weight_max": 21.634204864501953,
      "activations/layer12_attention_weight_min": -19.908863067626953,
      "activations/layer13_attention_weight_max": 22.511592864990234,
      "activations/layer13_attention_weight_min": -18.084732055664062,
      "activations/layer14_attention_weight_max": 24.191617965698242,
      "activations/layer14_attention_weight_min": -15.413476943969727,
      "activations/layer15_attention_weight_max": 19.350555419921875,
      "activations/layer15_attention_weight_min": -12.163637161254883,
      "activations/layer16_attention_weight_max": 24.556594848632812,
      "activations/layer16_attention_weight_min": -12.174774169921875,
      "activations/layer17_attention_weight_max": 20.510299682617188,
      "activations/layer17_attention_weight_min": -17.04169273376465,
      "activations/layer18_attention_weight_max": 25.97935676574707,
      "activations/layer18_attention_weight_min": -14.429740905761719,
      "activations/layer19_attention_weight_max": 20.951292037963867,
      "activations/layer19_attention_weight_min": -12.00325870513916,
      "activations/layer1_attention_weight_max": 17.22892951965332,
      "activations/layer1_attention_weight_min": -11.251407623291016,
      "activations/layer20_attention_weight_max": 19.68705940246582,
      "activations/layer20_attention_weight_min": -13.273555755615234,
      "activations/layer21_attention_weight_max": 22.548416137695312,
      "activations/layer21_attention_weight_min": -13.195053100585938,
      "activations/layer22_attention_weight_max": 13.399979591369629,
      "activations/layer22_attention_weight_min": -9.964110374450684,
      "activations/layer23_attention_weight_max": 12.566802024841309,
      "activations/layer23_attention_weight_min": -7.897932052612305,
      "activations/layer2_attention_weight_max": 25.880054473876953,
      "activations/layer2_attention_weight_min": -19.158611297607422,
      "activations/layer3_attention_weight_max": 61.062042236328125,
      "activations/layer3_attention_weight_min": -42.99093246459961,
      "activations/layer4_attention_weight_max": 93.74457550048828,
      "activations/layer4_attention_weight_min": -80.36849975585938,
      "activations/layer5_attention_weight_max": 172.19940185546875,
      "activations/layer5_attention_weight_min": -152.14651489257812,
      "activations/layer6_attention_weight_max": 149.40823364257812,
      "activations/layer6_attention_weight_min": -136.9667205810547,
      "activations/layer7_attention_weight_max": 118.25674438476562,
      "activations/layer7_attention_weight_min": -98.23975372314453,
      "activations/layer8_attention_weight_max": 82.06027221679688,
      "activations/layer8_attention_weight_min": -74.86553192138672,
      "activations/layer9_attention_weight_max": 79.8154525756836,
      "activations/layer9_attention_weight_min": -67.02128601074219,
      "epoch": 0.2,
      "learning_rate": 0.00014697015067564886,
      "loss": 3.6565,
      "step": 3400
    },
    {
      "activations/layer0_attention_weight_max": 12.226365089416504,
      "activations/layer0_attention_weight_min": -8.354793548583984,
      "activations/layer10_attention_weight_max": 68.21882629394531,
      "activations/layer10_attention_weight_min": -53.38620376586914,
      "activations/layer11_attention_weight_max": 29.549161911010742,
      "activations/layer11_attention_weight_min": -23.13747787475586,
      "activations/layer12_attention_weight_max": 22.064346313476562,
      "activations/layer12_attention_weight_min": -23.663421630859375,
      "activations/layer13_attention_weight_max": 22.572566986083984,
      "activations/layer13_attention_weight_min": -19.65735626220703,
      "activations/layer14_attention_weight_max": 19.682390213012695,
      "activations/layer14_attention_weight_min": -16.395429611206055,
      "activations/layer15_attention_weight_max": 18.490196228027344,
      "activations/layer15_attention_weight_min": -13.892056465148926,
      "activations/layer16_attention_weight_max": 21.727153778076172,
      "activations/layer16_attention_weight_min": -12.817939758300781,
      "activations/layer17_attention_weight_max": 18.7329158782959,
      "activations/layer17_attention_weight_min": -16.43723487854004,
      "activations/layer18_attention_weight_max": 25.586814880371094,
      "activations/layer18_attention_weight_min": -14.587540626525879,
      "activations/layer19_attention_weight_max": 19.457015991210938,
      "activations/layer19_attention_weight_min": -13.07360553741455,
      "activations/layer1_attention_weight_max": 17.5165958404541,
      "activations/layer1_attention_weight_min": -10.779875755310059,
      "activations/layer20_attention_weight_max": 20.43161392211914,
      "activations/layer20_attention_weight_min": -13.74959659576416,
      "activations/layer21_attention_weight_max": 16.19227409362793,
      "activations/layer21_attention_weight_min": -12.877287864685059,
      "activations/layer22_attention_weight_max": 14.738032341003418,
      "activations/layer22_attention_weight_min": -10.15013313293457,
      "activations/layer23_attention_weight_max": 13.375022888183594,
      "activations/layer23_attention_weight_min": -8.705967903137207,
      "activations/layer2_attention_weight_max": 25.177841186523438,
      "activations/layer2_attention_weight_min": -19.39594268798828,
      "activations/layer3_attention_weight_max": 57.941566467285156,
      "activations/layer3_attention_weight_min": -47.30384063720703,
      "activations/layer4_attention_weight_max": 95.5696792602539,
      "activations/layer4_attention_weight_min": -85.45952606201172,
      "activations/layer5_attention_weight_max": 175.70907592773438,
      "activations/layer5_attention_weight_min": -161.66815185546875,
      "activations/layer6_attention_weight_max": 145.28256225585938,
      "activations/layer6_attention_weight_min": -136.46363830566406,
      "activations/layer7_attention_weight_max": 111.60962677001953,
      "activations/layer7_attention_weight_min": -102.19143676757812,
      "activations/layer8_attention_weight_max": 82.22350311279297,
      "activations/layer8_attention_weight_min": -72.10462188720703,
      "activations/layer9_attention_weight_max": 75.51030731201172,
      "activations/layer9_attention_weight_min": -71.98580169677734,
      "epoch": 0.2,
      "learning_rate": 0.00014723549074437798,
      "loss": 3.6113,
      "step": 3450
    },
    {
      "activations/layer0_attention_weight_max": 12.821451187133789,
      "activations/layer0_attention_weight_min": -9.74906063079834,
      "activations/layer10_attention_weight_max": 67.05860900878906,
      "activations/layer10_attention_weight_min": -60.48072052001953,
      "activations/layer11_attention_weight_max": 30.59933090209961,
      "activations/layer11_attention_weight_min": -23.047698974609375,
      "activations/layer12_attention_weight_max": 22.514230728149414,
      "activations/layer12_attention_weight_min": -22.154993057250977,
      "activations/layer13_attention_weight_max": 23.377765655517578,
      "activations/layer13_attention_weight_min": -18.591711044311523,
      "activations/layer14_attention_weight_max": 20.997962951660156,
      "activations/layer14_attention_weight_min": -15.408513069152832,
      "activations/layer15_attention_weight_max": 19.980987548828125,
      "activations/layer15_attention_weight_min": -12.892483711242676,
      "activations/layer16_attention_weight_max": 22.90062713623047,
      "activations/layer16_attention_weight_min": -13.045561790466309,
      "activations/layer17_attention_weight_max": 21.1669864654541,
      "activations/layer17_attention_weight_min": -15.25881290435791,
      "activations/layer18_attention_weight_max": 27.945207595825195,
      "activations/layer18_attention_weight_min": -14.994508743286133,
      "activations/layer19_attention_weight_max": 16.452505111694336,
      "activations/layer19_attention_weight_min": -11.681110382080078,
      "activations/layer1_attention_weight_max": 16.989543914794922,
      "activations/layer1_attention_weight_min": -11.540115356445312,
      "activations/layer20_attention_weight_max": 20.661745071411133,
      "activations/layer20_attention_weight_min": -13.301100730895996,
      "activations/layer21_attention_weight_max": 17.761539459228516,
      "activations/layer21_attention_weight_min": -11.720174789428711,
      "activations/layer22_attention_weight_max": 13.585396766662598,
      "activations/layer22_attention_weight_min": -10.372526168823242,
      "activations/layer23_attention_weight_max": 14.291559219360352,
      "activations/layer23_attention_weight_min": -8.203351974487305,
      "activations/layer2_attention_weight_max": 26.335948944091797,
      "activations/layer2_attention_weight_min": -19.692317962646484,
      "activations/layer3_attention_weight_max": 59.761871337890625,
      "activations/layer3_attention_weight_min": -46.123443603515625,
      "activations/layer4_attention_weight_max": 94.59075164794922,
      "activations/layer4_attention_weight_min": -82.41381072998047,
      "activations/layer5_attention_weight_max": 173.6212615966797,
      "activations/layer5_attention_weight_min": -163.3558349609375,
      "activations/layer6_attention_weight_max": 138.09593200683594,
      "activations/layer6_attention_weight_min": -135.13699340820312,
      "activations/layer7_attention_weight_max": 100.97097778320312,
      "activations/layer7_attention_weight_min": -97.10211181640625,
      "activations/layer8_attention_weight_max": 77.61637878417969,
      "activations/layer8_attention_weight_min": -72.30332946777344,
      "activations/layer9_attention_weight_max": 73.5724868774414,
      "activations/layer9_attention_weight_min": -70.73880767822266,
      "epoch": 0.2,
      "learning_rate": 0.00014749699406668265,
      "loss": 3.6105,
      "step": 3500
    },
    {
      "activations/layer0_attention_weight_max": 12.016363143920898,
      "activations/layer0_attention_weight_min": -8.70706844329834,
      "activations/layer10_attention_weight_max": 65.81561279296875,
      "activations/layer10_attention_weight_min": -63.1967887878418,
      "activations/layer11_attention_weight_max": 29.85427474975586,
      "activations/layer11_attention_weight_min": -22.763038635253906,
      "activations/layer12_attention_weight_max": 23.95034408569336,
      "activations/layer12_attention_weight_min": -25.35875701904297,
      "activations/layer13_attention_weight_max": 23.87586784362793,
      "activations/layer13_attention_weight_min": -21.125587463378906,
      "activations/layer14_attention_weight_max": 24.15656852722168,
      "activations/layer14_attention_weight_min": -18.341846466064453,
      "activations/layer15_attention_weight_max": 19.3614559173584,
      "activations/layer15_attention_weight_min": -12.516695976257324,
      "activations/layer16_attention_weight_max": 22.540578842163086,
      "activations/layer16_attention_weight_min": -14.590599060058594,
      "activations/layer17_attention_weight_max": 18.52037239074707,
      "activations/layer17_attention_weight_min": -17.35917091369629,
      "activations/layer18_attention_weight_max": 27.97010612487793,
      "activations/layer18_attention_weight_min": -14.701303482055664,
      "activations/layer19_attention_weight_max": 16.848295211791992,
      "activations/layer19_attention_weight_min": -12.201519966125488,
      "activations/layer1_attention_weight_max": 17.565603256225586,
      "activations/layer1_attention_weight_min": -12.024829864501953,
      "activations/layer20_attention_weight_max": 17.997543334960938,
      "activations/layer20_attention_weight_min": -12.062396049499512,
      "activations/layer21_attention_weight_max": 19.660480499267578,
      "activations/layer21_attention_weight_min": -13.948143005371094,
      "activations/layer22_attention_weight_max": 17.32061004638672,
      "activations/layer22_attention_weight_min": -12.719803810119629,
      "activations/layer23_attention_weight_max": 11.373358726501465,
      "activations/layer23_attention_weight_min": -8.795616149902344,
      "activations/layer2_attention_weight_max": 25.156940460205078,
      "activations/layer2_attention_weight_min": -19.75653839111328,
      "activations/layer3_attention_weight_max": 58.22166061401367,
      "activations/layer3_attention_weight_min": -47.96907424926758,
      "activations/layer4_attention_weight_max": 91.63116455078125,
      "activations/layer4_attention_weight_min": -85.46381378173828,
      "activations/layer5_attention_weight_max": 170.7552490234375,
      "activations/layer5_attention_weight_min": -156.78875732421875,
      "activations/layer6_attention_weight_max": 144.30697631835938,
      "activations/layer6_attention_weight_min": -135.15432739257812,
      "activations/layer7_attention_weight_max": 102.9531021118164,
      "activations/layer7_attention_weight_min": -97.6333999633789,
      "activations/layer8_attention_weight_max": 74.400146484375,
      "activations/layer8_attention_weight_min": -74.05536651611328,
      "activations/layer9_attention_weight_max": 79.60652923583984,
      "activations/layer9_attention_weight_min": -79.47488403320312,
      "epoch": 0.21,
      "learning_rate": 0.00014775477001953373,
      "loss": 3.6049,
      "step": 3550
    },
    {
      "activations/layer0_attention_weight_max": 12.749702453613281,
      "activations/layer0_attention_weight_min": -9.313379287719727,
      "activations/layer10_attention_weight_max": 66.4647445678711,
      "activations/layer10_attention_weight_min": -61.684688568115234,
      "activations/layer11_attention_weight_max": 31.792295455932617,
      "activations/layer11_attention_weight_min": -22.470623016357422,
      "activations/layer12_attention_weight_max": 20.988916397094727,
      "activations/layer12_attention_weight_min": -24.994834899902344,
      "activations/layer13_attention_weight_max": 29.78264808654785,
      "activations/layer13_attention_weight_min": -21.75547218322754,
      "activations/layer14_attention_weight_max": 25.99385643005371,
      "activations/layer14_attention_weight_min": -16.417036056518555,
      "activations/layer15_attention_weight_max": 23.964004516601562,
      "activations/layer15_attention_weight_min": -15.106743812561035,
      "activations/layer16_attention_weight_max": 23.949846267700195,
      "activations/layer16_attention_weight_min": -12.634610176086426,
      "activations/layer17_attention_weight_max": 20.604658126831055,
      "activations/layer17_attention_weight_min": -16.51407814025879,
      "activations/layer18_attention_weight_max": 22.250476837158203,
      "activations/layer18_attention_weight_min": -15.700871467590332,
      "activations/layer19_attention_weight_max": 23.60329818725586,
      "activations/layer19_attention_weight_min": -10.814425468444824,
      "activations/layer1_attention_weight_max": 17.525096893310547,
      "activations/layer1_attention_weight_min": -11.964886665344238,
      "activations/layer20_attention_weight_max": 26.832944869995117,
      "activations/layer20_attention_weight_min": -12.654793739318848,
      "activations/layer21_attention_weight_max": 23.178176879882812,
      "activations/layer21_attention_weight_min": -12.656050682067871,
      "activations/layer22_attention_weight_max": 16.049489974975586,
      "activations/layer22_attention_weight_min": -10.453835487365723,
      "activations/layer23_attention_weight_max": 13.850130081176758,
      "activations/layer23_attention_weight_min": -10.188608169555664,
      "activations/layer2_attention_weight_max": 26.035306930541992,
      "activations/layer2_attention_weight_min": -20.129228591918945,
      "activations/layer3_attention_weight_max": 59.18916320800781,
      "activations/layer3_attention_weight_min": -46.15165710449219,
      "activations/layer4_attention_weight_max": 92.39965057373047,
      "activations/layer4_attention_weight_min": -82.9071044921875,
      "activations/layer5_attention_weight_max": 173.3681640625,
      "activations/layer5_attention_weight_min": -160.48423767089844,
      "activations/layer6_attention_weight_max": 145.59498596191406,
      "activations/layer6_attention_weight_min": -136.0997314453125,
      "activations/layer7_attention_weight_max": 107.34859466552734,
      "activations/layer7_attention_weight_min": -97.72428131103516,
      "activations/layer8_attention_weight_max": 79.87459564208984,
      "activations/layer8_attention_weight_min": -74.56422424316406,
      "activations/layer9_attention_weight_max": 74.718505859375,
      "activations/layer9_attention_weight_min": -70.0764389038086,
      "epoch": 0.21,
      "learning_rate": 0.00014800892336842126,
      "loss": 3.613,
      "step": 3600
    },
    {
      "activations/layer0_attention_weight_max": 12.815118789672852,
      "activations/layer0_attention_weight_min": -9.360034942626953,
      "activations/layer10_attention_weight_max": 65.62369537353516,
      "activations/layer10_attention_weight_min": -59.65790939331055,
      "activations/layer11_attention_weight_max": 26.474790573120117,
      "activations/layer11_attention_weight_min": -22.02927017211914,
      "activations/layer12_attention_weight_max": 20.971418380737305,
      "activations/layer12_attention_weight_min": -22.442785263061523,
      "activations/layer13_attention_weight_max": 22.40126609802246,
      "activations/layer13_attention_weight_min": -20.214130401611328,
      "activations/layer14_attention_weight_max": 23.211750030517578,
      "activations/layer14_attention_weight_min": -15.87343978881836,
      "activations/layer15_attention_weight_max": 17.960248947143555,
      "activations/layer15_attention_weight_min": -14.60689926147461,
      "activations/layer16_attention_weight_max": 21.85056495666504,
      "activations/layer16_attention_weight_min": -12.822554588317871,
      "activations/layer17_attention_weight_max": 18.99471092224121,
      "activations/layer17_attention_weight_min": -17.05617332458496,
      "activations/layer18_attention_weight_max": 23.788305282592773,
      "activations/layer18_attention_weight_min": -16.471044540405273,
      "activations/layer19_attention_weight_max": 20.94329261779785,
      "activations/layer19_attention_weight_min": -12.686677932739258,
      "activations/layer1_attention_weight_max": 17.633785247802734,
      "activations/layer1_attention_weight_min": -10.865104675292969,
      "activations/layer20_attention_weight_max": 22.441190719604492,
      "activations/layer20_attention_weight_min": -12.51696491241455,
      "activations/layer21_attention_weight_max": 19.410690307617188,
      "activations/layer21_attention_weight_min": -13.9468994140625,
      "activations/layer22_attention_weight_max": 15.477520942687988,
      "activations/layer22_attention_weight_min": -11.876492500305176,
      "activations/layer23_attention_weight_max": 13.054976463317871,
      "activations/layer23_attention_weight_min": -11.350417137145996,
      "activations/layer2_attention_weight_max": 24.506284713745117,
      "activations/layer2_attention_weight_min": -19.202110290527344,
      "activations/layer3_attention_weight_max": 58.56791687011719,
      "activations/layer3_attention_weight_min": -46.16126251220703,
      "activations/layer4_attention_weight_max": 92.96566009521484,
      "activations/layer4_attention_weight_min": -84.89976501464844,
      "activations/layer5_attention_weight_max": 172.68569946289062,
      "activations/layer5_attention_weight_min": -161.58705139160156,
      "activations/layer6_attention_weight_max": 148.06956481933594,
      "activations/layer6_attention_weight_min": -136.89047241210938,
      "activations/layer7_attention_weight_max": 112.50328063964844,
      "activations/layer7_attention_weight_min": -103.37425994873047,
      "activations/layer8_attention_weight_max": 85.48587799072266,
      "activations/layer8_attention_weight_min": -73.26860046386719,
      "activations/layer9_attention_weight_max": 75.72631072998047,
      "activations/layer9_attention_weight_min": -74.02376556396484,
      "epoch": 0.21,
      "learning_rate": 0.00014825955452300063,
      "loss": 3.5937,
      "step": 3650
    },
    {
      "activations/layer0_attention_weight_max": 13.53445053100586,
      "activations/layer0_attention_weight_min": -8.351463317871094,
      "activations/layer10_attention_weight_max": 61.352230072021484,
      "activations/layer10_attention_weight_min": -55.16697311401367,
      "activations/layer11_attention_weight_max": 29.604209899902344,
      "activations/layer11_attention_weight_min": -22.992874145507812,
      "activations/layer12_attention_weight_max": 21.65213966369629,
      "activations/layer12_attention_weight_min": -23.583988189697266,
      "activations/layer13_attention_weight_max": 23.532686233520508,
      "activations/layer13_attention_weight_min": -20.321632385253906,
      "activations/layer14_attention_weight_max": 22.34062385559082,
      "activations/layer14_attention_weight_min": -15.071659088134766,
      "activations/layer15_attention_weight_max": 22.26055145263672,
      "activations/layer15_attention_weight_min": -12.872932434082031,
      "activations/layer16_attention_weight_max": 23.401416778564453,
      "activations/layer16_attention_weight_min": -12.744213104248047,
      "activations/layer17_attention_weight_max": 19.57697105407715,
      "activations/layer17_attention_weight_min": -16.863344192504883,
      "activations/layer18_attention_weight_max": 24.901443481445312,
      "activations/layer18_attention_weight_min": -15.01204776763916,
      "activations/layer19_attention_weight_max": 21.16954231262207,
      "activations/layer19_attention_weight_min": -11.945131301879883,
      "activations/layer1_attention_weight_max": 17.148096084594727,
      "activations/layer1_attention_weight_min": -9.880916595458984,
      "activations/layer20_attention_weight_max": 20.564653396606445,
      "activations/layer20_attention_weight_min": -13.042119026184082,
      "activations/layer21_attention_weight_max": 22.128747940063477,
      "activations/layer21_attention_weight_min": -11.276705741882324,
      "activations/layer22_attention_weight_max": 14.1011381149292,
      "activations/layer22_attention_weight_min": -12.443440437316895,
      "activations/layer23_attention_weight_max": 12.157382011413574,
      "activations/layer23_attention_weight_min": -10.339015007019043,
      "activations/layer2_attention_weight_max": 25.5933895111084,
      "activations/layer2_attention_weight_min": -20.161956787109375,
      "activations/layer3_attention_weight_max": 56.41206741333008,
      "activations/layer3_attention_weight_min": -45.54859161376953,
      "activations/layer4_attention_weight_max": 92.59319305419922,
      "activations/layer4_attention_weight_min": -82.66387939453125,
      "activations/layer5_attention_weight_max": 167.6688995361328,
      "activations/layer5_attention_weight_min": -157.44479370117188,
      "activations/layer6_attention_weight_max": 142.46388244628906,
      "activations/layer6_attention_weight_min": -129.97842407226562,
      "activations/layer7_attention_weight_max": 106.19915008544922,
      "activations/layer7_attention_weight_min": -102.12135314941406,
      "activations/layer8_attention_weight_max": 74.80992126464844,
      "activations/layer8_attention_weight_min": -73.67949676513672,
      "activations/layer9_attention_weight_max": 73.49083709716797,
      "activations/layer9_attention_weight_min": -69.82615661621094,
      "epoch": 0.21,
      "learning_rate": 0.0001485067597752656,
      "loss": 3.6009,
      "step": 3700
    },
    {
      "activations/layer0_attention_weight_max": 13.445295333862305,
      "activations/layer0_attention_weight_min": -8.635425567626953,
      "activations/layer10_attention_weight_max": 60.009578704833984,
      "activations/layer10_attention_weight_min": -52.627227783203125,
      "activations/layer11_attention_weight_max": 28.965511322021484,
      "activations/layer11_attention_weight_min": -22.804052352905273,
      "activations/layer12_attention_weight_max": 23.139081954956055,
      "activations/layer12_attention_weight_min": -25.790430068969727,
      "activations/layer13_attention_weight_max": 29.569332122802734,
      "activations/layer13_attention_weight_min": -20.530128479003906,
      "activations/layer14_attention_weight_max": 25.529815673828125,
      "activations/layer14_attention_weight_min": -17.14649772644043,
      "activations/layer15_attention_weight_max": 21.261308670043945,
      "activations/layer15_attention_weight_min": -13.506779670715332,
      "activations/layer16_attention_weight_max": 25.823007583618164,
      "activations/layer16_attention_weight_min": -14.863052368164062,
      "activations/layer17_attention_weight_max": 21.066017150878906,
      "activations/layer17_attention_weight_min": -16.75417709350586,
      "activations/layer18_attention_weight_max": 28.264902114868164,
      "activations/layer18_attention_weight_min": -16.75814437866211,
      "activations/layer19_attention_weight_max": 24.449384689331055,
      "activations/layer19_attention_weight_min": -13.210339546203613,
      "activations/layer1_attention_weight_max": 18.06612777709961,
      "activations/layer1_attention_weight_min": -12.115008354187012,
      "activations/layer20_attention_weight_max": 26.221599578857422,
      "activations/layer20_attention_weight_min": -16.42966651916504,
      "activations/layer21_attention_weight_max": 21.554479598999023,
      "activations/layer21_attention_weight_min": -14.48130989074707,
      "activations/layer22_attention_weight_max": 15.89617919921875,
      "activations/layer22_attention_weight_min": -14.42270278930664,
      "activations/layer23_attention_weight_max": 14.368742942810059,
      "activations/layer23_attention_weight_min": -11.581416130065918,
      "activations/layer2_attention_weight_max": 26.24562644958496,
      "activations/layer2_attention_weight_min": -21.567304611206055,
      "activations/layer3_attention_weight_max": 59.243858337402344,
      "activations/layer3_attention_weight_min": -45.955718994140625,
      "activations/layer4_attention_weight_max": 93.2517318725586,
      "activations/layer4_attention_weight_min": -81.08890533447266,
      "activations/layer5_attention_weight_max": 169.31573486328125,
      "activations/layer5_attention_weight_min": -153.8240203857422,
      "activations/layer6_attention_weight_max": 152.39471435546875,
      "activations/layer6_attention_weight_min": -133.8427734375,
      "activations/layer7_attention_weight_max": 100.024169921875,
      "activations/layer7_attention_weight_min": -95.29004669189453,
      "activations/layer8_attention_weight_max": 75.7868423461914,
      "activations/layer8_attention_weight_min": -72.01818084716797,
      "activations/layer9_attention_weight_max": 71.71959686279297,
      "activations/layer9_attention_weight_min": -69.15568542480469,
      "epoch": 0.22,
      "learning_rate": 0.0001487506315216617,
      "loss": 3.597,
      "step": 3750
    },
    {
      "activations/layer0_attention_weight_max": 12.621782302856445,
      "activations/layer0_attention_weight_min": -9.032317161560059,
      "activations/layer10_attention_weight_max": 63.97673797607422,
      "activations/layer10_attention_weight_min": -56.024959564208984,
      "activations/layer11_attention_weight_max": 30.864723205566406,
      "activations/layer11_attention_weight_min": -21.193771362304688,
      "activations/layer12_attention_weight_max": 22.23023223876953,
      "activations/layer12_attention_weight_min": -20.017871856689453,
      "activations/layer13_attention_weight_max": 23.843875885009766,
      "activations/layer13_attention_weight_min": -21.696666717529297,
      "activations/layer14_attention_weight_max": 20.626508712768555,
      "activations/layer14_attention_weight_min": -15.690596580505371,
      "activations/layer15_attention_weight_max": 19.41185760498047,
      "activations/layer15_attention_weight_min": -14.067215919494629,
      "activations/layer16_attention_weight_max": 20.15386390686035,
      "activations/layer16_attention_weight_min": -11.557827949523926,
      "activations/layer17_attention_weight_max": 20.072277069091797,
      "activations/layer17_attention_weight_min": -16.638347625732422,
      "activations/layer18_attention_weight_max": 25.063596725463867,
      "activations/layer18_attention_weight_min": -13.452616691589355,
      "activations/layer19_attention_weight_max": 21.181983947753906,
      "activations/layer19_attention_weight_min": -10.444820404052734,
      "activations/layer1_attention_weight_max": 17.81648826599121,
      "activations/layer1_attention_weight_min": -11.748639106750488,
      "activations/layer20_attention_weight_max": 21.72018051147461,
      "activations/layer20_attention_weight_min": -11.510690689086914,
      "activations/layer21_attention_weight_max": 19.73949432373047,
      "activations/layer21_attention_weight_min": -12.110588073730469,
      "activations/layer22_attention_weight_max": 15.647417068481445,
      "activations/layer22_attention_weight_min": -10.44991683959961,
      "activations/layer23_attention_weight_max": 12.754667282104492,
      "activations/layer23_attention_weight_min": -9.59349536895752,
      "activations/layer2_attention_weight_max": 26.577781677246094,
      "activations/layer2_attention_weight_min": -20.24227523803711,
      "activations/layer3_attention_weight_max": 56.42533874511719,
      "activations/layer3_attention_weight_min": -45.67202377319336,
      "activations/layer4_attention_weight_max": 90.28780364990234,
      "activations/layer4_attention_weight_min": -81.85435485839844,
      "activations/layer5_attention_weight_max": 166.60765075683594,
      "activations/layer5_attention_weight_min": -158.72406005859375,
      "activations/layer6_attention_weight_max": 136.5283966064453,
      "activations/layer6_attention_weight_min": -131.43638610839844,
      "activations/layer7_attention_weight_max": 102.56668853759766,
      "activations/layer7_attention_weight_min": -100.52637481689453,
      "activations/layer8_attention_weight_max": 77.93101501464844,
      "activations/layer8_attention_weight_min": -73.4786605834961,
      "activations/layer9_attention_weight_max": 74.42549133300781,
      "activations/layer9_attention_weight_min": -70.89098358154297,
      "epoch": 0.22,
      "learning_rate": 0.00014899125847042232,
      "loss": 3.5969,
      "step": 3800
    },
    {
      "activations/layer0_attention_weight_max": 13.504745483398438,
      "activations/layer0_attention_weight_min": -8.808347702026367,
      "activations/layer10_attention_weight_max": 56.78990173339844,
      "activations/layer10_attention_weight_min": -55.37455749511719,
      "activations/layer11_attention_weight_max": 30.352087020874023,
      "activations/layer11_attention_weight_min": -21.788854598999023,
      "activations/layer12_attention_weight_max": 20.835535049438477,
      "activations/layer12_attention_weight_min": -22.4819278717041,
      "activations/layer13_attention_weight_max": 24.73810577392578,
      "activations/layer13_attention_weight_min": -22.333436965942383,
      "activations/layer14_attention_weight_max": 22.44388198852539,
      "activations/layer14_attention_weight_min": -17.092514038085938,
      "activations/layer15_attention_weight_max": 19.855493545532227,
      "activations/layer15_attention_weight_min": -13.396454811096191,
      "activations/layer16_attention_weight_max": 19.36029815673828,
      "activations/layer16_attention_weight_min": -12.375275611877441,
      "activations/layer17_attention_weight_max": 21.387557983398438,
      "activations/layer17_attention_weight_min": -18.423715591430664,
      "activations/layer18_attention_weight_max": 21.887155532836914,
      "activations/layer18_attention_weight_min": -14.118636131286621,
      "activations/layer19_attention_weight_max": 19.707700729370117,
      "activations/layer19_attention_weight_min": -12.491576194763184,
      "activations/layer1_attention_weight_max": 17.97222900390625,
      "activations/layer1_attention_weight_min": -11.520140647888184,
      "activations/layer20_attention_weight_max": 19.975017547607422,
      "activations/layer20_attention_weight_min": -12.160086631774902,
      "activations/layer21_attention_weight_max": 20.294878005981445,
      "activations/layer21_attention_weight_min": -12.810622215270996,
      "activations/layer22_attention_weight_max": 14.69676685333252,
      "activations/layer22_attention_weight_min": -11.097187042236328,
      "activations/layer23_attention_weight_max": 13.098222732543945,
      "activations/layer23_attention_weight_min": -11.273590087890625,
      "activations/layer2_attention_weight_max": 25.614334106445312,
      "activations/layer2_attention_weight_min": -20.493900299072266,
      "activations/layer3_attention_weight_max": 54.224853515625,
      "activations/layer3_attention_weight_min": -44.481441497802734,
      "activations/layer4_attention_weight_max": 87.35813903808594,
      "activations/layer4_attention_weight_min": -80.30093383789062,
      "activations/layer5_attention_weight_max": 162.34393310546875,
      "activations/layer5_attention_weight_min": -163.54998779296875,
      "activations/layer6_attention_weight_max": 127.47760009765625,
      "activations/layer6_attention_weight_min": -129.3253173828125,
      "activations/layer7_attention_weight_max": 96.2970199584961,
      "activations/layer7_attention_weight_min": -95.98194885253906,
      "activations/layer8_attention_weight_max": 76.75691986083984,
      "activations/layer8_attention_weight_min": -70.02547454833984,
      "activations/layer9_attention_weight_max": 72.01634216308594,
      "activations/layer9_attention_weight_min": -69.41862487792969,
      "epoch": 0.22,
      "learning_rate": 0.00014922872583529145,
      "loss": 3.5764,
      "step": 3850
    },
    {
      "activations/layer0_attention_weight_max": 12.692529678344727,
      "activations/layer0_attention_weight_min": -9.01816463470459,
      "activations/layer10_attention_weight_max": 66.44136810302734,
      "activations/layer10_attention_weight_min": -60.03363800048828,
      "activations/layer11_attention_weight_max": 27.9742431640625,
      "activations/layer11_attention_weight_min": -23.019733428955078,
      "activations/layer12_attention_weight_max": 21.147260665893555,
      "activations/layer12_attention_weight_min": -21.063093185424805,
      "activations/layer13_attention_weight_max": 27.17902946472168,
      "activations/layer13_attention_weight_min": -18.625715255737305,
      "activations/layer14_attention_weight_max": 22.81597900390625,
      "activations/layer14_attention_weight_min": -17.231529235839844,
      "activations/layer15_attention_weight_max": 23.031421661376953,
      "activations/layer15_attention_weight_min": -13.613897323608398,
      "activations/layer16_attention_weight_max": 23.189163208007812,
      "activations/layer16_attention_weight_min": -14.763348579406738,
      "activations/layer17_attention_weight_max": 19.601581573486328,
      "activations/layer17_attention_weight_min": -19.014623641967773,
      "activations/layer18_attention_weight_max": 25.384347915649414,
      "activations/layer18_attention_weight_min": -14.360973358154297,
      "activations/layer19_attention_weight_max": 21.0434627532959,
      "activations/layer19_attention_weight_min": -13.133589744567871,
      "activations/layer1_attention_weight_max": 17.85013771057129,
      "activations/layer1_attention_weight_min": -11.210392951965332,
      "activations/layer20_attention_weight_max": 21.730018615722656,
      "activations/layer20_attention_weight_min": -13.041680335998535,
      "activations/layer21_attention_weight_max": 22.103748321533203,
      "activations/layer21_attention_weight_min": -12.732179641723633,
      "activations/layer22_attention_weight_max": 16.45209503173828,
      "activations/layer22_attention_weight_min": -13.3548583984375,
      "activations/layer23_attention_weight_max": 14.00683879852295,
      "activations/layer23_attention_weight_min": -11.47171401977539,
      "activations/layer2_attention_weight_max": 27.40071678161621,
      "activations/layer2_attention_weight_min": -19.231014251708984,
      "activations/layer3_attention_weight_max": 58.98809814453125,
      "activations/layer3_attention_weight_min": -44.48561477661133,
      "activations/layer4_attention_weight_max": 94.68462371826172,
      "activations/layer4_attention_weight_min": -81.40943908691406,
      "activations/layer5_attention_weight_max": 168.8726806640625,
      "activations/layer5_attention_weight_min": -155.26022338867188,
      "activations/layer6_attention_weight_max": 136.59413146972656,
      "activations/layer6_attention_weight_min": -129.58401489257812,
      "activations/layer7_attention_weight_max": 102.97100067138672,
      "activations/layer7_attention_weight_min": -95.75619506835938,
      "activations/layer8_attention_weight_max": 77.28794860839844,
      "activations/layer8_attention_weight_min": -71.23091888427734,
      "activations/layer9_attention_weight_max": 77.59266662597656,
      "activations/layer9_attention_weight_min": -67.78838348388672,
      "epoch": 0.23,
      "learning_rate": 0.00014946311551669234,
      "loss": 3.558,
      "step": 3900
    },
    {
      "activations/layer0_attention_weight_max": 13.135930061340332,
      "activations/layer0_attention_weight_min": -9.324804306030273,
      "activations/layer10_attention_weight_max": 65.94796752929688,
      "activations/layer10_attention_weight_min": -61.842735290527344,
      "activations/layer11_attention_weight_max": 28.477697372436523,
      "activations/layer11_attention_weight_min": -23.15676498413086,
      "activations/layer12_attention_weight_max": 22.885786056518555,
      "activations/layer12_attention_weight_min": -22.512489318847656,
      "activations/layer13_attention_weight_max": 26.632061004638672,
      "activations/layer13_attention_weight_min": -22.685527801513672,
      "activations/layer14_attention_weight_max": 21.935762405395508,
      "activations/layer14_attention_weight_min": -15.745229721069336,
      "activations/layer15_attention_weight_max": 20.511276245117188,
      "activations/layer15_attention_weight_min": -13.210321426391602,
      "activations/layer16_attention_weight_max": 23.683401107788086,
      "activations/layer16_attention_weight_min": -13.525581359863281,
      "activations/layer17_attention_weight_max": 19.2646484375,
      "activations/layer17_attention_weight_min": -15.907620429992676,
      "activations/layer18_attention_weight_max": 24.059307098388672,
      "activations/layer18_attention_weight_min": -14.833829879760742,
      "activations/layer19_attention_weight_max": 19.616273880004883,
      "activations/layer19_attention_weight_min": -11.664192199707031,
      "activations/layer1_attention_weight_max": 17.546796798706055,
      "activations/layer1_attention_weight_min": -12.36135196685791,
      "activations/layer20_attention_weight_max": 20.369157791137695,
      "activations/layer20_attention_weight_min": -11.972268104553223,
      "activations/layer21_attention_weight_max": 21.7252197265625,
      "activations/layer21_attention_weight_min": -15.70152473449707,
      "activations/layer22_attention_weight_max": 15.974702835083008,
      "activations/layer22_attention_weight_min": -13.40966510772705,
      "activations/layer23_attention_weight_max": 12.4343900680542,
      "activations/layer23_attention_weight_min": -11.615072250366211,
      "activations/layer2_attention_weight_max": 25.670087814331055,
      "activations/layer2_attention_weight_min": -19.366287231445312,
      "activations/layer3_attention_weight_max": 56.91093444824219,
      "activations/layer3_attention_weight_min": -45.52425003051758,
      "activations/layer4_attention_weight_max": 90.4559097290039,
      "activations/layer4_attention_weight_min": -79.22278594970703,
      "activations/layer5_attention_weight_max": 168.37744140625,
      "activations/layer5_attention_weight_min": -153.31317138671875,
      "activations/layer6_attention_weight_max": 137.27401733398438,
      "activations/layer6_attention_weight_min": -131.64031982421875,
      "activations/layer7_attention_weight_max": 104.21283721923828,
      "activations/layer7_attention_weight_min": -96.74893951416016,
      "activations/layer8_attention_weight_max": 74.44356536865234,
      "activations/layer8_attention_weight_min": -71.70308685302734,
      "activations/layer9_attention_weight_max": 82.38521575927734,
      "activations/layer9_attention_weight_min": -75.99626159667969,
      "epoch": 0.23,
      "learning_rate": 0.00014969450627130542,
      "loss": 3.5574,
      "step": 3950
    },
    {
      "activations/layer0_attention_weight_max": 12.957918167114258,
      "activations/layer0_attention_weight_min": -8.723211288452148,
      "activations/layer10_attention_weight_max": 76.2961196899414,
      "activations/layer10_attention_weight_min": -55.904537200927734,
      "activations/layer11_attention_weight_max": 25.832149505615234,
      "activations/layer11_attention_weight_min": -22.694639205932617,
      "activations/layer12_attention_weight_max": 22.99510383605957,
      "activations/layer12_attention_weight_min": -21.189931869506836,
      "activations/layer13_attention_weight_max": 27.501333236694336,
      "activations/layer13_attention_weight_min": -20.188220977783203,
      "activations/layer14_attention_weight_max": 28.028593063354492,
      "activations/layer14_attention_weight_min": -16.32887077331543,
      "activations/layer15_attention_weight_max": 24.81849479675293,
      "activations/layer15_attention_weight_min": -14.83276081085205,
      "activations/layer16_attention_weight_max": 21.00318717956543,
      "activations/layer16_attention_weight_min": -11.659276008605957,
      "activations/layer17_attention_weight_max": 17.86273765563965,
      "activations/layer17_attention_weight_min": -14.681281089782715,
      "activations/layer18_attention_weight_max": 23.24046516418457,
      "activations/layer18_attention_weight_min": -12.83108139038086,
      "activations/layer19_attention_weight_max": 20.050382614135742,
      "activations/layer19_attention_weight_min": -12.201430320739746,
      "activations/layer1_attention_weight_max": 17.095272064208984,
      "activations/layer1_attention_weight_min": -10.668569564819336,
      "activations/layer20_attention_weight_max": 18.036754608154297,
      "activations/layer20_attention_weight_min": -10.819228172302246,
      "activations/layer21_attention_weight_max": 20.174457550048828,
      "activations/layer21_attention_weight_min": -12.463988304138184,
      "activations/layer22_attention_weight_max": 15.599241256713867,
      "activations/layer22_attention_weight_min": -11.275543212890625,
      "activations/layer23_attention_weight_max": 12.410475730895996,
      "activations/layer23_attention_weight_min": -9.83566665649414,
      "activations/layer2_attention_weight_max": 25.11482810974121,
      "activations/layer2_attention_weight_min": -19.464937210083008,
      "activations/layer3_attention_weight_max": 57.31867980957031,
      "activations/layer3_attention_weight_min": -45.762088775634766,
      "activations/layer4_attention_weight_max": 93.79393005371094,
      "activations/layer4_attention_weight_min": -81.47657775878906,
      "activations/layer5_attention_weight_max": 176.1510009765625,
      "activations/layer5_attention_weight_min": -152.98326110839844,
      "activations/layer6_attention_weight_max": 147.5010986328125,
      "activations/layer6_attention_weight_min": -125.9532470703125,
      "activations/layer7_attention_weight_max": 109.81608581542969,
      "activations/layer7_attention_weight_min": -97.1268539428711,
      "activations/layer8_attention_weight_max": 84.18408966064453,
      "activations/layer8_attention_weight_min": -69.90324401855469,
      "activations/layer9_attention_weight_max": 83.40620422363281,
      "activations/layer9_attention_weight_min": -66.54144287109375,
      "epoch": 0.23,
      "learning_rate": 0.00014992297387093407,
      "loss": 3.572,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_loss": 3.5,
      "eval_runtime": 8.2085,
      "eval_samples_per_second": 523.114,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_openwebtext_loss": 3.5,
      "eval_openwebtext_ppl": 33.11545195869231,
      "eval_openwebtext_runtime": 8.2085,
      "eval_openwebtext_samples_per_second": 523.114,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_wikitext_loss": 3.869140625,
      "eval_wikitext_ppl": 47.90120329104557,
      "eval_wikitext_runtime": 1.7093,
      "eval_wikitext_samples_per_second": 266.779,
      "step": 4000
    },
    {
      "epoch": 0.23,
      "eval_lambada_loss": 4.4453125,
      "eval_lambada_ppl": 85.2265069597304,
      "eval_lambada_runtime": 9.2831,
      "eval_lambada_samples_per_second": 524.503,
      "step": 4000
    },
    {
      "activations/layer0_attention_weight_max": 12.922432899475098,
      "activations/layer0_attention_weight_min": -9.341318130493164,
      "activations/layer10_attention_weight_max": 64.95115661621094,
      "activations/layer10_attention_weight_min": -50.944942474365234,
      "activations/layer11_attention_weight_max": 29.408170700073242,
      "activations/layer11_attention_weight_min": -23.216108322143555,
      "activations/layer12_attention_weight_max": 24.672571182250977,
      "activations/layer12_attention_weight_min": -22.113948822021484,
      "activations/layer13_attention_weight_max": 29.31346321105957,
      "activations/layer13_attention_weight_min": -20.832763671875,
      "activations/layer14_attention_weight_max": 25.269989013671875,
      "activations/layer14_attention_weight_min": -15.430288314819336,
      "activations/layer15_attention_weight_max": 24.425434112548828,
      "activations/layer15_attention_weight_min": -13.233510971069336,
      "activations/layer16_attention_weight_max": 24.611164093017578,
      "activations/layer16_attention_weight_min": -12.906489372253418,
      "activations/layer17_attention_weight_max": 23.505680084228516,
      "activations/layer17_attention_weight_min": -16.6563777923584,
      "activations/layer18_attention_weight_max": 25.455581665039062,
      "activations/layer18_attention_weight_min": -14.347898483276367,
      "activations/layer19_attention_weight_max": 22.185049057006836,
      "activations/layer19_attention_weight_min": -12.492766380310059,
      "activations/layer1_attention_weight_max": 18.80647850036621,
      "activations/layer1_attention_weight_min": -11.460125923156738,
      "activations/layer20_attention_weight_max": 25.442974090576172,
      "activations/layer20_attention_weight_min": -12.539347648620605,
      "activations/layer21_attention_weight_max": 21.672367095947266,
      "activations/layer21_attention_weight_min": -16.46725845336914,
      "activations/layer22_attention_weight_max": 16.63432502746582,
      "activations/layer22_attention_weight_min": -11.597770690917969,
      "activations/layer23_attention_weight_max": 14.632823944091797,
      "activations/layer23_attention_weight_min": -10.01895809173584,
      "activations/layer2_attention_weight_max": 28.23370361328125,
      "activations/layer2_attention_weight_min": -19.29131317138672,
      "activations/layer3_attention_weight_max": 57.881263732910156,
      "activations/layer3_attention_weight_min": -47.51893997192383,
      "activations/layer4_attention_weight_max": 88.4432601928711,
      "activations/layer4_attention_weight_min": -80.36808013916016,
      "activations/layer5_attention_weight_max": 160.2830352783203,
      "activations/layer5_attention_weight_min": -151.3256072998047,
      "activations/layer6_attention_weight_max": 129.09375,
      "activations/layer6_attention_weight_min": -122.03375244140625,
      "activations/layer7_attention_weight_max": 97.06200408935547,
      "activations/layer7_attention_weight_min": -94.59812927246094,
      "activations/layer8_attention_weight_max": 74.57205963134766,
      "activations/layer8_attention_weight_min": -69.39122009277344,
      "activations/layer9_attention_weight_max": 68.66924285888672,
      "activations/layer9_attention_weight_min": -69.85528564453125,
      "epoch": 0.24,
      "learning_rate": 0.00014998825757575756,
      "loss": 3.5429,
      "step": 4050
    },
    {
      "activations/layer0_attention_weight_max": 12.267108917236328,
      "activations/layer0_attention_weight_min": -9.222268104553223,
      "activations/layer10_attention_weight_max": 63.475521087646484,
      "activations/layer10_attention_weight_min": -56.571746826171875,
      "activations/layer11_attention_weight_max": 33.49913787841797,
      "activations/layer11_attention_weight_min": -23.326412200927734,
      "activations/layer12_attention_weight_max": 21.32343864440918,
      "activations/layer12_attention_weight_min": -22.934974670410156,
      "activations/layer13_attention_weight_max": 27.529644012451172,
      "activations/layer13_attention_weight_min": -22.66442108154297,
      "activations/layer14_attention_weight_max": 24.57703971862793,
      "activations/layer14_attention_weight_min": -16.759540557861328,
      "activations/layer15_attention_weight_max": 20.2943058013916,
      "activations/layer15_attention_weight_min": -13.5014009475708,
      "activations/layer16_attention_weight_max": 20.694421768188477,
      "activations/layer16_attention_weight_min": -13.507407188415527,
      "activations/layer17_attention_weight_max": 18.86433219909668,
      "activations/layer17_attention_weight_min": -15.417407989501953,
      "activations/layer18_attention_weight_max": 28.18563461303711,
      "activations/layer18_attention_weight_min": -16.5325870513916,
      "activations/layer19_attention_weight_max": 18.38149070739746,
      "activations/layer19_attention_weight_min": -12.4782133102417,
      "activations/layer1_attention_weight_max": 17.96207618713379,
      "activations/layer1_attention_weight_min": -11.644177436828613,
      "activations/layer20_attention_weight_max": 19.401599884033203,
      "activations/layer20_attention_weight_min": -12.337018966674805,
      "activations/layer21_attention_weight_max": 19.402786254882812,
      "activations/layer21_attention_weight_min": -11.622062683105469,
      "activations/layer22_attention_weight_max": 17.174196243286133,
      "activations/layer22_attention_weight_min": -11.962164878845215,
      "activations/layer23_attention_weight_max": 12.139823913574219,
      "activations/layer23_attention_weight_min": -9.641709327697754,
      "activations/layer2_attention_weight_max": 28.496440887451172,
      "activations/layer2_attention_weight_min": -19.11798095703125,
      "activations/layer3_attention_weight_max": 58.645389556884766,
      "activations/layer3_attention_weight_min": -44.687950134277344,
      "activations/layer4_attention_weight_max": 95.75899505615234,
      "activations/layer4_attention_weight_min": -80.67679595947266,
      "activations/layer5_attention_weight_max": 175.63351440429688,
      "activations/layer5_attention_weight_min": -156.3809814453125,
      "activations/layer6_attention_weight_max": 139.66015625,
      "activations/layer6_attention_weight_min": -126.76113891601562,
      "activations/layer7_attention_weight_max": 104.72409057617188,
      "activations/layer7_attention_weight_min": -93.68074798583984,
      "activations/layer8_attention_weight_max": 79.86241149902344,
      "activations/layer8_attention_weight_min": -71.33195495605469,
      "activations/layer9_attention_weight_max": 78.06491088867188,
      "activations/layer9_attention_weight_min": -72.98094940185547,
      "epoch": 0.24,
      "learning_rate": 0.00014996931818181817,
      "loss": 3.5661,
      "step": 4100
    },
    {
      "activations/layer0_attention_weight_max": 12.746415138244629,
      "activations/layer0_attention_weight_min": -10.45690631866455,
      "activations/layer10_attention_weight_max": 72.4345932006836,
      "activations/layer10_attention_weight_min": -56.866661071777344,
      "activations/layer11_attention_weight_max": 30.291278839111328,
      "activations/layer11_attention_weight_min": -22.690265655517578,
      "activations/layer12_attention_weight_max": 20.654253005981445,
      "activations/layer12_attention_weight_min": -23.89862060546875,
      "activations/layer13_attention_weight_max": 27.32077980041504,
      "activations/layer13_attention_weight_min": -21.303157806396484,
      "activations/layer14_attention_weight_max": 21.81394386291504,
      "activations/layer14_attention_weight_min": -15.478814125061035,
      "activations/layer15_attention_weight_max": 20.973493576049805,
      "activations/layer15_attention_weight_min": -15.939520835876465,
      "activations/layer16_attention_weight_max": 22.57499885559082,
      "activations/layer16_attention_weight_min": -13.851263046264648,
      "activations/layer17_attention_weight_max": 23.314319610595703,
      "activations/layer17_attention_weight_min": -14.409171104431152,
      "activations/layer18_attention_weight_max": 23.038057327270508,
      "activations/layer18_attention_weight_min": -13.212054252624512,
      "activations/layer19_attention_weight_max": 24.8039608001709,
      "activations/layer19_attention_weight_min": -11.056581497192383,
      "activations/layer1_attention_weight_max": 17.741714477539062,
      "activations/layer1_attention_weight_min": -10.86870002746582,
      "activations/layer20_attention_weight_max": 22.05202865600586,
      "activations/layer20_attention_weight_min": -10.559375762939453,
      "activations/layer21_attention_weight_max": 21.520305633544922,
      "activations/layer21_attention_weight_min": -14.402729988098145,
      "activations/layer22_attention_weight_max": 18.54267692565918,
      "activations/layer22_attention_weight_min": -13.583122253417969,
      "activations/layer23_attention_weight_max": 14.561025619506836,
      "activations/layer23_attention_weight_min": -13.023688316345215,
      "activations/layer2_attention_weight_max": 27.363447189331055,
      "activations/layer2_attention_weight_min": -18.79170036315918,
      "activations/layer3_attention_weight_max": 59.368858337402344,
      "activations/layer3_attention_weight_min": -45.13166427612305,
      "activations/layer4_attention_weight_max": 96.35430145263672,
      "activations/layer4_attention_weight_min": -78.59937286376953,
      "activations/layer5_attention_weight_max": 178.31016540527344,
      "activations/layer5_attention_weight_min": -153.65733337402344,
      "activations/layer6_attention_weight_max": 139.96099853515625,
      "activations/layer6_attention_weight_min": -127.5604476928711,
      "activations/layer7_attention_weight_max": 110.82166290283203,
      "activations/layer7_attention_weight_min": -90.11453247070312,
      "activations/layer8_attention_weight_max": 81.78443908691406,
      "activations/layer8_attention_weight_min": -70.22184753417969,
      "activations/layer9_attention_weight_max": 75.55384826660156,
      "activations/layer9_attention_weight_min": -67.11775970458984,
      "epoch": 0.24,
      "learning_rate": 0.00014995037878787876,
      "loss": 3.549,
      "step": 4150
    },
    {
      "activations/layer0_attention_weight_max": 13.354903221130371,
      "activations/layer0_attention_weight_min": -9.434210777282715,
      "activations/layer10_attention_weight_max": 65.544677734375,
      "activations/layer10_attention_weight_min": -55.647518157958984,
      "activations/layer11_attention_weight_max": 24.477066040039062,
      "activations/layer11_attention_weight_min": -24.628774642944336,
      "activations/layer12_attention_weight_max": 22.6064453125,
      "activations/layer12_attention_weight_min": -24.399349212646484,
      "activations/layer13_attention_weight_max": 24.70319366455078,
      "activations/layer13_attention_weight_min": -18.918821334838867,
      "activations/layer14_attention_weight_max": 22.431133270263672,
      "activations/layer14_attention_weight_min": -15.240964889526367,
      "activations/layer15_attention_weight_max": 22.191543579101562,
      "activations/layer15_attention_weight_min": -12.615792274475098,
      "activations/layer16_attention_weight_max": 22.529878616333008,
      "activations/layer16_attention_weight_min": -14.659379959106445,
      "activations/layer17_attention_weight_max": 18.798486709594727,
      "activations/layer17_attention_weight_min": -16.310400009155273,
      "activations/layer18_attention_weight_max": 24.91933822631836,
      "activations/layer18_attention_weight_min": -14.317955017089844,
      "activations/layer19_attention_weight_max": 20.092844009399414,
      "activations/layer19_attention_weight_min": -11.60526180267334,
      "activations/layer1_attention_weight_max": 19.004419326782227,
      "activations/layer1_attention_weight_min": -12.52000904083252,
      "activations/layer20_attention_weight_max": 18.478588104248047,
      "activations/layer20_attention_weight_min": -11.412520408630371,
      "activations/layer21_attention_weight_max": 20.214303970336914,
      "activations/layer21_attention_weight_min": -14.268193244934082,
      "activations/layer22_attention_weight_max": 17.471364974975586,
      "activations/layer22_attention_weight_min": -12.3079833984375,
      "activations/layer23_attention_weight_max": 11.801454544067383,
      "activations/layer23_attention_weight_min": -10.72241497039795,
      "activations/layer2_attention_weight_max": 24.413000106811523,
      "activations/layer2_attention_weight_min": -19.63800048828125,
      "activations/layer3_attention_weight_max": 55.52814865112305,
      "activations/layer3_attention_weight_min": -45.2667236328125,
      "activations/layer4_attention_weight_max": 89.32499694824219,
      "activations/layer4_attention_weight_min": -78.18624114990234,
      "activations/layer5_attention_weight_max": 164.39520263671875,
      "activations/layer5_attention_weight_min": -149.862548828125,
      "activations/layer6_attention_weight_max": 125.65673828125,
      "activations/layer6_attention_weight_min": -125.66138458251953,
      "activations/layer7_attention_weight_max": 97.00056457519531,
      "activations/layer7_attention_weight_min": -89.33293914794922,
      "activations/layer8_attention_weight_max": 69.44493865966797,
      "activations/layer8_attention_weight_min": -67.958740234375,
      "activations/layer9_attention_weight_max": 71.3185043334961,
      "activations/layer9_attention_weight_min": -70.15474700927734,
      "epoch": 0.24,
      "learning_rate": 0.00014993143939393938,
      "loss": 3.542,
      "step": 4200
    },
    {
      "activations/layer0_attention_weight_max": 13.558711051940918,
      "activations/layer0_attention_weight_min": -8.85112476348877,
      "activations/layer10_attention_weight_max": 57.90581512451172,
      "activations/layer10_attention_weight_min": -59.82111740112305,
      "activations/layer11_attention_weight_max": 29.93793487548828,
      "activations/layer11_attention_weight_min": -24.997005462646484,
      "activations/layer12_attention_weight_max": 21.63177490234375,
      "activations/layer12_attention_weight_min": -24.15626335144043,
      "activations/layer13_attention_weight_max": 26.259939193725586,
      "activations/layer13_attention_weight_min": -23.0556583404541,
      "activations/layer14_attention_weight_max": 22.39417839050293,
      "activations/layer14_attention_weight_min": -16.34595489501953,
      "activations/layer15_attention_weight_max": 22.826091766357422,
      "activations/layer15_attention_weight_min": -14.541379928588867,
      "activations/layer16_attention_weight_max": 22.335643768310547,
      "activations/layer16_attention_weight_min": -12.571181297302246,
      "activations/layer17_attention_weight_max": 21.308902740478516,
      "activations/layer17_attention_weight_min": -17.991785049438477,
      "activations/layer18_attention_weight_max": 25.200761795043945,
      "activations/layer18_attention_weight_min": -16.4256649017334,
      "activations/layer19_attention_weight_max": 24.4117488861084,
      "activations/layer19_attention_weight_min": -11.443881034851074,
      "activations/layer1_attention_weight_max": 17.498817443847656,
      "activations/layer1_attention_weight_min": -12.032804489135742,
      "activations/layer20_attention_weight_max": 20.86054801940918,
      "activations/layer20_attention_weight_min": -12.178925514221191,
      "activations/layer21_attention_weight_max": 21.904129028320312,
      "activations/layer21_attention_weight_min": -12.812423706054688,
      "activations/layer22_attention_weight_max": 17.59992218017578,
      "activations/layer22_attention_weight_min": -12.21886920928955,
      "activations/layer23_attention_weight_max": 13.66132926940918,
      "activations/layer23_attention_weight_min": -10.186129570007324,
      "activations/layer2_attention_weight_max": 24.570791244506836,
      "activations/layer2_attention_weight_min": -20.462066650390625,
      "activations/layer3_attention_weight_max": 55.814544677734375,
      "activations/layer3_attention_weight_min": -46.21213150024414,
      "activations/layer4_attention_weight_max": 88.07752227783203,
      "activations/layer4_attention_weight_min": -80.74237823486328,
      "activations/layer5_attention_weight_max": 162.6641845703125,
      "activations/layer5_attention_weight_min": -154.10110473632812,
      "activations/layer6_attention_weight_max": 126.95671081542969,
      "activations/layer6_attention_weight_min": -121.92704772949219,
      "activations/layer7_attention_weight_max": 95.49139404296875,
      "activations/layer7_attention_weight_min": -94.48226928710938,
      "activations/layer8_attention_weight_max": 70.1613998413086,
      "activations/layer8_attention_weight_min": -70.40367889404297,
      "activations/layer9_attention_weight_max": 68.2710189819336,
      "activations/layer9_attention_weight_min": -67.219482421875,
      "epoch": 0.25,
      "learning_rate": 0.0001499125,
      "loss": 3.5399,
      "step": 4250
    },
    {
      "activations/layer0_attention_weight_max": 13.456320762634277,
      "activations/layer0_attention_weight_min": -9.118531227111816,
      "activations/layer10_attention_weight_max": 64.59908294677734,
      "activations/layer10_attention_weight_min": -54.594078063964844,
      "activations/layer11_attention_weight_max": 26.85384750366211,
      "activations/layer11_attention_weight_min": -20.3439998626709,
      "activations/layer12_attention_weight_max": 21.99992561340332,
      "activations/layer12_attention_weight_min": -20.4299373626709,
      "activations/layer13_attention_weight_max": 27.824783325195312,
      "activations/layer13_attention_weight_min": -20.241374969482422,
      "activations/layer14_attention_weight_max": 32.99109649658203,
      "activations/layer14_attention_weight_min": -16.735151290893555,
      "activations/layer15_attention_weight_max": 23.253747940063477,
      "activations/layer15_attention_weight_min": -12.873764991760254,
      "activations/layer16_attention_weight_max": 22.90965461730957,
      "activations/layer16_attention_weight_min": -13.55341625213623,
      "activations/layer17_attention_weight_max": 19.861299514770508,
      "activations/layer17_attention_weight_min": -16.176244735717773,
      "activations/layer18_attention_weight_max": 21.681133270263672,
      "activations/layer18_attention_weight_min": -13.13558292388916,
      "activations/layer19_attention_weight_max": 17.43201446533203,
      "activations/layer19_attention_weight_min": -12.056109428405762,
      "activations/layer1_attention_weight_max": 17.967878341674805,
      "activations/layer1_attention_weight_min": -11.157538414001465,
      "activations/layer20_attention_weight_max": 18.890560150146484,
      "activations/layer20_attention_weight_min": -10.785750389099121,
      "activations/layer21_attention_weight_max": 20.79805564880371,
      "activations/layer21_attention_weight_min": -11.251370429992676,
      "activations/layer22_attention_weight_max": 17.011241912841797,
      "activations/layer22_attention_weight_min": -13.423237800598145,
      "activations/layer23_attention_weight_max": 13.89486312866211,
      "activations/layer23_attention_weight_min": -11.582895278930664,
      "activations/layer2_attention_weight_max": 26.95393180847168,
      "activations/layer2_attention_weight_min": -20.34664535522461,
      "activations/layer3_attention_weight_max": 59.053428649902344,
      "activations/layer3_attention_weight_min": -44.33647537231445,
      "activations/layer4_attention_weight_max": 94.64684295654297,
      "activations/layer4_attention_weight_min": -81.00093078613281,
      "activations/layer5_attention_weight_max": 168.96597290039062,
      "activations/layer5_attention_weight_min": -150.7110137939453,
      "activations/layer6_attention_weight_max": 131.26397705078125,
      "activations/layer6_attention_weight_min": -122.17352294921875,
      "activations/layer7_attention_weight_max": 101.21334838867188,
      "activations/layer7_attention_weight_min": -91.87762451171875,
      "activations/layer8_attention_weight_max": 73.84423065185547,
      "activations/layer8_attention_weight_min": -65.25532531738281,
      "activations/layer9_attention_weight_max": 75.3524398803711,
      "activations/layer9_attention_weight_min": -71.7279052734375,
      "epoch": 0.25,
      "learning_rate": 0.00014989356060606058,
      "loss": 3.5273,
      "step": 4300
    },
    {
      "activations/layer0_attention_weight_max": 12.661603927612305,
      "activations/layer0_attention_weight_min": -9.27232551574707,
      "activations/layer10_attention_weight_max": 62.44463348388672,
      "activations/layer10_attention_weight_min": -53.74207305908203,
      "activations/layer11_attention_weight_max": 31.427204132080078,
      "activations/layer11_attention_weight_min": -24.24825668334961,
      "activations/layer12_attention_weight_max": 22.620750427246094,
      "activations/layer12_attention_weight_min": -22.294111251831055,
      "activations/layer13_attention_weight_max": 27.819387435913086,
      "activations/layer13_attention_weight_min": -19.339637756347656,
      "activations/layer14_attention_weight_max": 23.901901245117188,
      "activations/layer14_attention_weight_min": -17.14377784729004,
      "activations/layer15_attention_weight_max": 25.19447898864746,
      "activations/layer15_attention_weight_min": -15.040491104125977,
      "activations/layer16_attention_weight_max": 23.77360725402832,
      "activations/layer16_attention_weight_min": -12.680893898010254,
      "activations/layer17_attention_weight_max": 22.003381729125977,
      "activations/layer17_attention_weight_min": -18.060131072998047,
      "activations/layer18_attention_weight_max": 25.16966438293457,
      "activations/layer18_attention_weight_min": -15.307615280151367,
      "activations/layer19_attention_weight_max": 23.81521224975586,
      "activations/layer19_attention_weight_min": -11.958783149719238,
      "activations/layer1_attention_weight_max": 18.760778427124023,
      "activations/layer1_attention_weight_min": -11.40529727935791,
      "activations/layer20_attention_weight_max": 24.015260696411133,
      "activations/layer20_attention_weight_min": -13.729653358459473,
      "activations/layer21_attention_weight_max": 21.343366622924805,
      "activations/layer21_attention_weight_min": -13.419381141662598,
      "activations/layer22_attention_weight_max": 19.824724197387695,
      "activations/layer22_attention_weight_min": -14.004472732543945,
      "activations/layer23_attention_weight_max": 14.916678428649902,
      "activations/layer23_attention_weight_min": -13.49143123626709,
      "activations/layer2_attention_weight_max": 24.483585357666016,
      "activations/layer2_attention_weight_min": -20.368247985839844,
      "activations/layer3_attention_weight_max": 57.16581726074219,
      "activations/layer3_attention_weight_min": -44.034053802490234,
      "activations/layer4_attention_weight_max": 91.982421875,
      "activations/layer4_attention_weight_min": -81.0694808959961,
      "activations/layer5_attention_weight_max": 172.53060913085938,
      "activations/layer5_attention_weight_min": -157.08346557617188,
      "activations/layer6_attention_weight_max": 131.89535522460938,
      "activations/layer6_attention_weight_min": -125.39191436767578,
      "activations/layer7_attention_weight_max": 105.92401123046875,
      "activations/layer7_attention_weight_min": -93.81713104248047,
      "activations/layer8_attention_weight_max": 77.92306518554688,
      "activations/layer8_attention_weight_min": -68.4742202758789,
      "activations/layer9_attention_weight_max": 75.10172271728516,
      "activations/layer9_attention_weight_min": -65.38784790039062,
      "epoch": 0.25,
      "learning_rate": 0.0001498746212121212,
      "loss": 3.5043,
      "step": 4350
    },
    {
      "activations/layer0_attention_weight_max": 12.845006942749023,
      "activations/layer0_attention_weight_min": -9.304282188415527,
      "activations/layer10_attention_weight_max": 72.28128814697266,
      "activations/layer10_attention_weight_min": -61.41264343261719,
      "activations/layer11_attention_weight_max": 36.013916015625,
      "activations/layer11_attention_weight_min": -23.67395782470703,
      "activations/layer12_attention_weight_max": 22.327117919921875,
      "activations/layer12_attention_weight_min": -22.444738388061523,
      "activations/layer13_attention_weight_max": 29.01951026916504,
      "activations/layer13_attention_weight_min": -22.400911331176758,
      "activations/layer14_attention_weight_max": 32.371795654296875,
      "activations/layer14_attention_weight_min": -17.168567657470703,
      "activations/layer15_attention_weight_max": 21.52582550048828,
      "activations/layer15_attention_weight_min": -14.172297477722168,
      "activations/layer16_attention_weight_max": 23.301733016967773,
      "activations/layer16_attention_weight_min": -12.908833503723145,
      "activations/layer17_attention_weight_max": 20.956485748291016,
      "activations/layer17_attention_weight_min": -16.004117965698242,
      "activations/layer18_attention_weight_max": 36.1447639465332,
      "activations/layer18_attention_weight_min": -16.178237915039062,
      "activations/layer19_attention_weight_max": 21.3549747467041,
      "activations/layer19_attention_weight_min": -12.653552055358887,
      "activations/layer1_attention_weight_max": 18.429492950439453,
      "activations/layer1_attention_weight_min": -11.185856819152832,
      "activations/layer20_attention_weight_max": 20.2883243560791,
      "activations/layer20_attention_weight_min": -12.834464073181152,
      "activations/layer21_attention_weight_max": 22.072280883789062,
      "activations/layer21_attention_weight_min": -13.597227096557617,
      "activations/layer22_attention_weight_max": 19.629873275756836,
      "activations/layer22_attention_weight_min": -12.184820175170898,
      "activations/layer23_attention_weight_max": 14.60811996459961,
      "activations/layer23_attention_weight_min": -10.548982620239258,
      "activations/layer2_attention_weight_max": 27.610530853271484,
      "activations/layer2_attention_weight_min": -19.682151794433594,
      "activations/layer3_attention_weight_max": 61.600738525390625,
      "activations/layer3_attention_weight_min": -45.457679748535156,
      "activations/layer4_attention_weight_max": 96.97479248046875,
      "activations/layer4_attention_weight_min": -78.83438873291016,
      "activations/layer5_attention_weight_max": 177.0876922607422,
      "activations/layer5_attention_weight_min": -150.81069946289062,
      "activations/layer6_attention_weight_max": 136.10482788085938,
      "activations/layer6_attention_weight_min": -119.57534790039062,
      "activations/layer7_attention_weight_max": 110.71841430664062,
      "activations/layer7_attention_weight_min": -89.09164428710938,
      "activations/layer8_attention_weight_max": 81.62151336669922,
      "activations/layer8_attention_weight_min": -68.1867446899414,
      "activations/layer9_attention_weight_max": 79.21355438232422,
      "activations/layer9_attention_weight_min": -71.98326873779297,
      "epoch": 0.26,
      "learning_rate": 0.0001498556818181818,
      "loss": 3.5308,
      "step": 4400
    },
    {
      "activations/layer0_attention_weight_max": 13.686346054077148,
      "activations/layer0_attention_weight_min": -9.046853065490723,
      "activations/layer10_attention_weight_max": 73.8586196899414,
      "activations/layer10_attention_weight_min": -60.85799026489258,
      "activations/layer11_attention_weight_max": 40.9711799621582,
      "activations/layer11_attention_weight_min": -23.010114669799805,
      "activations/layer12_attention_weight_max": 22.575397491455078,
      "activations/layer12_attention_weight_min": -22.546030044555664,
      "activations/layer13_attention_weight_max": 31.07318687438965,
      "activations/layer13_attention_weight_min": -21.397520065307617,
      "activations/layer14_attention_weight_max": 33.47918701171875,
      "activations/layer14_attention_weight_min": -17.055938720703125,
      "activations/layer15_attention_weight_max": 26.970726013183594,
      "activations/layer15_attention_weight_min": -13.788633346557617,
      "activations/layer16_attention_weight_max": 24.7782039642334,
      "activations/layer16_attention_weight_min": -14.911808967590332,
      "activations/layer17_attention_weight_max": 33.0462532043457,
      "activations/layer17_attention_weight_min": -18.22764015197754,
      "activations/layer18_attention_weight_max": 33.27082824707031,
      "activations/layer18_attention_weight_min": -15.339021682739258,
      "activations/layer19_attention_weight_max": 26.021530151367188,
      "activations/layer19_attention_weight_min": -13.379493713378906,
      "activations/layer1_attention_weight_max": 17.477724075317383,
      "activations/layer1_attention_weight_min": -10.543115615844727,
      "activations/layer20_attention_weight_max": 24.595951080322266,
      "activations/layer20_attention_weight_min": -12.488079071044922,
      "activations/layer21_attention_weight_max": 27.295011520385742,
      "activations/layer21_attention_weight_min": -12.22325325012207,
      "activations/layer22_attention_weight_max": 24.514726638793945,
      "activations/layer22_attention_weight_min": -11.951034545898438,
      "activations/layer23_attention_weight_max": 14.978559494018555,
      "activations/layer23_attention_weight_min": -10.218671798706055,
      "activations/layer2_attention_weight_max": 28.1378231048584,
      "activations/layer2_attention_weight_min": -20.089448928833008,
      "activations/layer3_attention_weight_max": 59.88615798950195,
      "activations/layer3_attention_weight_min": -44.02826690673828,
      "activations/layer4_attention_weight_max": 93.01519775390625,
      "activations/layer4_attention_weight_min": -79.26943969726562,
      "activations/layer5_attention_weight_max": 174.1182098388672,
      "activations/layer5_attention_weight_min": -155.92869567871094,
      "activations/layer6_attention_weight_max": 134.34698486328125,
      "activations/layer6_attention_weight_min": -121.51364135742188,
      "activations/layer7_attention_weight_max": 107.5551986694336,
      "activations/layer7_attention_weight_min": -93.74372863769531,
      "activations/layer8_attention_weight_max": 82.79801940917969,
      "activations/layer8_attention_weight_min": -66.82096862792969,
      "activations/layer9_attention_weight_max": 79.9529037475586,
      "activations/layer9_attention_weight_min": -70.00838470458984,
      "epoch": 0.26,
      "learning_rate": 0.00014983674242424242,
      "loss": 3.5072,
      "step": 4450
    },
    {
      "activations/layer0_attention_weight_max": 12.675057411193848,
      "activations/layer0_attention_weight_min": -9.082223892211914,
      "activations/layer10_attention_weight_max": 58.97068405151367,
      "activations/layer10_attention_weight_min": -51.67759323120117,
      "activations/layer11_attention_weight_max": 32.73347854614258,
      "activations/layer11_attention_weight_min": -26.018028259277344,
      "activations/layer12_attention_weight_max": 23.666202545166016,
      "activations/layer12_attention_weight_min": -22.401697158813477,
      "activations/layer13_attention_weight_max": 29.859619140625,
      "activations/layer13_attention_weight_min": -21.252880096435547,
      "activations/layer14_attention_weight_max": 32.57793426513672,
      "activations/layer14_attention_weight_min": -16.562458038330078,
      "activations/layer15_attention_weight_max": 26.911142349243164,
      "activations/layer15_attention_weight_min": -14.203825950622559,
      "activations/layer16_attention_weight_max": 23.470062255859375,
      "activations/layer16_attention_weight_min": -12.374408721923828,
      "activations/layer17_attention_weight_max": 26.328645706176758,
      "activations/layer17_attention_weight_min": -19.846134185791016,
      "activations/layer18_attention_weight_max": 29.375333786010742,
      "activations/layer18_attention_weight_min": -16.272397994995117,
      "activations/layer19_attention_weight_max": 21.767972946166992,
      "activations/layer19_attention_weight_min": -12.214611053466797,
      "activations/layer1_attention_weight_max": 17.956478118896484,
      "activations/layer1_attention_weight_min": -11.214990615844727,
      "activations/layer20_attention_weight_max": 20.844846725463867,
      "activations/layer20_attention_weight_min": -11.768166542053223,
      "activations/layer21_attention_weight_max": 20.973134994506836,
      "activations/layer21_attention_weight_min": -14.818942070007324,
      "activations/layer22_attention_weight_max": 17.719219207763672,
      "activations/layer22_attention_weight_min": -14.035696983337402,
      "activations/layer23_attention_weight_max": 14.53877067565918,
      "activations/layer23_attention_weight_min": -16.838275909423828,
      "activations/layer2_attention_weight_max": 26.947431564331055,
      "activations/layer2_attention_weight_min": -20.229576110839844,
      "activations/layer3_attention_weight_max": 58.630271911621094,
      "activations/layer3_attention_weight_min": -45.85211944580078,
      "activations/layer4_attention_weight_max": 92.6363525390625,
      "activations/layer4_attention_weight_min": -80.76580047607422,
      "activations/layer5_attention_weight_max": 178.2867889404297,
      "activations/layer5_attention_weight_min": -153.1334686279297,
      "activations/layer6_attention_weight_max": 136.38491821289062,
      "activations/layer6_attention_weight_min": -119.40091705322266,
      "activations/layer7_attention_weight_max": 104.86394500732422,
      "activations/layer7_attention_weight_min": -89.05716705322266,
      "activations/layer8_attention_weight_max": 79.30181884765625,
      "activations/layer8_attention_weight_min": -67.4845962524414,
      "activations/layer9_attention_weight_max": 77.82158660888672,
      "activations/layer9_attention_weight_min": -64.91011047363281,
      "epoch": 0.26,
      "learning_rate": 0.000149817803030303,
      "loss": 3.4985,
      "step": 4500
    },
    {
      "activations/layer0_attention_weight_max": 14.839950561523438,
      "activations/layer0_attention_weight_min": -9.400452613830566,
      "activations/layer10_attention_weight_max": 80.91511535644531,
      "activations/layer10_attention_weight_min": -62.90251159667969,
      "activations/layer11_attention_weight_max": 37.63102722167969,
      "activations/layer11_attention_weight_min": -24.83371353149414,
      "activations/layer12_attention_weight_max": 21.725074768066406,
      "activations/layer12_attention_weight_min": -25.944726943969727,
      "activations/layer13_attention_weight_max": 32.978084564208984,
      "activations/layer13_attention_weight_min": -21.56584930419922,
      "activations/layer14_attention_weight_max": 25.986083984375,
      "activations/layer14_attention_weight_min": -17.0704345703125,
      "activations/layer15_attention_weight_max": 23.29194450378418,
      "activations/layer15_attention_weight_min": -15.772809982299805,
      "activations/layer16_attention_weight_max": 24.784997940063477,
      "activations/layer16_attention_weight_min": -13.120928764343262,
      "activations/layer17_attention_weight_max": 19.995115280151367,
      "activations/layer17_attention_weight_min": -19.777124404907227,
      "activations/layer18_attention_weight_max": 27.621543884277344,
      "activations/layer18_attention_weight_min": -18.977434158325195,
      "activations/layer19_attention_weight_max": 21.5638370513916,
      "activations/layer19_attention_weight_min": -12.537760734558105,
      "activations/layer1_attention_weight_max": 17.593454360961914,
      "activations/layer1_attention_weight_min": -11.229025840759277,
      "activations/layer20_attention_weight_max": 19.27959442138672,
      "activations/layer20_attention_weight_min": -13.772204399108887,
      "activations/layer21_attention_weight_max": 20.885787963867188,
      "activations/layer21_attention_weight_min": -13.928011894226074,
      "activations/layer22_attention_weight_max": 16.783588409423828,
      "activations/layer22_attention_weight_min": -12.415035247802734,
      "activations/layer23_attention_weight_max": 12.404449462890625,
      "activations/layer23_attention_weight_min": -11.507186889648438,
      "activations/layer2_attention_weight_max": 28.00467300415039,
      "activations/layer2_attention_weight_min": -19.79474449157715,
      "activations/layer3_attention_weight_max": 61.22114562988281,
      "activations/layer3_attention_weight_min": -44.93788528442383,
      "activations/layer4_attention_weight_max": 92.16566467285156,
      "activations/layer4_attention_weight_min": -79.6989974975586,
      "activations/layer5_attention_weight_max": 181.06051635742188,
      "activations/layer5_attention_weight_min": -152.569091796875,
      "activations/layer6_attention_weight_max": 135.72540283203125,
      "activations/layer6_attention_weight_min": -119.69721221923828,
      "activations/layer7_attention_weight_max": 104.04470825195312,
      "activations/layer7_attention_weight_min": -92.764404296875,
      "activations/layer8_attention_weight_max": 83.41788482666016,
      "activations/layer8_attention_weight_min": -68.71995544433594,
      "activations/layer9_attention_weight_max": 92.6126937866211,
      "activations/layer9_attention_weight_min": -73.32410430908203,
      "epoch": 0.26,
      "learning_rate": 0.00014979886363636363,
      "loss": 3.4948,
      "step": 4550
    },
    {
      "activations/layer0_attention_weight_max": 12.673100471496582,
      "activations/layer0_attention_weight_min": -8.90933609008789,
      "activations/layer10_attention_weight_max": 73.64717864990234,
      "activations/layer10_attention_weight_min": -64.410888671875,
      "activations/layer11_attention_weight_max": 35.465457916259766,
      "activations/layer11_attention_weight_min": -23.5867977142334,
      "activations/layer12_attention_weight_max": 23.251480102539062,
      "activations/layer12_attention_weight_min": -21.029733657836914,
      "activations/layer13_attention_weight_max": 27.451770782470703,
      "activations/layer13_attention_weight_min": -25.173784255981445,
      "activations/layer14_attention_weight_max": 28.991069793701172,
      "activations/layer14_attention_weight_min": -16.353784561157227,
      "activations/layer15_attention_weight_max": 24.59739875793457,
      "activations/layer15_attention_weight_min": -13.627066612243652,
      "activations/layer16_attention_weight_max": 24.60844612121582,
      "activations/layer16_attention_weight_min": -14.578635215759277,
      "activations/layer17_attention_weight_max": 24.583032608032227,
      "activations/layer17_attention_weight_min": -16.65812110900879,
      "activations/layer18_attention_weight_max": 30.007497787475586,
      "activations/layer18_attention_weight_min": -15.43924617767334,
      "activations/layer19_attention_weight_max": 26.154998779296875,
      "activations/layer19_attention_weight_min": -12.34024715423584,
      "activations/layer1_attention_weight_max": 17.987024307250977,
      "activations/layer1_attention_weight_min": -10.544864654541016,
      "activations/layer20_attention_weight_max": 24.31801414489746,
      "activations/layer20_attention_weight_min": -11.974437713623047,
      "activations/layer21_attention_weight_max": 27.149978637695312,
      "activations/layer21_attention_weight_min": -15.125338554382324,
      "activations/layer22_attention_weight_max": 21.604341506958008,
      "activations/layer22_attention_weight_min": -12.024491310119629,
      "activations/layer23_attention_weight_max": 15.446870803833008,
      "activations/layer23_attention_weight_min": -13.112890243530273,
      "activations/layer2_attention_weight_max": 26.38923454284668,
      "activations/layer2_attention_weight_min": -19.373315811157227,
      "activations/layer3_attention_weight_max": 59.64888000488281,
      "activations/layer3_attention_weight_min": -45.678871154785156,
      "activations/layer4_attention_weight_max": 89.42333221435547,
      "activations/layer4_attention_weight_min": -78.69071197509766,
      "activations/layer5_attention_weight_max": 169.88636779785156,
      "activations/layer5_attention_weight_min": -151.55059814453125,
      "activations/layer6_attention_weight_max": 127.61697387695312,
      "activations/layer6_attention_weight_min": -115.81195831298828,
      "activations/layer7_attention_weight_max": 105.11682891845703,
      "activations/layer7_attention_weight_min": -93.89918518066406,
      "activations/layer8_attention_weight_max": 78.7751235961914,
      "activations/layer8_attention_weight_min": -72.4645767211914,
      "activations/layer9_attention_weight_max": 78.01472473144531,
      "activations/layer9_attention_weight_min": -68.18367767333984,
      "epoch": 0.27,
      "learning_rate": 0.00014977992424242424,
      "loss": 3.4896,
      "step": 4600
    },
    {
      "activations/layer0_attention_weight_max": 12.726202964782715,
      "activations/layer0_attention_weight_min": -9.019067764282227,
      "activations/layer10_attention_weight_max": 68.70946502685547,
      "activations/layer10_attention_weight_min": -55.25770950317383,
      "activations/layer11_attention_weight_max": 34.06050109863281,
      "activations/layer11_attention_weight_min": -24.989843368530273,
      "activations/layer12_attention_weight_max": 21.77338218688965,
      "activations/layer12_attention_weight_min": -21.263479232788086,
      "activations/layer13_attention_weight_max": 32.6903076171875,
      "activations/layer13_attention_weight_min": -19.50891876220703,
      "activations/layer14_attention_weight_max": 32.71598434448242,
      "activations/layer14_attention_weight_min": -15.660825729370117,
      "activations/layer15_attention_weight_max": 26.934606552124023,
      "activations/layer15_attention_weight_min": -16.30821418762207,
      "activations/layer16_attention_weight_max": 29.392709732055664,
      "activations/layer16_attention_weight_min": -14.040009498596191,
      "activations/layer17_attention_weight_max": 22.919111251831055,
      "activations/layer17_attention_weight_min": -17.695261001586914,
      "activations/layer18_attention_weight_max": 29.12324333190918,
      "activations/layer18_attention_weight_min": -15.754526138305664,
      "activations/layer19_attention_weight_max": 24.39807891845703,
      "activations/layer19_attention_weight_min": -11.012565612792969,
      "activations/layer1_attention_weight_max": 18.2110595703125,
      "activations/layer1_attention_weight_min": -12.301359176635742,
      "activations/layer20_attention_weight_max": 24.44472885131836,
      "activations/layer20_attention_weight_min": -12.088425636291504,
      "activations/layer21_attention_weight_max": 24.05791664123535,
      "activations/layer21_attention_weight_min": -15.476499557495117,
      "activations/layer22_attention_weight_max": 19.884389877319336,
      "activations/layer22_attention_weight_min": -11.861191749572754,
      "activations/layer23_attention_weight_max": 17.115665435791016,
      "activations/layer23_attention_weight_min": -14.744773864746094,
      "activations/layer2_attention_weight_max": 27.123722076416016,
      "activations/layer2_attention_weight_min": -19.607223510742188,
      "activations/layer3_attention_weight_max": 61.382957458496094,
      "activations/layer3_attention_weight_min": -45.08939743041992,
      "activations/layer4_attention_weight_max": 94.44770050048828,
      "activations/layer4_attention_weight_min": -79.4115982055664,
      "activations/layer5_attention_weight_max": 187.22265625,
      "activations/layer5_attention_weight_min": -152.12123107910156,
      "activations/layer6_attention_weight_max": 132.3082275390625,
      "activations/layer6_attention_weight_min": -117.2869873046875,
      "activations/layer7_attention_weight_max": 111.79042053222656,
      "activations/layer7_attention_weight_min": -94.96739196777344,
      "activations/layer8_attention_weight_max": 82.46868896484375,
      "activations/layer8_attention_weight_min": -67.9292221069336,
      "activations/layer9_attention_weight_max": 81.19906616210938,
      "activations/layer9_attention_weight_min": -67.67659759521484,
      "epoch": 0.27,
      "learning_rate": 0.00014976098484848483,
      "loss": 3.4891,
      "step": 4650
    },
    {
      "activations/layer0_attention_weight_max": 12.797769546508789,
      "activations/layer0_attention_weight_min": -10.24070930480957,
      "activations/layer10_attention_weight_max": 61.31517028808594,
      "activations/layer10_attention_weight_min": -58.332275390625,
      "activations/layer11_attention_weight_max": 29.04322624206543,
      "activations/layer11_attention_weight_min": -28.967939376831055,
      "activations/layer12_attention_weight_max": 20.777719497680664,
      "activations/layer12_attention_weight_min": -25.391820907592773,
      "activations/layer13_attention_weight_max": 28.132638931274414,
      "activations/layer13_attention_weight_min": -22.50246238708496,
      "activations/layer14_attention_weight_max": 32.690147399902344,
      "activations/layer14_attention_weight_min": -15.562431335449219,
      "activations/layer15_attention_weight_max": 22.603485107421875,
      "activations/layer15_attention_weight_min": -13.726103782653809,
      "activations/layer16_attention_weight_max": 22.019559860229492,
      "activations/layer16_attention_weight_min": -12.071788787841797,
      "activations/layer17_attention_weight_max": 24.85610580444336,
      "activations/layer17_attention_weight_min": -16.35250473022461,
      "activations/layer18_attention_weight_max": 30.031837463378906,
      "activations/layer18_attention_weight_min": -18.01894760131836,
      "activations/layer19_attention_weight_max": 19.39983367919922,
      "activations/layer19_attention_weight_min": -10.958706855773926,
      "activations/layer1_attention_weight_max": 18.847036361694336,
      "activations/layer1_attention_weight_min": -10.327107429504395,
      "activations/layer20_attention_weight_max": 17.908201217651367,
      "activations/layer20_attention_weight_min": -11.67480754852295,
      "activations/layer21_attention_weight_max": 20.58492660522461,
      "activations/layer21_attention_weight_min": -14.72750473022461,
      "activations/layer22_attention_weight_max": 15.41213321685791,
      "activations/layer22_attention_weight_min": -11.715564727783203,
      "activations/layer23_attention_weight_max": 16.17947769165039,
      "activations/layer23_attention_weight_min": -10.474100112915039,
      "activations/layer2_attention_weight_max": 26.558303833007812,
      "activations/layer2_attention_weight_min": -19.082216262817383,
      "activations/layer3_attention_weight_max": 59.54051971435547,
      "activations/layer3_attention_weight_min": -45.52118682861328,
      "activations/layer4_attention_weight_max": 93.34310150146484,
      "activations/layer4_attention_weight_min": -79.99669647216797,
      "activations/layer5_attention_weight_max": 175.97097778320312,
      "activations/layer5_attention_weight_min": -152.45950317382812,
      "activations/layer6_attention_weight_max": 137.16343688964844,
      "activations/layer6_attention_weight_min": -112.97563171386719,
      "activations/layer7_attention_weight_max": 102.38379669189453,
      "activations/layer7_attention_weight_min": -91.6835708618164,
      "activations/layer8_attention_weight_max": 72.4104995727539,
      "activations/layer8_attention_weight_min": -66.95095825195312,
      "activations/layer9_attention_weight_max": 75.42308044433594,
      "activations/layer9_attention_weight_min": -65.1363754272461,
      "epoch": 0.27,
      "learning_rate": 0.00014974204545454545,
      "loss": 3.4769,
      "step": 4700
    },
    {
      "activations/layer0_attention_weight_max": 15.41139030456543,
      "activations/layer0_attention_weight_min": -10.352238655090332,
      "activations/layer10_attention_weight_max": 69.43521881103516,
      "activations/layer10_attention_weight_min": -60.48286819458008,
      "activations/layer11_attention_weight_max": 32.946205139160156,
      "activations/layer11_attention_weight_min": -24.97657585144043,
      "activations/layer12_attention_weight_max": 21.63401222229004,
      "activations/layer12_attention_weight_min": -22.39495086669922,
      "activations/layer13_attention_weight_max": 30.71006965637207,
      "activations/layer13_attention_weight_min": -24.727643966674805,
      "activations/layer14_attention_weight_max": 28.867074966430664,
      "activations/layer14_attention_weight_min": -16.755247116088867,
      "activations/layer15_attention_weight_max": 23.822301864624023,
      "activations/layer15_attention_weight_min": -15.702807426452637,
      "activations/layer16_attention_weight_max": 24.44390869140625,
      "activations/layer16_attention_weight_min": -14.064407348632812,
      "activations/layer17_attention_weight_max": 24.066883087158203,
      "activations/layer17_attention_weight_min": -17.997297286987305,
      "activations/layer18_attention_weight_max": 26.509389877319336,
      "activations/layer18_attention_weight_min": -18.914716720581055,
      "activations/layer19_attention_weight_max": 26.5944766998291,
      "activations/layer19_attention_weight_min": -12.92927074432373,
      "activations/layer1_attention_weight_max": 19.112625122070312,
      "activations/layer1_attention_weight_min": -11.149188995361328,
      "activations/layer20_attention_weight_max": 25.02699851989746,
      "activations/layer20_attention_weight_min": -12.058565139770508,
      "activations/layer21_attention_weight_max": 27.92318344116211,
      "activations/layer21_attention_weight_min": -13.396224021911621,
      "activations/layer22_attention_weight_max": 22.849458694458008,
      "activations/layer22_attention_weight_min": -12.160465240478516,
      "activations/layer23_attention_weight_max": 16.324392318725586,
      "activations/layer23_attention_weight_min": -12.994112968444824,
      "activations/layer2_attention_weight_max": 27.078460693359375,
      "activations/layer2_attention_weight_min": -19.22960662841797,
      "activations/layer3_attention_weight_max": 59.35551834106445,
      "activations/layer3_attention_weight_min": -44.713294982910156,
      "activations/layer4_attention_weight_max": 90.78614807128906,
      "activations/layer4_attention_weight_min": -77.22554779052734,
      "activations/layer5_attention_weight_max": 169.26522827148438,
      "activations/layer5_attention_weight_min": -149.60946655273438,
      "activations/layer6_attention_weight_max": 120.6500473022461,
      "activations/layer6_attention_weight_min": -116.9950180053711,
      "activations/layer7_attention_weight_max": 99.9537582397461,
      "activations/layer7_attention_weight_min": -91.60018920898438,
      "activations/layer8_attention_weight_max": 80.56055450439453,
      "activations/layer8_attention_weight_min": -69.0615005493164,
      "activations/layer9_attention_weight_max": 76.2892837524414,
      "activations/layer9_attention_weight_min": -72.3727798461914,
      "epoch": 0.28,
      "learning_rate": 0.00014972310606060606,
      "loss": 3.4786,
      "step": 4750
    },
    {
      "activations/layer0_attention_weight_max": 14.746245384216309,
      "activations/layer0_attention_weight_min": -9.259185791015625,
      "activations/layer10_attention_weight_max": 63.939605712890625,
      "activations/layer10_attention_weight_min": -58.866756439208984,
      "activations/layer11_attention_weight_max": 29.799049377441406,
      "activations/layer11_attention_weight_min": -23.827804565429688,
      "activations/layer12_attention_weight_max": 21.551183700561523,
      "activations/layer12_attention_weight_min": -26.65243148803711,
      "activations/layer13_attention_weight_max": 32.861968994140625,
      "activations/layer13_attention_weight_min": -23.542631149291992,
      "activations/layer14_attention_weight_max": 28.277769088745117,
      "activations/layer14_attention_weight_min": -18.659236907958984,
      "activations/layer15_attention_weight_max": 24.072975158691406,
      "activations/layer15_attention_weight_min": -13.082907676696777,
      "activations/layer16_attention_weight_max": 23.15631866455078,
      "activations/layer16_attention_weight_min": -14.297805786132812,
      "activations/layer17_attention_weight_max": 25.864723205566406,
      "activations/layer17_attention_weight_min": -17.264352798461914,
      "activations/layer18_attention_weight_max": 30.955142974853516,
      "activations/layer18_attention_weight_min": -15.780421257019043,
      "activations/layer19_attention_weight_max": 21.62872886657715,
      "activations/layer19_attention_weight_min": -12.41757869720459,
      "activations/layer1_attention_weight_max": 18.50119400024414,
      "activations/layer1_attention_weight_min": -11.309021949768066,
      "activations/layer20_attention_weight_max": 22.22356605529785,
      "activations/layer20_attention_weight_min": -14.417649269104004,
      "activations/layer21_attention_weight_max": 26.055166244506836,
      "activations/layer21_attention_weight_min": -13.021312713623047,
      "activations/layer22_attention_weight_max": 22.63813018798828,
      "activations/layer22_attention_weight_min": -12.4440336227417,
      "activations/layer23_attention_weight_max": 16.64989471435547,
      "activations/layer23_attention_weight_min": -13.402331352233887,
      "activations/layer2_attention_weight_max": 28.00458335876465,
      "activations/layer2_attention_weight_min": -19.063749313354492,
      "activations/layer3_attention_weight_max": 60.57339096069336,
      "activations/layer3_attention_weight_min": -44.11891174316406,
      "activations/layer4_attention_weight_max": 89.93648529052734,
      "activations/layer4_attention_weight_min": -79.08647155761719,
      "activations/layer5_attention_weight_max": 163.47854614257812,
      "activations/layer5_attention_weight_min": -147.42825317382812,
      "activations/layer6_attention_weight_max": 123.39901733398438,
      "activations/layer6_attention_weight_min": -115.68289184570312,
      "activations/layer7_attention_weight_max": 99.5484848022461,
      "activations/layer7_attention_weight_min": -86.58488464355469,
      "activations/layer8_attention_weight_max": 74.42032623291016,
      "activations/layer8_attention_weight_min": -66.88020324707031,
      "activations/layer9_attention_weight_max": 74.67073822021484,
      "activations/layer9_attention_weight_min": -68.24884796142578,
      "epoch": 0.28,
      "learning_rate": 0.00014970416666666665,
      "loss": 3.4776,
      "step": 4800
    },
    {
      "activations/layer0_attention_weight_max": 13.613337516784668,
      "activations/layer0_attention_weight_min": -10.220705032348633,
      "activations/layer10_attention_weight_max": 72.3848876953125,
      "activations/layer10_attention_weight_min": -59.42305374145508,
      "activations/layer11_attention_weight_max": 33.30991744995117,
      "activations/layer11_attention_weight_min": -28.656497955322266,
      "activations/layer12_attention_weight_max": 22.905906677246094,
      "activations/layer12_attention_weight_min": -19.29861068725586,
      "activations/layer13_attention_weight_max": 28.91402816772461,
      "activations/layer13_attention_weight_min": -20.79814910888672,
      "activations/layer14_attention_weight_max": 24.114295959472656,
      "activations/layer14_attention_weight_min": -16.598920822143555,
      "activations/layer15_attention_weight_max": 24.27800750732422,
      "activations/layer15_attention_weight_min": -15.329615592956543,
      "activations/layer16_attention_weight_max": 21.23053741455078,
      "activations/layer16_attention_weight_min": -13.518381118774414,
      "activations/layer17_attention_weight_max": 19.497211456298828,
      "activations/layer17_attention_weight_min": -15.666375160217285,
      "activations/layer18_attention_weight_max": 20.741134643554688,
      "activations/layer18_attention_weight_min": -13.321332931518555,
      "activations/layer19_attention_weight_max": 18.956554412841797,
      "activations/layer19_attention_weight_min": -11.856675148010254,
      "activations/layer1_attention_weight_max": 18.17837142944336,
      "activations/layer1_attention_weight_min": -11.427931785583496,
      "activations/layer20_attention_weight_max": 17.779857635498047,
      "activations/layer20_attention_weight_min": -11.388968467712402,
      "activations/layer21_attention_weight_max": 20.751800537109375,
      "activations/layer21_attention_weight_min": -14.89319896697998,
      "activations/layer22_attention_weight_max": 19.06962776184082,
      "activations/layer22_attention_weight_min": -13.975337982177734,
      "activations/layer23_attention_weight_max": 14.849017143249512,
      "activations/layer23_attention_weight_min": -16.286970138549805,
      "activations/layer2_attention_weight_max": 28.643644332885742,
      "activations/layer2_attention_weight_min": -19.905019760131836,
      "activations/layer3_attention_weight_max": 59.984352111816406,
      "activations/layer3_attention_weight_min": -45.96456527709961,
      "activations/layer4_attention_weight_max": 91.20199584960938,
      "activations/layer4_attention_weight_min": -80.0624008178711,
      "activations/layer5_attention_weight_max": 171.86436462402344,
      "activations/layer5_attention_weight_min": -147.0898895263672,
      "activations/layer6_attention_weight_max": 125.3606948852539,
      "activations/layer6_attention_weight_min": -116.01152038574219,
      "activations/layer7_attention_weight_max": 99.86400604248047,
      "activations/layer7_attention_weight_min": -89.08147430419922,
      "activations/layer8_attention_weight_max": 80.32099914550781,
      "activations/layer8_attention_weight_min": -66.77552032470703,
      "activations/layer9_attention_weight_max": 74.16655731201172,
      "activations/layer9_attention_weight_min": -68.85820770263672,
      "epoch": 0.28,
      "learning_rate": 0.00014968522727272726,
      "loss": 3.4914,
      "step": 4850
    },
    {
      "activations/layer0_attention_weight_max": 14.147588729858398,
      "activations/layer0_attention_weight_min": -10.469592094421387,
      "activations/layer10_attention_weight_max": 62.97720718383789,
      "activations/layer10_attention_weight_min": -55.77484130859375,
      "activations/layer11_attention_weight_max": 32.009857177734375,
      "activations/layer11_attention_weight_min": -27.5208797454834,
      "activations/layer12_attention_weight_max": 23.682132720947266,
      "activations/layer12_attention_weight_min": -21.218278884887695,
      "activations/layer13_attention_weight_max": 29.062297821044922,
      "activations/layer13_attention_weight_min": -27.48699378967285,
      "activations/layer14_attention_weight_max": 25.875141143798828,
      "activations/layer14_attention_weight_min": -17.951602935791016,
      "activations/layer15_attention_weight_max": 25.42155647277832,
      "activations/layer15_attention_weight_min": -14.111952781677246,
      "activations/layer16_attention_weight_max": 25.365978240966797,
      "activations/layer16_attention_weight_min": -13.10130500793457,
      "activations/layer17_attention_weight_max": 21.53021812438965,
      "activations/layer17_attention_weight_min": -15.900853157043457,
      "activations/layer18_attention_weight_max": 26.292652130126953,
      "activations/layer18_attention_weight_min": -15.457282066345215,
      "activations/layer19_attention_weight_max": 20.87356185913086,
      "activations/layer19_attention_weight_min": -12.270946502685547,
      "activations/layer1_attention_weight_max": 19.25291633605957,
      "activations/layer1_attention_weight_min": -11.708532333374023,
      "activations/layer20_attention_weight_max": 23.54163360595703,
      "activations/layer20_attention_weight_min": -13.624489784240723,
      "activations/layer21_attention_weight_max": 25.707853317260742,
      "activations/layer21_attention_weight_min": -14.338793754577637,
      "activations/layer22_attention_weight_max": 23.4486026763916,
      "activations/layer22_attention_weight_min": -13.836217880249023,
      "activations/layer23_attention_weight_max": 17.3349552154541,
      "activations/layer23_attention_weight_min": -11.284441947937012,
      "activations/layer2_attention_weight_max": 24.517940521240234,
      "activations/layer2_attention_weight_min": -19.684436798095703,
      "activations/layer3_attention_weight_max": 54.71728515625,
      "activations/layer3_attention_weight_min": -43.177886962890625,
      "activations/layer4_attention_weight_max": 88.04586029052734,
      "activations/layer4_attention_weight_min": -80.50249481201172,
      "activations/layer5_attention_weight_max": 158.9105224609375,
      "activations/layer5_attention_weight_min": -148.32858276367188,
      "activations/layer6_attention_weight_max": 117.9591293334961,
      "activations/layer6_attention_weight_min": -114.31033325195312,
      "activations/layer7_attention_weight_max": 87.42630004882812,
      "activations/layer7_attention_weight_min": -86.80109405517578,
      "activations/layer8_attention_weight_max": 67.3729019165039,
      "activations/layer8_attention_weight_min": -66.62596130371094,
      "activations/layer9_attention_weight_max": 70.62464904785156,
      "activations/layer9_attention_weight_min": -68.35912322998047,
      "epoch": 0.28,
      "learning_rate": 0.00014966628787878788,
      "loss": 3.4736,
      "step": 4900
    },
    {
      "activations/layer0_attention_weight_max": 13.872538566589355,
      "activations/layer0_attention_weight_min": -9.84913444519043,
      "activations/layer10_attention_weight_max": 68.4332504272461,
      "activations/layer10_attention_weight_min": -61.06388473510742,
      "activations/layer11_attention_weight_max": 30.777210235595703,
      "activations/layer11_attention_weight_min": -26.286893844604492,
      "activations/layer12_attention_weight_max": 25.025104522705078,
      "activations/layer12_attention_weight_min": -22.59558868408203,
      "activations/layer13_attention_weight_max": 31.504295349121094,
      "activations/layer13_attention_weight_min": -23.11897850036621,
      "activations/layer14_attention_weight_max": 26.303972244262695,
      "activations/layer14_attention_weight_min": -17.124021530151367,
      "activations/layer15_attention_weight_max": 24.725481033325195,
      "activations/layer15_attention_weight_min": -15.101962089538574,
      "activations/layer16_attention_weight_max": 23.636043548583984,
      "activations/layer16_attention_weight_min": -14.812904357910156,
      "activations/layer17_attention_weight_max": 28.23623275756836,
      "activations/layer17_attention_weight_min": -21.55851173400879,
      "activations/layer18_attention_weight_max": 30.501449584960938,
      "activations/layer18_attention_weight_min": -17.27449607849121,
      "activations/layer19_attention_weight_max": 24.94809913635254,
      "activations/layer19_attention_weight_min": -12.232051849365234,
      "activations/layer1_attention_weight_max": 19.547800064086914,
      "activations/layer1_attention_weight_min": -11.399961471557617,
      "activations/layer20_attention_weight_max": 19.559328079223633,
      "activations/layer20_attention_weight_min": -14.619243621826172,
      "activations/layer21_attention_weight_max": 25.70269012451172,
      "activations/layer21_attention_weight_min": -14.876558303833008,
      "activations/layer22_attention_weight_max": 21.304325103759766,
      "activations/layer22_attention_weight_min": -13.6679048538208,
      "activations/layer23_attention_weight_max": 16.527320861816406,
      "activations/layer23_attention_weight_min": -15.348867416381836,
      "activations/layer2_attention_weight_max": 25.649688720703125,
      "activations/layer2_attention_weight_min": -19.08643341064453,
      "activations/layer3_attention_weight_max": 58.46590805053711,
      "activations/layer3_attention_weight_min": -44.74219512939453,
      "activations/layer4_attention_weight_max": 90.0353012084961,
      "activations/layer4_attention_weight_min": -77.63018035888672,
      "activations/layer5_attention_weight_max": 169.82098388671875,
      "activations/layer5_attention_weight_min": -146.99020385742188,
      "activations/layer6_attention_weight_max": 126.1641616821289,
      "activations/layer6_attention_weight_min": -112.0097427368164,
      "activations/layer7_attention_weight_max": 101.3750228881836,
      "activations/layer7_attention_weight_min": -84.6358413696289,
      "activations/layer8_attention_weight_max": 83.69284057617188,
      "activations/layer8_attention_weight_min": -66.39743041992188,
      "activations/layer9_attention_weight_max": 77.65592193603516,
      "activations/layer9_attention_weight_min": -69.34937286376953,
      "epoch": 0.29,
      "learning_rate": 0.00014964734848484847,
      "loss": 3.4619,
      "step": 4950
    },
    {
      "activations/layer0_attention_weight_max": 13.475381851196289,
      "activations/layer0_attention_weight_min": -9.685961723327637,
      "activations/layer10_attention_weight_max": 60.29509735107422,
      "activations/layer10_attention_weight_min": -60.40878677368164,
      "activations/layer11_attention_weight_max": 30.958364486694336,
      "activations/layer11_attention_weight_min": -24.508153915405273,
      "activations/layer12_attention_weight_max": 23.219966888427734,
      "activations/layer12_attention_weight_min": -23.767303466796875,
      "activations/layer13_attention_weight_max": 29.34227180480957,
      "activations/layer13_attention_weight_min": -22.199804306030273,
      "activations/layer14_attention_weight_max": 27.280363082885742,
      "activations/layer14_attention_weight_min": -16.773311614990234,
      "activations/layer15_attention_weight_max": 25.126331329345703,
      "activations/layer15_attention_weight_min": -13.496687889099121,
      "activations/layer16_attention_weight_max": 23.38597297668457,
      "activations/layer16_attention_weight_min": -14.429756164550781,
      "activations/layer17_attention_weight_max": 33.959407806396484,
      "activations/layer17_attention_weight_min": -17.8591365814209,
      "activations/layer18_attention_weight_max": 26.87301254272461,
      "activations/layer18_attention_weight_min": -17.00429344177246,
      "activations/layer19_attention_weight_max": 24.40248680114746,
      "activations/layer19_attention_weight_min": -14.037510871887207,
      "activations/layer1_attention_weight_max": 18.612062454223633,
      "activations/layer1_attention_weight_min": -11.929728507995605,
      "activations/layer20_attention_weight_max": 23.305622100830078,
      "activations/layer20_attention_weight_min": -13.389863967895508,
      "activations/layer21_attention_weight_max": 25.2679443359375,
      "activations/layer21_attention_weight_min": -15.29085636138916,
      "activations/layer22_attention_weight_max": 22.087757110595703,
      "activations/layer22_attention_weight_min": -13.136152267456055,
      "activations/layer23_attention_weight_max": 16.21639633178711,
      "activations/layer23_attention_weight_min": -12.223994255065918,
      "activations/layer2_attention_weight_max": 25.967464447021484,
      "activations/layer2_attention_weight_min": -18.679704666137695,
      "activations/layer3_attention_weight_max": 55.606483459472656,
      "activations/layer3_attention_weight_min": -44.65001678466797,
      "activations/layer4_attention_weight_max": 86.13317108154297,
      "activations/layer4_attention_weight_min": -78.70771789550781,
      "activations/layer5_attention_weight_max": 156.18545532226562,
      "activations/layer5_attention_weight_min": -148.9540557861328,
      "activations/layer6_attention_weight_max": 116.2686996459961,
      "activations/layer6_attention_weight_min": -109.78268432617188,
      "activations/layer7_attention_weight_max": 89.07669067382812,
      "activations/layer7_attention_weight_min": -86.19142150878906,
      "activations/layer8_attention_weight_max": 68.15126037597656,
      "activations/layer8_attention_weight_min": -65.72882843017578,
      "activations/layer9_attention_weight_max": 69.87399291992188,
      "activations/layer9_attention_weight_min": -64.24064636230469,
      "epoch": 0.29,
      "learning_rate": 0.00014962840909090908,
      "loss": 3.4486,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_loss": 3.404296875,
      "eval_runtime": 8.2052,
      "eval_samples_per_second": 523.326,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_openwebtext_loss": 3.404296875,
      "eval_openwebtext_ppl": 30.093129052017023,
      "eval_openwebtext_runtime": 8.2052,
      "eval_openwebtext_samples_per_second": 523.326,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_wikitext_loss": 3.7578125,
      "eval_wikitext_ppl": 42.854578979512624,
      "eval_wikitext_runtime": 1.706,
      "eval_wikitext_samples_per_second": 267.297,
      "step": 5000
    },
    {
      "epoch": 0.29,
      "eval_lambada_loss": 4.3046875,
      "eval_lambada_ppl": 74.04607243930377,
      "eval_lambada_runtime": 9.2664,
      "eval_lambada_samples_per_second": 525.449,
      "step": 5000
    },
    {
      "activations/layer0_attention_weight_max": 13.118768692016602,
      "activations/layer0_attention_weight_min": -9.25602912902832,
      "activations/layer10_attention_weight_max": 59.852073669433594,
      "activations/layer10_attention_weight_min": -52.85899353027344,
      "activations/layer11_attention_weight_max": 30.44550132751465,
      "activations/layer11_attention_weight_min": -25.15597915649414,
      "activations/layer12_attention_weight_max": 20.743803024291992,
      "activations/layer12_attention_weight_min": -21.653614044189453,
      "activations/layer13_attention_weight_max": 27.84296417236328,
      "activations/layer13_attention_weight_min": -23.840757369995117,
      "activations/layer14_attention_weight_max": 24.833087921142578,
      "activations/layer14_attention_weight_min": -15.843894004821777,
      "activations/layer15_attention_weight_max": 24.543766021728516,
      "activations/layer15_attention_weight_min": -15.95582103729248,
      "activations/layer16_attention_weight_max": 21.114099502563477,
      "activations/layer16_attention_weight_min": -14.936368942260742,
      "activations/layer17_attention_weight_max": 35.36444091796875,
      "activations/layer17_attention_weight_min": -20.198291778564453,
      "activations/layer18_attention_weight_max": 28.903451919555664,
      "activations/layer18_attention_weight_min": -15.777470588684082,
      "activations/layer19_attention_weight_max": 23.089717864990234,
      "activations/layer19_attention_weight_min": -11.767582893371582,
      "activations/layer1_attention_weight_max": 18.810697555541992,
      "activations/layer1_attention_weight_min": -10.330531120300293,
      "activations/layer20_attention_weight_max": 20.45414161682129,
      "activations/layer20_attention_weight_min": -11.751299858093262,
      "activations/layer21_attention_weight_max": 26.764421463012695,
      "activations/layer21_attention_weight_min": -15.418661117553711,
      "activations/layer22_attention_weight_max": 27.96947479248047,
      "activations/layer22_attention_weight_min": -14.062013626098633,
      "activations/layer23_attention_weight_max": 17.745019912719727,
      "activations/layer23_attention_weight_min": -10.781317710876465,
      "activations/layer2_attention_weight_max": 24.94025421142578,
      "activations/layer2_attention_weight_min": -18.803794860839844,
      "activations/layer3_attention_weight_max": 55.59376525878906,
      "activations/layer3_attention_weight_min": -43.32410430908203,
      "activations/layer4_attention_weight_max": 87.95919036865234,
      "activations/layer4_attention_weight_min": -76.39488983154297,
      "activations/layer5_attention_weight_max": 159.05294799804688,
      "activations/layer5_attention_weight_min": -142.73516845703125,
      "activations/layer6_attention_weight_max": 116.87776184082031,
      "activations/layer6_attention_weight_min": -113.78133392333984,
      "activations/layer7_attention_weight_max": 91.65672302246094,
      "activations/layer7_attention_weight_min": -84.48514556884766,
      "activations/layer8_attention_weight_max": 71.92113494873047,
      "activations/layer8_attention_weight_min": -64.17839050292969,
      "activations/layer9_attention_weight_max": 69.05960083007812,
      "activations/layer9_attention_weight_min": -65.71328735351562,
      "epoch": 0.29,
      "learning_rate": 0.00014960984848484847,
      "loss": 3.4656,
      "step": 5050
    },
    {
      "activations/layer0_attention_weight_max": 14.243789672851562,
      "activations/layer0_attention_weight_min": -10.12498950958252,
      "activations/layer10_attention_weight_max": 63.23017501831055,
      "activations/layer10_attention_weight_min": -55.17836380004883,
      "activations/layer11_attention_weight_max": 30.41905975341797,
      "activations/layer11_attention_weight_min": -26.79574203491211,
      "activations/layer12_attention_weight_max": 20.14100456237793,
      "activations/layer12_attention_weight_min": -26.900344848632812,
      "activations/layer13_attention_weight_max": 32.613868713378906,
      "activations/layer13_attention_weight_min": -24.60880470275879,
      "activations/layer14_attention_weight_max": 27.043941497802734,
      "activations/layer14_attention_weight_min": -17.26974868774414,
      "activations/layer15_attention_weight_max": 27.415966033935547,
      "activations/layer15_attention_weight_min": -17.770265579223633,
      "activations/layer16_attention_weight_max": 26.46243667602539,
      "activations/layer16_attention_weight_min": -14.28713321685791,
      "activations/layer17_attention_weight_max": 23.076570510864258,
      "activations/layer17_attention_weight_min": -18.721601486206055,
      "activations/layer18_attention_weight_max": 24.99395179748535,
      "activations/layer18_attention_weight_min": -17.702835083007812,
      "activations/layer19_attention_weight_max": 26.555570602416992,
      "activations/layer19_attention_weight_min": -12.409895896911621,
      "activations/layer1_attention_weight_max": 18.6071720123291,
      "activations/layer1_attention_weight_min": -10.730320930480957,
      "activations/layer20_attention_weight_max": 21.952213287353516,
      "activations/layer20_attention_weight_min": -12.932204246520996,
      "activations/layer21_attention_weight_max": 22.648683547973633,
      "activations/layer21_attention_weight_min": -15.767303466796875,
      "activations/layer22_attention_weight_max": 22.586139678955078,
      "activations/layer22_attention_weight_min": -14.164658546447754,
      "activations/layer23_attention_weight_max": 17.958927154541016,
      "activations/layer23_attention_weight_min": -15.55570125579834,
      "activations/layer2_attention_weight_max": 26.736522674560547,
      "activations/layer2_attention_weight_min": -19.544815063476562,
      "activations/layer3_attention_weight_max": 56.38867950439453,
      "activations/layer3_attention_weight_min": -45.571353912353516,
      "activations/layer4_attention_weight_max": 85.23942565917969,
      "activations/layer4_attention_weight_min": -74.95207214355469,
      "activations/layer5_attention_weight_max": 164.57278442382812,
      "activations/layer5_attention_weight_min": -146.02505493164062,
      "activations/layer6_attention_weight_max": 123.23869323730469,
      "activations/layer6_attention_weight_min": -113.87313079833984,
      "activations/layer7_attention_weight_max": 95.30084228515625,
      "activations/layer7_attention_weight_min": -89.76470184326172,
      "activations/layer8_attention_weight_max": 72.01609802246094,
      "activations/layer8_attention_weight_min": -65.32422637939453,
      "activations/layer9_attention_weight_max": 69.17058563232422,
      "activations/layer9_attention_weight_min": -63.26984786987305,
      "epoch": 0.3,
      "learning_rate": 0.0001495909090909091,
      "loss": 3.4652,
      "step": 5100
    },
    {
      "activations/layer0_attention_weight_max": 13.35186767578125,
      "activations/layer0_attention_weight_min": -10.704834938049316,
      "activations/layer10_attention_weight_max": 66.57634735107422,
      "activations/layer10_attention_weight_min": -57.46884536743164,
      "activations/layer11_attention_weight_max": 33.086402893066406,
      "activations/layer11_attention_weight_min": -26.267375946044922,
      "activations/layer12_attention_weight_max": 22.571186065673828,
      "activations/layer12_attention_weight_min": -22.793006896972656,
      "activations/layer13_attention_weight_max": 27.09428596496582,
      "activations/layer13_attention_weight_min": -22.41614532470703,
      "activations/layer14_attention_weight_max": 24.996353149414062,
      "activations/layer14_attention_weight_min": -16.863065719604492,
      "activations/layer15_attention_weight_max": 23.086299896240234,
      "activations/layer15_attention_weight_min": -15.598344802856445,
      "activations/layer16_attention_weight_max": 24.77519989013672,
      "activations/layer16_attention_weight_min": -13.478275299072266,
      "activations/layer17_attention_weight_max": 28.46781349182129,
      "activations/layer17_attention_weight_min": -16.76723289489746,
      "activations/layer18_attention_weight_max": 31.684978485107422,
      "activations/layer18_attention_weight_min": -17.670122146606445,
      "activations/layer19_attention_weight_max": 24.0980167388916,
      "activations/layer19_attention_weight_min": -12.373326301574707,
      "activations/layer1_attention_weight_max": 16.761064529418945,
      "activations/layer1_attention_weight_min": -10.42496109008789,
      "activations/layer20_attention_weight_max": 25.694316864013672,
      "activations/layer20_attention_weight_min": -11.660240173339844,
      "activations/layer21_attention_weight_max": 33.402252197265625,
      "activations/layer21_attention_weight_min": -16.502687454223633,
      "activations/layer22_attention_weight_max": 27.740333557128906,
      "activations/layer22_attention_weight_min": -14.349312782287598,
      "activations/layer23_attention_weight_max": 17.389005661010742,
      "activations/layer23_attention_weight_min": -13.198633193969727,
      "activations/layer2_attention_weight_max": 25.35797882080078,
      "activations/layer2_attention_weight_min": -19.84848976135254,
      "activations/layer3_attention_weight_max": 59.869529724121094,
      "activations/layer3_attention_weight_min": -44.306190490722656,
      "activations/layer4_attention_weight_max": 88.44207763671875,
      "activations/layer4_attention_weight_min": -74.99533081054688,
      "activations/layer5_attention_weight_max": 163.72300720214844,
      "activations/layer5_attention_weight_min": -143.016845703125,
      "activations/layer6_attention_weight_max": 117.9384994506836,
      "activations/layer6_attention_weight_min": -107.55811309814453,
      "activations/layer7_attention_weight_max": 100.6624526977539,
      "activations/layer7_attention_weight_min": -91.79239654541016,
      "activations/layer8_attention_weight_max": 73.71427154541016,
      "activations/layer8_attention_weight_min": -69.62876892089844,
      "activations/layer9_attention_weight_max": 74.97950744628906,
      "activations/layer9_attention_weight_min": -72.1888198852539,
      "epoch": 0.3,
      "learning_rate": 0.00014957196969696968,
      "loss": 3.464,
      "step": 5150
    },
    {
      "activations/layer0_attention_weight_max": 12.757944107055664,
      "activations/layer0_attention_weight_min": -9.39710521697998,
      "activations/layer10_attention_weight_max": 62.005943298339844,
      "activations/layer10_attention_weight_min": -55.885406494140625,
      "activations/layer11_attention_weight_max": 32.379112243652344,
      "activations/layer11_attention_weight_min": -25.502126693725586,
      "activations/layer12_attention_weight_max": 20.743574142456055,
      "activations/layer12_attention_weight_min": -21.47910499572754,
      "activations/layer13_attention_weight_max": 30.99125099182129,
      "activations/layer13_attention_weight_min": -22.12654685974121,
      "activations/layer14_attention_weight_max": 29.101268768310547,
      "activations/layer14_attention_weight_min": -19.01613998413086,
      "activations/layer15_attention_weight_max": 22.7674617767334,
      "activations/layer15_attention_weight_min": -15.792043685913086,
      "activations/layer16_attention_weight_max": 21.47992706298828,
      "activations/layer16_attention_weight_min": -13.673774719238281,
      "activations/layer17_attention_weight_max": 21.33432388305664,
      "activations/layer17_attention_weight_min": -18.32554054260254,
      "activations/layer18_attention_weight_max": 32.05791091918945,
      "activations/layer18_attention_weight_min": -16.621578216552734,
      "activations/layer19_attention_weight_max": 19.217737197875977,
      "activations/layer19_attention_weight_min": -12.261544227600098,
      "activations/layer1_attention_weight_max": 19.241392135620117,
      "activations/layer1_attention_weight_min": -11.062342643737793,
      "activations/layer20_attention_weight_max": 17.39054298400879,
      "activations/layer20_attention_weight_min": -11.624631881713867,
      "activations/layer21_attention_weight_max": 19.272354125976562,
      "activations/layer21_attention_weight_min": -14.103994369506836,
      "activations/layer22_attention_weight_max": 17.086652755737305,
      "activations/layer22_attention_weight_min": -12.378594398498535,
      "activations/layer23_attention_weight_max": 15.752229690551758,
      "activations/layer23_attention_weight_min": -12.57007884979248,
      "activations/layer2_attention_weight_max": 27.026325225830078,
      "activations/layer2_attention_weight_min": -19.604110717773438,
      "activations/layer3_attention_weight_max": 57.52351760864258,
      "activations/layer3_attention_weight_min": -44.324676513671875,
      "activations/layer4_attention_weight_max": 91.19256591796875,
      "activations/layer4_attention_weight_min": -79.44115447998047,
      "activations/layer5_attention_weight_max": 171.93417358398438,
      "activations/layer5_attention_weight_min": -148.06622314453125,
      "activations/layer6_attention_weight_max": 120.30679321289062,
      "activations/layer6_attention_weight_min": -108.03987121582031,
      "activations/layer7_attention_weight_max": 98.55879974365234,
      "activations/layer7_attention_weight_min": -90.19611358642578,
      "activations/layer8_attention_weight_max": 77.68853759765625,
      "activations/layer8_attention_weight_min": -69.15105438232422,
      "activations/layer9_attention_weight_max": 73.06354522705078,
      "activations/layer9_attention_weight_min": -67.41873168945312,
      "epoch": 0.3,
      "learning_rate": 0.0001495530303030303,
      "loss": 3.4492,
      "step": 5200
    },
    {
      "activations/layer0_attention_weight_max": 14.557649612426758,
      "activations/layer0_attention_weight_min": -9.841818809509277,
      "activations/layer10_attention_weight_max": 64.57783508300781,
      "activations/layer10_attention_weight_min": -52.56827163696289,
      "activations/layer11_attention_weight_max": 34.72541809082031,
      "activations/layer11_attention_weight_min": -25.267744064331055,
      "activations/layer12_attention_weight_max": 20.587114334106445,
      "activations/layer12_attention_weight_min": -21.630966186523438,
      "activations/layer13_attention_weight_max": 43.248111724853516,
      "activations/layer13_attention_weight_min": -28.33042335510254,
      "activations/layer14_attention_weight_max": 35.482295989990234,
      "activations/layer14_attention_weight_min": -15.702199935913086,
      "activations/layer15_attention_weight_max": 22.45358657836914,
      "activations/layer15_attention_weight_min": -14.898297309875488,
      "activations/layer16_attention_weight_max": 22.998674392700195,
      "activations/layer16_attention_weight_min": -13.478860855102539,
      "activations/layer17_attention_weight_max": 26.25768280029297,
      "activations/layer17_attention_weight_min": -19.494937896728516,
      "activations/layer18_attention_weight_max": 29.176305770874023,
      "activations/layer18_attention_weight_min": -16.85552215576172,
      "activations/layer19_attention_weight_max": 21.238311767578125,
      "activations/layer19_attention_weight_min": -11.69925594329834,
      "activations/layer1_attention_weight_max": 17.33545684814453,
      "activations/layer1_attention_weight_min": -11.055767059326172,
      "activations/layer20_attention_weight_max": 19.82470703125,
      "activations/layer20_attention_weight_min": -11.669185638427734,
      "activations/layer21_attention_weight_max": 26.183263778686523,
      "activations/layer21_attention_weight_min": -15.63908576965332,
      "activations/layer22_attention_weight_max": 23.331432342529297,
      "activations/layer22_attention_weight_min": -13.694856643676758,
      "activations/layer23_attention_weight_max": 18.136974334716797,
      "activations/layer23_attention_weight_min": -12.058059692382812,
      "activations/layer2_attention_weight_max": 26.54568862915039,
      "activations/layer2_attention_weight_min": -20.058307647705078,
      "activations/layer3_attention_weight_max": 58.1078987121582,
      "activations/layer3_attention_weight_min": -43.901512145996094,
      "activations/layer4_attention_weight_max": 87.40113830566406,
      "activations/layer4_attention_weight_min": -75.93455505371094,
      "activations/layer5_attention_weight_max": 161.37156677246094,
      "activations/layer5_attention_weight_min": -139.55519104003906,
      "activations/layer6_attention_weight_max": 116.38467407226562,
      "activations/layer6_attention_weight_min": -107.84300231933594,
      "activations/layer7_attention_weight_max": 100.0165023803711,
      "activations/layer7_attention_weight_min": -83.77947998046875,
      "activations/layer8_attention_weight_max": 75.71075439453125,
      "activations/layer8_attention_weight_min": -63.1697883605957,
      "activations/layer9_attention_weight_max": 83.71508026123047,
      "activations/layer9_attention_weight_min": -68.19608306884766,
      "epoch": 0.31,
      "learning_rate": 0.0001495340909090909,
      "loss": 3.4387,
      "step": 5250
    },
    {
      "activations/layer0_attention_weight_max": 13.112683296203613,
      "activations/layer0_attention_weight_min": -10.463080406188965,
      "activations/layer10_attention_weight_max": 65.67985534667969,
      "activations/layer10_attention_weight_min": -58.35930633544922,
      "activations/layer11_attention_weight_max": 35.81976318359375,
      "activations/layer11_attention_weight_min": -28.13093376159668,
      "activations/layer12_attention_weight_max": 25.264341354370117,
      "activations/layer12_attention_weight_min": -24.54070472717285,
      "activations/layer13_attention_weight_max": 34.92539978027344,
      "activations/layer13_attention_weight_min": -26.982807159423828,
      "activations/layer14_attention_weight_max": 30.334545135498047,
      "activations/layer14_attention_weight_min": -19.0579891204834,
      "activations/layer15_attention_weight_max": 27.47189712524414,
      "activations/layer15_attention_weight_min": -14.900893211364746,
      "activations/layer16_attention_weight_max": 24.938064575195312,
      "activations/layer16_attention_weight_min": -14.36600112915039,
      "activations/layer17_attention_weight_max": 28.036914825439453,
      "activations/layer17_attention_weight_min": -17.940040588378906,
      "activations/layer18_attention_weight_max": 24.77811050415039,
      "activations/layer18_attention_weight_min": -15.993941307067871,
      "activations/layer19_attention_weight_max": 24.880788803100586,
      "activations/layer19_attention_weight_min": -9.95401382446289,
      "activations/layer1_attention_weight_max": 19.31001853942871,
      "activations/layer1_attention_weight_min": -10.478796005249023,
      "activations/layer20_attention_weight_max": 23.728240966796875,
      "activations/layer20_attention_weight_min": -9.992177963256836,
      "activations/layer21_attention_weight_max": 24.423789978027344,
      "activations/layer21_attention_weight_min": -13.044978141784668,
      "activations/layer22_attention_weight_max": 21.89484977722168,
      "activations/layer22_attention_weight_min": -11.1585054397583,
      "activations/layer23_attention_weight_max": 15.697452545166016,
      "activations/layer23_attention_weight_min": -10.972253799438477,
      "activations/layer2_attention_weight_max": 24.393028259277344,
      "activations/layer2_attention_weight_min": -18.48830795288086,
      "activations/layer3_attention_weight_max": 56.21876907348633,
      "activations/layer3_attention_weight_min": -45.463478088378906,
      "activations/layer4_attention_weight_max": 88.9100112915039,
      "activations/layer4_attention_weight_min": -76.45259094238281,
      "activations/layer5_attention_weight_max": 166.88645935058594,
      "activations/layer5_attention_weight_min": -147.79461669921875,
      "activations/layer6_attention_weight_max": 116.39613342285156,
      "activations/layer6_attention_weight_min": -108.95885467529297,
      "activations/layer7_attention_weight_max": 92.12920379638672,
      "activations/layer7_attention_weight_min": -84.46453094482422,
      "activations/layer8_attention_weight_max": 70.88753509521484,
      "activations/layer8_attention_weight_min": -67.9665298461914,
      "activations/layer9_attention_weight_max": 71.4939193725586,
      "activations/layer9_attention_weight_min": -65.22899627685547,
      "epoch": 0.31,
      "learning_rate": 0.0001495151515151515,
      "loss": 3.4438,
      "step": 5300
    },
    {
      "activations/layer0_attention_weight_max": 12.72091007232666,
      "activations/layer0_attention_weight_min": -10.012916564941406,
      "activations/layer10_attention_weight_max": 67.62664794921875,
      "activations/layer10_attention_weight_min": -60.985923767089844,
      "activations/layer11_attention_weight_max": 31.326091766357422,
      "activations/layer11_attention_weight_min": -25.593860626220703,
      "activations/layer12_attention_weight_max": 21.803543090820312,
      "activations/layer12_attention_weight_min": -21.103153228759766,
      "activations/layer13_attention_weight_max": 29.062847137451172,
      "activations/layer13_attention_weight_min": -21.565488815307617,
      "activations/layer14_attention_weight_max": 26.437206268310547,
      "activations/layer14_attention_weight_min": -16.867778778076172,
      "activations/layer15_attention_weight_max": 23.29764747619629,
      "activations/layer15_attention_weight_min": -16.763717651367188,
      "activations/layer16_attention_weight_max": 21.981782913208008,
      "activations/layer16_attention_weight_min": -13.762807846069336,
      "activations/layer17_attention_weight_max": 22.91800880432129,
      "activations/layer17_attention_weight_min": -16.880971908569336,
      "activations/layer18_attention_weight_max": 33.92844009399414,
      "activations/layer18_attention_weight_min": -17.010372161865234,
      "activations/layer19_attention_weight_max": 22.66304588317871,
      "activations/layer19_attention_weight_min": -11.751235008239746,
      "activations/layer1_attention_weight_max": 17.6265926361084,
      "activations/layer1_attention_weight_min": -10.467738151550293,
      "activations/layer20_attention_weight_max": 20.837278366088867,
      "activations/layer20_attention_weight_min": -11.862814903259277,
      "activations/layer21_attention_weight_max": 20.684804916381836,
      "activations/layer21_attention_weight_min": -13.833324432373047,
      "activations/layer22_attention_weight_max": 20.135908126831055,
      "activations/layer22_attention_weight_min": -13.8355131149292,
      "activations/layer23_attention_weight_max": 15.681876182556152,
      "activations/layer23_attention_weight_min": -12.812786102294922,
      "activations/layer2_attention_weight_max": 25.505008697509766,
      "activations/layer2_attention_weight_min": -19.963359832763672,
      "activations/layer3_attention_weight_max": 57.023353576660156,
      "activations/layer3_attention_weight_min": -43.643028259277344,
      "activations/layer4_attention_weight_max": 84.73085021972656,
      "activations/layer4_attention_weight_min": -74.0067367553711,
      "activations/layer5_attention_weight_max": 154.478515625,
      "activations/layer5_attention_weight_min": -138.21243286132812,
      "activations/layer6_attention_weight_max": 108.7353286743164,
      "activations/layer6_attention_weight_min": -101.54703521728516,
      "activations/layer7_attention_weight_max": 92.29959869384766,
      "activations/layer7_attention_weight_min": -85.64689636230469,
      "activations/layer8_attention_weight_max": 71.87852478027344,
      "activations/layer8_attention_weight_min": -64.09745788574219,
      "activations/layer9_attention_weight_max": 70.92919158935547,
      "activations/layer9_attention_weight_min": -70.14257049560547,
      "epoch": 0.31,
      "learning_rate": 0.0001494962121212121,
      "loss": 3.433,
      "step": 5350
    },
    {
      "activations/layer0_attention_weight_max": 13.97622013092041,
      "activations/layer0_attention_weight_min": -10.214245796203613,
      "activations/layer10_attention_weight_max": 66.50031280517578,
      "activations/layer10_attention_weight_min": -59.23828887939453,
      "activations/layer11_attention_weight_max": 32.271156311035156,
      "activations/layer11_attention_weight_min": -28.126739501953125,
      "activations/layer12_attention_weight_max": 21.43742561340332,
      "activations/layer12_attention_weight_min": -21.943099975585938,
      "activations/layer13_attention_weight_max": 29.891864776611328,
      "activations/layer13_attention_weight_min": -22.603025436401367,
      "activations/layer14_attention_weight_max": 27.756376266479492,
      "activations/layer14_attention_weight_min": -20.073505401611328,
      "activations/layer15_attention_weight_max": 27.23796272277832,
      "activations/layer15_attention_weight_min": -16.10645866394043,
      "activations/layer16_attention_weight_max": 22.144304275512695,
      "activations/layer16_attention_weight_min": -13.018289566040039,
      "activations/layer17_attention_weight_max": 27.78782844543457,
      "activations/layer17_attention_weight_min": -16.621217727661133,
      "activations/layer18_attention_weight_max": 36.06959533691406,
      "activations/layer18_attention_weight_min": -18.67659568786621,
      "activations/layer19_attention_weight_max": 21.53508758544922,
      "activations/layer19_attention_weight_min": -11.375777244567871,
      "activations/layer1_attention_weight_max": 18.428653717041016,
      "activations/layer1_attention_weight_min": -11.078327178955078,
      "activations/layer20_attention_weight_max": 20.935956954956055,
      "activations/layer20_attention_weight_min": -11.515593528747559,
      "activations/layer21_attention_weight_max": 23.20155906677246,
      "activations/layer21_attention_weight_min": -12.844566345214844,
      "activations/layer22_attention_weight_max": 26.765119552612305,
      "activations/layer22_attention_weight_min": -11.283897399902344,
      "activations/layer23_attention_weight_max": 18.091896057128906,
      "activations/layer23_attention_weight_min": -11.68753719329834,
      "activations/layer2_attention_weight_max": 26.72563934326172,
      "activations/layer2_attention_weight_min": -19.055953979492188,
      "activations/layer3_attention_weight_max": 57.559932708740234,
      "activations/layer3_attention_weight_min": -42.62018966674805,
      "activations/layer4_attention_weight_max": 89.86443328857422,
      "activations/layer4_attention_weight_min": -75.06316375732422,
      "activations/layer5_attention_weight_max": 169.21214294433594,
      "activations/layer5_attention_weight_min": -144.0695343017578,
      "activations/layer6_attention_weight_max": 120.42283630371094,
      "activations/layer6_attention_weight_min": -103.73683166503906,
      "activations/layer7_attention_weight_max": 96.24178314208984,
      "activations/layer7_attention_weight_min": -85.92987823486328,
      "activations/layer8_attention_weight_max": 74.14186096191406,
      "activations/layer8_attention_weight_min": -62.94808578491211,
      "activations/layer9_attention_weight_max": 75.97569274902344,
      "activations/layer9_attention_weight_min": -69.25536346435547,
      "epoch": 0.31,
      "learning_rate": 0.0001494772727272727,
      "loss": 3.4404,
      "step": 5400
    },
    {
      "activations/layer0_attention_weight_max": 13.651528358459473,
      "activations/layer0_attention_weight_min": -10.76535701751709,
      "activations/layer10_attention_weight_max": 73.61502838134766,
      "activations/layer10_attention_weight_min": -58.73944854736328,
      "activations/layer11_attention_weight_max": 37.2883415222168,
      "activations/layer11_attention_weight_min": -25.316509246826172,
      "activations/layer12_attention_weight_max": 24.503395080566406,
      "activations/layer12_attention_weight_min": -21.783321380615234,
      "activations/layer13_attention_weight_max": 30.385082244873047,
      "activations/layer13_attention_weight_min": -22.450885772705078,
      "activations/layer14_attention_weight_max": 37.88450622558594,
      "activations/layer14_attention_weight_min": -19.643468856811523,
      "activations/layer15_attention_weight_max": 24.128211975097656,
      "activations/layer15_attention_weight_min": -15.452269554138184,
      "activations/layer16_attention_weight_max": 29.052196502685547,
      "activations/layer16_attention_weight_min": -13.39265251159668,
      "activations/layer17_attention_weight_max": 31.13673210144043,
      "activations/layer17_attention_weight_min": -18.402292251586914,
      "activations/layer18_attention_weight_max": 29.334003448486328,
      "activations/layer18_attention_weight_min": -14.671767234802246,
      "activations/layer19_attention_weight_max": 25.67813491821289,
      "activations/layer19_attention_weight_min": -10.82225227355957,
      "activations/layer1_attention_weight_max": 19.92037010192871,
      "activations/layer1_attention_weight_min": -10.401979446411133,
      "activations/layer20_attention_weight_max": 21.52293586730957,
      "activations/layer20_attention_weight_min": -13.9628267288208,
      "activations/layer21_attention_weight_max": 28.3380184173584,
      "activations/layer21_attention_weight_min": -13.803594589233398,
      "activations/layer22_attention_weight_max": 26.46501350402832,
      "activations/layer22_attention_weight_min": -11.913290023803711,
      "activations/layer23_attention_weight_max": 18.858970642089844,
      "activations/layer23_attention_weight_min": -11.895898818969727,
      "activations/layer2_attention_weight_max": 26.796226501464844,
      "activations/layer2_attention_weight_min": -18.842510223388672,
      "activations/layer3_attention_weight_max": 56.300472259521484,
      "activations/layer3_attention_weight_min": -45.07420349121094,
      "activations/layer4_attention_weight_max": 84.84986877441406,
      "activations/layer4_attention_weight_min": -77.14510345458984,
      "activations/layer5_attention_weight_max": 158.8584747314453,
      "activations/layer5_attention_weight_min": -141.65679931640625,
      "activations/layer6_attention_weight_max": 114.00253295898438,
      "activations/layer6_attention_weight_min": -102.49544525146484,
      "activations/layer7_attention_weight_max": 98.75203704833984,
      "activations/layer7_attention_weight_min": -84.47911834716797,
      "activations/layer8_attention_weight_max": 75.27540588378906,
      "activations/layer8_attention_weight_min": -65.11261749267578,
      "activations/layer9_attention_weight_max": 83.10781860351562,
      "activations/layer9_attention_weight_min": -70.66150665283203,
      "epoch": 0.32,
      "learning_rate": 0.0001494583333333333,
      "loss": 3.4267,
      "step": 5450
    },
    {
      "activations/layer0_attention_weight_max": 13.569883346557617,
      "activations/layer0_attention_weight_min": -9.81479263305664,
      "activations/layer10_attention_weight_max": 66.19033813476562,
      "activations/layer10_attention_weight_min": -54.72636413574219,
      "activations/layer11_attention_weight_max": 37.57258605957031,
      "activations/layer11_attention_weight_min": -28.615615844726562,
      "activations/layer12_attention_weight_max": 24.82927703857422,
      "activations/layer12_attention_weight_min": -22.42806625366211,
      "activations/layer13_attention_weight_max": 35.285980224609375,
      "activations/layer13_attention_weight_min": -23.648561477661133,
      "activations/layer14_attention_weight_max": 29.55139923095703,
      "activations/layer14_attention_weight_min": -19.771081924438477,
      "activations/layer15_attention_weight_max": 26.873205184936523,
      "activations/layer15_attention_weight_min": -18.057504653930664,
      "activations/layer16_attention_weight_max": 22.584827423095703,
      "activations/layer16_attention_weight_min": -14.462796211242676,
      "activations/layer17_attention_weight_max": 24.911287307739258,
      "activations/layer17_attention_weight_min": -18.439842224121094,
      "activations/layer18_attention_weight_max": 27.475332260131836,
      "activations/layer18_attention_weight_min": -20.390697479248047,
      "activations/layer19_attention_weight_max": 23.363447189331055,
      "activations/layer19_attention_weight_min": -13.959670066833496,
      "activations/layer1_attention_weight_max": 19.219852447509766,
      "activations/layer1_attention_weight_min": -11.024258613586426,
      "activations/layer20_attention_weight_max": 20.770185470581055,
      "activations/layer20_attention_weight_min": -15.237737655639648,
      "activations/layer21_attention_weight_max": 24.399309158325195,
      "activations/layer21_attention_weight_min": -15.857699394226074,
      "activations/layer22_attention_weight_max": 27.24243927001953,
      "activations/layer22_attention_weight_min": -13.62971305847168,
      "activations/layer23_attention_weight_max": 17.898117065429688,
      "activations/layer23_attention_weight_min": -13.143271446228027,
      "activations/layer2_attention_weight_max": 26.13331413269043,
      "activations/layer2_attention_weight_min": -19.87190055847168,
      "activations/layer3_attention_weight_max": 56.47671890258789,
      "activations/layer3_attention_weight_min": -45.722434997558594,
      "activations/layer4_attention_weight_max": 86.71842193603516,
      "activations/layer4_attention_weight_min": -76.57771301269531,
      "activations/layer5_attention_weight_max": 158.25308227539062,
      "activations/layer5_attention_weight_min": -139.39828491210938,
      "activations/layer6_attention_weight_max": 110.41822814941406,
      "activations/layer6_attention_weight_min": -100.05177307128906,
      "activations/layer7_attention_weight_max": 90.04694366455078,
      "activations/layer7_attention_weight_min": -84.09608459472656,
      "activations/layer8_attention_weight_max": 70.81842803955078,
      "activations/layer8_attention_weight_min": -65.11236572265625,
      "activations/layer9_attention_weight_max": 73.71236419677734,
      "activations/layer9_attention_weight_min": -66.2136459350586,
      "epoch": 0.32,
      "learning_rate": 0.00014943939393939393,
      "loss": 3.4288,
      "step": 5500
    },
    {
      "activations/layer0_attention_weight_max": 13.286920547485352,
      "activations/layer0_attention_weight_min": -10.160408020019531,
      "activations/layer10_attention_weight_max": 57.94882583618164,
      "activations/layer10_attention_weight_min": -52.3455924987793,
      "activations/layer11_attention_weight_max": 33.675628662109375,
      "activations/layer11_attention_weight_min": -27.933439254760742,
      "activations/layer12_attention_weight_max": 23.38944435119629,
      "activations/layer12_attention_weight_min": -20.782032012939453,
      "activations/layer13_attention_weight_max": 30.418764114379883,
      "activations/layer13_attention_weight_min": -24.293907165527344,
      "activations/layer14_attention_weight_max": 23.505983352661133,
      "activations/layer14_attention_weight_min": -18.09075927734375,
      "activations/layer15_attention_weight_max": 21.19736099243164,
      "activations/layer15_attention_weight_min": -15.325051307678223,
      "activations/layer16_attention_weight_max": 19.095691680908203,
      "activations/layer16_attention_weight_min": -13.990571975708008,
      "activations/layer17_attention_weight_max": 23.210891723632812,
      "activations/layer17_attention_weight_min": -20.817611694335938,
      "activations/layer18_attention_weight_max": 23.597328186035156,
      "activations/layer18_attention_weight_min": -16.043352127075195,
      "activations/layer19_attention_weight_max": 19.518550872802734,
      "activations/layer19_attention_weight_min": -10.83228588104248,
      "activations/layer1_attention_weight_max": 17.574050903320312,
      "activations/layer1_attention_weight_min": -12.34339714050293,
      "activations/layer20_attention_weight_max": 20.581485748291016,
      "activations/layer20_attention_weight_min": -12.581839561462402,
      "activations/layer21_attention_weight_max": 22.254091262817383,
      "activations/layer21_attention_weight_min": -14.326154708862305,
      "activations/layer22_attention_weight_max": 20.95685386657715,
      "activations/layer22_attention_weight_min": -12.993611335754395,
      "activations/layer23_attention_weight_max": 17.05596351623535,
      "activations/layer23_attention_weight_min": -11.814950942993164,
      "activations/layer2_attention_weight_max": 25.17607879638672,
      "activations/layer2_attention_weight_min": -20.198627471923828,
      "activations/layer3_attention_weight_max": 57.78828048706055,
      "activations/layer3_attention_weight_min": -44.335296630859375,
      "activations/layer4_attention_weight_max": 88.6455307006836,
      "activations/layer4_attention_weight_min": -75.02949523925781,
      "activations/layer5_attention_weight_max": 162.85897827148438,
      "activations/layer5_attention_weight_min": -140.43270874023438,
      "activations/layer6_attention_weight_max": 111.37747192382812,
      "activations/layer6_attention_weight_min": -105.26387023925781,
      "activations/layer7_attention_weight_max": 95.18213653564453,
      "activations/layer7_attention_weight_min": -90.02169036865234,
      "activations/layer8_attention_weight_max": 68.83814239501953,
      "activations/layer8_attention_weight_min": -65.23847961425781,
      "activations/layer9_attention_weight_max": 71.27964782714844,
      "activations/layer9_attention_weight_min": -66.02452850341797,
      "epoch": 0.32,
      "learning_rate": 0.00014942045454545451,
      "loss": 3.4124,
      "step": 5550
    },
    {
      "activations/layer0_attention_weight_max": 13.96718978881836,
      "activations/layer0_attention_weight_min": -10.407571792602539,
      "activations/layer10_attention_weight_max": 67.675537109375,
      "activations/layer10_attention_weight_min": -63.77664566040039,
      "activations/layer11_attention_weight_max": 35.527549743652344,
      "activations/layer11_attention_weight_min": -30.675334930419922,
      "activations/layer12_attention_weight_max": 20.78050422668457,
      "activations/layer12_attention_weight_min": -19.864948272705078,
      "activations/layer13_attention_weight_max": 31.637582778930664,
      "activations/layer13_attention_weight_min": -23.922609329223633,
      "activations/layer14_attention_weight_max": 31.076303482055664,
      "activations/layer14_attention_weight_min": -15.842394828796387,
      "activations/layer15_attention_weight_max": 27.011735916137695,
      "activations/layer15_attention_weight_min": -16.309932708740234,
      "activations/layer16_attention_weight_max": 26.47203254699707,
      "activations/layer16_attention_weight_min": -12.760407447814941,
      "activations/layer17_attention_weight_max": 27.732412338256836,
      "activations/layer17_attention_weight_min": -16.172014236450195,
      "activations/layer18_attention_weight_max": 36.59927749633789,
      "activations/layer18_attention_weight_min": -20.75458335876465,
      "activations/layer19_attention_weight_max": 26.16888999938965,
      "activations/layer19_attention_weight_min": -11.566132545471191,
      "activations/layer1_attention_weight_max": 19.218475341796875,
      "activations/layer1_attention_weight_min": -10.526541709899902,
      "activations/layer20_attention_weight_max": 22.28245735168457,
      "activations/layer20_attention_weight_min": -11.94335651397705,
      "activations/layer21_attention_weight_max": 24.04424476623535,
      "activations/layer21_attention_weight_min": -17.043075561523438,
      "activations/layer22_attention_weight_max": 23.695934295654297,
      "activations/layer22_attention_weight_min": -14.138076782226562,
      "activations/layer23_attention_weight_max": 17.22475242614746,
      "activations/layer23_attention_weight_min": -11.202818870544434,
      "activations/layer2_attention_weight_max": 26.55443572998047,
      "activations/layer2_attention_weight_min": -19.633041381835938,
      "activations/layer3_attention_weight_max": 58.331119537353516,
      "activations/layer3_attention_weight_min": -43.975746154785156,
      "activations/layer4_attention_weight_max": 90.92003631591797,
      "activations/layer4_attention_weight_min": -73.5837173461914,
      "activations/layer5_attention_weight_max": 162.46754455566406,
      "activations/layer5_attention_weight_min": -141.74468994140625,
      "activations/layer6_attention_weight_max": 112.99359893798828,
      "activations/layer6_attention_weight_min": -101.01720428466797,
      "activations/layer7_attention_weight_max": 96.9548110961914,
      "activations/layer7_attention_weight_min": -84.62897491455078,
      "activations/layer8_attention_weight_max": 79.84703063964844,
      "activations/layer8_attention_weight_min": -65.05522155761719,
      "activations/layer9_attention_weight_max": 74.16857147216797,
      "activations/layer9_attention_weight_min": -68.74333953857422,
      "epoch": 0.33,
      "learning_rate": 0.00014940151515151513,
      "loss": 3.4141,
      "step": 5600
    },
    {
      "activations/layer0_attention_weight_max": 13.03957462310791,
      "activations/layer0_attention_weight_min": -10.514177322387695,
      "activations/layer10_attention_weight_max": 63.22260665893555,
      "activations/layer10_attention_weight_min": -57.85023880004883,
      "activations/layer11_attention_weight_max": 35.644935607910156,
      "activations/layer11_attention_weight_min": -27.56624984741211,
      "activations/layer12_attention_weight_max": 21.748334884643555,
      "activations/layer12_attention_weight_min": -22.505380630493164,
      "activations/layer13_attention_weight_max": 33.23574447631836,
      "activations/layer13_attention_weight_min": -23.51095199584961,
      "activations/layer14_attention_weight_max": 30.704381942749023,
      "activations/layer14_attention_weight_min": -17.600936889648438,
      "activations/layer15_attention_weight_max": 28.586015701293945,
      "activations/layer15_attention_weight_min": -18.01034927368164,
      "activations/layer16_attention_weight_max": 21.643320083618164,
      "activations/layer16_attention_weight_min": -12.936147689819336,
      "activations/layer17_attention_weight_max": 23.389118194580078,
      "activations/layer17_attention_weight_min": -19.941747665405273,
      "activations/layer18_attention_weight_max": 37.25334930419922,
      "activations/layer18_attention_weight_min": -19.90966796875,
      "activations/layer19_attention_weight_max": 23.539838790893555,
      "activations/layer19_attention_weight_min": -10.11691951751709,
      "activations/layer1_attention_weight_max": 19.296527862548828,
      "activations/layer1_attention_weight_min": -11.134232521057129,
      "activations/layer20_attention_weight_max": 21.249160766601562,
      "activations/layer20_attention_weight_min": -12.560870170593262,
      "activations/layer21_attention_weight_max": 22.466033935546875,
      "activations/layer21_attention_weight_min": -12.27845287322998,
      "activations/layer22_attention_weight_max": 23.039567947387695,
      "activations/layer22_attention_weight_min": -10.499127388000488,
      "activations/layer23_attention_weight_max": 16.871307373046875,
      "activations/layer23_attention_weight_min": -10.111884117126465,
      "activations/layer2_attention_weight_max": 25.775615692138672,
      "activations/layer2_attention_weight_min": -20.0694580078125,
      "activations/layer3_attention_weight_max": 57.7000732421875,
      "activations/layer3_attention_weight_min": -44.25934600830078,
      "activations/layer4_attention_weight_max": 87.75484466552734,
      "activations/layer4_attention_weight_min": -76.82215881347656,
      "activations/layer5_attention_weight_max": 166.47018432617188,
      "activations/layer5_attention_weight_min": -142.72096252441406,
      "activations/layer6_attention_weight_max": 115.85398864746094,
      "activations/layer6_attention_weight_min": -99.44412994384766,
      "activations/layer7_attention_weight_max": 97.27655029296875,
      "activations/layer7_attention_weight_min": -83.82173919677734,
      "activations/layer8_attention_weight_max": 72.99950408935547,
      "activations/layer8_attention_weight_min": -62.66487121582031,
      "activations/layer9_attention_weight_max": 76.41747283935547,
      "activations/layer9_attention_weight_min": -66.92890167236328,
      "epoch": 0.33,
      "learning_rate": 0.00014938257575757574,
      "loss": 3.4109,
      "step": 5650
    },
    {
      "activations/layer0_attention_weight_max": 13.764920234680176,
      "activations/layer0_attention_weight_min": -9.912957191467285,
      "activations/layer10_attention_weight_max": 63.24934005737305,
      "activations/layer10_attention_weight_min": -57.62196731567383,
      "activations/layer11_attention_weight_max": 36.44075012207031,
      "activations/layer11_attention_weight_min": -30.280813217163086,
      "activations/layer12_attention_weight_max": 23.667449951171875,
      "activations/layer12_attention_weight_min": -21.14166259765625,
      "activations/layer13_attention_weight_max": 33.15217208862305,
      "activations/layer13_attention_weight_min": -28.046890258789062,
      "activations/layer14_attention_weight_max": 24.310134887695312,
      "activations/layer14_attention_weight_min": -18.96742057800293,
      "activations/layer15_attention_weight_max": 25.36065101623535,
      "activations/layer15_attention_weight_min": -18.505924224853516,
      "activations/layer16_attention_weight_max": 23.030668258666992,
      "activations/layer16_attention_weight_min": -14.632734298706055,
      "activations/layer17_attention_weight_max": 23.257274627685547,
      "activations/layer17_attention_weight_min": -18.82997703552246,
      "activations/layer18_attention_weight_max": 23.427600860595703,
      "activations/layer18_attention_weight_min": -14.340889930725098,
      "activations/layer19_attention_weight_max": 19.727983474731445,
      "activations/layer19_attention_weight_min": -10.45600414276123,
      "activations/layer1_attention_weight_max": 19.524669647216797,
      "activations/layer1_attention_weight_min": -11.032132148742676,
      "activations/layer20_attention_weight_max": 17.195646286010742,
      "activations/layer20_attention_weight_min": -11.697298049926758,
      "activations/layer21_attention_weight_max": 23.02897071838379,
      "activations/layer21_attention_weight_min": -13.170998573303223,
      "activations/layer22_attention_weight_max": 23.276439666748047,
      "activations/layer22_attention_weight_min": -13.090056419372559,
      "activations/layer23_attention_weight_max": 18.284774780273438,
      "activations/layer23_attention_weight_min": -11.83050537109375,
      "activations/layer2_attention_weight_max": 26.117719650268555,
      "activations/layer2_attention_weight_min": -20.355012893676758,
      "activations/layer3_attention_weight_max": 53.79332733154297,
      "activations/layer3_attention_weight_min": -45.0722770690918,
      "activations/layer4_attention_weight_max": 81.1085205078125,
      "activations/layer4_attention_weight_min": -75.24748229980469,
      "activations/layer5_attention_weight_max": 154.99327087402344,
      "activations/layer5_attention_weight_min": -139.02133178710938,
      "activations/layer6_attention_weight_max": 105.61808013916016,
      "activations/layer6_attention_weight_min": -103.18636322021484,
      "activations/layer7_attention_weight_max": 91.43714904785156,
      "activations/layer7_attention_weight_min": -89.3602294921875,
      "activations/layer8_attention_weight_max": 70.82183074951172,
      "activations/layer8_attention_weight_min": -64.75463104248047,
      "activations/layer9_attention_weight_max": 75.64692687988281,
      "activations/layer9_attention_weight_min": -66.99029541015625,
      "epoch": 0.33,
      "learning_rate": 0.00014936363636363636,
      "loss": 3.4205,
      "step": 5700
    },
    {
      "activations/layer0_attention_weight_max": 13.70713996887207,
      "activations/layer0_attention_weight_min": -10.915549278259277,
      "activations/layer10_attention_weight_max": 69.89651489257812,
      "activations/layer10_attention_weight_min": -56.113494873046875,
      "activations/layer11_attention_weight_max": 37.13568115234375,
      "activations/layer11_attention_weight_min": -27.039846420288086,
      "activations/layer12_attention_weight_max": 24.635475158691406,
      "activations/layer12_attention_weight_min": -23.127275466918945,
      "activations/layer13_attention_weight_max": 39.20389938354492,
      "activations/layer13_attention_weight_min": -24.88018226623535,
      "activations/layer14_attention_weight_max": 26.31879234313965,
      "activations/layer14_attention_weight_min": -16.700584411621094,
      "activations/layer15_attention_weight_max": 27.03963279724121,
      "activations/layer15_attention_weight_min": -15.802451133728027,
      "activations/layer16_attention_weight_max": 19.93724822998047,
      "activations/layer16_attention_weight_min": -12.989615440368652,
      "activations/layer17_attention_weight_max": 27.923044204711914,
      "activations/layer17_attention_weight_min": -16.783653259277344,
      "activations/layer18_attention_weight_max": 26.146560668945312,
      "activations/layer18_attention_weight_min": -14.751382827758789,
      "activations/layer19_attention_weight_max": 21.5333194732666,
      "activations/layer19_attention_weight_min": -11.652746200561523,
      "activations/layer1_attention_weight_max": 19.424175262451172,
      "activations/layer1_attention_weight_min": -10.45734691619873,
      "activations/layer20_attention_weight_max": 17.321327209472656,
      "activations/layer20_attention_weight_min": -11.683982849121094,
      "activations/layer21_attention_weight_max": 21.300460815429688,
      "activations/layer21_attention_weight_min": -14.320196151733398,
      "activations/layer22_attention_weight_max": 25.3225154876709,
      "activations/layer22_attention_weight_min": -11.923787117004395,
      "activations/layer23_attention_weight_max": 17.785852432250977,
      "activations/layer23_attention_weight_min": -11.411040306091309,
      "activations/layer2_attention_weight_max": 27.007230758666992,
      "activations/layer2_attention_weight_min": -20.583858489990234,
      "activations/layer3_attention_weight_max": 56.328182220458984,
      "activations/layer3_attention_weight_min": -43.63320541381836,
      "activations/layer4_attention_weight_max": 91.37765502929688,
      "activations/layer4_attention_weight_min": -73.56251525878906,
      "activations/layer5_attention_weight_max": 172.85263061523438,
      "activations/layer5_attention_weight_min": -141.31973266601562,
      "activations/layer6_attention_weight_max": 112.19119262695312,
      "activations/layer6_attention_weight_min": -101.0783462524414,
      "activations/layer7_attention_weight_max": 102.34098052978516,
      "activations/layer7_attention_weight_min": -81.724609375,
      "activations/layer8_attention_weight_max": 78.7179946899414,
      "activations/layer8_attention_weight_min": -64.00718688964844,
      "activations/layer9_attention_weight_max": 74.93510437011719,
      "activations/layer9_attention_weight_min": -62.52421951293945,
      "epoch": 0.33,
      "learning_rate": 0.00014934469696969695,
      "loss": 3.4046,
      "step": 5750
    },
    {
      "activations/layer0_attention_weight_max": 13.750094413757324,
      "activations/layer0_attention_weight_min": -10.190118789672852,
      "activations/layer10_attention_weight_max": 59.36922073364258,
      "activations/layer10_attention_weight_min": -57.23695755004883,
      "activations/layer11_attention_weight_max": 32.23657989501953,
      "activations/layer11_attention_weight_min": -29.540359497070312,
      "activations/layer12_attention_weight_max": 22.64625358581543,
      "activations/layer12_attention_weight_min": -22.634979248046875,
      "activations/layer13_attention_weight_max": 30.29802131652832,
      "activations/layer13_attention_weight_min": -26.356182098388672,
      "activations/layer14_attention_weight_max": 31.742286682128906,
      "activations/layer14_attention_weight_min": -18.075529098510742,
      "activations/layer15_attention_weight_max": 27.52029800415039,
      "activations/layer15_attention_weight_min": -19.28946876525879,
      "activations/layer16_attention_weight_max": 24.96891212463379,
      "activations/layer16_attention_weight_min": -14.39165210723877,
      "activations/layer17_attention_weight_max": 31.96137237548828,
      "activations/layer17_attention_weight_min": -20.960660934448242,
      "activations/layer18_attention_weight_max": 29.997421264648438,
      "activations/layer18_attention_weight_min": -17.91201400756836,
      "activations/layer19_attention_weight_max": 21.54767608642578,
      "activations/layer19_attention_weight_min": -13.70909309387207,
      "activations/layer1_attention_weight_max": 19.120033264160156,
      "activations/layer1_attention_weight_min": -11.909228324890137,
      "activations/layer20_attention_weight_max": 22.30591583251953,
      "activations/layer20_attention_weight_min": -12.930342674255371,
      "activations/layer21_attention_weight_max": 25.442119598388672,
      "activations/layer21_attention_weight_min": -15.490745544433594,
      "activations/layer22_attention_weight_max": 28.07904052734375,
      "activations/layer22_attention_weight_min": -13.476490020751953,
      "activations/layer23_attention_weight_max": 19.198936462402344,
      "activations/layer23_attention_weight_min": -14.974112510681152,
      "activations/layer2_attention_weight_max": 24.28510284423828,
      "activations/layer2_attention_weight_min": -20.23322296142578,
      "activations/layer3_attention_weight_max": 53.347957611083984,
      "activations/layer3_attention_weight_min": -44.46067810058594,
      "activations/layer4_attention_weight_max": 82.10716247558594,
      "activations/layer4_attention_weight_min": -75.88954162597656,
      "activations/layer5_attention_weight_max": 150.16854858398438,
      "activations/layer5_attention_weight_min": -139.805419921875,
      "activations/layer6_attention_weight_max": 97.84305572509766,
      "activations/layer6_attention_weight_min": -99.3136215209961,
      "activations/layer7_attention_weight_max": 82.174560546875,
      "activations/layer7_attention_weight_min": -84.17747497558594,
      "activations/layer8_attention_weight_max": 62.0252685546875,
      "activations/layer8_attention_weight_min": -63.92128372192383,
      "activations/layer9_attention_weight_max": 64.1942367553711,
      "activations/layer9_attention_weight_min": -65.80008697509766,
      "epoch": 0.34,
      "learning_rate": 0.00014932575757575756,
      "loss": 3.4124,
      "step": 5800
    },
    {
      "activations/layer0_attention_weight_max": 13.396368980407715,
      "activations/layer0_attention_weight_min": -10.60080337524414,
      "activations/layer10_attention_weight_max": 66.74578857421875,
      "activations/layer10_attention_weight_min": -55.1705322265625,
      "activations/layer11_attention_weight_max": 38.27838134765625,
      "activations/layer11_attention_weight_min": -29.302541732788086,
      "activations/layer12_attention_weight_max": 21.555810928344727,
      "activations/layer12_attention_weight_min": -23.76641082763672,
      "activations/layer13_attention_weight_max": 33.299678802490234,
      "activations/layer13_attention_weight_min": -33.66765594482422,
      "activations/layer14_attention_weight_max": 31.956031799316406,
      "activations/layer14_attention_weight_min": -25.418384552001953,
      "activations/layer15_attention_weight_max": 26.251903533935547,
      "activations/layer15_attention_weight_min": -19.445388793945312,
      "activations/layer16_attention_weight_max": 22.2537784576416,
      "activations/layer16_attention_weight_min": -17.609695434570312,
      "activations/layer17_attention_weight_max": 26.80491065979004,
      "activations/layer17_attention_weight_min": -24.44438934326172,
      "activations/layer18_attention_weight_max": 33.404205322265625,
      "activations/layer18_attention_weight_min": -21.472820281982422,
      "activations/layer19_attention_weight_max": 19.45037269592285,
      "activations/layer19_attention_weight_min": -15.348225593566895,
      "activations/layer1_attention_weight_max": 18.58902931213379,
      "activations/layer1_attention_weight_min": -10.917837142944336,
      "activations/layer20_attention_weight_max": 18.553821563720703,
      "activations/layer20_attention_weight_min": -14.110151290893555,
      "activations/layer21_attention_weight_max": 24.663286209106445,
      "activations/layer21_attention_weight_min": -16.421388626098633,
      "activations/layer22_attention_weight_max": 21.13912582397461,
      "activations/layer22_attention_weight_min": -13.174542427062988,
      "activations/layer23_attention_weight_max": 19.51053237915039,
      "activations/layer23_attention_weight_min": -13.816370010375977,
      "activations/layer2_attention_weight_max": 25.191566467285156,
      "activations/layer2_attention_weight_min": -20.518890380859375,
      "activations/layer3_attention_weight_max": 57.6197624206543,
      "activations/layer3_attention_weight_min": -43.18326187133789,
      "activations/layer4_attention_weight_max": 87.74526977539062,
      "activations/layer4_attention_weight_min": -76.07430267333984,
      "activations/layer5_attention_weight_max": 158.31175231933594,
      "activations/layer5_attention_weight_min": -141.11972045898438,
      "activations/layer6_attention_weight_max": 105.83840942382812,
      "activations/layer6_attention_weight_min": -101.53382110595703,
      "activations/layer7_attention_weight_max": 96.46179962158203,
      "activations/layer7_attention_weight_min": -82.1920166015625,
      "activations/layer8_attention_weight_max": 70.7386245727539,
      "activations/layer8_attention_weight_min": -63.385986328125,
      "activations/layer9_attention_weight_max": 75.99208068847656,
      "activations/layer9_attention_weight_min": -64.11958312988281,
      "epoch": 0.34,
      "learning_rate": 0.00014930681818181818,
      "loss": 3.3875,
      "step": 5850
    },
    {
      "activations/layer0_attention_weight_max": 13.248146057128906,
      "activations/layer0_attention_weight_min": -10.348957061767578,
      "activations/layer10_attention_weight_max": 66.87675476074219,
      "activations/layer10_attention_weight_min": -53.47955322265625,
      "activations/layer11_attention_weight_max": 37.727813720703125,
      "activations/layer11_attention_weight_min": -29.117971420288086,
      "activations/layer12_attention_weight_max": 22.18350601196289,
      "activations/layer12_attention_weight_min": -21.027393341064453,
      "activations/layer13_attention_weight_max": 38.15283203125,
      "activations/layer13_attention_weight_min": -24.072444915771484,
      "activations/layer14_attention_weight_max": 39.62156295776367,
      "activations/layer14_attention_weight_min": -21.077327728271484,
      "activations/layer15_attention_weight_max": 30.152767181396484,
      "activations/layer15_attention_weight_min": -17.80883026123047,
      "activations/layer16_attention_weight_max": 23.675825119018555,
      "activations/layer16_attention_weight_min": -14.286429405212402,
      "activations/layer17_attention_weight_max": 23.062400817871094,
      "activations/layer17_attention_weight_min": -20.378870010375977,
      "activations/layer18_attention_weight_max": 25.50395965576172,
      "activations/layer18_attention_weight_min": -17.445661544799805,
      "activations/layer19_attention_weight_max": 22.673810958862305,
      "activations/layer19_attention_weight_min": -11.8675537109375,
      "activations/layer1_attention_weight_max": 18.722389221191406,
      "activations/layer1_attention_weight_min": -11.349170684814453,
      "activations/layer20_attention_weight_max": 19.523157119750977,
      "activations/layer20_attention_weight_min": -13.690973281860352,
      "activations/layer21_attention_weight_max": 23.432933807373047,
      "activations/layer21_attention_weight_min": -15.859367370605469,
      "activations/layer22_attention_weight_max": 22.92701530456543,
      "activations/layer22_attention_weight_min": -13.261920928955078,
      "activations/layer23_attention_weight_max": 17.477079391479492,
      "activations/layer23_attention_weight_min": -12.638696670532227,
      "activations/layer2_attention_weight_max": 28.810834884643555,
      "activations/layer2_attention_weight_min": -19.684898376464844,
      "activations/layer3_attention_weight_max": 59.577247619628906,
      "activations/layer3_attention_weight_min": -43.76361083984375,
      "activations/layer4_attention_weight_max": 88.52169036865234,
      "activations/layer4_attention_weight_min": -74.9262924194336,
      "activations/layer5_attention_weight_max": 156.604248046875,
      "activations/layer5_attention_weight_min": -137.63302612304688,
      "activations/layer6_attention_weight_max": 109.59879302978516,
      "activations/layer6_attention_weight_min": -97.75724792480469,
      "activations/layer7_attention_weight_max": 96.0980224609375,
      "activations/layer7_attention_weight_min": -86.61238861083984,
      "activations/layer8_attention_weight_max": 80.35797119140625,
      "activations/layer8_attention_weight_min": -63.26665496826172,
      "activations/layer9_attention_weight_max": 78.17278289794922,
      "activations/layer9_attention_weight_min": -66.87101745605469,
      "epoch": 0.34,
      "learning_rate": 0.00014928787878787877,
      "loss": 3.3942,
      "step": 5900
    },
    {
      "activations/layer0_attention_weight_max": 12.792440414428711,
      "activations/layer0_attention_weight_min": -9.938492774963379,
      "activations/layer10_attention_weight_max": 65.43425750732422,
      "activations/layer10_attention_weight_min": -58.91686248779297,
      "activations/layer11_attention_weight_max": 30.858421325683594,
      "activations/layer11_attention_weight_min": -28.727611541748047,
      "activations/layer12_attention_weight_max": 25.2384090423584,
      "activations/layer12_attention_weight_min": -25.921344757080078,
      "activations/layer13_attention_weight_max": 41.94136428833008,
      "activations/layer13_attention_weight_min": -28.38271713256836,
      "activations/layer14_attention_weight_max": 36.515716552734375,
      "activations/layer14_attention_weight_min": -20.852136611938477,
      "activations/layer15_attention_weight_max": 25.99297332763672,
      "activations/layer15_attention_weight_min": -18.216157913208008,
      "activations/layer16_attention_weight_max": 26.086339950561523,
      "activations/layer16_attention_weight_min": -15.111265182495117,
      "activations/layer17_attention_weight_max": 30.112443923950195,
      "activations/layer17_attention_weight_min": -23.175548553466797,
      "activations/layer18_attention_weight_max": 41.05492401123047,
      "activations/layer18_attention_weight_min": -20.755456924438477,
      "activations/layer19_attention_weight_max": 22.426424026489258,
      "activations/layer19_attention_weight_min": -13.299727439880371,
      "activations/layer1_attention_weight_max": 17.717668533325195,
      "activations/layer1_attention_weight_min": -9.9030179977417,
      "activations/layer20_attention_weight_max": 22.446237564086914,
      "activations/layer20_attention_weight_min": -14.357501983642578,
      "activations/layer21_attention_weight_max": 23.19335174560547,
      "activations/layer21_attention_weight_min": -14.828057289123535,
      "activations/layer22_attention_weight_max": 20.416942596435547,
      "activations/layer22_attention_weight_min": -13.183449745178223,
      "activations/layer23_attention_weight_max": 14.894315719604492,
      "activations/layer23_attention_weight_min": -12.202738761901855,
      "activations/layer2_attention_weight_max": 23.33351707458496,
      "activations/layer2_attention_weight_min": -18.66790008544922,
      "activations/layer3_attention_weight_max": 52.945369720458984,
      "activations/layer3_attention_weight_min": -44.07801818847656,
      "activations/layer4_attention_weight_max": 84.72438049316406,
      "activations/layer4_attention_weight_min": -74.29356384277344,
      "activations/layer5_attention_weight_max": 156.25640869140625,
      "activations/layer5_attention_weight_min": -134.85134887695312,
      "activations/layer6_attention_weight_max": 106.84435272216797,
      "activations/layer6_attention_weight_min": -96.17206573486328,
      "activations/layer7_attention_weight_max": 88.05327606201172,
      "activations/layer7_attention_weight_min": -77.25679016113281,
      "activations/layer8_attention_weight_max": 68.90837097167969,
      "activations/layer8_attention_weight_min": -61.65978240966797,
      "activations/layer9_attention_weight_max": 84.6962890625,
      "activations/layer9_attention_weight_min": -66.20677947998047,
      "epoch": 0.35,
      "learning_rate": 0.00014926893939393938,
      "loss": 3.4008,
      "step": 5950
    },
    {
      "activations/layer0_attention_weight_max": 13.096402168273926,
      "activations/layer0_attention_weight_min": -11.067187309265137,
      "activations/layer10_attention_weight_max": 67.28857421875,
      "activations/layer10_attention_weight_min": -56.10234069824219,
      "activations/layer11_attention_weight_max": 35.07362365722656,
      "activations/layer11_attention_weight_min": -27.056211471557617,
      "activations/layer12_attention_weight_max": 20.546342849731445,
      "activations/layer12_attention_weight_min": -26.634944915771484,
      "activations/layer13_attention_weight_max": 33.06228256225586,
      "activations/layer13_attention_weight_min": -24.803937911987305,
      "activations/layer14_attention_weight_max": 28.534475326538086,
      "activations/layer14_attention_weight_min": -20.720191955566406,
      "activations/layer15_attention_weight_max": 26.109302520751953,
      "activations/layer15_attention_weight_min": -16.298227310180664,
      "activations/layer16_attention_weight_max": 22.7846622467041,
      "activations/layer16_attention_weight_min": -14.118684768676758,
      "activations/layer17_attention_weight_max": 22.0058536529541,
      "activations/layer17_attention_weight_min": -20.03911781311035,
      "activations/layer18_attention_weight_max": 32.10424041748047,
      "activations/layer18_attention_weight_min": -21.348146438598633,
      "activations/layer19_attention_weight_max": 21.547697067260742,
      "activations/layer19_attention_weight_min": -11.032988548278809,
      "activations/layer1_attention_weight_max": 17.284936904907227,
      "activations/layer1_attention_weight_min": -11.591910362243652,
      "activations/layer20_attention_weight_max": 16.60581398010254,
      "activations/layer20_attention_weight_min": -11.854351043701172,
      "activations/layer21_attention_weight_max": 21.81621742248535,
      "activations/layer21_attention_weight_min": -18.357648849487305,
      "activations/layer22_attention_weight_max": 21.73844337463379,
      "activations/layer22_attention_weight_min": -11.90643310546875,
      "activations/layer23_attention_weight_max": 16.209644317626953,
      "activations/layer23_attention_weight_min": -10.996306419372559,
      "activations/layer2_attention_weight_max": 25.92697525024414,
      "activations/layer2_attention_weight_min": -19.724594116210938,
      "activations/layer3_attention_weight_max": 54.524227142333984,
      "activations/layer3_attention_weight_min": -42.364173889160156,
      "activations/layer4_attention_weight_max": 83.9217758178711,
      "activations/layer4_attention_weight_min": -73.75733184814453,
      "activations/layer5_attention_weight_max": 167.57040405273438,
      "activations/layer5_attention_weight_min": -141.16043090820312,
      "activations/layer6_attention_weight_max": 104.96080017089844,
      "activations/layer6_attention_weight_min": -91.55128479003906,
      "activations/layer7_attention_weight_max": 96.55162811279297,
      "activations/layer7_attention_weight_min": -81.01273345947266,
      "activations/layer8_attention_weight_max": 71.20541381835938,
      "activations/layer8_attention_weight_min": -60.05194091796875,
      "activations/layer9_attention_weight_max": 72.72933959960938,
      "activations/layer9_attention_weight_min": -66.88265991210938,
      "epoch": 0.35,
      "learning_rate": 0.00014925,
      "loss": 3.4179,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_loss": 3.337890625,
      "eval_runtime": 8.2049,
      "eval_samples_per_second": 523.344,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_openwebtext_loss": 3.337890625,
      "eval_openwebtext_ppl": 28.15966472087403,
      "eval_openwebtext_runtime": 8.2049,
      "eval_openwebtext_samples_per_second": 523.344,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_wikitext_loss": 3.67578125,
      "eval_wikitext_ppl": 39.479488162280596,
      "eval_wikitext_runtime": 1.7066,
      "eval_wikitext_samples_per_second": 267.197,
      "step": 6000
    },
    {
      "epoch": 0.35,
      "eval_lambada_loss": 4.3046875,
      "eval_lambada_ppl": 74.04607243930377,
      "eval_lambada_runtime": 9.2542,
      "eval_lambada_samples_per_second": 526.137,
      "step": 6000
    },
    {
      "activations/layer0_attention_weight_max": 13.80350112915039,
      "activations/layer0_attention_weight_min": -10.463688850402832,
      "activations/layer10_attention_weight_max": 59.94817352294922,
      "activations/layer10_attention_weight_min": -64.14752197265625,
      "activations/layer11_attention_weight_max": 33.25714874267578,
      "activations/layer11_attention_weight_min": -32.2187614440918,
      "activations/layer12_attention_weight_max": 21.271970748901367,
      "activations/layer12_attention_weight_min": -21.338722229003906,
      "activations/layer13_attention_weight_max": 33.32118225097656,
      "activations/layer13_attention_weight_min": -27.105846405029297,
      "activations/layer14_attention_weight_max": 28.18255615234375,
      "activations/layer14_attention_weight_min": -21.76427459716797,
      "activations/layer15_attention_weight_max": 25.238832473754883,
      "activations/layer15_attention_weight_min": -16.28292465209961,
      "activations/layer16_attention_weight_max": 22.771955490112305,
      "activations/layer16_attention_weight_min": -15.71854019165039,
      "activations/layer17_attention_weight_max": 28.546838760375977,
      "activations/layer17_attention_weight_min": -18.8615665435791,
      "activations/layer18_attention_weight_max": 32.76622772216797,
      "activations/layer18_attention_weight_min": -16.28501319885254,
      "activations/layer19_attention_weight_max": 18.781511306762695,
      "activations/layer19_attention_weight_min": -10.861356735229492,
      "activations/layer1_attention_weight_max": 19.505401611328125,
      "activations/layer1_attention_weight_min": -10.306591033935547,
      "activations/layer20_attention_weight_max": 20.033733367919922,
      "activations/layer20_attention_weight_min": -10.299423217773438,
      "activations/layer21_attention_weight_max": 27.67540740966797,
      "activations/layer21_attention_weight_min": -13.453365325927734,
      "activations/layer22_attention_weight_max": 29.03663444519043,
      "activations/layer22_attention_weight_min": -12.547691345214844,
      "activations/layer23_attention_weight_max": 20.59760284423828,
      "activations/layer23_attention_weight_min": -11.715673446655273,
      "activations/layer2_attention_weight_max": 25.498737335205078,
      "activations/layer2_attention_weight_min": -19.475967407226562,
      "activations/layer3_attention_weight_max": 55.88517761230469,
      "activations/layer3_attention_weight_min": -43.76682662963867,
      "activations/layer4_attention_weight_max": 85.4947280883789,
      "activations/layer4_attention_weight_min": -75.9556884765625,
      "activations/layer5_attention_weight_max": 163.4443359375,
      "activations/layer5_attention_weight_min": -138.0382080078125,
      "activations/layer6_attention_weight_max": 99.9889907836914,
      "activations/layer6_attention_weight_min": -96.70909881591797,
      "activations/layer7_attention_weight_max": 86.04498291015625,
      "activations/layer7_attention_weight_min": -83.98150634765625,
      "activations/layer8_attention_weight_max": 71.67361450195312,
      "activations/layer8_attention_weight_min": -65.39642333984375,
      "activations/layer9_attention_weight_max": 73.44588470458984,
      "activations/layer9_attention_weight_min": -65.4027099609375,
      "epoch": 0.35,
      "learning_rate": 0.0001492314393939394,
      "loss": 3.3787,
      "step": 6050
    },
    {
      "activations/layer0_attention_weight_max": 14.08243465423584,
      "activations/layer0_attention_weight_min": -10.879465103149414,
      "activations/layer10_attention_weight_max": 63.96570587158203,
      "activations/layer10_attention_weight_min": -58.550045013427734,
      "activations/layer11_attention_weight_max": 39.20067596435547,
      "activations/layer11_attention_weight_min": -29.790889739990234,
      "activations/layer12_attention_weight_max": 19.629135131835938,
      "activations/layer12_attention_weight_min": -19.385244369506836,
      "activations/layer13_attention_weight_max": 30.889358520507812,
      "activations/layer13_attention_weight_min": -25.763097763061523,
      "activations/layer14_attention_weight_max": 24.539308547973633,
      "activations/layer14_attention_weight_min": -17.710163116455078,
      "activations/layer15_attention_weight_max": 23.40114974975586,
      "activations/layer15_attention_weight_min": -18.850099563598633,
      "activations/layer16_attention_weight_max": 18.111129760742188,
      "activations/layer16_attention_weight_min": -14.720451354980469,
      "activations/layer17_attention_weight_max": 25.231752395629883,
      "activations/layer17_attention_weight_min": -22.79987335205078,
      "activations/layer18_attention_weight_max": 25.867406845092773,
      "activations/layer18_attention_weight_min": -18.279829025268555,
      "activations/layer19_attention_weight_max": 19.575712203979492,
      "activations/layer19_attention_weight_min": -11.552119255065918,
      "activations/layer1_attention_weight_max": 18.97376251220703,
      "activations/layer1_attention_weight_min": -10.62117862701416,
      "activations/layer20_attention_weight_max": 16.917057037353516,
      "activations/layer20_attention_weight_min": -12.324512481689453,
      "activations/layer21_attention_weight_max": 23.118467330932617,
      "activations/layer21_attention_weight_min": -17.250934600830078,
      "activations/layer22_attention_weight_max": 23.423818588256836,
      "activations/layer22_attention_weight_min": -14.750511169433594,
      "activations/layer23_attention_weight_max": 16.780481338500977,
      "activations/layer23_attention_weight_min": -15.238924026489258,
      "activations/layer2_attention_weight_max": 26.125288009643555,
      "activations/layer2_attention_weight_min": -19.51971435546875,
      "activations/layer3_attention_weight_max": 54.61290740966797,
      "activations/layer3_attention_weight_min": -43.56135177612305,
      "activations/layer4_attention_weight_max": 83.35565185546875,
      "activations/layer4_attention_weight_min": -75.54977416992188,
      "activations/layer5_attention_weight_max": 152.08863830566406,
      "activations/layer5_attention_weight_min": -140.91387939453125,
      "activations/layer6_attention_weight_max": 100.46876525878906,
      "activations/layer6_attention_weight_min": -95.24617767333984,
      "activations/layer7_attention_weight_max": 87.39275360107422,
      "activations/layer7_attention_weight_min": -82.69378662109375,
      "activations/layer8_attention_weight_max": 72.06292724609375,
      "activations/layer8_attention_weight_min": -63.21091079711914,
      "activations/layer9_attention_weight_max": 76.9730224609375,
      "activations/layer9_attention_weight_min": -67.66515350341797,
      "epoch": 0.35,
      "learning_rate": 0.0001492125,
      "loss": 3.3917,
      "step": 6100
    },
    {
      "activations/layer0_attention_weight_max": 13.250016212463379,
      "activations/layer0_attention_weight_min": -10.830945014953613,
      "activations/layer10_attention_weight_max": 70.32630920410156,
      "activations/layer10_attention_weight_min": -55.680908203125,
      "activations/layer11_attention_weight_max": 39.28948974609375,
      "activations/layer11_attention_weight_min": -31.697265625,
      "activations/layer12_attention_weight_max": 28.72231674194336,
      "activations/layer12_attention_weight_min": -20.22882080078125,
      "activations/layer13_attention_weight_max": 38.819305419921875,
      "activations/layer13_attention_weight_min": -25.13265609741211,
      "activations/layer14_attention_weight_max": 30.19700813293457,
      "activations/layer14_attention_weight_min": -19.55451774597168,
      "activations/layer15_attention_weight_max": 31.620712280273438,
      "activations/layer15_attention_weight_min": -19.891193389892578,
      "activations/layer16_attention_weight_max": 25.981962203979492,
      "activations/layer16_attention_weight_min": -15.619050979614258,
      "activations/layer17_attention_weight_max": 33.10829544067383,
      "activations/layer17_attention_weight_min": -21.583370208740234,
      "activations/layer18_attention_weight_max": 40.400794982910156,
      "activations/layer18_attention_weight_min": -22.250490188598633,
      "activations/layer19_attention_weight_max": 23.32037925720215,
      "activations/layer19_attention_weight_min": -10.679676055908203,
      "activations/layer1_attention_weight_max": 19.827058792114258,
      "activations/layer1_attention_weight_min": -10.537959098815918,
      "activations/layer20_attention_weight_max": 21.166654586791992,
      "activations/layer20_attention_weight_min": -12.533782005310059,
      "activations/layer21_attention_weight_max": 26.1201229095459,
      "activations/layer21_attention_weight_min": -15.843804359436035,
      "activations/layer22_attention_weight_max": 24.03601837158203,
      "activations/layer22_attention_weight_min": -14.463533401489258,
      "activations/layer23_attention_weight_max": 20.48527717590332,
      "activations/layer23_attention_weight_min": -12.272429466247559,
      "activations/layer2_attention_weight_max": 25.78509521484375,
      "activations/layer2_attention_weight_min": -18.840484619140625,
      "activations/layer3_attention_weight_max": 56.54984664916992,
      "activations/layer3_attention_weight_min": -41.51768493652344,
      "activations/layer4_attention_weight_max": 87.08251190185547,
      "activations/layer4_attention_weight_min": -73.86223602294922,
      "activations/layer5_attention_weight_max": 163.575927734375,
      "activations/layer5_attention_weight_min": -137.61489868164062,
      "activations/layer6_attention_weight_max": 104.74842834472656,
      "activations/layer6_attention_weight_min": -95.87288665771484,
      "activations/layer7_attention_weight_max": 95.0924301147461,
      "activations/layer7_attention_weight_min": -81.99962615966797,
      "activations/layer8_attention_weight_max": 77.79251098632812,
      "activations/layer8_attention_weight_min": -63.17216110229492,
      "activations/layer9_attention_weight_max": 82.24542999267578,
      "activations/layer9_attention_weight_min": -67.74720001220703,
      "epoch": 0.36,
      "learning_rate": 0.0001491935606060606,
      "loss": 3.3775,
      "step": 6150
    },
    {
      "activations/layer0_attention_weight_max": 13.204985618591309,
      "activations/layer0_attention_weight_min": -11.242462158203125,
      "activations/layer10_attention_weight_max": 63.06439971923828,
      "activations/layer10_attention_weight_min": -57.71835708618164,
      "activations/layer11_attention_weight_max": 34.69668197631836,
      "activations/layer11_attention_weight_min": -28.0305233001709,
      "activations/layer12_attention_weight_max": 23.418901443481445,
      "activations/layer12_attention_weight_min": -23.53997802734375,
      "activations/layer13_attention_weight_max": 33.69872283935547,
      "activations/layer13_attention_weight_min": -27.049474716186523,
      "activations/layer14_attention_weight_max": 35.9318962097168,
      "activations/layer14_attention_weight_min": -21.52758026123047,
      "activations/layer15_attention_weight_max": 28.04579734802246,
      "activations/layer15_attention_weight_min": -18.82356834411621,
      "activations/layer16_attention_weight_max": 25.204782485961914,
      "activations/layer16_attention_weight_min": -14.708858489990234,
      "activations/layer17_attention_weight_max": 37.57368469238281,
      "activations/layer17_attention_weight_min": -21.4096736907959,
      "activations/layer18_attention_weight_max": 36.0375862121582,
      "activations/layer18_attention_weight_min": -19.571447372436523,
      "activations/layer19_attention_weight_max": 25.51447105407715,
      "activations/layer19_attention_weight_min": -11.725749015808105,
      "activations/layer1_attention_weight_max": 18.3279972076416,
      "activations/layer1_attention_weight_min": -11.339619636535645,
      "activations/layer20_attention_weight_max": 21.584182739257812,
      "activations/layer20_attention_weight_min": -11.675658226013184,
      "activations/layer21_attention_weight_max": 30.897747039794922,
      "activations/layer21_attention_weight_min": -12.866154670715332,
      "activations/layer22_attention_weight_max": 29.49024200439453,
      "activations/layer22_attention_weight_min": -11.51610279083252,
      "activations/layer23_attention_weight_max": 20.32037353515625,
      "activations/layer23_attention_weight_min": -11.578414916992188,
      "activations/layer2_attention_weight_max": 26.953868865966797,
      "activations/layer2_attention_weight_min": -19.622173309326172,
      "activations/layer3_attention_weight_max": 54.135440826416016,
      "activations/layer3_attention_weight_min": -44.49447250366211,
      "activations/layer4_attention_weight_max": 86.38927459716797,
      "activations/layer4_attention_weight_min": -74.31714630126953,
      "activations/layer5_attention_weight_max": 159.4373321533203,
      "activations/layer5_attention_weight_min": -135.67074584960938,
      "activations/layer6_attention_weight_max": 105.14064025878906,
      "activations/layer6_attention_weight_min": -91.14705657958984,
      "activations/layer7_attention_weight_max": 88.52889251708984,
      "activations/layer7_attention_weight_min": -79.83631134033203,
      "activations/layer8_attention_weight_max": 66.87075805664062,
      "activations/layer8_attention_weight_min": -60.12606430053711,
      "activations/layer9_attention_weight_max": 71.27999114990234,
      "activations/layer9_attention_weight_min": -67.07878875732422,
      "epoch": 0.36,
      "learning_rate": 0.0001491746212121212,
      "loss": 3.3808,
      "step": 6200
    },
    {
      "activations/layer0_attention_weight_max": 14.048428535461426,
      "activations/layer0_attention_weight_min": -11.366004943847656,
      "activations/layer10_attention_weight_max": 70.99471282958984,
      "activations/layer10_attention_weight_min": -61.09531784057617,
      "activations/layer11_attention_weight_max": 33.44367980957031,
      "activations/layer11_attention_weight_min": -28.460433959960938,
      "activations/layer12_attention_weight_max": 20.318517684936523,
      "activations/layer12_attention_weight_min": -23.037012100219727,
      "activations/layer13_attention_weight_max": 36.259315490722656,
      "activations/layer13_attention_weight_min": -27.878681182861328,
      "activations/layer14_attention_weight_max": 27.817380905151367,
      "activations/layer14_attention_weight_min": -18.692066192626953,
      "activations/layer15_attention_weight_max": 28.895505905151367,
      "activations/layer15_attention_weight_min": -18.169322967529297,
      "activations/layer16_attention_weight_max": 24.84269905090332,
      "activations/layer16_attention_weight_min": -15.424304008483887,
      "activations/layer17_attention_weight_max": 30.14786148071289,
      "activations/layer17_attention_weight_min": -19.10976791381836,
      "activations/layer18_attention_weight_max": 23.383634567260742,
      "activations/layer18_attention_weight_min": -14.846867561340332,
      "activations/layer19_attention_weight_max": 19.64399528503418,
      "activations/layer19_attention_weight_min": -11.128777503967285,
      "activations/layer1_attention_weight_max": 19.607126235961914,
      "activations/layer1_attention_weight_min": -11.775103569030762,
      "activations/layer20_attention_weight_max": 20.91033363342285,
      "activations/layer20_attention_weight_min": -12.909642219543457,
      "activations/layer21_attention_weight_max": 27.1949462890625,
      "activations/layer21_attention_weight_min": -15.078350067138672,
      "activations/layer22_attention_weight_max": 27.5850887298584,
      "activations/layer22_attention_weight_min": -11.3242769241333,
      "activations/layer23_attention_weight_max": 19.652605056762695,
      "activations/layer23_attention_weight_min": -11.838286399841309,
      "activations/layer2_attention_weight_max": 25.751670837402344,
      "activations/layer2_attention_weight_min": -20.576961517333984,
      "activations/layer3_attention_weight_max": 52.58759307861328,
      "activations/layer3_attention_weight_min": -44.389312744140625,
      "activations/layer4_attention_weight_max": 83.20343017578125,
      "activations/layer4_attention_weight_min": -72.9395980834961,
      "activations/layer5_attention_weight_max": 150.79083251953125,
      "activations/layer5_attention_weight_min": -135.55918884277344,
      "activations/layer6_attention_weight_max": 98.86476135253906,
      "activations/layer6_attention_weight_min": -91.74374389648438,
      "activations/layer7_attention_weight_max": 88.24734497070312,
      "activations/layer7_attention_weight_min": -83.19312286376953,
      "activations/layer8_attention_weight_max": 68.69135284423828,
      "activations/layer8_attention_weight_min": -62.79083251953125,
      "activations/layer9_attention_weight_max": 71.37916564941406,
      "activations/layer9_attention_weight_min": -65.82781982421875,
      "epoch": 0.36,
      "learning_rate": 0.00014915568181818182,
      "loss": 3.3974,
      "step": 6250
    },
    {
      "activations/layer0_attention_weight_max": 14.379776000976562,
      "activations/layer0_attention_weight_min": -10.649552345275879,
      "activations/layer10_attention_weight_max": 68.03375244140625,
      "activations/layer10_attention_weight_min": -50.66203689575195,
      "activations/layer11_attention_weight_max": 37.752647399902344,
      "activations/layer11_attention_weight_min": -29.2213134765625,
      "activations/layer12_attention_weight_max": 21.177602767944336,
      "activations/layer12_attention_weight_min": -19.85438346862793,
      "activations/layer13_attention_weight_max": 31.03606414794922,
      "activations/layer13_attention_weight_min": -23.50611114501953,
      "activations/layer14_attention_weight_max": 23.952497482299805,
      "activations/layer14_attention_weight_min": -18.9741153717041,
      "activations/layer15_attention_weight_max": 25.211938858032227,
      "activations/layer15_attention_weight_min": -18.677621841430664,
      "activations/layer16_attention_weight_max": 20.542585372924805,
      "activations/layer16_attention_weight_min": -14.772799491882324,
      "activations/layer17_attention_weight_max": 23.211336135864258,
      "activations/layer17_attention_weight_min": -17.234310150146484,
      "activations/layer18_attention_weight_max": 28.736055374145508,
      "activations/layer18_attention_weight_min": -16.968908309936523,
      "activations/layer19_attention_weight_max": 18.714801788330078,
      "activations/layer19_attention_weight_min": -10.058053970336914,
      "activations/layer1_attention_weight_max": 19.261823654174805,
      "activations/layer1_attention_weight_min": -10.970040321350098,
      "activations/layer20_attention_weight_max": 16.858335494995117,
      "activations/layer20_attention_weight_min": -11.162081718444824,
      "activations/layer21_attention_weight_max": 25.086673736572266,
      "activations/layer21_attention_weight_min": -14.549357414245605,
      "activations/layer22_attention_weight_max": 26.66306495666504,
      "activations/layer22_attention_weight_min": -10.746137619018555,
      "activations/layer23_attention_weight_max": 19.74120330810547,
      "activations/layer23_attention_weight_min": -12.671618461608887,
      "activations/layer2_attention_weight_max": 27.247325897216797,
      "activations/layer2_attention_weight_min": -19.427719116210938,
      "activations/layer3_attention_weight_max": 58.11539840698242,
      "activations/layer3_attention_weight_min": -44.51347351074219,
      "activations/layer4_attention_weight_max": 86.31694030761719,
      "activations/layer4_attention_weight_min": -73.23845672607422,
      "activations/layer5_attention_weight_max": 160.79812622070312,
      "activations/layer5_attention_weight_min": -138.37904357910156,
      "activations/layer6_attention_weight_max": 105.01333618164062,
      "activations/layer6_attention_weight_min": -96.09575653076172,
      "activations/layer7_attention_weight_max": 92.9297866821289,
      "activations/layer7_attention_weight_min": -82.51100158691406,
      "activations/layer8_attention_weight_max": 75.59671020507812,
      "activations/layer8_attention_weight_min": -61.87152099609375,
      "activations/layer9_attention_weight_max": 73.3791732788086,
      "activations/layer9_attention_weight_min": -78.3617172241211,
      "epoch": 0.37,
      "learning_rate": 0.0001491367424242424,
      "loss": 3.3866,
      "step": 6300
    },
    {
      "activations/layer0_attention_weight_max": 13.741262435913086,
      "activations/layer0_attention_weight_min": -11.241291999816895,
      "activations/layer10_attention_weight_max": 67.67943572998047,
      "activations/layer10_attention_weight_min": -56.01343536376953,
      "activations/layer11_attention_weight_max": 36.93307113647461,
      "activations/layer11_attention_weight_min": -32.16400909423828,
      "activations/layer12_attention_weight_max": 22.155208587646484,
      "activations/layer12_attention_weight_min": -21.63380241394043,
      "activations/layer13_attention_weight_max": 35.07380676269531,
      "activations/layer13_attention_weight_min": -24.172422409057617,
      "activations/layer14_attention_weight_max": 34.891944885253906,
      "activations/layer14_attention_weight_min": -20.65547752380371,
      "activations/layer15_attention_weight_max": 28.745954513549805,
      "activations/layer15_attention_weight_min": -17.56475830078125,
      "activations/layer16_attention_weight_max": 26.12493896484375,
      "activations/layer16_attention_weight_min": -15.305816650390625,
      "activations/layer17_attention_weight_max": 31.750757217407227,
      "activations/layer17_attention_weight_min": -21.301206588745117,
      "activations/layer18_attention_weight_max": 41.35393524169922,
      "activations/layer18_attention_weight_min": -21.4442195892334,
      "activations/layer19_attention_weight_max": 20.85332489013672,
      "activations/layer19_attention_weight_min": -11.404317855834961,
      "activations/layer1_attention_weight_max": 20.74973487854004,
      "activations/layer1_attention_weight_min": -11.27818775177002,
      "activations/layer20_attention_weight_max": 24.6673583984375,
      "activations/layer20_attention_weight_min": -11.987163543701172,
      "activations/layer21_attention_weight_max": 23.07270622253418,
      "activations/layer21_attention_weight_min": -13.178380966186523,
      "activations/layer22_attention_weight_max": 20.815288543701172,
      "activations/layer22_attention_weight_min": -13.030244827270508,
      "activations/layer23_attention_weight_max": 19.42316436767578,
      "activations/layer23_attention_weight_min": -11.528441429138184,
      "activations/layer2_attention_weight_max": 27.812244415283203,
      "activations/layer2_attention_weight_min": -20.153779983520508,
      "activations/layer3_attention_weight_max": 61.009010314941406,
      "activations/layer3_attention_weight_min": -46.593467712402344,
      "activations/layer4_attention_weight_max": 89.61845397949219,
      "activations/layer4_attention_weight_min": -73.81268310546875,
      "activations/layer5_attention_weight_max": 163.3033447265625,
      "activations/layer5_attention_weight_min": -135.52264404296875,
      "activations/layer6_attention_weight_max": 105.96147155761719,
      "activations/layer6_attention_weight_min": -92.80034637451172,
      "activations/layer7_attention_weight_max": 88.99243927001953,
      "activations/layer7_attention_weight_min": -79.33745574951172,
      "activations/layer8_attention_weight_max": 73.14261627197266,
      "activations/layer8_attention_weight_min": -64.24658966064453,
      "activations/layer9_attention_weight_max": 75.36198425292969,
      "activations/layer9_attention_weight_min": -65.0475082397461,
      "epoch": 0.37,
      "learning_rate": 0.00014911780303030302,
      "loss": 3.3499,
      "step": 6350
    },
    {
      "activations/layer0_attention_weight_max": 14.403847694396973,
      "activations/layer0_attention_weight_min": -12.589377403259277,
      "activations/layer10_attention_weight_max": 64.127197265625,
      "activations/layer10_attention_weight_min": -64.11015319824219,
      "activations/layer11_attention_weight_max": 39.38671875,
      "activations/layer11_attention_weight_min": -35.01990509033203,
      "activations/layer12_attention_weight_max": 24.714614868164062,
      "activations/layer12_attention_weight_min": -24.555524826049805,
      "activations/layer13_attention_weight_max": 36.54400634765625,
      "activations/layer13_attention_weight_min": -26.07948112487793,
      "activations/layer14_attention_weight_max": 30.13185691833496,
      "activations/layer14_attention_weight_min": -19.674531936645508,
      "activations/layer15_attention_weight_max": 29.83182144165039,
      "activations/layer15_attention_weight_min": -19.31345558166504,
      "activations/layer16_attention_weight_max": 23.660709381103516,
      "activations/layer16_attention_weight_min": -15.084242820739746,
      "activations/layer17_attention_weight_max": 28.972877502441406,
      "activations/layer17_attention_weight_min": -22.29831314086914,
      "activations/layer18_attention_weight_max": 28.8814754486084,
      "activations/layer18_attention_weight_min": -18.87565803527832,
      "activations/layer19_attention_weight_max": 19.961179733276367,
      "activations/layer19_attention_weight_min": -12.179076194763184,
      "activations/layer1_attention_weight_max": 19.71807861328125,
      "activations/layer1_attention_weight_min": -17.36551856994629,
      "activations/layer20_attention_weight_max": 21.685504913330078,
      "activations/layer20_attention_weight_min": -15.470258712768555,
      "activations/layer21_attention_weight_max": 27.283092498779297,
      "activations/layer21_attention_weight_min": -14.58187484741211,
      "activations/layer22_attention_weight_max": 27.54363250732422,
      "activations/layer22_attention_weight_min": -16.135488510131836,
      "activations/layer23_attention_weight_max": 19.745277404785156,
      "activations/layer23_attention_weight_min": -14.899733543395996,
      "activations/layer2_attention_weight_max": 19.87914276123047,
      "activations/layer2_attention_weight_min": -16.705556869506836,
      "activations/layer3_attention_weight_max": 43.536869049072266,
      "activations/layer3_attention_weight_min": -39.797969818115234,
      "activations/layer4_attention_weight_max": 66.64472198486328,
      "activations/layer4_attention_weight_min": -77.00289154052734,
      "activations/layer5_attention_weight_max": 148.85023498535156,
      "activations/layer5_attention_weight_min": -148.32864379882812,
      "activations/layer6_attention_weight_max": 86.32457733154297,
      "activations/layer6_attention_weight_min": -89.05716705322266,
      "activations/layer7_attention_weight_max": 78.29657745361328,
      "activations/layer7_attention_weight_min": -82.14675903320312,
      "activations/layer8_attention_weight_max": 73.86894989013672,
      "activations/layer8_attention_weight_min": -71.24429321289062,
      "activations/layer9_attention_weight_max": 67.60027313232422,
      "activations/layer9_attention_weight_min": -69.5067138671875,
      "epoch": 0.37,
      "learning_rate": 0.0001490992424242424,
      "loss": 3.5831,
      "step": 6400
    },
    {
      "activations/layer0_attention_weight_max": 13.648930549621582,
      "activations/layer0_attention_weight_min": -11.720470428466797,
      "activations/layer10_attention_weight_max": 72.15234375,
      "activations/layer10_attention_weight_min": -57.621028900146484,
      "activations/layer11_attention_weight_max": 39.4542236328125,
      "activations/layer11_attention_weight_min": -30.012662887573242,
      "activations/layer12_attention_weight_max": 27.631269454956055,
      "activations/layer12_attention_weight_min": -21.473342895507812,
      "activations/layer13_attention_weight_max": 41.683685302734375,
      "activations/layer13_attention_weight_min": -32.24076461791992,
      "activations/layer14_attention_weight_max": 41.6804313659668,
      "activations/layer14_attention_weight_min": -24.10683250427246,
      "activations/layer15_attention_weight_max": 26.821739196777344,
      "activations/layer15_attention_weight_min": -16.96794319152832,
      "activations/layer16_attention_weight_max": 23.654985427856445,
      "activations/layer16_attention_weight_min": -13.584450721740723,
      "activations/layer17_attention_weight_max": 29.48282814025879,
      "activations/layer17_attention_weight_min": -18.732254028320312,
      "activations/layer18_attention_weight_max": 37.262290954589844,
      "activations/layer18_attention_weight_min": -17.222280502319336,
      "activations/layer19_attention_weight_max": 22.557165145874023,
      "activations/layer19_attention_weight_min": -10.602578163146973,
      "activations/layer1_attention_weight_max": 17.533281326293945,
      "activations/layer1_attention_weight_min": -15.806769371032715,
      "activations/layer20_attention_weight_max": 20.250173568725586,
      "activations/layer20_attention_weight_min": -10.787542343139648,
      "activations/layer21_attention_weight_max": 23.78790283203125,
      "activations/layer21_attention_weight_min": -19.96792984008789,
      "activations/layer22_attention_weight_max": 23.80721092224121,
      "activations/layer22_attention_weight_min": -16.566152572631836,
      "activations/layer23_attention_weight_max": 16.616613388061523,
      "activations/layer23_attention_weight_min": -14.981082916259766,
      "activations/layer2_attention_weight_max": 20.652555465698242,
      "activations/layer2_attention_weight_min": -19.28708839416504,
      "activations/layer3_attention_weight_max": 43.625755310058594,
      "activations/layer3_attention_weight_min": -38.23638153076172,
      "activations/layer4_attention_weight_max": 69.63018798828125,
      "activations/layer4_attention_weight_min": -63.01418685913086,
      "activations/layer5_attention_weight_max": 137.57040405273438,
      "activations/layer5_attention_weight_min": -125.21875,
      "activations/layer6_attention_weight_max": 90.77729034423828,
      "activations/layer6_attention_weight_min": -82.97447967529297,
      "activations/layer7_attention_weight_max": 84.80706024169922,
      "activations/layer7_attention_weight_min": -74.82988739013672,
      "activations/layer8_attention_weight_max": 70.5408935546875,
      "activations/layer8_attention_weight_min": -65.58848571777344,
      "activations/layer9_attention_weight_max": 75.40650939941406,
      "activations/layer9_attention_weight_min": -64.23676300048828,
      "epoch": 0.37,
      "learning_rate": 0.00014908030303030303,
      "loss": 3.4504,
      "step": 6450
    },
    {
      "activations/layer0_attention_weight_max": 13.8930082321167,
      "activations/layer0_attention_weight_min": -11.219930648803711,
      "activations/layer10_attention_weight_max": 64.3704833984375,
      "activations/layer10_attention_weight_min": -54.57594680786133,
      "activations/layer11_attention_weight_max": 37.705718994140625,
      "activations/layer11_attention_weight_min": -29.952617645263672,
      "activations/layer12_attention_weight_max": 21.253162384033203,
      "activations/layer12_attention_weight_min": -24.08654022216797,
      "activations/layer13_attention_weight_max": 33.93094253540039,
      "activations/layer13_attention_weight_min": -28.170368194580078,
      "activations/layer14_attention_weight_max": 34.917049407958984,
      "activations/layer14_attention_weight_min": -20.16947364807129,
      "activations/layer15_attention_weight_max": 28.02407455444336,
      "activations/layer15_attention_weight_min": -18.977441787719727,
      "activations/layer16_attention_weight_max": 24.574127197265625,
      "activations/layer16_attention_weight_min": -16.124181747436523,
      "activations/layer17_attention_weight_max": 27.091426849365234,
      "activations/layer17_attention_weight_min": -20.77495574951172,
      "activations/layer18_attention_weight_max": 30.454097747802734,
      "activations/layer18_attention_weight_min": -20.26781463623047,
      "activations/layer19_attention_weight_max": 22.73915672302246,
      "activations/layer19_attention_weight_min": -10.909421920776367,
      "activations/layer1_attention_weight_max": 17.44305419921875,
      "activations/layer1_attention_weight_min": -13.344880104064941,
      "activations/layer20_attention_weight_max": 19.420194625854492,
      "activations/layer20_attention_weight_min": -13.407830238342285,
      "activations/layer21_attention_weight_max": 24.23810386657715,
      "activations/layer21_attention_weight_min": -16.222793579101562,
      "activations/layer22_attention_weight_max": 23.395877838134766,
      "activations/layer22_attention_weight_min": -14.825956344604492,
      "activations/layer23_attention_weight_max": 18.270313262939453,
      "activations/layer23_attention_weight_min": -14.919754028320312,
      "activations/layer2_attention_weight_max": 23.51796531677246,
      "activations/layer2_attention_weight_min": -18.851573944091797,
      "activations/layer3_attention_weight_max": 46.292911529541016,
      "activations/layer3_attention_weight_min": -39.34028625488281,
      "activations/layer4_attention_weight_max": 73.79806518554688,
      "activations/layer4_attention_weight_min": -63.70750045776367,
      "activations/layer5_attention_weight_max": 147.61959838867188,
      "activations/layer5_attention_weight_min": -124.7257080078125,
      "activations/layer6_attention_weight_max": 95.44816589355469,
      "activations/layer6_attention_weight_min": -84.92169189453125,
      "activations/layer7_attention_weight_max": 86.7701416015625,
      "activations/layer7_attention_weight_min": -76.20526123046875,
      "activations/layer8_attention_weight_max": 66.1781997680664,
      "activations/layer8_attention_weight_min": -61.09151840209961,
      "activations/layer9_attention_weight_max": 79.18182373046875,
      "activations/layer9_attention_weight_min": -62.87506103515625,
      "epoch": 0.38,
      "learning_rate": 0.00014906136363636362,
      "loss": 3.365,
      "step": 6500
    },
    {
      "activations/layer0_attention_weight_max": 13.762747764587402,
      "activations/layer0_attention_weight_min": -10.551603317260742,
      "activations/layer10_attention_weight_max": 65.48500061035156,
      "activations/layer10_attention_weight_min": -63.90750503540039,
      "activations/layer11_attention_weight_max": 38.46430969238281,
      "activations/layer11_attention_weight_min": -33.25704574584961,
      "activations/layer12_attention_weight_max": 23.05261993408203,
      "activations/layer12_attention_weight_min": -21.616403579711914,
      "activations/layer13_attention_weight_max": 41.49873733520508,
      "activations/layer13_attention_weight_min": -26.747318267822266,
      "activations/layer14_attention_weight_max": 30.228322982788086,
      "activations/layer14_attention_weight_min": -19.51568031311035,
      "activations/layer15_attention_weight_max": 34.373260498046875,
      "activations/layer15_attention_weight_min": -20.129440307617188,
      "activations/layer16_attention_weight_max": 23.585620880126953,
      "activations/layer16_attention_weight_min": -15.78533935546875,
      "activations/layer17_attention_weight_max": 31.52605628967285,
      "activations/layer17_attention_weight_min": -20.062387466430664,
      "activations/layer18_attention_weight_max": 34.6976432800293,
      "activations/layer18_attention_weight_min": -18.30593490600586,
      "activations/layer19_attention_weight_max": 23.620079040527344,
      "activations/layer19_attention_weight_min": -11.339088439941406,
      "activations/layer1_attention_weight_max": 17.529659271240234,
      "activations/layer1_attention_weight_min": -11.412482261657715,
      "activations/layer20_attention_weight_max": 18.86667823791504,
      "activations/layer20_attention_weight_min": -11.389778137207031,
      "activations/layer21_attention_weight_max": 25.14866828918457,
      "activations/layer21_attention_weight_min": -13.36766242980957,
      "activations/layer22_attention_weight_max": 24.927440643310547,
      "activations/layer22_attention_weight_min": -13.39482593536377,
      "activations/layer23_attention_weight_max": 21.372472763061523,
      "activations/layer23_attention_weight_min": -11.109272003173828,
      "activations/layer2_attention_weight_max": 24.005680084228516,
      "activations/layer2_attention_weight_min": -18.107044219970703,
      "activations/layer3_attention_weight_max": 48.4438591003418,
      "activations/layer3_attention_weight_min": -41.37793731689453,
      "activations/layer4_attention_weight_max": 74.45460510253906,
      "activations/layer4_attention_weight_min": -67.26065063476562,
      "activations/layer5_attention_weight_max": 137.81610107421875,
      "activations/layer5_attention_weight_min": -123.16819763183594,
      "activations/layer6_attention_weight_max": 91.13359832763672,
      "activations/layer6_attention_weight_min": -86.63168334960938,
      "activations/layer7_attention_weight_max": 88.80770111083984,
      "activations/layer7_attention_weight_min": -73.19158172607422,
      "activations/layer8_attention_weight_max": 69.92688751220703,
      "activations/layer8_attention_weight_min": -64.35807037353516,
      "activations/layer9_attention_weight_max": 69.67662811279297,
      "activations/layer9_attention_weight_min": -66.18159484863281,
      "epoch": 0.38,
      "learning_rate": 0.00014904242424242423,
      "loss": 3.3894,
      "step": 6550
    },
    {
      "activations/layer0_attention_weight_max": 13.80517578125,
      "activations/layer0_attention_weight_min": -10.03394603729248,
      "activations/layer10_attention_weight_max": 67.1015396118164,
      "activations/layer10_attention_weight_min": -56.0594596862793,
      "activations/layer11_attention_weight_max": 40.10686492919922,
      "activations/layer11_attention_weight_min": -29.58553123474121,
      "activations/layer12_attention_weight_max": 23.221506118774414,
      "activations/layer12_attention_weight_min": -21.337366104125977,
      "activations/layer13_attention_weight_max": 35.86437225341797,
      "activations/layer13_attention_weight_min": -29.48217010498047,
      "activations/layer14_attention_weight_max": 31.015165328979492,
      "activations/layer14_attention_weight_min": -21.37432289123535,
      "activations/layer15_attention_weight_max": 31.891603469848633,
      "activations/layer15_attention_weight_min": -19.029056549072266,
      "activations/layer16_attention_weight_max": 24.706092834472656,
      "activations/layer16_attention_weight_min": -15.011011123657227,
      "activations/layer17_attention_weight_max": 36.96307373046875,
      "activations/layer17_attention_weight_min": -22.75258445739746,
      "activations/layer18_attention_weight_max": 28.84650993347168,
      "activations/layer18_attention_weight_min": -15.199777603149414,
      "activations/layer19_attention_weight_max": 23.157817840576172,
      "activations/layer19_attention_weight_min": -10.738141059875488,
      "activations/layer1_attention_weight_max": 17.804027557373047,
      "activations/layer1_attention_weight_min": -12.104593276977539,
      "activations/layer20_attention_weight_max": 23.74952507019043,
      "activations/layer20_attention_weight_min": -11.93905258178711,
      "activations/layer21_attention_weight_max": 26.672563552856445,
      "activations/layer21_attention_weight_min": -15.510287284851074,
      "activations/layer22_attention_weight_max": 25.840545654296875,
      "activations/layer22_attention_weight_min": -11.535908699035645,
      "activations/layer23_attention_weight_max": 19.61304473876953,
      "activations/layer23_attention_weight_min": -10.623737335205078,
      "activations/layer2_attention_weight_max": 25.819225311279297,
      "activations/layer2_attention_weight_min": -19.014263153076172,
      "activations/layer3_attention_weight_max": 49.319915771484375,
      "activations/layer3_attention_weight_min": -41.4928092956543,
      "activations/layer4_attention_weight_max": 77.28732299804688,
      "activations/layer4_attention_weight_min": -67.58621215820312,
      "activations/layer5_attention_weight_max": 146.66653442382812,
      "activations/layer5_attention_weight_min": -124.7824935913086,
      "activations/layer6_attention_weight_max": 94.92281341552734,
      "activations/layer6_attention_weight_min": -83.1248779296875,
      "activations/layer7_attention_weight_max": 87.63451385498047,
      "activations/layer7_attention_weight_min": -78.13323974609375,
      "activations/layer8_attention_weight_max": 70.18583679199219,
      "activations/layer8_attention_weight_min": -61.14795684814453,
      "activations/layer9_attention_weight_max": 74.61761474609375,
      "activations/layer9_attention_weight_min": -69.46025848388672,
      "epoch": 0.38,
      "learning_rate": 0.00014902348484848485,
      "loss": 3.368,
      "step": 6600
    },
    {
      "activations/layer0_attention_weight_max": 15.886488914489746,
      "activations/layer0_attention_weight_min": -9.752370834350586,
      "activations/layer10_attention_weight_max": 65.40484619140625,
      "activations/layer10_attention_weight_min": -59.54206466674805,
      "activations/layer11_attention_weight_max": 38.91822814941406,
      "activations/layer11_attention_weight_min": -33.566593170166016,
      "activations/layer12_attention_weight_max": 23.609420776367188,
      "activations/layer12_attention_weight_min": -22.60529136657715,
      "activations/layer13_attention_weight_max": 37.97602081298828,
      "activations/layer13_attention_weight_min": -29.546865463256836,
      "activations/layer14_attention_weight_max": 30.265336990356445,
      "activations/layer14_attention_weight_min": -24.724084854125977,
      "activations/layer15_attention_weight_max": 28.143543243408203,
      "activations/layer15_attention_weight_min": -22.293161392211914,
      "activations/layer16_attention_weight_max": 23.023483276367188,
      "activations/layer16_attention_weight_min": -17.127948760986328,
      "activations/layer17_attention_weight_max": 32.684085845947266,
      "activations/layer17_attention_weight_min": -23.462629318237305,
      "activations/layer18_attention_weight_max": 34.07561492919922,
      "activations/layer18_attention_weight_min": -22.32186508178711,
      "activations/layer19_attention_weight_max": 19.553144454956055,
      "activations/layer19_attention_weight_min": -10.129923820495605,
      "activations/layer1_attention_weight_max": 18.321571350097656,
      "activations/layer1_attention_weight_min": -11.32176685333252,
      "activations/layer20_attention_weight_max": 18.688949584960938,
      "activations/layer20_attention_weight_min": -11.179354667663574,
      "activations/layer21_attention_weight_max": 22.836132049560547,
      "activations/layer21_attention_weight_min": -14.97578239440918,
      "activations/layer22_attention_weight_max": 26.271759033203125,
      "activations/layer22_attention_weight_min": -15.494306564331055,
      "activations/layer23_attention_weight_max": 18.01325225830078,
      "activations/layer23_attention_weight_min": -13.427011489868164,
      "activations/layer2_attention_weight_max": 25.143535614013672,
      "activations/layer2_attention_weight_min": -18.90161895751953,
      "activations/layer3_attention_weight_max": 49.60662078857422,
      "activations/layer3_attention_weight_min": -42.34181213378906,
      "activations/layer4_attention_weight_max": 77.09020233154297,
      "activations/layer4_attention_weight_min": -69.77577209472656,
      "activations/layer5_attention_weight_max": 144.51455688476562,
      "activations/layer5_attention_weight_min": -129.43740844726562,
      "activations/layer6_attention_weight_max": 92.94770812988281,
      "activations/layer6_attention_weight_min": -86.67857360839844,
      "activations/layer7_attention_weight_max": 84.27906799316406,
      "activations/layer7_attention_weight_min": -77.46075439453125,
      "activations/layer8_attention_weight_max": 69.30077362060547,
      "activations/layer8_attention_weight_min": -62.20201873779297,
      "activations/layer9_attention_weight_max": 73.24556732177734,
      "activations/layer9_attention_weight_min": -62.42777633666992,
      "epoch": 0.39,
      "learning_rate": 0.00014900454545454543,
      "loss": 3.3597,
      "step": 6650
    },
    {
      "activations/layer0_attention_weight_max": 13.580870628356934,
      "activations/layer0_attention_weight_min": -9.804112434387207,
      "activations/layer10_attention_weight_max": 64.30274963378906,
      "activations/layer10_attention_weight_min": -55.97967529296875,
      "activations/layer11_attention_weight_max": 38.27278137207031,
      "activations/layer11_attention_weight_min": -31.933509826660156,
      "activations/layer12_attention_weight_max": 24.048542022705078,
      "activations/layer12_attention_weight_min": -23.376800537109375,
      "activations/layer13_attention_weight_max": 36.95440673828125,
      "activations/layer13_attention_weight_min": -27.262670516967773,
      "activations/layer14_attention_weight_max": 29.858612060546875,
      "activations/layer14_attention_weight_min": -19.954713821411133,
      "activations/layer15_attention_weight_max": 26.470170974731445,
      "activations/layer15_attention_weight_min": -18.81522560119629,
      "activations/layer16_attention_weight_max": 19.76915740966797,
      "activations/layer16_attention_weight_min": -17.241235733032227,
      "activations/layer17_attention_weight_max": 30.55141258239746,
      "activations/layer17_attention_weight_min": -22.365074157714844,
      "activations/layer18_attention_weight_max": 32.58501434326172,
      "activations/layer18_attention_weight_min": -21.917055130004883,
      "activations/layer19_attention_weight_max": 20.893823623657227,
      "activations/layer19_attention_weight_min": -11.180602073669434,
      "activations/layer1_attention_weight_max": 17.102039337158203,
      "activations/layer1_attention_weight_min": -11.38209056854248,
      "activations/layer20_attention_weight_max": 19.539175033569336,
      "activations/layer20_attention_weight_min": -15.05208969116211,
      "activations/layer21_attention_weight_max": 20.794479370117188,
      "activations/layer21_attention_weight_min": -13.830324172973633,
      "activations/layer22_attention_weight_max": 21.78213882446289,
      "activations/layer22_attention_weight_min": -13.645242691040039,
      "activations/layer23_attention_weight_max": 16.49936866760254,
      "activations/layer23_attention_weight_min": -13.801458358764648,
      "activations/layer2_attention_weight_max": 25.660781860351562,
      "activations/layer2_attention_weight_min": -18.88570785522461,
      "activations/layer3_attention_weight_max": 52.88065719604492,
      "activations/layer3_attention_weight_min": -41.866554260253906,
      "activations/layer4_attention_weight_max": 80.72261810302734,
      "activations/layer4_attention_weight_min": -70.20186614990234,
      "activations/layer5_attention_weight_max": 148.84690856933594,
      "activations/layer5_attention_weight_min": -131.01947021484375,
      "activations/layer6_attention_weight_max": 92.4552230834961,
      "activations/layer6_attention_weight_min": -87.04623413085938,
      "activations/layer7_attention_weight_max": 85.24065399169922,
      "activations/layer7_attention_weight_min": -75.79449462890625,
      "activations/layer8_attention_weight_max": 70.56409454345703,
      "activations/layer8_attention_weight_min": -60.67155838012695,
      "activations/layer9_attention_weight_max": 89.56298828125,
      "activations/layer9_attention_weight_min": -70.51634979248047,
      "epoch": 0.39,
      "learning_rate": 0.00014898560606060605,
      "loss": 3.3714,
      "step": 6700
    },
    {
      "activations/layer0_attention_weight_max": 13.25387954711914,
      "activations/layer0_attention_weight_min": -9.168821334838867,
      "activations/layer10_attention_weight_max": 74.83195495605469,
      "activations/layer10_attention_weight_min": -60.59795379638672,
      "activations/layer11_attention_weight_max": 42.346431732177734,
      "activations/layer11_attention_weight_min": -30.936187744140625,
      "activations/layer12_attention_weight_max": 33.215824127197266,
      "activations/layer12_attention_weight_min": -24.19255256652832,
      "activations/layer13_attention_weight_max": 55.389122009277344,
      "activations/layer13_attention_weight_min": -33.210811614990234,
      "activations/layer14_attention_weight_max": 40.45045471191406,
      "activations/layer14_attention_weight_min": -20.119522094726562,
      "activations/layer15_attention_weight_max": 34.780792236328125,
      "activations/layer15_attention_weight_min": -19.04407501220703,
      "activations/layer16_attention_weight_max": 27.734846115112305,
      "activations/layer16_attention_weight_min": -16.524587631225586,
      "activations/layer17_attention_weight_max": 34.17732238769531,
      "activations/layer17_attention_weight_min": -20.798503875732422,
      "activations/layer18_attention_weight_max": 45.932743072509766,
      "activations/layer18_attention_weight_min": -21.206809997558594,
      "activations/layer19_attention_weight_max": 20.288612365722656,
      "activations/layer19_attention_weight_min": -10.741971969604492,
      "activations/layer1_attention_weight_max": 17.659955978393555,
      "activations/layer1_attention_weight_min": -11.796440124511719,
      "activations/layer20_attention_weight_max": 18.314308166503906,
      "activations/layer20_attention_weight_min": -9.8876953125,
      "activations/layer21_attention_weight_max": 25.760398864746094,
      "activations/layer21_attention_weight_min": -11.34174919128418,
      "activations/layer22_attention_weight_max": 26.332870483398438,
      "activations/layer22_attention_weight_min": -10.686225891113281,
      "activations/layer23_attention_weight_max": 18.53368377685547,
      "activations/layer23_attention_weight_min": -9.538705825805664,
      "activations/layer2_attention_weight_max": 25.169086456298828,
      "activations/layer2_attention_weight_min": -19.024280548095703,
      "activations/layer3_attention_weight_max": 54.982051849365234,
      "activations/layer3_attention_weight_min": -43.06553649902344,
      "activations/layer4_attention_weight_max": 83.67424774169922,
      "activations/layer4_attention_weight_min": -68.4225082397461,
      "activations/layer5_attention_weight_max": 154.07791137695312,
      "activations/layer5_attention_weight_min": -128.21969604492188,
      "activations/layer6_attention_weight_max": 93.1528091430664,
      "activations/layer6_attention_weight_min": -82.15283203125,
      "activations/layer7_attention_weight_max": 92.114990234375,
      "activations/layer7_attention_weight_min": -77.21468353271484,
      "activations/layer8_attention_weight_max": 71.5030288696289,
      "activations/layer8_attention_weight_min": -61.55712127685547,
      "activations/layer9_attention_weight_max": 82.13388061523438,
      "activations/layer9_attention_weight_min": -68.01820373535156,
      "epoch": 0.39,
      "learning_rate": 0.00014896666666666664,
      "loss": 3.362,
      "step": 6750
    },
    {
      "activations/layer0_attention_weight_max": 13.486300468444824,
      "activations/layer0_attention_weight_min": -9.991448402404785,
      "activations/layer10_attention_weight_max": 63.57056427001953,
      "activations/layer10_attention_weight_min": -60.3426628112793,
      "activations/layer11_attention_weight_max": 37.72542190551758,
      "activations/layer11_attention_weight_min": -33.529449462890625,
      "activations/layer12_attention_weight_max": 22.787290573120117,
      "activations/layer12_attention_weight_min": -24.173648834228516,
      "activations/layer13_attention_weight_max": 35.08551788330078,
      "activations/layer13_attention_weight_min": -30.83858871459961,
      "activations/layer14_attention_weight_max": 27.153778076171875,
      "activations/layer14_attention_weight_min": -22.565834045410156,
      "activations/layer15_attention_weight_max": 27.677976608276367,
      "activations/layer15_attention_weight_min": -18.500635147094727,
      "activations/layer16_attention_weight_max": 20.74728775024414,
      "activations/layer16_attention_weight_min": -16.47919464111328,
      "activations/layer17_attention_weight_max": 35.46013641357422,
      "activations/layer17_attention_weight_min": -25.845245361328125,
      "activations/layer18_attention_weight_max": 39.33332443237305,
      "activations/layer18_attention_weight_min": -18.787494659423828,
      "activations/layer19_attention_weight_max": 18.544178009033203,
      "activations/layer19_attention_weight_min": -10.186967849731445,
      "activations/layer1_attention_weight_max": 16.954647064208984,
      "activations/layer1_attention_weight_min": -11.161108016967773,
      "activations/layer20_attention_weight_max": 17.972118377685547,
      "activations/layer20_attention_weight_min": -10.564894676208496,
      "activations/layer21_attention_weight_max": 25.94744110107422,
      "activations/layer21_attention_weight_min": -12.0398530960083,
      "activations/layer22_attention_weight_max": 26.675262451171875,
      "activations/layer22_attention_weight_min": -14.784116744995117,
      "activations/layer23_attention_weight_max": 19.078636169433594,
      "activations/layer23_attention_weight_min": -14.928068161010742,
      "activations/layer2_attention_weight_max": 24.694469451904297,
      "activations/layer2_attention_weight_min": -19.159767150878906,
      "activations/layer3_attention_weight_max": 49.25062942504883,
      "activations/layer3_attention_weight_min": -44.172428131103516,
      "activations/layer4_attention_weight_max": 82.00516510009766,
      "activations/layer4_attention_weight_min": -70.75606536865234,
      "activations/layer5_attention_weight_max": 144.3935089111328,
      "activations/layer5_attention_weight_min": -128.88990783691406,
      "activations/layer6_attention_weight_max": 88.15902709960938,
      "activations/layer6_attention_weight_min": -87.6438217163086,
      "activations/layer7_attention_weight_max": 81.4461898803711,
      "activations/layer7_attention_weight_min": -77.14588165283203,
      "activations/layer8_attention_weight_max": 68.3776626586914,
      "activations/layer8_attention_weight_min": -61.36121368408203,
      "activations/layer9_attention_weight_max": 70.04827117919922,
      "activations/layer9_attention_weight_min": -68.48906707763672,
      "epoch": 0.4,
      "learning_rate": 0.00014894772727272725,
      "loss": 3.3581,
      "step": 6800
    },
    {
      "activations/layer0_attention_weight_max": 13.804740905761719,
      "activations/layer0_attention_weight_min": -10.013470649719238,
      "activations/layer10_attention_weight_max": 59.53432083129883,
      "activations/layer10_attention_weight_min": -58.38577651977539,
      "activations/layer11_attention_weight_max": 37.5228385925293,
      "activations/layer11_attention_weight_min": -32.40564727783203,
      "activations/layer12_attention_weight_max": 23.47832489013672,
      "activations/layer12_attention_weight_min": -24.944019317626953,
      "activations/layer13_attention_weight_max": 34.603858947753906,
      "activations/layer13_attention_weight_min": -28.803985595703125,
      "activations/layer14_attention_weight_max": 35.31708908081055,
      "activations/layer14_attention_weight_min": -24.270517349243164,
      "activations/layer15_attention_weight_max": 29.5820255279541,
      "activations/layer15_attention_weight_min": -21.487224578857422,
      "activations/layer16_attention_weight_max": 24.19782066345215,
      "activations/layer16_attention_weight_min": -18.7861385345459,
      "activations/layer17_attention_weight_max": 43.50046920776367,
      "activations/layer17_attention_weight_min": -28.884897232055664,
      "activations/layer18_attention_weight_max": 26.88496208190918,
      "activations/layer18_attention_weight_min": -23.49387550354004,
      "activations/layer19_attention_weight_max": 20.82549285888672,
      "activations/layer19_attention_weight_min": -17.297788619995117,
      "activations/layer1_attention_weight_max": 18.400230407714844,
      "activations/layer1_attention_weight_min": -10.798187255859375,
      "activations/layer20_attention_weight_max": 19.680391311645508,
      "activations/layer20_attention_weight_min": -18.609254837036133,
      "activations/layer21_attention_weight_max": 33.03425979614258,
      "activations/layer21_attention_weight_min": -22.96245765686035,
      "activations/layer22_attention_weight_max": 30.760726928710938,
      "activations/layer22_attention_weight_min": -18.89378547668457,
      "activations/layer23_attention_weight_max": 22.67898178100586,
      "activations/layer23_attention_weight_min": -19.082517623901367,
      "activations/layer2_attention_weight_max": 23.50237274169922,
      "activations/layer2_attention_weight_min": -19.428674697875977,
      "activations/layer3_attention_weight_max": 49.826995849609375,
      "activations/layer3_attention_weight_min": -43.193477630615234,
      "activations/layer4_attention_weight_max": 79.06920623779297,
      "activations/layer4_attention_weight_min": -71.71729278564453,
      "activations/layer5_attention_weight_max": 140.243408203125,
      "activations/layer5_attention_weight_min": -129.33071899414062,
      "activations/layer6_attention_weight_max": 87.16658782958984,
      "activations/layer6_attention_weight_min": -83.75279235839844,
      "activations/layer7_attention_weight_max": 81.72129821777344,
      "activations/layer7_attention_weight_min": -80.34262084960938,
      "activations/layer8_attention_weight_max": 65.50621795654297,
      "activations/layer8_attention_weight_min": -61.66630172729492,
      "activations/layer9_attention_weight_max": 69.11943817138672,
      "activations/layer9_attention_weight_min": -65.65080261230469,
      "epoch": 0.4,
      "learning_rate": 0.00014892878787878787,
      "loss": 3.3574,
      "step": 6850
    },
    {
      "activations/layer0_attention_weight_max": 13.358186721801758,
      "activations/layer0_attention_weight_min": -9.523544311523438,
      "activations/layer10_attention_weight_max": 66.22648620605469,
      "activations/layer10_attention_weight_min": -62.983978271484375,
      "activations/layer11_attention_weight_max": 37.99474334716797,
      "activations/layer11_attention_weight_min": -35.88334274291992,
      "activations/layer12_attention_weight_max": 23.48157501220703,
      "activations/layer12_attention_weight_min": -21.985918045043945,
      "activations/layer13_attention_weight_max": 39.05314636230469,
      "activations/layer13_attention_weight_min": -30.617738723754883,
      "activations/layer14_attention_weight_max": 29.386951446533203,
      "activations/layer14_attention_weight_min": -25.858705520629883,
      "activations/layer15_attention_weight_max": 33.17011642456055,
      "activations/layer15_attention_weight_min": -22.735450744628906,
      "activations/layer16_attention_weight_max": 25.767967224121094,
      "activations/layer16_attention_weight_min": -17.242403030395508,
      "activations/layer17_attention_weight_max": 36.905120849609375,
      "activations/layer17_attention_weight_min": -21.492210388183594,
      "activations/layer18_attention_weight_max": 25.334434509277344,
      "activations/layer18_attention_weight_min": -15.08559513092041,
      "activations/layer19_attention_weight_max": 22.240379333496094,
      "activations/layer19_attention_weight_min": -10.97316837310791,
      "activations/layer1_attention_weight_max": 18.35850715637207,
      "activations/layer1_attention_weight_min": -11.277865409851074,
      "activations/layer20_attention_weight_max": 18.517593383789062,
      "activations/layer20_attention_weight_min": -11.622106552124023,
      "activations/layer21_attention_weight_max": 26.081613540649414,
      "activations/layer21_attention_weight_min": -15.475954055786133,
      "activations/layer22_attention_weight_max": 26.982221603393555,
      "activations/layer22_attention_weight_min": -14.864913940429688,
      "activations/layer23_attention_weight_max": 19.85655975341797,
      "activations/layer23_attention_weight_min": -13.342669486999512,
      "activations/layer2_attention_weight_max": 25.301414489746094,
      "activations/layer2_attention_weight_min": -18.899253845214844,
      "activations/layer3_attention_weight_max": 52.54646682739258,
      "activations/layer3_attention_weight_min": -42.9111213684082,
      "activations/layer4_attention_weight_max": 84.00211334228516,
      "activations/layer4_attention_weight_min": -71.8005599975586,
      "activations/layer5_attention_weight_max": 148.00807189941406,
      "activations/layer5_attention_weight_min": -125.95763397216797,
      "activations/layer6_attention_weight_max": 92.90164184570312,
      "activations/layer6_attention_weight_min": -82.69256591796875,
      "activations/layer7_attention_weight_max": 84.90721130371094,
      "activations/layer7_attention_weight_min": -77.59677124023438,
      "activations/layer8_attention_weight_max": 71.54742431640625,
      "activations/layer8_attention_weight_min": -61.51885986328125,
      "activations/layer9_attention_weight_max": 77.4026107788086,
      "activations/layer9_attention_weight_min": -68.24726867675781,
      "epoch": 0.4,
      "learning_rate": 0.00014890984848484846,
      "loss": 3.3526,
      "step": 6900
    },
    {
      "activations/layer0_attention_weight_max": 14.693070411682129,
      "activations/layer0_attention_weight_min": -9.45885944366455,
      "activations/layer10_attention_weight_max": 59.13749313354492,
      "activations/layer10_attention_weight_min": -55.49551773071289,
      "activations/layer11_attention_weight_max": 40.804542541503906,
      "activations/layer11_attention_weight_min": -33.89680480957031,
      "activations/layer12_attention_weight_max": 21.804187774658203,
      "activations/layer12_attention_weight_min": -22.846105575561523,
      "activations/layer13_attention_weight_max": 34.961647033691406,
      "activations/layer13_attention_weight_min": -30.609827041625977,
      "activations/layer14_attention_weight_max": 30.255802154541016,
      "activations/layer14_attention_weight_min": -24.47734832763672,
      "activations/layer15_attention_weight_max": 29.824071884155273,
      "activations/layer15_attention_weight_min": -21.027572631835938,
      "activations/layer16_attention_weight_max": 23.007307052612305,
      "activations/layer16_attention_weight_min": -16.549325942993164,
      "activations/layer17_attention_weight_max": 38.5378532409668,
      "activations/layer17_attention_weight_min": -24.524940490722656,
      "activations/layer18_attention_weight_max": 39.812870025634766,
      "activations/layer18_attention_weight_min": -19.667390823364258,
      "activations/layer19_attention_weight_max": 20.698171615600586,
      "activations/layer19_attention_weight_min": -11.340256690979004,
      "activations/layer1_attention_weight_max": 17.049367904663086,
      "activations/layer1_attention_weight_min": -10.69343090057373,
      "activations/layer20_attention_weight_max": 21.085241317749023,
      "activations/layer20_attention_weight_min": -11.382330894470215,
      "activations/layer21_attention_weight_max": 30.308752059936523,
      "activations/layer21_attention_weight_min": -17.02508544921875,
      "activations/layer22_attention_weight_max": 25.50225830078125,
      "activations/layer22_attention_weight_min": -15.202621459960938,
      "activations/layer23_attention_weight_max": 19.912755966186523,
      "activations/layer23_attention_weight_min": -12.060809135437012,
      "activations/layer2_attention_weight_max": 25.42915153503418,
      "activations/layer2_attention_weight_min": -19.9610538482666,
      "activations/layer3_attention_weight_max": 51.8904914855957,
      "activations/layer3_attention_weight_min": -43.0529670715332,
      "activations/layer4_attention_weight_max": 75.97225189208984,
      "activations/layer4_attention_weight_min": -72.24080657958984,
      "activations/layer5_attention_weight_max": 143.38983154296875,
      "activations/layer5_attention_weight_min": -131.50692749023438,
      "activations/layer6_attention_weight_max": 88.92046356201172,
      "activations/layer6_attention_weight_min": -88.09479522705078,
      "activations/layer7_attention_weight_max": 84.06002044677734,
      "activations/layer7_attention_weight_min": -78.78276824951172,
      "activations/layer8_attention_weight_max": 66.65185546875,
      "activations/layer8_attention_weight_min": -59.76222610473633,
      "activations/layer9_attention_weight_max": 80.49238586425781,
      "activations/layer9_attention_weight_min": -70.72798156738281,
      "epoch": 0.4,
      "learning_rate": 0.00014889090909090907,
      "loss": 3.3444,
      "step": 6950
    },
    {
      "activations/layer0_attention_weight_max": 13.764240264892578,
      "activations/layer0_attention_weight_min": -9.658295631408691,
      "activations/layer10_attention_weight_max": 73.5185546875,
      "activations/layer10_attention_weight_min": -58.06864929199219,
      "activations/layer11_attention_weight_max": 47.66863250732422,
      "activations/layer11_attention_weight_min": -32.808563232421875,
      "activations/layer12_attention_weight_max": 27.335060119628906,
      "activations/layer12_attention_weight_min": -24.148807525634766,
      "activations/layer13_attention_weight_max": 46.98733901977539,
      "activations/layer13_attention_weight_min": -29.862579345703125,
      "activations/layer14_attention_weight_max": 36.52866744995117,
      "activations/layer14_attention_weight_min": -24.174732208251953,
      "activations/layer15_attention_weight_max": 37.200904846191406,
      "activations/layer15_attention_weight_min": -23.946277618408203,
      "activations/layer16_attention_weight_max": 28.279056549072266,
      "activations/layer16_attention_weight_min": -16.104745864868164,
      "activations/layer17_attention_weight_max": 35.06807327270508,
      "activations/layer17_attention_weight_min": -23.389345169067383,
      "activations/layer18_attention_weight_max": 39.15509796142578,
      "activations/layer18_attention_weight_min": -23.332420349121094,
      "activations/layer19_attention_weight_max": 22.216100692749023,
      "activations/layer19_attention_weight_min": -13.161410331726074,
      "activations/layer1_attention_weight_max": 17.928953170776367,
      "activations/layer1_attention_weight_min": -11.26213550567627,
      "activations/layer20_attention_weight_max": 18.13010025024414,
      "activations/layer20_attention_weight_min": -12.429764747619629,
      "activations/layer21_attention_weight_max": 21.588165283203125,
      "activations/layer21_attention_weight_min": -16.75235366821289,
      "activations/layer22_attention_weight_max": 19.698354721069336,
      "activations/layer22_attention_weight_min": -14.116772651672363,
      "activations/layer23_attention_weight_max": 17.12662696838379,
      "activations/layer23_attention_weight_min": -13.168773651123047,
      "activations/layer2_attention_weight_max": 26.73903465270996,
      "activations/layer2_attention_weight_min": -18.845197677612305,
      "activations/layer3_attention_weight_max": 55.13335037231445,
      "activations/layer3_attention_weight_min": -42.35593795776367,
      "activations/layer4_attention_weight_max": 83.4039535522461,
      "activations/layer4_attention_weight_min": -70.44585418701172,
      "activations/layer5_attention_weight_max": 158.88174438476562,
      "activations/layer5_attention_weight_min": -127.88106536865234,
      "activations/layer6_attention_weight_max": 96.42984008789062,
      "activations/layer6_attention_weight_min": -87.42082977294922,
      "activations/layer7_attention_weight_max": 94.50138092041016,
      "activations/layer7_attention_weight_min": -76.29019927978516,
      "activations/layer8_attention_weight_max": 75.8292236328125,
      "activations/layer8_attention_weight_min": -65.31028747558594,
      "activations/layer9_attention_weight_max": 79.80636596679688,
      "activations/layer9_attention_weight_min": -64.95958709716797,
      "epoch": 0.41,
      "learning_rate": 0.00014887196969696969,
      "loss": 3.3381,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_loss": 3.2890625,
      "eval_runtime": 8.1988,
      "eval_samples_per_second": 523.738,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_openwebtext_loss": 3.2890625,
      "eval_openwebtext_ppl": 26.817710263716034,
      "eval_openwebtext_runtime": 8.1988,
      "eval_openwebtext_samples_per_second": 523.738,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_wikitext_loss": 3.615234375,
      "eval_wikitext_ppl": 37.160054292819765,
      "eval_wikitext_runtime": 1.6694,
      "eval_wikitext_samples_per_second": 273.15,
      "step": 7000
    },
    {
      "epoch": 0.41,
      "eval_lambada_loss": 4.12109375,
      "eval_lambada_ppl": 61.626609520407456,
      "eval_lambada_runtime": 9.2546,
      "eval_lambada_samples_per_second": 526.115,
      "step": 7000
    },
    {
      "activations/layer0_attention_weight_max": 13.362887382507324,
      "activations/layer0_attention_weight_min": -9.685064315795898,
      "activations/layer10_attention_weight_max": 60.00214385986328,
      "activations/layer10_attention_weight_min": -54.59215545654297,
      "activations/layer11_attention_weight_max": 37.12126159667969,
      "activations/layer11_attention_weight_min": -32.07558059692383,
      "activations/layer12_attention_weight_max": 25.1716365814209,
      "activations/layer12_attention_weight_min": -22.852933883666992,
      "activations/layer13_attention_weight_max": 37.120399475097656,
      "activations/layer13_attention_weight_min": -33.344852447509766,
      "activations/layer14_attention_weight_max": 27.993793487548828,
      "activations/layer14_attention_weight_min": -22.30080223083496,
      "activations/layer15_attention_weight_max": 27.402584075927734,
      "activations/layer15_attention_weight_min": -24.906951904296875,
      "activations/layer16_attention_weight_max": 20.490097045898438,
      "activations/layer16_attention_weight_min": -19.125228881835938,
      "activations/layer17_attention_weight_max": 33.67403030395508,
      "activations/layer17_attention_weight_min": -28.767879486083984,
      "activations/layer18_attention_weight_max": 32.971622467041016,
      "activations/layer18_attention_weight_min": -18.765140533447266,
      "activations/layer19_attention_weight_max": 19.171825408935547,
      "activations/layer19_attention_weight_min": -11.028307914733887,
      "activations/layer1_attention_weight_max": 18.227710723876953,
      "activations/layer1_attention_weight_min": -11.011466026306152,
      "activations/layer20_attention_weight_max": 17.427722930908203,
      "activations/layer20_attention_weight_min": -12.922590255737305,
      "activations/layer21_attention_weight_max": 23.519412994384766,
      "activations/layer21_attention_weight_min": -15.514440536499023,
      "activations/layer22_attention_weight_max": 22.286457061767578,
      "activations/layer22_attention_weight_min": -14.0518798828125,
      "activations/layer23_attention_weight_max": 18.163433074951172,
      "activations/layer23_attention_weight_min": -14.129939079284668,
      "activations/layer2_attention_weight_max": 24.243682861328125,
      "activations/layer2_attention_weight_min": -19.743152618408203,
      "activations/layer3_attention_weight_max": 51.86943054199219,
      "activations/layer3_attention_weight_min": -43.174495697021484,
      "activations/layer4_attention_weight_max": 83.9970932006836,
      "activations/layer4_attention_weight_min": -71.28446197509766,
      "activations/layer5_attention_weight_max": 148.39768981933594,
      "activations/layer5_attention_weight_min": -126.70022583007812,
      "activations/layer6_attention_weight_max": 91.6451187133789,
      "activations/layer6_attention_weight_min": -81.80938720703125,
      "activations/layer7_attention_weight_max": 82.79120635986328,
      "activations/layer7_attention_weight_min": -75.34712982177734,
      "activations/layer8_attention_weight_max": 66.51263427734375,
      "activations/layer8_attention_weight_min": -58.09539794921875,
      "activations/layer9_attention_weight_max": 68.92472839355469,
      "activations/layer9_attention_weight_min": -64.96730041503906,
      "epoch": 0.41,
      "learning_rate": 0.00014885303030303027,
      "loss": 3.3345,
      "step": 7050
    },
    {
      "activations/layer0_attention_weight_max": 14.27627944946289,
      "activations/layer0_attention_weight_min": -9.323814392089844,
      "activations/layer10_attention_weight_max": 59.63712692260742,
      "activations/layer10_attention_weight_min": -57.83891296386719,
      "activations/layer11_attention_weight_max": 38.40033721923828,
      "activations/layer11_attention_weight_min": -30.438453674316406,
      "activations/layer12_attention_weight_max": 22.628110885620117,
      "activations/layer12_attention_weight_min": -21.5804443359375,
      "activations/layer13_attention_weight_max": 39.453590393066406,
      "activations/layer13_attention_weight_min": -30.563989639282227,
      "activations/layer14_attention_weight_max": 31.02813148498535,
      "activations/layer14_attention_weight_min": -20.98299217224121,
      "activations/layer15_attention_weight_max": 30.521879196166992,
      "activations/layer15_attention_weight_min": -22.161304473876953,
      "activations/layer16_attention_weight_max": 25.093603134155273,
      "activations/layer16_attention_weight_min": -16.17822265625,
      "activations/layer17_attention_weight_max": 35.78935623168945,
      "activations/layer17_attention_weight_min": -25.312503814697266,
      "activations/layer18_attention_weight_max": 43.38268280029297,
      "activations/layer18_attention_weight_min": -23.61522674560547,
      "activations/layer19_attention_weight_max": 21.87702178955078,
      "activations/layer19_attention_weight_min": -11.92855167388916,
      "activations/layer1_attention_weight_max": 18.858278274536133,
      "activations/layer1_attention_weight_min": -11.543943405151367,
      "activations/layer20_attention_weight_max": 19.50560188293457,
      "activations/layer20_attention_weight_min": -12.658587455749512,
      "activations/layer21_attention_weight_max": 25.54213523864746,
      "activations/layer21_attention_weight_min": -15.802022933959961,
      "activations/layer22_attention_weight_max": 23.22715187072754,
      "activations/layer22_attention_weight_min": -12.660078048706055,
      "activations/layer23_attention_weight_max": 18.949981689453125,
      "activations/layer23_attention_weight_min": -13.93646240234375,
      "activations/layer2_attention_weight_max": 24.000089645385742,
      "activations/layer2_attention_weight_min": -19.162796020507812,
      "activations/layer3_attention_weight_max": 52.61198806762695,
      "activations/layer3_attention_weight_min": -44.663291931152344,
      "activations/layer4_attention_weight_max": 81.81571960449219,
      "activations/layer4_attention_weight_min": -73.79125213623047,
      "activations/layer5_attention_weight_max": 158.80557250976562,
      "activations/layer5_attention_weight_min": -132.45823669433594,
      "activations/layer6_attention_weight_max": 100.34477996826172,
      "activations/layer6_attention_weight_min": -83.45553588867188,
      "activations/layer7_attention_weight_max": 89.21180725097656,
      "activations/layer7_attention_weight_min": -76.0859375,
      "activations/layer8_attention_weight_max": 76.2725830078125,
      "activations/layer8_attention_weight_min": -59.86385726928711,
      "activations/layer9_attention_weight_max": 72.52033996582031,
      "activations/layer9_attention_weight_min": -64.41226196289062,
      "epoch": 0.41,
      "learning_rate": 0.0001488340909090909,
      "loss": 3.3374,
      "step": 7100
    },
    {
      "activations/layer0_attention_weight_max": 14.224847793579102,
      "activations/layer0_attention_weight_min": -9.499247550964355,
      "activations/layer10_attention_weight_max": 65.71434020996094,
      "activations/layer10_attention_weight_min": -59.88658905029297,
      "activations/layer11_attention_weight_max": 39.07899475097656,
      "activations/layer11_attention_weight_min": -30.704723358154297,
      "activations/layer12_attention_weight_max": 21.737882614135742,
      "activations/layer12_attention_weight_min": -21.673053741455078,
      "activations/layer13_attention_weight_max": 42.52680206298828,
      "activations/layer13_attention_weight_min": -32.348445892333984,
      "activations/layer14_attention_weight_max": 31.80860137939453,
      "activations/layer14_attention_weight_min": -21.979135513305664,
      "activations/layer15_attention_weight_max": 33.48081588745117,
      "activations/layer15_attention_weight_min": -21.63418197631836,
      "activations/layer16_attention_weight_max": 23.8002872467041,
      "activations/layer16_attention_weight_min": -16.332454681396484,
      "activations/layer17_attention_weight_max": 40.42036056518555,
      "activations/layer17_attention_weight_min": -24.044551849365234,
      "activations/layer18_attention_weight_max": 35.91699981689453,
      "activations/layer18_attention_weight_min": -23.5024471282959,
      "activations/layer19_attention_weight_max": 22.151796340942383,
      "activations/layer19_attention_weight_min": -11.649508476257324,
      "activations/layer1_attention_weight_max": 18.471193313598633,
      "activations/layer1_attention_weight_min": -10.956987380981445,
      "activations/layer20_attention_weight_max": 20.05263328552246,
      "activations/layer20_attention_weight_min": -11.469208717346191,
      "activations/layer21_attention_weight_max": 29.651262283325195,
      "activations/layer21_attention_weight_min": -15.085591316223145,
      "activations/layer22_attention_weight_max": 24.61517906188965,
      "activations/layer22_attention_weight_min": -13.73238754272461,
      "activations/layer23_attention_weight_max": 17.079200744628906,
      "activations/layer23_attention_weight_min": -13.139144897460938,
      "activations/layer2_attention_weight_max": 25.857479095458984,
      "activations/layer2_attention_weight_min": -19.078466415405273,
      "activations/layer3_attention_weight_max": 53.06644058227539,
      "activations/layer3_attention_weight_min": -42.850433349609375,
      "activations/layer4_attention_weight_max": 78.32972717285156,
      "activations/layer4_attention_weight_min": -72.48336029052734,
      "activations/layer5_attention_weight_max": 142.75115966796875,
      "activations/layer5_attention_weight_min": -125.3629379272461,
      "activations/layer6_attention_weight_max": 89.70719146728516,
      "activations/layer6_attention_weight_min": -83.37486267089844,
      "activations/layer7_attention_weight_max": 81.72472381591797,
      "activations/layer7_attention_weight_min": -76.53468322753906,
      "activations/layer8_attention_weight_max": 72.73975372314453,
      "activations/layer8_attention_weight_min": -60.07251739501953,
      "activations/layer9_attention_weight_max": 73.91234588623047,
      "activations/layer9_attention_weight_min": -68.7686538696289,
      "epoch": 0.42,
      "learning_rate": 0.0001488151515151515,
      "loss": 3.3387,
      "step": 7150
    },
    {
      "activations/layer0_attention_weight_max": 14.472846031188965,
      "activations/layer0_attention_weight_min": -10.022391319274902,
      "activations/layer10_attention_weight_max": 61.32981872558594,
      "activations/layer10_attention_weight_min": -58.24869918823242,
      "activations/layer11_attention_weight_max": 39.97917175292969,
      "activations/layer11_attention_weight_min": -31.056350708007812,
      "activations/layer12_attention_weight_max": 21.159622192382812,
      "activations/layer12_attention_weight_min": -23.496122360229492,
      "activations/layer13_attention_weight_max": 44.75676727294922,
      "activations/layer13_attention_weight_min": -32.03047561645508,
      "activations/layer14_attention_weight_max": 28.956493377685547,
      "activations/layer14_attention_weight_min": -22.719165802001953,
      "activations/layer15_attention_weight_max": 32.49433135986328,
      "activations/layer15_attention_weight_min": -22.61186408996582,
      "activations/layer16_attention_weight_max": 22.905431747436523,
      "activations/layer16_attention_weight_min": -16.793434143066406,
      "activations/layer17_attention_weight_max": 35.71476364135742,
      "activations/layer17_attention_weight_min": -26.880510330200195,
      "activations/layer18_attention_weight_max": 31.49782943725586,
      "activations/layer18_attention_weight_min": -19.081506729125977,
      "activations/layer19_attention_weight_max": 20.560705184936523,
      "activations/layer19_attention_weight_min": -10.295843124389648,
      "activations/layer1_attention_weight_max": 17.972747802734375,
      "activations/layer1_attention_weight_min": -10.751032829284668,
      "activations/layer20_attention_weight_max": 18.050813674926758,
      "activations/layer20_attention_weight_min": -11.210872650146484,
      "activations/layer21_attention_weight_max": 31.199491500854492,
      "activations/layer21_attention_weight_min": -13.15199089050293,
      "activations/layer22_attention_weight_max": 33.19349670410156,
      "activations/layer22_attention_weight_min": -13.324647903442383,
      "activations/layer23_attention_weight_max": 24.479019165039062,
      "activations/layer23_attention_weight_min": -12.148199081420898,
      "activations/layer2_attention_weight_max": 25.20862579345703,
      "activations/layer2_attention_weight_min": -19.873554229736328,
      "activations/layer3_attention_weight_max": 52.79274368286133,
      "activations/layer3_attention_weight_min": -43.9214973449707,
      "activations/layer4_attention_weight_max": 81.70067596435547,
      "activations/layer4_attention_weight_min": -71.87114715576172,
      "activations/layer5_attention_weight_max": 145.60488891601562,
      "activations/layer5_attention_weight_min": -129.32260131835938,
      "activations/layer6_attention_weight_max": 87.57231140136719,
      "activations/layer6_attention_weight_min": -79.56929016113281,
      "activations/layer7_attention_weight_max": 81.60794067382812,
      "activations/layer7_attention_weight_min": -75.1758041381836,
      "activations/layer8_attention_weight_max": 62.44744873046875,
      "activations/layer8_attention_weight_min": -59.79348373413086,
      "activations/layer9_attention_weight_max": 69.02346801757812,
      "activations/layer9_attention_weight_min": -67.75865936279297,
      "epoch": 0.42,
      "learning_rate": 0.0001487962121212121,
      "loss": 3.356,
      "step": 7200
    },
    {
      "activations/layer0_attention_weight_max": 13.470682144165039,
      "activations/layer0_attention_weight_min": -10.120706558227539,
      "activations/layer10_attention_weight_max": 61.40482711791992,
      "activations/layer10_attention_weight_min": -56.5638542175293,
      "activations/layer11_attention_weight_max": 39.814414978027344,
      "activations/layer11_attention_weight_min": -30.41901397705078,
      "activations/layer12_attention_weight_max": 23.118478775024414,
      "activations/layer12_attention_weight_min": -23.25945281982422,
      "activations/layer13_attention_weight_max": 38.475868225097656,
      "activations/layer13_attention_weight_min": -32.472633361816406,
      "activations/layer14_attention_weight_max": 27.754962921142578,
      "activations/layer14_attention_weight_min": -24.100370407104492,
      "activations/layer15_attention_weight_max": 29.51781463623047,
      "activations/layer15_attention_weight_min": -23.108930587768555,
      "activations/layer16_attention_weight_max": 22.8568058013916,
      "activations/layer16_attention_weight_min": -18.209800720214844,
      "activations/layer17_attention_weight_max": 36.0844612121582,
      "activations/layer17_attention_weight_min": -26.140262603759766,
      "activations/layer18_attention_weight_max": 39.72758483886719,
      "activations/layer18_attention_weight_min": -21.54775619506836,
      "activations/layer19_attention_weight_max": 21.183164596557617,
      "activations/layer19_attention_weight_min": -11.48501968383789,
      "activations/layer1_attention_weight_max": 18.632659912109375,
      "activations/layer1_attention_weight_min": -10.905259132385254,
      "activations/layer20_attention_weight_max": 19.43254852294922,
      "activations/layer20_attention_weight_min": -11.418004989624023,
      "activations/layer21_attention_weight_max": 32.90463638305664,
      "activations/layer21_attention_weight_min": -16.110939025878906,
      "activations/layer22_attention_weight_max": 29.811744689941406,
      "activations/layer22_attention_weight_min": -14.220818519592285,
      "activations/layer23_attention_weight_max": 21.189563751220703,
      "activations/layer23_attention_weight_min": -12.829715728759766,
      "activations/layer2_attention_weight_max": 26.366872787475586,
      "activations/layer2_attention_weight_min": -19.281204223632812,
      "activations/layer3_attention_weight_max": 54.392601013183594,
      "activations/layer3_attention_weight_min": -43.329803466796875,
      "activations/layer4_attention_weight_max": 83.27350616455078,
      "activations/layer4_attention_weight_min": -72.30622100830078,
      "activations/layer5_attention_weight_max": 149.8072052001953,
      "activations/layer5_attention_weight_min": -132.48748779296875,
      "activations/layer6_attention_weight_max": 90.53610229492188,
      "activations/layer6_attention_weight_min": -80.56627655029297,
      "activations/layer7_attention_weight_max": 84.00392150878906,
      "activations/layer7_attention_weight_min": -76.03666687011719,
      "activations/layer8_attention_weight_max": 69.54132080078125,
      "activations/layer8_attention_weight_min": -60.25482940673828,
      "activations/layer9_attention_weight_max": 70.4958724975586,
      "activations/layer9_attention_weight_min": -70.4881591796875,
      "epoch": 0.42,
      "learning_rate": 0.0001487772727272727,
      "loss": 3.335,
      "step": 7250
    },
    {
      "activations/layer0_attention_weight_max": 13.907613754272461,
      "activations/layer0_attention_weight_min": -9.282835960388184,
      "activations/layer10_attention_weight_max": 64.43633270263672,
      "activations/layer10_attention_weight_min": -58.8485107421875,
      "activations/layer11_attention_weight_max": 42.33262252807617,
      "activations/layer11_attention_weight_min": -33.75433349609375,
      "activations/layer12_attention_weight_max": 27.215784072875977,
      "activations/layer12_attention_weight_min": -22.964534759521484,
      "activations/layer13_attention_weight_max": 41.803985595703125,
      "activations/layer13_attention_weight_min": -32.013710021972656,
      "activations/layer14_attention_weight_max": 26.468090057373047,
      "activations/layer14_attention_weight_min": -21.666179656982422,
      "activations/layer15_attention_weight_max": 30.060880661010742,
      "activations/layer15_attention_weight_min": -21.231374740600586,
      "activations/layer16_attention_weight_max": 24.130908966064453,
      "activations/layer16_attention_weight_min": -19.06661033630371,
      "activations/layer17_attention_weight_max": 36.4274787902832,
      "activations/layer17_attention_weight_min": -25.683277130126953,
      "activations/layer18_attention_weight_max": 44.28193664550781,
      "activations/layer18_attention_weight_min": -24.383544921875,
      "activations/layer19_attention_weight_max": 19.155860900878906,
      "activations/layer19_attention_weight_min": -14.071310997009277,
      "activations/layer1_attention_weight_max": 18.17238998413086,
      "activations/layer1_attention_weight_min": -11.40780258178711,
      "activations/layer20_attention_weight_max": 17.094200134277344,
      "activations/layer20_attention_weight_min": -15.698599815368652,
      "activations/layer21_attention_weight_max": 29.298187255859375,
      "activations/layer21_attention_weight_min": -18.45768928527832,
      "activations/layer22_attention_weight_max": 29.94196319580078,
      "activations/layer22_attention_weight_min": -16.66245460510254,
      "activations/layer23_attention_weight_max": 21.77292251586914,
      "activations/layer23_attention_weight_min": -14.409754753112793,
      "activations/layer2_attention_weight_max": 25.545209884643555,
      "activations/layer2_attention_weight_min": -20.898252487182617,
      "activations/layer3_attention_weight_max": 54.956031799316406,
      "activations/layer3_attention_weight_min": -43.63983154296875,
      "activations/layer4_attention_weight_max": 83.48847198486328,
      "activations/layer4_attention_weight_min": -72.5949935913086,
      "activations/layer5_attention_weight_max": 153.86276245117188,
      "activations/layer5_attention_weight_min": -128.79718017578125,
      "activations/layer6_attention_weight_max": 91.07450866699219,
      "activations/layer6_attention_weight_min": -80.23513793945312,
      "activations/layer7_attention_weight_max": 91.08775329589844,
      "activations/layer7_attention_weight_min": -75.9578857421875,
      "activations/layer8_attention_weight_max": 69.83268737792969,
      "activations/layer8_attention_weight_min": -63.75699234008789,
      "activations/layer9_attention_weight_max": 78.31734466552734,
      "activations/layer9_attention_weight_min": -68.91497802734375,
      "epoch": 0.42,
      "learning_rate": 0.00014875833333333332,
      "loss": 3.3277,
      "step": 7300
    },
    {
      "activations/layer0_attention_weight_max": 14.545772552490234,
      "activations/layer0_attention_weight_min": -8.939031600952148,
      "activations/layer10_attention_weight_max": 64.47900390625,
      "activations/layer10_attention_weight_min": -57.06883239746094,
      "activations/layer11_attention_weight_max": 39.16444396972656,
      "activations/layer11_attention_weight_min": -32.56534194946289,
      "activations/layer12_attention_weight_max": 25.11009407043457,
      "activations/layer12_attention_weight_min": -20.747636795043945,
      "activations/layer13_attention_weight_max": 41.345306396484375,
      "activations/layer13_attention_weight_min": -29.90932846069336,
      "activations/layer14_attention_weight_max": 30.58024024963379,
      "activations/layer14_attention_weight_min": -23.441242218017578,
      "activations/layer15_attention_weight_max": 30.887144088745117,
      "activations/layer15_attention_weight_min": -21.605443954467773,
      "activations/layer16_attention_weight_max": 20.962806701660156,
      "activations/layer16_attention_weight_min": -18.49358367919922,
      "activations/layer17_attention_weight_max": 31.69498062133789,
      "activations/layer17_attention_weight_min": -30.52037239074707,
      "activations/layer18_attention_weight_max": 26.993389129638672,
      "activations/layer18_attention_weight_min": -15.471999168395996,
      "activations/layer19_attention_weight_max": 19.696985244750977,
      "activations/layer19_attention_weight_min": -11.165425300598145,
      "activations/layer1_attention_weight_max": 17.78656768798828,
      "activations/layer1_attention_weight_min": -10.961390495300293,
      "activations/layer20_attention_weight_max": 18.221900939941406,
      "activations/layer20_attention_weight_min": -13.497783660888672,
      "activations/layer21_attention_weight_max": 26.316646575927734,
      "activations/layer21_attention_weight_min": -15.562450408935547,
      "activations/layer22_attention_weight_max": 26.4482364654541,
      "activations/layer22_attention_weight_min": -14.13265323638916,
      "activations/layer23_attention_weight_max": 19.527889251708984,
      "activations/layer23_attention_weight_min": -14.265002250671387,
      "activations/layer2_attention_weight_max": 27.2996826171875,
      "activations/layer2_attention_weight_min": -20.180803298950195,
      "activations/layer3_attention_weight_max": 55.3942756652832,
      "activations/layer3_attention_weight_min": -44.48967361450195,
      "activations/layer4_attention_weight_max": 84.20586395263672,
      "activations/layer4_attention_weight_min": -71.08251190185547,
      "activations/layer5_attention_weight_max": 149.98751831054688,
      "activations/layer5_attention_weight_min": -127.63430786132812,
      "activations/layer6_attention_weight_max": 92.08499908447266,
      "activations/layer6_attention_weight_min": -81.79096984863281,
      "activations/layer7_attention_weight_max": 82.4622573852539,
      "activations/layer7_attention_weight_min": -73.78551483154297,
      "activations/layer8_attention_weight_max": 71.3437271118164,
      "activations/layer8_attention_weight_min": -61.1456184387207,
      "activations/layer9_attention_weight_max": 75.71427917480469,
      "activations/layer9_attention_weight_min": -63.565185546875,
      "epoch": 0.43,
      "learning_rate": 0.00014873939393939394,
      "loss": 3.3202,
      "step": 7350
    },
    {
      "activations/layer0_attention_weight_max": 13.52713394165039,
      "activations/layer0_attention_weight_min": -9.404104232788086,
      "activations/layer10_attention_weight_max": 65.19122314453125,
      "activations/layer10_attention_weight_min": -62.33665084838867,
      "activations/layer11_attention_weight_max": 38.662841796875,
      "activations/layer11_attention_weight_min": -38.22466278076172,
      "activations/layer12_attention_weight_max": 23.283079147338867,
      "activations/layer12_attention_weight_min": -20.48346710205078,
      "activations/layer13_attention_weight_max": 44.83634948730469,
      "activations/layer13_attention_weight_min": -34.73625564575195,
      "activations/layer14_attention_weight_max": 29.051647186279297,
      "activations/layer14_attention_weight_min": -22.663232803344727,
      "activations/layer15_attention_weight_max": 30.753454208374023,
      "activations/layer15_attention_weight_min": -23.754018783569336,
      "activations/layer16_attention_weight_max": 23.6795654296875,
      "activations/layer16_attention_weight_min": -18.90212631225586,
      "activations/layer17_attention_weight_max": 36.65104293823242,
      "activations/layer17_attention_weight_min": -28.1312198638916,
      "activations/layer18_attention_weight_max": 38.40908432006836,
      "activations/layer18_attention_weight_min": -25.2891902923584,
      "activations/layer19_attention_weight_max": 19.60845947265625,
      "activations/layer19_attention_weight_min": -10.239964485168457,
      "activations/layer1_attention_weight_max": 19.075965881347656,
      "activations/layer1_attention_weight_min": -11.222872734069824,
      "activations/layer20_attention_weight_max": 17.74125862121582,
      "activations/layer20_attention_weight_min": -11.279547691345215,
      "activations/layer21_attention_weight_max": 26.460416793823242,
      "activations/layer21_attention_weight_min": -16.28754234313965,
      "activations/layer22_attention_weight_max": 30.811969757080078,
      "activations/layer22_attention_weight_min": -14.0360689163208,
      "activations/layer23_attention_weight_max": 22.547378540039062,
      "activations/layer23_attention_weight_min": -12.290633201599121,
      "activations/layer2_attention_weight_max": 26.30799674987793,
      "activations/layer2_attention_weight_min": -20.336261749267578,
      "activations/layer3_attention_weight_max": 51.344051361083984,
      "activations/layer3_attention_weight_min": -43.06139373779297,
      "activations/layer4_attention_weight_max": 79.48661804199219,
      "activations/layer4_attention_weight_min": -71.44274139404297,
      "activations/layer5_attention_weight_max": 144.78475952148438,
      "activations/layer5_attention_weight_min": -127.496337890625,
      "activations/layer6_attention_weight_max": 87.5898208618164,
      "activations/layer6_attention_weight_min": -80.04840087890625,
      "activations/layer7_attention_weight_max": 87.77869415283203,
      "activations/layer7_attention_weight_min": -78.08052062988281,
      "activations/layer8_attention_weight_max": 65.43901062011719,
      "activations/layer8_attention_weight_min": -58.63392639160156,
      "activations/layer9_attention_weight_max": 77.6174545288086,
      "activations/layer9_attention_weight_min": -68.05393981933594,
      "epoch": 0.43,
      "learning_rate": 0.00014872045454545452,
      "loss": 3.3442,
      "step": 7400
    },
    {
      "activations/layer0_attention_weight_max": 14.951250076293945,
      "activations/layer0_attention_weight_min": -9.28872013092041,
      "activations/layer10_attention_weight_max": 63.84607696533203,
      "activations/layer10_attention_weight_min": -58.50142288208008,
      "activations/layer11_attention_weight_max": 39.02100372314453,
      "activations/layer11_attention_weight_min": -32.585567474365234,
      "activations/layer12_attention_weight_max": 27.154027938842773,
      "activations/layer12_attention_weight_min": -22.327022552490234,
      "activations/layer13_attention_weight_max": 42.650291442871094,
      "activations/layer13_attention_weight_min": -31.894826889038086,
      "activations/layer14_attention_weight_max": 32.494483947753906,
      "activations/layer14_attention_weight_min": -23.7023868560791,
      "activations/layer15_attention_weight_max": 33.019222259521484,
      "activations/layer15_attention_weight_min": -23.456148147583008,
      "activations/layer16_attention_weight_max": 25.317569732666016,
      "activations/layer16_attention_weight_min": -18.436473846435547,
      "activations/layer17_attention_weight_max": 45.0632209777832,
      "activations/layer17_attention_weight_min": -29.11294937133789,
      "activations/layer18_attention_weight_max": 46.849403381347656,
      "activations/layer18_attention_weight_min": -27.109594345092773,
      "activations/layer19_attention_weight_max": 23.18338394165039,
      "activations/layer19_attention_weight_min": -12.053167343139648,
      "activations/layer1_attention_weight_max": 18.58494758605957,
      "activations/layer1_attention_weight_min": -11.040839195251465,
      "activations/layer20_attention_weight_max": 20.55895233154297,
      "activations/layer20_attention_weight_min": -13.12805461883545,
      "activations/layer21_attention_weight_max": 29.548233032226562,
      "activations/layer21_attention_weight_min": -15.222814559936523,
      "activations/layer22_attention_weight_max": 30.11275291442871,
      "activations/layer22_attention_weight_min": -17.2053279876709,
      "activations/layer23_attention_weight_max": 22.8961238861084,
      "activations/layer23_attention_weight_min": -15.487110137939453,
      "activations/layer2_attention_weight_max": 25.030187606811523,
      "activations/layer2_attention_weight_min": -20.146728515625,
      "activations/layer3_attention_weight_max": 54.149940490722656,
      "activations/layer3_attention_weight_min": -43.668190002441406,
      "activations/layer4_attention_weight_max": 80.52174377441406,
      "activations/layer4_attention_weight_min": -71.04308319091797,
      "activations/layer5_attention_weight_max": 143.88902282714844,
      "activations/layer5_attention_weight_min": -125.43254852294922,
      "activations/layer6_attention_weight_max": 87.25371551513672,
      "activations/layer6_attention_weight_min": -78.7680435180664,
      "activations/layer7_attention_weight_max": 83.70245361328125,
      "activations/layer7_attention_weight_min": -72.48538970947266,
      "activations/layer8_attention_weight_max": 65.48890686035156,
      "activations/layer8_attention_weight_min": -57.97532272338867,
      "activations/layer9_attention_weight_max": 72.36276245117188,
      "activations/layer9_attention_weight_min": -70.19313049316406,
      "epoch": 0.43,
      "learning_rate": 0.00014870151515151514,
      "loss": 3.3313,
      "step": 7450
    },
    {
      "activations/layer0_attention_weight_max": 13.456430435180664,
      "activations/layer0_attention_weight_min": -9.709835052490234,
      "activations/layer10_attention_weight_max": 62.402305603027344,
      "activations/layer10_attention_weight_min": -56.80231475830078,
      "activations/layer11_attention_weight_max": 40.182498931884766,
      "activations/layer11_attention_weight_min": -32.05708694458008,
      "activations/layer12_attention_weight_max": 23.09292221069336,
      "activations/layer12_attention_weight_min": -22.947284698486328,
      "activations/layer13_attention_weight_max": 42.1971549987793,
      "activations/layer13_attention_weight_min": -32.7646598815918,
      "activations/layer14_attention_weight_max": 32.09832000732422,
      "activations/layer14_attention_weight_min": -21.717079162597656,
      "activations/layer15_attention_weight_max": 34.06259536743164,
      "activations/layer15_attention_weight_min": -20.330801010131836,
      "activations/layer16_attention_weight_max": 24.520166397094727,
      "activations/layer16_attention_weight_min": -17.909013748168945,
      "activations/layer17_attention_weight_max": 38.56949234008789,
      "activations/layer17_attention_weight_min": -25.492273330688477,
      "activations/layer18_attention_weight_max": 38.749691009521484,
      "activations/layer18_attention_weight_min": -20.545440673828125,
      "activations/layer19_attention_weight_max": 20.66016387939453,
      "activations/layer19_attention_weight_min": -10.305122375488281,
      "activations/layer1_attention_weight_max": 17.476133346557617,
      "activations/layer1_attention_weight_min": -10.88735580444336,
      "activations/layer20_attention_weight_max": 18.289766311645508,
      "activations/layer20_attention_weight_min": -12.457371711730957,
      "activations/layer21_attention_weight_max": 33.09366226196289,
      "activations/layer21_attention_weight_min": -15.050979614257812,
      "activations/layer22_attention_weight_max": 32.689056396484375,
      "activations/layer22_attention_weight_min": -15.342719078063965,
      "activations/layer23_attention_weight_max": 22.90315055847168,
      "activations/layer23_attention_weight_min": -13.656745910644531,
      "activations/layer2_attention_weight_max": 26.61621856689453,
      "activations/layer2_attention_weight_min": -21.469139099121094,
      "activations/layer3_attention_weight_max": 56.65723419189453,
      "activations/layer3_attention_weight_min": -45.22220993041992,
      "activations/layer4_attention_weight_max": 85.51870727539062,
      "activations/layer4_attention_weight_min": -72.17411804199219,
      "activations/layer5_attention_weight_max": 151.34852600097656,
      "activations/layer5_attention_weight_min": -125.10069274902344,
      "activations/layer6_attention_weight_max": 86.86056518554688,
      "activations/layer6_attention_weight_min": -79.26104736328125,
      "activations/layer7_attention_weight_max": 84.87490844726562,
      "activations/layer7_attention_weight_min": -70.56714630126953,
      "activations/layer8_attention_weight_max": 66.93478393554688,
      "activations/layer8_attention_weight_min": -59.33781051635742,
      "activations/layer9_attention_weight_max": 75.99662780761719,
      "activations/layer9_attention_weight_min": -65.66194152832031,
      "epoch": 0.44,
      "learning_rate": 0.00014868257575757575,
      "loss": 3.3289,
      "step": 7500
    },
    {
      "activations/layer0_attention_weight_max": 14.414549827575684,
      "activations/layer0_attention_weight_min": -9.204059600830078,
      "activations/layer10_attention_weight_max": 66.18848419189453,
      "activations/layer10_attention_weight_min": -58.76920700073242,
      "activations/layer11_attention_weight_max": 41.3912353515625,
      "activations/layer11_attention_weight_min": -34.42243194580078,
      "activations/layer12_attention_weight_max": 22.48362922668457,
      "activations/layer12_attention_weight_min": -24.775436401367188,
      "activations/layer13_attention_weight_max": 42.45866012573242,
      "activations/layer13_attention_weight_min": -36.20117950439453,
      "activations/layer14_attention_weight_max": 35.78326416015625,
      "activations/layer14_attention_weight_min": -24.653396606445312,
      "activations/layer15_attention_weight_max": 35.99235916137695,
      "activations/layer15_attention_weight_min": -22.74987030029297,
      "activations/layer16_attention_weight_max": 25.657772064208984,
      "activations/layer16_attention_weight_min": -19.593420028686523,
      "activations/layer17_attention_weight_max": 36.95265197753906,
      "activations/layer17_attention_weight_min": -26.38085174560547,
      "activations/layer18_attention_weight_max": 36.564754486083984,
      "activations/layer18_attention_weight_min": -24.183446884155273,
      "activations/layer19_attention_weight_max": 18.887548446655273,
      "activations/layer19_attention_weight_min": -10.247560501098633,
      "activations/layer1_attention_weight_max": 18.574237823486328,
      "activations/layer1_attention_weight_min": -10.195260047912598,
      "activations/layer20_attention_weight_max": 18.12963104248047,
      "activations/layer20_attention_weight_min": -13.165755271911621,
      "activations/layer21_attention_weight_max": 27.39055061340332,
      "activations/layer21_attention_weight_min": -14.689305305480957,
      "activations/layer22_attention_weight_max": 25.977590560913086,
      "activations/layer22_attention_weight_min": -13.295711517333984,
      "activations/layer23_attention_weight_max": 19.966554641723633,
      "activations/layer23_attention_weight_min": -13.0833158493042,
      "activations/layer2_attention_weight_max": 26.1793212890625,
      "activations/layer2_attention_weight_min": -20.856666564941406,
      "activations/layer3_attention_weight_max": 58.189064025878906,
      "activations/layer3_attention_weight_min": -48.194610595703125,
      "activations/layer4_attention_weight_max": 87.54546356201172,
      "activations/layer4_attention_weight_min": -73.03756713867188,
      "activations/layer5_attention_weight_max": 154.436767578125,
      "activations/layer5_attention_weight_min": -130.15721130371094,
      "activations/layer6_attention_weight_max": 88.51826477050781,
      "activations/layer6_attention_weight_min": -80.65489959716797,
      "activations/layer7_attention_weight_max": 84.24121856689453,
      "activations/layer7_attention_weight_min": -75.6027603149414,
      "activations/layer8_attention_weight_max": 68.16455841064453,
      "activations/layer8_attention_weight_min": -60.5317268371582,
      "activations/layer9_attention_weight_max": 74.39022064208984,
      "activations/layer9_attention_weight_min": -63.0984992980957,
      "epoch": 0.44,
      "learning_rate": 0.00014866363636363637,
      "loss": 3.3157,
      "step": 7550
    },
    {
      "activations/layer0_attention_weight_max": 13.503265380859375,
      "activations/layer0_attention_weight_min": -9.78462028503418,
      "activations/layer10_attention_weight_max": 64.83978271484375,
      "activations/layer10_attention_weight_min": -59.75070571899414,
      "activations/layer11_attention_weight_max": 41.550689697265625,
      "activations/layer11_attention_weight_min": -35.82969284057617,
      "activations/layer12_attention_weight_max": 22.276233673095703,
      "activations/layer12_attention_weight_min": -22.20939826965332,
      "activations/layer13_attention_weight_max": 42.77641296386719,
      "activations/layer13_attention_weight_min": -30.11162757873535,
      "activations/layer14_attention_weight_max": 32.07714080810547,
      "activations/layer14_attention_weight_min": -24.80805015563965,
      "activations/layer15_attention_weight_max": 31.637264251708984,
      "activations/layer15_attention_weight_min": -23.63680648803711,
      "activations/layer16_attention_weight_max": 23.085363388061523,
      "activations/layer16_attention_weight_min": -18.417821884155273,
      "activations/layer17_attention_weight_max": 37.11656188964844,
      "activations/layer17_attention_weight_min": -25.0572509765625,
      "activations/layer18_attention_weight_max": 28.084213256835938,
      "activations/layer18_attention_weight_min": -23.446348190307617,
      "activations/layer19_attention_weight_max": 21.619089126586914,
      "activations/layer19_attention_weight_min": -11.26217269897461,
      "activations/layer1_attention_weight_max": 19.531095504760742,
      "activations/layer1_attention_weight_min": -10.1673583984375,
      "activations/layer20_attention_weight_max": 20.69292449951172,
      "activations/layer20_attention_weight_min": -12.104249000549316,
      "activations/layer21_attention_weight_max": 25.820180892944336,
      "activations/layer21_attention_weight_min": -14.477171897888184,
      "activations/layer22_attention_weight_max": 23.760271072387695,
      "activations/layer22_attention_weight_min": -14.798173904418945,
      "activations/layer23_attention_weight_max": 19.919218063354492,
      "activations/layer23_attention_weight_min": -13.169504165649414,
      "activations/layer2_attention_weight_max": 25.883352279663086,
      "activations/layer2_attention_weight_min": -19.77112579345703,
      "activations/layer3_attention_weight_max": 54.385562896728516,
      "activations/layer3_attention_weight_min": -44.27964401245117,
      "activations/layer4_attention_weight_max": 81.1815185546875,
      "activations/layer4_attention_weight_min": -71.20161437988281,
      "activations/layer5_attention_weight_max": 145.29360961914062,
      "activations/layer5_attention_weight_min": -130.5393524169922,
      "activations/layer6_attention_weight_max": 87.41078186035156,
      "activations/layer6_attention_weight_min": -80.46807098388672,
      "activations/layer7_attention_weight_max": 82.64131164550781,
      "activations/layer7_attention_weight_min": -73.92342376708984,
      "activations/layer8_attention_weight_max": 66.74678039550781,
      "activations/layer8_attention_weight_min": -60.11252212524414,
      "activations/layer9_attention_weight_max": 79.96514129638672,
      "activations/layer9_attention_weight_min": -70.7391586303711,
      "epoch": 0.44,
      "learning_rate": 0.00014864469696969696,
      "loss": 3.3156,
      "step": 7600
    },
    {
      "activations/layer0_attention_weight_max": 13.383418083190918,
      "activations/layer0_attention_weight_min": -9.613452911376953,
      "activations/layer10_attention_weight_max": 62.88329315185547,
      "activations/layer10_attention_weight_min": -63.57707214355469,
      "activations/layer11_attention_weight_max": 44.951499938964844,
      "activations/layer11_attention_weight_min": -33.525062561035156,
      "activations/layer12_attention_weight_max": 26.286643981933594,
      "activations/layer12_attention_weight_min": -21.624622344970703,
      "activations/layer13_attention_weight_max": 49.825069427490234,
      "activations/layer13_attention_weight_min": -35.29218673706055,
      "activations/layer14_attention_weight_max": 38.7486686706543,
      "activations/layer14_attention_weight_min": -24.756620407104492,
      "activations/layer15_attention_weight_max": 32.00675964355469,
      "activations/layer15_attention_weight_min": -26.298160552978516,
      "activations/layer16_attention_weight_max": 25.847309112548828,
      "activations/layer16_attention_weight_min": -19.051794052124023,
      "activations/layer17_attention_weight_max": 44.40681838989258,
      "activations/layer17_attention_weight_min": -25.924142837524414,
      "activations/layer18_attention_weight_max": 38.45595932006836,
      "activations/layer18_attention_weight_min": -23.906946182250977,
      "activations/layer19_attention_weight_max": 22.836904525756836,
      "activations/layer19_attention_weight_min": -12.441244125366211,
      "activations/layer1_attention_weight_max": 19.27298927307129,
      "activations/layer1_attention_weight_min": -10.806761741638184,
      "activations/layer20_attention_weight_max": 24.257179260253906,
      "activations/layer20_attention_weight_min": -13.58350658416748,
      "activations/layer21_attention_weight_max": 23.99167823791504,
      "activations/layer21_attention_weight_min": -15.858400344848633,
      "activations/layer22_attention_weight_max": 26.1661434173584,
      "activations/layer22_attention_weight_min": -15.606515884399414,
      "activations/layer23_attention_weight_max": 21.897869110107422,
      "activations/layer23_attention_weight_min": -15.879276275634766,
      "activations/layer2_attention_weight_max": 22.899250030517578,
      "activations/layer2_attention_weight_min": -19.227201461791992,
      "activations/layer3_attention_weight_max": 54.804901123046875,
      "activations/layer3_attention_weight_min": -42.750370025634766,
      "activations/layer4_attention_weight_max": 81.43272399902344,
      "activations/layer4_attention_weight_min": -73.03015899658203,
      "activations/layer5_attention_weight_max": 148.3043975830078,
      "activations/layer5_attention_weight_min": -132.02554321289062,
      "activations/layer6_attention_weight_max": 85.66796112060547,
      "activations/layer6_attention_weight_min": -79.80184173583984,
      "activations/layer7_attention_weight_max": 82.8842544555664,
      "activations/layer7_attention_weight_min": -74.25496673583984,
      "activations/layer8_attention_weight_max": 66.8760757446289,
      "activations/layer8_attention_weight_min": -60.19986343383789,
      "activations/layer9_attention_weight_max": 73.05938720703125,
      "activations/layer9_attention_weight_min": -63.538795471191406,
      "epoch": 0.44,
      "learning_rate": 0.00014862575757575757,
      "loss": 3.3175,
      "step": 7650
    },
    {
      "activations/layer0_attention_weight_max": 13.931187629699707,
      "activations/layer0_attention_weight_min": -9.504066467285156,
      "activations/layer10_attention_weight_max": 66.378662109375,
      "activations/layer10_attention_weight_min": -55.06129837036133,
      "activations/layer11_attention_weight_max": 36.794921875,
      "activations/layer11_attention_weight_min": -31.043373107910156,
      "activations/layer12_attention_weight_max": 22.740421295166016,
      "activations/layer12_attention_weight_min": -21.80674934387207,
      "activations/layer13_attention_weight_max": 43.180294036865234,
      "activations/layer13_attention_weight_min": -31.39945411682129,
      "activations/layer14_attention_weight_max": 30.739337921142578,
      "activations/layer14_attention_weight_min": -21.820110321044922,
      "activations/layer15_attention_weight_max": 32.46947479248047,
      "activations/layer15_attention_weight_min": -21.652263641357422,
      "activations/layer16_attention_weight_max": 23.031518936157227,
      "activations/layer16_attention_weight_min": -18.62714195251465,
      "activations/layer17_attention_weight_max": 41.208351135253906,
      "activations/layer17_attention_weight_min": -28.632930755615234,
      "activations/layer18_attention_weight_max": 40.8830451965332,
      "activations/layer18_attention_weight_min": -23.592208862304688,
      "activations/layer19_attention_weight_max": 17.996673583984375,
      "activations/layer19_attention_weight_min": -11.609004020690918,
      "activations/layer1_attention_weight_max": 19.567684173583984,
      "activations/layer1_attention_weight_min": -10.48595905303955,
      "activations/layer20_attention_weight_max": 17.75847053527832,
      "activations/layer20_attention_weight_min": -12.200133323669434,
      "activations/layer21_attention_weight_max": 26.69791603088379,
      "activations/layer21_attention_weight_min": -13.391048431396484,
      "activations/layer22_attention_weight_max": 23.94154930114746,
      "activations/layer22_attention_weight_min": -13.337764739990234,
      "activations/layer23_attention_weight_max": 19.35269546508789,
      "activations/layer23_attention_weight_min": -12.541189193725586,
      "activations/layer2_attention_weight_max": 27.822309494018555,
      "activations/layer2_attention_weight_min": -19.698795318603516,
      "activations/layer3_attention_weight_max": 54.99026870727539,
      "activations/layer3_attention_weight_min": -46.100379943847656,
      "activations/layer4_attention_weight_max": 83.99092102050781,
      "activations/layer4_attention_weight_min": -74.22533416748047,
      "activations/layer5_attention_weight_max": 153.7542266845703,
      "activations/layer5_attention_weight_min": -134.7719268798828,
      "activations/layer6_attention_weight_max": 86.3542251586914,
      "activations/layer6_attention_weight_min": -78.94482421875,
      "activations/layer7_attention_weight_max": 86.5385971069336,
      "activations/layer7_attention_weight_min": -75.37210845947266,
      "activations/layer8_attention_weight_max": 71.7914810180664,
      "activations/layer8_attention_weight_min": -63.32246780395508,
      "activations/layer9_attention_weight_max": 74.24930572509766,
      "activations/layer9_attention_weight_min": -67.2081298828125,
      "epoch": 0.45,
      "learning_rate": 0.0001486068181818182,
      "loss": 3.3008,
      "step": 7700
    },
    {
      "activations/layer0_attention_weight_max": 14.279446601867676,
      "activations/layer0_attention_weight_min": -9.651358604431152,
      "activations/layer10_attention_weight_max": 65.37672424316406,
      "activations/layer10_attention_weight_min": -58.41339874267578,
      "activations/layer11_attention_weight_max": 40.51600646972656,
      "activations/layer11_attention_weight_min": -35.0014533996582,
      "activations/layer12_attention_weight_max": 23.7176456451416,
      "activations/layer12_attention_weight_min": -21.50977325439453,
      "activations/layer13_attention_weight_max": 39.9552116394043,
      "activations/layer13_attention_weight_min": -34.20094680786133,
      "activations/layer14_attention_weight_max": 36.22410202026367,
      "activations/layer14_attention_weight_min": -26.792673110961914,
      "activations/layer15_attention_weight_max": 32.54800796508789,
      "activations/layer15_attention_weight_min": -24.73929214477539,
      "activations/layer16_attention_weight_max": 25.649642944335938,
      "activations/layer16_attention_weight_min": -17.264116287231445,
      "activations/layer17_attention_weight_max": 42.69561767578125,
      "activations/layer17_attention_weight_min": -29.10166358947754,
      "activations/layer18_attention_weight_max": 30.885026931762695,
      "activations/layer18_attention_weight_min": -21.68929672241211,
      "activations/layer19_attention_weight_max": 21.329843521118164,
      "activations/layer19_attention_weight_min": -10.984384536743164,
      "activations/layer1_attention_weight_max": 19.222850799560547,
      "activations/layer1_attention_weight_min": -11.121102333068848,
      "activations/layer20_attention_weight_max": 18.684646606445312,
      "activations/layer20_attention_weight_min": -12.14790153503418,
      "activations/layer21_attention_weight_max": 27.2933406829834,
      "activations/layer21_attention_weight_min": -15.587910652160645,
      "activations/layer22_attention_weight_max": 27.825519561767578,
      "activations/layer22_attention_weight_min": -14.903579711914062,
      "activations/layer23_attention_weight_max": 21.605688095092773,
      "activations/layer23_attention_weight_min": -14.394789695739746,
      "activations/layer2_attention_weight_max": 25.810585021972656,
      "activations/layer2_attention_weight_min": -20.205324172973633,
      "activations/layer3_attention_weight_max": 53.82028579711914,
      "activations/layer3_attention_weight_min": -44.17265701293945,
      "activations/layer4_attention_weight_max": 80.62377166748047,
      "activations/layer4_attention_weight_min": -69.86034393310547,
      "activations/layer5_attention_weight_max": 143.8129425048828,
      "activations/layer5_attention_weight_min": -125.24932098388672,
      "activations/layer6_attention_weight_max": 87.03622436523438,
      "activations/layer6_attention_weight_min": -77.40003967285156,
      "activations/layer7_attention_weight_max": 80.92018127441406,
      "activations/layer7_attention_weight_min": -72.46717071533203,
      "activations/layer8_attention_weight_max": 63.57964324951172,
      "activations/layer8_attention_weight_min": -56.62413024902344,
      "activations/layer9_attention_weight_max": 68.50926971435547,
      "activations/layer9_attention_weight_min": -63.37086868286133,
      "epoch": 0.45,
      "learning_rate": 0.00014858787878787878,
      "loss": 3.3119,
      "step": 7750
    },
    {
      "activations/layer0_attention_weight_max": 14.04392147064209,
      "activations/layer0_attention_weight_min": -9.87813663482666,
      "activations/layer10_attention_weight_max": 63.141719818115234,
      "activations/layer10_attention_weight_min": -57.04043960571289,
      "activations/layer11_attention_weight_max": 43.92013168334961,
      "activations/layer11_attention_weight_min": -34.478397369384766,
      "activations/layer12_attention_weight_max": 26.545000076293945,
      "activations/layer12_attention_weight_min": -23.16594123840332,
      "activations/layer13_attention_weight_max": 40.47108840942383,
      "activations/layer13_attention_weight_min": -30.991718292236328,
      "activations/layer14_attention_weight_max": 34.96285629272461,
      "activations/layer14_attention_weight_min": -25.74033546447754,
      "activations/layer15_attention_weight_max": 33.5184440612793,
      "activations/layer15_attention_weight_min": -22.380281448364258,
      "activations/layer16_attention_weight_max": 27.844940185546875,
      "activations/layer16_attention_weight_min": -21.36082649230957,
      "activations/layer17_attention_weight_max": 47.83797836303711,
      "activations/layer17_attention_weight_min": -26.291994094848633,
      "activations/layer18_attention_weight_max": 45.41709899902344,
      "activations/layer18_attention_weight_min": -23.052698135375977,
      "activations/layer19_attention_weight_max": 24.424808502197266,
      "activations/layer19_attention_weight_min": -11.246508598327637,
      "activations/layer1_attention_weight_max": 18.18122673034668,
      "activations/layer1_attention_weight_min": -10.923416137695312,
      "activations/layer20_attention_weight_max": 23.625347137451172,
      "activations/layer20_attention_weight_min": -12.53815746307373,
      "activations/layer21_attention_weight_max": 30.06986427307129,
      "activations/layer21_attention_weight_min": -14.999723434448242,
      "activations/layer22_attention_weight_max": 28.31662940979004,
      "activations/layer22_attention_weight_min": -16.378141403198242,
      "activations/layer23_attention_weight_max": 22.47283172607422,
      "activations/layer23_attention_weight_min": -13.471312522888184,
      "activations/layer2_attention_weight_max": 25.256855010986328,
      "activations/layer2_attention_weight_min": -20.74474334716797,
      "activations/layer3_attention_weight_max": 54.62906265258789,
      "activations/layer3_attention_weight_min": -43.93173599243164,
      "activations/layer4_attention_weight_max": 83.04315185546875,
      "activations/layer4_attention_weight_min": -71.18601989746094,
      "activations/layer5_attention_weight_max": 143.1849822998047,
      "activations/layer5_attention_weight_min": -128.6463165283203,
      "activations/layer6_attention_weight_max": 83.55780029296875,
      "activations/layer6_attention_weight_min": -75.7162094116211,
      "activations/layer7_attention_weight_max": 81.45636749267578,
      "activations/layer7_attention_weight_min": -75.02023315429688,
      "activations/layer8_attention_weight_max": 64.94672393798828,
      "activations/layer8_attention_weight_min": -57.90562057495117,
      "activations/layer9_attention_weight_max": 70.30962371826172,
      "activations/layer9_attention_weight_min": -67.1901626586914,
      "epoch": 0.45,
      "learning_rate": 0.0001485689393939394,
      "loss": 3.3201,
      "step": 7800
    },
    {
      "activations/layer0_attention_weight_max": 15.018194198608398,
      "activations/layer0_attention_weight_min": -9.9628324508667,
      "activations/layer10_attention_weight_max": 64.73780822753906,
      "activations/layer10_attention_weight_min": -63.62808609008789,
      "activations/layer11_attention_weight_max": 39.64009094238281,
      "activations/layer11_attention_weight_min": -35.83851623535156,
      "activations/layer12_attention_weight_max": 25.98357582092285,
      "activations/layer12_attention_weight_min": -22.739267349243164,
      "activations/layer13_attention_weight_max": 40.73878479003906,
      "activations/layer13_attention_weight_min": -38.69807052612305,
      "activations/layer14_attention_weight_max": 31.099220275878906,
      "activations/layer14_attention_weight_min": -27.653141021728516,
      "activations/layer15_attention_weight_max": 33.46241760253906,
      "activations/layer15_attention_weight_min": -25.649412155151367,
      "activations/layer16_attention_weight_max": 24.841753005981445,
      "activations/layer16_attention_weight_min": -20.062732696533203,
      "activations/layer17_attention_weight_max": 40.80376052856445,
      "activations/layer17_attention_weight_min": -34.21721649169922,
      "activations/layer18_attention_weight_max": 40.65912628173828,
      "activations/layer18_attention_weight_min": -23.0554256439209,
      "activations/layer19_attention_weight_max": 21.304819107055664,
      "activations/layer19_attention_weight_min": -11.461658477783203,
      "activations/layer1_attention_weight_max": 18.067049026489258,
      "activations/layer1_attention_weight_min": -11.11129379272461,
      "activations/layer20_attention_weight_max": 19.342973709106445,
      "activations/layer20_attention_weight_min": -12.220633506774902,
      "activations/layer21_attention_weight_max": 26.05997657775879,
      "activations/layer21_attention_weight_min": -16.461095809936523,
      "activations/layer22_attention_weight_max": 26.05859375,
      "activations/layer22_attention_weight_min": -15.07047176361084,
      "activations/layer23_attention_weight_max": 19.647218704223633,
      "activations/layer23_attention_weight_min": -12.940895080566406,
      "activations/layer2_attention_weight_max": 25.761985778808594,
      "activations/layer2_attention_weight_min": -20.382076263427734,
      "activations/layer3_attention_weight_max": 55.025753021240234,
      "activations/layer3_attention_weight_min": -46.25602340698242,
      "activations/layer4_attention_weight_max": 84.4094009399414,
      "activations/layer4_attention_weight_min": -73.8020248413086,
      "activations/layer5_attention_weight_max": 160.78790283203125,
      "activations/layer5_attention_weight_min": -137.001220703125,
      "activations/layer6_attention_weight_max": 87.99050903320312,
      "activations/layer6_attention_weight_min": -77.7718734741211,
      "activations/layer7_attention_weight_max": 82.49677276611328,
      "activations/layer7_attention_weight_min": -72.33245849609375,
      "activations/layer8_attention_weight_max": 67.34190368652344,
      "activations/layer8_attention_weight_min": -58.92933654785156,
      "activations/layer9_attention_weight_max": 74.17530059814453,
      "activations/layer9_attention_weight_min": -73.74060821533203,
      "epoch": 0.46,
      "learning_rate": 0.00014854999999999998,
      "loss": 3.2976,
      "step": 7850
    },
    {
      "activations/layer0_attention_weight_max": 13.211138725280762,
      "activations/layer0_attention_weight_min": -10.459869384765625,
      "activations/layer10_attention_weight_max": 71.12847900390625,
      "activations/layer10_attention_weight_min": -63.06553268432617,
      "activations/layer11_attention_weight_max": 45.48260498046875,
      "activations/layer11_attention_weight_min": -35.49833679199219,
      "activations/layer12_attention_weight_max": 25.189815521240234,
      "activations/layer12_attention_weight_min": -26.167360305786133,
      "activations/layer13_attention_weight_max": 45.22322082519531,
      "activations/layer13_attention_weight_min": -34.260337829589844,
      "activations/layer14_attention_weight_max": 31.86236572265625,
      "activations/layer14_attention_weight_min": -25.749448776245117,
      "activations/layer15_attention_weight_max": 30.19778060913086,
      "activations/layer15_attention_weight_min": -23.14974021911621,
      "activations/layer16_attention_weight_max": 24.844221115112305,
      "activations/layer16_attention_weight_min": -20.719425201416016,
      "activations/layer17_attention_weight_max": 38.43309020996094,
      "activations/layer17_attention_weight_min": -31.39288330078125,
      "activations/layer18_attention_weight_max": 44.692142486572266,
      "activations/layer18_attention_weight_min": -28.840425491333008,
      "activations/layer19_attention_weight_max": 19.88605308532715,
      "activations/layer19_attention_weight_min": -12.006518363952637,
      "activations/layer1_attention_weight_max": 17.777814865112305,
      "activations/layer1_attention_weight_min": -11.144645690917969,
      "activations/layer20_attention_weight_max": 20.485769271850586,
      "activations/layer20_attention_weight_min": -11.229434967041016,
      "activations/layer21_attention_weight_max": 26.353416442871094,
      "activations/layer21_attention_weight_min": -17.343204498291016,
      "activations/layer22_attention_weight_max": 26.88347053527832,
      "activations/layer22_attention_weight_min": -15.780835151672363,
      "activations/layer23_attention_weight_max": 18.168685913085938,
      "activations/layer23_attention_weight_min": -13.970926284790039,
      "activations/layer2_attention_weight_max": 25.452054977416992,
      "activations/layer2_attention_weight_min": -19.77398681640625,
      "activations/layer3_attention_weight_max": 57.81416702270508,
      "activations/layer3_attention_weight_min": -45.166934967041016,
      "activations/layer4_attention_weight_max": 83.00753021240234,
      "activations/layer4_attention_weight_min": -69.05191040039062,
      "activations/layer5_attention_weight_max": 147.03262329101562,
      "activations/layer5_attention_weight_min": -124.1125717163086,
      "activations/layer6_attention_weight_max": 85.84666442871094,
      "activations/layer6_attention_weight_min": -76.05241394042969,
      "activations/layer7_attention_weight_max": 84.73163604736328,
      "activations/layer7_attention_weight_min": -72.96427917480469,
      "activations/layer8_attention_weight_max": 71.20953369140625,
      "activations/layer8_attention_weight_min": -60.36533737182617,
      "activations/layer9_attention_weight_max": 72.02770233154297,
      "activations/layer9_attention_weight_min": -65.89811706542969,
      "epoch": 0.46,
      "learning_rate": 0.0001485310606060606,
      "loss": 3.2983,
      "step": 7900
    },
    {
      "activations/layer0_attention_weight_max": 13.98570442199707,
      "activations/layer0_attention_weight_min": -9.807310104370117,
      "activations/layer10_attention_weight_max": 67.68824005126953,
      "activations/layer10_attention_weight_min": -60.08155059814453,
      "activations/layer11_attention_weight_max": 43.378746032714844,
      "activations/layer11_attention_weight_min": -40.729774475097656,
      "activations/layer12_attention_weight_max": 25.2010498046875,
      "activations/layer12_attention_weight_min": -21.361934661865234,
      "activations/layer13_attention_weight_max": 45.54887771606445,
      "activations/layer13_attention_weight_min": -35.038421630859375,
      "activations/layer14_attention_weight_max": 47.38009262084961,
      "activations/layer14_attention_weight_min": -26.321931838989258,
      "activations/layer15_attention_weight_max": 38.78854751586914,
      "activations/layer15_attention_weight_min": -25.895597457885742,
      "activations/layer16_attention_weight_max": 32.779327392578125,
      "activations/layer16_attention_weight_min": -19.695817947387695,
      "activations/layer17_attention_weight_max": 45.45528030395508,
      "activations/layer17_attention_weight_min": -28.069129943847656,
      "activations/layer18_attention_weight_max": 42.3964958190918,
      "activations/layer18_attention_weight_min": -23.488201141357422,
      "activations/layer19_attention_weight_max": 21.65925407409668,
      "activations/layer19_attention_weight_min": -12.148238182067871,
      "activations/layer1_attention_weight_max": 19.06230926513672,
      "activations/layer1_attention_weight_min": -10.459166526794434,
      "activations/layer20_attention_weight_max": 20.02948570251465,
      "activations/layer20_attention_weight_min": -11.64474105834961,
      "activations/layer21_attention_weight_max": 33.842567443847656,
      "activations/layer21_attention_weight_min": -17.690185546875,
      "activations/layer22_attention_weight_max": 33.201560974121094,
      "activations/layer22_attention_weight_min": -13.837628364562988,
      "activations/layer23_attention_weight_max": 24.591398239135742,
      "activations/layer23_attention_weight_min": -15.451238632202148,
      "activations/layer2_attention_weight_max": 26.68218231201172,
      "activations/layer2_attention_weight_min": -19.893348693847656,
      "activations/layer3_attention_weight_max": 55.63200759887695,
      "activations/layer3_attention_weight_min": -45.41209411621094,
      "activations/layer4_attention_weight_max": 82.97270202636719,
      "activations/layer4_attention_weight_min": -70.4168472290039,
      "activations/layer5_attention_weight_max": 153.32086181640625,
      "activations/layer5_attention_weight_min": -129.561767578125,
      "activations/layer6_attention_weight_max": 90.01620483398438,
      "activations/layer6_attention_weight_min": -78.31723022460938,
      "activations/layer7_attention_weight_max": 87.9211654663086,
      "activations/layer7_attention_weight_min": -75.13839721679688,
      "activations/layer8_attention_weight_max": 65.05126190185547,
      "activations/layer8_attention_weight_min": -57.78076171875,
      "activations/layer9_attention_weight_max": 83.49347686767578,
      "activations/layer9_attention_weight_min": -69.06917572021484,
      "epoch": 0.46,
      "learning_rate": 0.0001485121212121212,
      "loss": 3.3058,
      "step": 7950
    },
    {
      "activations/layer0_attention_weight_max": 13.331512451171875,
      "activations/layer0_attention_weight_min": -9.843281745910645,
      "activations/layer10_attention_weight_max": 77.1852798461914,
      "activations/layer10_attention_weight_min": -59.990440368652344,
      "activations/layer11_attention_weight_max": 47.08144760131836,
      "activations/layer11_attention_weight_min": -43.52375030517578,
      "activations/layer12_attention_weight_max": 25.796384811401367,
      "activations/layer12_attention_weight_min": -22.8691463470459,
      "activations/layer13_attention_weight_max": 57.71168899536133,
      "activations/layer13_attention_weight_min": -39.85238265991211,
      "activations/layer14_attention_weight_max": 33.06343078613281,
      "activations/layer14_attention_weight_min": -28.566999435424805,
      "activations/layer15_attention_weight_max": 33.912315368652344,
      "activations/layer15_attention_weight_min": -25.597702026367188,
      "activations/layer16_attention_weight_max": 25.152557373046875,
      "activations/layer16_attention_weight_min": -18.69673728942871,
      "activations/layer17_attention_weight_max": 44.16789627075195,
      "activations/layer17_attention_weight_min": -29.10942268371582,
      "activations/layer18_attention_weight_max": 44.38144302368164,
      "activations/layer18_attention_weight_min": -23.437458038330078,
      "activations/layer19_attention_weight_max": 21.292856216430664,
      "activations/layer19_attention_weight_min": -11.517874717712402,
      "activations/layer1_attention_weight_max": 18.459232330322266,
      "activations/layer1_attention_weight_min": -10.743672370910645,
      "activations/layer20_attention_weight_max": 19.817584991455078,
      "activations/layer20_attention_weight_min": -12.169246673583984,
      "activations/layer21_attention_weight_max": 29.597091674804688,
      "activations/layer21_attention_weight_min": -15.780522346496582,
      "activations/layer22_attention_weight_max": 31.5523681640625,
      "activations/layer22_attention_weight_min": -17.04562759399414,
      "activations/layer23_attention_weight_max": 23.346050262451172,
      "activations/layer23_attention_weight_min": -16.37152099609375,
      "activations/layer2_attention_weight_max": 26.91250228881836,
      "activations/layer2_attention_weight_min": -21.107894897460938,
      "activations/layer3_attention_weight_max": 55.93089294433594,
      "activations/layer3_attention_weight_min": -44.52051544189453,
      "activations/layer4_attention_weight_max": 84.30780029296875,
      "activations/layer4_attention_weight_min": -71.93312072753906,
      "activations/layer5_attention_weight_max": 153.44583129882812,
      "activations/layer5_attention_weight_min": -127.84815979003906,
      "activations/layer6_attention_weight_max": 88.1214828491211,
      "activations/layer6_attention_weight_min": -74.88846588134766,
      "activations/layer7_attention_weight_max": 88.17985534667969,
      "activations/layer7_attention_weight_min": -76.20382690429688,
      "activations/layer8_attention_weight_max": 72.44808197021484,
      "activations/layer8_attention_weight_min": -57.46457290649414,
      "activations/layer9_attention_weight_max": 83.85319519042969,
      "activations/layer9_attention_weight_min": -66.9073486328125,
      "epoch": 0.46,
      "learning_rate": 0.0001484931818181818,
      "loss": 3.2841,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_loss": 3.244140625,
      "eval_runtime": 8.1939,
      "eval_samples_per_second": 524.05,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_openwebtext_loss": 3.244140625,
      "eval_openwebtext_ppl": 25.63966650134361,
      "eval_openwebtext_runtime": 8.1939,
      "eval_openwebtext_samples_per_second": 524.05,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_wikitext_loss": 3.560546875,
      "eval_wikitext_ppl": 35.18243227767251,
      "eval_wikitext_runtime": 1.6969,
      "eval_wikitext_samples_per_second": 268.721,
      "step": 8000
    },
    {
      "epoch": 0.46,
      "eval_lambada_loss": 4.01953125,
      "eval_lambada_ppl": 55.6750020519895,
      "eval_lambada_runtime": 9.2711,
      "eval_lambada_samples_per_second": 525.178,
      "step": 8000
    },
    {
      "activations/layer0_attention_weight_max": 14.349447250366211,
      "activations/layer0_attention_weight_min": -9.990484237670898,
      "activations/layer10_attention_weight_max": 60.996883392333984,
      "activations/layer10_attention_weight_min": -54.230682373046875,
      "activations/layer11_attention_weight_max": 42.81960678100586,
      "activations/layer11_attention_weight_min": -36.54503631591797,
      "activations/layer12_attention_weight_max": 24.388715744018555,
      "activations/layer12_attention_weight_min": -24.153913497924805,
      "activations/layer13_attention_weight_max": 48.582618713378906,
      "activations/layer13_attention_weight_min": -35.57585525512695,
      "activations/layer14_attention_weight_max": 33.409645080566406,
      "activations/layer14_attention_weight_min": -23.704484939575195,
      "activations/layer15_attention_weight_max": 31.792367935180664,
      "activations/layer15_attention_weight_min": -26.716007232666016,
      "activations/layer16_attention_weight_max": 28.11005973815918,
      "activations/layer16_attention_weight_min": -19.513933181762695,
      "activations/layer17_attention_weight_max": 44.94972610473633,
      "activations/layer17_attention_weight_min": -28.19102668762207,
      "activations/layer18_attention_weight_max": 35.452239990234375,
      "activations/layer18_attention_weight_min": -23.96534538269043,
      "activations/layer19_attention_weight_max": 21.8333740234375,
      "activations/layer19_attention_weight_min": -12.802248001098633,
      "activations/layer1_attention_weight_max": 17.88278579711914,
      "activations/layer1_attention_weight_min": -11.419795989990234,
      "activations/layer20_attention_weight_max": 20.471160888671875,
      "activations/layer20_attention_weight_min": -12.217015266418457,
      "activations/layer21_attention_weight_max": 27.286434173583984,
      "activations/layer21_attention_weight_min": -16.75567626953125,
      "activations/layer22_attention_weight_max": 28.12384796142578,
      "activations/layer22_attention_weight_min": -16.330488204956055,
      "activations/layer23_attention_weight_max": 21.72686004638672,
      "activations/layer23_attention_weight_min": -13.128738403320312,
      "activations/layer2_attention_weight_max": 26.507198333740234,
      "activations/layer2_attention_weight_min": -20.429452896118164,
      "activations/layer3_attention_weight_max": 56.091712951660156,
      "activations/layer3_attention_weight_min": -47.63346481323242,
      "activations/layer4_attention_weight_max": 82.11660766601562,
      "activations/layer4_attention_weight_min": -72.4698715209961,
      "activations/layer5_attention_weight_max": 149.4759521484375,
      "activations/layer5_attention_weight_min": -132.03436279296875,
      "activations/layer6_attention_weight_max": 83.5328140258789,
      "activations/layer6_attention_weight_min": -77.36766052246094,
      "activations/layer7_attention_weight_max": 84.31097412109375,
      "activations/layer7_attention_weight_min": -72.67486572265625,
      "activations/layer8_attention_weight_max": 67.59072875976562,
      "activations/layer8_attention_weight_min": -59.44221496582031,
      "activations/layer9_attention_weight_max": 76.02310180664062,
      "activations/layer9_attention_weight_min": -71.56419372558594,
      "epoch": 0.47,
      "learning_rate": 0.0001484742424242424,
      "loss": 3.2987,
      "step": 8050
    },
    {
      "activations/layer0_attention_weight_max": 13.085466384887695,
      "activations/layer0_attention_weight_min": -9.618142127990723,
      "activations/layer10_attention_weight_max": 67.0476303100586,
      "activations/layer10_attention_weight_min": -58.00257110595703,
      "activations/layer11_attention_weight_max": 37.76988220214844,
      "activations/layer11_attention_weight_min": -34.47176742553711,
      "activations/layer12_attention_weight_max": 25.714805603027344,
      "activations/layer12_attention_weight_min": -25.11814308166504,
      "activations/layer13_attention_weight_max": 38.47838592529297,
      "activations/layer13_attention_weight_min": -29.521780014038086,
      "activations/layer14_attention_weight_max": 34.29401397705078,
      "activations/layer14_attention_weight_min": -25.190391540527344,
      "activations/layer15_attention_weight_max": 34.624786376953125,
      "activations/layer15_attention_weight_min": -25.289588928222656,
      "activations/layer16_attention_weight_max": 29.050527572631836,
      "activations/layer16_attention_weight_min": -18.892610549926758,
      "activations/layer17_attention_weight_max": 46.91301345825195,
      "activations/layer17_attention_weight_min": -27.845523834228516,
      "activations/layer18_attention_weight_max": 53.24405288696289,
      "activations/layer18_attention_weight_min": -23.549585342407227,
      "activations/layer19_attention_weight_max": 23.138456344604492,
      "activations/layer19_attention_weight_min": -11.118328094482422,
      "activations/layer1_attention_weight_max": 18.169536590576172,
      "activations/layer1_attention_weight_min": -10.401311874389648,
      "activations/layer20_attention_weight_max": 22.49753761291504,
      "activations/layer20_attention_weight_min": -14.248170852661133,
      "activations/layer21_attention_weight_max": 31.65251922607422,
      "activations/layer21_attention_weight_min": -16.434179306030273,
      "activations/layer22_attention_weight_max": 31.15203857421875,
      "activations/layer22_attention_weight_min": -15.262816429138184,
      "activations/layer23_attention_weight_max": 20.50283432006836,
      "activations/layer23_attention_weight_min": -14.108145713806152,
      "activations/layer2_attention_weight_max": 26.667905807495117,
      "activations/layer2_attention_weight_min": -20.337291717529297,
      "activations/layer3_attention_weight_max": 57.79815673828125,
      "activations/layer3_attention_weight_min": -46.06256866455078,
      "activations/layer4_attention_weight_max": 84.22277069091797,
      "activations/layer4_attention_weight_min": -70.6365966796875,
      "activations/layer5_attention_weight_max": 146.37994384765625,
      "activations/layer5_attention_weight_min": -128.17877197265625,
      "activations/layer6_attention_weight_max": 84.69059753417969,
      "activations/layer6_attention_weight_min": -76.78715515136719,
      "activations/layer7_attention_weight_max": 78.68357849121094,
      "activations/layer7_attention_weight_min": -71.03340911865234,
      "activations/layer8_attention_weight_max": 64.4845962524414,
      "activations/layer8_attention_weight_min": -57.31937026977539,
      "activations/layer9_attention_weight_max": 73.50770568847656,
      "activations/layer9_attention_weight_min": -62.040771484375,
      "epoch": 0.47,
      "learning_rate": 0.00014845530303030303,
      "loss": 3.3056,
      "step": 8100
    },
    {
      "activations/layer0_attention_weight_max": 13.539613723754883,
      "activations/layer0_attention_weight_min": -10.1497163772583,
      "activations/layer10_attention_weight_max": 64.25312805175781,
      "activations/layer10_attention_weight_min": -57.10108184814453,
      "activations/layer11_attention_weight_max": 42.30349349975586,
      "activations/layer11_attention_weight_min": -33.96849060058594,
      "activations/layer12_attention_weight_max": 25.90300750732422,
      "activations/layer12_attention_weight_min": -22.035131454467773,
      "activations/layer13_attention_weight_max": 50.24496078491211,
      "activations/layer13_attention_weight_min": -35.731201171875,
      "activations/layer14_attention_weight_max": 42.10277557373047,
      "activations/layer14_attention_weight_min": -29.427885055541992,
      "activations/layer15_attention_weight_max": 36.1841926574707,
      "activations/layer15_attention_weight_min": -24.43672752380371,
      "activations/layer16_attention_weight_max": 26.21036148071289,
      "activations/layer16_attention_weight_min": -18.22226333618164,
      "activations/layer17_attention_weight_max": 38.66632843017578,
      "activations/layer17_attention_weight_min": -28.979257583618164,
      "activations/layer18_attention_weight_max": 39.779117584228516,
      "activations/layer18_attention_weight_min": -19.82017707824707,
      "activations/layer19_attention_weight_max": 19.795654296875,
      "activations/layer19_attention_weight_min": -11.438681602478027,
      "activations/layer1_attention_weight_max": 17.757915496826172,
      "activations/layer1_attention_weight_min": -10.773566246032715,
      "activations/layer20_attention_weight_max": 17.50296401977539,
      "activations/layer20_attention_weight_min": -12.52728271484375,
      "activations/layer21_attention_weight_max": 22.94040870666504,
      "activations/layer21_attention_weight_min": -15.191108703613281,
      "activations/layer22_attention_weight_max": 27.055070877075195,
      "activations/layer22_attention_weight_min": -13.055535316467285,
      "activations/layer23_attention_weight_max": 19.482419967651367,
      "activations/layer23_attention_weight_min": -14.5313138961792,
      "activations/layer2_attention_weight_max": 27.553783416748047,
      "activations/layer2_attention_weight_min": -20.535585403442383,
      "activations/layer3_attention_weight_max": 58.51728820800781,
      "activations/layer3_attention_weight_min": -46.650413513183594,
      "activations/layer4_attention_weight_max": 82.85967254638672,
      "activations/layer4_attention_weight_min": -70.06621551513672,
      "activations/layer5_attention_weight_max": 158.08297729492188,
      "activations/layer5_attention_weight_min": -124.93258666992188,
      "activations/layer6_attention_weight_max": 85.70814514160156,
      "activations/layer6_attention_weight_min": -73.38256072998047,
      "activations/layer7_attention_weight_max": 81.22189331054688,
      "activations/layer7_attention_weight_min": -71.34037017822266,
      "activations/layer8_attention_weight_max": 66.82472229003906,
      "activations/layer8_attention_weight_min": -57.32125473022461,
      "activations/layer9_attention_weight_max": 73.49591827392578,
      "activations/layer9_attention_weight_min": -63.93917465209961,
      "epoch": 0.47,
      "learning_rate": 0.00014843636363636361,
      "loss": 3.3012,
      "step": 8150
    },
    {
      "activations/layer0_attention_weight_max": 13.843520164489746,
      "activations/layer0_attention_weight_min": -10.037308692932129,
      "activations/layer10_attention_weight_max": 73.169677734375,
      "activations/layer10_attention_weight_min": -62.451072692871094,
      "activations/layer11_attention_weight_max": 42.03546905517578,
      "activations/layer11_attention_weight_min": -33.23590087890625,
      "activations/layer12_attention_weight_max": 26.750200271606445,
      "activations/layer12_attention_weight_min": -23.38174057006836,
      "activations/layer13_attention_weight_max": 48.11192321777344,
      "activations/layer13_attention_weight_min": -35.29621887207031,
      "activations/layer14_attention_weight_max": 34.61152267456055,
      "activations/layer14_attention_weight_min": -26.442718505859375,
      "activations/layer15_attention_weight_max": 34.23048782348633,
      "activations/layer15_attention_weight_min": -24.56068229675293,
      "activations/layer16_attention_weight_max": 24.389612197875977,
      "activations/layer16_attention_weight_min": -18.864904403686523,
      "activations/layer17_attention_weight_max": 43.484710693359375,
      "activations/layer17_attention_weight_min": -29.40932273864746,
      "activations/layer18_attention_weight_max": 28.724214553833008,
      "activations/layer18_attention_weight_min": -22.38642120361328,
      "activations/layer19_attention_weight_max": 19.06389045715332,
      "activations/layer19_attention_weight_min": -10.808945655822754,
      "activations/layer1_attention_weight_max": 17.610048294067383,
      "activations/layer1_attention_weight_min": -10.540262222290039,
      "activations/layer20_attention_weight_max": 16.68754005432129,
      "activations/layer20_attention_weight_min": -13.155010223388672,
      "activations/layer21_attention_weight_max": 23.646997451782227,
      "activations/layer21_attention_weight_min": -13.704526901245117,
      "activations/layer22_attention_weight_max": 25.494977951049805,
      "activations/layer22_attention_weight_min": -13.158060073852539,
      "activations/layer23_attention_weight_max": 21.233644485473633,
      "activations/layer23_attention_weight_min": -16.265981674194336,
      "activations/layer2_attention_weight_max": 28.17803955078125,
      "activations/layer2_attention_weight_min": -21.74312400817871,
      "activations/layer3_attention_weight_max": 54.074893951416016,
      "activations/layer3_attention_weight_min": -44.709678649902344,
      "activations/layer4_attention_weight_max": 81.12210845947266,
      "activations/layer4_attention_weight_min": -71.45272064208984,
      "activations/layer5_attention_weight_max": 151.744873046875,
      "activations/layer5_attention_weight_min": -126.02667236328125,
      "activations/layer6_attention_weight_max": 85.63565063476562,
      "activations/layer6_attention_weight_min": -73.84695434570312,
      "activations/layer7_attention_weight_max": 82.92982482910156,
      "activations/layer7_attention_weight_min": -72.94955444335938,
      "activations/layer8_attention_weight_max": 62.79560089111328,
      "activations/layer8_attention_weight_min": -57.34980773925781,
      "activations/layer9_attention_weight_max": 76.78414916992188,
      "activations/layer9_attention_weight_min": -65.1628646850586,
      "epoch": 0.48,
      "learning_rate": 0.00014841742424242423,
      "loss": 3.29,
      "step": 8200
    },
    {
      "activations/layer0_attention_weight_max": 13.430025100708008,
      "activations/layer0_attention_weight_min": -9.832818984985352,
      "activations/layer10_attention_weight_max": 64.68775939941406,
      "activations/layer10_attention_weight_min": -60.71453857421875,
      "activations/layer11_attention_weight_max": 40.909454345703125,
      "activations/layer11_attention_weight_min": -34.705623626708984,
      "activations/layer12_attention_weight_max": 29.352628707885742,
      "activations/layer12_attention_weight_min": -25.747129440307617,
      "activations/layer13_attention_weight_max": 46.95520782470703,
      "activations/layer13_attention_weight_min": -33.0967903137207,
      "activations/layer14_attention_weight_max": 49.71598815917969,
      "activations/layer14_attention_weight_min": -30.389507293701172,
      "activations/layer15_attention_weight_max": 37.34204864501953,
      "activations/layer15_attention_weight_min": -25.33461570739746,
      "activations/layer16_attention_weight_max": 30.930028915405273,
      "activations/layer16_attention_weight_min": -23.88945770263672,
      "activations/layer17_attention_weight_max": 53.346492767333984,
      "activations/layer17_attention_weight_min": -29.446767807006836,
      "activations/layer18_attention_weight_max": 39.70329666137695,
      "activations/layer18_attention_weight_min": -22.036725997924805,
      "activations/layer19_attention_weight_max": 20.979013442993164,
      "activations/layer19_attention_weight_min": -10.044366836547852,
      "activations/layer1_attention_weight_max": 16.596717834472656,
      "activations/layer1_attention_weight_min": -10.27570915222168,
      "activations/layer20_attention_weight_max": 22.395158767700195,
      "activations/layer20_attention_weight_min": -10.68918514251709,
      "activations/layer21_attention_weight_max": 33.52032470703125,
      "activations/layer21_attention_weight_min": -17.3635311126709,
      "activations/layer22_attention_weight_max": 34.910606384277344,
      "activations/layer22_attention_weight_min": -15.43287181854248,
      "activations/layer23_attention_weight_max": 23.873456954956055,
      "activations/layer23_attention_weight_min": -15.814998626708984,
      "activations/layer2_attention_weight_max": 24.955890655517578,
      "activations/layer2_attention_weight_min": -20.231639862060547,
      "activations/layer3_attention_weight_max": 53.90328598022461,
      "activations/layer3_attention_weight_min": -45.67061233520508,
      "activations/layer4_attention_weight_max": 83.07186889648438,
      "activations/layer4_attention_weight_min": -72.29283905029297,
      "activations/layer5_attention_weight_max": 149.39727783203125,
      "activations/layer5_attention_weight_min": -128.58599853515625,
      "activations/layer6_attention_weight_max": 82.03878021240234,
      "activations/layer6_attention_weight_min": -76.97151184082031,
      "activations/layer7_attention_weight_max": 80.16854858398438,
      "activations/layer7_attention_weight_min": -72.31428527832031,
      "activations/layer8_attention_weight_max": 66.46454620361328,
      "activations/layer8_attention_weight_min": -59.44780349731445,
      "activations/layer9_attention_weight_max": 70.58943939208984,
      "activations/layer9_attention_weight_min": -61.65077590942383,
      "epoch": 0.48,
      "learning_rate": 0.00014839848484848482,
      "loss": 3.3042,
      "step": 8250
    },
    {
      "activations/layer0_attention_weight_max": 14.824348449707031,
      "activations/layer0_attention_weight_min": -9.691533088684082,
      "activations/layer10_attention_weight_max": 68.24259185791016,
      "activations/layer10_attention_weight_min": -59.86302185058594,
      "activations/layer11_attention_weight_max": 44.934791564941406,
      "activations/layer11_attention_weight_min": -37.22551345825195,
      "activations/layer12_attention_weight_max": 28.860536575317383,
      "activations/layer12_attention_weight_min": -23.675174713134766,
      "activations/layer13_attention_weight_max": 44.597618103027344,
      "activations/layer13_attention_weight_min": -33.60327911376953,
      "activations/layer14_attention_weight_max": 44.46360397338867,
      "activations/layer14_attention_weight_min": -27.29183578491211,
      "activations/layer15_attention_weight_max": 41.51185989379883,
      "activations/layer15_attention_weight_min": -27.181293487548828,
      "activations/layer16_attention_weight_max": 33.33491134643555,
      "activations/layer16_attention_weight_min": -20.448091506958008,
      "activations/layer17_attention_weight_max": 52.88527297973633,
      "activations/layer17_attention_weight_min": -30.040931701660156,
      "activations/layer18_attention_weight_max": 59.416175842285156,
      "activations/layer18_attention_weight_min": -22.86094093322754,
      "activations/layer19_attention_weight_max": 26.29524040222168,
      "activations/layer19_attention_weight_min": -11.37427043914795,
      "activations/layer1_attention_weight_max": 15.74812126159668,
      "activations/layer1_attention_weight_min": -10.616073608398438,
      "activations/layer20_attention_weight_max": 18.996261596679688,
      "activations/layer20_attention_weight_min": -12.373096466064453,
      "activations/layer21_attention_weight_max": 29.310226440429688,
      "activations/layer21_attention_weight_min": -16.667774200439453,
      "activations/layer22_attention_weight_max": 28.124183654785156,
      "activations/layer22_attention_weight_min": -16.579256057739258,
      "activations/layer23_attention_weight_max": 20.261821746826172,
      "activations/layer23_attention_weight_min": -16.39151954650879,
      "activations/layer2_attention_weight_max": 26.380619049072266,
      "activations/layer2_attention_weight_min": -20.14864730834961,
      "activations/layer3_attention_weight_max": 59.350364685058594,
      "activations/layer3_attention_weight_min": -48.96723175048828,
      "activations/layer4_attention_weight_max": 85.98053741455078,
      "activations/layer4_attention_weight_min": -71.96575164794922,
      "activations/layer5_attention_weight_max": 160.8821563720703,
      "activations/layer5_attention_weight_min": -132.97305297851562,
      "activations/layer6_attention_weight_max": 87.75558471679688,
      "activations/layer6_attention_weight_min": -73.47368621826172,
      "activations/layer7_attention_weight_max": 84.61550903320312,
      "activations/layer7_attention_weight_min": -72.78636932373047,
      "activations/layer8_attention_weight_max": 66.61649322509766,
      "activations/layer8_attention_weight_min": -57.92146301269531,
      "activations/layer9_attention_weight_max": 76.05694580078125,
      "activations/layer9_attention_weight_min": -62.985595703125,
      "epoch": 0.48,
      "learning_rate": 0.00014837954545454543,
      "loss": 3.2865,
      "step": 8300
    },
    {
      "activations/layer0_attention_weight_max": 13.7601318359375,
      "activations/layer0_attention_weight_min": -9.490501403808594,
      "activations/layer10_attention_weight_max": 70.7061538696289,
      "activations/layer10_attention_weight_min": -60.76819610595703,
      "activations/layer11_attention_weight_max": 41.23278045654297,
      "activations/layer11_attention_weight_min": -34.813053131103516,
      "activations/layer12_attention_weight_max": 25.480300903320312,
      "activations/layer12_attention_weight_min": -21.045936584472656,
      "activations/layer13_attention_weight_max": 46.65879440307617,
      "activations/layer13_attention_weight_min": -34.213592529296875,
      "activations/layer14_attention_weight_max": 38.098060607910156,
      "activations/layer14_attention_weight_min": -27.944480895996094,
      "activations/layer15_attention_weight_max": 38.53882598876953,
      "activations/layer15_attention_weight_min": -25.302724838256836,
      "activations/layer16_attention_weight_max": 27.45909309387207,
      "activations/layer16_attention_weight_min": -20.358591079711914,
      "activations/layer17_attention_weight_max": 49.50497817993164,
      "activations/layer17_attention_weight_min": -28.68393898010254,
      "activations/layer18_attention_weight_max": 38.882476806640625,
      "activations/layer18_attention_weight_min": -23.832382202148438,
      "activations/layer19_attention_weight_max": 25.56529998779297,
      "activations/layer19_attention_weight_min": -12.540282249450684,
      "activations/layer1_attention_weight_max": 16.937631607055664,
      "activations/layer1_attention_weight_min": -10.150705337524414,
      "activations/layer20_attention_weight_max": 21.925016403198242,
      "activations/layer20_attention_weight_min": -11.579174995422363,
      "activations/layer21_attention_weight_max": 30.915348052978516,
      "activations/layer21_attention_weight_min": -15.700737953186035,
      "activations/layer22_attention_weight_max": 28.48573112487793,
      "activations/layer22_attention_weight_min": -14.7040433883667,
      "activations/layer23_attention_weight_max": 21.253774642944336,
      "activations/layer23_attention_weight_min": -14.331019401550293,
      "activations/layer2_attention_weight_max": 26.771610260009766,
      "activations/layer2_attention_weight_min": -19.389144897460938,
      "activations/layer3_attention_weight_max": 51.08303451538086,
      "activations/layer3_attention_weight_min": -43.463260650634766,
      "activations/layer4_attention_weight_max": 74.79723358154297,
      "activations/layer4_attention_weight_min": -68.29891967773438,
      "activations/layer5_attention_weight_max": 146.2032470703125,
      "activations/layer5_attention_weight_min": -123.92916107177734,
      "activations/layer6_attention_weight_max": 80.07366180419922,
      "activations/layer6_attention_weight_min": -72.81787109375,
      "activations/layer7_attention_weight_max": 77.08480072021484,
      "activations/layer7_attention_weight_min": -72.64389038085938,
      "activations/layer8_attention_weight_max": 62.8315315246582,
      "activations/layer8_attention_weight_min": -56.00339126586914,
      "activations/layer9_attention_weight_max": 81.65670013427734,
      "activations/layer9_attention_weight_min": -67.76753234863281,
      "epoch": 0.49,
      "learning_rate": 0.00014836060606060605,
      "loss": 3.2854,
      "step": 8350
    },
    {
      "activations/layer0_attention_weight_max": 13.714095115661621,
      "activations/layer0_attention_weight_min": -9.83145523071289,
      "activations/layer10_attention_weight_max": 65.89232635498047,
      "activations/layer10_attention_weight_min": -53.97673797607422,
      "activations/layer11_attention_weight_max": 46.5058708190918,
      "activations/layer11_attention_weight_min": -35.70695877075195,
      "activations/layer12_attention_weight_max": 24.832862854003906,
      "activations/layer12_attention_weight_min": -24.02255630493164,
      "activations/layer13_attention_weight_max": 48.96949768066406,
      "activations/layer13_attention_weight_min": -33.834835052490234,
      "activations/layer14_attention_weight_max": 37.731163024902344,
      "activations/layer14_attention_weight_min": -25.956785202026367,
      "activations/layer15_attention_weight_max": 37.883148193359375,
      "activations/layer15_attention_weight_min": -26.266460418701172,
      "activations/layer16_attention_weight_max": 28.00991439819336,
      "activations/layer16_attention_weight_min": -20.02252769470215,
      "activations/layer17_attention_weight_max": 49.61347961425781,
      "activations/layer17_attention_weight_min": -30.39474868774414,
      "activations/layer18_attention_weight_max": 59.1977424621582,
      "activations/layer18_attention_weight_min": -27.85198402404785,
      "activations/layer19_attention_weight_max": 21.798978805541992,
      "activations/layer19_attention_weight_min": -10.091779708862305,
      "activations/layer1_attention_weight_max": 18.92506980895996,
      "activations/layer1_attention_weight_min": -10.746784210205078,
      "activations/layer20_attention_weight_max": 19.084697723388672,
      "activations/layer20_attention_weight_min": -10.564589500427246,
      "activations/layer21_attention_weight_max": 30.822582244873047,
      "activations/layer21_attention_weight_min": -14.213523864746094,
      "activations/layer22_attention_weight_max": 30.7601318359375,
      "activations/layer22_attention_weight_min": -12.217070579528809,
      "activations/layer23_attention_weight_max": 20.436603546142578,
      "activations/layer23_attention_weight_min": -14.470918655395508,
      "activations/layer2_attention_weight_max": 28.179616928100586,
      "activations/layer2_attention_weight_min": -20.435945510864258,
      "activations/layer3_attention_weight_max": 55.8359260559082,
      "activations/layer3_attention_weight_min": -44.68497085571289,
      "activations/layer4_attention_weight_max": 82.71550750732422,
      "activations/layer4_attention_weight_min": -70.87245178222656,
      "activations/layer5_attention_weight_max": 149.9586181640625,
      "activations/layer5_attention_weight_min": -123.92717742919922,
      "activations/layer6_attention_weight_max": 82.83123016357422,
      "activations/layer6_attention_weight_min": -74.19880676269531,
      "activations/layer7_attention_weight_max": 84.08541107177734,
      "activations/layer7_attention_weight_min": -68.45734405517578,
      "activations/layer8_attention_weight_max": 69.50794219970703,
      "activations/layer8_attention_weight_min": -57.24850082397461,
      "activations/layer9_attention_weight_max": 73.71863555908203,
      "activations/layer9_attention_weight_min": -65.4101791381836,
      "epoch": 0.49,
      "learning_rate": 0.00014834204545454544,
      "loss": 3.281,
      "step": 8400
    },
    {
      "activations/layer0_attention_weight_max": 12.848217964172363,
      "activations/layer0_attention_weight_min": -9.801573753356934,
      "activations/layer10_attention_weight_max": 68.19966888427734,
      "activations/layer10_attention_weight_min": -61.85314178466797,
      "activations/layer11_attention_weight_max": 43.84977340698242,
      "activations/layer11_attention_weight_min": -35.78479766845703,
      "activations/layer12_attention_weight_max": 25.702289581298828,
      "activations/layer12_attention_weight_min": -21.563657760620117,
      "activations/layer13_attention_weight_max": 45.289756774902344,
      "activations/layer13_attention_weight_min": -36.18793487548828,
      "activations/layer14_attention_weight_max": 36.42525100708008,
      "activations/layer14_attention_weight_min": -28.158607482910156,
      "activations/layer15_attention_weight_max": 37.2639045715332,
      "activations/layer15_attention_weight_min": -26.53585433959961,
      "activations/layer16_attention_weight_max": 28.051300048828125,
      "activations/layer16_attention_weight_min": -19.827241897583008,
      "activations/layer17_attention_weight_max": 47.9771614074707,
      "activations/layer17_attention_weight_min": -31.44970703125,
      "activations/layer18_attention_weight_max": 36.51030349731445,
      "activations/layer18_attention_weight_min": -20.202924728393555,
      "activations/layer19_attention_weight_max": 21.819503784179688,
      "activations/layer19_attention_weight_min": -10.366728782653809,
      "activations/layer1_attention_weight_max": 16.71995735168457,
      "activations/layer1_attention_weight_min": -10.303741455078125,
      "activations/layer20_attention_weight_max": 20.840869903564453,
      "activations/layer20_attention_weight_min": -12.595155715942383,
      "activations/layer21_attention_weight_max": 30.600460052490234,
      "activations/layer21_attention_weight_min": -13.916486740112305,
      "activations/layer22_attention_weight_max": 27.84718132019043,
      "activations/layer22_attention_weight_min": -13.924583435058594,
      "activations/layer23_attention_weight_max": 21.72228240966797,
      "activations/layer23_attention_weight_min": -13.81849193572998,
      "activations/layer2_attention_weight_max": 26.131481170654297,
      "activations/layer2_attention_weight_min": -19.959075927734375,
      "activations/layer3_attention_weight_max": 52.10417938232422,
      "activations/layer3_attention_weight_min": -45.7120246887207,
      "activations/layer4_attention_weight_max": 78.59049224853516,
      "activations/layer4_attention_weight_min": -68.76036834716797,
      "activations/layer5_attention_weight_max": 139.65176391601562,
      "activations/layer5_attention_weight_min": -125.85823822021484,
      "activations/layer6_attention_weight_max": 80.75374603271484,
      "activations/layer6_attention_weight_min": -78.49533081054688,
      "activations/layer7_attention_weight_max": 84.1591796875,
      "activations/layer7_attention_weight_min": -77.34711456298828,
      "activations/layer8_attention_weight_max": 69.66085052490234,
      "activations/layer8_attention_weight_min": -61.99037551879883,
      "activations/layer9_attention_weight_max": 71.47222137451172,
      "activations/layer9_attention_weight_min": -66.47441101074219,
      "epoch": 0.49,
      "learning_rate": 0.00014832310606060603,
      "loss": 3.2938,
      "step": 8450
    },
    {
      "activations/layer0_attention_weight_max": 13.556452751159668,
      "activations/layer0_attention_weight_min": -9.703276634216309,
      "activations/layer10_attention_weight_max": 65.85588073730469,
      "activations/layer10_attention_weight_min": -54.46353530883789,
      "activations/layer11_attention_weight_max": 44.33245849609375,
      "activations/layer11_attention_weight_min": -34.97834777832031,
      "activations/layer12_attention_weight_max": 27.27719497680664,
      "activations/layer12_attention_weight_min": -22.75257110595703,
      "activations/layer13_attention_weight_max": 46.795494079589844,
      "activations/layer13_attention_weight_min": -34.385257720947266,
      "activations/layer14_attention_weight_max": 30.801984786987305,
      "activations/layer14_attention_weight_min": -24.833175659179688,
      "activations/layer15_attention_weight_max": 38.22929763793945,
      "activations/layer15_attention_weight_min": -25.54611587524414,
      "activations/layer16_attention_weight_max": 30.822574615478516,
      "activations/layer16_attention_weight_min": -21.734832763671875,
      "activations/layer17_attention_weight_max": 39.626304626464844,
      "activations/layer17_attention_weight_min": -29.14828872680664,
      "activations/layer18_attention_weight_max": 38.37263870239258,
      "activations/layer18_attention_weight_min": -25.340003967285156,
      "activations/layer19_attention_weight_max": 19.67581558227539,
      "activations/layer19_attention_weight_min": -11.127716064453125,
      "activations/layer1_attention_weight_max": 17.304981231689453,
      "activations/layer1_attention_weight_min": -10.949437141418457,
      "activations/layer20_attention_weight_max": 19.34177017211914,
      "activations/layer20_attention_weight_min": -12.349031448364258,
      "activations/layer21_attention_weight_max": 21.789085388183594,
      "activations/layer21_attention_weight_min": -17.571622848510742,
      "activations/layer22_attention_weight_max": 23.173099517822266,
      "activations/layer22_attention_weight_min": -15.795491218566895,
      "activations/layer23_attention_weight_max": 19.757404327392578,
      "activations/layer23_attention_weight_min": -16.61594009399414,
      "activations/layer2_attention_weight_max": 26.58749008178711,
      "activations/layer2_attention_weight_min": -20.225223541259766,
      "activations/layer3_attention_weight_max": 51.04717254638672,
      "activations/layer3_attention_weight_min": -44.039730072021484,
      "activations/layer4_attention_weight_max": 79.11608123779297,
      "activations/layer4_attention_weight_min": -67.77035522460938,
      "activations/layer5_attention_weight_max": 151.33676147460938,
      "activations/layer5_attention_weight_min": -126.62298583984375,
      "activations/layer6_attention_weight_max": 83.51778411865234,
      "activations/layer6_attention_weight_min": -72.85997772216797,
      "activations/layer7_attention_weight_max": 80.9922103881836,
      "activations/layer7_attention_weight_min": -71.45127868652344,
      "activations/layer8_attention_weight_max": 69.0390396118164,
      "activations/layer8_attention_weight_min": -59.15176773071289,
      "activations/layer9_attention_weight_max": 72.4074935913086,
      "activations/layer9_attention_weight_min": -67.37215423583984,
      "epoch": 0.49,
      "learning_rate": 0.00014830416666666664,
      "loss": 3.289,
      "step": 8500
    },
    {
      "activations/layer0_attention_weight_max": 14.12431526184082,
      "activations/layer0_attention_weight_min": -9.965960502624512,
      "activations/layer10_attention_weight_max": 68.50440979003906,
      "activations/layer10_attention_weight_min": -59.323482513427734,
      "activations/layer11_attention_weight_max": 43.69529724121094,
      "activations/layer11_attention_weight_min": -33.680328369140625,
      "activations/layer12_attention_weight_max": 24.066268920898438,
      "activations/layer12_attention_weight_min": -21.233230590820312,
      "activations/layer13_attention_weight_max": 43.6349983215332,
      "activations/layer13_attention_weight_min": -33.899227142333984,
      "activations/layer14_attention_weight_max": 34.37895965576172,
      "activations/layer14_attention_weight_min": -26.40089988708496,
      "activations/layer15_attention_weight_max": 39.50148010253906,
      "activations/layer15_attention_weight_min": -24.50868797302246,
      "activations/layer16_attention_weight_max": 28.581146240234375,
      "activations/layer16_attention_weight_min": -22.095062255859375,
      "activations/layer17_attention_weight_max": 45.98225784301758,
      "activations/layer17_attention_weight_min": -30.614620208740234,
      "activations/layer18_attention_weight_max": 42.43920135498047,
      "activations/layer18_attention_weight_min": -29.934579849243164,
      "activations/layer19_attention_weight_max": 20.479692459106445,
      "activations/layer19_attention_weight_min": -11.335576057434082,
      "activations/layer1_attention_weight_max": 16.59744644165039,
      "activations/layer1_attention_weight_min": -10.59577465057373,
      "activations/layer20_attention_weight_max": 20.554203033447266,
      "activations/layer20_attention_weight_min": -12.153903007507324,
      "activations/layer21_attention_weight_max": 28.403087615966797,
      "activations/layer21_attention_weight_min": -14.503326416015625,
      "activations/layer22_attention_weight_max": 27.876096725463867,
      "activations/layer22_attention_weight_min": -12.886449813842773,
      "activations/layer23_attention_weight_max": 21.71185874938965,
      "activations/layer23_attention_weight_min": -14.956253051757812,
      "activations/layer2_attention_weight_max": 26.662153244018555,
      "activations/layer2_attention_weight_min": -19.49984359741211,
      "activations/layer3_attention_weight_max": 52.30518341064453,
      "activations/layer3_attention_weight_min": -44.384281158447266,
      "activations/layer4_attention_weight_max": 80.32840728759766,
      "activations/layer4_attention_weight_min": -69.3481674194336,
      "activations/layer5_attention_weight_max": 142.43875122070312,
      "activations/layer5_attention_weight_min": -124.60255432128906,
      "activations/layer6_attention_weight_max": 79.52181243896484,
      "activations/layer6_attention_weight_min": -71.09718322753906,
      "activations/layer7_attention_weight_max": 78.82889556884766,
      "activations/layer7_attention_weight_min": -71.89676666259766,
      "activations/layer8_attention_weight_max": 68.59003448486328,
      "activations/layer8_attention_weight_min": -58.71516418457031,
      "activations/layer9_attention_weight_max": 72.05204010009766,
      "activations/layer9_attention_weight_min": -67.31884765625,
      "epoch": 0.5,
      "learning_rate": 0.00014828522727272726,
      "loss": 3.2848,
      "step": 8550
    },
    {
      "activations/layer0_attention_weight_max": 14.019922256469727,
      "activations/layer0_attention_weight_min": -10.403148651123047,
      "activations/layer10_attention_weight_max": 61.357566833496094,
      "activations/layer10_attention_weight_min": -57.52336120605469,
      "activations/layer11_attention_weight_max": 41.05183410644531,
      "activations/layer11_attention_weight_min": -35.65068817138672,
      "activations/layer12_attention_weight_max": 24.56394386291504,
      "activations/layer12_attention_weight_min": -20.94894790649414,
      "activations/layer13_attention_weight_max": 39.53036117553711,
      "activations/layer13_attention_weight_min": -33.691932678222656,
      "activations/layer14_attention_weight_max": 33.97319030761719,
      "activations/layer14_attention_weight_min": -26.20743751525879,
      "activations/layer15_attention_weight_max": 37.9627799987793,
      "activations/layer15_attention_weight_min": -31.54509735107422,
      "activations/layer16_attention_weight_max": 30.175512313842773,
      "activations/layer16_attention_weight_min": -25.54119300842285,
      "activations/layer17_attention_weight_max": 40.450111389160156,
      "activations/layer17_attention_weight_min": -30.97220802307129,
      "activations/layer18_attention_weight_max": 30.073110580444336,
      "activations/layer18_attention_weight_min": -22.18355941772461,
      "activations/layer19_attention_weight_max": 19.67070770263672,
      "activations/layer19_attention_weight_min": -12.242924690246582,
      "activations/layer1_attention_weight_max": 17.501264572143555,
      "activations/layer1_attention_weight_min": -10.53893756866455,
      "activations/layer20_attention_weight_max": 19.23677635192871,
      "activations/layer20_attention_weight_min": -12.03648567199707,
      "activations/layer21_attention_weight_max": 27.033884048461914,
      "activations/layer21_attention_weight_min": -17.116275787353516,
      "activations/layer22_attention_weight_max": 26.61408805847168,
      "activations/layer22_attention_weight_min": -13.69092845916748,
      "activations/layer23_attention_weight_max": 19.49987030029297,
      "activations/layer23_attention_weight_min": -16.727502822875977,
      "activations/layer2_attention_weight_max": 24.803632736206055,
      "activations/layer2_attention_weight_min": -21.22051239013672,
      "activations/layer3_attention_weight_max": 53.07193374633789,
      "activations/layer3_attention_weight_min": -46.271759033203125,
      "activations/layer4_attention_weight_max": 81.7920150756836,
      "activations/layer4_attention_weight_min": -70.3416976928711,
      "activations/layer5_attention_weight_max": 147.301513671875,
      "activations/layer5_attention_weight_min": -126.74876403808594,
      "activations/layer6_attention_weight_max": 78.88325500488281,
      "activations/layer6_attention_weight_min": -74.1610107421875,
      "activations/layer7_attention_weight_max": 78.87028503417969,
      "activations/layer7_attention_weight_min": -71.64049530029297,
      "activations/layer8_attention_weight_max": 62.05516815185547,
      "activations/layer8_attention_weight_min": -55.8717041015625,
      "activations/layer9_attention_weight_max": 74.60304260253906,
      "activations/layer9_attention_weight_min": -67.5534896850586,
      "epoch": 0.5,
      "learning_rate": 0.00014826628787878787,
      "loss": 3.3015,
      "step": 8600
    },
    {
      "activations/layer0_attention_weight_max": 13.31116771697998,
      "activations/layer0_attention_weight_min": -10.513517379760742,
      "activations/layer10_attention_weight_max": 72.12466430664062,
      "activations/layer10_attention_weight_min": -58.35350036621094,
      "activations/layer11_attention_weight_max": 42.6825065612793,
      "activations/layer11_attention_weight_min": -36.865875244140625,
      "activations/layer12_attention_weight_max": 25.4991512298584,
      "activations/layer12_attention_weight_min": -21.21791648864746,
      "activations/layer13_attention_weight_max": 55.639366149902344,
      "activations/layer13_attention_weight_min": -40.26301574707031,
      "activations/layer14_attention_weight_max": 43.63544845581055,
      "activations/layer14_attention_weight_min": -28.700664520263672,
      "activations/layer15_attention_weight_max": 44.24397659301758,
      "activations/layer15_attention_weight_min": -25.839763641357422,
      "activations/layer16_attention_weight_max": 35.81462097167969,
      "activations/layer16_attention_weight_min": -23.691011428833008,
      "activations/layer17_attention_weight_max": 52.9379997253418,
      "activations/layer17_attention_weight_min": -31.47152328491211,
      "activations/layer18_attention_weight_max": 42.30693817138672,
      "activations/layer18_attention_weight_min": -23.797595977783203,
      "activations/layer19_attention_weight_max": 22.358497619628906,
      "activations/layer19_attention_weight_min": -11.41387939453125,
      "activations/layer1_attention_weight_max": 18.375652313232422,
      "activations/layer1_attention_weight_min": -11.653641700744629,
      "activations/layer20_attention_weight_max": 20.077083587646484,
      "activations/layer20_attention_weight_min": -10.448643684387207,
      "activations/layer21_attention_weight_max": 32.3349609375,
      "activations/layer21_attention_weight_min": -17.19164276123047,
      "activations/layer22_attention_weight_max": 34.70170593261719,
      "activations/layer22_attention_weight_min": -15.856668472290039,
      "activations/layer23_attention_weight_max": 22.87140464782715,
      "activations/layer23_attention_weight_min": -16.978351593017578,
      "activations/layer2_attention_weight_max": 25.645984649658203,
      "activations/layer2_attention_weight_min": -20.80921173095703,
      "activations/layer3_attention_weight_max": 54.75749588012695,
      "activations/layer3_attention_weight_min": -47.12797927856445,
      "activations/layer4_attention_weight_max": 81.52891540527344,
      "activations/layer4_attention_weight_min": -70.75332641601562,
      "activations/layer5_attention_weight_max": 154.44284057617188,
      "activations/layer5_attention_weight_min": -122.49067687988281,
      "activations/layer6_attention_weight_max": 83.4346923828125,
      "activations/layer6_attention_weight_min": -73.01542663574219,
      "activations/layer7_attention_weight_max": 82.95550537109375,
      "activations/layer7_attention_weight_min": -71.504150390625,
      "activations/layer8_attention_weight_max": 68.87935638427734,
      "activations/layer8_attention_weight_min": -60.20689010620117,
      "activations/layer9_attention_weight_max": 71.95357513427734,
      "activations/layer9_attention_weight_min": -63.51507568359375,
      "epoch": 0.5,
      "learning_rate": 0.00014824734848484846,
      "loss": 3.2775,
      "step": 8650
    },
    {
      "activations/layer0_attention_weight_max": 13.997345924377441,
      "activations/layer0_attention_weight_min": -11.051678657531738,
      "activations/layer10_attention_weight_max": 72.81494140625,
      "activations/layer10_attention_weight_min": -61.73731231689453,
      "activations/layer11_attention_weight_max": 44.96352767944336,
      "activations/layer11_attention_weight_min": -36.652130126953125,
      "activations/layer12_attention_weight_max": 27.573776245117188,
      "activations/layer12_attention_weight_min": -22.482425689697266,
      "activations/layer13_attention_weight_max": 53.342166900634766,
      "activations/layer13_attention_weight_min": -38.644615173339844,
      "activations/layer14_attention_weight_max": 53.84932327270508,
      "activations/layer14_attention_weight_min": -30.879013061523438,
      "activations/layer15_attention_weight_max": 39.04000473022461,
      "activations/layer15_attention_weight_min": -27.395870208740234,
      "activations/layer16_attention_weight_max": 31.59857940673828,
      "activations/layer16_attention_weight_min": -22.529415130615234,
      "activations/layer17_attention_weight_max": 45.18627166748047,
      "activations/layer17_attention_weight_min": -32.32907485961914,
      "activations/layer18_attention_weight_max": 40.35142517089844,
      "activations/layer18_attention_weight_min": -24.289533615112305,
      "activations/layer19_attention_weight_max": 20.211156845092773,
      "activations/layer19_attention_weight_min": -14.285923957824707,
      "activations/layer1_attention_weight_max": 17.523372650146484,
      "activations/layer1_attention_weight_min": -10.666274070739746,
      "activations/layer20_attention_weight_max": 17.629621505737305,
      "activations/layer20_attention_weight_min": -12.949560165405273,
      "activations/layer21_attention_weight_max": 27.922409057617188,
      "activations/layer21_attention_weight_min": -16.80498695373535,
      "activations/layer22_attention_weight_max": 25.478668212890625,
      "activations/layer22_attention_weight_min": -13.814401626586914,
      "activations/layer23_attention_weight_max": 19.005361557006836,
      "activations/layer23_attention_weight_min": -17.281591415405273,
      "activations/layer2_attention_weight_max": 24.578527450561523,
      "activations/layer2_attention_weight_min": -20.122936248779297,
      "activations/layer3_attention_weight_max": 53.677608489990234,
      "activations/layer3_attention_weight_min": -45.556644439697266,
      "activations/layer4_attention_weight_max": 80.8193359375,
      "activations/layer4_attention_weight_min": -68.43661499023438,
      "activations/layer5_attention_weight_max": 148.1461944580078,
      "activations/layer5_attention_weight_min": -126.99607849121094,
      "activations/layer6_attention_weight_max": 82.52255249023438,
      "activations/layer6_attention_weight_min": -70.10165405273438,
      "activations/layer7_attention_weight_max": 82.9135971069336,
      "activations/layer7_attention_weight_min": -71.35456848144531,
      "activations/layer8_attention_weight_max": 64.36932373046875,
      "activations/layer8_attention_weight_min": -55.65680694580078,
      "activations/layer9_attention_weight_max": 86.88471221923828,
      "activations/layer9_attention_weight_min": -66.25615692138672,
      "epoch": 0.51,
      "learning_rate": 0.00014822840909090907,
      "loss": 3.2859,
      "step": 8700
    },
    {
      "activations/layer0_attention_weight_max": 13.494644165039062,
      "activations/layer0_attention_weight_min": -9.31330680847168,
      "activations/layer10_attention_weight_max": 65.63154602050781,
      "activations/layer10_attention_weight_min": -58.16838836669922,
      "activations/layer11_attention_weight_max": 47.73411560058594,
      "activations/layer11_attention_weight_min": -36.491668701171875,
      "activations/layer12_attention_weight_max": 25.49298095703125,
      "activations/layer12_attention_weight_min": -22.348617553710938,
      "activations/layer13_attention_weight_max": 60.71012496948242,
      "activations/layer13_attention_weight_min": -35.71559143066406,
      "activations/layer14_attention_weight_max": 55.21623992919922,
      "activations/layer14_attention_weight_min": -29.431686401367188,
      "activations/layer15_attention_weight_max": 44.19411849975586,
      "activations/layer15_attention_weight_min": -24.65789794921875,
      "activations/layer16_attention_weight_max": 32.595035552978516,
      "activations/layer16_attention_weight_min": -24.388402938842773,
      "activations/layer17_attention_weight_max": 62.69411087036133,
      "activations/layer17_attention_weight_min": -34.359527587890625,
      "activations/layer18_attention_weight_max": 51.4786491394043,
      "activations/layer18_attention_weight_min": -25.335641860961914,
      "activations/layer19_attention_weight_max": 22.021656036376953,
      "activations/layer19_attention_weight_min": -10.313738822937012,
      "activations/layer1_attention_weight_max": 17.758678436279297,
      "activations/layer1_attention_weight_min": -10.558305740356445,
      "activations/layer20_attention_weight_max": 18.285812377929688,
      "activations/layer20_attention_weight_min": -12.446860313415527,
      "activations/layer21_attention_weight_max": 23.914825439453125,
      "activations/layer21_attention_weight_min": -13.622200965881348,
      "activations/layer22_attention_weight_max": 26.516761779785156,
      "activations/layer22_attention_weight_min": -14.885501861572266,
      "activations/layer23_attention_weight_max": 19.976987838745117,
      "activations/layer23_attention_weight_min": -17.263320922851562,
      "activations/layer2_attention_weight_max": 26.13182830810547,
      "activations/layer2_attention_weight_min": -19.577457427978516,
      "activations/layer3_attention_weight_max": 52.83633804321289,
      "activations/layer3_attention_weight_min": -42.63453674316406,
      "activations/layer4_attention_weight_max": 78.15099334716797,
      "activations/layer4_attention_weight_min": -66.7772216796875,
      "activations/layer5_attention_weight_max": 149.04476928710938,
      "activations/layer5_attention_weight_min": -123.74505615234375,
      "activations/layer6_attention_weight_max": 84.19284057617188,
      "activations/layer6_attention_weight_min": -70.39337158203125,
      "activations/layer7_attention_weight_max": 85.88492584228516,
      "activations/layer7_attention_weight_min": -70.56391906738281,
      "activations/layer8_attention_weight_max": 73.83723449707031,
      "activations/layer8_attention_weight_min": -59.3570442199707,
      "activations/layer9_attention_weight_max": 76.26568603515625,
      "activations/layer9_attention_weight_min": -62.60264205932617,
      "epoch": 0.51,
      "learning_rate": 0.0001482094696969697,
      "loss": 3.2554,
      "step": 8750
    },
    {
      "activations/layer0_attention_weight_max": 16.578805923461914,
      "activations/layer0_attention_weight_min": -9.971942901611328,
      "activations/layer10_attention_weight_max": 66.59579467773438,
      "activations/layer10_attention_weight_min": -54.102745056152344,
      "activations/layer11_attention_weight_max": 39.99022674560547,
      "activations/layer11_attention_weight_min": -34.12164306640625,
      "activations/layer12_attention_weight_max": 23.80735969543457,
      "activations/layer12_attention_weight_min": -25.39531707763672,
      "activations/layer13_attention_weight_max": 43.46971893310547,
      "activations/layer13_attention_weight_min": -34.455989837646484,
      "activations/layer14_attention_weight_max": 37.16960144042969,
      "activations/layer14_attention_weight_min": -26.655990600585938,
      "activations/layer15_attention_weight_max": 34.99984359741211,
      "activations/layer15_attention_weight_min": -26.129531860351562,
      "activations/layer16_attention_weight_max": 32.72639083862305,
      "activations/layer16_attention_weight_min": -23.133268356323242,
      "activations/layer17_attention_weight_max": 50.30965042114258,
      "activations/layer17_attention_weight_min": -32.491127014160156,
      "activations/layer18_attention_weight_max": 44.4578971862793,
      "activations/layer18_attention_weight_min": -22.31200408935547,
      "activations/layer19_attention_weight_max": 20.30931854248047,
      "activations/layer19_attention_weight_min": -12.1041898727417,
      "activations/layer1_attention_weight_max": 17.39466667175293,
      "activations/layer1_attention_weight_min": -11.598387718200684,
      "activations/layer20_attention_weight_max": 20.446861267089844,
      "activations/layer20_attention_weight_min": -12.504521369934082,
      "activations/layer21_attention_weight_max": 28.448917388916016,
      "activations/layer21_attention_weight_min": -13.840179443359375,
      "activations/layer22_attention_weight_max": 27.64850616455078,
      "activations/layer22_attention_weight_min": -17.49149513244629,
      "activations/layer23_attention_weight_max": 21.094268798828125,
      "activations/layer23_attention_weight_min": -17.46381950378418,
      "activations/layer2_attention_weight_max": 26.68964958190918,
      "activations/layer2_attention_weight_min": -19.495418548583984,
      "activations/layer3_attention_weight_max": 53.986270904541016,
      "activations/layer3_attention_weight_min": -44.058197021484375,
      "activations/layer4_attention_weight_max": 84.79082489013672,
      "activations/layer4_attention_weight_min": -67.68441772460938,
      "activations/layer5_attention_weight_max": 155.07504272460938,
      "activations/layer5_attention_weight_min": -124.22621154785156,
      "activations/layer6_attention_weight_max": 83.1103744506836,
      "activations/layer6_attention_weight_min": -71.99935150146484,
      "activations/layer7_attention_weight_max": 83.88101196289062,
      "activations/layer7_attention_weight_min": -72.08026123046875,
      "activations/layer8_attention_weight_max": 66.77806091308594,
      "activations/layer8_attention_weight_min": -57.6137580871582,
      "activations/layer9_attention_weight_max": 86.23491668701172,
      "activations/layer9_attention_weight_min": -75.78632354736328,
      "epoch": 0.51,
      "learning_rate": 0.0001481905303030303,
      "loss": 3.2944,
      "step": 8800
    },
    {
      "activations/layer0_attention_weight_max": 13.154802322387695,
      "activations/layer0_attention_weight_min": -9.48983097076416,
      "activations/layer10_attention_weight_max": 68.4164810180664,
      "activations/layer10_attention_weight_min": -59.111324310302734,
      "activations/layer11_attention_weight_max": 41.52334213256836,
      "activations/layer11_attention_weight_min": -35.44743347167969,
      "activations/layer12_attention_weight_max": 26.53474235534668,
      "activations/layer12_attention_weight_min": -23.785614013671875,
      "activations/layer13_attention_weight_max": 54.222286224365234,
      "activations/layer13_attention_weight_min": -39.51696014404297,
      "activations/layer14_attention_weight_max": 42.94282531738281,
      "activations/layer14_attention_weight_min": -29.029878616333008,
      "activations/layer15_attention_weight_max": 39.2452278137207,
      "activations/layer15_attention_weight_min": -25.447914123535156,
      "activations/layer16_attention_weight_max": 34.43996810913086,
      "activations/layer16_attention_weight_min": -21.812589645385742,
      "activations/layer17_attention_weight_max": 52.125308990478516,
      "activations/layer17_attention_weight_min": -30.13762092590332,
      "activations/layer18_attention_weight_max": 46.11905288696289,
      "activations/layer18_attention_weight_min": -26.57209587097168,
      "activations/layer19_attention_weight_max": 18.80548858642578,
      "activations/layer19_attention_weight_min": -10.734847068786621,
      "activations/layer1_attention_weight_max": 18.377702713012695,
      "activations/layer1_attention_weight_min": -10.920563697814941,
      "activations/layer20_attention_weight_max": 19.408903121948242,
      "activations/layer20_attention_weight_min": -10.459439277648926,
      "activations/layer21_attention_weight_max": 31.987661361694336,
      "activations/layer21_attention_weight_min": -15.722582817077637,
      "activations/layer22_attention_weight_max": 30.928438186645508,
      "activations/layer22_attention_weight_min": -14.04354190826416,
      "activations/layer23_attention_weight_max": 21.52721405029297,
      "activations/layer23_attention_weight_min": -16.1159725189209,
      "activations/layer2_attention_weight_max": 25.888187408447266,
      "activations/layer2_attention_weight_min": -21.939680099487305,
      "activations/layer3_attention_weight_max": 59.774444580078125,
      "activations/layer3_attention_weight_min": -49.820579528808594,
      "activations/layer4_attention_weight_max": 90.53453063964844,
      "activations/layer4_attention_weight_min": -72.53741455078125,
      "activations/layer5_attention_weight_max": 158.779541015625,
      "activations/layer5_attention_weight_min": -125.86541748046875,
      "activations/layer6_attention_weight_max": 90.26193237304688,
      "activations/layer6_attention_weight_min": -73.02606201171875,
      "activations/layer7_attention_weight_max": 88.44512176513672,
      "activations/layer7_attention_weight_min": -71.8961410522461,
      "activations/layer8_attention_weight_max": 73.08068084716797,
      "activations/layer8_attention_weight_min": -62.86445999145508,
      "activations/layer9_attention_weight_max": 74.72032928466797,
      "activations/layer9_attention_weight_min": -61.35000228881836,
      "epoch": 0.51,
      "learning_rate": 0.0001481715909090909,
      "loss": 3.2567,
      "step": 8850
    },
    {
      "activations/layer0_attention_weight_max": 14.22570514678955,
      "activations/layer0_attention_weight_min": -10.057890892028809,
      "activations/layer10_attention_weight_max": 65.79315185546875,
      "activations/layer10_attention_weight_min": -56.2333984375,
      "activations/layer11_attention_weight_max": 45.618682861328125,
      "activations/layer11_attention_weight_min": -36.03533172607422,
      "activations/layer12_attention_weight_max": 31.500581741333008,
      "activations/layer12_attention_weight_min": -22.93464469909668,
      "activations/layer13_attention_weight_max": 49.742958068847656,
      "activations/layer13_attention_weight_min": -36.37211608886719,
      "activations/layer14_attention_weight_max": 39.1744499206543,
      "activations/layer14_attention_weight_min": -28.20105743408203,
      "activations/layer15_attention_weight_max": 41.22054672241211,
      "activations/layer15_attention_weight_min": -27.529586791992188,
      "activations/layer16_attention_weight_max": 31.188976287841797,
      "activations/layer16_attention_weight_min": -22.998353958129883,
      "activations/layer17_attention_weight_max": 50.111324310302734,
      "activations/layer17_attention_weight_min": -32.8402214050293,
      "activations/layer18_attention_weight_max": 38.6967887878418,
      "activations/layer18_attention_weight_min": -22.865474700927734,
      "activations/layer19_attention_weight_max": 19.561750411987305,
      "activations/layer19_attention_weight_min": -11.049856185913086,
      "activations/layer1_attention_weight_max": 18.430072784423828,
      "activations/layer1_attention_weight_min": -10.092369079589844,
      "activations/layer20_attention_weight_max": 20.056978225708008,
      "activations/layer20_attention_weight_min": -10.742025375366211,
      "activations/layer21_attention_weight_max": 32.55879211425781,
      "activations/layer21_attention_weight_min": -15.457183837890625,
      "activations/layer22_attention_weight_max": 31.488075256347656,
      "activations/layer22_attention_weight_min": -16.27128791809082,
      "activations/layer23_attention_weight_max": 22.375900268554688,
      "activations/layer23_attention_weight_min": -16.35152816772461,
      "activations/layer2_attention_weight_max": 25.3800048828125,
      "activations/layer2_attention_weight_min": -19.53653335571289,
      "activations/layer3_attention_weight_max": 57.022003173828125,
      "activations/layer3_attention_weight_min": -45.15386199951172,
      "activations/layer4_attention_weight_max": 82.73613739013672,
      "activations/layer4_attention_weight_min": -71.18705749511719,
      "activations/layer5_attention_weight_max": 151.9273223876953,
      "activations/layer5_attention_weight_min": -131.74412536621094,
      "activations/layer6_attention_weight_max": 81.1650161743164,
      "activations/layer6_attention_weight_min": -73.23764038085938,
      "activations/layer7_attention_weight_max": 81.62557983398438,
      "activations/layer7_attention_weight_min": -69.57162475585938,
      "activations/layer8_attention_weight_max": 68.9240493774414,
      "activations/layer8_attention_weight_min": -57.18694305419922,
      "activations/layer9_attention_weight_max": 72.37474822998047,
      "activations/layer9_attention_weight_min": -66.24960327148438,
      "epoch": 0.52,
      "learning_rate": 0.0001481526515151515,
      "loss": 3.2642,
      "step": 8900
    },
    {
      "activations/layer0_attention_weight_max": 13.916711807250977,
      "activations/layer0_attention_weight_min": -10.740120887756348,
      "activations/layer10_attention_weight_max": 62.11726760864258,
      "activations/layer10_attention_weight_min": -58.360652923583984,
      "activations/layer11_attention_weight_max": 49.47452926635742,
      "activations/layer11_attention_weight_min": -39.259857177734375,
      "activations/layer12_attention_weight_max": 25.31195831298828,
      "activations/layer12_attention_weight_min": -25.933719635009766,
      "activations/layer13_attention_weight_max": 55.28594207763672,
      "activations/layer13_attention_weight_min": -41.58076095581055,
      "activations/layer14_attention_weight_max": 43.42665481567383,
      "activations/layer14_attention_weight_min": -28.914445877075195,
      "activations/layer15_attention_weight_max": 38.88615417480469,
      "activations/layer15_attention_weight_min": -30.574125289916992,
      "activations/layer16_attention_weight_max": 31.59874725341797,
      "activations/layer16_attention_weight_min": -25.34564208984375,
      "activations/layer17_attention_weight_max": 48.97224044799805,
      "activations/layer17_attention_weight_min": -32.020652770996094,
      "activations/layer18_attention_weight_max": 34.12160110473633,
      "activations/layer18_attention_weight_min": -23.259851455688477,
      "activations/layer19_attention_weight_max": 19.352375030517578,
      "activations/layer19_attention_weight_min": -10.451148986816406,
      "activations/layer1_attention_weight_max": 16.536100387573242,
      "activations/layer1_attention_weight_min": -10.49026107788086,
      "activations/layer20_attention_weight_max": 18.447080612182617,
      "activations/layer20_attention_weight_min": -12.780731201171875,
      "activations/layer21_attention_weight_max": 27.789743423461914,
      "activations/layer21_attention_weight_min": -16.651020050048828,
      "activations/layer22_attention_weight_max": 29.98692512512207,
      "activations/layer22_attention_weight_min": -13.36658763885498,
      "activations/layer23_attention_weight_max": 22.53594970703125,
      "activations/layer23_attention_weight_min": -16.858341217041016,
      "activations/layer2_attention_weight_max": 24.057682037353516,
      "activations/layer2_attention_weight_min": -20.40386962890625,
      "activations/layer3_attention_weight_max": 52.53850173950195,
      "activations/layer3_attention_weight_min": -47.061668395996094,
      "activations/layer4_attention_weight_max": 80.79842376708984,
      "activations/layer4_attention_weight_min": -71.65387725830078,
      "activations/layer5_attention_weight_max": 140.62478637695312,
      "activations/layer5_attention_weight_min": -121.78907775878906,
      "activations/layer6_attention_weight_max": 76.39012908935547,
      "activations/layer6_attention_weight_min": -68.65345001220703,
      "activations/layer7_attention_weight_max": 73.89717864990234,
      "activations/layer7_attention_weight_min": -68.05685424804688,
      "activations/layer8_attention_weight_max": 64.43302154541016,
      "activations/layer8_attention_weight_min": -62.15050506591797,
      "activations/layer9_attention_weight_max": 87.4990234375,
      "activations/layer9_attention_weight_min": -70.6461410522461,
      "epoch": 0.52,
      "learning_rate": 0.00014813371212121212,
      "loss": 3.2794,
      "step": 8950
    },
    {
      "activations/layer0_attention_weight_max": 14.838214874267578,
      "activations/layer0_attention_weight_min": -10.41578197479248,
      "activations/layer10_attention_weight_max": 66.60967254638672,
      "activations/layer10_attention_weight_min": -62.15719985961914,
      "activations/layer11_attention_weight_max": 44.604698181152344,
      "activations/layer11_attention_weight_min": -36.78807067871094,
      "activations/layer12_attention_weight_max": 23.01611328125,
      "activations/layer12_attention_weight_min": -28.76799201965332,
      "activations/layer13_attention_weight_max": 46.542545318603516,
      "activations/layer13_attention_weight_min": -37.91484069824219,
      "activations/layer14_attention_weight_max": 37.161590576171875,
      "activations/layer14_attention_weight_min": -30.85797119140625,
      "activations/layer15_attention_weight_max": 38.04109573364258,
      "activations/layer15_attention_weight_min": -29.046588897705078,
      "activations/layer16_attention_weight_max": 36.92165756225586,
      "activations/layer16_attention_weight_min": -24.147052764892578,
      "activations/layer17_attention_weight_max": 48.779571533203125,
      "activations/layer17_attention_weight_min": -32.14342498779297,
      "activations/layer18_attention_weight_max": 56.086639404296875,
      "activations/layer18_attention_weight_min": -28.403907775878906,
      "activations/layer19_attention_weight_max": 22.208486557006836,
      "activations/layer19_attention_weight_min": -11.47038745880127,
      "activations/layer1_attention_weight_max": 18.021381378173828,
      "activations/layer1_attention_weight_min": -11.259932518005371,
      "activations/layer20_attention_weight_max": 21.962793350219727,
      "activations/layer20_attention_weight_min": -12.922052383422852,
      "activations/layer21_attention_weight_max": 28.33403205871582,
      "activations/layer21_attention_weight_min": -16.581439971923828,
      "activations/layer22_attention_weight_max": 30.28626823425293,
      "activations/layer22_attention_weight_min": -16.63117027282715,
      "activations/layer23_attention_weight_max": 20.679624557495117,
      "activations/layer23_attention_weight_min": -16.863977432250977,
      "activations/layer2_attention_weight_max": 26.07918357849121,
      "activations/layer2_attention_weight_min": -20.39851188659668,
      "activations/layer3_attention_weight_max": 54.041847229003906,
      "activations/layer3_attention_weight_min": -46.133052825927734,
      "activations/layer4_attention_weight_max": 84.6969985961914,
      "activations/layer4_attention_weight_min": -70.54805755615234,
      "activations/layer5_attention_weight_max": 151.7587890625,
      "activations/layer5_attention_weight_min": -130.46060180664062,
      "activations/layer6_attention_weight_max": 84.53486633300781,
      "activations/layer6_attention_weight_min": -68.8740463256836,
      "activations/layer7_attention_weight_max": 80.68004608154297,
      "activations/layer7_attention_weight_min": -70.5780029296875,
      "activations/layer8_attention_weight_max": 63.29439926147461,
      "activations/layer8_attention_weight_min": -57.64656448364258,
      "activations/layer9_attention_weight_max": 72.41693115234375,
      "activations/layer9_attention_weight_min": -72.29204559326172,
      "epoch": 0.52,
      "learning_rate": 0.0001481147727272727,
      "loss": 3.2777,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_loss": 3.208984375,
      "eval_runtime": 8.1955,
      "eval_samples_per_second": 523.946,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_openwebtext_loss": 3.208984375,
      "eval_openwebtext_ppl": 24.753932740556596,
      "eval_openwebtext_runtime": 8.1955,
      "eval_openwebtext_samples_per_second": 523.946,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_wikitext_loss": 3.51171875,
      "eval_wikitext_ppl": 33.50580642730562,
      "eval_wikitext_runtime": 1.697,
      "eval_wikitext_samples_per_second": 268.703,
      "step": 9000
    },
    {
      "epoch": 0.52,
      "eval_lambada_loss": 3.861328125,
      "eval_lambada_ppl": 47.52843316962603,
      "eval_lambada_runtime": 9.2691,
      "eval_lambada_samples_per_second": 525.293,
      "step": 9000
    },
    {
      "activations/layer0_attention_weight_max": 14.022317886352539,
      "activations/layer0_attention_weight_min": -10.704634666442871,
      "activations/layer10_attention_weight_max": 61.01535415649414,
      "activations/layer10_attention_weight_min": -60.13698196411133,
      "activations/layer11_attention_weight_max": 38.13657760620117,
      "activations/layer11_attention_weight_min": -33.39614486694336,
      "activations/layer12_attention_weight_max": 23.870044708251953,
      "activations/layer12_attention_weight_min": -23.42725944519043,
      "activations/layer13_attention_weight_max": 48.14900207519531,
      "activations/layer13_attention_weight_min": -37.018348693847656,
      "activations/layer14_attention_weight_max": 42.6552734375,
      "activations/layer14_attention_weight_min": -32.42039108276367,
      "activations/layer15_attention_weight_max": 37.101776123046875,
      "activations/layer15_attention_weight_min": -28.646217346191406,
      "activations/layer16_attention_weight_max": 28.490516662597656,
      "activations/layer16_attention_weight_min": -26.828454971313477,
      "activations/layer17_attention_weight_max": 47.910823822021484,
      "activations/layer17_attention_weight_min": -38.78284454345703,
      "activations/layer18_attention_weight_max": 43.10105514526367,
      "activations/layer18_attention_weight_min": -24.73301887512207,
      "activations/layer19_attention_weight_max": 17.927289962768555,
      "activations/layer19_attention_weight_min": -13.111002922058105,
      "activations/layer1_attention_weight_max": 16.849401473999023,
      "activations/layer1_attention_weight_min": -11.813011169433594,
      "activations/layer20_attention_weight_max": 19.189109802246094,
      "activations/layer20_attention_weight_min": -12.340465545654297,
      "activations/layer21_attention_weight_max": 25.327035903930664,
      "activations/layer21_attention_weight_min": -12.618748664855957,
      "activations/layer22_attention_weight_max": 24.14797019958496,
      "activations/layer22_attention_weight_min": -13.823448181152344,
      "activations/layer23_attention_weight_max": 17.73215675354004,
      "activations/layer23_attention_weight_min": -15.357104301452637,
      "activations/layer2_attention_weight_max": 25.522937774658203,
      "activations/layer2_attention_weight_min": -20.71569061279297,
      "activations/layer3_attention_weight_max": 52.04052734375,
      "activations/layer3_attention_weight_min": -44.37851333618164,
      "activations/layer4_attention_weight_max": 77.33741760253906,
      "activations/layer4_attention_weight_min": -68.1361312866211,
      "activations/layer5_attention_weight_max": 140.2003631591797,
      "activations/layer5_attention_weight_min": -120.18338012695312,
      "activations/layer6_attention_weight_max": 76.25276947021484,
      "activations/layer6_attention_weight_min": -66.81105041503906,
      "activations/layer7_attention_weight_max": 71.75252532958984,
      "activations/layer7_attention_weight_min": -67.42843627929688,
      "activations/layer8_attention_weight_max": 60.05011749267578,
      "activations/layer8_attention_weight_min": -54.82598114013672,
      "activations/layer9_attention_weight_max": 77.56494140625,
      "activations/layer9_attention_weight_min": -64.66436004638672,
      "epoch": 0.53,
      "learning_rate": 0.00014809583333333333,
      "loss": 3.2731,
      "step": 9050
    },
    {
      "activations/layer0_attention_weight_max": 15.076116561889648,
      "activations/layer0_attention_weight_min": -10.057971954345703,
      "activations/layer10_attention_weight_max": 63.078216552734375,
      "activations/layer10_attention_weight_min": -55.03992462158203,
      "activations/layer11_attention_weight_max": 40.524967193603516,
      "activations/layer11_attention_weight_min": -34.834171295166016,
      "activations/layer12_attention_weight_max": 26.646102905273438,
      "activations/layer12_attention_weight_min": -24.539520263671875,
      "activations/layer13_attention_weight_max": 44.50423812866211,
      "activations/layer13_attention_weight_min": -37.91688919067383,
      "activations/layer14_attention_weight_max": 39.503639221191406,
      "activations/layer14_attention_weight_min": -30.10321617126465,
      "activations/layer15_attention_weight_max": 34.13539123535156,
      "activations/layer15_attention_weight_min": -27.673723220825195,
      "activations/layer16_attention_weight_max": 34.5194206237793,
      "activations/layer16_attention_weight_min": -24.679744720458984,
      "activations/layer17_attention_weight_max": 48.48018264770508,
      "activations/layer17_attention_weight_min": -35.11977005004883,
      "activations/layer18_attention_weight_max": 34.32176971435547,
      "activations/layer18_attention_weight_min": -20.177175521850586,
      "activations/layer19_attention_weight_max": 23.843473434448242,
      "activations/layer19_attention_weight_min": -10.251463890075684,
      "activations/layer1_attention_weight_max": 16.32609748840332,
      "activations/layer1_attention_weight_min": -10.331880569458008,
      "activations/layer20_attention_weight_max": 22.407493591308594,
      "activations/layer20_attention_weight_min": -15.844219207763672,
      "activations/layer21_attention_weight_max": 30.044506072998047,
      "activations/layer21_attention_weight_min": -17.834274291992188,
      "activations/layer22_attention_weight_max": 27.226078033447266,
      "activations/layer22_attention_weight_min": -14.358158111572266,
      "activations/layer23_attention_weight_max": 20.579631805419922,
      "activations/layer23_attention_weight_min": -17.27638816833496,
      "activations/layer2_attention_weight_max": 26.642173767089844,
      "activations/layer2_attention_weight_min": -19.987167358398438,
      "activations/layer3_attention_weight_max": 52.678524017333984,
      "activations/layer3_attention_weight_min": -44.60868453979492,
      "activations/layer4_attention_weight_max": 76.56158447265625,
      "activations/layer4_attention_weight_min": -67.4360580444336,
      "activations/layer5_attention_weight_max": 145.42047119140625,
      "activations/layer5_attention_weight_min": -125.87123107910156,
      "activations/layer6_attention_weight_max": 79.77933502197266,
      "activations/layer6_attention_weight_min": -68.66775512695312,
      "activations/layer7_attention_weight_max": 78.00173950195312,
      "activations/layer7_attention_weight_min": -67.55429077148438,
      "activations/layer8_attention_weight_max": 60.81184387207031,
      "activations/layer8_attention_weight_min": -54.84020233154297,
      "activations/layer9_attention_weight_max": 80.58861541748047,
      "activations/layer9_attention_weight_min": -68.60982513427734,
      "epoch": 0.53,
      "learning_rate": 0.00014807689393939394,
      "loss": 3.28,
      "step": 9100
    },
    {
      "activations/layer0_attention_weight_max": 14.639427185058594,
      "activations/layer0_attention_weight_min": -10.367507934570312,
      "activations/layer10_attention_weight_max": 74.5545425415039,
      "activations/layer10_attention_weight_min": -60.59843063354492,
      "activations/layer11_attention_weight_max": 47.32041931152344,
      "activations/layer11_attention_weight_min": -35.74421691894531,
      "activations/layer12_attention_weight_max": 24.78056526184082,
      "activations/layer12_attention_weight_min": -22.66037368774414,
      "activations/layer13_attention_weight_max": 51.123661041259766,
      "activations/layer13_attention_weight_min": -35.89687728881836,
      "activations/layer14_attention_weight_max": 40.9789924621582,
      "activations/layer14_attention_weight_min": -31.259605407714844,
      "activations/layer15_attention_weight_max": 49.94308853149414,
      "activations/layer15_attention_weight_min": -30.92203140258789,
      "activations/layer16_attention_weight_max": 33.38713073730469,
      "activations/layer16_attention_weight_min": -23.730182647705078,
      "activations/layer17_attention_weight_max": 54.19640350341797,
      "activations/layer17_attention_weight_min": -31.099557876586914,
      "activations/layer18_attention_weight_max": 42.2833366394043,
      "activations/layer18_attention_weight_min": -25.925121307373047,
      "activations/layer19_attention_weight_max": 20.796873092651367,
      "activations/layer19_attention_weight_min": -12.379447937011719,
      "activations/layer1_attention_weight_max": 17.10594940185547,
      "activations/layer1_attention_weight_min": -10.681775093078613,
      "activations/layer20_attention_weight_max": 21.627153396606445,
      "activations/layer20_attention_weight_min": -13.264410018920898,
      "activations/layer21_attention_weight_max": 29.808452606201172,
      "activations/layer21_attention_weight_min": -15.533445358276367,
      "activations/layer22_attention_weight_max": 29.268123626708984,
      "activations/layer22_attention_weight_min": -14.251640319824219,
      "activations/layer23_attention_weight_max": 22.37031364440918,
      "activations/layer23_attention_weight_min": -18.697044372558594,
      "activations/layer2_attention_weight_max": 25.782917022705078,
      "activations/layer2_attention_weight_min": -20.45225715637207,
      "activations/layer3_attention_weight_max": 55.55525207519531,
      "activations/layer3_attention_weight_min": -46.50843811035156,
      "activations/layer4_attention_weight_max": 82.48505401611328,
      "activations/layer4_attention_weight_min": -68.96150207519531,
      "activations/layer5_attention_weight_max": 154.67333984375,
      "activations/layer5_attention_weight_min": -127.30441284179688,
      "activations/layer6_attention_weight_max": 85.76578521728516,
      "activations/layer6_attention_weight_min": -68.83683013916016,
      "activations/layer7_attention_weight_max": 83.70488739013672,
      "activations/layer7_attention_weight_min": -71.8829116821289,
      "activations/layer8_attention_weight_max": 67.1856918334961,
      "activations/layer8_attention_weight_min": -56.65663528442383,
      "activations/layer9_attention_weight_max": 87.12554168701172,
      "activations/layer9_attention_weight_min": -74.0775375366211,
      "epoch": 0.53,
      "learning_rate": 0.00014805795454545453,
      "loss": 3.2813,
      "step": 9150
    },
    {
      "activations/layer0_attention_weight_max": 14.546951293945312,
      "activations/layer0_attention_weight_min": -10.279472351074219,
      "activations/layer10_attention_weight_max": 72.23336791992188,
      "activations/layer10_attention_weight_min": -66.795166015625,
      "activations/layer11_attention_weight_max": 46.62206268310547,
      "activations/layer11_attention_weight_min": -36.861175537109375,
      "activations/layer12_attention_weight_max": 26.01032066345215,
      "activations/layer12_attention_weight_min": -22.350446701049805,
      "activations/layer13_attention_weight_max": 47.049354553222656,
      "activations/layer13_attention_weight_min": -37.98035430908203,
      "activations/layer14_attention_weight_max": 36.74125671386719,
      "activations/layer14_attention_weight_min": -29.8110408782959,
      "activations/layer15_attention_weight_max": 37.81467819213867,
      "activations/layer15_attention_weight_min": -28.975736618041992,
      "activations/layer16_attention_weight_max": 30.017488479614258,
      "activations/layer16_attention_weight_min": -25.939119338989258,
      "activations/layer17_attention_weight_max": 44.6307487487793,
      "activations/layer17_attention_weight_min": -39.477413177490234,
      "activations/layer18_attention_weight_max": 31.48504638671875,
      "activations/layer18_attention_weight_min": -23.401323318481445,
      "activations/layer19_attention_weight_max": 22.773149490356445,
      "activations/layer19_attention_weight_min": -11.306715965270996,
      "activations/layer1_attention_weight_max": 16.92414093017578,
      "activations/layer1_attention_weight_min": -10.999373435974121,
      "activations/layer20_attention_weight_max": 20.2064208984375,
      "activations/layer20_attention_weight_min": -12.961775779724121,
      "activations/layer21_attention_weight_max": 29.151931762695312,
      "activations/layer21_attention_weight_min": -15.481432914733887,
      "activations/layer22_attention_weight_max": 28.45272445678711,
      "activations/layer22_attention_weight_min": -15.455886840820312,
      "activations/layer23_attention_weight_max": 21.25053596496582,
      "activations/layer23_attention_weight_min": -17.197803497314453,
      "activations/layer2_attention_weight_max": 25.20039939880371,
      "activations/layer2_attention_weight_min": -20.41530990600586,
      "activations/layer3_attention_weight_max": 52.34144592285156,
      "activations/layer3_attention_weight_min": -44.821693420410156,
      "activations/layer4_attention_weight_max": 79.79959869384766,
      "activations/layer4_attention_weight_min": -68.5247573852539,
      "activations/layer5_attention_weight_max": 141.91038513183594,
      "activations/layer5_attention_weight_min": -121.88186645507812,
      "activations/layer6_attention_weight_max": 78.04588317871094,
      "activations/layer6_attention_weight_min": -67.89494323730469,
      "activations/layer7_attention_weight_max": 81.89386749267578,
      "activations/layer7_attention_weight_min": -71.24772644042969,
      "activations/layer8_attention_weight_max": 69.71082305908203,
      "activations/layer8_attention_weight_min": -58.95694351196289,
      "activations/layer9_attention_weight_max": 70.42364501953125,
      "activations/layer9_attention_weight_min": -62.413299560546875,
      "epoch": 0.53,
      "learning_rate": 0.00014803901515151514,
      "loss": 3.2384,
      "step": 9200
    },
    {
      "activations/layer0_attention_weight_max": 13.834420204162598,
      "activations/layer0_attention_weight_min": -9.447407722473145,
      "activations/layer10_attention_weight_max": 73.3411865234375,
      "activations/layer10_attention_weight_min": -56.7342643737793,
      "activations/layer11_attention_weight_max": 50.79765319824219,
      "activations/layer11_attention_weight_min": -34.957481384277344,
      "activations/layer12_attention_weight_max": 28.35603141784668,
      "activations/layer12_attention_weight_min": -21.61001205444336,
      "activations/layer13_attention_weight_max": 48.51203536987305,
      "activations/layer13_attention_weight_min": -35.08429718017578,
      "activations/layer14_attention_weight_max": 36.102149963378906,
      "activations/layer14_attention_weight_min": -28.478309631347656,
      "activations/layer15_attention_weight_max": 39.45426940917969,
      "activations/layer15_attention_weight_min": -29.2427921295166,
      "activations/layer16_attention_weight_max": 35.30228805541992,
      "activations/layer16_attention_weight_min": -25.453855514526367,
      "activations/layer17_attention_weight_max": 48.336700439453125,
      "activations/layer17_attention_weight_min": -33.048213958740234,
      "activations/layer18_attention_weight_max": 46.335750579833984,
      "activations/layer18_attention_weight_min": -28.206106185913086,
      "activations/layer19_attention_weight_max": 18.702970504760742,
      "activations/layer19_attention_weight_min": -11.189461708068848,
      "activations/layer1_attention_weight_max": 17.612245559692383,
      "activations/layer1_attention_weight_min": -10.771632194519043,
      "activations/layer20_attention_weight_max": 20.56139373779297,
      "activations/layer20_attention_weight_min": -12.537878036499023,
      "activations/layer21_attention_weight_max": 27.780790328979492,
      "activations/layer21_attention_weight_min": -16.097192764282227,
      "activations/layer22_attention_weight_max": 28.20819854736328,
      "activations/layer22_attention_weight_min": -14.498286247253418,
      "activations/layer23_attention_weight_max": 18.9409122467041,
      "activations/layer23_attention_weight_min": -17.77523422241211,
      "activations/layer2_attention_weight_max": 25.936290740966797,
      "activations/layer2_attention_weight_min": -21.153823852539062,
      "activations/layer3_attention_weight_max": 55.11665344238281,
      "activations/layer3_attention_weight_min": -47.67961120605469,
      "activations/layer4_attention_weight_max": 86.0644760131836,
      "activations/layer4_attention_weight_min": -71.96913146972656,
      "activations/layer5_attention_weight_max": 151.90887451171875,
      "activations/layer5_attention_weight_min": -121.2904281616211,
      "activations/layer6_attention_weight_max": 78.67164611816406,
      "activations/layer6_attention_weight_min": -68.05797576904297,
      "activations/layer7_attention_weight_max": 77.09832000732422,
      "activations/layer7_attention_weight_min": -66.41535949707031,
      "activations/layer8_attention_weight_max": 65.5045166015625,
      "activations/layer8_attention_weight_min": -57.06130599975586,
      "activations/layer9_attention_weight_max": 74.73149871826172,
      "activations/layer9_attention_weight_min": -65.12113189697266,
      "epoch": 0.54,
      "learning_rate": 0.00014802007575757576,
      "loss": 3.2592,
      "step": 9250
    },
    {
      "activations/layer0_attention_weight_max": 14.965167045593262,
      "activations/layer0_attention_weight_min": -9.977264404296875,
      "activations/layer10_attention_weight_max": 81.52297973632812,
      "activations/layer10_attention_weight_min": -62.81925582885742,
      "activations/layer11_attention_weight_max": 39.38190841674805,
      "activations/layer11_attention_weight_min": -32.31576919555664,
      "activations/layer12_attention_weight_max": 28.529056549072266,
      "activations/layer12_attention_weight_min": -26.37708854675293,
      "activations/layer13_attention_weight_max": 49.283409118652344,
      "activations/layer13_attention_weight_min": -38.70750045776367,
      "activations/layer14_attention_weight_max": 42.3217887878418,
      "activations/layer14_attention_weight_min": -27.225296020507812,
      "activations/layer15_attention_weight_max": 36.02180099487305,
      "activations/layer15_attention_weight_min": -27.72900390625,
      "activations/layer16_attention_weight_max": 32.69190979003906,
      "activations/layer16_attention_weight_min": -23.92059898376465,
      "activations/layer17_attention_weight_max": 51.80549621582031,
      "activations/layer17_attention_weight_min": -32.32124710083008,
      "activations/layer18_attention_weight_max": 45.418582916259766,
      "activations/layer18_attention_weight_min": -25.15995979309082,
      "activations/layer19_attention_weight_max": 18.86361312866211,
      "activations/layer19_attention_weight_min": -12.76755142211914,
      "activations/layer1_attention_weight_max": 18.813873291015625,
      "activations/layer1_attention_weight_min": -10.279540061950684,
      "activations/layer20_attention_weight_max": 20.08867645263672,
      "activations/layer20_attention_weight_min": -12.694629669189453,
      "activations/layer21_attention_weight_max": 28.848478317260742,
      "activations/layer21_attention_weight_min": -15.338896751403809,
      "activations/layer22_attention_weight_max": 27.868392944335938,
      "activations/layer22_attention_weight_min": -17.06932830810547,
      "activations/layer23_attention_weight_max": 18.427536010742188,
      "activations/layer23_attention_weight_min": -16.30401611328125,
      "activations/layer2_attention_weight_max": 24.500547409057617,
      "activations/layer2_attention_weight_min": -20.485544204711914,
      "activations/layer3_attention_weight_max": 53.806583404541016,
      "activations/layer3_attention_weight_min": -48.13117599487305,
      "activations/layer4_attention_weight_max": 84.27264404296875,
      "activations/layer4_attention_weight_min": -73.44709777832031,
      "activations/layer5_attention_weight_max": 165.96231079101562,
      "activations/layer5_attention_weight_min": -135.27517700195312,
      "activations/layer6_attention_weight_max": 92.19367218017578,
      "activations/layer6_attention_weight_min": -70.71068572998047,
      "activations/layer7_attention_weight_max": 84.78598022460938,
      "activations/layer7_attention_weight_min": -68.64672088623047,
      "activations/layer8_attention_weight_max": 68.84722900390625,
      "activations/layer8_attention_weight_min": -56.97885513305664,
      "activations/layer9_attention_weight_max": 87.83879089355469,
      "activations/layer9_attention_weight_min": -71.8899917602539,
      "epoch": 0.54,
      "learning_rate": 0.00014800113636363635,
      "loss": 3.2704,
      "step": 9300
    },
    {
      "activations/layer0_attention_weight_max": 14.670594215393066,
      "activations/layer0_attention_weight_min": -11.128974914550781,
      "activations/layer10_attention_weight_max": 61.85468673706055,
      "activations/layer10_attention_weight_min": -60.38348388671875,
      "activations/layer11_attention_weight_max": 45.28990936279297,
      "activations/layer11_attention_weight_min": -37.393890380859375,
      "activations/layer12_attention_weight_max": 23.65581512451172,
      "activations/layer12_attention_weight_min": -25.756851196289062,
      "activations/layer13_attention_weight_max": 44.60970687866211,
      "activations/layer13_attention_weight_min": -37.32592010498047,
      "activations/layer14_attention_weight_max": 42.04612731933594,
      "activations/layer14_attention_weight_min": -30.625049591064453,
      "activations/layer15_attention_weight_max": 40.42666244506836,
      "activations/layer15_attention_weight_min": -29.487064361572266,
      "activations/layer16_attention_weight_max": 29.149812698364258,
      "activations/layer16_attention_weight_min": -29.101865768432617,
      "activations/layer17_attention_weight_max": 47.388187408447266,
      "activations/layer17_attention_weight_min": -37.34268569946289,
      "activations/layer18_attention_weight_max": 38.66783905029297,
      "activations/layer18_attention_weight_min": -26.486024856567383,
      "activations/layer19_attention_weight_max": 18.133928298950195,
      "activations/layer19_attention_weight_min": -12.060641288757324,
      "activations/layer1_attention_weight_max": 18.298465728759766,
      "activations/layer1_attention_weight_min": -10.823626518249512,
      "activations/layer20_attention_weight_max": 15.835906982421875,
      "activations/layer20_attention_weight_min": -12.56704044342041,
      "activations/layer21_attention_weight_max": 27.81093978881836,
      "activations/layer21_attention_weight_min": -15.7550048828125,
      "activations/layer22_attention_weight_max": 27.430553436279297,
      "activations/layer22_attention_weight_min": -12.975432395935059,
      "activations/layer23_attention_weight_max": 21.58022117614746,
      "activations/layer23_attention_weight_min": -18.301918029785156,
      "activations/layer2_attention_weight_max": 24.29969596862793,
      "activations/layer2_attention_weight_min": -20.676528930664062,
      "activations/layer3_attention_weight_max": 55.218868255615234,
      "activations/layer3_attention_weight_min": -47.955169677734375,
      "activations/layer4_attention_weight_max": 84.0370864868164,
      "activations/layer4_attention_weight_min": -69.17156219482422,
      "activations/layer5_attention_weight_max": 153.31228637695312,
      "activations/layer5_attention_weight_min": -127.54229736328125,
      "activations/layer6_attention_weight_max": 80.3202133178711,
      "activations/layer6_attention_weight_min": -66.11357879638672,
      "activations/layer7_attention_weight_max": 77.42658996582031,
      "activations/layer7_attention_weight_min": -66.96832275390625,
      "activations/layer8_attention_weight_max": 61.06238555908203,
      "activations/layer8_attention_weight_min": -56.36757278442383,
      "activations/layer9_attention_weight_max": 82.44075775146484,
      "activations/layer9_attention_weight_min": -68.55628204345703,
      "epoch": 0.54,
      "learning_rate": 0.00014798219696969696,
      "loss": 3.2575,
      "step": 9350
    },
    {
      "activations/layer0_attention_weight_max": 14.347216606140137,
      "activations/layer0_attention_weight_min": -10.039717674255371,
      "activations/layer10_attention_weight_max": 69.2889175415039,
      "activations/layer10_attention_weight_min": -59.1959228515625,
      "activations/layer11_attention_weight_max": 44.12272262573242,
      "activations/layer11_attention_weight_min": -37.81686019897461,
      "activations/layer12_attention_weight_max": 32.20515060424805,
      "activations/layer12_attention_weight_min": -22.092924118041992,
      "activations/layer13_attention_weight_max": 53.68153762817383,
      "activations/layer13_attention_weight_min": -40.046844482421875,
      "activations/layer14_attention_weight_max": 38.99213790893555,
      "activations/layer14_attention_weight_min": -27.609891891479492,
      "activations/layer15_attention_weight_max": 40.452632904052734,
      "activations/layer15_attention_weight_min": -29.69842529296875,
      "activations/layer16_attention_weight_max": 36.71194839477539,
      "activations/layer16_attention_weight_min": -28.55023956298828,
      "activations/layer17_attention_weight_max": 51.51664733886719,
      "activations/layer17_attention_weight_min": -35.05440902709961,
      "activations/layer18_attention_weight_max": 45.41621780395508,
      "activations/layer18_attention_weight_min": -25.884353637695312,
      "activations/layer19_attention_weight_max": 20.137855529785156,
      "activations/layer19_attention_weight_min": -11.078829765319824,
      "activations/layer1_attention_weight_max": 18.898338317871094,
      "activations/layer1_attention_weight_min": -10.700284957885742,
      "activations/layer20_attention_weight_max": 18.148860931396484,
      "activations/layer20_attention_weight_min": -12.724102020263672,
      "activations/layer21_attention_weight_max": 26.94476890563965,
      "activations/layer21_attention_weight_min": -15.351454734802246,
      "activations/layer22_attention_weight_max": 26.658281326293945,
      "activations/layer22_attention_weight_min": -14.120466232299805,
      "activations/layer23_attention_weight_max": 20.46023178100586,
      "activations/layer23_attention_weight_min": -18.665679931640625,
      "activations/layer2_attention_weight_max": 25.850704193115234,
      "activations/layer2_attention_weight_min": -20.854957580566406,
      "activations/layer3_attention_weight_max": 51.63677978515625,
      "activations/layer3_attention_weight_min": -48.811607360839844,
      "activations/layer4_attention_weight_max": 84.8710708618164,
      "activations/layer4_attention_weight_min": -72.23870086669922,
      "activations/layer5_attention_weight_max": 152.9615478515625,
      "activations/layer5_attention_weight_min": -126.1514892578125,
      "activations/layer6_attention_weight_max": 79.37509155273438,
      "activations/layer6_attention_weight_min": -68.45207214355469,
      "activations/layer7_attention_weight_max": 77.75521087646484,
      "activations/layer7_attention_weight_min": -68.03721618652344,
      "activations/layer8_attention_weight_max": 64.49215698242188,
      "activations/layer8_attention_weight_min": -56.42226028442383,
      "activations/layer9_attention_weight_max": 72.94451141357422,
      "activations/layer9_attention_weight_min": -68.65534210205078,
      "epoch": 0.55,
      "learning_rate": 0.00014796363636363635,
      "loss": 3.2481,
      "step": 9400
    },
    {
      "activations/layer0_attention_weight_max": 14.28061294555664,
      "activations/layer0_attention_weight_min": -9.92420482635498,
      "activations/layer10_attention_weight_max": 59.46787643432617,
      "activations/layer10_attention_weight_min": -58.566917419433594,
      "activations/layer11_attention_weight_max": 36.80362319946289,
      "activations/layer11_attention_weight_min": -36.514808654785156,
      "activations/layer12_attention_weight_max": 31.320512771606445,
      "activations/layer12_attention_weight_min": -21.12193489074707,
      "activations/layer13_attention_weight_max": 54.15510177612305,
      "activations/layer13_attention_weight_min": -40.98992156982422,
      "activations/layer14_attention_weight_max": 50.811405181884766,
      "activations/layer14_attention_weight_min": -33.1793098449707,
      "activations/layer15_attention_weight_max": 35.95917510986328,
      "activations/layer15_attention_weight_min": -29.61116600036621,
      "activations/layer16_attention_weight_max": 35.77779006958008,
      "activations/layer16_attention_weight_min": -25.671443939208984,
      "activations/layer17_attention_weight_max": 48.70647430419922,
      "activations/layer17_attention_weight_min": -34.0886344909668,
      "activations/layer18_attention_weight_max": 44.449241638183594,
      "activations/layer18_attention_weight_min": -27.42776107788086,
      "activations/layer19_attention_weight_max": 21.915891647338867,
      "activations/layer19_attention_weight_min": -10.199041366577148,
      "activations/layer1_attention_weight_max": 16.848386764526367,
      "activations/layer1_attention_weight_min": -10.844576835632324,
      "activations/layer20_attention_weight_max": 20.065340042114258,
      "activations/layer20_attention_weight_min": -11.562116622924805,
      "activations/layer21_attention_weight_max": 26.117212295532227,
      "activations/layer21_attention_weight_min": -18.183149337768555,
      "activations/layer22_attention_weight_max": 26.1470947265625,
      "activations/layer22_attention_weight_min": -16.604333877563477,
      "activations/layer23_attention_weight_max": 23.226299285888672,
      "activations/layer23_attention_weight_min": -21.072399139404297,
      "activations/layer2_attention_weight_max": 24.839780807495117,
      "activations/layer2_attention_weight_min": -21.90380859375,
      "activations/layer3_attention_weight_max": 52.03974533081055,
      "activations/layer3_attention_weight_min": -48.769691467285156,
      "activations/layer4_attention_weight_max": 79.85453796386719,
      "activations/layer4_attention_weight_min": -70.93627166748047,
      "activations/layer5_attention_weight_max": 140.07369995117188,
      "activations/layer5_attention_weight_min": -127.69184875488281,
      "activations/layer6_attention_weight_max": 77.92476654052734,
      "activations/layer6_attention_weight_min": -66.80255126953125,
      "activations/layer7_attention_weight_max": 77.2759017944336,
      "activations/layer7_attention_weight_min": -73.33724212646484,
      "activations/layer8_attention_weight_max": 62.1763801574707,
      "activations/layer8_attention_weight_min": -58.20499038696289,
      "activations/layer9_attention_weight_max": 66.29244995117188,
      "activations/layer9_attention_weight_min": -65.17383575439453,
      "epoch": 0.55,
      "learning_rate": 0.00014794469696969697,
      "loss": 3.2381,
      "step": 9450
    },
    {
      "activations/layer0_attention_weight_max": 14.327938079833984,
      "activations/layer0_attention_weight_min": -9.84809684753418,
      "activations/layer10_attention_weight_max": 67.03443908691406,
      "activations/layer10_attention_weight_min": -57.29949951171875,
      "activations/layer11_attention_weight_max": 40.59637451171875,
      "activations/layer11_attention_weight_min": -36.953582763671875,
      "activations/layer12_attention_weight_max": 23.453954696655273,
      "activations/layer12_attention_weight_min": -21.416318893432617,
      "activations/layer13_attention_weight_max": 46.04212188720703,
      "activations/layer13_attention_weight_min": -38.29957962036133,
      "activations/layer14_attention_weight_max": 38.1349983215332,
      "activations/layer14_attention_weight_min": -27.721126556396484,
      "activations/layer15_attention_weight_max": 37.36357879638672,
      "activations/layer15_attention_weight_min": -29.436969757080078,
      "activations/layer16_attention_weight_max": 33.30470657348633,
      "activations/layer16_attention_weight_min": -26.495075225830078,
      "activations/layer17_attention_weight_max": 47.158302307128906,
      "activations/layer17_attention_weight_min": -31.928369522094727,
      "activations/layer18_attention_weight_max": 42.1508903503418,
      "activations/layer18_attention_weight_min": -27.139970779418945,
      "activations/layer19_attention_weight_max": 21.87527084350586,
      "activations/layer19_attention_weight_min": -11.625662803649902,
      "activations/layer1_attention_weight_max": 18.462474822998047,
      "activations/layer1_attention_weight_min": -13.98904037475586,
      "activations/layer20_attention_weight_max": 18.75560188293457,
      "activations/layer20_attention_weight_min": -14.333085060119629,
      "activations/layer21_attention_weight_max": 22.28435516357422,
      "activations/layer21_attention_weight_min": -15.692306518554688,
      "activations/layer22_attention_weight_max": 21.97732925415039,
      "activations/layer22_attention_weight_min": -15.303862571716309,
      "activations/layer23_attention_weight_max": 21.821226119995117,
      "activations/layer23_attention_weight_min": -20.370628356933594,
      "activations/layer2_attention_weight_max": 22.456998825073242,
      "activations/layer2_attention_weight_min": -21.311140060424805,
      "activations/layer3_attention_weight_max": 57.66416931152344,
      "activations/layer3_attention_weight_min": -59.982913970947266,
      "activations/layer4_attention_weight_max": 76.47000122070312,
      "activations/layer4_attention_weight_min": -82.4081039428711,
      "activations/layer5_attention_weight_max": 152.59707641601562,
      "activations/layer5_attention_weight_min": -147.8771514892578,
      "activations/layer6_attention_weight_max": 75.36992645263672,
      "activations/layer6_attention_weight_min": -74.55070495605469,
      "activations/layer7_attention_weight_max": 71.2359390258789,
      "activations/layer7_attention_weight_min": -71.93035125732422,
      "activations/layer8_attention_weight_max": 60.83364486694336,
      "activations/layer8_attention_weight_min": -55.88383865356445,
      "activations/layer9_attention_weight_max": 72.24786376953125,
      "activations/layer9_attention_weight_min": -66.59368133544922,
      "epoch": 0.55,
      "learning_rate": 0.00014792575757575756,
      "loss": 3.2476,
      "step": 9500
    },
    {
      "activations/layer0_attention_weight_max": 20.107736587524414,
      "activations/layer0_attention_weight_min": -25.106948852539062,
      "activations/layer10_attention_weight_max": 65.3250732421875,
      "activations/layer10_attention_weight_min": -58.700401306152344,
      "activations/layer11_attention_weight_max": 48.06665802001953,
      "activations/layer11_attention_weight_min": -39.16737365722656,
      "activations/layer12_attention_weight_max": 26.99372100830078,
      "activations/layer12_attention_weight_min": -24.823728561401367,
      "activations/layer13_attention_weight_max": 50.97182846069336,
      "activations/layer13_attention_weight_min": -38.15986633300781,
      "activations/layer14_attention_weight_max": 37.497703552246094,
      "activations/layer14_attention_weight_min": -33.27010726928711,
      "activations/layer15_attention_weight_max": 38.44371032714844,
      "activations/layer15_attention_weight_min": -26.5704288482666,
      "activations/layer16_attention_weight_max": 37.78166961669922,
      "activations/layer16_attention_weight_min": -25.832500457763672,
      "activations/layer17_attention_weight_max": 47.714988708496094,
      "activations/layer17_attention_weight_min": -31.53731346130371,
      "activations/layer18_attention_weight_max": 43.824974060058594,
      "activations/layer18_attention_weight_min": -27.951791763305664,
      "activations/layer19_attention_weight_max": 19.14561653137207,
      "activations/layer19_attention_weight_min": -12.649097442626953,
      "activations/layer1_attention_weight_max": 15.180716514587402,
      "activations/layer1_attention_weight_min": -14.663008689880371,
      "activations/layer20_attention_weight_max": 18.27366065979004,
      "activations/layer20_attention_weight_min": -11.189876556396484,
      "activations/layer21_attention_weight_max": 24.613828659057617,
      "activations/layer21_attention_weight_min": -15.363744735717773,
      "activations/layer22_attention_weight_max": 26.372278213500977,
      "activations/layer22_attention_weight_min": -14.058210372924805,
      "activations/layer23_attention_weight_max": 19.811758041381836,
      "activations/layer23_attention_weight_min": -13.065131187438965,
      "activations/layer2_attention_weight_max": 19.38100814819336,
      "activations/layer2_attention_weight_min": -16.668291091918945,
      "activations/layer3_attention_weight_max": 35.436607360839844,
      "activations/layer3_attention_weight_min": -32.918914794921875,
      "activations/layer4_attention_weight_max": 63.78298568725586,
      "activations/layer4_attention_weight_min": -63.6846923828125,
      "activations/layer5_attention_weight_max": 125.56233978271484,
      "activations/layer5_attention_weight_min": -118.15535736083984,
      "activations/layer6_attention_weight_max": 66.25101470947266,
      "activations/layer6_attention_weight_min": -62.19378662109375,
      "activations/layer7_attention_weight_max": 71.2145004272461,
      "activations/layer7_attention_weight_min": -62.024627685546875,
      "activations/layer8_attention_weight_max": 61.39607238769531,
      "activations/layer8_attention_weight_min": -53.12370681762695,
      "activations/layer9_attention_weight_max": 72.13274383544922,
      "activations/layer9_attention_weight_min": -62.8687629699707,
      "epoch": 0.55,
      "learning_rate": 0.00014790719696969695,
      "loss": 3.4564,
      "step": 9550
    },
    {
      "activations/layer0_attention_weight_max": 19.719594955444336,
      "activations/layer0_attention_weight_min": -21.971757888793945,
      "activations/layer10_attention_weight_max": 69.68731689453125,
      "activations/layer10_attention_weight_min": -55.03746795654297,
      "activations/layer11_attention_weight_max": 52.853755950927734,
      "activations/layer11_attention_weight_min": -38.88690948486328,
      "activations/layer12_attention_weight_max": 27.616477966308594,
      "activations/layer12_attention_weight_min": -22.13003158569336,
      "activations/layer13_attention_weight_max": 67.92875671386719,
      "activations/layer13_attention_weight_min": -37.44529342651367,
      "activations/layer14_attention_weight_max": 51.428707122802734,
      "activations/layer14_attention_weight_min": -28.389131546020508,
      "activations/layer15_attention_weight_max": 50.15977096557617,
      "activations/layer15_attention_weight_min": -28.204195022583008,
      "activations/layer16_attention_weight_max": 38.08883285522461,
      "activations/layer16_attention_weight_min": -27.060808181762695,
      "activations/layer17_attention_weight_max": 61.17402648925781,
      "activations/layer17_attention_weight_min": -38.2358283996582,
      "activations/layer18_attention_weight_max": 55.66237258911133,
      "activations/layer18_attention_weight_min": -25.625349044799805,
      "activations/layer19_attention_weight_max": 24.393211364746094,
      "activations/layer19_attention_weight_min": -11.812997817993164,
      "activations/layer1_attention_weight_max": 16.238380432128906,
      "activations/layer1_attention_weight_min": -14.30099105834961,
      "activations/layer20_attention_weight_max": 20.007671356201172,
      "activations/layer20_attention_weight_min": -12.155941009521484,
      "activations/layer21_attention_weight_max": 32.276920318603516,
      "activations/layer21_attention_weight_min": -13.906715393066406,
      "activations/layer22_attention_weight_max": 29.74181365966797,
      "activations/layer22_attention_weight_min": -12.683070182800293,
      "activations/layer23_attention_weight_max": 22.270084381103516,
      "activations/layer23_attention_weight_min": -18.146984100341797,
      "activations/layer2_attention_weight_max": 21.05139923095703,
      "activations/layer2_attention_weight_min": -19.597335815429688,
      "activations/layer3_attention_weight_max": 37.37718963623047,
      "activations/layer3_attention_weight_min": -35.95756149291992,
      "activations/layer4_attention_weight_max": 68.51834106445312,
      "activations/layer4_attention_weight_min": -70.99624633789062,
      "activations/layer5_attention_weight_max": 132.04891967773438,
      "activations/layer5_attention_weight_min": -120.63337707519531,
      "activations/layer6_attention_weight_max": 72.12776184082031,
      "activations/layer6_attention_weight_min": -65.36273193359375,
      "activations/layer7_attention_weight_max": 75.84120178222656,
      "activations/layer7_attention_weight_min": -65.57215881347656,
      "activations/layer8_attention_weight_max": 70.13187408447266,
      "activations/layer8_attention_weight_min": -55.648494720458984,
      "activations/layer9_attention_weight_max": 78.63507843017578,
      "activations/layer9_attention_weight_min": -62.670894622802734,
      "epoch": 0.56,
      "learning_rate": 0.00014788825757575756,
      "loss": 3.2795,
      "step": 9600
    },
    {
      "activations/layer0_attention_weight_max": 18.175704956054688,
      "activations/layer0_attention_weight_min": -21.55284881591797,
      "activations/layer10_attention_weight_max": 62.92295455932617,
      "activations/layer10_attention_weight_min": -58.34811782836914,
      "activations/layer11_attention_weight_max": 44.907470703125,
      "activations/layer11_attention_weight_min": -35.458099365234375,
      "activations/layer12_attention_weight_max": 24.511844635009766,
      "activations/layer12_attention_weight_min": -21.293949127197266,
      "activations/layer13_attention_weight_max": 52.84339904785156,
      "activations/layer13_attention_weight_min": -39.181373596191406,
      "activations/layer14_attention_weight_max": 44.35470199584961,
      "activations/layer14_attention_weight_min": -32.11711120605469,
      "activations/layer15_attention_weight_max": 42.43894958496094,
      "activations/layer15_attention_weight_min": -32.22148895263672,
      "activations/layer16_attention_weight_max": 38.96872329711914,
      "activations/layer16_attention_weight_min": -26.12273406982422,
      "activations/layer17_attention_weight_max": 56.41798782348633,
      "activations/layer17_attention_weight_min": -34.13153839111328,
      "activations/layer18_attention_weight_max": 35.656341552734375,
      "activations/layer18_attention_weight_min": -25.185007095336914,
      "activations/layer19_attention_weight_max": 21.5023136138916,
      "activations/layer19_attention_weight_min": -11.01347827911377,
      "activations/layer1_attention_weight_max": 15.71451473236084,
      "activations/layer1_attention_weight_min": -13.998261451721191,
      "activations/layer20_attention_weight_max": 19.875455856323242,
      "activations/layer20_attention_weight_min": -12.248808860778809,
      "activations/layer21_attention_weight_max": 33.701927185058594,
      "activations/layer21_attention_weight_min": -15.207021713256836,
      "activations/layer22_attention_weight_max": 30.931047439575195,
      "activations/layer22_attention_weight_min": -12.734163284301758,
      "activations/layer23_attention_weight_max": 21.47774887084961,
      "activations/layer23_attention_weight_min": -16.554733276367188,
      "activations/layer2_attention_weight_max": 21.34604263305664,
      "activations/layer2_attention_weight_min": -19.537090301513672,
      "activations/layer3_attention_weight_max": 39.621280670166016,
      "activations/layer3_attention_weight_min": -37.16380310058594,
      "activations/layer4_attention_weight_max": 75.2641830444336,
      "activations/layer4_attention_weight_min": -68.01913452148438,
      "activations/layer5_attention_weight_max": 135.49526977539062,
      "activations/layer5_attention_weight_min": -120.25865173339844,
      "activations/layer6_attention_weight_max": 70.83747100830078,
      "activations/layer6_attention_weight_min": -63.32140350341797,
      "activations/layer7_attention_weight_max": 73.66015625,
      "activations/layer7_attention_weight_min": -65.32298278808594,
      "activations/layer8_attention_weight_max": 61.53757095336914,
      "activations/layer8_attention_weight_min": -53.1663818359375,
      "activations/layer9_attention_weight_max": 69.38397979736328,
      "activations/layer9_attention_weight_min": -62.45539474487305,
      "epoch": 0.56,
      "learning_rate": 0.00014786931818181818,
      "loss": 3.2548,
      "step": 9650
    },
    {
      "activations/layer0_attention_weight_max": 17.497650146484375,
      "activations/layer0_attention_weight_min": -20.67789649963379,
      "activations/layer10_attention_weight_max": 69.60980224609375,
      "activations/layer10_attention_weight_min": -62.2373046875,
      "activations/layer11_attention_weight_max": 48.29248809814453,
      "activations/layer11_attention_weight_min": -38.54602813720703,
      "activations/layer12_attention_weight_max": 27.651161193847656,
      "activations/layer12_attention_weight_min": -22.201120376586914,
      "activations/layer13_attention_weight_max": 54.92136764526367,
      "activations/layer13_attention_weight_min": -41.7998046875,
      "activations/layer14_attention_weight_max": 47.212581634521484,
      "activations/layer14_attention_weight_min": -30.90237045288086,
      "activations/layer15_attention_weight_max": 43.74538040161133,
      "activations/layer15_attention_weight_min": -30.507354736328125,
      "activations/layer16_attention_weight_max": 38.04819107055664,
      "activations/layer16_attention_weight_min": -25.167598724365234,
      "activations/layer17_attention_weight_max": 50.68629837036133,
      "activations/layer17_attention_weight_min": -35.776981353759766,
      "activations/layer18_attention_weight_max": 42.787689208984375,
      "activations/layer18_attention_weight_min": -26.647632598876953,
      "activations/layer19_attention_weight_max": 22.015398025512695,
      "activations/layer19_attention_weight_min": -13.286128044128418,
      "activations/layer1_attention_weight_max": 16.248746871948242,
      "activations/layer1_attention_weight_min": -12.982589721679688,
      "activations/layer20_attention_weight_max": 18.08296775817871,
      "activations/layer20_attention_weight_min": -12.178936958312988,
      "activations/layer21_attention_weight_max": 28.048559188842773,
      "activations/layer21_attention_weight_min": -15.553220748901367,
      "activations/layer22_attention_weight_max": 29.49055290222168,
      "activations/layer22_attention_weight_min": -15.24002456665039,
      "activations/layer23_attention_weight_max": 19.286296844482422,
      "activations/layer23_attention_weight_min": -17.48356056213379,
      "activations/layer2_attention_weight_max": 22.335561752319336,
      "activations/layer2_attention_weight_min": -19.755353927612305,
      "activations/layer3_attention_weight_max": 41.309967041015625,
      "activations/layer3_attention_weight_min": -37.90332794189453,
      "activations/layer4_attention_weight_max": 74.2776870727539,
      "activations/layer4_attention_weight_min": -68.40148162841797,
      "activations/layer5_attention_weight_max": 136.18011474609375,
      "activations/layer5_attention_weight_min": -116.69917297363281,
      "activations/layer6_attention_weight_max": 72.99305725097656,
      "activations/layer6_attention_weight_min": -63.4888801574707,
      "activations/layer7_attention_weight_max": 71.25910949707031,
      "activations/layer7_attention_weight_min": -64.55581665039062,
      "activations/layer8_attention_weight_max": 61.80607223510742,
      "activations/layer8_attention_weight_min": -56.98746109008789,
      "activations/layer9_attention_weight_max": 70.4721908569336,
      "activations/layer9_attention_weight_min": -64.4784164428711,
      "epoch": 0.56,
      "learning_rate": 0.00014785037878787876,
      "loss": 3.2389,
      "step": 9700
    },
    {
      "activations/layer0_attention_weight_max": 16.823535919189453,
      "activations/layer0_attention_weight_min": -19.2343692779541,
      "activations/layer10_attention_weight_max": 60.5062255859375,
      "activations/layer10_attention_weight_min": -55.75025177001953,
      "activations/layer11_attention_weight_max": 43.05357360839844,
      "activations/layer11_attention_weight_min": -36.671966552734375,
      "activations/layer12_attention_weight_max": 23.8377685546875,
      "activations/layer12_attention_weight_min": -26.448583602905273,
      "activations/layer13_attention_weight_max": 51.84730529785156,
      "activations/layer13_attention_weight_min": -34.36252975463867,
      "activations/layer14_attention_weight_max": 56.168724060058594,
      "activations/layer14_attention_weight_min": -35.130828857421875,
      "activations/layer15_attention_weight_max": 39.04251480102539,
      "activations/layer15_attention_weight_min": -31.827373504638672,
      "activations/layer16_attention_weight_max": 32.87617492675781,
      "activations/layer16_attention_weight_min": -27.16505241394043,
      "activations/layer17_attention_weight_max": 55.207183837890625,
      "activations/layer17_attention_weight_min": -34.45427322387695,
      "activations/layer18_attention_weight_max": 47.67872619628906,
      "activations/layer18_attention_weight_min": -31.263221740722656,
      "activations/layer19_attention_weight_max": 18.218454360961914,
      "activations/layer19_attention_weight_min": -11.355199813842773,
      "activations/layer1_attention_weight_max": 15.687137603759766,
      "activations/layer1_attention_weight_min": -13.015883445739746,
      "activations/layer20_attention_weight_max": 16.414085388183594,
      "activations/layer20_attention_weight_min": -12.041433334350586,
      "activations/layer21_attention_weight_max": 26.110641479492188,
      "activations/layer21_attention_weight_min": -18.32938575744629,
      "activations/layer22_attention_weight_max": 28.823150634765625,
      "activations/layer22_attention_weight_min": -14.958330154418945,
      "activations/layer23_attention_weight_max": 20.84357452392578,
      "activations/layer23_attention_weight_min": -18.265722274780273,
      "activations/layer2_attention_weight_max": 23.816770553588867,
      "activations/layer2_attention_weight_min": -20.103107452392578,
      "activations/layer3_attention_weight_max": 44.452938079833984,
      "activations/layer3_attention_weight_min": -39.118568420410156,
      "activations/layer4_attention_weight_max": 80.40534210205078,
      "activations/layer4_attention_weight_min": -66.19034576416016,
      "activations/layer5_attention_weight_max": 145.0592041015625,
      "activations/layer5_attention_weight_min": -119.27169799804688,
      "activations/layer6_attention_weight_max": 75.86434936523438,
      "activations/layer6_attention_weight_min": -61.730655670166016,
      "activations/layer7_attention_weight_max": 72.82411193847656,
      "activations/layer7_attention_weight_min": -61.384674072265625,
      "activations/layer8_attention_weight_max": 56.55670166015625,
      "activations/layer8_attention_weight_min": -48.28312683105469,
      "activations/layer9_attention_weight_max": 80.99292755126953,
      "activations/layer9_attention_weight_min": -69.15785217285156,
      "epoch": 0.57,
      "learning_rate": 0.00014783143939393938,
      "loss": 3.2494,
      "step": 9750
    },
    {
      "activations/layer0_attention_weight_max": 16.494537353515625,
      "activations/layer0_attention_weight_min": -19.105432510375977,
      "activations/layer10_attention_weight_max": 70.56774139404297,
      "activations/layer10_attention_weight_min": -60.85236358642578,
      "activations/layer11_attention_weight_max": 40.882179260253906,
      "activations/layer11_attention_weight_min": -36.92906188964844,
      "activations/layer12_attention_weight_max": 27.880592346191406,
      "activations/layer12_attention_weight_min": -24.091842651367188,
      "activations/layer13_attention_weight_max": 53.43700408935547,
      "activations/layer13_attention_weight_min": -41.018001556396484,
      "activations/layer14_attention_weight_max": 40.56796646118164,
      "activations/layer14_attention_weight_min": -27.87875747680664,
      "activations/layer15_attention_weight_max": 40.99690628051758,
      "activations/layer15_attention_weight_min": -31.744647979736328,
      "activations/layer16_attention_weight_max": 36.21628189086914,
      "activations/layer16_attention_weight_min": -30.233470916748047,
      "activations/layer17_attention_weight_max": 48.9279899597168,
      "activations/layer17_attention_weight_min": -36.62242889404297,
      "activations/layer18_attention_weight_max": 47.53490447998047,
      "activations/layer18_attention_weight_min": -28.59237289428711,
      "activations/layer19_attention_weight_max": 22.337379455566406,
      "activations/layer19_attention_weight_min": -10.675373077392578,
      "activations/layer1_attention_weight_max": 15.603221893310547,
      "activations/layer1_attention_weight_min": -12.061997413635254,
      "activations/layer20_attention_weight_max": 17.393217086791992,
      "activations/layer20_attention_weight_min": -13.238903045654297,
      "activations/layer21_attention_weight_max": 24.606552124023438,
      "activations/layer21_attention_weight_min": -19.23836898803711,
      "activations/layer22_attention_weight_max": 24.738616943359375,
      "activations/layer22_attention_weight_min": -16.164018630981445,
      "activations/layer23_attention_weight_max": 21.8131160736084,
      "activations/layer23_attention_weight_min": -18.751392364501953,
      "activations/layer2_attention_weight_max": 22.204092025756836,
      "activations/layer2_attention_weight_min": -19.3393497467041,
      "activations/layer3_attention_weight_max": 43.882659912109375,
      "activations/layer3_attention_weight_min": -40.540958404541016,
      "activations/layer4_attention_weight_max": 78.26432037353516,
      "activations/layer4_attention_weight_min": -67.20409393310547,
      "activations/layer5_attention_weight_max": 140.8483428955078,
      "activations/layer5_attention_weight_min": -121.38988494873047,
      "activations/layer6_attention_weight_max": 74.19721984863281,
      "activations/layer6_attention_weight_min": -63.747596740722656,
      "activations/layer7_attention_weight_max": 73.63790893554688,
      "activations/layer7_attention_weight_min": -68.898193359375,
      "activations/layer8_attention_weight_max": 59.67024612426758,
      "activations/layer8_attention_weight_min": -52.309139251708984,
      "activations/layer9_attention_weight_max": 72.1768569946289,
      "activations/layer9_attention_weight_min": -66.5972900390625,
      "epoch": 0.57,
      "learning_rate": 0.0001478125,
      "loss": 3.2377,
      "step": 9800
    },
    {
      "activations/layer0_attention_weight_max": 16.860212326049805,
      "activations/layer0_attention_weight_min": -18.48625946044922,
      "activations/layer10_attention_weight_max": 68.8623046875,
      "activations/layer10_attention_weight_min": -62.09131622314453,
      "activations/layer11_attention_weight_max": 44.59995651245117,
      "activations/layer11_attention_weight_min": -33.4877815246582,
      "activations/layer12_attention_weight_max": 24.95937728881836,
      "activations/layer12_attention_weight_min": -21.59564208984375,
      "activations/layer13_attention_weight_max": 48.82284164428711,
      "activations/layer13_attention_weight_min": -35.404788970947266,
      "activations/layer14_attention_weight_max": 40.00857162475586,
      "activations/layer14_attention_weight_min": -34.80989456176758,
      "activations/layer15_attention_weight_max": 41.18678283691406,
      "activations/layer15_attention_weight_min": -28.536598205566406,
      "activations/layer16_attention_weight_max": 35.18069839477539,
      "activations/layer16_attention_weight_min": -28.97536849975586,
      "activations/layer17_attention_weight_max": 52.05889892578125,
      "activations/layer17_attention_weight_min": -35.69417190551758,
      "activations/layer18_attention_weight_max": 49.33928680419922,
      "activations/layer18_attention_weight_min": -27.810659408569336,
      "activations/layer19_attention_weight_max": 18.750097274780273,
      "activations/layer19_attention_weight_min": -11.73686408996582,
      "activations/layer1_attention_weight_max": 15.759258270263672,
      "activations/layer1_attention_weight_min": -13.318364143371582,
      "activations/layer20_attention_weight_max": 18.29750633239746,
      "activations/layer20_attention_weight_min": -12.287583351135254,
      "activations/layer21_attention_weight_max": 26.017847061157227,
      "activations/layer21_attention_weight_min": -15.258976936340332,
      "activations/layer22_attention_weight_max": 22.15201187133789,
      "activations/layer22_attention_weight_min": -15.406492233276367,
      "activations/layer23_attention_weight_max": 21.02707290649414,
      "activations/layer23_attention_weight_min": -20.375934600830078,
      "activations/layer2_attention_weight_max": 23.71318817138672,
      "activations/layer2_attention_weight_min": -20.632205963134766,
      "activations/layer3_attention_weight_max": 45.553958892822266,
      "activations/layer3_attention_weight_min": -39.43239974975586,
      "activations/layer4_attention_weight_max": 80.1561508178711,
      "activations/layer4_attention_weight_min": -68.73125457763672,
      "activations/layer5_attention_weight_max": 139.2227783203125,
      "activations/layer5_attention_weight_min": -122.28341674804688,
      "activations/layer6_attention_weight_max": 72.80595397949219,
      "activations/layer6_attention_weight_min": -64.40870666503906,
      "activations/layer7_attention_weight_max": 73.14022827148438,
      "activations/layer7_attention_weight_min": -66.13410949707031,
      "activations/layer8_attention_weight_max": 64.93851470947266,
      "activations/layer8_attention_weight_min": -55.84593200683594,
      "activations/layer9_attention_weight_max": 73.95038604736328,
      "activations/layer9_attention_weight_min": -64.44800567626953,
      "epoch": 0.57,
      "learning_rate": 0.00014779356060606058,
      "loss": 3.2446,
      "step": 9850
    },
    {
      "activations/layer0_attention_weight_max": 16.470190048217773,
      "activations/layer0_attention_weight_min": -18.318729400634766,
      "activations/layer10_attention_weight_max": 64.49076080322266,
      "activations/layer10_attention_weight_min": -59.74916458129883,
      "activations/layer11_attention_weight_max": 44.7152214050293,
      "activations/layer11_attention_weight_min": -41.677398681640625,
      "activations/layer12_attention_weight_max": 23.993026733398438,
      "activations/layer12_attention_weight_min": -23.20452880859375,
      "activations/layer13_attention_weight_max": 61.27778244018555,
      "activations/layer13_attention_weight_min": -41.73356246948242,
      "activations/layer14_attention_weight_max": 57.6611442565918,
      "activations/layer14_attention_weight_min": -34.51016616821289,
      "activations/layer15_attention_weight_max": 47.698726654052734,
      "activations/layer15_attention_weight_min": -35.0869026184082,
      "activations/layer16_attention_weight_max": 36.43442916870117,
      "activations/layer16_attention_weight_min": -27.319042205810547,
      "activations/layer17_attention_weight_max": 55.32146072387695,
      "activations/layer17_attention_weight_min": -36.376365661621094,
      "activations/layer18_attention_weight_max": 36.44977951049805,
      "activations/layer18_attention_weight_min": -27.035146713256836,
      "activations/layer19_attention_weight_max": 21.023305892944336,
      "activations/layer19_attention_weight_min": -10.596412658691406,
      "activations/layer1_attention_weight_max": 16.081052780151367,
      "activations/layer1_attention_weight_min": -12.786192893981934,
      "activations/layer20_attention_weight_max": 18.404800415039062,
      "activations/layer20_attention_weight_min": -14.05006217956543,
      "activations/layer21_attention_weight_max": 24.906885147094727,
      "activations/layer21_attention_weight_min": -15.527504920959473,
      "activations/layer22_attention_weight_max": 26.887123107910156,
      "activations/layer22_attention_weight_min": -15.804524421691895,
      "activations/layer23_attention_weight_max": 23.631980895996094,
      "activations/layer23_attention_weight_min": -19.88016128540039,
      "activations/layer2_attention_weight_max": 23.626171112060547,
      "activations/layer2_attention_weight_min": -20.46536636352539,
      "activations/layer3_attention_weight_max": 45.05878829956055,
      "activations/layer3_attention_weight_min": -41.29291915893555,
      "activations/layer4_attention_weight_max": 79.25230407714844,
      "activations/layer4_attention_weight_min": -70.9930648803711,
      "activations/layer5_attention_weight_max": 140.83287048339844,
      "activations/layer5_attention_weight_min": -118.01246643066406,
      "activations/layer6_attention_weight_max": 71.03140258789062,
      "activations/layer6_attention_weight_min": -63.888336181640625,
      "activations/layer7_attention_weight_max": 69.96821594238281,
      "activations/layer7_attention_weight_min": -65.69216918945312,
      "activations/layer8_attention_weight_max": 59.61520004272461,
      "activations/layer8_attention_weight_min": -53.91049575805664,
      "activations/layer9_attention_weight_max": 73.5045394897461,
      "activations/layer9_attention_weight_min": -64.03423309326172,
      "epoch": 0.58,
      "learning_rate": 0.0001477746212121212,
      "loss": 3.233,
      "step": 9900
    },
    {
      "activations/layer0_attention_weight_max": 16.245323181152344,
      "activations/layer0_attention_weight_min": -18.204753875732422,
      "activations/layer10_attention_weight_max": 67.02125549316406,
      "activations/layer10_attention_weight_min": -63.3875846862793,
      "activations/layer11_attention_weight_max": 40.69541931152344,
      "activations/layer11_attention_weight_min": -36.30946731567383,
      "activations/layer12_attention_weight_max": 27.75157356262207,
      "activations/layer12_attention_weight_min": -26.090425491333008,
      "activations/layer13_attention_weight_max": 42.335994720458984,
      "activations/layer13_attention_weight_min": -40.54853439331055,
      "activations/layer14_attention_weight_max": 35.925140380859375,
      "activations/layer14_attention_weight_min": -29.700735092163086,
      "activations/layer15_attention_weight_max": 42.16911315917969,
      "activations/layer15_attention_weight_min": -34.415672302246094,
      "activations/layer16_attention_weight_max": 31.43981170654297,
      "activations/layer16_attention_weight_min": -26.806669235229492,
      "activations/layer17_attention_weight_max": 49.12086868286133,
      "activations/layer17_attention_weight_min": -34.77214050292969,
      "activations/layer18_attention_weight_max": 37.55651092529297,
      "activations/layer18_attention_weight_min": -30.825414657592773,
      "activations/layer19_attention_weight_max": 17.045183181762695,
      "activations/layer19_attention_weight_min": -10.468648910522461,
      "activations/layer1_attention_weight_max": 16.924476623535156,
      "activations/layer1_attention_weight_min": -11.54823112487793,
      "activations/layer20_attention_weight_max": 17.36412239074707,
      "activations/layer20_attention_weight_min": -11.911637306213379,
      "activations/layer21_attention_weight_max": 25.25664520263672,
      "activations/layer21_attention_weight_min": -14.65038013458252,
      "activations/layer22_attention_weight_max": 24.563154220581055,
      "activations/layer22_attention_weight_min": -13.168564796447754,
      "activations/layer23_attention_weight_max": 19.18109893798828,
      "activations/layer23_attention_weight_min": -19.076631546020508,
      "activations/layer2_attention_weight_max": 21.500783920288086,
      "activations/layer2_attention_weight_min": -21.14874267578125,
      "activations/layer3_attention_weight_max": 44.61883544921875,
      "activations/layer3_attention_weight_min": -41.696903228759766,
      "activations/layer4_attention_weight_max": 78.92790985107422,
      "activations/layer4_attention_weight_min": -71.69933319091797,
      "activations/layer5_attention_weight_max": 137.69491577148438,
      "activations/layer5_attention_weight_min": -127.43270111083984,
      "activations/layer6_attention_weight_max": 68.34193420410156,
      "activations/layer6_attention_weight_min": -64.26571655273438,
      "activations/layer7_attention_weight_max": 68.69544982910156,
      "activations/layer7_attention_weight_min": -66.02701568603516,
      "activations/layer8_attention_weight_max": 61.97957992553711,
      "activations/layer8_attention_weight_min": -60.01535415649414,
      "activations/layer9_attention_weight_max": 77.8164291381836,
      "activations/layer9_attention_weight_min": -77.1791000366211,
      "epoch": 0.58,
      "learning_rate": 0.00014775568181818179,
      "loss": 3.2441,
      "step": 9950
    },
    {
      "activations/layer0_attention_weight_max": 16.260385513305664,
      "activations/layer0_attention_weight_min": -17.562301635742188,
      "activations/layer10_attention_weight_max": 71.21577453613281,
      "activations/layer10_attention_weight_min": -59.58986282348633,
      "activations/layer11_attention_weight_max": 48.74641036987305,
      "activations/layer11_attention_weight_min": -38.92372512817383,
      "activations/layer12_attention_weight_max": 24.43053436279297,
      "activations/layer12_attention_weight_min": -22.954999923706055,
      "activations/layer13_attention_weight_max": 54.27164077758789,
      "activations/layer13_attention_weight_min": -41.10804748535156,
      "activations/layer14_attention_weight_max": 40.133155822753906,
      "activations/layer14_attention_weight_min": -34.485992431640625,
      "activations/layer15_attention_weight_max": 38.16501235961914,
      "activations/layer15_attention_weight_min": -33.08576583862305,
      "activations/layer16_attention_weight_max": 34.89329528808594,
      "activations/layer16_attention_weight_min": -29.240095138549805,
      "activations/layer17_attention_weight_max": 54.44551467895508,
      "activations/layer17_attention_weight_min": -37.074371337890625,
      "activations/layer18_attention_weight_max": 46.66122055053711,
      "activations/layer18_attention_weight_min": -28.622636795043945,
      "activations/layer19_attention_weight_max": 21.113157272338867,
      "activations/layer19_attention_weight_min": -11.02094554901123,
      "activations/layer1_attention_weight_max": 17.215259552001953,
      "activations/layer1_attention_weight_min": -12.12183952331543,
      "activations/layer20_attention_weight_max": 16.341703414916992,
      "activations/layer20_attention_weight_min": -13.732040405273438,
      "activations/layer21_attention_weight_max": 29.7530574798584,
      "activations/layer21_attention_weight_min": -16.528833389282227,
      "activations/layer22_attention_weight_max": 27.881277084350586,
      "activations/layer22_attention_weight_min": -15.263056755065918,
      "activations/layer23_attention_weight_max": 21.841032028198242,
      "activations/layer23_attention_weight_min": -20.197420120239258,
      "activations/layer2_attention_weight_max": 23.59518051147461,
      "activations/layer2_attention_weight_min": -21.104656219482422,
      "activations/layer3_attention_weight_max": 43.8032341003418,
      "activations/layer3_attention_weight_min": -42.908512115478516,
      "activations/layer4_attention_weight_max": 77.4039535522461,
      "activations/layer4_attention_weight_min": -71.30742645263672,
      "activations/layer5_attention_weight_max": 136.7539520263672,
      "activations/layer5_attention_weight_min": -122.92912292480469,
      "activations/layer6_attention_weight_max": 71.7429428100586,
      "activations/layer6_attention_weight_min": -64.2486572265625,
      "activations/layer7_attention_weight_max": 75.06395721435547,
      "activations/layer7_attention_weight_min": -69.8701400756836,
      "activations/layer8_attention_weight_max": 62.28322982788086,
      "activations/layer8_attention_weight_min": -54.079307556152344,
      "activations/layer9_attention_weight_max": 70.58724212646484,
      "activations/layer9_attention_weight_min": -63.8784294128418,
      "epoch": 0.58,
      "learning_rate": 0.0001477367424242424,
      "loss": 3.2291,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_loss": 3.17578125,
      "eval_runtime": 8.2082,
      "eval_samples_per_second": 523.135,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_openwebtext_loss": 3.17578125,
      "eval_openwebtext_ppl": 23.94552000018515,
      "eval_openwebtext_runtime": 8.2082,
      "eval_openwebtext_samples_per_second": 523.135,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_wikitext_loss": 3.470703125,
      "eval_wikitext_ppl": 32.159346537604755,
      "eval_wikitext_runtime": 1.6968,
      "eval_wikitext_samples_per_second": 268.737,
      "step": 10000
    },
    {
      "epoch": 0.58,
      "eval_lambada_loss": 3.744140625,
      "eval_lambada_ppl": 42.27266353442275,
      "eval_lambada_runtime": 9.2782,
      "eval_lambada_samples_per_second": 524.78,
      "step": 10000
    },
    {
      "activations/layer0_attention_weight_max": 15.927287101745605,
      "activations/layer0_attention_weight_min": -17.206830978393555,
      "activations/layer10_attention_weight_max": 65.2261734008789,
      "activations/layer10_attention_weight_min": -59.89093780517578,
      "activations/layer11_attention_weight_max": 43.160675048828125,
      "activations/layer11_attention_weight_min": -37.165733337402344,
      "activations/layer12_attention_weight_max": 26.80842399597168,
      "activations/layer12_attention_weight_min": -23.098894119262695,
      "activations/layer13_attention_weight_max": 50.17066192626953,
      "activations/layer13_attention_weight_min": -39.376441955566406,
      "activations/layer14_attention_weight_max": 46.40489196777344,
      "activations/layer14_attention_weight_min": -33.5968132019043,
      "activations/layer15_attention_weight_max": 42.11018371582031,
      "activations/layer15_attention_weight_min": -29.694507598876953,
      "activations/layer16_attention_weight_max": 40.47043228149414,
      "activations/layer16_attention_weight_min": -27.009531021118164,
      "activations/layer17_attention_weight_max": 60.21243667602539,
      "activations/layer17_attention_weight_min": -35.039554595947266,
      "activations/layer18_attention_weight_max": 42.037864685058594,
      "activations/layer18_attention_weight_min": -28.2127742767334,
      "activations/layer19_attention_weight_max": 20.154491424560547,
      "activations/layer19_attention_weight_min": -10.964775085449219,
      "activations/layer1_attention_weight_max": 17.44295883178711,
      "activations/layer1_attention_weight_min": -12.543534278869629,
      "activations/layer20_attention_weight_max": 19.691387176513672,
      "activations/layer20_attention_weight_min": -11.559103012084961,
      "activations/layer21_attention_weight_max": 30.74074935913086,
      "activations/layer21_attention_weight_min": -16.24123191833496,
      "activations/layer22_attention_weight_max": 29.54863166809082,
      "activations/layer22_attention_weight_min": -14.860246658325195,
      "activations/layer23_attention_weight_max": 20.620250701904297,
      "activations/layer23_attention_weight_min": -21.89523696899414,
      "activations/layer2_attention_weight_max": 23.652252197265625,
      "activations/layer2_attention_weight_min": -20.412410736083984,
      "activations/layer3_attention_weight_max": 48.117164611816406,
      "activations/layer3_attention_weight_min": -42.374488830566406,
      "activations/layer4_attention_weight_max": 79.4070053100586,
      "activations/layer4_attention_weight_min": -68.73599243164062,
      "activations/layer5_attention_weight_max": 136.81771850585938,
      "activations/layer5_attention_weight_min": -123.86604309082031,
      "activations/layer6_attention_weight_max": 72.0657958984375,
      "activations/layer6_attention_weight_min": -63.3791389465332,
      "activations/layer7_attention_weight_max": 75.42308044433594,
      "activations/layer7_attention_weight_min": -65.69497680664062,
      "activations/layer8_attention_weight_max": 62.10662078857422,
      "activations/layer8_attention_weight_min": -55.362796783447266,
      "activations/layer9_attention_weight_max": 67.83405303955078,
      "activations/layer9_attention_weight_min": -62.70306396484375,
      "epoch": 0.58,
      "learning_rate": 0.00014771780303030302,
      "loss": 3.2392,
      "step": 10050
    },
    {
      "activations/layer0_attention_weight_max": 15.626829147338867,
      "activations/layer0_attention_weight_min": -17.243370056152344,
      "activations/layer10_attention_weight_max": 67.04469299316406,
      "activations/layer10_attention_weight_min": -64.17848205566406,
      "activations/layer11_attention_weight_max": 37.803802490234375,
      "activations/layer11_attention_weight_min": -34.85997772216797,
      "activations/layer12_attention_weight_max": 24.284021377563477,
      "activations/layer12_attention_weight_min": -25.05805778503418,
      "activations/layer13_attention_weight_max": 47.992557525634766,
      "activations/layer13_attention_weight_min": -40.09980010986328,
      "activations/layer14_attention_weight_max": 43.743743896484375,
      "activations/layer14_attention_weight_min": -36.04425048828125,
      "activations/layer15_attention_weight_max": 39.41285705566406,
      "activations/layer15_attention_weight_min": -32.724639892578125,
      "activations/layer16_attention_weight_max": 41.10323715209961,
      "activations/layer16_attention_weight_min": -30.282548904418945,
      "activations/layer17_attention_weight_max": 57.50353240966797,
      "activations/layer17_attention_weight_min": -34.13032150268555,
      "activations/layer18_attention_weight_max": 40.6818733215332,
      "activations/layer18_attention_weight_min": -24.843538284301758,
      "activations/layer19_attention_weight_max": 21.3587589263916,
      "activations/layer19_attention_weight_min": -11.220751762390137,
      "activations/layer1_attention_weight_max": 17.29003143310547,
      "activations/layer1_attention_weight_min": -12.211814880371094,
      "activations/layer20_attention_weight_max": 19.961368560791016,
      "activations/layer20_attention_weight_min": -12.390536308288574,
      "activations/layer21_attention_weight_max": 29.890756607055664,
      "activations/layer21_attention_weight_min": -14.96142864227295,
      "activations/layer22_attention_weight_max": 28.193208694458008,
      "activations/layer22_attention_weight_min": -14.256340980529785,
      "activations/layer23_attention_weight_max": 21.8431453704834,
      "activations/layer23_attention_weight_min": -21.781455993652344,
      "activations/layer2_attention_weight_max": 25.585285186767578,
      "activations/layer2_attention_weight_min": -21.417556762695312,
      "activations/layer3_attention_weight_max": 48.123924255371094,
      "activations/layer3_attention_weight_min": -44.13162612915039,
      "activations/layer4_attention_weight_max": 81.17991638183594,
      "activations/layer4_attention_weight_min": -71.0598373413086,
      "activations/layer5_attention_weight_max": 140.37628173828125,
      "activations/layer5_attention_weight_min": -126.33000946044922,
      "activations/layer6_attention_weight_max": 72.84053039550781,
      "activations/layer6_attention_weight_min": -64.37754821777344,
      "activations/layer7_attention_weight_max": 73.38226318359375,
      "activations/layer7_attention_weight_min": -68.27880096435547,
      "activations/layer8_attention_weight_max": 58.55302810668945,
      "activations/layer8_attention_weight_min": -56.63029479980469,
      "activations/layer9_attention_weight_max": 65.787353515625,
      "activations/layer9_attention_weight_min": -64.46905517578125,
      "epoch": 0.59,
      "learning_rate": 0.00014769886363636363,
      "loss": 3.2366,
      "step": 10100
    },
    {
      "activations/layer0_attention_weight_max": 15.227482795715332,
      "activations/layer0_attention_weight_min": -16.621198654174805,
      "activations/layer10_attention_weight_max": 74.53890991210938,
      "activations/layer10_attention_weight_min": -62.75202560424805,
      "activations/layer11_attention_weight_max": 46.401092529296875,
      "activations/layer11_attention_weight_min": -37.025726318359375,
      "activations/layer12_attention_weight_max": 27.744834899902344,
      "activations/layer12_attention_weight_min": -21.820383071899414,
      "activations/layer13_attention_weight_max": 57.859344482421875,
      "activations/layer13_attention_weight_min": -40.75275421142578,
      "activations/layer14_attention_weight_max": 42.16563415527344,
      "activations/layer14_attention_weight_min": -30.98936653137207,
      "activations/layer15_attention_weight_max": 45.938045501708984,
      "activations/layer15_attention_weight_min": -34.13801574707031,
      "activations/layer16_attention_weight_max": 36.6122932434082,
      "activations/layer16_attention_weight_min": -28.47785186767578,
      "activations/layer17_attention_weight_max": 52.14326858520508,
      "activations/layer17_attention_weight_min": -34.395660400390625,
      "activations/layer18_attention_weight_max": 48.172664642333984,
      "activations/layer18_attention_weight_min": -34.67790985107422,
      "activations/layer19_attention_weight_max": 18.99103355407715,
      "activations/layer19_attention_weight_min": -11.770758628845215,
      "activations/layer1_attention_weight_max": 16.0675106048584,
      "activations/layer1_attention_weight_min": -12.578564643859863,
      "activations/layer20_attention_weight_max": 17.903959274291992,
      "activations/layer20_attention_weight_min": -11.237503051757812,
      "activations/layer21_attention_weight_max": 28.70462417602539,
      "activations/layer21_attention_weight_min": -17.552230834960938,
      "activations/layer22_attention_weight_max": 26.537189483642578,
      "activations/layer22_attention_weight_min": -14.891337394714355,
      "activations/layer23_attention_weight_max": 20.09284210205078,
      "activations/layer23_attention_weight_min": -19.59372901916504,
      "activations/layer2_attention_weight_max": 24.98544692993164,
      "activations/layer2_attention_weight_min": -19.933650970458984,
      "activations/layer3_attention_weight_max": 48.01836013793945,
      "activations/layer3_attention_weight_min": -42.39908218383789,
      "activations/layer4_attention_weight_max": 81.29875946044922,
      "activations/layer4_attention_weight_min": -72.29103088378906,
      "activations/layer5_attention_weight_max": 139.44309997558594,
      "activations/layer5_attention_weight_min": -126.81826782226562,
      "activations/layer6_attention_weight_max": 74.33586120605469,
      "activations/layer6_attention_weight_min": -67.91801452636719,
      "activations/layer7_attention_weight_max": 74.74982452392578,
      "activations/layer7_attention_weight_min": -67.8941650390625,
      "activations/layer8_attention_weight_max": 66.74119567871094,
      "activations/layer8_attention_weight_min": -60.51597213745117,
      "activations/layer9_attention_weight_max": 68.9564437866211,
      "activations/layer9_attention_weight_min": -63.35160446166992,
      "epoch": 0.59,
      "learning_rate": 0.00014767992424242422,
      "loss": 3.2407,
      "step": 10150
    },
    {
      "activations/layer0_attention_weight_max": 15.548698425292969,
      "activations/layer0_attention_weight_min": -16.508956909179688,
      "activations/layer10_attention_weight_max": 75.924560546875,
      "activations/layer10_attention_weight_min": -60.765525817871094,
      "activations/layer11_attention_weight_max": 49.0175895690918,
      "activations/layer11_attention_weight_min": -40.719093322753906,
      "activations/layer12_attention_weight_max": 29.04139518737793,
      "activations/layer12_attention_weight_min": -21.102798461914062,
      "activations/layer13_attention_weight_max": 65.12310028076172,
      "activations/layer13_attention_weight_min": -42.24662399291992,
      "activations/layer14_attention_weight_max": 51.75020980834961,
      "activations/layer14_attention_weight_min": -34.042903900146484,
      "activations/layer15_attention_weight_max": 47.82773208618164,
      "activations/layer15_attention_weight_min": -29.93647575378418,
      "activations/layer16_attention_weight_max": 48.71176528930664,
      "activations/layer16_attention_weight_min": -29.588178634643555,
      "activations/layer17_attention_weight_max": 64.7567367553711,
      "activations/layer17_attention_weight_min": -38.742305755615234,
      "activations/layer18_attention_weight_max": 48.66425323486328,
      "activations/layer18_attention_weight_min": -28.16209602355957,
      "activations/layer19_attention_weight_max": 19.192228317260742,
      "activations/layer19_attention_weight_min": -10.944005012512207,
      "activations/layer1_attention_weight_max": 17.70241928100586,
      "activations/layer1_attention_weight_min": -11.620105743408203,
      "activations/layer20_attention_weight_max": 19.188283920288086,
      "activations/layer20_attention_weight_min": -11.660873413085938,
      "activations/layer21_attention_weight_max": 30.209800720214844,
      "activations/layer21_attention_weight_min": -15.67115592956543,
      "activations/layer22_attention_weight_max": 28.596101760864258,
      "activations/layer22_attention_weight_min": -14.56868839263916,
      "activations/layer23_attention_weight_max": 22.33661460876465,
      "activations/layer23_attention_weight_min": -20.95662498474121,
      "activations/layer2_attention_weight_max": 24.18488311767578,
      "activations/layer2_attention_weight_min": -20.251571655273438,
      "activations/layer3_attention_weight_max": 48.079898834228516,
      "activations/layer3_attention_weight_min": -44.06654739379883,
      "activations/layer4_attention_weight_max": 83.70509338378906,
      "activations/layer4_attention_weight_min": -73.08101654052734,
      "activations/layer5_attention_weight_max": 142.21798706054688,
      "activations/layer5_attention_weight_min": -121.73973083496094,
      "activations/layer6_attention_weight_max": 76.95339965820312,
      "activations/layer6_attention_weight_min": -64.82196807861328,
      "activations/layer7_attention_weight_max": 80.8044662475586,
      "activations/layer7_attention_weight_min": -66.08113861083984,
      "activations/layer8_attention_weight_max": 69.42340850830078,
      "activations/layer8_attention_weight_min": -55.97931671142578,
      "activations/layer9_attention_weight_max": 76.7304916381836,
      "activations/layer9_attention_weight_min": -66.30460357666016,
      "epoch": 0.59,
      "learning_rate": 0.00014766098484848483,
      "loss": 3.2312,
      "step": 10200
    },
    {
      "activations/layer0_attention_weight_max": 15.315384864807129,
      "activations/layer0_attention_weight_min": -17.170869827270508,
      "activations/layer10_attention_weight_max": 62.91443634033203,
      "activations/layer10_attention_weight_min": -60.037899017333984,
      "activations/layer11_attention_weight_max": 42.88429260253906,
      "activations/layer11_attention_weight_min": -35.23054504394531,
      "activations/layer12_attention_weight_max": 29.99720573425293,
      "activations/layer12_attention_weight_min": -24.132251739501953,
      "activations/layer13_attention_weight_max": 50.65572738647461,
      "activations/layer13_attention_weight_min": -35.978660583496094,
      "activations/layer14_attention_weight_max": 39.12474822998047,
      "activations/layer14_attention_weight_min": -27.62775993347168,
      "activations/layer15_attention_weight_max": 50.52621078491211,
      "activations/layer15_attention_weight_min": -33.11593246459961,
      "activations/layer16_attention_weight_max": 44.67988204956055,
      "activations/layer16_attention_weight_min": -30.50893211364746,
      "activations/layer17_attention_weight_max": 54.4045524597168,
      "activations/layer17_attention_weight_min": -35.477901458740234,
      "activations/layer18_attention_weight_max": 60.554874420166016,
      "activations/layer18_attention_weight_min": -32.1170768737793,
      "activations/layer19_attention_weight_max": 19.026344299316406,
      "activations/layer19_attention_weight_min": -11.332003593444824,
      "activations/layer1_attention_weight_max": 16.943815231323242,
      "activations/layer1_attention_weight_min": -12.293911933898926,
      "activations/layer20_attention_weight_max": 21.558916091918945,
      "activations/layer20_attention_weight_min": -11.356467247009277,
      "activations/layer21_attention_weight_max": 33.00786209106445,
      "activations/layer21_attention_weight_min": -14.818438529968262,
      "activations/layer22_attention_weight_max": 29.128694534301758,
      "activations/layer22_attention_weight_min": -14.110382080078125,
      "activations/layer23_attention_weight_max": 22.274084091186523,
      "activations/layer23_attention_weight_min": -21.88312530517578,
      "activations/layer2_attention_weight_max": 24.19385528564453,
      "activations/layer2_attention_weight_min": -21.230010986328125,
      "activations/layer3_attention_weight_max": 46.904998779296875,
      "activations/layer3_attention_weight_min": -45.49724197387695,
      "activations/layer4_attention_weight_max": 78.83687591552734,
      "activations/layer4_attention_weight_min": -70.0792236328125,
      "activations/layer5_attention_weight_max": 136.35955810546875,
      "activations/layer5_attention_weight_min": -121.15367889404297,
      "activations/layer6_attention_weight_max": 71.25871276855469,
      "activations/layer6_attention_weight_min": -65.2091064453125,
      "activations/layer7_attention_weight_max": 70.88158416748047,
      "activations/layer7_attention_weight_min": -66.83545684814453,
      "activations/layer8_attention_weight_max": 60.46780014038086,
      "activations/layer8_attention_weight_min": -52.903751373291016,
      "activations/layer9_attention_weight_max": 66.42721557617188,
      "activations/layer9_attention_weight_min": -67.81035614013672,
      "epoch": 0.6,
      "learning_rate": 0.00014764204545454545,
      "loss": 3.2278,
      "step": 10250
    },
    {
      "activations/layer0_attention_weight_max": 15.202072143554688,
      "activations/layer0_attention_weight_min": -16.138057708740234,
      "activations/layer10_attention_weight_max": 68.26090240478516,
      "activations/layer10_attention_weight_min": -61.370849609375,
      "activations/layer11_attention_weight_max": 48.506996154785156,
      "activations/layer11_attention_weight_min": -39.385047912597656,
      "activations/layer12_attention_weight_max": 25.55662727355957,
      "activations/layer12_attention_weight_min": -23.4035587310791,
      "activations/layer13_attention_weight_max": 51.14557647705078,
      "activations/layer13_attention_weight_min": -39.453208923339844,
      "activations/layer14_attention_weight_max": 46.66142654418945,
      "activations/layer14_attention_weight_min": -29.915294647216797,
      "activations/layer15_attention_weight_max": 42.544776916503906,
      "activations/layer15_attention_weight_min": -31.62331771850586,
      "activations/layer16_attention_weight_max": 37.24531936645508,
      "activations/layer16_attention_weight_min": -27.433395385742188,
      "activations/layer17_attention_weight_max": 55.705745697021484,
      "activations/layer17_attention_weight_min": -33.673465728759766,
      "activations/layer18_attention_weight_max": 49.939300537109375,
      "activations/layer18_attention_weight_min": -34.08541488647461,
      "activations/layer19_attention_weight_max": 20.069795608520508,
      "activations/layer19_attention_weight_min": -11.40282917022705,
      "activations/layer1_attention_weight_max": 17.46143341064453,
      "activations/layer1_attention_weight_min": -11.566247940063477,
      "activations/layer20_attention_weight_max": 20.88886260986328,
      "activations/layer20_attention_weight_min": -12.354269981384277,
      "activations/layer21_attention_weight_max": 27.690643310546875,
      "activations/layer21_attention_weight_min": -15.273735046386719,
      "activations/layer22_attention_weight_max": 28.29891586303711,
      "activations/layer22_attention_weight_min": -14.25863265991211,
      "activations/layer23_attention_weight_max": 20.43708038330078,
      "activations/layer23_attention_weight_min": -20.19411277770996,
      "activations/layer2_attention_weight_max": 26.215076446533203,
      "activations/layer2_attention_weight_min": -22.077489852905273,
      "activations/layer3_attention_weight_max": 50.6497802734375,
      "activations/layer3_attention_weight_min": -45.23033905029297,
      "activations/layer4_attention_weight_max": 80.94976043701172,
      "activations/layer4_attention_weight_min": -73.55630493164062,
      "activations/layer5_attention_weight_max": 138.60003662109375,
      "activations/layer5_attention_weight_min": -122.98661804199219,
      "activations/layer6_attention_weight_max": 75.07814025878906,
      "activations/layer6_attention_weight_min": -62.423484802246094,
      "activations/layer7_attention_weight_max": 72.69197845458984,
      "activations/layer7_attention_weight_min": -63.6292724609375,
      "activations/layer8_attention_weight_max": 71.41626739501953,
      "activations/layer8_attention_weight_min": -59.48479461669922,
      "activations/layer9_attention_weight_max": 72.58985137939453,
      "activations/layer9_attention_weight_min": -64.87529754638672,
      "epoch": 0.6,
      "learning_rate": 0.00014762310606060604,
      "loss": 3.2431,
      "step": 10300
    },
    {
      "activations/layer0_attention_weight_max": 15.38335132598877,
      "activations/layer0_attention_weight_min": -15.723649978637695,
      "activations/layer10_attention_weight_max": 65.88079071044922,
      "activations/layer10_attention_weight_min": -58.20511245727539,
      "activations/layer11_attention_weight_max": 48.40142822265625,
      "activations/layer11_attention_weight_min": -41.09120178222656,
      "activations/layer12_attention_weight_max": 25.77452278137207,
      "activations/layer12_attention_weight_min": -23.409420013427734,
      "activations/layer13_attention_weight_max": 56.96384811401367,
      "activations/layer13_attention_weight_min": -40.58980178833008,
      "activations/layer14_attention_weight_max": 46.8619270324707,
      "activations/layer14_attention_weight_min": -29.822784423828125,
      "activations/layer15_attention_weight_max": 46.51698684692383,
      "activations/layer15_attention_weight_min": -30.629560470581055,
      "activations/layer16_attention_weight_max": 39.74932098388672,
      "activations/layer16_attention_weight_min": -32.24955368041992,
      "activations/layer17_attention_weight_max": 48.51564407348633,
      "activations/layer17_attention_weight_min": -37.56077194213867,
      "activations/layer18_attention_weight_max": 42.32392501831055,
      "activations/layer18_attention_weight_min": -27.99249267578125,
      "activations/layer19_attention_weight_max": 19.340696334838867,
      "activations/layer19_attention_weight_min": -12.607994079589844,
      "activations/layer1_attention_weight_max": 19.491682052612305,
      "activations/layer1_attention_weight_min": -12.143935203552246,
      "activations/layer20_attention_weight_max": 19.285415649414062,
      "activations/layer20_attention_weight_min": -13.468694686889648,
      "activations/layer21_attention_weight_max": 23.529647827148438,
      "activations/layer21_attention_weight_min": -20.344491958618164,
      "activations/layer22_attention_weight_max": 23.133195877075195,
      "activations/layer22_attention_weight_min": -17.444128036499023,
      "activations/layer23_attention_weight_max": 21.012147903442383,
      "activations/layer23_attention_weight_min": -20.97633934020996,
      "activations/layer2_attention_weight_max": 28.811429977416992,
      "activations/layer2_attention_weight_min": -22.14961814880371,
      "activations/layer3_attention_weight_max": 49.00065231323242,
      "activations/layer3_attention_weight_min": -45.4569091796875,
      "activations/layer4_attention_weight_max": 82.69620513916016,
      "activations/layer4_attention_weight_min": -72.21348571777344,
      "activations/layer5_attention_weight_max": 139.59213256835938,
      "activations/layer5_attention_weight_min": -123.38224792480469,
      "activations/layer6_attention_weight_max": 73.82936096191406,
      "activations/layer6_attention_weight_min": -62.58595275878906,
      "activations/layer7_attention_weight_max": 74.26056671142578,
      "activations/layer7_attention_weight_min": -62.342708587646484,
      "activations/layer8_attention_weight_max": 67.84489440917969,
      "activations/layer8_attention_weight_min": -56.24183654785156,
      "activations/layer9_attention_weight_max": 74.15111541748047,
      "activations/layer9_attention_weight_min": -62.79777145385742,
      "epoch": 0.6,
      "learning_rate": 0.00014760416666666665,
      "loss": 3.2414,
      "step": 10350
    },
    {
      "activations/layer0_attention_weight_max": 14.823714256286621,
      "activations/layer0_attention_weight_min": -15.715258598327637,
      "activations/layer10_attention_weight_max": 73.19122314453125,
      "activations/layer10_attention_weight_min": -58.57155227661133,
      "activations/layer11_attention_weight_max": 42.809303283691406,
      "activations/layer11_attention_weight_min": -35.57467269897461,
      "activations/layer12_attention_weight_max": 25.80188751220703,
      "activations/layer12_attention_weight_min": -23.50028419494629,
      "activations/layer13_attention_weight_max": 53.42043685913086,
      "activations/layer13_attention_weight_min": -40.11836242675781,
      "activations/layer14_attention_weight_max": 42.73432540893555,
      "activations/layer14_attention_weight_min": -29.393997192382812,
      "activations/layer15_attention_weight_max": 41.17012405395508,
      "activations/layer15_attention_weight_min": -28.994415283203125,
      "activations/layer16_attention_weight_max": 39.97923278808594,
      "activations/layer16_attention_weight_min": -32.034427642822266,
      "activations/layer17_attention_weight_max": 52.540382385253906,
      "activations/layer17_attention_weight_min": -34.485469818115234,
      "activations/layer18_attention_weight_max": 32.341339111328125,
      "activations/layer18_attention_weight_min": -21.28632164001465,
      "activations/layer19_attention_weight_max": 21.0529727935791,
      "activations/layer19_attention_weight_min": -9.72957706451416,
      "activations/layer1_attention_weight_max": 16.974990844726562,
      "activations/layer1_attention_weight_min": -12.088354110717773,
      "activations/layer20_attention_weight_max": 19.823545455932617,
      "activations/layer20_attention_weight_min": -11.008697509765625,
      "activations/layer21_attention_weight_max": 29.93791389465332,
      "activations/layer21_attention_weight_min": -17.417184829711914,
      "activations/layer22_attention_weight_max": 27.67543601989746,
      "activations/layer22_attention_weight_min": -15.0435209274292,
      "activations/layer23_attention_weight_max": 21.81183624267578,
      "activations/layer23_attention_weight_min": -21.49411392211914,
      "activations/layer2_attention_weight_max": 25.794689178466797,
      "activations/layer2_attention_weight_min": -20.954126358032227,
      "activations/layer3_attention_weight_max": 50.93772888183594,
      "activations/layer3_attention_weight_min": -43.968299865722656,
      "activations/layer4_attention_weight_max": 81.97325897216797,
      "activations/layer4_attention_weight_min": -73.0434341430664,
      "activations/layer5_attention_weight_max": 143.47515869140625,
      "activations/layer5_attention_weight_min": -120.1861572265625,
      "activations/layer6_attention_weight_max": 73.81773376464844,
      "activations/layer6_attention_weight_min": -66.53236389160156,
      "activations/layer7_attention_weight_max": 76.42759704589844,
      "activations/layer7_attention_weight_min": -64.23384857177734,
      "activations/layer8_attention_weight_max": 64.65653991699219,
      "activations/layer8_attention_weight_min": -54.13083267211914,
      "activations/layer9_attention_weight_max": 76.73568725585938,
      "activations/layer9_attention_weight_min": -65.31653594970703,
      "epoch": 0.6,
      "learning_rate": 0.00014758522727272727,
      "loss": 3.225,
      "step": 10400
    },
    {
      "activations/layer0_attention_weight_max": 14.586791038513184,
      "activations/layer0_attention_weight_min": -15.491281509399414,
      "activations/layer10_attention_weight_max": 72.02692413330078,
      "activations/layer10_attention_weight_min": -60.85298538208008,
      "activations/layer11_attention_weight_max": 44.76707077026367,
      "activations/layer11_attention_weight_min": -39.9804801940918,
      "activations/layer12_attention_weight_max": 23.814552307128906,
      "activations/layer12_attention_weight_min": -23.536304473876953,
      "activations/layer13_attention_weight_max": 59.41251754760742,
      "activations/layer13_attention_weight_min": -41.225608825683594,
      "activations/layer14_attention_weight_max": 47.91620635986328,
      "activations/layer14_attention_weight_min": -32.87468338012695,
      "activations/layer15_attention_weight_max": 45.84244918823242,
      "activations/layer15_attention_weight_min": -32.80213165283203,
      "activations/layer16_attention_weight_max": 41.43939971923828,
      "activations/layer16_attention_weight_min": -29.878883361816406,
      "activations/layer17_attention_weight_max": 57.22138977050781,
      "activations/layer17_attention_weight_min": -35.388458251953125,
      "activations/layer18_attention_weight_max": 52.13146209716797,
      "activations/layer18_attention_weight_min": -29.587379455566406,
      "activations/layer19_attention_weight_max": 22.563251495361328,
      "activations/layer19_attention_weight_min": -13.637225151062012,
      "activations/layer1_attention_weight_max": 16.748798370361328,
      "activations/layer1_attention_weight_min": -11.482135772705078,
      "activations/layer20_attention_weight_max": 23.92205047607422,
      "activations/layer20_attention_weight_min": -11.522644996643066,
      "activations/layer21_attention_weight_max": 28.352340698242188,
      "activations/layer21_attention_weight_min": -12.363262176513672,
      "activations/layer22_attention_weight_max": 28.069814682006836,
      "activations/layer22_attention_weight_min": -13.464296340942383,
      "activations/layer23_attention_weight_max": 20.579986572265625,
      "activations/layer23_attention_weight_min": -20.75210952758789,
      "activations/layer2_attention_weight_max": 26.247175216674805,
      "activations/layer2_attention_weight_min": -21.50082778930664,
      "activations/layer3_attention_weight_max": 52.550228118896484,
      "activations/layer3_attention_weight_min": -46.25554656982422,
      "activations/layer4_attention_weight_max": 83.53482818603516,
      "activations/layer4_attention_weight_min": -73.69937133789062,
      "activations/layer5_attention_weight_max": 153.1741943359375,
      "activations/layer5_attention_weight_min": -129.331298828125,
      "activations/layer6_attention_weight_max": 83.94441223144531,
      "activations/layer6_attention_weight_min": -65.70750427246094,
      "activations/layer7_attention_weight_max": 89.53306579589844,
      "activations/layer7_attention_weight_min": -70.77583312988281,
      "activations/layer8_attention_weight_max": 72.15458679199219,
      "activations/layer8_attention_weight_min": -62.32100296020508,
      "activations/layer9_attention_weight_max": 74.81013488769531,
      "activations/layer9_attention_weight_min": -71.09111022949219,
      "epoch": 0.61,
      "learning_rate": 0.00014756628787878788,
      "loss": 3.2429,
      "step": 10450
    },
    {
      "activations/layer0_attention_weight_max": 14.370375633239746,
      "activations/layer0_attention_weight_min": -15.501481056213379,
      "activations/layer10_attention_weight_max": 67.2750015258789,
      "activations/layer10_attention_weight_min": -57.58249282836914,
      "activations/layer11_attention_weight_max": 42.966278076171875,
      "activations/layer11_attention_weight_min": -37.883785247802734,
      "activations/layer12_attention_weight_max": 28.89415740966797,
      "activations/layer12_attention_weight_min": -23.114131927490234,
      "activations/layer13_attention_weight_max": 45.47368240356445,
      "activations/layer13_attention_weight_min": -36.09220504760742,
      "activations/layer14_attention_weight_max": 47.42695236206055,
      "activations/layer14_attention_weight_min": -32.47953796386719,
      "activations/layer15_attention_weight_max": 45.53559875488281,
      "activations/layer15_attention_weight_min": -31.187931060791016,
      "activations/layer16_attention_weight_max": 34.72091293334961,
      "activations/layer16_attention_weight_min": -30.11294174194336,
      "activations/layer17_attention_weight_max": 53.82664108276367,
      "activations/layer17_attention_weight_min": -36.1158447265625,
      "activations/layer18_attention_weight_max": 53.34025955200195,
      "activations/layer18_attention_weight_min": -29.361652374267578,
      "activations/layer19_attention_weight_max": 19.252023696899414,
      "activations/layer19_attention_weight_min": -12.523460388183594,
      "activations/layer1_attention_weight_max": 17.046104431152344,
      "activations/layer1_attention_weight_min": -11.293498039245605,
      "activations/layer20_attention_weight_max": 21.044755935668945,
      "activations/layer20_attention_weight_min": -12.439217567443848,
      "activations/layer21_attention_weight_max": 26.908750534057617,
      "activations/layer21_attention_weight_min": -15.124946594238281,
      "activations/layer22_attention_weight_max": 23.93714141845703,
      "activations/layer22_attention_weight_min": -15.474531173706055,
      "activations/layer23_attention_weight_max": 18.967517852783203,
      "activations/layer23_attention_weight_min": -21.647430419921875,
      "activations/layer2_attention_weight_max": 25.763683319091797,
      "activations/layer2_attention_weight_min": -23.653064727783203,
      "activations/layer3_attention_weight_max": 48.659454345703125,
      "activations/layer3_attention_weight_min": -45.817787170410156,
      "activations/layer4_attention_weight_max": 77.60948944091797,
      "activations/layer4_attention_weight_min": -72.02919006347656,
      "activations/layer5_attention_weight_max": 138.1407928466797,
      "activations/layer5_attention_weight_min": -131.9017333984375,
      "activations/layer6_attention_weight_max": 75.8080062866211,
      "activations/layer6_attention_weight_min": -67.12825012207031,
      "activations/layer7_attention_weight_max": 79.9757308959961,
      "activations/layer7_attention_weight_min": -65.51913452148438,
      "activations/layer8_attention_weight_max": 67.34065246582031,
      "activations/layer8_attention_weight_min": -58.99626541137695,
      "activations/layer9_attention_weight_max": 89.52497100830078,
      "activations/layer9_attention_weight_min": -62.73298263549805,
      "epoch": 0.61,
      "learning_rate": 0.00014754734848484847,
      "loss": 3.2503,
      "step": 10500
    },
    {
      "activations/layer0_attention_weight_max": 14.374378204345703,
      "activations/layer0_attention_weight_min": -15.043856620788574,
      "activations/layer10_attention_weight_max": 64.61274719238281,
      "activations/layer10_attention_weight_min": -60.785240173339844,
      "activations/layer11_attention_weight_max": 49.59759521484375,
      "activations/layer11_attention_weight_min": -37.96259307861328,
      "activations/layer12_attention_weight_max": 27.283018112182617,
      "activations/layer12_attention_weight_min": -21.357816696166992,
      "activations/layer13_attention_weight_max": 49.24637222290039,
      "activations/layer13_attention_weight_min": -43.14180374145508,
      "activations/layer14_attention_weight_max": 34.678802490234375,
      "activations/layer14_attention_weight_min": -30.009428024291992,
      "activations/layer15_attention_weight_max": 38.66944122314453,
      "activations/layer15_attention_weight_min": -32.24832534790039,
      "activations/layer16_attention_weight_max": 34.12902069091797,
      "activations/layer16_attention_weight_min": -31.75691795349121,
      "activations/layer17_attention_weight_max": 51.6300163269043,
      "activations/layer17_attention_weight_min": -35.58896255493164,
      "activations/layer18_attention_weight_max": 37.777191162109375,
      "activations/layer18_attention_weight_min": -24.896703720092773,
      "activations/layer19_attention_weight_max": 19.438316345214844,
      "activations/layer19_attention_weight_min": -10.841742515563965,
      "activations/layer1_attention_weight_max": 17.044157028198242,
      "activations/layer1_attention_weight_min": -11.6517972946167,
      "activations/layer20_attention_weight_max": 22.33085823059082,
      "activations/layer20_attention_weight_min": -11.595199584960938,
      "activations/layer21_attention_weight_max": 28.821834564208984,
      "activations/layer21_attention_weight_min": -16.29947280883789,
      "activations/layer22_attention_weight_max": 27.793508529663086,
      "activations/layer22_attention_weight_min": -14.707375526428223,
      "activations/layer23_attention_weight_max": 20.219881057739258,
      "activations/layer23_attention_weight_min": -21.24304962158203,
      "activations/layer2_attention_weight_max": 27.166303634643555,
      "activations/layer2_attention_weight_min": -21.36864471435547,
      "activations/layer3_attention_weight_max": 49.931251525878906,
      "activations/layer3_attention_weight_min": -45.85517883300781,
      "activations/layer4_attention_weight_max": 79.89227294921875,
      "activations/layer4_attention_weight_min": -73.54984283447266,
      "activations/layer5_attention_weight_max": 141.19351196289062,
      "activations/layer5_attention_weight_min": -123.1556396484375,
      "activations/layer6_attention_weight_max": 70.77259063720703,
      "activations/layer6_attention_weight_min": -66.03941345214844,
      "activations/layer7_attention_weight_max": 79.5232162475586,
      "activations/layer7_attention_weight_min": -67.40995788574219,
      "activations/layer8_attention_weight_max": 61.81404495239258,
      "activations/layer8_attention_weight_min": -55.9227180480957,
      "activations/layer9_attention_weight_max": 67.9784164428711,
      "activations/layer9_attention_weight_min": -67.38225555419922,
      "epoch": 0.61,
      "learning_rate": 0.00014752840909090908,
      "loss": 3.2287,
      "step": 10550
    },
    {
      "activations/layer0_attention_weight_max": 13.833083152770996,
      "activations/layer0_attention_weight_min": -15.47925090789795,
      "activations/layer10_attention_weight_max": 65.36473846435547,
      "activations/layer10_attention_weight_min": -59.00633239746094,
      "activations/layer11_attention_weight_max": 50.254356384277344,
      "activations/layer11_attention_weight_min": -39.99405288696289,
      "activations/layer12_attention_weight_max": 25.65689468383789,
      "activations/layer12_attention_weight_min": -22.875097274780273,
      "activations/layer13_attention_weight_max": 52.51797103881836,
      "activations/layer13_attention_weight_min": -40.54336166381836,
      "activations/layer14_attention_weight_max": 45.79146957397461,
      "activations/layer14_attention_weight_min": -39.47517776489258,
      "activations/layer15_attention_weight_max": 40.05908966064453,
      "activations/layer15_attention_weight_min": -28.98792839050293,
      "activations/layer16_attention_weight_max": 38.35873794555664,
      "activations/layer16_attention_weight_min": -31.237409591674805,
      "activations/layer17_attention_weight_max": 56.97090530395508,
      "activations/layer17_attention_weight_min": -38.55015563964844,
      "activations/layer18_attention_weight_max": 40.417694091796875,
      "activations/layer18_attention_weight_min": -28.240116119384766,
      "activations/layer19_attention_weight_max": 20.554479598999023,
      "activations/layer19_attention_weight_min": -12.284622192382812,
      "activations/layer1_attention_weight_max": 18.90815544128418,
      "activations/layer1_attention_weight_min": -11.17126750946045,
      "activations/layer20_attention_weight_max": 21.190452575683594,
      "activations/layer20_attention_weight_min": -14.24782657623291,
      "activations/layer21_attention_weight_max": 30.67867660522461,
      "activations/layer21_attention_weight_min": -15.98250961303711,
      "activations/layer22_attention_weight_max": 30.193511962890625,
      "activations/layer22_attention_weight_min": -13.8565034866333,
      "activations/layer23_attention_weight_max": 22.274703979492188,
      "activations/layer23_attention_weight_min": -22.764135360717773,
      "activations/layer2_attention_weight_max": 26.82240104675293,
      "activations/layer2_attention_weight_min": -21.865554809570312,
      "activations/layer3_attention_weight_max": 51.1119384765625,
      "activations/layer3_attention_weight_min": -45.74972915649414,
      "activations/layer4_attention_weight_max": 82.72713470458984,
      "activations/layer4_attention_weight_min": -71.77238464355469,
      "activations/layer5_attention_weight_max": 144.2738037109375,
      "activations/layer5_attention_weight_min": -123.88072967529297,
      "activations/layer6_attention_weight_max": 73.29743957519531,
      "activations/layer6_attention_weight_min": -62.538795471191406,
      "activations/layer7_attention_weight_max": 79.27787017822266,
      "activations/layer7_attention_weight_min": -63.1010627746582,
      "activations/layer8_attention_weight_max": 60.13692092895508,
      "activations/layer8_attention_weight_min": -52.99293518066406,
      "activations/layer9_attention_weight_max": 75.26439666748047,
      "activations/layer9_attention_weight_min": -63.9989128112793,
      "epoch": 0.62,
      "learning_rate": 0.0001475094696969697,
      "loss": 3.2364,
      "step": 10600
    },
    {
      "activations/layer0_attention_weight_max": 14.097734451293945,
      "activations/layer0_attention_weight_min": -15.056230545043945,
      "activations/layer10_attention_weight_max": 71.3545150756836,
      "activations/layer10_attention_weight_min": -59.761653900146484,
      "activations/layer11_attention_weight_max": 50.539913177490234,
      "activations/layer11_attention_weight_min": -45.970947265625,
      "activations/layer12_attention_weight_max": 29.517671585083008,
      "activations/layer12_attention_weight_min": -25.2014217376709,
      "activations/layer13_attention_weight_max": 54.651771545410156,
      "activations/layer13_attention_weight_min": -42.71349334716797,
      "activations/layer14_attention_weight_max": 46.87619400024414,
      "activations/layer14_attention_weight_min": -35.00975036621094,
      "activations/layer15_attention_weight_max": 46.51344680786133,
      "activations/layer15_attention_weight_min": -31.182601928710938,
      "activations/layer16_attention_weight_max": 46.29957580566406,
      "activations/layer16_attention_weight_min": -32.03121566772461,
      "activations/layer17_attention_weight_max": 66.62034606933594,
      "activations/layer17_attention_weight_min": -37.927345275878906,
      "activations/layer18_attention_weight_max": 52.618186950683594,
      "activations/layer18_attention_weight_min": -31.132137298583984,
      "activations/layer19_attention_weight_max": 21.962690353393555,
      "activations/layer19_attention_weight_min": -11.909531593322754,
      "activations/layer1_attention_weight_max": 16.622264862060547,
      "activations/layer1_attention_weight_min": -10.649505615234375,
      "activations/layer20_attention_weight_max": 18.79063606262207,
      "activations/layer20_attention_weight_min": -11.952654838562012,
      "activations/layer21_attention_weight_max": 32.101722717285156,
      "activations/layer21_attention_weight_min": -16.928298950195312,
      "activations/layer22_attention_weight_max": 32.57208251953125,
      "activations/layer22_attention_weight_min": -13.255810737609863,
      "activations/layer23_attention_weight_max": 23.806652069091797,
      "activations/layer23_attention_weight_min": -22.991199493408203,
      "activations/layer2_attention_weight_max": 25.905624389648438,
      "activations/layer2_attention_weight_min": -21.132579803466797,
      "activations/layer3_attention_weight_max": 48.65862274169922,
      "activations/layer3_attention_weight_min": -44.75035858154297,
      "activations/layer4_attention_weight_max": 79.2291030883789,
      "activations/layer4_attention_weight_min": -72.63750457763672,
      "activations/layer5_attention_weight_max": 137.75416564941406,
      "activations/layer5_attention_weight_min": -121.24440002441406,
      "activations/layer6_attention_weight_max": 74.89323425292969,
      "activations/layer6_attention_weight_min": -64.82083129882812,
      "activations/layer7_attention_weight_max": 77.94646453857422,
      "activations/layer7_attention_weight_min": -63.96353530883789,
      "activations/layer8_attention_weight_max": 69.56156158447266,
      "activations/layer8_attention_weight_min": -55.92356872558594,
      "activations/layer9_attention_weight_max": 73.2762680053711,
      "activations/layer9_attention_weight_min": -65.17642211914062,
      "epoch": 0.62,
      "learning_rate": 0.0001474905303030303,
      "loss": 3.2188,
      "step": 10650
    },
    {
      "activations/layer0_attention_weight_max": 13.77144718170166,
      "activations/layer0_attention_weight_min": -15.536210060119629,
      "activations/layer10_attention_weight_max": 65.66571807861328,
      "activations/layer10_attention_weight_min": -59.40254592895508,
      "activations/layer11_attention_weight_max": 43.90220642089844,
      "activations/layer11_attention_weight_min": -38.042015075683594,
      "activations/layer12_attention_weight_max": 28.67723846435547,
      "activations/layer12_attention_weight_min": -23.33218765258789,
      "activations/layer13_attention_weight_max": 59.07143020629883,
      "activations/layer13_attention_weight_min": -45.15467071533203,
      "activations/layer14_attention_weight_max": 60.2130012512207,
      "activations/layer14_attention_weight_min": -41.56466293334961,
      "activations/layer15_attention_weight_max": 48.539283752441406,
      "activations/layer15_attention_weight_min": -30.86420440673828,
      "activations/layer16_attention_weight_max": 46.50425338745117,
      "activations/layer16_attention_weight_min": -32.14548873901367,
      "activations/layer17_attention_weight_max": 55.68520736694336,
      "activations/layer17_attention_weight_min": -33.46197509765625,
      "activations/layer18_attention_weight_max": 43.271263122558594,
      "activations/layer18_attention_weight_min": -28.797147750854492,
      "activations/layer19_attention_weight_max": 18.168424606323242,
      "activations/layer19_attention_weight_min": -11.076468467712402,
      "activations/layer1_attention_weight_max": 16.692623138427734,
      "activations/layer1_attention_weight_min": -11.254498481750488,
      "activations/layer20_attention_weight_max": 18.3128662109375,
      "activations/layer20_attention_weight_min": -12.972301483154297,
      "activations/layer21_attention_weight_max": 28.463909149169922,
      "activations/layer21_attention_weight_min": -15.2972412109375,
      "activations/layer22_attention_weight_max": 26.041181564331055,
      "activations/layer22_attention_weight_min": -12.772490501403809,
      "activations/layer23_attention_weight_max": 22.95919418334961,
      "activations/layer23_attention_weight_min": -22.294883728027344,
      "activations/layer2_attention_weight_max": 26.655141830444336,
      "activations/layer2_attention_weight_min": -22.12518310546875,
      "activations/layer3_attention_weight_max": 51.70851135253906,
      "activations/layer3_attention_weight_min": -48.87275695800781,
      "activations/layer4_attention_weight_max": 85.29642486572266,
      "activations/layer4_attention_weight_min": -72.32391357421875,
      "activations/layer5_attention_weight_max": 140.17178344726562,
      "activations/layer5_attention_weight_min": -128.40280151367188,
      "activations/layer6_attention_weight_max": 73.960693359375,
      "activations/layer6_attention_weight_min": -64.55921936035156,
      "activations/layer7_attention_weight_max": 72.37527465820312,
      "activations/layer7_attention_weight_min": -63.238529205322266,
      "activations/layer8_attention_weight_max": 59.95008087158203,
      "activations/layer8_attention_weight_min": -54.42469787597656,
      "activations/layer9_attention_weight_max": 71.95600891113281,
      "activations/layer9_attention_weight_min": -61.27535629272461,
      "epoch": 0.62,
      "learning_rate": 0.0001474715909090909,
      "loss": 3.2227,
      "step": 10700
    },
    {
      "activations/layer0_attention_weight_max": 13.586983680725098,
      "activations/layer0_attention_weight_min": -14.725873947143555,
      "activations/layer10_attention_weight_max": 68.77760314941406,
      "activations/layer10_attention_weight_min": -59.68460464477539,
      "activations/layer11_attention_weight_max": 45.773345947265625,
      "activations/layer11_attention_weight_min": -36.23938751220703,
      "activations/layer12_attention_weight_max": 28.5529727935791,
      "activations/layer12_attention_weight_min": -22.11029052734375,
      "activations/layer13_attention_weight_max": 60.07997512817383,
      "activations/layer13_attention_weight_min": -46.21903991699219,
      "activations/layer14_attention_weight_max": 42.92802429199219,
      "activations/layer14_attention_weight_min": -34.17621994018555,
      "activations/layer15_attention_weight_max": 45.49030303955078,
      "activations/layer15_attention_weight_min": -31.964073181152344,
      "activations/layer16_attention_weight_max": 40.91881561279297,
      "activations/layer16_attention_weight_min": -30.360301971435547,
      "activations/layer17_attention_weight_max": 55.69118118286133,
      "activations/layer17_attention_weight_min": -34.99928665161133,
      "activations/layer18_attention_weight_max": 48.704288482666016,
      "activations/layer18_attention_weight_min": -28.974210739135742,
      "activations/layer19_attention_weight_max": 22.094762802124023,
      "activations/layer19_attention_weight_min": -11.239924430847168,
      "activations/layer1_attention_weight_max": 17.38652992248535,
      "activations/layer1_attention_weight_min": -10.499261856079102,
      "activations/layer20_attention_weight_max": 21.976531982421875,
      "activations/layer20_attention_weight_min": -12.619697570800781,
      "activations/layer21_attention_weight_max": 31.88261604309082,
      "activations/layer21_attention_weight_min": -17.877084732055664,
      "activations/layer22_attention_weight_max": 31.37476921081543,
      "activations/layer22_attention_weight_min": -15.597818374633789,
      "activations/layer23_attention_weight_max": 23.505783081054688,
      "activations/layer23_attention_weight_min": -21.96820640563965,
      "activations/layer2_attention_weight_max": 26.338268280029297,
      "activations/layer2_attention_weight_min": -20.030746459960938,
      "activations/layer3_attention_weight_max": 48.889991760253906,
      "activations/layer3_attention_weight_min": -46.02034378051758,
      "activations/layer4_attention_weight_max": 80.4494400024414,
      "activations/layer4_attention_weight_min": -73.00393676757812,
      "activations/layer5_attention_weight_max": 137.53634643554688,
      "activations/layer5_attention_weight_min": -119.82463073730469,
      "activations/layer6_attention_weight_max": 74.0910415649414,
      "activations/layer6_attention_weight_min": -62.949440002441406,
      "activations/layer7_attention_weight_max": 68.62291717529297,
      "activations/layer7_attention_weight_min": -63.71218490600586,
      "activations/layer8_attention_weight_max": 60.48825454711914,
      "activations/layer8_attention_weight_min": -54.52583312988281,
      "activations/layer9_attention_weight_max": 75.42144775390625,
      "activations/layer9_attention_weight_min": -64.51714324951172,
      "epoch": 0.62,
      "learning_rate": 0.00014745265151515152,
      "loss": 3.2028,
      "step": 10750
    },
    {
      "activations/layer0_attention_weight_max": 13.204607009887695,
      "activations/layer0_attention_weight_min": -14.959281921386719,
      "activations/layer10_attention_weight_max": 73.3667221069336,
      "activations/layer10_attention_weight_min": -63.55763626098633,
      "activations/layer11_attention_weight_max": 49.969703674316406,
      "activations/layer11_attention_weight_min": -36.85630798339844,
      "activations/layer12_attention_weight_max": 28.002779006958008,
      "activations/layer12_attention_weight_min": -24.790159225463867,
      "activations/layer13_attention_weight_max": 54.47413635253906,
      "activations/layer13_attention_weight_min": -41.65321731567383,
      "activations/layer14_attention_weight_max": 43.71681594848633,
      "activations/layer14_attention_weight_min": -30.13043212890625,
      "activations/layer15_attention_weight_max": 47.480857849121094,
      "activations/layer15_attention_weight_min": -37.33120346069336,
      "activations/layer16_attention_weight_max": 38.79280471801758,
      "activations/layer16_attention_weight_min": -31.75194549560547,
      "activations/layer17_attention_weight_max": 58.45965576171875,
      "activations/layer17_attention_weight_min": -37.50125503540039,
      "activations/layer18_attention_weight_max": 37.746681213378906,
      "activations/layer18_attention_weight_min": -26.09303092956543,
      "activations/layer19_attention_weight_max": 21.841537475585938,
      "activations/layer19_attention_weight_min": -12.58337116241455,
      "activations/layer1_attention_weight_max": 16.882762908935547,
      "activations/layer1_attention_weight_min": -10.797030448913574,
      "activations/layer20_attention_weight_max": 17.90712547302246,
      "activations/layer20_attention_weight_min": -11.298254013061523,
      "activations/layer21_attention_weight_max": 31.46402931213379,
      "activations/layer21_attention_weight_min": -15.435163497924805,
      "activations/layer22_attention_weight_max": 30.063426971435547,
      "activations/layer22_attention_weight_min": -13.45128059387207,
      "activations/layer23_attention_weight_max": 23.252195358276367,
      "activations/layer23_attention_weight_min": -19.520164489746094,
      "activations/layer2_attention_weight_max": 29.361204147338867,
      "activations/layer2_attention_weight_min": -21.773860931396484,
      "activations/layer3_attention_weight_max": 53.42677307128906,
      "activations/layer3_attention_weight_min": -46.335479736328125,
      "activations/layer4_attention_weight_max": 84.85106658935547,
      "activations/layer4_attention_weight_min": -73.71099090576172,
      "activations/layer5_attention_weight_max": 149.68258666992188,
      "activations/layer5_attention_weight_min": -127.28316497802734,
      "activations/layer6_attention_weight_max": 76.4678726196289,
      "activations/layer6_attention_weight_min": -65.02873229980469,
      "activations/layer7_attention_weight_max": 74.64739990234375,
      "activations/layer7_attention_weight_min": -65.8346176147461,
      "activations/layer8_attention_weight_max": 66.0584716796875,
      "activations/layer8_attention_weight_min": -59.37852096557617,
      "activations/layer9_attention_weight_max": 73.45650482177734,
      "activations/layer9_attention_weight_min": -65.76675415039062,
      "epoch": 0.63,
      "learning_rate": 0.0001474337121212121,
      "loss": 3.2148,
      "step": 10800
    },
    {
      "activations/layer0_attention_weight_max": 13.72568416595459,
      "activations/layer0_attention_weight_min": -14.387795448303223,
      "activations/layer10_attention_weight_max": 66.32303619384766,
      "activations/layer10_attention_weight_min": -60.96969985961914,
      "activations/layer11_attention_weight_max": 43.301109313964844,
      "activations/layer11_attention_weight_min": -38.71099090576172,
      "activations/layer12_attention_weight_max": 23.48570442199707,
      "activations/layer12_attention_weight_min": -22.65232276916504,
      "activations/layer13_attention_weight_max": 52.619972229003906,
      "activations/layer13_attention_weight_min": -47.153446197509766,
      "activations/layer14_attention_weight_max": 53.21536636352539,
      "activations/layer14_attention_weight_min": -39.118839263916016,
      "activations/layer15_attention_weight_max": 40.27674102783203,
      "activations/layer15_attention_weight_min": -32.194000244140625,
      "activations/layer16_attention_weight_max": 41.682884216308594,
      "activations/layer16_attention_weight_min": -32.12331008911133,
      "activations/layer17_attention_weight_max": 50.92427444458008,
      "activations/layer17_attention_weight_min": -40.187110900878906,
      "activations/layer18_attention_weight_max": 35.62932205200195,
      "activations/layer18_attention_weight_min": -23.06544303894043,
      "activations/layer19_attention_weight_max": 20.191774368286133,
      "activations/layer19_attention_weight_min": -10.599777221679688,
      "activations/layer1_attention_weight_max": 17.406978607177734,
      "activations/layer1_attention_weight_min": -11.025239944458008,
      "activations/layer20_attention_weight_max": 15.8167142868042,
      "activations/layer20_attention_weight_min": -11.530182838439941,
      "activations/layer21_attention_weight_max": 28.255123138427734,
      "activations/layer21_attention_weight_min": -15.997225761413574,
      "activations/layer22_attention_weight_max": 23.129310607910156,
      "activations/layer22_attention_weight_min": -14.787640571594238,
      "activations/layer23_attention_weight_max": 21.54878044128418,
      "activations/layer23_attention_weight_min": -19.696002960205078,
      "activations/layer2_attention_weight_max": 27.120267868041992,
      "activations/layer2_attention_weight_min": -20.893264770507812,
      "activations/layer3_attention_weight_max": 47.052310943603516,
      "activations/layer3_attention_weight_min": -43.890506744384766,
      "activations/layer4_attention_weight_max": 78.17325592041016,
      "activations/layer4_attention_weight_min": -71.49100494384766,
      "activations/layer5_attention_weight_max": 137.89801025390625,
      "activations/layer5_attention_weight_min": -119.42071533203125,
      "activations/layer6_attention_weight_max": 71.29684448242188,
      "activations/layer6_attention_weight_min": -61.46408462524414,
      "activations/layer7_attention_weight_max": 77.69561004638672,
      "activations/layer7_attention_weight_min": -65.64726257324219,
      "activations/layer8_attention_weight_max": 62.76302719116211,
      "activations/layer8_attention_weight_min": -55.30213928222656,
      "activations/layer9_attention_weight_max": 79.42583465576172,
      "activations/layer9_attention_weight_min": -61.595680236816406,
      "epoch": 0.63,
      "learning_rate": 0.00014741477272727272,
      "loss": 3.2215,
      "step": 10850
    },
    {
      "activations/layer0_attention_weight_max": 14.061907768249512,
      "activations/layer0_attention_weight_min": -15.23155689239502,
      "activations/layer10_attention_weight_max": 66.78956604003906,
      "activations/layer10_attention_weight_min": -61.16001892089844,
      "activations/layer11_attention_weight_max": 46.378536224365234,
      "activations/layer11_attention_weight_min": -40.36494064331055,
      "activations/layer12_attention_weight_max": 24.910083770751953,
      "activations/layer12_attention_weight_min": -23.380096435546875,
      "activations/layer13_attention_weight_max": 62.23160171508789,
      "activations/layer13_attention_weight_min": -49.29274368286133,
      "activations/layer14_attention_weight_max": 44.43854904174805,
      "activations/layer14_attention_weight_min": -34.44123077392578,
      "activations/layer15_attention_weight_max": 42.793704986572266,
      "activations/layer15_attention_weight_min": -32.61874008178711,
      "activations/layer16_attention_weight_max": 39.493404388427734,
      "activations/layer16_attention_weight_min": -31.245372772216797,
      "activations/layer17_attention_weight_max": 47.757389068603516,
      "activations/layer17_attention_weight_min": -36.94804000854492,
      "activations/layer18_attention_weight_max": 52.61079406738281,
      "activations/layer18_attention_weight_min": -33.667327880859375,
      "activations/layer19_attention_weight_max": 18.238033294677734,
      "activations/layer19_attention_weight_min": -11.822897911071777,
      "activations/layer1_attention_weight_max": 17.27849769592285,
      "activations/layer1_attention_weight_min": -11.884856224060059,
      "activations/layer20_attention_weight_max": 18.899747848510742,
      "activations/layer20_attention_weight_min": -12.674090385437012,
      "activations/layer21_attention_weight_max": 31.18745994567871,
      "activations/layer21_attention_weight_min": -17.181095123291016,
      "activations/layer22_attention_weight_max": 31.038511276245117,
      "activations/layer22_attention_weight_min": -15.569175720214844,
      "activations/layer23_attention_weight_max": 22.28171157836914,
      "activations/layer23_attention_weight_min": -20.31206512451172,
      "activations/layer2_attention_weight_max": 27.318389892578125,
      "activations/layer2_attention_weight_min": -20.15438461303711,
      "activations/layer3_attention_weight_max": 48.807220458984375,
      "activations/layer3_attention_weight_min": -44.11740493774414,
      "activations/layer4_attention_weight_max": 79.15149688720703,
      "activations/layer4_attention_weight_min": -72.37290954589844,
      "activations/layer5_attention_weight_max": 137.56149291992188,
      "activations/layer5_attention_weight_min": -122.98175048828125,
      "activations/layer6_attention_weight_max": 71.9560317993164,
      "activations/layer6_attention_weight_min": -63.77043151855469,
      "activations/layer7_attention_weight_max": 70.7378921508789,
      "activations/layer7_attention_weight_min": -64.94036102294922,
      "activations/layer8_attention_weight_max": 59.32050323486328,
      "activations/layer8_attention_weight_min": -54.14765167236328,
      "activations/layer9_attention_weight_max": 71.86618041992188,
      "activations/layer9_attention_weight_min": -71.40850830078125,
      "epoch": 0.63,
      "learning_rate": 0.0001473958333333333,
      "loss": 3.2013,
      "step": 10900
    },
    {
      "activations/layer0_attention_weight_max": 13.417215347290039,
      "activations/layer0_attention_weight_min": -14.655022621154785,
      "activations/layer10_attention_weight_max": 69.91687774658203,
      "activations/layer10_attention_weight_min": -60.4399528503418,
      "activations/layer11_attention_weight_max": 43.06184387207031,
      "activations/layer11_attention_weight_min": -40.35930633544922,
      "activations/layer12_attention_weight_max": 27.77533721923828,
      "activations/layer12_attention_weight_min": -23.678852081298828,
      "activations/layer13_attention_weight_max": 54.596126556396484,
      "activations/layer13_attention_weight_min": -40.904396057128906,
      "activations/layer14_attention_weight_max": 45.2021598815918,
      "activations/layer14_attention_weight_min": -31.153221130371094,
      "activations/layer15_attention_weight_max": 45.72802734375,
      "activations/layer15_attention_weight_min": -30.57039451599121,
      "activations/layer16_attention_weight_max": 42.68928146362305,
      "activations/layer16_attention_weight_min": -31.538129806518555,
      "activations/layer17_attention_weight_max": 53.626068115234375,
      "activations/layer17_attention_weight_min": -34.320892333984375,
      "activations/layer18_attention_weight_max": 50.45753479003906,
      "activations/layer18_attention_weight_min": -29.497228622436523,
      "activations/layer19_attention_weight_max": 21.323421478271484,
      "activations/layer19_attention_weight_min": -10.669453620910645,
      "activations/layer1_attention_weight_max": 16.28886604309082,
      "activations/layer1_attention_weight_min": -11.384641647338867,
      "activations/layer20_attention_weight_max": 22.171480178833008,
      "activations/layer20_attention_weight_min": -11.465780258178711,
      "activations/layer21_attention_weight_max": 31.980680465698242,
      "activations/layer21_attention_weight_min": -15.488017082214355,
      "activations/layer22_attention_weight_max": 28.48684310913086,
      "activations/layer22_attention_weight_min": -14.65061092376709,
      "activations/layer23_attention_weight_max": 22.515012741088867,
      "activations/layer23_attention_weight_min": -19.52772331237793,
      "activations/layer2_attention_weight_max": 24.888595581054688,
      "activations/layer2_attention_weight_min": -21.961437225341797,
      "activations/layer3_attention_weight_max": 52.786808013916016,
      "activations/layer3_attention_weight_min": -47.37983703613281,
      "activations/layer4_attention_weight_max": 85.25098419189453,
      "activations/layer4_attention_weight_min": -73.85143280029297,
      "activations/layer5_attention_weight_max": 148.3412628173828,
      "activations/layer5_attention_weight_min": -124.15499877929688,
      "activations/layer6_attention_weight_max": 78.38668060302734,
      "activations/layer6_attention_weight_min": -66.48226928710938,
      "activations/layer7_attention_weight_max": 76.19715881347656,
      "activations/layer7_attention_weight_min": -65.17449951171875,
      "activations/layer8_attention_weight_max": 68.64014434814453,
      "activations/layer8_attention_weight_min": -56.24785614013672,
      "activations/layer9_attention_weight_max": 71.7843017578125,
      "activations/layer9_attention_weight_min": -63.96751022338867,
      "epoch": 0.64,
      "learning_rate": 0.00014737689393939392,
      "loss": 3.1997,
      "step": 10950
    },
    {
      "activations/layer0_attention_weight_max": 13.373489379882812,
      "activations/layer0_attention_weight_min": -14.176743507385254,
      "activations/layer10_attention_weight_max": 68.82618713378906,
      "activations/layer10_attention_weight_min": -60.36691665649414,
      "activations/layer11_attention_weight_max": 45.920711517333984,
      "activations/layer11_attention_weight_min": -38.54592514038086,
      "activations/layer12_attention_weight_max": 24.69013023376465,
      "activations/layer12_attention_weight_min": -28.975034713745117,
      "activations/layer13_attention_weight_max": 69.02053833007812,
      "activations/layer13_attention_weight_min": -47.103843688964844,
      "activations/layer14_attention_weight_max": 53.54086685180664,
      "activations/layer14_attention_weight_min": -36.41223907470703,
      "activations/layer15_attention_weight_max": 51.27627182006836,
      "activations/layer15_attention_weight_min": -35.78413772583008,
      "activations/layer16_attention_weight_max": 38.65892791748047,
      "activations/layer16_attention_weight_min": -31.25882911682129,
      "activations/layer17_attention_weight_max": 54.25962448120117,
      "activations/layer17_attention_weight_min": -38.9303092956543,
      "activations/layer18_attention_weight_max": 52.736331939697266,
      "activations/layer18_attention_weight_min": -32.83159637451172,
      "activations/layer19_attention_weight_max": 20.092561721801758,
      "activations/layer19_attention_weight_min": -11.501380920410156,
      "activations/layer1_attention_weight_max": 17.205472946166992,
      "activations/layer1_attention_weight_min": -10.502708435058594,
      "activations/layer20_attention_weight_max": 21.04821014404297,
      "activations/layer20_attention_weight_min": -14.761693954467773,
      "activations/layer21_attention_weight_max": 37.53310775756836,
      "activations/layer21_attention_weight_min": -15.783720970153809,
      "activations/layer22_attention_weight_max": 34.038944244384766,
      "activations/layer22_attention_weight_min": -14.9896879196167,
      "activations/layer23_attention_weight_max": 25.895702362060547,
      "activations/layer23_attention_weight_min": -22.515865325927734,
      "activations/layer2_attention_weight_max": 27.346088409423828,
      "activations/layer2_attention_weight_min": -21.90419578552246,
      "activations/layer3_attention_weight_max": 49.29397964477539,
      "activations/layer3_attention_weight_min": -46.4029541015625,
      "activations/layer4_attention_weight_max": 79.15778350830078,
      "activations/layer4_attention_weight_min": -71.44005584716797,
      "activations/layer5_attention_weight_max": 140.1254425048828,
      "activations/layer5_attention_weight_min": -122.739013671875,
      "activations/layer6_attention_weight_max": 75.73373413085938,
      "activations/layer6_attention_weight_min": -62.57771682739258,
      "activations/layer7_attention_weight_max": 75.04872131347656,
      "activations/layer7_attention_weight_min": -64.54694366455078,
      "activations/layer8_attention_weight_max": 68.58776092529297,
      "activations/layer8_attention_weight_min": -58.64077377319336,
      "activations/layer9_attention_weight_max": 82.19003295898438,
      "activations/layer9_attention_weight_min": -66.69402313232422,
      "epoch": 0.64,
      "learning_rate": 0.00014735795454545454,
      "loss": 3.2153,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_loss": 3.15234375,
      "eval_runtime": 8.2329,
      "eval_samples_per_second": 521.567,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_openwebtext_loss": 3.15234375,
      "eval_openwebtext_ppl": 23.390822626823073,
      "eval_openwebtext_runtime": 8.2329,
      "eval_openwebtext_samples_per_second": 521.567,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_wikitext_loss": 3.44140625,
      "eval_wikitext_ppl": 31.230845679407448,
      "eval_wikitext_runtime": 1.676,
      "eval_wikitext_samples_per_second": 272.076,
      "step": 11000
    },
    {
      "epoch": 0.64,
      "eval_lambada_loss": 3.712890625,
      "eval_lambada_ppl": 40.97207040739876,
      "eval_lambada_runtime": 9.2852,
      "eval_lambada_samples_per_second": 524.381,
      "step": 11000
    },
    {
      "activations/layer0_attention_weight_max": 13.706242561340332,
      "activations/layer0_attention_weight_min": -14.59172248840332,
      "activations/layer10_attention_weight_max": 61.20641326904297,
      "activations/layer10_attention_weight_min": -59.38896942138672,
      "activations/layer11_attention_weight_max": 47.30503845214844,
      "activations/layer11_attention_weight_min": -38.69221496582031,
      "activations/layer12_attention_weight_max": 27.913833618164062,
      "activations/layer12_attention_weight_min": -24.880401611328125,
      "activations/layer13_attention_weight_max": 50.1789436340332,
      "activations/layer13_attention_weight_min": -43.88669967651367,
      "activations/layer14_attention_weight_max": 44.733585357666016,
      "activations/layer14_attention_weight_min": -40.65188217163086,
      "activations/layer15_attention_weight_max": 42.063602447509766,
      "activations/layer15_attention_weight_min": -34.941864013671875,
      "activations/layer16_attention_weight_max": 43.20358657836914,
      "activations/layer16_attention_weight_min": -30.172622680664062,
      "activations/layer17_attention_weight_max": 53.65687561035156,
      "activations/layer17_attention_weight_min": -36.51755142211914,
      "activations/layer18_attention_weight_max": 39.875083923339844,
      "activations/layer18_attention_weight_min": -28.74769401550293,
      "activations/layer19_attention_weight_max": 18.2504825592041,
      "activations/layer19_attention_weight_min": -11.824494361877441,
      "activations/layer1_attention_weight_max": 17.380151748657227,
      "activations/layer1_attention_weight_min": -10.419751167297363,
      "activations/layer20_attention_weight_max": 18.177261352539062,
      "activations/layer20_attention_weight_min": -12.640743255615234,
      "activations/layer21_attention_weight_max": 32.07991027832031,
      "activations/layer21_attention_weight_min": -14.153438568115234,
      "activations/layer22_attention_weight_max": 28.539772033691406,
      "activations/layer22_attention_weight_min": -14.892254829406738,
      "activations/layer23_attention_weight_max": 20.870269775390625,
      "activations/layer23_attention_weight_min": -21.57290267944336,
      "activations/layer2_attention_weight_max": 25.595584869384766,
      "activations/layer2_attention_weight_min": -20.994449615478516,
      "activations/layer3_attention_weight_max": 48.77638244628906,
      "activations/layer3_attention_weight_min": -43.50291442871094,
      "activations/layer4_attention_weight_max": 75.08515167236328,
      "activations/layer4_attention_weight_min": -68.84668731689453,
      "activations/layer5_attention_weight_max": 134.67835998535156,
      "activations/layer5_attention_weight_min": -116.46807098388672,
      "activations/layer6_attention_weight_max": 69.73580169677734,
      "activations/layer6_attention_weight_min": -60.504764556884766,
      "activations/layer7_attention_weight_max": 67.85393524169922,
      "activations/layer7_attention_weight_min": -65.92147827148438,
      "activations/layer8_attention_weight_max": 56.71367263793945,
      "activations/layer8_attention_weight_min": -52.348209381103516,
      "activations/layer9_attention_weight_max": 66.66082000732422,
      "activations/layer9_attention_weight_min": -69.28719329833984,
      "epoch": 0.64,
      "learning_rate": 0.00014733901515151513,
      "loss": 3.2029,
      "step": 11050
    },
    {
      "activations/layer0_attention_weight_max": 13.742795944213867,
      "activations/layer0_attention_weight_min": -14.337897300720215,
      "activations/layer10_attention_weight_max": 66.93528747558594,
      "activations/layer10_attention_weight_min": -60.28248977661133,
      "activations/layer11_attention_weight_max": 47.44025802612305,
      "activations/layer11_attention_weight_min": -41.679603576660156,
      "activations/layer12_attention_weight_max": 27.261598587036133,
      "activations/layer12_attention_weight_min": -26.859479904174805,
      "activations/layer13_attention_weight_max": 53.64110565185547,
      "activations/layer13_attention_weight_min": -38.988826751708984,
      "activations/layer14_attention_weight_max": 42.23615264892578,
      "activations/layer14_attention_weight_min": -34.26045608520508,
      "activations/layer15_attention_weight_max": 40.77052688598633,
      "activations/layer15_attention_weight_min": -30.68630027770996,
      "activations/layer16_attention_weight_max": 40.482093811035156,
      "activations/layer16_attention_weight_min": -30.50395393371582,
      "activations/layer17_attention_weight_max": 56.65380859375,
      "activations/layer17_attention_weight_min": -34.91439437866211,
      "activations/layer18_attention_weight_max": 32.06520080566406,
      "activations/layer18_attention_weight_min": -24.825620651245117,
      "activations/layer19_attention_weight_max": 17.862131118774414,
      "activations/layer19_attention_weight_min": -11.538471221923828,
      "activations/layer1_attention_weight_max": 17.27728843688965,
      "activations/layer1_attention_weight_min": -10.495702743530273,
      "activations/layer20_attention_weight_max": 17.553340911865234,
      "activations/layer20_attention_weight_min": -13.021120071411133,
      "activations/layer21_attention_weight_max": 27.49807357788086,
      "activations/layer21_attention_weight_min": -13.817058563232422,
      "activations/layer22_attention_weight_max": 23.78188133239746,
      "activations/layer22_attention_weight_min": -13.3552885055542,
      "activations/layer23_attention_weight_max": 21.72018814086914,
      "activations/layer23_attention_weight_min": -22.29366683959961,
      "activations/layer2_attention_weight_max": 24.843053817749023,
      "activations/layer2_attention_weight_min": -21.601062774658203,
      "activations/layer3_attention_weight_max": 51.56028747558594,
      "activations/layer3_attention_weight_min": -47.64247131347656,
      "activations/layer4_attention_weight_max": 80.32537078857422,
      "activations/layer4_attention_weight_min": -72.73495483398438,
      "activations/layer5_attention_weight_max": 138.9080352783203,
      "activations/layer5_attention_weight_min": -122.35296630859375,
      "activations/layer6_attention_weight_max": 69.83055877685547,
      "activations/layer6_attention_weight_min": -63.99953079223633,
      "activations/layer7_attention_weight_max": 71.75418853759766,
      "activations/layer7_attention_weight_min": -68.78117370605469,
      "activations/layer8_attention_weight_max": 58.0565071105957,
      "activations/layer8_attention_weight_min": -52.73294448852539,
      "activations/layer9_attention_weight_max": 67.13727569580078,
      "activations/layer9_attention_weight_min": -68.82740020751953,
      "epoch": 0.64,
      "learning_rate": 0.00014732007575757574,
      "loss": 3.1966,
      "step": 11100
    },
    {
      "activations/layer0_attention_weight_max": 13.428759574890137,
      "activations/layer0_attention_weight_min": -14.091836929321289,
      "activations/layer10_attention_weight_max": 65.61170959472656,
      "activations/layer10_attention_weight_min": -55.367767333984375,
      "activations/layer11_attention_weight_max": 46.12370300292969,
      "activations/layer11_attention_weight_min": -36.91041946411133,
      "activations/layer12_attention_weight_max": 24.168920516967773,
      "activations/layer12_attention_weight_min": -25.29072380065918,
      "activations/layer13_attention_weight_max": 51.51333236694336,
      "activations/layer13_attention_weight_min": -39.10163116455078,
      "activations/layer14_attention_weight_max": 50.59071350097656,
      "activations/layer14_attention_weight_min": -41.23982620239258,
      "activations/layer15_attention_weight_max": 47.19346237182617,
      "activations/layer15_attention_weight_min": -33.826900482177734,
      "activations/layer16_attention_weight_max": 41.02302169799805,
      "activations/layer16_attention_weight_min": -28.850645065307617,
      "activations/layer17_attention_weight_max": 53.360225677490234,
      "activations/layer17_attention_weight_min": -36.185543060302734,
      "activations/layer18_attention_weight_max": 50.11148452758789,
      "activations/layer18_attention_weight_min": -31.424833297729492,
      "activations/layer19_attention_weight_max": 20.612506866455078,
      "activations/layer19_attention_weight_min": -11.754867553710938,
      "activations/layer1_attention_weight_max": 18.015689849853516,
      "activations/layer1_attention_weight_min": -11.478293418884277,
      "activations/layer20_attention_weight_max": 18.74810028076172,
      "activations/layer20_attention_weight_min": -12.135753631591797,
      "activations/layer21_attention_weight_max": 35.176334381103516,
      "activations/layer21_attention_weight_min": -15.907069206237793,
      "activations/layer22_attention_weight_max": 29.488759994506836,
      "activations/layer22_attention_weight_min": -13.771276473999023,
      "activations/layer23_attention_weight_max": 22.626728057861328,
      "activations/layer23_attention_weight_min": -21.69257354736328,
      "activations/layer2_attention_weight_max": 26.899084091186523,
      "activations/layer2_attention_weight_min": -21.9041748046875,
      "activations/layer3_attention_weight_max": 49.833927154541016,
      "activations/layer3_attention_weight_min": -45.41531753540039,
      "activations/layer4_attention_weight_max": 78.40731048583984,
      "activations/layer4_attention_weight_min": -71.80792999267578,
      "activations/layer5_attention_weight_max": 134.9402313232422,
      "activations/layer5_attention_weight_min": -127.27323913574219,
      "activations/layer6_attention_weight_max": 69.15597534179688,
      "activations/layer6_attention_weight_min": -62.37189865112305,
      "activations/layer7_attention_weight_max": 76.0474853515625,
      "activations/layer7_attention_weight_min": -62.80348587036133,
      "activations/layer8_attention_weight_max": 61.8644905090332,
      "activations/layer8_attention_weight_min": -52.514930725097656,
      "activations/layer9_attention_weight_max": 70.12523651123047,
      "activations/layer9_attention_weight_min": -63.456233978271484,
      "epoch": 0.65,
      "learning_rate": 0.00014730113636363636,
      "loss": 3.1983,
      "step": 11150
    },
    {
      "activations/layer0_attention_weight_max": 13.543000221252441,
      "activations/layer0_attention_weight_min": -13.46683406829834,
      "activations/layer10_attention_weight_max": 67.4268569946289,
      "activations/layer10_attention_weight_min": -56.89366149902344,
      "activations/layer11_attention_weight_max": 44.014015197753906,
      "activations/layer11_attention_weight_min": -37.63134002685547,
      "activations/layer12_attention_weight_max": 22.810890197753906,
      "activations/layer12_attention_weight_min": -23.120071411132812,
      "activations/layer13_attention_weight_max": 56.69477081298828,
      "activations/layer13_attention_weight_min": -40.54624938964844,
      "activations/layer14_attention_weight_max": 49.434932708740234,
      "activations/layer14_attention_weight_min": -31.86039924621582,
      "activations/layer15_attention_weight_max": 43.10700607299805,
      "activations/layer15_attention_weight_min": -31.69887351989746,
      "activations/layer16_attention_weight_max": 41.68332290649414,
      "activations/layer16_attention_weight_min": -32.65427780151367,
      "activations/layer17_attention_weight_max": 51.17609786987305,
      "activations/layer17_attention_weight_min": -40.68257522583008,
      "activations/layer18_attention_weight_max": 50.738277435302734,
      "activations/layer18_attention_weight_min": -36.2884407043457,
      "activations/layer19_attention_weight_max": 16.463415145874023,
      "activations/layer19_attention_weight_min": -12.102431297302246,
      "activations/layer1_attention_weight_max": 16.048425674438477,
      "activations/layer1_attention_weight_min": -11.080370903015137,
      "activations/layer20_attention_weight_max": 19.5594425201416,
      "activations/layer20_attention_weight_min": -13.05886173248291,
      "activations/layer21_attention_weight_max": 30.98733901977539,
      "activations/layer21_attention_weight_min": -17.960254669189453,
      "activations/layer22_attention_weight_max": 28.145919799804688,
      "activations/layer22_attention_weight_min": -16.300273895263672,
      "activations/layer23_attention_weight_max": 22.101224899291992,
      "activations/layer23_attention_weight_min": -21.13272476196289,
      "activations/layer2_attention_weight_max": 25.888103485107422,
      "activations/layer2_attention_weight_min": -21.925682067871094,
      "activations/layer3_attention_weight_max": 53.87327575683594,
      "activations/layer3_attention_weight_min": -48.671302795410156,
      "activations/layer4_attention_weight_max": 83.441162109375,
      "activations/layer4_attention_weight_min": -72.72246551513672,
      "activations/layer5_attention_weight_max": 154.62051391601562,
      "activations/layer5_attention_weight_min": -126.86492919921875,
      "activations/layer6_attention_weight_max": 77.99026489257812,
      "activations/layer6_attention_weight_min": -64.58977508544922,
      "activations/layer7_attention_weight_max": 77.78239440917969,
      "activations/layer7_attention_weight_min": -63.9414176940918,
      "activations/layer8_attention_weight_max": 63.23527908325195,
      "activations/layer8_attention_weight_min": -53.28573226928711,
      "activations/layer9_attention_weight_max": 73.29895782470703,
      "activations/layer9_attention_weight_min": -68.2149658203125,
      "epoch": 0.65,
      "learning_rate": 0.00014728219696969694,
      "loss": 3.2172,
      "step": 11200
    },
    {
      "activations/layer0_attention_weight_max": 13.36274242401123,
      "activations/layer0_attention_weight_min": -14.104531288146973,
      "activations/layer10_attention_weight_max": 63.028377532958984,
      "activations/layer10_attention_weight_min": -59.502315521240234,
      "activations/layer11_attention_weight_max": 44.714698791503906,
      "activations/layer11_attention_weight_min": -39.787315368652344,
      "activations/layer12_attention_weight_max": 26.306106567382812,
      "activations/layer12_attention_weight_min": -24.62222671508789,
      "activations/layer13_attention_weight_max": 54.600608825683594,
      "activations/layer13_attention_weight_min": -41.520381927490234,
      "activations/layer14_attention_weight_max": 51.810760498046875,
      "activations/layer14_attention_weight_min": -37.04037094116211,
      "activations/layer15_attention_weight_max": 43.90658187866211,
      "activations/layer15_attention_weight_min": -31.00187873840332,
      "activations/layer16_attention_weight_max": 44.72725296020508,
      "activations/layer16_attention_weight_min": -31.43644142150879,
      "activations/layer17_attention_weight_max": 54.332359313964844,
      "activations/layer17_attention_weight_min": -36.46306228637695,
      "activations/layer18_attention_weight_max": 56.972938537597656,
      "activations/layer18_attention_weight_min": -36.754295349121094,
      "activations/layer19_attention_weight_max": 17.466596603393555,
      "activations/layer19_attention_weight_min": -11.95383358001709,
      "activations/layer1_attention_weight_max": 16.76875877380371,
      "activations/layer1_attention_weight_min": -11.329028129577637,
      "activations/layer20_attention_weight_max": 18.628520965576172,
      "activations/layer20_attention_weight_min": -13.067407608032227,
      "activations/layer21_attention_weight_max": 36.28590393066406,
      "activations/layer21_attention_weight_min": -14.888242721557617,
      "activations/layer22_attention_weight_max": 33.543270111083984,
      "activations/layer22_attention_weight_min": -13.799116134643555,
      "activations/layer23_attention_weight_max": 24.064735412597656,
      "activations/layer23_attention_weight_min": -23.32259750366211,
      "activations/layer2_attention_weight_max": 25.643627166748047,
      "activations/layer2_attention_weight_min": -20.634187698364258,
      "activations/layer3_attention_weight_max": 50.39668273925781,
      "activations/layer3_attention_weight_min": -44.60886001586914,
      "activations/layer4_attention_weight_max": 79.99494171142578,
      "activations/layer4_attention_weight_min": -71.02338409423828,
      "activations/layer5_attention_weight_max": 134.60487365722656,
      "activations/layer5_attention_weight_min": -124.60796356201172,
      "activations/layer6_attention_weight_max": 69.02363586425781,
      "activations/layer6_attention_weight_min": -60.509490966796875,
      "activations/layer7_attention_weight_max": 68.6580581665039,
      "activations/layer7_attention_weight_min": -64.6976318359375,
      "activations/layer8_attention_weight_max": 60.72233963012695,
      "activations/layer8_attention_weight_min": -53.864418029785156,
      "activations/layer9_attention_weight_max": 68.40047454833984,
      "activations/layer9_attention_weight_min": -62.6460075378418,
      "epoch": 0.65,
      "learning_rate": 0.00014726325757575756,
      "loss": 3.1953,
      "step": 11250
    },
    {
      "activations/layer0_attention_weight_max": 13.501300811767578,
      "activations/layer0_attention_weight_min": -13.720429420471191,
      "activations/layer10_attention_weight_max": 66.80145263671875,
      "activations/layer10_attention_weight_min": -61.969940185546875,
      "activations/layer11_attention_weight_max": 44.93983459472656,
      "activations/layer11_attention_weight_min": -38.45454406738281,
      "activations/layer12_attention_weight_max": 26.329593658447266,
      "activations/layer12_attention_weight_min": -25.07218360900879,
      "activations/layer13_attention_weight_max": 54.7974967956543,
      "activations/layer13_attention_weight_min": -37.29714584350586,
      "activations/layer14_attention_weight_max": 48.700069427490234,
      "activations/layer14_attention_weight_min": -37.712032318115234,
      "activations/layer15_attention_weight_max": 45.219486236572266,
      "activations/layer15_attention_weight_min": -34.16352462768555,
      "activations/layer16_attention_weight_max": 40.863853454589844,
      "activations/layer16_attention_weight_min": -28.444198608398438,
      "activations/layer17_attention_weight_max": 50.00555419921875,
      "activations/layer17_attention_weight_min": -35.65711212158203,
      "activations/layer18_attention_weight_max": 59.309776306152344,
      "activations/layer18_attention_weight_min": -28.45693588256836,
      "activations/layer19_attention_weight_max": 18.03461265563965,
      "activations/layer19_attention_weight_min": -14.424338340759277,
      "activations/layer1_attention_weight_max": 16.2025203704834,
      "activations/layer1_attention_weight_min": -10.591910362243652,
      "activations/layer20_attention_weight_max": 20.51972770690918,
      "activations/layer20_attention_weight_min": -13.01842975616455,
      "activations/layer21_attention_weight_max": 28.098663330078125,
      "activations/layer21_attention_weight_min": -18.36166763305664,
      "activations/layer22_attention_weight_max": 25.99403953552246,
      "activations/layer22_attention_weight_min": -13.244194984436035,
      "activations/layer23_attention_weight_max": 20.507272720336914,
      "activations/layer23_attention_weight_min": -18.8802433013916,
      "activations/layer2_attention_weight_max": 26.15643882751465,
      "activations/layer2_attention_weight_min": -21.23670196533203,
      "activations/layer3_attention_weight_max": 52.7123908996582,
      "activations/layer3_attention_weight_min": -47.06839370727539,
      "activations/layer4_attention_weight_max": 84.9381103515625,
      "activations/layer4_attention_weight_min": -73.93405151367188,
      "activations/layer5_attention_weight_max": 143.13949584960938,
      "activations/layer5_attention_weight_min": -128.59019470214844,
      "activations/layer6_attention_weight_max": 77.12129211425781,
      "activations/layer6_attention_weight_min": -66.73870086669922,
      "activations/layer7_attention_weight_max": 77.56683349609375,
      "activations/layer7_attention_weight_min": -67.96800994873047,
      "activations/layer8_attention_weight_max": 66.74592590332031,
      "activations/layer8_attention_weight_min": -58.11626052856445,
      "activations/layer9_attention_weight_max": 73.44149017333984,
      "activations/layer9_attention_weight_min": -63.4036750793457,
      "epoch": 0.66,
      "learning_rate": 0.00014724431818181818,
      "loss": 3.2183,
      "step": 11300
    },
    {
      "activations/layer0_attention_weight_max": 13.164685249328613,
      "activations/layer0_attention_weight_min": -13.54292106628418,
      "activations/layer10_attention_weight_max": 61.48849868774414,
      "activations/layer10_attention_weight_min": -58.517852783203125,
      "activations/layer11_attention_weight_max": 40.4394645690918,
      "activations/layer11_attention_weight_min": -38.07063674926758,
      "activations/layer12_attention_weight_max": 26.54096031188965,
      "activations/layer12_attention_weight_min": -24.407867431640625,
      "activations/layer13_attention_weight_max": 46.76530075073242,
      "activations/layer13_attention_weight_min": -41.03985595703125,
      "activations/layer14_attention_weight_max": 44.66548156738281,
      "activations/layer14_attention_weight_min": -36.32879638671875,
      "activations/layer15_attention_weight_max": 45.84783935546875,
      "activations/layer15_attention_weight_min": -33.80682373046875,
      "activations/layer16_attention_weight_max": 39.842220306396484,
      "activations/layer16_attention_weight_min": -30.89684295654297,
      "activations/layer17_attention_weight_max": 54.042877197265625,
      "activations/layer17_attention_weight_min": -36.96016311645508,
      "activations/layer18_attention_weight_max": 48.785037994384766,
      "activations/layer18_attention_weight_min": -26.56675148010254,
      "activations/layer19_attention_weight_max": 21.468185424804688,
      "activations/layer19_attention_weight_min": -11.764616012573242,
      "activations/layer1_attention_weight_max": 16.95455551147461,
      "activations/layer1_attention_weight_min": -10.721389770507812,
      "activations/layer20_attention_weight_max": 21.163349151611328,
      "activations/layer20_attention_weight_min": -11.44193172454834,
      "activations/layer21_attention_weight_max": 32.26825714111328,
      "activations/layer21_attention_weight_min": -16.908226013183594,
      "activations/layer22_attention_weight_max": 29.150192260742188,
      "activations/layer22_attention_weight_min": -13.135764122009277,
      "activations/layer23_attention_weight_max": 21.504581451416016,
      "activations/layer23_attention_weight_min": -20.874393463134766,
      "activations/layer2_attention_weight_max": 24.517776489257812,
      "activations/layer2_attention_weight_min": -21.302824020385742,
      "activations/layer3_attention_weight_max": 51.25038528442383,
      "activations/layer3_attention_weight_min": -48.372779846191406,
      "activations/layer4_attention_weight_max": 81.46583557128906,
      "activations/layer4_attention_weight_min": -74.84834289550781,
      "activations/layer5_attention_weight_max": 138.11270141601562,
      "activations/layer5_attention_weight_min": -122.22747039794922,
      "activations/layer6_attention_weight_max": 68.41986083984375,
      "activations/layer6_attention_weight_min": -62.670143127441406,
      "activations/layer7_attention_weight_max": 71.29901885986328,
      "activations/layer7_attention_weight_min": -63.65046691894531,
      "activations/layer8_attention_weight_max": 65.58380889892578,
      "activations/layer8_attention_weight_min": -56.94424819946289,
      "activations/layer9_attention_weight_max": 67.38634490966797,
      "activations/layer9_attention_weight_min": -61.466163635253906,
      "epoch": 0.66,
      "learning_rate": 0.00014722537878787876,
      "loss": 3.2021,
      "step": 11350
    },
    {
      "activations/layer0_attention_weight_max": 14.008813858032227,
      "activations/layer0_attention_weight_min": -14.140948295593262,
      "activations/layer10_attention_weight_max": 64.93913269042969,
      "activations/layer10_attention_weight_min": -60.131591796875,
      "activations/layer11_attention_weight_max": 41.202232360839844,
      "activations/layer11_attention_weight_min": -39.58985137939453,
      "activations/layer12_attention_weight_max": 26.673242568969727,
      "activations/layer12_attention_weight_min": -23.15114974975586,
      "activations/layer13_attention_weight_max": 62.215431213378906,
      "activations/layer13_attention_weight_min": -45.794029235839844,
      "activations/layer14_attention_weight_max": 57.62115478515625,
      "activations/layer14_attention_weight_min": -40.61310577392578,
      "activations/layer15_attention_weight_max": 45.94624710083008,
      "activations/layer15_attention_weight_min": -31.354413986206055,
      "activations/layer16_attention_weight_max": 47.219757080078125,
      "activations/layer16_attention_weight_min": -32.72148895263672,
      "activations/layer17_attention_weight_max": 59.40670394897461,
      "activations/layer17_attention_weight_min": -34.56793975830078,
      "activations/layer18_attention_weight_max": 60.55541229248047,
      "activations/layer18_attention_weight_min": -40.75782012939453,
      "activations/layer19_attention_weight_max": 20.079875946044922,
      "activations/layer19_attention_weight_min": -11.299049377441406,
      "activations/layer1_attention_weight_max": 17.539905548095703,
      "activations/layer1_attention_weight_min": -10.787975311279297,
      "activations/layer20_attention_weight_max": 20.958032608032227,
      "activations/layer20_attention_weight_min": -11.831210136413574,
      "activations/layer21_attention_weight_max": 35.903472900390625,
      "activations/layer21_attention_weight_min": -15.908214569091797,
      "activations/layer22_attention_weight_max": 31.85643196105957,
      "activations/layer22_attention_weight_min": -12.58707332611084,
      "activations/layer23_attention_weight_max": 23.10628890991211,
      "activations/layer23_attention_weight_min": -21.859882354736328,
      "activations/layer2_attention_weight_max": 25.595746994018555,
      "activations/layer2_attention_weight_min": -21.35116195678711,
      "activations/layer3_attention_weight_max": 53.14058303833008,
      "activations/layer3_attention_weight_min": -48.4052734375,
      "activations/layer4_attention_weight_max": 81.95108795166016,
      "activations/layer4_attention_weight_min": -74.3062744140625,
      "activations/layer5_attention_weight_max": 133.97219848632812,
      "activations/layer5_attention_weight_min": -123.48556518554688,
      "activations/layer6_attention_weight_max": 69.14682006835938,
      "activations/layer6_attention_weight_min": -63.26780700683594,
      "activations/layer7_attention_weight_max": 69.16851043701172,
      "activations/layer7_attention_weight_min": -63.64849090576172,
      "activations/layer8_attention_weight_max": 57.84319305419922,
      "activations/layer8_attention_weight_min": -53.612770080566406,
      "activations/layer9_attention_weight_max": 70.54088592529297,
      "activations/layer9_attention_weight_min": -64.80963897705078,
      "epoch": 0.66,
      "learning_rate": 0.00014720643939393938,
      "loss": 3.2068,
      "step": 11400
    },
    {
      "activations/layer0_attention_weight_max": 13.570040702819824,
      "activations/layer0_attention_weight_min": -14.29306411743164,
      "activations/layer10_attention_weight_max": 66.44181060791016,
      "activations/layer10_attention_weight_min": -66.05107879638672,
      "activations/layer11_attention_weight_max": 46.821983337402344,
      "activations/layer11_attention_weight_min": -43.88801574707031,
      "activations/layer12_attention_weight_max": 23.567380905151367,
      "activations/layer12_attention_weight_min": -23.407882690429688,
      "activations/layer13_attention_weight_max": 55.06841278076172,
      "activations/layer13_attention_weight_min": -44.46165466308594,
      "activations/layer14_attention_weight_max": 42.072147369384766,
      "activations/layer14_attention_weight_min": -34.28036880493164,
      "activations/layer15_attention_weight_max": 40.56391906738281,
      "activations/layer15_attention_weight_min": -32.863555908203125,
      "activations/layer16_attention_weight_max": 37.67939376831055,
      "activations/layer16_attention_weight_min": -31.625370025634766,
      "activations/layer17_attention_weight_max": 52.26865005493164,
      "activations/layer17_attention_weight_min": -37.897186279296875,
      "activations/layer18_attention_weight_max": 51.60582733154297,
      "activations/layer18_attention_weight_min": -33.8480339050293,
      "activations/layer19_attention_weight_max": 19.2353572845459,
      "activations/layer19_attention_weight_min": -10.838301658630371,
      "activations/layer1_attention_weight_max": 16.416227340698242,
      "activations/layer1_attention_weight_min": -10.700501441955566,
      "activations/layer20_attention_weight_max": 18.15475845336914,
      "activations/layer20_attention_weight_min": -12.976812362670898,
      "activations/layer21_attention_weight_max": 29.388504028320312,
      "activations/layer21_attention_weight_min": -15.514144897460938,
      "activations/layer22_attention_weight_max": 27.74788475036621,
      "activations/layer22_attention_weight_min": -13.25328254699707,
      "activations/layer23_attention_weight_max": 20.799118041992188,
      "activations/layer23_attention_weight_min": -21.346965789794922,
      "activations/layer2_attention_weight_max": 26.899168014526367,
      "activations/layer2_attention_weight_min": -21.9776611328125,
      "activations/layer3_attention_weight_max": 52.04615783691406,
      "activations/layer3_attention_weight_min": -47.89253616333008,
      "activations/layer4_attention_weight_max": 84.2845458984375,
      "activations/layer4_attention_weight_min": -74.26773071289062,
      "activations/layer5_attention_weight_max": 146.59127807617188,
      "activations/layer5_attention_weight_min": -122.49626922607422,
      "activations/layer6_attention_weight_max": 73.65155029296875,
      "activations/layer6_attention_weight_min": -62.68336868286133,
      "activations/layer7_attention_weight_max": 69.40333557128906,
      "activations/layer7_attention_weight_min": -66.31431579589844,
      "activations/layer8_attention_weight_max": 57.21200180053711,
      "activations/layer8_attention_weight_min": -53.32685852050781,
      "activations/layer9_attention_weight_max": 68.36775970458984,
      "activations/layer9_attention_weight_min": -64.72158813476562,
      "epoch": 0.67,
      "learning_rate": 0.0001471875,
      "loss": 3.2076,
      "step": 11450
    },
    {
      "activations/layer0_attention_weight_max": 12.777819633483887,
      "activations/layer0_attention_weight_min": -13.348784446716309,
      "activations/layer10_attention_weight_max": 70.37675476074219,
      "activations/layer10_attention_weight_min": -60.64109802246094,
      "activations/layer11_attention_weight_max": 45.81867599487305,
      "activations/layer11_attention_weight_min": -40.191341400146484,
      "activations/layer12_attention_weight_max": 31.77125358581543,
      "activations/layer12_attention_weight_min": -24.670974731445312,
      "activations/layer13_attention_weight_max": 50.76691818237305,
      "activations/layer13_attention_weight_min": -36.0450325012207,
      "activations/layer14_attention_weight_max": 46.227874755859375,
      "activations/layer14_attention_weight_min": -31.138521194458008,
      "activations/layer15_attention_weight_max": 51.26919174194336,
      "activations/layer15_attention_weight_min": -37.97687530517578,
      "activations/layer16_attention_weight_max": 43.491661071777344,
      "activations/layer16_attention_weight_min": -33.19124221801758,
      "activations/layer17_attention_weight_max": 58.37826919555664,
      "activations/layer17_attention_weight_min": -36.8068733215332,
      "activations/layer18_attention_weight_max": 46.09400177001953,
      "activations/layer18_attention_weight_min": -29.736003875732422,
      "activations/layer19_attention_weight_max": 21.047842025756836,
      "activations/layer19_attention_weight_min": -11.698521614074707,
      "activations/layer1_attention_weight_max": 17.496183395385742,
      "activations/layer1_attention_weight_min": -11.015155792236328,
      "activations/layer20_attention_weight_max": 15.903322219848633,
      "activations/layer20_attention_weight_min": -12.52375602722168,
      "activations/layer21_attention_weight_max": 28.903966903686523,
      "activations/layer21_attention_weight_min": -16.635061264038086,
      "activations/layer22_attention_weight_max": 23.705097198486328,
      "activations/layer22_attention_weight_min": -15.317973136901855,
      "activations/layer23_attention_weight_max": 20.17147445678711,
      "activations/layer23_attention_weight_min": -19.79824447631836,
      "activations/layer2_attention_weight_max": 26.58993148803711,
      "activations/layer2_attention_weight_min": -23.03009796142578,
      "activations/layer3_attention_weight_max": 47.32338333129883,
      "activations/layer3_attention_weight_min": -44.822120666503906,
      "activations/layer4_attention_weight_max": 77.0335922241211,
      "activations/layer4_attention_weight_min": -71.01435852050781,
      "activations/layer5_attention_weight_max": 135.7674102783203,
      "activations/layer5_attention_weight_min": -120.94239807128906,
      "activations/layer6_attention_weight_max": 69.56875610351562,
      "activations/layer6_attention_weight_min": -61.34008026123047,
      "activations/layer7_attention_weight_max": 70.52611541748047,
      "activations/layer7_attention_weight_min": -63.78396987915039,
      "activations/layer8_attention_weight_max": 64.98894500732422,
      "activations/layer8_attention_weight_min": -60.64133834838867,
      "activations/layer9_attention_weight_max": 66.91094207763672,
      "activations/layer9_attention_weight_min": -60.563961029052734,
      "epoch": 0.67,
      "learning_rate": 0.00014716856060606058,
      "loss": 3.1785,
      "step": 11500
    },
    {
      "activations/layer0_attention_weight_max": 14.930190086364746,
      "activations/layer0_attention_weight_min": -13.511062622070312,
      "activations/layer10_attention_weight_max": 64.13215637207031,
      "activations/layer10_attention_weight_min": -60.15482711791992,
      "activations/layer11_attention_weight_max": 45.41367721557617,
      "activations/layer11_attention_weight_min": -45.43719482421875,
      "activations/layer12_attention_weight_max": 24.817832946777344,
      "activations/layer12_attention_weight_min": -22.96622657775879,
      "activations/layer13_attention_weight_max": 68.6249008178711,
      "activations/layer13_attention_weight_min": -44.355220794677734,
      "activations/layer14_attention_weight_max": 54.54914474487305,
      "activations/layer14_attention_weight_min": -34.09464645385742,
      "activations/layer15_attention_weight_max": 46.56544494628906,
      "activations/layer15_attention_weight_min": -32.22297668457031,
      "activations/layer16_attention_weight_max": 44.17727279663086,
      "activations/layer16_attention_weight_min": -30.98027801513672,
      "activations/layer17_attention_weight_max": 58.45790100097656,
      "activations/layer17_attention_weight_min": -36.55329895019531,
      "activations/layer18_attention_weight_max": 54.396427154541016,
      "activations/layer18_attention_weight_min": -33.85596466064453,
      "activations/layer19_attention_weight_max": 17.718801498413086,
      "activations/layer19_attention_weight_min": -12.788605690002441,
      "activations/layer1_attention_weight_max": 18.052845001220703,
      "activations/layer1_attention_weight_min": -11.152457237243652,
      "activations/layer20_attention_weight_max": 16.380868911743164,
      "activations/layer20_attention_weight_min": -11.591226577758789,
      "activations/layer21_attention_weight_max": 27.912145614624023,
      "activations/layer21_attention_weight_min": -16.95343017578125,
      "activations/layer22_attention_weight_max": 23.880779266357422,
      "activations/layer22_attention_weight_min": -14.102636337280273,
      "activations/layer23_attention_weight_max": 19.925724029541016,
      "activations/layer23_attention_weight_min": -20.828201293945312,
      "activations/layer2_attention_weight_max": 26.001325607299805,
      "activations/layer2_attention_weight_min": -21.452533721923828,
      "activations/layer3_attention_weight_max": 51.438621520996094,
      "activations/layer3_attention_weight_min": -47.79996109008789,
      "activations/layer4_attention_weight_max": 82.92340850830078,
      "activations/layer4_attention_weight_min": -73.38600158691406,
      "activations/layer5_attention_weight_max": 143.21066284179688,
      "activations/layer5_attention_weight_min": -125.6521224975586,
      "activations/layer6_attention_weight_max": 72.7180404663086,
      "activations/layer6_attention_weight_min": -65.39159393310547,
      "activations/layer7_attention_weight_max": 69.83641052246094,
      "activations/layer7_attention_weight_min": -66.31230926513672,
      "activations/layer8_attention_weight_max": 57.29294967651367,
      "activations/layer8_attention_weight_min": -54.56588363647461,
      "activations/layer9_attention_weight_max": 71.14522552490234,
      "activations/layer9_attention_weight_min": -64.10297393798828,
      "epoch": 0.67,
      "learning_rate": 0.00014714999999999997,
      "loss": 3.1776,
      "step": 11550
    },
    {
      "activations/layer0_attention_weight_max": 13.731880187988281,
      "activations/layer0_attention_weight_min": -13.389363288879395,
      "activations/layer10_attention_weight_max": 65.6691665649414,
      "activations/layer10_attention_weight_min": -57.65782928466797,
      "activations/layer11_attention_weight_max": 40.68018341064453,
      "activations/layer11_attention_weight_min": -42.17914962768555,
      "activations/layer12_attention_weight_max": 23.3430118560791,
      "activations/layer12_attention_weight_min": -23.18271827697754,
      "activations/layer13_attention_weight_max": 54.59319305419922,
      "activations/layer13_attention_weight_min": -41.18013000488281,
      "activations/layer14_attention_weight_max": 45.02488327026367,
      "activations/layer14_attention_weight_min": -30.4209041595459,
      "activations/layer15_attention_weight_max": 45.703636169433594,
      "activations/layer15_attention_weight_min": -36.85533905029297,
      "activations/layer16_attention_weight_max": 51.275665283203125,
      "activations/layer16_attention_weight_min": -35.73868179321289,
      "activations/layer17_attention_weight_max": 62.80055618286133,
      "activations/layer17_attention_weight_min": -41.14224624633789,
      "activations/layer18_attention_weight_max": 58.81311798095703,
      "activations/layer18_attention_weight_min": -36.33763885498047,
      "activations/layer19_attention_weight_max": 20.901042938232422,
      "activations/layer19_attention_weight_min": -11.236586570739746,
      "activations/layer1_attention_weight_max": 17.097318649291992,
      "activations/layer1_attention_weight_min": -10.962173461914062,
      "activations/layer20_attention_weight_max": 18.921480178833008,
      "activations/layer20_attention_weight_min": -15.430561065673828,
      "activations/layer21_attention_weight_max": 32.88795852661133,
      "activations/layer21_attention_weight_min": -16.886425018310547,
      "activations/layer22_attention_weight_max": 31.26075553894043,
      "activations/layer22_attention_weight_min": -16.254568099975586,
      "activations/layer23_attention_weight_max": 20.514293670654297,
      "activations/layer23_attention_weight_min": -20.340328216552734,
      "activations/layer2_attention_weight_max": 25.53292465209961,
      "activations/layer2_attention_weight_min": -21.586780548095703,
      "activations/layer3_attention_weight_max": 50.81520462036133,
      "activations/layer3_attention_weight_min": -47.00044250488281,
      "activations/layer4_attention_weight_max": 79.52208709716797,
      "activations/layer4_attention_weight_min": -71.73351287841797,
      "activations/layer5_attention_weight_max": 137.6611328125,
      "activations/layer5_attention_weight_min": -122.7296142578125,
      "activations/layer6_attention_weight_max": 69.95694732666016,
      "activations/layer6_attention_weight_min": -59.93938446044922,
      "activations/layer7_attention_weight_max": 69.59725952148438,
      "activations/layer7_attention_weight_min": -62.510948181152344,
      "activations/layer8_attention_weight_max": 57.38703155517578,
      "activations/layer8_attention_weight_min": -54.47309875488281,
      "activations/layer9_attention_weight_max": 67.40991973876953,
      "activations/layer9_attention_weight_min": -61.529510498046875,
      "epoch": 0.67,
      "learning_rate": 0.0001471310606060606,
      "loss": 3.1915,
      "step": 11600
    },
    {
      "activations/layer0_attention_weight_max": 13.005638122558594,
      "activations/layer0_attention_weight_min": -12.70446491241455,
      "activations/layer10_attention_weight_max": 66.07567596435547,
      "activations/layer10_attention_weight_min": -58.89936065673828,
      "activations/layer11_attention_weight_max": 46.050331115722656,
      "activations/layer11_attention_weight_min": -38.48267364501953,
      "activations/layer12_attention_weight_max": 30.097997665405273,
      "activations/layer12_attention_weight_min": -24.19791030883789,
      "activations/layer13_attention_weight_max": 59.35568618774414,
      "activations/layer13_attention_weight_min": -41.97222137451172,
      "activations/layer14_attention_weight_max": 42.03755187988281,
      "activations/layer14_attention_weight_min": -35.1373176574707,
      "activations/layer15_attention_weight_max": 50.75901794433594,
      "activations/layer15_attention_weight_min": -34.253700256347656,
      "activations/layer16_attention_weight_max": 50.190128326416016,
      "activations/layer16_attention_weight_min": -32.681419372558594,
      "activations/layer17_attention_weight_max": 54.97195053100586,
      "activations/layer17_attention_weight_min": -34.99116897583008,
      "activations/layer18_attention_weight_max": 60.11241912841797,
      "activations/layer18_attention_weight_min": -36.276611328125,
      "activations/layer19_attention_weight_max": 19.184415817260742,
      "activations/layer19_attention_weight_min": -11.346396446228027,
      "activations/layer1_attention_weight_max": 16.632112503051758,
      "activations/layer1_attention_weight_min": -11.266510963439941,
      "activations/layer20_attention_weight_max": 17.17107391357422,
      "activations/layer20_attention_weight_min": -12.269972801208496,
      "activations/layer21_attention_weight_max": 33.80012893676758,
      "activations/layer21_attention_weight_min": -16.677635192871094,
      "activations/layer22_attention_weight_max": 33.575592041015625,
      "activations/layer22_attention_weight_min": -16.580591201782227,
      "activations/layer23_attention_weight_max": 23.683698654174805,
      "activations/layer23_attention_weight_min": -20.683326721191406,
      "activations/layer2_attention_weight_max": 28.23810577392578,
      "activations/layer2_attention_weight_min": -22.73773765563965,
      "activations/layer3_attention_weight_max": 54.01131057739258,
      "activations/layer3_attention_weight_min": -48.521793365478516,
      "activations/layer4_attention_weight_max": 83.37347412109375,
      "activations/layer4_attention_weight_min": -74.07975769042969,
      "activations/layer5_attention_weight_max": 142.65402221679688,
      "activations/layer5_attention_weight_min": -127.730712890625,
      "activations/layer6_attention_weight_max": 71.07496643066406,
      "activations/layer6_attention_weight_min": -62.466644287109375,
      "activations/layer7_attention_weight_max": 72.1027603149414,
      "activations/layer7_attention_weight_min": -61.1346549987793,
      "activations/layer8_attention_weight_max": 63.48372268676758,
      "activations/layer8_attention_weight_min": -51.90272521972656,
      "activations/layer9_attention_weight_max": 68.15145111083984,
      "activations/layer9_attention_weight_min": -68.07247161865234,
      "epoch": 0.68,
      "learning_rate": 0.0001471121212121212,
      "loss": 3.1883,
      "step": 11650
    },
    {
      "activations/layer0_attention_weight_max": 13.155206680297852,
      "activations/layer0_attention_weight_min": -12.264871597290039,
      "activations/layer10_attention_weight_max": 74.67684936523438,
      "activations/layer10_attention_weight_min": -65.37501525878906,
      "activations/layer11_attention_weight_max": 45.43603515625,
      "activations/layer11_attention_weight_min": -38.161495208740234,
      "activations/layer12_attention_weight_max": 25.295547485351562,
      "activations/layer12_attention_weight_min": -24.30921173095703,
      "activations/layer13_attention_weight_max": 51.45607376098633,
      "activations/layer13_attention_weight_min": -37.068870544433594,
      "activations/layer14_attention_weight_max": 48.90531539916992,
      "activations/layer14_attention_weight_min": -40.30146408081055,
      "activations/layer15_attention_weight_max": 48.37976837158203,
      "activations/layer15_attention_weight_min": -33.95962905883789,
      "activations/layer16_attention_weight_max": 51.76359939575195,
      "activations/layer16_attention_weight_min": -37.49372482299805,
      "activations/layer17_attention_weight_max": 61.70100402832031,
      "activations/layer17_attention_weight_min": -40.080257415771484,
      "activations/layer18_attention_weight_max": 61.82093048095703,
      "activations/layer18_attention_weight_min": -35.269775390625,
      "activations/layer19_attention_weight_max": 23.4637508392334,
      "activations/layer19_attention_weight_min": -11.301129341125488,
      "activations/layer1_attention_weight_max": 15.772721290588379,
      "activations/layer1_attention_weight_min": -10.901771545410156,
      "activations/layer20_attention_weight_max": 23.896080017089844,
      "activations/layer20_attention_weight_min": -12.720447540283203,
      "activations/layer21_attention_weight_max": 30.438810348510742,
      "activations/layer21_attention_weight_min": -16.438236236572266,
      "activations/layer22_attention_weight_max": 30.289302825927734,
      "activations/layer22_attention_weight_min": -15.470541954040527,
      "activations/layer23_attention_weight_max": 28.16933822631836,
      "activations/layer23_attention_weight_min": -19.486576080322266,
      "activations/layer2_attention_weight_max": 25.494977951049805,
      "activations/layer2_attention_weight_min": -21.687009811401367,
      "activations/layer3_attention_weight_max": 51.20814514160156,
      "activations/layer3_attention_weight_min": -47.20662307739258,
      "activations/layer4_attention_weight_max": 84.0674057006836,
      "activations/layer4_attention_weight_min": -71.15165710449219,
      "activations/layer5_attention_weight_max": 142.7149658203125,
      "activations/layer5_attention_weight_min": -124.27684020996094,
      "activations/layer6_attention_weight_max": 71.46072387695312,
      "activations/layer6_attention_weight_min": -61.853004455566406,
      "activations/layer7_attention_weight_max": 72.48934936523438,
      "activations/layer7_attention_weight_min": -60.78321838378906,
      "activations/layer8_attention_weight_max": 61.0932502746582,
      "activations/layer8_attention_weight_min": -54.790977478027344,
      "activations/layer9_attention_weight_max": 68.52877044677734,
      "activations/layer9_attention_weight_min": -65.1364974975586,
      "epoch": 0.68,
      "learning_rate": 0.00014709318181818182,
      "loss": 3.1842,
      "step": 11700
    },
    {
      "activations/layer0_attention_weight_max": 13.99359130859375,
      "activations/layer0_attention_weight_min": -12.429255485534668,
      "activations/layer10_attention_weight_max": 60.04047775268555,
      "activations/layer10_attention_weight_min": -55.17084884643555,
      "activations/layer11_attention_weight_max": 40.485374450683594,
      "activations/layer11_attention_weight_min": -42.669063568115234,
      "activations/layer12_attention_weight_max": 22.910062789916992,
      "activations/layer12_attention_weight_min": -23.520660400390625,
      "activations/layer13_attention_weight_max": 48.338035583496094,
      "activations/layer13_attention_weight_min": -41.81974411010742,
      "activations/layer14_attention_weight_max": 38.823829650878906,
      "activations/layer14_attention_weight_min": -34.38188552856445,
      "activations/layer15_attention_weight_max": 42.693965911865234,
      "activations/layer15_attention_weight_min": -33.285911560058594,
      "activations/layer16_attention_weight_max": 44.1025276184082,
      "activations/layer16_attention_weight_min": -30.76342010498047,
      "activations/layer17_attention_weight_max": 54.958858489990234,
      "activations/layer17_attention_weight_min": -37.44831848144531,
      "activations/layer18_attention_weight_max": 41.596553802490234,
      "activations/layer18_attention_weight_min": -25.22968101501465,
      "activations/layer19_attention_weight_max": 20.10675621032715,
      "activations/layer19_attention_weight_min": -13.1274995803833,
      "activations/layer1_attention_weight_max": 17.621387481689453,
      "activations/layer1_attention_weight_min": -10.887693405151367,
      "activations/layer20_attention_weight_max": 19.672414779663086,
      "activations/layer20_attention_weight_min": -13.005651473999023,
      "activations/layer21_attention_weight_max": 34.901954650878906,
      "activations/layer21_attention_weight_min": -15.518416404724121,
      "activations/layer22_attention_weight_max": 32.53828430175781,
      "activations/layer22_attention_weight_min": -15.533426284790039,
      "activations/layer23_attention_weight_max": 26.11920166015625,
      "activations/layer23_attention_weight_min": -20.67479133605957,
      "activations/layer2_attention_weight_max": 26.844234466552734,
      "activations/layer2_attention_weight_min": -21.142566680908203,
      "activations/layer3_attention_weight_max": 50.673431396484375,
      "activations/layer3_attention_weight_min": -46.93844985961914,
      "activations/layer4_attention_weight_max": 79.80245208740234,
      "activations/layer4_attention_weight_min": -71.65670776367188,
      "activations/layer5_attention_weight_max": 135.7641143798828,
      "activations/layer5_attention_weight_min": -119.81979370117188,
      "activations/layer6_attention_weight_max": 66.0890121459961,
      "activations/layer6_attention_weight_min": -60.69123840332031,
      "activations/layer7_attention_weight_max": 64.4918441772461,
      "activations/layer7_attention_weight_min": -61.80473327636719,
      "activations/layer8_attention_weight_max": 54.411895751953125,
      "activations/layer8_attention_weight_min": -53.308448791503906,
      "activations/layer9_attention_weight_max": 65.2059326171875,
      "activations/layer9_attention_weight_min": -62.85052490234375,
      "epoch": 0.68,
      "learning_rate": 0.0001470742424242424,
      "loss": 3.1859,
      "step": 11750
    },
    {
      "activations/layer0_attention_weight_max": 13.327422142028809,
      "activations/layer0_attention_weight_min": -13.500067710876465,
      "activations/layer10_attention_weight_max": 71.98230743408203,
      "activations/layer10_attention_weight_min": -61.846519470214844,
      "activations/layer11_attention_weight_max": 48.68609619140625,
      "activations/layer11_attention_weight_min": -41.08428955078125,
      "activations/layer12_attention_weight_max": 25.24237060546875,
      "activations/layer12_attention_weight_min": -23.181333541870117,
      "activations/layer13_attention_weight_max": 53.803531646728516,
      "activations/layer13_attention_weight_min": -40.784881591796875,
      "activations/layer14_attention_weight_max": 50.837772369384766,
      "activations/layer14_attention_weight_min": -36.421138763427734,
      "activations/layer15_attention_weight_max": 53.72165298461914,
      "activations/layer15_attention_weight_min": -36.39851760864258,
      "activations/layer16_attention_weight_max": 49.7068977355957,
      "activations/layer16_attention_weight_min": -31.845727920532227,
      "activations/layer17_attention_weight_max": 61.12541961669922,
      "activations/layer17_attention_weight_min": -37.08031463623047,
      "activations/layer18_attention_weight_max": 73.5765151977539,
      "activations/layer18_attention_weight_min": -37.618621826171875,
      "activations/layer19_attention_weight_max": 23.78803825378418,
      "activations/layer19_attention_weight_min": -12.423637390136719,
      "activations/layer1_attention_weight_max": 16.840240478515625,
      "activations/layer1_attention_weight_min": -10.883895874023438,
      "activations/layer20_attention_weight_max": 19.053409576416016,
      "activations/layer20_attention_weight_min": -13.582983016967773,
      "activations/layer21_attention_weight_max": 38.60991287231445,
      "activations/layer21_attention_weight_min": -15.668047904968262,
      "activations/layer22_attention_weight_max": 34.37098693847656,
      "activations/layer22_attention_weight_min": -14.698821067810059,
      "activations/layer23_attention_weight_max": 25.40988540649414,
      "activations/layer23_attention_weight_min": -21.429828643798828,
      "activations/layer2_attention_weight_max": 25.140777587890625,
      "activations/layer2_attention_weight_min": -22.269229888916016,
      "activations/layer3_attention_weight_max": 51.59469985961914,
      "activations/layer3_attention_weight_min": -49.238258361816406,
      "activations/layer4_attention_weight_max": 78.49911499023438,
      "activations/layer4_attention_weight_min": -74.86494445800781,
      "activations/layer5_attention_weight_max": 137.80133056640625,
      "activations/layer5_attention_weight_min": -123.71520233154297,
      "activations/layer6_attention_weight_max": 69.03724670410156,
      "activations/layer6_attention_weight_min": -62.02362060546875,
      "activations/layer7_attention_weight_max": 72.85562896728516,
      "activations/layer7_attention_weight_min": -61.37356948852539,
      "activations/layer8_attention_weight_max": 61.7099609375,
      "activations/layer8_attention_weight_min": -55.9849853515625,
      "activations/layer9_attention_weight_max": 71.0671615600586,
      "activations/layer9_attention_weight_min": -65.5122299194336,
      "epoch": 0.69,
      "learning_rate": 0.00014705530303030302,
      "loss": 3.2052,
      "step": 11800
    },
    {
      "activations/layer0_attention_weight_max": 13.68764877319336,
      "activations/layer0_attention_weight_min": -12.878241539001465,
      "activations/layer10_attention_weight_max": 72.78336334228516,
      "activations/layer10_attention_weight_min": -62.33099365234375,
      "activations/layer11_attention_weight_max": 52.3698844909668,
      "activations/layer11_attention_weight_min": -43.12360382080078,
      "activations/layer12_attention_weight_max": 29.66925621032715,
      "activations/layer12_attention_weight_min": -24.20309066772461,
      "activations/layer13_attention_weight_max": 49.62717819213867,
      "activations/layer13_attention_weight_min": -39.39903259277344,
      "activations/layer14_attention_weight_max": 36.955482482910156,
      "activations/layer14_attention_weight_min": -32.290679931640625,
      "activations/layer15_attention_weight_max": 48.96943283081055,
      "activations/layer15_attention_weight_min": -40.61175537109375,
      "activations/layer16_attention_weight_max": 42.411643981933594,
      "activations/layer16_attention_weight_min": -31.724853515625,
      "activations/layer17_attention_weight_max": 51.965599060058594,
      "activations/layer17_attention_weight_min": -40.029327392578125,
      "activations/layer18_attention_weight_max": 51.78392791748047,
      "activations/layer18_attention_weight_min": -35.2359619140625,
      "activations/layer19_attention_weight_max": 20.351423263549805,
      "activations/layer19_attention_weight_min": -11.510931015014648,
      "activations/layer1_attention_weight_max": 16.63658905029297,
      "activations/layer1_attention_weight_min": -11.471386909484863,
      "activations/layer20_attention_weight_max": 17.159406661987305,
      "activations/layer20_attention_weight_min": -11.57589054107666,
      "activations/layer21_attention_weight_max": 29.701200485229492,
      "activations/layer21_attention_weight_min": -13.434670448303223,
      "activations/layer22_attention_weight_max": 28.74716567993164,
      "activations/layer22_attention_weight_min": -12.67608642578125,
      "activations/layer23_attention_weight_max": 21.212005615234375,
      "activations/layer23_attention_weight_min": -20.5363826751709,
      "activations/layer2_attention_weight_max": 26.829303741455078,
      "activations/layer2_attention_weight_min": -21.57473373413086,
      "activations/layer3_attention_weight_max": 51.03373718261719,
      "activations/layer3_attention_weight_min": -46.985870361328125,
      "activations/layer4_attention_weight_max": 78.89744567871094,
      "activations/layer4_attention_weight_min": -73.30297088623047,
      "activations/layer5_attention_weight_max": 142.2194061279297,
      "activations/layer5_attention_weight_min": -123.8631820678711,
      "activations/layer6_attention_weight_max": 72.7801284790039,
      "activations/layer6_attention_weight_min": -62.69013595581055,
      "activations/layer7_attention_weight_max": 71.69229888916016,
      "activations/layer7_attention_weight_min": -67.40122985839844,
      "activations/layer8_attention_weight_max": 62.12154769897461,
      "activations/layer8_attention_weight_min": -55.76701354980469,
      "activations/layer9_attention_weight_max": 72.21653747558594,
      "activations/layer9_attention_weight_min": -60.96501541137695,
      "epoch": 0.69,
      "learning_rate": 0.00014703636363636363,
      "loss": 3.1836,
      "step": 11850
    },
    {
      "activations/layer0_attention_weight_max": 13.115485191345215,
      "activations/layer0_attention_weight_min": -12.506465911865234,
      "activations/layer10_attention_weight_max": 68.90068817138672,
      "activations/layer10_attention_weight_min": -60.16111755371094,
      "activations/layer11_attention_weight_max": 48.0774040222168,
      "activations/layer11_attention_weight_min": -38.48554229736328,
      "activations/layer12_attention_weight_max": 30.47583770751953,
      "activations/layer12_attention_weight_min": -25.020248413085938,
      "activations/layer13_attention_weight_max": 69.44352722167969,
      "activations/layer13_attention_weight_min": -50.09865951538086,
      "activations/layer14_attention_weight_max": 61.360382080078125,
      "activations/layer14_attention_weight_min": -36.467041015625,
      "activations/layer15_attention_weight_max": 50.238914489746094,
      "activations/layer15_attention_weight_min": -32.79795455932617,
      "activations/layer16_attention_weight_max": 45.657352447509766,
      "activations/layer16_attention_weight_min": -35.11651611328125,
      "activations/layer17_attention_weight_max": 62.46444320678711,
      "activations/layer17_attention_weight_min": -35.93986511230469,
      "activations/layer18_attention_weight_max": 52.59465408325195,
      "activations/layer18_attention_weight_min": -33.415443420410156,
      "activations/layer19_attention_weight_max": 20.523855209350586,
      "activations/layer19_attention_weight_min": -11.47678279876709,
      "activations/layer1_attention_weight_max": 15.276762962341309,
      "activations/layer1_attention_weight_min": -12.107157707214355,
      "activations/layer20_attention_weight_max": 19.47314453125,
      "activations/layer20_attention_weight_min": -12.297774314880371,
      "activations/layer21_attention_weight_max": 31.71770477294922,
      "activations/layer21_attention_weight_min": -14.08108139038086,
      "activations/layer22_attention_weight_max": 28.001964569091797,
      "activations/layer22_attention_weight_min": -15.211637496948242,
      "activations/layer23_attention_weight_max": 20.052261352539062,
      "activations/layer23_attention_weight_min": -20.720979690551758,
      "activations/layer2_attention_weight_max": 25.7371883392334,
      "activations/layer2_attention_weight_min": -21.65887451171875,
      "activations/layer3_attention_weight_max": 51.1205940246582,
      "activations/layer3_attention_weight_min": -46.30388259887695,
      "activations/layer4_attention_weight_max": 85.3726577758789,
      "activations/layer4_attention_weight_min": -72.08780670166016,
      "activations/layer5_attention_weight_max": 151.04345703125,
      "activations/layer5_attention_weight_min": -122.83434295654297,
      "activations/layer6_attention_weight_max": 73.84028625488281,
      "activations/layer6_attention_weight_min": -62.14960861206055,
      "activations/layer7_attention_weight_max": 76.74500274658203,
      "activations/layer7_attention_weight_min": -64.95024108886719,
      "activations/layer8_attention_weight_max": 65.53347778320312,
      "activations/layer8_attention_weight_min": -54.2736701965332,
      "activations/layer9_attention_weight_max": 76.73420715332031,
      "activations/layer9_attention_weight_min": -63.66335678100586,
      "epoch": 0.69,
      "learning_rate": 0.00014701742424242425,
      "loss": 3.1671,
      "step": 11900
    },
    {
      "activations/layer0_attention_weight_max": 14.849591255187988,
      "activations/layer0_attention_weight_min": -12.438507080078125,
      "activations/layer10_attention_weight_max": 72.24423217773438,
      "activations/layer10_attention_weight_min": -61.69400405883789,
      "activations/layer11_attention_weight_max": 51.38414001464844,
      "activations/layer11_attention_weight_min": -40.564979553222656,
      "activations/layer12_attention_weight_max": 31.07520866394043,
      "activations/layer12_attention_weight_min": -23.73269271850586,
      "activations/layer13_attention_weight_max": 61.37977600097656,
      "activations/layer13_attention_weight_min": -43.599910736083984,
      "activations/layer14_attention_weight_max": 47.623722076416016,
      "activations/layer14_attention_weight_min": -38.91688919067383,
      "activations/layer15_attention_weight_max": 52.636260986328125,
      "activations/layer15_attention_weight_min": -37.662132263183594,
      "activations/layer16_attention_weight_max": 49.75600051879883,
      "activations/layer16_attention_weight_min": -34.404541015625,
      "activations/layer17_attention_weight_max": 53.97865676879883,
      "activations/layer17_attention_weight_min": -40.22597122192383,
      "activations/layer18_attention_weight_max": 47.99429702758789,
      "activations/layer18_attention_weight_min": -32.02126693725586,
      "activations/layer19_attention_weight_max": 20.491926193237305,
      "activations/layer19_attention_weight_min": -13.069323539733887,
      "activations/layer1_attention_weight_max": 16.984588623046875,
      "activations/layer1_attention_weight_min": -11.40639591217041,
      "activations/layer20_attention_weight_max": 17.733177185058594,
      "activations/layer20_attention_weight_min": -13.510375022888184,
      "activations/layer21_attention_weight_max": 30.896060943603516,
      "activations/layer21_attention_weight_min": -16.038400650024414,
      "activations/layer22_attention_weight_max": 28.296772003173828,
      "activations/layer22_attention_weight_min": -15.067609786987305,
      "activations/layer23_attention_weight_max": 21.357837677001953,
      "activations/layer23_attention_weight_min": -20.539640426635742,
      "activations/layer2_attention_weight_max": 25.38878631591797,
      "activations/layer2_attention_weight_min": -20.843273162841797,
      "activations/layer3_attention_weight_max": 52.25169372558594,
      "activations/layer3_attention_weight_min": -46.55448913574219,
      "activations/layer4_attention_weight_max": 80.4037857055664,
      "activations/layer4_attention_weight_min": -70.24967956542969,
      "activations/layer5_attention_weight_max": 147.59799194335938,
      "activations/layer5_attention_weight_min": -121.64370727539062,
      "activations/layer6_attention_weight_max": 74.61640930175781,
      "activations/layer6_attention_weight_min": -61.37364959716797,
      "activations/layer7_attention_weight_max": 77.58899688720703,
      "activations/layer7_attention_weight_min": -65.4796142578125,
      "activations/layer8_attention_weight_max": 60.73479461669922,
      "activations/layer8_attention_weight_min": -53.93349838256836,
      "activations/layer9_attention_weight_max": 70.43501281738281,
      "activations/layer9_attention_weight_min": -63.31007766723633,
      "epoch": 0.69,
      "learning_rate": 0.00014699848484848484,
      "loss": 3.1777,
      "step": 11950
    },
    {
      "activations/layer0_attention_weight_max": 13.000192642211914,
      "activations/layer0_attention_weight_min": -12.735729217529297,
      "activations/layer10_attention_weight_max": 62.5621337890625,
      "activations/layer10_attention_weight_min": -58.90098190307617,
      "activations/layer11_attention_weight_max": 43.73997497558594,
      "activations/layer11_attention_weight_min": -38.02594757080078,
      "activations/layer12_attention_weight_max": 25.199106216430664,
      "activations/layer12_attention_weight_min": -22.589929580688477,
      "activations/layer13_attention_weight_max": 52.279537200927734,
      "activations/layer13_attention_weight_min": -39.19642639160156,
      "activations/layer14_attention_weight_max": 44.11760330200195,
      "activations/layer14_attention_weight_min": -38.42532730102539,
      "activations/layer15_attention_weight_max": 46.888938903808594,
      "activations/layer15_attention_weight_min": -32.96499252319336,
      "activations/layer16_attention_weight_max": 43.88518142700195,
      "activations/layer16_attention_weight_min": -33.353145599365234,
      "activations/layer17_attention_weight_max": 58.13953399658203,
      "activations/layer17_attention_weight_min": -36.94878387451172,
      "activations/layer18_attention_weight_max": 63.02155685424805,
      "activations/layer18_attention_weight_min": -32.882843017578125,
      "activations/layer19_attention_weight_max": 21.51287841796875,
      "activations/layer19_attention_weight_min": -11.943601608276367,
      "activations/layer1_attention_weight_max": 17.4952449798584,
      "activations/layer1_attention_weight_min": -11.84155559539795,
      "activations/layer20_attention_weight_max": 21.49226188659668,
      "activations/layer20_attention_weight_min": -12.781821250915527,
      "activations/layer21_attention_weight_max": 32.95814514160156,
      "activations/layer21_attention_weight_min": -15.40014934539795,
      "activations/layer22_attention_weight_max": 30.190778732299805,
      "activations/layer22_attention_weight_min": -13.769378662109375,
      "activations/layer23_attention_weight_max": 21.215862274169922,
      "activations/layer23_attention_weight_min": -21.69171142578125,
      "activations/layer2_attention_weight_max": 27.371875762939453,
      "activations/layer2_attention_weight_min": -23.927335739135742,
      "activations/layer3_attention_weight_max": 52.927188873291016,
      "activations/layer3_attention_weight_min": -48.6777458190918,
      "activations/layer4_attention_weight_max": 80.21810913085938,
      "activations/layer4_attention_weight_min": -71.8438720703125,
      "activations/layer5_attention_weight_max": 142.31910705566406,
      "activations/layer5_attention_weight_min": -121.80935668945312,
      "activations/layer6_attention_weight_max": 71.15727996826172,
      "activations/layer6_attention_weight_min": -62.74458694458008,
      "activations/layer7_attention_weight_max": 69.5103759765625,
      "activations/layer7_attention_weight_min": -61.272377014160156,
      "activations/layer8_attention_weight_max": 58.64315414428711,
      "activations/layer8_attention_weight_min": -52.15371322631836,
      "activations/layer9_attention_weight_max": 71.7125015258789,
      "activations/layer9_attention_weight_min": -63.526058197021484,
      "epoch": 0.7,
      "learning_rate": 0.00014697954545454545,
      "loss": 3.1945,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_loss": 3.130859375,
      "eval_runtime": 8.2648,
      "eval_samples_per_second": 519.552,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_openwebtext_loss": 3.130859375,
      "eval_openwebtext_ppl": 22.893645317537526,
      "eval_openwebtext_runtime": 8.2648,
      "eval_openwebtext_samples_per_second": 519.552,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_wikitext_loss": 3.4140625,
      "eval_wikitext_ppl": 30.388446900124176,
      "eval_wikitext_runtime": 1.7145,
      "eval_wikitext_samples_per_second": 265.96,
      "step": 12000
    },
    {
      "epoch": 0.7,
      "eval_lambada_loss": 3.732421875,
      "eval_lambada_ppl": 41.780172086809024,
      "eval_lambada_runtime": 9.2943,
      "eval_lambada_samples_per_second": 523.872,
      "step": 12000
    },
    {
      "activations/layer0_attention_weight_max": 13.58771800994873,
      "activations/layer0_attention_weight_min": -12.728348731994629,
      "activations/layer10_attention_weight_max": 66.13534545898438,
      "activations/layer10_attention_weight_min": -61.53966522216797,
      "activations/layer11_attention_weight_max": 45.1910514831543,
      "activations/layer11_attention_weight_min": -39.801246643066406,
      "activations/layer12_attention_weight_max": 27.685367584228516,
      "activations/layer12_attention_weight_min": -24.94116973876953,
      "activations/layer13_attention_weight_max": 64.23677062988281,
      "activations/layer13_attention_weight_min": -48.087120056152344,
      "activations/layer14_attention_weight_max": 50.74686813354492,
      "activations/layer14_attention_weight_min": -35.962318420410156,
      "activations/layer15_attention_weight_max": 44.54578399658203,
      "activations/layer15_attention_weight_min": -36.31178665161133,
      "activations/layer16_attention_weight_max": 49.91252136230469,
      "activations/layer16_attention_weight_min": -38.455413818359375,
      "activations/layer17_attention_weight_max": 57.77304458618164,
      "activations/layer17_attention_weight_min": -40.12784957885742,
      "activations/layer18_attention_weight_max": 48.65871810913086,
      "activations/layer18_attention_weight_min": -37.56770706176758,
      "activations/layer19_attention_weight_max": 21.80696678161621,
      "activations/layer19_attention_weight_min": -11.916082382202148,
      "activations/layer1_attention_weight_max": 17.317420959472656,
      "activations/layer1_attention_weight_min": -12.379412651062012,
      "activations/layer20_attention_weight_max": 23.038755416870117,
      "activations/layer20_attention_weight_min": -13.825921058654785,
      "activations/layer21_attention_weight_max": 36.85035705566406,
      "activations/layer21_attention_weight_min": -17.695547103881836,
      "activations/layer22_attention_weight_max": 32.986576080322266,
      "activations/layer22_attention_weight_min": -15.827062606811523,
      "activations/layer23_attention_weight_max": 22.167224884033203,
      "activations/layer23_attention_weight_min": -20.49663543701172,
      "activations/layer2_attention_weight_max": 26.226821899414062,
      "activations/layer2_attention_weight_min": -22.863529205322266,
      "activations/layer3_attention_weight_max": 50.59526062011719,
      "activations/layer3_attention_weight_min": -49.43014144897461,
      "activations/layer4_attention_weight_max": 78.39566040039062,
      "activations/layer4_attention_weight_min": -74.68161010742188,
      "activations/layer5_attention_weight_max": 134.68995666503906,
      "activations/layer5_attention_weight_min": -124.71620178222656,
      "activations/layer6_attention_weight_max": 67.00878143310547,
      "activations/layer6_attention_weight_min": -62.041107177734375,
      "activations/layer7_attention_weight_max": 69.93195343017578,
      "activations/layer7_attention_weight_min": -63.40019226074219,
      "activations/layer8_attention_weight_max": 54.5715217590332,
      "activations/layer8_attention_weight_min": -57.48698806762695,
      "activations/layer9_attention_weight_max": 65.89971160888672,
      "activations/layer9_attention_weight_min": -62.96547317504883,
      "epoch": 0.7,
      "learning_rate": 0.00014696060606060604,
      "loss": 3.1808,
      "step": 12050
    },
    {
      "activations/layer0_attention_weight_max": 14.159042358398438,
      "activations/layer0_attention_weight_min": -12.095816612243652,
      "activations/layer10_attention_weight_max": 65.18669128417969,
      "activations/layer10_attention_weight_min": -59.89081573486328,
      "activations/layer11_attention_weight_max": 45.54475402832031,
      "activations/layer11_attention_weight_min": -38.890628814697266,
      "activations/layer12_attention_weight_max": 25.898761749267578,
      "activations/layer12_attention_weight_min": -23.773841857910156,
      "activations/layer13_attention_weight_max": 68.42625427246094,
      "activations/layer13_attention_weight_min": -41.647705078125,
      "activations/layer14_attention_weight_max": 57.762569427490234,
      "activations/layer14_attention_weight_min": -34.93403244018555,
      "activations/layer15_attention_weight_max": 45.8869743347168,
      "activations/layer15_attention_weight_min": -34.86800765991211,
      "activations/layer16_attention_weight_max": 44.71872329711914,
      "activations/layer16_attention_weight_min": -30.426651000976562,
      "activations/layer17_attention_weight_max": 56.64729690551758,
      "activations/layer17_attention_weight_min": -33.953670501708984,
      "activations/layer18_attention_weight_max": 64.32337951660156,
      "activations/layer18_attention_weight_min": -35.072853088378906,
      "activations/layer19_attention_weight_max": 18.827741622924805,
      "activations/layer19_attention_weight_min": -9.832489967346191,
      "activations/layer1_attention_weight_max": 17.38694190979004,
      "activations/layer1_attention_weight_min": -11.465357780456543,
      "activations/layer20_attention_weight_max": 18.743223190307617,
      "activations/layer20_attention_weight_min": -11.701885223388672,
      "activations/layer21_attention_weight_max": 35.416786193847656,
      "activations/layer21_attention_weight_min": -16.037586212158203,
      "activations/layer22_attention_weight_max": 29.87200164794922,
      "activations/layer22_attention_weight_min": -13.928167343139648,
      "activations/layer23_attention_weight_max": 20.154979705810547,
      "activations/layer23_attention_weight_min": -20.509387969970703,
      "activations/layer2_attention_weight_max": 26.349143981933594,
      "activations/layer2_attention_weight_min": -20.796794891357422,
      "activations/layer3_attention_weight_max": 52.50700759887695,
      "activations/layer3_attention_weight_min": -48.314476013183594,
      "activations/layer4_attention_weight_max": 83.0259780883789,
      "activations/layer4_attention_weight_min": -73.56375885009766,
      "activations/layer5_attention_weight_max": 146.0250701904297,
      "activations/layer5_attention_weight_min": -123.33147430419922,
      "activations/layer6_attention_weight_max": 71.6324691772461,
      "activations/layer6_attention_weight_min": -63.73782730102539,
      "activations/layer7_attention_weight_max": 76.25617980957031,
      "activations/layer7_attention_weight_min": -62.53754425048828,
      "activations/layer8_attention_weight_max": 62.09830856323242,
      "activations/layer8_attention_weight_min": -51.30128479003906,
      "activations/layer9_attention_weight_max": 68.45901489257812,
      "activations/layer9_attention_weight_min": -66.33895111083984,
      "epoch": 0.7,
      "learning_rate": 0.00014694166666666666,
      "loss": 3.1816,
      "step": 12100
    },
    {
      "activations/layer0_attention_weight_max": 12.689981460571289,
      "activations/layer0_attention_weight_min": -12.666592597961426,
      "activations/layer10_attention_weight_max": 75.2555923461914,
      "activations/layer10_attention_weight_min": -63.97838592529297,
      "activations/layer11_attention_weight_max": 50.63501739501953,
      "activations/layer11_attention_weight_min": -41.119041442871094,
      "activations/layer12_attention_weight_max": 30.233646392822266,
      "activations/layer12_attention_weight_min": -24.474157333374023,
      "activations/layer13_attention_weight_max": 83.62371826171875,
      "activations/layer13_attention_weight_min": -45.83750534057617,
      "activations/layer14_attention_weight_max": 76.68122100830078,
      "activations/layer14_attention_weight_min": -43.60578918457031,
      "activations/layer15_attention_weight_max": 60.783409118652344,
      "activations/layer15_attention_weight_min": -35.304683685302734,
      "activations/layer16_attention_weight_max": 50.51548385620117,
      "activations/layer16_attention_weight_min": -34.820011138916016,
      "activations/layer17_attention_weight_max": 65.6308364868164,
      "activations/layer17_attention_weight_min": -41.46873092651367,
      "activations/layer18_attention_weight_max": 60.6075553894043,
      "activations/layer18_attention_weight_min": -31.73861312866211,
      "activations/layer19_attention_weight_max": 20.540311813354492,
      "activations/layer19_attention_weight_min": -11.030147552490234,
      "activations/layer1_attention_weight_max": 16.467418670654297,
      "activations/layer1_attention_weight_min": -11.690258026123047,
      "activations/layer20_attention_weight_max": 16.536483764648438,
      "activations/layer20_attention_weight_min": -11.579519271850586,
      "activations/layer21_attention_weight_max": 31.430349349975586,
      "activations/layer21_attention_weight_min": -15.698073387145996,
      "activations/layer22_attention_weight_max": 27.91954803466797,
      "activations/layer22_attention_weight_min": -13.613199234008789,
      "activations/layer23_attention_weight_max": 23.707176208496094,
      "activations/layer23_attention_weight_min": -22.619916915893555,
      "activations/layer2_attention_weight_max": 25.846309661865234,
      "activations/layer2_attention_weight_min": -22.675092697143555,
      "activations/layer3_attention_weight_max": 52.706722259521484,
      "activations/layer3_attention_weight_min": -49.37080383300781,
      "activations/layer4_attention_weight_max": 86.57220458984375,
      "activations/layer4_attention_weight_min": -72.88967895507812,
      "activations/layer5_attention_weight_max": 150.47103881835938,
      "activations/layer5_attention_weight_min": -122.56441497802734,
      "activations/layer6_attention_weight_max": 76.46334838867188,
      "activations/layer6_attention_weight_min": -61.97282409667969,
      "activations/layer7_attention_weight_max": 75.1468276977539,
      "activations/layer7_attention_weight_min": -64.99986267089844,
      "activations/layer8_attention_weight_max": 67.30305480957031,
      "activations/layer8_attention_weight_min": -56.05049133300781,
      "activations/layer9_attention_weight_max": 83.03270721435547,
      "activations/layer9_attention_weight_min": -68.20308685302734,
      "epoch": 0.71,
      "learning_rate": 0.00014692272727272727,
      "loss": 3.1762,
      "step": 12150
    },
    {
      "activations/layer0_attention_weight_max": 12.791298866271973,
      "activations/layer0_attention_weight_min": -12.205020904541016,
      "activations/layer10_attention_weight_max": 72.80331420898438,
      "activations/layer10_attention_weight_min": -62.690940856933594,
      "activations/layer11_attention_weight_max": 46.56086730957031,
      "activations/layer11_attention_weight_min": -42.445091247558594,
      "activations/layer12_attention_weight_max": 23.875389099121094,
      "activations/layer12_attention_weight_min": -25.252918243408203,
      "activations/layer13_attention_weight_max": 59.81479263305664,
      "activations/layer13_attention_weight_min": -43.310203552246094,
      "activations/layer14_attention_weight_max": 49.71783447265625,
      "activations/layer14_attention_weight_min": -34.78419494628906,
      "activations/layer15_attention_weight_max": 49.516395568847656,
      "activations/layer15_attention_weight_min": -34.950138092041016,
      "activations/layer16_attention_weight_max": 44.417667388916016,
      "activations/layer16_attention_weight_min": -31.143627166748047,
      "activations/layer17_attention_weight_max": 56.62983322143555,
      "activations/layer17_attention_weight_min": -36.50043869018555,
      "activations/layer18_attention_weight_max": 59.1299934387207,
      "activations/layer18_attention_weight_min": -37.620521545410156,
      "activations/layer19_attention_weight_max": 19.486421585083008,
      "activations/layer19_attention_weight_min": -11.764277458190918,
      "activations/layer1_attention_weight_max": 16.4185848236084,
      "activations/layer1_attention_weight_min": -10.287558555603027,
      "activations/layer20_attention_weight_max": 16.183618545532227,
      "activations/layer20_attention_weight_min": -11.550277709960938,
      "activations/layer21_attention_weight_max": 29.500835418701172,
      "activations/layer21_attention_weight_min": -16.23763656616211,
      "activations/layer22_attention_weight_max": 25.457775115966797,
      "activations/layer22_attention_weight_min": -14.058981895446777,
      "activations/layer23_attention_weight_max": 20.86341667175293,
      "activations/layer23_attention_weight_min": -20.898303985595703,
      "activations/layer2_attention_weight_max": 25.38626480102539,
      "activations/layer2_attention_weight_min": -22.23969841003418,
      "activations/layer3_attention_weight_max": 51.44708251953125,
      "activations/layer3_attention_weight_min": -47.92368698120117,
      "activations/layer4_attention_weight_max": 79.74679565429688,
      "activations/layer4_attention_weight_min": -74.40208435058594,
      "activations/layer5_attention_weight_max": 141.09759521484375,
      "activations/layer5_attention_weight_min": -119.99980163574219,
      "activations/layer6_attention_weight_max": 68.24385070800781,
      "activations/layer6_attention_weight_min": -59.34357833862305,
      "activations/layer7_attention_weight_max": 69.00274658203125,
      "activations/layer7_attention_weight_min": -62.056602478027344,
      "activations/layer8_attention_weight_max": 61.40870666503906,
      "activations/layer8_attention_weight_min": -52.66365432739258,
      "activations/layer9_attention_weight_max": 68.69113159179688,
      "activations/layer9_attention_weight_min": -64.20274353027344,
      "epoch": 0.71,
      "learning_rate": 0.00014690378787878786,
      "loss": 3.1891,
      "step": 12200
    },
    {
      "activations/layer0_attention_weight_max": 13.922322273254395,
      "activations/layer0_attention_weight_min": -12.320218086242676,
      "activations/layer10_attention_weight_max": 65.80239868164062,
      "activations/layer10_attention_weight_min": -61.102352142333984,
      "activations/layer11_attention_weight_max": 42.99055099487305,
      "activations/layer11_attention_weight_min": -42.042598724365234,
      "activations/layer12_attention_weight_max": 26.046831130981445,
      "activations/layer12_attention_weight_min": -22.836626052856445,
      "activations/layer13_attention_weight_max": 53.42439270019531,
      "activations/layer13_attention_weight_min": -42.77448272705078,
      "activations/layer14_attention_weight_max": 40.280941009521484,
      "activations/layer14_attention_weight_min": -30.276126861572266,
      "activations/layer15_attention_weight_max": 47.7490119934082,
      "activations/layer15_attention_weight_min": -34.80520248413086,
      "activations/layer16_attention_weight_max": 40.190101623535156,
      "activations/layer16_attention_weight_min": -35.13406753540039,
      "activations/layer17_attention_weight_max": 54.706382751464844,
      "activations/layer17_attention_weight_min": -40.38299560546875,
      "activations/layer18_attention_weight_max": 44.08074188232422,
      "activations/layer18_attention_weight_min": -31.547405242919922,
      "activations/layer19_attention_weight_max": 19.714256286621094,
      "activations/layer19_attention_weight_min": -12.289433479309082,
      "activations/layer1_attention_weight_max": 15.8054780960083,
      "activations/layer1_attention_weight_min": -9.809175491333008,
      "activations/layer20_attention_weight_max": 17.08057403564453,
      "activations/layer20_attention_weight_min": -13.114368438720703,
      "activations/layer21_attention_weight_max": 28.02083396911621,
      "activations/layer21_attention_weight_min": -16.815156936645508,
      "activations/layer22_attention_weight_max": 26.70625114440918,
      "activations/layer22_attention_weight_min": -15.399725914001465,
      "activations/layer23_attention_weight_max": 21.56912612915039,
      "activations/layer23_attention_weight_min": -20.03853416442871,
      "activations/layer2_attention_weight_max": 28.540555953979492,
      "activations/layer2_attention_weight_min": -21.774826049804688,
      "activations/layer3_attention_weight_max": 50.03472900390625,
      "activations/layer3_attention_weight_min": -48.336891174316406,
      "activations/layer4_attention_weight_max": 78.69483184814453,
      "activations/layer4_attention_weight_min": -72.59630584716797,
      "activations/layer5_attention_weight_max": 141.2941131591797,
      "activations/layer5_attention_weight_min": -121.28213500976562,
      "activations/layer6_attention_weight_max": 71.92378234863281,
      "activations/layer6_attention_weight_min": -60.913063049316406,
      "activations/layer7_attention_weight_max": 71.45352935791016,
      "activations/layer7_attention_weight_min": -63.91407775878906,
      "activations/layer8_attention_weight_max": 59.7454719543457,
      "activations/layer8_attention_weight_min": -50.5865478515625,
      "activations/layer9_attention_weight_max": 69.78764343261719,
      "activations/layer9_attention_weight_min": -62.40047073364258,
      "epoch": 0.71,
      "learning_rate": 0.00014688484848484847,
      "loss": 3.1933,
      "step": 12250
    },
    {
      "activations/layer0_attention_weight_max": 13.175047874450684,
      "activations/layer0_attention_weight_min": -12.196479797363281,
      "activations/layer10_attention_weight_max": 71.7717514038086,
      "activations/layer10_attention_weight_min": -59.51127624511719,
      "activations/layer11_attention_weight_max": 49.26287078857422,
      "activations/layer11_attention_weight_min": -38.809085845947266,
      "activations/layer12_attention_weight_max": 31.645286560058594,
      "activations/layer12_attention_weight_min": -22.040536880493164,
      "activations/layer13_attention_weight_max": 57.35828399658203,
      "activations/layer13_attention_weight_min": -43.22748947143555,
      "activations/layer14_attention_weight_max": 54.78577423095703,
      "activations/layer14_attention_weight_min": -35.4037971496582,
      "activations/layer15_attention_weight_max": 49.62126159667969,
      "activations/layer15_attention_weight_min": -36.63552474975586,
      "activations/layer16_attention_weight_max": 49.14668273925781,
      "activations/layer16_attention_weight_min": -34.047576904296875,
      "activations/layer17_attention_weight_max": 60.89290237426758,
      "activations/layer17_attention_weight_min": -38.32188034057617,
      "activations/layer18_attention_weight_max": 64.58009338378906,
      "activations/layer18_attention_weight_min": -40.03772735595703,
      "activations/layer19_attention_weight_max": 20.00020408630371,
      "activations/layer19_attention_weight_min": -12.199565887451172,
      "activations/layer1_attention_weight_max": 16.69490623474121,
      "activations/layer1_attention_weight_min": -11.80690860748291,
      "activations/layer20_attention_weight_max": 18.33033561706543,
      "activations/layer20_attention_weight_min": -10.710321426391602,
      "activations/layer21_attention_weight_max": 34.629127502441406,
      "activations/layer21_attention_weight_min": -15.45711898803711,
      "activations/layer22_attention_weight_max": 30.425750732421875,
      "activations/layer22_attention_weight_min": -12.466876983642578,
      "activations/layer23_attention_weight_max": 23.661758422851562,
      "activations/layer23_attention_weight_min": -22.469226837158203,
      "activations/layer2_attention_weight_max": 25.78339195251465,
      "activations/layer2_attention_weight_min": -21.217769622802734,
      "activations/layer3_attention_weight_max": 51.191200256347656,
      "activations/layer3_attention_weight_min": -47.27337646484375,
      "activations/layer4_attention_weight_max": 78.21522521972656,
      "activations/layer4_attention_weight_min": -71.72654724121094,
      "activations/layer5_attention_weight_max": 142.23501586914062,
      "activations/layer5_attention_weight_min": -122.20895385742188,
      "activations/layer6_attention_weight_max": 70.80702209472656,
      "activations/layer6_attention_weight_min": -60.96913146972656,
      "activations/layer7_attention_weight_max": 71.92495727539062,
      "activations/layer7_attention_weight_min": -63.180233001708984,
      "activations/layer8_attention_weight_max": 63.41244888305664,
      "activations/layer8_attention_weight_min": -51.22792434692383,
      "activations/layer9_attention_weight_max": 73.871337890625,
      "activations/layer9_attention_weight_min": -64.17228698730469,
      "epoch": 0.71,
      "learning_rate": 0.0001468659090909091,
      "loss": 3.1942,
      "step": 12300
    },
    {
      "activations/layer0_attention_weight_max": 14.284214973449707,
      "activations/layer0_attention_weight_min": -11.646068572998047,
      "activations/layer10_attention_weight_max": 73.58344268798828,
      "activations/layer10_attention_weight_min": -66.81489562988281,
      "activations/layer11_attention_weight_max": 42.403961181640625,
      "activations/layer11_attention_weight_min": -35.582244873046875,
      "activations/layer12_attention_weight_max": 25.028507232666016,
      "activations/layer12_attention_weight_min": -24.641944885253906,
      "activations/layer13_attention_weight_max": 57.56807327270508,
      "activations/layer13_attention_weight_min": -45.144004821777344,
      "activations/layer14_attention_weight_max": 43.2449951171875,
      "activations/layer14_attention_weight_min": -33.36936950683594,
      "activations/layer15_attention_weight_max": 48.68981170654297,
      "activations/layer15_attention_weight_min": -34.16362380981445,
      "activations/layer16_attention_weight_max": 50.825950622558594,
      "activations/layer16_attention_weight_min": -34.064453125,
      "activations/layer17_attention_weight_max": 55.68476486206055,
      "activations/layer17_attention_weight_min": -36.11711883544922,
      "activations/layer18_attention_weight_max": 59.29615020751953,
      "activations/layer18_attention_weight_min": -31.582904815673828,
      "activations/layer19_attention_weight_max": 18.47134017944336,
      "activations/layer19_attention_weight_min": -11.649218559265137,
      "activations/layer1_attention_weight_max": 16.039154052734375,
      "activations/layer1_attention_weight_min": -10.832369804382324,
      "activations/layer20_attention_weight_max": 19.37413215637207,
      "activations/layer20_attention_weight_min": -11.580083847045898,
      "activations/layer21_attention_weight_max": 33.72035598754883,
      "activations/layer21_attention_weight_min": -16.364734649658203,
      "activations/layer22_attention_weight_max": 28.758209228515625,
      "activations/layer22_attention_weight_min": -14.55042552947998,
      "activations/layer23_attention_weight_max": 23.25153923034668,
      "activations/layer23_attention_weight_min": -20.164690017700195,
      "activations/layer2_attention_weight_max": 26.762069702148438,
      "activations/layer2_attention_weight_min": -22.59325408935547,
      "activations/layer3_attention_weight_max": 50.264957427978516,
      "activations/layer3_attention_weight_min": -50.20030212402344,
      "activations/layer4_attention_weight_max": 81.0736083984375,
      "activations/layer4_attention_weight_min": -73.09166717529297,
      "activations/layer5_attention_weight_max": 138.62814331054688,
      "activations/layer5_attention_weight_min": -119.31175231933594,
      "activations/layer6_attention_weight_max": 68.30269622802734,
      "activations/layer6_attention_weight_min": -58.27180099487305,
      "activations/layer7_attention_weight_max": 69.28897094726562,
      "activations/layer7_attention_weight_min": -58.36710739135742,
      "activations/layer8_attention_weight_max": 59.98403549194336,
      "activations/layer8_attention_weight_min": -52.88359451293945,
      "activations/layer9_attention_weight_max": 69.97289276123047,
      "activations/layer9_attention_weight_min": -68.72714233398438,
      "epoch": 0.72,
      "learning_rate": 0.00014684696969696968,
      "loss": 3.1813,
      "step": 12350
    },
    {
      "activations/layer0_attention_weight_max": 13.826001167297363,
      "activations/layer0_attention_weight_min": -11.908101081848145,
      "activations/layer10_attention_weight_max": 65.38838195800781,
      "activations/layer10_attention_weight_min": -59.960994720458984,
      "activations/layer11_attention_weight_max": 46.831077575683594,
      "activations/layer11_attention_weight_min": -40.904762268066406,
      "activations/layer12_attention_weight_max": 26.023435592651367,
      "activations/layer12_attention_weight_min": -25.438350677490234,
      "activations/layer13_attention_weight_max": 48.56143569946289,
      "activations/layer13_attention_weight_min": -40.05474853515625,
      "activations/layer14_attention_weight_max": 55.04271697998047,
      "activations/layer14_attention_weight_min": -44.592166900634766,
      "activations/layer15_attention_weight_max": 44.8345947265625,
      "activations/layer15_attention_weight_min": -34.80419921875,
      "activations/layer16_attention_weight_max": 45.56972885131836,
      "activations/layer16_attention_weight_min": -31.19989013671875,
      "activations/layer17_attention_weight_max": 52.39060592651367,
      "activations/layer17_attention_weight_min": -40.9470100402832,
      "activations/layer18_attention_weight_max": 52.216182708740234,
      "activations/layer18_attention_weight_min": -36.76758575439453,
      "activations/layer19_attention_weight_max": 19.32914924621582,
      "activations/layer19_attention_weight_min": -11.932229042053223,
      "activations/layer1_attention_weight_max": 16.326297760009766,
      "activations/layer1_attention_weight_min": -10.95881462097168,
      "activations/layer20_attention_weight_max": 17.025972366333008,
      "activations/layer20_attention_weight_min": -12.243213653564453,
      "activations/layer21_attention_weight_max": 32.838104248046875,
      "activations/layer21_attention_weight_min": -18.517654418945312,
      "activations/layer22_attention_weight_max": 30.852502822875977,
      "activations/layer22_attention_weight_min": -15.419147491455078,
      "activations/layer23_attention_weight_max": 22.678035736083984,
      "activations/layer23_attention_weight_min": -21.655540466308594,
      "activations/layer2_attention_weight_max": 27.232566833496094,
      "activations/layer2_attention_weight_min": -21.55824851989746,
      "activations/layer3_attention_weight_max": 53.09013366699219,
      "activations/layer3_attention_weight_min": -48.325927734375,
      "activations/layer4_attention_weight_max": 80.26897430419922,
      "activations/layer4_attention_weight_min": -72.84637451171875,
      "activations/layer5_attention_weight_max": 135.0126495361328,
      "activations/layer5_attention_weight_min": -116.8626708984375,
      "activations/layer6_attention_weight_max": 71.52963256835938,
      "activations/layer6_attention_weight_min": -59.36296844482422,
      "activations/layer7_attention_weight_max": 69.04572296142578,
      "activations/layer7_attention_weight_min": -61.59553146362305,
      "activations/layer8_attention_weight_max": 59.30239486694336,
      "activations/layer8_attention_weight_min": -53.30217742919922,
      "activations/layer9_attention_weight_max": 65.96382141113281,
      "activations/layer9_attention_weight_min": -65.98149871826172,
      "epoch": 0.72,
      "learning_rate": 0.0001468280303030303,
      "loss": 3.1707,
      "step": 12400
    },
    {
      "activations/layer0_attention_weight_max": 13.643682479858398,
      "activations/layer0_attention_weight_min": -11.666952133178711,
      "activations/layer10_attention_weight_max": 70.7571792602539,
      "activations/layer10_attention_weight_min": -63.02050018310547,
      "activations/layer11_attention_weight_max": 45.96443176269531,
      "activations/layer11_attention_weight_min": -40.5020751953125,
      "activations/layer12_attention_weight_max": 25.97343635559082,
      "activations/layer12_attention_weight_min": -23.742536544799805,
      "activations/layer13_attention_weight_max": 59.696964263916016,
      "activations/layer13_attention_weight_min": -46.007137298583984,
      "activations/layer14_attention_weight_max": 48.153076171875,
      "activations/layer14_attention_weight_min": -35.49114990234375,
      "activations/layer15_attention_weight_max": 49.780277252197266,
      "activations/layer15_attention_weight_min": -37.47407150268555,
      "activations/layer16_attention_weight_max": 45.5328369140625,
      "activations/layer16_attention_weight_min": -36.184818267822266,
      "activations/layer17_attention_weight_max": 60.51741027832031,
      "activations/layer17_attention_weight_min": -37.840938568115234,
      "activations/layer18_attention_weight_max": 47.27055358886719,
      "activations/layer18_attention_weight_min": -32.37847900390625,
      "activations/layer19_attention_weight_max": 20.337430953979492,
      "activations/layer19_attention_weight_min": -12.25144100189209,
      "activations/layer1_attention_weight_max": 16.625896453857422,
      "activations/layer1_attention_weight_min": -11.342337608337402,
      "activations/layer20_attention_weight_max": 19.463529586791992,
      "activations/layer20_attention_weight_min": -13.479293823242188,
      "activations/layer21_attention_weight_max": 38.51699447631836,
      "activations/layer21_attention_weight_min": -16.452388763427734,
      "activations/layer22_attention_weight_max": 30.896530151367188,
      "activations/layer22_attention_weight_min": -14.483224868774414,
      "activations/layer23_attention_weight_max": 24.506240844726562,
      "activations/layer23_attention_weight_min": -21.56159210205078,
      "activations/layer2_attention_weight_max": 25.3817138671875,
      "activations/layer2_attention_weight_min": -20.902782440185547,
      "activations/layer3_attention_weight_max": 50.66083908081055,
      "activations/layer3_attention_weight_min": -48.87864685058594,
      "activations/layer4_attention_weight_max": 79.50111389160156,
      "activations/layer4_attention_weight_min": -73.7789535522461,
      "activations/layer5_attention_weight_max": 134.9583282470703,
      "activations/layer5_attention_weight_min": -122.54460906982422,
      "activations/layer6_attention_weight_max": 69.31741333007812,
      "activations/layer6_attention_weight_min": -61.632774353027344,
      "activations/layer7_attention_weight_max": 70.88276672363281,
      "activations/layer7_attention_weight_min": -63.65568923950195,
      "activations/layer8_attention_weight_max": 58.18226623535156,
      "activations/layer8_attention_weight_min": -52.081016540527344,
      "activations/layer9_attention_weight_max": 70.76902770996094,
      "activations/layer9_attention_weight_min": -66.60281372070312,
      "epoch": 0.72,
      "learning_rate": 0.0001468090909090909,
      "loss": 3.1428,
      "step": 12450
    },
    {
      "activations/layer0_attention_weight_max": 13.475019454956055,
      "activations/layer0_attention_weight_min": -11.435821533203125,
      "activations/layer10_attention_weight_max": 63.71236038208008,
      "activations/layer10_attention_weight_min": -55.313499450683594,
      "activations/layer11_attention_weight_max": 44.365501403808594,
      "activations/layer11_attention_weight_min": -40.52174377441406,
      "activations/layer12_attention_weight_max": 27.441221237182617,
      "activations/layer12_attention_weight_min": -24.959653854370117,
      "activations/layer13_attention_weight_max": 56.58626174926758,
      "activations/layer13_attention_weight_min": -48.084877014160156,
      "activations/layer14_attention_weight_max": 47.73187255859375,
      "activations/layer14_attention_weight_min": -37.07579803466797,
      "activations/layer15_attention_weight_max": 46.945011138916016,
      "activations/layer15_attention_weight_min": -34.563758850097656,
      "activations/layer16_attention_weight_max": 46.001426696777344,
      "activations/layer16_attention_weight_min": -34.18751525878906,
      "activations/layer17_attention_weight_max": 53.18455123901367,
      "activations/layer17_attention_weight_min": -39.8845100402832,
      "activations/layer18_attention_weight_max": 52.59130096435547,
      "activations/layer18_attention_weight_min": -33.08272933959961,
      "activations/layer19_attention_weight_max": 22.73066520690918,
      "activations/layer19_attention_weight_min": -10.462660789489746,
      "activations/layer1_attention_weight_max": 17.81918716430664,
      "activations/layer1_attention_weight_min": -11.46878433227539,
      "activations/layer20_attention_weight_max": 16.595304489135742,
      "activations/layer20_attention_weight_min": -11.673858642578125,
      "activations/layer21_attention_weight_max": 34.930870056152344,
      "activations/layer21_attention_weight_min": -16.226564407348633,
      "activations/layer22_attention_weight_max": 27.730405807495117,
      "activations/layer22_attention_weight_min": -16.13231086730957,
      "activations/layer23_attention_weight_max": 22.630369186401367,
      "activations/layer23_attention_weight_min": -20.194499969482422,
      "activations/layer2_attention_weight_max": 25.47004508972168,
      "activations/layer2_attention_weight_min": -22.181211471557617,
      "activations/layer3_attention_weight_max": 50.84580612182617,
      "activations/layer3_attention_weight_min": -48.73199462890625,
      "activations/layer4_attention_weight_max": 80.71044158935547,
      "activations/layer4_attention_weight_min": -73.20086669921875,
      "activations/layer5_attention_weight_max": 135.5137176513672,
      "activations/layer5_attention_weight_min": -119.30856323242188,
      "activations/layer6_attention_weight_max": 64.45973205566406,
      "activations/layer6_attention_weight_min": -58.32098388671875,
      "activations/layer7_attention_weight_max": 68.64903259277344,
      "activations/layer7_attention_weight_min": -60.43208312988281,
      "activations/layer8_attention_weight_max": 56.373836517333984,
      "activations/layer8_attention_weight_min": -50.04243469238281,
      "activations/layer9_attention_weight_max": 72.53142547607422,
      "activations/layer9_attention_weight_min": -65.58419036865234,
      "epoch": 0.73,
      "learning_rate": 0.0001467901515151515,
      "loss": 3.1543,
      "step": 12500
    },
    {
      "activations/layer0_attention_weight_max": 13.795178413391113,
      "activations/layer0_attention_weight_min": -10.963705062866211,
      "activations/layer10_attention_weight_max": 60.968624114990234,
      "activations/layer10_attention_weight_min": -61.97733688354492,
      "activations/layer11_attention_weight_max": 46.281494140625,
      "activations/layer11_attention_weight_min": -43.195308685302734,
      "activations/layer12_attention_weight_max": 23.569677352905273,
      "activations/layer12_attention_weight_min": -24.05286407470703,
      "activations/layer13_attention_weight_max": 51.13105773925781,
      "activations/layer13_attention_weight_min": -44.657222747802734,
      "activations/layer14_attention_weight_max": 39.130775451660156,
      "activations/layer14_attention_weight_min": -34.91001892089844,
      "activations/layer15_attention_weight_max": 43.77265167236328,
      "activations/layer15_attention_weight_min": -35.12406539916992,
      "activations/layer16_attention_weight_max": 42.634281158447266,
      "activations/layer16_attention_weight_min": -35.37885665893555,
      "activations/layer17_attention_weight_max": 46.9686164855957,
      "activations/layer17_attention_weight_min": -41.51893615722656,
      "activations/layer18_attention_weight_max": 43.595428466796875,
      "activations/layer18_attention_weight_min": -33.2263069152832,
      "activations/layer19_attention_weight_max": 19.314790725708008,
      "activations/layer19_attention_weight_min": -11.4446382522583,
      "activations/layer1_attention_weight_max": 17.680688858032227,
      "activations/layer1_attention_weight_min": -10.170719146728516,
      "activations/layer20_attention_weight_max": 21.581499099731445,
      "activations/layer20_attention_weight_min": -13.475313186645508,
      "activations/layer21_attention_weight_max": 32.649810791015625,
      "activations/layer21_attention_weight_min": -16.622928619384766,
      "activations/layer22_attention_weight_max": 28.954561233520508,
      "activations/layer22_attention_weight_min": -15.697593688964844,
      "activations/layer23_attention_weight_max": 20.34453582763672,
      "activations/layer23_attention_weight_min": -20.35407257080078,
      "activations/layer2_attention_weight_max": 24.371097564697266,
      "activations/layer2_attention_weight_min": -21.337066650390625,
      "activations/layer3_attention_weight_max": 49.078182220458984,
      "activations/layer3_attention_weight_min": -46.744441986083984,
      "activations/layer4_attention_weight_max": 77.27327728271484,
      "activations/layer4_attention_weight_min": -71.7380142211914,
      "activations/layer5_attention_weight_max": 130.98402404785156,
      "activations/layer5_attention_weight_min": -124.75963592529297,
      "activations/layer6_attention_weight_max": 65.45967864990234,
      "activations/layer6_attention_weight_min": -61.91206359863281,
      "activations/layer7_attention_weight_max": 71.06964874267578,
      "activations/layer7_attention_weight_min": -62.186946868896484,
      "activations/layer8_attention_weight_max": 56.99032211303711,
      "activations/layer8_attention_weight_min": -52.820289611816406,
      "activations/layer9_attention_weight_max": 65.28675079345703,
      "activations/layer9_attention_weight_min": -63.132232666015625,
      "epoch": 0.73,
      "learning_rate": 0.00014677159090909089,
      "loss": 3.178,
      "step": 12550
    },
    {
      "activations/layer0_attention_weight_max": 13.070778846740723,
      "activations/layer0_attention_weight_min": -11.404945373535156,
      "activations/layer10_attention_weight_max": 67.04341125488281,
      "activations/layer10_attention_weight_min": -59.94782638549805,
      "activations/layer11_attention_weight_max": 49.14605712890625,
      "activations/layer11_attention_weight_min": -39.99571990966797,
      "activations/layer12_attention_weight_max": 27.850543975830078,
      "activations/layer12_attention_weight_min": -22.845458984375,
      "activations/layer13_attention_weight_max": 55.860252380371094,
      "activations/layer13_attention_weight_min": -43.74441909790039,
      "activations/layer14_attention_weight_max": 40.37273025512695,
      "activations/layer14_attention_weight_min": -32.566490173339844,
      "activations/layer15_attention_weight_max": 48.473487854003906,
      "activations/layer15_attention_weight_min": -38.9583854675293,
      "activations/layer16_attention_weight_max": 47.39790344238281,
      "activations/layer16_attention_weight_min": -32.66961669921875,
      "activations/layer17_attention_weight_max": 51.537933349609375,
      "activations/layer17_attention_weight_min": -41.68792724609375,
      "activations/layer18_attention_weight_max": 45.5699577331543,
      "activations/layer18_attention_weight_min": -30.969411849975586,
      "activations/layer19_attention_weight_max": 20.074121475219727,
      "activations/layer19_attention_weight_min": -11.347049713134766,
      "activations/layer1_attention_weight_max": 16.137937545776367,
      "activations/layer1_attention_weight_min": -10.870170593261719,
      "activations/layer20_attention_weight_max": 20.537025451660156,
      "activations/layer20_attention_weight_min": -13.694978713989258,
      "activations/layer21_attention_weight_max": 31.133352279663086,
      "activations/layer21_attention_weight_min": -20.26029396057129,
      "activations/layer22_attention_weight_max": 28.14672088623047,
      "activations/layer22_attention_weight_min": -14.804970741271973,
      "activations/layer23_attention_weight_max": 20.703811645507812,
      "activations/layer23_attention_weight_min": -20.37169075012207,
      "activations/layer2_attention_weight_max": 26.699848175048828,
      "activations/layer2_attention_weight_min": -21.09207534790039,
      "activations/layer3_attention_weight_max": 52.987335205078125,
      "activations/layer3_attention_weight_min": -49.839359283447266,
      "activations/layer4_attention_weight_max": 79.85424041748047,
      "activations/layer4_attention_weight_min": -72.90837860107422,
      "activations/layer5_attention_weight_max": 137.96603393554688,
      "activations/layer5_attention_weight_min": -123.82130432128906,
      "activations/layer6_attention_weight_max": 68.34477996826172,
      "activations/layer6_attention_weight_min": -62.6507568359375,
      "activations/layer7_attention_weight_max": 70.56605529785156,
      "activations/layer7_attention_weight_min": -65.23298645019531,
      "activations/layer8_attention_weight_max": 58.10569381713867,
      "activations/layer8_attention_weight_min": -52.21022415161133,
      "activations/layer9_attention_weight_max": 68.54378509521484,
      "activations/layer9_attention_weight_min": -67.41830444335938,
      "epoch": 0.73,
      "learning_rate": 0.0001467526515151515,
      "loss": 3.1696,
      "step": 12600
    },
    {
      "activations/layer0_attention_weight_max": 13.324729919433594,
      "activations/layer0_attention_weight_min": -11.343496322631836,
      "activations/layer10_attention_weight_max": 74.4017562866211,
      "activations/layer10_attention_weight_min": -65.3938980102539,
      "activations/layer11_attention_weight_max": 48.40980529785156,
      "activations/layer11_attention_weight_min": -42.708953857421875,
      "activations/layer12_attention_weight_max": 24.51123809814453,
      "activations/layer12_attention_weight_min": -26.754419326782227,
      "activations/layer13_attention_weight_max": 66.52095031738281,
      "activations/layer13_attention_weight_min": -46.38576889038086,
      "activations/layer14_attention_weight_max": 44.21418380737305,
      "activations/layer14_attention_weight_min": -33.682796478271484,
      "activations/layer15_attention_weight_max": 53.25703811645508,
      "activations/layer15_attention_weight_min": -35.42988586425781,
      "activations/layer16_attention_weight_max": 46.097312927246094,
      "activations/layer16_attention_weight_min": -33.54454040527344,
      "activations/layer17_attention_weight_max": 52.25896072387695,
      "activations/layer17_attention_weight_min": -35.30802917480469,
      "activations/layer18_attention_weight_max": 46.41194534301758,
      "activations/layer18_attention_weight_min": -30.747610092163086,
      "activations/layer19_attention_weight_max": 20.82000160217285,
      "activations/layer19_attention_weight_min": -11.260668754577637,
      "activations/layer1_attention_weight_max": 17.311573028564453,
      "activations/layer1_attention_weight_min": -11.065278053283691,
      "activations/layer20_attention_weight_max": 18.08540916442871,
      "activations/layer20_attention_weight_min": -12.285857200622559,
      "activations/layer21_attention_weight_max": 29.752120971679688,
      "activations/layer21_attention_weight_min": -14.810240745544434,
      "activations/layer22_attention_weight_max": 29.906293869018555,
      "activations/layer22_attention_weight_min": -13.6370849609375,
      "activations/layer23_attention_weight_max": 21.224559783935547,
      "activations/layer23_attention_weight_min": -18.779123306274414,
      "activations/layer2_attention_weight_max": 26.460887908935547,
      "activations/layer2_attention_weight_min": -21.351491928100586,
      "activations/layer3_attention_weight_max": 56.66460037231445,
      "activations/layer3_attention_weight_min": -50.981929779052734,
      "activations/layer4_attention_weight_max": 84.57050323486328,
      "activations/layer4_attention_weight_min": -73.90716552734375,
      "activations/layer5_attention_weight_max": 145.06161499023438,
      "activations/layer5_attention_weight_min": -125.54179382324219,
      "activations/layer6_attention_weight_max": 73.55779266357422,
      "activations/layer6_attention_weight_min": -60.07597351074219,
      "activations/layer7_attention_weight_max": 73.90821838378906,
      "activations/layer7_attention_weight_min": -59.6168327331543,
      "activations/layer8_attention_weight_max": 61.90071487426758,
      "activations/layer8_attention_weight_min": -52.53564453125,
      "activations/layer9_attention_weight_max": 69.14852142333984,
      "activations/layer9_attention_weight_min": -65.39008331298828,
      "epoch": 0.74,
      "learning_rate": 0.00014673371212121212,
      "loss": 3.1592,
      "step": 12650
    },
    {
      "activations/layer0_attention_weight_max": 13.723836898803711,
      "activations/layer0_attention_weight_min": -11.00959587097168,
      "activations/layer10_attention_weight_max": 63.20832443237305,
      "activations/layer10_attention_weight_min": -62.01459884643555,
      "activations/layer11_attention_weight_max": 43.56846237182617,
      "activations/layer11_attention_weight_min": -41.06821823120117,
      "activations/layer12_attention_weight_max": 28.363929748535156,
      "activations/layer12_attention_weight_min": -25.496274948120117,
      "activations/layer13_attention_weight_max": 54.49485778808594,
      "activations/layer13_attention_weight_min": -37.624874114990234,
      "activations/layer14_attention_weight_max": 53.92517852783203,
      "activations/layer14_attention_weight_min": -35.821678161621094,
      "activations/layer15_attention_weight_max": 46.61875534057617,
      "activations/layer15_attention_weight_min": -34.60199737548828,
      "activations/layer16_attention_weight_max": 47.555110931396484,
      "activations/layer16_attention_weight_min": -38.911197662353516,
      "activations/layer17_attention_weight_max": 56.205020904541016,
      "activations/layer17_attention_weight_min": -38.77014923095703,
      "activations/layer18_attention_weight_max": 54.067935943603516,
      "activations/layer18_attention_weight_min": -35.542327880859375,
      "activations/layer19_attention_weight_max": 18.28523826599121,
      "activations/layer19_attention_weight_min": -10.952945709228516,
      "activations/layer1_attention_weight_max": 17.15041160583496,
      "activations/layer1_attention_weight_min": -10.19803524017334,
      "activations/layer20_attention_weight_max": 18.496017456054688,
      "activations/layer20_attention_weight_min": -13.526016235351562,
      "activations/layer21_attention_weight_max": 35.256343841552734,
      "activations/layer21_attention_weight_min": -17.315629959106445,
      "activations/layer22_attention_weight_max": 31.80910873413086,
      "activations/layer22_attention_weight_min": -14.808843612670898,
      "activations/layer23_attention_weight_max": 21.519290924072266,
      "activations/layer23_attention_weight_min": -20.315540313720703,
      "activations/layer2_attention_weight_max": 25.60957908630371,
      "activations/layer2_attention_weight_min": -21.16808319091797,
      "activations/layer3_attention_weight_max": 49.12502670288086,
      "activations/layer3_attention_weight_min": -45.13790512084961,
      "activations/layer4_attention_weight_max": 75.9354248046875,
      "activations/layer4_attention_weight_min": -72.17977142333984,
      "activations/layer5_attention_weight_max": 133.7248992919922,
      "activations/layer5_attention_weight_min": -122.79283905029297,
      "activations/layer6_attention_weight_max": 67.20469665527344,
      "activations/layer6_attention_weight_min": -61.252967834472656,
      "activations/layer7_attention_weight_max": 70.3909912109375,
      "activations/layer7_attention_weight_min": -61.194923400878906,
      "activations/layer8_attention_weight_max": 58.78342819213867,
      "activations/layer8_attention_weight_min": -51.94338607788086,
      "activations/layer9_attention_weight_max": 67.7500991821289,
      "activations/layer9_attention_weight_min": -65.32120513916016,
      "epoch": 0.74,
      "learning_rate": 0.0001467147727272727,
      "loss": 3.1961,
      "step": 12700
    },
    {
      "activations/layer0_attention_weight_max": 13.339537620544434,
      "activations/layer0_attention_weight_min": -10.490782737731934,
      "activations/layer10_attention_weight_max": 75.709228515625,
      "activations/layer10_attention_weight_min": -66.11231994628906,
      "activations/layer11_attention_weight_max": 46.94342803955078,
      "activations/layer11_attention_weight_min": -40.09107208251953,
      "activations/layer12_attention_weight_max": 28.2900447845459,
      "activations/layer12_attention_weight_min": -23.624540328979492,
      "activations/layer13_attention_weight_max": 62.94879150390625,
      "activations/layer13_attention_weight_min": -41.24993133544922,
      "activations/layer14_attention_weight_max": 53.282596588134766,
      "activations/layer14_attention_weight_min": -34.750816345214844,
      "activations/layer15_attention_weight_max": 48.96206283569336,
      "activations/layer15_attention_weight_min": -35.617347717285156,
      "activations/layer16_attention_weight_max": 43.694786071777344,
      "activations/layer16_attention_weight_min": -31.841703414916992,
      "activations/layer17_attention_weight_max": 58.96811294555664,
      "activations/layer17_attention_weight_min": -37.519073486328125,
      "activations/layer18_attention_weight_max": 44.209537506103516,
      "activations/layer18_attention_weight_min": -33.183589935302734,
      "activations/layer19_attention_weight_max": 18.76375389099121,
      "activations/layer19_attention_weight_min": -12.961522102355957,
      "activations/layer1_attention_weight_max": 16.00952911376953,
      "activations/layer1_attention_weight_min": -10.289383888244629,
      "activations/layer20_attention_weight_max": 19.59349250793457,
      "activations/layer20_attention_weight_min": -13.774433135986328,
      "activations/layer21_attention_weight_max": 26.40087127685547,
      "activations/layer21_attention_weight_min": -15.192766189575195,
      "activations/layer22_attention_weight_max": 25.755037307739258,
      "activations/layer22_attention_weight_min": -18.41869354248047,
      "activations/layer23_attention_weight_max": 22.242626190185547,
      "activations/layer23_attention_weight_min": -19.98391342163086,
      "activations/layer2_attention_weight_max": 24.884830474853516,
      "activations/layer2_attention_weight_min": -21.320768356323242,
      "activations/layer3_attention_weight_max": 51.14759826660156,
      "activations/layer3_attention_weight_min": -49.205535888671875,
      "activations/layer4_attention_weight_max": 84.8770751953125,
      "activations/layer4_attention_weight_min": -76.16156768798828,
      "activations/layer5_attention_weight_max": 143.90463256835938,
      "activations/layer5_attention_weight_min": -127.39900970458984,
      "activations/layer6_attention_weight_max": 71.12826538085938,
      "activations/layer6_attention_weight_min": -61.34477615356445,
      "activations/layer7_attention_weight_max": 72.29521942138672,
      "activations/layer7_attention_weight_min": -64.09881591796875,
      "activations/layer8_attention_weight_max": 59.7686653137207,
      "activations/layer8_attention_weight_min": -50.815399169921875,
      "activations/layer9_attention_weight_max": 68.94377899169922,
      "activations/layer9_attention_weight_min": -60.48299026489258,
      "epoch": 0.74,
      "learning_rate": 0.00014669583333333332,
      "loss": 3.1578,
      "step": 12750
    },
    {
      "activations/layer0_attention_weight_max": 13.434840202331543,
      "activations/layer0_attention_weight_min": -10.78430461883545,
      "activations/layer10_attention_weight_max": 71.38508605957031,
      "activations/layer10_attention_weight_min": -61.89627456665039,
      "activations/layer11_attention_weight_max": 47.96698760986328,
      "activations/layer11_attention_weight_min": -40.92062759399414,
      "activations/layer12_attention_weight_max": 27.27016830444336,
      "activations/layer12_attention_weight_min": -24.26244354248047,
      "activations/layer13_attention_weight_max": 55.554569244384766,
      "activations/layer13_attention_weight_min": -44.54841995239258,
      "activations/layer14_attention_weight_max": 46.479339599609375,
      "activations/layer14_attention_weight_min": -39.458858489990234,
      "activations/layer15_attention_weight_max": 49.086795806884766,
      "activations/layer15_attention_weight_min": -40.46145248413086,
      "activations/layer16_attention_weight_max": 48.38846969604492,
      "activations/layer16_attention_weight_min": -35.88917541503906,
      "activations/layer17_attention_weight_max": 64.20658111572266,
      "activations/layer17_attention_weight_min": -42.27591323852539,
      "activations/layer18_attention_weight_max": 58.09166717529297,
      "activations/layer18_attention_weight_min": -40.597694396972656,
      "activations/layer19_attention_weight_max": 26.03836441040039,
      "activations/layer19_attention_weight_min": -11.747542381286621,
      "activations/layer1_attention_weight_max": 17.275794982910156,
      "activations/layer1_attention_weight_min": -10.346475601196289,
      "activations/layer20_attention_weight_max": 18.896133422851562,
      "activations/layer20_attention_weight_min": -12.444375038146973,
      "activations/layer21_attention_weight_max": 28.525358200073242,
      "activations/layer21_attention_weight_min": -18.99689292907715,
      "activations/layer22_attention_weight_max": 27.440088272094727,
      "activations/layer22_attention_weight_min": -16.766754150390625,
      "activations/layer23_attention_weight_max": 20.553447723388672,
      "activations/layer23_attention_weight_min": -21.99195671081543,
      "activations/layer2_attention_weight_max": 26.281673431396484,
      "activations/layer2_attention_weight_min": -21.483251571655273,
      "activations/layer3_attention_weight_max": 54.057315826416016,
      "activations/layer3_attention_weight_min": -51.75041198730469,
      "activations/layer4_attention_weight_max": 82.09589385986328,
      "activations/layer4_attention_weight_min": -72.48851776123047,
      "activations/layer5_attention_weight_max": 141.26437377929688,
      "activations/layer5_attention_weight_min": -125.413330078125,
      "activations/layer6_attention_weight_max": 71.00438690185547,
      "activations/layer6_attention_weight_min": -61.27952194213867,
      "activations/layer7_attention_weight_max": 72.33321380615234,
      "activations/layer7_attention_weight_min": -64.70047760009766,
      "activations/layer8_attention_weight_max": 61.74798583984375,
      "activations/layer8_attention_weight_min": -53.77528762817383,
      "activations/layer9_attention_weight_max": 79.09300994873047,
      "activations/layer9_attention_weight_min": -72.8703384399414,
      "epoch": 0.74,
      "learning_rate": 0.0001466768939393939,
      "loss": 3.1821,
      "step": 12800
    },
    {
      "activations/layer0_attention_weight_max": 13.354400634765625,
      "activations/layer0_attention_weight_min": -10.401224136352539,
      "activations/layer10_attention_weight_max": 69.24824523925781,
      "activations/layer10_attention_weight_min": -61.40824508666992,
      "activations/layer11_attention_weight_max": 49.40740203857422,
      "activations/layer11_attention_weight_min": -39.105499267578125,
      "activations/layer12_attention_weight_max": 28.954927444458008,
      "activations/layer12_attention_weight_min": -26.11095428466797,
      "activations/layer13_attention_weight_max": 50.99228286743164,
      "activations/layer13_attention_weight_min": -37.70609664916992,
      "activations/layer14_attention_weight_max": 46.11622619628906,
      "activations/layer14_attention_weight_min": -32.8569450378418,
      "activations/layer15_attention_weight_max": 54.440635681152344,
      "activations/layer15_attention_weight_min": -37.19862747192383,
      "activations/layer16_attention_weight_max": 46.2498664855957,
      "activations/layer16_attention_weight_min": -34.092926025390625,
      "activations/layer17_attention_weight_max": 55.260231018066406,
      "activations/layer17_attention_weight_min": -38.08143997192383,
      "activations/layer18_attention_weight_max": 51.11418914794922,
      "activations/layer18_attention_weight_min": -37.77513885498047,
      "activations/layer19_attention_weight_max": 17.167268753051758,
      "activations/layer19_attention_weight_min": -11.582442283630371,
      "activations/layer1_attention_weight_max": 16.222957611083984,
      "activations/layer1_attention_weight_min": -10.710969924926758,
      "activations/layer20_attention_weight_max": 19.38009262084961,
      "activations/layer20_attention_weight_min": -17.407859802246094,
      "activations/layer21_attention_weight_max": 29.394054412841797,
      "activations/layer21_attention_weight_min": -16.469104766845703,
      "activations/layer22_attention_weight_max": 26.70127296447754,
      "activations/layer22_attention_weight_min": -15.289685249328613,
      "activations/layer23_attention_weight_max": 21.0938720703125,
      "activations/layer23_attention_weight_min": -21.485097885131836,
      "activations/layer2_attention_weight_max": 24.57063102722168,
      "activations/layer2_attention_weight_min": -22.07754898071289,
      "activations/layer3_attention_weight_max": 49.942405700683594,
      "activations/layer3_attention_weight_min": -50.60774230957031,
      "activations/layer4_attention_weight_max": 83.31597137451172,
      "activations/layer4_attention_weight_min": -74.08870697021484,
      "activations/layer5_attention_weight_max": 139.6036834716797,
      "activations/layer5_attention_weight_min": -127.46005249023438,
      "activations/layer6_attention_weight_max": 69.28752136230469,
      "activations/layer6_attention_weight_min": -61.99896240234375,
      "activations/layer7_attention_weight_max": 71.85179138183594,
      "activations/layer7_attention_weight_min": -60.106754302978516,
      "activations/layer8_attention_weight_max": 64.60367584228516,
      "activations/layer8_attention_weight_min": -53.69012451171875,
      "activations/layer9_attention_weight_max": 78.12261962890625,
      "activations/layer9_attention_weight_min": -62.753448486328125,
      "epoch": 0.75,
      "learning_rate": 0.00014665795454545452,
      "loss": 3.1731,
      "step": 12850
    },
    {
      "activations/layer0_attention_weight_max": 13.398591995239258,
      "activations/layer0_attention_weight_min": -10.665042877197266,
      "activations/layer10_attention_weight_max": 68.15187072753906,
      "activations/layer10_attention_weight_min": -56.92798614501953,
      "activations/layer11_attention_weight_max": 47.34657287597656,
      "activations/layer11_attention_weight_min": -40.73274230957031,
      "activations/layer12_attention_weight_max": 34.36320114135742,
      "activations/layer12_attention_weight_min": -24.197736740112305,
      "activations/layer13_attention_weight_max": 53.796321868896484,
      "activations/layer13_attention_weight_min": -41.251155853271484,
      "activations/layer14_attention_weight_max": 44.43886947631836,
      "activations/layer14_attention_weight_min": -33.749000549316406,
      "activations/layer15_attention_weight_max": 52.73075866699219,
      "activations/layer15_attention_weight_min": -36.60582733154297,
      "activations/layer16_attention_weight_max": 51.48222351074219,
      "activations/layer16_attention_weight_min": -36.08668899536133,
      "activations/layer17_attention_weight_max": 67.19973754882812,
      "activations/layer17_attention_weight_min": -36.033199310302734,
      "activations/layer18_attention_weight_max": 51.0706901550293,
      "activations/layer18_attention_weight_min": -36.27253723144531,
      "activations/layer19_attention_weight_max": 22.74930763244629,
      "activations/layer19_attention_weight_min": -11.916952133178711,
      "activations/layer1_attention_weight_max": 17.164609909057617,
      "activations/layer1_attention_weight_min": -10.66079044342041,
      "activations/layer20_attention_weight_max": 19.651113510131836,
      "activations/layer20_attention_weight_min": -14.270210266113281,
      "activations/layer21_attention_weight_max": 33.76527786254883,
      "activations/layer21_attention_weight_min": -15.342451095581055,
      "activations/layer22_attention_weight_max": 33.74155044555664,
      "activations/layer22_attention_weight_min": -14.816813468933105,
      "activations/layer23_attention_weight_max": 25.64073944091797,
      "activations/layer23_attention_weight_min": -19.859628677368164,
      "activations/layer2_attention_weight_max": 26.10262107849121,
      "activations/layer2_attention_weight_min": -22.302289962768555,
      "activations/layer3_attention_weight_max": 52.01285934448242,
      "activations/layer3_attention_weight_min": -52.585601806640625,
      "activations/layer4_attention_weight_max": 77.80217742919922,
      "activations/layer4_attention_weight_min": -74.75188446044922,
      "activations/layer5_attention_weight_max": 139.14610290527344,
      "activations/layer5_attention_weight_min": -122.01141357421875,
      "activations/layer6_attention_weight_max": 72.14743041992188,
      "activations/layer6_attention_weight_min": -59.5238151550293,
      "activations/layer7_attention_weight_max": 73.47551727294922,
      "activations/layer7_attention_weight_min": -61.37334060668945,
      "activations/layer8_attention_weight_max": 62.18830871582031,
      "activations/layer8_attention_weight_min": -50.244140625,
      "activations/layer9_attention_weight_max": 76.11851501464844,
      "activations/layer9_attention_weight_min": -66.61935424804688,
      "epoch": 0.75,
      "learning_rate": 0.00014663901515151514,
      "loss": 3.1545,
      "step": 12900
    },
    {
      "activations/layer0_attention_weight_max": 14.230876922607422,
      "activations/layer0_attention_weight_min": -11.022916793823242,
      "activations/layer10_attention_weight_max": 68.52594757080078,
      "activations/layer10_attention_weight_min": -59.85211181640625,
      "activations/layer11_attention_weight_max": 52.316585540771484,
      "activations/layer11_attention_weight_min": -39.27447509765625,
      "activations/layer12_attention_weight_max": 29.59201431274414,
      "activations/layer12_attention_weight_min": -28.21883773803711,
      "activations/layer13_attention_weight_max": 57.03327560424805,
      "activations/layer13_attention_weight_min": -51.59623718261719,
      "activations/layer14_attention_weight_max": 57.641822814941406,
      "activations/layer14_attention_weight_min": -39.73447036743164,
      "activations/layer15_attention_weight_max": 54.44718551635742,
      "activations/layer15_attention_weight_min": -39.264583587646484,
      "activations/layer16_attention_weight_max": 40.037532806396484,
      "activations/layer16_attention_weight_min": -36.75272750854492,
      "activations/layer17_attention_weight_max": 49.33103561401367,
      "activations/layer17_attention_weight_min": -42.27809524536133,
      "activations/layer18_attention_weight_max": 48.06448745727539,
      "activations/layer18_attention_weight_min": -32.08881378173828,
      "activations/layer19_attention_weight_max": 17.020946502685547,
      "activations/layer19_attention_weight_min": -12.745989799499512,
      "activations/layer1_attention_weight_max": 16.790264129638672,
      "activations/layer1_attention_weight_min": -10.754887580871582,
      "activations/layer20_attention_weight_max": 18.509855270385742,
      "activations/layer20_attention_weight_min": -12.205263137817383,
      "activations/layer21_attention_weight_max": 32.35147476196289,
      "activations/layer21_attention_weight_min": -18.687530517578125,
      "activations/layer22_attention_weight_max": 29.230262756347656,
      "activations/layer22_attention_weight_min": -15.688919067382812,
      "activations/layer23_attention_weight_max": 19.30941390991211,
      "activations/layer23_attention_weight_min": -19.74237823486328,
      "activations/layer2_attention_weight_max": 26.346952438354492,
      "activations/layer2_attention_weight_min": -22.381378173828125,
      "activations/layer3_attention_weight_max": 53.10893630981445,
      "activations/layer3_attention_weight_min": -49.881404876708984,
      "activations/layer4_attention_weight_max": 82.45781707763672,
      "activations/layer4_attention_weight_min": -74.15702819824219,
      "activations/layer5_attention_weight_max": 144.63735961914062,
      "activations/layer5_attention_weight_min": -124.0332260131836,
      "activations/layer6_attention_weight_max": 77.05772399902344,
      "activations/layer6_attention_weight_min": -61.265525817871094,
      "activations/layer7_attention_weight_max": 77.59917449951172,
      "activations/layer7_attention_weight_min": -60.346656799316406,
      "activations/layer8_attention_weight_max": 61.85345458984375,
      "activations/layer8_attention_weight_min": -50.19102096557617,
      "activations/layer9_attention_weight_max": 79.70347595214844,
      "activations/layer9_attention_weight_min": -66.84579467773438,
      "epoch": 0.75,
      "learning_rate": 0.00014662007575757575,
      "loss": 3.163,
      "step": 12950
    },
    {
      "activations/layer0_attention_weight_max": 14.561610221862793,
      "activations/layer0_attention_weight_min": -10.956597328186035,
      "activations/layer10_attention_weight_max": 64.7433090209961,
      "activations/layer10_attention_weight_min": -59.97134780883789,
      "activations/layer11_attention_weight_max": 53.609474182128906,
      "activations/layer11_attention_weight_min": -42.96607971191406,
      "activations/layer12_attention_weight_max": 28.69659996032715,
      "activations/layer12_attention_weight_min": -26.618234634399414,
      "activations/layer13_attention_weight_max": 48.79060745239258,
      "activations/layer13_attention_weight_min": -41.446041107177734,
      "activations/layer14_attention_weight_max": 34.77751159667969,
      "activations/layer14_attention_weight_min": -29.710968017578125,
      "activations/layer15_attention_weight_max": 46.140159606933594,
      "activations/layer15_attention_weight_min": -35.84288787841797,
      "activations/layer16_attention_weight_max": 44.38273620605469,
      "activations/layer16_attention_weight_min": -36.30630874633789,
      "activations/layer17_attention_weight_max": 55.00442123413086,
      "activations/layer17_attention_weight_min": -39.4034309387207,
      "activations/layer18_attention_weight_max": 60.860511779785156,
      "activations/layer18_attention_weight_min": -34.364559173583984,
      "activations/layer19_attention_weight_max": 20.191913604736328,
      "activations/layer19_attention_weight_min": -12.131601333618164,
      "activations/layer1_attention_weight_max": 16.718534469604492,
      "activations/layer1_attention_weight_min": -10.655096054077148,
      "activations/layer20_attention_weight_max": 20.36052131652832,
      "activations/layer20_attention_weight_min": -12.909262657165527,
      "activations/layer21_attention_weight_max": 34.29143142700195,
      "activations/layer21_attention_weight_min": -18.141220092773438,
      "activations/layer22_attention_weight_max": 29.51495933532715,
      "activations/layer22_attention_weight_min": -16.503347396850586,
      "activations/layer23_attention_weight_max": 22.723112106323242,
      "activations/layer23_attention_weight_min": -19.311405181884766,
      "activations/layer2_attention_weight_max": 26.124710083007812,
      "activations/layer2_attention_weight_min": -22.424861907958984,
      "activations/layer3_attention_weight_max": 51.670166015625,
      "activations/layer3_attention_weight_min": -48.09843444824219,
      "activations/layer4_attention_weight_max": 82.49918365478516,
      "activations/layer4_attention_weight_min": -74.36119842529297,
      "activations/layer5_attention_weight_max": 137.85638427734375,
      "activations/layer5_attention_weight_min": -125.09568786621094,
      "activations/layer6_attention_weight_max": 72.40403747558594,
      "activations/layer6_attention_weight_min": -61.929115295410156,
      "activations/layer7_attention_weight_max": 73.74923706054688,
      "activations/layer7_attention_weight_min": -61.05460739135742,
      "activations/layer8_attention_weight_max": 61.00688552856445,
      "activations/layer8_attention_weight_min": -54.26243591308594,
      "activations/layer9_attention_weight_max": 72.48326873779297,
      "activations/layer9_attention_weight_min": -63.759151458740234,
      "epoch": 0.76,
      "learning_rate": 0.00014660113636363634,
      "loss": 3.1727,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_loss": 3.109375,
      "eval_runtime": 8.2261,
      "eval_samples_per_second": 521.994,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_openwebtext_loss": 3.109375,
      "eval_openwebtext_ppl": 22.40703562619394,
      "eval_openwebtext_runtime": 8.2261,
      "eval_openwebtext_samples_per_second": 521.994,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_wikitext_loss": 3.396484375,
      "eval_wikitext_ppl": 29.858942463791767,
      "eval_wikitext_runtime": 1.7133,
      "eval_wikitext_samples_per_second": 266.155,
      "step": 13000
    },
    {
      "epoch": 0.76,
      "eval_lambada_loss": 3.546875,
      "eval_lambada_ppl": 34.70469568042362,
      "eval_lambada_runtime": 9.2763,
      "eval_lambada_samples_per_second": 524.886,
      "step": 13000
    },
    {
      "activations/layer0_attention_weight_max": 14.556915283203125,
      "activations/layer0_attention_weight_min": -10.63121509552002,
      "activations/layer10_attention_weight_max": 62.70654296875,
      "activations/layer10_attention_weight_min": -62.76573944091797,
      "activations/layer11_attention_weight_max": 39.72512435913086,
      "activations/layer11_attention_weight_min": -38.26299285888672,
      "activations/layer12_attention_weight_max": 22.60192108154297,
      "activations/layer12_attention_weight_min": -23.002225875854492,
      "activations/layer13_attention_weight_max": 45.72745895385742,
      "activations/layer13_attention_weight_min": -40.06104278564453,
      "activations/layer14_attention_weight_max": 49.67532730102539,
      "activations/layer14_attention_weight_min": -41.69145965576172,
      "activations/layer15_attention_weight_max": 47.37685012817383,
      "activations/layer15_attention_weight_min": -37.63673400878906,
      "activations/layer16_attention_weight_max": 47.35856628417969,
      "activations/layer16_attention_weight_min": -36.58237075805664,
      "activations/layer17_attention_weight_max": 54.164146423339844,
      "activations/layer17_attention_weight_min": -36.48127365112305,
      "activations/layer18_attention_weight_max": 43.31184005737305,
      "activations/layer18_attention_weight_min": -32.49449920654297,
      "activations/layer19_attention_weight_max": 22.164594650268555,
      "activations/layer19_attention_weight_min": -13.119918823242188,
      "activations/layer1_attention_weight_max": 16.150835037231445,
      "activations/layer1_attention_weight_min": -10.828949928283691,
      "activations/layer20_attention_weight_max": 20.859203338623047,
      "activations/layer20_attention_weight_min": -12.600258827209473,
      "activations/layer21_attention_weight_max": 34.27573013305664,
      "activations/layer21_attention_weight_min": -14.032698631286621,
      "activations/layer22_attention_weight_max": 28.65831184387207,
      "activations/layer22_attention_weight_min": -16.73592758178711,
      "activations/layer23_attention_weight_max": 21.38162612915039,
      "activations/layer23_attention_weight_min": -21.02410125732422,
      "activations/layer2_attention_weight_max": 26.07221794128418,
      "activations/layer2_attention_weight_min": -21.3284912109375,
      "activations/layer3_attention_weight_max": 52.49740219116211,
      "activations/layer3_attention_weight_min": -48.61162185668945,
      "activations/layer4_attention_weight_max": 80.74727630615234,
      "activations/layer4_attention_weight_min": -71.31172180175781,
      "activations/layer5_attention_weight_max": 138.43124389648438,
      "activations/layer5_attention_weight_min": -125.1986083984375,
      "activations/layer6_attention_weight_max": 69.52168273925781,
      "activations/layer6_attention_weight_min": -59.60086441040039,
      "activations/layer7_attention_weight_max": 67.6265869140625,
      "activations/layer7_attention_weight_min": -59.025306701660156,
      "activations/layer8_attention_weight_max": 62.62428665161133,
      "activations/layer8_attention_weight_min": -55.827239990234375,
      "activations/layer9_attention_weight_max": 82.27713775634766,
      "activations/layer9_attention_weight_min": -73.83074951171875,
      "epoch": 0.76,
      "learning_rate": 0.00014658219696969696,
      "loss": 3.17,
      "step": 13050
    },
    {
      "activations/layer0_attention_weight_max": 15.494218826293945,
      "activations/layer0_attention_weight_min": -10.360762596130371,
      "activations/layer10_attention_weight_max": 63.62281036376953,
      "activations/layer10_attention_weight_min": -55.154624938964844,
      "activations/layer11_attention_weight_max": 44.73351287841797,
      "activations/layer11_attention_weight_min": -40.72949981689453,
      "activations/layer12_attention_weight_max": 27.125808715820312,
      "activations/layer12_attention_weight_min": -22.109174728393555,
      "activations/layer13_attention_weight_max": 51.538333892822266,
      "activations/layer13_attention_weight_min": -39.6956672668457,
      "activations/layer14_attention_weight_max": 47.973915100097656,
      "activations/layer14_attention_weight_min": -34.59585952758789,
      "activations/layer15_attention_weight_max": 52.572364807128906,
      "activations/layer15_attention_weight_min": -34.67888259887695,
      "activations/layer16_attention_weight_max": 48.039093017578125,
      "activations/layer16_attention_weight_min": -36.87889862060547,
      "activations/layer17_attention_weight_max": 57.24531936645508,
      "activations/layer17_attention_weight_min": -39.37474822998047,
      "activations/layer18_attention_weight_max": 50.19813537597656,
      "activations/layer18_attention_weight_min": -31.675729751586914,
      "activations/layer19_attention_weight_max": 20.21491050720215,
      "activations/layer19_attention_weight_min": -11.861502647399902,
      "activations/layer1_attention_weight_max": 16.262561798095703,
      "activations/layer1_attention_weight_min": -10.892852783203125,
      "activations/layer20_attention_weight_max": 19.264089584350586,
      "activations/layer20_attention_weight_min": -12.758735656738281,
      "activations/layer21_attention_weight_max": 34.07547378540039,
      "activations/layer21_attention_weight_min": -16.144729614257812,
      "activations/layer22_attention_weight_max": 28.37291717529297,
      "activations/layer22_attention_weight_min": -15.199926376342773,
      "activations/layer23_attention_weight_max": 23.79205322265625,
      "activations/layer23_attention_weight_min": -21.002822875976562,
      "activations/layer2_attention_weight_max": 26.650680541992188,
      "activations/layer2_attention_weight_min": -23.609548568725586,
      "activations/layer3_attention_weight_max": 51.98529052734375,
      "activations/layer3_attention_weight_min": -48.512516021728516,
      "activations/layer4_attention_weight_max": 81.51365661621094,
      "activations/layer4_attention_weight_min": -73.1512222290039,
      "activations/layer5_attention_weight_max": 141.3164520263672,
      "activations/layer5_attention_weight_min": -121.1427001953125,
      "activations/layer6_attention_weight_max": 74.08834838867188,
      "activations/layer6_attention_weight_min": -61.15968704223633,
      "activations/layer7_attention_weight_max": 69.26210021972656,
      "activations/layer7_attention_weight_min": -57.84014892578125,
      "activations/layer8_attention_weight_max": 55.20829772949219,
      "activations/layer8_attention_weight_min": -48.41834259033203,
      "activations/layer9_attention_weight_max": 76.6903305053711,
      "activations/layer9_attention_weight_min": -67.04774475097656,
      "epoch": 0.76,
      "learning_rate": 0.00014656325757575757,
      "loss": 3.1573,
      "step": 13100
    },
    {
      "activations/layer0_attention_weight_max": 15.517934799194336,
      "activations/layer0_attention_weight_min": -10.886691093444824,
      "activations/layer10_attention_weight_max": 70.91251373291016,
      "activations/layer10_attention_weight_min": -59.5835075378418,
      "activations/layer11_attention_weight_max": 41.388954162597656,
      "activations/layer11_attention_weight_min": -37.12987518310547,
      "activations/layer12_attention_weight_max": 25.719388961791992,
      "activations/layer12_attention_weight_min": -23.701019287109375,
      "activations/layer13_attention_weight_max": 59.492164611816406,
      "activations/layer13_attention_weight_min": -47.715511322021484,
      "activations/layer14_attention_weight_max": 57.64447021484375,
      "activations/layer14_attention_weight_min": -43.54168701171875,
      "activations/layer15_attention_weight_max": 57.90391159057617,
      "activations/layer15_attention_weight_min": -37.07536697387695,
      "activations/layer16_attention_weight_max": 46.21702194213867,
      "activations/layer16_attention_weight_min": -34.514163970947266,
      "activations/layer17_attention_weight_max": 53.3444709777832,
      "activations/layer17_attention_weight_min": -36.683837890625,
      "activations/layer18_attention_weight_max": 56.447479248046875,
      "activations/layer18_attention_weight_min": -40.04957580566406,
      "activations/layer19_attention_weight_max": 20.361604690551758,
      "activations/layer19_attention_weight_min": -12.152599334716797,
      "activations/layer1_attention_weight_max": 16.122234344482422,
      "activations/layer1_attention_weight_min": -11.509209632873535,
      "activations/layer20_attention_weight_max": 19.745296478271484,
      "activations/layer20_attention_weight_min": -13.72516918182373,
      "activations/layer21_attention_weight_max": 29.964445114135742,
      "activations/layer21_attention_weight_min": -15.117958068847656,
      "activations/layer22_attention_weight_max": 28.464794158935547,
      "activations/layer22_attention_weight_min": -15.905981063842773,
      "activations/layer23_attention_weight_max": 20.61638832092285,
      "activations/layer23_attention_weight_min": -20.648778915405273,
      "activations/layer2_attention_weight_max": 28.723663330078125,
      "activations/layer2_attention_weight_min": -22.633617401123047,
      "activations/layer3_attention_weight_max": 57.751834869384766,
      "activations/layer3_attention_weight_min": -52.399925231933594,
      "activations/layer4_attention_weight_max": 85.92317199707031,
      "activations/layer4_attention_weight_min": -74.41730499267578,
      "activations/layer5_attention_weight_max": 142.59890747070312,
      "activations/layer5_attention_weight_min": -123.84652709960938,
      "activations/layer6_attention_weight_max": 69.97185516357422,
      "activations/layer6_attention_weight_min": -63.91151809692383,
      "activations/layer7_attention_weight_max": 71.80654907226562,
      "activations/layer7_attention_weight_min": -63.53400802612305,
      "activations/layer8_attention_weight_max": 60.3399658203125,
      "activations/layer8_attention_weight_min": -54.17215347290039,
      "activations/layer9_attention_weight_max": 79.29700469970703,
      "activations/layer9_attention_weight_min": -68.16865539550781,
      "epoch": 0.76,
      "learning_rate": 0.00014654431818181816,
      "loss": 3.1712,
      "step": 13150
    },
    {
      "activations/layer0_attention_weight_max": 13.57106876373291,
      "activations/layer0_attention_weight_min": -10.615483283996582,
      "activations/layer10_attention_weight_max": 66.23065185546875,
      "activations/layer10_attention_weight_min": -58.5495719909668,
      "activations/layer11_attention_weight_max": 45.236419677734375,
      "activations/layer11_attention_weight_min": -42.43157196044922,
      "activations/layer12_attention_weight_max": 27.839136123657227,
      "activations/layer12_attention_weight_min": -24.67983627319336,
      "activations/layer13_attention_weight_max": 48.14655303955078,
      "activations/layer13_attention_weight_min": -39.47640609741211,
      "activations/layer14_attention_weight_max": 40.50932693481445,
      "activations/layer14_attention_weight_min": -34.0922737121582,
      "activations/layer15_attention_weight_max": 50.52254867553711,
      "activations/layer15_attention_weight_min": -36.819969177246094,
      "activations/layer16_attention_weight_max": 45.12586975097656,
      "activations/layer16_attention_weight_min": -31.359619140625,
      "activations/layer17_attention_weight_max": 55.873661041259766,
      "activations/layer17_attention_weight_min": -38.04380416870117,
      "activations/layer18_attention_weight_max": 59.75663757324219,
      "activations/layer18_attention_weight_min": -38.62611770629883,
      "activations/layer19_attention_weight_max": 18.952308654785156,
      "activations/layer19_attention_weight_min": -12.336166381835938,
      "activations/layer1_attention_weight_max": 17.2906551361084,
      "activations/layer1_attention_weight_min": -10.465993881225586,
      "activations/layer20_attention_weight_max": 18.528549194335938,
      "activations/layer20_attention_weight_min": -13.42238998413086,
      "activations/layer21_attention_weight_max": 29.550321578979492,
      "activations/layer21_attention_weight_min": -16.8291072845459,
      "activations/layer22_attention_weight_max": 25.659015655517578,
      "activations/layer22_attention_weight_min": -16.430213928222656,
      "activations/layer23_attention_weight_max": 21.90377426147461,
      "activations/layer23_attention_weight_min": -20.74663543701172,
      "activations/layer2_attention_weight_max": 24.292308807373047,
      "activations/layer2_attention_weight_min": -21.75350570678711,
      "activations/layer3_attention_weight_max": 48.159420013427734,
      "activations/layer3_attention_weight_min": -47.075016021728516,
      "activations/layer4_attention_weight_max": 75.49049377441406,
      "activations/layer4_attention_weight_min": -71.67760467529297,
      "activations/layer5_attention_weight_max": 127.56045532226562,
      "activations/layer5_attention_weight_min": -121.75144958496094,
      "activations/layer6_attention_weight_max": 62.1568489074707,
      "activations/layer6_attention_weight_min": -58.47528839111328,
      "activations/layer7_attention_weight_max": 63.726959228515625,
      "activations/layer7_attention_weight_min": -60.814720153808594,
      "activations/layer8_attention_weight_max": 55.895328521728516,
      "activations/layer8_attention_weight_min": -52.0576171875,
      "activations/layer9_attention_weight_max": 72.21708679199219,
      "activations/layer9_attention_weight_min": -64.12230682373047,
      "epoch": 0.77,
      "learning_rate": 0.00014652537878787877,
      "loss": 3.159,
      "step": 13200
    },
    {
      "activations/layer0_attention_weight_max": 13.369563102722168,
      "activations/layer0_attention_weight_min": -11.423781394958496,
      "activations/layer10_attention_weight_max": 83.42206573486328,
      "activations/layer10_attention_weight_min": -59.98278045654297,
      "activations/layer11_attention_weight_max": 53.6446533203125,
      "activations/layer11_attention_weight_min": -40.830772399902344,
      "activations/layer12_attention_weight_max": 27.568012237548828,
      "activations/layer12_attention_weight_min": -31.10106658935547,
      "activations/layer13_attention_weight_max": 70.12706756591797,
      "activations/layer13_attention_weight_min": -47.543209075927734,
      "activations/layer14_attention_weight_max": 52.713321685791016,
      "activations/layer14_attention_weight_min": -37.192222595214844,
      "activations/layer15_attention_weight_max": 49.45353698730469,
      "activations/layer15_attention_weight_min": -36.03239822387695,
      "activations/layer16_attention_weight_max": 43.3799934387207,
      "activations/layer16_attention_weight_min": -37.00608444213867,
      "activations/layer17_attention_weight_max": 56.57655715942383,
      "activations/layer17_attention_weight_min": -37.188480377197266,
      "activations/layer18_attention_weight_max": 47.62392044067383,
      "activations/layer18_attention_weight_min": -30.793432235717773,
      "activations/layer19_attention_weight_max": 20.866331100463867,
      "activations/layer19_attention_weight_min": -11.61607551574707,
      "activations/layer1_attention_weight_max": 16.576383590698242,
      "activations/layer1_attention_weight_min": -11.397974967956543,
      "activations/layer20_attention_weight_max": 21.307687759399414,
      "activations/layer20_attention_weight_min": -11.89338493347168,
      "activations/layer21_attention_weight_max": 33.76082229614258,
      "activations/layer21_attention_weight_min": -17.295358657836914,
      "activations/layer22_attention_weight_max": 30.84546661376953,
      "activations/layer22_attention_weight_min": -14.611421585083008,
      "activations/layer23_attention_weight_max": 21.71394920349121,
      "activations/layer23_attention_weight_min": -19.355979919433594,
      "activations/layer2_attention_weight_max": 29.134445190429688,
      "activations/layer2_attention_weight_min": -23.649097442626953,
      "activations/layer3_attention_weight_max": 55.163883209228516,
      "activations/layer3_attention_weight_min": -52.09994888305664,
      "activations/layer4_attention_weight_max": 84.25749969482422,
      "activations/layer4_attention_weight_min": -79.56134796142578,
      "activations/layer5_attention_weight_max": 155.27288818359375,
      "activations/layer5_attention_weight_min": -125.56118774414062,
      "activations/layer6_attention_weight_max": 79.62112426757812,
      "activations/layer6_attention_weight_min": -63.24885177612305,
      "activations/layer7_attention_weight_max": 78.78353118896484,
      "activations/layer7_attention_weight_min": -62.80429458618164,
      "activations/layer8_attention_weight_max": 68.09420776367188,
      "activations/layer8_attention_weight_min": -53.240848541259766,
      "activations/layer9_attention_weight_max": 84.14750671386719,
      "activations/layer9_attention_weight_min": -66.56475830078125,
      "epoch": 0.77,
      "learning_rate": 0.0001465064393939394,
      "loss": 3.1387,
      "step": 13250
    },
    {
      "activations/layer0_attention_weight_max": 13.728583335876465,
      "activations/layer0_attention_weight_min": -10.264278411865234,
      "activations/layer10_attention_weight_max": 72.67390441894531,
      "activations/layer10_attention_weight_min": -61.43474578857422,
      "activations/layer11_attention_weight_max": 51.31223678588867,
      "activations/layer11_attention_weight_min": -38.90037155151367,
      "activations/layer12_attention_weight_max": 28.2851505279541,
      "activations/layer12_attention_weight_min": -23.28343391418457,
      "activations/layer13_attention_weight_max": 56.78264617919922,
      "activations/layer13_attention_weight_min": -46.8802604675293,
      "activations/layer14_attention_weight_max": 58.27540969848633,
      "activations/layer14_attention_weight_min": -48.695823669433594,
      "activations/layer15_attention_weight_max": 48.95534133911133,
      "activations/layer15_attention_weight_min": -35.41877746582031,
      "activations/layer16_attention_weight_max": 46.25263214111328,
      "activations/layer16_attention_weight_min": -35.08475875854492,
      "activations/layer17_attention_weight_max": 54.68640899658203,
      "activations/layer17_attention_weight_min": -36.97487258911133,
      "activations/layer18_attention_weight_max": 63.25370788574219,
      "activations/layer18_attention_weight_min": -38.11421203613281,
      "activations/layer19_attention_weight_max": 21.092336654663086,
      "activations/layer19_attention_weight_min": -13.444908142089844,
      "activations/layer1_attention_weight_max": 16.614805221557617,
      "activations/layer1_attention_weight_min": -10.992232322692871,
      "activations/layer20_attention_weight_max": 22.1535701751709,
      "activations/layer20_attention_weight_min": -12.257216453552246,
      "activations/layer21_attention_weight_max": 38.99070739746094,
      "activations/layer21_attention_weight_min": -18.899450302124023,
      "activations/layer22_attention_weight_max": 32.45817565917969,
      "activations/layer22_attention_weight_min": -14.67757511138916,
      "activations/layer23_attention_weight_max": 23.253097534179688,
      "activations/layer23_attention_weight_min": -19.045074462890625,
      "activations/layer2_attention_weight_max": 27.63738250732422,
      "activations/layer2_attention_weight_min": -21.910764694213867,
      "activations/layer3_attention_weight_max": 53.68815231323242,
      "activations/layer3_attention_weight_min": -49.0439567565918,
      "activations/layer4_attention_weight_max": 82.03685760498047,
      "activations/layer4_attention_weight_min": -73.27696990966797,
      "activations/layer5_attention_weight_max": 134.8525848388672,
      "activations/layer5_attention_weight_min": -120.64564514160156,
      "activations/layer6_attention_weight_max": 70.64966583251953,
      "activations/layer6_attention_weight_min": -60.95676040649414,
      "activations/layer7_attention_weight_max": 68.27497100830078,
      "activations/layer7_attention_weight_min": -65.42021942138672,
      "activations/layer8_attention_weight_max": 60.56770706176758,
      "activations/layer8_attention_weight_min": -52.557647705078125,
      "activations/layer9_attention_weight_max": 77.74260711669922,
      "activations/layer9_attention_weight_min": -66.4268569946289,
      "epoch": 0.77,
      "learning_rate": 0.0001464875,
      "loss": 3.1529,
      "step": 13300
    },
    {
      "activations/layer0_attention_weight_max": 12.903306007385254,
      "activations/layer0_attention_weight_min": -10.410260200500488,
      "activations/layer10_attention_weight_max": 77.78514862060547,
      "activations/layer10_attention_weight_min": -62.71541213989258,
      "activations/layer11_attention_weight_max": 48.05980682373047,
      "activations/layer11_attention_weight_min": -39.768714904785156,
      "activations/layer12_attention_weight_max": 28.864036560058594,
      "activations/layer12_attention_weight_min": -21.49493408203125,
      "activations/layer13_attention_weight_max": 62.78509521484375,
      "activations/layer13_attention_weight_min": -50.2004280090332,
      "activations/layer14_attention_weight_max": 49.68973922729492,
      "activations/layer14_attention_weight_min": -38.65749740600586,
      "activations/layer15_attention_weight_max": 51.423667907714844,
      "activations/layer15_attention_weight_min": -37.667724609375,
      "activations/layer16_attention_weight_max": 51.76248550415039,
      "activations/layer16_attention_weight_min": -35.839576721191406,
      "activations/layer17_attention_weight_max": 58.67860794067383,
      "activations/layer17_attention_weight_min": -39.31532287597656,
      "activations/layer18_attention_weight_max": 65.46125793457031,
      "activations/layer18_attention_weight_min": -35.293479919433594,
      "activations/layer19_attention_weight_max": 20.449769973754883,
      "activations/layer19_attention_weight_min": -13.821341514587402,
      "activations/layer1_attention_weight_max": 16.508499145507812,
      "activations/layer1_attention_weight_min": -11.011466026306152,
      "activations/layer20_attention_weight_max": 20.131681442260742,
      "activations/layer20_attention_weight_min": -15.193814277648926,
      "activations/layer21_attention_weight_max": 31.403562545776367,
      "activations/layer21_attention_weight_min": -17.43356704711914,
      "activations/layer22_attention_weight_max": 30.844459533691406,
      "activations/layer22_attention_weight_min": -17.630582809448242,
      "activations/layer23_attention_weight_max": 23.0145263671875,
      "activations/layer23_attention_weight_min": -19.74212646484375,
      "activations/layer2_attention_weight_max": 27.840599060058594,
      "activations/layer2_attention_weight_min": -22.254392623901367,
      "activations/layer3_attention_weight_max": 54.93985366821289,
      "activations/layer3_attention_weight_min": -51.485076904296875,
      "activations/layer4_attention_weight_max": 82.7112045288086,
      "activations/layer4_attention_weight_min": -76.83612823486328,
      "activations/layer5_attention_weight_max": 141.8954315185547,
      "activations/layer5_attention_weight_min": -122.88194274902344,
      "activations/layer6_attention_weight_max": 71.6743392944336,
      "activations/layer6_attention_weight_min": -60.215675354003906,
      "activations/layer7_attention_weight_max": 76.32333374023438,
      "activations/layer7_attention_weight_min": -63.835323333740234,
      "activations/layer8_attention_weight_max": 60.86383056640625,
      "activations/layer8_attention_weight_min": -52.407440185546875,
      "activations/layer9_attention_weight_max": 70.6534652709961,
      "activations/layer9_attention_weight_min": -64.74827575683594,
      "epoch": 0.78,
      "learning_rate": 0.0001464685606060606,
      "loss": 3.162,
      "step": 13350
    },
    {
      "activations/layer0_attention_weight_max": 12.647640228271484,
      "activations/layer0_attention_weight_min": -10.991954803466797,
      "activations/layer10_attention_weight_max": 75.82136535644531,
      "activations/layer10_attention_weight_min": -69.30835723876953,
      "activations/layer11_attention_weight_max": 46.90388107299805,
      "activations/layer11_attention_weight_min": -38.23495101928711,
      "activations/layer12_attention_weight_max": 25.06964683532715,
      "activations/layer12_attention_weight_min": -23.887897491455078,
      "activations/layer13_attention_weight_max": 58.58543395996094,
      "activations/layer13_attention_weight_min": -41.27372741699219,
      "activations/layer14_attention_weight_max": 75.61695098876953,
      "activations/layer14_attention_weight_min": -41.928245544433594,
      "activations/layer15_attention_weight_max": 57.20677185058594,
      "activations/layer15_attention_weight_min": -35.27548599243164,
      "activations/layer16_attention_weight_max": 65.40599822998047,
      "activations/layer16_attention_weight_min": -36.493614196777344,
      "activations/layer17_attention_weight_max": 59.35192108154297,
      "activations/layer17_attention_weight_min": -40.91305160522461,
      "activations/layer18_attention_weight_max": 76.88409423828125,
      "activations/layer18_attention_weight_min": -38.29043960571289,
      "activations/layer19_attention_weight_max": 19.142263412475586,
      "activations/layer19_attention_weight_min": -11.028478622436523,
      "activations/layer1_attention_weight_max": 16.095394134521484,
      "activations/layer1_attention_weight_min": -10.188106536865234,
      "activations/layer20_attention_weight_max": 18.717187881469727,
      "activations/layer20_attention_weight_min": -15.985100746154785,
      "activations/layer21_attention_weight_max": 37.673095703125,
      "activations/layer21_attention_weight_min": -18.45407485961914,
      "activations/layer22_attention_weight_max": 31.82729721069336,
      "activations/layer22_attention_weight_min": -16.326873779296875,
      "activations/layer23_attention_weight_max": 22.82503890991211,
      "activations/layer23_attention_weight_min": -19.87464141845703,
      "activations/layer2_attention_weight_max": 25.6240234375,
      "activations/layer2_attention_weight_min": -20.632389068603516,
      "activations/layer3_attention_weight_max": 47.679466247558594,
      "activations/layer3_attention_weight_min": -44.32764434814453,
      "activations/layer4_attention_weight_max": 77.17201232910156,
      "activations/layer4_attention_weight_min": -69.59215545654297,
      "activations/layer5_attention_weight_max": 131.38665771484375,
      "activations/layer5_attention_weight_min": -118.9057846069336,
      "activations/layer6_attention_weight_max": 65.51852416992188,
      "activations/layer6_attention_weight_min": -57.54826736450195,
      "activations/layer7_attention_weight_max": 65.61409759521484,
      "activations/layer7_attention_weight_min": -60.51694869995117,
      "activations/layer8_attention_weight_max": 57.36133575439453,
      "activations/layer8_attention_weight_min": -51.1325569152832,
      "activations/layer9_attention_weight_max": 68.04051208496094,
      "activations/layer9_attention_weight_min": -64.9758529663086,
      "epoch": 0.78,
      "learning_rate": 0.0001464496212121212,
      "loss": 3.1461,
      "step": 13400
    },
    {
      "activations/layer0_attention_weight_max": 12.895611763000488,
      "activations/layer0_attention_weight_min": -10.4827880859375,
      "activations/layer10_attention_weight_max": 70.75275421142578,
      "activations/layer10_attention_weight_min": -64.9534683227539,
      "activations/layer11_attention_weight_max": 46.29217529296875,
      "activations/layer11_attention_weight_min": -43.15869140625,
      "activations/layer12_attention_weight_max": 26.582109451293945,
      "activations/layer12_attention_weight_min": -22.284194946289062,
      "activations/layer13_attention_weight_max": 63.61687088012695,
      "activations/layer13_attention_weight_min": -46.02098083496094,
      "activations/layer14_attention_weight_max": 51.03803253173828,
      "activations/layer14_attention_weight_min": -40.53722381591797,
      "activations/layer15_attention_weight_max": 53.431339263916016,
      "activations/layer15_attention_weight_min": -39.103538513183594,
      "activations/layer16_attention_weight_max": 49.96493148803711,
      "activations/layer16_attention_weight_min": -38.03351974487305,
      "activations/layer17_attention_weight_max": 55.98468017578125,
      "activations/layer17_attention_weight_min": -37.893394470214844,
      "activations/layer18_attention_weight_max": 56.13276672363281,
      "activations/layer18_attention_weight_min": -32.31688690185547,
      "activations/layer19_attention_weight_max": 17.640113830566406,
      "activations/layer19_attention_weight_min": -12.214937210083008,
      "activations/layer1_attention_weight_max": 16.909053802490234,
      "activations/layer1_attention_weight_min": -10.471388816833496,
      "activations/layer20_attention_weight_max": 20.07135009765625,
      "activations/layer20_attention_weight_min": -13.19246768951416,
      "activations/layer21_attention_weight_max": 28.354124069213867,
      "activations/layer21_attention_weight_min": -17.65639305114746,
      "activations/layer22_attention_weight_max": 26.033567428588867,
      "activations/layer22_attention_weight_min": -15.01846981048584,
      "activations/layer23_attention_weight_max": 22.278480529785156,
      "activations/layer23_attention_weight_min": -20.90730857849121,
      "activations/layer2_attention_weight_max": 26.081623077392578,
      "activations/layer2_attention_weight_min": -21.27666473388672,
      "activations/layer3_attention_weight_max": 52.66161346435547,
      "activations/layer3_attention_weight_min": -50.87074661254883,
      "activations/layer4_attention_weight_max": 81.1163558959961,
      "activations/layer4_attention_weight_min": -71.33134460449219,
      "activations/layer5_attention_weight_max": 138.65597534179688,
      "activations/layer5_attention_weight_min": -123.66670989990234,
      "activations/layer6_attention_weight_max": 69.22511291503906,
      "activations/layer6_attention_weight_min": -58.477806091308594,
      "activations/layer7_attention_weight_max": 68.2920913696289,
      "activations/layer7_attention_weight_min": -61.458641052246094,
      "activations/layer8_attention_weight_max": 57.83837890625,
      "activations/layer8_attention_weight_min": -51.932472229003906,
      "activations/layer9_attention_weight_max": 71.6790771484375,
      "activations/layer9_attention_weight_min": -65.03211975097656,
      "epoch": 0.78,
      "learning_rate": 0.00014643068181818182,
      "loss": 3.1615,
      "step": 13450
    },
    {
      "activations/layer0_attention_weight_max": 12.530423164367676,
      "activations/layer0_attention_weight_min": -10.664910316467285,
      "activations/layer10_attention_weight_max": 65.64322662353516,
      "activations/layer10_attention_weight_min": -55.88787078857422,
      "activations/layer11_attention_weight_max": 49.31594467163086,
      "activations/layer11_attention_weight_min": -45.33275604248047,
      "activations/layer12_attention_weight_max": 26.31156349182129,
      "activations/layer12_attention_weight_min": -22.427902221679688,
      "activations/layer13_attention_weight_max": 50.96665954589844,
      "activations/layer13_attention_weight_min": -40.22086715698242,
      "activations/layer14_attention_weight_max": 39.586204528808594,
      "activations/layer14_attention_weight_min": -34.30817794799805,
      "activations/layer15_attention_weight_max": 52.62348556518555,
      "activations/layer15_attention_weight_min": -41.1355094909668,
      "activations/layer16_attention_weight_max": 48.843387603759766,
      "activations/layer16_attention_weight_min": -36.289405822753906,
      "activations/layer17_attention_weight_max": 56.78355026245117,
      "activations/layer17_attention_weight_min": -39.45964813232422,
      "activations/layer18_attention_weight_max": 55.677734375,
      "activations/layer18_attention_weight_min": -45.04372787475586,
      "activations/layer19_attention_weight_max": 21.673160552978516,
      "activations/layer19_attention_weight_min": -12.641286849975586,
      "activations/layer1_attention_weight_max": 15.899162292480469,
      "activations/layer1_attention_weight_min": -10.045851707458496,
      "activations/layer20_attention_weight_max": 17.176254272460938,
      "activations/layer20_attention_weight_min": -14.421585083007812,
      "activations/layer21_attention_weight_max": 35.166709899902344,
      "activations/layer21_attention_weight_min": -16.847270965576172,
      "activations/layer22_attention_weight_max": 27.93302345275879,
      "activations/layer22_attention_weight_min": -14.987605094909668,
      "activations/layer23_attention_weight_max": 24.48119354248047,
      "activations/layer23_attention_weight_min": -19.756948471069336,
      "activations/layer2_attention_weight_max": 24.597991943359375,
      "activations/layer2_attention_weight_min": -21.23048210144043,
      "activations/layer3_attention_weight_max": 51.850006103515625,
      "activations/layer3_attention_weight_min": -48.255733489990234,
      "activations/layer4_attention_weight_max": 82.17815399169922,
      "activations/layer4_attention_weight_min": -73.6277847290039,
      "activations/layer5_attention_weight_max": 132.4587860107422,
      "activations/layer5_attention_weight_min": -119.16123962402344,
      "activations/layer6_attention_weight_max": 68.3919677734375,
      "activations/layer6_attention_weight_min": -60.40620040893555,
      "activations/layer7_attention_weight_max": 65.851318359375,
      "activations/layer7_attention_weight_min": -58.59355926513672,
      "activations/layer8_attention_weight_max": 55.591392517089844,
      "activations/layer8_attention_weight_min": -49.915000915527344,
      "activations/layer9_attention_weight_max": 78.48387908935547,
      "activations/layer9_attention_weight_min": -64.7770004272461,
      "epoch": 0.78,
      "learning_rate": 0.0001464117424242424,
      "loss": 3.1613,
      "step": 13500
    },
    {
      "activations/layer0_attention_weight_max": 13.737895011901855,
      "activations/layer0_attention_weight_min": -10.38529109954834,
      "activations/layer10_attention_weight_max": 77.18193817138672,
      "activations/layer10_attention_weight_min": -60.80623245239258,
      "activations/layer11_attention_weight_max": 51.50251388549805,
      "activations/layer11_attention_weight_min": -40.91782760620117,
      "activations/layer12_attention_weight_max": 30.623069763183594,
      "activations/layer12_attention_weight_min": -24.706378936767578,
      "activations/layer13_attention_weight_max": 62.31239318847656,
      "activations/layer13_attention_weight_min": -41.30730056762695,
      "activations/layer14_attention_weight_max": 57.887699127197266,
      "activations/layer14_attention_weight_min": -34.746315002441406,
      "activations/layer15_attention_weight_max": 55.56114196777344,
      "activations/layer15_attention_weight_min": -36.656463623046875,
      "activations/layer16_attention_weight_max": 43.01610565185547,
      "activations/layer16_attention_weight_min": -31.53813934326172,
      "activations/layer17_attention_weight_max": 55.38675308227539,
      "activations/layer17_attention_weight_min": -36.161048889160156,
      "activations/layer18_attention_weight_max": 65.0685043334961,
      "activations/layer18_attention_weight_min": -38.280147552490234,
      "activations/layer19_attention_weight_max": 21.145484924316406,
      "activations/layer19_attention_weight_min": -11.380228996276855,
      "activations/layer1_attention_weight_max": 16.527389526367188,
      "activations/layer1_attention_weight_min": -10.838086128234863,
      "activations/layer20_attention_weight_max": 16.786022186279297,
      "activations/layer20_attention_weight_min": -13.099217414855957,
      "activations/layer21_attention_weight_max": 30.214763641357422,
      "activations/layer21_attention_weight_min": -17.620685577392578,
      "activations/layer22_attention_weight_max": 29.54511833190918,
      "activations/layer22_attention_weight_min": -14.844340324401855,
      "activations/layer23_attention_weight_max": 21.469295501708984,
      "activations/layer23_attention_weight_min": -19.681053161621094,
      "activations/layer2_attention_weight_max": 25.64088249206543,
      "activations/layer2_attention_weight_min": -22.36186981201172,
      "activations/layer3_attention_weight_max": 54.044532775878906,
      "activations/layer3_attention_weight_min": -52.7251091003418,
      "activations/layer4_attention_weight_max": 86.8115463256836,
      "activations/layer4_attention_weight_min": -76.06609344482422,
      "activations/layer5_attention_weight_max": 152.59373474121094,
      "activations/layer5_attention_weight_min": -121.86778259277344,
      "activations/layer6_attention_weight_max": 75.3238754272461,
      "activations/layer6_attention_weight_min": -63.34269714355469,
      "activations/layer7_attention_weight_max": 74.18309783935547,
      "activations/layer7_attention_weight_min": -59.01322555541992,
      "activations/layer8_attention_weight_max": 66.56072998046875,
      "activations/layer8_attention_weight_min": -56.48598098754883,
      "activations/layer9_attention_weight_max": 91.38982391357422,
      "activations/layer9_attention_weight_min": -67.93232727050781,
      "epoch": 0.79,
      "learning_rate": 0.0001463931818181818,
      "loss": 3.1659,
      "step": 13550
    },
    {
      "activations/layer0_attention_weight_max": 14.504632949829102,
      "activations/layer0_attention_weight_min": -10.62070369720459,
      "activations/layer10_attention_weight_max": 71.5612564086914,
      "activations/layer10_attention_weight_min": -66.57605743408203,
      "activations/layer11_attention_weight_max": 48.426780700683594,
      "activations/layer11_attention_weight_min": -39.257293701171875,
      "activations/layer12_attention_weight_max": 28.394760131835938,
      "activations/layer12_attention_weight_min": -24.227127075195312,
      "activations/layer13_attention_weight_max": 63.240543365478516,
      "activations/layer13_attention_weight_min": -44.59577941894531,
      "activations/layer14_attention_weight_max": 57.889305114746094,
      "activations/layer14_attention_weight_min": -43.53782653808594,
      "activations/layer15_attention_weight_max": 49.53986358642578,
      "activations/layer15_attention_weight_min": -36.694034576416016,
      "activations/layer16_attention_weight_max": 51.099090576171875,
      "activations/layer16_attention_weight_min": -31.107133865356445,
      "activations/layer17_attention_weight_max": 58.99813461303711,
      "activations/layer17_attention_weight_min": -37.62600326538086,
      "activations/layer18_attention_weight_max": 54.255531311035156,
      "activations/layer18_attention_weight_min": -34.15122985839844,
      "activations/layer19_attention_weight_max": 22.190114974975586,
      "activations/layer19_attention_weight_min": -13.651529312133789,
      "activations/layer1_attention_weight_max": 16.835912704467773,
      "activations/layer1_attention_weight_min": -11.07189655303955,
      "activations/layer20_attention_weight_max": 18.083776473999023,
      "activations/layer20_attention_weight_min": -13.462072372436523,
      "activations/layer21_attention_weight_max": 33.468997955322266,
      "activations/layer21_attention_weight_min": -16.330251693725586,
      "activations/layer22_attention_weight_max": 29.981414794921875,
      "activations/layer22_attention_weight_min": -14.923376083374023,
      "activations/layer23_attention_weight_max": 20.581026077270508,
      "activations/layer23_attention_weight_min": -19.71271514892578,
      "activations/layer2_attention_weight_max": 25.06183433532715,
      "activations/layer2_attention_weight_min": -20.978361129760742,
      "activations/layer3_attention_weight_max": 50.226078033447266,
      "activations/layer3_attention_weight_min": -48.003028869628906,
      "activations/layer4_attention_weight_max": 78.13371276855469,
      "activations/layer4_attention_weight_min": -71.93827056884766,
      "activations/layer5_attention_weight_max": 135.7433624267578,
      "activations/layer5_attention_weight_min": -116.10328674316406,
      "activations/layer6_attention_weight_max": 68.64366912841797,
      "activations/layer6_attention_weight_min": -62.04450988769531,
      "activations/layer7_attention_weight_max": 68.50834655761719,
      "activations/layer7_attention_weight_min": -62.56024932861328,
      "activations/layer8_attention_weight_max": 56.21928787231445,
      "activations/layer8_attention_weight_min": -53.35527420043945,
      "activations/layer9_attention_weight_max": 79.78638458251953,
      "activations/layer9_attention_weight_min": -73.61296844482422,
      "epoch": 0.79,
      "learning_rate": 0.00014637424242424241,
      "loss": 3.1478,
      "step": 13600
    },
    {
      "activations/layer0_attention_weight_max": 12.660296440124512,
      "activations/layer0_attention_weight_min": -10.317694664001465,
      "activations/layer10_attention_weight_max": 79.94062805175781,
      "activations/layer10_attention_weight_min": -64.51312255859375,
      "activations/layer11_attention_weight_max": 43.63456344604492,
      "activations/layer11_attention_weight_min": -41.22911834716797,
      "activations/layer12_attention_weight_max": 32.33211135864258,
      "activations/layer12_attention_weight_min": -21.945146560668945,
      "activations/layer13_attention_weight_max": 75.57571411132812,
      "activations/layer13_attention_weight_min": -40.37309646606445,
      "activations/layer14_attention_weight_max": 98.50418090820312,
      "activations/layer14_attention_weight_min": -45.26709747314453,
      "activations/layer15_attention_weight_max": 62.712867736816406,
      "activations/layer15_attention_weight_min": -36.453224182128906,
      "activations/layer16_attention_weight_max": 49.04004669189453,
      "activations/layer16_attention_weight_min": -34.16679000854492,
      "activations/layer17_attention_weight_max": 59.61540603637695,
      "activations/layer17_attention_weight_min": -38.09282684326172,
      "activations/layer18_attention_weight_max": 54.445640563964844,
      "activations/layer18_attention_weight_min": -31.977365493774414,
      "activations/layer19_attention_weight_max": 17.583894729614258,
      "activations/layer19_attention_weight_min": -10.54136848449707,
      "activations/layer1_attention_weight_max": 16.71030044555664,
      "activations/layer1_attention_weight_min": -10.253799438476562,
      "activations/layer20_attention_weight_max": 19.404691696166992,
      "activations/layer20_attention_weight_min": -13.881856918334961,
      "activations/layer21_attention_weight_max": 27.264108657836914,
      "activations/layer21_attention_weight_min": -15.658366203308105,
      "activations/layer22_attention_weight_max": 24.64291000366211,
      "activations/layer22_attention_weight_min": -13.683162689208984,
      "activations/layer23_attention_weight_max": 18.94959259033203,
      "activations/layer23_attention_weight_min": -19.894590377807617,
      "activations/layer2_attention_weight_max": 24.015167236328125,
      "activations/layer2_attention_weight_min": -22.224346160888672,
      "activations/layer3_attention_weight_max": 50.41193771362305,
      "activations/layer3_attention_weight_min": -48.21977233886719,
      "activations/layer4_attention_weight_max": 76.5429458618164,
      "activations/layer4_attention_weight_min": -71.53722381591797,
      "activations/layer5_attention_weight_max": 138.7160186767578,
      "activations/layer5_attention_weight_min": -125.51554870605469,
      "activations/layer6_attention_weight_max": 71.75597381591797,
      "activations/layer6_attention_weight_min": -62.78952407836914,
      "activations/layer7_attention_weight_max": 72.48334503173828,
      "activations/layer7_attention_weight_min": -62.705238342285156,
      "activations/layer8_attention_weight_max": 61.82052993774414,
      "activations/layer8_attention_weight_min": -50.59941864013672,
      "activations/layer9_attention_weight_max": 87.06749725341797,
      "activations/layer9_attention_weight_min": -69.86817169189453,
      "epoch": 0.79,
      "learning_rate": 0.00014635530303030303,
      "loss": 3.1547,
      "step": 13650
    },
    {
      "activations/layer0_attention_weight_max": 14.364104270935059,
      "activations/layer0_attention_weight_min": -10.11684513092041,
      "activations/layer10_attention_weight_max": 75.6180648803711,
      "activations/layer10_attention_weight_min": -65.2093734741211,
      "activations/layer11_attention_weight_max": 46.50785827636719,
      "activations/layer11_attention_weight_min": -35.16145324707031,
      "activations/layer12_attention_weight_max": 27.96050453186035,
      "activations/layer12_attention_weight_min": -24.61347770690918,
      "activations/layer13_attention_weight_max": 59.802459716796875,
      "activations/layer13_attention_weight_min": -43.25298309326172,
      "activations/layer14_attention_weight_max": 50.783512115478516,
      "activations/layer14_attention_weight_min": -37.185462951660156,
      "activations/layer15_attention_weight_max": 50.3576774597168,
      "activations/layer15_attention_weight_min": -34.33040237426758,
      "activations/layer16_attention_weight_max": 48.45859146118164,
      "activations/layer16_attention_weight_min": -34.83992385864258,
      "activations/layer17_attention_weight_max": 60.81432342529297,
      "activations/layer17_attention_weight_min": -36.364356994628906,
      "activations/layer18_attention_weight_max": 48.13557052612305,
      "activations/layer18_attention_weight_min": -31.70974349975586,
      "activations/layer19_attention_weight_max": 19.27850341796875,
      "activations/layer19_attention_weight_min": -13.773773193359375,
      "activations/layer1_attention_weight_max": 15.815483093261719,
      "activations/layer1_attention_weight_min": -12.291902542114258,
      "activations/layer20_attention_weight_max": 16.791053771972656,
      "activations/layer20_attention_weight_min": -13.258092880249023,
      "activations/layer21_attention_weight_max": 29.90941619873047,
      "activations/layer21_attention_weight_min": -19.889984130859375,
      "activations/layer22_attention_weight_max": 26.7182674407959,
      "activations/layer22_attention_weight_min": -16.19183921813965,
      "activations/layer23_attention_weight_max": 20.989524841308594,
      "activations/layer23_attention_weight_min": -20.95322036743164,
      "activations/layer2_attention_weight_max": 25.111722946166992,
      "activations/layer2_attention_weight_min": -22.568378448486328,
      "activations/layer3_attention_weight_max": 54.17070770263672,
      "activations/layer3_attention_weight_min": -52.32674789428711,
      "activations/layer4_attention_weight_max": 84.5839614868164,
      "activations/layer4_attention_weight_min": -71.32090759277344,
      "activations/layer5_attention_weight_max": 146.90602111816406,
      "activations/layer5_attention_weight_min": -121.85238647460938,
      "activations/layer6_attention_weight_max": 73.58132934570312,
      "activations/layer6_attention_weight_min": -60.9505729675293,
      "activations/layer7_attention_weight_max": 70.98413848876953,
      "activations/layer7_attention_weight_min": -60.150753021240234,
      "activations/layer8_attention_weight_max": 59.60662078857422,
      "activations/layer8_attention_weight_min": -51.149539947509766,
      "activations/layer9_attention_weight_max": 72.0799331665039,
      "activations/layer9_attention_weight_min": -70.74622344970703,
      "epoch": 0.8,
      "learning_rate": 0.00014633636363636362,
      "loss": 3.1471,
      "step": 13700
    },
    {
      "activations/layer0_attention_weight_max": 13.282197952270508,
      "activations/layer0_attention_weight_min": -10.297094345092773,
      "activations/layer10_attention_weight_max": 83.15924835205078,
      "activations/layer10_attention_weight_min": -63.80708694458008,
      "activations/layer11_attention_weight_max": 49.91863250732422,
      "activations/layer11_attention_weight_min": -39.56053924560547,
      "activations/layer12_attention_weight_max": 24.359880447387695,
      "activations/layer12_attention_weight_min": -31.795141220092773,
      "activations/layer13_attention_weight_max": 58.4236946105957,
      "activations/layer13_attention_weight_min": -39.23218536376953,
      "activations/layer14_attention_weight_max": 59.885353088378906,
      "activations/layer14_attention_weight_min": -40.112510681152344,
      "activations/layer15_attention_weight_max": 49.377532958984375,
      "activations/layer15_attention_weight_min": -36.0382080078125,
      "activations/layer16_attention_weight_max": 46.78901290893555,
      "activations/layer16_attention_weight_min": -34.83433532714844,
      "activations/layer17_attention_weight_max": 59.13732147216797,
      "activations/layer17_attention_weight_min": -34.47180938720703,
      "activations/layer18_attention_weight_max": 44.563697814941406,
      "activations/layer18_attention_weight_min": -26.523792266845703,
      "activations/layer19_attention_weight_max": 21.275833129882812,
      "activations/layer19_attention_weight_min": -11.873344421386719,
      "activations/layer1_attention_weight_max": 16.706520080566406,
      "activations/layer1_attention_weight_min": -11.106677055358887,
      "activations/layer20_attention_weight_max": 21.987485885620117,
      "activations/layer20_attention_weight_min": -12.283571243286133,
      "activations/layer21_attention_weight_max": 32.96522521972656,
      "activations/layer21_attention_weight_min": -17.61637306213379,
      "activations/layer22_attention_weight_max": 34.01187515258789,
      "activations/layer22_attention_weight_min": -13.399909019470215,
      "activations/layer23_attention_weight_max": 22.64027214050293,
      "activations/layer23_attention_weight_min": -19.66888999938965,
      "activations/layer2_attention_weight_max": 23.49868392944336,
      "activations/layer2_attention_weight_min": -20.95700454711914,
      "activations/layer3_attention_weight_max": 54.619075775146484,
      "activations/layer3_attention_weight_min": -56.232261657714844,
      "activations/layer4_attention_weight_max": 87.83373260498047,
      "activations/layer4_attention_weight_min": -80.34668731689453,
      "activations/layer5_attention_weight_max": 158.3504180908203,
      "activations/layer5_attention_weight_min": -129.159912109375,
      "activations/layer6_attention_weight_max": 77.3145751953125,
      "activations/layer6_attention_weight_min": -63.10353088378906,
      "activations/layer7_attention_weight_max": 81.21290588378906,
      "activations/layer7_attention_weight_min": -60.49276351928711,
      "activations/layer8_attention_weight_max": 75.11553955078125,
      "activations/layer8_attention_weight_min": -57.08822250366211,
      "activations/layer9_attention_weight_max": 96.74244689941406,
      "activations/layer9_attention_weight_min": -74.21598815917969,
      "epoch": 0.8,
      "learning_rate": 0.00014631742424242423,
      "loss": 3.1653,
      "step": 13750
    },
    {
      "activations/layer0_attention_weight_max": 13.831522941589355,
      "activations/layer0_attention_weight_min": -10.579151153564453,
      "activations/layer10_attention_weight_max": 60.967010498046875,
      "activations/layer10_attention_weight_min": -54.95148849487305,
      "activations/layer11_attention_weight_max": 44.332271575927734,
      "activations/layer11_attention_weight_min": -39.35308837890625,
      "activations/layer12_attention_weight_max": 27.70481300354004,
      "activations/layer12_attention_weight_min": -23.136505126953125,
      "activations/layer13_attention_weight_max": 65.05461883544922,
      "activations/layer13_attention_weight_min": -55.54867935180664,
      "activations/layer14_attention_weight_max": 49.36789321899414,
      "activations/layer14_attention_weight_min": -39.889198303222656,
      "activations/layer15_attention_weight_max": 55.84550476074219,
      "activations/layer15_attention_weight_min": -40.333160400390625,
      "activations/layer16_attention_weight_max": 54.54253005981445,
      "activations/layer16_attention_weight_min": -40.990631103515625,
      "activations/layer17_attention_weight_max": 63.96294403076172,
      "activations/layer17_attention_weight_min": -40.06782531738281,
      "activations/layer18_attention_weight_max": 66.89210510253906,
      "activations/layer18_attention_weight_min": -40.0264892578125,
      "activations/layer19_attention_weight_max": 22.17512321472168,
      "activations/layer19_attention_weight_min": -12.294251441955566,
      "activations/layer1_attention_weight_max": 16.880178451538086,
      "activations/layer1_attention_weight_min": -10.089116096496582,
      "activations/layer20_attention_weight_max": 18.728376388549805,
      "activations/layer20_attention_weight_min": -12.127809524536133,
      "activations/layer21_attention_weight_max": 32.458858489990234,
      "activations/layer21_attention_weight_min": -17.834522247314453,
      "activations/layer22_attention_weight_max": 32.08679962158203,
      "activations/layer22_attention_weight_min": -17.82065773010254,
      "activations/layer23_attention_weight_max": 21.48876190185547,
      "activations/layer23_attention_weight_min": -19.134502410888672,
      "activations/layer2_attention_weight_max": 24.406278610229492,
      "activations/layer2_attention_weight_min": -22.130413055419922,
      "activations/layer3_attention_weight_max": 52.24518585205078,
      "activations/layer3_attention_weight_min": -52.00825500488281,
      "activations/layer4_attention_weight_max": 82.26559448242188,
      "activations/layer4_attention_weight_min": -74.63349151611328,
      "activations/layer5_attention_weight_max": 134.42259216308594,
      "activations/layer5_attention_weight_min": -124.14468383789062,
      "activations/layer6_attention_weight_max": 65.9877700805664,
      "activations/layer6_attention_weight_min": -57.209800720214844,
      "activations/layer7_attention_weight_max": 63.32376480102539,
      "activations/layer7_attention_weight_min": -57.56521987915039,
      "activations/layer8_attention_weight_max": 56.612274169921875,
      "activations/layer8_attention_weight_min": -49.673404693603516,
      "activations/layer9_attention_weight_max": 66.53155517578125,
      "activations/layer9_attention_weight_min": -66.78402709960938,
      "epoch": 0.8,
      "learning_rate": 0.00014629848484848485,
      "loss": 3.1533,
      "step": 13800
    },
    {
      "activations/layer0_attention_weight_max": 13.394918441772461,
      "activations/layer0_attention_weight_min": -10.38361644744873,
      "activations/layer10_attention_weight_max": 63.73047637939453,
      "activations/layer10_attention_weight_min": -55.40693283081055,
      "activations/layer11_attention_weight_max": 47.13084411621094,
      "activations/layer11_attention_weight_min": -42.28325653076172,
      "activations/layer12_attention_weight_max": 28.1708927154541,
      "activations/layer12_attention_weight_min": -24.193002700805664,
      "activations/layer13_attention_weight_max": 67.022216796875,
      "activations/layer13_attention_weight_min": -48.38045120239258,
      "activations/layer14_attention_weight_max": 46.916114807128906,
      "activations/layer14_attention_weight_min": -33.7114143371582,
      "activations/layer15_attention_weight_max": 52.58335876464844,
      "activations/layer15_attention_weight_min": -37.50138854980469,
      "activations/layer16_attention_weight_max": 54.97456359863281,
      "activations/layer16_attention_weight_min": -35.49430465698242,
      "activations/layer17_attention_weight_max": 56.75773620605469,
      "activations/layer17_attention_weight_min": -40.76900100708008,
      "activations/layer18_attention_weight_max": 51.33280563354492,
      "activations/layer18_attention_weight_min": -40.38316345214844,
      "activations/layer19_attention_weight_max": 18.87891387939453,
      "activations/layer19_attention_weight_min": -11.937387466430664,
      "activations/layer1_attention_weight_max": 16.08945083618164,
      "activations/layer1_attention_weight_min": -10.787970542907715,
      "activations/layer20_attention_weight_max": 17.285839080810547,
      "activations/layer20_attention_weight_min": -13.983550071716309,
      "activations/layer21_attention_weight_max": 31.634410858154297,
      "activations/layer21_attention_weight_min": -17.474611282348633,
      "activations/layer22_attention_weight_max": 27.30080795288086,
      "activations/layer22_attention_weight_min": -14.702143669128418,
      "activations/layer23_attention_weight_max": 20.53034210205078,
      "activations/layer23_attention_weight_min": -20.766315460205078,
      "activations/layer2_attention_weight_max": 24.28097152709961,
      "activations/layer2_attention_weight_min": -23.396652221679688,
      "activations/layer3_attention_weight_max": 52.80391311645508,
      "activations/layer3_attention_weight_min": -51.709877014160156,
      "activations/layer4_attention_weight_max": 83.41002655029297,
      "activations/layer4_attention_weight_min": -75.1944351196289,
      "activations/layer5_attention_weight_max": 141.12921142578125,
      "activations/layer5_attention_weight_min": -121.50728607177734,
      "activations/layer6_attention_weight_max": 68.51075744628906,
      "activations/layer6_attention_weight_min": -60.27943801879883,
      "activations/layer7_attention_weight_max": 69.24856567382812,
      "activations/layer7_attention_weight_min": -59.79239273071289,
      "activations/layer8_attention_weight_max": 65.44817352294922,
      "activations/layer8_attention_weight_min": -54.518924713134766,
      "activations/layer9_attention_weight_max": 69.38079071044922,
      "activations/layer9_attention_weight_min": -65.55592346191406,
      "epoch": 0.8,
      "learning_rate": 0.00014627954545454544,
      "loss": 3.1387,
      "step": 13850
    },
    {
      "activations/layer0_attention_weight_max": 15.263521194458008,
      "activations/layer0_attention_weight_min": -10.342899322509766,
      "activations/layer10_attention_weight_max": 74.9991683959961,
      "activations/layer10_attention_weight_min": -65.27195739746094,
      "activations/layer11_attention_weight_max": 45.55573654174805,
      "activations/layer11_attention_weight_min": -42.45163345336914,
      "activations/layer12_attention_weight_max": 27.431392669677734,
      "activations/layer12_attention_weight_min": -25.271299362182617,
      "activations/layer13_attention_weight_max": 59.5804557800293,
      "activations/layer13_attention_weight_min": -42.831809997558594,
      "activations/layer14_attention_weight_max": 50.85913848876953,
      "activations/layer14_attention_weight_min": -36.94070053100586,
      "activations/layer15_attention_weight_max": 51.55791473388672,
      "activations/layer15_attention_weight_min": -38.12211990356445,
      "activations/layer16_attention_weight_max": 50.09421157836914,
      "activations/layer16_attention_weight_min": -37.81230545043945,
      "activations/layer17_attention_weight_max": 56.459442138671875,
      "activations/layer17_attention_weight_min": -40.30781555175781,
      "activations/layer18_attention_weight_max": 53.20531463623047,
      "activations/layer18_attention_weight_min": -34.945369720458984,
      "activations/layer19_attention_weight_max": 20.819660186767578,
      "activations/layer19_attention_weight_min": -13.043177604675293,
      "activations/layer1_attention_weight_max": 17.511598587036133,
      "activations/layer1_attention_weight_min": -11.80734634399414,
      "activations/layer20_attention_weight_max": 19.602968215942383,
      "activations/layer20_attention_weight_min": -12.142683982849121,
      "activations/layer21_attention_weight_max": 31.90084457397461,
      "activations/layer21_attention_weight_min": -17.25704002380371,
      "activations/layer22_attention_weight_max": 33.21596145629883,
      "activations/layer22_attention_weight_min": -15.58138370513916,
      "activations/layer23_attention_weight_max": 23.27118492126465,
      "activations/layer23_attention_weight_min": -19.354135513305664,
      "activations/layer2_attention_weight_max": 24.266498565673828,
      "activations/layer2_attention_weight_min": -21.81508445739746,
      "activations/layer3_attention_weight_max": 51.16849136352539,
      "activations/layer3_attention_weight_min": -48.67329025268555,
      "activations/layer4_attention_weight_max": 78.54898071289062,
      "activations/layer4_attention_weight_min": -72.53649139404297,
      "activations/layer5_attention_weight_max": 133.00802612304688,
      "activations/layer5_attention_weight_min": -122.14379119873047,
      "activations/layer6_attention_weight_max": 64.67377471923828,
      "activations/layer6_attention_weight_min": -58.11576461791992,
      "activations/layer7_attention_weight_max": 65.60523986816406,
      "activations/layer7_attention_weight_min": -59.59742736816406,
      "activations/layer8_attention_weight_max": 59.34807205200195,
      "activations/layer8_attention_weight_min": -52.475379943847656,
      "activations/layer9_attention_weight_max": 89.64492797851562,
      "activations/layer9_attention_weight_min": -77.07880401611328,
      "epoch": 0.81,
      "learning_rate": 0.00014626060606060605,
      "loss": 3.1719,
      "step": 13900
    },
    {
      "activations/layer0_attention_weight_max": 15.588851928710938,
      "activations/layer0_attention_weight_min": -10.769325256347656,
      "activations/layer10_attention_weight_max": 61.96648406982422,
      "activations/layer10_attention_weight_min": -58.2255859375,
      "activations/layer11_attention_weight_max": 48.23215866088867,
      "activations/layer11_attention_weight_min": -39.33129119873047,
      "activations/layer12_attention_weight_max": 29.512557983398438,
      "activations/layer12_attention_weight_min": -22.4675350189209,
      "activations/layer13_attention_weight_max": 62.58388137817383,
      "activations/layer13_attention_weight_min": -46.52603530883789,
      "activations/layer14_attention_weight_max": 53.70378875732422,
      "activations/layer14_attention_weight_min": -37.545623779296875,
      "activations/layer15_attention_weight_max": 55.93775177001953,
      "activations/layer15_attention_weight_min": -40.216251373291016,
      "activations/layer16_attention_weight_max": 52.20389175415039,
      "activations/layer16_attention_weight_min": -33.17720031738281,
      "activations/layer17_attention_weight_max": 60.97299575805664,
      "activations/layer17_attention_weight_min": -39.3554801940918,
      "activations/layer18_attention_weight_max": 51.286766052246094,
      "activations/layer18_attention_weight_min": -31.885107040405273,
      "activations/layer19_attention_weight_max": 19.464994430541992,
      "activations/layer19_attention_weight_min": -14.583050727844238,
      "activations/layer1_attention_weight_max": 15.963764190673828,
      "activations/layer1_attention_weight_min": -10.865473747253418,
      "activations/layer20_attention_weight_max": 19.712980270385742,
      "activations/layer20_attention_weight_min": -13.443528175354004,
      "activations/layer21_attention_weight_max": 32.6292610168457,
      "activations/layer21_attention_weight_min": -17.709243774414062,
      "activations/layer22_attention_weight_max": 30.386859893798828,
      "activations/layer22_attention_weight_min": -17.870927810668945,
      "activations/layer23_attention_weight_max": 23.055166244506836,
      "activations/layer23_attention_weight_min": -18.985607147216797,
      "activations/layer2_attention_weight_max": 25.201147079467773,
      "activations/layer2_attention_weight_min": -21.17225456237793,
      "activations/layer3_attention_weight_max": 53.452537536621094,
      "activations/layer3_attention_weight_min": -51.41621780395508,
      "activations/layer4_attention_weight_max": 86.77708435058594,
      "activations/layer4_attention_weight_min": -75.9312973022461,
      "activations/layer5_attention_weight_max": 141.63699340820312,
      "activations/layer5_attention_weight_min": -129.1728515625,
      "activations/layer6_attention_weight_max": 66.715087890625,
      "activations/layer6_attention_weight_min": -58.77131652832031,
      "activations/layer7_attention_weight_max": 66.11709594726562,
      "activations/layer7_attention_weight_min": -60.00608444213867,
      "activations/layer8_attention_weight_max": 53.839881896972656,
      "activations/layer8_attention_weight_min": -48.83900833129883,
      "activations/layer9_attention_weight_max": 69.98004150390625,
      "activations/layer9_attention_weight_min": -65.47270965576172,
      "epoch": 0.81,
      "learning_rate": 0.00014624166666666664,
      "loss": 3.1521,
      "step": 13950
    },
    {
      "activations/layer0_attention_weight_max": 13.497725486755371,
      "activations/layer0_attention_weight_min": -10.760601997375488,
      "activations/layer10_attention_weight_max": 67.55555725097656,
      "activations/layer10_attention_weight_min": -61.94083786010742,
      "activations/layer11_attention_weight_max": 43.16355514526367,
      "activations/layer11_attention_weight_min": -39.14466094970703,
      "activations/layer12_attention_weight_max": 29.94294548034668,
      "activations/layer12_attention_weight_min": -23.792219161987305,
      "activations/layer13_attention_weight_max": 54.82793045043945,
      "activations/layer13_attention_weight_min": -42.80083465576172,
      "activations/layer14_attention_weight_max": 45.71257400512695,
      "activations/layer14_attention_weight_min": -35.43699264526367,
      "activations/layer15_attention_weight_max": 54.207481384277344,
      "activations/layer15_attention_weight_min": -36.9366340637207,
      "activations/layer16_attention_weight_max": 47.722408294677734,
      "activations/layer16_attention_weight_min": -37.337158203125,
      "activations/layer17_attention_weight_max": 59.68245315551758,
      "activations/layer17_attention_weight_min": -38.42694854736328,
      "activations/layer18_attention_weight_max": 59.21754837036133,
      "activations/layer18_attention_weight_min": -40.11421203613281,
      "activations/layer19_attention_weight_max": 20.652664184570312,
      "activations/layer19_attention_weight_min": -11.945478439331055,
      "activations/layer1_attention_weight_max": 16.961259841918945,
      "activations/layer1_attention_weight_min": -10.934550285339355,
      "activations/layer20_attention_weight_max": 21.290815353393555,
      "activations/layer20_attention_weight_min": -13.107246398925781,
      "activations/layer21_attention_weight_max": 35.767555236816406,
      "activations/layer21_attention_weight_min": -19.339126586914062,
      "activations/layer22_attention_weight_max": 32.36960983276367,
      "activations/layer22_attention_weight_min": -16.765695571899414,
      "activations/layer23_attention_weight_max": 23.52823257446289,
      "activations/layer23_attention_weight_min": -20.009471893310547,
      "activations/layer2_attention_weight_max": 25.778461456298828,
      "activations/layer2_attention_weight_min": -21.908653259277344,
      "activations/layer3_attention_weight_max": 51.936832427978516,
      "activations/layer3_attention_weight_min": -50.71181869506836,
      "activations/layer4_attention_weight_max": 80.31198120117188,
      "activations/layer4_attention_weight_min": -72.8380126953125,
      "activations/layer5_attention_weight_max": 139.5509033203125,
      "activations/layer5_attention_weight_min": -119.136962890625,
      "activations/layer6_attention_weight_max": 68.96630096435547,
      "activations/layer6_attention_weight_min": -59.32678985595703,
      "activations/layer7_attention_weight_max": 66.6397705078125,
      "activations/layer7_attention_weight_min": -59.90855407714844,
      "activations/layer8_attention_weight_max": 56.62122344970703,
      "activations/layer8_attention_weight_min": -52.80986404418945,
      "activations/layer9_attention_weight_max": 77.30970001220703,
      "activations/layer9_attention_weight_min": -63.805999755859375,
      "epoch": 0.81,
      "learning_rate": 0.00014622272727272725,
      "loss": 3.1325,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_loss": 3.091796875,
      "eval_runtime": 8.2448,
      "eval_samples_per_second": 520.814,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_openwebtext_loss": 3.091796875,
      "eval_openwebtext_ppl": 22.016603538363892,
      "eval_openwebtext_runtime": 8.2448,
      "eval_openwebtext_samples_per_second": 520.814,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_wikitext_loss": 3.384765625,
      "eval_wikitext_ppl": 29.511075247281195,
      "eval_wikitext_runtime": 1.7087,
      "eval_wikitext_samples_per_second": 266.875,
      "step": 14000
    },
    {
      "epoch": 0.81,
      "eval_lambada_loss": 3.630859375,
      "eval_lambada_ppl": 37.74523999888851,
      "eval_lambada_runtime": 9.3048,
      "eval_lambada_samples_per_second": 523.276,
      "step": 14000
    },
    {
      "activations/layer0_attention_weight_max": 13.845379829406738,
      "activations/layer0_attention_weight_min": -10.552481651306152,
      "activations/layer10_attention_weight_max": 66.61640167236328,
      "activations/layer10_attention_weight_min": -55.12092971801758,
      "activations/layer11_attention_weight_max": 43.670867919921875,
      "activations/layer11_attention_weight_min": -41.40845489501953,
      "activations/layer12_attention_weight_max": 24.80045509338379,
      "activations/layer12_attention_weight_min": -22.18531036376953,
      "activations/layer13_attention_weight_max": 53.25318908691406,
      "activations/layer13_attention_weight_min": -46.413944244384766,
      "activations/layer14_attention_weight_max": 43.72499465942383,
      "activations/layer14_attention_weight_min": -37.50347137451172,
      "activations/layer15_attention_weight_max": 50.91226577758789,
      "activations/layer15_attention_weight_min": -35.76864242553711,
      "activations/layer16_attention_weight_max": 48.27574920654297,
      "activations/layer16_attention_weight_min": -37.67329406738281,
      "activations/layer17_attention_weight_max": 55.36124038696289,
      "activations/layer17_attention_weight_min": -37.01845169067383,
      "activations/layer18_attention_weight_max": 69.45945739746094,
      "activations/layer18_attention_weight_min": -43.19988250732422,
      "activations/layer19_attention_weight_max": 18.36504554748535,
      "activations/layer19_attention_weight_min": -11.719264030456543,
      "activations/layer1_attention_weight_max": 15.997710227966309,
      "activations/layer1_attention_weight_min": -11.04154109954834,
      "activations/layer20_attention_weight_max": 18.07433319091797,
      "activations/layer20_attention_weight_min": -13.89681625366211,
      "activations/layer21_attention_weight_max": 31.17220115661621,
      "activations/layer21_attention_weight_min": -17.629798889160156,
      "activations/layer22_attention_weight_max": 30.720767974853516,
      "activations/layer22_attention_weight_min": -16.034910202026367,
      "activations/layer23_attention_weight_max": 22.094696044921875,
      "activations/layer23_attention_weight_min": -19.07967758178711,
      "activations/layer2_attention_weight_max": 24.44381332397461,
      "activations/layer2_attention_weight_min": -21.80366325378418,
      "activations/layer3_attention_weight_max": 52.082130432128906,
      "activations/layer3_attention_weight_min": -51.59189987182617,
      "activations/layer4_attention_weight_max": 80.32396697998047,
      "activations/layer4_attention_weight_min": -76.0780258178711,
      "activations/layer5_attention_weight_max": 131.16525268554688,
      "activations/layer5_attention_weight_min": -121.61393737792969,
      "activations/layer6_attention_weight_max": 65.39718627929688,
      "activations/layer6_attention_weight_min": -59.86687469482422,
      "activations/layer7_attention_weight_max": 63.480072021484375,
      "activations/layer7_attention_weight_min": -60.32979965209961,
      "activations/layer8_attention_weight_max": 57.46749496459961,
      "activations/layer8_attention_weight_min": -54.17059326171875,
      "activations/layer9_attention_weight_max": 69.03551483154297,
      "activations/layer9_attention_weight_min": -64.58260345458984,
      "epoch": 0.82,
      "learning_rate": 0.00014620378787878787,
      "loss": 3.1407,
      "step": 14050
    },
    {
      "activations/layer0_attention_weight_max": 12.999799728393555,
      "activations/layer0_attention_weight_min": -10.233355522155762,
      "activations/layer10_attention_weight_max": 74.39493560791016,
      "activations/layer10_attention_weight_min": -62.417076110839844,
      "activations/layer11_attention_weight_max": 46.45493698120117,
      "activations/layer11_attention_weight_min": -41.75660705566406,
      "activations/layer12_attention_weight_max": 29.299482345581055,
      "activations/layer12_attention_weight_min": -24.440704345703125,
      "activations/layer13_attention_weight_max": 65.83433532714844,
      "activations/layer13_attention_weight_min": -47.82807922363281,
      "activations/layer14_attention_weight_max": 60.09581756591797,
      "activations/layer14_attention_weight_min": -45.20370101928711,
      "activations/layer15_attention_weight_max": 54.232383728027344,
      "activations/layer15_attention_weight_min": -37.931365966796875,
      "activations/layer16_attention_weight_max": 55.45161437988281,
      "activations/layer16_attention_weight_min": -41.286991119384766,
      "activations/layer17_attention_weight_max": 65.08245086669922,
      "activations/layer17_attention_weight_min": -36.97809982299805,
      "activations/layer18_attention_weight_max": 68.04947662353516,
      "activations/layer18_attention_weight_min": -34.90885925292969,
      "activations/layer19_attention_weight_max": 26.764373779296875,
      "activations/layer19_attention_weight_min": -12.683144569396973,
      "activations/layer1_attention_weight_max": 15.27554702758789,
      "activations/layer1_attention_weight_min": -9.725436210632324,
      "activations/layer20_attention_weight_max": 16.413999557495117,
      "activations/layer20_attention_weight_min": -13.573354721069336,
      "activations/layer21_attention_weight_max": 29.593292236328125,
      "activations/layer21_attention_weight_min": -16.196287155151367,
      "activations/layer22_attention_weight_max": 28.308177947998047,
      "activations/layer22_attention_weight_min": -14.014135360717773,
      "activations/layer23_attention_weight_max": 22.59625244140625,
      "activations/layer23_attention_weight_min": -20.324596405029297,
      "activations/layer2_attention_weight_max": 23.990293502807617,
      "activations/layer2_attention_weight_min": -21.412933349609375,
      "activations/layer3_attention_weight_max": 50.080440521240234,
      "activations/layer3_attention_weight_min": -48.83462905883789,
      "activations/layer4_attention_weight_max": 79.69316864013672,
      "activations/layer4_attention_weight_min": -73.70831298828125,
      "activations/layer5_attention_weight_max": 138.33013916015625,
      "activations/layer5_attention_weight_min": -127.88777923583984,
      "activations/layer6_attention_weight_max": 69.77377319335938,
      "activations/layer6_attention_weight_min": -61.951988220214844,
      "activations/layer7_attention_weight_max": 72.74063873291016,
      "activations/layer7_attention_weight_min": -60.81597137451172,
      "activations/layer8_attention_weight_max": 56.10570526123047,
      "activations/layer8_attention_weight_min": -56.732601165771484,
      "activations/layer9_attention_weight_max": 81.51660919189453,
      "activations/layer9_attention_weight_min": -74.5947265625,
      "epoch": 0.82,
      "learning_rate": 0.00014618484848484846,
      "loss": 3.157,
      "step": 14100
    },
    {
      "activations/layer0_attention_weight_max": 14.48029899597168,
      "activations/layer0_attention_weight_min": -10.160350799560547,
      "activations/layer10_attention_weight_max": 66.03334045410156,
      "activations/layer10_attention_weight_min": -57.773067474365234,
      "activations/layer11_attention_weight_max": 46.2949333190918,
      "activations/layer11_attention_weight_min": -40.313812255859375,
      "activations/layer12_attention_weight_max": 35.221221923828125,
      "activations/layer12_attention_weight_min": -26.59927749633789,
      "activations/layer13_attention_weight_max": 92.77735137939453,
      "activations/layer13_attention_weight_min": -52.551448822021484,
      "activations/layer14_attention_weight_max": 119.15284729003906,
      "activations/layer14_attention_weight_min": -59.13798904418945,
      "activations/layer15_attention_weight_max": 59.66272735595703,
      "activations/layer15_attention_weight_min": -36.95109558105469,
      "activations/layer16_attention_weight_max": 45.37525177001953,
      "activations/layer16_attention_weight_min": -34.47930908203125,
      "activations/layer17_attention_weight_max": 56.108768463134766,
      "activations/layer17_attention_weight_min": -37.6208610534668,
      "activations/layer18_attention_weight_max": 59.81611633300781,
      "activations/layer18_attention_weight_min": -38.58698272705078,
      "activations/layer19_attention_weight_max": 19.02254295349121,
      "activations/layer19_attention_weight_min": -12.464643478393555,
      "activations/layer1_attention_weight_max": 17.674518585205078,
      "activations/layer1_attention_weight_min": -10.376508712768555,
      "activations/layer20_attention_weight_max": 16.985233306884766,
      "activations/layer20_attention_weight_min": -13.410970687866211,
      "activations/layer21_attention_weight_max": 34.70677185058594,
      "activations/layer21_attention_weight_min": -19.756542205810547,
      "activations/layer22_attention_weight_max": 32.621543884277344,
      "activations/layer22_attention_weight_min": -16.607704162597656,
      "activations/layer23_attention_weight_max": 22.866588592529297,
      "activations/layer23_attention_weight_min": -19.96337890625,
      "activations/layer2_attention_weight_max": 24.613147735595703,
      "activations/layer2_attention_weight_min": -22.03187370300293,
      "activations/layer3_attention_weight_max": 54.58564376831055,
      "activations/layer3_attention_weight_min": -51.320045471191406,
      "activations/layer4_attention_weight_max": 87.27947235107422,
      "activations/layer4_attention_weight_min": -72.59526824951172,
      "activations/layer5_attention_weight_max": 155.8224639892578,
      "activations/layer5_attention_weight_min": -128.97549438476562,
      "activations/layer6_attention_weight_max": 74.85892486572266,
      "activations/layer6_attention_weight_min": -60.28816223144531,
      "activations/layer7_attention_weight_max": 80.81490325927734,
      "activations/layer7_attention_weight_min": -60.6668701171875,
      "activations/layer8_attention_weight_max": 63.65816879272461,
      "activations/layer8_attention_weight_min": -48.974422454833984,
      "activations/layer9_attention_weight_max": 93.41798400878906,
      "activations/layer9_attention_weight_min": -71.85845947265625,
      "epoch": 0.82,
      "learning_rate": 0.00014616590909090907,
      "loss": 3.1344,
      "step": 14150
    },
    {
      "activations/layer0_attention_weight_max": 14.96630859375,
      "activations/layer0_attention_weight_min": -10.027473449707031,
      "activations/layer10_attention_weight_max": 68.16722106933594,
      "activations/layer10_attention_weight_min": -65.47552490234375,
      "activations/layer11_attention_weight_max": 45.794944763183594,
      "activations/layer11_attention_weight_min": -38.151519775390625,
      "activations/layer12_attention_weight_max": 28.086091995239258,
      "activations/layer12_attention_weight_min": -24.115015029907227,
      "activations/layer13_attention_weight_max": 53.84214782714844,
      "activations/layer13_attention_weight_min": -45.40592575073242,
      "activations/layer14_attention_weight_max": 53.902122497558594,
      "activations/layer14_attention_weight_min": -37.22517013549805,
      "activations/layer15_attention_weight_max": 50.044349670410156,
      "activations/layer15_attention_weight_min": -38.76193618774414,
      "activations/layer16_attention_weight_max": 50.90092468261719,
      "activations/layer16_attention_weight_min": -34.90732192993164,
      "activations/layer17_attention_weight_max": 62.02176284790039,
      "activations/layer17_attention_weight_min": -38.68159866333008,
      "activations/layer18_attention_weight_max": 57.28636932373047,
      "activations/layer18_attention_weight_min": -41.98457336425781,
      "activations/layer19_attention_weight_max": 19.662519454956055,
      "activations/layer19_attention_weight_min": -12.354334831237793,
      "activations/layer1_attention_weight_max": 16.83030128479004,
      "activations/layer1_attention_weight_min": -10.084684371948242,
      "activations/layer20_attention_weight_max": 16.500885009765625,
      "activations/layer20_attention_weight_min": -14.610389709472656,
      "activations/layer21_attention_weight_max": 36.71430587768555,
      "activations/layer21_attention_weight_min": -16.095312118530273,
      "activations/layer22_attention_weight_max": 33.12371063232422,
      "activations/layer22_attention_weight_min": -14.293106079101562,
      "activations/layer23_attention_weight_max": 20.98122787475586,
      "activations/layer23_attention_weight_min": -18.78761863708496,
      "activations/layer2_attention_weight_max": 24.18716049194336,
      "activations/layer2_attention_weight_min": -22.331045150756836,
      "activations/layer3_attention_weight_max": 50.015689849853516,
      "activations/layer3_attention_weight_min": -50.82479476928711,
      "activations/layer4_attention_weight_max": 82.63558197021484,
      "activations/layer4_attention_weight_min": -76.53547668457031,
      "activations/layer5_attention_weight_max": 135.30111694335938,
      "activations/layer5_attention_weight_min": -121.07726287841797,
      "activations/layer6_attention_weight_max": 63.573917388916016,
      "activations/layer6_attention_weight_min": -58.06848907470703,
      "activations/layer7_attention_weight_max": 62.424949645996094,
      "activations/layer7_attention_weight_min": -57.28146743774414,
      "activations/layer8_attention_weight_max": 51.4377555847168,
      "activations/layer8_attention_weight_min": -48.28216552734375,
      "activations/layer9_attention_weight_max": 80.56735229492188,
      "activations/layer9_attention_weight_min": -68.15716552734375,
      "epoch": 0.83,
      "learning_rate": 0.0001461469696969697,
      "loss": 3.1661,
      "step": 14200
    },
    {
      "activations/layer0_attention_weight_max": 13.523981094360352,
      "activations/layer0_attention_weight_min": -10.052443504333496,
      "activations/layer10_attention_weight_max": 72.03372955322266,
      "activations/layer10_attention_weight_min": -64.2847671508789,
      "activations/layer11_attention_weight_max": 43.45112228393555,
      "activations/layer11_attention_weight_min": -37.744476318359375,
      "activations/layer12_attention_weight_max": 29.628135681152344,
      "activations/layer12_attention_weight_min": -26.696903228759766,
      "activations/layer13_attention_weight_max": 65.33116912841797,
      "activations/layer13_attention_weight_min": -50.54890823364258,
      "activations/layer14_attention_weight_max": 56.70530319213867,
      "activations/layer14_attention_weight_min": -41.656272888183594,
      "activations/layer15_attention_weight_max": 53.32700729370117,
      "activations/layer15_attention_weight_min": -37.20848083496094,
      "activations/layer16_attention_weight_max": 50.46735763549805,
      "activations/layer16_attention_weight_min": -36.078086853027344,
      "activations/layer17_attention_weight_max": 62.32542037963867,
      "activations/layer17_attention_weight_min": -41.227317810058594,
      "activations/layer18_attention_weight_max": 58.3520393371582,
      "activations/layer18_attention_weight_min": -42.083396911621094,
      "activations/layer19_attention_weight_max": 19.646512985229492,
      "activations/layer19_attention_weight_min": -13.642155647277832,
      "activations/layer1_attention_weight_max": 17.007917404174805,
      "activations/layer1_attention_weight_min": -10.497065544128418,
      "activations/layer20_attention_weight_max": 20.335176467895508,
      "activations/layer20_attention_weight_min": -13.183467864990234,
      "activations/layer21_attention_weight_max": 33.64094924926758,
      "activations/layer21_attention_weight_min": -16.9989013671875,
      "activations/layer22_attention_weight_max": 27.602582931518555,
      "activations/layer22_attention_weight_min": -13.570653915405273,
      "activations/layer23_attention_weight_max": 24.196149826049805,
      "activations/layer23_attention_weight_min": -17.735836029052734,
      "activations/layer2_attention_weight_max": 25.443435668945312,
      "activations/layer2_attention_weight_min": -22.83922576904297,
      "activations/layer3_attention_weight_max": 53.5936164855957,
      "activations/layer3_attention_weight_min": -52.299644470214844,
      "activations/layer4_attention_weight_max": 83.02102661132812,
      "activations/layer4_attention_weight_min": -76.56038665771484,
      "activations/layer5_attention_weight_max": 148.43734741210938,
      "activations/layer5_attention_weight_min": -125.6097640991211,
      "activations/layer6_attention_weight_max": 69.48351287841797,
      "activations/layer6_attention_weight_min": -62.32926559448242,
      "activations/layer7_attention_weight_max": 71.5643539428711,
      "activations/layer7_attention_weight_min": -60.90614318847656,
      "activations/layer8_attention_weight_max": 55.40797424316406,
      "activations/layer8_attention_weight_min": -53.552574157714844,
      "activations/layer9_attention_weight_max": 76.32174682617188,
      "activations/layer9_attention_weight_min": -61.77470779418945,
      "epoch": 0.83,
      "learning_rate": 0.00014612803030303028,
      "loss": 3.1599,
      "step": 14250
    },
    {
      "activations/layer0_attention_weight_max": 14.916999816894531,
      "activations/layer0_attention_weight_min": -10.552581787109375,
      "activations/layer10_attention_weight_max": 63.572235107421875,
      "activations/layer10_attention_weight_min": -56.445457458496094,
      "activations/layer11_attention_weight_max": 42.08053970336914,
      "activations/layer11_attention_weight_min": -39.05067443847656,
      "activations/layer12_attention_weight_max": 25.8779354095459,
      "activations/layer12_attention_weight_min": -24.817367553710938,
      "activations/layer13_attention_weight_max": 54.95436477661133,
      "activations/layer13_attention_weight_min": -42.30693054199219,
      "activations/layer14_attention_weight_max": 49.40983200073242,
      "activations/layer14_attention_weight_min": -37.80691909790039,
      "activations/layer15_attention_weight_max": 51.011844635009766,
      "activations/layer15_attention_weight_min": -38.90333938598633,
      "activations/layer16_attention_weight_max": 46.71393585205078,
      "activations/layer16_attention_weight_min": -34.94484329223633,
      "activations/layer17_attention_weight_max": 55.851409912109375,
      "activations/layer17_attention_weight_min": -37.08207702636719,
      "activations/layer18_attention_weight_max": 48.07780456542969,
      "activations/layer18_attention_weight_min": -31.637956619262695,
      "activations/layer19_attention_weight_max": 17.01460075378418,
      "activations/layer19_attention_weight_min": -13.125784873962402,
      "activations/layer1_attention_weight_max": 16.500638961791992,
      "activations/layer1_attention_weight_min": -10.411188125610352,
      "activations/layer20_attention_weight_max": 17.995969772338867,
      "activations/layer20_attention_weight_min": -13.173306465148926,
      "activations/layer21_attention_weight_max": 32.545745849609375,
      "activations/layer21_attention_weight_min": -15.991600036621094,
      "activations/layer22_attention_weight_max": 29.488510131835938,
      "activations/layer22_attention_weight_min": -18.69200897216797,
      "activations/layer23_attention_weight_max": 19.840370178222656,
      "activations/layer23_attention_weight_min": -17.50872230529785,
      "activations/layer2_attention_weight_max": 24.432645797729492,
      "activations/layer2_attention_weight_min": -22.210906982421875,
      "activations/layer3_attention_weight_max": 52.0626335144043,
      "activations/layer3_attention_weight_min": -51.93466567993164,
      "activations/layer4_attention_weight_max": 80.64502716064453,
      "activations/layer4_attention_weight_min": -73.61955261230469,
      "activations/layer5_attention_weight_max": 136.63011169433594,
      "activations/layer5_attention_weight_min": -130.325439453125,
      "activations/layer6_attention_weight_max": 64.8239517211914,
      "activations/layer6_attention_weight_min": -57.86592102050781,
      "activations/layer7_attention_weight_max": 63.805641174316406,
      "activations/layer7_attention_weight_min": -58.413719177246094,
      "activations/layer8_attention_weight_max": 57.75297927856445,
      "activations/layer8_attention_weight_min": -48.75285720825195,
      "activations/layer9_attention_weight_max": 84.05830383300781,
      "activations/layer9_attention_weight_min": -70.6843490600586,
      "epoch": 0.83,
      "learning_rate": 0.0001461090909090909,
      "loss": 3.1346,
      "step": 14300
    },
    {
      "activations/layer0_attention_weight_max": 13.716309547424316,
      "activations/layer0_attention_weight_min": -9.870576858520508,
      "activations/layer10_attention_weight_max": 75.48930358886719,
      "activations/layer10_attention_weight_min": -62.50159454345703,
      "activations/layer11_attention_weight_max": 50.0653076171875,
      "activations/layer11_attention_weight_min": -42.08797836303711,
      "activations/layer12_attention_weight_max": 26.984451293945312,
      "activations/layer12_attention_weight_min": -26.340803146362305,
      "activations/layer13_attention_weight_max": 63.10182571411133,
      "activations/layer13_attention_weight_min": -40.22888946533203,
      "activations/layer14_attention_weight_max": 58.232933044433594,
      "activations/layer14_attention_weight_min": -36.4742431640625,
      "activations/layer15_attention_weight_max": 46.96446990966797,
      "activations/layer15_attention_weight_min": -38.01789474487305,
      "activations/layer16_attention_weight_max": 51.87808609008789,
      "activations/layer16_attention_weight_min": -37.789798736572266,
      "activations/layer17_attention_weight_max": 57.54846954345703,
      "activations/layer17_attention_weight_min": -37.49858093261719,
      "activations/layer18_attention_weight_max": 60.32837677001953,
      "activations/layer18_attention_weight_min": -38.9171028137207,
      "activations/layer19_attention_weight_max": 18.31818389892578,
      "activations/layer19_attention_weight_min": -12.57280445098877,
      "activations/layer1_attention_weight_max": 16.177473068237305,
      "activations/layer1_attention_weight_min": -10.955487251281738,
      "activations/layer20_attention_weight_max": 20.15386199951172,
      "activations/layer20_attention_weight_min": -14.177253723144531,
      "activations/layer21_attention_weight_max": 31.50948715209961,
      "activations/layer21_attention_weight_min": -18.270469665527344,
      "activations/layer22_attention_weight_max": 30.469865798950195,
      "activations/layer22_attention_weight_min": -16.58930206298828,
      "activations/layer23_attention_weight_max": 22.65894889831543,
      "activations/layer23_attention_weight_min": -18.734243392944336,
      "activations/layer2_attention_weight_max": 24.816749572753906,
      "activations/layer2_attention_weight_min": -22.06533432006836,
      "activations/layer3_attention_weight_max": 51.347774505615234,
      "activations/layer3_attention_weight_min": -51.09440231323242,
      "activations/layer4_attention_weight_max": 80.56071472167969,
      "activations/layer4_attention_weight_min": -74.14229583740234,
      "activations/layer5_attention_weight_max": 140.37069702148438,
      "activations/layer5_attention_weight_min": -124.47329711914062,
      "activations/layer6_attention_weight_max": 66.48130798339844,
      "activations/layer6_attention_weight_min": -58.33152770996094,
      "activations/layer7_attention_weight_max": 66.89253234863281,
      "activations/layer7_attention_weight_min": -60.253753662109375,
      "activations/layer8_attention_weight_max": 55.052371978759766,
      "activations/layer8_attention_weight_min": -49.04294967651367,
      "activations/layer9_attention_weight_max": 77.64923095703125,
      "activations/layer9_attention_weight_min": -69.12657928466797,
      "epoch": 0.83,
      "learning_rate": 0.0001460901515151515,
      "loss": 3.1507,
      "step": 14350
    },
    {
      "activations/layer0_attention_weight_max": 13.626922607421875,
      "activations/layer0_attention_weight_min": -10.32780647277832,
      "activations/layer10_attention_weight_max": 62.520809173583984,
      "activations/layer10_attention_weight_min": -58.77406692504883,
      "activations/layer11_attention_weight_max": 50.143150329589844,
      "activations/layer11_attention_weight_min": -41.15055847167969,
      "activations/layer12_attention_weight_max": 28.667856216430664,
      "activations/layer12_attention_weight_min": -24.272611618041992,
      "activations/layer13_attention_weight_max": 53.4516716003418,
      "activations/layer13_attention_weight_min": -44.484127044677734,
      "activations/layer14_attention_weight_max": 38.5572624206543,
      "activations/layer14_attention_weight_min": -36.32221221923828,
      "activations/layer15_attention_weight_max": 49.69779586791992,
      "activations/layer15_attention_weight_min": -38.487022399902344,
      "activations/layer16_attention_weight_max": 42.98065185546875,
      "activations/layer16_attention_weight_min": -34.38936233520508,
      "activations/layer17_attention_weight_max": 55.73830032348633,
      "activations/layer17_attention_weight_min": -37.78959655761719,
      "activations/layer18_attention_weight_max": 53.237606048583984,
      "activations/layer18_attention_weight_min": -38.4140739440918,
      "activations/layer19_attention_weight_max": 20.82817268371582,
      "activations/layer19_attention_weight_min": -12.504599571228027,
      "activations/layer1_attention_weight_max": 16.48311996459961,
      "activations/layer1_attention_weight_min": -10.657658576965332,
      "activations/layer20_attention_weight_max": 22.57783317565918,
      "activations/layer20_attention_weight_min": -14.94538688659668,
      "activations/layer21_attention_weight_max": 38.0609016418457,
      "activations/layer21_attention_weight_min": -16.67069435119629,
      "activations/layer22_attention_weight_max": 32.57661056518555,
      "activations/layer22_attention_weight_min": -14.948457717895508,
      "activations/layer23_attention_weight_max": 23.184654235839844,
      "activations/layer23_attention_weight_min": -20.124067306518555,
      "activations/layer2_attention_weight_max": 24.919309616088867,
      "activations/layer2_attention_weight_min": -21.850032806396484,
      "activations/layer3_attention_weight_max": 52.388893127441406,
      "activations/layer3_attention_weight_min": -50.96317672729492,
      "activations/layer4_attention_weight_max": 82.07475280761719,
      "activations/layer4_attention_weight_min": -72.45015716552734,
      "activations/layer5_attention_weight_max": 139.4385986328125,
      "activations/layer5_attention_weight_min": -122.40431213378906,
      "activations/layer6_attention_weight_max": 64.95901489257812,
      "activations/layer6_attention_weight_min": -57.52891159057617,
      "activations/layer7_attention_weight_max": 65.1280517578125,
      "activations/layer7_attention_weight_min": -57.885467529296875,
      "activations/layer8_attention_weight_max": 53.1138916015625,
      "activations/layer8_attention_weight_min": -52.548248291015625,
      "activations/layer9_attention_weight_max": 68.07140350341797,
      "activations/layer9_attention_weight_min": -63.095947265625,
      "epoch": 0.84,
      "learning_rate": 0.0001460712121212121,
      "loss": 3.1344,
      "step": 14400
    },
    {
      "activations/layer0_attention_weight_max": 13.308009147644043,
      "activations/layer0_attention_weight_min": -10.38424015045166,
      "activations/layer10_attention_weight_max": 75.4472885131836,
      "activations/layer10_attention_weight_min": -59.7324333190918,
      "activations/layer11_attention_weight_max": 50.45339584350586,
      "activations/layer11_attention_weight_min": -35.18457794189453,
      "activations/layer12_attention_weight_max": 27.218935012817383,
      "activations/layer12_attention_weight_min": -22.379175186157227,
      "activations/layer13_attention_weight_max": 65.96279907226562,
      "activations/layer13_attention_weight_min": -44.593101501464844,
      "activations/layer14_attention_weight_max": 55.33979034423828,
      "activations/layer14_attention_weight_min": -38.38111114501953,
      "activations/layer15_attention_weight_max": 53.8922119140625,
      "activations/layer15_attention_weight_min": -36.89344024658203,
      "activations/layer16_attention_weight_max": 52.52845001220703,
      "activations/layer16_attention_weight_min": -34.164424896240234,
      "activations/layer17_attention_weight_max": 60.17808151245117,
      "activations/layer17_attention_weight_min": -38.35546875,
      "activations/layer18_attention_weight_max": 72.8775405883789,
      "activations/layer18_attention_weight_min": -41.146270751953125,
      "activations/layer19_attention_weight_max": 22.275657653808594,
      "activations/layer19_attention_weight_min": -12.822331428527832,
      "activations/layer1_attention_weight_max": 16.237943649291992,
      "activations/layer1_attention_weight_min": -11.780147552490234,
      "activations/layer20_attention_weight_max": 18.205991744995117,
      "activations/layer20_attention_weight_min": -12.734831809997559,
      "activations/layer21_attention_weight_max": 36.827491760253906,
      "activations/layer21_attention_weight_min": -18.269775390625,
      "activations/layer22_attention_weight_max": 31.246057510375977,
      "activations/layer22_attention_weight_min": -14.242596626281738,
      "activations/layer23_attention_weight_max": 20.967002868652344,
      "activations/layer23_attention_weight_min": -18.424795150756836,
      "activations/layer2_attention_weight_max": 27.34023666381836,
      "activations/layer2_attention_weight_min": -22.381454467773438,
      "activations/layer3_attention_weight_max": 53.64381790161133,
      "activations/layer3_attention_weight_min": -52.593482971191406,
      "activations/layer4_attention_weight_max": 84.53594970703125,
      "activations/layer4_attention_weight_min": -76.17443084716797,
      "activations/layer5_attention_weight_max": 149.51885986328125,
      "activations/layer5_attention_weight_min": -130.43368530273438,
      "activations/layer6_attention_weight_max": 76.13603210449219,
      "activations/layer6_attention_weight_min": -59.783267974853516,
      "activations/layer7_attention_weight_max": 77.81182861328125,
      "activations/layer7_attention_weight_min": -60.744956970214844,
      "activations/layer8_attention_weight_max": 70.51443481445312,
      "activations/layer8_attention_weight_min": -54.375152587890625,
      "activations/layer9_attention_weight_max": 75.76371002197266,
      "activations/layer9_attention_weight_min": -66.6332015991211,
      "epoch": 0.84,
      "learning_rate": 0.0001460522727272727,
      "loss": 3.1309,
      "step": 14450
    },
    {
      "activations/layer0_attention_weight_max": 13.134247779846191,
      "activations/layer0_attention_weight_min": -10.777373313903809,
      "activations/layer10_attention_weight_max": 67.83535766601562,
      "activations/layer10_attention_weight_min": -65.89446258544922,
      "activations/layer11_attention_weight_max": 44.167022705078125,
      "activations/layer11_attention_weight_min": -38.81892395019531,
      "activations/layer12_attention_weight_max": 28.285253524780273,
      "activations/layer12_attention_weight_min": -26.735132217407227,
      "activations/layer13_attention_weight_max": 48.96723175048828,
      "activations/layer13_attention_weight_min": -40.28280258178711,
      "activations/layer14_attention_weight_max": 45.039573669433594,
      "activations/layer14_attention_weight_min": -41.217498779296875,
      "activations/layer15_attention_weight_max": 49.66716003417969,
      "activations/layer15_attention_weight_min": -40.68653869628906,
      "activations/layer16_attention_weight_max": 50.98268508911133,
      "activations/layer16_attention_weight_min": -38.60730743408203,
      "activations/layer17_attention_weight_max": 55.856746673583984,
      "activations/layer17_attention_weight_min": -41.81467056274414,
      "activations/layer18_attention_weight_max": 57.79773712158203,
      "activations/layer18_attention_weight_min": -35.7181282043457,
      "activations/layer19_attention_weight_max": 19.766456604003906,
      "activations/layer19_attention_weight_min": -12.015349388122559,
      "activations/layer1_attention_weight_max": 15.74329948425293,
      "activations/layer1_attention_weight_min": -10.731477737426758,
      "activations/layer20_attention_weight_max": 18.3773136138916,
      "activations/layer20_attention_weight_min": -14.3045072555542,
      "activations/layer21_attention_weight_max": 30.057321548461914,
      "activations/layer21_attention_weight_min": -17.55217933654785,
      "activations/layer22_attention_weight_max": 28.831886291503906,
      "activations/layer22_attention_weight_min": -14.505881309509277,
      "activations/layer23_attention_weight_max": 21.413105010986328,
      "activations/layer23_attention_weight_min": -20.275663375854492,
      "activations/layer2_attention_weight_max": 25.277372360229492,
      "activations/layer2_attention_weight_min": -20.986032485961914,
      "activations/layer3_attention_weight_max": 52.292823791503906,
      "activations/layer3_attention_weight_min": -48.62638473510742,
      "activations/layer4_attention_weight_max": 78.63663482666016,
      "activations/layer4_attention_weight_min": -71.74961853027344,
      "activations/layer5_attention_weight_max": 135.18722534179688,
      "activations/layer5_attention_weight_min": -124.53216552734375,
      "activations/layer6_attention_weight_max": 68.7779541015625,
      "activations/layer6_attention_weight_min": -58.374637603759766,
      "activations/layer7_attention_weight_max": 71.63508605957031,
      "activations/layer7_attention_weight_min": -59.72468185424805,
      "activations/layer8_attention_weight_max": 60.2231330871582,
      "activations/layer8_attention_weight_min": -50.69490051269531,
      "activations/layer9_attention_weight_max": 71.5741195678711,
      "activations/layer9_attention_weight_min": -69.79571533203125,
      "epoch": 0.84,
      "learning_rate": 0.00014603333333333332,
      "loss": 3.1362,
      "step": 14500
    },
    {
      "activations/layer0_attention_weight_max": 13.330272674560547,
      "activations/layer0_attention_weight_min": -10.799365043640137,
      "activations/layer10_attention_weight_max": 68.9071044921875,
      "activations/layer10_attention_weight_min": -60.0174674987793,
      "activations/layer11_attention_weight_max": 48.917884826660156,
      "activations/layer11_attention_weight_min": -43.21296310424805,
      "activations/layer12_attention_weight_max": 31.455730438232422,
      "activations/layer12_attention_weight_min": -25.43929672241211,
      "activations/layer13_attention_weight_max": 60.28063201904297,
      "activations/layer13_attention_weight_min": -47.938655853271484,
      "activations/layer14_attention_weight_max": 55.767215728759766,
      "activations/layer14_attention_weight_min": -35.4771614074707,
      "activations/layer15_attention_weight_max": 63.74972915649414,
      "activations/layer15_attention_weight_min": -41.136024475097656,
      "activations/layer16_attention_weight_max": 63.156028747558594,
      "activations/layer16_attention_weight_min": -36.65132141113281,
      "activations/layer17_attention_weight_max": 59.44565963745117,
      "activations/layer17_attention_weight_min": -38.68531036376953,
      "activations/layer18_attention_weight_max": 60.119083404541016,
      "activations/layer18_attention_weight_min": -35.794952392578125,
      "activations/layer19_attention_weight_max": 20.9522647857666,
      "activations/layer19_attention_weight_min": -15.520482063293457,
      "activations/layer1_attention_weight_max": 15.234803199768066,
      "activations/layer1_attention_weight_min": -10.872596740722656,
      "activations/layer20_attention_weight_max": 17.699535369873047,
      "activations/layer20_attention_weight_min": -16.869285583496094,
      "activations/layer21_attention_weight_max": 31.877599716186523,
      "activations/layer21_attention_weight_min": -22.471281051635742,
      "activations/layer22_attention_weight_max": 29.507463455200195,
      "activations/layer22_attention_weight_min": -17.246334075927734,
      "activations/layer23_attention_weight_max": 20.724544525146484,
      "activations/layer23_attention_weight_min": -19.582599639892578,
      "activations/layer2_attention_weight_max": 25.634214401245117,
      "activations/layer2_attention_weight_min": -23.10481834411621,
      "activations/layer3_attention_weight_max": 54.20713424682617,
      "activations/layer3_attention_weight_min": -51.110374450683594,
      "activations/layer4_attention_weight_max": 83.80919647216797,
      "activations/layer4_attention_weight_min": -70.51197052001953,
      "activations/layer5_attention_weight_max": 145.42196655273438,
      "activations/layer5_attention_weight_min": -128.70925903320312,
      "activations/layer6_attention_weight_max": 66.0706787109375,
      "activations/layer6_attention_weight_min": -58.56829833984375,
      "activations/layer7_attention_weight_max": 68.89942169189453,
      "activations/layer7_attention_weight_min": -58.94020080566406,
      "activations/layer8_attention_weight_max": 61.20960998535156,
      "activations/layer8_attention_weight_min": -49.37108612060547,
      "activations/layer9_attention_weight_max": 67.6103515625,
      "activations/layer9_attention_weight_min": -62.105709075927734,
      "epoch": 0.85,
      "learning_rate": 0.0001460151515151515,
      "loss": 3.1439,
      "step": 14550
    },
    {
      "activations/layer0_attention_weight_max": 13.247095108032227,
      "activations/layer0_attention_weight_min": -10.874934196472168,
      "activations/layer10_attention_weight_max": 67.9181900024414,
      "activations/layer10_attention_weight_min": -60.74228286743164,
      "activations/layer11_attention_weight_max": 46.80867004394531,
      "activations/layer11_attention_weight_min": -37.60499954223633,
      "activations/layer12_attention_weight_max": 26.935091018676758,
      "activations/layer12_attention_weight_min": -23.89904022216797,
      "activations/layer13_attention_weight_max": 77.33392333984375,
      "activations/layer13_attention_weight_min": -52.027870178222656,
      "activations/layer14_attention_weight_max": 68.19174194335938,
      "activations/layer14_attention_weight_min": -43.202415466308594,
      "activations/layer15_attention_weight_max": 55.992610931396484,
      "activations/layer15_attention_weight_min": -38.33859634399414,
      "activations/layer16_attention_weight_max": 47.138832092285156,
      "activations/layer16_attention_weight_min": -34.12114715576172,
      "activations/layer17_attention_weight_max": 61.71875762939453,
      "activations/layer17_attention_weight_min": -39.23686218261719,
      "activations/layer18_attention_weight_max": 72.83161163330078,
      "activations/layer18_attention_weight_min": -39.66535568237305,
      "activations/layer19_attention_weight_max": 20.302188873291016,
      "activations/layer19_attention_weight_min": -12.353639602661133,
      "activations/layer1_attention_weight_max": 15.88902759552002,
      "activations/layer1_attention_weight_min": -10.52012825012207,
      "activations/layer20_attention_weight_max": 19.856348037719727,
      "activations/layer20_attention_weight_min": -12.567790031433105,
      "activations/layer21_attention_weight_max": 28.685405731201172,
      "activations/layer21_attention_weight_min": -17.12948989868164,
      "activations/layer22_attention_weight_max": 26.353302001953125,
      "activations/layer22_attention_weight_min": -15.10106086730957,
      "activations/layer23_attention_weight_max": 20.7894229888916,
      "activations/layer23_attention_weight_min": -18.114965438842773,
      "activations/layer2_attention_weight_max": 27.215747833251953,
      "activations/layer2_attention_weight_min": -22.314632415771484,
      "activations/layer3_attention_weight_max": 49.73431396484375,
      "activations/layer3_attention_weight_min": -48.66807556152344,
      "activations/layer4_attention_weight_max": 78.13579559326172,
      "activations/layer4_attention_weight_min": -73.91004180908203,
      "activations/layer5_attention_weight_max": 134.60516357421875,
      "activations/layer5_attention_weight_min": -120.62010192871094,
      "activations/layer6_attention_weight_max": 66.31616973876953,
      "activations/layer6_attention_weight_min": -59.03508377075195,
      "activations/layer7_attention_weight_max": 63.60867691040039,
      "activations/layer7_attention_weight_min": -59.27524948120117,
      "activations/layer8_attention_weight_max": 54.46227264404297,
      "activations/layer8_attention_weight_min": -48.45462417602539,
      "activations/layer9_attention_weight_max": 75.03453063964844,
      "activations/layer9_attention_weight_min": -60.4783821105957,
      "epoch": 0.85,
      "learning_rate": 0.0001459962121212121,
      "loss": 3.1365,
      "step": 14600
    },
    {
      "activations/layer0_attention_weight_max": 13.746846199035645,
      "activations/layer0_attention_weight_min": -10.570268630981445,
      "activations/layer10_attention_weight_max": 68.9289779663086,
      "activations/layer10_attention_weight_min": -54.758235931396484,
      "activations/layer11_attention_weight_max": 46.40113067626953,
      "activations/layer11_attention_weight_min": -41.160072326660156,
      "activations/layer12_attention_weight_max": 25.646366119384766,
      "activations/layer12_attention_weight_min": -27.19267463684082,
      "activations/layer13_attention_weight_max": 55.625431060791016,
      "activations/layer13_attention_weight_min": -45.80345153808594,
      "activations/layer14_attention_weight_max": 58.91127014160156,
      "activations/layer14_attention_weight_min": -39.98381423950195,
      "activations/layer15_attention_weight_max": 46.16141891479492,
      "activations/layer15_attention_weight_min": -38.458404541015625,
      "activations/layer16_attention_weight_max": 48.61094665527344,
      "activations/layer16_attention_weight_min": -34.65079879760742,
      "activations/layer17_attention_weight_max": 58.13037109375,
      "activations/layer17_attention_weight_min": -38.98809051513672,
      "activations/layer18_attention_weight_max": 52.27053451538086,
      "activations/layer18_attention_weight_min": -37.694580078125,
      "activations/layer19_attention_weight_max": 20.143299102783203,
      "activations/layer19_attention_weight_min": -11.350211143493652,
      "activations/layer1_attention_weight_max": 15.419305801391602,
      "activations/layer1_attention_weight_min": -10.735041618347168,
      "activations/layer20_attention_weight_max": 20.105514526367188,
      "activations/layer20_attention_weight_min": -13.765189170837402,
      "activations/layer21_attention_weight_max": 44.06309509277344,
      "activations/layer21_attention_weight_min": -16.98355484008789,
      "activations/layer22_attention_weight_max": 37.813255310058594,
      "activations/layer22_attention_weight_min": -15.47311019897461,
      "activations/layer23_attention_weight_max": 26.1873836517334,
      "activations/layer23_attention_weight_min": -19.849172592163086,
      "activations/layer2_attention_weight_max": 27.14736557006836,
      "activations/layer2_attention_weight_min": -24.039783477783203,
      "activations/layer3_attention_weight_max": 53.10565948486328,
      "activations/layer3_attention_weight_min": -52.806331634521484,
      "activations/layer4_attention_weight_max": 87.22371673583984,
      "activations/layer4_attention_weight_min": -74.81331634521484,
      "activations/layer5_attention_weight_max": 142.51608276367188,
      "activations/layer5_attention_weight_min": -127.06947326660156,
      "activations/layer6_attention_weight_max": 68.71282958984375,
      "activations/layer6_attention_weight_min": -60.97880554199219,
      "activations/layer7_attention_weight_max": 69.84529113769531,
      "activations/layer7_attention_weight_min": -62.271358489990234,
      "activations/layer8_attention_weight_max": 56.50200271606445,
      "activations/layer8_attention_weight_min": -50.315956115722656,
      "activations/layer9_attention_weight_max": 88.16151428222656,
      "activations/layer9_attention_weight_min": -69.20809173583984,
      "epoch": 0.85,
      "learning_rate": 0.00014597727272727272,
      "loss": 3.1295,
      "step": 14650
    },
    {
      "activations/layer0_attention_weight_max": 14.010022163391113,
      "activations/layer0_attention_weight_min": -10.711750030517578,
      "activations/layer10_attention_weight_max": 58.06842041015625,
      "activations/layer10_attention_weight_min": -57.49789810180664,
      "activations/layer11_attention_weight_max": 45.07112121582031,
      "activations/layer11_attention_weight_min": -40.27907943725586,
      "activations/layer12_attention_weight_max": 25.194217681884766,
      "activations/layer12_attention_weight_min": -23.079370498657227,
      "activations/layer13_attention_weight_max": 52.54495620727539,
      "activations/layer13_attention_weight_min": -41.4343147277832,
      "activations/layer14_attention_weight_max": 43.18653106689453,
      "activations/layer14_attention_weight_min": -39.171695709228516,
      "activations/layer15_attention_weight_max": 48.11808395385742,
      "activations/layer15_attention_weight_min": -38.975337982177734,
      "activations/layer16_attention_weight_max": 44.3570671081543,
      "activations/layer16_attention_weight_min": -33.86252975463867,
      "activations/layer17_attention_weight_max": 51.58380126953125,
      "activations/layer17_attention_weight_min": -36.7650146484375,
      "activations/layer18_attention_weight_max": 60.30154037475586,
      "activations/layer18_attention_weight_min": -38.07113265991211,
      "activations/layer19_attention_weight_max": 19.185747146606445,
      "activations/layer19_attention_weight_min": -10.952006340026855,
      "activations/layer1_attention_weight_max": 15.789740562438965,
      "activations/layer1_attention_weight_min": -9.916224479675293,
      "activations/layer20_attention_weight_max": 17.276622772216797,
      "activations/layer20_attention_weight_min": -14.87653923034668,
      "activations/layer21_attention_weight_max": 31.75149154663086,
      "activations/layer21_attention_weight_min": -17.250350952148438,
      "activations/layer22_attention_weight_max": 30.579696655273438,
      "activations/layer22_attention_weight_min": -17.649995803833008,
      "activations/layer23_attention_weight_max": 20.287567138671875,
      "activations/layer23_attention_weight_min": -17.667678833007812,
      "activations/layer2_attention_weight_max": 25.709875106811523,
      "activations/layer2_attention_weight_min": -22.5479736328125,
      "activations/layer3_attention_weight_max": 48.61361312866211,
      "activations/layer3_attention_weight_min": -48.833351135253906,
      "activations/layer4_attention_weight_max": 75.88213348388672,
      "activations/layer4_attention_weight_min": -72.88518524169922,
      "activations/layer5_attention_weight_max": 126.57633972167969,
      "activations/layer5_attention_weight_min": -120.18206024169922,
      "activations/layer6_attention_weight_max": 62.56214141845703,
      "activations/layer6_attention_weight_min": -55.40177536010742,
      "activations/layer7_attention_weight_max": 60.72217559814453,
      "activations/layer7_attention_weight_min": -58.125328063964844,
      "activations/layer8_attention_weight_max": 56.475013732910156,
      "activations/layer8_attention_weight_min": -51.63302230834961,
      "activations/layer9_attention_weight_max": 68.11267852783203,
      "activations/layer9_attention_weight_min": -65.49221801757812,
      "epoch": 0.85,
      "learning_rate": 0.00014595833333333333,
      "loss": 3.1307,
      "step": 14700
    },
    {
      "activations/layer0_attention_weight_max": 14.180209159851074,
      "activations/layer0_attention_weight_min": -10.77202320098877,
      "activations/layer10_attention_weight_max": 68.47870635986328,
      "activations/layer10_attention_weight_min": -56.60425567626953,
      "activations/layer11_attention_weight_max": 48.53297424316406,
      "activations/layer11_attention_weight_min": -38.124000549316406,
      "activations/layer12_attention_weight_max": 26.92430305480957,
      "activations/layer12_attention_weight_min": -25.83806800842285,
      "activations/layer13_attention_weight_max": 55.05684280395508,
      "activations/layer13_attention_weight_min": -40.57485580444336,
      "activations/layer14_attention_weight_max": 44.28389358520508,
      "activations/layer14_attention_weight_min": -43.96430969238281,
      "activations/layer15_attention_weight_max": 48.66606521606445,
      "activations/layer15_attention_weight_min": -42.608402252197266,
      "activations/layer16_attention_weight_max": 50.33004379272461,
      "activations/layer16_attention_weight_min": -37.283409118652344,
      "activations/layer17_attention_weight_max": 56.24617385864258,
      "activations/layer17_attention_weight_min": -37.49645233154297,
      "activations/layer18_attention_weight_max": 52.14213180541992,
      "activations/layer18_attention_weight_min": -35.26592254638672,
      "activations/layer19_attention_weight_max": 20.17098617553711,
      "activations/layer19_attention_weight_min": -13.583572387695312,
      "activations/layer1_attention_weight_max": 15.233460426330566,
      "activations/layer1_attention_weight_min": -12.18947982788086,
      "activations/layer20_attention_weight_max": 21.737472534179688,
      "activations/layer20_attention_weight_min": -13.882840156555176,
      "activations/layer21_attention_weight_max": 37.2764892578125,
      "activations/layer21_attention_weight_min": -16.880197525024414,
      "activations/layer22_attention_weight_max": 31.05303955078125,
      "activations/layer22_attention_weight_min": -15.346781730651855,
      "activations/layer23_attention_weight_max": 22.18209457397461,
      "activations/layer23_attention_weight_min": -16.146591186523438,
      "activations/layer2_attention_weight_max": 25.58767318725586,
      "activations/layer2_attention_weight_min": -22.762624740600586,
      "activations/layer3_attention_weight_max": 52.31449890136719,
      "activations/layer3_attention_weight_min": -51.416934967041016,
      "activations/layer4_attention_weight_max": 81.61293029785156,
      "activations/layer4_attention_weight_min": -71.96548461914062,
      "activations/layer5_attention_weight_max": 141.373046875,
      "activations/layer5_attention_weight_min": -119.10211181640625,
      "activations/layer6_attention_weight_max": 72.1578598022461,
      "activations/layer6_attention_weight_min": -58.361900329589844,
      "activations/layer7_attention_weight_max": 75.87582397460938,
      "activations/layer7_attention_weight_min": -60.43565368652344,
      "activations/layer8_attention_weight_max": 59.1727294921875,
      "activations/layer8_attention_weight_min": -48.94824981689453,
      "activations/layer9_attention_weight_max": 69.55839538574219,
      "activations/layer9_attention_weight_min": -63.57331085205078,
      "epoch": 0.86,
      "learning_rate": 0.00014593939393939392,
      "loss": 3.1345,
      "step": 14750
    },
    {
      "activations/layer0_attention_weight_max": 14.207193374633789,
      "activations/layer0_attention_weight_min": -10.33248519897461,
      "activations/layer10_attention_weight_max": 74.54047393798828,
      "activations/layer10_attention_weight_min": -58.65393829345703,
      "activations/layer11_attention_weight_max": 45.13808822631836,
      "activations/layer11_attention_weight_min": -41.07676696777344,
      "activations/layer12_attention_weight_max": 30.067241668701172,
      "activations/layer12_attention_weight_min": -23.605192184448242,
      "activations/layer13_attention_weight_max": 55.53325653076172,
      "activations/layer13_attention_weight_min": -44.742591857910156,
      "activations/layer14_attention_weight_max": 52.276771545410156,
      "activations/layer14_attention_weight_min": -40.91230773925781,
      "activations/layer15_attention_weight_max": 56.03715515136719,
      "activations/layer15_attention_weight_min": -40.46712875366211,
      "activations/layer16_attention_weight_max": 59.54774856567383,
      "activations/layer16_attention_weight_min": -36.30939483642578,
      "activations/layer17_attention_weight_max": 56.272789001464844,
      "activations/layer17_attention_weight_min": -39.14591979980469,
      "activations/layer18_attention_weight_max": 65.6631088256836,
      "activations/layer18_attention_weight_min": -37.517581939697266,
      "activations/layer19_attention_weight_max": 22.638566970825195,
      "activations/layer19_attention_weight_min": -11.91474723815918,
      "activations/layer1_attention_weight_max": 16.550636291503906,
      "activations/layer1_attention_weight_min": -10.201144218444824,
      "activations/layer20_attention_weight_max": 18.716617584228516,
      "activations/layer20_attention_weight_min": -13.077688217163086,
      "activations/layer21_attention_weight_max": 33.099700927734375,
      "activations/layer21_attention_weight_min": -16.82965660095215,
      "activations/layer22_attention_weight_max": 30.99538803100586,
      "activations/layer22_attention_weight_min": -14.684691429138184,
      "activations/layer23_attention_weight_max": 22.767370223999023,
      "activations/layer23_attention_weight_min": -19.01186752319336,
      "activations/layer2_attention_weight_max": 25.29715347290039,
      "activations/layer2_attention_weight_min": -21.04353904724121,
      "activations/layer3_attention_weight_max": 52.14289474487305,
      "activations/layer3_attention_weight_min": -49.184139251708984,
      "activations/layer4_attention_weight_max": 83.00344848632812,
      "activations/layer4_attention_weight_min": -70.56433868408203,
      "activations/layer5_attention_weight_max": 129.48312377929688,
      "activations/layer5_attention_weight_min": -124.59274291992188,
      "activations/layer6_attention_weight_max": 64.36087036132812,
      "activations/layer6_attention_weight_min": -58.086753845214844,
      "activations/layer7_attention_weight_max": 64.38215637207031,
      "activations/layer7_attention_weight_min": -58.860137939453125,
      "activations/layer8_attention_weight_max": 57.03049850463867,
      "activations/layer8_attention_weight_min": -48.326324462890625,
      "activations/layer9_attention_weight_max": 72.11974334716797,
      "activations/layer9_attention_weight_min": -67.38590240478516,
      "epoch": 0.86,
      "learning_rate": 0.00014592045454545454,
      "loss": 3.1394,
      "step": 14800
    },
    {
      "activations/layer0_attention_weight_max": 14.293639183044434,
      "activations/layer0_attention_weight_min": -9.74474811553955,
      "activations/layer10_attention_weight_max": 72.20796203613281,
      "activations/layer10_attention_weight_min": -58.3101806640625,
      "activations/layer11_attention_weight_max": 44.79853439331055,
      "activations/layer11_attention_weight_min": -38.61989212036133,
      "activations/layer12_attention_weight_max": 28.368492126464844,
      "activations/layer12_attention_weight_min": -23.55222511291504,
      "activations/layer13_attention_weight_max": 70.77650451660156,
      "activations/layer13_attention_weight_min": -51.51522445678711,
      "activations/layer14_attention_weight_max": 57.93446731567383,
      "activations/layer14_attention_weight_min": -38.09574890136719,
      "activations/layer15_attention_weight_max": 54.015052795410156,
      "activations/layer15_attention_weight_min": -37.853973388671875,
      "activations/layer16_attention_weight_max": 57.15507507324219,
      "activations/layer16_attention_weight_min": -36.67018508911133,
      "activations/layer17_attention_weight_max": 61.32257843017578,
      "activations/layer17_attention_weight_min": -40.427032470703125,
      "activations/layer18_attention_weight_max": 56.091487884521484,
      "activations/layer18_attention_weight_min": -33.206275939941406,
      "activations/layer19_attention_weight_max": 23.096540451049805,
      "activations/layer19_attention_weight_min": -11.845784187316895,
      "activations/layer1_attention_weight_max": 15.781770706176758,
      "activations/layer1_attention_weight_min": -10.60824203491211,
      "activations/layer20_attention_weight_max": 19.69514274597168,
      "activations/layer20_attention_weight_min": -15.04456901550293,
      "activations/layer21_attention_weight_max": 35.927001953125,
      "activations/layer21_attention_weight_min": -16.134485244750977,
      "activations/layer22_attention_weight_max": 34.49763107299805,
      "activations/layer22_attention_weight_min": -16.298961639404297,
      "activations/layer23_attention_weight_max": 25.609630584716797,
      "activations/layer23_attention_weight_min": -18.608829498291016,
      "activations/layer2_attention_weight_max": 25.96709442138672,
      "activations/layer2_attention_weight_min": -23.020660400390625,
      "activations/layer3_attention_weight_max": 52.353477478027344,
      "activations/layer3_attention_weight_min": -49.013370513916016,
      "activations/layer4_attention_weight_max": 80.96119689941406,
      "activations/layer4_attention_weight_min": -71.3510513305664,
      "activations/layer5_attention_weight_max": 146.62957763671875,
      "activations/layer5_attention_weight_min": -127.09625244140625,
      "activations/layer6_attention_weight_max": 68.85598754882812,
      "activations/layer6_attention_weight_min": -59.92780303955078,
      "activations/layer7_attention_weight_max": 70.62506866455078,
      "activations/layer7_attention_weight_min": -58.80036926269531,
      "activations/layer8_attention_weight_max": 63.75551986694336,
      "activations/layer8_attention_weight_min": -57.80601119995117,
      "activations/layer9_attention_weight_max": 72.31300354003906,
      "activations/layer9_attention_weight_min": -65.41329193115234,
      "epoch": 0.86,
      "learning_rate": 0.00014590151515151515,
      "loss": 3.1474,
      "step": 14850
    },
    {
      "activations/layer0_attention_weight_max": 13.700843811035156,
      "activations/layer0_attention_weight_min": -10.291686058044434,
      "activations/layer10_attention_weight_max": 66.46666717529297,
      "activations/layer10_attention_weight_min": -60.13875198364258,
      "activations/layer11_attention_weight_max": 45.633522033691406,
      "activations/layer11_attention_weight_min": -39.640716552734375,
      "activations/layer12_attention_weight_max": 24.722232818603516,
      "activations/layer12_attention_weight_min": -22.140586853027344,
      "activations/layer13_attention_weight_max": 63.53968811035156,
      "activations/layer13_attention_weight_min": -53.83535385131836,
      "activations/layer14_attention_weight_max": 57.191612243652344,
      "activations/layer14_attention_weight_min": -49.330810546875,
      "activations/layer15_attention_weight_max": 48.0700569152832,
      "activations/layer15_attention_weight_min": -40.14999771118164,
      "activations/layer16_attention_weight_max": 49.33903884887695,
      "activations/layer16_attention_weight_min": -35.40965270996094,
      "activations/layer17_attention_weight_max": 53.634422302246094,
      "activations/layer17_attention_weight_min": -38.08780288696289,
      "activations/layer18_attention_weight_max": 58.52202606201172,
      "activations/layer18_attention_weight_min": -39.21717071533203,
      "activations/layer19_attention_weight_max": 20.913330078125,
      "activations/layer19_attention_weight_min": -12.09550952911377,
      "activations/layer1_attention_weight_max": 16.85713768005371,
      "activations/layer1_attention_weight_min": -11.26550579071045,
      "activations/layer20_attention_weight_max": 18.386125564575195,
      "activations/layer20_attention_weight_min": -14.809060096740723,
      "activations/layer21_attention_weight_max": 29.144893646240234,
      "activations/layer21_attention_weight_min": -15.680776596069336,
      "activations/layer22_attention_weight_max": 30.150890350341797,
      "activations/layer22_attention_weight_min": -15.404295921325684,
      "activations/layer23_attention_weight_max": 20.836223602294922,
      "activations/layer23_attention_weight_min": -18.69625473022461,
      "activations/layer2_attention_weight_max": 24.809024810791016,
      "activations/layer2_attention_weight_min": -22.755582809448242,
      "activations/layer3_attention_weight_max": 52.26323318481445,
      "activations/layer3_attention_weight_min": -52.32320022583008,
      "activations/layer4_attention_weight_max": 79.92689514160156,
      "activations/layer4_attention_weight_min": -72.54698181152344,
      "activations/layer5_attention_weight_max": 136.72689819335938,
      "activations/layer5_attention_weight_min": -121.25506591796875,
      "activations/layer6_attention_weight_max": 66.03165435791016,
      "activations/layer6_attention_weight_min": -57.92231369018555,
      "activations/layer7_attention_weight_max": 66.24549102783203,
      "activations/layer7_attention_weight_min": -56.12596893310547,
      "activations/layer8_attention_weight_max": 55.19621276855469,
      "activations/layer8_attention_weight_min": -49.70073699951172,
      "activations/layer9_attention_weight_max": 74.26856994628906,
      "activations/layer9_attention_weight_min": -73.92451477050781,
      "epoch": 0.87,
      "learning_rate": 0.00014588257575757574,
      "loss": 3.133,
      "step": 14900
    },
    {
      "activations/layer0_attention_weight_max": 12.948113441467285,
      "activations/layer0_attention_weight_min": -10.56338882446289,
      "activations/layer10_attention_weight_max": 69.98175048828125,
      "activations/layer10_attention_weight_min": -57.12534713745117,
      "activations/layer11_attention_weight_max": 45.971885681152344,
      "activations/layer11_attention_weight_min": -39.08629608154297,
      "activations/layer12_attention_weight_max": 24.612613677978516,
      "activations/layer12_attention_weight_min": -22.85124397277832,
      "activations/layer13_attention_weight_max": 56.01710510253906,
      "activations/layer13_attention_weight_min": -47.853580474853516,
      "activations/layer14_attention_weight_max": 44.87440872192383,
      "activations/layer14_attention_weight_min": -37.20453643798828,
      "activations/layer15_attention_weight_max": 48.175811767578125,
      "activations/layer15_attention_weight_min": -38.79240798950195,
      "activations/layer16_attention_weight_max": 45.64641189575195,
      "activations/layer16_attention_weight_min": -34.27682113647461,
      "activations/layer17_attention_weight_max": 49.49677276611328,
      "activations/layer17_attention_weight_min": -36.25425720214844,
      "activations/layer18_attention_weight_max": 47.38873291015625,
      "activations/layer18_attention_weight_min": -33.372222900390625,
      "activations/layer19_attention_weight_max": 21.409772872924805,
      "activations/layer19_attention_weight_min": -11.751622200012207,
      "activations/layer1_attention_weight_max": 16.4594783782959,
      "activations/layer1_attention_weight_min": -10.012514114379883,
      "activations/layer20_attention_weight_max": 18.25920867919922,
      "activations/layer20_attention_weight_min": -13.845710754394531,
      "activations/layer21_attention_weight_max": 27.609905242919922,
      "activations/layer21_attention_weight_min": -16.085432052612305,
      "activations/layer22_attention_weight_max": 26.806259155273438,
      "activations/layer22_attention_weight_min": -13.406607627868652,
      "activations/layer23_attention_weight_max": 19.576580047607422,
      "activations/layer23_attention_weight_min": -16.093997955322266,
      "activations/layer2_attention_weight_max": 24.904088973999023,
      "activations/layer2_attention_weight_min": -23.09088897705078,
      "activations/layer3_attention_weight_max": 50.38669967651367,
      "activations/layer3_attention_weight_min": -50.25383758544922,
      "activations/layer4_attention_weight_max": 77.2101058959961,
      "activations/layer4_attention_weight_min": -70.96461486816406,
      "activations/layer5_attention_weight_max": 131.31065368652344,
      "activations/layer5_attention_weight_min": -117.29412841796875,
      "activations/layer6_attention_weight_max": 62.126800537109375,
      "activations/layer6_attention_weight_min": -56.558563232421875,
      "activations/layer7_attention_weight_max": 68.6869125366211,
      "activations/layer7_attention_weight_min": -58.5145149230957,
      "activations/layer8_attention_weight_max": 53.18202209472656,
      "activations/layer8_attention_weight_min": -51.75721740722656,
      "activations/layer9_attention_weight_max": 76.89488220214844,
      "activations/layer9_attention_weight_min": -66.48112487792969,
      "epoch": 0.87,
      "learning_rate": 0.00014586363636363636,
      "loss": 3.148,
      "step": 14950
    },
    {
      "activations/layer0_attention_weight_max": 13.211494445800781,
      "activations/layer0_attention_weight_min": -10.528501510620117,
      "activations/layer10_attention_weight_max": 69.24832153320312,
      "activations/layer10_attention_weight_min": -57.5029296875,
      "activations/layer11_attention_weight_max": 48.928611755371094,
      "activations/layer11_attention_weight_min": -44.172420501708984,
      "activations/layer12_attention_weight_max": 27.941755294799805,
      "activations/layer12_attention_weight_min": -24.133529663085938,
      "activations/layer13_attention_weight_max": 70.27577209472656,
      "activations/layer13_attention_weight_min": -49.47624969482422,
      "activations/layer14_attention_weight_max": 63.71721649169922,
      "activations/layer14_attention_weight_min": -47.59128952026367,
      "activations/layer15_attention_weight_max": 52.22962951660156,
      "activations/layer15_attention_weight_min": -37.79314422607422,
      "activations/layer16_attention_weight_max": 47.676570892333984,
      "activations/layer16_attention_weight_min": -35.62921905517578,
      "activations/layer17_attention_weight_max": 62.09881591796875,
      "activations/layer17_attention_weight_min": -40.748111724853516,
      "activations/layer18_attention_weight_max": 56.42991256713867,
      "activations/layer18_attention_weight_min": -35.231178283691406,
      "activations/layer19_attention_weight_max": 21.838964462280273,
      "activations/layer19_attention_weight_min": -12.2857084274292,
      "activations/layer1_attention_weight_max": 16.07612419128418,
      "activations/layer1_attention_weight_min": -10.876036643981934,
      "activations/layer20_attention_weight_max": 17.224382400512695,
      "activations/layer20_attention_weight_min": -13.29378890991211,
      "activations/layer21_attention_weight_max": 27.943498611450195,
      "activations/layer21_attention_weight_min": -16.826679229736328,
      "activations/layer22_attention_weight_max": 31.592453002929688,
      "activations/layer22_attention_weight_min": -14.951275825500488,
      "activations/layer23_attention_weight_max": 23.15418815612793,
      "activations/layer23_attention_weight_min": -18.083240509033203,
      "activations/layer2_attention_weight_max": 27.5698299407959,
      "activations/layer2_attention_weight_min": -22.629547119140625,
      "activations/layer3_attention_weight_max": 52.22356414794922,
      "activations/layer3_attention_weight_min": -50.35299301147461,
      "activations/layer4_attention_weight_max": 87.11781311035156,
      "activations/layer4_attention_weight_min": -76.55862426757812,
      "activations/layer5_attention_weight_max": 146.04412841796875,
      "activations/layer5_attention_weight_min": -121.65190124511719,
      "activations/layer6_attention_weight_max": 66.9836196899414,
      "activations/layer6_attention_weight_min": -59.00783157348633,
      "activations/layer7_attention_weight_max": 67.24295043945312,
      "activations/layer7_attention_weight_min": -58.72468185424805,
      "activations/layer8_attention_weight_max": 60.93332290649414,
      "activations/layer8_attention_weight_min": -53.824851989746094,
      "activations/layer9_attention_weight_max": 82.68555450439453,
      "activations/layer9_attention_weight_min": -64.46619415283203,
      "epoch": 0.87,
      "learning_rate": 0.00014584469696969697,
      "loss": 3.1191,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_loss": 3.07421875,
      "eval_runtime": 8.2333,
      "eval_samples_per_second": 521.542,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_openwebtext_loss": 3.07421875,
      "eval_openwebtext_ppl": 21.632974546568068,
      "eval_openwebtext_runtime": 8.2333,
      "eval_openwebtext_samples_per_second": 521.542,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_wikitext_loss": 3.36328125,
      "eval_wikitext_ppl": 28.883810562338418,
      "eval_wikitext_runtime": 1.7028,
      "eval_wikitext_samples_per_second": 267.801,
      "step": 15000
    },
    {
      "epoch": 0.87,
      "eval_lambada_loss": 3.634765625,
      "eval_lambada_ppl": 37.89297069129353,
      "eval_lambada_runtime": 9.2814,
      "eval_lambada_samples_per_second": 524.595,
      "step": 15000
    },
    {
      "activations/layer0_attention_weight_max": 14.348001480102539,
      "activations/layer0_attention_weight_min": -11.469464302062988,
      "activations/layer10_attention_weight_max": 75.47035217285156,
      "activations/layer10_attention_weight_min": -71.21187591552734,
      "activations/layer11_attention_weight_max": 46.304969787597656,
      "activations/layer11_attention_weight_min": -41.65111541748047,
      "activations/layer12_attention_weight_max": 26.152009963989258,
      "activations/layer12_attention_weight_min": -26.032072067260742,
      "activations/layer13_attention_weight_max": 59.05506896972656,
      "activations/layer13_attention_weight_min": -42.808570861816406,
      "activations/layer14_attention_weight_max": 45.10265350341797,
      "activations/layer14_attention_weight_min": -38.012237548828125,
      "activations/layer15_attention_weight_max": 46.81645965576172,
      "activations/layer15_attention_weight_min": -34.459144592285156,
      "activations/layer16_attention_weight_max": 45.1654052734375,
      "activations/layer16_attention_weight_min": -36.243099212646484,
      "activations/layer17_attention_weight_max": 52.48033905029297,
      "activations/layer17_attention_weight_min": -38.50373077392578,
      "activations/layer18_attention_weight_max": 63.20629119873047,
      "activations/layer18_attention_weight_min": -34.833438873291016,
      "activations/layer19_attention_weight_max": 18.61713218688965,
      "activations/layer19_attention_weight_min": -11.375285148620605,
      "activations/layer1_attention_weight_max": 15.856298446655273,
      "activations/layer1_attention_weight_min": -9.624146461486816,
      "activations/layer20_attention_weight_max": 18.469524383544922,
      "activations/layer20_attention_weight_min": -13.438179969787598,
      "activations/layer21_attention_weight_max": 36.619361877441406,
      "activations/layer21_attention_weight_min": -13.867247581481934,
      "activations/layer22_attention_weight_max": 32.10526657104492,
      "activations/layer22_attention_weight_min": -14.02246379852295,
      "activations/layer23_attention_weight_max": 19.951107025146484,
      "activations/layer23_attention_weight_min": -18.431638717651367,
      "activations/layer2_attention_weight_max": 25.40380096435547,
      "activations/layer2_attention_weight_min": -22.346967697143555,
      "activations/layer3_attention_weight_max": 52.31013107299805,
      "activations/layer3_attention_weight_min": -49.72996139526367,
      "activations/layer4_attention_weight_max": 86.21685028076172,
      "activations/layer4_attention_weight_min": -71.11550903320312,
      "activations/layer5_attention_weight_max": 143.0489501953125,
      "activations/layer5_attention_weight_min": -122.49510955810547,
      "activations/layer6_attention_weight_max": 72.964599609375,
      "activations/layer6_attention_weight_min": -60.937713623046875,
      "activations/layer7_attention_weight_max": 69.84834289550781,
      "activations/layer7_attention_weight_min": -63.74260330200195,
      "activations/layer8_attention_weight_max": 62.35580825805664,
      "activations/layer8_attention_weight_min": -52.843292236328125,
      "activations/layer9_attention_weight_max": 89.08836364746094,
      "activations/layer9_attention_weight_min": -78.63932037353516,
      "epoch": 0.87,
      "learning_rate": 0.00014582575757575756,
      "loss": 3.1391,
      "step": 15050
    },
    {
      "activations/layer0_attention_weight_max": 16.257986068725586,
      "activations/layer0_attention_weight_min": -10.949601173400879,
      "activations/layer10_attention_weight_max": 65.41334533691406,
      "activations/layer10_attention_weight_min": -59.03244400024414,
      "activations/layer11_attention_weight_max": 43.54377365112305,
      "activations/layer11_attention_weight_min": -40.314781188964844,
      "activations/layer12_attention_weight_max": 29.483789443969727,
      "activations/layer12_attention_weight_min": -24.472686767578125,
      "activations/layer13_attention_weight_max": 57.420196533203125,
      "activations/layer13_attention_weight_min": -47.099891662597656,
      "activations/layer14_attention_weight_max": 46.52699279785156,
      "activations/layer14_attention_weight_min": -37.049373626708984,
      "activations/layer15_attention_weight_max": 45.13884353637695,
      "activations/layer15_attention_weight_min": -37.31058883666992,
      "activations/layer16_attention_weight_max": 45.04528045654297,
      "activations/layer16_attention_weight_min": -32.847686767578125,
      "activations/layer17_attention_weight_max": 57.69784164428711,
      "activations/layer17_attention_weight_min": -38.7758674621582,
      "activations/layer18_attention_weight_max": 54.43317794799805,
      "activations/layer18_attention_weight_min": -37.191558837890625,
      "activations/layer19_attention_weight_max": 20.980030059814453,
      "activations/layer19_attention_weight_min": -12.340692520141602,
      "activations/layer1_attention_weight_max": 15.624842643737793,
      "activations/layer1_attention_weight_min": -10.205531120300293,
      "activations/layer20_attention_weight_max": 17.294328689575195,
      "activations/layer20_attention_weight_min": -15.501161575317383,
      "activations/layer21_attention_weight_max": 32.25539779663086,
      "activations/layer21_attention_weight_min": -16.855409622192383,
      "activations/layer22_attention_weight_max": 30.557207107543945,
      "activations/layer22_attention_weight_min": -16.92159652709961,
      "activations/layer23_attention_weight_max": 21.51266098022461,
      "activations/layer23_attention_weight_min": -18.098102569580078,
      "activations/layer2_attention_weight_max": 25.179288864135742,
      "activations/layer2_attention_weight_min": -22.181072235107422,
      "activations/layer3_attention_weight_max": 50.122074127197266,
      "activations/layer3_attention_weight_min": -48.372108459472656,
      "activations/layer4_attention_weight_max": 75.85094451904297,
      "activations/layer4_attention_weight_min": -71.4291000366211,
      "activations/layer5_attention_weight_max": 132.55078125,
      "activations/layer5_attention_weight_min": -123.64305877685547,
      "activations/layer6_attention_weight_max": 65.89896392822266,
      "activations/layer6_attention_weight_min": -62.31683349609375,
      "activations/layer7_attention_weight_max": 64.93175506591797,
      "activations/layer7_attention_weight_min": -60.07582473754883,
      "activations/layer8_attention_weight_max": 56.61079025268555,
      "activations/layer8_attention_weight_min": -52.582725524902344,
      "activations/layer9_attention_weight_max": 73.6080093383789,
      "activations/layer9_attention_weight_min": -71.63806915283203,
      "epoch": 0.88,
      "learning_rate": 0.00014580681818181817,
      "loss": 3.1293,
      "step": 15100
    },
    {
      "activations/layer0_attention_weight_max": 13.45791244506836,
      "activations/layer0_attention_weight_min": -10.745765686035156,
      "activations/layer10_attention_weight_max": 68.516357421875,
      "activations/layer10_attention_weight_min": -58.63947296142578,
      "activations/layer11_attention_weight_max": 50.40813446044922,
      "activations/layer11_attention_weight_min": -39.02564239501953,
      "activations/layer12_attention_weight_max": 27.235700607299805,
      "activations/layer12_attention_weight_min": -22.923051834106445,
      "activations/layer13_attention_weight_max": 64.95362091064453,
      "activations/layer13_attention_weight_min": -50.53776931762695,
      "activations/layer14_attention_weight_max": 43.26502990722656,
      "activations/layer14_attention_weight_min": -35.03761672973633,
      "activations/layer15_attention_weight_max": 48.86032485961914,
      "activations/layer15_attention_weight_min": -37.135250091552734,
      "activations/layer16_attention_weight_max": 49.72856521606445,
      "activations/layer16_attention_weight_min": -35.51463317871094,
      "activations/layer17_attention_weight_max": 59.64276123046875,
      "activations/layer17_attention_weight_min": -39.06831741333008,
      "activations/layer18_attention_weight_max": 56.561100006103516,
      "activations/layer18_attention_weight_min": -35.13386154174805,
      "activations/layer19_attention_weight_max": 19.69624900817871,
      "activations/layer19_attention_weight_min": -13.011874198913574,
      "activations/layer1_attention_weight_max": 17.867443084716797,
      "activations/layer1_attention_weight_min": -10.553574562072754,
      "activations/layer20_attention_weight_max": 17.85256576538086,
      "activations/layer20_attention_weight_min": -13.568730354309082,
      "activations/layer21_attention_weight_max": 30.854076385498047,
      "activations/layer21_attention_weight_min": -17.578022003173828,
      "activations/layer22_attention_weight_max": 30.301198959350586,
      "activations/layer22_attention_weight_min": -15.46651840209961,
      "activations/layer23_attention_weight_max": 21.867895126342773,
      "activations/layer23_attention_weight_min": -18.186723709106445,
      "activations/layer2_attention_weight_max": 23.271068572998047,
      "activations/layer2_attention_weight_min": -21.809009552001953,
      "activations/layer3_attention_weight_max": 49.809783935546875,
      "activations/layer3_attention_weight_min": -49.20325469970703,
      "activations/layer4_attention_weight_max": 75.91603088378906,
      "activations/layer4_attention_weight_min": -73.48712921142578,
      "activations/layer5_attention_weight_max": 129.99575805664062,
      "activations/layer5_attention_weight_min": -120.17816162109375,
      "activations/layer6_attention_weight_max": 66.66258239746094,
      "activations/layer6_attention_weight_min": -58.17948532104492,
      "activations/layer7_attention_weight_max": 62.603546142578125,
      "activations/layer7_attention_weight_min": -56.83360290527344,
      "activations/layer8_attention_weight_max": 59.0977783203125,
      "activations/layer8_attention_weight_min": -52.63645935058594,
      "activations/layer9_attention_weight_max": 79.58901977539062,
      "activations/layer9_attention_weight_min": -62.00080490112305,
      "epoch": 0.88,
      "learning_rate": 0.0001457878787878788,
      "loss": 3.123,
      "step": 15150
    },
    {
      "activations/layer0_attention_weight_max": 14.229498863220215,
      "activations/layer0_attention_weight_min": -10.686569213867188,
      "activations/layer10_attention_weight_max": 67.13382720947266,
      "activations/layer10_attention_weight_min": -61.13022994995117,
      "activations/layer11_attention_weight_max": 44.90208053588867,
      "activations/layer11_attention_weight_min": -40.53543472290039,
      "activations/layer12_attention_weight_max": 28.3767032623291,
      "activations/layer12_attention_weight_min": -24.608762741088867,
      "activations/layer13_attention_weight_max": 61.74351501464844,
      "activations/layer13_attention_weight_min": -45.393836975097656,
      "activations/layer14_attention_weight_max": 48.4227180480957,
      "activations/layer14_attention_weight_min": -37.57592010498047,
      "activations/layer15_attention_weight_max": 54.409263610839844,
      "activations/layer15_attention_weight_min": -40.505035400390625,
      "activations/layer16_attention_weight_max": 49.63969802856445,
      "activations/layer16_attention_weight_min": -37.62417984008789,
      "activations/layer17_attention_weight_max": 55.016883850097656,
      "activations/layer17_attention_weight_min": -38.81343078613281,
      "activations/layer18_attention_weight_max": 57.89804458618164,
      "activations/layer18_attention_weight_min": -43.03398513793945,
      "activations/layer19_attention_weight_max": 21.30266761779785,
      "activations/layer19_attention_weight_min": -11.587416648864746,
      "activations/layer1_attention_weight_max": 15.8572416305542,
      "activations/layer1_attention_weight_min": -10.660026550292969,
      "activations/layer20_attention_weight_max": 16.937644958496094,
      "activations/layer20_attention_weight_min": -14.867835998535156,
      "activations/layer21_attention_weight_max": 33.2360954284668,
      "activations/layer21_attention_weight_min": -18.068384170532227,
      "activations/layer22_attention_weight_max": 31.671791076660156,
      "activations/layer22_attention_weight_min": -15.22884464263916,
      "activations/layer23_attention_weight_max": 21.67882537841797,
      "activations/layer23_attention_weight_min": -17.476055145263672,
      "activations/layer2_attention_weight_max": 27.243797302246094,
      "activations/layer2_attention_weight_min": -21.836959838867188,
      "activations/layer3_attention_weight_max": 52.356956481933594,
      "activations/layer3_attention_weight_min": -50.63320541381836,
      "activations/layer4_attention_weight_max": 81.63936614990234,
      "activations/layer4_attention_weight_min": -73.03839874267578,
      "activations/layer5_attention_weight_max": 137.52430725097656,
      "activations/layer5_attention_weight_min": -124.12679290771484,
      "activations/layer6_attention_weight_max": 70.01239013671875,
      "activations/layer6_attention_weight_min": -59.23103713989258,
      "activations/layer7_attention_weight_max": 68.38407897949219,
      "activations/layer7_attention_weight_min": -60.60308837890625,
      "activations/layer8_attention_weight_max": 57.08053970336914,
      "activations/layer8_attention_weight_min": -52.206241607666016,
      "activations/layer9_attention_weight_max": 80.90255737304688,
      "activations/layer9_attention_weight_min": -70.8164291381836,
      "epoch": 0.88,
      "learning_rate": 0.00014576893939393938,
      "loss": 3.1389,
      "step": 15200
    },
    {
      "activations/layer0_attention_weight_max": 13.161574363708496,
      "activations/layer0_attention_weight_min": -11.03652286529541,
      "activations/layer10_attention_weight_max": 80.41267395019531,
      "activations/layer10_attention_weight_min": -62.46365737915039,
      "activations/layer11_attention_weight_max": 47.46424865722656,
      "activations/layer11_attention_weight_min": -39.337921142578125,
      "activations/layer12_attention_weight_max": 32.429420471191406,
      "activations/layer12_attention_weight_min": -24.72034454345703,
      "activations/layer13_attention_weight_max": 73.89180755615234,
      "activations/layer13_attention_weight_min": -43.18161392211914,
      "activations/layer14_attention_weight_max": 68.1548080444336,
      "activations/layer14_attention_weight_min": -37.928165435791016,
      "activations/layer15_attention_weight_max": 69.4535140991211,
      "activations/layer15_attention_weight_min": -41.70868682861328,
      "activations/layer16_attention_weight_max": 64.5400161743164,
      "activations/layer16_attention_weight_min": -35.40235900878906,
      "activations/layer17_attention_weight_max": 72.51889038085938,
      "activations/layer17_attention_weight_min": -34.94941329956055,
      "activations/layer18_attention_weight_max": 89.51332092285156,
      "activations/layer18_attention_weight_min": -44.03478240966797,
      "activations/layer19_attention_weight_max": 22.492700576782227,
      "activations/layer19_attention_weight_min": -11.499347686767578,
      "activations/layer1_attention_weight_max": 16.076244354248047,
      "activations/layer1_attention_weight_min": -11.348888397216797,
      "activations/layer20_attention_weight_max": 20.080394744873047,
      "activations/layer20_attention_weight_min": -13.297943115234375,
      "activations/layer21_attention_weight_max": 39.652462005615234,
      "activations/layer21_attention_weight_min": -18.709421157836914,
      "activations/layer22_attention_weight_max": 35.72138214111328,
      "activations/layer22_attention_weight_min": -14.826168060302734,
      "activations/layer23_attention_weight_max": 26.170345306396484,
      "activations/layer23_attention_weight_min": -18.447193145751953,
      "activations/layer2_attention_weight_max": 25.53607940673828,
      "activations/layer2_attention_weight_min": -22.243810653686523,
      "activations/layer3_attention_weight_max": 52.6641731262207,
      "activations/layer3_attention_weight_min": -50.34208297729492,
      "activations/layer4_attention_weight_max": 79.41651153564453,
      "activations/layer4_attention_weight_min": -72.16262817382812,
      "activations/layer5_attention_weight_max": 138.21295166015625,
      "activations/layer5_attention_weight_min": -122.9701156616211,
      "activations/layer6_attention_weight_max": 64.30347442626953,
      "activations/layer6_attention_weight_min": -58.43206024169922,
      "activations/layer7_attention_weight_max": 68.41679382324219,
      "activations/layer7_attention_weight_min": -58.45936965942383,
      "activations/layer8_attention_weight_max": 58.4786262512207,
      "activations/layer8_attention_weight_min": -50.00670623779297,
      "activations/layer9_attention_weight_max": 77.62922668457031,
      "activations/layer9_attention_weight_min": -66.13912200927734,
      "epoch": 0.89,
      "learning_rate": 0.00014575,
      "loss": 3.1317,
      "step": 15250
    },
    {
      "activations/layer0_attention_weight_max": 14.026399612426758,
      "activations/layer0_attention_weight_min": -11.159618377685547,
      "activations/layer10_attention_weight_max": 64.59928131103516,
      "activations/layer10_attention_weight_min": -56.5416374206543,
      "activations/layer11_attention_weight_max": 44.70579147338867,
      "activations/layer11_attention_weight_min": -37.51081848144531,
      "activations/layer12_attention_weight_max": 25.462635040283203,
      "activations/layer12_attention_weight_min": -25.52419662475586,
      "activations/layer13_attention_weight_max": 62.782615661621094,
      "activations/layer13_attention_weight_min": -46.977394104003906,
      "activations/layer14_attention_weight_max": 54.75971984863281,
      "activations/layer14_attention_weight_min": -41.19898986816406,
      "activations/layer15_attention_weight_max": 50.494773864746094,
      "activations/layer15_attention_weight_min": -36.51926040649414,
      "activations/layer16_attention_weight_max": 47.12821960449219,
      "activations/layer16_attention_weight_min": -33.74400329589844,
      "activations/layer17_attention_weight_max": 63.4324836730957,
      "activations/layer17_attention_weight_min": -38.256954193115234,
      "activations/layer18_attention_weight_max": 61.802677154541016,
      "activations/layer18_attention_weight_min": -41.44198989868164,
      "activations/layer19_attention_weight_max": 20.780380249023438,
      "activations/layer19_attention_weight_min": -12.60851001739502,
      "activations/layer1_attention_weight_max": 15.439513206481934,
      "activations/layer1_attention_weight_min": -11.357868194580078,
      "activations/layer20_attention_weight_max": 19.979522705078125,
      "activations/layer20_attention_weight_min": -14.354694366455078,
      "activations/layer21_attention_weight_max": 34.10751724243164,
      "activations/layer21_attention_weight_min": -18.143381118774414,
      "activations/layer22_attention_weight_max": 32.40662384033203,
      "activations/layer22_attention_weight_min": -17.820432662963867,
      "activations/layer23_attention_weight_max": 23.503406524658203,
      "activations/layer23_attention_weight_min": -18.505144119262695,
      "activations/layer2_attention_weight_max": 25.371450424194336,
      "activations/layer2_attention_weight_min": -24.163734436035156,
      "activations/layer3_attention_weight_max": 51.54317092895508,
      "activations/layer3_attention_weight_min": -53.077735900878906,
      "activations/layer4_attention_weight_max": 80.2617416381836,
      "activations/layer4_attention_weight_min": -76.28408813476562,
      "activations/layer5_attention_weight_max": 138.07334899902344,
      "activations/layer5_attention_weight_min": -126.73705291748047,
      "activations/layer6_attention_weight_max": 68.43456268310547,
      "activations/layer6_attention_weight_min": -59.98863983154297,
      "activations/layer7_attention_weight_max": 66.96025848388672,
      "activations/layer7_attention_weight_min": -65.49589538574219,
      "activations/layer8_attention_weight_max": 56.99066162109375,
      "activations/layer8_attention_weight_min": -52.202545166015625,
      "activations/layer9_attention_weight_max": 70.5788803100586,
      "activations/layer9_attention_weight_min": -68.72457885742188,
      "epoch": 0.89,
      "learning_rate": 0.0001457310606060606,
      "loss": 3.1261,
      "step": 15300
    },
    {
      "activations/layer0_attention_weight_max": 13.197678565979004,
      "activations/layer0_attention_weight_min": -11.056574821472168,
      "activations/layer10_attention_weight_max": 77.34556579589844,
      "activations/layer10_attention_weight_min": -65.21092224121094,
      "activations/layer11_attention_weight_max": 49.46517562866211,
      "activations/layer11_attention_weight_min": -42.93159103393555,
      "activations/layer12_attention_weight_max": 26.983638763427734,
      "activations/layer12_attention_weight_min": -23.134172439575195,
      "activations/layer13_attention_weight_max": 60.156089782714844,
      "activations/layer13_attention_weight_min": -43.935062408447266,
      "activations/layer14_attention_weight_max": 51.48044204711914,
      "activations/layer14_attention_weight_min": -44.47692108154297,
      "activations/layer15_attention_weight_max": 52.42319869995117,
      "activations/layer15_attention_weight_min": -39.105587005615234,
      "activations/layer16_attention_weight_max": 55.760154724121094,
      "activations/layer16_attention_weight_min": -36.11534118652344,
      "activations/layer17_attention_weight_max": 72.31209564208984,
      "activations/layer17_attention_weight_min": -38.19039535522461,
      "activations/layer18_attention_weight_max": 71.78703308105469,
      "activations/layer18_attention_weight_min": -37.671478271484375,
      "activations/layer19_attention_weight_max": 23.767353057861328,
      "activations/layer19_attention_weight_min": -14.409685134887695,
      "activations/layer1_attention_weight_max": 15.63245964050293,
      "activations/layer1_attention_weight_min": -10.405322074890137,
      "activations/layer20_attention_weight_max": 24.671159744262695,
      "activations/layer20_attention_weight_min": -13.625138282775879,
      "activations/layer21_attention_weight_max": 36.17496871948242,
      "activations/layer21_attention_weight_min": -17.881793975830078,
      "activations/layer22_attention_weight_max": 36.13719940185547,
      "activations/layer22_attention_weight_min": -16.754846572875977,
      "activations/layer23_attention_weight_max": 23.0135498046875,
      "activations/layer23_attention_weight_min": -16.901653289794922,
      "activations/layer2_attention_weight_max": 24.95962905883789,
      "activations/layer2_attention_weight_min": -22.809703826904297,
      "activations/layer3_attention_weight_max": 53.61737060546875,
      "activations/layer3_attention_weight_min": -51.3964729309082,
      "activations/layer4_attention_weight_max": 84.73143768310547,
      "activations/layer4_attention_weight_min": -74.4116439819336,
      "activations/layer5_attention_weight_max": 137.35733032226562,
      "activations/layer5_attention_weight_min": -126.78459167480469,
      "activations/layer6_attention_weight_max": 66.44570922851562,
      "activations/layer6_attention_weight_min": -60.95079803466797,
      "activations/layer7_attention_weight_max": 66.99066162109375,
      "activations/layer7_attention_weight_min": -63.275146484375,
      "activations/layer8_attention_weight_max": 57.65366744995117,
      "activations/layer8_attention_weight_min": -51.232818603515625,
      "activations/layer9_attention_weight_max": 75.53079986572266,
      "activations/layer9_attention_weight_min": -66.20844268798828,
      "epoch": 0.89,
      "learning_rate": 0.0001457121212121212,
      "loss": 3.1053,
      "step": 15350
    },
    {
      "activations/layer0_attention_weight_max": 13.026448249816895,
      "activations/layer0_attention_weight_min": -11.305407524108887,
      "activations/layer10_attention_weight_max": 72.226806640625,
      "activations/layer10_attention_weight_min": -56.36933517456055,
      "activations/layer11_attention_weight_max": 45.2486686706543,
      "activations/layer11_attention_weight_min": -42.005374908447266,
      "activations/layer12_attention_weight_max": 31.609127044677734,
      "activations/layer12_attention_weight_min": -24.228368759155273,
      "activations/layer13_attention_weight_max": 111.56317138671875,
      "activations/layer13_attention_weight_min": -50.09825897216797,
      "activations/layer14_attention_weight_max": 126.47386169433594,
      "activations/layer14_attention_weight_min": -47.64913558959961,
      "activations/layer15_attention_weight_max": 60.30356979370117,
      "activations/layer15_attention_weight_min": -37.31517028808594,
      "activations/layer16_attention_weight_max": 45.79672622680664,
      "activations/layer16_attention_weight_min": -34.92521667480469,
      "activations/layer17_attention_weight_max": 53.561988830566406,
      "activations/layer17_attention_weight_min": -39.54865264892578,
      "activations/layer18_attention_weight_max": 48.57492446899414,
      "activations/layer18_attention_weight_min": -30.74346923828125,
      "activations/layer19_attention_weight_max": 19.97126579284668,
      "activations/layer19_attention_weight_min": -12.945052146911621,
      "activations/layer1_attention_weight_max": 15.103578567504883,
      "activations/layer1_attention_weight_min": -10.517779350280762,
      "activations/layer20_attention_weight_max": 22.75812339782715,
      "activations/layer20_attention_weight_min": -14.423206329345703,
      "activations/layer21_attention_weight_max": 36.741817474365234,
      "activations/layer21_attention_weight_min": -16.900768280029297,
      "activations/layer22_attention_weight_max": 31.606733322143555,
      "activations/layer22_attention_weight_min": -15.905784606933594,
      "activations/layer23_attention_weight_max": 22.2111873626709,
      "activations/layer23_attention_weight_min": -18.449398040771484,
      "activations/layer2_attention_weight_max": 26.468902587890625,
      "activations/layer2_attention_weight_min": -22.36429214477539,
      "activations/layer3_attention_weight_max": 53.88551330566406,
      "activations/layer3_attention_weight_min": -50.12198257446289,
      "activations/layer4_attention_weight_max": 84.93124389648438,
      "activations/layer4_attention_weight_min": -73.07200622558594,
      "activations/layer5_attention_weight_max": 151.4443817138672,
      "activations/layer5_attention_weight_min": -128.57595825195312,
      "activations/layer6_attention_weight_max": 70.03199768066406,
      "activations/layer6_attention_weight_min": -57.378787994384766,
      "activations/layer7_attention_weight_max": 73.21863555908203,
      "activations/layer7_attention_weight_min": -57.882568359375,
      "activations/layer8_attention_weight_max": 63.36198043823242,
      "activations/layer8_attention_weight_min": -52.306644439697266,
      "activations/layer9_attention_weight_max": 93.1709213256836,
      "activations/layer9_attention_weight_min": -72.66404724121094,
      "epoch": 0.89,
      "learning_rate": 0.0001456931818181818,
      "loss": 3.1155,
      "step": 15400
    },
    {
      "activations/layer0_attention_weight_max": 13.467784881591797,
      "activations/layer0_attention_weight_min": -10.584595680236816,
      "activations/layer10_attention_weight_max": 65.79912567138672,
      "activations/layer10_attention_weight_min": -56.43210983276367,
      "activations/layer11_attention_weight_max": 47.220703125,
      "activations/layer11_attention_weight_min": -37.57496643066406,
      "activations/layer12_attention_weight_max": 26.112537384033203,
      "activations/layer12_attention_weight_min": -22.3335018157959,
      "activations/layer13_attention_weight_max": 69.29479217529297,
      "activations/layer13_attention_weight_min": -43.005149841308594,
      "activations/layer14_attention_weight_max": 55.684852600097656,
      "activations/layer14_attention_weight_min": -36.3629150390625,
      "activations/layer15_attention_weight_max": 56.90660858154297,
      "activations/layer15_attention_weight_min": -35.93496322631836,
      "activations/layer16_attention_weight_max": 48.20789337158203,
      "activations/layer16_attention_weight_min": -32.2116584777832,
      "activations/layer17_attention_weight_max": 63.02407455444336,
      "activations/layer17_attention_weight_min": -36.741783142089844,
      "activations/layer18_attention_weight_max": 66.69345092773438,
      "activations/layer18_attention_weight_min": -36.368324279785156,
      "activations/layer19_attention_weight_max": 18.581119537353516,
      "activations/layer19_attention_weight_min": -11.490263938903809,
      "activations/layer1_attention_weight_max": 16.0753173828125,
      "activations/layer1_attention_weight_min": -11.124199867248535,
      "activations/layer20_attention_weight_max": 15.203401565551758,
      "activations/layer20_attention_weight_min": -11.477051734924316,
      "activations/layer21_attention_weight_max": 33.4948616027832,
      "activations/layer21_attention_weight_min": -18.857608795166016,
      "activations/layer22_attention_weight_max": 29.685821533203125,
      "activations/layer22_attention_weight_min": -16.017608642578125,
      "activations/layer23_attention_weight_max": 20.27310562133789,
      "activations/layer23_attention_weight_min": -16.149784088134766,
      "activations/layer2_attention_weight_max": 26.420942306518555,
      "activations/layer2_attention_weight_min": -23.85363006591797,
      "activations/layer3_attention_weight_max": 54.21670150756836,
      "activations/layer3_attention_weight_min": -53.35177230834961,
      "activations/layer4_attention_weight_max": 85.13019561767578,
      "activations/layer4_attention_weight_min": -73.20775604248047,
      "activations/layer5_attention_weight_max": 138.6878204345703,
      "activations/layer5_attention_weight_min": -122.7398681640625,
      "activations/layer6_attention_weight_max": 67.869384765625,
      "activations/layer6_attention_weight_min": -57.89323806762695,
      "activations/layer7_attention_weight_max": 65.03823852539062,
      "activations/layer7_attention_weight_min": -56.83452606201172,
      "activations/layer8_attention_weight_max": 57.49155807495117,
      "activations/layer8_attention_weight_min": -52.02592849731445,
      "activations/layer9_attention_weight_max": 76.56086730957031,
      "activations/layer9_attention_weight_min": -66.95207977294922,
      "epoch": 0.9,
      "learning_rate": 0.0001456742424242424,
      "loss": 3.1116,
      "step": 15450
    },
    {
      "activations/layer0_attention_weight_max": 14.751593589782715,
      "activations/layer0_attention_weight_min": -10.738582611083984,
      "activations/layer10_attention_weight_max": 67.72016906738281,
      "activations/layer10_attention_weight_min": -60.55731964111328,
      "activations/layer11_attention_weight_max": 44.10882568359375,
      "activations/layer11_attention_weight_min": -39.84645080566406,
      "activations/layer12_attention_weight_max": 27.610532760620117,
      "activations/layer12_attention_weight_min": -26.02932357788086,
      "activations/layer13_attention_weight_max": 65.06204223632812,
      "activations/layer13_attention_weight_min": -45.56687927246094,
      "activations/layer14_attention_weight_max": 39.50392150878906,
      "activations/layer14_attention_weight_min": -33.03679656982422,
      "activations/layer15_attention_weight_max": 50.50874710083008,
      "activations/layer15_attention_weight_min": -36.70277404785156,
      "activations/layer16_attention_weight_max": 44.56783676147461,
      "activations/layer16_attention_weight_min": -33.18656539916992,
      "activations/layer17_attention_weight_max": 53.42106246948242,
      "activations/layer17_attention_weight_min": -40.021297454833984,
      "activations/layer18_attention_weight_max": 58.79773712158203,
      "activations/layer18_attention_weight_min": -39.997840881347656,
      "activations/layer19_attention_weight_max": 20.78249740600586,
      "activations/layer19_attention_weight_min": -11.823083877563477,
      "activations/layer1_attention_weight_max": 15.599303245544434,
      "activations/layer1_attention_weight_min": -10.313283920288086,
      "activations/layer20_attention_weight_max": 17.46678924560547,
      "activations/layer20_attention_weight_min": -14.060698509216309,
      "activations/layer21_attention_weight_max": 31.166152954101562,
      "activations/layer21_attention_weight_min": -16.869308471679688,
      "activations/layer22_attention_weight_max": 29.939098358154297,
      "activations/layer22_attention_weight_min": -15.14223575592041,
      "activations/layer23_attention_weight_max": 19.954334259033203,
      "activations/layer23_attention_weight_min": -17.5142822265625,
      "activations/layer2_attention_weight_max": 25.34064483642578,
      "activations/layer2_attention_weight_min": -24.07330322265625,
      "activations/layer3_attention_weight_max": 53.29051208496094,
      "activations/layer3_attention_weight_min": -51.474246978759766,
      "activations/layer4_attention_weight_max": 85.50386810302734,
      "activations/layer4_attention_weight_min": -78.63998413085938,
      "activations/layer5_attention_weight_max": 153.76559448242188,
      "activations/layer5_attention_weight_min": -125.34471130371094,
      "activations/layer6_attention_weight_max": 72.60899353027344,
      "activations/layer6_attention_weight_min": -56.917724609375,
      "activations/layer7_attention_weight_max": 65.39996337890625,
      "activations/layer7_attention_weight_min": -57.64033889770508,
      "activations/layer8_attention_weight_max": 64.99364471435547,
      "activations/layer8_attention_weight_min": -52.45181655883789,
      "activations/layer9_attention_weight_max": 86.73026275634766,
      "activations/layer9_attention_weight_min": -74.88060760498047,
      "epoch": 0.9,
      "learning_rate": 0.000145655303030303,
      "loss": 3.1264,
      "step": 15500
    },
    {
      "activations/layer0_attention_weight_max": 13.99187183380127,
      "activations/layer0_attention_weight_min": -10.302084922790527,
      "activations/layer10_attention_weight_max": 70.2545166015625,
      "activations/layer10_attention_weight_min": -58.140865325927734,
      "activations/layer11_attention_weight_max": 50.00387191772461,
      "activations/layer11_attention_weight_min": -41.632503509521484,
      "activations/layer12_attention_weight_max": 35.01849365234375,
      "activations/layer12_attention_weight_min": -27.01078987121582,
      "activations/layer13_attention_weight_max": 64.21484375,
      "activations/layer13_attention_weight_min": -47.31275177001953,
      "activations/layer14_attention_weight_max": 45.69755172729492,
      "activations/layer14_attention_weight_min": -38.69536590576172,
      "activations/layer15_attention_weight_max": 66.08377838134766,
      "activations/layer15_attention_weight_min": -39.6822509765625,
      "activations/layer16_attention_weight_max": 49.92753601074219,
      "activations/layer16_attention_weight_min": -36.907745361328125,
      "activations/layer17_attention_weight_max": 59.75798416137695,
      "activations/layer17_attention_weight_min": -37.5698127746582,
      "activations/layer18_attention_weight_max": 77.48030853271484,
      "activations/layer18_attention_weight_min": -41.278411865234375,
      "activations/layer19_attention_weight_max": 22.10256576538086,
      "activations/layer19_attention_weight_min": -13.641679763793945,
      "activations/layer1_attention_weight_max": 16.245410919189453,
      "activations/layer1_attention_weight_min": -10.994510650634766,
      "activations/layer20_attention_weight_max": 22.533300399780273,
      "activations/layer20_attention_weight_min": -14.045470237731934,
      "activations/layer21_attention_weight_max": 37.94718933105469,
      "activations/layer21_attention_weight_min": -18.80911636352539,
      "activations/layer22_attention_weight_max": 35.705020904541016,
      "activations/layer22_attention_weight_min": -18.145179748535156,
      "activations/layer23_attention_weight_max": 23.91884422302246,
      "activations/layer23_attention_weight_min": -17.66387367248535,
      "activations/layer2_attention_weight_max": 25.41672134399414,
      "activations/layer2_attention_weight_min": -23.21951675415039,
      "activations/layer3_attention_weight_max": 56.24573516845703,
      "activations/layer3_attention_weight_min": -54.306480407714844,
      "activations/layer4_attention_weight_max": 80.70975494384766,
      "activations/layer4_attention_weight_min": -75.75767517089844,
      "activations/layer5_attention_weight_max": 138.46389770507812,
      "activations/layer5_attention_weight_min": -122.08236694335938,
      "activations/layer6_attention_weight_max": 68.42646026611328,
      "activations/layer6_attention_weight_min": -59.88490295410156,
      "activations/layer7_attention_weight_max": 69.84074401855469,
      "activations/layer7_attention_weight_min": -59.04616928100586,
      "activations/layer8_attention_weight_max": 62.06467819213867,
      "activations/layer8_attention_weight_min": -51.96087646484375,
      "activations/layer9_attention_weight_max": 74.46673583984375,
      "activations/layer9_attention_weight_min": -68.78305053710938,
      "epoch": 0.9,
      "learning_rate": 0.00014563636363636363,
      "loss": 3.1147,
      "step": 15550
    },
    {
      "activations/layer0_attention_weight_max": 15.546582221984863,
      "activations/layer0_attention_weight_min": -10.959351539611816,
      "activations/layer10_attention_weight_max": 66.56175231933594,
      "activations/layer10_attention_weight_min": -59.85699462890625,
      "activations/layer11_attention_weight_max": 45.3548698425293,
      "activations/layer11_attention_weight_min": -41.75297546386719,
      "activations/layer12_attention_weight_max": 26.061914443969727,
      "activations/layer12_attention_weight_min": -24.17850112915039,
      "activations/layer13_attention_weight_max": 60.03809356689453,
      "activations/layer13_attention_weight_min": -44.278724670410156,
      "activations/layer14_attention_weight_max": 53.41117477416992,
      "activations/layer14_attention_weight_min": -38.93204879760742,
      "activations/layer15_attention_weight_max": 53.14138412475586,
      "activations/layer15_attention_weight_min": -39.53266906738281,
      "activations/layer16_attention_weight_max": 51.32564926147461,
      "activations/layer16_attention_weight_min": -35.9424934387207,
      "activations/layer17_attention_weight_max": 59.83315658569336,
      "activations/layer17_attention_weight_min": -41.41511917114258,
      "activations/layer18_attention_weight_max": 70.54550170898438,
      "activations/layer18_attention_weight_min": -41.96857833862305,
      "activations/layer19_attention_weight_max": 21.882017135620117,
      "activations/layer19_attention_weight_min": -12.007522583007812,
      "activations/layer1_attention_weight_max": 14.455886840820312,
      "activations/layer1_attention_weight_min": -10.522014617919922,
      "activations/layer20_attention_weight_max": 20.788593292236328,
      "activations/layer20_attention_weight_min": -13.702597618103027,
      "activations/layer21_attention_weight_max": 34.2719612121582,
      "activations/layer21_attention_weight_min": -17.322532653808594,
      "activations/layer22_attention_weight_max": 33.55145263671875,
      "activations/layer22_attention_weight_min": -14.073752403259277,
      "activations/layer23_attention_weight_max": 20.103294372558594,
      "activations/layer23_attention_weight_min": -18.298322677612305,
      "activations/layer2_attention_weight_max": 25.829227447509766,
      "activations/layer2_attention_weight_min": -22.8294677734375,
      "activations/layer3_attention_weight_max": 51.999176025390625,
      "activations/layer3_attention_weight_min": -53.264434814453125,
      "activations/layer4_attention_weight_max": 81.49726867675781,
      "activations/layer4_attention_weight_min": -73.95833587646484,
      "activations/layer5_attention_weight_max": 138.9394989013672,
      "activations/layer5_attention_weight_min": -120.60926818847656,
      "activations/layer6_attention_weight_max": 66.82196807861328,
      "activations/layer6_attention_weight_min": -60.48518753051758,
      "activations/layer7_attention_weight_max": 67.16207122802734,
      "activations/layer7_attention_weight_min": -60.702388763427734,
      "activations/layer8_attention_weight_max": 57.44426727294922,
      "activations/layer8_attention_weight_min": -52.49778747558594,
      "activations/layer9_attention_weight_max": 78.00238800048828,
      "activations/layer9_attention_weight_min": -77.7868881225586,
      "epoch": 0.91,
      "learning_rate": 0.00014561742424242422,
      "loss": 3.1168,
      "step": 15600
    },
    {
      "activations/layer0_attention_weight_max": 13.417689323425293,
      "activations/layer0_attention_weight_min": -10.760546684265137,
      "activations/layer10_attention_weight_max": 69.58255767822266,
      "activations/layer10_attention_weight_min": -63.972591400146484,
      "activations/layer11_attention_weight_max": 48.6895751953125,
      "activations/layer11_attention_weight_min": -42.95815658569336,
      "activations/layer12_attention_weight_max": 28.739160537719727,
      "activations/layer12_attention_weight_min": -27.730012893676758,
      "activations/layer13_attention_weight_max": 55.00486373901367,
      "activations/layer13_attention_weight_min": -39.97621536254883,
      "activations/layer14_attention_weight_max": 48.5415153503418,
      "activations/layer14_attention_weight_min": -37.76557922363281,
      "activations/layer15_attention_weight_max": 53.581356048583984,
      "activations/layer15_attention_weight_min": -37.508480072021484,
      "activations/layer16_attention_weight_max": 49.18864822387695,
      "activations/layer16_attention_weight_min": -35.04190444946289,
      "activations/layer17_attention_weight_max": 56.280460357666016,
      "activations/layer17_attention_weight_min": -37.904014587402344,
      "activations/layer18_attention_weight_max": 69.1343002319336,
      "activations/layer18_attention_weight_min": -43.54051208496094,
      "activations/layer19_attention_weight_max": 20.23748779296875,
      "activations/layer19_attention_weight_min": -12.60226058959961,
      "activations/layer1_attention_weight_max": 15.814229011535645,
      "activations/layer1_attention_weight_min": -11.484814643859863,
      "activations/layer20_attention_weight_max": 18.086496353149414,
      "activations/layer20_attention_weight_min": -13.832449913024902,
      "activations/layer21_attention_weight_max": 33.1890869140625,
      "activations/layer21_attention_weight_min": -16.7497501373291,
      "activations/layer22_attention_weight_max": 30.069292068481445,
      "activations/layer22_attention_weight_min": -14.368404388427734,
      "activations/layer23_attention_weight_max": 20.232336044311523,
      "activations/layer23_attention_weight_min": -17.340158462524414,
      "activations/layer2_attention_weight_max": 26.049785614013672,
      "activations/layer2_attention_weight_min": -22.167156219482422,
      "activations/layer3_attention_weight_max": 53.21075439453125,
      "activations/layer3_attention_weight_min": -51.93534469604492,
      "activations/layer4_attention_weight_max": 83.32426452636719,
      "activations/layer4_attention_weight_min": -79.00971984863281,
      "activations/layer5_attention_weight_max": 141.13092041015625,
      "activations/layer5_attention_weight_min": -129.84727478027344,
      "activations/layer6_attention_weight_max": 64.9190902709961,
      "activations/layer6_attention_weight_min": -62.19881057739258,
      "activations/layer7_attention_weight_max": 68.8115234375,
      "activations/layer7_attention_weight_min": -58.622798919677734,
      "activations/layer8_attention_weight_max": 55.1266975402832,
      "activations/layer8_attention_weight_min": -51.853267669677734,
      "activations/layer9_attention_weight_max": 73.43730926513672,
      "activations/layer9_attention_weight_min": -70.84149169921875,
      "epoch": 0.91,
      "learning_rate": 0.00014559848484848483,
      "loss": 3.1362,
      "step": 15650
    },
    {
      "activations/layer0_attention_weight_max": 15.29134464263916,
      "activations/layer0_attention_weight_min": -10.8777437210083,
      "activations/layer10_attention_weight_max": 66.02857971191406,
      "activations/layer10_attention_weight_min": -57.542232513427734,
      "activations/layer11_attention_weight_max": 45.776771545410156,
      "activations/layer11_attention_weight_min": -36.44756317138672,
      "activations/layer12_attention_weight_max": 26.577594757080078,
      "activations/layer12_attention_weight_min": -23.693511962890625,
      "activations/layer13_attention_weight_max": 55.85759353637695,
      "activations/layer13_attention_weight_min": -42.42762756347656,
      "activations/layer14_attention_weight_max": 47.14228820800781,
      "activations/layer14_attention_weight_min": -32.622596740722656,
      "activations/layer15_attention_weight_max": 50.683963775634766,
      "activations/layer15_attention_weight_min": -37.00527572631836,
      "activations/layer16_attention_weight_max": 47.59088897705078,
      "activations/layer16_attention_weight_min": -34.28791809082031,
      "activations/layer17_attention_weight_max": 55.02614212036133,
      "activations/layer17_attention_weight_min": -35.74382400512695,
      "activations/layer18_attention_weight_max": 56.86820983886719,
      "activations/layer18_attention_weight_min": -41.2331428527832,
      "activations/layer19_attention_weight_max": 18.502527236938477,
      "activations/layer19_attention_weight_min": -12.3812255859375,
      "activations/layer1_attention_weight_max": 15.456274032592773,
      "activations/layer1_attention_weight_min": -10.99189281463623,
      "activations/layer20_attention_weight_max": 16.729021072387695,
      "activations/layer20_attention_weight_min": -13.456859588623047,
      "activations/layer21_attention_weight_max": 31.50425148010254,
      "activations/layer21_attention_weight_min": -17.336183547973633,
      "activations/layer22_attention_weight_max": 28.60966682434082,
      "activations/layer22_attention_weight_min": -16.73469352722168,
      "activations/layer23_attention_weight_max": 20.637008666992188,
      "activations/layer23_attention_weight_min": -17.537782669067383,
      "activations/layer2_attention_weight_max": 26.206281661987305,
      "activations/layer2_attention_weight_min": -21.756053924560547,
      "activations/layer3_attention_weight_max": 53.08258056640625,
      "activations/layer3_attention_weight_min": -49.29118347167969,
      "activations/layer4_attention_weight_max": 85.17609405517578,
      "activations/layer4_attention_weight_min": -73.62898254394531,
      "activations/layer5_attention_weight_max": 136.15957641601562,
      "activations/layer5_attention_weight_min": -117.38524627685547,
      "activations/layer6_attention_weight_max": 66.4400863647461,
      "activations/layer6_attention_weight_min": -57.86311721801758,
      "activations/layer7_attention_weight_max": 66.33757019042969,
      "activations/layer7_attention_weight_min": -60.09333801269531,
      "activations/layer8_attention_weight_max": 59.70423126220703,
      "activations/layer8_attention_weight_min": -51.4198112487793,
      "activations/layer9_attention_weight_max": 64.96993255615234,
      "activations/layer9_attention_weight_min": -62.40681076049805,
      "epoch": 0.91,
      "learning_rate": 0.00014557954545454545,
      "loss": 3.1313,
      "step": 15700
    },
    {
      "activations/layer0_attention_weight_max": 12.823466300964355,
      "activations/layer0_attention_weight_min": -10.381253242492676,
      "activations/layer10_attention_weight_max": 74.989990234375,
      "activations/layer10_attention_weight_min": -60.44664001464844,
      "activations/layer11_attention_weight_max": 57.94945526123047,
      "activations/layer11_attention_weight_min": -45.68413543701172,
      "activations/layer12_attention_weight_max": 30.051132202148438,
      "activations/layer12_attention_weight_min": -22.754562377929688,
      "activations/layer13_attention_weight_max": 86.62007904052734,
      "activations/layer13_attention_weight_min": -41.6021728515625,
      "activations/layer14_attention_weight_max": 96.93775177001953,
      "activations/layer14_attention_weight_min": -40.14325714111328,
      "activations/layer15_attention_weight_max": 53.89563751220703,
      "activations/layer15_attention_weight_min": -41.84331512451172,
      "activations/layer16_attention_weight_max": 57.810298919677734,
      "activations/layer16_attention_weight_min": -39.8496208190918,
      "activations/layer17_attention_weight_max": 59.50154113769531,
      "activations/layer17_attention_weight_min": -40.85781478881836,
      "activations/layer18_attention_weight_max": 58.745277404785156,
      "activations/layer18_attention_weight_min": -37.96284103393555,
      "activations/layer19_attention_weight_max": 16.892744064331055,
      "activations/layer19_attention_weight_min": -14.169212341308594,
      "activations/layer1_attention_weight_max": 16.865671157836914,
      "activations/layer1_attention_weight_min": -12.454132080078125,
      "activations/layer20_attention_weight_max": 17.908693313598633,
      "activations/layer20_attention_weight_min": -14.85872745513916,
      "activations/layer21_attention_weight_max": 28.451305389404297,
      "activations/layer21_attention_weight_min": -16.74528694152832,
      "activations/layer22_attention_weight_max": 26.624467849731445,
      "activations/layer22_attention_weight_min": -15.372323989868164,
      "activations/layer23_attention_weight_max": 18.215160369873047,
      "activations/layer23_attention_weight_min": -16.434667587280273,
      "activations/layer2_attention_weight_max": 26.823726654052734,
      "activations/layer2_attention_weight_min": -24.425939559936523,
      "activations/layer3_attention_weight_max": 53.3682975769043,
      "activations/layer3_attention_weight_min": -52.57884979248047,
      "activations/layer4_attention_weight_max": 81.02840423583984,
      "activations/layer4_attention_weight_min": -73.01903533935547,
      "activations/layer5_attention_weight_max": 139.5338134765625,
      "activations/layer5_attention_weight_min": -122.38885498046875,
      "activations/layer6_attention_weight_max": 67.54751586914062,
      "activations/layer6_attention_weight_min": -59.05982208251953,
      "activations/layer7_attention_weight_max": 72.32938385009766,
      "activations/layer7_attention_weight_min": -58.97773361206055,
      "activations/layer8_attention_weight_max": 59.58953857421875,
      "activations/layer8_attention_weight_min": -49.14086151123047,
      "activations/layer9_attention_weight_max": 77.6086654663086,
      "activations/layer9_attention_weight_min": -70.13594055175781,
      "epoch": 0.92,
      "learning_rate": 0.00014556060606060603,
      "loss": 3.1186,
      "step": 15750
    },
    {
      "activations/layer0_attention_weight_max": 13.16258716583252,
      "activations/layer0_attention_weight_min": -10.657134056091309,
      "activations/layer10_attention_weight_max": 65.21005249023438,
      "activations/layer10_attention_weight_min": -59.512474060058594,
      "activations/layer11_attention_weight_max": 44.12538146972656,
      "activations/layer11_attention_weight_min": -40.309776306152344,
      "activations/layer12_attention_weight_max": 31.858049392700195,
      "activations/layer12_attention_weight_min": -27.827302932739258,
      "activations/layer13_attention_weight_max": 55.00935745239258,
      "activations/layer13_attention_weight_min": -41.62977600097656,
      "activations/layer14_attention_weight_max": 45.57757568359375,
      "activations/layer14_attention_weight_min": -38.896400451660156,
      "activations/layer15_attention_weight_max": 52.4700927734375,
      "activations/layer15_attention_weight_min": -36.83540725708008,
      "activations/layer16_attention_weight_max": 45.79629898071289,
      "activations/layer16_attention_weight_min": -33.353187561035156,
      "activations/layer17_attention_weight_max": 53.95720291137695,
      "activations/layer17_attention_weight_min": -35.96720504760742,
      "activations/layer18_attention_weight_max": 55.503746032714844,
      "activations/layer18_attention_weight_min": -35.79616928100586,
      "activations/layer19_attention_weight_max": 20.18761444091797,
      "activations/layer19_attention_weight_min": -12.646505355834961,
      "activations/layer1_attention_weight_max": 15.087862968444824,
      "activations/layer1_attention_weight_min": -10.653578758239746,
      "activations/layer20_attention_weight_max": 18.917766571044922,
      "activations/layer20_attention_weight_min": -12.036160469055176,
      "activations/layer21_attention_weight_max": 35.08173370361328,
      "activations/layer21_attention_weight_min": -15.9468994140625,
      "activations/layer22_attention_weight_max": 32.60068130493164,
      "activations/layer22_attention_weight_min": -15.246753692626953,
      "activations/layer23_attention_weight_max": 21.91377067565918,
      "activations/layer23_attention_weight_min": -16.052852630615234,
      "activations/layer2_attention_weight_max": 26.461668014526367,
      "activations/layer2_attention_weight_min": -23.181495666503906,
      "activations/layer3_attention_weight_max": 51.2219123840332,
      "activations/layer3_attention_weight_min": -52.675804138183594,
      "activations/layer4_attention_weight_max": 79.8655014038086,
      "activations/layer4_attention_weight_min": -73.72840881347656,
      "activations/layer5_attention_weight_max": 132.23915100097656,
      "activations/layer5_attention_weight_min": -128.3444366455078,
      "activations/layer6_attention_weight_max": 61.56726837158203,
      "activations/layer6_attention_weight_min": -57.07002258300781,
      "activations/layer7_attention_weight_max": 66.87195587158203,
      "activations/layer7_attention_weight_min": -59.06184387207031,
      "activations/layer8_attention_weight_max": 59.96549606323242,
      "activations/layer8_attention_weight_min": -55.622928619384766,
      "activations/layer9_attention_weight_max": 67.97704315185547,
      "activations/layer9_attention_weight_min": -67.1795425415039,
      "epoch": 0.92,
      "learning_rate": 0.00014554166666666665,
      "loss": 3.1149,
      "step": 15800
    },
    {
      "activations/layer0_attention_weight_max": 13.250463485717773,
      "activations/layer0_attention_weight_min": -10.57806396484375,
      "activations/layer10_attention_weight_max": 67.47731018066406,
      "activations/layer10_attention_weight_min": -57.984710693359375,
      "activations/layer11_attention_weight_max": 48.489524841308594,
      "activations/layer11_attention_weight_min": -42.48120880126953,
      "activations/layer12_attention_weight_max": 30.225772857666016,
      "activations/layer12_attention_weight_min": -24.784873962402344,
      "activations/layer13_attention_weight_max": 52.43526077270508,
      "activations/layer13_attention_weight_min": -40.687408447265625,
      "activations/layer14_attention_weight_max": 44.76642608642578,
      "activations/layer14_attention_weight_min": -34.96847915649414,
      "activations/layer15_attention_weight_max": 50.871070861816406,
      "activations/layer15_attention_weight_min": -38.63899230957031,
      "activations/layer16_attention_weight_max": 41.28263473510742,
      "activations/layer16_attention_weight_min": -37.776493072509766,
      "activations/layer17_attention_weight_max": 56.07477569580078,
      "activations/layer17_attention_weight_min": -39.901729583740234,
      "activations/layer18_attention_weight_max": 70.15001678466797,
      "activations/layer18_attention_weight_min": -51.792232513427734,
      "activations/layer19_attention_weight_max": 17.310836791992188,
      "activations/layer19_attention_weight_min": -12.459210395812988,
      "activations/layer1_attention_weight_max": 15.50178050994873,
      "activations/layer1_attention_weight_min": -10.556020736694336,
      "activations/layer20_attention_weight_max": 20.768552780151367,
      "activations/layer20_attention_weight_min": -15.722502708435059,
      "activations/layer21_attention_weight_max": 32.8510856628418,
      "activations/layer21_attention_weight_min": -17.761154174804688,
      "activations/layer22_attention_weight_max": 29.555500030517578,
      "activations/layer22_attention_weight_min": -18.12354278564453,
      "activations/layer23_attention_weight_max": 20.42957305908203,
      "activations/layer23_attention_weight_min": -17.35272216796875,
      "activations/layer2_attention_weight_max": 27.69525909423828,
      "activations/layer2_attention_weight_min": -23.177955627441406,
      "activations/layer3_attention_weight_max": 53.26662826538086,
      "activations/layer3_attention_weight_min": -51.444271087646484,
      "activations/layer4_attention_weight_max": 85.48535919189453,
      "activations/layer4_attention_weight_min": -75.35271453857422,
      "activations/layer5_attention_weight_max": 146.53463745117188,
      "activations/layer5_attention_weight_min": -125.75779724121094,
      "activations/layer6_attention_weight_max": 67.92938232421875,
      "activations/layer6_attention_weight_min": -61.115501403808594,
      "activations/layer7_attention_weight_max": 66.3919677734375,
      "activations/layer7_attention_weight_min": -59.68785095214844,
      "activations/layer8_attention_weight_max": 61.39613723754883,
      "activations/layer8_attention_weight_min": -52.94662857055664,
      "activations/layer9_attention_weight_max": 76.58806610107422,
      "activations/layer9_attention_weight_min": -64.17029571533203,
      "epoch": 0.92,
      "learning_rate": 0.00014552272727272726,
      "loss": 3.1213,
      "step": 15850
    },
    {
      "activations/layer0_attention_weight_max": 14.449626922607422,
      "activations/layer0_attention_weight_min": -10.477818489074707,
      "activations/layer10_attention_weight_max": 68.03064727783203,
      "activations/layer10_attention_weight_min": -63.99213409423828,
      "activations/layer11_attention_weight_max": 49.48704147338867,
      "activations/layer11_attention_weight_min": -45.9228515625,
      "activations/layer12_attention_weight_max": 29.911649703979492,
      "activations/layer12_attention_weight_min": -26.75743865966797,
      "activations/layer13_attention_weight_max": 56.61933898925781,
      "activations/layer13_attention_weight_min": -49.95624923706055,
      "activations/layer14_attention_weight_max": 60.507164001464844,
      "activations/layer14_attention_weight_min": -44.01053237915039,
      "activations/layer15_attention_weight_max": 47.958885192871094,
      "activations/layer15_attention_weight_min": -35.912193298339844,
      "activations/layer16_attention_weight_max": 45.08699035644531,
      "activations/layer16_attention_weight_min": -35.48037338256836,
      "activations/layer17_attention_weight_max": 54.81157684326172,
      "activations/layer17_attention_weight_min": -36.04719924926758,
      "activations/layer18_attention_weight_max": 67.25103759765625,
      "activations/layer18_attention_weight_min": -46.16508102416992,
      "activations/layer19_attention_weight_max": 16.550453186035156,
      "activations/layer19_attention_weight_min": -12.403727531433105,
      "activations/layer1_attention_weight_max": 15.576516151428223,
      "activations/layer1_attention_weight_min": -10.965743064880371,
      "activations/layer20_attention_weight_max": 16.04792022705078,
      "activations/layer20_attention_weight_min": -14.184884071350098,
      "activations/layer21_attention_weight_max": 30.816448211669922,
      "activations/layer21_attention_weight_min": -15.638686180114746,
      "activations/layer22_attention_weight_max": 31.64978790283203,
      "activations/layer22_attention_weight_min": -15.220115661621094,
      "activations/layer23_attention_weight_max": 19.991357803344727,
      "activations/layer23_attention_weight_min": -17.667095184326172,
      "activations/layer2_attention_weight_max": 23.845258712768555,
      "activations/layer2_attention_weight_min": -22.485755920410156,
      "activations/layer3_attention_weight_max": 51.122440338134766,
      "activations/layer3_attention_weight_min": -51.78928756713867,
      "activations/layer4_attention_weight_max": 81.12448120117188,
      "activations/layer4_attention_weight_min": -73.3785171508789,
      "activations/layer5_attention_weight_max": 132.54376220703125,
      "activations/layer5_attention_weight_min": -121.36357116699219,
      "activations/layer6_attention_weight_max": 65.62459564208984,
      "activations/layer6_attention_weight_min": -60.82598876953125,
      "activations/layer7_attention_weight_max": 67.05854034423828,
      "activations/layer7_attention_weight_min": -57.21989440917969,
      "activations/layer8_attention_weight_max": 57.99409866333008,
      "activations/layer8_attention_weight_min": -50.30740737915039,
      "activations/layer9_attention_weight_max": 81.69865417480469,
      "activations/layer9_attention_weight_min": -72.37176513671875,
      "epoch": 0.92,
      "learning_rate": 0.00014550378787878785,
      "loss": 3.1105,
      "step": 15900
    },
    {
      "activations/layer0_attention_weight_max": 15.297858238220215,
      "activations/layer0_attention_weight_min": -10.898076057434082,
      "activations/layer10_attention_weight_max": 65.29620361328125,
      "activations/layer10_attention_weight_min": -59.27248001098633,
      "activations/layer11_attention_weight_max": 46.258724212646484,
      "activations/layer11_attention_weight_min": -37.867244720458984,
      "activations/layer12_attention_weight_max": 26.6657657623291,
      "activations/layer12_attention_weight_min": -25.26871681213379,
      "activations/layer13_attention_weight_max": 57.2667236328125,
      "activations/layer13_attention_weight_min": -45.52204895019531,
      "activations/layer14_attention_weight_max": 49.8934211730957,
      "activations/layer14_attention_weight_min": -39.006839752197266,
      "activations/layer15_attention_weight_max": 49.30643081665039,
      "activations/layer15_attention_weight_min": -37.50959014892578,
      "activations/layer16_attention_weight_max": 45.88726043701172,
      "activations/layer16_attention_weight_min": -34.571407318115234,
      "activations/layer17_attention_weight_max": 59.15256118774414,
      "activations/layer17_attention_weight_min": -42.804176330566406,
      "activations/layer18_attention_weight_max": 69.52528381347656,
      "activations/layer18_attention_weight_min": -40.38159942626953,
      "activations/layer19_attention_weight_max": 19.226945877075195,
      "activations/layer19_attention_weight_min": -12.18070125579834,
      "activations/layer1_attention_weight_max": 16.526811599731445,
      "activations/layer1_attention_weight_min": -10.472984313964844,
      "activations/layer20_attention_weight_max": 16.776248931884766,
      "activations/layer20_attention_weight_min": -13.402579307556152,
      "activations/layer21_attention_weight_max": 38.254791259765625,
      "activations/layer21_attention_weight_min": -17.95172119140625,
      "activations/layer22_attention_weight_max": 31.65973472595215,
      "activations/layer22_attention_weight_min": -15.087615966796875,
      "activations/layer23_attention_weight_max": 20.601619720458984,
      "activations/layer23_attention_weight_min": -16.18992042541504,
      "activations/layer2_attention_weight_max": 25.952993392944336,
      "activations/layer2_attention_weight_min": -21.93128776550293,
      "activations/layer3_attention_weight_max": 51.88904571533203,
      "activations/layer3_attention_weight_min": -48.482383728027344,
      "activations/layer4_attention_weight_max": 75.51168823242188,
      "activations/layer4_attention_weight_min": -71.1211166381836,
      "activations/layer5_attention_weight_max": 131.10012817382812,
      "activations/layer5_attention_weight_min": -118.13488006591797,
      "activations/layer6_attention_weight_max": 63.53556823730469,
      "activations/layer6_attention_weight_min": -58.21660232543945,
      "activations/layer7_attention_weight_max": 65.3231201171875,
      "activations/layer7_attention_weight_min": -60.18054962158203,
      "activations/layer8_attention_weight_max": 59.03891372680664,
      "activations/layer8_attention_weight_min": -56.344547271728516,
      "activations/layer9_attention_weight_max": 72.31010437011719,
      "activations/layer9_attention_weight_min": -70.56748962402344,
      "epoch": 0.93,
      "learning_rate": 0.00014548484848484847,
      "loss": 3.106,
      "step": 15950
    },
    {
      "activations/layer0_attention_weight_max": 14.087230682373047,
      "activations/layer0_attention_weight_min": -10.317804336547852,
      "activations/layer10_attention_weight_max": 65.99883270263672,
      "activations/layer10_attention_weight_min": -60.52684020996094,
      "activations/layer11_attention_weight_max": 53.58210372924805,
      "activations/layer11_attention_weight_min": -48.127967834472656,
      "activations/layer12_attention_weight_max": 27.486434936523438,
      "activations/layer12_attention_weight_min": -22.679758071899414,
      "activations/layer13_attention_weight_max": 56.37242126464844,
      "activations/layer13_attention_weight_min": -46.62409973144531,
      "activations/layer14_attention_weight_max": 49.280513763427734,
      "activations/layer14_attention_weight_min": -33.181793212890625,
      "activations/layer15_attention_weight_max": 54.733394622802734,
      "activations/layer15_attention_weight_min": -41.453460693359375,
      "activations/layer16_attention_weight_max": 51.016780853271484,
      "activations/layer16_attention_weight_min": -38.623653411865234,
      "activations/layer17_attention_weight_max": 57.39577102661133,
      "activations/layer17_attention_weight_min": -40.173561096191406,
      "activations/layer18_attention_weight_max": 56.80200958251953,
      "activations/layer18_attention_weight_min": -38.23323440551758,
      "activations/layer19_attention_weight_max": 20.633451461791992,
      "activations/layer19_attention_weight_min": -12.599529266357422,
      "activations/layer1_attention_weight_max": 14.688678741455078,
      "activations/layer1_attention_weight_min": -10.33585262298584,
      "activations/layer20_attention_weight_max": 21.76367950439453,
      "activations/layer20_attention_weight_min": -14.300741195678711,
      "activations/layer21_attention_weight_max": 33.55098342895508,
      "activations/layer21_attention_weight_min": -16.83445167541504,
      "activations/layer22_attention_weight_max": 31.728134155273438,
      "activations/layer22_attention_weight_min": -17.590295791625977,
      "activations/layer23_attention_weight_max": 21.139930725097656,
      "activations/layer23_attention_weight_min": -16.518077850341797,
      "activations/layer2_attention_weight_max": 25.797903060913086,
      "activations/layer2_attention_weight_min": -22.375782012939453,
      "activations/layer3_attention_weight_max": 54.623046875,
      "activations/layer3_attention_weight_min": -53.40694808959961,
      "activations/layer4_attention_weight_max": 81.02462768554688,
      "activations/layer4_attention_weight_min": -76.08370208740234,
      "activations/layer5_attention_weight_max": 133.295654296875,
      "activations/layer5_attention_weight_min": -127.3802490234375,
      "activations/layer6_attention_weight_max": 65.8348388671875,
      "activations/layer6_attention_weight_min": -59.630653381347656,
      "activations/layer7_attention_weight_max": 66.26006317138672,
      "activations/layer7_attention_weight_min": -57.942928314208984,
      "activations/layer8_attention_weight_max": 55.29515838623047,
      "activations/layer8_attention_weight_min": -48.8251838684082,
      "activations/layer9_attention_weight_max": 78.11605072021484,
      "activations/layer9_attention_weight_min": -71.59223175048828,
      "epoch": 0.93,
      "learning_rate": 0.00014546590909090908,
      "loss": 3.1058,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_loss": 3.060546875,
      "eval_runtime": 8.2646,
      "eval_samples_per_second": 519.565,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_openwebtext_loss": 3.060546875,
      "eval_openwebtext_ppl": 21.33922385967176,
      "eval_openwebtext_runtime": 8.2646,
      "eval_openwebtext_samples_per_second": 519.565,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_wikitext_loss": 3.3515625,
      "eval_wikitext_ppl": 28.547303976590324,
      "eval_wikitext_runtime": 1.7119,
      "eval_wikitext_samples_per_second": 266.372,
      "step": 16000
    },
    {
      "epoch": 0.93,
      "eval_lambada_loss": 3.4609375,
      "eval_lambada_ppl": 31.846818916590372,
      "eval_lambada_runtime": 9.2576,
      "eval_lambada_samples_per_second": 525.948,
      "step": 16000
    },
    {
      "activations/layer0_attention_weight_max": 13.562610626220703,
      "activations/layer0_attention_weight_min": -10.533447265625,
      "activations/layer10_attention_weight_max": 64.9554443359375,
      "activations/layer10_attention_weight_min": -59.092620849609375,
      "activations/layer11_attention_weight_max": 44.86011505126953,
      "activations/layer11_attention_weight_min": -39.84165573120117,
      "activations/layer12_attention_weight_max": 29.98019790649414,
      "activations/layer12_attention_weight_min": -28.45397186279297,
      "activations/layer13_attention_weight_max": 62.27461624145508,
      "activations/layer13_attention_weight_min": -45.05794143676758,
      "activations/layer14_attention_weight_max": 52.863197326660156,
      "activations/layer14_attention_weight_min": -38.24055862426758,
      "activations/layer15_attention_weight_max": 62.724876403808594,
      "activations/layer15_attention_weight_min": -37.7964973449707,
      "activations/layer16_attention_weight_max": 55.84593963623047,
      "activations/layer16_attention_weight_min": -37.57590103149414,
      "activations/layer17_attention_weight_max": 61.49617385864258,
      "activations/layer17_attention_weight_min": -38.65884017944336,
      "activations/layer18_attention_weight_max": 58.171836853027344,
      "activations/layer18_attention_weight_min": -46.780513763427734,
      "activations/layer19_attention_weight_max": 21.626073837280273,
      "activations/layer19_attention_weight_min": -11.345006942749023,
      "activations/layer1_attention_weight_max": 16.275691986083984,
      "activations/layer1_attention_weight_min": -10.744274139404297,
      "activations/layer20_attention_weight_max": 21.649057388305664,
      "activations/layer20_attention_weight_min": -16.10265350341797,
      "activations/layer21_attention_weight_max": 39.31499481201172,
      "activations/layer21_attention_weight_min": -17.31260108947754,
      "activations/layer22_attention_weight_max": 32.28033447265625,
      "activations/layer22_attention_weight_min": -15.21250057220459,
      "activations/layer23_attention_weight_max": 24.554485321044922,
      "activations/layer23_attention_weight_min": -16.122289657592773,
      "activations/layer2_attention_weight_max": 27.240554809570312,
      "activations/layer2_attention_weight_min": -22.610260009765625,
      "activations/layer3_attention_weight_max": 52.88040542602539,
      "activations/layer3_attention_weight_min": -55.59127426147461,
      "activations/layer4_attention_weight_max": 79.9603500366211,
      "activations/layer4_attention_weight_min": -74.17333984375,
      "activations/layer5_attention_weight_max": 134.8414306640625,
      "activations/layer5_attention_weight_min": -117.82627868652344,
      "activations/layer6_attention_weight_max": 68.33113098144531,
      "activations/layer6_attention_weight_min": -59.37863540649414,
      "activations/layer7_attention_weight_max": 67.65812683105469,
      "activations/layer7_attention_weight_min": -61.26182556152344,
      "activations/layer8_attention_weight_max": 62.93195343017578,
      "activations/layer8_attention_weight_min": -54.22420120239258,
      "activations/layer9_attention_weight_max": 87.6168441772461,
      "activations/layer9_attention_weight_min": -72.56412506103516,
      "epoch": 0.93,
      "learning_rate": 0.0001454469696969697,
      "loss": 3.1196,
      "step": 16050
    },
    {
      "activations/layer0_attention_weight_max": 16.378772735595703,
      "activations/layer0_attention_weight_min": -11.155153274536133,
      "activations/layer10_attention_weight_max": 73.78132629394531,
      "activations/layer10_attention_weight_min": -60.68052673339844,
      "activations/layer11_attention_weight_max": 48.42994689941406,
      "activations/layer11_attention_weight_min": -39.73919677734375,
      "activations/layer12_attention_weight_max": 25.803037643432617,
      "activations/layer12_attention_weight_min": -28.301816940307617,
      "activations/layer13_attention_weight_max": 64.52068328857422,
      "activations/layer13_attention_weight_min": -48.243934631347656,
      "activations/layer14_attention_weight_max": 50.291114807128906,
      "activations/layer14_attention_weight_min": -38.66249084472656,
      "activations/layer15_attention_weight_max": 51.33296585083008,
      "activations/layer15_attention_weight_min": -40.48398208618164,
      "activations/layer16_attention_weight_max": 48.78512954711914,
      "activations/layer16_attention_weight_min": -35.452674865722656,
      "activations/layer17_attention_weight_max": 57.57280731201172,
      "activations/layer17_attention_weight_min": -39.0787239074707,
      "activations/layer18_attention_weight_max": 68.15939331054688,
      "activations/layer18_attention_weight_min": -39.548763275146484,
      "activations/layer19_attention_weight_max": 20.666658401489258,
      "activations/layer19_attention_weight_min": -12.076730728149414,
      "activations/layer1_attention_weight_max": 15.593740463256836,
      "activations/layer1_attention_weight_min": -10.99354076385498,
      "activations/layer20_attention_weight_max": 18.53675651550293,
      "activations/layer20_attention_weight_min": -13.92437744140625,
      "activations/layer21_attention_weight_max": 31.008934020996094,
      "activations/layer21_attention_weight_min": -17.211009979248047,
      "activations/layer22_attention_weight_max": 28.484983444213867,
      "activations/layer22_attention_weight_min": -14.267967224121094,
      "activations/layer23_attention_weight_max": 19.766952514648438,
      "activations/layer23_attention_weight_min": -17.366395950317383,
      "activations/layer2_attention_weight_max": 23.84065055847168,
      "activations/layer2_attention_weight_min": -22.48836326599121,
      "activations/layer3_attention_weight_max": 50.83976364135742,
      "activations/layer3_attention_weight_min": -51.33665084838867,
      "activations/layer4_attention_weight_max": 80.36924743652344,
      "activations/layer4_attention_weight_min": -74.24574279785156,
      "activations/layer5_attention_weight_max": 139.53125,
      "activations/layer5_attention_weight_min": -123.40434265136719,
      "activations/layer6_attention_weight_max": 66.1984634399414,
      "activations/layer6_attention_weight_min": -58.03853225708008,
      "activations/layer7_attention_weight_max": 69.79119110107422,
      "activations/layer7_attention_weight_min": -58.90211868286133,
      "activations/layer8_attention_weight_max": 59.15300750732422,
      "activations/layer8_attention_weight_min": -48.9521484375,
      "activations/layer9_attention_weight_max": 95.1891098022461,
      "activations/layer9_attention_weight_min": -72.22053527832031,
      "epoch": 0.94,
      "learning_rate": 0.00014542803030303029,
      "loss": 3.1168,
      "step": 16100
    },
    {
      "activations/layer0_attention_weight_max": 14.609562873840332,
      "activations/layer0_attention_weight_min": -10.656243324279785,
      "activations/layer10_attention_weight_max": 69.22394561767578,
      "activations/layer10_attention_weight_min": -59.95921325683594,
      "activations/layer11_attention_weight_max": 51.04745101928711,
      "activations/layer11_attention_weight_min": -41.93775177001953,
      "activations/layer12_attention_weight_max": 26.440385818481445,
      "activations/layer12_attention_weight_min": -22.492950439453125,
      "activations/layer13_attention_weight_max": 75.93917846679688,
      "activations/layer13_attention_weight_min": -54.73644256591797,
      "activations/layer14_attention_weight_max": 55.42533493041992,
      "activations/layer14_attention_weight_min": -37.89154815673828,
      "activations/layer15_attention_weight_max": 56.86233901977539,
      "activations/layer15_attention_weight_min": -41.27235794067383,
      "activations/layer16_attention_weight_max": 50.86373519897461,
      "activations/layer16_attention_weight_min": -37.59402847290039,
      "activations/layer17_attention_weight_max": 57.77621841430664,
      "activations/layer17_attention_weight_min": -37.66596984863281,
      "activations/layer18_attention_weight_max": 51.79662322998047,
      "activations/layer18_attention_weight_min": -34.20661163330078,
      "activations/layer19_attention_weight_max": 17.57084083557129,
      "activations/layer19_attention_weight_min": -12.480752944946289,
      "activations/layer1_attention_weight_max": 16.88692855834961,
      "activations/layer1_attention_weight_min": -10.607125282287598,
      "activations/layer20_attention_weight_max": 18.28800392150879,
      "activations/layer20_attention_weight_min": -13.22443962097168,
      "activations/layer21_attention_weight_max": 33.80775451660156,
      "activations/layer21_attention_weight_min": -17.374116897583008,
      "activations/layer22_attention_weight_max": 31.102964401245117,
      "activations/layer22_attention_weight_min": -15.610029220581055,
      "activations/layer23_attention_weight_max": 21.083335876464844,
      "activations/layer23_attention_weight_min": -16.357879638671875,
      "activations/layer2_attention_weight_max": 25.139944076538086,
      "activations/layer2_attention_weight_min": -22.163986206054688,
      "activations/layer3_attention_weight_max": 53.54728698730469,
      "activations/layer3_attention_weight_min": -51.71908187866211,
      "activations/layer4_attention_weight_max": 82.41586303710938,
      "activations/layer4_attention_weight_min": -74.42985534667969,
      "activations/layer5_attention_weight_max": 130.00692749023438,
      "activations/layer5_attention_weight_min": -122.31448364257812,
      "activations/layer6_attention_weight_max": 64.26119232177734,
      "activations/layer6_attention_weight_min": -59.944915771484375,
      "activations/layer7_attention_weight_max": 66.46388244628906,
      "activations/layer7_attention_weight_min": -59.09593963623047,
      "activations/layer8_attention_weight_max": 55.70177459716797,
      "activations/layer8_attention_weight_min": -49.84331130981445,
      "activations/layer9_attention_weight_max": 68.02326202392578,
      "activations/layer9_attention_weight_min": -64.04364776611328,
      "epoch": 0.94,
      "learning_rate": 0.0001454090909090909,
      "loss": 3.1301,
      "step": 16150
    },
    {
      "activations/layer0_attention_weight_max": 15.557465553283691,
      "activations/layer0_attention_weight_min": -11.203821182250977,
      "activations/layer10_attention_weight_max": 67.92024993896484,
      "activations/layer10_attention_weight_min": -58.986873626708984,
      "activations/layer11_attention_weight_max": 43.44489669799805,
      "activations/layer11_attention_weight_min": -39.73216247558594,
      "activations/layer12_attention_weight_max": 25.589950561523438,
      "activations/layer12_attention_weight_min": -22.660734176635742,
      "activations/layer13_attention_weight_max": 73.16572570800781,
      "activations/layer13_attention_weight_min": -61.37508010864258,
      "activations/layer14_attention_weight_max": 78.16073608398438,
      "activations/layer14_attention_weight_min": -59.531150817871094,
      "activations/layer15_attention_weight_max": 55.65174865722656,
      "activations/layer15_attention_weight_min": -44.07261657714844,
      "activations/layer16_attention_weight_max": 56.93294906616211,
      "activations/layer16_attention_weight_min": -38.386329650878906,
      "activations/layer17_attention_weight_max": 58.70936965942383,
      "activations/layer17_attention_weight_min": -37.27887725830078,
      "activations/layer18_attention_weight_max": 83.35138702392578,
      "activations/layer18_attention_weight_min": -49.27952575683594,
      "activations/layer19_attention_weight_max": 20.76602554321289,
      "activations/layer19_attention_weight_min": -13.84239673614502,
      "activations/layer1_attention_weight_max": 15.289803504943848,
      "activations/layer1_attention_weight_min": -10.899827003479004,
      "activations/layer20_attention_weight_max": 19.58827018737793,
      "activations/layer20_attention_weight_min": -14.956713676452637,
      "activations/layer21_attention_weight_max": 36.342811584472656,
      "activations/layer21_attention_weight_min": -21.988365173339844,
      "activations/layer22_attention_weight_max": 33.85829162597656,
      "activations/layer22_attention_weight_min": -15.7813720703125,
      "activations/layer23_attention_weight_max": 22.66830825805664,
      "activations/layer23_attention_weight_min": -16.898895263671875,
      "activations/layer2_attention_weight_max": 25.795331954956055,
      "activations/layer2_attention_weight_min": -22.12819480895996,
      "activations/layer3_attention_weight_max": 52.57677459716797,
      "activations/layer3_attention_weight_min": -52.12987518310547,
      "activations/layer4_attention_weight_max": 80.44384002685547,
      "activations/layer4_attention_weight_min": -75.56858825683594,
      "activations/layer5_attention_weight_max": 131.44802856445312,
      "activations/layer5_attention_weight_min": -122.28157043457031,
      "activations/layer6_attention_weight_max": 64.241455078125,
      "activations/layer6_attention_weight_min": -59.07435607910156,
      "activations/layer7_attention_weight_max": 66.11981201171875,
      "activations/layer7_attention_weight_min": -60.85253143310547,
      "activations/layer8_attention_weight_max": 56.04008483886719,
      "activations/layer8_attention_weight_min": -49.144535064697266,
      "activations/layer9_attention_weight_max": 87.55919647216797,
      "activations/layer9_attention_weight_min": -69.52947235107422,
      "epoch": 0.94,
      "learning_rate": 0.00014539015151515152,
      "loss": 3.1077,
      "step": 16200
    },
    {
      "activations/layer0_attention_weight_max": 14.713047981262207,
      "activations/layer0_attention_weight_min": -11.176851272583008,
      "activations/layer10_attention_weight_max": 64.05016326904297,
      "activations/layer10_attention_weight_min": -57.12175750732422,
      "activations/layer11_attention_weight_max": 40.33184814453125,
      "activations/layer11_attention_weight_min": -38.748634338378906,
      "activations/layer12_attention_weight_max": 27.784486770629883,
      "activations/layer12_attention_weight_min": -27.36054801940918,
      "activations/layer13_attention_weight_max": 61.581119537353516,
      "activations/layer13_attention_weight_min": -45.299259185791016,
      "activations/layer14_attention_weight_max": 45.36960983276367,
      "activations/layer14_attention_weight_min": -35.13311767578125,
      "activations/layer15_attention_weight_max": 54.20096969604492,
      "activations/layer15_attention_weight_min": -41.97186279296875,
      "activations/layer16_attention_weight_max": 45.63224411010742,
      "activations/layer16_attention_weight_min": -37.45234680175781,
      "activations/layer17_attention_weight_max": 51.30727005004883,
      "activations/layer17_attention_weight_min": -37.440643310546875,
      "activations/layer18_attention_weight_max": 52.57371520996094,
      "activations/layer18_attention_weight_min": -37.54051971435547,
      "activations/layer19_attention_weight_max": 22.23363494873047,
      "activations/layer19_attention_weight_min": -13.613716125488281,
      "activations/layer1_attention_weight_max": 15.12714958190918,
      "activations/layer1_attention_weight_min": -11.106034278869629,
      "activations/layer20_attention_weight_max": 20.4672794342041,
      "activations/layer20_attention_weight_min": -13.485250473022461,
      "activations/layer21_attention_weight_max": 33.70703887939453,
      "activations/layer21_attention_weight_min": -17.174610137939453,
      "activations/layer22_attention_weight_max": 31.144392013549805,
      "activations/layer22_attention_weight_min": -16.56606101989746,
      "activations/layer23_attention_weight_max": 22.635704040527344,
      "activations/layer23_attention_weight_min": -16.605361938476562,
      "activations/layer2_attention_weight_max": 26.054357528686523,
      "activations/layer2_attention_weight_min": -22.634511947631836,
      "activations/layer3_attention_weight_max": 51.74116897583008,
      "activations/layer3_attention_weight_min": -52.09626770019531,
      "activations/layer4_attention_weight_max": 79.2427978515625,
      "activations/layer4_attention_weight_min": -75.72061157226562,
      "activations/layer5_attention_weight_max": 128.50503540039062,
      "activations/layer5_attention_weight_min": -120.80311584472656,
      "activations/layer6_attention_weight_max": 64.73023986816406,
      "activations/layer6_attention_weight_min": -59.16387939453125,
      "activations/layer7_attention_weight_max": 59.11698532104492,
      "activations/layer7_attention_weight_min": -56.20944595336914,
      "activations/layer8_attention_weight_max": 52.107261657714844,
      "activations/layer8_attention_weight_min": -51.37811279296875,
      "activations/layer9_attention_weight_max": 67.79957580566406,
      "activations/layer9_attention_weight_min": -67.94169616699219,
      "epoch": 0.94,
      "learning_rate": 0.0001453712121212121,
      "loss": 3.1111,
      "step": 16250
    },
    {
      "activations/layer0_attention_weight_max": 14.829092979431152,
      "activations/layer0_attention_weight_min": -10.947027206420898,
      "activations/layer10_attention_weight_max": 67.50537109375,
      "activations/layer10_attention_weight_min": -65.3252182006836,
      "activations/layer11_attention_weight_max": 43.34553527832031,
      "activations/layer11_attention_weight_min": -38.671173095703125,
      "activations/layer12_attention_weight_max": 27.129716873168945,
      "activations/layer12_attention_weight_min": -25.964216232299805,
      "activations/layer13_attention_weight_max": 54.85466384887695,
      "activations/layer13_attention_weight_min": -42.080772399902344,
      "activations/layer14_attention_weight_max": 54.02492904663086,
      "activations/layer14_attention_weight_min": -43.237335205078125,
      "activations/layer15_attention_weight_max": 53.03377151489258,
      "activations/layer15_attention_weight_min": -38.23371124267578,
      "activations/layer16_attention_weight_max": 51.9666862487793,
      "activations/layer16_attention_weight_min": -34.71207046508789,
      "activations/layer17_attention_weight_max": 62.17994689941406,
      "activations/layer17_attention_weight_min": -41.654964447021484,
      "activations/layer18_attention_weight_max": 79.93284606933594,
      "activations/layer18_attention_weight_min": -51.927825927734375,
      "activations/layer19_attention_weight_max": 23.056991577148438,
      "activations/layer19_attention_weight_min": -11.940458297729492,
      "activations/layer1_attention_weight_max": 15.817254066467285,
      "activations/layer1_attention_weight_min": -10.902670860290527,
      "activations/layer20_attention_weight_max": 18.737777709960938,
      "activations/layer20_attention_weight_min": -13.034529685974121,
      "activations/layer21_attention_weight_max": 37.68205642700195,
      "activations/layer21_attention_weight_min": -17.147371292114258,
      "activations/layer22_attention_weight_max": 32.38269805908203,
      "activations/layer22_attention_weight_min": -16.027450561523438,
      "activations/layer23_attention_weight_max": 24.157196044921875,
      "activations/layer23_attention_weight_min": -17.04810333251953,
      "activations/layer2_attention_weight_max": 24.746793746948242,
      "activations/layer2_attention_weight_min": -22.99002456665039,
      "activations/layer3_attention_weight_max": 52.259395599365234,
      "activations/layer3_attention_weight_min": -52.996456146240234,
      "activations/layer4_attention_weight_max": 78.46819305419922,
      "activations/layer4_attention_weight_min": -73.62895965576172,
      "activations/layer5_attention_weight_max": 136.04971313476562,
      "activations/layer5_attention_weight_min": -116.39924621582031,
      "activations/layer6_attention_weight_max": 63.167686462402344,
      "activations/layer6_attention_weight_min": -58.24481964111328,
      "activations/layer7_attention_weight_max": 62.04879379272461,
      "activations/layer7_attention_weight_min": -57.14254379272461,
      "activations/layer8_attention_weight_max": 53.909366607666016,
      "activations/layer8_attention_weight_min": -51.3656005859375,
      "activations/layer9_attention_weight_max": 69.18321990966797,
      "activations/layer9_attention_weight_min": -74.29118347167969,
      "epoch": 0.95,
      "learning_rate": 0.00014535227272727272,
      "loss": 3.1088,
      "step": 16300
    },
    {
      "activations/layer0_attention_weight_max": 16.294715881347656,
      "activations/layer0_attention_weight_min": -11.142524719238281,
      "activations/layer10_attention_weight_max": 74.40374755859375,
      "activations/layer10_attention_weight_min": -60.01530456542969,
      "activations/layer11_attention_weight_max": 46.896759033203125,
      "activations/layer11_attention_weight_min": -37.17451477050781,
      "activations/layer12_attention_weight_max": 30.390491485595703,
      "activations/layer12_attention_weight_min": -25.21854019165039,
      "activations/layer13_attention_weight_max": 72.63272094726562,
      "activations/layer13_attention_weight_min": -54.469600677490234,
      "activations/layer14_attention_weight_max": 62.76595687866211,
      "activations/layer14_attention_weight_min": -46.897151947021484,
      "activations/layer15_attention_weight_max": 56.1577033996582,
      "activations/layer15_attention_weight_min": -39.05836486816406,
      "activations/layer16_attention_weight_max": 54.12565612792969,
      "activations/layer16_attention_weight_min": -37.34817123413086,
      "activations/layer17_attention_weight_max": 64.74068450927734,
      "activations/layer17_attention_weight_min": -41.54552459716797,
      "activations/layer18_attention_weight_max": 73.822998046875,
      "activations/layer18_attention_weight_min": -41.90966033935547,
      "activations/layer19_attention_weight_max": 19.50884246826172,
      "activations/layer19_attention_weight_min": -12.61384391784668,
      "activations/layer1_attention_weight_max": 15.296693801879883,
      "activations/layer1_attention_weight_min": -11.642065048217773,
      "activations/layer20_attention_weight_max": 19.213136672973633,
      "activations/layer20_attention_weight_min": -13.3947172164917,
      "activations/layer21_attention_weight_max": 42.451438903808594,
      "activations/layer21_attention_weight_min": -20.177698135375977,
      "activations/layer22_attention_weight_max": 36.951377868652344,
      "activations/layer22_attention_weight_min": -17.79046058654785,
      "activations/layer23_attention_weight_max": 24.672588348388672,
      "activations/layer23_attention_weight_min": -17.35784339904785,
      "activations/layer2_attention_weight_max": 25.470937728881836,
      "activations/layer2_attention_weight_min": -22.675373077392578,
      "activations/layer3_attention_weight_max": 53.45488739013672,
      "activations/layer3_attention_weight_min": -53.42137908935547,
      "activations/layer4_attention_weight_max": 85.94298553466797,
      "activations/layer4_attention_weight_min": -73.64341735839844,
      "activations/layer5_attention_weight_max": 141.97906494140625,
      "activations/layer5_attention_weight_min": -124.99058532714844,
      "activations/layer6_attention_weight_max": 72.29690551757812,
      "activations/layer6_attention_weight_min": -62.35430145263672,
      "activations/layer7_attention_weight_max": 69.62186431884766,
      "activations/layer7_attention_weight_min": -61.47774124145508,
      "activations/layer8_attention_weight_max": 64.30351257324219,
      "activations/layer8_attention_weight_min": -50.79248809814453,
      "activations/layer9_attention_weight_max": 75.09423065185547,
      "activations/layer9_attention_weight_min": -68.11968231201172,
      "epoch": 0.95,
      "learning_rate": 0.00014533333333333333,
      "loss": 3.1032,
      "step": 16350
    },
    {
      "activations/layer0_attention_weight_max": 13.70602035522461,
      "activations/layer0_attention_weight_min": -11.37747859954834,
      "activations/layer10_attention_weight_max": 67.27571105957031,
      "activations/layer10_attention_weight_min": -55.990291595458984,
      "activations/layer11_attention_weight_max": 46.859657287597656,
      "activations/layer11_attention_weight_min": -41.076087951660156,
      "activations/layer12_attention_weight_max": 26.272058486938477,
      "activations/layer12_attention_weight_min": -22.90381622314453,
      "activations/layer13_attention_weight_max": 63.040130615234375,
      "activations/layer13_attention_weight_min": -49.16436004638672,
      "activations/layer14_attention_weight_max": 52.886871337890625,
      "activations/layer14_attention_weight_min": -40.060760498046875,
      "activations/layer15_attention_weight_max": 53.03258514404297,
      "activations/layer15_attention_weight_min": -41.462135314941406,
      "activations/layer16_attention_weight_max": 50.95285415649414,
      "activations/layer16_attention_weight_min": -38.86936569213867,
      "activations/layer17_attention_weight_max": 65.63933563232422,
      "activations/layer17_attention_weight_min": -40.33336639404297,
      "activations/layer18_attention_weight_max": 66.61962127685547,
      "activations/layer18_attention_weight_min": -38.31318283081055,
      "activations/layer19_attention_weight_max": 21.867088317871094,
      "activations/layer19_attention_weight_min": -13.44977855682373,
      "activations/layer1_attention_weight_max": 14.25724983215332,
      "activations/layer1_attention_weight_min": -10.443378448486328,
      "activations/layer20_attention_weight_max": 22.985429763793945,
      "activations/layer20_attention_weight_min": -14.508627891540527,
      "activations/layer21_attention_weight_max": 44.29729080200195,
      "activations/layer21_attention_weight_min": -18.60520362854004,
      "activations/layer22_attention_weight_max": 34.89203643798828,
      "activations/layer22_attention_weight_min": -16.58441162109375,
      "activations/layer23_attention_weight_max": 27.003002166748047,
      "activations/layer23_attention_weight_min": -16.588565826416016,
      "activations/layer2_attention_weight_max": 26.79635238647461,
      "activations/layer2_attention_weight_min": -23.282529830932617,
      "activations/layer3_attention_weight_max": 53.7986946105957,
      "activations/layer3_attention_weight_min": -51.73404312133789,
      "activations/layer4_attention_weight_max": 80.09357452392578,
      "activations/layer4_attention_weight_min": -75.0592041015625,
      "activations/layer5_attention_weight_max": 135.6058349609375,
      "activations/layer5_attention_weight_min": -125.63905334472656,
      "activations/layer6_attention_weight_max": 64.91114807128906,
      "activations/layer6_attention_weight_min": -61.99284362792969,
      "activations/layer7_attention_weight_max": 66.72676849365234,
      "activations/layer7_attention_weight_min": -58.68207550048828,
      "activations/layer8_attention_weight_max": 57.00461959838867,
      "activations/layer8_attention_weight_min": -51.87160110473633,
      "activations/layer9_attention_weight_max": 74.62460327148438,
      "activations/layer9_attention_weight_min": -63.44442367553711,
      "epoch": 0.95,
      "learning_rate": 0.00014531439393939392,
      "loss": 3.1002,
      "step": 16400
    },
    {
      "activations/layer0_attention_weight_max": 13.684195518493652,
      "activations/layer0_attention_weight_min": -11.61369800567627,
      "activations/layer10_attention_weight_max": 68.73834228515625,
      "activations/layer10_attention_weight_min": -62.82672882080078,
      "activations/layer11_attention_weight_max": 43.310787200927734,
      "activations/layer11_attention_weight_min": -37.132911682128906,
      "activations/layer12_attention_weight_max": 27.651826858520508,
      "activations/layer12_attention_weight_min": -23.11656379699707,
      "activations/layer13_attention_weight_max": 62.17988204956055,
      "activations/layer13_attention_weight_min": -50.51487350463867,
      "activations/layer14_attention_weight_max": 52.20279312133789,
      "activations/layer14_attention_weight_min": -34.385467529296875,
      "activations/layer15_attention_weight_max": 55.97701644897461,
      "activations/layer15_attention_weight_min": -39.9046516418457,
      "activations/layer16_attention_weight_max": 51.92525100708008,
      "activations/layer16_attention_weight_min": -38.118255615234375,
      "activations/layer17_attention_weight_max": 61.76419448852539,
      "activations/layer17_attention_weight_min": -38.336181640625,
      "activations/layer18_attention_weight_max": 61.437801361083984,
      "activations/layer18_attention_weight_min": -36.46467971801758,
      "activations/layer19_attention_weight_max": 19.992895126342773,
      "activations/layer19_attention_weight_min": -12.4073486328125,
      "activations/layer1_attention_weight_max": 14.470906257629395,
      "activations/layer1_attention_weight_min": -11.602313995361328,
      "activations/layer20_attention_weight_max": 20.83785629272461,
      "activations/layer20_attention_weight_min": -15.573287963867188,
      "activations/layer21_attention_weight_max": 36.94603729248047,
      "activations/layer21_attention_weight_min": -18.56032943725586,
      "activations/layer22_attention_weight_max": 37.359046936035156,
      "activations/layer22_attention_weight_min": -18.806406021118164,
      "activations/layer23_attention_weight_max": 24.47743797302246,
      "activations/layer23_attention_weight_min": -16.09343910217285,
      "activations/layer2_attention_weight_max": 25.406112670898438,
      "activations/layer2_attention_weight_min": -22.86842918395996,
      "activations/layer3_attention_weight_max": 49.66741180419922,
      "activations/layer3_attention_weight_min": -50.48504638671875,
      "activations/layer4_attention_weight_max": 76.8812484741211,
      "activations/layer4_attention_weight_min": -71.73467254638672,
      "activations/layer5_attention_weight_max": 131.35963439941406,
      "activations/layer5_attention_weight_min": -122.06365203857422,
      "activations/layer6_attention_weight_max": 69.09442138671875,
      "activations/layer6_attention_weight_min": -58.228309631347656,
      "activations/layer7_attention_weight_max": 65.9601058959961,
      "activations/layer7_attention_weight_min": -58.189910888671875,
      "activations/layer8_attention_weight_max": 58.43257522583008,
      "activations/layer8_attention_weight_min": -48.80682373046875,
      "activations/layer9_attention_weight_max": 71.82744598388672,
      "activations/layer9_attention_weight_min": -62.64036178588867,
      "epoch": 0.96,
      "learning_rate": 0.00014529545454545454,
      "loss": 3.0952,
      "step": 16450
    },
    {
      "activations/layer0_attention_weight_max": 14.198488235473633,
      "activations/layer0_attention_weight_min": -11.690587043762207,
      "activations/layer10_attention_weight_max": 77.3740234375,
      "activations/layer10_attention_weight_min": -60.94884490966797,
      "activations/layer11_attention_weight_max": 46.2949333190918,
      "activations/layer11_attention_weight_min": -39.111549377441406,
      "activations/layer12_attention_weight_max": 37.29474639892578,
      "activations/layer12_attention_weight_min": -25.69330406188965,
      "activations/layer13_attention_weight_max": 61.28861999511719,
      "activations/layer13_attention_weight_min": -46.17977523803711,
      "activations/layer14_attention_weight_max": 51.155662536621094,
      "activations/layer14_attention_weight_min": -37.81026840209961,
      "activations/layer15_attention_weight_max": 55.87424850463867,
      "activations/layer15_attention_weight_min": -41.556514739990234,
      "activations/layer16_attention_weight_max": 47.68692398071289,
      "activations/layer16_attention_weight_min": -36.344512939453125,
      "activations/layer17_attention_weight_max": 57.47597885131836,
      "activations/layer17_attention_weight_min": -38.500770568847656,
      "activations/layer18_attention_weight_max": 69.46529388427734,
      "activations/layer18_attention_weight_min": -43.34055709838867,
      "activations/layer19_attention_weight_max": 22.415037155151367,
      "activations/layer19_attention_weight_min": -12.253569602966309,
      "activations/layer1_attention_weight_max": 14.320950508117676,
      "activations/layer1_attention_weight_min": -9.97443675994873,
      "activations/layer20_attention_weight_max": 17.883508682250977,
      "activations/layer20_attention_weight_min": -13.007805824279785,
      "activations/layer21_attention_weight_max": 32.930328369140625,
      "activations/layer21_attention_weight_min": -16.994171142578125,
      "activations/layer22_attention_weight_max": 29.46491050720215,
      "activations/layer22_attention_weight_min": -15.352789878845215,
      "activations/layer23_attention_weight_max": 22.84343719482422,
      "activations/layer23_attention_weight_min": -15.982572555541992,
      "activations/layer2_attention_weight_max": 24.95345115661621,
      "activations/layer2_attention_weight_min": -21.11116600036621,
      "activations/layer3_attention_weight_max": 50.926597595214844,
      "activations/layer3_attention_weight_min": -48.193397521972656,
      "activations/layer4_attention_weight_max": 79.98751068115234,
      "activations/layer4_attention_weight_min": -70.27737426757812,
      "activations/layer5_attention_weight_max": 133.6981658935547,
      "activations/layer5_attention_weight_min": -119.716064453125,
      "activations/layer6_attention_weight_max": 64.28143310546875,
      "activations/layer6_attention_weight_min": -56.36649703979492,
      "activations/layer7_attention_weight_max": 67.10344696044922,
      "activations/layer7_attention_weight_min": -58.657188415527344,
      "activations/layer8_attention_weight_max": 55.94941329956055,
      "activations/layer8_attention_weight_min": -46.31000518798828,
      "activations/layer9_attention_weight_max": 78.59476470947266,
      "activations/layer9_attention_weight_min": -67.89147186279297,
      "epoch": 0.96,
      "learning_rate": 0.00014527651515151515,
      "loss": 3.0982,
      "step": 16500
    },
    {
      "activations/layer0_attention_weight_max": 14.770608901977539,
      "activations/layer0_attention_weight_min": -11.760977745056152,
      "activations/layer10_attention_weight_max": 68.6776351928711,
      "activations/layer10_attention_weight_min": -61.1202507019043,
      "activations/layer11_attention_weight_max": 49.987152099609375,
      "activations/layer11_attention_weight_min": -43.722190856933594,
      "activations/layer12_attention_weight_max": 26.399351119995117,
      "activations/layer12_attention_weight_min": -24.85556983947754,
      "activations/layer13_attention_weight_max": 76.44139862060547,
      "activations/layer13_attention_weight_min": -54.778751373291016,
      "activations/layer14_attention_weight_max": 78.70870208740234,
      "activations/layer14_attention_weight_min": -55.61305618286133,
      "activations/layer15_attention_weight_max": 56.12773513793945,
      "activations/layer15_attention_weight_min": -41.135215759277344,
      "activations/layer16_attention_weight_max": 47.86178970336914,
      "activations/layer16_attention_weight_min": -32.19923400878906,
      "activations/layer17_attention_weight_max": 65.6092300415039,
      "activations/layer17_attention_weight_min": -40.761024475097656,
      "activations/layer18_attention_weight_max": 64.07984924316406,
      "activations/layer18_attention_weight_min": -36.88213348388672,
      "activations/layer19_attention_weight_max": 22.067487716674805,
      "activations/layer19_attention_weight_min": -12.068777084350586,
      "activations/layer1_attention_weight_max": 15.955728530883789,
      "activations/layer1_attention_weight_min": -10.32157039642334,
      "activations/layer20_attention_weight_max": 17.02519416809082,
      "activations/layer20_attention_weight_min": -12.251850128173828,
      "activations/layer21_attention_weight_max": 30.144765853881836,
      "activations/layer21_attention_weight_min": -20.937149047851562,
      "activations/layer22_attention_weight_max": 28.4442081451416,
      "activations/layer22_attention_weight_min": -17.159114837646484,
      "activations/layer23_attention_weight_max": 21.696189880371094,
      "activations/layer23_attention_weight_min": -17.44206428527832,
      "activations/layer2_attention_weight_max": 25.69515037536621,
      "activations/layer2_attention_weight_min": -22.91299819946289,
      "activations/layer3_attention_weight_max": 50.984554290771484,
      "activations/layer3_attention_weight_min": -53.154151916503906,
      "activations/layer4_attention_weight_max": 84.56978607177734,
      "activations/layer4_attention_weight_min": -79.84954071044922,
      "activations/layer5_attention_weight_max": 143.77293395996094,
      "activations/layer5_attention_weight_min": -120.43365478515625,
      "activations/layer6_attention_weight_max": 67.22853088378906,
      "activations/layer6_attention_weight_min": -58.71205139160156,
      "activations/layer7_attention_weight_max": 65.05503845214844,
      "activations/layer7_attention_weight_min": -56.441837310791016,
      "activations/layer8_attention_weight_max": 55.391571044921875,
      "activations/layer8_attention_weight_min": -48.791927337646484,
      "activations/layer9_attention_weight_max": 74.25233459472656,
      "activations/layer9_attention_weight_min": -71.89962768554688,
      "epoch": 0.96,
      "learning_rate": 0.00014525795454545454,
      "loss": 3.1146,
      "step": 16550
    },
    {
      "activations/layer0_attention_weight_max": 13.908220291137695,
      "activations/layer0_attention_weight_min": -11.398886680603027,
      "activations/layer10_attention_weight_max": 59.785091400146484,
      "activations/layer10_attention_weight_min": -54.75361251831055,
      "activations/layer11_attention_weight_max": 41.836570739746094,
      "activations/layer11_attention_weight_min": -40.961814880371094,
      "activations/layer12_attention_weight_max": 29.221675872802734,
      "activations/layer12_attention_weight_min": -26.401107788085938,
      "activations/layer13_attention_weight_max": 56.42832946777344,
      "activations/layer13_attention_weight_min": -42.927879333496094,
      "activations/layer14_attention_weight_max": 53.10505676269531,
      "activations/layer14_attention_weight_min": -42.3527946472168,
      "activations/layer15_attention_weight_max": 50.47478103637695,
      "activations/layer15_attention_weight_min": -40.11379623413086,
      "activations/layer16_attention_weight_max": 48.642921447753906,
      "activations/layer16_attention_weight_min": -37.42420959472656,
      "activations/layer17_attention_weight_max": 55.01634216308594,
      "activations/layer17_attention_weight_min": -38.963253021240234,
      "activations/layer18_attention_weight_max": 66.35582733154297,
      "activations/layer18_attention_weight_min": -45.820255279541016,
      "activations/layer19_attention_weight_max": 19.888397216796875,
      "activations/layer19_attention_weight_min": -12.312315940856934,
      "activations/layer1_attention_weight_max": 15.416631698608398,
      "activations/layer1_attention_weight_min": -10.879095077514648,
      "activations/layer20_attention_weight_max": 17.88280487060547,
      "activations/layer20_attention_weight_min": -14.040030479431152,
      "activations/layer21_attention_weight_max": 31.578521728515625,
      "activations/layer21_attention_weight_min": -15.83195972442627,
      "activations/layer22_attention_weight_max": 27.37466812133789,
      "activations/layer22_attention_weight_min": -16.966768264770508,
      "activations/layer23_attention_weight_max": 20.07834243774414,
      "activations/layer23_attention_weight_min": -16.677757263183594,
      "activations/layer2_attention_weight_max": 24.670682907104492,
      "activations/layer2_attention_weight_min": -22.015058517456055,
      "activations/layer3_attention_weight_max": 50.354644775390625,
      "activations/layer3_attention_weight_min": -51.794403076171875,
      "activations/layer4_attention_weight_max": 74.36009979248047,
      "activations/layer4_attention_weight_min": -71.48495483398438,
      "activations/layer5_attention_weight_max": 124.35054016113281,
      "activations/layer5_attention_weight_min": -127.09950256347656,
      "activations/layer6_attention_weight_max": 58.812347412109375,
      "activations/layer6_attention_weight_min": -58.22528839111328,
      "activations/layer7_attention_weight_max": 58.34941101074219,
      "activations/layer7_attention_weight_min": -58.628211975097656,
      "activations/layer8_attention_weight_max": 49.82733917236328,
      "activations/layer8_attention_weight_min": -49.23423767089844,
      "activations/layer9_attention_weight_max": 69.09974670410156,
      "activations/layer9_attention_weight_min": -66.11603546142578,
      "epoch": 0.96,
      "learning_rate": 0.00014523901515151513,
      "loss": 3.1002,
      "step": 16600
    },
    {
      "activations/layer0_attention_weight_max": 15.409269332885742,
      "activations/layer0_attention_weight_min": -12.091591835021973,
      "activations/layer10_attention_weight_max": 73.60526275634766,
      "activations/layer10_attention_weight_min": -65.52232360839844,
      "activations/layer11_attention_weight_max": 49.30332565307617,
      "activations/layer11_attention_weight_min": -43.648475646972656,
      "activations/layer12_attention_weight_max": 27.674665451049805,
      "activations/layer12_attention_weight_min": -26.510948181152344,
      "activations/layer13_attention_weight_max": 54.02653121948242,
      "activations/layer13_attention_weight_min": -41.70459747314453,
      "activations/layer14_attention_weight_max": 48.460304260253906,
      "activations/layer14_attention_weight_min": -40.308868408203125,
      "activations/layer15_attention_weight_max": 53.23984146118164,
      "activations/layer15_attention_weight_min": -38.53144073486328,
      "activations/layer16_attention_weight_max": 44.42219161987305,
      "activations/layer16_attention_weight_min": -40.392852783203125,
      "activations/layer17_attention_weight_max": 51.253238677978516,
      "activations/layer17_attention_weight_min": -38.1472053527832,
      "activations/layer18_attention_weight_max": 59.40867233276367,
      "activations/layer18_attention_weight_min": -35.5042839050293,
      "activations/layer19_attention_weight_max": 19.022314071655273,
      "activations/layer19_attention_weight_min": -12.726054191589355,
      "activations/layer1_attention_weight_max": 14.922042846679688,
      "activations/layer1_attention_weight_min": -12.009326934814453,
      "activations/layer20_attention_weight_max": 19.14866828918457,
      "activations/layer20_attention_weight_min": -13.609315872192383,
      "activations/layer21_attention_weight_max": 37.2220458984375,
      "activations/layer21_attention_weight_min": -18.50643539428711,
      "activations/layer22_attention_weight_max": 32.77776336669922,
      "activations/layer22_attention_weight_min": -16.538188934326172,
      "activations/layer23_attention_weight_max": 21.73785972595215,
      "activations/layer23_attention_weight_min": -15.94900131225586,
      "activations/layer2_attention_weight_max": 25.935504913330078,
      "activations/layer2_attention_weight_min": -23.316875457763672,
      "activations/layer3_attention_weight_max": 52.48421096801758,
      "activations/layer3_attention_weight_min": -52.115108489990234,
      "activations/layer4_attention_weight_max": 82.52302551269531,
      "activations/layer4_attention_weight_min": -72.23516082763672,
      "activations/layer5_attention_weight_max": 137.3975830078125,
      "activations/layer5_attention_weight_min": -128.32431030273438,
      "activations/layer6_attention_weight_max": 64.36570739746094,
      "activations/layer6_attention_weight_min": -58.154788970947266,
      "activations/layer7_attention_weight_max": 67.37698364257812,
      "activations/layer7_attention_weight_min": -61.34503936767578,
      "activations/layer8_attention_weight_max": 58.21309280395508,
      "activations/layer8_attention_weight_min": -52.83586120605469,
      "activations/layer9_attention_weight_max": 89.327880859375,
      "activations/layer9_attention_weight_min": -79.20342254638672,
      "epoch": 0.97,
      "learning_rate": 0.00014522007575757575,
      "loss": 3.1027,
      "step": 16650
    },
    {
      "activations/layer0_attention_weight_max": 15.948047637939453,
      "activations/layer0_attention_weight_min": -11.920001983642578,
      "activations/layer10_attention_weight_max": 77.5412368774414,
      "activations/layer10_attention_weight_min": -60.54853439331055,
      "activations/layer11_attention_weight_max": 55.04335021972656,
      "activations/layer11_attention_weight_min": -39.000152587890625,
      "activations/layer12_attention_weight_max": 28.368244171142578,
      "activations/layer12_attention_weight_min": -28.63587760925293,
      "activations/layer13_attention_weight_max": 84.75813293457031,
      "activations/layer13_attention_weight_min": -49.50194549560547,
      "activations/layer14_attention_weight_max": 81.93358612060547,
      "activations/layer14_attention_weight_min": -41.52744674682617,
      "activations/layer15_attention_weight_max": 71.5732650756836,
      "activations/layer15_attention_weight_min": -37.86384963989258,
      "activations/layer16_attention_weight_max": 50.01069641113281,
      "activations/layer16_attention_weight_min": -33.356056213378906,
      "activations/layer17_attention_weight_max": 57.09870147705078,
      "activations/layer17_attention_weight_min": -38.96149826049805,
      "activations/layer18_attention_weight_max": 58.69289779663086,
      "activations/layer18_attention_weight_min": -38.52851104736328,
      "activations/layer19_attention_weight_max": 20.810850143432617,
      "activations/layer19_attention_weight_min": -12.041585922241211,
      "activations/layer1_attention_weight_max": 15.33360481262207,
      "activations/layer1_attention_weight_min": -10.28552532196045,
      "activations/layer20_attention_weight_max": 16.910795211791992,
      "activations/layer20_attention_weight_min": -13.492735862731934,
      "activations/layer21_attention_weight_max": 36.64809036254883,
      "activations/layer21_attention_weight_min": -16.770706176757812,
      "activations/layer22_attention_weight_max": 32.13926315307617,
      "activations/layer22_attention_weight_min": -14.97317123413086,
      "activations/layer23_attention_weight_max": 20.253496170043945,
      "activations/layer23_attention_weight_min": -16.52606964111328,
      "activations/layer2_attention_weight_max": 24.808425903320312,
      "activations/layer2_attention_weight_min": -23.63559341430664,
      "activations/layer3_attention_weight_max": 53.992103576660156,
      "activations/layer3_attention_weight_min": -54.370704650878906,
      "activations/layer4_attention_weight_max": 82.99812316894531,
      "activations/layer4_attention_weight_min": -76.91346740722656,
      "activations/layer5_attention_weight_max": 137.09754943847656,
      "activations/layer5_attention_weight_min": -124.1070556640625,
      "activations/layer6_attention_weight_max": 67.94974517822266,
      "activations/layer6_attention_weight_min": -58.03668975830078,
      "activations/layer7_attention_weight_max": 73.18584442138672,
      "activations/layer7_attention_weight_min": -62.67893600463867,
      "activations/layer8_attention_weight_max": 57.13057327270508,
      "activations/layer8_attention_weight_min": -50.587581634521484,
      "activations/layer9_attention_weight_max": 79.9809799194336,
      "activations/layer9_attention_weight_min": -67.2801513671875,
      "epoch": 0.97,
      "learning_rate": 0.00014520113636363636,
      "loss": 3.1058,
      "step": 16700
    },
    {
      "activations/layer0_attention_weight_max": 13.916258811950684,
      "activations/layer0_attention_weight_min": -11.444280624389648,
      "activations/layer10_attention_weight_max": 66.90596771240234,
      "activations/layer10_attention_weight_min": -57.53167724609375,
      "activations/layer11_attention_weight_max": 44.381996154785156,
      "activations/layer11_attention_weight_min": -35.51215744018555,
      "activations/layer12_attention_weight_max": 27.764419555664062,
      "activations/layer12_attention_weight_min": -24.235742568969727,
      "activations/layer13_attention_weight_max": 62.8795051574707,
      "activations/layer13_attention_weight_min": -44.02232360839844,
      "activations/layer14_attention_weight_max": 44.79487609863281,
      "activations/layer14_attention_weight_min": -29.944467544555664,
      "activations/layer15_attention_weight_max": 58.040401458740234,
      "activations/layer15_attention_weight_min": -40.62841033935547,
      "activations/layer16_attention_weight_max": 49.41523742675781,
      "activations/layer16_attention_weight_min": -31.69499397277832,
      "activations/layer17_attention_weight_max": 56.874176025390625,
      "activations/layer17_attention_weight_min": -35.498878479003906,
      "activations/layer18_attention_weight_max": 60.20773696899414,
      "activations/layer18_attention_weight_min": -34.051395416259766,
      "activations/layer19_attention_weight_max": 19.832286834716797,
      "activations/layer19_attention_weight_min": -11.584847450256348,
      "activations/layer1_attention_weight_max": 15.569684982299805,
      "activations/layer1_attention_weight_min": -11.396740913391113,
      "activations/layer20_attention_weight_max": 19.186120986938477,
      "activations/layer20_attention_weight_min": -14.034526824951172,
      "activations/layer21_attention_weight_max": 35.481170654296875,
      "activations/layer21_attention_weight_min": -18.615970611572266,
      "activations/layer22_attention_weight_max": 34.09232711791992,
      "activations/layer22_attention_weight_min": -15.695449829101562,
      "activations/layer23_attention_weight_max": 24.57552719116211,
      "activations/layer23_attention_weight_min": -16.06586456298828,
      "activations/layer2_attention_weight_max": 24.14691925048828,
      "activations/layer2_attention_weight_min": -22.683795928955078,
      "activations/layer3_attention_weight_max": 52.9146842956543,
      "activations/layer3_attention_weight_min": -54.969810485839844,
      "activations/layer4_attention_weight_max": 83.29418182373047,
      "activations/layer4_attention_weight_min": -75.97242736816406,
      "activations/layer5_attention_weight_max": 137.03707885742188,
      "activations/layer5_attention_weight_min": -119.64378356933594,
      "activations/layer6_attention_weight_max": 66.11589813232422,
      "activations/layer6_attention_weight_min": -57.5644645690918,
      "activations/layer7_attention_weight_max": 63.760677337646484,
      "activations/layer7_attention_weight_min": -56.31611633300781,
      "activations/layer8_attention_weight_max": 56.65851593017578,
      "activations/layer8_attention_weight_min": -49.10123825073242,
      "activations/layer9_attention_weight_max": 76.04733276367188,
      "activations/layer9_attention_weight_min": -64.37493133544922,
      "epoch": 0.97,
      "learning_rate": 0.00014518219696969695,
      "loss": 3.1044,
      "step": 16750
    },
    {
      "activations/layer0_attention_weight_max": 13.721940994262695,
      "activations/layer0_attention_weight_min": -11.063299179077148,
      "activations/layer10_attention_weight_max": 65.76237487792969,
      "activations/layer10_attention_weight_min": -59.82181167602539,
      "activations/layer11_attention_weight_max": 42.05965805053711,
      "activations/layer11_attention_weight_min": -41.34407043457031,
      "activations/layer12_attention_weight_max": 27.24989128112793,
      "activations/layer12_attention_weight_min": -25.02396011352539,
      "activations/layer13_attention_weight_max": 52.96803665161133,
      "activations/layer13_attention_weight_min": -41.731414794921875,
      "activations/layer14_attention_weight_max": 45.18500518798828,
      "activations/layer14_attention_weight_min": -39.95132064819336,
      "activations/layer15_attention_weight_max": 57.008968353271484,
      "activations/layer15_attention_weight_min": -44.953304290771484,
      "activations/layer16_attention_weight_max": 50.10602569580078,
      "activations/layer16_attention_weight_min": -37.845245361328125,
      "activations/layer17_attention_weight_max": 65.787109375,
      "activations/layer17_attention_weight_min": -40.176937103271484,
      "activations/layer18_attention_weight_max": 66.00425720214844,
      "activations/layer18_attention_weight_min": -41.637046813964844,
      "activations/layer19_attention_weight_max": 21.746641159057617,
      "activations/layer19_attention_weight_min": -12.869203567504883,
      "activations/layer1_attention_weight_max": 15.181142807006836,
      "activations/layer1_attention_weight_min": -10.853759765625,
      "activations/layer20_attention_weight_max": 19.219820022583008,
      "activations/layer20_attention_weight_min": -14.372859954833984,
      "activations/layer21_attention_weight_max": 36.65349578857422,
      "activations/layer21_attention_weight_min": -18.667509078979492,
      "activations/layer22_attention_weight_max": 36.078128814697266,
      "activations/layer22_attention_weight_min": -16.618633270263672,
      "activations/layer23_attention_weight_max": 25.633174896240234,
      "activations/layer23_attention_weight_min": -15.745809555053711,
      "activations/layer2_attention_weight_max": 26.072490692138672,
      "activations/layer2_attention_weight_min": -24.171852111816406,
      "activations/layer3_attention_weight_max": 52.743770599365234,
      "activations/layer3_attention_weight_min": -53.301177978515625,
      "activations/layer4_attention_weight_max": 82.10851287841797,
      "activations/layer4_attention_weight_min": -75.63530731201172,
      "activations/layer5_attention_weight_max": 135.4334716796875,
      "activations/layer5_attention_weight_min": -125.41593933105469,
      "activations/layer6_attention_weight_max": 66.39703369140625,
      "activations/layer6_attention_weight_min": -62.93383026123047,
      "activations/layer7_attention_weight_max": 61.97471237182617,
      "activations/layer7_attention_weight_min": -58.7452507019043,
      "activations/layer8_attention_weight_max": 54.954383850097656,
      "activations/layer8_attention_weight_min": -50.177242279052734,
      "activations/layer9_attention_weight_max": 65.4826431274414,
      "activations/layer9_attention_weight_min": -62.453102111816406,
      "epoch": 0.98,
      "learning_rate": 0.00014516325757575756,
      "loss": 3.0842,
      "step": 16800
    },
    {
      "activations/layer0_attention_weight_max": 12.981189727783203,
      "activations/layer0_attention_weight_min": -10.84350299835205,
      "activations/layer10_attention_weight_max": 75.09840393066406,
      "activations/layer10_attention_weight_min": -60.241817474365234,
      "activations/layer11_attention_weight_max": 50.22266387939453,
      "activations/layer11_attention_weight_min": -42.428131103515625,
      "activations/layer12_attention_weight_max": 28.470125198364258,
      "activations/layer12_attention_weight_min": -25.658748626708984,
      "activations/layer13_attention_weight_max": 66.55751037597656,
      "activations/layer13_attention_weight_min": -45.83797836303711,
      "activations/layer14_attention_weight_max": 55.32530212402344,
      "activations/layer14_attention_weight_min": -38.124759674072266,
      "activations/layer15_attention_weight_max": 57.731929779052734,
      "activations/layer15_attention_weight_min": -41.651939392089844,
      "activations/layer16_attention_weight_max": 47.50551223754883,
      "activations/layer16_attention_weight_min": -36.74264907836914,
      "activations/layer17_attention_weight_max": 55.36326599121094,
      "activations/layer17_attention_weight_min": -40.11294937133789,
      "activations/layer18_attention_weight_max": 74.6684799194336,
      "activations/layer18_attention_weight_min": -47.20425796508789,
      "activations/layer19_attention_weight_max": 21.464521408081055,
      "activations/layer19_attention_weight_min": -12.240126609802246,
      "activations/layer1_attention_weight_max": 15.39876937866211,
      "activations/layer1_attention_weight_min": -10.752107620239258,
      "activations/layer20_attention_weight_max": 19.335628509521484,
      "activations/layer20_attention_weight_min": -13.21924877166748,
      "activations/layer21_attention_weight_max": 38.310386657714844,
      "activations/layer21_attention_weight_min": -17.773887634277344,
      "activations/layer22_attention_weight_max": 31.3625545501709,
      "activations/layer22_attention_weight_min": -18.59792137145996,
      "activations/layer23_attention_weight_max": 26.62613296508789,
      "activations/layer23_attention_weight_min": -15.79773235321045,
      "activations/layer2_attention_weight_max": 24.952556610107422,
      "activations/layer2_attention_weight_min": -22.534259796142578,
      "activations/layer3_attention_weight_max": 53.513912200927734,
      "activations/layer3_attention_weight_min": -51.97697067260742,
      "activations/layer4_attention_weight_max": 84.10298919677734,
      "activations/layer4_attention_weight_min": -75.11073303222656,
      "activations/layer5_attention_weight_max": 140.224365234375,
      "activations/layer5_attention_weight_min": -131.2877960205078,
      "activations/layer6_attention_weight_max": 68.64356994628906,
      "activations/layer6_attention_weight_min": -63.69939041137695,
      "activations/layer7_attention_weight_max": 66.37041473388672,
      "activations/layer7_attention_weight_min": -60.524410247802734,
      "activations/layer8_attention_weight_max": 59.36207580566406,
      "activations/layer8_attention_weight_min": -49.72674560546875,
      "activations/layer9_attention_weight_max": 77.84982299804688,
      "activations/layer9_attention_weight_min": -71.1133041381836,
      "epoch": 0.98,
      "learning_rate": 0.00014514431818181818,
      "loss": 3.1129,
      "step": 16850
    },
    {
      "activations/layer0_attention_weight_max": 13.923656463623047,
      "activations/layer0_attention_weight_min": -10.781834602355957,
      "activations/layer10_attention_weight_max": 68.50482177734375,
      "activations/layer10_attention_weight_min": -58.701805114746094,
      "activations/layer11_attention_weight_max": 46.43387985229492,
      "activations/layer11_attention_weight_min": -38.635902404785156,
      "activations/layer12_attention_weight_max": 29.63425064086914,
      "activations/layer12_attention_weight_min": -23.747604370117188,
      "activations/layer13_attention_weight_max": 56.686546325683594,
      "activations/layer13_attention_weight_min": -47.171077728271484,
      "activations/layer14_attention_weight_max": 56.35671615600586,
      "activations/layer14_attention_weight_min": -50.08136749267578,
      "activations/layer15_attention_weight_max": 53.8970947265625,
      "activations/layer15_attention_weight_min": -40.36897659301758,
      "activations/layer16_attention_weight_max": 46.82215881347656,
      "activations/layer16_attention_weight_min": -38.34886932373047,
      "activations/layer17_attention_weight_max": 58.34283447265625,
      "activations/layer17_attention_weight_min": -43.81769943237305,
      "activations/layer18_attention_weight_max": 59.012638092041016,
      "activations/layer18_attention_weight_min": -35.64073944091797,
      "activations/layer19_attention_weight_max": 20.347322463989258,
      "activations/layer19_attention_weight_min": -13.624035835266113,
      "activations/layer1_attention_weight_max": 15.001290321350098,
      "activations/layer1_attention_weight_min": -11.546171188354492,
      "activations/layer20_attention_weight_max": 20.516748428344727,
      "activations/layer20_attention_weight_min": -13.403450965881348,
      "activations/layer21_attention_weight_max": 43.807373046875,
      "activations/layer21_attention_weight_min": -18.79580307006836,
      "activations/layer22_attention_weight_max": 37.01950454711914,
      "activations/layer22_attention_weight_min": -18.284027099609375,
      "activations/layer23_attention_weight_max": 23.684890747070312,
      "activations/layer23_attention_weight_min": -15.96525764465332,
      "activations/layer2_attention_weight_max": 25.944074630737305,
      "activations/layer2_attention_weight_min": -22.580747604370117,
      "activations/layer3_attention_weight_max": 52.61842727661133,
      "activations/layer3_attention_weight_min": -51.45075988769531,
      "activations/layer4_attention_weight_max": 79.40108489990234,
      "activations/layer4_attention_weight_min": -72.1701889038086,
      "activations/layer5_attention_weight_max": 139.9146728515625,
      "activations/layer5_attention_weight_min": -125.44164276123047,
      "activations/layer6_attention_weight_max": 65.57328796386719,
      "activations/layer6_attention_weight_min": -57.7204704284668,
      "activations/layer7_attention_weight_max": 70.48677062988281,
      "activations/layer7_attention_weight_min": -58.8671760559082,
      "activations/layer8_attention_weight_max": 52.1668586730957,
      "activations/layer8_attention_weight_min": -51.08125686645508,
      "activations/layer9_attention_weight_max": 70.93167114257812,
      "activations/layer9_attention_weight_min": -61.7183952331543,
      "epoch": 0.98,
      "learning_rate": 0.00014512537878787877,
      "loss": 3.1086,
      "step": 16900
    },
    {
      "activations/layer0_attention_weight_max": 13.69109058380127,
      "activations/layer0_attention_weight_min": -10.573287963867188,
      "activations/layer10_attention_weight_max": 74.60139465332031,
      "activations/layer10_attention_weight_min": -61.825523376464844,
      "activations/layer11_attention_weight_max": 51.11913299560547,
      "activations/layer11_attention_weight_min": -40.88703155517578,
      "activations/layer12_attention_weight_max": 30.872926712036133,
      "activations/layer12_attention_weight_min": -24.72955894470215,
      "activations/layer13_attention_weight_max": 57.95668411254883,
      "activations/layer13_attention_weight_min": -46.28108215332031,
      "activations/layer14_attention_weight_max": 52.20440673828125,
      "activations/layer14_attention_weight_min": -44.056175231933594,
      "activations/layer15_attention_weight_max": 52.595821380615234,
      "activations/layer15_attention_weight_min": -42.862342834472656,
      "activations/layer16_attention_weight_max": 49.88931655883789,
      "activations/layer16_attention_weight_min": -37.20314407348633,
      "activations/layer17_attention_weight_max": 61.72293472290039,
      "activations/layer17_attention_weight_min": -37.350643157958984,
      "activations/layer18_attention_weight_max": 67.8212661743164,
      "activations/layer18_attention_weight_min": -47.5598258972168,
      "activations/layer19_attention_weight_max": 20.701467514038086,
      "activations/layer19_attention_weight_min": -13.319609642028809,
      "activations/layer1_attention_weight_max": 15.705391883850098,
      "activations/layer1_attention_weight_min": -11.956429481506348,
      "activations/layer20_attention_weight_max": 20.58679962158203,
      "activations/layer20_attention_weight_min": -13.016683578491211,
      "activations/layer21_attention_weight_max": 36.13241195678711,
      "activations/layer21_attention_weight_min": -18.309585571289062,
      "activations/layer22_attention_weight_max": 34.6285400390625,
      "activations/layer22_attention_weight_min": -18.22101402282715,
      "activations/layer23_attention_weight_max": 26.118391036987305,
      "activations/layer23_attention_weight_min": -16.0242919921875,
      "activations/layer2_attention_weight_max": 25.664213180541992,
      "activations/layer2_attention_weight_min": -23.8255615234375,
      "activations/layer3_attention_weight_max": 52.63932800292969,
      "activations/layer3_attention_weight_min": -54.79403305053711,
      "activations/layer4_attention_weight_max": 83.14068603515625,
      "activations/layer4_attention_weight_min": -76.02132415771484,
      "activations/layer5_attention_weight_max": 134.83633422851562,
      "activations/layer5_attention_weight_min": -121.5780258178711,
      "activations/layer6_attention_weight_max": 63.57655715942383,
      "activations/layer6_attention_weight_min": -57.145591735839844,
      "activations/layer7_attention_weight_max": 65.5455093383789,
      "activations/layer7_attention_weight_min": -56.962806701660156,
      "activations/layer8_attention_weight_max": 55.91044616699219,
      "activations/layer8_attention_weight_min": -51.184722900390625,
      "activations/layer9_attention_weight_max": 70.9012680053711,
      "activations/layer9_attention_weight_min": -64.4704818725586,
      "epoch": 0.98,
      "learning_rate": 0.00014510643939393938,
      "loss": 3.1014,
      "step": 16950
    },
    {
      "activations/layer0_attention_weight_max": 14.11834716796875,
      "activations/layer0_attention_weight_min": -11.041470527648926,
      "activations/layer10_attention_weight_max": 69.01810455322266,
      "activations/layer10_attention_weight_min": -60.72466278076172,
      "activations/layer11_attention_weight_max": 44.763893127441406,
      "activations/layer11_attention_weight_min": -42.096187591552734,
      "activations/layer12_attention_weight_max": 28.18773078918457,
      "activations/layer12_attention_weight_min": -23.683561325073242,
      "activations/layer13_attention_weight_max": 67.12374877929688,
      "activations/layer13_attention_weight_min": -47.62178421020508,
      "activations/layer14_attention_weight_max": 45.849510192871094,
      "activations/layer14_attention_weight_min": -34.47587203979492,
      "activations/layer15_attention_weight_max": 55.576534271240234,
      "activations/layer15_attention_weight_min": -39.54072570800781,
      "activations/layer16_attention_weight_max": 60.744606018066406,
      "activations/layer16_attention_weight_min": -38.86186599731445,
      "activations/layer17_attention_weight_max": 62.38887405395508,
      "activations/layer17_attention_weight_min": -36.43986892700195,
      "activations/layer18_attention_weight_max": 68.03314208984375,
      "activations/layer18_attention_weight_min": -41.149044036865234,
      "activations/layer19_attention_weight_max": 24.09530258178711,
      "activations/layer19_attention_weight_min": -12.605814933776855,
      "activations/layer1_attention_weight_max": 16.196880340576172,
      "activations/layer1_attention_weight_min": -11.295692443847656,
      "activations/layer20_attention_weight_max": 18.576383590698242,
      "activations/layer20_attention_weight_min": -14.953335762023926,
      "activations/layer21_attention_weight_max": 37.228302001953125,
      "activations/layer21_attention_weight_min": -18.07711410522461,
      "activations/layer22_attention_weight_max": 32.03922653198242,
      "activations/layer22_attention_weight_min": -15.508954048156738,
      "activations/layer23_attention_weight_max": 22.980405807495117,
      "activations/layer23_attention_weight_min": -15.92438793182373,
      "activations/layer2_attention_weight_max": 25.241636276245117,
      "activations/layer2_attention_weight_min": -23.110862731933594,
      "activations/layer3_attention_weight_max": 52.92974853515625,
      "activations/layer3_attention_weight_min": -53.094417572021484,
      "activations/layer4_attention_weight_max": 81.15087890625,
      "activations/layer4_attention_weight_min": -74.64466094970703,
      "activations/layer5_attention_weight_max": 137.32301330566406,
      "activations/layer5_attention_weight_min": -128.1251220703125,
      "activations/layer6_attention_weight_max": 65.14431762695312,
      "activations/layer6_attention_weight_min": -59.7489013671875,
      "activations/layer7_attention_weight_max": 65.126708984375,
      "activations/layer7_attention_weight_min": -57.4355354309082,
      "activations/layer8_attention_weight_max": 59.45106506347656,
      "activations/layer8_attention_weight_min": -55.267822265625,
      "activations/layer9_attention_weight_max": 69.15455627441406,
      "activations/layer9_attention_weight_min": -70.71434783935547,
      "epoch": 0.99,
      "learning_rate": 0.0001450875,
      "loss": 3.1009,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_loss": 3.044921875,
      "eval_runtime": 8.3449,
      "eval_samples_per_second": 514.564,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_openwebtext_loss": 3.044921875,
      "eval_openwebtext_ppl": 21.008389858307645,
      "eval_openwebtext_runtime": 8.3449,
      "eval_openwebtext_samples_per_second": 514.564,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_wikitext_loss": 3.337890625,
      "eval_wikitext_ppl": 28.15966472087403,
      "eval_wikitext_runtime": 1.7193,
      "eval_wikitext_samples_per_second": 265.22,
      "step": 17000
    },
    {
      "epoch": 0.99,
      "eval_lambada_loss": 3.494140625,
      "eval_lambada_ppl": 32.92198346312245,
      "eval_lambada_runtime": 9.2994,
      "eval_lambada_samples_per_second": 523.583,
      "step": 17000
    },
    {
      "activations/layer0_attention_weight_max": 14.849527359008789,
      "activations/layer0_attention_weight_min": -10.74858283996582,
      "activations/layer10_attention_weight_max": 77.43505859375,
      "activations/layer10_attention_weight_min": -68.11276245117188,
      "activations/layer11_attention_weight_max": 54.84125900268555,
      "activations/layer11_attention_weight_min": -46.679927825927734,
      "activations/layer12_attention_weight_max": 30.175682067871094,
      "activations/layer12_attention_weight_min": -29.453367233276367,
      "activations/layer13_attention_weight_max": 73.49577331542969,
      "activations/layer13_attention_weight_min": -46.658687591552734,
      "activations/layer14_attention_weight_max": 62.58674621582031,
      "activations/layer14_attention_weight_min": -40.86723709106445,
      "activations/layer15_attention_weight_max": 49.58069610595703,
      "activations/layer15_attention_weight_min": -37.42573547363281,
      "activations/layer16_attention_weight_max": 51.8826789855957,
      "activations/layer16_attention_weight_min": -34.804351806640625,
      "activations/layer17_attention_weight_max": 63.1291389465332,
      "activations/layer17_attention_weight_min": -36.926795959472656,
      "activations/layer18_attention_weight_max": 83.26777648925781,
      "activations/layer18_attention_weight_min": -42.16352462768555,
      "activations/layer19_attention_weight_max": 20.392925262451172,
      "activations/layer19_attention_weight_min": -12.379778861999512,
      "activations/layer1_attention_weight_max": 14.237451553344727,
      "activations/layer1_attention_weight_min": -11.358811378479004,
      "activations/layer20_attention_weight_max": 18.141441345214844,
      "activations/layer20_attention_weight_min": -14.565258026123047,
      "activations/layer21_attention_weight_max": 36.568023681640625,
      "activations/layer21_attention_weight_min": -17.252553939819336,
      "activations/layer22_attention_weight_max": 33.691490173339844,
      "activations/layer22_attention_weight_min": -15.427814483642578,
      "activations/layer23_attention_weight_max": 19.958467483520508,
      "activations/layer23_attention_weight_min": -15.474363327026367,
      "activations/layer2_attention_weight_max": 27.203861236572266,
      "activations/layer2_attention_weight_min": -24.091941833496094,
      "activations/layer3_attention_weight_max": 56.7486686706543,
      "activations/layer3_attention_weight_min": -57.02032470703125,
      "activations/layer4_attention_weight_max": 88.2116470336914,
      "activations/layer4_attention_weight_min": -78.07183074951172,
      "activations/layer5_attention_weight_max": 144.88357543945312,
      "activations/layer5_attention_weight_min": -126.05876922607422,
      "activations/layer6_attention_weight_max": 69.75431060791016,
      "activations/layer6_attention_weight_min": -56.751625061035156,
      "activations/layer7_attention_weight_max": 70.78439331054688,
      "activations/layer7_attention_weight_min": -57.45134735107422,
      "activations/layer8_attention_weight_max": 60.68012237548828,
      "activations/layer8_attention_weight_min": -50.81752395629883,
      "activations/layer9_attention_weight_max": 89.87458801269531,
      "activations/layer9_attention_weight_min": -74.81700897216797,
      "epoch": 0.99,
      "learning_rate": 0.00014506856060606058,
      "loss": 3.099,
      "step": 17050
    },
    {
      "activations/layer0_attention_weight_max": 14.517221450805664,
      "activations/layer0_attention_weight_min": -10.928947448730469,
      "activations/layer10_attention_weight_max": 86.86622619628906,
      "activations/layer10_attention_weight_min": -63.543182373046875,
      "activations/layer11_attention_weight_max": 56.57771301269531,
      "activations/layer11_attention_weight_min": -43.618186950683594,
      "activations/layer12_attention_weight_max": 26.512651443481445,
      "activations/layer12_attention_weight_min": -24.514646530151367,
      "activations/layer13_attention_weight_max": 73.90286254882812,
      "activations/layer13_attention_weight_min": -44.99990463256836,
      "activations/layer14_attention_weight_max": 66.08182525634766,
      "activations/layer14_attention_weight_min": -38.85895538330078,
      "activations/layer15_attention_weight_max": 57.270050048828125,
      "activations/layer15_attention_weight_min": -36.47000503540039,
      "activations/layer16_attention_weight_max": 51.36990737915039,
      "activations/layer16_attention_weight_min": -33.952396392822266,
      "activations/layer17_attention_weight_max": 61.23580551147461,
      "activations/layer17_attention_weight_min": -40.60647964477539,
      "activations/layer18_attention_weight_max": 67.0647201538086,
      "activations/layer18_attention_weight_min": -43.08653259277344,
      "activations/layer19_attention_weight_max": 19.75111961364746,
      "activations/layer19_attention_weight_min": -12.204668998718262,
      "activations/layer1_attention_weight_max": 15.041391372680664,
      "activations/layer1_attention_weight_min": -11.666444778442383,
      "activations/layer20_attention_weight_max": 16.90363883972168,
      "activations/layer20_attention_weight_min": -15.779413223266602,
      "activations/layer21_attention_weight_max": 32.815616607666016,
      "activations/layer21_attention_weight_min": -18.302444458007812,
      "activations/layer22_attention_weight_max": 30.136751174926758,
      "activations/layer22_attention_weight_min": -16.085969924926758,
      "activations/layer23_attention_weight_max": 19.186935424804688,
      "activations/layer23_attention_weight_min": -15.611608505249023,
      "activations/layer2_attention_weight_max": 27.99786949157715,
      "activations/layer2_attention_weight_min": -25.643522262573242,
      "activations/layer3_attention_weight_max": 56.694637298583984,
      "activations/layer3_attention_weight_min": -57.96738815307617,
      "activations/layer4_attention_weight_max": 88.66320037841797,
      "activations/layer4_attention_weight_min": -77.23335266113281,
      "activations/layer5_attention_weight_max": 152.2771759033203,
      "activations/layer5_attention_weight_min": -125.90187072753906,
      "activations/layer6_attention_weight_max": 70.97488403320312,
      "activations/layer6_attention_weight_min": -57.31147766113281,
      "activations/layer7_attention_weight_max": 74.80640411376953,
      "activations/layer7_attention_weight_min": -57.540218353271484,
      "activations/layer8_attention_weight_max": 62.76244354248047,
      "activations/layer8_attention_weight_min": -49.46852493286133,
      "activations/layer9_attention_weight_max": 90.17798614501953,
      "activations/layer9_attention_weight_min": -73.3381576538086,
      "epoch": 0.99,
      "learning_rate": 0.0001450496212121212,
      "loss": 3.1086,
      "step": 17100
    },
    {
      "activations/layer0_attention_weight_max": 13.142065048217773,
      "activations/layer0_attention_weight_min": -11.209753036499023,
      "activations/layer10_attention_weight_max": 67.75695037841797,
      "activations/layer10_attention_weight_min": -59.253807067871094,
      "activations/layer11_attention_weight_max": 47.19935607910156,
      "activations/layer11_attention_weight_min": -41.9321174621582,
      "activations/layer12_attention_weight_max": 28.36665153503418,
      "activations/layer12_attention_weight_min": -25.605384826660156,
      "activations/layer13_attention_weight_max": 81.54794311523438,
      "activations/layer13_attention_weight_min": -50.096290588378906,
      "activations/layer14_attention_weight_max": 84.27810668945312,
      "activations/layer14_attention_weight_min": -50.52471923828125,
      "activations/layer15_attention_weight_max": 52.668025970458984,
      "activations/layer15_attention_weight_min": -38.7774543762207,
      "activations/layer16_attention_weight_max": 49.734683990478516,
      "activations/layer16_attention_weight_min": -36.6706657409668,
      "activations/layer17_attention_weight_max": 54.361595153808594,
      "activations/layer17_attention_weight_min": -37.52473831176758,
      "activations/layer18_attention_weight_max": 63.964599609375,
      "activations/layer18_attention_weight_min": -39.930885314941406,
      "activations/layer19_attention_weight_max": 20.130353927612305,
      "activations/layer19_attention_weight_min": -16.42940330505371,
      "activations/layer1_attention_weight_max": 15.322942733764648,
      "activations/layer1_attention_weight_min": -11.387993812561035,
      "activations/layer20_attention_weight_max": 21.667869567871094,
      "activations/layer20_attention_weight_min": -13.425487518310547,
      "activations/layer21_attention_weight_max": 35.6982421875,
      "activations/layer21_attention_weight_min": -18.07779312133789,
      "activations/layer22_attention_weight_max": 31.83399772644043,
      "activations/layer22_attention_weight_min": -16.132341384887695,
      "activations/layer23_attention_weight_max": 23.30617332458496,
      "activations/layer23_attention_weight_min": -15.388702392578125,
      "activations/layer2_attention_weight_max": 25.913291931152344,
      "activations/layer2_attention_weight_min": -23.664289474487305,
      "activations/layer3_attention_weight_max": 56.779945373535156,
      "activations/layer3_attention_weight_min": -55.13095474243164,
      "activations/layer4_attention_weight_max": 83.99345397949219,
      "activations/layer4_attention_weight_min": -76.5971450805664,
      "activations/layer5_attention_weight_max": 137.66513061523438,
      "activations/layer5_attention_weight_min": -122.46290588378906,
      "activations/layer6_attention_weight_max": 66.75191497802734,
      "activations/layer6_attention_weight_min": -55.615074157714844,
      "activations/layer7_attention_weight_max": 66.7605972290039,
      "activations/layer7_attention_weight_min": -57.933753967285156,
      "activations/layer8_attention_weight_max": 60.39961242675781,
      "activations/layer8_attention_weight_min": -54.219722747802734,
      "activations/layer9_attention_weight_max": 72.58849334716797,
      "activations/layer9_attention_weight_min": -65.16828155517578,
      "epoch": 1.0,
      "learning_rate": 0.0001450306818181818,
      "loss": 3.1078,
      "step": 17150
    },
    {
      "activations/layer0_attention_weight_max": 13.116823196411133,
      "activations/layer0_attention_weight_min": -10.625410079956055,
      "activations/layer10_attention_weight_max": 62.95521926879883,
      "activations/layer10_attention_weight_min": -62.52675247192383,
      "activations/layer11_attention_weight_max": 49.00273132324219,
      "activations/layer11_attention_weight_min": -45.59873580932617,
      "activations/layer12_attention_weight_max": 31.156696319580078,
      "activations/layer12_attention_weight_min": -26.533920288085938,
      "activations/layer13_attention_weight_max": 53.0713996887207,
      "activations/layer13_attention_weight_min": -42.08623504638672,
      "activations/layer14_attention_weight_max": 52.357994079589844,
      "activations/layer14_attention_weight_min": -40.70094680786133,
      "activations/layer15_attention_weight_max": 52.383602142333984,
      "activations/layer15_attention_weight_min": -42.29240417480469,
      "activations/layer16_attention_weight_max": 44.479530334472656,
      "activations/layer16_attention_weight_min": -32.413673400878906,
      "activations/layer17_attention_weight_max": 53.70899200439453,
      "activations/layer17_attention_weight_min": -39.16081619262695,
      "activations/layer18_attention_weight_max": 62.940834045410156,
      "activations/layer18_attention_weight_min": -42.05386734008789,
      "activations/layer19_attention_weight_max": 19.637676239013672,
      "activations/layer19_attention_weight_min": -12.664670944213867,
      "activations/layer1_attention_weight_max": 14.474934577941895,
      "activations/layer1_attention_weight_min": -9.973423957824707,
      "activations/layer20_attention_weight_max": 18.22646713256836,
      "activations/layer20_attention_weight_min": -15.148228645324707,
      "activations/layer21_attention_weight_max": 33.10721206665039,
      "activations/layer21_attention_weight_min": -18.689346313476562,
      "activations/layer22_attention_weight_max": 30.055580139160156,
      "activations/layer22_attention_weight_min": -15.876155853271484,
      "activations/layer23_attention_weight_max": 23.835371017456055,
      "activations/layer23_attention_weight_min": -16.76926040649414,
      "activations/layer2_attention_weight_max": 24.405813217163086,
      "activations/layer2_attention_weight_min": -21.806732177734375,
      "activations/layer3_attention_weight_max": 51.174461364746094,
      "activations/layer3_attention_weight_min": -52.41132354736328,
      "activations/layer4_attention_weight_max": 79.40380096435547,
      "activations/layer4_attention_weight_min": -72.5919418334961,
      "activations/layer5_attention_weight_max": 129.04815673828125,
      "activations/layer5_attention_weight_min": -123.9728012084961,
      "activations/layer6_attention_weight_max": 64.09834289550781,
      "activations/layer6_attention_weight_min": -58.11008834838867,
      "activations/layer7_attention_weight_max": 62.18996047973633,
      "activations/layer7_attention_weight_min": -57.44453048706055,
      "activations/layer8_attention_weight_max": 52.41196060180664,
      "activations/layer8_attention_weight_min": -49.75370788574219,
      "activations/layer9_attention_weight_max": 68.6651611328125,
      "activations/layer9_attention_weight_min": -63.4377555847168,
      "epoch": 1.0,
      "learning_rate": 0.0001450117424242424,
      "loss": 3.0852,
      "step": 17200
    },
    {
      "activations/layer0_attention_weight_max": 14.84309196472168,
      "activations/layer0_attention_weight_min": -10.99868392944336,
      "activations/layer10_attention_weight_max": 71.46615600585938,
      "activations/layer10_attention_weight_min": -57.17380905151367,
      "activations/layer11_attention_weight_max": 45.199928283691406,
      "activations/layer11_attention_weight_min": -42.39281463623047,
      "activations/layer12_attention_weight_max": 27.86435317993164,
      "activations/layer12_attention_weight_min": -22.61018180847168,
      "activations/layer13_attention_weight_max": 57.47780227661133,
      "activations/layer13_attention_weight_min": -42.13641357421875,
      "activations/layer14_attention_weight_max": 56.10715866088867,
      "activations/layer14_attention_weight_min": -40.050018310546875,
      "activations/layer15_attention_weight_max": 54.751426696777344,
      "activations/layer15_attention_weight_min": -42.70209884643555,
      "activations/layer16_attention_weight_max": 58.49037170410156,
      "activations/layer16_attention_weight_min": -36.89834976196289,
      "activations/layer17_attention_weight_max": 65.28623962402344,
      "activations/layer17_attention_weight_min": -37.1884765625,
      "activations/layer18_attention_weight_max": 70.72435760498047,
      "activations/layer18_attention_weight_min": -44.02048110961914,
      "activations/layer19_attention_weight_max": 21.97934913635254,
      "activations/layer19_attention_weight_min": -13.972940444946289,
      "activations/layer1_attention_weight_max": 14.864346504211426,
      "activations/layer1_attention_weight_min": -11.687180519104004,
      "activations/layer20_attention_weight_max": 20.473678588867188,
      "activations/layer20_attention_weight_min": -14.98213005065918,
      "activations/layer21_attention_weight_max": 39.40274429321289,
      "activations/layer21_attention_weight_min": -17.15896987915039,
      "activations/layer22_attention_weight_max": 35.52388381958008,
      "activations/layer22_attention_weight_min": -17.028282165527344,
      "activations/layer23_attention_weight_max": 23.309972763061523,
      "activations/layer23_attention_weight_min": -16.64896011352539,
      "activations/layer2_attention_weight_max": 25.98085594177246,
      "activations/layer2_attention_weight_min": -23.339954376220703,
      "activations/layer3_attention_weight_max": 54.06315231323242,
      "activations/layer3_attention_weight_min": -52.41096496582031,
      "activations/layer4_attention_weight_max": 84.19634246826172,
      "activations/layer4_attention_weight_min": -76.02045440673828,
      "activations/layer5_attention_weight_max": 140.00653076171875,
      "activations/layer5_attention_weight_min": -124.4310302734375,
      "activations/layer6_attention_weight_max": 65.36854553222656,
      "activations/layer6_attention_weight_min": -57.617225646972656,
      "activations/layer7_attention_weight_max": 67.75377655029297,
      "activations/layer7_attention_weight_min": -58.770206451416016,
      "activations/layer8_attention_weight_max": 56.6481819152832,
      "activations/layer8_attention_weight_min": -48.832435607910156,
      "activations/layer9_attention_weight_max": 68.69774627685547,
      "activations/layer9_attention_weight_min": -66.30281829833984,
      "epoch": 1.0,
      "learning_rate": 0.00014499280303030302,
      "loss": 3.1618,
      "step": 17250
    },
    {
      "activations/layer0_attention_weight_max": 13.92508602142334,
      "activations/layer0_attention_weight_min": -10.754505157470703,
      "activations/layer10_attention_weight_max": 64.49443054199219,
      "activations/layer10_attention_weight_min": -56.40562057495117,
      "activations/layer11_attention_weight_max": 44.575294494628906,
      "activations/layer11_attention_weight_min": -39.15149688720703,
      "activations/layer12_attention_weight_max": 32.06803512573242,
      "activations/layer12_attention_weight_min": -24.393333435058594,
      "activations/layer13_attention_weight_max": 51.78211975097656,
      "activations/layer13_attention_weight_min": -42.80164337158203,
      "activations/layer14_attention_weight_max": 45.9725341796875,
      "activations/layer14_attention_weight_min": -32.47174835205078,
      "activations/layer15_attention_weight_max": 55.52817153930664,
      "activations/layer15_attention_weight_min": -38.93822479248047,
      "activations/layer16_attention_weight_max": 48.09768295288086,
      "activations/layer16_attention_weight_min": -33.95032501220703,
      "activations/layer17_attention_weight_max": 55.97833251953125,
      "activations/layer17_attention_weight_min": -37.6512336730957,
      "activations/layer18_attention_weight_max": 59.780338287353516,
      "activations/layer18_attention_weight_min": -34.3084716796875,
      "activations/layer19_attention_weight_max": 21.1289119720459,
      "activations/layer19_attention_weight_min": -12.308392524719238,
      "activations/layer1_attention_weight_max": 14.821968078613281,
      "activations/layer1_attention_weight_min": -10.328177452087402,
      "activations/layer20_attention_weight_max": 19.23094940185547,
      "activations/layer20_attention_weight_min": -15.302239418029785,
      "activations/layer21_attention_weight_max": 41.84769058227539,
      "activations/layer21_attention_weight_min": -19.552385330200195,
      "activations/layer22_attention_weight_max": 34.27911376953125,
      "activations/layer22_attention_weight_min": -15.823695182800293,
      "activations/layer23_attention_weight_max": 23.294212341308594,
      "activations/layer23_attention_weight_min": -14.868534088134766,
      "activations/layer2_attention_weight_max": 24.54613494873047,
      "activations/layer2_attention_weight_min": -22.729507446289062,
      "activations/layer3_attention_weight_max": 50.76851272583008,
      "activations/layer3_attention_weight_min": -52.26041030883789,
      "activations/layer4_attention_weight_max": 82.32579803466797,
      "activations/layer4_attention_weight_min": -73.7101058959961,
      "activations/layer5_attention_weight_max": 138.79888916015625,
      "activations/layer5_attention_weight_min": -121.87370300292969,
      "activations/layer6_attention_weight_max": 65.86907958984375,
      "activations/layer6_attention_weight_min": -61.50621032714844,
      "activations/layer7_attention_weight_max": 66.96675872802734,
      "activations/layer7_attention_weight_min": -60.221858978271484,
      "activations/layer8_attention_weight_max": 58.17555618286133,
      "activations/layer8_attention_weight_min": -51.902530670166016,
      "activations/layer9_attention_weight_max": 75.3884506225586,
      "activations/layer9_attention_weight_min": -67.92103576660156,
      "epoch": 1.01,
      "learning_rate": 0.0001449738636363636,
      "loss": 3.0862,
      "step": 17300
    },
    {
      "activations/layer0_attention_weight_max": 13.9262056350708,
      "activations/layer0_attention_weight_min": -11.366722106933594,
      "activations/layer10_attention_weight_max": 68.14149475097656,
      "activations/layer10_attention_weight_min": -59.88078308105469,
      "activations/layer11_attention_weight_max": 48.046958923339844,
      "activations/layer11_attention_weight_min": -39.328330993652344,
      "activations/layer12_attention_weight_max": 27.44464874267578,
      "activations/layer12_attention_weight_min": -23.338462829589844,
      "activations/layer13_attention_weight_max": 69.86073303222656,
      "activations/layer13_attention_weight_min": -51.438316345214844,
      "activations/layer14_attention_weight_max": 66.58808898925781,
      "activations/layer14_attention_weight_min": -56.63151931762695,
      "activations/layer15_attention_weight_max": 59.140953063964844,
      "activations/layer15_attention_weight_min": -41.696537017822266,
      "activations/layer16_attention_weight_max": 55.75428009033203,
      "activations/layer16_attention_weight_min": -37.16568374633789,
      "activations/layer17_attention_weight_max": 60.14342498779297,
      "activations/layer17_attention_weight_min": -37.095584869384766,
      "activations/layer18_attention_weight_max": 59.055946350097656,
      "activations/layer18_attention_weight_min": -36.05257797241211,
      "activations/layer19_attention_weight_max": 19.063987731933594,
      "activations/layer19_attention_weight_min": -12.653550148010254,
      "activations/layer1_attention_weight_max": 14.645441055297852,
      "activations/layer1_attention_weight_min": -11.413860321044922,
      "activations/layer20_attention_weight_max": 20.12714385986328,
      "activations/layer20_attention_weight_min": -12.863636016845703,
      "activations/layer21_attention_weight_max": 39.16182327270508,
      "activations/layer21_attention_weight_min": -16.898483276367188,
      "activations/layer22_attention_weight_max": 34.237266540527344,
      "activations/layer22_attention_weight_min": -14.858154296875,
      "activations/layer23_attention_weight_max": 21.180980682373047,
      "activations/layer23_attention_weight_min": -15.70899486541748,
      "activations/layer2_attention_weight_max": 24.661245346069336,
      "activations/layer2_attention_weight_min": -22.413923263549805,
      "activations/layer3_attention_weight_max": 52.195343017578125,
      "activations/layer3_attention_weight_min": -54.393192291259766,
      "activations/layer4_attention_weight_max": 85.0295639038086,
      "activations/layer4_attention_weight_min": -74.18238830566406,
      "activations/layer5_attention_weight_max": 137.92376708984375,
      "activations/layer5_attention_weight_min": -124.9836654663086,
      "activations/layer6_attention_weight_max": 65.01979064941406,
      "activations/layer6_attention_weight_min": -59.18936538696289,
      "activations/layer7_attention_weight_max": 67.70436096191406,
      "activations/layer7_attention_weight_min": -61.935909271240234,
      "activations/layer8_attention_weight_max": 62.12020492553711,
      "activations/layer8_attention_weight_min": -56.33939743041992,
      "activations/layer9_attention_weight_max": 68.05514526367188,
      "activations/layer9_attention_weight_min": -68.24893188476562,
      "epoch": 1.01,
      "learning_rate": 0.00014495492424242422,
      "loss": 3.0805,
      "step": 17350
    },
    {
      "activations/layer0_attention_weight_max": 13.676044464111328,
      "activations/layer0_attention_weight_min": -11.072928428649902,
      "activations/layer10_attention_weight_max": 71.1419448852539,
      "activations/layer10_attention_weight_min": -66.95677185058594,
      "activations/layer11_attention_weight_max": 53.23780822753906,
      "activations/layer11_attention_weight_min": -39.82699966430664,
      "activations/layer12_attention_weight_max": 29.420917510986328,
      "activations/layer12_attention_weight_min": -24.86548614501953,
      "activations/layer13_attention_weight_max": 65.992919921875,
      "activations/layer13_attention_weight_min": -42.187896728515625,
      "activations/layer14_attention_weight_max": 44.19731903076172,
      "activations/layer14_attention_weight_min": -33.11726760864258,
      "activations/layer15_attention_weight_max": 57.72929763793945,
      "activations/layer15_attention_weight_min": -42.652000427246094,
      "activations/layer16_attention_weight_max": 48.53632736206055,
      "activations/layer16_attention_weight_min": -34.11691665649414,
      "activations/layer17_attention_weight_max": 60.10257339477539,
      "activations/layer17_attention_weight_min": -38.8127326965332,
      "activations/layer18_attention_weight_max": 76.3102035522461,
      "activations/layer18_attention_weight_min": -42.39777755737305,
      "activations/layer19_attention_weight_max": 21.469867706298828,
      "activations/layer19_attention_weight_min": -12.696074485778809,
      "activations/layer1_attention_weight_max": 15.603713035583496,
      "activations/layer1_attention_weight_min": -11.103096008300781,
      "activations/layer20_attention_weight_max": 17.97470474243164,
      "activations/layer20_attention_weight_min": -13.655447959899902,
      "activations/layer21_attention_weight_max": 33.87710189819336,
      "activations/layer21_attention_weight_min": -18.121706008911133,
      "activations/layer22_attention_weight_max": 32.730796813964844,
      "activations/layer22_attention_weight_min": -14.360428810119629,
      "activations/layer23_attention_weight_max": 23.003650665283203,
      "activations/layer23_attention_weight_min": -15.639535903930664,
      "activations/layer2_attention_weight_max": 25.78537368774414,
      "activations/layer2_attention_weight_min": -23.01442527770996,
      "activations/layer3_attention_weight_max": 51.797264099121094,
      "activations/layer3_attention_weight_min": -52.815940856933594,
      "activations/layer4_attention_weight_max": 82.92145538330078,
      "activations/layer4_attention_weight_min": -72.43826293945312,
      "activations/layer5_attention_weight_max": 143.95892333984375,
      "activations/layer5_attention_weight_min": -117.07328796386719,
      "activations/layer6_attention_weight_max": 68.6406021118164,
      "activations/layer6_attention_weight_min": -57.23942947387695,
      "activations/layer7_attention_weight_max": 74.19536590576172,
      "activations/layer7_attention_weight_min": -55.548683166503906,
      "activations/layer8_attention_weight_max": 58.40342330932617,
      "activations/layer8_attention_weight_min": -48.58742141723633,
      "activations/layer9_attention_weight_max": 76.57234954833984,
      "activations/layer9_attention_weight_min": -68.92461395263672,
      "epoch": 1.01,
      "learning_rate": 0.00014493598484848484,
      "loss": 3.0798,
      "step": 17400
    },
    {
      "activations/layer0_attention_weight_max": 14.453577995300293,
      "activations/layer0_attention_weight_min": -10.515213966369629,
      "activations/layer10_attention_weight_max": 70.55139923095703,
      "activations/layer10_attention_weight_min": -59.491146087646484,
      "activations/layer11_attention_weight_max": 43.47081756591797,
      "activations/layer11_attention_weight_min": -38.191322326660156,
      "activations/layer12_attention_weight_max": 27.577157974243164,
      "activations/layer12_attention_weight_min": -23.997636795043945,
      "activations/layer13_attention_weight_max": 69.74898529052734,
      "activations/layer13_attention_weight_min": -45.54604721069336,
      "activations/layer14_attention_weight_max": 70.24778747558594,
      "activations/layer14_attention_weight_min": -43.98126220703125,
      "activations/layer15_attention_weight_max": 56.030372619628906,
      "activations/layer15_attention_weight_min": -38.574771881103516,
      "activations/layer16_attention_weight_max": 47.44111251831055,
      "activations/layer16_attention_weight_min": -34.12633514404297,
      "activations/layer17_attention_weight_max": 60.86524200439453,
      "activations/layer17_attention_weight_min": -38.00661087036133,
      "activations/layer18_attention_weight_max": 62.20441818237305,
      "activations/layer18_attention_weight_min": -38.03787612915039,
      "activations/layer19_attention_weight_max": 21.21184730529785,
      "activations/layer19_attention_weight_min": -12.980570793151855,
      "activations/layer1_attention_weight_max": 15.755889892578125,
      "activations/layer1_attention_weight_min": -10.474453926086426,
      "activations/layer20_attention_weight_max": 19.378408432006836,
      "activations/layer20_attention_weight_min": -14.227585792541504,
      "activations/layer21_attention_weight_max": 39.49896240234375,
      "activations/layer21_attention_weight_min": -17.92947769165039,
      "activations/layer22_attention_weight_max": 32.33195877075195,
      "activations/layer22_attention_weight_min": -16.298635482788086,
      "activations/layer23_attention_weight_max": 19.651874542236328,
      "activations/layer23_attention_weight_min": -15.00737190246582,
      "activations/layer2_attention_weight_max": 25.37509536743164,
      "activations/layer2_attention_weight_min": -21.93147850036621,
      "activations/layer3_attention_weight_max": 48.22431182861328,
      "activations/layer3_attention_weight_min": -50.67408752441406,
      "activations/layer4_attention_weight_max": 75.00524139404297,
      "activations/layer4_attention_weight_min": -74.56035614013672,
      "activations/layer5_attention_weight_max": 125.18925476074219,
      "activations/layer5_attention_weight_min": -127.1703109741211,
      "activations/layer6_attention_weight_max": 58.89897155761719,
      "activations/layer6_attention_weight_min": -57.16836929321289,
      "activations/layer7_attention_weight_max": 64.22721099853516,
      "activations/layer7_attention_weight_min": -60.1591911315918,
      "activations/layer8_attention_weight_max": 51.47055435180664,
      "activations/layer8_attention_weight_min": -47.6037483215332,
      "activations/layer9_attention_weight_max": 66.74734497070312,
      "activations/layer9_attention_weight_min": -64.37667846679688,
      "epoch": 1.01,
      "learning_rate": 0.00014491704545454545,
      "loss": 3.0893,
      "step": 17450
    },
    {
      "activations/layer0_attention_weight_max": 16.872541427612305,
      "activations/layer0_attention_weight_min": -10.941642761230469,
      "activations/layer10_attention_weight_max": 86.90383911132812,
      "activations/layer10_attention_weight_min": -66.99234008789062,
      "activations/layer11_attention_weight_max": 57.29364013671875,
      "activations/layer11_attention_weight_min": -46.803192138671875,
      "activations/layer12_attention_weight_max": 28.183149337768555,
      "activations/layer12_attention_weight_min": -25.59022331237793,
      "activations/layer13_attention_weight_max": 56.03229904174805,
      "activations/layer13_attention_weight_min": -47.00899887084961,
      "activations/layer14_attention_weight_max": 68.18585968017578,
      "activations/layer14_attention_weight_min": -50.336578369140625,
      "activations/layer15_attention_weight_max": 60.79343795776367,
      "activations/layer15_attention_weight_min": -42.19838333129883,
      "activations/layer16_attention_weight_max": 50.69546127319336,
      "activations/layer16_attention_weight_min": -34.367950439453125,
      "activations/layer17_attention_weight_max": 61.25566482543945,
      "activations/layer17_attention_weight_min": -41.29655456542969,
      "activations/layer18_attention_weight_max": 72.4321060180664,
      "activations/layer18_attention_weight_min": -44.94207000732422,
      "activations/layer19_attention_weight_max": 21.3482723236084,
      "activations/layer19_attention_weight_min": -13.819498062133789,
      "activations/layer1_attention_weight_max": 15.562925338745117,
      "activations/layer1_attention_weight_min": -10.330350875854492,
      "activations/layer20_attention_weight_max": 20.895353317260742,
      "activations/layer20_attention_weight_min": -13.83746337890625,
      "activations/layer21_attention_weight_max": 33.379913330078125,
      "activations/layer21_attention_weight_min": -20.943992614746094,
      "activations/layer22_attention_weight_max": 30.677841186523438,
      "activations/layer22_attention_weight_min": -19.809459686279297,
      "activations/layer23_attention_weight_max": 19.526351928710938,
      "activations/layer23_attention_weight_min": -14.262758255004883,
      "activations/layer2_attention_weight_max": 27.036930084228516,
      "activations/layer2_attention_weight_min": -23.923572540283203,
      "activations/layer3_attention_weight_max": 54.1282958984375,
      "activations/layer3_attention_weight_min": -56.750572204589844,
      "activations/layer4_attention_weight_max": 83.8373031616211,
      "activations/layer4_attention_weight_min": -76.47612762451172,
      "activations/layer5_attention_weight_max": 145.92930603027344,
      "activations/layer5_attention_weight_min": -131.84022521972656,
      "activations/layer6_attention_weight_max": 64.68844604492188,
      "activations/layer6_attention_weight_min": -56.80363082885742,
      "activations/layer7_attention_weight_max": 64.82261657714844,
      "activations/layer7_attention_weight_min": -58.29599380493164,
      "activations/layer8_attention_weight_max": 66.11695098876953,
      "activations/layer8_attention_weight_min": -57.1470947265625,
      "activations/layer9_attention_weight_max": 91.22210693359375,
      "activations/layer9_attention_weight_min": -79.98745727539062,
      "epoch": 1.02,
      "learning_rate": 0.00014489810606060604,
      "loss": 3.0741,
      "step": 17500
    },
    {
      "activations/layer0_attention_weight_max": 14.40315055847168,
      "activations/layer0_attention_weight_min": -10.921106338500977,
      "activations/layer10_attention_weight_max": 64.8324203491211,
      "activations/layer10_attention_weight_min": -63.68366622924805,
      "activations/layer11_attention_weight_max": 47.77287292480469,
      "activations/layer11_attention_weight_min": -44.519622802734375,
      "activations/layer12_attention_weight_max": 27.70336151123047,
      "activations/layer12_attention_weight_min": -25.663654327392578,
      "activations/layer13_attention_weight_max": 54.05681610107422,
      "activations/layer13_attention_weight_min": -38.4033203125,
      "activations/layer14_attention_weight_max": 53.800804138183594,
      "activations/layer14_attention_weight_min": -33.20753479003906,
      "activations/layer15_attention_weight_max": 51.972686767578125,
      "activations/layer15_attention_weight_min": -37.10067367553711,
      "activations/layer16_attention_weight_max": 56.2386589050293,
      "activations/layer16_attention_weight_min": -32.8199462890625,
      "activations/layer17_attention_weight_max": 63.64194107055664,
      "activations/layer17_attention_weight_min": -39.878021240234375,
      "activations/layer18_attention_weight_max": 72.22957611083984,
      "activations/layer18_attention_weight_min": -44.10612487792969,
      "activations/layer19_attention_weight_max": 17.262502670288086,
      "activations/layer19_attention_weight_min": -11.548215866088867,
      "activations/layer1_attention_weight_max": 14.561922073364258,
      "activations/layer1_attention_weight_min": -9.6497163772583,
      "activations/layer20_attention_weight_max": 18.33661651611328,
      "activations/layer20_attention_weight_min": -13.055116653442383,
      "activations/layer21_attention_weight_max": 39.90033721923828,
      "activations/layer21_attention_weight_min": -17.82490348815918,
      "activations/layer22_attention_weight_max": 39.182373046875,
      "activations/layer22_attention_weight_min": -17.712085723876953,
      "activations/layer23_attention_weight_max": 26.172069549560547,
      "activations/layer23_attention_weight_min": -15.405620574951172,
      "activations/layer2_attention_weight_max": 24.344879150390625,
      "activations/layer2_attention_weight_min": -23.035945892333984,
      "activations/layer3_attention_weight_max": 50.72245407104492,
      "activations/layer3_attention_weight_min": -52.14280700683594,
      "activations/layer4_attention_weight_max": 76.56756591796875,
      "activations/layer4_attention_weight_min": -73.20735168457031,
      "activations/layer5_attention_weight_max": 128.24330139160156,
      "activations/layer5_attention_weight_min": -126.4939193725586,
      "activations/layer6_attention_weight_max": 61.31190872192383,
      "activations/layer6_attention_weight_min": -57.74434280395508,
      "activations/layer7_attention_weight_max": 63.99348068237305,
      "activations/layer7_attention_weight_min": -58.440216064453125,
      "activations/layer8_attention_weight_max": 57.56689453125,
      "activations/layer8_attention_weight_min": -52.78187561035156,
      "activations/layer9_attention_weight_max": 72.07864379882812,
      "activations/layer9_attention_weight_min": -68.77571868896484,
      "epoch": 1.02,
      "learning_rate": 0.00014487954545454543,
      "loss": 3.0781,
      "step": 17550
    },
    {
      "activations/layer0_attention_weight_max": 13.874333381652832,
      "activations/layer0_attention_weight_min": -10.598718643188477,
      "activations/layer10_attention_weight_max": 63.54896926879883,
      "activations/layer10_attention_weight_min": -59.24211120605469,
      "activations/layer11_attention_weight_max": 45.966793060302734,
      "activations/layer11_attention_weight_min": -39.144371032714844,
      "activations/layer12_attention_weight_max": 29.988407135009766,
      "activations/layer12_attention_weight_min": -23.92357635498047,
      "activations/layer13_attention_weight_max": 59.741859436035156,
      "activations/layer13_attention_weight_min": -42.563941955566406,
      "activations/layer14_attention_weight_max": 38.089332580566406,
      "activations/layer14_attention_weight_min": -37.813568115234375,
      "activations/layer15_attention_weight_max": 51.090789794921875,
      "activations/layer15_attention_weight_min": -41.41936492919922,
      "activations/layer16_attention_weight_max": 50.01288986206055,
      "activations/layer16_attention_weight_min": -35.9432373046875,
      "activations/layer17_attention_weight_max": 60.82781219482422,
      "activations/layer17_attention_weight_min": -36.99867248535156,
      "activations/layer18_attention_weight_max": 56.29910659790039,
      "activations/layer18_attention_weight_min": -35.00404357910156,
      "activations/layer19_attention_weight_max": 23.02937889099121,
      "activations/layer19_attention_weight_min": -13.736540794372559,
      "activations/layer1_attention_weight_max": 16.20359230041504,
      "activations/layer1_attention_weight_min": -10.486611366271973,
      "activations/layer20_attention_weight_max": 18.871402740478516,
      "activations/layer20_attention_weight_min": -15.791540145874023,
      "activations/layer21_attention_weight_max": 34.42375564575195,
      "activations/layer21_attention_weight_min": -16.463361740112305,
      "activations/layer22_attention_weight_max": 36.555965423583984,
      "activations/layer22_attention_weight_min": -16.533994674682617,
      "activations/layer23_attention_weight_max": 21.454505920410156,
      "activations/layer23_attention_weight_min": -18.645771026611328,
      "activations/layer2_attention_weight_max": 25.578609466552734,
      "activations/layer2_attention_weight_min": -22.31966209411621,
      "activations/layer3_attention_weight_max": 49.080989837646484,
      "activations/layer3_attention_weight_min": -51.81765365600586,
      "activations/layer4_attention_weight_max": 77.09112548828125,
      "activations/layer4_attention_weight_min": -72.86702728271484,
      "activations/layer5_attention_weight_max": 121.85631561279297,
      "activations/layer5_attention_weight_min": -121.35084533691406,
      "activations/layer6_attention_weight_max": 60.397796630859375,
      "activations/layer6_attention_weight_min": -55.727569580078125,
      "activations/layer7_attention_weight_max": 62.88913345336914,
      "activations/layer7_attention_weight_min": -58.29478073120117,
      "activations/layer8_attention_weight_max": 53.06592559814453,
      "activations/layer8_attention_weight_min": -46.504127502441406,
      "activations/layer9_attention_weight_max": 68.62214660644531,
      "activations/layer9_attention_weight_min": -65.88101196289062,
      "epoch": 1.02,
      "learning_rate": 0.00014486060606060604,
      "loss": 3.0751,
      "step": 17600
    },
    {
      "activations/layer0_attention_weight_max": 14.025341033935547,
      "activations/layer0_attention_weight_min": -10.862071990966797,
      "activations/layer10_attention_weight_max": 61.62797546386719,
      "activations/layer10_attention_weight_min": -54.95441818237305,
      "activations/layer11_attention_weight_max": 43.711082458496094,
      "activations/layer11_attention_weight_min": -39.02488327026367,
      "activations/layer12_attention_weight_max": 25.432056427001953,
      "activations/layer12_attention_weight_min": -27.787809371948242,
      "activations/layer13_attention_weight_max": 54.858985900878906,
      "activations/layer13_attention_weight_min": -43.75132369995117,
      "activations/layer14_attention_weight_max": 45.66284942626953,
      "activations/layer14_attention_weight_min": -39.78190612792969,
      "activations/layer15_attention_weight_max": 51.10175323486328,
      "activations/layer15_attention_weight_min": -41.535301208496094,
      "activations/layer16_attention_weight_max": 46.76176071166992,
      "activations/layer16_attention_weight_min": -34.79966735839844,
      "activations/layer17_attention_weight_max": 55.0720329284668,
      "activations/layer17_attention_weight_min": -37.543575286865234,
      "activations/layer18_attention_weight_max": 61.961181640625,
      "activations/layer18_attention_weight_min": -43.33891296386719,
      "activations/layer19_attention_weight_max": 21.640172958374023,
      "activations/layer19_attention_weight_min": -12.185614585876465,
      "activations/layer1_attention_weight_max": 15.494278907775879,
      "activations/layer1_attention_weight_min": -10.059316635131836,
      "activations/layer20_attention_weight_max": 17.339704513549805,
      "activations/layer20_attention_weight_min": -13.621773719787598,
      "activations/layer21_attention_weight_max": 44.57435607910156,
      "activations/layer21_attention_weight_min": -20.351608276367188,
      "activations/layer22_attention_weight_max": 33.521827697753906,
      "activations/layer22_attention_weight_min": -16.725013732910156,
      "activations/layer23_attention_weight_max": 21.598060607910156,
      "activations/layer23_attention_weight_min": -16.373397827148438,
      "activations/layer2_attention_weight_max": 25.53316307067871,
      "activations/layer2_attention_weight_min": -22.55609130859375,
      "activations/layer3_attention_weight_max": 50.8192138671875,
      "activations/layer3_attention_weight_min": -52.68390655517578,
      "activations/layer4_attention_weight_max": 79.39956665039062,
      "activations/layer4_attention_weight_min": -75.87487030029297,
      "activations/layer5_attention_weight_max": 127.1109619140625,
      "activations/layer5_attention_weight_min": -120.86691284179688,
      "activations/layer6_attention_weight_max": 59.861907958984375,
      "activations/layer6_attention_weight_min": -54.86996841430664,
      "activations/layer7_attention_weight_max": 64.48898315429688,
      "activations/layer7_attention_weight_min": -56.852561950683594,
      "activations/layer8_attention_weight_max": 51.94450759887695,
      "activations/layer8_attention_weight_min": -46.79899597167969,
      "activations/layer9_attention_weight_max": 67.36717987060547,
      "activations/layer9_attention_weight_min": -62.703094482421875,
      "epoch": 1.03,
      "learning_rate": 0.00014484166666666666,
      "loss": 3.0952,
      "step": 17650
    },
    {
      "activations/layer0_attention_weight_max": 14.356632232666016,
      "activations/layer0_attention_weight_min": -11.002561569213867,
      "activations/layer10_attention_weight_max": 67.16739654541016,
      "activations/layer10_attention_weight_min": -59.48290252685547,
      "activations/layer11_attention_weight_max": 48.114131927490234,
      "activations/layer11_attention_weight_min": -42.9080924987793,
      "activations/layer12_attention_weight_max": 28.418109893798828,
      "activations/layer12_attention_weight_min": -25.791303634643555,
      "activations/layer13_attention_weight_max": 54.172584533691406,
      "activations/layer13_attention_weight_min": -42.83176040649414,
      "activations/layer14_attention_weight_max": 61.41800308227539,
      "activations/layer14_attention_weight_min": -48.424320220947266,
      "activations/layer15_attention_weight_max": 55.291927337646484,
      "activations/layer15_attention_weight_min": -39.287628173828125,
      "activations/layer16_attention_weight_max": 52.062835693359375,
      "activations/layer16_attention_weight_min": -35.24570846557617,
      "activations/layer17_attention_weight_max": 61.88209915161133,
      "activations/layer17_attention_weight_min": -39.25407409667969,
      "activations/layer18_attention_weight_max": 61.53657150268555,
      "activations/layer18_attention_weight_min": -32.743629455566406,
      "activations/layer19_attention_weight_max": 19.64363670349121,
      "activations/layer19_attention_weight_min": -14.156814575195312,
      "activations/layer1_attention_weight_max": 15.11654281616211,
      "activations/layer1_attention_weight_min": -11.767603874206543,
      "activations/layer20_attention_weight_max": 21.476804733276367,
      "activations/layer20_attention_weight_min": -13.399690628051758,
      "activations/layer21_attention_weight_max": 38.28885269165039,
      "activations/layer21_attention_weight_min": -18.39089584350586,
      "activations/layer22_attention_weight_max": 34.73221969604492,
      "activations/layer22_attention_weight_min": -19.416696548461914,
      "activations/layer23_attention_weight_max": 26.846694946289062,
      "activations/layer23_attention_weight_min": -15.223464965820312,
      "activations/layer2_attention_weight_max": 25.8916072845459,
      "activations/layer2_attention_weight_min": -23.588621139526367,
      "activations/layer3_attention_weight_max": 51.1890754699707,
      "activations/layer3_attention_weight_min": -51.5345458984375,
      "activations/layer4_attention_weight_max": 82.9687728881836,
      "activations/layer4_attention_weight_min": -73.93441009521484,
      "activations/layer5_attention_weight_max": 139.4327392578125,
      "activations/layer5_attention_weight_min": -123.68710327148438,
      "activations/layer6_attention_weight_max": 61.46902847290039,
      "activations/layer6_attention_weight_min": -55.22250747680664,
      "activations/layer7_attention_weight_max": 67.10914611816406,
      "activations/layer7_attention_weight_min": -56.26041030883789,
      "activations/layer8_attention_weight_max": 57.30663299560547,
      "activations/layer8_attention_weight_min": -47.889015197753906,
      "activations/layer9_attention_weight_max": 69.44084167480469,
      "activations/layer9_attention_weight_min": -62.25083541870117,
      "epoch": 1.03,
      "learning_rate": 0.00014482272727272727,
      "loss": 3.0967,
      "step": 17700
    },
    {
      "activations/layer0_attention_weight_max": 13.824074745178223,
      "activations/layer0_attention_weight_min": -11.36949634552002,
      "activations/layer10_attention_weight_max": 66.24333190917969,
      "activations/layer10_attention_weight_min": -62.92222213745117,
      "activations/layer11_attention_weight_max": 40.41319274902344,
      "activations/layer11_attention_weight_min": -40.60288619995117,
      "activations/layer12_attention_weight_max": 24.292240142822266,
      "activations/layer12_attention_weight_min": -27.737545013427734,
      "activations/layer13_attention_weight_max": 55.01167297363281,
      "activations/layer13_attention_weight_min": -40.99156951904297,
      "activations/layer14_attention_weight_max": 53.988922119140625,
      "activations/layer14_attention_weight_min": -37.562129974365234,
      "activations/layer15_attention_weight_max": 51.299049377441406,
      "activations/layer15_attention_weight_min": -38.73401641845703,
      "activations/layer16_attention_weight_max": 43.058475494384766,
      "activations/layer16_attention_weight_min": -34.73059844970703,
      "activations/layer17_attention_weight_max": 51.944820404052734,
      "activations/layer17_attention_weight_min": -40.34932327270508,
      "activations/layer18_attention_weight_max": 66.6684799194336,
      "activations/layer18_attention_weight_min": -45.36856460571289,
      "activations/layer19_attention_weight_max": 18.12464714050293,
      "activations/layer19_attention_weight_min": -13.11441707611084,
      "activations/layer1_attention_weight_max": 15.619425773620605,
      "activations/layer1_attention_weight_min": -11.325072288513184,
      "activations/layer20_attention_weight_max": 19.136241912841797,
      "activations/layer20_attention_weight_min": -14.50575065612793,
      "activations/layer21_attention_weight_max": 41.48004913330078,
      "activations/layer21_attention_weight_min": -18.659942626953125,
      "activations/layer22_attention_weight_max": 36.963775634765625,
      "activations/layer22_attention_weight_min": -14.528162956237793,
      "activations/layer23_attention_weight_max": 22.034603118896484,
      "activations/layer23_attention_weight_min": -15.013739585876465,
      "activations/layer2_attention_weight_max": 24.02394676208496,
      "activations/layer2_attention_weight_min": -25.217029571533203,
      "activations/layer3_attention_weight_max": 54.13482666015625,
      "activations/layer3_attention_weight_min": -53.81560516357422,
      "activations/layer4_attention_weight_max": 83.42915344238281,
      "activations/layer4_attention_weight_min": -74.98116302490234,
      "activations/layer5_attention_weight_max": 127.00127410888672,
      "activations/layer5_attention_weight_min": -122.56700134277344,
      "activations/layer6_attention_weight_max": 63.73762893676758,
      "activations/layer6_attention_weight_min": -58.28325271606445,
      "activations/layer7_attention_weight_max": 60.502113342285156,
      "activations/layer7_attention_weight_min": -57.73924255371094,
      "activations/layer8_attention_weight_max": 54.384220123291016,
      "activations/layer8_attention_weight_min": -51.46940994262695,
      "activations/layer9_attention_weight_max": 64.8011703491211,
      "activations/layer9_attention_weight_min": -73.74917602539062,
      "epoch": 1.03,
      "learning_rate": 0.00014480378787878786,
      "loss": 3.0765,
      "step": 17750
    },
    {
      "activations/layer0_attention_weight_max": 13.929530143737793,
      "activations/layer0_attention_weight_min": -10.97981071472168,
      "activations/layer10_attention_weight_max": 73.73030090332031,
      "activations/layer10_attention_weight_min": -63.70942306518555,
      "activations/layer11_attention_weight_max": 52.340938568115234,
      "activations/layer11_attention_weight_min": -41.510833740234375,
      "activations/layer12_attention_weight_max": 29.56987762451172,
      "activations/layer12_attention_weight_min": -24.222898483276367,
      "activations/layer13_attention_weight_max": 69.24999237060547,
      "activations/layer13_attention_weight_min": -46.99068069458008,
      "activations/layer14_attention_weight_max": 57.11051559448242,
      "activations/layer14_attention_weight_min": -35.805572509765625,
      "activations/layer15_attention_weight_max": 60.973472595214844,
      "activations/layer15_attention_weight_min": -38.352020263671875,
      "activations/layer16_attention_weight_max": 47.857940673828125,
      "activations/layer16_attention_weight_min": -33.30743408203125,
      "activations/layer17_attention_weight_max": 61.64905548095703,
      "activations/layer17_attention_weight_min": -36.948089599609375,
      "activations/layer18_attention_weight_max": 77.58690643310547,
      "activations/layer18_attention_weight_min": -38.00873565673828,
      "activations/layer19_attention_weight_max": 25.710012435913086,
      "activations/layer19_attention_weight_min": -13.180871963500977,
      "activations/layer1_attention_weight_max": 14.467991828918457,
      "activations/layer1_attention_weight_min": -11.94566535949707,
      "activations/layer20_attention_weight_max": 20.15184211730957,
      "activations/layer20_attention_weight_min": -14.920675277709961,
      "activations/layer21_attention_weight_max": 33.000213623046875,
      "activations/layer21_attention_weight_min": -19.686336517333984,
      "activations/layer22_attention_weight_max": 34.7363395690918,
      "activations/layer22_attention_weight_min": -16.893131256103516,
      "activations/layer23_attention_weight_max": 26.42412567138672,
      "activations/layer23_attention_weight_min": -14.71817684173584,
      "activations/layer2_attention_weight_max": 26.998558044433594,
      "activations/layer2_attention_weight_min": -24.570341110229492,
      "activations/layer3_attention_weight_max": 52.75351333618164,
      "activations/layer3_attention_weight_min": -51.10078048706055,
      "activations/layer4_attention_weight_max": 80.37897491455078,
      "activations/layer4_attention_weight_min": -69.84955596923828,
      "activations/layer5_attention_weight_max": 133.02642822265625,
      "activations/layer5_attention_weight_min": -126.24858856201172,
      "activations/layer6_attention_weight_max": 64.85774230957031,
      "activations/layer6_attention_weight_min": -59.076053619384766,
      "activations/layer7_attention_weight_max": 67.73168182373047,
      "activations/layer7_attention_weight_min": -58.32149887084961,
      "activations/layer8_attention_weight_max": 62.44211959838867,
      "activations/layer8_attention_weight_min": -50.31768798828125,
      "activations/layer9_attention_weight_max": 72.88753509521484,
      "activations/layer9_attention_weight_min": -67.61405944824219,
      "epoch": 1.03,
      "learning_rate": 0.00014478484848484848,
      "loss": 3.087,
      "step": 17800
    },
    {
      "activations/layer0_attention_weight_max": 12.94706916809082,
      "activations/layer0_attention_weight_min": -11.475299835205078,
      "activations/layer10_attention_weight_max": 63.54034423828125,
      "activations/layer10_attention_weight_min": -59.45255661010742,
      "activations/layer11_attention_weight_max": 44.53660583496094,
      "activations/layer11_attention_weight_min": -40.91355514526367,
      "activations/layer12_attention_weight_max": 27.29619789123535,
      "activations/layer12_attention_weight_min": -22.65658187866211,
      "activations/layer13_attention_weight_max": 60.10785675048828,
      "activations/layer13_attention_weight_min": -43.29982376098633,
      "activations/layer14_attention_weight_max": 53.351348876953125,
      "activations/layer14_attention_weight_min": -37.65898513793945,
      "activations/layer15_attention_weight_max": 50.19628143310547,
      "activations/layer15_attention_weight_min": -38.86629104614258,
      "activations/layer16_attention_weight_max": 51.88917541503906,
      "activations/layer16_attention_weight_min": -33.70343780517578,
      "activations/layer17_attention_weight_max": 56.09526824951172,
      "activations/layer17_attention_weight_min": -36.54991149902344,
      "activations/layer18_attention_weight_max": 67.56455993652344,
      "activations/layer18_attention_weight_min": -44.557342529296875,
      "activations/layer19_attention_weight_max": 20.349157333374023,
      "activations/layer19_attention_weight_min": -12.591367721557617,
      "activations/layer1_attention_weight_max": 14.666421890258789,
      "activations/layer1_attention_weight_min": -11.823821067810059,
      "activations/layer20_attention_weight_max": 17.84836769104004,
      "activations/layer20_attention_weight_min": -13.978170394897461,
      "activations/layer21_attention_weight_max": 32.633697509765625,
      "activations/layer21_attention_weight_min": -18.344074249267578,
      "activations/layer22_attention_weight_max": 30.30362319946289,
      "activations/layer22_attention_weight_min": -16.31521987915039,
      "activations/layer23_attention_weight_max": 20.79463005065918,
      "activations/layer23_attention_weight_min": -15.617342948913574,
      "activations/layer2_attention_weight_max": 24.790176391601562,
      "activations/layer2_attention_weight_min": -23.849380493164062,
      "activations/layer3_attention_weight_max": 53.860206604003906,
      "activations/layer3_attention_weight_min": -55.78282928466797,
      "activations/layer4_attention_weight_max": 81.94872283935547,
      "activations/layer4_attention_weight_min": -77.06021881103516,
      "activations/layer5_attention_weight_max": 128.64569091796875,
      "activations/layer5_attention_weight_min": -127.31979370117188,
      "activations/layer6_attention_weight_max": 63.03099822998047,
      "activations/layer6_attention_weight_min": -59.68449783325195,
      "activations/layer7_attention_weight_max": 62.288970947265625,
      "activations/layer7_attention_weight_min": -56.183536529541016,
      "activations/layer8_attention_weight_max": 51.27470397949219,
      "activations/layer8_attention_weight_min": -51.09829330444336,
      "activations/layer9_attention_weight_max": 70.51177978515625,
      "activations/layer9_attention_weight_min": -65.55113983154297,
      "epoch": 1.04,
      "learning_rate": 0.0001447659090909091,
      "loss": 3.0803,
      "step": 17850
    },
    {
      "activations/layer0_attention_weight_max": 13.094376564025879,
      "activations/layer0_attention_weight_min": -11.6092529296875,
      "activations/layer10_attention_weight_max": 64.30615997314453,
      "activations/layer10_attention_weight_min": -57.76927185058594,
      "activations/layer11_attention_weight_max": 48.10289764404297,
      "activations/layer11_attention_weight_min": -44.1128044128418,
      "activations/layer12_attention_weight_max": 28.53205680847168,
      "activations/layer12_attention_weight_min": -25.211572647094727,
      "activations/layer13_attention_weight_max": 50.82496643066406,
      "activations/layer13_attention_weight_min": -42.76738739013672,
      "activations/layer14_attention_weight_max": 39.7304801940918,
      "activations/layer14_attention_weight_min": -39.22722244262695,
      "activations/layer15_attention_weight_max": 61.07673645019531,
      "activations/layer15_attention_weight_min": -47.4235954284668,
      "activations/layer16_attention_weight_max": 45.300235748291016,
      "activations/layer16_attention_weight_min": -36.767024993896484,
      "activations/layer17_attention_weight_max": 52.23610305786133,
      "activations/layer17_attention_weight_min": -40.07344436645508,
      "activations/layer18_attention_weight_max": 55.479774475097656,
      "activations/layer18_attention_weight_min": -36.6766471862793,
      "activations/layer19_attention_weight_max": 18.33789825439453,
      "activations/layer19_attention_weight_min": -12.494959831237793,
      "activations/layer1_attention_weight_max": 16.053667068481445,
      "activations/layer1_attention_weight_min": -10.795294761657715,
      "activations/layer20_attention_weight_max": 17.444791793823242,
      "activations/layer20_attention_weight_min": -13.124369621276855,
      "activations/layer21_attention_weight_max": 33.48802947998047,
      "activations/layer21_attention_weight_min": -18.842958450317383,
      "activations/layer22_attention_weight_max": 30.125825881958008,
      "activations/layer22_attention_weight_min": -19.0582218170166,
      "activations/layer23_attention_weight_max": 22.004764556884766,
      "activations/layer23_attention_weight_min": -16.532373428344727,
      "activations/layer2_attention_weight_max": 24.50874137878418,
      "activations/layer2_attention_weight_min": -23.650341033935547,
      "activations/layer3_attention_weight_max": 53.03681564331055,
      "activations/layer3_attention_weight_min": -54.45831298828125,
      "activations/layer4_attention_weight_max": 79.60411834716797,
      "activations/layer4_attention_weight_min": -76.29331970214844,
      "activations/layer5_attention_weight_max": 131.02743530273438,
      "activations/layer5_attention_weight_min": -134.66433715820312,
      "activations/layer6_attention_weight_max": 61.6619987487793,
      "activations/layer6_attention_weight_min": -59.74675369262695,
      "activations/layer7_attention_weight_max": 59.05635452270508,
      "activations/layer7_attention_weight_min": -55.6894416809082,
      "activations/layer8_attention_weight_max": 51.609649658203125,
      "activations/layer8_attention_weight_min": -48.75987243652344,
      "activations/layer9_attention_weight_max": 69.07198333740234,
      "activations/layer9_attention_weight_min": -61.646095275878906,
      "epoch": 1.04,
      "learning_rate": 0.00014474696969696968,
      "loss": 3.0992,
      "step": 17900
    },
    {
      "activations/layer0_attention_weight_max": 13.389322280883789,
      "activations/layer0_attention_weight_min": -11.542295455932617,
      "activations/layer10_attention_weight_max": 64.30360412597656,
      "activations/layer10_attention_weight_min": -57.744720458984375,
      "activations/layer11_attention_weight_max": 41.46913528442383,
      "activations/layer11_attention_weight_min": -39.75188064575195,
      "activations/layer12_attention_weight_max": 26.174596786499023,
      "activations/layer12_attention_weight_min": -24.933570861816406,
      "activations/layer13_attention_weight_max": 49.16990280151367,
      "activations/layer13_attention_weight_min": -43.74803161621094,
      "activations/layer14_attention_weight_max": 42.91606521606445,
      "activations/layer14_attention_weight_min": -39.831912994384766,
      "activations/layer15_attention_weight_max": 44.108856201171875,
      "activations/layer15_attention_weight_min": -39.33222961425781,
      "activations/layer16_attention_weight_max": 44.668121337890625,
      "activations/layer16_attention_weight_min": -35.58258819580078,
      "activations/layer17_attention_weight_max": 50.27033615112305,
      "activations/layer17_attention_weight_min": -38.591552734375,
      "activations/layer18_attention_weight_max": 61.5925178527832,
      "activations/layer18_attention_weight_min": -39.1384162902832,
      "activations/layer19_attention_weight_max": 21.993480682373047,
      "activations/layer19_attention_weight_min": -13.437049865722656,
      "activations/layer1_attention_weight_max": 15.439766883850098,
      "activations/layer1_attention_weight_min": -10.940914154052734,
      "activations/layer20_attention_weight_max": 19.66712188720703,
      "activations/layer20_attention_weight_min": -16.156564712524414,
      "activations/layer21_attention_weight_max": 36.08663558959961,
      "activations/layer21_attention_weight_min": -22.098947525024414,
      "activations/layer22_attention_weight_max": 33.03186798095703,
      "activations/layer22_attention_weight_min": -21.921720504760742,
      "activations/layer23_attention_weight_max": 24.035552978515625,
      "activations/layer23_attention_weight_min": -17.400390625,
      "activations/layer2_attention_weight_max": 26.227506637573242,
      "activations/layer2_attention_weight_min": -23.729698181152344,
      "activations/layer3_attention_weight_max": 50.15590286254883,
      "activations/layer3_attention_weight_min": -52.97525405883789,
      "activations/layer4_attention_weight_max": 77.4124526977539,
      "activations/layer4_attention_weight_min": -71.42668914794922,
      "activations/layer5_attention_weight_max": 122.43782043457031,
      "activations/layer5_attention_weight_min": -120.66342163085938,
      "activations/layer6_attention_weight_max": 58.97028732299805,
      "activations/layer6_attention_weight_min": -58.3796501159668,
      "activations/layer7_attention_weight_max": 62.05031204223633,
      "activations/layer7_attention_weight_min": -56.89789962768555,
      "activations/layer8_attention_weight_max": 52.18567657470703,
      "activations/layer8_attention_weight_min": -45.374324798583984,
      "activations/layer9_attention_weight_max": 63.72697830200195,
      "activations/layer9_attention_weight_min": -67.64336395263672,
      "epoch": 1.04,
      "learning_rate": 0.0001447280303030303,
      "loss": 3.0767,
      "step": 17950
    },
    {
      "activations/layer0_attention_weight_max": 14.118196487426758,
      "activations/layer0_attention_weight_min": -11.118891716003418,
      "activations/layer10_attention_weight_max": 76.0178451538086,
      "activations/layer10_attention_weight_min": -58.32364273071289,
      "activations/layer11_attention_weight_max": 47.816131591796875,
      "activations/layer11_attention_weight_min": -41.64290237426758,
      "activations/layer12_attention_weight_max": 29.42218589782715,
      "activations/layer12_attention_weight_min": -22.822940826416016,
      "activations/layer13_attention_weight_max": 64.25541687011719,
      "activations/layer13_attention_weight_min": -50.08167266845703,
      "activations/layer14_attention_weight_max": 55.019439697265625,
      "activations/layer14_attention_weight_min": -40.19636917114258,
      "activations/layer15_attention_weight_max": 57.131954193115234,
      "activations/layer15_attention_weight_min": -38.777103424072266,
      "activations/layer16_attention_weight_max": 50.879207611083984,
      "activations/layer16_attention_weight_min": -35.5501823425293,
      "activations/layer17_attention_weight_max": 70.30879974365234,
      "activations/layer17_attention_weight_min": -41.304405212402344,
      "activations/layer18_attention_weight_max": 68.34349822998047,
      "activations/layer18_attention_weight_min": -38.18996810913086,
      "activations/layer19_attention_weight_max": 20.931520462036133,
      "activations/layer19_attention_weight_min": -12.701143264770508,
      "activations/layer1_attention_weight_max": 15.66867733001709,
      "activations/layer1_attention_weight_min": -10.671936988830566,
      "activations/layer20_attention_weight_max": 18.722759246826172,
      "activations/layer20_attention_weight_min": -14.740463256835938,
      "activations/layer21_attention_weight_max": 42.84541702270508,
      "activations/layer21_attention_weight_min": -19.710844039916992,
      "activations/layer22_attention_weight_max": 36.161800384521484,
      "activations/layer22_attention_weight_min": -16.624385833740234,
      "activations/layer23_attention_weight_max": 23.411792755126953,
      "activations/layer23_attention_weight_min": -15.796714782714844,
      "activations/layer2_attention_weight_max": 25.241498947143555,
      "activations/layer2_attention_weight_min": -22.622867584228516,
      "activations/layer3_attention_weight_max": 51.71153259277344,
      "activations/layer3_attention_weight_min": -51.15692901611328,
      "activations/layer4_attention_weight_max": 80.63020324707031,
      "activations/layer4_attention_weight_min": -73.86884307861328,
      "activations/layer5_attention_weight_max": 135.45941162109375,
      "activations/layer5_attention_weight_min": -119.91104125976562,
      "activations/layer6_attention_weight_max": 63.27465057373047,
      "activations/layer6_attention_weight_min": -56.66822814941406,
      "activations/layer7_attention_weight_max": 65.8282241821289,
      "activations/layer7_attention_weight_min": -58.448577880859375,
      "activations/layer8_attention_weight_max": 57.58051300048828,
      "activations/layer8_attention_weight_min": -49.92093276977539,
      "activations/layer9_attention_weight_max": 67.43895721435547,
      "activations/layer9_attention_weight_min": -62.962127685546875,
      "epoch": 1.05,
      "learning_rate": 0.0001447090909090909,
      "loss": 3.0802,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_loss": 3.033203125,
      "eval_runtime": 8.4871,
      "eval_samples_per_second": 505.947,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_openwebtext_loss": 3.033203125,
      "eval_openwebtext_ppl": 20.763634702888442,
      "eval_openwebtext_runtime": 8.4871,
      "eval_openwebtext_samples_per_second": 505.947,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_wikitext_loss": 3.326171875,
      "eval_wikitext_ppl": 27.83159468972004,
      "eval_wikitext_runtime": 1.7116,
      "eval_wikitext_samples_per_second": 266.414,
      "step": 18000
    },
    {
      "epoch": 1.05,
      "eval_lambada_loss": 3.43359375,
      "eval_lambada_ppl": 30.987805310145493,
      "eval_lambada_runtime": 9.2819,
      "eval_lambada_samples_per_second": 524.567,
      "step": 18000
    },
    {
      "activations/layer0_attention_weight_max": 15.237883567810059,
      "activations/layer0_attention_weight_min": -11.238075256347656,
      "activations/layer10_attention_weight_max": 77.6124267578125,
      "activations/layer10_attention_weight_min": -63.181724548339844,
      "activations/layer11_attention_weight_max": 50.24226379394531,
      "activations/layer11_attention_weight_min": -39.097225189208984,
      "activations/layer12_attention_weight_max": 31.35312271118164,
      "activations/layer12_attention_weight_min": -24.68838882446289,
      "activations/layer13_attention_weight_max": 64.33159637451172,
      "activations/layer13_attention_weight_min": -48.17045593261719,
      "activations/layer14_attention_weight_max": 56.1990852355957,
      "activations/layer14_attention_weight_min": -41.6788330078125,
      "activations/layer15_attention_weight_max": 56.47981643676758,
      "activations/layer15_attention_weight_min": -45.48896789550781,
      "activations/layer16_attention_weight_max": 49.74418258666992,
      "activations/layer16_attention_weight_min": -35.47792053222656,
      "activations/layer17_attention_weight_max": 62.58671951293945,
      "activations/layer17_attention_weight_min": -39.394405364990234,
      "activations/layer18_attention_weight_max": 67.09747314453125,
      "activations/layer18_attention_weight_min": -36.62298583984375,
      "activations/layer19_attention_weight_max": 21.05414581298828,
      "activations/layer19_attention_weight_min": -13.827400207519531,
      "activations/layer1_attention_weight_max": 16.67801284790039,
      "activations/layer1_attention_weight_min": -10.682388305664062,
      "activations/layer20_attention_weight_max": 19.841535568237305,
      "activations/layer20_attention_weight_min": -15.553750038146973,
      "activations/layer21_attention_weight_max": 35.400299072265625,
      "activations/layer21_attention_weight_min": -20.078100204467773,
      "activations/layer22_attention_weight_max": 31.203372955322266,
      "activations/layer22_attention_weight_min": -18.920900344848633,
      "activations/layer23_attention_weight_max": 26.157943725585938,
      "activations/layer23_attention_weight_min": -14.714865684509277,
      "activations/layer2_attention_weight_max": 26.173152923583984,
      "activations/layer2_attention_weight_min": -23.009811401367188,
      "activations/layer3_attention_weight_max": 50.28610610961914,
      "activations/layer3_attention_weight_min": -54.02843475341797,
      "activations/layer4_attention_weight_max": 78.74517059326172,
      "activations/layer4_attention_weight_min": -76.57988739013672,
      "activations/layer5_attention_weight_max": 132.2183380126953,
      "activations/layer5_attention_weight_min": -118.82931518554688,
      "activations/layer6_attention_weight_max": 66.97555541992188,
      "activations/layer6_attention_weight_min": -58.22843551635742,
      "activations/layer7_attention_weight_max": 65.70491027832031,
      "activations/layer7_attention_weight_min": -56.55141830444336,
      "activations/layer8_attention_weight_max": 58.3420295715332,
      "activations/layer8_attention_weight_min": -49.71390151977539,
      "activations/layer9_attention_weight_max": 74.52055358886719,
      "activations/layer9_attention_weight_min": -71.05585479736328,
      "epoch": 1.05,
      "learning_rate": 0.0001446901515151515,
      "loss": 3.0883,
      "step": 18050
    },
    {
      "activations/layer0_attention_weight_max": 14.851613998413086,
      "activations/layer0_attention_weight_min": -11.861371994018555,
      "activations/layer10_attention_weight_max": 69.15518951416016,
      "activations/layer10_attention_weight_min": -57.161373138427734,
      "activations/layer11_attention_weight_max": 50.20459747314453,
      "activations/layer11_attention_weight_min": -38.64897155761719,
      "activations/layer12_attention_weight_max": 31.868549346923828,
      "activations/layer12_attention_weight_min": -26.339017868041992,
      "activations/layer13_attention_weight_max": 75.40716552734375,
      "activations/layer13_attention_weight_min": -48.65039825439453,
      "activations/layer14_attention_weight_max": 78.10069274902344,
      "activations/layer14_attention_weight_min": -45.69068145751953,
      "activations/layer15_attention_weight_max": 60.77260208129883,
      "activations/layer15_attention_weight_min": -40.197696685791016,
      "activations/layer16_attention_weight_max": 53.93381881713867,
      "activations/layer16_attention_weight_min": -38.51438522338867,
      "activations/layer17_attention_weight_max": 60.07364273071289,
      "activations/layer17_attention_weight_min": -39.46082305908203,
      "activations/layer18_attention_weight_max": 69.21720886230469,
      "activations/layer18_attention_weight_min": -42.725040435791016,
      "activations/layer19_attention_weight_max": 21.315534591674805,
      "activations/layer19_attention_weight_min": -13.113032341003418,
      "activations/layer1_attention_weight_max": 15.238736152648926,
      "activations/layer1_attention_weight_min": -11.717629432678223,
      "activations/layer20_attention_weight_max": 18.97939682006836,
      "activations/layer20_attention_weight_min": -14.16610050201416,
      "activations/layer21_attention_weight_max": 38.566734313964844,
      "activations/layer21_attention_weight_min": -20.22226905822754,
      "activations/layer22_attention_weight_max": 33.414241790771484,
      "activations/layer22_attention_weight_min": -17.14521026611328,
      "activations/layer23_attention_weight_max": 24.61948013305664,
      "activations/layer23_attention_weight_min": -14.956771850585938,
      "activations/layer2_attention_weight_max": 28.353126525878906,
      "activations/layer2_attention_weight_min": -23.67589569091797,
      "activations/layer3_attention_weight_max": 54.56652069091797,
      "activations/layer3_attention_weight_min": -54.1928596496582,
      "activations/layer4_attention_weight_max": 84.7135238647461,
      "activations/layer4_attention_weight_min": -77.47579193115234,
      "activations/layer5_attention_weight_max": 141.331787109375,
      "activations/layer5_attention_weight_min": -115.73692321777344,
      "activations/layer6_attention_weight_max": 66.35408020019531,
      "activations/layer6_attention_weight_min": -56.62076187133789,
      "activations/layer7_attention_weight_max": 68.614990234375,
      "activations/layer7_attention_weight_min": -55.50312042236328,
      "activations/layer8_attention_weight_max": 59.00485610961914,
      "activations/layer8_attention_weight_min": -48.82398223876953,
      "activations/layer9_attention_weight_max": 73.64280700683594,
      "activations/layer9_attention_weight_min": -71.0929183959961,
      "epoch": 1.05,
      "learning_rate": 0.0001446712121212121,
      "loss": 3.07,
      "step": 18100
    },
    {
      "activations/layer0_attention_weight_max": 13.36445426940918,
      "activations/layer0_attention_weight_min": -11.583245277404785,
      "activations/layer10_attention_weight_max": 66.36673736572266,
      "activations/layer10_attention_weight_min": -57.809173583984375,
      "activations/layer11_attention_weight_max": 44.78367233276367,
      "activations/layer11_attention_weight_min": -40.1650390625,
      "activations/layer12_attention_weight_max": 26.81705665588379,
      "activations/layer12_attention_weight_min": -23.453020095825195,
      "activations/layer13_attention_weight_max": 77.62772369384766,
      "activations/layer13_attention_weight_min": -52.188209533691406,
      "activations/layer14_attention_weight_max": 71.7694320678711,
      "activations/layer14_attention_weight_min": -47.741615295410156,
      "activations/layer15_attention_weight_max": 55.235313415527344,
      "activations/layer15_attention_weight_min": -38.76148223876953,
      "activations/layer16_attention_weight_max": 43.62805938720703,
      "activations/layer16_attention_weight_min": -35.14168167114258,
      "activations/layer17_attention_weight_max": 56.45640182495117,
      "activations/layer17_attention_weight_min": -36.129764556884766,
      "activations/layer18_attention_weight_max": 71.02063751220703,
      "activations/layer18_attention_weight_min": -45.20415496826172,
      "activations/layer19_attention_weight_max": 21.712236404418945,
      "activations/layer19_attention_weight_min": -13.259597778320312,
      "activations/layer1_attention_weight_max": 14.871346473693848,
      "activations/layer1_attention_weight_min": -10.21540641784668,
      "activations/layer20_attention_weight_max": 18.983314514160156,
      "activations/layer20_attention_weight_min": -15.10705280303955,
      "activations/layer21_attention_weight_max": 40.22922134399414,
      "activations/layer21_attention_weight_min": -20.191646575927734,
      "activations/layer22_attention_weight_max": 32.603458404541016,
      "activations/layer22_attention_weight_min": -16.186439514160156,
      "activations/layer23_attention_weight_max": 23.68093490600586,
      "activations/layer23_attention_weight_min": -15.045581817626953,
      "activations/layer2_attention_weight_max": 23.472309112548828,
      "activations/layer2_attention_weight_min": -22.683774948120117,
      "activations/layer3_attention_weight_max": 53.72290802001953,
      "activations/layer3_attention_weight_min": -54.42295455932617,
      "activations/layer4_attention_weight_max": 83.59917449951172,
      "activations/layer4_attention_weight_min": -76.02245330810547,
      "activations/layer5_attention_weight_max": 137.75765991210938,
      "activations/layer5_attention_weight_min": -122.44674682617188,
      "activations/layer6_attention_weight_max": 66.40518951416016,
      "activations/layer6_attention_weight_min": -57.85629653930664,
      "activations/layer7_attention_weight_max": 66.28164672851562,
      "activations/layer7_attention_weight_min": -58.12001037597656,
      "activations/layer8_attention_weight_max": 61.57854461669922,
      "activations/layer8_attention_weight_min": -49.412227630615234,
      "activations/layer9_attention_weight_max": 88.09712982177734,
      "activations/layer9_attention_weight_min": -65.57215881347656,
      "epoch": 1.05,
      "learning_rate": 0.00014465227272727273,
      "loss": 3.0733,
      "step": 18150
    },
    {
      "activations/layer0_attention_weight_max": 13.393829345703125,
      "activations/layer0_attention_weight_min": -11.343883514404297,
      "activations/layer10_attention_weight_max": 63.43018341064453,
      "activations/layer10_attention_weight_min": -62.02439498901367,
      "activations/layer11_attention_weight_max": 45.19696807861328,
      "activations/layer11_attention_weight_min": -43.4212646484375,
      "activations/layer12_attention_weight_max": 26.070785522460938,
      "activations/layer12_attention_weight_min": -24.853168487548828,
      "activations/layer13_attention_weight_max": 53.13103103637695,
      "activations/layer13_attention_weight_min": -46.35717010498047,
      "activations/layer14_attention_weight_max": 48.18885040283203,
      "activations/layer14_attention_weight_min": -44.493385314941406,
      "activations/layer15_attention_weight_max": 48.03960037231445,
      "activations/layer15_attention_weight_min": -39.304771423339844,
      "activations/layer16_attention_weight_max": 45.02576446533203,
      "activations/layer16_attention_weight_min": -37.030757904052734,
      "activations/layer17_attention_weight_max": 53.96940612792969,
      "activations/layer17_attention_weight_min": -40.38899230957031,
      "activations/layer18_attention_weight_max": 64.94412994384766,
      "activations/layer18_attention_weight_min": -38.31284713745117,
      "activations/layer19_attention_weight_max": 18.067800521850586,
      "activations/layer19_attention_weight_min": -13.478227615356445,
      "activations/layer1_attention_weight_max": 15.129419326782227,
      "activations/layer1_attention_weight_min": -12.124496459960938,
      "activations/layer20_attention_weight_max": 16.711223602294922,
      "activations/layer20_attention_weight_min": -14.003100395202637,
      "activations/layer21_attention_weight_max": 35.81737518310547,
      "activations/layer21_attention_weight_min": -20.274124145507812,
      "activations/layer22_attention_weight_max": 30.910737991333008,
      "activations/layer22_attention_weight_min": -17.23626136779785,
      "activations/layer23_attention_weight_max": 21.76594352722168,
      "activations/layer23_attention_weight_min": -16.257858276367188,
      "activations/layer2_attention_weight_max": 26.398099899291992,
      "activations/layer2_attention_weight_min": -23.791980743408203,
      "activations/layer3_attention_weight_max": 50.663692474365234,
      "activations/layer3_attention_weight_min": -51.374786376953125,
      "activations/layer4_attention_weight_max": 76.65843963623047,
      "activations/layer4_attention_weight_min": -73.68404388427734,
      "activations/layer5_attention_weight_max": 127.39773559570312,
      "activations/layer5_attention_weight_min": -122.62942504882812,
      "activations/layer6_attention_weight_max": 62.651309967041016,
      "activations/layer6_attention_weight_min": -60.503082275390625,
      "activations/layer7_attention_weight_max": 66.83983612060547,
      "activations/layer7_attention_weight_min": -63.491214752197266,
      "activations/layer8_attention_weight_max": 54.69493865966797,
      "activations/layer8_attention_weight_min": -55.5302734375,
      "activations/layer9_attention_weight_max": 67.78104400634766,
      "activations/layer9_attention_weight_min": -72.7198257446289,
      "epoch": 1.06,
      "learning_rate": 0.00014463333333333332,
      "loss": 3.0778,
      "step": 18200
    },
    {
      "activations/layer0_attention_weight_max": 13.24133014678955,
      "activations/layer0_attention_weight_min": -11.504420280456543,
      "activations/layer10_attention_weight_max": 63.883052825927734,
      "activations/layer10_attention_weight_min": -55.66946792602539,
      "activations/layer11_attention_weight_max": 45.29559326171875,
      "activations/layer11_attention_weight_min": -38.314430236816406,
      "activations/layer12_attention_weight_max": 26.47411346435547,
      "activations/layer12_attention_weight_min": -24.21101188659668,
      "activations/layer13_attention_weight_max": 62.2830810546875,
      "activations/layer13_attention_weight_min": -45.60540008544922,
      "activations/layer14_attention_weight_max": 71.87849426269531,
      "activations/layer14_attention_weight_min": -47.249332427978516,
      "activations/layer15_attention_weight_max": 54.06958770751953,
      "activations/layer15_attention_weight_min": -44.255184173583984,
      "activations/layer16_attention_weight_max": 50.36130142211914,
      "activations/layer16_attention_weight_min": -31.873767852783203,
      "activations/layer17_attention_weight_max": 53.8683967590332,
      "activations/layer17_attention_weight_min": -35.523406982421875,
      "activations/layer18_attention_weight_max": 61.43098068237305,
      "activations/layer18_attention_weight_min": -39.39230728149414,
      "activations/layer19_attention_weight_max": 20.371456146240234,
      "activations/layer19_attention_weight_min": -13.89621639251709,
      "activations/layer1_attention_weight_max": 14.966850280761719,
      "activations/layer1_attention_weight_min": -10.92491626739502,
      "activations/layer20_attention_weight_max": 20.471336364746094,
      "activations/layer20_attention_weight_min": -12.978554725646973,
      "activations/layer21_attention_weight_max": 36.33210754394531,
      "activations/layer21_attention_weight_min": -20.153778076171875,
      "activations/layer22_attention_weight_max": 35.50815963745117,
      "activations/layer22_attention_weight_min": -17.401996612548828,
      "activations/layer23_attention_weight_max": 24.295650482177734,
      "activations/layer23_attention_weight_min": -16.334823608398438,
      "activations/layer2_attention_weight_max": 24.326862335205078,
      "activations/layer2_attention_weight_min": -22.462615966796875,
      "activations/layer3_attention_weight_max": 49.670711517333984,
      "activations/layer3_attention_weight_min": -51.41216278076172,
      "activations/layer4_attention_weight_max": 73.78499603271484,
      "activations/layer4_attention_weight_min": -73.1661605834961,
      "activations/layer5_attention_weight_max": 120.74751281738281,
      "activations/layer5_attention_weight_min": -121.12706756591797,
      "activations/layer6_attention_weight_max": 58.93709182739258,
      "activations/layer6_attention_weight_min": -57.12187194824219,
      "activations/layer7_attention_weight_max": 58.17641067504883,
      "activations/layer7_attention_weight_min": -60.735252380371094,
      "activations/layer8_attention_weight_max": 51.33683776855469,
      "activations/layer8_attention_weight_min": -48.72927474975586,
      "activations/layer9_attention_weight_max": 64.60206604003906,
      "activations/layer9_attention_weight_min": -63.74982833862305,
      "epoch": 1.06,
      "learning_rate": 0.00014461439393939393,
      "loss": 3.0815,
      "step": 18250
    },
    {
      "activations/layer0_attention_weight_max": 13.614910125732422,
      "activations/layer0_attention_weight_min": -11.234126091003418,
      "activations/layer10_attention_weight_max": 63.29141616821289,
      "activations/layer10_attention_weight_min": -57.89525604248047,
      "activations/layer11_attention_weight_max": 43.66783142089844,
      "activations/layer11_attention_weight_min": -48.80310821533203,
      "activations/layer12_attention_weight_max": 27.59097671508789,
      "activations/layer12_attention_weight_min": -24.700571060180664,
      "activations/layer13_attention_weight_max": 54.43927764892578,
      "activations/layer13_attention_weight_min": -50.51055145263672,
      "activations/layer14_attention_weight_max": 47.000675201416016,
      "activations/layer14_attention_weight_min": -34.9752197265625,
      "activations/layer15_attention_weight_max": 54.32754898071289,
      "activations/layer15_attention_weight_min": -37.84026336669922,
      "activations/layer16_attention_weight_max": 46.18132400512695,
      "activations/layer16_attention_weight_min": -33.058860778808594,
      "activations/layer17_attention_weight_max": 53.658103942871094,
      "activations/layer17_attention_weight_min": -37.26173782348633,
      "activations/layer18_attention_weight_max": 54.90491485595703,
      "activations/layer18_attention_weight_min": -40.01704406738281,
      "activations/layer19_attention_weight_max": 18.91126823425293,
      "activations/layer19_attention_weight_min": -12.86091136932373,
      "activations/layer1_attention_weight_max": 15.182245254516602,
      "activations/layer1_attention_weight_min": -10.71241283416748,
      "activations/layer20_attention_weight_max": 16.567520141601562,
      "activations/layer20_attention_weight_min": -14.81062126159668,
      "activations/layer21_attention_weight_max": 31.43151092529297,
      "activations/layer21_attention_weight_min": -20.48046112060547,
      "activations/layer22_attention_weight_max": 29.1220760345459,
      "activations/layer22_attention_weight_min": -16.255815505981445,
      "activations/layer23_attention_weight_max": 20.11322784423828,
      "activations/layer23_attention_weight_min": -15.389179229736328,
      "activations/layer2_attention_weight_max": 25.50335693359375,
      "activations/layer2_attention_weight_min": -23.67477798461914,
      "activations/layer3_attention_weight_max": 51.89147186279297,
      "activations/layer3_attention_weight_min": -51.68244171142578,
      "activations/layer4_attention_weight_max": 81.41520690917969,
      "activations/layer4_attention_weight_min": -75.77445220947266,
      "activations/layer5_attention_weight_max": 137.76454162597656,
      "activations/layer5_attention_weight_min": -123.207275390625,
      "activations/layer6_attention_weight_max": 61.17380142211914,
      "activations/layer6_attention_weight_min": -57.16556167602539,
      "activations/layer7_attention_weight_max": 63.058692932128906,
      "activations/layer7_attention_weight_min": -56.4630241394043,
      "activations/layer8_attention_weight_max": 51.68678283691406,
      "activations/layer8_attention_weight_min": -54.47350311279297,
      "activations/layer9_attention_weight_max": 75.19767761230469,
      "activations/layer9_attention_weight_min": -79.82502746582031,
      "epoch": 1.06,
      "learning_rate": 0.00014459545454545452,
      "loss": 3.0698,
      "step": 18300
    },
    {
      "activations/layer0_attention_weight_max": 14.022284507751465,
      "activations/layer0_attention_weight_min": -11.285099983215332,
      "activations/layer10_attention_weight_max": 65.12677764892578,
      "activations/layer10_attention_weight_min": -63.00395965576172,
      "activations/layer11_attention_weight_max": 48.418212890625,
      "activations/layer11_attention_weight_min": -41.015907287597656,
      "activations/layer12_attention_weight_max": 29.932092666625977,
      "activations/layer12_attention_weight_min": -24.711780548095703,
      "activations/layer13_attention_weight_max": 61.55443572998047,
      "activations/layer13_attention_weight_min": -43.80541229248047,
      "activations/layer14_attention_weight_max": 44.411216735839844,
      "activations/layer14_attention_weight_min": -32.95317459106445,
      "activations/layer15_attention_weight_max": 51.71017074584961,
      "activations/layer15_attention_weight_min": -40.976932525634766,
      "activations/layer16_attention_weight_max": 42.4070930480957,
      "activations/layer16_attention_weight_min": -37.346290588378906,
      "activations/layer17_attention_weight_max": 52.23579406738281,
      "activations/layer17_attention_weight_min": -37.891944885253906,
      "activations/layer18_attention_weight_max": 62.69346237182617,
      "activations/layer18_attention_weight_min": -42.09199142456055,
      "activations/layer19_attention_weight_max": 18.159425735473633,
      "activations/layer19_attention_weight_min": -12.239182472229004,
      "activations/layer1_attention_weight_max": 14.703455924987793,
      "activations/layer1_attention_weight_min": -9.73633861541748,
      "activations/layer20_attention_weight_max": 17.933927536010742,
      "activations/layer20_attention_weight_min": -13.738691329956055,
      "activations/layer21_attention_weight_max": 32.482635498046875,
      "activations/layer21_attention_weight_min": -16.650531768798828,
      "activations/layer22_attention_weight_max": 30.03534507751465,
      "activations/layer22_attention_weight_min": -15.732285499572754,
      "activations/layer23_attention_weight_max": 20.11222267150879,
      "activations/layer23_attention_weight_min": -14.668609619140625,
      "activations/layer2_attention_weight_max": 23.450061798095703,
      "activations/layer2_attention_weight_min": -22.282283782958984,
      "activations/layer3_attention_weight_max": 52.4334602355957,
      "activations/layer3_attention_weight_min": -55.282405853271484,
      "activations/layer4_attention_weight_max": 81.7561264038086,
      "activations/layer4_attention_weight_min": -74.56775665283203,
      "activations/layer5_attention_weight_max": 135.12228393554688,
      "activations/layer5_attention_weight_min": -119.75009155273438,
      "activations/layer6_attention_weight_max": 65.24626159667969,
      "activations/layer6_attention_weight_min": -56.917236328125,
      "activations/layer7_attention_weight_max": 65.77503967285156,
      "activations/layer7_attention_weight_min": -58.47665023803711,
      "activations/layer8_attention_weight_max": 56.89345169067383,
      "activations/layer8_attention_weight_min": -51.85039138793945,
      "activations/layer9_attention_weight_max": 68.00519561767578,
      "activations/layer9_attention_weight_min": -65.60173797607422,
      "epoch": 1.07,
      "learning_rate": 0.00014457651515151513,
      "loss": 3.0928,
      "step": 18350
    },
    {
      "activations/layer0_attention_weight_max": 13.395149230957031,
      "activations/layer0_attention_weight_min": -11.16555404663086,
      "activations/layer10_attention_weight_max": 78.35343170166016,
      "activations/layer10_attention_weight_min": -65.93011474609375,
      "activations/layer11_attention_weight_max": 56.76520919799805,
      "activations/layer11_attention_weight_min": -41.38728332519531,
      "activations/layer12_attention_weight_max": 27.390098571777344,
      "activations/layer12_attention_weight_min": -23.65054702758789,
      "activations/layer13_attention_weight_max": 73.1968994140625,
      "activations/layer13_attention_weight_min": -44.02560806274414,
      "activations/layer14_attention_weight_max": 62.421531677246094,
      "activations/layer14_attention_weight_min": -34.79401397705078,
      "activations/layer15_attention_weight_max": 62.48075485229492,
      "activations/layer15_attention_weight_min": -38.98953628540039,
      "activations/layer16_attention_weight_max": 53.099849700927734,
      "activations/layer16_attention_weight_min": -33.9566650390625,
      "activations/layer17_attention_weight_max": 65.37362670898438,
      "activations/layer17_attention_weight_min": -37.894325256347656,
      "activations/layer18_attention_weight_max": 59.1988525390625,
      "activations/layer18_attention_weight_min": -38.706809997558594,
      "activations/layer19_attention_weight_max": 21.045211791992188,
      "activations/layer19_attention_weight_min": -14.07822322845459,
      "activations/layer1_attention_weight_max": 13.991958618164062,
      "activations/layer1_attention_weight_min": -10.205062866210938,
      "activations/layer20_attention_weight_max": 19.457918167114258,
      "activations/layer20_attention_weight_min": -13.825990676879883,
      "activations/layer21_attention_weight_max": 35.187313079833984,
      "activations/layer21_attention_weight_min": -17.83966064453125,
      "activations/layer22_attention_weight_max": 30.251665115356445,
      "activations/layer22_attention_weight_min": -16.62856101989746,
      "activations/layer23_attention_weight_max": 19.80887222290039,
      "activations/layer23_attention_weight_min": -15.545516967773438,
      "activations/layer2_attention_weight_max": 25.27956771850586,
      "activations/layer2_attention_weight_min": -23.401611328125,
      "activations/layer3_attention_weight_max": 56.32938003540039,
      "activations/layer3_attention_weight_min": -56.545631408691406,
      "activations/layer4_attention_weight_max": 93.88655853271484,
      "activations/layer4_attention_weight_min": -82.04978942871094,
      "activations/layer5_attention_weight_max": 160.92575073242188,
      "activations/layer5_attention_weight_min": -129.53372192382812,
      "activations/layer6_attention_weight_max": 77.65583801269531,
      "activations/layer6_attention_weight_min": -61.382720947265625,
      "activations/layer7_attention_weight_max": 82.06011962890625,
      "activations/layer7_attention_weight_min": -60.6317024230957,
      "activations/layer8_attention_weight_max": 70.68183898925781,
      "activations/layer8_attention_weight_min": -49.16314697265625,
      "activations/layer9_attention_weight_max": 85.88274383544922,
      "activations/layer9_attention_weight_min": -64.0848617553711,
      "epoch": 1.07,
      "learning_rate": 0.00014455757575757575,
      "loss": 3.0818,
      "step": 18400
    },
    {
      "activations/layer0_attention_weight_max": 13.029647827148438,
      "activations/layer0_attention_weight_min": -11.074044227600098,
      "activations/layer10_attention_weight_max": 72.34522247314453,
      "activations/layer10_attention_weight_min": -70.60670471191406,
      "activations/layer11_attention_weight_max": 47.868186950683594,
      "activations/layer11_attention_weight_min": -40.08061981201172,
      "activations/layer12_attention_weight_max": 28.312400817871094,
      "activations/layer12_attention_weight_min": -25.11066436767578,
      "activations/layer13_attention_weight_max": 77.40960693359375,
      "activations/layer13_attention_weight_min": -50.64219284057617,
      "activations/layer14_attention_weight_max": 63.40349197387695,
      "activations/layer14_attention_weight_min": -41.217018127441406,
      "activations/layer15_attention_weight_max": 59.03623580932617,
      "activations/layer15_attention_weight_min": -42.76448059082031,
      "activations/layer16_attention_weight_max": 57.05799865722656,
      "activations/layer16_attention_weight_min": -35.269405364990234,
      "activations/layer17_attention_weight_max": 56.7164421081543,
      "activations/layer17_attention_weight_min": -36.58845901489258,
      "activations/layer18_attention_weight_max": 80.39837646484375,
      "activations/layer18_attention_weight_min": -46.256797790527344,
      "activations/layer19_attention_weight_max": 24.471473693847656,
      "activations/layer19_attention_weight_min": -12.14078426361084,
      "activations/layer1_attention_weight_max": 14.515298843383789,
      "activations/layer1_attention_weight_min": -11.199019432067871,
      "activations/layer20_attention_weight_max": 19.16560935974121,
      "activations/layer20_attention_weight_min": -14.644381523132324,
      "activations/layer21_attention_weight_max": 31.721616744995117,
      "activations/layer21_attention_weight_min": -16.672189712524414,
      "activations/layer22_attention_weight_max": 31.758846282958984,
      "activations/layer22_attention_weight_min": -17.061683654785156,
      "activations/layer23_attention_weight_max": 22.826778411865234,
      "activations/layer23_attention_weight_min": -14.07343578338623,
      "activations/layer2_attention_weight_max": 25.285442352294922,
      "activations/layer2_attention_weight_min": -23.548017501831055,
      "activations/layer3_attention_weight_max": 52.824649810791016,
      "activations/layer3_attention_weight_min": -54.174320220947266,
      "activations/layer4_attention_weight_max": 81.44865417480469,
      "activations/layer4_attention_weight_min": -76.45056915283203,
      "activations/layer5_attention_weight_max": 137.07052612304688,
      "activations/layer5_attention_weight_min": -122.37306213378906,
      "activations/layer6_attention_weight_max": 63.781925201416016,
      "activations/layer6_attention_weight_min": -58.608482360839844,
      "activations/layer7_attention_weight_max": 62.79095458984375,
      "activations/layer7_attention_weight_min": -58.459815979003906,
      "activations/layer8_attention_weight_max": 52.22053909301758,
      "activations/layer8_attention_weight_min": -48.27132797241211,
      "activations/layer9_attention_weight_max": 74.96585845947266,
      "activations/layer9_attention_weight_min": -64.15142059326172,
      "epoch": 1.07,
      "learning_rate": 0.00014453863636363634,
      "loss": 3.0749,
      "step": 18450
    },
    {
      "activations/layer0_attention_weight_max": 13.602337837219238,
      "activations/layer0_attention_weight_min": -11.529151916503906,
      "activations/layer10_attention_weight_max": 67.05574798583984,
      "activations/layer10_attention_weight_min": -56.387874603271484,
      "activations/layer11_attention_weight_max": 44.029541015625,
      "activations/layer11_attention_weight_min": -39.34348678588867,
      "activations/layer12_attention_weight_max": 28.70648765563965,
      "activations/layer12_attention_weight_min": -23.902576446533203,
      "activations/layer13_attention_weight_max": 73.47135162353516,
      "activations/layer13_attention_weight_min": -52.251426696777344,
      "activations/layer14_attention_weight_max": 67.01427459716797,
      "activations/layer14_attention_weight_min": -48.59859848022461,
      "activations/layer15_attention_weight_max": 51.80696487426758,
      "activations/layer15_attention_weight_min": -42.04521560668945,
      "activations/layer16_attention_weight_max": 52.39252853393555,
      "activations/layer16_attention_weight_min": -34.49406051635742,
      "activations/layer17_attention_weight_max": 63.98605728149414,
      "activations/layer17_attention_weight_min": -40.59326934814453,
      "activations/layer18_attention_weight_max": 90.19561004638672,
      "activations/layer18_attention_weight_min": -41.2900505065918,
      "activations/layer19_attention_weight_max": 20.846349716186523,
      "activations/layer19_attention_weight_min": -14.350677490234375,
      "activations/layer1_attention_weight_max": 14.796537399291992,
      "activations/layer1_attention_weight_min": -11.156357765197754,
      "activations/layer20_attention_weight_max": 21.509477615356445,
      "activations/layer20_attention_weight_min": -15.367916107177734,
      "activations/layer21_attention_weight_max": 37.58620071411133,
      "activations/layer21_attention_weight_min": -22.846633911132812,
      "activations/layer22_attention_weight_max": 35.711700439453125,
      "activations/layer22_attention_weight_min": -17.074264526367188,
      "activations/layer23_attention_weight_max": 23.42443084716797,
      "activations/layer23_attention_weight_min": -14.846442222595215,
      "activations/layer2_attention_weight_max": 26.285200119018555,
      "activations/layer2_attention_weight_min": -24.27107810974121,
      "activations/layer3_attention_weight_max": 52.48260498046875,
      "activations/layer3_attention_weight_min": -55.32708740234375,
      "activations/layer4_attention_weight_max": 78.48463439941406,
      "activations/layer4_attention_weight_min": -73.39409637451172,
      "activations/layer5_attention_weight_max": 130.76806640625,
      "activations/layer5_attention_weight_min": -129.49850463867188,
      "activations/layer6_attention_weight_max": 63.17695236206055,
      "activations/layer6_attention_weight_min": -57.111209869384766,
      "activations/layer7_attention_weight_max": 61.903690338134766,
      "activations/layer7_attention_weight_min": -56.33169174194336,
      "activations/layer8_attention_weight_max": 55.16183853149414,
      "activations/layer8_attention_weight_min": -50.94043731689453,
      "activations/layer9_attention_weight_max": 66.59437561035156,
      "activations/layer9_attention_weight_min": -62.53155517578125,
      "epoch": 1.07,
      "learning_rate": 0.00014451969696969695,
      "loss": 3.0607,
      "step": 18500
    },
    {
      "activations/layer0_attention_weight_max": 14.866877555847168,
      "activations/layer0_attention_weight_min": -11.229266166687012,
      "activations/layer10_attention_weight_max": 71.756103515625,
      "activations/layer10_attention_weight_min": -61.764434814453125,
      "activations/layer11_attention_weight_max": 50.42991256713867,
      "activations/layer11_attention_weight_min": -38.63397979736328,
      "activations/layer12_attention_weight_max": 28.192411422729492,
      "activations/layer12_attention_weight_min": -25.295209884643555,
      "activations/layer13_attention_weight_max": 58.95043182373047,
      "activations/layer13_attention_weight_min": -47.791385650634766,
      "activations/layer14_attention_weight_max": 63.18439483642578,
      "activations/layer14_attention_weight_min": -44.29584503173828,
      "activations/layer15_attention_weight_max": 50.02642059326172,
      "activations/layer15_attention_weight_min": -38.28487014770508,
      "activations/layer16_attention_weight_max": 46.06097412109375,
      "activations/layer16_attention_weight_min": -32.39871597290039,
      "activations/layer17_attention_weight_max": 59.91683578491211,
      "activations/layer17_attention_weight_min": -35.38136291503906,
      "activations/layer18_attention_weight_max": 66.33633422851562,
      "activations/layer18_attention_weight_min": -36.143898010253906,
      "activations/layer19_attention_weight_max": 20.058109283447266,
      "activations/layer19_attention_weight_min": -13.447508811950684,
      "activations/layer1_attention_weight_max": 15.566079139709473,
      "activations/layer1_attention_weight_min": -11.266012191772461,
      "activations/layer20_attention_weight_max": 18.48292350769043,
      "activations/layer20_attention_weight_min": -13.313515663146973,
      "activations/layer21_attention_weight_max": 34.851253509521484,
      "activations/layer21_attention_weight_min": -17.764158248901367,
      "activations/layer22_attention_weight_max": 33.57974624633789,
      "activations/layer22_attention_weight_min": -15.56223201751709,
      "activations/layer23_attention_weight_max": 21.819625854492188,
      "activations/layer23_attention_weight_min": -16.0279541015625,
      "activations/layer2_attention_weight_max": 25.4608154296875,
      "activations/layer2_attention_weight_min": -22.811420440673828,
      "activations/layer3_attention_weight_max": 52.95491027832031,
      "activations/layer3_attention_weight_min": -53.673622131347656,
      "activations/layer4_attention_weight_max": 80.39995574951172,
      "activations/layer4_attention_weight_min": -75.16084289550781,
      "activations/layer5_attention_weight_max": 134.2022247314453,
      "activations/layer5_attention_weight_min": -128.11367797851562,
      "activations/layer6_attention_weight_max": 61.763980865478516,
      "activations/layer6_attention_weight_min": -58.493675231933594,
      "activations/layer7_attention_weight_max": 61.10475158691406,
      "activations/layer7_attention_weight_min": -61.24107360839844,
      "activations/layer8_attention_weight_max": 56.84465408325195,
      "activations/layer8_attention_weight_min": -50.4658088684082,
      "activations/layer9_attention_weight_max": 67.86766815185547,
      "activations/layer9_attention_weight_min": -65.0278091430664,
      "epoch": 1.08,
      "learning_rate": 0.00014450113636363634,
      "loss": 3.0849,
      "step": 18550
    },
    {
      "activations/layer0_attention_weight_max": 14.228170394897461,
      "activations/layer0_attention_weight_min": -10.65600872039795,
      "activations/layer10_attention_weight_max": 66.95980834960938,
      "activations/layer10_attention_weight_min": -56.09140396118164,
      "activations/layer11_attention_weight_max": 49.695430755615234,
      "activations/layer11_attention_weight_min": -40.60160827636719,
      "activations/layer12_attention_weight_max": 27.121782302856445,
      "activations/layer12_attention_weight_min": -23.35465431213379,
      "activations/layer13_attention_weight_max": 57.16843795776367,
      "activations/layer13_attention_weight_min": -42.85239791870117,
      "activations/layer14_attention_weight_max": 44.71798324584961,
      "activations/layer14_attention_weight_min": -36.147926330566406,
      "activations/layer15_attention_weight_max": 52.57006072998047,
      "activations/layer15_attention_weight_min": -39.69738006591797,
      "activations/layer16_attention_weight_max": 55.01757049560547,
      "activations/layer16_attention_weight_min": -35.12523651123047,
      "activations/layer17_attention_weight_max": 65.99052429199219,
      "activations/layer17_attention_weight_min": -39.873023986816406,
      "activations/layer18_attention_weight_max": 68.23101043701172,
      "activations/layer18_attention_weight_min": -40.99871826171875,
      "activations/layer19_attention_weight_max": 18.9096736907959,
      "activations/layer19_attention_weight_min": -13.1616849899292,
      "activations/layer1_attention_weight_max": 14.499706268310547,
      "activations/layer1_attention_weight_min": -10.619057655334473,
      "activations/layer20_attention_weight_max": 17.40851593017578,
      "activations/layer20_attention_weight_min": -14.743963241577148,
      "activations/layer21_attention_weight_max": 38.51313781738281,
      "activations/layer21_attention_weight_min": -18.739715576171875,
      "activations/layer22_attention_weight_max": 35.10245895385742,
      "activations/layer22_attention_weight_min": -16.345035552978516,
      "activations/layer23_attention_weight_max": 23.377696990966797,
      "activations/layer23_attention_weight_min": -13.108503341674805,
      "activations/layer2_attention_weight_max": 26.09621238708496,
      "activations/layer2_attention_weight_min": -24.5898494720459,
      "activations/layer3_attention_weight_max": 57.077308654785156,
      "activations/layer3_attention_weight_min": -57.21125030517578,
      "activations/layer4_attention_weight_max": 85.66828918457031,
      "activations/layer4_attention_weight_min": -76.7683334350586,
      "activations/layer5_attention_weight_max": 129.94912719726562,
      "activations/layer5_attention_weight_min": -128.44570922851562,
      "activations/layer6_attention_weight_max": 61.90858459472656,
      "activations/layer6_attention_weight_min": -57.128700256347656,
      "activations/layer7_attention_weight_max": 61.94541931152344,
      "activations/layer7_attention_weight_min": -58.568817138671875,
      "activations/layer8_attention_weight_max": 56.09298324584961,
      "activations/layer8_attention_weight_min": -50.85607147216797,
      "activations/layer9_attention_weight_max": 71.7181167602539,
      "activations/layer9_attention_weight_min": -66.31729125976562,
      "epoch": 1.08,
      "learning_rate": 0.00014448219696969696,
      "loss": 3.0927,
      "step": 18600
    },
    {
      "activations/layer0_attention_weight_max": 13.393134117126465,
      "activations/layer0_attention_weight_min": -11.862568855285645,
      "activations/layer10_attention_weight_max": 65.3272476196289,
      "activations/layer10_attention_weight_min": -57.14582443237305,
      "activations/layer11_attention_weight_max": 45.14481735229492,
      "activations/layer11_attention_weight_min": -42.52821350097656,
      "activations/layer12_attention_weight_max": 31.189359664916992,
      "activations/layer12_attention_weight_min": -26.17796516418457,
      "activations/layer13_attention_weight_max": 52.57938766479492,
      "activations/layer13_attention_weight_min": -39.472869873046875,
      "activations/layer14_attention_weight_max": 41.970916748046875,
      "activations/layer14_attention_weight_min": -39.2307014465332,
      "activations/layer15_attention_weight_max": 49.443695068359375,
      "activations/layer15_attention_weight_min": -40.5358772277832,
      "activations/layer16_attention_weight_max": 45.043190002441406,
      "activations/layer16_attention_weight_min": -34.66454315185547,
      "activations/layer17_attention_weight_max": 53.710594177246094,
      "activations/layer17_attention_weight_min": -36.98381805419922,
      "activations/layer18_attention_weight_max": 53.47395706176758,
      "activations/layer18_attention_weight_min": -35.237152099609375,
      "activations/layer19_attention_weight_max": 20.091230392456055,
      "activations/layer19_attention_weight_min": -12.781625747680664,
      "activations/layer1_attention_weight_max": 15.675093650817871,
      "activations/layer1_attention_weight_min": -10.9942045211792,
      "activations/layer20_attention_weight_max": 19.402080535888672,
      "activations/layer20_attention_weight_min": -14.098254203796387,
      "activations/layer21_attention_weight_max": 36.69862747192383,
      "activations/layer21_attention_weight_min": -20.74524688720703,
      "activations/layer22_attention_weight_max": 32.139892578125,
      "activations/layer22_attention_weight_min": -19.134328842163086,
      "activations/layer23_attention_weight_max": 20.28559112548828,
      "activations/layer23_attention_weight_min": -15.50312328338623,
      "activations/layer2_attention_weight_max": 25.52021026611328,
      "activations/layer2_attention_weight_min": -23.703369140625,
      "activations/layer3_attention_weight_max": 52.27680206298828,
      "activations/layer3_attention_weight_min": -54.91378402709961,
      "activations/layer4_attention_weight_max": 79.07320404052734,
      "activations/layer4_attention_weight_min": -74.56111907958984,
      "activations/layer5_attention_weight_max": 126.92549896240234,
      "activations/layer5_attention_weight_min": -119.6672592163086,
      "activations/layer6_attention_weight_max": 61.028160095214844,
      "activations/layer6_attention_weight_min": -59.76121520996094,
      "activations/layer7_attention_weight_max": 62.53569793701172,
      "activations/layer7_attention_weight_min": -59.30356216430664,
      "activations/layer8_attention_weight_max": 54.1621208190918,
      "activations/layer8_attention_weight_min": -50.27031326293945,
      "activations/layer9_attention_weight_max": 69.3901596069336,
      "activations/layer9_attention_weight_min": -68.83616638183594,
      "epoch": 1.08,
      "learning_rate": 0.00014446325757575755,
      "loss": 3.0823,
      "step": 18650
    },
    {
      "activations/layer0_attention_weight_max": 15.837900161743164,
      "activations/layer0_attention_weight_min": -11.876225471496582,
      "activations/layer10_attention_weight_max": 65.92549896240234,
      "activations/layer10_attention_weight_min": -59.595458984375,
      "activations/layer11_attention_weight_max": 43.21546173095703,
      "activations/layer11_attention_weight_min": -37.499916076660156,
      "activations/layer12_attention_weight_max": 26.193052291870117,
      "activations/layer12_attention_weight_min": -25.64706802368164,
      "activations/layer13_attention_weight_max": 58.61561965942383,
      "activations/layer13_attention_weight_min": -44.05105972290039,
      "activations/layer14_attention_weight_max": 56.426170349121094,
      "activations/layer14_attention_weight_min": -39.74964904785156,
      "activations/layer15_attention_weight_max": 62.24174499511719,
      "activations/layer15_attention_weight_min": -45.0893440246582,
      "activations/layer16_attention_weight_max": 49.23821258544922,
      "activations/layer16_attention_weight_min": -34.06154251098633,
      "activations/layer17_attention_weight_max": 59.13774871826172,
      "activations/layer17_attention_weight_min": -39.8906364440918,
      "activations/layer18_attention_weight_max": 67.50350189208984,
      "activations/layer18_attention_weight_min": -41.07733917236328,
      "activations/layer19_attention_weight_max": 19.992427825927734,
      "activations/layer19_attention_weight_min": -13.147093772888184,
      "activations/layer1_attention_weight_max": 16.284101486206055,
      "activations/layer1_attention_weight_min": -11.342436790466309,
      "activations/layer20_attention_weight_max": 17.20501708984375,
      "activations/layer20_attention_weight_min": -15.775379180908203,
      "activations/layer21_attention_weight_max": 42.82261276245117,
      "activations/layer21_attention_weight_min": -19.43387222290039,
      "activations/layer22_attention_weight_max": 39.31106948852539,
      "activations/layer22_attention_weight_min": -21.16019058227539,
      "activations/layer23_attention_weight_max": 26.420034408569336,
      "activations/layer23_attention_weight_min": -15.922080039978027,
      "activations/layer2_attention_weight_max": 24.577613830566406,
      "activations/layer2_attention_weight_min": -22.90680503845215,
      "activations/layer3_attention_weight_max": 49.75282287597656,
      "activations/layer3_attention_weight_min": -53.71958541870117,
      "activations/layer4_attention_weight_max": 77.34473419189453,
      "activations/layer4_attention_weight_min": -73.55745697021484,
      "activations/layer5_attention_weight_max": 131.79232788085938,
      "activations/layer5_attention_weight_min": -121.95991516113281,
      "activations/layer6_attention_weight_max": 63.76744842529297,
      "activations/layer6_attention_weight_min": -59.251007080078125,
      "activations/layer7_attention_weight_max": 63.05622863769531,
      "activations/layer7_attention_weight_min": -57.15849304199219,
      "activations/layer8_attention_weight_max": 54.97845458984375,
      "activations/layer8_attention_weight_min": -50.21663284301758,
      "activations/layer9_attention_weight_max": 83.11149597167969,
      "activations/layer9_attention_weight_min": -63.568233489990234,
      "epoch": 1.09,
      "learning_rate": 0.00014444431818181816,
      "loss": 3.0704,
      "step": 18700
    },
    {
      "activations/layer0_attention_weight_max": 13.665886878967285,
      "activations/layer0_attention_weight_min": -11.280793190002441,
      "activations/layer10_attention_weight_max": 81.422119140625,
      "activations/layer10_attention_weight_min": -73.08255767822266,
      "activations/layer11_attention_weight_max": 56.331336975097656,
      "activations/layer11_attention_weight_min": -50.042320251464844,
      "activations/layer12_attention_weight_max": 26.475194931030273,
      "activations/layer12_attention_weight_min": -26.6009464263916,
      "activations/layer13_attention_weight_max": 56.513267517089844,
      "activations/layer13_attention_weight_min": -44.30669403076172,
      "activations/layer14_attention_weight_max": 62.933387756347656,
      "activations/layer14_attention_weight_min": -44.85361862182617,
      "activations/layer15_attention_weight_max": 56.42259216308594,
      "activations/layer15_attention_weight_min": -40.53749084472656,
      "activations/layer16_attention_weight_max": 53.61077880859375,
      "activations/layer16_attention_weight_min": -39.692535400390625,
      "activations/layer17_attention_weight_max": 55.18197250366211,
      "activations/layer17_attention_weight_min": -40.83781433105469,
      "activations/layer18_attention_weight_max": 55.544761657714844,
      "activations/layer18_attention_weight_min": -38.948543548583984,
      "activations/layer19_attention_weight_max": 19.95209503173828,
      "activations/layer19_attention_weight_min": -12.18172836303711,
      "activations/layer1_attention_weight_max": 15.247366905212402,
      "activations/layer1_attention_weight_min": -11.352954864501953,
      "activations/layer20_attention_weight_max": 19.990495681762695,
      "activations/layer20_attention_weight_min": -15.31347370147705,
      "activations/layer21_attention_weight_max": 36.44709777832031,
      "activations/layer21_attention_weight_min": -19.861942291259766,
      "activations/layer22_attention_weight_max": 29.83779525756836,
      "activations/layer22_attention_weight_min": -15.190574645996094,
      "activations/layer23_attention_weight_max": 20.423786163330078,
      "activations/layer23_attention_weight_min": -14.398600578308105,
      "activations/layer2_attention_weight_max": 26.336395263671875,
      "activations/layer2_attention_weight_min": -24.206832885742188,
      "activations/layer3_attention_weight_max": 50.72119140625,
      "activations/layer3_attention_weight_min": -52.75325393676758,
      "activations/layer4_attention_weight_max": 79.46638488769531,
      "activations/layer4_attention_weight_min": -74.16425323486328,
      "activations/layer5_attention_weight_max": 126.6519775390625,
      "activations/layer5_attention_weight_min": -121.6336669921875,
      "activations/layer6_attention_weight_max": 60.25175094604492,
      "activations/layer6_attention_weight_min": -55.02826690673828,
      "activations/layer7_attention_weight_max": 63.34819412231445,
      "activations/layer7_attention_weight_min": -57.83124542236328,
      "activations/layer8_attention_weight_max": 57.16627883911133,
      "activations/layer8_attention_weight_min": -51.696189880371094,
      "activations/layer9_attention_weight_max": 70.44159698486328,
      "activations/layer9_attention_weight_min": -73.94157409667969,
      "epoch": 1.09,
      "learning_rate": 0.00014442537878787878,
      "loss": 3.0828,
      "step": 18750
    },
    {
      "activations/layer0_attention_weight_max": 12.846381187438965,
      "activations/layer0_attention_weight_min": -11.089404106140137,
      "activations/layer10_attention_weight_max": 70.72953033447266,
      "activations/layer10_attention_weight_min": -58.613067626953125,
      "activations/layer11_attention_weight_max": 45.84910583496094,
      "activations/layer11_attention_weight_min": -39.152923583984375,
      "activations/layer12_attention_weight_max": 33.28094482421875,
      "activations/layer12_attention_weight_min": -26.546932220458984,
      "activations/layer13_attention_weight_max": 56.21772384643555,
      "activations/layer13_attention_weight_min": -43.824546813964844,
      "activations/layer14_attention_weight_max": 62.52923583984375,
      "activations/layer14_attention_weight_min": -42.03701400756836,
      "activations/layer15_attention_weight_max": 59.808441162109375,
      "activations/layer15_attention_weight_min": -43.228145599365234,
      "activations/layer16_attention_weight_max": 53.53892135620117,
      "activations/layer16_attention_weight_min": -36.98723220825195,
      "activations/layer17_attention_weight_max": 59.50321578979492,
      "activations/layer17_attention_weight_min": -38.37965393066406,
      "activations/layer18_attention_weight_max": 66.07646179199219,
      "activations/layer18_attention_weight_min": -38.568878173828125,
      "activations/layer19_attention_weight_max": 22.155162811279297,
      "activations/layer19_attention_weight_min": -12.776459693908691,
      "activations/layer1_attention_weight_max": 14.4910249710083,
      "activations/layer1_attention_weight_min": -11.543070793151855,
      "activations/layer20_attention_weight_max": 18.387235641479492,
      "activations/layer20_attention_weight_min": -14.205781936645508,
      "activations/layer21_attention_weight_max": 42.20631790161133,
      "activations/layer21_attention_weight_min": -20.207626342773438,
      "activations/layer22_attention_weight_max": 35.31355667114258,
      "activations/layer22_attention_weight_min": -16.456602096557617,
      "activations/layer23_attention_weight_max": 22.466144561767578,
      "activations/layer23_attention_weight_min": -14.113479614257812,
      "activations/layer2_attention_weight_max": 25.734527587890625,
      "activations/layer2_attention_weight_min": -23.881771087646484,
      "activations/layer3_attention_weight_max": 52.9909782409668,
      "activations/layer3_attention_weight_min": -53.925270080566406,
      "activations/layer4_attention_weight_max": 83.71489715576172,
      "activations/layer4_attention_weight_min": -77.93814849853516,
      "activations/layer5_attention_weight_max": 137.08912658691406,
      "activations/layer5_attention_weight_min": -122.7020263671875,
      "activations/layer6_attention_weight_max": 66.6796875,
      "activations/layer6_attention_weight_min": -58.72580337524414,
      "activations/layer7_attention_weight_max": 67.91564178466797,
      "activations/layer7_attention_weight_min": -58.384056091308594,
      "activations/layer8_attention_weight_max": 54.40728759765625,
      "activations/layer8_attention_weight_min": -49.770362854003906,
      "activations/layer9_attention_weight_max": 75.79069519042969,
      "activations/layer9_attention_weight_min": -64.48606872558594,
      "epoch": 1.09,
      "learning_rate": 0.00014440643939393936,
      "loss": 3.0659,
      "step": 18800
    },
    {
      "activations/layer0_attention_weight_max": 14.602987289428711,
      "activations/layer0_attention_weight_min": -11.1132173538208,
      "activations/layer10_attention_weight_max": 72.93914031982422,
      "activations/layer10_attention_weight_min": -60.82927703857422,
      "activations/layer11_attention_weight_max": 49.20897674560547,
      "activations/layer11_attention_weight_min": -38.569252014160156,
      "activations/layer12_attention_weight_max": 29.085838317871094,
      "activations/layer12_attention_weight_min": -25.969467163085938,
      "activations/layer13_attention_weight_max": 59.58056640625,
      "activations/layer13_attention_weight_min": -45.05915069580078,
      "activations/layer14_attention_weight_max": 50.220638275146484,
      "activations/layer14_attention_weight_min": -37.290218353271484,
      "activations/layer15_attention_weight_max": 51.122291564941406,
      "activations/layer15_attention_weight_min": -38.882598876953125,
      "activations/layer16_attention_weight_max": 45.45637512207031,
      "activations/layer16_attention_weight_min": -32.97690200805664,
      "activations/layer17_attention_weight_max": 58.07478713989258,
      "activations/layer17_attention_weight_min": -37.54280090332031,
      "activations/layer18_attention_weight_max": 61.17461395263672,
      "activations/layer18_attention_weight_min": -38.69575500488281,
      "activations/layer19_attention_weight_max": 20.631742477416992,
      "activations/layer19_attention_weight_min": -14.527922630310059,
      "activations/layer1_attention_weight_max": 15.582751274108887,
      "activations/layer1_attention_weight_min": -10.86642837524414,
      "activations/layer20_attention_weight_max": 20.9888916015625,
      "activations/layer20_attention_weight_min": -13.901863098144531,
      "activations/layer21_attention_weight_max": 40.07770538330078,
      "activations/layer21_attention_weight_min": -21.67420768737793,
      "activations/layer22_attention_weight_max": 35.53638458251953,
      "activations/layer22_attention_weight_min": -16.812395095825195,
      "activations/layer23_attention_weight_max": 22.966642379760742,
      "activations/layer23_attention_weight_min": -15.044225692749023,
      "activations/layer2_attention_weight_max": 25.082481384277344,
      "activations/layer2_attention_weight_min": -22.388439178466797,
      "activations/layer3_attention_weight_max": 53.18914031982422,
      "activations/layer3_attention_weight_min": -53.38289260864258,
      "activations/layer4_attention_weight_max": 84.06153106689453,
      "activations/layer4_attention_weight_min": -72.88982391357422,
      "activations/layer5_attention_weight_max": 134.25123596191406,
      "activations/layer5_attention_weight_min": -120.66453552246094,
      "activations/layer6_attention_weight_max": 64.35628509521484,
      "activations/layer6_attention_weight_min": -56.2191047668457,
      "activations/layer7_attention_weight_max": 61.83549118041992,
      "activations/layer7_attention_weight_min": -57.156028747558594,
      "activations/layer8_attention_weight_max": 56.76525115966797,
      "activations/layer8_attention_weight_min": -47.424095153808594,
      "activations/layer9_attention_weight_max": 70.48973846435547,
      "activations/layer9_attention_weight_min": -64.54924011230469,
      "epoch": 1.1,
      "learning_rate": 0.00014438749999999998,
      "loss": 3.0773,
      "step": 18850
    },
    {
      "activations/layer0_attention_weight_max": 13.787109375,
      "activations/layer0_attention_weight_min": -11.436604499816895,
      "activations/layer10_attention_weight_max": 65.74200439453125,
      "activations/layer10_attention_weight_min": -56.629364013671875,
      "activations/layer11_attention_weight_max": 49.2613639831543,
      "activations/layer11_attention_weight_min": -41.30210494995117,
      "activations/layer12_attention_weight_max": 29.045751571655273,
      "activations/layer12_attention_weight_min": -24.671228408813477,
      "activations/layer13_attention_weight_max": 59.59343719482422,
      "activations/layer13_attention_weight_min": -44.04887008666992,
      "activations/layer14_attention_weight_max": 42.78824234008789,
      "activations/layer14_attention_weight_min": -31.618316650390625,
      "activations/layer15_attention_weight_max": 53.853118896484375,
      "activations/layer15_attention_weight_min": -38.26576614379883,
      "activations/layer16_attention_weight_max": 49.1789436340332,
      "activations/layer16_attention_weight_min": -34.45700454711914,
      "activations/layer17_attention_weight_max": 53.532405853271484,
      "activations/layer17_attention_weight_min": -37.231788635253906,
      "activations/layer18_attention_weight_max": 56.63742446899414,
      "activations/layer18_attention_weight_min": -34.30952072143555,
      "activations/layer19_attention_weight_max": 21.15829849243164,
      "activations/layer19_attention_weight_min": -12.243766784667969,
      "activations/layer1_attention_weight_max": 14.616968154907227,
      "activations/layer1_attention_weight_min": -11.796300888061523,
      "activations/layer20_attention_weight_max": 17.946369171142578,
      "activations/layer20_attention_weight_min": -15.85224723815918,
      "activations/layer21_attention_weight_max": 32.452392578125,
      "activations/layer21_attention_weight_min": -17.28628158569336,
      "activations/layer22_attention_weight_max": 28.0695858001709,
      "activations/layer22_attention_weight_min": -17.90532112121582,
      "activations/layer23_attention_weight_max": 21.245220184326172,
      "activations/layer23_attention_weight_min": -14.337026596069336,
      "activations/layer2_attention_weight_max": 24.198226928710938,
      "activations/layer2_attention_weight_min": -22.88789939880371,
      "activations/layer3_attention_weight_max": 53.116355895996094,
      "activations/layer3_attention_weight_min": -56.753501892089844,
      "activations/layer4_attention_weight_max": 86.40682220458984,
      "activations/layer4_attention_weight_min": -79.2962417602539,
      "activations/layer5_attention_weight_max": 137.603271484375,
      "activations/layer5_attention_weight_min": -128.33157348632812,
      "activations/layer6_attention_weight_max": 65.70147705078125,
      "activations/layer6_attention_weight_min": -61.56023025512695,
      "activations/layer7_attention_weight_max": 67.306396484375,
      "activations/layer7_attention_weight_min": -59.27774429321289,
      "activations/layer8_attention_weight_max": 55.9326057434082,
      "activations/layer8_attention_weight_min": -50.69696044921875,
      "activations/layer9_attention_weight_max": 68.34383392333984,
      "activations/layer9_attention_weight_min": -65.52080535888672,
      "epoch": 1.1,
      "learning_rate": 0.0001443685606060606,
      "loss": 3.0552,
      "step": 18900
    },
    {
      "activations/layer0_attention_weight_max": 13.822290420532227,
      "activations/layer0_attention_weight_min": -11.407341957092285,
      "activations/layer10_attention_weight_max": 59.77275085449219,
      "activations/layer10_attention_weight_min": -57.57767868041992,
      "activations/layer11_attention_weight_max": 42.109580993652344,
      "activations/layer11_attention_weight_min": -38.567108154296875,
      "activations/layer12_attention_weight_max": 26.15597152709961,
      "activations/layer12_attention_weight_min": -24.809545516967773,
      "activations/layer13_attention_weight_max": 62.66777420043945,
      "activations/layer13_attention_weight_min": -49.40571594238281,
      "activations/layer14_attention_weight_max": 61.73344802856445,
      "activations/layer14_attention_weight_min": -46.99343490600586,
      "activations/layer15_attention_weight_max": 52.59503936767578,
      "activations/layer15_attention_weight_min": -42.606964111328125,
      "activations/layer16_attention_weight_max": 45.189720153808594,
      "activations/layer16_attention_weight_min": -33.79523849487305,
      "activations/layer17_attention_weight_max": 53.89806365966797,
      "activations/layer17_attention_weight_min": -39.976112365722656,
      "activations/layer18_attention_weight_max": 66.94700622558594,
      "activations/layer18_attention_weight_min": -40.72671890258789,
      "activations/layer19_attention_weight_max": 19.79462432861328,
      "activations/layer19_attention_weight_min": -13.589479446411133,
      "activations/layer1_attention_weight_max": 16.4469051361084,
      "activations/layer1_attention_weight_min": -10.840500831604004,
      "activations/layer20_attention_weight_max": 21.55108642578125,
      "activations/layer20_attention_weight_min": -15.352532386779785,
      "activations/layer21_attention_weight_max": 35.669921875,
      "activations/layer21_attention_weight_min": -19.208038330078125,
      "activations/layer22_attention_weight_max": 33.434871673583984,
      "activations/layer22_attention_weight_min": -16.243886947631836,
      "activations/layer23_attention_weight_max": 22.71285057067871,
      "activations/layer23_attention_weight_min": -15.278541564941406,
      "activations/layer2_attention_weight_max": 26.587642669677734,
      "activations/layer2_attention_weight_min": -23.336721420288086,
      "activations/layer3_attention_weight_max": 52.5849723815918,
      "activations/layer3_attention_weight_min": -54.93033981323242,
      "activations/layer4_attention_weight_max": 83.36151885986328,
      "activations/layer4_attention_weight_min": -74.88456726074219,
      "activations/layer5_attention_weight_max": 131.8466796875,
      "activations/layer5_attention_weight_min": -126.8875961303711,
      "activations/layer6_attention_weight_max": 63.637821197509766,
      "activations/layer6_attention_weight_min": -57.943809509277344,
      "activations/layer7_attention_weight_max": 64.8594741821289,
      "activations/layer7_attention_weight_min": -56.06443405151367,
      "activations/layer8_attention_weight_max": 50.770084381103516,
      "activations/layer8_attention_weight_min": -45.51990509033203,
      "activations/layer9_attention_weight_max": 65.38709259033203,
      "activations/layer9_attention_weight_min": -59.551124572753906,
      "epoch": 1.1,
      "learning_rate": 0.0001443496212121212,
      "loss": 3.0833,
      "step": 18950
    },
    {
      "activations/layer0_attention_weight_max": 13.756367683410645,
      "activations/layer0_attention_weight_min": -11.20219898223877,
      "activations/layer10_attention_weight_max": 65.38312530517578,
      "activations/layer10_attention_weight_min": -60.79465866088867,
      "activations/layer11_attention_weight_max": 46.01359558105469,
      "activations/layer11_attention_weight_min": -39.637001037597656,
      "activations/layer12_attention_weight_max": 27.80051612854004,
      "activations/layer12_attention_weight_min": -25.331541061401367,
      "activations/layer13_attention_weight_max": 55.3348388671875,
      "activations/layer13_attention_weight_min": -40.927425384521484,
      "activations/layer14_attention_weight_max": 49.29127883911133,
      "activations/layer14_attention_weight_min": -39.14338684082031,
      "activations/layer15_attention_weight_max": 54.41819381713867,
      "activations/layer15_attention_weight_min": -37.753719329833984,
      "activations/layer16_attention_weight_max": 49.89975357055664,
      "activations/layer16_attention_weight_min": -38.41560363769531,
      "activations/layer17_attention_weight_max": 54.05410385131836,
      "activations/layer17_attention_weight_min": -38.06943130493164,
      "activations/layer18_attention_weight_max": 58.6929817199707,
      "activations/layer18_attention_weight_min": -39.513214111328125,
      "activations/layer19_attention_weight_max": 18.268962860107422,
      "activations/layer19_attention_weight_min": -13.465466499328613,
      "activations/layer1_attention_weight_max": 15.895686149597168,
      "activations/layer1_attention_weight_min": -11.687254905700684,
      "activations/layer20_attention_weight_max": 19.764923095703125,
      "activations/layer20_attention_weight_min": -14.786762237548828,
      "activations/layer21_attention_weight_max": 30.93364715576172,
      "activations/layer21_attention_weight_min": -17.150781631469727,
      "activations/layer22_attention_weight_max": 30.269943237304688,
      "activations/layer22_attention_weight_min": -15.323041915893555,
      "activations/layer23_attention_weight_max": 20.818592071533203,
      "activations/layer23_attention_weight_min": -16.338764190673828,
      "activations/layer2_attention_weight_max": 24.556312561035156,
      "activations/layer2_attention_weight_min": -22.890838623046875,
      "activations/layer3_attention_weight_max": 51.127525329589844,
      "activations/layer3_attention_weight_min": -52.9384765625,
      "activations/layer4_attention_weight_max": 78.25247955322266,
      "activations/layer4_attention_weight_min": -72.08651733398438,
      "activations/layer5_attention_weight_max": 132.3273162841797,
      "activations/layer5_attention_weight_min": -128.56048583984375,
      "activations/layer6_attention_weight_max": 64.49734497070312,
      "activations/layer6_attention_weight_min": -57.854576110839844,
      "activations/layer7_attention_weight_max": 61.577754974365234,
      "activations/layer7_attention_weight_min": -54.50918197631836,
      "activations/layer8_attention_weight_max": 48.82558059692383,
      "activations/layer8_attention_weight_min": -50.15306854248047,
      "activations/layer9_attention_weight_max": 66.8483657836914,
      "activations/layer9_attention_weight_min": -70.38280487060547,
      "epoch": 1.1,
      "learning_rate": 0.0001443306818181818,
      "loss": 3.0739,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_loss": 3.021484375,
      "eval_runtime": 8.6377,
      "eval_samples_per_second": 497.122,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_openwebtext_loss": 3.021484375,
      "eval_openwebtext_ppl": 20.521731031400577,
      "eval_openwebtext_runtime": 8.6377,
      "eval_openwebtext_samples_per_second": 497.122,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_wikitext_loss": 3.3125,
      "eval_wikitext_ppl": 27.4536739354601,
      "eval_wikitext_runtime": 1.5204,
      "eval_wikitext_samples_per_second": 299.918,
      "step": 19000
    },
    {
      "epoch": 1.1,
      "eval_lambada_loss": 3.412109375,
      "eval_lambada_ppl": 30.329152388418176,
      "eval_lambada_runtime": 9.2336,
      "eval_lambada_samples_per_second": 527.314,
      "step": 19000
    },
    {
      "activations/layer0_attention_weight_max": 14.506340026855469,
      "activations/layer0_attention_weight_min": -11.407258033752441,
      "activations/layer10_attention_weight_max": 69.3305435180664,
      "activations/layer10_attention_weight_min": -61.60361862182617,
      "activations/layer11_attention_weight_max": 50.38473129272461,
      "activations/layer11_attention_weight_min": -38.09327697753906,
      "activations/layer12_attention_weight_max": 33.55931854248047,
      "activations/layer12_attention_weight_min": -23.145599365234375,
      "activations/layer13_attention_weight_max": 62.548526763916016,
      "activations/layer13_attention_weight_min": -47.805076599121094,
      "activations/layer14_attention_weight_max": 61.021766662597656,
      "activations/layer14_attention_weight_min": -49.43037796020508,
      "activations/layer15_attention_weight_max": 65.66324615478516,
      "activations/layer15_attention_weight_min": -39.376522064208984,
      "activations/layer16_attention_weight_max": 49.957862854003906,
      "activations/layer16_attention_weight_min": -34.020809173583984,
      "activations/layer17_attention_weight_max": 59.8261604309082,
      "activations/layer17_attention_weight_min": -38.984928131103516,
      "activations/layer18_attention_weight_max": 68.2761459350586,
      "activations/layer18_attention_weight_min": -38.052398681640625,
      "activations/layer19_attention_weight_max": 20.326448440551758,
      "activations/layer19_attention_weight_min": -14.480499267578125,
      "activations/layer1_attention_weight_max": 15.681294441223145,
      "activations/layer1_attention_weight_min": -11.814736366271973,
      "activations/layer20_attention_weight_max": 19.629179000854492,
      "activations/layer20_attention_weight_min": -14.302419662475586,
      "activations/layer21_attention_weight_max": 39.05500411987305,
      "activations/layer21_attention_weight_min": -20.460739135742188,
      "activations/layer22_attention_weight_max": 33.93810272216797,
      "activations/layer22_attention_weight_min": -17.129674911499023,
      "activations/layer23_attention_weight_max": 21.271780014038086,
      "activations/layer23_attention_weight_min": -16.23723030090332,
      "activations/layer2_attention_weight_max": 27.940654754638672,
      "activations/layer2_attention_weight_min": -26.526723861694336,
      "activations/layer3_attention_weight_max": 54.26652908325195,
      "activations/layer3_attention_weight_min": -54.328556060791016,
      "activations/layer4_attention_weight_max": 81.52208709716797,
      "activations/layer4_attention_weight_min": -76.1376724243164,
      "activations/layer5_attention_weight_max": 137.05035400390625,
      "activations/layer5_attention_weight_min": -120.10884094238281,
      "activations/layer6_attention_weight_max": 67.87885284423828,
      "activations/layer6_attention_weight_min": -57.0800666809082,
      "activations/layer7_attention_weight_max": 67.58204650878906,
      "activations/layer7_attention_weight_min": -54.27814865112305,
      "activations/layer8_attention_weight_max": 62.20616912841797,
      "activations/layer8_attention_weight_min": -49.90705108642578,
      "activations/layer9_attention_weight_max": 78.82118225097656,
      "activations/layer9_attention_weight_min": -66.18270111083984,
      "epoch": 1.11,
      "learning_rate": 0.0001443117424242424,
      "loss": 3.0501,
      "step": 19050
    },
    {
      "activations/layer0_attention_weight_max": 14.309250831604004,
      "activations/layer0_attention_weight_min": -11.416489601135254,
      "activations/layer10_attention_weight_max": 71.19922637939453,
      "activations/layer10_attention_weight_min": -56.959163665771484,
      "activations/layer11_attention_weight_max": 44.03816223144531,
      "activations/layer11_attention_weight_min": -40.2879638671875,
      "activations/layer12_attention_weight_max": 27.513032913208008,
      "activations/layer12_attention_weight_min": -23.23210334777832,
      "activations/layer13_attention_weight_max": 79.07865142822266,
      "activations/layer13_attention_weight_min": -45.160457611083984,
      "activations/layer14_attention_weight_max": 75.33741760253906,
      "activations/layer14_attention_weight_min": -40.88359069824219,
      "activations/layer15_attention_weight_max": 63.45526885986328,
      "activations/layer15_attention_weight_min": -39.673980712890625,
      "activations/layer16_attention_weight_max": 56.812217712402344,
      "activations/layer16_attention_weight_min": -33.40242004394531,
      "activations/layer17_attention_weight_max": 60.46094512939453,
      "activations/layer17_attention_weight_min": -37.90968322753906,
      "activations/layer18_attention_weight_max": 78.65595245361328,
      "activations/layer18_attention_weight_min": -36.207252502441406,
      "activations/layer19_attention_weight_max": 19.517454147338867,
      "activations/layer19_attention_weight_min": -13.071919441223145,
      "activations/layer1_attention_weight_max": 14.969579696655273,
      "activations/layer1_attention_weight_min": -10.850810050964355,
      "activations/layer20_attention_weight_max": 19.523170471191406,
      "activations/layer20_attention_weight_min": -15.436772346496582,
      "activations/layer21_attention_weight_max": 34.511383056640625,
      "activations/layer21_attention_weight_min": -17.491580963134766,
      "activations/layer22_attention_weight_max": 34.59140396118164,
      "activations/layer22_attention_weight_min": -15.961549758911133,
      "activations/layer23_attention_weight_max": 22.771320343017578,
      "activations/layer23_attention_weight_min": -16.301651000976562,
      "activations/layer2_attention_weight_max": 23.699573516845703,
      "activations/layer2_attention_weight_min": -22.50958251953125,
      "activations/layer3_attention_weight_max": 52.58943557739258,
      "activations/layer3_attention_weight_min": -53.694175720214844,
      "activations/layer4_attention_weight_max": 84.01757049560547,
      "activations/layer4_attention_weight_min": -75.36463165283203,
      "activations/layer5_attention_weight_max": 139.42431640625,
      "activations/layer5_attention_weight_min": -123.48121643066406,
      "activations/layer6_attention_weight_max": 65.73772430419922,
      "activations/layer6_attention_weight_min": -57.368446350097656,
      "activations/layer7_attention_weight_max": 66.5561294555664,
      "activations/layer7_attention_weight_min": -55.842525482177734,
      "activations/layer8_attention_weight_max": 54.70131301879883,
      "activations/layer8_attention_weight_min": -47.89436721801758,
      "activations/layer9_attention_weight_max": 72.62541961669922,
      "activations/layer9_attention_weight_min": -68.75361633300781,
      "epoch": 1.11,
      "learning_rate": 0.00014429280303030303,
      "loss": 3.0749,
      "step": 19100
    },
    {
      "activations/layer0_attention_weight_max": 13.756808280944824,
      "activations/layer0_attention_weight_min": -10.998109817504883,
      "activations/layer10_attention_weight_max": 64.67764282226562,
      "activations/layer10_attention_weight_min": -66.97451782226562,
      "activations/layer11_attention_weight_max": 44.23656463623047,
      "activations/layer11_attention_weight_min": -45.704322814941406,
      "activations/layer12_attention_weight_max": 26.582149505615234,
      "activations/layer12_attention_weight_min": -25.51535987854004,
      "activations/layer13_attention_weight_max": 63.52592086791992,
      "activations/layer13_attention_weight_min": -44.430015563964844,
      "activations/layer14_attention_weight_max": 59.029937744140625,
      "activations/layer14_attention_weight_min": -45.78593826293945,
      "activations/layer15_attention_weight_max": 50.144508361816406,
      "activations/layer15_attention_weight_min": -40.42420959472656,
      "activations/layer16_attention_weight_max": 49.007415771484375,
      "activations/layer16_attention_weight_min": -37.00360107421875,
      "activations/layer17_attention_weight_max": 58.49980926513672,
      "activations/layer17_attention_weight_min": -39.9485969543457,
      "activations/layer18_attention_weight_max": 60.21694564819336,
      "activations/layer18_attention_weight_min": -39.16062545776367,
      "activations/layer19_attention_weight_max": 19.845491409301758,
      "activations/layer19_attention_weight_min": -12.809015274047852,
      "activations/layer1_attention_weight_max": 15.803271293640137,
      "activations/layer1_attention_weight_min": -11.603851318359375,
      "activations/layer20_attention_weight_max": 19.88610076904297,
      "activations/layer20_attention_weight_min": -16.033159255981445,
      "activations/layer21_attention_weight_max": 36.49221420288086,
      "activations/layer21_attention_weight_min": -26.102453231811523,
      "activations/layer22_attention_weight_max": 34.37861633300781,
      "activations/layer22_attention_weight_min": -19.651811599731445,
      "activations/layer23_attention_weight_max": 22.013662338256836,
      "activations/layer23_attention_weight_min": -13.866320610046387,
      "activations/layer2_attention_weight_max": 25.151382446289062,
      "activations/layer2_attention_weight_min": -24.510963439941406,
      "activations/layer3_attention_weight_max": 50.45762252807617,
      "activations/layer3_attention_weight_min": -52.19657516479492,
      "activations/layer4_attention_weight_max": 78.0215072631836,
      "activations/layer4_attention_weight_min": -72.6866683959961,
      "activations/layer5_attention_weight_max": 129.2664031982422,
      "activations/layer5_attention_weight_min": -119.42931365966797,
      "activations/layer6_attention_weight_max": 62.118980407714844,
      "activations/layer6_attention_weight_min": -60.179115295410156,
      "activations/layer7_attention_weight_max": 61.43356704711914,
      "activations/layer7_attention_weight_min": -55.527252197265625,
      "activations/layer8_attention_weight_max": 51.40668487548828,
      "activations/layer8_attention_weight_min": -51.22768020629883,
      "activations/layer9_attention_weight_max": 64.48757934570312,
      "activations/layer9_attention_weight_min": -66.02074432373047,
      "epoch": 1.11,
      "learning_rate": 0.00014427386363636364,
      "loss": 3.0843,
      "step": 19150
    },
    {
      "activations/layer0_attention_weight_max": 13.617573738098145,
      "activations/layer0_attention_weight_min": -11.241710662841797,
      "activations/layer10_attention_weight_max": 64.834228515625,
      "activations/layer10_attention_weight_min": -62.01886749267578,
      "activations/layer11_attention_weight_max": 41.96466827392578,
      "activations/layer11_attention_weight_min": -40.485374450683594,
      "activations/layer12_attention_weight_max": 24.1414852142334,
      "activations/layer12_attention_weight_min": -25.409854888916016,
      "activations/layer13_attention_weight_max": 65.39801025390625,
      "activations/layer13_attention_weight_min": -51.148868560791016,
      "activations/layer14_attention_weight_max": 57.70121383666992,
      "activations/layer14_attention_weight_min": -48.17290115356445,
      "activations/layer15_attention_weight_max": 55.47266387939453,
      "activations/layer15_attention_weight_min": -45.439266204833984,
      "activations/layer16_attention_weight_max": 49.917572021484375,
      "activations/layer16_attention_weight_min": -34.947479248046875,
      "activations/layer17_attention_weight_max": 60.10306167602539,
      "activations/layer17_attention_weight_min": -39.063812255859375,
      "activations/layer18_attention_weight_max": 67.75289916992188,
      "activations/layer18_attention_weight_min": -41.099365234375,
      "activations/layer19_attention_weight_max": 24.66551399230957,
      "activations/layer19_attention_weight_min": -13.299826622009277,
      "activations/layer1_attention_weight_max": 16.252742767333984,
      "activations/layer1_attention_weight_min": -10.83862590789795,
      "activations/layer20_attention_weight_max": 17.311765670776367,
      "activations/layer20_attention_weight_min": -14.368671417236328,
      "activations/layer21_attention_weight_max": 43.563323974609375,
      "activations/layer21_attention_weight_min": -22.382614135742188,
      "activations/layer22_attention_weight_max": 38.415855407714844,
      "activations/layer22_attention_weight_min": -20.62544059753418,
      "activations/layer23_attention_weight_max": 23.434659957885742,
      "activations/layer23_attention_weight_min": -14.19467544555664,
      "activations/layer2_attention_weight_max": 26.902488708496094,
      "activations/layer2_attention_weight_min": -25.188892364501953,
      "activations/layer3_attention_weight_max": 52.057186126708984,
      "activations/layer3_attention_weight_min": -54.186851501464844,
      "activations/layer4_attention_weight_max": 82.81291961669922,
      "activations/layer4_attention_weight_min": -75.15238189697266,
      "activations/layer5_attention_weight_max": 133.5000457763672,
      "activations/layer5_attention_weight_min": -124.70112609863281,
      "activations/layer6_attention_weight_max": 61.47234344482422,
      "activations/layer6_attention_weight_min": -58.46278381347656,
      "activations/layer7_attention_weight_max": 58.580745697021484,
      "activations/layer7_attention_weight_min": -55.97943878173828,
      "activations/layer8_attention_weight_max": 50.84337615966797,
      "activations/layer8_attention_weight_min": -46.89438247680664,
      "activations/layer9_attention_weight_max": 75.86385345458984,
      "activations/layer9_attention_weight_min": -67.9161605834961,
      "epoch": 1.12,
      "learning_rate": 0.00014425492424242423,
      "loss": 3.0756,
      "step": 19200
    },
    {
      "activations/layer0_attention_weight_max": 13.569469451904297,
      "activations/layer0_attention_weight_min": -11.159701347351074,
      "activations/layer10_attention_weight_max": 65.38349914550781,
      "activations/layer10_attention_weight_min": -54.906089782714844,
      "activations/layer11_attention_weight_max": 43.07428741455078,
      "activations/layer11_attention_weight_min": -36.44908905029297,
      "activations/layer12_attention_weight_max": 30.902524948120117,
      "activations/layer12_attention_weight_min": -25.909156799316406,
      "activations/layer13_attention_weight_max": 63.43522262573242,
      "activations/layer13_attention_weight_min": -49.26545333862305,
      "activations/layer14_attention_weight_max": 63.845794677734375,
      "activations/layer14_attention_weight_min": -43.088584899902344,
      "activations/layer15_attention_weight_max": 51.34577560424805,
      "activations/layer15_attention_weight_min": -38.15165710449219,
      "activations/layer16_attention_weight_max": 48.805320739746094,
      "activations/layer16_attention_weight_min": -30.7271728515625,
      "activations/layer17_attention_weight_max": 64.99039459228516,
      "activations/layer17_attention_weight_min": -37.07876205444336,
      "activations/layer18_attention_weight_max": 67.3595199584961,
      "activations/layer18_attention_weight_min": -39.578826904296875,
      "activations/layer19_attention_weight_max": 22.645715713500977,
      "activations/layer19_attention_weight_min": -14.069757461547852,
      "activations/layer1_attention_weight_max": 15.496447563171387,
      "activations/layer1_attention_weight_min": -11.555240631103516,
      "activations/layer20_attention_weight_max": 19.879276275634766,
      "activations/layer20_attention_weight_min": -14.64490032196045,
      "activations/layer21_attention_weight_max": 42.262664794921875,
      "activations/layer21_attention_weight_min": -18.704116821289062,
      "activations/layer22_attention_weight_max": 36.73920440673828,
      "activations/layer22_attention_weight_min": -16.55809783935547,
      "activations/layer23_attention_weight_max": 24.340984344482422,
      "activations/layer23_attention_weight_min": -16.491256713867188,
      "activations/layer2_attention_weight_max": 26.26116180419922,
      "activations/layer2_attention_weight_min": -23.623891830444336,
      "activations/layer3_attention_weight_max": 53.8831672668457,
      "activations/layer3_attention_weight_min": -51.53693389892578,
      "activations/layer4_attention_weight_max": 80.75444793701172,
      "activations/layer4_attention_weight_min": -73.11066436767578,
      "activations/layer5_attention_weight_max": 129.00177001953125,
      "activations/layer5_attention_weight_min": -118.57066345214844,
      "activations/layer6_attention_weight_max": 61.24481201171875,
      "activations/layer6_attention_weight_min": -54.44752502441406,
      "activations/layer7_attention_weight_max": 59.516998291015625,
      "activations/layer7_attention_weight_min": -55.05697250366211,
      "activations/layer8_attention_weight_max": 50.75749588012695,
      "activations/layer8_attention_weight_min": -48.84374237060547,
      "activations/layer9_attention_weight_max": 75.01532745361328,
      "activations/layer9_attention_weight_min": -63.420745849609375,
      "epoch": 1.12,
      "learning_rate": 0.00014423598484848485,
      "loss": 3.0817,
      "step": 19250
    },
    {
      "activations/layer0_attention_weight_max": 15.314531326293945,
      "activations/layer0_attention_weight_min": -11.28039264678955,
      "activations/layer10_attention_weight_max": 69.52574157714844,
      "activations/layer10_attention_weight_min": -60.74037551879883,
      "activations/layer11_attention_weight_max": 47.055198669433594,
      "activations/layer11_attention_weight_min": -38.092002868652344,
      "activations/layer12_attention_weight_max": 30.94778823852539,
      "activations/layer12_attention_weight_min": -26.013751983642578,
      "activations/layer13_attention_weight_max": 59.98091506958008,
      "activations/layer13_attention_weight_min": -47.303531646728516,
      "activations/layer14_attention_weight_max": 57.392154693603516,
      "activations/layer14_attention_weight_min": -44.60309600830078,
      "activations/layer15_attention_weight_max": 54.62070083618164,
      "activations/layer15_attention_weight_min": -39.94361114501953,
      "activations/layer16_attention_weight_max": 52.37419128417969,
      "activations/layer16_attention_weight_min": -34.38750076293945,
      "activations/layer17_attention_weight_max": 61.32719039916992,
      "activations/layer17_attention_weight_min": -36.83538818359375,
      "activations/layer18_attention_weight_max": 74.98329162597656,
      "activations/layer18_attention_weight_min": -38.42626190185547,
      "activations/layer19_attention_weight_max": 20.216581344604492,
      "activations/layer19_attention_weight_min": -13.143458366394043,
      "activations/layer1_attention_weight_max": 15.828530311584473,
      "activations/layer1_attention_weight_min": -10.7439603805542,
      "activations/layer20_attention_weight_max": 17.178739547729492,
      "activations/layer20_attention_weight_min": -14.083901405334473,
      "activations/layer21_attention_weight_max": 38.9583740234375,
      "activations/layer21_attention_weight_min": -19.813003540039062,
      "activations/layer22_attention_weight_max": 34.79912185668945,
      "activations/layer22_attention_weight_min": -18.516626358032227,
      "activations/layer23_attention_weight_max": 24.146787643432617,
      "activations/layer23_attention_weight_min": -15.377310752868652,
      "activations/layer2_attention_weight_max": 24.348140716552734,
      "activations/layer2_attention_weight_min": -22.669769287109375,
      "activations/layer3_attention_weight_max": 52.13101577758789,
      "activations/layer3_attention_weight_min": -54.127532958984375,
      "activations/layer4_attention_weight_max": 82.21700286865234,
      "activations/layer4_attention_weight_min": -76.59252166748047,
      "activations/layer5_attention_weight_max": 132.00894165039062,
      "activations/layer5_attention_weight_min": -117.14512634277344,
      "activations/layer6_attention_weight_max": 63.618072509765625,
      "activations/layer6_attention_weight_min": -57.19526290893555,
      "activations/layer7_attention_weight_max": 63.61814498901367,
      "activations/layer7_attention_weight_min": -55.98512649536133,
      "activations/layer8_attention_weight_max": 56.152748107910156,
      "activations/layer8_attention_weight_min": -48.484580993652344,
      "activations/layer9_attention_weight_max": 71.72369384765625,
      "activations/layer9_attention_weight_min": -60.92788314819336,
      "epoch": 1.12,
      "learning_rate": 0.00014421704545454546,
      "loss": 3.0593,
      "step": 19300
    },
    {
      "activations/layer0_attention_weight_max": 14.780339241027832,
      "activations/layer0_attention_weight_min": -11.680119514465332,
      "activations/layer10_attention_weight_max": 75.04772186279297,
      "activations/layer10_attention_weight_min": -63.71190643310547,
      "activations/layer11_attention_weight_max": 44.254119873046875,
      "activations/layer11_attention_weight_min": -36.877742767333984,
      "activations/layer12_attention_weight_max": 33.44063186645508,
      "activations/layer12_attention_weight_min": -23.264179229736328,
      "activations/layer13_attention_weight_max": 74.2439956665039,
      "activations/layer13_attention_weight_min": -43.50395202636719,
      "activations/layer14_attention_weight_max": 63.050411224365234,
      "activations/layer14_attention_weight_min": -36.86037063598633,
      "activations/layer15_attention_weight_max": 64.09236907958984,
      "activations/layer15_attention_weight_min": -39.18880844116211,
      "activations/layer16_attention_weight_max": 54.57966232299805,
      "activations/layer16_attention_weight_min": -35.74249267578125,
      "activations/layer17_attention_weight_max": 65.73619842529297,
      "activations/layer17_attention_weight_min": -39.250179290771484,
      "activations/layer18_attention_weight_max": 79.84818267822266,
      "activations/layer18_attention_weight_min": -41.83701705932617,
      "activations/layer19_attention_weight_max": 25.587261199951172,
      "activations/layer19_attention_weight_min": -14.245981216430664,
      "activations/layer1_attention_weight_max": 14.605436325073242,
      "activations/layer1_attention_weight_min": -10.890798568725586,
      "activations/layer20_attention_weight_max": 19.551929473876953,
      "activations/layer20_attention_weight_min": -15.289711952209473,
      "activations/layer21_attention_weight_max": 45.393104553222656,
      "activations/layer21_attention_weight_min": -23.661924362182617,
      "activations/layer22_attention_weight_max": 39.22144317626953,
      "activations/layer22_attention_weight_min": -17.987703323364258,
      "activations/layer23_attention_weight_max": 23.90169334411621,
      "activations/layer23_attention_weight_min": -13.822175979614258,
      "activations/layer2_attention_weight_max": 24.613487243652344,
      "activations/layer2_attention_weight_min": -24.144710540771484,
      "activations/layer3_attention_weight_max": 55.445220947265625,
      "activations/layer3_attention_weight_min": -57.231502532958984,
      "activations/layer4_attention_weight_max": 86.19113159179688,
      "activations/layer4_attention_weight_min": -82.53521728515625,
      "activations/layer5_attention_weight_max": 141.54702758789062,
      "activations/layer5_attention_weight_min": -124.30049896240234,
      "activations/layer6_attention_weight_max": 66.19802856445312,
      "activations/layer6_attention_weight_min": -58.848388671875,
      "activations/layer7_attention_weight_max": 70.0279541015625,
      "activations/layer7_attention_weight_min": -56.41486358642578,
      "activations/layer8_attention_weight_max": 54.112857818603516,
      "activations/layer8_attention_weight_min": -45.496559143066406,
      "activations/layer9_attention_weight_max": 75.59808349609375,
      "activations/layer9_attention_weight_min": -65.67330169677734,
      "epoch": 1.12,
      "learning_rate": 0.00014419810606060605,
      "loss": 3.0593,
      "step": 19350
    },
    {
      "activations/layer0_attention_weight_max": 13.697087287902832,
      "activations/layer0_attention_weight_min": -11.898000717163086,
      "activations/layer10_attention_weight_max": 66.85450744628906,
      "activations/layer10_attention_weight_min": -63.04912567138672,
      "activations/layer11_attention_weight_max": 44.13897705078125,
      "activations/layer11_attention_weight_min": -43.60375213623047,
      "activations/layer12_attention_weight_max": 28.232765197753906,
      "activations/layer12_attention_weight_min": -25.05388069152832,
      "activations/layer13_attention_weight_max": 55.678192138671875,
      "activations/layer13_attention_weight_min": -44.38523864746094,
      "activations/layer14_attention_weight_max": 44.23939514160156,
      "activations/layer14_attention_weight_min": -37.544654846191406,
      "activations/layer15_attention_weight_max": 52.55228805541992,
      "activations/layer15_attention_weight_min": -41.95478057861328,
      "activations/layer16_attention_weight_max": 44.225833892822266,
      "activations/layer16_attention_weight_min": -35.831871032714844,
      "activations/layer17_attention_weight_max": 52.5189094543457,
      "activations/layer17_attention_weight_min": -37.382869720458984,
      "activations/layer18_attention_weight_max": 66.26371765136719,
      "activations/layer18_attention_weight_min": -41.47671890258789,
      "activations/layer19_attention_weight_max": 18.910778045654297,
      "activations/layer19_attention_weight_min": -13.609933853149414,
      "activations/layer1_attention_weight_max": 17.093355178833008,
      "activations/layer1_attention_weight_min": -12.187176704406738,
      "activations/layer20_attention_weight_max": 17.441221237182617,
      "activations/layer20_attention_weight_min": -15.65191650390625,
      "activations/layer21_attention_weight_max": 37.360260009765625,
      "activations/layer21_attention_weight_min": -20.686782836914062,
      "activations/layer22_attention_weight_max": 34.992183685302734,
      "activations/layer22_attention_weight_min": -18.48829460144043,
      "activations/layer23_attention_weight_max": 22.62055015563965,
      "activations/layer23_attention_weight_min": -16.49264907836914,
      "activations/layer2_attention_weight_max": 24.400190353393555,
      "activations/layer2_attention_weight_min": -23.683319091796875,
      "activations/layer3_attention_weight_max": 57.9306526184082,
      "activations/layer3_attention_weight_min": -58.32597732543945,
      "activations/layer4_attention_weight_max": 83.38470458984375,
      "activations/layer4_attention_weight_min": -76.86387634277344,
      "activations/layer5_attention_weight_max": 127.97066497802734,
      "activations/layer5_attention_weight_min": -120.01592254638672,
      "activations/layer6_attention_weight_max": 62.22201919555664,
      "activations/layer6_attention_weight_min": -56.481689453125,
      "activations/layer7_attention_weight_max": 61.91579818725586,
      "activations/layer7_attention_weight_min": -56.511962890625,
      "activations/layer8_attention_weight_max": 50.49258804321289,
      "activations/layer8_attention_weight_min": -48.04086685180664,
      "activations/layer9_attention_weight_max": 69.30062103271484,
      "activations/layer9_attention_weight_min": -66.47612762451172,
      "epoch": 1.13,
      "learning_rate": 0.00014417916666666666,
      "loss": 3.0728,
      "step": 19400
    },
    {
      "activations/layer0_attention_weight_max": 14.037944793701172,
      "activations/layer0_attention_weight_min": -12.091784477233887,
      "activations/layer10_attention_weight_max": 61.19444274902344,
      "activations/layer10_attention_weight_min": -52.57612991333008,
      "activations/layer11_attention_weight_max": 43.23066329956055,
      "activations/layer11_attention_weight_min": -38.491065979003906,
      "activations/layer12_attention_weight_max": 26.546710968017578,
      "activations/layer12_attention_weight_min": -23.485509872436523,
      "activations/layer13_attention_weight_max": 53.645931243896484,
      "activations/layer13_attention_weight_min": -44.21791458129883,
      "activations/layer14_attention_weight_max": 45.29954528808594,
      "activations/layer14_attention_weight_min": -34.29632568359375,
      "activations/layer15_attention_weight_max": 56.266536712646484,
      "activations/layer15_attention_weight_min": -40.24529266357422,
      "activations/layer16_attention_weight_max": 49.28440475463867,
      "activations/layer16_attention_weight_min": -38.37263107299805,
      "activations/layer17_attention_weight_max": 65.38618469238281,
      "activations/layer17_attention_weight_min": -38.32710647583008,
      "activations/layer18_attention_weight_max": 65.00633239746094,
      "activations/layer18_attention_weight_min": -42.444175720214844,
      "activations/layer19_attention_weight_max": 21.452999114990234,
      "activations/layer19_attention_weight_min": -15.128084182739258,
      "activations/layer1_attention_weight_max": 14.7440767288208,
      "activations/layer1_attention_weight_min": -11.071642875671387,
      "activations/layer20_attention_weight_max": 20.374834060668945,
      "activations/layer20_attention_weight_min": -15.35693073272705,
      "activations/layer21_attention_weight_max": 40.08805847167969,
      "activations/layer21_attention_weight_min": -21.0435791015625,
      "activations/layer22_attention_weight_max": 34.7166748046875,
      "activations/layer22_attention_weight_min": -17.86301040649414,
      "activations/layer23_attention_weight_max": 22.459880828857422,
      "activations/layer23_attention_weight_min": -13.75937557220459,
      "activations/layer2_attention_weight_max": 25.598236083984375,
      "activations/layer2_attention_weight_min": -23.997825622558594,
      "activations/layer3_attention_weight_max": 52.663455963134766,
      "activations/layer3_attention_weight_min": -55.38653564453125,
      "activations/layer4_attention_weight_max": 84.12764739990234,
      "activations/layer4_attention_weight_min": -78.75052642822266,
      "activations/layer5_attention_weight_max": 135.10302734375,
      "activations/layer5_attention_weight_min": -119.6422348022461,
      "activations/layer6_attention_weight_max": 66.66828918457031,
      "activations/layer6_attention_weight_min": -57.28032302856445,
      "activations/layer7_attention_weight_max": 62.87804412841797,
      "activations/layer7_attention_weight_min": -56.12332534790039,
      "activations/layer8_attention_weight_max": 57.77309799194336,
      "activations/layer8_attention_weight_min": -49.10692596435547,
      "activations/layer9_attention_weight_max": 70.25123596191406,
      "activations/layer9_attention_weight_min": -64.80416107177734,
      "epoch": 1.13,
      "learning_rate": 0.00014416022727272725,
      "loss": 3.0913,
      "step": 19450
    },
    {
      "activations/layer0_attention_weight_max": 13.431999206542969,
      "activations/layer0_attention_weight_min": -12.374814987182617,
      "activations/layer10_attention_weight_max": 58.943355560302734,
      "activations/layer10_attention_weight_min": -55.26912307739258,
      "activations/layer11_attention_weight_max": 44.52456283569336,
      "activations/layer11_attention_weight_min": -41.62439727783203,
      "activations/layer12_attention_weight_max": 24.432167053222656,
      "activations/layer12_attention_weight_min": -23.370487213134766,
      "activations/layer13_attention_weight_max": 63.65678024291992,
      "activations/layer13_attention_weight_min": -49.50373458862305,
      "activations/layer14_attention_weight_max": 45.3935546875,
      "activations/layer14_attention_weight_min": -43.222328186035156,
      "activations/layer15_attention_weight_max": 47.07220458984375,
      "activations/layer15_attention_weight_min": -39.55556106567383,
      "activations/layer16_attention_weight_max": 50.7929801940918,
      "activations/layer16_attention_weight_min": -36.81093215942383,
      "activations/layer17_attention_weight_max": 53.4079475402832,
      "activations/layer17_attention_weight_min": -39.14302444458008,
      "activations/layer18_attention_weight_max": 58.737491607666016,
      "activations/layer18_attention_weight_min": -38.87871551513672,
      "activations/layer19_attention_weight_max": 21.879697799682617,
      "activations/layer19_attention_weight_min": -12.310927391052246,
      "activations/layer1_attention_weight_max": 14.55691146850586,
      "activations/layer1_attention_weight_min": -12.0907564163208,
      "activations/layer20_attention_weight_max": 18.852067947387695,
      "activations/layer20_attention_weight_min": -13.790523529052734,
      "activations/layer21_attention_weight_max": 32.53972625732422,
      "activations/layer21_attention_weight_min": -22.81293296813965,
      "activations/layer22_attention_weight_max": 34.140106201171875,
      "activations/layer22_attention_weight_min": -17.876813888549805,
      "activations/layer23_attention_weight_max": 21.584463119506836,
      "activations/layer23_attention_weight_min": -16.435272216796875,
      "activations/layer2_attention_weight_max": 25.065174102783203,
      "activations/layer2_attention_weight_min": -23.651975631713867,
      "activations/layer3_attention_weight_max": 52.34196090698242,
      "activations/layer3_attention_weight_min": -54.79347229003906,
      "activations/layer4_attention_weight_max": 79.74532318115234,
      "activations/layer4_attention_weight_min": -75.46078491210938,
      "activations/layer5_attention_weight_max": 125.30229187011719,
      "activations/layer5_attention_weight_min": -119.24012756347656,
      "activations/layer6_attention_weight_max": 58.52873229980469,
      "activations/layer6_attention_weight_min": -55.547584533691406,
      "activations/layer7_attention_weight_max": 53.82560729980469,
      "activations/layer7_attention_weight_min": -53.727237701416016,
      "activations/layer8_attention_weight_max": 44.9582405090332,
      "activations/layer8_attention_weight_min": -45.47368240356445,
      "activations/layer9_attention_weight_max": 72.54925537109375,
      "activations/layer9_attention_weight_min": -67.59564208984375,
      "epoch": 1.13,
      "learning_rate": 0.00014414128787878787,
      "loss": 3.0608,
      "step": 19500
    },
    {
      "activations/layer0_attention_weight_max": 13.694414138793945,
      "activations/layer0_attention_weight_min": -12.405379295349121,
      "activations/layer10_attention_weight_max": 64.8044204711914,
      "activations/layer10_attention_weight_min": -58.45347595214844,
      "activations/layer11_attention_weight_max": 45.65464782714844,
      "activations/layer11_attention_weight_min": -38.199951171875,
      "activations/layer12_attention_weight_max": 26.1234188079834,
      "activations/layer12_attention_weight_min": -25.12657356262207,
      "activations/layer13_attention_weight_max": 49.29977798461914,
      "activations/layer13_attention_weight_min": -37.14388656616211,
      "activations/layer14_attention_weight_max": 43.8317985534668,
      "activations/layer14_attention_weight_min": -37.51075744628906,
      "activations/layer15_attention_weight_max": 52.200965881347656,
      "activations/layer15_attention_weight_min": -40.27383804321289,
      "activations/layer16_attention_weight_max": 48.60899353027344,
      "activations/layer16_attention_weight_min": -34.454586029052734,
      "activations/layer17_attention_weight_max": 56.31441116333008,
      "activations/layer17_attention_weight_min": -39.62090301513672,
      "activations/layer18_attention_weight_max": 72.40035247802734,
      "activations/layer18_attention_weight_min": -44.359100341796875,
      "activations/layer19_attention_weight_max": 21.178483963012695,
      "activations/layer19_attention_weight_min": -13.805193901062012,
      "activations/layer1_attention_weight_max": 15.1841402053833,
      "activations/layer1_attention_weight_min": -12.821328163146973,
      "activations/layer20_attention_weight_max": 20.254648208618164,
      "activations/layer20_attention_weight_min": -15.712349891662598,
      "activations/layer21_attention_weight_max": 36.91043472290039,
      "activations/layer21_attention_weight_min": -21.510427474975586,
      "activations/layer22_attention_weight_max": 31.70742416381836,
      "activations/layer22_attention_weight_min": -16.78790283203125,
      "activations/layer23_attention_weight_max": 23.04383659362793,
      "activations/layer23_attention_weight_min": -14.902972221374512,
      "activations/layer2_attention_weight_max": 25.647764205932617,
      "activations/layer2_attention_weight_min": -22.56415557861328,
      "activations/layer3_attention_weight_max": 53.9031867980957,
      "activations/layer3_attention_weight_min": -55.16925048828125,
      "activations/layer4_attention_weight_max": 82.89525604248047,
      "activations/layer4_attention_weight_min": -74.96698760986328,
      "activations/layer5_attention_weight_max": 131.4626922607422,
      "activations/layer5_attention_weight_min": -120.50296020507812,
      "activations/layer6_attention_weight_max": 61.11821746826172,
      "activations/layer6_attention_weight_min": -59.57233428955078,
      "activations/layer7_attention_weight_max": 58.19618225097656,
      "activations/layer7_attention_weight_min": -55.037776947021484,
      "activations/layer8_attention_weight_max": 55.633811950683594,
      "activations/layer8_attention_weight_min": -47.914676666259766,
      "activations/layer9_attention_weight_max": 62.946571350097656,
      "activations/layer9_attention_weight_min": -62.802345275878906,
      "epoch": 1.14,
      "learning_rate": 0.00014412272727272726,
      "loss": 3.0604,
      "step": 19550
    },
    {
      "activations/layer0_attention_weight_max": 13.287873268127441,
      "activations/layer0_attention_weight_min": -12.092765808105469,
      "activations/layer10_attention_weight_max": 65.38520812988281,
      "activations/layer10_attention_weight_min": -57.8306999206543,
      "activations/layer11_attention_weight_max": 47.22674560546875,
      "activations/layer11_attention_weight_min": -41.51121139526367,
      "activations/layer12_attention_weight_max": 28.244474411010742,
      "activations/layer12_attention_weight_min": -23.322309494018555,
      "activations/layer13_attention_weight_max": 54.91222381591797,
      "activations/layer13_attention_weight_min": -41.487037658691406,
      "activations/layer14_attention_weight_max": 50.93422317504883,
      "activations/layer14_attention_weight_min": -36.094024658203125,
      "activations/layer15_attention_weight_max": 56.629539489746094,
      "activations/layer15_attention_weight_min": -40.0029182434082,
      "activations/layer16_attention_weight_max": 49.239715576171875,
      "activations/layer16_attention_weight_min": -32.97245407104492,
      "activations/layer17_attention_weight_max": 60.6142463684082,
      "activations/layer17_attention_weight_min": -37.222251892089844,
      "activations/layer18_attention_weight_max": 74.41252136230469,
      "activations/layer18_attention_weight_min": -39.51816177368164,
      "activations/layer19_attention_weight_max": 21.364587783813477,
      "activations/layer19_attention_weight_min": -13.087783813476562,
      "activations/layer1_attention_weight_max": 14.3287992477417,
      "activations/layer1_attention_weight_min": -11.480337142944336,
      "activations/layer20_attention_weight_max": 19.615297317504883,
      "activations/layer20_attention_weight_min": -15.273470878601074,
      "activations/layer21_attention_weight_max": 38.508331298828125,
      "activations/layer21_attention_weight_min": -20.66297149658203,
      "activations/layer22_attention_weight_max": 33.593528747558594,
      "activations/layer22_attention_weight_min": -18.93587875366211,
      "activations/layer23_attention_weight_max": 21.53639030456543,
      "activations/layer23_attention_weight_min": -14.081866264343262,
      "activations/layer2_attention_weight_max": 25.587461471557617,
      "activations/layer2_attention_weight_min": -22.619802474975586,
      "activations/layer3_attention_weight_max": 51.86073684692383,
      "activations/layer3_attention_weight_min": -52.93379592895508,
      "activations/layer4_attention_weight_max": 82.98641204833984,
      "activations/layer4_attention_weight_min": -71.5408935546875,
      "activations/layer5_attention_weight_max": 129.5845947265625,
      "activations/layer5_attention_weight_min": -122.68452453613281,
      "activations/layer6_attention_weight_max": 63.53307342529297,
      "activations/layer6_attention_weight_min": -55.566978454589844,
      "activations/layer7_attention_weight_max": 62.16989517211914,
      "activations/layer7_attention_weight_min": -56.85414505004883,
      "activations/layer8_attention_weight_max": 56.42417526245117,
      "activations/layer8_attention_weight_min": -51.2317008972168,
      "activations/layer9_attention_weight_max": 71.3797378540039,
      "activations/layer9_attention_weight_min": -71.07481384277344,
      "epoch": 1.14,
      "learning_rate": 0.00014410378787878787,
      "loss": 3.0574,
      "step": 19600
    },
    {
      "activations/layer0_attention_weight_max": 13.664948463439941,
      "activations/layer0_attention_weight_min": -12.361686706542969,
      "activations/layer10_attention_weight_max": 73.00082397460938,
      "activations/layer10_attention_weight_min": -63.65848922729492,
      "activations/layer11_attention_weight_max": 47.45451354980469,
      "activations/layer11_attention_weight_min": -45.05152893066406,
      "activations/layer12_attention_weight_max": 27.591663360595703,
      "activations/layer12_attention_weight_min": -27.125293731689453,
      "activations/layer13_attention_weight_max": 54.42399215698242,
      "activations/layer13_attention_weight_min": -40.00617599487305,
      "activations/layer14_attention_weight_max": 50.515838623046875,
      "activations/layer14_attention_weight_min": -33.86431121826172,
      "activations/layer15_attention_weight_max": 51.14393615722656,
      "activations/layer15_attention_weight_min": -37.780025482177734,
      "activations/layer16_attention_weight_max": 47.30561065673828,
      "activations/layer16_attention_weight_min": -32.90803146362305,
      "activations/layer17_attention_weight_max": 55.9875602722168,
      "activations/layer17_attention_weight_min": -35.159542083740234,
      "activations/layer18_attention_weight_max": 61.147300720214844,
      "activations/layer18_attention_weight_min": -38.66462326049805,
      "activations/layer19_attention_weight_max": 23.665618896484375,
      "activations/layer19_attention_weight_min": -13.440096855163574,
      "activations/layer1_attention_weight_max": 14.844536781311035,
      "activations/layer1_attention_weight_min": -10.222047805786133,
      "activations/layer20_attention_weight_max": 17.735050201416016,
      "activations/layer20_attention_weight_min": -14.981064796447754,
      "activations/layer21_attention_weight_max": 36.99456787109375,
      "activations/layer21_attention_weight_min": -21.145353317260742,
      "activations/layer22_attention_weight_max": 33.152366638183594,
      "activations/layer22_attention_weight_min": -17.40026092529297,
      "activations/layer23_attention_weight_max": 20.249034881591797,
      "activations/layer23_attention_weight_min": -15.074737548828125,
      "activations/layer2_attention_weight_max": 24.347740173339844,
      "activations/layer2_attention_weight_min": -23.04823875427246,
      "activations/layer3_attention_weight_max": 50.69332504272461,
      "activations/layer3_attention_weight_min": -54.936805725097656,
      "activations/layer4_attention_weight_max": 78.57587432861328,
      "activations/layer4_attention_weight_min": -74.28034973144531,
      "activations/layer5_attention_weight_max": 122.91476440429688,
      "activations/layer5_attention_weight_min": -118.9504623413086,
      "activations/layer6_attention_weight_max": 59.357582092285156,
      "activations/layer6_attention_weight_min": -54.860511779785156,
      "activations/layer7_attention_weight_max": 61.58189392089844,
      "activations/layer7_attention_weight_min": -57.63039779663086,
      "activations/layer8_attention_weight_max": 52.55162811279297,
      "activations/layer8_attention_weight_min": -52.63633728027344,
      "activations/layer9_attention_weight_max": 72.94612121582031,
      "activations/layer9_attention_weight_min": -70.54334259033203,
      "epoch": 1.14,
      "learning_rate": 0.00014408484848484846,
      "loss": 3.0595,
      "step": 19650
    },
    {
      "activations/layer0_attention_weight_max": 15.607033729553223,
      "activations/layer0_attention_weight_min": -11.822721481323242,
      "activations/layer10_attention_weight_max": 69.05250549316406,
      "activations/layer10_attention_weight_min": -59.80577087402344,
      "activations/layer11_attention_weight_max": 47.54325485229492,
      "activations/layer11_attention_weight_min": -39.45412063598633,
      "activations/layer12_attention_weight_max": 30.798908233642578,
      "activations/layer12_attention_weight_min": -22.998043060302734,
      "activations/layer13_attention_weight_max": 63.78035354614258,
      "activations/layer13_attention_weight_min": -39.932647705078125,
      "activations/layer14_attention_weight_max": 64.75687408447266,
      "activations/layer14_attention_weight_min": -37.06196594238281,
      "activations/layer15_attention_weight_max": 52.33031463623047,
      "activations/layer15_attention_weight_min": -40.97541046142578,
      "activations/layer16_attention_weight_max": 53.632606506347656,
      "activations/layer16_attention_weight_min": -32.840843200683594,
      "activations/layer17_attention_weight_max": 55.31193542480469,
      "activations/layer17_attention_weight_min": -37.0074462890625,
      "activations/layer18_attention_weight_max": 76.73774719238281,
      "activations/layer18_attention_weight_min": -41.444576263427734,
      "activations/layer19_attention_weight_max": 26.389480590820312,
      "activations/layer19_attention_weight_min": -12.154451370239258,
      "activations/layer1_attention_weight_max": 15.345731735229492,
      "activations/layer1_attention_weight_min": -10.499094009399414,
      "activations/layer20_attention_weight_max": 21.947364807128906,
      "activations/layer20_attention_weight_min": -17.03929328918457,
      "activations/layer21_attention_weight_max": 47.43949890136719,
      "activations/layer21_attention_weight_min": -20.031742095947266,
      "activations/layer22_attention_weight_max": 36.85777282714844,
      "activations/layer22_attention_weight_min": -16.312664031982422,
      "activations/layer23_attention_weight_max": 21.539594650268555,
      "activations/layer23_attention_weight_min": -14.181808471679688,
      "activations/layer2_attention_weight_max": 23.811697006225586,
      "activations/layer2_attention_weight_min": -24.326969146728516,
      "activations/layer3_attention_weight_max": 53.71371078491211,
      "activations/layer3_attention_weight_min": -58.11569595336914,
      "activations/layer4_attention_weight_max": 84.2286376953125,
      "activations/layer4_attention_weight_min": -76.65751647949219,
      "activations/layer5_attention_weight_max": 133.2034912109375,
      "activations/layer5_attention_weight_min": -128.04623413085938,
      "activations/layer6_attention_weight_max": 63.62238693237305,
      "activations/layer6_attention_weight_min": -59.64533615112305,
      "activations/layer7_attention_weight_max": 65.20152282714844,
      "activations/layer7_attention_weight_min": -55.530452728271484,
      "activations/layer8_attention_weight_max": 54.54691696166992,
      "activations/layer8_attention_weight_min": -46.891326904296875,
      "activations/layer9_attention_weight_max": 76.92792510986328,
      "activations/layer9_attention_weight_min": -66.51944732666016,
      "epoch": 1.14,
      "learning_rate": 0.00014406590909090908,
      "loss": 3.0694,
      "step": 19700
    },
    {
      "activations/layer0_attention_weight_max": 13.57890510559082,
      "activations/layer0_attention_weight_min": -12.654623985290527,
      "activations/layer10_attention_weight_max": 66.26373291015625,
      "activations/layer10_attention_weight_min": -64.28968811035156,
      "activations/layer11_attention_weight_max": 44.17655944824219,
      "activations/layer11_attention_weight_min": -41.768917083740234,
      "activations/layer12_attention_weight_max": 26.760181427001953,
      "activations/layer12_attention_weight_min": -24.07396697998047,
      "activations/layer13_attention_weight_max": 52.96106719970703,
      "activations/layer13_attention_weight_min": -45.17466735839844,
      "activations/layer14_attention_weight_max": 45.84444046020508,
      "activations/layer14_attention_weight_min": -40.500709533691406,
      "activations/layer15_attention_weight_max": 50.70033264160156,
      "activations/layer15_attention_weight_min": -39.43101501464844,
      "activations/layer16_attention_weight_max": 46.283447265625,
      "activations/layer16_attention_weight_min": -34.565269470214844,
      "activations/layer17_attention_weight_max": 58.5714111328125,
      "activations/layer17_attention_weight_min": -38.273956298828125,
      "activations/layer18_attention_weight_max": 63.17918014526367,
      "activations/layer18_attention_weight_min": -38.711299896240234,
      "activations/layer19_attention_weight_max": 21.55196189880371,
      "activations/layer19_attention_weight_min": -14.454426765441895,
      "activations/layer1_attention_weight_max": 16.139347076416016,
      "activations/layer1_attention_weight_min": -11.371696472167969,
      "activations/layer20_attention_weight_max": 21.678598403930664,
      "activations/layer20_attention_weight_min": -14.984106063842773,
      "activations/layer21_attention_weight_max": 40.81840896606445,
      "activations/layer21_attention_weight_min": -19.021961212158203,
      "activations/layer22_attention_weight_max": 35.99578094482422,
      "activations/layer22_attention_weight_min": -19.305322647094727,
      "activations/layer23_attention_weight_max": 23.408382415771484,
      "activations/layer23_attention_weight_min": -15.12045669555664,
      "activations/layer2_attention_weight_max": 25.2354793548584,
      "activations/layer2_attention_weight_min": -22.35199546813965,
      "activations/layer3_attention_weight_max": 52.28739547729492,
      "activations/layer3_attention_weight_min": -54.79051971435547,
      "activations/layer4_attention_weight_max": 82.32661437988281,
      "activations/layer4_attention_weight_min": -75.14370727539062,
      "activations/layer5_attention_weight_max": 134.20950317382812,
      "activations/layer5_attention_weight_min": -122.51109313964844,
      "activations/layer6_attention_weight_max": 63.01529312133789,
      "activations/layer6_attention_weight_min": -55.26741027832031,
      "activations/layer7_attention_weight_max": 62.194332122802734,
      "activations/layer7_attention_weight_min": -58.70591354370117,
      "activations/layer8_attention_weight_max": 53.55086898803711,
      "activations/layer8_attention_weight_min": -47.76872634887695,
      "activations/layer9_attention_weight_max": 70.72708129882812,
      "activations/layer9_attention_weight_min": -69.62520599365234,
      "epoch": 1.15,
      "learning_rate": 0.0001440469696969697,
      "loss": 3.0662,
      "step": 19750
    },
    {
      "activations/layer0_attention_weight_max": 13.050976753234863,
      "activations/layer0_attention_weight_min": -11.551180839538574,
      "activations/layer10_attention_weight_max": 80.70238494873047,
      "activations/layer10_attention_weight_min": -57.634273529052734,
      "activations/layer11_attention_weight_max": 57.33892822265625,
      "activations/layer11_attention_weight_min": -43.26459503173828,
      "activations/layer12_attention_weight_max": 35.30958557128906,
      "activations/layer12_attention_weight_min": -26.79660987854004,
      "activations/layer13_attention_weight_max": 75.87265014648438,
      "activations/layer13_attention_weight_min": -46.72563552856445,
      "activations/layer14_attention_weight_max": 60.97772216796875,
      "activations/layer14_attention_weight_min": -41.123634338378906,
      "activations/layer15_attention_weight_max": 59.62249755859375,
      "activations/layer15_attention_weight_min": -36.553611755371094,
      "activations/layer16_attention_weight_max": 54.69966506958008,
      "activations/layer16_attention_weight_min": -32.546207427978516,
      "activations/layer17_attention_weight_max": 67.5512466430664,
      "activations/layer17_attention_weight_min": -38.58111572265625,
      "activations/layer18_attention_weight_max": 74.11465454101562,
      "activations/layer18_attention_weight_min": -37.3467903137207,
      "activations/layer19_attention_weight_max": 24.233861923217773,
      "activations/layer19_attention_weight_min": -14.142613410949707,
      "activations/layer1_attention_weight_max": 15.506819725036621,
      "activations/layer1_attention_weight_min": -11.896021842956543,
      "activations/layer20_attention_weight_max": 21.305727005004883,
      "activations/layer20_attention_weight_min": -13.4783296585083,
      "activations/layer21_attention_weight_max": 39.345401763916016,
      "activations/layer21_attention_weight_min": -18.39177894592285,
      "activations/layer22_attention_weight_max": 36.57111740112305,
      "activations/layer22_attention_weight_min": -18.92472267150879,
      "activations/layer23_attention_weight_max": 27.429927825927734,
      "activations/layer23_attention_weight_min": -14.189537048339844,
      "activations/layer2_attention_weight_max": 26.016197204589844,
      "activations/layer2_attention_weight_min": -25.252422332763672,
      "activations/layer3_attention_weight_max": 59.668418884277344,
      "activations/layer3_attention_weight_min": -57.723628997802734,
      "activations/layer4_attention_weight_max": 89.62027740478516,
      "activations/layer4_attention_weight_min": -79.48651123046875,
      "activations/layer5_attention_weight_max": 150.7058563232422,
      "activations/layer5_attention_weight_min": -125.91746520996094,
      "activations/layer6_attention_weight_max": 68.14228057861328,
      "activations/layer6_attention_weight_min": -61.351051330566406,
      "activations/layer7_attention_weight_max": 69.8055191040039,
      "activations/layer7_attention_weight_min": -59.50343322753906,
      "activations/layer8_attention_weight_max": 61.80188751220703,
      "activations/layer8_attention_weight_min": -48.99028396606445,
      "activations/layer9_attention_weight_max": 78.32467651367188,
      "activations/layer9_attention_weight_min": -64.69168090820312,
      "epoch": 1.15,
      "learning_rate": 0.00014402803030303028,
      "loss": 3.0743,
      "step": 19800
    },
    {
      "activations/layer0_attention_weight_max": 13.506061553955078,
      "activations/layer0_attention_weight_min": -11.821629524230957,
      "activations/layer10_attention_weight_max": 66.404541015625,
      "activations/layer10_attention_weight_min": -59.079681396484375,
      "activations/layer11_attention_weight_max": 46.792701721191406,
      "activations/layer11_attention_weight_min": -43.64717102050781,
      "activations/layer12_attention_weight_max": 29.150815963745117,
      "activations/layer12_attention_weight_min": -24.95859146118164,
      "activations/layer13_attention_weight_max": 71.97513580322266,
      "activations/layer13_attention_weight_min": -57.07808303833008,
      "activations/layer14_attention_weight_max": 58.439510345458984,
      "activations/layer14_attention_weight_min": -44.3175048828125,
      "activations/layer15_attention_weight_max": 55.92129898071289,
      "activations/layer15_attention_weight_min": -45.379398345947266,
      "activations/layer16_attention_weight_max": 50.675315856933594,
      "activations/layer16_attention_weight_min": -36.908241271972656,
      "activations/layer17_attention_weight_max": 57.70896530151367,
      "activations/layer17_attention_weight_min": -40.652591705322266,
      "activations/layer18_attention_weight_max": 66.77603149414062,
      "activations/layer18_attention_weight_min": -38.4527702331543,
      "activations/layer19_attention_weight_max": 20.585315704345703,
      "activations/layer19_attention_weight_min": -14.392949104309082,
      "activations/layer1_attention_weight_max": 15.735760688781738,
      "activations/layer1_attention_weight_min": -11.273760795593262,
      "activations/layer20_attention_weight_max": 19.681785583496094,
      "activations/layer20_attention_weight_min": -15.23194694519043,
      "activations/layer21_attention_weight_max": 40.40514373779297,
      "activations/layer21_attention_weight_min": -24.203960418701172,
      "activations/layer22_attention_weight_max": 37.98927688598633,
      "activations/layer22_attention_weight_min": -20.542499542236328,
      "activations/layer23_attention_weight_max": 24.49978256225586,
      "activations/layer23_attention_weight_min": -15.242988586425781,
      "activations/layer2_attention_weight_max": 24.284748077392578,
      "activations/layer2_attention_weight_min": -23.267864227294922,
      "activations/layer3_attention_weight_max": 55.80632400512695,
      "activations/layer3_attention_weight_min": -57.04652404785156,
      "activations/layer4_attention_weight_max": 81.21269989013672,
      "activations/layer4_attention_weight_min": -75.29584503173828,
      "activations/layer5_attention_weight_max": 122.3806381225586,
      "activations/layer5_attention_weight_min": -119.67413330078125,
      "activations/layer6_attention_weight_max": 63.88603591918945,
      "activations/layer6_attention_weight_min": -56.78913497924805,
      "activations/layer7_attention_weight_max": 58.50099182128906,
      "activations/layer7_attention_weight_min": -55.1738166809082,
      "activations/layer8_attention_weight_max": 52.95991134643555,
      "activations/layer8_attention_weight_min": -49.84416580200195,
      "activations/layer9_attention_weight_max": 67.72148895263672,
      "activations/layer9_attention_weight_min": -72.52802276611328,
      "epoch": 1.15,
      "learning_rate": 0.0001440090909090909,
      "loss": 3.0686,
      "step": 19850
    },
    {
      "activations/layer0_attention_weight_max": 13.779187202453613,
      "activations/layer0_attention_weight_min": -12.4487886428833,
      "activations/layer10_attention_weight_max": 81.42597198486328,
      "activations/layer10_attention_weight_min": -67.97419738769531,
      "activations/layer11_attention_weight_max": 51.20161437988281,
      "activations/layer11_attention_weight_min": -44.5924186706543,
      "activations/layer12_attention_weight_max": 30.95054817199707,
      "activations/layer12_attention_weight_min": -25.28607749938965,
      "activations/layer13_attention_weight_max": 58.103336334228516,
      "activations/layer13_attention_weight_min": -41.49017333984375,
      "activations/layer14_attention_weight_max": 45.00830841064453,
      "activations/layer14_attention_weight_min": -30.175233840942383,
      "activations/layer15_attention_weight_max": 61.243499755859375,
      "activations/layer15_attention_weight_min": -39.66299057006836,
      "activations/layer16_attention_weight_max": 45.97946548461914,
      "activations/layer16_attention_weight_min": -35.91046905517578,
      "activations/layer17_attention_weight_max": 58.21400451660156,
      "activations/layer17_attention_weight_min": -36.66193389892578,
      "activations/layer18_attention_weight_max": 74.34362030029297,
      "activations/layer18_attention_weight_min": -41.26698303222656,
      "activations/layer19_attention_weight_max": 20.686498641967773,
      "activations/layer19_attention_weight_min": -12.722582817077637,
      "activations/layer1_attention_weight_max": 15.055706977844238,
      "activations/layer1_attention_weight_min": -12.001480102539062,
      "activations/layer20_attention_weight_max": 20.162151336669922,
      "activations/layer20_attention_weight_min": -14.684794425964355,
      "activations/layer21_attention_weight_max": 39.08662796020508,
      "activations/layer21_attention_weight_min": -17.66407585144043,
      "activations/layer22_attention_weight_max": 31.48265266418457,
      "activations/layer22_attention_weight_min": -17.90780258178711,
      "activations/layer23_attention_weight_max": 21.84430694580078,
      "activations/layer23_attention_weight_min": -13.794508934020996,
      "activations/layer2_attention_weight_max": 25.418975830078125,
      "activations/layer2_attention_weight_min": -24.678983688354492,
      "activations/layer3_attention_weight_max": 54.128170013427734,
      "activations/layer3_attention_weight_min": -54.569236755371094,
      "activations/layer4_attention_weight_max": 83.4597396850586,
      "activations/layer4_attention_weight_min": -76.81644439697266,
      "activations/layer5_attention_weight_max": 133.96754455566406,
      "activations/layer5_attention_weight_min": -120.85673522949219,
      "activations/layer6_attention_weight_max": 64.04989624023438,
      "activations/layer6_attention_weight_min": -60.033531188964844,
      "activations/layer7_attention_weight_max": 64.10720825195312,
      "activations/layer7_attention_weight_min": -60.573177337646484,
      "activations/layer8_attention_weight_max": 59.0237922668457,
      "activations/layer8_attention_weight_min": -54.707950592041016,
      "activations/layer9_attention_weight_max": 68.0459976196289,
      "activations/layer9_attention_weight_min": -66.50213623046875,
      "epoch": 1.16,
      "learning_rate": 0.0001439901515151515,
      "loss": 3.072,
      "step": 19900
    },
    {
      "activations/layer0_attention_weight_max": 13.285009384155273,
      "activations/layer0_attention_weight_min": -12.519055366516113,
      "activations/layer10_attention_weight_max": 71.75262451171875,
      "activations/layer10_attention_weight_min": -58.99887466430664,
      "activations/layer11_attention_weight_max": 44.3582878112793,
      "activations/layer11_attention_weight_min": -38.28455352783203,
      "activations/layer12_attention_weight_max": 30.09516143798828,
      "activations/layer12_attention_weight_min": -23.707284927368164,
      "activations/layer13_attention_weight_max": 65.7248764038086,
      "activations/layer13_attention_weight_min": -43.614097595214844,
      "activations/layer14_attention_weight_max": 56.422157287597656,
      "activations/layer14_attention_weight_min": -36.37154006958008,
      "activations/layer15_attention_weight_max": 55.024227142333984,
      "activations/layer15_attention_weight_min": -40.454193115234375,
      "activations/layer16_attention_weight_max": 50.95425796508789,
      "activations/layer16_attention_weight_min": -36.70127487182617,
      "activations/layer17_attention_weight_max": 62.50626754760742,
      "activations/layer17_attention_weight_min": -37.79878616333008,
      "activations/layer18_attention_weight_max": 61.90734100341797,
      "activations/layer18_attention_weight_min": -38.49810028076172,
      "activations/layer19_attention_weight_max": 19.85317611694336,
      "activations/layer19_attention_weight_min": -13.7857027053833,
      "activations/layer1_attention_weight_max": 15.370979309082031,
      "activations/layer1_attention_weight_min": -13.54444694519043,
      "activations/layer20_attention_weight_max": 19.55451202392578,
      "activations/layer20_attention_weight_min": -15.172112464904785,
      "activations/layer21_attention_weight_max": 39.290069580078125,
      "activations/layer21_attention_weight_min": -19.954147338867188,
      "activations/layer22_attention_weight_max": 35.891441345214844,
      "activations/layer22_attention_weight_min": -16.945613861083984,
      "activations/layer23_attention_weight_max": 24.054161071777344,
      "activations/layer23_attention_weight_min": -13.111628532409668,
      "activations/layer2_attention_weight_max": 26.489309310913086,
      "activations/layer2_attention_weight_min": -22.668724060058594,
      "activations/layer3_attention_weight_max": 52.9671630859375,
      "activations/layer3_attention_weight_min": -54.997833251953125,
      "activations/layer4_attention_weight_max": 82.49412536621094,
      "activations/layer4_attention_weight_min": -74.85404205322266,
      "activations/layer5_attention_weight_max": 127.901611328125,
      "activations/layer5_attention_weight_min": -120.74978637695312,
      "activations/layer6_attention_weight_max": 61.839134216308594,
      "activations/layer6_attention_weight_min": -56.005088806152344,
      "activations/layer7_attention_weight_max": 64.47178649902344,
      "activations/layer7_attention_weight_min": -56.80213928222656,
      "activations/layer8_attention_weight_max": 50.41161346435547,
      "activations/layer8_attention_weight_min": -45.642845153808594,
      "activations/layer9_attention_weight_max": 71.46570587158203,
      "activations/layer9_attention_weight_min": -68.00723266601562,
      "epoch": 1.16,
      "learning_rate": 0.0001439712121212121,
      "loss": 3.0668,
      "step": 19950
    },
    {
      "activations/layer0_attention_weight_max": 14.912202835083008,
      "activations/layer0_attention_weight_min": -12.416254997253418,
      "activations/layer10_attention_weight_max": 72.47696685791016,
      "activations/layer10_attention_weight_min": -60.598045349121094,
      "activations/layer11_attention_weight_max": 46.04340744018555,
      "activations/layer11_attention_weight_min": -39.54765701293945,
      "activations/layer12_attention_weight_max": 30.34512710571289,
      "activations/layer12_attention_weight_min": -24.269712448120117,
      "activations/layer13_attention_weight_max": 70.11622619628906,
      "activations/layer13_attention_weight_min": -52.63717269897461,
      "activations/layer14_attention_weight_max": 55.74242401123047,
      "activations/layer14_attention_weight_min": -47.014102935791016,
      "activations/layer15_attention_weight_max": 64.08147430419922,
      "activations/layer15_attention_weight_min": -43.85472106933594,
      "activations/layer16_attention_weight_max": 45.915504455566406,
      "activations/layer16_attention_weight_min": -35.979183197021484,
      "activations/layer17_attention_weight_max": 54.981441497802734,
      "activations/layer17_attention_weight_min": -37.6978645324707,
      "activations/layer18_attention_weight_max": 71.82572174072266,
      "activations/layer18_attention_weight_min": -41.86442565917969,
      "activations/layer19_attention_weight_max": 20.514171600341797,
      "activations/layer19_attention_weight_min": -14.253108024597168,
      "activations/layer1_attention_weight_max": 14.935792922973633,
      "activations/layer1_attention_weight_min": -10.778626441955566,
      "activations/layer20_attention_weight_max": 22.14628028869629,
      "activations/layer20_attention_weight_min": -15.339332580566406,
      "activations/layer21_attention_weight_max": 39.22285842895508,
      "activations/layer21_attention_weight_min": -19.210947036743164,
      "activations/layer22_attention_weight_max": 37.268856048583984,
      "activations/layer22_attention_weight_min": -17.690149307250977,
      "activations/layer23_attention_weight_max": 24.192245483398438,
      "activations/layer23_attention_weight_min": -16.334186553955078,
      "activations/layer2_attention_weight_max": 23.876768112182617,
      "activations/layer2_attention_weight_min": -22.736846923828125,
      "activations/layer3_attention_weight_max": 55.732177734375,
      "activations/layer3_attention_weight_min": -61.618499755859375,
      "activations/layer4_attention_weight_max": 85.31913757324219,
      "activations/layer4_attention_weight_min": -84.06597137451172,
      "activations/layer5_attention_weight_max": 136.1680908203125,
      "activations/layer5_attention_weight_min": -117.83287811279297,
      "activations/layer6_attention_weight_max": 63.888099670410156,
      "activations/layer6_attention_weight_min": -54.288143157958984,
      "activations/layer7_attention_weight_max": 65.55917358398438,
      "activations/layer7_attention_weight_min": -59.37838363647461,
      "activations/layer8_attention_weight_max": 57.47493362426758,
      "activations/layer8_attention_weight_min": -48.8770866394043,
      "activations/layer9_attention_weight_max": 70.77084350585938,
      "activations/layer9_attention_weight_min": -64.36212921142578,
      "epoch": 1.16,
      "learning_rate": 0.0001439522727272727,
      "loss": 3.066,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_loss": 3.01171875,
      "eval_runtime": 8.245,
      "eval_samples_per_second": 520.801,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_openwebtext_loss": 3.01171875,
      "eval_openwebtext_ppl": 20.322298876557472,
      "eval_openwebtext_runtime": 8.245,
      "eval_openwebtext_samples_per_second": 520.801,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_wikitext_loss": 3.294921875,
      "eval_wikitext_ppl": 26.975306541751994,
      "eval_wikitext_runtime": 1.5007,
      "eval_wikitext_samples_per_second": 303.858,
      "step": 20000
    },
    {
      "epoch": 1.16,
      "eval_lambada_loss": 3.267578125,
      "eval_lambada_ppl": 26.24769367033534,
      "eval_lambada_runtime": 9.0282,
      "eval_lambada_samples_per_second": 539.31,
      "step": 20000
    },
    {
      "activations/layer0_attention_weight_max": 14.628904342651367,
      "activations/layer0_attention_weight_min": -12.663318634033203,
      "activations/layer10_attention_weight_max": 74.51329803466797,
      "activations/layer10_attention_weight_min": -68.32827758789062,
      "activations/layer11_attention_weight_max": 49.71900177001953,
      "activations/layer11_attention_weight_min": -40.20780563354492,
      "activations/layer12_attention_weight_max": 28.547616958618164,
      "activations/layer12_attention_weight_min": -27.785602569580078,
      "activations/layer13_attention_weight_max": 65.20878601074219,
      "activations/layer13_attention_weight_min": -49.377445220947266,
      "activations/layer14_attention_weight_max": 48.30686950683594,
      "activations/layer14_attention_weight_min": -40.098487854003906,
      "activations/layer15_attention_weight_max": 55.882667541503906,
      "activations/layer15_attention_weight_min": -42.87745666503906,
      "activations/layer16_attention_weight_max": 47.777950286865234,
      "activations/layer16_attention_weight_min": -33.709259033203125,
      "activations/layer17_attention_weight_max": 53.248046875,
      "activations/layer17_attention_weight_min": -39.03973388671875,
      "activations/layer18_attention_weight_max": 69.35608673095703,
      "activations/layer18_attention_weight_min": -42.3117790222168,
      "activations/layer19_attention_weight_max": 19.48402214050293,
      "activations/layer19_attention_weight_min": -13.386674880981445,
      "activations/layer1_attention_weight_max": 15.248279571533203,
      "activations/layer1_attention_weight_min": -11.159127235412598,
      "activations/layer20_attention_weight_max": 17.01759147644043,
      "activations/layer20_attention_weight_min": -15.592927932739258,
      "activations/layer21_attention_weight_max": 35.288307189941406,
      "activations/layer21_attention_weight_min": -20.077190399169922,
      "activations/layer22_attention_weight_max": 32.028221130371094,
      "activations/layer22_attention_weight_min": -18.556306838989258,
      "activations/layer23_attention_weight_max": 22.455677032470703,
      "activations/layer23_attention_weight_min": -15.008987426757812,
      "activations/layer2_attention_weight_max": 23.710317611694336,
      "activations/layer2_attention_weight_min": -22.94274139404297,
      "activations/layer3_attention_weight_max": 55.381649017333984,
      "activations/layer3_attention_weight_min": -58.60084533691406,
      "activations/layer4_attention_weight_max": 85.01174926757812,
      "activations/layer4_attention_weight_min": -78.17189025878906,
      "activations/layer5_attention_weight_max": 135.55966186523438,
      "activations/layer5_attention_weight_min": -123.80024719238281,
      "activations/layer6_attention_weight_max": 63.47075653076172,
      "activations/layer6_attention_weight_min": -55.68977737426758,
      "activations/layer7_attention_weight_max": 61.5349235534668,
      "activations/layer7_attention_weight_min": -56.18415832519531,
      "activations/layer8_attention_weight_max": 58.49201202392578,
      "activations/layer8_attention_weight_min": -54.26607131958008,
      "activations/layer9_attention_weight_max": 80.84151458740234,
      "activations/layer9_attention_weight_min": -78.40984344482422,
      "epoch": 1.17,
      "learning_rate": 0.00014393333333333333,
      "loss": 3.0497,
      "step": 20050
    },
    {
      "activations/layer0_attention_weight_max": 13.581521987915039,
      "activations/layer0_attention_weight_min": -12.476822853088379,
      "activations/layer10_attention_weight_max": 74.62446594238281,
      "activations/layer10_attention_weight_min": -64.46778869628906,
      "activations/layer11_attention_weight_max": 41.71571350097656,
      "activations/layer11_attention_weight_min": -41.599143981933594,
      "activations/layer12_attention_weight_max": 31.26392364501953,
      "activations/layer12_attention_weight_min": -28.122419357299805,
      "activations/layer13_attention_weight_max": 52.87097930908203,
      "activations/layer13_attention_weight_min": -42.81700134277344,
      "activations/layer14_attention_weight_max": 43.72297286987305,
      "activations/layer14_attention_weight_min": -38.8654899597168,
      "activations/layer15_attention_weight_max": 53.90446090698242,
      "activations/layer15_attention_weight_min": -44.0091438293457,
      "activations/layer16_attention_weight_max": 44.859519958496094,
      "activations/layer16_attention_weight_min": -36.24675750732422,
      "activations/layer17_attention_weight_max": 52.05923843383789,
      "activations/layer17_attention_weight_min": -37.69785690307617,
      "activations/layer18_attention_weight_max": 60.88626480102539,
      "activations/layer18_attention_weight_min": -38.6658821105957,
      "activations/layer19_attention_weight_max": 20.683881759643555,
      "activations/layer19_attention_weight_min": -13.534693717956543,
      "activations/layer1_attention_weight_max": 14.919313430786133,
      "activations/layer1_attention_weight_min": -11.010000228881836,
      "activations/layer20_attention_weight_max": 18.808305740356445,
      "activations/layer20_attention_weight_min": -15.459774017333984,
      "activations/layer21_attention_weight_max": 37.608375549316406,
      "activations/layer21_attention_weight_min": -19.645488739013672,
      "activations/layer22_attention_weight_max": 35.82041549682617,
      "activations/layer22_attention_weight_min": -19.86865997314453,
      "activations/layer23_attention_weight_max": 24.74285125732422,
      "activations/layer23_attention_weight_min": -14.26498794555664,
      "activations/layer2_attention_weight_max": 26.590795516967773,
      "activations/layer2_attention_weight_min": -25.152616500854492,
      "activations/layer3_attention_weight_max": 51.104270935058594,
      "activations/layer3_attention_weight_min": -55.79752731323242,
      "activations/layer4_attention_weight_max": 78.6250991821289,
      "activations/layer4_attention_weight_min": -77.04181671142578,
      "activations/layer5_attention_weight_max": 125.4663314819336,
      "activations/layer5_attention_weight_min": -123.65715026855469,
      "activations/layer6_attention_weight_max": 60.01399230957031,
      "activations/layer6_attention_weight_min": -58.90279006958008,
      "activations/layer7_attention_weight_max": 59.27948760986328,
      "activations/layer7_attention_weight_min": -56.56355667114258,
      "activations/layer8_attention_weight_max": 51.116573333740234,
      "activations/layer8_attention_weight_min": -47.91918182373047,
      "activations/layer9_attention_weight_max": 71.3525161743164,
      "activations/layer9_attention_weight_min": -66.49620056152344,
      "epoch": 1.17,
      "learning_rate": 0.00014391439393939391,
      "loss": 3.0691,
      "step": 20100
    },
    {
      "activations/layer0_attention_weight_max": 14.993048667907715,
      "activations/layer0_attention_weight_min": -12.379611015319824,
      "activations/layer10_attention_weight_max": 63.820980072021484,
      "activations/layer10_attention_weight_min": -62.30091094970703,
      "activations/layer11_attention_weight_max": 48.59590148925781,
      "activations/layer11_attention_weight_min": -40.684051513671875,
      "activations/layer12_attention_weight_max": 26.525745391845703,
      "activations/layer12_attention_weight_min": -23.540786743164062,
      "activations/layer13_attention_weight_max": 54.31755828857422,
      "activations/layer13_attention_weight_min": -42.95247268676758,
      "activations/layer14_attention_weight_max": 48.57539367675781,
      "activations/layer14_attention_weight_min": -38.76264572143555,
      "activations/layer15_attention_weight_max": 54.426918029785156,
      "activations/layer15_attention_weight_min": -43.9091911315918,
      "activations/layer16_attention_weight_max": 48.83053207397461,
      "activations/layer16_attention_weight_min": -35.9193229675293,
      "activations/layer17_attention_weight_max": 57.27208709716797,
      "activations/layer17_attention_weight_min": -37.415626525878906,
      "activations/layer18_attention_weight_max": 58.645572662353516,
      "activations/layer18_attention_weight_min": -42.02084732055664,
      "activations/layer19_attention_weight_max": 21.295547485351562,
      "activations/layer19_attention_weight_min": -12.322937965393066,
      "activations/layer1_attention_weight_max": 15.2542142868042,
      "activations/layer1_attention_weight_min": -10.32509708404541,
      "activations/layer20_attention_weight_max": 18.561220169067383,
      "activations/layer20_attention_weight_min": -14.30481243133545,
      "activations/layer21_attention_weight_max": 35.3040657043457,
      "activations/layer21_attention_weight_min": -19.802932739257812,
      "activations/layer22_attention_weight_max": 34.04282760620117,
      "activations/layer22_attention_weight_min": -19.401811599731445,
      "activations/layer23_attention_weight_max": 21.872631072998047,
      "activations/layer23_attention_weight_min": -15.04761028289795,
      "activations/layer2_attention_weight_max": 25.048477172851562,
      "activations/layer2_attention_weight_min": -22.508644104003906,
      "activations/layer3_attention_weight_max": 52.954368591308594,
      "activations/layer3_attention_weight_min": -57.23236846923828,
      "activations/layer4_attention_weight_max": 78.37199401855469,
      "activations/layer4_attention_weight_min": -74.92407989501953,
      "activations/layer5_attention_weight_max": 126.34471130371094,
      "activations/layer5_attention_weight_min": -121.61772155761719,
      "activations/layer6_attention_weight_max": 62.47450637817383,
      "activations/layer6_attention_weight_min": -58.16857147216797,
      "activations/layer7_attention_weight_max": 58.74490737915039,
      "activations/layer7_attention_weight_min": -56.50601577758789,
      "activations/layer8_attention_weight_max": 49.8759651184082,
      "activations/layer8_attention_weight_min": -46.692909240722656,
      "activations/layer9_attention_weight_max": 67.98928833007812,
      "activations/layer9_attention_weight_min": -65.56926727294922,
      "epoch": 1.17,
      "learning_rate": 0.00014389545454545453,
      "loss": 3.0628,
      "step": 20150
    },
    {
      "activations/layer0_attention_weight_max": 15.498968124389648,
      "activations/layer0_attention_weight_min": -12.6474609375,
      "activations/layer10_attention_weight_max": 65.25846099853516,
      "activations/layer10_attention_weight_min": -56.810020446777344,
      "activations/layer11_attention_weight_max": 43.409423828125,
      "activations/layer11_attention_weight_min": -38.660400390625,
      "activations/layer12_attention_weight_max": 31.585250854492188,
      "activations/layer12_attention_weight_min": -22.97654914855957,
      "activations/layer13_attention_weight_max": 66.46563720703125,
      "activations/layer13_attention_weight_min": -51.62278366088867,
      "activations/layer14_attention_weight_max": 52.289459228515625,
      "activations/layer14_attention_weight_min": -41.99242401123047,
      "activations/layer15_attention_weight_max": 57.72928237915039,
      "activations/layer15_attention_weight_min": -44.06605529785156,
      "activations/layer16_attention_weight_max": 47.727596282958984,
      "activations/layer16_attention_weight_min": -36.64686584472656,
      "activations/layer17_attention_weight_max": 56.38352584838867,
      "activations/layer17_attention_weight_min": -38.878929138183594,
      "activations/layer18_attention_weight_max": 62.98635482788086,
      "activations/layer18_attention_weight_min": -37.895721435546875,
      "activations/layer19_attention_weight_max": 25.648244857788086,
      "activations/layer19_attention_weight_min": -13.221672058105469,
      "activations/layer1_attention_weight_max": 14.645269393920898,
      "activations/layer1_attention_weight_min": -11.061649322509766,
      "activations/layer20_attention_weight_max": 18.89066505432129,
      "activations/layer20_attention_weight_min": -15.146859169006348,
      "activations/layer21_attention_weight_max": 36.57883834838867,
      "activations/layer21_attention_weight_min": -21.22261619567871,
      "activations/layer22_attention_weight_max": 36.974369049072266,
      "activations/layer22_attention_weight_min": -18.73080825805664,
      "activations/layer23_attention_weight_max": 24.970333099365234,
      "activations/layer23_attention_weight_min": -14.527582168579102,
      "activations/layer2_attention_weight_max": 24.77780532836914,
      "activations/layer2_attention_weight_min": -23.0357666015625,
      "activations/layer3_attention_weight_max": 53.71707534790039,
      "activations/layer3_attention_weight_min": -53.44850540161133,
      "activations/layer4_attention_weight_max": 81.9570083618164,
      "activations/layer4_attention_weight_min": -72.92852020263672,
      "activations/layer5_attention_weight_max": 127.89918518066406,
      "activations/layer5_attention_weight_min": -115.25373840332031,
      "activations/layer6_attention_weight_max": 62.9897575378418,
      "activations/layer6_attention_weight_min": -58.19303512573242,
      "activations/layer7_attention_weight_max": 61.51351547241211,
      "activations/layer7_attention_weight_min": -55.757110595703125,
      "activations/layer8_attention_weight_max": 48.01832962036133,
      "activations/layer8_attention_weight_min": -46.517852783203125,
      "activations/layer9_attention_weight_max": 65.36724090576172,
      "activations/layer9_attention_weight_min": -66.9273910522461,
      "epoch": 1.17,
      "learning_rate": 0.00014387651515151514,
      "loss": 3.058,
      "step": 20200
    },
    {
      "activations/layer0_attention_weight_max": 13.691279411315918,
      "activations/layer0_attention_weight_min": -12.096360206604004,
      "activations/layer10_attention_weight_max": 70.45542907714844,
      "activations/layer10_attention_weight_min": -63.06184005737305,
      "activations/layer11_attention_weight_max": 49.87330627441406,
      "activations/layer11_attention_weight_min": -38.651485443115234,
      "activations/layer12_attention_weight_max": 31.374418258666992,
      "activations/layer12_attention_weight_min": -24.089588165283203,
      "activations/layer13_attention_weight_max": 56.26732635498047,
      "activations/layer13_attention_weight_min": -38.85818099975586,
      "activations/layer14_attention_weight_max": 60.57265090942383,
      "activations/layer14_attention_weight_min": -39.4012336730957,
      "activations/layer15_attention_weight_max": 64.38529968261719,
      "activations/layer15_attention_weight_min": -40.37589645385742,
      "activations/layer16_attention_weight_max": 54.710601806640625,
      "activations/layer16_attention_weight_min": -31.077404022216797,
      "activations/layer17_attention_weight_max": 61.37660598754883,
      "activations/layer17_attention_weight_min": -37.63941955566406,
      "activations/layer18_attention_weight_max": 68.66553497314453,
      "activations/layer18_attention_weight_min": -37.58330154418945,
      "activations/layer19_attention_weight_max": 24.39923858642578,
      "activations/layer19_attention_weight_min": -13.607867240905762,
      "activations/layer1_attention_weight_max": 14.84768295288086,
      "activations/layer1_attention_weight_min": -11.360737800598145,
      "activations/layer20_attention_weight_max": 19.993989944458008,
      "activations/layer20_attention_weight_min": -15.367965698242188,
      "activations/layer21_attention_weight_max": 42.011436462402344,
      "activations/layer21_attention_weight_min": -19.7943058013916,
      "activations/layer22_attention_weight_max": 43.80630874633789,
      "activations/layer22_attention_weight_min": -17.51386833190918,
      "activations/layer23_attention_weight_max": 24.95718765258789,
      "activations/layer23_attention_weight_min": -13.48283576965332,
      "activations/layer2_attention_weight_max": 24.51189422607422,
      "activations/layer2_attention_weight_min": -23.85475730895996,
      "activations/layer3_attention_weight_max": 53.196285247802734,
      "activations/layer3_attention_weight_min": -57.26708221435547,
      "activations/layer4_attention_weight_max": 85.8333511352539,
      "activations/layer4_attention_weight_min": -76.92362976074219,
      "activations/layer5_attention_weight_max": 136.99432373046875,
      "activations/layer5_attention_weight_min": -116.95989227294922,
      "activations/layer6_attention_weight_max": 65.58147430419922,
      "activations/layer6_attention_weight_min": -59.08511734008789,
      "activations/layer7_attention_weight_max": 65.20832061767578,
      "activations/layer7_attention_weight_min": -53.92112350463867,
      "activations/layer8_attention_weight_max": 53.383182525634766,
      "activations/layer8_attention_weight_min": -47.35032272338867,
      "activations/layer9_attention_weight_max": 68.11854553222656,
      "activations/layer9_attention_weight_min": -64.7229995727539,
      "epoch": 1.18,
      "learning_rate": 0.00014385757575757573,
      "loss": 3.0393,
      "step": 20250
    },
    {
      "activations/layer0_attention_weight_max": 13.542860984802246,
      "activations/layer0_attention_weight_min": -12.331144332885742,
      "activations/layer10_attention_weight_max": 67.06806945800781,
      "activations/layer10_attention_weight_min": -58.542198181152344,
      "activations/layer11_attention_weight_max": 48.99290466308594,
      "activations/layer11_attention_weight_min": -38.410770416259766,
      "activations/layer12_attention_weight_max": 30.8010196685791,
      "activations/layer12_attention_weight_min": -24.97136878967285,
      "activations/layer13_attention_weight_max": 64.48616790771484,
      "activations/layer13_attention_weight_min": -53.656700134277344,
      "activations/layer14_attention_weight_max": 57.96723556518555,
      "activations/layer14_attention_weight_min": -47.71144485473633,
      "activations/layer15_attention_weight_max": 53.34613037109375,
      "activations/layer15_attention_weight_min": -41.947879791259766,
      "activations/layer16_attention_weight_max": 42.28889846801758,
      "activations/layer16_attention_weight_min": -33.43815994262695,
      "activations/layer17_attention_weight_max": 52.376304626464844,
      "activations/layer17_attention_weight_min": -40.65739059448242,
      "activations/layer18_attention_weight_max": 57.14545822143555,
      "activations/layer18_attention_weight_min": -36.62562942504883,
      "activations/layer19_attention_weight_max": 20.49802017211914,
      "activations/layer19_attention_weight_min": -15.321423530578613,
      "activations/layer1_attention_weight_max": 14.38935661315918,
      "activations/layer1_attention_weight_min": -11.620786666870117,
      "activations/layer20_attention_weight_max": 16.85696792602539,
      "activations/layer20_attention_weight_min": -14.76501178741455,
      "activations/layer21_attention_weight_max": 33.45374298095703,
      "activations/layer21_attention_weight_min": -19.376352310180664,
      "activations/layer22_attention_weight_max": 30.133392333984375,
      "activations/layer22_attention_weight_min": -15.911670684814453,
      "activations/layer23_attention_weight_max": 21.643081665039062,
      "activations/layer23_attention_weight_min": -15.978923797607422,
      "activations/layer2_attention_weight_max": 25.62118148803711,
      "activations/layer2_attention_weight_min": -23.437362670898438,
      "activations/layer3_attention_weight_max": 56.590518951416016,
      "activations/layer3_attention_weight_min": -59.077274322509766,
      "activations/layer4_attention_weight_max": 87.3178939819336,
      "activations/layer4_attention_weight_min": -76.47212982177734,
      "activations/layer5_attention_weight_max": 138.48239135742188,
      "activations/layer5_attention_weight_min": -119.072265625,
      "activations/layer6_attention_weight_max": 67.12261199951172,
      "activations/layer6_attention_weight_min": -60.051597595214844,
      "activations/layer7_attention_weight_max": 64.5809326171875,
      "activations/layer7_attention_weight_min": -56.83692169189453,
      "activations/layer8_attention_weight_max": 51.166099548339844,
      "activations/layer8_attention_weight_min": -45.962669372558594,
      "activations/layer9_attention_weight_max": 76.78164672851562,
      "activations/layer9_attention_weight_min": -58.3398551940918,
      "epoch": 1.18,
      "learning_rate": 0.00014383863636363635,
      "loss": 3.0609,
      "step": 20300
    },
    {
      "activations/layer0_attention_weight_max": 14.201014518737793,
      "activations/layer0_attention_weight_min": -11.60360050201416,
      "activations/layer10_attention_weight_max": 68.78491973876953,
      "activations/layer10_attention_weight_min": -63.583953857421875,
      "activations/layer11_attention_weight_max": 43.57210922241211,
      "activations/layer11_attention_weight_min": -40.17204284667969,
      "activations/layer12_attention_weight_max": 25.25497055053711,
      "activations/layer12_attention_weight_min": -22.612361907958984,
      "activations/layer13_attention_weight_max": 52.572044372558594,
      "activations/layer13_attention_weight_min": -46.20464324951172,
      "activations/layer14_attention_weight_max": 53.88115692138672,
      "activations/layer14_attention_weight_min": -38.670257568359375,
      "activations/layer15_attention_weight_max": 51.24778366088867,
      "activations/layer15_attention_weight_min": -38.90823745727539,
      "activations/layer16_attention_weight_max": 48.700157165527344,
      "activations/layer16_attention_weight_min": -33.36183547973633,
      "activations/layer17_attention_weight_max": 55.579734802246094,
      "activations/layer17_attention_weight_min": -39.24895095825195,
      "activations/layer18_attention_weight_max": 65.9275131225586,
      "activations/layer18_attention_weight_min": -40.49102783203125,
      "activations/layer19_attention_weight_max": 19.017393112182617,
      "activations/layer19_attention_weight_min": -13.654088020324707,
      "activations/layer1_attention_weight_max": 14.830638885498047,
      "activations/layer1_attention_weight_min": -11.246264457702637,
      "activations/layer20_attention_weight_max": 18.54123878479004,
      "activations/layer20_attention_weight_min": -15.552776336669922,
      "activations/layer21_attention_weight_max": 37.122344970703125,
      "activations/layer21_attention_weight_min": -18.487396240234375,
      "activations/layer22_attention_weight_max": 34.031898498535156,
      "activations/layer22_attention_weight_min": -17.99166488647461,
      "activations/layer23_attention_weight_max": 22.732168197631836,
      "activations/layer23_attention_weight_min": -13.786952018737793,
      "activations/layer2_attention_weight_max": 24.746517181396484,
      "activations/layer2_attention_weight_min": -24.405315399169922,
      "activations/layer3_attention_weight_max": 54.67093276977539,
      "activations/layer3_attention_weight_min": -57.5316047668457,
      "activations/layer4_attention_weight_max": 82.60309600830078,
      "activations/layer4_attention_weight_min": -76.73503875732422,
      "activations/layer5_attention_weight_max": 129.08718872070312,
      "activations/layer5_attention_weight_min": -116.93531799316406,
      "activations/layer6_attention_weight_max": 62.632293701171875,
      "activations/layer6_attention_weight_min": -55.7661018371582,
      "activations/layer7_attention_weight_max": 60.55964279174805,
      "activations/layer7_attention_weight_min": -55.795379638671875,
      "activations/layer8_attention_weight_max": 52.88304138183594,
      "activations/layer8_attention_weight_min": -47.838897705078125,
      "activations/layer9_attention_weight_max": 66.91506958007812,
      "activations/layer9_attention_weight_min": -64.1552505493164,
      "epoch": 1.18,
      "learning_rate": 0.00014381969696969696,
      "loss": 3.0635,
      "step": 20350
    },
    {
      "activations/layer0_attention_weight_max": 14.947851181030273,
      "activations/layer0_attention_weight_min": -11.657344818115234,
      "activations/layer10_attention_weight_max": 73.60746002197266,
      "activations/layer10_attention_weight_min": -58.347103118896484,
      "activations/layer11_attention_weight_max": 47.497127532958984,
      "activations/layer11_attention_weight_min": -38.975608825683594,
      "activations/layer12_attention_weight_max": 31.836286544799805,
      "activations/layer12_attention_weight_min": -22.767513275146484,
      "activations/layer13_attention_weight_max": 65.99824523925781,
      "activations/layer13_attention_weight_min": -43.350345611572266,
      "activations/layer14_attention_weight_max": 65.8263168334961,
      "activations/layer14_attention_weight_min": -39.634193420410156,
      "activations/layer15_attention_weight_max": 58.24298095703125,
      "activations/layer15_attention_weight_min": -40.65092468261719,
      "activations/layer16_attention_weight_max": 53.80998229980469,
      "activations/layer16_attention_weight_min": -35.40287780761719,
      "activations/layer17_attention_weight_max": 59.30487060546875,
      "activations/layer17_attention_weight_min": -37.87017059326172,
      "activations/layer18_attention_weight_max": 81.19742584228516,
      "activations/layer18_attention_weight_min": -40.35399627685547,
      "activations/layer19_attention_weight_max": 23.245656967163086,
      "activations/layer19_attention_weight_min": -13.098695755004883,
      "activations/layer1_attention_weight_max": 15.70783519744873,
      "activations/layer1_attention_weight_min": -11.826573371887207,
      "activations/layer20_attention_weight_max": 21.420473098754883,
      "activations/layer20_attention_weight_min": -17.316740036010742,
      "activations/layer21_attention_weight_max": 38.04707717895508,
      "activations/layer21_attention_weight_min": -21.625642776489258,
      "activations/layer22_attention_weight_max": 39.69438934326172,
      "activations/layer22_attention_weight_min": -21.31096839904785,
      "activations/layer23_attention_weight_max": 23.57863998413086,
      "activations/layer23_attention_weight_min": -16.84190559387207,
      "activations/layer2_attention_weight_max": 26.530303955078125,
      "activations/layer2_attention_weight_min": -23.86347007751465,
      "activations/layer3_attention_weight_max": 52.92661666870117,
      "activations/layer3_attention_weight_min": -56.83753967285156,
      "activations/layer4_attention_weight_max": 84.31230926513672,
      "activations/layer4_attention_weight_min": -76.71336364746094,
      "activations/layer5_attention_weight_max": 130.08621215820312,
      "activations/layer5_attention_weight_min": -119.98079681396484,
      "activations/layer6_attention_weight_max": 64.48173522949219,
      "activations/layer6_attention_weight_min": -58.52979278564453,
      "activations/layer7_attention_weight_max": 67.75800323486328,
      "activations/layer7_attention_weight_min": -56.50075912475586,
      "activations/layer8_attention_weight_max": 58.81172561645508,
      "activations/layer8_attention_weight_min": -49.43186950683594,
      "activations/layer9_attention_weight_max": 70.47248077392578,
      "activations/layer9_attention_weight_min": -69.30870819091797,
      "epoch": 1.19,
      "learning_rate": 0.00014380075757575755,
      "loss": 3.0536,
      "step": 20400
    },
    {
      "activations/layer0_attention_weight_max": 14.243608474731445,
      "activations/layer0_attention_weight_min": -12.06267261505127,
      "activations/layer10_attention_weight_max": 70.22605895996094,
      "activations/layer10_attention_weight_min": -64.22413635253906,
      "activations/layer11_attention_weight_max": 51.396331787109375,
      "activations/layer11_attention_weight_min": -44.061946868896484,
      "activations/layer12_attention_weight_max": 30.023439407348633,
      "activations/layer12_attention_weight_min": -25.438199996948242,
      "activations/layer13_attention_weight_max": 77.17156982421875,
      "activations/layer13_attention_weight_min": -50.43159484863281,
      "activations/layer14_attention_weight_max": 89.72122192382812,
      "activations/layer14_attention_weight_min": -50.115482330322266,
      "activations/layer15_attention_weight_max": 56.883811950683594,
      "activations/layer15_attention_weight_min": -39.370792388916016,
      "activations/layer16_attention_weight_max": 59.537391662597656,
      "activations/layer16_attention_weight_min": -35.711181640625,
      "activations/layer17_attention_weight_max": 61.36980056762695,
      "activations/layer17_attention_weight_min": -37.5942497253418,
      "activations/layer18_attention_weight_max": 78.4292221069336,
      "activations/layer18_attention_weight_min": -41.64018249511719,
      "activations/layer19_attention_weight_max": 22.47197723388672,
      "activations/layer19_attention_weight_min": -15.257128715515137,
      "activations/layer1_attention_weight_max": 14.991561889648438,
      "activations/layer1_attention_weight_min": -11.347742080688477,
      "activations/layer20_attention_weight_max": 20.809619903564453,
      "activations/layer20_attention_weight_min": -13.791531562805176,
      "activations/layer21_attention_weight_max": 34.67242431640625,
      "activations/layer21_attention_weight_min": -18.603229522705078,
      "activations/layer22_attention_weight_max": 37.021663665771484,
      "activations/layer22_attention_weight_min": -18.243350982666016,
      "activations/layer23_attention_weight_max": 23.263717651367188,
      "activations/layer23_attention_weight_min": -14.531009674072266,
      "activations/layer2_attention_weight_max": 25.19430923461914,
      "activations/layer2_attention_weight_min": -23.05535888671875,
      "activations/layer3_attention_weight_max": 55.79596710205078,
      "activations/layer3_attention_weight_min": -55.68264389038086,
      "activations/layer4_attention_weight_max": 87.34788513183594,
      "activations/layer4_attention_weight_min": -81.00020599365234,
      "activations/layer5_attention_weight_max": 145.57089233398438,
      "activations/layer5_attention_weight_min": -122.1600570678711,
      "activations/layer6_attention_weight_max": 64.4250717163086,
      "activations/layer6_attention_weight_min": -56.814048767089844,
      "activations/layer7_attention_weight_max": 68.42063903808594,
      "activations/layer7_attention_weight_min": -64.96338653564453,
      "activations/layer8_attention_weight_max": 51.28252029418945,
      "activations/layer8_attention_weight_min": -51.304649353027344,
      "activations/layer9_attention_weight_max": 82.54066467285156,
      "activations/layer9_attention_weight_min": -71.41938018798828,
      "epoch": 1.19,
      "learning_rate": 0.00014378181818181817,
      "loss": 3.057,
      "step": 20450
    },
    {
      "activations/layer0_attention_weight_max": 14.49277400970459,
      "activations/layer0_attention_weight_min": -12.446639060974121,
      "activations/layer10_attention_weight_max": 64.36534881591797,
      "activations/layer10_attention_weight_min": -54.608604431152344,
      "activations/layer11_attention_weight_max": 47.7691650390625,
      "activations/layer11_attention_weight_min": -41.025550842285156,
      "activations/layer12_attention_weight_max": 30.785856246948242,
      "activations/layer12_attention_weight_min": -30.042644500732422,
      "activations/layer13_attention_weight_max": 65.578857421875,
      "activations/layer13_attention_weight_min": -43.603187561035156,
      "activations/layer14_attention_weight_max": 60.87991714477539,
      "activations/layer14_attention_weight_min": -39.3794059753418,
      "activations/layer15_attention_weight_max": 60.30963897705078,
      "activations/layer15_attention_weight_min": -39.94048309326172,
      "activations/layer16_attention_weight_max": 53.87165832519531,
      "activations/layer16_attention_weight_min": -32.95048141479492,
      "activations/layer17_attention_weight_max": 66.70795440673828,
      "activations/layer17_attention_weight_min": -37.93333435058594,
      "activations/layer18_attention_weight_max": 60.8004264831543,
      "activations/layer18_attention_weight_min": -35.08292770385742,
      "activations/layer19_attention_weight_max": 20.665401458740234,
      "activations/layer19_attention_weight_min": -13.918899536132812,
      "activations/layer1_attention_weight_max": 16.168060302734375,
      "activations/layer1_attention_weight_min": -13.239766120910645,
      "activations/layer20_attention_weight_max": 20.53108024597168,
      "activations/layer20_attention_weight_min": -14.912339210510254,
      "activations/layer21_attention_weight_max": 34.01228332519531,
      "activations/layer21_attention_weight_min": -18.44043731689453,
      "activations/layer22_attention_weight_max": 32.53049850463867,
      "activations/layer22_attention_weight_min": -19.73943519592285,
      "activations/layer23_attention_weight_max": 22.753538131713867,
      "activations/layer23_attention_weight_min": -13.473011016845703,
      "activations/layer2_attention_weight_max": 27.376251220703125,
      "activations/layer2_attention_weight_min": -25.76789665222168,
      "activations/layer3_attention_weight_max": 56.58842849731445,
      "activations/layer3_attention_weight_min": -58.57339096069336,
      "activations/layer4_attention_weight_max": 82.22206115722656,
      "activations/layer4_attention_weight_min": -77.4564437866211,
      "activations/layer5_attention_weight_max": 132.26589965820312,
      "activations/layer5_attention_weight_min": -126.4128646850586,
      "activations/layer6_attention_weight_max": 62.12098693847656,
      "activations/layer6_attention_weight_min": -57.888980865478516,
      "activations/layer7_attention_weight_max": 63.30116271972656,
      "activations/layer7_attention_weight_min": -55.40857696533203,
      "activations/layer8_attention_weight_max": 53.909488677978516,
      "activations/layer8_attention_weight_min": -48.4842414855957,
      "activations/layer9_attention_weight_max": 71.3343734741211,
      "activations/layer9_attention_weight_min": -66.00690460205078,
      "epoch": 1.19,
      "learning_rate": 0.00014376287878787878,
      "loss": 3.0545,
      "step": 20500
    },
    {
      "activations/layer0_attention_weight_max": 13.694483757019043,
      "activations/layer0_attention_weight_min": -12.274529457092285,
      "activations/layer10_attention_weight_max": 63.77939224243164,
      "activations/layer10_attention_weight_min": -53.44710159301758,
      "activations/layer11_attention_weight_max": 45.0451774597168,
      "activations/layer11_attention_weight_min": -36.450443267822266,
      "activations/layer12_attention_weight_max": 30.206266403198242,
      "activations/layer12_attention_weight_min": -25.940467834472656,
      "activations/layer13_attention_weight_max": 73.55958557128906,
      "activations/layer13_attention_weight_min": -51.36296844482422,
      "activations/layer14_attention_weight_max": 59.467830657958984,
      "activations/layer14_attention_weight_min": -40.53487014770508,
      "activations/layer15_attention_weight_max": 57.61067199707031,
      "activations/layer15_attention_weight_min": -35.84293746948242,
      "activations/layer16_attention_weight_max": 51.50507354736328,
      "activations/layer16_attention_weight_min": -31.76213836669922,
      "activations/layer17_attention_weight_max": 60.46280288696289,
      "activations/layer17_attention_weight_min": -34.72765350341797,
      "activations/layer18_attention_weight_max": 69.65560150146484,
      "activations/layer18_attention_weight_min": -37.353790283203125,
      "activations/layer19_attention_weight_max": 20.57920265197754,
      "activations/layer19_attention_weight_min": -15.519983291625977,
      "activations/layer1_attention_weight_max": 15.434621810913086,
      "activations/layer1_attention_weight_min": -9.924176216125488,
      "activations/layer20_attention_weight_max": 19.633329391479492,
      "activations/layer20_attention_weight_min": -13.999170303344727,
      "activations/layer21_attention_weight_max": 37.13977813720703,
      "activations/layer21_attention_weight_min": -18.627592086791992,
      "activations/layer22_attention_weight_max": 36.30632781982422,
      "activations/layer22_attention_weight_min": -17.485742568969727,
      "activations/layer23_attention_weight_max": 22.667144775390625,
      "activations/layer23_attention_weight_min": -14.223814964294434,
      "activations/layer2_attention_weight_max": 23.828929901123047,
      "activations/layer2_attention_weight_min": -23.293914794921875,
      "activations/layer3_attention_weight_max": 55.611183166503906,
      "activations/layer3_attention_weight_min": -56.14228820800781,
      "activations/layer4_attention_weight_max": 86.9898910522461,
      "activations/layer4_attention_weight_min": -76.17867279052734,
      "activations/layer5_attention_weight_max": 141.43792724609375,
      "activations/layer5_attention_weight_min": -124.1624755859375,
      "activations/layer6_attention_weight_max": 62.30763244628906,
      "activations/layer6_attention_weight_min": -55.29745864868164,
      "activations/layer7_attention_weight_max": 68.33851623535156,
      "activations/layer7_attention_weight_min": -58.715980529785156,
      "activations/layer8_attention_weight_max": 50.87526321411133,
      "activations/layer8_attention_weight_min": -49.683753967285156,
      "activations/layer9_attention_weight_max": 73.73088836669922,
      "activations/layer9_attention_weight_min": -69.43307495117188,
      "epoch": 1.19,
      "learning_rate": 0.0001437439393939394,
      "loss": 3.0478,
      "step": 20550
    },
    {
      "activations/layer0_attention_weight_max": 13.903768539428711,
      "activations/layer0_attention_weight_min": -12.481403350830078,
      "activations/layer10_attention_weight_max": 71.8028564453125,
      "activations/layer10_attention_weight_min": -60.28281784057617,
      "activations/layer11_attention_weight_max": 47.79759216308594,
      "activations/layer11_attention_weight_min": -37.596431732177734,
      "activations/layer12_attention_weight_max": 29.48920440673828,
      "activations/layer12_attention_weight_min": -26.217939376831055,
      "activations/layer13_attention_weight_max": 67.08747863769531,
      "activations/layer13_attention_weight_min": -45.51717758178711,
      "activations/layer14_attention_weight_max": 56.780086517333984,
      "activations/layer14_attention_weight_min": -42.838096618652344,
      "activations/layer15_attention_weight_max": 62.67976760864258,
      "activations/layer15_attention_weight_min": -40.37343978881836,
      "activations/layer16_attention_weight_max": 53.25110626220703,
      "activations/layer16_attention_weight_min": -32.611690521240234,
      "activations/layer17_attention_weight_max": 58.64243698120117,
      "activations/layer17_attention_weight_min": -38.93375015258789,
      "activations/layer18_attention_weight_max": 74.4253158569336,
      "activations/layer18_attention_weight_min": -38.70771408081055,
      "activations/layer19_attention_weight_max": 24.34969139099121,
      "activations/layer19_attention_weight_min": -15.338175773620605,
      "activations/layer1_attention_weight_max": 15.186935424804688,
      "activations/layer1_attention_weight_min": -11.155756950378418,
      "activations/layer20_attention_weight_max": 22.952056884765625,
      "activations/layer20_attention_weight_min": -13.246809959411621,
      "activations/layer21_attention_weight_max": 43.77461624145508,
      "activations/layer21_attention_weight_min": -20.469209671020508,
      "activations/layer22_attention_weight_max": 44.489898681640625,
      "activations/layer22_attention_weight_min": -16.5580997467041,
      "activations/layer23_attention_weight_max": 29.052236557006836,
      "activations/layer23_attention_weight_min": -16.11212158203125,
      "activations/layer2_attention_weight_max": 25.947538375854492,
      "activations/layer2_attention_weight_min": -24.766456604003906,
      "activations/layer3_attention_weight_max": 54.2558479309082,
      "activations/layer3_attention_weight_min": -57.036415100097656,
      "activations/layer4_attention_weight_max": 82.7094955444336,
      "activations/layer4_attention_weight_min": -73.51117706298828,
      "activations/layer5_attention_weight_max": 137.77053833007812,
      "activations/layer5_attention_weight_min": -120.74851989746094,
      "activations/layer6_attention_weight_max": 67.31639862060547,
      "activations/layer6_attention_weight_min": -54.690670013427734,
      "activations/layer7_attention_weight_max": 62.919586181640625,
      "activations/layer7_attention_weight_min": -55.229156494140625,
      "activations/layer8_attention_weight_max": 58.19206619262695,
      "activations/layer8_attention_weight_min": -47.31459426879883,
      "activations/layer9_attention_weight_max": 72.07842254638672,
      "activations/layer9_attention_weight_min": -61.5503044128418,
      "epoch": 1.2,
      "learning_rate": 0.00014372537878787879,
      "loss": 3.0619,
      "step": 20600
    },
    {
      "activations/layer0_attention_weight_max": 12.669957160949707,
      "activations/layer0_attention_weight_min": -11.988309860229492,
      "activations/layer10_attention_weight_max": 69.5616226196289,
      "activations/layer10_attention_weight_min": -60.554683685302734,
      "activations/layer11_attention_weight_max": 43.31287384033203,
      "activations/layer11_attention_weight_min": -42.89225387573242,
      "activations/layer12_attention_weight_max": 33.40227508544922,
      "activations/layer12_attention_weight_min": -26.18047523498535,
      "activations/layer13_attention_weight_max": 50.88096618652344,
      "activations/layer13_attention_weight_min": -42.75846862792969,
      "activations/layer14_attention_weight_max": 44.57273864746094,
      "activations/layer14_attention_weight_min": -38.320247650146484,
      "activations/layer15_attention_weight_max": 50.561187744140625,
      "activations/layer15_attention_weight_min": -40.017154693603516,
      "activations/layer16_attention_weight_max": 48.336238861083984,
      "activations/layer16_attention_weight_min": -34.058650970458984,
      "activations/layer17_attention_weight_max": 60.8293342590332,
      "activations/layer17_attention_weight_min": -42.706512451171875,
      "activations/layer18_attention_weight_max": 62.07862091064453,
      "activations/layer18_attention_weight_min": -42.95367431640625,
      "activations/layer19_attention_weight_max": 19.864273071289062,
      "activations/layer19_attention_weight_min": -14.145156860351562,
      "activations/layer1_attention_weight_max": 14.860284805297852,
      "activations/layer1_attention_weight_min": -9.93222427368164,
      "activations/layer20_attention_weight_max": 17.846351623535156,
      "activations/layer20_attention_weight_min": -13.815187454223633,
      "activations/layer21_attention_weight_max": 40.154762268066406,
      "activations/layer21_attention_weight_min": -21.54937171936035,
      "activations/layer22_attention_weight_max": 31.922027587890625,
      "activations/layer22_attention_weight_min": -19.212833404541016,
      "activations/layer23_attention_weight_max": 25.388397216796875,
      "activations/layer23_attention_weight_min": -14.665858268737793,
      "activations/layer2_attention_weight_max": 24.498931884765625,
      "activations/layer2_attention_weight_min": -23.643028259277344,
      "activations/layer3_attention_weight_max": 54.74594497680664,
      "activations/layer3_attention_weight_min": -57.213096618652344,
      "activations/layer4_attention_weight_max": 84.57469940185547,
      "activations/layer4_attention_weight_min": -77.37691497802734,
      "activations/layer5_attention_weight_max": 132.66268920898438,
      "activations/layer5_attention_weight_min": -122.44169616699219,
      "activations/layer6_attention_weight_max": 59.10242462158203,
      "activations/layer6_attention_weight_min": -54.00705337524414,
      "activations/layer7_attention_weight_max": 62.69523620605469,
      "activations/layer7_attention_weight_min": -55.659671783447266,
      "activations/layer8_attention_weight_max": 51.341102600097656,
      "activations/layer8_attention_weight_min": -49.28902053833008,
      "activations/layer9_attention_weight_max": 66.30940246582031,
      "activations/layer9_attention_weight_min": -63.356658935546875,
      "epoch": 1.2,
      "learning_rate": 0.00014370643939393937,
      "loss": 3.0607,
      "step": 20650
    },
    {
      "activations/layer0_attention_weight_max": 13.255881309509277,
      "activations/layer0_attention_weight_min": -11.833577156066895,
      "activations/layer10_attention_weight_max": 61.90247344970703,
      "activations/layer10_attention_weight_min": -60.8487548828125,
      "activations/layer11_attention_weight_max": 44.83088684082031,
      "activations/layer11_attention_weight_min": -41.02055358886719,
      "activations/layer12_attention_weight_max": 29.70972442626953,
      "activations/layer12_attention_weight_min": -27.643171310424805,
      "activations/layer13_attention_weight_max": 55.184329986572266,
      "activations/layer13_attention_weight_min": -43.27794647216797,
      "activations/layer14_attention_weight_max": 47.289363861083984,
      "activations/layer14_attention_weight_min": -36.973915100097656,
      "activations/layer15_attention_weight_max": 55.72694778442383,
      "activations/layer15_attention_weight_min": -39.9854736328125,
      "activations/layer16_attention_weight_max": 46.511741638183594,
      "activations/layer16_attention_weight_min": -33.202049255371094,
      "activations/layer17_attention_weight_max": 56.56890106201172,
      "activations/layer17_attention_weight_min": -38.26872634887695,
      "activations/layer18_attention_weight_max": 63.44314193725586,
      "activations/layer18_attention_weight_min": -42.39084243774414,
      "activations/layer19_attention_weight_max": 20.268308639526367,
      "activations/layer19_attention_weight_min": -12.682013511657715,
      "activations/layer1_attention_weight_max": 14.904547691345215,
      "activations/layer1_attention_weight_min": -11.378835678100586,
      "activations/layer20_attention_weight_max": 16.8970890045166,
      "activations/layer20_attention_weight_min": -13.635723114013672,
      "activations/layer21_attention_weight_max": 36.87456512451172,
      "activations/layer21_attention_weight_min": -18.208406448364258,
      "activations/layer22_attention_weight_max": 35.6728515625,
      "activations/layer22_attention_weight_min": -17.496469497680664,
      "activations/layer23_attention_weight_max": 23.014572143554688,
      "activations/layer23_attention_weight_min": -13.500736236572266,
      "activations/layer2_attention_weight_max": 24.940135955810547,
      "activations/layer2_attention_weight_min": -24.262004852294922,
      "activations/layer3_attention_weight_max": 52.71571731567383,
      "activations/layer3_attention_weight_min": -56.29592514038086,
      "activations/layer4_attention_weight_max": 79.4918441772461,
      "activations/layer4_attention_weight_min": -75.66732788085938,
      "activations/layer5_attention_weight_max": 130.42486572265625,
      "activations/layer5_attention_weight_min": -123.08302307128906,
      "activations/layer6_attention_weight_max": 61.53480529785156,
      "activations/layer6_attention_weight_min": -57.12739944458008,
      "activations/layer7_attention_weight_max": 60.90261459350586,
      "activations/layer7_attention_weight_min": -54.98681640625,
      "activations/layer8_attention_weight_max": 50.6279411315918,
      "activations/layer8_attention_weight_min": -48.605098724365234,
      "activations/layer9_attention_weight_max": 69.37095642089844,
      "activations/layer9_attention_weight_min": -69.6638412475586,
      "epoch": 1.2,
      "learning_rate": 0.0001436875,
      "loss": 3.0475,
      "step": 20700
    },
    {
      "activations/layer0_attention_weight_max": 15.049156188964844,
      "activations/layer0_attention_weight_min": -11.568798065185547,
      "activations/layer10_attention_weight_max": 70.2572250366211,
      "activations/layer10_attention_weight_min": -57.990901947021484,
      "activations/layer11_attention_weight_max": 42.895442962646484,
      "activations/layer11_attention_weight_min": -39.0666618347168,
      "activations/layer12_attention_weight_max": 27.09424591064453,
      "activations/layer12_attention_weight_min": -25.844324111938477,
      "activations/layer13_attention_weight_max": 71.15645599365234,
      "activations/layer13_attention_weight_min": -47.67477035522461,
      "activations/layer14_attention_weight_max": 69.78681945800781,
      "activations/layer14_attention_weight_min": -46.08767318725586,
      "activations/layer15_attention_weight_max": 53.88991165161133,
      "activations/layer15_attention_weight_min": -39.62506866455078,
      "activations/layer16_attention_weight_max": 44.09745788574219,
      "activations/layer16_attention_weight_min": -33.682342529296875,
      "activations/layer17_attention_weight_max": 52.13943099975586,
      "activations/layer17_attention_weight_min": -36.50899124145508,
      "activations/layer18_attention_weight_max": 64.91468811035156,
      "activations/layer18_attention_weight_min": -43.08654022216797,
      "activations/layer19_attention_weight_max": 22.06373405456543,
      "activations/layer19_attention_weight_min": -12.96258544921875,
      "activations/layer1_attention_weight_max": 14.209012985229492,
      "activations/layer1_attention_weight_min": -12.353116035461426,
      "activations/layer20_attention_weight_max": 20.22415542602539,
      "activations/layer20_attention_weight_min": -13.926673889160156,
      "activations/layer21_attention_weight_max": 37.05325698852539,
      "activations/layer21_attention_weight_min": -20.473718643188477,
      "activations/layer22_attention_weight_max": 32.15569305419922,
      "activations/layer22_attention_weight_min": -18.921207427978516,
      "activations/layer23_attention_weight_max": 22.269210815429688,
      "activations/layer23_attention_weight_min": -14.598119735717773,
      "activations/layer2_attention_weight_max": 27.13019561767578,
      "activations/layer2_attention_weight_min": -24.55078887939453,
      "activations/layer3_attention_weight_max": 55.28153991699219,
      "activations/layer3_attention_weight_min": -57.62562942504883,
      "activations/layer4_attention_weight_max": 84.4936752319336,
      "activations/layer4_attention_weight_min": -73.69866180419922,
      "activations/layer5_attention_weight_max": 129.62484741210938,
      "activations/layer5_attention_weight_min": -118.84039306640625,
      "activations/layer6_attention_weight_max": 62.652774810791016,
      "activations/layer6_attention_weight_min": -53.500709533691406,
      "activations/layer7_attention_weight_max": 59.77212142944336,
      "activations/layer7_attention_weight_min": -54.10585403442383,
      "activations/layer8_attention_weight_max": 49.15093231201172,
      "activations/layer8_attention_weight_min": -45.51826858520508,
      "activations/layer9_attention_weight_max": 72.6678237915039,
      "activations/layer9_attention_weight_min": -66.05828094482422,
      "epoch": 1.21,
      "learning_rate": 0.0001436685606060606,
      "loss": 3.0419,
      "step": 20750
    },
    {
      "activations/layer0_attention_weight_max": 14.136152267456055,
      "activations/layer0_attention_weight_min": -12.24301815032959,
      "activations/layer10_attention_weight_max": 65.31454467773438,
      "activations/layer10_attention_weight_min": -56.36741638183594,
      "activations/layer11_attention_weight_max": 41.58386993408203,
      "activations/layer11_attention_weight_min": -39.00495147705078,
      "activations/layer12_attention_weight_max": 26.840456008911133,
      "activations/layer12_attention_weight_min": -26.828340530395508,
      "activations/layer13_attention_weight_max": 76.3735580444336,
      "activations/layer13_attention_weight_min": -53.42430114746094,
      "activations/layer14_attention_weight_max": 56.80793762207031,
      "activations/layer14_attention_weight_min": -41.58719253540039,
      "activations/layer15_attention_weight_max": 51.392459869384766,
      "activations/layer15_attention_weight_min": -41.058738708496094,
      "activations/layer16_attention_weight_max": 46.153587341308594,
      "activations/layer16_attention_weight_min": -34.02745056152344,
      "activations/layer17_attention_weight_max": 52.128265380859375,
      "activations/layer17_attention_weight_min": -35.65058898925781,
      "activations/layer18_attention_weight_max": 62.49152755737305,
      "activations/layer18_attention_weight_min": -40.04412078857422,
      "activations/layer19_attention_weight_max": 21.813888549804688,
      "activations/layer19_attention_weight_min": -13.22106647491455,
      "activations/layer1_attention_weight_max": 15.586225509643555,
      "activations/layer1_attention_weight_min": -12.44737720489502,
      "activations/layer20_attention_weight_max": 23.60866355895996,
      "activations/layer20_attention_weight_min": -14.49465274810791,
      "activations/layer21_attention_weight_max": 34.618778228759766,
      "activations/layer21_attention_weight_min": -20.574050903320312,
      "activations/layer22_attention_weight_max": 36.667015075683594,
      "activations/layer22_attention_weight_min": -18.591136932373047,
      "activations/layer23_attention_weight_max": 23.95770835876465,
      "activations/layer23_attention_weight_min": -13.958236694335938,
      "activations/layer2_attention_weight_max": 27.085983276367188,
      "activations/layer2_attention_weight_min": -23.998348236083984,
      "activations/layer3_attention_weight_max": 53.64514923095703,
      "activations/layer3_attention_weight_min": -54.533714294433594,
      "activations/layer4_attention_weight_max": 78.95227813720703,
      "activations/layer4_attention_weight_min": -74.07422637939453,
      "activations/layer5_attention_weight_max": 129.73046875,
      "activations/layer5_attention_weight_min": -118.57200622558594,
      "activations/layer6_attention_weight_max": 62.363922119140625,
      "activations/layer6_attention_weight_min": -57.536376953125,
      "activations/layer7_attention_weight_max": 59.95191192626953,
      "activations/layer7_attention_weight_min": -54.740516662597656,
      "activations/layer8_attention_weight_max": 52.89012145996094,
      "activations/layer8_attention_weight_min": -48.20772933959961,
      "activations/layer9_attention_weight_max": 66.66024017333984,
      "activations/layer9_attention_weight_min": -61.62504959106445,
      "epoch": 1.21,
      "learning_rate": 0.00014364962121212122,
      "loss": 3.0671,
      "step": 20800
    },
    {
      "activations/layer0_attention_weight_max": 14.681649208068848,
      "activations/layer0_attention_weight_min": -12.38465404510498,
      "activations/layer10_attention_weight_max": 64.52020263671875,
      "activations/layer10_attention_weight_min": -58.45164489746094,
      "activations/layer11_attention_weight_max": 41.683223724365234,
      "activations/layer11_attention_weight_min": -38.253074645996094,
      "activations/layer12_attention_weight_max": 26.1803035736084,
      "activations/layer12_attention_weight_min": -25.590421676635742,
      "activations/layer13_attention_weight_max": 67.86814880371094,
      "activations/layer13_attention_weight_min": -55.70193862915039,
      "activations/layer14_attention_weight_max": 62.56377029418945,
      "activations/layer14_attention_weight_min": -52.806060791015625,
      "activations/layer15_attention_weight_max": 58.844520568847656,
      "activations/layer15_attention_weight_min": -42.39174270629883,
      "activations/layer16_attention_weight_max": 55.026336669921875,
      "activations/layer16_attention_weight_min": -36.88166809082031,
      "activations/layer17_attention_weight_max": 63.20507049560547,
      "activations/layer17_attention_weight_min": -41.4439582824707,
      "activations/layer18_attention_weight_max": 60.30358123779297,
      "activations/layer18_attention_weight_min": -36.754390716552734,
      "activations/layer19_attention_weight_max": 20.69411277770996,
      "activations/layer19_attention_weight_min": -14.047795295715332,
      "activations/layer1_attention_weight_max": 14.685874938964844,
      "activations/layer1_attention_weight_min": -12.016531944274902,
      "activations/layer20_attention_weight_max": 18.630435943603516,
      "activations/layer20_attention_weight_min": -16.862300872802734,
      "activations/layer21_attention_weight_max": 36.45802688598633,
      "activations/layer21_attention_weight_min": -21.295190811157227,
      "activations/layer22_attention_weight_max": 35.045650482177734,
      "activations/layer22_attention_weight_min": -21.26881980895996,
      "activations/layer23_attention_weight_max": 21.15129280090332,
      "activations/layer23_attention_weight_min": -16.810543060302734,
      "activations/layer2_attention_weight_max": 27.31816864013672,
      "activations/layer2_attention_weight_min": -24.362892150878906,
      "activations/layer3_attention_weight_max": 54.67559814453125,
      "activations/layer3_attention_weight_min": -54.01943588256836,
      "activations/layer4_attention_weight_max": 84.59163665771484,
      "activations/layer4_attention_weight_min": -75.87071228027344,
      "activations/layer5_attention_weight_max": 129.3736572265625,
      "activations/layer5_attention_weight_min": -122.93135070800781,
      "activations/layer6_attention_weight_max": 62.3140983581543,
      "activations/layer6_attention_weight_min": -55.518192291259766,
      "activations/layer7_attention_weight_max": 59.27903366088867,
      "activations/layer7_attention_weight_min": -53.64775085449219,
      "activations/layer8_attention_weight_max": 52.77706527709961,
      "activations/layer8_attention_weight_min": -45.66181945800781,
      "activations/layer9_attention_weight_max": 68.17436218261719,
      "activations/layer9_attention_weight_min": -63.522071838378906,
      "epoch": 1.21,
      "learning_rate": 0.0001436306818181818,
      "loss": 3.0369,
      "step": 20850
    },
    {
      "activations/layer0_attention_weight_max": 15.018818855285645,
      "activations/layer0_attention_weight_min": -11.711649894714355,
      "activations/layer10_attention_weight_max": 63.96345138549805,
      "activations/layer10_attention_weight_min": -53.970664978027344,
      "activations/layer11_attention_weight_max": 43.43687438964844,
      "activations/layer11_attention_weight_min": -41.022743225097656,
      "activations/layer12_attention_weight_max": 25.754291534423828,
      "activations/layer12_attention_weight_min": -28.635704040527344,
      "activations/layer13_attention_weight_max": 58.1434440612793,
      "activations/layer13_attention_weight_min": -46.18814468383789,
      "activations/layer14_attention_weight_max": 50.50895690917969,
      "activations/layer14_attention_weight_min": -37.70724868774414,
      "activations/layer15_attention_weight_max": 51.88675308227539,
      "activations/layer15_attention_weight_min": -39.64140701293945,
      "activations/layer16_attention_weight_max": 43.32267761230469,
      "activations/layer16_attention_weight_min": -32.174163818359375,
      "activations/layer17_attention_weight_max": 58.5574951171875,
      "activations/layer17_attention_weight_min": -37.83687973022461,
      "activations/layer18_attention_weight_max": 60.22360610961914,
      "activations/layer18_attention_weight_min": -35.239112854003906,
      "activations/layer19_attention_weight_max": 20.436254501342773,
      "activations/layer19_attention_weight_min": -13.8905029296875,
      "activations/layer1_attention_weight_max": 14.708295822143555,
      "activations/layer1_attention_weight_min": -11.79141616821289,
      "activations/layer20_attention_weight_max": 18.045589447021484,
      "activations/layer20_attention_weight_min": -14.082958221435547,
      "activations/layer21_attention_weight_max": 40.04542922973633,
      "activations/layer21_attention_weight_min": -20.457643508911133,
      "activations/layer22_attention_weight_max": 33.20975875854492,
      "activations/layer22_attention_weight_min": -17.50027847290039,
      "activations/layer23_attention_weight_max": 20.940540313720703,
      "activations/layer23_attention_weight_min": -14.44584846496582,
      "activations/layer2_attention_weight_max": 26.00363540649414,
      "activations/layer2_attention_weight_min": -24.48126983642578,
      "activations/layer3_attention_weight_max": 54.98506546020508,
      "activations/layer3_attention_weight_min": -56.90489959716797,
      "activations/layer4_attention_weight_max": 87.90388488769531,
      "activations/layer4_attention_weight_min": -78.26378631591797,
      "activations/layer5_attention_weight_max": 138.15316772460938,
      "activations/layer5_attention_weight_min": -120.20430755615234,
      "activations/layer6_attention_weight_max": 64.67816162109375,
      "activations/layer6_attention_weight_min": -58.04996109008789,
      "activations/layer7_attention_weight_max": 60.3442268371582,
      "activations/layer7_attention_weight_min": -55.905616760253906,
      "activations/layer8_attention_weight_max": 50.6196403503418,
      "activations/layer8_attention_weight_min": -47.66020965576172,
      "activations/layer9_attention_weight_max": 69.71526336669922,
      "activations/layer9_attention_weight_min": -66.95148468017578,
      "epoch": 1.21,
      "learning_rate": 0.00014361174242424242,
      "loss": 3.0532,
      "step": 20900
    },
    {
      "activations/layer0_attention_weight_max": 13.282824516296387,
      "activations/layer0_attention_weight_min": -11.85826301574707,
      "activations/layer10_attention_weight_max": 73.7953872680664,
      "activations/layer10_attention_weight_min": -63.318572998046875,
      "activations/layer11_attention_weight_max": 48.44868469238281,
      "activations/layer11_attention_weight_min": -42.5843620300293,
      "activations/layer12_attention_weight_max": 30.867063522338867,
      "activations/layer12_attention_weight_min": -27.18712615966797,
      "activations/layer13_attention_weight_max": 60.72867965698242,
      "activations/layer13_attention_weight_min": -40.68204879760742,
      "activations/layer14_attention_weight_max": 51.5759391784668,
      "activations/layer14_attention_weight_min": -36.04650115966797,
      "activations/layer15_attention_weight_max": 53.34839630126953,
      "activations/layer15_attention_weight_min": -41.552032470703125,
      "activations/layer16_attention_weight_max": 48.63853073120117,
      "activations/layer16_attention_weight_min": -39.26512145996094,
      "activations/layer17_attention_weight_max": 58.28131866455078,
      "activations/layer17_attention_weight_min": -38.24811935424805,
      "activations/layer18_attention_weight_max": 69.01664733886719,
      "activations/layer18_attention_weight_min": -45.3770866394043,
      "activations/layer19_attention_weight_max": 18.963443756103516,
      "activations/layer19_attention_weight_min": -14.04002857208252,
      "activations/layer1_attention_weight_max": 15.150481224060059,
      "activations/layer1_attention_weight_min": -12.329835891723633,
      "activations/layer20_attention_weight_max": 21.31976890563965,
      "activations/layer20_attention_weight_min": -14.962899208068848,
      "activations/layer21_attention_weight_max": 43.67510986328125,
      "activations/layer21_attention_weight_min": -20.524553298950195,
      "activations/layer22_attention_weight_max": 38.07014083862305,
      "activations/layer22_attention_weight_min": -21.724475860595703,
      "activations/layer23_attention_weight_max": 23.669225692749023,
      "activations/layer23_attention_weight_min": -13.83731746673584,
      "activations/layer2_attention_weight_max": 26.166746139526367,
      "activations/layer2_attention_weight_min": -24.741607666015625,
      "activations/layer3_attention_weight_max": 54.12116622924805,
      "activations/layer3_attention_weight_min": -56.32720184326172,
      "activations/layer4_attention_weight_max": 83.72762298583984,
      "activations/layer4_attention_weight_min": -74.78943634033203,
      "activations/layer5_attention_weight_max": 133.7575225830078,
      "activations/layer5_attention_weight_min": -117.83195495605469,
      "activations/layer6_attention_weight_max": 61.85590744018555,
      "activations/layer6_attention_weight_min": -55.1829833984375,
      "activations/layer7_attention_weight_max": 62.91411209106445,
      "activations/layer7_attention_weight_min": -55.739646911621094,
      "activations/layer8_attention_weight_max": 51.73030471801758,
      "activations/layer8_attention_weight_min": -43.721683502197266,
      "activations/layer9_attention_weight_max": 71.77197265625,
      "activations/layer9_attention_weight_min": -74.10704803466797,
      "epoch": 1.22,
      "learning_rate": 0.000143592803030303,
      "loss": 3.0542,
      "step": 20950
    },
    {
      "activations/layer0_attention_weight_max": 14.62033748626709,
      "activations/layer0_attention_weight_min": -12.033529281616211,
      "activations/layer10_attention_weight_max": 75.8646011352539,
      "activations/layer10_attention_weight_min": -58.021568298339844,
      "activations/layer11_attention_weight_max": 51.16221618652344,
      "activations/layer11_attention_weight_min": -39.16018295288086,
      "activations/layer12_attention_weight_max": 28.543962478637695,
      "activations/layer12_attention_weight_min": -23.544204711914062,
      "activations/layer13_attention_weight_max": 72.96305084228516,
      "activations/layer13_attention_weight_min": -48.91035079956055,
      "activations/layer14_attention_weight_max": 66.43974304199219,
      "activations/layer14_attention_weight_min": -41.02470397949219,
      "activations/layer15_attention_weight_max": 59.85152053833008,
      "activations/layer15_attention_weight_min": -39.29044723510742,
      "activations/layer16_attention_weight_max": 51.009742736816406,
      "activations/layer16_attention_weight_min": -31.496721267700195,
      "activations/layer17_attention_weight_max": 74.24211120605469,
      "activations/layer17_attention_weight_min": -37.5587158203125,
      "activations/layer18_attention_weight_max": 83.5628662109375,
      "activations/layer18_attention_weight_min": -39.662879943847656,
      "activations/layer19_attention_weight_max": 20.652807235717773,
      "activations/layer19_attention_weight_min": -12.632412910461426,
      "activations/layer1_attention_weight_max": 15.360334396362305,
      "activations/layer1_attention_weight_min": -11.138223648071289,
      "activations/layer20_attention_weight_max": 22.318052291870117,
      "activations/layer20_attention_weight_min": -13.67177963256836,
      "activations/layer21_attention_weight_max": 41.71413040161133,
      "activations/layer21_attention_weight_min": -19.864938735961914,
      "activations/layer22_attention_weight_max": 42.73453903198242,
      "activations/layer22_attention_weight_min": -19.897371292114258,
      "activations/layer23_attention_weight_max": 23.029813766479492,
      "activations/layer23_attention_weight_min": -15.970832824707031,
      "activations/layer2_attention_weight_max": 24.875768661499023,
      "activations/layer2_attention_weight_min": -23.915420532226562,
      "activations/layer3_attention_weight_max": 55.66258239746094,
      "activations/layer3_attention_weight_min": -58.18598175048828,
      "activations/layer4_attention_weight_max": 84.83271026611328,
      "activations/layer4_attention_weight_min": -77.7762222290039,
      "activations/layer5_attention_weight_max": 139.54013061523438,
      "activations/layer5_attention_weight_min": -128.18515014648438,
      "activations/layer6_attention_weight_max": 64.18666076660156,
      "activations/layer6_attention_weight_min": -57.36802291870117,
      "activations/layer7_attention_weight_max": 62.01094436645508,
      "activations/layer7_attention_weight_min": -56.51114273071289,
      "activations/layer8_attention_weight_max": 59.32643127441406,
      "activations/layer8_attention_weight_min": -49.26979446411133,
      "activations/layer9_attention_weight_max": 82.75381469726562,
      "activations/layer9_attention_weight_min": -70.54354858398438,
      "epoch": 1.22,
      "learning_rate": 0.00014357386363636363,
      "loss": 3.0674,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_loss": 3.0,
      "eval_runtime": 8.608,
      "eval_samples_per_second": 498.837,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_openwebtext_loss": 3.0,
      "eval_openwebtext_ppl": 20.085536923187668,
      "eval_openwebtext_runtime": 8.608,
      "eval_openwebtext_samples_per_second": 498.837,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_wikitext_loss": 3.29296875,
      "eval_wikitext_ppl": 26.922671813996086,
      "eval_wikitext_runtime": 1.7357,
      "eval_wikitext_samples_per_second": 262.725,
      "step": 21000
    },
    {
      "epoch": 1.22,
      "eval_lambada_loss": 3.337890625,
      "eval_lambada_ppl": 28.15966472087403,
      "eval_lambada_runtime": 9.3223,
      "eval_lambada_samples_per_second": 522.296,
      "step": 21000
    },
    {
      "activations/layer0_attention_weight_max": 13.168136596679688,
      "activations/layer0_attention_weight_min": -11.817103385925293,
      "activations/layer10_attention_weight_max": 69.62786102294922,
      "activations/layer10_attention_weight_min": -59.75139617919922,
      "activations/layer11_attention_weight_max": 44.86602020263672,
      "activations/layer11_attention_weight_min": -37.735958099365234,
      "activations/layer12_attention_weight_max": 27.93649673461914,
      "activations/layer12_attention_weight_min": -28.628822326660156,
      "activations/layer13_attention_weight_max": 51.57719039916992,
      "activations/layer13_attention_weight_min": -39.45198059082031,
      "activations/layer14_attention_weight_max": 49.469085693359375,
      "activations/layer14_attention_weight_min": -38.91103744506836,
      "activations/layer15_attention_weight_max": 51.86760711669922,
      "activations/layer15_attention_weight_min": -39.10889434814453,
      "activations/layer16_attention_weight_max": 51.94707489013672,
      "activations/layer16_attention_weight_min": -35.17410659790039,
      "activations/layer17_attention_weight_max": 56.70037841796875,
      "activations/layer17_attention_weight_min": -36.75465393066406,
      "activations/layer18_attention_weight_max": 58.694034576416016,
      "activations/layer18_attention_weight_min": -36.51789474487305,
      "activations/layer19_attention_weight_max": 20.32607650756836,
      "activations/layer19_attention_weight_min": -14.768638610839844,
      "activations/layer1_attention_weight_max": 15.498147964477539,
      "activations/layer1_attention_weight_min": -12.12092113494873,
      "activations/layer20_attention_weight_max": 20.382841110229492,
      "activations/layer20_attention_weight_min": -15.727835655212402,
      "activations/layer21_attention_weight_max": 32.630775451660156,
      "activations/layer21_attention_weight_min": -21.188034057617188,
      "activations/layer22_attention_weight_max": 30.741413116455078,
      "activations/layer22_attention_weight_min": -21.908716201782227,
      "activations/layer23_attention_weight_max": 19.958255767822266,
      "activations/layer23_attention_weight_min": -13.052108764648438,
      "activations/layer2_attention_weight_max": 23.958816528320312,
      "activations/layer2_attention_weight_min": -22.68549156188965,
      "activations/layer3_attention_weight_max": 55.389522552490234,
      "activations/layer3_attention_weight_min": -55.51271438598633,
      "activations/layer4_attention_weight_max": 87.11077117919922,
      "activations/layer4_attention_weight_min": -77.71929168701172,
      "activations/layer5_attention_weight_max": 138.29205322265625,
      "activations/layer5_attention_weight_min": -124.51145935058594,
      "activations/layer6_attention_weight_max": 63.04169464111328,
      "activations/layer6_attention_weight_min": -56.47307586669922,
      "activations/layer7_attention_weight_max": 66.41295623779297,
      "activations/layer7_attention_weight_min": -55.4403190612793,
      "activations/layer8_attention_weight_max": 54.07094192504883,
      "activations/layer8_attention_weight_min": -47.921939849853516,
      "activations/layer9_attention_weight_max": 75.70362854003906,
      "activations/layer9_attention_weight_min": -74.5907211303711,
      "epoch": 1.22,
      "learning_rate": 0.00014355492424242424,
      "loss": 3.0416,
      "step": 21050
    },
    {
      "activations/layer0_attention_weight_max": 12.80506706237793,
      "activations/layer0_attention_weight_min": -12.350678443908691,
      "activations/layer10_attention_weight_max": 69.52359771728516,
      "activations/layer10_attention_weight_min": -55.8203125,
      "activations/layer11_attention_weight_max": 50.947235107421875,
      "activations/layer11_attention_weight_min": -41.80226516723633,
      "activations/layer12_attention_weight_max": 29.204471588134766,
      "activations/layer12_attention_weight_min": -26.578229904174805,
      "activations/layer13_attention_weight_max": 64.80690002441406,
      "activations/layer13_attention_weight_min": -45.813377380371094,
      "activations/layer14_attention_weight_max": 53.691715240478516,
      "activations/layer14_attention_weight_min": -38.7017936706543,
      "activations/layer15_attention_weight_max": 59.59819030761719,
      "activations/layer15_attention_weight_min": -40.24198532104492,
      "activations/layer16_attention_weight_max": 50.04618453979492,
      "activations/layer16_attention_weight_min": -30.45648193359375,
      "activations/layer17_attention_weight_max": 56.54020690917969,
      "activations/layer17_attention_weight_min": -37.2448616027832,
      "activations/layer18_attention_weight_max": 77.34456634521484,
      "activations/layer18_attention_weight_min": -42.435001373291016,
      "activations/layer19_attention_weight_max": 22.294519424438477,
      "activations/layer19_attention_weight_min": -15.140263557434082,
      "activations/layer1_attention_weight_max": 14.71796703338623,
      "activations/layer1_attention_weight_min": -10.541106224060059,
      "activations/layer20_attention_weight_max": 17.370258331298828,
      "activations/layer20_attention_weight_min": -12.433887481689453,
      "activations/layer21_attention_weight_max": 36.578834533691406,
      "activations/layer21_attention_weight_min": -19.728317260742188,
      "activations/layer22_attention_weight_max": 34.396087646484375,
      "activations/layer22_attention_weight_min": -18.694141387939453,
      "activations/layer23_attention_weight_max": 22.903766632080078,
      "activations/layer23_attention_weight_min": -16.168668746948242,
      "activations/layer2_attention_weight_max": 25.27798843383789,
      "activations/layer2_attention_weight_min": -23.262340545654297,
      "activations/layer3_attention_weight_max": 53.70478439331055,
      "activations/layer3_attention_weight_min": -54.50128936767578,
      "activations/layer4_attention_weight_max": 82.49015045166016,
      "activations/layer4_attention_weight_min": -75.06129455566406,
      "activations/layer5_attention_weight_max": 137.59254455566406,
      "activations/layer5_attention_weight_min": -114.78252410888672,
      "activations/layer6_attention_weight_max": 64.67352294921875,
      "activations/layer6_attention_weight_min": -56.96533966064453,
      "activations/layer7_attention_weight_max": 68.99658966064453,
      "activations/layer7_attention_weight_min": -58.74557113647461,
      "activations/layer8_attention_weight_max": 60.13235092163086,
      "activations/layer8_attention_weight_min": -49.038936614990234,
      "activations/layer9_attention_weight_max": 73.29410552978516,
      "activations/layer9_attention_weight_min": -70.926513671875,
      "epoch": 1.23,
      "learning_rate": 0.00014353598484848483,
      "loss": 3.0583,
      "step": 21100
    },
    {
      "activations/layer0_attention_weight_max": 14.685050964355469,
      "activations/layer0_attention_weight_min": -13.170129776000977,
      "activations/layer10_attention_weight_max": 68.2835922241211,
      "activations/layer10_attention_weight_min": -57.57957458496094,
      "activations/layer11_attention_weight_max": 41.45720672607422,
      "activations/layer11_attention_weight_min": -39.620643615722656,
      "activations/layer12_attention_weight_max": 25.599592208862305,
      "activations/layer12_attention_weight_min": -24.8712158203125,
      "activations/layer13_attention_weight_max": 81.08883666992188,
      "activations/layer13_attention_weight_min": -53.231773376464844,
      "activations/layer14_attention_weight_max": 62.075252532958984,
      "activations/layer14_attention_weight_min": -40.72842788696289,
      "activations/layer15_attention_weight_max": 62.979862213134766,
      "activations/layer15_attention_weight_min": -38.28657913208008,
      "activations/layer16_attention_weight_max": 57.758487701416016,
      "activations/layer16_attention_weight_min": -36.15769577026367,
      "activations/layer17_attention_weight_max": 55.89396667480469,
      "activations/layer17_attention_weight_min": -35.22818374633789,
      "activations/layer18_attention_weight_max": 75.61260223388672,
      "activations/layer18_attention_weight_min": -38.77025604248047,
      "activations/layer19_attention_weight_max": 21.972148895263672,
      "activations/layer19_attention_weight_min": -13.353813171386719,
      "activations/layer1_attention_weight_max": 14.472774505615234,
      "activations/layer1_attention_weight_min": -13.33127498626709,
      "activations/layer20_attention_weight_max": 22.129858016967773,
      "activations/layer20_attention_weight_min": -13.446745872497559,
      "activations/layer21_attention_weight_max": 46.26443862915039,
      "activations/layer21_attention_weight_min": -19.505510330200195,
      "activations/layer22_attention_weight_max": 36.72270584106445,
      "activations/layer22_attention_weight_min": -18.955333709716797,
      "activations/layer23_attention_weight_max": 23.634382247924805,
      "activations/layer23_attention_weight_min": -16.56822395324707,
      "activations/layer2_attention_weight_max": 25.90656089782715,
      "activations/layer2_attention_weight_min": -24.105030059814453,
      "activations/layer3_attention_weight_max": 55.512428283691406,
      "activations/layer3_attention_weight_min": -60.27850341796875,
      "activations/layer4_attention_weight_max": 84.4398422241211,
      "activations/layer4_attention_weight_min": -76.17554473876953,
      "activations/layer5_attention_weight_max": 133.66482543945312,
      "activations/layer5_attention_weight_min": -126.33741760253906,
      "activations/layer6_attention_weight_max": 62.74352264404297,
      "activations/layer6_attention_weight_min": -57.0705451965332,
      "activations/layer7_attention_weight_max": 61.35890579223633,
      "activations/layer7_attention_weight_min": -56.46921920776367,
      "activations/layer8_attention_weight_max": 47.99839782714844,
      "activations/layer8_attention_weight_min": -45.3114013671875,
      "activations/layer9_attention_weight_max": 78.8175277709961,
      "activations/layer9_attention_weight_min": -65.79012298583984,
      "epoch": 1.23,
      "learning_rate": 0.00014351704545454544,
      "loss": 3.0507,
      "step": 21150
    },
    {
      "activations/layer0_attention_weight_max": 14.058157920837402,
      "activations/layer0_attention_weight_min": -12.884405136108398,
      "activations/layer10_attention_weight_max": 67.72334289550781,
      "activations/layer10_attention_weight_min": -56.821868896484375,
      "activations/layer11_attention_weight_max": 48.586578369140625,
      "activations/layer11_attention_weight_min": -44.645042419433594,
      "activations/layer12_attention_weight_max": 27.035078048706055,
      "activations/layer12_attention_weight_min": -23.14392852783203,
      "activations/layer13_attention_weight_max": 54.73347091674805,
      "activations/layer13_attention_weight_min": -40.096778869628906,
      "activations/layer14_attention_weight_max": 50.95411682128906,
      "activations/layer14_attention_weight_min": -43.18888854980469,
      "activations/layer15_attention_weight_max": 53.74474334716797,
      "activations/layer15_attention_weight_min": -40.508872985839844,
      "activations/layer16_attention_weight_max": 50.383216857910156,
      "activations/layer16_attention_weight_min": -33.33974838256836,
      "activations/layer17_attention_weight_max": 61.76476287841797,
      "activations/layer17_attention_weight_min": -36.6527099609375,
      "activations/layer18_attention_weight_max": 69.41062927246094,
      "activations/layer18_attention_weight_min": -39.56932067871094,
      "activations/layer19_attention_weight_max": 19.668638229370117,
      "activations/layer19_attention_weight_min": -16.11764907836914,
      "activations/layer1_attention_weight_max": 15.010009765625,
      "activations/layer1_attention_weight_min": -11.639673233032227,
      "activations/layer20_attention_weight_max": 17.329248428344727,
      "activations/layer20_attention_weight_min": -14.088431358337402,
      "activations/layer21_attention_weight_max": 37.21342086791992,
      "activations/layer21_attention_weight_min": -20.807382583618164,
      "activations/layer22_attention_weight_max": 32.124019622802734,
      "activations/layer22_attention_weight_min": -18.780263900756836,
      "activations/layer23_attention_weight_max": 23.330007553100586,
      "activations/layer23_attention_weight_min": -14.088640213012695,
      "activations/layer2_attention_weight_max": 24.402189254760742,
      "activations/layer2_attention_weight_min": -23.775434494018555,
      "activations/layer3_attention_weight_max": 51.668601989746094,
      "activations/layer3_attention_weight_min": -53.86973571777344,
      "activations/layer4_attention_weight_max": 79.68765258789062,
      "activations/layer4_attention_weight_min": -74.32112884521484,
      "activations/layer5_attention_weight_max": 126.14939880371094,
      "activations/layer5_attention_weight_min": -114.01482391357422,
      "activations/layer6_attention_weight_max": 60.10167694091797,
      "activations/layer6_attention_weight_min": -56.02863693237305,
      "activations/layer7_attention_weight_max": 63.5195198059082,
      "activations/layer7_attention_weight_min": -56.05287170410156,
      "activations/layer8_attention_weight_max": 52.24651336669922,
      "activations/layer8_attention_weight_min": -46.150001525878906,
      "activations/layer9_attention_weight_max": 69.25847625732422,
      "activations/layer9_attention_weight_min": -65.92821502685547,
      "epoch": 1.23,
      "learning_rate": 0.00014349810606060606,
      "loss": 3.0645,
      "step": 21200
    },
    {
      "activations/layer0_attention_weight_max": 14.172636032104492,
      "activations/layer0_attention_weight_min": -12.119277954101562,
      "activations/layer10_attention_weight_max": 69.58941650390625,
      "activations/layer10_attention_weight_min": -61.3685417175293,
      "activations/layer11_attention_weight_max": 44.66238021850586,
      "activations/layer11_attention_weight_min": -38.40476608276367,
      "activations/layer12_attention_weight_max": 29.3896541595459,
      "activations/layer12_attention_weight_min": -23.791704177856445,
      "activations/layer13_attention_weight_max": 61.27832794189453,
      "activations/layer13_attention_weight_min": -42.2118034362793,
      "activations/layer14_attention_weight_max": 52.18069839477539,
      "activations/layer14_attention_weight_min": -34.92658233642578,
      "activations/layer15_attention_weight_max": 52.99003601074219,
      "activations/layer15_attention_weight_min": -39.903106689453125,
      "activations/layer16_attention_weight_max": 45.30415725708008,
      "activations/layer16_attention_weight_min": -33.16521072387695,
      "activations/layer17_attention_weight_max": 54.421688079833984,
      "activations/layer17_attention_weight_min": -39.036502838134766,
      "activations/layer18_attention_weight_max": 65.00869750976562,
      "activations/layer18_attention_weight_min": -43.8716926574707,
      "activations/layer19_attention_weight_max": 20.74772834777832,
      "activations/layer19_attention_weight_min": -13.73493480682373,
      "activations/layer1_attention_weight_max": 15.154298782348633,
      "activations/layer1_attention_weight_min": -10.8561429977417,
      "activations/layer20_attention_weight_max": 18.276878356933594,
      "activations/layer20_attention_weight_min": -15.95937728881836,
      "activations/layer21_attention_weight_max": 36.77516555786133,
      "activations/layer21_attention_weight_min": -20.881946563720703,
      "activations/layer22_attention_weight_max": 35.494136810302734,
      "activations/layer22_attention_weight_min": -18.278968811035156,
      "activations/layer23_attention_weight_max": 25.851863861083984,
      "activations/layer23_attention_weight_min": -13.591949462890625,
      "activations/layer2_attention_weight_max": 23.91469383239746,
      "activations/layer2_attention_weight_min": -23.51263999938965,
      "activations/layer3_attention_weight_max": 54.10165023803711,
      "activations/layer3_attention_weight_min": -54.71281051635742,
      "activations/layer4_attention_weight_max": 84.94692993164062,
      "activations/layer4_attention_weight_min": -77.21187591552734,
      "activations/layer5_attention_weight_max": 132.82623291015625,
      "activations/layer5_attention_weight_min": -122.98294067382812,
      "activations/layer6_attention_weight_max": 62.536746978759766,
      "activations/layer6_attention_weight_min": -58.65223693847656,
      "activations/layer7_attention_weight_max": 63.765987396240234,
      "activations/layer7_attention_weight_min": -54.393096923828125,
      "activations/layer8_attention_weight_max": 51.423526763916016,
      "activations/layer8_attention_weight_min": -45.66633224487305,
      "activations/layer9_attention_weight_max": 64.63587951660156,
      "activations/layer9_attention_weight_min": -61.0587272644043,
      "epoch": 1.23,
      "learning_rate": 0.00014347916666666665,
      "loss": 3.0597,
      "step": 21250
    },
    {
      "activations/layer0_attention_weight_max": 14.093114852905273,
      "activations/layer0_attention_weight_min": -12.75157642364502,
      "activations/layer10_attention_weight_max": 59.04634475708008,
      "activations/layer10_attention_weight_min": -52.9928092956543,
      "activations/layer11_attention_weight_max": 46.844112396240234,
      "activations/layer11_attention_weight_min": -37.699947357177734,
      "activations/layer12_attention_weight_max": 25.94441032409668,
      "activations/layer12_attention_weight_min": -23.217729568481445,
      "activations/layer13_attention_weight_max": 54.55687713623047,
      "activations/layer13_attention_weight_min": -41.97935485839844,
      "activations/layer14_attention_weight_max": 46.010040283203125,
      "activations/layer14_attention_weight_min": -36.87107849121094,
      "activations/layer15_attention_weight_max": 53.740966796875,
      "activations/layer15_attention_weight_min": -40.778228759765625,
      "activations/layer16_attention_weight_max": 44.55524444580078,
      "activations/layer16_attention_weight_min": -33.10411071777344,
      "activations/layer17_attention_weight_max": 56.1846809387207,
      "activations/layer17_attention_weight_min": -36.430511474609375,
      "activations/layer18_attention_weight_max": 56.66590118408203,
      "activations/layer18_attention_weight_min": -39.55671310424805,
      "activations/layer19_attention_weight_max": 22.03358268737793,
      "activations/layer19_attention_weight_min": -13.405969619750977,
      "activations/layer1_attention_weight_max": 14.316378593444824,
      "activations/layer1_attention_weight_min": -12.75069808959961,
      "activations/layer20_attention_weight_max": 19.058494567871094,
      "activations/layer20_attention_weight_min": -14.422276496887207,
      "activations/layer21_attention_weight_max": 37.067298889160156,
      "activations/layer21_attention_weight_min": -19.67990493774414,
      "activations/layer22_attention_weight_max": 31.89820671081543,
      "activations/layer22_attention_weight_min": -17.25742530822754,
      "activations/layer23_attention_weight_max": 20.31914710998535,
      "activations/layer23_attention_weight_min": -15.497072219848633,
      "activations/layer2_attention_weight_max": 26.099071502685547,
      "activations/layer2_attention_weight_min": -24.249387741088867,
      "activations/layer3_attention_weight_max": 53.37681198120117,
      "activations/layer3_attention_weight_min": -55.19621658325195,
      "activations/layer4_attention_weight_max": 84.00167083740234,
      "activations/layer4_attention_weight_min": -74.6006088256836,
      "activations/layer5_attention_weight_max": 134.2445831298828,
      "activations/layer5_attention_weight_min": -117.85826110839844,
      "activations/layer6_attention_weight_max": 62.213035583496094,
      "activations/layer6_attention_weight_min": -56.20979690551758,
      "activations/layer7_attention_weight_max": 62.31934356689453,
      "activations/layer7_attention_weight_min": -54.17047119140625,
      "activations/layer8_attention_weight_max": 50.4453010559082,
      "activations/layer8_attention_weight_min": -45.433963775634766,
      "activations/layer9_attention_weight_max": 68.04562377929688,
      "activations/layer9_attention_weight_min": -63.07832717895508,
      "epoch": 1.24,
      "learning_rate": 0.00014346022727272726,
      "loss": 3.0425,
      "step": 21300
    },
    {
      "activations/layer0_attention_weight_max": 14.990442276000977,
      "activations/layer0_attention_weight_min": -12.37381649017334,
      "activations/layer10_attention_weight_max": 77.42431640625,
      "activations/layer10_attention_weight_min": -58.15944290161133,
      "activations/layer11_attention_weight_max": 50.84394454956055,
      "activations/layer11_attention_weight_min": -42.34978103637695,
      "activations/layer12_attention_weight_max": 35.03093338012695,
      "activations/layer12_attention_weight_min": -25.622018814086914,
      "activations/layer13_attention_weight_max": 106.25517272949219,
      "activations/layer13_attention_weight_min": -56.517372131347656,
      "activations/layer14_attention_weight_max": 114.22394561767578,
      "activations/layer14_attention_weight_min": -52.05434799194336,
      "activations/layer15_attention_weight_max": 75.17859649658203,
      "activations/layer15_attention_weight_min": -40.553714752197266,
      "activations/layer16_attention_weight_max": 66.91808319091797,
      "activations/layer16_attention_weight_min": -32.719242095947266,
      "activations/layer17_attention_weight_max": 79.84304809570312,
      "activations/layer17_attention_weight_min": -38.21287155151367,
      "activations/layer18_attention_weight_max": 79.92926788330078,
      "activations/layer18_attention_weight_min": -32.907989501953125,
      "activations/layer19_attention_weight_max": 27.557315826416016,
      "activations/layer19_attention_weight_min": -16.06855583190918,
      "activations/layer1_attention_weight_max": 14.884848594665527,
      "activations/layer1_attention_weight_min": -12.338056564331055,
      "activations/layer20_attention_weight_max": 22.980039596557617,
      "activations/layer20_attention_weight_min": -14.162796974182129,
      "activations/layer21_attention_weight_max": 48.67343521118164,
      "activations/layer21_attention_weight_min": -19.989185333251953,
      "activations/layer22_attention_weight_max": 42.70612716674805,
      "activations/layer22_attention_weight_min": -17.861270904541016,
      "activations/layer23_attention_weight_max": 27.9326171875,
      "activations/layer23_attention_weight_min": -15.278105735778809,
      "activations/layer2_attention_weight_max": 26.904048919677734,
      "activations/layer2_attention_weight_min": -25.378326416015625,
      "activations/layer3_attention_weight_max": 56.00517272949219,
      "activations/layer3_attention_weight_min": -58.637855529785156,
      "activations/layer4_attention_weight_max": 84.68684387207031,
      "activations/layer4_attention_weight_min": -77.23761749267578,
      "activations/layer5_attention_weight_max": 138.1611785888672,
      "activations/layer5_attention_weight_min": -122.22329711914062,
      "activations/layer6_attention_weight_max": 68.27577209472656,
      "activations/layer6_attention_weight_min": -57.93745040893555,
      "activations/layer7_attention_weight_max": 70.11702728271484,
      "activations/layer7_attention_weight_min": -56.553829193115234,
      "activations/layer8_attention_weight_max": 62.81740188598633,
      "activations/layer8_attention_weight_min": -58.64091110229492,
      "activations/layer9_attention_weight_max": 79.18849182128906,
      "activations/layer9_attention_weight_min": -72.2077865600586,
      "epoch": 1.24,
      "learning_rate": 0.00014344128787878785,
      "loss": 3.0426,
      "step": 21350
    },
    {
      "activations/layer0_attention_weight_max": 14.409531593322754,
      "activations/layer0_attention_weight_min": -12.525345802307129,
      "activations/layer10_attention_weight_max": 72.79692077636719,
      "activations/layer10_attention_weight_min": -59.62004089355469,
      "activations/layer11_attention_weight_max": 47.9290771484375,
      "activations/layer11_attention_weight_min": -39.57469940185547,
      "activations/layer12_attention_weight_max": 27.517196655273438,
      "activations/layer12_attention_weight_min": -23.60875701904297,
      "activations/layer13_attention_weight_max": 69.11331939697266,
      "activations/layer13_attention_weight_min": -45.366004943847656,
      "activations/layer14_attention_weight_max": 49.99361801147461,
      "activations/layer14_attention_weight_min": -36.01018524169922,
      "activations/layer15_attention_weight_max": 58.230587005615234,
      "activations/layer15_attention_weight_min": -42.57072830200195,
      "activations/layer16_attention_weight_max": 45.673133850097656,
      "activations/layer16_attention_weight_min": -33.08903503417969,
      "activations/layer17_attention_weight_max": 63.61904525756836,
      "activations/layer17_attention_weight_min": -37.90061950683594,
      "activations/layer18_attention_weight_max": 62.828800201416016,
      "activations/layer18_attention_weight_min": -37.75139617919922,
      "activations/layer19_attention_weight_max": 28.050241470336914,
      "activations/layer19_attention_weight_min": -14.581059455871582,
      "activations/layer1_attention_weight_max": 15.268516540527344,
      "activations/layer1_attention_weight_min": -12.593840599060059,
      "activations/layer20_attention_weight_max": 17.716917037963867,
      "activations/layer20_attention_weight_min": -14.463546752929688,
      "activations/layer21_attention_weight_max": 41.913482666015625,
      "activations/layer21_attention_weight_min": -21.095806121826172,
      "activations/layer22_attention_weight_max": 37.72779083251953,
      "activations/layer22_attention_weight_min": -21.012929916381836,
      "activations/layer23_attention_weight_max": 26.657011032104492,
      "activations/layer23_attention_weight_min": -13.98011589050293,
      "activations/layer2_attention_weight_max": 25.51339340209961,
      "activations/layer2_attention_weight_min": -24.969135284423828,
      "activations/layer3_attention_weight_max": 54.89068603515625,
      "activations/layer3_attention_weight_min": -56.89722442626953,
      "activations/layer4_attention_weight_max": 84.39657592773438,
      "activations/layer4_attention_weight_min": -78.00997924804688,
      "activations/layer5_attention_weight_max": 136.75537109375,
      "activations/layer5_attention_weight_min": -119.59697723388672,
      "activations/layer6_attention_weight_max": 71.1677474975586,
      "activations/layer6_attention_weight_min": -62.307647705078125,
      "activations/layer7_attention_weight_max": 70.32637023925781,
      "activations/layer7_attention_weight_min": -58.35853576660156,
      "activations/layer8_attention_weight_max": 58.84455871582031,
      "activations/layer8_attention_weight_min": -50.22993850708008,
      "activations/layer9_attention_weight_max": 72.90666961669922,
      "activations/layer9_attention_weight_min": -61.19363021850586,
      "epoch": 1.24,
      "learning_rate": 0.00014342234848484846,
      "loss": 3.0445,
      "step": 21400
    },
    {
      "activations/layer0_attention_weight_max": 12.937980651855469,
      "activations/layer0_attention_weight_min": -12.049177169799805,
      "activations/layer10_attention_weight_max": 74.06231689453125,
      "activations/layer10_attention_weight_min": -58.37449264526367,
      "activations/layer11_attention_weight_max": 48.17365264892578,
      "activations/layer11_attention_weight_min": -39.151084899902344,
      "activations/layer12_attention_weight_max": 31.42993927001953,
      "activations/layer12_attention_weight_min": -23.660850524902344,
      "activations/layer13_attention_weight_max": 61.92427062988281,
      "activations/layer13_attention_weight_min": -44.19770050048828,
      "activations/layer14_attention_weight_max": 41.17729568481445,
      "activations/layer14_attention_weight_min": -34.52815628051758,
      "activations/layer15_attention_weight_max": 53.38680648803711,
      "activations/layer15_attention_weight_min": -40.410030364990234,
      "activations/layer16_attention_weight_max": 48.9637565612793,
      "activations/layer16_attention_weight_min": -36.41033172607422,
      "activations/layer17_attention_weight_max": 54.89251708984375,
      "activations/layer17_attention_weight_min": -40.6803092956543,
      "activations/layer18_attention_weight_max": 63.8014030456543,
      "activations/layer18_attention_weight_min": -41.312740325927734,
      "activations/layer19_attention_weight_max": 20.60708236694336,
      "activations/layer19_attention_weight_min": -13.659136772155762,
      "activations/layer1_attention_weight_max": 14.393641471862793,
      "activations/layer1_attention_weight_min": -10.435759544372559,
      "activations/layer20_attention_weight_max": 19.968425750732422,
      "activations/layer20_attention_weight_min": -14.226468086242676,
      "activations/layer21_attention_weight_max": 35.52846908569336,
      "activations/layer21_attention_weight_min": -19.874408721923828,
      "activations/layer22_attention_weight_max": 32.61191177368164,
      "activations/layer22_attention_weight_min": -16.71521759033203,
      "activations/layer23_attention_weight_max": 24.632667541503906,
      "activations/layer23_attention_weight_min": -13.708181381225586,
      "activations/layer2_attention_weight_max": 23.973896026611328,
      "activations/layer2_attention_weight_min": -23.933452606201172,
      "activations/layer3_attention_weight_max": 54.72003173828125,
      "activations/layer3_attention_weight_min": -58.24381637573242,
      "activations/layer4_attention_weight_max": 82.66216278076172,
      "activations/layer4_attention_weight_min": -75.82102966308594,
      "activations/layer5_attention_weight_max": 132.7720947265625,
      "activations/layer5_attention_weight_min": -119.07250213623047,
      "activations/layer6_attention_weight_max": 64.63148498535156,
      "activations/layer6_attention_weight_min": -58.255943298339844,
      "activations/layer7_attention_weight_max": 63.25742721557617,
      "activations/layer7_attention_weight_min": -56.765926361083984,
      "activations/layer8_attention_weight_max": 53.72798156738281,
      "activations/layer8_attention_weight_min": -46.25932312011719,
      "activations/layer9_attention_weight_max": 71.84962463378906,
      "activations/layer9_attention_weight_min": -65.89244842529297,
      "epoch": 1.25,
      "learning_rate": 0.00014340378787878786,
      "loss": 3.0647,
      "step": 21450
    },
    {
      "activations/layer0_attention_weight_max": 13.489248275756836,
      "activations/layer0_attention_weight_min": -13.056416511535645,
      "activations/layer10_attention_weight_max": 64.86396789550781,
      "activations/layer10_attention_weight_min": -58.40154266357422,
      "activations/layer11_attention_weight_max": 40.023155212402344,
      "activations/layer11_attention_weight_min": -36.23880386352539,
      "activations/layer12_attention_weight_max": 28.16392707824707,
      "activations/layer12_attention_weight_min": -25.19232940673828,
      "activations/layer13_attention_weight_max": 53.89462661743164,
      "activations/layer13_attention_weight_min": -46.079002380371094,
      "activations/layer14_attention_weight_max": 53.138668060302734,
      "activations/layer14_attention_weight_min": -39.478302001953125,
      "activations/layer15_attention_weight_max": 54.07218933105469,
      "activations/layer15_attention_weight_min": -41.12297058105469,
      "activations/layer16_attention_weight_max": 44.9547119140625,
      "activations/layer16_attention_weight_min": -31.464937210083008,
      "activations/layer17_attention_weight_max": 60.29612350463867,
      "activations/layer17_attention_weight_min": -35.996192932128906,
      "activations/layer18_attention_weight_max": 62.12131118774414,
      "activations/layer18_attention_weight_min": -39.42170333862305,
      "activations/layer19_attention_weight_max": 22.570341110229492,
      "activations/layer19_attention_weight_min": -13.646387100219727,
      "activations/layer1_attention_weight_max": 14.610676765441895,
      "activations/layer1_attention_weight_min": -12.679696083068848,
      "activations/layer20_attention_weight_max": 21.614660263061523,
      "activations/layer20_attention_weight_min": -13.888448715209961,
      "activations/layer21_attention_weight_max": 37.77088165283203,
      "activations/layer21_attention_weight_min": -20.854232788085938,
      "activations/layer22_attention_weight_max": 34.51490020751953,
      "activations/layer22_attention_weight_min": -17.14871597290039,
      "activations/layer23_attention_weight_max": 21.83163070678711,
      "activations/layer23_attention_weight_min": -16.987445831298828,
      "activations/layer2_attention_weight_max": 25.460981369018555,
      "activations/layer2_attention_weight_min": -22.257877349853516,
      "activations/layer3_attention_weight_max": 54.85142135620117,
      "activations/layer3_attention_weight_min": -57.572574615478516,
      "activations/layer4_attention_weight_max": 80.81517791748047,
      "activations/layer4_attention_weight_min": -77.40400695800781,
      "activations/layer5_attention_weight_max": 134.7584686279297,
      "activations/layer5_attention_weight_min": -118.90702819824219,
      "activations/layer6_attention_weight_max": 62.082061767578125,
      "activations/layer6_attention_weight_min": -56.74468994140625,
      "activations/layer7_attention_weight_max": 62.38688278198242,
      "activations/layer7_attention_weight_min": -56.19289016723633,
      "activations/layer8_attention_weight_max": 50.431880950927734,
      "activations/layer8_attention_weight_min": -48.653785705566406,
      "activations/layer9_attention_weight_max": 67.7224349975586,
      "activations/layer9_attention_weight_min": -67.9634017944336,
      "epoch": 1.25,
      "learning_rate": 0.00014338484848484847,
      "loss": 3.0426,
      "step": 21500
    },
    {
      "activations/layer0_attention_weight_max": 12.301728248596191,
      "activations/layer0_attention_weight_min": -12.270528793334961,
      "activations/layer10_attention_weight_max": 69.72721099853516,
      "activations/layer10_attention_weight_min": -61.84701919555664,
      "activations/layer11_attention_weight_max": 42.935089111328125,
      "activations/layer11_attention_weight_min": -41.31688690185547,
      "activations/layer12_attention_weight_max": 27.247655868530273,
      "activations/layer12_attention_weight_min": -25.39352035522461,
      "activations/layer13_attention_weight_max": 67.12467956542969,
      "activations/layer13_attention_weight_min": -48.0914192199707,
      "activations/layer14_attention_weight_max": 58.801334381103516,
      "activations/layer14_attention_weight_min": -44.14677810668945,
      "activations/layer15_attention_weight_max": 53.87446975708008,
      "activations/layer15_attention_weight_min": -41.89298629760742,
      "activations/layer16_attention_weight_max": 49.60845947265625,
      "activations/layer16_attention_weight_min": -34.91255569458008,
      "activations/layer17_attention_weight_max": 57.58176803588867,
      "activations/layer17_attention_weight_min": -39.844688415527344,
      "activations/layer18_attention_weight_max": 63.62068176269531,
      "activations/layer18_attention_weight_min": -37.83152389526367,
      "activations/layer19_attention_weight_max": 20.921485900878906,
      "activations/layer19_attention_weight_min": -13.770545959472656,
      "activations/layer1_attention_weight_max": 15.291769981384277,
      "activations/layer1_attention_weight_min": -10.598349571228027,
      "activations/layer20_attention_weight_max": 18.288610458374023,
      "activations/layer20_attention_weight_min": -13.854876518249512,
      "activations/layer21_attention_weight_max": 35.37446594238281,
      "activations/layer21_attention_weight_min": -23.1746826171875,
      "activations/layer22_attention_weight_max": 35.02800369262695,
      "activations/layer22_attention_weight_min": -17.88203239440918,
      "activations/layer23_attention_weight_max": 25.538328170776367,
      "activations/layer23_attention_weight_min": -13.106683731079102,
      "activations/layer2_attention_weight_max": 24.203481674194336,
      "activations/layer2_attention_weight_min": -23.925819396972656,
      "activations/layer3_attention_weight_max": 54.110511779785156,
      "activations/layer3_attention_weight_min": -57.9173698425293,
      "activations/layer4_attention_weight_max": 79.50497436523438,
      "activations/layer4_attention_weight_min": -76.24549865722656,
      "activations/layer5_attention_weight_max": 123.62801361083984,
      "activations/layer5_attention_weight_min": -116.77062225341797,
      "activations/layer6_attention_weight_max": 58.47716522216797,
      "activations/layer6_attention_weight_min": -55.89457321166992,
      "activations/layer7_attention_weight_max": 57.628047943115234,
      "activations/layer7_attention_weight_min": -54.24465560913086,
      "activations/layer8_attention_weight_max": 47.83541488647461,
      "activations/layer8_attention_weight_min": -45.670501708984375,
      "activations/layer9_attention_weight_max": 70.59551239013672,
      "activations/layer9_attention_weight_min": -64.08403778076172,
      "epoch": 1.25,
      "learning_rate": 0.00014336590909090906,
      "loss": 3.051,
      "step": 21550
    },
    {
      "activations/layer0_attention_weight_max": 14.789806365966797,
      "activations/layer0_attention_weight_min": -12.696785926818848,
      "activations/layer10_attention_weight_max": 59.03070831298828,
      "activations/layer10_attention_weight_min": -57.330406188964844,
      "activations/layer11_attention_weight_max": 40.82192611694336,
      "activations/layer11_attention_weight_min": -41.199913024902344,
      "activations/layer12_attention_weight_max": 26.652318954467773,
      "activations/layer12_attention_weight_min": -22.75404167175293,
      "activations/layer13_attention_weight_max": 61.33758544921875,
      "activations/layer13_attention_weight_min": -49.486976623535156,
      "activations/layer14_attention_weight_max": 51.58848571777344,
      "activations/layer14_attention_weight_min": -39.936058044433594,
      "activations/layer15_attention_weight_max": 54.108436584472656,
      "activations/layer15_attention_weight_min": -42.5551643371582,
      "activations/layer16_attention_weight_max": 49.66288757324219,
      "activations/layer16_attention_weight_min": -34.45329666137695,
      "activations/layer17_attention_weight_max": 64.75631713867188,
      "activations/layer17_attention_weight_min": -40.12956619262695,
      "activations/layer18_attention_weight_max": 66.27281951904297,
      "activations/layer18_attention_weight_min": -37.04655456542969,
      "activations/layer19_attention_weight_max": 24.30242919921875,
      "activations/layer19_attention_weight_min": -15.408526420593262,
      "activations/layer1_attention_weight_max": 15.787445068359375,
      "activations/layer1_attention_weight_min": -10.878148078918457,
      "activations/layer20_attention_weight_max": 20.730619430541992,
      "activations/layer20_attention_weight_min": -15.545257568359375,
      "activations/layer21_attention_weight_max": 41.49652099609375,
      "activations/layer21_attention_weight_min": -20.973800659179688,
      "activations/layer22_attention_weight_max": 39.119808197021484,
      "activations/layer22_attention_weight_min": -20.682435989379883,
      "activations/layer23_attention_weight_max": 22.252696990966797,
      "activations/layer23_attention_weight_min": -14.184768676757812,
      "activations/layer2_attention_weight_max": 25.231618881225586,
      "activations/layer2_attention_weight_min": -24.079893112182617,
      "activations/layer3_attention_weight_max": 56.09984588623047,
      "activations/layer3_attention_weight_min": -58.662044525146484,
      "activations/layer4_attention_weight_max": 83.8690414428711,
      "activations/layer4_attention_weight_min": -78.81658935546875,
      "activations/layer5_attention_weight_max": 126.88516235351562,
      "activations/layer5_attention_weight_min": -119.50008392333984,
      "activations/layer6_attention_weight_max": 60.765296936035156,
      "activations/layer6_attention_weight_min": -57.206626892089844,
      "activations/layer7_attention_weight_max": 57.43568801879883,
      "activations/layer7_attention_weight_min": -56.015628814697266,
      "activations/layer8_attention_weight_max": 46.83260726928711,
      "activations/layer8_attention_weight_min": -45.74797821044922,
      "activations/layer9_attention_weight_max": 62.35195541381836,
      "activations/layer9_attention_weight_min": -60.433326721191406,
      "epoch": 1.26,
      "learning_rate": 0.00014334696969696967,
      "loss": 3.0476,
      "step": 21600
    },
    {
      "activations/layer0_attention_weight_max": 16.867637634277344,
      "activations/layer0_attention_weight_min": -12.502718925476074,
      "activations/layer10_attention_weight_max": 66.33436584472656,
      "activations/layer10_attention_weight_min": -55.61400604248047,
      "activations/layer11_attention_weight_max": 43.54070281982422,
      "activations/layer11_attention_weight_min": -38.21574783325195,
      "activations/layer12_attention_weight_max": 25.796953201293945,
      "activations/layer12_attention_weight_min": -23.426877975463867,
      "activations/layer13_attention_weight_max": 70.47770690917969,
      "activations/layer13_attention_weight_min": -55.942344665527344,
      "activations/layer14_attention_weight_max": 63.50432205200195,
      "activations/layer14_attention_weight_min": -50.05704116821289,
      "activations/layer15_attention_weight_max": 57.02293395996094,
      "activations/layer15_attention_weight_min": -40.00789260864258,
      "activations/layer16_attention_weight_max": 52.50510787963867,
      "activations/layer16_attention_weight_min": -35.176963806152344,
      "activations/layer17_attention_weight_max": 61.10381317138672,
      "activations/layer17_attention_weight_min": -39.62826919555664,
      "activations/layer18_attention_weight_max": 71.09766387939453,
      "activations/layer18_attention_weight_min": -38.35737228393555,
      "activations/layer19_attention_weight_max": 22.19355010986328,
      "activations/layer19_attention_weight_min": -13.428712844848633,
      "activations/layer1_attention_weight_max": 14.801339149475098,
      "activations/layer1_attention_weight_min": -11.85389232635498,
      "activations/layer20_attention_weight_max": 20.72224998474121,
      "activations/layer20_attention_weight_min": -15.777970314025879,
      "activations/layer21_attention_weight_max": 38.691410064697266,
      "activations/layer21_attention_weight_min": -22.823360443115234,
      "activations/layer22_attention_weight_max": 33.03256607055664,
      "activations/layer22_attention_weight_min": -19.774856567382812,
      "activations/layer23_attention_weight_max": 22.42713165283203,
      "activations/layer23_attention_weight_min": -15.974982261657715,
      "activations/layer2_attention_weight_max": 26.111892700195312,
      "activations/layer2_attention_weight_min": -23.970136642456055,
      "activations/layer3_attention_weight_max": 53.56190490722656,
      "activations/layer3_attention_weight_min": -56.477760314941406,
      "activations/layer4_attention_weight_max": 81.59619903564453,
      "activations/layer4_attention_weight_min": -77.05449676513672,
      "activations/layer5_attention_weight_max": 132.24290466308594,
      "activations/layer5_attention_weight_min": -113.70988464355469,
      "activations/layer6_attention_weight_max": 64.08369445800781,
      "activations/layer6_attention_weight_min": -55.285926818847656,
      "activations/layer7_attention_weight_max": 63.370262145996094,
      "activations/layer7_attention_weight_min": -55.667293548583984,
      "activations/layer8_attention_weight_max": 54.001060485839844,
      "activations/layer8_attention_weight_min": -50.437496185302734,
      "activations/layer9_attention_weight_max": 73.34275817871094,
      "activations/layer9_attention_weight_min": -66.00321197509766,
      "epoch": 1.26,
      "learning_rate": 0.0001433280303030303,
      "loss": 3.0603,
      "step": 21650
    },
    {
      "activations/layer0_attention_weight_max": 16.193145751953125,
      "activations/layer0_attention_weight_min": -12.254008293151855,
      "activations/layer10_attention_weight_max": 72.43517303466797,
      "activations/layer10_attention_weight_min": -58.96470642089844,
      "activations/layer11_attention_weight_max": 48.30022430419922,
      "activations/layer11_attention_weight_min": -41.08987808227539,
      "activations/layer12_attention_weight_max": 28.81814193725586,
      "activations/layer12_attention_weight_min": -25.583641052246094,
      "activations/layer13_attention_weight_max": 57.28062057495117,
      "activations/layer13_attention_weight_min": -43.60551452636719,
      "activations/layer14_attention_weight_max": 57.88956069946289,
      "activations/layer14_attention_weight_min": -35.86231994628906,
      "activations/layer15_attention_weight_max": 55.10576629638672,
      "activations/layer15_attention_weight_min": -38.794647216796875,
      "activations/layer16_attention_weight_max": 44.89265823364258,
      "activations/layer16_attention_weight_min": -29.682910919189453,
      "activations/layer17_attention_weight_max": 58.5386848449707,
      "activations/layer17_attention_weight_min": -36.37510681152344,
      "activations/layer18_attention_weight_max": 61.270442962646484,
      "activations/layer18_attention_weight_min": -39.8181037902832,
      "activations/layer19_attention_weight_max": 21.266860961914062,
      "activations/layer19_attention_weight_min": -13.747230529785156,
      "activations/layer1_attention_weight_max": 14.622199058532715,
      "activations/layer1_attention_weight_min": -11.353524208068848,
      "activations/layer20_attention_weight_max": 17.825037002563477,
      "activations/layer20_attention_weight_min": -14.027328491210938,
      "activations/layer21_attention_weight_max": 35.32967758178711,
      "activations/layer21_attention_weight_min": -19.145273208618164,
      "activations/layer22_attention_weight_max": 35.20218276977539,
      "activations/layer22_attention_weight_min": -19.06178855895996,
      "activations/layer23_attention_weight_max": 24.9814510345459,
      "activations/layer23_attention_weight_min": -14.4652099609375,
      "activations/layer2_attention_weight_max": 23.352378845214844,
      "activations/layer2_attention_weight_min": -23.827192306518555,
      "activations/layer3_attention_weight_max": 55.4697380065918,
      "activations/layer3_attention_weight_min": -59.19959259033203,
      "activations/layer4_attention_weight_max": 88.18209075927734,
      "activations/layer4_attention_weight_min": -79.55695343017578,
      "activations/layer5_attention_weight_max": 144.75830078125,
      "activations/layer5_attention_weight_min": -117.0489730834961,
      "activations/layer6_attention_weight_max": 64.93719482421875,
      "activations/layer6_attention_weight_min": -53.90155792236328,
      "activations/layer7_attention_weight_max": 65.36962127685547,
      "activations/layer7_attention_weight_min": -54.82674789428711,
      "activations/layer8_attention_weight_max": 58.07108688354492,
      "activations/layer8_attention_weight_min": -52.17068862915039,
      "activations/layer9_attention_weight_max": 86.1326675415039,
      "activations/layer9_attention_weight_min": -75.91838836669922,
      "epoch": 1.26,
      "learning_rate": 0.0001433090909090909,
      "loss": 3.0415,
      "step": 21700
    },
    {
      "activations/layer0_attention_weight_max": 13.891462326049805,
      "activations/layer0_attention_weight_min": -12.68077278137207,
      "activations/layer10_attention_weight_max": 69.68859100341797,
      "activations/layer10_attention_weight_min": -58.8677978515625,
      "activations/layer11_attention_weight_max": 43.8399658203125,
      "activations/layer11_attention_weight_min": -38.37174606323242,
      "activations/layer12_attention_weight_max": 26.981544494628906,
      "activations/layer12_attention_weight_min": -24.039121627807617,
      "activations/layer13_attention_weight_max": 67.67359924316406,
      "activations/layer13_attention_weight_min": -48.92927932739258,
      "activations/layer14_attention_weight_max": 57.32299041748047,
      "activations/layer14_attention_weight_min": -40.66268539428711,
      "activations/layer15_attention_weight_max": 59.085731506347656,
      "activations/layer15_attention_weight_min": -40.74880599975586,
      "activations/layer16_attention_weight_max": 48.02434158325195,
      "activations/layer16_attention_weight_min": -32.26871109008789,
      "activations/layer17_attention_weight_max": 56.99885940551758,
      "activations/layer17_attention_weight_min": -38.3330192565918,
      "activations/layer18_attention_weight_max": 65.51905059814453,
      "activations/layer18_attention_weight_min": -37.472694396972656,
      "activations/layer19_attention_weight_max": 21.112232208251953,
      "activations/layer19_attention_weight_min": -14.116449356079102,
      "activations/layer1_attention_weight_max": 14.861461639404297,
      "activations/layer1_attention_weight_min": -10.612317085266113,
      "activations/layer20_attention_weight_max": 22.964508056640625,
      "activations/layer20_attention_weight_min": -14.122124671936035,
      "activations/layer21_attention_weight_max": 47.18914794921875,
      "activations/layer21_attention_weight_min": -19.826852798461914,
      "activations/layer22_attention_weight_max": 40.7200813293457,
      "activations/layer22_attention_weight_min": -18.349878311157227,
      "activations/layer23_attention_weight_max": 27.267568588256836,
      "activations/layer23_attention_weight_min": -14.41526985168457,
      "activations/layer2_attention_weight_max": 24.807409286499023,
      "activations/layer2_attention_weight_min": -24.60584259033203,
      "activations/layer3_attention_weight_max": 53.748714447021484,
      "activations/layer3_attention_weight_min": -57.70243453979492,
      "activations/layer4_attention_weight_max": 81.19624328613281,
      "activations/layer4_attention_weight_min": -75.18059539794922,
      "activations/layer5_attention_weight_max": 122.71173095703125,
      "activations/layer5_attention_weight_min": -118.43806457519531,
      "activations/layer6_attention_weight_max": 58.770713806152344,
      "activations/layer6_attention_weight_min": -54.10145568847656,
      "activations/layer7_attention_weight_max": 57.220787048339844,
      "activations/layer7_attention_weight_min": -56.96839141845703,
      "activations/layer8_attention_weight_max": 51.73427200317383,
      "activations/layer8_attention_weight_min": -45.57573318481445,
      "activations/layer9_attention_weight_max": 66.77015686035156,
      "activations/layer9_attention_weight_min": -63.47541046142578,
      "epoch": 1.26,
      "learning_rate": 0.0001432901515151515,
      "loss": 3.0576,
      "step": 21750
    },
    {
      "activations/layer0_attention_weight_max": 15.485151290893555,
      "activations/layer0_attention_weight_min": -12.795143127441406,
      "activations/layer10_attention_weight_max": 72.6109390258789,
      "activations/layer10_attention_weight_min": -56.52553176879883,
      "activations/layer11_attention_weight_max": 57.10198211669922,
      "activations/layer11_attention_weight_min": -46.343284606933594,
      "activations/layer12_attention_weight_max": 28.645801544189453,
      "activations/layer12_attention_weight_min": -22.721799850463867,
      "activations/layer13_attention_weight_max": 70.59900665283203,
      "activations/layer13_attention_weight_min": -47.14207458496094,
      "activations/layer14_attention_weight_max": 51.09537124633789,
      "activations/layer14_attention_weight_min": -36.56196212768555,
      "activations/layer15_attention_weight_max": 61.65205764770508,
      "activations/layer15_attention_weight_min": -39.6984977722168,
      "activations/layer16_attention_weight_max": 43.866722106933594,
      "activations/layer16_attention_weight_min": -30.12715721130371,
      "activations/layer17_attention_weight_max": 54.94271469116211,
      "activations/layer17_attention_weight_min": -34.31877517700195,
      "activations/layer18_attention_weight_max": 56.99013137817383,
      "activations/layer18_attention_weight_min": -33.44133377075195,
      "activations/layer19_attention_weight_max": 21.0296573638916,
      "activations/layer19_attention_weight_min": -14.630142211914062,
      "activations/layer1_attention_weight_max": 14.96231460571289,
      "activations/layer1_attention_weight_min": -11.369662284851074,
      "activations/layer20_attention_weight_max": 18.565019607543945,
      "activations/layer20_attention_weight_min": -17.327795028686523,
      "activations/layer21_attention_weight_max": 33.09090805053711,
      "activations/layer21_attention_weight_min": -20.812891006469727,
      "activations/layer22_attention_weight_max": 36.81319808959961,
      "activations/layer22_attention_weight_min": -20.627527236938477,
      "activations/layer23_attention_weight_max": 24.253141403198242,
      "activations/layer23_attention_weight_min": -15.550376892089844,
      "activations/layer2_attention_weight_max": 24.808998107910156,
      "activations/layer2_attention_weight_min": -23.057628631591797,
      "activations/layer3_attention_weight_max": 54.30998611450195,
      "activations/layer3_attention_weight_min": -55.107643127441406,
      "activations/layer4_attention_weight_max": 86.0615234375,
      "activations/layer4_attention_weight_min": -73.8374252319336,
      "activations/layer5_attention_weight_max": 143.92446899414062,
      "activations/layer5_attention_weight_min": -123.964599609375,
      "activations/layer6_attention_weight_max": 67.64729309082031,
      "activations/layer6_attention_weight_min": -58.28248596191406,
      "activations/layer7_attention_weight_max": 70.471923828125,
      "activations/layer7_attention_weight_min": -56.845001220703125,
      "activations/layer8_attention_weight_max": 59.31819152832031,
      "activations/layer8_attention_weight_min": -48.6293830871582,
      "activations/layer9_attention_weight_max": 88.51387786865234,
      "activations/layer9_attention_weight_min": -77.19920349121094,
      "epoch": 1.27,
      "learning_rate": 0.0001432712121212121,
      "loss": 3.0446,
      "step": 21800
    },
    {
      "activations/layer0_attention_weight_max": 13.564114570617676,
      "activations/layer0_attention_weight_min": -12.955453872680664,
      "activations/layer10_attention_weight_max": 72.86196899414062,
      "activations/layer10_attention_weight_min": -55.11465835571289,
      "activations/layer11_attention_weight_max": 45.74869918823242,
      "activations/layer11_attention_weight_min": -37.12752151489258,
      "activations/layer12_attention_weight_max": 29.926448822021484,
      "activations/layer12_attention_weight_min": -23.737239837646484,
      "activations/layer13_attention_weight_max": 64.47579193115234,
      "activations/layer13_attention_weight_min": -42.85517501831055,
      "activations/layer14_attention_weight_max": 53.205345153808594,
      "activations/layer14_attention_weight_min": -43.38869094848633,
      "activations/layer15_attention_weight_max": 58.283416748046875,
      "activations/layer15_attention_weight_min": -40.610958099365234,
      "activations/layer16_attention_weight_max": 58.152225494384766,
      "activations/layer16_attention_weight_min": -34.41485595703125,
      "activations/layer17_attention_weight_max": 67.00789642333984,
      "activations/layer17_attention_weight_min": -39.683265686035156,
      "activations/layer18_attention_weight_max": 70.64151000976562,
      "activations/layer18_attention_weight_min": -40.74776840209961,
      "activations/layer19_attention_weight_max": 22.327913284301758,
      "activations/layer19_attention_weight_min": -15.746687889099121,
      "activations/layer1_attention_weight_max": 15.4942045211792,
      "activations/layer1_attention_weight_min": -11.191746711730957,
      "activations/layer20_attention_weight_max": 19.825218200683594,
      "activations/layer20_attention_weight_min": -14.769211769104004,
      "activations/layer21_attention_weight_max": 38.197566986083984,
      "activations/layer21_attention_weight_min": -20.214719772338867,
      "activations/layer22_attention_weight_max": 41.922367095947266,
      "activations/layer22_attention_weight_min": -18.785547256469727,
      "activations/layer23_attention_weight_max": 23.225135803222656,
      "activations/layer23_attention_weight_min": -13.670202255249023,
      "activations/layer2_attention_weight_max": 23.193450927734375,
      "activations/layer2_attention_weight_min": -22.610685348510742,
      "activations/layer3_attention_weight_max": 52.01732635498047,
      "activations/layer3_attention_weight_min": -53.55058288574219,
      "activations/layer4_attention_weight_max": 82.43708038330078,
      "activations/layer4_attention_weight_min": -75.03707122802734,
      "activations/layer5_attention_weight_max": 135.379638671875,
      "activations/layer5_attention_weight_min": -119.10787200927734,
      "activations/layer6_attention_weight_max": 63.53596496582031,
      "activations/layer6_attention_weight_min": -53.35218811035156,
      "activations/layer7_attention_weight_max": 65.37018585205078,
      "activations/layer7_attention_weight_min": -52.24810791015625,
      "activations/layer8_attention_weight_max": 57.3356819152832,
      "activations/layer8_attention_weight_min": -44.364383697509766,
      "activations/layer9_attention_weight_max": 73.67677307128906,
      "activations/layer9_attention_weight_min": -67.77674102783203,
      "epoch": 1.27,
      "learning_rate": 0.00014325227272727272,
      "loss": 3.0394,
      "step": 21850
    },
    {
      "activations/layer0_attention_weight_max": 14.273380279541016,
      "activations/layer0_attention_weight_min": -12.218077659606934,
      "activations/layer10_attention_weight_max": 63.40452194213867,
      "activations/layer10_attention_weight_min": -57.86960983276367,
      "activations/layer11_attention_weight_max": 38.81548309326172,
      "activations/layer11_attention_weight_min": -38.37092971801758,
      "activations/layer12_attention_weight_max": 27.267257690429688,
      "activations/layer12_attention_weight_min": -26.50095558166504,
      "activations/layer13_attention_weight_max": 59.59586715698242,
      "activations/layer13_attention_weight_min": -47.12804412841797,
      "activations/layer14_attention_weight_max": 57.77749252319336,
      "activations/layer14_attention_weight_min": -40.973175048828125,
      "activations/layer15_attention_weight_max": 50.3073844909668,
      "activations/layer15_attention_weight_min": -42.17211151123047,
      "activations/layer16_attention_weight_max": 45.622894287109375,
      "activations/layer16_attention_weight_min": -30.956615447998047,
      "activations/layer17_attention_weight_max": 58.63542938232422,
      "activations/layer17_attention_weight_min": -36.09425735473633,
      "activations/layer18_attention_weight_max": 60.982994079589844,
      "activations/layer18_attention_weight_min": -41.67447280883789,
      "activations/layer19_attention_weight_max": 20.306640625,
      "activations/layer19_attention_weight_min": -13.150484085083008,
      "activations/layer1_attention_weight_max": 14.773025512695312,
      "activations/layer1_attention_weight_min": -9.948701858520508,
      "activations/layer20_attention_weight_max": 18.634227752685547,
      "activations/layer20_attention_weight_min": -13.795519828796387,
      "activations/layer21_attention_weight_max": 34.86787796020508,
      "activations/layer21_attention_weight_min": -21.9132022857666,
      "activations/layer22_attention_weight_max": 37.86697769165039,
      "activations/layer22_attention_weight_min": -18.50482749938965,
      "activations/layer23_attention_weight_max": 23.755565643310547,
      "activations/layer23_attention_weight_min": -13.954019546508789,
      "activations/layer2_attention_weight_max": 25.13910484313965,
      "activations/layer2_attention_weight_min": -24.49675750732422,
      "activations/layer3_attention_weight_max": 55.534759521484375,
      "activations/layer3_attention_weight_min": -55.69009017944336,
      "activations/layer4_attention_weight_max": 84.64185333251953,
      "activations/layer4_attention_weight_min": -74.61473083496094,
      "activations/layer5_attention_weight_max": 126.7033920288086,
      "activations/layer5_attention_weight_min": -124.41293334960938,
      "activations/layer6_attention_weight_max": 60.45012283325195,
      "activations/layer6_attention_weight_min": -58.426300048828125,
      "activations/layer7_attention_weight_max": 62.41887283325195,
      "activations/layer7_attention_weight_min": -53.82609558105469,
      "activations/layer8_attention_weight_max": 47.54015350341797,
      "activations/layer8_attention_weight_min": -43.05495071411133,
      "activations/layer9_attention_weight_max": 67.84697723388672,
      "activations/layer9_attention_weight_min": -61.959468841552734,
      "epoch": 1.27,
      "learning_rate": 0.0001432333333333333,
      "loss": 3.0484,
      "step": 21900
    },
    {
      "activations/layer0_attention_weight_max": 13.46069622039795,
      "activations/layer0_attention_weight_min": -12.64107608795166,
      "activations/layer10_attention_weight_max": 65.11784362792969,
      "activations/layer10_attention_weight_min": -55.82439422607422,
      "activations/layer11_attention_weight_max": 46.23531723022461,
      "activations/layer11_attention_weight_min": -45.40909957885742,
      "activations/layer12_attention_weight_max": 25.320091247558594,
      "activations/layer12_attention_weight_min": -24.060672760009766,
      "activations/layer13_attention_weight_max": 63.915592193603516,
      "activations/layer13_attention_weight_min": -45.63222122192383,
      "activations/layer14_attention_weight_max": 50.41067123413086,
      "activations/layer14_attention_weight_min": -37.04646301269531,
      "activations/layer15_attention_weight_max": 54.03266143798828,
      "activations/layer15_attention_weight_min": -42.326175689697266,
      "activations/layer16_attention_weight_max": 44.43083953857422,
      "activations/layer16_attention_weight_min": -31.942432403564453,
      "activations/layer17_attention_weight_max": 54.598087310791016,
      "activations/layer17_attention_weight_min": -38.63655090332031,
      "activations/layer18_attention_weight_max": 54.83645248413086,
      "activations/layer18_attention_weight_min": -35.31153106689453,
      "activations/layer19_attention_weight_max": 21.978897094726562,
      "activations/layer19_attention_weight_min": -13.624534606933594,
      "activations/layer1_attention_weight_max": 15.634724617004395,
      "activations/layer1_attention_weight_min": -11.568198204040527,
      "activations/layer20_attention_weight_max": 18.761051177978516,
      "activations/layer20_attention_weight_min": -14.617950439453125,
      "activations/layer21_attention_weight_max": 38.43437957763672,
      "activations/layer21_attention_weight_min": -21.01476287841797,
      "activations/layer22_attention_weight_max": 36.96336364746094,
      "activations/layer22_attention_weight_min": -18.751102447509766,
      "activations/layer23_attention_weight_max": 22.56686019897461,
      "activations/layer23_attention_weight_min": -15.370097160339355,
      "activations/layer2_attention_weight_max": 23.892410278320312,
      "activations/layer2_attention_weight_min": -24.306509017944336,
      "activations/layer3_attention_weight_max": 55.04733657836914,
      "activations/layer3_attention_weight_min": -57.73817443847656,
      "activations/layer4_attention_weight_max": 86.88280487060547,
      "activations/layer4_attention_weight_min": -78.1306381225586,
      "activations/layer5_attention_weight_max": 133.71490478515625,
      "activations/layer5_attention_weight_min": -120.61278533935547,
      "activations/layer6_attention_weight_max": 63.133460998535156,
      "activations/layer6_attention_weight_min": -56.11659622192383,
      "activations/layer7_attention_weight_max": 63.946266174316406,
      "activations/layer7_attention_weight_min": -55.46304702758789,
      "activations/layer8_attention_weight_max": 57.00225830078125,
      "activations/layer8_attention_weight_min": -45.963172912597656,
      "activations/layer9_attention_weight_max": 71.31085205078125,
      "activations/layer9_attention_weight_min": -64.18707275390625,
      "epoch": 1.28,
      "learning_rate": 0.00014321439393939392,
      "loss": 3.0383,
      "step": 21950
    },
    {
      "activations/layer0_attention_weight_max": 13.22418212890625,
      "activations/layer0_attention_weight_min": -12.460587501525879,
      "activations/layer10_attention_weight_max": 69.62007141113281,
      "activations/layer10_attention_weight_min": -58.08735275268555,
      "activations/layer11_attention_weight_max": 46.72074890136719,
      "activations/layer11_attention_weight_min": -38.04563522338867,
      "activations/layer12_attention_weight_max": 30.261613845825195,
      "activations/layer12_attention_weight_min": -23.130836486816406,
      "activations/layer13_attention_weight_max": 67.72183227539062,
      "activations/layer13_attention_weight_min": -52.289329528808594,
      "activations/layer14_attention_weight_max": 59.56328201293945,
      "activations/layer14_attention_weight_min": -42.64582443237305,
      "activations/layer15_attention_weight_max": 58.379573822021484,
      "activations/layer15_attention_weight_min": -41.46784973144531,
      "activations/layer16_attention_weight_max": 47.30796432495117,
      "activations/layer16_attention_weight_min": -30.834156036376953,
      "activations/layer17_attention_weight_max": 54.739349365234375,
      "activations/layer17_attention_weight_min": -34.45101547241211,
      "activations/layer18_attention_weight_max": 67.45581817626953,
      "activations/layer18_attention_weight_min": -38.803462982177734,
      "activations/layer19_attention_weight_max": 19.922121047973633,
      "activations/layer19_attention_weight_min": -12.993481636047363,
      "activations/layer1_attention_weight_max": 14.698800086975098,
      "activations/layer1_attention_weight_min": -10.328680992126465,
      "activations/layer20_attention_weight_max": 18.70050048828125,
      "activations/layer20_attention_weight_min": -16.100868225097656,
      "activations/layer21_attention_weight_max": 41.82011032104492,
      "activations/layer21_attention_weight_min": -19.13553810119629,
      "activations/layer22_attention_weight_max": 37.36573028564453,
      "activations/layer22_attention_weight_min": -18.982528686523438,
      "activations/layer23_attention_weight_max": 24.753551483154297,
      "activations/layer23_attention_weight_min": -15.260664939880371,
      "activations/layer2_attention_weight_max": 23.080862045288086,
      "activations/layer2_attention_weight_min": -22.406822204589844,
      "activations/layer3_attention_weight_max": 53.219852447509766,
      "activations/layer3_attention_weight_min": -56.272823333740234,
      "activations/layer4_attention_weight_max": 84.80062103271484,
      "activations/layer4_attention_weight_min": -75.08391571044922,
      "activations/layer5_attention_weight_max": 135.65267944335938,
      "activations/layer5_attention_weight_min": -124.67378997802734,
      "activations/layer6_attention_weight_max": 63.8635368347168,
      "activations/layer6_attention_weight_min": -56.23636245727539,
      "activations/layer7_attention_weight_max": 60.99781799316406,
      "activations/layer7_attention_weight_min": -54.67510223388672,
      "activations/layer8_attention_weight_max": 51.55570983886719,
      "activations/layer8_attention_weight_min": -45.857276916503906,
      "activations/layer9_attention_weight_max": 73.4120101928711,
      "activations/layer9_attention_weight_min": -60.516693115234375,
      "epoch": 1.28,
      "learning_rate": 0.00014319545454545454,
      "loss": 3.0316,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_loss": 2.994140625,
      "eval_runtime": 8.6909,
      "eval_samples_per_second": 494.077,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_openwebtext_loss": 2.994140625,
      "eval_openwebtext_ppl": 19.96819234893607,
      "eval_openwebtext_runtime": 8.6909,
      "eval_openwebtext_samples_per_second": 494.077,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_wikitext_loss": 3.279296875,
      "eval_wikitext_ppl": 26.557093177483523,
      "eval_wikitext_runtime": 1.697,
      "eval_wikitext_samples_per_second": 268.703,
      "step": 22000
    },
    {
      "epoch": 1.28,
      "eval_lambada_loss": 3.29296875,
      "eval_lambada_ppl": 26.922671813996086,
      "eval_lambada_runtime": 9.3059,
      "eval_lambada_samples_per_second": 523.215,
      "step": 22000
    },
    {
      "activations/layer0_attention_weight_max": 13.9054594039917,
      "activations/layer0_attention_weight_min": -12.475214958190918,
      "activations/layer10_attention_weight_max": 75.27554321289062,
      "activations/layer10_attention_weight_min": -57.676300048828125,
      "activations/layer11_attention_weight_max": 54.01685333251953,
      "activations/layer11_attention_weight_min": -41.23057174682617,
      "activations/layer12_attention_weight_max": 26.934968948364258,
      "activations/layer12_attention_weight_min": -24.198841094970703,
      "activations/layer13_attention_weight_max": 57.80154800415039,
      "activations/layer13_attention_weight_min": -40.574127197265625,
      "activations/layer14_attention_weight_max": 55.58306121826172,
      "activations/layer14_attention_weight_min": -38.806732177734375,
      "activations/layer15_attention_weight_max": 49.15164566040039,
      "activations/layer15_attention_weight_min": -36.044376373291016,
      "activations/layer16_attention_weight_max": 45.69662094116211,
      "activations/layer16_attention_weight_min": -30.862659454345703,
      "activations/layer17_attention_weight_max": 57.44063186645508,
      "activations/layer17_attention_weight_min": -36.3509635925293,
      "activations/layer18_attention_weight_max": 54.95734786987305,
      "activations/layer18_attention_weight_min": -37.10981750488281,
      "activations/layer19_attention_weight_max": 19.807910919189453,
      "activations/layer19_attention_weight_min": -15.66852855682373,
      "activations/layer1_attention_weight_max": 15.764959335327148,
      "activations/layer1_attention_weight_min": -11.473603248596191,
      "activations/layer20_attention_weight_max": 18.70469856262207,
      "activations/layer20_attention_weight_min": -14.650151252746582,
      "activations/layer21_attention_weight_max": 42.41188430786133,
      "activations/layer21_attention_weight_min": -18.410961151123047,
      "activations/layer22_attention_weight_max": 33.984371185302734,
      "activations/layer22_attention_weight_min": -19.89023780822754,
      "activations/layer23_attention_weight_max": 20.60449981689453,
      "activations/layer23_attention_weight_min": -13.02418041229248,
      "activations/layer2_attention_weight_max": 25.46894073486328,
      "activations/layer2_attention_weight_min": -23.93090057373047,
      "activations/layer3_attention_weight_max": 59.761383056640625,
      "activations/layer3_attention_weight_min": -60.213565826416016,
      "activations/layer4_attention_weight_max": 85.61408996582031,
      "activations/layer4_attention_weight_min": -78.6072006225586,
      "activations/layer5_attention_weight_max": 135.0387725830078,
      "activations/layer5_attention_weight_min": -128.3665008544922,
      "activations/layer6_attention_weight_max": 64.0827865600586,
      "activations/layer6_attention_weight_min": -57.90134811401367,
      "activations/layer7_attention_weight_max": 61.4310417175293,
      "activations/layer7_attention_weight_min": -55.614784240722656,
      "activations/layer8_attention_weight_max": 61.43830871582031,
      "activations/layer8_attention_weight_min": -46.78409957885742,
      "activations/layer9_attention_weight_max": 89.71550750732422,
      "activations/layer9_attention_weight_min": -73.89543914794922,
      "epoch": 1.28,
      "learning_rate": 0.00014317651515151515,
      "loss": 3.0709,
      "step": 22050
    },
    {
      "activations/layer0_attention_weight_max": 12.523956298828125,
      "activations/layer0_attention_weight_min": -12.774396896362305,
      "activations/layer10_attention_weight_max": 71.29962158203125,
      "activations/layer10_attention_weight_min": -56.711181640625,
      "activations/layer11_attention_weight_max": 56.77128601074219,
      "activations/layer11_attention_weight_min": -42.058998107910156,
      "activations/layer12_attention_weight_max": 28.01548957824707,
      "activations/layer12_attention_weight_min": -27.555286407470703,
      "activations/layer13_attention_weight_max": 71.6866226196289,
      "activations/layer13_attention_weight_min": -42.344051361083984,
      "activations/layer14_attention_weight_max": 67.22247314453125,
      "activations/layer14_attention_weight_min": -39.00997543334961,
      "activations/layer15_attention_weight_max": 57.929237365722656,
      "activations/layer15_attention_weight_min": -39.944156646728516,
      "activations/layer16_attention_weight_max": 47.70415496826172,
      "activations/layer16_attention_weight_min": -33.8282356262207,
      "activations/layer17_attention_weight_max": 61.02033233642578,
      "activations/layer17_attention_weight_min": -34.83833312988281,
      "activations/layer18_attention_weight_max": 72.2071762084961,
      "activations/layer18_attention_weight_min": -37.34398651123047,
      "activations/layer19_attention_weight_max": 22.360258102416992,
      "activations/layer19_attention_weight_min": -13.823320388793945,
      "activations/layer1_attention_weight_max": 15.070247650146484,
      "activations/layer1_attention_weight_min": -11.621973037719727,
      "activations/layer20_attention_weight_max": 20.378137588500977,
      "activations/layer20_attention_weight_min": -14.179864883422852,
      "activations/layer21_attention_weight_max": 35.06550216674805,
      "activations/layer21_attention_weight_min": -20.26189613342285,
      "activations/layer22_attention_weight_max": 35.84144973754883,
      "activations/layer22_attention_weight_min": -19.267353057861328,
      "activations/layer23_attention_weight_max": 26.039440155029297,
      "activations/layer23_attention_weight_min": -13.131327629089355,
      "activations/layer2_attention_weight_max": 27.54266929626465,
      "activations/layer2_attention_weight_min": -26.711647033691406,
      "activations/layer3_attention_weight_max": 58.489620208740234,
      "activations/layer3_attention_weight_min": -62.17939376831055,
      "activations/layer4_attention_weight_max": 94.90604400634766,
      "activations/layer4_attention_weight_min": -83.3147964477539,
      "activations/layer5_attention_weight_max": 161.44808959960938,
      "activations/layer5_attention_weight_min": -124.93814086914062,
      "activations/layer6_attention_weight_max": 72.71277618408203,
      "activations/layer6_attention_weight_min": -54.84807205200195,
      "activations/layer7_attention_weight_max": 70.06338500976562,
      "activations/layer7_attention_weight_min": -62.37602233886719,
      "activations/layer8_attention_weight_max": 68.22445678710938,
      "activations/layer8_attention_weight_min": -55.18837356567383,
      "activations/layer9_attention_weight_max": 92.31365203857422,
      "activations/layer9_attention_weight_min": -77.92234802246094,
      "epoch": 1.28,
      "learning_rate": 0.00014315757575757574,
      "loss": 3.0456,
      "step": 22100
    },
    {
      "activations/layer0_attention_weight_max": 14.148954391479492,
      "activations/layer0_attention_weight_min": -12.128684997558594,
      "activations/layer10_attention_weight_max": 71.77567291259766,
      "activations/layer10_attention_weight_min": -54.26433563232422,
      "activations/layer11_attention_weight_max": 43.96504211425781,
      "activations/layer11_attention_weight_min": -39.58576202392578,
      "activations/layer12_attention_weight_max": 39.85719299316406,
      "activations/layer12_attention_weight_min": -25.00922966003418,
      "activations/layer13_attention_weight_max": 59.43181610107422,
      "activations/layer13_attention_weight_min": -46.81038284301758,
      "activations/layer14_attention_weight_max": 46.662052154541016,
      "activations/layer14_attention_weight_min": -36.98490524291992,
      "activations/layer15_attention_weight_max": 63.67844009399414,
      "activations/layer15_attention_weight_min": -49.1890869140625,
      "activations/layer16_attention_weight_max": 45.98273468017578,
      "activations/layer16_attention_weight_min": -35.123329162597656,
      "activations/layer17_attention_weight_max": 57.92110824584961,
      "activations/layer17_attention_weight_min": -36.88084411621094,
      "activations/layer18_attention_weight_max": 66.98419189453125,
      "activations/layer18_attention_weight_min": -38.729576110839844,
      "activations/layer19_attention_weight_max": 20.92687225341797,
      "activations/layer19_attention_weight_min": -13.478711128234863,
      "activations/layer1_attention_weight_max": 15.046480178833008,
      "activations/layer1_attention_weight_min": -11.5118408203125,
      "activations/layer20_attention_weight_max": 20.573198318481445,
      "activations/layer20_attention_weight_min": -16.397462844848633,
      "activations/layer21_attention_weight_max": 36.278297424316406,
      "activations/layer21_attention_weight_min": -19.31624412536621,
      "activations/layer22_attention_weight_max": 33.004844665527344,
      "activations/layer22_attention_weight_min": -19.393884658813477,
      "activations/layer23_attention_weight_max": 21.100379943847656,
      "activations/layer23_attention_weight_min": -15.432422637939453,
      "activations/layer2_attention_weight_max": 23.819744110107422,
      "activations/layer2_attention_weight_min": -24.013486862182617,
      "activations/layer3_attention_weight_max": 53.83340072631836,
      "activations/layer3_attention_weight_min": -54.93310546875,
      "activations/layer4_attention_weight_max": 83.17256927490234,
      "activations/layer4_attention_weight_min": -75.22124481201172,
      "activations/layer5_attention_weight_max": 131.92584228515625,
      "activations/layer5_attention_weight_min": -122.82463073730469,
      "activations/layer6_attention_weight_max": 60.84086608886719,
      "activations/layer6_attention_weight_min": -54.34766387939453,
      "activations/layer7_attention_weight_max": 62.048973083496094,
      "activations/layer7_attention_weight_min": -56.241111755371094,
      "activations/layer8_attention_weight_max": 51.35706329345703,
      "activations/layer8_attention_weight_min": -48.61088943481445,
      "activations/layer9_attention_weight_max": 70.37161254882812,
      "activations/layer9_attention_weight_min": -64.44143676757812,
      "epoch": 1.29,
      "learning_rate": 0.00014313863636363636,
      "loss": 3.0392,
      "step": 22150
    },
    {
      "activations/layer0_attention_weight_max": 15.260146141052246,
      "activations/layer0_attention_weight_min": -12.893556594848633,
      "activations/layer10_attention_weight_max": 75.95024871826172,
      "activations/layer10_attention_weight_min": -59.614898681640625,
      "activations/layer11_attention_weight_max": 54.99042510986328,
      "activations/layer11_attention_weight_min": -43.317665100097656,
      "activations/layer12_attention_weight_max": 29.674877166748047,
      "activations/layer12_attention_weight_min": -23.45047378540039,
      "activations/layer13_attention_weight_max": 66.5050277709961,
      "activations/layer13_attention_weight_min": -46.43881607055664,
      "activations/layer14_attention_weight_max": 64.6770248413086,
      "activations/layer14_attention_weight_min": -41.00800704956055,
      "activations/layer15_attention_weight_max": 56.070167541503906,
      "activations/layer15_attention_weight_min": -41.936763763427734,
      "activations/layer16_attention_weight_max": 45.1579704284668,
      "activations/layer16_attention_weight_min": -34.045310974121094,
      "activations/layer17_attention_weight_max": 58.19607162475586,
      "activations/layer17_attention_weight_min": -41.708805084228516,
      "activations/layer18_attention_weight_max": 70.2426986694336,
      "activations/layer18_attention_weight_min": -37.98488998413086,
      "activations/layer19_attention_weight_max": 22.690614700317383,
      "activations/layer19_attention_weight_min": -13.606338500976562,
      "activations/layer1_attention_weight_max": 14.516510963439941,
      "activations/layer1_attention_weight_min": -11.511823654174805,
      "activations/layer20_attention_weight_max": 18.64600944519043,
      "activations/layer20_attention_weight_min": -12.950910568237305,
      "activations/layer21_attention_weight_max": 40.14673614501953,
      "activations/layer21_attention_weight_min": -20.186988830566406,
      "activations/layer22_attention_weight_max": 34.121883392333984,
      "activations/layer22_attention_weight_min": -18.135820388793945,
      "activations/layer23_attention_weight_max": 21.280593872070312,
      "activations/layer23_attention_weight_min": -13.861686706542969,
      "activations/layer2_attention_weight_max": 25.527740478515625,
      "activations/layer2_attention_weight_min": -25.12084197998047,
      "activations/layer3_attention_weight_max": 58.17097473144531,
      "activations/layer3_attention_weight_min": -57.92643356323242,
      "activations/layer4_attention_weight_max": 86.90050506591797,
      "activations/layer4_attention_weight_min": -77.08831024169922,
      "activations/layer5_attention_weight_max": 137.36984252929688,
      "activations/layer5_attention_weight_min": -141.373291015625,
      "activations/layer6_attention_weight_max": 64.24694061279297,
      "activations/layer6_attention_weight_min": -63.20679473876953,
      "activations/layer7_attention_weight_max": 63.797218322753906,
      "activations/layer7_attention_weight_min": -57.602909088134766,
      "activations/layer8_attention_weight_max": 66.05542755126953,
      "activations/layer8_attention_weight_min": -56.71144485473633,
      "activations/layer9_attention_weight_max": 83.53079223632812,
      "activations/layer9_attention_weight_min": -72.9301528930664,
      "epoch": 1.29,
      "learning_rate": 0.00014311969696969697,
      "loss": 3.039,
      "step": 22200
    },
    {
      "activations/layer0_attention_weight_max": 13.234833717346191,
      "activations/layer0_attention_weight_min": -12.107135772705078,
      "activations/layer10_attention_weight_max": 68.8975830078125,
      "activations/layer10_attention_weight_min": -58.57343673706055,
      "activations/layer11_attention_weight_max": 45.743247985839844,
      "activations/layer11_attention_weight_min": -43.68102264404297,
      "activations/layer12_attention_weight_max": 28.127357482910156,
      "activations/layer12_attention_weight_min": -22.95580291748047,
      "activations/layer13_attention_weight_max": 68.4455795288086,
      "activations/layer13_attention_weight_min": -47.288841247558594,
      "activations/layer14_attention_weight_max": 52.556434631347656,
      "activations/layer14_attention_weight_min": -39.683902740478516,
      "activations/layer15_attention_weight_max": 53.96006393432617,
      "activations/layer15_attention_weight_min": -38.809814453125,
      "activations/layer16_attention_weight_max": 51.03491973876953,
      "activations/layer16_attention_weight_min": -35.786598205566406,
      "activations/layer17_attention_weight_max": 59.13832473754883,
      "activations/layer17_attention_weight_min": -36.970550537109375,
      "activations/layer18_attention_weight_max": 74.29056549072266,
      "activations/layer18_attention_weight_min": -42.86142349243164,
      "activations/layer19_attention_weight_max": 20.960508346557617,
      "activations/layer19_attention_weight_min": -14.045684814453125,
      "activations/layer1_attention_weight_max": 14.396183967590332,
      "activations/layer1_attention_weight_min": -12.0160493850708,
      "activations/layer20_attention_weight_max": 19.70848846435547,
      "activations/layer20_attention_weight_min": -13.46210765838623,
      "activations/layer21_attention_weight_max": 40.87867736816406,
      "activations/layer21_attention_weight_min": -20.139198303222656,
      "activations/layer22_attention_weight_max": 37.87289047241211,
      "activations/layer22_attention_weight_min": -19.54136085510254,
      "activations/layer23_attention_weight_max": 24.022891998291016,
      "activations/layer23_attention_weight_min": -16.330869674682617,
      "activations/layer2_attention_weight_max": 25.12013053894043,
      "activations/layer2_attention_weight_min": -24.41847038269043,
      "activations/layer3_attention_weight_max": 56.415557861328125,
      "activations/layer3_attention_weight_min": -60.688201904296875,
      "activations/layer4_attention_weight_max": 85.89510345458984,
      "activations/layer4_attention_weight_min": -78.216796875,
      "activations/layer5_attention_weight_max": 131.5489044189453,
      "activations/layer5_attention_weight_min": -122.3889389038086,
      "activations/layer6_attention_weight_max": 65.36288452148438,
      "activations/layer6_attention_weight_min": -57.20562744140625,
      "activations/layer7_attention_weight_max": 64.22163391113281,
      "activations/layer7_attention_weight_min": -61.0839958190918,
      "activations/layer8_attention_weight_max": 54.3076171875,
      "activations/layer8_attention_weight_min": -49.481624603271484,
      "activations/layer9_attention_weight_max": 65.4742202758789,
      "activations/layer9_attention_weight_min": -66.25715637207031,
      "epoch": 1.29,
      "learning_rate": 0.00014310075757575756,
      "loss": 3.0312,
      "step": 22250
    },
    {
      "activations/layer0_attention_weight_max": 13.846466064453125,
      "activations/layer0_attention_weight_min": -12.101953506469727,
      "activations/layer10_attention_weight_max": 66.02738952636719,
      "activations/layer10_attention_weight_min": -55.11553192138672,
      "activations/layer11_attention_weight_max": 46.8740348815918,
      "activations/layer11_attention_weight_min": -40.8530158996582,
      "activations/layer12_attention_weight_max": 30.433574676513672,
      "activations/layer12_attention_weight_min": -26.162973403930664,
      "activations/layer13_attention_weight_max": 56.20640563964844,
      "activations/layer13_attention_weight_min": -41.54804229736328,
      "activations/layer14_attention_weight_max": 49.099693298339844,
      "activations/layer14_attention_weight_min": -41.436500549316406,
      "activations/layer15_attention_weight_max": 58.625118255615234,
      "activations/layer15_attention_weight_min": -38.79470443725586,
      "activations/layer16_attention_weight_max": 45.350521087646484,
      "activations/layer16_attention_weight_min": -32.71437454223633,
      "activations/layer17_attention_weight_max": 58.75000762939453,
      "activations/layer17_attention_weight_min": -37.58254623413086,
      "activations/layer18_attention_weight_max": 64.65558624267578,
      "activations/layer18_attention_weight_min": -40.58014678955078,
      "activations/layer19_attention_weight_max": 20.92949676513672,
      "activations/layer19_attention_weight_min": -14.201927185058594,
      "activations/layer1_attention_weight_max": 14.670751571655273,
      "activations/layer1_attention_weight_min": -11.536983489990234,
      "activations/layer20_attention_weight_max": 19.249435424804688,
      "activations/layer20_attention_weight_min": -15.324053764343262,
      "activations/layer21_attention_weight_max": 37.785438537597656,
      "activations/layer21_attention_weight_min": -20.121490478515625,
      "activations/layer22_attention_weight_max": 34.97941970825195,
      "activations/layer22_attention_weight_min": -20.460237503051758,
      "activations/layer23_attention_weight_max": 25.053329467773438,
      "activations/layer23_attention_weight_min": -14.825008392333984,
      "activations/layer2_attention_weight_max": 25.924720764160156,
      "activations/layer2_attention_weight_min": -24.2871150970459,
      "activations/layer3_attention_weight_max": 53.944026947021484,
      "activations/layer3_attention_weight_min": -57.61899185180664,
      "activations/layer4_attention_weight_max": 84.3852767944336,
      "activations/layer4_attention_weight_min": -78.94625854492188,
      "activations/layer5_attention_weight_max": 126.82278442382812,
      "activations/layer5_attention_weight_min": -120.97782897949219,
      "activations/layer6_attention_weight_max": 60.313636779785156,
      "activations/layer6_attention_weight_min": -53.85411834716797,
      "activations/layer7_attention_weight_max": 59.02862548828125,
      "activations/layer7_attention_weight_min": -54.298274993896484,
      "activations/layer8_attention_weight_max": 53.2526969909668,
      "activations/layer8_attention_weight_min": -45.696189880371094,
      "activations/layer9_attention_weight_max": 71.94766998291016,
      "activations/layer9_attention_weight_min": -66.39952850341797,
      "epoch": 1.3,
      "learning_rate": 0.00014308181818181818,
      "loss": 3.044,
      "step": 22300
    },
    {
      "activations/layer0_attention_weight_max": 14.378571510314941,
      "activations/layer0_attention_weight_min": -12.424468994140625,
      "activations/layer10_attention_weight_max": 69.07363891601562,
      "activations/layer10_attention_weight_min": -58.94853210449219,
      "activations/layer11_attention_weight_max": 45.72447204589844,
      "activations/layer11_attention_weight_min": -38.845924377441406,
      "activations/layer12_attention_weight_max": 29.343318939208984,
      "activations/layer12_attention_weight_min": -25.676664352416992,
      "activations/layer13_attention_weight_max": 63.90174865722656,
      "activations/layer13_attention_weight_min": -46.07088088989258,
      "activations/layer14_attention_weight_max": 52.95657730102539,
      "activations/layer14_attention_weight_min": -39.888301849365234,
      "activations/layer15_attention_weight_max": 55.25617218017578,
      "activations/layer15_attention_weight_min": -38.46080780029297,
      "activations/layer16_attention_weight_max": 47.90336608886719,
      "activations/layer16_attention_weight_min": -30.774580001831055,
      "activations/layer17_attention_weight_max": 56.54170227050781,
      "activations/layer17_attention_weight_min": -37.900634765625,
      "activations/layer18_attention_weight_max": 65.67943572998047,
      "activations/layer18_attention_weight_min": -38.01777267456055,
      "activations/layer19_attention_weight_max": 18.586244583129883,
      "activations/layer19_attention_weight_min": -14.629005432128906,
      "activations/layer1_attention_weight_max": 16.085453033447266,
      "activations/layer1_attention_weight_min": -10.555728912353516,
      "activations/layer20_attention_weight_max": 19.879520416259766,
      "activations/layer20_attention_weight_min": -15.63253116607666,
      "activations/layer21_attention_weight_max": 36.41133499145508,
      "activations/layer21_attention_weight_min": -17.359241485595703,
      "activations/layer22_attention_weight_max": 38.2282600402832,
      "activations/layer22_attention_weight_min": -18.26038360595703,
      "activations/layer23_attention_weight_max": 24.91629409790039,
      "activations/layer23_attention_weight_min": -14.77802848815918,
      "activations/layer2_attention_weight_max": 24.822330474853516,
      "activations/layer2_attention_weight_min": -24.67431640625,
      "activations/layer3_attention_weight_max": 55.03020477294922,
      "activations/layer3_attention_weight_min": -56.37174606323242,
      "activations/layer4_attention_weight_max": 87.1779556274414,
      "activations/layer4_attention_weight_min": -78.23326110839844,
      "activations/layer5_attention_weight_max": 134.88548278808594,
      "activations/layer5_attention_weight_min": -116.16471862792969,
      "activations/layer6_attention_weight_max": 64.35965728759766,
      "activations/layer6_attention_weight_min": -56.157012939453125,
      "activations/layer7_attention_weight_max": 67.20243072509766,
      "activations/layer7_attention_weight_min": -55.45576095581055,
      "activations/layer8_attention_weight_max": 57.52709197998047,
      "activations/layer8_attention_weight_min": -47.57180404663086,
      "activations/layer9_attention_weight_max": 71.44209289550781,
      "activations/layer9_attention_weight_min": -63.629638671875,
      "epoch": 1.3,
      "learning_rate": 0.0001430628787878788,
      "loss": 3.0346,
      "step": 22350
    },
    {
      "activations/layer0_attention_weight_max": 14.20312786102295,
      "activations/layer0_attention_weight_min": -13.06035327911377,
      "activations/layer10_attention_weight_max": 67.38987731933594,
      "activations/layer10_attention_weight_min": -61.050811767578125,
      "activations/layer11_attention_weight_max": 50.910545349121094,
      "activations/layer11_attention_weight_min": -44.59164047241211,
      "activations/layer12_attention_weight_max": 25.11640167236328,
      "activations/layer12_attention_weight_min": -23.019378662109375,
      "activations/layer13_attention_weight_max": 65.83131408691406,
      "activations/layer13_attention_weight_min": -47.017921447753906,
      "activations/layer14_attention_weight_max": 65.98945617675781,
      "activations/layer14_attention_weight_min": -43.989654541015625,
      "activations/layer15_attention_weight_max": 61.672420501708984,
      "activations/layer15_attention_weight_min": -42.71221160888672,
      "activations/layer16_attention_weight_max": 54.252986907958984,
      "activations/layer16_attention_weight_min": -32.66246795654297,
      "activations/layer17_attention_weight_max": 62.70947265625,
      "activations/layer17_attention_weight_min": -40.21588897705078,
      "activations/layer18_attention_weight_max": 61.812313079833984,
      "activations/layer18_attention_weight_min": -39.13212203979492,
      "activations/layer19_attention_weight_max": 22.361604690551758,
      "activations/layer19_attention_weight_min": -13.836995124816895,
      "activations/layer1_attention_weight_max": 15.208105087280273,
      "activations/layer1_attention_weight_min": -11.05702018737793,
      "activations/layer20_attention_weight_max": 20.83260726928711,
      "activations/layer20_attention_weight_min": -14.672908782958984,
      "activations/layer21_attention_weight_max": 38.724700927734375,
      "activations/layer21_attention_weight_min": -20.26460075378418,
      "activations/layer22_attention_weight_max": 37.95684051513672,
      "activations/layer22_attention_weight_min": -19.817882537841797,
      "activations/layer23_attention_weight_max": 22.449216842651367,
      "activations/layer23_attention_weight_min": -13.928863525390625,
      "activations/layer2_attention_weight_max": 25.517309188842773,
      "activations/layer2_attention_weight_min": -24.758893966674805,
      "activations/layer3_attention_weight_max": 56.04798126220703,
      "activations/layer3_attention_weight_min": -57.243003845214844,
      "activations/layer4_attention_weight_max": 85.76737213134766,
      "activations/layer4_attention_weight_min": -78.17906951904297,
      "activations/layer5_attention_weight_max": 140.42349243164062,
      "activations/layer5_attention_weight_min": -124.61742401123047,
      "activations/layer6_attention_weight_max": 64.4612045288086,
      "activations/layer6_attention_weight_min": -59.549137115478516,
      "activations/layer7_attention_weight_max": 70.49124908447266,
      "activations/layer7_attention_weight_min": -57.470245361328125,
      "activations/layer8_attention_weight_max": 57.96391677856445,
      "activations/layer8_attention_weight_min": -50.9486198425293,
      "activations/layer9_attention_weight_max": 73.74493408203125,
      "activations/layer9_attention_weight_min": -67.2883529663086,
      "epoch": 1.3,
      "learning_rate": 0.00014304393939393938,
      "loss": 3.0264,
      "step": 22400
    },
    {
      "activations/layer0_attention_weight_max": 15.060832977294922,
      "activations/layer0_attention_weight_min": -12.664514541625977,
      "activations/layer10_attention_weight_max": 68.7739486694336,
      "activations/layer10_attention_weight_min": -55.726924896240234,
      "activations/layer11_attention_weight_max": 42.63226318359375,
      "activations/layer11_attention_weight_min": -36.255348205566406,
      "activations/layer12_attention_weight_max": 27.594112396240234,
      "activations/layer12_attention_weight_min": -27.973726272583008,
      "activations/layer13_attention_weight_max": 58.233978271484375,
      "activations/layer13_attention_weight_min": -42.78654861450195,
      "activations/layer14_attention_weight_max": 52.10323715209961,
      "activations/layer14_attention_weight_min": -35.29270935058594,
      "activations/layer15_attention_weight_max": 50.323673248291016,
      "activations/layer15_attention_weight_min": -43.77177047729492,
      "activations/layer16_attention_weight_max": 39.97471237182617,
      "activations/layer16_attention_weight_min": -30.61536407470703,
      "activations/layer17_attention_weight_max": 61.62661361694336,
      "activations/layer17_attention_weight_min": -37.59918975830078,
      "activations/layer18_attention_weight_max": 55.139739990234375,
      "activations/layer18_attention_weight_min": -33.92658615112305,
      "activations/layer19_attention_weight_max": 18.01177978515625,
      "activations/layer19_attention_weight_min": -13.66510009765625,
      "activations/layer1_attention_weight_max": 15.241659164428711,
      "activations/layer1_attention_weight_min": -10.988450050354004,
      "activations/layer20_attention_weight_max": 16.407419204711914,
      "activations/layer20_attention_weight_min": -14.167829513549805,
      "activations/layer21_attention_weight_max": 38.27741622924805,
      "activations/layer21_attention_weight_min": -21.875154495239258,
      "activations/layer22_attention_weight_max": 30.79665184020996,
      "activations/layer22_attention_weight_min": -18.329252243041992,
      "activations/layer23_attention_weight_max": 20.894943237304688,
      "activations/layer23_attention_weight_min": -16.382619857788086,
      "activations/layer2_attention_weight_max": 26.6759090423584,
      "activations/layer2_attention_weight_min": -23.912662506103516,
      "activations/layer3_attention_weight_max": 55.288028717041016,
      "activations/layer3_attention_weight_min": -57.01258087158203,
      "activations/layer4_attention_weight_max": 85.5674057006836,
      "activations/layer4_attention_weight_min": -82.61449432373047,
      "activations/layer5_attention_weight_max": 134.15252685546875,
      "activations/layer5_attention_weight_min": -124.55732727050781,
      "activations/layer6_attention_weight_max": 61.60487365722656,
      "activations/layer6_attention_weight_min": -56.362754821777344,
      "activations/layer7_attention_weight_max": 63.264469146728516,
      "activations/layer7_attention_weight_min": -57.99445724487305,
      "activations/layer8_attention_weight_max": 49.750877380371094,
      "activations/layer8_attention_weight_min": -45.33186340332031,
      "activations/layer9_attention_weight_max": 71.55215454101562,
      "activations/layer9_attention_weight_min": -60.286720275878906,
      "epoch": 1.3,
      "learning_rate": 0.000143025,
      "loss": 3.0342,
      "step": 22450
    },
    {
      "activations/layer0_attention_weight_max": 14.90965461730957,
      "activations/layer0_attention_weight_min": -12.784481048583984,
      "activations/layer10_attention_weight_max": 60.32683563232422,
      "activations/layer10_attention_weight_min": -49.9723014831543,
      "activations/layer11_attention_weight_max": 51.009979248046875,
      "activations/layer11_attention_weight_min": -37.054405212402344,
      "activations/layer12_attention_weight_max": 30.989233016967773,
      "activations/layer12_attention_weight_min": -35.67500686645508,
      "activations/layer13_attention_weight_max": 88.72793579101562,
      "activations/layer13_attention_weight_min": -59.28246307373047,
      "activations/layer14_attention_weight_max": 86.37115478515625,
      "activations/layer14_attention_weight_min": -57.92409896850586,
      "activations/layer15_attention_weight_max": 53.53565979003906,
      "activations/layer15_attention_weight_min": -42.6907958984375,
      "activations/layer16_attention_weight_max": 45.653709411621094,
      "activations/layer16_attention_weight_min": -30.197309494018555,
      "activations/layer17_attention_weight_max": 54.42070007324219,
      "activations/layer17_attention_weight_min": -37.169795989990234,
      "activations/layer18_attention_weight_max": 69.0907211303711,
      "activations/layer18_attention_weight_min": -37.718292236328125,
      "activations/layer19_attention_weight_max": 22.89973258972168,
      "activations/layer19_attention_weight_min": -14.986241340637207,
      "activations/layer1_attention_weight_max": 16.549509048461914,
      "activations/layer1_attention_weight_min": -14.028615951538086,
      "activations/layer20_attention_weight_max": 22.662738800048828,
      "activations/layer20_attention_weight_min": -14.56508731842041,
      "activations/layer21_attention_weight_max": 36.36726379394531,
      "activations/layer21_attention_weight_min": -20.767351150512695,
      "activations/layer22_attention_weight_max": 40.79715347290039,
      "activations/layer22_attention_weight_min": -18.614397048950195,
      "activations/layer23_attention_weight_max": 23.591304779052734,
      "activations/layer23_attention_weight_min": -14.035625457763672,
      "activations/layer2_attention_weight_max": 25.727462768554688,
      "activations/layer2_attention_weight_min": -25.356557846069336,
      "activations/layer3_attention_weight_max": 60.142459869384766,
      "activations/layer3_attention_weight_min": -60.76412582397461,
      "activations/layer4_attention_weight_max": 91.18352508544922,
      "activations/layer4_attention_weight_min": -81.00635528564453,
      "activations/layer5_attention_weight_max": 134.94821166992188,
      "activations/layer5_attention_weight_min": -118.0877685546875,
      "activations/layer6_attention_weight_max": 62.34394836425781,
      "activations/layer6_attention_weight_min": -53.05949783325195,
      "activations/layer7_attention_weight_max": 62.53724670410156,
      "activations/layer7_attention_weight_min": -54.236610412597656,
      "activations/layer8_attention_weight_max": 55.31928253173828,
      "activations/layer8_attention_weight_min": -47.27841567993164,
      "activations/layer9_attention_weight_max": 77.50021362304688,
      "activations/layer9_attention_weight_min": -64.42399597167969,
      "epoch": 1.31,
      "learning_rate": 0.0001430060606060606,
      "loss": 3.0258,
      "step": 22500
    },
    {
      "activations/layer0_attention_weight_max": 13.448299407958984,
      "activations/layer0_attention_weight_min": -12.671154975891113,
      "activations/layer10_attention_weight_max": 59.02347183227539,
      "activations/layer10_attention_weight_min": -56.08992385864258,
      "activations/layer11_attention_weight_max": 46.143035888671875,
      "activations/layer11_attention_weight_min": -40.938846588134766,
      "activations/layer12_attention_weight_max": 27.793319702148438,
      "activations/layer12_attention_weight_min": -24.853431701660156,
      "activations/layer13_attention_weight_max": 63.21282196044922,
      "activations/layer13_attention_weight_min": -51.4744758605957,
      "activations/layer14_attention_weight_max": 53.113853454589844,
      "activations/layer14_attention_weight_min": -41.09324645996094,
      "activations/layer15_attention_weight_max": 52.34103012084961,
      "activations/layer15_attention_weight_min": -40.83277893066406,
      "activations/layer16_attention_weight_max": 42.89386749267578,
      "activations/layer16_attention_weight_min": -32.696388244628906,
      "activations/layer17_attention_weight_max": 51.4912109375,
      "activations/layer17_attention_weight_min": -37.76896286010742,
      "activations/layer18_attention_weight_max": 64.5956039428711,
      "activations/layer18_attention_weight_min": -40.622955322265625,
      "activations/layer19_attention_weight_max": 22.291067123413086,
      "activations/layer19_attention_weight_min": -16.60917091369629,
      "activations/layer1_attention_weight_max": 15.63448715209961,
      "activations/layer1_attention_weight_min": -10.88585090637207,
      "activations/layer20_attention_weight_max": 21.09073829650879,
      "activations/layer20_attention_weight_min": -15.574784278869629,
      "activations/layer21_attention_weight_max": 39.27973937988281,
      "activations/layer21_attention_weight_min": -21.45564842224121,
      "activations/layer22_attention_weight_max": 34.46728515625,
      "activations/layer22_attention_weight_min": -19.822208404541016,
      "activations/layer23_attention_weight_max": 25.551036834716797,
      "activations/layer23_attention_weight_min": -15.9579496383667,
      "activations/layer2_attention_weight_max": 23.392324447631836,
      "activations/layer2_attention_weight_min": -24.96242332458496,
      "activations/layer3_attention_weight_max": 55.11698913574219,
      "activations/layer3_attention_weight_min": -57.7065544128418,
      "activations/layer4_attention_weight_max": 86.00926971435547,
      "activations/layer4_attention_weight_min": -76.51445770263672,
      "activations/layer5_attention_weight_max": 130.3740997314453,
      "activations/layer5_attention_weight_min": -121.24559783935547,
      "activations/layer6_attention_weight_max": 59.643333435058594,
      "activations/layer6_attention_weight_min": -53.823490142822266,
      "activations/layer7_attention_weight_max": 59.70973205566406,
      "activations/layer7_attention_weight_min": -59.20796203613281,
      "activations/layer8_attention_weight_max": 45.889137268066406,
      "activations/layer8_attention_weight_min": -44.52858352661133,
      "activations/layer9_attention_weight_max": 66.67964172363281,
      "activations/layer9_attention_weight_min": -64.18006134033203,
      "epoch": 1.31,
      "learning_rate": 0.0001429871212121212,
      "loss": 3.0545,
      "step": 22550
    },
    {
      "activations/layer0_attention_weight_max": 13.930774688720703,
      "activations/layer0_attention_weight_min": -13.0556640625,
      "activations/layer10_attention_weight_max": 87.44315338134766,
      "activations/layer10_attention_weight_min": -60.060020446777344,
      "activations/layer11_attention_weight_max": 49.72172546386719,
      "activations/layer11_attention_weight_min": -45.04121017456055,
      "activations/layer12_attention_weight_max": 31.36831283569336,
      "activations/layer12_attention_weight_min": -24.417564392089844,
      "activations/layer13_attention_weight_max": 68.39690399169922,
      "activations/layer13_attention_weight_min": -45.26710891723633,
      "activations/layer14_attention_weight_max": 62.78816604614258,
      "activations/layer14_attention_weight_min": -48.23664855957031,
      "activations/layer15_attention_weight_max": 56.44594955444336,
      "activations/layer15_attention_weight_min": -38.42855453491211,
      "activations/layer16_attention_weight_max": 45.7650260925293,
      "activations/layer16_attention_weight_min": -33.51282501220703,
      "activations/layer17_attention_weight_max": 58.29710006713867,
      "activations/layer17_attention_weight_min": -39.994911193847656,
      "activations/layer18_attention_weight_max": 64.20362091064453,
      "activations/layer18_attention_weight_min": -35.969512939453125,
      "activations/layer19_attention_weight_max": 18.978376388549805,
      "activations/layer19_attention_weight_min": -14.87033748626709,
      "activations/layer1_attention_weight_max": 15.219026565551758,
      "activations/layer1_attention_weight_min": -11.949352264404297,
      "activations/layer20_attention_weight_max": 19.377931594848633,
      "activations/layer20_attention_weight_min": -14.3660888671875,
      "activations/layer21_attention_weight_max": 36.699432373046875,
      "activations/layer21_attention_weight_min": -19.501562118530273,
      "activations/layer22_attention_weight_max": 32.32432174682617,
      "activations/layer22_attention_weight_min": -17.074934005737305,
      "activations/layer23_attention_weight_max": 20.04307746887207,
      "activations/layer23_attention_weight_min": -14.22366714477539,
      "activations/layer2_attention_weight_max": 28.135480880737305,
      "activations/layer2_attention_weight_min": -25.82242202758789,
      "activations/layer3_attention_weight_max": 59.60380172729492,
      "activations/layer3_attention_weight_min": -61.279842376708984,
      "activations/layer4_attention_weight_max": 92.0390853881836,
      "activations/layer4_attention_weight_min": -81.12987518310547,
      "activations/layer5_attention_weight_max": 151.62237548828125,
      "activations/layer5_attention_weight_min": -127.67536926269531,
      "activations/layer6_attention_weight_max": 71.6155776977539,
      "activations/layer6_attention_weight_min": -57.54719543457031,
      "activations/layer7_attention_weight_max": 74.54967498779297,
      "activations/layer7_attention_weight_min": -59.48652648925781,
      "activations/layer8_attention_weight_max": 67.92367553710938,
      "activations/layer8_attention_weight_min": -62.33677291870117,
      "activations/layer9_attention_weight_max": 87.27706909179688,
      "activations/layer9_attention_weight_min": -78.2110824584961,
      "epoch": 1.31,
      "learning_rate": 0.0001429681818181818,
      "loss": 3.0236,
      "step": 22600
    },
    {
      "activations/layer0_attention_weight_max": 13.209710121154785,
      "activations/layer0_attention_weight_min": -12.473604202270508,
      "activations/layer10_attention_weight_max": 62.59098815917969,
      "activations/layer10_attention_weight_min": -55.766624450683594,
      "activations/layer11_attention_weight_max": 42.4840202331543,
      "activations/layer11_attention_weight_min": -38.2041015625,
      "activations/layer12_attention_weight_max": 25.785140991210938,
      "activations/layer12_attention_weight_min": -22.65046501159668,
      "activations/layer13_attention_weight_max": 52.63511657714844,
      "activations/layer13_attention_weight_min": -43.59257125854492,
      "activations/layer14_attention_weight_max": 47.06358337402344,
      "activations/layer14_attention_weight_min": -38.5383186340332,
      "activations/layer15_attention_weight_max": 48.42628479003906,
      "activations/layer15_attention_weight_min": -38.78311538696289,
      "activations/layer16_attention_weight_max": 50.096309661865234,
      "activations/layer16_attention_weight_min": -34.4664192199707,
      "activations/layer17_attention_weight_max": 52.2191047668457,
      "activations/layer17_attention_weight_min": -37.240257263183594,
      "activations/layer18_attention_weight_max": 61.12287902832031,
      "activations/layer18_attention_weight_min": -35.36647415161133,
      "activations/layer19_attention_weight_max": 22.053173065185547,
      "activations/layer19_attention_weight_min": -18.208627700805664,
      "activations/layer1_attention_weight_max": 15.466803550720215,
      "activations/layer1_attention_weight_min": -12.055497169494629,
      "activations/layer20_attention_weight_max": 19.09109878540039,
      "activations/layer20_attention_weight_min": -14.682915687561035,
      "activations/layer21_attention_weight_max": 40.46376037597656,
      "activations/layer21_attention_weight_min": -20.68043327331543,
      "activations/layer22_attention_weight_max": 35.19696044921875,
      "activations/layer22_attention_weight_min": -19.601673126220703,
      "activations/layer23_attention_weight_max": 23.00371551513672,
      "activations/layer23_attention_weight_min": -18.073165893554688,
      "activations/layer2_attention_weight_max": 24.584354400634766,
      "activations/layer2_attention_weight_min": -23.711185455322266,
      "activations/layer3_attention_weight_max": 50.98213577270508,
      "activations/layer3_attention_weight_min": -54.19941711425781,
      "activations/layer4_attention_weight_max": 79.6568832397461,
      "activations/layer4_attention_weight_min": -73.98751068115234,
      "activations/layer5_attention_weight_max": 122.03436279296875,
      "activations/layer5_attention_weight_min": -119.14035034179688,
      "activations/layer6_attention_weight_max": 62.28394317626953,
      "activations/layer6_attention_weight_min": -54.24502182006836,
      "activations/layer7_attention_weight_max": 61.329078674316406,
      "activations/layer7_attention_weight_min": -55.811214447021484,
      "activations/layer8_attention_weight_max": 46.37724685668945,
      "activations/layer8_attention_weight_min": -45.67487716674805,
      "activations/layer9_attention_weight_max": 67.78050994873047,
      "activations/layer9_attention_weight_min": -63.21953201293945,
      "epoch": 1.32,
      "learning_rate": 0.0001429492424242424,
      "loss": 3.0251,
      "step": 22650
    },
    {
      "activations/layer0_attention_weight_max": 13.170254707336426,
      "activations/layer0_attention_weight_min": -12.88111400604248,
      "activations/layer10_attention_weight_max": 77.58692932128906,
      "activations/layer10_attention_weight_min": -64.1601333618164,
      "activations/layer11_attention_weight_max": 48.68388366699219,
      "activations/layer11_attention_weight_min": -37.91722106933594,
      "activations/layer12_attention_weight_max": 32.85112380981445,
      "activations/layer12_attention_weight_min": -24.07017707824707,
      "activations/layer13_attention_weight_max": 69.513671875,
      "activations/layer13_attention_weight_min": -46.95146942138672,
      "activations/layer14_attention_weight_max": 66.14053344726562,
      "activations/layer14_attention_weight_min": -42.66875076293945,
      "activations/layer15_attention_weight_max": 65.56686401367188,
      "activations/layer15_attention_weight_min": -44.15073776245117,
      "activations/layer16_attention_weight_max": 50.97468948364258,
      "activations/layer16_attention_weight_min": -34.44420623779297,
      "activations/layer17_attention_weight_max": 62.53300094604492,
      "activations/layer17_attention_weight_min": -40.02570724487305,
      "activations/layer18_attention_weight_max": 73.2088394165039,
      "activations/layer18_attention_weight_min": -40.229312896728516,
      "activations/layer19_attention_weight_max": 27.083730697631836,
      "activations/layer19_attention_weight_min": -13.48754596710205,
      "activations/layer1_attention_weight_max": 15.905205726623535,
      "activations/layer1_attention_weight_min": -11.616331100463867,
      "activations/layer20_attention_weight_max": 20.709447860717773,
      "activations/layer20_attention_weight_min": -14.285191535949707,
      "activations/layer21_attention_weight_max": 41.83094024658203,
      "activations/layer21_attention_weight_min": -23.08710289001465,
      "activations/layer22_attention_weight_max": 40.38447189331055,
      "activations/layer22_attention_weight_min": -20.452468872070312,
      "activations/layer23_attention_weight_max": 24.001708984375,
      "activations/layer23_attention_weight_min": -13.543837547302246,
      "activations/layer2_attention_weight_max": 24.884946823120117,
      "activations/layer2_attention_weight_min": -23.324146270751953,
      "activations/layer3_attention_weight_max": 54.16297149658203,
      "activations/layer3_attention_weight_min": -56.895225524902344,
      "activations/layer4_attention_weight_max": 82.65019226074219,
      "activations/layer4_attention_weight_min": -73.48351287841797,
      "activations/layer5_attention_weight_max": 129.56961059570312,
      "activations/layer5_attention_weight_min": -117.0400619506836,
      "activations/layer6_attention_weight_max": 60.65842056274414,
      "activations/layer6_attention_weight_min": -57.0101432800293,
      "activations/layer7_attention_weight_max": 65.65255737304688,
      "activations/layer7_attention_weight_min": -58.25231170654297,
      "activations/layer8_attention_weight_max": 54.84624481201172,
      "activations/layer8_attention_weight_min": -47.09098815917969,
      "activations/layer9_attention_weight_max": 74.60430145263672,
      "activations/layer9_attention_weight_min": -69.46822357177734,
      "epoch": 1.32,
      "learning_rate": 0.00014293030303030301,
      "loss": 3.0427,
      "step": 22700
    },
    {
      "activations/layer0_attention_weight_max": 15.002745628356934,
      "activations/layer0_attention_weight_min": -12.255229949951172,
      "activations/layer10_attention_weight_max": 72.34339141845703,
      "activations/layer10_attention_weight_min": -58.457794189453125,
      "activations/layer11_attention_weight_max": 53.968299865722656,
      "activations/layer11_attention_weight_min": -44.65726852416992,
      "activations/layer12_attention_weight_max": 29.44785499572754,
      "activations/layer12_attention_weight_min": -29.651342391967773,
      "activations/layer13_attention_weight_max": 67.83354187011719,
      "activations/layer13_attention_weight_min": -52.70101547241211,
      "activations/layer14_attention_weight_max": 57.854488372802734,
      "activations/layer14_attention_weight_min": -46.88596725463867,
      "activations/layer15_attention_weight_max": 57.7502326965332,
      "activations/layer15_attention_weight_min": -43.17171859741211,
      "activations/layer16_attention_weight_max": 43.79385757446289,
      "activations/layer16_attention_weight_min": -34.209224700927734,
      "activations/layer17_attention_weight_max": 53.6972541809082,
      "activations/layer17_attention_weight_min": -37.10916519165039,
      "activations/layer18_attention_weight_max": 59.2999153137207,
      "activations/layer18_attention_weight_min": -33.32212829589844,
      "activations/layer19_attention_weight_max": 18.094497680664062,
      "activations/layer19_attention_weight_min": -13.268324851989746,
      "activations/layer1_attention_weight_max": 14.813517570495605,
      "activations/layer1_attention_weight_min": -11.239958763122559,
      "activations/layer20_attention_weight_max": 18.572885513305664,
      "activations/layer20_attention_weight_min": -15.637105941772461,
      "activations/layer21_attention_weight_max": 30.08831024169922,
      "activations/layer21_attention_weight_min": -18.54606056213379,
      "activations/layer22_attention_weight_max": 27.49114990234375,
      "activations/layer22_attention_weight_min": -17.850765228271484,
      "activations/layer23_attention_weight_max": 18.348791122436523,
      "activations/layer23_attention_weight_min": -14.134291648864746,
      "activations/layer2_attention_weight_max": 26.395771026611328,
      "activations/layer2_attention_weight_min": -25.757465362548828,
      "activations/layer3_attention_weight_max": 56.38024139404297,
      "activations/layer3_attention_weight_min": -57.79492950439453,
      "activations/layer4_attention_weight_max": 85.03263854980469,
      "activations/layer4_attention_weight_min": -75.86437225341797,
      "activations/layer5_attention_weight_max": 128.37982177734375,
      "activations/layer5_attention_weight_min": -120.38685607910156,
      "activations/layer6_attention_weight_max": 60.96570587158203,
      "activations/layer6_attention_weight_min": -58.14517593383789,
      "activations/layer7_attention_weight_max": 60.992713928222656,
      "activations/layer7_attention_weight_min": -55.07810974121094,
      "activations/layer8_attention_weight_max": 57.88471984863281,
      "activations/layer8_attention_weight_min": -48.66079330444336,
      "activations/layer9_attention_weight_max": 81.84220886230469,
      "activations/layer9_attention_weight_min": -64.31857299804688,
      "epoch": 1.32,
      "learning_rate": 0.00014291136363636363,
      "loss": 3.0362,
      "step": 22750
    },
    {
      "activations/layer0_attention_weight_max": 13.390610694885254,
      "activations/layer0_attention_weight_min": -12.403554916381836,
      "activations/layer10_attention_weight_max": 68.94910430908203,
      "activations/layer10_attention_weight_min": -56.36827087402344,
      "activations/layer11_attention_weight_max": 45.471763610839844,
      "activations/layer11_attention_weight_min": -40.351158142089844,
      "activations/layer12_attention_weight_max": 29.492807388305664,
      "activations/layer12_attention_weight_min": -24.291704177856445,
      "activations/layer13_attention_weight_max": 66.4420166015625,
      "activations/layer13_attention_weight_min": -52.43880081176758,
      "activations/layer14_attention_weight_max": 53.33260726928711,
      "activations/layer14_attention_weight_min": -40.199066162109375,
      "activations/layer15_attention_weight_max": 62.28849792480469,
      "activations/layer15_attention_weight_min": -41.232398986816406,
      "activations/layer16_attention_weight_max": 52.86675262451172,
      "activations/layer16_attention_weight_min": -31.80021858215332,
      "activations/layer17_attention_weight_max": 63.16018295288086,
      "activations/layer17_attention_weight_min": -37.68867874145508,
      "activations/layer18_attention_weight_max": 65.89180755615234,
      "activations/layer18_attention_weight_min": -41.24346160888672,
      "activations/layer19_attention_weight_max": 21.743207931518555,
      "activations/layer19_attention_weight_min": -15.557339668273926,
      "activations/layer1_attention_weight_max": 15.097867965698242,
      "activations/layer1_attention_weight_min": -11.42673397064209,
      "activations/layer20_attention_weight_max": 21.45899200439453,
      "activations/layer20_attention_weight_min": -15.337191581726074,
      "activations/layer21_attention_weight_max": 37.29801559448242,
      "activations/layer21_attention_weight_min": -20.32123374938965,
      "activations/layer22_attention_weight_max": 36.978614807128906,
      "activations/layer22_attention_weight_min": -18.503122329711914,
      "activations/layer23_attention_weight_max": 22.99177360534668,
      "activations/layer23_attention_weight_min": -13.620420455932617,
      "activations/layer2_attention_weight_max": 25.968595504760742,
      "activations/layer2_attention_weight_min": -25.438461303710938,
      "activations/layer3_attention_weight_max": 56.89348220825195,
      "activations/layer3_attention_weight_min": -60.4466667175293,
      "activations/layer4_attention_weight_max": 87.8674087524414,
      "activations/layer4_attention_weight_min": -79.2441177368164,
      "activations/layer5_attention_weight_max": 134.80975341796875,
      "activations/layer5_attention_weight_min": -121.37171936035156,
      "activations/layer6_attention_weight_max": 66.1932601928711,
      "activations/layer6_attention_weight_min": -56.72834777832031,
      "activations/layer7_attention_weight_max": 65.13330078125,
      "activations/layer7_attention_weight_min": -58.6519889831543,
      "activations/layer8_attention_weight_max": 55.980098724365234,
      "activations/layer8_attention_weight_min": -51.47874069213867,
      "activations/layer9_attention_weight_max": 65.5883560180664,
      "activations/layer9_attention_weight_min": -62.18402099609375,
      "epoch": 1.32,
      "learning_rate": 0.00014289242424242422,
      "loss": 3.0516,
      "step": 22800
    },
    {
      "activations/layer0_attention_weight_max": 14.82303524017334,
      "activations/layer0_attention_weight_min": -12.91722583770752,
      "activations/layer10_attention_weight_max": 87.19036865234375,
      "activations/layer10_attention_weight_min": -61.37089538574219,
      "activations/layer11_attention_weight_max": 66.89651489257812,
      "activations/layer11_attention_weight_min": -43.48284912109375,
      "activations/layer12_attention_weight_max": 35.85250473022461,
      "activations/layer12_attention_weight_min": -27.91061019897461,
      "activations/layer13_attention_weight_max": 101.55879211425781,
      "activations/layer13_attention_weight_min": -51.43978500366211,
      "activations/layer14_attention_weight_max": 131.9860382080078,
      "activations/layer14_attention_weight_min": -63.113643646240234,
      "activations/layer15_attention_weight_max": 88.84300994873047,
      "activations/layer15_attention_weight_min": -41.697296142578125,
      "activations/layer16_attention_weight_max": 53.85306930541992,
      "activations/layer16_attention_weight_min": -31.794017791748047,
      "activations/layer17_attention_weight_max": 62.48541259765625,
      "activations/layer17_attention_weight_min": -36.77588653564453,
      "activations/layer18_attention_weight_max": 77.37737274169922,
      "activations/layer18_attention_weight_min": -41.40641403198242,
      "activations/layer19_attention_weight_max": 26.290393829345703,
      "activations/layer19_attention_weight_min": -14.419934272766113,
      "activations/layer1_attention_weight_max": 14.595817565917969,
      "activations/layer1_attention_weight_min": -11.496528625488281,
      "activations/layer20_attention_weight_max": 27.50698471069336,
      "activations/layer20_attention_weight_min": -14.207863807678223,
      "activations/layer21_attention_weight_max": 37.915340423583984,
      "activations/layer21_attention_weight_min": -18.55695343017578,
      "activations/layer22_attention_weight_max": 37.99649429321289,
      "activations/layer22_attention_weight_min": -18.93490982055664,
      "activations/layer23_attention_weight_max": 25.887348175048828,
      "activations/layer23_attention_weight_min": -16.559537887573242,
      "activations/layer2_attention_weight_max": 26.147632598876953,
      "activations/layer2_attention_weight_min": -25.359342575073242,
      "activations/layer3_attention_weight_max": 58.95274353027344,
      "activations/layer3_attention_weight_min": -60.4456901550293,
      "activations/layer4_attention_weight_max": 91.98956298828125,
      "activations/layer4_attention_weight_min": -80.4997787475586,
      "activations/layer5_attention_weight_max": 146.64926147460938,
      "activations/layer5_attention_weight_min": -128.9286651611328,
      "activations/layer6_attention_weight_max": 69.16679382324219,
      "activations/layer6_attention_weight_min": -57.16413116455078,
      "activations/layer7_attention_weight_max": 70.4476089477539,
      "activations/layer7_attention_weight_min": -58.99677658081055,
      "activations/layer8_attention_weight_max": 73.83943176269531,
      "activations/layer8_attention_weight_min": -53.283267974853516,
      "activations/layer9_attention_weight_max": 103.14791107177734,
      "activations/layer9_attention_weight_min": -76.66401672363281,
      "epoch": 1.33,
      "learning_rate": 0.00014287348484848483,
      "loss": 3.0352,
      "step": 22850
    },
    {
      "activations/layer0_attention_weight_max": 14.324335098266602,
      "activations/layer0_attention_weight_min": -12.747574806213379,
      "activations/layer10_attention_weight_max": 69.57687377929688,
      "activations/layer10_attention_weight_min": -63.36811447143555,
      "activations/layer11_attention_weight_max": 43.99055480957031,
      "activations/layer11_attention_weight_min": -38.71476745605469,
      "activations/layer12_attention_weight_max": 31.704866409301758,
      "activations/layer12_attention_weight_min": -23.450754165649414,
      "activations/layer13_attention_weight_max": 64.98379516601562,
      "activations/layer13_attention_weight_min": -50.64533996582031,
      "activations/layer14_attention_weight_max": 51.583709716796875,
      "activations/layer14_attention_weight_min": -36.853118896484375,
      "activations/layer15_attention_weight_max": 55.67156219482422,
      "activations/layer15_attention_weight_min": -42.602745056152344,
      "activations/layer16_attention_weight_max": 44.1645393371582,
      "activations/layer16_attention_weight_min": -33.5393180847168,
      "activations/layer17_attention_weight_max": 60.33871078491211,
      "activations/layer17_attention_weight_min": -39.09574508666992,
      "activations/layer18_attention_weight_max": 70.83544158935547,
      "activations/layer18_attention_weight_min": -38.48262405395508,
      "activations/layer19_attention_weight_max": 20.287797927856445,
      "activations/layer19_attention_weight_min": -14.037927627563477,
      "activations/layer1_attention_weight_max": 15.351585388183594,
      "activations/layer1_attention_weight_min": -12.743382453918457,
      "activations/layer20_attention_weight_max": 19.732929229736328,
      "activations/layer20_attention_weight_min": -13.838695526123047,
      "activations/layer21_attention_weight_max": 40.298301696777344,
      "activations/layer21_attention_weight_min": -21.551393508911133,
      "activations/layer22_attention_weight_max": 37.58605194091797,
      "activations/layer22_attention_weight_min": -20.52632713317871,
      "activations/layer23_attention_weight_max": 23.146177291870117,
      "activations/layer23_attention_weight_min": -15.369039535522461,
      "activations/layer2_attention_weight_max": 26.46054458618164,
      "activations/layer2_attention_weight_min": -24.694313049316406,
      "activations/layer3_attention_weight_max": 54.7287712097168,
      "activations/layer3_attention_weight_min": -57.8933219909668,
      "activations/layer4_attention_weight_max": 81.64620971679688,
      "activations/layer4_attention_weight_min": -75.3878402709961,
      "activations/layer5_attention_weight_max": 124.17227172851562,
      "activations/layer5_attention_weight_min": -118.18801879882812,
      "activations/layer6_attention_weight_max": 59.83189010620117,
      "activations/layer6_attention_weight_min": -57.97159194946289,
      "activations/layer7_attention_weight_max": 60.3144645690918,
      "activations/layer7_attention_weight_min": -54.7227668762207,
      "activations/layer8_attention_weight_max": 52.23931121826172,
      "activations/layer8_attention_weight_min": -47.34566116333008,
      "activations/layer9_attention_weight_max": 63.4322509765625,
      "activations/layer9_attention_weight_min": -59.452308654785156,
      "epoch": 1.33,
      "learning_rate": 0.00014285454545454545,
      "loss": 3.0372,
      "step": 22900
    },
    {
      "activations/layer0_attention_weight_max": 13.65666389465332,
      "activations/layer0_attention_weight_min": -12.453167915344238,
      "activations/layer10_attention_weight_max": 63.11973571777344,
      "activations/layer10_attention_weight_min": -57.939937591552734,
      "activations/layer11_attention_weight_max": 45.14978790283203,
      "activations/layer11_attention_weight_min": -39.58436965942383,
      "activations/layer12_attention_weight_max": 27.420499801635742,
      "activations/layer12_attention_weight_min": -25.659805297851562,
      "activations/layer13_attention_weight_max": 59.709190368652344,
      "activations/layer13_attention_weight_min": -47.01163101196289,
      "activations/layer14_attention_weight_max": 60.83989715576172,
      "activations/layer14_attention_weight_min": -41.498374938964844,
      "activations/layer15_attention_weight_max": 54.220523834228516,
      "activations/layer15_attention_weight_min": -42.63973617553711,
      "activations/layer16_attention_weight_max": 43.15798568725586,
      "activations/layer16_attention_weight_min": -32.901947021484375,
      "activations/layer17_attention_weight_max": 52.707061767578125,
      "activations/layer17_attention_weight_min": -35.402503967285156,
      "activations/layer18_attention_weight_max": 64.06165313720703,
      "activations/layer18_attention_weight_min": -44.30831527709961,
      "activations/layer19_attention_weight_max": 20.928546905517578,
      "activations/layer19_attention_weight_min": -14.868914604187012,
      "activations/layer1_attention_weight_max": 14.499009132385254,
      "activations/layer1_attention_weight_min": -10.37690544128418,
      "activations/layer20_attention_weight_max": 19.496749877929688,
      "activations/layer20_attention_weight_min": -16.41086769104004,
      "activations/layer21_attention_weight_max": 37.923683166503906,
      "activations/layer21_attention_weight_min": -19.71771240234375,
      "activations/layer22_attention_weight_max": 38.65394592285156,
      "activations/layer22_attention_weight_min": -18.321880340576172,
      "activations/layer23_attention_weight_max": 25.667064666748047,
      "activations/layer23_attention_weight_min": -16.679277420043945,
      "activations/layer2_attention_weight_max": 26.14983367919922,
      "activations/layer2_attention_weight_min": -25.712860107421875,
      "activations/layer3_attention_weight_max": 54.12702178955078,
      "activations/layer3_attention_weight_min": -55.79213333129883,
      "activations/layer4_attention_weight_max": 85.00183868408203,
      "activations/layer4_attention_weight_min": -78.62886047363281,
      "activations/layer5_attention_weight_max": 131.00634765625,
      "activations/layer5_attention_weight_min": -120.98509216308594,
      "activations/layer6_attention_weight_max": 66.90988159179688,
      "activations/layer6_attention_weight_min": -56.2925910949707,
      "activations/layer7_attention_weight_max": 62.63389205932617,
      "activations/layer7_attention_weight_min": -56.671756744384766,
      "activations/layer8_attention_weight_max": 53.512516021728516,
      "activations/layer8_attention_weight_min": -48.35560989379883,
      "activations/layer9_attention_weight_max": 67.40201568603516,
      "activations/layer9_attention_weight_min": -68.42293548583984,
      "epoch": 1.33,
      "learning_rate": 0.00014283560606060604,
      "loss": 3.0309,
      "step": 22950
    },
    {
      "activations/layer0_attention_weight_max": 13.859527587890625,
      "activations/layer0_attention_weight_min": -12.547955513000488,
      "activations/layer10_attention_weight_max": 66.24818420410156,
      "activations/layer10_attention_weight_min": -56.05612564086914,
      "activations/layer11_attention_weight_max": 40.120155334472656,
      "activations/layer11_attention_weight_min": -37.677513122558594,
      "activations/layer12_attention_weight_max": 33.7479133605957,
      "activations/layer12_attention_weight_min": -29.356046676635742,
      "activations/layer13_attention_weight_max": 55.263832092285156,
      "activations/layer13_attention_weight_min": -41.360557556152344,
      "activations/layer14_attention_weight_max": 47.00428771972656,
      "activations/layer14_attention_weight_min": -34.4703483581543,
      "activations/layer15_attention_weight_max": 49.36387252807617,
      "activations/layer15_attention_weight_min": -37.655372619628906,
      "activations/layer16_attention_weight_max": 45.050662994384766,
      "activations/layer16_attention_weight_min": -31.14356231689453,
      "activations/layer17_attention_weight_max": 63.84528350830078,
      "activations/layer17_attention_weight_min": -39.162681579589844,
      "activations/layer18_attention_weight_max": 62.174476623535156,
      "activations/layer18_attention_weight_min": -34.77926254272461,
      "activations/layer19_attention_weight_max": 21.465808868408203,
      "activations/layer19_attention_weight_min": -14.469813346862793,
      "activations/layer1_attention_weight_max": 14.89909553527832,
      "activations/layer1_attention_weight_min": -12.916365623474121,
      "activations/layer20_attention_weight_max": 20.052453994750977,
      "activations/layer20_attention_weight_min": -15.864246368408203,
      "activations/layer21_attention_weight_max": 41.63172912597656,
      "activations/layer21_attention_weight_min": -21.41227149963379,
      "activations/layer22_attention_weight_max": 43.558258056640625,
      "activations/layer22_attention_weight_min": -20.58893394470215,
      "activations/layer23_attention_weight_max": 25.347354888916016,
      "activations/layer23_attention_weight_min": -13.408251762390137,
      "activations/layer2_attention_weight_max": 23.6243953704834,
      "activations/layer2_attention_weight_min": -23.090774536132812,
      "activations/layer3_attention_weight_max": 52.78260040283203,
      "activations/layer3_attention_weight_min": -56.3488655090332,
      "activations/layer4_attention_weight_max": 81.73994445800781,
      "activations/layer4_attention_weight_min": -73.68790435791016,
      "activations/layer5_attention_weight_max": 123.02729797363281,
      "activations/layer5_attention_weight_min": -112.91127014160156,
      "activations/layer6_attention_weight_max": 56.327880859375,
      "activations/layer6_attention_weight_min": -53.86377716064453,
      "activations/layer7_attention_weight_max": 58.259239196777344,
      "activations/layer7_attention_weight_min": -57.535335540771484,
      "activations/layer8_attention_weight_max": 51.64955520629883,
      "activations/layer8_attention_weight_min": -45.05286407470703,
      "activations/layer9_attention_weight_max": 62.41202926635742,
      "activations/layer9_attention_weight_min": -63.65047073364258,
      "epoch": 1.34,
      "learning_rate": 0.00014281666666666665,
      "loss": 3.0175,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_loss": 2.984375,
      "eval_runtime": 8.4451,
      "eval_samples_per_second": 508.461,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_openwebtext_loss": 2.984375,
      "eval_openwebtext_ppl": 19.774139536218833,
      "eval_openwebtext_runtime": 8.4451,
      "eval_openwebtext_samples_per_second": 508.461,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_wikitext_loss": 3.267578125,
      "eval_wikitext_ppl": 26.24769367033534,
      "eval_wikitext_runtime": 1.7063,
      "eval_wikitext_samples_per_second": 267.243,
      "step": 23000
    },
    {
      "epoch": 1.34,
      "eval_lambada_loss": 3.298828125,
      "eval_lambada_ppl": 27.080884906424416,
      "eval_lambada_runtime": 9.33,
      "eval_lambada_samples_per_second": 521.865,
      "step": 23000
    },
    {
      "activations/layer0_attention_weight_max": 14.179786682128906,
      "activations/layer0_attention_weight_min": -12.463418006896973,
      "activations/layer10_attention_weight_max": 63.5230712890625,
      "activations/layer10_attention_weight_min": -57.363182067871094,
      "activations/layer11_attention_weight_max": 43.78426742553711,
      "activations/layer11_attention_weight_min": -38.63763427734375,
      "activations/layer12_attention_weight_max": 26.32579231262207,
      "activations/layer12_attention_weight_min": -25.50743865966797,
      "activations/layer13_attention_weight_max": 51.21684646606445,
      "activations/layer13_attention_weight_min": -41.208091735839844,
      "activations/layer14_attention_weight_max": 50.38179397583008,
      "activations/layer14_attention_weight_min": -44.78628158569336,
      "activations/layer15_attention_weight_max": 50.846954345703125,
      "activations/layer15_attention_weight_min": -42.204612731933594,
      "activations/layer16_attention_weight_max": 43.39912414550781,
      "activations/layer16_attention_weight_min": -33.055511474609375,
      "activations/layer17_attention_weight_max": 54.883365631103516,
      "activations/layer17_attention_weight_min": -37.08708572387695,
      "activations/layer18_attention_weight_max": 60.589683532714844,
      "activations/layer18_attention_weight_min": -39.34073257446289,
      "activations/layer19_attention_weight_max": 19.622936248779297,
      "activations/layer19_attention_weight_min": -15.778090476989746,
      "activations/layer1_attention_weight_max": 14.82490348815918,
      "activations/layer1_attention_weight_min": -12.233810424804688,
      "activations/layer20_attention_weight_max": 22.351097106933594,
      "activations/layer20_attention_weight_min": -15.025470733642578,
      "activations/layer21_attention_weight_max": 35.06903076171875,
      "activations/layer21_attention_weight_min": -21.751394271850586,
      "activations/layer22_attention_weight_max": 35.052650451660156,
      "activations/layer22_attention_weight_min": -22.563671112060547,
      "activations/layer23_attention_weight_max": 23.127256393432617,
      "activations/layer23_attention_weight_min": -14.264342308044434,
      "activations/layer2_attention_weight_max": 26.097347259521484,
      "activations/layer2_attention_weight_min": -24.21703338623047,
      "activations/layer3_attention_weight_max": 55.20745086669922,
      "activations/layer3_attention_weight_min": -55.48027420043945,
      "activations/layer4_attention_weight_max": 84.10745239257812,
      "activations/layer4_attention_weight_min": -74.93318176269531,
      "activations/layer5_attention_weight_max": 119.92218017578125,
      "activations/layer5_attention_weight_min": -125.16036224365234,
      "activations/layer6_attention_weight_max": 59.28641128540039,
      "activations/layer6_attention_weight_min": -61.22401809692383,
      "activations/layer7_attention_weight_max": 57.63737106323242,
      "activations/layer7_attention_weight_min": -54.630672454833984,
      "activations/layer8_attention_weight_max": 48.97816848754883,
      "activations/layer8_attention_weight_min": -45.812843322753906,
      "activations/layer9_attention_weight_max": 65.97369384765625,
      "activations/layer9_attention_weight_min": -65.6471176147461,
      "epoch": 1.34,
      "learning_rate": 0.00014279772727272727,
      "loss": 3.0426,
      "step": 23050
    },
    {
      "activations/layer0_attention_weight_max": 16.024099349975586,
      "activations/layer0_attention_weight_min": -12.615131378173828,
      "activations/layer10_attention_weight_max": 66.76039123535156,
      "activations/layer10_attention_weight_min": -56.15109634399414,
      "activations/layer11_attention_weight_max": 42.74674606323242,
      "activations/layer11_attention_weight_min": -37.951942443847656,
      "activations/layer12_attention_weight_max": 33.31604766845703,
      "activations/layer12_attention_weight_min": -24.515975952148438,
      "activations/layer13_attention_weight_max": 60.868106842041016,
      "activations/layer13_attention_weight_min": -46.71565628051758,
      "activations/layer14_attention_weight_max": 49.59356689453125,
      "activations/layer14_attention_weight_min": -43.62879943847656,
      "activations/layer15_attention_weight_max": 56.73975372314453,
      "activations/layer15_attention_weight_min": -42.21063995361328,
      "activations/layer16_attention_weight_max": 45.446720123291016,
      "activations/layer16_attention_weight_min": -35.386592864990234,
      "activations/layer17_attention_weight_max": 57.69659423828125,
      "activations/layer17_attention_weight_min": -35.657371520996094,
      "activations/layer18_attention_weight_max": 64.70308685302734,
      "activations/layer18_attention_weight_min": -38.552886962890625,
      "activations/layer19_attention_weight_max": 20.35694694519043,
      "activations/layer19_attention_weight_min": -13.874701499938965,
      "activations/layer1_attention_weight_max": 15.689772605895996,
      "activations/layer1_attention_weight_min": -11.722283363342285,
      "activations/layer20_attention_weight_max": 18.50609016418457,
      "activations/layer20_attention_weight_min": -15.347408294677734,
      "activations/layer21_attention_weight_max": 35.0972900390625,
      "activations/layer21_attention_weight_min": -20.678451538085938,
      "activations/layer22_attention_weight_max": 36.770931243896484,
      "activations/layer22_attention_weight_min": -20.535730361938477,
      "activations/layer23_attention_weight_max": 21.140216827392578,
      "activations/layer23_attention_weight_min": -13.896474838256836,
      "activations/layer2_attention_weight_max": 25.66388511657715,
      "activations/layer2_attention_weight_min": -24.089475631713867,
      "activations/layer3_attention_weight_max": 55.01601028442383,
      "activations/layer3_attention_weight_min": -57.499507904052734,
      "activations/layer4_attention_weight_max": 84.4119644165039,
      "activations/layer4_attention_weight_min": -76.43559265136719,
      "activations/layer5_attention_weight_max": 131.12091064453125,
      "activations/layer5_attention_weight_min": -119.865234375,
      "activations/layer6_attention_weight_max": 63.04573059082031,
      "activations/layer6_attention_weight_min": -52.70280075073242,
      "activations/layer7_attention_weight_max": 58.93392562866211,
      "activations/layer7_attention_weight_min": -53.57319259643555,
      "activations/layer8_attention_weight_max": 53.91806411743164,
      "activations/layer8_attention_weight_min": -43.986331939697266,
      "activations/layer9_attention_weight_max": 75.2658920288086,
      "activations/layer9_attention_weight_min": -69.54790496826172,
      "epoch": 1.34,
      "learning_rate": 0.00014277878787878785,
      "loss": 3.0327,
      "step": 23100
    },
    {
      "activations/layer0_attention_weight_max": 13.305953979492188,
      "activations/layer0_attention_weight_min": -12.509134292602539,
      "activations/layer10_attention_weight_max": 58.436859130859375,
      "activations/layer10_attention_weight_min": -52.85239028930664,
      "activations/layer11_attention_weight_max": 40.166324615478516,
      "activations/layer11_attention_weight_min": -36.04280090332031,
      "activations/layer12_attention_weight_max": 26.84425163269043,
      "activations/layer12_attention_weight_min": -23.49976921081543,
      "activations/layer13_attention_weight_max": 57.14027786254883,
      "activations/layer13_attention_weight_min": -44.60035705566406,
      "activations/layer14_attention_weight_max": 44.863441467285156,
      "activations/layer14_attention_weight_min": -35.79913330078125,
      "activations/layer15_attention_weight_max": 53.33321762084961,
      "activations/layer15_attention_weight_min": -38.65754699707031,
      "activations/layer16_attention_weight_max": 48.1136360168457,
      "activations/layer16_attention_weight_min": -31.982202529907227,
      "activations/layer17_attention_weight_max": 63.92448043823242,
      "activations/layer17_attention_weight_min": -39.62663650512695,
      "activations/layer18_attention_weight_max": 64.01023864746094,
      "activations/layer18_attention_weight_min": -37.285438537597656,
      "activations/layer19_attention_weight_max": 22.099105834960938,
      "activations/layer19_attention_weight_min": -15.972343444824219,
      "activations/layer1_attention_weight_max": 14.76344108581543,
      "activations/layer1_attention_weight_min": -11.110929489135742,
      "activations/layer20_attention_weight_max": 22.127361297607422,
      "activations/layer20_attention_weight_min": -15.612832069396973,
      "activations/layer21_attention_weight_max": 40.21364212036133,
      "activations/layer21_attention_weight_min": -21.15678596496582,
      "activations/layer22_attention_weight_max": 38.750831604003906,
      "activations/layer22_attention_weight_min": -18.51524543762207,
      "activations/layer23_attention_weight_max": 26.175247192382812,
      "activations/layer23_attention_weight_min": -13.138044357299805,
      "activations/layer2_attention_weight_max": 26.552602767944336,
      "activations/layer2_attention_weight_min": -23.580053329467773,
      "activations/layer3_attention_weight_max": 52.009788513183594,
      "activations/layer3_attention_weight_min": -56.13800048828125,
      "activations/layer4_attention_weight_max": 85.32353973388672,
      "activations/layer4_attention_weight_min": -79.0715560913086,
      "activations/layer5_attention_weight_max": 128.45193481445312,
      "activations/layer5_attention_weight_min": -117.24171447753906,
      "activations/layer6_attention_weight_max": 60.91008758544922,
      "activations/layer6_attention_weight_min": -56.27823257446289,
      "activations/layer7_attention_weight_max": 61.043941497802734,
      "activations/layer7_attention_weight_min": -53.30084991455078,
      "activations/layer8_attention_weight_max": 51.38422393798828,
      "activations/layer8_attention_weight_min": -46.139034271240234,
      "activations/layer9_attention_weight_max": 67.30209350585938,
      "activations/layer9_attention_weight_min": -61.82257843017578,
      "epoch": 1.35,
      "learning_rate": 0.00014275984848484847,
      "loss": 3.036,
      "step": 23150
    },
    {
      "activations/layer0_attention_weight_max": 16.184829711914062,
      "activations/layer0_attention_weight_min": -12.409063339233398,
      "activations/layer10_attention_weight_max": 66.8817367553711,
      "activations/layer10_attention_weight_min": -53.54380798339844,
      "activations/layer11_attention_weight_max": 47.15460968017578,
      "activations/layer11_attention_weight_min": -39.06987762451172,
      "activations/layer12_attention_weight_max": 30.51434898376465,
      "activations/layer12_attention_weight_min": -22.454252243041992,
      "activations/layer13_attention_weight_max": 67.15199279785156,
      "activations/layer13_attention_weight_min": -47.034305572509766,
      "activations/layer14_attention_weight_max": 59.779747009277344,
      "activations/layer14_attention_weight_min": -43.74559783935547,
      "activations/layer15_attention_weight_max": 52.303653717041016,
      "activations/layer15_attention_weight_min": -36.695316314697266,
      "activations/layer16_attention_weight_max": 44.96931838989258,
      "activations/layer16_attention_weight_min": -34.465328216552734,
      "activations/layer17_attention_weight_max": 60.91099166870117,
      "activations/layer17_attention_weight_min": -38.51857376098633,
      "activations/layer18_attention_weight_max": 69.96001434326172,
      "activations/layer18_attention_weight_min": -38.551170349121094,
      "activations/layer19_attention_weight_max": 24.323772430419922,
      "activations/layer19_attention_weight_min": -14.108633995056152,
      "activations/layer1_attention_weight_max": 14.675548553466797,
      "activations/layer1_attention_weight_min": -10.82445240020752,
      "activations/layer20_attention_weight_max": 22.491125106811523,
      "activations/layer20_attention_weight_min": -17.199796676635742,
      "activations/layer21_attention_weight_max": 40.60424041748047,
      "activations/layer21_attention_weight_min": -24.601335525512695,
      "activations/layer22_attention_weight_max": 38.10762023925781,
      "activations/layer22_attention_weight_min": -22.63977813720703,
      "activations/layer23_attention_weight_max": 23.322105407714844,
      "activations/layer23_attention_weight_min": -13.682779312133789,
      "activations/layer2_attention_weight_max": 25.21742057800293,
      "activations/layer2_attention_weight_min": -23.884761810302734,
      "activations/layer3_attention_weight_max": 52.739810943603516,
      "activations/layer3_attention_weight_min": -56.352386474609375,
      "activations/layer4_attention_weight_max": 80.94537353515625,
      "activations/layer4_attention_weight_min": -76.32914733886719,
      "activations/layer5_attention_weight_max": 134.48147583007812,
      "activations/layer5_attention_weight_min": -123.44485473632812,
      "activations/layer6_attention_weight_max": 61.29267120361328,
      "activations/layer6_attention_weight_min": -58.031097412109375,
      "activations/layer7_attention_weight_max": 65.03470611572266,
      "activations/layer7_attention_weight_min": -58.451202392578125,
      "activations/layer8_attention_weight_max": 55.90650177001953,
      "activations/layer8_attention_weight_min": -47.19546127319336,
      "activations/layer9_attention_weight_max": 76.76517486572266,
      "activations/layer9_attention_weight_min": -69.53392791748047,
      "epoch": 1.35,
      "learning_rate": 0.00014274090909090908,
      "loss": 3.0351,
      "step": 23200
    },
    {
      "activations/layer0_attention_weight_max": 14.62179183959961,
      "activations/layer0_attention_weight_min": -12.150371551513672,
      "activations/layer10_attention_weight_max": 61.35733413696289,
      "activations/layer10_attention_weight_min": -57.77397155761719,
      "activations/layer11_attention_weight_max": 43.271583557128906,
      "activations/layer11_attention_weight_min": -40.25501251220703,
      "activations/layer12_attention_weight_max": 24.650508880615234,
      "activations/layer12_attention_weight_min": -23.426664352416992,
      "activations/layer13_attention_weight_max": 65.51622009277344,
      "activations/layer13_attention_weight_min": -46.97984313964844,
      "activations/layer14_attention_weight_max": 51.48802185058594,
      "activations/layer14_attention_weight_min": -38.7609977722168,
      "activations/layer15_attention_weight_max": 56.547203063964844,
      "activations/layer15_attention_weight_min": -42.13157272338867,
      "activations/layer16_attention_weight_max": 48.524410247802734,
      "activations/layer16_attention_weight_min": -32.50382995605469,
      "activations/layer17_attention_weight_max": 54.594181060791016,
      "activations/layer17_attention_weight_min": -36.729652404785156,
      "activations/layer18_attention_weight_max": 63.69968795776367,
      "activations/layer18_attention_weight_min": -36.61219024658203,
      "activations/layer19_attention_weight_max": 22.88384437561035,
      "activations/layer19_attention_weight_min": -15.298413276672363,
      "activations/layer1_attention_weight_max": 15.968241691589355,
      "activations/layer1_attention_weight_min": -10.762506484985352,
      "activations/layer20_attention_weight_max": 22.460290908813477,
      "activations/layer20_attention_weight_min": -15.645049095153809,
      "activations/layer21_attention_weight_max": 35.81132507324219,
      "activations/layer21_attention_weight_min": -19.87101173400879,
      "activations/layer22_attention_weight_max": 35.438255310058594,
      "activations/layer22_attention_weight_min": -17.13195037841797,
      "activations/layer23_attention_weight_max": 23.347251892089844,
      "activations/layer23_attention_weight_min": -14.472965240478516,
      "activations/layer2_attention_weight_max": 24.872007369995117,
      "activations/layer2_attention_weight_min": -24.6839656829834,
      "activations/layer3_attention_weight_max": 58.25371551513672,
      "activations/layer3_attention_weight_min": -57.90110778808594,
      "activations/layer4_attention_weight_max": 87.1264877319336,
      "activations/layer4_attention_weight_min": -78.37194061279297,
      "activations/layer5_attention_weight_max": 133.96861267089844,
      "activations/layer5_attention_weight_min": -124.23670959472656,
      "activations/layer6_attention_weight_max": 61.03187561035156,
      "activations/layer6_attention_weight_min": -58.42964553833008,
      "activations/layer7_attention_weight_max": 60.78609848022461,
      "activations/layer7_attention_weight_min": -57.58102035522461,
      "activations/layer8_attention_weight_max": 51.269935607910156,
      "activations/layer8_attention_weight_min": -48.28374481201172,
      "activations/layer9_attention_weight_max": 72.33464050292969,
      "activations/layer9_attention_weight_min": -63.95155715942383,
      "epoch": 1.35,
      "learning_rate": 0.00014272196969696967,
      "loss": 3.0481,
      "step": 23250
    },
    {
      "activations/layer0_attention_weight_max": 13.892913818359375,
      "activations/layer0_attention_weight_min": -12.593972206115723,
      "activations/layer10_attention_weight_max": 64.49771118164062,
      "activations/layer10_attention_weight_min": -57.14472579956055,
      "activations/layer11_attention_weight_max": 45.30744934082031,
      "activations/layer11_attention_weight_min": -43.0186767578125,
      "activations/layer12_attention_weight_max": 34.90492630004883,
      "activations/layer12_attention_weight_min": -25.10429573059082,
      "activations/layer13_attention_weight_max": 59.982913970947266,
      "activations/layer13_attention_weight_min": -40.824188232421875,
      "activations/layer14_attention_weight_max": 57.78829574584961,
      "activations/layer14_attention_weight_min": -38.385711669921875,
      "activations/layer15_attention_weight_max": 49.93941879272461,
      "activations/layer15_attention_weight_min": -38.10744857788086,
      "activations/layer16_attention_weight_max": 49.83658218383789,
      "activations/layer16_attention_weight_min": -33.61891174316406,
      "activations/layer17_attention_weight_max": 61.9486083984375,
      "activations/layer17_attention_weight_min": -35.5124397277832,
      "activations/layer18_attention_weight_max": 66.16329956054688,
      "activations/layer18_attention_weight_min": -32.725067138671875,
      "activations/layer19_attention_weight_max": 21.225072860717773,
      "activations/layer19_attention_weight_min": -14.224577903747559,
      "activations/layer1_attention_weight_max": 15.564345359802246,
      "activations/layer1_attention_weight_min": -11.547074317932129,
      "activations/layer20_attention_weight_max": 22.1877384185791,
      "activations/layer20_attention_weight_min": -14.662109375,
      "activations/layer21_attention_weight_max": 41.78122329711914,
      "activations/layer21_attention_weight_min": -20.62044334411621,
      "activations/layer22_attention_weight_max": 39.562129974365234,
      "activations/layer22_attention_weight_min": -19.831974029541016,
      "activations/layer23_attention_weight_max": 22.572887420654297,
      "activations/layer23_attention_weight_min": -15.43399715423584,
      "activations/layer2_attention_weight_max": 26.806442260742188,
      "activations/layer2_attention_weight_min": -24.641735076904297,
      "activations/layer3_attention_weight_max": 53.71006393432617,
      "activations/layer3_attention_weight_min": -54.78474426269531,
      "activations/layer4_attention_weight_max": 83.0838851928711,
      "activations/layer4_attention_weight_min": -76.6530990600586,
      "activations/layer5_attention_weight_max": 130.21853637695312,
      "activations/layer5_attention_weight_min": -115.9397201538086,
      "activations/layer6_attention_weight_max": 58.99976348876953,
      "activations/layer6_attention_weight_min": -54.43098068237305,
      "activations/layer7_attention_weight_max": 60.589744567871094,
      "activations/layer7_attention_weight_min": -55.651588439941406,
      "activations/layer8_attention_weight_max": 51.4702033996582,
      "activations/layer8_attention_weight_min": -45.74933624267578,
      "activations/layer9_attention_weight_max": 73.06536102294922,
      "activations/layer9_attention_weight_min": -60.35150146484375,
      "epoch": 1.35,
      "learning_rate": 0.0001427030303030303,
      "loss": 3.0323,
      "step": 23300
    },
    {
      "activations/layer0_attention_weight_max": 14.918028831481934,
      "activations/layer0_attention_weight_min": -12.835662841796875,
      "activations/layer10_attention_weight_max": 68.37749481201172,
      "activations/layer10_attention_weight_min": -55.10247039794922,
      "activations/layer11_attention_weight_max": 53.870330810546875,
      "activations/layer11_attention_weight_min": -40.98728942871094,
      "activations/layer12_attention_weight_max": 27.04688835144043,
      "activations/layer12_attention_weight_min": -25.348058700561523,
      "activations/layer13_attention_weight_max": 65.07425689697266,
      "activations/layer13_attention_weight_min": -45.523136138916016,
      "activations/layer14_attention_weight_max": 62.68971252441406,
      "activations/layer14_attention_weight_min": -43.94483947753906,
      "activations/layer15_attention_weight_max": 57.87022018432617,
      "activations/layer15_attention_weight_min": -40.76840591430664,
      "activations/layer16_attention_weight_max": 51.928348541259766,
      "activations/layer16_attention_weight_min": -32.822025299072266,
      "activations/layer17_attention_weight_max": 68.35172271728516,
      "activations/layer17_attention_weight_min": -39.52756118774414,
      "activations/layer18_attention_weight_max": 70.69506072998047,
      "activations/layer18_attention_weight_min": -40.54426193237305,
      "activations/layer19_attention_weight_max": 20.61885643005371,
      "activations/layer19_attention_weight_min": -15.523381233215332,
      "activations/layer1_attention_weight_max": 16.38170051574707,
      "activations/layer1_attention_weight_min": -10.751976013183594,
      "activations/layer20_attention_weight_max": 23.194547653198242,
      "activations/layer20_attention_weight_min": -14.904115676879883,
      "activations/layer21_attention_weight_max": 38.10549545288086,
      "activations/layer21_attention_weight_min": -20.274497985839844,
      "activations/layer22_attention_weight_max": 38.32577896118164,
      "activations/layer22_attention_weight_min": -21.1729736328125,
      "activations/layer23_attention_weight_max": 23.876434326171875,
      "activations/layer23_attention_weight_min": -15.92544937133789,
      "activations/layer2_attention_weight_max": 26.324460983276367,
      "activations/layer2_attention_weight_min": -24.436267852783203,
      "activations/layer3_attention_weight_max": 60.32889938354492,
      "activations/layer3_attention_weight_min": -60.054222106933594,
      "activations/layer4_attention_weight_max": 92.73019409179688,
      "activations/layer4_attention_weight_min": -80.11284637451172,
      "activations/layer5_attention_weight_max": 154.34388732910156,
      "activations/layer5_attention_weight_min": -119.73528289794922,
      "activations/layer6_attention_weight_max": 67.64958953857422,
      "activations/layer6_attention_weight_min": -56.13048553466797,
      "activations/layer7_attention_weight_max": 65.79878234863281,
      "activations/layer7_attention_weight_min": -58.42871856689453,
      "activations/layer8_attention_weight_max": 54.899471282958984,
      "activations/layer8_attention_weight_min": -48.36361312866211,
      "activations/layer9_attention_weight_max": 85.46566009521484,
      "activations/layer9_attention_weight_min": -77.77429962158203,
      "epoch": 1.36,
      "learning_rate": 0.0001426840909090909,
      "loss": 3.0468,
      "step": 23350
    },
    {
      "activations/layer0_attention_weight_max": 14.17973518371582,
      "activations/layer0_attention_weight_min": -13.049115180969238,
      "activations/layer10_attention_weight_max": 68.64026641845703,
      "activations/layer10_attention_weight_min": -53.44744110107422,
      "activations/layer11_attention_weight_max": 44.05529022216797,
      "activations/layer11_attention_weight_min": -41.34867477416992,
      "activations/layer12_attention_weight_max": 27.01455307006836,
      "activations/layer12_attention_weight_min": -24.7283992767334,
      "activations/layer13_attention_weight_max": 50.95152282714844,
      "activations/layer13_attention_weight_min": -40.64545440673828,
      "activations/layer14_attention_weight_max": 50.135040283203125,
      "activations/layer14_attention_weight_min": -33.503292083740234,
      "activations/layer15_attention_weight_max": 61.11439895629883,
      "activations/layer15_attention_weight_min": -40.87038803100586,
      "activations/layer16_attention_weight_max": 52.7049674987793,
      "activations/layer16_attention_weight_min": -33.85295486450195,
      "activations/layer17_attention_weight_max": 64.4415054321289,
      "activations/layer17_attention_weight_min": -39.3912239074707,
      "activations/layer18_attention_weight_max": 65.8939437866211,
      "activations/layer18_attention_weight_min": -36.214290618896484,
      "activations/layer19_attention_weight_max": 24.064739227294922,
      "activations/layer19_attention_weight_min": -13.266326904296875,
      "activations/layer1_attention_weight_max": 15.132319450378418,
      "activations/layer1_attention_weight_min": -12.377348899841309,
      "activations/layer20_attention_weight_max": 23.261472702026367,
      "activations/layer20_attention_weight_min": -15.920144081115723,
      "activations/layer21_attention_weight_max": 35.56473159790039,
      "activations/layer21_attention_weight_min": -19.42222023010254,
      "activations/layer22_attention_weight_max": 36.56500244140625,
      "activations/layer22_attention_weight_min": -18.81255531311035,
      "activations/layer23_attention_weight_max": 23.257762908935547,
      "activations/layer23_attention_weight_min": -16.78018569946289,
      "activations/layer2_attention_weight_max": 25.84853172302246,
      "activations/layer2_attention_weight_min": -26.539653778076172,
      "activations/layer3_attention_weight_max": 56.583744049072266,
      "activations/layer3_attention_weight_min": -60.005802154541016,
      "activations/layer4_attention_weight_max": 85.6104736328125,
      "activations/layer4_attention_weight_min": -82.01924133300781,
      "activations/layer5_attention_weight_max": 133.32485961914062,
      "activations/layer5_attention_weight_min": -121.75609588623047,
      "activations/layer6_attention_weight_max": 63.141517639160156,
      "activations/layer6_attention_weight_min": -54.90934371948242,
      "activations/layer7_attention_weight_max": 60.597782135009766,
      "activations/layer7_attention_weight_min": -57.24274444580078,
      "activations/layer8_attention_weight_max": 50.895355224609375,
      "activations/layer8_attention_weight_min": -45.47664260864258,
      "activations/layer9_attention_weight_max": 78.98554229736328,
      "activations/layer9_attention_weight_min": -69.79007720947266,
      "epoch": 1.36,
      "learning_rate": 0.00014266515151515152,
      "loss": 3.0487,
      "step": 23400
    },
    {
      "activations/layer0_attention_weight_max": 14.004678726196289,
      "activations/layer0_attention_weight_min": -12.976529121398926,
      "activations/layer10_attention_weight_max": 62.779144287109375,
      "activations/layer10_attention_weight_min": -58.31421661376953,
      "activations/layer11_attention_weight_max": 41.117733001708984,
      "activations/layer11_attention_weight_min": -36.13030242919922,
      "activations/layer12_attention_weight_max": 29.345638275146484,
      "activations/layer12_attention_weight_min": -25.122554779052734,
      "activations/layer13_attention_weight_max": 56.24032974243164,
      "activations/layer13_attention_weight_min": -45.167198181152344,
      "activations/layer14_attention_weight_max": 59.9366455078125,
      "activations/layer14_attention_weight_min": -43.912864685058594,
      "activations/layer15_attention_weight_max": 57.02347183227539,
      "activations/layer15_attention_weight_min": -41.22062683105469,
      "activations/layer16_attention_weight_max": 45.503658294677734,
      "activations/layer16_attention_weight_min": -31.77423858642578,
      "activations/layer17_attention_weight_max": 60.88053512573242,
      "activations/layer17_attention_weight_min": -36.2311897277832,
      "activations/layer18_attention_weight_max": 58.0794792175293,
      "activations/layer18_attention_weight_min": -34.21015548706055,
      "activations/layer19_attention_weight_max": 22.36733055114746,
      "activations/layer19_attention_weight_min": -14.755180358886719,
      "activations/layer1_attention_weight_max": 15.909430503845215,
      "activations/layer1_attention_weight_min": -11.499987602233887,
      "activations/layer20_attention_weight_max": 20.043851852416992,
      "activations/layer20_attention_weight_min": -14.544988632202148,
      "activations/layer21_attention_weight_max": 39.53662109375,
      "activations/layer21_attention_weight_min": -21.602859497070312,
      "activations/layer22_attention_weight_max": 38.1268310546875,
      "activations/layer22_attention_weight_min": -19.068540573120117,
      "activations/layer23_attention_weight_max": 25.749835968017578,
      "activations/layer23_attention_weight_min": -16.368751525878906,
      "activations/layer2_attention_weight_max": 24.683963775634766,
      "activations/layer2_attention_weight_min": -25.185531616210938,
      "activations/layer3_attention_weight_max": 57.502506256103516,
      "activations/layer3_attention_weight_min": -59.30949783325195,
      "activations/layer4_attention_weight_max": 84.1732177734375,
      "activations/layer4_attention_weight_min": -78.2398910522461,
      "activations/layer5_attention_weight_max": 134.89651489257812,
      "activations/layer5_attention_weight_min": -115.77801513671875,
      "activations/layer6_attention_weight_max": 64.35157775878906,
      "activations/layer6_attention_weight_min": -56.35432815551758,
      "activations/layer7_attention_weight_max": 66.27778625488281,
      "activations/layer7_attention_weight_min": -55.952701568603516,
      "activations/layer8_attention_weight_max": 51.28094482421875,
      "activations/layer8_attention_weight_min": -47.59077835083008,
      "activations/layer9_attention_weight_max": 70.2711181640625,
      "activations/layer9_attention_weight_min": -70.63543701171875,
      "epoch": 1.36,
      "learning_rate": 0.0001426465909090909,
      "loss": 3.0253,
      "step": 23450
    },
    {
      "activations/layer0_attention_weight_max": 14.10139274597168,
      "activations/layer0_attention_weight_min": -12.547613143920898,
      "activations/layer10_attention_weight_max": 64.71939086914062,
      "activations/layer10_attention_weight_min": -54.186126708984375,
      "activations/layer11_attention_weight_max": 45.17829895019531,
      "activations/layer11_attention_weight_min": -38.97252655029297,
      "activations/layer12_attention_weight_max": 26.849136352539062,
      "activations/layer12_attention_weight_min": -25.25865936279297,
      "activations/layer13_attention_weight_max": 68.14566040039062,
      "activations/layer13_attention_weight_min": -46.578861236572266,
      "activations/layer14_attention_weight_max": 66.70660400390625,
      "activations/layer14_attention_weight_min": -40.99637222290039,
      "activations/layer15_attention_weight_max": 50.735164642333984,
      "activations/layer15_attention_weight_min": -36.23583221435547,
      "activations/layer16_attention_weight_max": 48.48177719116211,
      "activations/layer16_attention_weight_min": -34.669132232666016,
      "activations/layer17_attention_weight_max": 60.66288375854492,
      "activations/layer17_attention_weight_min": -38.068904876708984,
      "activations/layer18_attention_weight_max": 64.58915710449219,
      "activations/layer18_attention_weight_min": -32.59674835205078,
      "activations/layer19_attention_weight_max": 20.78499412536621,
      "activations/layer19_attention_weight_min": -14.95312786102295,
      "activations/layer1_attention_weight_max": 15.377700805664062,
      "activations/layer1_attention_weight_min": -12.838366508483887,
      "activations/layer20_attention_weight_max": 21.666296005249023,
      "activations/layer20_attention_weight_min": -14.38787841796875,
      "activations/layer21_attention_weight_max": 39.050140380859375,
      "activations/layer21_attention_weight_min": -19.758323669433594,
      "activations/layer22_attention_weight_max": 38.44379425048828,
      "activations/layer22_attention_weight_min": -19.742725372314453,
      "activations/layer23_attention_weight_max": 23.381032943725586,
      "activations/layer23_attention_weight_min": -13.613540649414062,
      "activations/layer2_attention_weight_max": 25.148876190185547,
      "activations/layer2_attention_weight_min": -24.121673583984375,
      "activations/layer3_attention_weight_max": 57.01732635498047,
      "activations/layer3_attention_weight_min": -59.50264358520508,
      "activations/layer4_attention_weight_max": 86.3171615600586,
      "activations/layer4_attention_weight_min": -79.80992126464844,
      "activations/layer5_attention_weight_max": 128.11380004882812,
      "activations/layer5_attention_weight_min": -119.52068328857422,
      "activations/layer6_attention_weight_max": 61.60968017578125,
      "activations/layer6_attention_weight_min": -52.385398864746094,
      "activations/layer7_attention_weight_max": 62.77032470703125,
      "activations/layer7_attention_weight_min": -52.61501693725586,
      "activations/layer8_attention_weight_max": 50.56926345825195,
      "activations/layer8_attention_weight_min": -42.203590393066406,
      "activations/layer9_attention_weight_max": 65.61490631103516,
      "activations/layer9_attention_weight_min": -61.037288665771484,
      "epoch": 1.37,
      "learning_rate": 0.0001426276515151515,
      "loss": 3.0318,
      "step": 23500
    },
    {
      "activations/layer0_attention_weight_max": 16.715673446655273,
      "activations/layer0_attention_weight_min": -12.686258316040039,
      "activations/layer10_attention_weight_max": 62.4840087890625,
      "activations/layer10_attention_weight_min": -55.79637145996094,
      "activations/layer11_attention_weight_max": 41.264225006103516,
      "activations/layer11_attention_weight_min": -38.084564208984375,
      "activations/layer12_attention_weight_max": 29.719581604003906,
      "activations/layer12_attention_weight_min": -25.050153732299805,
      "activations/layer13_attention_weight_max": 52.0039176940918,
      "activations/layer13_attention_weight_min": -41.55463790893555,
      "activations/layer14_attention_weight_max": 49.50185012817383,
      "activations/layer14_attention_weight_min": -41.4449348449707,
      "activations/layer15_attention_weight_max": 56.272945404052734,
      "activations/layer15_attention_weight_min": -39.02503204345703,
      "activations/layer16_attention_weight_max": 43.70102310180664,
      "activations/layer16_attention_weight_min": -31.97798728942871,
      "activations/layer17_attention_weight_max": 56.8226203918457,
      "activations/layer17_attention_weight_min": -40.405029296875,
      "activations/layer18_attention_weight_max": 60.13013458251953,
      "activations/layer18_attention_weight_min": -37.306617736816406,
      "activations/layer19_attention_weight_max": 21.238431930541992,
      "activations/layer19_attention_weight_min": -14.535809516906738,
      "activations/layer1_attention_weight_max": 15.541414260864258,
      "activations/layer1_attention_weight_min": -13.329880714416504,
      "activations/layer20_attention_weight_max": 19.035564422607422,
      "activations/layer20_attention_weight_min": -13.948973655700684,
      "activations/layer21_attention_weight_max": 39.33009719848633,
      "activations/layer21_attention_weight_min": -21.93558120727539,
      "activations/layer22_attention_weight_max": 37.91156768798828,
      "activations/layer22_attention_weight_min": -20.86016845703125,
      "activations/layer23_attention_weight_max": 23.954200744628906,
      "activations/layer23_attention_weight_min": -14.813152313232422,
      "activations/layer2_attention_weight_max": 24.743175506591797,
      "activations/layer2_attention_weight_min": -23.443782806396484,
      "activations/layer3_attention_weight_max": 53.342384338378906,
      "activations/layer3_attention_weight_min": -57.28794479370117,
      "activations/layer4_attention_weight_max": 82.912353515625,
      "activations/layer4_attention_weight_min": -74.79374694824219,
      "activations/layer5_attention_weight_max": 124.99345397949219,
      "activations/layer5_attention_weight_min": -117.82623291015625,
      "activations/layer6_attention_weight_max": 60.76179122924805,
      "activations/layer6_attention_weight_min": -54.69089126586914,
      "activations/layer7_attention_weight_max": 59.0416374206543,
      "activations/layer7_attention_weight_min": -54.44765853881836,
      "activations/layer8_attention_weight_max": 47.06105422973633,
      "activations/layer8_attention_weight_min": -44.134063720703125,
      "activations/layer9_attention_weight_max": 65.32976531982422,
      "activations/layer9_attention_weight_min": -64.85795593261719,
      "epoch": 1.37,
      "learning_rate": 0.0001426087121212121,
      "loss": 3.0313,
      "step": 23550
    },
    {
      "activations/layer0_attention_weight_max": 14.76528263092041,
      "activations/layer0_attention_weight_min": -12.934266090393066,
      "activations/layer10_attention_weight_max": 65.40409088134766,
      "activations/layer10_attention_weight_min": -54.18915557861328,
      "activations/layer11_attention_weight_max": 41.66804885864258,
      "activations/layer11_attention_weight_min": -37.336883544921875,
      "activations/layer12_attention_weight_max": 26.336360931396484,
      "activations/layer12_attention_weight_min": -24.7924861907959,
      "activations/layer13_attention_weight_max": 59.040008544921875,
      "activations/layer13_attention_weight_min": -56.6291618347168,
      "activations/layer14_attention_weight_max": 60.43597412109375,
      "activations/layer14_attention_weight_min": -40.22758483886719,
      "activations/layer15_attention_weight_max": 48.95061111450195,
      "activations/layer15_attention_weight_min": -42.083465576171875,
      "activations/layer16_attention_weight_max": 39.73174285888672,
      "activations/layer16_attention_weight_min": -30.029680252075195,
      "activations/layer17_attention_weight_max": 52.68252182006836,
      "activations/layer17_attention_weight_min": -36.67637252807617,
      "activations/layer18_attention_weight_max": 58.350379943847656,
      "activations/layer18_attention_weight_min": -39.17571258544922,
      "activations/layer19_attention_weight_max": 17.18804359436035,
      "activations/layer19_attention_weight_min": -14.843785285949707,
      "activations/layer1_attention_weight_max": 15.340466499328613,
      "activations/layer1_attention_weight_min": -11.533411026000977,
      "activations/layer20_attention_weight_max": 18.86161994934082,
      "activations/layer20_attention_weight_min": -14.687478065490723,
      "activations/layer21_attention_weight_max": 32.72108459472656,
      "activations/layer21_attention_weight_min": -19.932880401611328,
      "activations/layer22_attention_weight_max": 36.92802047729492,
      "activations/layer22_attention_weight_min": -20.78675079345703,
      "activations/layer23_attention_weight_max": 23.157184600830078,
      "activations/layer23_attention_weight_min": -13.848750114440918,
      "activations/layer2_attention_weight_max": 26.202194213867188,
      "activations/layer2_attention_weight_min": -24.467613220214844,
      "activations/layer3_attention_weight_max": 54.56755447387695,
      "activations/layer3_attention_weight_min": -58.3933219909668,
      "activations/layer4_attention_weight_max": 82.03761291503906,
      "activations/layer4_attention_weight_min": -75.66756439208984,
      "activations/layer5_attention_weight_max": 126.98344421386719,
      "activations/layer5_attention_weight_min": -116.43209075927734,
      "activations/layer6_attention_weight_max": 60.696617126464844,
      "activations/layer6_attention_weight_min": -60.26314163208008,
      "activations/layer7_attention_weight_max": 57.89333724975586,
      "activations/layer7_attention_weight_min": -54.5782470703125,
      "activations/layer8_attention_weight_max": 48.51845169067383,
      "activations/layer8_attention_weight_min": -44.79653549194336,
      "activations/layer9_attention_weight_max": 61.0345573425293,
      "activations/layer9_attention_weight_min": -63.205379486083984,
      "epoch": 1.37,
      "learning_rate": 0.00014258977272727273,
      "loss": 3.0504,
      "step": 23600
    },
    {
      "activations/layer0_attention_weight_max": 14.222813606262207,
      "activations/layer0_attention_weight_min": -12.966419219970703,
      "activations/layer10_attention_weight_max": 64.59559631347656,
      "activations/layer10_attention_weight_min": -54.554649353027344,
      "activations/layer11_attention_weight_max": 48.60561752319336,
      "activations/layer11_attention_weight_min": -39.35289764404297,
      "activations/layer12_attention_weight_max": 26.958925247192383,
      "activations/layer12_attention_weight_min": -26.726688385009766,
      "activations/layer13_attention_weight_max": 71.35800170898438,
      "activations/layer13_attention_weight_min": -45.01213073730469,
      "activations/layer14_attention_weight_max": 61.92015838623047,
      "activations/layer14_attention_weight_min": -42.442230224609375,
      "activations/layer15_attention_weight_max": 65.88933563232422,
      "activations/layer15_attention_weight_min": -37.27159118652344,
      "activations/layer16_attention_weight_max": 49.268367767333984,
      "activations/layer16_attention_weight_min": -32.27903747558594,
      "activations/layer17_attention_weight_max": 67.84259796142578,
      "activations/layer17_attention_weight_min": -34.168800354003906,
      "activations/layer18_attention_weight_max": 70.64964294433594,
      "activations/layer18_attention_weight_min": -32.1037483215332,
      "activations/layer19_attention_weight_max": 21.15842056274414,
      "activations/layer19_attention_weight_min": -14.29705810546875,
      "activations/layer1_attention_weight_max": 15.327452659606934,
      "activations/layer1_attention_weight_min": -11.436623573303223,
      "activations/layer20_attention_weight_max": 19.9649658203125,
      "activations/layer20_attention_weight_min": -15.211050987243652,
      "activations/layer21_attention_weight_max": 41.0243034362793,
      "activations/layer21_attention_weight_min": -22.508459091186523,
      "activations/layer22_attention_weight_max": 39.83116149902344,
      "activations/layer22_attention_weight_min": -19.255908966064453,
      "activations/layer23_attention_weight_max": 25.14600944519043,
      "activations/layer23_attention_weight_min": -18.953529357910156,
      "activations/layer2_attention_weight_max": 26.22041893005371,
      "activations/layer2_attention_weight_min": -24.09370994567871,
      "activations/layer3_attention_weight_max": 56.59078598022461,
      "activations/layer3_attention_weight_min": -58.64284133911133,
      "activations/layer4_attention_weight_max": 86.73912811279297,
      "activations/layer4_attention_weight_min": -77.81505584716797,
      "activations/layer5_attention_weight_max": 138.98837280273438,
      "activations/layer5_attention_weight_min": -115.5266342163086,
      "activations/layer6_attention_weight_max": 66.12437438964844,
      "activations/layer6_attention_weight_min": -55.39060974121094,
      "activations/layer7_attention_weight_max": 63.4748649597168,
      "activations/layer7_attention_weight_min": -54.26871871948242,
      "activations/layer8_attention_weight_max": 55.3655891418457,
      "activations/layer8_attention_weight_min": -50.39741897583008,
      "activations/layer9_attention_weight_max": 73.2129898071289,
      "activations/layer9_attention_weight_min": -66.17281341552734,
      "epoch": 1.37,
      "learning_rate": 0.00014257083333333334,
      "loss": 3.0329,
      "step": 23650
    },
    {
      "activations/layer0_attention_weight_max": 13.932708740234375,
      "activations/layer0_attention_weight_min": -12.8640775680542,
      "activations/layer10_attention_weight_max": 72.30823516845703,
      "activations/layer10_attention_weight_min": -67.55599975585938,
      "activations/layer11_attention_weight_max": 44.988765716552734,
      "activations/layer11_attention_weight_min": -37.150699615478516,
      "activations/layer12_attention_weight_max": 30.177566528320312,
      "activations/layer12_attention_weight_min": -26.85190200805664,
      "activations/layer13_attention_weight_max": 64.41867065429688,
      "activations/layer13_attention_weight_min": -36.76827621459961,
      "activations/layer14_attention_weight_max": 85.68392944335938,
      "activations/layer14_attention_weight_min": -44.180389404296875,
      "activations/layer15_attention_weight_max": 64.46726989746094,
      "activations/layer15_attention_weight_min": -38.60581588745117,
      "activations/layer16_attention_weight_max": 49.806392669677734,
      "activations/layer16_attention_weight_min": -28.858980178833008,
      "activations/layer17_attention_weight_max": 58.30400466918945,
      "activations/layer17_attention_weight_min": -34.05668640136719,
      "activations/layer18_attention_weight_max": 64.3855972290039,
      "activations/layer18_attention_weight_min": -35.29143142700195,
      "activations/layer19_attention_weight_max": 24.568422317504883,
      "activations/layer19_attention_weight_min": -13.866948127746582,
      "activations/layer1_attention_weight_max": 15.853649139404297,
      "activations/layer1_attention_weight_min": -10.836484909057617,
      "activations/layer20_attention_weight_max": 20.442710876464844,
      "activations/layer20_attention_weight_min": -16.057743072509766,
      "activations/layer21_attention_weight_max": 38.145477294921875,
      "activations/layer21_attention_weight_min": -22.597803115844727,
      "activations/layer22_attention_weight_max": 41.80619812011719,
      "activations/layer22_attention_weight_min": -19.578704833984375,
      "activations/layer23_attention_weight_max": 24.717208862304688,
      "activations/layer23_attention_weight_min": -14.576324462890625,
      "activations/layer2_attention_weight_max": 25.458608627319336,
      "activations/layer2_attention_weight_min": -24.607372283935547,
      "activations/layer3_attention_weight_max": 59.18793487548828,
      "activations/layer3_attention_weight_min": -60.90495300292969,
      "activations/layer4_attention_weight_max": 88.53856658935547,
      "activations/layer4_attention_weight_min": -83.00614929199219,
      "activations/layer5_attention_weight_max": 142.26312255859375,
      "activations/layer5_attention_weight_min": -118.10060119628906,
      "activations/layer6_attention_weight_max": 68.778564453125,
      "activations/layer6_attention_weight_min": -57.22450256347656,
      "activations/layer7_attention_weight_max": 65.97013092041016,
      "activations/layer7_attention_weight_min": -59.96830749511719,
      "activations/layer8_attention_weight_max": 55.21458435058594,
      "activations/layer8_attention_weight_min": -45.0066032409668,
      "activations/layer9_attention_weight_max": 75.33902740478516,
      "activations/layer9_attention_weight_min": -66.6766586303711,
      "epoch": 1.38,
      "learning_rate": 0.00014255189393939393,
      "loss": 3.0107,
      "step": 23700
    },
    {
      "activations/layer0_attention_weight_max": 13.498669624328613,
      "activations/layer0_attention_weight_min": -12.758362770080566,
      "activations/layer10_attention_weight_max": 76.61433410644531,
      "activations/layer10_attention_weight_min": -68.73541259765625,
      "activations/layer11_attention_weight_max": 44.816322326660156,
      "activations/layer11_attention_weight_min": -38.304447174072266,
      "activations/layer12_attention_weight_max": 30.29279899597168,
      "activations/layer12_attention_weight_min": -22.749164581298828,
      "activations/layer13_attention_weight_max": 62.383705139160156,
      "activations/layer13_attention_weight_min": -43.64664840698242,
      "activations/layer14_attention_weight_max": 54.07908248901367,
      "activations/layer14_attention_weight_min": -37.64284133911133,
      "activations/layer15_attention_weight_max": 58.797245025634766,
      "activations/layer15_attention_weight_min": -38.206485748291016,
      "activations/layer16_attention_weight_max": 46.99072265625,
      "activations/layer16_attention_weight_min": -30.990856170654297,
      "activations/layer17_attention_weight_max": 58.06026840209961,
      "activations/layer17_attention_weight_min": -35.614315032958984,
      "activations/layer18_attention_weight_max": 61.6181640625,
      "activations/layer18_attention_weight_min": -36.0948486328125,
      "activations/layer19_attention_weight_max": 21.3609676361084,
      "activations/layer19_attention_weight_min": -15.089198112487793,
      "activations/layer1_attention_weight_max": 15.348508834838867,
      "activations/layer1_attention_weight_min": -11.456696510314941,
      "activations/layer20_attention_weight_max": 21.025175094604492,
      "activations/layer20_attention_weight_min": -16.623830795288086,
      "activations/layer21_attention_weight_max": 37.62353515625,
      "activations/layer21_attention_weight_min": -21.220335006713867,
      "activations/layer22_attention_weight_max": 34.825950622558594,
      "activations/layer22_attention_weight_min": -22.48806381225586,
      "activations/layer23_attention_weight_max": 23.521747589111328,
      "activations/layer23_attention_weight_min": -13.34480094909668,
      "activations/layer2_attention_weight_max": 25.919971466064453,
      "activations/layer2_attention_weight_min": -24.733640670776367,
      "activations/layer3_attention_weight_max": 56.313045501708984,
      "activations/layer3_attention_weight_min": -60.07229232788086,
      "activations/layer4_attention_weight_max": 86.19001770019531,
      "activations/layer4_attention_weight_min": -82.30305480957031,
      "activations/layer5_attention_weight_max": 126.80464935302734,
      "activations/layer5_attention_weight_min": -120.11940002441406,
      "activations/layer6_attention_weight_max": 63.845767974853516,
      "activations/layer6_attention_weight_min": -53.43315124511719,
      "activations/layer7_attention_weight_max": 64.20162963867188,
      "activations/layer7_attention_weight_min": -55.994930267333984,
      "activations/layer8_attention_weight_max": 54.10961151123047,
      "activations/layer8_attention_weight_min": -47.9332389831543,
      "activations/layer9_attention_weight_max": 66.82179260253906,
      "activations/layer9_attention_weight_min": -61.560943603515625,
      "epoch": 1.38,
      "learning_rate": 0.00014253295454545454,
      "loss": 3.0321,
      "step": 23750
    },
    {
      "activations/layer0_attention_weight_max": 16.608909606933594,
      "activations/layer0_attention_weight_min": -13.174860000610352,
      "activations/layer10_attention_weight_max": 61.67194747924805,
      "activations/layer10_attention_weight_min": -53.02292251586914,
      "activations/layer11_attention_weight_max": 48.97623825073242,
      "activations/layer11_attention_weight_min": -40.551734924316406,
      "activations/layer12_attention_weight_max": 24.325096130371094,
      "activations/layer12_attention_weight_min": -26.55590057373047,
      "activations/layer13_attention_weight_max": 49.30338668823242,
      "activations/layer13_attention_weight_min": -46.17813491821289,
      "activations/layer14_attention_weight_max": 52.851104736328125,
      "activations/layer14_attention_weight_min": -41.3535041809082,
      "activations/layer15_attention_weight_max": 50.474464416503906,
      "activations/layer15_attention_weight_min": -41.60285568237305,
      "activations/layer16_attention_weight_max": 46.16521072387695,
      "activations/layer16_attention_weight_min": -33.952911376953125,
      "activations/layer17_attention_weight_max": 53.71389389038086,
      "activations/layer17_attention_weight_min": -37.537357330322266,
      "activations/layer18_attention_weight_max": 55.94619369506836,
      "activations/layer18_attention_weight_min": -35.497920989990234,
      "activations/layer19_attention_weight_max": 18.09162139892578,
      "activations/layer19_attention_weight_min": -14.129237174987793,
      "activations/layer1_attention_weight_max": 15.815162658691406,
      "activations/layer1_attention_weight_min": -10.511439323425293,
      "activations/layer20_attention_weight_max": 18.257854461669922,
      "activations/layer20_attention_weight_min": -14.467730522155762,
      "activations/layer21_attention_weight_max": 34.50439453125,
      "activations/layer21_attention_weight_min": -19.500900268554688,
      "activations/layer22_attention_weight_max": 37.45524215698242,
      "activations/layer22_attention_weight_min": -19.03105926513672,
      "activations/layer23_attention_weight_max": 21.402006149291992,
      "activations/layer23_attention_weight_min": -13.742476463317871,
      "activations/layer2_attention_weight_max": 23.89893341064453,
      "activations/layer2_attention_weight_min": -23.322494506835938,
      "activations/layer3_attention_weight_max": 53.957279205322266,
      "activations/layer3_attention_weight_min": -56.12663269042969,
      "activations/layer4_attention_weight_max": 80.98602294921875,
      "activations/layer4_attention_weight_min": -76.39051055908203,
      "activations/layer5_attention_weight_max": 126.47637176513672,
      "activations/layer5_attention_weight_min": -116.39507293701172,
      "activations/layer6_attention_weight_max": 59.91282272338867,
      "activations/layer6_attention_weight_min": -59.68321228027344,
      "activations/layer7_attention_weight_max": 64.55146789550781,
      "activations/layer7_attention_weight_min": -56.461082458496094,
      "activations/layer8_attention_weight_max": 57.09146499633789,
      "activations/layer8_attention_weight_min": -45.71159362792969,
      "activations/layer9_attention_weight_max": 76.33406829833984,
      "activations/layer9_attention_weight_min": -66.57196044921875,
      "epoch": 1.38,
      "learning_rate": 0.00014251401515151513,
      "loss": 3.0297,
      "step": 23800
    },
    {
      "activations/layer0_attention_weight_max": 13.406943321228027,
      "activations/layer0_attention_weight_min": -13.442078590393066,
      "activations/layer10_attention_weight_max": 70.23954010009766,
      "activations/layer10_attention_weight_min": -52.06742858886719,
      "activations/layer11_attention_weight_max": 52.39155197143555,
      "activations/layer11_attention_weight_min": -35.21332550048828,
      "activations/layer12_attention_weight_max": 30.134559631347656,
      "activations/layer12_attention_weight_min": -27.92032241821289,
      "activations/layer13_attention_weight_max": 96.59745025634766,
      "activations/layer13_attention_weight_min": -47.67210388183594,
      "activations/layer14_attention_weight_max": 98.15019989013672,
      "activations/layer14_attention_weight_min": -49.00587844848633,
      "activations/layer15_attention_weight_max": 59.205562591552734,
      "activations/layer15_attention_weight_min": -36.03534698486328,
      "activations/layer16_attention_weight_max": 58.77903747558594,
      "activations/layer16_attention_weight_min": -33.68167495727539,
      "activations/layer17_attention_weight_max": 68.37812805175781,
      "activations/layer17_attention_weight_min": -39.069393157958984,
      "activations/layer18_attention_weight_max": 71.05362701416016,
      "activations/layer18_attention_weight_min": -34.710567474365234,
      "activations/layer19_attention_weight_max": 23.519878387451172,
      "activations/layer19_attention_weight_min": -14.168420791625977,
      "activations/layer1_attention_weight_max": 16.304283142089844,
      "activations/layer1_attention_weight_min": -12.805879592895508,
      "activations/layer20_attention_weight_max": 20.06851577758789,
      "activations/layer20_attention_weight_min": -15.918789863586426,
      "activations/layer21_attention_weight_max": 38.299102783203125,
      "activations/layer21_attention_weight_min": -18.8873233795166,
      "activations/layer22_attention_weight_max": 39.77247619628906,
      "activations/layer22_attention_weight_min": -20.41211700439453,
      "activations/layer23_attention_weight_max": 23.459793090820312,
      "activations/layer23_attention_weight_min": -12.597070693969727,
      "activations/layer2_attention_weight_max": 28.146347045898438,
      "activations/layer2_attention_weight_min": -23.751501083374023,
      "activations/layer3_attention_weight_max": 60.899696350097656,
      "activations/layer3_attention_weight_min": -59.491432189941406,
      "activations/layer4_attention_weight_max": 97.60688781738281,
      "activations/layer4_attention_weight_min": -80.00794982910156,
      "activations/layer5_attention_weight_max": 147.98104858398438,
      "activations/layer5_attention_weight_min": -121.72542572021484,
      "activations/layer6_attention_weight_max": 68.8354721069336,
      "activations/layer6_attention_weight_min": -58.267547607421875,
      "activations/layer7_attention_weight_max": 73.14214324951172,
      "activations/layer7_attention_weight_min": -55.574913024902344,
      "activations/layer8_attention_weight_max": 66.24456024169922,
      "activations/layer8_attention_weight_min": -46.146236419677734,
      "activations/layer9_attention_weight_max": 93.73116302490234,
      "activations/layer9_attention_weight_min": -71.67579650878906,
      "epoch": 1.39,
      "learning_rate": 0.00014249507575757575,
      "loss": 3.0267,
      "step": 23850
    },
    {
      "activations/layer0_attention_weight_max": 13.173869132995605,
      "activations/layer0_attention_weight_min": -13.293217658996582,
      "activations/layer10_attention_weight_max": 62.637237548828125,
      "activations/layer10_attention_weight_min": -56.50628662109375,
      "activations/layer11_attention_weight_max": 42.38984680175781,
      "activations/layer11_attention_weight_min": -35.287025451660156,
      "activations/layer12_attention_weight_max": 25.82673454284668,
      "activations/layer12_attention_weight_min": -24.34983253479004,
      "activations/layer13_attention_weight_max": 78.69689178466797,
      "activations/layer13_attention_weight_min": -51.81946563720703,
      "activations/layer14_attention_weight_max": 82.03936767578125,
      "activations/layer14_attention_weight_min": -46.07238006591797,
      "activations/layer15_attention_weight_max": 60.55498123168945,
      "activations/layer15_attention_weight_min": -45.92991256713867,
      "activations/layer16_attention_weight_max": 53.5793571472168,
      "activations/layer16_attention_weight_min": -32.833404541015625,
      "activations/layer17_attention_weight_max": 63.61856460571289,
      "activations/layer17_attention_weight_min": -35.79556655883789,
      "activations/layer18_attention_weight_max": 69.5062255859375,
      "activations/layer18_attention_weight_min": -40.53201675415039,
      "activations/layer19_attention_weight_max": 23.119943618774414,
      "activations/layer19_attention_weight_min": -14.931933403015137,
      "activations/layer1_attention_weight_max": 15.341424942016602,
      "activations/layer1_attention_weight_min": -14.859139442443848,
      "activations/layer20_attention_weight_max": 19.874980926513672,
      "activations/layer20_attention_weight_min": -13.992860794067383,
      "activations/layer21_attention_weight_max": 38.840736389160156,
      "activations/layer21_attention_weight_min": -19.1988582611084,
      "activations/layer22_attention_weight_max": 41.915042877197266,
      "activations/layer22_attention_weight_min": -22.557689666748047,
      "activations/layer23_attention_weight_max": 21.390554428100586,
      "activations/layer23_attention_weight_min": -13.241450309753418,
      "activations/layer2_attention_weight_max": 27.86241912841797,
      "activations/layer2_attention_weight_min": -25.173301696777344,
      "activations/layer3_attention_weight_max": 55.21158218383789,
      "activations/layer3_attention_weight_min": -57.219818115234375,
      "activations/layer4_attention_weight_max": 86.48355102539062,
      "activations/layer4_attention_weight_min": -77.60000610351562,
      "activations/layer5_attention_weight_max": 125.41938781738281,
      "activations/layer5_attention_weight_min": -115.27062225341797,
      "activations/layer6_attention_weight_max": 59.65521240234375,
      "activations/layer6_attention_weight_min": -56.00606155395508,
      "activations/layer7_attention_weight_max": 57.0567741394043,
      "activations/layer7_attention_weight_min": -55.2895622253418,
      "activations/layer8_attention_weight_max": 47.2989387512207,
      "activations/layer8_attention_weight_min": -44.34467697143555,
      "activations/layer9_attention_weight_max": 67.09065246582031,
      "activations/layer9_attention_weight_min": -62.16398239135742,
      "epoch": 1.39,
      "learning_rate": 0.00014247613636363636,
      "loss": 3.0123,
      "step": 23900
    },
    {
      "activations/layer0_attention_weight_max": 14.116888999938965,
      "activations/layer0_attention_weight_min": -12.855780601501465,
      "activations/layer10_attention_weight_max": 61.3404541015625,
      "activations/layer10_attention_weight_min": -53.812744140625,
      "activations/layer11_attention_weight_max": 43.79580307006836,
      "activations/layer11_attention_weight_min": -38.20041275024414,
      "activations/layer12_attention_weight_max": 31.95402717590332,
      "activations/layer12_attention_weight_min": -24.27372169494629,
      "activations/layer13_attention_weight_max": 69.94434356689453,
      "activations/layer13_attention_weight_min": -54.599124908447266,
      "activations/layer14_attention_weight_max": 57.70696258544922,
      "activations/layer14_attention_weight_min": -45.931190490722656,
      "activations/layer15_attention_weight_max": 52.73125076293945,
      "activations/layer15_attention_weight_min": -42.11075210571289,
      "activations/layer16_attention_weight_max": 45.66529846191406,
      "activations/layer16_attention_weight_min": -33.0638427734375,
      "activations/layer17_attention_weight_max": 52.83474349975586,
      "activations/layer17_attention_weight_min": -36.14667892456055,
      "activations/layer18_attention_weight_max": 56.233642578125,
      "activations/layer18_attention_weight_min": -34.819271087646484,
      "activations/layer19_attention_weight_max": 21.482093811035156,
      "activations/layer19_attention_weight_min": -14.099286079406738,
      "activations/layer1_attention_weight_max": 14.797094345092773,
      "activations/layer1_attention_weight_min": -14.886414527893066,
      "activations/layer20_attention_weight_max": 20.30269432067871,
      "activations/layer20_attention_weight_min": -16.941726684570312,
      "activations/layer21_attention_weight_max": 36.09663391113281,
      "activations/layer21_attention_weight_min": -21.009212493896484,
      "activations/layer22_attention_weight_max": 33.17718505859375,
      "activations/layer22_attention_weight_min": -19.32111358642578,
      "activations/layer23_attention_weight_max": 23.737503051757812,
      "activations/layer23_attention_weight_min": -13.916586875915527,
      "activations/layer2_attention_weight_max": 25.185850143432617,
      "activations/layer2_attention_weight_min": -24.44312286376953,
      "activations/layer3_attention_weight_max": 56.26747512817383,
      "activations/layer3_attention_weight_min": -58.00811004638672,
      "activations/layer4_attention_weight_max": 82.95178985595703,
      "activations/layer4_attention_weight_min": -81.27025604248047,
      "activations/layer5_attention_weight_max": 126.88380432128906,
      "activations/layer5_attention_weight_min": -117.82510375976562,
      "activations/layer6_attention_weight_max": 61.24397277832031,
      "activations/layer6_attention_weight_min": -55.78264617919922,
      "activations/layer7_attention_weight_max": 59.51052474975586,
      "activations/layer7_attention_weight_min": -55.86021423339844,
      "activations/layer8_attention_weight_max": 50.512203216552734,
      "activations/layer8_attention_weight_min": -46.108131408691406,
      "activations/layer9_attention_weight_max": 69.40896606445312,
      "activations/layer9_attention_weight_min": -69.95174407958984,
      "epoch": 1.39,
      "learning_rate": 0.00014245719696969695,
      "loss": 3.0276,
      "step": 23950
    },
    {
      "activations/layer0_attention_weight_max": 13.651926040649414,
      "activations/layer0_attention_weight_min": -12.575019836425781,
      "activations/layer10_attention_weight_max": 69.86634826660156,
      "activations/layer10_attention_weight_min": -53.40596389770508,
      "activations/layer11_attention_weight_max": 49.884765625,
      "activations/layer11_attention_weight_min": -38.621978759765625,
      "activations/layer12_attention_weight_max": 29.666553497314453,
      "activations/layer12_attention_weight_min": -26.552579879760742,
      "activations/layer13_attention_weight_max": 76.73113250732422,
      "activations/layer13_attention_weight_min": -50.04191970825195,
      "activations/layer14_attention_weight_max": 59.098663330078125,
      "activations/layer14_attention_weight_min": -38.73366165161133,
      "activations/layer15_attention_weight_max": 62.23147201538086,
      "activations/layer15_attention_weight_min": -39.53818893432617,
      "activations/layer16_attention_weight_max": 44.424198150634766,
      "activations/layer16_attention_weight_min": -32.3167610168457,
      "activations/layer17_attention_weight_max": 58.82280731201172,
      "activations/layer17_attention_weight_min": -37.201602935791016,
      "activations/layer18_attention_weight_max": 61.84804153442383,
      "activations/layer18_attention_weight_min": -33.85585403442383,
      "activations/layer19_attention_weight_max": 21.916675567626953,
      "activations/layer19_attention_weight_min": -14.43217945098877,
      "activations/layer1_attention_weight_max": 14.779528617858887,
      "activations/layer1_attention_weight_min": -10.96560287475586,
      "activations/layer20_attention_weight_max": 19.461626052856445,
      "activations/layer20_attention_weight_min": -14.729960441589355,
      "activations/layer21_attention_weight_max": 37.792484283447266,
      "activations/layer21_attention_weight_min": -19.94045066833496,
      "activations/layer22_attention_weight_max": 35.01078414916992,
      "activations/layer22_attention_weight_min": -20.55544090270996,
      "activations/layer23_attention_weight_max": 22.494766235351562,
      "activations/layer23_attention_weight_min": -15.439176559448242,
      "activations/layer2_attention_weight_max": 24.00713348388672,
      "activations/layer2_attention_weight_min": -22.90924072265625,
      "activations/layer3_attention_weight_max": 54.2017822265625,
      "activations/layer3_attention_weight_min": -57.75541305541992,
      "activations/layer4_attention_weight_max": 86.7450942993164,
      "activations/layer4_attention_weight_min": -80.03633880615234,
      "activations/layer5_attention_weight_max": 133.26821899414062,
      "activations/layer5_attention_weight_min": -113.95265197753906,
      "activations/layer6_attention_weight_max": 65.34282684326172,
      "activations/layer6_attention_weight_min": -54.865543365478516,
      "activations/layer7_attention_weight_max": 66.79714965820312,
      "activations/layer7_attention_weight_min": -54.24392318725586,
      "activations/layer8_attention_weight_max": 51.1934928894043,
      "activations/layer8_attention_weight_min": -45.707279205322266,
      "activations/layer9_attention_weight_max": 69.64665222167969,
      "activations/layer9_attention_weight_min": -59.4177360534668,
      "epoch": 1.39,
      "learning_rate": 0.00014243825757575756,
      "loss": 3.0323,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_loss": 2.9765625,
      "eval_runtime": 8.6955,
      "eval_samples_per_second": 493.818,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_openwebtext_loss": 2.9765625,
      "eval_openwebtext_ppl": 19.620255961497417,
      "eval_openwebtext_runtime": 8.6955,
      "eval_openwebtext_samples_per_second": 493.818,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_wikitext_loss": 3.255859375,
      "eval_wikitext_ppl": 25.94189877662824,
      "eval_wikitext_runtime": 1.7428,
      "eval_wikitext_samples_per_second": 261.653,
      "step": 24000
    },
    {
      "epoch": 1.39,
      "eval_lambada_loss": 3.21484375,
      "eval_lambada_ppl": 24.89940107578365,
      "eval_lambada_runtime": 9.3234,
      "eval_lambada_samples_per_second": 522.232,
      "step": 24000
    },
    {
      "activations/layer0_attention_weight_max": 13.50627613067627,
      "activations/layer0_attention_weight_min": -13.02876091003418,
      "activations/layer10_attention_weight_max": 58.46601104736328,
      "activations/layer10_attention_weight_min": -49.573387145996094,
      "activations/layer11_attention_weight_max": 44.33771514892578,
      "activations/layer11_attention_weight_min": -36.19060516357422,
      "activations/layer12_attention_weight_max": 27.364286422729492,
      "activations/layer12_attention_weight_min": -24.251312255859375,
      "activations/layer13_attention_weight_max": 57.916961669921875,
      "activations/layer13_attention_weight_min": -40.6379280090332,
      "activations/layer14_attention_weight_max": 40.45490264892578,
      "activations/layer14_attention_weight_min": -34.63916015625,
      "activations/layer15_attention_weight_max": 50.298439025878906,
      "activations/layer15_attention_weight_min": -40.20342254638672,
      "activations/layer16_attention_weight_max": 42.89522171020508,
      "activations/layer16_attention_weight_min": -33.820682525634766,
      "activations/layer17_attention_weight_max": 57.21974182128906,
      "activations/layer17_attention_weight_min": -36.53213882446289,
      "activations/layer18_attention_weight_max": 54.36826705932617,
      "activations/layer18_attention_weight_min": -36.113861083984375,
      "activations/layer19_attention_weight_max": 19.50050926208496,
      "activations/layer19_attention_weight_min": -13.649746894836426,
      "activations/layer1_attention_weight_max": 14.545933723449707,
      "activations/layer1_attention_weight_min": -10.180585861206055,
      "activations/layer20_attention_weight_max": 18.56270408630371,
      "activations/layer20_attention_weight_min": -16.774112701416016,
      "activations/layer21_attention_weight_max": 32.051422119140625,
      "activations/layer21_attention_weight_min": -19.505081176757812,
      "activations/layer22_attention_weight_max": 37.05276870727539,
      "activations/layer22_attention_weight_min": -22.9316463470459,
      "activations/layer23_attention_weight_max": 21.44799041748047,
      "activations/layer23_attention_weight_min": -14.746988296508789,
      "activations/layer2_attention_weight_max": 25.004344940185547,
      "activations/layer2_attention_weight_min": -24.99338150024414,
      "activations/layer3_attention_weight_max": 55.309268951416016,
      "activations/layer3_attention_weight_min": -59.18495178222656,
      "activations/layer4_attention_weight_max": 82.24462127685547,
      "activations/layer4_attention_weight_min": -80.36592102050781,
      "activations/layer5_attention_weight_max": 124.77069091796875,
      "activations/layer5_attention_weight_min": -125.22740936279297,
      "activations/layer6_attention_weight_max": 60.030311584472656,
      "activations/layer6_attention_weight_min": -57.467079162597656,
      "activations/layer7_attention_weight_max": 56.84857177734375,
      "activations/layer7_attention_weight_min": -54.01334762573242,
      "activations/layer8_attention_weight_max": 46.66019058227539,
      "activations/layer8_attention_weight_min": -44.450931549072266,
      "activations/layer9_attention_weight_max": 70.21121215820312,
      "activations/layer9_attention_weight_min": -61.97904586791992,
      "epoch": 1.4,
      "learning_rate": 0.00014241931818181818,
      "loss": 3.0322,
      "step": 24050
    },
    {
      "activations/layer0_attention_weight_max": 15.438766479492188,
      "activations/layer0_attention_weight_min": -13.274527549743652,
      "activations/layer10_attention_weight_max": 70.92910766601562,
      "activations/layer10_attention_weight_min": -56.361572265625,
      "activations/layer11_attention_weight_max": 45.61507797241211,
      "activations/layer11_attention_weight_min": -38.129608154296875,
      "activations/layer12_attention_weight_max": 27.59195327758789,
      "activations/layer12_attention_weight_min": -24.898956298828125,
      "activations/layer13_attention_weight_max": 62.39097213745117,
      "activations/layer13_attention_weight_min": -45.07606887817383,
      "activations/layer14_attention_weight_max": 48.60882568359375,
      "activations/layer14_attention_weight_min": -33.213043212890625,
      "activations/layer15_attention_weight_max": 58.29201889038086,
      "activations/layer15_attention_weight_min": -40.22248840332031,
      "activations/layer16_attention_weight_max": 45.20560073852539,
      "activations/layer16_attention_weight_min": -31.946178436279297,
      "activations/layer17_attention_weight_max": 56.3752326965332,
      "activations/layer17_attention_weight_min": -39.9312744140625,
      "activations/layer18_attention_weight_max": 63.03797149658203,
      "activations/layer18_attention_weight_min": -37.5763053894043,
      "activations/layer19_attention_weight_max": 20.91810417175293,
      "activations/layer19_attention_weight_min": -15.837862014770508,
      "activations/layer1_attention_weight_max": 16.198413848876953,
      "activations/layer1_attention_weight_min": -11.736656188964844,
      "activations/layer20_attention_weight_max": 18.5931339263916,
      "activations/layer20_attention_weight_min": -14.578044891357422,
      "activations/layer21_attention_weight_max": 33.776527404785156,
      "activations/layer21_attention_weight_min": -19.342952728271484,
      "activations/layer22_attention_weight_max": 34.147605895996094,
      "activations/layer22_attention_weight_min": -18.780908584594727,
      "activations/layer23_attention_weight_max": 22.82008934020996,
      "activations/layer23_attention_weight_min": -13.691164016723633,
      "activations/layer2_attention_weight_max": 25.606441497802734,
      "activations/layer2_attention_weight_min": -26.301528930664062,
      "activations/layer3_attention_weight_max": 56.61869812011719,
      "activations/layer3_attention_weight_min": -60.2701301574707,
      "activations/layer4_attention_weight_max": 85.6886978149414,
      "activations/layer4_attention_weight_min": -79.97350311279297,
      "activations/layer5_attention_weight_max": 137.4816436767578,
      "activations/layer5_attention_weight_min": -117.39260864257812,
      "activations/layer6_attention_weight_max": 65.28292083740234,
      "activations/layer6_attention_weight_min": -60.12272644042969,
      "activations/layer7_attention_weight_max": 68.78424835205078,
      "activations/layer7_attention_weight_min": -56.915924072265625,
      "activations/layer8_attention_weight_max": 56.725868225097656,
      "activations/layer8_attention_weight_min": -47.95780563354492,
      "activations/layer9_attention_weight_max": 71.16747283935547,
      "activations/layer9_attention_weight_min": -65.7123031616211,
      "epoch": 1.4,
      "learning_rate": 0.00014240037878787877,
      "loss": 3.0282,
      "step": 24100
    },
    {
      "activations/layer0_attention_weight_max": 15.226627349853516,
      "activations/layer0_attention_weight_min": -12.516387939453125,
      "activations/layer10_attention_weight_max": 81.81197357177734,
      "activations/layer10_attention_weight_min": -59.8104362487793,
      "activations/layer11_attention_weight_max": 52.584251403808594,
      "activations/layer11_attention_weight_min": -39.39377975463867,
      "activations/layer12_attention_weight_max": 28.20038604736328,
      "activations/layer12_attention_weight_min": -25.37960433959961,
      "activations/layer13_attention_weight_max": 64.2637710571289,
      "activations/layer13_attention_weight_min": -43.20383834838867,
      "activations/layer14_attention_weight_max": 70.26288604736328,
      "activations/layer14_attention_weight_min": -41.65670394897461,
      "activations/layer15_attention_weight_max": 63.59346389770508,
      "activations/layer15_attention_weight_min": -38.87562561035156,
      "activations/layer16_attention_weight_max": 47.253089904785156,
      "activations/layer16_attention_weight_min": -32.1494026184082,
      "activations/layer17_attention_weight_max": 71.7001953125,
      "activations/layer17_attention_weight_min": -39.20018005371094,
      "activations/layer18_attention_weight_max": 62.51252746582031,
      "activations/layer18_attention_weight_min": -34.18971252441406,
      "activations/layer19_attention_weight_max": 20.39969253540039,
      "activations/layer19_attention_weight_min": -13.750821113586426,
      "activations/layer1_attention_weight_max": 15.485613822937012,
      "activations/layer1_attention_weight_min": -13.997514724731445,
      "activations/layer20_attention_weight_max": 18.337255477905273,
      "activations/layer20_attention_weight_min": -14.88868236541748,
      "activations/layer21_attention_weight_max": 39.12481689453125,
      "activations/layer21_attention_weight_min": -20.723155975341797,
      "activations/layer22_attention_weight_max": 33.512779235839844,
      "activations/layer22_attention_weight_min": -22.037504196166992,
      "activations/layer23_attention_weight_max": 20.581222534179688,
      "activations/layer23_attention_weight_min": -16.773828506469727,
      "activations/layer2_attention_weight_max": 26.13599967956543,
      "activations/layer2_attention_weight_min": -25.05183982849121,
      "activations/layer3_attention_weight_max": 60.50164794921875,
      "activations/layer3_attention_weight_min": -61.02524185180664,
      "activations/layer4_attention_weight_max": 91.29076385498047,
      "activations/layer4_attention_weight_min": -81.5388412475586,
      "activations/layer5_attention_weight_max": 141.25624084472656,
      "activations/layer5_attention_weight_min": -126.9253158569336,
      "activations/layer6_attention_weight_max": 70.40559387207031,
      "activations/layer6_attention_weight_min": -59.050235748291016,
      "activations/layer7_attention_weight_max": 66.48649597167969,
      "activations/layer7_attention_weight_min": -56.74802780151367,
      "activations/layer8_attention_weight_max": 60.11475372314453,
      "activations/layer8_attention_weight_min": -46.98899841308594,
      "activations/layer9_attention_weight_max": 76.23331451416016,
      "activations/layer9_attention_weight_min": -65.9069595336914,
      "epoch": 1.4,
      "learning_rate": 0.00014238143939393938,
      "loss": 3.0086,
      "step": 24150
    },
    {
      "activations/layer0_attention_weight_max": 13.56244945526123,
      "activations/layer0_attention_weight_min": -12.258931159973145,
      "activations/layer10_attention_weight_max": 62.30265808105469,
      "activations/layer10_attention_weight_min": -56.11627960205078,
      "activations/layer11_attention_weight_max": 45.652008056640625,
      "activations/layer11_attention_weight_min": -38.6484375,
      "activations/layer12_attention_weight_max": 27.407384872436523,
      "activations/layer12_attention_weight_min": -23.477649688720703,
      "activations/layer13_attention_weight_max": 67.7385482788086,
      "activations/layer13_attention_weight_min": -51.40937805175781,
      "activations/layer14_attention_weight_max": 47.247100830078125,
      "activations/layer14_attention_weight_min": -39.99436950683594,
      "activations/layer15_attention_weight_max": 51.065391540527344,
      "activations/layer15_attention_weight_min": -40.37384796142578,
      "activations/layer16_attention_weight_max": 46.256507873535156,
      "activations/layer16_attention_weight_min": -31.481800079345703,
      "activations/layer17_attention_weight_max": 59.13329315185547,
      "activations/layer17_attention_weight_min": -37.13655471801758,
      "activations/layer18_attention_weight_max": 69.08865356445312,
      "activations/layer18_attention_weight_min": -35.102169036865234,
      "activations/layer19_attention_weight_max": 21.534513473510742,
      "activations/layer19_attention_weight_min": -14.930098533630371,
      "activations/layer1_attention_weight_max": 15.286212921142578,
      "activations/layer1_attention_weight_min": -13.509632110595703,
      "activations/layer20_attention_weight_max": 20.322072982788086,
      "activations/layer20_attention_weight_min": -16.123106002807617,
      "activations/layer21_attention_weight_max": 44.791229248046875,
      "activations/layer21_attention_weight_min": -21.735937118530273,
      "activations/layer22_attention_weight_max": 37.77345275878906,
      "activations/layer22_attention_weight_min": -20.8243350982666,
      "activations/layer23_attention_weight_max": 24.122779846191406,
      "activations/layer23_attention_weight_min": -14.254343032836914,
      "activations/layer2_attention_weight_max": 25.13763999938965,
      "activations/layer2_attention_weight_min": -24.129222869873047,
      "activations/layer3_attention_weight_max": 54.46917724609375,
      "activations/layer3_attention_weight_min": -57.01394271850586,
      "activations/layer4_attention_weight_max": 83.40802001953125,
      "activations/layer4_attention_weight_min": -76.94696807861328,
      "activations/layer5_attention_weight_max": 125.17929077148438,
      "activations/layer5_attention_weight_min": -120.24874114990234,
      "activations/layer6_attention_weight_max": 60.15583038330078,
      "activations/layer6_attention_weight_min": -55.78678512573242,
      "activations/layer7_attention_weight_max": 58.42478561401367,
      "activations/layer7_attention_weight_min": -55.114322662353516,
      "activations/layer8_attention_weight_max": 48.82140350341797,
      "activations/layer8_attention_weight_min": -44.87239456176758,
      "activations/layer9_attention_weight_max": 66.23112487792969,
      "activations/layer9_attention_weight_min": -64.5287094116211,
      "epoch": 1.41,
      "learning_rate": 0.00014236249999999997,
      "loss": 3.012,
      "step": 24200
    },
    {
      "activations/layer0_attention_weight_max": 12.897750854492188,
      "activations/layer0_attention_weight_min": -12.21653938293457,
      "activations/layer10_attention_weight_max": 70.02738189697266,
      "activations/layer10_attention_weight_min": -60.2497673034668,
      "activations/layer11_attention_weight_max": 46.79270553588867,
      "activations/layer11_attention_weight_min": -37.75518798828125,
      "activations/layer12_attention_weight_max": 30.666120529174805,
      "activations/layer12_attention_weight_min": -26.224693298339844,
      "activations/layer13_attention_weight_max": 75.80900573730469,
      "activations/layer13_attention_weight_min": -48.34299850463867,
      "activations/layer14_attention_weight_max": 59.27529525756836,
      "activations/layer14_attention_weight_min": -35.833702087402344,
      "activations/layer15_attention_weight_max": 58.32659149169922,
      "activations/layer15_attention_weight_min": -38.85000991821289,
      "activations/layer16_attention_weight_max": 50.86335754394531,
      "activations/layer16_attention_weight_min": -32.01325225830078,
      "activations/layer17_attention_weight_max": 61.29362106323242,
      "activations/layer17_attention_weight_min": -36.318668365478516,
      "activations/layer18_attention_weight_max": 64.13650512695312,
      "activations/layer18_attention_weight_min": -34.968605041503906,
      "activations/layer19_attention_weight_max": 22.208904266357422,
      "activations/layer19_attention_weight_min": -13.757204055786133,
      "activations/layer1_attention_weight_max": 15.039410591125488,
      "activations/layer1_attention_weight_min": -11.02666187286377,
      "activations/layer20_attention_weight_max": 20.096946716308594,
      "activations/layer20_attention_weight_min": -13.997061729431152,
      "activations/layer21_attention_weight_max": 40.45420455932617,
      "activations/layer21_attention_weight_min": -20.84906578063965,
      "activations/layer22_attention_weight_max": 37.4745979309082,
      "activations/layer22_attention_weight_min": -19.28575325012207,
      "activations/layer23_attention_weight_max": 21.187578201293945,
      "activations/layer23_attention_weight_min": -15.32022762298584,
      "activations/layer2_attention_weight_max": 25.353429794311523,
      "activations/layer2_attention_weight_min": -24.281545639038086,
      "activations/layer3_attention_weight_max": 55.89447784423828,
      "activations/layer3_attention_weight_min": -57.24100875854492,
      "activations/layer4_attention_weight_max": 82.82762145996094,
      "activations/layer4_attention_weight_min": -77.41028594970703,
      "activations/layer5_attention_weight_max": 127.0168685913086,
      "activations/layer5_attention_weight_min": -115.45304870605469,
      "activations/layer6_attention_weight_max": 62.04866409301758,
      "activations/layer6_attention_weight_min": -56.743690490722656,
      "activations/layer7_attention_weight_max": 61.394535064697266,
      "activations/layer7_attention_weight_min": -55.11878967285156,
      "activations/layer8_attention_weight_max": 54.04678726196289,
      "activations/layer8_attention_weight_min": -45.483882904052734,
      "activations/layer9_attention_weight_max": 77.46810150146484,
      "activations/layer9_attention_weight_min": -64.33972930908203,
      "epoch": 1.41,
      "learning_rate": 0.00014234356060606059,
      "loss": 3.0264,
      "step": 24250
    },
    {
      "activations/layer0_attention_weight_max": 14.618703842163086,
      "activations/layer0_attention_weight_min": -12.325529098510742,
      "activations/layer10_attention_weight_max": 63.409969329833984,
      "activations/layer10_attention_weight_min": -54.752952575683594,
      "activations/layer11_attention_weight_max": 48.33183288574219,
      "activations/layer11_attention_weight_min": -41.89263916015625,
      "activations/layer12_attention_weight_max": 29.968645095825195,
      "activations/layer12_attention_weight_min": -24.41261100769043,
      "activations/layer13_attention_weight_max": 55.73406219482422,
      "activations/layer13_attention_weight_min": -45.35612869262695,
      "activations/layer14_attention_weight_max": 50.846736907958984,
      "activations/layer14_attention_weight_min": -38.038536071777344,
      "activations/layer15_attention_weight_max": 52.4161376953125,
      "activations/layer15_attention_weight_min": -38.93838119506836,
      "activations/layer16_attention_weight_max": 52.84152603149414,
      "activations/layer16_attention_weight_min": -30.52263069152832,
      "activations/layer17_attention_weight_max": 61.32240676879883,
      "activations/layer17_attention_weight_min": -36.282936096191406,
      "activations/layer18_attention_weight_max": 70.41657257080078,
      "activations/layer18_attention_weight_min": -38.90917205810547,
      "activations/layer19_attention_weight_max": 23.56330108642578,
      "activations/layer19_attention_weight_min": -14.14897346496582,
      "activations/layer1_attention_weight_max": 15.150246620178223,
      "activations/layer1_attention_weight_min": -11.392702102661133,
      "activations/layer20_attention_weight_max": 19.56365203857422,
      "activations/layer20_attention_weight_min": -17.368839263916016,
      "activations/layer21_attention_weight_max": 39.24782180786133,
      "activations/layer21_attention_weight_min": -21.973989486694336,
      "activations/layer22_attention_weight_max": 36.6410026550293,
      "activations/layer22_attention_weight_min": -19.597951889038086,
      "activations/layer23_attention_weight_max": 24.262678146362305,
      "activations/layer23_attention_weight_min": -14.357165336608887,
      "activations/layer2_attention_weight_max": 25.97846794128418,
      "activations/layer2_attention_weight_min": -27.381507873535156,
      "activations/layer3_attention_weight_max": 57.383705139160156,
      "activations/layer3_attention_weight_min": -60.55996322631836,
      "activations/layer4_attention_weight_max": 90.32965850830078,
      "activations/layer4_attention_weight_min": -80.056884765625,
      "activations/layer5_attention_weight_max": 132.93434143066406,
      "activations/layer5_attention_weight_min": -119.41432189941406,
      "activations/layer6_attention_weight_max": 63.243186950683594,
      "activations/layer6_attention_weight_min": -55.99205780029297,
      "activations/layer7_attention_weight_max": 61.63874053955078,
      "activations/layer7_attention_weight_min": -55.3112907409668,
      "activations/layer8_attention_weight_max": 49.21282958984375,
      "activations/layer8_attention_weight_min": -45.85608673095703,
      "activations/layer9_attention_weight_max": 71.14990234375,
      "activations/layer9_attention_weight_min": -67.51676177978516,
      "epoch": 1.41,
      "learning_rate": 0.0001423246212121212,
      "loss": 3.0265,
      "step": 24300
    },
    {
      "activations/layer0_attention_weight_max": 14.727998733520508,
      "activations/layer0_attention_weight_min": -12.234307289123535,
      "activations/layer10_attention_weight_max": 64.77803039550781,
      "activations/layer10_attention_weight_min": -55.236846923828125,
      "activations/layer11_attention_weight_max": 44.40385055541992,
      "activations/layer11_attention_weight_min": -37.89130401611328,
      "activations/layer12_attention_weight_max": 26.3554744720459,
      "activations/layer12_attention_weight_min": -26.622392654418945,
      "activations/layer13_attention_weight_max": 63.891326904296875,
      "activations/layer13_attention_weight_min": -50.969993591308594,
      "activations/layer14_attention_weight_max": 58.575157165527344,
      "activations/layer14_attention_weight_min": -44.59752655029297,
      "activations/layer15_attention_weight_max": 52.94962692260742,
      "activations/layer15_attention_weight_min": -41.25554275512695,
      "activations/layer16_attention_weight_max": 43.52423858642578,
      "activations/layer16_attention_weight_min": -33.21036148071289,
      "activations/layer17_attention_weight_max": 66.1366958618164,
      "activations/layer17_attention_weight_min": -39.40415573120117,
      "activations/layer18_attention_weight_max": 64.98204803466797,
      "activations/layer18_attention_weight_min": -38.92404556274414,
      "activations/layer19_attention_weight_max": 24.69577407836914,
      "activations/layer19_attention_weight_min": -15.016054153442383,
      "activations/layer1_attention_weight_max": 15.592799186706543,
      "activations/layer1_attention_weight_min": -11.923029899597168,
      "activations/layer20_attention_weight_max": 17.822021484375,
      "activations/layer20_attention_weight_min": -16.215242385864258,
      "activations/layer21_attention_weight_max": 40.71825408935547,
      "activations/layer21_attention_weight_min": -19.16216468811035,
      "activations/layer22_attention_weight_max": 36.378597259521484,
      "activations/layer22_attention_weight_min": -20.27535057067871,
      "activations/layer23_attention_weight_max": 25.6342830657959,
      "activations/layer23_attention_weight_min": -16.369487762451172,
      "activations/layer2_attention_weight_max": 25.760692596435547,
      "activations/layer2_attention_weight_min": -24.597122192382812,
      "activations/layer3_attention_weight_max": 56.54069900512695,
      "activations/layer3_attention_weight_min": -57.73938751220703,
      "activations/layer4_attention_weight_max": 87.62244415283203,
      "activations/layer4_attention_weight_min": -78.84769439697266,
      "activations/layer5_attention_weight_max": 128.05722045898438,
      "activations/layer5_attention_weight_min": -115.08635711669922,
      "activations/layer6_attention_weight_max": 63.54887771606445,
      "activations/layer6_attention_weight_min": -55.996952056884766,
      "activations/layer7_attention_weight_max": 59.95364761352539,
      "activations/layer7_attention_weight_min": -54.61688995361328,
      "activations/layer8_attention_weight_max": 52.241188049316406,
      "activations/layer8_attention_weight_min": -46.88150405883789,
      "activations/layer9_attention_weight_max": 73.82538604736328,
      "activations/layer9_attention_weight_min": -65.1905288696289,
      "epoch": 1.41,
      "learning_rate": 0.0001423056818181818,
      "loss": 3.0199,
      "step": 24350
    },
    {
      "activations/layer0_attention_weight_max": 14.24640941619873,
      "activations/layer0_attention_weight_min": -12.47737979888916,
      "activations/layer10_attention_weight_max": 69.13874816894531,
      "activations/layer10_attention_weight_min": -56.069332122802734,
      "activations/layer11_attention_weight_max": 46.12138366699219,
      "activations/layer11_attention_weight_min": -39.6351203918457,
      "activations/layer12_attention_weight_max": 31.539752960205078,
      "activations/layer12_attention_weight_min": -24.36063575744629,
      "activations/layer13_attention_weight_max": 58.1160888671875,
      "activations/layer13_attention_weight_min": -43.139122009277344,
      "activations/layer14_attention_weight_max": 53.32007598876953,
      "activations/layer14_attention_weight_min": -45.44151306152344,
      "activations/layer15_attention_weight_max": 63.634071350097656,
      "activations/layer15_attention_weight_min": -44.696563720703125,
      "activations/layer16_attention_weight_max": 46.0693359375,
      "activations/layer16_attention_weight_min": -33.95833969116211,
      "activations/layer17_attention_weight_max": 55.76329040527344,
      "activations/layer17_attention_weight_min": -39.766151428222656,
      "activations/layer18_attention_weight_max": 63.4207763671875,
      "activations/layer18_attention_weight_min": -38.73615646362305,
      "activations/layer19_attention_weight_max": 26.08819580078125,
      "activations/layer19_attention_weight_min": -15.51811695098877,
      "activations/layer1_attention_weight_max": 15.296781539916992,
      "activations/layer1_attention_weight_min": -11.518195152282715,
      "activations/layer20_attention_weight_max": 25.085304260253906,
      "activations/layer20_attention_weight_min": -14.061478614807129,
      "activations/layer21_attention_weight_max": 41.03993225097656,
      "activations/layer21_attention_weight_min": -22.62662696838379,
      "activations/layer22_attention_weight_max": 40.6445198059082,
      "activations/layer22_attention_weight_min": -19.93909454345703,
      "activations/layer23_attention_weight_max": 26.76581382751465,
      "activations/layer23_attention_weight_min": -15.391422271728516,
      "activations/layer2_attention_weight_max": 27.130859375,
      "activations/layer2_attention_weight_min": -26.110393524169922,
      "activations/layer3_attention_weight_max": 56.860496520996094,
      "activations/layer3_attention_weight_min": -58.901187896728516,
      "activations/layer4_attention_weight_max": 86.93231201171875,
      "activations/layer4_attention_weight_min": -77.3858871459961,
      "activations/layer5_attention_weight_max": 129.06497192382812,
      "activations/layer5_attention_weight_min": -118.00410461425781,
      "activations/layer6_attention_weight_max": 65.90032196044922,
      "activations/layer6_attention_weight_min": -54.40797805786133,
      "activations/layer7_attention_weight_max": 64.05333709716797,
      "activations/layer7_attention_weight_min": -54.464866638183594,
      "activations/layer8_attention_weight_max": 50.986900329589844,
      "activations/layer8_attention_weight_min": -45.361507415771484,
      "activations/layer9_attention_weight_max": 71.85469055175781,
      "activations/layer9_attention_weight_min": -64.74340057373047,
      "epoch": 1.42,
      "learning_rate": 0.0001422867424242424,
      "loss": 3.0123,
      "step": 24400
    },
    {
      "activations/layer0_attention_weight_max": 15.111893653869629,
      "activations/layer0_attention_weight_min": -12.683399200439453,
      "activations/layer10_attention_weight_max": 58.75706100463867,
      "activations/layer10_attention_weight_min": -51.213741302490234,
      "activations/layer11_attention_weight_max": 47.5838737487793,
      "activations/layer11_attention_weight_min": -41.87152862548828,
      "activations/layer12_attention_weight_max": 26.486730575561523,
      "activations/layer12_attention_weight_min": -23.6057186126709,
      "activations/layer13_attention_weight_max": 65.00838470458984,
      "activations/layer13_attention_weight_min": -50.13069534301758,
      "activations/layer14_attention_weight_max": 53.331756591796875,
      "activations/layer14_attention_weight_min": -39.13541030883789,
      "activations/layer15_attention_weight_max": 52.737308502197266,
      "activations/layer15_attention_weight_min": -38.006492614746094,
      "activations/layer16_attention_weight_max": 45.88712692260742,
      "activations/layer16_attention_weight_min": -35.603416442871094,
      "activations/layer17_attention_weight_max": 52.753414154052734,
      "activations/layer17_attention_weight_min": -42.27859115600586,
      "activations/layer18_attention_weight_max": 57.67760467529297,
      "activations/layer18_attention_weight_min": -40.912349700927734,
      "activations/layer19_attention_weight_max": 21.203420639038086,
      "activations/layer19_attention_weight_min": -14.14069652557373,
      "activations/layer1_attention_weight_max": 15.552395820617676,
      "activations/layer1_attention_weight_min": -11.924577713012695,
      "activations/layer20_attention_weight_max": 20.2519588470459,
      "activations/layer20_attention_weight_min": -14.545428276062012,
      "activations/layer21_attention_weight_max": 36.13945388793945,
      "activations/layer21_attention_weight_min": -21.108997344970703,
      "activations/layer22_attention_weight_max": 36.959896087646484,
      "activations/layer22_attention_weight_min": -19.023910522460938,
      "activations/layer23_attention_weight_max": 27.47024154663086,
      "activations/layer23_attention_weight_min": -13.808942794799805,
      "activations/layer2_attention_weight_max": 25.773283004760742,
      "activations/layer2_attention_weight_min": -24.647724151611328,
      "activations/layer3_attention_weight_max": 54.425872802734375,
      "activations/layer3_attention_weight_min": -57.19208526611328,
      "activations/layer4_attention_weight_max": 84.39421844482422,
      "activations/layer4_attention_weight_min": -78.03704833984375,
      "activations/layer5_attention_weight_max": 130.64283752441406,
      "activations/layer5_attention_weight_min": -118.3792953491211,
      "activations/layer6_attention_weight_max": 65.41912078857422,
      "activations/layer6_attention_weight_min": -55.794593811035156,
      "activations/layer7_attention_weight_max": 64.47023010253906,
      "activations/layer7_attention_weight_min": -54.57133865356445,
      "activations/layer8_attention_weight_max": 55.52762985229492,
      "activations/layer8_attention_weight_min": -48.41835403442383,
      "activations/layer9_attention_weight_max": 67.28155517578125,
      "activations/layer9_attention_weight_min": -62.8006477355957,
      "epoch": 1.42,
      "learning_rate": 0.0001422681818181818,
      "loss": 3.0231,
      "step": 24450
    },
    {
      "activations/layer0_attention_weight_max": 15.346596717834473,
      "activations/layer0_attention_weight_min": -12.27608871459961,
      "activations/layer10_attention_weight_max": 60.03555679321289,
      "activations/layer10_attention_weight_min": -50.429683685302734,
      "activations/layer11_attention_weight_max": 46.27879333496094,
      "activations/layer11_attention_weight_min": -38.897865295410156,
      "activations/layer12_attention_weight_max": 28.766563415527344,
      "activations/layer12_attention_weight_min": -23.506961822509766,
      "activations/layer13_attention_weight_max": 59.61573791503906,
      "activations/layer13_attention_weight_min": -44.04381561279297,
      "activations/layer14_attention_weight_max": 53.95125198364258,
      "activations/layer14_attention_weight_min": -36.0911865234375,
      "activations/layer15_attention_weight_max": 50.30744934082031,
      "activations/layer15_attention_weight_min": -38.20155715942383,
      "activations/layer16_attention_weight_max": 55.94546127319336,
      "activations/layer16_attention_weight_min": -33.95743942260742,
      "activations/layer17_attention_weight_max": 69.38511657714844,
      "activations/layer17_attention_weight_min": -38.58299255371094,
      "activations/layer18_attention_weight_max": 65.79949951171875,
      "activations/layer18_attention_weight_min": -37.281734466552734,
      "activations/layer19_attention_weight_max": 19.556211471557617,
      "activations/layer19_attention_weight_min": -16.63849449157715,
      "activations/layer1_attention_weight_max": 15.923667907714844,
      "activations/layer1_attention_weight_min": -11.980728149414062,
      "activations/layer20_attention_weight_max": 20.893835067749023,
      "activations/layer20_attention_weight_min": -16.88296127319336,
      "activations/layer21_attention_weight_max": 38.620140075683594,
      "activations/layer21_attention_weight_min": -21.183319091796875,
      "activations/layer22_attention_weight_max": 41.769405364990234,
      "activations/layer22_attention_weight_min": -20.93417739868164,
      "activations/layer23_attention_weight_max": 25.39809226989746,
      "activations/layer23_attention_weight_min": -15.81971549987793,
      "activations/layer2_attention_weight_max": 26.390071868896484,
      "activations/layer2_attention_weight_min": -25.273147583007812,
      "activations/layer3_attention_weight_max": 56.87252426147461,
      "activations/layer3_attention_weight_min": -61.04299545288086,
      "activations/layer4_attention_weight_max": 85.34651947021484,
      "activations/layer4_attention_weight_min": -74.88241577148438,
      "activations/layer5_attention_weight_max": 131.3777313232422,
      "activations/layer5_attention_weight_min": -114.6776123046875,
      "activations/layer6_attention_weight_max": 60.505836486816406,
      "activations/layer6_attention_weight_min": -53.72062683105469,
      "activations/layer7_attention_weight_max": 66.10738372802734,
      "activations/layer7_attention_weight_min": -56.264015197753906,
      "activations/layer8_attention_weight_max": 55.507755279541016,
      "activations/layer8_attention_weight_min": -48.37737274169922,
      "activations/layer9_attention_weight_max": 77.9820785522461,
      "activations/layer9_attention_weight_min": -68.04055786132812,
      "epoch": 1.42,
      "learning_rate": 0.0001422492424242424,
      "loss": 3.0158,
      "step": 24500
    },
    {
      "activations/layer0_attention_weight_max": 13.91734790802002,
      "activations/layer0_attention_weight_min": -11.737556457519531,
      "activations/layer10_attention_weight_max": 65.93685150146484,
      "activations/layer10_attention_weight_min": -56.60894775390625,
      "activations/layer11_attention_weight_max": 45.713829040527344,
      "activations/layer11_attention_weight_min": -35.42988586425781,
      "activations/layer12_attention_weight_max": 28.3190860748291,
      "activations/layer12_attention_weight_min": -25.551780700683594,
      "activations/layer13_attention_weight_max": 78.50994873046875,
      "activations/layer13_attention_weight_min": -51.76533126831055,
      "activations/layer14_attention_weight_max": 83.26057434082031,
      "activations/layer14_attention_weight_min": -48.013614654541016,
      "activations/layer15_attention_weight_max": 65.95071411132812,
      "activations/layer15_attention_weight_min": -41.13563537597656,
      "activations/layer16_attention_weight_max": 48.171546936035156,
      "activations/layer16_attention_weight_min": -34.3531379699707,
      "activations/layer17_attention_weight_max": 56.621089935302734,
      "activations/layer17_attention_weight_min": -39.09024429321289,
      "activations/layer18_attention_weight_max": 71.82422637939453,
      "activations/layer18_attention_weight_min": -38.221168518066406,
      "activations/layer19_attention_weight_max": 22.005109786987305,
      "activations/layer19_attention_weight_min": -14.91525936126709,
      "activations/layer1_attention_weight_max": 15.554535865783691,
      "activations/layer1_attention_weight_min": -10.974458694458008,
      "activations/layer20_attention_weight_max": 23.534757614135742,
      "activations/layer20_attention_weight_min": -17.851985931396484,
      "activations/layer21_attention_weight_max": 40.70899963378906,
      "activations/layer21_attention_weight_min": -20.780235290527344,
      "activations/layer22_attention_weight_max": 34.97697067260742,
      "activations/layer22_attention_weight_min": -21.759441375732422,
      "activations/layer23_attention_weight_max": 25.817907333374023,
      "activations/layer23_attention_weight_min": -15.146184921264648,
      "activations/layer2_attention_weight_max": 23.929838180541992,
      "activations/layer2_attention_weight_min": -23.81928825378418,
      "activations/layer3_attention_weight_max": 55.231014251708984,
      "activations/layer3_attention_weight_min": -59.85000228881836,
      "activations/layer4_attention_weight_max": 84.90471649169922,
      "activations/layer4_attention_weight_min": -75.63375854492188,
      "activations/layer5_attention_weight_max": 128.65695190429688,
      "activations/layer5_attention_weight_min": -112.9095687866211,
      "activations/layer6_attention_weight_max": 59.692604064941406,
      "activations/layer6_attention_weight_min": -53.26646041870117,
      "activations/layer7_attention_weight_max": 56.19401168823242,
      "activations/layer7_attention_weight_min": -53.740966796875,
      "activations/layer8_attention_weight_max": 49.78215789794922,
      "activations/layer8_attention_weight_min": -46.42115783691406,
      "activations/layer9_attention_weight_max": 67.52395629882812,
      "activations/layer9_attention_weight_min": -61.94941329956055,
      "epoch": 1.43,
      "learning_rate": 0.00014223030303030302,
      "loss": 3.0068,
      "step": 24550
    },
    {
      "activations/layer0_attention_weight_max": 14.399401664733887,
      "activations/layer0_attention_weight_min": -12.305777549743652,
      "activations/layer10_attention_weight_max": 61.91082000732422,
      "activations/layer10_attention_weight_min": -56.403526306152344,
      "activations/layer11_attention_weight_max": 44.035518646240234,
      "activations/layer11_attention_weight_min": -41.76265335083008,
      "activations/layer12_attention_weight_max": 28.173200607299805,
      "activations/layer12_attention_weight_min": -23.789283752441406,
      "activations/layer13_attention_weight_max": 57.004600524902344,
      "activations/layer13_attention_weight_min": -43.99770736694336,
      "activations/layer14_attention_weight_max": 51.06499099731445,
      "activations/layer14_attention_weight_min": -37.063194274902344,
      "activations/layer15_attention_weight_max": 49.0767822265625,
      "activations/layer15_attention_weight_min": -38.638877868652344,
      "activations/layer16_attention_weight_max": 43.514923095703125,
      "activations/layer16_attention_weight_min": -29.781892776489258,
      "activations/layer17_attention_weight_max": 53.42757034301758,
      "activations/layer17_attention_weight_min": -37.83283233642578,
      "activations/layer18_attention_weight_max": 56.599178314208984,
      "activations/layer18_attention_weight_min": -40.8515739440918,
      "activations/layer19_attention_weight_max": 21.344764709472656,
      "activations/layer19_attention_weight_min": -14.928812026977539,
      "activations/layer1_attention_weight_max": 15.339159965515137,
      "activations/layer1_attention_weight_min": -11.524442672729492,
      "activations/layer20_attention_weight_max": 19.762617111206055,
      "activations/layer20_attention_weight_min": -16.088712692260742,
      "activations/layer21_attention_weight_max": 35.92618179321289,
      "activations/layer21_attention_weight_min": -20.948318481445312,
      "activations/layer22_attention_weight_max": 35.11911392211914,
      "activations/layer22_attention_weight_min": -18.669292449951172,
      "activations/layer23_attention_weight_max": 23.173717498779297,
      "activations/layer23_attention_weight_min": -16.79841423034668,
      "activations/layer2_attention_weight_max": 25.065841674804688,
      "activations/layer2_attention_weight_min": -24.582721710205078,
      "activations/layer3_attention_weight_max": 54.146053314208984,
      "activations/layer3_attention_weight_min": -56.945465087890625,
      "activations/layer4_attention_weight_max": 83.41732788085938,
      "activations/layer4_attention_weight_min": -74.96016693115234,
      "activations/layer5_attention_weight_max": 129.144287109375,
      "activations/layer5_attention_weight_min": -114.22396087646484,
      "activations/layer6_attention_weight_max": 66.34571838378906,
      "activations/layer6_attention_weight_min": -54.38432693481445,
      "activations/layer7_attention_weight_max": 62.45107650756836,
      "activations/layer7_attention_weight_min": -51.782752990722656,
      "activations/layer8_attention_weight_max": 55.84946823120117,
      "activations/layer8_attention_weight_min": -45.8394889831543,
      "activations/layer9_attention_weight_max": 70.33885955810547,
      "activations/layer9_attention_weight_min": -65.78533172607422,
      "epoch": 1.43,
      "learning_rate": 0.0001422113636363636,
      "loss": 3.0252,
      "step": 24600
    },
    {
      "activations/layer0_attention_weight_max": 13.73840045928955,
      "activations/layer0_attention_weight_min": -12.535869598388672,
      "activations/layer10_attention_weight_max": 61.98630142211914,
      "activations/layer10_attention_weight_min": -54.52261734008789,
      "activations/layer11_attention_weight_max": 43.46908187866211,
      "activations/layer11_attention_weight_min": -35.41428756713867,
      "activations/layer12_attention_weight_max": 30.613384246826172,
      "activations/layer12_attention_weight_min": -26.13916778564453,
      "activations/layer13_attention_weight_max": 60.11857986450195,
      "activations/layer13_attention_weight_min": -46.069393157958984,
      "activations/layer14_attention_weight_max": 47.8170166015625,
      "activations/layer14_attention_weight_min": -37.995052337646484,
      "activations/layer15_attention_weight_max": 52.63554000854492,
      "activations/layer15_attention_weight_min": -38.271392822265625,
      "activations/layer16_attention_weight_max": 45.461158752441406,
      "activations/layer16_attention_weight_min": -30.757442474365234,
      "activations/layer17_attention_weight_max": 61.46480178833008,
      "activations/layer17_attention_weight_min": -38.67693328857422,
      "activations/layer18_attention_weight_max": 67.86603546142578,
      "activations/layer18_attention_weight_min": -37.880558013916016,
      "activations/layer19_attention_weight_max": 18.32063865661621,
      "activations/layer19_attention_weight_min": -14.295422554016113,
      "activations/layer1_attention_weight_max": 16.361345291137695,
      "activations/layer1_attention_weight_min": -12.961557388305664,
      "activations/layer20_attention_weight_max": 21.694839477539062,
      "activations/layer20_attention_weight_min": -14.398148536682129,
      "activations/layer21_attention_weight_max": 38.83333206176758,
      "activations/layer21_attention_weight_min": -19.15036964416504,
      "activations/layer22_attention_weight_max": 38.69465637207031,
      "activations/layer22_attention_weight_min": -19.816627502441406,
      "activations/layer23_attention_weight_max": 25.135726928710938,
      "activations/layer23_attention_weight_min": -13.74547004699707,
      "activations/layer2_attention_weight_max": 25.037193298339844,
      "activations/layer2_attention_weight_min": -25.232860565185547,
      "activations/layer3_attention_weight_max": 54.71831130981445,
      "activations/layer3_attention_weight_min": -58.681297302246094,
      "activations/layer4_attention_weight_max": 87.23226165771484,
      "activations/layer4_attention_weight_min": -77.6750259399414,
      "activations/layer5_attention_weight_max": 130.80657958984375,
      "activations/layer5_attention_weight_min": -119.58218383789062,
      "activations/layer6_attention_weight_max": 60.807769775390625,
      "activations/layer6_attention_weight_min": -55.35652542114258,
      "activations/layer7_attention_weight_max": 57.77809143066406,
      "activations/layer7_attention_weight_min": -53.84306716918945,
      "activations/layer8_attention_weight_max": 47.942317962646484,
      "activations/layer8_attention_weight_min": -43.38845443725586,
      "activations/layer9_attention_weight_max": 63.484737396240234,
      "activations/layer9_attention_weight_min": -66.20488739013672,
      "epoch": 1.43,
      "learning_rate": 0.00014219242424242423,
      "loss": 3.0249,
      "step": 24650
    },
    {
      "activations/layer0_attention_weight_max": 14.381817817687988,
      "activations/layer0_attention_weight_min": -12.476160049438477,
      "activations/layer10_attention_weight_max": 67.90191650390625,
      "activations/layer10_attention_weight_min": -52.97654724121094,
      "activations/layer11_attention_weight_max": 49.368282318115234,
      "activations/layer11_attention_weight_min": -38.109336853027344,
      "activations/layer12_attention_weight_max": 31.7330379486084,
      "activations/layer12_attention_weight_min": -26.965856552124023,
      "activations/layer13_attention_weight_max": 70.03682708740234,
      "activations/layer13_attention_weight_min": -53.19119644165039,
      "activations/layer14_attention_weight_max": 69.13308715820312,
      "activations/layer14_attention_weight_min": -44.934852600097656,
      "activations/layer15_attention_weight_max": 64.28189086914062,
      "activations/layer15_attention_weight_min": -39.91486358642578,
      "activations/layer16_attention_weight_max": 60.785865783691406,
      "activations/layer16_attention_weight_min": -32.88181686401367,
      "activations/layer17_attention_weight_max": 66.18624877929688,
      "activations/layer17_attention_weight_min": -37.76533508300781,
      "activations/layer18_attention_weight_max": 71.42255401611328,
      "activations/layer18_attention_weight_min": -35.453155517578125,
      "activations/layer19_attention_weight_max": 23.012418746948242,
      "activations/layer19_attention_weight_min": -15.907417297363281,
      "activations/layer1_attention_weight_max": 15.17525577545166,
      "activations/layer1_attention_weight_min": -12.262285232543945,
      "activations/layer20_attention_weight_max": 20.818206787109375,
      "activations/layer20_attention_weight_min": -14.554909706115723,
      "activations/layer21_attention_weight_max": 41.773902893066406,
      "activations/layer21_attention_weight_min": -23.86659812927246,
      "activations/layer22_attention_weight_max": 41.352928161621094,
      "activations/layer22_attention_weight_min": -19.988264083862305,
      "activations/layer23_attention_weight_max": 25.065872192382812,
      "activations/layer23_attention_weight_min": -15.418002128601074,
      "activations/layer2_attention_weight_max": 27.576581954956055,
      "activations/layer2_attention_weight_min": -24.079017639160156,
      "activations/layer3_attention_weight_max": 58.85921859741211,
      "activations/layer3_attention_weight_min": -59.664085388183594,
      "activations/layer4_attention_weight_max": 89.61862182617188,
      "activations/layer4_attention_weight_min": -84.40382385253906,
      "activations/layer5_attention_weight_max": 135.07061767578125,
      "activations/layer5_attention_weight_min": -122.85560607910156,
      "activations/layer6_attention_weight_max": 68.71389770507812,
      "activations/layer6_attention_weight_min": -59.17317581176758,
      "activations/layer7_attention_weight_max": 66.78800201416016,
      "activations/layer7_attention_weight_min": -61.38477325439453,
      "activations/layer8_attention_weight_max": 52.36026382446289,
      "activations/layer8_attention_weight_min": -47.050376892089844,
      "activations/layer9_attention_weight_max": 73.7902603149414,
      "activations/layer9_attention_weight_min": -62.57111740112305,
      "epoch": 1.44,
      "learning_rate": 0.00014217348484848484,
      "loss": 3.042,
      "step": 24700
    },
    {
      "activations/layer0_attention_weight_max": 13.986207962036133,
      "activations/layer0_attention_weight_min": -12.383245468139648,
      "activations/layer10_attention_weight_max": 65.53569793701172,
      "activations/layer10_attention_weight_min": -56.93878173828125,
      "activations/layer11_attention_weight_max": 47.07502746582031,
      "activations/layer11_attention_weight_min": -38.81855010986328,
      "activations/layer12_attention_weight_max": 31.885547637939453,
      "activations/layer12_attention_weight_min": -24.22257423400879,
      "activations/layer13_attention_weight_max": 78.79388427734375,
      "activations/layer13_attention_weight_min": -48.78406524658203,
      "activations/layer14_attention_weight_max": 57.4720573425293,
      "activations/layer14_attention_weight_min": -36.447364807128906,
      "activations/layer15_attention_weight_max": 55.52499771118164,
      "activations/layer15_attention_weight_min": -35.183738708496094,
      "activations/layer16_attention_weight_max": 55.929405212402344,
      "activations/layer16_attention_weight_min": -30.693078994750977,
      "activations/layer17_attention_weight_max": 61.15819549560547,
      "activations/layer17_attention_weight_min": -37.40375900268555,
      "activations/layer18_attention_weight_max": 70.13077545166016,
      "activations/layer18_attention_weight_min": -34.0852165222168,
      "activations/layer19_attention_weight_max": 21.507484436035156,
      "activations/layer19_attention_weight_min": -17.261653900146484,
      "activations/layer1_attention_weight_max": 16.74858283996582,
      "activations/layer1_attention_weight_min": -12.855834007263184,
      "activations/layer20_attention_weight_max": 17.77486801147461,
      "activations/layer20_attention_weight_min": -14.770788192749023,
      "activations/layer21_attention_weight_max": 39.84492874145508,
      "activations/layer21_attention_weight_min": -23.258220672607422,
      "activations/layer22_attention_weight_max": 38.41562271118164,
      "activations/layer22_attention_weight_min": -19.626304626464844,
      "activations/layer23_attention_weight_max": 27.209922790527344,
      "activations/layer23_attention_weight_min": -17.109432220458984,
      "activations/layer2_attention_weight_max": 26.518695831298828,
      "activations/layer2_attention_weight_min": -25.04891014099121,
      "activations/layer3_attention_weight_max": 53.85163116455078,
      "activations/layer3_attention_weight_min": -56.74405288696289,
      "activations/layer4_attention_weight_max": 85.57343292236328,
      "activations/layer4_attention_weight_min": -75.50603485107422,
      "activations/layer5_attention_weight_max": 143.4386749267578,
      "activations/layer5_attention_weight_min": -126.56455993652344,
      "activations/layer6_attention_weight_max": 63.006507873535156,
      "activations/layer6_attention_weight_min": -56.024017333984375,
      "activations/layer7_attention_weight_max": 66.86551666259766,
      "activations/layer7_attention_weight_min": -56.48871994018555,
      "activations/layer8_attention_weight_max": 57.59785461425781,
      "activations/layer8_attention_weight_min": -45.72825622558594,
      "activations/layer9_attention_weight_max": 76.26195526123047,
      "activations/layer9_attention_weight_min": -65.54827880859375,
      "epoch": 1.44,
      "learning_rate": 0.00014215454545454543,
      "loss": 3.0312,
      "step": 24750
    },
    {
      "activations/layer0_attention_weight_max": 14.00954818725586,
      "activations/layer0_attention_weight_min": -12.476634979248047,
      "activations/layer10_attention_weight_max": 65.81397247314453,
      "activations/layer10_attention_weight_min": -57.05755615234375,
      "activations/layer11_attention_weight_max": 45.17988586425781,
      "activations/layer11_attention_weight_min": -35.094871520996094,
      "activations/layer12_attention_weight_max": 28.95180892944336,
      "activations/layer12_attention_weight_min": -24.068775177001953,
      "activations/layer13_attention_weight_max": 66.97295379638672,
      "activations/layer13_attention_weight_min": -50.53068161010742,
      "activations/layer14_attention_weight_max": 66.72151947021484,
      "activations/layer14_attention_weight_min": -51.238075256347656,
      "activations/layer15_attention_weight_max": 48.39973068237305,
      "activations/layer15_attention_weight_min": -38.80801010131836,
      "activations/layer16_attention_weight_max": 47.38153839111328,
      "activations/layer16_attention_weight_min": -30.851041793823242,
      "activations/layer17_attention_weight_max": 60.7734375,
      "activations/layer17_attention_weight_min": -37.997222900390625,
      "activations/layer18_attention_weight_max": 62.32681655883789,
      "activations/layer18_attention_weight_min": -38.26449203491211,
      "activations/layer19_attention_weight_max": 21.730398178100586,
      "activations/layer19_attention_weight_min": -15.123356819152832,
      "activations/layer1_attention_weight_max": 15.728363990783691,
      "activations/layer1_attention_weight_min": -14.930596351623535,
      "activations/layer20_attention_weight_max": 24.1795654296875,
      "activations/layer20_attention_weight_min": -15.452244758605957,
      "activations/layer21_attention_weight_max": 40.08050537109375,
      "activations/layer21_attention_weight_min": -20.07284927368164,
      "activations/layer22_attention_weight_max": 36.071006774902344,
      "activations/layer22_attention_weight_min": -20.901687622070312,
      "activations/layer23_attention_weight_max": 27.236873626708984,
      "activations/layer23_attention_weight_min": -14.189696311950684,
      "activations/layer2_attention_weight_max": 26.30019187927246,
      "activations/layer2_attention_weight_min": -25.05103874206543,
      "activations/layer3_attention_weight_max": 58.903228759765625,
      "activations/layer3_attention_weight_min": -59.91455841064453,
      "activations/layer4_attention_weight_max": 88.63809204101562,
      "activations/layer4_attention_weight_min": -77.33023071289062,
      "activations/layer5_attention_weight_max": 126.76876831054688,
      "activations/layer5_attention_weight_min": -116.24015808105469,
      "activations/layer6_attention_weight_max": 61.12664031982422,
      "activations/layer6_attention_weight_min": -55.8807487487793,
      "activations/layer7_attention_weight_max": 58.63600540161133,
      "activations/layer7_attention_weight_min": -54.61438751220703,
      "activations/layer8_attention_weight_max": 50.19681167602539,
      "activations/layer8_attention_weight_min": -45.626686096191406,
      "activations/layer9_attention_weight_max": 67.69047546386719,
      "activations/layer9_attention_weight_min": -63.8311767578125,
      "epoch": 1.44,
      "learning_rate": 0.00014213560606060605,
      "loss": 3.0048,
      "step": 24800
    },
    {
      "activations/layer0_attention_weight_max": 13.840767860412598,
      "activations/layer0_attention_weight_min": -13.126118659973145,
      "activations/layer10_attention_weight_max": 66.02780151367188,
      "activations/layer10_attention_weight_min": -55.993072509765625,
      "activations/layer11_attention_weight_max": 47.54113006591797,
      "activations/layer11_attention_weight_min": -43.554710388183594,
      "activations/layer12_attention_weight_max": 29.47981071472168,
      "activations/layer12_attention_weight_min": -27.532236099243164,
      "activations/layer13_attention_weight_max": 64.0917739868164,
      "activations/layer13_attention_weight_min": -48.8933219909668,
      "activations/layer14_attention_weight_max": 51.93268966674805,
      "activations/layer14_attention_weight_min": -41.30693817138672,
      "activations/layer15_attention_weight_max": 54.44490051269531,
      "activations/layer15_attention_weight_min": -38.82271194458008,
      "activations/layer16_attention_weight_max": 45.07398223876953,
      "activations/layer16_attention_weight_min": -30.234195709228516,
      "activations/layer17_attention_weight_max": 58.83890151977539,
      "activations/layer17_attention_weight_min": -40.35743713378906,
      "activations/layer18_attention_weight_max": 66.98301696777344,
      "activations/layer18_attention_weight_min": -35.771114349365234,
      "activations/layer19_attention_weight_max": 22.86334800720215,
      "activations/layer19_attention_weight_min": -15.611342430114746,
      "activations/layer1_attention_weight_max": 15.476495742797852,
      "activations/layer1_attention_weight_min": -12.12714958190918,
      "activations/layer20_attention_weight_max": 18.604766845703125,
      "activations/layer20_attention_weight_min": -16.25371551513672,
      "activations/layer21_attention_weight_max": 38.07414627075195,
      "activations/layer21_attention_weight_min": -21.01981544494629,
      "activations/layer22_attention_weight_max": 36.06184768676758,
      "activations/layer22_attention_weight_min": -21.032163619995117,
      "activations/layer23_attention_weight_max": 23.466032028198242,
      "activations/layer23_attention_weight_min": -17.40261459350586,
      "activations/layer2_attention_weight_max": 26.554039001464844,
      "activations/layer2_attention_weight_min": -25.04739761352539,
      "activations/layer3_attention_weight_max": 54.29717254638672,
      "activations/layer3_attention_weight_min": -56.293907165527344,
      "activations/layer4_attention_weight_max": 86.56688690185547,
      "activations/layer4_attention_weight_min": -78.44953155517578,
      "activations/layer5_attention_weight_max": 135.14825439453125,
      "activations/layer5_attention_weight_min": -117.95460510253906,
      "activations/layer6_attention_weight_max": 62.80320358276367,
      "activations/layer6_attention_weight_min": -52.497066497802734,
      "activations/layer7_attention_weight_max": 61.581485748291016,
      "activations/layer7_attention_weight_min": -56.15781021118164,
      "activations/layer8_attention_weight_max": 55.842979431152344,
      "activations/layer8_attention_weight_min": -48.10576629638672,
      "activations/layer9_attention_weight_max": 72.97905731201172,
      "activations/layer9_attention_weight_min": -75.6345443725586,
      "epoch": 1.44,
      "learning_rate": 0.00014211666666666666,
      "loss": 3.0165,
      "step": 24850
    },
    {
      "activations/layer0_attention_weight_max": 14.985960006713867,
      "activations/layer0_attention_weight_min": -13.56251049041748,
      "activations/layer10_attention_weight_max": 62.544857025146484,
      "activations/layer10_attention_weight_min": -54.79026412963867,
      "activations/layer11_attention_weight_max": 44.92253875732422,
      "activations/layer11_attention_weight_min": -38.73461151123047,
      "activations/layer12_attention_weight_max": 25.38054656982422,
      "activations/layer12_attention_weight_min": -30.029333114624023,
      "activations/layer13_attention_weight_max": 63.524696350097656,
      "activations/layer13_attention_weight_min": -48.53676223754883,
      "activations/layer14_attention_weight_max": 61.93593215942383,
      "activations/layer14_attention_weight_min": -46.46138381958008,
      "activations/layer15_attention_weight_max": 52.08397674560547,
      "activations/layer15_attention_weight_min": -39.25175857543945,
      "activations/layer16_attention_weight_max": 47.58200454711914,
      "activations/layer16_attention_weight_min": -31.627092361450195,
      "activations/layer17_attention_weight_max": 61.76361846923828,
      "activations/layer17_attention_weight_min": -37.38016891479492,
      "activations/layer18_attention_weight_max": 57.8764533996582,
      "activations/layer18_attention_weight_min": -35.44087600708008,
      "activations/layer19_attention_weight_max": 22.41000747680664,
      "activations/layer19_attention_weight_min": -14.355856895446777,
      "activations/layer1_attention_weight_max": 15.77198314666748,
      "activations/layer1_attention_weight_min": -12.516169548034668,
      "activations/layer20_attention_weight_max": 21.056800842285156,
      "activations/layer20_attention_weight_min": -15.709314346313477,
      "activations/layer21_attention_weight_max": 36.378726959228516,
      "activations/layer21_attention_weight_min": -24.87788963317871,
      "activations/layer22_attention_weight_max": 36.890403747558594,
      "activations/layer22_attention_weight_min": -18.73686408996582,
      "activations/layer23_attention_weight_max": 25.976505279541016,
      "activations/layer23_attention_weight_min": -13.44584846496582,
      "activations/layer2_attention_weight_max": 24.068317413330078,
      "activations/layer2_attention_weight_min": -24.566394805908203,
      "activations/layer3_attention_weight_max": 59.251129150390625,
      "activations/layer3_attention_weight_min": -60.91454315185547,
      "activations/layer4_attention_weight_max": 92.1987075805664,
      "activations/layer4_attention_weight_min": -79.38544464111328,
      "activations/layer5_attention_weight_max": 128.61721801757812,
      "activations/layer5_attention_weight_min": -116.86199188232422,
      "activations/layer6_attention_weight_max": 62.66686248779297,
      "activations/layer6_attention_weight_min": -56.68701171875,
      "activations/layer7_attention_weight_max": 59.2841796875,
      "activations/layer7_attention_weight_min": -57.570552825927734,
      "activations/layer8_attention_weight_max": 50.61261749267578,
      "activations/layer8_attention_weight_min": -48.94681167602539,
      "activations/layer9_attention_weight_max": 74.82915496826172,
      "activations/layer9_attention_weight_min": -69.92461395263672,
      "epoch": 1.45,
      "learning_rate": 0.00014209772727272728,
      "loss": 3.0235,
      "step": 24900
    },
    {
      "activations/layer0_attention_weight_max": 15.613130569458008,
      "activations/layer0_attention_weight_min": -13.46277904510498,
      "activations/layer10_attention_weight_max": 61.23619079589844,
      "activations/layer10_attention_weight_min": -57.622711181640625,
      "activations/layer11_attention_weight_max": 49.530906677246094,
      "activations/layer11_attention_weight_min": -44.41292953491211,
      "activations/layer12_attention_weight_max": 24.258771896362305,
      "activations/layer12_attention_weight_min": -25.651901245117188,
      "activations/layer13_attention_weight_max": 55.53960037231445,
      "activations/layer13_attention_weight_min": -45.240238189697266,
      "activations/layer14_attention_weight_max": 46.59784698486328,
      "activations/layer14_attention_weight_min": -35.36203384399414,
      "activations/layer15_attention_weight_max": 48.340171813964844,
      "activations/layer15_attention_weight_min": -40.18275451660156,
      "activations/layer16_attention_weight_max": 38.11219024658203,
      "activations/layer16_attention_weight_min": -29.84918212890625,
      "activations/layer17_attention_weight_max": 51.33054733276367,
      "activations/layer17_attention_weight_min": -38.213775634765625,
      "activations/layer18_attention_weight_max": 52.1296272277832,
      "activations/layer18_attention_weight_min": -34.4369010925293,
      "activations/layer19_attention_weight_max": 22.89041519165039,
      "activations/layer19_attention_weight_min": -16.30215072631836,
      "activations/layer1_attention_weight_max": 16.35249137878418,
      "activations/layer1_attention_weight_min": -12.17482852935791,
      "activations/layer20_attention_weight_max": 20.147119522094727,
      "activations/layer20_attention_weight_min": -15.932272911071777,
      "activations/layer21_attention_weight_max": 34.35831832885742,
      "activations/layer21_attention_weight_min": -21.303604125976562,
      "activations/layer22_attention_weight_max": 35.06480026245117,
      "activations/layer22_attention_weight_min": -20.120920181274414,
      "activations/layer23_attention_weight_max": 22.940336227416992,
      "activations/layer23_attention_weight_min": -16.146467208862305,
      "activations/layer2_attention_weight_max": 25.386192321777344,
      "activations/layer2_attention_weight_min": -25.029111862182617,
      "activations/layer3_attention_weight_max": 55.53331756591797,
      "activations/layer3_attention_weight_min": -59.22727584838867,
      "activations/layer4_attention_weight_max": 87.97611236572266,
      "activations/layer4_attention_weight_min": -79.27755737304688,
      "activations/layer5_attention_weight_max": 137.13067626953125,
      "activations/layer5_attention_weight_min": -118.52302551269531,
      "activations/layer6_attention_weight_max": 61.9473876953125,
      "activations/layer6_attention_weight_min": -55.7983283996582,
      "activations/layer7_attention_weight_max": 63.795597076416016,
      "activations/layer7_attention_weight_min": -58.21742630004883,
      "activations/layer8_attention_weight_max": 57.15979766845703,
      "activations/layer8_attention_weight_min": -52.862117767333984,
      "activations/layer9_attention_weight_max": 76.1466293334961,
      "activations/layer9_attention_weight_min": -83.20477294921875,
      "epoch": 1.45,
      "learning_rate": 0.00014207878787878786,
      "loss": 3.0263,
      "step": 24950
    },
    {
      "activations/layer0_attention_weight_max": 14.659173011779785,
      "activations/layer0_attention_weight_min": -13.072336196899414,
      "activations/layer10_attention_weight_max": 61.06914520263672,
      "activations/layer10_attention_weight_min": -57.44435501098633,
      "activations/layer11_attention_weight_max": 50.46836853027344,
      "activations/layer11_attention_weight_min": -42.56449890136719,
      "activations/layer12_attention_weight_max": 25.81131362915039,
      "activations/layer12_attention_weight_min": -29.995603561401367,
      "activations/layer13_attention_weight_max": 88.75252532958984,
      "activations/layer13_attention_weight_min": -54.48426055908203,
      "activations/layer14_attention_weight_max": 115.21217346191406,
      "activations/layer14_attention_weight_min": -58.577056884765625,
      "activations/layer15_attention_weight_max": 68.08731842041016,
      "activations/layer15_attention_weight_min": -43.127811431884766,
      "activations/layer16_attention_weight_max": 45.638065338134766,
      "activations/layer16_attention_weight_min": -33.30247497558594,
      "activations/layer17_attention_weight_max": 56.689178466796875,
      "activations/layer17_attention_weight_min": -38.084930419921875,
      "activations/layer18_attention_weight_max": 55.369571685791016,
      "activations/layer18_attention_weight_min": -33.92530059814453,
      "activations/layer19_attention_weight_max": 20.426841735839844,
      "activations/layer19_attention_weight_min": -13.34369945526123,
      "activations/layer1_attention_weight_max": 16.007844924926758,
      "activations/layer1_attention_weight_min": -13.906095504760742,
      "activations/layer20_attention_weight_max": 18.897462844848633,
      "activations/layer20_attention_weight_min": -13.436666488647461,
      "activations/layer21_attention_weight_max": 33.63186264038086,
      "activations/layer21_attention_weight_min": -21.221803665161133,
      "activations/layer22_attention_weight_max": 34.477115631103516,
      "activations/layer22_attention_weight_min": -20.552278518676758,
      "activations/layer23_attention_weight_max": 22.54189109802246,
      "activations/layer23_attention_weight_min": -12.79835319519043,
      "activations/layer2_attention_weight_max": 24.621559143066406,
      "activations/layer2_attention_weight_min": -23.80402183532715,
      "activations/layer3_attention_weight_max": 57.04790496826172,
      "activations/layer3_attention_weight_min": -58.63203811645508,
      "activations/layer4_attention_weight_max": 93.00322723388672,
      "activations/layer4_attention_weight_min": -80.69029235839844,
      "activations/layer5_attention_weight_max": 140.91456604003906,
      "activations/layer5_attention_weight_min": -122.82350158691406,
      "activations/layer6_attention_weight_max": 63.71317672729492,
      "activations/layer6_attention_weight_min": -57.08600997924805,
      "activations/layer7_attention_weight_max": 70.3573226928711,
      "activations/layer7_attention_weight_min": -55.090110778808594,
      "activations/layer8_attention_weight_max": 56.409175872802734,
      "activations/layer8_attention_weight_min": -44.12062454223633,
      "activations/layer9_attention_weight_max": 79.48484802246094,
      "activations/layer9_attention_weight_min": -72.79431915283203,
      "epoch": 1.45,
      "learning_rate": 0.00014205984848484848,
      "loss": 3.041,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_loss": 2.966796875,
      "eval_runtime": 8.44,
      "eval_samples_per_second": 508.768,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_openwebtext_loss": 2.966796875,
      "eval_openwebtext_ppl": 19.429584428039178,
      "eval_openwebtext_runtime": 8.44,
      "eval_openwebtext_samples_per_second": 508.768,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_wikitext_loss": 3.2578125,
      "eval_wikitext_ppl": 25.992616060150944,
      "eval_wikitext_runtime": 1.7015,
      "eval_wikitext_samples_per_second": 268.004,
      "step": 25000
    },
    {
      "epoch": 1.45,
      "eval_lambada_loss": 3.23046875,
      "eval_lambada_ppl": 25.2915095879081,
      "eval_lambada_runtime": 9.2691,
      "eval_lambada_samples_per_second": 525.291,
      "step": 25000
    },
    {
      "activations/layer0_attention_weight_max": 13.33503246307373,
      "activations/layer0_attention_weight_min": -13.275644302368164,
      "activations/layer10_attention_weight_max": 67.768310546875,
      "activations/layer10_attention_weight_min": -61.330562591552734,
      "activations/layer11_attention_weight_max": 43.50385665893555,
      "activations/layer11_attention_weight_min": -38.618247985839844,
      "activations/layer12_attention_weight_max": 29.491840362548828,
      "activations/layer12_attention_weight_min": -27.27348518371582,
      "activations/layer13_attention_weight_max": 53.66667556762695,
      "activations/layer13_attention_weight_min": -47.515647888183594,
      "activations/layer14_attention_weight_max": 53.365745544433594,
      "activations/layer14_attention_weight_min": -38.10211944580078,
      "activations/layer15_attention_weight_max": 53.72344207763672,
      "activations/layer15_attention_weight_min": -41.546722412109375,
      "activations/layer16_attention_weight_max": 44.91583251953125,
      "activations/layer16_attention_weight_min": -33.48738479614258,
      "activations/layer17_attention_weight_max": 55.7397346496582,
      "activations/layer17_attention_weight_min": -38.380699157714844,
      "activations/layer18_attention_weight_max": 63.062435150146484,
      "activations/layer18_attention_weight_min": -40.07481384277344,
      "activations/layer19_attention_weight_max": 20.304929733276367,
      "activations/layer19_attention_weight_min": -14.189065933227539,
      "activations/layer1_attention_weight_max": 15.648919105529785,
      "activations/layer1_attention_weight_min": -12.151552200317383,
      "activations/layer20_attention_weight_max": 20.25959587097168,
      "activations/layer20_attention_weight_min": -15.200923919677734,
      "activations/layer21_attention_weight_max": 35.62370681762695,
      "activations/layer21_attention_weight_min": -23.618789672851562,
      "activations/layer22_attention_weight_max": 35.26656723022461,
      "activations/layer22_attention_weight_min": -21.420228958129883,
      "activations/layer23_attention_weight_max": 21.786985397338867,
      "activations/layer23_attention_weight_min": -13.46694564819336,
      "activations/layer2_attention_weight_max": 24.679988861083984,
      "activations/layer2_attention_weight_min": -23.96324920654297,
      "activations/layer3_attention_weight_max": 56.317020416259766,
      "activations/layer3_attention_weight_min": -57.968753814697266,
      "activations/layer4_attention_weight_max": 87.97602081298828,
      "activations/layer4_attention_weight_min": -80.53672790527344,
      "activations/layer5_attention_weight_max": 131.5513458251953,
      "activations/layer5_attention_weight_min": -119.53965759277344,
      "activations/layer6_attention_weight_max": 63.8464241027832,
      "activations/layer6_attention_weight_min": -58.4271125793457,
      "activations/layer7_attention_weight_max": 58.689937591552734,
      "activations/layer7_attention_weight_min": -54.31863021850586,
      "activations/layer8_attention_weight_max": 51.99696731567383,
      "activations/layer8_attention_weight_min": -44.07892990112305,
      "activations/layer9_attention_weight_max": 71.41725158691406,
      "activations/layer9_attention_weight_min": -62.18742752075195,
      "epoch": 1.46,
      "learning_rate": 0.0001420409090909091,
      "loss": 3.0286,
      "step": 25050
    },
    {
      "activations/layer0_attention_weight_max": 13.179295539855957,
      "activations/layer0_attention_weight_min": -12.757366180419922,
      "activations/layer10_attention_weight_max": 58.87687301635742,
      "activations/layer10_attention_weight_min": -52.93826675415039,
      "activations/layer11_attention_weight_max": 44.68589401245117,
      "activations/layer11_attention_weight_min": -36.09534454345703,
      "activations/layer12_attention_weight_max": 26.245546340942383,
      "activations/layer12_attention_weight_min": -24.13343048095703,
      "activations/layer13_attention_weight_max": 73.88468170166016,
      "activations/layer13_attention_weight_min": -50.40019607543945,
      "activations/layer14_attention_weight_max": 57.855751037597656,
      "activations/layer14_attention_weight_min": -42.6712646484375,
      "activations/layer15_attention_weight_max": 71.91483306884766,
      "activations/layer15_attention_weight_min": -46.3060188293457,
      "activations/layer16_attention_weight_max": 39.21123504638672,
      "activations/layer16_attention_weight_min": -38.49938201904297,
      "activations/layer17_attention_weight_max": 58.47257614135742,
      "activations/layer17_attention_weight_min": -44.23429870605469,
      "activations/layer18_attention_weight_max": 57.144744873046875,
      "activations/layer18_attention_weight_min": -37.454124450683594,
      "activations/layer19_attention_weight_max": 19.0158634185791,
      "activations/layer19_attention_weight_min": -14.4923734664917,
      "activations/layer1_attention_weight_max": 16.371746063232422,
      "activations/layer1_attention_weight_min": -13.967751502990723,
      "activations/layer20_attention_weight_max": 22.010438919067383,
      "activations/layer20_attention_weight_min": -16.503456115722656,
      "activations/layer21_attention_weight_max": 45.3990592956543,
      "activations/layer21_attention_weight_min": -24.44374656677246,
      "activations/layer22_attention_weight_max": 40.13469696044922,
      "activations/layer22_attention_weight_min": -24.750995635986328,
      "activations/layer23_attention_weight_max": 24.747594833374023,
      "activations/layer23_attention_weight_min": -16.095928192138672,
      "activations/layer2_attention_weight_max": 25.011642456054688,
      "activations/layer2_attention_weight_min": -24.825641632080078,
      "activations/layer3_attention_weight_max": 60.5438346862793,
      "activations/layer3_attention_weight_min": -62.8882942199707,
      "activations/layer4_attention_weight_max": 90.03660583496094,
      "activations/layer4_attention_weight_min": -79.25221252441406,
      "activations/layer5_attention_weight_max": 130.71710205078125,
      "activations/layer5_attention_weight_min": -119.15975189208984,
      "activations/layer6_attention_weight_max": 61.56427001953125,
      "activations/layer6_attention_weight_min": -56.756874084472656,
      "activations/layer7_attention_weight_max": 61.10108947753906,
      "activations/layer7_attention_weight_min": -57.163421630859375,
      "activations/layer8_attention_weight_max": 54.54941177368164,
      "activations/layer8_attention_weight_min": -49.2912483215332,
      "activations/layer9_attention_weight_max": 77.8327865600586,
      "activations/layer9_attention_weight_min": -80.43291473388672,
      "epoch": 1.46,
      "learning_rate": 0.00014202196969696968,
      "loss": 3.0227,
      "step": 25100
    },
    {
      "activations/layer0_attention_weight_max": 14.690009117126465,
      "activations/layer0_attention_weight_min": -12.930167198181152,
      "activations/layer10_attention_weight_max": 70.22435760498047,
      "activations/layer10_attention_weight_min": -58.55125045776367,
      "activations/layer11_attention_weight_max": 48.66779327392578,
      "activations/layer11_attention_weight_min": -42.52526092529297,
      "activations/layer12_attention_weight_max": 33.708770751953125,
      "activations/layer12_attention_weight_min": -28.229307174682617,
      "activations/layer13_attention_weight_max": 73.23675537109375,
      "activations/layer13_attention_weight_min": -52.872554779052734,
      "activations/layer14_attention_weight_max": 53.128448486328125,
      "activations/layer14_attention_weight_min": -39.47880172729492,
      "activations/layer15_attention_weight_max": 61.45764923095703,
      "activations/layer15_attention_weight_min": -43.6530647277832,
      "activations/layer16_attention_weight_max": 49.10801696777344,
      "activations/layer16_attention_weight_min": -34.74225616455078,
      "activations/layer17_attention_weight_max": 67.88858795166016,
      "activations/layer17_attention_weight_min": -40.74653244018555,
      "activations/layer18_attention_weight_max": 65.12608337402344,
      "activations/layer18_attention_weight_min": -35.44363784790039,
      "activations/layer19_attention_weight_max": 22.891704559326172,
      "activations/layer19_attention_weight_min": -13.355049133300781,
      "activations/layer1_attention_weight_max": 15.93498420715332,
      "activations/layer1_attention_weight_min": -13.29824161529541,
      "activations/layer20_attention_weight_max": 22.13561248779297,
      "activations/layer20_attention_weight_min": -15.709786415100098,
      "activations/layer21_attention_weight_max": 42.587745666503906,
      "activations/layer21_attention_weight_min": -20.2296199798584,
      "activations/layer22_attention_weight_max": 32.121822357177734,
      "activations/layer22_attention_weight_min": -18.86347198486328,
      "activations/layer23_attention_weight_max": 20.467777252197266,
      "activations/layer23_attention_weight_min": -12.99624252319336,
      "activations/layer2_attention_weight_max": 24.894683837890625,
      "activations/layer2_attention_weight_min": -24.896583557128906,
      "activations/layer3_attention_weight_max": 57.87959671020508,
      "activations/layer3_attention_weight_min": -59.263057708740234,
      "activations/layer4_attention_weight_max": 86.8899154663086,
      "activations/layer4_attention_weight_min": -79.49295806884766,
      "activations/layer5_attention_weight_max": 129.45982360839844,
      "activations/layer5_attention_weight_min": -123.81770324707031,
      "activations/layer6_attention_weight_max": 61.6316032409668,
      "activations/layer6_attention_weight_min": -55.96597671508789,
      "activations/layer7_attention_weight_max": 58.462562561035156,
      "activations/layer7_attention_weight_min": -53.282867431640625,
      "activations/layer8_attention_weight_max": 48.586647033691406,
      "activations/layer8_attention_weight_min": -45.213287353515625,
      "activations/layer9_attention_weight_max": 72.09544372558594,
      "activations/layer9_attention_weight_min": -70.47154998779297,
      "epoch": 1.46,
      "learning_rate": 0.0001420030303030303,
      "loss": 3.0274,
      "step": 25150
    },
    {
      "activations/layer0_attention_weight_max": 13.2606201171875,
      "activations/layer0_attention_weight_min": -12.82276725769043,
      "activations/layer10_attention_weight_max": 64.21485137939453,
      "activations/layer10_attention_weight_min": -59.93501663208008,
      "activations/layer11_attention_weight_max": 47.546932220458984,
      "activations/layer11_attention_weight_min": -37.51216125488281,
      "activations/layer12_attention_weight_max": 27.801755905151367,
      "activations/layer12_attention_weight_min": -24.15585708618164,
      "activations/layer13_attention_weight_max": 52.371009826660156,
      "activations/layer13_attention_weight_min": -42.56181335449219,
      "activations/layer14_attention_weight_max": 51.839515686035156,
      "activations/layer14_attention_weight_min": -40.133323669433594,
      "activations/layer15_attention_weight_max": 49.96877670288086,
      "activations/layer15_attention_weight_min": -39.66838455200195,
      "activations/layer16_attention_weight_max": 37.79872131347656,
      "activations/layer16_attention_weight_min": -31.64067268371582,
      "activations/layer17_attention_weight_max": 49.15628433227539,
      "activations/layer17_attention_weight_min": -35.92575454711914,
      "activations/layer18_attention_weight_max": 56.19313430786133,
      "activations/layer18_attention_weight_min": -33.28318786621094,
      "activations/layer19_attention_weight_max": 18.562793731689453,
      "activations/layer19_attention_weight_min": -14.078581809997559,
      "activations/layer1_attention_weight_max": 15.965887069702148,
      "activations/layer1_attention_weight_min": -11.629222869873047,
      "activations/layer20_attention_weight_max": 21.59256362915039,
      "activations/layer20_attention_weight_min": -18.853336334228516,
      "activations/layer21_attention_weight_max": 34.347564697265625,
      "activations/layer21_attention_weight_min": -21.098377227783203,
      "activations/layer22_attention_weight_max": 36.174720764160156,
      "activations/layer22_attention_weight_min": -20.681556701660156,
      "activations/layer23_attention_weight_max": 23.813634872436523,
      "activations/layer23_attention_weight_min": -12.921408653259277,
      "activations/layer2_attention_weight_max": 25.093740463256836,
      "activations/layer2_attention_weight_min": -25.00423240661621,
      "activations/layer3_attention_weight_max": 55.30412673950195,
      "activations/layer3_attention_weight_min": -57.8958740234375,
      "activations/layer4_attention_weight_max": 85.6212158203125,
      "activations/layer4_attention_weight_min": -78.27554321289062,
      "activations/layer5_attention_weight_max": 136.13021850585938,
      "activations/layer5_attention_weight_min": -117.81033325195312,
      "activations/layer6_attention_weight_max": 64.73072814941406,
      "activations/layer6_attention_weight_min": -56.30846405029297,
      "activations/layer7_attention_weight_max": 63.639007568359375,
      "activations/layer7_attention_weight_min": -58.74245071411133,
      "activations/layer8_attention_weight_max": 54.238739013671875,
      "activations/layer8_attention_weight_min": -45.13642883300781,
      "activations/layer9_attention_weight_max": 70.44751739501953,
      "activations/layer9_attention_weight_min": -67.67205810546875,
      "epoch": 1.46,
      "learning_rate": 0.0001419840909090909,
      "loss": 3.028,
      "step": 25200
    },
    {
      "activations/layer0_attention_weight_max": 13.485732078552246,
      "activations/layer0_attention_weight_min": -12.57767105102539,
      "activations/layer10_attention_weight_max": 65.8372573852539,
      "activations/layer10_attention_weight_min": -53.81730651855469,
      "activations/layer11_attention_weight_max": 45.158321380615234,
      "activations/layer11_attention_weight_min": -37.65727996826172,
      "activations/layer12_attention_weight_max": 28.725614547729492,
      "activations/layer12_attention_weight_min": -24.50513458251953,
      "activations/layer13_attention_weight_max": 62.68943405151367,
      "activations/layer13_attention_weight_min": -45.347557067871094,
      "activations/layer14_attention_weight_max": 52.185420989990234,
      "activations/layer14_attention_weight_min": -40.56724548339844,
      "activations/layer15_attention_weight_max": 55.66228485107422,
      "activations/layer15_attention_weight_min": -40.77499771118164,
      "activations/layer16_attention_weight_max": 46.882835388183594,
      "activations/layer16_attention_weight_min": -32.12019729614258,
      "activations/layer17_attention_weight_max": 53.648216247558594,
      "activations/layer17_attention_weight_min": -35.5156364440918,
      "activations/layer18_attention_weight_max": 57.51818084716797,
      "activations/layer18_attention_weight_min": -35.335693359375,
      "activations/layer19_attention_weight_max": 22.337438583374023,
      "activations/layer19_attention_weight_min": -14.693582534790039,
      "activations/layer1_attention_weight_max": 15.114001274108887,
      "activations/layer1_attention_weight_min": -12.128216743469238,
      "activations/layer20_attention_weight_max": 21.745967864990234,
      "activations/layer20_attention_weight_min": -19.06244659423828,
      "activations/layer21_attention_weight_max": 42.61132049560547,
      "activations/layer21_attention_weight_min": -22.1090030670166,
      "activations/layer22_attention_weight_max": 38.70512771606445,
      "activations/layer22_attention_weight_min": -20.217559814453125,
      "activations/layer23_attention_weight_max": 24.072284698486328,
      "activations/layer23_attention_weight_min": -15.805364608764648,
      "activations/layer2_attention_weight_max": 24.429576873779297,
      "activations/layer2_attention_weight_min": -24.539045333862305,
      "activations/layer3_attention_weight_max": 53.86207962036133,
      "activations/layer3_attention_weight_min": -58.97789764404297,
      "activations/layer4_attention_weight_max": 84.23306274414062,
      "activations/layer4_attention_weight_min": -79.01689147949219,
      "activations/layer5_attention_weight_max": 124.38341522216797,
      "activations/layer5_attention_weight_min": -117.88546752929688,
      "activations/layer6_attention_weight_max": 59.97505569458008,
      "activations/layer6_attention_weight_min": -53.466880798339844,
      "activations/layer7_attention_weight_max": 57.139286041259766,
      "activations/layer7_attention_weight_min": -52.402366638183594,
      "activations/layer8_attention_weight_max": 50.73721694946289,
      "activations/layer8_attention_weight_min": -43.242652893066406,
      "activations/layer9_attention_weight_max": 64.39006805419922,
      "activations/layer9_attention_weight_min": -59.622474670410156,
      "epoch": 1.47,
      "learning_rate": 0.0001419651515151515,
      "loss": 3.01,
      "step": 25250
    },
    {
      "activations/layer0_attention_weight_max": 14.109501838684082,
      "activations/layer0_attention_weight_min": -12.7965669631958,
      "activations/layer10_attention_weight_max": 69.7038345336914,
      "activations/layer10_attention_weight_min": -56.30099868774414,
      "activations/layer11_attention_weight_max": 42.51244354248047,
      "activations/layer11_attention_weight_min": -37.61653137207031,
      "activations/layer12_attention_weight_max": 27.720666885375977,
      "activations/layer12_attention_weight_min": -27.059179306030273,
      "activations/layer13_attention_weight_max": 56.24550247192383,
      "activations/layer13_attention_weight_min": -41.57887268066406,
      "activations/layer14_attention_weight_max": 41.75121307373047,
      "activations/layer14_attention_weight_min": -32.871131896972656,
      "activations/layer15_attention_weight_max": 53.47962188720703,
      "activations/layer15_attention_weight_min": -40.018585205078125,
      "activations/layer16_attention_weight_max": 43.9431266784668,
      "activations/layer16_attention_weight_min": -33.1064338684082,
      "activations/layer17_attention_weight_max": 60.1194953918457,
      "activations/layer17_attention_weight_min": -41.69731140136719,
      "activations/layer18_attention_weight_max": 62.09909439086914,
      "activations/layer18_attention_weight_min": -36.219337463378906,
      "activations/layer19_attention_weight_max": 21.53896713256836,
      "activations/layer19_attention_weight_min": -15.214034080505371,
      "activations/layer1_attention_weight_max": 15.221723556518555,
      "activations/layer1_attention_weight_min": -13.217288970947266,
      "activations/layer20_attention_weight_max": 20.093242645263672,
      "activations/layer20_attention_weight_min": -15.077117919921875,
      "activations/layer21_attention_weight_max": 38.005760192871094,
      "activations/layer21_attention_weight_min": -22.416105270385742,
      "activations/layer22_attention_weight_max": 38.02151870727539,
      "activations/layer22_attention_weight_min": -20.496318817138672,
      "activations/layer23_attention_weight_max": 27.21255111694336,
      "activations/layer23_attention_weight_min": -16.768789291381836,
      "activations/layer2_attention_weight_max": 25.092178344726562,
      "activations/layer2_attention_weight_min": -24.47911834716797,
      "activations/layer3_attention_weight_max": 58.32865905761719,
      "activations/layer3_attention_weight_min": -63.074180603027344,
      "activations/layer4_attention_weight_max": 88.61762237548828,
      "activations/layer4_attention_weight_min": -79.36598205566406,
      "activations/layer5_attention_weight_max": 128.00302124023438,
      "activations/layer5_attention_weight_min": -125.79705810546875,
      "activations/layer6_attention_weight_max": 60.17191696166992,
      "activations/layer6_attention_weight_min": -54.84511947631836,
      "activations/layer7_attention_weight_max": 58.468467712402344,
      "activations/layer7_attention_weight_min": -54.55085372924805,
      "activations/layer8_attention_weight_max": 48.037620544433594,
      "activations/layer8_attention_weight_min": -43.11688232421875,
      "activations/layer9_attention_weight_max": 74.0020980834961,
      "activations/layer9_attention_weight_min": -66.39031219482422,
      "epoch": 1.47,
      "learning_rate": 0.00014194621212121211,
      "loss": 3.0235,
      "step": 25300
    },
    {
      "activations/layer0_attention_weight_max": 16.289142608642578,
      "activations/layer0_attention_weight_min": -12.799805641174316,
      "activations/layer10_attention_weight_max": 64.63151550292969,
      "activations/layer10_attention_weight_min": -60.500057220458984,
      "activations/layer11_attention_weight_max": 44.67334747314453,
      "activations/layer11_attention_weight_min": -35.951751708984375,
      "activations/layer12_attention_weight_max": 28.30016326904297,
      "activations/layer12_attention_weight_min": -24.70475959777832,
      "activations/layer13_attention_weight_max": 66.65824890136719,
      "activations/layer13_attention_weight_min": -46.80287551879883,
      "activations/layer14_attention_weight_max": 48.69552230834961,
      "activations/layer14_attention_weight_min": -33.2879524230957,
      "activations/layer15_attention_weight_max": 60.65061569213867,
      "activations/layer15_attention_weight_min": -42.962913513183594,
      "activations/layer16_attention_weight_max": 45.71799087524414,
      "activations/layer16_attention_weight_min": -29.024850845336914,
      "activations/layer17_attention_weight_max": 58.1480712890625,
      "activations/layer17_attention_weight_min": -37.0388298034668,
      "activations/layer18_attention_weight_max": 58.097721099853516,
      "activations/layer18_attention_weight_min": -34.75979995727539,
      "activations/layer19_attention_weight_max": 21.276552200317383,
      "activations/layer19_attention_weight_min": -13.449374198913574,
      "activations/layer1_attention_weight_max": 15.825407981872559,
      "activations/layer1_attention_weight_min": -13.057130813598633,
      "activations/layer20_attention_weight_max": 19.678844451904297,
      "activations/layer20_attention_weight_min": -14.811382293701172,
      "activations/layer21_attention_weight_max": 37.81544876098633,
      "activations/layer21_attention_weight_min": -19.63709831237793,
      "activations/layer22_attention_weight_max": 34.600067138671875,
      "activations/layer22_attention_weight_min": -22.007719039916992,
      "activations/layer23_attention_weight_max": 26.459827423095703,
      "activations/layer23_attention_weight_min": -16.11146354675293,
      "activations/layer2_attention_weight_max": 24.15509796142578,
      "activations/layer2_attention_weight_min": -23.95168113708496,
      "activations/layer3_attention_weight_max": 54.737911224365234,
      "activations/layer3_attention_weight_min": -59.6744499206543,
      "activations/layer4_attention_weight_max": 85.1661148071289,
      "activations/layer4_attention_weight_min": -79.08771514892578,
      "activations/layer5_attention_weight_max": 128.65243530273438,
      "activations/layer5_attention_weight_min": -119.33978271484375,
      "activations/layer6_attention_weight_max": 61.359867095947266,
      "activations/layer6_attention_weight_min": -56.57011413574219,
      "activations/layer7_attention_weight_max": 61.75616455078125,
      "activations/layer7_attention_weight_min": -56.046043395996094,
      "activations/layer8_attention_weight_max": 51.92252731323242,
      "activations/layer8_attention_weight_min": -44.69719696044922,
      "activations/layer9_attention_weight_max": 76.93331146240234,
      "activations/layer9_attention_weight_min": -64.06157684326172,
      "epoch": 1.47,
      "learning_rate": 0.00014192727272727273,
      "loss": 3.0143,
      "step": 25350
    },
    {
      "activations/layer0_attention_weight_max": 14.383920669555664,
      "activations/layer0_attention_weight_min": -12.948981285095215,
      "activations/layer10_attention_weight_max": 60.08740997314453,
      "activations/layer10_attention_weight_min": -56.848388671875,
      "activations/layer11_attention_weight_max": 38.272430419921875,
      "activations/layer11_attention_weight_min": -36.363285064697266,
      "activations/layer12_attention_weight_max": 25.878101348876953,
      "activations/layer12_attention_weight_min": -24.25248908996582,
      "activations/layer13_attention_weight_max": 60.2279052734375,
      "activations/layer13_attention_weight_min": -44.15070724487305,
      "activations/layer14_attention_weight_max": 46.954345703125,
      "activations/layer14_attention_weight_min": -42.06069564819336,
      "activations/layer15_attention_weight_max": 45.36396789550781,
      "activations/layer15_attention_weight_min": -37.57167053222656,
      "activations/layer16_attention_weight_max": 39.93976593017578,
      "activations/layer16_attention_weight_min": -31.17106056213379,
      "activations/layer17_attention_weight_max": 51.80148696899414,
      "activations/layer17_attention_weight_min": -38.0545654296875,
      "activations/layer18_attention_weight_max": 54.51137924194336,
      "activations/layer18_attention_weight_min": -37.886348724365234,
      "activations/layer19_attention_weight_max": 18.9699764251709,
      "activations/layer19_attention_weight_min": -17.445741653442383,
      "activations/layer1_attention_weight_max": 16.412189483642578,
      "activations/layer1_attention_weight_min": -12.293209075927734,
      "activations/layer20_attention_weight_max": 18.446102142333984,
      "activations/layer20_attention_weight_min": -16.93929672241211,
      "activations/layer21_attention_weight_max": 36.56355285644531,
      "activations/layer21_attention_weight_min": -19.426563262939453,
      "activations/layer22_attention_weight_max": 36.42392349243164,
      "activations/layer22_attention_weight_min": -20.688846588134766,
      "activations/layer23_attention_weight_max": 22.85528564453125,
      "activations/layer23_attention_weight_min": -15.419161796569824,
      "activations/layer2_attention_weight_max": 24.732954025268555,
      "activations/layer2_attention_weight_min": -25.534513473510742,
      "activations/layer3_attention_weight_max": 57.22811508178711,
      "activations/layer3_attention_weight_min": -60.45930480957031,
      "activations/layer4_attention_weight_max": 81.96874237060547,
      "activations/layer4_attention_weight_min": -80.21050262451172,
      "activations/layer5_attention_weight_max": 124.4315185546875,
      "activations/layer5_attention_weight_min": -121.22183227539062,
      "activations/layer6_attention_weight_max": 60.05347442626953,
      "activations/layer6_attention_weight_min": -56.32041549682617,
      "activations/layer7_attention_weight_max": 57.865352630615234,
      "activations/layer7_attention_weight_min": -58.80024337768555,
      "activations/layer8_attention_weight_max": 46.80543899536133,
      "activations/layer8_attention_weight_min": -42.318328857421875,
      "activations/layer9_attention_weight_max": 71.18907928466797,
      "activations/layer9_attention_weight_min": -72.0952377319336,
      "epoch": 1.48,
      "learning_rate": 0.00014190833333333332,
      "loss": 3.0289,
      "step": 25400
    },
    {
      "activations/layer0_attention_weight_max": 14.859107971191406,
      "activations/layer0_attention_weight_min": -13.381587028503418,
      "activations/layer10_attention_weight_max": 63.602989196777344,
      "activations/layer10_attention_weight_min": -57.59455108642578,
      "activations/layer11_attention_weight_max": 42.348663330078125,
      "activations/layer11_attention_weight_min": -37.669612884521484,
      "activations/layer12_attention_weight_max": 29.695396423339844,
      "activations/layer12_attention_weight_min": -25.15074348449707,
      "activations/layer13_attention_weight_max": 64.7913589477539,
      "activations/layer13_attention_weight_min": -46.11301803588867,
      "activations/layer14_attention_weight_max": 46.12180709838867,
      "activations/layer14_attention_weight_min": -32.564369201660156,
      "activations/layer15_attention_weight_max": 59.4844970703125,
      "activations/layer15_attention_weight_min": -42.610599517822266,
      "activations/layer16_attention_weight_max": 45.66902542114258,
      "activations/layer16_attention_weight_min": -31.104324340820312,
      "activations/layer17_attention_weight_max": 53.04901885986328,
      "activations/layer17_attention_weight_min": -37.959041595458984,
      "activations/layer18_attention_weight_max": 55.91018295288086,
      "activations/layer18_attention_weight_min": -34.995357513427734,
      "activations/layer19_attention_weight_max": 18.89886474609375,
      "activations/layer19_attention_weight_min": -13.76676082611084,
      "activations/layer1_attention_weight_max": 15.436823844909668,
      "activations/layer1_attention_weight_min": -11.98789119720459,
      "activations/layer20_attention_weight_max": 19.374958038330078,
      "activations/layer20_attention_weight_min": -16.599660873413086,
      "activations/layer21_attention_weight_max": 34.53399658203125,
      "activations/layer21_attention_weight_min": -20.667678833007812,
      "activations/layer22_attention_weight_max": 32.35686111450195,
      "activations/layer22_attention_weight_min": -19.666906356811523,
      "activations/layer23_attention_weight_max": 22.396194458007812,
      "activations/layer23_attention_weight_min": -15.385454177856445,
      "activations/layer2_attention_weight_max": 24.87770652770996,
      "activations/layer2_attention_weight_min": -24.397878646850586,
      "activations/layer3_attention_weight_max": 56.53889083862305,
      "activations/layer3_attention_weight_min": -58.580772399902344,
      "activations/layer4_attention_weight_max": 83.4227523803711,
      "activations/layer4_attention_weight_min": -80.67745971679688,
      "activations/layer5_attention_weight_max": 126.90166473388672,
      "activations/layer5_attention_weight_min": -121.32476806640625,
      "activations/layer6_attention_weight_max": 63.889305114746094,
      "activations/layer6_attention_weight_min": -58.50260925292969,
      "activations/layer7_attention_weight_max": 61.14824676513672,
      "activations/layer7_attention_weight_min": -54.32713317871094,
      "activations/layer8_attention_weight_max": 49.138851165771484,
      "activations/layer8_attention_weight_min": -47.55347442626953,
      "activations/layer9_attention_weight_max": 66.63780975341797,
      "activations/layer9_attention_weight_min": -62.84891891479492,
      "epoch": 1.48,
      "learning_rate": 0.0001418897727272727,
      "loss": 3.0246,
      "step": 25450
    },
    {
      "activations/layer0_attention_weight_max": 13.73508071899414,
      "activations/layer0_attention_weight_min": -12.921516418457031,
      "activations/layer10_attention_weight_max": 67.28376007080078,
      "activations/layer10_attention_weight_min": -52.206329345703125,
      "activations/layer11_attention_weight_max": 44.752986907958984,
      "activations/layer11_attention_weight_min": -35.7479248046875,
      "activations/layer12_attention_weight_max": 32.678367614746094,
      "activations/layer12_attention_weight_min": -26.004642486572266,
      "activations/layer13_attention_weight_max": 53.68449401855469,
      "activations/layer13_attention_weight_min": -39.732784271240234,
      "activations/layer14_attention_weight_max": 47.453941345214844,
      "activations/layer14_attention_weight_min": -37.82235336303711,
      "activations/layer15_attention_weight_max": 53.759151458740234,
      "activations/layer15_attention_weight_min": -39.897464752197266,
      "activations/layer16_attention_weight_max": 45.24690246582031,
      "activations/layer16_attention_weight_min": -29.971826553344727,
      "activations/layer17_attention_weight_max": 63.81364059448242,
      "activations/layer17_attention_weight_min": -34.796749114990234,
      "activations/layer18_attention_weight_max": 56.6085205078125,
      "activations/layer18_attention_weight_min": -34.716156005859375,
      "activations/layer19_attention_weight_max": 20.553794860839844,
      "activations/layer19_attention_weight_min": -13.894986152648926,
      "activations/layer1_attention_weight_max": 15.85447883605957,
      "activations/layer1_attention_weight_min": -12.498873710632324,
      "activations/layer20_attention_weight_max": 20.97893524169922,
      "activations/layer20_attention_weight_min": -16.142900466918945,
      "activations/layer21_attention_weight_max": 39.33987045288086,
      "activations/layer21_attention_weight_min": -23.863462448120117,
      "activations/layer22_attention_weight_max": 31.93723487854004,
      "activations/layer22_attention_weight_min": -18.80954360961914,
      "activations/layer23_attention_weight_max": 20.0231990814209,
      "activations/layer23_attention_weight_min": -14.544955253601074,
      "activations/layer2_attention_weight_max": 23.907325744628906,
      "activations/layer2_attention_weight_min": -22.79306411743164,
      "activations/layer3_attention_weight_max": 57.446624755859375,
      "activations/layer3_attention_weight_min": -59.280494689941406,
      "activations/layer4_attention_weight_max": 89.31417846679688,
      "activations/layer4_attention_weight_min": -79.89399719238281,
      "activations/layer5_attention_weight_max": 133.79405212402344,
      "activations/layer5_attention_weight_min": -116.35763549804688,
      "activations/layer6_attention_weight_max": 68.4110107421875,
      "activations/layer6_attention_weight_min": -57.3439826965332,
      "activations/layer7_attention_weight_max": 65.08489990234375,
      "activations/layer7_attention_weight_min": -54.109500885009766,
      "activations/layer8_attention_weight_max": 52.72758865356445,
      "activations/layer8_attention_weight_min": -46.243438720703125,
      "activations/layer9_attention_weight_max": 70.03844451904297,
      "activations/layer9_attention_weight_min": -57.83004379272461,
      "epoch": 1.48,
      "learning_rate": 0.00014187083333333332,
      "loss": 3.0316,
      "step": 25500
    },
    {
      "activations/layer0_attention_weight_max": 14.343290328979492,
      "activations/layer0_attention_weight_min": -13.098843574523926,
      "activations/layer10_attention_weight_max": 67.30885314941406,
      "activations/layer10_attention_weight_min": -61.116180419921875,
      "activations/layer11_attention_weight_max": 47.19694137573242,
      "activations/layer11_attention_weight_min": -39.329986572265625,
      "activations/layer12_attention_weight_max": 33.28493881225586,
      "activations/layer12_attention_weight_min": -25.71562957763672,
      "activations/layer13_attention_weight_max": 61.6330680847168,
      "activations/layer13_attention_weight_min": -41.54962921142578,
      "activations/layer14_attention_weight_max": 60.470115661621094,
      "activations/layer14_attention_weight_min": -36.48603439331055,
      "activations/layer15_attention_weight_max": 51.83255386352539,
      "activations/layer15_attention_weight_min": -40.3605842590332,
      "activations/layer16_attention_weight_max": 50.39916229248047,
      "activations/layer16_attention_weight_min": -32.66592025756836,
      "activations/layer17_attention_weight_max": 61.07716369628906,
      "activations/layer17_attention_weight_min": -34.50860595703125,
      "activations/layer18_attention_weight_max": 69.0826187133789,
      "activations/layer18_attention_weight_min": -38.91424560546875,
      "activations/layer19_attention_weight_max": 22.766931533813477,
      "activations/layer19_attention_weight_min": -14.246070861816406,
      "activations/layer1_attention_weight_max": 15.939702033996582,
      "activations/layer1_attention_weight_min": -12.774375915527344,
      "activations/layer20_attention_weight_max": 21.064517974853516,
      "activations/layer20_attention_weight_min": -13.280442237854004,
      "activations/layer21_attention_weight_max": 38.1954460144043,
      "activations/layer21_attention_weight_min": -21.133255004882812,
      "activations/layer22_attention_weight_max": 38.37787628173828,
      "activations/layer22_attention_weight_min": -19.89714241027832,
      "activations/layer23_attention_weight_max": 24.687946319580078,
      "activations/layer23_attention_weight_min": -14.669268608093262,
      "activations/layer2_attention_weight_max": 25.06545639038086,
      "activations/layer2_attention_weight_min": -24.675796508789062,
      "activations/layer3_attention_weight_max": 56.40879440307617,
      "activations/layer3_attention_weight_min": -58.49020004272461,
      "activations/layer4_attention_weight_max": 86.89969635009766,
      "activations/layer4_attention_weight_min": -83.35494232177734,
      "activations/layer5_attention_weight_max": 133.4593963623047,
      "activations/layer5_attention_weight_min": -119.548583984375,
      "activations/layer6_attention_weight_max": 64.68731689453125,
      "activations/layer6_attention_weight_min": -54.08032989501953,
      "activations/layer7_attention_weight_max": 63.20772933959961,
      "activations/layer7_attention_weight_min": -54.43085479736328,
      "activations/layer8_attention_weight_max": 53.760276794433594,
      "activations/layer8_attention_weight_min": -43.76169204711914,
      "activations/layer9_attention_weight_max": 74.08219909667969,
      "activations/layer9_attention_weight_min": -62.88148880004883,
      "epoch": 1.48,
      "learning_rate": 0.00014185189393939394,
      "loss": 3.0356,
      "step": 25550
    },
    {
      "activations/layer0_attention_weight_max": 14.029581069946289,
      "activations/layer0_attention_weight_min": -13.09369945526123,
      "activations/layer10_attention_weight_max": 69.82474517822266,
      "activations/layer10_attention_weight_min": -58.04661178588867,
      "activations/layer11_attention_weight_max": 46.20135498046875,
      "activations/layer11_attention_weight_min": -34.22409439086914,
      "activations/layer12_attention_weight_max": 28.424240112304688,
      "activations/layer12_attention_weight_min": -23.488235473632812,
      "activations/layer13_attention_weight_max": 51.60593795776367,
      "activations/layer13_attention_weight_min": -38.88648986816406,
      "activations/layer14_attention_weight_max": 51.21319580078125,
      "activations/layer14_attention_weight_min": -33.89934158325195,
      "activations/layer15_attention_weight_max": 54.29631423950195,
      "activations/layer15_attention_weight_min": -39.380126953125,
      "activations/layer16_attention_weight_max": 44.21036911010742,
      "activations/layer16_attention_weight_min": -30.864660263061523,
      "activations/layer17_attention_weight_max": 60.47488021850586,
      "activations/layer17_attention_weight_min": -37.48406982421875,
      "activations/layer18_attention_weight_max": 66.85499572753906,
      "activations/layer18_attention_weight_min": -41.3441047668457,
      "activations/layer19_attention_weight_max": 25.762149810791016,
      "activations/layer19_attention_weight_min": -16.611360549926758,
      "activations/layer1_attention_weight_max": 16.009925842285156,
      "activations/layer1_attention_weight_min": -11.822949409484863,
      "activations/layer20_attention_weight_max": 18.98326873779297,
      "activations/layer20_attention_weight_min": -15.694061279296875,
      "activations/layer21_attention_weight_max": 49.92485427856445,
      "activations/layer21_attention_weight_min": -22.895849227905273,
      "activations/layer22_attention_weight_max": 41.472625732421875,
      "activations/layer22_attention_weight_min": -19.225658416748047,
      "activations/layer23_attention_weight_max": 26.660192489624023,
      "activations/layer23_attention_weight_min": -14.540384292602539,
      "activations/layer2_attention_weight_max": 24.971813201904297,
      "activations/layer2_attention_weight_min": -23.733692169189453,
      "activations/layer3_attention_weight_max": 58.22703552246094,
      "activations/layer3_attention_weight_min": -59.73289489746094,
      "activations/layer4_attention_weight_max": 86.6706771850586,
      "activations/layer4_attention_weight_min": -78.48509979248047,
      "activations/layer5_attention_weight_max": 123.72649383544922,
      "activations/layer5_attention_weight_min": -116.27873992919922,
      "activations/layer6_attention_weight_max": 61.976444244384766,
      "activations/layer6_attention_weight_min": -53.96566390991211,
      "activations/layer7_attention_weight_max": 58.89421463012695,
      "activations/layer7_attention_weight_min": -51.25682830810547,
      "activations/layer8_attention_weight_max": 48.7923698425293,
      "activations/layer8_attention_weight_min": -44.77669143676758,
      "activations/layer9_attention_weight_max": 65.14165496826172,
      "activations/layer9_attention_weight_min": -59.18438720703125,
      "epoch": 1.49,
      "learning_rate": 0.00014183295454545453,
      "loss": 3.0012,
      "step": 25600
    },
    {
      "activations/layer0_attention_weight_max": 12.780841827392578,
      "activations/layer0_attention_weight_min": -12.992974281311035,
      "activations/layer10_attention_weight_max": 76.01615142822266,
      "activations/layer10_attention_weight_min": -57.457969665527344,
      "activations/layer11_attention_weight_max": 60.414344787597656,
      "activations/layer11_attention_weight_min": -41.53070068359375,
      "activations/layer12_attention_weight_max": 27.56597328186035,
      "activations/layer12_attention_weight_min": -26.44892120361328,
      "activations/layer13_attention_weight_max": 65.04984283447266,
      "activations/layer13_attention_weight_min": -42.01554870605469,
      "activations/layer14_attention_weight_max": 53.000308990478516,
      "activations/layer14_attention_weight_min": -39.94204330444336,
      "activations/layer15_attention_weight_max": 53.123695373535156,
      "activations/layer15_attention_weight_min": -39.463279724121094,
      "activations/layer16_attention_weight_max": 46.468997955322266,
      "activations/layer16_attention_weight_min": -33.26831817626953,
      "activations/layer17_attention_weight_max": 60.01586151123047,
      "activations/layer17_attention_weight_min": -41.21632766723633,
      "activations/layer18_attention_weight_max": 59.579917907714844,
      "activations/layer18_attention_weight_min": -36.122127532958984,
      "activations/layer19_attention_weight_max": 19.08129119873047,
      "activations/layer19_attention_weight_min": -16.352405548095703,
      "activations/layer1_attention_weight_max": 15.566452980041504,
      "activations/layer1_attention_weight_min": -10.33623218536377,
      "activations/layer20_attention_weight_max": 19.763622283935547,
      "activations/layer20_attention_weight_min": -18.264549255371094,
      "activations/layer21_attention_weight_max": 39.00313186645508,
      "activations/layer21_attention_weight_min": -22.15212059020996,
      "activations/layer22_attention_weight_max": 35.99287033081055,
      "activations/layer22_attention_weight_min": -19.486997604370117,
      "activations/layer23_attention_weight_max": 21.75926971435547,
      "activations/layer23_attention_weight_min": -13.23969841003418,
      "activations/layer2_attention_weight_max": 26.95376205444336,
      "activations/layer2_attention_weight_min": -26.44198226928711,
      "activations/layer3_attention_weight_max": 58.22978973388672,
      "activations/layer3_attention_weight_min": -60.3806266784668,
      "activations/layer4_attention_weight_max": 93.2114028930664,
      "activations/layer4_attention_weight_min": -80.23681640625,
      "activations/layer5_attention_weight_max": 142.178955078125,
      "activations/layer5_attention_weight_min": -129.5843963623047,
      "activations/layer6_attention_weight_max": 64.80440521240234,
      "activations/layer6_attention_weight_min": -58.00932693481445,
      "activations/layer7_attention_weight_max": 66.44086456298828,
      "activations/layer7_attention_weight_min": -56.08660125732422,
      "activations/layer8_attention_weight_max": 69.26036071777344,
      "activations/layer8_attention_weight_min": -53.122859954833984,
      "activations/layer9_attention_weight_max": 84.33915710449219,
      "activations/layer9_attention_weight_min": -68.58828735351562,
      "epoch": 1.49,
      "learning_rate": 0.00014181401515151514,
      "loss": 3.0253,
      "step": 25650
    },
    {
      "activations/layer0_attention_weight_max": 15.63234806060791,
      "activations/layer0_attention_weight_min": -13.239603042602539,
      "activations/layer10_attention_weight_max": 64.41719055175781,
      "activations/layer10_attention_weight_min": -56.22675704956055,
      "activations/layer11_attention_weight_max": 45.320709228515625,
      "activations/layer11_attention_weight_min": -42.61857604980469,
      "activations/layer12_attention_weight_max": 30.39055633544922,
      "activations/layer12_attention_weight_min": -24.862171173095703,
      "activations/layer13_attention_weight_max": 68.13214111328125,
      "activations/layer13_attention_weight_min": -49.59144973754883,
      "activations/layer14_attention_weight_max": 64.19937896728516,
      "activations/layer14_attention_weight_min": -44.72187042236328,
      "activations/layer15_attention_weight_max": 60.56912612915039,
      "activations/layer15_attention_weight_min": -43.10680389404297,
      "activations/layer16_attention_weight_max": 49.62301254272461,
      "activations/layer16_attention_weight_min": -36.580204010009766,
      "activations/layer17_attention_weight_max": 56.9146614074707,
      "activations/layer17_attention_weight_min": -40.88936996459961,
      "activations/layer18_attention_weight_max": 57.76247787475586,
      "activations/layer18_attention_weight_min": -38.16628646850586,
      "activations/layer19_attention_weight_max": 20.833818435668945,
      "activations/layer19_attention_weight_min": -14.036381721496582,
      "activations/layer1_attention_weight_max": 15.4689302444458,
      "activations/layer1_attention_weight_min": -13.8887300491333,
      "activations/layer20_attention_weight_max": 19.778636932373047,
      "activations/layer20_attention_weight_min": -15.012100219726562,
      "activations/layer21_attention_weight_max": 37.66554641723633,
      "activations/layer21_attention_weight_min": -20.892791748046875,
      "activations/layer22_attention_weight_max": 39.7608528137207,
      "activations/layer22_attention_weight_min": -20.231685638427734,
      "activations/layer23_attention_weight_max": 24.336101531982422,
      "activations/layer23_attention_weight_min": -15.413707733154297,
      "activations/layer2_attention_weight_max": 25.030426025390625,
      "activations/layer2_attention_weight_min": -24.506149291992188,
      "activations/layer3_attention_weight_max": 59.338340759277344,
      "activations/layer3_attention_weight_min": -60.85649871826172,
      "activations/layer4_attention_weight_max": 89.50926971435547,
      "activations/layer4_attention_weight_min": -79.78954315185547,
      "activations/layer5_attention_weight_max": 135.37664794921875,
      "activations/layer5_attention_weight_min": -118.63819885253906,
      "activations/layer6_attention_weight_max": 64.26730346679688,
      "activations/layer6_attention_weight_min": -54.84083938598633,
      "activations/layer7_attention_weight_max": 61.4942512512207,
      "activations/layer7_attention_weight_min": -55.07099533081055,
      "activations/layer8_attention_weight_max": 50.92491149902344,
      "activations/layer8_attention_weight_min": -50.06976318359375,
      "activations/layer9_attention_weight_max": 69.2884292602539,
      "activations/layer9_attention_weight_min": -75.30220794677734,
      "epoch": 1.49,
      "learning_rate": 0.00014179507575757573,
      "loss": 3.0168,
      "step": 25700
    },
    {
      "activations/layer0_attention_weight_max": 15.250016212463379,
      "activations/layer0_attention_weight_min": -13.331462860107422,
      "activations/layer10_attention_weight_max": 63.82075881958008,
      "activations/layer10_attention_weight_min": -57.532432556152344,
      "activations/layer11_attention_weight_max": 44.809234619140625,
      "activations/layer11_attention_weight_min": -40.24778366088867,
      "activations/layer12_attention_weight_max": 26.715147018432617,
      "activations/layer12_attention_weight_min": -30.696651458740234,
      "activations/layer13_attention_weight_max": 53.57765579223633,
      "activations/layer13_attention_weight_min": -43.80691146850586,
      "activations/layer14_attention_weight_max": 48.686866760253906,
      "activations/layer14_attention_weight_min": -40.32343673706055,
      "activations/layer15_attention_weight_max": 47.81466293334961,
      "activations/layer15_attention_weight_min": -39.0589485168457,
      "activations/layer16_attention_weight_max": 39.59999465942383,
      "activations/layer16_attention_weight_min": -33.407936096191406,
      "activations/layer17_attention_weight_max": 58.60701370239258,
      "activations/layer17_attention_weight_min": -37.782108306884766,
      "activations/layer18_attention_weight_max": 68.5769271850586,
      "activations/layer18_attention_weight_min": -34.88961410522461,
      "activations/layer19_attention_weight_max": 18.868389129638672,
      "activations/layer19_attention_weight_min": -14.881019592285156,
      "activations/layer1_attention_weight_max": 15.211984634399414,
      "activations/layer1_attention_weight_min": -12.926736831665039,
      "activations/layer20_attention_weight_max": 17.217288970947266,
      "activations/layer20_attention_weight_min": -15.953442573547363,
      "activations/layer21_attention_weight_max": 35.317996978759766,
      "activations/layer21_attention_weight_min": -25.69192886352539,
      "activations/layer22_attention_weight_max": 35.77924346923828,
      "activations/layer22_attention_weight_min": -21.362255096435547,
      "activations/layer23_attention_weight_max": 20.99034881591797,
      "activations/layer23_attention_weight_min": -17.545408248901367,
      "activations/layer2_attention_weight_max": 25.953489303588867,
      "activations/layer2_attention_weight_min": -24.677719116210938,
      "activations/layer3_attention_weight_max": 61.41442108154297,
      "activations/layer3_attention_weight_min": -63.26506423950195,
      "activations/layer4_attention_weight_max": 88.5560073852539,
      "activations/layer4_attention_weight_min": -77.75768280029297,
      "activations/layer5_attention_weight_max": 134.28768920898438,
      "activations/layer5_attention_weight_min": -124.86920166015625,
      "activations/layer6_attention_weight_max": 66.79793548583984,
      "activations/layer6_attention_weight_min": -56.78617858886719,
      "activations/layer7_attention_weight_max": 63.379817962646484,
      "activations/layer7_attention_weight_min": -57.292213439941406,
      "activations/layer8_attention_weight_max": 48.894744873046875,
      "activations/layer8_attention_weight_min": -48.19051742553711,
      "activations/layer9_attention_weight_max": 71.02835083007812,
      "activations/layer9_attention_weight_min": -63.32522201538086,
      "epoch": 1.5,
      "learning_rate": 0.00014177613636363634,
      "loss": 3.0016,
      "step": 25750
    },
    {
      "activations/layer0_attention_weight_max": 13.935458183288574,
      "activations/layer0_attention_weight_min": -13.539207458496094,
      "activations/layer10_attention_weight_max": 66.65071105957031,
      "activations/layer10_attention_weight_min": -55.79197692871094,
      "activations/layer11_attention_weight_max": 46.2718505859375,
      "activations/layer11_attention_weight_min": -38.40504455566406,
      "activations/layer12_attention_weight_max": 38.77595138549805,
      "activations/layer12_attention_weight_min": -29.446517944335938,
      "activations/layer13_attention_weight_max": 64.16162872314453,
      "activations/layer13_attention_weight_min": -49.62649154663086,
      "activations/layer14_attention_weight_max": 59.254337310791016,
      "activations/layer14_attention_weight_min": -47.06660842895508,
      "activations/layer15_attention_weight_max": 54.304264068603516,
      "activations/layer15_attention_weight_min": -39.4132080078125,
      "activations/layer16_attention_weight_max": 47.91790008544922,
      "activations/layer16_attention_weight_min": -32.59920120239258,
      "activations/layer17_attention_weight_max": 58.74398422241211,
      "activations/layer17_attention_weight_min": -38.24550247192383,
      "activations/layer18_attention_weight_max": 66.3466567993164,
      "activations/layer18_attention_weight_min": -38.53468704223633,
      "activations/layer19_attention_weight_max": 24.469100952148438,
      "activations/layer19_attention_weight_min": -14.688517570495605,
      "activations/layer1_attention_weight_max": 15.256881713867188,
      "activations/layer1_attention_weight_min": -11.76418399810791,
      "activations/layer20_attention_weight_max": 20.431638717651367,
      "activations/layer20_attention_weight_min": -14.068342208862305,
      "activations/layer21_attention_weight_max": 37.3596076965332,
      "activations/layer21_attention_weight_min": -21.222930908203125,
      "activations/layer22_attention_weight_max": 36.910926818847656,
      "activations/layer22_attention_weight_min": -18.904325485229492,
      "activations/layer23_attention_weight_max": 24.06832504272461,
      "activations/layer23_attention_weight_min": -15.556196212768555,
      "activations/layer2_attention_weight_max": 25.37799835205078,
      "activations/layer2_attention_weight_min": -24.402359008789062,
      "activations/layer3_attention_weight_max": 54.66534423828125,
      "activations/layer3_attention_weight_min": -58.15662384033203,
      "activations/layer4_attention_weight_max": 89.3283920288086,
      "activations/layer4_attention_weight_min": -82.25118255615234,
      "activations/layer5_attention_weight_max": 131.62094116210938,
      "activations/layer5_attention_weight_min": -115.0823974609375,
      "activations/layer6_attention_weight_max": 61.600547790527344,
      "activations/layer6_attention_weight_min": -56.11867141723633,
      "activations/layer7_attention_weight_max": 60.5569953918457,
      "activations/layer7_attention_weight_min": -53.48329162597656,
      "activations/layer8_attention_weight_max": 50.93751525878906,
      "activations/layer8_attention_weight_min": -48.33535385131836,
      "activations/layer9_attention_weight_max": 65.44290161132812,
      "activations/layer9_attention_weight_min": -63.11143112182617,
      "epoch": 1.5,
      "learning_rate": 0.00014175719696969696,
      "loss": 3.0299,
      "step": 25800
    },
    {
      "activations/layer0_attention_weight_max": 14.460477828979492,
      "activations/layer0_attention_weight_min": -12.905421257019043,
      "activations/layer10_attention_weight_max": 66.93003845214844,
      "activations/layer10_attention_weight_min": -58.58654022216797,
      "activations/layer11_attention_weight_max": 48.01105880737305,
      "activations/layer11_attention_weight_min": -38.65950012207031,
      "activations/layer12_attention_weight_max": 30.672548294067383,
      "activations/layer12_attention_weight_min": -24.127338409423828,
      "activations/layer13_attention_weight_max": 74.1004638671875,
      "activations/layer13_attention_weight_min": -54.50279235839844,
      "activations/layer14_attention_weight_max": 69.54887390136719,
      "activations/layer14_attention_weight_min": -51.506778717041016,
      "activations/layer15_attention_weight_max": 56.11815643310547,
      "activations/layer15_attention_weight_min": -39.87284851074219,
      "activations/layer16_attention_weight_max": 44.528099060058594,
      "activations/layer16_attention_weight_min": -30.494279861450195,
      "activations/layer17_attention_weight_max": 68.19377899169922,
      "activations/layer17_attention_weight_min": -42.757469177246094,
      "activations/layer18_attention_weight_max": 65.92076110839844,
      "activations/layer18_attention_weight_min": -34.2506103515625,
      "activations/layer19_attention_weight_max": 22.031524658203125,
      "activations/layer19_attention_weight_min": -15.808314323425293,
      "activations/layer1_attention_weight_max": 15.23576831817627,
      "activations/layer1_attention_weight_min": -11.465411186218262,
      "activations/layer20_attention_weight_max": 22.86908531188965,
      "activations/layer20_attention_weight_min": -15.146570205688477,
      "activations/layer21_attention_weight_max": 42.781837463378906,
      "activations/layer21_attention_weight_min": -20.603025436401367,
      "activations/layer22_attention_weight_max": 38.60031509399414,
      "activations/layer22_attention_weight_min": -23.07684898376465,
      "activations/layer23_attention_weight_max": 22.78485107421875,
      "activations/layer23_attention_weight_min": -14.495150566101074,
      "activations/layer2_attention_weight_max": 24.566360473632812,
      "activations/layer2_attention_weight_min": -22.766216278076172,
      "activations/layer3_attention_weight_max": 52.38435363769531,
      "activations/layer3_attention_weight_min": -55.043983459472656,
      "activations/layer4_attention_weight_max": 82.2813491821289,
      "activations/layer4_attention_weight_min": -80.42098999023438,
      "activations/layer5_attention_weight_max": 128.58628845214844,
      "activations/layer5_attention_weight_min": -120.98455047607422,
      "activations/layer6_attention_weight_max": 64.25157928466797,
      "activations/layer6_attention_weight_min": -53.383724212646484,
      "activations/layer7_attention_weight_max": 60.077579498291016,
      "activations/layer7_attention_weight_min": -56.805259704589844,
      "activations/layer8_attention_weight_max": 51.40257263183594,
      "activations/layer8_attention_weight_min": -45.014198303222656,
      "activations/layer9_attention_weight_max": 67.8319320678711,
      "activations/layer9_attention_weight_min": -62.55024337768555,
      "epoch": 1.5,
      "learning_rate": 0.00014173825757575755,
      "loss": 2.9991,
      "step": 25850
    },
    {
      "activations/layer0_attention_weight_max": 16.119428634643555,
      "activations/layer0_attention_weight_min": -12.979470252990723,
      "activations/layer10_attention_weight_max": 71.14868927001953,
      "activations/layer10_attention_weight_min": -56.08182144165039,
      "activations/layer11_attention_weight_max": 56.873939514160156,
      "activations/layer11_attention_weight_min": -44.502044677734375,
      "activations/layer12_attention_weight_max": 30.02202796936035,
      "activations/layer12_attention_weight_min": -26.62079429626465,
      "activations/layer13_attention_weight_max": 68.48355102539062,
      "activations/layer13_attention_weight_min": -42.97449493408203,
      "activations/layer14_attention_weight_max": 72.0467300415039,
      "activations/layer14_attention_weight_min": -39.8990592956543,
      "activations/layer15_attention_weight_max": 54.185611724853516,
      "activations/layer15_attention_weight_min": -36.69751739501953,
      "activations/layer16_attention_weight_max": 56.650848388671875,
      "activations/layer16_attention_weight_min": -30.95412254333496,
      "activations/layer17_attention_weight_max": 67.33939361572266,
      "activations/layer17_attention_weight_min": -35.667057037353516,
      "activations/layer18_attention_weight_max": 80.40361022949219,
      "activations/layer18_attention_weight_min": -37.61674880981445,
      "activations/layer19_attention_weight_max": 23.562515258789062,
      "activations/layer19_attention_weight_min": -14.615274429321289,
      "activations/layer1_attention_weight_max": 15.571647644042969,
      "activations/layer1_attention_weight_min": -12.546871185302734,
      "activations/layer20_attention_weight_max": 20.435932159423828,
      "activations/layer20_attention_weight_min": -14.722585678100586,
      "activations/layer21_attention_weight_max": 44.077728271484375,
      "activations/layer21_attention_weight_min": -22.182010650634766,
      "activations/layer22_attention_weight_max": 34.449668884277344,
      "activations/layer22_attention_weight_min": -17.60443878173828,
      "activations/layer23_attention_weight_max": 22.987335205078125,
      "activations/layer23_attention_weight_min": -16.17822265625,
      "activations/layer2_attention_weight_max": 26.33011245727539,
      "activations/layer2_attention_weight_min": -26.139799118041992,
      "activations/layer3_attention_weight_max": 60.88672637939453,
      "activations/layer3_attention_weight_min": -60.815574645996094,
      "activations/layer4_attention_weight_max": 95.11457061767578,
      "activations/layer4_attention_weight_min": -83.69935607910156,
      "activations/layer5_attention_weight_max": 148.60653686523438,
      "activations/layer5_attention_weight_min": -121.78912353515625,
      "activations/layer6_attention_weight_max": 68.75638580322266,
      "activations/layer6_attention_weight_min": -54.06279373168945,
      "activations/layer7_attention_weight_max": 69.00738525390625,
      "activations/layer7_attention_weight_min": -58.82210159301758,
      "activations/layer8_attention_weight_max": 63.32453155517578,
      "activations/layer8_attention_weight_min": -52.922935485839844,
      "activations/layer9_attention_weight_max": 86.29564666748047,
      "activations/layer9_attention_weight_min": -75.25384521484375,
      "epoch": 1.5,
      "learning_rate": 0.00014171931818181816,
      "loss": 3.008,
      "step": 25900
    },
    {
      "activations/layer0_attention_weight_max": 14.085100173950195,
      "activations/layer0_attention_weight_min": -13.63523006439209,
      "activations/layer10_attention_weight_max": 70.75089263916016,
      "activations/layer10_attention_weight_min": -57.29523468017578,
      "activations/layer11_attention_weight_max": 46.25100326538086,
      "activations/layer11_attention_weight_min": -39.231842041015625,
      "activations/layer12_attention_weight_max": 27.49970245361328,
      "activations/layer12_attention_weight_min": -24.213029861450195,
      "activations/layer13_attention_weight_max": 56.47869110107422,
      "activations/layer13_attention_weight_min": -44.65602111816406,
      "activations/layer14_attention_weight_max": 59.98127365112305,
      "activations/layer14_attention_weight_min": -42.964176177978516,
      "activations/layer15_attention_weight_max": 53.97881317138672,
      "activations/layer15_attention_weight_min": -37.45274353027344,
      "activations/layer16_attention_weight_max": 50.06048583984375,
      "activations/layer16_attention_weight_min": -31.007823944091797,
      "activations/layer17_attention_weight_max": 63.07473373413086,
      "activations/layer17_attention_weight_min": -40.30475616455078,
      "activations/layer18_attention_weight_max": 63.88036346435547,
      "activations/layer18_attention_weight_min": -36.760459899902344,
      "activations/layer19_attention_weight_max": 20.54777717590332,
      "activations/layer19_attention_weight_min": -14.421744346618652,
      "activations/layer1_attention_weight_max": 15.271939277648926,
      "activations/layer1_attention_weight_min": -12.727124214172363,
      "activations/layer20_attention_weight_max": 20.821025848388672,
      "activations/layer20_attention_weight_min": -17.446495056152344,
      "activations/layer21_attention_weight_max": 40.699729919433594,
      "activations/layer21_attention_weight_min": -23.7278995513916,
      "activations/layer22_attention_weight_max": 39.862274169921875,
      "activations/layer22_attention_weight_min": -21.703311920166016,
      "activations/layer23_attention_weight_max": 23.64637565612793,
      "activations/layer23_attention_weight_min": -14.414538383483887,
      "activations/layer2_attention_weight_max": 28.05466079711914,
      "activations/layer2_attention_weight_min": -25.79677963256836,
      "activations/layer3_attention_weight_max": 56.934486389160156,
      "activations/layer3_attention_weight_min": -63.9245719909668,
      "activations/layer4_attention_weight_max": 88.1518783569336,
      "activations/layer4_attention_weight_min": -81.1533203125,
      "activations/layer5_attention_weight_max": 134.7242431640625,
      "activations/layer5_attention_weight_min": -115.52388763427734,
      "activations/layer6_attention_weight_max": 62.29538345336914,
      "activations/layer6_attention_weight_min": -54.37253952026367,
      "activations/layer7_attention_weight_max": 60.27777862548828,
      "activations/layer7_attention_weight_min": -55.26095199584961,
      "activations/layer8_attention_weight_max": 47.92934036254883,
      "activations/layer8_attention_weight_min": -43.47642135620117,
      "activations/layer9_attention_weight_max": 69.64046478271484,
      "activations/layer9_attention_weight_min": -60.21584701538086,
      "epoch": 1.51,
      "learning_rate": 0.00014170037878787878,
      "loss": 3.0106,
      "step": 25950
    },
    {
      "activations/layer0_attention_weight_max": 13.618005752563477,
      "activations/layer0_attention_weight_min": -12.947090148925781,
      "activations/layer10_attention_weight_max": 63.53483200073242,
      "activations/layer10_attention_weight_min": -55.40294647216797,
      "activations/layer11_attention_weight_max": 50.04732894897461,
      "activations/layer11_attention_weight_min": -45.892333984375,
      "activations/layer12_attention_weight_max": 27.268320083618164,
      "activations/layer12_attention_weight_min": -25.04001808166504,
      "activations/layer13_attention_weight_max": 58.89707565307617,
      "activations/layer13_attention_weight_min": -41.446502685546875,
      "activations/layer14_attention_weight_max": 49.67570495605469,
      "activations/layer14_attention_weight_min": -39.69147491455078,
      "activations/layer15_attention_weight_max": 58.30024337768555,
      "activations/layer15_attention_weight_min": -41.78529357910156,
      "activations/layer16_attention_weight_max": 45.04636764526367,
      "activations/layer16_attention_weight_min": -28.828777313232422,
      "activations/layer17_attention_weight_max": 56.8707389831543,
      "activations/layer17_attention_weight_min": -37.16789627075195,
      "activations/layer18_attention_weight_max": 63.396766662597656,
      "activations/layer18_attention_weight_min": -37.718238830566406,
      "activations/layer19_attention_weight_max": 20.75008773803711,
      "activations/layer19_attention_weight_min": -14.26185131072998,
      "activations/layer1_attention_weight_max": 17.022878646850586,
      "activations/layer1_attention_weight_min": -10.78415298461914,
      "activations/layer20_attention_weight_max": 18.06988525390625,
      "activations/layer20_attention_weight_min": -14.045923233032227,
      "activations/layer21_attention_weight_max": 37.0361442565918,
      "activations/layer21_attention_weight_min": -20.93453025817871,
      "activations/layer22_attention_weight_max": 34.56576156616211,
      "activations/layer22_attention_weight_min": -18.938936233520508,
      "activations/layer23_attention_weight_max": 22.928470611572266,
      "activations/layer23_attention_weight_min": -15.242561340332031,
      "activations/layer2_attention_weight_max": 25.2160701751709,
      "activations/layer2_attention_weight_min": -25.552200317382812,
      "activations/layer3_attention_weight_max": 58.948509216308594,
      "activations/layer3_attention_weight_min": -63.734291076660156,
      "activations/layer4_attention_weight_max": 88.1935043334961,
      "activations/layer4_attention_weight_min": -81.11344146728516,
      "activations/layer5_attention_weight_max": 131.27996826171875,
      "activations/layer5_attention_weight_min": -123.51792907714844,
      "activations/layer6_attention_weight_max": 61.85037612915039,
      "activations/layer6_attention_weight_min": -55.45854568481445,
      "activations/layer7_attention_weight_max": 60.742252349853516,
      "activations/layer7_attention_weight_min": -57.39760208129883,
      "activations/layer8_attention_weight_max": 51.21599197387695,
      "activations/layer8_attention_weight_min": -52.9306755065918,
      "activations/layer9_attention_weight_max": 71.19622039794922,
      "activations/layer9_attention_weight_min": -70.04463958740234,
      "epoch": 1.51,
      "learning_rate": 0.00014168143939393937,
      "loss": 3.0223,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_loss": 2.9609375,
      "eval_runtime": 8.5952,
      "eval_samples_per_second": 499.578,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_openwebtext_loss": 2.9609375,
      "eval_openwebtext_ppl": 19.31607208722833,
      "eval_openwebtext_runtime": 8.5952,
      "eval_openwebtext_samples_per_second": 499.578,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_wikitext_loss": 3.244140625,
      "eval_wikitext_ppl": 25.63966650134361,
      "eval_wikitext_runtime": 1.7034,
      "eval_wikitext_samples_per_second": 267.695,
      "step": 26000
    },
    {
      "epoch": 1.51,
      "eval_lambada_loss": 3.2890625,
      "eval_lambada_ppl": 26.817710263716034,
      "eval_lambada_runtime": 9.3387,
      "eval_lambada_samples_per_second": 521.378,
      "step": 26000
    },
    {
      "activations/layer0_attention_weight_max": 13.561419486999512,
      "activations/layer0_attention_weight_min": -13.482515335083008,
      "activations/layer10_attention_weight_max": 62.4110107421875,
      "activations/layer10_attention_weight_min": -53.38224411010742,
      "activations/layer11_attention_weight_max": 47.11513900756836,
      "activations/layer11_attention_weight_min": -38.06568145751953,
      "activations/layer12_attention_weight_max": 30.022422790527344,
      "activations/layer12_attention_weight_min": -24.542232513427734,
      "activations/layer13_attention_weight_max": 60.98139572143555,
      "activations/layer13_attention_weight_min": -46.06631088256836,
      "activations/layer14_attention_weight_max": 51.050941467285156,
      "activations/layer14_attention_weight_min": -40.65645980834961,
      "activations/layer15_attention_weight_max": 56.612403869628906,
      "activations/layer15_attention_weight_min": -38.966773986816406,
      "activations/layer16_attention_weight_max": 44.965457916259766,
      "activations/layer16_attention_weight_min": -28.40667724609375,
      "activations/layer17_attention_weight_max": 66.52542877197266,
      "activations/layer17_attention_weight_min": -39.25621795654297,
      "activations/layer18_attention_weight_max": 71.53681182861328,
      "activations/layer18_attention_weight_min": -34.785072326660156,
      "activations/layer19_attention_weight_max": 29.599267959594727,
      "activations/layer19_attention_weight_min": -14.282851219177246,
      "activations/layer1_attention_weight_max": 16.034677505493164,
      "activations/layer1_attention_weight_min": -12.02348518371582,
      "activations/layer20_attention_weight_max": 21.962623596191406,
      "activations/layer20_attention_weight_min": -16.776954650878906,
      "activations/layer21_attention_weight_max": 38.34560012817383,
      "activations/layer21_attention_weight_min": -21.074039459228516,
      "activations/layer22_attention_weight_max": 41.97529602050781,
      "activations/layer22_attention_weight_min": -19.21974754333496,
      "activations/layer23_attention_weight_max": 26.549314498901367,
      "activations/layer23_attention_weight_min": -14.47763729095459,
      "activations/layer2_attention_weight_max": 26.60364532470703,
      "activations/layer2_attention_weight_min": -25.045269012451172,
      "activations/layer3_attention_weight_max": 58.13727569580078,
      "activations/layer3_attention_weight_min": -61.89213562011719,
      "activations/layer4_attention_weight_max": 90.7345199584961,
      "activations/layer4_attention_weight_min": -83.4908676147461,
      "activations/layer5_attention_weight_max": 140.50193786621094,
      "activations/layer5_attention_weight_min": -119.39442443847656,
      "activations/layer6_attention_weight_max": 63.568992614746094,
      "activations/layer6_attention_weight_min": -56.89619064331055,
      "activations/layer7_attention_weight_max": 68.81285095214844,
      "activations/layer7_attention_weight_min": -55.10342788696289,
      "activations/layer8_attention_weight_max": 55.887935638427734,
      "activations/layer8_attention_weight_min": -44.810997009277344,
      "activations/layer9_attention_weight_max": 72.2842025756836,
      "activations/layer9_attention_weight_min": -60.9217414855957,
      "epoch": 1.51,
      "learning_rate": 0.00014166249999999998,
      "loss": 3.0087,
      "step": 26050
    },
    {
      "activations/layer0_attention_weight_max": 15.596275329589844,
      "activations/layer0_attention_weight_min": -12.966650009155273,
      "activations/layer10_attention_weight_max": 67.27217102050781,
      "activations/layer10_attention_weight_min": -54.27595901489258,
      "activations/layer11_attention_weight_max": 50.061424255371094,
      "activations/layer11_attention_weight_min": -37.969112396240234,
      "activations/layer12_attention_weight_max": 38.90996170043945,
      "activations/layer12_attention_weight_min": -32.06415557861328,
      "activations/layer13_attention_weight_max": 76.43142700195312,
      "activations/layer13_attention_weight_min": -48.300743103027344,
      "activations/layer14_attention_weight_max": 58.35079574584961,
      "activations/layer14_attention_weight_min": -42.35515213012695,
      "activations/layer15_attention_weight_max": 64.19522857666016,
      "activations/layer15_attention_weight_min": -42.14690017700195,
      "activations/layer16_attention_weight_max": 46.580081939697266,
      "activations/layer16_attention_weight_min": -34.7379150390625,
      "activations/layer17_attention_weight_max": 65.82029724121094,
      "activations/layer17_attention_weight_min": -43.756919860839844,
      "activations/layer18_attention_weight_max": 70.15402221679688,
      "activations/layer18_attention_weight_min": -38.819541931152344,
      "activations/layer19_attention_weight_max": 26.491994857788086,
      "activations/layer19_attention_weight_min": -14.610274314880371,
      "activations/layer1_attention_weight_max": 15.849933624267578,
      "activations/layer1_attention_weight_min": -14.582955360412598,
      "activations/layer20_attention_weight_max": 17.954002380371094,
      "activations/layer20_attention_weight_min": -15.023162841796875,
      "activations/layer21_attention_weight_max": 41.134037017822266,
      "activations/layer21_attention_weight_min": -23.81907844543457,
      "activations/layer22_attention_weight_max": 39.81607437133789,
      "activations/layer22_attention_weight_min": -22.52068328857422,
      "activations/layer23_attention_weight_max": 23.78333282470703,
      "activations/layer23_attention_weight_min": -13.658469200134277,
      "activations/layer2_attention_weight_max": 26.48888397216797,
      "activations/layer2_attention_weight_min": -24.38899040222168,
      "activations/layer3_attention_weight_max": 58.47317123413086,
      "activations/layer3_attention_weight_min": -61.02996826171875,
      "activations/layer4_attention_weight_max": 91.25984954833984,
      "activations/layer4_attention_weight_min": -82.65528869628906,
      "activations/layer5_attention_weight_max": 139.6685028076172,
      "activations/layer5_attention_weight_min": -122.5853271484375,
      "activations/layer6_attention_weight_max": 65.94366455078125,
      "activations/layer6_attention_weight_min": -55.0734748840332,
      "activations/layer7_attention_weight_max": 66.12356567382812,
      "activations/layer7_attention_weight_min": -54.570106506347656,
      "activations/layer8_attention_weight_max": 53.5400390625,
      "activations/layer8_attention_weight_min": -45.30054473876953,
      "activations/layer9_attention_weight_max": 70.4760971069336,
      "activations/layer9_attention_weight_min": -63.05872344970703,
      "epoch": 1.52,
      "learning_rate": 0.0001416435606060606,
      "loss": 2.98,
      "step": 26100
    },
    {
      "activations/layer0_attention_weight_max": 13.249371528625488,
      "activations/layer0_attention_weight_min": -13.543476104736328,
      "activations/layer10_attention_weight_max": 69.18009948730469,
      "activations/layer10_attention_weight_min": -52.24933624267578,
      "activations/layer11_attention_weight_max": 46.61924362182617,
      "activations/layer11_attention_weight_min": -38.286659240722656,
      "activations/layer12_attention_weight_max": 36.68798828125,
      "activations/layer12_attention_weight_min": -26.96169662475586,
      "activations/layer13_attention_weight_max": 63.29081344604492,
      "activations/layer13_attention_weight_min": -46.07218551635742,
      "activations/layer14_attention_weight_max": 50.9826545715332,
      "activations/layer14_attention_weight_min": -40.40839385986328,
      "activations/layer15_attention_weight_max": 60.41707229614258,
      "activations/layer15_attention_weight_min": -39.80684280395508,
      "activations/layer16_attention_weight_max": 48.90544128417969,
      "activations/layer16_attention_weight_min": -33.171104431152344,
      "activations/layer17_attention_weight_max": 72.0830307006836,
      "activations/layer17_attention_weight_min": -43.3259162902832,
      "activations/layer18_attention_weight_max": 66.245361328125,
      "activations/layer18_attention_weight_min": -40.38673782348633,
      "activations/layer19_attention_weight_max": 21.408933639526367,
      "activations/layer19_attention_weight_min": -14.655468940734863,
      "activations/layer1_attention_weight_max": 15.200663566589355,
      "activations/layer1_attention_weight_min": -13.578133583068848,
      "activations/layer20_attention_weight_max": 21.301843643188477,
      "activations/layer20_attention_weight_min": -18.082645416259766,
      "activations/layer21_attention_weight_max": 44.065914154052734,
      "activations/layer21_attention_weight_min": -22.485309600830078,
      "activations/layer22_attention_weight_max": 43.32632064819336,
      "activations/layer22_attention_weight_min": -22.623905181884766,
      "activations/layer23_attention_weight_max": 26.720685958862305,
      "activations/layer23_attention_weight_min": -14.9168119430542,
      "activations/layer2_attention_weight_max": 25.97472381591797,
      "activations/layer2_attention_weight_min": -24.99726104736328,
      "activations/layer3_attention_weight_max": 57.86912536621094,
      "activations/layer3_attention_weight_min": -59.01018142700195,
      "activations/layer4_attention_weight_max": 89.36613464355469,
      "activations/layer4_attention_weight_min": -78.94119262695312,
      "activations/layer5_attention_weight_max": 134.49688720703125,
      "activations/layer5_attention_weight_min": -118.54527282714844,
      "activations/layer6_attention_weight_max": 63.069278717041016,
      "activations/layer6_attention_weight_min": -52.96991729736328,
      "activations/layer7_attention_weight_max": 63.25185012817383,
      "activations/layer7_attention_weight_min": -51.65498733520508,
      "activations/layer8_attention_weight_max": 49.76234436035156,
      "activations/layer8_attention_weight_min": -42.70985412597656,
      "activations/layer9_attention_weight_max": 64.0994644165039,
      "activations/layer9_attention_weight_min": -61.5372314453125,
      "epoch": 1.52,
      "learning_rate": 0.0001416246212121212,
      "loss": 2.9807,
      "step": 26150
    },
    {
      "activations/layer0_attention_weight_max": 14.07694149017334,
      "activations/layer0_attention_weight_min": -13.045490264892578,
      "activations/layer10_attention_weight_max": 70.75019836425781,
      "activations/layer10_attention_weight_min": -56.95000457763672,
      "activations/layer11_attention_weight_max": 45.91822052001953,
      "activations/layer11_attention_weight_min": -38.97486114501953,
      "activations/layer12_attention_weight_max": 35.191707611083984,
      "activations/layer12_attention_weight_min": -29.779176712036133,
      "activations/layer13_attention_weight_max": 65.09506225585938,
      "activations/layer13_attention_weight_min": -48.45283889770508,
      "activations/layer14_attention_weight_max": 61.254669189453125,
      "activations/layer14_attention_weight_min": -37.44799041748047,
      "activations/layer15_attention_weight_max": 61.54792785644531,
      "activations/layer15_attention_weight_min": -38.63447952270508,
      "activations/layer16_attention_weight_max": 46.384422302246094,
      "activations/layer16_attention_weight_min": -33.46022415161133,
      "activations/layer17_attention_weight_max": 60.77462387084961,
      "activations/layer17_attention_weight_min": -39.2420539855957,
      "activations/layer18_attention_weight_max": 62.860286712646484,
      "activations/layer18_attention_weight_min": -38.592742919921875,
      "activations/layer19_attention_weight_max": 23.843381881713867,
      "activations/layer19_attention_weight_min": -13.635174751281738,
      "activations/layer1_attention_weight_max": 15.53322982788086,
      "activations/layer1_attention_weight_min": -11.981342315673828,
      "activations/layer20_attention_weight_max": 22.475440979003906,
      "activations/layer20_attention_weight_min": -13.800704956054688,
      "activations/layer21_attention_weight_max": 34.359073638916016,
      "activations/layer21_attention_weight_min": -22.122678756713867,
      "activations/layer22_attention_weight_max": 35.72711181640625,
      "activations/layer22_attention_weight_min": -23.752187728881836,
      "activations/layer23_attention_weight_max": 22.407032012939453,
      "activations/layer23_attention_weight_min": -15.575803756713867,
      "activations/layer2_attention_weight_max": 26.510663986206055,
      "activations/layer2_attention_weight_min": -25.464622497558594,
      "activations/layer3_attention_weight_max": 60.91835021972656,
      "activations/layer3_attention_weight_min": -67.07486724853516,
      "activations/layer4_attention_weight_max": 90.16317749023438,
      "activations/layer4_attention_weight_min": -82.98047637939453,
      "activations/layer5_attention_weight_max": 133.2059783935547,
      "activations/layer5_attention_weight_min": -111.52941131591797,
      "activations/layer6_attention_weight_max": 61.61181640625,
      "activations/layer6_attention_weight_min": -54.82326889038086,
      "activations/layer7_attention_weight_max": 62.019996643066406,
      "activations/layer7_attention_weight_min": -53.847774505615234,
      "activations/layer8_attention_weight_max": 52.25849533081055,
      "activations/layer8_attention_weight_min": -44.12451171875,
      "activations/layer9_attention_weight_max": 75.41520690917969,
      "activations/layer9_attention_weight_min": -64.61246490478516,
      "epoch": 1.52,
      "learning_rate": 0.0001416056818181818,
      "loss": 2.9879,
      "step": 26200
    },
    {
      "activations/layer0_attention_weight_max": 15.285019874572754,
      "activations/layer0_attention_weight_min": -13.246431350708008,
      "activations/layer10_attention_weight_max": 69.51782989501953,
      "activations/layer10_attention_weight_min": -56.511749267578125,
      "activations/layer11_attention_weight_max": 43.53300094604492,
      "activations/layer11_attention_weight_min": -38.82648468017578,
      "activations/layer12_attention_weight_max": 30.679489135742188,
      "activations/layer12_attention_weight_min": -24.269588470458984,
      "activations/layer13_attention_weight_max": 58.970184326171875,
      "activations/layer13_attention_weight_min": -50.07645797729492,
      "activations/layer14_attention_weight_max": 60.08992004394531,
      "activations/layer14_attention_weight_min": -46.12374496459961,
      "activations/layer15_attention_weight_max": 54.169090270996094,
      "activations/layer15_attention_weight_min": -43.7095832824707,
      "activations/layer16_attention_weight_max": 45.08806228637695,
      "activations/layer16_attention_weight_min": -33.090797424316406,
      "activations/layer17_attention_weight_max": 57.33525466918945,
      "activations/layer17_attention_weight_min": -40.60002517700195,
      "activations/layer18_attention_weight_max": 58.09181594848633,
      "activations/layer18_attention_weight_min": -41.34532928466797,
      "activations/layer19_attention_weight_max": 21.562145233154297,
      "activations/layer19_attention_weight_min": -15.058941841125488,
      "activations/layer1_attention_weight_max": 15.428244590759277,
      "activations/layer1_attention_weight_min": -12.05148983001709,
      "activations/layer20_attention_weight_max": 19.082351684570312,
      "activations/layer20_attention_weight_min": -15.07332706451416,
      "activations/layer21_attention_weight_max": 39.088985443115234,
      "activations/layer21_attention_weight_min": -22.20226287841797,
      "activations/layer22_attention_weight_max": 35.28837966918945,
      "activations/layer22_attention_weight_min": -20.448911666870117,
      "activations/layer23_attention_weight_max": 23.423559188842773,
      "activations/layer23_attention_weight_min": -16.246906280517578,
      "activations/layer2_attention_weight_max": 25.192047119140625,
      "activations/layer2_attention_weight_min": -24.554229736328125,
      "activations/layer3_attention_weight_max": 60.545162200927734,
      "activations/layer3_attention_weight_min": -65.29962158203125,
      "activations/layer4_attention_weight_max": 93.04534912109375,
      "activations/layer4_attention_weight_min": -83.42610931396484,
      "activations/layer5_attention_weight_max": 135.102783203125,
      "activations/layer5_attention_weight_min": -120.89427185058594,
      "activations/layer6_attention_weight_max": 68.65593719482422,
      "activations/layer6_attention_weight_min": -59.249053955078125,
      "activations/layer7_attention_weight_max": 63.667842864990234,
      "activations/layer7_attention_weight_min": -56.12348175048828,
      "activations/layer8_attention_weight_max": 50.60737609863281,
      "activations/layer8_attention_weight_min": -46.20866775512695,
      "activations/layer9_attention_weight_max": 69.03734588623047,
      "activations/layer9_attention_weight_min": -61.357421875,
      "epoch": 1.53,
      "learning_rate": 0.00014158674242424241,
      "loss": 2.9979,
      "step": 26250
    },
    {
      "activations/layer0_attention_weight_max": 14.733160972595215,
      "activations/layer0_attention_weight_min": -13.49831485748291,
      "activations/layer10_attention_weight_max": 73.662109375,
      "activations/layer10_attention_weight_min": -54.367088317871094,
      "activations/layer11_attention_weight_max": 58.02174377441406,
      "activations/layer11_attention_weight_min": -42.40332794189453,
      "activations/layer12_attention_weight_max": 30.43459701538086,
      "activations/layer12_attention_weight_min": -23.17733383178711,
      "activations/layer13_attention_weight_max": 91.2202377319336,
      "activations/layer13_attention_weight_min": -50.96839141845703,
      "activations/layer14_attention_weight_max": 84.15547943115234,
      "activations/layer14_attention_weight_min": -47.539371490478516,
      "activations/layer15_attention_weight_max": 66.5633544921875,
      "activations/layer15_attention_weight_min": -43.52341842651367,
      "activations/layer16_attention_weight_max": 63.75318908691406,
      "activations/layer16_attention_weight_min": -34.877986907958984,
      "activations/layer17_attention_weight_max": 70.90513610839844,
      "activations/layer17_attention_weight_min": -40.373435974121094,
      "activations/layer18_attention_weight_max": 71.33861541748047,
      "activations/layer18_attention_weight_min": -37.609779357910156,
      "activations/layer19_attention_weight_max": 26.410764694213867,
      "activations/layer19_attention_weight_min": -14.829731941223145,
      "activations/layer1_attention_weight_max": 15.482118606567383,
      "activations/layer1_attention_weight_min": -11.724533081054688,
      "activations/layer20_attention_weight_max": 23.645801544189453,
      "activations/layer20_attention_weight_min": -14.490721702575684,
      "activations/layer21_attention_weight_max": 46.20439529418945,
      "activations/layer21_attention_weight_min": -22.32869529724121,
      "activations/layer22_attention_weight_max": 43.342655181884766,
      "activations/layer22_attention_weight_min": -19.37445831298828,
      "activations/layer23_attention_weight_max": 27.632369995117188,
      "activations/layer23_attention_weight_min": -14.16553020477295,
      "activations/layer2_attention_weight_max": 25.225250244140625,
      "activations/layer2_attention_weight_min": -24.7513427734375,
      "activations/layer3_attention_weight_max": 60.09342956542969,
      "activations/layer3_attention_weight_min": -63.5632209777832,
      "activations/layer4_attention_weight_max": 99.12358856201172,
      "activations/layer4_attention_weight_min": -88.61182403564453,
      "activations/layer5_attention_weight_max": 163.85784912109375,
      "activations/layer5_attention_weight_min": -127.07701110839844,
      "activations/layer6_attention_weight_max": 76.83355712890625,
      "activations/layer6_attention_weight_min": -58.166385650634766,
      "activations/layer7_attention_weight_max": 78.53224182128906,
      "activations/layer7_attention_weight_min": -57.10068130493164,
      "activations/layer8_attention_weight_max": 69.9368896484375,
      "activations/layer8_attention_weight_min": -53.24525833129883,
      "activations/layer9_attention_weight_max": 89.47777557373047,
      "activations/layer9_attention_weight_min": -72.00779724121094,
      "epoch": 1.53,
      "learning_rate": 0.00014156780303030303,
      "loss": 2.989,
      "step": 26300
    },
    {
      "activations/layer0_attention_weight_max": 13.258615493774414,
      "activations/layer0_attention_weight_min": -13.229840278625488,
      "activations/layer10_attention_weight_max": 70.55731201171875,
      "activations/layer10_attention_weight_min": -56.63892364501953,
      "activations/layer11_attention_weight_max": 47.04963684082031,
      "activations/layer11_attention_weight_min": -36.50132369995117,
      "activations/layer12_attention_weight_max": 29.14029884338379,
      "activations/layer12_attention_weight_min": -23.74643325805664,
      "activations/layer13_attention_weight_max": 59.345314025878906,
      "activations/layer13_attention_weight_min": -41.48887634277344,
      "activations/layer14_attention_weight_max": 61.32653045654297,
      "activations/layer14_attention_weight_min": -39.012577056884766,
      "activations/layer15_attention_weight_max": 63.58034896850586,
      "activations/layer15_attention_weight_min": -40.050689697265625,
      "activations/layer16_attention_weight_max": 49.611454010009766,
      "activations/layer16_attention_weight_min": -32.621063232421875,
      "activations/layer17_attention_weight_max": 66.65208435058594,
      "activations/layer17_attention_weight_min": -38.43988800048828,
      "activations/layer18_attention_weight_max": 68.19203186035156,
      "activations/layer18_attention_weight_min": -34.829219818115234,
      "activations/layer19_attention_weight_max": 22.648653030395508,
      "activations/layer19_attention_weight_min": -16.150245666503906,
      "activations/layer1_attention_weight_max": 15.800714492797852,
      "activations/layer1_attention_weight_min": -14.626887321472168,
      "activations/layer20_attention_weight_max": 22.881797790527344,
      "activations/layer20_attention_weight_min": -15.21826171875,
      "activations/layer21_attention_weight_max": 40.28889083862305,
      "activations/layer21_attention_weight_min": -21.688934326171875,
      "activations/layer22_attention_weight_max": 36.154117584228516,
      "activations/layer22_attention_weight_min": -20.204425811767578,
      "activations/layer23_attention_weight_max": 23.575305938720703,
      "activations/layer23_attention_weight_min": -15.143309593200684,
      "activations/layer2_attention_weight_max": 26.12975311279297,
      "activations/layer2_attention_weight_min": -26.687530517578125,
      "activations/layer3_attention_weight_max": 60.910797119140625,
      "activations/layer3_attention_weight_min": -62.035160064697266,
      "activations/layer4_attention_weight_max": 91.28640747070312,
      "activations/layer4_attention_weight_min": -80.8990478515625,
      "activations/layer5_attention_weight_max": 135.52940368652344,
      "activations/layer5_attention_weight_min": -115.5959701538086,
      "activations/layer6_attention_weight_max": 65.11994171142578,
      "activations/layer6_attention_weight_min": -58.45582962036133,
      "activations/layer7_attention_weight_max": 64.04195404052734,
      "activations/layer7_attention_weight_min": -52.828487396240234,
      "activations/layer8_attention_weight_max": 53.06128692626953,
      "activations/layer8_attention_weight_min": -46.01021194458008,
      "activations/layer9_attention_weight_max": 68.13170623779297,
      "activations/layer9_attention_weight_min": -61.28683090209961,
      "epoch": 1.53,
      "learning_rate": 0.00014154886363636362,
      "loss": 2.9846,
      "step": 26350
    },
    {
      "activations/layer0_attention_weight_max": 12.825249671936035,
      "activations/layer0_attention_weight_min": -13.556506156921387,
      "activations/layer10_attention_weight_max": 68.61309051513672,
      "activations/layer10_attention_weight_min": -54.883209228515625,
      "activations/layer11_attention_weight_max": 44.89913558959961,
      "activations/layer11_attention_weight_min": -36.073829650878906,
      "activations/layer12_attention_weight_max": 25.561668395996094,
      "activations/layer12_attention_weight_min": -23.61832618713379,
      "activations/layer13_attention_weight_max": 54.666526794433594,
      "activations/layer13_attention_weight_min": -43.62598419189453,
      "activations/layer14_attention_weight_max": 49.038368225097656,
      "activations/layer14_attention_weight_min": -39.11729431152344,
      "activations/layer15_attention_weight_max": 53.67139434814453,
      "activations/layer15_attention_weight_min": -43.03788757324219,
      "activations/layer16_attention_weight_max": 49.00361251831055,
      "activations/layer16_attention_weight_min": -33.3577995300293,
      "activations/layer17_attention_weight_max": 63.88179016113281,
      "activations/layer17_attention_weight_min": -38.38837814331055,
      "activations/layer18_attention_weight_max": 69.82456970214844,
      "activations/layer18_attention_weight_min": -36.00298309326172,
      "activations/layer19_attention_weight_max": 21.556394577026367,
      "activations/layer19_attention_weight_min": -13.936596870422363,
      "activations/layer1_attention_weight_max": 14.840875625610352,
      "activations/layer1_attention_weight_min": -12.258891105651855,
      "activations/layer20_attention_weight_max": 18.754732131958008,
      "activations/layer20_attention_weight_min": -17.21493911743164,
      "activations/layer21_attention_weight_max": 35.420379638671875,
      "activations/layer21_attention_weight_min": -25.495908737182617,
      "activations/layer22_attention_weight_max": 36.5592155456543,
      "activations/layer22_attention_weight_min": -20.462160110473633,
      "activations/layer23_attention_weight_max": 25.445899963378906,
      "activations/layer23_attention_weight_min": -15.002531051635742,
      "activations/layer2_attention_weight_max": 23.362903594970703,
      "activations/layer2_attention_weight_min": -23.88246726989746,
      "activations/layer3_attention_weight_max": 58.54496383666992,
      "activations/layer3_attention_weight_min": -63.037254333496094,
      "activations/layer4_attention_weight_max": 88.67262268066406,
      "activations/layer4_attention_weight_min": -83.77963256835938,
      "activations/layer5_attention_weight_max": 134.39598083496094,
      "activations/layer5_attention_weight_min": -116.01882934570312,
      "activations/layer6_attention_weight_max": 63.6006965637207,
      "activations/layer6_attention_weight_min": -52.46444320678711,
      "activations/layer7_attention_weight_max": 66.15586853027344,
      "activations/layer7_attention_weight_min": -54.12600326538086,
      "activations/layer8_attention_weight_max": 52.742759704589844,
      "activations/layer8_attention_weight_min": -46.06853485107422,
      "activations/layer9_attention_weight_max": 67.70867156982422,
      "activations/layer9_attention_weight_min": -63.37907791137695,
      "epoch": 1.53,
      "learning_rate": 0.00014152992424242423,
      "loss": 2.9893,
      "step": 26400
    },
    {
      "activations/layer0_attention_weight_max": 13.986356735229492,
      "activations/layer0_attention_weight_min": -13.581974029541016,
      "activations/layer10_attention_weight_max": 57.73135757446289,
      "activations/layer10_attention_weight_min": -49.647918701171875,
      "activations/layer11_attention_weight_max": 41.7559928894043,
      "activations/layer11_attention_weight_min": -38.25059509277344,
      "activations/layer12_attention_weight_max": 29.706409454345703,
      "activations/layer12_attention_weight_min": -21.954151153564453,
      "activations/layer13_attention_weight_max": 57.03850555419922,
      "activations/layer13_attention_weight_min": -44.30699920654297,
      "activations/layer14_attention_weight_max": 45.805694580078125,
      "activations/layer14_attention_weight_min": -34.98104476928711,
      "activations/layer15_attention_weight_max": 53.51935577392578,
      "activations/layer15_attention_weight_min": -39.86994552612305,
      "activations/layer16_attention_weight_max": 47.03226089477539,
      "activations/layer16_attention_weight_min": -30.618030548095703,
      "activations/layer17_attention_weight_max": 61.07183837890625,
      "activations/layer17_attention_weight_min": -39.067543029785156,
      "activations/layer18_attention_weight_max": 57.40053176879883,
      "activations/layer18_attention_weight_min": -38.114566802978516,
      "activations/layer19_attention_weight_max": 21.751558303833008,
      "activations/layer19_attention_weight_min": -15.212355613708496,
      "activations/layer1_attention_weight_max": 15.83857250213623,
      "activations/layer1_attention_weight_min": -12.735023498535156,
      "activations/layer20_attention_weight_max": 19.0034236907959,
      "activations/layer20_attention_weight_min": -16.447261810302734,
      "activations/layer21_attention_weight_max": 39.63876724243164,
      "activations/layer21_attention_weight_min": -22.86992645263672,
      "activations/layer22_attention_weight_max": 35.70423889160156,
      "activations/layer22_attention_weight_min": -19.818038940429688,
      "activations/layer23_attention_weight_max": 20.49558448791504,
      "activations/layer23_attention_weight_min": -15.981574058532715,
      "activations/layer2_attention_weight_max": 27.15692138671875,
      "activations/layer2_attention_weight_min": -26.81305694580078,
      "activations/layer3_attention_weight_max": 59.732093811035156,
      "activations/layer3_attention_weight_min": -62.32158660888672,
      "activations/layer4_attention_weight_max": 88.35069274902344,
      "activations/layer4_attention_weight_min": -81.5722885131836,
      "activations/layer5_attention_weight_max": 129.81063842773438,
      "activations/layer5_attention_weight_min": -117.26055908203125,
      "activations/layer6_attention_weight_max": 61.10125732421875,
      "activations/layer6_attention_weight_min": -55.09233474731445,
      "activations/layer7_attention_weight_max": 57.81779861450195,
      "activations/layer7_attention_weight_min": -57.062217712402344,
      "activations/layer8_attention_weight_max": 47.885108947753906,
      "activations/layer8_attention_weight_min": -46.18806838989258,
      "activations/layer9_attention_weight_max": 66.10810852050781,
      "activations/layer9_attention_weight_min": -65.60930633544922,
      "epoch": 1.54,
      "learning_rate": 0.00014151136363636362,
      "loss": 2.972,
      "step": 26450
    },
    {
      "activations/layer0_attention_weight_max": 14.940631866455078,
      "activations/layer0_attention_weight_min": -13.527619361877441,
      "activations/layer10_attention_weight_max": 70.90665435791016,
      "activations/layer10_attention_weight_min": -51.28852462768555,
      "activations/layer11_attention_weight_max": 44.17369842529297,
      "activations/layer11_attention_weight_min": -35.37392807006836,
      "activations/layer12_attention_weight_max": 31.003646850585938,
      "activations/layer12_attention_weight_min": -25.295021057128906,
      "activations/layer13_attention_weight_max": 58.48442840576172,
      "activations/layer13_attention_weight_min": -44.56674575805664,
      "activations/layer14_attention_weight_max": 46.23906707763672,
      "activations/layer14_attention_weight_min": -33.34479522705078,
      "activations/layer15_attention_weight_max": 58.10493850708008,
      "activations/layer15_attention_weight_min": -37.67325973510742,
      "activations/layer16_attention_weight_max": 55.08020782470703,
      "activations/layer16_attention_weight_min": -35.41169738769531,
      "activations/layer17_attention_weight_max": 64.83880615234375,
      "activations/layer17_attention_weight_min": -39.53159713745117,
      "activations/layer18_attention_weight_max": 64.13199615478516,
      "activations/layer18_attention_weight_min": -38.71392059326172,
      "activations/layer19_attention_weight_max": 23.906187057495117,
      "activations/layer19_attention_weight_min": -13.480232238769531,
      "activations/layer1_attention_weight_max": 14.980246543884277,
      "activations/layer1_attention_weight_min": -11.588159561157227,
      "activations/layer20_attention_weight_max": 18.73705291748047,
      "activations/layer20_attention_weight_min": -17.36515998840332,
      "activations/layer21_attention_weight_max": 37.25429153442383,
      "activations/layer21_attention_weight_min": -21.183515548706055,
      "activations/layer22_attention_weight_max": 35.63814926147461,
      "activations/layer22_attention_weight_min": -20.088642120361328,
      "activations/layer23_attention_weight_max": 23.131467819213867,
      "activations/layer23_attention_weight_min": -13.428543090820312,
      "activations/layer2_attention_weight_max": 27.4886474609375,
      "activations/layer2_attention_weight_min": -24.892065048217773,
      "activations/layer3_attention_weight_max": 58.3408317565918,
      "activations/layer3_attention_weight_min": -61.62306594848633,
      "activations/layer4_attention_weight_max": 93.51522064208984,
      "activations/layer4_attention_weight_min": -81.92540740966797,
      "activations/layer5_attention_weight_max": 140.01654052734375,
      "activations/layer5_attention_weight_min": -117.75772094726562,
      "activations/layer6_attention_weight_max": 67.42507934570312,
      "activations/layer6_attention_weight_min": -56.72156524658203,
      "activations/layer7_attention_weight_max": 63.789093017578125,
      "activations/layer7_attention_weight_min": -54.88992691040039,
      "activations/layer8_attention_weight_max": 54.45296096801758,
      "activations/layer8_attention_weight_min": -46.745121002197266,
      "activations/layer9_attention_weight_max": 72.96077728271484,
      "activations/layer9_attention_weight_min": -61.72153854370117,
      "epoch": 1.54,
      "learning_rate": 0.00014149242424242424,
      "loss": 2.9789,
      "step": 26500
    },
    {
      "activations/layer0_attention_weight_max": 14.578794479370117,
      "activations/layer0_attention_weight_min": -13.678810119628906,
      "activations/layer10_attention_weight_max": 58.583248138427734,
      "activations/layer10_attention_weight_min": -52.264678955078125,
      "activations/layer11_attention_weight_max": 45.096763610839844,
      "activations/layer11_attention_weight_min": -38.72416305541992,
      "activations/layer12_attention_weight_max": 30.96288299560547,
      "activations/layer12_attention_weight_min": -28.31827735900879,
      "activations/layer13_attention_weight_max": 70.09818267822266,
      "activations/layer13_attention_weight_min": -51.27967834472656,
      "activations/layer14_attention_weight_max": 61.12175369262695,
      "activations/layer14_attention_weight_min": -41.93973159790039,
      "activations/layer15_attention_weight_max": 60.963314056396484,
      "activations/layer15_attention_weight_min": -43.39748001098633,
      "activations/layer16_attention_weight_max": 54.67776870727539,
      "activations/layer16_attention_weight_min": -32.68498611450195,
      "activations/layer17_attention_weight_max": 65.08143615722656,
      "activations/layer17_attention_weight_min": -39.20174026489258,
      "activations/layer18_attention_weight_max": 69.17205810546875,
      "activations/layer18_attention_weight_min": -35.75678634643555,
      "activations/layer19_attention_weight_max": 20.803932189941406,
      "activations/layer19_attention_weight_min": -14.94145679473877,
      "activations/layer1_attention_weight_max": 15.161046981811523,
      "activations/layer1_attention_weight_min": -13.603963851928711,
      "activations/layer20_attention_weight_max": 21.324588775634766,
      "activations/layer20_attention_weight_min": -14.533685684204102,
      "activations/layer21_attention_weight_max": 44.553436279296875,
      "activations/layer21_attention_weight_min": -21.12652587890625,
      "activations/layer22_attention_weight_max": 41.34611892700195,
      "activations/layer22_attention_weight_min": -21.569910049438477,
      "activations/layer23_attention_weight_max": 27.14674186706543,
      "activations/layer23_attention_weight_min": -12.845720291137695,
      "activations/layer2_attention_weight_max": 24.49401092529297,
      "activations/layer2_attention_weight_min": -24.97022247314453,
      "activations/layer3_attention_weight_max": 59.693580627441406,
      "activations/layer3_attention_weight_min": -62.74481201171875,
      "activations/layer4_attention_weight_max": 93.47982788085938,
      "activations/layer4_attention_weight_min": -83.7649154663086,
      "activations/layer5_attention_weight_max": 131.87930297851562,
      "activations/layer5_attention_weight_min": -122.01085662841797,
      "activations/layer6_attention_weight_max": 63.28041076660156,
      "activations/layer6_attention_weight_min": -53.22140121459961,
      "activations/layer7_attention_weight_max": 58.308326721191406,
      "activations/layer7_attention_weight_min": -54.50099182128906,
      "activations/layer8_attention_weight_max": 47.56149673461914,
      "activations/layer8_attention_weight_min": -43.751495361328125,
      "activations/layer9_attention_weight_max": 74.81568145751953,
      "activations/layer9_attention_weight_min": -63.048614501953125,
      "epoch": 1.54,
      "learning_rate": 0.00014147348484848485,
      "loss": 2.9911,
      "step": 26550
    },
    {
      "activations/layer0_attention_weight_max": 14.530389785766602,
      "activations/layer0_attention_weight_min": -13.28846263885498,
      "activations/layer10_attention_weight_max": 61.32757568359375,
      "activations/layer10_attention_weight_min": -55.3731689453125,
      "activations/layer11_attention_weight_max": 45.09276580810547,
      "activations/layer11_attention_weight_min": -38.93586730957031,
      "activations/layer12_attention_weight_max": 27.556808471679688,
      "activations/layer12_attention_weight_min": -24.876373291015625,
      "activations/layer13_attention_weight_max": 55.55014419555664,
      "activations/layer13_attention_weight_min": -41.216278076171875,
      "activations/layer14_attention_weight_max": 47.696468353271484,
      "activations/layer14_attention_weight_min": -34.552024841308594,
      "activations/layer15_attention_weight_max": 52.63613510131836,
      "activations/layer15_attention_weight_min": -44.49436569213867,
      "activations/layer16_attention_weight_max": 46.303314208984375,
      "activations/layer16_attention_weight_min": -31.561906814575195,
      "activations/layer17_attention_weight_max": 64.13715362548828,
      "activations/layer17_attention_weight_min": -36.885581970214844,
      "activations/layer18_attention_weight_max": 59.586181640625,
      "activations/layer18_attention_weight_min": -35.47228240966797,
      "activations/layer19_attention_weight_max": 21.38597297668457,
      "activations/layer19_attention_weight_min": -15.139566421508789,
      "activations/layer1_attention_weight_max": 15.17556381225586,
      "activations/layer1_attention_weight_min": -12.010302543640137,
      "activations/layer20_attention_weight_max": 17.941343307495117,
      "activations/layer20_attention_weight_min": -15.45057201385498,
      "activations/layer21_attention_weight_max": 43.421470642089844,
      "activations/layer21_attention_weight_min": -22.479949951171875,
      "activations/layer22_attention_weight_max": 35.591697692871094,
      "activations/layer22_attention_weight_min": -20.336294174194336,
      "activations/layer23_attention_weight_max": 24.214431762695312,
      "activations/layer23_attention_weight_min": -15.012592315673828,
      "activations/layer2_attention_weight_max": 26.312650680541992,
      "activations/layer2_attention_weight_min": -25.116622924804688,
      "activations/layer3_attention_weight_max": 56.33860397338867,
      "activations/layer3_attention_weight_min": -58.50650405883789,
      "activations/layer4_attention_weight_max": 88.73286437988281,
      "activations/layer4_attention_weight_min": -79.8569564819336,
      "activations/layer5_attention_weight_max": 127.38569641113281,
      "activations/layer5_attention_weight_min": -121.55068969726562,
      "activations/layer6_attention_weight_max": 59.1175422668457,
      "activations/layer6_attention_weight_min": -55.34455108642578,
      "activations/layer7_attention_weight_max": 56.1989860534668,
      "activations/layer7_attention_weight_min": -57.96989822387695,
      "activations/layer8_attention_weight_max": 47.66496658325195,
      "activations/layer8_attention_weight_min": -44.61253356933594,
      "activations/layer9_attention_weight_max": 62.6320686340332,
      "activations/layer9_attention_weight_min": -61.74402618408203,
      "epoch": 1.55,
      "learning_rate": 0.00014145454545454544,
      "loss": 2.9811,
      "step": 26600
    },
    {
      "activations/layer0_attention_weight_max": 14.261613845825195,
      "activations/layer0_attention_weight_min": -13.203521728515625,
      "activations/layer10_attention_weight_max": 60.223445892333984,
      "activations/layer10_attention_weight_min": -49.48456573486328,
      "activations/layer11_attention_weight_max": 41.361976623535156,
      "activations/layer11_attention_weight_min": -39.49193572998047,
      "activations/layer12_attention_weight_max": 27.49028778076172,
      "activations/layer12_attention_weight_min": -23.630720138549805,
      "activations/layer13_attention_weight_max": 55.89796447753906,
      "activations/layer13_attention_weight_min": -37.51738739013672,
      "activations/layer14_attention_weight_max": 50.20509338378906,
      "activations/layer14_attention_weight_min": -34.5779914855957,
      "activations/layer15_attention_weight_max": 53.061866760253906,
      "activations/layer15_attention_weight_min": -39.78030776977539,
      "activations/layer16_attention_weight_max": 47.13112258911133,
      "activations/layer16_attention_weight_min": -34.727516174316406,
      "activations/layer17_attention_weight_max": 55.25950241088867,
      "activations/layer17_attention_weight_min": -38.95219802856445,
      "activations/layer18_attention_weight_max": 61.6765022277832,
      "activations/layer18_attention_weight_min": -38.303401947021484,
      "activations/layer19_attention_weight_max": 19.779827117919922,
      "activations/layer19_attention_weight_min": -14.816264152526855,
      "activations/layer1_attention_weight_max": 15.080840110778809,
      "activations/layer1_attention_weight_min": -12.898990631103516,
      "activations/layer20_attention_weight_max": 17.899852752685547,
      "activations/layer20_attention_weight_min": -14.841424942016602,
      "activations/layer21_attention_weight_max": 35.980560302734375,
      "activations/layer21_attention_weight_min": -23.121105194091797,
      "activations/layer22_attention_weight_max": 35.66367721557617,
      "activations/layer22_attention_weight_min": -23.57388687133789,
      "activations/layer23_attention_weight_max": 24.224834442138672,
      "activations/layer23_attention_weight_min": -14.532759666442871,
      "activations/layer2_attention_weight_max": 25.36956787109375,
      "activations/layer2_attention_weight_min": -24.946155548095703,
      "activations/layer3_attention_weight_max": 58.09367752075195,
      "activations/layer3_attention_weight_min": -61.128562927246094,
      "activations/layer4_attention_weight_max": 88.05657196044922,
      "activations/layer4_attention_weight_min": -82.42668914794922,
      "activations/layer5_attention_weight_max": 120.6328353881836,
      "activations/layer5_attention_weight_min": -116.03170776367188,
      "activations/layer6_attention_weight_max": 57.66062927246094,
      "activations/layer6_attention_weight_min": -54.34230041503906,
      "activations/layer7_attention_weight_max": 57.88935089111328,
      "activations/layer7_attention_weight_min": -53.3193244934082,
      "activations/layer8_attention_weight_max": 48.01063919067383,
      "activations/layer8_attention_weight_min": -41.54984664916992,
      "activations/layer9_attention_weight_max": 69.31409454345703,
      "activations/layer9_attention_weight_min": -59.51002883911133,
      "epoch": 1.55,
      "learning_rate": 0.00014143560606060606,
      "loss": 2.9892,
      "step": 26650
    },
    {
      "activations/layer0_attention_weight_max": 16.247386932373047,
      "activations/layer0_attention_weight_min": -13.130879402160645,
      "activations/layer10_attention_weight_max": 67.2905044555664,
      "activations/layer10_attention_weight_min": -61.24579620361328,
      "activations/layer11_attention_weight_max": 44.50489807128906,
      "activations/layer11_attention_weight_min": -38.907691955566406,
      "activations/layer12_attention_weight_max": 30.02832794189453,
      "activations/layer12_attention_weight_min": -25.376178741455078,
      "activations/layer13_attention_weight_max": 63.79948806762695,
      "activations/layer13_attention_weight_min": -44.49227523803711,
      "activations/layer14_attention_weight_max": 55.14590072631836,
      "activations/layer14_attention_weight_min": -40.6834831237793,
      "activations/layer15_attention_weight_max": 61.104801177978516,
      "activations/layer15_attention_weight_min": -41.59295654296875,
      "activations/layer16_attention_weight_max": 46.76502990722656,
      "activations/layer16_attention_weight_min": -30.750829696655273,
      "activations/layer17_attention_weight_max": 64.4335708618164,
      "activations/layer17_attention_weight_min": -36.72847366333008,
      "activations/layer18_attention_weight_max": 66.7351303100586,
      "activations/layer18_attention_weight_min": -36.3601188659668,
      "activations/layer19_attention_weight_max": 22.630739212036133,
      "activations/layer19_attention_weight_min": -15.39335823059082,
      "activations/layer1_attention_weight_max": 15.072826385498047,
      "activations/layer1_attention_weight_min": -12.756341934204102,
      "activations/layer20_attention_weight_max": 20.53790283203125,
      "activations/layer20_attention_weight_min": -15.514013290405273,
      "activations/layer21_attention_weight_max": 36.599151611328125,
      "activations/layer21_attention_weight_min": -23.52326011657715,
      "activations/layer22_attention_weight_max": 35.055320739746094,
      "activations/layer22_attention_weight_min": -22.4404296875,
      "activations/layer23_attention_weight_max": 21.024402618408203,
      "activations/layer23_attention_weight_min": -15.959705352783203,
      "activations/layer2_attention_weight_max": 26.316389083862305,
      "activations/layer2_attention_weight_min": -24.952224731445312,
      "activations/layer3_attention_weight_max": 57.882442474365234,
      "activations/layer3_attention_weight_min": -59.6649055480957,
      "activations/layer4_attention_weight_max": 89.35047912597656,
      "activations/layer4_attention_weight_min": -79.88484954833984,
      "activations/layer5_attention_weight_max": 133.62062072753906,
      "activations/layer5_attention_weight_min": -122.87126922607422,
      "activations/layer6_attention_weight_max": 63.20428466796875,
      "activations/layer6_attention_weight_min": -56.161720275878906,
      "activations/layer7_attention_weight_max": 62.10665512084961,
      "activations/layer7_attention_weight_min": -54.81828308105469,
      "activations/layer8_attention_weight_max": 49.82571792602539,
      "activations/layer8_attention_weight_min": -45.57465744018555,
      "activations/layer9_attention_weight_max": 65.38399505615234,
      "activations/layer9_attention_weight_min": -58.96846389770508,
      "epoch": 1.55,
      "learning_rate": 0.00014141666666666667,
      "loss": 2.9955,
      "step": 26700
    },
    {
      "activations/layer0_attention_weight_max": 14.267621040344238,
      "activations/layer0_attention_weight_min": -13.144562721252441,
      "activations/layer10_attention_weight_max": 59.86690139770508,
      "activations/layer10_attention_weight_min": -52.0329475402832,
      "activations/layer11_attention_weight_max": 41.03534698486328,
      "activations/layer11_attention_weight_min": -38.82991027832031,
      "activations/layer12_attention_weight_max": 27.89215660095215,
      "activations/layer12_attention_weight_min": -23.61421012878418,
      "activations/layer13_attention_weight_max": 47.70491409301758,
      "activations/layer13_attention_weight_min": -42.3194694519043,
      "activations/layer14_attention_weight_max": 46.780765533447266,
      "activations/layer14_attention_weight_min": -37.58689498901367,
      "activations/layer15_attention_weight_max": 47.818870544433594,
      "activations/layer15_attention_weight_min": -41.90089797973633,
      "activations/layer16_attention_weight_max": 42.32966613769531,
      "activations/layer16_attention_weight_min": -34.763999938964844,
      "activations/layer17_attention_weight_max": 63.33747100830078,
      "activations/layer17_attention_weight_min": -41.76942825317383,
      "activations/layer18_attention_weight_max": 51.3940544128418,
      "activations/layer18_attention_weight_min": -36.74626922607422,
      "activations/layer19_attention_weight_max": 18.798694610595703,
      "activations/layer19_attention_weight_min": -14.342935562133789,
      "activations/layer1_attention_weight_max": 14.733649253845215,
      "activations/layer1_attention_weight_min": -11.73607349395752,
      "activations/layer20_attention_weight_max": 19.199077606201172,
      "activations/layer20_attention_weight_min": -15.427963256835938,
      "activations/layer21_attention_weight_max": 34.25021743774414,
      "activations/layer21_attention_weight_min": -21.671560287475586,
      "activations/layer22_attention_weight_max": 35.165592193603516,
      "activations/layer22_attention_weight_min": -20.3165283203125,
      "activations/layer23_attention_weight_max": 21.892576217651367,
      "activations/layer23_attention_weight_min": -15.537650108337402,
      "activations/layer2_attention_weight_max": 28.12830352783203,
      "activations/layer2_attention_weight_min": -27.12481689453125,
      "activations/layer3_attention_weight_max": 58.169090270996094,
      "activations/layer3_attention_weight_min": -61.929447174072266,
      "activations/layer4_attention_weight_max": 86.95214080810547,
      "activations/layer4_attention_weight_min": -79.2551498413086,
      "activations/layer5_attention_weight_max": 123.71946716308594,
      "activations/layer5_attention_weight_min": -119.52313232421875,
      "activations/layer6_attention_weight_max": 60.81412124633789,
      "activations/layer6_attention_weight_min": -56.09907150268555,
      "activations/layer7_attention_weight_max": 55.58226013183594,
      "activations/layer7_attention_weight_min": -53.84981155395508,
      "activations/layer8_attention_weight_max": 46.48593521118164,
      "activations/layer8_attention_weight_min": -44.275203704833984,
      "activations/layer9_attention_weight_max": 60.34287643432617,
      "activations/layer9_attention_weight_min": -60.254798889160156,
      "epoch": 1.55,
      "learning_rate": 0.00014139772727272726,
      "loss": 2.9934,
      "step": 26750
    },
    {
      "activations/layer0_attention_weight_max": 14.656523704528809,
      "activations/layer0_attention_weight_min": -13.511846542358398,
      "activations/layer10_attention_weight_max": 73.14508056640625,
      "activations/layer10_attention_weight_min": -59.912960052490234,
      "activations/layer11_attention_weight_max": 49.24102020263672,
      "activations/layer11_attention_weight_min": -39.08662033081055,
      "activations/layer12_attention_weight_max": 27.926450729370117,
      "activations/layer12_attention_weight_min": -25.90788459777832,
      "activations/layer13_attention_weight_max": 60.84983825683594,
      "activations/layer13_attention_weight_min": -44.743953704833984,
      "activations/layer14_attention_weight_max": 55.824371337890625,
      "activations/layer14_attention_weight_min": -40.098609924316406,
      "activations/layer15_attention_weight_max": 55.642173767089844,
      "activations/layer15_attention_weight_min": -41.10346984863281,
      "activations/layer16_attention_weight_max": 51.122276306152344,
      "activations/layer16_attention_weight_min": -33.17302703857422,
      "activations/layer17_attention_weight_max": 56.99540328979492,
      "activations/layer17_attention_weight_min": -38.27813720703125,
      "activations/layer18_attention_weight_max": 61.93550109863281,
      "activations/layer18_attention_weight_min": -34.377288818359375,
      "activations/layer19_attention_weight_max": 23.203046798706055,
      "activations/layer19_attention_weight_min": -16.4009952545166,
      "activations/layer1_attention_weight_max": 16.21446418762207,
      "activations/layer1_attention_weight_min": -12.108380317687988,
      "activations/layer20_attention_weight_max": 20.3255558013916,
      "activations/layer20_attention_weight_min": -14.550836563110352,
      "activations/layer21_attention_weight_max": 38.03927230834961,
      "activations/layer21_attention_weight_min": -23.71262550354004,
      "activations/layer22_attention_weight_max": 37.91548156738281,
      "activations/layer22_attention_weight_min": -20.140422821044922,
      "activations/layer23_attention_weight_max": 22.39438247680664,
      "activations/layer23_attention_weight_min": -14.20622444152832,
      "activations/layer2_attention_weight_max": 27.174224853515625,
      "activations/layer2_attention_weight_min": -24.915626525878906,
      "activations/layer3_attention_weight_max": 56.72446060180664,
      "activations/layer3_attention_weight_min": -57.76898956298828,
      "activations/layer4_attention_weight_max": 87.66412353515625,
      "activations/layer4_attention_weight_min": -80.23573303222656,
      "activations/layer5_attention_weight_max": 131.3274688720703,
      "activations/layer5_attention_weight_min": -116.94758605957031,
      "activations/layer6_attention_weight_max": 63.217872619628906,
      "activations/layer6_attention_weight_min": -55.008544921875,
      "activations/layer7_attention_weight_max": 66.97775268554688,
      "activations/layer7_attention_weight_min": -53.21842956542969,
      "activations/layer8_attention_weight_max": 56.15272903442383,
      "activations/layer8_attention_weight_min": -48.1790657043457,
      "activations/layer9_attention_weight_max": 73.81291198730469,
      "activations/layer9_attention_weight_min": -67.38768768310547,
      "epoch": 1.56,
      "learning_rate": 0.00014137878787878787,
      "loss": 3.0058,
      "step": 26800
    },
    {
      "activations/layer0_attention_weight_max": 15.166712760925293,
      "activations/layer0_attention_weight_min": -13.625046730041504,
      "activations/layer10_attention_weight_max": 62.75419616699219,
      "activations/layer10_attention_weight_min": -55.12525177001953,
      "activations/layer11_attention_weight_max": 40.49081039428711,
      "activations/layer11_attention_weight_min": -37.03602981567383,
      "activations/layer12_attention_weight_max": 33.39886474609375,
      "activations/layer12_attention_weight_min": -25.513885498046875,
      "activations/layer13_attention_weight_max": 59.414031982421875,
      "activations/layer13_attention_weight_min": -43.89931869506836,
      "activations/layer14_attention_weight_max": 45.47598648071289,
      "activations/layer14_attention_weight_min": -37.04700469970703,
      "activations/layer15_attention_weight_max": 54.218658447265625,
      "activations/layer15_attention_weight_min": -40.13319396972656,
      "activations/layer16_attention_weight_max": 45.07481002807617,
      "activations/layer16_attention_weight_min": -32.6966667175293,
      "activations/layer17_attention_weight_max": 61.79338455200195,
      "activations/layer17_attention_weight_min": -40.13801574707031,
      "activations/layer18_attention_weight_max": 61.803646087646484,
      "activations/layer18_attention_weight_min": -37.136871337890625,
      "activations/layer19_attention_weight_max": 20.127077102661133,
      "activations/layer19_attention_weight_min": -13.99235725402832,
      "activations/layer1_attention_weight_max": 15.466859817504883,
      "activations/layer1_attention_weight_min": -12.965448379516602,
      "activations/layer20_attention_weight_max": 21.181806564331055,
      "activations/layer20_attention_weight_min": -16.49393653869629,
      "activations/layer21_attention_weight_max": 41.32539367675781,
      "activations/layer21_attention_weight_min": -22.539119720458984,
      "activations/layer22_attention_weight_max": 38.66646194458008,
      "activations/layer22_attention_weight_min": -21.1684513092041,
      "activations/layer23_attention_weight_max": 24.015228271484375,
      "activations/layer23_attention_weight_min": -14.877302169799805,
      "activations/layer2_attention_weight_max": 24.93649673461914,
      "activations/layer2_attention_weight_min": -24.33275604248047,
      "activations/layer3_attention_weight_max": 59.738990783691406,
      "activations/layer3_attention_weight_min": -62.62870788574219,
      "activations/layer4_attention_weight_max": 89.96410369873047,
      "activations/layer4_attention_weight_min": -83.25835418701172,
      "activations/layer5_attention_weight_max": 124.7647705078125,
      "activations/layer5_attention_weight_min": -121.11375427246094,
      "activations/layer6_attention_weight_max": 60.37364959716797,
      "activations/layer6_attention_weight_min": -55.75766372680664,
      "activations/layer7_attention_weight_max": 59.974918365478516,
      "activations/layer7_attention_weight_min": -54.85682678222656,
      "activations/layer8_attention_weight_max": 47.14638137817383,
      "activations/layer8_attention_weight_min": -46.64912796020508,
      "activations/layer9_attention_weight_max": 62.114376068115234,
      "activations/layer9_attention_weight_min": -61.87235641479492,
      "epoch": 1.56,
      "learning_rate": 0.00014135984848484846,
      "loss": 2.9996,
      "step": 26850
    },
    {
      "activations/layer0_attention_weight_max": 14.56224250793457,
      "activations/layer0_attention_weight_min": -13.615776062011719,
      "activations/layer10_attention_weight_max": 60.88109588623047,
      "activations/layer10_attention_weight_min": -55.3778190612793,
      "activations/layer11_attention_weight_max": 44.634368896484375,
      "activations/layer11_attention_weight_min": -38.25054168701172,
      "activations/layer12_attention_weight_max": 26.79023551940918,
      "activations/layer12_attention_weight_min": -26.062299728393555,
      "activations/layer13_attention_weight_max": 79.327880859375,
      "activations/layer13_attention_weight_min": -62.540794372558594,
      "activations/layer14_attention_weight_max": 63.35224914550781,
      "activations/layer14_attention_weight_min": -47.49311828613281,
      "activations/layer15_attention_weight_max": 59.281517028808594,
      "activations/layer15_attention_weight_min": -45.272254943847656,
      "activations/layer16_attention_weight_max": 43.07173156738281,
      "activations/layer16_attention_weight_min": -31.893339157104492,
      "activations/layer17_attention_weight_max": 56.64896774291992,
      "activations/layer17_attention_weight_min": -40.65987777709961,
      "activations/layer18_attention_weight_max": 57.37685012817383,
      "activations/layer18_attention_weight_min": -35.70447540283203,
      "activations/layer19_attention_weight_max": 23.16419792175293,
      "activations/layer19_attention_weight_min": -14.097332954406738,
      "activations/layer1_attention_weight_max": 15.223556518554688,
      "activations/layer1_attention_weight_min": -11.93252182006836,
      "activations/layer20_attention_weight_max": 18.19162940979004,
      "activations/layer20_attention_weight_min": -16.10220718383789,
      "activations/layer21_attention_weight_max": 38.59785079956055,
      "activations/layer21_attention_weight_min": -21.429996490478516,
      "activations/layer22_attention_weight_max": 35.964176177978516,
      "activations/layer22_attention_weight_min": -21.46101951599121,
      "activations/layer23_attention_weight_max": 25.5274658203125,
      "activations/layer23_attention_weight_min": -14.97020149230957,
      "activations/layer2_attention_weight_max": 24.637042999267578,
      "activations/layer2_attention_weight_min": -23.883962631225586,
      "activations/layer3_attention_weight_max": 59.9906120300293,
      "activations/layer3_attention_weight_min": -63.23955154418945,
      "activations/layer4_attention_weight_max": 90.96001434326172,
      "activations/layer4_attention_weight_min": -84.76512908935547,
      "activations/layer5_attention_weight_max": 131.18641662597656,
      "activations/layer5_attention_weight_min": -117.05489349365234,
      "activations/layer6_attention_weight_max": 63.11872482299805,
      "activations/layer6_attention_weight_min": -52.46971130371094,
      "activations/layer7_attention_weight_max": 64.098388671875,
      "activations/layer7_attention_weight_min": -54.60181427001953,
      "activations/layer8_attention_weight_max": 52.59589385986328,
      "activations/layer8_attention_weight_min": -48.10076141357422,
      "activations/layer9_attention_weight_max": 75.02430725097656,
      "activations/layer9_attention_weight_min": -72.4663314819336,
      "epoch": 1.56,
      "learning_rate": 0.00014134090909090908,
      "loss": 3.0182,
      "step": 26900
    },
    {
      "activations/layer0_attention_weight_max": 14.322779655456543,
      "activations/layer0_attention_weight_min": -13.388663291931152,
      "activations/layer10_attention_weight_max": 68.314697265625,
      "activations/layer10_attention_weight_min": -51.94922637939453,
      "activations/layer11_attention_weight_max": 44.45806121826172,
      "activations/layer11_attention_weight_min": -38.85603332519531,
      "activations/layer12_attention_weight_max": 31.75049591064453,
      "activations/layer12_attention_weight_min": -24.75005340576172,
      "activations/layer13_attention_weight_max": 61.56501770019531,
      "activations/layer13_attention_weight_min": -42.82933807373047,
      "activations/layer14_attention_weight_max": 65.6281967163086,
      "activations/layer14_attention_weight_min": -38.85957336425781,
      "activations/layer15_attention_weight_max": 63.63223648071289,
      "activations/layer15_attention_weight_min": -38.9065055847168,
      "activations/layer16_attention_weight_max": 44.9781608581543,
      "activations/layer16_attention_weight_min": -29.398303985595703,
      "activations/layer17_attention_weight_max": 64.70999908447266,
      "activations/layer17_attention_weight_min": -34.176300048828125,
      "activations/layer18_attention_weight_max": 69.66236877441406,
      "activations/layer18_attention_weight_min": -33.65480041503906,
      "activations/layer19_attention_weight_max": 23.412931442260742,
      "activations/layer19_attention_weight_min": -16.266611099243164,
      "activations/layer1_attention_weight_max": 14.965072631835938,
      "activations/layer1_attention_weight_min": -12.910993576049805,
      "activations/layer20_attention_weight_max": 18.05129623413086,
      "activations/layer20_attention_weight_min": -19.31995964050293,
      "activations/layer21_attention_weight_max": 41.52833938598633,
      "activations/layer21_attention_weight_min": -20.819833755493164,
      "activations/layer22_attention_weight_max": 38.25038146972656,
      "activations/layer22_attention_weight_min": -21.24081802368164,
      "activations/layer23_attention_weight_max": 24.509071350097656,
      "activations/layer23_attention_weight_min": -17.99918556213379,
      "activations/layer2_attention_weight_max": 25.190336227416992,
      "activations/layer2_attention_weight_min": -25.317848205566406,
      "activations/layer3_attention_weight_max": 62.446102142333984,
      "activations/layer3_attention_weight_min": -65.08172607421875,
      "activations/layer4_attention_weight_max": 87.05787658691406,
      "activations/layer4_attention_weight_min": -80.65486145019531,
      "activations/layer5_attention_weight_max": 124.10359191894531,
      "activations/layer5_attention_weight_min": -118.85043334960938,
      "activations/layer6_attention_weight_max": 59.6887321472168,
      "activations/layer6_attention_weight_min": -55.067901611328125,
      "activations/layer7_attention_weight_max": 59.08241653442383,
      "activations/layer7_attention_weight_min": -54.39323806762695,
      "activations/layer8_attention_weight_max": 48.79522705078125,
      "activations/layer8_attention_weight_min": -40.99100112915039,
      "activations/layer9_attention_weight_max": 64.69586944580078,
      "activations/layer9_attention_weight_min": -62.676998138427734,
      "epoch": 1.57,
      "learning_rate": 0.0001413219696969697,
      "loss": 2.9964,
      "step": 26950
    },
    {
      "activations/layer0_attention_weight_max": 13.782964706420898,
      "activations/layer0_attention_weight_min": -13.760042190551758,
      "activations/layer10_attention_weight_max": 57.026519775390625,
      "activations/layer10_attention_weight_min": -49.178802490234375,
      "activations/layer11_attention_weight_max": 41.32087707519531,
      "activations/layer11_attention_weight_min": -38.55258560180664,
      "activations/layer12_attention_weight_max": 30.957120895385742,
      "activations/layer12_attention_weight_min": -24.742719650268555,
      "activations/layer13_attention_weight_max": 57.915374755859375,
      "activations/layer13_attention_weight_min": -47.895790100097656,
      "activations/layer14_attention_weight_max": 49.41647720336914,
      "activations/layer14_attention_weight_min": -38.860198974609375,
      "activations/layer15_attention_weight_max": 54.77116775512695,
      "activations/layer15_attention_weight_min": -39.84389114379883,
      "activations/layer16_attention_weight_max": 47.61607360839844,
      "activations/layer16_attention_weight_min": -30.624523162841797,
      "activations/layer17_attention_weight_max": 61.69157028198242,
      "activations/layer17_attention_weight_min": -36.90238952636719,
      "activations/layer18_attention_weight_max": 65.2856216430664,
      "activations/layer18_attention_weight_min": -33.30474853515625,
      "activations/layer19_attention_weight_max": 23.814756393432617,
      "activations/layer19_attention_weight_min": -14.586825370788574,
      "activations/layer1_attention_weight_max": 15.193778991699219,
      "activations/layer1_attention_weight_min": -12.742466926574707,
      "activations/layer20_attention_weight_max": 21.16666603088379,
      "activations/layer20_attention_weight_min": -15.956581115722656,
      "activations/layer21_attention_weight_max": 42.72841262817383,
      "activations/layer21_attention_weight_min": -21.080663681030273,
      "activations/layer22_attention_weight_max": 40.4234504699707,
      "activations/layer22_attention_weight_min": -19.655677795410156,
      "activations/layer23_attention_weight_max": 24.169652938842773,
      "activations/layer23_attention_weight_min": -16.728477478027344,
      "activations/layer2_attention_weight_max": 27.66448211669922,
      "activations/layer2_attention_weight_min": -25.4193172454834,
      "activations/layer3_attention_weight_max": 57.92542266845703,
      "activations/layer3_attention_weight_min": -60.01934051513672,
      "activations/layer4_attention_weight_max": 88.93226623535156,
      "activations/layer4_attention_weight_min": -79.0964584350586,
      "activations/layer5_attention_weight_max": 121.54347229003906,
      "activations/layer5_attention_weight_min": -113.27648162841797,
      "activations/layer6_attention_weight_max": 60.052433013916016,
      "activations/layer6_attention_weight_min": -53.53374481201172,
      "activations/layer7_attention_weight_max": 57.08108139038086,
      "activations/layer7_attention_weight_min": -53.684295654296875,
      "activations/layer8_attention_weight_max": 45.698829650878906,
      "activations/layer8_attention_weight_min": -41.95768356323242,
      "activations/layer9_attention_weight_max": 65.90766906738281,
      "activations/layer9_attention_weight_min": -60.4680061340332,
      "epoch": 1.57,
      "learning_rate": 0.00014130303030303028,
      "loss": 2.994,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_loss": 2.9453125,
      "eval_runtime": 8.4774,
      "eval_samples_per_second": 506.525,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_openwebtext_loss": 2.9453125,
      "eval_openwebtext_ppl": 19.016604146815915,
      "eval_openwebtext_runtime": 8.4774,
      "eval_openwebtext_samples_per_second": 506.525,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_wikitext_loss": 3.224609375,
      "eval_wikitext_ppl": 25.1437504597293,
      "eval_wikitext_runtime": 1.9479,
      "eval_wikitext_samples_per_second": 234.094,
      "step": 27000
    },
    {
      "epoch": 1.57,
      "eval_lambada_loss": 3.205078125,
      "eval_lambada_ppl": 24.657426302639607,
      "eval_lambada_runtime": 9.5765,
      "eval_lambada_samples_per_second": 508.432,
      "step": 27000
    },
    {
      "activations/layer0_attention_weight_max": 13.917512893676758,
      "activations/layer0_attention_weight_min": -13.888378143310547,
      "activations/layer10_attention_weight_max": 64.8532485961914,
      "activations/layer10_attention_weight_min": -54.99970626831055,
      "activations/layer11_attention_weight_max": 47.31909942626953,
      "activations/layer11_attention_weight_min": -47.89361572265625,
      "activations/layer12_attention_weight_max": 29.698436737060547,
      "activations/layer12_attention_weight_min": -25.148521423339844,
      "activations/layer13_attention_weight_max": 60.23615646362305,
      "activations/layer13_attention_weight_min": -44.48209762573242,
      "activations/layer14_attention_weight_max": 70.50718688964844,
      "activations/layer14_attention_weight_min": -44.383792877197266,
      "activations/layer15_attention_weight_max": 54.24887466430664,
      "activations/layer15_attention_weight_min": -41.44300842285156,
      "activations/layer16_attention_weight_max": 44.12666320800781,
      "activations/layer16_attention_weight_min": -31.912364959716797,
      "activations/layer17_attention_weight_max": 62.06355667114258,
      "activations/layer17_attention_weight_min": -37.67678451538086,
      "activations/layer18_attention_weight_max": 68.62940216064453,
      "activations/layer18_attention_weight_min": -33.27200698852539,
      "activations/layer19_attention_weight_max": 26.8927059173584,
      "activations/layer19_attention_weight_min": -14.5347900390625,
      "activations/layer1_attention_weight_max": 16.006269454956055,
      "activations/layer1_attention_weight_min": -12.360260009765625,
      "activations/layer20_attention_weight_max": 19.422510147094727,
      "activations/layer20_attention_weight_min": -15.212657928466797,
      "activations/layer21_attention_weight_max": 43.737510681152344,
      "activations/layer21_attention_weight_min": -21.522905349731445,
      "activations/layer22_attention_weight_max": 37.370052337646484,
      "activations/layer22_attention_weight_min": -22.916446685791016,
      "activations/layer23_attention_weight_max": 23.246631622314453,
      "activations/layer23_attention_weight_min": -13.500553131103516,
      "activations/layer2_attention_weight_max": 25.007421493530273,
      "activations/layer2_attention_weight_min": -24.53352928161621,
      "activations/layer3_attention_weight_max": 60.124996185302734,
      "activations/layer3_attention_weight_min": -63.209598541259766,
      "activations/layer4_attention_weight_max": 94.97578430175781,
      "activations/layer4_attention_weight_min": -84.95709991455078,
      "activations/layer5_attention_weight_max": 134.84439086914062,
      "activations/layer5_attention_weight_min": -120.53276062011719,
      "activations/layer6_attention_weight_max": 63.12176513671875,
      "activations/layer6_attention_weight_min": -54.81367492675781,
      "activations/layer7_attention_weight_max": 60.82078552246094,
      "activations/layer7_attention_weight_min": -62.43258285522461,
      "activations/layer8_attention_weight_max": 53.91914749145508,
      "activations/layer8_attention_weight_min": -49.1995735168457,
      "activations/layer9_attention_weight_max": 72.7685317993164,
      "activations/layer9_attention_weight_min": -71.24706268310547,
      "epoch": 1.57,
      "learning_rate": 0.0001412840909090909,
      "loss": 2.9896,
      "step": 27050
    },
    {
      "activations/layer0_attention_weight_max": 15.149675369262695,
      "activations/layer0_attention_weight_min": -13.553510665893555,
      "activations/layer10_attention_weight_max": 60.87894058227539,
      "activations/layer10_attention_weight_min": -53.73313522338867,
      "activations/layer11_attention_weight_max": 44.34381866455078,
      "activations/layer11_attention_weight_min": -36.59771728515625,
      "activations/layer12_attention_weight_max": 27.663978576660156,
      "activations/layer12_attention_weight_min": -25.208633422851562,
      "activations/layer13_attention_weight_max": 66.0489501953125,
      "activations/layer13_attention_weight_min": -50.65126419067383,
      "activations/layer14_attention_weight_max": 63.813594818115234,
      "activations/layer14_attention_weight_min": -45.10322570800781,
      "activations/layer15_attention_weight_max": 50.931724548339844,
      "activations/layer15_attention_weight_min": -40.81943893432617,
      "activations/layer16_attention_weight_max": 42.75137710571289,
      "activations/layer16_attention_weight_min": -32.273826599121094,
      "activations/layer17_attention_weight_max": 54.28767776489258,
      "activations/layer17_attention_weight_min": -35.69605255126953,
      "activations/layer18_attention_weight_max": 64.53328704833984,
      "activations/layer18_attention_weight_min": -33.634986877441406,
      "activations/layer19_attention_weight_max": 20.34600830078125,
      "activations/layer19_attention_weight_min": -14.154194831848145,
      "activations/layer1_attention_weight_max": 15.286730766296387,
      "activations/layer1_attention_weight_min": -14.760213851928711,
      "activations/layer20_attention_weight_max": 19.332950592041016,
      "activations/layer20_attention_weight_min": -14.4591703414917,
      "activations/layer21_attention_weight_max": 39.06401062011719,
      "activations/layer21_attention_weight_min": -20.91645050048828,
      "activations/layer22_attention_weight_max": 32.83794021606445,
      "activations/layer22_attention_weight_min": -18.94426155090332,
      "activations/layer23_attention_weight_max": 20.590185165405273,
      "activations/layer23_attention_weight_min": -13.296710014343262,
      "activations/layer2_attention_weight_max": 26.88100242614746,
      "activations/layer2_attention_weight_min": -24.52118682861328,
      "activations/layer3_attention_weight_max": 61.50489044189453,
      "activations/layer3_attention_weight_min": -60.894386291503906,
      "activations/layer4_attention_weight_max": 92.78121185302734,
      "activations/layer4_attention_weight_min": -83.22560119628906,
      "activations/layer5_attention_weight_max": 136.39642333984375,
      "activations/layer5_attention_weight_min": -118.03952026367188,
      "activations/layer6_attention_weight_max": 64.70301055908203,
      "activations/layer6_attention_weight_min": -56.30610656738281,
      "activations/layer7_attention_weight_max": 60.252166748046875,
      "activations/layer7_attention_weight_min": -55.828956604003906,
      "activations/layer8_attention_weight_max": 47.781272888183594,
      "activations/layer8_attention_weight_min": -42.878658294677734,
      "activations/layer9_attention_weight_max": 70.56108856201172,
      "activations/layer9_attention_weight_min": -64.35018157958984,
      "epoch": 1.57,
      "learning_rate": 0.0001412651515151515,
      "loss": 2.9913,
      "step": 27100
    },
    {
      "activations/layer0_attention_weight_max": 14.657511711120605,
      "activations/layer0_attention_weight_min": -13.741515159606934,
      "activations/layer10_attention_weight_max": 61.97904586791992,
      "activations/layer10_attention_weight_min": -56.730308532714844,
      "activations/layer11_attention_weight_max": 42.1363410949707,
      "activations/layer11_attention_weight_min": -38.48326873779297,
      "activations/layer12_attention_weight_max": 26.17145538330078,
      "activations/layer12_attention_weight_min": -23.549610137939453,
      "activations/layer13_attention_weight_max": 61.075016021728516,
      "activations/layer13_attention_weight_min": -50.52287292480469,
      "activations/layer14_attention_weight_max": 51.88443374633789,
      "activations/layer14_attention_weight_min": -40.71149444580078,
      "activations/layer15_attention_weight_max": 59.12488555908203,
      "activations/layer15_attention_weight_min": -41.946712493896484,
      "activations/layer16_attention_weight_max": 40.70130157470703,
      "activations/layer16_attention_weight_min": -34.64920425415039,
      "activations/layer17_attention_weight_max": 54.32216262817383,
      "activations/layer17_attention_weight_min": -38.905696868896484,
      "activations/layer18_attention_weight_max": 56.3228759765625,
      "activations/layer18_attention_weight_min": -34.25666809082031,
      "activations/layer19_attention_weight_max": 19.853010177612305,
      "activations/layer19_attention_weight_min": -16.622779846191406,
      "activations/layer1_attention_weight_max": 15.249109268188477,
      "activations/layer1_attention_weight_min": -12.092239379882812,
      "activations/layer20_attention_weight_max": 20.02994728088379,
      "activations/layer20_attention_weight_min": -15.928927421569824,
      "activations/layer21_attention_weight_max": 35.14339065551758,
      "activations/layer21_attention_weight_min": -23.25908660888672,
      "activations/layer22_attention_weight_max": 34.16813278198242,
      "activations/layer22_attention_weight_min": -24.891681671142578,
      "activations/layer23_attention_weight_max": 20.852142333984375,
      "activations/layer23_attention_weight_min": -17.743919372558594,
      "activations/layer2_attention_weight_max": 24.51422882080078,
      "activations/layer2_attention_weight_min": -24.618404388427734,
      "activations/layer3_attention_weight_max": 56.78800582885742,
      "activations/layer3_attention_weight_min": -59.579105377197266,
      "activations/layer4_attention_weight_max": 84.38267517089844,
      "activations/layer4_attention_weight_min": -80.12438201904297,
      "activations/layer5_attention_weight_max": 126.5338134765625,
      "activations/layer5_attention_weight_min": -121.34475708007812,
      "activations/layer6_attention_weight_max": 62.707176208496094,
      "activations/layer6_attention_weight_min": -53.10301208496094,
      "activations/layer7_attention_weight_max": 57.62601852416992,
      "activations/layer7_attention_weight_min": -56.20198440551758,
      "activations/layer8_attention_weight_max": 48.555789947509766,
      "activations/layer8_attention_weight_min": -43.63460159301758,
      "activations/layer9_attention_weight_max": 62.19961166381836,
      "activations/layer9_attention_weight_min": -60.09525680541992,
      "epoch": 1.58,
      "learning_rate": 0.0001412462121212121,
      "loss": 2.9994,
      "step": 27150
    },
    {
      "activations/layer0_attention_weight_max": 13.830236434936523,
      "activations/layer0_attention_weight_min": -13.237663269042969,
      "activations/layer10_attention_weight_max": 62.18115234375,
      "activations/layer10_attention_weight_min": -50.446678161621094,
      "activations/layer11_attention_weight_max": 46.06023406982422,
      "activations/layer11_attention_weight_min": -45.46171569824219,
      "activations/layer12_attention_weight_max": 33.580081939697266,
      "activations/layer12_attention_weight_min": -25.540708541870117,
      "activations/layer13_attention_weight_max": 66.67242431640625,
      "activations/layer13_attention_weight_min": -47.47843933105469,
      "activations/layer14_attention_weight_max": 61.86696243286133,
      "activations/layer14_attention_weight_min": -44.0335578918457,
      "activations/layer15_attention_weight_max": 58.0564079284668,
      "activations/layer15_attention_weight_min": -40.627254486083984,
      "activations/layer16_attention_weight_max": 47.46541213989258,
      "activations/layer16_attention_weight_min": -30.946271896362305,
      "activations/layer17_attention_weight_max": 59.98042297363281,
      "activations/layer17_attention_weight_min": -34.0593376159668,
      "activations/layer18_attention_weight_max": 57.60862350463867,
      "activations/layer18_attention_weight_min": -33.025943756103516,
      "activations/layer19_attention_weight_max": 22.032896041870117,
      "activations/layer19_attention_weight_min": -14.515215873718262,
      "activations/layer1_attention_weight_max": 14.946576118469238,
      "activations/layer1_attention_weight_min": -13.022533416748047,
      "activations/layer20_attention_weight_max": 20.269779205322266,
      "activations/layer20_attention_weight_min": -17.225135803222656,
      "activations/layer21_attention_weight_max": 35.840545654296875,
      "activations/layer21_attention_weight_min": -19.225727081298828,
      "activations/layer22_attention_weight_max": 35.31129455566406,
      "activations/layer22_attention_weight_min": -19.202016830444336,
      "activations/layer23_attention_weight_max": 23.275203704833984,
      "activations/layer23_attention_weight_min": -14.620319366455078,
      "activations/layer2_attention_weight_max": 25.002731323242188,
      "activations/layer2_attention_weight_min": -25.04327964782715,
      "activations/layer3_attention_weight_max": 57.74171829223633,
      "activations/layer3_attention_weight_min": -63.960636138916016,
      "activations/layer4_attention_weight_max": 87.34722137451172,
      "activations/layer4_attention_weight_min": -82.48907470703125,
      "activations/layer5_attention_weight_max": 125.84475708007812,
      "activations/layer5_attention_weight_min": -118.24247741699219,
      "activations/layer6_attention_weight_max": 59.26423645019531,
      "activations/layer6_attention_weight_min": -56.91777420043945,
      "activations/layer7_attention_weight_max": 59.87312698364258,
      "activations/layer7_attention_weight_min": -56.35054016113281,
      "activations/layer8_attention_weight_max": 48.29468536376953,
      "activations/layer8_attention_weight_min": -45.060970306396484,
      "activations/layer9_attention_weight_max": 59.36650466918945,
      "activations/layer9_attention_weight_min": -61.5687370300293,
      "epoch": 1.58,
      "learning_rate": 0.0001412272727272727,
      "loss": 2.9957,
      "step": 27200
    },
    {
      "activations/layer0_attention_weight_max": 14.85310173034668,
      "activations/layer0_attention_weight_min": -12.541419982910156,
      "activations/layer10_attention_weight_max": 59.24252700805664,
      "activations/layer10_attention_weight_min": -51.376495361328125,
      "activations/layer11_attention_weight_max": 46.762657165527344,
      "activations/layer11_attention_weight_min": -41.3551025390625,
      "activations/layer12_attention_weight_max": 29.044849395751953,
      "activations/layer12_attention_weight_min": -27.487930297851562,
      "activations/layer13_attention_weight_max": 62.09275817871094,
      "activations/layer13_attention_weight_min": -47.856258392333984,
      "activations/layer14_attention_weight_max": 63.53289794921875,
      "activations/layer14_attention_weight_min": -42.30948257446289,
      "activations/layer15_attention_weight_max": 52.31903839111328,
      "activations/layer15_attention_weight_min": -36.5220832824707,
      "activations/layer16_attention_weight_max": 43.31743621826172,
      "activations/layer16_attention_weight_min": -28.100122451782227,
      "activations/layer17_attention_weight_max": 56.60783767700195,
      "activations/layer17_attention_weight_min": -36.303428649902344,
      "activations/layer18_attention_weight_max": 58.99831771850586,
      "activations/layer18_attention_weight_min": -35.434791564941406,
      "activations/layer19_attention_weight_max": 21.909807205200195,
      "activations/layer19_attention_weight_min": -14.694360733032227,
      "activations/layer1_attention_weight_max": 15.5211820602417,
      "activations/layer1_attention_weight_min": -17.195642471313477,
      "activations/layer20_attention_weight_max": 21.996688842773438,
      "activations/layer20_attention_weight_min": -15.478548049926758,
      "activations/layer21_attention_weight_max": 43.63908004760742,
      "activations/layer21_attention_weight_min": -22.3470401763916,
      "activations/layer22_attention_weight_max": 40.62403106689453,
      "activations/layer22_attention_weight_min": -23.49144744873047,
      "activations/layer23_attention_weight_max": 25.109594345092773,
      "activations/layer23_attention_weight_min": -17.662857055664062,
      "activations/layer2_attention_weight_max": 27.643104553222656,
      "activations/layer2_attention_weight_min": -27.572959899902344,
      "activations/layer3_attention_weight_max": 61.39399337768555,
      "activations/layer3_attention_weight_min": -63.9775505065918,
      "activations/layer4_attention_weight_max": 90.42900085449219,
      "activations/layer4_attention_weight_min": -78.54405212402344,
      "activations/layer5_attention_weight_max": 127.39159393310547,
      "activations/layer5_attention_weight_min": -115.03387451171875,
      "activations/layer6_attention_weight_max": 62.59464645385742,
      "activations/layer6_attention_weight_min": -52.47439956665039,
      "activations/layer7_attention_weight_max": 60.6004753112793,
      "activations/layer7_attention_weight_min": -56.13465881347656,
      "activations/layer8_attention_weight_max": 50.8729133605957,
      "activations/layer8_attention_weight_min": -45.823089599609375,
      "activations/layer9_attention_weight_max": 67.03270721435547,
      "activations/layer9_attention_weight_min": -60.613990783691406,
      "epoch": 1.58,
      "learning_rate": 0.00014120833333333333,
      "loss": 2.9817,
      "step": 27250
    },
    {
      "activations/layer0_attention_weight_max": 14.514766693115234,
      "activations/layer0_attention_weight_min": -12.972528457641602,
      "activations/layer10_attention_weight_max": 66.30026245117188,
      "activations/layer10_attention_weight_min": -55.081146240234375,
      "activations/layer11_attention_weight_max": 41.767356872558594,
      "activations/layer11_attention_weight_min": -37.20884323120117,
      "activations/layer12_attention_weight_max": 29.134801864624023,
      "activations/layer12_attention_weight_min": -24.67318344116211,
      "activations/layer13_attention_weight_max": 56.47386169433594,
      "activations/layer13_attention_weight_min": -43.29441833496094,
      "activations/layer14_attention_weight_max": 59.664756774902344,
      "activations/layer14_attention_weight_min": -42.058250427246094,
      "activations/layer15_attention_weight_max": 56.452674865722656,
      "activations/layer15_attention_weight_min": -42.34284973144531,
      "activations/layer16_attention_weight_max": 47.22416687011719,
      "activations/layer16_attention_weight_min": -31.415925979614258,
      "activations/layer17_attention_weight_max": 56.228519439697266,
      "activations/layer17_attention_weight_min": -36.956146240234375,
      "activations/layer18_attention_weight_max": 59.29326629638672,
      "activations/layer18_attention_weight_min": -35.73505783081055,
      "activations/layer19_attention_weight_max": 19.266157150268555,
      "activations/layer19_attention_weight_min": -14.960833549499512,
      "activations/layer1_attention_weight_max": 15.595327377319336,
      "activations/layer1_attention_weight_min": -11.85758113861084,
      "activations/layer20_attention_weight_max": 20.56320571899414,
      "activations/layer20_attention_weight_min": -17.00125503540039,
      "activations/layer21_attention_weight_max": 40.94732666015625,
      "activations/layer21_attention_weight_min": -23.265615463256836,
      "activations/layer22_attention_weight_max": 36.17776870727539,
      "activations/layer22_attention_weight_min": -21.84694480895996,
      "activations/layer23_attention_weight_max": 23.481285095214844,
      "activations/layer23_attention_weight_min": -15.095012664794922,
      "activations/layer2_attention_weight_max": 25.203876495361328,
      "activations/layer2_attention_weight_min": -24.769987106323242,
      "activations/layer3_attention_weight_max": 58.33021545410156,
      "activations/layer3_attention_weight_min": -58.912757873535156,
      "activations/layer4_attention_weight_max": 89.43476867675781,
      "activations/layer4_attention_weight_min": -79.10980987548828,
      "activations/layer5_attention_weight_max": 128.0281219482422,
      "activations/layer5_attention_weight_min": -120.11781311035156,
      "activations/layer6_attention_weight_max": 60.87567138671875,
      "activations/layer6_attention_weight_min": -52.68991470336914,
      "activations/layer7_attention_weight_max": 59.39421844482422,
      "activations/layer7_attention_weight_min": -58.55698776245117,
      "activations/layer8_attention_weight_max": 47.444488525390625,
      "activations/layer8_attention_weight_min": -42.52559280395508,
      "activations/layer9_attention_weight_max": 73.0570297241211,
      "activations/layer9_attention_weight_min": -65.55665588378906,
      "epoch": 1.59,
      "learning_rate": 0.00014118939393939392,
      "loss": 3.0057,
      "step": 27300
    },
    {
      "activations/layer0_attention_weight_max": 14.581108093261719,
      "activations/layer0_attention_weight_min": -13.130133628845215,
      "activations/layer10_attention_weight_max": 65.14169311523438,
      "activations/layer10_attention_weight_min": -53.117305755615234,
      "activations/layer11_attention_weight_max": 46.602622985839844,
      "activations/layer11_attention_weight_min": -39.154544830322266,
      "activations/layer12_attention_weight_max": 26.63981056213379,
      "activations/layer12_attention_weight_min": -26.862831115722656,
      "activations/layer13_attention_weight_max": 64.4045639038086,
      "activations/layer13_attention_weight_min": -52.39836502075195,
      "activations/layer14_attention_weight_max": 45.769874572753906,
      "activations/layer14_attention_weight_min": -37.459835052490234,
      "activations/layer15_attention_weight_max": 66.73273468017578,
      "activations/layer15_attention_weight_min": -48.228599548339844,
      "activations/layer16_attention_weight_max": 43.37443923950195,
      "activations/layer16_attention_weight_min": -29.72907066345215,
      "activations/layer17_attention_weight_max": 58.04783248901367,
      "activations/layer17_attention_weight_min": -38.20365524291992,
      "activations/layer18_attention_weight_max": 57.657676696777344,
      "activations/layer18_attention_weight_min": -34.694698333740234,
      "activations/layer19_attention_weight_max": 19.883655548095703,
      "activations/layer19_attention_weight_min": -14.13322925567627,
      "activations/layer1_attention_weight_max": 15.968863487243652,
      "activations/layer1_attention_weight_min": -12.316969871520996,
      "activations/layer20_attention_weight_max": 21.51477813720703,
      "activations/layer20_attention_weight_min": -15.667132377624512,
      "activations/layer21_attention_weight_max": 38.236488342285156,
      "activations/layer21_attention_weight_min": -19.584835052490234,
      "activations/layer22_attention_weight_max": 37.56873321533203,
      "activations/layer22_attention_weight_min": -21.813243865966797,
      "activations/layer23_attention_weight_max": 23.6731014251709,
      "activations/layer23_attention_weight_min": -15.6502685546875,
      "activations/layer2_attention_weight_max": 24.790136337280273,
      "activations/layer2_attention_weight_min": -25.585851669311523,
      "activations/layer3_attention_weight_max": 58.40439987182617,
      "activations/layer3_attention_weight_min": -62.264312744140625,
      "activations/layer4_attention_weight_max": 87.14007568359375,
      "activations/layer4_attention_weight_min": -79.1259536743164,
      "activations/layer5_attention_weight_max": 122.50277709960938,
      "activations/layer5_attention_weight_min": -110.24958801269531,
      "activations/layer6_attention_weight_max": 60.037452697753906,
      "activations/layer6_attention_weight_min": -54.26123809814453,
      "activations/layer7_attention_weight_max": 58.10334777832031,
      "activations/layer7_attention_weight_min": -53.91380310058594,
      "activations/layer8_attention_weight_max": 45.868896484375,
      "activations/layer8_attention_weight_min": -42.80341339111328,
      "activations/layer9_attention_weight_max": 63.56193923950195,
      "activations/layer9_attention_weight_min": -66.26689147949219,
      "epoch": 1.59,
      "learning_rate": 0.00014117045454545453,
      "loss": 2.9936,
      "step": 27350
    },
    {
      "activations/layer0_attention_weight_max": 14.280889511108398,
      "activations/layer0_attention_weight_min": -12.762992858886719,
      "activations/layer10_attention_weight_max": 64.17313385009766,
      "activations/layer10_attention_weight_min": -51.893402099609375,
      "activations/layer11_attention_weight_max": 44.0643424987793,
      "activations/layer11_attention_weight_min": -36.9188117980957,
      "activations/layer12_attention_weight_max": 29.352970123291016,
      "activations/layer12_attention_weight_min": -24.742374420166016,
      "activations/layer13_attention_weight_max": 56.48231506347656,
      "activations/layer13_attention_weight_min": -40.374183654785156,
      "activations/layer14_attention_weight_max": 45.513999938964844,
      "activations/layer14_attention_weight_min": -34.71705627441406,
      "activations/layer15_attention_weight_max": 56.14574432373047,
      "activations/layer15_attention_weight_min": -38.40153884887695,
      "activations/layer16_attention_weight_max": 52.09855270385742,
      "activations/layer16_attention_weight_min": -31.375825881958008,
      "activations/layer17_attention_weight_max": 57.77142333984375,
      "activations/layer17_attention_weight_min": -36.076148986816406,
      "activations/layer18_attention_weight_max": 64.3199691772461,
      "activations/layer18_attention_weight_min": -33.1486701965332,
      "activations/layer19_attention_weight_max": 20.48542594909668,
      "activations/layer19_attention_weight_min": -13.644865989685059,
      "activations/layer1_attention_weight_max": 15.710312843322754,
      "activations/layer1_attention_weight_min": -10.849331855773926,
      "activations/layer20_attention_weight_max": 19.71129035949707,
      "activations/layer20_attention_weight_min": -16.89402198791504,
      "activations/layer21_attention_weight_max": 40.534297943115234,
      "activations/layer21_attention_weight_min": -23.2630558013916,
      "activations/layer22_attention_weight_max": 37.954402923583984,
      "activations/layer22_attention_weight_min": -20.913768768310547,
      "activations/layer23_attention_weight_max": 27.063344955444336,
      "activations/layer23_attention_weight_min": -14.467658996582031,
      "activations/layer2_attention_weight_max": 24.058883666992188,
      "activations/layer2_attention_weight_min": -24.475582122802734,
      "activations/layer3_attention_weight_max": 58.806724548339844,
      "activations/layer3_attention_weight_min": -60.210662841796875,
      "activations/layer4_attention_weight_max": 89.46475982666016,
      "activations/layer4_attention_weight_min": -81.06824493408203,
      "activations/layer5_attention_weight_max": 129.23727416992188,
      "activations/layer5_attention_weight_min": -114.17312622070312,
      "activations/layer6_attention_weight_max": 62.20133590698242,
      "activations/layer6_attention_weight_min": -53.06687927246094,
      "activations/layer7_attention_weight_max": 60.10429382324219,
      "activations/layer7_attention_weight_min": -55.745819091796875,
      "activations/layer8_attention_weight_max": 51.434627532958984,
      "activations/layer8_attention_weight_min": -43.45231628417969,
      "activations/layer9_attention_weight_max": 66.76461029052734,
      "activations/layer9_attention_weight_min": -56.5357551574707,
      "epoch": 1.59,
      "learning_rate": 0.00014115151515151512,
      "loss": 3.0003,
      "step": 27400
    },
    {
      "activations/layer0_attention_weight_max": 14.43673324584961,
      "activations/layer0_attention_weight_min": -12.732077598571777,
      "activations/layer10_attention_weight_max": 72.56436920166016,
      "activations/layer10_attention_weight_min": -59.81480026245117,
      "activations/layer11_attention_weight_max": 46.13208770751953,
      "activations/layer11_attention_weight_min": -43.30842590332031,
      "activations/layer12_attention_weight_max": 30.84425163269043,
      "activations/layer12_attention_weight_min": -24.779197692871094,
      "activations/layer13_attention_weight_max": 60.463626861572266,
      "activations/layer13_attention_weight_min": -43.568241119384766,
      "activations/layer14_attention_weight_max": 56.12276077270508,
      "activations/layer14_attention_weight_min": -35.915443420410156,
      "activations/layer15_attention_weight_max": 63.13276672363281,
      "activations/layer15_attention_weight_min": -40.53572082519531,
      "activations/layer16_attention_weight_max": 48.910682678222656,
      "activations/layer16_attention_weight_min": -30.583431243896484,
      "activations/layer17_attention_weight_max": 58.28305435180664,
      "activations/layer17_attention_weight_min": -37.7784538269043,
      "activations/layer18_attention_weight_max": 64.95240020751953,
      "activations/layer18_attention_weight_min": -36.183292388916016,
      "activations/layer19_attention_weight_max": 22.236093521118164,
      "activations/layer19_attention_weight_min": -16.625680923461914,
      "activations/layer1_attention_weight_max": 15.072304725646973,
      "activations/layer1_attention_weight_min": -11.864519119262695,
      "activations/layer20_attention_weight_max": 21.24008560180664,
      "activations/layer20_attention_weight_min": -15.419777870178223,
      "activations/layer21_attention_weight_max": 42.71708297729492,
      "activations/layer21_attention_weight_min": -22.017650604248047,
      "activations/layer22_attention_weight_max": 41.09660339355469,
      "activations/layer22_attention_weight_min": -25.185285568237305,
      "activations/layer23_attention_weight_max": 25.045185089111328,
      "activations/layer23_attention_weight_min": -16.173625946044922,
      "activations/layer2_attention_weight_max": 25.102184295654297,
      "activations/layer2_attention_weight_min": -26.071636199951172,
      "activations/layer3_attention_weight_max": 57.42443084716797,
      "activations/layer3_attention_weight_min": -61.834434509277344,
      "activations/layer4_attention_weight_max": 89.49040985107422,
      "activations/layer4_attention_weight_min": -84.83773803710938,
      "activations/layer5_attention_weight_max": 132.40997314453125,
      "activations/layer5_attention_weight_min": -118.95506286621094,
      "activations/layer6_attention_weight_max": 64.3613052368164,
      "activations/layer6_attention_weight_min": -57.44010543823242,
      "activations/layer7_attention_weight_max": 61.436370849609375,
      "activations/layer7_attention_weight_min": -56.721561431884766,
      "activations/layer8_attention_weight_max": 48.75650405883789,
      "activations/layer8_attention_weight_min": -46.9797248840332,
      "activations/layer9_attention_weight_max": 75.796142578125,
      "activations/layer9_attention_weight_min": -65.94867706298828,
      "epoch": 1.59,
      "learning_rate": 0.00014113295454545454,
      "loss": 2.9899,
      "step": 27450
    },
    {
      "activations/layer0_attention_weight_max": 13.859817504882812,
      "activations/layer0_attention_weight_min": -13.213842391967773,
      "activations/layer10_attention_weight_max": 67.86190032958984,
      "activations/layer10_attention_weight_min": -53.679344177246094,
      "activations/layer11_attention_weight_max": 46.570919036865234,
      "activations/layer11_attention_weight_min": -37.985443115234375,
      "activations/layer12_attention_weight_max": 29.077939987182617,
      "activations/layer12_attention_weight_min": -24.40424156188965,
      "activations/layer13_attention_weight_max": 65.73045349121094,
      "activations/layer13_attention_weight_min": -49.32535934448242,
      "activations/layer14_attention_weight_max": 50.42143249511719,
      "activations/layer14_attention_weight_min": -40.532867431640625,
      "activations/layer15_attention_weight_max": 53.69586944580078,
      "activations/layer15_attention_weight_min": -43.10654830932617,
      "activations/layer16_attention_weight_max": 46.69743347167969,
      "activations/layer16_attention_weight_min": -30.037010192871094,
      "activations/layer17_attention_weight_max": 56.37629699707031,
      "activations/layer17_attention_weight_min": -37.41439437866211,
      "activations/layer18_attention_weight_max": 59.5626335144043,
      "activations/layer18_attention_weight_min": -34.652889251708984,
      "activations/layer19_attention_weight_max": 28.006071090698242,
      "activations/layer19_attention_weight_min": -15.18608570098877,
      "activations/layer1_attention_weight_max": 15.563581466674805,
      "activations/layer1_attention_weight_min": -13.657870292663574,
      "activations/layer20_attention_weight_max": 20.879150390625,
      "activations/layer20_attention_weight_min": -15.483234405517578,
      "activations/layer21_attention_weight_max": 40.521095275878906,
      "activations/layer21_attention_weight_min": -22.261993408203125,
      "activations/layer22_attention_weight_max": 38.581443786621094,
      "activations/layer22_attention_weight_min": -21.122352600097656,
      "activations/layer23_attention_weight_max": 25.921451568603516,
      "activations/layer23_attention_weight_min": -14.810111999511719,
      "activations/layer2_attention_weight_max": 25.767356872558594,
      "activations/layer2_attention_weight_min": -24.88872528076172,
      "activations/layer3_attention_weight_max": 59.2874755859375,
      "activations/layer3_attention_weight_min": -61.222835540771484,
      "activations/layer4_attention_weight_max": 90.28662109375,
      "activations/layer4_attention_weight_min": -79.06948852539062,
      "activations/layer5_attention_weight_max": 133.06231689453125,
      "activations/layer5_attention_weight_min": -112.2030029296875,
      "activations/layer6_attention_weight_max": 61.74348068237305,
      "activations/layer6_attention_weight_min": -55.739356994628906,
      "activations/layer7_attention_weight_max": 59.83555603027344,
      "activations/layer7_attention_weight_min": -55.225685119628906,
      "activations/layer8_attention_weight_max": 50.8002815246582,
      "activations/layer8_attention_weight_min": -46.02531814575195,
      "activations/layer9_attention_weight_max": 69.22551727294922,
      "activations/layer9_attention_weight_min": -64.18486022949219,
      "epoch": 1.6,
      "learning_rate": 0.00014111401515151512,
      "loss": 2.9933,
      "step": 27500
    },
    {
      "activations/layer0_attention_weight_max": 14.284000396728516,
      "activations/layer0_attention_weight_min": -13.101602554321289,
      "activations/layer10_attention_weight_max": 65.05549621582031,
      "activations/layer10_attention_weight_min": -59.23741912841797,
      "activations/layer11_attention_weight_max": 46.58641815185547,
      "activations/layer11_attention_weight_min": -41.13444137573242,
      "activations/layer12_attention_weight_max": 30.910261154174805,
      "activations/layer12_attention_weight_min": -26.959501266479492,
      "activations/layer13_attention_weight_max": 56.73668670654297,
      "activations/layer13_attention_weight_min": -42.34388732910156,
      "activations/layer14_attention_weight_max": 55.79914855957031,
      "activations/layer14_attention_weight_min": -38.739784240722656,
      "activations/layer15_attention_weight_max": 52.27263259887695,
      "activations/layer15_attention_weight_min": -39.5565071105957,
      "activations/layer16_attention_weight_max": 47.46703338623047,
      "activations/layer16_attention_weight_min": -30.17601776123047,
      "activations/layer17_attention_weight_max": 63.18033981323242,
      "activations/layer17_attention_weight_min": -40.92428207397461,
      "activations/layer18_attention_weight_max": 61.35324478149414,
      "activations/layer18_attention_weight_min": -35.47002410888672,
      "activations/layer19_attention_weight_max": 20.768848419189453,
      "activations/layer19_attention_weight_min": -14.741960525512695,
      "activations/layer1_attention_weight_max": 15.649884223937988,
      "activations/layer1_attention_weight_min": -14.601171493530273,
      "activations/layer20_attention_weight_max": 19.593896865844727,
      "activations/layer20_attention_weight_min": -15.779448509216309,
      "activations/layer21_attention_weight_max": 38.11139678955078,
      "activations/layer21_attention_weight_min": -20.839984893798828,
      "activations/layer22_attention_weight_max": 38.48490905761719,
      "activations/layer22_attention_weight_min": -20.777090072631836,
      "activations/layer23_attention_weight_max": 24.88772201538086,
      "activations/layer23_attention_weight_min": -15.993570327758789,
      "activations/layer2_attention_weight_max": 26.332265853881836,
      "activations/layer2_attention_weight_min": -25.100500106811523,
      "activations/layer3_attention_weight_max": 60.50302505493164,
      "activations/layer3_attention_weight_min": -64.08157348632812,
      "activations/layer4_attention_weight_max": 89.1270523071289,
      "activations/layer4_attention_weight_min": -85.56988525390625,
      "activations/layer5_attention_weight_max": 130.5868377685547,
      "activations/layer5_attention_weight_min": -117.82688903808594,
      "activations/layer6_attention_weight_max": 62.4654655456543,
      "activations/layer6_attention_weight_min": -56.245758056640625,
      "activations/layer7_attention_weight_max": 58.15217208862305,
      "activations/layer7_attention_weight_min": -53.84602737426758,
      "activations/layer8_attention_weight_max": 48.4881706237793,
      "activations/layer8_attention_weight_min": -42.02622985839844,
      "activations/layer9_attention_weight_max": 73.96983337402344,
      "activations/layer9_attention_weight_min": -65.33946990966797,
      "epoch": 1.6,
      "learning_rate": 0.00014109507575757574,
      "loss": 2.9983,
      "step": 27550
    },
    {
      "activations/layer0_attention_weight_max": 14.217668533325195,
      "activations/layer0_attention_weight_min": -13.97253704071045,
      "activations/layer10_attention_weight_max": 65.10294342041016,
      "activations/layer10_attention_weight_min": -58.146297454833984,
      "activations/layer11_attention_weight_max": 46.86130905151367,
      "activations/layer11_attention_weight_min": -40.00176239013672,
      "activations/layer12_attention_weight_max": 29.277156829833984,
      "activations/layer12_attention_weight_min": -24.591796875,
      "activations/layer13_attention_weight_max": 62.263248443603516,
      "activations/layer13_attention_weight_min": -53.34306335449219,
      "activations/layer14_attention_weight_max": 57.49233627319336,
      "activations/layer14_attention_weight_min": -49.915771484375,
      "activations/layer15_attention_weight_max": 54.16539764404297,
      "activations/layer15_attention_weight_min": -45.21004104614258,
      "activations/layer16_attention_weight_max": 46.37526321411133,
      "activations/layer16_attention_weight_min": -32.35738754272461,
      "activations/layer17_attention_weight_max": 61.23801040649414,
      "activations/layer17_attention_weight_min": -40.67348861694336,
      "activations/layer18_attention_weight_max": 57.94164276123047,
      "activations/layer18_attention_weight_min": -37.454078674316406,
      "activations/layer19_attention_weight_max": 20.744338989257812,
      "activations/layer19_attention_weight_min": -14.454360008239746,
      "activations/layer1_attention_weight_max": 14.869977951049805,
      "activations/layer1_attention_weight_min": -12.57091236114502,
      "activations/layer20_attention_weight_max": 19.00477409362793,
      "activations/layer20_attention_weight_min": -14.975545883178711,
      "activations/layer21_attention_weight_max": 34.85321044921875,
      "activations/layer21_attention_weight_min": -22.919057846069336,
      "activations/layer22_attention_weight_max": 38.166175842285156,
      "activations/layer22_attention_weight_min": -23.23489761352539,
      "activations/layer23_attention_weight_max": 23.917518615722656,
      "activations/layer23_attention_weight_min": -14.850237846374512,
      "activations/layer2_attention_weight_max": 27.781614303588867,
      "activations/layer2_attention_weight_min": -26.764694213867188,
      "activations/layer3_attention_weight_max": 59.818382263183594,
      "activations/layer3_attention_weight_min": -62.00965881347656,
      "activations/layer4_attention_weight_max": 90.63538360595703,
      "activations/layer4_attention_weight_min": -81.11991119384766,
      "activations/layer5_attention_weight_max": 132.03492736816406,
      "activations/layer5_attention_weight_min": -119.47718811035156,
      "activations/layer6_attention_weight_max": 60.79909133911133,
      "activations/layer6_attention_weight_min": -53.489192962646484,
      "activations/layer7_attention_weight_max": 58.5165901184082,
      "activations/layer7_attention_weight_min": -51.36965560913086,
      "activations/layer8_attention_weight_max": 44.447242736816406,
      "activations/layer8_attention_weight_min": -43.549495697021484,
      "activations/layer9_attention_weight_max": 63.7372932434082,
      "activations/layer9_attention_weight_min": -61.0330810546875,
      "epoch": 1.6,
      "learning_rate": 0.00014107613636363635,
      "loss": 2.9857,
      "step": 27600
    },
    {
      "activations/layer0_attention_weight_max": 13.344696044921875,
      "activations/layer0_attention_weight_min": -13.52351188659668,
      "activations/layer10_attention_weight_max": 63.91048049926758,
      "activations/layer10_attention_weight_min": -49.99565124511719,
      "activations/layer11_attention_weight_max": 48.441017150878906,
      "activations/layer11_attention_weight_min": -38.89912033081055,
      "activations/layer12_attention_weight_max": 26.781551361083984,
      "activations/layer12_attention_weight_min": -28.223520278930664,
      "activations/layer13_attention_weight_max": 61.80998992919922,
      "activations/layer13_attention_weight_min": -44.41725158691406,
      "activations/layer14_attention_weight_max": 48.24664306640625,
      "activations/layer14_attention_weight_min": -36.30809783935547,
      "activations/layer15_attention_weight_max": 57.254581451416016,
      "activations/layer15_attention_weight_min": -42.23370361328125,
      "activations/layer16_attention_weight_max": 46.551185607910156,
      "activations/layer16_attention_weight_min": -32.31902313232422,
      "activations/layer17_attention_weight_max": 61.58816146850586,
      "activations/layer17_attention_weight_min": -40.99467849731445,
      "activations/layer18_attention_weight_max": 67.70926666259766,
      "activations/layer18_attention_weight_min": -35.67607498168945,
      "activations/layer19_attention_weight_max": 22.044511795043945,
      "activations/layer19_attention_weight_min": -15.424232482910156,
      "activations/layer1_attention_weight_max": 15.702187538146973,
      "activations/layer1_attention_weight_min": -12.159284591674805,
      "activations/layer20_attention_weight_max": 21.888395309448242,
      "activations/layer20_attention_weight_min": -15.436331748962402,
      "activations/layer21_attention_weight_max": 42.09221267700195,
      "activations/layer21_attention_weight_min": -27.92232322692871,
      "activations/layer22_attention_weight_max": 40.77322769165039,
      "activations/layer22_attention_weight_min": -24.249969482421875,
      "activations/layer23_attention_weight_max": 24.068815231323242,
      "activations/layer23_attention_weight_min": -21.661596298217773,
      "activations/layer2_attention_weight_max": 25.65582847595215,
      "activations/layer2_attention_weight_min": -26.26495933532715,
      "activations/layer3_attention_weight_max": 56.50139617919922,
      "activations/layer3_attention_weight_min": -62.33938217163086,
      "activations/layer4_attention_weight_max": 86.3170394897461,
      "activations/layer4_attention_weight_min": -76.77207946777344,
      "activations/layer5_attention_weight_max": 127.02696990966797,
      "activations/layer5_attention_weight_min": -116.03374481201172,
      "activations/layer6_attention_weight_max": 61.72098159790039,
      "activations/layer6_attention_weight_min": -55.12517547607422,
      "activations/layer7_attention_weight_max": 59.546958923339844,
      "activations/layer7_attention_weight_min": -53.194156646728516,
      "activations/layer8_attention_weight_max": 50.67346954345703,
      "activations/layer8_attention_weight_min": -42.52666091918945,
      "activations/layer9_attention_weight_max": 75.4134521484375,
      "activations/layer9_attention_weight_min": -64.07698822021484,
      "epoch": 1.61,
      "learning_rate": 0.00014105719696969694,
      "loss": 3.0014,
      "step": 27650
    },
    {
      "activations/layer0_attention_weight_max": 14.64724349975586,
      "activations/layer0_attention_weight_min": -13.679414749145508,
      "activations/layer10_attention_weight_max": 64.64103698730469,
      "activations/layer10_attention_weight_min": -53.19069290161133,
      "activations/layer11_attention_weight_max": 41.65507125854492,
      "activations/layer11_attention_weight_min": -36.79372024536133,
      "activations/layer12_attention_weight_max": 27.45585060119629,
      "activations/layer12_attention_weight_min": -23.02625846862793,
      "activations/layer13_attention_weight_max": 47.84843444824219,
      "activations/layer13_attention_weight_min": -43.89512634277344,
      "activations/layer14_attention_weight_max": 48.361228942871094,
      "activations/layer14_attention_weight_min": -38.15226745605469,
      "activations/layer15_attention_weight_max": 48.699954986572266,
      "activations/layer15_attention_weight_min": -39.03955078125,
      "activations/layer16_attention_weight_max": 47.61854934692383,
      "activations/layer16_attention_weight_min": -31.742019653320312,
      "activations/layer17_attention_weight_max": 54.52145004272461,
      "activations/layer17_attention_weight_min": -36.08985137939453,
      "activations/layer18_attention_weight_max": 52.527530670166016,
      "activations/layer18_attention_weight_min": -35.75889587402344,
      "activations/layer19_attention_weight_max": 21.26154899597168,
      "activations/layer19_attention_weight_min": -14.920048713684082,
      "activations/layer1_attention_weight_max": 15.479268074035645,
      "activations/layer1_attention_weight_min": -12.547125816345215,
      "activations/layer20_attention_weight_max": 20.0909481048584,
      "activations/layer20_attention_weight_min": -15.361061096191406,
      "activations/layer21_attention_weight_max": 42.13804244995117,
      "activations/layer21_attention_weight_min": -22.16615867614746,
      "activations/layer22_attention_weight_max": 35.656333923339844,
      "activations/layer22_attention_weight_min": -21.808368682861328,
      "activations/layer23_attention_weight_max": 23.41509246826172,
      "activations/layer23_attention_weight_min": -15.287565231323242,
      "activations/layer2_attention_weight_max": 26.253318786621094,
      "activations/layer2_attention_weight_min": -24.69307518005371,
      "activations/layer3_attention_weight_max": 56.650447845458984,
      "activations/layer3_attention_weight_min": -59.23252487182617,
      "activations/layer4_attention_weight_max": 87.16449737548828,
      "activations/layer4_attention_weight_min": -86.44065856933594,
      "activations/layer5_attention_weight_max": 127.80305480957031,
      "activations/layer5_attention_weight_min": -124.65225219726562,
      "activations/layer6_attention_weight_max": 60.1866340637207,
      "activations/layer6_attention_weight_min": -57.076904296875,
      "activations/layer7_attention_weight_max": 60.559913635253906,
      "activations/layer7_attention_weight_min": -54.48419189453125,
      "activations/layer8_attention_weight_max": 47.70634841918945,
      "activations/layer8_attention_weight_min": -48.189945220947266,
      "activations/layer9_attention_weight_max": 59.92626953125,
      "activations/layer9_attention_weight_min": -61.658668518066406,
      "epoch": 1.61,
      "learning_rate": 0.00014103825757575756,
      "loss": 3.0055,
      "step": 27700
    },
    {
      "activations/layer0_attention_weight_max": 14.731663703918457,
      "activations/layer0_attention_weight_min": -13.905434608459473,
      "activations/layer10_attention_weight_max": 64.13872528076172,
      "activations/layer10_attention_weight_min": -52.90933609008789,
      "activations/layer11_attention_weight_max": 46.195838928222656,
      "activations/layer11_attention_weight_min": -39.81016540527344,
      "activations/layer12_attention_weight_max": 32.87145233154297,
      "activations/layer12_attention_weight_min": -28.066057205200195,
      "activations/layer13_attention_weight_max": 67.44441223144531,
      "activations/layer13_attention_weight_min": -52.17243576049805,
      "activations/layer14_attention_weight_max": 58.1422233581543,
      "activations/layer14_attention_weight_min": -43.79985427856445,
      "activations/layer15_attention_weight_max": 58.44997024536133,
      "activations/layer15_attention_weight_min": -42.857215881347656,
      "activations/layer16_attention_weight_max": 43.01145553588867,
      "activations/layer16_attention_weight_min": -30.791711807250977,
      "activations/layer17_attention_weight_max": 55.91448211669922,
      "activations/layer17_attention_weight_min": -36.28687286376953,
      "activations/layer18_attention_weight_max": 62.9796028137207,
      "activations/layer18_attention_weight_min": -34.972999572753906,
      "activations/layer19_attention_weight_max": 23.87421417236328,
      "activations/layer19_attention_weight_min": -15.511454582214355,
      "activations/layer1_attention_weight_max": 15.434242248535156,
      "activations/layer1_attention_weight_min": -12.238485336303711,
      "activations/layer20_attention_weight_max": 22.00724220275879,
      "activations/layer20_attention_weight_min": -18.766956329345703,
      "activations/layer21_attention_weight_max": 42.50486373901367,
      "activations/layer21_attention_weight_min": -24.491727828979492,
      "activations/layer22_attention_weight_max": 38.93087387084961,
      "activations/layer22_attention_weight_min": -20.753761291503906,
      "activations/layer23_attention_weight_max": 26.786291122436523,
      "activations/layer23_attention_weight_min": -14.120203971862793,
      "activations/layer2_attention_weight_max": 24.854978561401367,
      "activations/layer2_attention_weight_min": -24.300586700439453,
      "activations/layer3_attention_weight_max": 59.573265075683594,
      "activations/layer3_attention_weight_min": -60.03460693359375,
      "activations/layer4_attention_weight_max": 89.37740325927734,
      "activations/layer4_attention_weight_min": -81.24787902832031,
      "activations/layer5_attention_weight_max": 130.59207153320312,
      "activations/layer5_attention_weight_min": -122.15599822998047,
      "activations/layer6_attention_weight_max": 61.30760192871094,
      "activations/layer6_attention_weight_min": -54.53470230102539,
      "activations/layer7_attention_weight_max": 61.5825080871582,
      "activations/layer7_attention_weight_min": -52.81126022338867,
      "activations/layer8_attention_weight_max": 48.604061126708984,
      "activations/layer8_attention_weight_min": -44.14043426513672,
      "activations/layer9_attention_weight_max": 63.99369430541992,
      "activations/layer9_attention_weight_min": -60.74781036376953,
      "epoch": 1.61,
      "learning_rate": 0.00014101931818181817,
      "loss": 3.0172,
      "step": 27750
    },
    {
      "activations/layer0_attention_weight_max": 13.549110412597656,
      "activations/layer0_attention_weight_min": -12.950071334838867,
      "activations/layer10_attention_weight_max": 64.70172882080078,
      "activations/layer10_attention_weight_min": -50.34403610229492,
      "activations/layer11_attention_weight_max": 43.6353759765625,
      "activations/layer11_attention_weight_min": -36.934120178222656,
      "activations/layer12_attention_weight_max": 31.404949188232422,
      "activations/layer12_attention_weight_min": -27.38797378540039,
      "activations/layer13_attention_weight_max": 76.17634582519531,
      "activations/layer13_attention_weight_min": -52.0194206237793,
      "activations/layer14_attention_weight_max": 72.8887710571289,
      "activations/layer14_attention_weight_min": -41.54151153564453,
      "activations/layer15_attention_weight_max": 63.72437286376953,
      "activations/layer15_attention_weight_min": -40.28082275390625,
      "activations/layer16_attention_weight_max": 47.055965423583984,
      "activations/layer16_attention_weight_min": -28.951183319091797,
      "activations/layer17_attention_weight_max": 63.65536117553711,
      "activations/layer17_attention_weight_min": -34.961151123046875,
      "activations/layer18_attention_weight_max": 73.79046630859375,
      "activations/layer18_attention_weight_min": -35.23133087158203,
      "activations/layer19_attention_weight_max": 24.86437225341797,
      "activations/layer19_attention_weight_min": -14.234687805175781,
      "activations/layer1_attention_weight_max": 15.545488357543945,
      "activations/layer1_attention_weight_min": -12.491483688354492,
      "activations/layer20_attention_weight_max": 21.681209564208984,
      "activations/layer20_attention_weight_min": -14.126437187194824,
      "activations/layer21_attention_weight_max": 47.05667495727539,
      "activations/layer21_attention_weight_min": -19.943862915039062,
      "activations/layer22_attention_weight_max": 37.403343200683594,
      "activations/layer22_attention_weight_min": -18.580053329467773,
      "activations/layer23_attention_weight_max": 26.707304000854492,
      "activations/layer23_attention_weight_min": -14.194101333618164,
      "activations/layer2_attention_weight_max": 28.779050827026367,
      "activations/layer2_attention_weight_min": -26.420141220092773,
      "activations/layer3_attention_weight_max": 57.23805236816406,
      "activations/layer3_attention_weight_min": -60.54713439941406,
      "activations/layer4_attention_weight_max": 88.82601928710938,
      "activations/layer4_attention_weight_min": -81.44457244873047,
      "activations/layer5_attention_weight_max": 125.90636444091797,
      "activations/layer5_attention_weight_min": -114.76068115234375,
      "activations/layer6_attention_weight_max": 62.30209732055664,
      "activations/layer6_attention_weight_min": -52.71527862548828,
      "activations/layer7_attention_weight_max": 63.32223129272461,
      "activations/layer7_attention_weight_min": -57.39970397949219,
      "activations/layer8_attention_weight_max": 46.646060943603516,
      "activations/layer8_attention_weight_min": -40.7653694152832,
      "activations/layer9_attention_weight_max": 67.53684997558594,
      "activations/layer9_attention_weight_min": -66.69783782958984,
      "epoch": 1.62,
      "learning_rate": 0.0001410003787878788,
      "loss": 2.9909,
      "step": 27800
    },
    {
      "activations/layer0_attention_weight_max": 14.527290344238281,
      "activations/layer0_attention_weight_min": -12.816272735595703,
      "activations/layer10_attention_weight_max": 61.55839538574219,
      "activations/layer10_attention_weight_min": -47.7188835144043,
      "activations/layer11_attention_weight_max": 43.725704193115234,
      "activations/layer11_attention_weight_min": -36.41761779785156,
      "activations/layer12_attention_weight_max": 30.575599670410156,
      "activations/layer12_attention_weight_min": -26.35832405090332,
      "activations/layer13_attention_weight_max": 68.01499938964844,
      "activations/layer13_attention_weight_min": -45.88298034667969,
      "activations/layer14_attention_weight_max": 68.29193878173828,
      "activations/layer14_attention_weight_min": -48.44560623168945,
      "activations/layer15_attention_weight_max": 53.498207092285156,
      "activations/layer15_attention_weight_min": -40.35164260864258,
      "activations/layer16_attention_weight_max": 42.68029022216797,
      "activations/layer16_attention_weight_min": -32.789093017578125,
      "activations/layer17_attention_weight_max": 58.9260139465332,
      "activations/layer17_attention_weight_min": -41.01898193359375,
      "activations/layer18_attention_weight_max": 61.832244873046875,
      "activations/layer18_attention_weight_min": -39.36335372924805,
      "activations/layer19_attention_weight_max": 19.095232009887695,
      "activations/layer19_attention_weight_min": -15.612906455993652,
      "activations/layer1_attention_weight_max": 16.111948013305664,
      "activations/layer1_attention_weight_min": -13.01015567779541,
      "activations/layer20_attention_weight_max": 18.79296112060547,
      "activations/layer20_attention_weight_min": -16.786497116088867,
      "activations/layer21_attention_weight_max": 36.125606536865234,
      "activations/layer21_attention_weight_min": -23.55785369873047,
      "activations/layer22_attention_weight_max": 34.10916519165039,
      "activations/layer22_attention_weight_min": -20.514753341674805,
      "activations/layer23_attention_weight_max": 23.676563262939453,
      "activations/layer23_attention_weight_min": -16.40863609313965,
      "activations/layer2_attention_weight_max": 26.604503631591797,
      "activations/layer2_attention_weight_min": -26.52496337890625,
      "activations/layer3_attention_weight_max": 63.240745544433594,
      "activations/layer3_attention_weight_min": -63.93547439575195,
      "activations/layer4_attention_weight_max": 95.22942352294922,
      "activations/layer4_attention_weight_min": -81.05553436279297,
      "activations/layer5_attention_weight_max": 139.2700653076172,
      "activations/layer5_attention_weight_min": -117.47654724121094,
      "activations/layer6_attention_weight_max": 63.89927673339844,
      "activations/layer6_attention_weight_min": -56.86127853393555,
      "activations/layer7_attention_weight_max": 59.69500732421875,
      "activations/layer7_attention_weight_min": -55.51807403564453,
      "activations/layer8_attention_weight_max": 46.08543014526367,
      "activations/layer8_attention_weight_min": -42.96059036254883,
      "activations/layer9_attention_weight_max": 69.9775390625,
      "activations/layer9_attention_weight_min": -64.58612060546875,
      "epoch": 1.62,
      "learning_rate": 0.00014098143939393938,
      "loss": 3.0015,
      "step": 27850
    },
    {
      "activations/layer0_attention_weight_max": 13.865239143371582,
      "activations/layer0_attention_weight_min": -13.45388126373291,
      "activations/layer10_attention_weight_max": 71.67618560791016,
      "activations/layer10_attention_weight_min": -59.44923782348633,
      "activations/layer11_attention_weight_max": 50.6285514831543,
      "activations/layer11_attention_weight_min": -43.64817428588867,
      "activations/layer12_attention_weight_max": 30.578475952148438,
      "activations/layer12_attention_weight_min": -24.194303512573242,
      "activations/layer13_attention_weight_max": 70.7915267944336,
      "activations/layer13_attention_weight_min": -42.43718719482422,
      "activations/layer14_attention_weight_max": 80.0478744506836,
      "activations/layer14_attention_weight_min": -43.82600784301758,
      "activations/layer15_attention_weight_max": 60.54765319824219,
      "activations/layer15_attention_weight_min": -36.858150482177734,
      "activations/layer16_attention_weight_max": 52.65714645385742,
      "activations/layer16_attention_weight_min": -28.190412521362305,
      "activations/layer17_attention_weight_max": 77.89053344726562,
      "activations/layer17_attention_weight_min": -34.8161735534668,
      "activations/layer18_attention_weight_max": 78.52969360351562,
      "activations/layer18_attention_weight_min": -35.50681686401367,
      "activations/layer19_attention_weight_max": 25.987751007080078,
      "activations/layer19_attention_weight_min": -15.983359336853027,
      "activations/layer1_attention_weight_max": 16.283464431762695,
      "activations/layer1_attention_weight_min": -13.674185752868652,
      "activations/layer20_attention_weight_max": 28.623016357421875,
      "activations/layer20_attention_weight_min": -14.926031112670898,
      "activations/layer21_attention_weight_max": 57.49783706665039,
      "activations/layer21_attention_weight_min": -20.898900985717773,
      "activations/layer22_attention_weight_max": 47.68130874633789,
      "activations/layer22_attention_weight_min": -19.891277313232422,
      "activations/layer23_attention_weight_max": 29.028392791748047,
      "activations/layer23_attention_weight_min": -13.374415397644043,
      "activations/layer2_attention_weight_max": 27.18310546875,
      "activations/layer2_attention_weight_min": -27.176334381103516,
      "activations/layer3_attention_weight_max": 59.93843078613281,
      "activations/layer3_attention_weight_min": -66.04243469238281,
      "activations/layer4_attention_weight_max": 89.72318267822266,
      "activations/layer4_attention_weight_min": -81.64021301269531,
      "activations/layer5_attention_weight_max": 135.93170166015625,
      "activations/layer5_attention_weight_min": -112.67632293701172,
      "activations/layer6_attention_weight_max": 61.483482360839844,
      "activations/layer6_attention_weight_min": -54.79702377319336,
      "activations/layer7_attention_weight_max": 70.78099060058594,
      "activations/layer7_attention_weight_min": -56.58961868286133,
      "activations/layer8_attention_weight_max": 55.653724670410156,
      "activations/layer8_attention_weight_min": -47.366188049316406,
      "activations/layer9_attention_weight_max": 77.03114318847656,
      "activations/layer9_attention_weight_min": -68.30899047851562,
      "epoch": 1.62,
      "learning_rate": 0.0001409625,
      "loss": 3.0058,
      "step": 27900
    },
    {
      "activations/layer0_attention_weight_max": 15.627143859863281,
      "activations/layer0_attention_weight_min": -13.237462043762207,
      "activations/layer10_attention_weight_max": 67.59571838378906,
      "activations/layer10_attention_weight_min": -56.718563079833984,
      "activations/layer11_attention_weight_max": 45.27098083496094,
      "activations/layer11_attention_weight_min": -32.918006896972656,
      "activations/layer12_attention_weight_max": 26.90142059326172,
      "activations/layer12_attention_weight_min": -23.798871994018555,
      "activations/layer13_attention_weight_max": 72.1385269165039,
      "activations/layer13_attention_weight_min": -42.30962371826172,
      "activations/layer14_attention_weight_max": 59.249881744384766,
      "activations/layer14_attention_weight_min": -43.09474182128906,
      "activations/layer15_attention_weight_max": 66.62494659423828,
      "activations/layer15_attention_weight_min": -41.41632080078125,
      "activations/layer16_attention_weight_max": 49.99324417114258,
      "activations/layer16_attention_weight_min": -29.360090255737305,
      "activations/layer17_attention_weight_max": 74.99906158447266,
      "activations/layer17_attention_weight_min": -35.543418884277344,
      "activations/layer18_attention_weight_max": 64.52613830566406,
      "activations/layer18_attention_weight_min": -34.794490814208984,
      "activations/layer19_attention_weight_max": 27.998823165893555,
      "activations/layer19_attention_weight_min": -14.13054084777832,
      "activations/layer1_attention_weight_max": 14.930444717407227,
      "activations/layer1_attention_weight_min": -12.3912353515625,
      "activations/layer20_attention_weight_max": 21.35186004638672,
      "activations/layer20_attention_weight_min": -16.84351921081543,
      "activations/layer21_attention_weight_max": 43.29916763305664,
      "activations/layer21_attention_weight_min": -21.76709747314453,
      "activations/layer22_attention_weight_max": 43.45499801635742,
      "activations/layer22_attention_weight_min": -22.51836395263672,
      "activations/layer23_attention_weight_max": 27.354835510253906,
      "activations/layer23_attention_weight_min": -15.651830673217773,
      "activations/layer2_attention_weight_max": 25.399539947509766,
      "activations/layer2_attention_weight_min": -23.87898063659668,
      "activations/layer3_attention_weight_max": 59.81208038330078,
      "activations/layer3_attention_weight_min": -65.1275405883789,
      "activations/layer4_attention_weight_max": 89.12508392333984,
      "activations/layer4_attention_weight_min": -81.1955795288086,
      "activations/layer5_attention_weight_max": 132.77012634277344,
      "activations/layer5_attention_weight_min": -114.32207489013672,
      "activations/layer6_attention_weight_max": 65.87340545654297,
      "activations/layer6_attention_weight_min": -56.66973114013672,
      "activations/layer7_attention_weight_max": 63.37924575805664,
      "activations/layer7_attention_weight_min": -52.03800964355469,
      "activations/layer8_attention_weight_max": 47.01274871826172,
      "activations/layer8_attention_weight_min": -40.835758209228516,
      "activations/layer9_attention_weight_max": 70.17549133300781,
      "activations/layer9_attention_weight_min": -61.5625,
      "epoch": 1.62,
      "learning_rate": 0.0001409435606060606,
      "loss": 2.9961,
      "step": 27950
    },
    {
      "activations/layer0_attention_weight_max": 13.915572166442871,
      "activations/layer0_attention_weight_min": -13.115283012390137,
      "activations/layer10_attention_weight_max": 57.16502380371094,
      "activations/layer10_attention_weight_min": -53.720516204833984,
      "activations/layer11_attention_weight_max": 49.043052673339844,
      "activations/layer11_attention_weight_min": -36.4737548828125,
      "activations/layer12_attention_weight_max": 24.757383346557617,
      "activations/layer12_attention_weight_min": -25.17847442626953,
      "activations/layer13_attention_weight_max": 53.91307830810547,
      "activations/layer13_attention_weight_min": -38.9298095703125,
      "activations/layer14_attention_weight_max": 47.1285514831543,
      "activations/layer14_attention_weight_min": -37.18126678466797,
      "activations/layer15_attention_weight_max": 47.20614242553711,
      "activations/layer15_attention_weight_min": -37.43935012817383,
      "activations/layer16_attention_weight_max": 47.13760757446289,
      "activations/layer16_attention_weight_min": -29.963638305664062,
      "activations/layer17_attention_weight_max": 53.89748001098633,
      "activations/layer17_attention_weight_min": -37.576393127441406,
      "activations/layer18_attention_weight_max": 54.22570037841797,
      "activations/layer18_attention_weight_min": -34.08619689941406,
      "activations/layer19_attention_weight_max": 19.856626510620117,
      "activations/layer19_attention_weight_min": -15.131352424621582,
      "activations/layer1_attention_weight_max": 15.966376304626465,
      "activations/layer1_attention_weight_min": -11.560853004455566,
      "activations/layer20_attention_weight_max": 17.46739959716797,
      "activations/layer20_attention_weight_min": -16.378238677978516,
      "activations/layer21_attention_weight_max": 30.00594139099121,
      "activations/layer21_attention_weight_min": -20.5947322845459,
      "activations/layer22_attention_weight_max": 29.620319366455078,
      "activations/layer22_attention_weight_min": -23.046480178833008,
      "activations/layer23_attention_weight_max": 20.06524658203125,
      "activations/layer23_attention_weight_min": -17.471858978271484,
      "activations/layer2_attention_weight_max": 26.356847763061523,
      "activations/layer2_attention_weight_min": -24.571819305419922,
      "activations/layer3_attention_weight_max": 56.55690383911133,
      "activations/layer3_attention_weight_min": -57.662574768066406,
      "activations/layer4_attention_weight_max": 81.7847671508789,
      "activations/layer4_attention_weight_min": -75.07818603515625,
      "activations/layer5_attention_weight_max": 117.3492431640625,
      "activations/layer5_attention_weight_min": -109.74819946289062,
      "activations/layer6_attention_weight_max": 60.70403289794922,
      "activations/layer6_attention_weight_min": -49.7730598449707,
      "activations/layer7_attention_weight_max": 56.09618377685547,
      "activations/layer7_attention_weight_min": -51.88428497314453,
      "activations/layer8_attention_weight_max": 44.564910888671875,
      "activations/layer8_attention_weight_min": -44.02937698364258,
      "activations/layer9_attention_weight_max": 70.739013671875,
      "activations/layer9_attention_weight_min": -61.3319091796875,
      "epoch": 1.63,
      "learning_rate": 0.0001409246212121212,
      "loss": 2.992,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_loss": 2.9453125,
      "eval_runtime": 8.4827,
      "eval_samples_per_second": 506.206,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_openwebtext_loss": 2.9453125,
      "eval_openwebtext_ppl": 19.016604146815915,
      "eval_openwebtext_runtime": 8.4827,
      "eval_openwebtext_samples_per_second": 506.206,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_wikitext_loss": 3.21875,
      "eval_wikitext_ppl": 24.9968545761854,
      "eval_wikitext_runtime": 1.9681,
      "eval_wikitext_samples_per_second": 231.692,
      "step": 28000
    },
    {
      "epoch": 1.63,
      "eval_lambada_loss": 3.263671875,
      "eval_lambada_ppl": 26.14536361045317,
      "eval_lambada_runtime": 9.5598,
      "eval_lambada_samples_per_second": 509.319,
      "step": 28000
    },
    {
      "activations/layer0_attention_weight_max": 13.61698055267334,
      "activations/layer0_attention_weight_min": -13.305771827697754,
      "activations/layer10_attention_weight_max": 65.74177551269531,
      "activations/layer10_attention_weight_min": -53.92500686645508,
      "activations/layer11_attention_weight_max": 44.71332931518555,
      "activations/layer11_attention_weight_min": -36.368377685546875,
      "activations/layer12_attention_weight_max": 27.471040725708008,
      "activations/layer12_attention_weight_min": -22.052635192871094,
      "activations/layer13_attention_weight_max": 61.57230758666992,
      "activations/layer13_attention_weight_min": -41.23993682861328,
      "activations/layer14_attention_weight_max": 55.21835708618164,
      "activations/layer14_attention_weight_min": -39.180728912353516,
      "activations/layer15_attention_weight_max": 55.4664192199707,
      "activations/layer15_attention_weight_min": -37.87015151977539,
      "activations/layer16_attention_weight_max": 52.69862365722656,
      "activations/layer16_attention_weight_min": -29.69947624206543,
      "activations/layer17_attention_weight_max": 58.45601272583008,
      "activations/layer17_attention_weight_min": -36.058746337890625,
      "activations/layer18_attention_weight_max": 61.258121490478516,
      "activations/layer18_attention_weight_min": -32.24256896972656,
      "activations/layer19_attention_weight_max": 24.185956954956055,
      "activations/layer19_attention_weight_min": -14.534628868103027,
      "activations/layer1_attention_weight_max": 16.08727264404297,
      "activations/layer1_attention_weight_min": -11.873180389404297,
      "activations/layer20_attention_weight_max": 19.313587188720703,
      "activations/layer20_attention_weight_min": -15.304351806640625,
      "activations/layer21_attention_weight_max": 38.61450958251953,
      "activations/layer21_attention_weight_min": -19.62796974182129,
      "activations/layer22_attention_weight_max": 38.78814697265625,
      "activations/layer22_attention_weight_min": -20.414567947387695,
      "activations/layer23_attention_weight_max": 22.376691818237305,
      "activations/layer23_attention_weight_min": -14.950095176696777,
      "activations/layer2_attention_weight_max": 26.56655502319336,
      "activations/layer2_attention_weight_min": -24.978065490722656,
      "activations/layer3_attention_weight_max": 60.21141052246094,
      "activations/layer3_attention_weight_min": -62.802574157714844,
      "activations/layer4_attention_weight_max": 85.22988891601562,
      "activations/layer4_attention_weight_min": -78.38219451904297,
      "activations/layer5_attention_weight_max": 122.97750091552734,
      "activations/layer5_attention_weight_min": -119.30049896240234,
      "activations/layer6_attention_weight_max": 59.0506477355957,
      "activations/layer6_attention_weight_min": -58.59547424316406,
      "activations/layer7_attention_weight_max": 57.55154037475586,
      "activations/layer7_attention_weight_min": -50.77833557128906,
      "activations/layer8_attention_weight_max": 49.5583381652832,
      "activations/layer8_attention_weight_min": -45.59265899658203,
      "activations/layer9_attention_weight_max": 73.23246765136719,
      "activations/layer9_attention_weight_min": -60.2542724609375,
      "epoch": 1.63,
      "learning_rate": 0.0001409056818181818,
      "loss": 2.9929,
      "step": 28050
    },
    {
      "activations/layer0_attention_weight_max": 15.469861030578613,
      "activations/layer0_attention_weight_min": -13.384106636047363,
      "activations/layer10_attention_weight_max": 69.38699340820312,
      "activations/layer10_attention_weight_min": -52.80282974243164,
      "activations/layer11_attention_weight_max": 47.89532470703125,
      "activations/layer11_attention_weight_min": -35.79840087890625,
      "activations/layer12_attention_weight_max": 31.116634368896484,
      "activations/layer12_attention_weight_min": -23.494407653808594,
      "activations/layer13_attention_weight_max": 67.09711456298828,
      "activations/layer13_attention_weight_min": -43.407955169677734,
      "activations/layer14_attention_weight_max": 78.83499908447266,
      "activations/layer14_attention_weight_min": -48.7855339050293,
      "activations/layer15_attention_weight_max": 55.758941650390625,
      "activations/layer15_attention_weight_min": -35.97754669189453,
      "activations/layer16_attention_weight_max": 56.197208404541016,
      "activations/layer16_attention_weight_min": -32.81239318847656,
      "activations/layer17_attention_weight_max": 70.28292846679688,
      "activations/layer17_attention_weight_min": -36.9024543762207,
      "activations/layer18_attention_weight_max": 67.6432113647461,
      "activations/layer18_attention_weight_min": -34.12055206298828,
      "activations/layer19_attention_weight_max": 23.572765350341797,
      "activations/layer19_attention_weight_min": -15.435591697692871,
      "activations/layer1_attention_weight_max": 16.041839599609375,
      "activations/layer1_attention_weight_min": -12.889693260192871,
      "activations/layer20_attention_weight_max": 22.790077209472656,
      "activations/layer20_attention_weight_min": -15.164847373962402,
      "activations/layer21_attention_weight_max": 38.5421028137207,
      "activations/layer21_attention_weight_min": -21.4520206451416,
      "activations/layer22_attention_weight_max": 46.890106201171875,
      "activations/layer22_attention_weight_min": -20.371694564819336,
      "activations/layer23_attention_weight_max": 26.071439743041992,
      "activations/layer23_attention_weight_min": -13.439177513122559,
      "activations/layer2_attention_weight_max": 25.333913803100586,
      "activations/layer2_attention_weight_min": -25.284360885620117,
      "activations/layer3_attention_weight_max": 58.83293914794922,
      "activations/layer3_attention_weight_min": -63.58233642578125,
      "activations/layer4_attention_weight_max": 90.65656280517578,
      "activations/layer4_attention_weight_min": -83.52459716796875,
      "activations/layer5_attention_weight_max": 135.77699279785156,
      "activations/layer5_attention_weight_min": -109.30978393554688,
      "activations/layer6_attention_weight_max": 62.676116943359375,
      "activations/layer6_attention_weight_min": -52.234867095947266,
      "activations/layer7_attention_weight_max": 63.26298904418945,
      "activations/layer7_attention_weight_min": -56.56315231323242,
      "activations/layer8_attention_weight_max": 57.97106170654297,
      "activations/layer8_attention_weight_min": -45.17854309082031,
      "activations/layer9_attention_weight_max": 79.77430725097656,
      "activations/layer9_attention_weight_min": -73.28939056396484,
      "epoch": 1.63,
      "learning_rate": 0.00014088674242424242,
      "loss": 3.0039,
      "step": 28100
    },
    {
      "activations/layer0_attention_weight_max": 14.34779167175293,
      "activations/layer0_attention_weight_min": -13.423507690429688,
      "activations/layer10_attention_weight_max": 62.56846618652344,
      "activations/layer10_attention_weight_min": -52.22364044189453,
      "activations/layer11_attention_weight_max": 45.28144454956055,
      "activations/layer11_attention_weight_min": -38.15269470214844,
      "activations/layer12_attention_weight_max": 28.910621643066406,
      "activations/layer12_attention_weight_min": -24.912412643432617,
      "activations/layer13_attention_weight_max": 58.50419998168945,
      "activations/layer13_attention_weight_min": -43.48088073730469,
      "activations/layer14_attention_weight_max": 51.39057540893555,
      "activations/layer14_attention_weight_min": -37.906105041503906,
      "activations/layer15_attention_weight_max": 54.282779693603516,
      "activations/layer15_attention_weight_min": -38.25849914550781,
      "activations/layer16_attention_weight_max": 43.05845642089844,
      "activations/layer16_attention_weight_min": -31.84159278869629,
      "activations/layer17_attention_weight_max": 53.726707458496094,
      "activations/layer17_attention_weight_min": -36.7029914855957,
      "activations/layer18_attention_weight_max": 62.74089813232422,
      "activations/layer18_attention_weight_min": -38.2503547668457,
      "activations/layer19_attention_weight_max": 22.8809871673584,
      "activations/layer19_attention_weight_min": -18.029512405395508,
      "activations/layer1_attention_weight_max": 16.263450622558594,
      "activations/layer1_attention_weight_min": -12.24563217163086,
      "activations/layer20_attention_weight_max": 22.365732192993164,
      "activations/layer20_attention_weight_min": -16.021507263183594,
      "activations/layer21_attention_weight_max": 36.25409698486328,
      "activations/layer21_attention_weight_min": -23.185392379760742,
      "activations/layer22_attention_weight_max": 36.420860290527344,
      "activations/layer22_attention_weight_min": -20.345863342285156,
      "activations/layer23_attention_weight_max": 21.48598289489746,
      "activations/layer23_attention_weight_min": -14.919878959655762,
      "activations/layer2_attention_weight_max": 26.17571258544922,
      "activations/layer2_attention_weight_min": -24.707744598388672,
      "activations/layer3_attention_weight_max": 56.2930908203125,
      "activations/layer3_attention_weight_min": -61.09869384765625,
      "activations/layer4_attention_weight_max": 86.26295471191406,
      "activations/layer4_attention_weight_min": -79.11211395263672,
      "activations/layer5_attention_weight_max": 123.90806579589844,
      "activations/layer5_attention_weight_min": -111.64114379882812,
      "activations/layer6_attention_weight_max": 60.79838180541992,
      "activations/layer6_attention_weight_min": -53.305213928222656,
      "activations/layer7_attention_weight_max": 57.21385192871094,
      "activations/layer7_attention_weight_min": -58.30423355102539,
      "activations/layer8_attention_weight_max": 46.087196350097656,
      "activations/layer8_attention_weight_min": -42.80519485473633,
      "activations/layer9_attention_weight_max": 62.94578170776367,
      "activations/layer9_attention_weight_min": -63.58417510986328,
      "epoch": 1.64,
      "learning_rate": 0.000140867803030303,
      "loss": 3.002,
      "step": 28150
    },
    {
      "activations/layer0_attention_weight_max": 13.867854118347168,
      "activations/layer0_attention_weight_min": -13.501296043395996,
      "activations/layer10_attention_weight_max": 59.86407470703125,
      "activations/layer10_attention_weight_min": -51.75032424926758,
      "activations/layer11_attention_weight_max": 44.04191207885742,
      "activations/layer11_attention_weight_min": -40.30266571044922,
      "activations/layer12_attention_weight_max": 29.339197158813477,
      "activations/layer12_attention_weight_min": -23.341306686401367,
      "activations/layer13_attention_weight_max": 84.71615600585938,
      "activations/layer13_attention_weight_min": -54.70925521850586,
      "activations/layer14_attention_weight_max": 96.98467254638672,
      "activations/layer14_attention_weight_min": -54.30078125,
      "activations/layer15_attention_weight_max": 52.11771774291992,
      "activations/layer15_attention_weight_min": -37.48867416381836,
      "activations/layer16_attention_weight_max": 45.66579055786133,
      "activations/layer16_attention_weight_min": -30.96233367919922,
      "activations/layer17_attention_weight_max": 66.44689178466797,
      "activations/layer17_attention_weight_min": -36.96058654785156,
      "activations/layer18_attention_weight_max": 71.82615661621094,
      "activations/layer18_attention_weight_min": -39.662940979003906,
      "activations/layer19_attention_weight_max": 21.41951560974121,
      "activations/layer19_attention_weight_min": -15.730597496032715,
      "activations/layer1_attention_weight_max": 15.856369018554688,
      "activations/layer1_attention_weight_min": -11.930523872375488,
      "activations/layer20_attention_weight_max": 23.393583297729492,
      "activations/layer20_attention_weight_min": -15.951226234436035,
      "activations/layer21_attention_weight_max": 43.6152229309082,
      "activations/layer21_attention_weight_min": -21.46891212463379,
      "activations/layer22_attention_weight_max": 39.072593688964844,
      "activations/layer22_attention_weight_min": -23.706890106201172,
      "activations/layer23_attention_weight_max": 23.940898895263672,
      "activations/layer23_attention_weight_min": -15.361215591430664,
      "activations/layer2_attention_weight_max": 25.39961051940918,
      "activations/layer2_attention_weight_min": -24.924964904785156,
      "activations/layer3_attention_weight_max": 56.5438346862793,
      "activations/layer3_attention_weight_min": -60.56730270385742,
      "activations/layer4_attention_weight_max": 87.62101745605469,
      "activations/layer4_attention_weight_min": -81.8080062866211,
      "activations/layer5_attention_weight_max": 129.56402587890625,
      "activations/layer5_attention_weight_min": -110.48783874511719,
      "activations/layer6_attention_weight_max": 65.04002380371094,
      "activations/layer6_attention_weight_min": -57.60234069824219,
      "activations/layer7_attention_weight_max": 61.916561126708984,
      "activations/layer7_attention_weight_min": -56.35011291503906,
      "activations/layer8_attention_weight_max": 49.96230697631836,
      "activations/layer8_attention_weight_min": -44.00719451904297,
      "activations/layer9_attention_weight_max": 70.42932891845703,
      "activations/layer9_attention_weight_min": -62.390785217285156,
      "epoch": 1.64,
      "learning_rate": 0.00014084886363636363,
      "loss": 3.0121,
      "step": 28200
    },
    {
      "activations/layer0_attention_weight_max": 17.116119384765625,
      "activations/layer0_attention_weight_min": -13.725021362304688,
      "activations/layer10_attention_weight_max": 69.67423248291016,
      "activations/layer10_attention_weight_min": -56.75450897216797,
      "activations/layer11_attention_weight_max": 45.045711517333984,
      "activations/layer11_attention_weight_min": -37.11848449707031,
      "activations/layer12_attention_weight_max": 29.848264694213867,
      "activations/layer12_attention_weight_min": -24.28080177307129,
      "activations/layer13_attention_weight_max": 82.93363189697266,
      "activations/layer13_attention_weight_min": -52.53312301635742,
      "activations/layer14_attention_weight_max": 70.45745086669922,
      "activations/layer14_attention_weight_min": -43.85124206542969,
      "activations/layer15_attention_weight_max": 64.88780975341797,
      "activations/layer15_attention_weight_min": -42.014488220214844,
      "activations/layer16_attention_weight_max": 50.50211715698242,
      "activations/layer16_attention_weight_min": -30.538818359375,
      "activations/layer17_attention_weight_max": 68.27030181884766,
      "activations/layer17_attention_weight_min": -37.28554153442383,
      "activations/layer18_attention_weight_max": 64.02459716796875,
      "activations/layer18_attention_weight_min": -35.222564697265625,
      "activations/layer19_attention_weight_max": 23.829456329345703,
      "activations/layer19_attention_weight_min": -15.319737434387207,
      "activations/layer1_attention_weight_max": 15.237703323364258,
      "activations/layer1_attention_weight_min": -14.08307933807373,
      "activations/layer20_attention_weight_max": 21.36786460876465,
      "activations/layer20_attention_weight_min": -19.77629280090332,
      "activations/layer21_attention_weight_max": 42.889808654785156,
      "activations/layer21_attention_weight_min": -23.464086532592773,
      "activations/layer22_attention_weight_max": 41.133235931396484,
      "activations/layer22_attention_weight_min": -22.858856201171875,
      "activations/layer23_attention_weight_max": 24.50196647644043,
      "activations/layer23_attention_weight_min": -14.162303924560547,
      "activations/layer2_attention_weight_max": 25.924087524414062,
      "activations/layer2_attention_weight_min": -25.50355339050293,
      "activations/layer3_attention_weight_max": 58.99272918701172,
      "activations/layer3_attention_weight_min": -62.81306076049805,
      "activations/layer4_attention_weight_max": 91.1175308227539,
      "activations/layer4_attention_weight_min": -85.07084655761719,
      "activations/layer5_attention_weight_max": 125.92521667480469,
      "activations/layer5_attention_weight_min": -115.44963073730469,
      "activations/layer6_attention_weight_max": 64.91868591308594,
      "activations/layer6_attention_weight_min": -57.9953727722168,
      "activations/layer7_attention_weight_max": 61.447021484375,
      "activations/layer7_attention_weight_min": -53.61752700805664,
      "activations/layer8_attention_weight_max": 53.16417694091797,
      "activations/layer8_attention_weight_min": -44.57039260864258,
      "activations/layer9_attention_weight_max": 66.58739471435547,
      "activations/layer9_attention_weight_min": -61.325653076171875,
      "epoch": 1.64,
      "learning_rate": 0.00014082992424242424,
      "loss": 2.9996,
      "step": 28250
    },
    {
      "activations/layer0_attention_weight_max": 15.110419273376465,
      "activations/layer0_attention_weight_min": -13.057808876037598,
      "activations/layer10_attention_weight_max": 63.229976654052734,
      "activations/layer10_attention_weight_min": -55.10558319091797,
      "activations/layer11_attention_weight_max": 41.11042785644531,
      "activations/layer11_attention_weight_min": -34.85268020629883,
      "activations/layer12_attention_weight_max": 32.72776412963867,
      "activations/layer12_attention_weight_min": -27.793027877807617,
      "activations/layer13_attention_weight_max": 55.797401428222656,
      "activations/layer13_attention_weight_min": -45.437278747558594,
      "activations/layer14_attention_weight_max": 68.85499572753906,
      "activations/layer14_attention_weight_min": -52.20960998535156,
      "activations/layer15_attention_weight_max": 47.67536926269531,
      "activations/layer15_attention_weight_min": -36.748634338378906,
      "activations/layer16_attention_weight_max": 44.43564224243164,
      "activations/layer16_attention_weight_min": -30.795026779174805,
      "activations/layer17_attention_weight_max": 62.20673370361328,
      "activations/layer17_attention_weight_min": -34.963619232177734,
      "activations/layer18_attention_weight_max": 56.759681701660156,
      "activations/layer18_attention_weight_min": -34.91569137573242,
      "activations/layer19_attention_weight_max": 18.303693771362305,
      "activations/layer19_attention_weight_min": -14.921833992004395,
      "activations/layer1_attention_weight_max": 15.119414329528809,
      "activations/layer1_attention_weight_min": -11.331808090209961,
      "activations/layer20_attention_weight_max": 16.146196365356445,
      "activations/layer20_attention_weight_min": -16.322996139526367,
      "activations/layer21_attention_weight_max": 33.96974182128906,
      "activations/layer21_attention_weight_min": -23.30324363708496,
      "activations/layer22_attention_weight_max": 34.02415466308594,
      "activations/layer22_attention_weight_min": -20.699687957763672,
      "activations/layer23_attention_weight_max": 20.58287811279297,
      "activations/layer23_attention_weight_min": -14.167522430419922,
      "activations/layer2_attention_weight_max": 26.5039119720459,
      "activations/layer2_attention_weight_min": -26.475364685058594,
      "activations/layer3_attention_weight_max": 59.32131576538086,
      "activations/layer3_attention_weight_min": -63.52505111694336,
      "activations/layer4_attention_weight_max": 88.48809051513672,
      "activations/layer4_attention_weight_min": -80.5686264038086,
      "activations/layer5_attention_weight_max": 125.32662963867188,
      "activations/layer5_attention_weight_min": -113.39633178710938,
      "activations/layer6_attention_weight_max": 63.22894287109375,
      "activations/layer6_attention_weight_min": -54.13138961791992,
      "activations/layer7_attention_weight_max": 58.25705337524414,
      "activations/layer7_attention_weight_min": -55.75619888305664,
      "activations/layer8_attention_weight_max": 49.163002014160156,
      "activations/layer8_attention_weight_min": -44.67180252075195,
      "activations/layer9_attention_weight_max": 65.95439147949219,
      "activations/layer9_attention_weight_min": -61.399627685546875,
      "epoch": 1.64,
      "learning_rate": 0.00014081098484848483,
      "loss": 3.008,
      "step": 28300
    },
    {
      "activations/layer0_attention_weight_max": 13.256794929504395,
      "activations/layer0_attention_weight_min": -12.770669937133789,
      "activations/layer10_attention_weight_max": 62.18628692626953,
      "activations/layer10_attention_weight_min": -52.25168991088867,
      "activations/layer11_attention_weight_max": 40.2777099609375,
      "activations/layer11_attention_weight_min": -36.32604217529297,
      "activations/layer12_attention_weight_max": 28.659303665161133,
      "activations/layer12_attention_weight_min": -26.866975784301758,
      "activations/layer13_attention_weight_max": 54.6551399230957,
      "activations/layer13_attention_weight_min": -41.00525665283203,
      "activations/layer14_attention_weight_max": 52.24484634399414,
      "activations/layer14_attention_weight_min": -38.934104919433594,
      "activations/layer15_attention_weight_max": 50.51525115966797,
      "activations/layer15_attention_weight_min": -39.637027740478516,
      "activations/layer16_attention_weight_max": 51.20745086669922,
      "activations/layer16_attention_weight_min": -32.06753921508789,
      "activations/layer17_attention_weight_max": 60.2619514465332,
      "activations/layer17_attention_weight_min": -40.57548904418945,
      "activations/layer18_attention_weight_max": 64.52967834472656,
      "activations/layer18_attention_weight_min": -37.6535530090332,
      "activations/layer19_attention_weight_max": 21.331010818481445,
      "activations/layer19_attention_weight_min": -15.01404094696045,
      "activations/layer1_attention_weight_max": 15.251411437988281,
      "activations/layer1_attention_weight_min": -13.76502799987793,
      "activations/layer20_attention_weight_max": 20.329601287841797,
      "activations/layer20_attention_weight_min": -16.969181060791016,
      "activations/layer21_attention_weight_max": 47.60693359375,
      "activations/layer21_attention_weight_min": -21.51177978515625,
      "activations/layer22_attention_weight_max": 43.638519287109375,
      "activations/layer22_attention_weight_min": -20.915782928466797,
      "activations/layer23_attention_weight_max": 25.68775749206543,
      "activations/layer23_attention_weight_min": -15.672893524169922,
      "activations/layer2_attention_weight_max": 26.24608612060547,
      "activations/layer2_attention_weight_min": -25.169178009033203,
      "activations/layer3_attention_weight_max": 58.038692474365234,
      "activations/layer3_attention_weight_min": -60.52357864379883,
      "activations/layer4_attention_weight_max": 87.07269287109375,
      "activations/layer4_attention_weight_min": -78.12054443359375,
      "activations/layer5_attention_weight_max": 122.57048797607422,
      "activations/layer5_attention_weight_min": -112.02095794677734,
      "activations/layer6_attention_weight_max": 59.05290222167969,
      "activations/layer6_attention_weight_min": -57.027183532714844,
      "activations/layer7_attention_weight_max": 57.037330627441406,
      "activations/layer7_attention_weight_min": -52.243682861328125,
      "activations/layer8_attention_weight_max": 48.31897735595703,
      "activations/layer8_attention_weight_min": -45.40955352783203,
      "activations/layer9_attention_weight_max": 62.0960693359375,
      "activations/layer9_attention_weight_min": -60.67292022705078,
      "epoch": 1.65,
      "learning_rate": 0.00014079204545454544,
      "loss": 2.9935,
      "step": 28350
    },
    {
      "activations/layer0_attention_weight_max": 15.829822540283203,
      "activations/layer0_attention_weight_min": -13.73881721496582,
      "activations/layer10_attention_weight_max": 61.414634704589844,
      "activations/layer10_attention_weight_min": -56.87922286987305,
      "activations/layer11_attention_weight_max": 50.999488830566406,
      "activations/layer11_attention_weight_min": -37.61041259765625,
      "activations/layer12_attention_weight_max": 26.5107479095459,
      "activations/layer12_attention_weight_min": -22.76563835144043,
      "activations/layer13_attention_weight_max": 57.531898498535156,
      "activations/layer13_attention_weight_min": -41.595298767089844,
      "activations/layer14_attention_weight_max": 52.03203582763672,
      "activations/layer14_attention_weight_min": -42.53879928588867,
      "activations/layer15_attention_weight_max": 57.82111740112305,
      "activations/layer15_attention_weight_min": -39.21160125732422,
      "activations/layer16_attention_weight_max": 40.20407485961914,
      "activations/layer16_attention_weight_min": -31.26005744934082,
      "activations/layer17_attention_weight_max": 54.81496047973633,
      "activations/layer17_attention_weight_min": -35.76065444946289,
      "activations/layer18_attention_weight_max": 60.95078659057617,
      "activations/layer18_attention_weight_min": -34.52265167236328,
      "activations/layer19_attention_weight_max": 23.4385986328125,
      "activations/layer19_attention_weight_min": -15.192044258117676,
      "activations/layer1_attention_weight_max": 15.786748886108398,
      "activations/layer1_attention_weight_min": -11.82831859588623,
      "activations/layer20_attention_weight_max": 18.753337860107422,
      "activations/layer20_attention_weight_min": -16.392539978027344,
      "activations/layer21_attention_weight_max": 39.77903747558594,
      "activations/layer21_attention_weight_min": -24.426889419555664,
      "activations/layer22_attention_weight_max": 39.36670684814453,
      "activations/layer22_attention_weight_min": -20.531963348388672,
      "activations/layer23_attention_weight_max": 26.517438888549805,
      "activations/layer23_attention_weight_min": -16.398202896118164,
      "activations/layer2_attention_weight_max": 24.420562744140625,
      "activations/layer2_attention_weight_min": -25.002262115478516,
      "activations/layer3_attention_weight_max": 57.18473434448242,
      "activations/layer3_attention_weight_min": -64.18699645996094,
      "activations/layer4_attention_weight_max": 87.7219467163086,
      "activations/layer4_attention_weight_min": -81.71141815185547,
      "activations/layer5_attention_weight_max": 127.37315368652344,
      "activations/layer5_attention_weight_min": -112.73257446289062,
      "activations/layer6_attention_weight_max": 60.76669692993164,
      "activations/layer6_attention_weight_min": -56.125614166259766,
      "activations/layer7_attention_weight_max": 63.73442459106445,
      "activations/layer7_attention_weight_min": -53.90167999267578,
      "activations/layer8_attention_weight_max": 51.80072784423828,
      "activations/layer8_attention_weight_min": -45.92756652832031,
      "activations/layer9_attention_weight_max": 84.76551055908203,
      "activations/layer9_attention_weight_min": -61.703853607177734,
      "epoch": 1.65,
      "learning_rate": 0.00014077310606060606,
      "loss": 2.9979,
      "step": 28400
    },
    {
      "activations/layer0_attention_weight_max": 17.242576599121094,
      "activations/layer0_attention_weight_min": -13.66868782043457,
      "activations/layer10_attention_weight_max": 66.88595581054688,
      "activations/layer10_attention_weight_min": -51.96259307861328,
      "activations/layer11_attention_weight_max": 47.358642578125,
      "activations/layer11_attention_weight_min": -35.61794662475586,
      "activations/layer12_attention_weight_max": 37.6762809753418,
      "activations/layer12_attention_weight_min": -24.637481689453125,
      "activations/layer13_attention_weight_max": 73.13668823242188,
      "activations/layer13_attention_weight_min": -43.84933853149414,
      "activations/layer14_attention_weight_max": 55.23964309692383,
      "activations/layer14_attention_weight_min": -37.3887825012207,
      "activations/layer15_attention_weight_max": 59.17546463012695,
      "activations/layer15_attention_weight_min": -38.5584602355957,
      "activations/layer16_attention_weight_max": 56.85380172729492,
      "activations/layer16_attention_weight_min": -30.338895797729492,
      "activations/layer17_attention_weight_max": 64.61160278320312,
      "activations/layer17_attention_weight_min": -36.80181121826172,
      "activations/layer18_attention_weight_max": 66.44000244140625,
      "activations/layer18_attention_weight_min": -33.59487533569336,
      "activations/layer19_attention_weight_max": 24.40520477294922,
      "activations/layer19_attention_weight_min": -14.574177742004395,
      "activations/layer1_attention_weight_max": 16.438549041748047,
      "activations/layer1_attention_weight_min": -14.495081901550293,
      "activations/layer20_attention_weight_max": 21.307292938232422,
      "activations/layer20_attention_weight_min": -15.345600128173828,
      "activations/layer21_attention_weight_max": 42.26683044433594,
      "activations/layer21_attention_weight_min": -21.743778228759766,
      "activations/layer22_attention_weight_max": 47.14727020263672,
      "activations/layer22_attention_weight_min": -21.128158569335938,
      "activations/layer23_attention_weight_max": 27.154624938964844,
      "activations/layer23_attention_weight_min": -14.47107982635498,
      "activations/layer2_attention_weight_max": 26.582509994506836,
      "activations/layer2_attention_weight_min": -25.640249252319336,
      "activations/layer3_attention_weight_max": 57.98896789550781,
      "activations/layer3_attention_weight_min": -61.74347686767578,
      "activations/layer4_attention_weight_max": 87.39537811279297,
      "activations/layer4_attention_weight_min": -80.3971176147461,
      "activations/layer5_attention_weight_max": 120.7474365234375,
      "activations/layer5_attention_weight_min": -115.04605102539062,
      "activations/layer6_attention_weight_max": 61.22531509399414,
      "activations/layer6_attention_weight_min": -54.1167106628418,
      "activations/layer7_attention_weight_max": 60.02055740356445,
      "activations/layer7_attention_weight_min": -52.797035217285156,
      "activations/layer8_attention_weight_max": 49.784568786621094,
      "activations/layer8_attention_weight_min": -43.26852035522461,
      "activations/layer9_attention_weight_max": 65.74815368652344,
      "activations/layer9_attention_weight_min": -63.7283935546875,
      "epoch": 1.65,
      "learning_rate": 0.00014075454545454545,
      "loss": 2.9904,
      "step": 28450
    },
    {
      "activations/layer0_attention_weight_max": 17.069799423217773,
      "activations/layer0_attention_weight_min": -13.873827934265137,
      "activations/layer10_attention_weight_max": 57.96953582763672,
      "activations/layer10_attention_weight_min": -47.78446960449219,
      "activations/layer11_attention_weight_max": 46.109859466552734,
      "activations/layer11_attention_weight_min": -39.363956451416016,
      "activations/layer12_attention_weight_max": 28.587800979614258,
      "activations/layer12_attention_weight_min": -25.34423828125,
      "activations/layer13_attention_weight_max": 59.684757232666016,
      "activations/layer13_attention_weight_min": -48.327083587646484,
      "activations/layer14_attention_weight_max": 50.473976135253906,
      "activations/layer14_attention_weight_min": -40.416873931884766,
      "activations/layer15_attention_weight_max": 58.111793518066406,
      "activations/layer15_attention_weight_min": -44.21731948852539,
      "activations/layer16_attention_weight_max": 44.07463073730469,
      "activations/layer16_attention_weight_min": -32.596290588378906,
      "activations/layer17_attention_weight_max": 60.6374397277832,
      "activations/layer17_attention_weight_min": -37.64497375488281,
      "activations/layer18_attention_weight_max": 68.17047119140625,
      "activations/layer18_attention_weight_min": -36.0514030456543,
      "activations/layer19_attention_weight_max": 23.21271324157715,
      "activations/layer19_attention_weight_min": -14.249648094177246,
      "activations/layer1_attention_weight_max": 15.850945472717285,
      "activations/layer1_attention_weight_min": -11.476178169250488,
      "activations/layer20_attention_weight_max": 21.108728408813477,
      "activations/layer20_attention_weight_min": -15.577616691589355,
      "activations/layer21_attention_weight_max": 42.699424743652344,
      "activations/layer21_attention_weight_min": -23.09220314025879,
      "activations/layer22_attention_weight_max": 39.43205642700195,
      "activations/layer22_attention_weight_min": -20.834882736206055,
      "activations/layer23_attention_weight_max": 27.76870346069336,
      "activations/layer23_attention_weight_min": -14.849451065063477,
      "activations/layer2_attention_weight_max": 24.97977066040039,
      "activations/layer2_attention_weight_min": -24.328153610229492,
      "activations/layer3_attention_weight_max": 57.22093963623047,
      "activations/layer3_attention_weight_min": -62.36681365966797,
      "activations/layer4_attention_weight_max": 87.6322021484375,
      "activations/layer4_attention_weight_min": -84.13119506835938,
      "activations/layer5_attention_weight_max": 127.66923522949219,
      "activations/layer5_attention_weight_min": -113.57522583007812,
      "activations/layer6_attention_weight_max": 61.138023376464844,
      "activations/layer6_attention_weight_min": -55.69218826293945,
      "activations/layer7_attention_weight_max": 61.858909606933594,
      "activations/layer7_attention_weight_min": -56.22402572631836,
      "activations/layer8_attention_weight_max": 52.89561462402344,
      "activations/layer8_attention_weight_min": -43.90153884887695,
      "activations/layer9_attention_weight_max": 82.11337280273438,
      "activations/layer9_attention_weight_min": -62.42542266845703,
      "epoch": 1.66,
      "learning_rate": 0.00014073560606060604,
      "loss": 3.0012,
      "step": 28500
    },
    {
      "activations/layer0_attention_weight_max": 13.544551849365234,
      "activations/layer0_attention_weight_min": -13.628291130065918,
      "activations/layer10_attention_weight_max": 60.08183288574219,
      "activations/layer10_attention_weight_min": -55.35895538330078,
      "activations/layer11_attention_weight_max": 39.2701416015625,
      "activations/layer11_attention_weight_min": -37.394683837890625,
      "activations/layer12_attention_weight_max": 29.450626373291016,
      "activations/layer12_attention_weight_min": -22.85297393798828,
      "activations/layer13_attention_weight_max": 82.72562408447266,
      "activations/layer13_attention_weight_min": -47.53760528564453,
      "activations/layer14_attention_weight_max": 80.77656555175781,
      "activations/layer14_attention_weight_min": -43.56300354003906,
      "activations/layer15_attention_weight_max": 60.873165130615234,
      "activations/layer15_attention_weight_min": -37.68427276611328,
      "activations/layer16_attention_weight_max": 48.96906280517578,
      "activations/layer16_attention_weight_min": -29.095176696777344,
      "activations/layer17_attention_weight_max": 66.19677734375,
      "activations/layer17_attention_weight_min": -38.10696792602539,
      "activations/layer18_attention_weight_max": 70.54273223876953,
      "activations/layer18_attention_weight_min": -33.12795639038086,
      "activations/layer19_attention_weight_max": 20.86507225036621,
      "activations/layer19_attention_weight_min": -15.229158401489258,
      "activations/layer1_attention_weight_max": 15.7946195602417,
      "activations/layer1_attention_weight_min": -13.727941513061523,
      "activations/layer20_attention_weight_max": 22.947223663330078,
      "activations/layer20_attention_weight_min": -18.057296752929688,
      "activations/layer21_attention_weight_max": 43.2171516418457,
      "activations/layer21_attention_weight_min": -22.325651168823242,
      "activations/layer22_attention_weight_max": 40.80746841430664,
      "activations/layer22_attention_weight_min": -22.907094955444336,
      "activations/layer23_attention_weight_max": 22.917560577392578,
      "activations/layer23_attention_weight_min": -13.95920467376709,
      "activations/layer2_attention_weight_max": 28.027305603027344,
      "activations/layer2_attention_weight_min": -24.772918701171875,
      "activations/layer3_attention_weight_max": 58.32613754272461,
      "activations/layer3_attention_weight_min": -59.402870178222656,
      "activations/layer4_attention_weight_max": 84.58042907714844,
      "activations/layer4_attention_weight_min": -77.17356872558594,
      "activations/layer5_attention_weight_max": 123.8495101928711,
      "activations/layer5_attention_weight_min": -116.3634033203125,
      "activations/layer6_attention_weight_max": 59.94184494018555,
      "activations/layer6_attention_weight_min": -55.46266555786133,
      "activations/layer7_attention_weight_max": 60.01173400878906,
      "activations/layer7_attention_weight_min": -52.91141891479492,
      "activations/layer8_attention_weight_max": 51.45665740966797,
      "activations/layer8_attention_weight_min": -45.56468963623047,
      "activations/layer9_attention_weight_max": 68.56584930419922,
      "activations/layer9_attention_weight_min": -59.65864181518555,
      "epoch": 1.66,
      "learning_rate": 0.00014071666666666665,
      "loss": 3.01,
      "step": 28550
    },
    {
      "activations/layer0_attention_weight_max": 13.369165420532227,
      "activations/layer0_attention_weight_min": -13.78182601928711,
      "activations/layer10_attention_weight_max": 56.6988525390625,
      "activations/layer10_attention_weight_min": -48.862396240234375,
      "activations/layer11_attention_weight_max": 44.81711959838867,
      "activations/layer11_attention_weight_min": -37.29764175415039,
      "activations/layer12_attention_weight_max": 28.513093948364258,
      "activations/layer12_attention_weight_min": -24.989181518554688,
      "activations/layer13_attention_weight_max": 62.69062042236328,
      "activations/layer13_attention_weight_min": -43.068660736083984,
      "activations/layer14_attention_weight_max": 57.20677947998047,
      "activations/layer14_attention_weight_min": -40.34435272216797,
      "activations/layer15_attention_weight_max": 56.92491149902344,
      "activations/layer15_attention_weight_min": -41.56278610229492,
      "activations/layer16_attention_weight_max": 45.53458786010742,
      "activations/layer16_attention_weight_min": -32.669132232666016,
      "activations/layer17_attention_weight_max": 56.587886810302734,
      "activations/layer17_attention_weight_min": -35.8670654296875,
      "activations/layer18_attention_weight_max": 58.48810958862305,
      "activations/layer18_attention_weight_min": -33.692901611328125,
      "activations/layer19_attention_weight_max": 23.835241317749023,
      "activations/layer19_attention_weight_min": -16.13636589050293,
      "activations/layer1_attention_weight_max": 15.516156196594238,
      "activations/layer1_attention_weight_min": -15.01268482208252,
      "activations/layer20_attention_weight_max": 24.833707809448242,
      "activations/layer20_attention_weight_min": -17.824783325195312,
      "activations/layer21_attention_weight_max": 39.70484161376953,
      "activations/layer21_attention_weight_min": -22.814184188842773,
      "activations/layer22_attention_weight_max": 39.27582931518555,
      "activations/layer22_attention_weight_min": -21.137123107910156,
      "activations/layer23_attention_weight_max": 23.898773193359375,
      "activations/layer23_attention_weight_min": -14.530989646911621,
      "activations/layer2_attention_weight_max": 26.5437068939209,
      "activations/layer2_attention_weight_min": -25.39727783203125,
      "activations/layer3_attention_weight_max": 58.44099044799805,
      "activations/layer3_attention_weight_min": -61.43528747558594,
      "activations/layer4_attention_weight_max": 89.5078353881836,
      "activations/layer4_attention_weight_min": -78.77107238769531,
      "activations/layer5_attention_weight_max": 129.69082641601562,
      "activations/layer5_attention_weight_min": -114.034912109375,
      "activations/layer6_attention_weight_max": 61.172576904296875,
      "activations/layer6_attention_weight_min": -54.970741271972656,
      "activations/layer7_attention_weight_max": 63.1253547668457,
      "activations/layer7_attention_weight_min": -56.84919738769531,
      "activations/layer8_attention_weight_max": 47.19953536987305,
      "activations/layer8_attention_weight_min": -40.6808967590332,
      "activations/layer9_attention_weight_max": 73.11910247802734,
      "activations/layer9_attention_weight_min": -66.47470092773438,
      "epoch": 1.66,
      "learning_rate": 0.00014069772727272727,
      "loss": 3.0086,
      "step": 28600
    },
    {
      "activations/layer0_attention_weight_max": 15.336075782775879,
      "activations/layer0_attention_weight_min": -13.369013786315918,
      "activations/layer10_attention_weight_max": 58.64447021484375,
      "activations/layer10_attention_weight_min": -49.825645446777344,
      "activations/layer11_attention_weight_max": 42.20421600341797,
      "activations/layer11_attention_weight_min": -37.859947204589844,
      "activations/layer12_attention_weight_max": 27.529373168945312,
      "activations/layer12_attention_weight_min": -25.567867279052734,
      "activations/layer13_attention_weight_max": 55.18329620361328,
      "activations/layer13_attention_weight_min": -43.808433532714844,
      "activations/layer14_attention_weight_max": 43.7423095703125,
      "activations/layer14_attention_weight_min": -35.60549545288086,
      "activations/layer15_attention_weight_max": 55.119625091552734,
      "activations/layer15_attention_weight_min": -41.14546585083008,
      "activations/layer16_attention_weight_max": 42.999046325683594,
      "activations/layer16_attention_weight_min": -30.750850677490234,
      "activations/layer17_attention_weight_max": 67.1278076171875,
      "activations/layer17_attention_weight_min": -41.08066940307617,
      "activations/layer18_attention_weight_max": 55.491416931152344,
      "activations/layer18_attention_weight_min": -33.17591857910156,
      "activations/layer19_attention_weight_max": 23.493982315063477,
      "activations/layer19_attention_weight_min": -15.690118789672852,
      "activations/layer1_attention_weight_max": 14.997960090637207,
      "activations/layer1_attention_weight_min": -12.894421577453613,
      "activations/layer20_attention_weight_max": 20.398189544677734,
      "activations/layer20_attention_weight_min": -15.813218116760254,
      "activations/layer21_attention_weight_max": 40.07619094848633,
      "activations/layer21_attention_weight_min": -23.233816146850586,
      "activations/layer22_attention_weight_max": 35.127830505371094,
      "activations/layer22_attention_weight_min": -22.360355377197266,
      "activations/layer23_attention_weight_max": 23.720714569091797,
      "activations/layer23_attention_weight_min": -15.749216079711914,
      "activations/layer2_attention_weight_max": 24.985538482666016,
      "activations/layer2_attention_weight_min": -26.704299926757812,
      "activations/layer3_attention_weight_max": 58.733642578125,
      "activations/layer3_attention_weight_min": -63.70514678955078,
      "activations/layer4_attention_weight_max": 90.59185028076172,
      "activations/layer4_attention_weight_min": -88.31063842773438,
      "activations/layer5_attention_weight_max": 127.10736083984375,
      "activations/layer5_attention_weight_min": -114.86451721191406,
      "activations/layer6_attention_weight_max": 62.408958435058594,
      "activations/layer6_attention_weight_min": -55.0501823425293,
      "activations/layer7_attention_weight_max": 58.250850677490234,
      "activations/layer7_attention_weight_min": -53.85391616821289,
      "activations/layer8_attention_weight_max": 45.80390167236328,
      "activations/layer8_attention_weight_min": -42.76845169067383,
      "activations/layer9_attention_weight_max": 65.27495574951172,
      "activations/layer9_attention_weight_min": -61.10007858276367,
      "epoch": 1.66,
      "learning_rate": 0.00014067878787878786,
      "loss": 3.0022,
      "step": 28650
    },
    {
      "activations/layer0_attention_weight_max": 14.59210205078125,
      "activations/layer0_attention_weight_min": -12.530905723571777,
      "activations/layer10_attention_weight_max": 66.01676940917969,
      "activations/layer10_attention_weight_min": -55.03406524658203,
      "activations/layer11_attention_weight_max": 50.931251525878906,
      "activations/layer11_attention_weight_min": -41.476890563964844,
      "activations/layer12_attention_weight_max": 29.83833122253418,
      "activations/layer12_attention_weight_min": -27.527530670166016,
      "activations/layer13_attention_weight_max": 61.23282241821289,
      "activations/layer13_attention_weight_min": -47.877357482910156,
      "activations/layer14_attention_weight_max": 49.810211181640625,
      "activations/layer14_attention_weight_min": -38.96763610839844,
      "activations/layer15_attention_weight_max": 59.2889289855957,
      "activations/layer15_attention_weight_min": -39.584510803222656,
      "activations/layer16_attention_weight_max": 41.35542297363281,
      "activations/layer16_attention_weight_min": -31.26682472229004,
      "activations/layer17_attention_weight_max": 57.39579391479492,
      "activations/layer17_attention_weight_min": -39.198917388916016,
      "activations/layer18_attention_weight_max": 58.09657669067383,
      "activations/layer18_attention_weight_min": -35.6799430847168,
      "activations/layer19_attention_weight_max": 22.09575653076172,
      "activations/layer19_attention_weight_min": -15.963396072387695,
      "activations/layer1_attention_weight_max": 15.448802947998047,
      "activations/layer1_attention_weight_min": -12.321869850158691,
      "activations/layer20_attention_weight_max": 22.0668888092041,
      "activations/layer20_attention_weight_min": -16.327661514282227,
      "activations/layer21_attention_weight_max": 36.173805236816406,
      "activations/layer21_attention_weight_min": -25.282548904418945,
      "activations/layer22_attention_weight_max": 34.09791946411133,
      "activations/layer22_attention_weight_min": -24.69227409362793,
      "activations/layer23_attention_weight_max": 22.65871238708496,
      "activations/layer23_attention_weight_min": -15.99293041229248,
      "activations/layer2_attention_weight_max": 25.40285301208496,
      "activations/layer2_attention_weight_min": -25.418272018432617,
      "activations/layer3_attention_weight_max": 57.88001251220703,
      "activations/layer3_attention_weight_min": -62.29493713378906,
      "activations/layer4_attention_weight_max": 88.80268096923828,
      "activations/layer4_attention_weight_min": -85.5412826538086,
      "activations/layer5_attention_weight_max": 128.80979919433594,
      "activations/layer5_attention_weight_min": -118.41166687011719,
      "activations/layer6_attention_weight_max": 64.09906005859375,
      "activations/layer6_attention_weight_min": -59.50725555419922,
      "activations/layer7_attention_weight_max": 61.878055572509766,
      "activations/layer7_attention_weight_min": -59.73579788208008,
      "activations/layer8_attention_weight_max": 59.43960189819336,
      "activations/layer8_attention_weight_min": -53.0403938293457,
      "activations/layer9_attention_weight_max": 83.45462799072266,
      "activations/layer9_attention_weight_min": -64.90216827392578,
      "epoch": 1.67,
      "learning_rate": 0.00014065984848484847,
      "loss": 3.003,
      "step": 28700
    },
    {
      "activations/layer0_attention_weight_max": 14.497722625732422,
      "activations/layer0_attention_weight_min": -12.54875373840332,
      "activations/layer10_attention_weight_max": 60.96451187133789,
      "activations/layer10_attention_weight_min": -54.385833740234375,
      "activations/layer11_attention_weight_max": 42.77677917480469,
      "activations/layer11_attention_weight_min": -38.071693420410156,
      "activations/layer12_attention_weight_max": 25.376359939575195,
      "activations/layer12_attention_weight_min": -25.484067916870117,
      "activations/layer13_attention_weight_max": 58.07920837402344,
      "activations/layer13_attention_weight_min": -41.109710693359375,
      "activations/layer14_attention_weight_max": 53.002010345458984,
      "activations/layer14_attention_weight_min": -34.99774169921875,
      "activations/layer15_attention_weight_max": 55.886680603027344,
      "activations/layer15_attention_weight_min": -39.906402587890625,
      "activations/layer16_attention_weight_max": 47.25514602661133,
      "activations/layer16_attention_weight_min": -29.57322883605957,
      "activations/layer17_attention_weight_max": 52.43177795410156,
      "activations/layer17_attention_weight_min": -35.759925842285156,
      "activations/layer18_attention_weight_max": 52.51066589355469,
      "activations/layer18_attention_weight_min": -34.94317626953125,
      "activations/layer19_attention_weight_max": 19.932287216186523,
      "activations/layer19_attention_weight_min": -15.369483947753906,
      "activations/layer1_attention_weight_max": 15.652104377746582,
      "activations/layer1_attention_weight_min": -11.916402816772461,
      "activations/layer20_attention_weight_max": 21.593717575073242,
      "activations/layer20_attention_weight_min": -15.945159912109375,
      "activations/layer21_attention_weight_max": 38.4827766418457,
      "activations/layer21_attention_weight_min": -21.42470359802246,
      "activations/layer22_attention_weight_max": 35.112613677978516,
      "activations/layer22_attention_weight_min": -19.59427833557129,
      "activations/layer23_attention_weight_max": 25.741268157958984,
      "activations/layer23_attention_weight_min": -15.214034080505371,
      "activations/layer2_attention_weight_max": 24.989368438720703,
      "activations/layer2_attention_weight_min": -24.12928009033203,
      "activations/layer3_attention_weight_max": 58.530338287353516,
      "activations/layer3_attention_weight_min": -61.454063415527344,
      "activations/layer4_attention_weight_max": 88.9894027709961,
      "activations/layer4_attention_weight_min": -81.02027130126953,
      "activations/layer5_attention_weight_max": 130.84567260742188,
      "activations/layer5_attention_weight_min": -113.83929443359375,
      "activations/layer6_attention_weight_max": 61.88380813598633,
      "activations/layer6_attention_weight_min": -56.26718521118164,
      "activations/layer7_attention_weight_max": 58.75579071044922,
      "activations/layer7_attention_weight_min": -58.298583984375,
      "activations/layer8_attention_weight_max": 48.234107971191406,
      "activations/layer8_attention_weight_min": -42.983089447021484,
      "activations/layer9_attention_weight_max": 68.41657257080078,
      "activations/layer9_attention_weight_min": -61.699989318847656,
      "epoch": 1.67,
      "learning_rate": 0.00014064090909090906,
      "loss": 3.0,
      "step": 28750
    },
    {
      "activations/layer0_attention_weight_max": 14.270076751708984,
      "activations/layer0_attention_weight_min": -13.356169700622559,
      "activations/layer10_attention_weight_max": 67.2003402709961,
      "activations/layer10_attention_weight_min": -55.57574462890625,
      "activations/layer11_attention_weight_max": 42.55811309814453,
      "activations/layer11_attention_weight_min": -35.754398345947266,
      "activations/layer12_attention_weight_max": 25.855504989624023,
      "activations/layer12_attention_weight_min": -25.16676139831543,
      "activations/layer13_attention_weight_max": 52.21932601928711,
      "activations/layer13_attention_weight_min": -41.4594841003418,
      "activations/layer14_attention_weight_max": 54.824195861816406,
      "activations/layer14_attention_weight_min": -41.082889556884766,
      "activations/layer15_attention_weight_max": 46.29102325439453,
      "activations/layer15_attention_weight_min": -36.56596755981445,
      "activations/layer16_attention_weight_max": 42.24240493774414,
      "activations/layer16_attention_weight_min": -29.912139892578125,
      "activations/layer17_attention_weight_max": 60.81380844116211,
      "activations/layer17_attention_weight_min": -38.417842864990234,
      "activations/layer18_attention_weight_max": 57.20641326904297,
      "activations/layer18_attention_weight_min": -34.587890625,
      "activations/layer19_attention_weight_max": 19.063642501831055,
      "activations/layer19_attention_weight_min": -17.060182571411133,
      "activations/layer1_attention_weight_max": 15.176946640014648,
      "activations/layer1_attention_weight_min": -12.189167976379395,
      "activations/layer20_attention_weight_max": 18.742740631103516,
      "activations/layer20_attention_weight_min": -14.863665580749512,
      "activations/layer21_attention_weight_max": 38.70962142944336,
      "activations/layer21_attention_weight_min": -21.354408264160156,
      "activations/layer22_attention_weight_max": 34.97016143798828,
      "activations/layer22_attention_weight_min": -21.576967239379883,
      "activations/layer23_attention_weight_max": 22.23883819580078,
      "activations/layer23_attention_weight_min": -15.252771377563477,
      "activations/layer2_attention_weight_max": 25.833890914916992,
      "activations/layer2_attention_weight_min": -25.639263153076172,
      "activations/layer3_attention_weight_max": 58.52210235595703,
      "activations/layer3_attention_weight_min": -63.70606994628906,
      "activations/layer4_attention_weight_max": 92.52769470214844,
      "activations/layer4_attention_weight_min": -81.46466827392578,
      "activations/layer5_attention_weight_max": 127.32244110107422,
      "activations/layer5_attention_weight_min": -113.78782653808594,
      "activations/layer6_attention_weight_max": 61.65464782714844,
      "activations/layer6_attention_weight_min": -55.0144157409668,
      "activations/layer7_attention_weight_max": 59.532440185546875,
      "activations/layer7_attention_weight_min": -53.091941833496094,
      "activations/layer8_attention_weight_max": 49.73477554321289,
      "activations/layer8_attention_weight_min": -43.80177688598633,
      "activations/layer9_attention_weight_max": 66.56978607177734,
      "activations/layer9_attention_weight_min": -57.80952072143555,
      "epoch": 1.67,
      "learning_rate": 0.00014062196969696967,
      "loss": 2.9864,
      "step": 28800
    },
    {
      "activations/layer0_attention_weight_max": 14.613941192626953,
      "activations/layer0_attention_weight_min": -13.243375778198242,
      "activations/layer10_attention_weight_max": 61.250946044921875,
      "activations/layer10_attention_weight_min": -53.77426528930664,
      "activations/layer11_attention_weight_max": 46.47251892089844,
      "activations/layer11_attention_weight_min": -37.54607391357422,
      "activations/layer12_attention_weight_max": 33.5122184753418,
      "activations/layer12_attention_weight_min": -27.226842880249023,
      "activations/layer13_attention_weight_max": 58.200706481933594,
      "activations/layer13_attention_weight_min": -42.1546630859375,
      "activations/layer14_attention_weight_max": 58.41672897338867,
      "activations/layer14_attention_weight_min": -39.14280319213867,
      "activations/layer15_attention_weight_max": 59.6864013671875,
      "activations/layer15_attention_weight_min": -42.128910064697266,
      "activations/layer16_attention_weight_max": 44.51730728149414,
      "activations/layer16_attention_weight_min": -31.22754669189453,
      "activations/layer17_attention_weight_max": 61.52186965942383,
      "activations/layer17_attention_weight_min": -37.83882522583008,
      "activations/layer18_attention_weight_max": 73.83546447753906,
      "activations/layer18_attention_weight_min": -35.016082763671875,
      "activations/layer19_attention_weight_max": 24.2171688079834,
      "activations/layer19_attention_weight_min": -14.711587905883789,
      "activations/layer1_attention_weight_max": 16.637950897216797,
      "activations/layer1_attention_weight_min": -12.623855590820312,
      "activations/layer20_attention_weight_max": 21.34075355529785,
      "activations/layer20_attention_weight_min": -16.0909366607666,
      "activations/layer21_attention_weight_max": 43.826934814453125,
      "activations/layer21_attention_weight_min": -21.478038787841797,
      "activations/layer22_attention_weight_max": 43.07320785522461,
      "activations/layer22_attention_weight_min": -22.49089813232422,
      "activations/layer23_attention_weight_max": 26.498939514160156,
      "activations/layer23_attention_weight_min": -18.338775634765625,
      "activations/layer2_attention_weight_max": 25.297130584716797,
      "activations/layer2_attention_weight_min": -24.949277877807617,
      "activations/layer3_attention_weight_max": 56.91053771972656,
      "activations/layer3_attention_weight_min": -62.301570892333984,
      "activations/layer4_attention_weight_max": 88.4364242553711,
      "activations/layer4_attention_weight_min": -83.0429916381836,
      "activations/layer5_attention_weight_max": 126.676513671875,
      "activations/layer5_attention_weight_min": -118.30244445800781,
      "activations/layer6_attention_weight_max": 62.5827751159668,
      "activations/layer6_attention_weight_min": -53.43785858154297,
      "activations/layer7_attention_weight_max": 58.97734451293945,
      "activations/layer7_attention_weight_min": -52.122982025146484,
      "activations/layer8_attention_weight_max": 49.96916580200195,
      "activations/layer8_attention_weight_min": -45.573917388916016,
      "activations/layer9_attention_weight_max": 67.72270965576172,
      "activations/layer9_attention_weight_min": -59.505672454833984,
      "epoch": 1.68,
      "learning_rate": 0.0001406030303030303,
      "loss": 3.0001,
      "step": 28850
    },
    {
      "activations/layer0_attention_weight_max": 15.852351188659668,
      "activations/layer0_attention_weight_min": -13.599791526794434,
      "activations/layer10_attention_weight_max": 60.42780303955078,
      "activations/layer10_attention_weight_min": -53.86344909667969,
      "activations/layer11_attention_weight_max": 46.567535400390625,
      "activations/layer11_attention_weight_min": -34.688087463378906,
      "activations/layer12_attention_weight_max": 32.67119598388672,
      "activations/layer12_attention_weight_min": -26.502033233642578,
      "activations/layer13_attention_weight_max": 69.62518310546875,
      "activations/layer13_attention_weight_min": -50.80479431152344,
      "activations/layer14_attention_weight_max": 63.52236557006836,
      "activations/layer14_attention_weight_min": -45.22646713256836,
      "activations/layer15_attention_weight_max": 51.54336166381836,
      "activations/layer15_attention_weight_min": -40.438968658447266,
      "activations/layer16_attention_weight_max": 44.24460220336914,
      "activations/layer16_attention_weight_min": -30.122156143188477,
      "activations/layer17_attention_weight_max": 59.50016403198242,
      "activations/layer17_attention_weight_min": -36.64429473876953,
      "activations/layer18_attention_weight_max": 61.81318283081055,
      "activations/layer18_attention_weight_min": -35.351646423339844,
      "activations/layer19_attention_weight_max": 19.157785415649414,
      "activations/layer19_attention_weight_min": -13.916923522949219,
      "activations/layer1_attention_weight_max": 16.31043243408203,
      "activations/layer1_attention_weight_min": -12.919990539550781,
      "activations/layer20_attention_weight_max": 17.406240463256836,
      "activations/layer20_attention_weight_min": -15.323497772216797,
      "activations/layer21_attention_weight_max": 40.604496002197266,
      "activations/layer21_attention_weight_min": -20.9320068359375,
      "activations/layer22_attention_weight_max": 36.1475830078125,
      "activations/layer22_attention_weight_min": -20.72267723083496,
      "activations/layer23_attention_weight_max": 21.142681121826172,
      "activations/layer23_attention_weight_min": -14.576465606689453,
      "activations/layer2_attention_weight_max": 26.246105194091797,
      "activations/layer2_attention_weight_min": -25.180870056152344,
      "activations/layer3_attention_weight_max": 57.19804000854492,
      "activations/layer3_attention_weight_min": -59.53034591674805,
      "activations/layer4_attention_weight_max": 86.20378875732422,
      "activations/layer4_attention_weight_min": -78.12124633789062,
      "activations/layer5_attention_weight_max": 122.17182922363281,
      "activations/layer5_attention_weight_min": -108.866943359375,
      "activations/layer6_attention_weight_max": 57.70924377441406,
      "activations/layer6_attention_weight_min": -52.250553131103516,
      "activations/layer7_attention_weight_max": 58.862548828125,
      "activations/layer7_attention_weight_min": -56.96527862548828,
      "activations/layer8_attention_weight_max": 45.202850341796875,
      "activations/layer8_attention_weight_min": -45.23906326293945,
      "activations/layer9_attention_weight_max": 72.04656982421875,
      "activations/layer9_attention_weight_min": -65.6982192993164,
      "epoch": 1.68,
      "learning_rate": 0.00014058409090909088,
      "loss": 3.0037,
      "step": 28900
    },
    {
      "activations/layer0_attention_weight_max": 16.707792282104492,
      "activations/layer0_attention_weight_min": -13.583900451660156,
      "activations/layer10_attention_weight_max": 74.01102447509766,
      "activations/layer10_attention_weight_min": -53.7427978515625,
      "activations/layer11_attention_weight_max": 50.90766525268555,
      "activations/layer11_attention_weight_min": -41.00499725341797,
      "activations/layer12_attention_weight_max": 33.58540725708008,
      "activations/layer12_attention_weight_min": -27.146488189697266,
      "activations/layer13_attention_weight_max": 72.03276062011719,
      "activations/layer13_attention_weight_min": -46.62783432006836,
      "activations/layer14_attention_weight_max": 79.05980682373047,
      "activations/layer14_attention_weight_min": -44.44649887084961,
      "activations/layer15_attention_weight_max": 50.97074890136719,
      "activations/layer15_attention_weight_min": -34.994056701660156,
      "activations/layer16_attention_weight_max": 38.3205451965332,
      "activations/layer16_attention_weight_min": -29.37368392944336,
      "activations/layer17_attention_weight_max": 55.03089141845703,
      "activations/layer17_attention_weight_min": -36.900367736816406,
      "activations/layer18_attention_weight_max": 52.120506286621094,
      "activations/layer18_attention_weight_min": -35.59789276123047,
      "activations/layer19_attention_weight_max": 20.646963119506836,
      "activations/layer19_attention_weight_min": -13.486510276794434,
      "activations/layer1_attention_weight_max": 15.703346252441406,
      "activations/layer1_attention_weight_min": -12.018508911132812,
      "activations/layer20_attention_weight_max": 20.083110809326172,
      "activations/layer20_attention_weight_min": -14.650839805603027,
      "activations/layer21_attention_weight_max": 38.857627868652344,
      "activations/layer21_attention_weight_min": -22.33036231994629,
      "activations/layer22_attention_weight_max": 36.36203384399414,
      "activations/layer22_attention_weight_min": -21.152503967285156,
      "activations/layer23_attention_weight_max": 21.92030906677246,
      "activations/layer23_attention_weight_min": -15.66163158416748,
      "activations/layer2_attention_weight_max": 26.131938934326172,
      "activations/layer2_attention_weight_min": -24.712276458740234,
      "activations/layer3_attention_weight_max": 59.15077590942383,
      "activations/layer3_attention_weight_min": -63.41030502319336,
      "activations/layer4_attention_weight_max": 88.17523956298828,
      "activations/layer4_attention_weight_min": -77.1529541015625,
      "activations/layer5_attention_weight_max": 138.0591583251953,
      "activations/layer5_attention_weight_min": -108.45384216308594,
      "activations/layer6_attention_weight_max": 64.38978576660156,
      "activations/layer6_attention_weight_min": -51.51591491699219,
      "activations/layer7_attention_weight_max": 61.27528381347656,
      "activations/layer7_attention_weight_min": -57.90118408203125,
      "activations/layer8_attention_weight_max": 65.87870025634766,
      "activations/layer8_attention_weight_min": -56.38838577270508,
      "activations/layer9_attention_weight_max": 84.42599487304688,
      "activations/layer9_attention_weight_min": -69.75828552246094,
      "epoch": 1.68,
      "learning_rate": 0.0001405651515151515,
      "loss": 2.9832,
      "step": 28950
    },
    {
      "activations/layer0_attention_weight_max": 14.041451454162598,
      "activations/layer0_attention_weight_min": -13.683956146240234,
      "activations/layer10_attention_weight_max": 55.513206481933594,
      "activations/layer10_attention_weight_min": -52.46574020385742,
      "activations/layer11_attention_weight_max": 38.857112884521484,
      "activations/layer11_attention_weight_min": -36.00896453857422,
      "activations/layer12_attention_weight_max": 24.660629272460938,
      "activations/layer12_attention_weight_min": -25.430376052856445,
      "activations/layer13_attention_weight_max": 59.93732833862305,
      "activations/layer13_attention_weight_min": -45.69717788696289,
      "activations/layer14_attention_weight_max": 58.91352844238281,
      "activations/layer14_attention_weight_min": -48.04189682006836,
      "activations/layer15_attention_weight_max": 52.47541427612305,
      "activations/layer15_attention_weight_min": -37.56832504272461,
      "activations/layer16_attention_weight_max": 40.27244567871094,
      "activations/layer16_attention_weight_min": -29.49405860900879,
      "activations/layer17_attention_weight_max": 52.72607421875,
      "activations/layer17_attention_weight_min": -34.675926208496094,
      "activations/layer18_attention_weight_max": 55.0119514465332,
      "activations/layer18_attention_weight_min": -34.49864959716797,
      "activations/layer19_attention_weight_max": 23.76423454284668,
      "activations/layer19_attention_weight_min": -14.778826713562012,
      "activations/layer1_attention_weight_max": 16.58555030822754,
      "activations/layer1_attention_weight_min": -12.144509315490723,
      "activations/layer20_attention_weight_max": 19.670469284057617,
      "activations/layer20_attention_weight_min": -15.758522033691406,
      "activations/layer21_attention_weight_max": 36.72883605957031,
      "activations/layer21_attention_weight_min": -21.58490753173828,
      "activations/layer22_attention_weight_max": 36.69582748413086,
      "activations/layer22_attention_weight_min": -24.553443908691406,
      "activations/layer23_attention_weight_max": 22.595792770385742,
      "activations/layer23_attention_weight_min": -13.92353343963623,
      "activations/layer2_attention_weight_max": 26.253849029541016,
      "activations/layer2_attention_weight_min": -24.73117446899414,
      "activations/layer3_attention_weight_max": 59.149810791015625,
      "activations/layer3_attention_weight_min": -61.721126556396484,
      "activations/layer4_attention_weight_max": 84.95917510986328,
      "activations/layer4_attention_weight_min": -82.70714569091797,
      "activations/layer5_attention_weight_max": 123.81564331054688,
      "activations/layer5_attention_weight_min": -114.93877410888672,
      "activations/layer6_attention_weight_max": 62.1456298828125,
      "activations/layer6_attention_weight_min": -57.2926139831543,
      "activations/layer7_attention_weight_max": 56.964996337890625,
      "activations/layer7_attention_weight_min": -54.14409255981445,
      "activations/layer8_attention_weight_max": 45.896949768066406,
      "activations/layer8_attention_weight_min": -47.054786682128906,
      "activations/layer9_attention_weight_max": 64.47239685058594,
      "activations/layer9_attention_weight_min": -61.3796501159668,
      "epoch": 1.69,
      "learning_rate": 0.0001405462121212121,
      "loss": 2.996,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_loss": 2.94140625,
      "eval_runtime": 8.4715,
      "eval_samples_per_second": 506.877,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_openwebtext_loss": 2.94140625,
      "eval_openwebtext_ppl": 18.942465433314446,
      "eval_openwebtext_runtime": 8.4715,
      "eval_openwebtext_samples_per_second": 506.877,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_wikitext_loss": 3.208984375,
      "eval_wikitext_ppl": 24.753932740556596,
      "eval_wikitext_runtime": 1.9444,
      "eval_wikitext_samples_per_second": 234.517,
      "step": 29000
    },
    {
      "epoch": 1.69,
      "eval_lambada_loss": 3.1484375,
      "eval_lambada_ppl": 23.299630451611073,
      "eval_lambada_runtime": 9.5481,
      "eval_lambada_samples_per_second": 509.945,
      "step": 29000
    },
    {
      "activations/layer0_attention_weight_max": 14.369081497192383,
      "activations/layer0_attention_weight_min": -13.54248332977295,
      "activations/layer10_attention_weight_max": 67.38787841796875,
      "activations/layer10_attention_weight_min": -52.79022979736328,
      "activations/layer11_attention_weight_max": 39.66400146484375,
      "activations/layer11_attention_weight_min": -38.73285675048828,
      "activations/layer12_attention_weight_max": 31.13351821899414,
      "activations/layer12_attention_weight_min": -25.525985717773438,
      "activations/layer13_attention_weight_max": 61.21553421020508,
      "activations/layer13_attention_weight_min": -44.11933898925781,
      "activations/layer14_attention_weight_max": 56.91250228881836,
      "activations/layer14_attention_weight_min": -37.30606460571289,
      "activations/layer15_attention_weight_max": 56.513492584228516,
      "activations/layer15_attention_weight_min": -39.61298370361328,
      "activations/layer16_attention_weight_max": 45.84494400024414,
      "activations/layer16_attention_weight_min": -28.81634521484375,
      "activations/layer17_attention_weight_max": 55.52751922607422,
      "activations/layer17_attention_weight_min": -35.068565368652344,
      "activations/layer18_attention_weight_max": 64.84940338134766,
      "activations/layer18_attention_weight_min": -35.188961029052734,
      "activations/layer19_attention_weight_max": 19.28975486755371,
      "activations/layer19_attention_weight_min": -14.282225608825684,
      "activations/layer1_attention_weight_max": 16.23106575012207,
      "activations/layer1_attention_weight_min": -11.761041641235352,
      "activations/layer20_attention_weight_max": 18.523103713989258,
      "activations/layer20_attention_weight_min": -15.623775482177734,
      "activations/layer21_attention_weight_max": 39.0439567565918,
      "activations/layer21_attention_weight_min": -20.148990631103516,
      "activations/layer22_attention_weight_max": 38.03221130371094,
      "activations/layer22_attention_weight_min": -19.64683723449707,
      "activations/layer23_attention_weight_max": 23.48157501220703,
      "activations/layer23_attention_weight_min": -13.854755401611328,
      "activations/layer2_attention_weight_max": 25.620288848876953,
      "activations/layer2_attention_weight_min": -25.386905670166016,
      "activations/layer3_attention_weight_max": 55.91979217529297,
      "activations/layer3_attention_weight_min": -59.9306755065918,
      "activations/layer4_attention_weight_max": 88.12529754638672,
      "activations/layer4_attention_weight_min": -80.90782928466797,
      "activations/layer5_attention_weight_max": 120.9753646850586,
      "activations/layer5_attention_weight_min": -114.49504089355469,
      "activations/layer6_attention_weight_max": 61.68292999267578,
      "activations/layer6_attention_weight_min": -50.06269073486328,
      "activations/layer7_attention_weight_max": 58.91309356689453,
      "activations/layer7_attention_weight_min": -61.05739974975586,
      "activations/layer8_attention_weight_max": 50.64360809326172,
      "activations/layer8_attention_weight_min": -47.0793342590332,
      "activations/layer9_attention_weight_max": 68.3199234008789,
      "activations/layer9_attention_weight_min": -64.67584991455078,
      "epoch": 1.69,
      "learning_rate": 0.00014052727272727272,
      "loss": 3.0062,
      "step": 29050
    },
    {
      "activations/layer0_attention_weight_max": 15.055591583251953,
      "activations/layer0_attention_weight_min": -13.547602653503418,
      "activations/layer10_attention_weight_max": 63.25485610961914,
      "activations/layer10_attention_weight_min": -49.855674743652344,
      "activations/layer11_attention_weight_max": 45.070159912109375,
      "activations/layer11_attention_weight_min": -37.12061309814453,
      "activations/layer12_attention_weight_max": 33.249732971191406,
      "activations/layer12_attention_weight_min": -26.455263137817383,
      "activations/layer13_attention_weight_max": 66.28570556640625,
      "activations/layer13_attention_weight_min": -41.75924301147461,
      "activations/layer14_attention_weight_max": 50.65974426269531,
      "activations/layer14_attention_weight_min": -35.38889694213867,
      "activations/layer15_attention_weight_max": 61.561798095703125,
      "activations/layer15_attention_weight_min": -43.6444206237793,
      "activations/layer16_attention_weight_max": 44.14122009277344,
      "activations/layer16_attention_weight_min": -31.512256622314453,
      "activations/layer17_attention_weight_max": 56.89584732055664,
      "activations/layer17_attention_weight_min": -36.79141616821289,
      "activations/layer18_attention_weight_max": 60.45705032348633,
      "activations/layer18_attention_weight_min": -34.00691223144531,
      "activations/layer19_attention_weight_max": 20.04310417175293,
      "activations/layer19_attention_weight_min": -16.302398681640625,
      "activations/layer1_attention_weight_max": 14.896037101745605,
      "activations/layer1_attention_weight_min": -12.755669593811035,
      "activations/layer20_attention_weight_max": 26.217266082763672,
      "activations/layer20_attention_weight_min": -16.434350967407227,
      "activations/layer21_attention_weight_max": 44.44126510620117,
      "activations/layer21_attention_weight_min": -22.05402946472168,
      "activations/layer22_attention_weight_max": 39.65105438232422,
      "activations/layer22_attention_weight_min": -20.184289932250977,
      "activations/layer23_attention_weight_max": 24.221097946166992,
      "activations/layer23_attention_weight_min": -13.953165054321289,
      "activations/layer2_attention_weight_max": 26.25501823425293,
      "activations/layer2_attention_weight_min": -25.416866302490234,
      "activations/layer3_attention_weight_max": 61.98490905761719,
      "activations/layer3_attention_weight_min": -65.32951354980469,
      "activations/layer4_attention_weight_max": 95.64811706542969,
      "activations/layer4_attention_weight_min": -86.48313903808594,
      "activations/layer5_attention_weight_max": 132.92413330078125,
      "activations/layer5_attention_weight_min": -115.43385314941406,
      "activations/layer6_attention_weight_max": 64.1681900024414,
      "activations/layer6_attention_weight_min": -55.84651184082031,
      "activations/layer7_attention_weight_max": 63.02003479003906,
      "activations/layer7_attention_weight_min": -54.013214111328125,
      "activations/layer8_attention_weight_max": 53.13992691040039,
      "activations/layer8_attention_weight_min": -44.28945541381836,
      "activations/layer9_attention_weight_max": 70.69503021240234,
      "activations/layer9_attention_weight_min": -61.221466064453125,
      "epoch": 1.69,
      "learning_rate": 0.0001405083333333333,
      "loss": 3.0104,
      "step": 29100
    },
    {
      "activations/layer0_attention_weight_max": 13.977828979492188,
      "activations/layer0_attention_weight_min": -13.676605224609375,
      "activations/layer10_attention_weight_max": 71.17207336425781,
      "activations/layer10_attention_weight_min": -51.849517822265625,
      "activations/layer11_attention_weight_max": 50.4987907409668,
      "activations/layer11_attention_weight_min": -39.25150680541992,
      "activations/layer12_attention_weight_max": 28.468921661376953,
      "activations/layer12_attention_weight_min": -23.131961822509766,
      "activations/layer13_attention_weight_max": 74.21500396728516,
      "activations/layer13_attention_weight_min": -44.91414260864258,
      "activations/layer14_attention_weight_max": 60.054656982421875,
      "activations/layer14_attention_weight_min": -36.44219207763672,
      "activations/layer15_attention_weight_max": 62.790348052978516,
      "activations/layer15_attention_weight_min": -40.34978103637695,
      "activations/layer16_attention_weight_max": 48.828697204589844,
      "activations/layer16_attention_weight_min": -29.721986770629883,
      "activations/layer17_attention_weight_max": 66.12259674072266,
      "activations/layer17_attention_weight_min": -37.498878479003906,
      "activations/layer18_attention_weight_max": 76.30828094482422,
      "activations/layer18_attention_weight_min": -37.63555908203125,
      "activations/layer19_attention_weight_max": 25.717166900634766,
      "activations/layer19_attention_weight_min": -16.60150718688965,
      "activations/layer1_attention_weight_max": 15.081086158752441,
      "activations/layer1_attention_weight_min": -16.03423500061035,
      "activations/layer20_attention_weight_max": 20.06369400024414,
      "activations/layer20_attention_weight_min": -17.277448654174805,
      "activations/layer21_attention_weight_max": 40.82209396362305,
      "activations/layer21_attention_weight_min": -21.306028366088867,
      "activations/layer22_attention_weight_max": 40.77065658569336,
      "activations/layer22_attention_weight_min": -21.839641571044922,
      "activations/layer23_attention_weight_max": 24.663541793823242,
      "activations/layer23_attention_weight_min": -15.17471981048584,
      "activations/layer2_attention_weight_max": 27.738744735717773,
      "activations/layer2_attention_weight_min": -27.035663604736328,
      "activations/layer3_attention_weight_max": 61.72701644897461,
      "activations/layer3_attention_weight_min": -64.9696044921875,
      "activations/layer4_attention_weight_max": 92.56591796875,
      "activations/layer4_attention_weight_min": -82.51264953613281,
      "activations/layer5_attention_weight_max": 144.217529296875,
      "activations/layer5_attention_weight_min": -118.74667358398438,
      "activations/layer6_attention_weight_max": 70.8113021850586,
      "activations/layer6_attention_weight_min": -57.49069595336914,
      "activations/layer7_attention_weight_max": 75.39682006835938,
      "activations/layer7_attention_weight_min": -56.14528274536133,
      "activations/layer8_attention_weight_max": 65.70653533935547,
      "activations/layer8_attention_weight_min": -48.551448822021484,
      "activations/layer9_attention_weight_max": 82.69139099121094,
      "activations/layer9_attention_weight_min": -61.37984848022461,
      "epoch": 1.69,
      "learning_rate": 0.00014048939393939393,
      "loss": 2.9905,
      "step": 29150
    },
    {
      "activations/layer0_attention_weight_max": 15.266637802124023,
      "activations/layer0_attention_weight_min": -13.325366020202637,
      "activations/layer10_attention_weight_max": 61.795894622802734,
      "activations/layer10_attention_weight_min": -51.624290466308594,
      "activations/layer11_attention_weight_max": 46.20383834838867,
      "activations/layer11_attention_weight_min": -39.90898132324219,
      "activations/layer12_attention_weight_max": 31.697603225708008,
      "activations/layer12_attention_weight_min": -24.186969757080078,
      "activations/layer13_attention_weight_max": 54.480587005615234,
      "activations/layer13_attention_weight_min": -42.18125915527344,
      "activations/layer14_attention_weight_max": 52.2960205078125,
      "activations/layer14_attention_weight_min": -40.81779861450195,
      "activations/layer15_attention_weight_max": 54.48232650756836,
      "activations/layer15_attention_weight_min": -39.28197479248047,
      "activations/layer16_attention_weight_max": 48.16926193237305,
      "activations/layer16_attention_weight_min": -30.44434356689453,
      "activations/layer17_attention_weight_max": 55.29094314575195,
      "activations/layer17_attention_weight_min": -37.01647186279297,
      "activations/layer18_attention_weight_max": 59.046199798583984,
      "activations/layer18_attention_weight_min": -34.525020599365234,
      "activations/layer19_attention_weight_max": 20.846086502075195,
      "activations/layer19_attention_weight_min": -14.147557258605957,
      "activations/layer1_attention_weight_max": 14.70844554901123,
      "activations/layer1_attention_weight_min": -13.565521240234375,
      "activations/layer20_attention_weight_max": 19.393436431884766,
      "activations/layer20_attention_weight_min": -16.194759368896484,
      "activations/layer21_attention_weight_max": 34.625762939453125,
      "activations/layer21_attention_weight_min": -21.674959182739258,
      "activations/layer22_attention_weight_max": 31.57021141052246,
      "activations/layer22_attention_weight_min": -20.669511795043945,
      "activations/layer23_attention_weight_max": 22.379802703857422,
      "activations/layer23_attention_weight_min": -15.619781494140625,
      "activations/layer2_attention_weight_max": 27.487510681152344,
      "activations/layer2_attention_weight_min": -25.18301010131836,
      "activations/layer3_attention_weight_max": 58.343143463134766,
      "activations/layer3_attention_weight_min": -62.86089324951172,
      "activations/layer4_attention_weight_max": 86.71057891845703,
      "activations/layer4_attention_weight_min": -77.44105529785156,
      "activations/layer5_attention_weight_max": 127.36514282226562,
      "activations/layer5_attention_weight_min": -111.2518539428711,
      "activations/layer6_attention_weight_max": 62.65445327758789,
      "activations/layer6_attention_weight_min": -56.03929138183594,
      "activations/layer7_attention_weight_max": 58.679969787597656,
      "activations/layer7_attention_weight_min": -55.12718963623047,
      "activations/layer8_attention_weight_max": 50.20314407348633,
      "activations/layer8_attention_weight_min": -43.18165969848633,
      "activations/layer9_attention_weight_max": 72.98267364501953,
      "activations/layer9_attention_weight_min": -66.41494750976562,
      "epoch": 1.7,
      "learning_rate": 0.00014047045454545454,
      "loss": 3.0121,
      "step": 29200
    },
    {
      "activations/layer0_attention_weight_max": 14.626261711120605,
      "activations/layer0_attention_weight_min": -13.768580436706543,
      "activations/layer10_attention_weight_max": 67.87440490722656,
      "activations/layer10_attention_weight_min": -55.78569793701172,
      "activations/layer11_attention_weight_max": 55.202266693115234,
      "activations/layer11_attention_weight_min": -41.09527587890625,
      "activations/layer12_attention_weight_max": 23.57117462158203,
      "activations/layer12_attention_weight_min": -28.026233673095703,
      "activations/layer13_attention_weight_max": 57.88870620727539,
      "activations/layer13_attention_weight_min": -43.67302322387695,
      "activations/layer14_attention_weight_max": 53.67835235595703,
      "activations/layer14_attention_weight_min": -38.675193786621094,
      "activations/layer15_attention_weight_max": 49.55512619018555,
      "activations/layer15_attention_weight_min": -36.95603942871094,
      "activations/layer16_attention_weight_max": 42.239017486572266,
      "activations/layer16_attention_weight_min": -30.39597511291504,
      "activations/layer17_attention_weight_max": 62.17557907104492,
      "activations/layer17_attention_weight_min": -36.4137077331543,
      "activations/layer18_attention_weight_max": 59.7657585144043,
      "activations/layer18_attention_weight_min": -31.444921493530273,
      "activations/layer19_attention_weight_max": 19.8255672454834,
      "activations/layer19_attention_weight_min": -14.631611824035645,
      "activations/layer1_attention_weight_max": 15.901147842407227,
      "activations/layer1_attention_weight_min": -14.071666717529297,
      "activations/layer20_attention_weight_max": 20.472307205200195,
      "activations/layer20_attention_weight_min": -15.84223461151123,
      "activations/layer21_attention_weight_max": 37.5731201171875,
      "activations/layer21_attention_weight_min": -20.114749908447266,
      "activations/layer22_attention_weight_max": 36.40921401977539,
      "activations/layer22_attention_weight_min": -21.903060913085938,
      "activations/layer23_attention_weight_max": 23.14305877685547,
      "activations/layer23_attention_weight_min": -14.054129600524902,
      "activations/layer2_attention_weight_max": 27.546972274780273,
      "activations/layer2_attention_weight_min": -27.202537536621094,
      "activations/layer3_attention_weight_max": 63.433162689208984,
      "activations/layer3_attention_weight_min": -64.67672729492188,
      "activations/layer4_attention_weight_max": 93.0411605834961,
      "activations/layer4_attention_weight_min": -84.16549682617188,
      "activations/layer5_attention_weight_max": 137.96786499023438,
      "activations/layer5_attention_weight_min": -123.10576629638672,
      "activations/layer6_attention_weight_max": 65.05365753173828,
      "activations/layer6_attention_weight_min": -55.22941589355469,
      "activations/layer7_attention_weight_max": 65.58916473388672,
      "activations/layer7_attention_weight_min": -56.01658630371094,
      "activations/layer8_attention_weight_max": 56.36054229736328,
      "activations/layer8_attention_weight_min": -48.14801025390625,
      "activations/layer9_attention_weight_max": 83.1407241821289,
      "activations/layer9_attention_weight_min": -74.71847534179688,
      "epoch": 1.7,
      "learning_rate": 0.00014045151515151516,
      "loss": 2.9784,
      "step": 29250
    },
    {
      "activations/layer0_attention_weight_max": 14.13262939453125,
      "activations/layer0_attention_weight_min": -13.16607666015625,
      "activations/layer10_attention_weight_max": 62.86507034301758,
      "activations/layer10_attention_weight_min": -60.03926467895508,
      "activations/layer11_attention_weight_max": 41.78769302368164,
      "activations/layer11_attention_weight_min": -40.34425735473633,
      "activations/layer12_attention_weight_max": 29.661563873291016,
      "activations/layer12_attention_weight_min": -27.490985870361328,
      "activations/layer13_attention_weight_max": 59.31441116333008,
      "activations/layer13_attention_weight_min": -45.818946838378906,
      "activations/layer14_attention_weight_max": 56.230281829833984,
      "activations/layer14_attention_weight_min": -42.309814453125,
      "activations/layer15_attention_weight_max": 52.83475112915039,
      "activations/layer15_attention_weight_min": -39.369178771972656,
      "activations/layer16_attention_weight_max": 42.17464828491211,
      "activations/layer16_attention_weight_min": -31.653499603271484,
      "activations/layer17_attention_weight_max": 56.28604507446289,
      "activations/layer17_attention_weight_min": -40.19905090332031,
      "activations/layer18_attention_weight_max": 57.4360466003418,
      "activations/layer18_attention_weight_min": -36.42240905761719,
      "activations/layer19_attention_weight_max": 19.67453956604004,
      "activations/layer19_attention_weight_min": -14.714428901672363,
      "activations/layer1_attention_weight_max": 15.532452583312988,
      "activations/layer1_attention_weight_min": -10.989349365234375,
      "activations/layer20_attention_weight_max": 21.590787887573242,
      "activations/layer20_attention_weight_min": -16.642427444458008,
      "activations/layer21_attention_weight_max": 41.58282470703125,
      "activations/layer21_attention_weight_min": -23.107141494750977,
      "activations/layer22_attention_weight_max": 35.47771072387695,
      "activations/layer22_attention_weight_min": -20.900287628173828,
      "activations/layer23_attention_weight_max": 25.94330596923828,
      "activations/layer23_attention_weight_min": -15.043059349060059,
      "activations/layer2_attention_weight_max": 25.63482666015625,
      "activations/layer2_attention_weight_min": -25.39044761657715,
      "activations/layer3_attention_weight_max": 57.535518646240234,
      "activations/layer3_attention_weight_min": -60.47472381591797,
      "activations/layer4_attention_weight_max": 87.49373626708984,
      "activations/layer4_attention_weight_min": -75.37328338623047,
      "activations/layer5_attention_weight_max": 120.67510223388672,
      "activations/layer5_attention_weight_min": -119.03966522216797,
      "activations/layer6_attention_weight_max": 59.64403533935547,
      "activations/layer6_attention_weight_min": -56.75382614135742,
      "activations/layer7_attention_weight_max": 57.349796295166016,
      "activations/layer7_attention_weight_min": -56.543617248535156,
      "activations/layer8_attention_weight_max": 45.78881072998047,
      "activations/layer8_attention_weight_min": -45.28424072265625,
      "activations/layer9_attention_weight_max": 74.3167953491211,
      "activations/layer9_attention_weight_min": -68.62921142578125,
      "epoch": 1.7,
      "learning_rate": 0.00014043257575757574,
      "loss": 2.9966,
      "step": 29300
    },
    {
      "activations/layer0_attention_weight_max": 16.434192657470703,
      "activations/layer0_attention_weight_min": -14.224752426147461,
      "activations/layer10_attention_weight_max": 60.682762145996094,
      "activations/layer10_attention_weight_min": -55.38663101196289,
      "activations/layer11_attention_weight_max": 46.1190185546875,
      "activations/layer11_attention_weight_min": -37.261878967285156,
      "activations/layer12_attention_weight_max": 26.709917068481445,
      "activations/layer12_attention_weight_min": -25.465105056762695,
      "activations/layer13_attention_weight_max": 51.07909393310547,
      "activations/layer13_attention_weight_min": -42.90041732788086,
      "activations/layer14_attention_weight_max": 43.670562744140625,
      "activations/layer14_attention_weight_min": -34.801239013671875,
      "activations/layer15_attention_weight_max": 45.7240104675293,
      "activations/layer15_attention_weight_min": -40.601463317871094,
      "activations/layer16_attention_weight_max": 38.58195114135742,
      "activations/layer16_attention_weight_min": -29.543270111083984,
      "activations/layer17_attention_weight_max": 50.66694641113281,
      "activations/layer17_attention_weight_min": -37.73611068725586,
      "activations/layer18_attention_weight_max": 52.09373474121094,
      "activations/layer18_attention_weight_min": -33.9071044921875,
      "activations/layer19_attention_weight_max": 18.625356674194336,
      "activations/layer19_attention_weight_min": -15.183588981628418,
      "activations/layer1_attention_weight_max": 16.01788330078125,
      "activations/layer1_attention_weight_min": -14.547754287719727,
      "activations/layer20_attention_weight_max": 18.065034866333008,
      "activations/layer20_attention_weight_min": -15.379839897155762,
      "activations/layer21_attention_weight_max": 36.1131477355957,
      "activations/layer21_attention_weight_min": -23.888498306274414,
      "activations/layer22_attention_weight_max": 37.42359924316406,
      "activations/layer22_attention_weight_min": -23.617403030395508,
      "activations/layer23_attention_weight_max": 22.393760681152344,
      "activations/layer23_attention_weight_min": -16.362548828125,
      "activations/layer2_attention_weight_max": 25.981931686401367,
      "activations/layer2_attention_weight_min": -26.103050231933594,
      "activations/layer3_attention_weight_max": 61.168052673339844,
      "activations/layer3_attention_weight_min": -61.91810989379883,
      "activations/layer4_attention_weight_max": 90.10391235351562,
      "activations/layer4_attention_weight_min": -82.8575668334961,
      "activations/layer5_attention_weight_max": 125.80567932128906,
      "activations/layer5_attention_weight_min": -108.05352020263672,
      "activations/layer6_attention_weight_max": 58.657676696777344,
      "activations/layer6_attention_weight_min": -54.53227615356445,
      "activations/layer7_attention_weight_max": 63.004268646240234,
      "activations/layer7_attention_weight_min": -65.22615051269531,
      "activations/layer8_attention_weight_max": 52.21358108520508,
      "activations/layer8_attention_weight_min": -42.54964065551758,
      "activations/layer9_attention_weight_max": 75.63874053955078,
      "activations/layer9_attention_weight_min": -69.1375961303711,
      "epoch": 1.71,
      "learning_rate": 0.00014041363636363636,
      "loss": 3.0014,
      "step": 29350
    },
    {
      "activations/layer0_attention_weight_max": 14.758317947387695,
      "activations/layer0_attention_weight_min": -13.705353736877441,
      "activations/layer10_attention_weight_max": 68.73394012451172,
      "activations/layer10_attention_weight_min": -51.914459228515625,
      "activations/layer11_attention_weight_max": 47.185035705566406,
      "activations/layer11_attention_weight_min": -34.71965408325195,
      "activations/layer12_attention_weight_max": 33.43313217163086,
      "activations/layer12_attention_weight_min": -25.556669235229492,
      "activations/layer13_attention_weight_max": 69.92588806152344,
      "activations/layer13_attention_weight_min": -47.98329544067383,
      "activations/layer14_attention_weight_max": 69.0009536743164,
      "activations/layer14_attention_weight_min": -44.76176834106445,
      "activations/layer15_attention_weight_max": 62.73884582519531,
      "activations/layer15_attention_weight_min": -39.69135284423828,
      "activations/layer16_attention_weight_max": 52.46733093261719,
      "activations/layer16_attention_weight_min": -29.885669708251953,
      "activations/layer17_attention_weight_max": 68.8396224975586,
      "activations/layer17_attention_weight_min": -36.85991287231445,
      "activations/layer18_attention_weight_max": 73.4101791381836,
      "activations/layer18_attention_weight_min": -32.994590759277344,
      "activations/layer19_attention_weight_max": 24.165298461914062,
      "activations/layer19_attention_weight_min": -14.634653091430664,
      "activations/layer1_attention_weight_max": 16.236572265625,
      "activations/layer1_attention_weight_min": -13.8795166015625,
      "activations/layer20_attention_weight_max": 22.900634765625,
      "activations/layer20_attention_weight_min": -17.193944931030273,
      "activations/layer21_attention_weight_max": 45.36913299560547,
      "activations/layer21_attention_weight_min": -24.59039306640625,
      "activations/layer22_attention_weight_max": 39.507591247558594,
      "activations/layer22_attention_weight_min": -23.169254302978516,
      "activations/layer23_attention_weight_max": 29.62557601928711,
      "activations/layer23_attention_weight_min": -14.974387168884277,
      "activations/layer2_attention_weight_max": 26.382795333862305,
      "activations/layer2_attention_weight_min": -25.18733024597168,
      "activations/layer3_attention_weight_max": 57.72247314453125,
      "activations/layer3_attention_weight_min": -62.45814895629883,
      "activations/layer4_attention_weight_max": 86.76630401611328,
      "activations/layer4_attention_weight_min": -79.95336151123047,
      "activations/layer5_attention_weight_max": 132.10360717773438,
      "activations/layer5_attention_weight_min": -109.06326293945312,
      "activations/layer6_attention_weight_max": 63.437347412109375,
      "activations/layer6_attention_weight_min": -56.36223220825195,
      "activations/layer7_attention_weight_max": 62.18714141845703,
      "activations/layer7_attention_weight_min": -54.188297271728516,
      "activations/layer8_attention_weight_max": 52.87239456176758,
      "activations/layer8_attention_weight_min": -43.475948333740234,
      "activations/layer9_attention_weight_max": 63.979827880859375,
      "activations/layer9_attention_weight_min": -61.0877685546875,
      "epoch": 1.71,
      "learning_rate": 0.00014039469696969697,
      "loss": 2.9838,
      "step": 29400
    },
    {
      "activations/layer0_attention_weight_max": 14.628266334533691,
      "activations/layer0_attention_weight_min": -13.981045722961426,
      "activations/layer10_attention_weight_max": 63.23094177246094,
      "activations/layer10_attention_weight_min": -52.48008346557617,
      "activations/layer11_attention_weight_max": 47.34667205810547,
      "activations/layer11_attention_weight_min": -36.831031799316406,
      "activations/layer12_attention_weight_max": 27.85214614868164,
      "activations/layer12_attention_weight_min": -27.25691032409668,
      "activations/layer13_attention_weight_max": 60.700767517089844,
      "activations/layer13_attention_weight_min": -44.0333137512207,
      "activations/layer14_attention_weight_max": 69.50385284423828,
      "activations/layer14_attention_weight_min": -47.33793640136719,
      "activations/layer15_attention_weight_max": 62.38945388793945,
      "activations/layer15_attention_weight_min": -42.9372673034668,
      "activations/layer16_attention_weight_max": 47.12164306640625,
      "activations/layer16_attention_weight_min": -32.86284255981445,
      "activations/layer17_attention_weight_max": 55.844390869140625,
      "activations/layer17_attention_weight_min": -38.055843353271484,
      "activations/layer18_attention_weight_max": 54.70410919189453,
      "activations/layer18_attention_weight_min": -36.41676712036133,
      "activations/layer19_attention_weight_max": 20.845596313476562,
      "activations/layer19_attention_weight_min": -15.103250503540039,
      "activations/layer1_attention_weight_max": 16.142051696777344,
      "activations/layer1_attention_weight_min": -11.926695823669434,
      "activations/layer20_attention_weight_max": 19.017831802368164,
      "activations/layer20_attention_weight_min": -15.645068168640137,
      "activations/layer21_attention_weight_max": 37.120948791503906,
      "activations/layer21_attention_weight_min": -21.845001220703125,
      "activations/layer22_attention_weight_max": 35.69819641113281,
      "activations/layer22_attention_weight_min": -21.800743103027344,
      "activations/layer23_attention_weight_max": 24.023677825927734,
      "activations/layer23_attention_weight_min": -13.445802688598633,
      "activations/layer2_attention_weight_max": 24.944839477539062,
      "activations/layer2_attention_weight_min": -24.307615280151367,
      "activations/layer3_attention_weight_max": 57.98119354248047,
      "activations/layer3_attention_weight_min": -65.78062438964844,
      "activations/layer4_attention_weight_max": 90.59495544433594,
      "activations/layer4_attention_weight_min": -84.53655242919922,
      "activations/layer5_attention_weight_max": 126.0736312866211,
      "activations/layer5_attention_weight_min": -116.24662780761719,
      "activations/layer6_attention_weight_max": 64.12308502197266,
      "activations/layer6_attention_weight_min": -55.59877395629883,
      "activations/layer7_attention_weight_max": 61.022239685058594,
      "activations/layer7_attention_weight_min": -58.99843215942383,
      "activations/layer8_attention_weight_max": 51.77492141723633,
      "activations/layer8_attention_weight_min": -44.92727279663086,
      "activations/layer9_attention_weight_max": 76.17700958251953,
      "activations/layer9_attention_weight_min": -64.68929290771484,
      "epoch": 1.71,
      "learning_rate": 0.00014037613636363636,
      "loss": 3.0084,
      "step": 29450
    },
    {
      "activations/layer0_attention_weight_max": 15.877873420715332,
      "activations/layer0_attention_weight_min": -13.517593383789062,
      "activations/layer10_attention_weight_max": 67.43412017822266,
      "activations/layer10_attention_weight_min": -50.9503173828125,
      "activations/layer11_attention_weight_max": 54.153656005859375,
      "activations/layer11_attention_weight_min": -37.38145065307617,
      "activations/layer12_attention_weight_max": 33.10942840576172,
      "activations/layer12_attention_weight_min": -29.237707138061523,
      "activations/layer13_attention_weight_max": 68.28563690185547,
      "activations/layer13_attention_weight_min": -48.83550262451172,
      "activations/layer14_attention_weight_max": 63.91246795654297,
      "activations/layer14_attention_weight_min": -43.448829650878906,
      "activations/layer15_attention_weight_max": 55.72397994995117,
      "activations/layer15_attention_weight_min": -39.033592224121094,
      "activations/layer16_attention_weight_max": 44.5836067199707,
      "activations/layer16_attention_weight_min": -32.48392105102539,
      "activations/layer17_attention_weight_max": 71.379638671875,
      "activations/layer17_attention_weight_min": -38.92155838012695,
      "activations/layer18_attention_weight_max": 62.18484115600586,
      "activations/layer18_attention_weight_min": -36.49732208251953,
      "activations/layer19_attention_weight_max": 21.330015182495117,
      "activations/layer19_attention_weight_min": -13.370965957641602,
      "activations/layer1_attention_weight_max": 15.414362907409668,
      "activations/layer1_attention_weight_min": -12.415799140930176,
      "activations/layer20_attention_weight_max": 18.244192123413086,
      "activations/layer20_attention_weight_min": -15.538552284240723,
      "activations/layer21_attention_weight_max": 37.558284759521484,
      "activations/layer21_attention_weight_min": -23.001129150390625,
      "activations/layer22_attention_weight_max": 33.431976318359375,
      "activations/layer22_attention_weight_min": -21.882619857788086,
      "activations/layer23_attention_weight_max": 21.091617584228516,
      "activations/layer23_attention_weight_min": -15.148908615112305,
      "activations/layer2_attention_weight_max": 28.469112396240234,
      "activations/layer2_attention_weight_min": -25.600112915039062,
      "activations/layer3_attention_weight_max": 64.19358825683594,
      "activations/layer3_attention_weight_min": -67.01461029052734,
      "activations/layer4_attention_weight_max": 99.14750671386719,
      "activations/layer4_attention_weight_min": -85.65543365478516,
      "activations/layer5_attention_weight_max": 144.0906219482422,
      "activations/layer5_attention_weight_min": -112.29307556152344,
      "activations/layer6_attention_weight_max": 67.68745422363281,
      "activations/layer6_attention_weight_min": -50.37638854980469,
      "activations/layer7_attention_weight_max": 66.33029174804688,
      "activations/layer7_attention_weight_min": -60.09815216064453,
      "activations/layer8_attention_weight_max": 60.52654266357422,
      "activations/layer8_attention_weight_min": -46.09941482543945,
      "activations/layer9_attention_weight_max": 79.49695587158203,
      "activations/layer9_attention_weight_min": -67.70158386230469,
      "epoch": 1.71,
      "learning_rate": 0.00014035719696969695,
      "loss": 2.996,
      "step": 29500
    },
    {
      "activations/layer0_attention_weight_max": 16.355939865112305,
      "activations/layer0_attention_weight_min": -13.67608642578125,
      "activations/layer10_attention_weight_max": 57.6984977722168,
      "activations/layer10_attention_weight_min": -51.45317459106445,
      "activations/layer11_attention_weight_max": 43.294761657714844,
      "activations/layer11_attention_weight_min": -39.18792724609375,
      "activations/layer12_attention_weight_max": 32.251853942871094,
      "activations/layer12_attention_weight_min": -26.070127487182617,
      "activations/layer13_attention_weight_max": 58.21613693237305,
      "activations/layer13_attention_weight_min": -46.34867858886719,
      "activations/layer14_attention_weight_max": 49.69076919555664,
      "activations/layer14_attention_weight_min": -37.344417572021484,
      "activations/layer15_attention_weight_max": 51.8714485168457,
      "activations/layer15_attention_weight_min": -38.465721130371094,
      "activations/layer16_attention_weight_max": 43.15538787841797,
      "activations/layer16_attention_weight_min": -31.902225494384766,
      "activations/layer17_attention_weight_max": 64.7311019897461,
      "activations/layer17_attention_weight_min": -36.85802459716797,
      "activations/layer18_attention_weight_max": 66.47190856933594,
      "activations/layer18_attention_weight_min": -33.03608322143555,
      "activations/layer19_attention_weight_max": 21.073396682739258,
      "activations/layer19_attention_weight_min": -14.887249946594238,
      "activations/layer1_attention_weight_max": 16.481021881103516,
      "activations/layer1_attention_weight_min": -14.100255012512207,
      "activations/layer20_attention_weight_max": 21.906877517700195,
      "activations/layer20_attention_weight_min": -17.731407165527344,
      "activations/layer21_attention_weight_max": 44.91093444824219,
      "activations/layer21_attention_weight_min": -22.104639053344727,
      "activations/layer22_attention_weight_max": 42.48433303833008,
      "activations/layer22_attention_weight_min": -21.939620971679688,
      "activations/layer23_attention_weight_max": 27.660297393798828,
      "activations/layer23_attention_weight_min": -15.628043174743652,
      "activations/layer2_attention_weight_max": 25.529808044433594,
      "activations/layer2_attention_weight_min": -25.84944725036621,
      "activations/layer3_attention_weight_max": 61.52098846435547,
      "activations/layer3_attention_weight_min": -63.50216293334961,
      "activations/layer4_attention_weight_max": 88.87084197998047,
      "activations/layer4_attention_weight_min": -84.6197738647461,
      "activations/layer5_attention_weight_max": 125.63275146484375,
      "activations/layer5_attention_weight_min": -119.18950653076172,
      "activations/layer6_attention_weight_max": 62.220680236816406,
      "activations/layer6_attention_weight_min": -55.21764373779297,
      "activations/layer7_attention_weight_max": 58.738704681396484,
      "activations/layer7_attention_weight_min": -60.27475357055664,
      "activations/layer8_attention_weight_max": 49.060489654541016,
      "activations/layer8_attention_weight_min": -45.16261672973633,
      "activations/layer9_attention_weight_max": 64.66271209716797,
      "activations/layer9_attention_weight_min": -59.397491455078125,
      "epoch": 1.72,
      "learning_rate": 0.00014033825757575757,
      "loss": 2.9999,
      "step": 29550
    },
    {
      "activations/layer0_attention_weight_max": 15.162711143493652,
      "activations/layer0_attention_weight_min": -13.956439971923828,
      "activations/layer10_attention_weight_max": 61.69091033935547,
      "activations/layer10_attention_weight_min": -50.98228073120117,
      "activations/layer11_attention_weight_max": 46.55244064331055,
      "activations/layer11_attention_weight_min": -33.49139404296875,
      "activations/layer12_attention_weight_max": 30.53067398071289,
      "activations/layer12_attention_weight_min": -25.644123077392578,
      "activations/layer13_attention_weight_max": 54.75446319580078,
      "activations/layer13_attention_weight_min": -48.5889778137207,
      "activations/layer14_attention_weight_max": 54.456729888916016,
      "activations/layer14_attention_weight_min": -39.193214416503906,
      "activations/layer15_attention_weight_max": 55.17902374267578,
      "activations/layer15_attention_weight_min": -40.456626892089844,
      "activations/layer16_attention_weight_max": 46.38254165649414,
      "activations/layer16_attention_weight_min": -31.899442672729492,
      "activations/layer17_attention_weight_max": 63.61769485473633,
      "activations/layer17_attention_weight_min": -37.8194465637207,
      "activations/layer18_attention_weight_max": 69.0015640258789,
      "activations/layer18_attention_weight_min": -37.22105407714844,
      "activations/layer19_attention_weight_max": 22.240724563598633,
      "activations/layer19_attention_weight_min": -14.921916007995605,
      "activations/layer1_attention_weight_max": 15.774931907653809,
      "activations/layer1_attention_weight_min": -12.588850975036621,
      "activations/layer20_attention_weight_max": 19.504117965698242,
      "activations/layer20_attention_weight_min": -14.619117736816406,
      "activations/layer21_attention_weight_max": 38.120784759521484,
      "activations/layer21_attention_weight_min": -22.936843872070312,
      "activations/layer22_attention_weight_max": 35.345848083496094,
      "activations/layer22_attention_weight_min": -21.975536346435547,
      "activations/layer23_attention_weight_max": 26.715131759643555,
      "activations/layer23_attention_weight_min": -14.917303085327148,
      "activations/layer2_attention_weight_max": 26.477951049804688,
      "activations/layer2_attention_weight_min": -25.02313232421875,
      "activations/layer3_attention_weight_max": 60.0938835144043,
      "activations/layer3_attention_weight_min": -64.20232391357422,
      "activations/layer4_attention_weight_max": 90.74364471435547,
      "activations/layer4_attention_weight_min": -79.69634246826172,
      "activations/layer5_attention_weight_max": 130.43621826171875,
      "activations/layer5_attention_weight_min": -112.05392456054688,
      "activations/layer6_attention_weight_max": 66.18750762939453,
      "activations/layer6_attention_weight_min": -53.96168899536133,
      "activations/layer7_attention_weight_max": 60.9875602722168,
      "activations/layer7_attention_weight_min": -52.949275970458984,
      "activations/layer8_attention_weight_max": 45.7580451965332,
      "activations/layer8_attention_weight_min": -40.859169006347656,
      "activations/layer9_attention_weight_max": 77.27312469482422,
      "activations/layer9_attention_weight_min": -61.22166061401367,
      "epoch": 1.72,
      "learning_rate": 0.00014031931818181818,
      "loss": 2.9898,
      "step": 29600
    },
    {
      "activations/layer0_attention_weight_max": 13.457525253295898,
      "activations/layer0_attention_weight_min": -13.19502067565918,
      "activations/layer10_attention_weight_max": 70.32097625732422,
      "activations/layer10_attention_weight_min": -53.227474212646484,
      "activations/layer11_attention_weight_max": 45.329078674316406,
      "activations/layer11_attention_weight_min": -37.084068298339844,
      "activations/layer12_attention_weight_max": 26.219831466674805,
      "activations/layer12_attention_weight_min": -25.41002655029297,
      "activations/layer13_attention_weight_max": 71.34178161621094,
      "activations/layer13_attention_weight_min": -41.343772888183594,
      "activations/layer14_attention_weight_max": 60.59292221069336,
      "activations/layer14_attention_weight_min": -36.86227035522461,
      "activations/layer15_attention_weight_max": 68.6578140258789,
      "activations/layer15_attention_weight_min": -39.53647994995117,
      "activations/layer16_attention_weight_max": 53.69438552856445,
      "activations/layer16_attention_weight_min": -29.80009651184082,
      "activations/layer17_attention_weight_max": 67.1977310180664,
      "activations/layer17_attention_weight_min": -36.49773025512695,
      "activations/layer18_attention_weight_max": 56.6302375793457,
      "activations/layer18_attention_weight_min": -32.490386962890625,
      "activations/layer19_attention_weight_max": 18.646989822387695,
      "activations/layer19_attention_weight_min": -14.857240676879883,
      "activations/layer1_attention_weight_max": 15.660118103027344,
      "activations/layer1_attention_weight_min": -12.581762313842773,
      "activations/layer20_attention_weight_max": 21.025854110717773,
      "activations/layer20_attention_weight_min": -15.216791152954102,
      "activations/layer21_attention_weight_max": 37.75129699707031,
      "activations/layer21_attention_weight_min": -22.805221557617188,
      "activations/layer22_attention_weight_max": 41.25886917114258,
      "activations/layer22_attention_weight_min": -23.32868003845215,
      "activations/layer23_attention_weight_max": 22.995624542236328,
      "activations/layer23_attention_weight_min": -15.586562156677246,
      "activations/layer2_attention_weight_max": 27.740453720092773,
      "activations/layer2_attention_weight_min": -27.45015525817871,
      "activations/layer3_attention_weight_max": 57.98719024658203,
      "activations/layer3_attention_weight_min": -61.863136291503906,
      "activations/layer4_attention_weight_max": 87.61971282958984,
      "activations/layer4_attention_weight_min": -77.78951263427734,
      "activations/layer5_attention_weight_max": 127.22109985351562,
      "activations/layer5_attention_weight_min": -113.04756927490234,
      "activations/layer6_attention_weight_max": 59.33422088623047,
      "activations/layer6_attention_weight_min": -53.925052642822266,
      "activations/layer7_attention_weight_max": 62.68117141723633,
      "activations/layer7_attention_weight_min": -52.7086296081543,
      "activations/layer8_attention_weight_max": 52.175907135009766,
      "activations/layer8_attention_weight_min": -44.11642837524414,
      "activations/layer9_attention_weight_max": 70.27995300292969,
      "activations/layer9_attention_weight_min": -61.15414047241211,
      "epoch": 1.72,
      "learning_rate": 0.00014030037878787877,
      "loss": 3.0025,
      "step": 29650
    },
    {
      "activations/layer0_attention_weight_max": 16.01729393005371,
      "activations/layer0_attention_weight_min": -14.083410263061523,
      "activations/layer10_attention_weight_max": 64.4704818725586,
      "activations/layer10_attention_weight_min": -56.246368408203125,
      "activations/layer11_attention_weight_max": 44.85940170288086,
      "activations/layer11_attention_weight_min": -38.5405387878418,
      "activations/layer12_attention_weight_max": 27.44801139831543,
      "activations/layer12_attention_weight_min": -24.68364143371582,
      "activations/layer13_attention_weight_max": 67.24767303466797,
      "activations/layer13_attention_weight_min": -48.12690353393555,
      "activations/layer14_attention_weight_max": 55.318023681640625,
      "activations/layer14_attention_weight_min": -38.938201904296875,
      "activations/layer15_attention_weight_max": 53.79697799682617,
      "activations/layer15_attention_weight_min": -40.63548278808594,
      "activations/layer16_attention_weight_max": 44.99916076660156,
      "activations/layer16_attention_weight_min": -28.456926345825195,
      "activations/layer17_attention_weight_max": 58.9669189453125,
      "activations/layer17_attention_weight_min": -34.711456298828125,
      "activations/layer18_attention_weight_max": 59.91865539550781,
      "activations/layer18_attention_weight_min": -34.01588439941406,
      "activations/layer19_attention_weight_max": 19.0578670501709,
      "activations/layer19_attention_weight_min": -15.60349178314209,
      "activations/layer1_attention_weight_max": 17.363725662231445,
      "activations/layer1_attention_weight_min": -14.361410140991211,
      "activations/layer20_attention_weight_max": 18.370582580566406,
      "activations/layer20_attention_weight_min": -14.369600296020508,
      "activations/layer21_attention_weight_max": 36.2228889465332,
      "activations/layer21_attention_weight_min": -23.640087127685547,
      "activations/layer22_attention_weight_max": 36.88336944580078,
      "activations/layer22_attention_weight_min": -21.279253005981445,
      "activations/layer23_attention_weight_max": 21.948726654052734,
      "activations/layer23_attention_weight_min": -15.546286582946777,
      "activations/layer2_attention_weight_max": 25.051786422729492,
      "activations/layer2_attention_weight_min": -26.249448776245117,
      "activations/layer3_attention_weight_max": 61.10819625854492,
      "activations/layer3_attention_weight_min": -64.19611358642578,
      "activations/layer4_attention_weight_max": 94.29796600341797,
      "activations/layer4_attention_weight_min": -84.94945526123047,
      "activations/layer5_attention_weight_max": 130.73626708984375,
      "activations/layer5_attention_weight_min": -111.99209594726562,
      "activations/layer6_attention_weight_max": 66.70045471191406,
      "activations/layer6_attention_weight_min": -57.92338562011719,
      "activations/layer7_attention_weight_max": 61.696205139160156,
      "activations/layer7_attention_weight_min": -54.94889831542969,
      "activations/layer8_attention_weight_max": 55.326847076416016,
      "activations/layer8_attention_weight_min": -42.773685455322266,
      "activations/layer9_attention_weight_max": 89.53128814697266,
      "activations/layer9_attention_weight_min": -68.70271301269531,
      "epoch": 1.73,
      "learning_rate": 0.00014028143939393939,
      "loss": 2.9877,
      "step": 29700
    },
    {
      "activations/layer0_attention_weight_max": 13.450332641601562,
      "activations/layer0_attention_weight_min": -13.124738693237305,
      "activations/layer10_attention_weight_max": 60.84659957885742,
      "activations/layer10_attention_weight_min": -51.772972106933594,
      "activations/layer11_attention_weight_max": 40.56342315673828,
      "activations/layer11_attention_weight_min": -33.432491302490234,
      "activations/layer12_attention_weight_max": 31.067848205566406,
      "activations/layer12_attention_weight_min": -25.68684196472168,
      "activations/layer13_attention_weight_max": 58.54217529296875,
      "activations/layer13_attention_weight_min": -41.887306213378906,
      "activations/layer14_attention_weight_max": 55.021881103515625,
      "activations/layer14_attention_weight_min": -41.84495544433594,
      "activations/layer15_attention_weight_max": 56.80390548706055,
      "activations/layer15_attention_weight_min": -36.595863342285156,
      "activations/layer16_attention_weight_max": 44.746009826660156,
      "activations/layer16_attention_weight_min": -31.177310943603516,
      "activations/layer17_attention_weight_max": 58.02886962890625,
      "activations/layer17_attention_weight_min": -39.19508743286133,
      "activations/layer18_attention_weight_max": 61.685096740722656,
      "activations/layer18_attention_weight_min": -36.73906326293945,
      "activations/layer19_attention_weight_max": 22.58715057373047,
      "activations/layer19_attention_weight_min": -14.4464693069458,
      "activations/layer1_attention_weight_max": 16.02178955078125,
      "activations/layer1_attention_weight_min": -12.86312198638916,
      "activations/layer20_attention_weight_max": 18.88117027282715,
      "activations/layer20_attention_weight_min": -14.972062110900879,
      "activations/layer21_attention_weight_max": 40.22359085083008,
      "activations/layer21_attention_weight_min": -23.273422241210938,
      "activations/layer22_attention_weight_max": 37.52812576293945,
      "activations/layer22_attention_weight_min": -23.302337646484375,
      "activations/layer23_attention_weight_max": 22.327709197998047,
      "activations/layer23_attention_weight_min": -13.122669219970703,
      "activations/layer2_attention_weight_max": 27.614177703857422,
      "activations/layer2_attention_weight_min": -27.113353729248047,
      "activations/layer3_attention_weight_max": 62.725921630859375,
      "activations/layer3_attention_weight_min": -67.47158813476562,
      "activations/layer4_attention_weight_max": 90.54336547851562,
      "activations/layer4_attention_weight_min": -85.49768829345703,
      "activations/layer5_attention_weight_max": 126.0517578125,
      "activations/layer5_attention_weight_min": -111.0821533203125,
      "activations/layer6_attention_weight_max": 61.55253982543945,
      "activations/layer6_attention_weight_min": -53.019691467285156,
      "activations/layer7_attention_weight_max": 57.63251876831055,
      "activations/layer7_attention_weight_min": -53.07707595825195,
      "activations/layer8_attention_weight_max": 46.58491897583008,
      "activations/layer8_attention_weight_min": -43.04909896850586,
      "activations/layer9_attention_weight_max": 62.76870346069336,
      "activations/layer9_attention_weight_min": -59.5565185546875,
      "epoch": 1.73,
      "learning_rate": 0.0001402625,
      "loss": 2.998,
      "step": 29750
    },
    {
      "activations/layer0_attention_weight_max": 15.604347229003906,
      "activations/layer0_attention_weight_min": -13.302882194519043,
      "activations/layer10_attention_weight_max": 65.35363006591797,
      "activations/layer10_attention_weight_min": -53.693382263183594,
      "activations/layer11_attention_weight_max": 43.86235046386719,
      "activations/layer11_attention_weight_min": -36.321990966796875,
      "activations/layer12_attention_weight_max": 26.153175354003906,
      "activations/layer12_attention_weight_min": -23.576290130615234,
      "activations/layer13_attention_weight_max": 56.23548126220703,
      "activations/layer13_attention_weight_min": -40.50325393676758,
      "activations/layer14_attention_weight_max": 51.71648406982422,
      "activations/layer14_attention_weight_min": -36.691184997558594,
      "activations/layer15_attention_weight_max": 52.7729606628418,
      "activations/layer15_attention_weight_min": -36.426231384277344,
      "activations/layer16_attention_weight_max": 48.36954879760742,
      "activations/layer16_attention_weight_min": -31.345836639404297,
      "activations/layer17_attention_weight_max": 59.89259719848633,
      "activations/layer17_attention_weight_min": -40.52596664428711,
      "activations/layer18_attention_weight_max": 65.94441986083984,
      "activations/layer18_attention_weight_min": -36.819610595703125,
      "activations/layer19_attention_weight_max": 21.10883903503418,
      "activations/layer19_attention_weight_min": -15.406329154968262,
      "activations/layer1_attention_weight_max": 15.14246654510498,
      "activations/layer1_attention_weight_min": -14.956632614135742,
      "activations/layer20_attention_weight_max": 23.779495239257812,
      "activations/layer20_attention_weight_min": -16.586368560791016,
      "activations/layer21_attention_weight_max": 40.78074264526367,
      "activations/layer21_attention_weight_min": -25.304380416870117,
      "activations/layer22_attention_weight_max": 43.475120544433594,
      "activations/layer22_attention_weight_min": -25.8859806060791,
      "activations/layer23_attention_weight_max": 24.683609008789062,
      "activations/layer23_attention_weight_min": -16.083606719970703,
      "activations/layer2_attention_weight_max": 27.028799057006836,
      "activations/layer2_attention_weight_min": -26.09046173095703,
      "activations/layer3_attention_weight_max": 59.636844635009766,
      "activations/layer3_attention_weight_min": -62.39077377319336,
      "activations/layer4_attention_weight_max": 90.80569458007812,
      "activations/layer4_attention_weight_min": -80.96321868896484,
      "activations/layer5_attention_weight_max": 134.8841552734375,
      "activations/layer5_attention_weight_min": -109.3750228881836,
      "activations/layer6_attention_weight_max": 60.30419921875,
      "activations/layer6_attention_weight_min": -52.4409065246582,
      "activations/layer7_attention_weight_max": 62.24177551269531,
      "activations/layer7_attention_weight_min": -57.76557922363281,
      "activations/layer8_attention_weight_max": 53.620147705078125,
      "activations/layer8_attention_weight_min": -43.57624816894531,
      "activations/layer9_attention_weight_max": 69.94120025634766,
      "activations/layer9_attention_weight_min": -60.53135299682617,
      "epoch": 1.73,
      "learning_rate": 0.0001402435606060606,
      "loss": 2.999,
      "step": 29800
    },
    {
      "activations/layer0_attention_weight_max": 13.456844329833984,
      "activations/layer0_attention_weight_min": -14.039765357971191,
      "activations/layer10_attention_weight_max": 64.54801177978516,
      "activations/layer10_attention_weight_min": -51.52827835083008,
      "activations/layer11_attention_weight_max": 43.257720947265625,
      "activations/layer11_attention_weight_min": -35.96284484863281,
      "activations/layer12_attention_weight_max": 32.3431510925293,
      "activations/layer12_attention_weight_min": -28.218067169189453,
      "activations/layer13_attention_weight_max": 63.330116271972656,
      "activations/layer13_attention_weight_min": -44.857059478759766,
      "activations/layer14_attention_weight_max": 62.21764373779297,
      "activations/layer14_attention_weight_min": -41.09191131591797,
      "activations/layer15_attention_weight_max": 49.16353988647461,
      "activations/layer15_attention_weight_min": -37.72066116333008,
      "activations/layer16_attention_weight_max": 43.20607376098633,
      "activations/layer16_attention_weight_min": -30.485090255737305,
      "activations/layer17_attention_weight_max": 50.129852294921875,
      "activations/layer17_attention_weight_min": -39.8824462890625,
      "activations/layer18_attention_weight_max": 54.64670181274414,
      "activations/layer18_attention_weight_min": -34.1127815246582,
      "activations/layer19_attention_weight_max": 19.90595817565918,
      "activations/layer19_attention_weight_min": -15.023600578308105,
      "activations/layer1_attention_weight_max": 17.410566329956055,
      "activations/layer1_attention_weight_min": -11.878692626953125,
      "activations/layer20_attention_weight_max": 20.09090805053711,
      "activations/layer20_attention_weight_min": -15.349101066589355,
      "activations/layer21_attention_weight_max": 38.638336181640625,
      "activations/layer21_attention_weight_min": -24.32448387145996,
      "activations/layer22_attention_weight_max": 34.16968536376953,
      "activations/layer22_attention_weight_min": -21.54206085205078,
      "activations/layer23_attention_weight_max": 20.99237823486328,
      "activations/layer23_attention_weight_min": -13.465707778930664,
      "activations/layer2_attention_weight_max": 23.78144645690918,
      "activations/layer2_attention_weight_min": -23.713232040405273,
      "activations/layer3_attention_weight_max": 54.246219635009766,
      "activations/layer3_attention_weight_min": -58.174808502197266,
      "activations/layer4_attention_weight_max": 84.45469665527344,
      "activations/layer4_attention_weight_min": -80.0645523071289,
      "activations/layer5_attention_weight_max": 120.3814697265625,
      "activations/layer5_attention_weight_min": -116.94786071777344,
      "activations/layer6_attention_weight_max": 59.760318756103516,
      "activations/layer6_attention_weight_min": -58.47861099243164,
      "activations/layer7_attention_weight_max": 57.833003997802734,
      "activations/layer7_attention_weight_min": -57.6243896484375,
      "activations/layer8_attention_weight_max": 47.96752166748047,
      "activations/layer8_attention_weight_min": -44.874786376953125,
      "activations/layer9_attention_weight_max": 66.5875473022461,
      "activations/layer9_attention_weight_min": -66.43624877929688,
      "epoch": 1.73,
      "learning_rate": 0.0001402246212121212,
      "loss": 2.995,
      "step": 29850
    },
    {
      "activations/layer0_attention_weight_max": 13.699687957763672,
      "activations/layer0_attention_weight_min": -13.759376525878906,
      "activations/layer10_attention_weight_max": 65.43531036376953,
      "activations/layer10_attention_weight_min": -51.52345657348633,
      "activations/layer11_attention_weight_max": 43.44033432006836,
      "activations/layer11_attention_weight_min": -40.562843322753906,
      "activations/layer12_attention_weight_max": 28.70650863647461,
      "activations/layer12_attention_weight_min": -27.815763473510742,
      "activations/layer13_attention_weight_max": 54.14973068237305,
      "activations/layer13_attention_weight_min": -41.28181838989258,
      "activations/layer14_attention_weight_max": 42.00423049926758,
      "activations/layer14_attention_weight_min": -33.75157928466797,
      "activations/layer15_attention_weight_max": 59.281307220458984,
      "activations/layer15_attention_weight_min": -38.309505462646484,
      "activations/layer16_attention_weight_max": 40.554630279541016,
      "activations/layer16_attention_weight_min": -29.249975204467773,
      "activations/layer17_attention_weight_max": 60.72055435180664,
      "activations/layer17_attention_weight_min": -41.42975616455078,
      "activations/layer18_attention_weight_max": 58.409488677978516,
      "activations/layer18_attention_weight_min": -32.76396560668945,
      "activations/layer19_attention_weight_max": 21.171247482299805,
      "activations/layer19_attention_weight_min": -14.684602737426758,
      "activations/layer1_attention_weight_max": 15.979585647583008,
      "activations/layer1_attention_weight_min": -13.804865837097168,
      "activations/layer20_attention_weight_max": 19.10150146484375,
      "activations/layer20_attention_weight_min": -16.057817459106445,
      "activations/layer21_attention_weight_max": 37.074729919433594,
      "activations/layer21_attention_weight_min": -21.811079025268555,
      "activations/layer22_attention_weight_max": 36.38471984863281,
      "activations/layer22_attention_weight_min": -21.5025577545166,
      "activations/layer23_attention_weight_max": 22.27686882019043,
      "activations/layer23_attention_weight_min": -15.386337280273438,
      "activations/layer2_attention_weight_max": 26.657150268554688,
      "activations/layer2_attention_weight_min": -25.150747299194336,
      "activations/layer3_attention_weight_max": 63.02939224243164,
      "activations/layer3_attention_weight_min": -66.13675689697266,
      "activations/layer4_attention_weight_max": 93.0842514038086,
      "activations/layer4_attention_weight_min": -88.25554656982422,
      "activations/layer5_attention_weight_max": 128.40557861328125,
      "activations/layer5_attention_weight_min": -111.25634002685547,
      "activations/layer6_attention_weight_max": 61.80940628051758,
      "activations/layer6_attention_weight_min": -54.14620590209961,
      "activations/layer7_attention_weight_max": 58.21668243408203,
      "activations/layer7_attention_weight_min": -52.31412887573242,
      "activations/layer8_attention_weight_max": 46.96894073486328,
      "activations/layer8_attention_weight_min": -43.33851623535156,
      "activations/layer9_attention_weight_max": 64.83354949951172,
      "activations/layer9_attention_weight_min": -57.39030838012695,
      "epoch": 1.74,
      "learning_rate": 0.0001402056818181818,
      "loss": 2.9724,
      "step": 29900
    },
    {
      "activations/layer0_attention_weight_max": 16.72372817993164,
      "activations/layer0_attention_weight_min": -14.06020450592041,
      "activations/layer10_attention_weight_max": 64.41983795166016,
      "activations/layer10_attention_weight_min": -52.275482177734375,
      "activations/layer11_attention_weight_max": 51.47227478027344,
      "activations/layer11_attention_weight_min": -41.30535125732422,
      "activations/layer12_attention_weight_max": 28.60797691345215,
      "activations/layer12_attention_weight_min": -29.19025993347168,
      "activations/layer13_attention_weight_max": 57.24388122558594,
      "activations/layer13_attention_weight_min": -48.66975021362305,
      "activations/layer14_attention_weight_max": 44.0883674621582,
      "activations/layer14_attention_weight_min": -35.35859680175781,
      "activations/layer15_attention_weight_max": 49.23088455200195,
      "activations/layer15_attention_weight_min": -37.87620544433594,
      "activations/layer16_attention_weight_max": 38.241268157958984,
      "activations/layer16_attention_weight_min": -30.60753631591797,
      "activations/layer17_attention_weight_max": 51.88788986206055,
      "activations/layer17_attention_weight_min": -34.34102249145508,
      "activations/layer18_attention_weight_max": 54.747406005859375,
      "activations/layer18_attention_weight_min": -36.11501693725586,
      "activations/layer19_attention_weight_max": 22.039289474487305,
      "activations/layer19_attention_weight_min": -15.46821117401123,
      "activations/layer1_attention_weight_max": 16.266807556152344,
      "activations/layer1_attention_weight_min": -13.740687370300293,
      "activations/layer20_attention_weight_max": 20.48267936706543,
      "activations/layer20_attention_weight_min": -15.97766399383545,
      "activations/layer21_attention_weight_max": 38.15011978149414,
      "activations/layer21_attention_weight_min": -22.349124908447266,
      "activations/layer22_attention_weight_max": 38.32344436645508,
      "activations/layer22_attention_weight_min": -21.32965087890625,
      "activations/layer23_attention_weight_max": 23.083263397216797,
      "activations/layer23_attention_weight_min": -15.934663772583008,
      "activations/layer2_attention_weight_max": 25.21135711669922,
      "activations/layer2_attention_weight_min": -24.413043975830078,
      "activations/layer3_attention_weight_max": 63.079322814941406,
      "activations/layer3_attention_weight_min": -65.22532653808594,
      "activations/layer4_attention_weight_max": 89.67814636230469,
      "activations/layer4_attention_weight_min": -78.3079833984375,
      "activations/layer5_attention_weight_max": 124.01560974121094,
      "activations/layer5_attention_weight_min": -116.56206512451172,
      "activations/layer6_attention_weight_max": 60.71136474609375,
      "activations/layer6_attention_weight_min": -52.42085647583008,
      "activations/layer7_attention_weight_max": 68.59556579589844,
      "activations/layer7_attention_weight_min": -58.17478942871094,
      "activations/layer8_attention_weight_max": 59.70475387573242,
      "activations/layer8_attention_weight_min": -45.25333023071289,
      "activations/layer9_attention_weight_max": 84.2914047241211,
      "activations/layer9_attention_weight_min": -75.05529022216797,
      "epoch": 1.74,
      "learning_rate": 0.0001401867424242424,
      "loss": 2.9997,
      "step": 29950
    },
    {
      "activations/layer0_attention_weight_max": 14.088912010192871,
      "activations/layer0_attention_weight_min": -13.707048416137695,
      "activations/layer10_attention_weight_max": 60.19261932373047,
      "activations/layer10_attention_weight_min": -47.365169525146484,
      "activations/layer11_attention_weight_max": 41.683372497558594,
      "activations/layer11_attention_weight_min": -36.5610466003418,
      "activations/layer12_attention_weight_max": 28.86671257019043,
      "activations/layer12_attention_weight_min": -22.421859741210938,
      "activations/layer13_attention_weight_max": 60.85401916503906,
      "activations/layer13_attention_weight_min": -43.355552673339844,
      "activations/layer14_attention_weight_max": 56.17531967163086,
      "activations/layer14_attention_weight_min": -37.659297943115234,
      "activations/layer15_attention_weight_max": 53.9464225769043,
      "activations/layer15_attention_weight_min": -38.87028121948242,
      "activations/layer16_attention_weight_max": 50.40097427368164,
      "activations/layer16_attention_weight_min": -33.251468658447266,
      "activations/layer17_attention_weight_max": 55.75859451293945,
      "activations/layer17_attention_weight_min": -39.65559387207031,
      "activations/layer18_attention_weight_max": 57.805110931396484,
      "activations/layer18_attention_weight_min": -36.772918701171875,
      "activations/layer19_attention_weight_max": 20.366018295288086,
      "activations/layer19_attention_weight_min": -15.866923332214355,
      "activations/layer1_attention_weight_max": 15.983331680297852,
      "activations/layer1_attention_weight_min": -13.22527027130127,
      "activations/layer20_attention_weight_max": 17.90751075744629,
      "activations/layer20_attention_weight_min": -16.246057510375977,
      "activations/layer21_attention_weight_max": 39.47489929199219,
      "activations/layer21_attention_weight_min": -21.50120735168457,
      "activations/layer22_attention_weight_max": 46.32977294921875,
      "activations/layer22_attention_weight_min": -22.428749084472656,
      "activations/layer23_attention_weight_max": 24.086776733398438,
      "activations/layer23_attention_weight_min": -14.73475170135498,
      "activations/layer2_attention_weight_max": 25.734230041503906,
      "activations/layer2_attention_weight_min": -25.55127716064453,
      "activations/layer3_attention_weight_max": 58.045413970947266,
      "activations/layer3_attention_weight_min": -59.82799530029297,
      "activations/layer4_attention_weight_max": 88.73702239990234,
      "activations/layer4_attention_weight_min": -81.22106170654297,
      "activations/layer5_attention_weight_max": 123.74783325195312,
      "activations/layer5_attention_weight_min": -121.84011840820312,
      "activations/layer6_attention_weight_max": 63.03315734863281,
      "activations/layer6_attention_weight_min": -55.65801239013672,
      "activations/layer7_attention_weight_max": 61.003990173339844,
      "activations/layer7_attention_weight_min": -50.00410842895508,
      "activations/layer8_attention_weight_max": 52.30550765991211,
      "activations/layer8_attention_weight_min": -41.414634704589844,
      "activations/layer9_attention_weight_max": 68.2372817993164,
      "activations/layer9_attention_weight_min": -56.572608947753906,
      "epoch": 1.74,
      "learning_rate": 0.00014016780303030302,
      "loss": 2.9651,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_loss": 2.9375,
      "eval_runtime": 8.4468,
      "eval_samples_per_second": 508.361,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_openwebtext_loss": 2.9375,
      "eval_openwebtext_ppl": 18.868615759264884,
      "eval_openwebtext_runtime": 8.4468,
      "eval_openwebtext_samples_per_second": 508.361,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_wikitext_loss": 3.21484375,
      "eval_wikitext_ppl": 24.89940107578365,
      "eval_wikitext_runtime": 1.9462,
      "eval_wikitext_samples_per_second": 234.306,
      "step": 30000
    },
    {
      "epoch": 1.74,
      "eval_lambada_loss": 3.31640625,
      "eval_lambada_ppl": 27.561124577174567,
      "eval_lambada_runtime": 9.5402,
      "eval_lambada_samples_per_second": 510.366,
      "step": 30000
    },
    {
      "activations/layer0_attention_weight_max": 13.481343269348145,
      "activations/layer0_attention_weight_min": -14.052239418029785,
      "activations/layer10_attention_weight_max": 58.6191520690918,
      "activations/layer10_attention_weight_min": -48.023468017578125,
      "activations/layer11_attention_weight_max": 41.7177734375,
      "activations/layer11_attention_weight_min": -34.256378173828125,
      "activations/layer12_attention_weight_max": 30.9081974029541,
      "activations/layer12_attention_weight_min": -25.86695098876953,
      "activations/layer13_attention_weight_max": 58.49668502807617,
      "activations/layer13_attention_weight_min": -44.010677337646484,
      "activations/layer14_attention_weight_max": 71.47552490234375,
      "activations/layer14_attention_weight_min": -44.94904327392578,
      "activations/layer15_attention_weight_max": 51.52308654785156,
      "activations/layer15_attention_weight_min": -38.077980041503906,
      "activations/layer16_attention_weight_max": 48.34684753417969,
      "activations/layer16_attention_weight_min": -30.006372451782227,
      "activations/layer17_attention_weight_max": 64.36798858642578,
      "activations/layer17_attention_weight_min": -35.510826110839844,
      "activations/layer18_attention_weight_max": 63.77834701538086,
      "activations/layer18_attention_weight_min": -32.49737548828125,
      "activations/layer19_attention_weight_max": 20.691022872924805,
      "activations/layer19_attention_weight_min": -16.257972717285156,
      "activations/layer1_attention_weight_max": 15.233274459838867,
      "activations/layer1_attention_weight_min": -13.417357444763184,
      "activations/layer20_attention_weight_max": 20.36302947998047,
      "activations/layer20_attention_weight_min": -14.889433860778809,
      "activations/layer21_attention_weight_max": 44.45893096923828,
      "activations/layer21_attention_weight_min": -24.871013641357422,
      "activations/layer22_attention_weight_max": 41.44233703613281,
      "activations/layer22_attention_weight_min": -22.718368530273438,
      "activations/layer23_attention_weight_max": 24.839229583740234,
      "activations/layer23_attention_weight_min": -14.477506637573242,
      "activations/layer2_attention_weight_max": 25.538982391357422,
      "activations/layer2_attention_weight_min": -25.311927795410156,
      "activations/layer3_attention_weight_max": 57.72804641723633,
      "activations/layer3_attention_weight_min": -63.17145538330078,
      "activations/layer4_attention_weight_max": 86.66233825683594,
      "activations/layer4_attention_weight_min": -76.59561920166016,
      "activations/layer5_attention_weight_max": 122.14134216308594,
      "activations/layer5_attention_weight_min": -112.31320190429688,
      "activations/layer6_attention_weight_max": 58.03214645385742,
      "activations/layer6_attention_weight_min": -52.9492301940918,
      "activations/layer7_attention_weight_max": 58.06577682495117,
      "activations/layer7_attention_weight_min": -55.85337829589844,
      "activations/layer8_attention_weight_max": 49.018733978271484,
      "activations/layer8_attention_weight_min": -47.52848815917969,
      "activations/layer9_attention_weight_max": 69.07072448730469,
      "activations/layer9_attention_weight_min": -65.4223403930664,
      "epoch": 1.75,
      "learning_rate": 0.0001401488636363636,
      "loss": 2.9915,
      "step": 30050
    },
    {
      "activations/layer0_attention_weight_max": 16.8643798828125,
      "activations/layer0_attention_weight_min": -14.060535430908203,
      "activations/layer10_attention_weight_max": 65.41645812988281,
      "activations/layer10_attention_weight_min": -54.51719665527344,
      "activations/layer11_attention_weight_max": 44.273536682128906,
      "activations/layer11_attention_weight_min": -39.42599105834961,
      "activations/layer12_attention_weight_max": 34.97886276245117,
      "activations/layer12_attention_weight_min": -28.52840805053711,
      "activations/layer13_attention_weight_max": 65.09966278076172,
      "activations/layer13_attention_weight_min": -49.27519607543945,
      "activations/layer14_attention_weight_max": 65.85308074951172,
      "activations/layer14_attention_weight_min": -48.67641830444336,
      "activations/layer15_attention_weight_max": 54.66948318481445,
      "activations/layer15_attention_weight_min": -37.978546142578125,
      "activations/layer16_attention_weight_max": 45.381309509277344,
      "activations/layer16_attention_weight_min": -31.734357833862305,
      "activations/layer17_attention_weight_max": 62.27504348754883,
      "activations/layer17_attention_weight_min": -38.363887786865234,
      "activations/layer18_attention_weight_max": 60.148983001708984,
      "activations/layer18_attention_weight_min": -34.344390869140625,
      "activations/layer19_attention_weight_max": 21.755252838134766,
      "activations/layer19_attention_weight_min": -16.201810836791992,
      "activations/layer1_attention_weight_max": 16.203798294067383,
      "activations/layer1_attention_weight_min": -12.382988929748535,
      "activations/layer20_attention_weight_max": 21.02580451965332,
      "activations/layer20_attention_weight_min": -15.1924467086792,
      "activations/layer21_attention_weight_max": 38.50030517578125,
      "activations/layer21_attention_weight_min": -23.476974487304688,
      "activations/layer22_attention_weight_max": 37.70917892456055,
      "activations/layer22_attention_weight_min": -25.09375762939453,
      "activations/layer23_attention_weight_max": 24.906835556030273,
      "activations/layer23_attention_weight_min": -17.77048683166504,
      "activations/layer2_attention_weight_max": 24.608760833740234,
      "activations/layer2_attention_weight_min": -25.524442672729492,
      "activations/layer3_attention_weight_max": 60.733917236328125,
      "activations/layer3_attention_weight_min": -63.30579376220703,
      "activations/layer4_attention_weight_max": 89.05532836914062,
      "activations/layer4_attention_weight_min": -80.65145111083984,
      "activations/layer5_attention_weight_max": 124.95799255371094,
      "activations/layer5_attention_weight_min": -116.13787841796875,
      "activations/layer6_attention_weight_max": 60.90338897705078,
      "activations/layer6_attention_weight_min": -56.21229553222656,
      "activations/layer7_attention_weight_max": 56.893455505371094,
      "activations/layer7_attention_weight_min": -53.945858001708984,
      "activations/layer8_attention_weight_max": 48.02455520629883,
      "activations/layer8_attention_weight_min": -41.261497497558594,
      "activations/layer9_attention_weight_max": 67.47307586669922,
      "activations/layer9_attention_weight_min": -58.31130599975586,
      "epoch": 1.75,
      "learning_rate": 0.00014012992424242422,
      "loss": 2.9697,
      "step": 30100
    },
    {
      "activations/layer0_attention_weight_max": 13.766657829284668,
      "activations/layer0_attention_weight_min": -13.454973220825195,
      "activations/layer10_attention_weight_max": 63.93814468383789,
      "activations/layer10_attention_weight_min": -51.993892669677734,
      "activations/layer11_attention_weight_max": 41.10866165161133,
      "activations/layer11_attention_weight_min": -37.68403625488281,
      "activations/layer12_attention_weight_max": 28.77030372619629,
      "activations/layer12_attention_weight_min": -26.24858856201172,
      "activations/layer13_attention_weight_max": 64.30199432373047,
      "activations/layer13_attention_weight_min": -43.244869232177734,
      "activations/layer14_attention_weight_max": 56.62150955200195,
      "activations/layer14_attention_weight_min": -39.75385284423828,
      "activations/layer15_attention_weight_max": 51.97420120239258,
      "activations/layer15_attention_weight_min": -37.372276306152344,
      "activations/layer16_attention_weight_max": 45.35552215576172,
      "activations/layer16_attention_weight_min": -29.59423828125,
      "activations/layer17_attention_weight_max": 59.25478744506836,
      "activations/layer17_attention_weight_min": -36.42783737182617,
      "activations/layer18_attention_weight_max": 60.91973114013672,
      "activations/layer18_attention_weight_min": -35.25545120239258,
      "activations/layer19_attention_weight_max": 20.2427921295166,
      "activations/layer19_attention_weight_min": -15.56421947479248,
      "activations/layer1_attention_weight_max": 15.55941390991211,
      "activations/layer1_attention_weight_min": -12.466347694396973,
      "activations/layer20_attention_weight_max": 18.406803131103516,
      "activations/layer20_attention_weight_min": -16.015378952026367,
      "activations/layer21_attention_weight_max": 35.41229248046875,
      "activations/layer21_attention_weight_min": -22.940521240234375,
      "activations/layer22_attention_weight_max": 35.30341339111328,
      "activations/layer22_attention_weight_min": -19.410207748413086,
      "activations/layer23_attention_weight_max": 24.588542938232422,
      "activations/layer23_attention_weight_min": -13.341133117675781,
      "activations/layer2_attention_weight_max": 26.241498947143555,
      "activations/layer2_attention_weight_min": -25.958877563476562,
      "activations/layer3_attention_weight_max": 57.92918395996094,
      "activations/layer3_attention_weight_min": -61.182796478271484,
      "activations/layer4_attention_weight_max": 88.47017669677734,
      "activations/layer4_attention_weight_min": -82.83757781982422,
      "activations/layer5_attention_weight_max": 124.26509094238281,
      "activations/layer5_attention_weight_min": -109.65615844726562,
      "activations/layer6_attention_weight_max": 62.44950866699219,
      "activations/layer6_attention_weight_min": -54.02866744995117,
      "activations/layer7_attention_weight_max": 59.69729995727539,
      "activations/layer7_attention_weight_min": -55.134769439697266,
      "activations/layer8_attention_weight_max": 45.41090774536133,
      "activations/layer8_attention_weight_min": -44.474544525146484,
      "activations/layer9_attention_weight_max": 65.59326171875,
      "activations/layer9_attention_weight_min": -58.3529052734375,
      "epoch": 1.75,
      "learning_rate": 0.00014011098484848484,
      "loss": 2.9811,
      "step": 30150
    },
    {
      "activations/layer0_attention_weight_max": 13.304619789123535,
      "activations/layer0_attention_weight_min": -13.986294746398926,
      "activations/layer10_attention_weight_max": 61.168575286865234,
      "activations/layer10_attention_weight_min": -49.715816497802734,
      "activations/layer11_attention_weight_max": 46.907222747802734,
      "activations/layer11_attention_weight_min": -38.655662536621094,
      "activations/layer12_attention_weight_max": 30.215171813964844,
      "activations/layer12_attention_weight_min": -25.60026741027832,
      "activations/layer13_attention_weight_max": 67.78640747070312,
      "activations/layer13_attention_weight_min": -45.997901916503906,
      "activations/layer14_attention_weight_max": 52.020084381103516,
      "activations/layer14_attention_weight_min": -39.29910659790039,
      "activations/layer15_attention_weight_max": 61.163360595703125,
      "activations/layer15_attention_weight_min": -44.860252380371094,
      "activations/layer16_attention_weight_max": 41.739803314208984,
      "activations/layer16_attention_weight_min": -30.973730087280273,
      "activations/layer17_attention_weight_max": 56.8934211730957,
      "activations/layer17_attention_weight_min": -37.691871643066406,
      "activations/layer18_attention_weight_max": 55.21194839477539,
      "activations/layer18_attention_weight_min": -34.893524169921875,
      "activations/layer19_attention_weight_max": 22.86214256286621,
      "activations/layer19_attention_weight_min": -14.985491752624512,
      "activations/layer1_attention_weight_max": 15.60198974609375,
      "activations/layer1_attention_weight_min": -13.429067611694336,
      "activations/layer20_attention_weight_max": 21.34044075012207,
      "activations/layer20_attention_weight_min": -18.506790161132812,
      "activations/layer21_attention_weight_max": 37.52344512939453,
      "activations/layer21_attention_weight_min": -22.249040603637695,
      "activations/layer22_attention_weight_max": 34.76835632324219,
      "activations/layer22_attention_weight_min": -19.938634872436523,
      "activations/layer23_attention_weight_max": 21.62701988220215,
      "activations/layer23_attention_weight_min": -14.507896423339844,
      "activations/layer2_attention_weight_max": 26.711164474487305,
      "activations/layer2_attention_weight_min": -24.480663299560547,
      "activations/layer3_attention_weight_max": 57.20751953125,
      "activations/layer3_attention_weight_min": -60.51445388793945,
      "activations/layer4_attention_weight_max": 85.37037658691406,
      "activations/layer4_attention_weight_min": -78.21965789794922,
      "activations/layer5_attention_weight_max": 124.24104309082031,
      "activations/layer5_attention_weight_min": -109.83212280273438,
      "activations/layer6_attention_weight_max": 61.80854797363281,
      "activations/layer6_attention_weight_min": -55.97038650512695,
      "activations/layer7_attention_weight_max": 59.91766357421875,
      "activations/layer7_attention_weight_min": -55.28339385986328,
      "activations/layer8_attention_weight_max": 51.14604949951172,
      "activations/layer8_attention_weight_min": -47.96077346801758,
      "activations/layer9_attention_weight_max": 70.582763671875,
      "activations/layer9_attention_weight_min": -62.5434684753418,
      "epoch": 1.75,
      "learning_rate": 0.00014009204545454543,
      "loss": 3.0018,
      "step": 30200
    },
    {
      "activations/layer0_attention_weight_max": 14.890029907226562,
      "activations/layer0_attention_weight_min": -13.332597732543945,
      "activations/layer10_attention_weight_max": 52.81806182861328,
      "activations/layer10_attention_weight_min": -47.33671188354492,
      "activations/layer11_attention_weight_max": 46.272682189941406,
      "activations/layer11_attention_weight_min": -37.36498260498047,
      "activations/layer12_attention_weight_max": 26.237361907958984,
      "activations/layer12_attention_weight_min": -22.989032745361328,
      "activations/layer13_attention_weight_max": 60.6456184387207,
      "activations/layer13_attention_weight_min": -47.401451110839844,
      "activations/layer14_attention_weight_max": 50.766197204589844,
      "activations/layer14_attention_weight_min": -37.30642318725586,
      "activations/layer15_attention_weight_max": 48.971866607666016,
      "activations/layer15_attention_weight_min": -37.8106689453125,
      "activations/layer16_attention_weight_max": 42.12544250488281,
      "activations/layer16_attention_weight_min": -31.444414138793945,
      "activations/layer17_attention_weight_max": 52.42360305786133,
      "activations/layer17_attention_weight_min": -37.578025817871094,
      "activations/layer18_attention_weight_max": 55.80600357055664,
      "activations/layer18_attention_weight_min": -33.1480712890625,
      "activations/layer19_attention_weight_max": 20.990768432617188,
      "activations/layer19_attention_weight_min": -15.179803848266602,
      "activations/layer1_attention_weight_max": 16.131437301635742,
      "activations/layer1_attention_weight_min": -12.038437843322754,
      "activations/layer20_attention_weight_max": 19.582752227783203,
      "activations/layer20_attention_weight_min": -15.607032775878906,
      "activations/layer21_attention_weight_max": 39.49424743652344,
      "activations/layer21_attention_weight_min": -23.91360855102539,
      "activations/layer22_attention_weight_max": 38.715885162353516,
      "activations/layer22_attention_weight_min": -22.70917320251465,
      "activations/layer23_attention_weight_max": 24.47443199157715,
      "activations/layer23_attention_weight_min": -15.236292839050293,
      "activations/layer2_attention_weight_max": 26.6470947265625,
      "activations/layer2_attention_weight_min": -27.53445816040039,
      "activations/layer3_attention_weight_max": 60.636993408203125,
      "activations/layer3_attention_weight_min": -65.15153503417969,
      "activations/layer4_attention_weight_max": 90.31108093261719,
      "activations/layer4_attention_weight_min": -83.10955047607422,
      "activations/layer5_attention_weight_max": 125.53755187988281,
      "activations/layer5_attention_weight_min": -111.29736328125,
      "activations/layer6_attention_weight_max": 60.588111877441406,
      "activations/layer6_attention_weight_min": -53.51993942260742,
      "activations/layer7_attention_weight_max": 55.85792922973633,
      "activations/layer7_attention_weight_min": -54.037811279296875,
      "activations/layer8_attention_weight_max": 43.95512390136719,
      "activations/layer8_attention_weight_min": -40.816734313964844,
      "activations/layer9_attention_weight_max": 64.2414779663086,
      "activations/layer9_attention_weight_min": -60.17135238647461,
      "epoch": 1.76,
      "learning_rate": 0.00014007310606060604,
      "loss": 2.9956,
      "step": 30250
    },
    {
      "activations/layer0_attention_weight_max": 14.035646438598633,
      "activations/layer0_attention_weight_min": -14.318957328796387,
      "activations/layer10_attention_weight_max": 60.999996185302734,
      "activations/layer10_attention_weight_min": -54.14729690551758,
      "activations/layer11_attention_weight_max": 39.21502685546875,
      "activations/layer11_attention_weight_min": -36.98835372924805,
      "activations/layer12_attention_weight_max": 30.253232955932617,
      "activations/layer12_attention_weight_min": -22.968246459960938,
      "activations/layer13_attention_weight_max": 60.92066192626953,
      "activations/layer13_attention_weight_min": -49.30283737182617,
      "activations/layer14_attention_weight_max": 55.72972106933594,
      "activations/layer14_attention_weight_min": -42.95801544189453,
      "activations/layer15_attention_weight_max": 51.52314376831055,
      "activations/layer15_attention_weight_min": -40.92313766479492,
      "activations/layer16_attention_weight_max": 42.12948226928711,
      "activations/layer16_attention_weight_min": -29.276535034179688,
      "activations/layer17_attention_weight_max": 56.01515197753906,
      "activations/layer17_attention_weight_min": -37.46767044067383,
      "activations/layer18_attention_weight_max": 60.82242202758789,
      "activations/layer18_attention_weight_min": -35.43560028076172,
      "activations/layer19_attention_weight_max": 23.905296325683594,
      "activations/layer19_attention_weight_min": -14.971964836120605,
      "activations/layer1_attention_weight_max": 15.535425186157227,
      "activations/layer1_attention_weight_min": -13.762231826782227,
      "activations/layer20_attention_weight_max": 21.88187599182129,
      "activations/layer20_attention_weight_min": -14.653209686279297,
      "activations/layer21_attention_weight_max": 42.972137451171875,
      "activations/layer21_attention_weight_min": -22.418081283569336,
      "activations/layer22_attention_weight_max": 37.229408264160156,
      "activations/layer22_attention_weight_min": -18.86204719543457,
      "activations/layer23_attention_weight_max": 23.22016716003418,
      "activations/layer23_attention_weight_min": -13.267111778259277,
      "activations/layer2_attention_weight_max": 25.679576873779297,
      "activations/layer2_attention_weight_min": -25.317790985107422,
      "activations/layer3_attention_weight_max": 58.141170501708984,
      "activations/layer3_attention_weight_min": -63.29867172241211,
      "activations/layer4_attention_weight_max": 88.66472625732422,
      "activations/layer4_attention_weight_min": -81.65025329589844,
      "activations/layer5_attention_weight_max": 126.29454803466797,
      "activations/layer5_attention_weight_min": -113.05348205566406,
      "activations/layer6_attention_weight_max": 60.641483306884766,
      "activations/layer6_attention_weight_min": -56.73012161254883,
      "activations/layer7_attention_weight_max": 56.50164794921875,
      "activations/layer7_attention_weight_min": -51.93836975097656,
      "activations/layer8_attention_weight_max": 46.74993896484375,
      "activations/layer8_attention_weight_min": -44.89411163330078,
      "activations/layer9_attention_weight_max": 61.865413665771484,
      "activations/layer9_attention_weight_min": -61.34809494018555,
      "epoch": 1.76,
      "learning_rate": 0.00014005416666666666,
      "loss": 2.9791,
      "step": 30300
    },
    {
      "activations/layer0_attention_weight_max": 13.270100593566895,
      "activations/layer0_attention_weight_min": -14.066543579101562,
      "activations/layer10_attention_weight_max": 66.6102294921875,
      "activations/layer10_attention_weight_min": -51.52313232421875,
      "activations/layer11_attention_weight_max": 44.049285888671875,
      "activations/layer11_attention_weight_min": -36.03802490234375,
      "activations/layer12_attention_weight_max": 26.967077255249023,
      "activations/layer12_attention_weight_min": -24.02043914794922,
      "activations/layer13_attention_weight_max": 61.719215393066406,
      "activations/layer13_attention_weight_min": -43.3604850769043,
      "activations/layer14_attention_weight_max": 53.860877990722656,
      "activations/layer14_attention_weight_min": -38.33393478393555,
      "activations/layer15_attention_weight_max": 51.39145278930664,
      "activations/layer15_attention_weight_min": -36.753231048583984,
      "activations/layer16_attention_weight_max": 43.44080352783203,
      "activations/layer16_attention_weight_min": -30.928218841552734,
      "activations/layer17_attention_weight_max": 61.06220245361328,
      "activations/layer17_attention_weight_min": -36.974517822265625,
      "activations/layer18_attention_weight_max": 59.52614974975586,
      "activations/layer18_attention_weight_min": -35.98251724243164,
      "activations/layer19_attention_weight_max": 24.680896759033203,
      "activations/layer19_attention_weight_min": -15.811247825622559,
      "activations/layer1_attention_weight_max": 16.081518173217773,
      "activations/layer1_attention_weight_min": -12.934741020202637,
      "activations/layer20_attention_weight_max": 19.428529739379883,
      "activations/layer20_attention_weight_min": -14.893576622009277,
      "activations/layer21_attention_weight_max": 39.63470458984375,
      "activations/layer21_attention_weight_min": -23.415176391601562,
      "activations/layer22_attention_weight_max": 33.93848419189453,
      "activations/layer22_attention_weight_min": -20.3662109375,
      "activations/layer23_attention_weight_max": 24.22480583190918,
      "activations/layer23_attention_weight_min": -16.688573837280273,
      "activations/layer2_attention_weight_max": 24.672534942626953,
      "activations/layer2_attention_weight_min": -24.594945907592773,
      "activations/layer3_attention_weight_max": 57.246891021728516,
      "activations/layer3_attention_weight_min": -60.46372604370117,
      "activations/layer4_attention_weight_max": 90.73387145996094,
      "activations/layer4_attention_weight_min": -80.52027893066406,
      "activations/layer5_attention_weight_max": 124.13530731201172,
      "activations/layer5_attention_weight_min": -108.50422668457031,
      "activations/layer6_attention_weight_max": 60.52693557739258,
      "activations/layer6_attention_weight_min": -50.86469650268555,
      "activations/layer7_attention_weight_max": 57.86173629760742,
      "activations/layer7_attention_weight_min": -52.65934753417969,
      "activations/layer8_attention_weight_max": 46.46770095825195,
      "activations/layer8_attention_weight_min": -45.02828598022461,
      "activations/layer9_attention_weight_max": 68.1109848022461,
      "activations/layer9_attention_weight_min": -60.21430587768555,
      "epoch": 1.76,
      "learning_rate": 0.00014003522727272725,
      "loss": 2.9881,
      "step": 30350
    },
    {
      "activations/layer0_attention_weight_max": 17.334869384765625,
      "activations/layer0_attention_weight_min": -13.507328033447266,
      "activations/layer10_attention_weight_max": 62.31730651855469,
      "activations/layer10_attention_weight_min": -51.37150955200195,
      "activations/layer11_attention_weight_max": 48.94548034667969,
      "activations/layer11_attention_weight_min": -38.23664093017578,
      "activations/layer12_attention_weight_max": 27.793357849121094,
      "activations/layer12_attention_weight_min": -25.605045318603516,
      "activations/layer13_attention_weight_max": 57.545989990234375,
      "activations/layer13_attention_weight_min": -39.711082458496094,
      "activations/layer14_attention_weight_max": 54.0656852722168,
      "activations/layer14_attention_weight_min": -39.613399505615234,
      "activations/layer15_attention_weight_max": 60.024688720703125,
      "activations/layer15_attention_weight_min": -40.42433166503906,
      "activations/layer16_attention_weight_max": 45.99976348876953,
      "activations/layer16_attention_weight_min": -29.228429794311523,
      "activations/layer17_attention_weight_max": 66.5126724243164,
      "activations/layer17_attention_weight_min": -36.78813552856445,
      "activations/layer18_attention_weight_max": 59.871986389160156,
      "activations/layer18_attention_weight_min": -32.275848388671875,
      "activations/layer19_attention_weight_max": 22.73661994934082,
      "activations/layer19_attention_weight_min": -14.244440078735352,
      "activations/layer1_attention_weight_max": 15.784543991088867,
      "activations/layer1_attention_weight_min": -11.674955368041992,
      "activations/layer20_attention_weight_max": 19.222461700439453,
      "activations/layer20_attention_weight_min": -17.08429527282715,
      "activations/layer21_attention_weight_max": 40.17169189453125,
      "activations/layer21_attention_weight_min": -21.312969207763672,
      "activations/layer22_attention_weight_max": 39.60060119628906,
      "activations/layer22_attention_weight_min": -22.15068244934082,
      "activations/layer23_attention_weight_max": 24.20514488220215,
      "activations/layer23_attention_weight_min": -15.751700401306152,
      "activations/layer2_attention_weight_max": 25.354225158691406,
      "activations/layer2_attention_weight_min": -24.813709259033203,
      "activations/layer3_attention_weight_max": 63.47834014892578,
      "activations/layer3_attention_weight_min": -66.90547180175781,
      "activations/layer4_attention_weight_max": 94.21724700927734,
      "activations/layer4_attention_weight_min": -82.7938003540039,
      "activations/layer5_attention_weight_max": 132.67726135253906,
      "activations/layer5_attention_weight_min": -109.5160140991211,
      "activations/layer6_attention_weight_max": 65.43622589111328,
      "activations/layer6_attention_weight_min": -53.24696350097656,
      "activations/layer7_attention_weight_max": 60.2320442199707,
      "activations/layer7_attention_weight_min": -55.10810852050781,
      "activations/layer8_attention_weight_max": 54.41371536254883,
      "activations/layer8_attention_weight_min": -42.01881408691406,
      "activations/layer9_attention_weight_max": 74.53727722167969,
      "activations/layer9_attention_weight_min": -59.749786376953125,
      "epoch": 1.77,
      "learning_rate": 0.00014001628787878786,
      "loss": 3.0024,
      "step": 30400
    },
    {
      "activations/layer0_attention_weight_max": 14.818543434143066,
      "activations/layer0_attention_weight_min": -13.519664764404297,
      "activations/layer10_attention_weight_max": 82.12466430664062,
      "activations/layer10_attention_weight_min": -56.369503021240234,
      "activations/layer11_attention_weight_max": 62.72640609741211,
      "activations/layer11_attention_weight_min": -41.78241729736328,
      "activations/layer12_attention_weight_max": 33.28120040893555,
      "activations/layer12_attention_weight_min": -25.75297737121582,
      "activations/layer13_attention_weight_max": 68.5648422241211,
      "activations/layer13_attention_weight_min": -53.542457580566406,
      "activations/layer14_attention_weight_max": 76.79360961914062,
      "activations/layer14_attention_weight_min": -49.362518310546875,
      "activations/layer15_attention_weight_max": 54.96650695800781,
      "activations/layer15_attention_weight_min": -38.30830764770508,
      "activations/layer16_attention_weight_max": 47.573455810546875,
      "activations/layer16_attention_weight_min": -31.733381271362305,
      "activations/layer17_attention_weight_max": 75.52902221679688,
      "activations/layer17_attention_weight_min": -39.81389617919922,
      "activations/layer18_attention_weight_max": 59.752044677734375,
      "activations/layer18_attention_weight_min": -33.554931640625,
      "activations/layer19_attention_weight_max": 22.798397064208984,
      "activations/layer19_attention_weight_min": -15.456573486328125,
      "activations/layer1_attention_weight_max": 17.221622467041016,
      "activations/layer1_attention_weight_min": -12.76828670501709,
      "activations/layer20_attention_weight_max": 21.313600540161133,
      "activations/layer20_attention_weight_min": -14.508447647094727,
      "activations/layer21_attention_weight_max": 46.642173767089844,
      "activations/layer21_attention_weight_min": -24.421422958374023,
      "activations/layer22_attention_weight_max": 36.424678802490234,
      "activations/layer22_attention_weight_min": -22.75283432006836,
      "activations/layer23_attention_weight_max": 21.94580841064453,
      "activations/layer23_attention_weight_min": -14.786115646362305,
      "activations/layer2_attention_weight_max": 26.146514892578125,
      "activations/layer2_attention_weight_min": -26.07783317565918,
      "activations/layer3_attention_weight_max": 61.088356018066406,
      "activations/layer3_attention_weight_min": -64.25686645507812,
      "activations/layer4_attention_weight_max": 94.2884750366211,
      "activations/layer4_attention_weight_min": -85.22148895263672,
      "activations/layer5_attention_weight_max": 139.17320251464844,
      "activations/layer5_attention_weight_min": -133.74757385253906,
      "activations/layer6_attention_weight_max": 64.1468734741211,
      "activations/layer6_attention_weight_min": -58.6397819519043,
      "activations/layer7_attention_weight_max": 69.42422485351562,
      "activations/layer7_attention_weight_min": -59.43859100341797,
      "activations/layer8_attention_weight_max": 64.80716705322266,
      "activations/layer8_attention_weight_min": -54.024070739746094,
      "activations/layer9_attention_weight_max": 82.02006530761719,
      "activations/layer9_attention_weight_min": -71.94267272949219,
      "epoch": 1.77,
      "learning_rate": 0.00013999772727272725,
      "loss": 3.0049,
      "step": 30450
    },
    {
      "activations/layer0_attention_weight_max": 15.140558242797852,
      "activations/layer0_attention_weight_min": -13.545001983642578,
      "activations/layer10_attention_weight_max": 74.15609741210938,
      "activations/layer10_attention_weight_min": -52.57480239868164,
      "activations/layer11_attention_weight_max": 49.1175537109375,
      "activations/layer11_attention_weight_min": -39.7221794128418,
      "activations/layer12_attention_weight_max": 29.780597686767578,
      "activations/layer12_attention_weight_min": -25.861143112182617,
      "activations/layer13_attention_weight_max": 63.90687561035156,
      "activations/layer13_attention_weight_min": -40.571895599365234,
      "activations/layer14_attention_weight_max": 57.19036102294922,
      "activations/layer14_attention_weight_min": -38.19221496582031,
      "activations/layer15_attention_weight_max": 59.23377227783203,
      "activations/layer15_attention_weight_min": -38.035545349121094,
      "activations/layer16_attention_weight_max": 50.365848541259766,
      "activations/layer16_attention_weight_min": -30.064496994018555,
      "activations/layer17_attention_weight_max": 62.29010009765625,
      "activations/layer17_attention_weight_min": -37.95917510986328,
      "activations/layer18_attention_weight_max": 70.38800048828125,
      "activations/layer18_attention_weight_min": -36.327476501464844,
      "activations/layer19_attention_weight_max": 24.535245895385742,
      "activations/layer19_attention_weight_min": -15.30256175994873,
      "activations/layer1_attention_weight_max": 16.034971237182617,
      "activations/layer1_attention_weight_min": -13.366503715515137,
      "activations/layer20_attention_weight_max": 20.68699836730957,
      "activations/layer20_attention_weight_min": -16.32001495361328,
      "activations/layer21_attention_weight_max": 38.63251495361328,
      "activations/layer21_attention_weight_min": -21.64203453063965,
      "activations/layer22_attention_weight_max": 39.622432708740234,
      "activations/layer22_attention_weight_min": -23.78130531311035,
      "activations/layer23_attention_weight_max": 25.939109802246094,
      "activations/layer23_attention_weight_min": -15.194289207458496,
      "activations/layer2_attention_weight_max": 24.671653747558594,
      "activations/layer2_attention_weight_min": -23.96282958984375,
      "activations/layer3_attention_weight_max": 61.69205856323242,
      "activations/layer3_attention_weight_min": -65.45865631103516,
      "activations/layer4_attention_weight_max": 95.52706909179688,
      "activations/layer4_attention_weight_min": -85.38018035888672,
      "activations/layer5_attention_weight_max": 136.7691650390625,
      "activations/layer5_attention_weight_min": -112.90240478515625,
      "activations/layer6_attention_weight_max": 66.65361022949219,
      "activations/layer6_attention_weight_min": -52.63886642456055,
      "activations/layer7_attention_weight_max": 62.5638542175293,
      "activations/layer7_attention_weight_min": -56.561729431152344,
      "activations/layer8_attention_weight_max": 50.243473052978516,
      "activations/layer8_attention_weight_min": -40.93751907348633,
      "activations/layer9_attention_weight_max": 76.03739929199219,
      "activations/layer9_attention_weight_min": -57.78744125366211,
      "epoch": 1.77,
      "learning_rate": 0.00013997878787878787,
      "loss": 2.9869,
      "step": 30500
    },
    {
      "activations/layer0_attention_weight_max": 12.970040321350098,
      "activations/layer0_attention_weight_min": -14.558292388916016,
      "activations/layer10_attention_weight_max": 58.66452407836914,
      "activations/layer10_attention_weight_min": -49.05852127075195,
      "activations/layer11_attention_weight_max": 43.15638732910156,
      "activations/layer11_attention_weight_min": -33.32019805908203,
      "activations/layer12_attention_weight_max": 30.472129821777344,
      "activations/layer12_attention_weight_min": -26.811450958251953,
      "activations/layer13_attention_weight_max": 52.01601791381836,
      "activations/layer13_attention_weight_min": -39.55409240722656,
      "activations/layer14_attention_weight_max": 43.7735710144043,
      "activations/layer14_attention_weight_min": -34.22378921508789,
      "activations/layer15_attention_weight_max": 47.17441940307617,
      "activations/layer15_attention_weight_min": -37.60163497924805,
      "activations/layer16_attention_weight_max": 43.54126739501953,
      "activations/layer16_attention_weight_min": -30.31803321838379,
      "activations/layer17_attention_weight_max": 51.795166015625,
      "activations/layer17_attention_weight_min": -40.16511154174805,
      "activations/layer18_attention_weight_max": 50.47834396362305,
      "activations/layer18_attention_weight_min": -33.482261657714844,
      "activations/layer19_attention_weight_max": 22.4263973236084,
      "activations/layer19_attention_weight_min": -13.633458137512207,
      "activations/layer1_attention_weight_max": 15.45388126373291,
      "activations/layer1_attention_weight_min": -13.216567993164062,
      "activations/layer20_attention_weight_max": 16.233976364135742,
      "activations/layer20_attention_weight_min": -16.266151428222656,
      "activations/layer21_attention_weight_max": 36.13108444213867,
      "activations/layer21_attention_weight_min": -20.685943603515625,
      "activations/layer22_attention_weight_max": 31.808841705322266,
      "activations/layer22_attention_weight_min": -23.53019142150879,
      "activations/layer23_attention_weight_max": 21.320940017700195,
      "activations/layer23_attention_weight_min": -16.471641540527344,
      "activations/layer2_attention_weight_max": 24.80567741394043,
      "activations/layer2_attention_weight_min": -25.857473373413086,
      "activations/layer3_attention_weight_max": 58.41343307495117,
      "activations/layer3_attention_weight_min": -62.51356506347656,
      "activations/layer4_attention_weight_max": 87.34261322021484,
      "activations/layer4_attention_weight_min": -81.18494415283203,
      "activations/layer5_attention_weight_max": 122.16841888427734,
      "activations/layer5_attention_weight_min": -109.59413146972656,
      "activations/layer6_attention_weight_max": 60.5411376953125,
      "activations/layer6_attention_weight_min": -54.5737419128418,
      "activations/layer7_attention_weight_max": 57.9100227355957,
      "activations/layer7_attention_weight_min": -53.844947814941406,
      "activations/layer8_attention_weight_max": 48.78628158569336,
      "activations/layer8_attention_weight_min": -40.66462707519531,
      "activations/layer9_attention_weight_max": 62.406982421875,
      "activations/layer9_attention_weight_min": -54.993560791015625,
      "epoch": 1.78,
      "learning_rate": 0.00013995984848484848,
      "loss": 2.98,
      "step": 30550
    },
    {
      "activations/layer0_attention_weight_max": 15.2113037109375,
      "activations/layer0_attention_weight_min": -13.762669563293457,
      "activations/layer10_attention_weight_max": 64.84425354003906,
      "activations/layer10_attention_weight_min": -52.1697883605957,
      "activations/layer11_attention_weight_max": 43.924537658691406,
      "activations/layer11_attention_weight_min": -40.89813995361328,
      "activations/layer12_attention_weight_max": 26.243623733520508,
      "activations/layer12_attention_weight_min": -25.546281814575195,
      "activations/layer13_attention_weight_max": 52.7851448059082,
      "activations/layer13_attention_weight_min": -39.345947265625,
      "activations/layer14_attention_weight_max": 44.82474136352539,
      "activations/layer14_attention_weight_min": -34.63145446777344,
      "activations/layer15_attention_weight_max": 48.663917541503906,
      "activations/layer15_attention_weight_min": -37.91201400756836,
      "activations/layer16_attention_weight_max": 44.68821334838867,
      "activations/layer16_attention_weight_min": -32.31616973876953,
      "activations/layer17_attention_weight_max": 70.26248931884766,
      "activations/layer17_attention_weight_min": -41.835418701171875,
      "activations/layer18_attention_weight_max": 56.47559356689453,
      "activations/layer18_attention_weight_min": -37.2926139831543,
      "activations/layer19_attention_weight_max": 21.033422470092773,
      "activations/layer19_attention_weight_min": -16.818204879760742,
      "activations/layer1_attention_weight_max": 15.96520709991455,
      "activations/layer1_attention_weight_min": -11.765035629272461,
      "activations/layer20_attention_weight_max": 21.154443740844727,
      "activations/layer20_attention_weight_min": -16.570411682128906,
      "activations/layer21_attention_weight_max": 42.30061721801758,
      "activations/layer21_attention_weight_min": -21.1615047454834,
      "activations/layer22_attention_weight_max": 36.04465866088867,
      "activations/layer22_attention_weight_min": -20.950088500976562,
      "activations/layer23_attention_weight_max": 23.953067779541016,
      "activations/layer23_attention_weight_min": -14.536310195922852,
      "activations/layer2_attention_weight_max": 25.05947494506836,
      "activations/layer2_attention_weight_min": -25.02741813659668,
      "activations/layer3_attention_weight_max": 58.30071258544922,
      "activations/layer3_attention_weight_min": -63.954620361328125,
      "activations/layer4_attention_weight_max": 90.56315612792969,
      "activations/layer4_attention_weight_min": -82.44142150878906,
      "activations/layer5_attention_weight_max": 122.2160873413086,
      "activations/layer5_attention_weight_min": -115.78636932373047,
      "activations/layer6_attention_weight_max": 62.35691833496094,
      "activations/layer6_attention_weight_min": -52.313907623291016,
      "activations/layer7_attention_weight_max": 60.46579360961914,
      "activations/layer7_attention_weight_min": -54.25691223144531,
      "activations/layer8_attention_weight_max": 44.49260330200195,
      "activations/layer8_attention_weight_min": -42.66925811767578,
      "activations/layer9_attention_weight_max": 64.64114379882812,
      "activations/layer9_attention_weight_min": -62.36027908325195,
      "epoch": 1.78,
      "learning_rate": 0.00013994090909090907,
      "loss": 2.9826,
      "step": 30600
    },
    {
      "activations/layer0_attention_weight_max": 14.21072769165039,
      "activations/layer0_attention_weight_min": -14.285889625549316,
      "activations/layer10_attention_weight_max": 58.195579528808594,
      "activations/layer10_attention_weight_min": -46.84648132324219,
      "activations/layer11_attention_weight_max": 42.09912109375,
      "activations/layer11_attention_weight_min": -34.23823547363281,
      "activations/layer12_attention_weight_max": 25.82497787475586,
      "activations/layer12_attention_weight_min": -26.142004013061523,
      "activations/layer13_attention_weight_max": 58.090606689453125,
      "activations/layer13_attention_weight_min": -43.50285339355469,
      "activations/layer14_attention_weight_max": 52.703208923339844,
      "activations/layer14_attention_weight_min": -38.64955520629883,
      "activations/layer15_attention_weight_max": 53.19635009765625,
      "activations/layer15_attention_weight_min": -38.46017837524414,
      "activations/layer16_attention_weight_max": 41.82878494262695,
      "activations/layer16_attention_weight_min": -29.156763076782227,
      "activations/layer17_attention_weight_max": 70.8069076538086,
      "activations/layer17_attention_weight_min": -37.23958206176758,
      "activations/layer18_attention_weight_max": 56.07313919067383,
      "activations/layer18_attention_weight_min": -35.505584716796875,
      "activations/layer19_attention_weight_max": 24.25131607055664,
      "activations/layer19_attention_weight_min": -15.180777549743652,
      "activations/layer1_attention_weight_max": 16.966808319091797,
      "activations/layer1_attention_weight_min": -12.02386474609375,
      "activations/layer20_attention_weight_max": 18.21961784362793,
      "activations/layer20_attention_weight_min": -16.707523345947266,
      "activations/layer21_attention_weight_max": 36.82521057128906,
      "activations/layer21_attention_weight_min": -23.20781898498535,
      "activations/layer22_attention_weight_max": 38.389366149902344,
      "activations/layer22_attention_weight_min": -21.40856170654297,
      "activations/layer23_attention_weight_max": 23.856117248535156,
      "activations/layer23_attention_weight_min": -13.714282989501953,
      "activations/layer2_attention_weight_max": 25.74423599243164,
      "activations/layer2_attention_weight_min": -24.070730209350586,
      "activations/layer3_attention_weight_max": 58.11747360229492,
      "activations/layer3_attention_weight_min": -63.03023147583008,
      "activations/layer4_attention_weight_max": 86.71548461914062,
      "activations/layer4_attention_weight_min": -80.24889373779297,
      "activations/layer5_attention_weight_max": 124.99766540527344,
      "activations/layer5_attention_weight_min": -110.69715881347656,
      "activations/layer6_attention_weight_max": 57.98993682861328,
      "activations/layer6_attention_weight_min": -52.81452560424805,
      "activations/layer7_attention_weight_max": 55.8414192199707,
      "activations/layer7_attention_weight_min": -54.531219482421875,
      "activations/layer8_attention_weight_max": 49.660396575927734,
      "activations/layer8_attention_weight_min": -43.332584381103516,
      "activations/layer9_attention_weight_max": 67.83161926269531,
      "activations/layer9_attention_weight_min": -60.20212936401367,
      "epoch": 1.78,
      "learning_rate": 0.00013992196969696968,
      "loss": 3.0038,
      "step": 30650
    },
    {
      "activations/layer0_attention_weight_max": 14.426589965820312,
      "activations/layer0_attention_weight_min": -13.642157554626465,
      "activations/layer10_attention_weight_max": 59.48600387573242,
      "activations/layer10_attention_weight_min": -51.778316497802734,
      "activations/layer11_attention_weight_max": 46.65478515625,
      "activations/layer11_attention_weight_min": -36.732120513916016,
      "activations/layer12_attention_weight_max": 32.868160247802734,
      "activations/layer12_attention_weight_min": -23.632566452026367,
      "activations/layer13_attention_weight_max": 67.10183715820312,
      "activations/layer13_attention_weight_min": -41.60536575317383,
      "activations/layer14_attention_weight_max": 61.3127326965332,
      "activations/layer14_attention_weight_min": -37.265586853027344,
      "activations/layer15_attention_weight_max": 50.91971969604492,
      "activations/layer15_attention_weight_min": -38.73332595825195,
      "activations/layer16_attention_weight_max": 45.4326057434082,
      "activations/layer16_attention_weight_min": -29.93538475036621,
      "activations/layer17_attention_weight_max": 66.13304138183594,
      "activations/layer17_attention_weight_min": -41.73588180541992,
      "activations/layer18_attention_weight_max": 55.431087493896484,
      "activations/layer18_attention_weight_min": -33.77971267700195,
      "activations/layer19_attention_weight_max": 22.36905288696289,
      "activations/layer19_attention_weight_min": -15.01541805267334,
      "activations/layer1_attention_weight_max": 16.05196189880371,
      "activations/layer1_attention_weight_min": -13.446646690368652,
      "activations/layer20_attention_weight_max": 25.735733032226562,
      "activations/layer20_attention_weight_min": -17.951139450073242,
      "activations/layer21_attention_weight_max": 40.899261474609375,
      "activations/layer21_attention_weight_min": -23.441152572631836,
      "activations/layer22_attention_weight_max": 45.99058151245117,
      "activations/layer22_attention_weight_min": -20.302576065063477,
      "activations/layer23_attention_weight_max": 28.13361358642578,
      "activations/layer23_attention_weight_min": -15.74838924407959,
      "activations/layer2_attention_weight_max": 25.29297637939453,
      "activations/layer2_attention_weight_min": -24.334087371826172,
      "activations/layer3_attention_weight_max": 60.59516143798828,
      "activations/layer3_attention_weight_min": -62.22843933105469,
      "activations/layer4_attention_weight_max": 93.37942504882812,
      "activations/layer4_attention_weight_min": -83.35289764404297,
      "activations/layer5_attention_weight_max": 124.51988220214844,
      "activations/layer5_attention_weight_min": -109.2154541015625,
      "activations/layer6_attention_weight_max": 60.70978927612305,
      "activations/layer6_attention_weight_min": -52.74711608886719,
      "activations/layer7_attention_weight_max": 64.39396667480469,
      "activations/layer7_attention_weight_min": -54.94261932373047,
      "activations/layer8_attention_weight_max": 51.27639389038086,
      "activations/layer8_attention_weight_min": -42.4925422668457,
      "activations/layer9_attention_weight_max": 71.53516387939453,
      "activations/layer9_attention_weight_min": -57.12263107299805,
      "epoch": 1.78,
      "learning_rate": 0.0001399030303030303,
      "loss": 2.9916,
      "step": 30700
    },
    {
      "activations/layer0_attention_weight_max": 15.286255836486816,
      "activations/layer0_attention_weight_min": -13.398792266845703,
      "activations/layer10_attention_weight_max": 61.53274917602539,
      "activations/layer10_attention_weight_min": -54.8978385925293,
      "activations/layer11_attention_weight_max": 46.36890411376953,
      "activations/layer11_attention_weight_min": -40.075157165527344,
      "activations/layer12_attention_weight_max": 36.65196990966797,
      "activations/layer12_attention_weight_min": -28.79629135131836,
      "activations/layer13_attention_weight_max": 53.508079528808594,
      "activations/layer13_attention_weight_min": -40.9312744140625,
      "activations/layer14_attention_weight_max": 51.58611297607422,
      "activations/layer14_attention_weight_min": -37.51162338256836,
      "activations/layer15_attention_weight_max": 48.80387878417969,
      "activations/layer15_attention_weight_min": -38.76657485961914,
      "activations/layer16_attention_weight_max": 40.48429489135742,
      "activations/layer16_attention_weight_min": -28.739192962646484,
      "activations/layer17_attention_weight_max": 53.44496536254883,
      "activations/layer17_attention_weight_min": -35.88398361206055,
      "activations/layer18_attention_weight_max": 57.908634185791016,
      "activations/layer18_attention_weight_min": -33.235687255859375,
      "activations/layer19_attention_weight_max": 21.213489532470703,
      "activations/layer19_attention_weight_min": -14.444220542907715,
      "activations/layer1_attention_weight_max": 16.20176124572754,
      "activations/layer1_attention_weight_min": -14.41572093963623,
      "activations/layer20_attention_weight_max": 21.72733497619629,
      "activations/layer20_attention_weight_min": -17.11507225036621,
      "activations/layer21_attention_weight_max": 41.411869049072266,
      "activations/layer21_attention_weight_min": -20.38819122314453,
      "activations/layer22_attention_weight_max": 33.16500473022461,
      "activations/layer22_attention_weight_min": -19.35841941833496,
      "activations/layer23_attention_weight_max": 22.238134384155273,
      "activations/layer23_attention_weight_min": -14.635257720947266,
      "activations/layer2_attention_weight_max": 27.04450225830078,
      "activations/layer2_attention_weight_min": -24.89130973815918,
      "activations/layer3_attention_weight_max": 61.751590728759766,
      "activations/layer3_attention_weight_min": -63.39035415649414,
      "activations/layer4_attention_weight_max": 91.6499252319336,
      "activations/layer4_attention_weight_min": -83.80802154541016,
      "activations/layer5_attention_weight_max": 129.80975341796875,
      "activations/layer5_attention_weight_min": -108.15182495117188,
      "activations/layer6_attention_weight_max": 60.50825500488281,
      "activations/layer6_attention_weight_min": -55.09275436401367,
      "activations/layer7_attention_weight_max": 58.80180740356445,
      "activations/layer7_attention_weight_min": -55.67194747924805,
      "activations/layer8_attention_weight_max": 52.524261474609375,
      "activations/layer8_attention_weight_min": -44.8041877746582,
      "activations/layer9_attention_weight_max": 72.62686920166016,
      "activations/layer9_attention_weight_min": -63.6434326171875,
      "epoch": 1.79,
      "learning_rate": 0.0001398840909090909,
      "loss": 2.9964,
      "step": 30750
    },
    {
      "activations/layer0_attention_weight_max": 14.164078712463379,
      "activations/layer0_attention_weight_min": -13.652698516845703,
      "activations/layer10_attention_weight_max": 60.50164031982422,
      "activations/layer10_attention_weight_min": -48.5093879699707,
      "activations/layer11_attention_weight_max": 42.89949035644531,
      "activations/layer11_attention_weight_min": -39.01959228515625,
      "activations/layer12_attention_weight_max": 28.631540298461914,
      "activations/layer12_attention_weight_min": -23.960254669189453,
      "activations/layer13_attention_weight_max": 65.3521957397461,
      "activations/layer13_attention_weight_min": -48.10481643676758,
      "activations/layer14_attention_weight_max": 53.988677978515625,
      "activations/layer14_attention_weight_min": -36.76845169067383,
      "activations/layer15_attention_weight_max": 51.313201904296875,
      "activations/layer15_attention_weight_min": -37.60173034667969,
      "activations/layer16_attention_weight_max": 45.66292953491211,
      "activations/layer16_attention_weight_min": -27.803083419799805,
      "activations/layer17_attention_weight_max": 73.8449935913086,
      "activations/layer17_attention_weight_min": -42.03540802001953,
      "activations/layer18_attention_weight_max": 59.40506362915039,
      "activations/layer18_attention_weight_min": -35.63199996948242,
      "activations/layer19_attention_weight_max": 21.58913803100586,
      "activations/layer19_attention_weight_min": -15.923239707946777,
      "activations/layer1_attention_weight_max": 16.523740768432617,
      "activations/layer1_attention_weight_min": -14.617874145507812,
      "activations/layer20_attention_weight_max": 23.436290740966797,
      "activations/layer20_attention_weight_min": -18.147905349731445,
      "activations/layer21_attention_weight_max": 42.783538818359375,
      "activations/layer21_attention_weight_min": -24.360565185546875,
      "activations/layer22_attention_weight_max": 40.93244552612305,
      "activations/layer22_attention_weight_min": -24.384187698364258,
      "activations/layer23_attention_weight_max": 25.829504013061523,
      "activations/layer23_attention_weight_min": -13.942439079284668,
      "activations/layer2_attention_weight_max": 26.134361267089844,
      "activations/layer2_attention_weight_min": -25.506534576416016,
      "activations/layer3_attention_weight_max": 62.01993179321289,
      "activations/layer3_attention_weight_min": -65.52194213867188,
      "activations/layer4_attention_weight_max": 94.0143814086914,
      "activations/layer4_attention_weight_min": -87.21990203857422,
      "activations/layer5_attention_weight_max": 133.675048828125,
      "activations/layer5_attention_weight_min": -114.20838928222656,
      "activations/layer6_attention_weight_max": 64.90794372558594,
      "activations/layer6_attention_weight_min": -54.470375061035156,
      "activations/layer7_attention_weight_max": 63.963157653808594,
      "activations/layer7_attention_weight_min": -55.5130729675293,
      "activations/layer8_attention_weight_max": 50.83517837524414,
      "activations/layer8_attention_weight_min": -43.454097747802734,
      "activations/layer9_attention_weight_max": 68.50704956054688,
      "activations/layer9_attention_weight_min": -63.890228271484375,
      "epoch": 1.79,
      "learning_rate": 0.0001398651515151515,
      "loss": 2.9829,
      "step": 30800
    },
    {
      "activations/layer0_attention_weight_max": 13.004115104675293,
      "activations/layer0_attention_weight_min": -14.13625717163086,
      "activations/layer10_attention_weight_max": 60.42185592651367,
      "activations/layer10_attention_weight_min": -53.195953369140625,
      "activations/layer11_attention_weight_max": 41.15826416015625,
      "activations/layer11_attention_weight_min": -36.87159729003906,
      "activations/layer12_attention_weight_max": 29.043092727661133,
      "activations/layer12_attention_weight_min": -24.710391998291016,
      "activations/layer13_attention_weight_max": 64.88841247558594,
      "activations/layer13_attention_weight_min": -45.15615463256836,
      "activations/layer14_attention_weight_max": 66.49345397949219,
      "activations/layer14_attention_weight_min": -43.54026794433594,
      "activations/layer15_attention_weight_max": 60.369468688964844,
      "activations/layer15_attention_weight_min": -41.29771423339844,
      "activations/layer16_attention_weight_max": 43.87869644165039,
      "activations/layer16_attention_weight_min": -30.745922088623047,
      "activations/layer17_attention_weight_max": 59.96810531616211,
      "activations/layer17_attention_weight_min": -38.78648376464844,
      "activations/layer18_attention_weight_max": 61.058170318603516,
      "activations/layer18_attention_weight_min": -34.464935302734375,
      "activations/layer19_attention_weight_max": 24.419891357421875,
      "activations/layer19_attention_weight_min": -18.528186798095703,
      "activations/layer1_attention_weight_max": 16.759912490844727,
      "activations/layer1_attention_weight_min": -13.6759614944458,
      "activations/layer20_attention_weight_max": 20.963376998901367,
      "activations/layer20_attention_weight_min": -16.567317962646484,
      "activations/layer21_attention_weight_max": 39.5888786315918,
      "activations/layer21_attention_weight_min": -25.457592010498047,
      "activations/layer22_attention_weight_max": 39.366573333740234,
      "activations/layer22_attention_weight_min": -21.624174118041992,
      "activations/layer23_attention_weight_max": 27.78650665283203,
      "activations/layer23_attention_weight_min": -15.62778091430664,
      "activations/layer2_attention_weight_max": 25.56802749633789,
      "activations/layer2_attention_weight_min": -24.994007110595703,
      "activations/layer3_attention_weight_max": 58.44327926635742,
      "activations/layer3_attention_weight_min": -61.229530334472656,
      "activations/layer4_attention_weight_max": 91.00362396240234,
      "activations/layer4_attention_weight_min": -82.40624237060547,
      "activations/layer5_attention_weight_max": 129.9410858154297,
      "activations/layer5_attention_weight_min": -115.27024841308594,
      "activations/layer6_attention_weight_max": 63.44516372680664,
      "activations/layer6_attention_weight_min": -55.023929595947266,
      "activations/layer7_attention_weight_max": 57.214988708496094,
      "activations/layer7_attention_weight_min": -58.2990837097168,
      "activations/layer8_attention_weight_max": 48.976776123046875,
      "activations/layer8_attention_weight_min": -44.4621467590332,
      "activations/layer9_attention_weight_max": 78.12060546875,
      "activations/layer9_attention_weight_min": -62.155799865722656,
      "epoch": 1.79,
      "learning_rate": 0.00013984621212121212,
      "loss": 2.9873,
      "step": 30850
    },
    {
      "activations/layer0_attention_weight_max": 15.279603958129883,
      "activations/layer0_attention_weight_min": -14.640108108520508,
      "activations/layer10_attention_weight_max": 67.83021545410156,
      "activations/layer10_attention_weight_min": -48.626338958740234,
      "activations/layer11_attention_weight_max": 53.80712890625,
      "activations/layer11_attention_weight_min": -39.491058349609375,
      "activations/layer12_attention_weight_max": 31.193103790283203,
      "activations/layer12_attention_weight_min": -24.769514083862305,
      "activations/layer13_attention_weight_max": 92.53614807128906,
      "activations/layer13_attention_weight_min": -57.72348403930664,
      "activations/layer14_attention_weight_max": 81.31254577636719,
      "activations/layer14_attention_weight_min": -44.481666564941406,
      "activations/layer15_attention_weight_max": 67.56522369384766,
      "activations/layer15_attention_weight_min": -38.58975601196289,
      "activations/layer16_attention_weight_max": 51.58788299560547,
      "activations/layer16_attention_weight_min": -27.41587257385254,
      "activations/layer17_attention_weight_max": 80.00313568115234,
      "activations/layer17_attention_weight_min": -39.035888671875,
      "activations/layer18_attention_weight_max": 66.02822875976562,
      "activations/layer18_attention_weight_min": -32.74270248413086,
      "activations/layer19_attention_weight_max": 26.7280330657959,
      "activations/layer19_attention_weight_min": -15.456171989440918,
      "activations/layer1_attention_weight_max": 16.151268005371094,
      "activations/layer1_attention_weight_min": -12.261228561401367,
      "activations/layer20_attention_weight_max": 24.04047393798828,
      "activations/layer20_attention_weight_min": -17.54083251953125,
      "activations/layer21_attention_weight_max": 52.2224006652832,
      "activations/layer21_attention_weight_min": -25.49222755432129,
      "activations/layer22_attention_weight_max": 43.269683837890625,
      "activations/layer22_attention_weight_min": -22.813344955444336,
      "activations/layer23_attention_weight_max": 24.316978454589844,
      "activations/layer23_attention_weight_min": -15.323379516601562,
      "activations/layer2_attention_weight_max": 27.1456298828125,
      "activations/layer2_attention_weight_min": -25.464887619018555,
      "activations/layer3_attention_weight_max": 64.37754821777344,
      "activations/layer3_attention_weight_min": -64.60582733154297,
      "activations/layer4_attention_weight_max": 96.68001556396484,
      "activations/layer4_attention_weight_min": -86.58550262451172,
      "activations/layer5_attention_weight_max": 143.4755096435547,
      "activations/layer5_attention_weight_min": -122.92916870117188,
      "activations/layer6_attention_weight_max": 67.42689514160156,
      "activations/layer6_attention_weight_min": -53.918052673339844,
      "activations/layer7_attention_weight_max": 66.72655487060547,
      "activations/layer7_attention_weight_min": -60.67838668823242,
      "activations/layer8_attention_weight_max": 54.78388595581055,
      "activations/layer8_attention_weight_min": -45.8655891418457,
      "activations/layer9_attention_weight_max": 76.41732025146484,
      "activations/layer9_attention_weight_min": -67.58573913574219,
      "epoch": 1.8,
      "learning_rate": 0.00013982727272727273,
      "loss": 2.9824,
      "step": 30900
    },
    {
      "activations/layer0_attention_weight_max": 13.712739944458008,
      "activations/layer0_attention_weight_min": -13.837960243225098,
      "activations/layer10_attention_weight_max": 59.4404411315918,
      "activations/layer10_attention_weight_min": -50.66923904418945,
      "activations/layer11_attention_weight_max": 40.10346221923828,
      "activations/layer11_attention_weight_min": -34.90929412841797,
      "activations/layer12_attention_weight_max": 30.924726486206055,
      "activations/layer12_attention_weight_min": -29.278120040893555,
      "activations/layer13_attention_weight_max": 49.50825881958008,
      "activations/layer13_attention_weight_min": -43.28656768798828,
      "activations/layer14_attention_weight_max": 54.54248046875,
      "activations/layer14_attention_weight_min": -39.35063934326172,
      "activations/layer15_attention_weight_max": 49.52595520019531,
      "activations/layer15_attention_weight_min": -37.63822555541992,
      "activations/layer16_attention_weight_max": 42.2252082824707,
      "activations/layer16_attention_weight_min": -29.55523681640625,
      "activations/layer17_attention_weight_max": 58.04924774169922,
      "activations/layer17_attention_weight_min": -36.6619758605957,
      "activations/layer18_attention_weight_max": 57.603919982910156,
      "activations/layer18_attention_weight_min": -33.74112319946289,
      "activations/layer19_attention_weight_max": 20.286556243896484,
      "activations/layer19_attention_weight_min": -14.95014476776123,
      "activations/layer1_attention_weight_max": 17.7695255279541,
      "activations/layer1_attention_weight_min": -13.095831871032715,
      "activations/layer20_attention_weight_max": 21.384197235107422,
      "activations/layer20_attention_weight_min": -16.37543487548828,
      "activations/layer21_attention_weight_max": 45.02981185913086,
      "activations/layer21_attention_weight_min": -25.08522605895996,
      "activations/layer22_attention_weight_max": 36.2220344543457,
      "activations/layer22_attention_weight_min": -21.19493865966797,
      "activations/layer23_attention_weight_max": 23.026512145996094,
      "activations/layer23_attention_weight_min": -15.408145904541016,
      "activations/layer2_attention_weight_max": 28.67102813720703,
      "activations/layer2_attention_weight_min": -26.334667205810547,
      "activations/layer3_attention_weight_max": 57.699119567871094,
      "activations/layer3_attention_weight_min": -61.2707405090332,
      "activations/layer4_attention_weight_max": 87.30044555664062,
      "activations/layer4_attention_weight_min": -80.39278411865234,
      "activations/layer5_attention_weight_max": 122.81381225585938,
      "activations/layer5_attention_weight_min": -117.12765502929688,
      "activations/layer6_attention_weight_max": 60.786537170410156,
      "activations/layer6_attention_weight_min": -56.39421463012695,
      "activations/layer7_attention_weight_max": 60.01008605957031,
      "activations/layer7_attention_weight_min": -58.05862808227539,
      "activations/layer8_attention_weight_max": 46.51221466064453,
      "activations/layer8_attention_weight_min": -43.825679779052734,
      "activations/layer9_attention_weight_max": 63.79417037963867,
      "activations/layer9_attention_weight_min": -60.37982940673828,
      "epoch": 1.8,
      "learning_rate": 0.00013980833333333332,
      "loss": 3.0034,
      "step": 30950
    },
    {
      "activations/layer0_attention_weight_max": 14.362992286682129,
      "activations/layer0_attention_weight_min": -14.025066375732422,
      "activations/layer10_attention_weight_max": 55.7967529296875,
      "activations/layer10_attention_weight_min": -49.887298583984375,
      "activations/layer11_attention_weight_max": 46.99950408935547,
      "activations/layer11_attention_weight_min": -37.75716781616211,
      "activations/layer12_attention_weight_max": 29.162382125854492,
      "activations/layer12_attention_weight_min": -23.78909683227539,
      "activations/layer13_attention_weight_max": 57.3907470703125,
      "activations/layer13_attention_weight_min": -43.22513961791992,
      "activations/layer14_attention_weight_max": 63.887699127197266,
      "activations/layer14_attention_weight_min": -36.39088439941406,
      "activations/layer15_attention_weight_max": 57.698944091796875,
      "activations/layer15_attention_weight_min": -40.601646423339844,
      "activations/layer16_attention_weight_max": 50.419349670410156,
      "activations/layer16_attention_weight_min": -30.34513282775879,
      "activations/layer17_attention_weight_max": 62.12833786010742,
      "activations/layer17_attention_weight_min": -37.754451751708984,
      "activations/layer18_attention_weight_max": 65.91806030273438,
      "activations/layer18_attention_weight_min": -34.22917556762695,
      "activations/layer19_attention_weight_max": 22.128467559814453,
      "activations/layer19_attention_weight_min": -15.71216106414795,
      "activations/layer1_attention_weight_max": 15.376084327697754,
      "activations/layer1_attention_weight_min": -12.545717239379883,
      "activations/layer20_attention_weight_max": 20.20194435119629,
      "activations/layer20_attention_weight_min": -17.926490783691406,
      "activations/layer21_attention_weight_max": 49.35001754760742,
      "activations/layer21_attention_weight_min": -27.52983283996582,
      "activations/layer22_attention_weight_max": 38.192466735839844,
      "activations/layer22_attention_weight_min": -20.771526336669922,
      "activations/layer23_attention_weight_max": 23.061717987060547,
      "activations/layer23_attention_weight_min": -13.753143310546875,
      "activations/layer2_attention_weight_max": 27.14351463317871,
      "activations/layer2_attention_weight_min": -26.21902084350586,
      "activations/layer3_attention_weight_max": 61.574729919433594,
      "activations/layer3_attention_weight_min": -65.95545196533203,
      "activations/layer4_attention_weight_max": 92.86458587646484,
      "activations/layer4_attention_weight_min": -80.73394775390625,
      "activations/layer5_attention_weight_max": 124.55290985107422,
      "activations/layer5_attention_weight_min": -111.08038330078125,
      "activations/layer6_attention_weight_max": 64.85330963134766,
      "activations/layer6_attention_weight_min": -51.56342697143555,
      "activations/layer7_attention_weight_max": 61.007659912109375,
      "activations/layer7_attention_weight_min": -52.851112365722656,
      "activations/layer8_attention_weight_max": 46.78480911254883,
      "activations/layer8_attention_weight_min": -40.80818557739258,
      "activations/layer9_attention_weight_max": 62.92975616455078,
      "activations/layer9_attention_weight_min": -56.914424896240234,
      "epoch": 1.8,
      "learning_rate": 0.00013978939393939394,
      "loss": 2.9812,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_loss": 2.93359375,
      "eval_runtime": 8.4546,
      "eval_samples_per_second": 507.888,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_openwebtext_loss": 2.93359375,
      "eval_openwebtext_ppl": 18.79505399780919,
      "eval_openwebtext_runtime": 8.4546,
      "eval_openwebtext_samples_per_second": 507.888,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_wikitext_loss": 3.201171875,
      "eval_wikitext_ppl": 24.561296107667808,
      "eval_wikitext_runtime": 1.9616,
      "eval_wikitext_samples_per_second": 232.466,
      "step": 31000
    },
    {
      "epoch": 1.8,
      "eval_lambada_loss": 3.19140625,
      "eval_lambada_ppl": 24.322607071104724,
      "eval_lambada_runtime": 9.5374,
      "eval_lambada_samples_per_second": 510.518,
      "step": 31000
    },
    {
      "activations/layer0_attention_weight_max": 14.900808334350586,
      "activations/layer0_attention_weight_min": -14.030317306518555,
      "activations/layer10_attention_weight_max": 67.5546875,
      "activations/layer10_attention_weight_min": -57.974876403808594,
      "activations/layer11_attention_weight_max": 44.63377380371094,
      "activations/layer11_attention_weight_min": -37.98847198486328,
      "activations/layer12_attention_weight_max": 25.4674015045166,
      "activations/layer12_attention_weight_min": -24.636714935302734,
      "activations/layer13_attention_weight_max": 62.3812255859375,
      "activations/layer13_attention_weight_min": -45.47842025756836,
      "activations/layer14_attention_weight_max": 63.68254470825195,
      "activations/layer14_attention_weight_min": -45.26118850708008,
      "activations/layer15_attention_weight_max": 57.194740295410156,
      "activations/layer15_attention_weight_min": -39.05311584472656,
      "activations/layer16_attention_weight_max": 46.78293228149414,
      "activations/layer16_attention_weight_min": -32.68975830078125,
      "activations/layer17_attention_weight_max": 61.75834274291992,
      "activations/layer17_attention_weight_min": -36.767269134521484,
      "activations/layer18_attention_weight_max": 58.57439422607422,
      "activations/layer18_attention_weight_min": -32.9276008605957,
      "activations/layer19_attention_weight_max": 21.35063362121582,
      "activations/layer19_attention_weight_min": -15.494972229003906,
      "activations/layer1_attention_weight_max": 15.612409591674805,
      "activations/layer1_attention_weight_min": -13.695662498474121,
      "activations/layer20_attention_weight_max": 20.507732391357422,
      "activations/layer20_attention_weight_min": -15.385159492492676,
      "activations/layer21_attention_weight_max": 39.826942443847656,
      "activations/layer21_attention_weight_min": -24.44230842590332,
      "activations/layer22_attention_weight_max": 37.25540542602539,
      "activations/layer22_attention_weight_min": -22.66558074951172,
      "activations/layer23_attention_weight_max": 23.507909774780273,
      "activations/layer23_attention_weight_min": -14.069393157958984,
      "activations/layer2_attention_weight_max": 27.785228729248047,
      "activations/layer2_attention_weight_min": -25.93063735961914,
      "activations/layer3_attention_weight_max": 63.7131233215332,
      "activations/layer3_attention_weight_min": -65.68160247802734,
      "activations/layer4_attention_weight_max": 89.19086456298828,
      "activations/layer4_attention_weight_min": -81.35980987548828,
      "activations/layer5_attention_weight_max": 128.65133666992188,
      "activations/layer5_attention_weight_min": -112.00686645507812,
      "activations/layer6_attention_weight_max": 61.46281814575195,
      "activations/layer6_attention_weight_min": -50.36888885498047,
      "activations/layer7_attention_weight_max": 57.57695007324219,
      "activations/layer7_attention_weight_min": -51.83244323730469,
      "activations/layer8_attention_weight_max": 47.74452209472656,
      "activations/layer8_attention_weight_min": -39.64613723754883,
      "activations/layer9_attention_weight_max": 61.30939865112305,
      "activations/layer9_attention_weight_min": -60.063575744628906,
      "epoch": 1.8,
      "learning_rate": 0.00013977045454545455,
      "loss": 2.9827,
      "step": 31050
    },
    {
      "activations/layer0_attention_weight_max": 17.19507598876953,
      "activations/layer0_attention_weight_min": -13.837044715881348,
      "activations/layer10_attention_weight_max": 63.132713317871094,
      "activations/layer10_attention_weight_min": -51.999629974365234,
      "activations/layer11_attention_weight_max": 47.895408630371094,
      "activations/layer11_attention_weight_min": -40.43449020385742,
      "activations/layer12_attention_weight_max": 28.979333877563477,
      "activations/layer12_attention_weight_min": -25.618406295776367,
      "activations/layer13_attention_weight_max": 65.52235412597656,
      "activations/layer13_attention_weight_min": -49.20292282104492,
      "activations/layer14_attention_weight_max": 59.475521087646484,
      "activations/layer14_attention_weight_min": -39.88272476196289,
      "activations/layer15_attention_weight_max": 53.83461380004883,
      "activations/layer15_attention_weight_min": -37.6181755065918,
      "activations/layer16_attention_weight_max": 43.49979019165039,
      "activations/layer16_attention_weight_min": -30.879283905029297,
      "activations/layer17_attention_weight_max": 78.3912353515625,
      "activations/layer17_attention_weight_min": -39.20857238769531,
      "activations/layer18_attention_weight_max": 60.45615005493164,
      "activations/layer18_attention_weight_min": -35.20839309692383,
      "activations/layer19_attention_weight_max": 24.827014923095703,
      "activations/layer19_attention_weight_min": -17.019126892089844,
      "activations/layer1_attention_weight_max": 17.04610824584961,
      "activations/layer1_attention_weight_min": -12.010987281799316,
      "activations/layer20_attention_weight_max": 23.618236541748047,
      "activations/layer20_attention_weight_min": -17.441335678100586,
      "activations/layer21_attention_weight_max": 44.791446685791016,
      "activations/layer21_attention_weight_min": -22.370895385742188,
      "activations/layer22_attention_weight_max": 41.354461669921875,
      "activations/layer22_attention_weight_min": -22.122629165649414,
      "activations/layer23_attention_weight_max": 24.21042251586914,
      "activations/layer23_attention_weight_min": -16.140918731689453,
      "activations/layer2_attention_weight_max": 27.68910026550293,
      "activations/layer2_attention_weight_min": -25.581520080566406,
      "activations/layer3_attention_weight_max": 60.2209358215332,
      "activations/layer3_attention_weight_min": -63.58932876586914,
      "activations/layer4_attention_weight_max": 91.66211700439453,
      "activations/layer4_attention_weight_min": -79.99633026123047,
      "activations/layer5_attention_weight_max": 120.02359008789062,
      "activations/layer5_attention_weight_min": -112.74748992919922,
      "activations/layer6_attention_weight_max": 58.37886428833008,
      "activations/layer6_attention_weight_min": -51.61795425415039,
      "activations/layer7_attention_weight_max": 55.42814254760742,
      "activations/layer7_attention_weight_min": -59.32688903808594,
      "activations/layer8_attention_weight_max": 48.54998779296875,
      "activations/layer8_attention_weight_min": -46.95790481567383,
      "activations/layer9_attention_weight_max": 80.39694213867188,
      "activations/layer9_attention_weight_min": -66.7623291015625,
      "epoch": 1.81,
      "learning_rate": 0.00013975151515151514,
      "loss": 2.9878,
      "step": 31100
    },
    {
      "activations/layer0_attention_weight_max": 14.388057708740234,
      "activations/layer0_attention_weight_min": -14.076590538024902,
      "activations/layer10_attention_weight_max": 72.69615936279297,
      "activations/layer10_attention_weight_min": -57.67247009277344,
      "activations/layer11_attention_weight_max": 48.27477264404297,
      "activations/layer11_attention_weight_min": -33.66429901123047,
      "activations/layer12_attention_weight_max": 34.87996292114258,
      "activations/layer12_attention_weight_min": -27.918521881103516,
      "activations/layer13_attention_weight_max": 54.76655578613281,
      "activations/layer13_attention_weight_min": -43.52849578857422,
      "activations/layer14_attention_weight_max": 56.19974899291992,
      "activations/layer14_attention_weight_min": -37.87704086303711,
      "activations/layer15_attention_weight_max": 57.39705276489258,
      "activations/layer15_attention_weight_min": -39.91938018798828,
      "activations/layer16_attention_weight_max": 47.30710220336914,
      "activations/layer16_attention_weight_min": -31.78209686279297,
      "activations/layer17_attention_weight_max": 58.11119842529297,
      "activations/layer17_attention_weight_min": -38.61003112792969,
      "activations/layer18_attention_weight_max": 62.66108322143555,
      "activations/layer18_attention_weight_min": -35.550376892089844,
      "activations/layer19_attention_weight_max": 23.793161392211914,
      "activations/layer19_attention_weight_min": -17.01112937927246,
      "activations/layer1_attention_weight_max": 15.769940376281738,
      "activations/layer1_attention_weight_min": -13.293743133544922,
      "activations/layer20_attention_weight_max": 20.40770721435547,
      "activations/layer20_attention_weight_min": -15.43525505065918,
      "activations/layer21_attention_weight_max": 49.263397216796875,
      "activations/layer21_attention_weight_min": -24.547557830810547,
      "activations/layer22_attention_weight_max": 40.50825881958008,
      "activations/layer22_attention_weight_min": -21.16420555114746,
      "activations/layer23_attention_weight_max": 23.545780181884766,
      "activations/layer23_attention_weight_min": -15.59298324584961,
      "activations/layer2_attention_weight_max": 27.2932186126709,
      "activations/layer2_attention_weight_min": -25.673259735107422,
      "activations/layer3_attention_weight_max": 58.39518737792969,
      "activations/layer3_attention_weight_min": -62.285865783691406,
      "activations/layer4_attention_weight_max": 90.66465759277344,
      "activations/layer4_attention_weight_min": -82.4638900756836,
      "activations/layer5_attention_weight_max": 127.74373626708984,
      "activations/layer5_attention_weight_min": -106.40858459472656,
      "activations/layer6_attention_weight_max": 61.91789245605469,
      "activations/layer6_attention_weight_min": -52.362159729003906,
      "activations/layer7_attention_weight_max": 59.9594612121582,
      "activations/layer7_attention_weight_min": -53.20981979370117,
      "activations/layer8_attention_weight_max": 46.98957061767578,
      "activations/layer8_attention_weight_min": -39.86758804321289,
      "activations/layer9_attention_weight_max": 66.5774154663086,
      "activations/layer9_attention_weight_min": -56.52323532104492,
      "epoch": 1.81,
      "learning_rate": 0.00013973257575757575,
      "loss": 2.9622,
      "step": 31150
    },
    {
      "activations/layer0_attention_weight_max": 15.464163780212402,
      "activations/layer0_attention_weight_min": -13.655084609985352,
      "activations/layer10_attention_weight_max": 56.633018493652344,
      "activations/layer10_attention_weight_min": -47.10419845581055,
      "activations/layer11_attention_weight_max": 40.7613525390625,
      "activations/layer11_attention_weight_min": -39.03999710083008,
      "activations/layer12_attention_weight_max": 31.442649841308594,
      "activations/layer12_attention_weight_min": -27.16771697998047,
      "activations/layer13_attention_weight_max": 63.29039764404297,
      "activations/layer13_attention_weight_min": -56.97923278808594,
      "activations/layer14_attention_weight_max": 50.27595901489258,
      "activations/layer14_attention_weight_min": -44.959781646728516,
      "activations/layer15_attention_weight_max": 58.135650634765625,
      "activations/layer15_attention_weight_min": -46.14146041870117,
      "activations/layer16_attention_weight_max": 41.67985153198242,
      "activations/layer16_attention_weight_min": -31.832300186157227,
      "activations/layer17_attention_weight_max": 65.05366516113281,
      "activations/layer17_attention_weight_min": -44.4690055847168,
      "activations/layer18_attention_weight_max": 60.040863037109375,
      "activations/layer18_attention_weight_min": -33.444984436035156,
      "activations/layer19_attention_weight_max": 21.22577476501465,
      "activations/layer19_attention_weight_min": -16.120386123657227,
      "activations/layer1_attention_weight_max": 15.790359497070312,
      "activations/layer1_attention_weight_min": -13.316559791564941,
      "activations/layer20_attention_weight_max": 19.22208023071289,
      "activations/layer20_attention_weight_min": -17.70773696899414,
      "activations/layer21_attention_weight_max": 38.75533676147461,
      "activations/layer21_attention_weight_min": -24.894811630249023,
      "activations/layer22_attention_weight_max": 36.47745132446289,
      "activations/layer22_attention_weight_min": -25.639482498168945,
      "activations/layer23_attention_weight_max": 22.553627014160156,
      "activations/layer23_attention_weight_min": -15.098516464233398,
      "activations/layer2_attention_weight_max": 28.23201560974121,
      "activations/layer2_attention_weight_min": -26.733081817626953,
      "activations/layer3_attention_weight_max": 59.372413635253906,
      "activations/layer3_attention_weight_min": -62.12189483642578,
      "activations/layer4_attention_weight_max": 88.29598236083984,
      "activations/layer4_attention_weight_min": -80.6246566772461,
      "activations/layer5_attention_weight_max": 124.90950012207031,
      "activations/layer5_attention_weight_min": -116.37139892578125,
      "activations/layer6_attention_weight_max": 59.03303146362305,
      "activations/layer6_attention_weight_min": -54.58100128173828,
      "activations/layer7_attention_weight_max": 62.58761978149414,
      "activations/layer7_attention_weight_min": -55.34846115112305,
      "activations/layer8_attention_weight_max": 51.54683303833008,
      "activations/layer8_attention_weight_min": -46.74361801147461,
      "activations/layer9_attention_weight_max": 64.53511047363281,
      "activations/layer9_attention_weight_min": -62.03396224975586,
      "epoch": 1.81,
      "learning_rate": 0.00013971363636363634,
      "loss": 2.9788,
      "step": 31200
    },
    {
      "activations/layer0_attention_weight_max": 14.898770332336426,
      "activations/layer0_attention_weight_min": -13.724725723266602,
      "activations/layer10_attention_weight_max": 65.91461944580078,
      "activations/layer10_attention_weight_min": -50.96779251098633,
      "activations/layer11_attention_weight_max": 43.90093231201172,
      "activations/layer11_attention_weight_min": -36.287681579589844,
      "activations/layer12_attention_weight_max": 27.283628463745117,
      "activations/layer12_attention_weight_min": -24.07790756225586,
      "activations/layer13_attention_weight_max": 66.82659912109375,
      "activations/layer13_attention_weight_min": -45.21492004394531,
      "activations/layer14_attention_weight_max": 58.33928680419922,
      "activations/layer14_attention_weight_min": -37.906837463378906,
      "activations/layer15_attention_weight_max": 57.74657440185547,
      "activations/layer15_attention_weight_min": -38.10446548461914,
      "activations/layer16_attention_weight_max": 44.99372482299805,
      "activations/layer16_attention_weight_min": -28.120512008666992,
      "activations/layer17_attention_weight_max": 64.3739242553711,
      "activations/layer17_attention_weight_min": -38.950130462646484,
      "activations/layer18_attention_weight_max": 58.69831466674805,
      "activations/layer18_attention_weight_min": -37.5964469909668,
      "activations/layer19_attention_weight_max": 20.91451072692871,
      "activations/layer19_attention_weight_min": -14.852691650390625,
      "activations/layer1_attention_weight_max": 17.186588287353516,
      "activations/layer1_attention_weight_min": -12.855062484741211,
      "activations/layer20_attention_weight_max": 20.03765106201172,
      "activations/layer20_attention_weight_min": -18.865985870361328,
      "activations/layer21_attention_weight_max": 38.57365798950195,
      "activations/layer21_attention_weight_min": -24.883758544921875,
      "activations/layer22_attention_weight_max": 37.295772552490234,
      "activations/layer22_attention_weight_min": -21.260255813598633,
      "activations/layer23_attention_weight_max": 26.941207885742188,
      "activations/layer23_attention_weight_min": -15.31466293334961,
      "activations/layer2_attention_weight_max": 27.32989501953125,
      "activations/layer2_attention_weight_min": -26.55740737915039,
      "activations/layer3_attention_weight_max": 57.1924934387207,
      "activations/layer3_attention_weight_min": -61.996482849121094,
      "activations/layer4_attention_weight_max": 87.87940216064453,
      "activations/layer4_attention_weight_min": -80.11441802978516,
      "activations/layer5_attention_weight_max": 120.56980895996094,
      "activations/layer5_attention_weight_min": -104.29592895507812,
      "activations/layer6_attention_weight_max": 59.57582092285156,
      "activations/layer6_attention_weight_min": -51.78252410888672,
      "activations/layer7_attention_weight_max": 60.075843811035156,
      "activations/layer7_attention_weight_min": -53.36357116699219,
      "activations/layer8_attention_weight_max": 47.001338958740234,
      "activations/layer8_attention_weight_min": -42.621158599853516,
      "activations/layer9_attention_weight_max": 65.48299407958984,
      "activations/layer9_attention_weight_min": -57.21692657470703,
      "epoch": 1.82,
      "learning_rate": 0.00013969469696969696,
      "loss": 2.998,
      "step": 31250
    },
    {
      "activations/layer0_attention_weight_max": 13.890684127807617,
      "activations/layer0_attention_weight_min": -13.94322395324707,
      "activations/layer10_attention_weight_max": 60.73228073120117,
      "activations/layer10_attention_weight_min": -49.29960632324219,
      "activations/layer11_attention_weight_max": 44.00651550292969,
      "activations/layer11_attention_weight_min": -34.33631896972656,
      "activations/layer12_attention_weight_max": 36.43220901489258,
      "activations/layer12_attention_weight_min": -29.409135818481445,
      "activations/layer13_attention_weight_max": 52.942955017089844,
      "activations/layer13_attention_weight_min": -36.7464599609375,
      "activations/layer14_attention_weight_max": 49.79216384887695,
      "activations/layer14_attention_weight_min": -32.345054626464844,
      "activations/layer15_attention_weight_max": 57.87812423706055,
      "activations/layer15_attention_weight_min": -38.8250846862793,
      "activations/layer16_attention_weight_max": 53.18294143676758,
      "activations/layer16_attention_weight_min": -30.382978439331055,
      "activations/layer17_attention_weight_max": 60.59532928466797,
      "activations/layer17_attention_weight_min": -37.92972183227539,
      "activations/layer18_attention_weight_max": 64.99470520019531,
      "activations/layer18_attention_weight_min": -34.61411666870117,
      "activations/layer19_attention_weight_max": 19.858816146850586,
      "activations/layer19_attention_weight_min": -16.702428817749023,
      "activations/layer1_attention_weight_max": 15.892230033874512,
      "activations/layer1_attention_weight_min": -12.782403945922852,
      "activations/layer20_attention_weight_max": 20.519290924072266,
      "activations/layer20_attention_weight_min": -16.14881706237793,
      "activations/layer21_attention_weight_max": 39.33280944824219,
      "activations/layer21_attention_weight_min": -22.648208618164062,
      "activations/layer22_attention_weight_max": 35.8029670715332,
      "activations/layer22_attention_weight_min": -21.284761428833008,
      "activations/layer23_attention_weight_max": 22.591266632080078,
      "activations/layer23_attention_weight_min": -14.532198905944824,
      "activations/layer2_attention_weight_max": 26.484901428222656,
      "activations/layer2_attention_weight_min": -24.555267333984375,
      "activations/layer3_attention_weight_max": 56.49637985229492,
      "activations/layer3_attention_weight_min": -59.8275146484375,
      "activations/layer4_attention_weight_max": 90.65977478027344,
      "activations/layer4_attention_weight_min": -80.6461181640625,
      "activations/layer5_attention_weight_max": 119.90048217773438,
      "activations/layer5_attention_weight_min": -104.24190521240234,
      "activations/layer6_attention_weight_max": 58.58624267578125,
      "activations/layer6_attention_weight_min": -50.15903091430664,
      "activations/layer7_attention_weight_max": 56.744293212890625,
      "activations/layer7_attention_weight_min": -50.46885299682617,
      "activations/layer8_attention_weight_max": 45.1456413269043,
      "activations/layer8_attention_weight_min": -41.40741729736328,
      "activations/layer9_attention_weight_max": 67.22676849365234,
      "activations/layer9_attention_weight_min": -59.78261184692383,
      "epoch": 1.82,
      "learning_rate": 0.00013967575757575757,
      "loss": 2.9878,
      "step": 31300
    },
    {
      "activations/layer0_attention_weight_max": 15.617734909057617,
      "activations/layer0_attention_weight_min": -14.006875038146973,
      "activations/layer10_attention_weight_max": 59.78216552734375,
      "activations/layer10_attention_weight_min": -51.35568618774414,
      "activations/layer11_attention_weight_max": 49.31708526611328,
      "activations/layer11_attention_weight_min": -35.377830505371094,
      "activations/layer12_attention_weight_max": 32.55956268310547,
      "activations/layer12_attention_weight_min": -24.33631706237793,
      "activations/layer13_attention_weight_max": 58.587799072265625,
      "activations/layer13_attention_weight_min": -42.46641540527344,
      "activations/layer14_attention_weight_max": 59.83787155151367,
      "activations/layer14_attention_weight_min": -39.030853271484375,
      "activations/layer15_attention_weight_max": 56.942291259765625,
      "activations/layer15_attention_weight_min": -40.759056091308594,
      "activations/layer16_attention_weight_max": 43.55771255493164,
      "activations/layer16_attention_weight_min": -30.146276473999023,
      "activations/layer17_attention_weight_max": 61.49028396606445,
      "activations/layer17_attention_weight_min": -38.99967956542969,
      "activations/layer18_attention_weight_max": 57.87412643432617,
      "activations/layer18_attention_weight_min": -34.332489013671875,
      "activations/layer19_attention_weight_max": 20.01862907409668,
      "activations/layer19_attention_weight_min": -14.722183227539062,
      "activations/layer1_attention_weight_max": 15.769248008728027,
      "activations/layer1_attention_weight_min": -14.779275894165039,
      "activations/layer20_attention_weight_max": 19.34963607788086,
      "activations/layer20_attention_weight_min": -15.527008056640625,
      "activations/layer21_attention_weight_max": 39.27823257446289,
      "activations/layer21_attention_weight_min": -23.718772888183594,
      "activations/layer22_attention_weight_max": 34.24565505981445,
      "activations/layer22_attention_weight_min": -21.14454460144043,
      "activations/layer23_attention_weight_max": 22.969833374023438,
      "activations/layer23_attention_weight_min": -14.471195220947266,
      "activations/layer2_attention_weight_max": 26.96087646484375,
      "activations/layer2_attention_weight_min": -27.9261531829834,
      "activations/layer3_attention_weight_max": 60.07643508911133,
      "activations/layer3_attention_weight_min": -63.400367736816406,
      "activations/layer4_attention_weight_max": 90.02877044677734,
      "activations/layer4_attention_weight_min": -81.52482604980469,
      "activations/layer5_attention_weight_max": 128.2200469970703,
      "activations/layer5_attention_weight_min": -111.14251708984375,
      "activations/layer6_attention_weight_max": 61.41305923461914,
      "activations/layer6_attention_weight_min": -54.84004592895508,
      "activations/layer7_attention_weight_max": 59.758975982666016,
      "activations/layer7_attention_weight_min": -54.35280227661133,
      "activations/layer8_attention_weight_max": 49.66709899902344,
      "activations/layer8_attention_weight_min": -43.39580535888672,
      "activations/layer9_attention_weight_max": 64.41085052490234,
      "activations/layer9_attention_weight_min": -54.26107406616211,
      "epoch": 1.82,
      "learning_rate": 0.00013965681818181816,
      "loss": 3.0016,
      "step": 31350
    },
    {
      "activations/layer0_attention_weight_max": 15.669140815734863,
      "activations/layer0_attention_weight_min": -14.006707191467285,
      "activations/layer10_attention_weight_max": 54.859676361083984,
      "activations/layer10_attention_weight_min": -51.573787689208984,
      "activations/layer11_attention_weight_max": 37.16401672363281,
      "activations/layer11_attention_weight_min": -34.60819625854492,
      "activations/layer12_attention_weight_max": 34.71073532104492,
      "activations/layer12_attention_weight_min": -26.153175354003906,
      "activations/layer13_attention_weight_max": 53.00725173950195,
      "activations/layer13_attention_weight_min": -41.43134307861328,
      "activations/layer14_attention_weight_max": 60.08072280883789,
      "activations/layer14_attention_weight_min": -46.18893051147461,
      "activations/layer15_attention_weight_max": 50.54531478881836,
      "activations/layer15_attention_weight_min": -37.795494079589844,
      "activations/layer16_attention_weight_max": 41.20417785644531,
      "activations/layer16_attention_weight_min": -26.99199676513672,
      "activations/layer17_attention_weight_max": 54.424957275390625,
      "activations/layer17_attention_weight_min": -35.56907653808594,
      "activations/layer18_attention_weight_max": 57.237762451171875,
      "activations/layer18_attention_weight_min": -33.08066177368164,
      "activations/layer19_attention_weight_max": 20.25849151611328,
      "activations/layer19_attention_weight_min": -16.16006851196289,
      "activations/layer1_attention_weight_max": 16.675127029418945,
      "activations/layer1_attention_weight_min": -13.883709907531738,
      "activations/layer20_attention_weight_max": 22.17791748046875,
      "activations/layer20_attention_weight_min": -16.400632858276367,
      "activations/layer21_attention_weight_max": 36.744285583496094,
      "activations/layer21_attention_weight_min": -22.998037338256836,
      "activations/layer22_attention_weight_max": 37.44258499145508,
      "activations/layer22_attention_weight_min": -21.85175323486328,
      "activations/layer23_attention_weight_max": 23.167377471923828,
      "activations/layer23_attention_weight_min": -13.687701225280762,
      "activations/layer2_attention_weight_max": 27.60719871520996,
      "activations/layer2_attention_weight_min": -27.690000534057617,
      "activations/layer3_attention_weight_max": 64.251708984375,
      "activations/layer3_attention_weight_min": -70.68865966796875,
      "activations/layer4_attention_weight_max": 92.3507308959961,
      "activations/layer4_attention_weight_min": -85.33885192871094,
      "activations/layer5_attention_weight_max": 127.31391906738281,
      "activations/layer5_attention_weight_min": -107.38275146484375,
      "activations/layer6_attention_weight_max": 60.815521240234375,
      "activations/layer6_attention_weight_min": -54.25376510620117,
      "activations/layer7_attention_weight_max": 57.87514114379883,
      "activations/layer7_attention_weight_min": -53.84621810913086,
      "activations/layer8_attention_weight_max": 45.0278205871582,
      "activations/layer8_attention_weight_min": -46.289093017578125,
      "activations/layer9_attention_weight_max": 60.77775192260742,
      "activations/layer9_attention_weight_min": -55.75410079956055,
      "epoch": 1.82,
      "learning_rate": 0.00013963787878787878,
      "loss": 2.9933,
      "step": 31400
    },
    {
      "activations/layer0_attention_weight_max": 14.05608081817627,
      "activations/layer0_attention_weight_min": -13.538705825805664,
      "activations/layer10_attention_weight_max": 71.62376403808594,
      "activations/layer10_attention_weight_min": -51.27936935424805,
      "activations/layer11_attention_weight_max": 50.009620666503906,
      "activations/layer11_attention_weight_min": -35.32215118408203,
      "activations/layer12_attention_weight_max": 29.27507209777832,
      "activations/layer12_attention_weight_min": -30.17943572998047,
      "activations/layer13_attention_weight_max": 70.03480529785156,
      "activations/layer13_attention_weight_min": -41.27302551269531,
      "activations/layer14_attention_weight_max": 67.84099578857422,
      "activations/layer14_attention_weight_min": -38.79991912841797,
      "activations/layer15_attention_weight_max": 52.58982849121094,
      "activations/layer15_attention_weight_min": -37.27413558959961,
      "activations/layer16_attention_weight_max": 40.27864074707031,
      "activations/layer16_attention_weight_min": -31.126270294189453,
      "activations/layer17_attention_weight_max": 54.189483642578125,
      "activations/layer17_attention_weight_min": -39.40327835083008,
      "activations/layer18_attention_weight_max": 59.661338806152344,
      "activations/layer18_attention_weight_min": -36.73270797729492,
      "activations/layer19_attention_weight_max": 23.926969528198242,
      "activations/layer19_attention_weight_min": -15.341534614562988,
      "activations/layer1_attention_weight_max": 16.396390914916992,
      "activations/layer1_attention_weight_min": -11.41876220703125,
      "activations/layer20_attention_weight_max": 19.636381149291992,
      "activations/layer20_attention_weight_min": -17.105236053466797,
      "activations/layer21_attention_weight_max": 42.65689468383789,
      "activations/layer21_attention_weight_min": -27.798179626464844,
      "activations/layer22_attention_weight_max": 33.4667854309082,
      "activations/layer22_attention_weight_min": -22.39086151123047,
      "activations/layer23_attention_weight_max": 20.924245834350586,
      "activations/layer23_attention_weight_min": -13.973017692565918,
      "activations/layer2_attention_weight_max": 26.677818298339844,
      "activations/layer2_attention_weight_min": -24.468677520751953,
      "activations/layer3_attention_weight_max": 61.39382553100586,
      "activations/layer3_attention_weight_min": -66.64250183105469,
      "activations/layer4_attention_weight_max": 93.16631317138672,
      "activations/layer4_attention_weight_min": -84.76244354248047,
      "activations/layer5_attention_weight_max": 136.46954345703125,
      "activations/layer5_attention_weight_min": -108.51136779785156,
      "activations/layer6_attention_weight_max": 65.0439682006836,
      "activations/layer6_attention_weight_min": -52.252296447753906,
      "activations/layer7_attention_weight_max": 63.18392562866211,
      "activations/layer7_attention_weight_min": -59.73920440673828,
      "activations/layer8_attention_weight_max": 64.64151763916016,
      "activations/layer8_attention_weight_min": -46.45548629760742,
      "activations/layer9_attention_weight_max": 82.49358367919922,
      "activations/layer9_attention_weight_min": -63.20978927612305,
      "epoch": 1.83,
      "learning_rate": 0.00013961931818181817,
      "loss": 2.9853,
      "step": 31450
    },
    {
      "activations/layer0_attention_weight_max": 14.449697494506836,
      "activations/layer0_attention_weight_min": -13.3577241897583,
      "activations/layer10_attention_weight_max": 58.562557220458984,
      "activations/layer10_attention_weight_min": -49.800262451171875,
      "activations/layer11_attention_weight_max": 45.65393829345703,
      "activations/layer11_attention_weight_min": -35.147926330566406,
      "activations/layer12_attention_weight_max": 26.02665138244629,
      "activations/layer12_attention_weight_min": -25.807308197021484,
      "activations/layer13_attention_weight_max": 68.59777069091797,
      "activations/layer13_attention_weight_min": -42.074745178222656,
      "activations/layer14_attention_weight_max": 42.8679084777832,
      "activations/layer14_attention_weight_min": -30.798959732055664,
      "activations/layer15_attention_weight_max": 54.1055908203125,
      "activations/layer15_attention_weight_min": -38.732810974121094,
      "activations/layer16_attention_weight_max": 49.41309356689453,
      "activations/layer16_attention_weight_min": -30.473169326782227,
      "activations/layer17_attention_weight_max": 63.71457290649414,
      "activations/layer17_attention_weight_min": -36.395973205566406,
      "activations/layer18_attention_weight_max": 57.92516326904297,
      "activations/layer18_attention_weight_min": -30.692556381225586,
      "activations/layer19_attention_weight_max": 22.145143508911133,
      "activations/layer19_attention_weight_min": -15.51232624053955,
      "activations/layer1_attention_weight_max": 17.139360427856445,
      "activations/layer1_attention_weight_min": -14.461280822753906,
      "activations/layer20_attention_weight_max": 21.77391242980957,
      "activations/layer20_attention_weight_min": -17.60109519958496,
      "activations/layer21_attention_weight_max": 41.258052825927734,
      "activations/layer21_attention_weight_min": -20.023712158203125,
      "activations/layer22_attention_weight_max": 36.73038864135742,
      "activations/layer22_attention_weight_min": -19.726348876953125,
      "activations/layer23_attention_weight_max": 22.756467819213867,
      "activations/layer23_attention_weight_min": -15.551952362060547,
      "activations/layer2_attention_weight_max": 27.387853622436523,
      "activations/layer2_attention_weight_min": -26.030075073242188,
      "activations/layer3_attention_weight_max": 62.20290756225586,
      "activations/layer3_attention_weight_min": -65.54252624511719,
      "activations/layer4_attention_weight_max": 91.95039367675781,
      "activations/layer4_attention_weight_min": -85.28912353515625,
      "activations/layer5_attention_weight_max": 125.61409759521484,
      "activations/layer5_attention_weight_min": -110.4072265625,
      "activations/layer6_attention_weight_max": 60.49502944946289,
      "activations/layer6_attention_weight_min": -54.44182586669922,
      "activations/layer7_attention_weight_max": 61.82850646972656,
      "activations/layer7_attention_weight_min": -51.96440124511719,
      "activations/layer8_attention_weight_max": 50.368839263916016,
      "activations/layer8_attention_weight_min": -43.99684524536133,
      "activations/layer9_attention_weight_max": 68.56643676757812,
      "activations/layer9_attention_weight_min": -57.89075469970703,
      "epoch": 1.83,
      "learning_rate": 0.00013960037878787878,
      "loss": 2.9864,
      "step": 31500
    },
    {
      "activations/layer0_attention_weight_max": 14.346274375915527,
      "activations/layer0_attention_weight_min": -13.369829177856445,
      "activations/layer10_attention_weight_max": 63.86641311645508,
      "activations/layer10_attention_weight_min": -59.024261474609375,
      "activations/layer11_attention_weight_max": 39.002498626708984,
      "activations/layer11_attention_weight_min": -34.169761657714844,
      "activations/layer12_attention_weight_max": 31.267820358276367,
      "activations/layer12_attention_weight_min": -27.34107780456543,
      "activations/layer13_attention_weight_max": 64.7795639038086,
      "activations/layer13_attention_weight_min": -51.29527282714844,
      "activations/layer14_attention_weight_max": 49.338260650634766,
      "activations/layer14_attention_weight_min": -43.410552978515625,
      "activations/layer15_attention_weight_max": 52.87725830078125,
      "activations/layer15_attention_weight_min": -40.139278411865234,
      "activations/layer16_attention_weight_max": 50.370914459228516,
      "activations/layer16_attention_weight_min": -28.47740364074707,
      "activations/layer17_attention_weight_max": 61.40007781982422,
      "activations/layer17_attention_weight_min": -39.12376022338867,
      "activations/layer18_attention_weight_max": 61.489707946777344,
      "activations/layer18_attention_weight_min": -35.27782440185547,
      "activations/layer19_attention_weight_max": 21.713102340698242,
      "activations/layer19_attention_weight_min": -13.573390007019043,
      "activations/layer1_attention_weight_max": 16.02106285095215,
      "activations/layer1_attention_weight_min": -13.51809310913086,
      "activations/layer20_attention_weight_max": 17.622142791748047,
      "activations/layer20_attention_weight_min": -20.330129623413086,
      "activations/layer21_attention_weight_max": 43.032222747802734,
      "activations/layer21_attention_weight_min": -24.349943161010742,
      "activations/layer22_attention_weight_max": 42.235801696777344,
      "activations/layer22_attention_weight_min": -22.306936264038086,
      "activations/layer23_attention_weight_max": 22.869171142578125,
      "activations/layer23_attention_weight_min": -16.65500259399414,
      "activations/layer2_attention_weight_max": 25.305461883544922,
      "activations/layer2_attention_weight_min": -25.320541381835938,
      "activations/layer3_attention_weight_max": 57.81890106201172,
      "activations/layer3_attention_weight_min": -60.86835861206055,
      "activations/layer4_attention_weight_max": 87.07997131347656,
      "activations/layer4_attention_weight_min": -84.5034408569336,
      "activations/layer5_attention_weight_max": 119.14041137695312,
      "activations/layer5_attention_weight_min": -108.5945053100586,
      "activations/layer6_attention_weight_max": 57.418853759765625,
      "activations/layer6_attention_weight_min": -54.317874908447266,
      "activations/layer7_attention_weight_max": 56.143985748291016,
      "activations/layer7_attention_weight_min": -55.36509323120117,
      "activations/layer8_attention_weight_max": 44.76117706298828,
      "activations/layer8_attention_weight_min": -42.76361083984375,
      "activations/layer9_attention_weight_max": 63.56837844848633,
      "activations/layer9_attention_weight_min": -55.221412658691406,
      "epoch": 1.83,
      "learning_rate": 0.00013958143939393937,
      "loss": 2.9737,
      "step": 31550
    },
    {
      "activations/layer0_attention_weight_max": 14.989591598510742,
      "activations/layer0_attention_weight_min": -13.212907791137695,
      "activations/layer10_attention_weight_max": 61.30817413330078,
      "activations/layer10_attention_weight_min": -54.29829788208008,
      "activations/layer11_attention_weight_max": 40.741458892822266,
      "activations/layer11_attention_weight_min": -35.971683502197266,
      "activations/layer12_attention_weight_max": 28.25094985961914,
      "activations/layer12_attention_weight_min": -24.75446128845215,
      "activations/layer13_attention_weight_max": 48.40335464477539,
      "activations/layer13_attention_weight_min": -41.19603729248047,
      "activations/layer14_attention_weight_max": 41.94226837158203,
      "activations/layer14_attention_weight_min": -37.45760726928711,
      "activations/layer15_attention_weight_max": 44.69301986694336,
      "activations/layer15_attention_weight_min": -35.421165466308594,
      "activations/layer16_attention_weight_max": 41.099605560302734,
      "activations/layer16_attention_weight_min": -30.62640380859375,
      "activations/layer17_attention_weight_max": 53.13422393798828,
      "activations/layer17_attention_weight_min": -37.54057693481445,
      "activations/layer18_attention_weight_max": 57.920223236083984,
      "activations/layer18_attention_weight_min": -34.9362678527832,
      "activations/layer19_attention_weight_max": 20.05739402770996,
      "activations/layer19_attention_weight_min": -17.708337783813477,
      "activations/layer1_attention_weight_max": 15.59919548034668,
      "activations/layer1_attention_weight_min": -12.325201988220215,
      "activations/layer20_attention_weight_max": 18.015663146972656,
      "activations/layer20_attention_weight_min": -18.240209579467773,
      "activations/layer21_attention_weight_max": 35.61378860473633,
      "activations/layer21_attention_weight_min": -20.854656219482422,
      "activations/layer22_attention_weight_max": 31.960969924926758,
      "activations/layer22_attention_weight_min": -19.933177947998047,
      "activations/layer23_attention_weight_max": 20.49608612060547,
      "activations/layer23_attention_weight_min": -14.50934886932373,
      "activations/layer2_attention_weight_max": 26.419017791748047,
      "activations/layer2_attention_weight_min": -25.0327205657959,
      "activations/layer3_attention_weight_max": 61.0677375793457,
      "activations/layer3_attention_weight_min": -68.223876953125,
      "activations/layer4_attention_weight_max": 88.38170623779297,
      "activations/layer4_attention_weight_min": -82.27326965332031,
      "activations/layer5_attention_weight_max": 122.64944458007812,
      "activations/layer5_attention_weight_min": -114.81361389160156,
      "activations/layer6_attention_weight_max": 59.03717041015625,
      "activations/layer6_attention_weight_min": -55.547203063964844,
      "activations/layer7_attention_weight_max": 57.919246673583984,
      "activations/layer7_attention_weight_min": -53.057098388671875,
      "activations/layer8_attention_weight_max": 47.15382766723633,
      "activations/layer8_attention_weight_min": -42.29768753051758,
      "activations/layer9_attention_weight_max": 63.44633102416992,
      "activations/layer9_attention_weight_min": -56.805023193359375,
      "epoch": 1.84,
      "learning_rate": 0.00013956249999999998,
      "loss": 2.9712,
      "step": 31600
    },
    {
      "activations/layer0_attention_weight_max": 13.681506156921387,
      "activations/layer0_attention_weight_min": -13.581753730773926,
      "activations/layer10_attention_weight_max": 62.685970306396484,
      "activations/layer10_attention_weight_min": -50.61716079711914,
      "activations/layer11_attention_weight_max": 46.36626434326172,
      "activations/layer11_attention_weight_min": -38.79613494873047,
      "activations/layer12_attention_weight_max": 28.7553653717041,
      "activations/layer12_attention_weight_min": -26.382530212402344,
      "activations/layer13_attention_weight_max": 61.43202590942383,
      "activations/layer13_attention_weight_min": -43.29581832885742,
      "activations/layer14_attention_weight_max": 61.59424591064453,
      "activations/layer14_attention_weight_min": -38.416561126708984,
      "activations/layer15_attention_weight_max": 60.45235824584961,
      "activations/layer15_attention_weight_min": -38.03092956542969,
      "activations/layer16_attention_weight_max": 52.82498550415039,
      "activations/layer16_attention_weight_min": -31.998355865478516,
      "activations/layer17_attention_weight_max": 66.00534057617188,
      "activations/layer17_attention_weight_min": -35.967769622802734,
      "activations/layer18_attention_weight_max": 67.03377532958984,
      "activations/layer18_attention_weight_min": -33.39696502685547,
      "activations/layer19_attention_weight_max": 20.56350326538086,
      "activations/layer19_attention_weight_min": -14.929213523864746,
      "activations/layer1_attention_weight_max": 15.764286994934082,
      "activations/layer1_attention_weight_min": -12.2702054977417,
      "activations/layer20_attention_weight_max": 22.336467742919922,
      "activations/layer20_attention_weight_min": -13.322720527648926,
      "activations/layer21_attention_weight_max": 43.32777404785156,
      "activations/layer21_attention_weight_min": -26.161672592163086,
      "activations/layer22_attention_weight_max": 38.28584289550781,
      "activations/layer22_attention_weight_min": -21.605867385864258,
      "activations/layer23_attention_weight_max": 25.120595932006836,
      "activations/layer23_attention_weight_min": -14.550772666931152,
      "activations/layer2_attention_weight_max": 25.858516693115234,
      "activations/layer2_attention_weight_min": -25.226545333862305,
      "activations/layer3_attention_weight_max": 59.4360237121582,
      "activations/layer3_attention_weight_min": -65.2750244140625,
      "activations/layer4_attention_weight_max": 87.33341979980469,
      "activations/layer4_attention_weight_min": -81.6406478881836,
      "activations/layer5_attention_weight_max": 129.99368286132812,
      "activations/layer5_attention_weight_min": -106.9996337890625,
      "activations/layer6_attention_weight_max": 61.99391555786133,
      "activations/layer6_attention_weight_min": -53.36585998535156,
      "activations/layer7_attention_weight_max": 58.31467056274414,
      "activations/layer7_attention_weight_min": -55.090816497802734,
      "activations/layer8_attention_weight_max": 51.2164306640625,
      "activations/layer8_attention_weight_min": -48.115352630615234,
      "activations/layer9_attention_weight_max": 67.3650131225586,
      "activations/layer9_attention_weight_min": -63.12114334106445,
      "epoch": 1.84,
      "learning_rate": 0.0001395435606060606,
      "loss": 3.0045,
      "step": 31650
    },
    {
      "activations/layer0_attention_weight_max": 14.888440132141113,
      "activations/layer0_attention_weight_min": -13.55553150177002,
      "activations/layer10_attention_weight_max": 58.63496017456055,
      "activations/layer10_attention_weight_min": -48.68516159057617,
      "activations/layer11_attention_weight_max": 42.983177185058594,
      "activations/layer11_attention_weight_min": -35.93746566772461,
      "activations/layer12_attention_weight_max": 28.877500534057617,
      "activations/layer12_attention_weight_min": -23.68996810913086,
      "activations/layer13_attention_weight_max": 52.23173141479492,
      "activations/layer13_attention_weight_min": -44.23923873901367,
      "activations/layer14_attention_weight_max": 51.74675750732422,
      "activations/layer14_attention_weight_min": -44.4082145690918,
      "activations/layer15_attention_weight_max": 51.049415588378906,
      "activations/layer15_attention_weight_min": -38.83232116699219,
      "activations/layer16_attention_weight_max": 39.30536651611328,
      "activations/layer16_attention_weight_min": -27.880353927612305,
      "activations/layer17_attention_weight_max": 60.63213348388672,
      "activations/layer17_attention_weight_min": -40.40053939819336,
      "activations/layer18_attention_weight_max": 54.86405944824219,
      "activations/layer18_attention_weight_min": -35.286808013916016,
      "activations/layer19_attention_weight_max": 21.959178924560547,
      "activations/layer19_attention_weight_min": -15.894556999206543,
      "activations/layer1_attention_weight_max": 17.0612850189209,
      "activations/layer1_attention_weight_min": -13.48946762084961,
      "activations/layer20_attention_weight_max": 19.802724838256836,
      "activations/layer20_attention_weight_min": -18.801969528198242,
      "activations/layer21_attention_weight_max": 36.76680374145508,
      "activations/layer21_attention_weight_min": -23.912334442138672,
      "activations/layer22_attention_weight_max": 35.376136779785156,
      "activations/layer22_attention_weight_min": -23.9990177154541,
      "activations/layer23_attention_weight_max": 24.260879516601562,
      "activations/layer23_attention_weight_min": -14.535345077514648,
      "activations/layer2_attention_weight_max": 26.73470687866211,
      "activations/layer2_attention_weight_min": -27.573604583740234,
      "activations/layer3_attention_weight_max": 58.09665298461914,
      "activations/layer3_attention_weight_min": -61.413902282714844,
      "activations/layer4_attention_weight_max": 86.92583465576172,
      "activations/layer4_attention_weight_min": -78.92150115966797,
      "activations/layer5_attention_weight_max": 122.39476776123047,
      "activations/layer5_attention_weight_min": -102.19695281982422,
      "activations/layer6_attention_weight_max": 59.898834228515625,
      "activations/layer6_attention_weight_min": -49.360816955566406,
      "activations/layer7_attention_weight_max": 54.05805206298828,
      "activations/layer7_attention_weight_min": -50.86491775512695,
      "activations/layer8_attention_weight_max": 42.16969680786133,
      "activations/layer8_attention_weight_min": -39.934932708740234,
      "activations/layer9_attention_weight_max": 58.48798370361328,
      "activations/layer9_attention_weight_min": -59.39722442626953,
      "epoch": 1.84,
      "learning_rate": 0.0001395246212121212,
      "loss": 2.9728,
      "step": 31700
    },
    {
      "activations/layer0_attention_weight_max": 14.89130973815918,
      "activations/layer0_attention_weight_min": -13.332977294921875,
      "activations/layer10_attention_weight_max": 58.5141716003418,
      "activations/layer10_attention_weight_min": -52.48850631713867,
      "activations/layer11_attention_weight_max": 39.7408561706543,
      "activations/layer11_attention_weight_min": -33.972068786621094,
      "activations/layer12_attention_weight_max": 29.371797561645508,
      "activations/layer12_attention_weight_min": -23.07864761352539,
      "activations/layer13_attention_weight_max": 65.35234832763672,
      "activations/layer13_attention_weight_min": -52.89879608154297,
      "activations/layer14_attention_weight_max": 55.539573669433594,
      "activations/layer14_attention_weight_min": -40.65894317626953,
      "activations/layer15_attention_weight_max": 53.611045837402344,
      "activations/layer15_attention_weight_min": -39.93378448486328,
      "activations/layer16_attention_weight_max": 43.37717056274414,
      "activations/layer16_attention_weight_min": -30.45769500732422,
      "activations/layer17_attention_weight_max": 59.65859603881836,
      "activations/layer17_attention_weight_min": -37.96328353881836,
      "activations/layer18_attention_weight_max": 55.3271369934082,
      "activations/layer18_attention_weight_min": -35.424705505371094,
      "activations/layer19_attention_weight_max": 23.51701545715332,
      "activations/layer19_attention_weight_min": -15.221236228942871,
      "activations/layer1_attention_weight_max": 16.170841217041016,
      "activations/layer1_attention_weight_min": -12.72131061553955,
      "activations/layer20_attention_weight_max": 19.909452438354492,
      "activations/layer20_attention_weight_min": -15.163981437683105,
      "activations/layer21_attention_weight_max": 37.19150161743164,
      "activations/layer21_attention_weight_min": -23.758220672607422,
      "activations/layer22_attention_weight_max": 32.002830505371094,
      "activations/layer22_attention_weight_min": -20.65511131286621,
      "activations/layer23_attention_weight_max": 24.63515853881836,
      "activations/layer23_attention_weight_min": -14.035849571228027,
      "activations/layer2_attention_weight_max": 28.105581283569336,
      "activations/layer2_attention_weight_min": -24.82555389404297,
      "activations/layer3_attention_weight_max": 58.54176712036133,
      "activations/layer3_attention_weight_min": -63.876041412353516,
      "activations/layer4_attention_weight_max": 86.67074584960938,
      "activations/layer4_attention_weight_min": -80.4343032836914,
      "activations/layer5_attention_weight_max": 125.30224609375,
      "activations/layer5_attention_weight_min": -107.5723876953125,
      "activations/layer6_attention_weight_max": 61.02567672729492,
      "activations/layer6_attention_weight_min": -51.604713439941406,
      "activations/layer7_attention_weight_max": 59.819175720214844,
      "activations/layer7_attention_weight_min": -52.14852523803711,
      "activations/layer8_attention_weight_max": 44.07710647583008,
      "activations/layer8_attention_weight_min": -42.017311096191406,
      "activations/layer9_attention_weight_max": 62.800811767578125,
      "activations/layer9_attention_weight_min": -62.94061279296875,
      "epoch": 1.84,
      "learning_rate": 0.0001395056818181818,
      "loss": 2.9769,
      "step": 31750
    },
    {
      "activations/layer0_attention_weight_max": 15.949868202209473,
      "activations/layer0_attention_weight_min": -13.745394706726074,
      "activations/layer10_attention_weight_max": 63.075439453125,
      "activations/layer10_attention_weight_min": -50.232757568359375,
      "activations/layer11_attention_weight_max": 44.68478012084961,
      "activations/layer11_attention_weight_min": -36.745574951171875,
      "activations/layer12_attention_weight_max": 35.929786682128906,
      "activations/layer12_attention_weight_min": -27.725648880004883,
      "activations/layer13_attention_weight_max": 77.7657241821289,
      "activations/layer13_attention_weight_min": -49.8131217956543,
      "activations/layer14_attention_weight_max": 62.74136734008789,
      "activations/layer14_attention_weight_min": -39.061561584472656,
      "activations/layer15_attention_weight_max": 56.9610710144043,
      "activations/layer15_attention_weight_min": -36.12580108642578,
      "activations/layer16_attention_weight_max": 50.22891616821289,
      "activations/layer16_attention_weight_min": -28.74503517150879,
      "activations/layer17_attention_weight_max": 69.62324523925781,
      "activations/layer17_attention_weight_min": -40.10648727416992,
      "activations/layer18_attention_weight_max": 69.2413558959961,
      "activations/layer18_attention_weight_min": -36.48627853393555,
      "activations/layer19_attention_weight_max": 24.415258407592773,
      "activations/layer19_attention_weight_min": -17.518892288208008,
      "activations/layer1_attention_weight_max": 16.75510025024414,
      "activations/layer1_attention_weight_min": -13.99046802520752,
      "activations/layer20_attention_weight_max": 23.320110321044922,
      "activations/layer20_attention_weight_min": -16.155059814453125,
      "activations/layer21_attention_weight_max": 49.013851165771484,
      "activations/layer21_attention_weight_min": -23.02497100830078,
      "activations/layer22_attention_weight_max": 44.70549774169922,
      "activations/layer22_attention_weight_min": -23.44940948486328,
      "activations/layer23_attention_weight_max": 27.479740142822266,
      "activations/layer23_attention_weight_min": -16.03058433532715,
      "activations/layer2_attention_weight_max": 27.683277130126953,
      "activations/layer2_attention_weight_min": -27.01186180114746,
      "activations/layer3_attention_weight_max": 62.628604888916016,
      "activations/layer3_attention_weight_min": -63.24372100830078,
      "activations/layer4_attention_weight_max": 93.1787338256836,
      "activations/layer4_attention_weight_min": -84.2608642578125,
      "activations/layer5_attention_weight_max": 131.38941955566406,
      "activations/layer5_attention_weight_min": -108.38817596435547,
      "activations/layer6_attention_weight_max": 61.34831237792969,
      "activations/layer6_attention_weight_min": -51.62118148803711,
      "activations/layer7_attention_weight_max": 62.01658630371094,
      "activations/layer7_attention_weight_min": -56.604949951171875,
      "activations/layer8_attention_weight_max": 53.43095397949219,
      "activations/layer8_attention_weight_min": -45.356910705566406,
      "activations/layer9_attention_weight_max": 77.38737487792969,
      "activations/layer9_attention_weight_min": -70.30413055419922,
      "epoch": 1.85,
      "learning_rate": 0.0001394871212121212,
      "loss": 2.9827,
      "step": 31800
    },
    {
      "activations/layer0_attention_weight_max": 15.081135749816895,
      "activations/layer0_attention_weight_min": -14.097726821899414,
      "activations/layer10_attention_weight_max": 63.87232208251953,
      "activations/layer10_attention_weight_min": -52.49724578857422,
      "activations/layer11_attention_weight_max": 53.859779357910156,
      "activations/layer11_attention_weight_min": -36.52216339111328,
      "activations/layer12_attention_weight_max": 29.496055603027344,
      "activations/layer12_attention_weight_min": -22.026870727539062,
      "activations/layer13_attention_weight_max": 79.38874816894531,
      "activations/layer13_attention_weight_min": -50.045902252197266,
      "activations/layer14_attention_weight_max": 100.219970703125,
      "activations/layer14_attention_weight_min": -55.360748291015625,
      "activations/layer15_attention_weight_max": 61.181800842285156,
      "activations/layer15_attention_weight_min": -40.23938751220703,
      "activations/layer16_attention_weight_max": 51.211517333984375,
      "activations/layer16_attention_weight_min": -30.272571563720703,
      "activations/layer17_attention_weight_max": 68.6746826171875,
      "activations/layer17_attention_weight_min": -35.57097244262695,
      "activations/layer18_attention_weight_max": 66.87482452392578,
      "activations/layer18_attention_weight_min": -32.953033447265625,
      "activations/layer19_attention_weight_max": 29.133127212524414,
      "activations/layer19_attention_weight_min": -17.369884490966797,
      "activations/layer1_attention_weight_max": 16.220293045043945,
      "activations/layer1_attention_weight_min": -15.19147777557373,
      "activations/layer20_attention_weight_max": 23.259990692138672,
      "activations/layer20_attention_weight_min": -16.902341842651367,
      "activations/layer21_attention_weight_max": 47.1409912109375,
      "activations/layer21_attention_weight_min": -25.1207218170166,
      "activations/layer22_attention_weight_max": 46.865787506103516,
      "activations/layer22_attention_weight_min": -22.695432662963867,
      "activations/layer23_attention_weight_max": 27.127559661865234,
      "activations/layer23_attention_weight_min": -15.433696746826172,
      "activations/layer2_attention_weight_max": 27.38576889038086,
      "activations/layer2_attention_weight_min": -28.794265747070312,
      "activations/layer3_attention_weight_max": 59.463462829589844,
      "activations/layer3_attention_weight_min": -61.730159759521484,
      "activations/layer4_attention_weight_max": 88.30536651611328,
      "activations/layer4_attention_weight_min": -85.67943572998047,
      "activations/layer5_attention_weight_max": 123.48784637451172,
      "activations/layer5_attention_weight_min": -106.45072937011719,
      "activations/layer6_attention_weight_max": 62.464263916015625,
      "activations/layer6_attention_weight_min": -53.387027740478516,
      "activations/layer7_attention_weight_max": 58.73476028442383,
      "activations/layer7_attention_weight_min": -55.39028549194336,
      "activations/layer8_attention_weight_max": 47.31984329223633,
      "activations/layer8_attention_weight_min": -42.20551681518555,
      "activations/layer9_attention_weight_max": 66.60779571533203,
      "activations/layer9_attention_weight_min": -64.4466781616211,
      "epoch": 1.85,
      "learning_rate": 0.0001394681818181818,
      "loss": 2.9948,
      "step": 31850
    },
    {
      "activations/layer0_attention_weight_max": 14.886293411254883,
      "activations/layer0_attention_weight_min": -14.268351554870605,
      "activations/layer10_attention_weight_max": 62.07216262817383,
      "activations/layer10_attention_weight_min": -49.24705505371094,
      "activations/layer11_attention_weight_max": 50.761680603027344,
      "activations/layer11_attention_weight_min": -43.715972900390625,
      "activations/layer12_attention_weight_max": 28.157106399536133,
      "activations/layer12_attention_weight_min": -26.35348129272461,
      "activations/layer13_attention_weight_max": 78.93106842041016,
      "activations/layer13_attention_weight_min": -48.39894485473633,
      "activations/layer14_attention_weight_max": 89.77815246582031,
      "activations/layer14_attention_weight_min": -52.238040924072266,
      "activations/layer15_attention_weight_max": 50.731929779052734,
      "activations/layer15_attention_weight_min": -37.26719284057617,
      "activations/layer16_attention_weight_max": 42.72971725463867,
      "activations/layer16_attention_weight_min": -28.264286041259766,
      "activations/layer17_attention_weight_max": 62.44194412231445,
      "activations/layer17_attention_weight_min": -39.80847930908203,
      "activations/layer18_attention_weight_max": 64.05320739746094,
      "activations/layer18_attention_weight_min": -34.89342498779297,
      "activations/layer19_attention_weight_max": 21.44209861755371,
      "activations/layer19_attention_weight_min": -15.915536880493164,
      "activations/layer1_attention_weight_max": 15.950650215148926,
      "activations/layer1_attention_weight_min": -14.704814910888672,
      "activations/layer20_attention_weight_max": 21.020915985107422,
      "activations/layer20_attention_weight_min": -16.71917724609375,
      "activations/layer21_attention_weight_max": 42.22294235229492,
      "activations/layer21_attention_weight_min": -23.17899513244629,
      "activations/layer22_attention_weight_max": 42.126991271972656,
      "activations/layer22_attention_weight_min": -23.781696319580078,
      "activations/layer23_attention_weight_max": 25.694984436035156,
      "activations/layer23_attention_weight_min": -18.48137664794922,
      "activations/layer2_attention_weight_max": 27.09857940673828,
      "activations/layer2_attention_weight_min": -27.747074127197266,
      "activations/layer3_attention_weight_max": 62.844852447509766,
      "activations/layer3_attention_weight_min": -67.36481475830078,
      "activations/layer4_attention_weight_max": 97.00385284423828,
      "activations/layer4_attention_weight_min": -86.90894317626953,
      "activations/layer5_attention_weight_max": 149.18702697753906,
      "activations/layer5_attention_weight_min": -114.47062683105469,
      "activations/layer6_attention_weight_max": 72.42574310302734,
      "activations/layer6_attention_weight_min": -53.791805267333984,
      "activations/layer7_attention_weight_max": 68.0444564819336,
      "activations/layer7_attention_weight_min": -53.554405212402344,
      "activations/layer8_attention_weight_max": 63.23537063598633,
      "activations/layer8_attention_weight_min": -50.196739196777344,
      "activations/layer9_attention_weight_max": 88.4720687866211,
      "activations/layer9_attention_weight_min": -70.21529388427734,
      "epoch": 1.85,
      "learning_rate": 0.0001394492424242424,
      "loss": 2.9842,
      "step": 31900
    },
    {
      "activations/layer0_attention_weight_max": 13.570293426513672,
      "activations/layer0_attention_weight_min": -13.700287818908691,
      "activations/layer10_attention_weight_max": 59.330379486083984,
      "activations/layer10_attention_weight_min": -50.498897552490234,
      "activations/layer11_attention_weight_max": 44.00732421875,
      "activations/layer11_attention_weight_min": -38.190555572509766,
      "activations/layer12_attention_weight_max": 26.363584518432617,
      "activations/layer12_attention_weight_min": -24.58055877685547,
      "activations/layer13_attention_weight_max": 55.83139419555664,
      "activations/layer13_attention_weight_min": -43.99513244628906,
      "activations/layer14_attention_weight_max": 50.63431167602539,
      "activations/layer14_attention_weight_min": -39.15754318237305,
      "activations/layer15_attention_weight_max": 56.63972091674805,
      "activations/layer15_attention_weight_min": -39.3714485168457,
      "activations/layer16_attention_weight_max": 40.733699798583984,
      "activations/layer16_attention_weight_min": -30.635473251342773,
      "activations/layer17_attention_weight_max": 75.84246826171875,
      "activations/layer17_attention_weight_min": -44.65789794921875,
      "activations/layer18_attention_weight_max": 70.72996520996094,
      "activations/layer18_attention_weight_min": -33.9533576965332,
      "activations/layer19_attention_weight_max": 21.72039222717285,
      "activations/layer19_attention_weight_min": -15.522668838500977,
      "activations/layer1_attention_weight_max": 15.982860565185547,
      "activations/layer1_attention_weight_min": -13.142447471618652,
      "activations/layer20_attention_weight_max": 23.182065963745117,
      "activations/layer20_attention_weight_min": -18.384920120239258,
      "activations/layer21_attention_weight_max": 41.73463821411133,
      "activations/layer21_attention_weight_min": -23.865798950195312,
      "activations/layer22_attention_weight_max": 35.86994552612305,
      "activations/layer22_attention_weight_min": -21.28270149230957,
      "activations/layer23_attention_weight_max": 22.075450897216797,
      "activations/layer23_attention_weight_min": -15.233745574951172,
      "activations/layer2_attention_weight_max": 26.96969223022461,
      "activations/layer2_attention_weight_min": -25.984912872314453,
      "activations/layer3_attention_weight_max": 60.57970428466797,
      "activations/layer3_attention_weight_min": -62.34061050415039,
      "activations/layer4_attention_weight_max": 95.4784164428711,
      "activations/layer4_attention_weight_min": -83.51753997802734,
      "activations/layer5_attention_weight_max": 132.7776336669922,
      "activations/layer5_attention_weight_min": -116.95803833007812,
      "activations/layer6_attention_weight_max": 62.137359619140625,
      "activations/layer6_attention_weight_min": -52.9754524230957,
      "activations/layer7_attention_weight_max": 56.91522216796875,
      "activations/layer7_attention_weight_min": -53.10966491699219,
      "activations/layer8_attention_weight_max": 46.24342346191406,
      "activations/layer8_attention_weight_min": -41.01857376098633,
      "activations/layer9_attention_weight_max": 65.31808471679688,
      "activations/layer9_attention_weight_min": -60.67342758178711,
      "epoch": 1.86,
      "learning_rate": 0.000139430303030303,
      "loss": 2.9953,
      "step": 31950
    },
    {
      "activations/layer0_attention_weight_max": 14.44226360321045,
      "activations/layer0_attention_weight_min": -13.922375679016113,
      "activations/layer10_attention_weight_max": 60.90789031982422,
      "activations/layer10_attention_weight_min": -49.34793472290039,
      "activations/layer11_attention_weight_max": 45.09741973876953,
      "activations/layer11_attention_weight_min": -37.04282760620117,
      "activations/layer12_attention_weight_max": 28.639135360717773,
      "activations/layer12_attention_weight_min": -24.66191864013672,
      "activations/layer13_attention_weight_max": 63.12267303466797,
      "activations/layer13_attention_weight_min": -44.08181381225586,
      "activations/layer14_attention_weight_max": 60.84717559814453,
      "activations/layer14_attention_weight_min": -36.16440200805664,
      "activations/layer15_attention_weight_max": 53.46343994140625,
      "activations/layer15_attention_weight_min": -40.459266662597656,
      "activations/layer16_attention_weight_max": 45.3538703918457,
      "activations/layer16_attention_weight_min": -29.399499893188477,
      "activations/layer17_attention_weight_max": 58.46814727783203,
      "activations/layer17_attention_weight_min": -34.790000915527344,
      "activations/layer18_attention_weight_max": 58.89213562011719,
      "activations/layer18_attention_weight_min": -33.065982818603516,
      "activations/layer19_attention_weight_max": 18.777830123901367,
      "activations/layer19_attention_weight_min": -14.384368896484375,
      "activations/layer1_attention_weight_max": 15.858024597167969,
      "activations/layer1_attention_weight_min": -12.58822250366211,
      "activations/layer20_attention_weight_max": 18.15424346923828,
      "activations/layer20_attention_weight_min": -17.06800651550293,
      "activations/layer21_attention_weight_max": 39.41571044921875,
      "activations/layer21_attention_weight_min": -21.888952255249023,
      "activations/layer22_attention_weight_max": 36.507869720458984,
      "activations/layer22_attention_weight_min": -20.661792755126953,
      "activations/layer23_attention_weight_max": 21.42182159423828,
      "activations/layer23_attention_weight_min": -15.844878196716309,
      "activations/layer2_attention_weight_max": 28.25487518310547,
      "activations/layer2_attention_weight_min": -27.651601791381836,
      "activations/layer3_attention_weight_max": 61.976646423339844,
      "activations/layer3_attention_weight_min": -66.23448181152344,
      "activations/layer4_attention_weight_max": 90.44811248779297,
      "activations/layer4_attention_weight_min": -84.16836547851562,
      "activations/layer5_attention_weight_max": 122.12971496582031,
      "activations/layer5_attention_weight_min": -113.65675354003906,
      "activations/layer6_attention_weight_max": 58.85315704345703,
      "activations/layer6_attention_weight_min": -55.70427322387695,
      "activations/layer7_attention_weight_max": 58.355648040771484,
      "activations/layer7_attention_weight_min": -52.40660095214844,
      "activations/layer8_attention_weight_max": 50.78560256958008,
      "activations/layer8_attention_weight_min": -43.2199821472168,
      "activations/layer9_attention_weight_max": 67.83268737792969,
      "activations/layer9_attention_weight_min": -59.13227081298828,
      "epoch": 1.86,
      "learning_rate": 0.00013941136363636363,
      "loss": 2.9787,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_loss": 2.927734375,
      "eval_runtime": 8.4493,
      "eval_samples_per_second": 508.211,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_openwebtext_loss": 2.927734375,
      "eval_openwebtext_ppl": 18.6852487375444,
      "eval_openwebtext_runtime": 8.4493,
      "eval_openwebtext_samples_per_second": 508.211,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_wikitext_loss": 3.203125,
      "eval_wikitext_ppl": 24.609314266596865,
      "eval_wikitext_runtime": 1.9417,
      "eval_wikitext_samples_per_second": 234.846,
      "step": 32000
    },
    {
      "epoch": 1.86,
      "eval_lambada_loss": 3.169921875,
      "eval_lambada_ppl": 23.80562446936611,
      "eval_lambada_runtime": 9.5425,
      "eval_lambada_samples_per_second": 510.242,
      "step": 32000
    },
    {
      "activations/layer0_attention_weight_max": 13.494091987609863,
      "activations/layer0_attention_weight_min": -13.868914604187012,
      "activations/layer10_attention_weight_max": 62.17706298828125,
      "activations/layer10_attention_weight_min": -50.6366081237793,
      "activations/layer11_attention_weight_max": 45.35358810424805,
      "activations/layer11_attention_weight_min": -37.089874267578125,
      "activations/layer12_attention_weight_max": 33.709903717041016,
      "activations/layer12_attention_weight_min": -25.14500617980957,
      "activations/layer13_attention_weight_max": 65.44819641113281,
      "activations/layer13_attention_weight_min": -44.8830680847168,
      "activations/layer14_attention_weight_max": 64.64476776123047,
      "activations/layer14_attention_weight_min": -38.254337310791016,
      "activations/layer15_attention_weight_max": 58.37177276611328,
      "activations/layer15_attention_weight_min": -38.53955078125,
      "activations/layer16_attention_weight_max": 45.16041946411133,
      "activations/layer16_attention_weight_min": -28.878007888793945,
      "activations/layer17_attention_weight_max": 62.01457977294922,
      "activations/layer17_attention_weight_min": -36.20598220825195,
      "activations/layer18_attention_weight_max": 64.84728240966797,
      "activations/layer18_attention_weight_min": -35.76625442504883,
      "activations/layer19_attention_weight_max": 21.773420333862305,
      "activations/layer19_attention_weight_min": -15.09222412109375,
      "activations/layer1_attention_weight_max": 16.09403419494629,
      "activations/layer1_attention_weight_min": -12.128605842590332,
      "activations/layer20_attention_weight_max": 22.70284652709961,
      "activations/layer20_attention_weight_min": -16.339990615844727,
      "activations/layer21_attention_weight_max": 42.92202377319336,
      "activations/layer21_attention_weight_min": -24.799558639526367,
      "activations/layer22_attention_weight_max": 38.7031364440918,
      "activations/layer22_attention_weight_min": -21.516551971435547,
      "activations/layer23_attention_weight_max": 25.261442184448242,
      "activations/layer23_attention_weight_min": -15.21549129486084,
      "activations/layer2_attention_weight_max": 25.547260284423828,
      "activations/layer2_attention_weight_min": -25.954906463623047,
      "activations/layer3_attention_weight_max": 59.47195816040039,
      "activations/layer3_attention_weight_min": -62.63264083862305,
      "activations/layer4_attention_weight_max": 86.46747589111328,
      "activations/layer4_attention_weight_min": -78.87763214111328,
      "activations/layer5_attention_weight_max": 125.8221435546875,
      "activations/layer5_attention_weight_min": -105.1362533569336,
      "activations/layer6_attention_weight_max": 61.168861389160156,
      "activations/layer6_attention_weight_min": -51.57046890258789,
      "activations/layer7_attention_weight_max": 60.13462448120117,
      "activations/layer7_attention_weight_min": -55.001861572265625,
      "activations/layer8_attention_weight_max": 49.777496337890625,
      "activations/layer8_attention_weight_min": -40.54580307006836,
      "activations/layer9_attention_weight_max": 58.8625602722168,
      "activations/layer9_attention_weight_min": -54.60378646850586,
      "epoch": 1.86,
      "learning_rate": 0.00013939242424242424,
      "loss": 2.9749,
      "step": 32050
    },
    {
      "activations/layer0_attention_weight_max": 16.161361694335938,
      "activations/layer0_attention_weight_min": -13.896020889282227,
      "activations/layer10_attention_weight_max": 65.41159057617188,
      "activations/layer10_attention_weight_min": -52.69704055786133,
      "activations/layer11_attention_weight_max": 46.15470504760742,
      "activations/layer11_attention_weight_min": -36.9869270324707,
      "activations/layer12_attention_weight_max": 29.707216262817383,
      "activations/layer12_attention_weight_min": -26.29582977294922,
      "activations/layer13_attention_weight_max": 64.23919677734375,
      "activations/layer13_attention_weight_min": -44.38739776611328,
      "activations/layer14_attention_weight_max": 62.66154861450195,
      "activations/layer14_attention_weight_min": -43.44575119018555,
      "activations/layer15_attention_weight_max": 51.45632553100586,
      "activations/layer15_attention_weight_min": -36.8470573425293,
      "activations/layer16_attention_weight_max": 43.76247787475586,
      "activations/layer16_attention_weight_min": -28.604761123657227,
      "activations/layer17_attention_weight_max": 56.83466720581055,
      "activations/layer17_attention_weight_min": -36.8690071105957,
      "activations/layer18_attention_weight_max": 58.569541931152344,
      "activations/layer18_attention_weight_min": -33.20783615112305,
      "activations/layer19_attention_weight_max": 22.840068817138672,
      "activations/layer19_attention_weight_min": -16.248600006103516,
      "activations/layer1_attention_weight_max": 17.858110427856445,
      "activations/layer1_attention_weight_min": -13.056564331054688,
      "activations/layer20_attention_weight_max": 19.594852447509766,
      "activations/layer20_attention_weight_min": -18.69817543029785,
      "activations/layer21_attention_weight_max": 37.08429718017578,
      "activations/layer21_attention_weight_min": -23.18816375732422,
      "activations/layer22_attention_weight_max": 34.37488555908203,
      "activations/layer22_attention_weight_min": -22.9851016998291,
      "activations/layer23_attention_weight_max": 24.987380981445312,
      "activations/layer23_attention_weight_min": -14.667928695678711,
      "activations/layer2_attention_weight_max": 25.893959045410156,
      "activations/layer2_attention_weight_min": -26.883995056152344,
      "activations/layer3_attention_weight_max": 60.8029670715332,
      "activations/layer3_attention_weight_min": -68.98387908935547,
      "activations/layer4_attention_weight_max": 90.90388488769531,
      "activations/layer4_attention_weight_min": -86.6142807006836,
      "activations/layer5_attention_weight_max": 127.6227035522461,
      "activations/layer5_attention_weight_min": -108.10284423828125,
      "activations/layer6_attention_weight_max": 62.469764709472656,
      "activations/layer6_attention_weight_min": -52.54775619506836,
      "activations/layer7_attention_weight_max": 58.97445297241211,
      "activations/layer7_attention_weight_min": -55.52211380004883,
      "activations/layer8_attention_weight_max": 48.223262786865234,
      "activations/layer8_attention_weight_min": -42.33887481689453,
      "activations/layer9_attention_weight_max": 71.46842193603516,
      "activations/layer9_attention_weight_min": -65.21891021728516,
      "epoch": 1.87,
      "learning_rate": 0.00013937348484848483,
      "loss": 2.9993,
      "step": 32100
    },
    {
      "activations/layer0_attention_weight_max": 15.147394180297852,
      "activations/layer0_attention_weight_min": -13.739825248718262,
      "activations/layer10_attention_weight_max": 69.79642486572266,
      "activations/layer10_attention_weight_min": -52.73432159423828,
      "activations/layer11_attention_weight_max": 51.244380950927734,
      "activations/layer11_attention_weight_min": -37.64508056640625,
      "activations/layer12_attention_weight_max": 43.886592864990234,
      "activations/layer12_attention_weight_min": -25.74162483215332,
      "activations/layer13_attention_weight_max": 65.18514251708984,
      "activations/layer13_attention_weight_min": -42.07338333129883,
      "activations/layer14_attention_weight_max": 47.950714111328125,
      "activations/layer14_attention_weight_min": -36.9948616027832,
      "activations/layer15_attention_weight_max": 60.48249435424805,
      "activations/layer15_attention_weight_min": -37.86854553222656,
      "activations/layer16_attention_weight_max": 41.95024108886719,
      "activations/layer16_attention_weight_min": -28.222373962402344,
      "activations/layer17_attention_weight_max": 73.99130249023438,
      "activations/layer17_attention_weight_min": -40.53217697143555,
      "activations/layer18_attention_weight_max": 57.698326110839844,
      "activations/layer18_attention_weight_min": -35.373809814453125,
      "activations/layer19_attention_weight_max": 19.309968948364258,
      "activations/layer19_attention_weight_min": -14.629780769348145,
      "activations/layer1_attention_weight_max": 17.258678436279297,
      "activations/layer1_attention_weight_min": -12.316248893737793,
      "activations/layer20_attention_weight_max": 20.719816207885742,
      "activations/layer20_attention_weight_min": -18.51785659790039,
      "activations/layer21_attention_weight_max": 43.66534423828125,
      "activations/layer21_attention_weight_min": -26.722238540649414,
      "activations/layer22_attention_weight_max": 37.29878234863281,
      "activations/layer22_attention_weight_min": -22.03337860107422,
      "activations/layer23_attention_weight_max": 24.005796432495117,
      "activations/layer23_attention_weight_min": -15.262548446655273,
      "activations/layer2_attention_weight_max": 27.775938034057617,
      "activations/layer2_attention_weight_min": -27.539609909057617,
      "activations/layer3_attention_weight_max": 62.205562591552734,
      "activations/layer3_attention_weight_min": -65.59373474121094,
      "activations/layer4_attention_weight_max": 100.8237075805664,
      "activations/layer4_attention_weight_min": -87.69805908203125,
      "activations/layer5_attention_weight_max": 143.12548828125,
      "activations/layer5_attention_weight_min": -114.52777099609375,
      "activations/layer6_attention_weight_max": 66.17308807373047,
      "activations/layer6_attention_weight_min": -54.65609359741211,
      "activations/layer7_attention_weight_max": 64.53873443603516,
      "activations/layer7_attention_weight_min": -59.23853302001953,
      "activations/layer8_attention_weight_max": 54.65753936767578,
      "activations/layer8_attention_weight_min": -43.57904815673828,
      "activations/layer9_attention_weight_max": 68.78085327148438,
      "activations/layer9_attention_weight_min": -58.90828323364258,
      "epoch": 1.87,
      "learning_rate": 0.00013935454545454544,
      "loss": 2.9533,
      "step": 32150
    },
    {
      "activations/layer0_attention_weight_max": 14.47549057006836,
      "activations/layer0_attention_weight_min": -14.166726112365723,
      "activations/layer10_attention_weight_max": 68.44648742675781,
      "activations/layer10_attention_weight_min": -57.0145263671875,
      "activations/layer11_attention_weight_max": 42.448387145996094,
      "activations/layer11_attention_weight_min": -40.097145080566406,
      "activations/layer12_attention_weight_max": 26.612735748291016,
      "activations/layer12_attention_weight_min": -24.157005310058594,
      "activations/layer13_attention_weight_max": 68.1094741821289,
      "activations/layer13_attention_weight_min": -48.197261810302734,
      "activations/layer14_attention_weight_max": 55.96628952026367,
      "activations/layer14_attention_weight_min": -37.29875183105469,
      "activations/layer15_attention_weight_max": 53.2323112487793,
      "activations/layer15_attention_weight_min": -38.34804916381836,
      "activations/layer16_attention_weight_max": 41.52346420288086,
      "activations/layer16_attention_weight_min": -29.53582191467285,
      "activations/layer17_attention_weight_max": 73.06330871582031,
      "activations/layer17_attention_weight_min": -38.18864822387695,
      "activations/layer18_attention_weight_max": 64.05387115478516,
      "activations/layer18_attention_weight_min": -35.217437744140625,
      "activations/layer19_attention_weight_max": 24.88372230529785,
      "activations/layer19_attention_weight_min": -15.075111389160156,
      "activations/layer1_attention_weight_max": 16.294452667236328,
      "activations/layer1_attention_weight_min": -12.919788360595703,
      "activations/layer20_attention_weight_max": 20.850858688354492,
      "activations/layer20_attention_weight_min": -15.635305404663086,
      "activations/layer21_attention_weight_max": 45.11808776855469,
      "activations/layer21_attention_weight_min": -21.956504821777344,
      "activations/layer22_attention_weight_max": 38.7884635925293,
      "activations/layer22_attention_weight_min": -21.410776138305664,
      "activations/layer23_attention_weight_max": 25.680509567260742,
      "activations/layer23_attention_weight_min": -13.83698844909668,
      "activations/layer2_attention_weight_max": 26.404541015625,
      "activations/layer2_attention_weight_min": -26.34773826599121,
      "activations/layer3_attention_weight_max": 62.951934814453125,
      "activations/layer3_attention_weight_min": -65.01677703857422,
      "activations/layer4_attention_weight_max": 94.2416763305664,
      "activations/layer4_attention_weight_min": -83.7386703491211,
      "activations/layer5_attention_weight_max": 126.35952758789062,
      "activations/layer5_attention_weight_min": -116.4609375,
      "activations/layer6_attention_weight_max": 60.6502799987793,
      "activations/layer6_attention_weight_min": -55.00285720825195,
      "activations/layer7_attention_weight_max": 57.9890022277832,
      "activations/layer7_attention_weight_min": -58.11326217651367,
      "activations/layer8_attention_weight_max": 51.60847091674805,
      "activations/layer8_attention_weight_min": -51.92908477783203,
      "activations/layer9_attention_weight_max": 65.85104370117188,
      "activations/layer9_attention_weight_min": -63.88412094116211,
      "epoch": 1.87,
      "learning_rate": 0.00013933560606060606,
      "loss": 2.974,
      "step": 32200
    },
    {
      "activations/layer0_attention_weight_max": 17.266576766967773,
      "activations/layer0_attention_weight_min": -14.451723098754883,
      "activations/layer10_attention_weight_max": 61.92241668701172,
      "activations/layer10_attention_weight_min": -48.07793045043945,
      "activations/layer11_attention_weight_max": 40.47366714477539,
      "activations/layer11_attention_weight_min": -32.80619812011719,
      "activations/layer12_attention_weight_max": 27.31183624267578,
      "activations/layer12_attention_weight_min": -23.386924743652344,
      "activations/layer13_attention_weight_max": 67.41912841796875,
      "activations/layer13_attention_weight_min": -48.58145523071289,
      "activations/layer14_attention_weight_max": 54.214900970458984,
      "activations/layer14_attention_weight_min": -39.411766052246094,
      "activations/layer15_attention_weight_max": 52.252071380615234,
      "activations/layer15_attention_weight_min": -37.27444076538086,
      "activations/layer16_attention_weight_max": 46.50949478149414,
      "activations/layer16_attention_weight_min": -30.211353302001953,
      "activations/layer17_attention_weight_max": 55.8072624206543,
      "activations/layer17_attention_weight_min": -36.575321197509766,
      "activations/layer18_attention_weight_max": 58.58803176879883,
      "activations/layer18_attention_weight_min": -32.42627716064453,
      "activations/layer19_attention_weight_max": 22.04916000366211,
      "activations/layer19_attention_weight_min": -14.858926773071289,
      "activations/layer1_attention_weight_max": 15.690108299255371,
      "activations/layer1_attention_weight_min": -11.745426177978516,
      "activations/layer20_attention_weight_max": 23.208112716674805,
      "activations/layer20_attention_weight_min": -16.658889770507812,
      "activations/layer21_attention_weight_max": 42.0139045715332,
      "activations/layer21_attention_weight_min": -24.192333221435547,
      "activations/layer22_attention_weight_max": 36.136348724365234,
      "activations/layer22_attention_weight_min": -20.918811798095703,
      "activations/layer23_attention_weight_max": 24.487567901611328,
      "activations/layer23_attention_weight_min": -15.22033977508545,
      "activations/layer2_attention_weight_max": 26.865070343017578,
      "activations/layer2_attention_weight_min": -26.509952545166016,
      "activations/layer3_attention_weight_max": 63.1187629699707,
      "activations/layer3_attention_weight_min": -66.20823669433594,
      "activations/layer4_attention_weight_max": 92.55158996582031,
      "activations/layer4_attention_weight_min": -85.3301010131836,
      "activations/layer5_attention_weight_max": 123.29933166503906,
      "activations/layer5_attention_weight_min": -108.00838470458984,
      "activations/layer6_attention_weight_max": 61.48828125,
      "activations/layer6_attention_weight_min": -55.112632751464844,
      "activations/layer7_attention_weight_max": 58.58562469482422,
      "activations/layer7_attention_weight_min": -54.56126403808594,
      "activations/layer8_attention_weight_max": 43.45603561401367,
      "activations/layer8_attention_weight_min": -42.212066650390625,
      "activations/layer9_attention_weight_max": 69.93912506103516,
      "activations/layer9_attention_weight_min": -55.34626388549805,
      "epoch": 1.87,
      "learning_rate": 0.00013931666666666665,
      "loss": 2.9857,
      "step": 32250
    },
    {
      "activations/layer0_attention_weight_max": 14.600245475769043,
      "activations/layer0_attention_weight_min": -14.127974510192871,
      "activations/layer10_attention_weight_max": 62.180625915527344,
      "activations/layer10_attention_weight_min": -47.17006301879883,
      "activations/layer11_attention_weight_max": 45.13648986816406,
      "activations/layer11_attention_weight_min": -35.27349853515625,
      "activations/layer12_attention_weight_max": 32.93720626831055,
      "activations/layer12_attention_weight_min": -26.23563003540039,
      "activations/layer13_attention_weight_max": 62.48881912231445,
      "activations/layer13_attention_weight_min": -43.63994598388672,
      "activations/layer14_attention_weight_max": 54.02456283569336,
      "activations/layer14_attention_weight_min": -42.16600036621094,
      "activations/layer15_attention_weight_max": 55.11822509765625,
      "activations/layer15_attention_weight_min": -41.073028564453125,
      "activations/layer16_attention_weight_max": 41.055416107177734,
      "activations/layer16_attention_weight_min": -30.76102638244629,
      "activations/layer17_attention_weight_max": 62.70522689819336,
      "activations/layer17_attention_weight_min": -36.82704162597656,
      "activations/layer18_attention_weight_max": 59.872764587402344,
      "activations/layer18_attention_weight_min": -33.18869400024414,
      "activations/layer19_attention_weight_max": 22.526594161987305,
      "activations/layer19_attention_weight_min": -16.44412612915039,
      "activations/layer1_attention_weight_max": 15.90296459197998,
      "activations/layer1_attention_weight_min": -13.58154010772705,
      "activations/layer20_attention_weight_max": 18.548648834228516,
      "activations/layer20_attention_weight_min": -17.432451248168945,
      "activations/layer21_attention_weight_max": 39.03663635253906,
      "activations/layer21_attention_weight_min": -23.8604793548584,
      "activations/layer22_attention_weight_max": 33.78181076049805,
      "activations/layer22_attention_weight_min": -21.192724227905273,
      "activations/layer23_attention_weight_max": 22.981788635253906,
      "activations/layer23_attention_weight_min": -16.40106964111328,
      "activations/layer2_attention_weight_max": 25.01639175415039,
      "activations/layer2_attention_weight_min": -24.676626205444336,
      "activations/layer3_attention_weight_max": 59.244110107421875,
      "activations/layer3_attention_weight_min": -61.67629623413086,
      "activations/layer4_attention_weight_max": 91.13607025146484,
      "activations/layer4_attention_weight_min": -83.64891815185547,
      "activations/layer5_attention_weight_max": 124.39749145507812,
      "activations/layer5_attention_weight_min": -104.90647888183594,
      "activations/layer6_attention_weight_max": 61.191505432128906,
      "activations/layer6_attention_weight_min": -51.27938461303711,
      "activations/layer7_attention_weight_max": 59.05716323852539,
      "activations/layer7_attention_weight_min": -56.47282028198242,
      "activations/layer8_attention_weight_max": 48.463417053222656,
      "activations/layer8_attention_weight_min": -40.4030647277832,
      "activations/layer9_attention_weight_max": 61.44122314453125,
      "activations/layer9_attention_weight_min": -58.339962005615234,
      "epoch": 1.88,
      "learning_rate": 0.00013929772727272726,
      "loss": 2.9893,
      "step": 32300
    },
    {
      "activations/layer0_attention_weight_max": 16.566295623779297,
      "activations/layer0_attention_weight_min": -13.592584609985352,
      "activations/layer10_attention_weight_max": 64.89991760253906,
      "activations/layer10_attention_weight_min": -49.55046463012695,
      "activations/layer11_attention_weight_max": 50.53997039794922,
      "activations/layer11_attention_weight_min": -34.30910110473633,
      "activations/layer12_attention_weight_max": 27.8560733795166,
      "activations/layer12_attention_weight_min": -27.737625122070312,
      "activations/layer13_attention_weight_max": 56.03749084472656,
      "activations/layer13_attention_weight_min": -39.69585037231445,
      "activations/layer14_attention_weight_max": 48.901451110839844,
      "activations/layer14_attention_weight_min": -38.53255081176758,
      "activations/layer15_attention_weight_max": 54.96682357788086,
      "activations/layer15_attention_weight_min": -37.663063049316406,
      "activations/layer16_attention_weight_max": 44.43190002441406,
      "activations/layer16_attention_weight_min": -29.87718391418457,
      "activations/layer17_attention_weight_max": 62.0571174621582,
      "activations/layer17_attention_weight_min": -38.04011917114258,
      "activations/layer18_attention_weight_max": 56.43606948852539,
      "activations/layer18_attention_weight_min": -36.22526550292969,
      "activations/layer19_attention_weight_max": 23.595867156982422,
      "activations/layer19_attention_weight_min": -16.262874603271484,
      "activations/layer1_attention_weight_max": 16.363462448120117,
      "activations/layer1_attention_weight_min": -12.176154136657715,
      "activations/layer20_attention_weight_max": 20.713157653808594,
      "activations/layer20_attention_weight_min": -15.631693840026855,
      "activations/layer21_attention_weight_max": 42.517486572265625,
      "activations/layer21_attention_weight_min": -23.649412155151367,
      "activations/layer22_attention_weight_max": 37.07957077026367,
      "activations/layer22_attention_weight_min": -21.176639556884766,
      "activations/layer23_attention_weight_max": 23.798023223876953,
      "activations/layer23_attention_weight_min": -14.23930549621582,
      "activations/layer2_attention_weight_max": 26.604541778564453,
      "activations/layer2_attention_weight_min": -26.328567504882812,
      "activations/layer3_attention_weight_max": 62.35831832885742,
      "activations/layer3_attention_weight_min": -62.01750183105469,
      "activations/layer4_attention_weight_max": 94.1247329711914,
      "activations/layer4_attention_weight_min": -80.52300262451172,
      "activations/layer5_attention_weight_max": 121.60650634765625,
      "activations/layer5_attention_weight_min": -101.0361557006836,
      "activations/layer6_attention_weight_max": 60.78764724731445,
      "activations/layer6_attention_weight_min": -49.94057083129883,
      "activations/layer7_attention_weight_max": 58.80859375,
      "activations/layer7_attention_weight_min": -54.89561080932617,
      "activations/layer8_attention_weight_max": 47.45619583129883,
      "activations/layer8_attention_weight_min": -39.01982116699219,
      "activations/layer9_attention_weight_max": 64.4306411743164,
      "activations/layer9_attention_weight_min": -59.228187561035156,
      "epoch": 1.88,
      "learning_rate": 0.00013927878787878788,
      "loss": 2.9757,
      "step": 32350
    },
    {
      "activations/layer0_attention_weight_max": 13.914945602416992,
      "activations/layer0_attention_weight_min": -14.074973106384277,
      "activations/layer10_attention_weight_max": 58.73728942871094,
      "activations/layer10_attention_weight_min": -50.89555740356445,
      "activations/layer11_attention_weight_max": 48.51902770996094,
      "activations/layer11_attention_weight_min": -34.86323165893555,
      "activations/layer12_attention_weight_max": 32.49469757080078,
      "activations/layer12_attention_weight_min": -26.192724227905273,
      "activations/layer13_attention_weight_max": 68.02818298339844,
      "activations/layer13_attention_weight_min": -48.30400466918945,
      "activations/layer14_attention_weight_max": 53.251914978027344,
      "activations/layer14_attention_weight_min": -37.50381088256836,
      "activations/layer15_attention_weight_max": 53.76095199584961,
      "activations/layer15_attention_weight_min": -38.856101989746094,
      "activations/layer16_attention_weight_max": 49.713645935058594,
      "activations/layer16_attention_weight_min": -30.977039337158203,
      "activations/layer17_attention_weight_max": 61.844970703125,
      "activations/layer17_attention_weight_min": -38.105690002441406,
      "activations/layer18_attention_weight_max": 60.43085861206055,
      "activations/layer18_attention_weight_min": -32.430694580078125,
      "activations/layer19_attention_weight_max": 21.05748748779297,
      "activations/layer19_attention_weight_min": -15.422714233398438,
      "activations/layer1_attention_weight_max": 16.304040908813477,
      "activations/layer1_attention_weight_min": -12.238384246826172,
      "activations/layer20_attention_weight_max": 19.243976593017578,
      "activations/layer20_attention_weight_min": -18.077545166015625,
      "activations/layer21_attention_weight_max": 40.52015686035156,
      "activations/layer21_attention_weight_min": -25.764177322387695,
      "activations/layer22_attention_weight_max": 38.993614196777344,
      "activations/layer22_attention_weight_min": -23.30710792541504,
      "activations/layer23_attention_weight_max": 25.522258758544922,
      "activations/layer23_attention_weight_min": -14.999167442321777,
      "activations/layer2_attention_weight_max": 28.44681167602539,
      "activations/layer2_attention_weight_min": -26.5950927734375,
      "activations/layer3_attention_weight_max": 60.559730529785156,
      "activations/layer3_attention_weight_min": -63.755775451660156,
      "activations/layer4_attention_weight_max": 90.25341033935547,
      "activations/layer4_attention_weight_min": -79.46627044677734,
      "activations/layer5_attention_weight_max": 122.62062072753906,
      "activations/layer5_attention_weight_min": -107.3353500366211,
      "activations/layer6_attention_weight_max": 59.78493118286133,
      "activations/layer6_attention_weight_min": -53.5909309387207,
      "activations/layer7_attention_weight_max": 56.820343017578125,
      "activations/layer7_attention_weight_min": -55.4525032043457,
      "activations/layer8_attention_weight_max": 45.15262985229492,
      "activations/layer8_attention_weight_min": -40.898399353027344,
      "activations/layer9_attention_weight_max": 63.83979415893555,
      "activations/layer9_attention_weight_min": -58.7263069152832,
      "epoch": 1.88,
      "learning_rate": 0.0001392598484848485,
      "loss": 2.979,
      "step": 32400
    },
    {
      "activations/layer0_attention_weight_max": 13.402297019958496,
      "activations/layer0_attention_weight_min": -13.789358139038086,
      "activations/layer10_attention_weight_max": 56.426170349121094,
      "activations/layer10_attention_weight_min": -46.24387741088867,
      "activations/layer11_attention_weight_max": 42.81916427612305,
      "activations/layer11_attention_weight_min": -37.58935546875,
      "activations/layer12_attention_weight_max": 30.471149444580078,
      "activations/layer12_attention_weight_min": -24.11899185180664,
      "activations/layer13_attention_weight_max": 63.53251266479492,
      "activations/layer13_attention_weight_min": -45.1755256652832,
      "activations/layer14_attention_weight_max": 46.246543884277344,
      "activations/layer14_attention_weight_min": -34.885833740234375,
      "activations/layer15_attention_weight_max": 50.08729553222656,
      "activations/layer15_attention_weight_min": -42.35750198364258,
      "activations/layer16_attention_weight_max": 41.031219482421875,
      "activations/layer16_attention_weight_min": -30.60640525817871,
      "activations/layer17_attention_weight_max": 55.360107421875,
      "activations/layer17_attention_weight_min": -36.67030334472656,
      "activations/layer18_attention_weight_max": 61.21921157836914,
      "activations/layer18_attention_weight_min": -35.29459762573242,
      "activations/layer19_attention_weight_max": 20.308441162109375,
      "activations/layer19_attention_weight_min": -16.131336212158203,
      "activations/layer1_attention_weight_max": 17.293920516967773,
      "activations/layer1_attention_weight_min": -13.973811149597168,
      "activations/layer20_attention_weight_max": 17.96041488647461,
      "activations/layer20_attention_weight_min": -16.13958740234375,
      "activations/layer21_attention_weight_max": 42.38873291015625,
      "activations/layer21_attention_weight_min": -23.597524642944336,
      "activations/layer22_attention_weight_max": 35.20927429199219,
      "activations/layer22_attention_weight_min": -20.823911666870117,
      "activations/layer23_attention_weight_max": 21.928632736206055,
      "activations/layer23_attention_weight_min": -15.572200775146484,
      "activations/layer2_attention_weight_max": 27.521512985229492,
      "activations/layer2_attention_weight_min": -26.433032989501953,
      "activations/layer3_attention_weight_max": 60.38896942138672,
      "activations/layer3_attention_weight_min": -62.79437255859375,
      "activations/layer4_attention_weight_max": 94.11475372314453,
      "activations/layer4_attention_weight_min": -81.61039733886719,
      "activations/layer5_attention_weight_max": 133.83555603027344,
      "activations/layer5_attention_weight_min": -110.67778778076172,
      "activations/layer6_attention_weight_max": 63.91968536376953,
      "activations/layer6_attention_weight_min": -54.092796325683594,
      "activations/layer7_attention_weight_max": 64.9871597290039,
      "activations/layer7_attention_weight_min": -53.34103012084961,
      "activations/layer8_attention_weight_max": 49.662715911865234,
      "activations/layer8_attention_weight_min": -41.640159606933594,
      "activations/layer9_attention_weight_max": 65.30171966552734,
      "activations/layer9_attention_weight_min": -63.651649475097656,
      "epoch": 1.89,
      "learning_rate": 0.00013924090909090908,
      "loss": 2.9668,
      "step": 32450
    },
    {
      "activations/layer0_attention_weight_max": 14.535847663879395,
      "activations/layer0_attention_weight_min": -14.050004005432129,
      "activations/layer10_attention_weight_max": 60.95872497558594,
      "activations/layer10_attention_weight_min": -53.93280029296875,
      "activations/layer11_attention_weight_max": 45.06606674194336,
      "activations/layer11_attention_weight_min": -39.65827941894531,
      "activations/layer12_attention_weight_max": 26.063840866088867,
      "activations/layer12_attention_weight_min": -23.435707092285156,
      "activations/layer13_attention_weight_max": 73.38488006591797,
      "activations/layer13_attention_weight_min": -50.478477478027344,
      "activations/layer14_attention_weight_max": 51.131690979003906,
      "activations/layer14_attention_weight_min": -34.77561950683594,
      "activations/layer15_attention_weight_max": 64.34295654296875,
      "activations/layer15_attention_weight_min": -42.38426971435547,
      "activations/layer16_attention_weight_max": 52.8365364074707,
      "activations/layer16_attention_weight_min": -35.58253479003906,
      "activations/layer17_attention_weight_max": 73.1928939819336,
      "activations/layer17_attention_weight_min": -39.71888732910156,
      "activations/layer18_attention_weight_max": 74.52340698242188,
      "activations/layer18_attention_weight_min": -37.75593566894531,
      "activations/layer19_attention_weight_max": 23.81416893005371,
      "activations/layer19_attention_weight_min": -17.39031410217285,
      "activations/layer1_attention_weight_max": 15.771021842956543,
      "activations/layer1_attention_weight_min": -11.693860054016113,
      "activations/layer20_attention_weight_max": 23.177827835083008,
      "activations/layer20_attention_weight_min": -17.123647689819336,
      "activations/layer21_attention_weight_max": 44.505680084228516,
      "activations/layer21_attention_weight_min": -24.450952529907227,
      "activations/layer22_attention_weight_max": 41.76064682006836,
      "activations/layer22_attention_weight_min": -23.041053771972656,
      "activations/layer23_attention_weight_max": 26.599143981933594,
      "activations/layer23_attention_weight_min": -14.75294303894043,
      "activations/layer2_attention_weight_max": 25.284833908081055,
      "activations/layer2_attention_weight_min": -25.772369384765625,
      "activations/layer3_attention_weight_max": 59.23988342285156,
      "activations/layer3_attention_weight_min": -62.415164947509766,
      "activations/layer4_attention_weight_max": 89.19190216064453,
      "activations/layer4_attention_weight_min": -79.72930145263672,
      "activations/layer5_attention_weight_max": 119.45738983154297,
      "activations/layer5_attention_weight_min": -108.25236511230469,
      "activations/layer6_attention_weight_max": 61.96410369873047,
      "activations/layer6_attention_weight_min": -56.96247100830078,
      "activations/layer7_attention_weight_max": 59.1235237121582,
      "activations/layer7_attention_weight_min": -57.989356994628906,
      "activations/layer8_attention_weight_max": 51.656246185302734,
      "activations/layer8_attention_weight_min": -43.52873992919922,
      "activations/layer9_attention_weight_max": 63.36739730834961,
      "activations/layer9_attention_weight_min": -59.239070892333984,
      "epoch": 1.89,
      "learning_rate": 0.0001392219696969697,
      "loss": 2.9667,
      "step": 32500
    },
    {
      "activations/layer0_attention_weight_max": 15.60181713104248,
      "activations/layer0_attention_weight_min": -13.740550994873047,
      "activations/layer10_attention_weight_max": 67.65748596191406,
      "activations/layer10_attention_weight_min": -54.95494842529297,
      "activations/layer11_attention_weight_max": 46.991302490234375,
      "activations/layer11_attention_weight_min": -34.36391067504883,
      "activations/layer12_attention_weight_max": 32.3804931640625,
      "activations/layer12_attention_weight_min": -25.125534057617188,
      "activations/layer13_attention_weight_max": 74.3132553100586,
      "activations/layer13_attention_weight_min": -47.31112289428711,
      "activations/layer14_attention_weight_max": 59.07339096069336,
      "activations/layer14_attention_weight_min": -36.67294692993164,
      "activations/layer15_attention_weight_max": 57.00578308105469,
      "activations/layer15_attention_weight_min": -42.13565444946289,
      "activations/layer16_attention_weight_max": 46.77323913574219,
      "activations/layer16_attention_weight_min": -29.587738037109375,
      "activations/layer17_attention_weight_max": 63.38873291015625,
      "activations/layer17_attention_weight_min": -36.03378677368164,
      "activations/layer18_attention_weight_max": 58.97556686401367,
      "activations/layer18_attention_weight_min": -34.74293518066406,
      "activations/layer19_attention_weight_max": 21.08308982849121,
      "activations/layer19_attention_weight_min": -15.029492378234863,
      "activations/layer1_attention_weight_max": 16.537675857543945,
      "activations/layer1_attention_weight_min": -11.784356117248535,
      "activations/layer20_attention_weight_max": 18.93520164489746,
      "activations/layer20_attention_weight_min": -16.517074584960938,
      "activations/layer21_attention_weight_max": 35.421539306640625,
      "activations/layer21_attention_weight_min": -22.374603271484375,
      "activations/layer22_attention_weight_max": 36.78732681274414,
      "activations/layer22_attention_weight_min": -19.279253005981445,
      "activations/layer23_attention_weight_max": 22.700698852539062,
      "activations/layer23_attention_weight_min": -14.694441795349121,
      "activations/layer2_attention_weight_max": 28.533855438232422,
      "activations/layer2_attention_weight_min": -26.54513168334961,
      "activations/layer3_attention_weight_max": 60.871639251708984,
      "activations/layer3_attention_weight_min": -63.60879898071289,
      "activations/layer4_attention_weight_max": 95.0129623413086,
      "activations/layer4_attention_weight_min": -82.9629898071289,
      "activations/layer5_attention_weight_max": 129.7628173828125,
      "activations/layer5_attention_weight_min": -102.75587463378906,
      "activations/layer6_attention_weight_max": 63.060943603515625,
      "activations/layer6_attention_weight_min": -50.261226654052734,
      "activations/layer7_attention_weight_max": 66.45577239990234,
      "activations/layer7_attention_weight_min": -54.70852279663086,
      "activations/layer8_attention_weight_max": 51.89219665527344,
      "activations/layer8_attention_weight_min": -42.333740234375,
      "activations/layer9_attention_weight_max": 71.90178680419922,
      "activations/layer9_attention_weight_min": -60.69143295288086,
      "epoch": 1.89,
      "learning_rate": 0.00013920303030303028,
      "loss": 2.9633,
      "step": 32550
    },
    {
      "activations/layer0_attention_weight_max": 16.026023864746094,
      "activations/layer0_attention_weight_min": -13.540847778320312,
      "activations/layer10_attention_weight_max": 65.1437759399414,
      "activations/layer10_attention_weight_min": -53.238983154296875,
      "activations/layer11_attention_weight_max": 51.389427185058594,
      "activations/layer11_attention_weight_min": -38.018211364746094,
      "activations/layer12_attention_weight_max": 31.99915313720703,
      "activations/layer12_attention_weight_min": -27.425817489624023,
      "activations/layer13_attention_weight_max": 67.25457000732422,
      "activations/layer13_attention_weight_min": -48.11813735961914,
      "activations/layer14_attention_weight_max": 63.41725158691406,
      "activations/layer14_attention_weight_min": -42.15976333618164,
      "activations/layer15_attention_weight_max": 55.95765686035156,
      "activations/layer15_attention_weight_min": -39.50666046142578,
      "activations/layer16_attention_weight_max": 42.835445404052734,
      "activations/layer16_attention_weight_min": -28.235048294067383,
      "activations/layer17_attention_weight_max": 57.63933181762695,
      "activations/layer17_attention_weight_min": -37.81386947631836,
      "activations/layer18_attention_weight_max": 59.15747833251953,
      "activations/layer18_attention_weight_min": -34.519378662109375,
      "activations/layer19_attention_weight_max": 20.212553024291992,
      "activations/layer19_attention_weight_min": -16.08729362487793,
      "activations/layer1_attention_weight_max": 16.572772979736328,
      "activations/layer1_attention_weight_min": -13.607730865478516,
      "activations/layer20_attention_weight_max": 19.505945205688477,
      "activations/layer20_attention_weight_min": -17.61273765563965,
      "activations/layer21_attention_weight_max": 39.89666748046875,
      "activations/layer21_attention_weight_min": -22.595117568969727,
      "activations/layer22_attention_weight_max": 36.474727630615234,
      "activations/layer22_attention_weight_min": -20.650903701782227,
      "activations/layer23_attention_weight_max": 21.43850326538086,
      "activations/layer23_attention_weight_min": -14.040122985839844,
      "activations/layer2_attention_weight_max": 28.760154724121094,
      "activations/layer2_attention_weight_min": -28.303993225097656,
      "activations/layer3_attention_weight_max": 65.23379516601562,
      "activations/layer3_attention_weight_min": -67.80093383789062,
      "activations/layer4_attention_weight_max": 92.669189453125,
      "activations/layer4_attention_weight_min": -86.03888702392578,
      "activations/layer5_attention_weight_max": 131.68833923339844,
      "activations/layer5_attention_weight_min": -116.22467041015625,
      "activations/layer6_attention_weight_max": 63.56475067138672,
      "activations/layer6_attention_weight_min": -56.57743835449219,
      "activations/layer7_attention_weight_max": 63.95782470703125,
      "activations/layer7_attention_weight_min": -55.38167190551758,
      "activations/layer8_attention_weight_max": 54.89388656616211,
      "activations/layer8_attention_weight_min": -46.28004455566406,
      "activations/layer9_attention_weight_max": 81.14073944091797,
      "activations/layer9_attention_weight_min": -67.01212310791016,
      "epoch": 1.89,
      "learning_rate": 0.0001391840909090909,
      "loss": 2.9894,
      "step": 32600
    },
    {
      "activations/layer0_attention_weight_max": 13.820231437683105,
      "activations/layer0_attention_weight_min": -13.09190845489502,
      "activations/layer10_attention_weight_max": 54.04093933105469,
      "activations/layer10_attention_weight_min": -49.13350296020508,
      "activations/layer11_attention_weight_max": 44.602195739746094,
      "activations/layer11_attention_weight_min": -36.244834899902344,
      "activations/layer12_attention_weight_max": 33.40877151489258,
      "activations/layer12_attention_weight_min": -25.178714752197266,
      "activations/layer13_attention_weight_max": 57.07477951049805,
      "activations/layer13_attention_weight_min": -46.08106994628906,
      "activations/layer14_attention_weight_max": 48.39039993286133,
      "activations/layer14_attention_weight_min": -37.14826202392578,
      "activations/layer15_attention_weight_max": 47.83418655395508,
      "activations/layer15_attention_weight_min": -35.69593811035156,
      "activations/layer16_attention_weight_max": 42.422698974609375,
      "activations/layer16_attention_weight_min": -32.23112487792969,
      "activations/layer17_attention_weight_max": 58.52587890625,
      "activations/layer17_attention_weight_min": -37.276893615722656,
      "activations/layer18_attention_weight_max": 55.5573616027832,
      "activations/layer18_attention_weight_min": -36.437286376953125,
      "activations/layer19_attention_weight_max": 20.640790939331055,
      "activations/layer19_attention_weight_min": -15.385225296020508,
      "activations/layer1_attention_weight_max": 17.946060180664062,
      "activations/layer1_attention_weight_min": -12.438948631286621,
      "activations/layer20_attention_weight_max": 19.20081329345703,
      "activations/layer20_attention_weight_min": -17.326641082763672,
      "activations/layer21_attention_weight_max": 42.05378341674805,
      "activations/layer21_attention_weight_min": -21.927082061767578,
      "activations/layer22_attention_weight_max": 36.53398895263672,
      "activations/layer22_attention_weight_min": -22.41304588317871,
      "activations/layer23_attention_weight_max": 22.407217025756836,
      "activations/layer23_attention_weight_min": -15.42658519744873,
      "activations/layer2_attention_weight_max": 26.61785125732422,
      "activations/layer2_attention_weight_min": -26.26373863220215,
      "activations/layer3_attention_weight_max": 61.695953369140625,
      "activations/layer3_attention_weight_min": -66.85384368896484,
      "activations/layer4_attention_weight_max": 90.05096435546875,
      "activations/layer4_attention_weight_min": -82.85442352294922,
      "activations/layer5_attention_weight_max": 124.81111907958984,
      "activations/layer5_attention_weight_min": -112.90202331542969,
      "activations/layer6_attention_weight_max": 58.78846740722656,
      "activations/layer6_attention_weight_min": -54.448341369628906,
      "activations/layer7_attention_weight_max": 55.88637924194336,
      "activations/layer7_attention_weight_min": -54.445167541503906,
      "activations/layer8_attention_weight_max": 48.55571365356445,
      "activations/layer8_attention_weight_min": -44.89693069458008,
      "activations/layer9_attention_weight_max": 67.96749114990234,
      "activations/layer9_attention_weight_min": -62.92722702026367,
      "epoch": 1.9,
      "learning_rate": 0.0001391651515151515,
      "loss": 2.9721,
      "step": 32650
    },
    {
      "activations/layer0_attention_weight_max": 14.183415412902832,
      "activations/layer0_attention_weight_min": -14.099336624145508,
      "activations/layer10_attention_weight_max": 56.43633270263672,
      "activations/layer10_attention_weight_min": -51.130489349365234,
      "activations/layer11_attention_weight_max": 47.492156982421875,
      "activations/layer11_attention_weight_min": -35.23982238769531,
      "activations/layer12_attention_weight_max": 32.68931198120117,
      "activations/layer12_attention_weight_min": -26.949851989746094,
      "activations/layer13_attention_weight_max": 55.75728225708008,
      "activations/layer13_attention_weight_min": -39.61281204223633,
      "activations/layer14_attention_weight_max": 52.06425094604492,
      "activations/layer14_attention_weight_min": -40.67507553100586,
      "activations/layer15_attention_weight_max": 47.854515075683594,
      "activations/layer15_attention_weight_min": -36.181121826171875,
      "activations/layer16_attention_weight_max": 44.685298919677734,
      "activations/layer16_attention_weight_min": -29.678462982177734,
      "activations/layer17_attention_weight_max": 58.78297424316406,
      "activations/layer17_attention_weight_min": -40.16887283325195,
      "activations/layer18_attention_weight_max": 60.98418426513672,
      "activations/layer18_attention_weight_min": -32.76905059814453,
      "activations/layer19_attention_weight_max": 18.937490463256836,
      "activations/layer19_attention_weight_min": -17.603620529174805,
      "activations/layer1_attention_weight_max": 15.837334632873535,
      "activations/layer1_attention_weight_min": -12.373181343078613,
      "activations/layer20_attention_weight_max": 19.728174209594727,
      "activations/layer20_attention_weight_min": -17.631839752197266,
      "activations/layer21_attention_weight_max": 42.09476852416992,
      "activations/layer21_attention_weight_min": -24.387041091918945,
      "activations/layer22_attention_weight_max": 35.07063293457031,
      "activations/layer22_attention_weight_min": -22.080650329589844,
      "activations/layer23_attention_weight_max": 24.210567474365234,
      "activations/layer23_attention_weight_min": -15.816795349121094,
      "activations/layer2_attention_weight_max": 26.440542221069336,
      "activations/layer2_attention_weight_min": -25.974103927612305,
      "activations/layer3_attention_weight_max": 61.05437088012695,
      "activations/layer3_attention_weight_min": -62.288719177246094,
      "activations/layer4_attention_weight_max": 92.43900299072266,
      "activations/layer4_attention_weight_min": -78.17874145507812,
      "activations/layer5_attention_weight_max": 117.95094299316406,
      "activations/layer5_attention_weight_min": -106.24544525146484,
      "activations/layer6_attention_weight_max": 57.0751953125,
      "activations/layer6_attention_weight_min": -52.764400482177734,
      "activations/layer7_attention_weight_max": 55.317718505859375,
      "activations/layer7_attention_weight_min": -52.30655288696289,
      "activations/layer8_attention_weight_max": 44.893436431884766,
      "activations/layer8_attention_weight_min": -42.95869064331055,
      "activations/layer9_attention_weight_max": 59.482051849365234,
      "activations/layer9_attention_weight_min": -54.0458869934082,
      "epoch": 1.9,
      "learning_rate": 0.0001391462121212121,
      "loss": 2.9924,
      "step": 32700
    },
    {
      "activations/layer0_attention_weight_max": 13.36325740814209,
      "activations/layer0_attention_weight_min": -13.276656150817871,
      "activations/layer10_attention_weight_max": 63.870140075683594,
      "activations/layer10_attention_weight_min": -50.35276794433594,
      "activations/layer11_attention_weight_max": 41.065921783447266,
      "activations/layer11_attention_weight_min": -37.90010070800781,
      "activations/layer12_attention_weight_max": 33.56198501586914,
      "activations/layer12_attention_weight_min": -28.438085556030273,
      "activations/layer13_attention_weight_max": 51.29299545288086,
      "activations/layer13_attention_weight_min": -39.740169525146484,
      "activations/layer14_attention_weight_max": 50.14846420288086,
      "activations/layer14_attention_weight_min": -34.48328399658203,
      "activations/layer15_attention_weight_max": 51.359622955322266,
      "activations/layer15_attention_weight_min": -38.62187576293945,
      "activations/layer16_attention_weight_max": 46.122684478759766,
      "activations/layer16_attention_weight_min": -29.282270431518555,
      "activations/layer17_attention_weight_max": 54.769683837890625,
      "activations/layer17_attention_weight_min": -34.29045104980469,
      "activations/layer18_attention_weight_max": 53.34267044067383,
      "activations/layer18_attention_weight_min": -31.93359375,
      "activations/layer19_attention_weight_max": 25.12398910522461,
      "activations/layer19_attention_weight_min": -14.150120735168457,
      "activations/layer1_attention_weight_max": 15.842440605163574,
      "activations/layer1_attention_weight_min": -12.091585159301758,
      "activations/layer20_attention_weight_max": 21.21824073791504,
      "activations/layer20_attention_weight_min": -17.156064987182617,
      "activations/layer21_attention_weight_max": 40.77065658569336,
      "activations/layer21_attention_weight_min": -23.92009735107422,
      "activations/layer22_attention_weight_max": 38.31148147583008,
      "activations/layer22_attention_weight_min": -20.113704681396484,
      "activations/layer23_attention_weight_max": 23.985713958740234,
      "activations/layer23_attention_weight_min": -14.212115287780762,
      "activations/layer2_attention_weight_max": 25.6097412109375,
      "activations/layer2_attention_weight_min": -25.201858520507812,
      "activations/layer3_attention_weight_max": 59.68049621582031,
      "activations/layer3_attention_weight_min": -61.996681213378906,
      "activations/layer4_attention_weight_max": 90.4347152709961,
      "activations/layer4_attention_weight_min": -80.44874572753906,
      "activations/layer5_attention_weight_max": 123.54500579833984,
      "activations/layer5_attention_weight_min": -111.0079574584961,
      "activations/layer6_attention_weight_max": 60.689048767089844,
      "activations/layer6_attention_weight_min": -51.758792877197266,
      "activations/layer7_attention_weight_max": 57.1920166015625,
      "activations/layer7_attention_weight_min": -55.09702682495117,
      "activations/layer8_attention_weight_max": 44.39155578613281,
      "activations/layer8_attention_weight_min": -40.36035919189453,
      "activations/layer9_attention_weight_max": 58.427825927734375,
      "activations/layer9_attention_weight_min": -55.18607711791992,
      "epoch": 1.9,
      "learning_rate": 0.00013912727272727272,
      "loss": 2.9714,
      "step": 32750
    },
    {
      "activations/layer0_attention_weight_max": 16.09441375732422,
      "activations/layer0_attention_weight_min": -14.087968826293945,
      "activations/layer10_attention_weight_max": 61.9559326171875,
      "activations/layer10_attention_weight_min": -46.1913948059082,
      "activations/layer11_attention_weight_max": 45.5457878112793,
      "activations/layer11_attention_weight_min": -32.972164154052734,
      "activations/layer12_attention_weight_max": 36.259822845458984,
      "activations/layer12_attention_weight_min": -27.4052791595459,
      "activations/layer13_attention_weight_max": 58.23178482055664,
      "activations/layer13_attention_weight_min": -45.9981803894043,
      "activations/layer14_attention_weight_max": 54.01463317871094,
      "activations/layer14_attention_weight_min": -40.492095947265625,
      "activations/layer15_attention_weight_max": 53.84099197387695,
      "activations/layer15_attention_weight_min": -36.93094253540039,
      "activations/layer16_attention_weight_max": 43.335304260253906,
      "activations/layer16_attention_weight_min": -28.603408813476562,
      "activations/layer17_attention_weight_max": 66.7339096069336,
      "activations/layer17_attention_weight_min": -36.842933654785156,
      "activations/layer18_attention_weight_max": 60.4892692565918,
      "activations/layer18_attention_weight_min": -32.537132263183594,
      "activations/layer19_attention_weight_max": 23.61477279663086,
      "activations/layer19_attention_weight_min": -16.68512535095215,
      "activations/layer1_attention_weight_max": 16.400697708129883,
      "activations/layer1_attention_weight_min": -12.534937858581543,
      "activations/layer20_attention_weight_max": 20.625444412231445,
      "activations/layer20_attention_weight_min": -16.23692512512207,
      "activations/layer21_attention_weight_max": 43.83539581298828,
      "activations/layer21_attention_weight_min": -23.116493225097656,
      "activations/layer22_attention_weight_max": 37.01683044433594,
      "activations/layer22_attention_weight_min": -22.31866455078125,
      "activations/layer23_attention_weight_max": 25.473983764648438,
      "activations/layer23_attention_weight_min": -15.201313018798828,
      "activations/layer2_attention_weight_max": 26.336902618408203,
      "activations/layer2_attention_weight_min": -25.967636108398438,
      "activations/layer3_attention_weight_max": 63.54829406738281,
      "activations/layer3_attention_weight_min": -65.76782989501953,
      "activations/layer4_attention_weight_max": 95.2359390258789,
      "activations/layer4_attention_weight_min": -84.8349838256836,
      "activations/layer5_attention_weight_max": 130.897705078125,
      "activations/layer5_attention_weight_min": -109.60637664794922,
      "activations/layer6_attention_weight_max": 59.1829719543457,
      "activations/layer6_attention_weight_min": -53.156471252441406,
      "activations/layer7_attention_weight_max": 62.806068420410156,
      "activations/layer7_attention_weight_min": -57.7126350402832,
      "activations/layer8_attention_weight_max": 55.28746032714844,
      "activations/layer8_attention_weight_min": -41.58552551269531,
      "activations/layer9_attention_weight_max": 73.67656707763672,
      "activations/layer9_attention_weight_min": -59.2286491394043,
      "epoch": 1.91,
      "learning_rate": 0.00013910833333333333,
      "loss": 2.9857,
      "step": 32800
    },
    {
      "activations/layer0_attention_weight_max": 14.082287788391113,
      "activations/layer0_attention_weight_min": -13.877235412597656,
      "activations/layer10_attention_weight_max": 56.39854049682617,
      "activations/layer10_attention_weight_min": -47.27789306640625,
      "activations/layer11_attention_weight_max": 42.209068298339844,
      "activations/layer11_attention_weight_min": -32.91701126098633,
      "activations/layer12_attention_weight_max": 27.974044799804688,
      "activations/layer12_attention_weight_min": -24.863683700561523,
      "activations/layer13_attention_weight_max": 57.93304443359375,
      "activations/layer13_attention_weight_min": -42.02610778808594,
      "activations/layer14_attention_weight_max": 51.3212890625,
      "activations/layer14_attention_weight_min": -37.86805725097656,
      "activations/layer15_attention_weight_max": 49.68105697631836,
      "activations/layer15_attention_weight_min": -36.48790740966797,
      "activations/layer16_attention_weight_max": 42.378414154052734,
      "activations/layer16_attention_weight_min": -30.883989334106445,
      "activations/layer17_attention_weight_max": 72.955322265625,
      "activations/layer17_attention_weight_min": -35.918338775634766,
      "activations/layer18_attention_weight_max": 54.43376922607422,
      "activations/layer18_attention_weight_min": -35.76216125488281,
      "activations/layer19_attention_weight_max": 21.0815486907959,
      "activations/layer19_attention_weight_min": -16.143226623535156,
      "activations/layer1_attention_weight_max": 17.91926383972168,
      "activations/layer1_attention_weight_min": -12.108352661132812,
      "activations/layer20_attention_weight_max": 19.94738006591797,
      "activations/layer20_attention_weight_min": -16.191123962402344,
      "activations/layer21_attention_weight_max": 44.24860382080078,
      "activations/layer21_attention_weight_min": -25.68223762512207,
      "activations/layer22_attention_weight_max": 38.66310119628906,
      "activations/layer22_attention_weight_min": -20.252716064453125,
      "activations/layer23_attention_weight_max": 23.289424896240234,
      "activations/layer23_attention_weight_min": -15.037954330444336,
      "activations/layer2_attention_weight_max": 28.057804107666016,
      "activations/layer2_attention_weight_min": -27.72443389892578,
      "activations/layer3_attention_weight_max": 64.1143798828125,
      "activations/layer3_attention_weight_min": -67.36976623535156,
      "activations/layer4_attention_weight_max": 96.46304321289062,
      "activations/layer4_attention_weight_min": -88.18091583251953,
      "activations/layer5_attention_weight_max": 132.0321502685547,
      "activations/layer5_attention_weight_min": -107.84494018554688,
      "activations/layer6_attention_weight_max": 62.13064956665039,
      "activations/layer6_attention_weight_min": -55.007896423339844,
      "activations/layer7_attention_weight_max": 58.0263786315918,
      "activations/layer7_attention_weight_min": -54.543575286865234,
      "activations/layer8_attention_weight_max": 47.62870788574219,
      "activations/layer8_attention_weight_min": -40.10890579223633,
      "activations/layer9_attention_weight_max": 65.44742584228516,
      "activations/layer9_attention_weight_min": -59.98516845703125,
      "epoch": 1.91,
      "learning_rate": 0.00013908939393939392,
      "loss": 2.9611,
      "step": 32850
    },
    {
      "activations/layer0_attention_weight_max": 13.918313026428223,
      "activations/layer0_attention_weight_min": -14.044084548950195,
      "activations/layer10_attention_weight_max": 57.19683074951172,
      "activations/layer10_attention_weight_min": -51.45258712768555,
      "activations/layer11_attention_weight_max": 46.67121505737305,
      "activations/layer11_attention_weight_min": -39.506858825683594,
      "activations/layer12_attention_weight_max": 27.806262969970703,
      "activations/layer12_attention_weight_min": -24.94302749633789,
      "activations/layer13_attention_weight_max": 65.94371032714844,
      "activations/layer13_attention_weight_min": -46.87269592285156,
      "activations/layer14_attention_weight_max": 52.785789489746094,
      "activations/layer14_attention_weight_min": -35.342620849609375,
      "activations/layer15_attention_weight_max": 58.45117950439453,
      "activations/layer15_attention_weight_min": -38.42251968383789,
      "activations/layer16_attention_weight_max": 41.07063674926758,
      "activations/layer16_attention_weight_min": -27.719593048095703,
      "activations/layer17_attention_weight_max": 65.52715301513672,
      "activations/layer17_attention_weight_min": -35.792442321777344,
      "activations/layer18_attention_weight_max": 64.54920959472656,
      "activations/layer18_attention_weight_min": -35.504058837890625,
      "activations/layer19_attention_weight_max": 25.8105411529541,
      "activations/layer19_attention_weight_min": -17.707565307617188,
      "activations/layer1_attention_weight_max": 15.331605911254883,
      "activations/layer1_attention_weight_min": -13.286362648010254,
      "activations/layer20_attention_weight_max": 20.66396713256836,
      "activations/layer20_attention_weight_min": -16.16704750061035,
      "activations/layer21_attention_weight_max": 42.02312088012695,
      "activations/layer21_attention_weight_min": -25.956701278686523,
      "activations/layer22_attention_weight_max": 39.141075134277344,
      "activations/layer22_attention_weight_min": -20.91980743408203,
      "activations/layer23_attention_weight_max": 23.65614128112793,
      "activations/layer23_attention_weight_min": -13.762861251831055,
      "activations/layer2_attention_weight_max": 27.67293357849121,
      "activations/layer2_attention_weight_min": -25.71224021911621,
      "activations/layer3_attention_weight_max": 59.835693359375,
      "activations/layer3_attention_weight_min": -63.698577880859375,
      "activations/layer4_attention_weight_max": 90.38876342773438,
      "activations/layer4_attention_weight_min": -81.82649230957031,
      "activations/layer5_attention_weight_max": 123.17587280273438,
      "activations/layer5_attention_weight_min": -109.33326721191406,
      "activations/layer6_attention_weight_max": 58.85441970825195,
      "activations/layer6_attention_weight_min": -54.8984375,
      "activations/layer7_attention_weight_max": 58.94453048706055,
      "activations/layer7_attention_weight_min": -51.58742141723633,
      "activations/layer8_attention_weight_max": 46.97118377685547,
      "activations/layer8_attention_weight_min": -40.58283996582031,
      "activations/layer9_attention_weight_max": 64.03408813476562,
      "activations/layer9_attention_weight_min": -58.11489486694336,
      "epoch": 1.91,
      "learning_rate": 0.00013907045454545453,
      "loss": 2.9794,
      "step": 32900
    },
    {
      "activations/layer0_attention_weight_max": 15.408902168273926,
      "activations/layer0_attention_weight_min": -13.489350318908691,
      "activations/layer10_attention_weight_max": 59.7024040222168,
      "activations/layer10_attention_weight_min": -49.075347900390625,
      "activations/layer11_attention_weight_max": 46.03017807006836,
      "activations/layer11_attention_weight_min": -36.779842376708984,
      "activations/layer12_attention_weight_max": 34.36259078979492,
      "activations/layer12_attention_weight_min": -24.22749137878418,
      "activations/layer13_attention_weight_max": 61.90164566040039,
      "activations/layer13_attention_weight_min": -46.620601654052734,
      "activations/layer14_attention_weight_max": 57.089508056640625,
      "activations/layer14_attention_weight_min": -42.841678619384766,
      "activations/layer15_attention_weight_max": 58.289852142333984,
      "activations/layer15_attention_weight_min": -40.98541259765625,
      "activations/layer16_attention_weight_max": 44.53229904174805,
      "activations/layer16_attention_weight_min": -32.3451042175293,
      "activations/layer17_attention_weight_max": 64.40563201904297,
      "activations/layer17_attention_weight_min": -38.86928939819336,
      "activations/layer18_attention_weight_max": 60.49775695800781,
      "activations/layer18_attention_weight_min": -38.28140640258789,
      "activations/layer19_attention_weight_max": 22.2039852142334,
      "activations/layer19_attention_weight_min": -17.649578094482422,
      "activations/layer1_attention_weight_max": 15.690571784973145,
      "activations/layer1_attention_weight_min": -12.467005729675293,
      "activations/layer20_attention_weight_max": 21.83445930480957,
      "activations/layer20_attention_weight_min": -16.754060745239258,
      "activations/layer21_attention_weight_max": 44.949928283691406,
      "activations/layer21_attention_weight_min": -28.06901741027832,
      "activations/layer22_attention_weight_max": 41.34318161010742,
      "activations/layer22_attention_weight_min": -23.18317222595215,
      "activations/layer23_attention_weight_max": 27.27579116821289,
      "activations/layer23_attention_weight_min": -15.202868461608887,
      "activations/layer2_attention_weight_max": 31.38553237915039,
      "activations/layer2_attention_weight_min": -28.687057495117188,
      "activations/layer3_attention_weight_max": 61.590572357177734,
      "activations/layer3_attention_weight_min": -68.72132110595703,
      "activations/layer4_attention_weight_max": 97.26109313964844,
      "activations/layer4_attention_weight_min": -87.01090240478516,
      "activations/layer5_attention_weight_max": 132.955810546875,
      "activations/layer5_attention_weight_min": -108.40916442871094,
      "activations/layer6_attention_weight_max": 64.06920623779297,
      "activations/layer6_attention_weight_min": -54.6313591003418,
      "activations/layer7_attention_weight_max": 59.189002990722656,
      "activations/layer7_attention_weight_min": -56.02860641479492,
      "activations/layer8_attention_weight_max": 49.281959533691406,
      "activations/layer8_attention_weight_min": -42.60420608520508,
      "activations/layer9_attention_weight_max": 77.18537139892578,
      "activations/layer9_attention_weight_min": -65.10557556152344,
      "epoch": 1.91,
      "learning_rate": 0.00013905151515151515,
      "loss": 2.9802,
      "step": 32950
    },
    {
      "activations/layer0_attention_weight_max": 14.134883880615234,
      "activations/layer0_attention_weight_min": -13.702315330505371,
      "activations/layer10_attention_weight_max": 59.82138442993164,
      "activations/layer10_attention_weight_min": -51.947593688964844,
      "activations/layer11_attention_weight_max": 44.87008285522461,
      "activations/layer11_attention_weight_min": -37.138710021972656,
      "activations/layer12_attention_weight_max": 28.568880081176758,
      "activations/layer12_attention_weight_min": -23.646400451660156,
      "activations/layer13_attention_weight_max": 75.8045654296875,
      "activations/layer13_attention_weight_min": -44.52698516845703,
      "activations/layer14_attention_weight_max": 51.101593017578125,
      "activations/layer14_attention_weight_min": -38.476959228515625,
      "activations/layer15_attention_weight_max": 56.814998626708984,
      "activations/layer15_attention_weight_min": -40.523536682128906,
      "activations/layer16_attention_weight_max": 45.78067398071289,
      "activations/layer16_attention_weight_min": -27.961345672607422,
      "activations/layer17_attention_weight_max": 58.89229965209961,
      "activations/layer17_attention_weight_min": -35.5319709777832,
      "activations/layer18_attention_weight_max": 59.35985565185547,
      "activations/layer18_attention_weight_min": -33.593475341796875,
      "activations/layer19_attention_weight_max": 22.425268173217773,
      "activations/layer19_attention_weight_min": -15.217254638671875,
      "activations/layer1_attention_weight_max": 16.476551055908203,
      "activations/layer1_attention_weight_min": -14.316081047058105,
      "activations/layer20_attention_weight_max": 17.65842628479004,
      "activations/layer20_attention_weight_min": -16.928781509399414,
      "activations/layer21_attention_weight_max": 42.16436004638672,
      "activations/layer21_attention_weight_min": -22.325712203979492,
      "activations/layer22_attention_weight_max": 41.02080535888672,
      "activations/layer22_attention_weight_min": -21.524497985839844,
      "activations/layer23_attention_weight_max": 22.044700622558594,
      "activations/layer23_attention_weight_min": -14.479743003845215,
      "activations/layer2_attention_weight_max": 26.372785568237305,
      "activations/layer2_attention_weight_min": -25.097288131713867,
      "activations/layer3_attention_weight_max": 61.31236267089844,
      "activations/layer3_attention_weight_min": -64.92965698242188,
      "activations/layer4_attention_weight_max": 90.23675537109375,
      "activations/layer4_attention_weight_min": -84.62615203857422,
      "activations/layer5_attention_weight_max": 121.99278259277344,
      "activations/layer5_attention_weight_min": -109.50655364990234,
      "activations/layer6_attention_weight_max": 63.723724365234375,
      "activations/layer6_attention_weight_min": -55.084693908691406,
      "activations/layer7_attention_weight_max": 55.49632263183594,
      "activations/layer7_attention_weight_min": -52.1268424987793,
      "activations/layer8_attention_weight_max": 43.60670852661133,
      "activations/layer8_attention_weight_min": -39.33837127685547,
      "activations/layer9_attention_weight_max": 61.277320861816406,
      "activations/layer9_attention_weight_min": -52.90789794921875,
      "epoch": 1.92,
      "learning_rate": 0.00013903257575757574,
      "loss": 2.9722,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_loss": 2.923828125,
      "eval_runtime": 8.4653,
      "eval_samples_per_second": 507.247,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_openwebtext_loss": 2.923828125,
      "eval_openwebtext_ppl": 18.61240185635776,
      "eval_openwebtext_runtime": 8.4653,
      "eval_openwebtext_samples_per_second": 507.247,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_wikitext_loss": 3.19921875,
      "eval_wikitext_ppl": 24.513371642677637,
      "eval_wikitext_runtime": 1.9837,
      "eval_wikitext_samples_per_second": 229.872,
      "step": 33000
    },
    {
      "epoch": 1.92,
      "eval_lambada_loss": 3.142578125,
      "eval_lambada_ppl": 23.16350836406023,
      "eval_lambada_runtime": 9.5743,
      "eval_lambada_samples_per_second": 508.547,
      "step": 33000
    },
    {
      "activations/layer0_attention_weight_max": 15.260095596313477,
      "activations/layer0_attention_weight_min": -13.338860511779785,
      "activations/layer10_attention_weight_max": 71.23188781738281,
      "activations/layer10_attention_weight_min": -57.01990509033203,
      "activations/layer11_attention_weight_max": 58.06928253173828,
      "activations/layer11_attention_weight_min": -40.418094635009766,
      "activations/layer12_attention_weight_max": 37.5428352355957,
      "activations/layer12_attention_weight_min": -25.241403579711914,
      "activations/layer13_attention_weight_max": 80.37828063964844,
      "activations/layer13_attention_weight_min": -53.480289459228516,
      "activations/layer14_attention_weight_max": 62.888160705566406,
      "activations/layer14_attention_weight_min": -40.138694763183594,
      "activations/layer15_attention_weight_max": 59.00701141357422,
      "activations/layer15_attention_weight_min": -40.126983642578125,
      "activations/layer16_attention_weight_max": 44.492733001708984,
      "activations/layer16_attention_weight_min": -29.997142791748047,
      "activations/layer17_attention_weight_max": 71.61112213134766,
      "activations/layer17_attention_weight_min": -36.63607406616211,
      "activations/layer18_attention_weight_max": 58.41635513305664,
      "activations/layer18_attention_weight_min": -35.326087951660156,
      "activations/layer19_attention_weight_max": 24.295530319213867,
      "activations/layer19_attention_weight_min": -17.25011444091797,
      "activations/layer1_attention_weight_max": 15.483983039855957,
      "activations/layer1_attention_weight_min": -12.72903823852539,
      "activations/layer20_attention_weight_max": 20.049619674682617,
      "activations/layer20_attention_weight_min": -16.578067779541016,
      "activations/layer21_attention_weight_max": 38.968963623046875,
      "activations/layer21_attention_weight_min": -23.57173728942871,
      "activations/layer22_attention_weight_max": 35.331336975097656,
      "activations/layer22_attention_weight_min": -21.983707427978516,
      "activations/layer23_attention_weight_max": 22.138015747070312,
      "activations/layer23_attention_weight_min": -14.485372543334961,
      "activations/layer2_attention_weight_max": 25.895187377929688,
      "activations/layer2_attention_weight_min": -24.98248863220215,
      "activations/layer3_attention_weight_max": 63.80012512207031,
      "activations/layer3_attention_weight_min": -66.29866790771484,
      "activations/layer4_attention_weight_max": 95.14078521728516,
      "activations/layer4_attention_weight_min": -86.33258056640625,
      "activations/layer5_attention_weight_max": 135.26162719726562,
      "activations/layer5_attention_weight_min": -114.73912811279297,
      "activations/layer6_attention_weight_max": 65.67155456542969,
      "activations/layer6_attention_weight_min": -56.54277420043945,
      "activations/layer7_attention_weight_max": 63.93751907348633,
      "activations/layer7_attention_weight_min": -56.59359359741211,
      "activations/layer8_attention_weight_max": 56.617740631103516,
      "activations/layer8_attention_weight_min": -41.85080337524414,
      "activations/layer9_attention_weight_max": 76.94196319580078,
      "activations/layer9_attention_weight_min": -57.19314193725586,
      "epoch": 1.92,
      "learning_rate": 0.00013901363636363635,
      "loss": 2.9753,
      "step": 33050
    },
    {
      "activations/layer0_attention_weight_max": 15.486140251159668,
      "activations/layer0_attention_weight_min": -13.808881759643555,
      "activations/layer10_attention_weight_max": 64.65110778808594,
      "activations/layer10_attention_weight_min": -49.193939208984375,
      "activations/layer11_attention_weight_max": 46.347591400146484,
      "activations/layer11_attention_weight_min": -34.986385345458984,
      "activations/layer12_attention_weight_max": 29.765623092651367,
      "activations/layer12_attention_weight_min": -24.796653747558594,
      "activations/layer13_attention_weight_max": 63.49112319946289,
      "activations/layer13_attention_weight_min": -40.07256317138672,
      "activations/layer14_attention_weight_max": 54.24381637573242,
      "activations/layer14_attention_weight_min": -36.0021858215332,
      "activations/layer15_attention_weight_max": 52.36100769042969,
      "activations/layer15_attention_weight_min": -38.63530731201172,
      "activations/layer16_attention_weight_max": 45.04999923706055,
      "activations/layer16_attention_weight_min": -26.939594268798828,
      "activations/layer17_attention_weight_max": 58.46318435668945,
      "activations/layer17_attention_weight_min": -35.442283630371094,
      "activations/layer18_attention_weight_max": 59.21826171875,
      "activations/layer18_attention_weight_min": -32.032676696777344,
      "activations/layer19_attention_weight_max": 22.662294387817383,
      "activations/layer19_attention_weight_min": -16.181903839111328,
      "activations/layer1_attention_weight_max": 17.161901473999023,
      "activations/layer1_attention_weight_min": -13.189117431640625,
      "activations/layer20_attention_weight_max": 19.194353103637695,
      "activations/layer20_attention_weight_min": -15.300054550170898,
      "activations/layer21_attention_weight_max": 41.610504150390625,
      "activations/layer21_attention_weight_min": -24.78917694091797,
      "activations/layer22_attention_weight_max": 35.44772720336914,
      "activations/layer22_attention_weight_min": -20.4143123626709,
      "activations/layer23_attention_weight_max": 21.874298095703125,
      "activations/layer23_attention_weight_min": -15.40650749206543,
      "activations/layer2_attention_weight_max": 26.972816467285156,
      "activations/layer2_attention_weight_min": -26.846044540405273,
      "activations/layer3_attention_weight_max": 65.3964614868164,
      "activations/layer3_attention_weight_min": -71.76423645019531,
      "activations/layer4_attention_weight_max": 93.61734771728516,
      "activations/layer4_attention_weight_min": -86.18878173828125,
      "activations/layer5_attention_weight_max": 136.06051635742188,
      "activations/layer5_attention_weight_min": -110.57881927490234,
      "activations/layer6_attention_weight_max": 63.17145919799805,
      "activations/layer6_attention_weight_min": -55.03682327270508,
      "activations/layer7_attention_weight_max": 59.36410903930664,
      "activations/layer7_attention_weight_min": -58.245914459228516,
      "activations/layer8_attention_weight_max": 56.10269546508789,
      "activations/layer8_attention_weight_min": -41.33791732788086,
      "activations/layer9_attention_weight_max": 87.46329498291016,
      "activations/layer9_attention_weight_min": -67.77747344970703,
      "epoch": 1.92,
      "learning_rate": 0.00013899469696969694,
      "loss": 2.9756,
      "step": 33100
    },
    {
      "activations/layer0_attention_weight_max": 14.646184921264648,
      "activations/layer0_attention_weight_min": -13.618582725524902,
      "activations/layer10_attention_weight_max": 63.66600036621094,
      "activations/layer10_attention_weight_min": -50.39334487915039,
      "activations/layer11_attention_weight_max": 40.86561584472656,
      "activations/layer11_attention_weight_min": -33.70998001098633,
      "activations/layer12_attention_weight_max": 25.833969116210938,
      "activations/layer12_attention_weight_min": -25.245323181152344,
      "activations/layer13_attention_weight_max": 66.58458709716797,
      "activations/layer13_attention_weight_min": -50.3953742980957,
      "activations/layer14_attention_weight_max": 58.0662841796875,
      "activations/layer14_attention_weight_min": -39.65912628173828,
      "activations/layer15_attention_weight_max": 55.216712951660156,
      "activations/layer15_attention_weight_min": -38.92920684814453,
      "activations/layer16_attention_weight_max": 42.312679290771484,
      "activations/layer16_attention_weight_min": -29.144195556640625,
      "activations/layer17_attention_weight_max": 71.88536071777344,
      "activations/layer17_attention_weight_min": -40.02886199951172,
      "activations/layer18_attention_weight_max": 65.86962890625,
      "activations/layer18_attention_weight_min": -33.22822952270508,
      "activations/layer19_attention_weight_max": 25.3990535736084,
      "activations/layer19_attention_weight_min": -16.99898910522461,
      "activations/layer1_attention_weight_max": 15.554277420043945,
      "activations/layer1_attention_weight_min": -12.518325805664062,
      "activations/layer20_attention_weight_max": 21.43596076965332,
      "activations/layer20_attention_weight_min": -16.83271026611328,
      "activations/layer21_attention_weight_max": 47.112545013427734,
      "activations/layer21_attention_weight_min": -25.351194381713867,
      "activations/layer22_attention_weight_max": 38.84228515625,
      "activations/layer22_attention_weight_min": -19.40958023071289,
      "activations/layer23_attention_weight_max": 25.96242904663086,
      "activations/layer23_attention_weight_min": -14.0148344039917,
      "activations/layer2_attention_weight_max": 26.233600616455078,
      "activations/layer2_attention_weight_min": -26.592090606689453,
      "activations/layer3_attention_weight_max": 63.49632263183594,
      "activations/layer3_attention_weight_min": -65.52830505371094,
      "activations/layer4_attention_weight_max": 100.23414611816406,
      "activations/layer4_attention_weight_min": -85.60926818847656,
      "activations/layer5_attention_weight_max": 134.09140014648438,
      "activations/layer5_attention_weight_min": -111.74214172363281,
      "activations/layer6_attention_weight_max": 64.74139404296875,
      "activations/layer6_attention_weight_min": -54.89936447143555,
      "activations/layer7_attention_weight_max": 60.49702072143555,
      "activations/layer7_attention_weight_min": -53.45839309692383,
      "activations/layer8_attention_weight_max": 48.0503044128418,
      "activations/layer8_attention_weight_min": -40.3807373046875,
      "activations/layer9_attention_weight_max": 61.95782470703125,
      "activations/layer9_attention_weight_min": -57.252479553222656,
      "epoch": 1.93,
      "learning_rate": 0.00013897575757575756,
      "loss": 2.9934,
      "step": 33150
    },
    {
      "activations/layer0_attention_weight_max": 13.859476089477539,
      "activations/layer0_attention_weight_min": -13.609651565551758,
      "activations/layer10_attention_weight_max": 62.331329345703125,
      "activations/layer10_attention_weight_min": -48.18149185180664,
      "activations/layer11_attention_weight_max": 47.219390869140625,
      "activations/layer11_attention_weight_min": -37.98426818847656,
      "activations/layer12_attention_weight_max": 30.337955474853516,
      "activations/layer12_attention_weight_min": -27.665658950805664,
      "activations/layer13_attention_weight_max": 66.49270629882812,
      "activations/layer13_attention_weight_min": -49.83147430419922,
      "activations/layer14_attention_weight_max": 61.319786071777344,
      "activations/layer14_attention_weight_min": -47.52717590332031,
      "activations/layer15_attention_weight_max": 55.63628005981445,
      "activations/layer15_attention_weight_min": -41.06645965576172,
      "activations/layer16_attention_weight_max": 48.22945022583008,
      "activations/layer16_attention_weight_min": -30.895429611206055,
      "activations/layer17_attention_weight_max": 61.25497817993164,
      "activations/layer17_attention_weight_min": -36.67746353149414,
      "activations/layer18_attention_weight_max": 61.114707946777344,
      "activations/layer18_attention_weight_min": -33.23912811279297,
      "activations/layer19_attention_weight_max": 19.794179916381836,
      "activations/layer19_attention_weight_min": -14.859103202819824,
      "activations/layer1_attention_weight_max": 16.816299438476562,
      "activations/layer1_attention_weight_min": -13.766850471496582,
      "activations/layer20_attention_weight_max": 19.792749404907227,
      "activations/layer20_attention_weight_min": -14.162328720092773,
      "activations/layer21_attention_weight_max": 41.450927734375,
      "activations/layer21_attention_weight_min": -24.58879280090332,
      "activations/layer22_attention_weight_max": 34.3943977355957,
      "activations/layer22_attention_weight_min": -20.94917106628418,
      "activations/layer23_attention_weight_max": 25.225112915039062,
      "activations/layer23_attention_weight_min": -15.686687469482422,
      "activations/layer2_attention_weight_max": 25.694387435913086,
      "activations/layer2_attention_weight_min": -25.46851348876953,
      "activations/layer3_attention_weight_max": 62.413551330566406,
      "activations/layer3_attention_weight_min": -66.2332763671875,
      "activations/layer4_attention_weight_max": 94.76521301269531,
      "activations/layer4_attention_weight_min": -87.13166809082031,
      "activations/layer5_attention_weight_max": 139.32398986816406,
      "activations/layer5_attention_weight_min": -111.78340148925781,
      "activations/layer6_attention_weight_max": 69.61119842529297,
      "activations/layer6_attention_weight_min": -58.86555099487305,
      "activations/layer7_attention_weight_max": 68.11335754394531,
      "activations/layer7_attention_weight_min": -57.0174560546875,
      "activations/layer8_attention_weight_max": 53.153751373291016,
      "activations/layer8_attention_weight_min": -48.712486267089844,
      "activations/layer9_attention_weight_max": 70.24871063232422,
      "activations/layer9_attention_weight_min": -69.65238189697266,
      "epoch": 1.93,
      "learning_rate": 0.00013895681818181817,
      "loss": 2.9761,
      "step": 33200
    },
    {
      "activations/layer0_attention_weight_max": 13.802018165588379,
      "activations/layer0_attention_weight_min": -13.522640228271484,
      "activations/layer10_attention_weight_max": 56.779449462890625,
      "activations/layer10_attention_weight_min": -47.89093017578125,
      "activations/layer11_attention_weight_max": 47.23464584350586,
      "activations/layer11_attention_weight_min": -36.249691009521484,
      "activations/layer12_attention_weight_max": 31.175626754760742,
      "activations/layer12_attention_weight_min": -26.939411163330078,
      "activations/layer13_attention_weight_max": 56.70758056640625,
      "activations/layer13_attention_weight_min": -41.3090705871582,
      "activations/layer14_attention_weight_max": 51.997798919677734,
      "activations/layer14_attention_weight_min": -36.82538986206055,
      "activations/layer15_attention_weight_max": 56.63497543334961,
      "activations/layer15_attention_weight_min": -42.08958435058594,
      "activations/layer16_attention_weight_max": 39.40446472167969,
      "activations/layer16_attention_weight_min": -30.772720336914062,
      "activations/layer17_attention_weight_max": 61.22665023803711,
      "activations/layer17_attention_weight_min": -37.115875244140625,
      "activations/layer18_attention_weight_max": 58.96736145019531,
      "activations/layer18_attention_weight_min": -34.21088409423828,
      "activations/layer19_attention_weight_max": 20.163246154785156,
      "activations/layer19_attention_weight_min": -17.743167877197266,
      "activations/layer1_attention_weight_max": 16.751420974731445,
      "activations/layer1_attention_weight_min": -13.467327117919922,
      "activations/layer20_attention_weight_max": 18.690568923950195,
      "activations/layer20_attention_weight_min": -17.32948875427246,
      "activations/layer21_attention_weight_max": 40.487648010253906,
      "activations/layer21_attention_weight_min": -23.296798706054688,
      "activations/layer22_attention_weight_max": 33.67732238769531,
      "activations/layer22_attention_weight_min": -23.105453491210938,
      "activations/layer23_attention_weight_max": 24.30104637145996,
      "activations/layer23_attention_weight_min": -17.22967529296875,
      "activations/layer2_attention_weight_max": 26.843311309814453,
      "activations/layer2_attention_weight_min": -25.9044189453125,
      "activations/layer3_attention_weight_max": 60.15721893310547,
      "activations/layer3_attention_weight_min": -60.24693298339844,
      "activations/layer4_attention_weight_max": 90.01033782958984,
      "activations/layer4_attention_weight_min": -79.6988296508789,
      "activations/layer5_attention_weight_max": 126.4989242553711,
      "activations/layer5_attention_weight_min": -102.44503021240234,
      "activations/layer6_attention_weight_max": 60.010135650634766,
      "activations/layer6_attention_weight_min": -53.64488983154297,
      "activations/layer7_attention_weight_max": 56.02663040161133,
      "activations/layer7_attention_weight_min": -51.083396911621094,
      "activations/layer8_attention_weight_max": 44.42192840576172,
      "activations/layer8_attention_weight_min": -44.69853973388672,
      "activations/layer9_attention_weight_max": 62.259212493896484,
      "activations/layer9_attention_weight_min": -57.84977340698242,
      "epoch": 1.93,
      "learning_rate": 0.00013893787878787876,
      "loss": 2.9703,
      "step": 33250
    },
    {
      "activations/layer0_attention_weight_max": 14.247730255126953,
      "activations/layer0_attention_weight_min": -13.948615074157715,
      "activations/layer10_attention_weight_max": 56.96417999267578,
      "activations/layer10_attention_weight_min": -45.775108337402344,
      "activations/layer11_attention_weight_max": 43.25239181518555,
      "activations/layer11_attention_weight_min": -35.009239196777344,
      "activations/layer12_attention_weight_max": 26.06406593322754,
      "activations/layer12_attention_weight_min": -22.61860466003418,
      "activations/layer13_attention_weight_max": 71.38914489746094,
      "activations/layer13_attention_weight_min": -46.347347259521484,
      "activations/layer14_attention_weight_max": 56.84519577026367,
      "activations/layer14_attention_weight_min": -33.967350006103516,
      "activations/layer15_attention_weight_max": 54.95998764038086,
      "activations/layer15_attention_weight_min": -35.925201416015625,
      "activations/layer16_attention_weight_max": 44.61923599243164,
      "activations/layer16_attention_weight_min": -29.171764373779297,
      "activations/layer17_attention_weight_max": 54.53919982910156,
      "activations/layer17_attention_weight_min": -38.336814880371094,
      "activations/layer18_attention_weight_max": 57.21186828613281,
      "activations/layer18_attention_weight_min": -32.44419479370117,
      "activations/layer19_attention_weight_max": 22.52886962890625,
      "activations/layer19_attention_weight_min": -15.366244316101074,
      "activations/layer1_attention_weight_max": 15.401759147644043,
      "activations/layer1_attention_weight_min": -13.25747299194336,
      "activations/layer20_attention_weight_max": 20.95720100402832,
      "activations/layer20_attention_weight_min": -15.83056640625,
      "activations/layer21_attention_weight_max": 37.203590393066406,
      "activations/layer21_attention_weight_min": -21.291584014892578,
      "activations/layer22_attention_weight_max": 35.296104431152344,
      "activations/layer22_attention_weight_min": -21.990385055541992,
      "activations/layer23_attention_weight_max": 23.622703552246094,
      "activations/layer23_attention_weight_min": -15.348209381103516,
      "activations/layer2_attention_weight_max": 25.5387020111084,
      "activations/layer2_attention_weight_min": -25.549182891845703,
      "activations/layer3_attention_weight_max": 59.35633850097656,
      "activations/layer3_attention_weight_min": -62.518951416015625,
      "activations/layer4_attention_weight_max": 88.49496459960938,
      "activations/layer4_attention_weight_min": -79.1817626953125,
      "activations/layer5_attention_weight_max": 130.75863647460938,
      "activations/layer5_attention_weight_min": -108.70445251464844,
      "activations/layer6_attention_weight_max": 61.24107360839844,
      "activations/layer6_attention_weight_min": -52.84327697753906,
      "activations/layer7_attention_weight_max": 59.656856536865234,
      "activations/layer7_attention_weight_min": -55.33740997314453,
      "activations/layer8_attention_weight_max": 53.815887451171875,
      "activations/layer8_attention_weight_min": -40.94069290161133,
      "activations/layer9_attention_weight_max": 68.82836151123047,
      "activations/layer9_attention_weight_min": -56.30153274536133,
      "epoch": 1.93,
      "learning_rate": 0.00013891893939393937,
      "loss": 2.987,
      "step": 33300
    },
    {
      "activations/layer0_attention_weight_max": 16.27871322631836,
      "activations/layer0_attention_weight_min": -13.928971290588379,
      "activations/layer10_attention_weight_max": 55.835479736328125,
      "activations/layer10_attention_weight_min": -45.05825424194336,
      "activations/layer11_attention_weight_max": 50.151283264160156,
      "activations/layer11_attention_weight_min": -38.813236236572266,
      "activations/layer12_attention_weight_max": 31.644319534301758,
      "activations/layer12_attention_weight_min": -25.358600616455078,
      "activations/layer13_attention_weight_max": 56.285465240478516,
      "activations/layer13_attention_weight_min": -39.7282829284668,
      "activations/layer14_attention_weight_max": 56.256534576416016,
      "activations/layer14_attention_weight_min": -36.14351272583008,
      "activations/layer15_attention_weight_max": 46.98318862915039,
      "activations/layer15_attention_weight_min": -36.0095100402832,
      "activations/layer16_attention_weight_max": 41.95730209350586,
      "activations/layer16_attention_weight_min": -29.43807029724121,
      "activations/layer17_attention_weight_max": 60.08850860595703,
      "activations/layer17_attention_weight_min": -46.02275085449219,
      "activations/layer18_attention_weight_max": 56.664100646972656,
      "activations/layer18_attention_weight_min": -35.37493133544922,
      "activations/layer19_attention_weight_max": 21.54269790649414,
      "activations/layer19_attention_weight_min": -14.608403205871582,
      "activations/layer1_attention_weight_max": 16.371726989746094,
      "activations/layer1_attention_weight_min": -13.906282424926758,
      "activations/layer20_attention_weight_max": 19.937063217163086,
      "activations/layer20_attention_weight_min": -16.31377410888672,
      "activations/layer21_attention_weight_max": 35.84265899658203,
      "activations/layer21_attention_weight_min": -21.576671600341797,
      "activations/layer22_attention_weight_max": 34.36736297607422,
      "activations/layer22_attention_weight_min": -20.111915588378906,
      "activations/layer23_attention_weight_max": 21.7449951171875,
      "activations/layer23_attention_weight_min": -14.239859580993652,
      "activations/layer2_attention_weight_max": 27.423656463623047,
      "activations/layer2_attention_weight_min": -27.032987594604492,
      "activations/layer3_attention_weight_max": 62.6976432800293,
      "activations/layer3_attention_weight_min": -64.78845977783203,
      "activations/layer4_attention_weight_max": 95.3465576171875,
      "activations/layer4_attention_weight_min": -85.44548797607422,
      "activations/layer5_attention_weight_max": 129.36676025390625,
      "activations/layer5_attention_weight_min": -105.67390441894531,
      "activations/layer6_attention_weight_max": 62.054203033447266,
      "activations/layer6_attention_weight_min": -53.94684982299805,
      "activations/layer7_attention_weight_max": 64.80217742919922,
      "activations/layer7_attention_weight_min": -58.86848831176758,
      "activations/layer8_attention_weight_max": 56.232566833496094,
      "activations/layer8_attention_weight_min": -43.388397216796875,
      "activations/layer9_attention_weight_max": 74.52718353271484,
      "activations/layer9_attention_weight_min": -68.71754455566406,
      "epoch": 1.94,
      "learning_rate": 0.0001389,
      "loss": 2.9779,
      "step": 33350
    },
    {
      "activations/layer0_attention_weight_max": 13.734025001525879,
      "activations/layer0_attention_weight_min": -13.253361701965332,
      "activations/layer10_attention_weight_max": 71.63748168945312,
      "activations/layer10_attention_weight_min": -53.0703125,
      "activations/layer11_attention_weight_max": 40.15153884887695,
      "activations/layer11_attention_weight_min": -33.07598114013672,
      "activations/layer12_attention_weight_max": 32.16836929321289,
      "activations/layer12_attention_weight_min": -24.95861053466797,
      "activations/layer13_attention_weight_max": 64.3410415649414,
      "activations/layer13_attention_weight_min": -47.58357238769531,
      "activations/layer14_attention_weight_max": 47.035621643066406,
      "activations/layer14_attention_weight_min": -33.8117790222168,
      "activations/layer15_attention_weight_max": 52.978126525878906,
      "activations/layer15_attention_weight_min": -40.721405029296875,
      "activations/layer16_attention_weight_max": 43.781654357910156,
      "activations/layer16_attention_weight_min": -32.09012985229492,
      "activations/layer17_attention_weight_max": 57.95167541503906,
      "activations/layer17_attention_weight_min": -36.84211730957031,
      "activations/layer18_attention_weight_max": 58.442222595214844,
      "activations/layer18_attention_weight_min": -33.885162353515625,
      "activations/layer19_attention_weight_max": 22.830476760864258,
      "activations/layer19_attention_weight_min": -14.91535472869873,
      "activations/layer1_attention_weight_max": 15.733602523803711,
      "activations/layer1_attention_weight_min": -14.624091148376465,
      "activations/layer20_attention_weight_max": 17.193662643432617,
      "activations/layer20_attention_weight_min": -17.901155471801758,
      "activations/layer21_attention_weight_max": 36.873619079589844,
      "activations/layer21_attention_weight_min": -25.791893005371094,
      "activations/layer22_attention_weight_max": 33.88309097290039,
      "activations/layer22_attention_weight_min": -22.948179244995117,
      "activations/layer23_attention_weight_max": 21.971227645874023,
      "activations/layer23_attention_weight_min": -13.713846206665039,
      "activations/layer2_attention_weight_max": 26.78144073486328,
      "activations/layer2_attention_weight_min": -25.51809310913086,
      "activations/layer3_attention_weight_max": 58.447227478027344,
      "activations/layer3_attention_weight_min": -62.181556701660156,
      "activations/layer4_attention_weight_max": 89.03792572021484,
      "activations/layer4_attention_weight_min": -82.22136688232422,
      "activations/layer5_attention_weight_max": 121.88067626953125,
      "activations/layer5_attention_weight_min": -106.18678283691406,
      "activations/layer6_attention_weight_max": 64.84916687011719,
      "activations/layer6_attention_weight_min": -51.481300354003906,
      "activations/layer7_attention_weight_max": 55.75977325439453,
      "activations/layer7_attention_weight_min": -55.23957443237305,
      "activations/layer8_attention_weight_max": 46.06521987915039,
      "activations/layer8_attention_weight_min": -40.530330657958984,
      "activations/layer9_attention_weight_max": 61.590972900390625,
      "activations/layer9_attention_weight_min": -60.75913619995117,
      "epoch": 1.94,
      "learning_rate": 0.0001388810606060606,
      "loss": 2.9768,
      "step": 33400
    },
    {
      "activations/layer0_attention_weight_max": 14.065694808959961,
      "activations/layer0_attention_weight_min": -13.9320068359375,
      "activations/layer10_attention_weight_max": 67.6768569946289,
      "activations/layer10_attention_weight_min": -51.53123474121094,
      "activations/layer11_attention_weight_max": 45.264976501464844,
      "activations/layer11_attention_weight_min": -37.651432037353516,
      "activations/layer12_attention_weight_max": 30.970212936401367,
      "activations/layer12_attention_weight_min": -27.722976684570312,
      "activations/layer13_attention_weight_max": 61.34557342529297,
      "activations/layer13_attention_weight_min": -45.36064529418945,
      "activations/layer14_attention_weight_max": 51.33719253540039,
      "activations/layer14_attention_weight_min": -37.53923416137695,
      "activations/layer15_attention_weight_max": 51.881370544433594,
      "activations/layer15_attention_weight_min": -36.60650634765625,
      "activations/layer16_attention_weight_max": 41.926475524902344,
      "activations/layer16_attention_weight_min": -30.23650360107422,
      "activations/layer17_attention_weight_max": 60.45768356323242,
      "activations/layer17_attention_weight_min": -36.8213005065918,
      "activations/layer18_attention_weight_max": 62.38972473144531,
      "activations/layer18_attention_weight_min": -36.24149703979492,
      "activations/layer19_attention_weight_max": 29.20564842224121,
      "activations/layer19_attention_weight_min": -15.736139297485352,
      "activations/layer1_attention_weight_max": 18.381479263305664,
      "activations/layer1_attention_weight_min": -11.801433563232422,
      "activations/layer20_attention_weight_max": 21.130138397216797,
      "activations/layer20_attention_weight_min": -16.161945343017578,
      "activations/layer21_attention_weight_max": 38.13242721557617,
      "activations/layer21_attention_weight_min": -24.30887794494629,
      "activations/layer22_attention_weight_max": 36.55266571044922,
      "activations/layer22_attention_weight_min": -22.034332275390625,
      "activations/layer23_attention_weight_max": 26.287429809570312,
      "activations/layer23_attention_weight_min": -14.998781204223633,
      "activations/layer2_attention_weight_max": 27.33232879638672,
      "activations/layer2_attention_weight_min": -26.864891052246094,
      "activations/layer3_attention_weight_max": 61.084617614746094,
      "activations/layer3_attention_weight_min": -64.09990692138672,
      "activations/layer4_attention_weight_max": 94.84382629394531,
      "activations/layer4_attention_weight_min": -83.89936828613281,
      "activations/layer5_attention_weight_max": 131.41021728515625,
      "activations/layer5_attention_weight_min": -105.93034362792969,
      "activations/layer6_attention_weight_max": 63.66222381591797,
      "activations/layer6_attention_weight_min": -54.43461227416992,
      "activations/layer7_attention_weight_max": 62.417503356933594,
      "activations/layer7_attention_weight_min": -55.56019973754883,
      "activations/layer8_attention_weight_max": 48.71770095825195,
      "activations/layer8_attention_weight_min": -41.53997802734375,
      "activations/layer9_attention_weight_max": 66.47171020507812,
      "activations/layer9_attention_weight_min": -57.26649856567383,
      "epoch": 1.94,
      "learning_rate": 0.0001388621212121212,
      "loss": 2.9673,
      "step": 33450
    },
    {
      "activations/layer0_attention_weight_max": 14.066999435424805,
      "activations/layer0_attention_weight_min": -14.109973907470703,
      "activations/layer10_attention_weight_max": 70.50743103027344,
      "activations/layer10_attention_weight_min": -52.3499870300293,
      "activations/layer11_attention_weight_max": 53.38491439819336,
      "activations/layer11_attention_weight_min": -39.72687911987305,
      "activations/layer12_attention_weight_max": 28.241479873657227,
      "activations/layer12_attention_weight_min": -26.113765716552734,
      "activations/layer13_attention_weight_max": 71.39571380615234,
      "activations/layer13_attention_weight_min": -52.1904411315918,
      "activations/layer14_attention_weight_max": 58.21904373168945,
      "activations/layer14_attention_weight_min": -40.93467330932617,
      "activations/layer15_attention_weight_max": 56.81681442260742,
      "activations/layer15_attention_weight_min": -38.806785583496094,
      "activations/layer16_attention_weight_max": 38.778507232666016,
      "activations/layer16_attention_weight_min": -28.255931854248047,
      "activations/layer17_attention_weight_max": 58.36793518066406,
      "activations/layer17_attention_weight_min": -37.325828552246094,
      "activations/layer18_attention_weight_max": 63.93897247314453,
      "activations/layer18_attention_weight_min": -32.9128303527832,
      "activations/layer19_attention_weight_max": 22.028343200683594,
      "activations/layer19_attention_weight_min": -14.449174880981445,
      "activations/layer1_attention_weight_max": 16.761425018310547,
      "activations/layer1_attention_weight_min": -12.42280387878418,
      "activations/layer20_attention_weight_max": 19.538127899169922,
      "activations/layer20_attention_weight_min": -18.062788009643555,
      "activations/layer21_attention_weight_max": 41.066680908203125,
      "activations/layer21_attention_weight_min": -23.695276260375977,
      "activations/layer22_attention_weight_max": 34.8424072265625,
      "activations/layer22_attention_weight_min": -23.047279357910156,
      "activations/layer23_attention_weight_max": 24.268131256103516,
      "activations/layer23_attention_weight_min": -14.619105339050293,
      "activations/layer2_attention_weight_max": 25.617542266845703,
      "activations/layer2_attention_weight_min": -25.197790145874023,
      "activations/layer3_attention_weight_max": 62.10558319091797,
      "activations/layer3_attention_weight_min": -67.43769073486328,
      "activations/layer4_attention_weight_max": 88.61427307128906,
      "activations/layer4_attention_weight_min": -82.22914123535156,
      "activations/layer5_attention_weight_max": 123.96917724609375,
      "activations/layer5_attention_weight_min": -108.80596923828125,
      "activations/layer6_attention_weight_max": 58.010684967041016,
      "activations/layer6_attention_weight_min": -51.07111358642578,
      "activations/layer7_attention_weight_max": 63.24262237548828,
      "activations/layer7_attention_weight_min": -55.30965805053711,
      "activations/layer8_attention_weight_max": 62.6423225402832,
      "activations/layer8_attention_weight_min": -47.86698913574219,
      "activations/layer9_attention_weight_max": 86.61595916748047,
      "activations/layer9_attention_weight_min": -71.96293640136719,
      "epoch": 1.95,
      "learning_rate": 0.0001388431818181818,
      "loss": 2.9619,
      "step": 33500
    },
    {
      "activations/layer0_attention_weight_max": 15.615079879760742,
      "activations/layer0_attention_weight_min": -13.65386962890625,
      "activations/layer10_attention_weight_max": 58.33475112915039,
      "activations/layer10_attention_weight_min": -46.02593231201172,
      "activations/layer11_attention_weight_max": 46.30491638183594,
      "activations/layer11_attention_weight_min": -39.614051818847656,
      "activations/layer12_attention_weight_max": 31.010360717773438,
      "activations/layer12_attention_weight_min": -22.832855224609375,
      "activations/layer13_attention_weight_max": 74.11770629882812,
      "activations/layer13_attention_weight_min": -50.642311096191406,
      "activations/layer14_attention_weight_max": 69.51920318603516,
      "activations/layer14_attention_weight_min": -44.64971160888672,
      "activations/layer15_attention_weight_max": 49.53588104248047,
      "activations/layer15_attention_weight_min": -38.209877014160156,
      "activations/layer16_attention_weight_max": 40.40797424316406,
      "activations/layer16_attention_weight_min": -29.60209846496582,
      "activations/layer17_attention_weight_max": 65.8092269897461,
      "activations/layer17_attention_weight_min": -37.68735122680664,
      "activations/layer18_attention_weight_max": 65.15442657470703,
      "activations/layer18_attention_weight_min": -32.9290657043457,
      "activations/layer19_attention_weight_max": 21.6104736328125,
      "activations/layer19_attention_weight_min": -15.6969575881958,
      "activations/layer1_attention_weight_max": 16.154022216796875,
      "activations/layer1_attention_weight_min": -13.733274459838867,
      "activations/layer20_attention_weight_max": 20.61338996887207,
      "activations/layer20_attention_weight_min": -16.533254623413086,
      "activations/layer21_attention_weight_max": 41.346336364746094,
      "activations/layer21_attention_weight_min": -22.143224716186523,
      "activations/layer22_attention_weight_max": 33.420066833496094,
      "activations/layer22_attention_weight_min": -20.731945037841797,
      "activations/layer23_attention_weight_max": 24.080745697021484,
      "activations/layer23_attention_weight_min": -14.133539199829102,
      "activations/layer2_attention_weight_max": 26.2791748046875,
      "activations/layer2_attention_weight_min": -25.96767807006836,
      "activations/layer3_attention_weight_max": 60.32067108154297,
      "activations/layer3_attention_weight_min": -66.09578704833984,
      "activations/layer4_attention_weight_max": 92.42497253417969,
      "activations/layer4_attention_weight_min": -86.44383239746094,
      "activations/layer5_attention_weight_max": 128.9163818359375,
      "activations/layer5_attention_weight_min": -114.44975280761719,
      "activations/layer6_attention_weight_max": 61.33325958251953,
      "activations/layer6_attention_weight_min": -54.90398406982422,
      "activations/layer7_attention_weight_max": 64.31390380859375,
      "activations/layer7_attention_weight_min": -57.716148376464844,
      "activations/layer8_attention_weight_max": 52.26011276245117,
      "activations/layer8_attention_weight_min": -43.4279899597168,
      "activations/layer9_attention_weight_max": 70.02889251708984,
      "activations/layer9_attention_weight_min": -61.53972244262695,
      "epoch": 1.95,
      "learning_rate": 0.00013882424242424242,
      "loss": 2.9707,
      "step": 33550
    },
    {
      "activations/layer0_attention_weight_max": 14.332088470458984,
      "activations/layer0_attention_weight_min": -13.585827827453613,
      "activations/layer10_attention_weight_max": 59.908042907714844,
      "activations/layer10_attention_weight_min": -49.830162048339844,
      "activations/layer11_attention_weight_max": 44.19935607910156,
      "activations/layer11_attention_weight_min": -32.687767028808594,
      "activations/layer12_attention_weight_max": 29.049222946166992,
      "activations/layer12_attention_weight_min": -24.190505981445312,
      "activations/layer13_attention_weight_max": 66.50634002685547,
      "activations/layer13_attention_weight_min": -46.36686325073242,
      "activations/layer14_attention_weight_max": 55.66498947143555,
      "activations/layer14_attention_weight_min": -37.11140060424805,
      "activations/layer15_attention_weight_max": 58.53656005859375,
      "activations/layer15_attention_weight_min": -37.727230072021484,
      "activations/layer16_attention_weight_max": 48.02302169799805,
      "activations/layer16_attention_weight_min": -28.348901748657227,
      "activations/layer17_attention_weight_max": 59.99825668334961,
      "activations/layer17_attention_weight_min": -38.64350128173828,
      "activations/layer18_attention_weight_max": 54.723716735839844,
      "activations/layer18_attention_weight_min": -32.41718292236328,
      "activations/layer19_attention_weight_max": 22.26231575012207,
      "activations/layer19_attention_weight_min": -15.3829345703125,
      "activations/layer1_attention_weight_max": 16.463138580322266,
      "activations/layer1_attention_weight_min": -16.218740463256836,
      "activations/layer20_attention_weight_max": 21.61638641357422,
      "activations/layer20_attention_weight_min": -15.419777870178223,
      "activations/layer21_attention_weight_max": 42.799415588378906,
      "activations/layer21_attention_weight_min": -22.980152130126953,
      "activations/layer22_attention_weight_max": 37.04056167602539,
      "activations/layer22_attention_weight_min": -23.26902961730957,
      "activations/layer23_attention_weight_max": 23.832901000976562,
      "activations/layer23_attention_weight_min": -15.334001541137695,
      "activations/layer2_attention_weight_max": 29.94931411743164,
      "activations/layer2_attention_weight_min": -29.335407257080078,
      "activations/layer3_attention_weight_max": 64.80164337158203,
      "activations/layer3_attention_weight_min": -70.20333099365234,
      "activations/layer4_attention_weight_max": 95.2906265258789,
      "activations/layer4_attention_weight_min": -88.16680908203125,
      "activations/layer5_attention_weight_max": 127.48632049560547,
      "activations/layer5_attention_weight_min": -109.25177001953125,
      "activations/layer6_attention_weight_max": 61.37278747558594,
      "activations/layer6_attention_weight_min": -52.290279388427734,
      "activations/layer7_attention_weight_max": 57.485504150390625,
      "activations/layer7_attention_weight_min": -51.66945266723633,
      "activations/layer8_attention_weight_max": 44.69807052612305,
      "activations/layer8_attention_weight_min": -41.594181060791016,
      "activations/layer9_attention_weight_max": 63.869476318359375,
      "activations/layer9_attention_weight_min": -56.876953125,
      "epoch": 1.95,
      "learning_rate": 0.000138805303030303,
      "loss": 2.9655,
      "step": 33600
    },
    {
      "activations/layer0_attention_weight_max": 13.401845932006836,
      "activations/layer0_attention_weight_min": -14.071100234985352,
      "activations/layer10_attention_weight_max": 58.883872985839844,
      "activations/layer10_attention_weight_min": -48.55817794799805,
      "activations/layer11_attention_weight_max": 43.297462463378906,
      "activations/layer11_attention_weight_min": -44.35828399658203,
      "activations/layer12_attention_weight_max": 27.114770889282227,
      "activations/layer12_attention_weight_min": -25.399219512939453,
      "activations/layer13_attention_weight_max": 75.17489624023438,
      "activations/layer13_attention_weight_min": -45.76847457885742,
      "activations/layer14_attention_weight_max": 54.82865524291992,
      "activations/layer14_attention_weight_min": -36.2988395690918,
      "activations/layer15_attention_weight_max": 63.723411560058594,
      "activations/layer15_attention_weight_min": -39.99919891357422,
      "activations/layer16_attention_weight_max": 50.07072067260742,
      "activations/layer16_attention_weight_min": -28.92506980895996,
      "activations/layer17_attention_weight_max": 67.0734634399414,
      "activations/layer17_attention_weight_min": -35.52151107788086,
      "activations/layer18_attention_weight_max": 65.3509750366211,
      "activations/layer18_attention_weight_min": -33.669578552246094,
      "activations/layer19_attention_weight_max": 24.734018325805664,
      "activations/layer19_attention_weight_min": -15.712531089782715,
      "activations/layer1_attention_weight_max": 15.905901908874512,
      "activations/layer1_attention_weight_min": -13.54840087890625,
      "activations/layer20_attention_weight_max": 21.54023551940918,
      "activations/layer20_attention_weight_min": -17.945356369018555,
      "activations/layer21_attention_weight_max": 39.166481018066406,
      "activations/layer21_attention_weight_min": -25.724468231201172,
      "activations/layer22_attention_weight_max": 39.43390655517578,
      "activations/layer22_attention_weight_min": -23.373260498046875,
      "activations/layer23_attention_weight_max": 26.55214500427246,
      "activations/layer23_attention_weight_min": -16.94235610961914,
      "activations/layer2_attention_weight_max": 25.749698638916016,
      "activations/layer2_attention_weight_min": -24.278989791870117,
      "activations/layer3_attention_weight_max": 60.488887786865234,
      "activations/layer3_attention_weight_min": -63.87133026123047,
      "activations/layer4_attention_weight_max": 90.63323211669922,
      "activations/layer4_attention_weight_min": -83.20519256591797,
      "activations/layer5_attention_weight_max": 122.17485046386719,
      "activations/layer5_attention_weight_min": -107.41710662841797,
      "activations/layer6_attention_weight_max": 58.55851364135742,
      "activations/layer6_attention_weight_min": -49.427635192871094,
      "activations/layer7_attention_weight_max": 56.15169143676758,
      "activations/layer7_attention_weight_min": -52.63969421386719,
      "activations/layer8_attention_weight_max": 43.42268753051758,
      "activations/layer8_attention_weight_min": -42.34614181518555,
      "activations/layer9_attention_weight_max": 64.14482116699219,
      "activations/layer9_attention_weight_min": -67.4910659790039,
      "epoch": 1.96,
      "learning_rate": 0.00013878636363636362,
      "loss": 2.9717,
      "step": 33650
    },
    {
      "activations/layer0_attention_weight_max": 14.620182037353516,
      "activations/layer0_attention_weight_min": -13.912070274353027,
      "activations/layer10_attention_weight_max": 64.1456069946289,
      "activations/layer10_attention_weight_min": -44.90726852416992,
      "activations/layer11_attention_weight_max": 43.86663818359375,
      "activations/layer11_attention_weight_min": -36.55007553100586,
      "activations/layer12_attention_weight_max": 32.96939468383789,
      "activations/layer12_attention_weight_min": -28.197696685791016,
      "activations/layer13_attention_weight_max": 99.05597686767578,
      "activations/layer13_attention_weight_min": -48.380252838134766,
      "activations/layer14_attention_weight_max": 119.36283874511719,
      "activations/layer14_attention_weight_min": -48.712242126464844,
      "activations/layer15_attention_weight_max": 78.29338073730469,
      "activations/layer15_attention_weight_min": -41.17893600463867,
      "activations/layer16_attention_weight_max": 52.71003723144531,
      "activations/layer16_attention_weight_min": -28.068370819091797,
      "activations/layer17_attention_weight_max": 63.34161376953125,
      "activations/layer17_attention_weight_min": -37.31071853637695,
      "activations/layer18_attention_weight_max": 57.96971893310547,
      "activations/layer18_attention_weight_min": -32.61859893798828,
      "activations/layer19_attention_weight_max": 19.05153465270996,
      "activations/layer19_attention_weight_min": -17.795652389526367,
      "activations/layer1_attention_weight_max": 16.630481719970703,
      "activations/layer1_attention_weight_min": -13.701221466064453,
      "activations/layer20_attention_weight_max": 20.772478103637695,
      "activations/layer20_attention_weight_min": -16.892879486083984,
      "activations/layer21_attention_weight_max": 39.36600875854492,
      "activations/layer21_attention_weight_min": -23.17061424255371,
      "activations/layer22_attention_weight_max": 34.431602478027344,
      "activations/layer22_attention_weight_min": -21.141551971435547,
      "activations/layer23_attention_weight_max": 24.83643913269043,
      "activations/layer23_attention_weight_min": -14.852802276611328,
      "activations/layer2_attention_weight_max": 27.388233184814453,
      "activations/layer2_attention_weight_min": -27.428752899169922,
      "activations/layer3_attention_weight_max": 65.6429214477539,
      "activations/layer3_attention_weight_min": -64.97232055664062,
      "activations/layer4_attention_weight_max": 95.80690002441406,
      "activations/layer4_attention_weight_min": -82.38700866699219,
      "activations/layer5_attention_weight_max": 134.79078674316406,
      "activations/layer5_attention_weight_min": -105.43059539794922,
      "activations/layer6_attention_weight_max": 64.97293090820312,
      "activations/layer6_attention_weight_min": -51.78131866455078,
      "activations/layer7_attention_weight_max": 61.532073974609375,
      "activations/layer7_attention_weight_min": -56.261966705322266,
      "activations/layer8_attention_weight_max": 51.56076431274414,
      "activations/layer8_attention_weight_min": -39.81039047241211,
      "activations/layer9_attention_weight_max": 77.23107147216797,
      "activations/layer9_attention_weight_min": -58.91553497314453,
      "epoch": 1.96,
      "learning_rate": 0.00013876742424242424,
      "loss": 2.9604,
      "step": 33700
    },
    {
      "activations/layer0_attention_weight_max": 13.24704360961914,
      "activations/layer0_attention_weight_min": -14.02407455444336,
      "activations/layer10_attention_weight_max": 54.79667282104492,
      "activations/layer10_attention_weight_min": -48.0441780090332,
      "activations/layer11_attention_weight_max": 49.544700622558594,
      "activations/layer11_attention_weight_min": -39.01215744018555,
      "activations/layer12_attention_weight_max": 27.12360191345215,
      "activations/layer12_attention_weight_min": -24.28874397277832,
      "activations/layer13_attention_weight_max": 54.848228454589844,
      "activations/layer13_attention_weight_min": -43.67391586303711,
      "activations/layer14_attention_weight_max": 53.63935852050781,
      "activations/layer14_attention_weight_min": -39.615257263183594,
      "activations/layer15_attention_weight_max": 52.734989166259766,
      "activations/layer15_attention_weight_min": -37.831146240234375,
      "activations/layer16_attention_weight_max": 38.96061325073242,
      "activations/layer16_attention_weight_min": -30.056072235107422,
      "activations/layer17_attention_weight_max": 60.45724105834961,
      "activations/layer17_attention_weight_min": -36.56661605834961,
      "activations/layer18_attention_weight_max": 53.49789810180664,
      "activations/layer18_attention_weight_min": -33.256778717041016,
      "activations/layer19_attention_weight_max": 18.93486785888672,
      "activations/layer19_attention_weight_min": -14.811110496520996,
      "activations/layer1_attention_weight_max": 16.054256439208984,
      "activations/layer1_attention_weight_min": -11.857953071594238,
      "activations/layer20_attention_weight_max": 17.992876052856445,
      "activations/layer20_attention_weight_min": -18.199750900268555,
      "activations/layer21_attention_weight_max": 38.10535430908203,
      "activations/layer21_attention_weight_min": -24.31028175354004,
      "activations/layer22_attention_weight_max": 35.227210998535156,
      "activations/layer22_attention_weight_min": -20.663698196411133,
      "activations/layer23_attention_weight_max": 22.211688995361328,
      "activations/layer23_attention_weight_min": -15.94477653503418,
      "activations/layer2_attention_weight_max": 28.37252426147461,
      "activations/layer2_attention_weight_min": -26.67877769470215,
      "activations/layer3_attention_weight_max": 60.913700103759766,
      "activations/layer3_attention_weight_min": -63.91828918457031,
      "activations/layer4_attention_weight_max": 91.63604736328125,
      "activations/layer4_attention_weight_min": -82.5329818725586,
      "activations/layer5_attention_weight_max": 132.2552947998047,
      "activations/layer5_attention_weight_min": -104.38365173339844,
      "activations/layer6_attention_weight_max": 61.13303756713867,
      "activations/layer6_attention_weight_min": -52.320674896240234,
      "activations/layer7_attention_weight_max": 61.14126968383789,
      "activations/layer7_attention_weight_min": -53.47052764892578,
      "activations/layer8_attention_weight_max": 46.486202239990234,
      "activations/layer8_attention_weight_min": -41.16461181640625,
      "activations/layer9_attention_weight_max": 61.330074310302734,
      "activations/layer9_attention_weight_min": -54.98987579345703,
      "epoch": 1.96,
      "learning_rate": 0.00013874848484848485,
      "loss": 2.9802,
      "step": 33750
    },
    {
      "activations/layer0_attention_weight_max": 15.012490272521973,
      "activations/layer0_attention_weight_min": -13.600046157836914,
      "activations/layer10_attention_weight_max": 69.88697814941406,
      "activations/layer10_attention_weight_min": -49.109737396240234,
      "activations/layer11_attention_weight_max": 50.834716796875,
      "activations/layer11_attention_weight_min": -35.226863861083984,
      "activations/layer12_attention_weight_max": 34.86455154418945,
      "activations/layer12_attention_weight_min": -27.932775497436523,
      "activations/layer13_attention_weight_max": 63.03556442260742,
      "activations/layer13_attention_weight_min": -43.32353973388672,
      "activations/layer14_attention_weight_max": 60.13018798828125,
      "activations/layer14_attention_weight_min": -42.90542221069336,
      "activations/layer15_attention_weight_max": 61.12416458129883,
      "activations/layer15_attention_weight_min": -40.73260498046875,
      "activations/layer16_attention_weight_max": 50.1875,
      "activations/layer16_attention_weight_min": -30.815196990966797,
      "activations/layer17_attention_weight_max": 65.18749237060547,
      "activations/layer17_attention_weight_min": -39.02328872680664,
      "activations/layer18_attention_weight_max": 63.54711151123047,
      "activations/layer18_attention_weight_min": -36.840152740478516,
      "activations/layer19_attention_weight_max": 20.910017013549805,
      "activations/layer19_attention_weight_min": -15.51679515838623,
      "activations/layer1_attention_weight_max": 16.3009033203125,
      "activations/layer1_attention_weight_min": -13.66907024383545,
      "activations/layer20_attention_weight_max": 19.963375091552734,
      "activations/layer20_attention_weight_min": -15.384090423583984,
      "activations/layer21_attention_weight_max": 42.28851318359375,
      "activations/layer21_attention_weight_min": -24.875913619995117,
      "activations/layer22_attention_weight_max": 35.615074157714844,
      "activations/layer22_attention_weight_min": -23.28533935546875,
      "activations/layer23_attention_weight_max": 23.80629539489746,
      "activations/layer23_attention_weight_min": -16.036334991455078,
      "activations/layer2_attention_weight_max": 26.749189376831055,
      "activations/layer2_attention_weight_min": -25.688289642333984,
      "activations/layer3_attention_weight_max": 63.314693450927734,
      "activations/layer3_attention_weight_min": -64.5249252319336,
      "activations/layer4_attention_weight_max": 93.67871856689453,
      "activations/layer4_attention_weight_min": -81.40155792236328,
      "activations/layer5_attention_weight_max": 135.984619140625,
      "activations/layer5_attention_weight_min": -108.21467590332031,
      "activations/layer6_attention_weight_max": 66.04717254638672,
      "activations/layer6_attention_weight_min": -53.50012969970703,
      "activations/layer7_attention_weight_max": 61.096004486083984,
      "activations/layer7_attention_weight_min": -52.562156677246094,
      "activations/layer8_attention_weight_max": 50.793357849121094,
      "activations/layer8_attention_weight_min": -40.18003463745117,
      "activations/layer9_attention_weight_max": 63.47134017944336,
      "activations/layer9_attention_weight_min": -59.574459075927734,
      "epoch": 1.96,
      "learning_rate": 0.00013872992424242425,
      "loss": 2.9788,
      "step": 33800
    },
    {
      "activations/layer0_attention_weight_max": 13.574353218078613,
      "activations/layer0_attention_weight_min": -14.227428436279297,
      "activations/layer10_attention_weight_max": 62.550010681152344,
      "activations/layer10_attention_weight_min": -49.7774772644043,
      "activations/layer11_attention_weight_max": 40.68781661987305,
      "activations/layer11_attention_weight_min": -34.37843704223633,
      "activations/layer12_attention_weight_max": 37.25115966796875,
      "activations/layer12_attention_weight_min": -26.49416732788086,
      "activations/layer13_attention_weight_max": 71.49403381347656,
      "activations/layer13_attention_weight_min": -48.61451721191406,
      "activations/layer14_attention_weight_max": 49.104549407958984,
      "activations/layer14_attention_weight_min": -37.903724670410156,
      "activations/layer15_attention_weight_max": 59.38471603393555,
      "activations/layer15_attention_weight_min": -36.5211181640625,
      "activations/layer16_attention_weight_max": 40.12517547607422,
      "activations/layer16_attention_weight_min": -28.004087448120117,
      "activations/layer17_attention_weight_max": 66.01783752441406,
      "activations/layer17_attention_weight_min": -36.659263610839844,
      "activations/layer18_attention_weight_max": 58.684879302978516,
      "activations/layer18_attention_weight_min": -30.495159149169922,
      "activations/layer19_attention_weight_max": 24.917381286621094,
      "activations/layer19_attention_weight_min": -16.065576553344727,
      "activations/layer1_attention_weight_max": 15.994195938110352,
      "activations/layer1_attention_weight_min": -13.542977333068848,
      "activations/layer20_attention_weight_max": 22.098644256591797,
      "activations/layer20_attention_weight_min": -18.48862075805664,
      "activations/layer21_attention_weight_max": 39.20136260986328,
      "activations/layer21_attention_weight_min": -27.72764015197754,
      "activations/layer22_attention_weight_max": 37.61366653442383,
      "activations/layer22_attention_weight_min": -22.13945960998535,
      "activations/layer23_attention_weight_max": 22.215221405029297,
      "activations/layer23_attention_weight_min": -16.255386352539062,
      "activations/layer2_attention_weight_max": 25.657276153564453,
      "activations/layer2_attention_weight_min": -25.585233688354492,
      "activations/layer3_attention_weight_max": 59.506141662597656,
      "activations/layer3_attention_weight_min": -63.59129333496094,
      "activations/layer4_attention_weight_max": 91.24089813232422,
      "activations/layer4_attention_weight_min": -84.07388305664062,
      "activations/layer5_attention_weight_max": 141.84341430664062,
      "activations/layer5_attention_weight_min": -113.89112854003906,
      "activations/layer6_attention_weight_max": 65.71688842773438,
      "activations/layer6_attention_weight_min": -54.360191345214844,
      "activations/layer7_attention_weight_max": 59.4585075378418,
      "activations/layer7_attention_weight_min": -54.73268127441406,
      "activations/layer8_attention_weight_max": 47.60797119140625,
      "activations/layer8_attention_weight_min": -41.97410583496094,
      "activations/layer9_attention_weight_max": 66.69434356689453,
      "activations/layer9_attention_weight_min": -58.84434127807617,
      "epoch": 1.97,
      "learning_rate": 0.00013871098484848483,
      "loss": 2.9769,
      "step": 33850
    },
    {
      "activations/layer0_attention_weight_max": 15.343498229980469,
      "activations/layer0_attention_weight_min": -14.284967422485352,
      "activations/layer10_attention_weight_max": 57.891475677490234,
      "activations/layer10_attention_weight_min": -50.25928497314453,
      "activations/layer11_attention_weight_max": 54.72361373901367,
      "activations/layer11_attention_weight_min": -36.3636360168457,
      "activations/layer12_attention_weight_max": 31.824262619018555,
      "activations/layer12_attention_weight_min": -22.624662399291992,
      "activations/layer13_attention_weight_max": 54.25886535644531,
      "activations/layer13_attention_weight_min": -39.293880462646484,
      "activations/layer14_attention_weight_max": 42.84782409667969,
      "activations/layer14_attention_weight_min": -32.90194320678711,
      "activations/layer15_attention_weight_max": 52.671661376953125,
      "activations/layer15_attention_weight_min": -36.16278839111328,
      "activations/layer16_attention_weight_max": 40.08340835571289,
      "activations/layer16_attention_weight_min": -29.8979434967041,
      "activations/layer17_attention_weight_max": 56.37579345703125,
      "activations/layer17_attention_weight_min": -37.44765090942383,
      "activations/layer18_attention_weight_max": 53.515228271484375,
      "activations/layer18_attention_weight_min": -36.651885986328125,
      "activations/layer19_attention_weight_max": 21.230920791625977,
      "activations/layer19_attention_weight_min": -13.441359519958496,
      "activations/layer1_attention_weight_max": 17.174320220947266,
      "activations/layer1_attention_weight_min": -12.873859405517578,
      "activations/layer20_attention_weight_max": 18.56638526916504,
      "activations/layer20_attention_weight_min": -16.50777816772461,
      "activations/layer21_attention_weight_max": 37.06087112426758,
      "activations/layer21_attention_weight_min": -23.089027404785156,
      "activations/layer22_attention_weight_max": 32.84238052368164,
      "activations/layer22_attention_weight_min": -21.01241683959961,
      "activations/layer23_attention_weight_max": 21.616214752197266,
      "activations/layer23_attention_weight_min": -13.502120971679688,
      "activations/layer2_attention_weight_max": 26.287254333496094,
      "activations/layer2_attention_weight_min": -26.918691635131836,
      "activations/layer3_attention_weight_max": 61.99945068359375,
      "activations/layer3_attention_weight_min": -66.4832534790039,
      "activations/layer4_attention_weight_max": 90.58289337158203,
      "activations/layer4_attention_weight_min": -83.11750030517578,
      "activations/layer5_attention_weight_max": 128.26519775390625,
      "activations/layer5_attention_weight_min": -110.50152587890625,
      "activations/layer6_attention_weight_max": 59.06092834472656,
      "activations/layer6_attention_weight_min": -53.278602600097656,
      "activations/layer7_attention_weight_max": 60.020694732666016,
      "activations/layer7_attention_weight_min": -56.60602951049805,
      "activations/layer8_attention_weight_max": 48.90336990356445,
      "activations/layer8_attention_weight_min": -42.39906692504883,
      "activations/layer9_attention_weight_max": 69.53459930419922,
      "activations/layer9_attention_weight_min": -61.2321662902832,
      "epoch": 1.97,
      "learning_rate": 0.00013869204545454545,
      "loss": 2.989,
      "step": 33900
    },
    {
      "activations/layer0_attention_weight_max": 14.628893852233887,
      "activations/layer0_attention_weight_min": -14.342236518859863,
      "activations/layer10_attention_weight_max": 59.46352005004883,
      "activations/layer10_attention_weight_min": -49.53086853027344,
      "activations/layer11_attention_weight_max": 44.01569747924805,
      "activations/layer11_attention_weight_min": -37.604209899902344,
      "activations/layer12_attention_weight_max": 31.482507705688477,
      "activations/layer12_attention_weight_min": -23.28311538696289,
      "activations/layer13_attention_weight_max": 69.89836120605469,
      "activations/layer13_attention_weight_min": -48.52655792236328,
      "activations/layer14_attention_weight_max": 73.8265609741211,
      "activations/layer14_attention_weight_min": -49.1882209777832,
      "activations/layer15_attention_weight_max": 56.5084342956543,
      "activations/layer15_attention_weight_min": -39.14189910888672,
      "activations/layer16_attention_weight_max": 43.41276550292969,
      "activations/layer16_attention_weight_min": -29.054458618164062,
      "activations/layer17_attention_weight_max": 68.75460815429688,
      "activations/layer17_attention_weight_min": -41.86949157714844,
      "activations/layer18_attention_weight_max": 64.64942932128906,
      "activations/layer18_attention_weight_min": -33.743927001953125,
      "activations/layer19_attention_weight_max": 24.738239288330078,
      "activations/layer19_attention_weight_min": -15.588047981262207,
      "activations/layer1_attention_weight_max": 15.972013473510742,
      "activations/layer1_attention_weight_min": -13.020248413085938,
      "activations/layer20_attention_weight_max": 23.21552848815918,
      "activations/layer20_attention_weight_min": -16.334009170532227,
      "activations/layer21_attention_weight_max": 46.02037811279297,
      "activations/layer21_attention_weight_min": -26.414981842041016,
      "activations/layer22_attention_weight_max": 40.14577865600586,
      "activations/layer22_attention_weight_min": -21.98656463623047,
      "activations/layer23_attention_weight_max": 25.38349151611328,
      "activations/layer23_attention_weight_min": -15.5700101852417,
      "activations/layer2_attention_weight_max": 26.103103637695312,
      "activations/layer2_attention_weight_min": -24.8602352142334,
      "activations/layer3_attention_weight_max": 63.60020446777344,
      "activations/layer3_attention_weight_min": -68.79277801513672,
      "activations/layer4_attention_weight_max": 97.27494812011719,
      "activations/layer4_attention_weight_min": -90.2505874633789,
      "activations/layer5_attention_weight_max": 134.4466552734375,
      "activations/layer5_attention_weight_min": -111.14250946044922,
      "activations/layer6_attention_weight_max": 63.198184967041016,
      "activations/layer6_attention_weight_min": -52.1922492980957,
      "activations/layer7_attention_weight_max": 61.14236831665039,
      "activations/layer7_attention_weight_min": -56.02985763549805,
      "activations/layer8_attention_weight_max": 53.64226150512695,
      "activations/layer8_attention_weight_min": -45.609066009521484,
      "activations/layer9_attention_weight_max": 69.1299819946289,
      "activations/layer9_attention_weight_min": -62.02500534057617,
      "epoch": 1.97,
      "learning_rate": 0.00013867310606060606,
      "loss": 2.9749,
      "step": 33950
    },
    {
      "activations/layer0_attention_weight_max": 14.596525192260742,
      "activations/layer0_attention_weight_min": -13.61841106414795,
      "activations/layer10_attention_weight_max": 68.93766021728516,
      "activations/layer10_attention_weight_min": -45.987876892089844,
      "activations/layer11_attention_weight_max": 47.95408248901367,
      "activations/layer11_attention_weight_min": -37.62017822265625,
      "activations/layer12_attention_weight_max": 37.72915267944336,
      "activations/layer12_attention_weight_min": -25.326208114624023,
      "activations/layer13_attention_weight_max": 72.77279663085938,
      "activations/layer13_attention_weight_min": -41.28322219848633,
      "activations/layer14_attention_weight_max": 81.14386749267578,
      "activations/layer14_attention_weight_min": -40.13316345214844,
      "activations/layer15_attention_weight_max": 52.137630462646484,
      "activations/layer15_attention_weight_min": -35.65127944946289,
      "activations/layer16_attention_weight_max": 44.62794876098633,
      "activations/layer16_attention_weight_min": -28.74033546447754,
      "activations/layer17_attention_weight_max": 68.94932556152344,
      "activations/layer17_attention_weight_min": -40.37733840942383,
      "activations/layer18_attention_weight_max": 66.13673400878906,
      "activations/layer18_attention_weight_min": -34.74515914916992,
      "activations/layer19_attention_weight_max": 24.004194259643555,
      "activations/layer19_attention_weight_min": -14.582178115844727,
      "activations/layer1_attention_weight_max": 15.97555923461914,
      "activations/layer1_attention_weight_min": -15.389464378356934,
      "activations/layer20_attention_weight_max": 25.29718017578125,
      "activations/layer20_attention_weight_min": -15.602574348449707,
      "activations/layer21_attention_weight_max": 45.4411506652832,
      "activations/layer21_attention_weight_min": -25.867935180664062,
      "activations/layer22_attention_weight_max": 39.556819915771484,
      "activations/layer22_attention_weight_min": -22.09760093688965,
      "activations/layer23_attention_weight_max": 25.187633514404297,
      "activations/layer23_attention_weight_min": -15.13729476928711,
      "activations/layer2_attention_weight_max": 26.743797302246094,
      "activations/layer2_attention_weight_min": -25.79592514038086,
      "activations/layer3_attention_weight_max": 64.13155364990234,
      "activations/layer3_attention_weight_min": -65.38226318359375,
      "activations/layer4_attention_weight_max": 94.99018096923828,
      "activations/layer4_attention_weight_min": -82.89706420898438,
      "activations/layer5_attention_weight_max": 129.1207275390625,
      "activations/layer5_attention_weight_min": -102.95701599121094,
      "activations/layer6_attention_weight_max": 61.246978759765625,
      "activations/layer6_attention_weight_min": -54.249752044677734,
      "activations/layer7_attention_weight_max": 76.17748260498047,
      "activations/layer7_attention_weight_min": -57.479530334472656,
      "activations/layer8_attention_weight_max": 57.60407257080078,
      "activations/layer8_attention_weight_min": -41.23006057739258,
      "activations/layer9_attention_weight_max": 79.50338745117188,
      "activations/layer9_attention_weight_min": -57.06584548950195,
      "epoch": 1.98,
      "learning_rate": 0.00013865416666666665,
      "loss": 2.9691,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_loss": 2.91796875,
      "eval_runtime": 8.4752,
      "eval_samples_per_second": 506.655,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_openwebtext_loss": 2.91796875,
      "eval_openwebtext_ppl": 18.503663694167408,
      "eval_openwebtext_runtime": 8.4752,
      "eval_openwebtext_samples_per_second": 506.655,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_wikitext_loss": 3.1796875,
      "eval_wikitext_ppl": 24.039240115615065,
      "eval_wikitext_runtime": 1.958,
      "eval_wikitext_samples_per_second": 232.888,
      "step": 34000
    },
    {
      "epoch": 1.98,
      "eval_lambada_loss": 3.09375,
      "eval_lambada_ppl": 22.059646737841184,
      "eval_lambada_runtime": 9.5546,
      "eval_lambada_samples_per_second": 509.597,
      "step": 34000
    },
    {
      "activations/layer0_attention_weight_max": 15.184072494506836,
      "activations/layer0_attention_weight_min": -13.780646324157715,
      "activations/layer10_attention_weight_max": 62.58881378173828,
      "activations/layer10_attention_weight_min": -52.81760787963867,
      "activations/layer11_attention_weight_max": 45.74007797241211,
      "activations/layer11_attention_weight_min": -34.47988510131836,
      "activations/layer12_attention_weight_max": 31.222257614135742,
      "activations/layer12_attention_weight_min": -24.218826293945312,
      "activations/layer13_attention_weight_max": 57.34545135498047,
      "activations/layer13_attention_weight_min": -40.0977668762207,
      "activations/layer14_attention_weight_max": 50.03224182128906,
      "activations/layer14_attention_weight_min": -33.52313995361328,
      "activations/layer15_attention_weight_max": 61.18833923339844,
      "activations/layer15_attention_weight_min": -39.34547424316406,
      "activations/layer16_attention_weight_max": 45.83632278442383,
      "activations/layer16_attention_weight_min": -31.335790634155273,
      "activations/layer17_attention_weight_max": 64.2719497680664,
      "activations/layer17_attention_weight_min": -40.80397415161133,
      "activations/layer18_attention_weight_max": 60.72016906738281,
      "activations/layer18_attention_weight_min": -34.5981559753418,
      "activations/layer19_attention_weight_max": 27.406784057617188,
      "activations/layer19_attention_weight_min": -16.880146026611328,
      "activations/layer1_attention_weight_max": 15.721999168395996,
      "activations/layer1_attention_weight_min": -13.201549530029297,
      "activations/layer20_attention_weight_max": 21.715288162231445,
      "activations/layer20_attention_weight_min": -18.13518524169922,
      "activations/layer21_attention_weight_max": 46.412471771240234,
      "activations/layer21_attention_weight_min": -24.53325080871582,
      "activations/layer22_attention_weight_max": 34.4244270324707,
      "activations/layer22_attention_weight_min": -23.58949089050293,
      "activations/layer23_attention_weight_max": 22.94367027282715,
      "activations/layer23_attention_weight_min": -15.608663558959961,
      "activations/layer2_attention_weight_max": 26.461400985717773,
      "activations/layer2_attention_weight_min": -25.615097045898438,
      "activations/layer3_attention_weight_max": 61.88786315917969,
      "activations/layer3_attention_weight_min": -65.60485076904297,
      "activations/layer4_attention_weight_max": 89.43941497802734,
      "activations/layer4_attention_weight_min": -83.68264770507812,
      "activations/layer5_attention_weight_max": 126.63548278808594,
      "activations/layer5_attention_weight_min": -105.18296813964844,
      "activations/layer6_attention_weight_max": 61.05002212524414,
      "activations/layer6_attention_weight_min": -52.62213897705078,
      "activations/layer7_attention_weight_max": 61.00600814819336,
      "activations/layer7_attention_weight_min": -52.868614196777344,
      "activations/layer8_attention_weight_max": 53.1688346862793,
      "activations/layer8_attention_weight_min": -41.28645324707031,
      "activations/layer9_attention_weight_max": 64.01203155517578,
      "activations/layer9_attention_weight_min": -56.52462387084961,
      "epoch": 1.98,
      "learning_rate": 0.00013863522727272727,
      "loss": 2.9716,
      "step": 34050
    },
    {
      "activations/layer0_attention_weight_max": 14.447416305541992,
      "activations/layer0_attention_weight_min": -14.060806274414062,
      "activations/layer10_attention_weight_max": 57.99247360229492,
      "activations/layer10_attention_weight_min": -47.42068862915039,
      "activations/layer11_attention_weight_max": 40.341270446777344,
      "activations/layer11_attention_weight_min": -34.17606735229492,
      "activations/layer12_attention_weight_max": 25.35881805419922,
      "activations/layer12_attention_weight_min": -22.650142669677734,
      "activations/layer13_attention_weight_max": 61.82935333251953,
      "activations/layer13_attention_weight_min": -43.854042053222656,
      "activations/layer14_attention_weight_max": 57.4810791015625,
      "activations/layer14_attention_weight_min": -37.34769821166992,
      "activations/layer15_attention_weight_max": 50.046356201171875,
      "activations/layer15_attention_weight_min": -43.237815856933594,
      "activations/layer16_attention_weight_max": 44.13533020019531,
      "activations/layer16_attention_weight_min": -35.4278450012207,
      "activations/layer17_attention_weight_max": 55.42556381225586,
      "activations/layer17_attention_weight_min": -39.412757873535156,
      "activations/layer18_attention_weight_max": 59.49901580810547,
      "activations/layer18_attention_weight_min": -35.0125732421875,
      "activations/layer19_attention_weight_max": 20.598979949951172,
      "activations/layer19_attention_weight_min": -15.357657432556152,
      "activations/layer1_attention_weight_max": 16.357454299926758,
      "activations/layer1_attention_weight_min": -12.405797004699707,
      "activations/layer20_attention_weight_max": 19.54149055480957,
      "activations/layer20_attention_weight_min": -19.239336013793945,
      "activations/layer21_attention_weight_max": 39.89837646484375,
      "activations/layer21_attention_weight_min": -23.072242736816406,
      "activations/layer22_attention_weight_max": 37.804325103759766,
      "activations/layer22_attention_weight_min": -22.12973403930664,
      "activations/layer23_attention_weight_max": 26.12991714477539,
      "activations/layer23_attention_weight_min": -14.935220718383789,
      "activations/layer2_attention_weight_max": 25.396482467651367,
      "activations/layer2_attention_weight_min": -25.29262351989746,
      "activations/layer3_attention_weight_max": 60.40414047241211,
      "activations/layer3_attention_weight_min": -65.20674896240234,
      "activations/layer4_attention_weight_max": 94.81002044677734,
      "activations/layer4_attention_weight_min": -82.23780822753906,
      "activations/layer5_attention_weight_max": 124.13973999023438,
      "activations/layer5_attention_weight_min": -106.51902770996094,
      "activations/layer6_attention_weight_max": 59.68708419799805,
      "activations/layer6_attention_weight_min": -52.98191452026367,
      "activations/layer7_attention_weight_max": 57.05279541015625,
      "activations/layer7_attention_weight_min": -50.26673126220703,
      "activations/layer8_attention_weight_max": 45.096126556396484,
      "activations/layer8_attention_weight_min": -41.170841217041016,
      "activations/layer9_attention_weight_max": 61.680355072021484,
      "activations/layer9_attention_weight_min": -53.689300537109375,
      "epoch": 1.98,
      "learning_rate": 0.00013861628787878788,
      "loss": 2.9597,
      "step": 34100
    },
    {
      "activations/layer0_attention_weight_max": 15.335986137390137,
      "activations/layer0_attention_weight_min": -14.259963035583496,
      "activations/layer10_attention_weight_max": 61.678585052490234,
      "activations/layer10_attention_weight_min": -47.87847900390625,
      "activations/layer11_attention_weight_max": 40.28099060058594,
      "activations/layer11_attention_weight_min": -32.78834533691406,
      "activations/layer12_attention_weight_max": 27.347644805908203,
      "activations/layer12_attention_weight_min": -28.533069610595703,
      "activations/layer13_attention_weight_max": 60.53327560424805,
      "activations/layer13_attention_weight_min": -44.863582611083984,
      "activations/layer14_attention_weight_max": 56.42580032348633,
      "activations/layer14_attention_weight_min": -39.945186614990234,
      "activations/layer15_attention_weight_max": 54.622493743896484,
      "activations/layer15_attention_weight_min": -37.652626037597656,
      "activations/layer16_attention_weight_max": 43.672855377197266,
      "activations/layer16_attention_weight_min": -27.649932861328125,
      "activations/layer17_attention_weight_max": 59.39250946044922,
      "activations/layer17_attention_weight_min": -34.524593353271484,
      "activations/layer18_attention_weight_max": 58.3537712097168,
      "activations/layer18_attention_weight_min": -33.127288818359375,
      "activations/layer19_attention_weight_max": 23.494335174560547,
      "activations/layer19_attention_weight_min": -16.051145553588867,
      "activations/layer1_attention_weight_max": 17.2362060546875,
      "activations/layer1_attention_weight_min": -12.387297630310059,
      "activations/layer20_attention_weight_max": 21.158414840698242,
      "activations/layer20_attention_weight_min": -16.888898849487305,
      "activations/layer21_attention_weight_max": 42.23070526123047,
      "activations/layer21_attention_weight_min": -25.209836959838867,
      "activations/layer22_attention_weight_max": 39.51139450073242,
      "activations/layer22_attention_weight_min": -22.71731185913086,
      "activations/layer23_attention_weight_max": 23.37389373779297,
      "activations/layer23_attention_weight_min": -15.088655471801758,
      "activations/layer2_attention_weight_max": 25.02912139892578,
      "activations/layer2_attention_weight_min": -25.45448875427246,
      "activations/layer3_attention_weight_max": 61.29707336425781,
      "activations/layer3_attention_weight_min": -61.88597106933594,
      "activations/layer4_attention_weight_max": 90.13430786132812,
      "activations/layer4_attention_weight_min": -82.25811004638672,
      "activations/layer5_attention_weight_max": 128.38436889648438,
      "activations/layer5_attention_weight_min": -105.77621459960938,
      "activations/layer6_attention_weight_max": 60.83115005493164,
      "activations/layer6_attention_weight_min": -53.24019241333008,
      "activations/layer7_attention_weight_max": 57.68925476074219,
      "activations/layer7_attention_weight_min": -51.57791519165039,
      "activations/layer8_attention_weight_max": 48.77335739135742,
      "activations/layer8_attention_weight_min": -40.66509246826172,
      "activations/layer9_attention_weight_max": 60.24171829223633,
      "activations/layer9_attention_weight_min": -56.29566192626953,
      "epoch": 1.98,
      "learning_rate": 0.00013859734848484847,
      "loss": 2.9791,
      "step": 34150
    },
    {
      "activations/layer0_attention_weight_max": 14.994840621948242,
      "activations/layer0_attention_weight_min": -13.907922744750977,
      "activations/layer10_attention_weight_max": 56.88661575317383,
      "activations/layer10_attention_weight_min": -48.34241485595703,
      "activations/layer11_attention_weight_max": 39.80330276489258,
      "activations/layer11_attention_weight_min": -35.64342498779297,
      "activations/layer12_attention_weight_max": 25.086605072021484,
      "activations/layer12_attention_weight_min": -27.503177642822266,
      "activations/layer13_attention_weight_max": 55.14698791503906,
      "activations/layer13_attention_weight_min": -43.77733612060547,
      "activations/layer14_attention_weight_max": 55.36439895629883,
      "activations/layer14_attention_weight_min": -41.9284553527832,
      "activations/layer15_attention_weight_max": 48.782325744628906,
      "activations/layer15_attention_weight_min": -41.139381408691406,
      "activations/layer16_attention_weight_max": 43.77348327636719,
      "activations/layer16_attention_weight_min": -29.15479850769043,
      "activations/layer17_attention_weight_max": 64.20365142822266,
      "activations/layer17_attention_weight_min": -39.142364501953125,
      "activations/layer18_attention_weight_max": 63.72031784057617,
      "activations/layer18_attention_weight_min": -34.453975677490234,
      "activations/layer19_attention_weight_max": 22.17144203186035,
      "activations/layer19_attention_weight_min": -18.205638885498047,
      "activations/layer1_attention_weight_max": 16.437158584594727,
      "activations/layer1_attention_weight_min": -13.107892990112305,
      "activations/layer20_attention_weight_max": 27.35131072998047,
      "activations/layer20_attention_weight_min": -16.636125564575195,
      "activations/layer21_attention_weight_max": 42.53834915161133,
      "activations/layer21_attention_weight_min": -23.79861831665039,
      "activations/layer22_attention_weight_max": 42.43461227416992,
      "activations/layer22_attention_weight_min": -22.309751510620117,
      "activations/layer23_attention_weight_max": 26.34413719177246,
      "activations/layer23_attention_weight_min": -14.135339736938477,
      "activations/layer2_attention_weight_max": 27.60072135925293,
      "activations/layer2_attention_weight_min": -26.876150131225586,
      "activations/layer3_attention_weight_max": 60.901851654052734,
      "activations/layer3_attention_weight_min": -63.81821823120117,
      "activations/layer4_attention_weight_max": 89.18640899658203,
      "activations/layer4_attention_weight_min": -81.47151947021484,
      "activations/layer5_attention_weight_max": 122.6246566772461,
      "activations/layer5_attention_weight_min": -111.92964935302734,
      "activations/layer6_attention_weight_max": 60.24311447143555,
      "activations/layer6_attention_weight_min": -52.421146392822266,
      "activations/layer7_attention_weight_max": 56.99003601074219,
      "activations/layer7_attention_weight_min": -53.19144821166992,
      "activations/layer8_attention_weight_max": 44.3508186340332,
      "activations/layer8_attention_weight_min": -41.57061004638672,
      "activations/layer9_attention_weight_max": 58.57123565673828,
      "activations/layer9_attention_weight_min": -55.67045211791992,
      "epoch": 1.99,
      "learning_rate": 0.00013857840909090908,
      "loss": 2.9812,
      "step": 34200
    },
    {
      "activations/layer0_attention_weight_max": 16.02849006652832,
      "activations/layer0_attention_weight_min": -13.216131210327148,
      "activations/layer10_attention_weight_max": 62.91436767578125,
      "activations/layer10_attention_weight_min": -49.23136520385742,
      "activations/layer11_attention_weight_max": 43.27455139160156,
      "activations/layer11_attention_weight_min": -34.88621520996094,
      "activations/layer12_attention_weight_max": 30.658710479736328,
      "activations/layer12_attention_weight_min": -25.445556640625,
      "activations/layer13_attention_weight_max": 62.806095123291016,
      "activations/layer13_attention_weight_min": -44.6259880065918,
      "activations/layer14_attention_weight_max": 80.10530090332031,
      "activations/layer14_attention_weight_min": -50.78518295288086,
      "activations/layer15_attention_weight_max": 50.21025085449219,
      "activations/layer15_attention_weight_min": -36.71302032470703,
      "activations/layer16_attention_weight_max": 43.595298767089844,
      "activations/layer16_attention_weight_min": -28.789688110351562,
      "activations/layer17_attention_weight_max": 58.0694580078125,
      "activations/layer17_attention_weight_min": -34.53523635864258,
      "activations/layer18_attention_weight_max": 63.30405044555664,
      "activations/layer18_attention_weight_min": -34.671451568603516,
      "activations/layer19_attention_weight_max": 23.59120750427246,
      "activations/layer19_attention_weight_min": -17.414182662963867,
      "activations/layer1_attention_weight_max": 16.386653900146484,
      "activations/layer1_attention_weight_min": -14.368570327758789,
      "activations/layer20_attention_weight_max": 25.665861129760742,
      "activations/layer20_attention_weight_min": -15.53884220123291,
      "activations/layer21_attention_weight_max": 41.080814361572266,
      "activations/layer21_attention_weight_min": -21.560710906982422,
      "activations/layer22_attention_weight_max": 37.7999153137207,
      "activations/layer22_attention_weight_min": -20.42178726196289,
      "activations/layer23_attention_weight_max": 22.974262237548828,
      "activations/layer23_attention_weight_min": -12.476581573486328,
      "activations/layer2_attention_weight_max": 26.569618225097656,
      "activations/layer2_attention_weight_min": -25.00244140625,
      "activations/layer3_attention_weight_max": 64.5057144165039,
      "activations/layer3_attention_weight_min": -65.7484130859375,
      "activations/layer4_attention_weight_max": 94.17899322509766,
      "activations/layer4_attention_weight_min": -83.08727264404297,
      "activations/layer5_attention_weight_max": 135.06289672851562,
      "activations/layer5_attention_weight_min": -112.9291000366211,
      "activations/layer6_attention_weight_max": 67.75680541992188,
      "activations/layer6_attention_weight_min": -52.789878845214844,
      "activations/layer7_attention_weight_max": 66.27629089355469,
      "activations/layer7_attention_weight_min": -61.79433822631836,
      "activations/layer8_attention_weight_max": 47.81624984741211,
      "activations/layer8_attention_weight_min": -44.90348434448242,
      "activations/layer9_attention_weight_max": 64.74471282958984,
      "activations/layer9_attention_weight_min": -56.01401901245117,
      "epoch": 1.99,
      "learning_rate": 0.00013855946969696967,
      "loss": 2.9773,
      "step": 34250
    },
    {
      "activations/layer0_attention_weight_max": 13.745966911315918,
      "activations/layer0_attention_weight_min": -13.408679962158203,
      "activations/layer10_attention_weight_max": 62.11851501464844,
      "activations/layer10_attention_weight_min": -52.164974212646484,
      "activations/layer11_attention_weight_max": 47.96732711791992,
      "activations/layer11_attention_weight_min": -35.64058303833008,
      "activations/layer12_attention_weight_max": 24.90628433227539,
      "activations/layer12_attention_weight_min": -23.64881134033203,
      "activations/layer13_attention_weight_max": 78.9318618774414,
      "activations/layer13_attention_weight_min": -52.181026458740234,
      "activations/layer14_attention_weight_max": 52.84272384643555,
      "activations/layer14_attention_weight_min": -38.97705841064453,
      "activations/layer15_attention_weight_max": 64.52286529541016,
      "activations/layer15_attention_weight_min": -44.550514221191406,
      "activations/layer16_attention_weight_max": 41.2735710144043,
      "activations/layer16_attention_weight_min": -29.20176124572754,
      "activations/layer17_attention_weight_max": 56.87117385864258,
      "activations/layer17_attention_weight_min": -36.104042053222656,
      "activations/layer18_attention_weight_max": 51.8676872253418,
      "activations/layer18_attention_weight_min": -31.986055374145508,
      "activations/layer19_attention_weight_max": 19.119997024536133,
      "activations/layer19_attention_weight_min": -14.000958442687988,
      "activations/layer1_attention_weight_max": 16.217039108276367,
      "activations/layer1_attention_weight_min": -12.453977584838867,
      "activations/layer20_attention_weight_max": 18.202363967895508,
      "activations/layer20_attention_weight_min": -16.825462341308594,
      "activations/layer21_attention_weight_max": 37.38568878173828,
      "activations/layer21_attention_weight_min": -25.1035099029541,
      "activations/layer22_attention_weight_max": 33.65092468261719,
      "activations/layer22_attention_weight_min": -23.608266830444336,
      "activations/layer23_attention_weight_max": 21.39859390258789,
      "activations/layer23_attention_weight_min": -14.146839141845703,
      "activations/layer2_attention_weight_max": 26.44772720336914,
      "activations/layer2_attention_weight_min": -25.7125244140625,
      "activations/layer3_attention_weight_max": 59.72368621826172,
      "activations/layer3_attention_weight_min": -64.81331634521484,
      "activations/layer4_attention_weight_max": 86.52141571044922,
      "activations/layer4_attention_weight_min": -86.04130554199219,
      "activations/layer5_attention_weight_max": 123.93479919433594,
      "activations/layer5_attention_weight_min": -105.20738220214844,
      "activations/layer6_attention_weight_max": 61.26966857910156,
      "activations/layer6_attention_weight_min": -51.10021209716797,
      "activations/layer7_attention_weight_max": 58.181861877441406,
      "activations/layer7_attention_weight_min": -54.65970993041992,
      "activations/layer8_attention_weight_max": 48.76736831665039,
      "activations/layer8_attention_weight_min": -40.93677520751953,
      "activations/layer9_attention_weight_max": 62.13323974609375,
      "activations/layer9_attention_weight_min": -53.931732177734375,
      "epoch": 1.99,
      "learning_rate": 0.0001385405303030303,
      "loss": 2.9781,
      "step": 34300
    },
    {
      "activations/layer0_attention_weight_max": 14.4002685546875,
      "activations/layer0_attention_weight_min": -14.114809036254883,
      "activations/layer10_attention_weight_max": 62.3402099609375,
      "activations/layer10_attention_weight_min": -46.49129104614258,
      "activations/layer11_attention_weight_max": 44.95214080810547,
      "activations/layer11_attention_weight_min": -34.221641540527344,
      "activations/layer12_attention_weight_max": 25.960115432739258,
      "activations/layer12_attention_weight_min": -23.18387794494629,
      "activations/layer13_attention_weight_max": 80.54654693603516,
      "activations/layer13_attention_weight_min": -54.95758056640625,
      "activations/layer14_attention_weight_max": 62.92365646362305,
      "activations/layer14_attention_weight_min": -42.36398696899414,
      "activations/layer15_attention_weight_max": 58.860042572021484,
      "activations/layer15_attention_weight_min": -38.6328010559082,
      "activations/layer16_attention_weight_max": 43.01054763793945,
      "activations/layer16_attention_weight_min": -27.968088150024414,
      "activations/layer17_attention_weight_max": 62.42863845825195,
      "activations/layer17_attention_weight_min": -36.64917755126953,
      "activations/layer18_attention_weight_max": 69.56806182861328,
      "activations/layer18_attention_weight_min": -35.203468322753906,
      "activations/layer19_attention_weight_max": 27.432342529296875,
      "activations/layer19_attention_weight_min": -16.154369354248047,
      "activations/layer1_attention_weight_max": 16.39678955078125,
      "activations/layer1_attention_weight_min": -12.448119163513184,
      "activations/layer20_attention_weight_max": 21.692014694213867,
      "activations/layer20_attention_weight_min": -16.47316551208496,
      "activations/layer21_attention_weight_max": 43.08864212036133,
      "activations/layer21_attention_weight_min": -23.972227096557617,
      "activations/layer22_attention_weight_max": 41.53798294067383,
      "activations/layer22_attention_weight_min": -21.440513610839844,
      "activations/layer23_attention_weight_max": 28.140609741210938,
      "activations/layer23_attention_weight_min": -16.304615020751953,
      "activations/layer2_attention_weight_max": 26.535022735595703,
      "activations/layer2_attention_weight_min": -26.009098052978516,
      "activations/layer3_attention_weight_max": 62.09799575805664,
      "activations/layer3_attention_weight_min": -64.56014251708984,
      "activations/layer4_attention_weight_max": 94.45497131347656,
      "activations/layer4_attention_weight_min": -82.95272064208984,
      "activations/layer5_attention_weight_max": 124.42588806152344,
      "activations/layer5_attention_weight_min": -105.1767578125,
      "activations/layer6_attention_weight_max": 59.0489616394043,
      "activations/layer6_attention_weight_min": -50.12271499633789,
      "activations/layer7_attention_weight_max": 58.88899230957031,
      "activations/layer7_attention_weight_min": -57.175315856933594,
      "activations/layer8_attention_weight_max": 46.544795989990234,
      "activations/layer8_attention_weight_min": -40.925376892089844,
      "activations/layer9_attention_weight_max": 67.22038269042969,
      "activations/layer9_attention_weight_min": -56.1071662902832,
      "epoch": 2.0,
      "learning_rate": 0.0001385215909090909,
      "loss": 2.9837,
      "step": 34350
    },
    {
      "activations/layer0_attention_weight_max": 13.616290092468262,
      "activations/layer0_attention_weight_min": -13.986693382263184,
      "activations/layer10_attention_weight_max": 59.843292236328125,
      "activations/layer10_attention_weight_min": -50.72615432739258,
      "activations/layer11_attention_weight_max": 48.14341735839844,
      "activations/layer11_attention_weight_min": -40.998416900634766,
      "activations/layer12_attention_weight_max": 25.426374435424805,
      "activations/layer12_attention_weight_min": -25.679471969604492,
      "activations/layer13_attention_weight_max": 65.43863677978516,
      "activations/layer13_attention_weight_min": -44.83100509643555,
      "activations/layer14_attention_weight_max": 54.531944274902344,
      "activations/layer14_attention_weight_min": -34.81986618041992,
      "activations/layer15_attention_weight_max": 58.49809265136719,
      "activations/layer15_attention_weight_min": -39.25840759277344,
      "activations/layer16_attention_weight_max": 43.10310745239258,
      "activations/layer16_attention_weight_min": -27.822956085205078,
      "activations/layer17_attention_weight_max": 77.8668212890625,
      "activations/layer17_attention_weight_min": -39.96217346191406,
      "activations/layer18_attention_weight_max": 59.78606414794922,
      "activations/layer18_attention_weight_min": -32.71575164794922,
      "activations/layer19_attention_weight_max": 21.89338493347168,
      "activations/layer19_attention_weight_min": -16.342073440551758,
      "activations/layer1_attention_weight_max": 17.676801681518555,
      "activations/layer1_attention_weight_min": -13.034857749938965,
      "activations/layer20_attention_weight_max": 23.627460479736328,
      "activations/layer20_attention_weight_min": -16.987247467041016,
      "activations/layer21_attention_weight_max": 40.04938507080078,
      "activations/layer21_attention_weight_min": -23.764577865600586,
      "activations/layer22_attention_weight_max": 36.58914566040039,
      "activations/layer22_attention_weight_min": -23.95728874206543,
      "activations/layer23_attention_weight_max": 22.806682586669922,
      "activations/layer23_attention_weight_min": -14.267223358154297,
      "activations/layer2_attention_weight_max": 28.083921432495117,
      "activations/layer2_attention_weight_min": -26.20728302001953,
      "activations/layer3_attention_weight_max": 64.77859497070312,
      "activations/layer3_attention_weight_min": -67.43171691894531,
      "activations/layer4_attention_weight_max": 97.43331146240234,
      "activations/layer4_attention_weight_min": -90.1651611328125,
      "activations/layer5_attention_weight_max": 135.71083068847656,
      "activations/layer5_attention_weight_min": -113.46145629882812,
      "activations/layer6_attention_weight_max": 65.19429016113281,
      "activations/layer6_attention_weight_min": -55.39670181274414,
      "activations/layer7_attention_weight_max": 61.07509231567383,
      "activations/layer7_attention_weight_min": -60.41109085083008,
      "activations/layer8_attention_weight_max": 50.44431686401367,
      "activations/layer8_attention_weight_min": -53.265830993652344,
      "activations/layer9_attention_weight_max": 69.12928009033203,
      "activations/layer9_attention_weight_min": -67.01222229003906,
      "epoch": 2.0,
      "learning_rate": 0.0001385026515151515,
      "loss": 2.9523,
      "step": 34400
    },
    {
      "activations/layer0_attention_weight_max": 13.135703086853027,
      "activations/layer0_attention_weight_min": -13.398776054382324,
      "activations/layer10_attention_weight_max": 58.57469177246094,
      "activations/layer10_attention_weight_min": -48.13566207885742,
      "activations/layer11_attention_weight_max": 45.417030334472656,
      "activations/layer11_attention_weight_min": -35.330955505371094,
      "activations/layer12_attention_weight_max": 27.30207061767578,
      "activations/layer12_attention_weight_min": -23.06731605529785,
      "activations/layer13_attention_weight_max": 79.51385498046875,
      "activations/layer13_attention_weight_min": -51.87166213989258,
      "activations/layer14_attention_weight_max": 68.1178207397461,
      "activations/layer14_attention_weight_min": -44.02907943725586,
      "activations/layer15_attention_weight_max": 56.707557678222656,
      "activations/layer15_attention_weight_min": -38.70871353149414,
      "activations/layer16_attention_weight_max": 43.043697357177734,
      "activations/layer16_attention_weight_min": -28.559587478637695,
      "activations/layer17_attention_weight_max": 57.02370071411133,
      "activations/layer17_attention_weight_min": -36.4573860168457,
      "activations/layer18_attention_weight_max": 60.7576789855957,
      "activations/layer18_attention_weight_min": -35.027976989746094,
      "activations/layer19_attention_weight_max": 21.70125389099121,
      "activations/layer19_attention_weight_min": -15.467638969421387,
      "activations/layer1_attention_weight_max": 16.610963821411133,
      "activations/layer1_attention_weight_min": -15.128613471984863,
      "activations/layer20_attention_weight_max": 21.420310974121094,
      "activations/layer20_attention_weight_min": -19.78038787841797,
      "activations/layer21_attention_weight_max": 44.41708755493164,
      "activations/layer21_attention_weight_min": -25.78500747680664,
      "activations/layer22_attention_weight_max": 35.88180923461914,
      "activations/layer22_attention_weight_min": -21.98712158203125,
      "activations/layer23_attention_weight_max": 25.84199333190918,
      "activations/layer23_attention_weight_min": -14.440530776977539,
      "activations/layer2_attention_weight_max": 25.833829879760742,
      "activations/layer2_attention_weight_min": -25.145160675048828,
      "activations/layer3_attention_weight_max": 61.07741165161133,
      "activations/layer3_attention_weight_min": -65.24385070800781,
      "activations/layer4_attention_weight_max": 88.14370727539062,
      "activations/layer4_attention_weight_min": -81.13109588623047,
      "activations/layer5_attention_weight_max": 116.41771697998047,
      "activations/layer5_attention_weight_min": -107.82722473144531,
      "activations/layer6_attention_weight_max": 62.60902786254883,
      "activations/layer6_attention_weight_min": -53.89659881591797,
      "activations/layer7_attention_weight_max": 56.55754852294922,
      "activations/layer7_attention_weight_min": -52.37309646606445,
      "activations/layer8_attention_weight_max": 43.2988166809082,
      "activations/layer8_attention_weight_min": -39.60114669799805,
      "activations/layer9_attention_weight_max": 67.24234008789062,
      "activations/layer9_attention_weight_min": -55.581459045410156,
      "epoch": 2.0,
      "learning_rate": 0.0001384837121212121,
      "loss": 2.9991,
      "step": 34450
    },
    {
      "activations/layer0_attention_weight_max": 13.637654304504395,
      "activations/layer0_attention_weight_min": -13.917037963867188,
      "activations/layer10_attention_weight_max": 65.30064392089844,
      "activations/layer10_attention_weight_min": -49.993988037109375,
      "activations/layer11_attention_weight_max": 49.060752868652344,
      "activations/layer11_attention_weight_min": -40.10554885864258,
      "activations/layer12_attention_weight_max": 44.61365509033203,
      "activations/layer12_attention_weight_min": -29.802350997924805,
      "activations/layer13_attention_weight_max": 67.59693908691406,
      "activations/layer13_attention_weight_min": -43.444766998291016,
      "activations/layer14_attention_weight_max": 52.943504333496094,
      "activations/layer14_attention_weight_min": -40.4354133605957,
      "activations/layer15_attention_weight_max": 57.42546081542969,
      "activations/layer15_attention_weight_min": -38.87565612792969,
      "activations/layer16_attention_weight_max": 43.51004409790039,
      "activations/layer16_attention_weight_min": -30.269344329833984,
      "activations/layer17_attention_weight_max": 72.70777893066406,
      "activations/layer17_attention_weight_min": -37.50701904296875,
      "activations/layer18_attention_weight_max": 58.118896484375,
      "activations/layer18_attention_weight_min": -33.9935302734375,
      "activations/layer19_attention_weight_max": 21.064319610595703,
      "activations/layer19_attention_weight_min": -14.765312194824219,
      "activations/layer1_attention_weight_max": 16.740339279174805,
      "activations/layer1_attention_weight_min": -12.968603134155273,
      "activations/layer20_attention_weight_max": 20.132461547851562,
      "activations/layer20_attention_weight_min": -18.003311157226562,
      "activations/layer21_attention_weight_max": 37.007144927978516,
      "activations/layer21_attention_weight_min": -24.52474021911621,
      "activations/layer22_attention_weight_max": 37.34785461425781,
      "activations/layer22_attention_weight_min": -22.20931053161621,
      "activations/layer23_attention_weight_max": 22.594314575195312,
      "activations/layer23_attention_weight_min": -14.220647811889648,
      "activations/layer2_attention_weight_max": 25.68146324157715,
      "activations/layer2_attention_weight_min": -24.455821990966797,
      "activations/layer3_attention_weight_max": 64.47444152832031,
      "activations/layer3_attention_weight_min": -64.55074310302734,
      "activations/layer4_attention_weight_max": 96.53741455078125,
      "activations/layer4_attention_weight_min": -85.60958099365234,
      "activations/layer5_attention_weight_max": 132.63665771484375,
      "activations/layer5_attention_weight_min": -105.38009643554688,
      "activations/layer6_attention_weight_max": 61.28562927246094,
      "activations/layer6_attention_weight_min": -51.21051788330078,
      "activations/layer7_attention_weight_max": 60.79865264892578,
      "activations/layer7_attention_weight_min": -53.660282135009766,
      "activations/layer8_attention_weight_max": 55.51578903198242,
      "activations/layer8_attention_weight_min": -47.1734504699707,
      "activations/layer9_attention_weight_max": 72.00944519042969,
      "activations/layer9_attention_weight_min": -60.648956298828125,
      "epoch": 2.0,
      "learning_rate": 0.00013846477272727272,
      "loss": 2.9712,
      "step": 34500
    },
    {
      "activations/layer0_attention_weight_max": 16.534378051757812,
      "activations/layer0_attention_weight_min": -13.8406982421875,
      "activations/layer10_attention_weight_max": 56.09099578857422,
      "activations/layer10_attention_weight_min": -46.98573684692383,
      "activations/layer11_attention_weight_max": 41.84748458862305,
      "activations/layer11_attention_weight_min": -37.661869049072266,
      "activations/layer12_attention_weight_max": 32.2061767578125,
      "activations/layer12_attention_weight_min": -23.23794937133789,
      "activations/layer13_attention_weight_max": 62.66901779174805,
      "activations/layer13_attention_weight_min": -45.41510772705078,
      "activations/layer14_attention_weight_max": 59.01915740966797,
      "activations/layer14_attention_weight_min": -42.67914962768555,
      "activations/layer15_attention_weight_max": 55.54521942138672,
      "activations/layer15_attention_weight_min": -41.49645233154297,
      "activations/layer16_attention_weight_max": 43.01758575439453,
      "activations/layer16_attention_weight_min": -30.300228118896484,
      "activations/layer17_attention_weight_max": 55.657527923583984,
      "activations/layer17_attention_weight_min": -37.19578552246094,
      "activations/layer18_attention_weight_max": 53.14459991455078,
      "activations/layer18_attention_weight_min": -35.65143585205078,
      "activations/layer19_attention_weight_max": 19.930723190307617,
      "activations/layer19_attention_weight_min": -16.582380294799805,
      "activations/layer1_attention_weight_max": 16.457141876220703,
      "activations/layer1_attention_weight_min": -13.869807243347168,
      "activations/layer20_attention_weight_max": 20.23958969116211,
      "activations/layer20_attention_weight_min": -18.131122589111328,
      "activations/layer21_attention_weight_max": 38.02371597290039,
      "activations/layer21_attention_weight_min": -23.50655174255371,
      "activations/layer22_attention_weight_max": 39.44460678100586,
      "activations/layer22_attention_weight_min": -22.901042938232422,
      "activations/layer23_attention_weight_max": 21.20667266845703,
      "activations/layer23_attention_weight_min": -15.06107234954834,
      "activations/layer2_attention_weight_max": 25.074657440185547,
      "activations/layer2_attention_weight_min": -26.367053985595703,
      "activations/layer3_attention_weight_max": 58.053749084472656,
      "activations/layer3_attention_weight_min": -63.923091888427734,
      "activations/layer4_attention_weight_max": 88.98780822753906,
      "activations/layer4_attention_weight_min": -82.78365325927734,
      "activations/layer5_attention_weight_max": 124.9922103881836,
      "activations/layer5_attention_weight_min": -108.12638854980469,
      "activations/layer6_attention_weight_max": 63.30510330200195,
      "activations/layer6_attention_weight_min": -55.25596237182617,
      "activations/layer7_attention_weight_max": 58.228511810302734,
      "activations/layer7_attention_weight_min": -54.00326919555664,
      "activations/layer8_attention_weight_max": 46.232852935791016,
      "activations/layer8_attention_weight_min": -41.26396560668945,
      "activations/layer9_attention_weight_max": 63.67024612426758,
      "activations/layer9_attention_weight_min": -57.55335235595703,
      "epoch": 2.01,
      "learning_rate": 0.0001384458333333333,
      "loss": 2.9578,
      "step": 34550
    },
    {
      "activations/layer0_attention_weight_max": 13.702534675598145,
      "activations/layer0_attention_weight_min": -13.882891654968262,
      "activations/layer10_attention_weight_max": 59.663291931152344,
      "activations/layer10_attention_weight_min": -46.22296905517578,
      "activations/layer11_attention_weight_max": 45.362945556640625,
      "activations/layer11_attention_weight_min": -33.71436309814453,
      "activations/layer12_attention_weight_max": 32.98405075073242,
      "activations/layer12_attention_weight_min": -25.411603927612305,
      "activations/layer13_attention_weight_max": 61.5307731628418,
      "activations/layer13_attention_weight_min": -39.666500091552734,
      "activations/layer14_attention_weight_max": 62.59065628051758,
      "activations/layer14_attention_weight_min": -39.768314361572266,
      "activations/layer15_attention_weight_max": 60.629249572753906,
      "activations/layer15_attention_weight_min": -38.5078239440918,
      "activations/layer16_attention_weight_max": 45.55712890625,
      "activations/layer16_attention_weight_min": -28.693771362304688,
      "activations/layer17_attention_weight_max": 67.59005737304688,
      "activations/layer17_attention_weight_min": -36.85957717895508,
      "activations/layer18_attention_weight_max": 63.310546875,
      "activations/layer18_attention_weight_min": -34.816368103027344,
      "activations/layer19_attention_weight_max": 25.03713035583496,
      "activations/layer19_attention_weight_min": -18.146305084228516,
      "activations/layer1_attention_weight_max": 16.986637115478516,
      "activations/layer1_attention_weight_min": -12.867594718933105,
      "activations/layer20_attention_weight_max": 25.06759262084961,
      "activations/layer20_attention_weight_min": -15.933050155639648,
      "activations/layer21_attention_weight_max": 48.78641891479492,
      "activations/layer21_attention_weight_min": -24.285032272338867,
      "activations/layer22_attention_weight_max": 47.33876037597656,
      "activations/layer22_attention_weight_min": -26.09857749938965,
      "activations/layer23_attention_weight_max": 30.314495086669922,
      "activations/layer23_attention_weight_min": -15.449687004089355,
      "activations/layer2_attention_weight_max": 26.324996948242188,
      "activations/layer2_attention_weight_min": -25.484180450439453,
      "activations/layer3_attention_weight_max": 59.242374420166016,
      "activations/layer3_attention_weight_min": -61.657562255859375,
      "activations/layer4_attention_weight_max": 93.49858856201172,
      "activations/layer4_attention_weight_min": -80.520263671875,
      "activations/layer5_attention_weight_max": 131.10995483398438,
      "activations/layer5_attention_weight_min": -103.59831237792969,
      "activations/layer6_attention_weight_max": 63.41922378540039,
      "activations/layer6_attention_weight_min": -52.88380432128906,
      "activations/layer7_attention_weight_max": 62.56465530395508,
      "activations/layer7_attention_weight_min": -53.20809555053711,
      "activations/layer8_attention_weight_max": 48.40065383911133,
      "activations/layer8_attention_weight_min": -40.81627655029297,
      "activations/layer9_attention_weight_max": 71.42269134521484,
      "activations/layer9_attention_weight_min": -60.194923400878906,
      "epoch": 2.01,
      "learning_rate": 0.00013842689393939392,
      "loss": 2.9635,
      "step": 34600
    },
    {
      "activations/layer0_attention_weight_max": 15.013025283813477,
      "activations/layer0_attention_weight_min": -14.123579025268555,
      "activations/layer10_attention_weight_max": 54.4804801940918,
      "activations/layer10_attention_weight_min": -44.135562896728516,
      "activations/layer11_attention_weight_max": 47.547359466552734,
      "activations/layer11_attention_weight_min": -39.1297721862793,
      "activations/layer12_attention_weight_max": 26.64359474182129,
      "activations/layer12_attention_weight_min": -27.7908878326416,
      "activations/layer13_attention_weight_max": 67.27945709228516,
      "activations/layer13_attention_weight_min": -48.192039489746094,
      "activations/layer14_attention_weight_max": 65.78546905517578,
      "activations/layer14_attention_weight_min": -44.32744216918945,
      "activations/layer15_attention_weight_max": 52.89189529418945,
      "activations/layer15_attention_weight_min": -37.337093353271484,
      "activations/layer16_attention_weight_max": 44.428619384765625,
      "activations/layer16_attention_weight_min": -27.373994827270508,
      "activations/layer17_attention_weight_max": 59.04151916503906,
      "activations/layer17_attention_weight_min": -35.50273513793945,
      "activations/layer18_attention_weight_max": 63.86956024169922,
      "activations/layer18_attention_weight_min": -34.58512496948242,
      "activations/layer19_attention_weight_max": 22.968538284301758,
      "activations/layer19_attention_weight_min": -15.671374320983887,
      "activations/layer1_attention_weight_max": 16.882823944091797,
      "activations/layer1_attention_weight_min": -13.10969066619873,
      "activations/layer20_attention_weight_max": 20.017784118652344,
      "activations/layer20_attention_weight_min": -16.835979461669922,
      "activations/layer21_attention_weight_max": 39.89912796020508,
      "activations/layer21_attention_weight_min": -24.354747772216797,
      "activations/layer22_attention_weight_max": 33.298851013183594,
      "activations/layer22_attention_weight_min": -24.28156280517578,
      "activations/layer23_attention_weight_max": 23.21007537841797,
      "activations/layer23_attention_weight_min": -18.791322708129883,
      "activations/layer2_attention_weight_max": 29.285781860351562,
      "activations/layer2_attention_weight_min": -26.98386001586914,
      "activations/layer3_attention_weight_max": 64.39918518066406,
      "activations/layer3_attention_weight_min": -64.15739440917969,
      "activations/layer4_attention_weight_max": 100.06163024902344,
      "activations/layer4_attention_weight_min": -82.50547790527344,
      "activations/layer5_attention_weight_max": 134.44223022460938,
      "activations/layer5_attention_weight_min": -109.48263549804688,
      "activations/layer6_attention_weight_max": 63.91304397583008,
      "activations/layer6_attention_weight_min": -51.75450134277344,
      "activations/layer7_attention_weight_max": 64.56340026855469,
      "activations/layer7_attention_weight_min": -51.08567428588867,
      "activations/layer8_attention_weight_max": 53.173580169677734,
      "activations/layer8_attention_weight_min": -43.77284622192383,
      "activations/layer9_attention_weight_max": 74.99378967285156,
      "activations/layer9_attention_weight_min": -58.44971466064453,
      "epoch": 2.01,
      "learning_rate": 0.00013840795454545454,
      "loss": 2.9594,
      "step": 34650
    },
    {
      "activations/layer0_attention_weight_max": 14.078132629394531,
      "activations/layer0_attention_weight_min": -13.974146842956543,
      "activations/layer10_attention_weight_max": 57.7337760925293,
      "activations/layer10_attention_weight_min": -49.3975715637207,
      "activations/layer11_attention_weight_max": 40.630958557128906,
      "activations/layer11_attention_weight_min": -33.32688903808594,
      "activations/layer12_attention_weight_max": 30.641809463500977,
      "activations/layer12_attention_weight_min": -26.724308013916016,
      "activations/layer13_attention_weight_max": 78.5466079711914,
      "activations/layer13_attention_weight_min": -57.1704216003418,
      "activations/layer14_attention_weight_max": 70.2647705078125,
      "activations/layer14_attention_weight_min": -50.91888427734375,
      "activations/layer15_attention_weight_max": 63.108245849609375,
      "activations/layer15_attention_weight_min": -45.89247131347656,
      "activations/layer16_attention_weight_max": 42.01036834716797,
      "activations/layer16_attention_weight_min": -29.822477340698242,
      "activations/layer17_attention_weight_max": 56.28313064575195,
      "activations/layer17_attention_weight_min": -35.524879455566406,
      "activations/layer18_attention_weight_max": 57.1169548034668,
      "activations/layer18_attention_weight_min": -32.06374740600586,
      "activations/layer19_attention_weight_max": 21.716684341430664,
      "activations/layer19_attention_weight_min": -17.08514404296875,
      "activations/layer1_attention_weight_max": 16.757869720458984,
      "activations/layer1_attention_weight_min": -12.301488876342773,
      "activations/layer20_attention_weight_max": 18.95220184326172,
      "activations/layer20_attention_weight_min": -16.75844383239746,
      "activations/layer21_attention_weight_max": 38.46000289916992,
      "activations/layer21_attention_weight_min": -23.053558349609375,
      "activations/layer22_attention_weight_max": 33.67622375488281,
      "activations/layer22_attention_weight_min": -24.19261360168457,
      "activations/layer23_attention_weight_max": 24.288307189941406,
      "activations/layer23_attention_weight_min": -15.16541576385498,
      "activations/layer2_attention_weight_max": 27.144620895385742,
      "activations/layer2_attention_weight_min": -26.6661434173584,
      "activations/layer3_attention_weight_max": 62.7022705078125,
      "activations/layer3_attention_weight_min": -65.57559967041016,
      "activations/layer4_attention_weight_max": 92.11030578613281,
      "activations/layer4_attention_weight_min": -81.6488265991211,
      "activations/layer5_attention_weight_max": 127.75723266601562,
      "activations/layer5_attention_weight_min": -111.00355529785156,
      "activations/layer6_attention_weight_max": 63.404850006103516,
      "activations/layer6_attention_weight_min": -53.548152923583984,
      "activations/layer7_attention_weight_max": 55.93399429321289,
      "activations/layer7_attention_weight_min": -54.36850357055664,
      "activations/layer8_attention_weight_max": 46.27113723754883,
      "activations/layer8_attention_weight_min": -41.750362396240234,
      "activations/layer9_attention_weight_max": 61.52803421020508,
      "activations/layer9_attention_weight_min": -53.780548095703125,
      "epoch": 2.02,
      "learning_rate": 0.00013838901515151513,
      "loss": 2.9705,
      "step": 34700
    },
    {
      "activations/layer0_attention_weight_max": 13.933541297912598,
      "activations/layer0_attention_weight_min": -13.948911666870117,
      "activations/layer10_attention_weight_max": 58.832794189453125,
      "activations/layer10_attention_weight_min": -47.0051155090332,
      "activations/layer11_attention_weight_max": 41.786102294921875,
      "activations/layer11_attention_weight_min": -35.83951950073242,
      "activations/layer12_attention_weight_max": 30.334863662719727,
      "activations/layer12_attention_weight_min": -24.69865608215332,
      "activations/layer13_attention_weight_max": 52.85762405395508,
      "activations/layer13_attention_weight_min": -39.62062072753906,
      "activations/layer14_attention_weight_max": 47.65690994262695,
      "activations/layer14_attention_weight_min": -35.18756103515625,
      "activations/layer15_attention_weight_max": 53.00905990600586,
      "activations/layer15_attention_weight_min": -35.32910919189453,
      "activations/layer16_attention_weight_max": 40.24484634399414,
      "activations/layer16_attention_weight_min": -28.213768005371094,
      "activations/layer17_attention_weight_max": 56.65074920654297,
      "activations/layer17_attention_weight_min": -37.4139289855957,
      "activations/layer18_attention_weight_max": 59.34992599487305,
      "activations/layer18_attention_weight_min": -32.58784103393555,
      "activations/layer19_attention_weight_max": 22.03507423400879,
      "activations/layer19_attention_weight_min": -15.361780166625977,
      "activations/layer1_attention_weight_max": 15.605475425720215,
      "activations/layer1_attention_weight_min": -12.956733703613281,
      "activations/layer20_attention_weight_max": 20.216398239135742,
      "activations/layer20_attention_weight_min": -16.91502571105957,
      "activations/layer21_attention_weight_max": 35.458290100097656,
      "activations/layer21_attention_weight_min": -23.084491729736328,
      "activations/layer22_attention_weight_max": 36.70793533325195,
      "activations/layer22_attention_weight_min": -21.900672912597656,
      "activations/layer23_attention_weight_max": 22.935649871826172,
      "activations/layer23_attention_weight_min": -13.697699546813965,
      "activations/layer2_attention_weight_max": 27.007848739624023,
      "activations/layer2_attention_weight_min": -25.487747192382812,
      "activations/layer3_attention_weight_max": 61.824527740478516,
      "activations/layer3_attention_weight_min": -66.99991607666016,
      "activations/layer4_attention_weight_max": 91.16346740722656,
      "activations/layer4_attention_weight_min": -86.72134399414062,
      "activations/layer5_attention_weight_max": 122.87321472167969,
      "activations/layer5_attention_weight_min": -105.23433685302734,
      "activations/layer6_attention_weight_max": 61.02141571044922,
      "activations/layer6_attention_weight_min": -51.17585372924805,
      "activations/layer7_attention_weight_max": 55.609169006347656,
      "activations/layer7_attention_weight_min": -51.861629486083984,
      "activations/layer8_attention_weight_max": 44.28324508666992,
      "activations/layer8_attention_weight_min": -38.8260383605957,
      "activations/layer9_attention_weight_max": 58.64925003051758,
      "activations/layer9_attention_weight_min": -54.82125473022461,
      "epoch": 2.02,
      "learning_rate": 0.00013837007575757574,
      "loss": 2.9513,
      "step": 34750
    },
    {
      "activations/layer0_attention_weight_max": 16.895668029785156,
      "activations/layer0_attention_weight_min": -13.8341064453125,
      "activations/layer10_attention_weight_max": 62.35931396484375,
      "activations/layer10_attention_weight_min": -46.55060577392578,
      "activations/layer11_attention_weight_max": 50.40423583984375,
      "activations/layer11_attention_weight_min": -35.68256378173828,
      "activations/layer12_attention_weight_max": 44.33165740966797,
      "activations/layer12_attention_weight_min": -29.519563674926758,
      "activations/layer13_attention_weight_max": 59.72733688354492,
      "activations/layer13_attention_weight_min": -43.94392776489258,
      "activations/layer14_attention_weight_max": 54.833492279052734,
      "activations/layer14_attention_weight_min": -41.16144561767578,
      "activations/layer15_attention_weight_max": 49.96854019165039,
      "activations/layer15_attention_weight_min": -34.47727966308594,
      "activations/layer16_attention_weight_max": 43.46309280395508,
      "activations/layer16_attention_weight_min": -27.465850830078125,
      "activations/layer17_attention_weight_max": 62.02111053466797,
      "activations/layer17_attention_weight_min": -37.92290115356445,
      "activations/layer18_attention_weight_max": 61.434776306152344,
      "activations/layer18_attention_weight_min": -34.672996520996094,
      "activations/layer19_attention_weight_max": 20.16788101196289,
      "activations/layer19_attention_weight_min": -16.249174118041992,
      "activations/layer1_attention_weight_max": 15.614779472351074,
      "activations/layer1_attention_weight_min": -13.598666191101074,
      "activations/layer20_attention_weight_max": 20.44192886352539,
      "activations/layer20_attention_weight_min": -16.695796966552734,
      "activations/layer21_attention_weight_max": 41.25895690917969,
      "activations/layer21_attention_weight_min": -24.37932014465332,
      "activations/layer22_attention_weight_max": 35.54744338989258,
      "activations/layer22_attention_weight_min": -21.992517471313477,
      "activations/layer23_attention_weight_max": 23.663692474365234,
      "activations/layer23_attention_weight_min": -16.41651153564453,
      "activations/layer2_attention_weight_max": 29.344785690307617,
      "activations/layer2_attention_weight_min": -27.72450828552246,
      "activations/layer3_attention_weight_max": 64.58445739746094,
      "activations/layer3_attention_weight_min": -68.1684799194336,
      "activations/layer4_attention_weight_max": 97.87571716308594,
      "activations/layer4_attention_weight_min": -86.09020233154297,
      "activations/layer5_attention_weight_max": 135.88473510742188,
      "activations/layer5_attention_weight_min": -114.88395690917969,
      "activations/layer6_attention_weight_max": 67.07379150390625,
      "activations/layer6_attention_weight_min": -51.19879913330078,
      "activations/layer7_attention_weight_max": 67.23272705078125,
      "activations/layer7_attention_weight_min": -55.43781280517578,
      "activations/layer8_attention_weight_max": 50.80519104003906,
      "activations/layer8_attention_weight_min": -40.503414154052734,
      "activations/layer9_attention_weight_max": 64.5911865234375,
      "activations/layer9_attention_weight_min": -56.338775634765625,
      "epoch": 2.02,
      "learning_rate": 0.00013835151515151513,
      "loss": 2.9675,
      "step": 34800
    },
    {
      "activations/layer0_attention_weight_max": 13.672323226928711,
      "activations/layer0_attention_weight_min": -13.34702205657959,
      "activations/layer10_attention_weight_max": 58.5083122253418,
      "activations/layer10_attention_weight_min": -48.93350601196289,
      "activations/layer11_attention_weight_max": 43.0767822265625,
      "activations/layer11_attention_weight_min": -36.50130081176758,
      "activations/layer12_attention_weight_max": 28.327913284301758,
      "activations/layer12_attention_weight_min": -24.7402400970459,
      "activations/layer13_attention_weight_max": 83.35808563232422,
      "activations/layer13_attention_weight_min": -60.47246170043945,
      "activations/layer14_attention_weight_max": 64.35693359375,
      "activations/layer14_attention_weight_min": -50.5665168762207,
      "activations/layer15_attention_weight_max": 51.61103057861328,
      "activations/layer15_attention_weight_min": -36.43625259399414,
      "activations/layer16_attention_weight_max": 41.33510208129883,
      "activations/layer16_attention_weight_min": -29.75548553466797,
      "activations/layer17_attention_weight_max": 61.86030578613281,
      "activations/layer17_attention_weight_min": -37.258445739746094,
      "activations/layer18_attention_weight_max": 58.52372741699219,
      "activations/layer18_attention_weight_min": -33.46223449707031,
      "activations/layer19_attention_weight_max": 23.176427841186523,
      "activations/layer19_attention_weight_min": -16.262434005737305,
      "activations/layer1_attention_weight_max": 16.23378562927246,
      "activations/layer1_attention_weight_min": -14.334628105163574,
      "activations/layer20_attention_weight_max": 22.36138153076172,
      "activations/layer20_attention_weight_min": -16.812334060668945,
      "activations/layer21_attention_weight_max": 51.38679885864258,
      "activations/layer21_attention_weight_min": -24.703617095947266,
      "activations/layer22_attention_weight_max": 39.443389892578125,
      "activations/layer22_attention_weight_min": -22.275592803955078,
      "activations/layer23_attention_weight_max": 25.966405868530273,
      "activations/layer23_attention_weight_min": -15.297843933105469,
      "activations/layer2_attention_weight_max": 28.954208374023438,
      "activations/layer2_attention_weight_min": -28.089345932006836,
      "activations/layer3_attention_weight_max": 60.12904357910156,
      "activations/layer3_attention_weight_min": -63.448062896728516,
      "activations/layer4_attention_weight_max": 88.0933837890625,
      "activations/layer4_attention_weight_min": -85.13668060302734,
      "activations/layer5_attention_weight_max": 125.70552825927734,
      "activations/layer5_attention_weight_min": -108.67073822021484,
      "activations/layer6_attention_weight_max": 63.200767517089844,
      "activations/layer6_attention_weight_min": -52.28968048095703,
      "activations/layer7_attention_weight_max": 60.13361358642578,
      "activations/layer7_attention_weight_min": -55.5589599609375,
      "activations/layer8_attention_weight_max": 46.48520278930664,
      "activations/layer8_attention_weight_min": -38.36034393310547,
      "activations/layer9_attention_weight_max": 63.63604736328125,
      "activations/layer9_attention_weight_min": -57.21601104736328,
      "epoch": 2.02,
      "learning_rate": 0.00013833257575757575,
      "loss": 2.958,
      "step": 34850
    },
    {
      "activations/layer0_attention_weight_max": 13.490781784057617,
      "activations/layer0_attention_weight_min": -13.796321868896484,
      "activations/layer10_attention_weight_max": 56.7179069519043,
      "activations/layer10_attention_weight_min": -48.5316162109375,
      "activations/layer11_attention_weight_max": 40.056243896484375,
      "activations/layer11_attention_weight_min": -33.32926940917969,
      "activations/layer12_attention_weight_max": 28.020719528198242,
      "activations/layer12_attention_weight_min": -23.646167755126953,
      "activations/layer13_attention_weight_max": 51.41206359863281,
      "activations/layer13_attention_weight_min": -43.07723617553711,
      "activations/layer14_attention_weight_max": 50.201087951660156,
      "activations/layer14_attention_weight_min": -39.679664611816406,
      "activations/layer15_attention_weight_max": 49.274497985839844,
      "activations/layer15_attention_weight_min": -38.20235824584961,
      "activations/layer16_attention_weight_max": 38.350929260253906,
      "activations/layer16_attention_weight_min": -28.2514705657959,
      "activations/layer17_attention_weight_max": 63.79804229736328,
      "activations/layer17_attention_weight_min": -35.818302154541016,
      "activations/layer18_attention_weight_max": 57.49303436279297,
      "activations/layer18_attention_weight_min": -34.154571533203125,
      "activations/layer19_attention_weight_max": 19.70981788635254,
      "activations/layer19_attention_weight_min": -17.04347038269043,
      "activations/layer1_attention_weight_max": 15.69198989868164,
      "activations/layer1_attention_weight_min": -13.79360294342041,
      "activations/layer20_attention_weight_max": 18.678312301635742,
      "activations/layer20_attention_weight_min": -17.559661865234375,
      "activations/layer21_attention_weight_max": 40.17138671875,
      "activations/layer21_attention_weight_min": -25.615951538085938,
      "activations/layer22_attention_weight_max": 34.08647537231445,
      "activations/layer22_attention_weight_min": -21.951732635498047,
      "activations/layer23_attention_weight_max": 22.33393096923828,
      "activations/layer23_attention_weight_min": -16.043376922607422,
      "activations/layer2_attention_weight_max": 28.016197204589844,
      "activations/layer2_attention_weight_min": -25.749099731445312,
      "activations/layer3_attention_weight_max": 59.79026794433594,
      "activations/layer3_attention_weight_min": -64.89717864990234,
      "activations/layer4_attention_weight_max": 89.63092803955078,
      "activations/layer4_attention_weight_min": -80.1963119506836,
      "activations/layer5_attention_weight_max": 125.41696166992188,
      "activations/layer5_attention_weight_min": -103.93214416503906,
      "activations/layer6_attention_weight_max": 59.22507095336914,
      "activations/layer6_attention_weight_min": -51.76599884033203,
      "activations/layer7_attention_weight_max": 54.98870849609375,
      "activations/layer7_attention_weight_min": -48.6387825012207,
      "activations/layer8_attention_weight_max": 45.407772064208984,
      "activations/layer8_attention_weight_min": -43.28226089477539,
      "activations/layer9_attention_weight_max": 56.25020217895508,
      "activations/layer9_attention_weight_min": -52.77766036987305,
      "epoch": 2.03,
      "learning_rate": 0.00013831363636363636,
      "loss": 2.9537,
      "step": 34900
    },
    {
      "activations/layer0_attention_weight_max": 15.319193840026855,
      "activations/layer0_attention_weight_min": -13.488373756408691,
      "activations/layer10_attention_weight_max": 68.2529525756836,
      "activations/layer10_attention_weight_min": -52.87886428833008,
      "activations/layer11_attention_weight_max": 47.4691162109375,
      "activations/layer11_attention_weight_min": -37.90483093261719,
      "activations/layer12_attention_weight_max": 33.780094146728516,
      "activations/layer12_attention_weight_min": -26.311906814575195,
      "activations/layer13_attention_weight_max": 82.87618255615234,
      "activations/layer13_attention_weight_min": -51.660499572753906,
      "activations/layer14_attention_weight_max": 80.13780975341797,
      "activations/layer14_attention_weight_min": -47.19438171386719,
      "activations/layer15_attention_weight_max": 75.45970153808594,
      "activations/layer15_attention_weight_min": -39.665836334228516,
      "activations/layer16_attention_weight_max": 47.82889175415039,
      "activations/layer16_attention_weight_min": -27.500211715698242,
      "activations/layer17_attention_weight_max": 66.68148803710938,
      "activations/layer17_attention_weight_min": -33.47113037109375,
      "activations/layer18_attention_weight_max": 65.13666534423828,
      "activations/layer18_attention_weight_min": -33.11696243286133,
      "activations/layer19_attention_weight_max": 22.947967529296875,
      "activations/layer19_attention_weight_min": -14.551325798034668,
      "activations/layer1_attention_weight_max": 17.154630661010742,
      "activations/layer1_attention_weight_min": -12.243243217468262,
      "activations/layer20_attention_weight_max": 22.837364196777344,
      "activations/layer20_attention_weight_min": -15.601263046264648,
      "activations/layer21_attention_weight_max": 49.388824462890625,
      "activations/layer21_attention_weight_min": -25.051219940185547,
      "activations/layer22_attention_weight_max": 36.63174819946289,
      "activations/layer22_attention_weight_min": -20.660612106323242,
      "activations/layer23_attention_weight_max": 22.084190368652344,
      "activations/layer23_attention_weight_min": -14.857230186462402,
      "activations/layer2_attention_weight_max": 27.638654708862305,
      "activations/layer2_attention_weight_min": -26.995328903198242,
      "activations/layer3_attention_weight_max": 67.14726257324219,
      "activations/layer3_attention_weight_min": -67.3101577758789,
      "activations/layer4_attention_weight_max": 104.58171844482422,
      "activations/layer4_attention_weight_min": -88.99082946777344,
      "activations/layer5_attention_weight_max": 135.27996826171875,
      "activations/layer5_attention_weight_min": -108.49583435058594,
      "activations/layer6_attention_weight_max": 67.53382110595703,
      "activations/layer6_attention_weight_min": -52.52653503417969,
      "activations/layer7_attention_weight_max": 66.8408203125,
      "activations/layer7_attention_weight_min": -56.15451431274414,
      "activations/layer8_attention_weight_max": 50.96464920043945,
      "activations/layer8_attention_weight_min": -42.994659423828125,
      "activations/layer9_attention_weight_max": 67.6107406616211,
      "activations/layer9_attention_weight_min": -60.75971603393555,
      "epoch": 2.03,
      "learning_rate": 0.00013829469696969695,
      "loss": 2.9395,
      "step": 34950
    },
    {
      "activations/layer0_attention_weight_max": 14.279669761657715,
      "activations/layer0_attention_weight_min": -13.611274719238281,
      "activations/layer10_attention_weight_max": 65.5723648071289,
      "activations/layer10_attention_weight_min": -50.96236038208008,
      "activations/layer11_attention_weight_max": 45.64952850341797,
      "activations/layer11_attention_weight_min": -33.05656433105469,
      "activations/layer12_attention_weight_max": 29.20401382446289,
      "activations/layer12_attention_weight_min": -23.546993255615234,
      "activations/layer13_attention_weight_max": 73.18086242675781,
      "activations/layer13_attention_weight_min": -47.940608978271484,
      "activations/layer14_attention_weight_max": 69.16181945800781,
      "activations/layer14_attention_weight_min": -47.08442306518555,
      "activations/layer15_attention_weight_max": 53.24644088745117,
      "activations/layer15_attention_weight_min": -36.68689727783203,
      "activations/layer16_attention_weight_max": 47.053646087646484,
      "activations/layer16_attention_weight_min": -29.236553192138672,
      "activations/layer17_attention_weight_max": 63.40156936645508,
      "activations/layer17_attention_weight_min": -37.668331146240234,
      "activations/layer18_attention_weight_max": 63.97111511230469,
      "activations/layer18_attention_weight_min": -35.6368408203125,
      "activations/layer19_attention_weight_max": 26.66937255859375,
      "activations/layer19_attention_weight_min": -17.41504669189453,
      "activations/layer1_attention_weight_max": 16.425596237182617,
      "activations/layer1_attention_weight_min": -11.85281753540039,
      "activations/layer20_attention_weight_max": 24.5382137298584,
      "activations/layer20_attention_weight_min": -16.55307960510254,
      "activations/layer21_attention_weight_max": 44.410064697265625,
      "activations/layer21_attention_weight_min": -24.424381256103516,
      "activations/layer22_attention_weight_max": 38.30130386352539,
      "activations/layer22_attention_weight_min": -25.0693302154541,
      "activations/layer23_attention_weight_max": 26.201826095581055,
      "activations/layer23_attention_weight_min": -15.924985885620117,
      "activations/layer2_attention_weight_max": 26.153852462768555,
      "activations/layer2_attention_weight_min": -24.82624053955078,
      "activations/layer3_attention_weight_max": 63.65431594848633,
      "activations/layer3_attention_weight_min": -70.12544250488281,
      "activations/layer4_attention_weight_max": 90.6598129272461,
      "activations/layer4_attention_weight_min": -85.53807067871094,
      "activations/layer5_attention_weight_max": 119.8027572631836,
      "activations/layer5_attention_weight_min": -108.36431121826172,
      "activations/layer6_attention_weight_max": 59.36956024169922,
      "activations/layer6_attention_weight_min": -51.61499786376953,
      "activations/layer7_attention_weight_max": 54.00782012939453,
      "activations/layer7_attention_weight_min": -53.96914291381836,
      "activations/layer8_attention_weight_max": 47.08833694458008,
      "activations/layer8_attention_weight_min": -41.68571853637695,
      "activations/layer9_attention_weight_max": 64.1368408203125,
      "activations/layer9_attention_weight_min": -58.48796463012695,
      "epoch": 2.03,
      "learning_rate": 0.00013827575757575757,
      "loss": 2.9726,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_loss": 2.9140625,
      "eval_runtime": 8.5197,
      "eval_samples_per_second": 504.007,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_openwebtext_loss": 2.9140625,
      "eval_openwebtext_ppl": 18.431524745974645,
      "eval_openwebtext_runtime": 8.5197,
      "eval_openwebtext_samples_per_second": 504.007,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_wikitext_loss": 3.1796875,
      "eval_wikitext_ppl": 24.039240115615065,
      "eval_wikitext_runtime": 1.9224,
      "eval_wikitext_samples_per_second": 237.197,
      "step": 35000
    },
    {
      "epoch": 2.03,
      "eval_lambada_loss": 3.130859375,
      "eval_lambada_ppl": 22.893645317537526,
      "eval_lambada_runtime": 9.5481,
      "eval_lambada_samples_per_second": 509.944,
      "step": 35000
    },
    {
      "activations/layer0_attention_weight_max": 12.94295597076416,
      "activations/layer0_attention_weight_min": -13.502874374389648,
      "activations/layer10_attention_weight_max": 68.55720520019531,
      "activations/layer10_attention_weight_min": -53.34962844848633,
      "activations/layer11_attention_weight_max": 51.65502166748047,
      "activations/layer11_attention_weight_min": -37.80792236328125,
      "activations/layer12_attention_weight_max": 28.592744827270508,
      "activations/layer12_attention_weight_min": -25.006759643554688,
      "activations/layer13_attention_weight_max": 66.24560546875,
      "activations/layer13_attention_weight_min": -47.29338455200195,
      "activations/layer14_attention_weight_max": 76.98047637939453,
      "activations/layer14_attention_weight_min": -47.89791488647461,
      "activations/layer15_attention_weight_max": 57.34282302856445,
      "activations/layer15_attention_weight_min": -40.839805603027344,
      "activations/layer16_attention_weight_max": 42.45705032348633,
      "activations/layer16_attention_weight_min": -26.80719757080078,
      "activations/layer17_attention_weight_max": 60.09942626953125,
      "activations/layer17_attention_weight_min": -37.56376647949219,
      "activations/layer18_attention_weight_max": 56.09116744995117,
      "activations/layer18_attention_weight_min": -33.2138786315918,
      "activations/layer19_attention_weight_max": 21.16770362854004,
      "activations/layer19_attention_weight_min": -15.925944328308105,
      "activations/layer1_attention_weight_max": 16.221263885498047,
      "activations/layer1_attention_weight_min": -12.622673034667969,
      "activations/layer20_attention_weight_max": 19.937089920043945,
      "activations/layer20_attention_weight_min": -17.943744659423828,
      "activations/layer21_attention_weight_max": 41.3624382019043,
      "activations/layer21_attention_weight_min": -23.320526123046875,
      "activations/layer22_attention_weight_max": 32.384517669677734,
      "activations/layer22_attention_weight_min": -22.783538818359375,
      "activations/layer23_attention_weight_max": 23.330657958984375,
      "activations/layer23_attention_weight_min": -15.173837661743164,
      "activations/layer2_attention_weight_max": 27.641767501831055,
      "activations/layer2_attention_weight_min": -25.86139678955078,
      "activations/layer3_attention_weight_max": 65.59500885009766,
      "activations/layer3_attention_weight_min": -64.74775695800781,
      "activations/layer4_attention_weight_max": 91.40402221679688,
      "activations/layer4_attention_weight_min": -86.03922271728516,
      "activations/layer5_attention_weight_max": 136.13082885742188,
      "activations/layer5_attention_weight_min": -107.57445526123047,
      "activations/layer6_attention_weight_max": 64.9928207397461,
      "activations/layer6_attention_weight_min": -53.6089973449707,
      "activations/layer7_attention_weight_max": 70.33637237548828,
      "activations/layer7_attention_weight_min": -65.41481018066406,
      "activations/layer8_attention_weight_max": 63.72144317626953,
      "activations/layer8_attention_weight_min": -51.162288665771484,
      "activations/layer9_attention_weight_max": 79.83537292480469,
      "activations/layer9_attention_weight_min": -71.32616424560547,
      "epoch": 2.04,
      "learning_rate": 0.00013825681818181818,
      "loss": 2.9488,
      "step": 35050
    },
    {
      "activations/layer0_attention_weight_max": 14.566737174987793,
      "activations/layer0_attention_weight_min": -13.525619506835938,
      "activations/layer10_attention_weight_max": 59.714263916015625,
      "activations/layer10_attention_weight_min": -46.06640625,
      "activations/layer11_attention_weight_max": 42.049076080322266,
      "activations/layer11_attention_weight_min": -32.869773864746094,
      "activations/layer12_attention_weight_max": 27.446399688720703,
      "activations/layer12_attention_weight_min": -23.36240577697754,
      "activations/layer13_attention_weight_max": 63.8464469909668,
      "activations/layer13_attention_weight_min": -44.27361297607422,
      "activations/layer14_attention_weight_max": 52.2227783203125,
      "activations/layer14_attention_weight_min": -38.089447021484375,
      "activations/layer15_attention_weight_max": 54.36057662963867,
      "activations/layer15_attention_weight_min": -37.290714263916016,
      "activations/layer16_attention_weight_max": 43.17609786987305,
      "activations/layer16_attention_weight_min": -27.595932006835938,
      "activations/layer17_attention_weight_max": 66.02615356445312,
      "activations/layer17_attention_weight_min": -40.77738571166992,
      "activations/layer18_attention_weight_max": 69.69096374511719,
      "activations/layer18_attention_weight_min": -33.25639724731445,
      "activations/layer19_attention_weight_max": 22.249900817871094,
      "activations/layer19_attention_weight_min": -16.90544891357422,
      "activations/layer1_attention_weight_max": 15.730652809143066,
      "activations/layer1_attention_weight_min": -12.716409683227539,
      "activations/layer20_attention_weight_max": 22.202871322631836,
      "activations/layer20_attention_weight_min": -18.84249496459961,
      "activations/layer21_attention_weight_max": 41.88869094848633,
      "activations/layer21_attention_weight_min": -28.734731674194336,
      "activations/layer22_attention_weight_max": 34.51365280151367,
      "activations/layer22_attention_weight_min": -23.30548667907715,
      "activations/layer23_attention_weight_max": 23.412952423095703,
      "activations/layer23_attention_weight_min": -15.626241683959961,
      "activations/layer2_attention_weight_max": 26.94989013671875,
      "activations/layer2_attention_weight_min": -25.40105438232422,
      "activations/layer3_attention_weight_max": 60.03278350830078,
      "activations/layer3_attention_weight_min": -61.979103088378906,
      "activations/layer4_attention_weight_max": 93.126220703125,
      "activations/layer4_attention_weight_min": -82.84534454345703,
      "activations/layer5_attention_weight_max": 127.44368743896484,
      "activations/layer5_attention_weight_min": -106.37562561035156,
      "activations/layer6_attention_weight_max": 62.38042068481445,
      "activations/layer6_attention_weight_min": -52.66532516479492,
      "activations/layer7_attention_weight_max": 58.74631881713867,
      "activations/layer7_attention_weight_min": -52.42543411254883,
      "activations/layer8_attention_weight_max": 47.298851013183594,
      "activations/layer8_attention_weight_min": -40.000213623046875,
      "activations/layer9_attention_weight_max": 58.8369140625,
      "activations/layer9_attention_weight_min": -57.56965255737305,
      "epoch": 2.04,
      "learning_rate": 0.00013823787878787877,
      "loss": 2.9473,
      "step": 35100
    },
    {
      "activations/layer0_attention_weight_max": 14.515674591064453,
      "activations/layer0_attention_weight_min": -14.259905815124512,
      "activations/layer10_attention_weight_max": 57.84294891357422,
      "activations/layer10_attention_weight_min": -45.385101318359375,
      "activations/layer11_attention_weight_max": 41.97283935546875,
      "activations/layer11_attention_weight_min": -32.916595458984375,
      "activations/layer12_attention_weight_max": 28.993627548217773,
      "activations/layer12_attention_weight_min": -22.667667388916016,
      "activations/layer13_attention_weight_max": 72.31584167480469,
      "activations/layer13_attention_weight_min": -42.12286376953125,
      "activations/layer14_attention_weight_max": 77.80929565429688,
      "activations/layer14_attention_weight_min": -36.07724380493164,
      "activations/layer15_attention_weight_max": 59.990203857421875,
      "activations/layer15_attention_weight_min": -37.23305130004883,
      "activations/layer16_attention_weight_max": 47.25595474243164,
      "activations/layer16_attention_weight_min": -27.632793426513672,
      "activations/layer17_attention_weight_max": 69.29434967041016,
      "activations/layer17_attention_weight_min": -36.5844841003418,
      "activations/layer18_attention_weight_max": 70.52838134765625,
      "activations/layer18_attention_weight_min": -32.33001708984375,
      "activations/layer19_attention_weight_max": 25.47454071044922,
      "activations/layer19_attention_weight_min": -14.87322998046875,
      "activations/layer1_attention_weight_max": 16.284637451171875,
      "activations/layer1_attention_weight_min": -13.028160095214844,
      "activations/layer20_attention_weight_max": 25.31609535217285,
      "activations/layer20_attention_weight_min": -18.288097381591797,
      "activations/layer21_attention_weight_max": 39.961753845214844,
      "activations/layer21_attention_weight_min": -24.930326461791992,
      "activations/layer22_attention_weight_max": 37.34364318847656,
      "activations/layer22_attention_weight_min": -26.07456398010254,
      "activations/layer23_attention_weight_max": 23.758808135986328,
      "activations/layer23_attention_weight_min": -13.664020538330078,
      "activations/layer2_attention_weight_max": 26.79713249206543,
      "activations/layer2_attention_weight_min": -25.523334503173828,
      "activations/layer3_attention_weight_max": 61.610042572021484,
      "activations/layer3_attention_weight_min": -64.27131652832031,
      "activations/layer4_attention_weight_max": 95.52030181884766,
      "activations/layer4_attention_weight_min": -82.32206726074219,
      "activations/layer5_attention_weight_max": 136.934814453125,
      "activations/layer5_attention_weight_min": -108.45665740966797,
      "activations/layer6_attention_weight_max": 65.34752655029297,
      "activations/layer6_attention_weight_min": -56.02494812011719,
      "activations/layer7_attention_weight_max": 61.43010711669922,
      "activations/layer7_attention_weight_min": -56.64259719848633,
      "activations/layer8_attention_weight_max": 48.26555633544922,
      "activations/layer8_attention_weight_min": -45.350093841552734,
      "activations/layer9_attention_weight_max": 69.0235595703125,
      "activations/layer9_attention_weight_min": -55.86003494262695,
      "epoch": 2.04,
      "learning_rate": 0.00013821893939393938,
      "loss": 2.9707,
      "step": 35150
    },
    {
      "activations/layer0_attention_weight_max": 16.082475662231445,
      "activations/layer0_attention_weight_min": -13.522926330566406,
      "activations/layer10_attention_weight_max": 67.1766357421875,
      "activations/layer10_attention_weight_min": -49.184505462646484,
      "activations/layer11_attention_weight_max": 44.960140228271484,
      "activations/layer11_attention_weight_min": -32.9071044921875,
      "activations/layer12_attention_weight_max": 40.081878662109375,
      "activations/layer12_attention_weight_min": -27.225257873535156,
      "activations/layer13_attention_weight_max": 63.93324279785156,
      "activations/layer13_attention_weight_min": -42.83777618408203,
      "activations/layer14_attention_weight_max": 56.9134407043457,
      "activations/layer14_attention_weight_min": -35.47175216674805,
      "activations/layer15_attention_weight_max": 55.69919967651367,
      "activations/layer15_attention_weight_min": -36.725486755371094,
      "activations/layer16_attention_weight_max": 44.13534164428711,
      "activations/layer16_attention_weight_min": -27.471237182617188,
      "activations/layer17_attention_weight_max": 63.40504455566406,
      "activations/layer17_attention_weight_min": -34.588077545166016,
      "activations/layer18_attention_weight_max": 57.58597183227539,
      "activations/layer18_attention_weight_min": -32.336578369140625,
      "activations/layer19_attention_weight_max": 25.674036026000977,
      "activations/layer19_attention_weight_min": -15.680659294128418,
      "activations/layer1_attention_weight_max": 17.590930938720703,
      "activations/layer1_attention_weight_min": -12.5892915725708,
      "activations/layer20_attention_weight_max": 20.959104537963867,
      "activations/layer20_attention_weight_min": -16.697818756103516,
      "activations/layer21_attention_weight_max": 51.25871276855469,
      "activations/layer21_attention_weight_min": -24.97064208984375,
      "activations/layer22_attention_weight_max": 39.675716400146484,
      "activations/layer22_attention_weight_min": -21.603450775146484,
      "activations/layer23_attention_weight_max": 26.797927856445312,
      "activations/layer23_attention_weight_min": -13.891054153442383,
      "activations/layer2_attention_weight_max": 27.022083282470703,
      "activations/layer2_attention_weight_min": -25.658828735351562,
      "activations/layer3_attention_weight_max": 62.854591369628906,
      "activations/layer3_attention_weight_min": -65.15144348144531,
      "activations/layer4_attention_weight_max": 96.33397674560547,
      "activations/layer4_attention_weight_min": -87.8075180053711,
      "activations/layer5_attention_weight_max": 137.77578735351562,
      "activations/layer5_attention_weight_min": -107.36190795898438,
      "activations/layer6_attention_weight_max": 68.8335189819336,
      "activations/layer6_attention_weight_min": -56.12042999267578,
      "activations/layer7_attention_weight_max": 63.809539794921875,
      "activations/layer7_attention_weight_min": -54.5182991027832,
      "activations/layer8_attention_weight_max": 51.94426727294922,
      "activations/layer8_attention_weight_min": -42.49216842651367,
      "activations/layer9_attention_weight_max": 73.11260223388672,
      "activations/layer9_attention_weight_min": -60.436988830566406,
      "epoch": 2.05,
      "learning_rate": 0.0001382,
      "loss": 2.9618,
      "step": 35200
    },
    {
      "activations/layer0_attention_weight_max": 13.561739921569824,
      "activations/layer0_attention_weight_min": -14.438631057739258,
      "activations/layer10_attention_weight_max": 57.92204284667969,
      "activations/layer10_attention_weight_min": -49.55664825439453,
      "activations/layer11_attention_weight_max": 45.847564697265625,
      "activations/layer11_attention_weight_min": -34.635704040527344,
      "activations/layer12_attention_weight_max": 28.177112579345703,
      "activations/layer12_attention_weight_min": -25.672304153442383,
      "activations/layer13_attention_weight_max": 57.495994567871094,
      "activations/layer13_attention_weight_min": -40.723777770996094,
      "activations/layer14_attention_weight_max": 64.67350006103516,
      "activations/layer14_attention_weight_min": -41.77729034423828,
      "activations/layer15_attention_weight_max": 53.28276443481445,
      "activations/layer15_attention_weight_min": -40.41627502441406,
      "activations/layer16_attention_weight_max": 40.60966873168945,
      "activations/layer16_attention_weight_min": -30.996931076049805,
      "activations/layer17_attention_weight_max": 62.86762237548828,
      "activations/layer17_attention_weight_min": -35.15415954589844,
      "activations/layer18_attention_weight_max": 59.24710464477539,
      "activations/layer18_attention_weight_min": -33.4569091796875,
      "activations/layer19_attention_weight_max": 23.57863998413086,
      "activations/layer19_attention_weight_min": -14.76098346710205,
      "activations/layer1_attention_weight_max": 16.427207946777344,
      "activations/layer1_attention_weight_min": -12.960524559020996,
      "activations/layer20_attention_weight_max": 22.161243438720703,
      "activations/layer20_attention_weight_min": -16.842975616455078,
      "activations/layer21_attention_weight_max": 36.50041580200195,
      "activations/layer21_attention_weight_min": -26.596538543701172,
      "activations/layer22_attention_weight_max": 33.90431594848633,
      "activations/layer22_attention_weight_min": -22.046916961669922,
      "activations/layer23_attention_weight_max": 25.878089904785156,
      "activations/layer23_attention_weight_min": -16.765628814697266,
      "activations/layer2_attention_weight_max": 26.85895538330078,
      "activations/layer2_attention_weight_min": -27.034591674804688,
      "activations/layer3_attention_weight_max": 63.02156448364258,
      "activations/layer3_attention_weight_min": -64.56343841552734,
      "activations/layer4_attention_weight_max": 92.55332946777344,
      "activations/layer4_attention_weight_min": -79.8522720336914,
      "activations/layer5_attention_weight_max": 131.78086853027344,
      "activations/layer5_attention_weight_min": -104.96517944335938,
      "activations/layer6_attention_weight_max": 62.47836685180664,
      "activations/layer6_attention_weight_min": -52.191688537597656,
      "activations/layer7_attention_weight_max": 59.83295440673828,
      "activations/layer7_attention_weight_min": -53.49104309082031,
      "activations/layer8_attention_weight_max": 47.90034866333008,
      "activations/layer8_attention_weight_min": -39.38254165649414,
      "activations/layer9_attention_weight_max": 71.65582275390625,
      "activations/layer9_attention_weight_min": -57.51446533203125,
      "epoch": 2.05,
      "learning_rate": 0.0001381810606060606,
      "loss": 2.9535,
      "step": 35250
    },
    {
      "activations/layer0_attention_weight_max": 15.209468841552734,
      "activations/layer0_attention_weight_min": -13.812273025512695,
      "activations/layer10_attention_weight_max": 63.5606803894043,
      "activations/layer10_attention_weight_min": -46.301605224609375,
      "activations/layer11_attention_weight_max": 45.969947814941406,
      "activations/layer11_attention_weight_min": -36.893123626708984,
      "activations/layer12_attention_weight_max": 34.973426818847656,
      "activations/layer12_attention_weight_min": -24.945005416870117,
      "activations/layer13_attention_weight_max": 62.99830627441406,
      "activations/layer13_attention_weight_min": -45.14333724975586,
      "activations/layer14_attention_weight_max": 54.29637145996094,
      "activations/layer14_attention_weight_min": -33.68239974975586,
      "activations/layer15_attention_weight_max": 54.56581497192383,
      "activations/layer15_attention_weight_min": -37.397315979003906,
      "activations/layer16_attention_weight_max": 42.55174255371094,
      "activations/layer16_attention_weight_min": -28.84792709350586,
      "activations/layer17_attention_weight_max": 64.45804595947266,
      "activations/layer17_attention_weight_min": -36.817726135253906,
      "activations/layer18_attention_weight_max": 54.65251159667969,
      "activations/layer18_attention_weight_min": -34.146549224853516,
      "activations/layer19_attention_weight_max": 21.261011123657227,
      "activations/layer19_attention_weight_min": -15.577308654785156,
      "activations/layer1_attention_weight_max": 16.41299057006836,
      "activations/layer1_attention_weight_min": -16.34960174560547,
      "activations/layer20_attention_weight_max": 18.68864631652832,
      "activations/layer20_attention_weight_min": -16.15823745727539,
      "activations/layer21_attention_weight_max": 39.86174392700195,
      "activations/layer21_attention_weight_min": -25.479093551635742,
      "activations/layer22_attention_weight_max": 36.3809700012207,
      "activations/layer22_attention_weight_min": -21.88414192199707,
      "activations/layer23_attention_weight_max": 22.599645614624023,
      "activations/layer23_attention_weight_min": -17.164581298828125,
      "activations/layer2_attention_weight_max": 28.273244857788086,
      "activations/layer2_attention_weight_min": -27.080875396728516,
      "activations/layer3_attention_weight_max": 63.24013900756836,
      "activations/layer3_attention_weight_min": -64.67623901367188,
      "activations/layer4_attention_weight_max": 91.96717834472656,
      "activations/layer4_attention_weight_min": -84.26176452636719,
      "activations/layer5_attention_weight_max": 122.31333923339844,
      "activations/layer5_attention_weight_min": -107.57395935058594,
      "activations/layer6_attention_weight_max": 61.1129035949707,
      "activations/layer6_attention_weight_min": -51.94133377075195,
      "activations/layer7_attention_weight_max": 58.697364807128906,
      "activations/layer7_attention_weight_min": -56.07177734375,
      "activations/layer8_attention_weight_max": 45.424713134765625,
      "activations/layer8_attention_weight_min": -39.186859130859375,
      "activations/layer9_attention_weight_max": 64.09365844726562,
      "activations/layer9_attention_weight_min": -55.53797149658203,
      "epoch": 2.05,
      "learning_rate": 0.0001381621212121212,
      "loss": 2.9677,
      "step": 35300
    },
    {
      "activations/layer0_attention_weight_max": 14.246826171875,
      "activations/layer0_attention_weight_min": -13.955833435058594,
      "activations/layer10_attention_weight_max": 59.286766052246094,
      "activations/layer10_attention_weight_min": -45.0518913269043,
      "activations/layer11_attention_weight_max": 55.37745666503906,
      "activations/layer11_attention_weight_min": -34.007102966308594,
      "activations/layer12_attention_weight_max": 31.143192291259766,
      "activations/layer12_attention_weight_min": -26.439838409423828,
      "activations/layer13_attention_weight_max": 58.78238296508789,
      "activations/layer13_attention_weight_min": -44.49046325683594,
      "activations/layer14_attention_weight_max": 54.518436431884766,
      "activations/layer14_attention_weight_min": -39.526519775390625,
      "activations/layer15_attention_weight_max": 49.3860969543457,
      "activations/layer15_attention_weight_min": -36.29698181152344,
      "activations/layer16_attention_weight_max": 40.47780990600586,
      "activations/layer16_attention_weight_min": -27.816692352294922,
      "activations/layer17_attention_weight_max": 63.55926513671875,
      "activations/layer17_attention_weight_min": -37.80217742919922,
      "activations/layer18_attention_weight_max": 54.971473693847656,
      "activations/layer18_attention_weight_min": -33.483741760253906,
      "activations/layer19_attention_weight_max": 21.65333366394043,
      "activations/layer19_attention_weight_min": -16.537208557128906,
      "activations/layer1_attention_weight_max": 15.923822402954102,
      "activations/layer1_attention_weight_min": -13.883051872253418,
      "activations/layer20_attention_weight_max": 21.145252227783203,
      "activations/layer20_attention_weight_min": -15.702808380126953,
      "activations/layer21_attention_weight_max": 38.2912483215332,
      "activations/layer21_attention_weight_min": -23.245086669921875,
      "activations/layer22_attention_weight_max": 36.22654724121094,
      "activations/layer22_attention_weight_min": -21.853580474853516,
      "activations/layer23_attention_weight_max": 26.821523666381836,
      "activations/layer23_attention_weight_min": -15.863853454589844,
      "activations/layer2_attention_weight_max": 27.84650993347168,
      "activations/layer2_attention_weight_min": -27.033063888549805,
      "activations/layer3_attention_weight_max": 68.59324645996094,
      "activations/layer3_attention_weight_min": -70.47454833984375,
      "activations/layer4_attention_weight_max": 98.10575103759766,
      "activations/layer4_attention_weight_min": -89.52798461914062,
      "activations/layer5_attention_weight_max": 128.5867156982422,
      "activations/layer5_attention_weight_min": -109.8485107421875,
      "activations/layer6_attention_weight_max": 59.8120002746582,
      "activations/layer6_attention_weight_min": -55.126060485839844,
      "activations/layer7_attention_weight_max": 58.06392288208008,
      "activations/layer7_attention_weight_min": -58.15813446044922,
      "activations/layer8_attention_weight_max": 49.77443313598633,
      "activations/layer8_attention_weight_min": -39.30437088012695,
      "activations/layer9_attention_weight_max": 71.19386291503906,
      "activations/layer9_attention_weight_min": -58.40128707885742,
      "epoch": 2.05,
      "learning_rate": 0.00013814318181818182,
      "loss": 2.9532,
      "step": 35350
    },
    {
      "activations/layer0_attention_weight_max": 13.870234489440918,
      "activations/layer0_attention_weight_min": -13.69796085357666,
      "activations/layer10_attention_weight_max": 57.96604919433594,
      "activations/layer10_attention_weight_min": -43.028160095214844,
      "activations/layer11_attention_weight_max": 43.875457763671875,
      "activations/layer11_attention_weight_min": -32.17656707763672,
      "activations/layer12_attention_weight_max": 26.430875778198242,
      "activations/layer12_attention_weight_min": -23.911455154418945,
      "activations/layer13_attention_weight_max": 60.76209259033203,
      "activations/layer13_attention_weight_min": -38.95771026611328,
      "activations/layer14_attention_weight_max": 54.833763122558594,
      "activations/layer14_attention_weight_min": -35.83822250366211,
      "activations/layer15_attention_weight_max": 56.93564224243164,
      "activations/layer15_attention_weight_min": -36.87630081176758,
      "activations/layer16_attention_weight_max": 42.886905670166016,
      "activations/layer16_attention_weight_min": -29.1301212310791,
      "activations/layer17_attention_weight_max": 58.46207046508789,
      "activations/layer17_attention_weight_min": -37.56450271606445,
      "activations/layer18_attention_weight_max": 59.2371826171875,
      "activations/layer18_attention_weight_min": -33.15309143066406,
      "activations/layer19_attention_weight_max": 21.367456436157227,
      "activations/layer19_attention_weight_min": -16.192506790161133,
      "activations/layer1_attention_weight_max": 17.19558334350586,
      "activations/layer1_attention_weight_min": -12.868806838989258,
      "activations/layer20_attention_weight_max": 19.888011932373047,
      "activations/layer20_attention_weight_min": -18.48283576965332,
      "activations/layer21_attention_weight_max": 39.23476028442383,
      "activations/layer21_attention_weight_min": -24.21182632446289,
      "activations/layer22_attention_weight_max": 35.71222686767578,
      "activations/layer22_attention_weight_min": -21.94558334350586,
      "activations/layer23_attention_weight_max": 23.369152069091797,
      "activations/layer23_attention_weight_min": -15.946223258972168,
      "activations/layer2_attention_weight_max": 26.664653778076172,
      "activations/layer2_attention_weight_min": -26.523658752441406,
      "activations/layer3_attention_weight_max": 62.08338928222656,
      "activations/layer3_attention_weight_min": -64.9623794555664,
      "activations/layer4_attention_weight_max": 89.56256866455078,
      "activations/layer4_attention_weight_min": -80.18660736083984,
      "activations/layer5_attention_weight_max": 125.8504867553711,
      "activations/layer5_attention_weight_min": -100.77120971679688,
      "activations/layer6_attention_weight_max": 60.908267974853516,
      "activations/layer6_attention_weight_min": -51.335880279541016,
      "activations/layer7_attention_weight_max": 55.87890625,
      "activations/layer7_attention_weight_min": -53.146392822265625,
      "activations/layer8_attention_weight_max": 42.74954605102539,
      "activations/layer8_attention_weight_min": -38.78507614135742,
      "activations/layer9_attention_weight_max": 62.94800567626953,
      "activations/layer9_attention_weight_min": -55.3111457824707,
      "epoch": 2.06,
      "learning_rate": 0.0001381242424242424,
      "loss": 2.9551,
      "step": 35400
    },
    {
      "activations/layer0_attention_weight_max": 15.215006828308105,
      "activations/layer0_attention_weight_min": -13.92581558227539,
      "activations/layer10_attention_weight_max": 60.7368278503418,
      "activations/layer10_attention_weight_min": -48.8505973815918,
      "activations/layer11_attention_weight_max": 47.723175048828125,
      "activations/layer11_attention_weight_min": -36.21664047241211,
      "activations/layer12_attention_weight_max": 40.12605667114258,
      "activations/layer12_attention_weight_min": -26.195226669311523,
      "activations/layer13_attention_weight_max": 61.40037536621094,
      "activations/layer13_attention_weight_min": -44.22780990600586,
      "activations/layer14_attention_weight_max": 57.79842758178711,
      "activations/layer14_attention_weight_min": -43.798526763916016,
      "activations/layer15_attention_weight_max": 54.453529357910156,
      "activations/layer15_attention_weight_min": -36.427616119384766,
      "activations/layer16_attention_weight_max": 47.716026306152344,
      "activations/layer16_attention_weight_min": -31.100000381469727,
      "activations/layer17_attention_weight_max": 64.7634048461914,
      "activations/layer17_attention_weight_min": -35.16147994995117,
      "activations/layer18_attention_weight_max": 58.02878189086914,
      "activations/layer18_attention_weight_min": -32.589385986328125,
      "activations/layer19_attention_weight_max": 24.35295295715332,
      "activations/layer19_attention_weight_min": -17.203758239746094,
      "activations/layer1_attention_weight_max": 15.934187889099121,
      "activations/layer1_attention_weight_min": -13.614727973937988,
      "activations/layer20_attention_weight_max": 21.281620025634766,
      "activations/layer20_attention_weight_min": -18.01300048828125,
      "activations/layer21_attention_weight_max": 50.0013313293457,
      "activations/layer21_attention_weight_min": -25.295494079589844,
      "activations/layer22_attention_weight_max": 37.675350189208984,
      "activations/layer22_attention_weight_min": -21.37158966064453,
      "activations/layer23_attention_weight_max": 27.057348251342773,
      "activations/layer23_attention_weight_min": -16.27618408203125,
      "activations/layer2_attention_weight_max": 28.378311157226562,
      "activations/layer2_attention_weight_min": -27.794185638427734,
      "activations/layer3_attention_weight_max": 64.91285705566406,
      "activations/layer3_attention_weight_min": -66.8307876586914,
      "activations/layer4_attention_weight_max": 98.4460678100586,
      "activations/layer4_attention_weight_min": -86.02760314941406,
      "activations/layer5_attention_weight_max": 142.9310302734375,
      "activations/layer5_attention_weight_min": -110.60055541992188,
      "activations/layer6_attention_weight_max": 66.62218475341797,
      "activations/layer6_attention_weight_min": -55.72880172729492,
      "activations/layer7_attention_weight_max": 69.19867706298828,
      "activations/layer7_attention_weight_min": -54.21222686767578,
      "activations/layer8_attention_weight_max": 63.32320785522461,
      "activations/layer8_attention_weight_min": -47.526947021484375,
      "activations/layer9_attention_weight_max": 85.54827117919922,
      "activations/layer9_attention_weight_min": -64.37181854248047,
      "epoch": 2.06,
      "learning_rate": 0.00013810530303030302,
      "loss": 2.9511,
      "step": 35450
    },
    {
      "activations/layer0_attention_weight_max": 14.772231101989746,
      "activations/layer0_attention_weight_min": -13.892398834228516,
      "activations/layer10_attention_weight_max": 53.72258758544922,
      "activations/layer10_attention_weight_min": -43.23398971557617,
      "activations/layer11_attention_weight_max": 38.73903274536133,
      "activations/layer11_attention_weight_min": -34.619075775146484,
      "activations/layer12_attention_weight_max": 26.14059066772461,
      "activations/layer12_attention_weight_min": -27.494264602661133,
      "activations/layer13_attention_weight_max": 60.3197135925293,
      "activations/layer13_attention_weight_min": -39.018585205078125,
      "activations/layer14_attention_weight_max": 38.81031799316406,
      "activations/layer14_attention_weight_min": -32.108951568603516,
      "activations/layer15_attention_weight_max": 47.435302734375,
      "activations/layer15_attention_weight_min": -35.233646392822266,
      "activations/layer16_attention_weight_max": 41.08009338378906,
      "activations/layer16_attention_weight_min": -27.79322052001953,
      "activations/layer17_attention_weight_max": 67.03826141357422,
      "activations/layer17_attention_weight_min": -39.81864547729492,
      "activations/layer18_attention_weight_max": 59.83938217163086,
      "activations/layer18_attention_weight_min": -33.23603439331055,
      "activations/layer19_attention_weight_max": 21.653318405151367,
      "activations/layer19_attention_weight_min": -14.684196472167969,
      "activations/layer1_attention_weight_max": 15.742257118225098,
      "activations/layer1_attention_weight_min": -14.162763595581055,
      "activations/layer20_attention_weight_max": 16.65025520324707,
      "activations/layer20_attention_weight_min": -18.972675323486328,
      "activations/layer21_attention_weight_max": 38.12885284423828,
      "activations/layer21_attention_weight_min": -27.323928833007812,
      "activations/layer22_attention_weight_max": 36.45792007446289,
      "activations/layer22_attention_weight_min": -23.253925323486328,
      "activations/layer23_attention_weight_max": 23.435192108154297,
      "activations/layer23_attention_weight_min": -14.0755615234375,
      "activations/layer2_attention_weight_max": 26.713199615478516,
      "activations/layer2_attention_weight_min": -25.04554557800293,
      "activations/layer3_attention_weight_max": 61.74016189575195,
      "activations/layer3_attention_weight_min": -65.72360229492188,
      "activations/layer4_attention_weight_max": 92.75898742675781,
      "activations/layer4_attention_weight_min": -83.7735824584961,
      "activations/layer5_attention_weight_max": 131.070068359375,
      "activations/layer5_attention_weight_min": -109.27568054199219,
      "activations/layer6_attention_weight_max": 61.10245132446289,
      "activations/layer6_attention_weight_min": -54.039161682128906,
      "activations/layer7_attention_weight_max": 60.097389221191406,
      "activations/layer7_attention_weight_min": -56.60386657714844,
      "activations/layer8_attention_weight_max": 44.74369430541992,
      "activations/layer8_attention_weight_min": -38.514469146728516,
      "activations/layer9_attention_weight_max": 61.07332992553711,
      "activations/layer9_attention_weight_min": -62.27366256713867,
      "epoch": 2.06,
      "learning_rate": 0.00013808636363636363,
      "loss": 2.9545,
      "step": 35500
    },
    {
      "activations/layer0_attention_weight_max": 14.053139686584473,
      "activations/layer0_attention_weight_min": -14.43882942199707,
      "activations/layer10_attention_weight_max": 62.752464294433594,
      "activations/layer10_attention_weight_min": -51.14139175415039,
      "activations/layer11_attention_weight_max": 39.51118469238281,
      "activations/layer11_attention_weight_min": -36.73651123046875,
      "activations/layer12_attention_weight_max": 31.71405792236328,
      "activations/layer12_attention_weight_min": -28.34923553466797,
      "activations/layer13_attention_weight_max": 55.39688491821289,
      "activations/layer13_attention_weight_min": -39.291297912597656,
      "activations/layer14_attention_weight_max": 54.43575668334961,
      "activations/layer14_attention_weight_min": -37.19164276123047,
      "activations/layer15_attention_weight_max": 56.23759460449219,
      "activations/layer15_attention_weight_min": -39.64126205444336,
      "activations/layer16_attention_weight_max": 41.09158706665039,
      "activations/layer16_attention_weight_min": -27.957731246948242,
      "activations/layer17_attention_weight_max": 74.00244140625,
      "activations/layer17_attention_weight_min": -43.81389236450195,
      "activations/layer18_attention_weight_max": 55.90892791748047,
      "activations/layer18_attention_weight_min": -32.5673713684082,
      "activations/layer19_attention_weight_max": 23.581539154052734,
      "activations/layer19_attention_weight_min": -16.942232131958008,
      "activations/layer1_attention_weight_max": 17.541622161865234,
      "activations/layer1_attention_weight_min": -12.473934173583984,
      "activations/layer20_attention_weight_max": 20.57644271850586,
      "activations/layer20_attention_weight_min": -18.653053283691406,
      "activations/layer21_attention_weight_max": 42.08855056762695,
      "activations/layer21_attention_weight_min": -22.598175048828125,
      "activations/layer22_attention_weight_max": 39.60586166381836,
      "activations/layer22_attention_weight_min": -20.698381423950195,
      "activations/layer23_attention_weight_max": 25.832698822021484,
      "activations/layer23_attention_weight_min": -12.344108581542969,
      "activations/layer2_attention_weight_max": 28.04273796081543,
      "activations/layer2_attention_weight_min": -27.29792594909668,
      "activations/layer3_attention_weight_max": 65.00125122070312,
      "activations/layer3_attention_weight_min": -68.98424530029297,
      "activations/layer4_attention_weight_max": 97.32328796386719,
      "activations/layer4_attention_weight_min": -86.91593933105469,
      "activations/layer5_attention_weight_max": 129.82992553710938,
      "activations/layer5_attention_weight_min": -118.8538818359375,
      "activations/layer6_attention_weight_max": 62.29573440551758,
      "activations/layer6_attention_weight_min": -57.408512115478516,
      "activations/layer7_attention_weight_max": 62.84864807128906,
      "activations/layer7_attention_weight_min": -54.494937896728516,
      "activations/layer8_attention_weight_max": 47.160118103027344,
      "activations/layer8_attention_weight_min": -39.177974700927734,
      "activations/layer9_attention_weight_max": 73.1583023071289,
      "activations/layer9_attention_weight_min": -67.53411102294922,
      "epoch": 2.07,
      "learning_rate": 0.00013806742424242422,
      "loss": 2.9462,
      "step": 35550
    },
    {
      "activations/layer0_attention_weight_max": 13.934597969055176,
      "activations/layer0_attention_weight_min": -14.108235359191895,
      "activations/layer10_attention_weight_max": 55.67569351196289,
      "activations/layer10_attention_weight_min": -46.97379684448242,
      "activations/layer11_attention_weight_max": 37.88917541503906,
      "activations/layer11_attention_weight_min": -34.39625549316406,
      "activations/layer12_attention_weight_max": 25.327362060546875,
      "activations/layer12_attention_weight_min": -23.879343032836914,
      "activations/layer13_attention_weight_max": 51.44429397583008,
      "activations/layer13_attention_weight_min": -40.51295471191406,
      "activations/layer14_attention_weight_max": 51.471397399902344,
      "activations/layer14_attention_weight_min": -41.506072998046875,
      "activations/layer15_attention_weight_max": 47.2347412109375,
      "activations/layer15_attention_weight_min": -37.427860260009766,
      "activations/layer16_attention_weight_max": 42.91234588623047,
      "activations/layer16_attention_weight_min": -29.309221267700195,
      "activations/layer17_attention_weight_max": 61.06938552856445,
      "activations/layer17_attention_weight_min": -38.28045654296875,
      "activations/layer18_attention_weight_max": 58.16985321044922,
      "activations/layer18_attention_weight_min": -34.28121566772461,
      "activations/layer19_attention_weight_max": 19.808090209960938,
      "activations/layer19_attention_weight_min": -14.966765403747559,
      "activations/layer1_attention_weight_max": 16.838367462158203,
      "activations/layer1_attention_weight_min": -13.700798988342285,
      "activations/layer20_attention_weight_max": 19.614036560058594,
      "activations/layer20_attention_weight_min": -19.750404357910156,
      "activations/layer21_attention_weight_max": 36.36171340942383,
      "activations/layer21_attention_weight_min": -23.953088760375977,
      "activations/layer22_attention_weight_max": 40.178001403808594,
      "activations/layer22_attention_weight_min": -24.624542236328125,
      "activations/layer23_attention_weight_max": 22.344566345214844,
      "activations/layer23_attention_weight_min": -14.749717712402344,
      "activations/layer2_attention_weight_max": 27.10749626159668,
      "activations/layer2_attention_weight_min": -26.145301818847656,
      "activations/layer3_attention_weight_max": 61.91890335083008,
      "activations/layer3_attention_weight_min": -66.8675308227539,
      "activations/layer4_attention_weight_max": 92.8278579711914,
      "activations/layer4_attention_weight_min": -81.7582778930664,
      "activations/layer5_attention_weight_max": 125.93040466308594,
      "activations/layer5_attention_weight_min": -104.33782196044922,
      "activations/layer6_attention_weight_max": 59.3187255859375,
      "activations/layer6_attention_weight_min": -53.04935836791992,
      "activations/layer7_attention_weight_max": 56.72352981567383,
      "activations/layer7_attention_weight_min": -57.03865051269531,
      "activations/layer8_attention_weight_max": 42.1881103515625,
      "activations/layer8_attention_weight_min": -37.99293899536133,
      "activations/layer9_attention_weight_max": 63.6020393371582,
      "activations/layer9_attention_weight_min": -60.58563995361328,
      "epoch": 2.07,
      "learning_rate": 0.00013804848484848484,
      "loss": 2.9486,
      "step": 35600
    },
    {
      "activations/layer0_attention_weight_max": 13.991394996643066,
      "activations/layer0_attention_weight_min": -13.878873825073242,
      "activations/layer10_attention_weight_max": 57.55671310424805,
      "activations/layer10_attention_weight_min": -45.24673843383789,
      "activations/layer11_attention_weight_max": 44.833404541015625,
      "activations/layer11_attention_weight_min": -33.61128234863281,
      "activations/layer12_attention_weight_max": 33.584136962890625,
      "activations/layer12_attention_weight_min": -26.968698501586914,
      "activations/layer13_attention_weight_max": 81.06593322753906,
      "activations/layer13_attention_weight_min": -49.08576583862305,
      "activations/layer14_attention_weight_max": 76.03791809082031,
      "activations/layer14_attention_weight_min": -40.48067092895508,
      "activations/layer15_attention_weight_max": 55.793758392333984,
      "activations/layer15_attention_weight_min": -36.005767822265625,
      "activations/layer16_attention_weight_max": 42.751930236816406,
      "activations/layer16_attention_weight_min": -26.959239959716797,
      "activations/layer17_attention_weight_max": 62.7321662902832,
      "activations/layer17_attention_weight_min": -35.84577941894531,
      "activations/layer18_attention_weight_max": 67.94042205810547,
      "activations/layer18_attention_weight_min": -34.88990020751953,
      "activations/layer19_attention_weight_max": 24.22188377380371,
      "activations/layer19_attention_weight_min": -16.48219108581543,
      "activations/layer1_attention_weight_max": 17.293333053588867,
      "activations/layer1_attention_weight_min": -12.7976655960083,
      "activations/layer20_attention_weight_max": 19.81653594970703,
      "activations/layer20_attention_weight_min": -18.263826370239258,
      "activations/layer21_attention_weight_max": 47.05032730102539,
      "activations/layer21_attention_weight_min": -25.8786678314209,
      "activations/layer22_attention_weight_max": 37.73763656616211,
      "activations/layer22_attention_weight_min": -20.410703659057617,
      "activations/layer23_attention_weight_max": 24.290754318237305,
      "activations/layer23_attention_weight_min": -15.87479305267334,
      "activations/layer2_attention_weight_max": 26.8106746673584,
      "activations/layer2_attention_weight_min": -27.438695907592773,
      "activations/layer3_attention_weight_max": 61.70513153076172,
      "activations/layer3_attention_weight_min": -67.40548706054688,
      "activations/layer4_attention_weight_max": 94.78305053710938,
      "activations/layer4_attention_weight_min": -81.95894622802734,
      "activations/layer5_attention_weight_max": 133.35986328125,
      "activations/layer5_attention_weight_min": -104.21568298339844,
      "activations/layer6_attention_weight_max": 64.1624984741211,
      "activations/layer6_attention_weight_min": -52.39604568481445,
      "activations/layer7_attention_weight_max": 61.90305709838867,
      "activations/layer7_attention_weight_min": -57.13606262207031,
      "activations/layer8_attention_weight_max": 48.089622497558594,
      "activations/layer8_attention_weight_min": -41.7947998046875,
      "activations/layer9_attention_weight_max": 62.2240104675293,
      "activations/layer9_attention_weight_min": -55.263343811035156,
      "epoch": 2.07,
      "learning_rate": 0.00013802954545454545,
      "loss": 2.9671,
      "step": 35650
    },
    {
      "activations/layer0_attention_weight_max": 14.481230735778809,
      "activations/layer0_attention_weight_min": -11.485102653503418,
      "activations/layer10_attention_weight_max": 51.69028091430664,
      "activations/layer10_attention_weight_min": -50.112911224365234,
      "activations/layer11_attention_weight_max": 44.135398864746094,
      "activations/layer11_attention_weight_min": -43.95048522949219,
      "activations/layer12_attention_weight_max": 29.91156578063965,
      "activations/layer12_attention_weight_min": -23.554758071899414,
      "activations/layer13_attention_weight_max": 64.6222152709961,
      "activations/layer13_attention_weight_min": -40.636009216308594,
      "activations/layer14_attention_weight_max": 54.57583236694336,
      "activations/layer14_attention_weight_min": -37.58315658569336,
      "activations/layer15_attention_weight_max": 62.410953521728516,
      "activations/layer15_attention_weight_min": -42.57561492919922,
      "activations/layer16_attention_weight_max": 46.5800895690918,
      "activations/layer16_attention_weight_min": -30.15301513671875,
      "activations/layer17_attention_weight_max": 61.04071807861328,
      "activations/layer17_attention_weight_min": -43.60841751098633,
      "activations/layer18_attention_weight_max": 63.09457778930664,
      "activations/layer18_attention_weight_min": -35.98512649536133,
      "activations/layer19_attention_weight_max": 21.08180809020996,
      "activations/layer19_attention_weight_min": -16.641521453857422,
      "activations/layer1_attention_weight_max": 21.967201232910156,
      "activations/layer1_attention_weight_min": -18.52309799194336,
      "activations/layer20_attention_weight_max": 23.82195281982422,
      "activations/layer20_attention_weight_min": -18.838516235351562,
      "activations/layer21_attention_weight_max": 39.99645233154297,
      "activations/layer21_attention_weight_min": -26.923391342163086,
      "activations/layer22_attention_weight_max": 35.88895034790039,
      "activations/layer22_attention_weight_min": -24.379331588745117,
      "activations/layer23_attention_weight_max": 28.815969467163086,
      "activations/layer23_attention_weight_min": -16.903770446777344,
      "activations/layer2_attention_weight_max": 26.61871337890625,
      "activations/layer2_attention_weight_min": -27.641836166381836,
      "activations/layer3_attention_weight_max": 61.36302947998047,
      "activations/layer3_attention_weight_min": -59.625667572021484,
      "activations/layer4_attention_weight_max": 84.35057067871094,
      "activations/layer4_attention_weight_min": -71.07836151123047,
      "activations/layer5_attention_weight_max": 145.65907287597656,
      "activations/layer5_attention_weight_min": -114.81065368652344,
      "activations/layer6_attention_weight_max": 72.13736724853516,
      "activations/layer6_attention_weight_min": -63.585723876953125,
      "activations/layer7_attention_weight_max": 65.33029174804688,
      "activations/layer7_attention_weight_min": -67.4439926147461,
      "activations/layer8_attention_weight_max": 49.44990921020508,
      "activations/layer8_attention_weight_min": -50.795352935791016,
      "activations/layer9_attention_weight_max": 49.7550048828125,
      "activations/layer9_attention_weight_min": -62.557003021240234,
      "epoch": 2.07,
      "learning_rate": 0.0001380121212121212,
      "loss": 3.1618,
      "step": 35700
    },
    {
      "activations/layer0_attention_weight_max": 15.373350143432617,
      "activations/layer0_attention_weight_min": -11.295166015625,
      "activations/layer10_attention_weight_max": 59.7540283203125,
      "activations/layer10_attention_weight_min": -49.21238327026367,
      "activations/layer11_attention_weight_max": 46.956092834472656,
      "activations/layer11_attention_weight_min": -33.61138916015625,
      "activations/layer12_attention_weight_max": 31.54623794555664,
      "activations/layer12_attention_weight_min": -23.63197135925293,
      "activations/layer13_attention_weight_max": 70.34939575195312,
      "activations/layer13_attention_weight_min": -43.27497863769531,
      "activations/layer14_attention_weight_max": 55.52643585205078,
      "activations/layer14_attention_weight_min": -35.59109115600586,
      "activations/layer15_attention_weight_max": 56.140228271484375,
      "activations/layer15_attention_weight_min": -37.676055908203125,
      "activations/layer16_attention_weight_max": 42.64912796020508,
      "activations/layer16_attention_weight_min": -30.80464744567871,
      "activations/layer17_attention_weight_max": 62.51248550415039,
      "activations/layer17_attention_weight_min": -38.51256561279297,
      "activations/layer18_attention_weight_max": 64.46453094482422,
      "activations/layer18_attention_weight_min": -33.32144546508789,
      "activations/layer19_attention_weight_max": 25.113994598388672,
      "activations/layer19_attention_weight_min": -16.567808151245117,
      "activations/layer1_attention_weight_max": 20.153247833251953,
      "activations/layer1_attention_weight_min": -15.271519660949707,
      "activations/layer20_attention_weight_max": 21.768138885498047,
      "activations/layer20_attention_weight_min": -13.34126091003418,
      "activations/layer21_attention_weight_max": 50.11273956298828,
      "activations/layer21_attention_weight_min": -25.362211227416992,
      "activations/layer22_attention_weight_max": 36.28095626831055,
      "activations/layer22_attention_weight_min": -21.836902618408203,
      "activations/layer23_attention_weight_max": 24.738048553466797,
      "activations/layer23_attention_weight_min": -13.7862548828125,
      "activations/layer2_attention_weight_max": 24.377208709716797,
      "activations/layer2_attention_weight_min": -21.81043243408203,
      "activations/layer3_attention_weight_max": 53.53696823120117,
      "activations/layer3_attention_weight_min": -56.404117584228516,
      "activations/layer4_attention_weight_max": 80.663818359375,
      "activations/layer4_attention_weight_min": -73.07816314697266,
      "activations/layer5_attention_weight_max": 116.33737182617188,
      "activations/layer5_attention_weight_min": -99.21539306640625,
      "activations/layer6_attention_weight_max": 56.54800796508789,
      "activations/layer6_attention_weight_min": -51.94083786010742,
      "activations/layer7_attention_weight_max": 54.470611572265625,
      "activations/layer7_attention_weight_min": -57.01261520385742,
      "activations/layer8_attention_weight_max": 43.065311431884766,
      "activations/layer8_attention_weight_min": -36.83948516845703,
      "activations/layer9_attention_weight_max": 57.881744384765625,
      "activations/layer9_attention_weight_min": -56.16845703125,
      "epoch": 2.08,
      "learning_rate": 0.0001379931818181818,
      "loss": 3.0626,
      "step": 35750
    },
    {
      "activations/layer0_attention_weight_max": 14.741151809692383,
      "activations/layer0_attention_weight_min": -11.180558204650879,
      "activations/layer10_attention_weight_max": 54.65519332885742,
      "activations/layer10_attention_weight_min": -47.266475677490234,
      "activations/layer11_attention_weight_max": 39.222862243652344,
      "activations/layer11_attention_weight_min": -34.200653076171875,
      "activations/layer12_attention_weight_max": 24.833145141601562,
      "activations/layer12_attention_weight_min": -24.497329711914062,
      "activations/layer13_attention_weight_max": 61.60279083251953,
      "activations/layer13_attention_weight_min": -48.02227020263672,
      "activations/layer14_attention_weight_max": 48.878387451171875,
      "activations/layer14_attention_weight_min": -39.64335632324219,
      "activations/layer15_attention_weight_max": 56.62800598144531,
      "activations/layer15_attention_weight_min": -37.30045700073242,
      "activations/layer16_attention_weight_max": 45.67273712158203,
      "activations/layer16_attention_weight_min": -31.16590118408203,
      "activations/layer17_attention_weight_max": 63.3693733215332,
      "activations/layer17_attention_weight_min": -37.34529495239258,
      "activations/layer18_attention_weight_max": 57.11018371582031,
      "activations/layer18_attention_weight_min": -31.991228103637695,
      "activations/layer19_attention_weight_max": 22.400632858276367,
      "activations/layer19_attention_weight_min": -16.85283088684082,
      "activations/layer1_attention_weight_max": 16.986135482788086,
      "activations/layer1_attention_weight_min": -14.941306114196777,
      "activations/layer20_attention_weight_max": 19.61847686767578,
      "activations/layer20_attention_weight_min": -18.368812561035156,
      "activations/layer21_attention_weight_max": 45.73824691772461,
      "activations/layer21_attention_weight_min": -24.944236755371094,
      "activations/layer22_attention_weight_max": 38.24530792236328,
      "activations/layer22_attention_weight_min": -21.967132568359375,
      "activations/layer23_attention_weight_max": 28.676467895507812,
      "activations/layer23_attention_weight_min": -15.787002563476562,
      "activations/layer2_attention_weight_max": 26.07216453552246,
      "activations/layer2_attention_weight_min": -22.80194854736328,
      "activations/layer3_attention_weight_max": 53.33686065673828,
      "activations/layer3_attention_weight_min": -56.71317672729492,
      "activations/layer4_attention_weight_max": 79.48262786865234,
      "activations/layer4_attention_weight_min": -73.27250671386719,
      "activations/layer5_attention_weight_max": 109.16667938232422,
      "activations/layer5_attention_weight_min": -92.7049789428711,
      "activations/layer6_attention_weight_max": 58.4200439453125,
      "activations/layer6_attention_weight_min": -46.89606857299805,
      "activations/layer7_attention_weight_max": 54.34189987182617,
      "activations/layer7_attention_weight_min": -51.281070709228516,
      "activations/layer8_attention_weight_max": 43.52836227416992,
      "activations/layer8_attention_weight_min": -38.83261489868164,
      "activations/layer9_attention_weight_max": 58.7189826965332,
      "activations/layer9_attention_weight_min": -55.98968505859375,
      "epoch": 2.08,
      "learning_rate": 0.0001379742424242424,
      "loss": 2.9677,
      "step": 35800
    },
    {
      "activations/layer0_attention_weight_max": 14.31587028503418,
      "activations/layer0_attention_weight_min": -11.733519554138184,
      "activations/layer10_attention_weight_max": 58.12083435058594,
      "activations/layer10_attention_weight_min": -44.42631912231445,
      "activations/layer11_attention_weight_max": 40.16284942626953,
      "activations/layer11_attention_weight_min": -33.64671325683594,
      "activations/layer12_attention_weight_max": 27.62592887878418,
      "activations/layer12_attention_weight_min": -26.434791564941406,
      "activations/layer13_attention_weight_max": 75.3337631225586,
      "activations/layer13_attention_weight_min": -47.556297302246094,
      "activations/layer14_attention_weight_max": 66.8670883178711,
      "activations/layer14_attention_weight_min": -46.53047561645508,
      "activations/layer15_attention_weight_max": 64.42391204833984,
      "activations/layer15_attention_weight_min": -41.3373908996582,
      "activations/layer16_attention_weight_max": 51.804508209228516,
      "activations/layer16_attention_weight_min": -30.194047927856445,
      "activations/layer17_attention_weight_max": 82.18592071533203,
      "activations/layer17_attention_weight_min": -38.477115631103516,
      "activations/layer18_attention_weight_max": 68.60264587402344,
      "activations/layer18_attention_weight_min": -32.20180892944336,
      "activations/layer19_attention_weight_max": 25.106992721557617,
      "activations/layer19_attention_weight_min": -15.818852424621582,
      "activations/layer1_attention_weight_max": 17.025272369384766,
      "activations/layer1_attention_weight_min": -14.63079833984375,
      "activations/layer20_attention_weight_max": 24.561227798461914,
      "activations/layer20_attention_weight_min": -17.38524055480957,
      "activations/layer21_attention_weight_max": 41.00559997558594,
      "activations/layer21_attention_weight_min": -24.291488647460938,
      "activations/layer22_attention_weight_max": 44.90788650512695,
      "activations/layer22_attention_weight_min": -22.426002502441406,
      "activations/layer23_attention_weight_max": 23.813007354736328,
      "activations/layer23_attention_weight_min": -13.879081726074219,
      "activations/layer2_attention_weight_max": 26.50225830078125,
      "activations/layer2_attention_weight_min": -23.14565086364746,
      "activations/layer3_attention_weight_max": 58.14128875732422,
      "activations/layer3_attention_weight_min": -59.59956741333008,
      "activations/layer4_attention_weight_max": 85.88362121582031,
      "activations/layer4_attention_weight_min": -84.45575714111328,
      "activations/layer5_attention_weight_max": 121.42706298828125,
      "activations/layer5_attention_weight_min": -98.037841796875,
      "activations/layer6_attention_weight_max": 59.34079360961914,
      "activations/layer6_attention_weight_min": -48.5438117980957,
      "activations/layer7_attention_weight_max": 53.6258659362793,
      "activations/layer7_attention_weight_min": -53.64095687866211,
      "activations/layer8_attention_weight_max": 47.2164306640625,
      "activations/layer8_attention_weight_min": -40.70646286010742,
      "activations/layer9_attention_weight_max": 60.72745895385742,
      "activations/layer9_attention_weight_min": -56.036048889160156,
      "epoch": 2.08,
      "learning_rate": 0.00013795530303030302,
      "loss": 2.9792,
      "step": 35850
    },
    {
      "activations/layer0_attention_weight_max": 15.158129692077637,
      "activations/layer0_attention_weight_min": -11.196348190307617,
      "activations/layer10_attention_weight_max": 58.21979904174805,
      "activations/layer10_attention_weight_min": -47.32733917236328,
      "activations/layer11_attention_weight_max": 42.05803680419922,
      "activations/layer11_attention_weight_min": -34.43149948120117,
      "activations/layer12_attention_weight_max": 33.98329544067383,
      "activations/layer12_attention_weight_min": -27.590330123901367,
      "activations/layer13_attention_weight_max": 70.11676788330078,
      "activations/layer13_attention_weight_min": -53.10670852661133,
      "activations/layer14_attention_weight_max": 48.38789749145508,
      "activations/layer14_attention_weight_min": -35.944091796875,
      "activations/layer15_attention_weight_max": 63.194332122802734,
      "activations/layer15_attention_weight_min": -43.086204528808594,
      "activations/layer16_attention_weight_max": 41.13669204711914,
      "activations/layer16_attention_weight_min": -28.70176124572754,
      "activations/layer17_attention_weight_max": 61.56955337524414,
      "activations/layer17_attention_weight_min": -37.98845672607422,
      "activations/layer18_attention_weight_max": 59.83218002319336,
      "activations/layer18_attention_weight_min": -31.1370792388916,
      "activations/layer19_attention_weight_max": 20.212730407714844,
      "activations/layer19_attention_weight_min": -14.770649909973145,
      "activations/layer1_attention_weight_max": 15.765130996704102,
      "activations/layer1_attention_weight_min": -14.967867851257324,
      "activations/layer20_attention_weight_max": 21.102039337158203,
      "activations/layer20_attention_weight_min": -16.66048240661621,
      "activations/layer21_attention_weight_max": 42.3390007019043,
      "activations/layer21_attention_weight_min": -24.718469619750977,
      "activations/layer22_attention_weight_max": 33.81834030151367,
      "activations/layer22_attention_weight_min": -23.377178192138672,
      "activations/layer23_attention_weight_max": 24.37860870361328,
      "activations/layer23_attention_weight_min": -15.001171112060547,
      "activations/layer2_attention_weight_max": 26.655418395996094,
      "activations/layer2_attention_weight_min": -22.62415313720703,
      "activations/layer3_attention_weight_max": 55.322837829589844,
      "activations/layer3_attention_weight_min": -58.899295806884766,
      "activations/layer4_attention_weight_max": 84.35035705566406,
      "activations/layer4_attention_weight_min": -78.89888763427734,
      "activations/layer5_attention_weight_max": 117.98652648925781,
      "activations/layer5_attention_weight_min": -96.18822479248047,
      "activations/layer6_attention_weight_max": 57.21659469604492,
      "activations/layer6_attention_weight_min": -49.78641128540039,
      "activations/layer7_attention_weight_max": 65.9057388305664,
      "activations/layer7_attention_weight_min": -54.07025909423828,
      "activations/layer8_attention_weight_max": 49.12327194213867,
      "activations/layer8_attention_weight_min": -38.516361236572266,
      "activations/layer9_attention_weight_max": 78.49726867675781,
      "activations/layer9_attention_weight_min": -61.32371139526367,
      "epoch": 2.09,
      "learning_rate": 0.00013793636363636363,
      "loss": 2.9521,
      "step": 35900
    },
    {
      "activations/layer0_attention_weight_max": 13.947452545166016,
      "activations/layer0_attention_weight_min": -10.712059020996094,
      "activations/layer10_attention_weight_max": 54.82487487792969,
      "activations/layer10_attention_weight_min": -46.526611328125,
      "activations/layer11_attention_weight_max": 39.988861083984375,
      "activations/layer11_attention_weight_min": -33.280818939208984,
      "activations/layer12_attention_weight_max": 29.189289093017578,
      "activations/layer12_attention_weight_min": -25.07097816467285,
      "activations/layer13_attention_weight_max": 52.48915481567383,
      "activations/layer13_attention_weight_min": -38.645111083984375,
      "activations/layer14_attention_weight_max": 71.27766418457031,
      "activations/layer14_attention_weight_min": -45.07378005981445,
      "activations/layer15_attention_weight_max": 47.702117919921875,
      "activations/layer15_attention_weight_min": -37.043983459472656,
      "activations/layer16_attention_weight_max": 45.17693328857422,
      "activations/layer16_attention_weight_min": -27.125507354736328,
      "activations/layer17_attention_weight_max": 62.75676345825195,
      "activations/layer17_attention_weight_min": -39.29275131225586,
      "activations/layer18_attention_weight_max": 62.66413879394531,
      "activations/layer18_attention_weight_min": -34.15549087524414,
      "activations/layer19_attention_weight_max": 22.184246063232422,
      "activations/layer19_attention_weight_min": -17.01051902770996,
      "activations/layer1_attention_weight_max": 15.859087944030762,
      "activations/layer1_attention_weight_min": -13.170209884643555,
      "activations/layer20_attention_weight_max": 22.6170654296875,
      "activations/layer20_attention_weight_min": -20.503341674804688,
      "activations/layer21_attention_weight_max": 42.658687591552734,
      "activations/layer21_attention_weight_min": -24.1981201171875,
      "activations/layer22_attention_weight_max": 34.75446319580078,
      "activations/layer22_attention_weight_min": -20.554643630981445,
      "activations/layer23_attention_weight_max": 23.830732345581055,
      "activations/layer23_attention_weight_min": -15.708704948425293,
      "activations/layer2_attention_weight_max": 27.26021385192871,
      "activations/layer2_attention_weight_min": -23.505781173706055,
      "activations/layer3_attention_weight_max": 57.1686897277832,
      "activations/layer3_attention_weight_min": -65.26531219482422,
      "activations/layer4_attention_weight_max": 85.08094024658203,
      "activations/layer4_attention_weight_min": -83.41382598876953,
      "activations/layer5_attention_weight_max": 117.59523010253906,
      "activations/layer5_attention_weight_min": -98.99786376953125,
      "activations/layer6_attention_weight_max": 57.054908752441406,
      "activations/layer6_attention_weight_min": -50.40177536010742,
      "activations/layer7_attention_weight_max": 53.27816390991211,
      "activations/layer7_attention_weight_min": -48.55261993408203,
      "activations/layer8_attention_weight_max": 44.14759826660156,
      "activations/layer8_attention_weight_min": -38.13129806518555,
      "activations/layer9_attention_weight_max": 62.437435150146484,
      "activations/layer9_attention_weight_min": -53.67550277709961,
      "epoch": 2.09,
      "learning_rate": 0.00013791742424242422,
      "loss": 2.9553,
      "step": 35950
    },
    {
      "activations/layer0_attention_weight_max": 13.505230903625488,
      "activations/layer0_attention_weight_min": -11.578940391540527,
      "activations/layer10_attention_weight_max": 56.31119918823242,
      "activations/layer10_attention_weight_min": -48.20562744140625,
      "activations/layer11_attention_weight_max": 41.35816192626953,
      "activations/layer11_attention_weight_min": -37.901615142822266,
      "activations/layer12_attention_weight_max": 29.49061393737793,
      "activations/layer12_attention_weight_min": -28.07558822631836,
      "activations/layer13_attention_weight_max": 57.252349853515625,
      "activations/layer13_attention_weight_min": -42.15129470825195,
      "activations/layer14_attention_weight_max": 53.58319854736328,
      "activations/layer14_attention_weight_min": -39.2408332824707,
      "activations/layer15_attention_weight_max": 65.0374526977539,
      "activations/layer15_attention_weight_min": -47.15626525878906,
      "activations/layer16_attention_weight_max": 40.269996643066406,
      "activations/layer16_attention_weight_min": -29.400737762451172,
      "activations/layer17_attention_weight_max": 59.42367172241211,
      "activations/layer17_attention_weight_min": -37.17790985107422,
      "activations/layer18_attention_weight_max": 56.96687316894531,
      "activations/layer18_attention_weight_min": -33.69294738769531,
      "activations/layer19_attention_weight_max": 19.789730072021484,
      "activations/layer19_attention_weight_min": -18.360626220703125,
      "activations/layer1_attention_weight_max": 17.051584243774414,
      "activations/layer1_attention_weight_min": -15.287790298461914,
      "activations/layer20_attention_weight_max": 19.255207061767578,
      "activations/layer20_attention_weight_min": -15.273567199707031,
      "activations/layer21_attention_weight_max": 48.53976821899414,
      "activations/layer21_attention_weight_min": -26.295822143554688,
      "activations/layer22_attention_weight_max": 39.365360260009766,
      "activations/layer22_attention_weight_min": -21.649641036987305,
      "activations/layer23_attention_weight_max": 24.37889862060547,
      "activations/layer23_attention_weight_min": -14.863493919372559,
      "activations/layer2_attention_weight_max": 28.385114669799805,
      "activations/layer2_attention_weight_min": -24.37708854675293,
      "activations/layer3_attention_weight_max": 57.326568603515625,
      "activations/layer3_attention_weight_min": -61.620826721191406,
      "activations/layer4_attention_weight_max": 84.0695571899414,
      "activations/layer4_attention_weight_min": -79.22020721435547,
      "activations/layer5_attention_weight_max": 118.49659729003906,
      "activations/layer5_attention_weight_min": -102.9764404296875,
      "activations/layer6_attention_weight_max": 58.719276428222656,
      "activations/layer6_attention_weight_min": -54.351566314697266,
      "activations/layer7_attention_weight_max": 56.7347297668457,
      "activations/layer7_attention_weight_min": -54.05052185058594,
      "activations/layer8_attention_weight_max": 43.39033126831055,
      "activations/layer8_attention_weight_min": -37.898311614990234,
      "activations/layer9_attention_weight_max": 55.327999114990234,
      "activations/layer9_attention_weight_min": -53.047176361083984,
      "epoch": 2.09,
      "learning_rate": 0.00013789848484848483,
      "loss": 2.9563,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_loss": 2.91015625,
      "eval_runtime": 8.482,
      "eval_samples_per_second": 506.246,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_openwebtext_loss": 2.91015625,
      "eval_openwebtext_ppl": 18.359667040887704,
      "eval_openwebtext_runtime": 8.482,
      "eval_openwebtext_samples_per_second": 506.246,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_wikitext_loss": 3.15625,
      "eval_wikitext_ppl": 23.482371718117374,
      "eval_wikitext_runtime": 1.966,
      "eval_wikitext_samples_per_second": 231.944,
      "step": 36000
    },
    {
      "epoch": 2.09,
      "eval_lambada_loss": 3.111328125,
      "eval_lambada_ppl": 22.450842133517945,
      "eval_lambada_runtime": 9.5411,
      "eval_lambada_samples_per_second": 510.316,
      "step": 36000
    },
    {
      "activations/layer0_attention_weight_max": 14.286991119384766,
      "activations/layer0_attention_weight_min": -11.356785774230957,
      "activations/layer10_attention_weight_max": 60.616363525390625,
      "activations/layer10_attention_weight_min": -51.45889663696289,
      "activations/layer11_attention_weight_max": 44.759765625,
      "activations/layer11_attention_weight_min": -33.566184997558594,
      "activations/layer12_attention_weight_max": 26.40650749206543,
      "activations/layer12_attention_weight_min": -28.72126579284668,
      "activations/layer13_attention_weight_max": 61.580894470214844,
      "activations/layer13_attention_weight_min": -51.52643966674805,
      "activations/layer14_attention_weight_max": 53.35041427612305,
      "activations/layer14_attention_weight_min": -37.0167350769043,
      "activations/layer15_attention_weight_max": 45.90714645385742,
      "activations/layer15_attention_weight_min": -36.85829162597656,
      "activations/layer16_attention_weight_max": 44.03363037109375,
      "activations/layer16_attention_weight_min": -32.60848617553711,
      "activations/layer17_attention_weight_max": 59.46401596069336,
      "activations/layer17_attention_weight_min": -34.88916778564453,
      "activations/layer18_attention_weight_max": 53.81293869018555,
      "activations/layer18_attention_weight_min": -34.61780548095703,
      "activations/layer19_attention_weight_max": 20.25318717956543,
      "activations/layer19_attention_weight_min": -15.291426658630371,
      "activations/layer1_attention_weight_max": 16.983673095703125,
      "activations/layer1_attention_weight_min": -16.219223022460938,
      "activations/layer20_attention_weight_max": 20.19706153869629,
      "activations/layer20_attention_weight_min": -17.369915008544922,
      "activations/layer21_attention_weight_max": 38.44218063354492,
      "activations/layer21_attention_weight_min": -23.09183120727539,
      "activations/layer22_attention_weight_max": 37.861236572265625,
      "activations/layer22_attention_weight_min": -21.299734115600586,
      "activations/layer23_attention_weight_max": 24.938451766967773,
      "activations/layer23_attention_weight_min": -15.924609184265137,
      "activations/layer2_attention_weight_max": 25.834564208984375,
      "activations/layer2_attention_weight_min": -23.315961837768555,
      "activations/layer3_attention_weight_max": 63.41837692260742,
      "activations/layer3_attention_weight_min": -63.25493621826172,
      "activations/layer4_attention_weight_max": 90.88141632080078,
      "activations/layer4_attention_weight_min": -80.394775390625,
      "activations/layer5_attention_weight_max": 125.60482025146484,
      "activations/layer5_attention_weight_min": -106.11860656738281,
      "activations/layer6_attention_weight_max": 57.401859283447266,
      "activations/layer6_attention_weight_min": -50.87360763549805,
      "activations/layer7_attention_weight_max": 57.35249328613281,
      "activations/layer7_attention_weight_min": -58.611785888671875,
      "activations/layer8_attention_weight_max": 52.62538146972656,
      "activations/layer8_attention_weight_min": -42.959651947021484,
      "activations/layer9_attention_weight_max": 72.30364227294922,
      "activations/layer9_attention_weight_min": -66.13338470458984,
      "epoch": 2.09,
      "learning_rate": 0.00013787954545454545,
      "loss": 2.9606,
      "step": 36050
    },
    {
      "activations/layer0_attention_weight_max": 13.389384269714355,
      "activations/layer0_attention_weight_min": -11.632134437561035,
      "activations/layer10_attention_weight_max": 53.32440948486328,
      "activations/layer10_attention_weight_min": -42.47413635253906,
      "activations/layer11_attention_weight_max": 40.156349182128906,
      "activations/layer11_attention_weight_min": -32.35688018798828,
      "activations/layer12_attention_weight_max": 29.348581314086914,
      "activations/layer12_attention_weight_min": -24.00302505493164,
      "activations/layer13_attention_weight_max": 53.48903274536133,
      "activations/layer13_attention_weight_min": -37.77989196777344,
      "activations/layer14_attention_weight_max": 44.0803337097168,
      "activations/layer14_attention_weight_min": -33.97006607055664,
      "activations/layer15_attention_weight_max": 47.43470764160156,
      "activations/layer15_attention_weight_min": -34.01834487915039,
      "activations/layer16_attention_weight_max": 44.111324310302734,
      "activations/layer16_attention_weight_min": -28.43471908569336,
      "activations/layer17_attention_weight_max": 64.76627349853516,
      "activations/layer17_attention_weight_min": -38.338253021240234,
      "activations/layer18_attention_weight_max": 57.07633972167969,
      "activations/layer18_attention_weight_min": -31.35177230834961,
      "activations/layer19_attention_weight_max": 22.76527976989746,
      "activations/layer19_attention_weight_min": -17.19500160217285,
      "activations/layer1_attention_weight_max": 17.18093490600586,
      "activations/layer1_attention_weight_min": -14.213329315185547,
      "activations/layer20_attention_weight_max": 23.00092124938965,
      "activations/layer20_attention_weight_min": -17.49905776977539,
      "activations/layer21_attention_weight_max": 46.55839538574219,
      "activations/layer21_attention_weight_min": -23.24970245361328,
      "activations/layer22_attention_weight_max": 35.40086364746094,
      "activations/layer22_attention_weight_min": -24.0508975982666,
      "activations/layer23_attention_weight_max": 25.71693229675293,
      "activations/layer23_attention_weight_min": -15.222835540771484,
      "activations/layer2_attention_weight_max": 26.341054916381836,
      "activations/layer2_attention_weight_min": -24.34402084350586,
      "activations/layer3_attention_weight_max": 57.6360969543457,
      "activations/layer3_attention_weight_min": -60.43036651611328,
      "activations/layer4_attention_weight_max": 82.30184173583984,
      "activations/layer4_attention_weight_min": -77.35174560546875,
      "activations/layer5_attention_weight_max": 117.95346069335938,
      "activations/layer5_attention_weight_min": -98.76057434082031,
      "activations/layer6_attention_weight_max": 56.08201217651367,
      "activations/layer6_attention_weight_min": -49.93132019042969,
      "activations/layer7_attention_weight_max": 53.29591369628906,
      "activations/layer7_attention_weight_min": -51.451683044433594,
      "activations/layer8_attention_weight_max": 42.927772521972656,
      "activations/layer8_attention_weight_min": -38.237369537353516,
      "activations/layer9_attention_weight_max": 55.56538772583008,
      "activations/layer9_attention_weight_min": -55.256103515625,
      "epoch": 2.1,
      "learning_rate": 0.00013786060606060604,
      "loss": 2.9401,
      "step": 36100
    },
    {
      "activations/layer0_attention_weight_max": 14.487911224365234,
      "activations/layer0_attention_weight_min": -12.156407356262207,
      "activations/layer10_attention_weight_max": 65.82006072998047,
      "activations/layer10_attention_weight_min": -49.564693450927734,
      "activations/layer11_attention_weight_max": 43.81233215332031,
      "activations/layer11_attention_weight_min": -38.83015441894531,
      "activations/layer12_attention_weight_max": 31.569475173950195,
      "activations/layer12_attention_weight_min": -26.5002384185791,
      "activations/layer13_attention_weight_max": 77.18405151367188,
      "activations/layer13_attention_weight_min": -48.52561569213867,
      "activations/layer14_attention_weight_max": 56.91565704345703,
      "activations/layer14_attention_weight_min": -43.51202392578125,
      "activations/layer15_attention_weight_max": 65.59178924560547,
      "activations/layer15_attention_weight_min": -41.76417922973633,
      "activations/layer16_attention_weight_max": 46.163089752197266,
      "activations/layer16_attention_weight_min": -29.70160675048828,
      "activations/layer17_attention_weight_max": 65.31008911132812,
      "activations/layer17_attention_weight_min": -39.87849426269531,
      "activations/layer18_attention_weight_max": 74.12474060058594,
      "activations/layer18_attention_weight_min": -38.47789764404297,
      "activations/layer19_attention_weight_max": 25.100080490112305,
      "activations/layer19_attention_weight_min": -16.225603103637695,
      "activations/layer1_attention_weight_max": 19.117412567138672,
      "activations/layer1_attention_weight_min": -13.131792068481445,
      "activations/layer20_attention_weight_max": 22.976369857788086,
      "activations/layer20_attention_weight_min": -17.230060577392578,
      "activations/layer21_attention_weight_max": 49.98069381713867,
      "activations/layer21_attention_weight_min": -26.219144821166992,
      "activations/layer22_attention_weight_max": 43.788333892822266,
      "activations/layer22_attention_weight_min": -23.464557647705078,
      "activations/layer23_attention_weight_max": 31.04975128173828,
      "activations/layer23_attention_weight_min": -17.595531463623047,
      "activations/layer2_attention_weight_max": 27.339780807495117,
      "activations/layer2_attention_weight_min": -25.23257064819336,
      "activations/layer3_attention_weight_max": 63.68400192260742,
      "activations/layer3_attention_weight_min": -64.68085479736328,
      "activations/layer4_attention_weight_max": 88.93016052246094,
      "activations/layer4_attention_weight_min": -82.65670013427734,
      "activations/layer5_attention_weight_max": 125.02838134765625,
      "activations/layer5_attention_weight_min": -102.73402404785156,
      "activations/layer6_attention_weight_max": 62.33601760864258,
      "activations/layer6_attention_weight_min": -52.212459564208984,
      "activations/layer7_attention_weight_max": 60.08234405517578,
      "activations/layer7_attention_weight_min": -52.95244598388672,
      "activations/layer8_attention_weight_max": 51.21121597290039,
      "activations/layer8_attention_weight_min": -44.597076416015625,
      "activations/layer9_attention_weight_max": 65.05154418945312,
      "activations/layer9_attention_weight_min": -66.10965728759766,
      "epoch": 2.1,
      "learning_rate": 0.00013784166666666665,
      "loss": 2.9663,
      "step": 36150
    },
    {
      "activations/layer0_attention_weight_max": 15.48550796508789,
      "activations/layer0_attention_weight_min": -12.185079574584961,
      "activations/layer10_attention_weight_max": 59.119930267333984,
      "activations/layer10_attention_weight_min": -46.14207077026367,
      "activations/layer11_attention_weight_max": 42.242042541503906,
      "activations/layer11_attention_weight_min": -32.90690612792969,
      "activations/layer12_attention_weight_max": 29.107606887817383,
      "activations/layer12_attention_weight_min": -24.122962951660156,
      "activations/layer13_attention_weight_max": 61.19947814941406,
      "activations/layer13_attention_weight_min": -41.43933868408203,
      "activations/layer14_attention_weight_max": 61.146026611328125,
      "activations/layer14_attention_weight_min": -41.42250442504883,
      "activations/layer15_attention_weight_max": 57.74287033081055,
      "activations/layer15_attention_weight_min": -37.14156723022461,
      "activations/layer16_attention_weight_max": 41.304161071777344,
      "activations/layer16_attention_weight_min": -29.10576057434082,
      "activations/layer17_attention_weight_max": 67.5089111328125,
      "activations/layer17_attention_weight_min": -37.519309997558594,
      "activations/layer18_attention_weight_max": 55.78618621826172,
      "activations/layer18_attention_weight_min": -31.899145126342773,
      "activations/layer19_attention_weight_max": 20.3649845123291,
      "activations/layer19_attention_weight_min": -16.128013610839844,
      "activations/layer1_attention_weight_max": 16.73053550720215,
      "activations/layer1_attention_weight_min": -16.26516342163086,
      "activations/layer20_attention_weight_max": 20.023937225341797,
      "activations/layer20_attention_weight_min": -17.05997085571289,
      "activations/layer21_attention_weight_max": 42.57748794555664,
      "activations/layer21_attention_weight_min": -23.841711044311523,
      "activations/layer22_attention_weight_max": 38.055782318115234,
      "activations/layer22_attention_weight_min": -22.386268615722656,
      "activations/layer23_attention_weight_max": 21.852176666259766,
      "activations/layer23_attention_weight_min": -14.414003372192383,
      "activations/layer2_attention_weight_max": 25.400001525878906,
      "activations/layer2_attention_weight_min": -24.481794357299805,
      "activations/layer3_attention_weight_max": 63.545188903808594,
      "activations/layer3_attention_weight_min": -65.99483489990234,
      "activations/layer4_attention_weight_max": 94.41783905029297,
      "activations/layer4_attention_weight_min": -88.0403823852539,
      "activations/layer5_attention_weight_max": 127.11174774169922,
      "activations/layer5_attention_weight_min": -105.7593994140625,
      "activations/layer6_attention_weight_max": 61.861820220947266,
      "activations/layer6_attention_weight_min": -51.465553283691406,
      "activations/layer7_attention_weight_max": 61.06190872192383,
      "activations/layer7_attention_weight_min": -54.852333068847656,
      "activations/layer8_attention_weight_max": 48.76725387573242,
      "activations/layer8_attention_weight_min": -40.43486785888672,
      "activations/layer9_attention_weight_max": 69.85279083251953,
      "activations/layer9_attention_weight_min": -63.25627517700195,
      "epoch": 2.1,
      "learning_rate": 0.00013782272727272727,
      "loss": 2.9503,
      "step": 36200
    },
    {
      "activations/layer0_attention_weight_max": 14.579061508178711,
      "activations/layer0_attention_weight_min": -11.913799285888672,
      "activations/layer10_attention_weight_max": 55.013607025146484,
      "activations/layer10_attention_weight_min": -43.99460220336914,
      "activations/layer11_attention_weight_max": 43.590362548828125,
      "activations/layer11_attention_weight_min": -31.535520553588867,
      "activations/layer12_attention_weight_max": 27.89365577697754,
      "activations/layer12_attention_weight_min": -25.02945899963379,
      "activations/layer13_attention_weight_max": 54.71145248413086,
      "activations/layer13_attention_weight_min": -39.31401062011719,
      "activations/layer14_attention_weight_max": 48.56520080566406,
      "activations/layer14_attention_weight_min": -36.0592041015625,
      "activations/layer15_attention_weight_max": 50.03438949584961,
      "activations/layer15_attention_weight_min": -35.81572341918945,
      "activations/layer16_attention_weight_max": 44.65204620361328,
      "activations/layer16_attention_weight_min": -28.673301696777344,
      "activations/layer17_attention_weight_max": 69.15838623046875,
      "activations/layer17_attention_weight_min": -36.68521499633789,
      "activations/layer18_attention_weight_max": 57.26872253417969,
      "activations/layer18_attention_weight_min": -31.448625564575195,
      "activations/layer19_attention_weight_max": 18.164897918701172,
      "activations/layer19_attention_weight_min": -16.684831619262695,
      "activations/layer1_attention_weight_max": 15.65645694732666,
      "activations/layer1_attention_weight_min": -13.05999755859375,
      "activations/layer20_attention_weight_max": 22.930421829223633,
      "activations/layer20_attention_weight_min": -17.836170196533203,
      "activations/layer21_attention_weight_max": 45.97260665893555,
      "activations/layer21_attention_weight_min": -26.4345760345459,
      "activations/layer22_attention_weight_max": 35.55388259887695,
      "activations/layer22_attention_weight_min": -21.587493896484375,
      "activations/layer23_attention_weight_max": 23.69753646850586,
      "activations/layer23_attention_weight_min": -14.518217086791992,
      "activations/layer2_attention_weight_max": 27.876502990722656,
      "activations/layer2_attention_weight_min": -26.26848602294922,
      "activations/layer3_attention_weight_max": 61.358863830566406,
      "activations/layer3_attention_weight_min": -65.05697631835938,
      "activations/layer4_attention_weight_max": 87.25141906738281,
      "activations/layer4_attention_weight_min": -82.42935180664062,
      "activations/layer5_attention_weight_max": 118.87068176269531,
      "activations/layer5_attention_weight_min": -105.03298950195312,
      "activations/layer6_attention_weight_max": 57.962730407714844,
      "activations/layer6_attention_weight_min": -50.25205993652344,
      "activations/layer7_attention_weight_max": 56.49370193481445,
      "activations/layer7_attention_weight_min": -55.39382553100586,
      "activations/layer8_attention_weight_max": 44.01426696777344,
      "activations/layer8_attention_weight_min": -36.8661994934082,
      "activations/layer9_attention_weight_max": 60.3110237121582,
      "activations/layer9_attention_weight_min": -55.529083251953125,
      "epoch": 2.11,
      "learning_rate": 0.00013780378787878785,
      "loss": 2.9427,
      "step": 36250
    },
    {
      "activations/layer0_attention_weight_max": 14.19814682006836,
      "activations/layer0_attention_weight_min": -12.8468599319458,
      "activations/layer10_attention_weight_max": 66.11688232421875,
      "activations/layer10_attention_weight_min": -51.22000503540039,
      "activations/layer11_attention_weight_max": 52.90003204345703,
      "activations/layer11_attention_weight_min": -40.872615814208984,
      "activations/layer12_attention_weight_max": 31.300920486450195,
      "activations/layer12_attention_weight_min": -24.775936126708984,
      "activations/layer13_attention_weight_max": 59.431854248046875,
      "activations/layer13_attention_weight_min": -39.44853210449219,
      "activations/layer14_attention_weight_max": 54.85080337524414,
      "activations/layer14_attention_weight_min": -33.111942291259766,
      "activations/layer15_attention_weight_max": 50.46697235107422,
      "activations/layer15_attention_weight_min": -34.843162536621094,
      "activations/layer16_attention_weight_max": 45.03937911987305,
      "activations/layer16_attention_weight_min": -29.68351936340332,
      "activations/layer17_attention_weight_max": 64.94657135009766,
      "activations/layer17_attention_weight_min": -36.841896057128906,
      "activations/layer18_attention_weight_max": 59.14402389526367,
      "activations/layer18_attention_weight_min": -31.95372200012207,
      "activations/layer19_attention_weight_max": 28.48794937133789,
      "activations/layer19_attention_weight_min": -17.929819107055664,
      "activations/layer1_attention_weight_max": 15.98308277130127,
      "activations/layer1_attention_weight_min": -17.220775604248047,
      "activations/layer20_attention_weight_max": 22.36296272277832,
      "activations/layer20_attention_weight_min": -18.29277992248535,
      "activations/layer21_attention_weight_max": 46.313289642333984,
      "activations/layer21_attention_weight_min": -24.0279598236084,
      "activations/layer22_attention_weight_max": 39.728206634521484,
      "activations/layer22_attention_weight_min": -23.485227584838867,
      "activations/layer23_attention_weight_max": 28.188512802124023,
      "activations/layer23_attention_weight_min": -15.334753036499023,
      "activations/layer2_attention_weight_max": 28.221120834350586,
      "activations/layer2_attention_weight_min": -24.901107788085938,
      "activations/layer3_attention_weight_max": 63.82194900512695,
      "activations/layer3_attention_weight_min": -67.05138397216797,
      "activations/layer4_attention_weight_max": 95.82618713378906,
      "activations/layer4_attention_weight_min": -91.84612274169922,
      "activations/layer5_attention_weight_max": 138.56333923339844,
      "activations/layer5_attention_weight_min": -106.08473205566406,
      "activations/layer6_attention_weight_max": 67.75790405273438,
      "activations/layer6_attention_weight_min": -51.06993865966797,
      "activations/layer7_attention_weight_max": 63.62487030029297,
      "activations/layer7_attention_weight_min": -62.497615814208984,
      "activations/layer8_attention_weight_max": 52.352115631103516,
      "activations/layer8_attention_weight_min": -45.10082244873047,
      "activations/layer9_attention_weight_max": 73.51000213623047,
      "activations/layer9_attention_weight_min": -61.1869010925293,
      "epoch": 2.11,
      "learning_rate": 0.00013778484848484847,
      "loss": 2.9555,
      "step": 36300
    },
    {
      "activations/layer0_attention_weight_max": 15.83828067779541,
      "activations/layer0_attention_weight_min": -13.040360450744629,
      "activations/layer10_attention_weight_max": 57.31926727294922,
      "activations/layer10_attention_weight_min": -48.534786224365234,
      "activations/layer11_attention_weight_max": 42.24174499511719,
      "activations/layer11_attention_weight_min": -37.524078369140625,
      "activations/layer12_attention_weight_max": 36.71782302856445,
      "activations/layer12_attention_weight_min": -26.70456314086914,
      "activations/layer13_attention_weight_max": 57.399173736572266,
      "activations/layer13_attention_weight_min": -42.527706146240234,
      "activations/layer14_attention_weight_max": 55.319828033447266,
      "activations/layer14_attention_weight_min": -38.94703674316406,
      "activations/layer15_attention_weight_max": 54.00773239135742,
      "activations/layer15_attention_weight_min": -36.852386474609375,
      "activations/layer16_attention_weight_max": 47.07334518432617,
      "activations/layer16_attention_weight_min": -27.56913185119629,
      "activations/layer17_attention_weight_max": 66.6685562133789,
      "activations/layer17_attention_weight_min": -37.52935791015625,
      "activations/layer18_attention_weight_max": 61.19579315185547,
      "activations/layer18_attention_weight_min": -35.6472282409668,
      "activations/layer19_attention_weight_max": 28.368539810180664,
      "activations/layer19_attention_weight_min": -14.400801658630371,
      "activations/layer1_attention_weight_max": 15.613334655761719,
      "activations/layer1_attention_weight_min": -13.37281322479248,
      "activations/layer20_attention_weight_max": 20.71538543701172,
      "activations/layer20_attention_weight_min": -18.754920959472656,
      "activations/layer21_attention_weight_max": 40.873836517333984,
      "activations/layer21_attention_weight_min": -23.205053329467773,
      "activations/layer22_attention_weight_max": 36.472225189208984,
      "activations/layer22_attention_weight_min": -24.821767807006836,
      "activations/layer23_attention_weight_max": 23.28021812438965,
      "activations/layer23_attention_weight_min": -14.985858917236328,
      "activations/layer2_attention_weight_max": 27.04745864868164,
      "activations/layer2_attention_weight_min": -24.57088279724121,
      "activations/layer3_attention_weight_max": 59.43640899658203,
      "activations/layer3_attention_weight_min": -66.10969543457031,
      "activations/layer4_attention_weight_max": 88.28160858154297,
      "activations/layer4_attention_weight_min": -79.9624252319336,
      "activations/layer5_attention_weight_max": 116.76861572265625,
      "activations/layer5_attention_weight_min": -103.12889862060547,
      "activations/layer6_attention_weight_max": 57.218910217285156,
      "activations/layer6_attention_weight_min": -52.8410530090332,
      "activations/layer7_attention_weight_max": 55.80958557128906,
      "activations/layer7_attention_weight_min": -51.468631744384766,
      "activations/layer8_attention_weight_max": 41.17018508911133,
      "activations/layer8_attention_weight_min": -39.755149841308594,
      "activations/layer9_attention_weight_max": 63.3843879699707,
      "activations/layer9_attention_weight_min": -50.43206787109375,
      "epoch": 2.11,
      "learning_rate": 0.00013776590909090908,
      "loss": 2.951,
      "step": 36350
    },
    {
      "activations/layer0_attention_weight_max": 14.273822784423828,
      "activations/layer0_attention_weight_min": -12.810699462890625,
      "activations/layer10_attention_weight_max": 54.51682662963867,
      "activations/layer10_attention_weight_min": -45.07977294921875,
      "activations/layer11_attention_weight_max": 43.197593688964844,
      "activations/layer11_attention_weight_min": -32.55157470703125,
      "activations/layer12_attention_weight_max": 30.567611694335938,
      "activations/layer12_attention_weight_min": -25.40445327758789,
      "activations/layer13_attention_weight_max": 51.315330505371094,
      "activations/layer13_attention_weight_min": -38.490169525146484,
      "activations/layer14_attention_weight_max": 40.132102966308594,
      "activations/layer14_attention_weight_min": -33.73414611816406,
      "activations/layer15_attention_weight_max": 50.476234436035156,
      "activations/layer15_attention_weight_min": -39.03047561645508,
      "activations/layer16_attention_weight_max": 40.82765579223633,
      "activations/layer16_attention_weight_min": -27.92425537109375,
      "activations/layer17_attention_weight_max": 62.77804946899414,
      "activations/layer17_attention_weight_min": -36.963871002197266,
      "activations/layer18_attention_weight_max": 57.87441635131836,
      "activations/layer18_attention_weight_min": -34.52749252319336,
      "activations/layer19_attention_weight_max": 20.905885696411133,
      "activations/layer19_attention_weight_min": -14.08232307434082,
      "activations/layer1_attention_weight_max": 15.851595878601074,
      "activations/layer1_attention_weight_min": -13.96265697479248,
      "activations/layer20_attention_weight_max": 20.72563934326172,
      "activations/layer20_attention_weight_min": -16.7360782623291,
      "activations/layer21_attention_weight_max": 38.62860107421875,
      "activations/layer21_attention_weight_min": -23.616241455078125,
      "activations/layer22_attention_weight_max": 40.966331481933594,
      "activations/layer22_attention_weight_min": -22.52162742614746,
      "activations/layer23_attention_weight_max": 26.13846206665039,
      "activations/layer23_attention_weight_min": -15.18391227722168,
      "activations/layer2_attention_weight_max": 27.496566772460938,
      "activations/layer2_attention_weight_min": -24.287254333496094,
      "activations/layer3_attention_weight_max": 63.5765380859375,
      "activations/layer3_attention_weight_min": -65.32077026367188,
      "activations/layer4_attention_weight_max": 91.51103973388672,
      "activations/layer4_attention_weight_min": -81.85616302490234,
      "activations/layer5_attention_weight_max": 128.26092529296875,
      "activations/layer5_attention_weight_min": -102.281494140625,
      "activations/layer6_attention_weight_max": 61.834842681884766,
      "activations/layer6_attention_weight_min": -49.774845123291016,
      "activations/layer7_attention_weight_max": 55.827796936035156,
      "activations/layer7_attention_weight_min": -50.099891662597656,
      "activations/layer8_attention_weight_max": 46.06833267211914,
      "activations/layer8_attention_weight_min": -39.253177642822266,
      "activations/layer9_attention_weight_max": 70.25288391113281,
      "activations/layer9_attention_weight_min": -54.28767776489258,
      "epoch": 2.12,
      "learning_rate": 0.00013774696969696967,
      "loss": 2.9369,
      "step": 36400
    },
    {
      "activations/layer0_attention_weight_max": 14.517316818237305,
      "activations/layer0_attention_weight_min": -12.660723686218262,
      "activations/layer10_attention_weight_max": 58.765960693359375,
      "activations/layer10_attention_weight_min": -44.74373245239258,
      "activations/layer11_attention_weight_max": 44.94215393066406,
      "activations/layer11_attention_weight_min": -33.24488067626953,
      "activations/layer12_attention_weight_max": 29.118412017822266,
      "activations/layer12_attention_weight_min": -26.37442398071289,
      "activations/layer13_attention_weight_max": 53.2981071472168,
      "activations/layer13_attention_weight_min": -43.46656799316406,
      "activations/layer14_attention_weight_max": 49.85982131958008,
      "activations/layer14_attention_weight_min": -35.66460418701172,
      "activations/layer15_attention_weight_max": 50.93470001220703,
      "activations/layer15_attention_weight_min": -38.25675964355469,
      "activations/layer16_attention_weight_max": 41.758583068847656,
      "activations/layer16_attention_weight_min": -29.975263595581055,
      "activations/layer17_attention_weight_max": 64.11959838867188,
      "activations/layer17_attention_weight_min": -38.62881088256836,
      "activations/layer18_attention_weight_max": 57.97263717651367,
      "activations/layer18_attention_weight_min": -33.067081451416016,
      "activations/layer19_attention_weight_max": 23.39936637878418,
      "activations/layer19_attention_weight_min": -15.049094200134277,
      "activations/layer1_attention_weight_max": 16.25094223022461,
      "activations/layer1_attention_weight_min": -13.591511726379395,
      "activations/layer20_attention_weight_max": 21.016971588134766,
      "activations/layer20_attention_weight_min": -16.318246841430664,
      "activations/layer21_attention_weight_max": 46.709228515625,
      "activations/layer21_attention_weight_min": -23.508316040039062,
      "activations/layer22_attention_weight_max": 39.01323318481445,
      "activations/layer22_attention_weight_min": -21.51495361328125,
      "activations/layer23_attention_weight_max": 24.898754119873047,
      "activations/layer23_attention_weight_min": -13.621736526489258,
      "activations/layer2_attention_weight_max": 27.72895622253418,
      "activations/layer2_attention_weight_min": -26.369159698486328,
      "activations/layer3_attention_weight_max": 61.702579498291016,
      "activations/layer3_attention_weight_min": -66.9585189819336,
      "activations/layer4_attention_weight_max": 91.76225280761719,
      "activations/layer4_attention_weight_min": -86.82052612304688,
      "activations/layer5_attention_weight_max": 128.9869384765625,
      "activations/layer5_attention_weight_min": -109.28477478027344,
      "activations/layer6_attention_weight_max": 61.15474319458008,
      "activations/layer6_attention_weight_min": -50.85051727294922,
      "activations/layer7_attention_weight_max": 58.78861999511719,
      "activations/layer7_attention_weight_min": -61.54558181762695,
      "activations/layer8_attention_weight_max": 49.44297790527344,
      "activations/layer8_attention_weight_min": -41.1345100402832,
      "activations/layer9_attention_weight_max": 62.47998046875,
      "activations/layer9_attention_weight_min": -58.51982498168945,
      "epoch": 2.12,
      "learning_rate": 0.0001377280303030303,
      "loss": 2.9575,
      "step": 36450
    },
    {
      "activations/layer0_attention_weight_max": 13.505236625671387,
      "activations/layer0_attention_weight_min": -12.79322338104248,
      "activations/layer10_attention_weight_max": 58.78889846801758,
      "activations/layer10_attention_weight_min": -50.97620391845703,
      "activations/layer11_attention_weight_max": 42.201210021972656,
      "activations/layer11_attention_weight_min": -33.66206359863281,
      "activations/layer12_attention_weight_max": 28.40873908996582,
      "activations/layer12_attention_weight_min": -29.382158279418945,
      "activations/layer13_attention_weight_max": 54.700496673583984,
      "activations/layer13_attention_weight_min": -40.05824279785156,
      "activations/layer14_attention_weight_max": 42.544410705566406,
      "activations/layer14_attention_weight_min": -31.71706771850586,
      "activations/layer15_attention_weight_max": 51.19328308105469,
      "activations/layer15_attention_weight_min": -37.179595947265625,
      "activations/layer16_attention_weight_max": 39.37940216064453,
      "activations/layer16_attention_weight_min": -29.246906280517578,
      "activations/layer17_attention_weight_max": 60.0609016418457,
      "activations/layer17_attention_weight_min": -38.73335266113281,
      "activations/layer18_attention_weight_max": 55.673789978027344,
      "activations/layer18_attention_weight_min": -36.23994445800781,
      "activations/layer19_attention_weight_max": 19.68250274658203,
      "activations/layer19_attention_weight_min": -14.570619583129883,
      "activations/layer1_attention_weight_max": 16.884347915649414,
      "activations/layer1_attention_weight_min": -13.540360450744629,
      "activations/layer20_attention_weight_max": 18.282691955566406,
      "activations/layer20_attention_weight_min": -18.880868911743164,
      "activations/layer21_attention_weight_max": 38.80939483642578,
      "activations/layer21_attention_weight_min": -24.997642517089844,
      "activations/layer22_attention_weight_max": 34.087684631347656,
      "activations/layer22_attention_weight_min": -22.246490478515625,
      "activations/layer23_attention_weight_max": 22.641691207885742,
      "activations/layer23_attention_weight_min": -14.809588432312012,
      "activations/layer2_attention_weight_max": 27.292308807373047,
      "activations/layer2_attention_weight_min": -24.49146842956543,
      "activations/layer3_attention_weight_max": 60.1287956237793,
      "activations/layer3_attention_weight_min": -61.11104202270508,
      "activations/layer4_attention_weight_max": 95.72957611083984,
      "activations/layer4_attention_weight_min": -84.20549011230469,
      "activations/layer5_attention_weight_max": 128.39724731445312,
      "activations/layer5_attention_weight_min": -105.3900375366211,
      "activations/layer6_attention_weight_max": 60.77973556518555,
      "activations/layer6_attention_weight_min": -52.75666046142578,
      "activations/layer7_attention_weight_max": 59.68648147583008,
      "activations/layer7_attention_weight_min": -56.13399887084961,
      "activations/layer8_attention_weight_max": 44.55437088012695,
      "activations/layer8_attention_weight_min": -39.59610366821289,
      "activations/layer9_attention_weight_max": 69.00074005126953,
      "activations/layer9_attention_weight_min": -60.439876556396484,
      "epoch": 2.12,
      "learning_rate": 0.0001377090909090909,
      "loss": 2.9625,
      "step": 36500
    },
    {
      "activations/layer0_attention_weight_max": 13.712789535522461,
      "activations/layer0_attention_weight_min": -12.779415130615234,
      "activations/layer10_attention_weight_max": 52.99774932861328,
      "activations/layer10_attention_weight_min": -45.904415130615234,
      "activations/layer11_attention_weight_max": 40.95237350463867,
      "activations/layer11_attention_weight_min": -34.36454772949219,
      "activations/layer12_attention_weight_max": 26.920307159423828,
      "activations/layer12_attention_weight_min": -23.928546905517578,
      "activations/layer13_attention_weight_max": 62.15373992919922,
      "activations/layer13_attention_weight_min": -44.69734191894531,
      "activations/layer14_attention_weight_max": 52.32822799682617,
      "activations/layer14_attention_weight_min": -37.89842224121094,
      "activations/layer15_attention_weight_max": 50.941680908203125,
      "activations/layer15_attention_weight_min": -34.66432189941406,
      "activations/layer16_attention_weight_max": 44.37784194946289,
      "activations/layer16_attention_weight_min": -30.089122772216797,
      "activations/layer17_attention_weight_max": 61.45360565185547,
      "activations/layer17_attention_weight_min": -39.699398040771484,
      "activations/layer18_attention_weight_max": 62.41739273071289,
      "activations/layer18_attention_weight_min": -35.91205978393555,
      "activations/layer19_attention_weight_max": 25.272706985473633,
      "activations/layer19_attention_weight_min": -16.536428451538086,
      "activations/layer1_attention_weight_max": 14.9423828125,
      "activations/layer1_attention_weight_min": -12.775206565856934,
      "activations/layer20_attention_weight_max": 23.540281295776367,
      "activations/layer20_attention_weight_min": -16.274030685424805,
      "activations/layer21_attention_weight_max": 38.61421585083008,
      "activations/layer21_attention_weight_min": -23.82432746887207,
      "activations/layer22_attention_weight_max": 40.03800964355469,
      "activations/layer22_attention_weight_min": -25.68724822998047,
      "activations/layer23_attention_weight_max": 25.408666610717773,
      "activations/layer23_attention_weight_min": -15.688727378845215,
      "activations/layer2_attention_weight_max": 26.23712730407715,
      "activations/layer2_attention_weight_min": -24.072654724121094,
      "activations/layer3_attention_weight_max": 63.564720153808594,
      "activations/layer3_attention_weight_min": -64.25843811035156,
      "activations/layer4_attention_weight_max": 88.65485382080078,
      "activations/layer4_attention_weight_min": -82.7865982055664,
      "activations/layer5_attention_weight_max": 120.63739013671875,
      "activations/layer5_attention_weight_min": -102.79611206054688,
      "activations/layer6_attention_weight_max": 57.6121711730957,
      "activations/layer6_attention_weight_min": -51.7103157043457,
      "activations/layer7_attention_weight_max": 56.119571685791016,
      "activations/layer7_attention_weight_min": -54.471099853515625,
      "activations/layer8_attention_weight_max": 43.799415588378906,
      "activations/layer8_attention_weight_min": -37.80437088012695,
      "activations/layer9_attention_weight_max": 63.7744026184082,
      "activations/layer9_attention_weight_min": -54.28232955932617,
      "epoch": 2.12,
      "learning_rate": 0.0001376901515151515,
      "loss": 2.9569,
      "step": 36550
    },
    {
      "activations/layer0_attention_weight_max": 15.015642166137695,
      "activations/layer0_attention_weight_min": -12.729985237121582,
      "activations/layer10_attention_weight_max": 57.38011932373047,
      "activations/layer10_attention_weight_min": -46.34661102294922,
      "activations/layer11_attention_weight_max": 39.813594818115234,
      "activations/layer11_attention_weight_min": -34.82978820800781,
      "activations/layer12_attention_weight_max": 28.507814407348633,
      "activations/layer12_attention_weight_min": -25.99040985107422,
      "activations/layer13_attention_weight_max": 50.965599060058594,
      "activations/layer13_attention_weight_min": -38.435508728027344,
      "activations/layer14_attention_weight_max": 44.37013244628906,
      "activations/layer14_attention_weight_min": -34.795711517333984,
      "activations/layer15_attention_weight_max": 50.44395446777344,
      "activations/layer15_attention_weight_min": -38.7982292175293,
      "activations/layer16_attention_weight_max": 41.9243278503418,
      "activations/layer16_attention_weight_min": -29.252878189086914,
      "activations/layer17_attention_weight_max": 58.53470230102539,
      "activations/layer17_attention_weight_min": -35.768638610839844,
      "activations/layer18_attention_weight_max": 61.31137466430664,
      "activations/layer18_attention_weight_min": -33.64237976074219,
      "activations/layer19_attention_weight_max": 22.142812728881836,
      "activations/layer19_attention_weight_min": -16.646854400634766,
      "activations/layer1_attention_weight_max": 16.2666072845459,
      "activations/layer1_attention_weight_min": -14.488961219787598,
      "activations/layer20_attention_weight_max": 21.818321228027344,
      "activations/layer20_attention_weight_min": -18.705766677856445,
      "activations/layer21_attention_weight_max": 46.072509765625,
      "activations/layer21_attention_weight_min": -25.886552810668945,
      "activations/layer22_attention_weight_max": 38.55197525024414,
      "activations/layer22_attention_weight_min": -22.84455108642578,
      "activations/layer23_attention_weight_max": 29.60887908935547,
      "activations/layer23_attention_weight_min": -15.441378593444824,
      "activations/layer2_attention_weight_max": 26.806928634643555,
      "activations/layer2_attention_weight_min": -25.879064559936523,
      "activations/layer3_attention_weight_max": 61.07809829711914,
      "activations/layer3_attention_weight_min": -65.9462890625,
      "activations/layer4_attention_weight_max": 90.75240325927734,
      "activations/layer4_attention_weight_min": -81.29515075683594,
      "activations/layer5_attention_weight_max": 116.1407470703125,
      "activations/layer5_attention_weight_min": -101.63423919677734,
      "activations/layer6_attention_weight_max": 56.90650939941406,
      "activations/layer6_attention_weight_min": -51.03622817993164,
      "activations/layer7_attention_weight_max": 55.454830169677734,
      "activations/layer7_attention_weight_min": -50.53280258178711,
      "activations/layer8_attention_weight_max": 42.01799392700195,
      "activations/layer8_attention_weight_min": -39.96730041503906,
      "activations/layer9_attention_weight_max": 53.611793518066406,
      "activations/layer9_attention_weight_min": -50.5548210144043,
      "epoch": 2.13,
      "learning_rate": 0.0001376712121212121,
      "loss": 2.9485,
      "step": 36600
    },
    {
      "activations/layer0_attention_weight_max": 16.258562088012695,
      "activations/layer0_attention_weight_min": -13.129016876220703,
      "activations/layer10_attention_weight_max": 55.86712646484375,
      "activations/layer10_attention_weight_min": -45.98317337036133,
      "activations/layer11_attention_weight_max": 45.020721435546875,
      "activations/layer11_attention_weight_min": -32.20829772949219,
      "activations/layer12_attention_weight_max": 31.151227951049805,
      "activations/layer12_attention_weight_min": -24.090744018554688,
      "activations/layer13_attention_weight_max": 59.70950698852539,
      "activations/layer13_attention_weight_min": -45.64420700073242,
      "activations/layer14_attention_weight_max": 52.93614196777344,
      "activations/layer14_attention_weight_min": -36.683589935302734,
      "activations/layer15_attention_weight_max": 64.49237060546875,
      "activations/layer15_attention_weight_min": -40.05156326293945,
      "activations/layer16_attention_weight_max": 39.68904113769531,
      "activations/layer16_attention_weight_min": -27.598690032958984,
      "activations/layer17_attention_weight_max": 58.09760284423828,
      "activations/layer17_attention_weight_min": -36.443546295166016,
      "activations/layer18_attention_weight_max": 60.728797912597656,
      "activations/layer18_attention_weight_min": -32.960784912109375,
      "activations/layer19_attention_weight_max": 20.946094512939453,
      "activations/layer19_attention_weight_min": -16.8182430267334,
      "activations/layer1_attention_weight_max": 17.472190856933594,
      "activations/layer1_attention_weight_min": -13.831539154052734,
      "activations/layer20_attention_weight_max": 22.670194625854492,
      "activations/layer20_attention_weight_min": -18.954435348510742,
      "activations/layer21_attention_weight_max": 44.96320724487305,
      "activations/layer21_attention_weight_min": -25.80199432373047,
      "activations/layer22_attention_weight_max": 39.93467712402344,
      "activations/layer22_attention_weight_min": -23.634037017822266,
      "activations/layer23_attention_weight_max": 25.773670196533203,
      "activations/layer23_attention_weight_min": -14.534411430358887,
      "activations/layer2_attention_weight_max": 26.172534942626953,
      "activations/layer2_attention_weight_min": -26.278013229370117,
      "activations/layer3_attention_weight_max": 67.48306274414062,
      "activations/layer3_attention_weight_min": -72.35330200195312,
      "activations/layer4_attention_weight_max": 105.52950286865234,
      "activations/layer4_attention_weight_min": -91.61824798583984,
      "activations/layer5_attention_weight_max": 135.2404327392578,
      "activations/layer5_attention_weight_min": -109.52147674560547,
      "activations/layer6_attention_weight_max": 56.53974914550781,
      "activations/layer6_attention_weight_min": -49.22663116455078,
      "activations/layer7_attention_weight_max": 60.708152770996094,
      "activations/layer7_attention_weight_min": -51.177757263183594,
      "activations/layer8_attention_weight_max": 45.01731491088867,
      "activations/layer8_attention_weight_min": -36.15310287475586,
      "activations/layer9_attention_weight_max": 65.23114776611328,
      "activations/layer9_attention_weight_min": -54.79865264892578,
      "epoch": 2.13,
      "learning_rate": 0.00013765227272727272,
      "loss": 2.9632,
      "step": 36650
    },
    {
      "activations/layer0_attention_weight_max": 15.671467781066895,
      "activations/layer0_attention_weight_min": -13.321932792663574,
      "activations/layer10_attention_weight_max": 63.06459426879883,
      "activations/layer10_attention_weight_min": -46.22248458862305,
      "activations/layer11_attention_weight_max": 46.41069793701172,
      "activations/layer11_attention_weight_min": -35.868186950683594,
      "activations/layer12_attention_weight_max": 31.747211456298828,
      "activations/layer12_attention_weight_min": -25.05225372314453,
      "activations/layer13_attention_weight_max": 69.14259338378906,
      "activations/layer13_attention_weight_min": -46.407814025878906,
      "activations/layer14_attention_weight_max": 58.5898551940918,
      "activations/layer14_attention_weight_min": -37.212364196777344,
      "activations/layer15_attention_weight_max": 56.779850006103516,
      "activations/layer15_attention_weight_min": -37.22856140136719,
      "activations/layer16_attention_weight_max": 39.14182662963867,
      "activations/layer16_attention_weight_min": -31.21710968017578,
      "activations/layer17_attention_weight_max": 72.124267578125,
      "activations/layer17_attention_weight_min": -41.874229431152344,
      "activations/layer18_attention_weight_max": 59.53050994873047,
      "activations/layer18_attention_weight_min": -32.6323127746582,
      "activations/layer19_attention_weight_max": 21.66248321533203,
      "activations/layer19_attention_weight_min": -16.05585289001465,
      "activations/layer1_attention_weight_max": 16.52659034729004,
      "activations/layer1_attention_weight_min": -19.381717681884766,
      "activations/layer20_attention_weight_max": 23.875865936279297,
      "activations/layer20_attention_weight_min": -16.181241989135742,
      "activations/layer21_attention_weight_max": 40.236881256103516,
      "activations/layer21_attention_weight_min": -24.920019149780273,
      "activations/layer22_attention_weight_max": 40.707950592041016,
      "activations/layer22_attention_weight_min": -24.915435791015625,
      "activations/layer23_attention_weight_max": 28.243919372558594,
      "activations/layer23_attention_weight_min": -14.579450607299805,
      "activations/layer2_attention_weight_max": 28.4509334564209,
      "activations/layer2_attention_weight_min": -27.932554244995117,
      "activations/layer3_attention_weight_max": 64.69402313232422,
      "activations/layer3_attention_weight_min": -67.1806411743164,
      "activations/layer4_attention_weight_max": 97.31178283691406,
      "activations/layer4_attention_weight_min": -85.44346618652344,
      "activations/layer5_attention_weight_max": 127.63204956054688,
      "activations/layer5_attention_weight_min": -108.31207275390625,
      "activations/layer6_attention_weight_max": 60.55115509033203,
      "activations/layer6_attention_weight_min": -51.64141082763672,
      "activations/layer7_attention_weight_max": 62.901187896728516,
      "activations/layer7_attention_weight_min": -53.50986099243164,
      "activations/layer8_attention_weight_max": 51.58075714111328,
      "activations/layer8_attention_weight_min": -36.74033737182617,
      "activations/layer9_attention_weight_max": 69.82890319824219,
      "activations/layer9_attention_weight_min": -53.88209915161133,
      "epoch": 2.13,
      "learning_rate": 0.00013763333333333334,
      "loss": 2.9594,
      "step": 36700
    },
    {
      "activations/layer0_attention_weight_max": 14.595532417297363,
      "activations/layer0_attention_weight_min": -13.08125114440918,
      "activations/layer10_attention_weight_max": 59.867393493652344,
      "activations/layer10_attention_weight_min": -44.519954681396484,
      "activations/layer11_attention_weight_max": 46.553489685058594,
      "activations/layer11_attention_weight_min": -34.67689514160156,
      "activations/layer12_attention_weight_max": 24.53106117248535,
      "activations/layer12_attention_weight_min": -24.26603889465332,
      "activations/layer13_attention_weight_max": 56.06282424926758,
      "activations/layer13_attention_weight_min": -35.481422424316406,
      "activations/layer14_attention_weight_max": 54.63327407836914,
      "activations/layer14_attention_weight_min": -33.70284652709961,
      "activations/layer15_attention_weight_max": 56.34957504272461,
      "activations/layer15_attention_weight_min": -39.64140701293945,
      "activations/layer16_attention_weight_max": 43.324012756347656,
      "activations/layer16_attention_weight_min": -29.07549476623535,
      "activations/layer17_attention_weight_max": 63.43765640258789,
      "activations/layer17_attention_weight_min": -37.49309158325195,
      "activations/layer18_attention_weight_max": 62.525672912597656,
      "activations/layer18_attention_weight_min": -30.914108276367188,
      "activations/layer19_attention_weight_max": 25.70613670349121,
      "activations/layer19_attention_weight_min": -16.38926124572754,
      "activations/layer1_attention_weight_max": 16.589492797851562,
      "activations/layer1_attention_weight_min": -12.14668083190918,
      "activations/layer20_attention_weight_max": 24.923954010009766,
      "activations/layer20_attention_weight_min": -18.851072311401367,
      "activations/layer21_attention_weight_max": 44.0854377746582,
      "activations/layer21_attention_weight_min": -26.759702682495117,
      "activations/layer22_attention_weight_max": 36.100215911865234,
      "activations/layer22_attention_weight_min": -23.89333152770996,
      "activations/layer23_attention_weight_max": 27.5953369140625,
      "activations/layer23_attention_weight_min": -15.443151473999023,
      "activations/layer2_attention_weight_max": 28.246076583862305,
      "activations/layer2_attention_weight_min": -28.674022674560547,
      "activations/layer3_attention_weight_max": 62.55031204223633,
      "activations/layer3_attention_weight_min": -70.03253936767578,
      "activations/layer4_attention_weight_max": 92.63194274902344,
      "activations/layer4_attention_weight_min": -84.24638366699219,
      "activations/layer5_attention_weight_max": 131.1044158935547,
      "activations/layer5_attention_weight_min": -104.83029174804688,
      "activations/layer6_attention_weight_max": 62.94923400878906,
      "activations/layer6_attention_weight_min": -50.61384582519531,
      "activations/layer7_attention_weight_max": 66.30025482177734,
      "activations/layer7_attention_weight_min": -55.957786560058594,
      "activations/layer8_attention_weight_max": 48.915645599365234,
      "activations/layer8_attention_weight_min": -39.77093505859375,
      "activations/layer9_attention_weight_max": 68.72846221923828,
      "activations/layer9_attention_weight_min": -60.58521270751953,
      "epoch": 2.14,
      "learning_rate": 0.00013761439393939392,
      "loss": 2.964,
      "step": 36750
    },
    {
      "activations/layer0_attention_weight_max": 15.038336753845215,
      "activations/layer0_attention_weight_min": -13.73115348815918,
      "activations/layer10_attention_weight_max": 61.08787536621094,
      "activations/layer10_attention_weight_min": -52.088130950927734,
      "activations/layer11_attention_weight_max": 46.977943420410156,
      "activations/layer11_attention_weight_min": -34.063636779785156,
      "activations/layer12_attention_weight_max": 29.39250373840332,
      "activations/layer12_attention_weight_min": -24.661239624023438,
      "activations/layer13_attention_weight_max": 54.57101821899414,
      "activations/layer13_attention_weight_min": -38.79256057739258,
      "activations/layer14_attention_weight_max": 55.140682220458984,
      "activations/layer14_attention_weight_min": -32.44002151489258,
      "activations/layer15_attention_weight_max": 55.94911193847656,
      "activations/layer15_attention_weight_min": -39.71095657348633,
      "activations/layer16_attention_weight_max": 47.27890396118164,
      "activations/layer16_attention_weight_min": -30.691715240478516,
      "activations/layer17_attention_weight_max": 58.08002471923828,
      "activations/layer17_attention_weight_min": -39.67870330810547,
      "activations/layer18_attention_weight_max": 57.47878646850586,
      "activations/layer18_attention_weight_min": -35.91045379638672,
      "activations/layer19_attention_weight_max": 22.272756576538086,
      "activations/layer19_attention_weight_min": -15.695780754089355,
      "activations/layer1_attention_weight_max": 15.105923652648926,
      "activations/layer1_attention_weight_min": -12.976285934448242,
      "activations/layer20_attention_weight_max": 20.501970291137695,
      "activations/layer20_attention_weight_min": -19.044864654541016,
      "activations/layer21_attention_weight_max": 41.74904251098633,
      "activations/layer21_attention_weight_min": -25.310781478881836,
      "activations/layer22_attention_weight_max": 35.62923812866211,
      "activations/layer22_attention_weight_min": -22.410127639770508,
      "activations/layer23_attention_weight_max": 28.306842803955078,
      "activations/layer23_attention_weight_min": -15.727533340454102,
      "activations/layer2_attention_weight_max": 26.951454162597656,
      "activations/layer2_attention_weight_min": -25.668785095214844,
      "activations/layer3_attention_weight_max": 59.86754608154297,
      "activations/layer3_attention_weight_min": -61.098445892333984,
      "activations/layer4_attention_weight_max": 91.17056274414062,
      "activations/layer4_attention_weight_min": -83.00679779052734,
      "activations/layer5_attention_weight_max": 116.84480285644531,
      "activations/layer5_attention_weight_min": -100.92210388183594,
      "activations/layer6_attention_weight_max": 59.30400466918945,
      "activations/layer6_attention_weight_min": -50.5672492980957,
      "activations/layer7_attention_weight_max": 55.21030807495117,
      "activations/layer7_attention_weight_min": -51.66349411010742,
      "activations/layer8_attention_weight_max": 45.71711730957031,
      "activations/layer8_attention_weight_min": -37.649620056152344,
      "activations/layer9_attention_weight_max": 59.795257568359375,
      "activations/layer9_attention_weight_min": -52.51343536376953,
      "epoch": 2.14,
      "learning_rate": 0.00013759545454545454,
      "loss": 2.9629,
      "step": 36800
    },
    {
      "activations/layer0_attention_weight_max": 14.858263969421387,
      "activations/layer0_attention_weight_min": -13.314260482788086,
      "activations/layer10_attention_weight_max": 57.49369812011719,
      "activations/layer10_attention_weight_min": -50.60099792480469,
      "activations/layer11_attention_weight_max": 41.971214294433594,
      "activations/layer11_attention_weight_min": -35.33068084716797,
      "activations/layer12_attention_weight_max": 26.72483253479004,
      "activations/layer12_attention_weight_min": -25.037755966186523,
      "activations/layer13_attention_weight_max": 53.47028732299805,
      "activations/layer13_attention_weight_min": -39.88639450073242,
      "activations/layer14_attention_weight_max": 44.1932487487793,
      "activations/layer14_attention_weight_min": -36.608245849609375,
      "activations/layer15_attention_weight_max": 45.93678665161133,
      "activations/layer15_attention_weight_min": -36.93572235107422,
      "activations/layer16_attention_weight_max": 41.04991149902344,
      "activations/layer16_attention_weight_min": -28.50809669494629,
      "activations/layer17_attention_weight_max": 53.88499069213867,
      "activations/layer17_attention_weight_min": -36.31021499633789,
      "activations/layer18_attention_weight_max": 50.67628860473633,
      "activations/layer18_attention_weight_min": -31.71307945251465,
      "activations/layer19_attention_weight_max": 20.465795516967773,
      "activations/layer19_attention_weight_min": -16.811634063720703,
      "activations/layer1_attention_weight_max": 15.977045059204102,
      "activations/layer1_attention_weight_min": -15.235150337219238,
      "activations/layer20_attention_weight_max": 20.92127799987793,
      "activations/layer20_attention_weight_min": -16.282119750976562,
      "activations/layer21_attention_weight_max": 33.75754928588867,
      "activations/layer21_attention_weight_min": -24.599674224853516,
      "activations/layer22_attention_weight_max": 32.6420783996582,
      "activations/layer22_attention_weight_min": -23.75929832458496,
      "activations/layer23_attention_weight_max": 22.00006103515625,
      "activations/layer23_attention_weight_min": -13.963996887207031,
      "activations/layer2_attention_weight_max": 27.072134017944336,
      "activations/layer2_attention_weight_min": -26.047740936279297,
      "activations/layer3_attention_weight_max": 64.91160583496094,
      "activations/layer3_attention_weight_min": -63.31684875488281,
      "activations/layer4_attention_weight_max": 97.8355484008789,
      "activations/layer4_attention_weight_min": -84.20250701904297,
      "activations/layer5_attention_weight_max": 126.4530258178711,
      "activations/layer5_attention_weight_min": -103.23250579833984,
      "activations/layer6_attention_weight_max": 60.608055114746094,
      "activations/layer6_attention_weight_min": -49.05552673339844,
      "activations/layer7_attention_weight_max": 62.21954345703125,
      "activations/layer7_attention_weight_min": -53.1989860534668,
      "activations/layer8_attention_weight_max": 45.9898567199707,
      "activations/layer8_attention_weight_min": -38.802059173583984,
      "activations/layer9_attention_weight_max": 67.2055435180664,
      "activations/layer9_attention_weight_min": -56.12775421142578,
      "epoch": 2.14,
      "learning_rate": 0.00013757651515151515,
      "loss": 2.9528,
      "step": 36850
    },
    {
      "activations/layer0_attention_weight_max": 14.07192325592041,
      "activations/layer0_attention_weight_min": -13.333331108093262,
      "activations/layer10_attention_weight_max": 56.59596633911133,
      "activations/layer10_attention_weight_min": -46.98617935180664,
      "activations/layer11_attention_weight_max": 45.139251708984375,
      "activations/layer11_attention_weight_min": -33.82649230957031,
      "activations/layer12_attention_weight_max": 33.1798210144043,
      "activations/layer12_attention_weight_min": -23.003173828125,
      "activations/layer13_attention_weight_max": 59.1715202331543,
      "activations/layer13_attention_weight_min": -45.3050651550293,
      "activations/layer14_attention_weight_max": 45.928836822509766,
      "activations/layer14_attention_weight_min": -36.603824615478516,
      "activations/layer15_attention_weight_max": 47.361324310302734,
      "activations/layer15_attention_weight_min": -36.562255859375,
      "activations/layer16_attention_weight_max": 44.07413101196289,
      "activations/layer16_attention_weight_min": -27.877601623535156,
      "activations/layer17_attention_weight_max": 59.68076705932617,
      "activations/layer17_attention_weight_min": -38.04744338989258,
      "activations/layer18_attention_weight_max": 58.58259582519531,
      "activations/layer18_attention_weight_min": -33.47985076904297,
      "activations/layer19_attention_weight_max": 22.6187801361084,
      "activations/layer19_attention_weight_min": -17.948604583740234,
      "activations/layer1_attention_weight_max": 15.477131843566895,
      "activations/layer1_attention_weight_min": -14.329381942749023,
      "activations/layer20_attention_weight_max": 20.51799201965332,
      "activations/layer20_attention_weight_min": -18.607345581054688,
      "activations/layer21_attention_weight_max": 45.56864929199219,
      "activations/layer21_attention_weight_min": -24.56920623779297,
      "activations/layer22_attention_weight_max": 38.082027435302734,
      "activations/layer22_attention_weight_min": -23.173282623291016,
      "activations/layer23_attention_weight_max": 26.73377227783203,
      "activations/layer23_attention_weight_min": -16.53593635559082,
      "activations/layer2_attention_weight_max": 27.17467498779297,
      "activations/layer2_attention_weight_min": -27.334522247314453,
      "activations/layer3_attention_weight_max": 64.0486831665039,
      "activations/layer3_attention_weight_min": -64.6953353881836,
      "activations/layer4_attention_weight_max": 89.14151763916016,
      "activations/layer4_attention_weight_min": -84.05107116699219,
      "activations/layer5_attention_weight_max": 122.40797424316406,
      "activations/layer5_attention_weight_min": -104.41634368896484,
      "activations/layer6_attention_weight_max": 59.20549011230469,
      "activations/layer6_attention_weight_min": -51.7728385925293,
      "activations/layer7_attention_weight_max": 54.001502990722656,
      "activations/layer7_attention_weight_min": -50.42034149169922,
      "activations/layer8_attention_weight_max": 41.218013763427734,
      "activations/layer8_attention_weight_min": -39.24428176879883,
      "activations/layer9_attention_weight_max": 56.53894805908203,
      "activations/layer9_attention_weight_min": -51.131507873535156,
      "epoch": 2.14,
      "learning_rate": 0.00013755757575757574,
      "loss": 2.9697,
      "step": 36900
    },
    {
      "activations/layer0_attention_weight_max": 15.008530616760254,
      "activations/layer0_attention_weight_min": -13.206646919250488,
      "activations/layer10_attention_weight_max": 53.70948028564453,
      "activations/layer10_attention_weight_min": -41.89181137084961,
      "activations/layer11_attention_weight_max": 41.01193618774414,
      "activations/layer11_attention_weight_min": -30.135116577148438,
      "activations/layer12_attention_weight_max": 34.894779205322266,
      "activations/layer12_attention_weight_min": -27.994361877441406,
      "activations/layer13_attention_weight_max": 56.290679931640625,
      "activations/layer13_attention_weight_min": -40.80537796020508,
      "activations/layer14_attention_weight_max": 56.052146911621094,
      "activations/layer14_attention_weight_min": -40.102394104003906,
      "activations/layer15_attention_weight_max": 50.113643646240234,
      "activations/layer15_attention_weight_min": -36.862648010253906,
      "activations/layer16_attention_weight_max": 37.02562713623047,
      "activations/layer16_attention_weight_min": -33.1352424621582,
      "activations/layer17_attention_weight_max": 57.7620849609375,
      "activations/layer17_attention_weight_min": -36.64128112792969,
      "activations/layer18_attention_weight_max": 60.2197151184082,
      "activations/layer18_attention_weight_min": -32.133514404296875,
      "activations/layer19_attention_weight_max": 21.260225296020508,
      "activations/layer19_attention_weight_min": -16.67327308654785,
      "activations/layer1_attention_weight_max": 17.02708625793457,
      "activations/layer1_attention_weight_min": -13.338969230651855,
      "activations/layer20_attention_weight_max": 20.39356231689453,
      "activations/layer20_attention_weight_min": -17.712575912475586,
      "activations/layer21_attention_weight_max": 41.94797897338867,
      "activations/layer21_attention_weight_min": -26.104354858398438,
      "activations/layer22_attention_weight_max": 34.13343048095703,
      "activations/layer22_attention_weight_min": -23.027498245239258,
      "activations/layer23_attention_weight_max": 23.344257354736328,
      "activations/layer23_attention_weight_min": -18.0997314453125,
      "activations/layer2_attention_weight_max": 28.542158126831055,
      "activations/layer2_attention_weight_min": -28.323049545288086,
      "activations/layer3_attention_weight_max": 62.758243560791016,
      "activations/layer3_attention_weight_min": -65.35567474365234,
      "activations/layer4_attention_weight_max": 91.61236572265625,
      "activations/layer4_attention_weight_min": -80.37667846679688,
      "activations/layer5_attention_weight_max": 124.16069030761719,
      "activations/layer5_attention_weight_min": -99.84451293945312,
      "activations/layer6_attention_weight_max": 62.20411682128906,
      "activations/layer6_attention_weight_min": -49.40586471557617,
      "activations/layer7_attention_weight_max": 57.99788284301758,
      "activations/layer7_attention_weight_min": -56.06044006347656,
      "activations/layer8_attention_weight_max": 43.630577087402344,
      "activations/layer8_attention_weight_min": -37.224849700927734,
      "activations/layer9_attention_weight_max": 58.304237365722656,
      "activations/layer9_attention_weight_min": -56.6083869934082,
      "epoch": 2.15,
      "learning_rate": 0.00013753863636363636,
      "loss": 2.9443,
      "step": 36950
    },
    {
      "activations/layer0_attention_weight_max": 15.00344467163086,
      "activations/layer0_attention_weight_min": -13.362129211425781,
      "activations/layer10_attention_weight_max": 50.79391860961914,
      "activations/layer10_attention_weight_min": -43.8834342956543,
      "activations/layer11_attention_weight_max": 38.49089050292969,
      "activations/layer11_attention_weight_min": -33.605594635009766,
      "activations/layer12_attention_weight_max": 27.468137741088867,
      "activations/layer12_attention_weight_min": -23.378009796142578,
      "activations/layer13_attention_weight_max": 51.22148132324219,
      "activations/layer13_attention_weight_min": -38.41450119018555,
      "activations/layer14_attention_weight_max": 40.188438415527344,
      "activations/layer14_attention_weight_min": -29.62785530090332,
      "activations/layer15_attention_weight_max": 52.078895568847656,
      "activations/layer15_attention_weight_min": -34.72222137451172,
      "activations/layer16_attention_weight_max": 42.509029388427734,
      "activations/layer16_attention_weight_min": -29.85565757751465,
      "activations/layer17_attention_weight_max": 59.63180923461914,
      "activations/layer17_attention_weight_min": -37.0735969543457,
      "activations/layer18_attention_weight_max": 53.88814926147461,
      "activations/layer18_attention_weight_min": -33.7567253112793,
      "activations/layer19_attention_weight_max": 21.059080123901367,
      "activations/layer19_attention_weight_min": -16.512481689453125,
      "activations/layer1_attention_weight_max": 15.9187650680542,
      "activations/layer1_attention_weight_min": -12.52632999420166,
      "activations/layer20_attention_weight_max": 20.218767166137695,
      "activations/layer20_attention_weight_min": -19.128643035888672,
      "activations/layer21_attention_weight_max": 42.28982162475586,
      "activations/layer21_attention_weight_min": -26.660308837890625,
      "activations/layer22_attention_weight_max": 35.45370864868164,
      "activations/layer22_attention_weight_min": -26.00429916381836,
      "activations/layer23_attention_weight_max": 23.040164947509766,
      "activations/layer23_attention_weight_min": -15.138608932495117,
      "activations/layer2_attention_weight_max": 26.48699951171875,
      "activations/layer2_attention_weight_min": -26.583906173706055,
      "activations/layer3_attention_weight_max": 62.10895919799805,
      "activations/layer3_attention_weight_min": -64.85464477539062,
      "activations/layer4_attention_weight_max": 95.68254852294922,
      "activations/layer4_attention_weight_min": -87.75704193115234,
      "activations/layer5_attention_weight_max": 124.61155700683594,
      "activations/layer5_attention_weight_min": -103.4033432006836,
      "activations/layer6_attention_weight_max": 60.8343620300293,
      "activations/layer6_attention_weight_min": -51.07305145263672,
      "activations/layer7_attention_weight_max": 55.0378532409668,
      "activations/layer7_attention_weight_min": -52.38172912597656,
      "activations/layer8_attention_weight_max": 41.82280349731445,
      "activations/layer8_attention_weight_min": -39.60396194458008,
      "activations/layer9_attention_weight_max": 56.01163864135742,
      "activations/layer9_attention_weight_min": -52.673004150390625,
      "epoch": 2.15,
      "learning_rate": 0.00013751969696969697,
      "loss": 2.9603,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_loss": 2.90625,
      "eval_runtime": 8.4655,
      "eval_samples_per_second": 507.236,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_openwebtext_loss": 2.90625,
      "eval_openwebtext_ppl": 18.28808948244362,
      "eval_openwebtext_runtime": 8.4655,
      "eval_openwebtext_samples_per_second": 507.236,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_wikitext_loss": 3.158203125,
      "eval_wikitext_ppl": 23.52828054362271,
      "eval_wikitext_runtime": 1.9382,
      "eval_wikitext_samples_per_second": 235.269,
      "step": 37000
    },
    {
      "epoch": 2.15,
      "eval_lambada_loss": 3.17578125,
      "eval_lambada_ppl": 23.94552000018515,
      "eval_lambada_runtime": 9.5562,
      "eval_lambada_samples_per_second": 509.512,
      "step": 37000
    },
    {
      "activations/layer0_attention_weight_max": 14.85330867767334,
      "activations/layer0_attention_weight_min": -13.557381629943848,
      "activations/layer10_attention_weight_max": 62.55417251586914,
      "activations/layer10_attention_weight_min": -44.619239807128906,
      "activations/layer11_attention_weight_max": 45.258018493652344,
      "activations/layer11_attention_weight_min": -32.93267822265625,
      "activations/layer12_attention_weight_max": 32.48456954956055,
      "activations/layer12_attention_weight_min": -25.267799377441406,
      "activations/layer13_attention_weight_max": 59.7878303527832,
      "activations/layer13_attention_weight_min": -45.039466857910156,
      "activations/layer14_attention_weight_max": 43.69234848022461,
      "activations/layer14_attention_weight_min": -32.33405303955078,
      "activations/layer15_attention_weight_max": 46.48406219482422,
      "activations/layer15_attention_weight_min": -34.74196243286133,
      "activations/layer16_attention_weight_max": 42.487552642822266,
      "activations/layer16_attention_weight_min": -27.777416229248047,
      "activations/layer17_attention_weight_max": 60.3209228515625,
      "activations/layer17_attention_weight_min": -40.79130172729492,
      "activations/layer18_attention_weight_max": 59.47168731689453,
      "activations/layer18_attention_weight_min": -33.829551696777344,
      "activations/layer19_attention_weight_max": 24.136262893676758,
      "activations/layer19_attention_weight_min": -17.38245964050293,
      "activations/layer1_attention_weight_max": 18.038572311401367,
      "activations/layer1_attention_weight_min": -13.676620483398438,
      "activations/layer20_attention_weight_max": 20.71533203125,
      "activations/layer20_attention_weight_min": -18.317981719970703,
      "activations/layer21_attention_weight_max": 42.575927734375,
      "activations/layer21_attention_weight_min": -26.018596649169922,
      "activations/layer22_attention_weight_max": 36.3334846496582,
      "activations/layer22_attention_weight_min": -25.561750411987305,
      "activations/layer23_attention_weight_max": 26.43471908569336,
      "activations/layer23_attention_weight_min": -16.14429473876953,
      "activations/layer2_attention_weight_max": 26.450727462768555,
      "activations/layer2_attention_weight_min": -26.020484924316406,
      "activations/layer3_attention_weight_max": 63.10761642456055,
      "activations/layer3_attention_weight_min": -65.81696319580078,
      "activations/layer4_attention_weight_max": 95.36650848388672,
      "activations/layer4_attention_weight_min": -84.415283203125,
      "activations/layer5_attention_weight_max": 122.63945770263672,
      "activations/layer5_attention_weight_min": -103.18233489990234,
      "activations/layer6_attention_weight_max": 59.70271682739258,
      "activations/layer6_attention_weight_min": -49.69218444824219,
      "activations/layer7_attention_weight_max": 58.51854705810547,
      "activations/layer7_attention_weight_min": -50.511260986328125,
      "activations/layer8_attention_weight_max": 43.36213684082031,
      "activations/layer8_attention_weight_min": -37.36963653564453,
      "activations/layer9_attention_weight_max": 59.7464599609375,
      "activations/layer9_attention_weight_min": -56.8651123046875,
      "epoch": 2.15,
      "learning_rate": 0.00013750075757575756,
      "loss": 2.9618,
      "step": 37050
    },
    {
      "activations/layer0_attention_weight_max": 15.142709732055664,
      "activations/layer0_attention_weight_min": -13.290544509887695,
      "activations/layer10_attention_weight_max": 55.24083709716797,
      "activations/layer10_attention_weight_min": -46.03860855102539,
      "activations/layer11_attention_weight_max": 52.82887268066406,
      "activations/layer11_attention_weight_min": -33.78026580810547,
      "activations/layer12_attention_weight_max": 32.89269256591797,
      "activations/layer12_attention_weight_min": -24.84477996826172,
      "activations/layer13_attention_weight_max": 67.38387298583984,
      "activations/layer13_attention_weight_min": -43.060306549072266,
      "activations/layer14_attention_weight_max": 44.404876708984375,
      "activations/layer14_attention_weight_min": -35.45010757446289,
      "activations/layer15_attention_weight_max": 62.83298110961914,
      "activations/layer15_attention_weight_min": -42.06138229370117,
      "activations/layer16_attention_weight_max": 39.50042724609375,
      "activations/layer16_attention_weight_min": -30.42284393310547,
      "activations/layer17_attention_weight_max": 62.94197463989258,
      "activations/layer17_attention_weight_min": -40.05393981933594,
      "activations/layer18_attention_weight_max": 55.780029296875,
      "activations/layer18_attention_weight_min": -35.0784797668457,
      "activations/layer19_attention_weight_max": 17.778139114379883,
      "activations/layer19_attention_weight_min": -14.765913963317871,
      "activations/layer1_attention_weight_max": 16.39912223815918,
      "activations/layer1_attention_weight_min": -13.846749305725098,
      "activations/layer20_attention_weight_max": 19.75646209716797,
      "activations/layer20_attention_weight_min": -15.73286247253418,
      "activations/layer21_attention_weight_max": 34.70500946044922,
      "activations/layer21_attention_weight_min": -25.213760375976562,
      "activations/layer22_attention_weight_max": 36.37571334838867,
      "activations/layer22_attention_weight_min": -23.5704345703125,
      "activations/layer23_attention_weight_max": 26.022571563720703,
      "activations/layer23_attention_weight_min": -17.62921905517578,
      "activations/layer2_attention_weight_max": 26.739919662475586,
      "activations/layer2_attention_weight_min": -25.14813232421875,
      "activations/layer3_attention_weight_max": 61.94816589355469,
      "activations/layer3_attention_weight_min": -66.39042663574219,
      "activations/layer4_attention_weight_max": 91.75127410888672,
      "activations/layer4_attention_weight_min": -83.82366180419922,
      "activations/layer5_attention_weight_max": 125.14562225341797,
      "activations/layer5_attention_weight_min": -102.60519409179688,
      "activations/layer6_attention_weight_max": 61.84207534790039,
      "activations/layer6_attention_weight_min": -51.77616500854492,
      "activations/layer7_attention_weight_max": 56.20966720581055,
      "activations/layer7_attention_weight_min": -52.32685089111328,
      "activations/layer8_attention_weight_max": 42.84433364868164,
      "activations/layer8_attention_weight_min": -36.642417907714844,
      "activations/layer9_attention_weight_max": 61.584327697753906,
      "activations/layer9_attention_weight_min": -55.93324661254883,
      "epoch": 2.16,
      "learning_rate": 0.00013748181818181817,
      "loss": 2.9562,
      "step": 37100
    },
    {
      "activations/layer0_attention_weight_max": 16.000364303588867,
      "activations/layer0_attention_weight_min": -14.04555606842041,
      "activations/layer10_attention_weight_max": 63.13493347167969,
      "activations/layer10_attention_weight_min": -46.229949951171875,
      "activations/layer11_attention_weight_max": 42.225772857666016,
      "activations/layer11_attention_weight_min": -34.19139862060547,
      "activations/layer12_attention_weight_max": 25.390329360961914,
      "activations/layer12_attention_weight_min": -24.797204971313477,
      "activations/layer13_attention_weight_max": 64.21884155273438,
      "activations/layer13_attention_weight_min": -43.808250427246094,
      "activations/layer14_attention_weight_max": 61.26806640625,
      "activations/layer14_attention_weight_min": -44.54251480102539,
      "activations/layer15_attention_weight_max": 52.17445373535156,
      "activations/layer15_attention_weight_min": -36.53227615356445,
      "activations/layer16_attention_weight_max": 42.62890625,
      "activations/layer16_attention_weight_min": -33.77688217163086,
      "activations/layer17_attention_weight_max": 63.55411911010742,
      "activations/layer17_attention_weight_min": -40.91447830200195,
      "activations/layer18_attention_weight_max": 57.35147476196289,
      "activations/layer18_attention_weight_min": -33.89794921875,
      "activations/layer19_attention_weight_max": 22.94194221496582,
      "activations/layer19_attention_weight_min": -15.285600662231445,
      "activations/layer1_attention_weight_max": 16.47968864440918,
      "activations/layer1_attention_weight_min": -14.07175064086914,
      "activations/layer20_attention_weight_max": 20.870006561279297,
      "activations/layer20_attention_weight_min": -18.06564712524414,
      "activations/layer21_attention_weight_max": 38.552879333496094,
      "activations/layer21_attention_weight_min": -26.261383056640625,
      "activations/layer22_attention_weight_max": 39.175437927246094,
      "activations/layer22_attention_weight_min": -24.03679847717285,
      "activations/layer23_attention_weight_max": 24.716856002807617,
      "activations/layer23_attention_weight_min": -14.956033706665039,
      "activations/layer2_attention_weight_max": 27.623088836669922,
      "activations/layer2_attention_weight_min": -27.52191925048828,
      "activations/layer3_attention_weight_max": 63.967857360839844,
      "activations/layer3_attention_weight_min": -67.3907241821289,
      "activations/layer4_attention_weight_max": 97.1930923461914,
      "activations/layer4_attention_weight_min": -85.23187255859375,
      "activations/layer5_attention_weight_max": 123.77458190917969,
      "activations/layer5_attention_weight_min": -100.29246520996094,
      "activations/layer6_attention_weight_max": 63.40476989746094,
      "activations/layer6_attention_weight_min": -47.26823043823242,
      "activations/layer7_attention_weight_max": 56.81129455566406,
      "activations/layer7_attention_weight_min": -52.45612716674805,
      "activations/layer8_attention_weight_max": 44.180416107177734,
      "activations/layer8_attention_weight_min": -37.25725555419922,
      "activations/layer9_attention_weight_max": 62.79936599731445,
      "activations/layer9_attention_weight_min": -52.616512298583984,
      "epoch": 2.16,
      "learning_rate": 0.0001374628787878788,
      "loss": 2.9393,
      "step": 37150
    },
    {
      "activations/layer0_attention_weight_max": 14.125968933105469,
      "activations/layer0_attention_weight_min": -13.245445251464844,
      "activations/layer10_attention_weight_max": 61.895164489746094,
      "activations/layer10_attention_weight_min": -50.3143424987793,
      "activations/layer11_attention_weight_max": 44.6468505859375,
      "activations/layer11_attention_weight_min": -31.87963104248047,
      "activations/layer12_attention_weight_max": 33.31097412109375,
      "activations/layer12_attention_weight_min": -23.34621810913086,
      "activations/layer13_attention_weight_max": 68.09603881835938,
      "activations/layer13_attention_weight_min": -41.19142150878906,
      "activations/layer14_attention_weight_max": 51.16298294067383,
      "activations/layer14_attention_weight_min": -33.92502212524414,
      "activations/layer15_attention_weight_max": 57.13591384887695,
      "activations/layer15_attention_weight_min": -38.71016311645508,
      "activations/layer16_attention_weight_max": 41.00928497314453,
      "activations/layer16_attention_weight_min": -30.238935470581055,
      "activations/layer17_attention_weight_max": 71.3026351928711,
      "activations/layer17_attention_weight_min": -38.76116180419922,
      "activations/layer18_attention_weight_max": 61.75585174560547,
      "activations/layer18_attention_weight_min": -34.176048278808594,
      "activations/layer19_attention_weight_max": 25.27519416809082,
      "activations/layer19_attention_weight_min": -16.307424545288086,
      "activations/layer1_attention_weight_max": 16.691415786743164,
      "activations/layer1_attention_weight_min": -13.545783996582031,
      "activations/layer20_attention_weight_max": 21.304655075073242,
      "activations/layer20_attention_weight_min": -16.36883544921875,
      "activations/layer21_attention_weight_max": 47.4374885559082,
      "activations/layer21_attention_weight_min": -26.951921463012695,
      "activations/layer22_attention_weight_max": 37.20303726196289,
      "activations/layer22_attention_weight_min": -25.516273498535156,
      "activations/layer23_attention_weight_max": 28.89967155456543,
      "activations/layer23_attention_weight_min": -17.513254165649414,
      "activations/layer2_attention_weight_max": 28.256561279296875,
      "activations/layer2_attention_weight_min": -26.833656311035156,
      "activations/layer3_attention_weight_max": 63.13731384277344,
      "activations/layer3_attention_weight_min": -66.17050170898438,
      "activations/layer4_attention_weight_max": 100.78446197509766,
      "activations/layer4_attention_weight_min": -83.07205200195312,
      "activations/layer5_attention_weight_max": 130.86614990234375,
      "activations/layer5_attention_weight_min": -106.47068786621094,
      "activations/layer6_attention_weight_max": 65.82669067382812,
      "activations/layer6_attention_weight_min": -53.0705680847168,
      "activations/layer7_attention_weight_max": 61.498756408691406,
      "activations/layer7_attention_weight_min": -50.68777084350586,
      "activations/layer8_attention_weight_max": 46.92633819580078,
      "activations/layer8_attention_weight_min": -39.11679458618164,
      "activations/layer9_attention_weight_max": 62.77802276611328,
      "activations/layer9_attention_weight_min": -53.3731803894043,
      "epoch": 2.16,
      "learning_rate": 0.00013744393939393938,
      "loss": 2.9457,
      "step": 37200
    },
    {
      "activations/layer0_attention_weight_max": 14.291757583618164,
      "activations/layer0_attention_weight_min": -13.670705795288086,
      "activations/layer10_attention_weight_max": 57.530086517333984,
      "activations/layer10_attention_weight_min": -54.400047302246094,
      "activations/layer11_attention_weight_max": 45.06789779663086,
      "activations/layer11_attention_weight_min": -33.54518508911133,
      "activations/layer12_attention_weight_max": 27.050342559814453,
      "activations/layer12_attention_weight_min": -26.834348678588867,
      "activations/layer13_attention_weight_max": 66.37256622314453,
      "activations/layer13_attention_weight_min": -45.74752426147461,
      "activations/layer14_attention_weight_max": 61.96919250488281,
      "activations/layer14_attention_weight_min": -46.07350158691406,
      "activations/layer15_attention_weight_max": 55.722076416015625,
      "activations/layer15_attention_weight_min": -37.25138854980469,
      "activations/layer16_attention_weight_max": 40.14854431152344,
      "activations/layer16_attention_weight_min": -27.439697265625,
      "activations/layer17_attention_weight_max": 60.83707046508789,
      "activations/layer17_attention_weight_min": -35.92759704589844,
      "activations/layer18_attention_weight_max": 55.07712173461914,
      "activations/layer18_attention_weight_min": -36.600555419921875,
      "activations/layer19_attention_weight_max": 22.115568161010742,
      "activations/layer19_attention_weight_min": -17.62381362915039,
      "activations/layer1_attention_weight_max": 16.489187240600586,
      "activations/layer1_attention_weight_min": -14.892475128173828,
      "activations/layer20_attention_weight_max": 20.875885009765625,
      "activations/layer20_attention_weight_min": -17.13039779663086,
      "activations/layer21_attention_weight_max": 38.005531311035156,
      "activations/layer21_attention_weight_min": -23.91806411743164,
      "activations/layer22_attention_weight_max": 37.43173599243164,
      "activations/layer22_attention_weight_min": -22.335296630859375,
      "activations/layer23_attention_weight_max": 23.480175018310547,
      "activations/layer23_attention_weight_min": -15.828977584838867,
      "activations/layer2_attention_weight_max": 28.29549789428711,
      "activations/layer2_attention_weight_min": -28.74160385131836,
      "activations/layer3_attention_weight_max": 64.1895751953125,
      "activations/layer3_attention_weight_min": -64.41647338867188,
      "activations/layer4_attention_weight_max": 96.39595031738281,
      "activations/layer4_attention_weight_min": -82.07561492919922,
      "activations/layer5_attention_weight_max": 120.27499389648438,
      "activations/layer5_attention_weight_min": -99.747314453125,
      "activations/layer6_attention_weight_max": 59.80012893676758,
      "activations/layer6_attention_weight_min": -50.7962760925293,
      "activations/layer7_attention_weight_max": 54.389129638671875,
      "activations/layer7_attention_weight_min": -51.63447189331055,
      "activations/layer8_attention_weight_max": 43.23555374145508,
      "activations/layer8_attention_weight_min": -38.028465270996094,
      "activations/layer9_attention_weight_max": 57.53765869140625,
      "activations/layer9_attention_weight_min": -51.60309982299805,
      "epoch": 2.16,
      "learning_rate": 0.000137425,
      "loss": 2.9714,
      "step": 37250
    },
    {
      "activations/layer0_attention_weight_max": 14.817240715026855,
      "activations/layer0_attention_weight_min": -13.504064559936523,
      "activations/layer10_attention_weight_max": 66.51504516601562,
      "activations/layer10_attention_weight_min": -45.95415115356445,
      "activations/layer11_attention_weight_max": 55.43928146362305,
      "activations/layer11_attention_weight_min": -35.31883239746094,
      "activations/layer12_attention_weight_max": 39.00520706176758,
      "activations/layer12_attention_weight_min": -26.038286209106445,
      "activations/layer13_attention_weight_max": 97.30033874511719,
      "activations/layer13_attention_weight_min": -47.318668365478516,
      "activations/layer14_attention_weight_max": 119.54481506347656,
      "activations/layer14_attention_weight_min": -49.008148193359375,
      "activations/layer15_attention_weight_max": 66.7157211303711,
      "activations/layer15_attention_weight_min": -36.51538848876953,
      "activations/layer16_attention_weight_max": 51.90712356567383,
      "activations/layer16_attention_weight_min": -28.158103942871094,
      "activations/layer17_attention_weight_max": 67.43983459472656,
      "activations/layer17_attention_weight_min": -36.05098342895508,
      "activations/layer18_attention_weight_max": 66.41974639892578,
      "activations/layer18_attention_weight_min": -33.98611068725586,
      "activations/layer19_attention_weight_max": 22.011707305908203,
      "activations/layer19_attention_weight_min": -17.305648803710938,
      "activations/layer1_attention_weight_max": 16.698244094848633,
      "activations/layer1_attention_weight_min": -13.630170822143555,
      "activations/layer20_attention_weight_max": 20.58582878112793,
      "activations/layer20_attention_weight_min": -17.231956481933594,
      "activations/layer21_attention_weight_max": 42.417720794677734,
      "activations/layer21_attention_weight_min": -26.004934310913086,
      "activations/layer22_attention_weight_max": 35.49441146850586,
      "activations/layer22_attention_weight_min": -20.455717086791992,
      "activations/layer23_attention_weight_max": 23.03023338317871,
      "activations/layer23_attention_weight_min": -16.13690948486328,
      "activations/layer2_attention_weight_max": 28.711524963378906,
      "activations/layer2_attention_weight_min": -27.002857208251953,
      "activations/layer3_attention_weight_max": 69.731689453125,
      "activations/layer3_attention_weight_min": -72.1497802734375,
      "activations/layer4_attention_weight_max": 102.6088638305664,
      "activations/layer4_attention_weight_min": -92.7857437133789,
      "activations/layer5_attention_weight_max": 146.44430541992188,
      "activations/layer5_attention_weight_min": -119.43936157226562,
      "activations/layer6_attention_weight_max": 72.18843841552734,
      "activations/layer6_attention_weight_min": -56.23546600341797,
      "activations/layer7_attention_weight_max": 70.84986877441406,
      "activations/layer7_attention_weight_min": -59.94845199584961,
      "activations/layer8_attention_weight_max": 66.85006713867188,
      "activations/layer8_attention_weight_min": -45.59215545654297,
      "activations/layer9_attention_weight_max": 86.77753448486328,
      "activations/layer9_attention_weight_min": -62.08844757080078,
      "epoch": 2.17,
      "learning_rate": 0.00013740606060606058,
      "loss": 2.9594,
      "step": 37300
    },
    {
      "activations/layer0_attention_weight_max": 14.066374778747559,
      "activations/layer0_attention_weight_min": -13.976994514465332,
      "activations/layer10_attention_weight_max": 56.04434585571289,
      "activations/layer10_attention_weight_min": -45.88264465332031,
      "activations/layer11_attention_weight_max": 44.98539352416992,
      "activations/layer11_attention_weight_min": -36.045867919921875,
      "activations/layer12_attention_weight_max": 29.724777221679688,
      "activations/layer12_attention_weight_min": -24.090682983398438,
      "activations/layer13_attention_weight_max": 86.49696350097656,
      "activations/layer13_attention_weight_min": -60.43071365356445,
      "activations/layer14_attention_weight_max": 78.27664184570312,
      "activations/layer14_attention_weight_min": -50.970088958740234,
      "activations/layer15_attention_weight_max": 61.481868743896484,
      "activations/layer15_attention_weight_min": -35.958805084228516,
      "activations/layer16_attention_weight_max": 43.59604263305664,
      "activations/layer16_attention_weight_min": -27.34398651123047,
      "activations/layer17_attention_weight_max": 68.15739440917969,
      "activations/layer17_attention_weight_min": -39.82188415527344,
      "activations/layer18_attention_weight_max": 57.7567024230957,
      "activations/layer18_attention_weight_min": -34.61931610107422,
      "activations/layer19_attention_weight_max": 25.081329345703125,
      "activations/layer19_attention_weight_min": -17.90098762512207,
      "activations/layer1_attention_weight_max": 15.759842872619629,
      "activations/layer1_attention_weight_min": -14.629742622375488,
      "activations/layer20_attention_weight_max": 23.65411949157715,
      "activations/layer20_attention_weight_min": -16.744544982910156,
      "activations/layer21_attention_weight_max": 46.95878219604492,
      "activations/layer21_attention_weight_min": -27.3282470703125,
      "activations/layer22_attention_weight_max": 39.87083053588867,
      "activations/layer22_attention_weight_min": -22.151580810546875,
      "activations/layer23_attention_weight_max": 27.91482162475586,
      "activations/layer23_attention_weight_min": -15.38630199432373,
      "activations/layer2_attention_weight_max": 27.008472442626953,
      "activations/layer2_attention_weight_min": -27.12592887878418,
      "activations/layer3_attention_weight_max": 63.19123077392578,
      "activations/layer3_attention_weight_min": -65.79866790771484,
      "activations/layer4_attention_weight_max": 91.57073211669922,
      "activations/layer4_attention_weight_min": -88.11055755615234,
      "activations/layer5_attention_weight_max": 119.43252563476562,
      "activations/layer5_attention_weight_min": -102.15692138671875,
      "activations/layer6_attention_weight_max": 59.21544647216797,
      "activations/layer6_attention_weight_min": -49.14179992675781,
      "activations/layer7_attention_weight_max": 54.40629196166992,
      "activations/layer7_attention_weight_min": -52.14357376098633,
      "activations/layer8_attention_weight_max": 45.79057693481445,
      "activations/layer8_attention_weight_min": -40.67884826660156,
      "activations/layer9_attention_weight_max": 69.78096771240234,
      "activations/layer9_attention_weight_min": -58.39502716064453,
      "epoch": 2.17,
      "learning_rate": 0.0001373871212121212,
      "loss": 2.9539,
      "step": 37350
    },
    {
      "activations/layer0_attention_weight_max": 13.434368133544922,
      "activations/layer0_attention_weight_min": -13.777200698852539,
      "activations/layer10_attention_weight_max": 53.201171875,
      "activations/layer10_attention_weight_min": -43.31826400756836,
      "activations/layer11_attention_weight_max": 42.6277961730957,
      "activations/layer11_attention_weight_min": -32.026302337646484,
      "activations/layer12_attention_weight_max": 30.60495376586914,
      "activations/layer12_attention_weight_min": -23.107166290283203,
      "activations/layer13_attention_weight_max": 54.27216339111328,
      "activations/layer13_attention_weight_min": -39.779541015625,
      "activations/layer14_attention_weight_max": 51.04216384887695,
      "activations/layer14_attention_weight_min": -35.731136322021484,
      "activations/layer15_attention_weight_max": 50.19203186035156,
      "activations/layer15_attention_weight_min": -36.43540954589844,
      "activations/layer16_attention_weight_max": 43.24527359008789,
      "activations/layer16_attention_weight_min": -27.849445343017578,
      "activations/layer17_attention_weight_max": 66.94792175292969,
      "activations/layer17_attention_weight_min": -38.12820053100586,
      "activations/layer18_attention_weight_max": 56.01033020019531,
      "activations/layer18_attention_weight_min": -34.371437072753906,
      "activations/layer19_attention_weight_max": 19.98492431640625,
      "activations/layer19_attention_weight_min": -17.503501892089844,
      "activations/layer1_attention_weight_max": 17.175132751464844,
      "activations/layer1_attention_weight_min": -14.760196685791016,
      "activations/layer20_attention_weight_max": 21.472820281982422,
      "activations/layer20_attention_weight_min": -19.26224708557129,
      "activations/layer21_attention_weight_max": 41.547706604003906,
      "activations/layer21_attention_weight_min": -26.887100219726562,
      "activations/layer22_attention_weight_max": 33.258384704589844,
      "activations/layer22_attention_weight_min": -21.759103775024414,
      "activations/layer23_attention_weight_max": 22.371177673339844,
      "activations/layer23_attention_weight_min": -16.523311614990234,
      "activations/layer2_attention_weight_max": 27.27764129638672,
      "activations/layer2_attention_weight_min": -26.850521087646484,
      "activations/layer3_attention_weight_max": 63.046878814697266,
      "activations/layer3_attention_weight_min": -67.07476043701172,
      "activations/layer4_attention_weight_max": 89.6887435913086,
      "activations/layer4_attention_weight_min": -82.8628921508789,
      "activations/layer5_attention_weight_max": 123.02639770507812,
      "activations/layer5_attention_weight_min": -105.30818939208984,
      "activations/layer6_attention_weight_max": 59.64985656738281,
      "activations/layer6_attention_weight_min": -50.536827087402344,
      "activations/layer7_attention_weight_max": 58.191322326660156,
      "activations/layer7_attention_weight_min": -48.84001922607422,
      "activations/layer8_attention_weight_max": 41.59758377075195,
      "activations/layer8_attention_weight_min": -39.51390075683594,
      "activations/layer9_attention_weight_max": 53.848182678222656,
      "activations/layer9_attention_weight_min": -49.62949752807617,
      "epoch": 2.17,
      "learning_rate": 0.0001373681818181818,
      "loss": 2.9543,
      "step": 37400
    },
    {
      "activations/layer0_attention_weight_max": 15.022229194641113,
      "activations/layer0_attention_weight_min": -13.624632835388184,
      "activations/layer10_attention_weight_max": 59.60393524169922,
      "activations/layer10_attention_weight_min": -47.40998458862305,
      "activations/layer11_attention_weight_max": 43.41089630126953,
      "activations/layer11_attention_weight_min": -34.83118438720703,
      "activations/layer12_attention_weight_max": 32.257415771484375,
      "activations/layer12_attention_weight_min": -25.518144607543945,
      "activations/layer13_attention_weight_max": 53.58904266357422,
      "activations/layer13_attention_weight_min": -40.52312088012695,
      "activations/layer14_attention_weight_max": 47.35673141479492,
      "activations/layer14_attention_weight_min": -34.92587661743164,
      "activations/layer15_attention_weight_max": 48.45771789550781,
      "activations/layer15_attention_weight_min": -34.38935852050781,
      "activations/layer16_attention_weight_max": 38.08481216430664,
      "activations/layer16_attention_weight_min": -27.87206268310547,
      "activations/layer17_attention_weight_max": 59.32568359375,
      "activations/layer17_attention_weight_min": -38.530059814453125,
      "activations/layer18_attention_weight_max": 54.89147186279297,
      "activations/layer18_attention_weight_min": -32.39673614501953,
      "activations/layer19_attention_weight_max": 22.05547523498535,
      "activations/layer19_attention_weight_min": -15.670465469360352,
      "activations/layer1_attention_weight_max": 16.1822452545166,
      "activations/layer1_attention_weight_min": -14.229598045349121,
      "activations/layer20_attention_weight_max": 19.914859771728516,
      "activations/layer20_attention_weight_min": -16.762371063232422,
      "activations/layer21_attention_weight_max": 36.53363800048828,
      "activations/layer21_attention_weight_min": -29.962417602539062,
      "activations/layer22_attention_weight_max": 30.316370010375977,
      "activations/layer22_attention_weight_min": -21.886438369750977,
      "activations/layer23_attention_weight_max": 24.062164306640625,
      "activations/layer23_attention_weight_min": -13.788115501403809,
      "activations/layer2_attention_weight_max": 27.412979125976562,
      "activations/layer2_attention_weight_min": -25.81074333190918,
      "activations/layer3_attention_weight_max": 62.35211181640625,
      "activations/layer3_attention_weight_min": -68.51881408691406,
      "activations/layer4_attention_weight_max": 92.24024200439453,
      "activations/layer4_attention_weight_min": -93.06979370117188,
      "activations/layer5_attention_weight_max": 120.79750061035156,
      "activations/layer5_attention_weight_min": -112.63931274414062,
      "activations/layer6_attention_weight_max": 61.64293670654297,
      "activations/layer6_attention_weight_min": -54.30652618408203,
      "activations/layer7_attention_weight_max": 59.64031982421875,
      "activations/layer7_attention_weight_min": -53.55887222290039,
      "activations/layer8_attention_weight_max": 46.98704528808594,
      "activations/layer8_attention_weight_min": -39.44389724731445,
      "activations/layer9_attention_weight_max": 76.73522186279297,
      "activations/layer9_attention_weight_min": -58.181617736816406,
      "epoch": 2.18,
      "learning_rate": 0.0001373492424242424,
      "loss": 2.9487,
      "step": 37450
    },
    {
      "activations/layer0_attention_weight_max": 14.537822723388672,
      "activations/layer0_attention_weight_min": -14.129183769226074,
      "activations/layer10_attention_weight_max": 56.32505416870117,
      "activations/layer10_attention_weight_min": -43.18101119995117,
      "activations/layer11_attention_weight_max": 42.61698913574219,
      "activations/layer11_attention_weight_min": -32.40478515625,
      "activations/layer12_attention_weight_max": 27.509235382080078,
      "activations/layer12_attention_weight_min": -26.66533851623535,
      "activations/layer13_attention_weight_max": 67.32527923583984,
      "activations/layer13_attention_weight_min": -41.460689544677734,
      "activations/layer14_attention_weight_max": 73.27560424804688,
      "activations/layer14_attention_weight_min": -41.534088134765625,
      "activations/layer15_attention_weight_max": 49.916358947753906,
      "activations/layer15_attention_weight_min": -37.90432357788086,
      "activations/layer16_attention_weight_max": 39.07582092285156,
      "activations/layer16_attention_weight_min": -28.28662109375,
      "activations/layer17_attention_weight_max": 59.97865676879883,
      "activations/layer17_attention_weight_min": -35.502906799316406,
      "activations/layer18_attention_weight_max": 53.248008728027344,
      "activations/layer18_attention_weight_min": -32.79155349731445,
      "activations/layer19_attention_weight_max": 22.14902687072754,
      "activations/layer19_attention_weight_min": -15.870834350585938,
      "activations/layer1_attention_weight_max": 15.70920467376709,
      "activations/layer1_attention_weight_min": -14.91287899017334,
      "activations/layer20_attention_weight_max": 20.265056610107422,
      "activations/layer20_attention_weight_min": -17.672178268432617,
      "activations/layer21_attention_weight_max": 43.207191467285156,
      "activations/layer21_attention_weight_min": -25.96910858154297,
      "activations/layer22_attention_weight_max": 34.584129333496094,
      "activations/layer22_attention_weight_min": -22.647777557373047,
      "activations/layer23_attention_weight_max": 24.130653381347656,
      "activations/layer23_attention_weight_min": -13.877355575561523,
      "activations/layer2_attention_weight_max": 27.355070114135742,
      "activations/layer2_attention_weight_min": -27.63033676147461,
      "activations/layer3_attention_weight_max": 68.2882308959961,
      "activations/layer3_attention_weight_min": -67.15257263183594,
      "activations/layer4_attention_weight_max": 96.37957763671875,
      "activations/layer4_attention_weight_min": -83.86089324951172,
      "activations/layer5_attention_weight_max": 127.57311248779297,
      "activations/layer5_attention_weight_min": -102.16580200195312,
      "activations/layer6_attention_weight_max": 62.88029861450195,
      "activations/layer6_attention_weight_min": -50.994991302490234,
      "activations/layer7_attention_weight_max": 61.16508483886719,
      "activations/layer7_attention_weight_min": -51.72234344482422,
      "activations/layer8_attention_weight_max": 45.776790618896484,
      "activations/layer8_attention_weight_min": -39.203285217285156,
      "activations/layer9_attention_weight_max": 71.29683685302734,
      "activations/layer9_attention_weight_min": -54.979427337646484,
      "epoch": 2.18,
      "learning_rate": 0.00013733030303030301,
      "loss": 2.9596,
      "step": 37500
    },
    {
      "activations/layer0_attention_weight_max": 13.633039474487305,
      "activations/layer0_attention_weight_min": -13.667320251464844,
      "activations/layer10_attention_weight_max": 55.89414596557617,
      "activations/layer10_attention_weight_min": -45.01054763793945,
      "activations/layer11_attention_weight_max": 37.82202911376953,
      "activations/layer11_attention_weight_min": -31.729061126708984,
      "activations/layer12_attention_weight_max": 28.55009651184082,
      "activations/layer12_attention_weight_min": -23.732471466064453,
      "activations/layer13_attention_weight_max": 54.39192581176758,
      "activations/layer13_attention_weight_min": -42.175872802734375,
      "activations/layer14_attention_weight_max": 44.95973205566406,
      "activations/layer14_attention_weight_min": -36.10936737060547,
      "activations/layer15_attention_weight_max": 48.131629943847656,
      "activations/layer15_attention_weight_min": -36.426937103271484,
      "activations/layer16_attention_weight_max": 40.53598403930664,
      "activations/layer16_attention_weight_min": -29.653444290161133,
      "activations/layer17_attention_weight_max": 69.95000457763672,
      "activations/layer17_attention_weight_min": -40.0238037109375,
      "activations/layer18_attention_weight_max": 51.35304641723633,
      "activations/layer18_attention_weight_min": -34.33605194091797,
      "activations/layer19_attention_weight_max": 22.436723709106445,
      "activations/layer19_attention_weight_min": -16.420106887817383,
      "activations/layer1_attention_weight_max": 16.180044174194336,
      "activations/layer1_attention_weight_min": -12.793455123901367,
      "activations/layer20_attention_weight_max": 20.886014938354492,
      "activations/layer20_attention_weight_min": -19.735658645629883,
      "activations/layer21_attention_weight_max": 45.33272933959961,
      "activations/layer21_attention_weight_min": -29.204954147338867,
      "activations/layer22_attention_weight_max": 38.85822677612305,
      "activations/layer22_attention_weight_min": -23.38324737548828,
      "activations/layer23_attention_weight_max": 23.992408752441406,
      "activations/layer23_attention_weight_min": -16.15863609313965,
      "activations/layer2_attention_weight_max": 28.0063533782959,
      "activations/layer2_attention_weight_min": -27.9320125579834,
      "activations/layer3_attention_weight_max": 61.24748992919922,
      "activations/layer3_attention_weight_min": -66.54536437988281,
      "activations/layer4_attention_weight_max": 94.20880889892578,
      "activations/layer4_attention_weight_min": -85.6287612915039,
      "activations/layer5_attention_weight_max": 128.32119750976562,
      "activations/layer5_attention_weight_min": -101.14505004882812,
      "activations/layer6_attention_weight_max": 62.93681716918945,
      "activations/layer6_attention_weight_min": -53.48963928222656,
      "activations/layer7_attention_weight_max": 57.97201919555664,
      "activations/layer7_attention_weight_min": -49.89534378051758,
      "activations/layer8_attention_weight_max": 43.78822326660156,
      "activations/layer8_attention_weight_min": -37.585391998291016,
      "activations/layer9_attention_weight_max": 59.11533737182617,
      "activations/layer9_attention_weight_min": -52.515045166015625,
      "epoch": 2.18,
      "learning_rate": 0.00013731136363636363,
      "loss": 2.9533,
      "step": 37550
    },
    {
      "activations/layer0_attention_weight_max": 14.38622760772705,
      "activations/layer0_attention_weight_min": -14.124837875366211,
      "activations/layer10_attention_weight_max": 52.45814895629883,
      "activations/layer10_attention_weight_min": -44.77305221557617,
      "activations/layer11_attention_weight_max": 39.63616943359375,
      "activations/layer11_attention_weight_min": -31.728809356689453,
      "activations/layer12_attention_weight_max": 25.79503631591797,
      "activations/layer12_attention_weight_min": -21.899152755737305,
      "activations/layer13_attention_weight_max": 61.750701904296875,
      "activations/layer13_attention_weight_min": -45.9533805847168,
      "activations/layer14_attention_weight_max": 44.03133010864258,
      "activations/layer14_attention_weight_min": -39.91757583618164,
      "activations/layer15_attention_weight_max": 51.81706619262695,
      "activations/layer15_attention_weight_min": -41.286109924316406,
      "activations/layer16_attention_weight_max": 36.76300811767578,
      "activations/layer16_attention_weight_min": -27.342803955078125,
      "activations/layer17_attention_weight_max": 58.0732421875,
      "activations/layer17_attention_weight_min": -37.36636734008789,
      "activations/layer18_attention_weight_max": 49.019596099853516,
      "activations/layer18_attention_weight_min": -32.0133171081543,
      "activations/layer19_attention_weight_max": 20.234390258789062,
      "activations/layer19_attention_weight_min": -16.159759521484375,
      "activations/layer1_attention_weight_max": 16.770448684692383,
      "activations/layer1_attention_weight_min": -12.778191566467285,
      "activations/layer20_attention_weight_max": 18.420822143554688,
      "activations/layer20_attention_weight_min": -17.446840286254883,
      "activations/layer21_attention_weight_max": 36.16923904418945,
      "activations/layer21_attention_weight_min": -23.73650550842285,
      "activations/layer22_attention_weight_max": 33.317684173583984,
      "activations/layer22_attention_weight_min": -21.87483787536621,
      "activations/layer23_attention_weight_max": 25.72322654724121,
      "activations/layer23_attention_weight_min": -14.979033470153809,
      "activations/layer2_attention_weight_max": 26.481769561767578,
      "activations/layer2_attention_weight_min": -26.267200469970703,
      "activations/layer3_attention_weight_max": 62.31471633911133,
      "activations/layer3_attention_weight_min": -65.54013061523438,
      "activations/layer4_attention_weight_max": 94.43578338623047,
      "activations/layer4_attention_weight_min": -84.8367691040039,
      "activations/layer5_attention_weight_max": 122.2586898803711,
      "activations/layer5_attention_weight_min": -106.49583435058594,
      "activations/layer6_attention_weight_max": 65.19020080566406,
      "activations/layer6_attention_weight_min": -56.27582550048828,
      "activations/layer7_attention_weight_max": 56.50448226928711,
      "activations/layer7_attention_weight_min": -50.91667938232422,
      "activations/layer8_attention_weight_max": 41.25838088989258,
      "activations/layer8_attention_weight_min": -39.09147644042969,
      "activations/layer9_attention_weight_max": 56.867366790771484,
      "activations/layer9_attention_weight_min": -52.77531814575195,
      "epoch": 2.18,
      "learning_rate": 0.00013729242424242422,
      "loss": 2.9474,
      "step": 37600
    },
    {
      "activations/layer0_attention_weight_max": 14.85028076171875,
      "activations/layer0_attention_weight_min": -13.920063018798828,
      "activations/layer10_attention_weight_max": 54.92841339111328,
      "activations/layer10_attention_weight_min": -44.30359649658203,
      "activations/layer11_attention_weight_max": 42.749229431152344,
      "activations/layer11_attention_weight_min": -33.264495849609375,
      "activations/layer12_attention_weight_max": 24.062437057495117,
      "activations/layer12_attention_weight_min": -22.52318572998047,
      "activations/layer13_attention_weight_max": 74.20240783691406,
      "activations/layer13_attention_weight_min": -46.189327239990234,
      "activations/layer14_attention_weight_max": 75.88839721679688,
      "activations/layer14_attention_weight_min": -40.545223236083984,
      "activations/layer15_attention_weight_max": 52.67889404296875,
      "activations/layer15_attention_weight_min": -36.6297721862793,
      "activations/layer16_attention_weight_max": 41.37525939941406,
      "activations/layer16_attention_weight_min": -27.716602325439453,
      "activations/layer17_attention_weight_max": 66.43863677978516,
      "activations/layer17_attention_weight_min": -38.3897590637207,
      "activations/layer18_attention_weight_max": 55.49425506591797,
      "activations/layer18_attention_weight_min": -34.119873046875,
      "activations/layer19_attention_weight_max": 23.365034103393555,
      "activations/layer19_attention_weight_min": -16.40651512145996,
      "activations/layer1_attention_weight_max": 16.090377807617188,
      "activations/layer1_attention_weight_min": -13.6842041015625,
      "activations/layer20_attention_weight_max": 20.693857192993164,
      "activations/layer20_attention_weight_min": -18.654743194580078,
      "activations/layer21_attention_weight_max": 38.97236251831055,
      "activations/layer21_attention_weight_min": -26.309810638427734,
      "activations/layer22_attention_weight_max": 31.866519927978516,
      "activations/layer22_attention_weight_min": -22.72804069519043,
      "activations/layer23_attention_weight_max": 22.66781997680664,
      "activations/layer23_attention_weight_min": -14.474308013916016,
      "activations/layer2_attention_weight_max": 26.722820281982422,
      "activations/layer2_attention_weight_min": -27.075443267822266,
      "activations/layer3_attention_weight_max": 62.868587493896484,
      "activations/layer3_attention_weight_min": -68.81814575195312,
      "activations/layer4_attention_weight_max": 93.59728240966797,
      "activations/layer4_attention_weight_min": -88.30917358398438,
      "activations/layer5_attention_weight_max": 124.35336303710938,
      "activations/layer5_attention_weight_min": -104.819580078125,
      "activations/layer6_attention_weight_max": 66.93687438964844,
      "activations/layer6_attention_weight_min": -51.35903549194336,
      "activations/layer7_attention_weight_max": 58.07190704345703,
      "activations/layer7_attention_weight_min": -50.74258041381836,
      "activations/layer8_attention_weight_max": 45.25038528442383,
      "activations/layer8_attention_weight_min": -36.133052825927734,
      "activations/layer9_attention_weight_max": 63.180145263671875,
      "activations/layer9_attention_weight_min": -52.8575439453125,
      "epoch": 2.19,
      "learning_rate": 0.00013727348484848483,
      "loss": 2.9549,
      "step": 37650
    },
    {
      "activations/layer0_attention_weight_max": 13.338812828063965,
      "activations/layer0_attention_weight_min": -14.059425354003906,
      "activations/layer10_attention_weight_max": 58.82548522949219,
      "activations/layer10_attention_weight_min": -49.732269287109375,
      "activations/layer11_attention_weight_max": 48.058448791503906,
      "activations/layer11_attention_weight_min": -34.29973220825195,
      "activations/layer12_attention_weight_max": 30.722896575927734,
      "activations/layer12_attention_weight_min": -24.184295654296875,
      "activations/layer13_attention_weight_max": 62.5520133972168,
      "activations/layer13_attention_weight_min": -47.4283332824707,
      "activations/layer14_attention_weight_max": 58.365745544433594,
      "activations/layer14_attention_weight_min": -44.8647575378418,
      "activations/layer15_attention_weight_max": 53.00910568237305,
      "activations/layer15_attention_weight_min": -40.0900764465332,
      "activations/layer16_attention_weight_max": 46.04042053222656,
      "activations/layer16_attention_weight_min": -29.62421417236328,
      "activations/layer17_attention_weight_max": 64.09233093261719,
      "activations/layer17_attention_weight_min": -37.72138977050781,
      "activations/layer18_attention_weight_max": 74.26007080078125,
      "activations/layer18_attention_weight_min": -34.798641204833984,
      "activations/layer19_attention_weight_max": 24.133710861206055,
      "activations/layer19_attention_weight_min": -15.958407402038574,
      "activations/layer1_attention_weight_max": 16.232860565185547,
      "activations/layer1_attention_weight_min": -13.471632957458496,
      "activations/layer20_attention_weight_max": 23.213998794555664,
      "activations/layer20_attention_weight_min": -16.670461654663086,
      "activations/layer21_attention_weight_max": 48.047691345214844,
      "activations/layer21_attention_weight_min": -30.521230697631836,
      "activations/layer22_attention_weight_max": 43.87358856201172,
      "activations/layer22_attention_weight_min": -23.540283203125,
      "activations/layer23_attention_weight_max": 28.101362228393555,
      "activations/layer23_attention_weight_min": -16.083919525146484,
      "activations/layer2_attention_weight_max": 26.193634033203125,
      "activations/layer2_attention_weight_min": -26.08303451538086,
      "activations/layer3_attention_weight_max": 61.917930603027344,
      "activations/layer3_attention_weight_min": -63.87745666503906,
      "activations/layer4_attention_weight_max": 91.90034484863281,
      "activations/layer4_attention_weight_min": -80.66203308105469,
      "activations/layer5_attention_weight_max": 117.69447326660156,
      "activations/layer5_attention_weight_min": -98.89055633544922,
      "activations/layer6_attention_weight_max": 64.5282211303711,
      "activations/layer6_attention_weight_min": -49.2317008972168,
      "activations/layer7_attention_weight_max": 58.009193420410156,
      "activations/layer7_attention_weight_min": -51.693965911865234,
      "activations/layer8_attention_weight_max": 45.12180709838867,
      "activations/layer8_attention_weight_min": -36.95309066772461,
      "activations/layer9_attention_weight_max": 56.65339279174805,
      "activations/layer9_attention_weight_min": -54.90267562866211,
      "epoch": 2.19,
      "learning_rate": 0.00013725454545454545,
      "loss": 2.9424,
      "step": 37700
    },
    {
      "activations/layer0_attention_weight_max": 13.288700103759766,
      "activations/layer0_attention_weight_min": -13.57923412322998,
      "activations/layer10_attention_weight_max": 56.428890228271484,
      "activations/layer10_attention_weight_min": -43.16924285888672,
      "activations/layer11_attention_weight_max": 43.64747619628906,
      "activations/layer11_attention_weight_min": -35.90656280517578,
      "activations/layer12_attention_weight_max": 30.635881423950195,
      "activations/layer12_attention_weight_min": -27.73255729675293,
      "activations/layer13_attention_weight_max": 80.20355224609375,
      "activations/layer13_attention_weight_min": -43.7510871887207,
      "activations/layer14_attention_weight_max": 74.99029541015625,
      "activations/layer14_attention_weight_min": -39.445491790771484,
      "activations/layer15_attention_weight_max": 61.38009262084961,
      "activations/layer15_attention_weight_min": -36.7150993347168,
      "activations/layer16_attention_weight_max": 47.07034683227539,
      "activations/layer16_attention_weight_min": -28.105459213256836,
      "activations/layer17_attention_weight_max": 83.45480346679688,
      "activations/layer17_attention_weight_min": -35.17903518676758,
      "activations/layer18_attention_weight_max": 71.8802261352539,
      "activations/layer18_attention_weight_min": -34.177337646484375,
      "activations/layer19_attention_weight_max": 29.005374908447266,
      "activations/layer19_attention_weight_min": -18.000934600830078,
      "activations/layer1_attention_weight_max": 16.20233917236328,
      "activations/layer1_attention_weight_min": -13.095406532287598,
      "activations/layer20_attention_weight_max": 30.325626373291016,
      "activations/layer20_attention_weight_min": -17.60727310180664,
      "activations/layer21_attention_weight_max": 58.65336608886719,
      "activations/layer21_attention_weight_min": -23.422073364257812,
      "activations/layer22_attention_weight_max": 48.343101501464844,
      "activations/layer22_attention_weight_min": -25.385902404785156,
      "activations/layer23_attention_weight_max": 31.83692741394043,
      "activations/layer23_attention_weight_min": -15.03826904296875,
      "activations/layer2_attention_weight_max": 28.03734588623047,
      "activations/layer2_attention_weight_min": -27.476181030273438,
      "activations/layer3_attention_weight_max": 65.68659210205078,
      "activations/layer3_attention_weight_min": -68.06336975097656,
      "activations/layer4_attention_weight_max": 94.72817993164062,
      "activations/layer4_attention_weight_min": -82.67196655273438,
      "activations/layer5_attention_weight_max": 129.39215087890625,
      "activations/layer5_attention_weight_min": -102.64012145996094,
      "activations/layer6_attention_weight_max": 65.517822265625,
      "activations/layer6_attention_weight_min": -54.74970245361328,
      "activations/layer7_attention_weight_max": 59.70133972167969,
      "activations/layer7_attention_weight_min": -48.910335540771484,
      "activations/layer8_attention_weight_max": 44.4024543762207,
      "activations/layer8_attention_weight_min": -38.07292556762695,
      "activations/layer9_attention_weight_max": 62.777313232421875,
      "activations/layer9_attention_weight_min": -60.16750717163086,
      "epoch": 2.19,
      "learning_rate": 0.00013723560606060603,
      "loss": 2.9465,
      "step": 37750
    },
    {
      "activations/layer0_attention_weight_max": 15.592605590820312,
      "activations/layer0_attention_weight_min": -13.70175552368164,
      "activations/layer10_attention_weight_max": 53.419281005859375,
      "activations/layer10_attention_weight_min": -43.743072509765625,
      "activations/layer11_attention_weight_max": 41.1780891418457,
      "activations/layer11_attention_weight_min": -35.04933166503906,
      "activations/layer12_attention_weight_max": 27.477191925048828,
      "activations/layer12_attention_weight_min": -25.803592681884766,
      "activations/layer13_attention_weight_max": 63.90276336669922,
      "activations/layer13_attention_weight_min": -47.00147247314453,
      "activations/layer14_attention_weight_max": 63.14148712158203,
      "activations/layer14_attention_weight_min": -48.13893127441406,
      "activations/layer15_attention_weight_max": 52.7869873046875,
      "activations/layer15_attention_weight_min": -38.0283203125,
      "activations/layer16_attention_weight_max": 49.734214782714844,
      "activations/layer16_attention_weight_min": -31.01734161376953,
      "activations/layer17_attention_weight_max": 57.37875747680664,
      "activations/layer17_attention_weight_min": -37.2264518737793,
      "activations/layer18_attention_weight_max": 55.0140266418457,
      "activations/layer18_attention_weight_min": -33.16185760498047,
      "activations/layer19_attention_weight_max": 21.0554256439209,
      "activations/layer19_attention_weight_min": -16.10662269592285,
      "activations/layer1_attention_weight_max": 15.974396705627441,
      "activations/layer1_attention_weight_min": -13.998570442199707,
      "activations/layer20_attention_weight_max": 21.518047332763672,
      "activations/layer20_attention_weight_min": -16.99358367919922,
      "activations/layer21_attention_weight_max": 36.46832275390625,
      "activations/layer21_attention_weight_min": -22.24979591369629,
      "activations/layer22_attention_weight_max": 40.72800064086914,
      "activations/layer22_attention_weight_min": -21.920360565185547,
      "activations/layer23_attention_weight_max": 25.36514663696289,
      "activations/layer23_attention_weight_min": -15.28509521484375,
      "activations/layer2_attention_weight_max": 26.53582000732422,
      "activations/layer2_attention_weight_min": -26.27513885498047,
      "activations/layer3_attention_weight_max": 62.119171142578125,
      "activations/layer3_attention_weight_min": -67.61892700195312,
      "activations/layer4_attention_weight_max": 91.27188873291016,
      "activations/layer4_attention_weight_min": -84.679931640625,
      "activations/layer5_attention_weight_max": 120.6982421875,
      "activations/layer5_attention_weight_min": -107.42695617675781,
      "activations/layer6_attention_weight_max": 61.34395217895508,
      "activations/layer6_attention_weight_min": -52.80636978149414,
      "activations/layer7_attention_weight_max": 56.73065185546875,
      "activations/layer7_attention_weight_min": -53.24658203125,
      "activations/layer8_attention_weight_max": 45.906700134277344,
      "activations/layer8_attention_weight_min": -45.4171028137207,
      "activations/layer9_attention_weight_max": 60.7389030456543,
      "activations/layer9_attention_weight_min": -56.29895782470703,
      "epoch": 2.2,
      "learning_rate": 0.00013721666666666665,
      "loss": 2.9622,
      "step": 37800
    },
    {
      "activations/layer0_attention_weight_max": 16.125267028808594,
      "activations/layer0_attention_weight_min": -13.813549995422363,
      "activations/layer10_attention_weight_max": 54.24761199951172,
      "activations/layer10_attention_weight_min": -44.99992370605469,
      "activations/layer11_attention_weight_max": 40.01972579956055,
      "activations/layer11_attention_weight_min": -33.623802185058594,
      "activations/layer12_attention_weight_max": 41.950801849365234,
      "activations/layer12_attention_weight_min": -28.053258895874023,
      "activations/layer13_attention_weight_max": 53.17947769165039,
      "activations/layer13_attention_weight_min": -39.211700439453125,
      "activations/layer14_attention_weight_max": 45.98200988769531,
      "activations/layer14_attention_weight_min": -33.315086364746094,
      "activations/layer15_attention_weight_max": 52.01705551147461,
      "activations/layer15_attention_weight_min": -36.463462829589844,
      "activations/layer16_attention_weight_max": 42.89955520629883,
      "activations/layer16_attention_weight_min": -29.63474464416504,
      "activations/layer17_attention_weight_max": 56.40663528442383,
      "activations/layer17_attention_weight_min": -36.20984649658203,
      "activations/layer18_attention_weight_max": 58.72832489013672,
      "activations/layer18_attention_weight_min": -31.845369338989258,
      "activations/layer19_attention_weight_max": 18.33977699279785,
      "activations/layer19_attention_weight_min": -17.821149826049805,
      "activations/layer1_attention_weight_max": 16.161643981933594,
      "activations/layer1_attention_weight_min": -13.499444007873535,
      "activations/layer20_attention_weight_max": 23.618608474731445,
      "activations/layer20_attention_weight_min": -20.477943420410156,
      "activations/layer21_attention_weight_max": 44.739166259765625,
      "activations/layer21_attention_weight_min": -23.814451217651367,
      "activations/layer22_attention_weight_max": 37.6099967956543,
      "activations/layer22_attention_weight_min": -24.148218154907227,
      "activations/layer23_attention_weight_max": 22.65627098083496,
      "activations/layer23_attention_weight_min": -15.40210247039795,
      "activations/layer2_attention_weight_max": 27.030967712402344,
      "activations/layer2_attention_weight_min": -25.648754119873047,
      "activations/layer3_attention_weight_max": 65.47298431396484,
      "activations/layer3_attention_weight_min": -65.45873260498047,
      "activations/layer4_attention_weight_max": 91.56779479980469,
      "activations/layer4_attention_weight_min": -89.52021026611328,
      "activations/layer5_attention_weight_max": 128.66552734375,
      "activations/layer5_attention_weight_min": -104.25495910644531,
      "activations/layer6_attention_weight_max": 60.19917678833008,
      "activations/layer6_attention_weight_min": -51.30929946899414,
      "activations/layer7_attention_weight_max": 56.24266052246094,
      "activations/layer7_attention_weight_min": -49.03019332885742,
      "activations/layer8_attention_weight_max": 45.02589797973633,
      "activations/layer8_attention_weight_min": -37.7083625793457,
      "activations/layer9_attention_weight_max": 57.9207878112793,
      "activations/layer9_attention_weight_min": -49.53367233276367,
      "epoch": 2.2,
      "learning_rate": 0.00013719772727272727,
      "loss": 2.9277,
      "step": 37850
    },
    {
      "activations/layer0_attention_weight_max": 13.043302536010742,
      "activations/layer0_attention_weight_min": -13.907539367675781,
      "activations/layer10_attention_weight_max": 60.13678741455078,
      "activations/layer10_attention_weight_min": -48.86491012573242,
      "activations/layer11_attention_weight_max": 41.580543518066406,
      "activations/layer11_attention_weight_min": -37.53972625732422,
      "activations/layer12_attention_weight_max": 38.57514572143555,
      "activations/layer12_attention_weight_min": -27.33426856994629,
      "activations/layer13_attention_weight_max": 60.11687088012695,
      "activations/layer13_attention_weight_min": -40.3791389465332,
      "activations/layer14_attention_weight_max": 49.19293212890625,
      "activations/layer14_attention_weight_min": -32.67997360229492,
      "activations/layer15_attention_weight_max": 54.50229263305664,
      "activations/layer15_attention_weight_min": -38.2661247253418,
      "activations/layer16_attention_weight_max": 44.534454345703125,
      "activations/layer16_attention_weight_min": -27.202898025512695,
      "activations/layer17_attention_weight_max": 63.59419250488281,
      "activations/layer17_attention_weight_min": -38.76982498168945,
      "activations/layer18_attention_weight_max": 62.02664566040039,
      "activations/layer18_attention_weight_min": -32.49884033203125,
      "activations/layer19_attention_weight_max": 22.837432861328125,
      "activations/layer19_attention_weight_min": -15.581216812133789,
      "activations/layer1_attention_weight_max": 16.501527786254883,
      "activations/layer1_attention_weight_min": -12.677616119384766,
      "activations/layer20_attention_weight_max": 18.582029342651367,
      "activations/layer20_attention_weight_min": -17.43056297302246,
      "activations/layer21_attention_weight_max": 45.99770736694336,
      "activations/layer21_attention_weight_min": -25.984664916992188,
      "activations/layer22_attention_weight_max": 38.799739837646484,
      "activations/layer22_attention_weight_min": -21.959535598754883,
      "activations/layer23_attention_weight_max": 24.034164428710938,
      "activations/layer23_attention_weight_min": -18.625667572021484,
      "activations/layer2_attention_weight_max": 28.479114532470703,
      "activations/layer2_attention_weight_min": -28.241966247558594,
      "activations/layer3_attention_weight_max": 63.35988235473633,
      "activations/layer3_attention_weight_min": -67.98259735107422,
      "activations/layer4_attention_weight_max": 93.6198501586914,
      "activations/layer4_attention_weight_min": -85.36103820800781,
      "activations/layer5_attention_weight_max": 125.26885223388672,
      "activations/layer5_attention_weight_min": -102.9911117553711,
      "activations/layer6_attention_weight_max": 66.83768463134766,
      "activations/layer6_attention_weight_min": -53.30083084106445,
      "activations/layer7_attention_weight_max": 61.8685417175293,
      "activations/layer7_attention_weight_min": -49.94679641723633,
      "activations/layer8_attention_weight_max": 46.61610412597656,
      "activations/layer8_attention_weight_min": -38.765201568603516,
      "activations/layer9_attention_weight_max": 55.71442413330078,
      "activations/layer9_attention_weight_min": -53.173683166503906,
      "epoch": 2.2,
      "learning_rate": 0.00013717878787878785,
      "loss": 2.9412,
      "step": 37900
    },
    {
      "activations/layer0_attention_weight_max": 14.032505989074707,
      "activations/layer0_attention_weight_min": -14.106411933898926,
      "activations/layer10_attention_weight_max": 64.84770965576172,
      "activations/layer10_attention_weight_min": -48.88703536987305,
      "activations/layer11_attention_weight_max": 45.874935150146484,
      "activations/layer11_attention_weight_min": -33.303993225097656,
      "activations/layer12_attention_weight_max": 31.682985305786133,
      "activations/layer12_attention_weight_min": -26.267887115478516,
      "activations/layer13_attention_weight_max": 59.62054443359375,
      "activations/layer13_attention_weight_min": -45.79981231689453,
      "activations/layer14_attention_weight_max": 41.81660842895508,
      "activations/layer14_attention_weight_min": -30.89409065246582,
      "activations/layer15_attention_weight_max": 56.46736526489258,
      "activations/layer15_attention_weight_min": -38.36642837524414,
      "activations/layer16_attention_weight_max": 44.82461929321289,
      "activations/layer16_attention_weight_min": -31.13134765625,
      "activations/layer17_attention_weight_max": 60.98740005493164,
      "activations/layer17_attention_weight_min": -37.2214469909668,
      "activations/layer18_attention_weight_max": 58.871376037597656,
      "activations/layer18_attention_weight_min": -34.402706146240234,
      "activations/layer19_attention_weight_max": 20.450956344604492,
      "activations/layer19_attention_weight_min": -16.243988037109375,
      "activations/layer1_attention_weight_max": 16.93290138244629,
      "activations/layer1_attention_weight_min": -13.368276596069336,
      "activations/layer20_attention_weight_max": 21.694643020629883,
      "activations/layer20_attention_weight_min": -18.764978408813477,
      "activations/layer21_attention_weight_max": 53.603118896484375,
      "activations/layer21_attention_weight_min": -27.029897689819336,
      "activations/layer22_attention_weight_max": 38.247310638427734,
      "activations/layer22_attention_weight_min": -20.892032623291016,
      "activations/layer23_attention_weight_max": 23.889602661132812,
      "activations/layer23_attention_weight_min": -13.31234359741211,
      "activations/layer2_attention_weight_max": 29.473674774169922,
      "activations/layer2_attention_weight_min": -28.170970916748047,
      "activations/layer3_attention_weight_max": 66.10491180419922,
      "activations/layer3_attention_weight_min": -68.38162231445312,
      "activations/layer4_attention_weight_max": 92.75981903076172,
      "activations/layer4_attention_weight_min": -86.37618255615234,
      "activations/layer5_attention_weight_max": 128.72021484375,
      "activations/layer5_attention_weight_min": -101.42179870605469,
      "activations/layer6_attention_weight_max": 63.30900955200195,
      "activations/layer6_attention_weight_min": -54.1457633972168,
      "activations/layer7_attention_weight_max": 63.02409362792969,
      "activations/layer7_attention_weight_min": -51.36003875732422,
      "activations/layer8_attention_weight_max": 45.25100326538086,
      "activations/layer8_attention_weight_min": -40.17201614379883,
      "activations/layer9_attention_weight_max": 58.184486389160156,
      "activations/layer9_attention_weight_min": -56.538604736328125,
      "epoch": 2.21,
      "learning_rate": 0.00013715984848484847,
      "loss": 2.9602,
      "step": 37950
    },
    {
      "activations/layer0_attention_weight_max": 15.442001342773438,
      "activations/layer0_attention_weight_min": -14.03438663482666,
      "activations/layer10_attention_weight_max": 60.647911071777344,
      "activations/layer10_attention_weight_min": -49.103851318359375,
      "activations/layer11_attention_weight_max": 44.58026885986328,
      "activations/layer11_attention_weight_min": -38.711509704589844,
      "activations/layer12_attention_weight_max": 29.88311195373535,
      "activations/layer12_attention_weight_min": -26.14628028869629,
      "activations/layer13_attention_weight_max": 67.2772445678711,
      "activations/layer13_attention_weight_min": -47.88328170776367,
      "activations/layer14_attention_weight_max": 58.75174331665039,
      "activations/layer14_attention_weight_min": -37.24895477294922,
      "activations/layer15_attention_weight_max": 59.02225875854492,
      "activations/layer15_attention_weight_min": -37.15774917602539,
      "activations/layer16_attention_weight_max": 43.33878707885742,
      "activations/layer16_attention_weight_min": -26.62501335144043,
      "activations/layer17_attention_weight_max": 64.14248657226562,
      "activations/layer17_attention_weight_min": -37.46318817138672,
      "activations/layer18_attention_weight_max": 60.20405197143555,
      "activations/layer18_attention_weight_min": -31.143033981323242,
      "activations/layer19_attention_weight_max": 21.754018783569336,
      "activations/layer19_attention_weight_min": -17.274229049682617,
      "activations/layer1_attention_weight_max": 16.618297576904297,
      "activations/layer1_attention_weight_min": -12.368414878845215,
      "activations/layer20_attention_weight_max": 18.685049057006836,
      "activations/layer20_attention_weight_min": -16.994239807128906,
      "activations/layer21_attention_weight_max": 43.297157287597656,
      "activations/layer21_attention_weight_min": -23.944543838500977,
      "activations/layer22_attention_weight_max": 36.56078338623047,
      "activations/layer22_attention_weight_min": -21.935022354125977,
      "activations/layer23_attention_weight_max": 23.16506004333496,
      "activations/layer23_attention_weight_min": -16.06609344482422,
      "activations/layer2_attention_weight_max": 26.719364166259766,
      "activations/layer2_attention_weight_min": -27.09479522705078,
      "activations/layer3_attention_weight_max": 61.739498138427734,
      "activations/layer3_attention_weight_min": -65.03777313232422,
      "activations/layer4_attention_weight_max": 95.96981048583984,
      "activations/layer4_attention_weight_min": -84.02561950683594,
      "activations/layer5_attention_weight_max": 119.96746063232422,
      "activations/layer5_attention_weight_min": -99.70944213867188,
      "activations/layer6_attention_weight_max": 61.2054328918457,
      "activations/layer6_attention_weight_min": -52.13434982299805,
      "activations/layer7_attention_weight_max": 58.873497009277344,
      "activations/layer7_attention_weight_min": -58.21379089355469,
      "activations/layer8_attention_weight_max": 43.02825164794922,
      "activations/layer8_attention_weight_min": -39.47893524169922,
      "activations/layer9_attention_weight_max": 63.1656608581543,
      "activations/layer9_attention_weight_min": -56.34321212768555,
      "epoch": 2.21,
      "learning_rate": 0.00013714090909090908,
      "loss": 2.9479,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_loss": 2.90234375,
      "eval_runtime": 8.4786,
      "eval_samples_per_second": 506.454,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_openwebtext_loss": 2.90234375,
      "eval_openwebtext_ppl": 18.216790978454146,
      "eval_openwebtext_runtime": 8.4786,
      "eval_openwebtext_samples_per_second": 506.454,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_wikitext_loss": 3.15625,
      "eval_wikitext_ppl": 23.482371718117374,
      "eval_wikitext_runtime": 1.9309,
      "eval_wikitext_samples_per_second": 236.161,
      "step": 38000
    },
    {
      "epoch": 2.21,
      "eval_lambada_loss": 3.14453125,
      "eval_lambada_ppl": 23.20879380099744,
      "eval_lambada_runtime": 9.5429,
      "eval_lambada_samples_per_second": 510.224,
      "step": 38000
    },
    {
      "activations/layer0_attention_weight_max": 14.276302337646484,
      "activations/layer0_attention_weight_min": -14.473891258239746,
      "activations/layer10_attention_weight_max": 55.939170837402344,
      "activations/layer10_attention_weight_min": -45.36509323120117,
      "activations/layer11_attention_weight_max": 43.57221221923828,
      "activations/layer11_attention_weight_min": -33.521217346191406,
      "activations/layer12_attention_weight_max": 31.289608001708984,
      "activations/layer12_attention_weight_min": -24.231061935424805,
      "activations/layer13_attention_weight_max": 50.88320541381836,
      "activations/layer13_attention_weight_min": -40.632320404052734,
      "activations/layer14_attention_weight_max": 44.6190299987793,
      "activations/layer14_attention_weight_min": -33.14656448364258,
      "activations/layer15_attention_weight_max": 52.60281753540039,
      "activations/layer15_attention_weight_min": -38.05574417114258,
      "activations/layer16_attention_weight_max": 39.8322868347168,
      "activations/layer16_attention_weight_min": -29.99908447265625,
      "activations/layer17_attention_weight_max": 74.13822174072266,
      "activations/layer17_attention_weight_min": -39.57329559326172,
      "activations/layer18_attention_weight_max": 70.36637878417969,
      "activations/layer18_attention_weight_min": -36.38997268676758,
      "activations/layer19_attention_weight_max": 22.22659683227539,
      "activations/layer19_attention_weight_min": -15.540300369262695,
      "activations/layer1_attention_weight_max": 16.2880802154541,
      "activations/layer1_attention_weight_min": -13.895424842834473,
      "activations/layer20_attention_weight_max": 20.91873550415039,
      "activations/layer20_attention_weight_min": -21.150056838989258,
      "activations/layer21_attention_weight_max": 45.138526916503906,
      "activations/layer21_attention_weight_min": -25.909587860107422,
      "activations/layer22_attention_weight_max": 37.18326950073242,
      "activations/layer22_attention_weight_min": -24.750261306762695,
      "activations/layer23_attention_weight_max": 25.025373458862305,
      "activations/layer23_attention_weight_min": -14.929052352905273,
      "activations/layer2_attention_weight_max": 28.89150619506836,
      "activations/layer2_attention_weight_min": -26.965065002441406,
      "activations/layer3_attention_weight_max": 64.135498046875,
      "activations/layer3_attention_weight_min": -66.12220764160156,
      "activations/layer4_attention_weight_max": 92.50711059570312,
      "activations/layer4_attention_weight_min": -85.64732360839844,
      "activations/layer5_attention_weight_max": 120.641357421875,
      "activations/layer5_attention_weight_min": -98.4228515625,
      "activations/layer6_attention_weight_max": 61.684349060058594,
      "activations/layer6_attention_weight_min": -51.2930793762207,
      "activations/layer7_attention_weight_max": 57.74879455566406,
      "activations/layer7_attention_weight_min": -51.16714096069336,
      "activations/layer8_attention_weight_max": 44.391109466552734,
      "activations/layer8_attention_weight_min": -38.64146041870117,
      "activations/layer9_attention_weight_max": 62.3354377746582,
      "activations/layer9_attention_weight_min": -54.13833236694336,
      "epoch": 2.21,
      "learning_rate": 0.0001371219696969697,
      "loss": 2.9331,
      "step": 38050
    },
    {
      "activations/layer0_attention_weight_max": 14.442119598388672,
      "activations/layer0_attention_weight_min": -14.335546493530273,
      "activations/layer10_attention_weight_max": 54.38799285888672,
      "activations/layer10_attention_weight_min": -44.4234504699707,
      "activations/layer11_attention_weight_max": 41.50517272949219,
      "activations/layer11_attention_weight_min": -31.984657287597656,
      "activations/layer12_attention_weight_max": 32.59614181518555,
      "activations/layer12_attention_weight_min": -26.327238082885742,
      "activations/layer13_attention_weight_max": 64.58163452148438,
      "activations/layer13_attention_weight_min": -43.817745208740234,
      "activations/layer14_attention_weight_max": 60.0649528503418,
      "activations/layer14_attention_weight_min": -41.87187576293945,
      "activations/layer15_attention_weight_max": 52.4855842590332,
      "activations/layer15_attention_weight_min": -37.37895965576172,
      "activations/layer16_attention_weight_max": 42.09945297241211,
      "activations/layer16_attention_weight_min": -27.663782119750977,
      "activations/layer17_attention_weight_max": 59.48388671875,
      "activations/layer17_attention_weight_min": -36.90791702270508,
      "activations/layer18_attention_weight_max": 61.18757629394531,
      "activations/layer18_attention_weight_min": -34.0272331237793,
      "activations/layer19_attention_weight_max": 20.908395767211914,
      "activations/layer19_attention_weight_min": -15.322470664978027,
      "activations/layer1_attention_weight_max": 16.210065841674805,
      "activations/layer1_attention_weight_min": -14.950516700744629,
      "activations/layer20_attention_weight_max": 20.49885368347168,
      "activations/layer20_attention_weight_min": -16.49960708618164,
      "activations/layer21_attention_weight_max": 40.45841598510742,
      "activations/layer21_attention_weight_min": -23.232946395874023,
      "activations/layer22_attention_weight_max": 34.05271530151367,
      "activations/layer22_attention_weight_min": -20.869192123413086,
      "activations/layer23_attention_weight_max": 24.934720993041992,
      "activations/layer23_attention_weight_min": -16.94432830810547,
      "activations/layer2_attention_weight_max": 27.392074584960938,
      "activations/layer2_attention_weight_min": -25.980121612548828,
      "activations/layer3_attention_weight_max": 63.82647705078125,
      "activations/layer3_attention_weight_min": -67.16094970703125,
      "activations/layer4_attention_weight_max": 90.76909637451172,
      "activations/layer4_attention_weight_min": -81.21220397949219,
      "activations/layer5_attention_weight_max": 121.15450286865234,
      "activations/layer5_attention_weight_min": -105.97154998779297,
      "activations/layer6_attention_weight_max": 60.78221893310547,
      "activations/layer6_attention_weight_min": -52.44042205810547,
      "activations/layer7_attention_weight_max": 58.495235443115234,
      "activations/layer7_attention_weight_min": -55.49764633178711,
      "activations/layer8_attention_weight_max": 44.3011360168457,
      "activations/layer8_attention_weight_min": -36.99189758300781,
      "activations/layer9_attention_weight_max": 54.669857025146484,
      "activations/layer9_attention_weight_min": -53.30751419067383,
      "epoch": 2.21,
      "learning_rate": 0.00013710303030303029,
      "loss": 2.9395,
      "step": 38100
    },
    {
      "activations/layer0_attention_weight_max": 14.089752197265625,
      "activations/layer0_attention_weight_min": -14.079695701599121,
      "activations/layer10_attention_weight_max": 67.88517761230469,
      "activations/layer10_attention_weight_min": -53.26249694824219,
      "activations/layer11_attention_weight_max": 45.890907287597656,
      "activations/layer11_attention_weight_min": -32.27534103393555,
      "activations/layer12_attention_weight_max": 41.09907150268555,
      "activations/layer12_attention_weight_min": -29.76479721069336,
      "activations/layer13_attention_weight_max": 49.938838958740234,
      "activations/layer13_attention_weight_min": -38.86739730834961,
      "activations/layer14_attention_weight_max": 55.18683624267578,
      "activations/layer14_attention_weight_min": -37.31966781616211,
      "activations/layer15_attention_weight_max": 47.63407516479492,
      "activations/layer15_attention_weight_min": -38.0684700012207,
      "activations/layer16_attention_weight_max": 38.80623245239258,
      "activations/layer16_attention_weight_min": -27.232805252075195,
      "activations/layer17_attention_weight_max": 51.29753494262695,
      "activations/layer17_attention_weight_min": -35.72599792480469,
      "activations/layer18_attention_weight_max": 50.64126205444336,
      "activations/layer18_attention_weight_min": -33.54716491699219,
      "activations/layer19_attention_weight_max": 19.7747745513916,
      "activations/layer19_attention_weight_min": -15.698144912719727,
      "activations/layer1_attention_weight_max": 16.663349151611328,
      "activations/layer1_attention_weight_min": -13.403539657592773,
      "activations/layer20_attention_weight_max": 18.764738082885742,
      "activations/layer20_attention_weight_min": -18.4815731048584,
      "activations/layer21_attention_weight_max": 40.91055679321289,
      "activations/layer21_attention_weight_min": -25.21788215637207,
      "activations/layer22_attention_weight_max": 38.21131896972656,
      "activations/layer22_attention_weight_min": -22.94127082824707,
      "activations/layer23_attention_weight_max": 20.907384872436523,
      "activations/layer23_attention_weight_min": -13.92179012298584,
      "activations/layer2_attention_weight_max": 25.502235412597656,
      "activations/layer2_attention_weight_min": -24.671796798706055,
      "activations/layer3_attention_weight_max": 60.818695068359375,
      "activations/layer3_attention_weight_min": -64.14259338378906,
      "activations/layer4_attention_weight_max": 89.58946228027344,
      "activations/layer4_attention_weight_min": -81.09220886230469,
      "activations/layer5_attention_weight_max": 113.93510437011719,
      "activations/layer5_attention_weight_min": -99.26375579833984,
      "activations/layer6_attention_weight_max": 59.862388610839844,
      "activations/layer6_attention_weight_min": -50.726253509521484,
      "activations/layer7_attention_weight_max": 57.833770751953125,
      "activations/layer7_attention_weight_min": -49.151390075683594,
      "activations/layer8_attention_weight_max": 41.029476165771484,
      "activations/layer8_attention_weight_min": -36.98088836669922,
      "activations/layer9_attention_weight_max": 60.20170974731445,
      "activations/layer9_attention_weight_min": -51.05318069458008,
      "epoch": 2.22,
      "learning_rate": 0.0001370840909090909,
      "loss": 2.9429,
      "step": 38150
    },
    {
      "activations/layer0_attention_weight_max": 13.776803016662598,
      "activations/layer0_attention_weight_min": -13.260333061218262,
      "activations/layer10_attention_weight_max": 56.73589324951172,
      "activations/layer10_attention_weight_min": -44.34955978393555,
      "activations/layer11_attention_weight_max": 46.433555603027344,
      "activations/layer11_attention_weight_min": -34.21232223510742,
      "activations/layer12_attention_weight_max": 31.340038299560547,
      "activations/layer12_attention_weight_min": -23.85350227355957,
      "activations/layer13_attention_weight_max": 56.864471435546875,
      "activations/layer13_attention_weight_min": -36.03684997558594,
      "activations/layer14_attention_weight_max": 52.11326599121094,
      "activations/layer14_attention_weight_min": -33.05413818359375,
      "activations/layer15_attention_weight_max": 55.5956916809082,
      "activations/layer15_attention_weight_min": -34.06972885131836,
      "activations/layer16_attention_weight_max": 42.61102294921875,
      "activations/layer16_attention_weight_min": -28.483905792236328,
      "activations/layer17_attention_weight_max": 65.32305908203125,
      "activations/layer17_attention_weight_min": -36.991756439208984,
      "activations/layer18_attention_weight_max": 68.73919677734375,
      "activations/layer18_attention_weight_min": -34.4274787902832,
      "activations/layer19_attention_weight_max": 23.48102378845215,
      "activations/layer19_attention_weight_min": -15.961746215820312,
      "activations/layer1_attention_weight_max": 16.696746826171875,
      "activations/layer1_attention_weight_min": -12.607141494750977,
      "activations/layer20_attention_weight_max": 22.91288185119629,
      "activations/layer20_attention_weight_min": -16.68583106994629,
      "activations/layer21_attention_weight_max": 42.59775924682617,
      "activations/layer21_attention_weight_min": -25.611038208007812,
      "activations/layer22_attention_weight_max": 41.11427688598633,
      "activations/layer22_attention_weight_min": -23.968551635742188,
      "activations/layer23_attention_weight_max": 27.578571319580078,
      "activations/layer23_attention_weight_min": -14.353984832763672,
      "activations/layer2_attention_weight_max": 27.198932647705078,
      "activations/layer2_attention_weight_min": -25.431997299194336,
      "activations/layer3_attention_weight_max": 62.968318939208984,
      "activations/layer3_attention_weight_min": -66.03255462646484,
      "activations/layer4_attention_weight_max": 92.75606536865234,
      "activations/layer4_attention_weight_min": -85.01882934570312,
      "activations/layer5_attention_weight_max": 131.40692138671875,
      "activations/layer5_attention_weight_min": -102.56343078613281,
      "activations/layer6_attention_weight_max": 65.41436004638672,
      "activations/layer6_attention_weight_min": -50.66157531738281,
      "activations/layer7_attention_weight_max": 61.128501892089844,
      "activations/layer7_attention_weight_min": -51.009552001953125,
      "activations/layer8_attention_weight_max": 46.91397476196289,
      "activations/layer8_attention_weight_min": -34.53303909301758,
      "activations/layer9_attention_weight_max": 67.2125015258789,
      "activations/layer9_attention_weight_min": -60.26194381713867,
      "epoch": 2.22,
      "learning_rate": 0.00013706515151515152,
      "loss": 2.9362,
      "step": 38200
    },
    {
      "activations/layer0_attention_weight_max": 13.953660011291504,
      "activations/layer0_attention_weight_min": -13.89398193359375,
      "activations/layer10_attention_weight_max": 58.40340042114258,
      "activations/layer10_attention_weight_min": -45.72163391113281,
      "activations/layer11_attention_weight_max": 46.354393005371094,
      "activations/layer11_attention_weight_min": -34.04332733154297,
      "activations/layer12_attention_weight_max": 29.23407745361328,
      "activations/layer12_attention_weight_min": -25.45858383178711,
      "activations/layer13_attention_weight_max": 50.53920364379883,
      "activations/layer13_attention_weight_min": -39.593379974365234,
      "activations/layer14_attention_weight_max": 51.080379486083984,
      "activations/layer14_attention_weight_min": -42.55183792114258,
      "activations/layer15_attention_weight_max": 51.7635612487793,
      "activations/layer15_attention_weight_min": -35.963462829589844,
      "activations/layer16_attention_weight_max": 47.12295150756836,
      "activations/layer16_attention_weight_min": -29.577919006347656,
      "activations/layer17_attention_weight_max": 59.96232986450195,
      "activations/layer17_attention_weight_min": -39.51261901855469,
      "activations/layer18_attention_weight_max": 64.3957290649414,
      "activations/layer18_attention_weight_min": -32.50620651245117,
      "activations/layer19_attention_weight_max": 21.672924041748047,
      "activations/layer19_attention_weight_min": -17.13289451599121,
      "activations/layer1_attention_weight_max": 17.440813064575195,
      "activations/layer1_attention_weight_min": -12.721345901489258,
      "activations/layer20_attention_weight_max": 20.301620483398438,
      "activations/layer20_attention_weight_min": -16.666521072387695,
      "activations/layer21_attention_weight_max": 40.81211471557617,
      "activations/layer21_attention_weight_min": -25.342050552368164,
      "activations/layer22_attention_weight_max": 37.548583984375,
      "activations/layer22_attention_weight_min": -21.07500457763672,
      "activations/layer23_attention_weight_max": 24.045066833496094,
      "activations/layer23_attention_weight_min": -14.466100692749023,
      "activations/layer2_attention_weight_max": 26.489730834960938,
      "activations/layer2_attention_weight_min": -24.50954246520996,
      "activations/layer3_attention_weight_max": 62.23548889160156,
      "activations/layer3_attention_weight_min": -63.12811279296875,
      "activations/layer4_attention_weight_max": 90.90585327148438,
      "activations/layer4_attention_weight_min": -81.7060546875,
      "activations/layer5_attention_weight_max": 125.57254028320312,
      "activations/layer5_attention_weight_min": -98.4840316772461,
      "activations/layer6_attention_weight_max": 60.35011291503906,
      "activations/layer6_attention_weight_min": -48.82027816772461,
      "activations/layer7_attention_weight_max": 55.93906784057617,
      "activations/layer7_attention_weight_min": -53.08494186401367,
      "activations/layer8_attention_weight_max": 44.6746826171875,
      "activations/layer8_attention_weight_min": -35.73844528198242,
      "activations/layer9_attention_weight_max": 62.7969856262207,
      "activations/layer9_attention_weight_min": -52.86106491088867,
      "epoch": 2.22,
      "learning_rate": 0.0001370462121212121,
      "loss": 2.9625,
      "step": 38250
    },
    {
      "activations/layer0_attention_weight_max": 14.653005599975586,
      "activations/layer0_attention_weight_min": -14.194948196411133,
      "activations/layer10_attention_weight_max": 58.9058837890625,
      "activations/layer10_attention_weight_min": -44.29060363769531,
      "activations/layer11_attention_weight_max": 49.692256927490234,
      "activations/layer11_attention_weight_min": -34.963890075683594,
      "activations/layer12_attention_weight_max": 29.15087127685547,
      "activations/layer12_attention_weight_min": -26.79623794555664,
      "activations/layer13_attention_weight_max": 56.24530029296875,
      "activations/layer13_attention_weight_min": -42.930992126464844,
      "activations/layer14_attention_weight_max": 53.061214447021484,
      "activations/layer14_attention_weight_min": -38.825931549072266,
      "activations/layer15_attention_weight_max": 53.870758056640625,
      "activations/layer15_attention_weight_min": -37.243186950683594,
      "activations/layer16_attention_weight_max": 40.114994049072266,
      "activations/layer16_attention_weight_min": -27.932905197143555,
      "activations/layer17_attention_weight_max": 68.16385650634766,
      "activations/layer17_attention_weight_min": -36.8151969909668,
      "activations/layer18_attention_weight_max": 58.99258804321289,
      "activations/layer18_attention_weight_min": -36.650123596191406,
      "activations/layer19_attention_weight_max": 22.79690933227539,
      "activations/layer19_attention_weight_min": -16.32939910888672,
      "activations/layer1_attention_weight_max": 17.012252807617188,
      "activations/layer1_attention_weight_min": -14.238725662231445,
      "activations/layer20_attention_weight_max": 23.773021697998047,
      "activations/layer20_attention_weight_min": -17.673227310180664,
      "activations/layer21_attention_weight_max": 40.49114990234375,
      "activations/layer21_attention_weight_min": -25.5621280670166,
      "activations/layer22_attention_weight_max": 38.2469482421875,
      "activations/layer22_attention_weight_min": -22.196556091308594,
      "activations/layer23_attention_weight_max": 24.923465728759766,
      "activations/layer23_attention_weight_min": -15.7066650390625,
      "activations/layer2_attention_weight_max": 26.902488708496094,
      "activations/layer2_attention_weight_min": -26.124290466308594,
      "activations/layer3_attention_weight_max": 62.82316970825195,
      "activations/layer3_attention_weight_min": -66.48147583007812,
      "activations/layer4_attention_weight_max": 91.49443817138672,
      "activations/layer4_attention_weight_min": -82.86905670166016,
      "activations/layer5_attention_weight_max": 120.85966491699219,
      "activations/layer5_attention_weight_min": -100.85375213623047,
      "activations/layer6_attention_weight_max": 58.44526290893555,
      "activations/layer6_attention_weight_min": -50.88499069213867,
      "activations/layer7_attention_weight_max": 54.84162902832031,
      "activations/layer7_attention_weight_min": -54.52810287475586,
      "activations/layer8_attention_weight_max": 44.550437927246094,
      "activations/layer8_attention_weight_min": -41.805442810058594,
      "activations/layer9_attention_weight_max": 68.03084564208984,
      "activations/layer9_attention_weight_min": -56.73601150512695,
      "epoch": 2.23,
      "learning_rate": 0.00013702727272727272,
      "loss": 2.944,
      "step": 38300
    },
    {
      "activations/layer0_attention_weight_max": 14.294466972351074,
      "activations/layer0_attention_weight_min": -14.162592887878418,
      "activations/layer10_attention_weight_max": 55.20873260498047,
      "activations/layer10_attention_weight_min": -46.42778396606445,
      "activations/layer11_attention_weight_max": 43.7908935546875,
      "activations/layer11_attention_weight_min": -33.66896057128906,
      "activations/layer12_attention_weight_max": 29.109058380126953,
      "activations/layer12_attention_weight_min": -26.768009185791016,
      "activations/layer13_attention_weight_max": 67.17998504638672,
      "activations/layer13_attention_weight_min": -42.98653793334961,
      "activations/layer14_attention_weight_max": 65.49620819091797,
      "activations/layer14_attention_weight_min": -39.16310501098633,
      "activations/layer15_attention_weight_max": 63.039188385009766,
      "activations/layer15_attention_weight_min": -40.81822204589844,
      "activations/layer16_attention_weight_max": 40.202850341796875,
      "activations/layer16_attention_weight_min": -28.57139015197754,
      "activations/layer17_attention_weight_max": 65.55085754394531,
      "activations/layer17_attention_weight_min": -36.23884963989258,
      "activations/layer18_attention_weight_max": 59.3858642578125,
      "activations/layer18_attention_weight_min": -35.039466857910156,
      "activations/layer19_attention_weight_max": 22.483705520629883,
      "activations/layer19_attention_weight_min": -17.846323013305664,
      "activations/layer1_attention_weight_max": 17.274677276611328,
      "activations/layer1_attention_weight_min": -14.285893440246582,
      "activations/layer20_attention_weight_max": 21.218544006347656,
      "activations/layer20_attention_weight_min": -16.685087203979492,
      "activations/layer21_attention_weight_max": 45.946475982666016,
      "activations/layer21_attention_weight_min": -27.53997802734375,
      "activations/layer22_attention_weight_max": 36.54592514038086,
      "activations/layer22_attention_weight_min": -23.248416900634766,
      "activations/layer23_attention_weight_max": 23.69170379638672,
      "activations/layer23_attention_weight_min": -15.382123947143555,
      "activations/layer2_attention_weight_max": 29.617111206054688,
      "activations/layer2_attention_weight_min": -26.32185935974121,
      "activations/layer3_attention_weight_max": 64.87202453613281,
      "activations/layer3_attention_weight_min": -68.1919937133789,
      "activations/layer4_attention_weight_max": 93.73860931396484,
      "activations/layer4_attention_weight_min": -86.37703704833984,
      "activations/layer5_attention_weight_max": 118.54415130615234,
      "activations/layer5_attention_weight_min": -102.08345794677734,
      "activations/layer6_attention_weight_max": 59.417884826660156,
      "activations/layer6_attention_weight_min": -51.96732711791992,
      "activations/layer7_attention_weight_max": 61.87208557128906,
      "activations/layer7_attention_weight_min": -54.64606857299805,
      "activations/layer8_attention_weight_max": 48.56686019897461,
      "activations/layer8_attention_weight_min": -38.75056838989258,
      "activations/layer9_attention_weight_max": 75.45207977294922,
      "activations/layer9_attention_weight_min": -60.10834503173828,
      "epoch": 2.23,
      "learning_rate": 0.00013700833333333333,
      "loss": 2.9475,
      "step": 38350
    },
    {
      "activations/layer0_attention_weight_max": 16.80303192138672,
      "activations/layer0_attention_weight_min": -13.791665077209473,
      "activations/layer10_attention_weight_max": 51.733001708984375,
      "activations/layer10_attention_weight_min": -45.369873046875,
      "activations/layer11_attention_weight_max": 37.56608581542969,
      "activations/layer11_attention_weight_min": -30.899417877197266,
      "activations/layer12_attention_weight_max": 28.16830062866211,
      "activations/layer12_attention_weight_min": -25.030757904052734,
      "activations/layer13_attention_weight_max": 54.6276969909668,
      "activations/layer13_attention_weight_min": -39.80632781982422,
      "activations/layer14_attention_weight_max": 57.576236724853516,
      "activations/layer14_attention_weight_min": -43.85016632080078,
      "activations/layer15_attention_weight_max": 47.165016174316406,
      "activations/layer15_attention_weight_min": -35.88535690307617,
      "activations/layer16_attention_weight_max": 42.191993713378906,
      "activations/layer16_attention_weight_min": -27.34428596496582,
      "activations/layer17_attention_weight_max": 59.14812088012695,
      "activations/layer17_attention_weight_min": -39.42245864868164,
      "activations/layer18_attention_weight_max": 53.282081604003906,
      "activations/layer18_attention_weight_min": -33.054298400878906,
      "activations/layer19_attention_weight_max": 22.43044662475586,
      "activations/layer19_attention_weight_min": -19.58891487121582,
      "activations/layer1_attention_weight_max": 16.949113845825195,
      "activations/layer1_attention_weight_min": -14.116426467895508,
      "activations/layer20_attention_weight_max": 20.701648712158203,
      "activations/layer20_attention_weight_min": -21.36798858642578,
      "activations/layer21_attention_weight_max": 39.156986236572266,
      "activations/layer21_attention_weight_min": -27.651512145996094,
      "activations/layer22_attention_weight_max": 38.93397521972656,
      "activations/layer22_attention_weight_min": -23.40315055847168,
      "activations/layer23_attention_weight_max": 22.403518676757812,
      "activations/layer23_attention_weight_min": -14.958949089050293,
      "activations/layer2_attention_weight_max": 27.33088493347168,
      "activations/layer2_attention_weight_min": -27.071317672729492,
      "activations/layer3_attention_weight_max": 65.20272064208984,
      "activations/layer3_attention_weight_min": -67.12650299072266,
      "activations/layer4_attention_weight_max": 93.3596420288086,
      "activations/layer4_attention_weight_min": -88.24657440185547,
      "activations/layer5_attention_weight_max": 127.29473114013672,
      "activations/layer5_attention_weight_min": -102.07670593261719,
      "activations/layer6_attention_weight_max": 65.3922348022461,
      "activations/layer6_attention_weight_min": -50.38393783569336,
      "activations/layer7_attention_weight_max": 58.3858528137207,
      "activations/layer7_attention_weight_min": -57.658477783203125,
      "activations/layer8_attention_weight_max": 42.99718475341797,
      "activations/layer8_attention_weight_min": -38.339622497558594,
      "activations/layer9_attention_weight_max": 61.40609359741211,
      "activations/layer9_attention_weight_min": -58.8228645324707,
      "epoch": 2.23,
      "learning_rate": 0.00013698939393939392,
      "loss": 2.9535,
      "step": 38400
    },
    {
      "activations/layer0_attention_weight_max": 13.42983627319336,
      "activations/layer0_attention_weight_min": -13.398707389831543,
      "activations/layer10_attention_weight_max": 57.552337646484375,
      "activations/layer10_attention_weight_min": -44.33217239379883,
      "activations/layer11_attention_weight_max": 48.35963439941406,
      "activations/layer11_attention_weight_min": -35.058956146240234,
      "activations/layer12_attention_weight_max": 30.744434356689453,
      "activations/layer12_attention_weight_min": -28.53253746032715,
      "activations/layer13_attention_weight_max": 61.74871063232422,
      "activations/layer13_attention_weight_min": -41.666072845458984,
      "activations/layer14_attention_weight_max": 67.07158660888672,
      "activations/layer14_attention_weight_min": -45.011173248291016,
      "activations/layer15_attention_weight_max": 51.13576889038086,
      "activations/layer15_attention_weight_min": -34.91040802001953,
      "activations/layer16_attention_weight_max": 40.47554397583008,
      "activations/layer16_attention_weight_min": -26.87334442138672,
      "activations/layer17_attention_weight_max": 59.4883918762207,
      "activations/layer17_attention_weight_min": -36.09371566772461,
      "activations/layer18_attention_weight_max": 52.97318649291992,
      "activations/layer18_attention_weight_min": -32.257511138916016,
      "activations/layer19_attention_weight_max": 20.82895851135254,
      "activations/layer19_attention_weight_min": -20.04877471923828,
      "activations/layer1_attention_weight_max": 17.50448226928711,
      "activations/layer1_attention_weight_min": -14.173144340515137,
      "activations/layer20_attention_weight_max": 24.734067916870117,
      "activations/layer20_attention_weight_min": -17.88470458984375,
      "activations/layer21_attention_weight_max": 43.479949951171875,
      "activations/layer21_attention_weight_min": -27.955211639404297,
      "activations/layer22_attention_weight_max": 36.8700065612793,
      "activations/layer22_attention_weight_min": -23.83649444580078,
      "activations/layer23_attention_weight_max": 25.985591888427734,
      "activations/layer23_attention_weight_min": -14.621562004089355,
      "activations/layer2_attention_weight_max": 30.256240844726562,
      "activations/layer2_attention_weight_min": -28.020606994628906,
      "activations/layer3_attention_weight_max": 69.04219818115234,
      "activations/layer3_attention_weight_min": -68.05989837646484,
      "activations/layer4_attention_weight_max": 103.0409164428711,
      "activations/layer4_attention_weight_min": -86.52706146240234,
      "activations/layer5_attention_weight_max": 134.42886352539062,
      "activations/layer5_attention_weight_min": -103.63497924804688,
      "activations/layer6_attention_weight_max": 64.61566925048828,
      "activations/layer6_attention_weight_min": -52.00087356567383,
      "activations/layer7_attention_weight_max": 64.27427673339844,
      "activations/layer7_attention_weight_min": -56.40118408203125,
      "activations/layer8_attention_weight_max": 49.73895263671875,
      "activations/layer8_attention_weight_min": -46.21763610839844,
      "activations/layer9_attention_weight_max": 67.0538558959961,
      "activations/layer9_attention_weight_min": -58.0505485534668,
      "epoch": 2.23,
      "learning_rate": 0.00013697045454545454,
      "loss": 2.9495,
      "step": 38450
    },
    {
      "activations/layer0_attention_weight_max": 14.152327537536621,
      "activations/layer0_attention_weight_min": -13.89803409576416,
      "activations/layer10_attention_weight_max": 52.28912353515625,
      "activations/layer10_attention_weight_min": -40.814544677734375,
      "activations/layer11_attention_weight_max": 43.82640075683594,
      "activations/layer11_attention_weight_min": -32.113277435302734,
      "activations/layer12_attention_weight_max": 28.34008026123047,
      "activations/layer12_attention_weight_min": -23.782577514648438,
      "activations/layer13_attention_weight_max": 55.79141616821289,
      "activations/layer13_attention_weight_min": -38.06446075439453,
      "activations/layer14_attention_weight_max": 60.110877990722656,
      "activations/layer14_attention_weight_min": -35.78057098388672,
      "activations/layer15_attention_weight_max": 61.775264739990234,
      "activations/layer15_attention_weight_min": -38.15838623046875,
      "activations/layer16_attention_weight_max": 43.9545783996582,
      "activations/layer16_attention_weight_min": -28.842100143432617,
      "activations/layer17_attention_weight_max": 66.26622772216797,
      "activations/layer17_attention_weight_min": -39.205589294433594,
      "activations/layer18_attention_weight_max": 61.693424224853516,
      "activations/layer18_attention_weight_min": -31.0234432220459,
      "activations/layer19_attention_weight_max": 27.637836456298828,
      "activations/layer19_attention_weight_min": -17.38347816467285,
      "activations/layer1_attention_weight_max": 17.303050994873047,
      "activations/layer1_attention_weight_min": -12.858552932739258,
      "activations/layer20_attention_weight_max": 22.04376792907715,
      "activations/layer20_attention_weight_min": -20.30588150024414,
      "activations/layer21_attention_weight_max": 41.39364242553711,
      "activations/layer21_attention_weight_min": -22.459152221679688,
      "activations/layer22_attention_weight_max": 38.39177322387695,
      "activations/layer22_attention_weight_min": -23.898149490356445,
      "activations/layer23_attention_weight_max": 26.30687713623047,
      "activations/layer23_attention_weight_min": -15.15456771850586,
      "activations/layer2_attention_weight_max": 26.795026779174805,
      "activations/layer2_attention_weight_min": -26.81427764892578,
      "activations/layer3_attention_weight_max": 64.37104797363281,
      "activations/layer3_attention_weight_min": -70.6159439086914,
      "activations/layer4_attention_weight_max": 85.59733581542969,
      "activations/layer4_attention_weight_min": -86.0785140991211,
      "activations/layer5_attention_weight_max": 111.79857635498047,
      "activations/layer5_attention_weight_min": -104.1233139038086,
      "activations/layer6_attention_weight_max": 57.19144058227539,
      "activations/layer6_attention_weight_min": -51.6398811340332,
      "activations/layer7_attention_weight_max": 50.446922302246094,
      "activations/layer7_attention_weight_min": -49.34962844848633,
      "activations/layer8_attention_weight_max": 38.241943359375,
      "activations/layer8_attention_weight_min": -36.34321975708008,
      "activations/layer9_attention_weight_max": 55.07712936401367,
      "activations/layer9_attention_weight_min": -51.2059211730957,
      "epoch": 2.24,
      "learning_rate": 0.00013695151515151515,
      "loss": 2.9587,
      "step": 38500
    },
    {
      "activations/layer0_attention_weight_max": 14.258295059204102,
      "activations/layer0_attention_weight_min": -13.849225997924805,
      "activations/layer10_attention_weight_max": 60.336692810058594,
      "activations/layer10_attention_weight_min": -48.08241653442383,
      "activations/layer11_attention_weight_max": 47.66666793823242,
      "activations/layer11_attention_weight_min": -33.59757995605469,
      "activations/layer12_attention_weight_max": 33.852420806884766,
      "activations/layer12_attention_weight_min": -26.341890335083008,
      "activations/layer13_attention_weight_max": 58.79010772705078,
      "activations/layer13_attention_weight_min": -38.62342071533203,
      "activations/layer14_attention_weight_max": 52.748287200927734,
      "activations/layer14_attention_weight_min": -32.632469177246094,
      "activations/layer15_attention_weight_max": 60.06821060180664,
      "activations/layer15_attention_weight_min": -38.92042541503906,
      "activations/layer16_attention_weight_max": 41.65530776977539,
      "activations/layer16_attention_weight_min": -32.427276611328125,
      "activations/layer17_attention_weight_max": 58.75210189819336,
      "activations/layer17_attention_weight_min": -36.84032440185547,
      "activations/layer18_attention_weight_max": 54.37113952636719,
      "activations/layer18_attention_weight_min": -31.161287307739258,
      "activations/layer19_attention_weight_max": 21.19321632385254,
      "activations/layer19_attention_weight_min": -15.144523620605469,
      "activations/layer1_attention_weight_max": 17.114748001098633,
      "activations/layer1_attention_weight_min": -14.12411117553711,
      "activations/layer20_attention_weight_max": 20.152658462524414,
      "activations/layer20_attention_weight_min": -15.538799285888672,
      "activations/layer21_attention_weight_max": 42.918914794921875,
      "activations/layer21_attention_weight_min": -25.67007827758789,
      "activations/layer22_attention_weight_max": 36.052734375,
      "activations/layer22_attention_weight_min": -24.245012283325195,
      "activations/layer23_attention_weight_max": 25.095754623413086,
      "activations/layer23_attention_weight_min": -16.136816024780273,
      "activations/layer2_attention_weight_max": 28.135417938232422,
      "activations/layer2_attention_weight_min": -27.688255310058594,
      "activations/layer3_attention_weight_max": 63.693885803222656,
      "activations/layer3_attention_weight_min": -67.08889770507812,
      "activations/layer4_attention_weight_max": 91.40853881835938,
      "activations/layer4_attention_weight_min": -84.23332214355469,
      "activations/layer5_attention_weight_max": 119.71015930175781,
      "activations/layer5_attention_weight_min": -103.46168518066406,
      "activations/layer6_attention_weight_max": 58.210208892822266,
      "activations/layer6_attention_weight_min": -52.50441360473633,
      "activations/layer7_attention_weight_max": 60.85287094116211,
      "activations/layer7_attention_weight_min": -52.90319061279297,
      "activations/layer8_attention_weight_max": 44.422786712646484,
      "activations/layer8_attention_weight_min": -36.75943374633789,
      "activations/layer9_attention_weight_max": 56.4437141418457,
      "activations/layer9_attention_weight_min": -52.8240966796875,
      "epoch": 2.24,
      "learning_rate": 0.00013693257575757574,
      "loss": 2.9523,
      "step": 38550
    },
    {
      "activations/layer0_attention_weight_max": 14.37871265411377,
      "activations/layer0_attention_weight_min": -13.719889640808105,
      "activations/layer10_attention_weight_max": 51.45072937011719,
      "activations/layer10_attention_weight_min": -44.322303771972656,
      "activations/layer11_attention_weight_max": 43.27946090698242,
      "activations/layer11_attention_weight_min": -32.99134063720703,
      "activations/layer12_attention_weight_max": 26.78452491760254,
      "activations/layer12_attention_weight_min": -24.57461166381836,
      "activations/layer13_attention_weight_max": 56.666812896728516,
      "activations/layer13_attention_weight_min": -48.362369537353516,
      "activations/layer14_attention_weight_max": 61.182373046875,
      "activations/layer14_attention_weight_min": -48.754146575927734,
      "activations/layer15_attention_weight_max": 56.75541305541992,
      "activations/layer15_attention_weight_min": -40.9251594543457,
      "activations/layer16_attention_weight_max": 46.17750549316406,
      "activations/layer16_attention_weight_min": -29.884586334228516,
      "activations/layer17_attention_weight_max": 58.72157287597656,
      "activations/layer17_attention_weight_min": -36.393211364746094,
      "activations/layer18_attention_weight_max": 54.01580047607422,
      "activations/layer18_attention_weight_min": -32.829803466796875,
      "activations/layer19_attention_weight_max": 20.914663314819336,
      "activations/layer19_attention_weight_min": -16.307329177856445,
      "activations/layer1_attention_weight_max": 17.704561233520508,
      "activations/layer1_attention_weight_min": -12.851372718811035,
      "activations/layer20_attention_weight_max": 20.854290008544922,
      "activations/layer20_attention_weight_min": -16.780412673950195,
      "activations/layer21_attention_weight_max": 45.493080139160156,
      "activations/layer21_attention_weight_min": -29.270156860351562,
      "activations/layer22_attention_weight_max": 37.71384811401367,
      "activations/layer22_attention_weight_min": -25.16996955871582,
      "activations/layer23_attention_weight_max": 24.585590362548828,
      "activations/layer23_attention_weight_min": -14.407991409301758,
      "activations/layer2_attention_weight_max": 27.911781311035156,
      "activations/layer2_attention_weight_min": -28.639373779296875,
      "activations/layer3_attention_weight_max": 64.71968841552734,
      "activations/layer3_attention_weight_min": -71.39157104492188,
      "activations/layer4_attention_weight_max": 95.62600708007812,
      "activations/layer4_attention_weight_min": -87.66280364990234,
      "activations/layer5_attention_weight_max": 126.61083984375,
      "activations/layer5_attention_weight_min": -102.06017303466797,
      "activations/layer6_attention_weight_max": 63.35090637207031,
      "activations/layer6_attention_weight_min": -52.30521011352539,
      "activations/layer7_attention_weight_max": 55.677162170410156,
      "activations/layer7_attention_weight_min": -55.9369010925293,
      "activations/layer8_attention_weight_max": 43.33449935913086,
      "activations/layer8_attention_weight_min": -41.07827377319336,
      "activations/layer9_attention_weight_max": 58.111202239990234,
      "activations/layer9_attention_weight_min": -57.21061325073242,
      "epoch": 2.24,
      "learning_rate": 0.00013691363636363636,
      "loss": 2.9594,
      "step": 38600
    },
    {
      "activations/layer0_attention_weight_max": 15.437512397766113,
      "activations/layer0_attention_weight_min": -14.034306526184082,
      "activations/layer10_attention_weight_max": 60.91270446777344,
      "activations/layer10_attention_weight_min": -45.914554595947266,
      "activations/layer11_attention_weight_max": 48.28022384643555,
      "activations/layer11_attention_weight_min": -36.06378173828125,
      "activations/layer12_attention_weight_max": 28.562313079833984,
      "activations/layer12_attention_weight_min": -28.147172927856445,
      "activations/layer13_attention_weight_max": 61.8722038269043,
      "activations/layer13_attention_weight_min": -44.043338775634766,
      "activations/layer14_attention_weight_max": 93.34317016601562,
      "activations/layer14_attention_weight_min": -48.21415710449219,
      "activations/layer15_attention_weight_max": 62.461326599121094,
      "activations/layer15_attention_weight_min": -33.841976165771484,
      "activations/layer16_attention_weight_max": 38.4813232421875,
      "activations/layer16_attention_weight_min": -27.388744354248047,
      "activations/layer17_attention_weight_max": 66.89173126220703,
      "activations/layer17_attention_weight_min": -35.780269622802734,
      "activations/layer18_attention_weight_max": 54.93010711669922,
      "activations/layer18_attention_weight_min": -31.222354888916016,
      "activations/layer19_attention_weight_max": 20.737760543823242,
      "activations/layer19_attention_weight_min": -16.171018600463867,
      "activations/layer1_attention_weight_max": 17.325645446777344,
      "activations/layer1_attention_weight_min": -16.419347763061523,
      "activations/layer20_attention_weight_max": 22.213584899902344,
      "activations/layer20_attention_weight_min": -16.66278839111328,
      "activations/layer21_attention_weight_max": 39.86687469482422,
      "activations/layer21_attention_weight_min": -24.77855682373047,
      "activations/layer22_attention_weight_max": 33.58667755126953,
      "activations/layer22_attention_weight_min": -22.80388069152832,
      "activations/layer23_attention_weight_max": 24.80611801147461,
      "activations/layer23_attention_weight_min": -16.146944046020508,
      "activations/layer2_attention_weight_max": 27.81964683532715,
      "activations/layer2_attention_weight_min": -27.912527084350586,
      "activations/layer3_attention_weight_max": 65.00674438476562,
      "activations/layer3_attention_weight_min": -67.90800476074219,
      "activations/layer4_attention_weight_max": 99.34423065185547,
      "activations/layer4_attention_weight_min": -87.76856994628906,
      "activations/layer5_attention_weight_max": 131.23074340820312,
      "activations/layer5_attention_weight_min": -111.54718017578125,
      "activations/layer6_attention_weight_max": 62.20419692993164,
      "activations/layer6_attention_weight_min": -53.399147033691406,
      "activations/layer7_attention_weight_max": 63.08663558959961,
      "activations/layer7_attention_weight_min": -53.14578628540039,
      "activations/layer8_attention_weight_max": 50.53819274902344,
      "activations/layer8_attention_weight_min": -40.39365768432617,
      "activations/layer9_attention_weight_max": 72.71288299560547,
      "activations/layer9_attention_weight_min": -63.57898712158203,
      "epoch": 2.25,
      "learning_rate": 0.00013689469696969697,
      "loss": 2.9489,
      "step": 38650
    },
    {
      "activations/layer0_attention_weight_max": 14.690098762512207,
      "activations/layer0_attention_weight_min": -13.895896911621094,
      "activations/layer10_attention_weight_max": 56.18913269042969,
      "activations/layer10_attention_weight_min": -43.71613311767578,
      "activations/layer11_attention_weight_max": 43.71171188354492,
      "activations/layer11_attention_weight_min": -33.726409912109375,
      "activations/layer12_attention_weight_max": 29.18537712097168,
      "activations/layer12_attention_weight_min": -25.365577697753906,
      "activations/layer13_attention_weight_max": 62.652740478515625,
      "activations/layer13_attention_weight_min": -42.08512496948242,
      "activations/layer14_attention_weight_max": 57.9224739074707,
      "activations/layer14_attention_weight_min": -36.027130126953125,
      "activations/layer15_attention_weight_max": 55.2725830078125,
      "activations/layer15_attention_weight_min": -35.73359298706055,
      "activations/layer16_attention_weight_max": 41.09386444091797,
      "activations/layer16_attention_weight_min": -29.06419563293457,
      "activations/layer17_attention_weight_max": 71.14353942871094,
      "activations/layer17_attention_weight_min": -38.04015350341797,
      "activations/layer18_attention_weight_max": 66.94686126708984,
      "activations/layer18_attention_weight_min": -35.04556655883789,
      "activations/layer19_attention_weight_max": 24.752910614013672,
      "activations/layer19_attention_weight_min": -15.389830589294434,
      "activations/layer1_attention_weight_max": 15.479512214660645,
      "activations/layer1_attention_weight_min": -17.870258331298828,
      "activations/layer20_attention_weight_max": 21.986440658569336,
      "activations/layer20_attention_weight_min": -17.062334060668945,
      "activations/layer21_attention_weight_max": 45.59186935424805,
      "activations/layer21_attention_weight_min": -23.368431091308594,
      "activations/layer22_attention_weight_max": 37.575618743896484,
      "activations/layer22_attention_weight_min": -22.238407135009766,
      "activations/layer23_attention_weight_max": 25.7679443359375,
      "activations/layer23_attention_weight_min": -14.180583953857422,
      "activations/layer2_attention_weight_max": 29.186687469482422,
      "activations/layer2_attention_weight_min": -27.90333366394043,
      "activations/layer3_attention_weight_max": 63.96739196777344,
      "activations/layer3_attention_weight_min": -69.676513671875,
      "activations/layer4_attention_weight_max": 95.57966613769531,
      "activations/layer4_attention_weight_min": -84.80242156982422,
      "activations/layer5_attention_weight_max": 122.46844482421875,
      "activations/layer5_attention_weight_min": -110.02379608154297,
      "activations/layer6_attention_weight_max": 62.17832565307617,
      "activations/layer6_attention_weight_min": -51.00199890136719,
      "activations/layer7_attention_weight_max": 54.95743942260742,
      "activations/layer7_attention_weight_min": -49.272926330566406,
      "activations/layer8_attention_weight_max": 43.926231384277344,
      "activations/layer8_attention_weight_min": -38.22832489013672,
      "activations/layer9_attention_weight_max": 56.67432403564453,
      "activations/layer9_attention_weight_min": -52.1942024230957,
      "epoch": 2.25,
      "learning_rate": 0.00013687575757575756,
      "loss": 2.954,
      "step": 38700
    },
    {
      "activations/layer0_attention_weight_max": 13.563788414001465,
      "activations/layer0_attention_weight_min": -13.729323387145996,
      "activations/layer10_attention_weight_max": 55.36253356933594,
      "activations/layer10_attention_weight_min": -43.1143798828125,
      "activations/layer11_attention_weight_max": 41.91209411621094,
      "activations/layer11_attention_weight_min": -32.138214111328125,
      "activations/layer12_attention_weight_max": 31.37221908569336,
      "activations/layer12_attention_weight_min": -27.96006965637207,
      "activations/layer13_attention_weight_max": 52.275325775146484,
      "activations/layer13_attention_weight_min": -43.14588928222656,
      "activations/layer14_attention_weight_max": 54.94565200805664,
      "activations/layer14_attention_weight_min": -40.26164245605469,
      "activations/layer15_attention_weight_max": 50.80889129638672,
      "activations/layer15_attention_weight_min": -39.55810546875,
      "activations/layer16_attention_weight_max": 38.83005905151367,
      "activations/layer16_attention_weight_min": -26.001649856567383,
      "activations/layer17_attention_weight_max": 61.43285369873047,
      "activations/layer17_attention_weight_min": -38.46091079711914,
      "activations/layer18_attention_weight_max": 56.4605827331543,
      "activations/layer18_attention_weight_min": -31.724088668823242,
      "activations/layer19_attention_weight_max": 22.42613983154297,
      "activations/layer19_attention_weight_min": -15.75443172454834,
      "activations/layer1_attention_weight_max": 16.13450050354004,
      "activations/layer1_attention_weight_min": -14.130913734436035,
      "activations/layer20_attention_weight_max": 19.890178680419922,
      "activations/layer20_attention_weight_min": -19.378103256225586,
      "activations/layer21_attention_weight_max": 42.02535629272461,
      "activations/layer21_attention_weight_min": -25.950448989868164,
      "activations/layer22_attention_weight_max": 34.61994552612305,
      "activations/layer22_attention_weight_min": -19.559839248657227,
      "activations/layer23_attention_weight_max": 23.265151977539062,
      "activations/layer23_attention_weight_min": -15.798162460327148,
      "activations/layer2_attention_weight_max": 27.866287231445312,
      "activations/layer2_attention_weight_min": -27.03437614440918,
      "activations/layer3_attention_weight_max": 62.498565673828125,
      "activations/layer3_attention_weight_min": -68.52383422851562,
      "activations/layer4_attention_weight_max": 93.94022369384766,
      "activations/layer4_attention_weight_min": -88.22946166992188,
      "activations/layer5_attention_weight_max": 117.9843978881836,
      "activations/layer5_attention_weight_min": -103.51998138427734,
      "activations/layer6_attention_weight_max": 57.945770263671875,
      "activations/layer6_attention_weight_min": -50.670223236083984,
      "activations/layer7_attention_weight_max": 52.23452377319336,
      "activations/layer7_attention_weight_min": -51.5682373046875,
      "activations/layer8_attention_weight_max": 42.186767578125,
      "activations/layer8_attention_weight_min": -36.40072250366211,
      "activations/layer9_attention_weight_max": 57.16779708862305,
      "activations/layer9_attention_weight_min": -51.681312561035156,
      "epoch": 2.25,
      "learning_rate": 0.00013685681818181817,
      "loss": 2.9517,
      "step": 38750
    },
    {
      "activations/layer0_attention_weight_max": 13.293924331665039,
      "activations/layer0_attention_weight_min": -13.954191207885742,
      "activations/layer10_attention_weight_max": 50.60990905761719,
      "activations/layer10_attention_weight_min": -41.83366775512695,
      "activations/layer11_attention_weight_max": 39.736778259277344,
      "activations/layer11_attention_weight_min": -29.874069213867188,
      "activations/layer12_attention_weight_max": 29.03992462158203,
      "activations/layer12_attention_weight_min": -24.022396087646484,
      "activations/layer13_attention_weight_max": 58.351444244384766,
      "activations/layer13_attention_weight_min": -47.058406829833984,
      "activations/layer14_attention_weight_max": 53.50305938720703,
      "activations/layer14_attention_weight_min": -42.162879943847656,
      "activations/layer15_attention_weight_max": 54.55976486206055,
      "activations/layer15_attention_weight_min": -39.89848327636719,
      "activations/layer16_attention_weight_max": 41.378597259521484,
      "activations/layer16_attention_weight_min": -28.793701171875,
      "activations/layer17_attention_weight_max": 64.88609313964844,
      "activations/layer17_attention_weight_min": -35.82609558105469,
      "activations/layer18_attention_weight_max": 63.69282531738281,
      "activations/layer18_attention_weight_min": -33.00657272338867,
      "activations/layer19_attention_weight_max": 20.66361427307129,
      "activations/layer19_attention_weight_min": -17.160398483276367,
      "activations/layer1_attention_weight_max": 16.85648536682129,
      "activations/layer1_attention_weight_min": -14.300490379333496,
      "activations/layer20_attention_weight_max": 23.524681091308594,
      "activations/layer20_attention_weight_min": -17.854143142700195,
      "activations/layer21_attention_weight_max": 51.13608932495117,
      "activations/layer21_attention_weight_min": -30.090496063232422,
      "activations/layer22_attention_weight_max": 38.410980224609375,
      "activations/layer22_attention_weight_min": -24.810380935668945,
      "activations/layer23_attention_weight_max": 25.061092376708984,
      "activations/layer23_attention_weight_min": -17.780452728271484,
      "activations/layer2_attention_weight_max": 27.917152404785156,
      "activations/layer2_attention_weight_min": -27.945837020874023,
      "activations/layer3_attention_weight_max": 67.08879852294922,
      "activations/layer3_attention_weight_min": -70.18009948730469,
      "activations/layer4_attention_weight_max": 93.27416229248047,
      "activations/layer4_attention_weight_min": -86.4200439453125,
      "activations/layer5_attention_weight_max": 121.6502914428711,
      "activations/layer5_attention_weight_min": -107.76910400390625,
      "activations/layer6_attention_weight_max": 61.45708465576172,
      "activations/layer6_attention_weight_min": -53.76316452026367,
      "activations/layer7_attention_weight_max": 54.912078857421875,
      "activations/layer7_attention_weight_min": -50.90031814575195,
      "activations/layer8_attention_weight_max": 43.080448150634766,
      "activations/layer8_attention_weight_min": -37.06767654418945,
      "activations/layer9_attention_weight_max": 61.54519271850586,
      "activations/layer9_attention_weight_min": -55.72794723510742,
      "epoch": 2.25,
      "learning_rate": 0.00013683787878787876,
      "loss": 2.9662,
      "step": 38800
    },
    {
      "activations/layer0_attention_weight_max": 13.4212007522583,
      "activations/layer0_attention_weight_min": -14.268719673156738,
      "activations/layer10_attention_weight_max": 57.0262565612793,
      "activations/layer10_attention_weight_min": -42.853660583496094,
      "activations/layer11_attention_weight_max": 41.80950164794922,
      "activations/layer11_attention_weight_min": -32.0565071105957,
      "activations/layer12_attention_weight_max": 32.159446716308594,
      "activations/layer12_attention_weight_min": -25.311628341674805,
      "activations/layer13_attention_weight_max": 59.06097412109375,
      "activations/layer13_attention_weight_min": -40.476165771484375,
      "activations/layer14_attention_weight_max": 52.15584182739258,
      "activations/layer14_attention_weight_min": -34.149532318115234,
      "activations/layer15_attention_weight_max": 52.42145919799805,
      "activations/layer15_attention_weight_min": -34.62166213989258,
      "activations/layer16_attention_weight_max": 39.36793899536133,
      "activations/layer16_attention_weight_min": -28.346296310424805,
      "activations/layer17_attention_weight_max": 67.23944854736328,
      "activations/layer17_attention_weight_min": -37.558441162109375,
      "activations/layer18_attention_weight_max": 54.16753387451172,
      "activations/layer18_attention_weight_min": -32.45899963378906,
      "activations/layer19_attention_weight_max": 23.248674392700195,
      "activations/layer19_attention_weight_min": -16.64320182800293,
      "activations/layer1_attention_weight_max": 16.38268280029297,
      "activations/layer1_attention_weight_min": -14.486743927001953,
      "activations/layer20_attention_weight_max": 20.44413948059082,
      "activations/layer20_attention_weight_min": -18.473188400268555,
      "activations/layer21_attention_weight_max": 42.17940902709961,
      "activations/layer21_attention_weight_min": -24.74861717224121,
      "activations/layer22_attention_weight_max": 34.740150451660156,
      "activations/layer22_attention_weight_min": -20.54573631286621,
      "activations/layer23_attention_weight_max": 27.691205978393555,
      "activations/layer23_attention_weight_min": -14.998157501220703,
      "activations/layer2_attention_weight_max": 28.18733024597168,
      "activations/layer2_attention_weight_min": -27.253273010253906,
      "activations/layer3_attention_weight_max": 67.18456268310547,
      "activations/layer3_attention_weight_min": -72.6512222290039,
      "activations/layer4_attention_weight_max": 98.62884521484375,
      "activations/layer4_attention_weight_min": -89.58867645263672,
      "activations/layer5_attention_weight_max": 124.40228271484375,
      "activations/layer5_attention_weight_min": -102.67019653320312,
      "activations/layer6_attention_weight_max": 64.0895767211914,
      "activations/layer6_attention_weight_min": -51.7833137512207,
      "activations/layer7_attention_weight_max": 59.473472595214844,
      "activations/layer7_attention_weight_min": -49.96943283081055,
      "activations/layer8_attention_weight_max": 46.7412223815918,
      "activations/layer8_attention_weight_min": -37.98921203613281,
      "activations/layer9_attention_weight_max": 60.58051681518555,
      "activations/layer9_attention_weight_min": -49.7868766784668,
      "epoch": 2.26,
      "learning_rate": 0.00013681893939393938,
      "loss": 2.9755,
      "step": 38850
    },
    {
      "activations/layer0_attention_weight_max": 15.278508186340332,
      "activations/layer0_attention_weight_min": -14.447676658630371,
      "activations/layer10_attention_weight_max": 65.43255615234375,
      "activations/layer10_attention_weight_min": -41.96052932739258,
      "activations/layer11_attention_weight_max": 49.7335319519043,
      "activations/layer11_attention_weight_min": -33.83698272705078,
      "activations/layer12_attention_weight_max": 28.483842849731445,
      "activations/layer12_attention_weight_min": -27.6589298248291,
      "activations/layer13_attention_weight_max": 77.72594451904297,
      "activations/layer13_attention_weight_min": -45.49305725097656,
      "activations/layer14_attention_weight_max": 79.30781555175781,
      "activations/layer14_attention_weight_min": -44.63462448120117,
      "activations/layer15_attention_weight_max": 62.94194793701172,
      "activations/layer15_attention_weight_min": -35.888343811035156,
      "activations/layer16_attention_weight_max": 49.62384796142578,
      "activations/layer16_attention_weight_min": -25.51927375793457,
      "activations/layer17_attention_weight_max": 72.40036010742188,
      "activations/layer17_attention_weight_min": -39.124183654785156,
      "activations/layer18_attention_weight_max": 61.76613235473633,
      "activations/layer18_attention_weight_min": -32.82307815551758,
      "activations/layer19_attention_weight_max": 28.805845260620117,
      "activations/layer19_attention_weight_min": -17.192480087280273,
      "activations/layer1_attention_weight_max": 17.014976501464844,
      "activations/layer1_attention_weight_min": -13.83460521697998,
      "activations/layer20_attention_weight_max": 20.423276901245117,
      "activations/layer20_attention_weight_min": -19.272724151611328,
      "activations/layer21_attention_weight_max": 53.52301025390625,
      "activations/layer21_attention_weight_min": -25.96776580810547,
      "activations/layer22_attention_weight_max": 39.6525764465332,
      "activations/layer22_attention_weight_min": -24.026702880859375,
      "activations/layer23_attention_weight_max": 25.438125610351562,
      "activations/layer23_attention_weight_min": -17.025794982910156,
      "activations/layer2_attention_weight_max": 27.691946029663086,
      "activations/layer2_attention_weight_min": -25.331470489501953,
      "activations/layer3_attention_weight_max": 64.10894775390625,
      "activations/layer3_attention_weight_min": -70.47223663330078,
      "activations/layer4_attention_weight_max": 95.9831314086914,
      "activations/layer4_attention_weight_min": -93.05523681640625,
      "activations/layer5_attention_weight_max": 134.40452575683594,
      "activations/layer5_attention_weight_min": -106.81307220458984,
      "activations/layer6_attention_weight_max": 65.52560424804688,
      "activations/layer6_attention_weight_min": -54.602901458740234,
      "activations/layer7_attention_weight_max": 64.33917236328125,
      "activations/layer7_attention_weight_min": -65.08888244628906,
      "activations/layer8_attention_weight_max": 55.1276969909668,
      "activations/layer8_attention_weight_min": -44.298316955566406,
      "activations/layer9_attention_weight_max": 74.2880630493164,
      "activations/layer9_attention_weight_min": -61.96345901489258,
      "epoch": 2.26,
      "learning_rate": 0.0001368,
      "loss": 2.9516,
      "step": 38900
    },
    {
      "activations/layer0_attention_weight_max": 13.477959632873535,
      "activations/layer0_attention_weight_min": -13.756437301635742,
      "activations/layer10_attention_weight_max": 51.415863037109375,
      "activations/layer10_attention_weight_min": -43.09352111816406,
      "activations/layer11_attention_weight_max": 48.259483337402344,
      "activations/layer11_attention_weight_min": -33.883968353271484,
      "activations/layer12_attention_weight_max": 28.71981430053711,
      "activations/layer12_attention_weight_min": -23.486106872558594,
      "activations/layer13_attention_weight_max": 53.11806106567383,
      "activations/layer13_attention_weight_min": -41.96776580810547,
      "activations/layer14_attention_weight_max": 51.56108093261719,
      "activations/layer14_attention_weight_min": -38.84489822387695,
      "activations/layer15_attention_weight_max": 50.95998001098633,
      "activations/layer15_attention_weight_min": -34.5327262878418,
      "activations/layer16_attention_weight_max": 38.67490005493164,
      "activations/layer16_attention_weight_min": -27.046527862548828,
      "activations/layer17_attention_weight_max": 57.87887191772461,
      "activations/layer17_attention_weight_min": -36.592594146728516,
      "activations/layer18_attention_weight_max": 54.271697998046875,
      "activations/layer18_attention_weight_min": -32.35398483276367,
      "activations/layer19_attention_weight_max": 20.79944610595703,
      "activations/layer19_attention_weight_min": -18.17946434020996,
      "activations/layer1_attention_weight_max": 17.218358993530273,
      "activations/layer1_attention_weight_min": -12.250829696655273,
      "activations/layer20_attention_weight_max": 18.46136474609375,
      "activations/layer20_attention_weight_min": -16.55063247680664,
      "activations/layer21_attention_weight_max": 41.4301643371582,
      "activations/layer21_attention_weight_min": -26.14360237121582,
      "activations/layer22_attention_weight_max": 32.56058120727539,
      "activations/layer22_attention_weight_min": -22.745420455932617,
      "activations/layer23_attention_weight_max": 22.99811553955078,
      "activations/layer23_attention_weight_min": -14.028377532958984,
      "activations/layer2_attention_weight_max": 26.976726531982422,
      "activations/layer2_attention_weight_min": -26.063289642333984,
      "activations/layer3_attention_weight_max": 62.01350784301758,
      "activations/layer3_attention_weight_min": -67.70036315917969,
      "activations/layer4_attention_weight_max": 90.4459457397461,
      "activations/layer4_attention_weight_min": -85.07564544677734,
      "activations/layer5_attention_weight_max": 120.51187133789062,
      "activations/layer5_attention_weight_min": -101.44713592529297,
      "activations/layer6_attention_weight_max": 58.67582321166992,
      "activations/layer6_attention_weight_min": -51.3286247253418,
      "activations/layer7_attention_weight_max": 51.0088005065918,
      "activations/layer7_attention_weight_min": -49.13291549682617,
      "activations/layer8_attention_weight_max": 40.33317565917969,
      "activations/layer8_attention_weight_min": -36.68291473388672,
      "activations/layer9_attention_weight_max": 57.0652961730957,
      "activations/layer9_attention_weight_min": -51.342864990234375,
      "epoch": 2.26,
      "learning_rate": 0.00013678106060606058,
      "loss": 2.9516,
      "step": 38950
    },
    {
      "activations/layer0_attention_weight_max": 15.683808326721191,
      "activations/layer0_attention_weight_min": -13.906031608581543,
      "activations/layer10_attention_weight_max": 54.41748809814453,
      "activations/layer10_attention_weight_min": -44.16291046142578,
      "activations/layer11_attention_weight_max": 39.73473358154297,
      "activations/layer11_attention_weight_min": -32.526580810546875,
      "activations/layer12_attention_weight_max": 27.102497100830078,
      "activations/layer12_attention_weight_min": -26.127487182617188,
      "activations/layer13_attention_weight_max": 51.52296829223633,
      "activations/layer13_attention_weight_min": -40.202735900878906,
      "activations/layer14_attention_weight_max": 52.23334884643555,
      "activations/layer14_attention_weight_min": -37.620147705078125,
      "activations/layer15_attention_weight_max": 48.8954963684082,
      "activations/layer15_attention_weight_min": -37.419185638427734,
      "activations/layer16_attention_weight_max": 45.5216064453125,
      "activations/layer16_attention_weight_min": -29.36494255065918,
      "activations/layer17_attention_weight_max": 65.4826431274414,
      "activations/layer17_attention_weight_min": -38.284629821777344,
      "activations/layer18_attention_weight_max": 58.832069396972656,
      "activations/layer18_attention_weight_min": -33.33930969238281,
      "activations/layer19_attention_weight_max": 24.224172592163086,
      "activations/layer19_attention_weight_min": -17.496143341064453,
      "activations/layer1_attention_weight_max": 18.339004516601562,
      "activations/layer1_attention_weight_min": -15.776883125305176,
      "activations/layer20_attention_weight_max": 22.145118713378906,
      "activations/layer20_attention_weight_min": -19.08877944946289,
      "activations/layer21_attention_weight_max": 46.89121627807617,
      "activations/layer21_attention_weight_min": -25.71335220336914,
      "activations/layer22_attention_weight_max": 39.21923828125,
      "activations/layer22_attention_weight_min": -24.85118293762207,
      "activations/layer23_attention_weight_max": 27.553571701049805,
      "activations/layer23_attention_weight_min": -16.502498626708984,
      "activations/layer2_attention_weight_max": 28.849700927734375,
      "activations/layer2_attention_weight_min": -27.621803283691406,
      "activations/layer3_attention_weight_max": 66.6683578491211,
      "activations/layer3_attention_weight_min": -69.50645446777344,
      "activations/layer4_attention_weight_max": 96.01731872558594,
      "activations/layer4_attention_weight_min": -82.33385467529297,
      "activations/layer5_attention_weight_max": 129.77171325683594,
      "activations/layer5_attention_weight_min": -99.47313690185547,
      "activations/layer6_attention_weight_max": 63.766014099121094,
      "activations/layer6_attention_weight_min": -51.1883430480957,
      "activations/layer7_attention_weight_max": 57.99196243286133,
      "activations/layer7_attention_weight_min": -52.03155517578125,
      "activations/layer8_attention_weight_max": 42.65113067626953,
      "activations/layer8_attention_weight_min": -36.73516082763672,
      "activations/layer9_attention_weight_max": 58.46479415893555,
      "activations/layer9_attention_weight_min": -54.017913818359375,
      "epoch": 2.27,
      "learning_rate": 0.0001367621212121212,
      "loss": 2.95,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_loss": 2.896484375,
      "eval_runtime": 8.4739,
      "eval_samples_per_second": 506.735,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_openwebtext_loss": 2.896484375,
      "eval_openwebtext_ppl": 18.110364070885183,
      "eval_openwebtext_runtime": 8.4739,
      "eval_openwebtext_samples_per_second": 506.735,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_wikitext_loss": 3.15234375,
      "eval_wikitext_ppl": 23.390822626823073,
      "eval_wikitext_runtime": 1.9656,
      "eval_wikitext_samples_per_second": 231.985,
      "step": 39000
    },
    {
      "epoch": 2.27,
      "eval_lambada_loss": 3.083984375,
      "eval_lambada_ppl": 21.845268970328604,
      "eval_lambada_runtime": 9.5445,
      "eval_lambada_samples_per_second": 510.137,
      "step": 39000
    },
    {
      "activations/layer0_attention_weight_max": 15.330100059509277,
      "activations/layer0_attention_weight_min": -13.435968399047852,
      "activations/layer10_attention_weight_max": 49.98601150512695,
      "activations/layer10_attention_weight_min": -42.11448287963867,
      "activations/layer11_attention_weight_max": 38.253631591796875,
      "activations/layer11_attention_weight_min": -32.868385314941406,
      "activations/layer12_attention_weight_max": 26.901838302612305,
      "activations/layer12_attention_weight_min": -25.75519371032715,
      "activations/layer13_attention_weight_max": 69.76268005371094,
      "activations/layer13_attention_weight_min": -47.02186965942383,
      "activations/layer14_attention_weight_max": 60.912662506103516,
      "activations/layer14_attention_weight_min": -38.96513366699219,
      "activations/layer15_attention_weight_max": 54.57619857788086,
      "activations/layer15_attention_weight_min": -38.1181640625,
      "activations/layer16_attention_weight_max": 41.82686233520508,
      "activations/layer16_attention_weight_min": -27.41797637939453,
      "activations/layer17_attention_weight_max": 60.92022705078125,
      "activations/layer17_attention_weight_min": -37.341773986816406,
      "activations/layer18_attention_weight_max": 51.70669937133789,
      "activations/layer18_attention_weight_min": -34.016536712646484,
      "activations/layer19_attention_weight_max": 22.380022048950195,
      "activations/layer19_attention_weight_min": -15.916481018066406,
      "activations/layer1_attention_weight_max": 17.656156539916992,
      "activations/layer1_attention_weight_min": -13.302519798278809,
      "activations/layer20_attention_weight_max": 22.008502960205078,
      "activations/layer20_attention_weight_min": -21.19548225402832,
      "activations/layer21_attention_weight_max": 39.8966178894043,
      "activations/layer21_attention_weight_min": -24.56783103942871,
      "activations/layer22_attention_weight_max": 38.41532516479492,
      "activations/layer22_attention_weight_min": -23.334758758544922,
      "activations/layer23_attention_weight_max": 22.807174682617188,
      "activations/layer23_attention_weight_min": -14.035815238952637,
      "activations/layer2_attention_weight_max": 29.043439865112305,
      "activations/layer2_attention_weight_min": -28.258695602416992,
      "activations/layer3_attention_weight_max": 65.76746368408203,
      "activations/layer3_attention_weight_min": -66.88945007324219,
      "activations/layer4_attention_weight_max": 92.19601440429688,
      "activations/layer4_attention_weight_min": -81.4847640991211,
      "activations/layer5_attention_weight_max": 117.69308471679688,
      "activations/layer5_attention_weight_min": -97.84371948242188,
      "activations/layer6_attention_weight_max": 59.66874694824219,
      "activations/layer6_attention_weight_min": -50.293113708496094,
      "activations/layer7_attention_weight_max": 55.31493377685547,
      "activations/layer7_attention_weight_min": -50.62502670288086,
      "activations/layer8_attention_weight_max": 40.843204498291016,
      "activations/layer8_attention_weight_min": -36.65138626098633,
      "activations/layer9_attention_weight_max": 64.10589599609375,
      "activations/layer9_attention_weight_min": -51.26534652709961,
      "epoch": 2.27,
      "learning_rate": 0.0001367431818181818,
      "loss": 2.9384,
      "step": 39050
    },
    {
      "activations/layer0_attention_weight_max": 15.291365623474121,
      "activations/layer0_attention_weight_min": -14.029195785522461,
      "activations/layer10_attention_weight_max": 59.53842544555664,
      "activations/layer10_attention_weight_min": -44.093345642089844,
      "activations/layer11_attention_weight_max": 52.364925384521484,
      "activations/layer11_attention_weight_min": -33.21490478515625,
      "activations/layer12_attention_weight_max": 31.677427291870117,
      "activations/layer12_attention_weight_min": -25.492610931396484,
      "activations/layer13_attention_weight_max": 79.13445281982422,
      "activations/layer13_attention_weight_min": -47.58127975463867,
      "activations/layer14_attention_weight_max": 79.88715362548828,
      "activations/layer14_attention_weight_min": -47.360374450683594,
      "activations/layer15_attention_weight_max": 49.0793342590332,
      "activations/layer15_attention_weight_min": -35.924137115478516,
      "activations/layer16_attention_weight_max": 40.015960693359375,
      "activations/layer16_attention_weight_min": -26.23110008239746,
      "activations/layer17_attention_weight_max": 55.79543685913086,
      "activations/layer17_attention_weight_min": -37.6782112121582,
      "activations/layer18_attention_weight_max": 57.297645568847656,
      "activations/layer18_attention_weight_min": -32.033111572265625,
      "activations/layer19_attention_weight_max": 19.220325469970703,
      "activations/layer19_attention_weight_min": -16.958303451538086,
      "activations/layer1_attention_weight_max": 16.632041931152344,
      "activations/layer1_attention_weight_min": -14.539194107055664,
      "activations/layer20_attention_weight_max": 20.619901657104492,
      "activations/layer20_attention_weight_min": -18.740093231201172,
      "activations/layer21_attention_weight_max": 41.53240203857422,
      "activations/layer21_attention_weight_min": -27.32238006591797,
      "activations/layer22_attention_weight_max": 34.640323638916016,
      "activations/layer22_attention_weight_min": -23.91643714904785,
      "activations/layer23_attention_weight_max": 25.452749252319336,
      "activations/layer23_attention_weight_min": -15.659126281738281,
      "activations/layer2_attention_weight_max": 30.286972045898438,
      "activations/layer2_attention_weight_min": -28.24146270751953,
      "activations/layer3_attention_weight_max": 68.49008178710938,
      "activations/layer3_attention_weight_min": -72.34541320800781,
      "activations/layer4_attention_weight_max": 98.42298889160156,
      "activations/layer4_attention_weight_min": -87.55824279785156,
      "activations/layer5_attention_weight_max": 138.09902954101562,
      "activations/layer5_attention_weight_min": -104.6644515991211,
      "activations/layer6_attention_weight_max": 69.36324310302734,
      "activations/layer6_attention_weight_min": -52.31205368041992,
      "activations/layer7_attention_weight_max": 63.7862548828125,
      "activations/layer7_attention_weight_min": -54.12714385986328,
      "activations/layer8_attention_weight_max": 58.35734939575195,
      "activations/layer8_attention_weight_min": -40.73732376098633,
      "activations/layer9_attention_weight_max": 79.01751708984375,
      "activations/layer9_attention_weight_min": -68.0609359741211,
      "epoch": 2.27,
      "learning_rate": 0.0001367242424242424,
      "loss": 2.9483,
      "step": 39100
    },
    {
      "activations/layer0_attention_weight_max": 14.865639686584473,
      "activations/layer0_attention_weight_min": -13.746953964233398,
      "activations/layer10_attention_weight_max": 52.32871627807617,
      "activations/layer10_attention_weight_min": -42.1532096862793,
      "activations/layer11_attention_weight_max": 39.0750732421875,
      "activations/layer11_attention_weight_min": -31.272083282470703,
      "activations/layer12_attention_weight_max": 25.08226203918457,
      "activations/layer12_attention_weight_min": -27.645105361938477,
      "activations/layer13_attention_weight_max": 55.95917892456055,
      "activations/layer13_attention_weight_min": -44.590476989746094,
      "activations/layer14_attention_weight_max": 54.71517562866211,
      "activations/layer14_attention_weight_min": -38.210411071777344,
      "activations/layer15_attention_weight_max": 46.200103759765625,
      "activations/layer15_attention_weight_min": -35.22195053100586,
      "activations/layer16_attention_weight_max": 42.90950393676758,
      "activations/layer16_attention_weight_min": -27.01524543762207,
      "activations/layer17_attention_weight_max": 61.04637908935547,
      "activations/layer17_attention_weight_min": -35.373531341552734,
      "activations/layer18_attention_weight_max": 62.53491973876953,
      "activations/layer18_attention_weight_min": -33.339202880859375,
      "activations/layer19_attention_weight_max": 21.194887161254883,
      "activations/layer19_attention_weight_min": -16.020505905151367,
      "activations/layer1_attention_weight_max": 18.048847198486328,
      "activations/layer1_attention_weight_min": -14.36805534362793,
      "activations/layer20_attention_weight_max": 21.719255447387695,
      "activations/layer20_attention_weight_min": -16.774921417236328,
      "activations/layer21_attention_weight_max": 40.026580810546875,
      "activations/layer21_attention_weight_min": -24.333786010742188,
      "activations/layer22_attention_weight_max": 38.19710922241211,
      "activations/layer22_attention_weight_min": -22.588804244995117,
      "activations/layer23_attention_weight_max": 22.905027389526367,
      "activations/layer23_attention_weight_min": -14.02099895477295,
      "activations/layer2_attention_weight_max": 27.781246185302734,
      "activations/layer2_attention_weight_min": -27.796810150146484,
      "activations/layer3_attention_weight_max": 66.19031524658203,
      "activations/layer3_attention_weight_min": -68.31745910644531,
      "activations/layer4_attention_weight_max": 93.77830505371094,
      "activations/layer4_attention_weight_min": -87.45210266113281,
      "activations/layer5_attention_weight_max": 120.81454467773438,
      "activations/layer5_attention_weight_min": -99.04216003417969,
      "activations/layer6_attention_weight_max": 63.433292388916016,
      "activations/layer6_attention_weight_min": -51.16847610473633,
      "activations/layer7_attention_weight_max": 56.81764221191406,
      "activations/layer7_attention_weight_min": -47.79383850097656,
      "activations/layer8_attention_weight_max": 42.98323440551758,
      "activations/layer8_attention_weight_min": -34.20286178588867,
      "activations/layer9_attention_weight_max": 63.03940963745117,
      "activations/layer9_attention_weight_min": -60.8881721496582,
      "epoch": 2.27,
      "learning_rate": 0.000136705303030303,
      "loss": 2.9559,
      "step": 39150
    },
    {
      "activations/layer0_attention_weight_max": 15.113869667053223,
      "activations/layer0_attention_weight_min": -13.265094757080078,
      "activations/layer10_attention_weight_max": 51.22998809814453,
      "activations/layer10_attention_weight_min": -47.06315994262695,
      "activations/layer11_attention_weight_max": 41.944175720214844,
      "activations/layer11_attention_weight_min": -33.342044830322266,
      "activations/layer12_attention_weight_max": 24.669687271118164,
      "activations/layer12_attention_weight_min": -27.45509147644043,
      "activations/layer13_attention_weight_max": 65.01107788085938,
      "activations/layer13_attention_weight_min": -46.33566665649414,
      "activations/layer14_attention_weight_max": 69.04484558105469,
      "activations/layer14_attention_weight_min": -50.27546691894531,
      "activations/layer15_attention_weight_max": 49.973758697509766,
      "activations/layer15_attention_weight_min": -37.570465087890625,
      "activations/layer16_attention_weight_max": 38.508697509765625,
      "activations/layer16_attention_weight_min": -27.468055725097656,
      "activations/layer17_attention_weight_max": 56.488643646240234,
      "activations/layer17_attention_weight_min": -35.464744567871094,
      "activations/layer18_attention_weight_max": 53.40492248535156,
      "activations/layer18_attention_weight_min": -31.6684513092041,
      "activations/layer19_attention_weight_max": 22.9212646484375,
      "activations/layer19_attention_weight_min": -16.30544090270996,
      "activations/layer1_attention_weight_max": 17.356645584106445,
      "activations/layer1_attention_weight_min": -13.362544059753418,
      "activations/layer20_attention_weight_max": 19.45416259765625,
      "activations/layer20_attention_weight_min": -17.345972061157227,
      "activations/layer21_attention_weight_max": 38.588470458984375,
      "activations/layer21_attention_weight_min": -26.27379608154297,
      "activations/layer22_attention_weight_max": 36.59361267089844,
      "activations/layer22_attention_weight_min": -21.300954818725586,
      "activations/layer23_attention_weight_max": 23.627965927124023,
      "activations/layer23_attention_weight_min": -15.026117324829102,
      "activations/layer2_attention_weight_max": 27.436153411865234,
      "activations/layer2_attention_weight_min": -27.79986572265625,
      "activations/layer3_attention_weight_max": 64.67276000976562,
      "activations/layer3_attention_weight_min": -68.03748321533203,
      "activations/layer4_attention_weight_max": 99.46240234375,
      "activations/layer4_attention_weight_min": -87.72689819335938,
      "activations/layer5_attention_weight_max": 131.44337463378906,
      "activations/layer5_attention_weight_min": -108.35699462890625,
      "activations/layer6_attention_weight_max": 63.067649841308594,
      "activations/layer6_attention_weight_min": -53.0195426940918,
      "activations/layer7_attention_weight_max": 58.9725456237793,
      "activations/layer7_attention_weight_min": -54.78718948364258,
      "activations/layer8_attention_weight_max": 47.45542907714844,
      "activations/layer8_attention_weight_min": -37.63932800292969,
      "activations/layer9_attention_weight_max": 72.10328674316406,
      "activations/layer9_attention_weight_min": -60.82584762573242,
      "epoch": 2.28,
      "learning_rate": 0.00013668636363636363,
      "loss": 2.9578,
      "step": 39200
    },
    {
      "activations/layer0_attention_weight_max": 13.378381729125977,
      "activations/layer0_attention_weight_min": -13.536396980285645,
      "activations/layer10_attention_weight_max": 60.55618667602539,
      "activations/layer10_attention_weight_min": -50.520809173583984,
      "activations/layer11_attention_weight_max": 49.141578674316406,
      "activations/layer11_attention_weight_min": -36.16345977783203,
      "activations/layer12_attention_weight_max": 35.564083099365234,
      "activations/layer12_attention_weight_min": -26.756956100463867,
      "activations/layer13_attention_weight_max": 76.84030151367188,
      "activations/layer13_attention_weight_min": -50.155216217041016,
      "activations/layer14_attention_weight_max": 67.5936508178711,
      "activations/layer14_attention_weight_min": -40.01288986206055,
      "activations/layer15_attention_weight_max": 53.430213928222656,
      "activations/layer15_attention_weight_min": -36.824562072753906,
      "activations/layer16_attention_weight_max": 46.68385696411133,
      "activations/layer16_attention_weight_min": -29.253799438476562,
      "activations/layer17_attention_weight_max": 64.8495864868164,
      "activations/layer17_attention_weight_min": -39.62344741821289,
      "activations/layer18_attention_weight_max": 60.047183990478516,
      "activations/layer18_attention_weight_min": -31.173311233520508,
      "activations/layer19_attention_weight_max": 20.738605499267578,
      "activations/layer19_attention_weight_min": -17.565641403198242,
      "activations/layer1_attention_weight_max": 16.587997436523438,
      "activations/layer1_attention_weight_min": -15.06885814666748,
      "activations/layer20_attention_weight_max": 20.68395233154297,
      "activations/layer20_attention_weight_min": -16.17308235168457,
      "activations/layer21_attention_weight_max": 43.27617645263672,
      "activations/layer21_attention_weight_min": -26.231456756591797,
      "activations/layer22_attention_weight_max": 43.08757019042969,
      "activations/layer22_attention_weight_min": -24.68645668029785,
      "activations/layer23_attention_weight_max": 23.98200225830078,
      "activations/layer23_attention_weight_min": -15.388533592224121,
      "activations/layer2_attention_weight_max": 29.59686279296875,
      "activations/layer2_attention_weight_min": -28.51144027709961,
      "activations/layer3_attention_weight_max": 67.19474792480469,
      "activations/layer3_attention_weight_min": -68.50988006591797,
      "activations/layer4_attention_weight_max": 94.55695343017578,
      "activations/layer4_attention_weight_min": -86.3460464477539,
      "activations/layer5_attention_weight_max": 122.75518798828125,
      "activations/layer5_attention_weight_min": -101.2911605834961,
      "activations/layer6_attention_weight_max": 59.685569763183594,
      "activations/layer6_attention_weight_min": -49.4324836730957,
      "activations/layer7_attention_weight_max": 55.38123321533203,
      "activations/layer7_attention_weight_min": -50.075740814208984,
      "activations/layer8_attention_weight_max": 43.47556686401367,
      "activations/layer8_attention_weight_min": -37.704917907714844,
      "activations/layer9_attention_weight_max": 62.184303283691406,
      "activations/layer9_attention_weight_min": -53.6956672668457,
      "epoch": 2.28,
      "learning_rate": 0.00013666742424242422,
      "loss": 2.9462,
      "step": 39250
    },
    {
      "activations/layer0_attention_weight_max": 14.128544807434082,
      "activations/layer0_attention_weight_min": -13.559669494628906,
      "activations/layer10_attention_weight_max": 53.05960464477539,
      "activations/layer10_attention_weight_min": -42.25278854370117,
      "activations/layer11_attention_weight_max": 46.048622131347656,
      "activations/layer11_attention_weight_min": -32.51890563964844,
      "activations/layer12_attention_weight_max": 35.06795883178711,
      "activations/layer12_attention_weight_min": -25.231958389282227,
      "activations/layer13_attention_weight_max": 60.848121643066406,
      "activations/layer13_attention_weight_min": -39.8896598815918,
      "activations/layer14_attention_weight_max": 49.55257034301758,
      "activations/layer14_attention_weight_min": -31.354854583740234,
      "activations/layer15_attention_weight_max": 54.35147476196289,
      "activations/layer15_attention_weight_min": -36.86184310913086,
      "activations/layer16_attention_weight_max": 43.33799743652344,
      "activations/layer16_attention_weight_min": -27.212175369262695,
      "activations/layer17_attention_weight_max": 60.6024169921875,
      "activations/layer17_attention_weight_min": -37.93052673339844,
      "activations/layer18_attention_weight_max": 54.94731140136719,
      "activations/layer18_attention_weight_min": -33.711544036865234,
      "activations/layer19_attention_weight_max": 22.94268226623535,
      "activations/layer19_attention_weight_min": -15.966517448425293,
      "activations/layer1_attention_weight_max": 16.367475509643555,
      "activations/layer1_attention_weight_min": -14.287906646728516,
      "activations/layer20_attention_weight_max": 21.489683151245117,
      "activations/layer20_attention_weight_min": -18.555625915527344,
      "activations/layer21_attention_weight_max": 43.471492767333984,
      "activations/layer21_attention_weight_min": -26.489830017089844,
      "activations/layer22_attention_weight_max": 37.23246383666992,
      "activations/layer22_attention_weight_min": -25.731178283691406,
      "activations/layer23_attention_weight_max": 24.22027587890625,
      "activations/layer23_attention_weight_min": -15.611322402954102,
      "activations/layer2_attention_weight_max": 28.138652801513672,
      "activations/layer2_attention_weight_min": -26.35772705078125,
      "activations/layer3_attention_weight_max": 61.53264236450195,
      "activations/layer3_attention_weight_min": -67.38706970214844,
      "activations/layer4_attention_weight_max": 94.69249725341797,
      "activations/layer4_attention_weight_min": -83.41378021240234,
      "activations/layer5_attention_weight_max": 128.45729064941406,
      "activations/layer5_attention_weight_min": -100.43218231201172,
      "activations/layer6_attention_weight_max": 61.14037322998047,
      "activations/layer6_attention_weight_min": -52.618167877197266,
      "activations/layer7_attention_weight_max": 58.45901870727539,
      "activations/layer7_attention_weight_min": -51.24510192871094,
      "activations/layer8_attention_weight_max": 43.485260009765625,
      "activations/layer8_attention_weight_min": -36.98198699951172,
      "activations/layer9_attention_weight_max": 59.06571578979492,
      "activations/layer9_attention_weight_min": -55.077430725097656,
      "epoch": 2.28,
      "learning_rate": 0.00013664848484848483,
      "loss": 2.9502,
      "step": 39300
    },
    {
      "activations/layer0_attention_weight_max": 15.571240425109863,
      "activations/layer0_attention_weight_min": -13.592287063598633,
      "activations/layer10_attention_weight_max": 48.899410247802734,
      "activations/layer10_attention_weight_min": -43.75303649902344,
      "activations/layer11_attention_weight_max": 40.7012825012207,
      "activations/layer11_attention_weight_min": -32.671661376953125,
      "activations/layer12_attention_weight_max": 27.909156799316406,
      "activations/layer12_attention_weight_min": -25.1090030670166,
      "activations/layer13_attention_weight_max": 52.365379333496094,
      "activations/layer13_attention_weight_min": -39.33561325073242,
      "activations/layer14_attention_weight_max": 42.85016632080078,
      "activations/layer14_attention_weight_min": -31.306779861450195,
      "activations/layer15_attention_weight_max": 48.58122634887695,
      "activations/layer15_attention_weight_min": -33.089698791503906,
      "activations/layer16_attention_weight_max": 39.79745101928711,
      "activations/layer16_attention_weight_min": -27.62334632873535,
      "activations/layer17_attention_weight_max": 58.08876419067383,
      "activations/layer17_attention_weight_min": -37.936092376708984,
      "activations/layer18_attention_weight_max": 67.2540054321289,
      "activations/layer18_attention_weight_min": -33.910789489746094,
      "activations/layer19_attention_weight_max": 21.177385330200195,
      "activations/layer19_attention_weight_min": -17.215795516967773,
      "activations/layer1_attention_weight_max": 17.964710235595703,
      "activations/layer1_attention_weight_min": -13.050548553466797,
      "activations/layer20_attention_weight_max": 20.12171173095703,
      "activations/layer20_attention_weight_min": -19.81740379333496,
      "activations/layer21_attention_weight_max": 43.66993713378906,
      "activations/layer21_attention_weight_min": -28.2808837890625,
      "activations/layer22_attention_weight_max": 34.66370391845703,
      "activations/layer22_attention_weight_min": -23.073816299438477,
      "activations/layer23_attention_weight_max": 22.326034545898438,
      "activations/layer23_attention_weight_min": -15.012430191040039,
      "activations/layer2_attention_weight_max": 28.45893669128418,
      "activations/layer2_attention_weight_min": -27.696029663085938,
      "activations/layer3_attention_weight_max": 61.60512161254883,
      "activations/layer3_attention_weight_min": -65.58848571777344,
      "activations/layer4_attention_weight_max": 88.45828247070312,
      "activations/layer4_attention_weight_min": -85.52816772460938,
      "activations/layer5_attention_weight_max": 120.19590759277344,
      "activations/layer5_attention_weight_min": -101.2176742553711,
      "activations/layer6_attention_weight_max": 57.66989517211914,
      "activations/layer6_attention_weight_min": -49.525978088378906,
      "activations/layer7_attention_weight_max": 58.20025634765625,
      "activations/layer7_attention_weight_min": -53.68791580200195,
      "activations/layer8_attention_weight_max": 43.04865646362305,
      "activations/layer8_attention_weight_min": -36.67287826538086,
      "activations/layer9_attention_weight_max": 64.71025848388672,
      "activations/layer9_attention_weight_min": -54.295654296875,
      "epoch": 2.29,
      "learning_rate": 0.00013662954545454545,
      "loss": 2.9496,
      "step": 39350
    },
    {
      "activations/layer0_attention_weight_max": 14.786924362182617,
      "activations/layer0_attention_weight_min": -14.159067153930664,
      "activations/layer10_attention_weight_max": 52.24845886230469,
      "activations/layer10_attention_weight_min": -43.91091537475586,
      "activations/layer11_attention_weight_max": 40.983436584472656,
      "activations/layer11_attention_weight_min": -33.11975860595703,
      "activations/layer12_attention_weight_max": 28.222034454345703,
      "activations/layer12_attention_weight_min": -22.993383407592773,
      "activations/layer13_attention_weight_max": 72.57699584960938,
      "activations/layer13_attention_weight_min": -47.078636169433594,
      "activations/layer14_attention_weight_max": 70.5896987915039,
      "activations/layer14_attention_weight_min": -46.0368766784668,
      "activations/layer15_attention_weight_max": 55.64544677734375,
      "activations/layer15_attention_weight_min": -38.59641647338867,
      "activations/layer16_attention_weight_max": 37.9387321472168,
      "activations/layer16_attention_weight_min": -31.074295043945312,
      "activations/layer17_attention_weight_max": 55.14961624145508,
      "activations/layer17_attention_weight_min": -35.65367889404297,
      "activations/layer18_attention_weight_max": 63.2783088684082,
      "activations/layer18_attention_weight_min": -31.335012435913086,
      "activations/layer19_attention_weight_max": 19.105009078979492,
      "activations/layer19_attention_weight_min": -15.292695999145508,
      "activations/layer1_attention_weight_max": 18.174226760864258,
      "activations/layer1_attention_weight_min": -13.987957000732422,
      "activations/layer20_attention_weight_max": 22.721656799316406,
      "activations/layer20_attention_weight_min": -18.135936737060547,
      "activations/layer21_attention_weight_max": 44.2518196105957,
      "activations/layer21_attention_weight_min": -27.296310424804688,
      "activations/layer22_attention_weight_max": 36.40499496459961,
      "activations/layer22_attention_weight_min": -21.73371124267578,
      "activations/layer23_attention_weight_max": 23.277088165283203,
      "activations/layer23_attention_weight_min": -16.172931671142578,
      "activations/layer2_attention_weight_max": 27.356109619140625,
      "activations/layer2_attention_weight_min": -26.17823028564453,
      "activations/layer3_attention_weight_max": 62.56959533691406,
      "activations/layer3_attention_weight_min": -67.94149780273438,
      "activations/layer4_attention_weight_max": 92.1386489868164,
      "activations/layer4_attention_weight_min": -83.14886474609375,
      "activations/layer5_attention_weight_max": 117.11515808105469,
      "activations/layer5_attention_weight_min": -100.89424896240234,
      "activations/layer6_attention_weight_max": 56.91286087036133,
      "activations/layer6_attention_weight_min": -48.74317932128906,
      "activations/layer7_attention_weight_max": 54.319610595703125,
      "activations/layer7_attention_weight_min": -53.147369384765625,
      "activations/layer8_attention_weight_max": 40.12657928466797,
      "activations/layer8_attention_weight_min": -35.9975471496582,
      "activations/layer9_attention_weight_max": 59.57855224609375,
      "activations/layer9_attention_weight_min": -54.16372299194336,
      "epoch": 2.29,
      "learning_rate": 0.00013661060606060606,
      "loss": 2.9446,
      "step": 39400
    },
    {
      "activations/layer0_attention_weight_max": 14.534390449523926,
      "activations/layer0_attention_weight_min": -13.80582332611084,
      "activations/layer10_attention_weight_max": 55.82197952270508,
      "activations/layer10_attention_weight_min": -45.06004333496094,
      "activations/layer11_attention_weight_max": 40.98028564453125,
      "activations/layer11_attention_weight_min": -36.50755310058594,
      "activations/layer12_attention_weight_max": 28.22832489013672,
      "activations/layer12_attention_weight_min": -23.555580139160156,
      "activations/layer13_attention_weight_max": 68.3760986328125,
      "activations/layer13_attention_weight_min": -52.72132110595703,
      "activations/layer14_attention_weight_max": 50.71026611328125,
      "activations/layer14_attention_weight_min": -40.00053024291992,
      "activations/layer15_attention_weight_max": 55.22260665893555,
      "activations/layer15_attention_weight_min": -41.37103271484375,
      "activations/layer16_attention_weight_max": 40.17831802368164,
      "activations/layer16_attention_weight_min": -30.527217864990234,
      "activations/layer17_attention_weight_max": 61.6302490234375,
      "activations/layer17_attention_weight_min": -38.88080978393555,
      "activations/layer18_attention_weight_max": 50.30636215209961,
      "activations/layer18_attention_weight_min": -33.77706527709961,
      "activations/layer19_attention_weight_max": 20.370098114013672,
      "activations/layer19_attention_weight_min": -18.344350814819336,
      "activations/layer1_attention_weight_max": 17.562959671020508,
      "activations/layer1_attention_weight_min": -12.731446266174316,
      "activations/layer20_attention_weight_max": 18.235050201416016,
      "activations/layer20_attention_weight_min": -16.60630226135254,
      "activations/layer21_attention_weight_max": 37.23397445678711,
      "activations/layer21_attention_weight_min": -25.705703735351562,
      "activations/layer22_attention_weight_max": 33.021812438964844,
      "activations/layer22_attention_weight_min": -25.078041076660156,
      "activations/layer23_attention_weight_max": 22.396915435791016,
      "activations/layer23_attention_weight_min": -17.252201080322266,
      "activations/layer2_attention_weight_max": 27.242359161376953,
      "activations/layer2_attention_weight_min": -26.926025390625,
      "activations/layer3_attention_weight_max": 62.6141471862793,
      "activations/layer3_attention_weight_min": -65.48463439941406,
      "activations/layer4_attention_weight_max": 90.20671844482422,
      "activations/layer4_attention_weight_min": -84.07666015625,
      "activations/layer5_attention_weight_max": 125.68916320800781,
      "activations/layer5_attention_weight_min": -108.63059997558594,
      "activations/layer6_attention_weight_max": 60.2169189453125,
      "activations/layer6_attention_weight_min": -54.20142364501953,
      "activations/layer7_attention_weight_max": 57.920597076416016,
      "activations/layer7_attention_weight_min": -53.4452018737793,
      "activations/layer8_attention_weight_max": 42.93356704711914,
      "activations/layer8_attention_weight_min": -37.41487503051758,
      "activations/layer9_attention_weight_max": 58.91640090942383,
      "activations/layer9_attention_weight_min": -51.8101806640625,
      "epoch": 2.29,
      "learning_rate": 0.00013659166666666665,
      "loss": 2.9449,
      "step": 39450
    },
    {
      "activations/layer0_attention_weight_max": 14.388150215148926,
      "activations/layer0_attention_weight_min": -13.405326843261719,
      "activations/layer10_attention_weight_max": 61.51774215698242,
      "activations/layer10_attention_weight_min": -45.26814651489258,
      "activations/layer11_attention_weight_max": 44.83811950683594,
      "activations/layer11_attention_weight_min": -33.91306686401367,
      "activations/layer12_attention_weight_max": 28.817655563354492,
      "activations/layer12_attention_weight_min": -28.287023544311523,
      "activations/layer13_attention_weight_max": 60.900550842285156,
      "activations/layer13_attention_weight_min": -42.190765380859375,
      "activations/layer14_attention_weight_max": 54.63239288330078,
      "activations/layer14_attention_weight_min": -37.66107940673828,
      "activations/layer15_attention_weight_max": 49.794219970703125,
      "activations/layer15_attention_weight_min": -33.770606994628906,
      "activations/layer16_attention_weight_max": 37.589725494384766,
      "activations/layer16_attention_weight_min": -29.84083366394043,
      "activations/layer17_attention_weight_max": 61.18780517578125,
      "activations/layer17_attention_weight_min": -37.54010772705078,
      "activations/layer18_attention_weight_max": 55.97434616088867,
      "activations/layer18_attention_weight_min": -33.580997467041016,
      "activations/layer19_attention_weight_max": 19.86909294128418,
      "activations/layer19_attention_weight_min": -18.587202072143555,
      "activations/layer1_attention_weight_max": 17.15308380126953,
      "activations/layer1_attention_weight_min": -12.592029571533203,
      "activations/layer20_attention_weight_max": 18.976085662841797,
      "activations/layer20_attention_weight_min": -19.487590789794922,
      "activations/layer21_attention_weight_max": 42.8641242980957,
      "activations/layer21_attention_weight_min": -26.319753646850586,
      "activations/layer22_attention_weight_max": 32.72108459472656,
      "activations/layer22_attention_weight_min": -22.536537170410156,
      "activations/layer23_attention_weight_max": 24.963417053222656,
      "activations/layer23_attention_weight_min": -15.683446884155273,
      "activations/layer2_attention_weight_max": 29.314342498779297,
      "activations/layer2_attention_weight_min": -27.226097106933594,
      "activations/layer3_attention_weight_max": 67.18458557128906,
      "activations/layer3_attention_weight_min": -70.68804168701172,
      "activations/layer4_attention_weight_max": 99.7158432006836,
      "activations/layer4_attention_weight_min": -88.83546447753906,
      "activations/layer5_attention_weight_max": 137.7438201904297,
      "activations/layer5_attention_weight_min": -106.35131072998047,
      "activations/layer6_attention_weight_max": 63.6823616027832,
      "activations/layer6_attention_weight_min": -52.651268005371094,
      "activations/layer7_attention_weight_max": 69.2953109741211,
      "activations/layer7_attention_weight_min": -55.20815658569336,
      "activations/layer8_attention_weight_max": 53.33247756958008,
      "activations/layer8_attention_weight_min": -37.11818313598633,
      "activations/layer9_attention_weight_max": 74.29426574707031,
      "activations/layer9_attention_weight_min": -58.352882385253906,
      "epoch": 2.3,
      "learning_rate": 0.00013657272727272726,
      "loss": 2.9346,
      "step": 39500
    },
    {
      "activations/layer0_attention_weight_max": 15.369707107543945,
      "activations/layer0_attention_weight_min": -14.150765419006348,
      "activations/layer10_attention_weight_max": 51.556819915771484,
      "activations/layer10_attention_weight_min": -43.28928756713867,
      "activations/layer11_attention_weight_max": 36.15932083129883,
      "activations/layer11_attention_weight_min": -32.02861022949219,
      "activations/layer12_attention_weight_max": 24.979887008666992,
      "activations/layer12_attention_weight_min": -30.715944290161133,
      "activations/layer13_attention_weight_max": 49.327457427978516,
      "activations/layer13_attention_weight_min": -39.80107498168945,
      "activations/layer14_attention_weight_max": 38.046478271484375,
      "activations/layer14_attention_weight_min": -32.11037826538086,
      "activations/layer15_attention_weight_max": 45.59233856201172,
      "activations/layer15_attention_weight_min": -36.2733154296875,
      "activations/layer16_attention_weight_max": 37.31587219238281,
      "activations/layer16_attention_weight_min": -28.118837356567383,
      "activations/layer17_attention_weight_max": 51.81615447998047,
      "activations/layer17_attention_weight_min": -41.79692077636719,
      "activations/layer18_attention_weight_max": 45.91006851196289,
      "activations/layer18_attention_weight_min": -33.61037826538086,
      "activations/layer19_attention_weight_max": 21.58760643005371,
      "activations/layer19_attention_weight_min": -17.871328353881836,
      "activations/layer1_attention_weight_max": 16.809986114501953,
      "activations/layer1_attention_weight_min": -12.151660919189453,
      "activations/layer20_attention_weight_max": 20.644107818603516,
      "activations/layer20_attention_weight_min": -19.95431900024414,
      "activations/layer21_attention_weight_max": 34.77289962768555,
      "activations/layer21_attention_weight_min": -25.078386306762695,
      "activations/layer22_attention_weight_max": 32.670753479003906,
      "activations/layer22_attention_weight_min": -25.650300979614258,
      "activations/layer23_attention_weight_max": 21.758743286132812,
      "activations/layer23_attention_weight_min": -14.503592491149902,
      "activations/layer2_attention_weight_max": 26.23828887939453,
      "activations/layer2_attention_weight_min": -25.399757385253906,
      "activations/layer3_attention_weight_max": 67.15198516845703,
      "activations/layer3_attention_weight_min": -66.71236419677734,
      "activations/layer4_attention_weight_max": 104.3863754272461,
      "activations/layer4_attention_weight_min": -93.32103729248047,
      "activations/layer5_attention_weight_max": 132.81597900390625,
      "activations/layer5_attention_weight_min": -106.42938232421875,
      "activations/layer6_attention_weight_max": 59.34169006347656,
      "activations/layer6_attention_weight_min": -51.8706169128418,
      "activations/layer7_attention_weight_max": 58.05837631225586,
      "activations/layer7_attention_weight_min": -50.834293365478516,
      "activations/layer8_attention_weight_max": 42.06370162963867,
      "activations/layer8_attention_weight_min": -36.447052001953125,
      "activations/layer9_attention_weight_max": 61.63407516479492,
      "activations/layer9_attention_weight_min": -55.386756896972656,
      "epoch": 2.3,
      "learning_rate": 0.00013655378787878788,
      "loss": 2.9344,
      "step": 39550
    },
    {
      "activations/layer0_attention_weight_max": 14.247258186340332,
      "activations/layer0_attention_weight_min": -13.726469993591309,
      "activations/layer10_attention_weight_max": 51.664180755615234,
      "activations/layer10_attention_weight_min": -44.27535629272461,
      "activations/layer11_attention_weight_max": 37.68610382080078,
      "activations/layer11_attention_weight_min": -33.22936248779297,
      "activations/layer12_attention_weight_max": 29.514148712158203,
      "activations/layer12_attention_weight_min": -22.914377212524414,
      "activations/layer13_attention_weight_max": 48.701881408691406,
      "activations/layer13_attention_weight_min": -36.508514404296875,
      "activations/layer14_attention_weight_max": 40.68915557861328,
      "activations/layer14_attention_weight_min": -32.33721160888672,
      "activations/layer15_attention_weight_max": 45.083526611328125,
      "activations/layer15_attention_weight_min": -37.57822799682617,
      "activations/layer16_attention_weight_max": 35.19144058227539,
      "activations/layer16_attention_weight_min": -27.863563537597656,
      "activations/layer17_attention_weight_max": 58.62913131713867,
      "activations/layer17_attention_weight_min": -36.638031005859375,
      "activations/layer18_attention_weight_max": 51.08483123779297,
      "activations/layer18_attention_weight_min": -31.538578033447266,
      "activations/layer19_attention_weight_max": 19.17791175842285,
      "activations/layer19_attention_weight_min": -16.46833610534668,
      "activations/layer1_attention_weight_max": 16.25821876525879,
      "activations/layer1_attention_weight_min": -14.271025657653809,
      "activations/layer20_attention_weight_max": 20.981388092041016,
      "activations/layer20_attention_weight_min": -21.27567481994629,
      "activations/layer21_attention_weight_max": 39.48271560668945,
      "activations/layer21_attention_weight_min": -23.93048095703125,
      "activations/layer22_attention_weight_max": 36.0982666015625,
      "activations/layer22_attention_weight_min": -21.154544830322266,
      "activations/layer23_attention_weight_max": 23.48187255859375,
      "activations/layer23_attention_weight_min": -14.657144546508789,
      "activations/layer2_attention_weight_max": 27.622303009033203,
      "activations/layer2_attention_weight_min": -26.899211883544922,
      "activations/layer3_attention_weight_max": 66.89912414550781,
      "activations/layer3_attention_weight_min": -68.76673889160156,
      "activations/layer4_attention_weight_max": 92.83296966552734,
      "activations/layer4_attention_weight_min": -88.28604888916016,
      "activations/layer5_attention_weight_max": 116.86641693115234,
      "activations/layer5_attention_weight_min": -105.98384094238281,
      "activations/layer6_attention_weight_max": 56.287353515625,
      "activations/layer6_attention_weight_min": -55.17109298706055,
      "activations/layer7_attention_weight_max": 55.80573654174805,
      "activations/layer7_attention_weight_min": -52.95125961303711,
      "activations/layer8_attention_weight_max": 43.04616928100586,
      "activations/layer8_attention_weight_min": -38.79390335083008,
      "activations/layer9_attention_weight_max": 63.4408073425293,
      "activations/layer9_attention_weight_min": -51.945648193359375,
      "epoch": 2.3,
      "learning_rate": 0.0001365348484848485,
      "loss": 2.9454,
      "step": 39600
    },
    {
      "activations/layer0_attention_weight_max": 14.743037223815918,
      "activations/layer0_attention_weight_min": -13.577098846435547,
      "activations/layer10_attention_weight_max": 52.67222213745117,
      "activations/layer10_attention_weight_min": -45.466400146484375,
      "activations/layer11_attention_weight_max": 42.25922393798828,
      "activations/layer11_attention_weight_min": -37.586647033691406,
      "activations/layer12_attention_weight_max": 38.47667694091797,
      "activations/layer12_attention_weight_min": -26.768251419067383,
      "activations/layer13_attention_weight_max": 53.073307037353516,
      "activations/layer13_attention_weight_min": -42.68397903442383,
      "activations/layer14_attention_weight_max": 52.17236328125,
      "activations/layer14_attention_weight_min": -39.48196792602539,
      "activations/layer15_attention_weight_max": 44.92341613769531,
      "activations/layer15_attention_weight_min": -34.92780685424805,
      "activations/layer16_attention_weight_max": 42.88900375366211,
      "activations/layer16_attention_weight_min": -26.192955017089844,
      "activations/layer17_attention_weight_max": 61.52184295654297,
      "activations/layer17_attention_weight_min": -35.607425689697266,
      "activations/layer18_attention_weight_max": 53.307228088378906,
      "activations/layer18_attention_weight_min": -32.27192687988281,
      "activations/layer19_attention_weight_max": 21.75417709350586,
      "activations/layer19_attention_weight_min": -16.552974700927734,
      "activations/layer1_attention_weight_max": 17.722848892211914,
      "activations/layer1_attention_weight_min": -13.521129608154297,
      "activations/layer20_attention_weight_max": 20.371719360351562,
      "activations/layer20_attention_weight_min": -16.170684814453125,
      "activations/layer21_attention_weight_max": 39.31364822387695,
      "activations/layer21_attention_weight_min": -25.392881393432617,
      "activations/layer22_attention_weight_max": 35.713157653808594,
      "activations/layer22_attention_weight_min": -21.19672393798828,
      "activations/layer23_attention_weight_max": 24.6627254486084,
      "activations/layer23_attention_weight_min": -16.810060501098633,
      "activations/layer2_attention_weight_max": 25.95030975341797,
      "activations/layer2_attention_weight_min": -26.174163818359375,
      "activations/layer3_attention_weight_max": 61.01264190673828,
      "activations/layer3_attention_weight_min": -65.93464660644531,
      "activations/layer4_attention_weight_max": 93.37316131591797,
      "activations/layer4_attention_weight_min": -84.4664306640625,
      "activations/layer5_attention_weight_max": 124.09779357910156,
      "activations/layer5_attention_weight_min": -104.19427490234375,
      "activations/layer6_attention_weight_max": 61.72416305541992,
      "activations/layer6_attention_weight_min": -50.49692916870117,
      "activations/layer7_attention_weight_max": 58.576168060302734,
      "activations/layer7_attention_weight_min": -52.33018112182617,
      "activations/layer8_attention_weight_max": 45.609336853027344,
      "activations/layer8_attention_weight_min": -41.89003372192383,
      "activations/layer9_attention_weight_max": 61.2679557800293,
      "activations/layer9_attention_weight_min": -58.82813262939453,
      "epoch": 2.3,
      "learning_rate": 0.00013651590909090908,
      "loss": 2.939,
      "step": 39650
    },
    {
      "activations/layer0_attention_weight_max": 13.921298027038574,
      "activations/layer0_attention_weight_min": -14.152870178222656,
      "activations/layer10_attention_weight_max": 50.873374938964844,
      "activations/layer10_attention_weight_min": -45.12431716918945,
      "activations/layer11_attention_weight_max": 38.71889877319336,
      "activations/layer11_attention_weight_min": -32.70440673828125,
      "activations/layer12_attention_weight_max": 27.014387130737305,
      "activations/layer12_attention_weight_min": -23.791091918945312,
      "activations/layer13_attention_weight_max": 54.49422073364258,
      "activations/layer13_attention_weight_min": -37.64659881591797,
      "activations/layer14_attention_weight_max": 46.3019905090332,
      "activations/layer14_attention_weight_min": -32.499046325683594,
      "activations/layer15_attention_weight_max": 51.22490692138672,
      "activations/layer15_attention_weight_min": -39.04901885986328,
      "activations/layer16_attention_weight_max": 42.363128662109375,
      "activations/layer16_attention_weight_min": -28.69080924987793,
      "activations/layer17_attention_weight_max": 56.110469818115234,
      "activations/layer17_attention_weight_min": -36.228485107421875,
      "activations/layer18_attention_weight_max": 58.63982391357422,
      "activations/layer18_attention_weight_min": -35.4642219543457,
      "activations/layer19_attention_weight_max": 21.86691665649414,
      "activations/layer19_attention_weight_min": -16.727638244628906,
      "activations/layer1_attention_weight_max": 16.87653923034668,
      "activations/layer1_attention_weight_min": -12.566128730773926,
      "activations/layer20_attention_weight_max": 21.44913101196289,
      "activations/layer20_attention_weight_min": -18.162622451782227,
      "activations/layer21_attention_weight_max": 41.667781829833984,
      "activations/layer21_attention_weight_min": -27.607009887695312,
      "activations/layer22_attention_weight_max": 35.929683685302734,
      "activations/layer22_attention_weight_min": -24.290996551513672,
      "activations/layer23_attention_weight_max": 25.203285217285156,
      "activations/layer23_attention_weight_min": -17.843883514404297,
      "activations/layer2_attention_weight_max": 25.548017501831055,
      "activations/layer2_attention_weight_min": -25.818693161010742,
      "activations/layer3_attention_weight_max": 61.73468780517578,
      "activations/layer3_attention_weight_min": -67.89579010009766,
      "activations/layer4_attention_weight_max": 95.90065002441406,
      "activations/layer4_attention_weight_min": -88.76414489746094,
      "activations/layer5_attention_weight_max": 113.34103393554688,
      "activations/layer5_attention_weight_min": -99.240966796875,
      "activations/layer6_attention_weight_max": 58.81000518798828,
      "activations/layer6_attention_weight_min": -49.864017486572266,
      "activations/layer7_attention_weight_max": 56.84347152709961,
      "activations/layer7_attention_weight_min": -50.661277770996094,
      "activations/layer8_attention_weight_max": 39.85558319091797,
      "activations/layer8_attention_weight_min": -37.73659896850586,
      "activations/layer9_attention_weight_max": 55.79691696166992,
      "activations/layer9_attention_weight_min": -47.89559555053711,
      "epoch": 2.31,
      "learning_rate": 0.0001364969696969697,
      "loss": 2.9556,
      "step": 39700
    },
    {
      "activations/layer0_attention_weight_max": 14.1781587600708,
      "activations/layer0_attention_weight_min": -14.807719230651855,
      "activations/layer10_attention_weight_max": 53.330284118652344,
      "activations/layer10_attention_weight_min": -42.86649703979492,
      "activations/layer11_attention_weight_max": 42.94001388549805,
      "activations/layer11_attention_weight_min": -35.40563201904297,
      "activations/layer12_attention_weight_max": 27.656312942504883,
      "activations/layer12_attention_weight_min": -24.725242614746094,
      "activations/layer13_attention_weight_max": 63.000946044921875,
      "activations/layer13_attention_weight_min": -46.10154342651367,
      "activations/layer14_attention_weight_max": 51.229957580566406,
      "activations/layer14_attention_weight_min": -36.27943420410156,
      "activations/layer15_attention_weight_max": 53.616004943847656,
      "activations/layer15_attention_weight_min": -35.927825927734375,
      "activations/layer16_attention_weight_max": 41.41761779785156,
      "activations/layer16_attention_weight_min": -28.298513412475586,
      "activations/layer17_attention_weight_max": 71.54718017578125,
      "activations/layer17_attention_weight_min": -36.857391357421875,
      "activations/layer18_attention_weight_max": 56.75673294067383,
      "activations/layer18_attention_weight_min": -32.590248107910156,
      "activations/layer19_attention_weight_max": 24.49005699157715,
      "activations/layer19_attention_weight_min": -18.586408615112305,
      "activations/layer1_attention_weight_max": 16.82834243774414,
      "activations/layer1_attention_weight_min": -13.362447738647461,
      "activations/layer20_attention_weight_max": 21.99888801574707,
      "activations/layer20_attention_weight_min": -17.46739959716797,
      "activations/layer21_attention_weight_max": 42.3776969909668,
      "activations/layer21_attention_weight_min": -29.37693214416504,
      "activations/layer22_attention_weight_max": 36.053279876708984,
      "activations/layer22_attention_weight_min": -22.2865047454834,
      "activations/layer23_attention_weight_max": 22.73070526123047,
      "activations/layer23_attention_weight_min": -14.86860466003418,
      "activations/layer2_attention_weight_max": 28.305007934570312,
      "activations/layer2_attention_weight_min": -27.27884292602539,
      "activations/layer3_attention_weight_max": 64.41060638427734,
      "activations/layer3_attention_weight_min": -64.39576721191406,
      "activations/layer4_attention_weight_max": 90.98734283447266,
      "activations/layer4_attention_weight_min": -82.19662475585938,
      "activations/layer5_attention_weight_max": 117.77874755859375,
      "activations/layer5_attention_weight_min": -97.7886962890625,
      "activations/layer6_attention_weight_max": 59.066654205322266,
      "activations/layer6_attention_weight_min": -52.348228454589844,
      "activations/layer7_attention_weight_max": 56.18476104736328,
      "activations/layer7_attention_weight_min": -48.62300491333008,
      "activations/layer8_attention_weight_max": 43.63540267944336,
      "activations/layer8_attention_weight_min": -37.73549270629883,
      "activations/layer9_attention_weight_max": 58.817237854003906,
      "activations/layer9_attention_weight_min": -51.2059440612793,
      "epoch": 2.31,
      "learning_rate": 0.00013647803030303028,
      "loss": 2.9598,
      "step": 39750
    },
    {
      "activations/layer0_attention_weight_max": 13.832195281982422,
      "activations/layer0_attention_weight_min": -13.7898588180542,
      "activations/layer10_attention_weight_max": 52.2490119934082,
      "activations/layer10_attention_weight_min": -45.17155456542969,
      "activations/layer11_attention_weight_max": 45.68104553222656,
      "activations/layer11_attention_weight_min": -34.634403228759766,
      "activations/layer12_attention_weight_max": 27.808542251586914,
      "activations/layer12_attention_weight_min": -29.306528091430664,
      "activations/layer13_attention_weight_max": 62.20295715332031,
      "activations/layer13_attention_weight_min": -48.67115783691406,
      "activations/layer14_attention_weight_max": 43.69890594482422,
      "activations/layer14_attention_weight_min": -34.77688980102539,
      "activations/layer15_attention_weight_max": 53.696834564208984,
      "activations/layer15_attention_weight_min": -40.84867477416992,
      "activations/layer16_attention_weight_max": 37.875328063964844,
      "activations/layer16_attention_weight_min": -27.803569793701172,
      "activations/layer17_attention_weight_max": 54.5903434753418,
      "activations/layer17_attention_weight_min": -36.991580963134766,
      "activations/layer18_attention_weight_max": 50.846229553222656,
      "activations/layer18_attention_weight_min": -30.61077308654785,
      "activations/layer19_attention_weight_max": 22.591432571411133,
      "activations/layer19_attention_weight_min": -16.288982391357422,
      "activations/layer1_attention_weight_max": 15.573713302612305,
      "activations/layer1_attention_weight_min": -12.48660945892334,
      "activations/layer20_attention_weight_max": 19.78514862060547,
      "activations/layer20_attention_weight_min": -16.88678550720215,
      "activations/layer21_attention_weight_max": 40.34226989746094,
      "activations/layer21_attention_weight_min": -28.57234764099121,
      "activations/layer22_attention_weight_max": 33.374290466308594,
      "activations/layer22_attention_weight_min": -20.923198699951172,
      "activations/layer23_attention_weight_max": 22.261089324951172,
      "activations/layer23_attention_weight_min": -15.532562255859375,
      "activations/layer2_attention_weight_max": 27.731239318847656,
      "activations/layer2_attention_weight_min": -25.200408935546875,
      "activations/layer3_attention_weight_max": 61.65602493286133,
      "activations/layer3_attention_weight_min": -63.66398239135742,
      "activations/layer4_attention_weight_max": 91.814208984375,
      "activations/layer4_attention_weight_min": -83.57172393798828,
      "activations/layer5_attention_weight_max": 116.28260803222656,
      "activations/layer5_attention_weight_min": -100.17822265625,
      "activations/layer6_attention_weight_max": 57.09292984008789,
      "activations/layer6_attention_weight_min": -52.15786361694336,
      "activations/layer7_attention_weight_max": 53.913421630859375,
      "activations/layer7_attention_weight_min": -51.168819427490234,
      "activations/layer8_attention_weight_max": 38.81196212768555,
      "activations/layer8_attention_weight_min": -38.341758728027344,
      "activations/layer9_attention_weight_max": 55.033416748046875,
      "activations/layer9_attention_weight_min": -58.25394821166992,
      "epoch": 2.31,
      "learning_rate": 0.0001364590909090909,
      "loss": 2.9444,
      "step": 39800
    },
    {
      "activations/layer0_attention_weight_max": 13.412630081176758,
      "activations/layer0_attention_weight_min": -13.8857421875,
      "activations/layer10_attention_weight_max": 55.216522216796875,
      "activations/layer10_attention_weight_min": -42.6385383605957,
      "activations/layer11_attention_weight_max": 46.748470306396484,
      "activations/layer11_attention_weight_min": -33.870731353759766,
      "activations/layer12_attention_weight_max": 37.286617279052734,
      "activations/layer12_attention_weight_min": -23.950191497802734,
      "activations/layer13_attention_weight_max": 57.4450798034668,
      "activations/layer13_attention_weight_min": -40.36188507080078,
      "activations/layer14_attention_weight_max": 54.06424331665039,
      "activations/layer14_attention_weight_min": -36.6595573425293,
      "activations/layer15_attention_weight_max": 51.892086029052734,
      "activations/layer15_attention_weight_min": -36.731178283691406,
      "activations/layer16_attention_weight_max": 49.581321716308594,
      "activations/layer16_attention_weight_min": -31.36393165588379,
      "activations/layer17_attention_weight_max": 63.01251220703125,
      "activations/layer17_attention_weight_min": -37.33283996582031,
      "activations/layer18_attention_weight_max": 60.233489990234375,
      "activations/layer18_attention_weight_min": -33.74238204956055,
      "activations/layer19_attention_weight_max": 24.83777618408203,
      "activations/layer19_attention_weight_min": -15.787155151367188,
      "activations/layer1_attention_weight_max": 17.313488006591797,
      "activations/layer1_attention_weight_min": -15.730928421020508,
      "activations/layer20_attention_weight_max": 22.149517059326172,
      "activations/layer20_attention_weight_min": -19.441177368164062,
      "activations/layer21_attention_weight_max": 43.96038818359375,
      "activations/layer21_attention_weight_min": -27.18399429321289,
      "activations/layer22_attention_weight_max": 39.188255310058594,
      "activations/layer22_attention_weight_min": -23.909761428833008,
      "activations/layer23_attention_weight_max": 26.944644927978516,
      "activations/layer23_attention_weight_min": -14.817233085632324,
      "activations/layer2_attention_weight_max": 28.512279510498047,
      "activations/layer2_attention_weight_min": -27.546672821044922,
      "activations/layer3_attention_weight_max": 64.2253189086914,
      "activations/layer3_attention_weight_min": -68.39563751220703,
      "activations/layer4_attention_weight_max": 95.19712829589844,
      "activations/layer4_attention_weight_min": -88.06322479248047,
      "activations/layer5_attention_weight_max": 124.12325286865234,
      "activations/layer5_attention_weight_min": -101.96548461914062,
      "activations/layer6_attention_weight_max": 61.23585891723633,
      "activations/layer6_attention_weight_min": -50.17243957519531,
      "activations/layer7_attention_weight_max": 58.796260833740234,
      "activations/layer7_attention_weight_min": -50.355987548828125,
      "activations/layer8_attention_weight_max": 45.140052795410156,
      "activations/layer8_attention_weight_min": -37.96364974975586,
      "activations/layer9_attention_weight_max": 66.30167388916016,
      "activations/layer9_attention_weight_min": -55.96371078491211,
      "epoch": 2.32,
      "learning_rate": 0.00013644015151515151,
      "loss": 2.9473,
      "step": 39850
    },
    {
      "activations/layer0_attention_weight_max": 13.507146835327148,
      "activations/layer0_attention_weight_min": -13.719486236572266,
      "activations/layer10_attention_weight_max": 54.12424087524414,
      "activations/layer10_attention_weight_min": -42.476409912109375,
      "activations/layer11_attention_weight_max": 39.826377868652344,
      "activations/layer11_attention_weight_min": -31.483592987060547,
      "activations/layer12_attention_weight_max": 25.422481536865234,
      "activations/layer12_attention_weight_min": -22.57297134399414,
      "activations/layer13_attention_weight_max": 71.53556823730469,
      "activations/layer13_attention_weight_min": -52.90333938598633,
      "activations/layer14_attention_weight_max": 57.31937026977539,
      "activations/layer14_attention_weight_min": -37.68593215942383,
      "activations/layer15_attention_weight_max": 64.42080688476562,
      "activations/layer15_attention_weight_min": -45.12197494506836,
      "activations/layer16_attention_weight_max": 38.0450553894043,
      "activations/layer16_attention_weight_min": -28.65562629699707,
      "activations/layer17_attention_weight_max": 64.70064544677734,
      "activations/layer17_attention_weight_min": -39.85408020019531,
      "activations/layer18_attention_weight_max": 56.075992584228516,
      "activations/layer18_attention_weight_min": -33.382484436035156,
      "activations/layer19_attention_weight_max": 19.237878799438477,
      "activations/layer19_attention_weight_min": -18.110612869262695,
      "activations/layer1_attention_weight_max": 17.028034210205078,
      "activations/layer1_attention_weight_min": -12.614114761352539,
      "activations/layer20_attention_weight_max": 20.75556755065918,
      "activations/layer20_attention_weight_min": -19.022258758544922,
      "activations/layer21_attention_weight_max": 42.02289581298828,
      "activations/layer21_attention_weight_min": -27.140066146850586,
      "activations/layer22_attention_weight_max": 33.0518913269043,
      "activations/layer22_attention_weight_min": -22.17180061340332,
      "activations/layer23_attention_weight_max": 21.46323013305664,
      "activations/layer23_attention_weight_min": -14.988368034362793,
      "activations/layer2_attention_weight_max": 28.092056274414062,
      "activations/layer2_attention_weight_min": -25.717679977416992,
      "activations/layer3_attention_weight_max": 61.11973190307617,
      "activations/layer3_attention_weight_min": -66.76396942138672,
      "activations/layer4_attention_weight_max": 88.61328125,
      "activations/layer4_attention_weight_min": -86.35884094238281,
      "activations/layer5_attention_weight_max": 117.83525848388672,
      "activations/layer5_attention_weight_min": -106.53900146484375,
      "activations/layer6_attention_weight_max": 56.867156982421875,
      "activations/layer6_attention_weight_min": -51.32797622680664,
      "activations/layer7_attention_weight_max": 58.84672164916992,
      "activations/layer7_attention_weight_min": -51.809471130371094,
      "activations/layer8_attention_weight_max": 42.68501663208008,
      "activations/layer8_attention_weight_min": -38.87335205078125,
      "activations/layer9_attention_weight_max": 62.80159378051758,
      "activations/layer9_attention_weight_min": -52.70710372924805,
      "epoch": 2.32,
      "learning_rate": 0.0001364212121212121,
      "loss": 2.9439,
      "step": 39900
    },
    {
      "activations/layer0_attention_weight_max": 14.440263748168945,
      "activations/layer0_attention_weight_min": -14.106070518493652,
      "activations/layer10_attention_weight_max": 50.22316360473633,
      "activations/layer10_attention_weight_min": -43.14735794067383,
      "activations/layer11_attention_weight_max": 39.771705627441406,
      "activations/layer11_attention_weight_min": -32.33790969848633,
      "activations/layer12_attention_weight_max": 29.100788116455078,
      "activations/layer12_attention_weight_min": -26.49851417541504,
      "activations/layer13_attention_weight_max": 50.08869552612305,
      "activations/layer13_attention_weight_min": -41.35601806640625,
      "activations/layer14_attention_weight_max": 41.47998046875,
      "activations/layer14_attention_weight_min": -30.687915802001953,
      "activations/layer15_attention_weight_max": 53.70466232299805,
      "activations/layer15_attention_weight_min": -38.65536880493164,
      "activations/layer16_attention_weight_max": 42.75834274291992,
      "activations/layer16_attention_weight_min": -28.29553985595703,
      "activations/layer17_attention_weight_max": 61.53209686279297,
      "activations/layer17_attention_weight_min": -36.76374816894531,
      "activations/layer18_attention_weight_max": 61.88489532470703,
      "activations/layer18_attention_weight_min": -34.007442474365234,
      "activations/layer19_attention_weight_max": 25.280010223388672,
      "activations/layer19_attention_weight_min": -17.99464225769043,
      "activations/layer1_attention_weight_max": 16.079843521118164,
      "activations/layer1_attention_weight_min": -13.788186073303223,
      "activations/layer20_attention_weight_max": 21.659896850585938,
      "activations/layer20_attention_weight_min": -18.288957595825195,
      "activations/layer21_attention_weight_max": 42.57085037231445,
      "activations/layer21_attention_weight_min": -27.741575241088867,
      "activations/layer22_attention_weight_max": 37.71078872680664,
      "activations/layer22_attention_weight_min": -23.29389190673828,
      "activations/layer23_attention_weight_max": 27.066608428955078,
      "activations/layer23_attention_weight_min": -15.784673690795898,
      "activations/layer2_attention_weight_max": 26.384435653686523,
      "activations/layer2_attention_weight_min": -26.13143539428711,
      "activations/layer3_attention_weight_max": 63.59807205200195,
      "activations/layer3_attention_weight_min": -64.35958862304688,
      "activations/layer4_attention_weight_max": 95.9764404296875,
      "activations/layer4_attention_weight_min": -84.6956558227539,
      "activations/layer5_attention_weight_max": 123.40160369873047,
      "activations/layer5_attention_weight_min": -99.26124572753906,
      "activations/layer6_attention_weight_max": 63.77862548828125,
      "activations/layer6_attention_weight_min": -53.11118698120117,
      "activations/layer7_attention_weight_max": 57.493019104003906,
      "activations/layer7_attention_weight_min": -51.31376647949219,
      "activations/layer8_attention_weight_max": 41.38680648803711,
      "activations/layer8_attention_weight_min": -37.59672927856445,
      "activations/layer9_attention_weight_max": 58.86651611328125,
      "activations/layer9_attention_weight_min": -54.370697021484375,
      "epoch": 2.32,
      "learning_rate": 0.00013640227272727272,
      "loss": 2.9441,
      "step": 39950
    },
    {
      "activations/layer0_attention_weight_max": 14.120780944824219,
      "activations/layer0_attention_weight_min": -14.330272674560547,
      "activations/layer10_attention_weight_max": 54.99210739135742,
      "activations/layer10_attention_weight_min": -43.75019454956055,
      "activations/layer11_attention_weight_max": 49.86429977416992,
      "activations/layer11_attention_weight_min": -33.960060119628906,
      "activations/layer12_attention_weight_max": 27.9754581451416,
      "activations/layer12_attention_weight_min": -24.24399757385254,
      "activations/layer13_attention_weight_max": 75.0430908203125,
      "activations/layer13_attention_weight_min": -47.86701202392578,
      "activations/layer14_attention_weight_max": 69.09015655517578,
      "activations/layer14_attention_weight_min": -44.44768524169922,
      "activations/layer15_attention_weight_max": 55.63896942138672,
      "activations/layer15_attention_weight_min": -38.49637222290039,
      "activations/layer16_attention_weight_max": 42.203369140625,
      "activations/layer16_attention_weight_min": -28.22196388244629,
      "activations/layer17_attention_weight_max": 70.47074127197266,
      "activations/layer17_attention_weight_min": -33.89072799682617,
      "activations/layer18_attention_weight_max": 73.11763763427734,
      "activations/layer18_attention_weight_min": -32.80367660522461,
      "activations/layer19_attention_weight_max": 27.042770385742188,
      "activations/layer19_attention_weight_min": -17.76179313659668,
      "activations/layer1_attention_weight_max": 16.959068298339844,
      "activations/layer1_attention_weight_min": -14.198270797729492,
      "activations/layer20_attention_weight_max": 22.8330020904541,
      "activations/layer20_attention_weight_min": -18.185585021972656,
      "activations/layer21_attention_weight_max": 54.66529083251953,
      "activations/layer21_attention_weight_min": -27.442459106445312,
      "activations/layer22_attention_weight_max": 42.65003204345703,
      "activations/layer22_attention_weight_min": -23.29608917236328,
      "activations/layer23_attention_weight_max": 30.286651611328125,
      "activations/layer23_attention_weight_min": -16.37876319885254,
      "activations/layer2_attention_weight_max": 29.310691833496094,
      "activations/layer2_attention_weight_min": -26.504484176635742,
      "activations/layer3_attention_weight_max": 64.04366302490234,
      "activations/layer3_attention_weight_min": -68.76578521728516,
      "activations/layer4_attention_weight_max": 91.02095794677734,
      "activations/layer4_attention_weight_min": -84.8506088256836,
      "activations/layer5_attention_weight_max": 118.16142272949219,
      "activations/layer5_attention_weight_min": -104.37245178222656,
      "activations/layer6_attention_weight_max": 59.7796516418457,
      "activations/layer6_attention_weight_min": -51.90441131591797,
      "activations/layer7_attention_weight_max": 58.36601638793945,
      "activations/layer7_attention_weight_min": -52.871280670166016,
      "activations/layer8_attention_weight_max": 41.3931884765625,
      "activations/layer8_attention_weight_min": -37.8641242980957,
      "activations/layer9_attention_weight_max": 58.65302276611328,
      "activations/layer9_attention_weight_min": -49.46794509887695,
      "epoch": 2.32,
      "learning_rate": 0.00013638333333333333,
      "loss": 2.9385,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_loss": 2.89453125,
      "eval_runtime": 8.4886,
      "eval_samples_per_second": 505.856,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_openwebtext_loss": 2.89453125,
      "eval_openwebtext_ppl": 18.075026786359576,
      "eval_openwebtext_runtime": 8.4886,
      "eval_openwebtext_samples_per_second": 505.856,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_wikitext_loss": 3.146484375,
      "eval_wikitext_ppl": 23.25416777248505,
      "eval_wikitext_runtime": 1.9565,
      "eval_wikitext_samples_per_second": 233.069,
      "step": 40000
    },
    {
      "epoch": 2.32,
      "eval_lambada_loss": 3.115234375,
      "eval_lambada_ppl": 22.538712245181248,
      "eval_lambada_runtime": 9.5193,
      "eval_lambada_samples_per_second": 511.485,
      "step": 40000
    },
    {
      "activations/layer0_attention_weight_max": 14.916448593139648,
      "activations/layer0_attention_weight_min": -14.21473217010498,
      "activations/layer10_attention_weight_max": 51.92362594604492,
      "activations/layer10_attention_weight_min": -48.68853759765625,
      "activations/layer11_attention_weight_max": 42.03197479248047,
      "activations/layer11_attention_weight_min": -31.668163299560547,
      "activations/layer12_attention_weight_max": 26.59170913696289,
      "activations/layer12_attention_weight_min": -24.387222290039062,
      "activations/layer13_attention_weight_max": 57.59977722167969,
      "activations/layer13_attention_weight_min": -42.95980453491211,
      "activations/layer14_attention_weight_max": 62.67535400390625,
      "activations/layer14_attention_weight_min": -39.573726654052734,
      "activations/layer15_attention_weight_max": 59.93281936645508,
      "activations/layer15_attention_weight_min": -42.69089126586914,
      "activations/layer16_attention_weight_max": 54.49465560913086,
      "activations/layer16_attention_weight_min": -27.5678653717041,
      "activations/layer17_attention_weight_max": 69.02613830566406,
      "activations/layer17_attention_weight_min": -34.56609344482422,
      "activations/layer18_attention_weight_max": 67.47566223144531,
      "activations/layer18_attention_weight_min": -32.137298583984375,
      "activations/layer19_attention_weight_max": 29.044530868530273,
      "activations/layer19_attention_weight_min": -17.0850887298584,
      "activations/layer1_attention_weight_max": 18.046859741210938,
      "activations/layer1_attention_weight_min": -12.714177131652832,
      "activations/layer20_attention_weight_max": 26.773664474487305,
      "activations/layer20_attention_weight_min": -19.003990173339844,
      "activations/layer21_attention_weight_max": 43.30979919433594,
      "activations/layer21_attention_weight_min": -24.72936248779297,
      "activations/layer22_attention_weight_max": 42.529090881347656,
      "activations/layer22_attention_weight_min": -22.664413452148438,
      "activations/layer23_attention_weight_max": 32.218746185302734,
      "activations/layer23_attention_weight_min": -14.255414962768555,
      "activations/layer2_attention_weight_max": 26.77327537536621,
      "activations/layer2_attention_weight_min": -25.837909698486328,
      "activations/layer3_attention_weight_max": 63.311317443847656,
      "activations/layer3_attention_weight_min": -70.41758728027344,
      "activations/layer4_attention_weight_max": 90.71134185791016,
      "activations/layer4_attention_weight_min": -85.46013641357422,
      "activations/layer5_attention_weight_max": 115.73178100585938,
      "activations/layer5_attention_weight_min": -102.06814575195312,
      "activations/layer6_attention_weight_max": 59.11759567260742,
      "activations/layer6_attention_weight_min": -50.87368392944336,
      "activations/layer7_attention_weight_max": 54.37745666503906,
      "activations/layer7_attention_weight_min": -54.38825225830078,
      "activations/layer8_attention_weight_max": 42.21833038330078,
      "activations/layer8_attention_weight_min": -38.476707458496094,
      "activations/layer9_attention_weight_max": 58.26160430908203,
      "activations/layer9_attention_weight_min": -51.41107177734375,
      "epoch": 2.33,
      "learning_rate": 0.00013636439393939392,
      "loss": 2.9553,
      "step": 40050
    },
    {
      "activations/layer0_attention_weight_max": 16.708044052124023,
      "activations/layer0_attention_weight_min": -14.00257396697998,
      "activations/layer10_attention_weight_max": 52.14213180541992,
      "activations/layer10_attention_weight_min": -40.481658935546875,
      "activations/layer11_attention_weight_max": 38.52217102050781,
      "activations/layer11_attention_weight_min": -31.28164291381836,
      "activations/layer12_attention_weight_max": 29.133163452148438,
      "activations/layer12_attention_weight_min": -24.711057662963867,
      "activations/layer13_attention_weight_max": 54.408878326416016,
      "activations/layer13_attention_weight_min": -43.53738021850586,
      "activations/layer14_attention_weight_max": 47.70582580566406,
      "activations/layer14_attention_weight_min": -39.354278564453125,
      "activations/layer15_attention_weight_max": 49.35508728027344,
      "activations/layer15_attention_weight_min": -36.116180419921875,
      "activations/layer16_attention_weight_max": 37.23701477050781,
      "activations/layer16_attention_weight_min": -27.150991439819336,
      "activations/layer17_attention_weight_max": 52.40878677368164,
      "activations/layer17_attention_weight_min": -36.7238655090332,
      "activations/layer18_attention_weight_max": 52.588436126708984,
      "activations/layer18_attention_weight_min": -34.237823486328125,
      "activations/layer19_attention_weight_max": 23.85540771484375,
      "activations/layer19_attention_weight_min": -16.393651962280273,
      "activations/layer1_attention_weight_max": 17.16261863708496,
      "activations/layer1_attention_weight_min": -12.16953182220459,
      "activations/layer20_attention_weight_max": 18.462142944335938,
      "activations/layer20_attention_weight_min": -17.546449661254883,
      "activations/layer21_attention_weight_max": 42.84857940673828,
      "activations/layer21_attention_weight_min": -25.540725708007812,
      "activations/layer22_attention_weight_max": 33.584983825683594,
      "activations/layer22_attention_weight_min": -22.389652252197266,
      "activations/layer23_attention_weight_max": 25.728527069091797,
      "activations/layer23_attention_weight_min": -14.318133354187012,
      "activations/layer2_attention_weight_max": 27.504491806030273,
      "activations/layer2_attention_weight_min": -25.334701538085938,
      "activations/layer3_attention_weight_max": 64.73471069335938,
      "activations/layer3_attention_weight_min": -65.7697525024414,
      "activations/layer4_attention_weight_max": 98.95449829101562,
      "activations/layer4_attention_weight_min": -92.2290267944336,
      "activations/layer5_attention_weight_max": 122.88567352294922,
      "activations/layer5_attention_weight_min": -106.47817993164062,
      "activations/layer6_attention_weight_max": 63.71107482910156,
      "activations/layer6_attention_weight_min": -53.436031341552734,
      "activations/layer7_attention_weight_max": 59.16814041137695,
      "activations/layer7_attention_weight_min": -52.01565933227539,
      "activations/layer8_attention_weight_max": 44.72570037841797,
      "activations/layer8_attention_weight_min": -39.96367263793945,
      "activations/layer9_attention_weight_max": 68.70805358886719,
      "activations/layer9_attention_weight_min": -55.96295928955078,
      "epoch": 2.33,
      "learning_rate": 0.00013634545454545454,
      "loss": 2.9434,
      "step": 40100
    },
    {
      "activations/layer0_attention_weight_max": 13.582145690917969,
      "activations/layer0_attention_weight_min": -13.854447364807129,
      "activations/layer10_attention_weight_max": 53.606109619140625,
      "activations/layer10_attention_weight_min": -43.34019088745117,
      "activations/layer11_attention_weight_max": 39.227760314941406,
      "activations/layer11_attention_weight_min": -30.706396102905273,
      "activations/layer12_attention_weight_max": 27.0306453704834,
      "activations/layer12_attention_weight_min": -23.611047744750977,
      "activations/layer13_attention_weight_max": 60.996376037597656,
      "activations/layer13_attention_weight_min": -39.62047576904297,
      "activations/layer14_attention_weight_max": 54.29426193237305,
      "activations/layer14_attention_weight_min": -34.97700500488281,
      "activations/layer15_attention_weight_max": 62.66158676147461,
      "activations/layer15_attention_weight_min": -38.98542404174805,
      "activations/layer16_attention_weight_max": 44.14488983154297,
      "activations/layer16_attention_weight_min": -29.235525131225586,
      "activations/layer17_attention_weight_max": 59.49662399291992,
      "activations/layer17_attention_weight_min": -35.27308654785156,
      "activations/layer18_attention_weight_max": 64.1239242553711,
      "activations/layer18_attention_weight_min": -33.827152252197266,
      "activations/layer19_attention_weight_max": 23.396209716796875,
      "activations/layer19_attention_weight_min": -15.917627334594727,
      "activations/layer1_attention_weight_max": 16.3271427154541,
      "activations/layer1_attention_weight_min": -12.541467666625977,
      "activations/layer20_attention_weight_max": 23.401567459106445,
      "activations/layer20_attention_weight_min": -18.365135192871094,
      "activations/layer21_attention_weight_max": 42.759029388427734,
      "activations/layer21_attention_weight_min": -25.571996688842773,
      "activations/layer22_attention_weight_max": 36.52101516723633,
      "activations/layer22_attention_weight_min": -24.960798263549805,
      "activations/layer23_attention_weight_max": 24.99365997314453,
      "activations/layer23_attention_weight_min": -17.928909301757812,
      "activations/layer2_attention_weight_max": 26.720176696777344,
      "activations/layer2_attention_weight_min": -27.44825553894043,
      "activations/layer3_attention_weight_max": 62.30420684814453,
      "activations/layer3_attention_weight_min": -65.04383850097656,
      "activations/layer4_attention_weight_max": 92.06829071044922,
      "activations/layer4_attention_weight_min": -83.17288970947266,
      "activations/layer5_attention_weight_max": 115.81947326660156,
      "activations/layer5_attention_weight_min": -97.13967895507812,
      "activations/layer6_attention_weight_max": 58.50126647949219,
      "activations/layer6_attention_weight_min": -49.24380111694336,
      "activations/layer7_attention_weight_max": 52.8149299621582,
      "activations/layer7_attention_weight_min": -50.12004470825195,
      "activations/layer8_attention_weight_max": 40.459747314453125,
      "activations/layer8_attention_weight_min": -35.098297119140625,
      "activations/layer9_attention_weight_max": 57.122535705566406,
      "activations/layer9_attention_weight_min": -49.89789962768555,
      "epoch": 2.33,
      "learning_rate": 0.00013632651515151515,
      "loss": 2.9396,
      "step": 40150
    },
    {
      "activations/layer0_attention_weight_max": 15.433887481689453,
      "activations/layer0_attention_weight_min": -14.557167053222656,
      "activations/layer10_attention_weight_max": 55.21078872680664,
      "activations/layer10_attention_weight_min": -46.820396423339844,
      "activations/layer11_attention_weight_max": 39.9949951171875,
      "activations/layer11_attention_weight_min": -32.780235290527344,
      "activations/layer12_attention_weight_max": 31.76287269592285,
      "activations/layer12_attention_weight_min": -23.5447940826416,
      "activations/layer13_attention_weight_max": 56.87120819091797,
      "activations/layer13_attention_weight_min": -42.20453643798828,
      "activations/layer14_attention_weight_max": 58.44742202758789,
      "activations/layer14_attention_weight_min": -32.70589828491211,
      "activations/layer15_attention_weight_max": 48.05715560913086,
      "activations/layer15_attention_weight_min": -35.99858474731445,
      "activations/layer16_attention_weight_max": 38.8582649230957,
      "activations/layer16_attention_weight_min": -26.275136947631836,
      "activations/layer17_attention_weight_max": 63.72281265258789,
      "activations/layer17_attention_weight_min": -40.77132034301758,
      "activations/layer18_attention_weight_max": 58.10257339477539,
      "activations/layer18_attention_weight_min": -36.559120178222656,
      "activations/layer19_attention_weight_max": 20.524337768554688,
      "activations/layer19_attention_weight_min": -16.498422622680664,
      "activations/layer1_attention_weight_max": 16.720447540283203,
      "activations/layer1_attention_weight_min": -12.670111656188965,
      "activations/layer20_attention_weight_max": 18.6912784576416,
      "activations/layer20_attention_weight_min": -17.91107749938965,
      "activations/layer21_attention_weight_max": 43.6553840637207,
      "activations/layer21_attention_weight_min": -27.45091438293457,
      "activations/layer22_attention_weight_max": 32.69388961791992,
      "activations/layer22_attention_weight_min": -20.779993057250977,
      "activations/layer23_attention_weight_max": 20.45511245727539,
      "activations/layer23_attention_weight_min": -20.08875274658203,
      "activations/layer2_attention_weight_max": 27.846431732177734,
      "activations/layer2_attention_weight_min": -27.46864128112793,
      "activations/layer3_attention_weight_max": 63.70433044433594,
      "activations/layer3_attention_weight_min": -70.53306579589844,
      "activations/layer4_attention_weight_max": 91.97754669189453,
      "activations/layer4_attention_weight_min": -87.3016586303711,
      "activations/layer5_attention_weight_max": 119.77789306640625,
      "activations/layer5_attention_weight_min": -102.76295471191406,
      "activations/layer6_attention_weight_max": 58.29035186767578,
      "activations/layer6_attention_weight_min": -50.95623016357422,
      "activations/layer7_attention_weight_max": 57.778099060058594,
      "activations/layer7_attention_weight_min": -51.44853210449219,
      "activations/layer8_attention_weight_max": 45.384281158447266,
      "activations/layer8_attention_weight_min": -38.55464553833008,
      "activations/layer9_attention_weight_max": 58.065277099609375,
      "activations/layer9_attention_weight_min": -47.74813461303711,
      "epoch": 2.34,
      "learning_rate": 0.00013630757575757574,
      "loss": 2.9307,
      "step": 40200
    },
    {
      "activations/layer0_attention_weight_max": 13.910202026367188,
      "activations/layer0_attention_weight_min": -14.247734069824219,
      "activations/layer10_attention_weight_max": 57.2733154296875,
      "activations/layer10_attention_weight_min": -45.37309646606445,
      "activations/layer11_attention_weight_max": 42.74079513549805,
      "activations/layer11_attention_weight_min": -32.36847686767578,
      "activations/layer12_attention_weight_max": 25.617725372314453,
      "activations/layer12_attention_weight_min": -24.616857528686523,
      "activations/layer13_attention_weight_max": 65.74063873291016,
      "activations/layer13_attention_weight_min": -49.88019943237305,
      "activations/layer14_attention_weight_max": 57.705257415771484,
      "activations/layer14_attention_weight_min": -42.36432647705078,
      "activations/layer15_attention_weight_max": 48.35969543457031,
      "activations/layer15_attention_weight_min": -36.52573776245117,
      "activations/layer16_attention_weight_max": 47.006649017333984,
      "activations/layer16_attention_weight_min": -27.680517196655273,
      "activations/layer17_attention_weight_max": 60.17034912109375,
      "activations/layer17_attention_weight_min": -35.43820571899414,
      "activations/layer18_attention_weight_max": 63.225257873535156,
      "activations/layer18_attention_weight_min": -33.6909294128418,
      "activations/layer19_attention_weight_max": 23.359743118286133,
      "activations/layer19_attention_weight_min": -16.115095138549805,
      "activations/layer1_attention_weight_max": 17.764448165893555,
      "activations/layer1_attention_weight_min": -15.654590606689453,
      "activations/layer20_attention_weight_max": 21.503231048583984,
      "activations/layer20_attention_weight_min": -16.191516876220703,
      "activations/layer21_attention_weight_max": 42.76491165161133,
      "activations/layer21_attention_weight_min": -28.846481323242188,
      "activations/layer22_attention_weight_max": 39.00537872314453,
      "activations/layer22_attention_weight_min": -22.153484344482422,
      "activations/layer23_attention_weight_max": 23.996807098388672,
      "activations/layer23_attention_weight_min": -16.684118270874023,
      "activations/layer2_attention_weight_max": 27.712499618530273,
      "activations/layer2_attention_weight_min": -26.710655212402344,
      "activations/layer3_attention_weight_max": 67.88203430175781,
      "activations/layer3_attention_weight_min": -69.08822631835938,
      "activations/layer4_attention_weight_max": 94.36693572998047,
      "activations/layer4_attention_weight_min": -84.56747436523438,
      "activations/layer5_attention_weight_max": 117.1998519897461,
      "activations/layer5_attention_weight_min": -102.34514617919922,
      "activations/layer6_attention_weight_max": 57.957069396972656,
      "activations/layer6_attention_weight_min": -50.15363311767578,
      "activations/layer7_attention_weight_max": 56.583133697509766,
      "activations/layer7_attention_weight_min": -53.10235595703125,
      "activations/layer8_attention_weight_max": 42.58064651489258,
      "activations/layer8_attention_weight_min": -37.85395812988281,
      "activations/layer9_attention_weight_max": 56.18915557861328,
      "activations/layer9_attention_weight_min": -54.17727279663086,
      "epoch": 2.34,
      "learning_rate": 0.00013628863636363635,
      "loss": 2.9486,
      "step": 40250
    },
    {
      "activations/layer0_attention_weight_max": 13.30447006225586,
      "activations/layer0_attention_weight_min": -13.4472017288208,
      "activations/layer10_attention_weight_max": 58.99484634399414,
      "activations/layer10_attention_weight_min": -45.203460693359375,
      "activations/layer11_attention_weight_max": 49.35051727294922,
      "activations/layer11_attention_weight_min": -30.48737907409668,
      "activations/layer12_attention_weight_max": 33.76851272583008,
      "activations/layer12_attention_weight_min": -24.072059631347656,
      "activations/layer13_attention_weight_max": 59.250980377197266,
      "activations/layer13_attention_weight_min": -42.794803619384766,
      "activations/layer14_attention_weight_max": 52.48569869995117,
      "activations/layer14_attention_weight_min": -41.05797576904297,
      "activations/layer15_attention_weight_max": 54.73130416870117,
      "activations/layer15_attention_weight_min": -34.686004638671875,
      "activations/layer16_attention_weight_max": 42.14473342895508,
      "activations/layer16_attention_weight_min": -27.35380744934082,
      "activations/layer17_attention_weight_max": 64.01376342773438,
      "activations/layer17_attention_weight_min": -36.46863555908203,
      "activations/layer18_attention_weight_max": 63.3814697265625,
      "activations/layer18_attention_weight_min": -35.11373519897461,
      "activations/layer19_attention_weight_max": 21.24308204650879,
      "activations/layer19_attention_weight_min": -17.081607818603516,
      "activations/layer1_attention_weight_max": 16.861249923706055,
      "activations/layer1_attention_weight_min": -12.99164867401123,
      "activations/layer20_attention_weight_max": 19.698774337768555,
      "activations/layer20_attention_weight_min": -19.770164489746094,
      "activations/layer21_attention_weight_max": 59.61903762817383,
      "activations/layer21_attention_weight_min": -27.906742095947266,
      "activations/layer22_attention_weight_max": 37.82678985595703,
      "activations/layer22_attention_weight_min": -24.165477752685547,
      "activations/layer23_attention_weight_max": 26.699752807617188,
      "activations/layer23_attention_weight_min": -16.023571014404297,
      "activations/layer2_attention_weight_max": 27.565107345581055,
      "activations/layer2_attention_weight_min": -26.375869750976562,
      "activations/layer3_attention_weight_max": 65.22462463378906,
      "activations/layer3_attention_weight_min": -71.27589416503906,
      "activations/layer4_attention_weight_max": 92.48628997802734,
      "activations/layer4_attention_weight_min": -84.01670837402344,
      "activations/layer5_attention_weight_max": 121.09471130371094,
      "activations/layer5_attention_weight_min": -103.88331604003906,
      "activations/layer6_attention_weight_max": 57.14847183227539,
      "activations/layer6_attention_weight_min": -50.49759292602539,
      "activations/layer7_attention_weight_max": 57.912899017333984,
      "activations/layer7_attention_weight_min": -52.61205291748047,
      "activations/layer8_attention_weight_max": 46.1713752746582,
      "activations/layer8_attention_weight_min": -36.33583068847656,
      "activations/layer9_attention_weight_max": 58.20037841796875,
      "activations/layer9_attention_weight_min": -54.4415168762207,
      "epoch": 2.34,
      "learning_rate": 0.00013626969696969694,
      "loss": 2.9385,
      "step": 40300
    },
    {
      "activations/layer0_attention_weight_max": 13.117676734924316,
      "activations/layer0_attention_weight_min": -14.302289009094238,
      "activations/layer10_attention_weight_max": 61.02702331542969,
      "activations/layer10_attention_weight_min": -46.655418395996094,
      "activations/layer11_attention_weight_max": 44.08336639404297,
      "activations/layer11_attention_weight_min": -32.47885513305664,
      "activations/layer12_attention_weight_max": 35.68400192260742,
      "activations/layer12_attention_weight_min": -26.798851013183594,
      "activations/layer13_attention_weight_max": 67.2528305053711,
      "activations/layer13_attention_weight_min": -52.41122055053711,
      "activations/layer14_attention_weight_max": 52.98443603515625,
      "activations/layer14_attention_weight_min": -39.14341735839844,
      "activations/layer15_attention_weight_max": 55.465335845947266,
      "activations/layer15_attention_weight_min": -40.68141555786133,
      "activations/layer16_attention_weight_max": 39.755245208740234,
      "activations/layer16_attention_weight_min": -27.75092887878418,
      "activations/layer17_attention_weight_max": 62.29277801513672,
      "activations/layer17_attention_weight_min": -39.23032760620117,
      "activations/layer18_attention_weight_max": 56.68053436279297,
      "activations/layer18_attention_weight_min": -32.5179557800293,
      "activations/layer19_attention_weight_max": 23.064960479736328,
      "activations/layer19_attention_weight_min": -15.649184226989746,
      "activations/layer1_attention_weight_max": 16.608449935913086,
      "activations/layer1_attention_weight_min": -12.366286277770996,
      "activations/layer20_attention_weight_max": 20.77008819580078,
      "activations/layer20_attention_weight_min": -19.20842742919922,
      "activations/layer21_attention_weight_max": 45.94706344604492,
      "activations/layer21_attention_weight_min": -25.597381591796875,
      "activations/layer22_attention_weight_max": 36.8696174621582,
      "activations/layer22_attention_weight_min": -21.0181884765625,
      "activations/layer23_attention_weight_max": 26.128337860107422,
      "activations/layer23_attention_weight_min": -16.241111755371094,
      "activations/layer2_attention_weight_max": 27.26534652709961,
      "activations/layer2_attention_weight_min": -26.72938346862793,
      "activations/layer3_attention_weight_max": 65.8113021850586,
      "activations/layer3_attention_weight_min": -67.9234390258789,
      "activations/layer4_attention_weight_max": 89.70310974121094,
      "activations/layer4_attention_weight_min": -88.35608673095703,
      "activations/layer5_attention_weight_max": 119.11571502685547,
      "activations/layer5_attention_weight_min": -103.97962951660156,
      "activations/layer6_attention_weight_max": 63.71685028076172,
      "activations/layer6_attention_weight_min": -49.65507888793945,
      "activations/layer7_attention_weight_max": 54.768226623535156,
      "activations/layer7_attention_weight_min": -49.99920654296875,
      "activations/layer8_attention_weight_max": 44.05644607543945,
      "activations/layer8_attention_weight_min": -34.66659927368164,
      "activations/layer9_attention_weight_max": 54.51673126220703,
      "activations/layer9_attention_weight_min": -46.83588790893555,
      "epoch": 2.34,
      "learning_rate": 0.00013625075757575756,
      "loss": 2.9365,
      "step": 40350
    },
    {
      "activations/layer0_attention_weight_max": 14.489311218261719,
      "activations/layer0_attention_weight_min": -14.163071632385254,
      "activations/layer10_attention_weight_max": 63.85291290283203,
      "activations/layer10_attention_weight_min": -47.57769775390625,
      "activations/layer11_attention_weight_max": 46.194671630859375,
      "activations/layer11_attention_weight_min": -29.838239669799805,
      "activations/layer12_attention_weight_max": 41.15437698364258,
      "activations/layer12_attention_weight_min": -24.231550216674805,
      "activations/layer13_attention_weight_max": 64.05022430419922,
      "activations/layer13_attention_weight_min": -42.40061569213867,
      "activations/layer14_attention_weight_max": 59.107635498046875,
      "activations/layer14_attention_weight_min": -38.3123664855957,
      "activations/layer15_attention_weight_max": 64.55491638183594,
      "activations/layer15_attention_weight_min": -41.588016510009766,
      "activations/layer16_attention_weight_max": 44.23478317260742,
      "activations/layer16_attention_weight_min": -26.78460693359375,
      "activations/layer17_attention_weight_max": 57.85047149658203,
      "activations/layer17_attention_weight_min": -34.6319580078125,
      "activations/layer18_attention_weight_max": 62.388362884521484,
      "activations/layer18_attention_weight_min": -34.527366638183594,
      "activations/layer19_attention_weight_max": 21.7165584564209,
      "activations/layer19_attention_weight_min": -15.48130989074707,
      "activations/layer1_attention_weight_max": 16.606552124023438,
      "activations/layer1_attention_weight_min": -14.76885986328125,
      "activations/layer20_attention_weight_max": 18.500232696533203,
      "activations/layer20_attention_weight_min": -18.350088119506836,
      "activations/layer21_attention_weight_max": 40.053192138671875,
      "activations/layer21_attention_weight_min": -25.927305221557617,
      "activations/layer22_attention_weight_max": 34.91287612915039,
      "activations/layer22_attention_weight_min": -21.710643768310547,
      "activations/layer23_attention_weight_max": 21.20123291015625,
      "activations/layer23_attention_weight_min": -15.10377311706543,
      "activations/layer2_attention_weight_max": 28.012348175048828,
      "activations/layer2_attention_weight_min": -25.99941062927246,
      "activations/layer3_attention_weight_max": 63.834842681884766,
      "activations/layer3_attention_weight_min": -66.07463073730469,
      "activations/layer4_attention_weight_max": 91.2695083618164,
      "activations/layer4_attention_weight_min": -83.56507873535156,
      "activations/layer5_attention_weight_max": 123.35832214355469,
      "activations/layer5_attention_weight_min": -101.19723510742188,
      "activations/layer6_attention_weight_max": 61.336219787597656,
      "activations/layer6_attention_weight_min": -50.44486618041992,
      "activations/layer7_attention_weight_max": 59.91596221923828,
      "activations/layer7_attention_weight_min": -50.93771743774414,
      "activations/layer8_attention_weight_max": 47.6889762878418,
      "activations/layer8_attention_weight_min": -35.36612319946289,
      "activations/layer9_attention_weight_max": 60.61265182495117,
      "activations/layer9_attention_weight_min": -53.566253662109375,
      "epoch": 2.35,
      "learning_rate": 0.00013623181818181817,
      "loss": 2.9496,
      "step": 40400
    },
    {
      "activations/layer0_attention_weight_max": 15.042265892028809,
      "activations/layer0_attention_weight_min": -13.945368766784668,
      "activations/layer10_attention_weight_max": 64.17325592041016,
      "activations/layer10_attention_weight_min": -51.73012161254883,
      "activations/layer11_attention_weight_max": 39.94529342651367,
      "activations/layer11_attention_weight_min": -32.155059814453125,
      "activations/layer12_attention_weight_max": 25.481082916259766,
      "activations/layer12_attention_weight_min": -22.44710922241211,
      "activations/layer13_attention_weight_max": 62.8813362121582,
      "activations/layer13_attention_weight_min": -47.40813064575195,
      "activations/layer14_attention_weight_max": 42.943450927734375,
      "activations/layer14_attention_weight_min": -34.258506774902344,
      "activations/layer15_attention_weight_max": 50.127357482910156,
      "activations/layer15_attention_weight_min": -35.747554779052734,
      "activations/layer16_attention_weight_max": 36.13760757446289,
      "activations/layer16_attention_weight_min": -27.462369918823242,
      "activations/layer17_attention_weight_max": 57.1833381652832,
      "activations/layer17_attention_weight_min": -37.18016052246094,
      "activations/layer18_attention_weight_max": 49.928958892822266,
      "activations/layer18_attention_weight_min": -32.49552536010742,
      "activations/layer19_attention_weight_max": 19.17531394958496,
      "activations/layer19_attention_weight_min": -17.37531089782715,
      "activations/layer1_attention_weight_max": 16.61281967163086,
      "activations/layer1_attention_weight_min": -13.052142143249512,
      "activations/layer20_attention_weight_max": 21.935359954833984,
      "activations/layer20_attention_weight_min": -17.774824142456055,
      "activations/layer21_attention_weight_max": 40.53310775756836,
      "activations/layer21_attention_weight_min": -28.7572021484375,
      "activations/layer22_attention_weight_max": 30.844791412353516,
      "activations/layer22_attention_weight_min": -21.824871063232422,
      "activations/layer23_attention_weight_max": 22.133832931518555,
      "activations/layer23_attention_weight_min": -15.684741020202637,
      "activations/layer2_attention_weight_max": 29.27963638305664,
      "activations/layer2_attention_weight_min": -27.404197692871094,
      "activations/layer3_attention_weight_max": 67.24559020996094,
      "activations/layer3_attention_weight_min": -71.03279876708984,
      "activations/layer4_attention_weight_max": 93.3847885131836,
      "activations/layer4_attention_weight_min": -86.13744354248047,
      "activations/layer5_attention_weight_max": 110.0215072631836,
      "activations/layer5_attention_weight_min": -99.06138610839844,
      "activations/layer6_attention_weight_max": 56.593841552734375,
      "activations/layer6_attention_weight_min": -50.93872833251953,
      "activations/layer7_attention_weight_max": 56.43798065185547,
      "activations/layer7_attention_weight_min": -53.60318374633789,
      "activations/layer8_attention_weight_max": 41.429683685302734,
      "activations/layer8_attention_weight_min": -35.73721694946289,
      "activations/layer9_attention_weight_max": 57.7485466003418,
      "activations/layer9_attention_weight_min": -50.443050384521484,
      "epoch": 2.35,
      "learning_rate": 0.00013621287878787876,
      "loss": 2.9313,
      "step": 40450
    },
    {
      "activations/layer0_attention_weight_max": 14.46723461151123,
      "activations/layer0_attention_weight_min": -13.849474906921387,
      "activations/layer10_attention_weight_max": 47.84721755981445,
      "activations/layer10_attention_weight_min": -44.137939453125,
      "activations/layer11_attention_weight_max": 41.466102600097656,
      "activations/layer11_attention_weight_min": -30.601062774658203,
      "activations/layer12_attention_weight_max": 23.607105255126953,
      "activations/layer12_attention_weight_min": -27.543859481811523,
      "activations/layer13_attention_weight_max": 47.090091705322266,
      "activations/layer13_attention_weight_min": -37.515804290771484,
      "activations/layer14_attention_weight_max": 42.63260269165039,
      "activations/layer14_attention_weight_min": -34.43586349487305,
      "activations/layer15_attention_weight_max": 45.83633041381836,
      "activations/layer15_attention_weight_min": -36.71274948120117,
      "activations/layer16_attention_weight_max": 34.66002655029297,
      "activations/layer16_attention_weight_min": -27.543577194213867,
      "activations/layer17_attention_weight_max": 60.83685302734375,
      "activations/layer17_attention_weight_min": -37.79350662231445,
      "activations/layer18_attention_weight_max": 46.570194244384766,
      "activations/layer18_attention_weight_min": -32.581260681152344,
      "activations/layer19_attention_weight_max": 19.003232955932617,
      "activations/layer19_attention_weight_min": -14.550015449523926,
      "activations/layer1_attention_weight_max": 16.223831176757812,
      "activations/layer1_attention_weight_min": -12.622281074523926,
      "activations/layer20_attention_weight_max": 17.387685775756836,
      "activations/layer20_attention_weight_min": -17.74108123779297,
      "activations/layer21_attention_weight_max": 35.40544509887695,
      "activations/layer21_attention_weight_min": -26.315841674804688,
      "activations/layer22_attention_weight_max": 31.130645751953125,
      "activations/layer22_attention_weight_min": -20.92867088317871,
      "activations/layer23_attention_weight_max": 21.727148056030273,
      "activations/layer23_attention_weight_min": -14.644851684570312,
      "activations/layer2_attention_weight_max": 26.992294311523438,
      "activations/layer2_attention_weight_min": -26.54205894470215,
      "activations/layer3_attention_weight_max": 63.52318572998047,
      "activations/layer3_attention_weight_min": -68.75373840332031,
      "activations/layer4_attention_weight_max": 90.41173553466797,
      "activations/layer4_attention_weight_min": -85.78406524658203,
      "activations/layer5_attention_weight_max": 116.12467956542969,
      "activations/layer5_attention_weight_min": -104.07783508300781,
      "activations/layer6_attention_weight_max": 58.062347412109375,
      "activations/layer6_attention_weight_min": -51.52949905395508,
      "activations/layer7_attention_weight_max": 53.0234489440918,
      "activations/layer7_attention_weight_min": -57.49737548828125,
      "activations/layer8_attention_weight_max": 39.79497528076172,
      "activations/layer8_attention_weight_min": -36.60239028930664,
      "activations/layer9_attention_weight_max": 52.596431732177734,
      "activations/layer9_attention_weight_min": -49.510005950927734,
      "epoch": 2.35,
      "learning_rate": 0.00013619393939393937,
      "loss": 2.9452,
      "step": 40500
    },
    {
      "activations/layer0_attention_weight_max": 15.232498168945312,
      "activations/layer0_attention_weight_min": -14.43774700164795,
      "activations/layer10_attention_weight_max": 52.1507682800293,
      "activations/layer10_attention_weight_min": -42.95389938354492,
      "activations/layer11_attention_weight_max": 48.46753692626953,
      "activations/layer11_attention_weight_min": -35.52870178222656,
      "activations/layer12_attention_weight_max": 26.5784969329834,
      "activations/layer12_attention_weight_min": -26.41728401184082,
      "activations/layer13_attention_weight_max": 58.35313415527344,
      "activations/layer13_attention_weight_min": -40.68177032470703,
      "activations/layer14_attention_weight_max": 54.45890426635742,
      "activations/layer14_attention_weight_min": -35.31515884399414,
      "activations/layer15_attention_weight_max": 49.8822135925293,
      "activations/layer15_attention_weight_min": -38.33169174194336,
      "activations/layer16_attention_weight_max": 45.97077178955078,
      "activations/layer16_attention_weight_min": -30.31437873840332,
      "activations/layer17_attention_weight_max": 70.38572692871094,
      "activations/layer17_attention_weight_min": -40.72239685058594,
      "activations/layer18_attention_weight_max": 58.46931076049805,
      "activations/layer18_attention_weight_min": -33.70606994628906,
      "activations/layer19_attention_weight_max": 20.040212631225586,
      "activations/layer19_attention_weight_min": -16.03458595275879,
      "activations/layer1_attention_weight_max": 17.050352096557617,
      "activations/layer1_attention_weight_min": -13.65780258178711,
      "activations/layer20_attention_weight_max": 21.36661148071289,
      "activations/layer20_attention_weight_min": -21.127239227294922,
      "activations/layer21_attention_weight_max": 37.875709533691406,
      "activations/layer21_attention_weight_min": -28.295085906982422,
      "activations/layer22_attention_weight_max": 35.01356506347656,
      "activations/layer22_attention_weight_min": -22.884538650512695,
      "activations/layer23_attention_weight_max": 25.721572875976562,
      "activations/layer23_attention_weight_min": -15.916131973266602,
      "activations/layer2_attention_weight_max": 27.088064193725586,
      "activations/layer2_attention_weight_min": -26.432172775268555,
      "activations/layer3_attention_weight_max": 67.3574447631836,
      "activations/layer3_attention_weight_min": -68.67442321777344,
      "activations/layer4_attention_weight_max": 95.43047332763672,
      "activations/layer4_attention_weight_min": -85.18045043945312,
      "activations/layer5_attention_weight_max": 130.02902221679688,
      "activations/layer5_attention_weight_min": -104.76083374023438,
      "activations/layer6_attention_weight_max": 62.331016540527344,
      "activations/layer6_attention_weight_min": -50.33016586303711,
      "activations/layer7_attention_weight_max": 58.71401596069336,
      "activations/layer7_attention_weight_min": -55.7690544128418,
      "activations/layer8_attention_weight_max": 46.42544174194336,
      "activations/layer8_attention_weight_min": -40.64821243286133,
      "activations/layer9_attention_weight_max": 63.587074279785156,
      "activations/layer9_attention_weight_min": -56.205135345458984,
      "epoch": 2.36,
      "learning_rate": 0.000136175,
      "loss": 2.9331,
      "step": 40550
    },
    {
      "activations/layer0_attention_weight_max": 14.15764045715332,
      "activations/layer0_attention_weight_min": -14.151642799377441,
      "activations/layer10_attention_weight_max": 56.816009521484375,
      "activations/layer10_attention_weight_min": -47.076961517333984,
      "activations/layer11_attention_weight_max": 40.41979217529297,
      "activations/layer11_attention_weight_min": -30.750965118408203,
      "activations/layer12_attention_weight_max": 26.834918975830078,
      "activations/layer12_attention_weight_min": -24.163928985595703,
      "activations/layer13_attention_weight_max": 57.19859313964844,
      "activations/layer13_attention_weight_min": -39.045982360839844,
      "activations/layer14_attention_weight_max": 45.0267448425293,
      "activations/layer14_attention_weight_min": -34.34080123901367,
      "activations/layer15_attention_weight_max": 54.96968460083008,
      "activations/layer15_attention_weight_min": -36.85474395751953,
      "activations/layer16_attention_weight_max": 37.582515716552734,
      "activations/layer16_attention_weight_min": -27.94017219543457,
      "activations/layer17_attention_weight_max": 65.32272338867188,
      "activations/layer17_attention_weight_min": -38.860572814941406,
      "activations/layer18_attention_weight_max": 57.25647735595703,
      "activations/layer18_attention_weight_min": -33.57882308959961,
      "activations/layer19_attention_weight_max": 20.931900024414062,
      "activations/layer19_attention_weight_min": -17.51220703125,
      "activations/layer1_attention_weight_max": 17.715768814086914,
      "activations/layer1_attention_weight_min": -12.034348487854004,
      "activations/layer20_attention_weight_max": 20.23517608642578,
      "activations/layer20_attention_weight_min": -18.318506240844727,
      "activations/layer21_attention_weight_max": 40.75162124633789,
      "activations/layer21_attention_weight_min": -26.750553131103516,
      "activations/layer22_attention_weight_max": 37.97953414916992,
      "activations/layer22_attention_weight_min": -22.571273803710938,
      "activations/layer23_attention_weight_max": 24.140586853027344,
      "activations/layer23_attention_weight_min": -15.79637336730957,
      "activations/layer2_attention_weight_max": 28.099943161010742,
      "activations/layer2_attention_weight_min": -27.215543746948242,
      "activations/layer3_attention_weight_max": 65.84967041015625,
      "activations/layer3_attention_weight_min": -69.59376525878906,
      "activations/layer4_attention_weight_max": 92.9700698852539,
      "activations/layer4_attention_weight_min": -84.32317352294922,
      "activations/layer5_attention_weight_max": 120.24047088623047,
      "activations/layer5_attention_weight_min": -105.84606170654297,
      "activations/layer6_attention_weight_max": 59.146080017089844,
      "activations/layer6_attention_weight_min": -51.719234466552734,
      "activations/layer7_attention_weight_max": 55.894195556640625,
      "activations/layer7_attention_weight_min": -50.31810760498047,
      "activations/layer8_attention_weight_max": 41.29222869873047,
      "activations/layer8_attention_weight_min": -37.9505729675293,
      "activations/layer9_attention_weight_max": 54.29265213012695,
      "activations/layer9_attention_weight_min": -53.544677734375,
      "epoch": 2.36,
      "learning_rate": 0.00013615606060606058,
      "loss": 2.9409,
      "step": 40600
    },
    {
      "activations/layer0_attention_weight_max": 14.497156143188477,
      "activations/layer0_attention_weight_min": -14.380607604980469,
      "activations/layer10_attention_weight_max": 53.30508804321289,
      "activations/layer10_attention_weight_min": -40.64375686645508,
      "activations/layer11_attention_weight_max": 41.36760711669922,
      "activations/layer11_attention_weight_min": -29.634868621826172,
      "activations/layer12_attention_weight_max": 32.48329162597656,
      "activations/layer12_attention_weight_min": -25.595783233642578,
      "activations/layer13_attention_weight_max": 62.253662109375,
      "activations/layer13_attention_weight_min": -43.41577911376953,
      "activations/layer14_attention_weight_max": 64.94377136230469,
      "activations/layer14_attention_weight_min": -39.78830337524414,
      "activations/layer15_attention_weight_max": 56.49728775024414,
      "activations/layer15_attention_weight_min": -37.180110931396484,
      "activations/layer16_attention_weight_max": 48.714210510253906,
      "activations/layer16_attention_weight_min": -28.976795196533203,
      "activations/layer17_attention_weight_max": 86.94512939453125,
      "activations/layer17_attention_weight_min": -40.557498931884766,
      "activations/layer18_attention_weight_max": 82.5927963256836,
      "activations/layer18_attention_weight_min": -33.45669174194336,
      "activations/layer19_attention_weight_max": 29.74252700805664,
      "activations/layer19_attention_weight_min": -17.18146324157715,
      "activations/layer1_attention_weight_max": 16.85934829711914,
      "activations/layer1_attention_weight_min": -15.997535705566406,
      "activations/layer20_attention_weight_max": 25.287723541259766,
      "activations/layer20_attention_weight_min": -18.046384811401367,
      "activations/layer21_attention_weight_max": 57.08858108520508,
      "activations/layer21_attention_weight_min": -27.121206283569336,
      "activations/layer22_attention_weight_max": 43.331077575683594,
      "activations/layer22_attention_weight_min": -22.33401870727539,
      "activations/layer23_attention_weight_max": 28.05270767211914,
      "activations/layer23_attention_weight_min": -14.49583625793457,
      "activations/layer2_attention_weight_max": 28.158742904663086,
      "activations/layer2_attention_weight_min": -27.11351776123047,
      "activations/layer3_attention_weight_max": 65.3040771484375,
      "activations/layer3_attention_weight_min": -69.9458236694336,
      "activations/layer4_attention_weight_max": 93.07621765136719,
      "activations/layer4_attention_weight_min": -81.77754974365234,
      "activations/layer5_attention_weight_max": 124.40987396240234,
      "activations/layer5_attention_weight_min": -99.85377502441406,
      "activations/layer6_attention_weight_max": 60.84812545776367,
      "activations/layer6_attention_weight_min": -51.48693084716797,
      "activations/layer7_attention_weight_max": 55.789825439453125,
      "activations/layer7_attention_weight_min": -52.327476501464844,
      "activations/layer8_attention_weight_max": 46.450199127197266,
      "activations/layer8_attention_weight_min": -39.11192321777344,
      "activations/layer9_attention_weight_max": 60.70548629760742,
      "activations/layer9_attention_weight_min": -54.00421905517578,
      "epoch": 2.36,
      "learning_rate": 0.0001361371212121212,
      "loss": 2.9433,
      "step": 40650
    },
    {
      "activations/layer0_attention_weight_max": 14.865614891052246,
      "activations/layer0_attention_weight_min": -14.21828556060791,
      "activations/layer10_attention_weight_max": 57.21238327026367,
      "activations/layer10_attention_weight_min": -45.8451042175293,
      "activations/layer11_attention_weight_max": 41.26072692871094,
      "activations/layer11_attention_weight_min": -31.835899353027344,
      "activations/layer12_attention_weight_max": 24.93579864501953,
      "activations/layer12_attention_weight_min": -25.836271286010742,
      "activations/layer13_attention_weight_max": 56.7456169128418,
      "activations/layer13_attention_weight_min": -40.15394973754883,
      "activations/layer14_attention_weight_max": 51.98781967163086,
      "activations/layer14_attention_weight_min": -35.33353805541992,
      "activations/layer15_attention_weight_max": 50.34546661376953,
      "activations/layer15_attention_weight_min": -39.61518859863281,
      "activations/layer16_attention_weight_max": 41.066402435302734,
      "activations/layer16_attention_weight_min": -26.72078514099121,
      "activations/layer17_attention_weight_max": 60.98819351196289,
      "activations/layer17_attention_weight_min": -36.50310134887695,
      "activations/layer18_attention_weight_max": 52.09421157836914,
      "activations/layer18_attention_weight_min": -36.01406478881836,
      "activations/layer19_attention_weight_max": 22.031314849853516,
      "activations/layer19_attention_weight_min": -16.89662742614746,
      "activations/layer1_attention_weight_max": 16.461042404174805,
      "activations/layer1_attention_weight_min": -14.720393180847168,
      "activations/layer20_attention_weight_max": 20.737171173095703,
      "activations/layer20_attention_weight_min": -17.15243911743164,
      "activations/layer21_attention_weight_max": 37.18560791015625,
      "activations/layer21_attention_weight_min": -26.038284301757812,
      "activations/layer22_attention_weight_max": 33.77354049682617,
      "activations/layer22_attention_weight_min": -21.890342712402344,
      "activations/layer23_attention_weight_max": 24.519437789916992,
      "activations/layer23_attention_weight_min": -17.4212703704834,
      "activations/layer2_attention_weight_max": 26.590559005737305,
      "activations/layer2_attention_weight_min": -25.56789779663086,
      "activations/layer3_attention_weight_max": 64.35047149658203,
      "activations/layer3_attention_weight_min": -67.3854751586914,
      "activations/layer4_attention_weight_max": 93.13009643554688,
      "activations/layer4_attention_weight_min": -90.42776489257812,
      "activations/layer5_attention_weight_max": 124.83601379394531,
      "activations/layer5_attention_weight_min": -100.75071716308594,
      "activations/layer6_attention_weight_max": 62.184783935546875,
      "activations/layer6_attention_weight_min": -50.95399475097656,
      "activations/layer7_attention_weight_max": 54.76667404174805,
      "activations/layer7_attention_weight_min": -53.373504638671875,
      "activations/layer8_attention_weight_max": 40.3585319519043,
      "activations/layer8_attention_weight_min": -36.20469665527344,
      "activations/layer9_attention_weight_max": 52.33271026611328,
      "activations/layer9_attention_weight_min": -50.04226303100586,
      "epoch": 2.36,
      "learning_rate": 0.00013611856060606058,
      "loss": 2.951,
      "step": 40700
    },
    {
      "activations/layer0_attention_weight_max": 14.44835376739502,
      "activations/layer0_attention_weight_min": -14.252791404724121,
      "activations/layer10_attention_weight_max": 51.6107292175293,
      "activations/layer10_attention_weight_min": -43.7857780456543,
      "activations/layer11_attention_weight_max": 42.5274543762207,
      "activations/layer11_attention_weight_min": -30.525901794433594,
      "activations/layer12_attention_weight_max": 27.213916778564453,
      "activations/layer12_attention_weight_min": -25.418664932250977,
      "activations/layer13_attention_weight_max": 63.28398513793945,
      "activations/layer13_attention_weight_min": -46.17182922363281,
      "activations/layer14_attention_weight_max": 61.04015350341797,
      "activations/layer14_attention_weight_min": -44.97785186767578,
      "activations/layer15_attention_weight_max": 51.87260055541992,
      "activations/layer15_attention_weight_min": -34.656463623046875,
      "activations/layer16_attention_weight_max": 41.12439727783203,
      "activations/layer16_attention_weight_min": -28.55571937561035,
      "activations/layer17_attention_weight_max": 55.94949722290039,
      "activations/layer17_attention_weight_min": -38.61606216430664,
      "activations/layer18_attention_weight_max": 49.832096099853516,
      "activations/layer18_attention_weight_min": -32.04780960083008,
      "activations/layer19_attention_weight_max": 19.497304916381836,
      "activations/layer19_attention_weight_min": -17.625728607177734,
      "activations/layer1_attention_weight_max": 17.986080169677734,
      "activations/layer1_attention_weight_min": -13.79116439819336,
      "activations/layer20_attention_weight_max": 21.101518630981445,
      "activations/layer20_attention_weight_min": -16.358436584472656,
      "activations/layer21_attention_weight_max": 40.221893310546875,
      "activations/layer21_attention_weight_min": -26.8980770111084,
      "activations/layer22_attention_weight_max": 41.31836700439453,
      "activations/layer22_attention_weight_min": -21.13555145263672,
      "activations/layer23_attention_weight_max": 23.354660034179688,
      "activations/layer23_attention_weight_min": -15.9743070602417,
      "activations/layer2_attention_weight_max": 29.348661422729492,
      "activations/layer2_attention_weight_min": -26.919483184814453,
      "activations/layer3_attention_weight_max": 63.585365295410156,
      "activations/layer3_attention_weight_min": -68.82865142822266,
      "activations/layer4_attention_weight_max": 93.26775360107422,
      "activations/layer4_attention_weight_min": -84.4814682006836,
      "activations/layer5_attention_weight_max": 119.18895721435547,
      "activations/layer5_attention_weight_min": -103.21208190917969,
      "activations/layer6_attention_weight_max": 56.78554153442383,
      "activations/layer6_attention_weight_min": -51.02324676513672,
      "activations/layer7_attention_weight_max": 68.54969787597656,
      "activations/layer7_attention_weight_min": -54.7120361328125,
      "activations/layer8_attention_weight_max": 48.4786262512207,
      "activations/layer8_attention_weight_min": -38.65192794799805,
      "activations/layer9_attention_weight_max": 67.65473937988281,
      "activations/layer9_attention_weight_min": -60.684783935546875,
      "epoch": 2.37,
      "learning_rate": 0.0001360996212121212,
      "loss": 2.9357,
      "step": 40750
    },
    {
      "activations/layer0_attention_weight_max": 15.222527503967285,
      "activations/layer0_attention_weight_min": -14.400182723999023,
      "activations/layer10_attention_weight_max": 53.00054931640625,
      "activations/layer10_attention_weight_min": -44.306087493896484,
      "activations/layer11_attention_weight_max": 43.70325469970703,
      "activations/layer11_attention_weight_min": -32.1978759765625,
      "activations/layer12_attention_weight_max": 34.453758239746094,
      "activations/layer12_attention_weight_min": -27.98303985595703,
      "activations/layer13_attention_weight_max": 58.3226203918457,
      "activations/layer13_attention_weight_min": -42.50109100341797,
      "activations/layer14_attention_weight_max": 44.78457260131836,
      "activations/layer14_attention_weight_min": -33.10090255737305,
      "activations/layer15_attention_weight_max": 48.3418083190918,
      "activations/layer15_attention_weight_min": -36.05466842651367,
      "activations/layer16_attention_weight_max": 43.91275405883789,
      "activations/layer16_attention_weight_min": -28.269821166992188,
      "activations/layer17_attention_weight_max": 60.79894256591797,
      "activations/layer17_attention_weight_min": -36.64753341674805,
      "activations/layer18_attention_weight_max": 58.1795539855957,
      "activations/layer18_attention_weight_min": -33.62924575805664,
      "activations/layer19_attention_weight_max": 22.037946701049805,
      "activations/layer19_attention_weight_min": -16.393064498901367,
      "activations/layer1_attention_weight_max": 16.371702194213867,
      "activations/layer1_attention_weight_min": -12.983720779418945,
      "activations/layer20_attention_weight_max": 20.969676971435547,
      "activations/layer20_attention_weight_min": -17.541555404663086,
      "activations/layer21_attention_weight_max": 50.73987579345703,
      "activations/layer21_attention_weight_min": -29.3303279876709,
      "activations/layer22_attention_weight_max": 42.26212692260742,
      "activations/layer22_attention_weight_min": -21.945159912109375,
      "activations/layer23_attention_weight_max": 24.580677032470703,
      "activations/layer23_attention_weight_min": -14.157816886901855,
      "activations/layer2_attention_weight_max": 27.938922882080078,
      "activations/layer2_attention_weight_min": -27.928983688354492,
      "activations/layer3_attention_weight_max": 68.51630401611328,
      "activations/layer3_attention_weight_min": -68.35662841796875,
      "activations/layer4_attention_weight_max": 99.90087890625,
      "activations/layer4_attention_weight_min": -87.91781616210938,
      "activations/layer5_attention_weight_max": 124.46330261230469,
      "activations/layer5_attention_weight_min": -110.42137908935547,
      "activations/layer6_attention_weight_max": 60.57889175415039,
      "activations/layer6_attention_weight_min": -53.08479690551758,
      "activations/layer7_attention_weight_max": 56.700191497802734,
      "activations/layer7_attention_weight_min": -54.247314453125,
      "activations/layer8_attention_weight_max": 45.36641311645508,
      "activations/layer8_attention_weight_min": -37.402652740478516,
      "activations/layer9_attention_weight_max": 59.56385040283203,
      "activations/layer9_attention_weight_min": -53.296566009521484,
      "epoch": 2.37,
      "learning_rate": 0.0001360806818181818,
      "loss": 2.9344,
      "step": 40800
    },
    {
      "activations/layer0_attention_weight_max": 14.476483345031738,
      "activations/layer0_attention_weight_min": -14.202445983886719,
      "activations/layer10_attention_weight_max": 52.910003662109375,
      "activations/layer10_attention_weight_min": -43.33186340332031,
      "activations/layer11_attention_weight_max": 41.396392822265625,
      "activations/layer11_attention_weight_min": -32.58778381347656,
      "activations/layer12_attention_weight_max": 27.201879501342773,
      "activations/layer12_attention_weight_min": -25.30787467956543,
      "activations/layer13_attention_weight_max": 61.03788757324219,
      "activations/layer13_attention_weight_min": -41.50740051269531,
      "activations/layer14_attention_weight_max": 59.527408599853516,
      "activations/layer14_attention_weight_min": -39.53636169433594,
      "activations/layer15_attention_weight_max": 49.08553695678711,
      "activations/layer15_attention_weight_min": -35.762699127197266,
      "activations/layer16_attention_weight_max": 39.103599548339844,
      "activations/layer16_attention_weight_min": -29.105695724487305,
      "activations/layer17_attention_weight_max": 58.84291458129883,
      "activations/layer17_attention_weight_min": -36.86331558227539,
      "activations/layer18_attention_weight_max": 54.49625015258789,
      "activations/layer18_attention_weight_min": -34.051273345947266,
      "activations/layer19_attention_weight_max": 20.907846450805664,
      "activations/layer19_attention_weight_min": -16.063749313354492,
      "activations/layer1_attention_weight_max": 16.811004638671875,
      "activations/layer1_attention_weight_min": -13.205849647521973,
      "activations/layer20_attention_weight_max": 18.986473083496094,
      "activations/layer20_attention_weight_min": -18.18375587463379,
      "activations/layer21_attention_weight_max": 43.812530517578125,
      "activations/layer21_attention_weight_min": -26.918485641479492,
      "activations/layer22_attention_weight_max": 32.89124298095703,
      "activations/layer22_attention_weight_min": -22.517650604248047,
      "activations/layer23_attention_weight_max": 25.627126693725586,
      "activations/layer23_attention_weight_min": -15.423574447631836,
      "activations/layer2_attention_weight_max": 27.0262451171875,
      "activations/layer2_attention_weight_min": -26.179977416992188,
      "activations/layer3_attention_weight_max": 64.22482299804688,
      "activations/layer3_attention_weight_min": -67.17253112792969,
      "activations/layer4_attention_weight_max": 91.73189544677734,
      "activations/layer4_attention_weight_min": -81.3392105102539,
      "activations/layer5_attention_weight_max": 120.42479705810547,
      "activations/layer5_attention_weight_min": -98.35176086425781,
      "activations/layer6_attention_weight_max": 57.2359619140625,
      "activations/layer6_attention_weight_min": -49.30564498901367,
      "activations/layer7_attention_weight_max": 54.686485290527344,
      "activations/layer7_attention_weight_min": -53.97098159790039,
      "activations/layer8_attention_weight_max": 42.50373840332031,
      "activations/layer8_attention_weight_min": -37.26707458496094,
      "activations/layer9_attention_weight_max": 63.69636917114258,
      "activations/layer9_attention_weight_min": -54.9951286315918,
      "epoch": 2.37,
      "learning_rate": 0.0001360617424242424,
      "loss": 2.9399,
      "step": 40850
    },
    {
      "activations/layer0_attention_weight_max": 14.390535354614258,
      "activations/layer0_attention_weight_min": -14.082743644714355,
      "activations/layer10_attention_weight_max": 52.89451217651367,
      "activations/layer10_attention_weight_min": -45.29169845581055,
      "activations/layer11_attention_weight_max": 39.34861373901367,
      "activations/layer11_attention_weight_min": -31.83961296081543,
      "activations/layer12_attention_weight_max": 26.439529418945312,
      "activations/layer12_attention_weight_min": -25.234506607055664,
      "activations/layer13_attention_weight_max": 53.4346809387207,
      "activations/layer13_attention_weight_min": -40.872352600097656,
      "activations/layer14_attention_weight_max": 58.20048904418945,
      "activations/layer14_attention_weight_min": -37.386802673339844,
      "activations/layer15_attention_weight_max": 49.58622360229492,
      "activations/layer15_attention_weight_min": -33.57328796386719,
      "activations/layer16_attention_weight_max": 38.14348220825195,
      "activations/layer16_attention_weight_min": -30.36426544189453,
      "activations/layer17_attention_weight_max": 61.61288070678711,
      "activations/layer17_attention_weight_min": -35.37982940673828,
      "activations/layer18_attention_weight_max": 60.56074142456055,
      "activations/layer18_attention_weight_min": -34.136627197265625,
      "activations/layer19_attention_weight_max": 23.782899856567383,
      "activations/layer19_attention_weight_min": -19.957862854003906,
      "activations/layer1_attention_weight_max": 16.26109504699707,
      "activations/layer1_attention_weight_min": -14.997647285461426,
      "activations/layer20_attention_weight_max": 22.246171951293945,
      "activations/layer20_attention_weight_min": -17.274290084838867,
      "activations/layer21_attention_weight_max": 47.967735290527344,
      "activations/layer21_attention_weight_min": -27.630393981933594,
      "activations/layer22_attention_weight_max": 39.1490364074707,
      "activations/layer22_attention_weight_min": -23.39481544494629,
      "activations/layer23_attention_weight_max": 23.952863693237305,
      "activations/layer23_attention_weight_min": -16.280956268310547,
      "activations/layer2_attention_weight_max": 29.481359481811523,
      "activations/layer2_attention_weight_min": -27.50710105895996,
      "activations/layer3_attention_weight_max": 65.61561584472656,
      "activations/layer3_attention_weight_min": -67.35990142822266,
      "activations/layer4_attention_weight_max": 96.83221435546875,
      "activations/layer4_attention_weight_min": -85.87389373779297,
      "activations/layer5_attention_weight_max": 122.15576171875,
      "activations/layer5_attention_weight_min": -97.61270141601562,
      "activations/layer6_attention_weight_max": 60.64860153198242,
      "activations/layer6_attention_weight_min": -49.15702438354492,
      "activations/layer7_attention_weight_max": 55.11688995361328,
      "activations/layer7_attention_weight_min": -49.8287467956543,
      "activations/layer8_attention_weight_max": 41.68495178222656,
      "activations/layer8_attention_weight_min": -36.51882553100586,
      "activations/layer9_attention_weight_max": 58.79132080078125,
      "activations/layer9_attention_weight_min": -50.37096405029297,
      "epoch": 2.38,
      "learning_rate": 0.00013604280303030302,
      "loss": 2.9166,
      "step": 40900
    },
    {
      "activations/layer0_attention_weight_max": 14.371194839477539,
      "activations/layer0_attention_weight_min": -14.904858589172363,
      "activations/layer10_attention_weight_max": 56.532318115234375,
      "activations/layer10_attention_weight_min": -43.87456512451172,
      "activations/layer11_attention_weight_max": 43.374061584472656,
      "activations/layer11_attention_weight_min": -31.651451110839844,
      "activations/layer12_attention_weight_max": 29.43065071105957,
      "activations/layer12_attention_weight_min": -23.59973907470703,
      "activations/layer13_attention_weight_max": 79.31358337402344,
      "activations/layer13_attention_weight_min": -48.67954635620117,
      "activations/layer14_attention_weight_max": 73.6471176147461,
      "activations/layer14_attention_weight_min": -38.90581130981445,
      "activations/layer15_attention_weight_max": 56.13249588012695,
      "activations/layer15_attention_weight_min": -38.446083068847656,
      "activations/layer16_attention_weight_max": 45.35704803466797,
      "activations/layer16_attention_weight_min": -28.510812759399414,
      "activations/layer17_attention_weight_max": 62.7840576171875,
      "activations/layer17_attention_weight_min": -35.240325927734375,
      "activations/layer18_attention_weight_max": 58.20558547973633,
      "activations/layer18_attention_weight_min": -32.82694625854492,
      "activations/layer19_attention_weight_max": 21.54854393005371,
      "activations/layer19_attention_weight_min": -17.049652099609375,
      "activations/layer1_attention_weight_max": 17.703264236450195,
      "activations/layer1_attention_weight_min": -12.14781379699707,
      "activations/layer20_attention_weight_max": 24.065895080566406,
      "activations/layer20_attention_weight_min": -18.141704559326172,
      "activations/layer21_attention_weight_max": 55.4183349609375,
      "activations/layer21_attention_weight_min": -31.18729591369629,
      "activations/layer22_attention_weight_max": 41.182247161865234,
      "activations/layer22_attention_weight_min": -22.901203155517578,
      "activations/layer23_attention_weight_max": 27.86074447631836,
      "activations/layer23_attention_weight_min": -16.339736938476562,
      "activations/layer2_attention_weight_max": 27.15631866455078,
      "activations/layer2_attention_weight_min": -27.542949676513672,
      "activations/layer3_attention_weight_max": 63.10569763183594,
      "activations/layer3_attention_weight_min": -68.5116195678711,
      "activations/layer4_attention_weight_max": 93.61776733398438,
      "activations/layer4_attention_weight_min": -87.17560577392578,
      "activations/layer5_attention_weight_max": 125.23067474365234,
      "activations/layer5_attention_weight_min": -103.34129333496094,
      "activations/layer6_attention_weight_max": 65.51667022705078,
      "activations/layer6_attention_weight_min": -51.9744987487793,
      "activations/layer7_attention_weight_max": 58.80420684814453,
      "activations/layer7_attention_weight_min": -51.54102325439453,
      "activations/layer8_attention_weight_max": 41.77902603149414,
      "activations/layer8_attention_weight_min": -37.641048431396484,
      "activations/layer9_attention_weight_max": 61.464599609375,
      "activations/layer9_attention_weight_min": -51.527374267578125,
      "epoch": 2.38,
      "learning_rate": 0.00013602386363636363,
      "loss": 2.9425,
      "step": 40950
    },
    {
      "activations/layer0_attention_weight_max": 14.32691478729248,
      "activations/layer0_attention_weight_min": -14.212418556213379,
      "activations/layer10_attention_weight_max": 50.37118911743164,
      "activations/layer10_attention_weight_min": -41.2414665222168,
      "activations/layer11_attention_weight_max": 43.4266471862793,
      "activations/layer11_attention_weight_min": -32.28478240966797,
      "activations/layer12_attention_weight_max": 25.463159561157227,
      "activations/layer12_attention_weight_min": -22.757001876831055,
      "activations/layer13_attention_weight_max": 59.02324295043945,
      "activations/layer13_attention_weight_min": -39.76301193237305,
      "activations/layer14_attention_weight_max": 44.748355865478516,
      "activations/layer14_attention_weight_min": -31.49515724182129,
      "activations/layer15_attention_weight_max": 56.476104736328125,
      "activations/layer15_attention_weight_min": -34.466712951660156,
      "activations/layer16_attention_weight_max": 40.304561614990234,
      "activations/layer16_attention_weight_min": -30.586360931396484,
      "activations/layer17_attention_weight_max": 59.82337188720703,
      "activations/layer17_attention_weight_min": -37.24809646606445,
      "activations/layer18_attention_weight_max": 55.45204544067383,
      "activations/layer18_attention_weight_min": -33.07229995727539,
      "activations/layer19_attention_weight_max": 19.63701057434082,
      "activations/layer19_attention_weight_min": -16.441984176635742,
      "activations/layer1_attention_weight_max": 16.772855758666992,
      "activations/layer1_attention_weight_min": -13.497743606567383,
      "activations/layer20_attention_weight_max": 20.18680191040039,
      "activations/layer20_attention_weight_min": -19.257701873779297,
      "activations/layer21_attention_weight_max": 42.068145751953125,
      "activations/layer21_attention_weight_min": -24.759933471679688,
      "activations/layer22_attention_weight_max": 31.98242950439453,
      "activations/layer22_attention_weight_min": -20.58856201171875,
      "activations/layer23_attention_weight_max": 22.169353485107422,
      "activations/layer23_attention_weight_min": -15.326006889343262,
      "activations/layer2_attention_weight_max": 27.014816284179688,
      "activations/layer2_attention_weight_min": -26.384716033935547,
      "activations/layer3_attention_weight_max": 64.25041198730469,
      "activations/layer3_attention_weight_min": -70.22962188720703,
      "activations/layer4_attention_weight_max": 97.33235168457031,
      "activations/layer4_attention_weight_min": -89.6363296508789,
      "activations/layer5_attention_weight_max": 125.32698822021484,
      "activations/layer5_attention_weight_min": -105.69506072998047,
      "activations/layer6_attention_weight_max": 63.392032623291016,
      "activations/layer6_attention_weight_min": -53.59361267089844,
      "activations/layer7_attention_weight_max": 56.887447357177734,
      "activations/layer7_attention_weight_min": -50.796287536621094,
      "activations/layer8_attention_weight_max": 42.027549743652344,
      "activations/layer8_attention_weight_min": -37.55339813232422,
      "activations/layer9_attention_weight_max": 55.51435470581055,
      "activations/layer9_attention_weight_min": -51.71393966674805,
      "epoch": 2.38,
      "learning_rate": 0.00013600492424242425,
      "loss": 2.952,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_loss": 2.892578125,
      "eval_runtime": 8.4813,
      "eval_samples_per_second": 506.292,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_openwebtext_loss": 2.892578125,
      "eval_openwebtext_ppl": 18.039758452611142,
      "eval_openwebtext_runtime": 8.4813,
      "eval_openwebtext_samples_per_second": 506.292,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_wikitext_loss": 3.162109375,
      "eval_wikitext_ppl": 23.620367629992042,
      "eval_wikitext_runtime": 1.9165,
      "eval_wikitext_samples_per_second": 237.939,
      "step": 41000
    },
    {
      "epoch": 2.38,
      "eval_lambada_loss": 3.03515625,
      "eval_lambada_ppl": 20.8042283062039,
      "eval_lambada_runtime": 9.5257,
      "eval_lambada_samples_per_second": 511.142,
      "step": 41000
    },
    {
      "activations/layer0_attention_weight_max": 14.930609703063965,
      "activations/layer0_attention_weight_min": -14.864485740661621,
      "activations/layer10_attention_weight_max": 49.20673751831055,
      "activations/layer10_attention_weight_min": -43.445377349853516,
      "activations/layer11_attention_weight_max": 42.746009826660156,
      "activations/layer11_attention_weight_min": -33.63264465332031,
      "activations/layer12_attention_weight_max": 26.396516799926758,
      "activations/layer12_attention_weight_min": -24.26499366760254,
      "activations/layer13_attention_weight_max": 47.342933654785156,
      "activations/layer13_attention_weight_min": -38.93541717529297,
      "activations/layer14_attention_weight_max": 49.74891662597656,
      "activations/layer14_attention_weight_min": -32.762969970703125,
      "activations/layer15_attention_weight_max": 48.70439147949219,
      "activations/layer15_attention_weight_min": -36.269493103027344,
      "activations/layer16_attention_weight_max": 36.01374435424805,
      "activations/layer16_attention_weight_min": -26.684406280517578,
      "activations/layer17_attention_weight_max": 54.587493896484375,
      "activations/layer17_attention_weight_min": -39.035640716552734,
      "activations/layer18_attention_weight_max": 57.15469741821289,
      "activations/layer18_attention_weight_min": -33.4716682434082,
      "activations/layer19_attention_weight_max": 21.168195724487305,
      "activations/layer19_attention_weight_min": -18.5477352142334,
      "activations/layer1_attention_weight_max": 16.64735221862793,
      "activations/layer1_attention_weight_min": -13.675944328308105,
      "activations/layer20_attention_weight_max": 23.613536834716797,
      "activations/layer20_attention_weight_min": -18.269298553466797,
      "activations/layer21_attention_weight_max": 37.96989822387695,
      "activations/layer21_attention_weight_min": -26.087875366210938,
      "activations/layer22_attention_weight_max": 33.22522735595703,
      "activations/layer22_attention_weight_min": -22.717615127563477,
      "activations/layer23_attention_weight_max": 22.525470733642578,
      "activations/layer23_attention_weight_min": -15.282614707946777,
      "activations/layer2_attention_weight_max": 27.064800262451172,
      "activations/layer2_attention_weight_min": -25.637624740600586,
      "activations/layer3_attention_weight_max": 64.39130401611328,
      "activations/layer3_attention_weight_min": -66.02503204345703,
      "activations/layer4_attention_weight_max": 93.10031127929688,
      "activations/layer4_attention_weight_min": -86.12672424316406,
      "activations/layer5_attention_weight_max": 116.9247055053711,
      "activations/layer5_attention_weight_min": -106.22771453857422,
      "activations/layer6_attention_weight_max": 57.11894607543945,
      "activations/layer6_attention_weight_min": -52.31438446044922,
      "activations/layer7_attention_weight_max": 52.676605224609375,
      "activations/layer7_attention_weight_min": -53.20861053466797,
      "activations/layer8_attention_weight_max": 39.3964958190918,
      "activations/layer8_attention_weight_min": -36.883602142333984,
      "activations/layer9_attention_weight_max": 52.5024528503418,
      "activations/layer9_attention_weight_min": -53.01394271850586,
      "epoch": 2.39,
      "learning_rate": 0.00013598598484848483,
      "loss": 2.9575,
      "step": 41050
    },
    {
      "activations/layer0_attention_weight_max": 14.75777816772461,
      "activations/layer0_attention_weight_min": -14.762067794799805,
      "activations/layer10_attention_weight_max": 52.471092224121094,
      "activations/layer10_attention_weight_min": -43.22148513793945,
      "activations/layer11_attention_weight_max": 44.25640106201172,
      "activations/layer11_attention_weight_min": -30.305578231811523,
      "activations/layer12_attention_weight_max": 31.637466430664062,
      "activations/layer12_attention_weight_min": -25.449874877929688,
      "activations/layer13_attention_weight_max": 68.06362915039062,
      "activations/layer13_attention_weight_min": -46.52388000488281,
      "activations/layer14_attention_weight_max": 68.62501525878906,
      "activations/layer14_attention_weight_min": -46.50318908691406,
      "activations/layer15_attention_weight_max": 53.83357238769531,
      "activations/layer15_attention_weight_min": -38.395748138427734,
      "activations/layer16_attention_weight_max": 40.24924087524414,
      "activations/layer16_attention_weight_min": -29.74934196472168,
      "activations/layer17_attention_weight_max": 61.46845245361328,
      "activations/layer17_attention_weight_min": -35.35630798339844,
      "activations/layer18_attention_weight_max": 53.04658508300781,
      "activations/layer18_attention_weight_min": -32.22953414916992,
      "activations/layer19_attention_weight_max": 25.181968688964844,
      "activations/layer19_attention_weight_min": -17.646224975585938,
      "activations/layer1_attention_weight_max": 18.360742568969727,
      "activations/layer1_attention_weight_min": -13.827391624450684,
      "activations/layer20_attention_weight_max": 25.06657600402832,
      "activations/layer20_attention_weight_min": -15.696300506591797,
      "activations/layer21_attention_weight_max": 41.16590881347656,
      "activations/layer21_attention_weight_min": -25.04336929321289,
      "activations/layer22_attention_weight_max": 37.214439392089844,
      "activations/layer22_attention_weight_min": -21.693321228027344,
      "activations/layer23_attention_weight_max": 25.784894943237305,
      "activations/layer23_attention_weight_min": -12.826518058776855,
      "activations/layer2_attention_weight_max": 28.022605895996094,
      "activations/layer2_attention_weight_min": -25.935867309570312,
      "activations/layer3_attention_weight_max": 69.052978515625,
      "activations/layer3_attention_weight_min": -68.44607543945312,
      "activations/layer4_attention_weight_max": 104.2699203491211,
      "activations/layer4_attention_weight_min": -88.86869049072266,
      "activations/layer5_attention_weight_max": 130.861572265625,
      "activations/layer5_attention_weight_min": -103.73997497558594,
      "activations/layer6_attention_weight_max": 62.124603271484375,
      "activations/layer6_attention_weight_min": -51.3549690246582,
      "activations/layer7_attention_weight_max": 62.477928161621094,
      "activations/layer7_attention_weight_min": -56.35300827026367,
      "activations/layer8_attention_weight_max": 46.638885498046875,
      "activations/layer8_attention_weight_min": -40.39365768432617,
      "activations/layer9_attention_weight_max": 62.09517288208008,
      "activations/layer9_attention_weight_min": -54.34038162231445,
      "epoch": 2.39,
      "learning_rate": 0.00013596704545454545,
      "loss": 2.9249,
      "step": 41100
    },
    {
      "activations/layer0_attention_weight_max": 13.837201118469238,
      "activations/layer0_attention_weight_min": -14.575688362121582,
      "activations/layer10_attention_weight_max": 53.07028579711914,
      "activations/layer10_attention_weight_min": -43.461143493652344,
      "activations/layer11_attention_weight_max": 41.05055618286133,
      "activations/layer11_attention_weight_min": -32.569644927978516,
      "activations/layer12_attention_weight_max": 28.6866397857666,
      "activations/layer12_attention_weight_min": -24.381423950195312,
      "activations/layer13_attention_weight_max": 49.80516052246094,
      "activations/layer13_attention_weight_min": -40.063331604003906,
      "activations/layer14_attention_weight_max": 47.224430084228516,
      "activations/layer14_attention_weight_min": -37.22501754760742,
      "activations/layer15_attention_weight_max": 49.89331817626953,
      "activations/layer15_attention_weight_min": -36.659725189208984,
      "activations/layer16_attention_weight_max": 38.61467361450195,
      "activations/layer16_attention_weight_min": -27.857023239135742,
      "activations/layer17_attention_weight_max": 58.98442840576172,
      "activations/layer17_attention_weight_min": -37.81591033935547,
      "activations/layer18_attention_weight_max": 53.87718963623047,
      "activations/layer18_attention_weight_min": -34.85403060913086,
      "activations/layer19_attention_weight_max": 20.63360595703125,
      "activations/layer19_attention_weight_min": -17.4532470703125,
      "activations/layer1_attention_weight_max": 16.03363037109375,
      "activations/layer1_attention_weight_min": -13.07496166229248,
      "activations/layer20_attention_weight_max": 20.059566497802734,
      "activations/layer20_attention_weight_min": -17.410640716552734,
      "activations/layer21_attention_weight_max": 45.269649505615234,
      "activations/layer21_attention_weight_min": -24.638090133666992,
      "activations/layer22_attention_weight_max": 32.71900939941406,
      "activations/layer22_attention_weight_min": -21.978078842163086,
      "activations/layer23_attention_weight_max": 21.36156463623047,
      "activations/layer23_attention_weight_min": -17.147634506225586,
      "activations/layer2_attention_weight_max": 28.082489013671875,
      "activations/layer2_attention_weight_min": -28.038658142089844,
      "activations/layer3_attention_weight_max": 65.32598114013672,
      "activations/layer3_attention_weight_min": -73.0634994506836,
      "activations/layer4_attention_weight_max": 94.60419464111328,
      "activations/layer4_attention_weight_min": -91.2034912109375,
      "activations/layer5_attention_weight_max": 116.88470458984375,
      "activations/layer5_attention_weight_min": -100.89663696289062,
      "activations/layer6_attention_weight_max": 59.60625457763672,
      "activations/layer6_attention_weight_min": -48.727333068847656,
      "activations/layer7_attention_weight_max": 54.775306701660156,
      "activations/layer7_attention_weight_min": -49.84425354003906,
      "activations/layer8_attention_weight_max": 41.860267639160156,
      "activations/layer8_attention_weight_min": -35.87974166870117,
      "activations/layer9_attention_weight_max": 56.57981491088867,
      "activations/layer9_attention_weight_min": -52.23115921020508,
      "epoch": 2.39,
      "learning_rate": 0.00013594810606060606,
      "loss": 2.9499,
      "step": 41150
    },
    {
      "activations/layer0_attention_weight_max": 14.714847564697266,
      "activations/layer0_attention_weight_min": -13.956379890441895,
      "activations/layer10_attention_weight_max": 49.51779556274414,
      "activations/layer10_attention_weight_min": -42.75975799560547,
      "activations/layer11_attention_weight_max": 46.010276794433594,
      "activations/layer11_attention_weight_min": -31.465307235717773,
      "activations/layer12_attention_weight_max": 27.097856521606445,
      "activations/layer12_attention_weight_min": -24.16236686706543,
      "activations/layer13_attention_weight_max": 57.958900451660156,
      "activations/layer13_attention_weight_min": -42.784523010253906,
      "activations/layer14_attention_weight_max": 53.815547943115234,
      "activations/layer14_attention_weight_min": -32.648216247558594,
      "activations/layer15_attention_weight_max": 60.94597244262695,
      "activations/layer15_attention_weight_min": -35.44635009765625,
      "activations/layer16_attention_weight_max": 47.336612701416016,
      "activations/layer16_attention_weight_min": -27.60883331298828,
      "activations/layer17_attention_weight_max": 73.0959243774414,
      "activations/layer17_attention_weight_min": -39.508445739746094,
      "activations/layer18_attention_weight_max": 62.2327766418457,
      "activations/layer18_attention_weight_min": -33.960784912109375,
      "activations/layer19_attention_weight_max": 21.982120513916016,
      "activations/layer19_attention_weight_min": -16.430906295776367,
      "activations/layer1_attention_weight_max": 17.347837448120117,
      "activations/layer1_attention_weight_min": -12.572330474853516,
      "activations/layer20_attention_weight_max": 26.57841682434082,
      "activations/layer20_attention_weight_min": -20.501710891723633,
      "activations/layer21_attention_weight_max": 44.23761749267578,
      "activations/layer21_attention_weight_min": -26.345029830932617,
      "activations/layer22_attention_weight_max": 41.795082092285156,
      "activations/layer22_attention_weight_min": -21.171350479125977,
      "activations/layer23_attention_weight_max": 31.396284103393555,
      "activations/layer23_attention_weight_min": -16.11978530883789,
      "activations/layer2_attention_weight_max": 27.671964645385742,
      "activations/layer2_attention_weight_min": -27.061573028564453,
      "activations/layer3_attention_weight_max": 63.444095611572266,
      "activations/layer3_attention_weight_min": -68.46366119384766,
      "activations/layer4_attention_weight_max": 94.72924041748047,
      "activations/layer4_attention_weight_min": -87.3852310180664,
      "activations/layer5_attention_weight_max": 128.45606994628906,
      "activations/layer5_attention_weight_min": -102.779541015625,
      "activations/layer6_attention_weight_max": 62.20429611206055,
      "activations/layer6_attention_weight_min": -52.136924743652344,
      "activations/layer7_attention_weight_max": 55.73737716674805,
      "activations/layer7_attention_weight_min": -49.83916473388672,
      "activations/layer8_attention_weight_max": 42.00126266479492,
      "activations/layer8_attention_weight_min": -41.7725944519043,
      "activations/layer9_attention_weight_max": 57.0850715637207,
      "activations/layer9_attention_weight_min": -49.416351318359375,
      "epoch": 2.39,
      "learning_rate": 0.00013592916666666665,
      "loss": 2.927,
      "step": 41200
    },
    {
      "activations/layer0_attention_weight_max": 15.060114860534668,
      "activations/layer0_attention_weight_min": -14.58642292022705,
      "activations/layer10_attention_weight_max": 50.102294921875,
      "activations/layer10_attention_weight_min": -40.8796272277832,
      "activations/layer11_attention_weight_max": 41.05701446533203,
      "activations/layer11_attention_weight_min": -33.28276062011719,
      "activations/layer12_attention_weight_max": 25.74321937561035,
      "activations/layer12_attention_weight_min": -25.83452606201172,
      "activations/layer13_attention_weight_max": 53.40794372558594,
      "activations/layer13_attention_weight_min": -43.758426666259766,
      "activations/layer14_attention_weight_max": 53.38325881958008,
      "activations/layer14_attention_weight_min": -39.64452362060547,
      "activations/layer15_attention_weight_max": 49.80476379394531,
      "activations/layer15_attention_weight_min": -40.0532341003418,
      "activations/layer16_attention_weight_max": 35.07888412475586,
      "activations/layer16_attention_weight_min": -25.489309310913086,
      "activations/layer17_attention_weight_max": 56.02570343017578,
      "activations/layer17_attention_weight_min": -37.44841003417969,
      "activations/layer18_attention_weight_max": 47.89046096801758,
      "activations/layer18_attention_weight_min": -32.517486572265625,
      "activations/layer19_attention_weight_max": 19.578205108642578,
      "activations/layer19_attention_weight_min": -16.352508544921875,
      "activations/layer1_attention_weight_max": 17.148456573486328,
      "activations/layer1_attention_weight_min": -12.860630989074707,
      "activations/layer20_attention_weight_max": 18.828454971313477,
      "activations/layer20_attention_weight_min": -17.447776794433594,
      "activations/layer21_attention_weight_max": 39.37500762939453,
      "activations/layer21_attention_weight_min": -25.322378158569336,
      "activations/layer22_attention_weight_max": 33.211483001708984,
      "activations/layer22_attention_weight_min": -20.833913803100586,
      "activations/layer23_attention_weight_max": 26.01427459716797,
      "activations/layer23_attention_weight_min": -17.479732513427734,
      "activations/layer2_attention_weight_max": 26.605480194091797,
      "activations/layer2_attention_weight_min": -25.39461898803711,
      "activations/layer3_attention_weight_max": 64.88288116455078,
      "activations/layer3_attention_weight_min": -68.70760345458984,
      "activations/layer4_attention_weight_max": 90.76307678222656,
      "activations/layer4_attention_weight_min": -82.57679748535156,
      "activations/layer5_attention_weight_max": 115.67073059082031,
      "activations/layer5_attention_weight_min": -97.71587371826172,
      "activations/layer6_attention_weight_max": 57.666263580322266,
      "activations/layer6_attention_weight_min": -50.36330032348633,
      "activations/layer7_attention_weight_max": 56.36891174316406,
      "activations/layer7_attention_weight_min": -49.5680046081543,
      "activations/layer8_attention_weight_max": 40.812461853027344,
      "activations/layer8_attention_weight_min": -35.551109313964844,
      "activations/layer9_attention_weight_max": 60.79151153564453,
      "activations/layer9_attention_weight_min": -57.1710205078125,
      "epoch": 2.4,
      "learning_rate": 0.00013591022727272727,
      "loss": 2.9393,
      "step": 41250
    },
    {
      "activations/layer0_attention_weight_max": 13.999119758605957,
      "activations/layer0_attention_weight_min": -14.097261428833008,
      "activations/layer10_attention_weight_max": 56.238487243652344,
      "activations/layer10_attention_weight_min": -43.396427154541016,
      "activations/layer11_attention_weight_max": 42.24412155151367,
      "activations/layer11_attention_weight_min": -31.45393180847168,
      "activations/layer12_attention_weight_max": 28.56328773498535,
      "activations/layer12_attention_weight_min": -35.032859802246094,
      "activations/layer13_attention_weight_max": 57.7190055847168,
      "activations/layer13_attention_weight_min": -42.9535026550293,
      "activations/layer14_attention_weight_max": 39.246253967285156,
      "activations/layer14_attention_weight_min": -31.008621215820312,
      "activations/layer15_attention_weight_max": 51.458152770996094,
      "activations/layer15_attention_weight_min": -36.82294464111328,
      "activations/layer16_attention_weight_max": 39.04759216308594,
      "activations/layer16_attention_weight_min": -26.715499877929688,
      "activations/layer17_attention_weight_max": 59.83430480957031,
      "activations/layer17_attention_weight_min": -35.88908767700195,
      "activations/layer18_attention_weight_max": 60.60471725463867,
      "activations/layer18_attention_weight_min": -32.66714859008789,
      "activations/layer19_attention_weight_max": 22.208812713623047,
      "activations/layer19_attention_weight_min": -16.080488204956055,
      "activations/layer1_attention_weight_max": 16.585121154785156,
      "activations/layer1_attention_weight_min": -12.778964042663574,
      "activations/layer20_attention_weight_max": 17.87033462524414,
      "activations/layer20_attention_weight_min": -18.19567108154297,
      "activations/layer21_attention_weight_max": 41.82606887817383,
      "activations/layer21_attention_weight_min": -27.247337341308594,
      "activations/layer22_attention_weight_max": 35.48200988769531,
      "activations/layer22_attention_weight_min": -21.80340576171875,
      "activations/layer23_attention_weight_max": 24.63130760192871,
      "activations/layer23_attention_weight_min": -16.523591995239258,
      "activations/layer2_attention_weight_max": 30.84791374206543,
      "activations/layer2_attention_weight_min": -26.012821197509766,
      "activations/layer3_attention_weight_max": 68.60425567626953,
      "activations/layer3_attention_weight_min": -71.46408081054688,
      "activations/layer4_attention_weight_max": 104.01786041259766,
      "activations/layer4_attention_weight_min": -94.86226654052734,
      "activations/layer5_attention_weight_max": 130.00955200195312,
      "activations/layer5_attention_weight_min": -102.72073364257812,
      "activations/layer6_attention_weight_max": 60.56001663208008,
      "activations/layer6_attention_weight_min": -53.12131881713867,
      "activations/layer7_attention_weight_max": 61.28334045410156,
      "activations/layer7_attention_weight_min": -53.41120147705078,
      "activations/layer8_attention_weight_max": 42.65864944458008,
      "activations/layer8_attention_weight_min": -37.28432083129883,
      "activations/layer9_attention_weight_max": 64.43486785888672,
      "activations/layer9_attention_weight_min": -61.60237503051758,
      "epoch": 2.4,
      "learning_rate": 0.00013589128787878788,
      "loss": 2.9359,
      "step": 41300
    },
    {
      "activations/layer0_attention_weight_max": 13.827431678771973,
      "activations/layer0_attention_weight_min": -14.483592987060547,
      "activations/layer10_attention_weight_max": 52.47739791870117,
      "activations/layer10_attention_weight_min": -44.29447555541992,
      "activations/layer11_attention_weight_max": 39.225372314453125,
      "activations/layer11_attention_weight_min": -32.20407485961914,
      "activations/layer12_attention_weight_max": 37.390113830566406,
      "activations/layer12_attention_weight_min": -22.086267471313477,
      "activations/layer13_attention_weight_max": 52.611873626708984,
      "activations/layer13_attention_weight_min": -40.150047302246094,
      "activations/layer14_attention_weight_max": 52.833946228027344,
      "activations/layer14_attention_weight_min": -37.20642852783203,
      "activations/layer15_attention_weight_max": 48.534828186035156,
      "activations/layer15_attention_weight_min": -35.173343658447266,
      "activations/layer16_attention_weight_max": 42.38652420043945,
      "activations/layer16_attention_weight_min": -26.882484436035156,
      "activations/layer17_attention_weight_max": 58.9991340637207,
      "activations/layer17_attention_weight_min": -34.71138381958008,
      "activations/layer18_attention_weight_max": 59.62021255493164,
      "activations/layer18_attention_weight_min": -35.39116668701172,
      "activations/layer19_attention_weight_max": 24.560672760009766,
      "activations/layer19_attention_weight_min": -17.68513298034668,
      "activations/layer1_attention_weight_max": 16.26261329650879,
      "activations/layer1_attention_weight_min": -14.948132514953613,
      "activations/layer20_attention_weight_max": 20.751365661621094,
      "activations/layer20_attention_weight_min": -18.994253158569336,
      "activations/layer21_attention_weight_max": 44.40802764892578,
      "activations/layer21_attention_weight_min": -27.54990005493164,
      "activations/layer22_attention_weight_max": 36.40763473510742,
      "activations/layer22_attention_weight_min": -21.29389190673828,
      "activations/layer23_attention_weight_max": 21.741743087768555,
      "activations/layer23_attention_weight_min": -14.868989944458008,
      "activations/layer2_attention_weight_max": 28.395427703857422,
      "activations/layer2_attention_weight_min": -27.672758102416992,
      "activations/layer3_attention_weight_max": 66.53227996826172,
      "activations/layer3_attention_weight_min": -69.00696563720703,
      "activations/layer4_attention_weight_max": 93.8015365600586,
      "activations/layer4_attention_weight_min": -87.71102142333984,
      "activations/layer5_attention_weight_max": 119.76783752441406,
      "activations/layer5_attention_weight_min": -104.22163391113281,
      "activations/layer6_attention_weight_max": 58.45249938964844,
      "activations/layer6_attention_weight_min": -50.46030044555664,
      "activations/layer7_attention_weight_max": 57.35486602783203,
      "activations/layer7_attention_weight_min": -50.12949752807617,
      "activations/layer8_attention_weight_max": 43.31216049194336,
      "activations/layer8_attention_weight_min": -37.28778839111328,
      "activations/layer9_attention_weight_max": 59.931060791015625,
      "activations/layer9_attention_weight_min": -52.009578704833984,
      "epoch": 2.4,
      "learning_rate": 0.00013587234848484847,
      "loss": 2.9331,
      "step": 41350
    },
    {
      "activations/layer0_attention_weight_max": 14.427886962890625,
      "activations/layer0_attention_weight_min": -13.645983695983887,
      "activations/layer10_attention_weight_max": 61.72255325317383,
      "activations/layer10_attention_weight_min": -47.77005386352539,
      "activations/layer11_attention_weight_max": 42.293270111083984,
      "activations/layer11_attention_weight_min": -30.750591278076172,
      "activations/layer12_attention_weight_max": 28.07862663269043,
      "activations/layer12_attention_weight_min": -25.879837036132812,
      "activations/layer13_attention_weight_max": 65.71619415283203,
      "activations/layer13_attention_weight_min": -43.916114807128906,
      "activations/layer14_attention_weight_max": 66.9285888671875,
      "activations/layer14_attention_weight_min": -36.710243225097656,
      "activations/layer15_attention_weight_max": 53.86676788330078,
      "activations/layer15_attention_weight_min": -36.332401275634766,
      "activations/layer16_attention_weight_max": 42.30543899536133,
      "activations/layer16_attention_weight_min": -28.134050369262695,
      "activations/layer17_attention_weight_max": 63.04505920410156,
      "activations/layer17_attention_weight_min": -36.75876235961914,
      "activations/layer18_attention_weight_max": 56.25349426269531,
      "activations/layer18_attention_weight_min": -31.090248107910156,
      "activations/layer19_attention_weight_max": 23.218236923217773,
      "activations/layer19_attention_weight_min": -18.234424591064453,
      "activations/layer1_attention_weight_max": 16.87670135498047,
      "activations/layer1_attention_weight_min": -13.93246078491211,
      "activations/layer20_attention_weight_max": 21.58361053466797,
      "activations/layer20_attention_weight_min": -19.784107208251953,
      "activations/layer21_attention_weight_max": 39.8137321472168,
      "activations/layer21_attention_weight_min": -27.928251266479492,
      "activations/layer22_attention_weight_max": 36.44095230102539,
      "activations/layer22_attention_weight_min": -23.375078201293945,
      "activations/layer23_attention_weight_max": 27.26735496520996,
      "activations/layer23_attention_weight_min": -14.695625305175781,
      "activations/layer2_attention_weight_max": 26.464168548583984,
      "activations/layer2_attention_weight_min": -26.367605209350586,
      "activations/layer3_attention_weight_max": 67.3465347290039,
      "activations/layer3_attention_weight_min": -69.45293426513672,
      "activations/layer4_attention_weight_max": 90.0913314819336,
      "activations/layer4_attention_weight_min": -85.64067077636719,
      "activations/layer5_attention_weight_max": 117.66934204101562,
      "activations/layer5_attention_weight_min": -99.52001190185547,
      "activations/layer6_attention_weight_max": 58.169639587402344,
      "activations/layer6_attention_weight_min": -48.52169418334961,
      "activations/layer7_attention_weight_max": 53.888545989990234,
      "activations/layer7_attention_weight_min": -49.629852294921875,
      "activations/layer8_attention_weight_max": 39.52363204956055,
      "activations/layer8_attention_weight_min": -35.91582107543945,
      "activations/layer9_attention_weight_max": 60.084468841552734,
      "activations/layer9_attention_weight_min": -51.60932159423828,
      "epoch": 2.41,
      "learning_rate": 0.00013585340909090909,
      "loss": 2.9565,
      "step": 41400
    },
    {
      "activations/layer0_attention_weight_max": 14.052119255065918,
      "activations/layer0_attention_weight_min": -14.891655921936035,
      "activations/layer10_attention_weight_max": 59.7304801940918,
      "activations/layer10_attention_weight_min": -49.14331817626953,
      "activations/layer11_attention_weight_max": 47.381771087646484,
      "activations/layer11_attention_weight_min": -34.771820068359375,
      "activations/layer12_attention_weight_max": 31.549755096435547,
      "activations/layer12_attention_weight_min": -26.954862594604492,
      "activations/layer13_attention_weight_max": 59.108734130859375,
      "activations/layer13_attention_weight_min": -40.44713592529297,
      "activations/layer14_attention_weight_max": 54.13411331176758,
      "activations/layer14_attention_weight_min": -35.87805938720703,
      "activations/layer15_attention_weight_max": 52.17241287231445,
      "activations/layer15_attention_weight_min": -34.838741302490234,
      "activations/layer16_attention_weight_max": 37.12624740600586,
      "activations/layer16_attention_weight_min": -26.3763427734375,
      "activations/layer17_attention_weight_max": 57.68739700317383,
      "activations/layer17_attention_weight_min": -36.41481018066406,
      "activations/layer18_attention_weight_max": 57.5338020324707,
      "activations/layer18_attention_weight_min": -31.856664657592773,
      "activations/layer19_attention_weight_max": 21.33796501159668,
      "activations/layer19_attention_weight_min": -16.395353317260742,
      "activations/layer1_attention_weight_max": 17.70110511779785,
      "activations/layer1_attention_weight_min": -14.747145652770996,
      "activations/layer20_attention_weight_max": 19.030040740966797,
      "activations/layer20_attention_weight_min": -18.014080047607422,
      "activations/layer21_attention_weight_max": 43.8820686340332,
      "activations/layer21_attention_weight_min": -29.39504623413086,
      "activations/layer22_attention_weight_max": 33.22145080566406,
      "activations/layer22_attention_weight_min": -25.29413604736328,
      "activations/layer23_attention_weight_max": 21.48303985595703,
      "activations/layer23_attention_weight_min": -17.700571060180664,
      "activations/layer2_attention_weight_max": 27.753355026245117,
      "activations/layer2_attention_weight_min": -26.18844985961914,
      "activations/layer3_attention_weight_max": 62.56121063232422,
      "activations/layer3_attention_weight_min": -66.43460845947266,
      "activations/layer4_attention_weight_max": 93.03092193603516,
      "activations/layer4_attention_weight_min": -84.47799682617188,
      "activations/layer5_attention_weight_max": 119.36336517333984,
      "activations/layer5_attention_weight_min": -96.6739730834961,
      "activations/layer6_attention_weight_max": 61.49930953979492,
      "activations/layer6_attention_weight_min": -51.031436920166016,
      "activations/layer7_attention_weight_max": 66.06241607666016,
      "activations/layer7_attention_weight_min": -61.814659118652344,
      "activations/layer8_attention_weight_max": 48.071407318115234,
      "activations/layer8_attention_weight_min": -39.27324295043945,
      "activations/layer9_attention_weight_max": 66.46129608154297,
      "activations/layer9_attention_weight_min": -56.53764724731445,
      "epoch": 2.41,
      "learning_rate": 0.00013583446969696967,
      "loss": 2.9458,
      "step": 41450
    },
    {
      "activations/layer0_attention_weight_max": 15.450698852539062,
      "activations/layer0_attention_weight_min": -14.640207290649414,
      "activations/layer10_attention_weight_max": 54.17517852783203,
      "activations/layer10_attention_weight_min": -42.345638275146484,
      "activations/layer11_attention_weight_max": 40.662113189697266,
      "activations/layer11_attention_weight_min": -31.609840393066406,
      "activations/layer12_attention_weight_max": 29.622570037841797,
      "activations/layer12_attention_weight_min": -23.705408096313477,
      "activations/layer13_attention_weight_max": 58.4882698059082,
      "activations/layer13_attention_weight_min": -41.11321258544922,
      "activations/layer14_attention_weight_max": 54.49998092651367,
      "activations/layer14_attention_weight_min": -39.52693176269531,
      "activations/layer15_attention_weight_max": 57.81528854370117,
      "activations/layer15_attention_weight_min": -37.84819030761719,
      "activations/layer16_attention_weight_max": 45.968318939208984,
      "activations/layer16_attention_weight_min": -28.562591552734375,
      "activations/layer17_attention_weight_max": 61.54530334472656,
      "activations/layer17_attention_weight_min": -39.015811920166016,
      "activations/layer18_attention_weight_max": 54.85771942138672,
      "activations/layer18_attention_weight_min": -33.58774948120117,
      "activations/layer19_attention_weight_max": 23.601680755615234,
      "activations/layer19_attention_weight_min": -16.179250717163086,
      "activations/layer1_attention_weight_max": 16.827817916870117,
      "activations/layer1_attention_weight_min": -14.881950378417969,
      "activations/layer20_attention_weight_max": 20.674339294433594,
      "activations/layer20_attention_weight_min": -18.458086013793945,
      "activations/layer21_attention_weight_max": 48.495384216308594,
      "activations/layer21_attention_weight_min": -29.563884735107422,
      "activations/layer22_attention_weight_max": 42.23708724975586,
      "activations/layer22_attention_weight_min": -21.39424705505371,
      "activations/layer23_attention_weight_max": 24.527542114257812,
      "activations/layer23_attention_weight_min": -15.755844116210938,
      "activations/layer2_attention_weight_max": 28.47020721435547,
      "activations/layer2_attention_weight_min": -26.959274291992188,
      "activations/layer3_attention_weight_max": 64.93121337890625,
      "activations/layer3_attention_weight_min": -68.20350646972656,
      "activations/layer4_attention_weight_max": 89.76167297363281,
      "activations/layer4_attention_weight_min": -84.86614227294922,
      "activations/layer5_attention_weight_max": 120.13072204589844,
      "activations/layer5_attention_weight_min": -105.20491790771484,
      "activations/layer6_attention_weight_max": 59.281455993652344,
      "activations/layer6_attention_weight_min": -50.81831741333008,
      "activations/layer7_attention_weight_max": 56.78052520751953,
      "activations/layer7_attention_weight_min": -51.71452713012695,
      "activations/layer8_attention_weight_max": 43.838348388671875,
      "activations/layer8_attention_weight_min": -37.23909378051758,
      "activations/layer9_attention_weight_max": 60.0784912109375,
      "activations/layer9_attention_weight_min": -53.251312255859375,
      "epoch": 2.41,
      "learning_rate": 0.0001358155303030303,
      "loss": 2.9571,
      "step": 41500
    },
    {
      "activations/layer0_attention_weight_max": 13.91150188446045,
      "activations/layer0_attention_weight_min": -14.467432975769043,
      "activations/layer10_attention_weight_max": 52.82676696777344,
      "activations/layer10_attention_weight_min": -43.4432258605957,
      "activations/layer11_attention_weight_max": 41.048606872558594,
      "activations/layer11_attention_weight_min": -31.547515869140625,
      "activations/layer12_attention_weight_max": 35.30910873413086,
      "activations/layer12_attention_weight_min": -25.07038688659668,
      "activations/layer13_attention_weight_max": 53.817928314208984,
      "activations/layer13_attention_weight_min": -35.51210021972656,
      "activations/layer14_attention_weight_max": 46.44704818725586,
      "activations/layer14_attention_weight_min": -30.810834884643555,
      "activations/layer15_attention_weight_max": 50.22016525268555,
      "activations/layer15_attention_weight_min": -34.398963928222656,
      "activations/layer16_attention_weight_max": 40.444488525390625,
      "activations/layer16_attention_weight_min": -28.126863479614258,
      "activations/layer17_attention_weight_max": 67.18527221679688,
      "activations/layer17_attention_weight_min": -35.95335388183594,
      "activations/layer18_attention_weight_max": 54.24357604980469,
      "activations/layer18_attention_weight_min": -34.12895202636719,
      "activations/layer19_attention_weight_max": 26.847944259643555,
      "activations/layer19_attention_weight_min": -16.059968948364258,
      "activations/layer1_attention_weight_max": 16.258325576782227,
      "activations/layer1_attention_weight_min": -13.002862930297852,
      "activations/layer20_attention_weight_max": 23.48202896118164,
      "activations/layer20_attention_weight_min": -17.485960006713867,
      "activations/layer21_attention_weight_max": 47.53361129760742,
      "activations/layer21_attention_weight_min": -26.7746639251709,
      "activations/layer22_attention_weight_max": 38.51384353637695,
      "activations/layer22_attention_weight_min": -23.056621551513672,
      "activations/layer23_attention_weight_max": 24.90083122253418,
      "activations/layer23_attention_weight_min": -15.485648155212402,
      "activations/layer2_attention_weight_max": 28.283588409423828,
      "activations/layer2_attention_weight_min": -26.502058029174805,
      "activations/layer3_attention_weight_max": 66.8882827758789,
      "activations/layer3_attention_weight_min": -69.67564392089844,
      "activations/layer4_attention_weight_max": 93.2580795288086,
      "activations/layer4_attention_weight_min": -86.41703033447266,
      "activations/layer5_attention_weight_max": 126.19003295898438,
      "activations/layer5_attention_weight_min": -102.45909881591797,
      "activations/layer6_attention_weight_max": 64.78341674804688,
      "activations/layer6_attention_weight_min": -52.90442657470703,
      "activations/layer7_attention_weight_max": 58.26621627807617,
      "activations/layer7_attention_weight_min": -53.834571838378906,
      "activations/layer8_attention_weight_max": 44.34580612182617,
      "activations/layer8_attention_weight_min": -36.42206573486328,
      "activations/layer9_attention_weight_max": 64.09297180175781,
      "activations/layer9_attention_weight_min": -56.80194854736328,
      "epoch": 2.41,
      "learning_rate": 0.0001357965909090909,
      "loss": 2.9318,
      "step": 41550
    },
    {
      "activations/layer0_attention_weight_max": 15.632414817810059,
      "activations/layer0_attention_weight_min": -14.419925689697266,
      "activations/layer10_attention_weight_max": 54.466365814208984,
      "activations/layer10_attention_weight_min": -45.29353332519531,
      "activations/layer11_attention_weight_max": 39.967796325683594,
      "activations/layer11_attention_weight_min": -30.698989868164062,
      "activations/layer12_attention_weight_max": 27.84223175048828,
      "activations/layer12_attention_weight_min": -25.523189544677734,
      "activations/layer13_attention_weight_max": 68.2034912109375,
      "activations/layer13_attention_weight_min": -46.31239318847656,
      "activations/layer14_attention_weight_max": 64.03042602539062,
      "activations/layer14_attention_weight_min": -43.21116256713867,
      "activations/layer15_attention_weight_max": 56.85626983642578,
      "activations/layer15_attention_weight_min": -36.621944427490234,
      "activations/layer16_attention_weight_max": 44.20114517211914,
      "activations/layer16_attention_weight_min": -28.25016975402832,
      "activations/layer17_attention_weight_max": 58.45525360107422,
      "activations/layer17_attention_weight_min": -36.95960998535156,
      "activations/layer18_attention_weight_max": 57.53769302368164,
      "activations/layer18_attention_weight_min": -33.88113784790039,
      "activations/layer19_attention_weight_max": 24.033584594726562,
      "activations/layer19_attention_weight_min": -16.795310974121094,
      "activations/layer1_attention_weight_max": 17.16131591796875,
      "activations/layer1_attention_weight_min": -16.35207748413086,
      "activations/layer20_attention_weight_max": 21.12204360961914,
      "activations/layer20_attention_weight_min": -18.36650848388672,
      "activations/layer21_attention_weight_max": 39.34930419921875,
      "activations/layer21_attention_weight_min": -27.808561325073242,
      "activations/layer22_attention_weight_max": 38.027278900146484,
      "activations/layer22_attention_weight_min": -22.56273651123047,
      "activations/layer23_attention_weight_max": 25.132549285888672,
      "activations/layer23_attention_weight_min": -14.789058685302734,
      "activations/layer2_attention_weight_max": 28.340675354003906,
      "activations/layer2_attention_weight_min": -26.67264175415039,
      "activations/layer3_attention_weight_max": 64.45567321777344,
      "activations/layer3_attention_weight_min": -70.50214385986328,
      "activations/layer4_attention_weight_max": 90.39704132080078,
      "activations/layer4_attention_weight_min": -84.41261291503906,
      "activations/layer5_attention_weight_max": 118.04644775390625,
      "activations/layer5_attention_weight_min": -97.72433471679688,
      "activations/layer6_attention_weight_max": 56.83702850341797,
      "activations/layer6_attention_weight_min": -48.16804885864258,
      "activations/layer7_attention_weight_max": 55.68187713623047,
      "activations/layer7_attention_weight_min": -48.98229217529297,
      "activations/layer8_attention_weight_max": 40.938114166259766,
      "activations/layer8_attention_weight_min": -36.832210540771484,
      "activations/layer9_attention_weight_max": 63.5449104309082,
      "activations/layer9_attention_weight_min": -52.49601364135742,
      "epoch": 2.42,
      "learning_rate": 0.0001357780303030303,
      "loss": 2.9319,
      "step": 41600
    },
    {
      "activations/layer0_attention_weight_max": 15.37700366973877,
      "activations/layer0_attention_weight_min": -14.059569358825684,
      "activations/layer10_attention_weight_max": 67.50274658203125,
      "activations/layer10_attention_weight_min": -45.079925537109375,
      "activations/layer11_attention_weight_max": 52.04996871948242,
      "activations/layer11_attention_weight_min": -39.714088439941406,
      "activations/layer12_attention_weight_max": 31.819902420043945,
      "activations/layer12_attention_weight_min": -24.635648727416992,
      "activations/layer13_attention_weight_max": 62.3834114074707,
      "activations/layer13_attention_weight_min": -48.255043029785156,
      "activations/layer14_attention_weight_max": 65.85005950927734,
      "activations/layer14_attention_weight_min": -43.99958801269531,
      "activations/layer15_attention_weight_max": 55.22693634033203,
      "activations/layer15_attention_weight_min": -40.7590217590332,
      "activations/layer16_attention_weight_max": 38.94351577758789,
      "activations/layer16_attention_weight_min": -26.74329376220703,
      "activations/layer17_attention_weight_max": 63.14902114868164,
      "activations/layer17_attention_weight_min": -36.37742233276367,
      "activations/layer18_attention_weight_max": 56.09074401855469,
      "activations/layer18_attention_weight_min": -32.11583709716797,
      "activations/layer19_attention_weight_max": 19.613065719604492,
      "activations/layer19_attention_weight_min": -19.458457946777344,
      "activations/layer1_attention_weight_max": 18.512535095214844,
      "activations/layer1_attention_weight_min": -13.923649787902832,
      "activations/layer20_attention_weight_max": 19.796537399291992,
      "activations/layer20_attention_weight_min": -17.043119430541992,
      "activations/layer21_attention_weight_max": 40.215919494628906,
      "activations/layer21_attention_weight_min": -25.46358299255371,
      "activations/layer22_attention_weight_max": 34.4619026184082,
      "activations/layer22_attention_weight_min": -20.88437843322754,
      "activations/layer23_attention_weight_max": 23.887699127197266,
      "activations/layer23_attention_weight_min": -15.997000694274902,
      "activations/layer2_attention_weight_max": 28.042152404785156,
      "activations/layer2_attention_weight_min": -27.867408752441406,
      "activations/layer3_attention_weight_max": 68.0405502319336,
      "activations/layer3_attention_weight_min": -70.19586181640625,
      "activations/layer4_attention_weight_max": 94.38603973388672,
      "activations/layer4_attention_weight_min": -85.83586120605469,
      "activations/layer5_attention_weight_max": 129.1599578857422,
      "activations/layer5_attention_weight_min": -107.47006225585938,
      "activations/layer6_attention_weight_max": 64.4505844116211,
      "activations/layer6_attention_weight_min": -52.66912078857422,
      "activations/layer7_attention_weight_max": 69.09672546386719,
      "activations/layer7_attention_weight_min": -60.089691162109375,
      "activations/layer8_attention_weight_max": 56.3738899230957,
      "activations/layer8_attention_weight_min": -43.02297592163086,
      "activations/layer9_attention_weight_max": 76.060546875,
      "activations/layer9_attention_weight_min": -62.810943603515625,
      "epoch": 2.42,
      "learning_rate": 0.00013575909090909088,
      "loss": 2.9332,
      "step": 41650
    },
    {
      "activations/layer0_attention_weight_max": 14.806273460388184,
      "activations/layer0_attention_weight_min": -15.342124938964844,
      "activations/layer10_attention_weight_max": 57.1510009765625,
      "activations/layer10_attention_weight_min": -41.47719192504883,
      "activations/layer11_attention_weight_max": 39.393531799316406,
      "activations/layer11_attention_weight_min": -32.73195266723633,
      "activations/layer12_attention_weight_max": 26.494022369384766,
      "activations/layer12_attention_weight_min": -23.560941696166992,
      "activations/layer13_attention_weight_max": 62.92424392700195,
      "activations/layer13_attention_weight_min": -44.42058181762695,
      "activations/layer14_attention_weight_max": 62.483009338378906,
      "activations/layer14_attention_weight_min": -40.4272346496582,
      "activations/layer15_attention_weight_max": 47.50006866455078,
      "activations/layer15_attention_weight_min": -34.5302619934082,
      "activations/layer16_attention_weight_max": 37.97418975830078,
      "activations/layer16_attention_weight_min": -30.09636878967285,
      "activations/layer17_attention_weight_max": 54.49049758911133,
      "activations/layer17_attention_weight_min": -35.4269905090332,
      "activations/layer18_attention_weight_max": 56.15349578857422,
      "activations/layer18_attention_weight_min": -33.88654327392578,
      "activations/layer19_attention_weight_max": 20.550573348999023,
      "activations/layer19_attention_weight_min": -16.062763214111328,
      "activations/layer1_attention_weight_max": 16.678686141967773,
      "activations/layer1_attention_weight_min": -13.743285179138184,
      "activations/layer20_attention_weight_max": 20.784927368164062,
      "activations/layer20_attention_weight_min": -17.470361709594727,
      "activations/layer21_attention_weight_max": 41.61617660522461,
      "activations/layer21_attention_weight_min": -29.946231842041016,
      "activations/layer22_attention_weight_max": 35.65167999267578,
      "activations/layer22_attention_weight_min": -23.789997100830078,
      "activations/layer23_attention_weight_max": 22.800853729248047,
      "activations/layer23_attention_weight_min": -14.313952445983887,
      "activations/layer2_attention_weight_max": 28.017494201660156,
      "activations/layer2_attention_weight_min": -27.077314376831055,
      "activations/layer3_attention_weight_max": 67.06289672851562,
      "activations/layer3_attention_weight_min": -69.63175201416016,
      "activations/layer4_attention_weight_max": 92.78377532958984,
      "activations/layer4_attention_weight_min": -84.71985626220703,
      "activations/layer5_attention_weight_max": 117.36101531982422,
      "activations/layer5_attention_weight_min": -96.86074829101562,
      "activations/layer6_attention_weight_max": 56.959632873535156,
      "activations/layer6_attention_weight_min": -50.643428802490234,
      "activations/layer7_attention_weight_max": 52.8529167175293,
      "activations/layer7_attention_weight_min": -50.94563293457031,
      "activations/layer8_attention_weight_max": 39.700439453125,
      "activations/layer8_attention_weight_min": -34.267417907714844,
      "activations/layer9_attention_weight_max": 53.89699172973633,
      "activations/layer9_attention_weight_min": -49.94475555419922,
      "epoch": 2.42,
      "learning_rate": 0.0001357401515151515,
      "loss": 2.9372,
      "step": 41700
    },
    {
      "activations/layer0_attention_weight_max": 13.033966064453125,
      "activations/layer0_attention_weight_min": -15.114641189575195,
      "activations/layer10_attention_weight_max": 54.37277603149414,
      "activations/layer10_attention_weight_min": -40.00596618652344,
      "activations/layer11_attention_weight_max": 42.82297897338867,
      "activations/layer11_attention_weight_min": -29.252660751342773,
      "activations/layer12_attention_weight_max": 30.56629753112793,
      "activations/layer12_attention_weight_min": -27.5134334564209,
      "activations/layer13_attention_weight_max": 55.940330505371094,
      "activations/layer13_attention_weight_min": -40.292205810546875,
      "activations/layer14_attention_weight_max": 47.1446418762207,
      "activations/layer14_attention_weight_min": -35.64947509765625,
      "activations/layer15_attention_weight_max": 51.57273483276367,
      "activations/layer15_attention_weight_min": -33.52324295043945,
      "activations/layer16_attention_weight_max": 37.955081939697266,
      "activations/layer16_attention_weight_min": -29.842714309692383,
      "activations/layer17_attention_weight_max": 67.74696350097656,
      "activations/layer17_attention_weight_min": -36.27454376220703,
      "activations/layer18_attention_weight_max": 57.819053649902344,
      "activations/layer18_attention_weight_min": -32.022056579589844,
      "activations/layer19_attention_weight_max": 20.69722557067871,
      "activations/layer19_attention_weight_min": -14.963650703430176,
      "activations/layer1_attention_weight_max": 17.14527130126953,
      "activations/layer1_attention_weight_min": -12.932243347167969,
      "activations/layer20_attention_weight_max": 20.245689392089844,
      "activations/layer20_attention_weight_min": -20.743064880371094,
      "activations/layer21_attention_weight_max": 44.03047180175781,
      "activations/layer21_attention_weight_min": -24.09080696105957,
      "activations/layer22_attention_weight_max": 34.74089431762695,
      "activations/layer22_attention_weight_min": -21.20416259765625,
      "activations/layer23_attention_weight_max": 21.904434204101562,
      "activations/layer23_attention_weight_min": -14.79630184173584,
      "activations/layer2_attention_weight_max": 26.5754451751709,
      "activations/layer2_attention_weight_min": -26.519371032714844,
      "activations/layer3_attention_weight_max": 61.35007858276367,
      "activations/layer3_attention_weight_min": -64.89299774169922,
      "activations/layer4_attention_weight_max": 91.0456771850586,
      "activations/layer4_attention_weight_min": -83.46015167236328,
      "activations/layer5_attention_weight_max": 120.74275207519531,
      "activations/layer5_attention_weight_min": -99.21517944335938,
      "activations/layer6_attention_weight_max": 58.44114303588867,
      "activations/layer6_attention_weight_min": -51.113075256347656,
      "activations/layer7_attention_weight_max": 54.533668518066406,
      "activations/layer7_attention_weight_min": -52.770565032958984,
      "activations/layer8_attention_weight_max": 40.882015228271484,
      "activations/layer8_attention_weight_min": -36.609397888183594,
      "activations/layer9_attention_weight_max": 54.4933967590332,
      "activations/layer9_attention_weight_min": -49.49393844604492,
      "epoch": 2.43,
      "learning_rate": 0.0001357212121212121,
      "loss": 2.9425,
      "step": 41750
    },
    {
      "activations/layer0_attention_weight_max": 15.454313278198242,
      "activations/layer0_attention_weight_min": -14.48845386505127,
      "activations/layer10_attention_weight_max": 47.454833984375,
      "activations/layer10_attention_weight_min": -41.23026657104492,
      "activations/layer11_attention_weight_max": 34.400047302246094,
      "activations/layer11_attention_weight_min": -31.638248443603516,
      "activations/layer12_attention_weight_max": 29.681398391723633,
      "activations/layer12_attention_weight_min": -25.896888732910156,
      "activations/layer13_attention_weight_max": 52.188011169433594,
      "activations/layer13_attention_weight_min": -44.09553909301758,
      "activations/layer14_attention_weight_max": 46.757930755615234,
      "activations/layer14_attention_weight_min": -34.055198669433594,
      "activations/layer15_attention_weight_max": 55.94099807739258,
      "activations/layer15_attention_weight_min": -41.14105987548828,
      "activations/layer16_attention_weight_max": 38.16493225097656,
      "activations/layer16_attention_weight_min": -27.46067237854004,
      "activations/layer17_attention_weight_max": 57.40117263793945,
      "activations/layer17_attention_weight_min": -37.66553497314453,
      "activations/layer18_attention_weight_max": 48.88077926635742,
      "activations/layer18_attention_weight_min": -31.002620697021484,
      "activations/layer19_attention_weight_max": 20.871261596679688,
      "activations/layer19_attention_weight_min": -16.43994140625,
      "activations/layer1_attention_weight_max": 17.469242095947266,
      "activations/layer1_attention_weight_min": -14.826899528503418,
      "activations/layer20_attention_weight_max": 20.652650833129883,
      "activations/layer20_attention_weight_min": -19.3530330657959,
      "activations/layer21_attention_weight_max": 45.936500549316406,
      "activations/layer21_attention_weight_min": -28.57291030883789,
      "activations/layer22_attention_weight_max": 36.331138610839844,
      "activations/layer22_attention_weight_min": -23.67424201965332,
      "activations/layer23_attention_weight_max": 21.986486434936523,
      "activations/layer23_attention_weight_min": -17.208518981933594,
      "activations/layer2_attention_weight_max": 28.091711044311523,
      "activations/layer2_attention_weight_min": -26.08051300048828,
      "activations/layer3_attention_weight_max": 63.86705780029297,
      "activations/layer3_attention_weight_min": -68.42448425292969,
      "activations/layer4_attention_weight_max": 97.75362396240234,
      "activations/layer4_attention_weight_min": -87.5419692993164,
      "activations/layer5_attention_weight_max": 124.38912200927734,
      "activations/layer5_attention_weight_min": -101.3808822631836,
      "activations/layer6_attention_weight_max": 62.84038543701172,
      "activations/layer6_attention_weight_min": -53.54707717895508,
      "activations/layer7_attention_weight_max": 59.37361145019531,
      "activations/layer7_attention_weight_min": -54.95589828491211,
      "activations/layer8_attention_weight_max": 39.87309646606445,
      "activations/layer8_attention_weight_min": -35.27146530151367,
      "activations/layer9_attention_weight_max": 53.58613204956055,
      "activations/layer9_attention_weight_min": -52.119911193847656,
      "epoch": 2.43,
      "learning_rate": 0.0001357022727272727,
      "loss": 2.9258,
      "step": 41800
    },
    {
      "activations/layer0_attention_weight_max": 14.673839569091797,
      "activations/layer0_attention_weight_min": -14.502376556396484,
      "activations/layer10_attention_weight_max": 49.80213165283203,
      "activations/layer10_attention_weight_min": -38.18753433227539,
      "activations/layer11_attention_weight_max": 37.036258697509766,
      "activations/layer11_attention_weight_min": -32.612388610839844,
      "activations/layer12_attention_weight_max": 30.85120391845703,
      "activations/layer12_attention_weight_min": -22.819305419921875,
      "activations/layer13_attention_weight_max": 50.804019927978516,
      "activations/layer13_attention_weight_min": -37.20964431762695,
      "activations/layer14_attention_weight_max": 44.376808166503906,
      "activations/layer14_attention_weight_min": -31.15146255493164,
      "activations/layer15_attention_weight_max": 49.898948669433594,
      "activations/layer15_attention_weight_min": -34.98557662963867,
      "activations/layer16_attention_weight_max": 43.194400787353516,
      "activations/layer16_attention_weight_min": -30.10798454284668,
      "activations/layer17_attention_weight_max": 59.4794921875,
      "activations/layer17_attention_weight_min": -37.718021392822266,
      "activations/layer18_attention_weight_max": 62.019954681396484,
      "activations/layer18_attention_weight_min": -31.73232650756836,
      "activations/layer19_attention_weight_max": 21.968839645385742,
      "activations/layer19_attention_weight_min": -19.588838577270508,
      "activations/layer1_attention_weight_max": 17.534433364868164,
      "activations/layer1_attention_weight_min": -14.16397476196289,
      "activations/layer20_attention_weight_max": 20.360389709472656,
      "activations/layer20_attention_weight_min": -18.4346866607666,
      "activations/layer21_attention_weight_max": 45.38127899169922,
      "activations/layer21_attention_weight_min": -25.29254722595215,
      "activations/layer22_attention_weight_max": 39.23081588745117,
      "activations/layer22_attention_weight_min": -23.84200096130371,
      "activations/layer23_attention_weight_max": 24.60793113708496,
      "activations/layer23_attention_weight_min": -13.635214805603027,
      "activations/layer2_attention_weight_max": 28.2178955078125,
      "activations/layer2_attention_weight_min": -27.490339279174805,
      "activations/layer3_attention_weight_max": 65.768798828125,
      "activations/layer3_attention_weight_min": -69.92240142822266,
      "activations/layer4_attention_weight_max": 94.074951171875,
      "activations/layer4_attention_weight_min": -84.15592193603516,
      "activations/layer5_attention_weight_max": 121.3811264038086,
      "activations/layer5_attention_weight_min": -99.05911254882812,
      "activations/layer6_attention_weight_max": 59.31404495239258,
      "activations/layer6_attention_weight_min": -48.835548400878906,
      "activations/layer7_attention_weight_max": 57.43254089355469,
      "activations/layer7_attention_weight_min": -51.30079650878906,
      "activations/layer8_attention_weight_max": 44.34877395629883,
      "activations/layer8_attention_weight_min": -37.1645393371582,
      "activations/layer9_attention_weight_max": 66.89077758789062,
      "activations/layer9_attention_weight_min": -52.83948516845703,
      "epoch": 2.43,
      "learning_rate": 0.00013568333333333332,
      "loss": 2.9456,
      "step": 41850
    },
    {
      "activations/layer0_attention_weight_max": 14.041719436645508,
      "activations/layer0_attention_weight_min": -15.014077186584473,
      "activations/layer10_attention_weight_max": 60.391719818115234,
      "activations/layer10_attention_weight_min": -44.950439453125,
      "activations/layer11_attention_weight_max": 42.50338363647461,
      "activations/layer11_attention_weight_min": -31.498062133789062,
      "activations/layer12_attention_weight_max": 36.32514953613281,
      "activations/layer12_attention_weight_min": -22.977035522460938,
      "activations/layer13_attention_weight_max": 60.82831573486328,
      "activations/layer13_attention_weight_min": -43.23463821411133,
      "activations/layer14_attention_weight_max": 54.162174224853516,
      "activations/layer14_attention_weight_min": -40.855892181396484,
      "activations/layer15_attention_weight_max": 55.009647369384766,
      "activations/layer15_attention_weight_min": -35.137298583984375,
      "activations/layer16_attention_weight_max": 41.06494140625,
      "activations/layer16_attention_weight_min": -28.043331146240234,
      "activations/layer17_attention_weight_max": 64.41029357910156,
      "activations/layer17_attention_weight_min": -38.131195068359375,
      "activations/layer18_attention_weight_max": 56.79662322998047,
      "activations/layer18_attention_weight_min": -31.7568302154541,
      "activations/layer19_attention_weight_max": 24.628524780273438,
      "activations/layer19_attention_weight_min": -17.07706642150879,
      "activations/layer1_attention_weight_max": 17.32365608215332,
      "activations/layer1_attention_weight_min": -14.491982460021973,
      "activations/layer20_attention_weight_max": 22.140453338623047,
      "activations/layer20_attention_weight_min": -16.69756317138672,
      "activations/layer21_attention_weight_max": 44.345306396484375,
      "activations/layer21_attention_weight_min": -29.458049774169922,
      "activations/layer22_attention_weight_max": 37.770469665527344,
      "activations/layer22_attention_weight_min": -23.598026275634766,
      "activations/layer23_attention_weight_max": 28.30750846862793,
      "activations/layer23_attention_weight_min": -16.05251693725586,
      "activations/layer2_attention_weight_max": 28.008682250976562,
      "activations/layer2_attention_weight_min": -26.988479614257812,
      "activations/layer3_attention_weight_max": 65.13768005371094,
      "activations/layer3_attention_weight_min": -69.48800659179688,
      "activations/layer4_attention_weight_max": 92.04150390625,
      "activations/layer4_attention_weight_min": -87.42269897460938,
      "activations/layer5_attention_weight_max": 123.61604309082031,
      "activations/layer5_attention_weight_min": -104.65975952148438,
      "activations/layer6_attention_weight_max": 64.31655883789062,
      "activations/layer6_attention_weight_min": -54.12328338623047,
      "activations/layer7_attention_weight_max": 56.4255485534668,
      "activations/layer7_attention_weight_min": -54.36460494995117,
      "activations/layer8_attention_weight_max": 44.80558395385742,
      "activations/layer8_attention_weight_min": -37.81333541870117,
      "activations/layer9_attention_weight_max": 59.08706283569336,
      "activations/layer9_attention_weight_min": -54.83720016479492,
      "epoch": 2.43,
      "learning_rate": 0.00013566439393939393,
      "loss": 2.9434,
      "step": 41900
    },
    {
      "activations/layer0_attention_weight_max": 15.523103713989258,
      "activations/layer0_attention_weight_min": -14.589664459228516,
      "activations/layer10_attention_weight_max": 48.68143844604492,
      "activations/layer10_attention_weight_min": -39.207767486572266,
      "activations/layer11_attention_weight_max": 44.07224655151367,
      "activations/layer11_attention_weight_min": -31.001129150390625,
      "activations/layer12_attention_weight_max": 30.25905990600586,
      "activations/layer12_attention_weight_min": -23.812456130981445,
      "activations/layer13_attention_weight_max": 63.58704376220703,
      "activations/layer13_attention_weight_min": -42.8432502746582,
      "activations/layer14_attention_weight_max": 58.12625503540039,
      "activations/layer14_attention_weight_min": -39.05801010131836,
      "activations/layer15_attention_weight_max": 55.80227279663086,
      "activations/layer15_attention_weight_min": -38.91056442260742,
      "activations/layer16_attention_weight_max": 39.78718185424805,
      "activations/layer16_attention_weight_min": -26.782114028930664,
      "activations/layer17_attention_weight_max": 66.27433776855469,
      "activations/layer17_attention_weight_min": -37.19097137451172,
      "activations/layer18_attention_weight_max": 54.393890380859375,
      "activations/layer18_attention_weight_min": -33.61053466796875,
      "activations/layer19_attention_weight_max": 18.10770606994629,
      "activations/layer19_attention_weight_min": -14.951828002929688,
      "activations/layer1_attention_weight_max": 17.1374568939209,
      "activations/layer1_attention_weight_min": -13.381595611572266,
      "activations/layer20_attention_weight_max": 19.00337791442871,
      "activations/layer20_attention_weight_min": -15.738791465759277,
      "activations/layer21_attention_weight_max": 42.80231475830078,
      "activations/layer21_attention_weight_min": -28.13035011291504,
      "activations/layer22_attention_weight_max": 33.7569694519043,
      "activations/layer22_attention_weight_min": -21.918794631958008,
      "activations/layer23_attention_weight_max": 22.537860870361328,
      "activations/layer23_attention_weight_min": -15.067062377929688,
      "activations/layer2_attention_weight_max": 26.082447052001953,
      "activations/layer2_attention_weight_min": -24.930356979370117,
      "activations/layer3_attention_weight_max": 65.52815246582031,
      "activations/layer3_attention_weight_min": -68.55397033691406,
      "activations/layer4_attention_weight_max": 95.65292358398438,
      "activations/layer4_attention_weight_min": -86.0716781616211,
      "activations/layer5_attention_weight_max": 113.28388977050781,
      "activations/layer5_attention_weight_min": -96.34832763671875,
      "activations/layer6_attention_weight_max": 59.24661636352539,
      "activations/layer6_attention_weight_min": -49.24354934692383,
      "activations/layer7_attention_weight_max": 57.38706970214844,
      "activations/layer7_attention_weight_min": -51.214866638183594,
      "activations/layer8_attention_weight_max": 40.23295974731445,
      "activations/layer8_attention_weight_min": -34.17128372192383,
      "activations/layer9_attention_weight_max": 58.267669677734375,
      "activations/layer9_attention_weight_min": -51.466976165771484,
      "epoch": 2.44,
      "learning_rate": 0.00013564545454545452,
      "loss": 2.9495,
      "step": 41950
    },
    {
      "activations/layer0_attention_weight_max": 14.67903995513916,
      "activations/layer0_attention_weight_min": -14.26483154296875,
      "activations/layer10_attention_weight_max": 52.465232849121094,
      "activations/layer10_attention_weight_min": -40.9369010925293,
      "activations/layer11_attention_weight_max": 40.797828674316406,
      "activations/layer11_attention_weight_min": -30.940673828125,
      "activations/layer12_attention_weight_max": 31.964839935302734,
      "activations/layer12_attention_weight_min": -25.66584014892578,
      "activations/layer13_attention_weight_max": 61.47444534301758,
      "activations/layer13_attention_weight_min": -41.826377868652344,
      "activations/layer14_attention_weight_max": 54.832908630371094,
      "activations/layer14_attention_weight_min": -39.634605407714844,
      "activations/layer15_attention_weight_max": 51.182735443115234,
      "activations/layer15_attention_weight_min": -35.4437255859375,
      "activations/layer16_attention_weight_max": 36.60622787475586,
      "activations/layer16_attention_weight_min": -28.647886276245117,
      "activations/layer17_attention_weight_max": 58.93091583251953,
      "activations/layer17_attention_weight_min": -37.081398010253906,
      "activations/layer18_attention_weight_max": 55.04445266723633,
      "activations/layer18_attention_weight_min": -31.319034576416016,
      "activations/layer19_attention_weight_max": 24.420114517211914,
      "activations/layer19_attention_weight_min": -16.420299530029297,
      "activations/layer1_attention_weight_max": 16.914152145385742,
      "activations/layer1_attention_weight_min": -12.59431266784668,
      "activations/layer20_attention_weight_max": 21.934635162353516,
      "activations/layer20_attention_weight_min": -16.9025821685791,
      "activations/layer21_attention_weight_max": 35.93082809448242,
      "activations/layer21_attention_weight_min": -27.328655242919922,
      "activations/layer22_attention_weight_max": 33.859188079833984,
      "activations/layer22_attention_weight_min": -22.444551467895508,
      "activations/layer23_attention_weight_max": 24.87031364440918,
      "activations/layer23_attention_weight_min": -16.172008514404297,
      "activations/layer2_attention_weight_max": 28.510536193847656,
      "activations/layer2_attention_weight_min": -27.1084041595459,
      "activations/layer3_attention_weight_max": 66.68062591552734,
      "activations/layer3_attention_weight_min": -72.61566162109375,
      "activations/layer4_attention_weight_max": 97.70514678955078,
      "activations/layer4_attention_weight_min": -85.73570251464844,
      "activations/layer5_attention_weight_max": 127.6939926147461,
      "activations/layer5_attention_weight_min": -98.9931640625,
      "activations/layer6_attention_weight_max": 61.86487579345703,
      "activations/layer6_attention_weight_min": -49.895469665527344,
      "activations/layer7_attention_weight_max": 57.37580490112305,
      "activations/layer7_attention_weight_min": -55.28123474121094,
      "activations/layer8_attention_weight_max": 46.915592193603516,
      "activations/layer8_attention_weight_min": -36.128639221191406,
      "activations/layer9_attention_weight_max": 63.21148681640625,
      "activations/layer9_attention_weight_min": -51.436031341552734,
      "epoch": 2.44,
      "learning_rate": 0.00013562651515151513,
      "loss": 2.9515,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_loss": 2.888671875,
      "eval_runtime": 8.4899,
      "eval_samples_per_second": 505.78,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_openwebtext_loss": 2.888671875,
      "eval_openwebtext_ppl": 17.969428099556087,
      "eval_openwebtext_runtime": 8.4899,
      "eval_openwebtext_samples_per_second": 505.78,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_wikitext_loss": 3.15625,
      "eval_wikitext_ppl": 23.482371718117374,
      "eval_wikitext_runtime": 1.968,
      "eval_wikitext_samples_per_second": 231.704,
      "step": 42000
    },
    {
      "epoch": 2.44,
      "eval_lambada_loss": 3.087890625,
      "eval_lambada_ppl": 21.930768935644906,
      "eval_lambada_runtime": 9.61,
      "eval_lambada_samples_per_second": 506.657,
      "step": 42000
    },
    {
      "activations/layer0_attention_weight_max": 14.530264854431152,
      "activations/layer0_attention_weight_min": -14.521895408630371,
      "activations/layer10_attention_weight_max": 57.22899627685547,
      "activations/layer10_attention_weight_min": -45.45904541015625,
      "activations/layer11_attention_weight_max": 41.04551696777344,
      "activations/layer11_attention_weight_min": -30.86422348022461,
      "activations/layer12_attention_weight_max": 32.07548522949219,
      "activations/layer12_attention_weight_min": -25.482635498046875,
      "activations/layer13_attention_weight_max": 59.249271392822266,
      "activations/layer13_attention_weight_min": -38.89522171020508,
      "activations/layer14_attention_weight_max": 56.45224380493164,
      "activations/layer14_attention_weight_min": -34.80877685546875,
      "activations/layer15_attention_weight_max": 52.533626556396484,
      "activations/layer15_attention_weight_min": -36.46139144897461,
      "activations/layer16_attention_weight_max": 41.2528076171875,
      "activations/layer16_attention_weight_min": -28.37516212463379,
      "activations/layer17_attention_weight_max": 60.50703811645508,
      "activations/layer17_attention_weight_min": -38.80204772949219,
      "activations/layer18_attention_weight_max": 55.00481414794922,
      "activations/layer18_attention_weight_min": -34.76517105102539,
      "activations/layer19_attention_weight_max": 26.579492568969727,
      "activations/layer19_attention_weight_min": -16.59193229675293,
      "activations/layer1_attention_weight_max": 19.005918502807617,
      "activations/layer1_attention_weight_min": -13.712581634521484,
      "activations/layer20_attention_weight_max": 20.31161880493164,
      "activations/layer20_attention_weight_min": -18.135112762451172,
      "activations/layer21_attention_weight_max": 37.18586349487305,
      "activations/layer21_attention_weight_min": -27.684085845947266,
      "activations/layer22_attention_weight_max": 35.06697463989258,
      "activations/layer22_attention_weight_min": -23.356159210205078,
      "activations/layer23_attention_weight_max": 24.849716186523438,
      "activations/layer23_attention_weight_min": -17.735858917236328,
      "activations/layer2_attention_weight_max": 28.34789276123047,
      "activations/layer2_attention_weight_min": -28.04111099243164,
      "activations/layer3_attention_weight_max": 66.74247741699219,
      "activations/layer3_attention_weight_min": -68.82009887695312,
      "activations/layer4_attention_weight_max": 96.12934112548828,
      "activations/layer4_attention_weight_min": -85.5552749633789,
      "activations/layer5_attention_weight_max": 126.23751068115234,
      "activations/layer5_attention_weight_min": -99.79173278808594,
      "activations/layer6_attention_weight_max": 61.36798858642578,
      "activations/layer6_attention_weight_min": -50.67761993408203,
      "activations/layer7_attention_weight_max": 55.64021682739258,
      "activations/layer7_attention_weight_min": -51.39567184448242,
      "activations/layer8_attention_weight_max": 45.255130767822266,
      "activations/layer8_attention_weight_min": -34.40675354003906,
      "activations/layer9_attention_weight_max": 59.8614616394043,
      "activations/layer9_attention_weight_min": -53.96627426147461,
      "epoch": 2.44,
      "learning_rate": 0.00013560757575757575,
      "loss": 2.9378,
      "step": 42050
    },
    {
      "activations/layer0_attention_weight_max": 15.276103973388672,
      "activations/layer0_attention_weight_min": -14.541929244995117,
      "activations/layer10_attention_weight_max": 54.99641036987305,
      "activations/layer10_attention_weight_min": -39.88471221923828,
      "activations/layer11_attention_weight_max": 41.102256774902344,
      "activations/layer11_attention_weight_min": -32.38669967651367,
      "activations/layer12_attention_weight_max": 27.47529411315918,
      "activations/layer12_attention_weight_min": -30.359634399414062,
      "activations/layer13_attention_weight_max": 69.20491790771484,
      "activations/layer13_attention_weight_min": -43.51555633544922,
      "activations/layer14_attention_weight_max": 94.19083404541016,
      "activations/layer14_attention_weight_min": -47.879520416259766,
      "activations/layer15_attention_weight_max": 65.08550262451172,
      "activations/layer15_attention_weight_min": -39.181583404541016,
      "activations/layer16_attention_weight_max": 40.31562423706055,
      "activations/layer16_attention_weight_min": -25.5849609375,
      "activations/layer17_attention_weight_max": 59.21511459350586,
      "activations/layer17_attention_weight_min": -40.26319122314453,
      "activations/layer18_attention_weight_max": 60.496543884277344,
      "activations/layer18_attention_weight_min": -32.9006462097168,
      "activations/layer19_attention_weight_max": 26.82790184020996,
      "activations/layer19_attention_weight_min": -16.93842124938965,
      "activations/layer1_attention_weight_max": 18.087966918945312,
      "activations/layer1_attention_weight_min": -13.259825706481934,
      "activations/layer20_attention_weight_max": 18.757253646850586,
      "activations/layer20_attention_weight_min": -19.820375442504883,
      "activations/layer21_attention_weight_max": 44.56635665893555,
      "activations/layer21_attention_weight_min": -28.67692756652832,
      "activations/layer22_attention_weight_max": 36.578243255615234,
      "activations/layer22_attention_weight_min": -23.390033721923828,
      "activations/layer23_attention_weight_max": 22.80365753173828,
      "activations/layer23_attention_weight_min": -18.027759552001953,
      "activations/layer2_attention_weight_max": 27.399778366088867,
      "activations/layer2_attention_weight_min": -26.128986358642578,
      "activations/layer3_attention_weight_max": 71.15270233154297,
      "activations/layer3_attention_weight_min": -70.85576629638672,
      "activations/layer4_attention_weight_max": 107.48846435546875,
      "activations/layer4_attention_weight_min": -91.01919555664062,
      "activations/layer5_attention_weight_max": 133.97195434570312,
      "activations/layer5_attention_weight_min": -109.88714599609375,
      "activations/layer6_attention_weight_max": 64.12579345703125,
      "activations/layer6_attention_weight_min": -52.413536071777344,
      "activations/layer7_attention_weight_max": 66.62393951416016,
      "activations/layer7_attention_weight_min": -55.76448059082031,
      "activations/layer8_attention_weight_max": 47.01445770263672,
      "activations/layer8_attention_weight_min": -38.644832611083984,
      "activations/layer9_attention_weight_max": 63.09527587890625,
      "activations/layer9_attention_weight_min": -55.99360275268555,
      "epoch": 2.45,
      "learning_rate": 0.00013558863636363634,
      "loss": 2.9332,
      "step": 42100
    },
    {
      "activations/layer0_attention_weight_max": 13.28212833404541,
      "activations/layer0_attention_weight_min": -14.610045433044434,
      "activations/layer10_attention_weight_max": 52.00288772583008,
      "activations/layer10_attention_weight_min": -39.422584533691406,
      "activations/layer11_attention_weight_max": 43.70823669433594,
      "activations/layer11_attention_weight_min": -30.31839370727539,
      "activations/layer12_attention_weight_max": 29.591312408447266,
      "activations/layer12_attention_weight_min": -24.874540328979492,
      "activations/layer13_attention_weight_max": 69.87964630126953,
      "activations/layer13_attention_weight_min": -49.34935760498047,
      "activations/layer14_attention_weight_max": 71.68269348144531,
      "activations/layer14_attention_weight_min": -49.99546432495117,
      "activations/layer15_attention_weight_max": 55.521148681640625,
      "activations/layer15_attention_weight_min": -35.208736419677734,
      "activations/layer16_attention_weight_max": 45.03373336791992,
      "activations/layer16_attention_weight_min": -26.800968170166016,
      "activations/layer17_attention_weight_max": 60.81808090209961,
      "activations/layer17_attention_weight_min": -36.74717712402344,
      "activations/layer18_attention_weight_max": 56.865901947021484,
      "activations/layer18_attention_weight_min": -33.588470458984375,
      "activations/layer19_attention_weight_max": 21.20914649963379,
      "activations/layer19_attention_weight_min": -17.55963706970215,
      "activations/layer1_attention_weight_max": 17.870267868041992,
      "activations/layer1_attention_weight_min": -13.773297309875488,
      "activations/layer20_attention_weight_max": 20.408016204833984,
      "activations/layer20_attention_weight_min": -21.486642837524414,
      "activations/layer21_attention_weight_max": 42.30575180053711,
      "activations/layer21_attention_weight_min": -28.86502456665039,
      "activations/layer22_attention_weight_max": 39.37515640258789,
      "activations/layer22_attention_weight_min": -22.796344757080078,
      "activations/layer23_attention_weight_max": 23.32672691345215,
      "activations/layer23_attention_weight_min": -14.560198783874512,
      "activations/layer2_attention_weight_max": 27.628515243530273,
      "activations/layer2_attention_weight_min": -27.096845626831055,
      "activations/layer3_attention_weight_max": 65.4111557006836,
      "activations/layer3_attention_weight_min": -70.7379150390625,
      "activations/layer4_attention_weight_max": 99.3316879272461,
      "activations/layer4_attention_weight_min": -88.44627380371094,
      "activations/layer5_attention_weight_max": 126.47600555419922,
      "activations/layer5_attention_weight_min": -103.83218383789062,
      "activations/layer6_attention_weight_max": 64.4671630859375,
      "activations/layer6_attention_weight_min": -50.74501419067383,
      "activations/layer7_attention_weight_max": 57.18632507324219,
      "activations/layer7_attention_weight_min": -50.256996154785156,
      "activations/layer8_attention_weight_max": 42.94062805175781,
      "activations/layer8_attention_weight_min": -37.26334762573242,
      "activations/layer9_attention_weight_max": 59.06929397583008,
      "activations/layer9_attention_weight_min": -52.1940803527832,
      "epoch": 2.45,
      "learning_rate": 0.00013556969696969695,
      "loss": 2.938,
      "step": 42150
    },
    {
      "activations/layer0_attention_weight_max": 14.219038963317871,
      "activations/layer0_attention_weight_min": -14.96079158782959,
      "activations/layer10_attention_weight_max": 51.938472747802734,
      "activations/layer10_attention_weight_min": -40.3626708984375,
      "activations/layer11_attention_weight_max": 38.638397216796875,
      "activations/layer11_attention_weight_min": -31.4002742767334,
      "activations/layer12_attention_weight_max": 23.66960906982422,
      "activations/layer12_attention_weight_min": -33.9085693359375,
      "activations/layer13_attention_weight_max": 49.32831573486328,
      "activations/layer13_attention_weight_min": -36.265872955322266,
      "activations/layer14_attention_weight_max": 40.68598175048828,
      "activations/layer14_attention_weight_min": -35.60543441772461,
      "activations/layer15_attention_weight_max": 43.83744812011719,
      "activations/layer15_attention_weight_min": -35.270301818847656,
      "activations/layer16_attention_weight_max": 34.52302932739258,
      "activations/layer16_attention_weight_min": -28.610393524169922,
      "activations/layer17_attention_weight_max": 56.477046966552734,
      "activations/layer17_attention_weight_min": -38.29888916015625,
      "activations/layer18_attention_weight_max": 49.13605499267578,
      "activations/layer18_attention_weight_min": -31.963369369506836,
      "activations/layer19_attention_weight_max": 17.71502685546875,
      "activations/layer19_attention_weight_min": -15.81541919708252,
      "activations/layer1_attention_weight_max": 16.799583435058594,
      "activations/layer1_attention_weight_min": -13.782645225524902,
      "activations/layer20_attention_weight_max": 17.711204528808594,
      "activations/layer20_attention_weight_min": -15.929214477539062,
      "activations/layer21_attention_weight_max": 33.7946662902832,
      "activations/layer21_attention_weight_min": -27.3924560546875,
      "activations/layer22_attention_weight_max": 33.215171813964844,
      "activations/layer22_attention_weight_min": -20.684755325317383,
      "activations/layer23_attention_weight_max": 21.392364501953125,
      "activations/layer23_attention_weight_min": -14.89068603515625,
      "activations/layer2_attention_weight_max": 27.823253631591797,
      "activations/layer2_attention_weight_min": -25.604154586791992,
      "activations/layer3_attention_weight_max": 70.13339233398438,
      "activations/layer3_attention_weight_min": -68.43551635742188,
      "activations/layer4_attention_weight_max": 97.55402374267578,
      "activations/layer4_attention_weight_min": -84.80915832519531,
      "activations/layer5_attention_weight_max": 130.34432983398438,
      "activations/layer5_attention_weight_min": -98.7944564819336,
      "activations/layer6_attention_weight_max": 62.699066162109375,
      "activations/layer6_attention_weight_min": -48.09505081176758,
      "activations/layer7_attention_weight_max": 57.23844909667969,
      "activations/layer7_attention_weight_min": -57.950260162353516,
      "activations/layer8_attention_weight_max": 40.87211608886719,
      "activations/layer8_attention_weight_min": -35.336997985839844,
      "activations/layer9_attention_weight_max": 59.6735954284668,
      "activations/layer9_attention_weight_min": -66.40950775146484,
      "epoch": 2.45,
      "learning_rate": 0.00013555075757575757,
      "loss": 2.9434,
      "step": 42200
    },
    {
      "activations/layer0_attention_weight_max": 15.114867210388184,
      "activations/layer0_attention_weight_min": -15.239720344543457,
      "activations/layer10_attention_weight_max": 50.466209411621094,
      "activations/layer10_attention_weight_min": -40.81067657470703,
      "activations/layer11_attention_weight_max": 42.68067169189453,
      "activations/layer11_attention_weight_min": -30.748945236206055,
      "activations/layer12_attention_weight_max": 24.088834762573242,
      "activations/layer12_attention_weight_min": -26.376155853271484,
      "activations/layer13_attention_weight_max": 66.86321258544922,
      "activations/layer13_attention_weight_min": -42.56380844116211,
      "activations/layer14_attention_weight_max": 65.11209106445312,
      "activations/layer14_attention_weight_min": -43.0816764831543,
      "activations/layer15_attention_weight_max": 48.4813117980957,
      "activations/layer15_attention_weight_min": -34.91444778442383,
      "activations/layer16_attention_weight_max": 43.94955062866211,
      "activations/layer16_attention_weight_min": -27.865785598754883,
      "activations/layer17_attention_weight_max": 59.71042251586914,
      "activations/layer17_attention_weight_min": -37.07637405395508,
      "activations/layer18_attention_weight_max": 53.126277923583984,
      "activations/layer18_attention_weight_min": -31.612773895263672,
      "activations/layer19_attention_weight_max": 24.262279510498047,
      "activations/layer19_attention_weight_min": -17.86872673034668,
      "activations/layer1_attention_weight_max": 17.262893676757812,
      "activations/layer1_attention_weight_min": -13.708687782287598,
      "activations/layer20_attention_weight_max": 20.551145553588867,
      "activations/layer20_attention_weight_min": -17.33366584777832,
      "activations/layer21_attention_weight_max": 41.61839294433594,
      "activations/layer21_attention_weight_min": -27.69535255432129,
      "activations/layer22_attention_weight_max": 44.96392059326172,
      "activations/layer22_attention_weight_min": -20.815715789794922,
      "activations/layer23_attention_weight_max": 24.84828758239746,
      "activations/layer23_attention_weight_min": -13.742517471313477,
      "activations/layer2_attention_weight_max": 30.60732650756836,
      "activations/layer2_attention_weight_min": -27.83694839477539,
      "activations/layer3_attention_weight_max": 67.44389343261719,
      "activations/layer3_attention_weight_min": -69.46420288085938,
      "activations/layer4_attention_weight_max": 102.9197998046875,
      "activations/layer4_attention_weight_min": -88.6123275756836,
      "activations/layer5_attention_weight_max": 129.31402587890625,
      "activations/layer5_attention_weight_min": -102.44474792480469,
      "activations/layer6_attention_weight_max": 65.08406829833984,
      "activations/layer6_attention_weight_min": -51.24073028564453,
      "activations/layer7_attention_weight_max": 59.34854507446289,
      "activations/layer7_attention_weight_min": -49.49555969238281,
      "activations/layer8_attention_weight_max": 47.51525115966797,
      "activations/layer8_attention_weight_min": -35.123313903808594,
      "activations/layer9_attention_weight_max": 65.6116714477539,
      "activations/layer9_attention_weight_min": -59.0357666015625,
      "epoch": 2.45,
      "learning_rate": 0.00013553181818181818,
      "loss": 2.949,
      "step": 42250
    },
    {
      "activations/layer0_attention_weight_max": 15.645776748657227,
      "activations/layer0_attention_weight_min": -14.04958724975586,
      "activations/layer10_attention_weight_max": 60.426361083984375,
      "activations/layer10_attention_weight_min": -42.57453155517578,
      "activations/layer11_attention_weight_max": 44.16801071166992,
      "activations/layer11_attention_weight_min": -32.78048324584961,
      "activations/layer12_attention_weight_max": 32.75533676147461,
      "activations/layer12_attention_weight_min": -31.080835342407227,
      "activations/layer13_attention_weight_max": 57.19483947753906,
      "activations/layer13_attention_weight_min": -38.5400505065918,
      "activations/layer14_attention_weight_max": 41.1405143737793,
      "activations/layer14_attention_weight_min": -29.194421768188477,
      "activations/layer15_attention_weight_max": 47.62568664550781,
      "activations/layer15_attention_weight_min": -33.798709869384766,
      "activations/layer16_attention_weight_max": 39.0549430847168,
      "activations/layer16_attention_weight_min": -25.47386360168457,
      "activations/layer17_attention_weight_max": 66.08432006835938,
      "activations/layer17_attention_weight_min": -36.47220993041992,
      "activations/layer18_attention_weight_max": 51.95927810668945,
      "activations/layer18_attention_weight_min": -31.70263671875,
      "activations/layer19_attention_weight_max": 21.62909507751465,
      "activations/layer19_attention_weight_min": -15.994552612304688,
      "activations/layer1_attention_weight_max": 18.43761444091797,
      "activations/layer1_attention_weight_min": -14.088521003723145,
      "activations/layer20_attention_weight_max": 18.37797737121582,
      "activations/layer20_attention_weight_min": -16.28577423095703,
      "activations/layer21_attention_weight_max": 35.85124588012695,
      "activations/layer21_attention_weight_min": -26.41227149963379,
      "activations/layer22_attention_weight_max": 31.969661712646484,
      "activations/layer22_attention_weight_min": -23.40856170654297,
      "activations/layer23_attention_weight_max": 24.90304946899414,
      "activations/layer23_attention_weight_min": -16.019760131835938,
      "activations/layer2_attention_weight_max": 28.4249324798584,
      "activations/layer2_attention_weight_min": -27.06259536743164,
      "activations/layer3_attention_weight_max": 67.79360961914062,
      "activations/layer3_attention_weight_min": -71.50991821289062,
      "activations/layer4_attention_weight_max": 99.52774047851562,
      "activations/layer4_attention_weight_min": -89.16911315917969,
      "activations/layer5_attention_weight_max": 136.25262451171875,
      "activations/layer5_attention_weight_min": -99.36204528808594,
      "activations/layer6_attention_weight_max": 68.39928436279297,
      "activations/layer6_attention_weight_min": -49.072837829589844,
      "activations/layer7_attention_weight_max": 64.69869995117188,
      "activations/layer7_attention_weight_min": -58.594181060791016,
      "activations/layer8_attention_weight_max": 54.23616409301758,
      "activations/layer8_attention_weight_min": -36.44169998168945,
      "activations/layer9_attention_weight_max": 70.2188949584961,
      "activations/layer9_attention_weight_min": -55.554718017578125,
      "epoch": 2.46,
      "learning_rate": 0.00013551287878787877,
      "loss": 2.9376,
      "step": 42300
    },
    {
      "activations/layer0_attention_weight_max": 15.958890914916992,
      "activations/layer0_attention_weight_min": -14.596402168273926,
      "activations/layer10_attention_weight_max": 58.07059097290039,
      "activations/layer10_attention_weight_min": -41.1021614074707,
      "activations/layer11_attention_weight_max": 43.7854118347168,
      "activations/layer11_attention_weight_min": -33.14338302612305,
      "activations/layer12_attention_weight_max": 30.908992767333984,
      "activations/layer12_attention_weight_min": -26.20342445373535,
      "activations/layer13_attention_weight_max": 74.60107421875,
      "activations/layer13_attention_weight_min": -44.20179748535156,
      "activations/layer14_attention_weight_max": 80.53741455078125,
      "activations/layer14_attention_weight_min": -48.66626739501953,
      "activations/layer15_attention_weight_max": 55.450904846191406,
      "activations/layer15_attention_weight_min": -36.186580657958984,
      "activations/layer16_attention_weight_max": 39.20926284790039,
      "activations/layer16_attention_weight_min": -29.193296432495117,
      "activations/layer17_attention_weight_max": 57.82529830932617,
      "activations/layer17_attention_weight_min": -37.58082962036133,
      "activations/layer18_attention_weight_max": 52.617401123046875,
      "activations/layer18_attention_weight_min": -32.62822723388672,
      "activations/layer19_attention_weight_max": 21.529848098754883,
      "activations/layer19_attention_weight_min": -15.7074613571167,
      "activations/layer1_attention_weight_max": 16.45808982849121,
      "activations/layer1_attention_weight_min": -13.40421199798584,
      "activations/layer20_attention_weight_max": 17.070035934448242,
      "activations/layer20_attention_weight_min": -18.60364532470703,
      "activations/layer21_attention_weight_max": 41.24630355834961,
      "activations/layer21_attention_weight_min": -22.54909896850586,
      "activations/layer22_attention_weight_max": 33.555233001708984,
      "activations/layer22_attention_weight_min": -21.51608657836914,
      "activations/layer23_attention_weight_max": 21.982999801635742,
      "activations/layer23_attention_weight_min": -14.723411560058594,
      "activations/layer2_attention_weight_max": 29.645627975463867,
      "activations/layer2_attention_weight_min": -26.837759017944336,
      "activations/layer3_attention_weight_max": 66.51161193847656,
      "activations/layer3_attention_weight_min": -69.5100326538086,
      "activations/layer4_attention_weight_max": 98.94750213623047,
      "activations/layer4_attention_weight_min": -86.49617767333984,
      "activations/layer5_attention_weight_max": 130.25489807128906,
      "activations/layer5_attention_weight_min": -103.681396484375,
      "activations/layer6_attention_weight_max": 61.93666076660156,
      "activations/layer6_attention_weight_min": -50.34013748168945,
      "activations/layer7_attention_weight_max": 64.4499740600586,
      "activations/layer7_attention_weight_min": -58.79170608520508,
      "activations/layer8_attention_weight_max": 49.25944137573242,
      "activations/layer8_attention_weight_min": -37.283836364746094,
      "activations/layer9_attention_weight_max": 71.91696166992188,
      "activations/layer9_attention_weight_min": -62.6106071472168,
      "epoch": 2.46,
      "learning_rate": 0.00013549393939393938,
      "loss": 2.9344,
      "step": 42350
    },
    {
      "activations/layer0_attention_weight_max": 16.780824661254883,
      "activations/layer0_attention_weight_min": -15.015015602111816,
      "activations/layer10_attention_weight_max": 53.40937042236328,
      "activations/layer10_attention_weight_min": -39.9249153137207,
      "activations/layer11_attention_weight_max": 41.29763412475586,
      "activations/layer11_attention_weight_min": -29.934852600097656,
      "activations/layer12_attention_weight_max": 31.201078414916992,
      "activations/layer12_attention_weight_min": -31.847902297973633,
      "activations/layer13_attention_weight_max": 54.74712371826172,
      "activations/layer13_attention_weight_min": -39.302337646484375,
      "activations/layer14_attention_weight_max": 48.80276107788086,
      "activations/layer14_attention_weight_min": -34.21586227416992,
      "activations/layer15_attention_weight_max": 52.72669219970703,
      "activations/layer15_attention_weight_min": -33.12834930419922,
      "activations/layer16_attention_weight_max": 40.534263610839844,
      "activations/layer16_attention_weight_min": -25.91802215576172,
      "activations/layer17_attention_weight_max": 59.47064208984375,
      "activations/layer17_attention_weight_min": -35.54841232299805,
      "activations/layer18_attention_weight_max": 53.85070037841797,
      "activations/layer18_attention_weight_min": -30.372953414916992,
      "activations/layer19_attention_weight_max": 23.838682174682617,
      "activations/layer19_attention_weight_min": -16.8301944732666,
      "activations/layer1_attention_weight_max": 17.61948585510254,
      "activations/layer1_attention_weight_min": -12.607736587524414,
      "activations/layer20_attention_weight_max": 21.446474075317383,
      "activations/layer20_attention_weight_min": -17.282320022583008,
      "activations/layer21_attention_weight_max": 39.57971954345703,
      "activations/layer21_attention_weight_min": -27.602928161621094,
      "activations/layer22_attention_weight_max": 39.37274932861328,
      "activations/layer22_attention_weight_min": -22.834672927856445,
      "activations/layer23_attention_weight_max": 28.29047966003418,
      "activations/layer23_attention_weight_min": -15.315569877624512,
      "activations/layer2_attention_weight_max": 27.209476470947266,
      "activations/layer2_attention_weight_min": -26.030141830444336,
      "activations/layer3_attention_weight_max": 62.76755905151367,
      "activations/layer3_attention_weight_min": -66.32657623291016,
      "activations/layer4_attention_weight_max": 98.747802734375,
      "activations/layer4_attention_weight_min": -84.36380767822266,
      "activations/layer5_attention_weight_max": 129.55438232421875,
      "activations/layer5_attention_weight_min": -99.99644470214844,
      "activations/layer6_attention_weight_max": 62.481048583984375,
      "activations/layer6_attention_weight_min": -49.43478775024414,
      "activations/layer7_attention_weight_max": 64.87013244628906,
      "activations/layer7_attention_weight_min": -53.25349807739258,
      "activations/layer8_attention_weight_max": 50.66475296020508,
      "activations/layer8_attention_weight_min": -35.8159065246582,
      "activations/layer9_attention_weight_max": 73.57632446289062,
      "activations/layer9_attention_weight_min": -54.82221603393555,
      "epoch": 2.46,
      "learning_rate": 0.000135475,
      "loss": 2.9363,
      "step": 42400
    },
    {
      "activations/layer0_attention_weight_max": 13.585437774658203,
      "activations/layer0_attention_weight_min": -14.367486000061035,
      "activations/layer10_attention_weight_max": 48.66794967651367,
      "activations/layer10_attention_weight_min": -40.011329650878906,
      "activations/layer11_attention_weight_max": 36.36851501464844,
      "activations/layer11_attention_weight_min": -30.19948387145996,
      "activations/layer12_attention_weight_max": 25.296321868896484,
      "activations/layer12_attention_weight_min": -21.137008666992188,
      "activations/layer13_attention_weight_max": 59.0886116027832,
      "activations/layer13_attention_weight_min": -49.35498809814453,
      "activations/layer14_attention_weight_max": 55.954261779785156,
      "activations/layer14_attention_weight_min": -42.5885009765625,
      "activations/layer15_attention_weight_max": 46.27426528930664,
      "activations/layer15_attention_weight_min": -34.366783142089844,
      "activations/layer16_attention_weight_max": 36.910587310791016,
      "activations/layer16_attention_weight_min": -27.356624603271484,
      "activations/layer17_attention_weight_max": 56.627662658691406,
      "activations/layer17_attention_weight_min": -35.223968505859375,
      "activations/layer18_attention_weight_max": 50.1978874206543,
      "activations/layer18_attention_weight_min": -33.94134521484375,
      "activations/layer19_attention_weight_max": 19.909425735473633,
      "activations/layer19_attention_weight_min": -16.600732803344727,
      "activations/layer1_attention_weight_max": 16.256303787231445,
      "activations/layer1_attention_weight_min": -14.990777969360352,
      "activations/layer20_attention_weight_max": 20.776334762573242,
      "activations/layer20_attention_weight_min": -19.349105834960938,
      "activations/layer21_attention_weight_max": 40.7296257019043,
      "activations/layer21_attention_weight_min": -27.94516944885254,
      "activations/layer22_attention_weight_max": 37.13533020019531,
      "activations/layer22_attention_weight_min": -25.641077041625977,
      "activations/layer23_attention_weight_max": 24.384021759033203,
      "activations/layer23_attention_weight_min": -16.362422943115234,
      "activations/layer2_attention_weight_max": 27.15547752380371,
      "activations/layer2_attention_weight_min": -26.830211639404297,
      "activations/layer3_attention_weight_max": 66.03913879394531,
      "activations/layer3_attention_weight_min": -69.52576446533203,
      "activations/layer4_attention_weight_max": 94.04910278320312,
      "activations/layer4_attention_weight_min": -83.96009063720703,
      "activations/layer5_attention_weight_max": 121.46354675292969,
      "activations/layer5_attention_weight_min": -98.45296478271484,
      "activations/layer6_attention_weight_max": 60.91010665893555,
      "activations/layer6_attention_weight_min": -51.22871398925781,
      "activations/layer7_attention_weight_max": 52.89595031738281,
      "activations/layer7_attention_weight_min": -53.080718994140625,
      "activations/layer8_attention_weight_max": 38.06694412231445,
      "activations/layer8_attention_weight_min": -34.37089157104492,
      "activations/layer9_attention_weight_max": 54.57925033569336,
      "activations/layer9_attention_weight_min": -50.57851028442383,
      "epoch": 2.47,
      "learning_rate": 0.00013545606060606061,
      "loss": 2.9281,
      "step": 42450
    },
    {
      "activations/layer0_attention_weight_max": 13.918909072875977,
      "activations/layer0_attention_weight_min": -14.81901741027832,
      "activations/layer10_attention_weight_max": 53.04331970214844,
      "activations/layer10_attention_weight_min": -42.12997055053711,
      "activations/layer11_attention_weight_max": 37.41486358642578,
      "activations/layer11_attention_weight_min": -30.021263122558594,
      "activations/layer12_attention_weight_max": 24.90981674194336,
      "activations/layer12_attention_weight_min": -25.126144409179688,
      "activations/layer13_attention_weight_max": 48.39105987548828,
      "activations/layer13_attention_weight_min": -40.24673080444336,
      "activations/layer14_attention_weight_max": 38.04793930053711,
      "activations/layer14_attention_weight_min": -31.464996337890625,
      "activations/layer15_attention_weight_max": 46.06412887573242,
      "activations/layer15_attention_weight_min": -32.26590347290039,
      "activations/layer16_attention_weight_max": 36.308624267578125,
      "activations/layer16_attention_weight_min": -27.25096321105957,
      "activations/layer17_attention_weight_max": 53.53071975708008,
      "activations/layer17_attention_weight_min": -36.75505447387695,
      "activations/layer18_attention_weight_max": 54.825504302978516,
      "activations/layer18_attention_weight_min": -29.36268424987793,
      "activations/layer19_attention_weight_max": 20.812429428100586,
      "activations/layer19_attention_weight_min": -16.090787887573242,
      "activations/layer1_attention_weight_max": 16.89113998413086,
      "activations/layer1_attention_weight_min": -14.154936790466309,
      "activations/layer20_attention_weight_max": 19.819091796875,
      "activations/layer20_attention_weight_min": -16.792829513549805,
      "activations/layer21_attention_weight_max": 41.44842529296875,
      "activations/layer21_attention_weight_min": -26.456310272216797,
      "activations/layer22_attention_weight_max": 33.32135009765625,
      "activations/layer22_attention_weight_min": -23.800317764282227,
      "activations/layer23_attention_weight_max": 24.451099395751953,
      "activations/layer23_attention_weight_min": -17.32705307006836,
      "activations/layer2_attention_weight_max": 28.29475212097168,
      "activations/layer2_attention_weight_min": -27.573707580566406,
      "activations/layer3_attention_weight_max": 64.1801986694336,
      "activations/layer3_attention_weight_min": -69.32897186279297,
      "activations/layer4_attention_weight_max": 90.20684051513672,
      "activations/layer4_attention_weight_min": -84.14339447021484,
      "activations/layer5_attention_weight_max": 115.6454086303711,
      "activations/layer5_attention_weight_min": -98.17388916015625,
      "activations/layer6_attention_weight_max": 60.681739807128906,
      "activations/layer6_attention_weight_min": -49.101341247558594,
      "activations/layer7_attention_weight_max": 56.103450775146484,
      "activations/layer7_attention_weight_min": -49.43693161010742,
      "activations/layer8_attention_weight_max": 42.36389923095703,
      "activations/layer8_attention_weight_min": -35.75448226928711,
      "activations/layer9_attention_weight_max": 56.439388275146484,
      "activations/layer9_attention_weight_min": -51.51096725463867,
      "epoch": 2.47,
      "learning_rate": 0.0001354371212121212,
      "loss": 2.9194,
      "step": 42500
    },
    {
      "activations/layer0_attention_weight_max": 14.923550605773926,
      "activations/layer0_attention_weight_min": -14.611271858215332,
      "activations/layer10_attention_weight_max": 50.93958282470703,
      "activations/layer10_attention_weight_min": -41.42487716674805,
      "activations/layer11_attention_weight_max": 43.09729766845703,
      "activations/layer11_attention_weight_min": -33.50898742675781,
      "activations/layer12_attention_weight_max": 26.475269317626953,
      "activations/layer12_attention_weight_min": -29.342618942260742,
      "activations/layer13_attention_weight_max": 74.55179595947266,
      "activations/layer13_attention_weight_min": -47.39359664916992,
      "activations/layer14_attention_weight_max": 58.83515548706055,
      "activations/layer14_attention_weight_min": -36.5814094543457,
      "activations/layer15_attention_weight_max": 52.84122085571289,
      "activations/layer15_attention_weight_min": -35.66318893432617,
      "activations/layer16_attention_weight_max": 42.68796157836914,
      "activations/layer16_attention_weight_min": -28.397310256958008,
      "activations/layer17_attention_weight_max": 68.11172485351562,
      "activations/layer17_attention_weight_min": -36.76280975341797,
      "activations/layer18_attention_weight_max": 58.343231201171875,
      "activations/layer18_attention_weight_min": -30.314815521240234,
      "activations/layer19_attention_weight_max": 23.740320205688477,
      "activations/layer19_attention_weight_min": -16.735380172729492,
      "activations/layer1_attention_weight_max": 16.24966049194336,
      "activations/layer1_attention_weight_min": -15.005420684814453,
      "activations/layer20_attention_weight_max": 21.15654754638672,
      "activations/layer20_attention_weight_min": -18.64937400817871,
      "activations/layer21_attention_weight_max": 45.9732780456543,
      "activations/layer21_attention_weight_min": -27.01393699645996,
      "activations/layer22_attention_weight_max": 41.113441467285156,
      "activations/layer22_attention_weight_min": -25.09080696105957,
      "activations/layer23_attention_weight_max": 27.665952682495117,
      "activations/layer23_attention_weight_min": -17.55382537841797,
      "activations/layer2_attention_weight_max": 29.10317611694336,
      "activations/layer2_attention_weight_min": -28.692028045654297,
      "activations/layer3_attention_weight_max": 70.1978988647461,
      "activations/layer3_attention_weight_min": -71.93111419677734,
      "activations/layer4_attention_weight_max": 100.97850036621094,
      "activations/layer4_attention_weight_min": -93.68181610107422,
      "activations/layer5_attention_weight_max": 120.62680053710938,
      "activations/layer5_attention_weight_min": -106.99652099609375,
      "activations/layer6_attention_weight_max": 64.18025207519531,
      "activations/layer6_attention_weight_min": -56.9775505065918,
      "activations/layer7_attention_weight_max": 62.074951171875,
      "activations/layer7_attention_weight_min": -55.961517333984375,
      "activations/layer8_attention_weight_max": 45.54011535644531,
      "activations/layer8_attention_weight_min": -37.28368377685547,
      "activations/layer9_attention_weight_max": 66.34469604492188,
      "activations/layer9_attention_weight_min": -53.67130661010742,
      "epoch": 2.47,
      "learning_rate": 0.00013541818181818182,
      "loss": 2.9382,
      "step": 42550
    },
    {
      "activations/layer0_attention_weight_max": 13.620508193969727,
      "activations/layer0_attention_weight_min": -14.956790924072266,
      "activations/layer10_attention_weight_max": 50.57512283325195,
      "activations/layer10_attention_weight_min": -41.8830680847168,
      "activations/layer11_attention_weight_max": 34.98847961425781,
      "activations/layer11_attention_weight_min": -30.87145233154297,
      "activations/layer12_attention_weight_max": 25.972883224487305,
      "activations/layer12_attention_weight_min": -25.94112777709961,
      "activations/layer13_attention_weight_max": 53.83631134033203,
      "activations/layer13_attention_weight_min": -40.40599060058594,
      "activations/layer14_attention_weight_max": 45.373390197753906,
      "activations/layer14_attention_weight_min": -35.811279296875,
      "activations/layer15_attention_weight_max": 52.27043533325195,
      "activations/layer15_attention_weight_min": -38.23707962036133,
      "activations/layer16_attention_weight_max": 39.199554443359375,
      "activations/layer16_attention_weight_min": -28.4654541015625,
      "activations/layer17_attention_weight_max": 60.67957305908203,
      "activations/layer17_attention_weight_min": -34.89081573486328,
      "activations/layer18_attention_weight_max": 54.37697219848633,
      "activations/layer18_attention_weight_min": -32.792015075683594,
      "activations/layer19_attention_weight_max": 21.34083366394043,
      "activations/layer19_attention_weight_min": -18.839506149291992,
      "activations/layer1_attention_weight_max": 17.126401901245117,
      "activations/layer1_attention_weight_min": -15.079334259033203,
      "activations/layer20_attention_weight_max": 20.33419418334961,
      "activations/layer20_attention_weight_min": -18.89586067199707,
      "activations/layer21_attention_weight_max": 41.11265182495117,
      "activations/layer21_attention_weight_min": -27.39805793762207,
      "activations/layer22_attention_weight_max": 34.77266311645508,
      "activations/layer22_attention_weight_min": -21.686260223388672,
      "activations/layer23_attention_weight_max": 23.573558807373047,
      "activations/layer23_attention_weight_min": -15.961685180664062,
      "activations/layer2_attention_weight_max": 28.34967803955078,
      "activations/layer2_attention_weight_min": -27.767230987548828,
      "activations/layer3_attention_weight_max": 68.163818359375,
      "activations/layer3_attention_weight_min": -68.47260284423828,
      "activations/layer4_attention_weight_max": 97.6361312866211,
      "activations/layer4_attention_weight_min": -88.74601745605469,
      "activations/layer5_attention_weight_max": 127.46536254882812,
      "activations/layer5_attention_weight_min": -110.86444091796875,
      "activations/layer6_attention_weight_max": 61.77558135986328,
      "activations/layer6_attention_weight_min": -54.27651596069336,
      "activations/layer7_attention_weight_max": 55.3574104309082,
      "activations/layer7_attention_weight_min": -54.954437255859375,
      "activations/layer8_attention_weight_max": 42.57720947265625,
      "activations/layer8_attention_weight_min": -37.34660720825195,
      "activations/layer9_attention_weight_max": 55.27754592895508,
      "activations/layer9_attention_weight_min": -53.05277633666992,
      "epoch": 2.48,
      "learning_rate": 0.0001353992424242424,
      "loss": 2.9445,
      "step": 42600
    },
    {
      "activations/layer0_attention_weight_max": 14.08536434173584,
      "activations/layer0_attention_weight_min": -14.726993560791016,
      "activations/layer10_attention_weight_max": 50.082366943359375,
      "activations/layer10_attention_weight_min": -41.19014358520508,
      "activations/layer11_attention_weight_max": 38.83489990234375,
      "activations/layer11_attention_weight_min": -29.724727630615234,
      "activations/layer12_attention_weight_max": 28.26835060119629,
      "activations/layer12_attention_weight_min": -24.550615310668945,
      "activations/layer13_attention_weight_max": 56.70048141479492,
      "activations/layer13_attention_weight_min": -41.245147705078125,
      "activations/layer14_attention_weight_max": 46.23661422729492,
      "activations/layer14_attention_weight_min": -32.895294189453125,
      "activations/layer15_attention_weight_max": 47.65388870239258,
      "activations/layer15_attention_weight_min": -35.30791091918945,
      "activations/layer16_attention_weight_max": 39.39065933227539,
      "activations/layer16_attention_weight_min": -29.545516967773438,
      "activations/layer17_attention_weight_max": 70.53353881835938,
      "activations/layer17_attention_weight_min": -38.61784744262695,
      "activations/layer18_attention_weight_max": 53.3293571472168,
      "activations/layer18_attention_weight_min": -33.26841354370117,
      "activations/layer19_attention_weight_max": 21.136730194091797,
      "activations/layer19_attention_weight_min": -16.48181915283203,
      "activations/layer1_attention_weight_max": 16.51259994506836,
      "activations/layer1_attention_weight_min": -13.775959014892578,
      "activations/layer20_attention_weight_max": 20.393810272216797,
      "activations/layer20_attention_weight_min": -22.752744674682617,
      "activations/layer21_attention_weight_max": 44.094696044921875,
      "activations/layer21_attention_weight_min": -30.211055755615234,
      "activations/layer22_attention_weight_max": 36.27250289916992,
      "activations/layer22_attention_weight_min": -24.549436569213867,
      "activations/layer23_attention_weight_max": 23.610004425048828,
      "activations/layer23_attention_weight_min": -16.708099365234375,
      "activations/layer2_attention_weight_max": 28.1414794921875,
      "activations/layer2_attention_weight_min": -26.06328582763672,
      "activations/layer3_attention_weight_max": 63.94162368774414,
      "activations/layer3_attention_weight_min": -64.67652130126953,
      "activations/layer4_attention_weight_max": 95.81797790527344,
      "activations/layer4_attention_weight_min": -83.2949447631836,
      "activations/layer5_attention_weight_max": 119.54804992675781,
      "activations/layer5_attention_weight_min": -99.27044677734375,
      "activations/layer6_attention_weight_max": 58.2098388671875,
      "activations/layer6_attention_weight_min": -48.900779724121094,
      "activations/layer7_attention_weight_max": 55.04671096801758,
      "activations/layer7_attention_weight_min": -54.43732833862305,
      "activations/layer8_attention_weight_max": 42.31936264038086,
      "activations/layer8_attention_weight_min": -36.690433502197266,
      "activations/layer9_attention_weight_max": 59.25344467163086,
      "activations/layer9_attention_weight_min": -52.89595413208008,
      "epoch": 2.48,
      "learning_rate": 0.00013538030303030302,
      "loss": 2.9425,
      "step": 42650
    },
    {
      "activations/layer0_attention_weight_max": 14.575846672058105,
      "activations/layer0_attention_weight_min": -14.916024208068848,
      "activations/layer10_attention_weight_max": 57.73907470703125,
      "activations/layer10_attention_weight_min": -43.09072494506836,
      "activations/layer11_attention_weight_max": 45.251312255859375,
      "activations/layer11_attention_weight_min": -34.338768005371094,
      "activations/layer12_attention_weight_max": 31.429166793823242,
      "activations/layer12_attention_weight_min": -25.718542098999023,
      "activations/layer13_attention_weight_max": 66.62290954589844,
      "activations/layer13_attention_weight_min": -44.439327239990234,
      "activations/layer14_attention_weight_max": 71.47914123535156,
      "activations/layer14_attention_weight_min": -42.526710510253906,
      "activations/layer15_attention_weight_max": 51.824676513671875,
      "activations/layer15_attention_weight_min": -33.02475357055664,
      "activations/layer16_attention_weight_max": 34.197452545166016,
      "activations/layer16_attention_weight_min": -29.196958541870117,
      "activations/layer17_attention_weight_max": 62.12889862060547,
      "activations/layer17_attention_weight_min": -35.658267974853516,
      "activations/layer18_attention_weight_max": 50.78349685668945,
      "activations/layer18_attention_weight_min": -34.16975784301758,
      "activations/layer19_attention_weight_max": 21.989498138427734,
      "activations/layer19_attention_weight_min": -17.85406494140625,
      "activations/layer1_attention_weight_max": 17.942399978637695,
      "activations/layer1_attention_weight_min": -15.300904273986816,
      "activations/layer20_attention_weight_max": 21.929515838623047,
      "activations/layer20_attention_weight_min": -18.014286041259766,
      "activations/layer21_attention_weight_max": 40.006866455078125,
      "activations/layer21_attention_weight_min": -28.51822280883789,
      "activations/layer22_attention_weight_max": 36.028289794921875,
      "activations/layer22_attention_weight_min": -22.389726638793945,
      "activations/layer23_attention_weight_max": 23.27450180053711,
      "activations/layer23_attention_weight_min": -16.67901611328125,
      "activations/layer2_attention_weight_max": 29.131168365478516,
      "activations/layer2_attention_weight_min": -27.38652801513672,
      "activations/layer3_attention_weight_max": 68.74866485595703,
      "activations/layer3_attention_weight_min": -70.83155822753906,
      "activations/layer4_attention_weight_max": 97.22918701171875,
      "activations/layer4_attention_weight_min": -87.06615447998047,
      "activations/layer5_attention_weight_max": 125.9105224609375,
      "activations/layer5_attention_weight_min": -103.76558685302734,
      "activations/layer6_attention_weight_max": 63.1122932434082,
      "activations/layer6_attention_weight_min": -50.612770080566406,
      "activations/layer7_attention_weight_max": 64.7844009399414,
      "activations/layer7_attention_weight_min": -54.166255950927734,
      "activations/layer8_attention_weight_max": 48.0450325012207,
      "activations/layer8_attention_weight_min": -38.578651428222656,
      "activations/layer9_attention_weight_max": 64.0766372680664,
      "activations/layer9_attention_weight_min": -56.300228118896484,
      "epoch": 2.48,
      "learning_rate": 0.00013536136363636364,
      "loss": 2.9376,
      "step": 42700
    },
    {
      "activations/layer0_attention_weight_max": 14.21947956085205,
      "activations/layer0_attention_weight_min": -14.869636535644531,
      "activations/layer10_attention_weight_max": 53.7010498046875,
      "activations/layer10_attention_weight_min": -41.41717529296875,
      "activations/layer11_attention_weight_max": 38.547386169433594,
      "activations/layer11_attention_weight_min": -30.57244110107422,
      "activations/layer12_attention_weight_max": 31.08082389831543,
      "activations/layer12_attention_weight_min": -25.488216400146484,
      "activations/layer13_attention_weight_max": 67.95835876464844,
      "activations/layer13_attention_weight_min": -48.361080169677734,
      "activations/layer14_attention_weight_max": 62.595333099365234,
      "activations/layer14_attention_weight_min": -38.337833404541016,
      "activations/layer15_attention_weight_max": 52.779239654541016,
      "activations/layer15_attention_weight_min": -36.75526428222656,
      "activations/layer16_attention_weight_max": 49.90768814086914,
      "activations/layer16_attention_weight_min": -30.827733993530273,
      "activations/layer17_attention_weight_max": 59.46657943725586,
      "activations/layer17_attention_weight_min": -36.421260833740234,
      "activations/layer18_attention_weight_max": 56.08967590332031,
      "activations/layer18_attention_weight_min": -29.848575592041016,
      "activations/layer19_attention_weight_max": 20.39848518371582,
      "activations/layer19_attention_weight_min": -16.12148094177246,
      "activations/layer1_attention_weight_max": 17.043699264526367,
      "activations/layer1_attention_weight_min": -12.893052101135254,
      "activations/layer20_attention_weight_max": 22.18951988220215,
      "activations/layer20_attention_weight_min": -15.969648361206055,
      "activations/layer21_attention_weight_max": 43.88797378540039,
      "activations/layer21_attention_weight_min": -25.467355728149414,
      "activations/layer22_attention_weight_max": 42.04158401489258,
      "activations/layer22_attention_weight_min": -23.55072593688965,
      "activations/layer23_attention_weight_max": 27.301061630249023,
      "activations/layer23_attention_weight_min": -15.07552719116211,
      "activations/layer2_attention_weight_max": 26.63311004638672,
      "activations/layer2_attention_weight_min": -25.904335021972656,
      "activations/layer3_attention_weight_max": 68.10575866699219,
      "activations/layer3_attention_weight_min": -68.68389129638672,
      "activations/layer4_attention_weight_max": 102.47865295410156,
      "activations/layer4_attention_weight_min": -88.90486907958984,
      "activations/layer5_attention_weight_max": 130.20794677734375,
      "activations/layer5_attention_weight_min": -105.77513885498047,
      "activations/layer6_attention_weight_max": 64.72251892089844,
      "activations/layer6_attention_weight_min": -54.81315231323242,
      "activations/layer7_attention_weight_max": 59.87333679199219,
      "activations/layer7_attention_weight_min": -55.28592300415039,
      "activations/layer8_attention_weight_max": 42.92570495605469,
      "activations/layer8_attention_weight_min": -36.55584716796875,
      "activations/layer9_attention_weight_max": 57.516448974609375,
      "activations/layer9_attention_weight_min": -49.68223571777344,
      "epoch": 2.48,
      "learning_rate": 0.00013534242424242422,
      "loss": 2.9421,
      "step": 42750
    },
    {
      "activations/layer0_attention_weight_max": 14.401152610778809,
      "activations/layer0_attention_weight_min": -14.984715461730957,
      "activations/layer10_attention_weight_max": 49.0059928894043,
      "activations/layer10_attention_weight_min": -40.946815490722656,
      "activations/layer11_attention_weight_max": 39.793880462646484,
      "activations/layer11_attention_weight_min": -30.7659969329834,
      "activations/layer12_attention_weight_max": 28.355321884155273,
      "activations/layer12_attention_weight_min": -24.391191482543945,
      "activations/layer13_attention_weight_max": 51.35297393798828,
      "activations/layer13_attention_weight_min": -39.68164825439453,
      "activations/layer14_attention_weight_max": 53.570499420166016,
      "activations/layer14_attention_weight_min": -40.03710174560547,
      "activations/layer15_attention_weight_max": 47.73276901245117,
      "activations/layer15_attention_weight_min": -36.15926742553711,
      "activations/layer16_attention_weight_max": 37.88011932373047,
      "activations/layer16_attention_weight_min": -28.240983963012695,
      "activations/layer17_attention_weight_max": 62.9589729309082,
      "activations/layer17_attention_weight_min": -37.61498260498047,
      "activations/layer18_attention_weight_max": 57.8444938659668,
      "activations/layer18_attention_weight_min": -32.74617385864258,
      "activations/layer19_attention_weight_max": 19.270814895629883,
      "activations/layer19_attention_weight_min": -21.371946334838867,
      "activations/layer1_attention_weight_max": 17.265897750854492,
      "activations/layer1_attention_weight_min": -14.579965591430664,
      "activations/layer20_attention_weight_max": 22.586837768554688,
      "activations/layer20_attention_weight_min": -17.590282440185547,
      "activations/layer21_attention_weight_max": 42.10745620727539,
      "activations/layer21_attention_weight_min": -25.9856014251709,
      "activations/layer22_attention_weight_max": 35.50286865234375,
      "activations/layer22_attention_weight_min": -22.25223159790039,
      "activations/layer23_attention_weight_max": 24.28227996826172,
      "activations/layer23_attention_weight_min": -16.023605346679688,
      "activations/layer2_attention_weight_max": 27.489261627197266,
      "activations/layer2_attention_weight_min": -27.75537872314453,
      "activations/layer3_attention_weight_max": 64.24921417236328,
      "activations/layer3_attention_weight_min": -70.8917007446289,
      "activations/layer4_attention_weight_max": 93.08837127685547,
      "activations/layer4_attention_weight_min": -89.45394134521484,
      "activations/layer5_attention_weight_max": 120.27058410644531,
      "activations/layer5_attention_weight_min": -103.90166473388672,
      "activations/layer6_attention_weight_max": 62.95758056640625,
      "activations/layer6_attention_weight_min": -51.46670913696289,
      "activations/layer7_attention_weight_max": 54.46894073486328,
      "activations/layer7_attention_weight_min": -52.820919036865234,
      "activations/layer8_attention_weight_max": 41.09375,
      "activations/layer8_attention_weight_min": -34.86818313598633,
      "activations/layer9_attention_weight_max": 52.209815979003906,
      "activations/layer9_attention_weight_min": -49.099483489990234,
      "epoch": 2.49,
      "learning_rate": 0.00013532348484848484,
      "loss": 2.9424,
      "step": 42800
    },
    {
      "activations/layer0_attention_weight_max": 15.548282623291016,
      "activations/layer0_attention_weight_min": -14.204667091369629,
      "activations/layer10_attention_weight_max": 47.634788513183594,
      "activations/layer10_attention_weight_min": -42.11311721801758,
      "activations/layer11_attention_weight_max": 36.89982604980469,
      "activations/layer11_attention_weight_min": -29.622602462768555,
      "activations/layer12_attention_weight_max": 26.22738265991211,
      "activations/layer12_attention_weight_min": -25.001480102539062,
      "activations/layer13_attention_weight_max": 59.06198501586914,
      "activations/layer13_attention_weight_min": -43.08816909790039,
      "activations/layer14_attention_weight_max": 55.93394088745117,
      "activations/layer14_attention_weight_min": -36.0417594909668,
      "activations/layer15_attention_weight_max": 47.510860443115234,
      "activations/layer15_attention_weight_min": -35.63850784301758,
      "activations/layer16_attention_weight_max": 38.596736907958984,
      "activations/layer16_attention_weight_min": -29.254911422729492,
      "activations/layer17_attention_weight_max": 61.7916259765625,
      "activations/layer17_attention_weight_min": -36.48670196533203,
      "activations/layer18_attention_weight_max": 56.501399993896484,
      "activations/layer18_attention_weight_min": -32.86732864379883,
      "activations/layer19_attention_weight_max": 21.7561092376709,
      "activations/layer19_attention_weight_min": -17.246110916137695,
      "activations/layer1_attention_weight_max": 17.12767791748047,
      "activations/layer1_attention_weight_min": -15.13722038269043,
      "activations/layer20_attention_weight_max": 22.307270050048828,
      "activations/layer20_attention_weight_min": -20.20710563659668,
      "activations/layer21_attention_weight_max": 48.186641693115234,
      "activations/layer21_attention_weight_min": -25.991439819335938,
      "activations/layer22_attention_weight_max": 35.813602447509766,
      "activations/layer22_attention_weight_min": -26.97407341003418,
      "activations/layer23_attention_weight_max": 25.73361587524414,
      "activations/layer23_attention_weight_min": -18.8963623046875,
      "activations/layer2_attention_weight_max": 29.53547477722168,
      "activations/layer2_attention_weight_min": -26.783676147460938,
      "activations/layer3_attention_weight_max": 66.4345474243164,
      "activations/layer3_attention_weight_min": -68.45901489257812,
      "activations/layer4_attention_weight_max": 93.50420379638672,
      "activations/layer4_attention_weight_min": -82.92167663574219,
      "activations/layer5_attention_weight_max": 117.62896728515625,
      "activations/layer5_attention_weight_min": -101.85792541503906,
      "activations/layer6_attention_weight_max": 60.661888122558594,
      "activations/layer6_attention_weight_min": -53.684608459472656,
      "activations/layer7_attention_weight_max": 55.394065856933594,
      "activations/layer7_attention_weight_min": -54.46763229370117,
      "activations/layer8_attention_weight_max": 39.58449172973633,
      "activations/layer8_attention_weight_min": -37.2588005065918,
      "activations/layer9_attention_weight_max": 53.47689437866211,
      "activations/layer9_attention_weight_min": -51.91769027709961,
      "epoch": 2.49,
      "learning_rate": 0.00013530454545454545,
      "loss": 2.9421,
      "step": 42850
    },
    {
      "activations/layer0_attention_weight_max": 13.204707145690918,
      "activations/layer0_attention_weight_min": -14.8353910446167,
      "activations/layer10_attention_weight_max": 55.408111572265625,
      "activations/layer10_attention_weight_min": -46.088661193847656,
      "activations/layer11_attention_weight_max": 43.19171905517578,
      "activations/layer11_attention_weight_min": -30.955093383789062,
      "activations/layer12_attention_weight_max": 29.095407485961914,
      "activations/layer12_attention_weight_min": -22.66473960876465,
      "activations/layer13_attention_weight_max": 57.55649185180664,
      "activations/layer13_attention_weight_min": -45.037357330322266,
      "activations/layer14_attention_weight_max": 51.231964111328125,
      "activations/layer14_attention_weight_min": -34.8459358215332,
      "activations/layer15_attention_weight_max": 54.750484466552734,
      "activations/layer15_attention_weight_min": -36.77071762084961,
      "activations/layer16_attention_weight_max": 43.377479553222656,
      "activations/layer16_attention_weight_min": -28.032569885253906,
      "activations/layer17_attention_weight_max": 61.91572570800781,
      "activations/layer17_attention_weight_min": -36.2322883605957,
      "activations/layer18_attention_weight_max": 58.338050842285156,
      "activations/layer18_attention_weight_min": -32.01405715942383,
      "activations/layer19_attention_weight_max": 22.20989990234375,
      "activations/layer19_attention_weight_min": -15.737497329711914,
      "activations/layer1_attention_weight_max": 17.726125717163086,
      "activations/layer1_attention_weight_min": -15.738886833190918,
      "activations/layer20_attention_weight_max": 20.942874908447266,
      "activations/layer20_attention_weight_min": -20.842012405395508,
      "activations/layer21_attention_weight_max": 44.508880615234375,
      "activations/layer21_attention_weight_min": -26.295486450195312,
      "activations/layer22_attention_weight_max": 34.68087387084961,
      "activations/layer22_attention_weight_min": -25.777347564697266,
      "activations/layer23_attention_weight_max": 26.090011596679688,
      "activations/layer23_attention_weight_min": -14.03182601928711,
      "activations/layer2_attention_weight_max": 28.10125732421875,
      "activations/layer2_attention_weight_min": -26.63571548461914,
      "activations/layer3_attention_weight_max": 66.70359802246094,
      "activations/layer3_attention_weight_min": -71.36656188964844,
      "activations/layer4_attention_weight_max": 100.77806854248047,
      "activations/layer4_attention_weight_min": -88.61839294433594,
      "activations/layer5_attention_weight_max": 125.37066650390625,
      "activations/layer5_attention_weight_min": -102.3004379272461,
      "activations/layer6_attention_weight_max": 59.8873291015625,
      "activations/layer6_attention_weight_min": -54.21669387817383,
      "activations/layer7_attention_weight_max": 57.99688720703125,
      "activations/layer7_attention_weight_min": -52.39211654663086,
      "activations/layer8_attention_weight_max": 43.606746673583984,
      "activations/layer8_attention_weight_min": -37.500701904296875,
      "activations/layer9_attention_weight_max": 60.168697357177734,
      "activations/layer9_attention_weight_min": -49.762325286865234,
      "epoch": 2.49,
      "learning_rate": 0.00013528560606060604,
      "loss": 2.9342,
      "step": 42900
    },
    {
      "activations/layer0_attention_weight_max": 15.776666641235352,
      "activations/layer0_attention_weight_min": -14.501164436340332,
      "activations/layer10_attention_weight_max": 54.88194274902344,
      "activations/layer10_attention_weight_min": -41.76066207885742,
      "activations/layer11_attention_weight_max": 44.17863464355469,
      "activations/layer11_attention_weight_min": -34.699806213378906,
      "activations/layer12_attention_weight_max": 28.82221221923828,
      "activations/layer12_attention_weight_min": -29.017433166503906,
      "activations/layer13_attention_weight_max": 69.44775390625,
      "activations/layer13_attention_weight_min": -43.827735900878906,
      "activations/layer14_attention_weight_max": 67.13968658447266,
      "activations/layer14_attention_weight_min": -42.64385986328125,
      "activations/layer15_attention_weight_max": 55.410308837890625,
      "activations/layer15_attention_weight_min": -36.760318756103516,
      "activations/layer16_attention_weight_max": 39.7939567565918,
      "activations/layer16_attention_weight_min": -27.329713821411133,
      "activations/layer17_attention_weight_max": 69.66061401367188,
      "activations/layer17_attention_weight_min": -37.36967086791992,
      "activations/layer18_attention_weight_max": 67.7610092163086,
      "activations/layer18_attention_weight_min": -32.59064865112305,
      "activations/layer19_attention_weight_max": 24.46074867248535,
      "activations/layer19_attention_weight_min": -18.311201095581055,
      "activations/layer1_attention_weight_max": 17.048290252685547,
      "activations/layer1_attention_weight_min": -13.93034839630127,
      "activations/layer20_attention_weight_max": 25.237333297729492,
      "activations/layer20_attention_weight_min": -20.00724983215332,
      "activations/layer21_attention_weight_max": 57.78335952758789,
      "activations/layer21_attention_weight_min": -33.51081085205078,
      "activations/layer22_attention_weight_max": 38.78422164916992,
      "activations/layer22_attention_weight_min": -25.42444610595703,
      "activations/layer23_attention_weight_max": 26.7822265625,
      "activations/layer23_attention_weight_min": -15.890090942382812,
      "activations/layer2_attention_weight_max": 28.2321834564209,
      "activations/layer2_attention_weight_min": -25.59895133972168,
      "activations/layer3_attention_weight_max": 64.39396667480469,
      "activations/layer3_attention_weight_min": -67.9815902709961,
      "activations/layer4_attention_weight_max": 95.55602264404297,
      "activations/layer4_attention_weight_min": -84.43973541259766,
      "activations/layer5_attention_weight_max": 128.884521484375,
      "activations/layer5_attention_weight_min": -103.45072937011719,
      "activations/layer6_attention_weight_max": 62.50288009643555,
      "activations/layer6_attention_weight_min": -52.72193908691406,
      "activations/layer7_attention_weight_max": 65.81603240966797,
      "activations/layer7_attention_weight_min": -58.50020217895508,
      "activations/layer8_attention_weight_max": 52.620323181152344,
      "activations/layer8_attention_weight_min": -40.44682312011719,
      "activations/layer9_attention_weight_max": 73.45494842529297,
      "activations/layer9_attention_weight_min": -64.88875579833984,
      "epoch": 2.5,
      "learning_rate": 0.00013526666666666666,
      "loss": 2.9483,
      "step": 42950
    },
    {
      "activations/layer0_attention_weight_max": 14.987070083618164,
      "activations/layer0_attention_weight_min": -14.673035621643066,
      "activations/layer10_attention_weight_max": 46.7631721496582,
      "activations/layer10_attention_weight_min": -41.28963088989258,
      "activations/layer11_attention_weight_max": 36.69945526123047,
      "activations/layer11_attention_weight_min": -30.060331344604492,
      "activations/layer12_attention_weight_max": 23.816431045532227,
      "activations/layer12_attention_weight_min": -23.677343368530273,
      "activations/layer13_attention_weight_max": 53.74860382080078,
      "activations/layer13_attention_weight_min": -40.213157653808594,
      "activations/layer14_attention_weight_max": 45.260807037353516,
      "activations/layer14_attention_weight_min": -33.174259185791016,
      "activations/layer15_attention_weight_max": 48.30905532836914,
      "activations/layer15_attention_weight_min": -34.54425811767578,
      "activations/layer16_attention_weight_max": 40.96080780029297,
      "activations/layer16_attention_weight_min": -28.265247344970703,
      "activations/layer17_attention_weight_max": 65.69007110595703,
      "activations/layer17_attention_weight_min": -36.35127639770508,
      "activations/layer18_attention_weight_max": 57.9703369140625,
      "activations/layer18_attention_weight_min": -32.148101806640625,
      "activations/layer19_attention_weight_max": 22.274255752563477,
      "activations/layer19_attention_weight_min": -19.415508270263672,
      "activations/layer1_attention_weight_max": 16.873023986816406,
      "activations/layer1_attention_weight_min": -15.401717185974121,
      "activations/layer20_attention_weight_max": 21.972414016723633,
      "activations/layer20_attention_weight_min": -18.557205200195312,
      "activations/layer21_attention_weight_max": 42.11812973022461,
      "activations/layer21_attention_weight_min": -26.286954879760742,
      "activations/layer22_attention_weight_max": 36.0267219543457,
      "activations/layer22_attention_weight_min": -22.071557998657227,
      "activations/layer23_attention_weight_max": 24.66522789001465,
      "activations/layer23_attention_weight_min": -16.36712074279785,
      "activations/layer2_attention_weight_max": 28.76088523864746,
      "activations/layer2_attention_weight_min": -26.409589767456055,
      "activations/layer3_attention_weight_max": 68.24617767333984,
      "activations/layer3_attention_weight_min": -73.18085479736328,
      "activations/layer4_attention_weight_max": 96.8951416015625,
      "activations/layer4_attention_weight_min": -89.86551666259766,
      "activations/layer5_attention_weight_max": 121.83440399169922,
      "activations/layer5_attention_weight_min": -102.94952392578125,
      "activations/layer6_attention_weight_max": 58.79890060424805,
      "activations/layer6_attention_weight_min": -51.39925003051758,
      "activations/layer7_attention_weight_max": 55.20372772216797,
      "activations/layer7_attention_weight_min": -47.24530029296875,
      "activations/layer8_attention_weight_max": 42.71729278564453,
      "activations/layer8_attention_weight_min": -34.792091369628906,
      "activations/layer9_attention_weight_max": 55.0643310546875,
      "activations/layer9_attention_weight_min": -52.01375961303711,
      "epoch": 2.5,
      "learning_rate": 0.00013524772727272727,
      "loss": 2.9366,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_loss": 2.8828125,
      "eval_runtime": 8.4829,
      "eval_samples_per_second": 506.195,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_openwebtext_loss": 2.8828125,
      "eval_openwebtext_ppl": 17.864446345871784,
      "eval_openwebtext_runtime": 8.4829,
      "eval_openwebtext_samples_per_second": 506.195,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_wikitext_loss": 3.138671875,
      "eval_wikitext_ppl": 23.073202403172917,
      "eval_wikitext_runtime": 1.9588,
      "eval_wikitext_samples_per_second": 232.8,
      "step": 43000
    },
    {
      "epoch": 2.5,
      "eval_lambada_loss": 2.990234375,
      "eval_lambada_ppl": 19.89034374461794,
      "eval_lambada_runtime": 9.5543,
      "eval_lambada_samples_per_second": 509.614,
      "step": 43000
    },
    {
      "activations/layer0_attention_weight_max": 14.98271369934082,
      "activations/layer0_attention_weight_min": -14.515386581420898,
      "activations/layer10_attention_weight_max": 48.506126403808594,
      "activations/layer10_attention_weight_min": -41.0102424621582,
      "activations/layer11_attention_weight_max": 39.210044860839844,
      "activations/layer11_attention_weight_min": -31.339576721191406,
      "activations/layer12_attention_weight_max": 32.53141403198242,
      "activations/layer12_attention_weight_min": -23.760786056518555,
      "activations/layer13_attention_weight_max": 63.20706558227539,
      "activations/layer13_attention_weight_min": -46.96992874145508,
      "activations/layer14_attention_weight_max": 49.862709045410156,
      "activations/layer14_attention_weight_min": -36.2835693359375,
      "activations/layer15_attention_weight_max": 49.67445373535156,
      "activations/layer15_attention_weight_min": -36.528987884521484,
      "activations/layer16_attention_weight_max": 38.646949768066406,
      "activations/layer16_attention_weight_min": -27.744279861450195,
      "activations/layer17_attention_weight_max": 62.80753707885742,
      "activations/layer17_attention_weight_min": -38.45956039428711,
      "activations/layer18_attention_weight_max": 55.19949722290039,
      "activations/layer18_attention_weight_min": -34.655879974365234,
      "activations/layer19_attention_weight_max": 22.090085983276367,
      "activations/layer19_attention_weight_min": -18.67995262145996,
      "activations/layer1_attention_weight_max": 16.50278091430664,
      "activations/layer1_attention_weight_min": -14.41100025177002,
      "activations/layer20_attention_weight_max": 21.59424591064453,
      "activations/layer20_attention_weight_min": -19.782529830932617,
      "activations/layer21_attention_weight_max": 45.982784271240234,
      "activations/layer21_attention_weight_min": -29.098331451416016,
      "activations/layer22_attention_weight_max": 35.811439514160156,
      "activations/layer22_attention_weight_min": -22.41389274597168,
      "activations/layer23_attention_weight_max": 22.08965301513672,
      "activations/layer23_attention_weight_min": -14.778755187988281,
      "activations/layer2_attention_weight_max": 29.138858795166016,
      "activations/layer2_attention_weight_min": -27.138578414916992,
      "activations/layer3_attention_weight_max": 65.95185089111328,
      "activations/layer3_attention_weight_min": -70.49490356445312,
      "activations/layer4_attention_weight_max": 98.28240203857422,
      "activations/layer4_attention_weight_min": -87.88298034667969,
      "activations/layer5_attention_weight_max": 122.5958251953125,
      "activations/layer5_attention_weight_min": -104.12034606933594,
      "activations/layer6_attention_weight_max": 61.15901565551758,
      "activations/layer6_attention_weight_min": -56.2264404296875,
      "activations/layer7_attention_weight_max": 53.03033447265625,
      "activations/layer7_attention_weight_min": -54.104766845703125,
      "activations/layer8_attention_weight_max": 41.45590591430664,
      "activations/layer8_attention_weight_min": -39.3304328918457,
      "activations/layer9_attention_weight_max": 51.040618896484375,
      "activations/layer9_attention_weight_min": -52.21120071411133,
      "epoch": 2.5,
      "learning_rate": 0.00013522878787878786,
      "loss": 2.9242,
      "step": 43050
    },
    {
      "activations/layer0_attention_weight_max": 13.69118595123291,
      "activations/layer0_attention_weight_min": -14.714825630187988,
      "activations/layer10_attention_weight_max": 52.782249450683594,
      "activations/layer10_attention_weight_min": -38.23714828491211,
      "activations/layer11_attention_weight_max": 40.342979431152344,
      "activations/layer11_attention_weight_min": -29.38062858581543,
      "activations/layer12_attention_weight_max": 29.178937911987305,
      "activations/layer12_attention_weight_min": -22.271625518798828,
      "activations/layer13_attention_weight_max": 58.11336135864258,
      "activations/layer13_attention_weight_min": -44.30119705200195,
      "activations/layer14_attention_weight_max": 59.49711227416992,
      "activations/layer14_attention_weight_min": -42.608158111572266,
      "activations/layer15_attention_weight_max": 51.51346206665039,
      "activations/layer15_attention_weight_min": -37.47391128540039,
      "activations/layer16_attention_weight_max": 38.09518051147461,
      "activations/layer16_attention_weight_min": -27.09129524230957,
      "activations/layer17_attention_weight_max": 60.43796157836914,
      "activations/layer17_attention_weight_min": -37.55534362792969,
      "activations/layer18_attention_weight_max": 55.04873275756836,
      "activations/layer18_attention_weight_min": -32.649749755859375,
      "activations/layer19_attention_weight_max": 22.46653938293457,
      "activations/layer19_attention_weight_min": -16.774850845336914,
      "activations/layer1_attention_weight_max": 18.083850860595703,
      "activations/layer1_attention_weight_min": -15.4987211227417,
      "activations/layer20_attention_weight_max": 20.703510284423828,
      "activations/layer20_attention_weight_min": -19.818870544433594,
      "activations/layer21_attention_weight_max": 41.834510803222656,
      "activations/layer21_attention_weight_min": -29.379669189453125,
      "activations/layer22_attention_weight_max": 38.17403030395508,
      "activations/layer22_attention_weight_min": -23.65658950805664,
      "activations/layer23_attention_weight_max": 25.246185302734375,
      "activations/layer23_attention_weight_min": -18.52271842956543,
      "activations/layer2_attention_weight_max": 27.31652069091797,
      "activations/layer2_attention_weight_min": -26.133527755737305,
      "activations/layer3_attention_weight_max": 65.39234924316406,
      "activations/layer3_attention_weight_min": -69.6861572265625,
      "activations/layer4_attention_weight_max": 96.30750274658203,
      "activations/layer4_attention_weight_min": -87.51200103759766,
      "activations/layer5_attention_weight_max": 130.3726806640625,
      "activations/layer5_attention_weight_min": -102.53047180175781,
      "activations/layer6_attention_weight_max": 64.99122619628906,
      "activations/layer6_attention_weight_min": -54.06080627441406,
      "activations/layer7_attention_weight_max": 58.32430648803711,
      "activations/layer7_attention_weight_min": -51.56179428100586,
      "activations/layer8_attention_weight_max": 43.39714431762695,
      "activations/layer8_attention_weight_min": -37.81868362426758,
      "activations/layer9_attention_weight_max": 52.4308967590332,
      "activations/layer9_attention_weight_min": -49.547698974609375,
      "epoch": 2.5,
      "learning_rate": 0.00013520984848484847,
      "loss": 2.9386,
      "step": 43100
    },
    {
      "activations/layer0_attention_weight_max": 14.202495574951172,
      "activations/layer0_attention_weight_min": -14.670151710510254,
      "activations/layer10_attention_weight_max": 48.196434020996094,
      "activations/layer10_attention_weight_min": -41.38946533203125,
      "activations/layer11_attention_weight_max": 36.30955123901367,
      "activations/layer11_attention_weight_min": -29.603702545166016,
      "activations/layer12_attention_weight_max": 29.26590919494629,
      "activations/layer12_attention_weight_min": -26.72673225402832,
      "activations/layer13_attention_weight_max": 63.92229080200195,
      "activations/layer13_attention_weight_min": -45.45524597167969,
      "activations/layer14_attention_weight_max": 46.27101516723633,
      "activations/layer14_attention_weight_min": -32.060585021972656,
      "activations/layer15_attention_weight_max": 56.584617614746094,
      "activations/layer15_attention_weight_min": -38.488155364990234,
      "activations/layer16_attention_weight_max": 37.62982940673828,
      "activations/layer16_attention_weight_min": -26.81511878967285,
      "activations/layer17_attention_weight_max": 60.5878791809082,
      "activations/layer17_attention_weight_min": -36.80636215209961,
      "activations/layer18_attention_weight_max": 52.44278335571289,
      "activations/layer18_attention_weight_min": -29.632902145385742,
      "activations/layer19_attention_weight_max": 24.534090042114258,
      "activations/layer19_attention_weight_min": -19.41231346130371,
      "activations/layer1_attention_weight_max": 17.912500381469727,
      "activations/layer1_attention_weight_min": -13.668734550476074,
      "activations/layer20_attention_weight_max": 20.09807586669922,
      "activations/layer20_attention_weight_min": -19.129335403442383,
      "activations/layer21_attention_weight_max": 41.45743179321289,
      "activations/layer21_attention_weight_min": -27.272811889648438,
      "activations/layer22_attention_weight_max": 33.85060501098633,
      "activations/layer22_attention_weight_min": -21.942691802978516,
      "activations/layer23_attention_weight_max": 21.17607307434082,
      "activations/layer23_attention_weight_min": -15.585433006286621,
      "activations/layer2_attention_weight_max": 26.533939361572266,
      "activations/layer2_attention_weight_min": -25.35057830810547,
      "activations/layer3_attention_weight_max": 64.44573211669922,
      "activations/layer3_attention_weight_min": -69.2859878540039,
      "activations/layer4_attention_weight_max": 92.8948974609375,
      "activations/layer4_attention_weight_min": -83.68563079833984,
      "activations/layer5_attention_weight_max": 121.38845825195312,
      "activations/layer5_attention_weight_min": -99.60841369628906,
      "activations/layer6_attention_weight_max": 64.8708267211914,
      "activations/layer6_attention_weight_min": -49.35314178466797,
      "activations/layer7_attention_weight_max": 55.99203109741211,
      "activations/layer7_attention_weight_min": -50.23849868774414,
      "activations/layer8_attention_weight_max": 39.519622802734375,
      "activations/layer8_attention_weight_min": -34.127925872802734,
      "activations/layer9_attention_weight_max": 53.95681381225586,
      "activations/layer9_attention_weight_min": -52.30495071411133,
      "epoch": 2.51,
      "learning_rate": 0.00013519090909090906,
      "loss": 2.9267,
      "step": 43150
    },
    {
      "activations/layer0_attention_weight_max": 14.07359790802002,
      "activations/layer0_attention_weight_min": -15.263541221618652,
      "activations/layer10_attention_weight_max": 51.80754470825195,
      "activations/layer10_attention_weight_min": -43.3410530090332,
      "activations/layer11_attention_weight_max": 40.67536926269531,
      "activations/layer11_attention_weight_min": -34.34297561645508,
      "activations/layer12_attention_weight_max": 24.669631958007812,
      "activations/layer12_attention_weight_min": -23.462547302246094,
      "activations/layer13_attention_weight_max": 62.024452209472656,
      "activations/layer13_attention_weight_min": -45.59566116333008,
      "activations/layer14_attention_weight_max": 54.408573150634766,
      "activations/layer14_attention_weight_min": -42.330718994140625,
      "activations/layer15_attention_weight_max": 53.101661682128906,
      "activations/layer15_attention_weight_min": -38.19272994995117,
      "activations/layer16_attention_weight_max": 39.96219253540039,
      "activations/layer16_attention_weight_min": -28.21436882019043,
      "activations/layer17_attention_weight_max": 55.44191360473633,
      "activations/layer17_attention_weight_min": -40.276573181152344,
      "activations/layer18_attention_weight_max": 52.2313117980957,
      "activations/layer18_attention_weight_min": -32.397674560546875,
      "activations/layer19_attention_weight_max": 21.69207191467285,
      "activations/layer19_attention_weight_min": -16.444990158081055,
      "activations/layer1_attention_weight_max": 17.871084213256836,
      "activations/layer1_attention_weight_min": -14.566802978515625,
      "activations/layer20_attention_weight_max": 24.495838165283203,
      "activations/layer20_attention_weight_min": -18.09497833251953,
      "activations/layer21_attention_weight_max": 38.60956954956055,
      "activations/layer21_attention_weight_min": -30.87668800354004,
      "activations/layer22_attention_weight_max": 33.09804916381836,
      "activations/layer22_attention_weight_min": -23.922645568847656,
      "activations/layer23_attention_weight_max": 23.871871948242188,
      "activations/layer23_attention_weight_min": -16.440284729003906,
      "activations/layer2_attention_weight_max": 28.251333236694336,
      "activations/layer2_attention_weight_min": -27.812149047851562,
      "activations/layer3_attention_weight_max": 65.05824279785156,
      "activations/layer3_attention_weight_min": -73.55975341796875,
      "activations/layer4_attention_weight_max": 95.34414672851562,
      "activations/layer4_attention_weight_min": -92.95692443847656,
      "activations/layer5_attention_weight_max": 121.80169677734375,
      "activations/layer5_attention_weight_min": -105.74954223632812,
      "activations/layer6_attention_weight_max": 58.136566162109375,
      "activations/layer6_attention_weight_min": -53.94249725341797,
      "activations/layer7_attention_weight_max": 53.79219055175781,
      "activations/layer7_attention_weight_min": -52.940826416015625,
      "activations/layer8_attention_weight_max": 42.998443603515625,
      "activations/layer8_attention_weight_min": -37.10942077636719,
      "activations/layer9_attention_weight_max": 57.08742141723633,
      "activations/layer9_attention_weight_min": -53.78607940673828,
      "epoch": 2.51,
      "learning_rate": 0.00013517196969696968,
      "loss": 2.9405,
      "step": 43200
    },
    {
      "activations/layer0_attention_weight_max": 14.457387924194336,
      "activations/layer0_attention_weight_min": -14.673174858093262,
      "activations/layer10_attention_weight_max": 61.354820251464844,
      "activations/layer10_attention_weight_min": -46.56245803833008,
      "activations/layer11_attention_weight_max": 45.98936080932617,
      "activations/layer11_attention_weight_min": -39.146907806396484,
      "activations/layer12_attention_weight_max": 31.31073760986328,
      "activations/layer12_attention_weight_min": -24.491085052490234,
      "activations/layer13_attention_weight_max": 58.00524139404297,
      "activations/layer13_attention_weight_min": -40.303428649902344,
      "activations/layer14_attention_weight_max": 64.19322204589844,
      "activations/layer14_attention_weight_min": -37.420310974121094,
      "activations/layer15_attention_weight_max": 53.271427154541016,
      "activations/layer15_attention_weight_min": -35.07230758666992,
      "activations/layer16_attention_weight_max": 41.87582015991211,
      "activations/layer16_attention_weight_min": -28.237442016601562,
      "activations/layer17_attention_weight_max": 68.93028259277344,
      "activations/layer17_attention_weight_min": -35.41286849975586,
      "activations/layer18_attention_weight_max": 60.16181945800781,
      "activations/layer18_attention_weight_min": -33.167137145996094,
      "activations/layer19_attention_weight_max": 24.87143898010254,
      "activations/layer19_attention_weight_min": -18.40980339050293,
      "activations/layer1_attention_weight_max": 17.05891990661621,
      "activations/layer1_attention_weight_min": -13.70361042022705,
      "activations/layer20_attention_weight_max": 26.100589752197266,
      "activations/layer20_attention_weight_min": -19.44449806213379,
      "activations/layer21_attention_weight_max": 45.98865509033203,
      "activations/layer21_attention_weight_min": -24.149559020996094,
      "activations/layer22_attention_weight_max": 44.1371955871582,
      "activations/layer22_attention_weight_min": -22.38282012939453,
      "activations/layer23_attention_weight_max": 33.594810485839844,
      "activations/layer23_attention_weight_min": -16.482784271240234,
      "activations/layer2_attention_weight_max": 27.992856979370117,
      "activations/layer2_attention_weight_min": -26.726165771484375,
      "activations/layer3_attention_weight_max": 67.1195297241211,
      "activations/layer3_attention_weight_min": -72.32597351074219,
      "activations/layer4_attention_weight_max": 99.12301635742188,
      "activations/layer4_attention_weight_min": -91.27751922607422,
      "activations/layer5_attention_weight_max": 134.34609985351562,
      "activations/layer5_attention_weight_min": -102.21733856201172,
      "activations/layer6_attention_weight_max": 66.69297790527344,
      "activations/layer6_attention_weight_min": -51.62283706665039,
      "activations/layer7_attention_weight_max": 60.46063232421875,
      "activations/layer7_attention_weight_min": -52.736087799072266,
      "activations/layer8_attention_weight_max": 50.131378173828125,
      "activations/layer8_attention_weight_min": -39.150718688964844,
      "activations/layer9_attention_weight_max": 75.92943572998047,
      "activations/layer9_attention_weight_min": -58.98948287963867,
      "epoch": 2.51,
      "learning_rate": 0.0001351530303030303,
      "loss": 2.9509,
      "step": 43250
    },
    {
      "activations/layer0_attention_weight_max": 14.101982116699219,
      "activations/layer0_attention_weight_min": -15.419878959655762,
      "activations/layer10_attention_weight_max": 50.94499588012695,
      "activations/layer10_attention_weight_min": -42.1806755065918,
      "activations/layer11_attention_weight_max": 38.63798904418945,
      "activations/layer11_attention_weight_min": -31.234935760498047,
      "activations/layer12_attention_weight_max": 25.401121139526367,
      "activations/layer12_attention_weight_min": -24.873016357421875,
      "activations/layer13_attention_weight_max": 64.5837173461914,
      "activations/layer13_attention_weight_min": -46.51689529418945,
      "activations/layer14_attention_weight_max": 53.88905715942383,
      "activations/layer14_attention_weight_min": -36.91579818725586,
      "activations/layer15_attention_weight_max": 47.8353157043457,
      "activations/layer15_attention_weight_min": -35.27268600463867,
      "activations/layer16_attention_weight_max": 38.57200622558594,
      "activations/layer16_attention_weight_min": -28.536144256591797,
      "activations/layer17_attention_weight_max": 57.73546600341797,
      "activations/layer17_attention_weight_min": -39.05924606323242,
      "activations/layer18_attention_weight_max": 49.89212417602539,
      "activations/layer18_attention_weight_min": -33.222412109375,
      "activations/layer19_attention_weight_max": 21.197589874267578,
      "activations/layer19_attention_weight_min": -20.572439193725586,
      "activations/layer1_attention_weight_max": 17.529312133789062,
      "activations/layer1_attention_weight_min": -17.43300437927246,
      "activations/layer20_attention_weight_max": 21.364471435546875,
      "activations/layer20_attention_weight_min": -19.960041046142578,
      "activations/layer21_attention_weight_max": 47.08262634277344,
      "activations/layer21_attention_weight_min": -30.613143920898438,
      "activations/layer22_attention_weight_max": 31.651628494262695,
      "activations/layer22_attention_weight_min": -24.95254898071289,
      "activations/layer23_attention_weight_max": 23.44569206237793,
      "activations/layer23_attention_weight_min": -17.331954956054688,
      "activations/layer2_attention_weight_max": 28.905521392822266,
      "activations/layer2_attention_weight_min": -26.25739288330078,
      "activations/layer3_attention_weight_max": 66.06463623046875,
      "activations/layer3_attention_weight_min": -66.40446472167969,
      "activations/layer4_attention_weight_max": 99.71904754638672,
      "activations/layer4_attention_weight_min": -88.71520233154297,
      "activations/layer5_attention_weight_max": 125.50502014160156,
      "activations/layer5_attention_weight_min": -101.90568542480469,
      "activations/layer6_attention_weight_max": 64.92074584960938,
      "activations/layer6_attention_weight_min": -51.68906784057617,
      "activations/layer7_attention_weight_max": 59.05142593383789,
      "activations/layer7_attention_weight_min": -52.87531280517578,
      "activations/layer8_attention_weight_max": 42.86558532714844,
      "activations/layer8_attention_weight_min": -35.50020217895508,
      "activations/layer9_attention_weight_max": 54.821468353271484,
      "activations/layer9_attention_weight_min": -56.0382080078125,
      "epoch": 2.52,
      "learning_rate": 0.00013513409090909088,
      "loss": 2.9206,
      "step": 43300
    },
    {
      "activations/layer0_attention_weight_max": 14.321710586547852,
      "activations/layer0_attention_weight_min": -14.524317741394043,
      "activations/layer10_attention_weight_max": 49.305233001708984,
      "activations/layer10_attention_weight_min": -37.99215316772461,
      "activations/layer11_attention_weight_max": 42.70698165893555,
      "activations/layer11_attention_weight_min": -32.84503936767578,
      "activations/layer12_attention_weight_max": 28.73038673400879,
      "activations/layer12_attention_weight_min": -24.11029624938965,
      "activations/layer13_attention_weight_max": 56.43402099609375,
      "activations/layer13_attention_weight_min": -42.4888916015625,
      "activations/layer14_attention_weight_max": 50.02899932861328,
      "activations/layer14_attention_weight_min": -34.62990951538086,
      "activations/layer15_attention_weight_max": 51.42905044555664,
      "activations/layer15_attention_weight_min": -34.558719635009766,
      "activations/layer16_attention_weight_max": 46.66869354248047,
      "activations/layer16_attention_weight_min": -27.810131072998047,
      "activations/layer17_attention_weight_max": 59.9236946105957,
      "activations/layer17_attention_weight_min": -36.41238784790039,
      "activations/layer18_attention_weight_max": 56.05772018432617,
      "activations/layer18_attention_weight_min": -33.985931396484375,
      "activations/layer19_attention_weight_max": 23.862266540527344,
      "activations/layer19_attention_weight_min": -16.6918888092041,
      "activations/layer1_attention_weight_max": 16.767297744750977,
      "activations/layer1_attention_weight_min": -13.751655578613281,
      "activations/layer20_attention_weight_max": 20.71163558959961,
      "activations/layer20_attention_weight_min": -21.177244186401367,
      "activations/layer21_attention_weight_max": 52.07755661010742,
      "activations/layer21_attention_weight_min": -30.788558959960938,
      "activations/layer22_attention_weight_max": 36.307926177978516,
      "activations/layer22_attention_weight_min": -23.39984893798828,
      "activations/layer23_attention_weight_max": 25.459861755371094,
      "activations/layer23_attention_weight_min": -15.196757316589355,
      "activations/layer2_attention_weight_max": 27.474912643432617,
      "activations/layer2_attention_weight_min": -26.199079513549805,
      "activations/layer3_attention_weight_max": 67.82439422607422,
      "activations/layer3_attention_weight_min": -68.7745590209961,
      "activations/layer4_attention_weight_max": 99.41304779052734,
      "activations/layer4_attention_weight_min": -88.27855682373047,
      "activations/layer5_attention_weight_max": 123.59685516357422,
      "activations/layer5_attention_weight_min": -101.93289184570312,
      "activations/layer6_attention_weight_max": 61.89350509643555,
      "activations/layer6_attention_weight_min": -53.19956588745117,
      "activations/layer7_attention_weight_max": 61.16122817993164,
      "activations/layer7_attention_weight_min": -55.23679733276367,
      "activations/layer8_attention_weight_max": 40.447689056396484,
      "activations/layer8_attention_weight_min": -35.44660568237305,
      "activations/layer9_attention_weight_max": 57.110816955566406,
      "activations/layer9_attention_weight_min": -52.423072814941406,
      "epoch": 2.52,
      "learning_rate": 0.0001351151515151515,
      "loss": 2.9276,
      "step": 43350
    },
    {
      "activations/layer0_attention_weight_max": 14.953231811523438,
      "activations/layer0_attention_weight_min": -14.447527885437012,
      "activations/layer10_attention_weight_max": 47.94404220581055,
      "activations/layer10_attention_weight_min": -39.22296142578125,
      "activations/layer11_attention_weight_max": 36.651214599609375,
      "activations/layer11_attention_weight_min": -30.954313278198242,
      "activations/layer12_attention_weight_max": 32.36532211303711,
      "activations/layer12_attention_weight_min": -25.642200469970703,
      "activations/layer13_attention_weight_max": 55.697975158691406,
      "activations/layer13_attention_weight_min": -44.51203155517578,
      "activations/layer14_attention_weight_max": 48.63389587402344,
      "activations/layer14_attention_weight_min": -39.779605865478516,
      "activations/layer15_attention_weight_max": 45.97821044921875,
      "activations/layer15_attention_weight_min": -34.515071868896484,
      "activations/layer16_attention_weight_max": 38.79756546020508,
      "activations/layer16_attention_weight_min": -28.767898559570312,
      "activations/layer17_attention_weight_max": 57.35755157470703,
      "activations/layer17_attention_weight_min": -36.390953063964844,
      "activations/layer18_attention_weight_max": 57.440643310546875,
      "activations/layer18_attention_weight_min": -33.927955627441406,
      "activations/layer19_attention_weight_max": 24.342164993286133,
      "activations/layer19_attention_weight_min": -16.994243621826172,
      "activations/layer1_attention_weight_max": 16.072006225585938,
      "activations/layer1_attention_weight_min": -16.12527847290039,
      "activations/layer20_attention_weight_max": 20.703847885131836,
      "activations/layer20_attention_weight_min": -18.66997528076172,
      "activations/layer21_attention_weight_max": 45.759246826171875,
      "activations/layer21_attention_weight_min": -26.07560157775879,
      "activations/layer22_attention_weight_max": 37.89739227294922,
      "activations/layer22_attention_weight_min": -25.32058334350586,
      "activations/layer23_attention_weight_max": 29.911911010742188,
      "activations/layer23_attention_weight_min": -17.720386505126953,
      "activations/layer2_attention_weight_max": 27.814064025878906,
      "activations/layer2_attention_weight_min": -27.307575225830078,
      "activations/layer3_attention_weight_max": 67.60262298583984,
      "activations/layer3_attention_weight_min": -70.68939208984375,
      "activations/layer4_attention_weight_max": 95.69488525390625,
      "activations/layer4_attention_weight_min": -90.8355941772461,
      "activations/layer5_attention_weight_max": 122.37861633300781,
      "activations/layer5_attention_weight_min": -103.765869140625,
      "activations/layer6_attention_weight_max": 59.97517013549805,
      "activations/layer6_attention_weight_min": -54.3791389465332,
      "activations/layer7_attention_weight_max": 54.247371673583984,
      "activations/layer7_attention_weight_min": -56.569644927978516,
      "activations/layer8_attention_weight_max": 39.050437927246094,
      "activations/layer8_attention_weight_min": -35.537418365478516,
      "activations/layer9_attention_weight_max": 55.7322883605957,
      "activations/layer9_attention_weight_min": -50.2769775390625,
      "epoch": 2.52,
      "learning_rate": 0.0001350962121212121,
      "loss": 2.9329,
      "step": 43400
    },
    {
      "activations/layer0_attention_weight_max": 14.855378150939941,
      "activations/layer0_attention_weight_min": -14.660085678100586,
      "activations/layer10_attention_weight_max": 49.445430755615234,
      "activations/layer10_attention_weight_min": -41.4345588684082,
      "activations/layer11_attention_weight_max": 40.00124740600586,
      "activations/layer11_attention_weight_min": -33.5339469909668,
      "activations/layer12_attention_weight_max": 24.4869384765625,
      "activations/layer12_attention_weight_min": -26.04244613647461,
      "activations/layer13_attention_weight_max": 52.860572814941406,
      "activations/layer13_attention_weight_min": -41.71119689941406,
      "activations/layer14_attention_weight_max": 49.92226028442383,
      "activations/layer14_attention_weight_min": -35.50303268432617,
      "activations/layer15_attention_weight_max": 50.70227813720703,
      "activations/layer15_attention_weight_min": -38.38346481323242,
      "activations/layer16_attention_weight_max": 40.59280776977539,
      "activations/layer16_attention_weight_min": -28.31342315673828,
      "activations/layer17_attention_weight_max": 68.89948272705078,
      "activations/layer17_attention_weight_min": -40.29718017578125,
      "activations/layer18_attention_weight_max": 58.55022430419922,
      "activations/layer18_attention_weight_min": -32.81094741821289,
      "activations/layer19_attention_weight_max": 21.231721878051758,
      "activations/layer19_attention_weight_min": -19.684059143066406,
      "activations/layer1_attention_weight_max": 17.46493911743164,
      "activations/layer1_attention_weight_min": -12.154312133789062,
      "activations/layer20_attention_weight_max": 22.959142684936523,
      "activations/layer20_attention_weight_min": -17.294469833374023,
      "activations/layer21_attention_weight_max": 42.7094612121582,
      "activations/layer21_attention_weight_min": -27.327816009521484,
      "activations/layer22_attention_weight_max": 41.09244155883789,
      "activations/layer22_attention_weight_min": -25.480533599853516,
      "activations/layer23_attention_weight_max": 25.058828353881836,
      "activations/layer23_attention_weight_min": -17.26015281677246,
      "activations/layer2_attention_weight_max": 26.743635177612305,
      "activations/layer2_attention_weight_min": -26.01625633239746,
      "activations/layer3_attention_weight_max": 66.30887603759766,
      "activations/layer3_attention_weight_min": -66.46565246582031,
      "activations/layer4_attention_weight_max": 104.7499771118164,
      "activations/layer4_attention_weight_min": -88.8864974975586,
      "activations/layer5_attention_weight_max": 140.5537109375,
      "activations/layer5_attention_weight_min": -101.69860076904297,
      "activations/layer6_attention_weight_max": 67.19371032714844,
      "activations/layer6_attention_weight_min": -49.720645904541016,
      "activations/layer7_attention_weight_max": 63.57786178588867,
      "activations/layer7_attention_weight_min": -54.31913757324219,
      "activations/layer8_attention_weight_max": 44.651859283447266,
      "activations/layer8_attention_weight_min": -36.02375030517578,
      "activations/layer9_attention_weight_max": 55.22307205200195,
      "activations/layer9_attention_weight_min": -52.760711669921875,
      "epoch": 2.52,
      "learning_rate": 0.0001350772727272727,
      "loss": 2.9284,
      "step": 43450
    },
    {
      "activations/layer0_attention_weight_max": 14.057829856872559,
      "activations/layer0_attention_weight_min": -15.403241157531738,
      "activations/layer10_attention_weight_max": 60.0714111328125,
      "activations/layer10_attention_weight_min": -42.6746711730957,
      "activations/layer11_attention_weight_max": 48.97486877441406,
      "activations/layer11_attention_weight_min": -30.744369506835938,
      "activations/layer12_attention_weight_max": 30.059799194335938,
      "activations/layer12_attention_weight_min": -26.982620239257812,
      "activations/layer13_attention_weight_max": 57.28300476074219,
      "activations/layer13_attention_weight_min": -39.042781829833984,
      "activations/layer14_attention_weight_max": 49.4584846496582,
      "activations/layer14_attention_weight_min": -34.616207122802734,
      "activations/layer15_attention_weight_max": 54.5569953918457,
      "activations/layer15_attention_weight_min": -37.13214874267578,
      "activations/layer16_attention_weight_max": 42.90656280517578,
      "activations/layer16_attention_weight_min": -27.005237579345703,
      "activations/layer17_attention_weight_max": 75.58506774902344,
      "activations/layer17_attention_weight_min": -41.18085861206055,
      "activations/layer18_attention_weight_max": 61.648277282714844,
      "activations/layer18_attention_weight_min": -32.745365142822266,
      "activations/layer19_attention_weight_max": 22.66398811340332,
      "activations/layer19_attention_weight_min": -16.655616760253906,
      "activations/layer1_attention_weight_max": 16.724496841430664,
      "activations/layer1_attention_weight_min": -14.475529670715332,
      "activations/layer20_attention_weight_max": 22.192724227905273,
      "activations/layer20_attention_weight_min": -20.09147834777832,
      "activations/layer21_attention_weight_max": 43.53749465942383,
      "activations/layer21_attention_weight_min": -27.078514099121094,
      "activations/layer22_attention_weight_max": 43.243896484375,
      "activations/layer22_attention_weight_min": -24.441057205200195,
      "activations/layer23_attention_weight_max": 26.103116989135742,
      "activations/layer23_attention_weight_min": -15.843649864196777,
      "activations/layer2_attention_weight_max": 28.746999740600586,
      "activations/layer2_attention_weight_min": -27.71898651123047,
      "activations/layer3_attention_weight_max": 67.40987396240234,
      "activations/layer3_attention_weight_min": -73.8263931274414,
      "activations/layer4_attention_weight_max": 98.26480102539062,
      "activations/layer4_attention_weight_min": -88.89107513427734,
      "activations/layer5_attention_weight_max": 127.61495971679688,
      "activations/layer5_attention_weight_min": -105.25129699707031,
      "activations/layer6_attention_weight_max": 64.63946533203125,
      "activations/layer6_attention_weight_min": -52.09897232055664,
      "activations/layer7_attention_weight_max": 59.70333480834961,
      "activations/layer7_attention_weight_min": -53.649051666259766,
      "activations/layer8_attention_weight_max": 43.407501220703125,
      "activations/layer8_attention_weight_min": -36.13612747192383,
      "activations/layer9_attention_weight_max": 58.116878509521484,
      "activations/layer9_attention_weight_min": -56.0875244140625,
      "epoch": 2.53,
      "learning_rate": 0.00013505833333333331,
      "loss": 2.9296,
      "step": 43500
    },
    {
      "activations/layer0_attention_weight_max": 16.151172637939453,
      "activations/layer0_attention_weight_min": -15.084325790405273,
      "activations/layer10_attention_weight_max": 51.43994903564453,
      "activations/layer10_attention_weight_min": -41.32536697387695,
      "activations/layer11_attention_weight_max": 39.49317932128906,
      "activations/layer11_attention_weight_min": -30.984798431396484,
      "activations/layer12_attention_weight_max": 41.94902801513672,
      "activations/layer12_attention_weight_min": -28.08928108215332,
      "activations/layer13_attention_weight_max": 55.43145751953125,
      "activations/layer13_attention_weight_min": -39.60833740234375,
      "activations/layer14_attention_weight_max": 50.035770416259766,
      "activations/layer14_attention_weight_min": -33.10663986206055,
      "activations/layer15_attention_weight_max": 53.3807258605957,
      "activations/layer15_attention_weight_min": -34.514522552490234,
      "activations/layer16_attention_weight_max": 44.4031867980957,
      "activations/layer16_attention_weight_min": -26.60994529724121,
      "activations/layer17_attention_weight_max": 64.21804809570312,
      "activations/layer17_attention_weight_min": -39.9711799621582,
      "activations/layer18_attention_weight_max": 60.0340576171875,
      "activations/layer18_attention_weight_min": -33.39973449707031,
      "activations/layer19_attention_weight_max": 22.921728134155273,
      "activations/layer19_attention_weight_min": -18.101669311523438,
      "activations/layer1_attention_weight_max": 16.787994384765625,
      "activations/layer1_attention_weight_min": -15.134072303771973,
      "activations/layer20_attention_weight_max": 23.10759735107422,
      "activations/layer20_attention_weight_min": -18.464900970458984,
      "activations/layer21_attention_weight_max": 44.28866195678711,
      "activations/layer21_attention_weight_min": -27.32927703857422,
      "activations/layer22_attention_weight_max": 34.282474517822266,
      "activations/layer22_attention_weight_min": -24.240203857421875,
      "activations/layer23_attention_weight_max": 26.687841415405273,
      "activations/layer23_attention_weight_min": -16.472129821777344,
      "activations/layer2_attention_weight_max": 28.64173698425293,
      "activations/layer2_attention_weight_min": -27.399396896362305,
      "activations/layer3_attention_weight_max": 68.78848266601562,
      "activations/layer3_attention_weight_min": -73.15101623535156,
      "activations/layer4_attention_weight_max": 96.71625518798828,
      "activations/layer4_attention_weight_min": -92.4282455444336,
      "activations/layer5_attention_weight_max": 125.33248138427734,
      "activations/layer5_attention_weight_min": -105.69161224365234,
      "activations/layer6_attention_weight_max": 62.97687530517578,
      "activations/layer6_attention_weight_min": -53.991310119628906,
      "activations/layer7_attention_weight_max": 58.32658004760742,
      "activations/layer7_attention_weight_min": -53.65458297729492,
      "activations/layer8_attention_weight_max": 42.24081802368164,
      "activations/layer8_attention_weight_min": -37.65232849121094,
      "activations/layer9_attention_weight_max": 54.72096633911133,
      "activations/layer9_attention_weight_min": -51.06723403930664,
      "epoch": 2.53,
      "learning_rate": 0.00013503939393939393,
      "loss": 2.9406,
      "step": 43550
    },
    {
      "activations/layer0_attention_weight_max": 12.982949256896973,
      "activations/layer0_attention_weight_min": -15.662398338317871,
      "activations/layer10_attention_weight_max": 57.800777435302734,
      "activations/layer10_attention_weight_min": -42.293434143066406,
      "activations/layer11_attention_weight_max": 42.53942108154297,
      "activations/layer11_attention_weight_min": -34.068153381347656,
      "activations/layer12_attention_weight_max": 33.58500671386719,
      "activations/layer12_attention_weight_min": -29.73676300048828,
      "activations/layer13_attention_weight_max": 63.59953308105469,
      "activations/layer13_attention_weight_min": -43.10094451904297,
      "activations/layer14_attention_weight_max": 53.96380615234375,
      "activations/layer14_attention_weight_min": -34.38897705078125,
      "activations/layer15_attention_weight_max": 51.713436126708984,
      "activations/layer15_attention_weight_min": -34.57931137084961,
      "activations/layer16_attention_weight_max": 39.2649040222168,
      "activations/layer16_attention_weight_min": -32.02465057373047,
      "activations/layer17_attention_weight_max": 65.78749084472656,
      "activations/layer17_attention_weight_min": -35.60980224609375,
      "activations/layer18_attention_weight_max": 62.00503921508789,
      "activations/layer18_attention_weight_min": -31.871849060058594,
      "activations/layer19_attention_weight_max": 21.6650390625,
      "activations/layer19_attention_weight_min": -15.921553611755371,
      "activations/layer1_attention_weight_max": 17.51618003845215,
      "activations/layer1_attention_weight_min": -14.002288818359375,
      "activations/layer20_attention_weight_max": 19.072856903076172,
      "activations/layer20_attention_weight_min": -17.260221481323242,
      "activations/layer21_attention_weight_max": 43.95772171020508,
      "activations/layer21_attention_weight_min": -28.92459487915039,
      "activations/layer22_attention_weight_max": 37.956974029541016,
      "activations/layer22_attention_weight_min": -21.97182273864746,
      "activations/layer23_attention_weight_max": 28.32455825805664,
      "activations/layer23_attention_weight_min": -16.01025390625,
      "activations/layer2_attention_weight_max": 28.575912475585938,
      "activations/layer2_attention_weight_min": -29.63934326171875,
      "activations/layer3_attention_weight_max": 70.92021179199219,
      "activations/layer3_attention_weight_min": -75.65528869628906,
      "activations/layer4_attention_weight_max": 101.39260864257812,
      "activations/layer4_attention_weight_min": -93.27605438232422,
      "activations/layer5_attention_weight_max": 136.0887451171875,
      "activations/layer5_attention_weight_min": -104.90664672851562,
      "activations/layer6_attention_weight_max": 65.52941131591797,
      "activations/layer6_attention_weight_min": -55.05972671508789,
      "activations/layer7_attention_weight_max": 63.33991622924805,
      "activations/layer7_attention_weight_min": -54.76594543457031,
      "activations/layer8_attention_weight_max": 46.61769485473633,
      "activations/layer8_attention_weight_min": -39.15641403198242,
      "activations/layer9_attention_weight_max": 58.845272064208984,
      "activations/layer9_attention_weight_min": -53.414241790771484,
      "epoch": 2.53,
      "learning_rate": 0.0001350212121212121,
      "loss": 2.9429,
      "step": 43600
    },
    {
      "activations/layer0_attention_weight_max": 14.290939331054688,
      "activations/layer0_attention_weight_min": -14.889986991882324,
      "activations/layer10_attention_weight_max": 50.36459732055664,
      "activations/layer10_attention_weight_min": -40.639347076416016,
      "activations/layer11_attention_weight_max": 42.7440185546875,
      "activations/layer11_attention_weight_min": -30.835391998291016,
      "activations/layer12_attention_weight_max": 28.827655792236328,
      "activations/layer12_attention_weight_min": -21.690317153930664,
      "activations/layer13_attention_weight_max": 50.31462097167969,
      "activations/layer13_attention_weight_min": -37.8394775390625,
      "activations/layer14_attention_weight_max": 47.037132263183594,
      "activations/layer14_attention_weight_min": -33.5983772277832,
      "activations/layer15_attention_weight_max": 48.14339828491211,
      "activations/layer15_attention_weight_min": -35.39642333984375,
      "activations/layer16_attention_weight_max": 39.93472671508789,
      "activations/layer16_attention_weight_min": -27.43168067932129,
      "activations/layer17_attention_weight_max": 63.21489334106445,
      "activations/layer17_attention_weight_min": -37.77898025512695,
      "activations/layer18_attention_weight_max": 60.13280487060547,
      "activations/layer18_attention_weight_min": -32.99378204345703,
      "activations/layer19_attention_weight_max": 23.413623809814453,
      "activations/layer19_attention_weight_min": -20.152729034423828,
      "activations/layer1_attention_weight_max": 17.4352970123291,
      "activations/layer1_attention_weight_min": -14.376116752624512,
      "activations/layer20_attention_weight_max": 20.65091323852539,
      "activations/layer20_attention_weight_min": -17.60573387145996,
      "activations/layer21_attention_weight_max": 45.61101150512695,
      "activations/layer21_attention_weight_min": -26.511960983276367,
      "activations/layer22_attention_weight_max": 32.60503387451172,
      "activations/layer22_attention_weight_min": -22.366060256958008,
      "activations/layer23_attention_weight_max": 23.820707321166992,
      "activations/layer23_attention_weight_min": -14.862544059753418,
      "activations/layer2_attention_weight_max": 28.988971710205078,
      "activations/layer2_attention_weight_min": -27.690595626831055,
      "activations/layer3_attention_weight_max": 67.83260345458984,
      "activations/layer3_attention_weight_min": -71.01287841796875,
      "activations/layer4_attention_weight_max": 95.48723602294922,
      "activations/layer4_attention_weight_min": -88.48928833007812,
      "activations/layer5_attention_weight_max": 121.97069549560547,
      "activations/layer5_attention_weight_min": -103.6509017944336,
      "activations/layer6_attention_weight_max": 59.08152389526367,
      "activations/layer6_attention_weight_min": -52.47956085205078,
      "activations/layer7_attention_weight_max": 54.64801025390625,
      "activations/layer7_attention_weight_min": -53.83021545410156,
      "activations/layer8_attention_weight_max": 41.38795471191406,
      "activations/layer8_attention_weight_min": -36.7194709777832,
      "activations/layer9_attention_weight_max": 54.00913619995117,
      "activations/layer9_attention_weight_min": -50.967769622802734,
      "epoch": 2.54,
      "learning_rate": 0.0001350022727272727,
      "loss": 2.925,
      "step": 43650
    },
    {
      "activations/layer0_attention_weight_max": 13.162952423095703,
      "activations/layer0_attention_weight_min": -14.586260795593262,
      "activations/layer10_attention_weight_max": 48.25095748901367,
      "activations/layer10_attention_weight_min": -43.373653411865234,
      "activations/layer11_attention_weight_max": 35.32673645019531,
      "activations/layer11_attention_weight_min": -29.606258392333984,
      "activations/layer12_attention_weight_max": 22.06100082397461,
      "activations/layer12_attention_weight_min": -23.821857452392578,
      "activations/layer13_attention_weight_max": 50.1938591003418,
      "activations/layer13_attention_weight_min": -35.68084716796875,
      "activations/layer14_attention_weight_max": 47.08941650390625,
      "activations/layer14_attention_weight_min": -40.031070709228516,
      "activations/layer15_attention_weight_max": 45.37924575805664,
      "activations/layer15_attention_weight_min": -32.28974151611328,
      "activations/layer16_attention_weight_max": 37.64247131347656,
      "activations/layer16_attention_weight_min": -26.67992401123047,
      "activations/layer17_attention_weight_max": 64.83512115478516,
      "activations/layer17_attention_weight_min": -40.16395950317383,
      "activations/layer18_attention_weight_max": 58.42673110961914,
      "activations/layer18_attention_weight_min": -32.68077850341797,
      "activations/layer19_attention_weight_max": 23.111042022705078,
      "activations/layer19_attention_weight_min": -18.888769149780273,
      "activations/layer1_attention_weight_max": 16.658605575561523,
      "activations/layer1_attention_weight_min": -12.471230506896973,
      "activations/layer20_attention_weight_max": 21.53261375427246,
      "activations/layer20_attention_weight_min": -21.260290145874023,
      "activations/layer21_attention_weight_max": 38.865089416503906,
      "activations/layer21_attention_weight_min": -26.406539916992188,
      "activations/layer22_attention_weight_max": 38.74755096435547,
      "activations/layer22_attention_weight_min": -21.533946990966797,
      "activations/layer23_attention_weight_max": 24.58978843688965,
      "activations/layer23_attention_weight_min": -16.95165252685547,
      "activations/layer2_attention_weight_max": 28.897483825683594,
      "activations/layer2_attention_weight_min": -26.649959564208984,
      "activations/layer3_attention_weight_max": 68.04322814941406,
      "activations/layer3_attention_weight_min": -72.77552032470703,
      "activations/layer4_attention_weight_max": 96.670654296875,
      "activations/layer4_attention_weight_min": -90.00170135498047,
      "activations/layer5_attention_weight_max": 117.72642517089844,
      "activations/layer5_attention_weight_min": -97.353271484375,
      "activations/layer6_attention_weight_max": 61.034088134765625,
      "activations/layer6_attention_weight_min": -50.85262680053711,
      "activations/layer7_attention_weight_max": 54.22457504272461,
      "activations/layer7_attention_weight_min": -49.23521041870117,
      "activations/layer8_attention_weight_max": 40.9932861328125,
      "activations/layer8_attention_weight_min": -35.12348937988281,
      "activations/layer9_attention_weight_max": 60.190059661865234,
      "activations/layer9_attention_weight_min": -49.25585174560547,
      "epoch": 2.54,
      "learning_rate": 0.00013498333333333333,
      "loss": 2.9268,
      "step": 43700
    },
    {
      "activations/layer0_attention_weight_max": 13.975242614746094,
      "activations/layer0_attention_weight_min": -14.18014144897461,
      "activations/layer10_attention_weight_max": 52.7779655456543,
      "activations/layer10_attention_weight_min": -39.1224250793457,
      "activations/layer11_attention_weight_max": 47.210201263427734,
      "activations/layer11_attention_weight_min": -31.695270538330078,
      "activations/layer12_attention_weight_max": 25.71039581298828,
      "activations/layer12_attention_weight_min": -24.042707443237305,
      "activations/layer13_attention_weight_max": 56.82240295410156,
      "activations/layer13_attention_weight_min": -39.216880798339844,
      "activations/layer14_attention_weight_max": 44.733154296875,
      "activations/layer14_attention_weight_min": -31.232746124267578,
      "activations/layer15_attention_weight_max": 52.25016784667969,
      "activations/layer15_attention_weight_min": -36.839290618896484,
      "activations/layer16_attention_weight_max": 35.962486267089844,
      "activations/layer16_attention_weight_min": -27.79098892211914,
      "activations/layer17_attention_weight_max": 57.09773254394531,
      "activations/layer17_attention_weight_min": -36.132713317871094,
      "activations/layer18_attention_weight_max": 55.63300704956055,
      "activations/layer18_attention_weight_min": -33.21375274658203,
      "activations/layer19_attention_weight_max": 19.47115135192871,
      "activations/layer19_attention_weight_min": -17.975858688354492,
      "activations/layer1_attention_weight_max": 17.68352508544922,
      "activations/layer1_attention_weight_min": -15.306350708007812,
      "activations/layer20_attention_weight_max": 19.217145919799805,
      "activations/layer20_attention_weight_min": -20.322484970092773,
      "activations/layer21_attention_weight_max": 39.57636260986328,
      "activations/layer21_attention_weight_min": -28.94131851196289,
      "activations/layer22_attention_weight_max": 32.35239028930664,
      "activations/layer22_attention_weight_min": -22.375858306884766,
      "activations/layer23_attention_weight_max": 23.183204650878906,
      "activations/layer23_attention_weight_min": -16.10797882080078,
      "activations/layer2_attention_weight_max": 26.87445068359375,
      "activations/layer2_attention_weight_min": -26.746673583984375,
      "activations/layer3_attention_weight_max": 63.783355712890625,
      "activations/layer3_attention_weight_min": -69.05609893798828,
      "activations/layer4_attention_weight_max": 92.06015014648438,
      "activations/layer4_attention_weight_min": -82.46575927734375,
      "activations/layer5_attention_weight_max": 115.42030334472656,
      "activations/layer5_attention_weight_min": -96.44271087646484,
      "activations/layer6_attention_weight_max": 58.79816436767578,
      "activations/layer6_attention_weight_min": -47.1196403503418,
      "activations/layer7_attention_weight_max": 55.83431625366211,
      "activations/layer7_attention_weight_min": -51.905845642089844,
      "activations/layer8_attention_weight_max": 42.884307861328125,
      "activations/layer8_attention_weight_min": -34.06856155395508,
      "activations/layer9_attention_weight_max": 52.67078399658203,
      "activations/layer9_attention_weight_min": -50.814083099365234,
      "epoch": 2.54,
      "learning_rate": 0.00013496439393939394,
      "loss": 2.9229,
      "step": 43750
    },
    {
      "activations/layer0_attention_weight_max": 15.712189674377441,
      "activations/layer0_attention_weight_min": -14.752391815185547,
      "activations/layer10_attention_weight_max": 48.422847747802734,
      "activations/layer10_attention_weight_min": -39.63105773925781,
      "activations/layer11_attention_weight_max": 37.89212417602539,
      "activations/layer11_attention_weight_min": -30.956411361694336,
      "activations/layer12_attention_weight_max": 25.222551345825195,
      "activations/layer12_attention_weight_min": -24.138639450073242,
      "activations/layer13_attention_weight_max": 54.44145202636719,
      "activations/layer13_attention_weight_min": -40.40277862548828,
      "activations/layer14_attention_weight_max": 42.93243408203125,
      "activations/layer14_attention_weight_min": -32.71620178222656,
      "activations/layer15_attention_weight_max": 52.22994613647461,
      "activations/layer15_attention_weight_min": -35.3415412902832,
      "activations/layer16_attention_weight_max": 36.405364990234375,
      "activations/layer16_attention_weight_min": -25.895158767700195,
      "activations/layer17_attention_weight_max": 56.05628204345703,
      "activations/layer17_attention_weight_min": -35.8729133605957,
      "activations/layer18_attention_weight_max": 51.60371780395508,
      "activations/layer18_attention_weight_min": -30.005704879760742,
      "activations/layer19_attention_weight_max": 21.42183494567871,
      "activations/layer19_attention_weight_min": -14.882502555847168,
      "activations/layer1_attention_weight_max": 16.11615753173828,
      "activations/layer1_attention_weight_min": -13.541853904724121,
      "activations/layer20_attention_weight_max": 20.2210693359375,
      "activations/layer20_attention_weight_min": -21.78761863708496,
      "activations/layer21_attention_weight_max": 37.5052604675293,
      "activations/layer21_attention_weight_min": -27.735008239746094,
      "activations/layer22_attention_weight_max": 34.71294021606445,
      "activations/layer22_attention_weight_min": -19.867536544799805,
      "activations/layer23_attention_weight_max": 24.873960494995117,
      "activations/layer23_attention_weight_min": -13.92100715637207,
      "activations/layer2_attention_weight_max": 29.34825897216797,
      "activations/layer2_attention_weight_min": -27.651884078979492,
      "activations/layer3_attention_weight_max": 67.98931121826172,
      "activations/layer3_attention_weight_min": -73.13864135742188,
      "activations/layer4_attention_weight_max": 98.95081329345703,
      "activations/layer4_attention_weight_min": -85.67597961425781,
      "activations/layer5_attention_weight_max": 126.93090057373047,
      "activations/layer5_attention_weight_min": -99.77119445800781,
      "activations/layer6_attention_weight_max": 65.82066345214844,
      "activations/layer6_attention_weight_min": -49.59886932373047,
      "activations/layer7_attention_weight_max": 58.62804412841797,
      "activations/layer7_attention_weight_min": -55.94926071166992,
      "activations/layer8_attention_weight_max": 41.7392463684082,
      "activations/layer8_attention_weight_min": -35.076805114746094,
      "activations/layer9_attention_weight_max": 61.63201904296875,
      "activations/layer9_attention_weight_min": -50.35211944580078,
      "epoch": 2.55,
      "learning_rate": 0.00013494545454545453,
      "loss": 2.9421,
      "step": 43800
    },
    {
      "activations/layer0_attention_weight_max": 13.997876167297363,
      "activations/layer0_attention_weight_min": -14.345751762390137,
      "activations/layer10_attention_weight_max": 58.991756439208984,
      "activations/layer10_attention_weight_min": -46.91436767578125,
      "activations/layer11_attention_weight_max": 48.3217887878418,
      "activations/layer11_attention_weight_min": -31.533950805664062,
      "activations/layer12_attention_weight_max": 31.717077255249023,
      "activations/layer12_attention_weight_min": -24.84781265258789,
      "activations/layer13_attention_weight_max": 52.786922454833984,
      "activations/layer13_attention_weight_min": -37.44856262207031,
      "activations/layer14_attention_weight_max": 57.32883071899414,
      "activations/layer14_attention_weight_min": -37.04793930053711,
      "activations/layer15_attention_weight_max": 50.69277572631836,
      "activations/layer15_attention_weight_min": -34.41656494140625,
      "activations/layer16_attention_weight_max": 44.5785026550293,
      "activations/layer16_attention_weight_min": -27.707340240478516,
      "activations/layer17_attention_weight_max": 74.617431640625,
      "activations/layer17_attention_weight_min": -43.375465393066406,
      "activations/layer18_attention_weight_max": 67.48406982421875,
      "activations/layer18_attention_weight_min": -35.7686882019043,
      "activations/layer19_attention_weight_max": 27.27235984802246,
      "activations/layer19_attention_weight_min": -17.20443344116211,
      "activations/layer1_attention_weight_max": 18.179256439208984,
      "activations/layer1_attention_weight_min": -15.347614288330078,
      "activations/layer20_attention_weight_max": 21.636276245117188,
      "activations/layer20_attention_weight_min": -18.762765884399414,
      "activations/layer21_attention_weight_max": 44.3973388671875,
      "activations/layer21_attention_weight_min": -29.08871078491211,
      "activations/layer22_attention_weight_max": 40.51578140258789,
      "activations/layer22_attention_weight_min": -26.66034698486328,
      "activations/layer23_attention_weight_max": 29.463857650756836,
      "activations/layer23_attention_weight_min": -18.14830207824707,
      "activations/layer2_attention_weight_max": 27.830968856811523,
      "activations/layer2_attention_weight_min": -26.4738826751709,
      "activations/layer3_attention_weight_max": 64.98709106445312,
      "activations/layer3_attention_weight_min": -70.56462097167969,
      "activations/layer4_attention_weight_max": 94.08580780029297,
      "activations/layer4_attention_weight_min": -91.2241439819336,
      "activations/layer5_attention_weight_max": 121.92937469482422,
      "activations/layer5_attention_weight_min": -102.13097381591797,
      "activations/layer6_attention_weight_max": 60.90013885498047,
      "activations/layer6_attention_weight_min": -50.31612014770508,
      "activations/layer7_attention_weight_max": 58.20337677001953,
      "activations/layer7_attention_weight_min": -52.11966323852539,
      "activations/layer8_attention_weight_max": 41.68004608154297,
      "activations/layer8_attention_weight_min": -37.651206970214844,
      "activations/layer9_attention_weight_max": 69.17540740966797,
      "activations/layer9_attention_weight_min": -55.155242919921875,
      "epoch": 2.55,
      "learning_rate": 0.00013492651515151514,
      "loss": 2.9412,
      "step": 43850
    },
    {
      "activations/layer0_attention_weight_max": 13.951184272766113,
      "activations/layer0_attention_weight_min": -14.737680435180664,
      "activations/layer10_attention_weight_max": 50.808189392089844,
      "activations/layer10_attention_weight_min": -41.71757507324219,
      "activations/layer11_attention_weight_max": 42.7364616394043,
      "activations/layer11_attention_weight_min": -30.72066879272461,
      "activations/layer12_attention_weight_max": 28.074020385742188,
      "activations/layer12_attention_weight_min": -25.052162170410156,
      "activations/layer13_attention_weight_max": 69.3941879272461,
      "activations/layer13_attention_weight_min": -45.44416046142578,
      "activations/layer14_attention_weight_max": 61.232845306396484,
      "activations/layer14_attention_weight_min": -41.290435791015625,
      "activations/layer15_attention_weight_max": 54.966758728027344,
      "activations/layer15_attention_weight_min": -36.963016510009766,
      "activations/layer16_attention_weight_max": 42.220848083496094,
      "activations/layer16_attention_weight_min": -27.86223602294922,
      "activations/layer17_attention_weight_max": 65.4517593383789,
      "activations/layer17_attention_weight_min": -33.692771911621094,
      "activations/layer18_attention_weight_max": 56.7036247253418,
      "activations/layer18_attention_weight_min": -31.916372299194336,
      "activations/layer19_attention_weight_max": 27.1159610748291,
      "activations/layer19_attention_weight_min": -15.582841873168945,
      "activations/layer1_attention_weight_max": 17.066314697265625,
      "activations/layer1_attention_weight_min": -14.085813522338867,
      "activations/layer20_attention_weight_max": 23.686254501342773,
      "activations/layer20_attention_weight_min": -17.754104614257812,
      "activations/layer21_attention_weight_max": 49.84333038330078,
      "activations/layer21_attention_weight_min": -27.92923927307129,
      "activations/layer22_attention_weight_max": 45.96856689453125,
      "activations/layer22_attention_weight_min": -23.18630027770996,
      "activations/layer23_attention_weight_max": 30.682552337646484,
      "activations/layer23_attention_weight_min": -14.492663383483887,
      "activations/layer2_attention_weight_max": 28.820999145507812,
      "activations/layer2_attention_weight_min": -28.185033798217773,
      "activations/layer3_attention_weight_max": 67.62442016601562,
      "activations/layer3_attention_weight_min": -70.39575958251953,
      "activations/layer4_attention_weight_max": 95.35650634765625,
      "activations/layer4_attention_weight_min": -87.52771759033203,
      "activations/layer5_attention_weight_max": 125.39131927490234,
      "activations/layer5_attention_weight_min": -98.96208190917969,
      "activations/layer6_attention_weight_max": 63.679039001464844,
      "activations/layer6_attention_weight_min": -50.23658752441406,
      "activations/layer7_attention_weight_max": 56.30974578857422,
      "activations/layer7_attention_weight_min": -54.60797119140625,
      "activations/layer8_attention_weight_max": 43.08146286010742,
      "activations/layer8_attention_weight_min": -34.4815559387207,
      "activations/layer9_attention_weight_max": 60.14471435546875,
      "activations/layer9_attention_weight_min": -50.61979675292969,
      "epoch": 2.55,
      "learning_rate": 0.00013490757575757576,
      "loss": 2.9207,
      "step": 43900
    },
    {
      "activations/layer0_attention_weight_max": 13.668665885925293,
      "activations/layer0_attention_weight_min": -14.77627944946289,
      "activations/layer10_attention_weight_max": 49.42746353149414,
      "activations/layer10_attention_weight_min": -40.06216049194336,
      "activations/layer11_attention_weight_max": 46.43365478515625,
      "activations/layer11_attention_weight_min": -32.714107513427734,
      "activations/layer12_attention_weight_max": 29.445655822753906,
      "activations/layer12_attention_weight_min": -23.467365264892578,
      "activations/layer13_attention_weight_max": 70.0096435546875,
      "activations/layer13_attention_weight_min": -50.30323028564453,
      "activations/layer14_attention_weight_max": 59.40251922607422,
      "activations/layer14_attention_weight_min": -43.006797790527344,
      "activations/layer15_attention_weight_max": 58.50778579711914,
      "activations/layer15_attention_weight_min": -35.366146087646484,
      "activations/layer16_attention_weight_max": 42.925262451171875,
      "activations/layer16_attention_weight_min": -28.030784606933594,
      "activations/layer17_attention_weight_max": 62.47819900512695,
      "activations/layer17_attention_weight_min": -36.28200149536133,
      "activations/layer18_attention_weight_max": 61.301116943359375,
      "activations/layer18_attention_weight_min": -33.64728927612305,
      "activations/layer19_attention_weight_max": 27.244131088256836,
      "activations/layer19_attention_weight_min": -16.484783172607422,
      "activations/layer1_attention_weight_max": 16.740238189697266,
      "activations/layer1_attention_weight_min": -13.725908279418945,
      "activations/layer20_attention_weight_max": 21.437156677246094,
      "activations/layer20_attention_weight_min": -19.083293914794922,
      "activations/layer21_attention_weight_max": 42.482295989990234,
      "activations/layer21_attention_weight_min": -28.10061264038086,
      "activations/layer22_attention_weight_max": 38.511314392089844,
      "activations/layer22_attention_weight_min": -22.246788024902344,
      "activations/layer23_attention_weight_max": 24.031774520874023,
      "activations/layer23_attention_weight_min": -15.57380485534668,
      "activations/layer2_attention_weight_max": 28.851764678955078,
      "activations/layer2_attention_weight_min": -28.014793395996094,
      "activations/layer3_attention_weight_max": 71.4867172241211,
      "activations/layer3_attention_weight_min": -71.80479431152344,
      "activations/layer4_attention_weight_max": 100.84821319580078,
      "activations/layer4_attention_weight_min": -89.08540344238281,
      "activations/layer5_attention_weight_max": 130.06204223632812,
      "activations/layer5_attention_weight_min": -103.41280364990234,
      "activations/layer6_attention_weight_max": 65.48616027832031,
      "activations/layer6_attention_weight_min": -51.216575622558594,
      "activations/layer7_attention_weight_max": 64.1496810913086,
      "activations/layer7_attention_weight_min": -55.260765075683594,
      "activations/layer8_attention_weight_max": 48.23165512084961,
      "activations/layer8_attention_weight_min": -36.353641510009766,
      "activations/layer9_attention_weight_max": 66.16630554199219,
      "activations/layer9_attention_weight_min": -51.747371673583984,
      "epoch": 2.55,
      "learning_rate": 0.00013488863636363635,
      "loss": 2.9215,
      "step": 43950
    },
    {
      "activations/layer0_attention_weight_max": 14.673501014709473,
      "activations/layer0_attention_weight_min": -14.215387344360352,
      "activations/layer10_attention_weight_max": 49.45589065551758,
      "activations/layer10_attention_weight_min": -44.08417892456055,
      "activations/layer11_attention_weight_max": 36.108612060546875,
      "activations/layer11_attention_weight_min": -30.742725372314453,
      "activations/layer12_attention_weight_max": 26.699655532836914,
      "activations/layer12_attention_weight_min": -25.380746841430664,
      "activations/layer13_attention_weight_max": 61.531455993652344,
      "activations/layer13_attention_weight_min": -46.05823516845703,
      "activations/layer14_attention_weight_max": 70.08134460449219,
      "activations/layer14_attention_weight_min": -41.65810012817383,
      "activations/layer15_attention_weight_max": 49.38079071044922,
      "activations/layer15_attention_weight_min": -36.56777572631836,
      "activations/layer16_attention_weight_max": 39.579254150390625,
      "activations/layer16_attention_weight_min": -27.27975082397461,
      "activations/layer17_attention_weight_max": 65.85865783691406,
      "activations/layer17_attention_weight_min": -39.35048294067383,
      "activations/layer18_attention_weight_max": 52.319358825683594,
      "activations/layer18_attention_weight_min": -32.99058151245117,
      "activations/layer19_attention_weight_max": 20.607316970825195,
      "activations/layer19_attention_weight_min": -18.012903213500977,
      "activations/layer1_attention_weight_max": 16.464595794677734,
      "activations/layer1_attention_weight_min": -16.419645309448242,
      "activations/layer20_attention_weight_max": 20.085956573486328,
      "activations/layer20_attention_weight_min": -21.32280731201172,
      "activations/layer21_attention_weight_max": 47.39908218383789,
      "activations/layer21_attention_weight_min": -27.097543716430664,
      "activations/layer22_attention_weight_max": 35.04938507080078,
      "activations/layer22_attention_weight_min": -22.179616928100586,
      "activations/layer23_attention_weight_max": 26.040695190429688,
      "activations/layer23_attention_weight_min": -17.883949279785156,
      "activations/layer2_attention_weight_max": 29.69894027709961,
      "activations/layer2_attention_weight_min": -27.846332550048828,
      "activations/layer3_attention_weight_max": 63.66582489013672,
      "activations/layer3_attention_weight_min": -69.22091674804688,
      "activations/layer4_attention_weight_max": 91.07830047607422,
      "activations/layer4_attention_weight_min": -82.0459213256836,
      "activations/layer5_attention_weight_max": 119.83089447021484,
      "activations/layer5_attention_weight_min": -104.55391693115234,
      "activations/layer6_attention_weight_max": 58.602874755859375,
      "activations/layer6_attention_weight_min": -49.90912628173828,
      "activations/layer7_attention_weight_max": 52.44913101196289,
      "activations/layer7_attention_weight_min": -53.02629852294922,
      "activations/layer8_attention_weight_max": 37.70402908325195,
      "activations/layer8_attention_weight_min": -34.46967315673828,
      "activations/layer9_attention_weight_max": 56.016693115234375,
      "activations/layer9_attention_weight_min": -47.42999267578125,
      "epoch": 2.56,
      "learning_rate": 0.00013486969696969696,
      "loss": 2.9341,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_loss": 2.87890625,
      "eval_runtime": 8.4549,
      "eval_samples_per_second": 507.87,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_openwebtext_loss": 2.87890625,
      "eval_openwebtext_ppl": 17.79479946994832,
      "eval_openwebtext_runtime": 8.4549,
      "eval_openwebtext_samples_per_second": 507.87,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_wikitext_loss": 3.13671875,
      "eval_wikitext_ppl": 23.028181534732802,
      "eval_wikitext_runtime": 1.9419,
      "eval_wikitext_samples_per_second": 234.816,
      "step": 44000
    },
    {
      "epoch": 2.56,
      "eval_lambada_loss": 3.087890625,
      "eval_lambada_ppl": 21.930768935644906,
      "eval_lambada_runtime": 9.5712,
      "eval_lambada_samples_per_second": 508.712,
      "step": 44000
    },
    {
      "activations/layer0_attention_weight_max": 14.373072624206543,
      "activations/layer0_attention_weight_min": -14.248544692993164,
      "activations/layer10_attention_weight_max": 50.06414794921875,
      "activations/layer10_attention_weight_min": -38.6221809387207,
      "activations/layer11_attention_weight_max": 38.806884765625,
      "activations/layer11_attention_weight_min": -30.166173934936523,
      "activations/layer12_attention_weight_max": 33.98045349121094,
      "activations/layer12_attention_weight_min": -27.674415588378906,
      "activations/layer13_attention_weight_max": 60.527381896972656,
      "activations/layer13_attention_weight_min": -43.48249816894531,
      "activations/layer14_attention_weight_max": 52.04769515991211,
      "activations/layer14_attention_weight_min": -36.41417694091797,
      "activations/layer15_attention_weight_max": 56.065025329589844,
      "activations/layer15_attention_weight_min": -37.25656509399414,
      "activations/layer16_attention_weight_max": 44.84212875366211,
      "activations/layer16_attention_weight_min": -28.806137084960938,
      "activations/layer17_attention_weight_max": 62.42633056640625,
      "activations/layer17_attention_weight_min": -36.9440803527832,
      "activations/layer18_attention_weight_max": 54.194549560546875,
      "activations/layer18_attention_weight_min": -32.29241180419922,
      "activations/layer19_attention_weight_max": 22.843664169311523,
      "activations/layer19_attention_weight_min": -17.73116111755371,
      "activations/layer1_attention_weight_max": 16.74742889404297,
      "activations/layer1_attention_weight_min": -13.324210166931152,
      "activations/layer20_attention_weight_max": 23.26331901550293,
      "activations/layer20_attention_weight_min": -19.291763305664062,
      "activations/layer21_attention_weight_max": 44.21583938598633,
      "activations/layer21_attention_weight_min": -32.37370681762695,
      "activations/layer22_attention_weight_max": 35.638816833496094,
      "activations/layer22_attention_weight_min": -22.98700523376465,
      "activations/layer23_attention_weight_max": 26.46750259399414,
      "activations/layer23_attention_weight_min": -17.22730255126953,
      "activations/layer2_attention_weight_max": 28.789566040039062,
      "activations/layer2_attention_weight_min": -26.489356994628906,
      "activations/layer3_attention_weight_max": 66.6428451538086,
      "activations/layer3_attention_weight_min": -69.28255462646484,
      "activations/layer4_attention_weight_max": 98.50721740722656,
      "activations/layer4_attention_weight_min": -90.31182861328125,
      "activations/layer5_attention_weight_max": 132.09378051757812,
      "activations/layer5_attention_weight_min": -103.35823059082031,
      "activations/layer6_attention_weight_max": 66.95382690429688,
      "activations/layer6_attention_weight_min": -52.6905632019043,
      "activations/layer7_attention_weight_max": 59.01370620727539,
      "activations/layer7_attention_weight_min": -51.897918701171875,
      "activations/layer8_attention_weight_max": 40.056217193603516,
      "activations/layer8_attention_weight_min": -36.588768005371094,
      "activations/layer9_attention_weight_max": 53.69392776489258,
      "activations/layer9_attention_weight_min": -51.509185791015625,
      "epoch": 2.56,
      "learning_rate": 0.00013485075757575758,
      "loss": 2.9258,
      "step": 44050
    },
    {
      "activations/layer0_attention_weight_max": 14.651915550231934,
      "activations/layer0_attention_weight_min": -14.858373641967773,
      "activations/layer10_attention_weight_max": 50.00706481933594,
      "activations/layer10_attention_weight_min": -39.679237365722656,
      "activations/layer11_attention_weight_max": 42.104522705078125,
      "activations/layer11_attention_weight_min": -34.01422882080078,
      "activations/layer12_attention_weight_max": 29.528573989868164,
      "activations/layer12_attention_weight_min": -23.71950912475586,
      "activations/layer13_attention_weight_max": 62.816253662109375,
      "activations/layer13_attention_weight_min": -44.163265228271484,
      "activations/layer14_attention_weight_max": 71.5934066772461,
      "activations/layer14_attention_weight_min": -42.95405960083008,
      "activations/layer15_attention_weight_max": 52.90599822998047,
      "activations/layer15_attention_weight_min": -37.612091064453125,
      "activations/layer16_attention_weight_max": 40.26286315917969,
      "activations/layer16_attention_weight_min": -27.896465301513672,
      "activations/layer17_attention_weight_max": 59.66873550415039,
      "activations/layer17_attention_weight_min": -35.57882308959961,
      "activations/layer18_attention_weight_max": 58.21780776977539,
      "activations/layer18_attention_weight_min": -33.57167053222656,
      "activations/layer19_attention_weight_max": 23.880237579345703,
      "activations/layer19_attention_weight_min": -16.18987464904785,
      "activations/layer1_attention_weight_max": 17.752735137939453,
      "activations/layer1_attention_weight_min": -16.551021575927734,
      "activations/layer20_attention_weight_max": 21.81800079345703,
      "activations/layer20_attention_weight_min": -18.061996459960938,
      "activations/layer21_attention_weight_max": 42.16838073730469,
      "activations/layer21_attention_weight_min": -26.520370483398438,
      "activations/layer22_attention_weight_max": 34.611473083496094,
      "activations/layer22_attention_weight_min": -23.56220817565918,
      "activations/layer23_attention_weight_max": 22.84453582763672,
      "activations/layer23_attention_weight_min": -14.816397666931152,
      "activations/layer2_attention_weight_max": 28.621261596679688,
      "activations/layer2_attention_weight_min": -26.82271385192871,
      "activations/layer3_attention_weight_max": 71.86517333984375,
      "activations/layer3_attention_weight_min": -72.15331268310547,
      "activations/layer4_attention_weight_max": 103.96092987060547,
      "activations/layer4_attention_weight_min": -91.17330169677734,
      "activations/layer5_attention_weight_max": 127.12823486328125,
      "activations/layer5_attention_weight_min": -108.68621826171875,
      "activations/layer6_attention_weight_max": 63.65971374511719,
      "activations/layer6_attention_weight_min": -53.60361099243164,
      "activations/layer7_attention_weight_max": 59.084163665771484,
      "activations/layer7_attention_weight_min": -59.43471908569336,
      "activations/layer8_attention_weight_max": 47.78398513793945,
      "activations/layer8_attention_weight_min": -39.578216552734375,
      "activations/layer9_attention_weight_max": 60.33596420288086,
      "activations/layer9_attention_weight_min": -70.96392822265625,
      "epoch": 2.56,
      "learning_rate": 0.00013483181818181816,
      "loss": 2.9375,
      "step": 44100
    },
    {
      "activations/layer0_attention_weight_max": 14.907041549682617,
      "activations/layer0_attention_weight_min": -14.32940673828125,
      "activations/layer10_attention_weight_max": 49.99323654174805,
      "activations/layer10_attention_weight_min": -39.7421875,
      "activations/layer11_attention_weight_max": 39.03143310546875,
      "activations/layer11_attention_weight_min": -32.63180160522461,
      "activations/layer12_attention_weight_max": 26.676124572753906,
      "activations/layer12_attention_weight_min": -28.772022247314453,
      "activations/layer13_attention_weight_max": 60.03490447998047,
      "activations/layer13_attention_weight_min": -38.18437194824219,
      "activations/layer14_attention_weight_max": 62.62207794189453,
      "activations/layer14_attention_weight_min": -40.85945129394531,
      "activations/layer15_attention_weight_max": 50.80876922607422,
      "activations/layer15_attention_weight_min": -34.17350769042969,
      "activations/layer16_attention_weight_max": 39.262901306152344,
      "activations/layer16_attention_weight_min": -28.78274917602539,
      "activations/layer17_attention_weight_max": 65.40787506103516,
      "activations/layer17_attention_weight_min": -37.94435501098633,
      "activations/layer18_attention_weight_max": 57.038639068603516,
      "activations/layer18_attention_weight_min": -32.6164665222168,
      "activations/layer19_attention_weight_max": 21.788978576660156,
      "activations/layer19_attention_weight_min": -16.730594635009766,
      "activations/layer1_attention_weight_max": 16.46967315673828,
      "activations/layer1_attention_weight_min": -13.136279106140137,
      "activations/layer20_attention_weight_max": 23.91451644897461,
      "activations/layer20_attention_weight_min": -19.661344528198242,
      "activations/layer21_attention_weight_max": 42.5680046081543,
      "activations/layer21_attention_weight_min": -26.620176315307617,
      "activations/layer22_attention_weight_max": 37.104488372802734,
      "activations/layer22_attention_weight_min": -23.68210792541504,
      "activations/layer23_attention_weight_max": 24.84871482849121,
      "activations/layer23_attention_weight_min": -18.116147994995117,
      "activations/layer2_attention_weight_max": 29.227903366088867,
      "activations/layer2_attention_weight_min": -27.59760093688965,
      "activations/layer3_attention_weight_max": 65.82540130615234,
      "activations/layer3_attention_weight_min": -68.48918914794922,
      "activations/layer4_attention_weight_max": 96.54285430908203,
      "activations/layer4_attention_weight_min": -87.98028564453125,
      "activations/layer5_attention_weight_max": 129.1863555908203,
      "activations/layer5_attention_weight_min": -100.4671859741211,
      "activations/layer6_attention_weight_max": 62.18412780761719,
      "activations/layer6_attention_weight_min": -49.939273834228516,
      "activations/layer7_attention_weight_max": 55.60544967651367,
      "activations/layer7_attention_weight_min": -50.97825622558594,
      "activations/layer8_attention_weight_max": 43.80097198486328,
      "activations/layer8_attention_weight_min": -38.742610931396484,
      "activations/layer9_attention_weight_max": 56.49129104614258,
      "activations/layer9_attention_weight_min": -50.61337661743164,
      "epoch": 2.57,
      "learning_rate": 0.00013481287878787878,
      "loss": 2.9292,
      "step": 44150
    },
    {
      "activations/layer0_attention_weight_max": 14.994647979736328,
      "activations/layer0_attention_weight_min": -15.070298194885254,
      "activations/layer10_attention_weight_max": 59.373138427734375,
      "activations/layer10_attention_weight_min": -38.48819351196289,
      "activations/layer11_attention_weight_max": 41.84051513671875,
      "activations/layer11_attention_weight_min": -31.610076904296875,
      "activations/layer12_attention_weight_max": 26.775392532348633,
      "activations/layer12_attention_weight_min": -24.343341827392578,
      "activations/layer13_attention_weight_max": 89.58122253417969,
      "activations/layer13_attention_weight_min": -46.51789093017578,
      "activations/layer14_attention_weight_max": 90.9657974243164,
      "activations/layer14_attention_weight_min": -45.008060455322266,
      "activations/layer15_attention_weight_max": 66.60014343261719,
      "activations/layer15_attention_weight_min": -35.464942932128906,
      "activations/layer16_attention_weight_max": 55.52040100097656,
      "activations/layer16_attention_weight_min": -27.499963760375977,
      "activations/layer17_attention_weight_max": 71.89359283447266,
      "activations/layer17_attention_weight_min": -33.7082405090332,
      "activations/layer18_attention_weight_max": 64.96875762939453,
      "activations/layer18_attention_weight_min": -30.359073638916016,
      "activations/layer19_attention_weight_max": 28.950109481811523,
      "activations/layer19_attention_weight_min": -16.17683982849121,
      "activations/layer1_attention_weight_max": 17.985790252685547,
      "activations/layer1_attention_weight_min": -13.347618103027344,
      "activations/layer20_attention_weight_max": 22.761857986450195,
      "activations/layer20_attention_weight_min": -18.247013092041016,
      "activations/layer21_attention_weight_max": 51.0319709777832,
      "activations/layer21_attention_weight_min": -22.05887794494629,
      "activations/layer22_attention_weight_max": 48.531639099121094,
      "activations/layer22_attention_weight_min": -20.92017936706543,
      "activations/layer23_attention_weight_max": 26.24652099609375,
      "activations/layer23_attention_weight_min": -15.737859725952148,
      "activations/layer2_attention_weight_max": 29.289974212646484,
      "activations/layer2_attention_weight_min": -26.191150665283203,
      "activations/layer3_attention_weight_max": 72.44722747802734,
      "activations/layer3_attention_weight_min": -70.7099838256836,
      "activations/layer4_attention_weight_max": 105.40138244628906,
      "activations/layer4_attention_weight_min": -87.52790069580078,
      "activations/layer5_attention_weight_max": 148.55377197265625,
      "activations/layer5_attention_weight_min": -104.05709075927734,
      "activations/layer6_attention_weight_max": 69.85218811035156,
      "activations/layer6_attention_weight_min": -51.972511291503906,
      "activations/layer7_attention_weight_max": 62.40144729614258,
      "activations/layer7_attention_weight_min": -55.90653610229492,
      "activations/layer8_attention_weight_max": 46.44761276245117,
      "activations/layer8_attention_weight_min": -36.01544189453125,
      "activations/layer9_attention_weight_max": 62.0725212097168,
      "activations/layer9_attention_weight_min": -52.73969650268555,
      "epoch": 2.57,
      "learning_rate": 0.0001347939393939394,
      "loss": 2.9313,
      "step": 44200
    },
    {
      "activations/layer0_attention_weight_max": 14.748847961425781,
      "activations/layer0_attention_weight_min": -15.040273666381836,
      "activations/layer10_attention_weight_max": 47.6994743347168,
      "activations/layer10_attention_weight_min": -45.03489303588867,
      "activations/layer11_attention_weight_max": 37.93819808959961,
      "activations/layer11_attention_weight_min": -29.169761657714844,
      "activations/layer12_attention_weight_max": 27.133590698242188,
      "activations/layer12_attention_weight_min": -25.006967544555664,
      "activations/layer13_attention_weight_max": 51.38850402832031,
      "activations/layer13_attention_weight_min": -37.54133987426758,
      "activations/layer14_attention_weight_max": 53.21135711669922,
      "activations/layer14_attention_weight_min": -34.57279586791992,
      "activations/layer15_attention_weight_max": 47.27641677856445,
      "activations/layer15_attention_weight_min": -36.654762268066406,
      "activations/layer16_attention_weight_max": 41.404056549072266,
      "activations/layer16_attention_weight_min": -28.247577667236328,
      "activations/layer17_attention_weight_max": 57.73286819458008,
      "activations/layer17_attention_weight_min": -37.54501724243164,
      "activations/layer18_attention_weight_max": 53.06125259399414,
      "activations/layer18_attention_weight_min": -32.390045166015625,
      "activations/layer19_attention_weight_max": 20.704383850097656,
      "activations/layer19_attention_weight_min": -18.543439865112305,
      "activations/layer1_attention_weight_max": 17.511629104614258,
      "activations/layer1_attention_weight_min": -13.364151000976562,
      "activations/layer20_attention_weight_max": 19.770231246948242,
      "activations/layer20_attention_weight_min": -17.67906951904297,
      "activations/layer21_attention_weight_max": 48.431236267089844,
      "activations/layer21_attention_weight_min": -29.875,
      "activations/layer22_attention_weight_max": 33.971309661865234,
      "activations/layer22_attention_weight_min": -22.90326690673828,
      "activations/layer23_attention_weight_max": 21.797073364257812,
      "activations/layer23_attention_weight_min": -14.899784088134766,
      "activations/layer2_attention_weight_max": 29.780750274658203,
      "activations/layer2_attention_weight_min": -26.664230346679688,
      "activations/layer3_attention_weight_max": 66.31200408935547,
      "activations/layer3_attention_weight_min": -67.24329376220703,
      "activations/layer4_attention_weight_max": 93.15677642822266,
      "activations/layer4_attention_weight_min": -84.04383850097656,
      "activations/layer5_attention_weight_max": 118.26555633544922,
      "activations/layer5_attention_weight_min": -96.57699584960938,
      "activations/layer6_attention_weight_max": 57.51681900024414,
      "activations/layer6_attention_weight_min": -51.697391510009766,
      "activations/layer7_attention_weight_max": 56.31000900268555,
      "activations/layer7_attention_weight_min": -50.36869430541992,
      "activations/layer8_attention_weight_max": 41.84103012084961,
      "activations/layer8_attention_weight_min": -34.353736877441406,
      "activations/layer9_attention_weight_max": 58.476680755615234,
      "activations/layer9_attention_weight_min": -47.0064582824707,
      "epoch": 2.57,
      "learning_rate": 0.00013477499999999998,
      "loss": 2.9282,
      "step": 44250
    },
    {
      "activations/layer0_attention_weight_max": 15.938860893249512,
      "activations/layer0_attention_weight_min": -14.436356544494629,
      "activations/layer10_attention_weight_max": 51.17710494995117,
      "activations/layer10_attention_weight_min": -41.14008712768555,
      "activations/layer11_attention_weight_max": 39.136451721191406,
      "activations/layer11_attention_weight_min": -30.707395553588867,
      "activations/layer12_attention_weight_max": 24.061399459838867,
      "activations/layer12_attention_weight_min": -25.983261108398438,
      "activations/layer13_attention_weight_max": 60.60697555541992,
      "activations/layer13_attention_weight_min": -45.084205627441406,
      "activations/layer14_attention_weight_max": 55.947940826416016,
      "activations/layer14_attention_weight_min": -34.80388641357422,
      "activations/layer15_attention_weight_max": 56.96804428100586,
      "activations/layer15_attention_weight_min": -34.520572662353516,
      "activations/layer16_attention_weight_max": 38.73279571533203,
      "activations/layer16_attention_weight_min": -28.32999038696289,
      "activations/layer17_attention_weight_max": 63.02350616455078,
      "activations/layer17_attention_weight_min": -36.731563568115234,
      "activations/layer18_attention_weight_max": 53.76994705200195,
      "activations/layer18_attention_weight_min": -32.33930587768555,
      "activations/layer19_attention_weight_max": 20.769058227539062,
      "activations/layer19_attention_weight_min": -18.696775436401367,
      "activations/layer1_attention_weight_max": 16.6176700592041,
      "activations/layer1_attention_weight_min": -18.392290115356445,
      "activations/layer20_attention_weight_max": 20.646568298339844,
      "activations/layer20_attention_weight_min": -17.807292938232422,
      "activations/layer21_attention_weight_max": 45.405887603759766,
      "activations/layer21_attention_weight_min": -28.324338912963867,
      "activations/layer22_attention_weight_max": 35.61689758300781,
      "activations/layer22_attention_weight_min": -22.021873474121094,
      "activations/layer23_attention_weight_max": 24.915180206298828,
      "activations/layer23_attention_weight_min": -17.074111938476562,
      "activations/layer2_attention_weight_max": 29.090885162353516,
      "activations/layer2_attention_weight_min": -27.851863861083984,
      "activations/layer3_attention_weight_max": 67.77840423583984,
      "activations/layer3_attention_weight_min": -71.91016387939453,
      "activations/layer4_attention_weight_max": 97.66551208496094,
      "activations/layer4_attention_weight_min": -89.38687896728516,
      "activations/layer5_attention_weight_max": 121.90086364746094,
      "activations/layer5_attention_weight_min": -104.73017883300781,
      "activations/layer6_attention_weight_max": 61.831581115722656,
      "activations/layer6_attention_weight_min": -54.3575553894043,
      "activations/layer7_attention_weight_max": 60.86439895629883,
      "activations/layer7_attention_weight_min": -53.514713287353516,
      "activations/layer8_attention_weight_max": 42.289794921875,
      "activations/layer8_attention_weight_min": -36.4107780456543,
      "activations/layer9_attention_weight_max": 55.38715362548828,
      "activations/layer9_attention_weight_min": -49.46574401855469,
      "epoch": 2.57,
      "learning_rate": 0.0001347560606060606,
      "loss": 2.9426,
      "step": 44300
    },
    {
      "activations/layer0_attention_weight_max": 16.594417572021484,
      "activations/layer0_attention_weight_min": -13.732237815856934,
      "activations/layer10_attention_weight_max": 50.993125915527344,
      "activations/layer10_attention_weight_min": -44.62145233154297,
      "activations/layer11_attention_weight_max": 40.902828216552734,
      "activations/layer11_attention_weight_min": -34.84578323364258,
      "activations/layer12_attention_weight_max": 36.06105422973633,
      "activations/layer12_attention_weight_min": -27.166263580322266,
      "activations/layer13_attention_weight_max": 51.9465446472168,
      "activations/layer13_attention_weight_min": -42.682167053222656,
      "activations/layer14_attention_weight_max": 40.54546356201172,
      "activations/layer14_attention_weight_min": -29.97623634338379,
      "activations/layer15_attention_weight_max": 44.7301025390625,
      "activations/layer15_attention_weight_min": -35.062007904052734,
      "activations/layer16_attention_weight_max": 33.162540435791016,
      "activations/layer16_attention_weight_min": -28.36144256591797,
      "activations/layer17_attention_weight_max": 56.2870979309082,
      "activations/layer17_attention_weight_min": -37.8194465637207,
      "activations/layer18_attention_weight_max": 47.812896728515625,
      "activations/layer18_attention_weight_min": -32.395076751708984,
      "activations/layer19_attention_weight_max": 18.164777755737305,
      "activations/layer19_attention_weight_min": -17.56561851501465,
      "activations/layer1_attention_weight_max": 17.6353702545166,
      "activations/layer1_attention_weight_min": -14.297542572021484,
      "activations/layer20_attention_weight_max": 17.366153717041016,
      "activations/layer20_attention_weight_min": -19.24134063720703,
      "activations/layer21_attention_weight_max": 41.78300476074219,
      "activations/layer21_attention_weight_min": -31.143966674804688,
      "activations/layer22_attention_weight_max": 29.866561889648438,
      "activations/layer22_attention_weight_min": -24.35441780090332,
      "activations/layer23_attention_weight_max": 22.64453887939453,
      "activations/layer23_attention_weight_min": -15.464056968688965,
      "activations/layer2_attention_weight_max": 28.685331344604492,
      "activations/layer2_attention_weight_min": -27.794124603271484,
      "activations/layer3_attention_weight_max": 69.18585205078125,
      "activations/layer3_attention_weight_min": -70.45464324951172,
      "activations/layer4_attention_weight_max": 98.9060287475586,
      "activations/layer4_attention_weight_min": -86.20600128173828,
      "activations/layer5_attention_weight_max": 128.61053466796875,
      "activations/layer5_attention_weight_min": -102.4200668334961,
      "activations/layer6_attention_weight_max": 63.16252136230469,
      "activations/layer6_attention_weight_min": -55.292991638183594,
      "activations/layer7_attention_weight_max": 58.300193786621094,
      "activations/layer7_attention_weight_min": -53.779815673828125,
      "activations/layer8_attention_weight_max": 42.290504455566406,
      "activations/layer8_attention_weight_min": -36.11033630371094,
      "activations/layer9_attention_weight_max": 63.22119903564453,
      "activations/layer9_attention_weight_min": -53.211605072021484,
      "epoch": 2.58,
      "learning_rate": 0.0001347371212121212,
      "loss": 2.9294,
      "step": 44350
    },
    {
      "activations/layer0_attention_weight_max": 13.860968589782715,
      "activations/layer0_attention_weight_min": -13.572155952453613,
      "activations/layer10_attention_weight_max": 50.470672607421875,
      "activations/layer10_attention_weight_min": -42.7773551940918,
      "activations/layer11_attention_weight_max": 38.99689483642578,
      "activations/layer11_attention_weight_min": -29.86922836303711,
      "activations/layer12_attention_weight_max": 23.814970016479492,
      "activations/layer12_attention_weight_min": -28.259082794189453,
      "activations/layer13_attention_weight_max": 52.6004753112793,
      "activations/layer13_attention_weight_min": -34.99974822998047,
      "activations/layer14_attention_weight_max": 47.75166320800781,
      "activations/layer14_attention_weight_min": -32.18439865112305,
      "activations/layer15_attention_weight_max": 48.90902328491211,
      "activations/layer15_attention_weight_min": -34.266944885253906,
      "activations/layer16_attention_weight_max": 40.10251235961914,
      "activations/layer16_attention_weight_min": -24.961063385009766,
      "activations/layer17_attention_weight_max": 62.40667724609375,
      "activations/layer17_attention_weight_min": -38.023681640625,
      "activations/layer18_attention_weight_max": 48.85018539428711,
      "activations/layer18_attention_weight_min": -30.21824836730957,
      "activations/layer19_attention_weight_max": 22.109949111938477,
      "activations/layer19_attention_weight_min": -16.32691764831543,
      "activations/layer1_attention_weight_max": 17.43483543395996,
      "activations/layer1_attention_weight_min": -16.523605346679688,
      "activations/layer20_attention_weight_max": 20.57189178466797,
      "activations/layer20_attention_weight_min": -19.36741828918457,
      "activations/layer21_attention_weight_max": 38.33333206176758,
      "activations/layer21_attention_weight_min": -28.397024154663086,
      "activations/layer22_attention_weight_max": 32.674861907958984,
      "activations/layer22_attention_weight_min": -23.33745574951172,
      "activations/layer23_attention_weight_max": 24.2340087890625,
      "activations/layer23_attention_weight_min": -15.428327560424805,
      "activations/layer2_attention_weight_max": 28.526203155517578,
      "activations/layer2_attention_weight_min": -26.958984375,
      "activations/layer3_attention_weight_max": 67.70777893066406,
      "activations/layer3_attention_weight_min": -69.10074615478516,
      "activations/layer4_attention_weight_max": 100.53813934326172,
      "activations/layer4_attention_weight_min": -86.74073028564453,
      "activations/layer5_attention_weight_max": 130.0173797607422,
      "activations/layer5_attention_weight_min": -102.88261413574219,
      "activations/layer6_attention_weight_max": 63.36785125732422,
      "activations/layer6_attention_weight_min": -53.87348937988281,
      "activations/layer7_attention_weight_max": 55.368526458740234,
      "activations/layer7_attention_weight_min": -52.795623779296875,
      "activations/layer8_attention_weight_max": 40.33988571166992,
      "activations/layer8_attention_weight_min": -34.90358352661133,
      "activations/layer9_attention_weight_max": 58.4009895324707,
      "activations/layer9_attention_weight_min": -52.88900375366211,
      "epoch": 2.58,
      "learning_rate": 0.0001347181818181818,
      "loss": 2.933,
      "step": 44400
    },
    {
      "activations/layer0_attention_weight_max": 14.967536926269531,
      "activations/layer0_attention_weight_min": -14.020276069641113,
      "activations/layer10_attention_weight_max": 53.873199462890625,
      "activations/layer10_attention_weight_min": -37.77594757080078,
      "activations/layer11_attention_weight_max": 46.55418014526367,
      "activations/layer11_attention_weight_min": -31.65966796875,
      "activations/layer12_attention_weight_max": 29.142515182495117,
      "activations/layer12_attention_weight_min": -24.42028045654297,
      "activations/layer13_attention_weight_max": 54.89570617675781,
      "activations/layer13_attention_weight_min": -40.262264251708984,
      "activations/layer14_attention_weight_max": 42.64064025878906,
      "activations/layer14_attention_weight_min": -31.08649253845215,
      "activations/layer15_attention_weight_max": 51.0769157409668,
      "activations/layer15_attention_weight_min": -34.666847229003906,
      "activations/layer16_attention_weight_max": 43.98862075805664,
      "activations/layer16_attention_weight_min": -29.490957260131836,
      "activations/layer17_attention_weight_max": 63.39950942993164,
      "activations/layer17_attention_weight_min": -37.706825256347656,
      "activations/layer18_attention_weight_max": 63.33051681518555,
      "activations/layer18_attention_weight_min": -31.56498908996582,
      "activations/layer19_attention_weight_max": 23.079282760620117,
      "activations/layer19_attention_weight_min": -16.872459411621094,
      "activations/layer1_attention_weight_max": 16.813447952270508,
      "activations/layer1_attention_weight_min": -14.123812675476074,
      "activations/layer20_attention_weight_max": 21.61716079711914,
      "activations/layer20_attention_weight_min": -18.936141967773438,
      "activations/layer21_attention_weight_max": 45.84989547729492,
      "activations/layer21_attention_weight_min": -27.61183738708496,
      "activations/layer22_attention_weight_max": 38.981163024902344,
      "activations/layer22_attention_weight_min": -24.352630615234375,
      "activations/layer23_attention_weight_max": 28.4984130859375,
      "activations/layer23_attention_weight_min": -17.84532928466797,
      "activations/layer2_attention_weight_max": 27.759008407592773,
      "activations/layer2_attention_weight_min": -24.782569885253906,
      "activations/layer3_attention_weight_max": 68.88489532470703,
      "activations/layer3_attention_weight_min": -71.53720092773438,
      "activations/layer4_attention_weight_max": 97.89493560791016,
      "activations/layer4_attention_weight_min": -88.7532730102539,
      "activations/layer5_attention_weight_max": 128.34426879882812,
      "activations/layer5_attention_weight_min": -97.82955932617188,
      "activations/layer6_attention_weight_max": 63.2750358581543,
      "activations/layer6_attention_weight_min": -50.36056900024414,
      "activations/layer7_attention_weight_max": 59.12090301513672,
      "activations/layer7_attention_weight_min": -51.7177619934082,
      "activations/layer8_attention_weight_max": 44.499488830566406,
      "activations/layer8_attention_weight_min": -35.68025207519531,
      "activations/layer9_attention_weight_max": 56.422420501708984,
      "activations/layer9_attention_weight_min": -49.48616027832031,
      "epoch": 2.58,
      "learning_rate": 0.00013469924242424242,
      "loss": 2.9258,
      "step": 44450
    },
    {
      "activations/layer0_attention_weight_max": 14.583882331848145,
      "activations/layer0_attention_weight_min": -13.891562461853027,
      "activations/layer10_attention_weight_max": 53.262630462646484,
      "activations/layer10_attention_weight_min": -42.508270263671875,
      "activations/layer11_attention_weight_max": 44.13075256347656,
      "activations/layer11_attention_weight_min": -31.053119659423828,
      "activations/layer12_attention_weight_max": 28.079431533813477,
      "activations/layer12_attention_weight_min": -26.37009048461914,
      "activations/layer13_attention_weight_max": 53.09355926513672,
      "activations/layer13_attention_weight_min": -36.209716796875,
      "activations/layer14_attention_weight_max": 46.16038131713867,
      "activations/layer14_attention_weight_min": -33.04309844970703,
      "activations/layer15_attention_weight_max": 46.92230987548828,
      "activations/layer15_attention_weight_min": -33.42824935913086,
      "activations/layer16_attention_weight_max": 45.554969787597656,
      "activations/layer16_attention_weight_min": -28.655431747436523,
      "activations/layer17_attention_weight_max": 68.33646392822266,
      "activations/layer17_attention_weight_min": -37.703125,
      "activations/layer18_attention_weight_max": 56.23139572143555,
      "activations/layer18_attention_weight_min": -31.82258415222168,
      "activations/layer19_attention_weight_max": 22.187793731689453,
      "activations/layer19_attention_weight_min": -18.13020896911621,
      "activations/layer1_attention_weight_max": 17.193607330322266,
      "activations/layer1_attention_weight_min": -12.98814582824707,
      "activations/layer20_attention_weight_max": 18.648303985595703,
      "activations/layer20_attention_weight_min": -17.052316665649414,
      "activations/layer21_attention_weight_max": 40.61980056762695,
      "activations/layer21_attention_weight_min": -27.08598518371582,
      "activations/layer22_attention_weight_max": 37.17148208618164,
      "activations/layer22_attention_weight_min": -23.96120262145996,
      "activations/layer23_attention_weight_max": 22.65790557861328,
      "activations/layer23_attention_weight_min": -16.773239135742188,
      "activations/layer2_attention_weight_max": 29.41150665283203,
      "activations/layer2_attention_weight_min": -26.296293258666992,
      "activations/layer3_attention_weight_max": 71.38723754882812,
      "activations/layer3_attention_weight_min": -73.36067962646484,
      "activations/layer4_attention_weight_max": 97.14641571044922,
      "activations/layer4_attention_weight_min": -88.81951141357422,
      "activations/layer5_attention_weight_max": 122.19792938232422,
      "activations/layer5_attention_weight_min": -99.54129028320312,
      "activations/layer6_attention_weight_max": 61.825965881347656,
      "activations/layer6_attention_weight_min": -51.801509857177734,
      "activations/layer7_attention_weight_max": 61.80485534667969,
      "activations/layer7_attention_weight_min": -54.49360275268555,
      "activations/layer8_attention_weight_max": 40.76224136352539,
      "activations/layer8_attention_weight_min": -36.52281951904297,
      "activations/layer9_attention_weight_max": 56.20115280151367,
      "activations/layer9_attention_weight_min": -49.04402160644531,
      "epoch": 2.59,
      "learning_rate": 0.000134680303030303,
      "loss": 2.9431,
      "step": 44500
    },
    {
      "activations/layer0_attention_weight_max": 14.352008819580078,
      "activations/layer0_attention_weight_min": -13.514909744262695,
      "activations/layer10_attention_weight_max": 48.568843841552734,
      "activations/layer10_attention_weight_min": -41.70466995239258,
      "activations/layer11_attention_weight_max": 38.668792724609375,
      "activations/layer11_attention_weight_min": -29.821149826049805,
      "activations/layer12_attention_weight_max": 35.15755081176758,
      "activations/layer12_attention_weight_min": -29.016576766967773,
      "activations/layer13_attention_weight_max": 54.54179763793945,
      "activations/layer13_attention_weight_min": -39.028255462646484,
      "activations/layer14_attention_weight_max": 35.9082145690918,
      "activations/layer14_attention_weight_min": -28.802804946899414,
      "activations/layer15_attention_weight_max": 46.99113464355469,
      "activations/layer15_attention_weight_min": -34.78948974609375,
      "activations/layer16_attention_weight_max": 36.67523193359375,
      "activations/layer16_attention_weight_min": -25.896581649780273,
      "activations/layer17_attention_weight_max": 63.01844024658203,
      "activations/layer17_attention_weight_min": -37.944705963134766,
      "activations/layer18_attention_weight_max": 49.01669692993164,
      "activations/layer18_attention_weight_min": -30.0178165435791,
      "activations/layer19_attention_weight_max": 19.616430282592773,
      "activations/layer19_attention_weight_min": -18.32892417907715,
      "activations/layer1_attention_weight_max": 15.994837760925293,
      "activations/layer1_attention_weight_min": -12.669154167175293,
      "activations/layer20_attention_weight_max": 20.22389030456543,
      "activations/layer20_attention_weight_min": -17.695940017700195,
      "activations/layer21_attention_weight_max": 38.56801223754883,
      "activations/layer21_attention_weight_min": -28.377025604248047,
      "activations/layer22_attention_weight_max": 33.404640197753906,
      "activations/layer22_attention_weight_min": -22.773406982421875,
      "activations/layer23_attention_weight_max": 25.159557342529297,
      "activations/layer23_attention_weight_min": -14.016799926757812,
      "activations/layer2_attention_weight_max": 29.740642547607422,
      "activations/layer2_attention_weight_min": -28.241466522216797,
      "activations/layer3_attention_weight_max": 68.87120056152344,
      "activations/layer3_attention_weight_min": -72.39906311035156,
      "activations/layer4_attention_weight_max": 98.42147064208984,
      "activations/layer4_attention_weight_min": -87.2962875366211,
      "activations/layer5_attention_weight_max": 118.86001586914062,
      "activations/layer5_attention_weight_min": -96.90419006347656,
      "activations/layer6_attention_weight_max": 59.25526809692383,
      "activations/layer6_attention_weight_min": -51.60319137573242,
      "activations/layer7_attention_weight_max": 57.257965087890625,
      "activations/layer7_attention_weight_min": -55.36627197265625,
      "activations/layer8_attention_weight_max": 42.04337692260742,
      "activations/layer8_attention_weight_min": -36.526798248291016,
      "activations/layer9_attention_weight_max": 56.80279541015625,
      "activations/layer9_attention_weight_min": -47.50173568725586,
      "epoch": 2.59,
      "learning_rate": 0.00013466136363636362,
      "loss": 2.9426,
      "step": 44550
    },
    {
      "activations/layer0_attention_weight_max": 15.051923751831055,
      "activations/layer0_attention_weight_min": -13.820868492126465,
      "activations/layer10_attention_weight_max": 52.44684600830078,
      "activations/layer10_attention_weight_min": -39.57670974731445,
      "activations/layer11_attention_weight_max": 45.569374084472656,
      "activations/layer11_attention_weight_min": -32.53243637084961,
      "activations/layer12_attention_weight_max": 29.120859146118164,
      "activations/layer12_attention_weight_min": -24.70147705078125,
      "activations/layer13_attention_weight_max": 62.50609588623047,
      "activations/layer13_attention_weight_min": -41.160091400146484,
      "activations/layer14_attention_weight_max": 65.76737213134766,
      "activations/layer14_attention_weight_min": -42.153221130371094,
      "activations/layer15_attention_weight_max": 51.73141098022461,
      "activations/layer15_attention_weight_min": -35.287471771240234,
      "activations/layer16_attention_weight_max": 39.53005599975586,
      "activations/layer16_attention_weight_min": -25.90960121154785,
      "activations/layer17_attention_weight_max": 61.27899932861328,
      "activations/layer17_attention_weight_min": -37.70462417602539,
      "activations/layer18_attention_weight_max": 53.78133773803711,
      "activations/layer18_attention_weight_min": -35.80138397216797,
      "activations/layer19_attention_weight_max": 21.5947208404541,
      "activations/layer19_attention_weight_min": -16.758541107177734,
      "activations/layer1_attention_weight_max": 16.12272834777832,
      "activations/layer1_attention_weight_min": -15.927021980285645,
      "activations/layer20_attention_weight_max": 20.751632690429688,
      "activations/layer20_attention_weight_min": -19.957679748535156,
      "activations/layer21_attention_weight_max": 47.317222595214844,
      "activations/layer21_attention_weight_min": -28.08009910583496,
      "activations/layer22_attention_weight_max": 35.134220123291016,
      "activations/layer22_attention_weight_min": -23.13764190673828,
      "activations/layer23_attention_weight_max": 23.410709381103516,
      "activations/layer23_attention_weight_min": -16.73841094970703,
      "activations/layer2_attention_weight_max": 29.89211654663086,
      "activations/layer2_attention_weight_min": -28.933929443359375,
      "activations/layer3_attention_weight_max": 69.502685546875,
      "activations/layer3_attention_weight_min": -71.73298645019531,
      "activations/layer4_attention_weight_max": 98.76921844482422,
      "activations/layer4_attention_weight_min": -87.37279510498047,
      "activations/layer5_attention_weight_max": 124.28785705566406,
      "activations/layer5_attention_weight_min": -102.23495483398438,
      "activations/layer6_attention_weight_max": 61.62883377075195,
      "activations/layer6_attention_weight_min": -52.37578201293945,
      "activations/layer7_attention_weight_max": 64.6179428100586,
      "activations/layer7_attention_weight_min": -55.31522750854492,
      "activations/layer8_attention_weight_max": 44.2746467590332,
      "activations/layer8_attention_weight_min": -36.82073211669922,
      "activations/layer9_attention_weight_max": 59.47468948364258,
      "activations/layer9_attention_weight_min": -50.12650680541992,
      "epoch": 2.59,
      "learning_rate": 0.00013464242424242423,
      "loss": 2.9197,
      "step": 44600
    },
    {
      "activations/layer0_attention_weight_max": 14.472554206848145,
      "activations/layer0_attention_weight_min": -14.283547401428223,
      "activations/layer10_attention_weight_max": 61.999019622802734,
      "activations/layer10_attention_weight_min": -42.37809371948242,
      "activations/layer11_attention_weight_max": 43.399593353271484,
      "activations/layer11_attention_weight_min": -28.897754669189453,
      "activations/layer12_attention_weight_max": 29.795108795166016,
      "activations/layer12_attention_weight_min": -25.66419219970703,
      "activations/layer13_attention_weight_max": 64.87421417236328,
      "activations/layer13_attention_weight_min": -37.31109619140625,
      "activations/layer14_attention_weight_max": 56.02638626098633,
      "activations/layer14_attention_weight_min": -36.77350997924805,
      "activations/layer15_attention_weight_max": 51.33755111694336,
      "activations/layer15_attention_weight_min": -36.68767166137695,
      "activations/layer16_attention_weight_max": 40.76566696166992,
      "activations/layer16_attention_weight_min": -31.64214324951172,
      "activations/layer17_attention_weight_max": 69.28770446777344,
      "activations/layer17_attention_weight_min": -38.33298873901367,
      "activations/layer18_attention_weight_max": 69.52183532714844,
      "activations/layer18_attention_weight_min": -32.111183166503906,
      "activations/layer19_attention_weight_max": 24.391075134277344,
      "activations/layer19_attention_weight_min": -16.981813430786133,
      "activations/layer1_attention_weight_max": 18.219316482543945,
      "activations/layer1_attention_weight_min": -13.679667472839355,
      "activations/layer20_attention_weight_max": 25.222349166870117,
      "activations/layer20_attention_weight_min": -17.978227615356445,
      "activations/layer21_attention_weight_max": 50.87508010864258,
      "activations/layer21_attention_weight_min": -29.64461898803711,
      "activations/layer22_attention_weight_max": 37.59259033203125,
      "activations/layer22_attention_weight_min": -23.969371795654297,
      "activations/layer23_attention_weight_max": 30.86530113220215,
      "activations/layer23_attention_weight_min": -16.413728713989258,
      "activations/layer2_attention_weight_max": 29.888341903686523,
      "activations/layer2_attention_weight_min": -26.919553756713867,
      "activations/layer3_attention_weight_max": 66.7911605834961,
      "activations/layer3_attention_weight_min": -71.3978500366211,
      "activations/layer4_attention_weight_max": 95.94200897216797,
      "activations/layer4_attention_weight_min": -85.23983001708984,
      "activations/layer5_attention_weight_max": 124.66241455078125,
      "activations/layer5_attention_weight_min": -101.81755828857422,
      "activations/layer6_attention_weight_max": 61.959754943847656,
      "activations/layer6_attention_weight_min": -51.25629425048828,
      "activations/layer7_attention_weight_max": 58.70082092285156,
      "activations/layer7_attention_weight_min": -54.26511764526367,
      "activations/layer8_attention_weight_max": 43.89448547363281,
      "activations/layer8_attention_weight_min": -34.62858581542969,
      "activations/layer9_attention_weight_max": 55.27857208251953,
      "activations/layer9_attention_weight_min": -50.77790451049805,
      "epoch": 2.59,
      "learning_rate": 0.00013462348484848482,
      "loss": 2.9102,
      "step": 44650
    },
    {
      "activations/layer0_attention_weight_max": 15.12147331237793,
      "activations/layer0_attention_weight_min": -14.247017860412598,
      "activations/layer10_attention_weight_max": 52.91977310180664,
      "activations/layer10_attention_weight_min": -40.32007598876953,
      "activations/layer11_attention_weight_max": 38.59729766845703,
      "activations/layer11_attention_weight_min": -28.83816146850586,
      "activations/layer12_attention_weight_max": 25.1455078125,
      "activations/layer12_attention_weight_min": -26.857295989990234,
      "activations/layer13_attention_weight_max": 57.23236083984375,
      "activations/layer13_attention_weight_min": -38.32917404174805,
      "activations/layer14_attention_weight_max": 49.965431213378906,
      "activations/layer14_attention_weight_min": -32.75804901123047,
      "activations/layer15_attention_weight_max": 49.216102600097656,
      "activations/layer15_attention_weight_min": -35.52375793457031,
      "activations/layer16_attention_weight_max": 40.694602966308594,
      "activations/layer16_attention_weight_min": -26.016233444213867,
      "activations/layer17_attention_weight_max": 63.95972442626953,
      "activations/layer17_attention_weight_min": -38.93870162963867,
      "activations/layer18_attention_weight_max": 54.60265350341797,
      "activations/layer18_attention_weight_min": -33.205291748046875,
      "activations/layer19_attention_weight_max": 23.08393096923828,
      "activations/layer19_attention_weight_min": -15.984756469726562,
      "activations/layer1_attention_weight_max": 17.035776138305664,
      "activations/layer1_attention_weight_min": -13.337099075317383,
      "activations/layer20_attention_weight_max": 22.6331844329834,
      "activations/layer20_attention_weight_min": -17.33901023864746,
      "activations/layer21_attention_weight_max": 38.70851516723633,
      "activations/layer21_attention_weight_min": -28.636287689208984,
      "activations/layer22_attention_weight_max": 35.87009048461914,
      "activations/layer22_attention_weight_min": -25.17763328552246,
      "activations/layer23_attention_weight_max": 22.04290008544922,
      "activations/layer23_attention_weight_min": -15.83093547821045,
      "activations/layer2_attention_weight_max": 28.885883331298828,
      "activations/layer2_attention_weight_min": -26.335926055908203,
      "activations/layer3_attention_weight_max": 69.02628326416016,
      "activations/layer3_attention_weight_min": -66.58551788330078,
      "activations/layer4_attention_weight_max": 97.70579528808594,
      "activations/layer4_attention_weight_min": -84.10131072998047,
      "activations/layer5_attention_weight_max": 123.908935546875,
      "activations/layer5_attention_weight_min": -96.90769958496094,
      "activations/layer6_attention_weight_max": 60.92697525024414,
      "activations/layer6_attention_weight_min": -49.237205505371094,
      "activations/layer7_attention_weight_max": 60.84295654296875,
      "activations/layer7_attention_weight_min": -52.54755783081055,
      "activations/layer8_attention_weight_max": 47.24579620361328,
      "activations/layer8_attention_weight_min": -35.19266128540039,
      "activations/layer9_attention_weight_max": 67.32927703857422,
      "activations/layer9_attention_weight_min": -53.13308334350586,
      "epoch": 2.6,
      "learning_rate": 0.00013460454545454544,
      "loss": 2.927,
      "step": 44700
    },
    {
      "activations/layer0_attention_weight_max": 16.84476089477539,
      "activations/layer0_attention_weight_min": -14.880607604980469,
      "activations/layer10_attention_weight_max": 55.885982513427734,
      "activations/layer10_attention_weight_min": -42.85752487182617,
      "activations/layer11_attention_weight_max": 47.468467712402344,
      "activations/layer11_attention_weight_min": -35.20153045654297,
      "activations/layer12_attention_weight_max": 39.747066497802734,
      "activations/layer12_attention_weight_min": -23.47121810913086,
      "activations/layer13_attention_weight_max": 60.74213409423828,
      "activations/layer13_attention_weight_min": -43.34040069580078,
      "activations/layer14_attention_weight_max": 49.06723403930664,
      "activations/layer14_attention_weight_min": -33.990203857421875,
      "activations/layer15_attention_weight_max": 54.42074203491211,
      "activations/layer15_attention_weight_min": -37.72170639038086,
      "activations/layer16_attention_weight_max": 41.72939682006836,
      "activations/layer16_attention_weight_min": -26.760658264160156,
      "activations/layer17_attention_weight_max": 64.90856170654297,
      "activations/layer17_attention_weight_min": -34.43061828613281,
      "activations/layer18_attention_weight_max": 60.86054992675781,
      "activations/layer18_attention_weight_min": -31.587276458740234,
      "activations/layer19_attention_weight_max": 24.308570861816406,
      "activations/layer19_attention_weight_min": -18.682022094726562,
      "activations/layer1_attention_weight_max": 16.310205459594727,
      "activations/layer1_attention_weight_min": -14.71101188659668,
      "activations/layer20_attention_weight_max": 17.945934295654297,
      "activations/layer20_attention_weight_min": -16.544347763061523,
      "activations/layer21_attention_weight_max": 45.13833999633789,
      "activations/layer21_attention_weight_min": -25.763525009155273,
      "activations/layer22_attention_weight_max": 34.72340393066406,
      "activations/layer22_attention_weight_min": -21.447940826416016,
      "activations/layer23_attention_weight_max": 21.132949829101562,
      "activations/layer23_attention_weight_min": -13.911120414733887,
      "activations/layer2_attention_weight_max": 30.47900390625,
      "activations/layer2_attention_weight_min": -27.575307846069336,
      "activations/layer3_attention_weight_max": 74.97525024414062,
      "activations/layer3_attention_weight_min": -73.74142456054688,
      "activations/layer4_attention_weight_max": 105.75150299072266,
      "activations/layer4_attention_weight_min": -93.89991760253906,
      "activations/layer5_attention_weight_max": 140.02468872070312,
      "activations/layer5_attention_weight_min": -111.04632568359375,
      "activations/layer6_attention_weight_max": 68.09129333496094,
      "activations/layer6_attention_weight_min": -55.05643081665039,
      "activations/layer7_attention_weight_max": 74.50236511230469,
      "activations/layer7_attention_weight_min": -62.97262191772461,
      "activations/layer8_attention_weight_max": 58.80597686767578,
      "activations/layer8_attention_weight_min": -38.20442581176758,
      "activations/layer9_attention_weight_max": 78.82024383544922,
      "activations/layer9_attention_weight_min": -55.75299072265625,
      "epoch": 2.6,
      "learning_rate": 0.00013458560606060605,
      "loss": 2.9476,
      "step": 44750
    },
    {
      "activations/layer0_attention_weight_max": 14.332557678222656,
      "activations/layer0_attention_weight_min": -14.1699800491333,
      "activations/layer10_attention_weight_max": 49.240901947021484,
      "activations/layer10_attention_weight_min": -41.77778625488281,
      "activations/layer11_attention_weight_max": 37.558189392089844,
      "activations/layer11_attention_weight_min": -32.470680236816406,
      "activations/layer12_attention_weight_max": 28.85486602783203,
      "activations/layer12_attention_weight_min": -26.097322463989258,
      "activations/layer13_attention_weight_max": 60.91234588623047,
      "activations/layer13_attention_weight_min": -44.17298889160156,
      "activations/layer14_attention_weight_max": 63.96918869018555,
      "activations/layer14_attention_weight_min": -40.02528381347656,
      "activations/layer15_attention_weight_max": 52.741371154785156,
      "activations/layer15_attention_weight_min": -34.54841232299805,
      "activations/layer16_attention_weight_max": 41.919315338134766,
      "activations/layer16_attention_weight_min": -25.975021362304688,
      "activations/layer17_attention_weight_max": 64.03546142578125,
      "activations/layer17_attention_weight_min": -35.504634857177734,
      "activations/layer18_attention_weight_max": 55.601600646972656,
      "activations/layer18_attention_weight_min": -31.889684677124023,
      "activations/layer19_attention_weight_max": 21.369733810424805,
      "activations/layer19_attention_weight_min": -16.733989715576172,
      "activations/layer1_attention_weight_max": 17.121074676513672,
      "activations/layer1_attention_weight_min": -13.403349876403809,
      "activations/layer20_attention_weight_max": 22.03369140625,
      "activations/layer20_attention_weight_min": -19.669517517089844,
      "activations/layer21_attention_weight_max": 39.650657653808594,
      "activations/layer21_attention_weight_min": -25.426029205322266,
      "activations/layer22_attention_weight_max": 38.05887985229492,
      "activations/layer22_attention_weight_min": -21.727529525756836,
      "activations/layer23_attention_weight_max": 25.567289352416992,
      "activations/layer23_attention_weight_min": -14.626260757446289,
      "activations/layer2_attention_weight_max": 27.154582977294922,
      "activations/layer2_attention_weight_min": -26.078929901123047,
      "activations/layer3_attention_weight_max": 65.3777847290039,
      "activations/layer3_attention_weight_min": -68.71601104736328,
      "activations/layer4_attention_weight_max": 95.2497329711914,
      "activations/layer4_attention_weight_min": -86.1287612915039,
      "activations/layer5_attention_weight_max": 118.52300262451172,
      "activations/layer5_attention_weight_min": -98.16049194335938,
      "activations/layer6_attention_weight_max": 61.19188690185547,
      "activations/layer6_attention_weight_min": -47.228702545166016,
      "activations/layer7_attention_weight_max": 57.392337799072266,
      "activations/layer7_attention_weight_min": -55.44471740722656,
      "activations/layer8_attention_weight_max": 39.20330810546875,
      "activations/layer8_attention_weight_min": -36.18793869018555,
      "activations/layer9_attention_weight_max": 56.32383346557617,
      "activations/layer9_attention_weight_min": -51.54813766479492,
      "epoch": 2.6,
      "learning_rate": 0.00013456666666666664,
      "loss": 2.9375,
      "step": 44800
    },
    {
      "activations/layer0_attention_weight_max": 14.462751388549805,
      "activations/layer0_attention_weight_min": -13.94389820098877,
      "activations/layer10_attention_weight_max": 51.4085693359375,
      "activations/layer10_attention_weight_min": -40.539710998535156,
      "activations/layer11_attention_weight_max": 40.96192932128906,
      "activations/layer11_attention_weight_min": -30.7515926361084,
      "activations/layer12_attention_weight_max": 27.289142608642578,
      "activations/layer12_attention_weight_min": -21.689498901367188,
      "activations/layer13_attention_weight_max": 56.685550689697266,
      "activations/layer13_attention_weight_min": -37.38806915283203,
      "activations/layer14_attention_weight_max": 46.918128967285156,
      "activations/layer14_attention_weight_min": -32.06932067871094,
      "activations/layer15_attention_weight_max": 54.086265563964844,
      "activations/layer15_attention_weight_min": -36.3729133605957,
      "activations/layer16_attention_weight_max": 37.60944366455078,
      "activations/layer16_attention_weight_min": -27.498401641845703,
      "activations/layer17_attention_weight_max": 61.90218734741211,
      "activations/layer17_attention_weight_min": -35.6282958984375,
      "activations/layer18_attention_weight_max": 55.93738555908203,
      "activations/layer18_attention_weight_min": -33.04462432861328,
      "activations/layer19_attention_weight_max": 21.862104415893555,
      "activations/layer19_attention_weight_min": -19.326797485351562,
      "activations/layer1_attention_weight_max": 17.11646270751953,
      "activations/layer1_attention_weight_min": -14.569137573242188,
      "activations/layer20_attention_weight_max": 22.264122009277344,
      "activations/layer20_attention_weight_min": -20.937271118164062,
      "activations/layer21_attention_weight_max": 41.6799201965332,
      "activations/layer21_attention_weight_min": -30.144784927368164,
      "activations/layer22_attention_weight_max": 34.75052261352539,
      "activations/layer22_attention_weight_min": -24.21730613708496,
      "activations/layer23_attention_weight_max": 24.926910400390625,
      "activations/layer23_attention_weight_min": -15.460521697998047,
      "activations/layer2_attention_weight_max": 28.474876403808594,
      "activations/layer2_attention_weight_min": -27.42299461364746,
      "activations/layer3_attention_weight_max": 70.01557159423828,
      "activations/layer3_attention_weight_min": -71.39286041259766,
      "activations/layer4_attention_weight_max": 96.21896362304688,
      "activations/layer4_attention_weight_min": -89.9605484008789,
      "activations/layer5_attention_weight_max": 126.202392578125,
      "activations/layer5_attention_weight_min": -102.5346908569336,
      "activations/layer6_attention_weight_max": 63.579742431640625,
      "activations/layer6_attention_weight_min": -50.78594207763672,
      "activations/layer7_attention_weight_max": 58.48389434814453,
      "activations/layer7_attention_weight_min": -62.5308723449707,
      "activations/layer8_attention_weight_max": 39.55617141723633,
      "activations/layer8_attention_weight_min": -35.82411193847656,
      "activations/layer9_attention_weight_max": 53.606842041015625,
      "activations/layer9_attention_weight_min": -51.0340690612793,
      "epoch": 2.61,
      "learning_rate": 0.00013454772727272725,
      "loss": 2.9158,
      "step": 44850
    },
    {
      "activations/layer0_attention_weight_max": 14.407544136047363,
      "activations/layer0_attention_weight_min": -13.842537879943848,
      "activations/layer10_attention_weight_max": 54.719635009765625,
      "activations/layer10_attention_weight_min": -40.6552619934082,
      "activations/layer11_attention_weight_max": 40.721466064453125,
      "activations/layer11_attention_weight_min": -29.53955841064453,
      "activations/layer12_attention_weight_max": 35.68974685668945,
      "activations/layer12_attention_weight_min": -25.023229598999023,
      "activations/layer13_attention_weight_max": 67.75277709960938,
      "activations/layer13_attention_weight_min": -41.64189910888672,
      "activations/layer14_attention_weight_max": 50.549293518066406,
      "activations/layer14_attention_weight_min": -32.51565170288086,
      "activations/layer15_attention_weight_max": 51.9251594543457,
      "activations/layer15_attention_weight_min": -34.525569915771484,
      "activations/layer16_attention_weight_max": 45.02885055541992,
      "activations/layer16_attention_weight_min": -26.232608795166016,
      "activations/layer17_attention_weight_max": 66.58604431152344,
      "activations/layer17_attention_weight_min": -35.3376350402832,
      "activations/layer18_attention_weight_max": 59.4807243347168,
      "activations/layer18_attention_weight_min": -30.992948532104492,
      "activations/layer19_attention_weight_max": 28.99062156677246,
      "activations/layer19_attention_weight_min": -18.16480255126953,
      "activations/layer1_attention_weight_max": 17.125307083129883,
      "activations/layer1_attention_weight_min": -17.58930015563965,
      "activations/layer20_attention_weight_max": 24.29147720336914,
      "activations/layer20_attention_weight_min": -17.047998428344727,
      "activations/layer21_attention_weight_max": 48.7484130859375,
      "activations/layer21_attention_weight_min": -28.873394012451172,
      "activations/layer22_attention_weight_max": 40.5550422668457,
      "activations/layer22_attention_weight_min": -21.985857009887695,
      "activations/layer23_attention_weight_max": 28.268957138061523,
      "activations/layer23_attention_weight_min": -16.51886558532715,
      "activations/layer2_attention_weight_max": 28.28510093688965,
      "activations/layer2_attention_weight_min": -27.60955047607422,
      "activations/layer3_attention_weight_max": 67.0682373046875,
      "activations/layer3_attention_weight_min": -69.84342193603516,
      "activations/layer4_attention_weight_max": 95.10530853271484,
      "activations/layer4_attention_weight_min": -84.79228973388672,
      "activations/layer5_attention_weight_max": 129.70286560058594,
      "activations/layer5_attention_weight_min": -105.41316986083984,
      "activations/layer6_attention_weight_max": 65.2831039428711,
      "activations/layer6_attention_weight_min": -52.123233795166016,
      "activations/layer7_attention_weight_max": 60.265724182128906,
      "activations/layer7_attention_weight_min": -54.87129592895508,
      "activations/layer8_attention_weight_max": 44.70606231689453,
      "activations/layer8_attention_weight_min": -37.059898376464844,
      "activations/layer9_attention_weight_max": 61.70166778564453,
      "activations/layer9_attention_weight_min": -54.41847610473633,
      "epoch": 2.61,
      "learning_rate": 0.00013452878787878787,
      "loss": 2.9225,
      "step": 44900
    },
    {
      "activations/layer0_attention_weight_max": 14.832185745239258,
      "activations/layer0_attention_weight_min": -14.53933334350586,
      "activations/layer10_attention_weight_max": 47.62188720703125,
      "activations/layer10_attention_weight_min": -38.20266342163086,
      "activations/layer11_attention_weight_max": 35.15602493286133,
      "activations/layer11_attention_weight_min": -30.9468994140625,
      "activations/layer12_attention_weight_max": 28.03014373779297,
      "activations/layer12_attention_weight_min": -25.20421600341797,
      "activations/layer13_attention_weight_max": 57.714908599853516,
      "activations/layer13_attention_weight_min": -47.5069694519043,
      "activations/layer14_attention_weight_max": 60.00027084350586,
      "activations/layer14_attention_weight_min": -46.18877410888672,
      "activations/layer15_attention_weight_max": 44.9350700378418,
      "activations/layer15_attention_weight_min": -36.929473876953125,
      "activations/layer16_attention_weight_max": 38.47336196899414,
      "activations/layer16_attention_weight_min": -27.065673828125,
      "activations/layer17_attention_weight_max": 54.69831848144531,
      "activations/layer17_attention_weight_min": -35.63478469848633,
      "activations/layer18_attention_weight_max": 62.0479621887207,
      "activations/layer18_attention_weight_min": -31.422481536865234,
      "activations/layer19_attention_weight_max": 19.91206932067871,
      "activations/layer19_attention_weight_min": -16.024274826049805,
      "activations/layer1_attention_weight_max": 17.397504806518555,
      "activations/layer1_attention_weight_min": -16.147932052612305,
      "activations/layer20_attention_weight_max": 20.715375900268555,
      "activations/layer20_attention_weight_min": -18.15302848815918,
      "activations/layer21_attention_weight_max": 42.961334228515625,
      "activations/layer21_attention_weight_min": -28.999588012695312,
      "activations/layer22_attention_weight_max": 32.6568489074707,
      "activations/layer22_attention_weight_min": -23.163206100463867,
      "activations/layer23_attention_weight_max": 24.959684371948242,
      "activations/layer23_attention_weight_min": -16.686477661132812,
      "activations/layer2_attention_weight_max": 27.77236557006836,
      "activations/layer2_attention_weight_min": -27.351421356201172,
      "activations/layer3_attention_weight_max": 66.04004669189453,
      "activations/layer3_attention_weight_min": -68.1836166381836,
      "activations/layer4_attention_weight_max": 95.30403137207031,
      "activations/layer4_attention_weight_min": -86.75861358642578,
      "activations/layer5_attention_weight_max": 114.10594177246094,
      "activations/layer5_attention_weight_min": -100.01982879638672,
      "activations/layer6_attention_weight_max": 56.6618537902832,
      "activations/layer6_attention_weight_min": -49.49455261230469,
      "activations/layer7_attention_weight_max": 60.205345153808594,
      "activations/layer7_attention_weight_min": -55.9937744140625,
      "activations/layer8_attention_weight_max": 44.346824645996094,
      "activations/layer8_attention_weight_min": -37.080501556396484,
      "activations/layer9_attention_weight_max": 66.15129852294922,
      "activations/layer9_attention_weight_min": -52.719329833984375,
      "epoch": 2.61,
      "learning_rate": 0.00013450984848484846,
      "loss": 2.9202,
      "step": 44950
    },
    {
      "activations/layer0_attention_weight_max": 14.224998474121094,
      "activations/layer0_attention_weight_min": -14.597421646118164,
      "activations/layer10_attention_weight_max": 47.847835540771484,
      "activations/layer10_attention_weight_min": -39.308067321777344,
      "activations/layer11_attention_weight_max": 42.70902633666992,
      "activations/layer11_attention_weight_min": -30.342594146728516,
      "activations/layer12_attention_weight_max": 27.96285629272461,
      "activations/layer12_attention_weight_min": -24.620405197143555,
      "activations/layer13_attention_weight_max": 70.37752532958984,
      "activations/layer13_attention_weight_min": -49.99368667602539,
      "activations/layer14_attention_weight_max": 56.78017807006836,
      "activations/layer14_attention_weight_min": -40.28166198730469,
      "activations/layer15_attention_weight_max": 56.232051849365234,
      "activations/layer15_attention_weight_min": -36.90013122558594,
      "activations/layer16_attention_weight_max": 37.34208297729492,
      "activations/layer16_attention_weight_min": -28.528154373168945,
      "activations/layer17_attention_weight_max": 59.68625259399414,
      "activations/layer17_attention_weight_min": -36.49290084838867,
      "activations/layer18_attention_weight_max": 55.53068542480469,
      "activations/layer18_attention_weight_min": -34.3476676940918,
      "activations/layer19_attention_weight_max": 22.558462142944336,
      "activations/layer19_attention_weight_min": -18.018962860107422,
      "activations/layer1_attention_weight_max": 17.661828994750977,
      "activations/layer1_attention_weight_min": -14.473859786987305,
      "activations/layer20_attention_weight_max": 20.375364303588867,
      "activations/layer20_attention_weight_min": -16.88768196105957,
      "activations/layer21_attention_weight_max": 45.0395622253418,
      "activations/layer21_attention_weight_min": -29.67915916442871,
      "activations/layer22_attention_weight_max": 37.36507034301758,
      "activations/layer22_attention_weight_min": -21.480445861816406,
      "activations/layer23_attention_weight_max": 24.159133911132812,
      "activations/layer23_attention_weight_min": -14.734539031982422,
      "activations/layer2_attention_weight_max": 27.97535514831543,
      "activations/layer2_attention_weight_min": -26.207820892333984,
      "activations/layer3_attention_weight_max": 64.36817932128906,
      "activations/layer3_attention_weight_min": -69.8600845336914,
      "activations/layer4_attention_weight_max": 94.87702941894531,
      "activations/layer4_attention_weight_min": -86.61332702636719,
      "activations/layer5_attention_weight_max": 116.72669982910156,
      "activations/layer5_attention_weight_min": -97.21087646484375,
      "activations/layer6_attention_weight_max": 60.302764892578125,
      "activations/layer6_attention_weight_min": -48.797996520996094,
      "activations/layer7_attention_weight_max": 54.712554931640625,
      "activations/layer7_attention_weight_min": -52.01388168334961,
      "activations/layer8_attention_weight_max": 37.83156204223633,
      "activations/layer8_attention_weight_min": -33.12118148803711,
      "activations/layer9_attention_weight_max": 53.94334030151367,
      "activations/layer9_attention_weight_min": -49.121036529541016,
      "epoch": 2.61,
      "learning_rate": 0.00013449090909090907,
      "loss": 2.9351,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_loss": 2.876953125,
      "eval_runtime": 8.0642,
      "eval_samples_per_second": 532.476,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_openwebtext_loss": 2.876953125,
      "eval_openwebtext_ppl": 17.76007792103387,
      "eval_openwebtext_runtime": 8.0642,
      "eval_openwebtext_samples_per_second": 532.476,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_wikitext_loss": 3.130859375,
      "eval_wikitext_ppl": 22.893645317537526,
      "eval_wikitext_runtime": 1.5974,
      "eval_wikitext_samples_per_second": 285.465,
      "step": 45000
    },
    {
      "epoch": 2.61,
      "eval_lambada_loss": 2.951171875,
      "eval_lambada_ppl": 19.128356641900336,
      "eval_lambada_runtime": 9.2377,
      "eval_lambada_samples_per_second": 527.078,
      "step": 45000
    },
    {
      "activations/layer0_attention_weight_max": 14.759462356567383,
      "activations/layer0_attention_weight_min": -14.277252197265625,
      "activations/layer10_attention_weight_max": 56.562496185302734,
      "activations/layer10_attention_weight_min": -41.091339111328125,
      "activations/layer11_attention_weight_max": 43.921630859375,
      "activations/layer11_attention_weight_min": -35.51020050048828,
      "activations/layer12_attention_weight_max": 26.054725646972656,
      "activations/layer12_attention_weight_min": -27.358463287353516,
      "activations/layer13_attention_weight_max": 59.531959533691406,
      "activations/layer13_attention_weight_min": -41.94013595581055,
      "activations/layer14_attention_weight_max": 52.640480041503906,
      "activations/layer14_attention_weight_min": -39.3925666809082,
      "activations/layer15_attention_weight_max": 55.18159866333008,
      "activations/layer15_attention_weight_min": -40.478031158447266,
      "activations/layer16_attention_weight_max": 39.993064880371094,
      "activations/layer16_attention_weight_min": -25.991594314575195,
      "activations/layer17_attention_weight_max": 63.85871887207031,
      "activations/layer17_attention_weight_min": -34.82424545288086,
      "activations/layer18_attention_weight_max": 56.81521224975586,
      "activations/layer18_attention_weight_min": -31.952701568603516,
      "activations/layer19_attention_weight_max": 21.1832332611084,
      "activations/layer19_attention_weight_min": -15.59289264678955,
      "activations/layer1_attention_weight_max": 17.86446189880371,
      "activations/layer1_attention_weight_min": -14.27433967590332,
      "activations/layer20_attention_weight_max": 20.128767013549805,
      "activations/layer20_attention_weight_min": -16.64598846435547,
      "activations/layer21_attention_weight_max": 40.706756591796875,
      "activations/layer21_attention_weight_min": -26.195735931396484,
      "activations/layer22_attention_weight_max": 34.15717315673828,
      "activations/layer22_attention_weight_min": -21.277833938598633,
      "activations/layer23_attention_weight_max": 24.99911880493164,
      "activations/layer23_attention_weight_min": -16.144765853881836,
      "activations/layer2_attention_weight_max": 27.400875091552734,
      "activations/layer2_attention_weight_min": -26.250526428222656,
      "activations/layer3_attention_weight_max": 64.9973373413086,
      "activations/layer3_attention_weight_min": -68.11392211914062,
      "activations/layer4_attention_weight_max": 96.94239807128906,
      "activations/layer4_attention_weight_min": -85.49816131591797,
      "activations/layer5_attention_weight_max": 130.48887634277344,
      "activations/layer5_attention_weight_min": -104.486083984375,
      "activations/layer6_attention_weight_max": 65.26753997802734,
      "activations/layer6_attention_weight_min": -53.04049301147461,
      "activations/layer7_attention_weight_max": 67.36869049072266,
      "activations/layer7_attention_weight_min": -56.40037536621094,
      "activations/layer8_attention_weight_max": 55.157039642333984,
      "activations/layer8_attention_weight_min": -39.350826263427734,
      "activations/layer9_attention_weight_max": 81.40308380126953,
      "activations/layer9_attention_weight_min": -60.00347900390625,
      "epoch": 2.62,
      "learning_rate": 0.0001344719696969697,
      "loss": 2.9178,
      "step": 45050
    },
    {
      "activations/layer0_attention_weight_max": 13.295455932617188,
      "activations/layer0_attention_weight_min": -14.647712707519531,
      "activations/layer10_attention_weight_max": 48.49845504760742,
      "activations/layer10_attention_weight_min": -37.65278244018555,
      "activations/layer11_attention_weight_max": 40.09967803955078,
      "activations/layer11_attention_weight_min": -29.159025192260742,
      "activations/layer12_attention_weight_max": 28.26306915283203,
      "activations/layer12_attention_weight_min": -23.12644386291504,
      "activations/layer13_attention_weight_max": 55.910438537597656,
      "activations/layer13_attention_weight_min": -40.328369140625,
      "activations/layer14_attention_weight_max": 43.76742935180664,
      "activations/layer14_attention_weight_min": -33.34281921386719,
      "activations/layer15_attention_weight_max": 48.601993560791016,
      "activations/layer15_attention_weight_min": -35.55765914916992,
      "activations/layer16_attention_weight_max": 37.997474670410156,
      "activations/layer16_attention_weight_min": -25.834903717041016,
      "activations/layer17_attention_weight_max": 57.41261291503906,
      "activations/layer17_attention_weight_min": -34.74416732788086,
      "activations/layer18_attention_weight_max": 52.41569900512695,
      "activations/layer18_attention_weight_min": -29.32989501953125,
      "activations/layer19_attention_weight_max": 22.70970344543457,
      "activations/layer19_attention_weight_min": -17.96809959411621,
      "activations/layer1_attention_weight_max": 19.932226181030273,
      "activations/layer1_attention_weight_min": -15.163910865783691,
      "activations/layer20_attention_weight_max": 19.59376335144043,
      "activations/layer20_attention_weight_min": -18.576932907104492,
      "activations/layer21_attention_weight_max": 42.90559768676758,
      "activations/layer21_attention_weight_min": -30.196794509887695,
      "activations/layer22_attention_weight_max": 33.91971969604492,
      "activations/layer22_attention_weight_min": -21.393348693847656,
      "activations/layer23_attention_weight_max": 25.721942901611328,
      "activations/layer23_attention_weight_min": -15.5816650390625,
      "activations/layer2_attention_weight_max": 28.82068634033203,
      "activations/layer2_attention_weight_min": -26.578563690185547,
      "activations/layer3_attention_weight_max": 63.32535171508789,
      "activations/layer3_attention_weight_min": -69.78154754638672,
      "activations/layer4_attention_weight_max": 96.48784637451172,
      "activations/layer4_attention_weight_min": -87.53927612304688,
      "activations/layer5_attention_weight_max": 124.99093627929688,
      "activations/layer5_attention_weight_min": -101.57268524169922,
      "activations/layer6_attention_weight_max": 60.11949157714844,
      "activations/layer6_attention_weight_min": -49.881874084472656,
      "activations/layer7_attention_weight_max": 60.010868072509766,
      "activations/layer7_attention_weight_min": -51.009971618652344,
      "activations/layer8_attention_weight_max": 39.77522659301758,
      "activations/layer8_attention_weight_min": -37.11855697631836,
      "activations/layer9_attention_weight_max": 51.95431137084961,
      "activations/layer9_attention_weight_min": -47.89109420776367,
      "epoch": 2.62,
      "learning_rate": 0.0001344530303030303,
      "loss": 2.9223,
      "step": 45100
    },
    {
      "activations/layer0_attention_weight_max": 13.795692443847656,
      "activations/layer0_attention_weight_min": -14.452357292175293,
      "activations/layer10_attention_weight_max": 57.65201950073242,
      "activations/layer10_attention_weight_min": -38.45882034301758,
      "activations/layer11_attention_weight_max": 39.82656478881836,
      "activations/layer11_attention_weight_min": -29.54671859741211,
      "activations/layer12_attention_weight_max": 35.314964294433594,
      "activations/layer12_attention_weight_min": -26.93012237548828,
      "activations/layer13_attention_weight_max": 53.99961471557617,
      "activations/layer13_attention_weight_min": -36.916587829589844,
      "activations/layer14_attention_weight_max": 57.61113357543945,
      "activations/layer14_attention_weight_min": -36.905250549316406,
      "activations/layer15_attention_weight_max": 57.038429260253906,
      "activations/layer15_attention_weight_min": -35.80855941772461,
      "activations/layer16_attention_weight_max": 42.737735748291016,
      "activations/layer16_attention_weight_min": -30.847278594970703,
      "activations/layer17_attention_weight_max": 60.80491256713867,
      "activations/layer17_attention_weight_min": -35.71314239501953,
      "activations/layer18_attention_weight_max": 65.66071319580078,
      "activations/layer18_attention_weight_min": -34.0805549621582,
      "activations/layer19_attention_weight_max": 22.80914878845215,
      "activations/layer19_attention_weight_min": -17.610864639282227,
      "activations/layer1_attention_weight_max": 17.494829177856445,
      "activations/layer1_attention_weight_min": -15.114848136901855,
      "activations/layer20_attention_weight_max": 20.80792236328125,
      "activations/layer20_attention_weight_min": -17.418296813964844,
      "activations/layer21_attention_weight_max": 44.68196487426758,
      "activations/layer21_attention_weight_min": -27.049327850341797,
      "activations/layer22_attention_weight_max": 41.285552978515625,
      "activations/layer22_attention_weight_min": -24.39369010925293,
      "activations/layer23_attention_weight_max": 27.43416976928711,
      "activations/layer23_attention_weight_min": -16.69923210144043,
      "activations/layer2_attention_weight_max": 27.754188537597656,
      "activations/layer2_attention_weight_min": -25.931499481201172,
      "activations/layer3_attention_weight_max": 68.91653442382812,
      "activations/layer3_attention_weight_min": -74.1076431274414,
      "activations/layer4_attention_weight_max": 97.35169219970703,
      "activations/layer4_attention_weight_min": -91.30717468261719,
      "activations/layer5_attention_weight_max": 121.57199096679688,
      "activations/layer5_attention_weight_min": -98.8197250366211,
      "activations/layer6_attention_weight_max": 61.74448013305664,
      "activations/layer6_attention_weight_min": -47.96154022216797,
      "activations/layer7_attention_weight_max": 60.372676849365234,
      "activations/layer7_attention_weight_min": -51.9636344909668,
      "activations/layer8_attention_weight_max": 43.95753860473633,
      "activations/layer8_attention_weight_min": -36.10512161254883,
      "activations/layer9_attention_weight_max": 58.16459274291992,
      "activations/layer9_attention_weight_min": -49.33340072631836,
      "epoch": 2.62,
      "learning_rate": 0.0001344340909090909,
      "loss": 2.9405,
      "step": 45150
    },
    {
      "activations/layer0_attention_weight_max": 14.957132339477539,
      "activations/layer0_attention_weight_min": -14.193791389465332,
      "activations/layer10_attention_weight_max": 58.81364440917969,
      "activations/layer10_attention_weight_min": -40.592803955078125,
      "activations/layer11_attention_weight_max": 52.2226676940918,
      "activations/layer11_attention_weight_min": -34.01061248779297,
      "activations/layer12_attention_weight_max": 25.09109115600586,
      "activations/layer12_attention_weight_min": -25.95262908935547,
      "activations/layer13_attention_weight_max": 92.93555450439453,
      "activations/layer13_attention_weight_min": -49.64226531982422,
      "activations/layer14_attention_weight_max": 105.0497055053711,
      "activations/layer14_attention_weight_min": -47.76926040649414,
      "activations/layer15_attention_weight_max": 70.10173034667969,
      "activations/layer15_attention_weight_min": -34.1279182434082,
      "activations/layer16_attention_weight_max": 38.95180130004883,
      "activations/layer16_attention_weight_min": -26.460025787353516,
      "activations/layer17_attention_weight_max": 53.9751091003418,
      "activations/layer17_attention_weight_min": -34.94227981567383,
      "activations/layer18_attention_weight_max": 53.83588790893555,
      "activations/layer18_attention_weight_min": -28.68358039855957,
      "activations/layer19_attention_weight_max": 21.448537826538086,
      "activations/layer19_attention_weight_min": -15.843249320983887,
      "activations/layer1_attention_weight_max": 17.573543548583984,
      "activations/layer1_attention_weight_min": -13.728782653808594,
      "activations/layer20_attention_weight_max": 19.976234436035156,
      "activations/layer20_attention_weight_min": -20.07264518737793,
      "activations/layer21_attention_weight_max": 33.79964065551758,
      "activations/layer21_attention_weight_min": -24.812522888183594,
      "activations/layer22_attention_weight_max": 30.46293830871582,
      "activations/layer22_attention_weight_min": -21.646326065063477,
      "activations/layer23_attention_weight_max": 23.63668441772461,
      "activations/layer23_attention_weight_min": -14.574737548828125,
      "activations/layer2_attention_weight_max": 30.431211471557617,
      "activations/layer2_attention_weight_min": -28.62274169921875,
      "activations/layer3_attention_weight_max": 74.59434509277344,
      "activations/layer3_attention_weight_min": -74.29017639160156,
      "activations/layer4_attention_weight_max": 119.64952850341797,
      "activations/layer4_attention_weight_min": -103.68521881103516,
      "activations/layer5_attention_weight_max": 163.54354858398438,
      "activations/layer5_attention_weight_min": -124.313232421875,
      "activations/layer6_attention_weight_max": 83.32323455810547,
      "activations/layer6_attention_weight_min": -61.97108459472656,
      "activations/layer7_attention_weight_max": 77.91478729248047,
      "activations/layer7_attention_weight_min": -59.18546676635742,
      "activations/layer8_attention_weight_max": 65.5888442993164,
      "activations/layer8_attention_weight_min": -48.02628707885742,
      "activations/layer9_attention_weight_max": 89.9615707397461,
      "activations/layer9_attention_weight_min": -62.647560119628906,
      "epoch": 2.63,
      "learning_rate": 0.0001344151515151515,
      "loss": 2.9332,
      "step": 45200
    },
    {
      "activations/layer0_attention_weight_max": 14.524388313293457,
      "activations/layer0_attention_weight_min": -15.14538288116455,
      "activations/layer10_attention_weight_max": 47.60939407348633,
      "activations/layer10_attention_weight_min": -39.60160827636719,
      "activations/layer11_attention_weight_max": 36.56516647338867,
      "activations/layer11_attention_weight_min": -30.314693450927734,
      "activations/layer12_attention_weight_max": 29.095918655395508,
      "activations/layer12_attention_weight_min": -25.50033187866211,
      "activations/layer13_attention_weight_max": 52.63072204589844,
      "activations/layer13_attention_weight_min": -38.084449768066406,
      "activations/layer14_attention_weight_max": 62.167720794677734,
      "activations/layer14_attention_weight_min": -41.7497444152832,
      "activations/layer15_attention_weight_max": 46.449981689453125,
      "activations/layer15_attention_weight_min": -33.07358169555664,
      "activations/layer16_attention_weight_max": 41.78553009033203,
      "activations/layer16_attention_weight_min": -27.949663162231445,
      "activations/layer17_attention_weight_max": 60.79127502441406,
      "activations/layer17_attention_weight_min": -36.97552490234375,
      "activations/layer18_attention_weight_max": 53.871952056884766,
      "activations/layer18_attention_weight_min": -31.32647132873535,
      "activations/layer19_attention_weight_max": 23.31651496887207,
      "activations/layer19_attention_weight_min": -18.10878562927246,
      "activations/layer1_attention_weight_max": 17.973936080932617,
      "activations/layer1_attention_weight_min": -13.566158294677734,
      "activations/layer20_attention_weight_max": 22.231937408447266,
      "activations/layer20_attention_weight_min": -19.943456649780273,
      "activations/layer21_attention_weight_max": 45.868408203125,
      "activations/layer21_attention_weight_min": -26.354026794433594,
      "activations/layer22_attention_weight_max": 40.506980895996094,
      "activations/layer22_attention_weight_min": -21.769445419311523,
      "activations/layer23_attention_weight_max": 26.817642211914062,
      "activations/layer23_attention_weight_min": -16.225984573364258,
      "activations/layer2_attention_weight_max": 27.665822982788086,
      "activations/layer2_attention_weight_min": -26.927959442138672,
      "activations/layer3_attention_weight_max": 70.6629638671875,
      "activations/layer3_attention_weight_min": -69.52359008789062,
      "activations/layer4_attention_weight_max": 98.31038665771484,
      "activations/layer4_attention_weight_min": -87.7146224975586,
      "activations/layer5_attention_weight_max": 119.66514587402344,
      "activations/layer5_attention_weight_min": -101.16793823242188,
      "activations/layer6_attention_weight_max": 62.55033874511719,
      "activations/layer6_attention_weight_min": -54.680912017822266,
      "activations/layer7_attention_weight_max": 57.405357360839844,
      "activations/layer7_attention_weight_min": -54.730369567871094,
      "activations/layer8_attention_weight_max": 39.919525146484375,
      "activations/layer8_attention_weight_min": -34.43402099609375,
      "activations/layer9_attention_weight_max": 52.422340393066406,
      "activations/layer9_attention_weight_min": -48.18117141723633,
      "epoch": 2.63,
      "learning_rate": 0.00013439621212121212,
      "loss": 2.9162,
      "step": 45250
    },
    {
      "activations/layer0_attention_weight_max": 14.175680160522461,
      "activations/layer0_attention_weight_min": -14.411680221557617,
      "activations/layer10_attention_weight_max": 43.71388626098633,
      "activations/layer10_attention_weight_min": -36.5457649230957,
      "activations/layer11_attention_weight_max": 39.165382385253906,
      "activations/layer11_attention_weight_min": -30.499038696289062,
      "activations/layer12_attention_weight_max": 22.9765567779541,
      "activations/layer12_attention_weight_min": -24.90704345703125,
      "activations/layer13_attention_weight_max": 57.918460845947266,
      "activations/layer13_attention_weight_min": -41.37129211425781,
      "activations/layer14_attention_weight_max": 58.568885803222656,
      "activations/layer14_attention_weight_min": -38.32048034667969,
      "activations/layer15_attention_weight_max": 53.35140609741211,
      "activations/layer15_attention_weight_min": -38.19622802734375,
      "activations/layer16_attention_weight_max": 39.272647857666016,
      "activations/layer16_attention_weight_min": -29.320268630981445,
      "activations/layer17_attention_weight_max": 59.24376678466797,
      "activations/layer17_attention_weight_min": -36.58281707763672,
      "activations/layer18_attention_weight_max": 50.8851432800293,
      "activations/layer18_attention_weight_min": -36.88323974609375,
      "activations/layer19_attention_weight_max": 23.09159278869629,
      "activations/layer19_attention_weight_min": -16.548715591430664,
      "activations/layer1_attention_weight_max": 17.372644424438477,
      "activations/layer1_attention_weight_min": -14.833706855773926,
      "activations/layer20_attention_weight_max": 21.080242156982422,
      "activations/layer20_attention_weight_min": -18.678953170776367,
      "activations/layer21_attention_weight_max": 39.70661163330078,
      "activations/layer21_attention_weight_min": -31.846782684326172,
      "activations/layer22_attention_weight_max": 37.84449768066406,
      "activations/layer22_attention_weight_min": -27.026195526123047,
      "activations/layer23_attention_weight_max": 26.842174530029297,
      "activations/layer23_attention_weight_min": -18.02756118774414,
      "activations/layer2_attention_weight_max": 28.80891227722168,
      "activations/layer2_attention_weight_min": -27.203433990478516,
      "activations/layer3_attention_weight_max": 67.51264190673828,
      "activations/layer3_attention_weight_min": -71.3440170288086,
      "activations/layer4_attention_weight_max": 95.67825317382812,
      "activations/layer4_attention_weight_min": -86.97075653076172,
      "activations/layer5_attention_weight_max": 119.91770935058594,
      "activations/layer5_attention_weight_min": -97.89303588867188,
      "activations/layer6_attention_weight_max": 61.9383659362793,
      "activations/layer6_attention_weight_min": -50.167701721191406,
      "activations/layer7_attention_weight_max": 55.091800689697266,
      "activations/layer7_attention_weight_min": -50.92881774902344,
      "activations/layer8_attention_weight_max": 38.47041702270508,
      "activations/layer8_attention_weight_min": -33.83484649658203,
      "activations/layer9_attention_weight_max": 54.80002975463867,
      "activations/layer9_attention_weight_min": -46.67020797729492,
      "epoch": 2.63,
      "learning_rate": 0.0001343772727272727,
      "loss": 2.9372,
      "step": 45300
    },
    {
      "activations/layer0_attention_weight_max": 14.644933700561523,
      "activations/layer0_attention_weight_min": -14.039365768432617,
      "activations/layer10_attention_weight_max": 47.269432067871094,
      "activations/layer10_attention_weight_min": -36.607017517089844,
      "activations/layer11_attention_weight_max": 39.23381042480469,
      "activations/layer11_attention_weight_min": -27.469974517822266,
      "activations/layer12_attention_weight_max": 27.053268432617188,
      "activations/layer12_attention_weight_min": -23.961576461791992,
      "activations/layer13_attention_weight_max": 62.04376983642578,
      "activations/layer13_attention_weight_min": -38.441566467285156,
      "activations/layer14_attention_weight_max": 51.168949127197266,
      "activations/layer14_attention_weight_min": -35.28029251098633,
      "activations/layer15_attention_weight_max": 45.78768539428711,
      "activations/layer15_attention_weight_min": -33.41315841674805,
      "activations/layer16_attention_weight_max": 43.26383972167969,
      "activations/layer16_attention_weight_min": -28.02033042907715,
      "activations/layer17_attention_weight_max": 61.60149002075195,
      "activations/layer17_attention_weight_min": -37.195560455322266,
      "activations/layer18_attention_weight_max": 54.77782440185547,
      "activations/layer18_attention_weight_min": -30.71160316467285,
      "activations/layer19_attention_weight_max": 21.93244743347168,
      "activations/layer19_attention_weight_min": -15.894932746887207,
      "activations/layer1_attention_weight_max": 17.271724700927734,
      "activations/layer1_attention_weight_min": -13.549965858459473,
      "activations/layer20_attention_weight_max": 21.33147430419922,
      "activations/layer20_attention_weight_min": -19.794918060302734,
      "activations/layer21_attention_weight_max": 49.39692306518555,
      "activations/layer21_attention_weight_min": -28.59770965576172,
      "activations/layer22_attention_weight_max": 37.73225402832031,
      "activations/layer22_attention_weight_min": -21.698638916015625,
      "activations/layer23_attention_weight_max": 23.104110717773438,
      "activations/layer23_attention_weight_min": -16.259124755859375,
      "activations/layer2_attention_weight_max": 27.72173309326172,
      "activations/layer2_attention_weight_min": -26.555927276611328,
      "activations/layer3_attention_weight_max": 66.47882080078125,
      "activations/layer3_attention_weight_min": -71.51490783691406,
      "activations/layer4_attention_weight_max": 96.90254974365234,
      "activations/layer4_attention_weight_min": -89.6273193359375,
      "activations/layer5_attention_weight_max": 119.41993713378906,
      "activations/layer5_attention_weight_min": -100.96080780029297,
      "activations/layer6_attention_weight_max": 60.5882682800293,
      "activations/layer6_attention_weight_min": -49.08347702026367,
      "activations/layer7_attention_weight_max": 54.084312438964844,
      "activations/layer7_attention_weight_min": -50.2837028503418,
      "activations/layer8_attention_weight_max": 40.01637268066406,
      "activations/layer8_attention_weight_min": -34.70683670043945,
      "activations/layer9_attention_weight_max": 55.20241165161133,
      "activations/layer9_attention_weight_min": -46.864261627197266,
      "epoch": 2.64,
      "learning_rate": 0.00013435833333333332,
      "loss": 2.9313,
      "step": 45350
    },
    {
      "activations/layer0_attention_weight_max": 15.51820182800293,
      "activations/layer0_attention_weight_min": -14.585095405578613,
      "activations/layer10_attention_weight_max": 54.45871353149414,
      "activations/layer10_attention_weight_min": -41.13641357421875,
      "activations/layer11_attention_weight_max": 42.23551940917969,
      "activations/layer11_attention_weight_min": -30.03455924987793,
      "activations/layer12_attention_weight_max": 27.970314025878906,
      "activations/layer12_attention_weight_min": -23.10944175720215,
      "activations/layer13_attention_weight_max": 67.01016998291016,
      "activations/layer13_attention_weight_min": -43.211795806884766,
      "activations/layer14_attention_weight_max": 57.03067398071289,
      "activations/layer14_attention_weight_min": -37.36283493041992,
      "activations/layer15_attention_weight_max": 57.2451057434082,
      "activations/layer15_attention_weight_min": -35.10793685913086,
      "activations/layer16_attention_weight_max": 41.78439712524414,
      "activations/layer16_attention_weight_min": -26.508140563964844,
      "activations/layer17_attention_weight_max": 65.05111694335938,
      "activations/layer17_attention_weight_min": -37.62735366821289,
      "activations/layer18_attention_weight_max": 59.64663314819336,
      "activations/layer18_attention_weight_min": -32.63361740112305,
      "activations/layer19_attention_weight_max": 24.854251861572266,
      "activations/layer19_attention_weight_min": -18.33673667907715,
      "activations/layer1_attention_weight_max": 16.404239654541016,
      "activations/layer1_attention_weight_min": -13.007486343383789,
      "activations/layer20_attention_weight_max": 26.87115478515625,
      "activations/layer20_attention_weight_min": -18.25177574157715,
      "activations/layer21_attention_weight_max": 44.88774871826172,
      "activations/layer21_attention_weight_min": -27.765756607055664,
      "activations/layer22_attention_weight_max": 37.56422805786133,
      "activations/layer22_attention_weight_min": -22.147090911865234,
      "activations/layer23_attention_weight_max": 23.936872482299805,
      "activations/layer23_attention_weight_min": -16.30232810974121,
      "activations/layer2_attention_weight_max": 29.052099227905273,
      "activations/layer2_attention_weight_min": -26.800886154174805,
      "activations/layer3_attention_weight_max": 69.43328094482422,
      "activations/layer3_attention_weight_min": -71.86346435546875,
      "activations/layer4_attention_weight_max": 96.45244598388672,
      "activations/layer4_attention_weight_min": -88.68359375,
      "activations/layer5_attention_weight_max": 118.57473754882812,
      "activations/layer5_attention_weight_min": -100.47078704833984,
      "activations/layer6_attention_weight_max": 55.70460891723633,
      "activations/layer6_attention_weight_min": -51.27812194824219,
      "activations/layer7_attention_weight_max": 55.177181243896484,
      "activations/layer7_attention_weight_min": -51.11030960083008,
      "activations/layer8_attention_weight_max": 38.98004913330078,
      "activations/layer8_attention_weight_min": -33.89288330078125,
      "activations/layer9_attention_weight_max": 56.600154876708984,
      "activations/layer9_attention_weight_min": -50.44124984741211,
      "epoch": 2.64,
      "learning_rate": 0.00013433939393939394,
      "loss": 2.9165,
      "step": 45400
    },
    {
      "activations/layer0_attention_weight_max": 13.8914794921875,
      "activations/layer0_attention_weight_min": -14.749499320983887,
      "activations/layer10_attention_weight_max": 53.72774124145508,
      "activations/layer10_attention_weight_min": -39.6064453125,
      "activations/layer11_attention_weight_max": 38.800636291503906,
      "activations/layer11_attention_weight_min": -28.439258575439453,
      "activations/layer12_attention_weight_max": 26.674739837646484,
      "activations/layer12_attention_weight_min": -23.836013793945312,
      "activations/layer13_attention_weight_max": 67.41285705566406,
      "activations/layer13_attention_weight_min": -44.09941101074219,
      "activations/layer14_attention_weight_max": 59.04514694213867,
      "activations/layer14_attention_weight_min": -41.873939514160156,
      "activations/layer15_attention_weight_max": 52.79035949707031,
      "activations/layer15_attention_weight_min": -34.27962875366211,
      "activations/layer16_attention_weight_max": 42.64332962036133,
      "activations/layer16_attention_weight_min": -27.541345596313477,
      "activations/layer17_attention_weight_max": 66.81283569335938,
      "activations/layer17_attention_weight_min": -34.705841064453125,
      "activations/layer18_attention_weight_max": 70.63954162597656,
      "activations/layer18_attention_weight_min": -33.67190933227539,
      "activations/layer19_attention_weight_max": 24.471410751342773,
      "activations/layer19_attention_weight_min": -16.766387939453125,
      "activations/layer1_attention_weight_max": 17.697580337524414,
      "activations/layer1_attention_weight_min": -13.485944747924805,
      "activations/layer20_attention_weight_max": 22.998275756835938,
      "activations/layer20_attention_weight_min": -21.17442512512207,
      "activations/layer21_attention_weight_max": 48.44873046875,
      "activations/layer21_attention_weight_min": -29.138057708740234,
      "activations/layer22_attention_weight_max": 39.676387786865234,
      "activations/layer22_attention_weight_min": -23.655786514282227,
      "activations/layer23_attention_weight_max": 27.217113494873047,
      "activations/layer23_attention_weight_min": -16.76034164428711,
      "activations/layer2_attention_weight_max": 30.184972763061523,
      "activations/layer2_attention_weight_min": -28.908966064453125,
      "activations/layer3_attention_weight_max": 68.57093048095703,
      "activations/layer3_attention_weight_min": -74.4844970703125,
      "activations/layer4_attention_weight_max": 104.38478088378906,
      "activations/layer4_attention_weight_min": -96.24132537841797,
      "activations/layer5_attention_weight_max": 131.7623748779297,
      "activations/layer5_attention_weight_min": -107.33683776855469,
      "activations/layer6_attention_weight_max": 63.59195327758789,
      "activations/layer6_attention_weight_min": -54.77095031738281,
      "activations/layer7_attention_weight_max": 60.9975471496582,
      "activations/layer7_attention_weight_min": -55.27891159057617,
      "activations/layer8_attention_weight_max": 42.749427795410156,
      "activations/layer8_attention_weight_min": -37.87057113647461,
      "activations/layer9_attention_weight_max": 61.576454162597656,
      "activations/layer9_attention_weight_min": -55.73757553100586,
      "epoch": 2.64,
      "learning_rate": 0.00013432045454545453,
      "loss": 2.9395,
      "step": 45450
    },
    {
      "activations/layer0_attention_weight_max": 14.376296043395996,
      "activations/layer0_attention_weight_min": -14.52768325805664,
      "activations/layer10_attention_weight_max": 46.576595306396484,
      "activations/layer10_attention_weight_min": -42.217288970947266,
      "activations/layer11_attention_weight_max": 38.12477111816406,
      "activations/layer11_attention_weight_min": -28.921993255615234,
      "activations/layer12_attention_weight_max": 28.038959503173828,
      "activations/layer12_attention_weight_min": -23.386592864990234,
      "activations/layer13_attention_weight_max": 47.98006820678711,
      "activations/layer13_attention_weight_min": -38.921932220458984,
      "activations/layer14_attention_weight_max": 40.3862419128418,
      "activations/layer14_attention_weight_min": -31.279956817626953,
      "activations/layer15_attention_weight_max": 46.810001373291016,
      "activations/layer15_attention_weight_min": -34.037166595458984,
      "activations/layer16_attention_weight_max": 40.381771087646484,
      "activations/layer16_attention_weight_min": -27.58958625793457,
      "activations/layer17_attention_weight_max": 67.48307037353516,
      "activations/layer17_attention_weight_min": -36.62143325805664,
      "activations/layer18_attention_weight_max": 56.90534591674805,
      "activations/layer18_attention_weight_min": -32.753177642822266,
      "activations/layer19_attention_weight_max": 25.071943283081055,
      "activations/layer19_attention_weight_min": -15.016621589660645,
      "activations/layer1_attention_weight_max": 17.14217758178711,
      "activations/layer1_attention_weight_min": -12.941841125488281,
      "activations/layer20_attention_weight_max": 20.803152084350586,
      "activations/layer20_attention_weight_min": -18.721378326416016,
      "activations/layer21_attention_weight_max": 38.42241287231445,
      "activations/layer21_attention_weight_min": -28.41754150390625,
      "activations/layer22_attention_weight_max": 33.2279167175293,
      "activations/layer22_attention_weight_min": -22.91550636291504,
      "activations/layer23_attention_weight_max": 25.116291046142578,
      "activations/layer23_attention_weight_min": -17.042837142944336,
      "activations/layer2_attention_weight_max": 30.62373161315918,
      "activations/layer2_attention_weight_min": -26.82415008544922,
      "activations/layer3_attention_weight_max": 68.95838928222656,
      "activations/layer3_attention_weight_min": -70.26689147949219,
      "activations/layer4_attention_weight_max": 96.68212127685547,
      "activations/layer4_attention_weight_min": -90.49231719970703,
      "activations/layer5_attention_weight_max": 124.63377380371094,
      "activations/layer5_attention_weight_min": -103.9954833984375,
      "activations/layer6_attention_weight_max": 62.262962341308594,
      "activations/layer6_attention_weight_min": -51.68470764160156,
      "activations/layer7_attention_weight_max": 59.29388427734375,
      "activations/layer7_attention_weight_min": -53.195404052734375,
      "activations/layer8_attention_weight_max": 39.42352294921875,
      "activations/layer8_attention_weight_min": -36.45758819580078,
      "activations/layer9_attention_weight_max": 58.8494758605957,
      "activations/layer9_attention_weight_min": -50.59587478637695,
      "epoch": 2.64,
      "learning_rate": 0.00013430151515151514,
      "loss": 2.9263,
      "step": 45500
    },
    {
      "activations/layer0_attention_weight_max": 13.86296558380127,
      "activations/layer0_attention_weight_min": -14.653176307678223,
      "activations/layer10_attention_weight_max": 47.95023727416992,
      "activations/layer10_attention_weight_min": -39.78069305419922,
      "activations/layer11_attention_weight_max": 45.02272033691406,
      "activations/layer11_attention_weight_min": -29.466548919677734,
      "activations/layer12_attention_weight_max": 26.684837341308594,
      "activations/layer12_attention_weight_min": -23.72301483154297,
      "activations/layer13_attention_weight_max": 57.462615966796875,
      "activations/layer13_attention_weight_min": -41.256996154785156,
      "activations/layer14_attention_weight_max": 56.84734344482422,
      "activations/layer14_attention_weight_min": -38.39077377319336,
      "activations/layer15_attention_weight_max": 49.03404235839844,
      "activations/layer15_attention_weight_min": -36.509002685546875,
      "activations/layer16_attention_weight_max": 43.64234924316406,
      "activations/layer16_attention_weight_min": -28.943157196044922,
      "activations/layer17_attention_weight_max": 65.90260314941406,
      "activations/layer17_attention_weight_min": -36.459327697753906,
      "activations/layer18_attention_weight_max": 68.90459442138672,
      "activations/layer18_attention_weight_min": -35.576080322265625,
      "activations/layer19_attention_weight_max": 27.1728572845459,
      "activations/layer19_attention_weight_min": -18.095809936523438,
      "activations/layer1_attention_weight_max": 17.197866439819336,
      "activations/layer1_attention_weight_min": -15.364226341247559,
      "activations/layer20_attention_weight_max": 26.912824630737305,
      "activations/layer20_attention_weight_min": -18.92957305908203,
      "activations/layer21_attention_weight_max": 45.798614501953125,
      "activations/layer21_attention_weight_min": -28.273244857788086,
      "activations/layer22_attention_weight_max": 45.144405364990234,
      "activations/layer22_attention_weight_min": -24.06580924987793,
      "activations/layer23_attention_weight_max": 27.69179344177246,
      "activations/layer23_attention_weight_min": -16.211502075195312,
      "activations/layer2_attention_weight_max": 29.375587463378906,
      "activations/layer2_attention_weight_min": -26.87967300415039,
      "activations/layer3_attention_weight_max": 68.23403930664062,
      "activations/layer3_attention_weight_min": -69.29723358154297,
      "activations/layer4_attention_weight_max": 95.37753295898438,
      "activations/layer4_attention_weight_min": -86.47740173339844,
      "activations/layer5_attention_weight_max": 119.68817138671875,
      "activations/layer5_attention_weight_min": -99.89215087890625,
      "activations/layer6_attention_weight_max": 59.39646530151367,
      "activations/layer6_attention_weight_min": -50.42909240722656,
      "activations/layer7_attention_weight_max": 54.57328414916992,
      "activations/layer7_attention_weight_min": -55.20198440551758,
      "activations/layer8_attention_weight_max": 40.13593292236328,
      "activations/layer8_attention_weight_min": -35.585933685302734,
      "activations/layer9_attention_weight_max": 53.525604248046875,
      "activations/layer9_attention_weight_min": -49.22703170776367,
      "epoch": 2.65,
      "learning_rate": 0.00013428257575757576,
      "loss": 2.9413,
      "step": 45550
    },
    {
      "activations/layer0_attention_weight_max": 13.7946138381958,
      "activations/layer0_attention_weight_min": -14.99052619934082,
      "activations/layer10_attention_weight_max": 47.2010383605957,
      "activations/layer10_attention_weight_min": -39.00873565673828,
      "activations/layer11_attention_weight_max": 48.204444885253906,
      "activations/layer11_attention_weight_min": -31.70380973815918,
      "activations/layer12_attention_weight_max": 24.38653564453125,
      "activations/layer12_attention_weight_min": -26.711177825927734,
      "activations/layer13_attention_weight_max": 63.206424713134766,
      "activations/layer13_attention_weight_min": -44.016578674316406,
      "activations/layer14_attention_weight_max": 51.85853576660156,
      "activations/layer14_attention_weight_min": -34.96253967285156,
      "activations/layer15_attention_weight_max": 51.680660247802734,
      "activations/layer15_attention_weight_min": -33.45176315307617,
      "activations/layer16_attention_weight_max": 37.43118667602539,
      "activations/layer16_attention_weight_min": -27.720996856689453,
      "activations/layer17_attention_weight_max": 57.32498550415039,
      "activations/layer17_attention_weight_min": -35.603824615478516,
      "activations/layer18_attention_weight_max": 49.5091667175293,
      "activations/layer18_attention_weight_min": -29.76388168334961,
      "activations/layer19_attention_weight_max": 21.786823272705078,
      "activations/layer19_attention_weight_min": -17.473857879638672,
      "activations/layer1_attention_weight_max": 16.542299270629883,
      "activations/layer1_attention_weight_min": -15.334249496459961,
      "activations/layer20_attention_weight_max": 22.78707504272461,
      "activations/layer20_attention_weight_min": -18.785463333129883,
      "activations/layer21_attention_weight_max": 42.33137130737305,
      "activations/layer21_attention_weight_min": -27.685293197631836,
      "activations/layer22_attention_weight_max": 35.34576416015625,
      "activations/layer22_attention_weight_min": -21.782730102539062,
      "activations/layer23_attention_weight_max": 22.89220428466797,
      "activations/layer23_attention_weight_min": -17.051727294921875,
      "activations/layer2_attention_weight_max": 28.694744110107422,
      "activations/layer2_attention_weight_min": -27.168075561523438,
      "activations/layer3_attention_weight_max": 71.16182708740234,
      "activations/layer3_attention_weight_min": -73.86136627197266,
      "activations/layer4_attention_weight_max": 103.57592010498047,
      "activations/layer4_attention_weight_min": -94.14092254638672,
      "activations/layer5_attention_weight_max": 135.0047607421875,
      "activations/layer5_attention_weight_min": -108.27726745605469,
      "activations/layer6_attention_weight_max": 66.26270294189453,
      "activations/layer6_attention_weight_min": -53.4911003112793,
      "activations/layer7_attention_weight_max": 58.27878952026367,
      "activations/layer7_attention_weight_min": -55.36371612548828,
      "activations/layer8_attention_weight_max": 40.950050354003906,
      "activations/layer8_attention_weight_min": -36.839569091796875,
      "activations/layer9_attention_weight_max": 54.024513244628906,
      "activations/layer9_attention_weight_min": -62.08041000366211,
      "epoch": 2.65,
      "learning_rate": 0.00013426401515151515,
      "loss": 2.9257,
      "step": 45600
    },
    {
      "activations/layer0_attention_weight_max": 14.039972305297852,
      "activations/layer0_attention_weight_min": -14.594452857971191,
      "activations/layer10_attention_weight_max": 49.22815704345703,
      "activations/layer10_attention_weight_min": -37.23543930053711,
      "activations/layer11_attention_weight_max": 41.189231872558594,
      "activations/layer11_attention_weight_min": -29.30307388305664,
      "activations/layer12_attention_weight_max": 30.259483337402344,
      "activations/layer12_attention_weight_min": -23.16988754272461,
      "activations/layer13_attention_weight_max": 60.930633544921875,
      "activations/layer13_attention_weight_min": -40.928932189941406,
      "activations/layer14_attention_weight_max": 49.30061721801758,
      "activations/layer14_attention_weight_min": -33.355690002441406,
      "activations/layer15_attention_weight_max": 52.9235954284668,
      "activations/layer15_attention_weight_min": -33.599708557128906,
      "activations/layer16_attention_weight_max": 47.80458450317383,
      "activations/layer16_attention_weight_min": -27.708478927612305,
      "activations/layer17_attention_weight_max": 64.28923034667969,
      "activations/layer17_attention_weight_min": -34.87114334106445,
      "activations/layer18_attention_weight_max": 64.88589477539062,
      "activations/layer18_attention_weight_min": -32.19099044799805,
      "activations/layer19_attention_weight_max": 24.597076416015625,
      "activations/layer19_attention_weight_min": -18.019956588745117,
      "activations/layer1_attention_weight_max": 16.492643356323242,
      "activations/layer1_attention_weight_min": -13.021599769592285,
      "activations/layer20_attention_weight_max": 25.944133758544922,
      "activations/layer20_attention_weight_min": -17.677947998046875,
      "activations/layer21_attention_weight_max": 48.83615493774414,
      "activations/layer21_attention_weight_min": -28.43841552734375,
      "activations/layer22_attention_weight_max": 38.50701141357422,
      "activations/layer22_attention_weight_min": -21.62701416015625,
      "activations/layer23_attention_weight_max": 29.48729705810547,
      "activations/layer23_attention_weight_min": -14.942099571228027,
      "activations/layer2_attention_weight_max": 28.79848289489746,
      "activations/layer2_attention_weight_min": -25.99929428100586,
      "activations/layer3_attention_weight_max": 68.39884185791016,
      "activations/layer3_attention_weight_min": -70.29993438720703,
      "activations/layer4_attention_weight_max": 98.46861267089844,
      "activations/layer4_attention_weight_min": -85.97705078125,
      "activations/layer5_attention_weight_max": 125.31912231445312,
      "activations/layer5_attention_weight_min": -97.98420715332031,
      "activations/layer6_attention_weight_max": 63.78889846801758,
      "activations/layer6_attention_weight_min": -48.539634704589844,
      "activations/layer7_attention_weight_max": 63.66792678833008,
      "activations/layer7_attention_weight_min": -54.967323303222656,
      "activations/layer8_attention_weight_max": 43.204856872558594,
      "activations/layer8_attention_weight_min": -38.6595344543457,
      "activations/layer9_attention_weight_max": 63.02214431762695,
      "activations/layer9_attention_weight_min": -53.86259078979492,
      "epoch": 2.65,
      "learning_rate": 0.00013424507575757576,
      "loss": 2.9361,
      "step": 45650
    },
    {
      "activations/layer0_attention_weight_max": 13.803455352783203,
      "activations/layer0_attention_weight_min": -14.140543937683105,
      "activations/layer10_attention_weight_max": 48.65288162231445,
      "activations/layer10_attention_weight_min": -42.163848876953125,
      "activations/layer11_attention_weight_max": 36.918853759765625,
      "activations/layer11_attention_weight_min": -28.970844268798828,
      "activations/layer12_attention_weight_max": 25.93467903137207,
      "activations/layer12_attention_weight_min": -25.389860153198242,
      "activations/layer13_attention_weight_max": 75.50402069091797,
      "activations/layer13_attention_weight_min": -49.003746032714844,
      "activations/layer14_attention_weight_max": 58.99763107299805,
      "activations/layer14_attention_weight_min": -40.202144622802734,
      "activations/layer15_attention_weight_max": 55.3382568359375,
      "activations/layer15_attention_weight_min": -34.76577377319336,
      "activations/layer16_attention_weight_max": 43.16803741455078,
      "activations/layer16_attention_weight_min": -27.424013137817383,
      "activations/layer17_attention_weight_max": 62.46229934692383,
      "activations/layer17_attention_weight_min": -36.22095489501953,
      "activations/layer18_attention_weight_max": 59.8094596862793,
      "activations/layer18_attention_weight_min": -32.9883918762207,
      "activations/layer19_attention_weight_max": 21.637876510620117,
      "activations/layer19_attention_weight_min": -17.669384002685547,
      "activations/layer1_attention_weight_max": 16.876140594482422,
      "activations/layer1_attention_weight_min": -13.995076179504395,
      "activations/layer20_attention_weight_max": 21.306564331054688,
      "activations/layer20_attention_weight_min": -18.37874412536621,
      "activations/layer21_attention_weight_max": 40.16651153564453,
      "activations/layer21_attention_weight_min": -28.15099334716797,
      "activations/layer22_attention_weight_max": 36.20305252075195,
      "activations/layer22_attention_weight_min": -23.732555389404297,
      "activations/layer23_attention_weight_max": 23.84969711303711,
      "activations/layer23_attention_weight_min": -16.54471206665039,
      "activations/layer2_attention_weight_max": 29.946205139160156,
      "activations/layer2_attention_weight_min": -28.75080680847168,
      "activations/layer3_attention_weight_max": 66.73246765136719,
      "activations/layer3_attention_weight_min": -72.0704345703125,
      "activations/layer4_attention_weight_max": 93.5540771484375,
      "activations/layer4_attention_weight_min": -87.24810028076172,
      "activations/layer5_attention_weight_max": 118.27374267578125,
      "activations/layer5_attention_weight_min": -100.46537780761719,
      "activations/layer6_attention_weight_max": 61.36540603637695,
      "activations/layer6_attention_weight_min": -50.48891067504883,
      "activations/layer7_attention_weight_max": 64.75435638427734,
      "activations/layer7_attention_weight_min": -56.146724700927734,
      "activations/layer8_attention_weight_max": 40.69004440307617,
      "activations/layer8_attention_weight_min": -37.73509979248047,
      "activations/layer9_attention_weight_max": 62.530517578125,
      "activations/layer9_attention_weight_min": -54.33541488647461,
      "epoch": 2.66,
      "learning_rate": 0.00013422613636363635,
      "loss": 2.9235,
      "step": 45700
    },
    {
      "activations/layer0_attention_weight_max": 14.745210647583008,
      "activations/layer0_attention_weight_min": -14.027099609375,
      "activations/layer10_attention_weight_max": 47.117340087890625,
      "activations/layer10_attention_weight_min": -38.54964065551758,
      "activations/layer11_attention_weight_max": 38.43559646606445,
      "activations/layer11_attention_weight_min": -31.380393981933594,
      "activations/layer12_attention_weight_max": 25.87241554260254,
      "activations/layer12_attention_weight_min": -23.528173446655273,
      "activations/layer13_attention_weight_max": 70.91839599609375,
      "activations/layer13_attention_weight_min": -60.06541061401367,
      "activations/layer14_attention_weight_max": 68.431396484375,
      "activations/layer14_attention_weight_min": -51.0120735168457,
      "activations/layer15_attention_weight_max": 47.07735824584961,
      "activations/layer15_attention_weight_min": -32.86638259887695,
      "activations/layer16_attention_weight_max": 34.081031799316406,
      "activations/layer16_attention_weight_min": -26.157108306884766,
      "activations/layer17_attention_weight_max": 53.300148010253906,
      "activations/layer17_attention_weight_min": -33.90839767456055,
      "activations/layer18_attention_weight_max": 49.268035888671875,
      "activations/layer18_attention_weight_min": -30.216678619384766,
      "activations/layer19_attention_weight_max": 16.764760971069336,
      "activations/layer19_attention_weight_min": -18.0146541595459,
      "activations/layer1_attention_weight_max": 17.965717315673828,
      "activations/layer1_attention_weight_min": -14.397506713867188,
      "activations/layer20_attention_weight_max": 19.95298194885254,
      "activations/layer20_attention_weight_min": -17.284181594848633,
      "activations/layer21_attention_weight_max": 36.0327033996582,
      "activations/layer21_attention_weight_min": -26.112634658813477,
      "activations/layer22_attention_weight_max": 34.54624938964844,
      "activations/layer22_attention_weight_min": -20.58670997619629,
      "activations/layer23_attention_weight_max": 22.321582794189453,
      "activations/layer23_attention_weight_min": -16.357942581176758,
      "activations/layer2_attention_weight_max": 26.526203155517578,
      "activations/layer2_attention_weight_min": -24.54218101501465,
      "activations/layer3_attention_weight_max": 63.24905014038086,
      "activations/layer3_attention_weight_min": -64.9924087524414,
      "activations/layer4_attention_weight_max": 95.1236801147461,
      "activations/layer4_attention_weight_min": -84.49164581298828,
      "activations/layer5_attention_weight_max": 122.5821762084961,
      "activations/layer5_attention_weight_min": -94.02473449707031,
      "activations/layer6_attention_weight_max": 58.66936111450195,
      "activations/layer6_attention_weight_min": -48.315269470214844,
      "activations/layer7_attention_weight_max": 64.70391845703125,
      "activations/layer7_attention_weight_min": -51.123470306396484,
      "activations/layer8_attention_weight_max": 52.44721221923828,
      "activations/layer8_attention_weight_min": -36.70071029663086,
      "activations/layer9_attention_weight_max": 77.2536849975586,
      "activations/layer9_attention_weight_min": -55.20415115356445,
      "epoch": 2.66,
      "learning_rate": 0.00013420719696969697,
      "loss": 2.9082,
      "step": 45750
    },
    {
      "activations/layer0_attention_weight_max": 15.459272384643555,
      "activations/layer0_attention_weight_min": -14.246635437011719,
      "activations/layer10_attention_weight_max": 56.44645309448242,
      "activations/layer10_attention_weight_min": -40.34218215942383,
      "activations/layer11_attention_weight_max": 44.66300582885742,
      "activations/layer11_attention_weight_min": -30.005216598510742,
      "activations/layer12_attention_weight_max": 30.735445022583008,
      "activations/layer12_attention_weight_min": -26.564937591552734,
      "activations/layer13_attention_weight_max": 60.71381759643555,
      "activations/layer13_attention_weight_min": -39.35401916503906,
      "activations/layer14_attention_weight_max": 43.416664123535156,
      "activations/layer14_attention_weight_min": -28.959976196289062,
      "activations/layer15_attention_weight_max": 48.38936233520508,
      "activations/layer15_attention_weight_min": -36.30617904663086,
      "activations/layer16_attention_weight_max": 40.75070571899414,
      "activations/layer16_attention_weight_min": -28.420454025268555,
      "activations/layer17_attention_weight_max": 52.32414627075195,
      "activations/layer17_attention_weight_min": -35.4697380065918,
      "activations/layer18_attention_weight_max": 57.03574752807617,
      "activations/layer18_attention_weight_min": -34.345726013183594,
      "activations/layer19_attention_weight_max": 22.426172256469727,
      "activations/layer19_attention_weight_min": -15.684816360473633,
      "activations/layer1_attention_weight_max": 17.009662628173828,
      "activations/layer1_attention_weight_min": -13.100292205810547,
      "activations/layer20_attention_weight_max": 20.218605041503906,
      "activations/layer20_attention_weight_min": -17.973217010498047,
      "activations/layer21_attention_weight_max": 39.02015686035156,
      "activations/layer21_attention_weight_min": -28.93453598022461,
      "activations/layer22_attention_weight_max": 35.74660110473633,
      "activations/layer22_attention_weight_min": -23.434059143066406,
      "activations/layer23_attention_weight_max": 25.979450225830078,
      "activations/layer23_attention_weight_min": -15.765280723571777,
      "activations/layer2_attention_weight_max": 30.97103500366211,
      "activations/layer2_attention_weight_min": -28.052709579467773,
      "activations/layer3_attention_weight_max": 75.2510757446289,
      "activations/layer3_attention_weight_min": -76.31219482421875,
      "activations/layer4_attention_weight_max": 108.68816375732422,
      "activations/layer4_attention_weight_min": -94.28713989257812,
      "activations/layer5_attention_weight_max": 138.32022094726562,
      "activations/layer5_attention_weight_min": -107.72933197021484,
      "activations/layer6_attention_weight_max": 64.22566986083984,
      "activations/layer6_attention_weight_min": -51.700523376464844,
      "activations/layer7_attention_weight_max": 73.61618041992188,
      "activations/layer7_attention_weight_min": -54.31695556640625,
      "activations/layer8_attention_weight_max": 48.245079040527344,
      "activations/layer8_attention_weight_min": -37.08220291137695,
      "activations/layer9_attention_weight_max": 74.17293548583984,
      "activations/layer9_attention_weight_min": -54.46509552001953,
      "epoch": 2.66,
      "learning_rate": 0.00013418825757575755,
      "loss": 2.9329,
      "step": 45800
    },
    {
      "activations/layer0_attention_weight_max": 14.119603157043457,
      "activations/layer0_attention_weight_min": -14.657081604003906,
      "activations/layer10_attention_weight_max": 47.2652473449707,
      "activations/layer10_attention_weight_min": -38.151004791259766,
      "activations/layer11_attention_weight_max": 39.428855895996094,
      "activations/layer11_attention_weight_min": -30.215124130249023,
      "activations/layer12_attention_weight_max": 28.705421447753906,
      "activations/layer12_attention_weight_min": -23.768415451049805,
      "activations/layer13_attention_weight_max": 53.2445182800293,
      "activations/layer13_attention_weight_min": -44.48841857910156,
      "activations/layer14_attention_weight_max": 43.09309005737305,
      "activations/layer14_attention_weight_min": -36.34084701538086,
      "activations/layer15_attention_weight_max": 48.370872497558594,
      "activations/layer15_attention_weight_min": -37.10165786743164,
      "activations/layer16_attention_weight_max": 36.44032287597656,
      "activations/layer16_attention_weight_min": -27.87254524230957,
      "activations/layer17_attention_weight_max": 58.63056182861328,
      "activations/layer17_attention_weight_min": -35.63644027709961,
      "activations/layer18_attention_weight_max": 47.28479766845703,
      "activations/layer18_attention_weight_min": -30.39349937438965,
      "activations/layer19_attention_weight_max": 19.914127349853516,
      "activations/layer19_attention_weight_min": -16.138601303100586,
      "activations/layer1_attention_weight_max": 16.60176658630371,
      "activations/layer1_attention_weight_min": -16.361478805541992,
      "activations/layer20_attention_weight_max": 20.280794143676758,
      "activations/layer20_attention_weight_min": -21.97238540649414,
      "activations/layer21_attention_weight_max": 39.50580596923828,
      "activations/layer21_attention_weight_min": -26.210819244384766,
      "activations/layer22_attention_weight_max": 36.156288146972656,
      "activations/layer22_attention_weight_min": -22.992374420166016,
      "activations/layer23_attention_weight_max": 21.852895736694336,
      "activations/layer23_attention_weight_min": -14.896661758422852,
      "activations/layer2_attention_weight_max": 26.79884910583496,
      "activations/layer2_attention_weight_min": -25.026090621948242,
      "activations/layer3_attention_weight_max": 67.00664520263672,
      "activations/layer3_attention_weight_min": -69.79859161376953,
      "activations/layer4_attention_weight_max": 92.8539047241211,
      "activations/layer4_attention_weight_min": -84.22997283935547,
      "activations/layer5_attention_weight_max": 119.93440246582031,
      "activations/layer5_attention_weight_min": -98.22103881835938,
      "activations/layer6_attention_weight_max": 59.5340576171875,
      "activations/layer6_attention_weight_min": -50.64925765991211,
      "activations/layer7_attention_weight_max": 53.33697509765625,
      "activations/layer7_attention_weight_min": -54.017791748046875,
      "activations/layer8_attention_weight_max": 39.389190673828125,
      "activations/layer8_attention_weight_min": -35.10307693481445,
      "activations/layer9_attention_weight_max": 53.9150276184082,
      "activations/layer9_attention_weight_min": -49.888607025146484,
      "epoch": 2.66,
      "learning_rate": 0.00013416969696969697,
      "loss": 2.9444,
      "step": 45850
    },
    {
      "activations/layer0_attention_weight_max": 14.713777542114258,
      "activations/layer0_attention_weight_min": -14.634710311889648,
      "activations/layer10_attention_weight_max": 49.83545684814453,
      "activations/layer10_attention_weight_min": -41.43117904663086,
      "activations/layer11_attention_weight_max": 40.01494598388672,
      "activations/layer11_attention_weight_min": -33.467411041259766,
      "activations/layer12_attention_weight_max": 28.857669830322266,
      "activations/layer12_attention_weight_min": -27.470863342285156,
      "activations/layer13_attention_weight_max": 92.84823608398438,
      "activations/layer13_attention_weight_min": -60.19518280029297,
      "activations/layer14_attention_weight_max": 81.03153991699219,
      "activations/layer14_attention_weight_min": -52.810333251953125,
      "activations/layer15_attention_weight_max": 56.447967529296875,
      "activations/layer15_attention_weight_min": -36.48588180541992,
      "activations/layer16_attention_weight_max": 42.59095764160156,
      "activations/layer16_attention_weight_min": -26.54343605041504,
      "activations/layer17_attention_weight_max": 60.7825813293457,
      "activations/layer17_attention_weight_min": -36.146812438964844,
      "activations/layer18_attention_weight_max": 56.552059173583984,
      "activations/layer18_attention_weight_min": -31.273906707763672,
      "activations/layer19_attention_weight_max": 22.659658432006836,
      "activations/layer19_attention_weight_min": -18.240299224853516,
      "activations/layer1_attention_weight_max": 18.066062927246094,
      "activations/layer1_attention_weight_min": -14.418363571166992,
      "activations/layer20_attention_weight_max": 20.494413375854492,
      "activations/layer20_attention_weight_min": -17.918054580688477,
      "activations/layer21_attention_weight_max": 47.45143508911133,
      "activations/layer21_attention_weight_min": -27.751312255859375,
      "activations/layer22_attention_weight_max": 38.092918395996094,
      "activations/layer22_attention_weight_min": -21.933486938476562,
      "activations/layer23_attention_weight_max": 24.506671905517578,
      "activations/layer23_attention_weight_min": -15.951330184936523,
      "activations/layer2_attention_weight_max": 28.314754486083984,
      "activations/layer2_attention_weight_min": -26.325183868408203,
      "activations/layer3_attention_weight_max": 67.83634185791016,
      "activations/layer3_attention_weight_min": -71.78375244140625,
      "activations/layer4_attention_weight_max": 99.05180358886719,
      "activations/layer4_attention_weight_min": -89.84832000732422,
      "activations/layer5_attention_weight_max": 123.21935272216797,
      "activations/layer5_attention_weight_min": -100.38249206542969,
      "activations/layer6_attention_weight_max": 62.136558532714844,
      "activations/layer6_attention_weight_min": -50.1410026550293,
      "activations/layer7_attention_weight_max": 57.799171447753906,
      "activations/layer7_attention_weight_min": -54.797550201416016,
      "activations/layer8_attention_weight_max": 43.8060188293457,
      "activations/layer8_attention_weight_min": -36.19649124145508,
      "activations/layer9_attention_weight_max": 54.76943588256836,
      "activations/layer9_attention_weight_min": -49.32609939575195,
      "epoch": 2.67,
      "learning_rate": 0.00013415075757575756,
      "loss": 2.9493,
      "step": 45900
    },
    {
      "activations/layer0_attention_weight_max": 13.682807922363281,
      "activations/layer0_attention_weight_min": -15.062280654907227,
      "activations/layer10_attention_weight_max": 49.202327728271484,
      "activations/layer10_attention_weight_min": -38.32033157348633,
      "activations/layer11_attention_weight_max": 40.012733459472656,
      "activations/layer11_attention_weight_min": -29.345409393310547,
      "activations/layer12_attention_weight_max": 28.987106323242188,
      "activations/layer12_attention_weight_min": -26.48827362060547,
      "activations/layer13_attention_weight_max": 54.28055191040039,
      "activations/layer13_attention_weight_min": -38.553985595703125,
      "activations/layer14_attention_weight_max": 48.073951721191406,
      "activations/layer14_attention_weight_min": -33.61469268798828,
      "activations/layer15_attention_weight_max": 55.685142517089844,
      "activations/layer15_attention_weight_min": -34.64813232421875,
      "activations/layer16_attention_weight_max": 40.195838928222656,
      "activations/layer16_attention_weight_min": -28.58403968811035,
      "activations/layer17_attention_weight_max": 68.4600601196289,
      "activations/layer17_attention_weight_min": -34.809391021728516,
      "activations/layer18_attention_weight_max": 53.72232437133789,
      "activations/layer18_attention_weight_min": -32.08357238769531,
      "activations/layer19_attention_weight_max": 21.201669692993164,
      "activations/layer19_attention_weight_min": -18.341646194458008,
      "activations/layer1_attention_weight_max": 17.973712921142578,
      "activations/layer1_attention_weight_min": -13.07176399230957,
      "activations/layer20_attention_weight_max": 23.833824157714844,
      "activations/layer20_attention_weight_min": -19.766098022460938,
      "activations/layer21_attention_weight_max": 43.462074279785156,
      "activations/layer21_attention_weight_min": -25.527036666870117,
      "activations/layer22_attention_weight_max": 35.023807525634766,
      "activations/layer22_attention_weight_min": -22.311792373657227,
      "activations/layer23_attention_weight_max": 23.716711044311523,
      "activations/layer23_attention_weight_min": -16.331005096435547,
      "activations/layer2_attention_weight_max": 27.952293395996094,
      "activations/layer2_attention_weight_min": -26.200292587280273,
      "activations/layer3_attention_weight_max": 67.11955261230469,
      "activations/layer3_attention_weight_min": -71.36740112304688,
      "activations/layer4_attention_weight_max": 96.75739288330078,
      "activations/layer4_attention_weight_min": -86.88033294677734,
      "activations/layer5_attention_weight_max": 121.8837890625,
      "activations/layer5_attention_weight_min": -102.38450622558594,
      "activations/layer6_attention_weight_max": 62.87251281738281,
      "activations/layer6_attention_weight_min": -51.51283645629883,
      "activations/layer7_attention_weight_max": 58.4003791809082,
      "activations/layer7_attention_weight_min": -53.53281021118164,
      "activations/layer8_attention_weight_max": 42.177581787109375,
      "activations/layer8_attention_weight_min": -37.27702331542969,
      "activations/layer9_attention_weight_max": 55.37006378173828,
      "activations/layer9_attention_weight_min": -54.62504196166992,
      "epoch": 2.67,
      "learning_rate": 0.00013413181818181817,
      "loss": 2.9314,
      "step": 45950
    },
    {
      "activations/layer0_attention_weight_max": 13.212992668151855,
      "activations/layer0_attention_weight_min": -14.462471961975098,
      "activations/layer10_attention_weight_max": 56.13429641723633,
      "activations/layer10_attention_weight_min": -40.82563400268555,
      "activations/layer11_attention_weight_max": 43.1690788269043,
      "activations/layer11_attention_weight_min": -31.874629974365234,
      "activations/layer12_attention_weight_max": 40.67015838623047,
      "activations/layer12_attention_weight_min": -24.62724494934082,
      "activations/layer13_attention_weight_max": 59.282657623291016,
      "activations/layer13_attention_weight_min": -39.82168197631836,
      "activations/layer14_attention_weight_max": 51.74258804321289,
      "activations/layer14_attention_weight_min": -34.5584602355957,
      "activations/layer15_attention_weight_max": 54.4408073425293,
      "activations/layer15_attention_weight_min": -36.20854949951172,
      "activations/layer16_attention_weight_max": 41.129905700683594,
      "activations/layer16_attention_weight_min": -28.195207595825195,
      "activations/layer17_attention_weight_max": 80.46340942382812,
      "activations/layer17_attention_weight_min": -43.506595611572266,
      "activations/layer18_attention_weight_max": 58.2344856262207,
      "activations/layer18_attention_weight_min": -34.18785858154297,
      "activations/layer19_attention_weight_max": 23.7465877532959,
      "activations/layer19_attention_weight_min": -17.370864868164062,
      "activations/layer1_attention_weight_max": 17.3077449798584,
      "activations/layer1_attention_weight_min": -14.240318298339844,
      "activations/layer20_attention_weight_max": 21.824018478393555,
      "activations/layer20_attention_weight_min": -18.80699920654297,
      "activations/layer21_attention_weight_max": 47.31681823730469,
      "activations/layer21_attention_weight_min": -29.04339599609375,
      "activations/layer22_attention_weight_max": 38.92732238769531,
      "activations/layer22_attention_weight_min": -24.498634338378906,
      "activations/layer23_attention_weight_max": 24.12994384765625,
      "activations/layer23_attention_weight_min": -15.833094596862793,
      "activations/layer2_attention_weight_max": 29.92262077331543,
      "activations/layer2_attention_weight_min": -27.652095794677734,
      "activations/layer3_attention_weight_max": 70.57097625732422,
      "activations/layer3_attention_weight_min": -76.20857238769531,
      "activations/layer4_attention_weight_max": 101.2901840209961,
      "activations/layer4_attention_weight_min": -94.23650360107422,
      "activations/layer5_attention_weight_max": 130.23095703125,
      "activations/layer5_attention_weight_min": -109.8643798828125,
      "activations/layer6_attention_weight_max": 66.15159606933594,
      "activations/layer6_attention_weight_min": -53.95160675048828,
      "activations/layer7_attention_weight_max": 58.052433013916016,
      "activations/layer7_attention_weight_min": -54.68745422363281,
      "activations/layer8_attention_weight_max": 45.69853591918945,
      "activations/layer8_attention_weight_min": -35.24217987060547,
      "activations/layer9_attention_weight_max": 58.71046829223633,
      "activations/layer9_attention_weight_min": -48.949405670166016,
      "epoch": 2.67,
      "learning_rate": 0.00013411287878787876,
      "loss": 2.9243,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_loss": 2.875,
      "eval_runtime": 8.4933,
      "eval_samples_per_second": 505.578,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_openwebtext_loss": 2.875,
      "eval_openwebtext_ppl": 17.725424121461643,
      "eval_openwebtext_runtime": 8.4933,
      "eval_openwebtext_samples_per_second": 505.578,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_wikitext_loss": 3.123046875,
      "eval_wikitext_ppl": 22.715485556228362,
      "eval_wikitext_runtime": 1.9382,
      "eval_wikitext_samples_per_second": 235.265,
      "step": 46000
    },
    {
      "epoch": 2.67,
      "eval_lambada_loss": 2.953125,
      "eval_lambada_ppl": 19.165753221675935,
      "eval_lambada_runtime": 9.5401,
      "eval_lambada_samples_per_second": 510.375,
      "step": 46000
    },
    {
      "activations/layer0_attention_weight_max": 14.626579284667969,
      "activations/layer0_attention_weight_min": -15.765973091125488,
      "activations/layer10_attention_weight_max": 58.778045654296875,
      "activations/layer10_attention_weight_min": -39.538482666015625,
      "activations/layer11_attention_weight_max": 47.6793327331543,
      "activations/layer11_attention_weight_min": -32.989749908447266,
      "activations/layer12_attention_weight_max": 28.65862464904785,
      "activations/layer12_attention_weight_min": -25.088790893554688,
      "activations/layer13_attention_weight_max": 73.099609375,
      "activations/layer13_attention_weight_min": -42.57981491088867,
      "activations/layer14_attention_weight_max": 67.995361328125,
      "activations/layer14_attention_weight_min": -39.28465270996094,
      "activations/layer15_attention_weight_max": 55.7725944519043,
      "activations/layer15_attention_weight_min": -35.30760955810547,
      "activations/layer16_attention_weight_max": 45.26607131958008,
      "activations/layer16_attention_weight_min": -28.237455368041992,
      "activations/layer17_attention_weight_max": 67.96732330322266,
      "activations/layer17_attention_weight_min": -35.65922927856445,
      "activations/layer18_attention_weight_max": 64.93487548828125,
      "activations/layer18_attention_weight_min": -33.311195373535156,
      "activations/layer19_attention_weight_max": 29.575672149658203,
      "activations/layer19_attention_weight_min": -16.741806030273438,
      "activations/layer1_attention_weight_max": 17.50420379638672,
      "activations/layer1_attention_weight_min": -13.759848594665527,
      "activations/layer20_attention_weight_max": 23.78651237487793,
      "activations/layer20_attention_weight_min": -18.682178497314453,
      "activations/layer21_attention_weight_max": 42.78285598754883,
      "activations/layer21_attention_weight_min": -27.866647720336914,
      "activations/layer22_attention_weight_max": 39.39078140258789,
      "activations/layer22_attention_weight_min": -21.852893829345703,
      "activations/layer23_attention_weight_max": 24.373455047607422,
      "activations/layer23_attention_weight_min": -15.624918937683105,
      "activations/layer2_attention_weight_max": 28.03322410583496,
      "activations/layer2_attention_weight_min": -26.681608200073242,
      "activations/layer3_attention_weight_max": 74.6441879272461,
      "activations/layer3_attention_weight_min": -72.37902069091797,
      "activations/layer4_attention_weight_max": 105.88521575927734,
      "activations/layer4_attention_weight_min": -89.30175018310547,
      "activations/layer5_attention_weight_max": 136.25921630859375,
      "activations/layer5_attention_weight_min": -102.82255554199219,
      "activations/layer6_attention_weight_max": 68.87041473388672,
      "activations/layer6_attention_weight_min": -51.30115509033203,
      "activations/layer7_attention_weight_max": 71.70698547363281,
      "activations/layer7_attention_weight_min": -54.20181655883789,
      "activations/layer8_attention_weight_max": 46.49692153930664,
      "activations/layer8_attention_weight_min": -38.26538848876953,
      "activations/layer9_attention_weight_max": 63.48682403564453,
      "activations/layer9_attention_weight_min": -53.76676559448242,
      "epoch": 2.68,
      "learning_rate": 0.00013409393939393938,
      "loss": 2.9339,
      "step": 46050
    },
    {
      "activations/layer0_attention_weight_max": 13.78249454498291,
      "activations/layer0_attention_weight_min": -14.905077934265137,
      "activations/layer10_attention_weight_max": 48.27848815917969,
      "activations/layer10_attention_weight_min": -38.2796630859375,
      "activations/layer11_attention_weight_max": 39.41292190551758,
      "activations/layer11_attention_weight_min": -32.213218688964844,
      "activations/layer12_attention_weight_max": 27.863981246948242,
      "activations/layer12_attention_weight_min": -22.409881591796875,
      "activations/layer13_attention_weight_max": 55.247283935546875,
      "activations/layer13_attention_weight_min": -41.624359130859375,
      "activations/layer14_attention_weight_max": 51.934043884277344,
      "activations/layer14_attention_weight_min": -37.503475189208984,
      "activations/layer15_attention_weight_max": 52.96968078613281,
      "activations/layer15_attention_weight_min": -34.47294616699219,
      "activations/layer16_attention_weight_max": 43.65271759033203,
      "activations/layer16_attention_weight_min": -26.3392276763916,
      "activations/layer17_attention_weight_max": 61.12004852294922,
      "activations/layer17_attention_weight_min": -36.61452102661133,
      "activations/layer18_attention_weight_max": 57.755191802978516,
      "activations/layer18_attention_weight_min": -31.507911682128906,
      "activations/layer19_attention_weight_max": 20.277029037475586,
      "activations/layer19_attention_weight_min": -18.180967330932617,
      "activations/layer1_attention_weight_max": 18.028316497802734,
      "activations/layer1_attention_weight_min": -13.605337142944336,
      "activations/layer20_attention_weight_max": 20.995481491088867,
      "activations/layer20_attention_weight_min": -18.32084846496582,
      "activations/layer21_attention_weight_max": 42.86162185668945,
      "activations/layer21_attention_weight_min": -27.315114974975586,
      "activations/layer22_attention_weight_max": 38.20850372314453,
      "activations/layer22_attention_weight_min": -26.10471534729004,
      "activations/layer23_attention_weight_max": 26.174419403076172,
      "activations/layer23_attention_weight_min": -16.344362258911133,
      "activations/layer2_attention_weight_max": 28.818431854248047,
      "activations/layer2_attention_weight_min": -28.92372703552246,
      "activations/layer3_attention_weight_max": 65.11479949951172,
      "activations/layer3_attention_weight_min": -71.69599914550781,
      "activations/layer4_attention_weight_max": 98.11712646484375,
      "activations/layer4_attention_weight_min": -89.43001556396484,
      "activations/layer5_attention_weight_max": 118.160888671875,
      "activations/layer5_attention_weight_min": -99.12133026123047,
      "activations/layer6_attention_weight_max": 62.445926666259766,
      "activations/layer6_attention_weight_min": -50.38130187988281,
      "activations/layer7_attention_weight_max": 55.97601318359375,
      "activations/layer7_attention_weight_min": -52.523162841796875,
      "activations/layer8_attention_weight_max": 40.00922393798828,
      "activations/layer8_attention_weight_min": -33.917510986328125,
      "activations/layer9_attention_weight_max": 56.88433837890625,
      "activations/layer9_attention_weight_min": -47.086238861083984,
      "epoch": 2.68,
      "learning_rate": 0.000134075,
      "loss": 2.9282,
      "step": 46100
    },
    {
      "activations/layer0_attention_weight_max": 15.304221153259277,
      "activations/layer0_attention_weight_min": -14.7800931930542,
      "activations/layer10_attention_weight_max": 52.285823822021484,
      "activations/layer10_attention_weight_min": -40.052520751953125,
      "activations/layer11_attention_weight_max": 42.47112274169922,
      "activations/layer11_attention_weight_min": -29.26814079284668,
      "activations/layer12_attention_weight_max": 31.043251037597656,
      "activations/layer12_attention_weight_min": -23.39422035217285,
      "activations/layer13_attention_weight_max": 51.67390823364258,
      "activations/layer13_attention_weight_min": -40.908241271972656,
      "activations/layer14_attention_weight_max": 46.97307205200195,
      "activations/layer14_attention_weight_min": -37.22386169433594,
      "activations/layer15_attention_weight_max": 51.494224548339844,
      "activations/layer15_attention_weight_min": -32.96122360229492,
      "activations/layer16_attention_weight_max": 37.573238372802734,
      "activations/layer16_attention_weight_min": -27.131364822387695,
      "activations/layer17_attention_weight_max": 61.07644271850586,
      "activations/layer17_attention_weight_min": -38.42481231689453,
      "activations/layer18_attention_weight_max": 61.13896942138672,
      "activations/layer18_attention_weight_min": -33.84901428222656,
      "activations/layer19_attention_weight_max": 21.33296775817871,
      "activations/layer19_attention_weight_min": -19.49672508239746,
      "activations/layer1_attention_weight_max": 17.380971908569336,
      "activations/layer1_attention_weight_min": -13.880149841308594,
      "activations/layer20_attention_weight_max": 20.944700241088867,
      "activations/layer20_attention_weight_min": -17.165267944335938,
      "activations/layer21_attention_weight_max": 44.151710510253906,
      "activations/layer21_attention_weight_min": -31.709415435791016,
      "activations/layer22_attention_weight_max": 33.40446472167969,
      "activations/layer22_attention_weight_min": -22.61797523498535,
      "activations/layer23_attention_weight_max": 24.774503707885742,
      "activations/layer23_attention_weight_min": -16.285470962524414,
      "activations/layer2_attention_weight_max": 28.620893478393555,
      "activations/layer2_attention_weight_min": -26.8450984954834,
      "activations/layer3_attention_weight_max": 69.27752685546875,
      "activations/layer3_attention_weight_min": -70.61739349365234,
      "activations/layer4_attention_weight_max": 100.55340576171875,
      "activations/layer4_attention_weight_min": -86.33171844482422,
      "activations/layer5_attention_weight_max": 124.36430358886719,
      "activations/layer5_attention_weight_min": -96.69951629638672,
      "activations/layer6_attention_weight_max": 61.71290969848633,
      "activations/layer6_attention_weight_min": -50.90550231933594,
      "activations/layer7_attention_weight_max": 59.84857940673828,
      "activations/layer7_attention_weight_min": -53.63511657714844,
      "activations/layer8_attention_weight_max": 40.339599609375,
      "activations/layer8_attention_weight_min": -32.399444580078125,
      "activations/layer9_attention_weight_max": 53.936073303222656,
      "activations/layer9_attention_weight_min": -46.980072021484375,
      "epoch": 2.68,
      "learning_rate": 0.00013405643939393938,
      "loss": 2.914,
      "step": 46150
    },
    {
      "activations/layer0_attention_weight_max": 14.752829551696777,
      "activations/layer0_attention_weight_min": -15.138794898986816,
      "activations/layer10_attention_weight_max": 52.69913101196289,
      "activations/layer10_attention_weight_min": -40.112998962402344,
      "activations/layer11_attention_weight_max": 41.70283889770508,
      "activations/layer11_attention_weight_min": -28.78238296508789,
      "activations/layer12_attention_weight_max": 31.97997283935547,
      "activations/layer12_attention_weight_min": -24.553913116455078,
      "activations/layer13_attention_weight_max": 60.92505645751953,
      "activations/layer13_attention_weight_min": -41.261043548583984,
      "activations/layer14_attention_weight_max": 46.94692611694336,
      "activations/layer14_attention_weight_min": -30.243715286254883,
      "activations/layer15_attention_weight_max": 51.97665023803711,
      "activations/layer15_attention_weight_min": -36.78282928466797,
      "activations/layer16_attention_weight_max": 43.05419158935547,
      "activations/layer16_attention_weight_min": -26.89069175720215,
      "activations/layer17_attention_weight_max": 63.52771759033203,
      "activations/layer17_attention_weight_min": -36.5690803527832,
      "activations/layer18_attention_weight_max": 57.995418548583984,
      "activations/layer18_attention_weight_min": -30.293025970458984,
      "activations/layer19_attention_weight_max": 27.9648380279541,
      "activations/layer19_attention_weight_min": -16.335617065429688,
      "activations/layer1_attention_weight_max": 16.769136428833008,
      "activations/layer1_attention_weight_min": -13.971501350402832,
      "activations/layer20_attention_weight_max": 27.57512092590332,
      "activations/layer20_attention_weight_min": -20.34331512451172,
      "activations/layer21_attention_weight_max": 47.43429946899414,
      "activations/layer21_attention_weight_min": -29.036251068115234,
      "activations/layer22_attention_weight_max": 38.92496871948242,
      "activations/layer22_attention_weight_min": -22.55954360961914,
      "activations/layer23_attention_weight_max": 27.2491455078125,
      "activations/layer23_attention_weight_min": -15.734895706176758,
      "activations/layer2_attention_weight_max": 27.391096115112305,
      "activations/layer2_attention_weight_min": -26.755874633789062,
      "activations/layer3_attention_weight_max": 66.00682067871094,
      "activations/layer3_attention_weight_min": -67.62147521972656,
      "activations/layer4_attention_weight_max": 93.28610229492188,
      "activations/layer4_attention_weight_min": -85.9305648803711,
      "activations/layer5_attention_weight_max": 118.88810729980469,
      "activations/layer5_attention_weight_min": -99.10438537597656,
      "activations/layer6_attention_weight_max": 60.37437438964844,
      "activations/layer6_attention_weight_min": -49.245853424072266,
      "activations/layer7_attention_weight_max": 58.00831604003906,
      "activations/layer7_attention_weight_min": -52.906925201416016,
      "activations/layer8_attention_weight_max": 43.11577224731445,
      "activations/layer8_attention_weight_min": -33.47526931762695,
      "activations/layer9_attention_weight_max": 60.7808952331543,
      "activations/layer9_attention_weight_min": -51.451087951660156,
      "epoch": 2.68,
      "learning_rate": 0.00013403749999999997,
      "loss": 2.9172,
      "step": 46200
    },
    {
      "activations/layer0_attention_weight_max": 15.973700523376465,
      "activations/layer0_attention_weight_min": -14.548114776611328,
      "activations/layer10_attention_weight_max": 50.332515716552734,
      "activations/layer10_attention_weight_min": -41.7148551940918,
      "activations/layer11_attention_weight_max": 41.4072265625,
      "activations/layer11_attention_weight_min": -33.04393768310547,
      "activations/layer12_attention_weight_max": 31.464054107666016,
      "activations/layer12_attention_weight_min": -24.296300888061523,
      "activations/layer13_attention_weight_max": 61.79180908203125,
      "activations/layer13_attention_weight_min": -44.23848342895508,
      "activations/layer14_attention_weight_max": 58.125885009765625,
      "activations/layer14_attention_weight_min": -38.44143295288086,
      "activations/layer15_attention_weight_max": 51.962467193603516,
      "activations/layer15_attention_weight_min": -34.290931701660156,
      "activations/layer16_attention_weight_max": 47.15129089355469,
      "activations/layer16_attention_weight_min": -26.966880798339844,
      "activations/layer17_attention_weight_max": 61.56233596801758,
      "activations/layer17_attention_weight_min": -37.2476921081543,
      "activations/layer18_attention_weight_max": 54.670745849609375,
      "activations/layer18_attention_weight_min": -32.63376235961914,
      "activations/layer19_attention_weight_max": 24.810409545898438,
      "activations/layer19_attention_weight_min": -16.631351470947266,
      "activations/layer1_attention_weight_max": 17.46443748474121,
      "activations/layer1_attention_weight_min": -14.027944564819336,
      "activations/layer20_attention_weight_max": 17.665109634399414,
      "activations/layer20_attention_weight_min": -18.973474502563477,
      "activations/layer21_attention_weight_max": 42.66668701171875,
      "activations/layer21_attention_weight_min": -27.215824127197266,
      "activations/layer22_attention_weight_max": 39.16830825805664,
      "activations/layer22_attention_weight_min": -21.958066940307617,
      "activations/layer23_attention_weight_max": 22.69855308532715,
      "activations/layer23_attention_weight_min": -14.974799156188965,
      "activations/layer2_attention_weight_max": 27.78643798828125,
      "activations/layer2_attention_weight_min": -26.350997924804688,
      "activations/layer3_attention_weight_max": 66.69569396972656,
      "activations/layer3_attention_weight_min": -73.44677734375,
      "activations/layer4_attention_weight_max": 100.47307586669922,
      "activations/layer4_attention_weight_min": -89.11092376708984,
      "activations/layer5_attention_weight_max": 119.29467010498047,
      "activations/layer5_attention_weight_min": -102.29586029052734,
      "activations/layer6_attention_weight_max": 59.94306182861328,
      "activations/layer6_attention_weight_min": -50.05411148071289,
      "activations/layer7_attention_weight_max": 71.70523834228516,
      "activations/layer7_attention_weight_min": -60.995094299316406,
      "activations/layer8_attention_weight_max": 47.5809326171875,
      "activations/layer8_attention_weight_min": -34.71062088012695,
      "activations/layer9_attention_weight_max": 62.99222946166992,
      "activations/layer9_attention_weight_min": -56.260799407958984,
      "epoch": 2.69,
      "learning_rate": 0.0001340185606060606,
      "loss": 2.9281,
      "step": 46250
    },
    {
      "activations/layer0_attention_weight_max": 13.487462043762207,
      "activations/layer0_attention_weight_min": -14.726916313171387,
      "activations/layer10_attention_weight_max": 44.45246124267578,
      "activations/layer10_attention_weight_min": -37.57947540283203,
      "activations/layer11_attention_weight_max": 39.74247360229492,
      "activations/layer11_attention_weight_min": -28.281814575195312,
      "activations/layer12_attention_weight_max": 26.945022583007812,
      "activations/layer12_attention_weight_min": -23.78514862060547,
      "activations/layer13_attention_weight_max": 51.715389251708984,
      "activations/layer13_attention_weight_min": -41.03710174560547,
      "activations/layer14_attention_weight_max": 48.71268844604492,
      "activations/layer14_attention_weight_min": -33.38071823120117,
      "activations/layer15_attention_weight_max": 52.327980041503906,
      "activations/layer15_attention_weight_min": -34.255577087402344,
      "activations/layer16_attention_weight_max": 35.40338897705078,
      "activations/layer16_attention_weight_min": -28.36981964111328,
      "activations/layer17_attention_weight_max": 56.32478332519531,
      "activations/layer17_attention_weight_min": -36.84341812133789,
      "activations/layer18_attention_weight_max": 52.34203338623047,
      "activations/layer18_attention_weight_min": -32.808006286621094,
      "activations/layer19_attention_weight_max": 20.872758865356445,
      "activations/layer19_attention_weight_min": -18.077733993530273,
      "activations/layer1_attention_weight_max": 17.513761520385742,
      "activations/layer1_attention_weight_min": -14.091511726379395,
      "activations/layer20_attention_weight_max": 22.099105834960938,
      "activations/layer20_attention_weight_min": -17.07175064086914,
      "activations/layer21_attention_weight_max": 36.38055419921875,
      "activations/layer21_attention_weight_min": -26.763763427734375,
      "activations/layer22_attention_weight_max": 35.34070587158203,
      "activations/layer22_attention_weight_min": -25.267072677612305,
      "activations/layer23_attention_weight_max": 21.633941650390625,
      "activations/layer23_attention_weight_min": -16.156360626220703,
      "activations/layer2_attention_weight_max": 27.6488037109375,
      "activations/layer2_attention_weight_min": -25.934852600097656,
      "activations/layer3_attention_weight_max": 66.32127380371094,
      "activations/layer3_attention_weight_min": -69.85614776611328,
      "activations/layer4_attention_weight_max": 97.68534088134766,
      "activations/layer4_attention_weight_min": -86.89065551757812,
      "activations/layer5_attention_weight_max": 115.64688110351562,
      "activations/layer5_attention_weight_min": -98.68922424316406,
      "activations/layer6_attention_weight_max": 60.52830505371094,
      "activations/layer6_attention_weight_min": -51.19157409667969,
      "activations/layer7_attention_weight_max": 54.76735305786133,
      "activations/layer7_attention_weight_min": -53.86949920654297,
      "activations/layer8_attention_weight_max": 36.94828414916992,
      "activations/layer8_attention_weight_min": -33.71085739135742,
      "activations/layer9_attention_weight_max": 52.306396484375,
      "activations/layer9_attention_weight_min": -45.53880310058594,
      "epoch": 2.69,
      "learning_rate": 0.0001339996212121212,
      "loss": 2.9375,
      "step": 46300
    },
    {
      "activations/layer0_attention_weight_max": 14.456661224365234,
      "activations/layer0_attention_weight_min": -14.703375816345215,
      "activations/layer10_attention_weight_max": 47.39961624145508,
      "activations/layer10_attention_weight_min": -37.7165412902832,
      "activations/layer11_attention_weight_max": 38.7625732421875,
      "activations/layer11_attention_weight_min": -30.096296310424805,
      "activations/layer12_attention_weight_max": 31.640621185302734,
      "activations/layer12_attention_weight_min": -25.976764678955078,
      "activations/layer13_attention_weight_max": 59.888694763183594,
      "activations/layer13_attention_weight_min": -42.95450973510742,
      "activations/layer14_attention_weight_max": 55.42894744873047,
      "activations/layer14_attention_weight_min": -38.56046676635742,
      "activations/layer15_attention_weight_max": 52.63552474975586,
      "activations/layer15_attention_weight_min": -32.80593490600586,
      "activations/layer16_attention_weight_max": 46.03056716918945,
      "activations/layer16_attention_weight_min": -27.037187576293945,
      "activations/layer17_attention_weight_max": 60.68224334716797,
      "activations/layer17_attention_weight_min": -37.4633674621582,
      "activations/layer18_attention_weight_max": 62.4603385925293,
      "activations/layer18_attention_weight_min": -31.097759246826172,
      "activations/layer19_attention_weight_max": 25.356218338012695,
      "activations/layer19_attention_weight_min": -18.10787010192871,
      "activations/layer1_attention_weight_max": 17.5150146484375,
      "activations/layer1_attention_weight_min": -13.891396522521973,
      "activations/layer20_attention_weight_max": 20.6954288482666,
      "activations/layer20_attention_weight_min": -19.967313766479492,
      "activations/layer21_attention_weight_max": 44.01195526123047,
      "activations/layer21_attention_weight_min": -26.6096248626709,
      "activations/layer22_attention_weight_max": 38.25630187988281,
      "activations/layer22_attention_weight_min": -21.50453758239746,
      "activations/layer23_attention_weight_max": 25.13392448425293,
      "activations/layer23_attention_weight_min": -13.858024597167969,
      "activations/layer2_attention_weight_max": 28.94648551940918,
      "activations/layer2_attention_weight_min": -27.06397247314453,
      "activations/layer3_attention_weight_max": 67.35758209228516,
      "activations/layer3_attention_weight_min": -71.29247283935547,
      "activations/layer4_attention_weight_max": 99.31718444824219,
      "activations/layer4_attention_weight_min": -89.95136260986328,
      "activations/layer5_attention_weight_max": 123.95143127441406,
      "activations/layer5_attention_weight_min": -103.64311218261719,
      "activations/layer6_attention_weight_max": 59.8745002746582,
      "activations/layer6_attention_weight_min": -51.581199645996094,
      "activations/layer7_attention_weight_max": 56.84666061401367,
      "activations/layer7_attention_weight_min": -56.05867385864258,
      "activations/layer8_attention_weight_max": 41.34700012207031,
      "activations/layer8_attention_weight_min": -35.47356414794922,
      "activations/layer9_attention_weight_max": 62.67194366455078,
      "activations/layer9_attention_weight_min": -48.80671310424805,
      "epoch": 2.69,
      "learning_rate": 0.0001339806818181818,
      "loss": 2.912,
      "step": 46350
    },
    {
      "activations/layer0_attention_weight_max": 14.16138744354248,
      "activations/layer0_attention_weight_min": -14.961627960205078,
      "activations/layer10_attention_weight_max": 50.63132095336914,
      "activations/layer10_attention_weight_min": -41.73713302612305,
      "activations/layer11_attention_weight_max": 41.276695251464844,
      "activations/layer11_attention_weight_min": -28.19029998779297,
      "activations/layer12_attention_weight_max": 28.044233322143555,
      "activations/layer12_attention_weight_min": -23.327510833740234,
      "activations/layer13_attention_weight_max": 53.025779724121094,
      "activations/layer13_attention_weight_min": -37.270973205566406,
      "activations/layer14_attention_weight_max": 46.36420822143555,
      "activations/layer14_attention_weight_min": -33.11676025390625,
      "activations/layer15_attention_weight_max": 47.0153694152832,
      "activations/layer15_attention_weight_min": -34.48779296875,
      "activations/layer16_attention_weight_max": 37.5263671875,
      "activations/layer16_attention_weight_min": -28.82427215576172,
      "activations/layer17_attention_weight_max": 67.10209655761719,
      "activations/layer17_attention_weight_min": -38.83397674560547,
      "activations/layer18_attention_weight_max": 55.872520446777344,
      "activations/layer18_attention_weight_min": -30.588356018066406,
      "activations/layer19_attention_weight_max": 21.9931583404541,
      "activations/layer19_attention_weight_min": -17.91362762451172,
      "activations/layer1_attention_weight_max": 17.63626480102539,
      "activations/layer1_attention_weight_min": -15.754053115844727,
      "activations/layer20_attention_weight_max": 19.731372833251953,
      "activations/layer20_attention_weight_min": -18.72173309326172,
      "activations/layer21_attention_weight_max": 44.09686279296875,
      "activations/layer21_attention_weight_min": -24.826536178588867,
      "activations/layer22_attention_weight_max": 34.66827392578125,
      "activations/layer22_attention_weight_min": -24.28141975402832,
      "activations/layer23_attention_weight_max": 24.3038387298584,
      "activations/layer23_attention_weight_min": -15.919341087341309,
      "activations/layer2_attention_weight_max": 29.73970603942871,
      "activations/layer2_attention_weight_min": -26.840129852294922,
      "activations/layer3_attention_weight_max": 66.542724609375,
      "activations/layer3_attention_weight_min": -71.00291442871094,
      "activations/layer4_attention_weight_max": 98.4400863647461,
      "activations/layer4_attention_weight_min": -90.22095489501953,
      "activations/layer5_attention_weight_max": 125.21861267089844,
      "activations/layer5_attention_weight_min": -102.34954833984375,
      "activations/layer6_attention_weight_max": 63.821983337402344,
      "activations/layer6_attention_weight_min": -50.30415344238281,
      "activations/layer7_attention_weight_max": 58.431514739990234,
      "activations/layer7_attention_weight_min": -50.13407897949219,
      "activations/layer8_attention_weight_max": 40.899803161621094,
      "activations/layer8_attention_weight_min": -33.810733795166016,
      "activations/layer9_attention_weight_max": 57.45425033569336,
      "activations/layer9_attention_weight_min": -50.71249008178711,
      "epoch": 2.7,
      "learning_rate": 0.0001339617424242424,
      "loss": 2.914,
      "step": 46400
    },
    {
      "activations/layer0_attention_weight_max": 14.750505447387695,
      "activations/layer0_attention_weight_min": -15.39695930480957,
      "activations/layer10_attention_weight_max": 48.72186279296875,
      "activations/layer10_attention_weight_min": -38.23388671875,
      "activations/layer11_attention_weight_max": 46.586036682128906,
      "activations/layer11_attention_weight_min": -33.77251434326172,
      "activations/layer12_attention_weight_max": 30.792400360107422,
      "activations/layer12_attention_weight_min": -25.139116287231445,
      "activations/layer13_attention_weight_max": 58.91953659057617,
      "activations/layer13_attention_weight_min": -42.66572952270508,
      "activations/layer14_attention_weight_max": 53.0877799987793,
      "activations/layer14_attention_weight_min": -35.769195556640625,
      "activations/layer15_attention_weight_max": 47.315223693847656,
      "activations/layer15_attention_weight_min": -34.27645492553711,
      "activations/layer16_attention_weight_max": 38.93659973144531,
      "activations/layer16_attention_weight_min": -27.322477340698242,
      "activations/layer17_attention_weight_max": 57.11484909057617,
      "activations/layer17_attention_weight_min": -36.23178482055664,
      "activations/layer18_attention_weight_max": 54.54124069213867,
      "activations/layer18_attention_weight_min": -33.089317321777344,
      "activations/layer19_attention_weight_max": 22.794919967651367,
      "activations/layer19_attention_weight_min": -16.52216148376465,
      "activations/layer1_attention_weight_max": 19.767305374145508,
      "activations/layer1_attention_weight_min": -15.101776123046875,
      "activations/layer20_attention_weight_max": 25.791349411010742,
      "activations/layer20_attention_weight_min": -19.207151412963867,
      "activations/layer21_attention_weight_max": 41.55159378051758,
      "activations/layer21_attention_weight_min": -27.084739685058594,
      "activations/layer22_attention_weight_max": 34.89714050292969,
      "activations/layer22_attention_weight_min": -22.274639129638672,
      "activations/layer23_attention_weight_max": 27.781082153320312,
      "activations/layer23_attention_weight_min": -16.121999740600586,
      "activations/layer2_attention_weight_max": 29.857532501220703,
      "activations/layer2_attention_weight_min": -26.646053314208984,
      "activations/layer3_attention_weight_max": 67.26055145263672,
      "activations/layer3_attention_weight_min": -70.25165557861328,
      "activations/layer4_attention_weight_max": 93.99163055419922,
      "activations/layer4_attention_weight_min": -86.09456634521484,
      "activations/layer5_attention_weight_max": 124.33760833740234,
      "activations/layer5_attention_weight_min": -104.87973022460938,
      "activations/layer6_attention_weight_max": 59.90164566040039,
      "activations/layer6_attention_weight_min": -50.778480529785156,
      "activations/layer7_attention_weight_max": 66.8209457397461,
      "activations/layer7_attention_weight_min": -67.64794158935547,
      "activations/layer8_attention_weight_max": 47.451114654541016,
      "activations/layer8_attention_weight_min": -38.50851058959961,
      "activations/layer9_attention_weight_max": 61.242671966552734,
      "activations/layer9_attention_weight_min": -57.725582122802734,
      "epoch": 2.7,
      "learning_rate": 0.00013394280303030302,
      "loss": 2.9236,
      "step": 46450
    },
    {
      "activations/layer0_attention_weight_max": 13.840398788452148,
      "activations/layer0_attention_weight_min": -15.343486785888672,
      "activations/layer10_attention_weight_max": 46.48868179321289,
      "activations/layer10_attention_weight_min": -37.64127731323242,
      "activations/layer11_attention_weight_max": 33.79009246826172,
      "activations/layer11_attention_weight_min": -27.847700119018555,
      "activations/layer12_attention_weight_max": 25.46512794494629,
      "activations/layer12_attention_weight_min": -24.595428466796875,
      "activations/layer13_attention_weight_max": 58.45622634887695,
      "activations/layer13_attention_weight_min": -39.196590423583984,
      "activations/layer14_attention_weight_max": 44.57277297973633,
      "activations/layer14_attention_weight_min": -34.12601852416992,
      "activations/layer15_attention_weight_max": 48.375526428222656,
      "activations/layer15_attention_weight_min": -35.29195785522461,
      "activations/layer16_attention_weight_max": 39.68904495239258,
      "activations/layer16_attention_weight_min": -27.015483856201172,
      "activations/layer17_attention_weight_max": 62.09120178222656,
      "activations/layer17_attention_weight_min": -38.56536865234375,
      "activations/layer18_attention_weight_max": 51.05118179321289,
      "activations/layer18_attention_weight_min": -31.315265655517578,
      "activations/layer19_attention_weight_max": 23.868913650512695,
      "activations/layer19_attention_weight_min": -18.48719596862793,
      "activations/layer1_attention_weight_max": 18.077587127685547,
      "activations/layer1_attention_weight_min": -14.733062744140625,
      "activations/layer20_attention_weight_max": 21.787294387817383,
      "activations/layer20_attention_weight_min": -18.60222053527832,
      "activations/layer21_attention_weight_max": 42.67752456665039,
      "activations/layer21_attention_weight_min": -29.283109664916992,
      "activations/layer22_attention_weight_max": 34.15523910522461,
      "activations/layer22_attention_weight_min": -21.729694366455078,
      "activations/layer23_attention_weight_max": 22.73495101928711,
      "activations/layer23_attention_weight_min": -17.67747688293457,
      "activations/layer2_attention_weight_max": 30.013418197631836,
      "activations/layer2_attention_weight_min": -26.775663375854492,
      "activations/layer3_attention_weight_max": 70.31197357177734,
      "activations/layer3_attention_weight_min": -75.77394104003906,
      "activations/layer4_attention_weight_max": 99.07586669921875,
      "activations/layer4_attention_weight_min": -91.83148193359375,
      "activations/layer5_attention_weight_max": 128.12583923339844,
      "activations/layer5_attention_weight_min": -108.2944564819336,
      "activations/layer6_attention_weight_max": 65.63530731201172,
      "activations/layer6_attention_weight_min": -55.6162109375,
      "activations/layer7_attention_weight_max": 58.248043060302734,
      "activations/layer7_attention_weight_min": -51.50590515136719,
      "activations/layer8_attention_weight_max": 39.96187210083008,
      "activations/layer8_attention_weight_min": -35.6668701171875,
      "activations/layer9_attention_weight_max": 53.613033294677734,
      "activations/layer9_attention_weight_min": -47.18805694580078,
      "epoch": 2.7,
      "learning_rate": 0.0001339238636363636,
      "loss": 2.9202,
      "step": 46500
    },
    {
      "activations/layer0_attention_weight_max": 14.642603874206543,
      "activations/layer0_attention_weight_min": -15.661534309387207,
      "activations/layer10_attention_weight_max": 49.51657485961914,
      "activations/layer10_attention_weight_min": -40.15027618408203,
      "activations/layer11_attention_weight_max": 41.05561828613281,
      "activations/layer11_attention_weight_min": -31.02056884765625,
      "activations/layer12_attention_weight_max": 29.793678283691406,
      "activations/layer12_attention_weight_min": -27.620237350463867,
      "activations/layer13_attention_weight_max": 54.05750274658203,
      "activations/layer13_attention_weight_min": -39.54364776611328,
      "activations/layer14_attention_weight_max": 50.17964172363281,
      "activations/layer14_attention_weight_min": -35.329586029052734,
      "activations/layer15_attention_weight_max": 48.72618103027344,
      "activations/layer15_attention_weight_min": -33.52241134643555,
      "activations/layer16_attention_weight_max": 41.098785400390625,
      "activations/layer16_attention_weight_min": -27.215667724609375,
      "activations/layer17_attention_weight_max": 63.57080841064453,
      "activations/layer17_attention_weight_min": -35.375999450683594,
      "activations/layer18_attention_weight_max": 63.23664474487305,
      "activations/layer18_attention_weight_min": -35.04014587402344,
      "activations/layer19_attention_weight_max": 23.43021011352539,
      "activations/layer19_attention_weight_min": -19.74373435974121,
      "activations/layer1_attention_weight_max": 17.69040870666504,
      "activations/layer1_attention_weight_min": -13.65051555633545,
      "activations/layer20_attention_weight_max": 19.8817081451416,
      "activations/layer20_attention_weight_min": -16.940893173217773,
      "activations/layer21_attention_weight_max": 49.26615905761719,
      "activations/layer21_attention_weight_min": -30.117380142211914,
      "activations/layer22_attention_weight_max": 38.383583068847656,
      "activations/layer22_attention_weight_min": -21.248050689697266,
      "activations/layer23_attention_weight_max": 24.078609466552734,
      "activations/layer23_attention_weight_min": -16.803682327270508,
      "activations/layer2_attention_weight_max": 28.926883697509766,
      "activations/layer2_attention_weight_min": -26.619665145874023,
      "activations/layer3_attention_weight_max": 67.36829376220703,
      "activations/layer3_attention_weight_min": -74.7594985961914,
      "activations/layer4_attention_weight_max": 102.185546875,
      "activations/layer4_attention_weight_min": -93.34315490722656,
      "activations/layer5_attention_weight_max": 131.2950897216797,
      "activations/layer5_attention_weight_min": -99.61455535888672,
      "activations/layer6_attention_weight_max": 61.010353088378906,
      "activations/layer6_attention_weight_min": -49.17448043823242,
      "activations/layer7_attention_weight_max": 58.4571418762207,
      "activations/layer7_attention_weight_min": -52.44664001464844,
      "activations/layer8_attention_weight_max": 38.27815628051758,
      "activations/layer8_attention_weight_min": -35.405799865722656,
      "activations/layer9_attention_weight_max": 55.44863510131836,
      "activations/layer9_attention_weight_min": -49.02849578857422,
      "epoch": 2.7,
      "learning_rate": 0.00013390492424242422,
      "loss": 2.932,
      "step": 46550
    },
    {
      "activations/layer0_attention_weight_max": 13.600900650024414,
      "activations/layer0_attention_weight_min": -14.608607292175293,
      "activations/layer10_attention_weight_max": 43.37812423706055,
      "activations/layer10_attention_weight_min": -34.689971923828125,
      "activations/layer11_attention_weight_max": 38.58686065673828,
      "activations/layer11_attention_weight_min": -27.856178283691406,
      "activations/layer12_attention_weight_max": 28.202211380004883,
      "activations/layer12_attention_weight_min": -21.688858032226562,
      "activations/layer13_attention_weight_max": 65.41868591308594,
      "activations/layer13_attention_weight_min": -52.26555252075195,
      "activations/layer14_attention_weight_max": 51.83655548095703,
      "activations/layer14_attention_weight_min": -36.46810531616211,
      "activations/layer15_attention_weight_max": 50.7084846496582,
      "activations/layer15_attention_weight_min": -38.12171936035156,
      "activations/layer16_attention_weight_max": 37.59757995605469,
      "activations/layer16_attention_weight_min": -25.70058250427246,
      "activations/layer17_attention_weight_max": 59.07692337036133,
      "activations/layer17_attention_weight_min": -39.060298919677734,
      "activations/layer18_attention_weight_max": 51.67178726196289,
      "activations/layer18_attention_weight_min": -31.209575653076172,
      "activations/layer19_attention_weight_max": 21.775983810424805,
      "activations/layer19_attention_weight_min": -17.45412254333496,
      "activations/layer1_attention_weight_max": 17.371309280395508,
      "activations/layer1_attention_weight_min": -14.243401527404785,
      "activations/layer20_attention_weight_max": 25.100553512573242,
      "activations/layer20_attention_weight_min": -17.043466567993164,
      "activations/layer21_attention_weight_max": 43.11450958251953,
      "activations/layer21_attention_weight_min": -26.72465705871582,
      "activations/layer22_attention_weight_max": 38.35310745239258,
      "activations/layer22_attention_weight_min": -22.641904830932617,
      "activations/layer23_attention_weight_max": 26.929349899291992,
      "activations/layer23_attention_weight_min": -16.204328536987305,
      "activations/layer2_attention_weight_max": 26.76350212097168,
      "activations/layer2_attention_weight_min": -26.56220054626465,
      "activations/layer3_attention_weight_max": 65.86507415771484,
      "activations/layer3_attention_weight_min": -66.71629333496094,
      "activations/layer4_attention_weight_max": 93.42555236816406,
      "activations/layer4_attention_weight_min": -87.26070404052734,
      "activations/layer5_attention_weight_max": 118.97732543945312,
      "activations/layer5_attention_weight_min": -99.75325012207031,
      "activations/layer6_attention_weight_max": 57.56101608276367,
      "activations/layer6_attention_weight_min": -50.139869689941406,
      "activations/layer7_attention_weight_max": 56.29907989501953,
      "activations/layer7_attention_weight_min": -53.02652359008789,
      "activations/layer8_attention_weight_max": 38.46230697631836,
      "activations/layer8_attention_weight_min": -34.23319625854492,
      "activations/layer9_attention_weight_max": 52.94482421875,
      "activations/layer9_attention_weight_min": -49.81637191772461,
      "epoch": 2.71,
      "learning_rate": 0.00013388598484848484,
      "loss": 2.9267,
      "step": 46600
    },
    {
      "activations/layer0_attention_weight_max": 14.767817497253418,
      "activations/layer0_attention_weight_min": -14.843609809875488,
      "activations/layer10_attention_weight_max": 48.16496658325195,
      "activations/layer10_attention_weight_min": -36.584205627441406,
      "activations/layer11_attention_weight_max": 38.80711364746094,
      "activations/layer11_attention_weight_min": -29.75074577331543,
      "activations/layer12_attention_weight_max": 25.04391098022461,
      "activations/layer12_attention_weight_min": -23.67246437072754,
      "activations/layer13_attention_weight_max": 59.14235305786133,
      "activations/layer13_attention_weight_min": -44.755409240722656,
      "activations/layer14_attention_weight_max": 57.49919891357422,
      "activations/layer14_attention_weight_min": -37.41693878173828,
      "activations/layer15_attention_weight_max": 52.42875289916992,
      "activations/layer15_attention_weight_min": -38.16747283935547,
      "activations/layer16_attention_weight_max": 41.7612419128418,
      "activations/layer16_attention_weight_min": -28.513643264770508,
      "activations/layer17_attention_weight_max": 61.25989532470703,
      "activations/layer17_attention_weight_min": -33.92107391357422,
      "activations/layer18_attention_weight_max": 60.865081787109375,
      "activations/layer18_attention_weight_min": -32.11799240112305,
      "activations/layer19_attention_weight_max": 25.31300926208496,
      "activations/layer19_attention_weight_min": -17.974842071533203,
      "activations/layer1_attention_weight_max": 16.991992950439453,
      "activations/layer1_attention_weight_min": -14.756086349487305,
      "activations/layer20_attention_weight_max": 20.14838409423828,
      "activations/layer20_attention_weight_min": -18.8198299407959,
      "activations/layer21_attention_weight_max": 43.09497833251953,
      "activations/layer21_attention_weight_min": -28.258968353271484,
      "activations/layer22_attention_weight_max": 35.327552795410156,
      "activations/layer22_attention_weight_min": -22.623397827148438,
      "activations/layer23_attention_weight_max": 27.641754150390625,
      "activations/layer23_attention_weight_min": -17.268375396728516,
      "activations/layer2_attention_weight_max": 29.06539535522461,
      "activations/layer2_attention_weight_min": -29.610260009765625,
      "activations/layer3_attention_weight_max": 64.758544921875,
      "activations/layer3_attention_weight_min": -70.66377258300781,
      "activations/layer4_attention_weight_max": 96.23290252685547,
      "activations/layer4_attention_weight_min": -92.58309936523438,
      "activations/layer5_attention_weight_max": 124.18914031982422,
      "activations/layer5_attention_weight_min": -104.96334075927734,
      "activations/layer6_attention_weight_max": 62.579071044921875,
      "activations/layer6_attention_weight_min": -52.08237075805664,
      "activations/layer7_attention_weight_max": 56.98157501220703,
      "activations/layer7_attention_weight_min": -52.31498718261719,
      "activations/layer8_attention_weight_max": 40.9053840637207,
      "activations/layer8_attention_weight_min": -34.65004348754883,
      "activations/layer9_attention_weight_max": 62.04734420776367,
      "activations/layer9_attention_weight_min": -51.306434631347656,
      "epoch": 2.71,
      "learning_rate": 0.00013386704545454545,
      "loss": 2.9336,
      "step": 46650
    },
    {
      "activations/layer0_attention_weight_max": 15.837170600891113,
      "activations/layer0_attention_weight_min": -14.875128746032715,
      "activations/layer10_attention_weight_max": 55.12649917602539,
      "activations/layer10_attention_weight_min": -40.508056640625,
      "activations/layer11_attention_weight_max": 41.27275085449219,
      "activations/layer11_attention_weight_min": -29.18722915649414,
      "activations/layer12_attention_weight_max": 32.42957305908203,
      "activations/layer12_attention_weight_min": -24.471208572387695,
      "activations/layer13_attention_weight_max": 57.940879821777344,
      "activations/layer13_attention_weight_min": -38.882843017578125,
      "activations/layer14_attention_weight_max": 56.82056427001953,
      "activations/layer14_attention_weight_min": -33.70738983154297,
      "activations/layer15_attention_weight_max": 49.64901351928711,
      "activations/layer15_attention_weight_min": -35.68824768066406,
      "activations/layer16_attention_weight_max": 43.33014678955078,
      "activations/layer16_attention_weight_min": -28.91217041015625,
      "activations/layer17_attention_weight_max": 70.1897964477539,
      "activations/layer17_attention_weight_min": -37.16282653808594,
      "activations/layer18_attention_weight_max": 57.906097412109375,
      "activations/layer18_attention_weight_min": -31.381731033325195,
      "activations/layer19_attention_weight_max": 24.87388801574707,
      "activations/layer19_attention_weight_min": -17.753633499145508,
      "activations/layer1_attention_weight_max": 18.342227935791016,
      "activations/layer1_attention_weight_min": -12.927370071411133,
      "activations/layer20_attention_weight_max": 22.843320846557617,
      "activations/layer20_attention_weight_min": -18.07329559326172,
      "activations/layer21_attention_weight_max": 50.513607025146484,
      "activations/layer21_attention_weight_min": -28.964128494262695,
      "activations/layer22_attention_weight_max": 36.89185333251953,
      "activations/layer22_attention_weight_min": -25.039838790893555,
      "activations/layer23_attention_weight_max": 27.835256576538086,
      "activations/layer23_attention_weight_min": -16.82630157470703,
      "activations/layer2_attention_weight_max": 27.635969161987305,
      "activations/layer2_attention_weight_min": -26.7299861907959,
      "activations/layer3_attention_weight_max": 66.25890350341797,
      "activations/layer3_attention_weight_min": -67.70551300048828,
      "activations/layer4_attention_weight_max": 97.81806182861328,
      "activations/layer4_attention_weight_min": -87.37860107421875,
      "activations/layer5_attention_weight_max": 120.91169738769531,
      "activations/layer5_attention_weight_min": -103.55960083007812,
      "activations/layer6_attention_weight_max": 60.44407653808594,
      "activations/layer6_attention_weight_min": -52.252342224121094,
      "activations/layer7_attention_weight_max": 59.74663543701172,
      "activations/layer7_attention_weight_min": -54.725982666015625,
      "activations/layer8_attention_weight_max": 41.58375930786133,
      "activations/layer8_attention_weight_min": -35.61079788208008,
      "activations/layer9_attention_weight_max": 56.944454193115234,
      "activations/layer9_attention_weight_min": -48.07444381713867,
      "epoch": 2.71,
      "learning_rate": 0.00013384810606060604,
      "loss": 2.9381,
      "step": 46700
    },
    {
      "activations/layer0_attention_weight_max": 14.983620643615723,
      "activations/layer0_attention_weight_min": -15.11803913116455,
      "activations/layer10_attention_weight_max": 51.22481155395508,
      "activations/layer10_attention_weight_min": -38.031593322753906,
      "activations/layer11_attention_weight_max": 41.49053192138672,
      "activations/layer11_attention_weight_min": -28.812788009643555,
      "activations/layer12_attention_weight_max": 31.465105056762695,
      "activations/layer12_attention_weight_min": -26.27841567993164,
      "activations/layer13_attention_weight_max": 58.856727600097656,
      "activations/layer13_attention_weight_min": -40.523223876953125,
      "activations/layer14_attention_weight_max": 58.69203567504883,
      "activations/layer14_attention_weight_min": -36.889644622802734,
      "activations/layer15_attention_weight_max": 51.737911224365234,
      "activations/layer15_attention_weight_min": -32.56619644165039,
      "activations/layer16_attention_weight_max": 39.80063247680664,
      "activations/layer16_attention_weight_min": -27.1814022064209,
      "activations/layer17_attention_weight_max": 69.65345001220703,
      "activations/layer17_attention_weight_min": -38.90653991699219,
      "activations/layer18_attention_weight_max": 59.61304473876953,
      "activations/layer18_attention_weight_min": -31.034170150756836,
      "activations/layer19_attention_weight_max": 20.4501953125,
      "activations/layer19_attention_weight_min": -17.3476619720459,
      "activations/layer1_attention_weight_max": 17.736724853515625,
      "activations/layer1_attention_weight_min": -15.160781860351562,
      "activations/layer20_attention_weight_max": 19.49778175354004,
      "activations/layer20_attention_weight_min": -18.041263580322266,
      "activations/layer21_attention_weight_max": 45.7055778503418,
      "activations/layer21_attention_weight_min": -26.518356323242188,
      "activations/layer22_attention_weight_max": 37.31764221191406,
      "activations/layer22_attention_weight_min": -22.98255157470703,
      "activations/layer23_attention_weight_max": 27.79974937438965,
      "activations/layer23_attention_weight_min": -15.674327850341797,
      "activations/layer2_attention_weight_max": 28.66901206970215,
      "activations/layer2_attention_weight_min": -27.62006950378418,
      "activations/layer3_attention_weight_max": 69.1672134399414,
      "activations/layer3_attention_weight_min": -73.78045654296875,
      "activations/layer4_attention_weight_max": 103.66175842285156,
      "activations/layer4_attention_weight_min": -91.49545288085938,
      "activations/layer5_attention_weight_max": 132.2091064453125,
      "activations/layer5_attention_weight_min": -107.52490234375,
      "activations/layer6_attention_weight_max": 63.82676315307617,
      "activations/layer6_attention_weight_min": -52.44225311279297,
      "activations/layer7_attention_weight_max": 67.03449249267578,
      "activations/layer7_attention_weight_min": -59.61646270751953,
      "activations/layer8_attention_weight_max": 43.5858039855957,
      "activations/layer8_attention_weight_min": -36.38050079345703,
      "activations/layer9_attention_weight_max": 56.52942657470703,
      "activations/layer9_attention_weight_min": -50.219486236572266,
      "epoch": 2.72,
      "learning_rate": 0.00013382916666666666,
      "loss": 2.9366,
      "step": 46750
    },
    {
      "activations/layer0_attention_weight_max": 15.190443992614746,
      "activations/layer0_attention_weight_min": -15.15198040008545,
      "activations/layer10_attention_weight_max": 53.37489700317383,
      "activations/layer10_attention_weight_min": -38.84564208984375,
      "activations/layer11_attention_weight_max": 41.575462341308594,
      "activations/layer11_attention_weight_min": -30.026844024658203,
      "activations/layer12_attention_weight_max": 37.41263961791992,
      "activations/layer12_attention_weight_min": -23.99262237548828,
      "activations/layer13_attention_weight_max": 66.93293762207031,
      "activations/layer13_attention_weight_min": -41.70730209350586,
      "activations/layer14_attention_weight_max": 56.87760543823242,
      "activations/layer14_attention_weight_min": -35.29475784301758,
      "activations/layer15_attention_weight_max": 61.886329650878906,
      "activations/layer15_attention_weight_min": -36.31596374511719,
      "activations/layer16_attention_weight_max": 39.81059646606445,
      "activations/layer16_attention_weight_min": -29.696680068969727,
      "activations/layer17_attention_weight_max": 72.19905090332031,
      "activations/layer17_attention_weight_min": -36.960208892822266,
      "activations/layer18_attention_weight_max": 55.4727668762207,
      "activations/layer18_attention_weight_min": -33.29915237426758,
      "activations/layer19_attention_weight_max": 27.187667846679688,
      "activations/layer19_attention_weight_min": -18.135480880737305,
      "activations/layer1_attention_weight_max": 17.38688087463379,
      "activations/layer1_attention_weight_min": -12.684687614440918,
      "activations/layer20_attention_weight_max": 19.292875289916992,
      "activations/layer20_attention_weight_min": -20.728605270385742,
      "activations/layer21_attention_weight_max": 42.24260330200195,
      "activations/layer21_attention_weight_min": -29.876811981201172,
      "activations/layer22_attention_weight_max": 34.04257583618164,
      "activations/layer22_attention_weight_min": -23.3306884765625,
      "activations/layer23_attention_weight_max": 27.582035064697266,
      "activations/layer23_attention_weight_min": -16.191394805908203,
      "activations/layer2_attention_weight_max": 27.599842071533203,
      "activations/layer2_attention_weight_min": -26.743179321289062,
      "activations/layer3_attention_weight_max": 65.50078582763672,
      "activations/layer3_attention_weight_min": -71.5062484741211,
      "activations/layer4_attention_weight_max": 97.95695495605469,
      "activations/layer4_attention_weight_min": -90.73524475097656,
      "activations/layer5_attention_weight_max": 126.87077331542969,
      "activations/layer5_attention_weight_min": -105.6021728515625,
      "activations/layer6_attention_weight_max": 64.3421859741211,
      "activations/layer6_attention_weight_min": -54.32228469848633,
      "activations/layer7_attention_weight_max": 64.15090942382812,
      "activations/layer7_attention_weight_min": -55.47248840332031,
      "activations/layer8_attention_weight_max": 44.043052673339844,
      "activations/layer8_attention_weight_min": -37.50828170776367,
      "activations/layer9_attention_weight_max": 59.76179122924805,
      "activations/layer9_attention_weight_min": -52.9718132019043,
      "epoch": 2.72,
      "learning_rate": 0.00013381022727272727,
      "loss": 2.9228,
      "step": 46800
    },
    {
      "activations/layer0_attention_weight_max": 14.083954811096191,
      "activations/layer0_attention_weight_min": -15.508316040039062,
      "activations/layer10_attention_weight_max": 50.20471954345703,
      "activations/layer10_attention_weight_min": -38.95740509033203,
      "activations/layer11_attention_weight_max": 36.73741912841797,
      "activations/layer11_attention_weight_min": -29.218280792236328,
      "activations/layer12_attention_weight_max": 28.79317855834961,
      "activations/layer12_attention_weight_min": -27.65109634399414,
      "activations/layer13_attention_weight_max": 61.383644104003906,
      "activations/layer13_attention_weight_min": -42.326904296875,
      "activations/layer14_attention_weight_max": 64.23575592041016,
      "activations/layer14_attention_weight_min": -40.563602447509766,
      "activations/layer15_attention_weight_max": 50.113407135009766,
      "activations/layer15_attention_weight_min": -34.26453399658203,
      "activations/layer16_attention_weight_max": 37.31319808959961,
      "activations/layer16_attention_weight_min": -29.575159072875977,
      "activations/layer17_attention_weight_max": 62.74843215942383,
      "activations/layer17_attention_weight_min": -37.59475326538086,
      "activations/layer18_attention_weight_max": 55.54582977294922,
      "activations/layer18_attention_weight_min": -30.8192081451416,
      "activations/layer19_attention_weight_max": 23.5758113861084,
      "activations/layer19_attention_weight_min": -18.15321159362793,
      "activations/layer1_attention_weight_max": 19.562835693359375,
      "activations/layer1_attention_weight_min": -14.359622955322266,
      "activations/layer20_attention_weight_max": 20.794355392456055,
      "activations/layer20_attention_weight_min": -18.053165435791016,
      "activations/layer21_attention_weight_max": 44.9764289855957,
      "activations/layer21_attention_weight_min": -25.978044509887695,
      "activations/layer22_attention_weight_max": 36.53395462036133,
      "activations/layer22_attention_weight_min": -25.9476375579834,
      "activations/layer23_attention_weight_max": 24.335073471069336,
      "activations/layer23_attention_weight_min": -14.47146224975586,
      "activations/layer2_attention_weight_max": 28.929012298583984,
      "activations/layer2_attention_weight_min": -26.30040740966797,
      "activations/layer3_attention_weight_max": 69.1390380859375,
      "activations/layer3_attention_weight_min": -71.10868835449219,
      "activations/layer4_attention_weight_max": 97.33902740478516,
      "activations/layer4_attention_weight_min": -89.62496185302734,
      "activations/layer5_attention_weight_max": 119.05523681640625,
      "activations/layer5_attention_weight_min": -101.48869323730469,
      "activations/layer6_attention_weight_max": 59.34599685668945,
      "activations/layer6_attention_weight_min": -49.843589782714844,
      "activations/layer7_attention_weight_max": 58.98003005981445,
      "activations/layer7_attention_weight_min": -55.46766662597656,
      "activations/layer8_attention_weight_max": 39.11997985839844,
      "activations/layer8_attention_weight_min": -35.04262924194336,
      "activations/layer9_attention_weight_max": 59.3220100402832,
      "activations/layer9_attention_weight_min": -52.033748626708984,
      "epoch": 2.72,
      "learning_rate": 0.00013379128787878789,
      "loss": 2.9432,
      "step": 46850
    },
    {
      "activations/layer0_attention_weight_max": 14.189963340759277,
      "activations/layer0_attention_weight_min": -14.884257316589355,
      "activations/layer10_attention_weight_max": 46.33549499511719,
      "activations/layer10_attention_weight_min": -39.24005126953125,
      "activations/layer11_attention_weight_max": 39.09973907470703,
      "activations/layer11_attention_weight_min": -29.610164642333984,
      "activations/layer12_attention_weight_max": 24.24759864807129,
      "activations/layer12_attention_weight_min": -26.56377410888672,
      "activations/layer13_attention_weight_max": 53.89055633544922,
      "activations/layer13_attention_weight_min": -43.46247100830078,
      "activations/layer14_attention_weight_max": 49.34539794921875,
      "activations/layer14_attention_weight_min": -36.866355895996094,
      "activations/layer15_attention_weight_max": 50.336273193359375,
      "activations/layer15_attention_weight_min": -36.5118408203125,
      "activations/layer16_attention_weight_max": 38.19620132446289,
      "activations/layer16_attention_weight_min": -27.003032684326172,
      "activations/layer17_attention_weight_max": 64.06083679199219,
      "activations/layer17_attention_weight_min": -37.154876708984375,
      "activations/layer18_attention_weight_max": 60.612483978271484,
      "activations/layer18_attention_weight_min": -33.40642547607422,
      "activations/layer19_attention_weight_max": 22.26664924621582,
      "activations/layer19_attention_weight_min": -18.096691131591797,
      "activations/layer1_attention_weight_max": 18.311735153198242,
      "activations/layer1_attention_weight_min": -13.679889678955078,
      "activations/layer20_attention_weight_max": 22.100711822509766,
      "activations/layer20_attention_weight_min": -17.68516731262207,
      "activations/layer21_attention_weight_max": 45.02606201171875,
      "activations/layer21_attention_weight_min": -27.744735717773438,
      "activations/layer22_attention_weight_max": 34.30112838745117,
      "activations/layer22_attention_weight_min": -23.4945125579834,
      "activations/layer23_attention_weight_max": 24.82668685913086,
      "activations/layer23_attention_weight_min": -16.026325225830078,
      "activations/layer2_attention_weight_max": 28.06847381591797,
      "activations/layer2_attention_weight_min": -26.955259323120117,
      "activations/layer3_attention_weight_max": 63.475982666015625,
      "activations/layer3_attention_weight_min": -67.86317443847656,
      "activations/layer4_attention_weight_max": 97.16588592529297,
      "activations/layer4_attention_weight_min": -83.50041961669922,
      "activations/layer5_attention_weight_max": 124.99385833740234,
      "activations/layer5_attention_weight_min": -100.89092254638672,
      "activations/layer6_attention_weight_max": 66.0571517944336,
      "activations/layer6_attention_weight_min": -50.38945388793945,
      "activations/layer7_attention_weight_max": 57.492645263671875,
      "activations/layer7_attention_weight_min": -49.36691665649414,
      "activations/layer8_attention_weight_max": 38.62584686279297,
      "activations/layer8_attention_weight_min": -34.989471435546875,
      "activations/layer9_attention_weight_max": 50.63996124267578,
      "activations/layer9_attention_weight_min": -45.91745376586914,
      "epoch": 2.73,
      "learning_rate": 0.00013377234848484847,
      "loss": 2.9282,
      "step": 46900
    },
    {
      "activations/layer0_attention_weight_max": 15.667533874511719,
      "activations/layer0_attention_weight_min": -15.44621753692627,
      "activations/layer10_attention_weight_max": 52.50522994995117,
      "activations/layer10_attention_weight_min": -37.46766662597656,
      "activations/layer11_attention_weight_max": 43.17023849487305,
      "activations/layer11_attention_weight_min": -28.747541427612305,
      "activations/layer12_attention_weight_max": 25.890419006347656,
      "activations/layer12_attention_weight_min": -24.516565322875977,
      "activations/layer13_attention_weight_max": 67.90322875976562,
      "activations/layer13_attention_weight_min": -41.48078155517578,
      "activations/layer14_attention_weight_max": 59.75637435913086,
      "activations/layer14_attention_weight_min": -32.02394485473633,
      "activations/layer15_attention_weight_max": 46.612728118896484,
      "activations/layer15_attention_weight_min": -34.271575927734375,
      "activations/layer16_attention_weight_max": 41.48139953613281,
      "activations/layer16_attention_weight_min": -27.699554443359375,
      "activations/layer17_attention_weight_max": 55.26570510864258,
      "activations/layer17_attention_weight_min": -34.824195861816406,
      "activations/layer18_attention_weight_max": 49.39936828613281,
      "activations/layer18_attention_weight_min": -34.8507194519043,
      "activations/layer19_attention_weight_max": 21.3210506439209,
      "activations/layer19_attention_weight_min": -16.171741485595703,
      "activations/layer1_attention_weight_max": 16.973325729370117,
      "activations/layer1_attention_weight_min": -14.202178955078125,
      "activations/layer20_attention_weight_max": 21.12495994567871,
      "activations/layer20_attention_weight_min": -18.48309898376465,
      "activations/layer21_attention_weight_max": 38.59403991699219,
      "activations/layer21_attention_weight_min": -26.322011947631836,
      "activations/layer22_attention_weight_max": 35.99313735961914,
      "activations/layer22_attention_weight_min": -21.405553817749023,
      "activations/layer23_attention_weight_max": 23.354368209838867,
      "activations/layer23_attention_weight_min": -16.28675651550293,
      "activations/layer2_attention_weight_max": 27.238975524902344,
      "activations/layer2_attention_weight_min": -26.743452072143555,
      "activations/layer3_attention_weight_max": 73.61632537841797,
      "activations/layer3_attention_weight_min": -74.7154541015625,
      "activations/layer4_attention_weight_max": 105.88142395019531,
      "activations/layer4_attention_weight_min": -95.43929290771484,
      "activations/layer5_attention_weight_max": 136.84344482421875,
      "activations/layer5_attention_weight_min": -108.61662292480469,
      "activations/layer6_attention_weight_max": 67.10570526123047,
      "activations/layer6_attention_weight_min": -52.49961471557617,
      "activations/layer7_attention_weight_max": 65.56982421875,
      "activations/layer7_attention_weight_min": -59.54667282104492,
      "activations/layer8_attention_weight_max": 48.02235794067383,
      "activations/layer8_attention_weight_min": -36.69838333129883,
      "activations/layer9_attention_weight_max": 67.63739013671875,
      "activations/layer9_attention_weight_min": -59.66337966918945,
      "epoch": 2.73,
      "learning_rate": 0.0001337534090909091,
      "loss": 2.9373,
      "step": 46950
    },
    {
      "activations/layer0_attention_weight_max": 13.23934555053711,
      "activations/layer0_attention_weight_min": -14.735126495361328,
      "activations/layer10_attention_weight_max": 46.155391693115234,
      "activations/layer10_attention_weight_min": -38.67443084716797,
      "activations/layer11_attention_weight_max": 37.71310043334961,
      "activations/layer11_attention_weight_min": -31.192916870117188,
      "activations/layer12_attention_weight_max": 24.12084197998047,
      "activations/layer12_attention_weight_min": -22.777400970458984,
      "activations/layer13_attention_weight_max": 55.24809265136719,
      "activations/layer13_attention_weight_min": -36.996665954589844,
      "activations/layer14_attention_weight_max": 47.47764205932617,
      "activations/layer14_attention_weight_min": -32.82450485229492,
      "activations/layer15_attention_weight_max": 46.856014251708984,
      "activations/layer15_attention_weight_min": -33.53462600708008,
      "activations/layer16_attention_weight_max": 38.12765121459961,
      "activations/layer16_attention_weight_min": -25.720247268676758,
      "activations/layer17_attention_weight_max": 54.3474006652832,
      "activations/layer17_attention_weight_min": -34.47520065307617,
      "activations/layer18_attention_weight_max": 55.267887115478516,
      "activations/layer18_attention_weight_min": -30.08755111694336,
      "activations/layer19_attention_weight_max": 21.286684036254883,
      "activations/layer19_attention_weight_min": -17.470897674560547,
      "activations/layer1_attention_weight_max": 17.52202033996582,
      "activations/layer1_attention_weight_min": -13.046220779418945,
      "activations/layer20_attention_weight_max": 19.618906021118164,
      "activations/layer20_attention_weight_min": -19.841812133789062,
      "activations/layer21_attention_weight_max": 40.331947326660156,
      "activations/layer21_attention_weight_min": -30.13509178161621,
      "activations/layer22_attention_weight_max": 34.25812530517578,
      "activations/layer22_attention_weight_min": -22.726661682128906,
      "activations/layer23_attention_weight_max": 25.34624481201172,
      "activations/layer23_attention_weight_min": -16.55540657043457,
      "activations/layer2_attention_weight_max": 26.31395721435547,
      "activations/layer2_attention_weight_min": -25.13666343688965,
      "activations/layer3_attention_weight_max": 65.74956512451172,
      "activations/layer3_attention_weight_min": -71.35653686523438,
      "activations/layer4_attention_weight_max": 101.12157440185547,
      "activations/layer4_attention_weight_min": -90.90972137451172,
      "activations/layer5_attention_weight_max": 130.1179962158203,
      "activations/layer5_attention_weight_min": -106.89418029785156,
      "activations/layer6_attention_weight_max": 66.14247131347656,
      "activations/layer6_attention_weight_min": -52.29391098022461,
      "activations/layer7_attention_weight_max": 58.932228088378906,
      "activations/layer7_attention_weight_min": -52.14704895019531,
      "activations/layer8_attention_weight_max": 41.94743347167969,
      "activations/layer8_attention_weight_min": -34.989559173583984,
      "activations/layer9_attention_weight_max": 57.342891693115234,
      "activations/layer9_attention_weight_min": -52.050331115722656,
      "epoch": 2.73,
      "learning_rate": 0.0001337344696969697,
      "loss": 2.9261,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_loss": 2.87109375,
      "eval_runtime": 8.497,
      "eval_samples_per_second": 505.355,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_openwebtext_loss": 2.87109375,
      "eval_openwebtext_ppl": 17.656319241826594,
      "eval_openwebtext_runtime": 8.497,
      "eval_openwebtext_samples_per_second": 505.355,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_wikitext_loss": 3.146484375,
      "eval_wikitext_ppl": 23.25416777248505,
      "eval_wikitext_runtime": 1.9345,
      "eval_wikitext_samples_per_second": 235.722,
      "step": 47000
    },
    {
      "epoch": 2.73,
      "eval_lambada_loss": 2.990234375,
      "eval_lambada_ppl": 19.89034374461794,
      "eval_lambada_runtime": 9.5514,
      "eval_lambada_samples_per_second": 509.768,
      "step": 47000
    },
    {
      "activations/layer0_attention_weight_max": 14.274373054504395,
      "activations/layer0_attention_weight_min": -14.670141220092773,
      "activations/layer10_attention_weight_max": 51.93659591674805,
      "activations/layer10_attention_weight_min": -39.97919464111328,
      "activations/layer11_attention_weight_max": 40.63661193847656,
      "activations/layer11_attention_weight_min": -33.39350128173828,
      "activations/layer12_attention_weight_max": 26.211774826049805,
      "activations/layer12_attention_weight_min": -24.432422637939453,
      "activations/layer13_attention_weight_max": 47.381568908691406,
      "activations/layer13_attention_weight_min": -37.64973068237305,
      "activations/layer14_attention_weight_max": 44.88431167602539,
      "activations/layer14_attention_weight_min": -37.03125762939453,
      "activations/layer15_attention_weight_max": 50.376853942871094,
      "activations/layer15_attention_weight_min": -34.128028869628906,
      "activations/layer16_attention_weight_max": 43.195213317871094,
      "activations/layer16_attention_weight_min": -26.570642471313477,
      "activations/layer17_attention_weight_max": 62.1151008605957,
      "activations/layer17_attention_weight_min": -36.83650588989258,
      "activations/layer18_attention_weight_max": 54.01689147949219,
      "activations/layer18_attention_weight_min": -30.953899383544922,
      "activations/layer19_attention_weight_max": 22.930885314941406,
      "activations/layer19_attention_weight_min": -17.75149154663086,
      "activations/layer1_attention_weight_max": 16.815610885620117,
      "activations/layer1_attention_weight_min": -13.227957725524902,
      "activations/layer20_attention_weight_max": 21.43967056274414,
      "activations/layer20_attention_weight_min": -18.30406951904297,
      "activations/layer21_attention_weight_max": 41.02265930175781,
      "activations/layer21_attention_weight_min": -27.897064208984375,
      "activations/layer22_attention_weight_max": 35.98924255371094,
      "activations/layer22_attention_weight_min": -24.07265281677246,
      "activations/layer23_attention_weight_max": 23.787044525146484,
      "activations/layer23_attention_weight_min": -15.341227531433105,
      "activations/layer2_attention_weight_max": 27.661705017089844,
      "activations/layer2_attention_weight_min": -25.689186096191406,
      "activations/layer3_attention_weight_max": 74.13345336914062,
      "activations/layer3_attention_weight_min": -76.0529556274414,
      "activations/layer4_attention_weight_max": 102.7314682006836,
      "activations/layer4_attention_weight_min": -90.8853759765625,
      "activations/layer5_attention_weight_max": 132.41949462890625,
      "activations/layer5_attention_weight_min": -110.05672454833984,
      "activations/layer6_attention_weight_max": 66.10313415527344,
      "activations/layer6_attention_weight_min": -53.069820404052734,
      "activations/layer7_attention_weight_max": 61.568729400634766,
      "activations/layer7_attention_weight_min": -57.464576721191406,
      "activations/layer8_attention_weight_max": 44.89661407470703,
      "activations/layer8_attention_weight_min": -39.50035858154297,
      "activations/layer9_attention_weight_max": 60.643218994140625,
      "activations/layer9_attention_weight_min": -52.85721969604492,
      "epoch": 2.73,
      "learning_rate": 0.0001337155303030303,
      "loss": 2.9157,
      "step": 47050
    },
    {
      "activations/layer0_attention_weight_max": 14.947607040405273,
      "activations/layer0_attention_weight_min": -14.493691444396973,
      "activations/layer10_attention_weight_max": 43.4069938659668,
      "activations/layer10_attention_weight_min": -37.035194396972656,
      "activations/layer11_attention_weight_max": 32.87791442871094,
      "activations/layer11_attention_weight_min": -29.282672882080078,
      "activations/layer12_attention_weight_max": 22.847354888916016,
      "activations/layer12_attention_weight_min": -23.899133682250977,
      "activations/layer13_attention_weight_max": 56.31166458129883,
      "activations/layer13_attention_weight_min": -42.944698333740234,
      "activations/layer14_attention_weight_max": 45.76087951660156,
      "activations/layer14_attention_weight_min": -35.78497314453125,
      "activations/layer15_attention_weight_max": 46.69683074951172,
      "activations/layer15_attention_weight_min": -35.318084716796875,
      "activations/layer16_attention_weight_max": 37.90406799316406,
      "activations/layer16_attention_weight_min": -27.759870529174805,
      "activations/layer17_attention_weight_max": 54.8385009765625,
      "activations/layer17_attention_weight_min": -37.2867431640625,
      "activations/layer18_attention_weight_max": 50.32943344116211,
      "activations/layer18_attention_weight_min": -31.228622436523438,
      "activations/layer19_attention_weight_max": 22.522748947143555,
      "activations/layer19_attention_weight_min": -18.435306549072266,
      "activations/layer1_attention_weight_max": 17.2808837890625,
      "activations/layer1_attention_weight_min": -13.844486236572266,
      "activations/layer20_attention_weight_max": 18.6096134185791,
      "activations/layer20_attention_weight_min": -21.052812576293945,
      "activations/layer21_attention_weight_max": 36.72809600830078,
      "activations/layer21_attention_weight_min": -29.710519790649414,
      "activations/layer22_attention_weight_max": 33.9421272277832,
      "activations/layer22_attention_weight_min": -21.35173988342285,
      "activations/layer23_attention_weight_max": 24.100181579589844,
      "activations/layer23_attention_weight_min": -16.139022827148438,
      "activations/layer2_attention_weight_max": 27.117733001708984,
      "activations/layer2_attention_weight_min": -26.86189842224121,
      "activations/layer3_attention_weight_max": 64.64921569824219,
      "activations/layer3_attention_weight_min": -73.48287200927734,
      "activations/layer4_attention_weight_max": 96.09810638427734,
      "activations/layer4_attention_weight_min": -92.1815185546875,
      "activations/layer5_attention_weight_max": 121.46318054199219,
      "activations/layer5_attention_weight_min": -102.38739013671875,
      "activations/layer6_attention_weight_max": 61.45888137817383,
      "activations/layer6_attention_weight_min": -50.67843246459961,
      "activations/layer7_attention_weight_max": 54.168251037597656,
      "activations/layer7_attention_weight_min": -52.447784423828125,
      "activations/layer8_attention_weight_max": 38.90696716308594,
      "activations/layer8_attention_weight_min": -36.057640075683594,
      "activations/layer9_attention_weight_max": 49.847557067871094,
      "activations/layer9_attention_weight_min": -51.89813232421875,
      "epoch": 2.74,
      "learning_rate": 0.0001336965909090909,
      "loss": 2.9234,
      "step": 47100
    },
    {
      "activations/layer0_attention_weight_max": 13.73134708404541,
      "activations/layer0_attention_weight_min": -14.323634147644043,
      "activations/layer10_attention_weight_max": 56.67639923095703,
      "activations/layer10_attention_weight_min": -39.59453201293945,
      "activations/layer11_attention_weight_max": 39.288734436035156,
      "activations/layer11_attention_weight_min": -30.972631454467773,
      "activations/layer12_attention_weight_max": 45.19718933105469,
      "activations/layer12_attention_weight_min": -26.62893295288086,
      "activations/layer13_attention_weight_max": 70.88955688476562,
      "activations/layer13_attention_weight_min": -42.698177337646484,
      "activations/layer14_attention_weight_max": 55.44563674926758,
      "activations/layer14_attention_weight_min": -39.41973114013672,
      "activations/layer15_attention_weight_max": 57.55427169799805,
      "activations/layer15_attention_weight_min": -38.50562286376953,
      "activations/layer16_attention_weight_max": 36.83537673950195,
      "activations/layer16_attention_weight_min": -27.34000015258789,
      "activations/layer17_attention_weight_max": 63.21429443359375,
      "activations/layer17_attention_weight_min": -40.201534271240234,
      "activations/layer18_attention_weight_max": 53.47275924682617,
      "activations/layer18_attention_weight_min": -33.323177337646484,
      "activations/layer19_attention_weight_max": 19.61272621154785,
      "activations/layer19_attention_weight_min": -17.252653121948242,
      "activations/layer1_attention_weight_max": 16.790645599365234,
      "activations/layer1_attention_weight_min": -15.861444473266602,
      "activations/layer20_attention_weight_max": 21.454204559326172,
      "activations/layer20_attention_weight_min": -18.009370803833008,
      "activations/layer21_attention_weight_max": 39.85147476196289,
      "activations/layer21_attention_weight_min": -26.80529022216797,
      "activations/layer22_attention_weight_max": 35.05714797973633,
      "activations/layer22_attention_weight_min": -23.66863250732422,
      "activations/layer23_attention_weight_max": 22.803298950195312,
      "activations/layer23_attention_weight_min": -15.685138702392578,
      "activations/layer2_attention_weight_max": 30.314300537109375,
      "activations/layer2_attention_weight_min": -27.963672637939453,
      "activations/layer3_attention_weight_max": 67.9759521484375,
      "activations/layer3_attention_weight_min": -76.33565521240234,
      "activations/layer4_attention_weight_max": 96.50859832763672,
      "activations/layer4_attention_weight_min": -88.25493621826172,
      "activations/layer5_attention_weight_max": 121.86886596679688,
      "activations/layer5_attention_weight_min": -103.47084045410156,
      "activations/layer6_attention_weight_max": 58.7685661315918,
      "activations/layer6_attention_weight_min": -50.28837966918945,
      "activations/layer7_attention_weight_max": 58.23158264160156,
      "activations/layer7_attention_weight_min": -53.05703353881836,
      "activations/layer8_attention_weight_max": 42.66085433959961,
      "activations/layer8_attention_weight_min": -38.107852935791016,
      "activations/layer9_attention_weight_max": 52.819976806640625,
      "activations/layer9_attention_weight_min": -47.72262191772461,
      "epoch": 2.74,
      "learning_rate": 0.0001336776515151515,
      "loss": 2.9158,
      "step": 47150
    },
    {
      "activations/layer0_attention_weight_max": 13.880192756652832,
      "activations/layer0_attention_weight_min": -14.490900039672852,
      "activations/layer10_attention_weight_max": 43.67396926879883,
      "activations/layer10_attention_weight_min": -36.37105178833008,
      "activations/layer11_attention_weight_max": 35.380340576171875,
      "activations/layer11_attention_weight_min": -28.005910873413086,
      "activations/layer12_attention_weight_max": 23.397062301635742,
      "activations/layer12_attention_weight_min": -26.315792083740234,
      "activations/layer13_attention_weight_max": 63.77513885498047,
      "activations/layer13_attention_weight_min": -49.66022872924805,
      "activations/layer14_attention_weight_max": 52.18391799926758,
      "activations/layer14_attention_weight_min": -38.43888854980469,
      "activations/layer15_attention_weight_max": 57.12626266479492,
      "activations/layer15_attention_weight_min": -40.0963249206543,
      "activations/layer16_attention_weight_max": 38.068851470947266,
      "activations/layer16_attention_weight_min": -28.65959358215332,
      "activations/layer17_attention_weight_max": 59.42089080810547,
      "activations/layer17_attention_weight_min": -35.973663330078125,
      "activations/layer18_attention_weight_max": 51.53194046020508,
      "activations/layer18_attention_weight_min": -31.176225662231445,
      "activations/layer19_attention_weight_max": 23.766477584838867,
      "activations/layer19_attention_weight_min": -21.506378173828125,
      "activations/layer1_attention_weight_max": 18.42264747619629,
      "activations/layer1_attention_weight_min": -13.354243278503418,
      "activations/layer20_attention_weight_max": 21.12492561340332,
      "activations/layer20_attention_weight_min": -18.479087829589844,
      "activations/layer21_attention_weight_max": 39.77762985229492,
      "activations/layer21_attention_weight_min": -26.587913513183594,
      "activations/layer22_attention_weight_max": 34.471771240234375,
      "activations/layer22_attention_weight_min": -24.08022117614746,
      "activations/layer23_attention_weight_max": 29.54366683959961,
      "activations/layer23_attention_weight_min": -17.719989776611328,
      "activations/layer2_attention_weight_max": 28.127609252929688,
      "activations/layer2_attention_weight_min": -26.564809799194336,
      "activations/layer3_attention_weight_max": 66.62222290039062,
      "activations/layer3_attention_weight_min": -71.47039031982422,
      "activations/layer4_attention_weight_max": 94.49662017822266,
      "activations/layer4_attention_weight_min": -87.31859588623047,
      "activations/layer5_attention_weight_max": 122.36492919921875,
      "activations/layer5_attention_weight_min": -103.33300018310547,
      "activations/layer6_attention_weight_max": 58.91120910644531,
      "activations/layer6_attention_weight_min": -49.06608963012695,
      "activations/layer7_attention_weight_max": 56.51122283935547,
      "activations/layer7_attention_weight_min": -56.579044342041016,
      "activations/layer8_attention_weight_max": 39.69678497314453,
      "activations/layer8_attention_weight_min": -35.154701232910156,
      "activations/layer9_attention_weight_max": 57.31930160522461,
      "activations/layer9_attention_weight_min": -53.83131790161133,
      "epoch": 2.74,
      "learning_rate": 0.0001336587121212121,
      "loss": 2.8999,
      "step": 47200
    },
    {
      "activations/layer0_attention_weight_max": 15.228205680847168,
      "activations/layer0_attention_weight_min": -14.218976974487305,
      "activations/layer10_attention_weight_max": 50.19718551635742,
      "activations/layer10_attention_weight_min": -39.53339767456055,
      "activations/layer11_attention_weight_max": 46.82221603393555,
      "activations/layer11_attention_weight_min": -32.492958068847656,
      "activations/layer12_attention_weight_max": 27.66459846496582,
      "activations/layer12_attention_weight_min": -23.392196655273438,
      "activations/layer13_attention_weight_max": 56.75792694091797,
      "activations/layer13_attention_weight_min": -36.761009216308594,
      "activations/layer14_attention_weight_max": 61.53776550292969,
      "activations/layer14_attention_weight_min": -35.05187225341797,
      "activations/layer15_attention_weight_max": 46.71464157104492,
      "activations/layer15_attention_weight_min": -36.32943344116211,
      "activations/layer16_attention_weight_max": 39.00114822387695,
      "activations/layer16_attention_weight_min": -29.097742080688477,
      "activations/layer17_attention_weight_max": 55.55648422241211,
      "activations/layer17_attention_weight_min": -38.571876525878906,
      "activations/layer18_attention_weight_max": 51.50070571899414,
      "activations/layer18_attention_weight_min": -38.573097229003906,
      "activations/layer19_attention_weight_max": 20.531747817993164,
      "activations/layer19_attention_weight_min": -17.058286666870117,
      "activations/layer1_attention_weight_max": 17.669523239135742,
      "activations/layer1_attention_weight_min": -13.282875061035156,
      "activations/layer20_attention_weight_max": 18.89533233642578,
      "activations/layer20_attention_weight_min": -19.335851669311523,
      "activations/layer21_attention_weight_max": 38.50796890258789,
      "activations/layer21_attention_weight_min": -29.61949920654297,
      "activations/layer22_attention_weight_max": 33.36381149291992,
      "activations/layer22_attention_weight_min": -24.522315979003906,
      "activations/layer23_attention_weight_max": 23.445363998413086,
      "activations/layer23_attention_weight_min": -15.834269523620605,
      "activations/layer2_attention_weight_max": 28.971553802490234,
      "activations/layer2_attention_weight_min": -27.18636131286621,
      "activations/layer3_attention_weight_max": 67.56285095214844,
      "activations/layer3_attention_weight_min": -72.72930908203125,
      "activations/layer4_attention_weight_max": 96.50370025634766,
      "activations/layer4_attention_weight_min": -89.29241180419922,
      "activations/layer5_attention_weight_max": 120.27093505859375,
      "activations/layer5_attention_weight_min": -101.63833618164062,
      "activations/layer6_attention_weight_max": 59.54449462890625,
      "activations/layer6_attention_weight_min": -49.7066535949707,
      "activations/layer7_attention_weight_max": 58.08662033081055,
      "activations/layer7_attention_weight_min": -52.3609733581543,
      "activations/layer8_attention_weight_max": 39.52162551879883,
      "activations/layer8_attention_weight_min": -35.13236618041992,
      "activations/layer9_attention_weight_max": 62.1133918762207,
      "activations/layer9_attention_weight_min": -49.78529357910156,
      "epoch": 2.75,
      "learning_rate": 0.00013363977272727272,
      "loss": 2.9138,
      "step": 47250
    },
    {
      "activations/layer0_attention_weight_max": 14.84453010559082,
      "activations/layer0_attention_weight_min": -14.24085807800293,
      "activations/layer10_attention_weight_max": 53.2099723815918,
      "activations/layer10_attention_weight_min": -38.80250930786133,
      "activations/layer11_attention_weight_max": 42.03244400024414,
      "activations/layer11_attention_weight_min": -31.198780059814453,
      "activations/layer12_attention_weight_max": 30.736534118652344,
      "activations/layer12_attention_weight_min": -28.857892990112305,
      "activations/layer13_attention_weight_max": 89.76351165771484,
      "activations/layer13_attention_weight_min": -54.941627502441406,
      "activations/layer14_attention_weight_max": 99.81987762451172,
      "activations/layer14_attention_weight_min": -52.81556701660156,
      "activations/layer15_attention_weight_max": 51.79582977294922,
      "activations/layer15_attention_weight_min": -33.408084869384766,
      "activations/layer16_attention_weight_max": 58.07316970825195,
      "activations/layer16_attention_weight_min": -27.015932083129883,
      "activations/layer17_attention_weight_max": 70.77371978759766,
      "activations/layer17_attention_weight_min": -34.93571853637695,
      "activations/layer18_attention_weight_max": 52.03013229370117,
      "activations/layer18_attention_weight_min": -30.400066375732422,
      "activations/layer19_attention_weight_max": 19.678815841674805,
      "activations/layer19_attention_weight_min": -17.0411319732666,
      "activations/layer1_attention_weight_max": 17.075027465820312,
      "activations/layer1_attention_weight_min": -15.51386547088623,
      "activations/layer20_attention_weight_max": 24.725040435791016,
      "activations/layer20_attention_weight_min": -18.521568298339844,
      "activations/layer21_attention_weight_max": 56.015438079833984,
      "activations/layer21_attention_weight_min": -24.98277473449707,
      "activations/layer22_attention_weight_max": 32.77031707763672,
      "activations/layer22_attention_weight_min": -24.892406463623047,
      "activations/layer23_attention_weight_max": 25.105329513549805,
      "activations/layer23_attention_weight_min": -16.68093490600586,
      "activations/layer2_attention_weight_max": 31.4801025390625,
      "activations/layer2_attention_weight_min": -28.284597396850586,
      "activations/layer3_attention_weight_max": 76.96527862548828,
      "activations/layer3_attention_weight_min": -76.01535034179688,
      "activations/layer4_attention_weight_max": 106.6922836303711,
      "activations/layer4_attention_weight_min": -92.95284271240234,
      "activations/layer5_attention_weight_max": 139.43783569335938,
      "activations/layer5_attention_weight_min": -104.85881805419922,
      "activations/layer6_attention_weight_max": 70.4820785522461,
      "activations/layer6_attention_weight_min": -52.005794525146484,
      "activations/layer7_attention_weight_max": 70.69803619384766,
      "activations/layer7_attention_weight_min": -57.79384231567383,
      "activations/layer8_attention_weight_max": 47.33308792114258,
      "activations/layer8_attention_weight_min": -39.96723556518555,
      "activations/layer9_attention_weight_max": 68.72516632080078,
      "activations/layer9_attention_weight_min": -54.7070426940918,
      "epoch": 2.75,
      "learning_rate": 0.0001336208333333333,
      "loss": 2.8977,
      "step": 47300
    },
    {
      "activations/layer0_attention_weight_max": 13.165382385253906,
      "activations/layer0_attention_weight_min": -14.77384090423584,
      "activations/layer10_attention_weight_max": 50.056697845458984,
      "activations/layer10_attention_weight_min": -38.390174865722656,
      "activations/layer11_attention_weight_max": 43.19874572753906,
      "activations/layer11_attention_weight_min": -29.19776153564453,
      "activations/layer12_attention_weight_max": 26.560665130615234,
      "activations/layer12_attention_weight_min": -23.775094985961914,
      "activations/layer13_attention_weight_max": 60.729225158691406,
      "activations/layer13_attention_weight_min": -44.17665481567383,
      "activations/layer14_attention_weight_max": 51.24393081665039,
      "activations/layer14_attention_weight_min": -37.50397872924805,
      "activations/layer15_attention_weight_max": 52.5621223449707,
      "activations/layer15_attention_weight_min": -35.01138687133789,
      "activations/layer16_attention_weight_max": 39.538516998291016,
      "activations/layer16_attention_weight_min": -26.062353134155273,
      "activations/layer17_attention_weight_max": 62.68559646606445,
      "activations/layer17_attention_weight_min": -37.738033294677734,
      "activations/layer18_attention_weight_max": 54.7459716796875,
      "activations/layer18_attention_weight_min": -31.861173629760742,
      "activations/layer19_attention_weight_max": 23.734792709350586,
      "activations/layer19_attention_weight_min": -15.829228401184082,
      "activations/layer1_attention_weight_max": 17.096750259399414,
      "activations/layer1_attention_weight_min": -16.084991455078125,
      "activations/layer20_attention_weight_max": 22.718616485595703,
      "activations/layer20_attention_weight_min": -20.00779914855957,
      "activations/layer21_attention_weight_max": 40.69698715209961,
      "activations/layer21_attention_weight_min": -26.554349899291992,
      "activations/layer22_attention_weight_max": 36.17661666870117,
      "activations/layer22_attention_weight_min": -23.79668617248535,
      "activations/layer23_attention_weight_max": 23.586484909057617,
      "activations/layer23_attention_weight_min": -16.645278930664062,
      "activations/layer2_attention_weight_max": 30.584121704101562,
      "activations/layer2_attention_weight_min": -28.930173873901367,
      "activations/layer3_attention_weight_max": 68.66620635986328,
      "activations/layer3_attention_weight_min": -73.962646484375,
      "activations/layer4_attention_weight_max": 96.39400482177734,
      "activations/layer4_attention_weight_min": -93.04326629638672,
      "activations/layer5_attention_weight_max": 123.76997375488281,
      "activations/layer5_attention_weight_min": -113.86637878417969,
      "activations/layer6_attention_weight_max": 61.06839370727539,
      "activations/layer6_attention_weight_min": -52.421566009521484,
      "activations/layer7_attention_weight_max": 55.5490837097168,
      "activations/layer7_attention_weight_min": -57.26482391357422,
      "activations/layer8_attention_weight_max": 40.40023422241211,
      "activations/layer8_attention_weight_min": -36.062442779541016,
      "activations/layer9_attention_weight_max": 55.057430267333984,
      "activations/layer9_attention_weight_min": -50.043514251708984,
      "epoch": 2.75,
      "learning_rate": 0.00013360189393939393,
      "loss": 2.9226,
      "step": 47350
    },
    {
      "activations/layer0_attention_weight_max": 13.944774627685547,
      "activations/layer0_attention_weight_min": -15.01675796508789,
      "activations/layer10_attention_weight_max": 57.839351654052734,
      "activations/layer10_attention_weight_min": -39.41959762573242,
      "activations/layer11_attention_weight_max": 46.79096984863281,
      "activations/layer11_attention_weight_min": -31.805204391479492,
      "activations/layer12_attention_weight_max": 48.97603225708008,
      "activations/layer12_attention_weight_min": -28.966184616088867,
      "activations/layer13_attention_weight_max": 57.46413040161133,
      "activations/layer13_attention_weight_min": -42.86592102050781,
      "activations/layer14_attention_weight_max": 51.59871292114258,
      "activations/layer14_attention_weight_min": -38.52407455444336,
      "activations/layer15_attention_weight_max": 43.82814407348633,
      "activations/layer15_attention_weight_min": -35.14151382446289,
      "activations/layer16_attention_weight_max": 33.904212951660156,
      "activations/layer16_attention_weight_min": -25.837764739990234,
      "activations/layer17_attention_weight_max": 57.2578125,
      "activations/layer17_attention_weight_min": -36.80385971069336,
      "activations/layer18_attention_weight_max": 49.556854248046875,
      "activations/layer18_attention_weight_min": -30.10369300842285,
      "activations/layer19_attention_weight_max": 21.013322830200195,
      "activations/layer19_attention_weight_min": -15.5787992477417,
      "activations/layer1_attention_weight_max": 17.101076126098633,
      "activations/layer1_attention_weight_min": -18.07694435119629,
      "activations/layer20_attention_weight_max": 18.515945434570312,
      "activations/layer20_attention_weight_min": -19.07794761657715,
      "activations/layer21_attention_weight_max": 38.70142364501953,
      "activations/layer21_attention_weight_min": -28.97624969482422,
      "activations/layer22_attention_weight_max": 32.90178298950195,
      "activations/layer22_attention_weight_min": -22.015329360961914,
      "activations/layer23_attention_weight_max": 20.61042022705078,
      "activations/layer23_attention_weight_min": -15.58415412902832,
      "activations/layer2_attention_weight_max": 31.25312614440918,
      "activations/layer2_attention_weight_min": -28.58606719970703,
      "activations/layer3_attention_weight_max": 71.51333618164062,
      "activations/layer3_attention_weight_min": -75.88387298583984,
      "activations/layer4_attention_weight_max": 105.76411437988281,
      "activations/layer4_attention_weight_min": -91.30072784423828,
      "activations/layer5_attention_weight_max": 127.39046478271484,
      "activations/layer5_attention_weight_min": -100.1097640991211,
      "activations/layer6_attention_weight_max": 63.03477096557617,
      "activations/layer6_attention_weight_min": -49.194480895996094,
      "activations/layer7_attention_weight_max": 64.89447021484375,
      "activations/layer7_attention_weight_min": -53.76657485961914,
      "activations/layer8_attention_weight_max": 51.98078918457031,
      "activations/layer8_attention_weight_min": -37.0185546875,
      "activations/layer9_attention_weight_max": 72.06893920898438,
      "activations/layer9_attention_weight_min": -56.03995895385742,
      "epoch": 2.75,
      "learning_rate": 0.00013358295454545454,
      "loss": 2.928,
      "step": 47400
    },
    {
      "activations/layer0_attention_weight_max": 14.733819007873535,
      "activations/layer0_attention_weight_min": -14.018900871276855,
      "activations/layer10_attention_weight_max": 50.64777755737305,
      "activations/layer10_attention_weight_min": -40.503055572509766,
      "activations/layer11_attention_weight_max": 43.45079803466797,
      "activations/layer11_attention_weight_min": -28.918739318847656,
      "activations/layer12_attention_weight_max": 28.80179786682129,
      "activations/layer12_attention_weight_min": -23.363672256469727,
      "activations/layer13_attention_weight_max": 50.57932662963867,
      "activations/layer13_attention_weight_min": -40.31752395629883,
      "activations/layer14_attention_weight_max": 51.5676383972168,
      "activations/layer14_attention_weight_min": -37.26178741455078,
      "activations/layer15_attention_weight_max": 50.91312789916992,
      "activations/layer15_attention_weight_min": -36.27705764770508,
      "activations/layer16_attention_weight_max": 39.73694610595703,
      "activations/layer16_attention_weight_min": -28.476015090942383,
      "activations/layer17_attention_weight_max": 60.31079864501953,
      "activations/layer17_attention_weight_min": -39.42670822143555,
      "activations/layer18_attention_weight_max": 59.022525787353516,
      "activations/layer18_attention_weight_min": -35.25040054321289,
      "activations/layer19_attention_weight_max": 29.342405319213867,
      "activations/layer19_attention_weight_min": -17.10301399230957,
      "activations/layer1_attention_weight_max": 16.629222869873047,
      "activations/layer1_attention_weight_min": -13.205543518066406,
      "activations/layer20_attention_weight_max": 23.95315933227539,
      "activations/layer20_attention_weight_min": -19.556018829345703,
      "activations/layer21_attention_weight_max": 48.607574462890625,
      "activations/layer21_attention_weight_min": -30.989309310913086,
      "activations/layer22_attention_weight_max": 36.14987564086914,
      "activations/layer22_attention_weight_min": -24.80462646484375,
      "activations/layer23_attention_weight_max": 26.586219787597656,
      "activations/layer23_attention_weight_min": -17.807830810546875,
      "activations/layer2_attention_weight_max": 30.06661605834961,
      "activations/layer2_attention_weight_min": -28.160612106323242,
      "activations/layer3_attention_weight_max": 73.41388702392578,
      "activations/layer3_attention_weight_min": -78.1785659790039,
      "activations/layer4_attention_weight_max": 107.24647521972656,
      "activations/layer4_attention_weight_min": -94.88713073730469,
      "activations/layer5_attention_weight_max": 131.05894470214844,
      "activations/layer5_attention_weight_min": -106.1534652709961,
      "activations/layer6_attention_weight_max": 62.93516540527344,
      "activations/layer6_attention_weight_min": -51.44910430908203,
      "activations/layer7_attention_weight_max": 58.28281784057617,
      "activations/layer7_attention_weight_min": -55.14616775512695,
      "activations/layer8_attention_weight_max": 43.97206497192383,
      "activations/layer8_attention_weight_min": -36.14210510253906,
      "activations/layer9_attention_weight_max": 57.083709716796875,
      "activations/layer9_attention_weight_min": -50.04648971557617,
      "epoch": 2.76,
      "learning_rate": 0.00013356401515151513,
      "loss": 2.9148,
      "step": 47450
    },
    {
      "activations/layer0_attention_weight_max": 14.934057235717773,
      "activations/layer0_attention_weight_min": -14.575276374816895,
      "activations/layer10_attention_weight_max": 48.583740234375,
      "activations/layer10_attention_weight_min": -40.96088790893555,
      "activations/layer11_attention_weight_max": 42.230098724365234,
      "activations/layer11_attention_weight_min": -35.42863845825195,
      "activations/layer12_attention_weight_max": 29.22295570373535,
      "activations/layer12_attention_weight_min": -25.21677017211914,
      "activations/layer13_attention_weight_max": 65.72332000732422,
      "activations/layer13_attention_weight_min": -46.19255828857422,
      "activations/layer14_attention_weight_max": 63.80897521972656,
      "activations/layer14_attention_weight_min": -44.06947708129883,
      "activations/layer15_attention_weight_max": 47.50136947631836,
      "activations/layer15_attention_weight_min": -35.75221252441406,
      "activations/layer16_attention_weight_max": 35.489017486572266,
      "activations/layer16_attention_weight_min": -26.20403480529785,
      "activations/layer17_attention_weight_max": 61.6964111328125,
      "activations/layer17_attention_weight_min": -35.84553909301758,
      "activations/layer18_attention_weight_max": 54.2122688293457,
      "activations/layer18_attention_weight_min": -31.978105545043945,
      "activations/layer19_attention_weight_max": 26.6231746673584,
      "activations/layer19_attention_weight_min": -17.04009437561035,
      "activations/layer1_attention_weight_max": 16.978565216064453,
      "activations/layer1_attention_weight_min": -13.138620376586914,
      "activations/layer20_attention_weight_max": 21.801048278808594,
      "activations/layer20_attention_weight_min": -18.093547821044922,
      "activations/layer21_attention_weight_max": 42.76332473754883,
      "activations/layer21_attention_weight_min": -26.709997177124023,
      "activations/layer22_attention_weight_max": 34.29251480102539,
      "activations/layer22_attention_weight_min": -22.62666893005371,
      "activations/layer23_attention_weight_max": 24.822467803955078,
      "activations/layer23_attention_weight_min": -17.83160972595215,
      "activations/layer2_attention_weight_max": 28.10883140563965,
      "activations/layer2_attention_weight_min": -28.394399642944336,
      "activations/layer3_attention_weight_max": 69.65821838378906,
      "activations/layer3_attention_weight_min": -70.60148620605469,
      "activations/layer4_attention_weight_max": 102.9384536743164,
      "activations/layer4_attention_weight_min": -87.92182159423828,
      "activations/layer5_attention_weight_max": 122.5824203491211,
      "activations/layer5_attention_weight_min": -100.30581665039062,
      "activations/layer6_attention_weight_max": 61.43727111816406,
      "activations/layer6_attention_weight_min": -51.28664779663086,
      "activations/layer7_attention_weight_max": 61.198875427246094,
      "activations/layer7_attention_weight_min": -56.36334991455078,
      "activations/layer8_attention_weight_max": 44.99687957763672,
      "activations/layer8_attention_weight_min": -37.1292610168457,
      "activations/layer9_attention_weight_max": 69.98912048339844,
      "activations/layer9_attention_weight_min": -57.333065032958984,
      "epoch": 2.76,
      "learning_rate": 0.00013354507575757575,
      "loss": 2.9068,
      "step": 47500
    },
    {
      "activations/layer0_attention_weight_max": 14.7410249710083,
      "activations/layer0_attention_weight_min": -13.833331108093262,
      "activations/layer10_attention_weight_max": 56.40351486206055,
      "activations/layer10_attention_weight_min": -36.11744689941406,
      "activations/layer11_attention_weight_max": 40.29884338378906,
      "activations/layer11_attention_weight_min": -28.16458511352539,
      "activations/layer12_attention_weight_max": 31.20783805847168,
      "activations/layer12_attention_weight_min": -23.56352996826172,
      "activations/layer13_attention_weight_max": 71.4503402709961,
      "activations/layer13_attention_weight_min": -37.87779998779297,
      "activations/layer14_attention_weight_max": 57.352542877197266,
      "activations/layer14_attention_weight_min": -31.409826278686523,
      "activations/layer15_attention_weight_max": 53.147613525390625,
      "activations/layer15_attention_weight_min": -31.14198112487793,
      "activations/layer16_attention_weight_max": 43.86050796508789,
      "activations/layer16_attention_weight_min": -26.318925857543945,
      "activations/layer17_attention_weight_max": 63.55082321166992,
      "activations/layer17_attention_weight_min": -34.32719802856445,
      "activations/layer18_attention_weight_max": 55.84978485107422,
      "activations/layer18_attention_weight_min": -30.595726013183594,
      "activations/layer19_attention_weight_max": 23.64802360534668,
      "activations/layer19_attention_weight_min": -17.028684616088867,
      "activations/layer1_attention_weight_max": 17.09784507751465,
      "activations/layer1_attention_weight_min": -14.770705223083496,
      "activations/layer20_attention_weight_max": 21.616657257080078,
      "activations/layer20_attention_weight_min": -17.483776092529297,
      "activations/layer21_attention_weight_max": 44.274658203125,
      "activations/layer21_attention_weight_min": -28.365392684936523,
      "activations/layer22_attention_weight_max": 35.12894821166992,
      "activations/layer22_attention_weight_min": -22.935659408569336,
      "activations/layer23_attention_weight_max": 29.165966033935547,
      "activations/layer23_attention_weight_min": -16.210803985595703,
      "activations/layer2_attention_weight_max": 28.782695770263672,
      "activations/layer2_attention_weight_min": -28.451492309570312,
      "activations/layer3_attention_weight_max": 72.12538146972656,
      "activations/layer3_attention_weight_min": -73.6640853881836,
      "activations/layer4_attention_weight_max": 103.1269760131836,
      "activations/layer4_attention_weight_min": -87.58588409423828,
      "activations/layer5_attention_weight_max": 127.97425842285156,
      "activations/layer5_attention_weight_min": -100.56672668457031,
      "activations/layer6_attention_weight_max": 61.582862854003906,
      "activations/layer6_attention_weight_min": -51.700016021728516,
      "activations/layer7_attention_weight_max": 60.0250244140625,
      "activations/layer7_attention_weight_min": -55.841514587402344,
      "activations/layer8_attention_weight_max": 41.66276931762695,
      "activations/layer8_attention_weight_min": -35.07424545288086,
      "activations/layer9_attention_weight_max": 60.599769592285156,
      "activations/layer9_attention_weight_min": -49.7368049621582,
      "epoch": 2.76,
      "learning_rate": 0.00013352613636363636,
      "loss": 2.9231,
      "step": 47550
    },
    {
      "activations/layer0_attention_weight_max": 15.097380638122559,
      "activations/layer0_attention_weight_min": -14.589805603027344,
      "activations/layer10_attention_weight_max": 45.16058349609375,
      "activations/layer10_attention_weight_min": -38.3941764831543,
      "activations/layer11_attention_weight_max": 38.16027069091797,
      "activations/layer11_attention_weight_min": -28.56311798095703,
      "activations/layer12_attention_weight_max": 24.892005920410156,
      "activations/layer12_attention_weight_min": -28.55453109741211,
      "activations/layer13_attention_weight_max": 45.275386810302734,
      "activations/layer13_attention_weight_min": -33.783905029296875,
      "activations/layer14_attention_weight_max": 50.56147003173828,
      "activations/layer14_attention_weight_min": -33.3037109375,
      "activations/layer15_attention_weight_max": 47.37837600708008,
      "activations/layer15_attention_weight_min": -32.33536911010742,
      "activations/layer16_attention_weight_max": 44.86994934082031,
      "activations/layer16_attention_weight_min": -29.124467849731445,
      "activations/layer17_attention_weight_max": 64.55338287353516,
      "activations/layer17_attention_weight_min": -36.007301330566406,
      "activations/layer18_attention_weight_max": 51.16809844970703,
      "activations/layer18_attention_weight_min": -30.690649032592773,
      "activations/layer19_attention_weight_max": 22.61192512512207,
      "activations/layer19_attention_weight_min": -18.81428337097168,
      "activations/layer1_attention_weight_max": 17.46927261352539,
      "activations/layer1_attention_weight_min": -14.70571517944336,
      "activations/layer20_attention_weight_max": 18.4665470123291,
      "activations/layer20_attention_weight_min": -19.825159072875977,
      "activations/layer21_attention_weight_max": 39.03961944580078,
      "activations/layer21_attention_weight_min": -27.126501083374023,
      "activations/layer22_attention_weight_max": 36.222103118896484,
      "activations/layer22_attention_weight_min": -23.093692779541016,
      "activations/layer23_attention_weight_max": 23.384002685546875,
      "activations/layer23_attention_weight_min": -16.841815948486328,
      "activations/layer2_attention_weight_max": 29.31855010986328,
      "activations/layer2_attention_weight_min": -26.978544235229492,
      "activations/layer3_attention_weight_max": 68.13014221191406,
      "activations/layer3_attention_weight_min": -69.57227325439453,
      "activations/layer4_attention_weight_max": 98.10250091552734,
      "activations/layer4_attention_weight_min": -88.2064437866211,
      "activations/layer5_attention_weight_max": 121.92881774902344,
      "activations/layer5_attention_weight_min": -94.54313659667969,
      "activations/layer6_attention_weight_max": 58.870121002197266,
      "activations/layer6_attention_weight_min": -47.79408264160156,
      "activations/layer7_attention_weight_max": 58.63371658325195,
      "activations/layer7_attention_weight_min": -51.954681396484375,
      "activations/layer8_attention_weight_max": 41.378055572509766,
      "activations/layer8_attention_weight_min": -34.53691864013672,
      "activations/layer9_attention_weight_max": 60.7428092956543,
      "activations/layer9_attention_weight_min": -53.4607048034668,
      "epoch": 2.77,
      "learning_rate": 0.00013350719696969695,
      "loss": 2.9267,
      "step": 47600
    },
    {
      "activations/layer0_attention_weight_max": 15.377325057983398,
      "activations/layer0_attention_weight_min": -14.355278968811035,
      "activations/layer10_attention_weight_max": 50.181026458740234,
      "activations/layer10_attention_weight_min": -37.8345832824707,
      "activations/layer11_attention_weight_max": 41.44682312011719,
      "activations/layer11_attention_weight_min": -31.435461044311523,
      "activations/layer12_attention_weight_max": 30.139076232910156,
      "activations/layer12_attention_weight_min": -24.455034255981445,
      "activations/layer13_attention_weight_max": 73.06507873535156,
      "activations/layer13_attention_weight_min": -47.84994888305664,
      "activations/layer14_attention_weight_max": 73.8038101196289,
      "activations/layer14_attention_weight_min": -42.942020416259766,
      "activations/layer15_attention_weight_max": 53.31817626953125,
      "activations/layer15_attention_weight_min": -35.92736053466797,
      "activations/layer16_attention_weight_max": 41.41575622558594,
      "activations/layer16_attention_weight_min": -26.321744918823242,
      "activations/layer17_attention_weight_max": 65.02945709228516,
      "activations/layer17_attention_weight_min": -37.64247512817383,
      "activations/layer18_attention_weight_max": 52.18820571899414,
      "activations/layer18_attention_weight_min": -31.08783531188965,
      "activations/layer19_attention_weight_max": 20.43621063232422,
      "activations/layer19_attention_weight_min": -16.339685440063477,
      "activations/layer1_attention_weight_max": 18.405973434448242,
      "activations/layer1_attention_weight_min": -13.155046463012695,
      "activations/layer20_attention_weight_max": 19.453340530395508,
      "activations/layer20_attention_weight_min": -20.125385284423828,
      "activations/layer21_attention_weight_max": 38.24268341064453,
      "activations/layer21_attention_weight_min": -27.114421844482422,
      "activations/layer22_attention_weight_max": 33.456947326660156,
      "activations/layer22_attention_weight_min": -25.002012252807617,
      "activations/layer23_attention_weight_max": 23.04749870300293,
      "activations/layer23_attention_weight_min": -16.55956268310547,
      "activations/layer2_attention_weight_max": 29.336170196533203,
      "activations/layer2_attention_weight_min": -26.861181259155273,
      "activations/layer3_attention_weight_max": 68.93724060058594,
      "activations/layer3_attention_weight_min": -69.232177734375,
      "activations/layer4_attention_weight_max": 102.2783203125,
      "activations/layer4_attention_weight_min": -90.31780242919922,
      "activations/layer5_attention_weight_max": 129.99172973632812,
      "activations/layer5_attention_weight_min": -104.16998291015625,
      "activations/layer6_attention_weight_max": 64.34920501708984,
      "activations/layer6_attention_weight_min": -53.474971771240234,
      "activations/layer7_attention_weight_max": 65.39122009277344,
      "activations/layer7_attention_weight_min": -61.7289924621582,
      "activations/layer8_attention_weight_max": 43.4210205078125,
      "activations/layer8_attention_weight_min": -37.35871505737305,
      "activations/layer9_attention_weight_max": 57.839881896972656,
      "activations/layer9_attention_weight_min": -55.6300163269043,
      "epoch": 2.77,
      "learning_rate": 0.00013348825757575756,
      "loss": 2.9155,
      "step": 47650
    },
    {
      "activations/layer0_attention_weight_max": 13.958086967468262,
      "activations/layer0_attention_weight_min": -14.628448486328125,
      "activations/layer10_attention_weight_max": 47.26033401489258,
      "activations/layer10_attention_weight_min": -38.399261474609375,
      "activations/layer11_attention_weight_max": 38.785888671875,
      "activations/layer11_attention_weight_min": -29.61325454711914,
      "activations/layer12_attention_weight_max": 30.10562515258789,
      "activations/layer12_attention_weight_min": -25.652999877929688,
      "activations/layer13_attention_weight_max": 52.63874435424805,
      "activations/layer13_attention_weight_min": -37.90899658203125,
      "activations/layer14_attention_weight_max": 52.227203369140625,
      "activations/layer14_attention_weight_min": -35.14989471435547,
      "activations/layer15_attention_weight_max": 49.48329162597656,
      "activations/layer15_attention_weight_min": -32.694759368896484,
      "activations/layer16_attention_weight_max": 40.205013275146484,
      "activations/layer16_attention_weight_min": -28.535558700561523,
      "activations/layer17_attention_weight_max": 57.93402099609375,
      "activations/layer17_attention_weight_min": -35.552703857421875,
      "activations/layer18_attention_weight_max": 49.156219482421875,
      "activations/layer18_attention_weight_min": -34.17375183105469,
      "activations/layer19_attention_weight_max": 22.173688888549805,
      "activations/layer19_attention_weight_min": -20.416112899780273,
      "activations/layer1_attention_weight_max": 17.44438934326172,
      "activations/layer1_attention_weight_min": -13.116336822509766,
      "activations/layer20_attention_weight_max": 20.19256019592285,
      "activations/layer20_attention_weight_min": -23.192705154418945,
      "activations/layer21_attention_weight_max": 41.059173583984375,
      "activations/layer21_attention_weight_min": -31.896432876586914,
      "activations/layer22_attention_weight_max": 34.03584289550781,
      "activations/layer22_attention_weight_min": -24.613496780395508,
      "activations/layer23_attention_weight_max": 24.051671981811523,
      "activations/layer23_attention_weight_min": -16.926000595092773,
      "activations/layer2_attention_weight_max": 29.161251068115234,
      "activations/layer2_attention_weight_min": -27.32270622253418,
      "activations/layer3_attention_weight_max": 65.21834564208984,
      "activations/layer3_attention_weight_min": -68.47494506835938,
      "activations/layer4_attention_weight_max": 96.46111297607422,
      "activations/layer4_attention_weight_min": -88.59236907958984,
      "activations/layer5_attention_weight_max": 123.50048065185547,
      "activations/layer5_attention_weight_min": -100.34745788574219,
      "activations/layer6_attention_weight_max": 59.832054138183594,
      "activations/layer6_attention_weight_min": -51.29264831542969,
      "activations/layer7_attention_weight_max": 56.46057891845703,
      "activations/layer7_attention_weight_min": -54.86152648925781,
      "activations/layer8_attention_weight_max": 39.46358871459961,
      "activations/layer8_attention_weight_min": -36.144248962402344,
      "activations/layer9_attention_weight_max": 61.71406936645508,
      "activations/layer9_attention_weight_min": -48.107112884521484,
      "epoch": 2.77,
      "learning_rate": 0.00013346931818181815,
      "loss": 2.9087,
      "step": 47700
    },
    {
      "activations/layer0_attention_weight_max": 14.644224166870117,
      "activations/layer0_attention_weight_min": -15.008055686950684,
      "activations/layer10_attention_weight_max": 49.61842727661133,
      "activations/layer10_attention_weight_min": -40.66187286376953,
      "activations/layer11_attention_weight_max": 40.585304260253906,
      "activations/layer11_attention_weight_min": -31.08922576904297,
      "activations/layer12_attention_weight_max": 25.649085998535156,
      "activations/layer12_attention_weight_min": -27.939451217651367,
      "activations/layer13_attention_weight_max": 68.05846405029297,
      "activations/layer13_attention_weight_min": -49.13949203491211,
      "activations/layer14_attention_weight_max": 66.1175308227539,
      "activations/layer14_attention_weight_min": -45.703529357910156,
      "activations/layer15_attention_weight_max": 55.175872802734375,
      "activations/layer15_attention_weight_min": -40.449825286865234,
      "activations/layer16_attention_weight_max": 39.20572280883789,
      "activations/layer16_attention_weight_min": -26.291847229003906,
      "activations/layer17_attention_weight_max": 57.45444869995117,
      "activations/layer17_attention_weight_min": -33.370689392089844,
      "activations/layer18_attention_weight_max": 45.59043502807617,
      "activations/layer18_attention_weight_min": -30.664043426513672,
      "activations/layer19_attention_weight_max": 23.037662506103516,
      "activations/layer19_attention_weight_min": -18.390066146850586,
      "activations/layer1_attention_weight_max": 18.641700744628906,
      "activations/layer1_attention_weight_min": -18.13558578491211,
      "activations/layer20_attention_weight_max": 20.159183502197266,
      "activations/layer20_attention_weight_min": -20.642248153686523,
      "activations/layer21_attention_weight_max": 44.90291976928711,
      "activations/layer21_attention_weight_min": -32.13521194458008,
      "activations/layer22_attention_weight_max": 32.21529769897461,
      "activations/layer22_attention_weight_min": -22.660085678100586,
      "activations/layer23_attention_weight_max": 21.723159790039062,
      "activations/layer23_attention_weight_min": -18.21961212158203,
      "activations/layer2_attention_weight_max": 30.934560775756836,
      "activations/layer2_attention_weight_min": -28.395263671875,
      "activations/layer3_attention_weight_max": 69.78512573242188,
      "activations/layer3_attention_weight_min": -72.60620880126953,
      "activations/layer4_attention_weight_max": 98.68965148925781,
      "activations/layer4_attention_weight_min": -91.38872528076172,
      "activations/layer5_attention_weight_max": 127.50765991210938,
      "activations/layer5_attention_weight_min": -105.51165771484375,
      "activations/layer6_attention_weight_max": 64.49907684326172,
      "activations/layer6_attention_weight_min": -51.813880920410156,
      "activations/layer7_attention_weight_max": 64.15910339355469,
      "activations/layer7_attention_weight_min": -61.505149841308594,
      "activations/layer8_attention_weight_max": 44.4187126159668,
      "activations/layer8_attention_weight_min": -40.834476470947266,
      "activations/layer9_attention_weight_max": 55.39274215698242,
      "activations/layer9_attention_weight_min": -55.73383712768555,
      "epoch": 2.77,
      "learning_rate": 0.00013345037878787877,
      "loss": 2.9177,
      "step": 47750
    },
    {
      "activations/layer0_attention_weight_max": 14.277548789978027,
      "activations/layer0_attention_weight_min": -14.38244915008545,
      "activations/layer10_attention_weight_max": 54.34415054321289,
      "activations/layer10_attention_weight_min": -38.4054069519043,
      "activations/layer11_attention_weight_max": 39.21432113647461,
      "activations/layer11_attention_weight_min": -27.69219207763672,
      "activations/layer12_attention_weight_max": 32.49098587036133,
      "activations/layer12_attention_weight_min": -23.9304141998291,
      "activations/layer13_attention_weight_max": 52.21055603027344,
      "activations/layer13_attention_weight_min": -37.57024002075195,
      "activations/layer14_attention_weight_max": 50.283592224121094,
      "activations/layer14_attention_weight_min": -33.199974060058594,
      "activations/layer15_attention_weight_max": 47.632442474365234,
      "activations/layer15_attention_weight_min": -37.70890808105469,
      "activations/layer16_attention_weight_max": 37.40200424194336,
      "activations/layer16_attention_weight_min": -26.426380157470703,
      "activations/layer17_attention_weight_max": 61.58366775512695,
      "activations/layer17_attention_weight_min": -36.9503288269043,
      "activations/layer18_attention_weight_max": 57.2524299621582,
      "activations/layer18_attention_weight_min": -33.9859733581543,
      "activations/layer19_attention_weight_max": 22.777128219604492,
      "activations/layer19_attention_weight_min": -17.29121208190918,
      "activations/layer1_attention_weight_max": 17.85293197631836,
      "activations/layer1_attention_weight_min": -15.517056465148926,
      "activations/layer20_attention_weight_max": 19.227121353149414,
      "activations/layer20_attention_weight_min": -18.00817108154297,
      "activations/layer21_attention_weight_max": 40.145477294921875,
      "activations/layer21_attention_weight_min": -30.552854537963867,
      "activations/layer22_attention_weight_max": 37.31689453125,
      "activations/layer22_attention_weight_min": -24.4279727935791,
      "activations/layer23_attention_weight_max": 23.72798728942871,
      "activations/layer23_attention_weight_min": -16.700040817260742,
      "activations/layer2_attention_weight_max": 28.65013885498047,
      "activations/layer2_attention_weight_min": -26.700834274291992,
      "activations/layer3_attention_weight_max": 65.21890258789062,
      "activations/layer3_attention_weight_min": -69.18766784667969,
      "activations/layer4_attention_weight_max": 98.22686004638672,
      "activations/layer4_attention_weight_min": -88.49292755126953,
      "activations/layer5_attention_weight_max": 120.00051879882812,
      "activations/layer5_attention_weight_min": -104.44471740722656,
      "activations/layer6_attention_weight_max": 61.5184326171875,
      "activations/layer6_attention_weight_min": -53.74891662597656,
      "activations/layer7_attention_weight_max": 62.65277862548828,
      "activations/layer7_attention_weight_min": -56.263309478759766,
      "activations/layer8_attention_weight_max": 39.77406311035156,
      "activations/layer8_attention_weight_min": -34.6282844543457,
      "activations/layer9_attention_weight_max": 53.68230056762695,
      "activations/layer9_attention_weight_min": -50.74995040893555,
      "epoch": 2.78,
      "learning_rate": 0.00013343143939393938,
      "loss": 2.9221,
      "step": 47800
    },
    {
      "activations/layer0_attention_weight_max": 13.948384284973145,
      "activations/layer0_attention_weight_min": -14.102670669555664,
      "activations/layer10_attention_weight_max": 47.6192626953125,
      "activations/layer10_attention_weight_min": -37.7876091003418,
      "activations/layer11_attention_weight_max": 37.58955001831055,
      "activations/layer11_attention_weight_min": -31.006603240966797,
      "activations/layer12_attention_weight_max": 28.832393646240234,
      "activations/layer12_attention_weight_min": -26.08514976501465,
      "activations/layer13_attention_weight_max": 57.56719207763672,
      "activations/layer13_attention_weight_min": -42.38839340209961,
      "activations/layer14_attention_weight_max": 52.47785949707031,
      "activations/layer14_attention_weight_min": -38.75715637207031,
      "activations/layer15_attention_weight_max": 51.601383209228516,
      "activations/layer15_attention_weight_min": -35.212581634521484,
      "activations/layer16_attention_weight_max": 40.88554000854492,
      "activations/layer16_attention_weight_min": -27.597366333007812,
      "activations/layer17_attention_weight_max": 66.66209411621094,
      "activations/layer17_attention_weight_min": -35.13772201538086,
      "activations/layer18_attention_weight_max": 56.208099365234375,
      "activations/layer18_attention_weight_min": -33.09772872924805,
      "activations/layer19_attention_weight_max": 24.75836753845215,
      "activations/layer19_attention_weight_min": -16.009048461914062,
      "activations/layer1_attention_weight_max": 17.441680908203125,
      "activations/layer1_attention_weight_min": -15.05250358581543,
      "activations/layer20_attention_weight_max": 21.658353805541992,
      "activations/layer20_attention_weight_min": -16.388456344604492,
      "activations/layer21_attention_weight_max": 41.22227096557617,
      "activations/layer21_attention_weight_min": -27.79510498046875,
      "activations/layer22_attention_weight_max": 41.60944366455078,
      "activations/layer22_attention_weight_min": -23.28311538696289,
      "activations/layer23_attention_weight_max": 26.286304473876953,
      "activations/layer23_attention_weight_min": -16.69510269165039,
      "activations/layer2_attention_weight_max": 29.83230972290039,
      "activations/layer2_attention_weight_min": -27.44780731201172,
      "activations/layer3_attention_weight_max": 72.36598205566406,
      "activations/layer3_attention_weight_min": -71.91592407226562,
      "activations/layer4_attention_weight_max": 104.56401824951172,
      "activations/layer4_attention_weight_min": -94.86000061035156,
      "activations/layer5_attention_weight_max": 134.55380249023438,
      "activations/layer5_attention_weight_min": -107.22078704833984,
      "activations/layer6_attention_weight_max": 67.87736511230469,
      "activations/layer6_attention_weight_min": -52.51482391357422,
      "activations/layer7_attention_weight_max": 62.72059631347656,
      "activations/layer7_attention_weight_min": -53.424110412597656,
      "activations/layer8_attention_weight_max": 44.41426467895508,
      "activations/layer8_attention_weight_min": -39.044105529785156,
      "activations/layer9_attention_weight_max": 61.9003791809082,
      "activations/layer9_attention_weight_min": -51.6976432800293,
      "epoch": 2.78,
      "learning_rate": 0.00013341249999999997,
      "loss": 2.9147,
      "step": 47850
    },
    {
      "activations/layer0_attention_weight_max": 15.985610008239746,
      "activations/layer0_attention_weight_min": -14.179224967956543,
      "activations/layer10_attention_weight_max": 47.512420654296875,
      "activations/layer10_attention_weight_min": -35.68294143676758,
      "activations/layer11_attention_weight_max": 36.309791564941406,
      "activations/layer11_attention_weight_min": -27.379444122314453,
      "activations/layer12_attention_weight_max": 29.22042465209961,
      "activations/layer12_attention_weight_min": -28.71933364868164,
      "activations/layer13_attention_weight_max": 64.1660385131836,
      "activations/layer13_attention_weight_min": -44.59685134887695,
      "activations/layer14_attention_weight_max": 68.77796936035156,
      "activations/layer14_attention_weight_min": -42.53669357299805,
      "activations/layer15_attention_weight_max": 45.420936584472656,
      "activations/layer15_attention_weight_min": -33.44670486450195,
      "activations/layer16_attention_weight_max": 42.42339324951172,
      "activations/layer16_attention_weight_min": -28.301984786987305,
      "activations/layer17_attention_weight_max": 59.50940704345703,
      "activations/layer17_attention_weight_min": -36.14537811279297,
      "activations/layer18_attention_weight_max": 56.3079948425293,
      "activations/layer18_attention_weight_min": -32.7765998840332,
      "activations/layer19_attention_weight_max": 20.613019943237305,
      "activations/layer19_attention_weight_min": -17.048154830932617,
      "activations/layer1_attention_weight_max": 17.94202423095703,
      "activations/layer1_attention_weight_min": -13.921561241149902,
      "activations/layer20_attention_weight_max": 23.825300216674805,
      "activations/layer20_attention_weight_min": -19.5472469329834,
      "activations/layer21_attention_weight_max": 45.42412567138672,
      "activations/layer21_attention_weight_min": -27.11861801147461,
      "activations/layer22_attention_weight_max": 39.0910530090332,
      "activations/layer22_attention_weight_min": -23.770122528076172,
      "activations/layer23_attention_weight_max": 24.753990173339844,
      "activations/layer23_attention_weight_min": -19.040267944335938,
      "activations/layer2_attention_weight_max": 30.847427368164062,
      "activations/layer2_attention_weight_min": -28.306381225585938,
      "activations/layer3_attention_weight_max": 71.82940673828125,
      "activations/layer3_attention_weight_min": -71.71070861816406,
      "activations/layer4_attention_weight_max": 105.56705474853516,
      "activations/layer4_attention_weight_min": -87.80561065673828,
      "activations/layer5_attention_weight_max": 131.24755859375,
      "activations/layer5_attention_weight_min": -98.94097900390625,
      "activations/layer6_attention_weight_max": 63.99352264404297,
      "activations/layer6_attention_weight_min": -50.618309020996094,
      "activations/layer7_attention_weight_max": 67.85405731201172,
      "activations/layer7_attention_weight_min": -57.1278076171875,
      "activations/layer8_attention_weight_max": 41.09074783325195,
      "activations/layer8_attention_weight_min": -34.97370147705078,
      "activations/layer9_attention_weight_max": 60.34017562866211,
      "activations/layer9_attention_weight_min": -54.709449768066406,
      "epoch": 2.78,
      "learning_rate": 0.00013339356060606058,
      "loss": 2.921,
      "step": 47900
    },
    {
      "activations/layer0_attention_weight_max": 14.761943817138672,
      "activations/layer0_attention_weight_min": -13.91954231262207,
      "activations/layer10_attention_weight_max": 50.99335479736328,
      "activations/layer10_attention_weight_min": -39.29539108276367,
      "activations/layer11_attention_weight_max": 37.712249755859375,
      "activations/layer11_attention_weight_min": -32.535118103027344,
      "activations/layer12_attention_weight_max": 36.5709228515625,
      "activations/layer12_attention_weight_min": -24.53731918334961,
      "activations/layer13_attention_weight_max": 53.40753936767578,
      "activations/layer13_attention_weight_min": -42.47801971435547,
      "activations/layer14_attention_weight_max": 45.55315017700195,
      "activations/layer14_attention_weight_min": -34.76549530029297,
      "activations/layer15_attention_weight_max": 45.97355270385742,
      "activations/layer15_attention_weight_min": -36.314430236816406,
      "activations/layer16_attention_weight_max": 37.8133544921875,
      "activations/layer16_attention_weight_min": -26.61711311340332,
      "activations/layer17_attention_weight_max": 62.30099105834961,
      "activations/layer17_attention_weight_min": -35.89533996582031,
      "activations/layer18_attention_weight_max": 51.49949264526367,
      "activations/layer18_attention_weight_min": -32.125946044921875,
      "activations/layer19_attention_weight_max": 21.843061447143555,
      "activations/layer19_attention_weight_min": -16.86091423034668,
      "activations/layer1_attention_weight_max": 17.62921142578125,
      "activations/layer1_attention_weight_min": -14.08633041381836,
      "activations/layer20_attention_weight_max": 20.37494468688965,
      "activations/layer20_attention_weight_min": -17.74803924560547,
      "activations/layer21_attention_weight_max": 47.2014045715332,
      "activations/layer21_attention_weight_min": -30.442081451416016,
      "activations/layer22_attention_weight_max": 33.23183059692383,
      "activations/layer22_attention_weight_min": -22.4327335357666,
      "activations/layer23_attention_weight_max": 29.331069946289062,
      "activations/layer23_attention_weight_min": -15.44281005859375,
      "activations/layer2_attention_weight_max": 27.727575302124023,
      "activations/layer2_attention_weight_min": -27.01788902282715,
      "activations/layer3_attention_weight_max": 68.27972412109375,
      "activations/layer3_attention_weight_min": -72.32206726074219,
      "activations/layer4_attention_weight_max": 96.4885025024414,
      "activations/layer4_attention_weight_min": -89.21411895751953,
      "activations/layer5_attention_weight_max": 117.80697631835938,
      "activations/layer5_attention_weight_min": -103.92866516113281,
      "activations/layer6_attention_weight_max": 62.64662170410156,
      "activations/layer6_attention_weight_min": -50.490623474121094,
      "activations/layer7_attention_weight_max": 59.68848419189453,
      "activations/layer7_attention_weight_min": -54.591209411621094,
      "activations/layer8_attention_weight_max": 41.13568878173828,
      "activations/layer8_attention_weight_min": -36.349220275878906,
      "activations/layer9_attention_weight_max": 53.83307647705078,
      "activations/layer9_attention_weight_min": -46.29973602294922,
      "epoch": 2.79,
      "learning_rate": 0.0001333746212121212,
      "loss": 2.9238,
      "step": 47950
    },
    {
      "activations/layer0_attention_weight_max": 13.645364761352539,
      "activations/layer0_attention_weight_min": -14.120492935180664,
      "activations/layer10_attention_weight_max": 46.1641960144043,
      "activations/layer10_attention_weight_min": -37.374576568603516,
      "activations/layer11_attention_weight_max": 37.663360595703125,
      "activations/layer11_attention_weight_min": -27.034717559814453,
      "activations/layer12_attention_weight_max": 25.93607521057129,
      "activations/layer12_attention_weight_min": -26.269060134887695,
      "activations/layer13_attention_weight_max": 51.377891540527344,
      "activations/layer13_attention_weight_min": -37.965450286865234,
      "activations/layer14_attention_weight_max": 47.43262481689453,
      "activations/layer14_attention_weight_min": -36.842342376708984,
      "activations/layer15_attention_weight_max": 55.35702133178711,
      "activations/layer15_attention_weight_min": -35.180694580078125,
      "activations/layer16_attention_weight_max": 38.1830940246582,
      "activations/layer16_attention_weight_min": -26.97768783569336,
      "activations/layer17_attention_weight_max": 59.55999755859375,
      "activations/layer17_attention_weight_min": -35.997005462646484,
      "activations/layer18_attention_weight_max": 56.34101486206055,
      "activations/layer18_attention_weight_min": -32.53135299682617,
      "activations/layer19_attention_weight_max": 22.717702865600586,
      "activations/layer19_attention_weight_min": -16.615415573120117,
      "activations/layer1_attention_weight_max": 17.654409408569336,
      "activations/layer1_attention_weight_min": -15.226696014404297,
      "activations/layer20_attention_weight_max": 21.803428649902344,
      "activations/layer20_attention_weight_min": -19.6207218170166,
      "activations/layer21_attention_weight_max": 43.513641357421875,
      "activations/layer21_attention_weight_min": -30.425146102905273,
      "activations/layer22_attention_weight_max": 36.97590637207031,
      "activations/layer22_attention_weight_min": -24.27305030822754,
      "activations/layer23_attention_weight_max": 25.53034782409668,
      "activations/layer23_attention_weight_min": -15.870899200439453,
      "activations/layer2_attention_weight_max": 29.47138023376465,
      "activations/layer2_attention_weight_min": -25.994873046875,
      "activations/layer3_attention_weight_max": 68.27338409423828,
      "activations/layer3_attention_weight_min": -72.83797454833984,
      "activations/layer4_attention_weight_max": 97.4024887084961,
      "activations/layer4_attention_weight_min": -88.6924819946289,
      "activations/layer5_attention_weight_max": 121.85166931152344,
      "activations/layer5_attention_weight_min": -103.57662200927734,
      "activations/layer6_attention_weight_max": 63.3977165222168,
      "activations/layer6_attention_weight_min": -54.294002532958984,
      "activations/layer7_attention_weight_max": 58.77180099487305,
      "activations/layer7_attention_weight_min": -55.263301849365234,
      "activations/layer8_attention_weight_max": 41.7764892578125,
      "activations/layer8_attention_weight_min": -35.486297607421875,
      "activations/layer9_attention_weight_max": 57.77020263671875,
      "activations/layer9_attention_weight_min": -47.18845748901367,
      "epoch": 2.79,
      "learning_rate": 0.00013335568181818182,
      "loss": 2.9162,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_loss": 2.869140625,
      "eval_runtime": 8.4092,
      "eval_samples_per_second": 510.629,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_openwebtext_loss": 2.869140625,
      "eval_openwebtext_ppl": 17.6218678981495,
      "eval_openwebtext_runtime": 8.4092,
      "eval_openwebtext_samples_per_second": 510.629,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_wikitext_loss": 3.115234375,
      "eval_wikitext_ppl": 22.538712245181248,
      "eval_wikitext_runtime": 1.954,
      "eval_wikitext_samples_per_second": 233.372,
      "step": 48000
    },
    {
      "epoch": 2.79,
      "eval_lambada_loss": 3.013671875,
      "eval_lambada_ppl": 20.36202965350754,
      "eval_lambada_runtime": 9.5326,
      "eval_lambada_samples_per_second": 510.774,
      "step": 48000
    },
    {
      "activations/layer0_attention_weight_max": 15.962884902954102,
      "activations/layer0_attention_weight_min": -14.371111869812012,
      "activations/layer10_attention_weight_max": 45.19816589355469,
      "activations/layer10_attention_weight_min": -38.455963134765625,
      "activations/layer11_attention_weight_max": 38.295413970947266,
      "activations/layer11_attention_weight_min": -30.03676986694336,
      "activations/layer12_attention_weight_max": 25.299076080322266,
      "activations/layer12_attention_weight_min": -26.255084991455078,
      "activations/layer13_attention_weight_max": 56.832401275634766,
      "activations/layer13_attention_weight_min": -36.16781997680664,
      "activations/layer14_attention_weight_max": 65.35482788085938,
      "activations/layer14_attention_weight_min": -39.90229034423828,
      "activations/layer15_attention_weight_max": 52.18512725830078,
      "activations/layer15_attention_weight_min": -33.51099395751953,
      "activations/layer16_attention_weight_max": 40.217124938964844,
      "activations/layer16_attention_weight_min": -25.605268478393555,
      "activations/layer17_attention_weight_max": 63.55293273925781,
      "activations/layer17_attention_weight_min": -33.237220764160156,
      "activations/layer18_attention_weight_max": 53.55559158325195,
      "activations/layer18_attention_weight_min": -31.838472366333008,
      "activations/layer19_attention_weight_max": 21.031761169433594,
      "activations/layer19_attention_weight_min": -17.900617599487305,
      "activations/layer1_attention_weight_max": 17.415325164794922,
      "activations/layer1_attention_weight_min": -13.348685264587402,
      "activations/layer20_attention_weight_max": 19.782930374145508,
      "activations/layer20_attention_weight_min": -19.08807373046875,
      "activations/layer21_attention_weight_max": 46.90732955932617,
      "activations/layer21_attention_weight_min": -29.032329559326172,
      "activations/layer22_attention_weight_max": 38.04759216308594,
      "activations/layer22_attention_weight_min": -21.821401596069336,
      "activations/layer23_attention_weight_max": 24.05158805847168,
      "activations/layer23_attention_weight_min": -18.28412437438965,
      "activations/layer2_attention_weight_max": 28.46384048461914,
      "activations/layer2_attention_weight_min": -27.22736358642578,
      "activations/layer3_attention_weight_max": 69.8028793334961,
      "activations/layer3_attention_weight_min": -74.59355926513672,
      "activations/layer4_attention_weight_max": 100.8140640258789,
      "activations/layer4_attention_weight_min": -102.44412994384766,
      "activations/layer5_attention_weight_max": 122.5462875366211,
      "activations/layer5_attention_weight_min": -111.44014739990234,
      "activations/layer6_attention_weight_max": 65.1849594116211,
      "activations/layer6_attention_weight_min": -51.0586051940918,
      "activations/layer7_attention_weight_max": 61.26932907104492,
      "activations/layer7_attention_weight_min": -57.38169479370117,
      "activations/layer8_attention_weight_max": 40.32127380371094,
      "activations/layer8_attention_weight_min": -35.90883255004883,
      "activations/layer9_attention_weight_max": 58.92041015625,
      "activations/layer9_attention_weight_min": -52.47797775268555,
      "epoch": 2.79,
      "learning_rate": 0.0001333367424242424,
      "loss": 2.9279,
      "step": 48050
    },
    {
      "activations/layer0_attention_weight_max": 15.27532958984375,
      "activations/layer0_attention_weight_min": -14.733309745788574,
      "activations/layer10_attention_weight_max": 46.2304573059082,
      "activations/layer10_attention_weight_min": -36.504730224609375,
      "activations/layer11_attention_weight_max": 37.79892349243164,
      "activations/layer11_attention_weight_min": -29.854841232299805,
      "activations/layer12_attention_weight_max": 27.070419311523438,
      "activations/layer12_attention_weight_min": -27.152694702148438,
      "activations/layer13_attention_weight_max": 59.80425262451172,
      "activations/layer13_attention_weight_min": -40.38142776489258,
      "activations/layer14_attention_weight_max": 56.98672866821289,
      "activations/layer14_attention_weight_min": -36.959232330322266,
      "activations/layer15_attention_weight_max": 45.26123046875,
      "activations/layer15_attention_weight_min": -33.20587921142578,
      "activations/layer16_attention_weight_max": 44.7149658203125,
      "activations/layer16_attention_weight_min": -27.32354736328125,
      "activations/layer17_attention_weight_max": 60.4489631652832,
      "activations/layer17_attention_weight_min": -39.10646057128906,
      "activations/layer18_attention_weight_max": 54.70421600341797,
      "activations/layer18_attention_weight_min": -32.948387145996094,
      "activations/layer19_attention_weight_max": 23.024202346801758,
      "activations/layer19_attention_weight_min": -18.478967666625977,
      "activations/layer1_attention_weight_max": 17.56012725830078,
      "activations/layer1_attention_weight_min": -15.570974349975586,
      "activations/layer20_attention_weight_max": 19.33015251159668,
      "activations/layer20_attention_weight_min": -19.683935165405273,
      "activations/layer21_attention_weight_max": 43.555084228515625,
      "activations/layer21_attention_weight_min": -29.966440200805664,
      "activations/layer22_attention_weight_max": 34.784507751464844,
      "activations/layer22_attention_weight_min": -24.154497146606445,
      "activations/layer23_attention_weight_max": 24.20412254333496,
      "activations/layer23_attention_weight_min": -15.627924919128418,
      "activations/layer2_attention_weight_max": 29.049360275268555,
      "activations/layer2_attention_weight_min": -27.893651962280273,
      "activations/layer3_attention_weight_max": 71.56099700927734,
      "activations/layer3_attention_weight_min": -72.28629302978516,
      "activations/layer4_attention_weight_max": 103.58491516113281,
      "activations/layer4_attention_weight_min": -91.08837890625,
      "activations/layer5_attention_weight_max": 124.50970458984375,
      "activations/layer5_attention_weight_min": -102.3222885131836,
      "activations/layer6_attention_weight_max": 60.56094741821289,
      "activations/layer6_attention_weight_min": -50.971946716308594,
      "activations/layer7_attention_weight_max": 60.4064826965332,
      "activations/layer7_attention_weight_min": -53.20692825317383,
      "activations/layer8_attention_weight_max": 39.416481018066406,
      "activations/layer8_attention_weight_min": -34.866825103759766,
      "activations/layer9_attention_weight_max": 56.0363883972168,
      "activations/layer9_attention_weight_min": -48.70208740234375,
      "epoch": 2.79,
      "learning_rate": 0.00013331780303030302,
      "loss": 2.9231,
      "step": 48100
    },
    {
      "activations/layer0_attention_weight_max": 14.395685195922852,
      "activations/layer0_attention_weight_min": -14.307417869567871,
      "activations/layer10_attention_weight_max": 57.63962173461914,
      "activations/layer10_attention_weight_min": -44.793060302734375,
      "activations/layer11_attention_weight_max": 40.110557556152344,
      "activations/layer11_attention_weight_min": -28.11871910095215,
      "activations/layer12_attention_weight_max": 24.35034942626953,
      "activations/layer12_attention_weight_min": -31.072961807250977,
      "activations/layer13_attention_weight_max": 59.81763458251953,
      "activations/layer13_attention_weight_min": -42.58641052246094,
      "activations/layer14_attention_weight_max": 54.45656967163086,
      "activations/layer14_attention_weight_min": -35.4495849609375,
      "activations/layer15_attention_weight_max": 49.717952728271484,
      "activations/layer15_attention_weight_min": -34.29724884033203,
      "activations/layer16_attention_weight_max": 38.940250396728516,
      "activations/layer16_attention_weight_min": -27.06355094909668,
      "activations/layer17_attention_weight_max": 56.60731887817383,
      "activations/layer17_attention_weight_min": -35.770870208740234,
      "activations/layer18_attention_weight_max": 51.162479400634766,
      "activations/layer18_attention_weight_min": -33.65290451049805,
      "activations/layer19_attention_weight_max": 21.931535720825195,
      "activations/layer19_attention_weight_min": -18.9514217376709,
      "activations/layer1_attention_weight_max": 16.682477951049805,
      "activations/layer1_attention_weight_min": -13.753148078918457,
      "activations/layer20_attention_weight_max": 20.87645149230957,
      "activations/layer20_attention_weight_min": -17.87334442138672,
      "activations/layer21_attention_weight_max": 42.81306838989258,
      "activations/layer21_attention_weight_min": -26.45610237121582,
      "activations/layer22_attention_weight_max": 35.05298614501953,
      "activations/layer22_attention_weight_min": -22.73310661315918,
      "activations/layer23_attention_weight_max": 25.047603607177734,
      "activations/layer23_attention_weight_min": -16.155517578125,
      "activations/layer2_attention_weight_max": 28.499792098999023,
      "activations/layer2_attention_weight_min": -26.419368743896484,
      "activations/layer3_attention_weight_max": 70.3001937866211,
      "activations/layer3_attention_weight_min": -70.94488525390625,
      "activations/layer4_attention_weight_max": 101.48816680908203,
      "activations/layer4_attention_weight_min": -89.27141571044922,
      "activations/layer5_attention_weight_max": 128.77590942382812,
      "activations/layer5_attention_weight_min": -99.63766479492188,
      "activations/layer6_attention_weight_max": 61.46967697143555,
      "activations/layer6_attention_weight_min": -50.95475387573242,
      "activations/layer7_attention_weight_max": 63.03849792480469,
      "activations/layer7_attention_weight_min": -54.847442626953125,
      "activations/layer8_attention_weight_max": 40.88119888305664,
      "activations/layer8_attention_weight_min": -34.968414306640625,
      "activations/layer9_attention_weight_max": 53.15985870361328,
      "activations/layer9_attention_weight_min": -48.069881439208984,
      "epoch": 2.8,
      "learning_rate": 0.00013329886363636363,
      "loss": 2.9024,
      "step": 48150
    },
    {
      "activations/layer0_attention_weight_max": 15.11011028289795,
      "activations/layer0_attention_weight_min": -14.59825325012207,
      "activations/layer10_attention_weight_max": 47.62943649291992,
      "activations/layer10_attention_weight_min": -35.78300476074219,
      "activations/layer11_attention_weight_max": 41.672027587890625,
      "activations/layer11_attention_weight_min": -28.022253036499023,
      "activations/layer12_attention_weight_max": 23.700077056884766,
      "activations/layer12_attention_weight_min": -26.18375587463379,
      "activations/layer13_attention_weight_max": 50.85517883300781,
      "activations/layer13_attention_weight_min": -36.35295867919922,
      "activations/layer14_attention_weight_max": 44.81461715698242,
      "activations/layer14_attention_weight_min": -29.6923828125,
      "activations/layer15_attention_weight_max": 51.17929458618164,
      "activations/layer15_attention_weight_min": -32.20759963989258,
      "activations/layer16_attention_weight_max": 40.651981353759766,
      "activations/layer16_attention_weight_min": -25.46114158630371,
      "activations/layer17_attention_weight_max": 60.52017593383789,
      "activations/layer17_attention_weight_min": -34.72383499145508,
      "activations/layer18_attention_weight_max": 52.10089111328125,
      "activations/layer18_attention_weight_min": -30.12383460998535,
      "activations/layer19_attention_weight_max": 20.237783432006836,
      "activations/layer19_attention_weight_min": -16.100261688232422,
      "activations/layer1_attention_weight_max": 18.201902389526367,
      "activations/layer1_attention_weight_min": -13.366355895996094,
      "activations/layer20_attention_weight_max": 18.658615112304688,
      "activations/layer20_attention_weight_min": -19.306135177612305,
      "activations/layer21_attention_weight_max": 41.20375442504883,
      "activations/layer21_attention_weight_min": -26.301528930664062,
      "activations/layer22_attention_weight_max": 33.15011978149414,
      "activations/layer22_attention_weight_min": -19.768230438232422,
      "activations/layer23_attention_weight_max": 22.03890037536621,
      "activations/layer23_attention_weight_min": -16.16318130493164,
      "activations/layer2_attention_weight_max": 28.963550567626953,
      "activations/layer2_attention_weight_min": -27.736492156982422,
      "activations/layer3_attention_weight_max": 66.86117553710938,
      "activations/layer3_attention_weight_min": -72.10591125488281,
      "activations/layer4_attention_weight_max": 97.97574615478516,
      "activations/layer4_attention_weight_min": -89.82405853271484,
      "activations/layer5_attention_weight_max": 131.6002197265625,
      "activations/layer5_attention_weight_min": -99.52974700927734,
      "activations/layer6_attention_weight_max": 68.573974609375,
      "activations/layer6_attention_weight_min": -49.73011779785156,
      "activations/layer7_attention_weight_max": 60.169193267822266,
      "activations/layer7_attention_weight_min": -53.494625091552734,
      "activations/layer8_attention_weight_max": 43.22456741333008,
      "activations/layer8_attention_weight_min": -35.50908279418945,
      "activations/layer9_attention_weight_max": 53.02882766723633,
      "activations/layer9_attention_weight_min": -46.945926666259766,
      "epoch": 2.8,
      "learning_rate": 0.00013327992424242425,
      "loss": 2.9136,
      "step": 48200
    },
    {
      "activations/layer0_attention_weight_max": 14.792783737182617,
      "activations/layer0_attention_weight_min": -14.568968772888184,
      "activations/layer10_attention_weight_max": 50.76301193237305,
      "activations/layer10_attention_weight_min": -41.61176681518555,
      "activations/layer11_attention_weight_max": 43.51597595214844,
      "activations/layer11_attention_weight_min": -31.779361724853516,
      "activations/layer12_attention_weight_max": 29.42961311340332,
      "activations/layer12_attention_weight_min": -23.343521118164062,
      "activations/layer13_attention_weight_max": 63.414127349853516,
      "activations/layer13_attention_weight_min": -40.69566345214844,
      "activations/layer14_attention_weight_max": 57.955692291259766,
      "activations/layer14_attention_weight_min": -45.87351989746094,
      "activations/layer15_attention_weight_max": 58.20155715942383,
      "activations/layer15_attention_weight_min": -35.352298736572266,
      "activations/layer16_attention_weight_max": 44.95648193359375,
      "activations/layer16_attention_weight_min": -27.015331268310547,
      "activations/layer17_attention_weight_max": 70.24087524414062,
      "activations/layer17_attention_weight_min": -36.712982177734375,
      "activations/layer18_attention_weight_max": 73.17889404296875,
      "activations/layer18_attention_weight_min": -32.52827835083008,
      "activations/layer19_attention_weight_max": 23.400226593017578,
      "activations/layer19_attention_weight_min": -16.808378219604492,
      "activations/layer1_attention_weight_max": 17.291458129882812,
      "activations/layer1_attention_weight_min": -16.837562561035156,
      "activations/layer20_attention_weight_max": 23.551311492919922,
      "activations/layer20_attention_weight_min": -19.680105209350586,
      "activations/layer21_attention_weight_max": 45.83985137939453,
      "activations/layer21_attention_weight_min": -27.814115524291992,
      "activations/layer22_attention_weight_max": 37.83802795410156,
      "activations/layer22_attention_weight_min": -25.21381950378418,
      "activations/layer23_attention_weight_max": 28.286588668823242,
      "activations/layer23_attention_weight_min": -16.57947540283203,
      "activations/layer2_attention_weight_max": 29.871417999267578,
      "activations/layer2_attention_weight_min": -27.66085433959961,
      "activations/layer3_attention_weight_max": 70.62525939941406,
      "activations/layer3_attention_weight_min": -72.45857238769531,
      "activations/layer4_attention_weight_max": 102.49874114990234,
      "activations/layer4_attention_weight_min": -93.01954650878906,
      "activations/layer5_attention_weight_max": 122.60127258300781,
      "activations/layer5_attention_weight_min": -102.56767272949219,
      "activations/layer6_attention_weight_max": 60.77960968017578,
      "activations/layer6_attention_weight_min": -53.458927154541016,
      "activations/layer7_attention_weight_max": 57.332096099853516,
      "activations/layer7_attention_weight_min": -53.140281677246094,
      "activations/layer8_attention_weight_max": 40.53703689575195,
      "activations/layer8_attention_weight_min": -37.391258239746094,
      "activations/layer9_attention_weight_max": 52.11520004272461,
      "activations/layer9_attention_weight_min": -51.66592025756836,
      "epoch": 2.8,
      "learning_rate": 0.00013326098484848484,
      "loss": 2.9386,
      "step": 48250
    },
    {
      "activations/layer0_attention_weight_max": 16.090179443359375,
      "activations/layer0_attention_weight_min": -14.387025833129883,
      "activations/layer10_attention_weight_max": 45.92927932739258,
      "activations/layer10_attention_weight_min": -37.146636962890625,
      "activations/layer11_attention_weight_max": 41.703163146972656,
      "activations/layer11_attention_weight_min": -28.237308502197266,
      "activations/layer12_attention_weight_max": 35.00068283081055,
      "activations/layer12_attention_weight_min": -27.179710388183594,
      "activations/layer13_attention_weight_max": 58.48250198364258,
      "activations/layer13_attention_weight_min": -45.29142761230469,
      "activations/layer14_attention_weight_max": 62.066097259521484,
      "activations/layer14_attention_weight_min": -45.32136917114258,
      "activations/layer15_attention_weight_max": 49.01797103881836,
      "activations/layer15_attention_weight_min": -39.074371337890625,
      "activations/layer16_attention_weight_max": 38.56655502319336,
      "activations/layer16_attention_weight_min": -28.520816802978516,
      "activations/layer17_attention_weight_max": 62.71105194091797,
      "activations/layer17_attention_weight_min": -37.743507385253906,
      "activations/layer18_attention_weight_max": 49.182830810546875,
      "activations/layer18_attention_weight_min": -31.30120849609375,
      "activations/layer19_attention_weight_max": 21.313833236694336,
      "activations/layer19_attention_weight_min": -16.070371627807617,
      "activations/layer1_attention_weight_max": 18.24331283569336,
      "activations/layer1_attention_weight_min": -13.035634994506836,
      "activations/layer20_attention_weight_max": 19.643436431884766,
      "activations/layer20_attention_weight_min": -20.354402542114258,
      "activations/layer21_attention_weight_max": 38.88458251953125,
      "activations/layer21_attention_weight_min": -28.064054489135742,
      "activations/layer22_attention_weight_max": 36.59725570678711,
      "activations/layer22_attention_weight_min": -23.279260635375977,
      "activations/layer23_attention_weight_max": 21.508075714111328,
      "activations/layer23_attention_weight_min": -17.38726043701172,
      "activations/layer2_attention_weight_max": 31.37087631225586,
      "activations/layer2_attention_weight_min": -29.254173278808594,
      "activations/layer3_attention_weight_max": 81.06256103515625,
      "activations/layer3_attention_weight_min": -81.53157806396484,
      "activations/layer4_attention_weight_max": 112.2394027709961,
      "activations/layer4_attention_weight_min": -94.65204620361328,
      "activations/layer5_attention_weight_max": 133.74844360351562,
      "activations/layer5_attention_weight_min": -104.27864074707031,
      "activations/layer6_attention_weight_max": 62.605987548828125,
      "activations/layer6_attention_weight_min": -52.054195404052734,
      "activations/layer7_attention_weight_max": 61.14414978027344,
      "activations/layer7_attention_weight_min": -59.00945281982422,
      "activations/layer8_attention_weight_max": 40.54560089111328,
      "activations/layer8_attention_weight_min": -39.53865051269531,
      "activations/layer9_attention_weight_max": 55.536476135253906,
      "activations/layer9_attention_weight_min": -57.935028076171875,
      "epoch": 2.81,
      "learning_rate": 0.00013324204545454545,
      "loss": 2.9173,
      "step": 48300
    },
    {
      "activations/layer0_attention_weight_max": 13.723447799682617,
      "activations/layer0_attention_weight_min": -14.49852466583252,
      "activations/layer10_attention_weight_max": 50.59866714477539,
      "activations/layer10_attention_weight_min": -38.132564544677734,
      "activations/layer11_attention_weight_max": 39.06890869140625,
      "activations/layer11_attention_weight_min": -29.824758529663086,
      "activations/layer12_attention_weight_max": 28.11597442626953,
      "activations/layer12_attention_weight_min": -25.800600051879883,
      "activations/layer13_attention_weight_max": 56.22123718261719,
      "activations/layer13_attention_weight_min": -40.36809539794922,
      "activations/layer14_attention_weight_max": 51.89704895019531,
      "activations/layer14_attention_weight_min": -34.622703552246094,
      "activations/layer15_attention_weight_max": 50.01914596557617,
      "activations/layer15_attention_weight_min": -32.64811325073242,
      "activations/layer16_attention_weight_max": 41.30203628540039,
      "activations/layer16_attention_weight_min": -26.210050582885742,
      "activations/layer17_attention_weight_max": 62.73577880859375,
      "activations/layer17_attention_weight_min": -35.69414138793945,
      "activations/layer18_attention_weight_max": 54.30863952636719,
      "activations/layer18_attention_weight_min": -31.683561325073242,
      "activations/layer19_attention_weight_max": 22.952543258666992,
      "activations/layer19_attention_weight_min": -17.941015243530273,
      "activations/layer1_attention_weight_max": 17.87270736694336,
      "activations/layer1_attention_weight_min": -15.47231674194336,
      "activations/layer20_attention_weight_max": 20.819665908813477,
      "activations/layer20_attention_weight_min": -22.603839874267578,
      "activations/layer21_attention_weight_max": 47.982933044433594,
      "activations/layer21_attention_weight_min": -29.47385597229004,
      "activations/layer22_attention_weight_max": 40.078521728515625,
      "activations/layer22_attention_weight_min": -22.213024139404297,
      "activations/layer23_attention_weight_max": 26.100881576538086,
      "activations/layer23_attention_weight_min": -13.740504264831543,
      "activations/layer2_attention_weight_max": 30.937850952148438,
      "activations/layer2_attention_weight_min": -27.851760864257812,
      "activations/layer3_attention_weight_max": 70.52247619628906,
      "activations/layer3_attention_weight_min": -71.51205444335938,
      "activations/layer4_attention_weight_max": 98.0649185180664,
      "activations/layer4_attention_weight_min": -87.93065643310547,
      "activations/layer5_attention_weight_max": 123.970703125,
      "activations/layer5_attention_weight_min": -98.31021118164062,
      "activations/layer6_attention_weight_max": 62.960304260253906,
      "activations/layer6_attention_weight_min": -48.980045318603516,
      "activations/layer7_attention_weight_max": 57.235801696777344,
      "activations/layer7_attention_weight_min": -52.59833908081055,
      "activations/layer8_attention_weight_max": 45.414329528808594,
      "activations/layer8_attention_weight_min": -40.4683723449707,
      "activations/layer9_attention_weight_max": 61.42312240600586,
      "activations/layer9_attention_weight_min": -52.466609954833984,
      "epoch": 2.81,
      "learning_rate": 0.00013322310606060607,
      "loss": 2.9108,
      "step": 48350
    },
    {
      "activations/layer0_attention_weight_max": 13.587233543395996,
      "activations/layer0_attention_weight_min": -14.568224906921387,
      "activations/layer10_attention_weight_max": 53.549800872802734,
      "activations/layer10_attention_weight_min": -36.86768341064453,
      "activations/layer11_attention_weight_max": 38.981712341308594,
      "activations/layer11_attention_weight_min": -28.037704467773438,
      "activations/layer12_attention_weight_max": 51.36452865600586,
      "activations/layer12_attention_weight_min": -27.090314865112305,
      "activations/layer13_attention_weight_max": 61.3624153137207,
      "activations/layer13_attention_weight_min": -41.245147705078125,
      "activations/layer14_attention_weight_max": 67.6806869506836,
      "activations/layer14_attention_weight_min": -42.75917434692383,
      "activations/layer15_attention_weight_max": 52.42140579223633,
      "activations/layer15_attention_weight_min": -32.81852722167969,
      "activations/layer16_attention_weight_max": 41.332313537597656,
      "activations/layer16_attention_weight_min": -27.94826316833496,
      "activations/layer17_attention_weight_max": 66.73898315429688,
      "activations/layer17_attention_weight_min": -36.980499267578125,
      "activations/layer18_attention_weight_max": 54.97471237182617,
      "activations/layer18_attention_weight_min": -32.52108383178711,
      "activations/layer19_attention_weight_max": 25.914884567260742,
      "activations/layer19_attention_weight_min": -21.67504119873047,
      "activations/layer1_attention_weight_max": 17.74024772644043,
      "activations/layer1_attention_weight_min": -14.16087818145752,
      "activations/layer20_attention_weight_max": 24.575000762939453,
      "activations/layer20_attention_weight_min": -19.700307846069336,
      "activations/layer21_attention_weight_max": 48.24388122558594,
      "activations/layer21_attention_weight_min": -27.18798065185547,
      "activations/layer22_attention_weight_max": 38.74057388305664,
      "activations/layer22_attention_weight_min": -22.88567543029785,
      "activations/layer23_attention_weight_max": 30.634353637695312,
      "activations/layer23_attention_weight_min": -18.541242599487305,
      "activations/layer2_attention_weight_max": 29.5589656829834,
      "activations/layer2_attention_weight_min": -27.7022762298584,
      "activations/layer3_attention_weight_max": 71.0455322265625,
      "activations/layer3_attention_weight_min": -72.9507827758789,
      "activations/layer4_attention_weight_max": 98.4415512084961,
      "activations/layer4_attention_weight_min": -89.54358673095703,
      "activations/layer5_attention_weight_max": 122.64091491699219,
      "activations/layer5_attention_weight_min": -101.7120361328125,
      "activations/layer6_attention_weight_max": 61.044803619384766,
      "activations/layer6_attention_weight_min": -52.4648551940918,
      "activations/layer7_attention_weight_max": 56.38172912597656,
      "activations/layer7_attention_weight_min": -53.56171798706055,
      "activations/layer8_attention_weight_max": 43.54904556274414,
      "activations/layer8_attention_weight_min": -36.19413375854492,
      "activations/layer9_attention_weight_max": 54.02580642700195,
      "activations/layer9_attention_weight_min": -48.9571533203125,
      "epoch": 2.81,
      "learning_rate": 0.00013320416666666665,
      "loss": 2.9198,
      "step": 48400
    },
    {
      "activations/layer0_attention_weight_max": 14.72977352142334,
      "activations/layer0_attention_weight_min": -14.24817943572998,
      "activations/layer10_attention_weight_max": 49.122222900390625,
      "activations/layer10_attention_weight_min": -37.87623977661133,
      "activations/layer11_attention_weight_max": 47.68044662475586,
      "activations/layer11_attention_weight_min": -30.183576583862305,
      "activations/layer12_attention_weight_max": 27.535978317260742,
      "activations/layer12_attention_weight_min": -31.93464469909668,
      "activations/layer13_attention_weight_max": 52.141353607177734,
      "activations/layer13_attention_weight_min": -38.71138000488281,
      "activations/layer14_attention_weight_max": 75.24464416503906,
      "activations/layer14_attention_weight_min": -47.69379425048828,
      "activations/layer15_attention_weight_max": 57.91972732543945,
      "activations/layer15_attention_weight_min": -37.565284729003906,
      "activations/layer16_attention_weight_max": 42.72930145263672,
      "activations/layer16_attention_weight_min": -28.343679428100586,
      "activations/layer17_attention_weight_max": 66.40145874023438,
      "activations/layer17_attention_weight_min": -37.84840774536133,
      "activations/layer18_attention_weight_max": 65.08866119384766,
      "activations/layer18_attention_weight_min": -31.98855209350586,
      "activations/layer19_attention_weight_max": 20.326597213745117,
      "activations/layer19_attention_weight_min": -17.2391414642334,
      "activations/layer1_attention_weight_max": 20.390655517578125,
      "activations/layer1_attention_weight_min": -12.380757331848145,
      "activations/layer20_attention_weight_max": 21.194822311401367,
      "activations/layer20_attention_weight_min": -18.22158432006836,
      "activations/layer21_attention_weight_max": 40.92937469482422,
      "activations/layer21_attention_weight_min": -29.02520751953125,
      "activations/layer22_attention_weight_max": 35.58045196533203,
      "activations/layer22_attention_weight_min": -22.792037963867188,
      "activations/layer23_attention_weight_max": 25.15134048461914,
      "activations/layer23_attention_weight_min": -15.875472068786621,
      "activations/layer2_attention_weight_max": 28.240921020507812,
      "activations/layer2_attention_weight_min": -27.871829986572266,
      "activations/layer3_attention_weight_max": 67.70358276367188,
      "activations/layer3_attention_weight_min": -72.31734466552734,
      "activations/layer4_attention_weight_max": 100.41043853759766,
      "activations/layer4_attention_weight_min": -88.5012435913086,
      "activations/layer5_attention_weight_max": 125.02281951904297,
      "activations/layer5_attention_weight_min": -97.1488037109375,
      "activations/layer6_attention_weight_max": 64.797119140625,
      "activations/layer6_attention_weight_min": -49.87154006958008,
      "activations/layer7_attention_weight_max": 57.663291931152344,
      "activations/layer7_attention_weight_min": -52.092655181884766,
      "activations/layer8_attention_weight_max": 41.34962463378906,
      "activations/layer8_attention_weight_min": -35.28522872924805,
      "activations/layer9_attention_weight_max": 58.919837951660156,
      "activations/layer9_attention_weight_min": -52.42391586303711,
      "epoch": 2.82,
      "learning_rate": 0.00013318522727272727,
      "loss": 2.9332,
      "step": 48450
    },
    {
      "activations/layer0_attention_weight_max": 15.733672142028809,
      "activations/layer0_attention_weight_min": -13.974124908447266,
      "activations/layer10_attention_weight_max": 50.0711669921875,
      "activations/layer10_attention_weight_min": -38.15950393676758,
      "activations/layer11_attention_weight_max": 43.322364807128906,
      "activations/layer11_attention_weight_min": -29.983577728271484,
      "activations/layer12_attention_weight_max": 27.83028793334961,
      "activations/layer12_attention_weight_min": -26.380517959594727,
      "activations/layer13_attention_weight_max": 54.294090270996094,
      "activations/layer13_attention_weight_min": -35.611351013183594,
      "activations/layer14_attention_weight_max": 54.786529541015625,
      "activations/layer14_attention_weight_min": -33.55131530761719,
      "activations/layer15_attention_weight_max": 47.214500427246094,
      "activations/layer15_attention_weight_min": -33.05172348022461,
      "activations/layer16_attention_weight_max": 40.062957763671875,
      "activations/layer16_attention_weight_min": -26.598037719726562,
      "activations/layer17_attention_weight_max": 60.40489959716797,
      "activations/layer17_attention_weight_min": -35.28248977661133,
      "activations/layer18_attention_weight_max": 53.09464645385742,
      "activations/layer18_attention_weight_min": -31.719051361083984,
      "activations/layer19_attention_weight_max": 23.916893005371094,
      "activations/layer19_attention_weight_min": -17.56553840637207,
      "activations/layer1_attention_weight_max": 17.352388381958008,
      "activations/layer1_attention_weight_min": -16.29094696044922,
      "activations/layer20_attention_weight_max": 18.39480972290039,
      "activations/layer20_attention_weight_min": -18.640104293823242,
      "activations/layer21_attention_weight_max": 45.44962692260742,
      "activations/layer21_attention_weight_min": -27.544628143310547,
      "activations/layer22_attention_weight_max": 31.51237678527832,
      "activations/layer22_attention_weight_min": -23.620656967163086,
      "activations/layer23_attention_weight_max": 21.27204704284668,
      "activations/layer23_attention_weight_min": -17.108043670654297,
      "activations/layer2_attention_weight_max": 29.43198013305664,
      "activations/layer2_attention_weight_min": -27.86450958251953,
      "activations/layer3_attention_weight_max": 71.41281127929688,
      "activations/layer3_attention_weight_min": -77.24932098388672,
      "activations/layer4_attention_weight_max": 100.97273254394531,
      "activations/layer4_attention_weight_min": -94.76612854003906,
      "activations/layer5_attention_weight_max": 119.16988372802734,
      "activations/layer5_attention_weight_min": -102.94307708740234,
      "activations/layer6_attention_weight_max": 61.94608688354492,
      "activations/layer6_attention_weight_min": -51.196048736572266,
      "activations/layer7_attention_weight_max": 69.46855163574219,
      "activations/layer7_attention_weight_min": -59.93156814575195,
      "activations/layer8_attention_weight_max": 48.480934143066406,
      "activations/layer8_attention_weight_min": -35.07255172729492,
      "activations/layer9_attention_weight_max": 70.92711639404297,
      "activations/layer9_attention_weight_min": -60.665740966796875,
      "epoch": 2.82,
      "learning_rate": 0.00013316666666666666,
      "loss": 2.9329,
      "step": 48500
    },
    {
      "activations/layer0_attention_weight_max": 14.774250984191895,
      "activations/layer0_attention_weight_min": -14.264864921569824,
      "activations/layer10_attention_weight_max": 46.08745193481445,
      "activations/layer10_attention_weight_min": -38.64372634887695,
      "activations/layer11_attention_weight_max": 36.11594009399414,
      "activations/layer11_attention_weight_min": -28.531423568725586,
      "activations/layer12_attention_weight_max": 22.873899459838867,
      "activations/layer12_attention_weight_min": -24.960357666015625,
      "activations/layer13_attention_weight_max": 58.359779357910156,
      "activations/layer13_attention_weight_min": -40.39087677001953,
      "activations/layer14_attention_weight_max": 54.01245880126953,
      "activations/layer14_attention_weight_min": -36.879066467285156,
      "activations/layer15_attention_weight_max": 51.237335205078125,
      "activations/layer15_attention_weight_min": -36.091800689697266,
      "activations/layer16_attention_weight_max": 40.613487243652344,
      "activations/layer16_attention_weight_min": -28.223522186279297,
      "activations/layer17_attention_weight_max": 63.37855529785156,
      "activations/layer17_attention_weight_min": -37.173553466796875,
      "activations/layer18_attention_weight_max": 54.992733001708984,
      "activations/layer18_attention_weight_min": -32.20633316040039,
      "activations/layer19_attention_weight_max": 19.327497482299805,
      "activations/layer19_attention_weight_min": -17.86955451965332,
      "activations/layer1_attention_weight_max": 17.37256622314453,
      "activations/layer1_attention_weight_min": -15.498734474182129,
      "activations/layer20_attention_weight_max": 21.085783004760742,
      "activations/layer20_attention_weight_min": -19.90398406982422,
      "activations/layer21_attention_weight_max": 42.53123474121094,
      "activations/layer21_attention_weight_min": -27.40312957763672,
      "activations/layer22_attention_weight_max": 34.07144546508789,
      "activations/layer22_attention_weight_min": -23.929149627685547,
      "activations/layer23_attention_weight_max": 27.50666046142578,
      "activations/layer23_attention_weight_min": -16.181224822998047,
      "activations/layer2_attention_weight_max": 28.922500610351562,
      "activations/layer2_attention_weight_min": -27.067846298217773,
      "activations/layer3_attention_weight_max": 69.96023559570312,
      "activations/layer3_attention_weight_min": -73.83329010009766,
      "activations/layer4_attention_weight_max": 97.7196273803711,
      "activations/layer4_attention_weight_min": -89.52368927001953,
      "activations/layer5_attention_weight_max": 118.80082702636719,
      "activations/layer5_attention_weight_min": -100.67030334472656,
      "activations/layer6_attention_weight_max": 61.85325622558594,
      "activations/layer6_attention_weight_min": -48.749210357666016,
      "activations/layer7_attention_weight_max": 60.954498291015625,
      "activations/layer7_attention_weight_min": -54.81013870239258,
      "activations/layer8_attention_weight_max": 40.0408935546875,
      "activations/layer8_attention_weight_min": -34.584129333496094,
      "activations/layer9_attention_weight_max": 54.551002502441406,
      "activations/layer9_attention_weight_min": -45.39787673950195,
      "epoch": 2.82,
      "learning_rate": 0.00013314772727272728,
      "loss": 2.912,
      "step": 48550
    },
    {
      "activations/layer0_attention_weight_max": 14.391749382019043,
      "activations/layer0_attention_weight_min": -14.336499214172363,
      "activations/layer10_attention_weight_max": 46.03409194946289,
      "activations/layer10_attention_weight_min": -34.542335510253906,
      "activations/layer11_attention_weight_max": 43.45811080932617,
      "activations/layer11_attention_weight_min": -28.986059188842773,
      "activations/layer12_attention_weight_max": 29.904720306396484,
      "activations/layer12_attention_weight_min": -23.792407989501953,
      "activations/layer13_attention_weight_max": 60.521026611328125,
      "activations/layer13_attention_weight_min": -45.90734100341797,
      "activations/layer14_attention_weight_max": 56.51256561279297,
      "activations/layer14_attention_weight_min": -33.04936981201172,
      "activations/layer15_attention_weight_max": 59.14183807373047,
      "activations/layer15_attention_weight_min": -36.27215576171875,
      "activations/layer16_attention_weight_max": 46.70783233642578,
      "activations/layer16_attention_weight_min": -26.76774024963379,
      "activations/layer17_attention_weight_max": 68.92510986328125,
      "activations/layer17_attention_weight_min": -35.538841247558594,
      "activations/layer18_attention_weight_max": 61.75675964355469,
      "activations/layer18_attention_weight_min": -33.371028900146484,
      "activations/layer19_attention_weight_max": 28.06498146057129,
      "activations/layer19_attention_weight_min": -17.727798461914062,
      "activations/layer1_attention_weight_max": 17.59231948852539,
      "activations/layer1_attention_weight_min": -13.215948104858398,
      "activations/layer20_attention_weight_max": 23.00090217590332,
      "activations/layer20_attention_weight_min": -19.092613220214844,
      "activations/layer21_attention_weight_max": 45.35902404785156,
      "activations/layer21_attention_weight_min": -24.97501564025879,
      "activations/layer22_attention_weight_max": 39.86647033691406,
      "activations/layer22_attention_weight_min": -21.92457389831543,
      "activations/layer23_attention_weight_max": 27.83346176147461,
      "activations/layer23_attention_weight_min": -14.942056655883789,
      "activations/layer2_attention_weight_max": 28.85443115234375,
      "activations/layer2_attention_weight_min": -25.735164642333984,
      "activations/layer3_attention_weight_max": 70.50785827636719,
      "activations/layer3_attention_weight_min": -71.24819946289062,
      "activations/layer4_attention_weight_max": 98.65850830078125,
      "activations/layer4_attention_weight_min": -88.15877532958984,
      "activations/layer5_attention_weight_max": 125.58260345458984,
      "activations/layer5_attention_weight_min": -102.0592269897461,
      "activations/layer6_attention_weight_max": 60.53337860107422,
      "activations/layer6_attention_weight_min": -51.738651275634766,
      "activations/layer7_attention_weight_max": 56.61316680908203,
      "activations/layer7_attention_weight_min": -53.32381820678711,
      "activations/layer8_attention_weight_max": 39.95205307006836,
      "activations/layer8_attention_weight_min": -35.11301040649414,
      "activations/layer9_attention_weight_max": 57.432167053222656,
      "activations/layer9_attention_weight_min": -53.835304260253906,
      "epoch": 2.82,
      "learning_rate": 0.00013312878787878786,
      "loss": 2.9185,
      "step": 48600
    },
    {
      "activations/layer0_attention_weight_max": 14.252705574035645,
      "activations/layer0_attention_weight_min": -14.364895820617676,
      "activations/layer10_attention_weight_max": 45.60313034057617,
      "activations/layer10_attention_weight_min": -35.897735595703125,
      "activations/layer11_attention_weight_max": 38.35296630859375,
      "activations/layer11_attention_weight_min": -27.760696411132812,
      "activations/layer12_attention_weight_max": 23.324613571166992,
      "activations/layer12_attention_weight_min": -25.94235610961914,
      "activations/layer13_attention_weight_max": 59.09660720825195,
      "activations/layer13_attention_weight_min": -45.28652572631836,
      "activations/layer14_attention_weight_max": 41.6148681640625,
      "activations/layer14_attention_weight_min": -32.02245330810547,
      "activations/layer15_attention_weight_max": 48.79473876953125,
      "activations/layer15_attention_weight_min": -37.935035705566406,
      "activations/layer16_attention_weight_max": 33.3904914855957,
      "activations/layer16_attention_weight_min": -28.676279067993164,
      "activations/layer17_attention_weight_max": 53.01174545288086,
      "activations/layer17_attention_weight_min": -34.51893997192383,
      "activations/layer18_attention_weight_max": 49.79111099243164,
      "activations/layer18_attention_weight_min": -32.587127685546875,
      "activations/layer19_attention_weight_max": 18.771141052246094,
      "activations/layer19_attention_weight_min": -20.07871437072754,
      "activations/layer1_attention_weight_max": 16.996374130249023,
      "activations/layer1_attention_weight_min": -15.267014503479004,
      "activations/layer20_attention_weight_max": 21.088090896606445,
      "activations/layer20_attention_weight_min": -19.633012771606445,
      "activations/layer21_attention_weight_max": 40.039833068847656,
      "activations/layer21_attention_weight_min": -27.954477310180664,
      "activations/layer22_attention_weight_max": 30.180084228515625,
      "activations/layer22_attention_weight_min": -23.940752029418945,
      "activations/layer23_attention_weight_max": 24.19623565673828,
      "activations/layer23_attention_weight_min": -18.569059371948242,
      "activations/layer2_attention_weight_max": 28.351741790771484,
      "activations/layer2_attention_weight_min": -26.936145782470703,
      "activations/layer3_attention_weight_max": 69.23092651367188,
      "activations/layer3_attention_weight_min": -73.0727310180664,
      "activations/layer4_attention_weight_max": 97.9688720703125,
      "activations/layer4_attention_weight_min": -85.87213897705078,
      "activations/layer5_attention_weight_max": 125.67674255371094,
      "activations/layer5_attention_weight_min": -98.61766052246094,
      "activations/layer6_attention_weight_max": 61.236854553222656,
      "activations/layer6_attention_weight_min": -49.68539047241211,
      "activations/layer7_attention_weight_max": 55.028541564941406,
      "activations/layer7_attention_weight_min": -52.89869689941406,
      "activations/layer8_attention_weight_max": 39.39297866821289,
      "activations/layer8_attention_weight_min": -34.87382888793945,
      "activations/layer9_attention_weight_max": 55.1511116027832,
      "activations/layer9_attention_weight_min": -49.472721099853516,
      "epoch": 2.83,
      "learning_rate": 0.00013310984848484848,
      "loss": 2.9137,
      "step": 48650
    },
    {
      "activations/layer0_attention_weight_max": 13.861783981323242,
      "activations/layer0_attention_weight_min": -14.23920726776123,
      "activations/layer10_attention_weight_max": 42.361995697021484,
      "activations/layer10_attention_weight_min": -34.24188232421875,
      "activations/layer11_attention_weight_max": 39.69437789916992,
      "activations/layer11_attention_weight_min": -27.426572799682617,
      "activations/layer12_attention_weight_max": 23.270179748535156,
      "activations/layer12_attention_weight_min": -23.08521270751953,
      "activations/layer13_attention_weight_max": 46.45759582519531,
      "activations/layer13_attention_weight_min": -38.496517181396484,
      "activations/layer14_attention_weight_max": 45.16040802001953,
      "activations/layer14_attention_weight_min": -33.5593376159668,
      "activations/layer15_attention_weight_max": 50.30630111694336,
      "activations/layer15_attention_weight_min": -35.06667709350586,
      "activations/layer16_attention_weight_max": 39.87249755859375,
      "activations/layer16_attention_weight_min": -26.15666961669922,
      "activations/layer17_attention_weight_max": 63.72652053833008,
      "activations/layer17_attention_weight_min": -37.02589797973633,
      "activations/layer18_attention_weight_max": 54.93118667602539,
      "activations/layer18_attention_weight_min": -34.35495376586914,
      "activations/layer19_attention_weight_max": 20.045692443847656,
      "activations/layer19_attention_weight_min": -17.378551483154297,
      "activations/layer1_attention_weight_max": 17.89252281188965,
      "activations/layer1_attention_weight_min": -13.742281913757324,
      "activations/layer20_attention_weight_max": 21.400402069091797,
      "activations/layer20_attention_weight_min": -18.08197784423828,
      "activations/layer21_attention_weight_max": 36.744049072265625,
      "activations/layer21_attention_weight_min": -24.78693199157715,
      "activations/layer22_attention_weight_max": 33.19662857055664,
      "activations/layer22_attention_weight_min": -21.519134521484375,
      "activations/layer23_attention_weight_max": 25.60464096069336,
      "activations/layer23_attention_weight_min": -15.864439010620117,
      "activations/layer2_attention_weight_max": 28.251707077026367,
      "activations/layer2_attention_weight_min": -25.67828941345215,
      "activations/layer3_attention_weight_max": 67.0562515258789,
      "activations/layer3_attention_weight_min": -68.9094009399414,
      "activations/layer4_attention_weight_max": 97.06935119628906,
      "activations/layer4_attention_weight_min": -88.25230407714844,
      "activations/layer5_attention_weight_max": 121.02754211425781,
      "activations/layer5_attention_weight_min": -102.2220458984375,
      "activations/layer6_attention_weight_max": 62.55091094970703,
      "activations/layer6_attention_weight_min": -51.955162048339844,
      "activations/layer7_attention_weight_max": 59.391212463378906,
      "activations/layer7_attention_weight_min": -52.1912956237793,
      "activations/layer8_attention_weight_max": 40.568824768066406,
      "activations/layer8_attention_weight_min": -34.37113571166992,
      "activations/layer9_attention_weight_max": 53.70156478881836,
      "activations/layer9_attention_weight_min": -42.83485794067383,
      "epoch": 2.83,
      "learning_rate": 0.0001330909090909091,
      "loss": 2.9127,
      "step": 48700
    },
    {
      "activations/layer0_attention_weight_max": 13.531923294067383,
      "activations/layer0_attention_weight_min": -14.00854778289795,
      "activations/layer10_attention_weight_max": 48.868988037109375,
      "activations/layer10_attention_weight_min": -37.30341339111328,
      "activations/layer11_attention_weight_max": 42.8258056640625,
      "activations/layer11_attention_weight_min": -29.271228790283203,
      "activations/layer12_attention_weight_max": 46.266937255859375,
      "activations/layer12_attention_weight_min": -29.439414978027344,
      "activations/layer13_attention_weight_max": 55.15966796875,
      "activations/layer13_attention_weight_min": -40.1329345703125,
      "activations/layer14_attention_weight_max": 58.44609451293945,
      "activations/layer14_attention_weight_min": -36.11154556274414,
      "activations/layer15_attention_weight_max": 58.32853698730469,
      "activations/layer15_attention_weight_min": -35.32253646850586,
      "activations/layer16_attention_weight_max": 37.27869415283203,
      "activations/layer16_attention_weight_min": -26.460420608520508,
      "activations/layer17_attention_weight_max": 56.07460403442383,
      "activations/layer17_attention_weight_min": -36.59185791015625,
      "activations/layer18_attention_weight_max": 56.17304611206055,
      "activations/layer18_attention_weight_min": -30.072410583496094,
      "activations/layer19_attention_weight_max": 24.67014503479004,
      "activations/layer19_attention_weight_min": -18.35114288330078,
      "activations/layer1_attention_weight_max": 16.7153263092041,
      "activations/layer1_attention_weight_min": -14.12210464477539,
      "activations/layer20_attention_weight_max": 17.24681282043457,
      "activations/layer20_attention_weight_min": -22.274311065673828,
      "activations/layer21_attention_weight_max": 45.63088607788086,
      "activations/layer21_attention_weight_min": -29.554672241210938,
      "activations/layer22_attention_weight_max": 33.87030029296875,
      "activations/layer22_attention_weight_min": -24.194334030151367,
      "activations/layer23_attention_weight_max": 24.78504180908203,
      "activations/layer23_attention_weight_min": -17.158649444580078,
      "activations/layer2_attention_weight_max": 30.863365173339844,
      "activations/layer2_attention_weight_min": -27.42499542236328,
      "activations/layer3_attention_weight_max": 70.76673126220703,
      "activations/layer3_attention_weight_min": -74.21766662597656,
      "activations/layer4_attention_weight_max": 101.475830078125,
      "activations/layer4_attention_weight_min": -88.88684844970703,
      "activations/layer5_attention_weight_max": 122.81951141357422,
      "activations/layer5_attention_weight_min": -103.79130554199219,
      "activations/layer6_attention_weight_max": 62.502777099609375,
      "activations/layer6_attention_weight_min": -51.50637435913086,
      "activations/layer7_attention_weight_max": 58.57441329956055,
      "activations/layer7_attention_weight_min": -56.654361724853516,
      "activations/layer8_attention_weight_max": 43.004364013671875,
      "activations/layer8_attention_weight_min": -35.66617965698242,
      "activations/layer9_attention_weight_max": 55.950069427490234,
      "activations/layer9_attention_weight_min": -50.77157974243164,
      "epoch": 2.83,
      "learning_rate": 0.00013307196969696968,
      "loss": 2.9223,
      "step": 48750
    },
    {
      "activations/layer0_attention_weight_max": 13.850905418395996,
      "activations/layer0_attention_weight_min": -14.884449005126953,
      "activations/layer10_attention_weight_max": 48.54742431640625,
      "activations/layer10_attention_weight_min": -39.857696533203125,
      "activations/layer11_attention_weight_max": 37.08951950073242,
      "activations/layer11_attention_weight_min": -29.169357299804688,
      "activations/layer12_attention_weight_max": 27.497974395751953,
      "activations/layer12_attention_weight_min": -25.160770416259766,
      "activations/layer13_attention_weight_max": 50.711666107177734,
      "activations/layer13_attention_weight_min": -39.051204681396484,
      "activations/layer14_attention_weight_max": 45.71382522583008,
      "activations/layer14_attention_weight_min": -32.78874969482422,
      "activations/layer15_attention_weight_max": 48.645606994628906,
      "activations/layer15_attention_weight_min": -33.6894645690918,
      "activations/layer16_attention_weight_max": 40.81657409667969,
      "activations/layer16_attention_weight_min": -26.389570236206055,
      "activations/layer17_attention_weight_max": 56.623661041259766,
      "activations/layer17_attention_weight_min": -37.08755111694336,
      "activations/layer18_attention_weight_max": 57.77410125732422,
      "activations/layer18_attention_weight_min": -35.606719970703125,
      "activations/layer19_attention_weight_max": 23.720500946044922,
      "activations/layer19_attention_weight_min": -16.654752731323242,
      "activations/layer1_attention_weight_max": 16.62873649597168,
      "activations/layer1_attention_weight_min": -13.551898956298828,
      "activations/layer20_attention_weight_max": 22.940980911254883,
      "activations/layer20_attention_weight_min": -21.370088577270508,
      "activations/layer21_attention_weight_max": 45.82637023925781,
      "activations/layer21_attention_weight_min": -28.585872650146484,
      "activations/layer22_attention_weight_max": 37.129756927490234,
      "activations/layer22_attention_weight_min": -22.563371658325195,
      "activations/layer23_attention_weight_max": 24.66033935546875,
      "activations/layer23_attention_weight_min": -16.466535568237305,
      "activations/layer2_attention_weight_max": 28.957172393798828,
      "activations/layer2_attention_weight_min": -26.875606536865234,
      "activations/layer3_attention_weight_max": 67.70896911621094,
      "activations/layer3_attention_weight_min": -70.5708236694336,
      "activations/layer4_attention_weight_max": 100.66917419433594,
      "activations/layer4_attention_weight_min": -90.54010772705078,
      "activations/layer5_attention_weight_max": 126.39588928222656,
      "activations/layer5_attention_weight_min": -104.92607879638672,
      "activations/layer6_attention_weight_max": 62.72338104248047,
      "activations/layer6_attention_weight_min": -55.2482795715332,
      "activations/layer7_attention_weight_max": 59.165855407714844,
      "activations/layer7_attention_weight_min": -55.22221755981445,
      "activations/layer8_attention_weight_max": 42.3311653137207,
      "activations/layer8_attention_weight_min": -37.8836555480957,
      "activations/layer9_attention_weight_max": 57.26076889038086,
      "activations/layer9_attention_weight_min": -51.55318069458008,
      "epoch": 2.84,
      "learning_rate": 0.0001330530303030303,
      "loss": 2.9227,
      "step": 48800
    },
    {
      "activations/layer0_attention_weight_max": 14.520508766174316,
      "activations/layer0_attention_weight_min": -14.860377311706543,
      "activations/layer10_attention_weight_max": 44.9769172668457,
      "activations/layer10_attention_weight_min": -35.83794403076172,
      "activations/layer11_attention_weight_max": 34.965972900390625,
      "activations/layer11_attention_weight_min": -28.936481475830078,
      "activations/layer12_attention_weight_max": 31.771862030029297,
      "activations/layer12_attention_weight_min": -26.946298599243164,
      "activations/layer13_attention_weight_max": 58.66341018676758,
      "activations/layer13_attention_weight_min": -40.61767578125,
      "activations/layer14_attention_weight_max": 52.035030364990234,
      "activations/layer14_attention_weight_min": -35.41518783569336,
      "activations/layer15_attention_weight_max": 52.263240814208984,
      "activations/layer15_attention_weight_min": -35.35512161254883,
      "activations/layer16_attention_weight_max": 38.67673873901367,
      "activations/layer16_attention_weight_min": -26.620712280273438,
      "activations/layer17_attention_weight_max": 65.91122436523438,
      "activations/layer17_attention_weight_min": -38.88048553466797,
      "activations/layer18_attention_weight_max": 57.17259979248047,
      "activations/layer18_attention_weight_min": -32.97712326049805,
      "activations/layer19_attention_weight_max": 24.964147567749023,
      "activations/layer19_attention_weight_min": -19.077611923217773,
      "activations/layer1_attention_weight_max": 18.00304412841797,
      "activations/layer1_attention_weight_min": -15.349610328674316,
      "activations/layer20_attention_weight_max": 21.52835464477539,
      "activations/layer20_attention_weight_min": -18.71961784362793,
      "activations/layer21_attention_weight_max": 46.9094352722168,
      "activations/layer21_attention_weight_min": -25.36236000061035,
      "activations/layer22_attention_weight_max": 38.7069206237793,
      "activations/layer22_attention_weight_min": -24.047405242919922,
      "activations/layer23_attention_weight_max": 28.297340393066406,
      "activations/layer23_attention_weight_min": -16.17691421508789,
      "activations/layer2_attention_weight_max": 27.79657745361328,
      "activations/layer2_attention_weight_min": -25.966190338134766,
      "activations/layer3_attention_weight_max": 67.13706970214844,
      "activations/layer3_attention_weight_min": -69.37451171875,
      "activations/layer4_attention_weight_max": 97.538818359375,
      "activations/layer4_attention_weight_min": -87.58043670654297,
      "activations/layer5_attention_weight_max": 123.55300903320312,
      "activations/layer5_attention_weight_min": -96.71240234375,
      "activations/layer6_attention_weight_max": 63.13141632080078,
      "activations/layer6_attention_weight_min": -50.96620178222656,
      "activations/layer7_attention_weight_max": 61.31591796875,
      "activations/layer7_attention_weight_min": -52.70676040649414,
      "activations/layer8_attention_weight_max": 40.56025314331055,
      "activations/layer8_attention_weight_min": -34.23830795288086,
      "activations/layer9_attention_weight_max": 56.96621322631836,
      "activations/layer9_attention_weight_min": -46.95380401611328,
      "epoch": 2.84,
      "learning_rate": 0.00013303409090909088,
      "loss": 2.9275,
      "step": 48850
    },
    {
      "activations/layer0_attention_weight_max": 13.555978775024414,
      "activations/layer0_attention_weight_min": -14.443266868591309,
      "activations/layer10_attention_weight_max": 47.42771911621094,
      "activations/layer10_attention_weight_min": -34.63072967529297,
      "activations/layer11_attention_weight_max": 41.656700134277344,
      "activations/layer11_attention_weight_min": -30.936603546142578,
      "activations/layer12_attention_weight_max": 25.053680419921875,
      "activations/layer12_attention_weight_min": -24.87127113342285,
      "activations/layer13_attention_weight_max": 56.54106903076172,
      "activations/layer13_attention_weight_min": -43.02357864379883,
      "activations/layer14_attention_weight_max": 53.91645812988281,
      "activations/layer14_attention_weight_min": -37.421321868896484,
      "activations/layer15_attention_weight_max": 46.65659713745117,
      "activations/layer15_attention_weight_min": -31.428070068359375,
      "activations/layer16_attention_weight_max": 41.51649475097656,
      "activations/layer16_attention_weight_min": -25.868896484375,
      "activations/layer17_attention_weight_max": 60.85398483276367,
      "activations/layer17_attention_weight_min": -35.82059860229492,
      "activations/layer18_attention_weight_max": 57.9415168762207,
      "activations/layer18_attention_weight_min": -32.670345306396484,
      "activations/layer19_attention_weight_max": 25.4492244720459,
      "activations/layer19_attention_weight_min": -17.459367752075195,
      "activations/layer1_attention_weight_max": 18.444530487060547,
      "activations/layer1_attention_weight_min": -14.036858558654785,
      "activations/layer20_attention_weight_max": 22.240564346313477,
      "activations/layer20_attention_weight_min": -18.831789016723633,
      "activations/layer21_attention_weight_max": 39.15564727783203,
      "activations/layer21_attention_weight_min": -26.730430603027344,
      "activations/layer22_attention_weight_max": 35.06214141845703,
      "activations/layer22_attention_weight_min": -26.814241409301758,
      "activations/layer23_attention_weight_max": 26.207561492919922,
      "activations/layer23_attention_weight_min": -16.55107879638672,
      "activations/layer2_attention_weight_max": 29.05306053161621,
      "activations/layer2_attention_weight_min": -29.76699447631836,
      "activations/layer3_attention_weight_max": 69.92417907714844,
      "activations/layer3_attention_weight_min": -73.51837921142578,
      "activations/layer4_attention_weight_max": 96.95618438720703,
      "activations/layer4_attention_weight_min": -93.76993560791016,
      "activations/layer5_attention_weight_max": 120.25099182128906,
      "activations/layer5_attention_weight_min": -101.69728088378906,
      "activations/layer6_attention_weight_max": 59.35050964355469,
      "activations/layer6_attention_weight_min": -52.54256820678711,
      "activations/layer7_attention_weight_max": 58.188507080078125,
      "activations/layer7_attention_weight_min": -57.17274856567383,
      "activations/layer8_attention_weight_max": 40.320030212402344,
      "activations/layer8_attention_weight_min": -35.31965255737305,
      "activations/layer9_attention_weight_max": 59.35467529296875,
      "activations/layer9_attention_weight_min": -50.31420135498047,
      "epoch": 2.84,
      "learning_rate": 0.0001330151515151515,
      "loss": 2.8998,
      "step": 48900
    },
    {
      "activations/layer0_attention_weight_max": 14.43417739868164,
      "activations/layer0_attention_weight_min": -14.50537109375,
      "activations/layer10_attention_weight_max": 51.10302734375,
      "activations/layer10_attention_weight_min": -38.524497985839844,
      "activations/layer11_attention_weight_max": 50.90144348144531,
      "activations/layer11_attention_weight_min": -31.35968589782715,
      "activations/layer12_attention_weight_max": 32.190914154052734,
      "activations/layer12_attention_weight_min": -27.92868423461914,
      "activations/layer13_attention_weight_max": 60.41523361206055,
      "activations/layer13_attention_weight_min": -42.63056182861328,
      "activations/layer14_attention_weight_max": 59.8868408203125,
      "activations/layer14_attention_weight_min": -35.09831237792969,
      "activations/layer15_attention_weight_max": 50.07304382324219,
      "activations/layer15_attention_weight_min": -34.752681732177734,
      "activations/layer16_attention_weight_max": 39.409393310546875,
      "activations/layer16_attention_weight_min": -27.485044479370117,
      "activations/layer17_attention_weight_max": 60.77808380126953,
      "activations/layer17_attention_weight_min": -35.05352783203125,
      "activations/layer18_attention_weight_max": 56.88618469238281,
      "activations/layer18_attention_weight_min": -33.96875,
      "activations/layer19_attention_weight_max": 24.28347396850586,
      "activations/layer19_attention_weight_min": -17.09914207458496,
      "activations/layer1_attention_weight_max": 17.878528594970703,
      "activations/layer1_attention_weight_min": -14.592840194702148,
      "activations/layer20_attention_weight_max": 25.322153091430664,
      "activations/layer20_attention_weight_min": -18.970726013183594,
      "activations/layer21_attention_weight_max": 47.47989273071289,
      "activations/layer21_attention_weight_min": -28.209346771240234,
      "activations/layer22_attention_weight_max": 40.00923156738281,
      "activations/layer22_attention_weight_min": -22.866365432739258,
      "activations/layer23_attention_weight_max": 26.508193969726562,
      "activations/layer23_attention_weight_min": -15.182573318481445,
      "activations/layer2_attention_weight_max": 29.519956588745117,
      "activations/layer2_attention_weight_min": -28.212926864624023,
      "activations/layer3_attention_weight_max": 71.30389404296875,
      "activations/layer3_attention_weight_min": -74.49654388427734,
      "activations/layer4_attention_weight_max": 98.5495834350586,
      "activations/layer4_attention_weight_min": -92.3416519165039,
      "activations/layer5_attention_weight_max": 127.62986755371094,
      "activations/layer5_attention_weight_min": -107.78439331054688,
      "activations/layer6_attention_weight_max": 63.2524299621582,
      "activations/layer6_attention_weight_min": -54.645206451416016,
      "activations/layer7_attention_weight_max": 73.54791259765625,
      "activations/layer7_attention_weight_min": -55.84199142456055,
      "activations/layer8_attention_weight_max": 53.42804718017578,
      "activations/layer8_attention_weight_min": -35.62607192993164,
      "activations/layer9_attention_weight_max": 82.44053649902344,
      "activations/layer9_attention_weight_min": -59.15741729736328,
      "epoch": 2.84,
      "learning_rate": 0.00013299621212121211,
      "loss": 2.909,
      "step": 48950
    },
    {
      "activations/layer0_attention_weight_max": 13.93981647491455,
      "activations/layer0_attention_weight_min": -14.38769245147705,
      "activations/layer10_attention_weight_max": 47.698299407958984,
      "activations/layer10_attention_weight_min": -38.9388542175293,
      "activations/layer11_attention_weight_max": 40.236541748046875,
      "activations/layer11_attention_weight_min": -29.294172286987305,
      "activations/layer12_attention_weight_max": 25.01091957092285,
      "activations/layer12_attention_weight_min": -22.497535705566406,
      "activations/layer13_attention_weight_max": 60.006805419921875,
      "activations/layer13_attention_weight_min": -44.81196594238281,
      "activations/layer14_attention_weight_max": 47.952850341796875,
      "activations/layer14_attention_weight_min": -36.520851135253906,
      "activations/layer15_attention_weight_max": 46.57754135131836,
      "activations/layer15_attention_weight_min": -37.168697357177734,
      "activations/layer16_attention_weight_max": 41.05873107910156,
      "activations/layer16_attention_weight_min": -28.6965389251709,
      "activations/layer17_attention_weight_max": 60.3521728515625,
      "activations/layer17_attention_weight_min": -35.253238677978516,
      "activations/layer18_attention_weight_max": 60.08640670776367,
      "activations/layer18_attention_weight_min": -32.12831497192383,
      "activations/layer19_attention_weight_max": 21.724014282226562,
      "activations/layer19_attention_weight_min": -16.81062889099121,
      "activations/layer1_attention_weight_max": 17.8913516998291,
      "activations/layer1_attention_weight_min": -14.308603286743164,
      "activations/layer20_attention_weight_max": 20.637088775634766,
      "activations/layer20_attention_weight_min": -17.801128387451172,
      "activations/layer21_attention_weight_max": 39.6696891784668,
      "activations/layer21_attention_weight_min": -27.252132415771484,
      "activations/layer22_attention_weight_max": 36.92979049682617,
      "activations/layer22_attention_weight_min": -22.772235870361328,
      "activations/layer23_attention_weight_max": 25.534297943115234,
      "activations/layer23_attention_weight_min": -16.703269958496094,
      "activations/layer2_attention_weight_max": 27.872533798217773,
      "activations/layer2_attention_weight_min": -27.004674911499023,
      "activations/layer3_attention_weight_max": 68.96878814697266,
      "activations/layer3_attention_weight_min": -74.53661346435547,
      "activations/layer4_attention_weight_max": 97.21858215332031,
      "activations/layer4_attention_weight_min": -92.15123748779297,
      "activations/layer5_attention_weight_max": 121.36122131347656,
      "activations/layer5_attention_weight_min": -102.51695251464844,
      "activations/layer6_attention_weight_max": 60.178409576416016,
      "activations/layer6_attention_weight_min": -52.201412200927734,
      "activations/layer7_attention_weight_max": 57.451656341552734,
      "activations/layer7_attention_weight_min": -54.570987701416016,
      "activations/layer8_attention_weight_max": 42.38654708862305,
      "activations/layer8_attention_weight_min": -36.504940032958984,
      "activations/layer9_attention_weight_max": 52.60184860229492,
      "activations/layer9_attention_weight_min": -51.4510383605957,
      "epoch": 2.85,
      "learning_rate": 0.0001329772727272727,
      "loss": 2.9079,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_loss": 2.865234375,
      "eval_runtime": 8.4614,
      "eval_samples_per_second": 507.478,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_openwebtext_loss": 2.865234375,
      "eval_openwebtext_ppl": 17.553166745968216,
      "eval_openwebtext_runtime": 8.4614,
      "eval_openwebtext_samples_per_second": 507.478,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_wikitext_loss": 3.1171875,
      "eval_wikitext_ppl": 22.582776184718522,
      "eval_wikitext_runtime": 1.9425,
      "eval_wikitext_samples_per_second": 234.749,
      "step": 49000
    },
    {
      "epoch": 2.85,
      "eval_lambada_loss": 3.06640625,
      "eval_lambada_ppl": 21.464625403032095,
      "eval_lambada_runtime": 9.4973,
      "eval_lambada_samples_per_second": 512.671,
      "step": 49000
    },
    {
      "activations/layer0_attention_weight_max": 14.394109725952148,
      "activations/layer0_attention_weight_min": -14.526325225830078,
      "activations/layer10_attention_weight_max": 49.285823822021484,
      "activations/layer10_attention_weight_min": -36.65175247192383,
      "activations/layer11_attention_weight_max": 47.391578674316406,
      "activations/layer11_attention_weight_min": -29.16275405883789,
      "activations/layer12_attention_weight_max": 25.39989471435547,
      "activations/layer12_attention_weight_min": -25.983108520507812,
      "activations/layer13_attention_weight_max": 50.43015670776367,
      "activations/layer13_attention_weight_min": -41.6690559387207,
      "activations/layer14_attention_weight_max": 55.435333251953125,
      "activations/layer14_attention_weight_min": -32.69746780395508,
      "activations/layer15_attention_weight_max": 40.818721771240234,
      "activations/layer15_attention_weight_min": -32.68452072143555,
      "activations/layer16_attention_weight_max": 33.52090072631836,
      "activations/layer16_attention_weight_min": -26.393970489501953,
      "activations/layer17_attention_weight_max": 58.31817626953125,
      "activations/layer17_attention_weight_min": -40.05462646484375,
      "activations/layer18_attention_weight_max": 52.10927963256836,
      "activations/layer18_attention_weight_min": -30.535696029663086,
      "activations/layer19_attention_weight_max": 22.50261878967285,
      "activations/layer19_attention_weight_min": -20.431718826293945,
      "activations/layer1_attention_weight_max": 18.635923385620117,
      "activations/layer1_attention_weight_min": -15.700901985168457,
      "activations/layer20_attention_weight_max": 20.531614303588867,
      "activations/layer20_attention_weight_min": -20.214712142944336,
      "activations/layer21_attention_weight_max": 37.66419982910156,
      "activations/layer21_attention_weight_min": -27.85162353515625,
      "activations/layer22_attention_weight_max": 32.18763732910156,
      "activations/layer22_attention_weight_min": -24.12351417541504,
      "activations/layer23_attention_weight_max": 22.937522888183594,
      "activations/layer23_attention_weight_min": -14.398798942565918,
      "activations/layer2_attention_weight_max": 29.951473236083984,
      "activations/layer2_attention_weight_min": -27.18206024169922,
      "activations/layer3_attention_weight_max": 66.39694213867188,
      "activations/layer3_attention_weight_min": -70.67377471923828,
      "activations/layer4_attention_weight_max": 96.619140625,
      "activations/layer4_attention_weight_min": -86.4629135131836,
      "activations/layer5_attention_weight_max": 132.57620239257812,
      "activations/layer5_attention_weight_min": -111.00714111328125,
      "activations/layer6_attention_weight_max": 66.9646987915039,
      "activations/layer6_attention_weight_min": -51.54242706298828,
      "activations/layer7_attention_weight_max": 77.61408996582031,
      "activations/layer7_attention_weight_min": -55.96465301513672,
      "activations/layer8_attention_weight_max": 58.33899688720703,
      "activations/layer8_attention_weight_min": -39.622703552246094,
      "activations/layer9_attention_weight_max": 84.9605941772461,
      "activations/layer9_attention_weight_min": -54.07406234741211,
      "epoch": 2.85,
      "learning_rate": 0.00013295833333333332,
      "loss": 2.9112,
      "step": 49050
    },
    {
      "activations/layer0_attention_weight_max": 15.420825004577637,
      "activations/layer0_attention_weight_min": -14.152074813842773,
      "activations/layer10_attention_weight_max": 47.90451431274414,
      "activations/layer10_attention_weight_min": -36.11566162109375,
      "activations/layer11_attention_weight_max": 41.724342346191406,
      "activations/layer11_attention_weight_min": -29.470401763916016,
      "activations/layer12_attention_weight_max": 26.75625991821289,
      "activations/layer12_attention_weight_min": -24.858081817626953,
      "activations/layer13_attention_weight_max": 64.09307861328125,
      "activations/layer13_attention_weight_min": -42.044090270996094,
      "activations/layer14_attention_weight_max": 67.19949340820312,
      "activations/layer14_attention_weight_min": -38.572113037109375,
      "activations/layer15_attention_weight_max": 49.69907760620117,
      "activations/layer15_attention_weight_min": -33.113319396972656,
      "activations/layer16_attention_weight_max": 47.68931579589844,
      "activations/layer16_attention_weight_min": -27.142871856689453,
      "activations/layer17_attention_weight_max": 66.33917999267578,
      "activations/layer17_attention_weight_min": -39.6546516418457,
      "activations/layer18_attention_weight_max": 56.556068420410156,
      "activations/layer18_attention_weight_min": -32.400943756103516,
      "activations/layer19_attention_weight_max": 23.73105812072754,
      "activations/layer19_attention_weight_min": -18.114713668823242,
      "activations/layer1_attention_weight_max": 18.736339569091797,
      "activations/layer1_attention_weight_min": -14.570252418518066,
      "activations/layer20_attention_weight_max": 23.171926498413086,
      "activations/layer20_attention_weight_min": -20.912763595581055,
      "activations/layer21_attention_weight_max": 52.59677505493164,
      "activations/layer21_attention_weight_min": -29.57375144958496,
      "activations/layer22_attention_weight_max": 40.0013427734375,
      "activations/layer22_attention_weight_min": -24.893096923828125,
      "activations/layer23_attention_weight_max": 22.774303436279297,
      "activations/layer23_attention_weight_min": -15.641006469726562,
      "activations/layer2_attention_weight_max": 30.880172729492188,
      "activations/layer2_attention_weight_min": -28.31917953491211,
      "activations/layer3_attention_weight_max": 68.15089416503906,
      "activations/layer3_attention_weight_min": -72.1303939819336,
      "activations/layer4_attention_weight_max": 101.43016052246094,
      "activations/layer4_attention_weight_min": -89.01396179199219,
      "activations/layer5_attention_weight_max": 128.4170379638672,
      "activations/layer5_attention_weight_min": -102.68108367919922,
      "activations/layer6_attention_weight_max": 61.818634033203125,
      "activations/layer6_attention_weight_min": -50.69595718383789,
      "activations/layer7_attention_weight_max": 55.98444366455078,
      "activations/layer7_attention_weight_min": -54.86665344238281,
      "activations/layer8_attention_weight_max": 40.27400588989258,
      "activations/layer8_attention_weight_min": -34.25829315185547,
      "activations/layer9_attention_weight_max": 53.03950119018555,
      "activations/layer9_attention_weight_min": -47.612606048583984,
      "epoch": 2.85,
      "learning_rate": 0.00013293939393939393,
      "loss": 2.9172,
      "step": 49100
    },
    {
      "activations/layer0_attention_weight_max": 13.850726127624512,
      "activations/layer0_attention_weight_min": -14.146721839904785,
      "activations/layer10_attention_weight_max": 49.051692962646484,
      "activations/layer10_attention_weight_min": -37.04072952270508,
      "activations/layer11_attention_weight_max": 40.40877151489258,
      "activations/layer11_attention_weight_min": -29.350341796875,
      "activations/layer12_attention_weight_max": 28.003950119018555,
      "activations/layer12_attention_weight_min": -24.156835556030273,
      "activations/layer13_attention_weight_max": 49.61071014404297,
      "activations/layer13_attention_weight_min": -39.607704162597656,
      "activations/layer14_attention_weight_max": 44.67966842651367,
      "activations/layer14_attention_weight_min": -38.04281234741211,
      "activations/layer15_attention_weight_max": 43.08580017089844,
      "activations/layer15_attention_weight_min": -34.14946746826172,
      "activations/layer16_attention_weight_max": 41.153236389160156,
      "activations/layer16_attention_weight_min": -28.67851448059082,
      "activations/layer17_attention_weight_max": 60.4500732421875,
      "activations/layer17_attention_weight_min": -37.34033203125,
      "activations/layer18_attention_weight_max": 62.06520462036133,
      "activations/layer18_attention_weight_min": -32.27527618408203,
      "activations/layer19_attention_weight_max": 24.529294967651367,
      "activations/layer19_attention_weight_min": -17.59849739074707,
      "activations/layer1_attention_weight_max": 17.536760330200195,
      "activations/layer1_attention_weight_min": -14.060498237609863,
      "activations/layer20_attention_weight_max": 22.360990524291992,
      "activations/layer20_attention_weight_min": -22.37682342529297,
      "activations/layer21_attention_weight_max": 44.69209289550781,
      "activations/layer21_attention_weight_min": -30.736013412475586,
      "activations/layer22_attention_weight_max": 36.11570739746094,
      "activations/layer22_attention_weight_min": -23.795469284057617,
      "activations/layer23_attention_weight_max": 27.601146697998047,
      "activations/layer23_attention_weight_min": -17.315319061279297,
      "activations/layer2_attention_weight_max": 29.116933822631836,
      "activations/layer2_attention_weight_min": -27.178281784057617,
      "activations/layer3_attention_weight_max": 73.59815979003906,
      "activations/layer3_attention_weight_min": -74.1156005859375,
      "activations/layer4_attention_weight_max": 100.7168197631836,
      "activations/layer4_attention_weight_min": -93.37673950195312,
      "activations/layer5_attention_weight_max": 119.85216522216797,
      "activations/layer5_attention_weight_min": -107.10519409179688,
      "activations/layer6_attention_weight_max": 59.113346099853516,
      "activations/layer6_attention_weight_min": -51.53107452392578,
      "activations/layer7_attention_weight_max": 70.82396697998047,
      "activations/layer7_attention_weight_min": -57.72909927368164,
      "activations/layer8_attention_weight_max": 45.00387191772461,
      "activations/layer8_attention_weight_min": -33.71391677856445,
      "activations/layer9_attention_weight_max": 67.42800903320312,
      "activations/layer9_attention_weight_min": -54.02393341064453,
      "epoch": 2.86,
      "learning_rate": 0.00013292045454545452,
      "loss": 2.9046,
      "step": 49150
    },
    {
      "activations/layer0_attention_weight_max": 14.514630317687988,
      "activations/layer0_attention_weight_min": -14.440712928771973,
      "activations/layer10_attention_weight_max": 48.797630310058594,
      "activations/layer10_attention_weight_min": -37.20613479614258,
      "activations/layer11_attention_weight_max": 41.246490478515625,
      "activations/layer11_attention_weight_min": -31.2832088470459,
      "activations/layer12_attention_weight_max": 35.382049560546875,
      "activations/layer12_attention_weight_min": -24.792327880859375,
      "activations/layer13_attention_weight_max": 61.279441833496094,
      "activations/layer13_attention_weight_min": -47.135215759277344,
      "activations/layer14_attention_weight_max": 53.43996047973633,
      "activations/layer14_attention_weight_min": -39.82402420043945,
      "activations/layer15_attention_weight_max": 57.00939178466797,
      "activations/layer15_attention_weight_min": -37.016353607177734,
      "activations/layer16_attention_weight_max": 35.90242385864258,
      "activations/layer16_attention_weight_min": -28.114521026611328,
      "activations/layer17_attention_weight_max": 59.7601203918457,
      "activations/layer17_attention_weight_min": -34.739315032958984,
      "activations/layer18_attention_weight_max": 56.18019485473633,
      "activations/layer18_attention_weight_min": -32.79887008666992,
      "activations/layer19_attention_weight_max": 24.12405014038086,
      "activations/layer19_attention_weight_min": -18.57620620727539,
      "activations/layer1_attention_weight_max": 17.597503662109375,
      "activations/layer1_attention_weight_min": -14.137666702270508,
      "activations/layer20_attention_weight_max": 19.474321365356445,
      "activations/layer20_attention_weight_min": -17.203969955444336,
      "activations/layer21_attention_weight_max": 45.2834358215332,
      "activations/layer21_attention_weight_min": -26.74907684326172,
      "activations/layer22_attention_weight_max": 33.63035202026367,
      "activations/layer22_attention_weight_min": -21.354583740234375,
      "activations/layer23_attention_weight_max": 23.990690231323242,
      "activations/layer23_attention_weight_min": -16.79840850830078,
      "activations/layer2_attention_weight_max": 29.24367332458496,
      "activations/layer2_attention_weight_min": -27.620704650878906,
      "activations/layer3_attention_weight_max": 71.95704650878906,
      "activations/layer3_attention_weight_min": -76.5201644897461,
      "activations/layer4_attention_weight_max": 101.15496826171875,
      "activations/layer4_attention_weight_min": -92.27645111083984,
      "activations/layer5_attention_weight_max": 129.37179565429688,
      "activations/layer5_attention_weight_min": -102.5908432006836,
      "activations/layer6_attention_weight_max": 62.044044494628906,
      "activations/layer6_attention_weight_min": -51.2730827331543,
      "activations/layer7_attention_weight_max": 58.76211929321289,
      "activations/layer7_attention_weight_min": -57.75503158569336,
      "activations/layer8_attention_weight_max": 42.075923919677734,
      "activations/layer8_attention_weight_min": -38.053199768066406,
      "activations/layer9_attention_weight_max": 55.52482986450195,
      "activations/layer9_attention_weight_min": -50.530296325683594,
      "epoch": 2.86,
      "learning_rate": 0.00013290151515151514,
      "loss": 2.9217,
      "step": 49200
    },
    {
      "activations/layer0_attention_weight_max": 15.102337837219238,
      "activations/layer0_attention_weight_min": -14.250293731689453,
      "activations/layer10_attention_weight_max": 47.13603591918945,
      "activations/layer10_attention_weight_min": -37.101234436035156,
      "activations/layer11_attention_weight_max": 36.87471389770508,
      "activations/layer11_attention_weight_min": -27.757484436035156,
      "activations/layer12_attention_weight_max": 27.908933639526367,
      "activations/layer12_attention_weight_min": -24.18233871459961,
      "activations/layer13_attention_weight_max": 55.04420471191406,
      "activations/layer13_attention_weight_min": -41.87773895263672,
      "activations/layer14_attention_weight_max": 46.92041778564453,
      "activations/layer14_attention_weight_min": -35.020652770996094,
      "activations/layer15_attention_weight_max": 47.751853942871094,
      "activations/layer15_attention_weight_min": -34.212093353271484,
      "activations/layer16_attention_weight_max": 37.95780944824219,
      "activations/layer16_attention_weight_min": -27.13652229309082,
      "activations/layer17_attention_weight_max": 59.6864013671875,
      "activations/layer17_attention_weight_min": -37.3918342590332,
      "activations/layer18_attention_weight_max": 54.86557388305664,
      "activations/layer18_attention_weight_min": -33.75648880004883,
      "activations/layer19_attention_weight_max": 24.03647232055664,
      "activations/layer19_attention_weight_min": -18.599882125854492,
      "activations/layer1_attention_weight_max": 18.136449813842773,
      "activations/layer1_attention_weight_min": -17.843963623046875,
      "activations/layer20_attention_weight_max": 21.82335662841797,
      "activations/layer20_attention_weight_min": -18.204681396484375,
      "activations/layer21_attention_weight_max": 41.405982971191406,
      "activations/layer21_attention_weight_min": -27.210887908935547,
      "activations/layer22_attention_weight_max": 37.11686325073242,
      "activations/layer22_attention_weight_min": -22.43575096130371,
      "activations/layer23_attention_weight_max": 24.26361083984375,
      "activations/layer23_attention_weight_min": -15.48337459564209,
      "activations/layer2_attention_weight_max": 31.05060386657715,
      "activations/layer2_attention_weight_min": -29.595325469970703,
      "activations/layer3_attention_weight_max": 71.86019134521484,
      "activations/layer3_attention_weight_min": -80.1088638305664,
      "activations/layer4_attention_weight_max": 98.77315521240234,
      "activations/layer4_attention_weight_min": -98.21360778808594,
      "activations/layer5_attention_weight_max": 126.94930267333984,
      "activations/layer5_attention_weight_min": -109.40287780761719,
      "activations/layer6_attention_weight_max": 61.39463424682617,
      "activations/layer6_attention_weight_min": -56.79510498046875,
      "activations/layer7_attention_weight_max": 62.07786560058594,
      "activations/layer7_attention_weight_min": -56.18693923950195,
      "activations/layer8_attention_weight_max": 42.02064895629883,
      "activations/layer8_attention_weight_min": -36.16080856323242,
      "activations/layer9_attention_weight_max": 53.115413665771484,
      "activations/layer9_attention_weight_min": -51.082130432128906,
      "epoch": 2.86,
      "learning_rate": 0.00013288257575757575,
      "loss": 2.9207,
      "step": 49250
    },
    {
      "activations/layer0_attention_weight_max": 15.395564079284668,
      "activations/layer0_attention_weight_min": -14.626887321472168,
      "activations/layer10_attention_weight_max": 46.50067901611328,
      "activations/layer10_attention_weight_min": -37.14167785644531,
      "activations/layer11_attention_weight_max": 41.46446228027344,
      "activations/layer11_attention_weight_min": -29.421720504760742,
      "activations/layer12_attention_weight_max": 24.873777389526367,
      "activations/layer12_attention_weight_min": -27.25180435180664,
      "activations/layer13_attention_weight_max": 50.37696075439453,
      "activations/layer13_attention_weight_min": -38.733642578125,
      "activations/layer14_attention_weight_max": 39.99660110473633,
      "activations/layer14_attention_weight_min": -34.751766204833984,
      "activations/layer15_attention_weight_max": 43.844383239746094,
      "activations/layer15_attention_weight_min": -32.963809967041016,
      "activations/layer16_attention_weight_max": 36.02560806274414,
      "activations/layer16_attention_weight_min": -25.78611946105957,
      "activations/layer17_attention_weight_max": 64.18907165527344,
      "activations/layer17_attention_weight_min": -39.848304748535156,
      "activations/layer18_attention_weight_max": 54.77690887451172,
      "activations/layer18_attention_weight_min": -31.477577209472656,
      "activations/layer19_attention_weight_max": 19.310991287231445,
      "activations/layer19_attention_weight_min": -20.054418563842773,
      "activations/layer1_attention_weight_max": 17.684913635253906,
      "activations/layer1_attention_weight_min": -15.075296401977539,
      "activations/layer20_attention_weight_max": 20.311662673950195,
      "activations/layer20_attention_weight_min": -18.663497924804688,
      "activations/layer21_attention_weight_max": 44.35639190673828,
      "activations/layer21_attention_weight_min": -26.46682357788086,
      "activations/layer22_attention_weight_max": 34.32920455932617,
      "activations/layer22_attention_weight_min": -23.378707885742188,
      "activations/layer23_attention_weight_max": 24.089584350585938,
      "activations/layer23_attention_weight_min": -16.242902755737305,
      "activations/layer2_attention_weight_max": 26.42207908630371,
      "activations/layer2_attention_weight_min": -25.72564697265625,
      "activations/layer3_attention_weight_max": 68.35761260986328,
      "activations/layer3_attention_weight_min": -72.1255874633789,
      "activations/layer4_attention_weight_max": 103.6746826171875,
      "activations/layer4_attention_weight_min": -93.00328826904297,
      "activations/layer5_attention_weight_max": 124.89340209960938,
      "activations/layer5_attention_weight_min": -108.03826904296875,
      "activations/layer6_attention_weight_max": 61.8315315246582,
      "activations/layer6_attention_weight_min": -51.99128723144531,
      "activations/layer7_attention_weight_max": 61.87159729003906,
      "activations/layer7_attention_weight_min": -54.69019317626953,
      "activations/layer8_attention_weight_max": 41.21885299682617,
      "activations/layer8_attention_weight_min": -35.03364562988281,
      "activations/layer9_attention_weight_max": 63.15761947631836,
      "activations/layer9_attention_weight_min": -54.697357177734375,
      "epoch": 2.86,
      "learning_rate": 0.00013286363636363634,
      "loss": 2.9301,
      "step": 49300
    },
    {
      "activations/layer0_attention_weight_max": 15.616153717041016,
      "activations/layer0_attention_weight_min": -15.089632987976074,
      "activations/layer10_attention_weight_max": 49.19296646118164,
      "activations/layer10_attention_weight_min": -37.31675720214844,
      "activations/layer11_attention_weight_max": 39.737579345703125,
      "activations/layer11_attention_weight_min": -29.718677520751953,
      "activations/layer12_attention_weight_max": 32.349098205566406,
      "activations/layer12_attention_weight_min": -26.80715560913086,
      "activations/layer13_attention_weight_max": 59.699317932128906,
      "activations/layer13_attention_weight_min": -41.23991394042969,
      "activations/layer14_attention_weight_max": 57.067970275878906,
      "activations/layer14_attention_weight_min": -36.49191665649414,
      "activations/layer15_attention_weight_max": 49.08506774902344,
      "activations/layer15_attention_weight_min": -33.65386962890625,
      "activations/layer16_attention_weight_max": 41.23420333862305,
      "activations/layer16_attention_weight_min": -29.129667282104492,
      "activations/layer17_attention_weight_max": 68.85623931884766,
      "activations/layer17_attention_weight_min": -35.96908187866211,
      "activations/layer18_attention_weight_max": 55.322509765625,
      "activations/layer18_attention_weight_min": -33.46216583251953,
      "activations/layer19_attention_weight_max": 26.409835815429688,
      "activations/layer19_attention_weight_min": -17.127647399902344,
      "activations/layer1_attention_weight_max": 17.6442813873291,
      "activations/layer1_attention_weight_min": -14.838626861572266,
      "activations/layer20_attention_weight_max": 23.092185974121094,
      "activations/layer20_attention_weight_min": -18.88471031188965,
      "activations/layer21_attention_weight_max": 48.00144577026367,
      "activations/layer21_attention_weight_min": -34.93239212036133,
      "activations/layer22_attention_weight_max": 39.82917022705078,
      "activations/layer22_attention_weight_min": -22.756088256835938,
      "activations/layer23_attention_weight_max": 27.221599578857422,
      "activations/layer23_attention_weight_min": -16.355731964111328,
      "activations/layer2_attention_weight_max": 29.77692413330078,
      "activations/layer2_attention_weight_min": -27.960744857788086,
      "activations/layer3_attention_weight_max": 69.8207015991211,
      "activations/layer3_attention_weight_min": -73.2852554321289,
      "activations/layer4_attention_weight_max": 97.68065643310547,
      "activations/layer4_attention_weight_min": -89.92485809326172,
      "activations/layer5_attention_weight_max": 124.8116455078125,
      "activations/layer5_attention_weight_min": -101.53129577636719,
      "activations/layer6_attention_weight_max": 63.278968811035156,
      "activations/layer6_attention_weight_min": -51.609886169433594,
      "activations/layer7_attention_weight_max": 60.26721954345703,
      "activations/layer7_attention_weight_min": -54.976985931396484,
      "activations/layer8_attention_weight_max": 41.027854919433594,
      "activations/layer8_attention_weight_min": -35.189674377441406,
      "activations/layer9_attention_weight_max": 52.6955680847168,
      "activations/layer9_attention_weight_min": -49.10498046875,
      "epoch": 2.87,
      "learning_rate": 0.00013284469696969695,
      "loss": 2.9174,
      "step": 49350
    },
    {
      "activations/layer0_attention_weight_max": 14.228557586669922,
      "activations/layer0_attention_weight_min": -14.825061798095703,
      "activations/layer10_attention_weight_max": 48.069854736328125,
      "activations/layer10_attention_weight_min": -35.820648193359375,
      "activations/layer11_attention_weight_max": 43.721092224121094,
      "activations/layer11_attention_weight_min": -29.012481689453125,
      "activations/layer12_attention_weight_max": 25.28658676147461,
      "activations/layer12_attention_weight_min": -25.83888053894043,
      "activations/layer13_attention_weight_max": 66.70323181152344,
      "activations/layer13_attention_weight_min": -40.68730545043945,
      "activations/layer14_attention_weight_max": 80.9041519165039,
      "activations/layer14_attention_weight_min": -44.90850830078125,
      "activations/layer15_attention_weight_max": 49.98497772216797,
      "activations/layer15_attention_weight_min": -35.71895217895508,
      "activations/layer16_attention_weight_max": 46.80240249633789,
      "activations/layer16_attention_weight_min": -26.214078903198242,
      "activations/layer17_attention_weight_max": 68.9670181274414,
      "activations/layer17_attention_weight_min": -36.613365173339844,
      "activations/layer18_attention_weight_max": 62.85542297363281,
      "activations/layer18_attention_weight_min": -32.34592819213867,
      "activations/layer19_attention_weight_max": 23.966033935546875,
      "activations/layer19_attention_weight_min": -19.352109909057617,
      "activations/layer1_attention_weight_max": 17.513765335083008,
      "activations/layer1_attention_weight_min": -16.142690658569336,
      "activations/layer20_attention_weight_max": 27.167455673217773,
      "activations/layer20_attention_weight_min": -18.053300857543945,
      "activations/layer21_attention_weight_max": 54.85344696044922,
      "activations/layer21_attention_weight_min": -27.111692428588867,
      "activations/layer22_attention_weight_max": 43.8670768737793,
      "activations/layer22_attention_weight_min": -22.216354370117188,
      "activations/layer23_attention_weight_max": 24.801090240478516,
      "activations/layer23_attention_weight_min": -16.312496185302734,
      "activations/layer2_attention_weight_max": 28.277278900146484,
      "activations/layer2_attention_weight_min": -27.43731689453125,
      "activations/layer3_attention_weight_max": 70.50434112548828,
      "activations/layer3_attention_weight_min": -74.96064758300781,
      "activations/layer4_attention_weight_max": 105.5413818359375,
      "activations/layer4_attention_weight_min": -96.3282470703125,
      "activations/layer5_attention_weight_max": 127.19000244140625,
      "activations/layer5_attention_weight_min": -107.4900131225586,
      "activations/layer6_attention_weight_max": 65.88352966308594,
      "activations/layer6_attention_weight_min": -55.61552810668945,
      "activations/layer7_attention_weight_max": 59.82798767089844,
      "activations/layer7_attention_weight_min": -54.79054260253906,
      "activations/layer8_attention_weight_max": 40.851078033447266,
      "activations/layer8_attention_weight_min": -36.351200103759766,
      "activations/layer9_attention_weight_max": 61.407691955566406,
      "activations/layer9_attention_weight_min": -50.97379684448242,
      "epoch": 2.87,
      "learning_rate": 0.00013282575757575757,
      "loss": 2.9225,
      "step": 49400
    },
    {
      "activations/layer0_attention_weight_max": 13.774799346923828,
      "activations/layer0_attention_weight_min": -14.982337951660156,
      "activations/layer10_attention_weight_max": 44.55070877075195,
      "activations/layer10_attention_weight_min": -35.44324493408203,
      "activations/layer11_attention_weight_max": 38.34001922607422,
      "activations/layer11_attention_weight_min": -28.741504669189453,
      "activations/layer12_attention_weight_max": 28.316179275512695,
      "activations/layer12_attention_weight_min": -25.706613540649414,
      "activations/layer13_attention_weight_max": 54.998924255371094,
      "activations/layer13_attention_weight_min": -39.663570404052734,
      "activations/layer14_attention_weight_max": 51.04319381713867,
      "activations/layer14_attention_weight_min": -32.201019287109375,
      "activations/layer15_attention_weight_max": 46.711727142333984,
      "activations/layer15_attention_weight_min": -31.710214614868164,
      "activations/layer16_attention_weight_max": 39.561737060546875,
      "activations/layer16_attention_weight_min": -25.998308181762695,
      "activations/layer17_attention_weight_max": 61.06073760986328,
      "activations/layer17_attention_weight_min": -36.97837448120117,
      "activations/layer18_attention_weight_max": 52.38656234741211,
      "activations/layer18_attention_weight_min": -31.810749053955078,
      "activations/layer19_attention_weight_max": 23.5711612701416,
      "activations/layer19_attention_weight_min": -17.6197452545166,
      "activations/layer1_attention_weight_max": 17.04621124267578,
      "activations/layer1_attention_weight_min": -15.346219062805176,
      "activations/layer20_attention_weight_max": 21.127119064331055,
      "activations/layer20_attention_weight_min": -19.01085090637207,
      "activations/layer21_attention_weight_max": 45.95573806762695,
      "activations/layer21_attention_weight_min": -26.73028564453125,
      "activations/layer22_attention_weight_max": 36.50871658325195,
      "activations/layer22_attention_weight_min": -21.429201126098633,
      "activations/layer23_attention_weight_max": 27.890533447265625,
      "activations/layer23_attention_weight_min": -15.343832969665527,
      "activations/layer2_attention_weight_max": 28.176513671875,
      "activations/layer2_attention_weight_min": -29.527252197265625,
      "activations/layer3_attention_weight_max": 67.80032348632812,
      "activations/layer3_attention_weight_min": -69.0947265625,
      "activations/layer4_attention_weight_max": 103.04328918457031,
      "activations/layer4_attention_weight_min": -87.0739517211914,
      "activations/layer5_attention_weight_max": 128.50009155273438,
      "activations/layer5_attention_weight_min": -105.13015747070312,
      "activations/layer6_attention_weight_max": 61.599098205566406,
      "activations/layer6_attention_weight_min": -52.2982063293457,
      "activations/layer7_attention_weight_max": 57.51557159423828,
      "activations/layer7_attention_weight_min": -53.432708740234375,
      "activations/layer8_attention_weight_max": 37.79323959350586,
      "activations/layer8_attention_weight_min": -34.47158432006836,
      "activations/layer9_attention_weight_max": 48.34300231933594,
      "activations/layer9_attention_weight_min": -50.32312774658203,
      "epoch": 2.87,
      "learning_rate": 0.00013280681818181818,
      "loss": 2.9182,
      "step": 49450
    },
    {
      "activations/layer0_attention_weight_max": 14.069132804870605,
      "activations/layer0_attention_weight_min": -14.618203163146973,
      "activations/layer10_attention_weight_max": 47.55241775512695,
      "activations/layer10_attention_weight_min": -35.36818313598633,
      "activations/layer11_attention_weight_max": 38.20112609863281,
      "activations/layer11_attention_weight_min": -29.516767501831055,
      "activations/layer12_attention_weight_max": 27.56787872314453,
      "activations/layer12_attention_weight_min": -23.693592071533203,
      "activations/layer13_attention_weight_max": 67.13634490966797,
      "activations/layer13_attention_weight_min": -40.47550964355469,
      "activations/layer14_attention_weight_max": 59.10112380981445,
      "activations/layer14_attention_weight_min": -34.688934326171875,
      "activations/layer15_attention_weight_max": 53.81483459472656,
      "activations/layer15_attention_weight_min": -34.264068603515625,
      "activations/layer16_attention_weight_max": 37.65800094604492,
      "activations/layer16_attention_weight_min": -27.664262771606445,
      "activations/layer17_attention_weight_max": 70.30659484863281,
      "activations/layer17_attention_weight_min": -41.93083190917969,
      "activations/layer18_attention_weight_max": 63.45655059814453,
      "activations/layer18_attention_weight_min": -31.885290145874023,
      "activations/layer19_attention_weight_max": 22.63823127746582,
      "activations/layer19_attention_weight_min": -21.380380630493164,
      "activations/layer1_attention_weight_max": 17.94474983215332,
      "activations/layer1_attention_weight_min": -14.873811721801758,
      "activations/layer20_attention_weight_max": 24.278417587280273,
      "activations/layer20_attention_weight_min": -20.73416519165039,
      "activations/layer21_attention_weight_max": 50.42387008666992,
      "activations/layer21_attention_weight_min": -27.087486267089844,
      "activations/layer22_attention_weight_max": 39.92316818237305,
      "activations/layer22_attention_weight_min": -22.759511947631836,
      "activations/layer23_attention_weight_max": 28.033329010009766,
      "activations/layer23_attention_weight_min": -16.80276870727539,
      "activations/layer2_attention_weight_max": 28.020490646362305,
      "activations/layer2_attention_weight_min": -27.071212768554688,
      "activations/layer3_attention_weight_max": 69.17337799072266,
      "activations/layer3_attention_weight_min": -72.27662658691406,
      "activations/layer4_attention_weight_max": 94.81016540527344,
      "activations/layer4_attention_weight_min": -86.42180633544922,
      "activations/layer5_attention_weight_max": 124.34748840332031,
      "activations/layer5_attention_weight_min": -96.3967514038086,
      "activations/layer6_attention_weight_max": 61.683433532714844,
      "activations/layer6_attention_weight_min": -48.44864273071289,
      "activations/layer7_attention_weight_max": 60.438804626464844,
      "activations/layer7_attention_weight_min": -51.2044677734375,
      "activations/layer8_attention_weight_max": 40.59842300415039,
      "activations/layer8_attention_weight_min": -34.31557083129883,
      "activations/layer9_attention_weight_max": 54.06220245361328,
      "activations/layer9_attention_weight_min": -47.511566162109375,
      "epoch": 2.88,
      "learning_rate": 0.00013278787878787877,
      "loss": 2.9188,
      "step": 49500
    },
    {
      "activations/layer0_attention_weight_max": 14.270474433898926,
      "activations/layer0_attention_weight_min": -15.244504928588867,
      "activations/layer10_attention_weight_max": 47.720245361328125,
      "activations/layer10_attention_weight_min": -39.056488037109375,
      "activations/layer11_attention_weight_max": 38.41327667236328,
      "activations/layer11_attention_weight_min": -29.18012237548828,
      "activations/layer12_attention_weight_max": 25.943832397460938,
      "activations/layer12_attention_weight_min": -26.33454704284668,
      "activations/layer13_attention_weight_max": 54.95417404174805,
      "activations/layer13_attention_weight_min": -42.46660614013672,
      "activations/layer14_attention_weight_max": 44.26057052612305,
      "activations/layer14_attention_weight_min": -34.52229690551758,
      "activations/layer15_attention_weight_max": 48.73585891723633,
      "activations/layer15_attention_weight_min": -35.153114318847656,
      "activations/layer16_attention_weight_max": 40.61225128173828,
      "activations/layer16_attention_weight_min": -27.05186653137207,
      "activations/layer17_attention_weight_max": 69.78836059570312,
      "activations/layer17_attention_weight_min": -36.5823860168457,
      "activations/layer18_attention_weight_max": 62.14850997924805,
      "activations/layer18_attention_weight_min": -33.07350540161133,
      "activations/layer19_attention_weight_max": 25.05483055114746,
      "activations/layer19_attention_weight_min": -16.417888641357422,
      "activations/layer1_attention_weight_max": 17.413320541381836,
      "activations/layer1_attention_weight_min": -15.300713539123535,
      "activations/layer20_attention_weight_max": 22.2713623046875,
      "activations/layer20_attention_weight_min": -20.334686279296875,
      "activations/layer21_attention_weight_max": 40.56947708129883,
      "activations/layer21_attention_weight_min": -28.142263412475586,
      "activations/layer22_attention_weight_max": 37.73297882080078,
      "activations/layer22_attention_weight_min": -23.911846160888672,
      "activations/layer23_attention_weight_max": 25.04842185974121,
      "activations/layer23_attention_weight_min": -15.603265762329102,
      "activations/layer2_attention_weight_max": 29.65456199645996,
      "activations/layer2_attention_weight_min": -28.164608001708984,
      "activations/layer3_attention_weight_max": 69.66249084472656,
      "activations/layer3_attention_weight_min": -75.89824676513672,
      "activations/layer4_attention_weight_max": 98.80801391601562,
      "activations/layer4_attention_weight_min": -96.902099609375,
      "activations/layer5_attention_weight_max": 120.48204803466797,
      "activations/layer5_attention_weight_min": -104.02763366699219,
      "activations/layer6_attention_weight_max": 61.0167121887207,
      "activations/layer6_attention_weight_min": -53.1733283996582,
      "activations/layer7_attention_weight_max": 59.03268051147461,
      "activations/layer7_attention_weight_min": -57.56443786621094,
      "activations/layer8_attention_weight_max": 40.564876556396484,
      "activations/layer8_attention_weight_min": -34.62786102294922,
      "activations/layer9_attention_weight_max": 60.56874465942383,
      "activations/layer9_attention_weight_min": -49.27536392211914,
      "epoch": 2.88,
      "learning_rate": 0.00013276893939393939,
      "loss": 2.9452,
      "step": 49550
    },
    {
      "activations/layer0_attention_weight_max": 14.534295082092285,
      "activations/layer0_attention_weight_min": -14.469965934753418,
      "activations/layer10_attention_weight_max": 48.46073532104492,
      "activations/layer10_attention_weight_min": -37.132896423339844,
      "activations/layer11_attention_weight_max": 38.963623046875,
      "activations/layer11_attention_weight_min": -28.044723510742188,
      "activations/layer12_attention_weight_max": 29.23951530456543,
      "activations/layer12_attention_weight_min": -25.66350746154785,
      "activations/layer13_attention_weight_max": 56.45124435424805,
      "activations/layer13_attention_weight_min": -39.60410690307617,
      "activations/layer14_attention_weight_max": 55.473915100097656,
      "activations/layer14_attention_weight_min": -38.64173889160156,
      "activations/layer15_attention_weight_max": 51.2305908203125,
      "activations/layer15_attention_weight_min": -35.43485641479492,
      "activations/layer16_attention_weight_max": 38.55850601196289,
      "activations/layer16_attention_weight_min": -27.292787551879883,
      "activations/layer17_attention_weight_max": 68.26789093017578,
      "activations/layer17_attention_weight_min": -36.48509979248047,
      "activations/layer18_attention_weight_max": 59.50715637207031,
      "activations/layer18_attention_weight_min": -32.67095184326172,
      "activations/layer19_attention_weight_max": 21.58316421508789,
      "activations/layer19_attention_weight_min": -17.230098724365234,
      "activations/layer1_attention_weight_max": 17.246061325073242,
      "activations/layer1_attention_weight_min": -13.90839958190918,
      "activations/layer20_attention_weight_max": 22.915969848632812,
      "activations/layer20_attention_weight_min": -17.126972198486328,
      "activations/layer21_attention_weight_max": 44.18885803222656,
      "activations/layer21_attention_weight_min": -29.448101043701172,
      "activations/layer22_attention_weight_max": 35.898067474365234,
      "activations/layer22_attention_weight_min": -24.002445220947266,
      "activations/layer23_attention_weight_max": 28.633209228515625,
      "activations/layer23_attention_weight_min": -15.708971977233887,
      "activations/layer2_attention_weight_max": 28.902738571166992,
      "activations/layer2_attention_weight_min": -27.260494232177734,
      "activations/layer3_attention_weight_max": 70.30795288085938,
      "activations/layer3_attention_weight_min": -76.05709075927734,
      "activations/layer4_attention_weight_max": 100.02023315429688,
      "activations/layer4_attention_weight_min": -97.3533935546875,
      "activations/layer5_attention_weight_max": 134.68516540527344,
      "activations/layer5_attention_weight_min": -103.39859008789062,
      "activations/layer6_attention_weight_max": 66.97644805908203,
      "activations/layer6_attention_weight_min": -51.445579528808594,
      "activations/layer7_attention_weight_max": 66.23834991455078,
      "activations/layer7_attention_weight_min": -52.15371322631836,
      "activations/layer8_attention_weight_max": 41.56005096435547,
      "activations/layer8_attention_weight_min": -35.05976486206055,
      "activations/layer9_attention_weight_max": 62.27585983276367,
      "activations/layer9_attention_weight_min": -54.92110061645508,
      "epoch": 2.88,
      "learning_rate": 0.00013275,
      "loss": 2.9092,
      "step": 49600
    },
    {
      "activations/layer0_attention_weight_max": 13.853079795837402,
      "activations/layer0_attention_weight_min": -15.002360343933105,
      "activations/layer10_attention_weight_max": 53.987274169921875,
      "activations/layer10_attention_weight_min": -38.04359817504883,
      "activations/layer11_attention_weight_max": 36.95378112792969,
      "activations/layer11_attention_weight_min": -29.03791046142578,
      "activations/layer12_attention_weight_max": 25.268754959106445,
      "activations/layer12_attention_weight_min": -23.44538116455078,
      "activations/layer13_attention_weight_max": 58.88018798828125,
      "activations/layer13_attention_weight_min": -43.712913513183594,
      "activations/layer14_attention_weight_max": 61.42036056518555,
      "activations/layer14_attention_weight_min": -41.14048385620117,
      "activations/layer15_attention_weight_max": 45.358551025390625,
      "activations/layer15_attention_weight_min": -32.107322692871094,
      "activations/layer16_attention_weight_max": 41.89328384399414,
      "activations/layer16_attention_weight_min": -25.27934455871582,
      "activations/layer17_attention_weight_max": 65.7862548828125,
      "activations/layer17_attention_weight_min": -39.617393493652344,
      "activations/layer18_attention_weight_max": 53.77896499633789,
      "activations/layer18_attention_weight_min": -31.902870178222656,
      "activations/layer19_attention_weight_max": 20.01259994506836,
      "activations/layer19_attention_weight_min": -16.66444206237793,
      "activations/layer1_attention_weight_max": 17.715925216674805,
      "activations/layer1_attention_weight_min": -16.253293991088867,
      "activations/layer20_attention_weight_max": 21.02686882019043,
      "activations/layer20_attention_weight_min": -17.278820037841797,
      "activations/layer21_attention_weight_max": 43.67867660522461,
      "activations/layer21_attention_weight_min": -30.078502655029297,
      "activations/layer22_attention_weight_max": 34.64716720581055,
      "activations/layer22_attention_weight_min": -23.852869033813477,
      "activations/layer23_attention_weight_max": 26.471609115600586,
      "activations/layer23_attention_weight_min": -15.429931640625,
      "activations/layer2_attention_weight_max": 29.957225799560547,
      "activations/layer2_attention_weight_min": -27.377208709716797,
      "activations/layer3_attention_weight_max": 71.95245361328125,
      "activations/layer3_attention_weight_min": -71.07904052734375,
      "activations/layer4_attention_weight_max": 103.0905532836914,
      "activations/layer4_attention_weight_min": -89.82781219482422,
      "activations/layer5_attention_weight_max": 133.4935302734375,
      "activations/layer5_attention_weight_min": -102.19972229003906,
      "activations/layer6_attention_weight_max": 66.24099731445312,
      "activations/layer6_attention_weight_min": -50.807212829589844,
      "activations/layer7_attention_weight_max": 62.1558837890625,
      "activations/layer7_attention_weight_min": -56.924278259277344,
      "activations/layer8_attention_weight_max": 44.716819763183594,
      "activations/layer8_attention_weight_min": -35.59706115722656,
      "activations/layer9_attention_weight_max": 58.22144317626953,
      "activations/layer9_attention_weight_min": -51.803680419921875,
      "epoch": 2.88,
      "learning_rate": 0.0001327310606060606,
      "loss": 2.9259,
      "step": 49650
    },
    {
      "activations/layer0_attention_weight_max": 14.089930534362793,
      "activations/layer0_attention_weight_min": -15.058311462402344,
      "activations/layer10_attention_weight_max": 47.03361892700195,
      "activations/layer10_attention_weight_min": -37.20592498779297,
      "activations/layer11_attention_weight_max": 36.108516693115234,
      "activations/layer11_attention_weight_min": -28.231605529785156,
      "activations/layer12_attention_weight_max": 28.99931526184082,
      "activations/layer12_attention_weight_min": -27.52533721923828,
      "activations/layer13_attention_weight_max": 58.46388626098633,
      "activations/layer13_attention_weight_min": -38.49324417114258,
      "activations/layer14_attention_weight_max": 58.27381896972656,
      "activations/layer14_attention_weight_min": -38.125038146972656,
      "activations/layer15_attention_weight_max": 47.64426803588867,
      "activations/layer15_attention_weight_min": -32.00868225097656,
      "activations/layer16_attention_weight_max": 42.2969856262207,
      "activations/layer16_attention_weight_min": -27.40937614440918,
      "activations/layer17_attention_weight_max": 63.87296676635742,
      "activations/layer17_attention_weight_min": -39.09865188598633,
      "activations/layer18_attention_weight_max": 56.18515396118164,
      "activations/layer18_attention_weight_min": -33.05975341796875,
      "activations/layer19_attention_weight_max": 22.757999420166016,
      "activations/layer19_attention_weight_min": -17.57879066467285,
      "activations/layer1_attention_weight_max": 17.109939575195312,
      "activations/layer1_attention_weight_min": -13.747149467468262,
      "activations/layer20_attention_weight_max": 23.09838104248047,
      "activations/layer20_attention_weight_min": -17.0819034576416,
      "activations/layer21_attention_weight_max": 47.649559020996094,
      "activations/layer21_attention_weight_min": -29.19792366027832,
      "activations/layer22_attention_weight_max": 43.78855514526367,
      "activations/layer22_attention_weight_min": -22.63048553466797,
      "activations/layer23_attention_weight_max": 26.939517974853516,
      "activations/layer23_attention_weight_min": -15.959577560424805,
      "activations/layer2_attention_weight_max": 29.642284393310547,
      "activations/layer2_attention_weight_min": -27.41802978515625,
      "activations/layer3_attention_weight_max": 71.18533325195312,
      "activations/layer3_attention_weight_min": -74.92741394042969,
      "activations/layer4_attention_weight_max": 102.07898712158203,
      "activations/layer4_attention_weight_min": -92.03185272216797,
      "activations/layer5_attention_weight_max": 123.07597351074219,
      "activations/layer5_attention_weight_min": -98.55870056152344,
      "activations/layer6_attention_weight_max": 59.60816955566406,
      "activations/layer6_attention_weight_min": -49.50483322143555,
      "activations/layer7_attention_weight_max": 57.8156623840332,
      "activations/layer7_attention_weight_min": -52.52396774291992,
      "activations/layer8_attention_weight_max": 39.272056579589844,
      "activations/layer8_attention_weight_min": -34.01567459106445,
      "activations/layer9_attention_weight_max": 52.6511116027832,
      "activations/layer9_attention_weight_min": -47.821571350097656,
      "epoch": 2.89,
      "learning_rate": 0.0001327121212121212,
      "loss": 2.9241,
      "step": 49700
    },
    {
      "activations/layer0_attention_weight_max": 15.284064292907715,
      "activations/layer0_attention_weight_min": -14.770596504211426,
      "activations/layer10_attention_weight_max": 45.37541198730469,
      "activations/layer10_attention_weight_min": -37.32929992675781,
      "activations/layer11_attention_weight_max": 43.41103744506836,
      "activations/layer11_attention_weight_min": -32.28581619262695,
      "activations/layer12_attention_weight_max": 32.163429260253906,
      "activations/layer12_attention_weight_min": -24.817060470581055,
      "activations/layer13_attention_weight_max": 70.9651870727539,
      "activations/layer13_attention_weight_min": -45.28971481323242,
      "activations/layer14_attention_weight_max": 86.57209777832031,
      "activations/layer14_attention_weight_min": -52.24164962768555,
      "activations/layer15_attention_weight_max": 56.60121536254883,
      "activations/layer15_attention_weight_min": -35.06742477416992,
      "activations/layer16_attention_weight_max": 43.72540283203125,
      "activations/layer16_attention_weight_min": -27.481210708618164,
      "activations/layer17_attention_weight_max": 61.77090072631836,
      "activations/layer17_attention_weight_min": -39.573028564453125,
      "activations/layer18_attention_weight_max": 57.78434753417969,
      "activations/layer18_attention_weight_min": -32.362205505371094,
      "activations/layer19_attention_weight_max": 26.212488174438477,
      "activations/layer19_attention_weight_min": -16.421598434448242,
      "activations/layer1_attention_weight_max": 19.553741455078125,
      "activations/layer1_attention_weight_min": -14.233037948608398,
      "activations/layer20_attention_weight_max": 22.025806427001953,
      "activations/layer20_attention_weight_min": -18.514741897583008,
      "activations/layer21_attention_weight_max": 41.41840362548828,
      "activations/layer21_attention_weight_min": -28.207401275634766,
      "activations/layer22_attention_weight_max": 35.21522521972656,
      "activations/layer22_attention_weight_min": -22.740816116333008,
      "activations/layer23_attention_weight_max": 25.388587951660156,
      "activations/layer23_attention_weight_min": -16.380407333374023,
      "activations/layer2_attention_weight_max": 27.062002182006836,
      "activations/layer2_attention_weight_min": -26.505966186523438,
      "activations/layer3_attention_weight_max": 67.31405639648438,
      "activations/layer3_attention_weight_min": -72.1572494506836,
      "activations/layer4_attention_weight_max": 101.57485961914062,
      "activations/layer4_attention_weight_min": -93.27117919921875,
      "activations/layer5_attention_weight_max": 125.48570251464844,
      "activations/layer5_attention_weight_min": -105.83028411865234,
      "activations/layer6_attention_weight_max": 62.6141242980957,
      "activations/layer6_attention_weight_min": -54.790931701660156,
      "activations/layer7_attention_weight_max": 73.68071746826172,
      "activations/layer7_attention_weight_min": -56.5128288269043,
      "activations/layer8_attention_weight_max": 55.68645477294922,
      "activations/layer8_attention_weight_min": -38.66919708251953,
      "activations/layer9_attention_weight_max": 80.34033203125,
      "activations/layer9_attention_weight_min": -58.54822540283203,
      "epoch": 2.89,
      "learning_rate": 0.00013269318181818182,
      "loss": 2.9058,
      "step": 49750
    },
    {
      "activations/layer0_attention_weight_max": 14.569443702697754,
      "activations/layer0_attention_weight_min": -14.326845169067383,
      "activations/layer10_attention_weight_max": 43.28998947143555,
      "activations/layer10_attention_weight_min": -35.27339553833008,
      "activations/layer11_attention_weight_max": 35.70793151855469,
      "activations/layer11_attention_weight_min": -27.145681381225586,
      "activations/layer12_attention_weight_max": 28.88103675842285,
      "activations/layer12_attention_weight_min": -24.06793975830078,
      "activations/layer13_attention_weight_max": 54.77848434448242,
      "activations/layer13_attention_weight_min": -37.7997932434082,
      "activations/layer14_attention_weight_max": 52.501708984375,
      "activations/layer14_attention_weight_min": -32.88582992553711,
      "activations/layer15_attention_weight_max": 50.58749771118164,
      "activations/layer15_attention_weight_min": -34.30059051513672,
      "activations/layer16_attention_weight_max": 38.4913330078125,
      "activations/layer16_attention_weight_min": -28.501436233520508,
      "activations/layer17_attention_weight_max": 66.74562072753906,
      "activations/layer17_attention_weight_min": -36.500484466552734,
      "activations/layer18_attention_weight_max": 55.67619323730469,
      "activations/layer18_attention_weight_min": -32.737762451171875,
      "activations/layer19_attention_weight_max": 25.810678482055664,
      "activations/layer19_attention_weight_min": -19.515846252441406,
      "activations/layer1_attention_weight_max": 18.1019287109375,
      "activations/layer1_attention_weight_min": -14.822400093078613,
      "activations/layer20_attention_weight_max": 23.510866165161133,
      "activations/layer20_attention_weight_min": -21.097402572631836,
      "activations/layer21_attention_weight_max": 42.79192352294922,
      "activations/layer21_attention_weight_min": -28.35636329650879,
      "activations/layer22_attention_weight_max": 37.13039779663086,
      "activations/layer22_attention_weight_min": -22.45354652404785,
      "activations/layer23_attention_weight_max": 26.744443893432617,
      "activations/layer23_attention_weight_min": -15.363384246826172,
      "activations/layer2_attention_weight_max": 28.431785583496094,
      "activations/layer2_attention_weight_min": -27.82136344909668,
      "activations/layer3_attention_weight_max": 66.19606018066406,
      "activations/layer3_attention_weight_min": -68.78800964355469,
      "activations/layer4_attention_weight_max": 96.10041809082031,
      "activations/layer4_attention_weight_min": -88.82878112792969,
      "activations/layer5_attention_weight_max": 123.0077133178711,
      "activations/layer5_attention_weight_min": -97.67216491699219,
      "activations/layer6_attention_weight_max": 63.24066925048828,
      "activations/layer6_attention_weight_min": -48.889747619628906,
      "activations/layer7_attention_weight_max": 56.88812255859375,
      "activations/layer7_attention_weight_min": -54.47294616699219,
      "activations/layer8_attention_weight_max": 37.515193939208984,
      "activations/layer8_attention_weight_min": -34.92753601074219,
      "activations/layer9_attention_weight_max": 52.71753692626953,
      "activations/layer9_attention_weight_min": -47.452938079833984,
      "epoch": 2.89,
      "learning_rate": 0.0001326742424242424,
      "loss": 2.9103,
      "step": 49800
    },
    {
      "activations/layer0_attention_weight_max": 14.52231502532959,
      "activations/layer0_attention_weight_min": -15.150386810302734,
      "activations/layer10_attention_weight_max": 46.68608474731445,
      "activations/layer10_attention_weight_min": -36.896392822265625,
      "activations/layer11_attention_weight_max": 40.747398376464844,
      "activations/layer11_attention_weight_min": -30.50499725341797,
      "activations/layer12_attention_weight_max": 32.253562927246094,
      "activations/layer12_attention_weight_min": -23.241748809814453,
      "activations/layer13_attention_weight_max": 57.56754684448242,
      "activations/layer13_attention_weight_min": -40.07789993286133,
      "activations/layer14_attention_weight_max": 46.980403900146484,
      "activations/layer14_attention_weight_min": -36.48746109008789,
      "activations/layer15_attention_weight_max": 53.79423141479492,
      "activations/layer15_attention_weight_min": -36.38304138183594,
      "activations/layer16_attention_weight_max": 40.56016159057617,
      "activations/layer16_attention_weight_min": -28.91434669494629,
      "activations/layer17_attention_weight_max": 60.24665069580078,
      "activations/layer17_attention_weight_min": -36.548011779785156,
      "activations/layer18_attention_weight_max": 63.691097259521484,
      "activations/layer18_attention_weight_min": -32.75505065917969,
      "activations/layer19_attention_weight_max": 21.645854949951172,
      "activations/layer19_attention_weight_min": -17.29395866394043,
      "activations/layer1_attention_weight_max": 17.769847869873047,
      "activations/layer1_attention_weight_min": -15.087737083435059,
      "activations/layer20_attention_weight_max": 19.8563289642334,
      "activations/layer20_attention_weight_min": -17.55974006652832,
      "activations/layer21_attention_weight_max": 49.73691177368164,
      "activations/layer21_attention_weight_min": -32.65470504760742,
      "activations/layer22_attention_weight_max": 36.31199645996094,
      "activations/layer22_attention_weight_min": -22.4713191986084,
      "activations/layer23_attention_weight_max": 26.48630142211914,
      "activations/layer23_attention_weight_min": -17.45514488220215,
      "activations/layer2_attention_weight_max": 29.196773529052734,
      "activations/layer2_attention_weight_min": -28.19122886657715,
      "activations/layer3_attention_weight_max": 70.34638977050781,
      "activations/layer3_attention_weight_min": -76.2421646118164,
      "activations/layer4_attention_weight_max": 98.77095794677734,
      "activations/layer4_attention_weight_min": -92.98831176757812,
      "activations/layer5_attention_weight_max": 122.927001953125,
      "activations/layer5_attention_weight_min": -103.37773132324219,
      "activations/layer6_attention_weight_max": 62.750003814697266,
      "activations/layer6_attention_weight_min": -51.35100555419922,
      "activations/layer7_attention_weight_max": 60.20248031616211,
      "activations/layer7_attention_weight_min": -56.53180694580078,
      "activations/layer8_attention_weight_max": 38.80356979370117,
      "activations/layer8_attention_weight_min": -34.5050048828125,
      "activations/layer9_attention_weight_max": 51.54342269897461,
      "activations/layer9_attention_weight_min": -48.679744720458984,
      "epoch": 2.9,
      "learning_rate": 0.00013265530303030302,
      "loss": 2.9235,
      "step": 49850
    },
    {
      "activations/layer0_attention_weight_max": 15.31887149810791,
      "activations/layer0_attention_weight_min": -14.461897850036621,
      "activations/layer10_attention_weight_max": 60.47775650024414,
      "activations/layer10_attention_weight_min": -39.599143981933594,
      "activations/layer11_attention_weight_max": 52.977779388427734,
      "activations/layer11_attention_weight_min": -34.173919677734375,
      "activations/layer12_attention_weight_max": 35.29450988769531,
      "activations/layer12_attention_weight_min": -23.6895694732666,
      "activations/layer13_attention_weight_max": 86.37422943115234,
      "activations/layer13_attention_weight_min": -51.87728500366211,
      "activations/layer14_attention_weight_max": 74.67386627197266,
      "activations/layer14_attention_weight_min": -42.96388626098633,
      "activations/layer15_attention_weight_max": 58.075660705566406,
      "activations/layer15_attention_weight_min": -36.63676452636719,
      "activations/layer16_attention_weight_max": 43.82971954345703,
      "activations/layer16_attention_weight_min": -25.85954475402832,
      "activations/layer17_attention_weight_max": 61.81867218017578,
      "activations/layer17_attention_weight_min": -37.9484748840332,
      "activations/layer18_attention_weight_max": 55.12651062011719,
      "activations/layer18_attention_weight_min": -33.451080322265625,
      "activations/layer19_attention_weight_max": 24.38613510131836,
      "activations/layer19_attention_weight_min": -17.260953903198242,
      "activations/layer1_attention_weight_max": 18.10535430908203,
      "activations/layer1_attention_weight_min": -14.796455383300781,
      "activations/layer20_attention_weight_max": 18.902271270751953,
      "activations/layer20_attention_weight_min": -19.857301712036133,
      "activations/layer21_attention_weight_max": 40.53349304199219,
      "activations/layer21_attention_weight_min": -25.949106216430664,
      "activations/layer22_attention_weight_max": 33.2413215637207,
      "activations/layer22_attention_weight_min": -21.18981170654297,
      "activations/layer23_attention_weight_max": 25.494625091552734,
      "activations/layer23_attention_weight_min": -14.67282485961914,
      "activations/layer2_attention_weight_max": 28.789756774902344,
      "activations/layer2_attention_weight_min": -27.852153778076172,
      "activations/layer3_attention_weight_max": 71.28165435791016,
      "activations/layer3_attention_weight_min": -74.81365966796875,
      "activations/layer4_attention_weight_max": 104.703125,
      "activations/layer4_attention_weight_min": -98.04251861572266,
      "activations/layer5_attention_weight_max": 128.33984375,
      "activations/layer5_attention_weight_min": -113.02838134765625,
      "activations/layer6_attention_weight_max": 63.84098434448242,
      "activations/layer6_attention_weight_min": -53.15226745605469,
      "activations/layer7_attention_weight_max": 69.13787841796875,
      "activations/layer7_attention_weight_min": -59.27962875366211,
      "activations/layer8_attention_weight_max": 52.399452209472656,
      "activations/layer8_attention_weight_min": -40.27323532104492,
      "activations/layer9_attention_weight_max": 73.48751068115234,
      "activations/layer9_attention_weight_min": -57.70893096923828,
      "epoch": 2.9,
      "learning_rate": 0.00013263636363636364,
      "loss": 2.9153,
      "step": 49900
    },
    {
      "activations/layer0_attention_weight_max": 13.947733879089355,
      "activations/layer0_attention_weight_min": -14.861482620239258,
      "activations/layer10_attention_weight_max": 51.858184814453125,
      "activations/layer10_attention_weight_min": -36.98814010620117,
      "activations/layer11_attention_weight_max": 39.50596618652344,
      "activations/layer11_attention_weight_min": -27.38433837890625,
      "activations/layer12_attention_weight_max": 27.115442276000977,
      "activations/layer12_attention_weight_min": -23.908700942993164,
      "activations/layer13_attention_weight_max": 56.81303787231445,
      "activations/layer13_attention_weight_min": -36.90876388549805,
      "activations/layer14_attention_weight_max": 55.819725036621094,
      "activations/layer14_attention_weight_min": -32.971412658691406,
      "activations/layer15_attention_weight_max": 49.12981033325195,
      "activations/layer15_attention_weight_min": -32.32209014892578,
      "activations/layer16_attention_weight_max": 35.32619094848633,
      "activations/layer16_attention_weight_min": -25.73752212524414,
      "activations/layer17_attention_weight_max": 60.49153137207031,
      "activations/layer17_attention_weight_min": -36.01458740234375,
      "activations/layer18_attention_weight_max": 60.5130615234375,
      "activations/layer18_attention_weight_min": -33.89425277709961,
      "activations/layer19_attention_weight_max": 24.227954864501953,
      "activations/layer19_attention_weight_min": -17.020971298217773,
      "activations/layer1_attention_weight_max": 17.70537567138672,
      "activations/layer1_attention_weight_min": -15.169607162475586,
      "activations/layer20_attention_weight_max": 22.959989547729492,
      "activations/layer20_attention_weight_min": -18.901844024658203,
      "activations/layer21_attention_weight_max": 44.946449279785156,
      "activations/layer21_attention_weight_min": -29.245952606201172,
      "activations/layer22_attention_weight_max": 36.627098083496094,
      "activations/layer22_attention_weight_min": -25.13356590270996,
      "activations/layer23_attention_weight_max": 27.724407196044922,
      "activations/layer23_attention_weight_min": -15.460819244384766,
      "activations/layer2_attention_weight_max": 30.37689971923828,
      "activations/layer2_attention_weight_min": -28.926902770996094,
      "activations/layer3_attention_weight_max": 73.20545196533203,
      "activations/layer3_attention_weight_min": -77.26567077636719,
      "activations/layer4_attention_weight_max": 101.63855743408203,
      "activations/layer4_attention_weight_min": -95.27757263183594,
      "activations/layer5_attention_weight_max": 126.2206039428711,
      "activations/layer5_attention_weight_min": -101.02749633789062,
      "activations/layer6_attention_weight_max": 64.54682922363281,
      "activations/layer6_attention_weight_min": -52.54074478149414,
      "activations/layer7_attention_weight_max": 58.138919830322266,
      "activations/layer7_attention_weight_min": -55.695579528808594,
      "activations/layer8_attention_weight_max": 42.39115905761719,
      "activations/layer8_attention_weight_min": -35.42430877685547,
      "activations/layer9_attention_weight_max": 51.00870132446289,
      "activations/layer9_attention_weight_min": -45.92564392089844,
      "epoch": 2.9,
      "learning_rate": 0.00013261742424242423,
      "loss": 2.9273,
      "step": 49950
    },
    {
      "activations/layer0_attention_weight_max": 14.124056816101074,
      "activations/layer0_attention_weight_min": -15.000011444091797,
      "activations/layer10_attention_weight_max": 44.66412353515625,
      "activations/layer10_attention_weight_min": -35.98894119262695,
      "activations/layer11_attention_weight_max": 36.82128143310547,
      "activations/layer11_attention_weight_min": -27.47228240966797,
      "activations/layer12_attention_weight_max": 25.28106117248535,
      "activations/layer12_attention_weight_min": -25.45201873779297,
      "activations/layer13_attention_weight_max": 53.73780822753906,
      "activations/layer13_attention_weight_min": -35.1767578125,
      "activations/layer14_attention_weight_max": 48.077205657958984,
      "activations/layer14_attention_weight_min": -34.0595817565918,
      "activations/layer15_attention_weight_max": 46.81792449951172,
      "activations/layer15_attention_weight_min": -32.84972381591797,
      "activations/layer16_attention_weight_max": 32.27201843261719,
      "activations/layer16_attention_weight_min": -25.066377639770508,
      "activations/layer17_attention_weight_max": 57.86098098754883,
      "activations/layer17_attention_weight_min": -33.374820709228516,
      "activations/layer18_attention_weight_max": 51.36594009399414,
      "activations/layer18_attention_weight_min": -30.5413875579834,
      "activations/layer19_attention_weight_max": 20.816200256347656,
      "activations/layer19_attention_weight_min": -17.995105743408203,
      "activations/layer1_attention_weight_max": 18.104991912841797,
      "activations/layer1_attention_weight_min": -13.071616172790527,
      "activations/layer20_attention_weight_max": 20.592239379882812,
      "activations/layer20_attention_weight_min": -20.16141128540039,
      "activations/layer21_attention_weight_max": 37.521629333496094,
      "activations/layer21_attention_weight_min": -28.263187408447266,
      "activations/layer22_attention_weight_max": 31.825708389282227,
      "activations/layer22_attention_weight_min": -23.326231002807617,
      "activations/layer23_attention_weight_max": 25.945425033569336,
      "activations/layer23_attention_weight_min": -15.65625286102295,
      "activations/layer2_attention_weight_max": 31.696090698242188,
      "activations/layer2_attention_weight_min": -30.206907272338867,
      "activations/layer3_attention_weight_max": 68.93695831298828,
      "activations/layer3_attention_weight_min": -73.37553405761719,
      "activations/layer4_attention_weight_max": 99.65291595458984,
      "activations/layer4_attention_weight_min": -91.58784484863281,
      "activations/layer5_attention_weight_max": 123.07554626464844,
      "activations/layer5_attention_weight_min": -106.09871673583984,
      "activations/layer6_attention_weight_max": 62.35922622680664,
      "activations/layer6_attention_weight_min": -53.52524185180664,
      "activations/layer7_attention_weight_max": 55.450462341308594,
      "activations/layer7_attention_weight_min": -55.18659210205078,
      "activations/layer8_attention_weight_max": 41.208309173583984,
      "activations/layer8_attention_weight_min": -34.27522659301758,
      "activations/layer9_attention_weight_max": 49.907020568847656,
      "activations/layer9_attention_weight_min": -48.286319732666016,
      "epoch": 2.91,
      "learning_rate": 0.00013259848484848484,
      "loss": 2.9125,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_loss": 2.86328125,
      "eval_runtime": 8.4633,
      "eval_samples_per_second": 507.366,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_openwebtext_loss": 2.86328125,
      "eval_openwebtext_ppl": 17.51891667538985,
      "eval_openwebtext_runtime": 8.4633,
      "eval_openwebtext_samples_per_second": 507.366,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_wikitext_loss": 3.1171875,
      "eval_wikitext_ppl": 22.582776184718522,
      "eval_wikitext_runtime": 1.9926,
      "eval_wikitext_samples_per_second": 228.847,
      "step": 50000
    },
    {
      "epoch": 2.91,
      "eval_lambada_loss": 3.056640625,
      "eval_lambada_ppl": 21.256030110048442,
      "eval_lambada_runtime": 9.5463,
      "eval_lambada_samples_per_second": 510.041,
      "step": 50000
    },
    {
      "activations/layer0_attention_weight_max": 15.999788284301758,
      "activations/layer0_attention_weight_min": -14.964816093444824,
      "activations/layer10_attention_weight_max": 49.19907760620117,
      "activations/layer10_attention_weight_min": -38.60679244995117,
      "activations/layer11_attention_weight_max": 37.2589111328125,
      "activations/layer11_attention_weight_min": -27.7938232421875,
      "activations/layer12_attention_weight_max": 27.743854522705078,
      "activations/layer12_attention_weight_min": -24.96656036376953,
      "activations/layer13_attention_weight_max": 52.50322341918945,
      "activations/layer13_attention_weight_min": -39.464988708496094,
      "activations/layer14_attention_weight_max": 42.93596267700195,
      "activations/layer14_attention_weight_min": -31.545581817626953,
      "activations/layer15_attention_weight_max": 51.48133087158203,
      "activations/layer15_attention_weight_min": -33.214969635009766,
      "activations/layer16_attention_weight_max": 43.98247146606445,
      "activations/layer16_attention_weight_min": -30.56760597229004,
      "activations/layer17_attention_weight_max": 61.84099197387695,
      "activations/layer17_attention_weight_min": -37.501426696777344,
      "activations/layer18_attention_weight_max": 50.99869155883789,
      "activations/layer18_attention_weight_min": -33.930606842041016,
      "activations/layer19_attention_weight_max": 26.163602828979492,
      "activations/layer19_attention_weight_min": -18.035663604736328,
      "activations/layer1_attention_weight_max": 17.989864349365234,
      "activations/layer1_attention_weight_min": -15.680728912353516,
      "activations/layer20_attention_weight_max": 23.732969284057617,
      "activations/layer20_attention_weight_min": -20.785308837890625,
      "activations/layer21_attention_weight_max": 42.3885498046875,
      "activations/layer21_attention_weight_min": -29.198503494262695,
      "activations/layer22_attention_weight_max": 37.74574279785156,
      "activations/layer22_attention_weight_min": -25.10782814025879,
      "activations/layer23_attention_weight_max": 26.886701583862305,
      "activations/layer23_attention_weight_min": -16.71286392211914,
      "activations/layer2_attention_weight_max": 29.408981323242188,
      "activations/layer2_attention_weight_min": -28.010765075683594,
      "activations/layer3_attention_weight_max": 71.12671661376953,
      "activations/layer3_attention_weight_min": -73.79622650146484,
      "activations/layer4_attention_weight_max": 101.06663513183594,
      "activations/layer4_attention_weight_min": -91.29059600830078,
      "activations/layer5_attention_weight_max": 123.52926635742188,
      "activations/layer5_attention_weight_min": -102.0047836303711,
      "activations/layer6_attention_weight_max": 63.79368209838867,
      "activations/layer6_attention_weight_min": -52.02863693237305,
      "activations/layer7_attention_weight_max": 57.02782440185547,
      "activations/layer7_attention_weight_min": -53.10128402709961,
      "activations/layer8_attention_weight_max": 42.739967346191406,
      "activations/layer8_attention_weight_min": -35.75325393676758,
      "activations/layer9_attention_weight_max": 56.43648910522461,
      "activations/layer9_attention_weight_min": -47.24379348754883,
      "epoch": 2.91,
      "learning_rate": 0.00013257954545454546,
      "loss": 2.9331,
      "step": 50050
    },
    {
      "activations/layer0_attention_weight_max": 14.920466423034668,
      "activations/layer0_attention_weight_min": -15.074474334716797,
      "activations/layer10_attention_weight_max": 49.22300720214844,
      "activations/layer10_attention_weight_min": -38.70256805419922,
      "activations/layer11_attention_weight_max": 37.11186981201172,
      "activations/layer11_attention_weight_min": -28.12934684753418,
      "activations/layer12_attention_weight_max": 25.21996307373047,
      "activations/layer12_attention_weight_min": -26.120365142822266,
      "activations/layer13_attention_weight_max": 61.20780944824219,
      "activations/layer13_attention_weight_min": -44.30283737182617,
      "activations/layer14_attention_weight_max": 63.064544677734375,
      "activations/layer14_attention_weight_min": -37.82581329345703,
      "activations/layer15_attention_weight_max": 49.005348205566406,
      "activations/layer15_attention_weight_min": -33.50612258911133,
      "activations/layer16_attention_weight_max": 42.140716552734375,
      "activations/layer16_attention_weight_min": -26.215198516845703,
      "activations/layer17_attention_weight_max": 63.83208084106445,
      "activations/layer17_attention_weight_min": -36.23438262939453,
      "activations/layer18_attention_weight_max": 56.024871826171875,
      "activations/layer18_attention_weight_min": -32.996681213378906,
      "activations/layer19_attention_weight_max": 20.863174438476562,
      "activations/layer19_attention_weight_min": -18.7608642578125,
      "activations/layer1_attention_weight_max": 17.072736740112305,
      "activations/layer1_attention_weight_min": -13.681158065795898,
      "activations/layer20_attention_weight_max": 21.402746200561523,
      "activations/layer20_attention_weight_min": -19.77478790283203,
      "activations/layer21_attention_weight_max": 39.742431640625,
      "activations/layer21_attention_weight_min": -25.69102668762207,
      "activations/layer22_attention_weight_max": 34.551979064941406,
      "activations/layer22_attention_weight_min": -22.999828338623047,
      "activations/layer23_attention_weight_max": 27.074932098388672,
      "activations/layer23_attention_weight_min": -18.6197452545166,
      "activations/layer2_attention_weight_max": 27.951414108276367,
      "activations/layer2_attention_weight_min": -27.390277862548828,
      "activations/layer3_attention_weight_max": 73.58673858642578,
      "activations/layer3_attention_weight_min": -76.16393280029297,
      "activations/layer4_attention_weight_max": 102.87310028076172,
      "activations/layer4_attention_weight_min": -94.04006958007812,
      "activations/layer5_attention_weight_max": 123.2490234375,
      "activations/layer5_attention_weight_min": -103.24954223632812,
      "activations/layer6_attention_weight_max": 62.15250015258789,
      "activations/layer6_attention_weight_min": -53.1181755065918,
      "activations/layer7_attention_weight_max": 58.265926361083984,
      "activations/layer7_attention_weight_min": -54.7002067565918,
      "activations/layer8_attention_weight_max": 37.40145492553711,
      "activations/layer8_attention_weight_min": -35.03816604614258,
      "activations/layer9_attention_weight_max": 53.090450286865234,
      "activations/layer9_attention_weight_min": -50.3767204284668,
      "epoch": 2.91,
      "learning_rate": 0.00013256060606060604,
      "loss": 2.8998,
      "step": 50100
    },
    {
      "activations/layer0_attention_weight_max": 14.604635238647461,
      "activations/layer0_attention_weight_min": -15.336860656738281,
      "activations/layer10_attention_weight_max": 45.71575927734375,
      "activations/layer10_attention_weight_min": -36.54399490356445,
      "activations/layer11_attention_weight_max": 34.15631866455078,
      "activations/layer11_attention_weight_min": -30.15880584716797,
      "activations/layer12_attention_weight_max": 26.734188079833984,
      "activations/layer12_attention_weight_min": -24.697824478149414,
      "activations/layer13_attention_weight_max": 58.52874755859375,
      "activations/layer13_attention_weight_min": -39.878841400146484,
      "activations/layer14_attention_weight_max": 47.497093200683594,
      "activations/layer14_attention_weight_min": -31.391860961914062,
      "activations/layer15_attention_weight_max": 55.40398406982422,
      "activations/layer15_attention_weight_min": -35.07453918457031,
      "activations/layer16_attention_weight_max": 48.16291809082031,
      "activations/layer16_attention_weight_min": -26.918270111083984,
      "activations/layer17_attention_weight_max": 68.49241638183594,
      "activations/layer17_attention_weight_min": -37.12649154663086,
      "activations/layer18_attention_weight_max": 56.249794006347656,
      "activations/layer18_attention_weight_min": -32.96560287475586,
      "activations/layer19_attention_weight_max": 25.00691032409668,
      "activations/layer19_attention_weight_min": -16.705839157104492,
      "activations/layer1_attention_weight_max": 17.610389709472656,
      "activations/layer1_attention_weight_min": -15.532581329345703,
      "activations/layer20_attention_weight_max": 21.104736328125,
      "activations/layer20_attention_weight_min": -19.392465591430664,
      "activations/layer21_attention_weight_max": 44.727867126464844,
      "activations/layer21_attention_weight_min": -33.121070861816406,
      "activations/layer22_attention_weight_max": 42.1937370300293,
      "activations/layer22_attention_weight_min": -23.612289428710938,
      "activations/layer23_attention_weight_max": 31.288558959960938,
      "activations/layer23_attention_weight_min": -16.14704704284668,
      "activations/layer2_attention_weight_max": 29.305334091186523,
      "activations/layer2_attention_weight_min": -26.635589599609375,
      "activations/layer3_attention_weight_max": 74.13775634765625,
      "activations/layer3_attention_weight_min": -73.25942993164062,
      "activations/layer4_attention_weight_max": 97.67422485351562,
      "activations/layer4_attention_weight_min": -91.1401596069336,
      "activations/layer5_attention_weight_max": 124.93084716796875,
      "activations/layer5_attention_weight_min": -105.2193603515625,
      "activations/layer6_attention_weight_max": 62.522918701171875,
      "activations/layer6_attention_weight_min": -53.95670700073242,
      "activations/layer7_attention_weight_max": 61.602500915527344,
      "activations/layer7_attention_weight_min": -57.65806579589844,
      "activations/layer8_attention_weight_max": 40.302764892578125,
      "activations/layer8_attention_weight_min": -36.117713928222656,
      "activations/layer9_attention_weight_max": 51.19065475463867,
      "activations/layer9_attention_weight_min": -51.55398178100586,
      "epoch": 2.91,
      "learning_rate": 0.00013254166666666666,
      "loss": 2.9109,
      "step": 50150
    },
    {
      "activations/layer0_attention_weight_max": 13.684158325195312,
      "activations/layer0_attention_weight_min": -15.168966293334961,
      "activations/layer10_attention_weight_max": 43.96499252319336,
      "activations/layer10_attention_weight_min": -36.24319839477539,
      "activations/layer11_attention_weight_max": 35.242347717285156,
      "activations/layer11_attention_weight_min": -29.453460693359375,
      "activations/layer12_attention_weight_max": 24.27557945251465,
      "activations/layer12_attention_weight_min": -25.5755672454834,
      "activations/layer13_attention_weight_max": 54.43366622924805,
      "activations/layer13_attention_weight_min": -42.16425323486328,
      "activations/layer14_attention_weight_max": 45.89869689941406,
      "activations/layer14_attention_weight_min": -36.927974700927734,
      "activations/layer15_attention_weight_max": 43.15468215942383,
      "activations/layer15_attention_weight_min": -34.93235397338867,
      "activations/layer16_attention_weight_max": 36.263492584228516,
      "activations/layer16_attention_weight_min": -25.341327667236328,
      "activations/layer17_attention_weight_max": 63.15801239013672,
      "activations/layer17_attention_weight_min": -34.985408782958984,
      "activations/layer18_attention_weight_max": 56.50094985961914,
      "activations/layer18_attention_weight_min": -33.621761322021484,
      "activations/layer19_attention_weight_max": 23.187837600708008,
      "activations/layer19_attention_weight_min": -21.329984664916992,
      "activations/layer1_attention_weight_max": 17.40505027770996,
      "activations/layer1_attention_weight_min": -14.728381156921387,
      "activations/layer20_attention_weight_max": 25.085847854614258,
      "activations/layer20_attention_weight_min": -19.269428253173828,
      "activations/layer21_attention_weight_max": 41.89446258544922,
      "activations/layer21_attention_weight_min": -29.081817626953125,
      "activations/layer22_attention_weight_max": 36.21928405761719,
      "activations/layer22_attention_weight_min": -23.72029685974121,
      "activations/layer23_attention_weight_max": 29.529685974121094,
      "activations/layer23_attention_weight_min": -16.668682098388672,
      "activations/layer2_attention_weight_max": 26.946380615234375,
      "activations/layer2_attention_weight_min": -25.759601593017578,
      "activations/layer3_attention_weight_max": 69.28093719482422,
      "activations/layer3_attention_weight_min": -73.6175765991211,
      "activations/layer4_attention_weight_max": 94.35181427001953,
      "activations/layer4_attention_weight_min": -88.05965423583984,
      "activations/layer5_attention_weight_max": 118.70417785644531,
      "activations/layer5_attention_weight_min": -98.11116027832031,
      "activations/layer6_attention_weight_max": 61.29826354980469,
      "activations/layer6_attention_weight_min": -51.21062469482422,
      "activations/layer7_attention_weight_max": 55.77111053466797,
      "activations/layer7_attention_weight_min": -56.257503509521484,
      "activations/layer8_attention_weight_max": 40.199317932128906,
      "activations/layer8_attention_weight_min": -33.62555694580078,
      "activations/layer9_attention_weight_max": 53.78175735473633,
      "activations/layer9_attention_weight_min": -50.246517181396484,
      "epoch": 2.92,
      "learning_rate": 0.00013252272727272727,
      "loss": 2.9284,
      "step": 50200
    },
    {
      "activations/layer0_attention_weight_max": 14.476096153259277,
      "activations/layer0_attention_weight_min": -15.647261619567871,
      "activations/layer10_attention_weight_max": 41.8588981628418,
      "activations/layer10_attention_weight_min": -36.91352081298828,
      "activations/layer11_attention_weight_max": 31.873855590820312,
      "activations/layer11_attention_weight_min": -28.658788681030273,
      "activations/layer12_attention_weight_max": 23.37169075012207,
      "activations/layer12_attention_weight_min": -23.12824249267578,
      "activations/layer13_attention_weight_max": 54.217628479003906,
      "activations/layer13_attention_weight_min": -40.20711135864258,
      "activations/layer14_attention_weight_max": 44.828128814697266,
      "activations/layer14_attention_weight_min": -32.42205810546875,
      "activations/layer15_attention_weight_max": 43.201271057128906,
      "activations/layer15_attention_weight_min": -32.99813461303711,
      "activations/layer16_attention_weight_max": 41.543724060058594,
      "activations/layer16_attention_weight_min": -27.545307159423828,
      "activations/layer17_attention_weight_max": 62.20956802368164,
      "activations/layer17_attention_weight_min": -35.620670318603516,
      "activations/layer18_attention_weight_max": 52.31529998779297,
      "activations/layer18_attention_weight_min": -32.755645751953125,
      "activations/layer19_attention_weight_max": 21.80767250061035,
      "activations/layer19_attention_weight_min": -17.467161178588867,
      "activations/layer1_attention_weight_max": 17.686546325683594,
      "activations/layer1_attention_weight_min": -14.894137382507324,
      "activations/layer20_attention_weight_max": 22.033166885375977,
      "activations/layer20_attention_weight_min": -18.862247467041016,
      "activations/layer21_attention_weight_max": 43.979923248291016,
      "activations/layer21_attention_weight_min": -27.175161361694336,
      "activations/layer22_attention_weight_max": 37.04240417480469,
      "activations/layer22_attention_weight_min": -22.909648895263672,
      "activations/layer23_attention_weight_max": 24.05860710144043,
      "activations/layer23_attention_weight_min": -15.736818313598633,
      "activations/layer2_attention_weight_max": 27.339414596557617,
      "activations/layer2_attention_weight_min": -25.93505096435547,
      "activations/layer3_attention_weight_max": 68.76271057128906,
      "activations/layer3_attention_weight_min": -73.69368743896484,
      "activations/layer4_attention_weight_max": 97.97325897216797,
      "activations/layer4_attention_weight_min": -91.17903900146484,
      "activations/layer5_attention_weight_max": 121.15608978271484,
      "activations/layer5_attention_weight_min": -99.00685119628906,
      "activations/layer6_attention_weight_max": 61.845916748046875,
      "activations/layer6_attention_weight_min": -48.70985412597656,
      "activations/layer7_attention_weight_max": 57.98333740234375,
      "activations/layer7_attention_weight_min": -52.733943939208984,
      "activations/layer8_attention_weight_max": 40.91524124145508,
      "activations/layer8_attention_weight_min": -33.54570770263672,
      "activations/layer9_attention_weight_max": 48.741451263427734,
      "activations/layer9_attention_weight_min": -47.949302673339844,
      "epoch": 2.92,
      "learning_rate": 0.00013250416666666666,
      "loss": 2.9128,
      "step": 50250
    },
    {
      "activations/layer0_attention_weight_max": 15.238994598388672,
      "activations/layer0_attention_weight_min": -14.560162544250488,
      "activations/layer10_attention_weight_max": 46.34791564941406,
      "activations/layer10_attention_weight_min": -35.036861419677734,
      "activations/layer11_attention_weight_max": 37.16412353515625,
      "activations/layer11_attention_weight_min": -27.39613151550293,
      "activations/layer12_attention_weight_max": 23.95993995666504,
      "activations/layer12_attention_weight_min": -28.766324996948242,
      "activations/layer13_attention_weight_max": 49.97574234008789,
      "activations/layer13_attention_weight_min": -37.6566047668457,
      "activations/layer14_attention_weight_max": 41.75621795654297,
      "activations/layer14_attention_weight_min": -30.703659057617188,
      "activations/layer15_attention_weight_max": 43.91069793701172,
      "activations/layer15_attention_weight_min": -35.446083068847656,
      "activations/layer16_attention_weight_max": 37.67428970336914,
      "activations/layer16_attention_weight_min": -26.20832633972168,
      "activations/layer17_attention_weight_max": 58.08934783935547,
      "activations/layer17_attention_weight_min": -36.58409118652344,
      "activations/layer18_attention_weight_max": 54.515445709228516,
      "activations/layer18_attention_weight_min": -31.41502571105957,
      "activations/layer19_attention_weight_max": 23.203960418701172,
      "activations/layer19_attention_weight_min": -18.0526180267334,
      "activations/layer1_attention_weight_max": 18.358495712280273,
      "activations/layer1_attention_weight_min": -14.664474487304688,
      "activations/layer20_attention_weight_max": 22.045122146606445,
      "activations/layer20_attention_weight_min": -19.523738861083984,
      "activations/layer21_attention_weight_max": 39.67363739013672,
      "activations/layer21_attention_weight_min": -28.041921615600586,
      "activations/layer22_attention_weight_max": 33.69801330566406,
      "activations/layer22_attention_weight_min": -24.336803436279297,
      "activations/layer23_attention_weight_max": 27.03132438659668,
      "activations/layer23_attention_weight_min": -17.774290084838867,
      "activations/layer2_attention_weight_max": 27.758121490478516,
      "activations/layer2_attention_weight_min": -26.003170013427734,
      "activations/layer3_attention_weight_max": 68.40117645263672,
      "activations/layer3_attention_weight_min": -72.0589370727539,
      "activations/layer4_attention_weight_max": 97.67872619628906,
      "activations/layer4_attention_weight_min": -87.21868896484375,
      "activations/layer5_attention_weight_max": 120.01531219482422,
      "activations/layer5_attention_weight_min": -97.74978637695312,
      "activations/layer6_attention_weight_max": 58.83910369873047,
      "activations/layer6_attention_weight_min": -50.307254791259766,
      "activations/layer7_attention_weight_max": 61.0169677734375,
      "activations/layer7_attention_weight_min": -52.71731948852539,
      "activations/layer8_attention_weight_max": 38.993629455566406,
      "activations/layer8_attention_weight_min": -35.05769729614258,
      "activations/layer9_attention_weight_max": 56.449798583984375,
      "activations/layer9_attention_weight_min": -47.12844467163086,
      "epoch": 2.92,
      "learning_rate": 0.00013248522727272725,
      "loss": 2.9272,
      "step": 50300
    },
    {
      "activations/layer0_attention_weight_max": 15.097790718078613,
      "activations/layer0_attention_weight_min": -14.836956024169922,
      "activations/layer10_attention_weight_max": 46.73427200317383,
      "activations/layer10_attention_weight_min": -35.18110656738281,
      "activations/layer11_attention_weight_max": 42.04936981201172,
      "activations/layer11_attention_weight_min": -30.27699851989746,
      "activations/layer12_attention_weight_max": 25.277748107910156,
      "activations/layer12_attention_weight_min": -24.974740982055664,
      "activations/layer13_attention_weight_max": 58.80411911010742,
      "activations/layer13_attention_weight_min": -40.08941650390625,
      "activations/layer14_attention_weight_max": 55.98006820678711,
      "activations/layer14_attention_weight_min": -37.64961242675781,
      "activations/layer15_attention_weight_max": 50.20503234863281,
      "activations/layer15_attention_weight_min": -33.47113037109375,
      "activations/layer16_attention_weight_max": 38.05887222290039,
      "activations/layer16_attention_weight_min": -25.78990364074707,
      "activations/layer17_attention_weight_max": 54.02428436279297,
      "activations/layer17_attention_weight_min": -35.931556701660156,
      "activations/layer18_attention_weight_max": 52.55788040161133,
      "activations/layer18_attention_weight_min": -31.590608596801758,
      "activations/layer19_attention_weight_max": 22.012439727783203,
      "activations/layer19_attention_weight_min": -18.25153160095215,
      "activations/layer1_attention_weight_max": 18.263839721679688,
      "activations/layer1_attention_weight_min": -15.39903736114502,
      "activations/layer20_attention_weight_max": 21.805334091186523,
      "activations/layer20_attention_weight_min": -18.996183395385742,
      "activations/layer21_attention_weight_max": 37.8103141784668,
      "activations/layer21_attention_weight_min": -26.997129440307617,
      "activations/layer22_attention_weight_max": 35.34684753417969,
      "activations/layer22_attention_weight_min": -22.454004287719727,
      "activations/layer23_attention_weight_max": 22.2840576171875,
      "activations/layer23_attention_weight_min": -17.643739700317383,
      "activations/layer2_attention_weight_max": 28.296772003173828,
      "activations/layer2_attention_weight_min": -26.8791561126709,
      "activations/layer3_attention_weight_max": 72.9820785522461,
      "activations/layer3_attention_weight_min": -75.66609954833984,
      "activations/layer4_attention_weight_max": 103.84974670410156,
      "activations/layer4_attention_weight_min": -94.77892303466797,
      "activations/layer5_attention_weight_max": 119.55236053466797,
      "activations/layer5_attention_weight_min": -98.36209106445312,
      "activations/layer6_attention_weight_max": 60.069252014160156,
      "activations/layer6_attention_weight_min": -49.2185173034668,
      "activations/layer7_attention_weight_max": 70.88268280029297,
      "activations/layer7_attention_weight_min": -56.855857849121094,
      "activations/layer8_attention_weight_max": 48.81209182739258,
      "activations/layer8_attention_weight_min": -35.624271392822266,
      "activations/layer9_attention_weight_max": 75.25585174560547,
      "activations/layer9_attention_weight_min": -58.1252326965332,
      "epoch": 2.93,
      "learning_rate": 0.00013246628787878787,
      "loss": 2.9151,
      "step": 50350
    },
    {
      "activations/layer0_attention_weight_max": 14.374509811401367,
      "activations/layer0_attention_weight_min": -14.62623119354248,
      "activations/layer10_attention_weight_max": 45.57722854614258,
      "activations/layer10_attention_weight_min": -37.320987701416016,
      "activations/layer11_attention_weight_max": 34.821128845214844,
      "activations/layer11_attention_weight_min": -27.3094539642334,
      "activations/layer12_attention_weight_max": 26.226882934570312,
      "activations/layer12_attention_weight_min": -24.26017189025879,
      "activations/layer13_attention_weight_max": 53.97108840942383,
      "activations/layer13_attention_weight_min": -38.505733489990234,
      "activations/layer14_attention_weight_max": 52.853248596191406,
      "activations/layer14_attention_weight_min": -32.60268783569336,
      "activations/layer15_attention_weight_max": 50.06732177734375,
      "activations/layer15_attention_weight_min": -32.19607162475586,
      "activations/layer16_attention_weight_max": 40.51648712158203,
      "activations/layer16_attention_weight_min": -26.75109100341797,
      "activations/layer17_attention_weight_max": 67.13616180419922,
      "activations/layer17_attention_weight_min": -35.4295654296875,
      "activations/layer18_attention_weight_max": 56.8952751159668,
      "activations/layer18_attention_weight_min": -32.4575309753418,
      "activations/layer19_attention_weight_max": 26.02301025390625,
      "activations/layer19_attention_weight_min": -17.770977020263672,
      "activations/layer1_attention_weight_max": 17.12413215637207,
      "activations/layer1_attention_weight_min": -14.591306686401367,
      "activations/layer20_attention_weight_max": 25.013858795166016,
      "activations/layer20_attention_weight_min": -18.934295654296875,
      "activations/layer21_attention_weight_max": 50.3012580871582,
      "activations/layer21_attention_weight_min": -26.199039459228516,
      "activations/layer22_attention_weight_max": 39.26270294189453,
      "activations/layer22_attention_weight_min": -22.200817108154297,
      "activations/layer23_attention_weight_max": 27.623756408691406,
      "activations/layer23_attention_weight_min": -17.119760513305664,
      "activations/layer2_attention_weight_max": 27.187185287475586,
      "activations/layer2_attention_weight_min": -26.000776290893555,
      "activations/layer3_attention_weight_max": 67.45481872558594,
      "activations/layer3_attention_weight_min": -77.96092987060547,
      "activations/layer4_attention_weight_max": 95.1559066772461,
      "activations/layer4_attention_weight_min": -88.92675018310547,
      "activations/layer5_attention_weight_max": 114.92797088623047,
      "activations/layer5_attention_weight_min": -96.3310775756836,
      "activations/layer6_attention_weight_max": 60.295936584472656,
      "activations/layer6_attention_weight_min": -50.33884048461914,
      "activations/layer7_attention_weight_max": 59.36827850341797,
      "activations/layer7_attention_weight_min": -53.38712692260742,
      "activations/layer8_attention_weight_max": 40.02195358276367,
      "activations/layer8_attention_weight_min": -34.8800163269043,
      "activations/layer9_attention_weight_max": 52.588409423828125,
      "activations/layer9_attention_weight_min": -46.9698486328125,
      "epoch": 2.93,
      "learning_rate": 0.00013244734848484848,
      "loss": 2.9247,
      "step": 50400
    },
    {
      "activations/layer0_attention_weight_max": 13.94005012512207,
      "activations/layer0_attention_weight_min": -15.50047779083252,
      "activations/layer10_attention_weight_max": 45.390933990478516,
      "activations/layer10_attention_weight_min": -38.09125900268555,
      "activations/layer11_attention_weight_max": 37.65538787841797,
      "activations/layer11_attention_weight_min": -30.754886627197266,
      "activations/layer12_attention_weight_max": 28.96805763244629,
      "activations/layer12_attention_weight_min": -25.199243545532227,
      "activations/layer13_attention_weight_max": 67.11290740966797,
      "activations/layer13_attention_weight_min": -47.580806732177734,
      "activations/layer14_attention_weight_max": 55.162940979003906,
      "activations/layer14_attention_weight_min": -40.46215057373047,
      "activations/layer15_attention_weight_max": 54.265010833740234,
      "activations/layer15_attention_weight_min": -37.13837432861328,
      "activations/layer16_attention_weight_max": 43.00986099243164,
      "activations/layer16_attention_weight_min": -27.992412567138672,
      "activations/layer17_attention_weight_max": 67.08110809326172,
      "activations/layer17_attention_weight_min": -35.977272033691406,
      "activations/layer18_attention_weight_max": 55.1886100769043,
      "activations/layer18_attention_weight_min": -33.820030212402344,
      "activations/layer19_attention_weight_max": 27.8660888671875,
      "activations/layer19_attention_weight_min": -19.5527286529541,
      "activations/layer1_attention_weight_max": 19.212007522583008,
      "activations/layer1_attention_weight_min": -15.239224433898926,
      "activations/layer20_attention_weight_max": 25.757230758666992,
      "activations/layer20_attention_weight_min": -17.315383911132812,
      "activations/layer21_attention_weight_max": 48.462852478027344,
      "activations/layer21_attention_weight_min": -29.48708724975586,
      "activations/layer22_attention_weight_max": 36.65315246582031,
      "activations/layer22_attention_weight_min": -22.84003448486328,
      "activations/layer23_attention_weight_max": 27.10724449157715,
      "activations/layer23_attention_weight_min": -16.015422821044922,
      "activations/layer2_attention_weight_max": 29.297462463378906,
      "activations/layer2_attention_weight_min": -26.613216400146484,
      "activations/layer3_attention_weight_max": 74.32061767578125,
      "activations/layer3_attention_weight_min": -75.95271301269531,
      "activations/layer4_attention_weight_max": 104.86146545410156,
      "activations/layer4_attention_weight_min": -92.35704040527344,
      "activations/layer5_attention_weight_max": 131.12033081054688,
      "activations/layer5_attention_weight_min": -105.77339172363281,
      "activations/layer6_attention_weight_max": 65.0980224609375,
      "activations/layer6_attention_weight_min": -52.13948440551758,
      "activations/layer7_attention_weight_max": 68.54436492919922,
      "activations/layer7_attention_weight_min": -54.84901809692383,
      "activations/layer8_attention_weight_max": 42.6879997253418,
      "activations/layer8_attention_weight_min": -36.35079574584961,
      "activations/layer9_attention_weight_max": 58.578285217285156,
      "activations/layer9_attention_weight_min": -52.088348388671875,
      "epoch": 2.93,
      "learning_rate": 0.00013242840909090907,
      "loss": 2.9163,
      "step": 50450
    },
    {
      "activations/layer0_attention_weight_max": 14.769880294799805,
      "activations/layer0_attention_weight_min": -15.141003608703613,
      "activations/layer10_attention_weight_max": 44.38262939453125,
      "activations/layer10_attention_weight_min": -37.6691780090332,
      "activations/layer11_attention_weight_max": 38.681312561035156,
      "activations/layer11_attention_weight_min": -27.555173873901367,
      "activations/layer12_attention_weight_max": 28.16715431213379,
      "activations/layer12_attention_weight_min": -23.66975212097168,
      "activations/layer13_attention_weight_max": 50.31452560424805,
      "activations/layer13_attention_weight_min": -37.90138244628906,
      "activations/layer14_attention_weight_max": 46.38641357421875,
      "activations/layer14_attention_weight_min": -29.280986785888672,
      "activations/layer15_attention_weight_max": 52.77898406982422,
      "activations/layer15_attention_weight_min": -36.0049934387207,
      "activations/layer16_attention_weight_max": 38.125816345214844,
      "activations/layer16_attention_weight_min": -27.285736083984375,
      "activations/layer17_attention_weight_max": 58.98235321044922,
      "activations/layer17_attention_weight_min": -35.41725540161133,
      "activations/layer18_attention_weight_max": 54.89780044555664,
      "activations/layer18_attention_weight_min": -32.471187591552734,
      "activations/layer19_attention_weight_max": 23.032878875732422,
      "activations/layer19_attention_weight_min": -16.65108299255371,
      "activations/layer1_attention_weight_max": 17.585372924804688,
      "activations/layer1_attention_weight_min": -14.580962181091309,
      "activations/layer20_attention_weight_max": 21.561813354492188,
      "activations/layer20_attention_weight_min": -20.73399543762207,
      "activations/layer21_attention_weight_max": 36.86924362182617,
      "activations/layer21_attention_weight_min": -25.3936824798584,
      "activations/layer22_attention_weight_max": 38.10758972167969,
      "activations/layer22_attention_weight_min": -21.843669891357422,
      "activations/layer23_attention_weight_max": 29.093801498413086,
      "activations/layer23_attention_weight_min": -15.301525115966797,
      "activations/layer2_attention_weight_max": 29.108951568603516,
      "activations/layer2_attention_weight_min": -27.816450119018555,
      "activations/layer3_attention_weight_max": 75.2353286743164,
      "activations/layer3_attention_weight_min": -81.62699127197266,
      "activations/layer4_attention_weight_max": 101.69550323486328,
      "activations/layer4_attention_weight_min": -89.24497985839844,
      "activations/layer5_attention_weight_max": 122.53598022460938,
      "activations/layer5_attention_weight_min": -100.67696380615234,
      "activations/layer6_attention_weight_max": 61.03702163696289,
      "activations/layer6_attention_weight_min": -50.72638702392578,
      "activations/layer7_attention_weight_max": 56.59847640991211,
      "activations/layer7_attention_weight_min": -56.36042404174805,
      "activations/layer8_attention_weight_max": 41.22113800048828,
      "activations/layer8_attention_weight_min": -38.73430252075195,
      "activations/layer9_attention_weight_max": 57.651153564453125,
      "activations/layer9_attention_weight_min": -52.28752517700195,
      "epoch": 2.93,
      "learning_rate": 0.00013240946969696969,
      "loss": 2.9126,
      "step": 50500
    },
    {
      "activations/layer0_attention_weight_max": 14.460005760192871,
      "activations/layer0_attention_weight_min": -14.709100723266602,
      "activations/layer10_attention_weight_max": 50.39514923095703,
      "activations/layer10_attention_weight_min": -38.530792236328125,
      "activations/layer11_attention_weight_max": 37.85331726074219,
      "activations/layer11_attention_weight_min": -28.184051513671875,
      "activations/layer12_attention_weight_max": 32.2718505859375,
      "activations/layer12_attention_weight_min": -26.643465042114258,
      "activations/layer13_attention_weight_max": 69.28482055664062,
      "activations/layer13_attention_weight_min": -48.44181823730469,
      "activations/layer14_attention_weight_max": 61.08926773071289,
      "activations/layer14_attention_weight_min": -46.440528869628906,
      "activations/layer15_attention_weight_max": 50.91758346557617,
      "activations/layer15_attention_weight_min": -35.7526969909668,
      "activations/layer16_attention_weight_max": 43.40911865234375,
      "activations/layer16_attention_weight_min": -29.10333251953125,
      "activations/layer17_attention_weight_max": 66.55699157714844,
      "activations/layer17_attention_weight_min": -37.820594787597656,
      "activations/layer18_attention_weight_max": 59.143150329589844,
      "activations/layer18_attention_weight_min": -36.08191680908203,
      "activations/layer19_attention_weight_max": 28.906864166259766,
      "activations/layer19_attention_weight_min": -15.675088882446289,
      "activations/layer1_attention_weight_max": 17.63019561767578,
      "activations/layer1_attention_weight_min": -16.062965393066406,
      "activations/layer20_attention_weight_max": 23.05278778076172,
      "activations/layer20_attention_weight_min": -19.829410552978516,
      "activations/layer21_attention_weight_max": 50.949119567871094,
      "activations/layer21_attention_weight_min": -30.118131637573242,
      "activations/layer22_attention_weight_max": 41.49771499633789,
      "activations/layer22_attention_weight_min": -24.15541648864746,
      "activations/layer23_attention_weight_max": 28.250518798828125,
      "activations/layer23_attention_weight_min": -16.75708770751953,
      "activations/layer2_attention_weight_max": 28.137142181396484,
      "activations/layer2_attention_weight_min": -26.586181640625,
      "activations/layer3_attention_weight_max": 71.35237121582031,
      "activations/layer3_attention_weight_min": -73.38314056396484,
      "activations/layer4_attention_weight_max": 97.28836822509766,
      "activations/layer4_attention_weight_min": -92.16947174072266,
      "activations/layer5_attention_weight_max": 123.84111785888672,
      "activations/layer5_attention_weight_min": -104.57329559326172,
      "activations/layer6_attention_weight_max": 61.35248565673828,
      "activations/layer6_attention_weight_min": -51.96653747558594,
      "activations/layer7_attention_weight_max": 61.677520751953125,
      "activations/layer7_attention_weight_min": -55.98552703857422,
      "activations/layer8_attention_weight_max": 44.17171096801758,
      "activations/layer8_attention_weight_min": -36.44560241699219,
      "activations/layer9_attention_weight_max": 52.07526779174805,
      "activations/layer9_attention_weight_min": -47.225643157958984,
      "epoch": 2.94,
      "learning_rate": 0.00013239053030303027,
      "loss": 2.9154,
      "step": 50550
    },
    {
      "activations/layer0_attention_weight_max": 13.957989692687988,
      "activations/layer0_attention_weight_min": -14.885340690612793,
      "activations/layer10_attention_weight_max": 45.37354278564453,
      "activations/layer10_attention_weight_min": -37.65846633911133,
      "activations/layer11_attention_weight_max": 36.76830291748047,
      "activations/layer11_attention_weight_min": -30.402420043945312,
      "activations/layer12_attention_weight_max": 34.584228515625,
      "activations/layer12_attention_weight_min": -27.82193374633789,
      "activations/layer13_attention_weight_max": 62.44730758666992,
      "activations/layer13_attention_weight_min": -44.668251037597656,
      "activations/layer14_attention_weight_max": 54.39157485961914,
      "activations/layer14_attention_weight_min": -31.956066131591797,
      "activations/layer15_attention_weight_max": 53.886959075927734,
      "activations/layer15_attention_weight_min": -33.488189697265625,
      "activations/layer16_attention_weight_max": 56.555606842041016,
      "activations/layer16_attention_weight_min": -26.619495391845703,
      "activations/layer17_attention_weight_max": 69.83539581298828,
      "activations/layer17_attention_weight_min": -36.29576110839844,
      "activations/layer18_attention_weight_max": 57.3798828125,
      "activations/layer18_attention_weight_min": -32.06178283691406,
      "activations/layer19_attention_weight_max": 25.9157772064209,
      "activations/layer19_attention_weight_min": -17.07476043701172,
      "activations/layer1_attention_weight_max": 18.13504981994629,
      "activations/layer1_attention_weight_min": -14.82165813446045,
      "activations/layer20_attention_weight_max": 24.177518844604492,
      "activations/layer20_attention_weight_min": -17.58205795288086,
      "activations/layer21_attention_weight_max": 43.75395584106445,
      "activations/layer21_attention_weight_min": -28.21968650817871,
      "activations/layer22_attention_weight_max": 38.6212158203125,
      "activations/layer22_attention_weight_min": -21.887853622436523,
      "activations/layer23_attention_weight_max": 30.63372802734375,
      "activations/layer23_attention_weight_min": -15.103527069091797,
      "activations/layer2_attention_weight_max": 29.308746337890625,
      "activations/layer2_attention_weight_min": -27.82241439819336,
      "activations/layer3_attention_weight_max": 74.3935546875,
      "activations/layer3_attention_weight_min": -75.15713500976562,
      "activations/layer4_attention_weight_max": 102.30803680419922,
      "activations/layer4_attention_weight_min": -94.29019927978516,
      "activations/layer5_attention_weight_max": 124.07334899902344,
      "activations/layer5_attention_weight_min": -101.73744201660156,
      "activations/layer6_attention_weight_max": 63.63832473754883,
      "activations/layer6_attention_weight_min": -51.54478454589844,
      "activations/layer7_attention_weight_max": 64.28412628173828,
      "activations/layer7_attention_weight_min": -56.98417663574219,
      "activations/layer8_attention_weight_max": 41.9389533996582,
      "activations/layer8_attention_weight_min": -36.11723709106445,
      "activations/layer9_attention_weight_max": 55.0882453918457,
      "activations/layer9_attention_weight_min": -51.87186813354492,
      "epoch": 2.94,
      "learning_rate": 0.0001323715909090909,
      "loss": 2.9102,
      "step": 50600
    },
    {
      "activations/layer0_attention_weight_max": 14.542975425720215,
      "activations/layer0_attention_weight_min": -14.149081230163574,
      "activations/layer10_attention_weight_max": 46.798824310302734,
      "activations/layer10_attention_weight_min": -36.389793395996094,
      "activations/layer11_attention_weight_max": 38.3886604309082,
      "activations/layer11_attention_weight_min": -28.153743743896484,
      "activations/layer12_attention_weight_max": 28.07012176513672,
      "activations/layer12_attention_weight_min": -26.604143142700195,
      "activations/layer13_attention_weight_max": 48.67170715332031,
      "activations/layer13_attention_weight_min": -39.42109298706055,
      "activations/layer14_attention_weight_max": 50.7617301940918,
      "activations/layer14_attention_weight_min": -31.53338050842285,
      "activations/layer15_attention_weight_max": 46.00533676147461,
      "activations/layer15_attention_weight_min": -31.59492301940918,
      "activations/layer16_attention_weight_max": 38.900115966796875,
      "activations/layer16_attention_weight_min": -25.104814529418945,
      "activations/layer17_attention_weight_max": 55.47398376464844,
      "activations/layer17_attention_weight_min": -34.949588775634766,
      "activations/layer18_attention_weight_max": 54.62950897216797,
      "activations/layer18_attention_weight_min": -31.663530349731445,
      "activations/layer19_attention_weight_max": 22.965314865112305,
      "activations/layer19_attention_weight_min": -16.3008975982666,
      "activations/layer1_attention_weight_max": 18.944114685058594,
      "activations/layer1_attention_weight_min": -13.297338485717773,
      "activations/layer20_attention_weight_max": 20.305070877075195,
      "activations/layer20_attention_weight_min": -17.5043888092041,
      "activations/layer21_attention_weight_max": 44.80665588378906,
      "activations/layer21_attention_weight_min": -25.960050582885742,
      "activations/layer22_attention_weight_max": 37.57780838012695,
      "activations/layer22_attention_weight_min": -22.445764541625977,
      "activations/layer23_attention_weight_max": 25.986248016357422,
      "activations/layer23_attention_weight_min": -14.724374771118164,
      "activations/layer2_attention_weight_max": 27.68780517578125,
      "activations/layer2_attention_weight_min": -26.213741302490234,
      "activations/layer3_attention_weight_max": 74.20838165283203,
      "activations/layer3_attention_weight_min": -74.28040313720703,
      "activations/layer4_attention_weight_max": 100.12848663330078,
      "activations/layer4_attention_weight_min": -87.05164337158203,
      "activations/layer5_attention_weight_max": 124.17144775390625,
      "activations/layer5_attention_weight_min": -100.11589050292969,
      "activations/layer6_attention_weight_max": 59.12236785888672,
      "activations/layer6_attention_weight_min": -50.62278366088867,
      "activations/layer7_attention_weight_max": 59.46275329589844,
      "activations/layer7_attention_weight_min": -53.74220275878906,
      "activations/layer8_attention_weight_max": 38.51477813720703,
      "activations/layer8_attention_weight_min": -36.794395446777344,
      "activations/layer9_attention_weight_max": 59.5452766418457,
      "activations/layer9_attention_weight_min": -51.586761474609375,
      "epoch": 2.94,
      "learning_rate": 0.0001323526515151515,
      "loss": 2.9288,
      "step": 50650
    },
    {
      "activations/layer0_attention_weight_max": 14.024189949035645,
      "activations/layer0_attention_weight_min": -14.867353439331055,
      "activations/layer10_attention_weight_max": 45.36281204223633,
      "activations/layer10_attention_weight_min": -37.520362854003906,
      "activations/layer11_attention_weight_max": 42.99088668823242,
      "activations/layer11_attention_weight_min": -28.83055877685547,
      "activations/layer12_attention_weight_max": 23.475116729736328,
      "activations/layer12_attention_weight_min": -24.757719039916992,
      "activations/layer13_attention_weight_max": 69.39520263671875,
      "activations/layer13_attention_weight_min": -45.50935363769531,
      "activations/layer14_attention_weight_max": 57.18798065185547,
      "activations/layer14_attention_weight_min": -36.538917541503906,
      "activations/layer15_attention_weight_max": 55.38602066040039,
      "activations/layer15_attention_weight_min": -36.119224548339844,
      "activations/layer16_attention_weight_max": 43.68553924560547,
      "activations/layer16_attention_weight_min": -32.09429168701172,
      "activations/layer17_attention_weight_max": 64.56919860839844,
      "activations/layer17_attention_weight_min": -37.25917434692383,
      "activations/layer18_attention_weight_max": 55.168296813964844,
      "activations/layer18_attention_weight_min": -32.988494873046875,
      "activations/layer19_attention_weight_max": 25.038679122924805,
      "activations/layer19_attention_weight_min": -20.5762939453125,
      "activations/layer1_attention_weight_max": 16.831085205078125,
      "activations/layer1_attention_weight_min": -16.09598159790039,
      "activations/layer20_attention_weight_max": 23.094070434570312,
      "activations/layer20_attention_weight_min": -19.51895523071289,
      "activations/layer21_attention_weight_max": 40.574745178222656,
      "activations/layer21_attention_weight_min": -29.838415145874023,
      "activations/layer22_attention_weight_max": 37.980751037597656,
      "activations/layer22_attention_weight_min": -23.33038902282715,
      "activations/layer23_attention_weight_max": 27.186935424804688,
      "activations/layer23_attention_weight_min": -15.839832305908203,
      "activations/layer2_attention_weight_max": 30.412456512451172,
      "activations/layer2_attention_weight_min": -27.545913696289062,
      "activations/layer3_attention_weight_max": 75.22132110595703,
      "activations/layer3_attention_weight_min": -79.18610382080078,
      "activations/layer4_attention_weight_max": 101.73165130615234,
      "activations/layer4_attention_weight_min": -97.92572784423828,
      "activations/layer5_attention_weight_max": 128.89273071289062,
      "activations/layer5_attention_weight_min": -108.87542724609375,
      "activations/layer6_attention_weight_max": 63.53171157836914,
      "activations/layer6_attention_weight_min": -53.6357536315918,
      "activations/layer7_attention_weight_max": 61.33798599243164,
      "activations/layer7_attention_weight_min": -57.54975128173828,
      "activations/layer8_attention_weight_max": 40.87528610229492,
      "activations/layer8_attention_weight_min": -38.59244918823242,
      "activations/layer9_attention_weight_max": 54.50061798095703,
      "activations/layer9_attention_weight_min": -52.543983459472656,
      "epoch": 2.95,
      "learning_rate": 0.0001323337121212121,
      "loss": 2.9206,
      "step": 50700
    },
    {
      "activations/layer0_attention_weight_max": 15.584013938903809,
      "activations/layer0_attention_weight_min": -15.073740005493164,
      "activations/layer10_attention_weight_max": 51.42176055908203,
      "activations/layer10_attention_weight_min": -38.775753021240234,
      "activations/layer11_attention_weight_max": 40.92884063720703,
      "activations/layer11_attention_weight_min": -29.151647567749023,
      "activations/layer12_attention_weight_max": 30.325590133666992,
      "activations/layer12_attention_weight_min": -24.448261260986328,
      "activations/layer13_attention_weight_max": 67.86384582519531,
      "activations/layer13_attention_weight_min": -42.44891357421875,
      "activations/layer14_attention_weight_max": 58.08877182006836,
      "activations/layer14_attention_weight_min": -35.89643096923828,
      "activations/layer15_attention_weight_max": 58.002559661865234,
      "activations/layer15_attention_weight_min": -34.98887634277344,
      "activations/layer16_attention_weight_max": 41.75616455078125,
      "activations/layer16_attention_weight_min": -26.99952507019043,
      "activations/layer17_attention_weight_max": 67.58454895019531,
      "activations/layer17_attention_weight_min": -39.4903450012207,
      "activations/layer18_attention_weight_max": 57.04767990112305,
      "activations/layer18_attention_weight_min": -34.89421844482422,
      "activations/layer19_attention_weight_max": 25.0331974029541,
      "activations/layer19_attention_weight_min": -16.648759841918945,
      "activations/layer1_attention_weight_max": 17.827924728393555,
      "activations/layer1_attention_weight_min": -14.969807624816895,
      "activations/layer20_attention_weight_max": 24.571670532226562,
      "activations/layer20_attention_weight_min": -18.699016571044922,
      "activations/layer21_attention_weight_max": 52.41824722290039,
      "activations/layer21_attention_weight_min": -26.965770721435547,
      "activations/layer22_attention_weight_max": 42.749549865722656,
      "activations/layer22_attention_weight_min": -23.665679931640625,
      "activations/layer23_attention_weight_max": 31.5308895111084,
      "activations/layer23_attention_weight_min": -18.570724487304688,
      "activations/layer2_attention_weight_max": 29.40753173828125,
      "activations/layer2_attention_weight_min": -27.32301139831543,
      "activations/layer3_attention_weight_max": 68.60563659667969,
      "activations/layer3_attention_weight_min": -72.33236694335938,
      "activations/layer4_attention_weight_max": 104.42015838623047,
      "activations/layer4_attention_weight_min": -94.38331604003906,
      "activations/layer5_attention_weight_max": 136.80271911621094,
      "activations/layer5_attention_weight_min": -105.0251693725586,
      "activations/layer6_attention_weight_max": 69.05439758300781,
      "activations/layer6_attention_weight_min": -50.32185363769531,
      "activations/layer7_attention_weight_max": 63.15705490112305,
      "activations/layer7_attention_weight_min": -56.20551681518555,
      "activations/layer8_attention_weight_max": 42.83538818359375,
      "activations/layer8_attention_weight_min": -37.800010681152344,
      "activations/layer9_attention_weight_max": 57.607662200927734,
      "activations/layer9_attention_weight_min": -49.58494186401367,
      "epoch": 2.95,
      "learning_rate": 0.0001323147727272727,
      "loss": 2.9065,
      "step": 50750
    },
    {
      "activations/layer0_attention_weight_max": 14.696590423583984,
      "activations/layer0_attention_weight_min": -14.242220878601074,
      "activations/layer10_attention_weight_max": 52.304561614990234,
      "activations/layer10_attention_weight_min": -38.464290618896484,
      "activations/layer11_attention_weight_max": 45.72921371459961,
      "activations/layer11_attention_weight_min": -33.55683517456055,
      "activations/layer12_attention_weight_max": 27.947328567504883,
      "activations/layer12_attention_weight_min": -24.317411422729492,
      "activations/layer13_attention_weight_max": 51.38740158081055,
      "activations/layer13_attention_weight_min": -40.4891242980957,
      "activations/layer14_attention_weight_max": 52.206565856933594,
      "activations/layer14_attention_weight_min": -37.374637603759766,
      "activations/layer15_attention_weight_max": 48.757537841796875,
      "activations/layer15_attention_weight_min": -34.590999603271484,
      "activations/layer16_attention_weight_max": 34.641517639160156,
      "activations/layer16_attention_weight_min": -26.00448989868164,
      "activations/layer17_attention_weight_max": 56.94858932495117,
      "activations/layer17_attention_weight_min": -36.092140197753906,
      "activations/layer18_attention_weight_max": 51.090572357177734,
      "activations/layer18_attention_weight_min": -33.49188232421875,
      "activations/layer19_attention_weight_max": 20.44853401184082,
      "activations/layer19_attention_weight_min": -15.572319030761719,
      "activations/layer1_attention_weight_max": 17.84101676940918,
      "activations/layer1_attention_weight_min": -13.428571701049805,
      "activations/layer20_attention_weight_max": 19.816795349121094,
      "activations/layer20_attention_weight_min": -20.63030433654785,
      "activations/layer21_attention_weight_max": 43.51537322998047,
      "activations/layer21_attention_weight_min": -32.31655502319336,
      "activations/layer22_attention_weight_max": 33.241455078125,
      "activations/layer22_attention_weight_min": -22.749319076538086,
      "activations/layer23_attention_weight_max": 24.31587028503418,
      "activations/layer23_attention_weight_min": -19.424936294555664,
      "activations/layer2_attention_weight_max": 29.19481658935547,
      "activations/layer2_attention_weight_min": -26.030475616455078,
      "activations/layer3_attention_weight_max": 72.7798080444336,
      "activations/layer3_attention_weight_min": -74.2693099975586,
      "activations/layer4_attention_weight_max": 108.70851135253906,
      "activations/layer4_attention_weight_min": -95.60503387451172,
      "activations/layer5_attention_weight_max": 137.62571716308594,
      "activations/layer5_attention_weight_min": -113.76377868652344,
      "activations/layer6_attention_weight_max": 68.078369140625,
      "activations/layer6_attention_weight_min": -58.00263595581055,
      "activations/layer7_attention_weight_max": 72.4245834350586,
      "activations/layer7_attention_weight_min": -59.190425872802734,
      "activations/layer8_attention_weight_max": 50.070960998535156,
      "activations/layer8_attention_weight_min": -38.46228790283203,
      "activations/layer9_attention_weight_max": 70.30450439453125,
      "activations/layer9_attention_weight_min": -59.31736373901367,
      "epoch": 2.95,
      "learning_rate": 0.00013229583333333332,
      "loss": 2.9034,
      "step": 50800
    },
    {
      "activations/layer0_attention_weight_max": 14.276001930236816,
      "activations/layer0_attention_weight_min": -14.457419395446777,
      "activations/layer10_attention_weight_max": 47.75246047973633,
      "activations/layer10_attention_weight_min": -38.64051818847656,
      "activations/layer11_attention_weight_max": 38.79887008666992,
      "activations/layer11_attention_weight_min": -29.886241912841797,
      "activations/layer12_attention_weight_max": 25.452070236206055,
      "activations/layer12_attention_weight_min": -26.868680953979492,
      "activations/layer13_attention_weight_max": 65.98713684082031,
      "activations/layer13_attention_weight_min": -41.51311492919922,
      "activations/layer14_attention_weight_max": 82.50326538085938,
      "activations/layer14_attention_weight_min": -45.714439392089844,
      "activations/layer15_attention_weight_max": 55.715885162353516,
      "activations/layer15_attention_weight_min": -33.99384689331055,
      "activations/layer16_attention_weight_max": 40.7369270324707,
      "activations/layer16_attention_weight_min": -28.005765914916992,
      "activations/layer17_attention_weight_max": 66.3118667602539,
      "activations/layer17_attention_weight_min": -37.81644821166992,
      "activations/layer18_attention_weight_max": 60.19655990600586,
      "activations/layer18_attention_weight_min": -32.87710952758789,
      "activations/layer19_attention_weight_max": 23.419023513793945,
      "activations/layer19_attention_weight_min": -18.562841415405273,
      "activations/layer1_attention_weight_max": 17.658266067504883,
      "activations/layer1_attention_weight_min": -14.003924369812012,
      "activations/layer20_attention_weight_max": 26.473283767700195,
      "activations/layer20_attention_weight_min": -19.092741012573242,
      "activations/layer21_attention_weight_max": 51.73675537109375,
      "activations/layer21_attention_weight_min": -28.52958106994629,
      "activations/layer22_attention_weight_max": 40.03394317626953,
      "activations/layer22_attention_weight_min": -23.95846939086914,
      "activations/layer23_attention_weight_max": 26.485815048217773,
      "activations/layer23_attention_weight_min": -17.787269592285156,
      "activations/layer2_attention_weight_max": 29.036277770996094,
      "activations/layer2_attention_weight_min": -26.481735229492188,
      "activations/layer3_attention_weight_max": 73.93770599365234,
      "activations/layer3_attention_weight_min": -76.4363784790039,
      "activations/layer4_attention_weight_max": 103.8151626586914,
      "activations/layer4_attention_weight_min": -96.05475616455078,
      "activations/layer5_attention_weight_max": 132.403076171875,
      "activations/layer5_attention_weight_min": -108.20433807373047,
      "activations/layer6_attention_weight_max": 67.51049041748047,
      "activations/layer6_attention_weight_min": -51.2809944152832,
      "activations/layer7_attention_weight_max": 65.92000579833984,
      "activations/layer7_attention_weight_min": -60.01662063598633,
      "activations/layer8_attention_weight_max": 45.05530548095703,
      "activations/layer8_attention_weight_min": -38.13908386230469,
      "activations/layer9_attention_weight_max": 67.56649017333984,
      "activations/layer9_attention_weight_min": -49.75067901611328,
      "epoch": 2.95,
      "learning_rate": 0.00013227689393939394,
      "loss": 2.9305,
      "step": 50850
    },
    {
      "activations/layer0_attention_weight_max": 14.691512107849121,
      "activations/layer0_attention_weight_min": -14.820213317871094,
      "activations/layer10_attention_weight_max": 46.396697998046875,
      "activations/layer10_attention_weight_min": -36.32245635986328,
      "activations/layer11_attention_weight_max": 40.46772003173828,
      "activations/layer11_attention_weight_min": -28.49714469909668,
      "activations/layer12_attention_weight_max": 25.272098541259766,
      "activations/layer12_attention_weight_min": -29.898283004760742,
      "activations/layer13_attention_weight_max": 59.31849670410156,
      "activations/layer13_attention_weight_min": -40.70884323120117,
      "activations/layer14_attention_weight_max": 44.65993118286133,
      "activations/layer14_attention_weight_min": -35.768218994140625,
      "activations/layer15_attention_weight_max": 55.57864761352539,
      "activations/layer15_attention_weight_min": -35.4544792175293,
      "activations/layer16_attention_weight_max": 38.91490936279297,
      "activations/layer16_attention_weight_min": -27.563364028930664,
      "activations/layer17_attention_weight_max": 60.11064529418945,
      "activations/layer17_attention_weight_min": -37.9036979675293,
      "activations/layer18_attention_weight_max": 60.29609680175781,
      "activations/layer18_attention_weight_min": -34.90684509277344,
      "activations/layer19_attention_weight_max": 24.20517349243164,
      "activations/layer19_attention_weight_min": -20.27961540222168,
      "activations/layer1_attention_weight_max": 17.517826080322266,
      "activations/layer1_attention_weight_min": -14.1320161819458,
      "activations/layer20_attention_weight_max": 20.98489761352539,
      "activations/layer20_attention_weight_min": -19.490821838378906,
      "activations/layer21_attention_weight_max": 46.00935745239258,
      "activations/layer21_attention_weight_min": -30.396194458007812,
      "activations/layer22_attention_weight_max": 37.66441345214844,
      "activations/layer22_attention_weight_min": -24.520519256591797,
      "activations/layer23_attention_weight_max": 25.518474578857422,
      "activations/layer23_attention_weight_min": -15.307455062866211,
      "activations/layer2_attention_weight_max": 29.714038848876953,
      "activations/layer2_attention_weight_min": -27.309326171875,
      "activations/layer3_attention_weight_max": 70.16387176513672,
      "activations/layer3_attention_weight_min": -76.5534896850586,
      "activations/layer4_attention_weight_max": 102.1950912475586,
      "activations/layer4_attention_weight_min": -90.80883026123047,
      "activations/layer5_attention_weight_max": 125.68087005615234,
      "activations/layer5_attention_weight_min": -100.25367736816406,
      "activations/layer6_attention_weight_max": 61.17325210571289,
      "activations/layer6_attention_weight_min": -51.24718475341797,
      "activations/layer7_attention_weight_max": 57.83137130737305,
      "activations/layer7_attention_weight_min": -53.85618591308594,
      "activations/layer8_attention_weight_max": 38.40277862548828,
      "activations/layer8_attention_weight_min": -36.49156951904297,
      "activations/layer9_attention_weight_max": 47.99380111694336,
      "activations/layer9_attention_weight_min": -45.61454391479492,
      "epoch": 2.96,
      "learning_rate": 0.00013225795454545452,
      "loss": 2.9173,
      "step": 50900
    },
    {
      "activations/layer0_attention_weight_max": 14.037101745605469,
      "activations/layer0_attention_weight_min": -14.696941375732422,
      "activations/layer10_attention_weight_max": 44.41910934448242,
      "activations/layer10_attention_weight_min": -38.04866409301758,
      "activations/layer11_attention_weight_max": 36.01597595214844,
      "activations/layer11_attention_weight_min": -28.1230525970459,
      "activations/layer12_attention_weight_max": 23.200332641601562,
      "activations/layer12_attention_weight_min": -23.860034942626953,
      "activations/layer13_attention_weight_max": 55.7940673828125,
      "activations/layer13_attention_weight_min": -41.31184005737305,
      "activations/layer14_attention_weight_max": 42.7614860534668,
      "activations/layer14_attention_weight_min": -31.142677307128906,
      "activations/layer15_attention_weight_max": 43.666358947753906,
      "activations/layer15_attention_weight_min": -32.519264221191406,
      "activations/layer16_attention_weight_max": 32.928077697753906,
      "activations/layer16_attention_weight_min": -26.229509353637695,
      "activations/layer17_attention_weight_max": 60.01826858520508,
      "activations/layer17_attention_weight_min": -39.5843391418457,
      "activations/layer18_attention_weight_max": 55.831947326660156,
      "activations/layer18_attention_weight_min": -31.000831604003906,
      "activations/layer19_attention_weight_max": 21.689115524291992,
      "activations/layer19_attention_weight_min": -17.503339767456055,
      "activations/layer1_attention_weight_max": 17.892799377441406,
      "activations/layer1_attention_weight_min": -13.196916580200195,
      "activations/layer20_attention_weight_max": 19.237533569335938,
      "activations/layer20_attention_weight_min": -22.108257293701172,
      "activations/layer21_attention_weight_max": 40.282142639160156,
      "activations/layer21_attention_weight_min": -27.152557373046875,
      "activations/layer22_attention_weight_max": 34.79666519165039,
      "activations/layer22_attention_weight_min": -20.993934631347656,
      "activations/layer23_attention_weight_max": 23.43556022644043,
      "activations/layer23_attention_weight_min": -15.221484184265137,
      "activations/layer2_attention_weight_max": 27.80734634399414,
      "activations/layer2_attention_weight_min": -27.543941497802734,
      "activations/layer3_attention_weight_max": 68.18489837646484,
      "activations/layer3_attention_weight_min": -73.65480041503906,
      "activations/layer4_attention_weight_max": 99.9687728881836,
      "activations/layer4_attention_weight_min": -92.67923736572266,
      "activations/layer5_attention_weight_max": 120.55238342285156,
      "activations/layer5_attention_weight_min": -103.39434051513672,
      "activations/layer6_attention_weight_max": 56.56596755981445,
      "activations/layer6_attention_weight_min": -50.96333694458008,
      "activations/layer7_attention_weight_max": 57.022499084472656,
      "activations/layer7_attention_weight_min": -54.64143753051758,
      "activations/layer8_attention_weight_max": 37.68291473388672,
      "activations/layer8_attention_weight_min": -34.84401321411133,
      "activations/layer9_attention_weight_max": 52.51813888549805,
      "activations/layer9_attention_weight_min": -48.67496871948242,
      "epoch": 2.96,
      "learning_rate": 0.00013223901515151514,
      "loss": 2.9061,
      "step": 50950
    },
    {
      "activations/layer0_attention_weight_max": 15.857967376708984,
      "activations/layer0_attention_weight_min": -14.353165626525879,
      "activations/layer10_attention_weight_max": 47.12510681152344,
      "activations/layer10_attention_weight_min": -35.06331253051758,
      "activations/layer11_attention_weight_max": 36.82618713378906,
      "activations/layer11_attention_weight_min": -28.3886775970459,
      "activations/layer12_attention_weight_max": 25.475830078125,
      "activations/layer12_attention_weight_min": -26.042306900024414,
      "activations/layer13_attention_weight_max": 53.403892517089844,
      "activations/layer13_attention_weight_min": -38.98016357421875,
      "activations/layer14_attention_weight_max": 47.72516632080078,
      "activations/layer14_attention_weight_min": -32.68567657470703,
      "activations/layer15_attention_weight_max": 49.95914840698242,
      "activations/layer15_attention_weight_min": -33.401405334472656,
      "activations/layer16_attention_weight_max": 42.30936050415039,
      "activations/layer16_attention_weight_min": -28.56109046936035,
      "activations/layer17_attention_weight_max": 68.05594635009766,
      "activations/layer17_attention_weight_min": -36.10237503051758,
      "activations/layer18_attention_weight_max": 60.85805892944336,
      "activations/layer18_attention_weight_min": -33.36632537841797,
      "activations/layer19_attention_weight_max": 21.146350860595703,
      "activations/layer19_attention_weight_min": -18.03554916381836,
      "activations/layer1_attention_weight_max": 17.816574096679688,
      "activations/layer1_attention_weight_min": -14.226611137390137,
      "activations/layer20_attention_weight_max": 20.61281394958496,
      "activations/layer20_attention_weight_min": -19.30504035949707,
      "activations/layer21_attention_weight_max": 51.04059982299805,
      "activations/layer21_attention_weight_min": -30.620275497436523,
      "activations/layer22_attention_weight_max": 34.81416320800781,
      "activations/layer22_attention_weight_min": -22.28464126586914,
      "activations/layer23_attention_weight_max": 23.64015007019043,
      "activations/layer23_attention_weight_min": -16.54660415649414,
      "activations/layer2_attention_weight_max": 30.359233856201172,
      "activations/layer2_attention_weight_min": -28.072797775268555,
      "activations/layer3_attention_weight_max": 71.36175537109375,
      "activations/layer3_attention_weight_min": -74.8472671508789,
      "activations/layer4_attention_weight_max": 98.75019073486328,
      "activations/layer4_attention_weight_min": -88.70442199707031,
      "activations/layer5_attention_weight_max": 118.13223266601562,
      "activations/layer5_attention_weight_min": -96.93498992919922,
      "activations/layer6_attention_weight_max": 63.56198501586914,
      "activations/layer6_attention_weight_min": -52.96529006958008,
      "activations/layer7_attention_weight_max": 61.41788864135742,
      "activations/layer7_attention_weight_min": -54.774925231933594,
      "activations/layer8_attention_weight_max": 42.67305374145508,
      "activations/layer8_attention_weight_min": -35.08649444580078,
      "activations/layer9_attention_weight_max": 53.085792541503906,
      "activations/layer9_attention_weight_min": -48.5029182434082,
      "epoch": 2.96,
      "learning_rate": 0.00013222007575757575,
      "loss": 2.9109,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_loss": 2.861328125,
      "eval_runtime": 8.5091,
      "eval_samples_per_second": 504.634,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_openwebtext_loss": 2.861328125,
      "eval_openwebtext_ppl": 17.484733434196265,
      "eval_openwebtext_runtime": 8.5091,
      "eval_openwebtext_samples_per_second": 504.634,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_wikitext_loss": 3.111328125,
      "eval_wikitext_ppl": 22.450842133517945,
      "eval_wikitext_runtime": 1.9931,
      "eval_wikitext_samples_per_second": 228.787,
      "step": 51000
    },
    {
      "epoch": 2.96,
      "eval_lambada_loss": 3.009765625,
      "eval_lambada_ppl": 20.282645623050005,
      "eval_lambada_runtime": 9.5388,
      "eval_lambada_samples_per_second": 510.444,
      "step": 51000
    },
    {
      "activations/layer0_attention_weight_max": 14.762843132019043,
      "activations/layer0_attention_weight_min": -15.428960800170898,
      "activations/layer10_attention_weight_max": 46.26533126831055,
      "activations/layer10_attention_weight_min": -35.984710693359375,
      "activations/layer11_attention_weight_max": 39.22590637207031,
      "activations/layer11_attention_weight_min": -28.150449752807617,
      "activations/layer12_attention_weight_max": 25.783639907836914,
      "activations/layer12_attention_weight_min": -25.036762237548828,
      "activations/layer13_attention_weight_max": 56.58160400390625,
      "activations/layer13_attention_weight_min": -34.88279724121094,
      "activations/layer14_attention_weight_max": 48.102210998535156,
      "activations/layer14_attention_weight_min": -32.18229293823242,
      "activations/layer15_attention_weight_max": 48.48794937133789,
      "activations/layer15_attention_weight_min": -32.41209030151367,
      "activations/layer16_attention_weight_max": 39.73569107055664,
      "activations/layer16_attention_weight_min": -28.072845458984375,
      "activations/layer17_attention_weight_max": 64.55307006835938,
      "activations/layer17_attention_weight_min": -36.975242614746094,
      "activations/layer18_attention_weight_max": 60.64491653442383,
      "activations/layer18_attention_weight_min": -33.23984909057617,
      "activations/layer19_attention_weight_max": 26.29938316345215,
      "activations/layer19_attention_weight_min": -18.505231857299805,
      "activations/layer1_attention_weight_max": 17.985666275024414,
      "activations/layer1_attention_weight_min": -16.266555786132812,
      "activations/layer20_attention_weight_max": 22.570512771606445,
      "activations/layer20_attention_weight_min": -21.050792694091797,
      "activations/layer21_attention_weight_max": 52.42185974121094,
      "activations/layer21_attention_weight_min": -32.6500358581543,
      "activations/layer22_attention_weight_max": 36.16177749633789,
      "activations/layer22_attention_weight_min": -23.212541580200195,
      "activations/layer23_attention_weight_max": 26.909360885620117,
      "activations/layer23_attention_weight_min": -15.187253952026367,
      "activations/layer2_attention_weight_max": 29.632841110229492,
      "activations/layer2_attention_weight_min": -27.658246994018555,
      "activations/layer3_attention_weight_max": 70.44678497314453,
      "activations/layer3_attention_weight_min": -71.37640380859375,
      "activations/layer4_attention_weight_max": 103.37383270263672,
      "activations/layer4_attention_weight_min": -87.8651351928711,
      "activations/layer5_attention_weight_max": 128.84986877441406,
      "activations/layer5_attention_weight_min": -97.04563903808594,
      "activations/layer6_attention_weight_max": 63.66312026977539,
      "activations/layer6_attention_weight_min": -49.30457305908203,
      "activations/layer7_attention_weight_max": 59.68470764160156,
      "activations/layer7_attention_weight_min": -52.58611297607422,
      "activations/layer8_attention_weight_max": 41.86936950683594,
      "activations/layer8_attention_weight_min": -34.903350830078125,
      "activations/layer9_attention_weight_max": 54.12525177001953,
      "activations/layer9_attention_weight_min": -49.2834587097168,
      "epoch": 2.97,
      "learning_rate": 0.00013220113636363637,
      "loss": 2.9106,
      "step": 51050
    },
    {
      "activations/layer0_attention_weight_max": 13.675115585327148,
      "activations/layer0_attention_weight_min": -15.084443092346191,
      "activations/layer10_attention_weight_max": 56.49052047729492,
      "activations/layer10_attention_weight_min": -38.0886344909668,
      "activations/layer11_attention_weight_max": 37.907352447509766,
      "activations/layer11_attention_weight_min": -27.915206909179688,
      "activations/layer12_attention_weight_max": 25.313343048095703,
      "activations/layer12_attention_weight_min": -25.690046310424805,
      "activations/layer13_attention_weight_max": 69.34833526611328,
      "activations/layer13_attention_weight_min": -42.17091369628906,
      "activations/layer14_attention_weight_max": 58.587764739990234,
      "activations/layer14_attention_weight_min": -33.99077224731445,
      "activations/layer15_attention_weight_max": 62.74607467651367,
      "activations/layer15_attention_weight_min": -37.20448303222656,
      "activations/layer16_attention_weight_max": 39.18045425415039,
      "activations/layer16_attention_weight_min": -25.61021614074707,
      "activations/layer17_attention_weight_max": 70.17923736572266,
      "activations/layer17_attention_weight_min": -35.939815521240234,
      "activations/layer18_attention_weight_max": 65.48590087890625,
      "activations/layer18_attention_weight_min": -32.30586624145508,
      "activations/layer19_attention_weight_max": 24.132932662963867,
      "activations/layer19_attention_weight_min": -18.01651954650879,
      "activations/layer1_attention_weight_max": 17.72402000427246,
      "activations/layer1_attention_weight_min": -16.339860916137695,
      "activations/layer20_attention_weight_max": 22.14687156677246,
      "activations/layer20_attention_weight_min": -16.028629302978516,
      "activations/layer21_attention_weight_max": 39.24816131591797,
      "activations/layer21_attention_weight_min": -27.08474349975586,
      "activations/layer22_attention_weight_max": 36.43242263793945,
      "activations/layer22_attention_weight_min": -22.622772216796875,
      "activations/layer23_attention_weight_max": 26.96820831298828,
      "activations/layer23_attention_weight_min": -16.166603088378906,
      "activations/layer2_attention_weight_max": 28.72757339477539,
      "activations/layer2_attention_weight_min": -25.597923278808594,
      "activations/layer3_attention_weight_max": 68.2078857421875,
      "activations/layer3_attention_weight_min": -72.0510025024414,
      "activations/layer4_attention_weight_max": 100.45880889892578,
      "activations/layer4_attention_weight_min": -92.97813415527344,
      "activations/layer5_attention_weight_max": 123.18523406982422,
      "activations/layer5_attention_weight_min": -104.15815734863281,
      "activations/layer6_attention_weight_max": 61.16399383544922,
      "activations/layer6_attention_weight_min": -51.833614349365234,
      "activations/layer7_attention_weight_max": 59.16859817504883,
      "activations/layer7_attention_weight_min": -57.576805114746094,
      "activations/layer8_attention_weight_max": 43.67251205444336,
      "activations/layer8_attention_weight_min": -35.25046157836914,
      "activations/layer9_attention_weight_max": 53.02478790283203,
      "activations/layer9_attention_weight_min": -50.82799530029297,
      "epoch": 2.97,
      "learning_rate": 0.00013218219696969696,
      "loss": 2.9059,
      "step": 51100
    },
    {
      "activations/layer0_attention_weight_max": 14.665178298950195,
      "activations/layer0_attention_weight_min": -14.35537052154541,
      "activations/layer10_attention_weight_max": 45.292179107666016,
      "activations/layer10_attention_weight_min": -35.628761291503906,
      "activations/layer11_attention_weight_max": 41.014793395996094,
      "activations/layer11_attention_weight_min": -28.45572280883789,
      "activations/layer12_attention_weight_max": 27.137964248657227,
      "activations/layer12_attention_weight_min": -28.617053985595703,
      "activations/layer13_attention_weight_max": 84.88893127441406,
      "activations/layer13_attention_weight_min": -51.534080505371094,
      "activations/layer14_attention_weight_max": 78.87443542480469,
      "activations/layer14_attention_weight_min": -50.6473503112793,
      "activations/layer15_attention_weight_max": 50.592559814453125,
      "activations/layer15_attention_weight_min": -33.58678436279297,
      "activations/layer16_attention_weight_max": 37.789100646972656,
      "activations/layer16_attention_weight_min": -26.81100845336914,
      "activations/layer17_attention_weight_max": 53.79642105102539,
      "activations/layer17_attention_weight_min": -34.447174072265625,
      "activations/layer18_attention_weight_max": 47.51552963256836,
      "activations/layer18_attention_weight_min": -31.951496124267578,
      "activations/layer19_attention_weight_max": 21.764570236206055,
      "activations/layer19_attention_weight_min": -18.307655334472656,
      "activations/layer1_attention_weight_max": 19.19581413269043,
      "activations/layer1_attention_weight_min": -13.156411170959473,
      "activations/layer20_attention_weight_max": 20.641223907470703,
      "activations/layer20_attention_weight_min": -17.626222610473633,
      "activations/layer21_attention_weight_max": 38.354793548583984,
      "activations/layer21_attention_weight_min": -26.36026954650879,
      "activations/layer22_attention_weight_max": 32.959068298339844,
      "activations/layer22_attention_weight_min": -22.56169891357422,
      "activations/layer23_attention_weight_max": 22.892345428466797,
      "activations/layer23_attention_weight_min": -15.201778411865234,
      "activations/layer2_attention_weight_max": 29.220966339111328,
      "activations/layer2_attention_weight_min": -26.95535659790039,
      "activations/layer3_attention_weight_max": 73.17881774902344,
      "activations/layer3_attention_weight_min": -81.86817932128906,
      "activations/layer4_attention_weight_max": 102.00507354736328,
      "activations/layer4_attention_weight_min": -101.92774200439453,
      "activations/layer5_attention_weight_max": 125.55580139160156,
      "activations/layer5_attention_weight_min": -106.32805633544922,
      "activations/layer6_attention_weight_max": 65.11412811279297,
      "activations/layer6_attention_weight_min": -52.29222869873047,
      "activations/layer7_attention_weight_max": 63.07086181640625,
      "activations/layer7_attention_weight_min": -52.97474670410156,
      "activations/layer8_attention_weight_max": 42.283287048339844,
      "activations/layer8_attention_weight_min": -34.28827667236328,
      "activations/layer9_attention_weight_max": 55.84534454345703,
      "activations/layer9_attention_weight_min": -51.0778923034668,
      "epoch": 2.97,
      "learning_rate": 0.00013216325757575757,
      "loss": 2.9043,
      "step": 51150
    },
    {
      "activations/layer0_attention_weight_max": 14.380044937133789,
      "activations/layer0_attention_weight_min": -14.27510929107666,
      "activations/layer10_attention_weight_max": 47.33072280883789,
      "activations/layer10_attention_weight_min": -37.2027473449707,
      "activations/layer11_attention_weight_max": 36.51287078857422,
      "activations/layer11_attention_weight_min": -28.616649627685547,
      "activations/layer12_attention_weight_max": 30.826431274414062,
      "activations/layer12_attention_weight_min": -21.809894561767578,
      "activations/layer13_attention_weight_max": 63.5330924987793,
      "activations/layer13_attention_weight_min": -41.063385009765625,
      "activations/layer14_attention_weight_max": 56.73597717285156,
      "activations/layer14_attention_weight_min": -33.43073654174805,
      "activations/layer15_attention_weight_max": 57.41382598876953,
      "activations/layer15_attention_weight_min": -34.466529846191406,
      "activations/layer16_attention_weight_max": 39.924049377441406,
      "activations/layer16_attention_weight_min": -26.74798011779785,
      "activations/layer17_attention_weight_max": 68.4309310913086,
      "activations/layer17_attention_weight_min": -35.652618408203125,
      "activations/layer18_attention_weight_max": 51.898773193359375,
      "activations/layer18_attention_weight_min": -32.13519287109375,
      "activations/layer19_attention_weight_max": 26.659414291381836,
      "activations/layer19_attention_weight_min": -16.375478744506836,
      "activations/layer1_attention_weight_max": 16.702037811279297,
      "activations/layer1_attention_weight_min": -12.838074684143066,
      "activations/layer20_attention_weight_max": 23.820016860961914,
      "activations/layer20_attention_weight_min": -20.559154510498047,
      "activations/layer21_attention_weight_max": 48.44647216796875,
      "activations/layer21_attention_weight_min": -27.79153060913086,
      "activations/layer22_attention_weight_max": 38.31087875366211,
      "activations/layer22_attention_weight_min": -22.47662925720215,
      "activations/layer23_attention_weight_max": 28.248023986816406,
      "activations/layer23_attention_weight_min": -15.911657333374023,
      "activations/layer2_attention_weight_max": 27.35909080505371,
      "activations/layer2_attention_weight_min": -26.331157684326172,
      "activations/layer3_attention_weight_max": 67.6838607788086,
      "activations/layer3_attention_weight_min": -69.84973907470703,
      "activations/layer4_attention_weight_max": 93.46287536621094,
      "activations/layer4_attention_weight_min": -84.55424499511719,
      "activations/layer5_attention_weight_max": 118.87059783935547,
      "activations/layer5_attention_weight_min": -98.30685424804688,
      "activations/layer6_attention_weight_max": 58.30020523071289,
      "activations/layer6_attention_weight_min": -51.62260437011719,
      "activations/layer7_attention_weight_max": 60.7144660949707,
      "activations/layer7_attention_weight_min": -53.11927795410156,
      "activations/layer8_attention_weight_max": 39.376564025878906,
      "activations/layer8_attention_weight_min": -35.12883377075195,
      "activations/layer9_attention_weight_max": 50.45394515991211,
      "activations/layer9_attention_weight_min": -47.97532272338867,
      "epoch": 2.97,
      "learning_rate": 0.0001321443181818182,
      "loss": 2.9206,
      "step": 51200
    },
    {
      "activations/layer0_attention_weight_max": 13.542572975158691,
      "activations/layer0_attention_weight_min": -14.459004402160645,
      "activations/layer10_attention_weight_max": 47.580955505371094,
      "activations/layer10_attention_weight_min": -37.14118194580078,
      "activations/layer11_attention_weight_max": 37.308128356933594,
      "activations/layer11_attention_weight_min": -28.809797286987305,
      "activations/layer12_attention_weight_max": 27.6064510345459,
      "activations/layer12_attention_weight_min": -26.58597183227539,
      "activations/layer13_attention_weight_max": 50.55479049682617,
      "activations/layer13_attention_weight_min": -40.7482795715332,
      "activations/layer14_attention_weight_max": 60.305660247802734,
      "activations/layer14_attention_weight_min": -39.324710845947266,
      "activations/layer15_attention_weight_max": 49.472251892089844,
      "activations/layer15_attention_weight_min": -36.611297607421875,
      "activations/layer16_attention_weight_max": 40.54307174682617,
      "activations/layer16_attention_weight_min": -28.03006362915039,
      "activations/layer17_attention_weight_max": 60.43194580078125,
      "activations/layer17_attention_weight_min": -37.62250518798828,
      "activations/layer18_attention_weight_max": 57.83576583862305,
      "activations/layer18_attention_weight_min": -34.03956604003906,
      "activations/layer19_attention_weight_max": 25.982213973999023,
      "activations/layer19_attention_weight_min": -18.028392791748047,
      "activations/layer1_attention_weight_max": 19.064655303955078,
      "activations/layer1_attention_weight_min": -13.670768737792969,
      "activations/layer20_attention_weight_max": 19.883811950683594,
      "activations/layer20_attention_weight_min": -20.279346466064453,
      "activations/layer21_attention_weight_max": 43.00311279296875,
      "activations/layer21_attention_weight_min": -28.68674659729004,
      "activations/layer22_attention_weight_max": 38.181034088134766,
      "activations/layer22_attention_weight_min": -23.724124908447266,
      "activations/layer23_attention_weight_max": 23.917091369628906,
      "activations/layer23_attention_weight_min": -15.843528747558594,
      "activations/layer2_attention_weight_max": 28.270469665527344,
      "activations/layer2_attention_weight_min": -26.320388793945312,
      "activations/layer3_attention_weight_max": 73.21979522705078,
      "activations/layer3_attention_weight_min": -76.01679992675781,
      "activations/layer4_attention_weight_max": 103.69606018066406,
      "activations/layer4_attention_weight_min": -90.65856170654297,
      "activations/layer5_attention_weight_max": 128.7337646484375,
      "activations/layer5_attention_weight_min": -104.09835815429688,
      "activations/layer6_attention_weight_max": 66.0111083984375,
      "activations/layer6_attention_weight_min": -52.41743087768555,
      "activations/layer7_attention_weight_max": 65.15180969238281,
      "activations/layer7_attention_weight_min": -55.708274841308594,
      "activations/layer8_attention_weight_max": 41.93833541870117,
      "activations/layer8_attention_weight_min": -39.1763801574707,
      "activations/layer9_attention_weight_max": 56.51966094970703,
      "activations/layer9_attention_weight_min": -56.148719787597656,
      "epoch": 2.98,
      "learning_rate": 0.00013212537878787878,
      "loss": 2.9178,
      "step": 51250
    },
    {
      "activations/layer0_attention_weight_max": 16.029977798461914,
      "activations/layer0_attention_weight_min": -14.537428855895996,
      "activations/layer10_attention_weight_max": 46.6209602355957,
      "activations/layer10_attention_weight_min": -35.86228942871094,
      "activations/layer11_attention_weight_max": 39.380008697509766,
      "activations/layer11_attention_weight_min": -29.331340789794922,
      "activations/layer12_attention_weight_max": 30.33487319946289,
      "activations/layer12_attention_weight_min": -23.227169036865234,
      "activations/layer13_attention_weight_max": 73.25347900390625,
      "activations/layer13_attention_weight_min": -45.54762268066406,
      "activations/layer14_attention_weight_max": 59.61722183227539,
      "activations/layer14_attention_weight_min": -35.01474380493164,
      "activations/layer15_attention_weight_max": 55.06989288330078,
      "activations/layer15_attention_weight_min": -33.50519561767578,
      "activations/layer16_attention_weight_max": 50.891639709472656,
      "activations/layer16_attention_weight_min": -26.198699951171875,
      "activations/layer17_attention_weight_max": 64.2651596069336,
      "activations/layer17_attention_weight_min": -37.21164321899414,
      "activations/layer18_attention_weight_max": 62.12959289550781,
      "activations/layer18_attention_weight_min": -32.734962463378906,
      "activations/layer19_attention_weight_max": 29.328720092773438,
      "activations/layer19_attention_weight_min": -18.51934814453125,
      "activations/layer1_attention_weight_max": 17.41660499572754,
      "activations/layer1_attention_weight_min": -13.383986473083496,
      "activations/layer20_attention_weight_max": 25.16805076599121,
      "activations/layer20_attention_weight_min": -18.708477020263672,
      "activations/layer21_attention_weight_max": 55.08927917480469,
      "activations/layer21_attention_weight_min": -25.53084373474121,
      "activations/layer22_attention_weight_max": 44.97418975830078,
      "activations/layer22_attention_weight_min": -23.456560134887695,
      "activations/layer23_attention_weight_max": 29.63446044921875,
      "activations/layer23_attention_weight_min": -17.835861206054688,
      "activations/layer2_attention_weight_max": 29.0213680267334,
      "activations/layer2_attention_weight_min": -27.601730346679688,
      "activations/layer3_attention_weight_max": 69.4609375,
      "activations/layer3_attention_weight_min": -74.19926452636719,
      "activations/layer4_attention_weight_max": 98.67515563964844,
      "activations/layer4_attention_weight_min": -89.0986099243164,
      "activations/layer5_attention_weight_max": 123.10098266601562,
      "activations/layer5_attention_weight_min": -104.2663803100586,
      "activations/layer6_attention_weight_max": 63.24578857421875,
      "activations/layer6_attention_weight_min": -50.42524337768555,
      "activations/layer7_attention_weight_max": 59.385093688964844,
      "activations/layer7_attention_weight_min": -56.416168212890625,
      "activations/layer8_attention_weight_max": 43.12580490112305,
      "activations/layer8_attention_weight_min": -36.86072540283203,
      "activations/layer9_attention_weight_max": 54.795692443847656,
      "activations/layer9_attention_weight_min": -47.35218811035156,
      "epoch": 2.98,
      "learning_rate": 0.0001321064393939394,
      "loss": 2.9056,
      "step": 51300
    },
    {
      "activations/layer0_attention_weight_max": 14.223556518554688,
      "activations/layer0_attention_weight_min": -14.553787231445312,
      "activations/layer10_attention_weight_max": 49.96550369262695,
      "activations/layer10_attention_weight_min": -34.93336486816406,
      "activations/layer11_attention_weight_max": 44.2226676940918,
      "activations/layer11_attention_weight_min": -28.485210418701172,
      "activations/layer12_attention_weight_max": 25.591508865356445,
      "activations/layer12_attention_weight_min": -31.34113121032715,
      "activations/layer13_attention_weight_max": 75.90274810791016,
      "activations/layer13_attention_weight_min": -44.67561340332031,
      "activations/layer14_attention_weight_max": 70.41282653808594,
      "activations/layer14_attention_weight_min": -37.330345153808594,
      "activations/layer15_attention_weight_max": 53.431392669677734,
      "activations/layer15_attention_weight_min": -31.8512020111084,
      "activations/layer16_attention_weight_max": 38.154090881347656,
      "activations/layer16_attention_weight_min": -25.51654052734375,
      "activations/layer17_attention_weight_max": 67.77149200439453,
      "activations/layer17_attention_weight_min": -34.83926773071289,
      "activations/layer18_attention_weight_max": 56.887638092041016,
      "activations/layer18_attention_weight_min": -32.33263397216797,
      "activations/layer19_attention_weight_max": 24.870805740356445,
      "activations/layer19_attention_weight_min": -17.585500717163086,
      "activations/layer1_attention_weight_max": 18.820714950561523,
      "activations/layer1_attention_weight_min": -14.666972160339355,
      "activations/layer20_attention_weight_max": 20.85274314880371,
      "activations/layer20_attention_weight_min": -17.480653762817383,
      "activations/layer21_attention_weight_max": 43.7210807800293,
      "activations/layer21_attention_weight_min": -25.42841148376465,
      "activations/layer22_attention_weight_max": 34.273189544677734,
      "activations/layer22_attention_weight_min": -21.785736083984375,
      "activations/layer23_attention_weight_max": 28.579723358154297,
      "activations/layer23_attention_weight_min": -16.1627197265625,
      "activations/layer2_attention_weight_max": 28.403411865234375,
      "activations/layer2_attention_weight_min": -27.87834930419922,
      "activations/layer3_attention_weight_max": 73.91869354248047,
      "activations/layer3_attention_weight_min": -75.75464630126953,
      "activations/layer4_attention_weight_max": 106.82258605957031,
      "activations/layer4_attention_weight_min": -91.5749740600586,
      "activations/layer5_attention_weight_max": 125.17211151123047,
      "activations/layer5_attention_weight_min": -97.96200561523438,
      "activations/layer6_attention_weight_max": 61.25159454345703,
      "activations/layer6_attention_weight_min": -51.32987976074219,
      "activations/layer7_attention_weight_max": 70.178466796875,
      "activations/layer7_attention_weight_min": -56.702457427978516,
      "activations/layer8_attention_weight_max": 43.61001205444336,
      "activations/layer8_attention_weight_min": -34.00689697265625,
      "activations/layer9_attention_weight_max": 63.32454299926758,
      "activations/layer9_attention_weight_min": -50.53835678100586,
      "epoch": 2.98,
      "learning_rate": 0.0001320875,
      "loss": 2.8907,
      "step": 51350
    },
    {
      "activations/layer0_attention_weight_max": 15.17154598236084,
      "activations/layer0_attention_weight_min": -15.016120910644531,
      "activations/layer10_attention_weight_max": 49.12131881713867,
      "activations/layer10_attention_weight_min": -34.782203674316406,
      "activations/layer11_attention_weight_max": 42.537696838378906,
      "activations/layer11_attention_weight_min": -28.53200340270996,
      "activations/layer12_attention_weight_max": 27.48213005065918,
      "activations/layer12_attention_weight_min": -26.64183807373047,
      "activations/layer13_attention_weight_max": 71.57249450683594,
      "activations/layer13_attention_weight_min": -39.12384796142578,
      "activations/layer14_attention_weight_max": 61.688568115234375,
      "activations/layer14_attention_weight_min": -33.38694763183594,
      "activations/layer15_attention_weight_max": 58.86623001098633,
      "activations/layer15_attention_weight_min": -32.39336013793945,
      "activations/layer16_attention_weight_max": 42.186038970947266,
      "activations/layer16_attention_weight_min": -25.923940658569336,
      "activations/layer17_attention_weight_max": 63.916748046875,
      "activations/layer17_attention_weight_min": -38.9821891784668,
      "activations/layer18_attention_weight_max": 59.77631759643555,
      "activations/layer18_attention_weight_min": -31.951435089111328,
      "activations/layer19_attention_weight_max": 27.306777954101562,
      "activations/layer19_attention_weight_min": -18.544326782226562,
      "activations/layer1_attention_weight_max": 18.015058517456055,
      "activations/layer1_attention_weight_min": -14.77548599243164,
      "activations/layer20_attention_weight_max": 23.79794692993164,
      "activations/layer20_attention_weight_min": -18.749282836914062,
      "activations/layer21_attention_weight_max": 43.45783615112305,
      "activations/layer21_attention_weight_min": -28.794897079467773,
      "activations/layer22_attention_weight_max": 35.95017623901367,
      "activations/layer22_attention_weight_min": -24.797821044921875,
      "activations/layer23_attention_weight_max": 25.35840606689453,
      "activations/layer23_attention_weight_min": -16.74505615234375,
      "activations/layer2_attention_weight_max": 30.623294830322266,
      "activations/layer2_attention_weight_min": -28.598236083984375,
      "activations/layer3_attention_weight_max": 76.53782653808594,
      "activations/layer3_attention_weight_min": -78.2567367553711,
      "activations/layer4_attention_weight_max": 110.70697021484375,
      "activations/layer4_attention_weight_min": -93.19646453857422,
      "activations/layer5_attention_weight_max": 131.97824096679688,
      "activations/layer5_attention_weight_min": -101.91576385498047,
      "activations/layer6_attention_weight_max": 68.09162902832031,
      "activations/layer6_attention_weight_min": -51.14023208618164,
      "activations/layer7_attention_weight_max": 68.5925064086914,
      "activations/layer7_attention_weight_min": -58.83313751220703,
      "activations/layer8_attention_weight_max": 42.079715728759766,
      "activations/layer8_attention_weight_min": -36.05406951904297,
      "activations/layer9_attention_weight_max": 59.39039993286133,
      "activations/layer9_attention_weight_min": -47.4854621887207,
      "epoch": 2.99,
      "learning_rate": 0.0001320685606060606,
      "loss": 2.9124,
      "step": 51400
    },
    {
      "activations/layer0_attention_weight_max": 14.69674015045166,
      "activations/layer0_attention_weight_min": -14.792365074157715,
      "activations/layer10_attention_weight_max": 43.94102096557617,
      "activations/layer10_attention_weight_min": -39.054542541503906,
      "activations/layer11_attention_weight_max": 35.547245025634766,
      "activations/layer11_attention_weight_min": -30.41570281982422,
      "activations/layer12_attention_weight_max": 28.192020416259766,
      "activations/layer12_attention_weight_min": -24.125667572021484,
      "activations/layer13_attention_weight_max": 52.11888122558594,
      "activations/layer13_attention_weight_min": -40.07343673706055,
      "activations/layer14_attention_weight_max": 43.12639617919922,
      "activations/layer14_attention_weight_min": -29.75981330871582,
      "activations/layer15_attention_weight_max": 46.744747161865234,
      "activations/layer15_attention_weight_min": -34.165828704833984,
      "activations/layer16_attention_weight_max": 37.85271453857422,
      "activations/layer16_attention_weight_min": -29.961484909057617,
      "activations/layer17_attention_weight_max": 59.750244140625,
      "activations/layer17_attention_weight_min": -36.191253662109375,
      "activations/layer18_attention_weight_max": 57.564300537109375,
      "activations/layer18_attention_weight_min": -33.483585357666016,
      "activations/layer19_attention_weight_max": 19.625980377197266,
      "activations/layer19_attention_weight_min": -17.290321350097656,
      "activations/layer1_attention_weight_max": 19.177494049072266,
      "activations/layer1_attention_weight_min": -15.0704927444458,
      "activations/layer20_attention_weight_max": 20.52730941772461,
      "activations/layer20_attention_weight_min": -16.955106735229492,
      "activations/layer21_attention_weight_max": 50.20169448852539,
      "activations/layer21_attention_weight_min": -28.785152435302734,
      "activations/layer22_attention_weight_max": 32.61259841918945,
      "activations/layer22_attention_weight_min": -21.82339859008789,
      "activations/layer23_attention_weight_max": 24.27051544189453,
      "activations/layer23_attention_weight_min": -17.475276947021484,
      "activations/layer2_attention_weight_max": 29.148725509643555,
      "activations/layer2_attention_weight_min": -27.39021873474121,
      "activations/layer3_attention_weight_max": 73.45428466796875,
      "activations/layer3_attention_weight_min": -74.93030548095703,
      "activations/layer4_attention_weight_max": 102.80870819091797,
      "activations/layer4_attention_weight_min": -91.69915008544922,
      "activations/layer5_attention_weight_max": 125.09623718261719,
      "activations/layer5_attention_weight_min": -105.72502136230469,
      "activations/layer6_attention_weight_max": 62.342254638671875,
      "activations/layer6_attention_weight_min": -49.489994049072266,
      "activations/layer7_attention_weight_max": 60.898040771484375,
      "activations/layer7_attention_weight_min": -55.83656692504883,
      "activations/layer8_attention_weight_max": 41.301307678222656,
      "activations/layer8_attention_weight_min": -35.16192626953125,
      "activations/layer9_attention_weight_max": 54.07292556762695,
      "activations/layer9_attention_weight_min": -49.45186233520508,
      "epoch": 2.99,
      "learning_rate": 0.00013204999999999998,
      "loss": 2.9192,
      "step": 51450
    },
    {
      "activations/layer0_attention_weight_max": 13.450092315673828,
      "activations/layer0_attention_weight_min": -14.654211044311523,
      "activations/layer10_attention_weight_max": 45.9382438659668,
      "activations/layer10_attention_weight_min": -37.72776794433594,
      "activations/layer11_attention_weight_max": 36.40755844116211,
      "activations/layer11_attention_weight_min": -30.50363540649414,
      "activations/layer12_attention_weight_max": 25.111373901367188,
      "activations/layer12_attention_weight_min": -26.706470489501953,
      "activations/layer13_attention_weight_max": 64.20487213134766,
      "activations/layer13_attention_weight_min": -41.674339294433594,
      "activations/layer14_attention_weight_max": 56.82876968383789,
      "activations/layer14_attention_weight_min": -34.0589485168457,
      "activations/layer15_attention_weight_max": 46.73988342285156,
      "activations/layer15_attention_weight_min": -34.46152114868164,
      "activations/layer16_attention_weight_max": 36.94218063354492,
      "activations/layer16_attention_weight_min": -27.58732032775879,
      "activations/layer17_attention_weight_max": 57.03871154785156,
      "activations/layer17_attention_weight_min": -37.315887451171875,
      "activations/layer18_attention_weight_max": 47.57303237915039,
      "activations/layer18_attention_weight_min": -31.29376220703125,
      "activations/layer19_attention_weight_max": 21.3607177734375,
      "activations/layer19_attention_weight_min": -20.062528610229492,
      "activations/layer1_attention_weight_max": 17.83110237121582,
      "activations/layer1_attention_weight_min": -14.646263122558594,
      "activations/layer20_attention_weight_max": 19.932218551635742,
      "activations/layer20_attention_weight_min": -22.97991180419922,
      "activations/layer21_attention_weight_max": 42.26516342163086,
      "activations/layer21_attention_weight_min": -26.75465965270996,
      "activations/layer22_attention_weight_max": 37.44424057006836,
      "activations/layer22_attention_weight_min": -23.047746658325195,
      "activations/layer23_attention_weight_max": 25.435176849365234,
      "activations/layer23_attention_weight_min": -15.90418815612793,
      "activations/layer2_attention_weight_max": 32.36745071411133,
      "activations/layer2_attention_weight_min": -30.612478256225586,
      "activations/layer3_attention_weight_max": 76.23146057128906,
      "activations/layer3_attention_weight_min": -79.76243591308594,
      "activations/layer4_attention_weight_max": 102.1963119506836,
      "activations/layer4_attention_weight_min": -97.88671875,
      "activations/layer5_attention_weight_max": 131.10391235351562,
      "activations/layer5_attention_weight_min": -110.66033172607422,
      "activations/layer6_attention_weight_max": 63.221595764160156,
      "activations/layer6_attention_weight_min": -53.91851043701172,
      "activations/layer7_attention_weight_max": 61.18540573120117,
      "activations/layer7_attention_weight_min": -56.21707534790039,
      "activations/layer8_attention_weight_max": 44.09717559814453,
      "activations/layer8_attention_weight_min": -38.87031555175781,
      "activations/layer9_attention_weight_max": 55.8603515625,
      "activations/layer9_attention_weight_min": -51.83636474609375,
      "epoch": 2.99,
      "learning_rate": 0.0001320310606060606,
      "loss": 2.9139,
      "step": 51500
    },
    {
      "activations/layer0_attention_weight_max": 15.024176597595215,
      "activations/layer0_attention_weight_min": -14.362764358520508,
      "activations/layer10_attention_weight_max": 46.96326446533203,
      "activations/layer10_attention_weight_min": -34.74722671508789,
      "activations/layer11_attention_weight_max": 37.68022155761719,
      "activations/layer11_attention_weight_min": -26.867338180541992,
      "activations/layer12_attention_weight_max": 28.492446899414062,
      "activations/layer12_attention_weight_min": -24.62482452392578,
      "activations/layer13_attention_weight_max": 59.6495361328125,
      "activations/layer13_attention_weight_min": -40.54506301879883,
      "activations/layer14_attention_weight_max": 52.87210464477539,
      "activations/layer14_attention_weight_min": -32.0151252746582,
      "activations/layer15_attention_weight_max": 46.753379821777344,
      "activations/layer15_attention_weight_min": -33.057437896728516,
      "activations/layer16_attention_weight_max": 44.01361083984375,
      "activations/layer16_attention_weight_min": -27.856460571289062,
      "activations/layer17_attention_weight_max": 56.4868278503418,
      "activations/layer17_attention_weight_min": -40.4901008605957,
      "activations/layer18_attention_weight_max": 50.92735290527344,
      "activations/layer18_attention_weight_min": -32.33823776245117,
      "activations/layer19_attention_weight_max": 20.6624813079834,
      "activations/layer19_attention_weight_min": -18.87123680114746,
      "activations/layer1_attention_weight_max": 16.899391174316406,
      "activations/layer1_attention_weight_min": -16.009090423583984,
      "activations/layer20_attention_weight_max": 18.870759963989258,
      "activations/layer20_attention_weight_min": -23.418012619018555,
      "activations/layer21_attention_weight_max": 42.01768112182617,
      "activations/layer21_attention_weight_min": -26.201580047607422,
      "activations/layer22_attention_weight_max": 38.29120635986328,
      "activations/layer22_attention_weight_min": -21.33032989501953,
      "activations/layer23_attention_weight_max": 24.570819854736328,
      "activations/layer23_attention_weight_min": -16.878150939941406,
      "activations/layer2_attention_weight_max": 29.969884872436523,
      "activations/layer2_attention_weight_min": -28.211002349853516,
      "activations/layer3_attention_weight_max": 68.13339233398438,
      "activations/layer3_attention_weight_min": -70.48831176757812,
      "activations/layer4_attention_weight_max": 101.22935485839844,
      "activations/layer4_attention_weight_min": -84.83549499511719,
      "activations/layer5_attention_weight_max": 117.58113098144531,
      "activations/layer5_attention_weight_min": -99.49346923828125,
      "activations/layer6_attention_weight_max": 59.90757369995117,
      "activations/layer6_attention_weight_min": -51.16053009033203,
      "activations/layer7_attention_weight_max": 57.35546875,
      "activations/layer7_attention_weight_min": -55.197715759277344,
      "activations/layer8_attention_weight_max": 38.5289192199707,
      "activations/layer8_attention_weight_min": -33.962684631347656,
      "activations/layer9_attention_weight_max": 52.60238265991211,
      "activations/layer9_attention_weight_min": -45.79772186279297,
      "epoch": 3.0,
      "learning_rate": 0.00013201212121212121,
      "loss": 2.9242,
      "step": 51550
    },
    {
      "activations/layer0_attention_weight_max": 13.76194953918457,
      "activations/layer0_attention_weight_min": -14.37716007232666,
      "activations/layer10_attention_weight_max": 42.0206298828125,
      "activations/layer10_attention_weight_min": -36.176612854003906,
      "activations/layer11_attention_weight_max": 34.6911735534668,
      "activations/layer11_attention_weight_min": -29.021316528320312,
      "activations/layer12_attention_weight_max": 25.588043212890625,
      "activations/layer12_attention_weight_min": -27.708864212036133,
      "activations/layer13_attention_weight_max": 45.27561569213867,
      "activations/layer13_attention_weight_min": -38.181312561035156,
      "activations/layer14_attention_weight_max": 41.31894302368164,
      "activations/layer14_attention_weight_min": -32.873260498046875,
      "activations/layer15_attention_weight_max": 41.18704605102539,
      "activations/layer15_attention_weight_min": -33.033935546875,
      "activations/layer16_attention_weight_max": 33.715415954589844,
      "activations/layer16_attention_weight_min": -26.870681762695312,
      "activations/layer17_attention_weight_max": 53.32658386230469,
      "activations/layer17_attention_weight_min": -34.96770095825195,
      "activations/layer18_attention_weight_max": 45.5888671875,
      "activations/layer18_attention_weight_min": -30.70219612121582,
      "activations/layer19_attention_weight_max": 19.4609317779541,
      "activations/layer19_attention_weight_min": -17.334850311279297,
      "activations/layer1_attention_weight_max": 17.396318435668945,
      "activations/layer1_attention_weight_min": -16.716873168945312,
      "activations/layer20_attention_weight_max": 18.543346405029297,
      "activations/layer20_attention_weight_min": -19.100954055786133,
      "activations/layer21_attention_weight_max": 35.735595703125,
      "activations/layer21_attention_weight_min": -28.95231819152832,
      "activations/layer22_attention_weight_max": 34.00971603393555,
      "activations/layer22_attention_weight_min": -22.149513244628906,
      "activations/layer23_attention_weight_max": 22.075958251953125,
      "activations/layer23_attention_weight_min": -14.67054557800293,
      "activations/layer2_attention_weight_max": 30.536100387573242,
      "activations/layer2_attention_weight_min": -29.486263275146484,
      "activations/layer3_attention_weight_max": 72.66334533691406,
      "activations/layer3_attention_weight_min": -75.15009307861328,
      "activations/layer4_attention_weight_max": 101.53911590576172,
      "activations/layer4_attention_weight_min": -92.8375473022461,
      "activations/layer5_attention_weight_max": 123.89823913574219,
      "activations/layer5_attention_weight_min": -104.99449157714844,
      "activations/layer6_attention_weight_max": 64.52315521240234,
      "activations/layer6_attention_weight_min": -51.98634719848633,
      "activations/layer7_attention_weight_max": 58.38741683959961,
      "activations/layer7_attention_weight_min": -56.67573547363281,
      "activations/layer8_attention_weight_max": 36.862876892089844,
      "activations/layer8_attention_weight_min": -38.36247634887695,
      "activations/layer9_attention_weight_max": 58.020381927490234,
      "activations/layer9_attention_weight_min": -55.143951416015625,
      "epoch": 3.0,
      "learning_rate": 0.0001319931818181818,
      "loss": 2.9027,
      "step": 51600
    },
    {
      "activations/layer0_attention_weight_max": 14.127700805664062,
      "activations/layer0_attention_weight_min": -14.542671203613281,
      "activations/layer10_attention_weight_max": 48.9644660949707,
      "activations/layer10_attention_weight_min": -36.95610046386719,
      "activations/layer11_attention_weight_max": 35.4693489074707,
      "activations/layer11_attention_weight_min": -29.180633544921875,
      "activations/layer12_attention_weight_max": 41.57598876953125,
      "activations/layer12_attention_weight_min": -28.37854766845703,
      "activations/layer13_attention_weight_max": 63.814151763916016,
      "activations/layer13_attention_weight_min": -48.673763275146484,
      "activations/layer14_attention_weight_max": 50.69552993774414,
      "activations/layer14_attention_weight_min": -38.32679748535156,
      "activations/layer15_attention_weight_max": 48.83238220214844,
      "activations/layer15_attention_weight_min": -35.09925079345703,
      "activations/layer16_attention_weight_max": 37.70361328125,
      "activations/layer16_attention_weight_min": -26.27766227722168,
      "activations/layer17_attention_weight_max": 61.67532730102539,
      "activations/layer17_attention_weight_min": -34.769649505615234,
      "activations/layer18_attention_weight_max": 55.14089584350586,
      "activations/layer18_attention_weight_min": -31.196048736572266,
      "activations/layer19_attention_weight_max": 26.003217697143555,
      "activations/layer19_attention_weight_min": -22.43096923828125,
      "activations/layer1_attention_weight_max": 17.350805282592773,
      "activations/layer1_attention_weight_min": -14.79710578918457,
      "activations/layer20_attention_weight_max": 22.06769371032715,
      "activations/layer20_attention_weight_min": -19.05811309814453,
      "activations/layer21_attention_weight_max": 40.3913459777832,
      "activations/layer21_attention_weight_min": -27.17715072631836,
      "activations/layer22_attention_weight_max": 35.61601257324219,
      "activations/layer22_attention_weight_min": -21.332202911376953,
      "activations/layer23_attention_weight_max": 23.65435791015625,
      "activations/layer23_attention_weight_min": -15.889652252197266,
      "activations/layer2_attention_weight_max": 30.045700073242188,
      "activations/layer2_attention_weight_min": -28.093503952026367,
      "activations/layer3_attention_weight_max": 72.32958221435547,
      "activations/layer3_attention_weight_min": -76.54033660888672,
      "activations/layer4_attention_weight_max": 105.1691665649414,
      "activations/layer4_attention_weight_min": -92.2224349975586,
      "activations/layer5_attention_weight_max": 124.29441833496094,
      "activations/layer5_attention_weight_min": -104.20606994628906,
      "activations/layer6_attention_weight_max": 60.055320739746094,
      "activations/layer6_attention_weight_min": -51.83103942871094,
      "activations/layer7_attention_weight_max": 63.523765563964844,
      "activations/layer7_attention_weight_min": -55.29108810424805,
      "activations/layer8_attention_weight_max": 42.07073974609375,
      "activations/layer8_attention_weight_min": -37.824134826660156,
      "activations/layer9_attention_weight_max": 55.02736282348633,
      "activations/layer9_attention_weight_min": -45.89360046386719,
      "epoch": 3.0,
      "learning_rate": 0.00013197386363636361,
      "loss": 2.9505,
      "step": 51650
    },
    {
      "activations/layer0_attention_weight_max": 13.33355712890625,
      "activations/layer0_attention_weight_min": -13.87847900390625,
      "activations/layer10_attention_weight_max": 43.12882995605469,
      "activations/layer10_attention_weight_min": -35.33645248413086,
      "activations/layer11_attention_weight_max": 38.372474670410156,
      "activations/layer11_attention_weight_min": -26.30719757080078,
      "activations/layer12_attention_weight_max": 25.512596130371094,
      "activations/layer12_attention_weight_min": -24.160444259643555,
      "activations/layer13_attention_weight_max": 52.164127349853516,
      "activations/layer13_attention_weight_min": -37.821044921875,
      "activations/layer14_attention_weight_max": 45.151649475097656,
      "activations/layer14_attention_weight_min": -32.11202621459961,
      "activations/layer15_attention_weight_max": 46.437503814697266,
      "activations/layer15_attention_weight_min": -31.631603240966797,
      "activations/layer16_attention_weight_max": 36.5279655456543,
      "activations/layer16_attention_weight_min": -26.03396987915039,
      "activations/layer17_attention_weight_max": 58.66440963745117,
      "activations/layer17_attention_weight_min": -36.226253509521484,
      "activations/layer18_attention_weight_max": 50.236270904541016,
      "activations/layer18_attention_weight_min": -30.95458984375,
      "activations/layer19_attention_weight_max": 19.32657241821289,
      "activations/layer19_attention_weight_min": -18.652023315429688,
      "activations/layer1_attention_weight_max": 19.025087356567383,
      "activations/layer1_attention_weight_min": -14.255793571472168,
      "activations/layer20_attention_weight_max": 20.137577056884766,
      "activations/layer20_attention_weight_min": -19.978055953979492,
      "activations/layer21_attention_weight_max": 38.47690963745117,
      "activations/layer21_attention_weight_min": -25.014310836791992,
      "activations/layer22_attention_weight_max": 37.07062911987305,
      "activations/layer22_attention_weight_min": -24.804325103759766,
      "activations/layer23_attention_weight_max": 23.880504608154297,
      "activations/layer23_attention_weight_min": -16.875093460083008,
      "activations/layer2_attention_weight_max": 28.506065368652344,
      "activations/layer2_attention_weight_min": -26.717863082885742,
      "activations/layer3_attention_weight_max": 70.6597671508789,
      "activations/layer3_attention_weight_min": -77.68717193603516,
      "activations/layer4_attention_weight_max": 99.55010986328125,
      "activations/layer4_attention_weight_min": -90.40438842773438,
      "activations/layer5_attention_weight_max": 118.13525390625,
      "activations/layer5_attention_weight_min": -101.51232147216797,
      "activations/layer6_attention_weight_max": 58.42451095581055,
      "activations/layer6_attention_weight_min": -50.205589294433594,
      "activations/layer7_attention_weight_max": 59.81744384765625,
      "activations/layer7_attention_weight_min": -54.52519226074219,
      "activations/layer8_attention_weight_max": 36.60796356201172,
      "activations/layer8_attention_weight_min": -34.07660675048828,
      "activations/layer9_attention_weight_max": 50.762062072753906,
      "activations/layer9_attention_weight_min": -47.30624771118164,
      "epoch": 3.0,
      "learning_rate": 0.00013195492424242423,
      "loss": 2.8867,
      "step": 51700
    },
    {
      "activations/layer0_attention_weight_max": 15.241124153137207,
      "activations/layer0_attention_weight_min": -15.209320068359375,
      "activations/layer10_attention_weight_max": 48.812313079833984,
      "activations/layer10_attention_weight_min": -36.913631439208984,
      "activations/layer11_attention_weight_max": 36.46092987060547,
      "activations/layer11_attention_weight_min": -27.052661895751953,
      "activations/layer12_attention_weight_max": 24.731464385986328,
      "activations/layer12_attention_weight_min": -23.741113662719727,
      "activations/layer13_attention_weight_max": 64.56397247314453,
      "activations/layer13_attention_weight_min": -44.039791107177734,
      "activations/layer14_attention_weight_max": 55.52594757080078,
      "activations/layer14_attention_weight_min": -36.83285903930664,
      "activations/layer15_attention_weight_max": 55.424522399902344,
      "activations/layer15_attention_weight_min": -33.65787887573242,
      "activations/layer16_attention_weight_max": 39.535926818847656,
      "activations/layer16_attention_weight_min": -27.096792221069336,
      "activations/layer17_attention_weight_max": 67.56134796142578,
      "activations/layer17_attention_weight_min": -36.62853240966797,
      "activations/layer18_attention_weight_max": 55.587249755859375,
      "activations/layer18_attention_weight_min": -31.49837875366211,
      "activations/layer19_attention_weight_max": 25.733325958251953,
      "activations/layer19_attention_weight_min": -17.609643936157227,
      "activations/layer1_attention_weight_max": 17.977800369262695,
      "activations/layer1_attention_weight_min": -16.007543563842773,
      "activations/layer20_attention_weight_max": 22.731576919555664,
      "activations/layer20_attention_weight_min": -17.36206817626953,
      "activations/layer21_attention_weight_max": 43.481929779052734,
      "activations/layer21_attention_weight_min": -27.205406188964844,
      "activations/layer22_attention_weight_max": 36.379112243652344,
      "activations/layer22_attention_weight_min": -24.088281631469727,
      "activations/layer23_attention_weight_max": 26.192623138427734,
      "activations/layer23_attention_weight_min": -19.978988647460938,
      "activations/layer2_attention_weight_max": 27.370044708251953,
      "activations/layer2_attention_weight_min": -26.64437484741211,
      "activations/layer3_attention_weight_max": 68.40147399902344,
      "activations/layer3_attention_weight_min": -71.60953521728516,
      "activations/layer4_attention_weight_max": 100.28052520751953,
      "activations/layer4_attention_weight_min": -89.05509948730469,
      "activations/layer5_attention_weight_max": 129.34352111816406,
      "activations/layer5_attention_weight_min": -105.18769836425781,
      "activations/layer6_attention_weight_max": 64.94567108154297,
      "activations/layer6_attention_weight_min": -51.34846878051758,
      "activations/layer7_attention_weight_max": 58.88551712036133,
      "activations/layer7_attention_weight_min": -52.55732727050781,
      "activations/layer8_attention_weight_max": 40.945457458496094,
      "activations/layer8_attention_weight_min": -35.95979690551758,
      "activations/layer9_attention_weight_max": 50.13711166381836,
      "activations/layer9_attention_weight_min": -48.040889739990234,
      "epoch": 3.01,
      "learning_rate": 0.00013193598484848484,
      "loss": 2.8982,
      "step": 51750
    },
    {
      "activations/layer0_attention_weight_max": 14.264483451843262,
      "activations/layer0_attention_weight_min": -14.856576919555664,
      "activations/layer10_attention_weight_max": 46.65849304199219,
      "activations/layer10_attention_weight_min": -36.15253448486328,
      "activations/layer11_attention_weight_max": 39.28226089477539,
      "activations/layer11_attention_weight_min": -29.079193115234375,
      "activations/layer12_attention_weight_max": 32.78882598876953,
      "activations/layer12_attention_weight_min": -24.744888305664062,
      "activations/layer13_attention_weight_max": 49.84375762939453,
      "activations/layer13_attention_weight_min": -35.142738342285156,
      "activations/layer14_attention_weight_max": 54.94952392578125,
      "activations/layer14_attention_weight_min": -35.99518966674805,
      "activations/layer15_attention_weight_max": 48.5928840637207,
      "activations/layer15_attention_weight_min": -33.88349914550781,
      "activations/layer16_attention_weight_max": 41.572288513183594,
      "activations/layer16_attention_weight_min": -29.863672256469727,
      "activations/layer17_attention_weight_max": 63.23421859741211,
      "activations/layer17_attention_weight_min": -36.43356704711914,
      "activations/layer18_attention_weight_max": 62.687435150146484,
      "activations/layer18_attention_weight_min": -32.711246490478516,
      "activations/layer19_attention_weight_max": 27.189899444580078,
      "activations/layer19_attention_weight_min": -17.0542049407959,
      "activations/layer1_attention_weight_max": 18.160070419311523,
      "activations/layer1_attention_weight_min": -15.561769485473633,
      "activations/layer20_attention_weight_max": 26.371768951416016,
      "activations/layer20_attention_weight_min": -17.529895782470703,
      "activations/layer21_attention_weight_max": 49.62031173706055,
      "activations/layer21_attention_weight_min": -29.96160888671875,
      "activations/layer22_attention_weight_max": 40.36054229736328,
      "activations/layer22_attention_weight_min": -23.589313507080078,
      "activations/layer23_attention_weight_max": 29.157424926757812,
      "activations/layer23_attention_weight_min": -15.407249450683594,
      "activations/layer2_attention_weight_max": 28.05714225769043,
      "activations/layer2_attention_weight_min": -27.167743682861328,
      "activations/layer3_attention_weight_max": 70.96955871582031,
      "activations/layer3_attention_weight_min": -71.76819610595703,
      "activations/layer4_attention_weight_max": 95.75386810302734,
      "activations/layer4_attention_weight_min": -87.4122085571289,
      "activations/layer5_attention_weight_max": 120.66220092773438,
      "activations/layer5_attention_weight_min": -100.86602783203125,
      "activations/layer6_attention_weight_max": 60.80391311645508,
      "activations/layer6_attention_weight_min": -50.066463470458984,
      "activations/layer7_attention_weight_max": 58.60700988769531,
      "activations/layer7_attention_weight_min": -53.755802154541016,
      "activations/layer8_attention_weight_max": 38.973548889160156,
      "activations/layer8_attention_weight_min": -34.19532012939453,
      "activations/layer9_attention_weight_max": 54.402889251708984,
      "activations/layer9_attention_weight_min": -48.03276443481445,
      "epoch": 3.01,
      "learning_rate": 0.00013191704545454543,
      "loss": 2.9143,
      "step": 51800
    },
    {
      "activations/layer0_attention_weight_max": 14.225115776062012,
      "activations/layer0_attention_weight_min": -14.693288803100586,
      "activations/layer10_attention_weight_max": 42.92524719238281,
      "activations/layer10_attention_weight_min": -38.52314376831055,
      "activations/layer11_attention_weight_max": 35.26813507080078,
      "activations/layer11_attention_weight_min": -27.14627456665039,
      "activations/layer12_attention_weight_max": 26.569231033325195,
      "activations/layer12_attention_weight_min": -24.32282257080078,
      "activations/layer13_attention_weight_max": 62.84021759033203,
      "activations/layer13_attention_weight_min": -41.038177490234375,
      "activations/layer14_attention_weight_max": 77.51110076904297,
      "activations/layer14_attention_weight_min": -42.56976318359375,
      "activations/layer15_attention_weight_max": 56.496402740478516,
      "activations/layer15_attention_weight_min": -35.12747573852539,
      "activations/layer16_attention_weight_max": 39.376644134521484,
      "activations/layer16_attention_weight_min": -26.940969467163086,
      "activations/layer17_attention_weight_max": 58.64910125732422,
      "activations/layer17_attention_weight_min": -35.89091873168945,
      "activations/layer18_attention_weight_max": 52.512027740478516,
      "activations/layer18_attention_weight_min": -32.3168830871582,
      "activations/layer19_attention_weight_max": 26.476598739624023,
      "activations/layer19_attention_weight_min": -18.304824829101562,
      "activations/layer1_attention_weight_max": 17.359630584716797,
      "activations/layer1_attention_weight_min": -14.322871208190918,
      "activations/layer20_attention_weight_max": 25.47344398498535,
      "activations/layer20_attention_weight_min": -21.300174713134766,
      "activations/layer21_attention_weight_max": 46.867340087890625,
      "activations/layer21_attention_weight_min": -26.951358795166016,
      "activations/layer22_attention_weight_max": 39.77381134033203,
      "activations/layer22_attention_weight_min": -23.589885711669922,
      "activations/layer23_attention_weight_max": 28.012571334838867,
      "activations/layer23_attention_weight_min": -18.820045471191406,
      "activations/layer2_attention_weight_max": 30.11453628540039,
      "activations/layer2_attention_weight_min": -27.233448028564453,
      "activations/layer3_attention_weight_max": 72.69203186035156,
      "activations/layer3_attention_weight_min": -70.17256927490234,
      "activations/layer4_attention_weight_max": 102.23455810546875,
      "activations/layer4_attention_weight_min": -93.59276580810547,
      "activations/layer5_attention_weight_max": 126.58106994628906,
      "activations/layer5_attention_weight_min": -100.60528564453125,
      "activations/layer6_attention_weight_max": 63.41324234008789,
      "activations/layer6_attention_weight_min": -53.34345626831055,
      "activations/layer7_attention_weight_max": 61.71153259277344,
      "activations/layer7_attention_weight_min": -52.739620208740234,
      "activations/layer8_attention_weight_max": 40.19649887084961,
      "activations/layer8_attention_weight_min": -34.84478759765625,
      "activations/layer9_attention_weight_max": 59.77523422241211,
      "activations/layer9_attention_weight_min": -48.39828109741211,
      "epoch": 3.01,
      "learning_rate": 0.00013189810606060605,
      "loss": 2.8904,
      "step": 51850
    },
    {
      "activations/layer0_attention_weight_max": 14.769553184509277,
      "activations/layer0_attention_weight_min": -14.550597190856934,
      "activations/layer10_attention_weight_max": 43.632041931152344,
      "activations/layer10_attention_weight_min": -35.47502517700195,
      "activations/layer11_attention_weight_max": 35.34382629394531,
      "activations/layer11_attention_weight_min": -28.094993591308594,
      "activations/layer12_attention_weight_max": 27.292484283447266,
      "activations/layer12_attention_weight_min": -23.407934188842773,
      "activations/layer13_attention_weight_max": 52.04330825805664,
      "activations/layer13_attention_weight_min": -35.71169662475586,
      "activations/layer14_attention_weight_max": 44.84367370605469,
      "activations/layer14_attention_weight_min": -33.838199615478516,
      "activations/layer15_attention_weight_max": 48.204017639160156,
      "activations/layer15_attention_weight_min": -33.49845504760742,
      "activations/layer16_attention_weight_max": 42.098514556884766,
      "activations/layer16_attention_weight_min": -26.996212005615234,
      "activations/layer17_attention_weight_max": 65.29638671875,
      "activations/layer17_attention_weight_min": -40.128562927246094,
      "activations/layer18_attention_weight_max": 59.43769454956055,
      "activations/layer18_attention_weight_min": -31.85987663269043,
      "activations/layer19_attention_weight_max": 27.639911651611328,
      "activations/layer19_attention_weight_min": -19.27226448059082,
      "activations/layer1_attention_weight_max": 16.77761459350586,
      "activations/layer1_attention_weight_min": -13.460458755493164,
      "activations/layer20_attention_weight_max": 22.76604461669922,
      "activations/layer20_attention_weight_min": -17.638896942138672,
      "activations/layer21_attention_weight_max": 47.70951843261719,
      "activations/layer21_attention_weight_min": -28.320241928100586,
      "activations/layer22_attention_weight_max": 38.04804992675781,
      "activations/layer22_attention_weight_min": -24.256364822387695,
      "activations/layer23_attention_weight_max": 26.003887176513672,
      "activations/layer23_attention_weight_min": -17.403907775878906,
      "activations/layer2_attention_weight_max": 30.15530776977539,
      "activations/layer2_attention_weight_min": -27.280006408691406,
      "activations/layer3_attention_weight_max": 70.16950988769531,
      "activations/layer3_attention_weight_min": -74.88805389404297,
      "activations/layer4_attention_weight_max": 97.2208251953125,
      "activations/layer4_attention_weight_min": -88.34317016601562,
      "activations/layer5_attention_weight_max": 118.15352630615234,
      "activations/layer5_attention_weight_min": -98.606201171875,
      "activations/layer6_attention_weight_max": 64.56748962402344,
      "activations/layer6_attention_weight_min": -49.07503890991211,
      "activations/layer7_attention_weight_max": 58.54127502441406,
      "activations/layer7_attention_weight_min": -51.74497604370117,
      "activations/layer8_attention_weight_max": 39.34335708618164,
      "activations/layer8_attention_weight_min": -37.46548843383789,
      "activations/layer9_attention_weight_max": 49.666465759277344,
      "activations/layer9_attention_weight_min": -50.18275451660156,
      "epoch": 3.02,
      "learning_rate": 0.00013187916666666664,
      "loss": 2.93,
      "step": 51900
    },
    {
      "activations/layer0_attention_weight_max": 15.844167709350586,
      "activations/layer0_attention_weight_min": -14.226736068725586,
      "activations/layer10_attention_weight_max": 51.408931732177734,
      "activations/layer10_attention_weight_min": -36.68953323364258,
      "activations/layer11_attention_weight_max": 43.29645919799805,
      "activations/layer11_attention_weight_min": -29.42046356201172,
      "activations/layer12_attention_weight_max": 31.572446823120117,
      "activations/layer12_attention_weight_min": -24.953750610351562,
      "activations/layer13_attention_weight_max": 59.61973571777344,
      "activations/layer13_attention_weight_min": -40.52570724487305,
      "activations/layer14_attention_weight_max": 77.00509643554688,
      "activations/layer14_attention_weight_min": -45.1279296875,
      "activations/layer15_attention_weight_max": 58.33717346191406,
      "activations/layer15_attention_weight_min": -34.05699920654297,
      "activations/layer16_attention_weight_max": 44.576133728027344,
      "activations/layer16_attention_weight_min": -27.130422592163086,
      "activations/layer17_attention_weight_max": 63.84490966796875,
      "activations/layer17_attention_weight_min": -36.425472259521484,
      "activations/layer18_attention_weight_max": 62.297447204589844,
      "activations/layer18_attention_weight_min": -34.922420501708984,
      "activations/layer19_attention_weight_max": 29.741683959960938,
      "activations/layer19_attention_weight_min": -16.00226593017578,
      "activations/layer1_attention_weight_max": 18.256855010986328,
      "activations/layer1_attention_weight_min": -13.12403392791748,
      "activations/layer20_attention_weight_max": 24.6875057220459,
      "activations/layer20_attention_weight_min": -16.247514724731445,
      "activations/layer21_attention_weight_max": 55.84382247924805,
      "activations/layer21_attention_weight_min": -30.901256561279297,
      "activations/layer22_attention_weight_max": 41.56702423095703,
      "activations/layer22_attention_weight_min": -20.878232955932617,
      "activations/layer23_attention_weight_max": 30.457618713378906,
      "activations/layer23_attention_weight_min": -15.411205291748047,
      "activations/layer2_attention_weight_max": 30.76248550415039,
      "activations/layer2_attention_weight_min": -28.830947875976562,
      "activations/layer3_attention_weight_max": 68.2942123413086,
      "activations/layer3_attention_weight_min": -72.7549057006836,
      "activations/layer4_attention_weight_max": 98.68889617919922,
      "activations/layer4_attention_weight_min": -89.51860809326172,
      "activations/layer5_attention_weight_max": 127.43584442138672,
      "activations/layer5_attention_weight_min": -107.45925903320312,
      "activations/layer6_attention_weight_max": 64.03044128417969,
      "activations/layer6_attention_weight_min": -54.48495864868164,
      "activations/layer7_attention_weight_max": 68.8257827758789,
      "activations/layer7_attention_weight_min": -63.03950881958008,
      "activations/layer8_attention_weight_max": 52.91551971435547,
      "activations/layer8_attention_weight_min": -40.29106521606445,
      "activations/layer9_attention_weight_max": 74.76180267333984,
      "activations/layer9_attention_weight_min": -54.937374114990234,
      "epoch": 3.02,
      "learning_rate": 0.00013186022727272725,
      "loss": 2.911,
      "step": 51950
    },
    {
      "activations/layer0_attention_weight_max": 13.598959922790527,
      "activations/layer0_attention_weight_min": -14.682929992675781,
      "activations/layer10_attention_weight_max": 49.04899215698242,
      "activations/layer10_attention_weight_min": -37.93170166015625,
      "activations/layer11_attention_weight_max": 42.9582633972168,
      "activations/layer11_attention_weight_min": -26.523151397705078,
      "activations/layer12_attention_weight_max": 28.349058151245117,
      "activations/layer12_attention_weight_min": -24.425201416015625,
      "activations/layer13_attention_weight_max": 61.02602005004883,
      "activations/layer13_attention_weight_min": -40.38788604736328,
      "activations/layer14_attention_weight_max": 47.2181510925293,
      "activations/layer14_attention_weight_min": -31.386795043945312,
      "activations/layer15_attention_weight_max": 52.23003387451172,
      "activations/layer15_attention_weight_min": -33.81132507324219,
      "activations/layer16_attention_weight_max": 40.21193313598633,
      "activations/layer16_attention_weight_min": -25.962297439575195,
      "activations/layer17_attention_weight_max": 56.8364143371582,
      "activations/layer17_attention_weight_min": -34.024959564208984,
      "activations/layer18_attention_weight_max": 48.53609848022461,
      "activations/layer18_attention_weight_min": -32.088172912597656,
      "activations/layer19_attention_weight_max": 21.812305450439453,
      "activations/layer19_attention_weight_min": -19.15033721923828,
      "activations/layer1_attention_weight_max": 17.276042938232422,
      "activations/layer1_attention_weight_min": -15.22708511352539,
      "activations/layer20_attention_weight_max": 20.32285499572754,
      "activations/layer20_attention_weight_min": -20.922374725341797,
      "activations/layer21_attention_weight_max": 42.16553497314453,
      "activations/layer21_attention_weight_min": -28.203861236572266,
      "activations/layer22_attention_weight_max": 35.925331115722656,
      "activations/layer22_attention_weight_min": -25.35101318359375,
      "activations/layer23_attention_weight_max": 28.1751766204834,
      "activations/layer23_attention_weight_min": -18.893966674804688,
      "activations/layer2_attention_weight_max": 28.20553970336914,
      "activations/layer2_attention_weight_min": -26.589935302734375,
      "activations/layer3_attention_weight_max": 71.0777359008789,
      "activations/layer3_attention_weight_min": -74.62290954589844,
      "activations/layer4_attention_weight_max": 101.75885009765625,
      "activations/layer4_attention_weight_min": -95.90010833740234,
      "activations/layer5_attention_weight_max": 121.72142028808594,
      "activations/layer5_attention_weight_min": -95.69918823242188,
      "activations/layer6_attention_weight_max": 59.5422248840332,
      "activations/layer6_attention_weight_min": -49.40946960449219,
      "activations/layer7_attention_weight_max": 59.56843948364258,
      "activations/layer7_attention_weight_min": -58.273006439208984,
      "activations/layer8_attention_weight_max": 42.42550277709961,
      "activations/layer8_attention_weight_min": -37.27973175048828,
      "activations/layer9_attention_weight_max": 54.01477813720703,
      "activations/layer9_attention_weight_min": -49.66775131225586,
      "epoch": 3.02,
      "learning_rate": 0.00013184128787878787,
      "loss": 2.8849,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_loss": 2.857421875,
      "eval_runtime": 8.537,
      "eval_samples_per_second": 502.984,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_openwebtext_loss": 2.857421875,
      "eval_openwebtext_ppl": 17.416566918622852,
      "eval_openwebtext_runtime": 8.537,
      "eval_openwebtext_samples_per_second": 502.984,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_wikitext_loss": 3.107421875,
      "eval_wikitext_ppl": 22.36331459495464,
      "eval_wikitext_runtime": 1.6468,
      "eval_wikitext_samples_per_second": 276.897,
      "step": 52000
    },
    {
      "epoch": 3.02,
      "eval_lambada_loss": 3.0234375,
      "eval_lambada_ppl": 20.56185170491249,
      "eval_lambada_runtime": 9.2114,
      "eval_lambada_samples_per_second": 528.585,
      "step": 52000
    },
    {
      "activations/layer0_attention_weight_max": 16.27885627746582,
      "activations/layer0_attention_weight_min": -14.6343994140625,
      "activations/layer10_attention_weight_max": 45.88079833984375,
      "activations/layer10_attention_weight_min": -39.74837112426758,
      "activations/layer11_attention_weight_max": 35.659088134765625,
      "activations/layer11_attention_weight_min": -28.073993682861328,
      "activations/layer12_attention_weight_max": 29.748600006103516,
      "activations/layer12_attention_weight_min": -28.624391555786133,
      "activations/layer13_attention_weight_max": 49.99312210083008,
      "activations/layer13_attention_weight_min": -39.27317810058594,
      "activations/layer14_attention_weight_max": 39.39139938354492,
      "activations/layer14_attention_weight_min": -31.589658737182617,
      "activations/layer15_attention_weight_max": 42.307498931884766,
      "activations/layer15_attention_weight_min": -32.22064971923828,
      "activations/layer16_attention_weight_max": 36.33330535888672,
      "activations/layer16_attention_weight_min": -26.872753143310547,
      "activations/layer17_attention_weight_max": 56.94851303100586,
      "activations/layer17_attention_weight_min": -37.83012008666992,
      "activations/layer18_attention_weight_max": 48.7796516418457,
      "activations/layer18_attention_weight_min": -31.76472282409668,
      "activations/layer19_attention_weight_max": 20.3463134765625,
      "activations/layer19_attention_weight_min": -19.089704513549805,
      "activations/layer1_attention_weight_max": 17.55243492126465,
      "activations/layer1_attention_weight_min": -12.737235069274902,
      "activations/layer20_attention_weight_max": 20.955663681030273,
      "activations/layer20_attention_weight_min": -17.594791412353516,
      "activations/layer21_attention_weight_max": 36.33321762084961,
      "activations/layer21_attention_weight_min": -26.36556625366211,
      "activations/layer22_attention_weight_max": 32.68155288696289,
      "activations/layer22_attention_weight_min": -24.0706729888916,
      "activations/layer23_attention_weight_max": 24.270170211791992,
      "activations/layer23_attention_weight_min": -16.60173988342285,
      "activations/layer2_attention_weight_max": 28.388519287109375,
      "activations/layer2_attention_weight_min": -27.457685470581055,
      "activations/layer3_attention_weight_max": 70.76455688476562,
      "activations/layer3_attention_weight_min": -75.34757995605469,
      "activations/layer4_attention_weight_max": 94.48465728759766,
      "activations/layer4_attention_weight_min": -89.4527359008789,
      "activations/layer5_attention_weight_max": 117.00634765625,
      "activations/layer5_attention_weight_min": -101.9188232421875,
      "activations/layer6_attention_weight_max": 59.258880615234375,
      "activations/layer6_attention_weight_min": -51.40745544433594,
      "activations/layer7_attention_weight_max": 59.65824890136719,
      "activations/layer7_attention_weight_min": -57.985633850097656,
      "activations/layer8_attention_weight_max": 40.37764358520508,
      "activations/layer8_attention_weight_min": -37.1148681640625,
      "activations/layer9_attention_weight_max": 50.781700134277344,
      "activations/layer9_attention_weight_min": -48.264339447021484,
      "epoch": 3.02,
      "learning_rate": 0.00013182234848484845,
      "loss": 2.8961,
      "step": 52050
    },
    {
      "activations/layer0_attention_weight_max": 13.647168159484863,
      "activations/layer0_attention_weight_min": -15.141507148742676,
      "activations/layer10_attention_weight_max": 51.168052673339844,
      "activations/layer10_attention_weight_min": -36.82118606567383,
      "activations/layer11_attention_weight_max": 37.82832717895508,
      "activations/layer11_attention_weight_min": -28.968490600585938,
      "activations/layer12_attention_weight_max": 29.41977882385254,
      "activations/layer12_attention_weight_min": -25.473648071289062,
      "activations/layer13_attention_weight_max": 54.06559371948242,
      "activations/layer13_attention_weight_min": -36.191009521484375,
      "activations/layer14_attention_weight_max": 48.36180114746094,
      "activations/layer14_attention_weight_min": -32.23542404174805,
      "activations/layer15_attention_weight_max": 47.58146667480469,
      "activations/layer15_attention_weight_min": -31.549087524414062,
      "activations/layer16_attention_weight_max": 36.525394439697266,
      "activations/layer16_attention_weight_min": -26.141021728515625,
      "activations/layer17_attention_weight_max": 64.87252044677734,
      "activations/layer17_attention_weight_min": -34.62824630737305,
      "activations/layer18_attention_weight_max": 58.109588623046875,
      "activations/layer18_attention_weight_min": -31.878803253173828,
      "activations/layer19_attention_weight_max": 24.31739616394043,
      "activations/layer19_attention_weight_min": -18.18844223022461,
      "activations/layer1_attention_weight_max": 18.28308868408203,
      "activations/layer1_attention_weight_min": -16.00060272216797,
      "activations/layer20_attention_weight_max": 24.217313766479492,
      "activations/layer20_attention_weight_min": -20.513547897338867,
      "activations/layer21_attention_weight_max": 56.75640869140625,
      "activations/layer21_attention_weight_min": -27.305889129638672,
      "activations/layer22_attention_weight_max": 38.65557098388672,
      "activations/layer22_attention_weight_min": -22.18311309814453,
      "activations/layer23_attention_weight_max": 27.57785415649414,
      "activations/layer23_attention_weight_min": -18.053543090820312,
      "activations/layer2_attention_weight_max": 27.639806747436523,
      "activations/layer2_attention_weight_min": -26.76991844177246,
      "activations/layer3_attention_weight_max": 69.77025604248047,
      "activations/layer3_attention_weight_min": -73.9129867553711,
      "activations/layer4_attention_weight_max": 98.5809326171875,
      "activations/layer4_attention_weight_min": -89.05357360839844,
      "activations/layer5_attention_weight_max": 125.4788818359375,
      "activations/layer5_attention_weight_min": -97.12570190429688,
      "activations/layer6_attention_weight_max": 62.437294006347656,
      "activations/layer6_attention_weight_min": -50.163211822509766,
      "activations/layer7_attention_weight_max": 60.21616744995117,
      "activations/layer7_attention_weight_min": -54.00437545776367,
      "activations/layer8_attention_weight_max": 41.46489715576172,
      "activations/layer8_attention_weight_min": -36.0810432434082,
      "activations/layer9_attention_weight_max": 52.68059158325195,
      "activations/layer9_attention_weight_min": -45.16941452026367,
      "epoch": 3.03,
      "learning_rate": 0.00013180340909090907,
      "loss": 2.8977,
      "step": 52100
    },
    {
      "activations/layer0_attention_weight_max": 15.134330749511719,
      "activations/layer0_attention_weight_min": -14.894735336303711,
      "activations/layer10_attention_weight_max": 45.456024169921875,
      "activations/layer10_attention_weight_min": -37.21464920043945,
      "activations/layer11_attention_weight_max": 35.571990966796875,
      "activations/layer11_attention_weight_min": -29.09325408935547,
      "activations/layer12_attention_weight_max": 31.59585189819336,
      "activations/layer12_attention_weight_min": -25.747154235839844,
      "activations/layer13_attention_weight_max": 53.83443832397461,
      "activations/layer13_attention_weight_min": -40.4609489440918,
      "activations/layer14_attention_weight_max": 53.226192474365234,
      "activations/layer14_attention_weight_min": -36.054054260253906,
      "activations/layer15_attention_weight_max": 48.317352294921875,
      "activations/layer15_attention_weight_min": -37.87413787841797,
      "activations/layer16_attention_weight_max": 38.961307525634766,
      "activations/layer16_attention_weight_min": -26.418508529663086,
      "activations/layer17_attention_weight_max": 64.44632720947266,
      "activations/layer17_attention_weight_min": -37.82143020629883,
      "activations/layer18_attention_weight_max": 49.71245193481445,
      "activations/layer18_attention_weight_min": -35.04570388793945,
      "activations/layer19_attention_weight_max": 22.100601196289062,
      "activations/layer19_attention_weight_min": -18.59181785583496,
      "activations/layer1_attention_weight_max": 17.296846389770508,
      "activations/layer1_attention_weight_min": -14.24871826171875,
      "activations/layer20_attention_weight_max": 21.74217987060547,
      "activations/layer20_attention_weight_min": -18.486783981323242,
      "activations/layer21_attention_weight_max": 44.276763916015625,
      "activations/layer21_attention_weight_min": -25.992029190063477,
      "activations/layer22_attention_weight_max": 36.07093811035156,
      "activations/layer22_attention_weight_min": -21.384197235107422,
      "activations/layer23_attention_weight_max": 27.58380699157715,
      "activations/layer23_attention_weight_min": -18.660120010375977,
      "activations/layer2_attention_weight_max": 28.07435417175293,
      "activations/layer2_attention_weight_min": -25.93704605102539,
      "activations/layer3_attention_weight_max": 70.59077453613281,
      "activations/layer3_attention_weight_min": -74.88798522949219,
      "activations/layer4_attention_weight_max": 94.11001586914062,
      "activations/layer4_attention_weight_min": -89.34758758544922,
      "activations/layer5_attention_weight_max": 119.84435272216797,
      "activations/layer5_attention_weight_min": -99.33854675292969,
      "activations/layer6_attention_weight_max": 60.973609924316406,
      "activations/layer6_attention_weight_min": -52.30901336669922,
      "activations/layer7_attention_weight_max": 62.623687744140625,
      "activations/layer7_attention_weight_min": -56.469078063964844,
      "activations/layer8_attention_weight_max": 38.14585494995117,
      "activations/layer8_attention_weight_min": -34.902000427246094,
      "activations/layer9_attention_weight_max": 53.744606018066406,
      "activations/layer9_attention_weight_min": -48.134429931640625,
      "epoch": 3.03,
      "learning_rate": 0.00013178446969696968,
      "loss": 2.8967,
      "step": 52150
    },
    {
      "activations/layer0_attention_weight_max": 14.17849349975586,
      "activations/layer0_attention_weight_min": -14.686142921447754,
      "activations/layer10_attention_weight_max": 40.43933868408203,
      "activations/layer10_attention_weight_min": -35.524696350097656,
      "activations/layer11_attention_weight_max": 35.675846099853516,
      "activations/layer11_attention_weight_min": -28.681188583374023,
      "activations/layer12_attention_weight_max": 22.94569206237793,
      "activations/layer12_attention_weight_min": -23.719449996948242,
      "activations/layer13_attention_weight_max": 72.43071746826172,
      "activations/layer13_attention_weight_min": -44.85402297973633,
      "activations/layer14_attention_weight_max": 54.214019775390625,
      "activations/layer14_attention_weight_min": -31.32938003540039,
      "activations/layer15_attention_weight_max": 56.927555084228516,
      "activations/layer15_attention_weight_min": -34.57743453979492,
      "activations/layer16_attention_weight_max": 42.78767776489258,
      "activations/layer16_attention_weight_min": -28.00557518005371,
      "activations/layer17_attention_weight_max": 63.06549835205078,
      "activations/layer17_attention_weight_min": -35.86669158935547,
      "activations/layer18_attention_weight_max": 56.41090774536133,
      "activations/layer18_attention_weight_min": -32.59564971923828,
      "activations/layer19_attention_weight_max": 25.14419937133789,
      "activations/layer19_attention_weight_min": -18.722766876220703,
      "activations/layer1_attention_weight_max": 17.59087371826172,
      "activations/layer1_attention_weight_min": -13.669615745544434,
      "activations/layer20_attention_weight_max": 24.037723541259766,
      "activations/layer20_attention_weight_min": -20.93787384033203,
      "activations/layer21_attention_weight_max": 49.307647705078125,
      "activations/layer21_attention_weight_min": -31.186885833740234,
      "activations/layer22_attention_weight_max": 36.227294921875,
      "activations/layer22_attention_weight_min": -24.39645767211914,
      "activations/layer23_attention_weight_max": 26.270885467529297,
      "activations/layer23_attention_weight_min": -17.656824111938477,
      "activations/layer2_attention_weight_max": 26.555646896362305,
      "activations/layer2_attention_weight_min": -25.43514633178711,
      "activations/layer3_attention_weight_max": 67.33394622802734,
      "activations/layer3_attention_weight_min": -72.31559753417969,
      "activations/layer4_attention_weight_max": 100.73992156982422,
      "activations/layer4_attention_weight_min": -93.22969818115234,
      "activations/layer5_attention_weight_max": 121.17269897460938,
      "activations/layer5_attention_weight_min": -104.01776123046875,
      "activations/layer6_attention_weight_max": 60.66835021972656,
      "activations/layer6_attention_weight_min": -49.29448699951172,
      "activations/layer7_attention_weight_max": 58.18575668334961,
      "activations/layer7_attention_weight_min": -60.857749938964844,
      "activations/layer8_attention_weight_max": 37.58122634887695,
      "activations/layer8_attention_weight_min": -36.263790130615234,
      "activations/layer9_attention_weight_max": 52.53540802001953,
      "activations/layer9_attention_weight_min": -49.723114013671875,
      "epoch": 3.03,
      "learning_rate": 0.0001317655303030303,
      "loss": 2.8944,
      "step": 52200
    },
    {
      "activations/layer0_attention_weight_max": 14.146981239318848,
      "activations/layer0_attention_weight_min": -14.902417182922363,
      "activations/layer10_attention_weight_max": 43.0058708190918,
      "activations/layer10_attention_weight_min": -34.26747512817383,
      "activations/layer11_attention_weight_max": 36.791297912597656,
      "activations/layer11_attention_weight_min": -29.19146728515625,
      "activations/layer12_attention_weight_max": 26.613094329833984,
      "activations/layer12_attention_weight_min": -25.475021362304688,
      "activations/layer13_attention_weight_max": 54.44252014160156,
      "activations/layer13_attention_weight_min": -39.0434684753418,
      "activations/layer14_attention_weight_max": 54.94249725341797,
      "activations/layer14_attention_weight_min": -34.09432601928711,
      "activations/layer15_attention_weight_max": 44.353965759277344,
      "activations/layer15_attention_weight_min": -31.970867156982422,
      "activations/layer16_attention_weight_max": 38.20600509643555,
      "activations/layer16_attention_weight_min": -27.904603958129883,
      "activations/layer17_attention_weight_max": 60.07859420776367,
      "activations/layer17_attention_weight_min": -37.20521926879883,
      "activations/layer18_attention_weight_max": 51.32785415649414,
      "activations/layer18_attention_weight_min": -33.59341812133789,
      "activations/layer19_attention_weight_max": 20.800626754760742,
      "activations/layer19_attention_weight_min": -18.26711082458496,
      "activations/layer1_attention_weight_max": 17.099384307861328,
      "activations/layer1_attention_weight_min": -13.823917388916016,
      "activations/layer20_attention_weight_max": 23.267887115478516,
      "activations/layer20_attention_weight_min": -18.968477249145508,
      "activations/layer21_attention_weight_max": 42.41181564331055,
      "activations/layer21_attention_weight_min": -27.627975463867188,
      "activations/layer22_attention_weight_max": 34.03053283691406,
      "activations/layer22_attention_weight_min": -22.65984535217285,
      "activations/layer23_attention_weight_max": 26.23664093017578,
      "activations/layer23_attention_weight_min": -17.17171287536621,
      "activations/layer2_attention_weight_max": 28.227642059326172,
      "activations/layer2_attention_weight_min": -28.033279418945312,
      "activations/layer3_attention_weight_max": 67.69438171386719,
      "activations/layer3_attention_weight_min": -71.20523071289062,
      "activations/layer4_attention_weight_max": 93.58287048339844,
      "activations/layer4_attention_weight_min": -87.49669647216797,
      "activations/layer5_attention_weight_max": 116.95466613769531,
      "activations/layer5_attention_weight_min": -95.58601379394531,
      "activations/layer6_attention_weight_max": 59.99934387207031,
      "activations/layer6_attention_weight_min": -49.24321746826172,
      "activations/layer7_attention_weight_max": 63.951595306396484,
      "activations/layer7_attention_weight_min": -54.03645324707031,
      "activations/layer8_attention_weight_max": 38.1087532043457,
      "activations/layer8_attention_weight_min": -35.34525680541992,
      "activations/layer9_attention_weight_max": 56.794734954833984,
      "activations/layer9_attention_weight_min": -50.13986587524414,
      "epoch": 3.04,
      "learning_rate": 0.0001317465909090909,
      "loss": 2.9037,
      "step": 52250
    },
    {
      "activations/layer0_attention_weight_max": 14.657430648803711,
      "activations/layer0_attention_weight_min": -14.879762649536133,
      "activations/layer10_attention_weight_max": 41.16469955444336,
      "activations/layer10_attention_weight_min": -35.71854782104492,
      "activations/layer11_attention_weight_max": 36.558509826660156,
      "activations/layer11_attention_weight_min": -27.86153793334961,
      "activations/layer12_attention_weight_max": 24.660837173461914,
      "activations/layer12_attention_weight_min": -25.240474700927734,
      "activations/layer13_attention_weight_max": 49.79167556762695,
      "activations/layer13_attention_weight_min": -39.4709358215332,
      "activations/layer14_attention_weight_max": 36.33058166503906,
      "activations/layer14_attention_weight_min": -28.88992691040039,
      "activations/layer15_attention_weight_max": 50.39269256591797,
      "activations/layer15_attention_weight_min": -34.930633544921875,
      "activations/layer16_attention_weight_max": 36.179351806640625,
      "activations/layer16_attention_weight_min": -25.565488815307617,
      "activations/layer17_attention_weight_max": 60.66294479370117,
      "activations/layer17_attention_weight_min": -37.17204284667969,
      "activations/layer18_attention_weight_max": 51.40214157104492,
      "activations/layer18_attention_weight_min": -33.3685417175293,
      "activations/layer19_attention_weight_max": 23.896963119506836,
      "activations/layer19_attention_weight_min": -18.15086555480957,
      "activations/layer1_attention_weight_max": 16.782428741455078,
      "activations/layer1_attention_weight_min": -14.168354034423828,
      "activations/layer20_attention_weight_max": 21.23346710205078,
      "activations/layer20_attention_weight_min": -20.799985885620117,
      "activations/layer21_attention_weight_max": 44.42625427246094,
      "activations/layer21_attention_weight_min": -31.340160369873047,
      "activations/layer22_attention_weight_max": 36.05794143676758,
      "activations/layer22_attention_weight_min": -24.30376434326172,
      "activations/layer23_attention_weight_max": 25.84786033630371,
      "activations/layer23_attention_weight_min": -16.573734283447266,
      "activations/layer2_attention_weight_max": 29.464637756347656,
      "activations/layer2_attention_weight_min": -27.65494155883789,
      "activations/layer3_attention_weight_max": 71.63756561279297,
      "activations/layer3_attention_weight_min": -72.68762969970703,
      "activations/layer4_attention_weight_max": 99.64672088623047,
      "activations/layer4_attention_weight_min": -86.98949432373047,
      "activations/layer5_attention_weight_max": 118.67501068115234,
      "activations/layer5_attention_weight_min": -98.46263122558594,
      "activations/layer6_attention_weight_max": 59.422645568847656,
      "activations/layer6_attention_weight_min": -51.27314376831055,
      "activations/layer7_attention_weight_max": 58.371376037597656,
      "activations/layer7_attention_weight_min": -54.696346282958984,
      "activations/layer8_attention_weight_max": 38.85206985473633,
      "activations/layer8_attention_weight_min": -34.74763107299805,
      "activations/layer9_attention_weight_max": 49.99275588989258,
      "activations/layer9_attention_weight_min": -46.64183807373047,
      "epoch": 3.04,
      "learning_rate": 0.0001317276515151515,
      "loss": 2.9062,
      "step": 52300
    },
    {
      "activations/layer0_attention_weight_max": 13.464054107666016,
      "activations/layer0_attention_weight_min": -14.928947448730469,
      "activations/layer10_attention_weight_max": 49.09147262573242,
      "activations/layer10_attention_weight_min": -38.424076080322266,
      "activations/layer11_attention_weight_max": 39.38908386230469,
      "activations/layer11_attention_weight_min": -27.800504684448242,
      "activations/layer12_attention_weight_max": 44.053932189941406,
      "activations/layer12_attention_weight_min": -30.86553192138672,
      "activations/layer13_attention_weight_max": 56.217044830322266,
      "activations/layer13_attention_weight_min": -42.08961868286133,
      "activations/layer14_attention_weight_max": 46.41743469238281,
      "activations/layer14_attention_weight_min": -31.777307510375977,
      "activations/layer15_attention_weight_max": 49.80976486206055,
      "activations/layer15_attention_weight_min": -34.995811462402344,
      "activations/layer16_attention_weight_max": 40.02458572387695,
      "activations/layer16_attention_weight_min": -26.49117088317871,
      "activations/layer17_attention_weight_max": 61.61809158325195,
      "activations/layer17_attention_weight_min": -41.61030960083008,
      "activations/layer18_attention_weight_max": 51.80268096923828,
      "activations/layer18_attention_weight_min": -33.00698471069336,
      "activations/layer19_attention_weight_max": 24.521780014038086,
      "activations/layer19_attention_weight_min": -18.168354034423828,
      "activations/layer1_attention_weight_max": 19.164823532104492,
      "activations/layer1_attention_weight_min": -16.668180465698242,
      "activations/layer20_attention_weight_max": 21.818157196044922,
      "activations/layer20_attention_weight_min": -19.456314086914062,
      "activations/layer21_attention_weight_max": 40.33179473876953,
      "activations/layer21_attention_weight_min": -27.825420379638672,
      "activations/layer22_attention_weight_max": 35.52924346923828,
      "activations/layer22_attention_weight_min": -22.453752517700195,
      "activations/layer23_attention_weight_max": 25.195579528808594,
      "activations/layer23_attention_weight_min": -15.662489891052246,
      "activations/layer2_attention_weight_max": 28.41037368774414,
      "activations/layer2_attention_weight_min": -26.98976707458496,
      "activations/layer3_attention_weight_max": 70.70064544677734,
      "activations/layer3_attention_weight_min": -77.82429504394531,
      "activations/layer4_attention_weight_max": 99.22248077392578,
      "activations/layer4_attention_weight_min": -99.0177230834961,
      "activations/layer5_attention_weight_max": 124.02935791015625,
      "activations/layer5_attention_weight_min": -106.22737121582031,
      "activations/layer6_attention_weight_max": 62.01958084106445,
      "activations/layer6_attention_weight_min": -51.900028228759766,
      "activations/layer7_attention_weight_max": 57.16231155395508,
      "activations/layer7_attention_weight_min": -58.041996002197266,
      "activations/layer8_attention_weight_max": 41.491233825683594,
      "activations/layer8_attention_weight_min": -35.96244812011719,
      "activations/layer9_attention_weight_max": 49.284095764160156,
      "activations/layer9_attention_weight_min": -49.042198181152344,
      "epoch": 3.04,
      "learning_rate": 0.00013170871212121212,
      "loss": 2.9083,
      "step": 52350
    },
    {
      "activations/layer0_attention_weight_max": 14.44301700592041,
      "activations/layer0_attention_weight_min": -14.499465942382812,
      "activations/layer10_attention_weight_max": 48.26301956176758,
      "activations/layer10_attention_weight_min": -37.850685119628906,
      "activations/layer11_attention_weight_max": 35.370914459228516,
      "activations/layer11_attention_weight_min": -28.494049072265625,
      "activations/layer12_attention_weight_max": 28.346940994262695,
      "activations/layer12_attention_weight_min": -23.831146240234375,
      "activations/layer13_attention_weight_max": 58.77635955810547,
      "activations/layer13_attention_weight_min": -48.943241119384766,
      "activations/layer14_attention_weight_max": 65.78102111816406,
      "activations/layer14_attention_weight_min": -50.402442932128906,
      "activations/layer15_attention_weight_max": 50.51166915893555,
      "activations/layer15_attention_weight_min": -39.693695068359375,
      "activations/layer16_attention_weight_max": 49.506778717041016,
      "activations/layer16_attention_weight_min": -26.20728302001953,
      "activations/layer17_attention_weight_max": 69.83392333984375,
      "activations/layer17_attention_weight_min": -34.86488723754883,
      "activations/layer18_attention_weight_max": 61.8790168762207,
      "activations/layer18_attention_weight_min": -30.419164657592773,
      "activations/layer19_attention_weight_max": 24.33453369140625,
      "activations/layer19_attention_weight_min": -17.81467056274414,
      "activations/layer1_attention_weight_max": 17.9844970703125,
      "activations/layer1_attention_weight_min": -18.298656463623047,
      "activations/layer20_attention_weight_max": 25.570411682128906,
      "activations/layer20_attention_weight_min": -19.51748275756836,
      "activations/layer21_attention_weight_max": 47.37689971923828,
      "activations/layer21_attention_weight_min": -26.008251190185547,
      "activations/layer22_attention_weight_max": 42.785179138183594,
      "activations/layer22_attention_weight_min": -22.715208053588867,
      "activations/layer23_attention_weight_max": 25.46526336669922,
      "activations/layer23_attention_weight_min": -17.041030883789062,
      "activations/layer2_attention_weight_max": 30.487350463867188,
      "activations/layer2_attention_weight_min": -28.396240234375,
      "activations/layer3_attention_weight_max": 72.40270233154297,
      "activations/layer3_attention_weight_min": -75.90133666992188,
      "activations/layer4_attention_weight_max": 101.4803466796875,
      "activations/layer4_attention_weight_min": -92.49813842773438,
      "activations/layer5_attention_weight_max": 126.75311279296875,
      "activations/layer5_attention_weight_min": -100.9384994506836,
      "activations/layer6_attention_weight_max": 61.777732849121094,
      "activations/layer6_attention_weight_min": -56.969730377197266,
      "activations/layer7_attention_weight_max": 64.60016632080078,
      "activations/layer7_attention_weight_min": -54.88970184326172,
      "activations/layer8_attention_weight_max": 45.09286880493164,
      "activations/layer8_attention_weight_min": -38.04665756225586,
      "activations/layer9_attention_weight_max": 59.50269317626953,
      "activations/layer9_attention_weight_min": -49.048004150390625,
      "epoch": 3.04,
      "learning_rate": 0.00013168977272727273,
      "loss": 2.8999,
      "step": 52400
    },
    {
      "activations/layer0_attention_weight_max": 14.569710731506348,
      "activations/layer0_attention_weight_min": -14.625151634216309,
      "activations/layer10_attention_weight_max": 44.20845413208008,
      "activations/layer10_attention_weight_min": -36.32294845581055,
      "activations/layer11_attention_weight_max": 38.02311706542969,
      "activations/layer11_attention_weight_min": -30.056806564331055,
      "activations/layer12_attention_weight_max": 26.242345809936523,
      "activations/layer12_attention_weight_min": -26.887853622436523,
      "activations/layer13_attention_weight_max": 55.79192352294922,
      "activations/layer13_attention_weight_min": -37.322200775146484,
      "activations/layer14_attention_weight_max": 48.479576110839844,
      "activations/layer14_attention_weight_min": -33.82276916503906,
      "activations/layer15_attention_weight_max": 45.211795806884766,
      "activations/layer15_attention_weight_min": -32.40886688232422,
      "activations/layer16_attention_weight_max": 44.0208854675293,
      "activations/layer16_attention_weight_min": -26.602664947509766,
      "activations/layer17_attention_weight_max": 65.3392333984375,
      "activations/layer17_attention_weight_min": -34.80852127075195,
      "activations/layer18_attention_weight_max": 51.55500793457031,
      "activations/layer18_attention_weight_min": -30.573299407958984,
      "activations/layer19_attention_weight_max": 22.416791915893555,
      "activations/layer19_attention_weight_min": -15.847543716430664,
      "activations/layer1_attention_weight_max": 16.86519432067871,
      "activations/layer1_attention_weight_min": -15.159001350402832,
      "activations/layer20_attention_weight_max": 19.129777908325195,
      "activations/layer20_attention_weight_min": -20.871294021606445,
      "activations/layer21_attention_weight_max": 45.32433319091797,
      "activations/layer21_attention_weight_min": -26.63713836669922,
      "activations/layer22_attention_weight_max": 43.109100341796875,
      "activations/layer22_attention_weight_min": -23.93667984008789,
      "activations/layer23_attention_weight_max": 26.21728515625,
      "activations/layer23_attention_weight_min": -17.262168884277344,
      "activations/layer2_attention_weight_max": 29.62087631225586,
      "activations/layer2_attention_weight_min": -27.103092193603516,
      "activations/layer3_attention_weight_max": 74.80724334716797,
      "activations/layer3_attention_weight_min": -76.0071029663086,
      "activations/layer4_attention_weight_max": 101.27208709716797,
      "activations/layer4_attention_weight_min": -91.9199447631836,
      "activations/layer5_attention_weight_max": 123.99021911621094,
      "activations/layer5_attention_weight_min": -100.03977966308594,
      "activations/layer6_attention_weight_max": 62.61848068237305,
      "activations/layer6_attention_weight_min": -50.1082878112793,
      "activations/layer7_attention_weight_max": 60.916595458984375,
      "activations/layer7_attention_weight_min": -56.461151123046875,
      "activations/layer8_attention_weight_max": 37.97817611694336,
      "activations/layer8_attention_weight_min": -34.43217849731445,
      "activations/layer9_attention_weight_max": 52.18440628051758,
      "activations/layer9_attention_weight_min": -48.08684158325195,
      "epoch": 3.05,
      "learning_rate": 0.00013167083333333332,
      "loss": 2.9005,
      "step": 52450
    },
    {
      "activations/layer0_attention_weight_max": 13.863478660583496,
      "activations/layer0_attention_weight_min": -15.232406616210938,
      "activations/layer10_attention_weight_max": 42.77342987060547,
      "activations/layer10_attention_weight_min": -34.41944122314453,
      "activations/layer11_attention_weight_max": 37.77008056640625,
      "activations/layer11_attention_weight_min": -26.019184112548828,
      "activations/layer12_attention_weight_max": 31.578340530395508,
      "activations/layer12_attention_weight_min": -24.70363998413086,
      "activations/layer13_attention_weight_max": 48.30974197387695,
      "activations/layer13_attention_weight_min": -36.54335403442383,
      "activations/layer14_attention_weight_max": 47.4338493347168,
      "activations/layer14_attention_weight_min": -32.20534896850586,
      "activations/layer15_attention_weight_max": 44.9692497253418,
      "activations/layer15_attention_weight_min": -34.05026626586914,
      "activations/layer16_attention_weight_max": 38.764198303222656,
      "activations/layer16_attention_weight_min": -27.631223678588867,
      "activations/layer17_attention_weight_max": 58.05909729003906,
      "activations/layer17_attention_weight_min": -34.948482513427734,
      "activations/layer18_attention_weight_max": 55.34803009033203,
      "activations/layer18_attention_weight_min": -33.56906509399414,
      "activations/layer19_attention_weight_max": 23.28481101989746,
      "activations/layer19_attention_weight_min": -17.33970832824707,
      "activations/layer1_attention_weight_max": 17.673032760620117,
      "activations/layer1_attention_weight_min": -15.090535163879395,
      "activations/layer20_attention_weight_max": 21.59592056274414,
      "activations/layer20_attention_weight_min": -21.613298416137695,
      "activations/layer21_attention_weight_max": 44.90725326538086,
      "activations/layer21_attention_weight_min": -28.090896606445312,
      "activations/layer22_attention_weight_max": 36.4935188293457,
      "activations/layer22_attention_weight_min": -23.664215087890625,
      "activations/layer23_attention_weight_max": 29.222267150878906,
      "activations/layer23_attention_weight_min": -16.852087020874023,
      "activations/layer2_attention_weight_max": 29.000761032104492,
      "activations/layer2_attention_weight_min": -27.864429473876953,
      "activations/layer3_attention_weight_max": 69.76716613769531,
      "activations/layer3_attention_weight_min": -72.6801986694336,
      "activations/layer4_attention_weight_max": 99.30500030517578,
      "activations/layer4_attention_weight_min": -88.36929321289062,
      "activations/layer5_attention_weight_max": 121.24839782714844,
      "activations/layer5_attention_weight_min": -100.4140853881836,
      "activations/layer6_attention_weight_max": 61.25197219848633,
      "activations/layer6_attention_weight_min": -49.178871154785156,
      "activations/layer7_attention_weight_max": 59.075191497802734,
      "activations/layer7_attention_weight_min": -54.542476654052734,
      "activations/layer8_attention_weight_max": 39.78068161010742,
      "activations/layer8_attention_weight_min": -33.31481170654297,
      "activations/layer9_attention_weight_max": 56.27287673950195,
      "activations/layer9_attention_weight_min": -49.24342727661133,
      "epoch": 3.05,
      "learning_rate": 0.00013165189393939393,
      "loss": 2.8862,
      "step": 52500
    },
    {
      "activations/layer0_attention_weight_max": 13.418644905090332,
      "activations/layer0_attention_weight_min": -14.485755920410156,
      "activations/layer10_attention_weight_max": 45.6937141418457,
      "activations/layer10_attention_weight_min": -35.4741096496582,
      "activations/layer11_attention_weight_max": 37.339115142822266,
      "activations/layer11_attention_weight_min": -28.503406524658203,
      "activations/layer12_attention_weight_max": 28.201757431030273,
      "activations/layer12_attention_weight_min": -26.924026489257812,
      "activations/layer13_attention_weight_max": 51.69377899169922,
      "activations/layer13_attention_weight_min": -41.21934127807617,
      "activations/layer14_attention_weight_max": 42.73671340942383,
      "activations/layer14_attention_weight_min": -33.8473014831543,
      "activations/layer15_attention_weight_max": 44.49017333984375,
      "activations/layer15_attention_weight_min": -31.76995086669922,
      "activations/layer16_attention_weight_max": 36.3631591796875,
      "activations/layer16_attention_weight_min": -26.7750244140625,
      "activations/layer17_attention_weight_max": 54.6364631652832,
      "activations/layer17_attention_weight_min": -33.87448501586914,
      "activations/layer18_attention_weight_max": 49.123191833496094,
      "activations/layer18_attention_weight_min": -31.488309860229492,
      "activations/layer19_attention_weight_max": 21.002971649169922,
      "activations/layer19_attention_weight_min": -16.099029541015625,
      "activations/layer1_attention_weight_max": 17.616365432739258,
      "activations/layer1_attention_weight_min": -14.47586727142334,
      "activations/layer20_attention_weight_max": 21.47052574157715,
      "activations/layer20_attention_weight_min": -18.792964935302734,
      "activations/layer21_attention_weight_max": 45.35272979736328,
      "activations/layer21_attention_weight_min": -28.246984481811523,
      "activations/layer22_attention_weight_max": 32.149356842041016,
      "activations/layer22_attention_weight_min": -21.45294952392578,
      "activations/layer23_attention_weight_max": 25.512771606445312,
      "activations/layer23_attention_weight_min": -15.901954650878906,
      "activations/layer2_attention_weight_max": 28.807392120361328,
      "activations/layer2_attention_weight_min": -27.464542388916016,
      "activations/layer3_attention_weight_max": 68.79916381835938,
      "activations/layer3_attention_weight_min": -75.67549133300781,
      "activations/layer4_attention_weight_max": 95.89503479003906,
      "activations/layer4_attention_weight_min": -91.88195037841797,
      "activations/layer5_attention_weight_max": 117.40288543701172,
      "activations/layer5_attention_weight_min": -104.19810485839844,
      "activations/layer6_attention_weight_max": 58.94630432128906,
      "activations/layer6_attention_weight_min": -53.640865325927734,
      "activations/layer7_attention_weight_max": 66.5789566040039,
      "activations/layer7_attention_weight_min": -57.045005798339844,
      "activations/layer8_attention_weight_max": 39.37409210205078,
      "activations/layer8_attention_weight_min": -35.12514114379883,
      "activations/layer9_attention_weight_max": 53.312801361083984,
      "activations/layer9_attention_weight_min": -51.50080490112305,
      "epoch": 3.05,
      "learning_rate": 0.00013163295454545455,
      "loss": 2.8991,
      "step": 52550
    },
    {
      "activations/layer0_attention_weight_max": 14.136266708374023,
      "activations/layer0_attention_weight_min": -14.333283424377441,
      "activations/layer10_attention_weight_max": 49.31352233886719,
      "activations/layer10_attention_weight_min": -38.16802215576172,
      "activations/layer11_attention_weight_max": 37.68351745605469,
      "activations/layer11_attention_weight_min": -28.275968551635742,
      "activations/layer12_attention_weight_max": 26.800880432128906,
      "activations/layer12_attention_weight_min": -27.382890701293945,
      "activations/layer13_attention_weight_max": 55.10432815551758,
      "activations/layer13_attention_weight_min": -40.61288070678711,
      "activations/layer14_attention_weight_max": 55.37752151489258,
      "activations/layer14_attention_weight_min": -32.412559509277344,
      "activations/layer15_attention_weight_max": 48.53871154785156,
      "activations/layer15_attention_weight_min": -36.54985046386719,
      "activations/layer16_attention_weight_max": 37.81308364868164,
      "activations/layer16_attention_weight_min": -29.019508361816406,
      "activations/layer17_attention_weight_max": 60.87295150756836,
      "activations/layer17_attention_weight_min": -39.89747619628906,
      "activations/layer18_attention_weight_max": 55.93894577026367,
      "activations/layer18_attention_weight_min": -34.304168701171875,
      "activations/layer19_attention_weight_max": 27.256561279296875,
      "activations/layer19_attention_weight_min": -19.132291793823242,
      "activations/layer1_attention_weight_max": 18.320558547973633,
      "activations/layer1_attention_weight_min": -15.433048248291016,
      "activations/layer20_attention_weight_max": 23.90813446044922,
      "activations/layer20_attention_weight_min": -19.458148956298828,
      "activations/layer21_attention_weight_max": 44.44673156738281,
      "activations/layer21_attention_weight_min": -28.782127380371094,
      "activations/layer22_attention_weight_max": 35.116493225097656,
      "activations/layer22_attention_weight_min": -24.053857803344727,
      "activations/layer23_attention_weight_max": 25.828786849975586,
      "activations/layer23_attention_weight_min": -18.92540168762207,
      "activations/layer2_attention_weight_max": 30.68999481201172,
      "activations/layer2_attention_weight_min": -29.0599422454834,
      "activations/layer3_attention_weight_max": 75.03276824951172,
      "activations/layer3_attention_weight_min": -77.85286712646484,
      "activations/layer4_attention_weight_max": 104.23644256591797,
      "activations/layer4_attention_weight_min": -96.03539276123047,
      "activations/layer5_attention_weight_max": 125.56362915039062,
      "activations/layer5_attention_weight_min": -107.9696273803711,
      "activations/layer6_attention_weight_max": 62.3875846862793,
      "activations/layer6_attention_weight_min": -54.67829513549805,
      "activations/layer7_attention_weight_max": 62.060516357421875,
      "activations/layer7_attention_weight_min": -56.582862854003906,
      "activations/layer8_attention_weight_max": 41.76216506958008,
      "activations/layer8_attention_weight_min": -36.95510482788086,
      "activations/layer9_attention_weight_max": 52.72083282470703,
      "activations/layer9_attention_weight_min": -48.941524505615234,
      "epoch": 3.06,
      "learning_rate": 0.00013161401515151514,
      "loss": 2.9042,
      "step": 52600
    },
    {
      "activations/layer0_attention_weight_max": 15.2401123046875,
      "activations/layer0_attention_weight_min": -14.887648582458496,
      "activations/layer10_attention_weight_max": 41.69493865966797,
      "activations/layer10_attention_weight_min": -34.509002685546875,
      "activations/layer11_attention_weight_max": 36.53821563720703,
      "activations/layer11_attention_weight_min": -30.048986434936523,
      "activations/layer12_attention_weight_max": 25.36729621887207,
      "activations/layer12_attention_weight_min": -28.015445709228516,
      "activations/layer13_attention_weight_max": 75.72905731201172,
      "activations/layer13_attention_weight_min": -52.7562141418457,
      "activations/layer14_attention_weight_max": 77.98614501953125,
      "activations/layer14_attention_weight_min": -52.01424026489258,
      "activations/layer15_attention_weight_max": 58.70777130126953,
      "activations/layer15_attention_weight_min": -39.89079284667969,
      "activations/layer16_attention_weight_max": 37.465423583984375,
      "activations/layer16_attention_weight_min": -26.703075408935547,
      "activations/layer17_attention_weight_max": 62.35106658935547,
      "activations/layer17_attention_weight_min": -35.12251663208008,
      "activations/layer18_attention_weight_max": 54.78984069824219,
      "activations/layer18_attention_weight_min": -35.41515350341797,
      "activations/layer19_attention_weight_max": 24.551422119140625,
      "activations/layer19_attention_weight_min": -16.88161849975586,
      "activations/layer1_attention_weight_max": 17.14813995361328,
      "activations/layer1_attention_weight_min": -15.158754348754883,
      "activations/layer20_attention_weight_max": 22.321643829345703,
      "activations/layer20_attention_weight_min": -22.708253860473633,
      "activations/layer21_attention_weight_max": 45.76070022583008,
      "activations/layer21_attention_weight_min": -30.827213287353516,
      "activations/layer22_attention_weight_max": 35.924034118652344,
      "activations/layer22_attention_weight_min": -24.849559783935547,
      "activations/layer23_attention_weight_max": 27.2127742767334,
      "activations/layer23_attention_weight_min": -18.385168075561523,
      "activations/layer2_attention_weight_max": 27.668006896972656,
      "activations/layer2_attention_weight_min": -26.203990936279297,
      "activations/layer3_attention_weight_max": 69.43379974365234,
      "activations/layer3_attention_weight_min": -74.41632843017578,
      "activations/layer4_attention_weight_max": 98.09636688232422,
      "activations/layer4_attention_weight_min": -89.7662353515625,
      "activations/layer5_attention_weight_max": 117.43310546875,
      "activations/layer5_attention_weight_min": -97.04890441894531,
      "activations/layer6_attention_weight_max": 58.91233444213867,
      "activations/layer6_attention_weight_min": -48.73434829711914,
      "activations/layer7_attention_weight_max": 60.287620544433594,
      "activations/layer7_attention_weight_min": -55.53079605102539,
      "activations/layer8_attention_weight_max": 39.848453521728516,
      "activations/layer8_attention_weight_min": -35.77919387817383,
      "activations/layer9_attention_weight_max": 59.152687072753906,
      "activations/layer9_attention_weight_min": -55.94732666015625,
      "epoch": 3.06,
      "learning_rate": 0.00013159507575757575,
      "loss": 2.8857,
      "step": 52650
    },
    {
      "activations/layer0_attention_weight_max": 14.551239013671875,
      "activations/layer0_attention_weight_min": -14.496818542480469,
      "activations/layer10_attention_weight_max": 46.626708984375,
      "activations/layer10_attention_weight_min": -35.416748046875,
      "activations/layer11_attention_weight_max": 35.61750411987305,
      "activations/layer11_attention_weight_min": -28.63030242919922,
      "activations/layer12_attention_weight_max": 31.479381561279297,
      "activations/layer12_attention_weight_min": -25.106428146362305,
      "activations/layer13_attention_weight_max": 46.33366394042969,
      "activations/layer13_attention_weight_min": -37.14895248413086,
      "activations/layer14_attention_weight_max": 44.4847412109375,
      "activations/layer14_attention_weight_min": -31.82659339904785,
      "activations/layer15_attention_weight_max": 47.68443298339844,
      "activations/layer15_attention_weight_min": -34.0860710144043,
      "activations/layer16_attention_weight_max": 41.17367935180664,
      "activations/layer16_attention_weight_min": -27.109783172607422,
      "activations/layer17_attention_weight_max": 62.73236846923828,
      "activations/layer17_attention_weight_min": -39.577964782714844,
      "activations/layer18_attention_weight_max": 55.040489196777344,
      "activations/layer18_attention_weight_min": -33.75648880004883,
      "activations/layer19_attention_weight_max": 23.851625442504883,
      "activations/layer19_attention_weight_min": -18.043071746826172,
      "activations/layer1_attention_weight_max": 17.449459075927734,
      "activations/layer1_attention_weight_min": -14.158940315246582,
      "activations/layer20_attention_weight_max": 23.238059997558594,
      "activations/layer20_attention_weight_min": -19.957120895385742,
      "activations/layer21_attention_weight_max": 42.80938720703125,
      "activations/layer21_attention_weight_min": -27.45250129699707,
      "activations/layer22_attention_weight_max": 40.328125,
      "activations/layer22_attention_weight_min": -24.94272804260254,
      "activations/layer23_attention_weight_max": 27.84418487548828,
      "activations/layer23_attention_weight_min": -16.312761306762695,
      "activations/layer2_attention_weight_max": 28.766624450683594,
      "activations/layer2_attention_weight_min": -27.24854850769043,
      "activations/layer3_attention_weight_max": 71.86479187011719,
      "activations/layer3_attention_weight_min": -74.26966857910156,
      "activations/layer4_attention_weight_max": 101.91104125976562,
      "activations/layer4_attention_weight_min": -85.24198913574219,
      "activations/layer5_attention_weight_max": 123.82887268066406,
      "activations/layer5_attention_weight_min": -97.52544403076172,
      "activations/layer6_attention_weight_max": 66.59954833984375,
      "activations/layer6_attention_weight_min": -50.59846878051758,
      "activations/layer7_attention_weight_max": 64.2674789428711,
      "activations/layer7_attention_weight_min": -57.5309944152832,
      "activations/layer8_attention_weight_max": 41.03709411621094,
      "activations/layer8_attention_weight_min": -34.37348556518555,
      "activations/layer9_attention_weight_max": 56.471553802490234,
      "activations/layer9_attention_weight_min": -45.66111373901367,
      "epoch": 3.06,
      "learning_rate": 0.00013157613636363637,
      "loss": 2.8921,
      "step": 52700
    },
    {
      "activations/layer0_attention_weight_max": 14.200196266174316,
      "activations/layer0_attention_weight_min": -14.628320693969727,
      "activations/layer10_attention_weight_max": 47.48307800292969,
      "activations/layer10_attention_weight_min": -34.24589538574219,
      "activations/layer11_attention_weight_max": 45.659942626953125,
      "activations/layer11_attention_weight_min": -30.623899459838867,
      "activations/layer12_attention_weight_max": 29.814268112182617,
      "activations/layer12_attention_weight_min": -26.036760330200195,
      "activations/layer13_attention_weight_max": 87.32036590576172,
      "activations/layer13_attention_weight_min": -46.069149017333984,
      "activations/layer14_attention_weight_max": 66.5747299194336,
      "activations/layer14_attention_weight_min": -36.128047943115234,
      "activations/layer15_attention_weight_max": 50.694580078125,
      "activations/layer15_attention_weight_min": -35.28936767578125,
      "activations/layer16_attention_weight_max": 41.42520523071289,
      "activations/layer16_attention_weight_min": -26.61137580871582,
      "activations/layer17_attention_weight_max": 64.6615982055664,
      "activations/layer17_attention_weight_min": -36.800628662109375,
      "activations/layer18_attention_weight_max": 57.86375045776367,
      "activations/layer18_attention_weight_min": -31.619352340698242,
      "activations/layer19_attention_weight_max": 25.88926124572754,
      "activations/layer19_attention_weight_min": -18.082862854003906,
      "activations/layer1_attention_weight_max": 16.84971046447754,
      "activations/layer1_attention_weight_min": -13.860688209533691,
      "activations/layer20_attention_weight_max": 23.68035125732422,
      "activations/layer20_attention_weight_min": -17.44359588623047,
      "activations/layer21_attention_weight_max": 50.300010681152344,
      "activations/layer21_attention_weight_min": -27.25958251953125,
      "activations/layer22_attention_weight_max": 43.148983001708984,
      "activations/layer22_attention_weight_min": -24.311325073242188,
      "activations/layer23_attention_weight_max": 28.660049438476562,
      "activations/layer23_attention_weight_min": -17.31279945373535,
      "activations/layer2_attention_weight_max": 30.095645904541016,
      "activations/layer2_attention_weight_min": -27.31125259399414,
      "activations/layer3_attention_weight_max": 75.20833587646484,
      "activations/layer3_attention_weight_min": -79.82562255859375,
      "activations/layer4_attention_weight_max": 105.48475646972656,
      "activations/layer4_attention_weight_min": -99.05767059326172,
      "activations/layer5_attention_weight_max": 125.25395965576172,
      "activations/layer5_attention_weight_min": -102.4549789428711,
      "activations/layer6_attention_weight_max": 63.131675720214844,
      "activations/layer6_attention_weight_min": -51.12923049926758,
      "activations/layer7_attention_weight_max": 60.385196685791016,
      "activations/layer7_attention_weight_min": -57.63587188720703,
      "activations/layer8_attention_weight_max": 41.62845993041992,
      "activations/layer8_attention_weight_min": -35.9536018371582,
      "activations/layer9_attention_weight_max": 54.24689865112305,
      "activations/layer9_attention_weight_min": -48.857723236083984,
      "epoch": 3.07,
      "learning_rate": 0.00013155719696969696,
      "loss": 2.9012,
      "step": 52750
    },
    {
      "activations/layer0_attention_weight_max": 13.936341285705566,
      "activations/layer0_attention_weight_min": -14.42602825164795,
      "activations/layer10_attention_weight_max": 42.598716735839844,
      "activations/layer10_attention_weight_min": -35.866939544677734,
      "activations/layer11_attention_weight_max": 36.3986930847168,
      "activations/layer11_attention_weight_min": -26.537273406982422,
      "activations/layer12_attention_weight_max": 26.191654205322266,
      "activations/layer12_attention_weight_min": -25.76043128967285,
      "activations/layer13_attention_weight_max": 54.49276351928711,
      "activations/layer13_attention_weight_min": -39.602394104003906,
      "activations/layer14_attention_weight_max": 62.29180908203125,
      "activations/layer14_attention_weight_min": -41.764732360839844,
      "activations/layer15_attention_weight_max": 55.47575378417969,
      "activations/layer15_attention_weight_min": -35.97184753417969,
      "activations/layer16_attention_weight_max": 42.65903091430664,
      "activations/layer16_attention_weight_min": -28.570369720458984,
      "activations/layer17_attention_weight_max": 59.57378387451172,
      "activations/layer17_attention_weight_min": -38.00935745239258,
      "activations/layer18_attention_weight_max": 50.253753662109375,
      "activations/layer18_attention_weight_min": -30.915605545043945,
      "activations/layer19_attention_weight_max": 22.671607971191406,
      "activations/layer19_attention_weight_min": -19.11165428161621,
      "activations/layer1_attention_weight_max": 17.510759353637695,
      "activations/layer1_attention_weight_min": -14.261517524719238,
      "activations/layer20_attention_weight_max": 20.616697311401367,
      "activations/layer20_attention_weight_min": -18.301902770996094,
      "activations/layer21_attention_weight_max": 44.30902099609375,
      "activations/layer21_attention_weight_min": -28.4730224609375,
      "activations/layer22_attention_weight_max": 36.473663330078125,
      "activations/layer22_attention_weight_min": -22.497417449951172,
      "activations/layer23_attention_weight_max": 27.158348083496094,
      "activations/layer23_attention_weight_min": -16.265968322753906,
      "activations/layer2_attention_weight_max": 29.44308853149414,
      "activations/layer2_attention_weight_min": -26.747234344482422,
      "activations/layer3_attention_weight_max": 73.5363998413086,
      "activations/layer3_attention_weight_min": -75.8448486328125,
      "activations/layer4_attention_weight_max": 100.36778259277344,
      "activations/layer4_attention_weight_min": -90.44425201416016,
      "activations/layer5_attention_weight_max": 122.43446350097656,
      "activations/layer5_attention_weight_min": -96.11650848388672,
      "activations/layer6_attention_weight_max": 62.84100341796875,
      "activations/layer6_attention_weight_min": -49.45412063598633,
      "activations/layer7_attention_weight_max": 57.81124496459961,
      "activations/layer7_attention_weight_min": -52.86957550048828,
      "activations/layer8_attention_weight_max": 39.4527587890625,
      "activations/layer8_attention_weight_min": -34.132659912109375,
      "activations/layer9_attention_weight_max": 54.74164962768555,
      "activations/layer9_attention_weight_min": -49.2261962890625,
      "epoch": 3.07,
      "learning_rate": 0.00013153825757575757,
      "loss": 2.8916,
      "step": 52800
    },
    {
      "activations/layer0_attention_weight_max": 13.633185386657715,
      "activations/layer0_attention_weight_min": -14.650300025939941,
      "activations/layer10_attention_weight_max": 43.0962028503418,
      "activations/layer10_attention_weight_min": -31.99860191345215,
      "activations/layer11_attention_weight_max": 33.89907455444336,
      "activations/layer11_attention_weight_min": -26.383834838867188,
      "activations/layer12_attention_weight_max": 26.736513137817383,
      "activations/layer12_attention_weight_min": -28.86033821105957,
      "activations/layer13_attention_weight_max": 56.701663970947266,
      "activations/layer13_attention_weight_min": -40.78681945800781,
      "activations/layer14_attention_weight_max": 48.75197982788086,
      "activations/layer14_attention_weight_min": -29.57480239868164,
      "activations/layer15_attention_weight_max": 48.68891906738281,
      "activations/layer15_attention_weight_min": -34.973567962646484,
      "activations/layer16_attention_weight_max": 40.21638488769531,
      "activations/layer16_attention_weight_min": -27.038143157958984,
      "activations/layer17_attention_weight_max": 64.81281280517578,
      "activations/layer17_attention_weight_min": -36.58595657348633,
      "activations/layer18_attention_weight_max": 55.7514533996582,
      "activations/layer18_attention_weight_min": -30.257253646850586,
      "activations/layer19_attention_weight_max": 23.893526077270508,
      "activations/layer19_attention_weight_min": -21.60107421875,
      "activations/layer1_attention_weight_max": 17.31693458557129,
      "activations/layer1_attention_weight_min": -16.180679321289062,
      "activations/layer20_attention_weight_max": 22.4516544342041,
      "activations/layer20_attention_weight_min": -17.91407585144043,
      "activations/layer21_attention_weight_max": 40.25726318359375,
      "activations/layer21_attention_weight_min": -27.59142303466797,
      "activations/layer22_attention_weight_max": 36.3980598449707,
      "activations/layer22_attention_weight_min": -20.965591430664062,
      "activations/layer23_attention_weight_max": 24.49476432800293,
      "activations/layer23_attention_weight_min": -15.204719543457031,
      "activations/layer2_attention_weight_max": 28.89284896850586,
      "activations/layer2_attention_weight_min": -26.96599578857422,
      "activations/layer3_attention_weight_max": 71.36183166503906,
      "activations/layer3_attention_weight_min": -74.88218688964844,
      "activations/layer4_attention_weight_max": 97.51079559326172,
      "activations/layer4_attention_weight_min": -91.69808197021484,
      "activations/layer5_attention_weight_max": 119.25149536132812,
      "activations/layer5_attention_weight_min": -101.70540618896484,
      "activations/layer6_attention_weight_max": 61.98020553588867,
      "activations/layer6_attention_weight_min": -50.21059036254883,
      "activations/layer7_attention_weight_max": 59.55916213989258,
      "activations/layer7_attention_weight_min": -53.55039596557617,
      "activations/layer8_attention_weight_max": 42.05766296386719,
      "activations/layer8_attention_weight_min": -33.33673095703125,
      "activations/layer9_attention_weight_max": 52.52717971801758,
      "activations/layer9_attention_weight_min": -51.145809173583984,
      "epoch": 3.07,
      "learning_rate": 0.00013151931818181819,
      "loss": 2.9003,
      "step": 52850
    },
    {
      "activations/layer0_attention_weight_max": 13.353219032287598,
      "activations/layer0_attention_weight_min": -14.596891403198242,
      "activations/layer10_attention_weight_max": 48.5286750793457,
      "activations/layer10_attention_weight_min": -37.58231735229492,
      "activations/layer11_attention_weight_max": 37.79470443725586,
      "activations/layer11_attention_weight_min": -27.777008056640625,
      "activations/layer12_attention_weight_max": 32.51923370361328,
      "activations/layer12_attention_weight_min": -23.87419319152832,
      "activations/layer13_attention_weight_max": 53.82204055786133,
      "activations/layer13_attention_weight_min": -39.84935760498047,
      "activations/layer14_attention_weight_max": 43.53821563720703,
      "activations/layer14_attention_weight_min": -31.469091415405273,
      "activations/layer15_attention_weight_max": 51.45196533203125,
      "activations/layer15_attention_weight_min": -34.36689376831055,
      "activations/layer16_attention_weight_max": 43.885501861572266,
      "activations/layer16_attention_weight_min": -27.550613403320312,
      "activations/layer17_attention_weight_max": 60.92823791503906,
      "activations/layer17_attention_weight_min": -36.217742919921875,
      "activations/layer18_attention_weight_max": 57.01464080810547,
      "activations/layer18_attention_weight_min": -31.5642147064209,
      "activations/layer19_attention_weight_max": 26.47499656677246,
      "activations/layer19_attention_weight_min": -16.82898712158203,
      "activations/layer1_attention_weight_max": 18.16649055480957,
      "activations/layer1_attention_weight_min": -13.969578742980957,
      "activations/layer20_attention_weight_max": 18.34166717529297,
      "activations/layer20_attention_weight_min": -18.316455841064453,
      "activations/layer21_attention_weight_max": 47.5001106262207,
      "activations/layer21_attention_weight_min": -27.83493995666504,
      "activations/layer22_attention_weight_max": 35.67605209350586,
      "activations/layer22_attention_weight_min": -21.358774185180664,
      "activations/layer23_attention_weight_max": 25.896560668945312,
      "activations/layer23_attention_weight_min": -15.861536979675293,
      "activations/layer2_attention_weight_max": 29.27410888671875,
      "activations/layer2_attention_weight_min": -27.873340606689453,
      "activations/layer3_attention_weight_max": 73.7727279663086,
      "activations/layer3_attention_weight_min": -80.38986206054688,
      "activations/layer4_attention_weight_max": 105.58978271484375,
      "activations/layer4_attention_weight_min": -99.97282409667969,
      "activations/layer5_attention_weight_max": 128.28321838378906,
      "activations/layer5_attention_weight_min": -109.37274169921875,
      "activations/layer6_attention_weight_max": 64.8855209350586,
      "activations/layer6_attention_weight_min": -58.44004440307617,
      "activations/layer7_attention_weight_max": 59.57582092285156,
      "activations/layer7_attention_weight_min": -57.850547790527344,
      "activations/layer8_attention_weight_max": 40.47228240966797,
      "activations/layer8_attention_weight_min": -38.9888801574707,
      "activations/layer9_attention_weight_max": 52.082855224609375,
      "activations/layer9_attention_weight_min": -49.48067092895508,
      "epoch": 3.07,
      "learning_rate": 0.00013150037878787877,
      "loss": 2.919,
      "step": 52900
    },
    {
      "activations/layer0_attention_weight_max": 14.289082527160645,
      "activations/layer0_attention_weight_min": -15.004507064819336,
      "activations/layer10_attention_weight_max": 44.83591842651367,
      "activations/layer10_attention_weight_min": -35.719573974609375,
      "activations/layer11_attention_weight_max": 39.335487365722656,
      "activations/layer11_attention_weight_min": -25.953824996948242,
      "activations/layer12_attention_weight_max": 26.049232482910156,
      "activations/layer12_attention_weight_min": -26.261003494262695,
      "activations/layer13_attention_weight_max": 58.73124694824219,
      "activations/layer13_attention_weight_min": -38.58170700073242,
      "activations/layer14_attention_weight_max": 51.83678436279297,
      "activations/layer14_attention_weight_min": -32.673797607421875,
      "activations/layer15_attention_weight_max": 54.22704315185547,
      "activations/layer15_attention_weight_min": -35.908260345458984,
      "activations/layer16_attention_weight_max": 39.93355178833008,
      "activations/layer16_attention_weight_min": -28.241798400878906,
      "activations/layer17_attention_weight_max": 62.99869918823242,
      "activations/layer17_attention_weight_min": -36.12688064575195,
      "activations/layer18_attention_weight_max": 54.91316223144531,
      "activations/layer18_attention_weight_min": -31.118350982666016,
      "activations/layer19_attention_weight_max": 19.66290855407715,
      "activations/layer19_attention_weight_min": -17.776405334472656,
      "activations/layer1_attention_weight_max": 17.82749366760254,
      "activations/layer1_attention_weight_min": -16.09125518798828,
      "activations/layer20_attention_weight_max": 21.13592529296875,
      "activations/layer20_attention_weight_min": -18.3934268951416,
      "activations/layer21_attention_weight_max": 42.408409118652344,
      "activations/layer21_attention_weight_min": -28.366924285888672,
      "activations/layer22_attention_weight_max": 36.35654830932617,
      "activations/layer22_attention_weight_min": -24.513126373291016,
      "activations/layer23_attention_weight_max": 25.36012840270996,
      "activations/layer23_attention_weight_min": -18.040912628173828,
      "activations/layer2_attention_weight_max": 28.516071319580078,
      "activations/layer2_attention_weight_min": -28.698814392089844,
      "activations/layer3_attention_weight_max": 75.10448455810547,
      "activations/layer3_attention_weight_min": -74.81797790527344,
      "activations/layer4_attention_weight_max": 102.19046783447266,
      "activations/layer4_attention_weight_min": -90.8999252319336,
      "activations/layer5_attention_weight_max": 127.74224090576172,
      "activations/layer5_attention_weight_min": -102.70335388183594,
      "activations/layer6_attention_weight_max": 63.99789810180664,
      "activations/layer6_attention_weight_min": -53.148250579833984,
      "activations/layer7_attention_weight_max": 60.372188568115234,
      "activations/layer7_attention_weight_min": -60.102420806884766,
      "activations/layer8_attention_weight_max": 41.13317108154297,
      "activations/layer8_attention_weight_min": -36.79425811767578,
      "activations/layer9_attention_weight_max": 49.33179473876953,
      "activations/layer9_attention_weight_min": -52.93121337890625,
      "epoch": 3.08,
      "learning_rate": 0.0001314814393939394,
      "loss": 2.9041,
      "step": 52950
    },
    {
      "activations/layer0_attention_weight_max": 15.00379753112793,
      "activations/layer0_attention_weight_min": -14.410582542419434,
      "activations/layer10_attention_weight_max": 44.41366195678711,
      "activations/layer10_attention_weight_min": -34.26205825805664,
      "activations/layer11_attention_weight_max": 35.351905822753906,
      "activations/layer11_attention_weight_min": -25.75296401977539,
      "activations/layer12_attention_weight_max": 23.736541748046875,
      "activations/layer12_attention_weight_min": -25.13109588623047,
      "activations/layer13_attention_weight_max": 53.57152557373047,
      "activations/layer13_attention_weight_min": -38.01524353027344,
      "activations/layer14_attention_weight_max": 56.897071838378906,
      "activations/layer14_attention_weight_min": -35.78911590576172,
      "activations/layer15_attention_weight_max": 49.731746673583984,
      "activations/layer15_attention_weight_min": -34.85485076904297,
      "activations/layer16_attention_weight_max": 42.449832916259766,
      "activations/layer16_attention_weight_min": -26.15642738342285,
      "activations/layer17_attention_weight_max": 67.10082244873047,
      "activations/layer17_attention_weight_min": -34.99874496459961,
      "activations/layer18_attention_weight_max": 52.8453254699707,
      "activations/layer18_attention_weight_min": -31.498233795166016,
      "activations/layer19_attention_weight_max": 25.433713912963867,
      "activations/layer19_attention_weight_min": -17.315168380737305,
      "activations/layer1_attention_weight_max": 17.827051162719727,
      "activations/layer1_attention_weight_min": -13.577099800109863,
      "activations/layer20_attention_weight_max": 23.227121353149414,
      "activations/layer20_attention_weight_min": -19.84398078918457,
      "activations/layer21_attention_weight_max": 43.138702392578125,
      "activations/layer21_attention_weight_min": -27.12608528137207,
      "activations/layer22_attention_weight_max": 35.33919906616211,
      "activations/layer22_attention_weight_min": -21.899478912353516,
      "activations/layer23_attention_weight_max": 32.01177978515625,
      "activations/layer23_attention_weight_min": -15.475604057312012,
      "activations/layer2_attention_weight_max": 29.262306213378906,
      "activations/layer2_attention_weight_min": -27.403425216674805,
      "activations/layer3_attention_weight_max": 70.47845458984375,
      "activations/layer3_attention_weight_min": -73.58551788330078,
      "activations/layer4_attention_weight_max": 98.95110321044922,
      "activations/layer4_attention_weight_min": -90.0993881225586,
      "activations/layer5_attention_weight_max": 113.6079330444336,
      "activations/layer5_attention_weight_min": -101.83200073242188,
      "activations/layer6_attention_weight_max": 59.35669708251953,
      "activations/layer6_attention_weight_min": -51.12388610839844,
      "activations/layer7_attention_weight_max": 55.8779296875,
      "activations/layer7_attention_weight_min": -56.4619255065918,
      "activations/layer8_attention_weight_max": 38.763214111328125,
      "activations/layer8_attention_weight_min": -34.931785583496094,
      "activations/layer9_attention_weight_max": 50.03108215332031,
      "activations/layer9_attention_weight_min": -46.107704162597656,
      "epoch": 3.08,
      "learning_rate": 0.00013146249999999998,
      "loss": 2.8918,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_loss": 2.85546875,
      "eval_runtime": 8.5234,
      "eval_samples_per_second": 503.792,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_openwebtext_loss": 2.85546875,
      "eval_openwebtext_ppl": 17.382583384208317,
      "eval_openwebtext_runtime": 8.5234,
      "eval_openwebtext_samples_per_second": 503.792,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_wikitext_loss": 3.11328125,
      "eval_wikitext_ppl": 22.494734284035275,
      "eval_wikitext_runtime": 1.9283,
      "eval_wikitext_samples_per_second": 236.483,
      "step": 53000
    },
    {
      "epoch": 3.08,
      "eval_lambada_loss": 3.01171875,
      "eval_lambada_ppl": 20.322298876557472,
      "eval_lambada_runtime": 9.551,
      "eval_lambada_samples_per_second": 509.791,
      "step": 53000
    },
    {
      "activations/layer0_attention_weight_max": 14.054973602294922,
      "activations/layer0_attention_weight_min": -14.58267879486084,
      "activations/layer10_attention_weight_max": 44.97134780883789,
      "activations/layer10_attention_weight_min": -34.51884841918945,
      "activations/layer11_attention_weight_max": 36.384071350097656,
      "activations/layer11_attention_weight_min": -25.992359161376953,
      "activations/layer12_attention_weight_max": 32.8241081237793,
      "activations/layer12_attention_weight_min": -24.05144500732422,
      "activations/layer13_attention_weight_max": 57.32556915283203,
      "activations/layer13_attention_weight_min": -39.891845703125,
      "activations/layer14_attention_weight_max": 54.45302963256836,
      "activations/layer14_attention_weight_min": -35.2003288269043,
      "activations/layer15_attention_weight_max": 53.33030700683594,
      "activations/layer15_attention_weight_min": -33.38676071166992,
      "activations/layer16_attention_weight_max": 44.221683502197266,
      "activations/layer16_attention_weight_min": -25.9162540435791,
      "activations/layer17_attention_weight_max": 68.8421401977539,
      "activations/layer17_attention_weight_min": -37.03129577636719,
      "activations/layer18_attention_weight_max": 54.34634780883789,
      "activations/layer18_attention_weight_min": -31.884075164794922,
      "activations/layer19_attention_weight_max": 28.582965850830078,
      "activations/layer19_attention_weight_min": -18.01996612548828,
      "activations/layer1_attention_weight_max": 17.02833366394043,
      "activations/layer1_attention_weight_min": -15.615415573120117,
      "activations/layer20_attention_weight_max": 22.081146240234375,
      "activations/layer20_attention_weight_min": -20.003353118896484,
      "activations/layer21_attention_weight_max": 47.057640075683594,
      "activations/layer21_attention_weight_min": -30.708641052246094,
      "activations/layer22_attention_weight_max": 44.192657470703125,
      "activations/layer22_attention_weight_min": -22.922109603881836,
      "activations/layer23_attention_weight_max": 25.169788360595703,
      "activations/layer23_attention_weight_min": -17.329051971435547,
      "activations/layer2_attention_weight_max": 31.719215393066406,
      "activations/layer2_attention_weight_min": -26.73345947265625,
      "activations/layer3_attention_weight_max": 70.88489532470703,
      "activations/layer3_attention_weight_min": -74.33696746826172,
      "activations/layer4_attention_weight_max": 103.85003662109375,
      "activations/layer4_attention_weight_min": -88.10013580322266,
      "activations/layer5_attention_weight_max": 127.41471862792969,
      "activations/layer5_attention_weight_min": -101.01091003417969,
      "activations/layer6_attention_weight_max": 60.01548385620117,
      "activations/layer6_attention_weight_min": -50.60190963745117,
      "activations/layer7_attention_weight_max": 61.39583206176758,
      "activations/layer7_attention_weight_min": -56.59269332885742,
      "activations/layer8_attention_weight_max": 42.15939712524414,
      "activations/layer8_attention_weight_min": -36.896080017089844,
      "activations/layer9_attention_weight_max": 50.99599838256836,
      "activations/layer9_attention_weight_min": -45.85390853881836,
      "epoch": 3.08,
      "learning_rate": 0.0001314435606060606,
      "loss": 2.9014,
      "step": 53050
    },
    {
      "activations/layer0_attention_weight_max": 15.24190616607666,
      "activations/layer0_attention_weight_min": -14.466715812683105,
      "activations/layer10_attention_weight_max": 45.4376220703125,
      "activations/layer10_attention_weight_min": -35.626014709472656,
      "activations/layer11_attention_weight_max": 38.41456985473633,
      "activations/layer11_attention_weight_min": -29.469097137451172,
      "activations/layer12_attention_weight_max": 27.03582000732422,
      "activations/layer12_attention_weight_min": -26.196596145629883,
      "activations/layer13_attention_weight_max": 70.57475280761719,
      "activations/layer13_attention_weight_min": -45.85649871826172,
      "activations/layer14_attention_weight_max": 79.30015563964844,
      "activations/layer14_attention_weight_min": -48.098228454589844,
      "activations/layer15_attention_weight_max": 57.56509780883789,
      "activations/layer15_attention_weight_min": -35.604496002197266,
      "activations/layer16_attention_weight_max": 40.16345977783203,
      "activations/layer16_attention_weight_min": -28.988876342773438,
      "activations/layer17_attention_weight_max": 67.03907775878906,
      "activations/layer17_attention_weight_min": -36.22502517700195,
      "activations/layer18_attention_weight_max": 53.19272232055664,
      "activations/layer18_attention_weight_min": -33.913917541503906,
      "activations/layer19_attention_weight_max": 24.428892135620117,
      "activations/layer19_attention_weight_min": -18.453733444213867,
      "activations/layer1_attention_weight_max": 17.03663444519043,
      "activations/layer1_attention_weight_min": -14.092899322509766,
      "activations/layer20_attention_weight_max": 22.83846092224121,
      "activations/layer20_attention_weight_min": -18.155393600463867,
      "activations/layer21_attention_weight_max": 43.79652786254883,
      "activations/layer21_attention_weight_min": -28.17205810546875,
      "activations/layer22_attention_weight_max": 40.75947570800781,
      "activations/layer22_attention_weight_min": -23.515005111694336,
      "activations/layer23_attention_weight_max": 28.34787368774414,
      "activations/layer23_attention_weight_min": -15.753721237182617,
      "activations/layer2_attention_weight_max": 29.826641082763672,
      "activations/layer2_attention_weight_min": -29.081022262573242,
      "activations/layer3_attention_weight_max": 73.11105346679688,
      "activations/layer3_attention_weight_min": -69.53073120117188,
      "activations/layer4_attention_weight_max": 98.88561248779297,
      "activations/layer4_attention_weight_min": -87.06659698486328,
      "activations/layer5_attention_weight_max": 129.59832763671875,
      "activations/layer5_attention_weight_min": -102.86302185058594,
      "activations/layer6_attention_weight_max": 67.00704193115234,
      "activations/layer6_attention_weight_min": -52.24049377441406,
      "activations/layer7_attention_weight_max": 60.49131393432617,
      "activations/layer7_attention_weight_min": -55.163421630859375,
      "activations/layer8_attention_weight_max": 46.26344680786133,
      "activations/layer8_attention_weight_min": -37.59611129760742,
      "activations/layer9_attention_weight_max": 56.5059928894043,
      "activations/layer9_attention_weight_min": -45.49467086791992,
      "epoch": 3.09,
      "learning_rate": 0.0001314246212121212,
      "loss": 2.9087,
      "step": 53100
    },
    {
      "activations/layer0_attention_weight_max": 14.278735160827637,
      "activations/layer0_attention_weight_min": -14.964655876159668,
      "activations/layer10_attention_weight_max": 43.743019104003906,
      "activations/layer10_attention_weight_min": -36.542022705078125,
      "activations/layer11_attention_weight_max": 35.57352828979492,
      "activations/layer11_attention_weight_min": -28.49590301513672,
      "activations/layer12_attention_weight_max": 27.934322357177734,
      "activations/layer12_attention_weight_min": -26.37040901184082,
      "activations/layer13_attention_weight_max": 49.93547821044922,
      "activations/layer13_attention_weight_min": -37.571205139160156,
      "activations/layer14_attention_weight_max": 51.50203323364258,
      "activations/layer14_attention_weight_min": -34.51393127441406,
      "activations/layer15_attention_weight_max": 46.15467834472656,
      "activations/layer15_attention_weight_min": -33.68800354003906,
      "activations/layer16_attention_weight_max": 45.120670318603516,
      "activations/layer16_attention_weight_min": -25.5971736907959,
      "activations/layer17_attention_weight_max": 60.13229751586914,
      "activations/layer17_attention_weight_min": -34.667320251464844,
      "activations/layer18_attention_weight_max": 54.94215774536133,
      "activations/layer18_attention_weight_min": -29.98138427734375,
      "activations/layer19_attention_weight_max": 24.15108299255371,
      "activations/layer19_attention_weight_min": -16.51872444152832,
      "activations/layer1_attention_weight_max": 18.141555786132812,
      "activations/layer1_attention_weight_min": -13.688314437866211,
      "activations/layer20_attention_weight_max": 24.078927993774414,
      "activations/layer20_attention_weight_min": -19.30461311340332,
      "activations/layer21_attention_weight_max": 44.531009674072266,
      "activations/layer21_attention_weight_min": -25.701745986938477,
      "activations/layer22_attention_weight_max": 36.321651458740234,
      "activations/layer22_attention_weight_min": -20.68684959411621,
      "activations/layer23_attention_weight_max": 29.149513244628906,
      "activations/layer23_attention_weight_min": -14.716986656188965,
      "activations/layer2_attention_weight_max": 28.864776611328125,
      "activations/layer2_attention_weight_min": -27.335350036621094,
      "activations/layer3_attention_weight_max": 72.39958190917969,
      "activations/layer3_attention_weight_min": -75.96488952636719,
      "activations/layer4_attention_weight_max": 101.09996795654297,
      "activations/layer4_attention_weight_min": -87.61736297607422,
      "activations/layer5_attention_weight_max": 120.04326629638672,
      "activations/layer5_attention_weight_min": -100.34172058105469,
      "activations/layer6_attention_weight_max": 60.491249084472656,
      "activations/layer6_attention_weight_min": -50.71224594116211,
      "activations/layer7_attention_weight_max": 67.37489318847656,
      "activations/layer7_attention_weight_min": -57.07933044433594,
      "activations/layer8_attention_weight_max": 40.06032180786133,
      "activations/layer8_attention_weight_min": -34.35442352294922,
      "activations/layer9_attention_weight_max": 57.8978157043457,
      "activations/layer9_attention_weight_min": -45.38063430786133,
      "epoch": 3.09,
      "learning_rate": 0.0001314056818181818,
      "loss": 2.9044,
      "step": 53150
    },
    {
      "activations/layer0_attention_weight_max": 14.78257942199707,
      "activations/layer0_attention_weight_min": -15.255514144897461,
      "activations/layer10_attention_weight_max": 55.022430419921875,
      "activations/layer10_attention_weight_min": -37.506954193115234,
      "activations/layer11_attention_weight_max": 46.1727294921875,
      "activations/layer11_attention_weight_min": -29.922409057617188,
      "activations/layer12_attention_weight_max": 33.60813522338867,
      "activations/layer12_attention_weight_min": -23.052839279174805,
      "activations/layer13_attention_weight_max": 66.90126037597656,
      "activations/layer13_attention_weight_min": -45.62627410888672,
      "activations/layer14_attention_weight_max": 54.58549118041992,
      "activations/layer14_attention_weight_min": -32.73432540893555,
      "activations/layer15_attention_weight_max": 68.98584747314453,
      "activations/layer15_attention_weight_min": -36.97145080566406,
      "activations/layer16_attention_weight_max": 39.74488830566406,
      "activations/layer16_attention_weight_min": -25.775497436523438,
      "activations/layer17_attention_weight_max": 74.07698822021484,
      "activations/layer17_attention_weight_min": -36.00432205200195,
      "activations/layer18_attention_weight_max": 57.147789001464844,
      "activations/layer18_attention_weight_min": -31.660192489624023,
      "activations/layer19_attention_weight_max": 26.687414169311523,
      "activations/layer19_attention_weight_min": -17.6014404296875,
      "activations/layer1_attention_weight_max": 16.857099533081055,
      "activations/layer1_attention_weight_min": -14.487788200378418,
      "activations/layer20_attention_weight_max": 23.834089279174805,
      "activations/layer20_attention_weight_min": -20.77604103088379,
      "activations/layer21_attention_weight_max": 49.09477615356445,
      "activations/layer21_attention_weight_min": -28.827573776245117,
      "activations/layer22_attention_weight_max": 36.874271392822266,
      "activations/layer22_attention_weight_min": -21.641109466552734,
      "activations/layer23_attention_weight_max": 27.426916122436523,
      "activations/layer23_attention_weight_min": -15.502382278442383,
      "activations/layer2_attention_weight_max": 27.994342803955078,
      "activations/layer2_attention_weight_min": -28.063209533691406,
      "activations/layer3_attention_weight_max": 70.93923950195312,
      "activations/layer3_attention_weight_min": -76.48442840576172,
      "activations/layer4_attention_weight_max": 105.12007141113281,
      "activations/layer4_attention_weight_min": -99.77813720703125,
      "activations/layer5_attention_weight_max": 138.5606231689453,
      "activations/layer5_attention_weight_min": -110.60134887695312,
      "activations/layer6_attention_weight_max": 69.98735046386719,
      "activations/layer6_attention_weight_min": -57.54741668701172,
      "activations/layer7_attention_weight_max": 62.99895095825195,
      "activations/layer7_attention_weight_min": -56.35746383666992,
      "activations/layer8_attention_weight_max": 45.0020751953125,
      "activations/layer8_attention_weight_min": -39.13010025024414,
      "activations/layer9_attention_weight_max": 57.820980072021484,
      "activations/layer9_attention_weight_min": -49.97030258178711,
      "epoch": 3.09,
      "learning_rate": 0.0001313867424242424,
      "loss": 2.898,
      "step": 53200
    },
    {
      "activations/layer0_attention_weight_max": 14.594865798950195,
      "activations/layer0_attention_weight_min": -14.906888008117676,
      "activations/layer10_attention_weight_max": 43.38560485839844,
      "activations/layer10_attention_weight_min": -34.970947265625,
      "activations/layer11_attention_weight_max": 43.767677307128906,
      "activations/layer11_attention_weight_min": -31.57135772705078,
      "activations/layer12_attention_weight_max": 25.680679321289062,
      "activations/layer12_attention_weight_min": -26.640958786010742,
      "activations/layer13_attention_weight_max": 58.95871353149414,
      "activations/layer13_attention_weight_min": -38.6348762512207,
      "activations/layer14_attention_weight_max": 48.958885192871094,
      "activations/layer14_attention_weight_min": -30.58062744140625,
      "activations/layer15_attention_weight_max": 46.84565353393555,
      "activations/layer15_attention_weight_min": -34.53986358642578,
      "activations/layer16_attention_weight_max": 38.31796646118164,
      "activations/layer16_attention_weight_min": -29.01687240600586,
      "activations/layer17_attention_weight_max": 64.16708374023438,
      "activations/layer17_attention_weight_min": -38.13643264770508,
      "activations/layer18_attention_weight_max": 59.17737579345703,
      "activations/layer18_attention_weight_min": -34.217098236083984,
      "activations/layer19_attention_weight_max": 22.003042221069336,
      "activations/layer19_attention_weight_min": -18.712499618530273,
      "activations/layer1_attention_weight_max": 17.18990707397461,
      "activations/layer1_attention_weight_min": -13.45678997039795,
      "activations/layer20_attention_weight_max": 19.928457260131836,
      "activations/layer20_attention_weight_min": -19.285505294799805,
      "activations/layer21_attention_weight_max": 39.5738525390625,
      "activations/layer21_attention_weight_min": -30.00362777709961,
      "activations/layer22_attention_weight_max": 35.32695007324219,
      "activations/layer22_attention_weight_min": -23.62926483154297,
      "activations/layer23_attention_weight_max": 23.861473083496094,
      "activations/layer23_attention_weight_min": -17.268383026123047,
      "activations/layer2_attention_weight_max": 28.413501739501953,
      "activations/layer2_attention_weight_min": -27.06178855895996,
      "activations/layer3_attention_weight_max": 68.18022918701172,
      "activations/layer3_attention_weight_min": -71.55535125732422,
      "activations/layer4_attention_weight_max": 95.1146240234375,
      "activations/layer4_attention_weight_min": -88.97582244873047,
      "activations/layer5_attention_weight_max": 117.58447265625,
      "activations/layer5_attention_weight_min": -98.5562973022461,
      "activations/layer6_attention_weight_max": 59.547698974609375,
      "activations/layer6_attention_weight_min": -50.70707321166992,
      "activations/layer7_attention_weight_max": 60.16929244995117,
      "activations/layer7_attention_weight_min": -57.34278869628906,
      "activations/layer8_attention_weight_max": 39.66069030761719,
      "activations/layer8_attention_weight_min": -38.04463195800781,
      "activations/layer9_attention_weight_max": 57.72697067260742,
      "activations/layer9_attention_weight_min": -51.24202346801758,
      "epoch": 3.09,
      "learning_rate": 0.00013136780303030302,
      "loss": 2.8874,
      "step": 53250
    },
    {
      "activations/layer0_attention_weight_max": 13.97219181060791,
      "activations/layer0_attention_weight_min": -14.432897567749023,
      "activations/layer10_attention_weight_max": 43.293304443359375,
      "activations/layer10_attention_weight_min": -33.30370330810547,
      "activations/layer11_attention_weight_max": 40.30043029785156,
      "activations/layer11_attention_weight_min": -28.010313034057617,
      "activations/layer12_attention_weight_max": 27.207590103149414,
      "activations/layer12_attention_weight_min": -23.81877326965332,
      "activations/layer13_attention_weight_max": 55.971405029296875,
      "activations/layer13_attention_weight_min": -36.58882141113281,
      "activations/layer14_attention_weight_max": 44.272865295410156,
      "activations/layer14_attention_weight_min": -37.96672058105469,
      "activations/layer15_attention_weight_max": 44.825767517089844,
      "activations/layer15_attention_weight_min": -31.69078826904297,
      "activations/layer16_attention_weight_max": 35.75922775268555,
      "activations/layer16_attention_weight_min": -26.658262252807617,
      "activations/layer17_attention_weight_max": 62.50264358520508,
      "activations/layer17_attention_weight_min": -35.437198638916016,
      "activations/layer18_attention_weight_max": 57.450618743896484,
      "activations/layer18_attention_weight_min": -31.527339935302734,
      "activations/layer19_attention_weight_max": 20.52399253845215,
      "activations/layer19_attention_weight_min": -19.366870880126953,
      "activations/layer1_attention_weight_max": 18.23818016052246,
      "activations/layer1_attention_weight_min": -13.945919036865234,
      "activations/layer20_attention_weight_max": 19.584455490112305,
      "activations/layer20_attention_weight_min": -18.51414680480957,
      "activations/layer21_attention_weight_max": 48.06404495239258,
      "activations/layer21_attention_weight_min": -29.64633560180664,
      "activations/layer22_attention_weight_max": 34.86527633666992,
      "activations/layer22_attention_weight_min": -23.3461856842041,
      "activations/layer23_attention_weight_max": 25.617328643798828,
      "activations/layer23_attention_weight_min": -15.853358268737793,
      "activations/layer2_attention_weight_max": 30.431537628173828,
      "activations/layer2_attention_weight_min": -29.15724754333496,
      "activations/layer3_attention_weight_max": 71.48686981201172,
      "activations/layer3_attention_weight_min": -77.8653793334961,
      "activations/layer4_attention_weight_max": 96.6251449584961,
      "activations/layer4_attention_weight_min": -88.54881286621094,
      "activations/layer5_attention_weight_max": 119.18185424804688,
      "activations/layer5_attention_weight_min": -96.10791015625,
      "activations/layer6_attention_weight_max": 59.266578674316406,
      "activations/layer6_attention_weight_min": -49.47575378417969,
      "activations/layer7_attention_weight_max": 60.27495193481445,
      "activations/layer7_attention_weight_min": -54.578651428222656,
      "activations/layer8_attention_weight_max": 39.26169204711914,
      "activations/layer8_attention_weight_min": -34.29597854614258,
      "activations/layer9_attention_weight_max": 53.5746955871582,
      "activations/layer9_attention_weight_min": -50.970062255859375,
      "epoch": 3.1,
      "learning_rate": 0.0001313488636363636,
      "loss": 2.908,
      "step": 53300
    },
    {
      "activations/layer0_attention_weight_max": 14.466778755187988,
      "activations/layer0_attention_weight_min": -14.861271858215332,
      "activations/layer10_attention_weight_max": 46.01303482055664,
      "activations/layer10_attention_weight_min": -37.991424560546875,
      "activations/layer11_attention_weight_max": 35.10398483276367,
      "activations/layer11_attention_weight_min": -26.88454818725586,
      "activations/layer12_attention_weight_max": 30.737995147705078,
      "activations/layer12_attention_weight_min": -27.277986526489258,
      "activations/layer13_attention_weight_max": 49.939029693603516,
      "activations/layer13_attention_weight_min": -36.6630744934082,
      "activations/layer14_attention_weight_max": 55.098384857177734,
      "activations/layer14_attention_weight_min": -37.6244010925293,
      "activations/layer15_attention_weight_max": 46.32784652709961,
      "activations/layer15_attention_weight_min": -33.33916091918945,
      "activations/layer16_attention_weight_max": 37.59811782836914,
      "activations/layer16_attention_weight_min": -26.46999168395996,
      "activations/layer17_attention_weight_max": 58.66912841796875,
      "activations/layer17_attention_weight_min": -40.1817741394043,
      "activations/layer18_attention_weight_max": 51.22352600097656,
      "activations/layer18_attention_weight_min": -31.816097259521484,
      "activations/layer19_attention_weight_max": 20.544614791870117,
      "activations/layer19_attention_weight_min": -16.55735969543457,
      "activations/layer1_attention_weight_max": 18.419607162475586,
      "activations/layer1_attention_weight_min": -16.51407241821289,
      "activations/layer20_attention_weight_max": 20.96637725830078,
      "activations/layer20_attention_weight_min": -21.094602584838867,
      "activations/layer21_attention_weight_max": 42.249961853027344,
      "activations/layer21_attention_weight_min": -30.672361373901367,
      "activations/layer22_attention_weight_max": 33.835723876953125,
      "activations/layer22_attention_weight_min": -22.326488494873047,
      "activations/layer23_attention_weight_max": 24.794769287109375,
      "activations/layer23_attention_weight_min": -20.36022186279297,
      "activations/layer2_attention_weight_max": 30.191875457763672,
      "activations/layer2_attention_weight_min": -27.328840255737305,
      "activations/layer3_attention_weight_max": 76.32003021240234,
      "activations/layer3_attention_weight_min": -74.8226318359375,
      "activations/layer4_attention_weight_max": 106.16043853759766,
      "activations/layer4_attention_weight_min": -94.02478790283203,
      "activations/layer5_attention_weight_max": 123.59230041503906,
      "activations/layer5_attention_weight_min": -103.54989624023438,
      "activations/layer6_attention_weight_max": 59.59911346435547,
      "activations/layer6_attention_weight_min": -53.65622329711914,
      "activations/layer7_attention_weight_max": 64.26438903808594,
      "activations/layer7_attention_weight_min": -55.60739517211914,
      "activations/layer8_attention_weight_max": 38.77973556518555,
      "activations/layer8_attention_weight_min": -35.81489562988281,
      "activations/layer9_attention_weight_max": 53.66079330444336,
      "activations/layer9_attention_weight_min": -49.66155242919922,
      "epoch": 3.1,
      "learning_rate": 0.00013132992424242423,
      "loss": 2.9034,
      "step": 53350
    },
    {
      "activations/layer0_attention_weight_max": 16.687929153442383,
      "activations/layer0_attention_weight_min": -15.264128684997559,
      "activations/layer10_attention_weight_max": 44.14854049682617,
      "activations/layer10_attention_weight_min": -38.72420120239258,
      "activations/layer11_attention_weight_max": 33.9283332824707,
      "activations/layer11_attention_weight_min": -27.397544860839844,
      "activations/layer12_attention_weight_max": 27.506885528564453,
      "activations/layer12_attention_weight_min": -27.307146072387695,
      "activations/layer13_attention_weight_max": 51.080753326416016,
      "activations/layer13_attention_weight_min": -38.76026916503906,
      "activations/layer14_attention_weight_max": 46.14586639404297,
      "activations/layer14_attention_weight_min": -32.54106140136719,
      "activations/layer15_attention_weight_max": 47.59343719482422,
      "activations/layer15_attention_weight_min": -33.20595932006836,
      "activations/layer16_attention_weight_max": 38.712650299072266,
      "activations/layer16_attention_weight_min": -26.92390251159668,
      "activations/layer17_attention_weight_max": 59.52239227294922,
      "activations/layer17_attention_weight_min": -35.27585220336914,
      "activations/layer18_attention_weight_max": 54.746185302734375,
      "activations/layer18_attention_weight_min": -33.6882209777832,
      "activations/layer19_attention_weight_max": 21.373146057128906,
      "activations/layer19_attention_weight_min": -17.518104553222656,
      "activations/layer1_attention_weight_max": 17.558942794799805,
      "activations/layer1_attention_weight_min": -13.58041000366211,
      "activations/layer20_attention_weight_max": 22.29511260986328,
      "activations/layer20_attention_weight_min": -19.827787399291992,
      "activations/layer21_attention_weight_max": 45.381256103515625,
      "activations/layer21_attention_weight_min": -27.21834945678711,
      "activations/layer22_attention_weight_max": 38.19864273071289,
      "activations/layer22_attention_weight_min": -23.674541473388672,
      "activations/layer23_attention_weight_max": 26.914648056030273,
      "activations/layer23_attention_weight_min": -15.730085372924805,
      "activations/layer2_attention_weight_max": 28.674800872802734,
      "activations/layer2_attention_weight_min": -26.68220329284668,
      "activations/layer3_attention_weight_max": 70.78253936767578,
      "activations/layer3_attention_weight_min": -71.74861907958984,
      "activations/layer4_attention_weight_max": 101.2092514038086,
      "activations/layer4_attention_weight_min": -90.97164916992188,
      "activations/layer5_attention_weight_max": 119.42967224121094,
      "activations/layer5_attention_weight_min": -98.49851989746094,
      "activations/layer6_attention_weight_max": 60.774513244628906,
      "activations/layer6_attention_weight_min": -53.20962142944336,
      "activations/layer7_attention_weight_max": 63.70730209350586,
      "activations/layer7_attention_weight_min": -55.833168029785156,
      "activations/layer8_attention_weight_max": 39.06406784057617,
      "activations/layer8_attention_weight_min": -36.320919036865234,
      "activations/layer9_attention_weight_max": 58.87261962890625,
      "activations/layer9_attention_weight_min": -46.12178039550781,
      "epoch": 3.1,
      "learning_rate": 0.00013131098484848482,
      "loss": 2.9148,
      "step": 53400
    },
    {
      "activations/layer0_attention_weight_max": 15.46343994140625,
      "activations/layer0_attention_weight_min": -14.538119316101074,
      "activations/layer10_attention_weight_max": 39.079627990722656,
      "activations/layer10_attention_weight_min": -35.66671371459961,
      "activations/layer11_attention_weight_max": 35.02482223510742,
      "activations/layer11_attention_weight_min": -26.026962280273438,
      "activations/layer12_attention_weight_max": 23.850278854370117,
      "activations/layer12_attention_weight_min": -24.845626831054688,
      "activations/layer13_attention_weight_max": 55.973411560058594,
      "activations/layer13_attention_weight_min": -44.89193344116211,
      "activations/layer14_attention_weight_max": 66.48226928710938,
      "activations/layer14_attention_weight_min": -46.798423767089844,
      "activations/layer15_attention_weight_max": 46.29302215576172,
      "activations/layer15_attention_weight_min": -31.20699691772461,
      "activations/layer16_attention_weight_max": 35.76811599731445,
      "activations/layer16_attention_weight_min": -27.699844360351562,
      "activations/layer17_attention_weight_max": 55.68095779418945,
      "activations/layer17_attention_weight_min": -34.129817962646484,
      "activations/layer18_attention_weight_max": 52.18559646606445,
      "activations/layer18_attention_weight_min": -32.61150360107422,
      "activations/layer19_attention_weight_max": 23.083097457885742,
      "activations/layer19_attention_weight_min": -17.898183822631836,
      "activations/layer1_attention_weight_max": 17.633390426635742,
      "activations/layer1_attention_weight_min": -15.831599235534668,
      "activations/layer20_attention_weight_max": 21.59910011291504,
      "activations/layer20_attention_weight_min": -20.002517700195312,
      "activations/layer21_attention_weight_max": 39.91046142578125,
      "activations/layer21_attention_weight_min": -28.237096786499023,
      "activations/layer22_attention_weight_max": 37.09883117675781,
      "activations/layer22_attention_weight_min": -23.672687530517578,
      "activations/layer23_attention_weight_max": 28.34882926940918,
      "activations/layer23_attention_weight_min": -16.346054077148438,
      "activations/layer2_attention_weight_max": 29.89926528930664,
      "activations/layer2_attention_weight_min": -28.675731658935547,
      "activations/layer3_attention_weight_max": 73.78765106201172,
      "activations/layer3_attention_weight_min": -78.36762237548828,
      "activations/layer4_attention_weight_max": 95.02367401123047,
      "activations/layer4_attention_weight_min": -89.51525115966797,
      "activations/layer5_attention_weight_max": 110.8989486694336,
      "activations/layer5_attention_weight_min": -102.224853515625,
      "activations/layer6_attention_weight_max": 57.50865936279297,
      "activations/layer6_attention_weight_min": -51.595252990722656,
      "activations/layer7_attention_weight_max": 63.313655853271484,
      "activations/layer7_attention_weight_min": -59.23967742919922,
      "activations/layer8_attention_weight_max": 38.44450759887695,
      "activations/layer8_attention_weight_min": -35.254676818847656,
      "activations/layer9_attention_weight_max": 52.74749755859375,
      "activations/layer9_attention_weight_min": -49.512813568115234,
      "epoch": 3.11,
      "learning_rate": 0.00013129242424242423,
      "loss": 2.8926,
      "step": 53450
    },
    {
      "activations/layer0_attention_weight_max": 14.430535316467285,
      "activations/layer0_attention_weight_min": -14.314432144165039,
      "activations/layer10_attention_weight_max": 43.6500244140625,
      "activations/layer10_attention_weight_min": -34.73637771606445,
      "activations/layer11_attention_weight_max": 35.76073455810547,
      "activations/layer11_attention_weight_min": -26.410396575927734,
      "activations/layer12_attention_weight_max": 31.30290412902832,
      "activations/layer12_attention_weight_min": -26.487869262695312,
      "activations/layer13_attention_weight_max": 69.12277221679688,
      "activations/layer13_attention_weight_min": -40.407142639160156,
      "activations/layer14_attention_weight_max": 65.43538665771484,
      "activations/layer14_attention_weight_min": -34.93992614746094,
      "activations/layer15_attention_weight_max": 51.61301040649414,
      "activations/layer15_attention_weight_min": -32.37654113769531,
      "activations/layer16_attention_weight_max": 45.233333587646484,
      "activations/layer16_attention_weight_min": -26.006725311279297,
      "activations/layer17_attention_weight_max": 91.63134002685547,
      "activations/layer17_attention_weight_min": -37.99460220336914,
      "activations/layer18_attention_weight_max": 65.0694351196289,
      "activations/layer18_attention_weight_min": -33.0677490234375,
      "activations/layer19_attention_weight_max": 30.685922622680664,
      "activations/layer19_attention_weight_min": -20.915176391601562,
      "activations/layer1_attention_weight_max": 17.275371551513672,
      "activations/layer1_attention_weight_min": -15.491775512695312,
      "activations/layer20_attention_weight_max": 28.688907623291016,
      "activations/layer20_attention_weight_min": -20.0718994140625,
      "activations/layer21_attention_weight_max": 58.990814208984375,
      "activations/layer21_attention_weight_min": -24.477928161621094,
      "activations/layer22_attention_weight_max": 45.11056900024414,
      "activations/layer22_attention_weight_min": -22.636211395263672,
      "activations/layer23_attention_weight_max": 30.761558532714844,
      "activations/layer23_attention_weight_min": -16.351123809814453,
      "activations/layer2_attention_weight_max": 30.913679122924805,
      "activations/layer2_attention_weight_min": -28.29867172241211,
      "activations/layer3_attention_weight_max": 69.76374816894531,
      "activations/layer3_attention_weight_min": -70.26094818115234,
      "activations/layer4_attention_weight_max": 99.30321502685547,
      "activations/layer4_attention_weight_min": -89.81744384765625,
      "activations/layer5_attention_weight_max": 127.67720031738281,
      "activations/layer5_attention_weight_min": -98.3333969116211,
      "activations/layer6_attention_weight_max": 67.97608184814453,
      "activations/layer6_attention_weight_min": -52.02161407470703,
      "activations/layer7_attention_weight_max": 66.56632232666016,
      "activations/layer7_attention_weight_min": -58.208866119384766,
      "activations/layer8_attention_weight_max": 37.068912506103516,
      "activations/layer8_attention_weight_min": -37.90446853637695,
      "activations/layer9_attention_weight_max": 48.108882904052734,
      "activations/layer9_attention_weight_min": -45.72028732299805,
      "epoch": 3.11,
      "learning_rate": 0.00013127348484848482,
      "loss": 2.896,
      "step": 53500
    },
    {
      "activations/layer0_attention_weight_max": 13.855359077453613,
      "activations/layer0_attention_weight_min": -14.294330596923828,
      "activations/layer10_attention_weight_max": 43.58736038208008,
      "activations/layer10_attention_weight_min": -37.495460510253906,
      "activations/layer11_attention_weight_max": 37.313926696777344,
      "activations/layer11_attention_weight_min": -29.402759552001953,
      "activations/layer12_attention_weight_max": 29.3146915435791,
      "activations/layer12_attention_weight_min": -30.81793785095215,
      "activations/layer13_attention_weight_max": 61.78464889526367,
      "activations/layer13_attention_weight_min": -48.40599822998047,
      "activations/layer14_attention_weight_max": 56.60358810424805,
      "activations/layer14_attention_weight_min": -40.84156799316406,
      "activations/layer15_attention_weight_max": 52.337486267089844,
      "activations/layer15_attention_weight_min": -37.55765151977539,
      "activations/layer16_attention_weight_max": 38.1926155090332,
      "activations/layer16_attention_weight_min": -27.118215560913086,
      "activations/layer17_attention_weight_max": 61.84750747680664,
      "activations/layer17_attention_weight_min": -34.87506103515625,
      "activations/layer18_attention_weight_max": 59.37726974487305,
      "activations/layer18_attention_weight_min": -33.41490173339844,
      "activations/layer19_attention_weight_max": 20.141332626342773,
      "activations/layer19_attention_weight_min": -16.8964900970459,
      "activations/layer1_attention_weight_max": 17.7131404876709,
      "activations/layer1_attention_weight_min": -16.1970272064209,
      "activations/layer20_attention_weight_max": 21.770462036132812,
      "activations/layer20_attention_weight_min": -19.690568923950195,
      "activations/layer21_attention_weight_max": 41.78418731689453,
      "activations/layer21_attention_weight_min": -27.596464157104492,
      "activations/layer22_attention_weight_max": 38.73069381713867,
      "activations/layer22_attention_weight_min": -23.328950881958008,
      "activations/layer23_attention_weight_max": 24.406341552734375,
      "activations/layer23_attention_weight_min": -15.825759887695312,
      "activations/layer2_attention_weight_max": 28.208148956298828,
      "activations/layer2_attention_weight_min": -27.168441772460938,
      "activations/layer3_attention_weight_max": 73.80166625976562,
      "activations/layer3_attention_weight_min": -76.63964080810547,
      "activations/layer4_attention_weight_max": 101.2511978149414,
      "activations/layer4_attention_weight_min": -90.94750213623047,
      "activations/layer5_attention_weight_max": 118.77056121826172,
      "activations/layer5_attention_weight_min": -102.0130844116211,
      "activations/layer6_attention_weight_max": 63.715579986572266,
      "activations/layer6_attention_weight_min": -53.94447708129883,
      "activations/layer7_attention_weight_max": 58.205299377441406,
      "activations/layer7_attention_weight_min": -59.79187774658203,
      "activations/layer8_attention_weight_max": 40.17850875854492,
      "activations/layer8_attention_weight_min": -38.104736328125,
      "activations/layer9_attention_weight_max": 48.601070404052734,
      "activations/layer9_attention_weight_min": -49.33363723754883,
      "epoch": 3.11,
      "learning_rate": 0.00013125454545454544,
      "loss": 2.9062,
      "step": 53550
    },
    {
      "activations/layer0_attention_weight_max": 14.201728820800781,
      "activations/layer0_attention_weight_min": -15.106840133666992,
      "activations/layer10_attention_weight_max": 48.31349563598633,
      "activations/layer10_attention_weight_min": -35.89002227783203,
      "activations/layer11_attention_weight_max": 36.6712646484375,
      "activations/layer11_attention_weight_min": -26.62643814086914,
      "activations/layer12_attention_weight_max": 27.868131637573242,
      "activations/layer12_attention_weight_min": -26.89252471923828,
      "activations/layer13_attention_weight_max": 65.86031341552734,
      "activations/layer13_attention_weight_min": -52.257511138916016,
      "activations/layer14_attention_weight_max": 48.682376861572266,
      "activations/layer14_attention_weight_min": -39.619850158691406,
      "activations/layer15_attention_weight_max": 50.26066589355469,
      "activations/layer15_attention_weight_min": -38.11791229248047,
      "activations/layer16_attention_weight_max": 38.00162887573242,
      "activations/layer16_attention_weight_min": -25.54090690612793,
      "activations/layer17_attention_weight_max": 62.61338424682617,
      "activations/layer17_attention_weight_min": -41.45685577392578,
      "activations/layer18_attention_weight_max": 62.07335662841797,
      "activations/layer18_attention_weight_min": -32.055572509765625,
      "activations/layer19_attention_weight_max": 21.038610458374023,
      "activations/layer19_attention_weight_min": -15.010281562805176,
      "activations/layer1_attention_weight_max": 17.05485725402832,
      "activations/layer1_attention_weight_min": -12.977116584777832,
      "activations/layer20_attention_weight_max": 19.436748504638672,
      "activations/layer20_attention_weight_min": -21.081432342529297,
      "activations/layer21_attention_weight_max": 35.74929428100586,
      "activations/layer21_attention_weight_min": -27.627506256103516,
      "activations/layer22_attention_weight_max": 34.16517639160156,
      "activations/layer22_attention_weight_min": -21.82402229309082,
      "activations/layer23_attention_weight_max": 26.394023895263672,
      "activations/layer23_attention_weight_min": -17.261573791503906,
      "activations/layer2_attention_weight_max": 27.287189483642578,
      "activations/layer2_attention_weight_min": -27.097122192382812,
      "activations/layer3_attention_weight_max": 70.6254653930664,
      "activations/layer3_attention_weight_min": -75.97106170654297,
      "activations/layer4_attention_weight_max": 101.2051010131836,
      "activations/layer4_attention_weight_min": -89.63973236083984,
      "activations/layer5_attention_weight_max": 124.42745208740234,
      "activations/layer5_attention_weight_min": -103.7733383178711,
      "activations/layer6_attention_weight_max": 63.30126190185547,
      "activations/layer6_attention_weight_min": -52.043006896972656,
      "activations/layer7_attention_weight_max": 60.107582092285156,
      "activations/layer7_attention_weight_min": -54.7923583984375,
      "activations/layer8_attention_weight_max": 39.62997055053711,
      "activations/layer8_attention_weight_min": -33.53009796142578,
      "activations/layer9_attention_weight_max": 56.6905517578125,
      "activations/layer9_attention_weight_min": -47.47703552246094,
      "epoch": 3.11,
      "learning_rate": 0.00013123560606060605,
      "loss": 2.9075,
      "step": 53600
    },
    {
      "activations/layer0_attention_weight_max": 14.808867454528809,
      "activations/layer0_attention_weight_min": -14.876790046691895,
      "activations/layer10_attention_weight_max": 43.9160041809082,
      "activations/layer10_attention_weight_min": -34.21988296508789,
      "activations/layer11_attention_weight_max": 37.86485290527344,
      "activations/layer11_attention_weight_min": -27.526161193847656,
      "activations/layer12_attention_weight_max": 30.531299591064453,
      "activations/layer12_attention_weight_min": -24.238298416137695,
      "activations/layer13_attention_weight_max": 50.291770935058594,
      "activations/layer13_attention_weight_min": -39.67366409301758,
      "activations/layer14_attention_weight_max": 42.69749450683594,
      "activations/layer14_attention_weight_min": -33.288909912109375,
      "activations/layer15_attention_weight_max": 43.166046142578125,
      "activations/layer15_attention_weight_min": -33.2859992980957,
      "activations/layer16_attention_weight_max": 41.756649017333984,
      "activations/layer16_attention_weight_min": -28.65434455871582,
      "activations/layer17_attention_weight_max": 60.48622512817383,
      "activations/layer17_attention_weight_min": -37.08940887451172,
      "activations/layer18_attention_weight_max": 50.88422393798828,
      "activations/layer18_attention_weight_min": -32.642677307128906,
      "activations/layer19_attention_weight_max": 20.64592933654785,
      "activations/layer19_attention_weight_min": -18.263547897338867,
      "activations/layer1_attention_weight_max": 17.28672981262207,
      "activations/layer1_attention_weight_min": -15.581027030944824,
      "activations/layer20_attention_weight_max": 20.760250091552734,
      "activations/layer20_attention_weight_min": -18.554203033447266,
      "activations/layer21_attention_weight_max": 42.5349006652832,
      "activations/layer21_attention_weight_min": -31.367502212524414,
      "activations/layer22_attention_weight_max": 32.81687927246094,
      "activations/layer22_attention_weight_min": -24.67359161376953,
      "activations/layer23_attention_weight_max": 23.143699645996094,
      "activations/layer23_attention_weight_min": -14.882457733154297,
      "activations/layer2_attention_weight_max": 29.901046752929688,
      "activations/layer2_attention_weight_min": -28.19039535522461,
      "activations/layer3_attention_weight_max": 70.3196792602539,
      "activations/layer3_attention_weight_min": -73.85566711425781,
      "activations/layer4_attention_weight_max": 93.46040344238281,
      "activations/layer4_attention_weight_min": -87.5583724975586,
      "activations/layer5_attention_weight_max": 116.73039245605469,
      "activations/layer5_attention_weight_min": -95.20469665527344,
      "activations/layer6_attention_weight_max": 58.46466827392578,
      "activations/layer6_attention_weight_min": -50.76430130004883,
      "activations/layer7_attention_weight_max": 56.56796646118164,
      "activations/layer7_attention_weight_min": -57.46392059326172,
      "activations/layer8_attention_weight_max": 38.54335403442383,
      "activations/layer8_attention_weight_min": -35.86653518676758,
      "activations/layer9_attention_weight_max": 46.53959274291992,
      "activations/layer9_attention_weight_min": -46.56781768798828,
      "epoch": 3.12,
      "learning_rate": 0.00013121666666666667,
      "loss": 2.8851,
      "step": 53650
    },
    {
      "activations/layer0_attention_weight_max": 14.172513008117676,
      "activations/layer0_attention_weight_min": -14.24545669555664,
      "activations/layer10_attention_weight_max": 39.596954345703125,
      "activations/layer10_attention_weight_min": -34.89081573486328,
      "activations/layer11_attention_weight_max": 31.889244079589844,
      "activations/layer11_attention_weight_min": -27.673805236816406,
      "activations/layer12_attention_weight_max": 23.970382690429688,
      "activations/layer12_attention_weight_min": -23.232805252075195,
      "activations/layer13_attention_weight_max": 50.96255874633789,
      "activations/layer13_attention_weight_min": -35.956260681152344,
      "activations/layer14_attention_weight_max": 54.124961853027344,
      "activations/layer14_attention_weight_min": -33.30573654174805,
      "activations/layer15_attention_weight_max": 44.653411865234375,
      "activations/layer15_attention_weight_min": -33.53422546386719,
      "activations/layer16_attention_weight_max": 37.572715759277344,
      "activations/layer16_attention_weight_min": -27.88602066040039,
      "activations/layer17_attention_weight_max": 57.85483932495117,
      "activations/layer17_attention_weight_min": -37.768585205078125,
      "activations/layer18_attention_weight_max": 48.84720230102539,
      "activations/layer18_attention_weight_min": -31.06934928894043,
      "activations/layer19_attention_weight_max": 20.205419540405273,
      "activations/layer19_attention_weight_min": -16.679527282714844,
      "activations/layer1_attention_weight_max": 17.77760124206543,
      "activations/layer1_attention_weight_min": -14.24645709991455,
      "activations/layer20_attention_weight_max": 21.387638092041016,
      "activations/layer20_attention_weight_min": -19.8194637298584,
      "activations/layer21_attention_weight_max": 38.47471237182617,
      "activations/layer21_attention_weight_min": -26.73186492919922,
      "activations/layer22_attention_weight_max": 34.55564498901367,
      "activations/layer22_attention_weight_min": -22.050996780395508,
      "activations/layer23_attention_weight_max": 25.9116268157959,
      "activations/layer23_attention_weight_min": -15.661582946777344,
      "activations/layer2_attention_weight_max": 29.726802825927734,
      "activations/layer2_attention_weight_min": -27.28741455078125,
      "activations/layer3_attention_weight_max": 71.66841888427734,
      "activations/layer3_attention_weight_min": -74.50830078125,
      "activations/layer4_attention_weight_max": 100.40895080566406,
      "activations/layer4_attention_weight_min": -94.23359680175781,
      "activations/layer5_attention_weight_max": 119.71896362304688,
      "activations/layer5_attention_weight_min": -103.20821380615234,
      "activations/layer6_attention_weight_max": 60.055885314941406,
      "activations/layer6_attention_weight_min": -53.90391540527344,
      "activations/layer7_attention_weight_max": 59.0726318359375,
      "activations/layer7_attention_weight_min": -54.7823486328125,
      "activations/layer8_attention_weight_max": 37.61937713623047,
      "activations/layer8_attention_weight_min": -36.369476318359375,
      "activations/layer9_attention_weight_max": 54.956275939941406,
      "activations/layer9_attention_weight_min": -47.84884262084961,
      "epoch": 3.12,
      "learning_rate": 0.00013119772727272725,
      "loss": 2.9004,
      "step": 53700
    },
    {
      "activations/layer0_attention_weight_max": 14.448060035705566,
      "activations/layer0_attention_weight_min": -14.822748184204102,
      "activations/layer10_attention_weight_max": 42.61743927001953,
      "activations/layer10_attention_weight_min": -36.068878173828125,
      "activations/layer11_attention_weight_max": 35.9096565246582,
      "activations/layer11_attention_weight_min": -26.587888717651367,
      "activations/layer12_attention_weight_max": 28.933717727661133,
      "activations/layer12_attention_weight_min": -23.32189178466797,
      "activations/layer13_attention_weight_max": 45.228450775146484,
      "activations/layer13_attention_weight_min": -37.90657043457031,
      "activations/layer14_attention_weight_max": 37.0722770690918,
      "activations/layer14_attention_weight_min": -26.770233154296875,
      "activations/layer15_attention_weight_max": 40.81673049926758,
      "activations/layer15_attention_weight_min": -31.081348419189453,
      "activations/layer16_attention_weight_max": 34.40394973754883,
      "activations/layer16_attention_weight_min": -26.93028450012207,
      "activations/layer17_attention_weight_max": 59.21531295776367,
      "activations/layer17_attention_weight_min": -34.82878494262695,
      "activations/layer18_attention_weight_max": 48.69419479370117,
      "activations/layer18_attention_weight_min": -30.730628967285156,
      "activations/layer19_attention_weight_max": 21.482709884643555,
      "activations/layer19_attention_weight_min": -18.750646591186523,
      "activations/layer1_attention_weight_max": 18.409324645996094,
      "activations/layer1_attention_weight_min": -13.377561569213867,
      "activations/layer20_attention_weight_max": 20.90260887145996,
      "activations/layer20_attention_weight_min": -21.871131896972656,
      "activations/layer21_attention_weight_max": 41.216270446777344,
      "activations/layer21_attention_weight_min": -31.42792320251465,
      "activations/layer22_attention_weight_max": 34.86661911010742,
      "activations/layer22_attention_weight_min": -25.06614112854004,
      "activations/layer23_attention_weight_max": 25.294919967651367,
      "activations/layer23_attention_weight_min": -16.827299118041992,
      "activations/layer2_attention_weight_max": 28.477256774902344,
      "activations/layer2_attention_weight_min": -26.416614532470703,
      "activations/layer3_attention_weight_max": 70.83173370361328,
      "activations/layer3_attention_weight_min": -72.21257781982422,
      "activations/layer4_attention_weight_max": 100.16850280761719,
      "activations/layer4_attention_weight_min": -90.11489868164062,
      "activations/layer5_attention_weight_max": 119.98229217529297,
      "activations/layer5_attention_weight_min": -99.71217346191406,
      "activations/layer6_attention_weight_max": 59.53837585449219,
      "activations/layer6_attention_weight_min": -51.34492492675781,
      "activations/layer7_attention_weight_max": 56.821598052978516,
      "activations/layer7_attention_weight_min": -55.74028778076172,
      "activations/layer8_attention_weight_max": 37.318359375,
      "activations/layer8_attention_weight_min": -35.48227310180664,
      "activations/layer9_attention_weight_max": 46.665504455566406,
      "activations/layer9_attention_weight_min": -47.364601135253906,
      "epoch": 3.12,
      "learning_rate": 0.00013117878787878787,
      "loss": 2.9175,
      "step": 53750
    },
    {
      "activations/layer0_attention_weight_max": 14.487794876098633,
      "activations/layer0_attention_weight_min": -14.581258773803711,
      "activations/layer10_attention_weight_max": 44.236297607421875,
      "activations/layer10_attention_weight_min": -35.19928741455078,
      "activations/layer11_attention_weight_max": 34.418296813964844,
      "activations/layer11_attention_weight_min": -27.90089225769043,
      "activations/layer12_attention_weight_max": 29.79279899597168,
      "activations/layer12_attention_weight_min": -27.749961853027344,
      "activations/layer13_attention_weight_max": 65.52224731445312,
      "activations/layer13_attention_weight_min": -48.21788787841797,
      "activations/layer14_attention_weight_max": 74.59114837646484,
      "activations/layer14_attention_weight_min": -46.73545455932617,
      "activations/layer15_attention_weight_max": 49.58892059326172,
      "activations/layer15_attention_weight_min": -32.750797271728516,
      "activations/layer16_attention_weight_max": 40.405555725097656,
      "activations/layer16_attention_weight_min": -25.734018325805664,
      "activations/layer17_attention_weight_max": 63.40386199951172,
      "activations/layer17_attention_weight_min": -36.03799819946289,
      "activations/layer18_attention_weight_max": 52.76409149169922,
      "activations/layer18_attention_weight_min": -31.97735023498535,
      "activations/layer19_attention_weight_max": 20.672826766967773,
      "activations/layer19_attention_weight_min": -18.699949264526367,
      "activations/layer1_attention_weight_max": 18.463342666625977,
      "activations/layer1_attention_weight_min": -14.758965492248535,
      "activations/layer20_attention_weight_max": 22.21522331237793,
      "activations/layer20_attention_weight_min": -22.910940170288086,
      "activations/layer21_attention_weight_max": 43.04265594482422,
      "activations/layer21_attention_weight_min": -27.920867919921875,
      "activations/layer22_attention_weight_max": 39.05366897583008,
      "activations/layer22_attention_weight_min": -23.112186431884766,
      "activations/layer23_attention_weight_max": 24.18289566040039,
      "activations/layer23_attention_weight_min": -17.864648818969727,
      "activations/layer2_attention_weight_max": 30.473674774169922,
      "activations/layer2_attention_weight_min": -27.21942710876465,
      "activations/layer3_attention_weight_max": 72.5672378540039,
      "activations/layer3_attention_weight_min": -79.46376037597656,
      "activations/layer4_attention_weight_max": 102.05549621582031,
      "activations/layer4_attention_weight_min": -92.87794494628906,
      "activations/layer5_attention_weight_max": 126.04354095458984,
      "activations/layer5_attention_weight_min": -104.38543701171875,
      "activations/layer6_attention_weight_max": 61.44520568847656,
      "activations/layer6_attention_weight_min": -53.05466079711914,
      "activations/layer7_attention_weight_max": 61.58090591430664,
      "activations/layer7_attention_weight_min": -59.684410095214844,
      "activations/layer8_attention_weight_max": 43.33545684814453,
      "activations/layer8_attention_weight_min": -37.53865432739258,
      "activations/layer9_attention_weight_max": 56.80884552001953,
      "activations/layer9_attention_weight_min": -51.992034912109375,
      "epoch": 3.13,
      "learning_rate": 0.00013115984848484848,
      "loss": 2.8883,
      "step": 53800
    },
    {
      "activations/layer0_attention_weight_max": 14.261629104614258,
      "activations/layer0_attention_weight_min": -14.339237213134766,
      "activations/layer10_attention_weight_max": 44.50272750854492,
      "activations/layer10_attention_weight_min": -37.057373046875,
      "activations/layer11_attention_weight_max": 32.31476974487305,
      "activations/layer11_attention_weight_min": -26.590688705444336,
      "activations/layer12_attention_weight_max": 27.25960922241211,
      "activations/layer12_attention_weight_min": -28.89057159423828,
      "activations/layer13_attention_weight_max": 47.47813034057617,
      "activations/layer13_attention_weight_min": -35.80105972290039,
      "activations/layer14_attention_weight_max": 45.113548278808594,
      "activations/layer14_attention_weight_min": -34.907005310058594,
      "activations/layer15_attention_weight_max": 47.57610321044922,
      "activations/layer15_attention_weight_min": -32.55480194091797,
      "activations/layer16_attention_weight_max": 36.087650299072266,
      "activations/layer16_attention_weight_min": -27.996055603027344,
      "activations/layer17_attention_weight_max": 61.04718780517578,
      "activations/layer17_attention_weight_min": -36.70884323120117,
      "activations/layer18_attention_weight_max": 58.61561965942383,
      "activations/layer18_attention_weight_min": -30.695261001586914,
      "activations/layer19_attention_weight_max": 19.4008731842041,
      "activations/layer19_attention_weight_min": -19.21185874938965,
      "activations/layer1_attention_weight_max": 17.50040626525879,
      "activations/layer1_attention_weight_min": -14.970657348632812,
      "activations/layer20_attention_weight_max": 22.502714157104492,
      "activations/layer20_attention_weight_min": -19.74382209777832,
      "activations/layer21_attention_weight_max": 48.02012634277344,
      "activations/layer21_attention_weight_min": -28.681682586669922,
      "activations/layer22_attention_weight_max": 38.443809509277344,
      "activations/layer22_attention_weight_min": -23.714813232421875,
      "activations/layer23_attention_weight_max": 24.374067306518555,
      "activations/layer23_attention_weight_min": -18.07759666442871,
      "activations/layer2_attention_weight_max": 28.85834312438965,
      "activations/layer2_attention_weight_min": -26.718259811401367,
      "activations/layer3_attention_weight_max": 70.31688690185547,
      "activations/layer3_attention_weight_min": -76.43321228027344,
      "activations/layer4_attention_weight_max": 99.07305908203125,
      "activations/layer4_attention_weight_min": -90.90186309814453,
      "activations/layer5_attention_weight_max": 115.396484375,
      "activations/layer5_attention_weight_min": -101.88390350341797,
      "activations/layer6_attention_weight_max": 63.46540451049805,
      "activations/layer6_attention_weight_min": -52.02797317504883,
      "activations/layer7_attention_weight_max": 59.078182220458984,
      "activations/layer7_attention_weight_min": -56.258419036865234,
      "activations/layer8_attention_weight_max": 38.48488235473633,
      "activations/layer8_attention_weight_min": -36.30202865600586,
      "activations/layer9_attention_weight_max": 47.2283935546875,
      "activations/layer9_attention_weight_min": -45.087730407714844,
      "epoch": 3.13,
      "learning_rate": 0.00013114090909090907,
      "loss": 2.8905,
      "step": 53850
    },
    {
      "activations/layer0_attention_weight_max": 15.33277416229248,
      "activations/layer0_attention_weight_min": -14.35994815826416,
      "activations/layer10_attention_weight_max": 44.50688171386719,
      "activations/layer10_attention_weight_min": -34.5570068359375,
      "activations/layer11_attention_weight_max": 35.6331787109375,
      "activations/layer11_attention_weight_min": -26.37383270263672,
      "activations/layer12_attention_weight_max": 26.984067916870117,
      "activations/layer12_attention_weight_min": -23.866472244262695,
      "activations/layer13_attention_weight_max": 56.35057830810547,
      "activations/layer13_attention_weight_min": -34.74347686767578,
      "activations/layer14_attention_weight_max": 46.093994140625,
      "activations/layer14_attention_weight_min": -31.4255313873291,
      "activations/layer15_attention_weight_max": 48.867244720458984,
      "activations/layer15_attention_weight_min": -30.85665512084961,
      "activations/layer16_attention_weight_max": 41.88217544555664,
      "activations/layer16_attention_weight_min": -27.305070877075195,
      "activations/layer17_attention_weight_max": 60.64640426635742,
      "activations/layer17_attention_weight_min": -37.79396438598633,
      "activations/layer18_attention_weight_max": 57.538021087646484,
      "activations/layer18_attention_weight_min": -33.49346160888672,
      "activations/layer19_attention_weight_max": 21.36858367919922,
      "activations/layer19_attention_weight_min": -16.647123336791992,
      "activations/layer1_attention_weight_max": 17.214384078979492,
      "activations/layer1_attention_weight_min": -14.503019332885742,
      "activations/layer20_attention_weight_max": 19.935338973999023,
      "activations/layer20_attention_weight_min": -19.921226501464844,
      "activations/layer21_attention_weight_max": 42.90857696533203,
      "activations/layer21_attention_weight_min": -26.91398811340332,
      "activations/layer22_attention_weight_max": 36.751441955566406,
      "activations/layer22_attention_weight_min": -21.902217864990234,
      "activations/layer23_attention_weight_max": 26.449413299560547,
      "activations/layer23_attention_weight_min": -17.246244430541992,
      "activations/layer2_attention_weight_max": 28.844602584838867,
      "activations/layer2_attention_weight_min": -25.7919921875,
      "activations/layer3_attention_weight_max": 70.5833511352539,
      "activations/layer3_attention_weight_min": -75.09049987792969,
      "activations/layer4_attention_weight_max": 99.99440002441406,
      "activations/layer4_attention_weight_min": -90.93077087402344,
      "activations/layer5_attention_weight_max": 119.75677490234375,
      "activations/layer5_attention_weight_min": -98.59260559082031,
      "activations/layer6_attention_weight_max": 60.916419982910156,
      "activations/layer6_attention_weight_min": -49.61032485961914,
      "activations/layer7_attention_weight_max": 60.42692184448242,
      "activations/layer7_attention_weight_min": -54.44646072387695,
      "activations/layer8_attention_weight_max": 41.20138168334961,
      "activations/layer8_attention_weight_min": -36.290218353271484,
      "activations/layer9_attention_weight_max": 52.45789337158203,
      "activations/layer9_attention_weight_min": -45.51176452636719,
      "epoch": 3.13,
      "learning_rate": 0.0001311219696969697,
      "loss": 2.918,
      "step": 53900
    },
    {
      "activations/layer0_attention_weight_max": 14.018717765808105,
      "activations/layer0_attention_weight_min": -14.164618492126465,
      "activations/layer10_attention_weight_max": 46.83573532104492,
      "activations/layer10_attention_weight_min": -35.37671661376953,
      "activations/layer11_attention_weight_max": 40.120906829833984,
      "activations/layer11_attention_weight_min": -29.012760162353516,
      "activations/layer12_attention_weight_max": 30.16476821899414,
      "activations/layer12_attention_weight_min": -26.592100143432617,
      "activations/layer13_attention_weight_max": 60.95840835571289,
      "activations/layer13_attention_weight_min": -43.90201187133789,
      "activations/layer14_attention_weight_max": 73.28118133544922,
      "activations/layer14_attention_weight_min": -42.62644577026367,
      "activations/layer15_attention_weight_max": 55.81980895996094,
      "activations/layer15_attention_weight_min": -33.22422409057617,
      "activations/layer16_attention_weight_max": 44.34058380126953,
      "activations/layer16_attention_weight_min": -28.032869338989258,
      "activations/layer17_attention_weight_max": 66.69454956054688,
      "activations/layer17_attention_weight_min": -35.179927825927734,
      "activations/layer18_attention_weight_max": 66.38139343261719,
      "activations/layer18_attention_weight_min": -33.1267204284668,
      "activations/layer19_attention_weight_max": 32.90031814575195,
      "activations/layer19_attention_weight_min": -18.04002571105957,
      "activations/layer1_attention_weight_max": 17.545820236206055,
      "activations/layer1_attention_weight_min": -14.669283866882324,
      "activations/layer20_attention_weight_max": 23.555526733398438,
      "activations/layer20_attention_weight_min": -16.794221878051758,
      "activations/layer21_attention_weight_max": 59.21385955810547,
      "activations/layer21_attention_weight_min": -28.180089950561523,
      "activations/layer22_attention_weight_max": 41.805084228515625,
      "activations/layer22_attention_weight_min": -22.62390899658203,
      "activations/layer23_attention_weight_max": 28.396053314208984,
      "activations/layer23_attention_weight_min": -16.452678680419922,
      "activations/layer2_attention_weight_max": 31.648855209350586,
      "activations/layer2_attention_weight_min": -28.495031356811523,
      "activations/layer3_attention_weight_max": 71.75548553466797,
      "activations/layer3_attention_weight_min": -74.63322448730469,
      "activations/layer4_attention_weight_max": 105.3989486694336,
      "activations/layer4_attention_weight_min": -93.4337387084961,
      "activations/layer5_attention_weight_max": 128.443115234375,
      "activations/layer5_attention_weight_min": -105.05534362792969,
      "activations/layer6_attention_weight_max": 67.4961166381836,
      "activations/layer6_attention_weight_min": -53.1482048034668,
      "activations/layer7_attention_weight_max": 61.7708625793457,
      "activations/layer7_attention_weight_min": -55.232444763183594,
      "activations/layer8_attention_weight_max": 43.005821228027344,
      "activations/layer8_attention_weight_min": -37.19792556762695,
      "activations/layer9_attention_weight_max": 52.592620849609375,
      "activations/layer9_attention_weight_min": -47.653743743896484,
      "epoch": 3.13,
      "learning_rate": 0.0001311030303030303,
      "loss": 2.8906,
      "step": 53950
    },
    {
      "activations/layer0_attention_weight_max": 13.847660064697266,
      "activations/layer0_attention_weight_min": -14.243012428283691,
      "activations/layer10_attention_weight_max": 42.8320198059082,
      "activations/layer10_attention_weight_min": -35.65349197387695,
      "activations/layer11_attention_weight_max": 34.44599533081055,
      "activations/layer11_attention_weight_min": -29.451496124267578,
      "activations/layer12_attention_weight_max": 24.35723114013672,
      "activations/layer12_attention_weight_min": -26.612520217895508,
      "activations/layer13_attention_weight_max": 60.6877555847168,
      "activations/layer13_attention_weight_min": -44.68244171142578,
      "activations/layer14_attention_weight_max": 75.62022399902344,
      "activations/layer14_attention_weight_min": -53.43900680541992,
      "activations/layer15_attention_weight_max": 51.80313491821289,
      "activations/layer15_attention_weight_min": -36.692596435546875,
      "activations/layer16_attention_weight_max": 36.31440734863281,
      "activations/layer16_attention_weight_min": -28.873260498046875,
      "activations/layer17_attention_weight_max": 59.57537078857422,
      "activations/layer17_attention_weight_min": -37.12201690673828,
      "activations/layer18_attention_weight_max": 48.6936149597168,
      "activations/layer18_attention_weight_min": -32.8768424987793,
      "activations/layer19_attention_weight_max": 21.048450469970703,
      "activations/layer19_attention_weight_min": -18.029827117919922,
      "activations/layer1_attention_weight_max": 18.880619049072266,
      "activations/layer1_attention_weight_min": -14.961307525634766,
      "activations/layer20_attention_weight_max": 20.33489227294922,
      "activations/layer20_attention_weight_min": -19.690296173095703,
      "activations/layer21_attention_weight_max": 42.78358459472656,
      "activations/layer21_attention_weight_min": -29.869384765625,
      "activations/layer22_attention_weight_max": 33.287410736083984,
      "activations/layer22_attention_weight_min": -23.871585845947266,
      "activations/layer23_attention_weight_max": 29.25389862060547,
      "activations/layer23_attention_weight_min": -18.669342041015625,
      "activations/layer2_attention_weight_max": 29.141563415527344,
      "activations/layer2_attention_weight_min": -27.24506950378418,
      "activations/layer3_attention_weight_max": 70.88050842285156,
      "activations/layer3_attention_weight_min": -77.77684020996094,
      "activations/layer4_attention_weight_max": 101.2082748413086,
      "activations/layer4_attention_weight_min": -91.30919647216797,
      "activations/layer5_attention_weight_max": 119.9691162109375,
      "activations/layer5_attention_weight_min": -104.61404418945312,
      "activations/layer6_attention_weight_max": 60.11701965332031,
      "activations/layer6_attention_weight_min": -51.40323257446289,
      "activations/layer7_attention_weight_max": 60.918067932128906,
      "activations/layer7_attention_weight_min": -58.63223648071289,
      "activations/layer8_attention_weight_max": 40.9323616027832,
      "activations/layer8_attention_weight_min": -38.713409423828125,
      "activations/layer9_attention_weight_max": 56.106903076171875,
      "activations/layer9_attention_weight_min": -50.41803741455078,
      "epoch": 3.14,
      "learning_rate": 0.00013108409090909092,
      "loss": 2.9092,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_loss": 2.853515625,
      "eval_runtime": 8.5057,
      "eval_samples_per_second": 504.838,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_openwebtext_loss": 2.853515625,
      "eval_openwebtext_ppl": 17.348666159108166,
      "eval_openwebtext_runtime": 8.5057,
      "eval_openwebtext_samples_per_second": 504.838,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_wikitext_loss": 3.10546875,
      "eval_wikitext_ppl": 22.319678873017494,
      "eval_wikitext_runtime": 2.0032,
      "eval_wikitext_samples_per_second": 227.632,
      "step": 54000
    },
    {
      "epoch": 3.14,
      "eval_lambada_loss": 3.08984375,
      "eval_lambada_ppl": 21.973644325590612,
      "eval_lambada_runtime": 9.6368,
      "eval_lambada_samples_per_second": 505.25,
      "step": 54000
    },
    {
      "activations/layer0_attention_weight_max": 15.19623851776123,
      "activations/layer0_attention_weight_min": -14.200100898742676,
      "activations/layer10_attention_weight_max": 49.00441360473633,
      "activations/layer10_attention_weight_min": -37.590553283691406,
      "activations/layer11_attention_weight_max": 42.73545837402344,
      "activations/layer11_attention_weight_min": -29.710350036621094,
      "activations/layer12_attention_weight_max": 32.0200080871582,
      "activations/layer12_attention_weight_min": -23.971601486206055,
      "activations/layer13_attention_weight_max": 68.40209197998047,
      "activations/layer13_attention_weight_min": -42.98915100097656,
      "activations/layer14_attention_weight_max": 53.16063690185547,
      "activations/layer14_attention_weight_min": -33.38452911376953,
      "activations/layer15_attention_weight_max": 51.97917175292969,
      "activations/layer15_attention_weight_min": -35.670928955078125,
      "activations/layer16_attention_weight_max": 40.07744598388672,
      "activations/layer16_attention_weight_min": -26.560943603515625,
      "activations/layer17_attention_weight_max": 66.05914306640625,
      "activations/layer17_attention_weight_min": -36.2493782043457,
      "activations/layer18_attention_weight_max": 60.80535125732422,
      "activations/layer18_attention_weight_min": -33.6323356628418,
      "activations/layer19_attention_weight_max": 25.40407371520996,
      "activations/layer19_attention_weight_min": -17.636085510253906,
      "activations/layer1_attention_weight_max": 18.042390823364258,
      "activations/layer1_attention_weight_min": -14.479647636413574,
      "activations/layer20_attention_weight_max": 24.630659103393555,
      "activations/layer20_attention_weight_min": -19.55613136291504,
      "activations/layer21_attention_weight_max": 55.31275177001953,
      "activations/layer21_attention_weight_min": -32.10295486450195,
      "activations/layer22_attention_weight_max": 41.20473861694336,
      "activations/layer22_attention_weight_min": -24.01334571838379,
      "activations/layer23_attention_weight_max": 35.78746795654297,
      "activations/layer23_attention_weight_min": -16.165918350219727,
      "activations/layer2_attention_weight_max": 31.13509178161621,
      "activations/layer2_attention_weight_min": -27.12087059020996,
      "activations/layer3_attention_weight_max": 75.51968383789062,
      "activations/layer3_attention_weight_min": -77.6298599243164,
      "activations/layer4_attention_weight_max": 101.80630493164062,
      "activations/layer4_attention_weight_min": -90.89876556396484,
      "activations/layer5_attention_weight_max": 125.73612976074219,
      "activations/layer5_attention_weight_min": -102.51399993896484,
      "activations/layer6_attention_weight_max": 62.564842224121094,
      "activations/layer6_attention_weight_min": -53.16673278808594,
      "activations/layer7_attention_weight_max": 69.17700958251953,
      "activations/layer7_attention_weight_min": -59.42986297607422,
      "activations/layer8_attention_weight_max": 46.98856735229492,
      "activations/layer8_attention_weight_min": -37.78642654418945,
      "activations/layer9_attention_weight_max": 63.865692138671875,
      "activations/layer9_attention_weight_min": -52.66889572143555,
      "epoch": 3.14,
      "learning_rate": 0.0001310651515151515,
      "loss": 2.9108,
      "step": 54050
    },
    {
      "activations/layer0_attention_weight_max": 14.070903778076172,
      "activations/layer0_attention_weight_min": -13.604654312133789,
      "activations/layer10_attention_weight_max": 49.798763275146484,
      "activations/layer10_attention_weight_min": -36.624420166015625,
      "activations/layer11_attention_weight_max": 41.97860336303711,
      "activations/layer11_attention_weight_min": -29.370010375976562,
      "activations/layer12_attention_weight_max": 26.99131202697754,
      "activations/layer12_attention_weight_min": -24.97527503967285,
      "activations/layer13_attention_weight_max": 61.052040100097656,
      "activations/layer13_attention_weight_min": -38.06770324707031,
      "activations/layer14_attention_weight_max": 56.94206619262695,
      "activations/layer14_attention_weight_min": -32.65966033935547,
      "activations/layer15_attention_weight_max": 54.37220764160156,
      "activations/layer15_attention_weight_min": -32.29094696044922,
      "activations/layer16_attention_weight_max": 42.11112594604492,
      "activations/layer16_attention_weight_min": -25.770742416381836,
      "activations/layer17_attention_weight_max": 71.44055938720703,
      "activations/layer17_attention_weight_min": -34.91536331176758,
      "activations/layer18_attention_weight_max": 57.8222770690918,
      "activations/layer18_attention_weight_min": -30.56899642944336,
      "activations/layer19_attention_weight_max": 26.40289306640625,
      "activations/layer19_attention_weight_min": -19.245161056518555,
      "activations/layer1_attention_weight_max": 17.66039276123047,
      "activations/layer1_attention_weight_min": -14.810264587402344,
      "activations/layer20_attention_weight_max": 25.3455810546875,
      "activations/layer20_attention_weight_min": -19.332651138305664,
      "activations/layer21_attention_weight_max": 52.422996520996094,
      "activations/layer21_attention_weight_min": -29.98735809326172,
      "activations/layer22_attention_weight_max": 38.89622116088867,
      "activations/layer22_attention_weight_min": -23.487979888916016,
      "activations/layer23_attention_weight_max": 29.44127655029297,
      "activations/layer23_attention_weight_min": -17.36286163330078,
      "activations/layer2_attention_weight_max": 30.987546920776367,
      "activations/layer2_attention_weight_min": -28.5831298828125,
      "activations/layer3_attention_weight_max": 73.25721740722656,
      "activations/layer3_attention_weight_min": -80.17089080810547,
      "activations/layer4_attention_weight_max": 99.75978088378906,
      "activations/layer4_attention_weight_min": -94.35619354248047,
      "activations/layer5_attention_weight_max": 116.21380615234375,
      "activations/layer5_attention_weight_min": -104.85076904296875,
      "activations/layer6_attention_weight_max": 58.87842559814453,
      "activations/layer6_attention_weight_min": -49.640987396240234,
      "activations/layer7_attention_weight_max": 59.4891357421875,
      "activations/layer7_attention_weight_min": -55.51666259765625,
      "activations/layer8_attention_weight_max": 38.74905014038086,
      "activations/layer8_attention_weight_min": -37.01360321044922,
      "activations/layer9_attention_weight_max": 56.767024993896484,
      "activations/layer9_attention_weight_min": -46.20759201049805,
      "epoch": 3.14,
      "learning_rate": 0.00013104621212121212,
      "loss": 2.9064,
      "step": 54100
    },
    {
      "activations/layer0_attention_weight_max": 14.630446434020996,
      "activations/layer0_attention_weight_min": -14.193954467773438,
      "activations/layer10_attention_weight_max": 41.616764068603516,
      "activations/layer10_attention_weight_min": -35.64875411987305,
      "activations/layer11_attention_weight_max": 33.366737365722656,
      "activations/layer11_attention_weight_min": -26.956884384155273,
      "activations/layer12_attention_weight_max": 23.235736846923828,
      "activations/layer12_attention_weight_min": -23.013708114624023,
      "activations/layer13_attention_weight_max": 47.44581604003906,
      "activations/layer13_attention_weight_min": -39.21778106689453,
      "activations/layer14_attention_weight_max": 42.00853729248047,
      "activations/layer14_attention_weight_min": -33.06829071044922,
      "activations/layer15_attention_weight_max": 44.34056854248047,
      "activations/layer15_attention_weight_min": -33.46773910522461,
      "activations/layer16_attention_weight_max": 36.35847091674805,
      "activations/layer16_attention_weight_min": -27.274381637573242,
      "activations/layer17_attention_weight_max": 52.52136993408203,
      "activations/layer17_attention_weight_min": -41.66563034057617,
      "activations/layer18_attention_weight_max": 50.09694290161133,
      "activations/layer18_attention_weight_min": -33.103973388671875,
      "activations/layer19_attention_weight_max": 21.414825439453125,
      "activations/layer19_attention_weight_min": -25.288673400878906,
      "activations/layer1_attention_weight_max": 17.329395294189453,
      "activations/layer1_attention_weight_min": -16.92327117919922,
      "activations/layer20_attention_weight_max": 21.52033042907715,
      "activations/layer20_attention_weight_min": -20.276805877685547,
      "activations/layer21_attention_weight_max": 38.99806594848633,
      "activations/layer21_attention_weight_min": -30.60167121887207,
      "activations/layer22_attention_weight_max": 31.76366424560547,
      "activations/layer22_attention_weight_min": -24.04454803466797,
      "activations/layer23_attention_weight_max": 24.200267791748047,
      "activations/layer23_attention_weight_min": -16.887985229492188,
      "activations/layer2_attention_weight_max": 30.451839447021484,
      "activations/layer2_attention_weight_min": -28.222896575927734,
      "activations/layer3_attention_weight_max": 74.908935546875,
      "activations/layer3_attention_weight_min": -78.69630432128906,
      "activations/layer4_attention_weight_max": 99.68473052978516,
      "activations/layer4_attention_weight_min": -94.77837371826172,
      "activations/layer5_attention_weight_max": 118.39932250976562,
      "activations/layer5_attention_weight_min": -103.88680267333984,
      "activations/layer6_attention_weight_max": 60.96921920776367,
      "activations/layer6_attention_weight_min": -53.922828674316406,
      "activations/layer7_attention_weight_max": 56.70095443725586,
      "activations/layer7_attention_weight_min": -54.416664123535156,
      "activations/layer8_attention_weight_max": 38.59260940551758,
      "activations/layer8_attention_weight_min": -36.81009292602539,
      "activations/layer9_attention_weight_max": 57.07882308959961,
      "activations/layer9_attention_weight_min": -48.94309616088867,
      "epoch": 3.15,
      "learning_rate": 0.0001310272727272727,
      "loss": 2.9027,
      "step": 54150
    },
    {
      "activations/layer0_attention_weight_max": 13.862716674804688,
      "activations/layer0_attention_weight_min": -13.887580871582031,
      "activations/layer10_attention_weight_max": 48.94556427001953,
      "activations/layer10_attention_weight_min": -36.661529541015625,
      "activations/layer11_attention_weight_max": 36.00830841064453,
      "activations/layer11_attention_weight_min": -28.535375595092773,
      "activations/layer12_attention_weight_max": 28.179264068603516,
      "activations/layer12_attention_weight_min": -27.62161636352539,
      "activations/layer13_attention_weight_max": 58.96329879760742,
      "activations/layer13_attention_weight_min": -44.73453903198242,
      "activations/layer14_attention_weight_max": 59.049739837646484,
      "activations/layer14_attention_weight_min": -33.07217788696289,
      "activations/layer15_attention_weight_max": 48.16559600830078,
      "activations/layer15_attention_weight_min": -32.58585739135742,
      "activations/layer16_attention_weight_max": 40.746002197265625,
      "activations/layer16_attention_weight_min": -26.061832427978516,
      "activations/layer17_attention_weight_max": 61.83924865722656,
      "activations/layer17_attention_weight_min": -34.715301513671875,
      "activations/layer18_attention_weight_max": 54.70703125,
      "activations/layer18_attention_weight_min": -30.454998016357422,
      "activations/layer19_attention_weight_max": 24.02716636657715,
      "activations/layer19_attention_weight_min": -17.43453025817871,
      "activations/layer1_attention_weight_max": 18.449085235595703,
      "activations/layer1_attention_weight_min": -15.033697128295898,
      "activations/layer20_attention_weight_max": 20.987451553344727,
      "activations/layer20_attention_weight_min": -19.998546600341797,
      "activations/layer21_attention_weight_max": 42.19268798828125,
      "activations/layer21_attention_weight_min": -28.026681900024414,
      "activations/layer22_attention_weight_max": 35.95890808105469,
      "activations/layer22_attention_weight_min": -23.80870819091797,
      "activations/layer23_attention_weight_max": 27.00668716430664,
      "activations/layer23_attention_weight_min": -16.998058319091797,
      "activations/layer2_attention_weight_max": 29.393997192382812,
      "activations/layer2_attention_weight_min": -27.0356502532959,
      "activations/layer3_attention_weight_max": 68.71426391601562,
      "activations/layer3_attention_weight_min": -72.87733459472656,
      "activations/layer4_attention_weight_max": 101.46721649169922,
      "activations/layer4_attention_weight_min": -91.70243835449219,
      "activations/layer5_attention_weight_max": 123.72130584716797,
      "activations/layer5_attention_weight_min": -97.51832580566406,
      "activations/layer6_attention_weight_max": 61.14573287963867,
      "activations/layer6_attention_weight_min": -48.74055480957031,
      "activations/layer7_attention_weight_max": 59.236995697021484,
      "activations/layer7_attention_weight_min": -54.91238021850586,
      "activations/layer8_attention_weight_max": 39.05317687988281,
      "activations/layer8_attention_weight_min": -34.721885681152344,
      "activations/layer9_attention_weight_max": 51.06292724609375,
      "activations/layer9_attention_weight_min": -47.88917541503906,
      "epoch": 3.15,
      "learning_rate": 0.00013100833333333332,
      "loss": 2.9057,
      "step": 54200
    },
    {
      "activations/layer0_attention_weight_max": 13.608418464660645,
      "activations/layer0_attention_weight_min": -15.185029029846191,
      "activations/layer10_attention_weight_max": 47.471336364746094,
      "activations/layer10_attention_weight_min": -36.13441848754883,
      "activations/layer11_attention_weight_max": 35.80458068847656,
      "activations/layer11_attention_weight_min": -27.566905975341797,
      "activations/layer12_attention_weight_max": 25.296476364135742,
      "activations/layer12_attention_weight_min": -22.914079666137695,
      "activations/layer13_attention_weight_max": 53.00296401977539,
      "activations/layer13_attention_weight_min": -38.27411651611328,
      "activations/layer14_attention_weight_max": 51.115150451660156,
      "activations/layer14_attention_weight_min": -36.7044563293457,
      "activations/layer15_attention_weight_max": 51.21383285522461,
      "activations/layer15_attention_weight_min": -31.40561294555664,
      "activations/layer16_attention_weight_max": 35.92681121826172,
      "activations/layer16_attention_weight_min": -26.512752532958984,
      "activations/layer17_attention_weight_max": 59.82255172729492,
      "activations/layer17_attention_weight_min": -35.921409606933594,
      "activations/layer18_attention_weight_max": 52.137474060058594,
      "activations/layer18_attention_weight_min": -30.955787658691406,
      "activations/layer19_attention_weight_max": 19.657426834106445,
      "activations/layer19_attention_weight_min": -16.579360961914062,
      "activations/layer1_attention_weight_max": 17.607446670532227,
      "activations/layer1_attention_weight_min": -14.29758358001709,
      "activations/layer20_attention_weight_max": 22.58588409423828,
      "activations/layer20_attention_weight_min": -20.006757736206055,
      "activations/layer21_attention_weight_max": 47.008724212646484,
      "activations/layer21_attention_weight_min": -27.055553436279297,
      "activations/layer22_attention_weight_max": 40.31084060668945,
      "activations/layer22_attention_weight_min": -22.220184326171875,
      "activations/layer23_attention_weight_max": 28.31440544128418,
      "activations/layer23_attention_weight_min": -15.09913158416748,
      "activations/layer2_attention_weight_max": 27.540786743164062,
      "activations/layer2_attention_weight_min": -27.37790298461914,
      "activations/layer3_attention_weight_max": 73.72994995117188,
      "activations/layer3_attention_weight_min": -80.17406463623047,
      "activations/layer4_attention_weight_max": 102.05331420898438,
      "activations/layer4_attention_weight_min": -91.95635986328125,
      "activations/layer5_attention_weight_max": 126.99560546875,
      "activations/layer5_attention_weight_min": -101.32620239257812,
      "activations/layer6_attention_weight_max": 66.66984558105469,
      "activations/layer6_attention_weight_min": -51.09856414794922,
      "activations/layer7_attention_weight_max": 63.23154830932617,
      "activations/layer7_attention_weight_min": -60.80758285522461,
      "activations/layer8_attention_weight_max": 38.8212776184082,
      "activations/layer8_attention_weight_min": -39.4859619140625,
      "activations/layer9_attention_weight_max": 53.768402099609375,
      "activations/layer9_attention_weight_min": -46.99814224243164,
      "epoch": 3.15,
      "learning_rate": 0.00013098939393939394,
      "loss": 2.9018,
      "step": 54250
    },
    {
      "activations/layer0_attention_weight_max": 15.632543563842773,
      "activations/layer0_attention_weight_min": -14.272122383117676,
      "activations/layer10_attention_weight_max": 44.00445556640625,
      "activations/layer10_attention_weight_min": -34.614280700683594,
      "activations/layer11_attention_weight_max": 38.81590270996094,
      "activations/layer11_attention_weight_min": -25.934234619140625,
      "activations/layer12_attention_weight_max": 25.35643196105957,
      "activations/layer12_attention_weight_min": -25.015649795532227,
      "activations/layer13_attention_weight_max": 54.22937774658203,
      "activations/layer13_attention_weight_min": -37.442039489746094,
      "activations/layer14_attention_weight_max": 49.77842330932617,
      "activations/layer14_attention_weight_min": -30.192846298217773,
      "activations/layer15_attention_weight_max": 44.80247497558594,
      "activations/layer15_attention_weight_min": -31.47159767150879,
      "activations/layer16_attention_weight_max": 33.441043853759766,
      "activations/layer16_attention_weight_min": -26.487239837646484,
      "activations/layer17_attention_weight_max": 56.441776275634766,
      "activations/layer17_attention_weight_min": -37.17371368408203,
      "activations/layer18_attention_weight_max": 50.17190933227539,
      "activations/layer18_attention_weight_min": -31.993257522583008,
      "activations/layer19_attention_weight_max": 22.701337814331055,
      "activations/layer19_attention_weight_min": -16.921669006347656,
      "activations/layer1_attention_weight_max": 17.710599899291992,
      "activations/layer1_attention_weight_min": -12.508467674255371,
      "activations/layer20_attention_weight_max": 23.253232955932617,
      "activations/layer20_attention_weight_min": -21.78688621520996,
      "activations/layer21_attention_weight_max": 45.31682205200195,
      "activations/layer21_attention_weight_min": -30.150005340576172,
      "activations/layer22_attention_weight_max": 33.87863540649414,
      "activations/layer22_attention_weight_min": -24.22870635986328,
      "activations/layer23_attention_weight_max": 26.27608871459961,
      "activations/layer23_attention_weight_min": -17.06517219543457,
      "activations/layer2_attention_weight_max": 29.21371078491211,
      "activations/layer2_attention_weight_min": -27.72530746459961,
      "activations/layer3_attention_weight_max": 73.07444763183594,
      "activations/layer3_attention_weight_min": -76.2403564453125,
      "activations/layer4_attention_weight_max": 100.72700500488281,
      "activations/layer4_attention_weight_min": -92.85130310058594,
      "activations/layer5_attention_weight_max": 121.97810363769531,
      "activations/layer5_attention_weight_min": -103.69243621826172,
      "activations/layer6_attention_weight_max": 62.591094970703125,
      "activations/layer6_attention_weight_min": -53.7032585144043,
      "activations/layer7_attention_weight_max": 65.46207427978516,
      "activations/layer7_attention_weight_min": -56.172882080078125,
      "activations/layer8_attention_weight_max": 43.929325103759766,
      "activations/layer8_attention_weight_min": -37.34877395629883,
      "activations/layer9_attention_weight_max": 49.967952728271484,
      "activations/layer9_attention_weight_min": -46.59878158569336,
      "epoch": 3.16,
      "learning_rate": 0.00013097083333333333,
      "loss": 2.8843,
      "step": 54300
    },
    {
      "activations/layer0_attention_weight_max": 13.790993690490723,
      "activations/layer0_attention_weight_min": -14.843963623046875,
      "activations/layer10_attention_weight_max": 42.283836364746094,
      "activations/layer10_attention_weight_min": -35.36183547973633,
      "activations/layer11_attention_weight_max": 40.38532638549805,
      "activations/layer11_attention_weight_min": -32.41534423828125,
      "activations/layer12_attention_weight_max": 26.816068649291992,
      "activations/layer12_attention_weight_min": -22.648990631103516,
      "activations/layer13_attention_weight_max": 51.272850036621094,
      "activations/layer13_attention_weight_min": -38.863128662109375,
      "activations/layer14_attention_weight_max": 50.25796127319336,
      "activations/layer14_attention_weight_min": -34.85536575317383,
      "activations/layer15_attention_weight_max": 49.911808013916016,
      "activations/layer15_attention_weight_min": -34.74836349487305,
      "activations/layer16_attention_weight_max": 37.24477767944336,
      "activations/layer16_attention_weight_min": -27.891773223876953,
      "activations/layer17_attention_weight_max": 61.49386978149414,
      "activations/layer17_attention_weight_min": -39.56755828857422,
      "activations/layer18_attention_weight_max": 49.36771011352539,
      "activations/layer18_attention_weight_min": -34.81167984008789,
      "activations/layer19_attention_weight_max": 24.381919860839844,
      "activations/layer19_attention_weight_min": -19.881568908691406,
      "activations/layer1_attention_weight_max": 18.32297706604004,
      "activations/layer1_attention_weight_min": -13.53636360168457,
      "activations/layer20_attention_weight_max": 21.28390884399414,
      "activations/layer20_attention_weight_min": -20.154396057128906,
      "activations/layer21_attention_weight_max": 44.20317840576172,
      "activations/layer21_attention_weight_min": -27.585453033447266,
      "activations/layer22_attention_weight_max": 40.7916145324707,
      "activations/layer22_attention_weight_min": -22.25103759765625,
      "activations/layer23_attention_weight_max": 28.326948165893555,
      "activations/layer23_attention_weight_min": -18.675636291503906,
      "activations/layer2_attention_weight_max": 29.211669921875,
      "activations/layer2_attention_weight_min": -27.668407440185547,
      "activations/layer3_attention_weight_max": 73.95427703857422,
      "activations/layer3_attention_weight_min": -77.63904571533203,
      "activations/layer4_attention_weight_max": 102.25311279296875,
      "activations/layer4_attention_weight_min": -92.50254821777344,
      "activations/layer5_attention_weight_max": 121.32417297363281,
      "activations/layer5_attention_weight_min": -99.31205749511719,
      "activations/layer6_attention_weight_max": 62.80400848388672,
      "activations/layer6_attention_weight_min": -50.59941482543945,
      "activations/layer7_attention_weight_max": 60.365631103515625,
      "activations/layer7_attention_weight_min": -57.413330078125,
      "activations/layer8_attention_weight_max": 38.7049560546875,
      "activations/layer8_attention_weight_min": -34.62701416015625,
      "activations/layer9_attention_weight_max": 49.512752532958984,
      "activations/layer9_attention_weight_min": -47.644004821777344,
      "epoch": 3.16,
      "learning_rate": 0.00013095189393939392,
      "loss": 2.8975,
      "step": 54350
    },
    {
      "activations/layer0_attention_weight_max": 15.816009521484375,
      "activations/layer0_attention_weight_min": -14.774600982666016,
      "activations/layer10_attention_weight_max": 46.35693359375,
      "activations/layer10_attention_weight_min": -36.69528579711914,
      "activations/layer11_attention_weight_max": 36.07524108886719,
      "activations/layer11_attention_weight_min": -28.104063034057617,
      "activations/layer12_attention_weight_max": 29.540912628173828,
      "activations/layer12_attention_weight_min": -28.564773559570312,
      "activations/layer13_attention_weight_max": 64.92090606689453,
      "activations/layer13_attention_weight_min": -40.39901351928711,
      "activations/layer14_attention_weight_max": 61.696895599365234,
      "activations/layer14_attention_weight_min": -35.06925964355469,
      "activations/layer15_attention_weight_max": 50.994571685791016,
      "activations/layer15_attention_weight_min": -32.25101089477539,
      "activations/layer16_attention_weight_max": 47.1745719909668,
      "activations/layer16_attention_weight_min": -27.603057861328125,
      "activations/layer17_attention_weight_max": 69.5196304321289,
      "activations/layer17_attention_weight_min": -35.528812408447266,
      "activations/layer18_attention_weight_max": 63.21589279174805,
      "activations/layer18_attention_weight_min": -36.85967254638672,
      "activations/layer19_attention_weight_max": 29.270578384399414,
      "activations/layer19_attention_weight_min": -19.00176429748535,
      "activations/layer1_attention_weight_max": 17.94623374938965,
      "activations/layer1_attention_weight_min": -13.804226875305176,
      "activations/layer20_attention_weight_max": 23.570837020874023,
      "activations/layer20_attention_weight_min": -16.70292091369629,
      "activations/layer21_attention_weight_max": 44.01735305786133,
      "activations/layer21_attention_weight_min": -27.436376571655273,
      "activations/layer22_attention_weight_max": 42.671756744384766,
      "activations/layer22_attention_weight_min": -21.90909194946289,
      "activations/layer23_attention_weight_max": 24.795578002929688,
      "activations/layer23_attention_weight_min": -15.594268798828125,
      "activations/layer2_attention_weight_max": 30.06515121459961,
      "activations/layer2_attention_weight_min": -28.240711212158203,
      "activations/layer3_attention_weight_max": 74.47628784179688,
      "activations/layer3_attention_weight_min": -73.54995727539062,
      "activations/layer4_attention_weight_max": 103.99226379394531,
      "activations/layer4_attention_weight_min": -89.7371826171875,
      "activations/layer5_attention_weight_max": 123.78009033203125,
      "activations/layer5_attention_weight_min": -100.08477020263672,
      "activations/layer6_attention_weight_max": 62.95792770385742,
      "activations/layer6_attention_weight_min": -51.28012466430664,
      "activations/layer7_attention_weight_max": 63.047119140625,
      "activations/layer7_attention_weight_min": -56.438331604003906,
      "activations/layer8_attention_weight_max": 46.805938720703125,
      "activations/layer8_attention_weight_min": -37.19458770751953,
      "activations/layer9_attention_weight_max": 59.915260314941406,
      "activations/layer9_attention_weight_min": -51.13998794555664,
      "epoch": 3.16,
      "learning_rate": 0.00013093295454545453,
      "loss": 2.882,
      "step": 54400
    },
    {
      "activations/layer0_attention_weight_max": 15.075372695922852,
      "activations/layer0_attention_weight_min": -14.887068748474121,
      "activations/layer10_attention_weight_max": 51.6477165222168,
      "activations/layer10_attention_weight_min": -36.04494857788086,
      "activations/layer11_attention_weight_max": 35.79378890991211,
      "activations/layer11_attention_weight_min": -28.291654586791992,
      "activations/layer12_attention_weight_max": 31.474258422851562,
      "activations/layer12_attention_weight_min": -24.161828994750977,
      "activations/layer13_attention_weight_max": 67.38067626953125,
      "activations/layer13_attention_weight_min": -38.522151947021484,
      "activations/layer14_attention_weight_max": 62.7604866027832,
      "activations/layer14_attention_weight_min": -31.492536544799805,
      "activations/layer15_attention_weight_max": 54.676815032958984,
      "activations/layer15_attention_weight_min": -36.64093017578125,
      "activations/layer16_attention_weight_max": 43.04737854003906,
      "activations/layer16_attention_weight_min": -26.456233978271484,
      "activations/layer17_attention_weight_max": 66.09953308105469,
      "activations/layer17_attention_weight_min": -37.83191680908203,
      "activations/layer18_attention_weight_max": 57.25197219848633,
      "activations/layer18_attention_weight_min": -32.034027099609375,
      "activations/layer19_attention_weight_max": 23.772924423217773,
      "activations/layer19_attention_weight_min": -18.25459098815918,
      "activations/layer1_attention_weight_max": 18.007200241088867,
      "activations/layer1_attention_weight_min": -15.355424880981445,
      "activations/layer20_attention_weight_max": 26.156299591064453,
      "activations/layer20_attention_weight_min": -20.595035552978516,
      "activations/layer21_attention_weight_max": 49.48459243774414,
      "activations/layer21_attention_weight_min": -27.53201675415039,
      "activations/layer22_attention_weight_max": 39.44032669067383,
      "activations/layer22_attention_weight_min": -23.178863525390625,
      "activations/layer23_attention_weight_max": 29.71094512939453,
      "activations/layer23_attention_weight_min": -17.103500366210938,
      "activations/layer2_attention_weight_max": 28.38240623474121,
      "activations/layer2_attention_weight_min": -27.45932388305664,
      "activations/layer3_attention_weight_max": 71.536865234375,
      "activations/layer3_attention_weight_min": -74.6291732788086,
      "activations/layer4_attention_weight_max": 103.88313293457031,
      "activations/layer4_attention_weight_min": -92.18964385986328,
      "activations/layer5_attention_weight_max": 135.86825561523438,
      "activations/layer5_attention_weight_min": -102.39332580566406,
      "activations/layer6_attention_weight_max": 70.74649047851562,
      "activations/layer6_attention_weight_min": -52.513607025146484,
      "activations/layer7_attention_weight_max": 71.16847229003906,
      "activations/layer7_attention_weight_min": -57.84554672241211,
      "activations/layer8_attention_weight_max": 39.210784912109375,
      "activations/layer8_attention_weight_min": -36.49866485595703,
      "activations/layer9_attention_weight_max": 55.270179748535156,
      "activations/layer9_attention_weight_min": -49.13983917236328,
      "epoch": 3.16,
      "learning_rate": 0.00013091401515151515,
      "loss": 2.904,
      "step": 54450
    },
    {
      "activations/layer0_attention_weight_max": 16.395915985107422,
      "activations/layer0_attention_weight_min": -14.510388374328613,
      "activations/layer10_attention_weight_max": 50.15932083129883,
      "activations/layer10_attention_weight_min": -35.31049346923828,
      "activations/layer11_attention_weight_max": 42.07292938232422,
      "activations/layer11_attention_weight_min": -28.50800895690918,
      "activations/layer12_attention_weight_max": 39.66815185546875,
      "activations/layer12_attention_weight_min": -25.30161476135254,
      "activations/layer13_attention_weight_max": 72.54032897949219,
      "activations/layer13_attention_weight_min": -42.54658508300781,
      "activations/layer14_attention_weight_max": 61.36275100708008,
      "activations/layer14_attention_weight_min": -33.89984893798828,
      "activations/layer15_attention_weight_max": 57.882164001464844,
      "activations/layer15_attention_weight_min": -33.780941009521484,
      "activations/layer16_attention_weight_max": 53.351131439208984,
      "activations/layer16_attention_weight_min": -26.293766021728516,
      "activations/layer17_attention_weight_max": 65.01802825927734,
      "activations/layer17_attention_weight_min": -37.5959587097168,
      "activations/layer18_attention_weight_max": 61.49768829345703,
      "activations/layer18_attention_weight_min": -32.06976318359375,
      "activations/layer19_attention_weight_max": 23.524150848388672,
      "activations/layer19_attention_weight_min": -17.57888412475586,
      "activations/layer1_attention_weight_max": 16.962871551513672,
      "activations/layer1_attention_weight_min": -14.479072570800781,
      "activations/layer20_attention_weight_max": 22.64388656616211,
      "activations/layer20_attention_weight_min": -20.774980545043945,
      "activations/layer21_attention_weight_max": 42.65372085571289,
      "activations/layer21_attention_weight_min": -29.401748657226562,
      "activations/layer22_attention_weight_max": 41.86446762084961,
      "activations/layer22_attention_weight_min": -24.05120277404785,
      "activations/layer23_attention_weight_max": 26.72427749633789,
      "activations/layer23_attention_weight_min": -19.045066833496094,
      "activations/layer2_attention_weight_max": 31.98458480834961,
      "activations/layer2_attention_weight_min": -28.57501792907715,
      "activations/layer3_attention_weight_max": 76.09100341796875,
      "activations/layer3_attention_weight_min": -76.84429931640625,
      "activations/layer4_attention_weight_max": 105.46858215332031,
      "activations/layer4_attention_weight_min": -92.51283264160156,
      "activations/layer5_attention_weight_max": 123.17234802246094,
      "activations/layer5_attention_weight_min": -102.44213104248047,
      "activations/layer6_attention_weight_max": 63.96574783325195,
      "activations/layer6_attention_weight_min": -52.3260612487793,
      "activations/layer7_attention_weight_max": 70.05892944335938,
      "activations/layer7_attention_weight_min": -61.11860275268555,
      "activations/layer8_attention_weight_max": 46.80906295776367,
      "activations/layer8_attention_weight_min": -35.87779998779297,
      "activations/layer9_attention_weight_max": 65.89038848876953,
      "activations/layer9_attention_weight_min": -51.27567672729492,
      "epoch": 3.17,
      "learning_rate": 0.00013089507575757574,
      "loss": 2.9081,
      "step": 54500
    },
    {
      "activations/layer0_attention_weight_max": 14.283692359924316,
      "activations/layer0_attention_weight_min": -14.921344757080078,
      "activations/layer10_attention_weight_max": 42.87186813354492,
      "activations/layer10_attention_weight_min": -36.381961822509766,
      "activations/layer11_attention_weight_max": 40.948455810546875,
      "activations/layer11_attention_weight_min": -29.377727508544922,
      "activations/layer12_attention_weight_max": 30.010587692260742,
      "activations/layer12_attention_weight_min": -24.47772979736328,
      "activations/layer13_attention_weight_max": 53.56607437133789,
      "activations/layer13_attention_weight_min": -38.847747802734375,
      "activations/layer14_attention_weight_max": 54.380191802978516,
      "activations/layer14_attention_weight_min": -35.86701583862305,
      "activations/layer15_attention_weight_max": 44.500850677490234,
      "activations/layer15_attention_weight_min": -31.97358512878418,
      "activations/layer16_attention_weight_max": 40.94711685180664,
      "activations/layer16_attention_weight_min": -26.9613037109375,
      "activations/layer17_attention_weight_max": 60.5692024230957,
      "activations/layer17_attention_weight_min": -37.643470764160156,
      "activations/layer18_attention_weight_max": 48.242835998535156,
      "activations/layer18_attention_weight_min": -30.95134162902832,
      "activations/layer19_attention_weight_max": 23.59941291809082,
      "activations/layer19_attention_weight_min": -19.2159366607666,
      "activations/layer1_attention_weight_max": 17.709192276000977,
      "activations/layer1_attention_weight_min": -13.883371353149414,
      "activations/layer20_attention_weight_max": 25.117046356201172,
      "activations/layer20_attention_weight_min": -20.067018508911133,
      "activations/layer21_attention_weight_max": 40.879730224609375,
      "activations/layer21_attention_weight_min": -28.550737380981445,
      "activations/layer22_attention_weight_max": 43.71873092651367,
      "activations/layer22_attention_weight_min": -25.016817092895508,
      "activations/layer23_attention_weight_max": 28.369993209838867,
      "activations/layer23_attention_weight_min": -16.84233283996582,
      "activations/layer2_attention_weight_max": 30.230411529541016,
      "activations/layer2_attention_weight_min": -28.293113708496094,
      "activations/layer3_attention_weight_max": 76.93938446044922,
      "activations/layer3_attention_weight_min": -78.66287994384766,
      "activations/layer4_attention_weight_max": 106.34415435791016,
      "activations/layer4_attention_weight_min": -96.66488647460938,
      "activations/layer5_attention_weight_max": 122.584228515625,
      "activations/layer5_attention_weight_min": -102.06634521484375,
      "activations/layer6_attention_weight_max": 63.59931945800781,
      "activations/layer6_attention_weight_min": -53.51081466674805,
      "activations/layer7_attention_weight_max": 61.31119918823242,
      "activations/layer7_attention_weight_min": -59.40479278564453,
      "activations/layer8_attention_weight_max": 39.2274169921875,
      "activations/layer8_attention_weight_min": -37.337120056152344,
      "activations/layer9_attention_weight_max": 55.09444808959961,
      "activations/layer9_attention_weight_min": -49.916358947753906,
      "epoch": 3.17,
      "learning_rate": 0.00013087613636363635,
      "loss": 2.8974,
      "step": 54550
    },
    {
      "activations/layer0_attention_weight_max": 14.49425983428955,
      "activations/layer0_attention_weight_min": -14.762598991394043,
      "activations/layer10_attention_weight_max": 48.07866287231445,
      "activations/layer10_attention_weight_min": -36.396018981933594,
      "activations/layer11_attention_weight_max": 37.17970275878906,
      "activations/layer11_attention_weight_min": -28.741374969482422,
      "activations/layer12_attention_weight_max": 46.10565185546875,
      "activations/layer12_attention_weight_min": -27.245664596557617,
      "activations/layer13_attention_weight_max": 56.71417236328125,
      "activations/layer13_attention_weight_min": -37.51879119873047,
      "activations/layer14_attention_weight_max": 50.93485641479492,
      "activations/layer14_attention_weight_min": -32.01053237915039,
      "activations/layer15_attention_weight_max": 54.676055908203125,
      "activations/layer15_attention_weight_min": -31.862442016601562,
      "activations/layer16_attention_weight_max": 42.646053314208984,
      "activations/layer16_attention_weight_min": -26.47087860107422,
      "activations/layer17_attention_weight_max": 74.88770294189453,
      "activations/layer17_attention_weight_min": -39.942806243896484,
      "activations/layer18_attention_weight_max": 64.79779052734375,
      "activations/layer18_attention_weight_min": -35.188377380371094,
      "activations/layer19_attention_weight_max": 26.040393829345703,
      "activations/layer19_attention_weight_min": -16.923831939697266,
      "activations/layer1_attention_weight_max": 18.045866012573242,
      "activations/layer1_attention_weight_min": -14.533098220825195,
      "activations/layer20_attention_weight_max": 24.694196701049805,
      "activations/layer20_attention_weight_min": -21.969921112060547,
      "activations/layer21_attention_weight_max": 52.47087097167969,
      "activations/layer21_attention_weight_min": -26.087461471557617,
      "activations/layer22_attention_weight_max": 38.868316650390625,
      "activations/layer22_attention_weight_min": -25.486604690551758,
      "activations/layer23_attention_weight_max": 30.446561813354492,
      "activations/layer23_attention_weight_min": -16.064205169677734,
      "activations/layer2_attention_weight_max": 30.397192001342773,
      "activations/layer2_attention_weight_min": -28.709348678588867,
      "activations/layer3_attention_weight_max": 73.34793090820312,
      "activations/layer3_attention_weight_min": -75.8299789428711,
      "activations/layer4_attention_weight_max": 102.38705444335938,
      "activations/layer4_attention_weight_min": -92.0110092163086,
      "activations/layer5_attention_weight_max": 118.3345947265625,
      "activations/layer5_attention_weight_min": -99.03668212890625,
      "activations/layer6_attention_weight_max": 57.77094650268555,
      "activations/layer6_attention_weight_min": -50.133296966552734,
      "activations/layer7_attention_weight_max": 61.11207580566406,
      "activations/layer7_attention_weight_min": -55.41681671142578,
      "activations/layer8_attention_weight_max": 41.09608840942383,
      "activations/layer8_attention_weight_min": -35.31597137451172,
      "activations/layer9_attention_weight_max": 52.650245666503906,
      "activations/layer9_attention_weight_min": -49.15328598022461,
      "epoch": 3.17,
      "learning_rate": 0.00013085719696969697,
      "loss": 2.9059,
      "step": 54600
    },
    {
      "activations/layer0_attention_weight_max": 14.318770408630371,
      "activations/layer0_attention_weight_min": -14.862720489501953,
      "activations/layer10_attention_weight_max": 50.88045120239258,
      "activations/layer10_attention_weight_min": -38.856197357177734,
      "activations/layer11_attention_weight_max": 42.738033294677734,
      "activations/layer11_attention_weight_min": -29.00969696044922,
      "activations/layer12_attention_weight_max": 31.197919845581055,
      "activations/layer12_attention_weight_min": -23.75521469116211,
      "activations/layer13_attention_weight_max": 63.26197052001953,
      "activations/layer13_attention_weight_min": -37.32701110839844,
      "activations/layer14_attention_weight_max": 52.47162628173828,
      "activations/layer14_attention_weight_min": -29.97796630859375,
      "activations/layer15_attention_weight_max": 45.766300201416016,
      "activations/layer15_attention_weight_min": -30.698579788208008,
      "activations/layer16_attention_weight_max": 38.20828628540039,
      "activations/layer16_attention_weight_min": -26.68514060974121,
      "activations/layer17_attention_weight_max": 62.58240509033203,
      "activations/layer17_attention_weight_min": -34.6411018371582,
      "activations/layer18_attention_weight_max": 50.13325881958008,
      "activations/layer18_attention_weight_min": -32.60029602050781,
      "activations/layer19_attention_weight_max": 22.112712860107422,
      "activations/layer19_attention_weight_min": -21.196163177490234,
      "activations/layer1_attention_weight_max": 18.935447692871094,
      "activations/layer1_attention_weight_min": -13.692192077636719,
      "activations/layer20_attention_weight_max": 20.284915924072266,
      "activations/layer20_attention_weight_min": -20.253883361816406,
      "activations/layer21_attention_weight_max": 38.867713928222656,
      "activations/layer21_attention_weight_min": -27.406661987304688,
      "activations/layer22_attention_weight_max": 37.07036209106445,
      "activations/layer22_attention_weight_min": -21.49017333984375,
      "activations/layer23_attention_weight_max": 27.132291793823242,
      "activations/layer23_attention_weight_min": -15.334938049316406,
      "activations/layer2_attention_weight_max": 30.597301483154297,
      "activations/layer2_attention_weight_min": -30.289268493652344,
      "activations/layer3_attention_weight_max": 76.33743286132812,
      "activations/layer3_attention_weight_min": -82.3397445678711,
      "activations/layer4_attention_weight_max": 113.73908996582031,
      "activations/layer4_attention_weight_min": -102.80443572998047,
      "activations/layer5_attention_weight_max": 143.5491943359375,
      "activations/layer5_attention_weight_min": -114.64408874511719,
      "activations/layer6_attention_weight_max": 71.21668243408203,
      "activations/layer6_attention_weight_min": -56.27485275268555,
      "activations/layer7_attention_weight_max": 71.71812438964844,
      "activations/layer7_attention_weight_min": -58.522377014160156,
      "activations/layer8_attention_weight_max": 51.73138427734375,
      "activations/layer8_attention_weight_min": -39.681312561035156,
      "activations/layer9_attention_weight_max": 66.25129699707031,
      "activations/layer9_attention_weight_min": -54.04132080078125,
      "epoch": 3.18,
      "learning_rate": 0.00013083825757575755,
      "loss": 2.9053,
      "step": 54650
    },
    {
      "activations/layer0_attention_weight_max": 14.628741264343262,
      "activations/layer0_attention_weight_min": -15.321708679199219,
      "activations/layer10_attention_weight_max": 49.39829635620117,
      "activations/layer10_attention_weight_min": -38.87335968017578,
      "activations/layer11_attention_weight_max": 43.69017028808594,
      "activations/layer11_attention_weight_min": -28.192825317382812,
      "activations/layer12_attention_weight_max": 28.2775821685791,
      "activations/layer12_attention_weight_min": -24.966659545898438,
      "activations/layer13_attention_weight_max": 61.7935791015625,
      "activations/layer13_attention_weight_min": -41.070213317871094,
      "activations/layer14_attention_weight_max": 53.037845611572266,
      "activations/layer14_attention_weight_min": -33.17779541015625,
      "activations/layer15_attention_weight_max": 55.14912796020508,
      "activations/layer15_attention_weight_min": -37.693912506103516,
      "activations/layer16_attention_weight_max": 49.80551528930664,
      "activations/layer16_attention_weight_min": -27.678613662719727,
      "activations/layer17_attention_weight_max": 67.8522720336914,
      "activations/layer17_attention_weight_min": -34.87410354614258,
      "activations/layer18_attention_weight_max": 63.15734100341797,
      "activations/layer18_attention_weight_min": -34.7769889831543,
      "activations/layer19_attention_weight_max": 26.36225700378418,
      "activations/layer19_attention_weight_min": -21.010801315307617,
      "activations/layer1_attention_weight_max": 18.548463821411133,
      "activations/layer1_attention_weight_min": -13.62851333618164,
      "activations/layer20_attention_weight_max": 26.40471649169922,
      "activations/layer20_attention_weight_min": -20.218660354614258,
      "activations/layer21_attention_weight_max": 48.091819763183594,
      "activations/layer21_attention_weight_min": -28.2485408782959,
      "activations/layer22_attention_weight_max": 36.99911117553711,
      "activations/layer22_attention_weight_min": -24.14189338684082,
      "activations/layer23_attention_weight_max": 34.93977355957031,
      "activations/layer23_attention_weight_min": -19.47007179260254,
      "activations/layer2_attention_weight_max": 30.231077194213867,
      "activations/layer2_attention_weight_min": -28.92578887939453,
      "activations/layer3_attention_weight_max": 68.49699401855469,
      "activations/layer3_attention_weight_min": -75.10050201416016,
      "activations/layer4_attention_weight_max": 96.19670867919922,
      "activations/layer4_attention_weight_min": -89.59270477294922,
      "activations/layer5_attention_weight_max": 117.70623779296875,
      "activations/layer5_attention_weight_min": -100.51304626464844,
      "activations/layer6_attention_weight_max": 58.54655075073242,
      "activations/layer6_attention_weight_min": -51.67524337768555,
      "activations/layer7_attention_weight_max": 62.14624786376953,
      "activations/layer7_attention_weight_min": -57.59153747558594,
      "activations/layer8_attention_weight_max": 41.77176284790039,
      "activations/layer8_attention_weight_min": -36.40450668334961,
      "activations/layer9_attention_weight_max": 54.49222946166992,
      "activations/layer9_attention_weight_min": -50.170265197753906,
      "epoch": 3.18,
      "learning_rate": 0.00013081931818181817,
      "loss": 2.8871,
      "step": 54700
    },
    {
      "activations/layer0_attention_weight_max": 14.177194595336914,
      "activations/layer0_attention_weight_min": -14.132279396057129,
      "activations/layer10_attention_weight_max": 44.01102828979492,
      "activations/layer10_attention_weight_min": -36.457454681396484,
      "activations/layer11_attention_weight_max": 33.96250915527344,
      "activations/layer11_attention_weight_min": -27.994869232177734,
      "activations/layer12_attention_weight_max": 26.497798919677734,
      "activations/layer12_attention_weight_min": -24.62346076965332,
      "activations/layer13_attention_weight_max": 54.01908493041992,
      "activations/layer13_attention_weight_min": -41.26149368286133,
      "activations/layer14_attention_weight_max": 57.15956115722656,
      "activations/layer14_attention_weight_min": -40.45143127441406,
      "activations/layer15_attention_weight_max": 45.18474578857422,
      "activations/layer15_attention_weight_min": -33.39780044555664,
      "activations/layer16_attention_weight_max": 36.93184280395508,
      "activations/layer16_attention_weight_min": -28.093355178833008,
      "activations/layer17_attention_weight_max": 56.07858657836914,
      "activations/layer17_attention_weight_min": -36.339324951171875,
      "activations/layer18_attention_weight_max": 52.85276794433594,
      "activations/layer18_attention_weight_min": -31.62669563293457,
      "activations/layer19_attention_weight_max": 21.477460861206055,
      "activations/layer19_attention_weight_min": -17.85335350036621,
      "activations/layer1_attention_weight_max": 17.40025520324707,
      "activations/layer1_attention_weight_min": -14.316319465637207,
      "activations/layer20_attention_weight_max": 22.3110408782959,
      "activations/layer20_attention_weight_min": -20.72892951965332,
      "activations/layer21_attention_weight_max": 44.7021484375,
      "activations/layer21_attention_weight_min": -29.923892974853516,
      "activations/layer22_attention_weight_max": 34.93338394165039,
      "activations/layer22_attention_weight_min": -22.778833389282227,
      "activations/layer23_attention_weight_max": 24.760986328125,
      "activations/layer23_attention_weight_min": -15.27585506439209,
      "activations/layer2_attention_weight_max": 30.547687530517578,
      "activations/layer2_attention_weight_min": -27.743457794189453,
      "activations/layer3_attention_weight_max": 76.40959167480469,
      "activations/layer3_attention_weight_min": -80.6532974243164,
      "activations/layer4_attention_weight_max": 99.79072570800781,
      "activations/layer4_attention_weight_min": -94.75436401367188,
      "activations/layer5_attention_weight_max": 117.67659759521484,
      "activations/layer5_attention_weight_min": -98.69949340820312,
      "activations/layer6_attention_weight_max": 57.97270584106445,
      "activations/layer6_attention_weight_min": -50.6151237487793,
      "activations/layer7_attention_weight_max": 56.71269607543945,
      "activations/layer7_attention_weight_min": -55.95533752441406,
      "activations/layer8_attention_weight_max": 37.19063186645508,
      "activations/layer8_attention_weight_min": -36.25813674926758,
      "activations/layer9_attention_weight_max": 52.901641845703125,
      "activations/layer9_attention_weight_min": -47.91547775268555,
      "epoch": 3.18,
      "learning_rate": 0.00013080037878787878,
      "loss": 2.9016,
      "step": 54750
    },
    {
      "activations/layer0_attention_weight_max": 15.854717254638672,
      "activations/layer0_attention_weight_min": -14.429418563842773,
      "activations/layer10_attention_weight_max": 42.35529327392578,
      "activations/layer10_attention_weight_min": -33.76318359375,
      "activations/layer11_attention_weight_max": 37.18183898925781,
      "activations/layer11_attention_weight_min": -28.31191635131836,
      "activations/layer12_attention_weight_max": 32.683021545410156,
      "activations/layer12_attention_weight_min": -22.957061767578125,
      "activations/layer13_attention_weight_max": 58.324790954589844,
      "activations/layer13_attention_weight_min": -39.27203369140625,
      "activations/layer14_attention_weight_max": 53.99598693847656,
      "activations/layer14_attention_weight_min": -36.84794998168945,
      "activations/layer15_attention_weight_max": 52.24985885620117,
      "activations/layer15_attention_weight_min": -35.90949630737305,
      "activations/layer16_attention_weight_max": 41.8022575378418,
      "activations/layer16_attention_weight_min": -27.5142879486084,
      "activations/layer17_attention_weight_max": 60.18364334106445,
      "activations/layer17_attention_weight_min": -40.219390869140625,
      "activations/layer18_attention_weight_max": 56.711181640625,
      "activations/layer18_attention_weight_min": -33.48516082763672,
      "activations/layer19_attention_weight_max": 26.76247215270996,
      "activations/layer19_attention_weight_min": -16.520437240600586,
      "activations/layer1_attention_weight_max": 17.959556579589844,
      "activations/layer1_attention_weight_min": -14.702075958251953,
      "activations/layer20_attention_weight_max": 21.79757308959961,
      "activations/layer20_attention_weight_min": -16.582439422607422,
      "activations/layer21_attention_weight_max": 41.858028411865234,
      "activations/layer21_attention_weight_min": -26.05539894104004,
      "activations/layer22_attention_weight_max": 43.67991638183594,
      "activations/layer22_attention_weight_min": -23.17656135559082,
      "activations/layer23_attention_weight_max": 29.82301139831543,
      "activations/layer23_attention_weight_min": -15.63909912109375,
      "activations/layer2_attention_weight_max": 29.784740447998047,
      "activations/layer2_attention_weight_min": -27.92605209350586,
      "activations/layer3_attention_weight_max": 74.8858871459961,
      "activations/layer3_attention_weight_min": -79.04085540771484,
      "activations/layer4_attention_weight_max": 99.1880874633789,
      "activations/layer4_attention_weight_min": -94.6557846069336,
      "activations/layer5_attention_weight_max": 119.88972473144531,
      "activations/layer5_attention_weight_min": -100.80999755859375,
      "activations/layer6_attention_weight_max": 60.649356842041016,
      "activations/layer6_attention_weight_min": -50.45248031616211,
      "activations/layer7_attention_weight_max": 64.00239562988281,
      "activations/layer7_attention_weight_min": -56.54062271118164,
      "activations/layer8_attention_weight_max": 40.4688720703125,
      "activations/layer8_attention_weight_min": -35.5572395324707,
      "activations/layer9_attention_weight_max": 55.41128158569336,
      "activations/layer9_attention_weight_min": -49.59326171875,
      "epoch": 3.18,
      "learning_rate": 0.00013078143939393937,
      "loss": 2.889,
      "step": 54800
    },
    {
      "activations/layer0_attention_weight_max": 14.59665584564209,
      "activations/layer0_attention_weight_min": -14.532575607299805,
      "activations/layer10_attention_weight_max": 44.30845260620117,
      "activations/layer10_attention_weight_min": -35.6434211730957,
      "activations/layer11_attention_weight_max": 37.68201446533203,
      "activations/layer11_attention_weight_min": -26.989303588867188,
      "activations/layer12_attention_weight_max": 24.9100341796875,
      "activations/layer12_attention_weight_min": -29.740171432495117,
      "activations/layer13_attention_weight_max": 69.38448333740234,
      "activations/layer13_attention_weight_min": -51.59370040893555,
      "activations/layer14_attention_weight_max": 57.59683609008789,
      "activations/layer14_attention_weight_min": -43.30806350708008,
      "activations/layer15_attention_weight_max": 50.420928955078125,
      "activations/layer15_attention_weight_min": -37.52867126464844,
      "activations/layer16_attention_weight_max": 37.9118766784668,
      "activations/layer16_attention_weight_min": -26.033863067626953,
      "activations/layer17_attention_weight_max": 58.30396270751953,
      "activations/layer17_attention_weight_min": -35.750308990478516,
      "activations/layer18_attention_weight_max": 49.53274917602539,
      "activations/layer18_attention_weight_min": -31.07649040222168,
      "activations/layer19_attention_weight_max": 22.030515670776367,
      "activations/layer19_attention_weight_min": -18.17961883544922,
      "activations/layer1_attention_weight_max": 17.99057960510254,
      "activations/layer1_attention_weight_min": -15.47216510772705,
      "activations/layer20_attention_weight_max": 22.428014755249023,
      "activations/layer20_attention_weight_min": -19.486797332763672,
      "activations/layer21_attention_weight_max": 39.8131103515625,
      "activations/layer21_attention_weight_min": -26.996337890625,
      "activations/layer22_attention_weight_max": 35.25263214111328,
      "activations/layer22_attention_weight_min": -21.272415161132812,
      "activations/layer23_attention_weight_max": 26.256664276123047,
      "activations/layer23_attention_weight_min": -15.842825889587402,
      "activations/layer2_attention_weight_max": 31.050636291503906,
      "activations/layer2_attention_weight_min": -28.391494750976562,
      "activations/layer3_attention_weight_max": 73.02935791015625,
      "activations/layer3_attention_weight_min": -74.69953918457031,
      "activations/layer4_attention_weight_max": 100.4332046508789,
      "activations/layer4_attention_weight_min": -96.88960266113281,
      "activations/layer5_attention_weight_max": 120.33197021484375,
      "activations/layer5_attention_weight_min": -109.6175537109375,
      "activations/layer6_attention_weight_max": 60.70555877685547,
      "activations/layer6_attention_weight_min": -51.14243698120117,
      "activations/layer7_attention_weight_max": 59.96865463256836,
      "activations/layer7_attention_weight_min": -59.326663970947266,
      "activations/layer8_attention_weight_max": 39.23992919921875,
      "activations/layer8_attention_weight_min": -39.86796951293945,
      "activations/layer9_attention_weight_max": 53.84717559814453,
      "activations/layer9_attention_weight_min": -50.51413345336914,
      "epoch": 3.19,
      "learning_rate": 0.0001307625,
      "loss": 2.9059,
      "step": 54850
    },
    {
      "activations/layer0_attention_weight_max": 14.183350563049316,
      "activations/layer0_attention_weight_min": -14.757761001586914,
      "activations/layer10_attention_weight_max": 43.61375045776367,
      "activations/layer10_attention_weight_min": -34.26328659057617,
      "activations/layer11_attention_weight_max": 37.48350143432617,
      "activations/layer11_attention_weight_min": -26.14729118347168,
      "activations/layer12_attention_weight_max": 26.585041046142578,
      "activations/layer12_attention_weight_min": -22.647184371948242,
      "activations/layer13_attention_weight_max": 50.64073181152344,
      "activations/layer13_attention_weight_min": -38.24250793457031,
      "activations/layer14_attention_weight_max": 48.675025939941406,
      "activations/layer14_attention_weight_min": -33.552154541015625,
      "activations/layer15_attention_weight_max": 48.51331329345703,
      "activations/layer15_attention_weight_min": -33.4023323059082,
      "activations/layer16_attention_weight_max": 42.959712982177734,
      "activations/layer16_attention_weight_min": -27.644878387451172,
      "activations/layer17_attention_weight_max": 58.9904670715332,
      "activations/layer17_attention_weight_min": -35.327423095703125,
      "activations/layer18_attention_weight_max": 57.53230285644531,
      "activations/layer18_attention_weight_min": -32.83580780029297,
      "activations/layer19_attention_weight_max": 24.37269401550293,
      "activations/layer19_attention_weight_min": -19.51227378845215,
      "activations/layer1_attention_weight_max": 18.334369659423828,
      "activations/layer1_attention_weight_min": -14.140443801879883,
      "activations/layer20_attention_weight_max": 23.218706130981445,
      "activations/layer20_attention_weight_min": -20.004276275634766,
      "activations/layer21_attention_weight_max": 45.01189041137695,
      "activations/layer21_attention_weight_min": -26.603811264038086,
      "activations/layer22_attention_weight_max": 38.99415588378906,
      "activations/layer22_attention_weight_min": -24.376079559326172,
      "activations/layer23_attention_weight_max": 24.696495056152344,
      "activations/layer23_attention_weight_min": -17.14693260192871,
      "activations/layer2_attention_weight_max": 27.093109130859375,
      "activations/layer2_attention_weight_min": -26.04525375366211,
      "activations/layer3_attention_weight_max": 67.8740463256836,
      "activations/layer3_attention_weight_min": -74.3925552368164,
      "activations/layer4_attention_weight_max": 95.84982299804688,
      "activations/layer4_attention_weight_min": -95.27510070800781,
      "activations/layer5_attention_weight_max": 112.20582580566406,
      "activations/layer5_attention_weight_min": -98.73774719238281,
      "activations/layer6_attention_weight_max": 56.5325813293457,
      "activations/layer6_attention_weight_min": -50.62895202636719,
      "activations/layer7_attention_weight_max": 57.788002014160156,
      "activations/layer7_attention_weight_min": -54.67437744140625,
      "activations/layer8_attention_weight_max": 37.11381149291992,
      "activations/layer8_attention_weight_min": -34.00060272216797,
      "activations/layer9_attention_weight_max": 46.9564208984375,
      "activations/layer9_attention_weight_min": -46.235233306884766,
      "epoch": 3.19,
      "learning_rate": 0.00013074393939393938,
      "loss": 2.8959,
      "step": 54900
    },
    {
      "activations/layer0_attention_weight_max": 15.244233131408691,
      "activations/layer0_attention_weight_min": -14.736013412475586,
      "activations/layer10_attention_weight_max": 45.63452911376953,
      "activations/layer10_attention_weight_min": -34.82747268676758,
      "activations/layer11_attention_weight_max": 37.59837341308594,
      "activations/layer11_attention_weight_min": -27.57198143005371,
      "activations/layer12_attention_weight_max": 28.485322952270508,
      "activations/layer12_attention_weight_min": -26.247318267822266,
      "activations/layer13_attention_weight_max": 55.15638732910156,
      "activations/layer13_attention_weight_min": -38.36329650878906,
      "activations/layer14_attention_weight_max": 50.3640022277832,
      "activations/layer14_attention_weight_min": -33.630165100097656,
      "activations/layer15_attention_weight_max": 48.89839172363281,
      "activations/layer15_attention_weight_min": -34.11904525756836,
      "activations/layer16_attention_weight_max": 41.700164794921875,
      "activations/layer16_attention_weight_min": -27.564899444580078,
      "activations/layer17_attention_weight_max": 59.20396041870117,
      "activations/layer17_attention_weight_min": -37.10519790649414,
      "activations/layer18_attention_weight_max": 49.552303314208984,
      "activations/layer18_attention_weight_min": -33.894195556640625,
      "activations/layer19_attention_weight_max": 21.5031795501709,
      "activations/layer19_attention_weight_min": -16.814504623413086,
      "activations/layer1_attention_weight_max": 17.557376861572266,
      "activations/layer1_attention_weight_min": -12.661019325256348,
      "activations/layer20_attention_weight_max": 19.4119873046875,
      "activations/layer20_attention_weight_min": -19.115062713623047,
      "activations/layer21_attention_weight_max": 42.09505081176758,
      "activations/layer21_attention_weight_min": -28.41830825805664,
      "activations/layer22_attention_weight_max": 34.51232147216797,
      "activations/layer22_attention_weight_min": -23.937397003173828,
      "activations/layer23_attention_weight_max": 24.022436141967773,
      "activations/layer23_attention_weight_min": -16.719350814819336,
      "activations/layer2_attention_weight_max": 30.15326690673828,
      "activations/layer2_attention_weight_min": -28.21082305908203,
      "activations/layer3_attention_weight_max": 78.03521728515625,
      "activations/layer3_attention_weight_min": -79.1376724243164,
      "activations/layer4_attention_weight_max": 97.69085693359375,
      "activations/layer4_attention_weight_min": -93.66246032714844,
      "activations/layer5_attention_weight_max": 121.38734436035156,
      "activations/layer5_attention_weight_min": -99.10038757324219,
      "activations/layer6_attention_weight_max": 58.99495315551758,
      "activations/layer6_attention_weight_min": -49.31440734863281,
      "activations/layer7_attention_weight_max": 64.3919906616211,
      "activations/layer7_attention_weight_min": -64.29052734375,
      "activations/layer8_attention_weight_max": 39.50238037109375,
      "activations/layer8_attention_weight_min": -35.97053909301758,
      "activations/layer9_attention_weight_max": 57.366573333740234,
      "activations/layer9_attention_weight_min": -49.061824798583984,
      "epoch": 3.19,
      "learning_rate": 0.000130725,
      "loss": 2.8986,
      "step": 54950
    },
    {
      "activations/layer0_attention_weight_max": 16.27768325805664,
      "activations/layer0_attention_weight_min": -14.562743186950684,
      "activations/layer10_attention_weight_max": 45.441925048828125,
      "activations/layer10_attention_weight_min": -36.29609680175781,
      "activations/layer11_attention_weight_max": 36.88109588623047,
      "activations/layer11_attention_weight_min": -26.77039909362793,
      "activations/layer12_attention_weight_max": 29.342599868774414,
      "activations/layer12_attention_weight_min": -26.65532112121582,
      "activations/layer13_attention_weight_max": 45.68333053588867,
      "activations/layer13_attention_weight_min": -34.626495361328125,
      "activations/layer14_attention_weight_max": 41.82984161376953,
      "activations/layer14_attention_weight_min": -31.749958038330078,
      "activations/layer15_attention_weight_max": 44.41808319091797,
      "activations/layer15_attention_weight_min": -31.781291961669922,
      "activations/layer16_attention_weight_max": 35.755611419677734,
      "activations/layer16_attention_weight_min": -25.875396728515625,
      "activations/layer17_attention_weight_max": 59.75809860229492,
      "activations/layer17_attention_weight_min": -35.16086959838867,
      "activations/layer18_attention_weight_max": 53.78495407104492,
      "activations/layer18_attention_weight_min": -31.63149642944336,
      "activations/layer19_attention_weight_max": 23.845783233642578,
      "activations/layer19_attention_weight_min": -18.4743595123291,
      "activations/layer1_attention_weight_max": 17.327259063720703,
      "activations/layer1_attention_weight_min": -15.809907913208008,
      "activations/layer20_attention_weight_max": 21.321626663208008,
      "activations/layer20_attention_weight_min": -19.43498992919922,
      "activations/layer21_attention_weight_max": 43.448062896728516,
      "activations/layer21_attention_weight_min": -26.58414649963379,
      "activations/layer22_attention_weight_max": 35.05881118774414,
      "activations/layer22_attention_weight_min": -22.185462951660156,
      "activations/layer23_attention_weight_max": 25.241567611694336,
      "activations/layer23_attention_weight_min": -19.03666114807129,
      "activations/layer2_attention_weight_max": 29.637866973876953,
      "activations/layer2_attention_weight_min": -27.60477066040039,
      "activations/layer3_attention_weight_max": 72.33905029296875,
      "activations/layer3_attention_weight_min": -75.72775268554688,
      "activations/layer4_attention_weight_max": 100.60594940185547,
      "activations/layer4_attention_weight_min": -93.38274383544922,
      "activations/layer5_attention_weight_max": 113.25556945800781,
      "activations/layer5_attention_weight_min": -96.40914154052734,
      "activations/layer6_attention_weight_max": 60.43716812133789,
      "activations/layer6_attention_weight_min": -52.64274215698242,
      "activations/layer7_attention_weight_max": 60.9348030090332,
      "activations/layer7_attention_weight_min": -59.29636764526367,
      "activations/layer8_attention_weight_max": 37.02968215942383,
      "activations/layer8_attention_weight_min": -35.49370574951172,
      "activations/layer9_attention_weight_max": 58.522518157958984,
      "activations/layer9_attention_weight_min": -47.49178695678711,
      "epoch": 3.2,
      "learning_rate": 0.00013070606060606058,
      "loss": 2.8952,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_loss": 2.849609375,
      "eval_runtime": 8.4922,
      "eval_samples_per_second": 505.643,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_openwebtext_loss": 2.849609375,
      "eval_openwebtext_ppl": 17.28103011956753,
      "eval_openwebtext_runtime": 8.4922,
      "eval_openwebtext_samples_per_second": 505.643,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_wikitext_loss": 3.09375,
      "eval_wikitext_ppl": 22.059646737841184,
      "eval_wikitext_runtime": 1.9212,
      "eval_wikitext_samples_per_second": 237.347,
      "step": 55000
    },
    {
      "epoch": 3.2,
      "eval_lambada_loss": 2.921875,
      "eval_lambada_ppl": 18.576084986220376,
      "eval_lambada_runtime": 9.5449,
      "eval_lambada_samples_per_second": 510.113,
      "step": 55000
    },
    {
      "activations/layer0_attention_weight_max": 14.125502586364746,
      "activations/layer0_attention_weight_min": -14.904568672180176,
      "activations/layer10_attention_weight_max": 41.4654541015625,
      "activations/layer10_attention_weight_min": -34.62430191040039,
      "activations/layer11_attention_weight_max": 36.32368469238281,
      "activations/layer11_attention_weight_min": -26.964725494384766,
      "activations/layer12_attention_weight_max": 25.1706600189209,
      "activations/layer12_attention_weight_min": -24.2574405670166,
      "activations/layer13_attention_weight_max": 53.57089614868164,
      "activations/layer13_attention_weight_min": -39.56145477294922,
      "activations/layer14_attention_weight_max": 46.09868621826172,
      "activations/layer14_attention_weight_min": -29.850568771362305,
      "activations/layer15_attention_weight_max": 55.17924499511719,
      "activations/layer15_attention_weight_min": -31.786420822143555,
      "activations/layer16_attention_weight_max": 37.56416320800781,
      "activations/layer16_attention_weight_min": -26.3170108795166,
      "activations/layer17_attention_weight_max": 58.8952522277832,
      "activations/layer17_attention_weight_min": -35.451171875,
      "activations/layer18_attention_weight_max": 54.53310012817383,
      "activations/layer18_attention_weight_min": -31.97738265991211,
      "activations/layer19_attention_weight_max": 21.286212921142578,
      "activations/layer19_attention_weight_min": -19.535263061523438,
      "activations/layer1_attention_weight_max": 17.507631301879883,
      "activations/layer1_attention_weight_min": -15.822829246520996,
      "activations/layer20_attention_weight_max": 21.2025146484375,
      "activations/layer20_attention_weight_min": -19.00131607055664,
      "activations/layer21_attention_weight_max": 42.610748291015625,
      "activations/layer21_attention_weight_min": -29.004638671875,
      "activations/layer22_attention_weight_max": 35.17721176147461,
      "activations/layer22_attention_weight_min": -23.216672897338867,
      "activations/layer23_attention_weight_max": 25.495885848999023,
      "activations/layer23_attention_weight_min": -16.198993682861328,
      "activations/layer2_attention_weight_max": 28.554723739624023,
      "activations/layer2_attention_weight_min": -27.106130599975586,
      "activations/layer3_attention_weight_max": 70.3218002319336,
      "activations/layer3_attention_weight_min": -75.3311767578125,
      "activations/layer4_attention_weight_max": 99.64842224121094,
      "activations/layer4_attention_weight_min": -88.91162872314453,
      "activations/layer5_attention_weight_max": 118.45869445800781,
      "activations/layer5_attention_weight_min": -97.87806701660156,
      "activations/layer6_attention_weight_max": 60.672245025634766,
      "activations/layer6_attention_weight_min": -50.386234283447266,
      "activations/layer7_attention_weight_max": 57.271671295166016,
      "activations/layer7_attention_weight_min": -57.043636322021484,
      "activations/layer8_attention_weight_max": 38.911651611328125,
      "activations/layer8_attention_weight_min": -36.207542419433594,
      "activations/layer9_attention_weight_max": 48.04798889160156,
      "activations/layer9_attention_weight_min": -45.3687629699707,
      "epoch": 3.2,
      "learning_rate": 0.0001306871212121212,
      "loss": 2.8844,
      "step": 55050
    },
    {
      "activations/layer0_attention_weight_max": 13.828744888305664,
      "activations/layer0_attention_weight_min": -14.615774154663086,
      "activations/layer10_attention_weight_max": 45.948368072509766,
      "activations/layer10_attention_weight_min": -34.9615364074707,
      "activations/layer11_attention_weight_max": 37.78765106201172,
      "activations/layer11_attention_weight_min": -26.50505828857422,
      "activations/layer12_attention_weight_max": 24.761844635009766,
      "activations/layer12_attention_weight_min": -25.89841651916504,
      "activations/layer13_attention_weight_max": 60.8466911315918,
      "activations/layer13_attention_weight_min": -39.75617980957031,
      "activations/layer14_attention_weight_max": 50.67792892456055,
      "activations/layer14_attention_weight_min": -31.274003982543945,
      "activations/layer15_attention_weight_max": 56.53944778442383,
      "activations/layer15_attention_weight_min": -35.875553131103516,
      "activations/layer16_attention_weight_max": 38.54750061035156,
      "activations/layer16_attention_weight_min": -28.0411376953125,
      "activations/layer17_attention_weight_max": 68.00471496582031,
      "activations/layer17_attention_weight_min": -39.72203063964844,
      "activations/layer18_attention_weight_max": 55.246543884277344,
      "activations/layer18_attention_weight_min": -33.23114776611328,
      "activations/layer19_attention_weight_max": 23.611906051635742,
      "activations/layer19_attention_weight_min": -17.658321380615234,
      "activations/layer1_attention_weight_max": 17.772672653198242,
      "activations/layer1_attention_weight_min": -14.63359260559082,
      "activations/layer20_attention_weight_max": 20.814781188964844,
      "activations/layer20_attention_weight_min": -21.2276611328125,
      "activations/layer21_attention_weight_max": 43.57841110229492,
      "activations/layer21_attention_weight_min": -27.61815643310547,
      "activations/layer22_attention_weight_max": 32.222511291503906,
      "activations/layer22_attention_weight_min": -24.263994216918945,
      "activations/layer23_attention_weight_max": 27.520870208740234,
      "activations/layer23_attention_weight_min": -17.523292541503906,
      "activations/layer2_attention_weight_max": 30.31488609313965,
      "activations/layer2_attention_weight_min": -28.091352462768555,
      "activations/layer3_attention_weight_max": 76.00645446777344,
      "activations/layer3_attention_weight_min": -82.78466033935547,
      "activations/layer4_attention_weight_max": 105.05323791503906,
      "activations/layer4_attention_weight_min": -98.72124481201172,
      "activations/layer5_attention_weight_max": 122.93931579589844,
      "activations/layer5_attention_weight_min": -109.36168670654297,
      "activations/layer6_attention_weight_max": 62.9785041809082,
      "activations/layer6_attention_weight_min": -54.12240219116211,
      "activations/layer7_attention_weight_max": 60.948665618896484,
      "activations/layer7_attention_weight_min": -60.67963790893555,
      "activations/layer8_attention_weight_max": 43.986515045166016,
      "activations/layer8_attention_weight_min": -38.385589599609375,
      "activations/layer9_attention_weight_max": 56.113258361816406,
      "activations/layer9_attention_weight_min": -48.5200309753418,
      "epoch": 3.2,
      "learning_rate": 0.0001306681818181818,
      "loss": 2.8714,
      "step": 55100
    },
    {
      "activations/layer0_attention_weight_max": 14.328997611999512,
      "activations/layer0_attention_weight_min": -14.337982177734375,
      "activations/layer10_attention_weight_max": 42.03342819213867,
      "activations/layer10_attention_weight_min": -35.09649658203125,
      "activations/layer11_attention_weight_max": 36.616844177246094,
      "activations/layer11_attention_weight_min": -26.288557052612305,
      "activations/layer12_attention_weight_max": 28.242067337036133,
      "activations/layer12_attention_weight_min": -24.302465438842773,
      "activations/layer13_attention_weight_max": 54.49699401855469,
      "activations/layer13_attention_weight_min": -39.730072021484375,
      "activations/layer14_attention_weight_max": 48.94956970214844,
      "activations/layer14_attention_weight_min": -39.390750885009766,
      "activations/layer15_attention_weight_max": 47.02517318725586,
      "activations/layer15_attention_weight_min": -34.3861198425293,
      "activations/layer16_attention_weight_max": 38.40870666503906,
      "activations/layer16_attention_weight_min": -26.664854049682617,
      "activations/layer17_attention_weight_max": 58.81620407104492,
      "activations/layer17_attention_weight_min": -36.14693832397461,
      "activations/layer18_attention_weight_max": 57.5086669921875,
      "activations/layer18_attention_weight_min": -32.8455924987793,
      "activations/layer19_attention_weight_max": 21.484102249145508,
      "activations/layer19_attention_weight_min": -16.507165908813477,
      "activations/layer1_attention_weight_max": 17.942703247070312,
      "activations/layer1_attention_weight_min": -15.52335262298584,
      "activations/layer20_attention_weight_max": 22.749778747558594,
      "activations/layer20_attention_weight_min": -19.431657791137695,
      "activations/layer21_attention_weight_max": 46.90573501586914,
      "activations/layer21_attention_weight_min": -30.1435546875,
      "activations/layer22_attention_weight_max": 35.68464660644531,
      "activations/layer22_attention_weight_min": -24.721282958984375,
      "activations/layer23_attention_weight_max": 30.464786529541016,
      "activations/layer23_attention_weight_min": -17.75795555114746,
      "activations/layer2_attention_weight_max": 30.210960388183594,
      "activations/layer2_attention_weight_min": -27.341421127319336,
      "activations/layer3_attention_weight_max": 73.3330307006836,
      "activations/layer3_attention_weight_min": -75.8805923461914,
      "activations/layer4_attention_weight_max": 97.57906341552734,
      "activations/layer4_attention_weight_min": -89.83318328857422,
      "activations/layer5_attention_weight_max": 118.2805404663086,
      "activations/layer5_attention_weight_min": -99.83415222167969,
      "activations/layer6_attention_weight_max": 58.62175369262695,
      "activations/layer6_attention_weight_min": -49.27696990966797,
      "activations/layer7_attention_weight_max": 59.48236846923828,
      "activations/layer7_attention_weight_min": -57.92259979248047,
      "activations/layer8_attention_weight_max": 38.72695541381836,
      "activations/layer8_attention_weight_min": -36.87392807006836,
      "activations/layer9_attention_weight_max": 51.04676818847656,
      "activations/layer9_attention_weight_min": -47.59184646606445,
      "epoch": 3.2,
      "learning_rate": 0.0001306492424242424,
      "loss": 2.8768,
      "step": 55150
    },
    {
      "activations/layer0_attention_weight_max": 14.249027252197266,
      "activations/layer0_attention_weight_min": -15.121041297912598,
      "activations/layer10_attention_weight_max": 51.53629684448242,
      "activations/layer10_attention_weight_min": -37.29829788208008,
      "activations/layer11_attention_weight_max": 40.68737030029297,
      "activations/layer11_attention_weight_min": -29.929622650146484,
      "activations/layer12_attention_weight_max": 26.38770866394043,
      "activations/layer12_attention_weight_min": -23.328392028808594,
      "activations/layer13_attention_weight_max": 63.2973747253418,
      "activations/layer13_attention_weight_min": -41.89863586425781,
      "activations/layer14_attention_weight_max": 53.057640075683594,
      "activations/layer14_attention_weight_min": -33.60704040527344,
      "activations/layer15_attention_weight_max": 58.05617904663086,
      "activations/layer15_attention_weight_min": -33.5645751953125,
      "activations/layer16_attention_weight_max": 42.383182525634766,
      "activations/layer16_attention_weight_min": -27.429136276245117,
      "activations/layer17_attention_weight_max": 63.84946823120117,
      "activations/layer17_attention_weight_min": -39.09531784057617,
      "activations/layer18_attention_weight_max": 54.9412727355957,
      "activations/layer18_attention_weight_min": -32.93611526489258,
      "activations/layer19_attention_weight_max": 22.078079223632812,
      "activations/layer19_attention_weight_min": -19.340412139892578,
      "activations/layer1_attention_weight_max": 18.537118911743164,
      "activations/layer1_attention_weight_min": -15.117429733276367,
      "activations/layer20_attention_weight_max": 23.59360694885254,
      "activations/layer20_attention_weight_min": -20.030790328979492,
      "activations/layer21_attention_weight_max": 47.02032470703125,
      "activations/layer21_attention_weight_min": -28.093908309936523,
      "activations/layer22_attention_weight_max": 37.89628982543945,
      "activations/layer22_attention_weight_min": -23.97547721862793,
      "activations/layer23_attention_weight_max": 28.073871612548828,
      "activations/layer23_attention_weight_min": -16.890274047851562,
      "activations/layer2_attention_weight_max": 29.474010467529297,
      "activations/layer2_attention_weight_min": -28.752166748046875,
      "activations/layer3_attention_weight_max": 73.02831268310547,
      "activations/layer3_attention_weight_min": -78.66775512695312,
      "activations/layer4_attention_weight_max": 100.65384674072266,
      "activations/layer4_attention_weight_min": -94.44721221923828,
      "activations/layer5_attention_weight_max": 119.98806762695312,
      "activations/layer5_attention_weight_min": -100.64479064941406,
      "activations/layer6_attention_weight_max": 61.91571807861328,
      "activations/layer6_attention_weight_min": -50.39020919799805,
      "activations/layer7_attention_weight_max": 64.49703979492188,
      "activations/layer7_attention_weight_min": -63.87526321411133,
      "activations/layer8_attention_weight_max": 40.46305847167969,
      "activations/layer8_attention_weight_min": -41.04374694824219,
      "activations/layer9_attention_weight_max": 52.948455810546875,
      "activations/layer9_attention_weight_min": -48.5313720703125,
      "epoch": 3.21,
      "learning_rate": 0.00013063030303030301,
      "loss": 2.8646,
      "step": 55200
    },
    {
      "activations/layer0_attention_weight_max": 14.830968856811523,
      "activations/layer0_attention_weight_min": -15.043685913085938,
      "activations/layer10_attention_weight_max": 46.03913879394531,
      "activations/layer10_attention_weight_min": -35.827125549316406,
      "activations/layer11_attention_weight_max": 38.626651763916016,
      "activations/layer11_attention_weight_min": -27.176761627197266,
      "activations/layer12_attention_weight_max": 33.62458038330078,
      "activations/layer12_attention_weight_min": -24.11876678466797,
      "activations/layer13_attention_weight_max": 67.66565704345703,
      "activations/layer13_attention_weight_min": -43.22683334350586,
      "activations/layer14_attention_weight_max": 76.2070083618164,
      "activations/layer14_attention_weight_min": -38.404014587402344,
      "activations/layer15_attention_weight_max": 53.156150817871094,
      "activations/layer15_attention_weight_min": -32.00593566894531,
      "activations/layer16_attention_weight_max": 44.229583740234375,
      "activations/layer16_attention_weight_min": -28.94011116027832,
      "activations/layer17_attention_weight_max": 71.73191833496094,
      "activations/layer17_attention_weight_min": -38.065738677978516,
      "activations/layer18_attention_weight_max": 59.626060485839844,
      "activations/layer18_attention_weight_min": -32.95064163208008,
      "activations/layer19_attention_weight_max": 26.171232223510742,
      "activations/layer19_attention_weight_min": -19.420150756835938,
      "activations/layer1_attention_weight_max": 19.4218692779541,
      "activations/layer1_attention_weight_min": -13.079949378967285,
      "activations/layer20_attention_weight_max": 26.984195709228516,
      "activations/layer20_attention_weight_min": -18.618350982666016,
      "activations/layer21_attention_weight_max": 47.55869674682617,
      "activations/layer21_attention_weight_min": -28.336299896240234,
      "activations/layer22_attention_weight_max": 43.5791015625,
      "activations/layer22_attention_weight_min": -21.978792190551758,
      "activations/layer23_attention_weight_max": 29.5609130859375,
      "activations/layer23_attention_weight_min": -18.199234008789062,
      "activations/layer2_attention_weight_max": 29.45453643798828,
      "activations/layer2_attention_weight_min": -28.15581512451172,
      "activations/layer3_attention_weight_max": 73.88560485839844,
      "activations/layer3_attention_weight_min": -79.34031677246094,
      "activations/layer4_attention_weight_max": 109.3772964477539,
      "activations/layer4_attention_weight_min": -98.1545181274414,
      "activations/layer5_attention_weight_max": 131.218994140625,
      "activations/layer5_attention_weight_min": -103.26439666748047,
      "activations/layer6_attention_weight_max": 61.771018981933594,
      "activations/layer6_attention_weight_min": -51.15719223022461,
      "activations/layer7_attention_weight_max": 66.01089477539062,
      "activations/layer7_attention_weight_min": -59.19334411621094,
      "activations/layer8_attention_weight_max": 45.72324752807617,
      "activations/layer8_attention_weight_min": -37.70412063598633,
      "activations/layer9_attention_weight_max": 60.6759033203125,
      "activations/layer9_attention_weight_min": -52.608680725097656,
      "epoch": 3.21,
      "learning_rate": 0.00013061136363636363,
      "loss": 2.8885,
      "step": 55250
    },
    {
      "activations/layer0_attention_weight_max": 13.837178230285645,
      "activations/layer0_attention_weight_min": -14.967686653137207,
      "activations/layer10_attention_weight_max": 43.24436569213867,
      "activations/layer10_attention_weight_min": -35.73175048828125,
      "activations/layer11_attention_weight_max": 33.2103385925293,
      "activations/layer11_attention_weight_min": -26.065502166748047,
      "activations/layer12_attention_weight_max": 26.193851470947266,
      "activations/layer12_attention_weight_min": -22.57244110107422,
      "activations/layer13_attention_weight_max": 61.610084533691406,
      "activations/layer13_attention_weight_min": -44.28987121582031,
      "activations/layer14_attention_weight_max": 58.8837776184082,
      "activations/layer14_attention_weight_min": -41.917633056640625,
      "activations/layer15_attention_weight_max": 47.98660659790039,
      "activations/layer15_attention_weight_min": -32.385066986083984,
      "activations/layer16_attention_weight_max": 39.21075439453125,
      "activations/layer16_attention_weight_min": -27.281845092773438,
      "activations/layer17_attention_weight_max": 58.25539016723633,
      "activations/layer17_attention_weight_min": -39.45807647705078,
      "activations/layer18_attention_weight_max": 52.82429122924805,
      "activations/layer18_attention_weight_min": -33.689483642578125,
      "activations/layer19_attention_weight_max": 22.3584041595459,
      "activations/layer19_attention_weight_min": -17.967145919799805,
      "activations/layer1_attention_weight_max": 16.982261657714844,
      "activations/layer1_attention_weight_min": -13.578516006469727,
      "activations/layer20_attention_weight_max": 23.183443069458008,
      "activations/layer20_attention_weight_min": -20.876876831054688,
      "activations/layer21_attention_weight_max": 45.52079391479492,
      "activations/layer21_attention_weight_min": -28.570640563964844,
      "activations/layer22_attention_weight_max": 36.75559616088867,
      "activations/layer22_attention_weight_min": -22.554359436035156,
      "activations/layer23_attention_weight_max": 27.95246696472168,
      "activations/layer23_attention_weight_min": -17.057411193847656,
      "activations/layer2_attention_weight_max": 28.93006134033203,
      "activations/layer2_attention_weight_min": -28.447277069091797,
      "activations/layer3_attention_weight_max": 71.5716552734375,
      "activations/layer3_attention_weight_min": -76.0372085571289,
      "activations/layer4_attention_weight_max": 102.6707763671875,
      "activations/layer4_attention_weight_min": -91.9635238647461,
      "activations/layer5_attention_weight_max": 118.64540100097656,
      "activations/layer5_attention_weight_min": -99.10124206542969,
      "activations/layer6_attention_weight_max": 58.858551025390625,
      "activations/layer6_attention_weight_min": -51.79490280151367,
      "activations/layer7_attention_weight_max": 62.24251174926758,
      "activations/layer7_attention_weight_min": -58.15768051147461,
      "activations/layer8_attention_weight_max": 38.95665740966797,
      "activations/layer8_attention_weight_min": -36.081851959228516,
      "activations/layer9_attention_weight_max": 51.12339782714844,
      "activations/layer9_attention_weight_min": -49.20936965942383,
      "epoch": 3.21,
      "learning_rate": 0.00013059242424242424,
      "loss": 2.8875,
      "step": 55300
    },
    {
      "activations/layer0_attention_weight_max": 14.896733283996582,
      "activations/layer0_attention_weight_min": -14.737650871276855,
      "activations/layer10_attention_weight_max": 44.40191650390625,
      "activations/layer10_attention_weight_min": -31.931791305541992,
      "activations/layer11_attention_weight_max": 33.33802032470703,
      "activations/layer11_attention_weight_min": -25.876415252685547,
      "activations/layer12_attention_weight_max": 23.51348304748535,
      "activations/layer12_attention_weight_min": -24.55076789855957,
      "activations/layer13_attention_weight_max": 57.21225357055664,
      "activations/layer13_attention_weight_min": -37.95082473754883,
      "activations/layer14_attention_weight_max": 52.511409759521484,
      "activations/layer14_attention_weight_min": -34.474449157714844,
      "activations/layer15_attention_weight_max": 52.72948455810547,
      "activations/layer15_attention_weight_min": -33.165611267089844,
      "activations/layer16_attention_weight_max": 37.71763229370117,
      "activations/layer16_attention_weight_min": -26.659988403320312,
      "activations/layer17_attention_weight_max": 69.08858489990234,
      "activations/layer17_attention_weight_min": -33.9304084777832,
      "activations/layer18_attention_weight_max": 59.6406135559082,
      "activations/layer18_attention_weight_min": -30.393465042114258,
      "activations/layer19_attention_weight_max": 25.7248477935791,
      "activations/layer19_attention_weight_min": -18.430707931518555,
      "activations/layer1_attention_weight_max": 18.790569305419922,
      "activations/layer1_attention_weight_min": -14.411258697509766,
      "activations/layer20_attention_weight_max": 26.866466522216797,
      "activations/layer20_attention_weight_min": -20.528688430786133,
      "activations/layer21_attention_weight_max": 44.6522331237793,
      "activations/layer21_attention_weight_min": -28.514617919921875,
      "activations/layer22_attention_weight_max": 39.57733917236328,
      "activations/layer22_attention_weight_min": -22.83389663696289,
      "activations/layer23_attention_weight_max": 28.368263244628906,
      "activations/layer23_attention_weight_min": -15.875606536865234,
      "activations/layer2_attention_weight_max": 28.27509307861328,
      "activations/layer2_attention_weight_min": -27.798315048217773,
      "activations/layer3_attention_weight_max": 76.71834564208984,
      "activations/layer3_attention_weight_min": -77.76578521728516,
      "activations/layer4_attention_weight_max": 108.69658660888672,
      "activations/layer4_attention_weight_min": -98.9500961303711,
      "activations/layer5_attention_weight_max": 126.29795837402344,
      "activations/layer5_attention_weight_min": -104.78020477294922,
      "activations/layer6_attention_weight_max": 61.675682067871094,
      "activations/layer6_attention_weight_min": -52.875328063964844,
      "activations/layer7_attention_weight_max": 60.49839782714844,
      "activations/layer7_attention_weight_min": -57.847862243652344,
      "activations/layer8_attention_weight_max": 42.37491226196289,
      "activations/layer8_attention_weight_min": -35.60615158081055,
      "activations/layer9_attention_weight_max": 55.26654815673828,
      "activations/layer9_attention_weight_min": -45.95994186401367,
      "epoch": 3.22,
      "learning_rate": 0.00013057348484848483,
      "loss": 2.8852,
      "step": 55350
    },
    {
      "activations/layer0_attention_weight_max": 14.731328964233398,
      "activations/layer0_attention_weight_min": -14.7979736328125,
      "activations/layer10_attention_weight_max": 43.38660430908203,
      "activations/layer10_attention_weight_min": -34.824214935302734,
      "activations/layer11_attention_weight_max": 37.26507568359375,
      "activations/layer11_attention_weight_min": -27.480146408081055,
      "activations/layer12_attention_weight_max": 26.434476852416992,
      "activations/layer12_attention_weight_min": -24.932514190673828,
      "activations/layer13_attention_weight_max": 54.29069137573242,
      "activations/layer13_attention_weight_min": -40.76588821411133,
      "activations/layer14_attention_weight_max": 47.598243713378906,
      "activations/layer14_attention_weight_min": -33.492794036865234,
      "activations/layer15_attention_weight_max": 47.514591217041016,
      "activations/layer15_attention_weight_min": -34.741554260253906,
      "activations/layer16_attention_weight_max": 34.03929901123047,
      "activations/layer16_attention_weight_min": -27.680644989013672,
      "activations/layer17_attention_weight_max": 61.72554397583008,
      "activations/layer17_attention_weight_min": -34.422096252441406,
      "activations/layer18_attention_weight_max": 55.117244720458984,
      "activations/layer18_attention_weight_min": -30.109413146972656,
      "activations/layer19_attention_weight_max": 22.16303825378418,
      "activations/layer19_attention_weight_min": -18.671825408935547,
      "activations/layer1_attention_weight_max": 17.710250854492188,
      "activations/layer1_attention_weight_min": -13.09805965423584,
      "activations/layer20_attention_weight_max": 22.650449752807617,
      "activations/layer20_attention_weight_min": -20.652326583862305,
      "activations/layer21_attention_weight_max": 46.844993591308594,
      "activations/layer21_attention_weight_min": -30.101314544677734,
      "activations/layer22_attention_weight_max": 35.90116882324219,
      "activations/layer22_attention_weight_min": -22.474834442138672,
      "activations/layer23_attention_weight_max": 23.31420135498047,
      "activations/layer23_attention_weight_min": -17.298446655273438,
      "activations/layer2_attention_weight_max": 28.44849395751953,
      "activations/layer2_attention_weight_min": -26.766887664794922,
      "activations/layer3_attention_weight_max": 70.47271728515625,
      "activations/layer3_attention_weight_min": -77.60936737060547,
      "activations/layer4_attention_weight_max": 96.50656127929688,
      "activations/layer4_attention_weight_min": -91.91093444824219,
      "activations/layer5_attention_weight_max": 116.51271057128906,
      "activations/layer5_attention_weight_min": -96.26799011230469,
      "activations/layer6_attention_weight_max": 58.92084503173828,
      "activations/layer6_attention_weight_min": -49.77973937988281,
      "activations/layer7_attention_weight_max": 61.75858688354492,
      "activations/layer7_attention_weight_min": -55.81691360473633,
      "activations/layer8_attention_weight_max": 39.67982864379883,
      "activations/layer8_attention_weight_min": -36.080474853515625,
      "activations/layer9_attention_weight_max": 53.183528900146484,
      "activations/layer9_attention_weight_min": -45.730403900146484,
      "epoch": 3.22,
      "learning_rate": 0.00013055454545454545,
      "loss": 2.8779,
      "step": 55400
    },
    {
      "activations/layer0_attention_weight_max": 14.149435997009277,
      "activations/layer0_attention_weight_min": -14.793485641479492,
      "activations/layer10_attention_weight_max": 43.96300506591797,
      "activations/layer10_attention_weight_min": -34.700923919677734,
      "activations/layer11_attention_weight_max": 38.87468719482422,
      "activations/layer11_attention_weight_min": -26.367919921875,
      "activations/layer12_attention_weight_max": 28.448701858520508,
      "activations/layer12_attention_weight_min": -26.475255966186523,
      "activations/layer13_attention_weight_max": 63.04209518432617,
      "activations/layer13_attention_weight_min": -41.466285705566406,
      "activations/layer14_attention_weight_max": 64.15026092529297,
      "activations/layer14_attention_weight_min": -35.572845458984375,
      "activations/layer15_attention_weight_max": 55.057647705078125,
      "activations/layer15_attention_weight_min": -34.14751434326172,
      "activations/layer16_attention_weight_max": 46.509525299072266,
      "activations/layer16_attention_weight_min": -26.266613006591797,
      "activations/layer17_attention_weight_max": 76.82188415527344,
      "activations/layer17_attention_weight_min": -37.85251998901367,
      "activations/layer18_attention_weight_max": 62.71207046508789,
      "activations/layer18_attention_weight_min": -31.958255767822266,
      "activations/layer19_attention_weight_max": 25.7833194732666,
      "activations/layer19_attention_weight_min": -17.73638916015625,
      "activations/layer1_attention_weight_max": 16.92818260192871,
      "activations/layer1_attention_weight_min": -15.588593482971191,
      "activations/layer20_attention_weight_max": 28.114906311035156,
      "activations/layer20_attention_weight_min": -18.980321884155273,
      "activations/layer21_attention_weight_max": 52.268333435058594,
      "activations/layer21_attention_weight_min": -25.760284423828125,
      "activations/layer22_attention_weight_max": 40.06208038330078,
      "activations/layer22_attention_weight_min": -21.435510635375977,
      "activations/layer23_attention_weight_max": 29.835186004638672,
      "activations/layer23_attention_weight_min": -15.341581344604492,
      "activations/layer2_attention_weight_max": 31.318870544433594,
      "activations/layer2_attention_weight_min": -27.318164825439453,
      "activations/layer3_attention_weight_max": 79.19548034667969,
      "activations/layer3_attention_weight_min": -79.22061920166016,
      "activations/layer4_attention_weight_max": 105.19221496582031,
      "activations/layer4_attention_weight_min": -92.76721954345703,
      "activations/layer5_attention_weight_max": 126.94780731201172,
      "activations/layer5_attention_weight_min": -107.77861022949219,
      "activations/layer6_attention_weight_max": 62.525516510009766,
      "activations/layer6_attention_weight_min": -52.39118957519531,
      "activations/layer7_attention_weight_max": 65.06819152832031,
      "activations/layer7_attention_weight_min": -55.78900909423828,
      "activations/layer8_attention_weight_max": 39.99113845825195,
      "activations/layer8_attention_weight_min": -35.9209098815918,
      "activations/layer9_attention_weight_max": 53.67214584350586,
      "activations/layer9_attention_weight_min": -48.577274322509766,
      "epoch": 3.22,
      "learning_rate": 0.00013053560606060606,
      "loss": 2.881,
      "step": 55450
    },
    {
      "activations/layer0_attention_weight_max": 14.61769962310791,
      "activations/layer0_attention_weight_min": -14.943449020385742,
      "activations/layer10_attention_weight_max": 41.875,
      "activations/layer10_attention_weight_min": -35.5873908996582,
      "activations/layer11_attention_weight_max": 36.28163146972656,
      "activations/layer11_attention_weight_min": -28.381324768066406,
      "activations/layer12_attention_weight_max": 29.80352020263672,
      "activations/layer12_attention_weight_min": -24.34001922607422,
      "activations/layer13_attention_weight_max": 53.68257522583008,
      "activations/layer13_attention_weight_min": -38.176841735839844,
      "activations/layer14_attention_weight_max": 51.0791015625,
      "activations/layer14_attention_weight_min": -34.51219940185547,
      "activations/layer15_attention_weight_max": 47.46132278442383,
      "activations/layer15_attention_weight_min": -31.347620010375977,
      "activations/layer16_attention_weight_max": 44.32027816772461,
      "activations/layer16_attention_weight_min": -26.941259384155273,
      "activations/layer17_attention_weight_max": 56.95389175415039,
      "activations/layer17_attention_weight_min": -37.29549789428711,
      "activations/layer18_attention_weight_max": 55.881412506103516,
      "activations/layer18_attention_weight_min": -33.43387985229492,
      "activations/layer19_attention_weight_max": 21.290708541870117,
      "activations/layer19_attention_weight_min": -18.90458869934082,
      "activations/layer1_attention_weight_max": 16.78380012512207,
      "activations/layer1_attention_weight_min": -13.664015769958496,
      "activations/layer20_attention_weight_max": 22.535959243774414,
      "activations/layer20_attention_weight_min": -20.55715560913086,
      "activations/layer21_attention_weight_max": 47.26133728027344,
      "activations/layer21_attention_weight_min": -26.076568603515625,
      "activations/layer22_attention_weight_max": 37.59136962890625,
      "activations/layer22_attention_weight_min": -24.9659423828125,
      "activations/layer23_attention_weight_max": 27.879568099975586,
      "activations/layer23_attention_weight_min": -16.371280670166016,
      "activations/layer2_attention_weight_max": 29.42486572265625,
      "activations/layer2_attention_weight_min": -27.678382873535156,
      "activations/layer3_attention_weight_max": 73.77375793457031,
      "activations/layer3_attention_weight_min": -77.160400390625,
      "activations/layer4_attention_weight_max": 99.05139923095703,
      "activations/layer4_attention_weight_min": -97.05261993408203,
      "activations/layer5_attention_weight_max": 120.42007446289062,
      "activations/layer5_attention_weight_min": -100.54548645019531,
      "activations/layer6_attention_weight_max": 59.76588439941406,
      "activations/layer6_attention_weight_min": -52.87932586669922,
      "activations/layer7_attention_weight_max": 74.29437255859375,
      "activations/layer7_attention_weight_min": -58.7569694519043,
      "activations/layer8_attention_weight_max": 38.837528228759766,
      "activations/layer8_attention_weight_min": -35.760501861572266,
      "activations/layer9_attention_weight_max": 54.47933578491211,
      "activations/layer9_attention_weight_min": -49.80388259887695,
      "epoch": 3.22,
      "learning_rate": 0.00013051666666666665,
      "loss": 2.8749,
      "step": 55500
    },
    {
      "activations/layer0_attention_weight_max": 15.817523956298828,
      "activations/layer0_attention_weight_min": -14.578020095825195,
      "activations/layer10_attention_weight_max": 50.81622314453125,
      "activations/layer10_attention_weight_min": -36.91451644897461,
      "activations/layer11_attention_weight_max": 40.78132247924805,
      "activations/layer11_attention_weight_min": -27.447006225585938,
      "activations/layer12_attention_weight_max": 31.42975425720215,
      "activations/layer12_attention_weight_min": -24.895200729370117,
      "activations/layer13_attention_weight_max": 93.59947204589844,
      "activations/layer13_attention_weight_min": -58.86964416503906,
      "activations/layer14_attention_weight_max": 84.94427490234375,
      "activations/layer14_attention_weight_min": -47.141990661621094,
      "activations/layer15_attention_weight_max": 59.67025375366211,
      "activations/layer15_attention_weight_min": -34.52653121948242,
      "activations/layer16_attention_weight_max": 40.19532775878906,
      "activations/layer16_attention_weight_min": -26.818161010742188,
      "activations/layer17_attention_weight_max": 67.81510162353516,
      "activations/layer17_attention_weight_min": -36.39040756225586,
      "activations/layer18_attention_weight_max": 55.05436706542969,
      "activations/layer18_attention_weight_min": -31.875,
      "activations/layer19_attention_weight_max": 25.54852294921875,
      "activations/layer19_attention_weight_min": -19.235183715820312,
      "activations/layer1_attention_weight_max": 17.473949432373047,
      "activations/layer1_attention_weight_min": -15.599393844604492,
      "activations/layer20_attention_weight_max": 24.58586883544922,
      "activations/layer20_attention_weight_min": -20.861249923706055,
      "activations/layer21_attention_weight_max": 43.851688385009766,
      "activations/layer21_attention_weight_min": -28.308467864990234,
      "activations/layer22_attention_weight_max": 39.68398666381836,
      "activations/layer22_attention_weight_min": -23.51056671142578,
      "activations/layer23_attention_weight_max": 27.934612274169922,
      "activations/layer23_attention_weight_min": -16.558704376220703,
      "activations/layer2_attention_weight_max": 30.887622833251953,
      "activations/layer2_attention_weight_min": -28.702186584472656,
      "activations/layer3_attention_weight_max": 73.59391021728516,
      "activations/layer3_attention_weight_min": -76.60285949707031,
      "activations/layer4_attention_weight_max": 105.88253021240234,
      "activations/layer4_attention_weight_min": -93.33184814453125,
      "activations/layer5_attention_weight_max": 132.81735229492188,
      "activations/layer5_attention_weight_min": -104.83171844482422,
      "activations/layer6_attention_weight_max": 64.60652923583984,
      "activations/layer6_attention_weight_min": -55.56984329223633,
      "activations/layer7_attention_weight_max": 70.52257537841797,
      "activations/layer7_attention_weight_min": -60.19953536987305,
      "activations/layer8_attention_weight_max": 47.32163619995117,
      "activations/layer8_attention_weight_min": -38.440181732177734,
      "activations/layer9_attention_weight_max": 66.5439224243164,
      "activations/layer9_attention_weight_min": -52.377716064453125,
      "epoch": 3.23,
      "learning_rate": 0.00013049772727272726,
      "loss": 2.8693,
      "step": 55550
    },
    {
      "activations/layer0_attention_weight_max": 14.795299530029297,
      "activations/layer0_attention_weight_min": -14.622387886047363,
      "activations/layer10_attention_weight_max": 44.21882629394531,
      "activations/layer10_attention_weight_min": -35.717857360839844,
      "activations/layer11_attention_weight_max": 37.748779296875,
      "activations/layer11_attention_weight_min": -27.11532211303711,
      "activations/layer12_attention_weight_max": 27.640764236450195,
      "activations/layer12_attention_weight_min": -27.661544799804688,
      "activations/layer13_attention_weight_max": 60.7843132019043,
      "activations/layer13_attention_weight_min": -40.274452209472656,
      "activations/layer14_attention_weight_max": 44.26387023925781,
      "activations/layer14_attention_weight_min": -29.075923919677734,
      "activations/layer15_attention_weight_max": 49.30527114868164,
      "activations/layer15_attention_weight_min": -35.89311599731445,
      "activations/layer16_attention_weight_max": 39.63028335571289,
      "activations/layer16_attention_weight_min": -24.625762939453125,
      "activations/layer17_attention_weight_max": 64.34935760498047,
      "activations/layer17_attention_weight_min": -36.540279388427734,
      "activations/layer18_attention_weight_max": 53.213687896728516,
      "activations/layer18_attention_weight_min": -31.715185165405273,
      "activations/layer19_attention_weight_max": 22.519760131835938,
      "activations/layer19_attention_weight_min": -16.26662826538086,
      "activations/layer1_attention_weight_max": 19.21954345703125,
      "activations/layer1_attention_weight_min": -17.26454734802246,
      "activations/layer20_attention_weight_max": 20.04256820678711,
      "activations/layer20_attention_weight_min": -22.132801055908203,
      "activations/layer21_attention_weight_max": 45.540245056152344,
      "activations/layer21_attention_weight_min": -32.43165969848633,
      "activations/layer22_attention_weight_max": 37.41935348510742,
      "activations/layer22_attention_weight_min": -23.92357635498047,
      "activations/layer23_attention_weight_max": 23.986873626708984,
      "activations/layer23_attention_weight_min": -15.599746704101562,
      "activations/layer2_attention_weight_max": 31.883237838745117,
      "activations/layer2_attention_weight_min": -29.83235740661621,
      "activations/layer3_attention_weight_max": 74.5824966430664,
      "activations/layer3_attention_weight_min": -80.74006652832031,
      "activations/layer4_attention_weight_max": 102.79188537597656,
      "activations/layer4_attention_weight_min": -94.85802459716797,
      "activations/layer5_attention_weight_max": 116.92503356933594,
      "activations/layer5_attention_weight_min": -103.11289978027344,
      "activations/layer6_attention_weight_max": 59.01288986206055,
      "activations/layer6_attention_weight_min": -54.43293762207031,
      "activations/layer7_attention_weight_max": 62.960296630859375,
      "activations/layer7_attention_weight_min": -57.94300079345703,
      "activations/layer8_attention_weight_max": 38.22821044921875,
      "activations/layer8_attention_weight_min": -37.38758087158203,
      "activations/layer9_attention_weight_max": 51.2093505859375,
      "activations/layer9_attention_weight_min": -47.16176986694336,
      "epoch": 3.23,
      "learning_rate": 0.00013047878787878788,
      "loss": 2.8968,
      "step": 55600
    },
    {
      "activations/layer0_attention_weight_max": 14.580537796020508,
      "activations/layer0_attention_weight_min": -14.128284454345703,
      "activations/layer10_attention_weight_max": 39.531124114990234,
      "activations/layer10_attention_weight_min": -35.593475341796875,
      "activations/layer11_attention_weight_max": 32.29389572143555,
      "activations/layer11_attention_weight_min": -28.973682403564453,
      "activations/layer12_attention_weight_max": 23.208675384521484,
      "activations/layer12_attention_weight_min": -24.080968856811523,
      "activations/layer13_attention_weight_max": 57.89716339111328,
      "activations/layer13_attention_weight_min": -43.239139556884766,
      "activations/layer14_attention_weight_max": 49.788429260253906,
      "activations/layer14_attention_weight_min": -37.02170944213867,
      "activations/layer15_attention_weight_max": 42.56608963012695,
      "activations/layer15_attention_weight_min": -32.74102020263672,
      "activations/layer16_attention_weight_max": 35.66744613647461,
      "activations/layer16_attention_weight_min": -26.05259895324707,
      "activations/layer17_attention_weight_max": 56.59370040893555,
      "activations/layer17_attention_weight_min": -37.38978576660156,
      "activations/layer18_attention_weight_max": 46.511924743652344,
      "activations/layer18_attention_weight_min": -33.616912841796875,
      "activations/layer19_attention_weight_max": 21.177356719970703,
      "activations/layer19_attention_weight_min": -18.556705474853516,
      "activations/layer1_attention_weight_max": 17.139358520507812,
      "activations/layer1_attention_weight_min": -13.081710815429688,
      "activations/layer20_attention_weight_max": 20.66057014465332,
      "activations/layer20_attention_weight_min": -19.688568115234375,
      "activations/layer21_attention_weight_max": 38.375694274902344,
      "activations/layer21_attention_weight_min": -30.989439010620117,
      "activations/layer22_attention_weight_max": 35.60670471191406,
      "activations/layer22_attention_weight_min": -24.86093521118164,
      "activations/layer23_attention_weight_max": 26.191160202026367,
      "activations/layer23_attention_weight_min": -16.25802230834961,
      "activations/layer2_attention_weight_max": 29.556987762451172,
      "activations/layer2_attention_weight_min": -27.404617309570312,
      "activations/layer3_attention_weight_max": 73.30472564697266,
      "activations/layer3_attention_weight_min": -76.42060852050781,
      "activations/layer4_attention_weight_max": 103.3281021118164,
      "activations/layer4_attention_weight_min": -90.74003601074219,
      "activations/layer5_attention_weight_max": 122.50057220458984,
      "activations/layer5_attention_weight_min": -101.01811218261719,
      "activations/layer6_attention_weight_max": 63.04200744628906,
      "activations/layer6_attention_weight_min": -55.81293487548828,
      "activations/layer7_attention_weight_max": 59.40249252319336,
      "activations/layer7_attention_weight_min": -57.27103042602539,
      "activations/layer8_attention_weight_max": 38.02550506591797,
      "activations/layer8_attention_weight_min": -39.60647201538086,
      "activations/layer9_attention_weight_max": 47.56362533569336,
      "activations/layer9_attention_weight_min": -49.047462463378906,
      "epoch": 3.23,
      "learning_rate": 0.00013045984848484847,
      "loss": 2.8981,
      "step": 55650
    },
    {
      "activations/layer0_attention_weight_max": 14.79043960571289,
      "activations/layer0_attention_weight_min": -14.65841007232666,
      "activations/layer10_attention_weight_max": 44.77106857299805,
      "activations/layer10_attention_weight_min": -35.90152359008789,
      "activations/layer11_attention_weight_max": 38.630401611328125,
      "activations/layer11_attention_weight_min": -29.460798263549805,
      "activations/layer12_attention_weight_max": 25.618505477905273,
      "activations/layer12_attention_weight_min": -27.532943725585938,
      "activations/layer13_attention_weight_max": 56.38112258911133,
      "activations/layer13_attention_weight_min": -34.823368072509766,
      "activations/layer14_attention_weight_max": 47.37510299682617,
      "activations/layer14_attention_weight_min": -32.969459533691406,
      "activations/layer15_attention_weight_max": 48.4222297668457,
      "activations/layer15_attention_weight_min": -31.885501861572266,
      "activations/layer16_attention_weight_max": 41.860572814941406,
      "activations/layer16_attention_weight_min": -25.068992614746094,
      "activations/layer17_attention_weight_max": 59.52077102661133,
      "activations/layer17_attention_weight_min": -34.0764045715332,
      "activations/layer18_attention_weight_max": 52.90829849243164,
      "activations/layer18_attention_weight_min": -33.47636413574219,
      "activations/layer19_attention_weight_max": 24.433591842651367,
      "activations/layer19_attention_weight_min": -17.28282928466797,
      "activations/layer1_attention_weight_max": 18.014984130859375,
      "activations/layer1_attention_weight_min": -12.79590892791748,
      "activations/layer20_attention_weight_max": 22.14755630493164,
      "activations/layer20_attention_weight_min": -17.553970336914062,
      "activations/layer21_attention_weight_max": 41.58913040161133,
      "activations/layer21_attention_weight_min": -25.43814468383789,
      "activations/layer22_attention_weight_max": 36.704219818115234,
      "activations/layer22_attention_weight_min": -20.73589515686035,
      "activations/layer23_attention_weight_max": 26.423877716064453,
      "activations/layer23_attention_weight_min": -15.062704086303711,
      "activations/layer2_attention_weight_max": 28.20116424560547,
      "activations/layer2_attention_weight_min": -26.484905242919922,
      "activations/layer3_attention_weight_max": 75.56290435791016,
      "activations/layer3_attention_weight_min": -80.84613800048828,
      "activations/layer4_attention_weight_max": 108.45291900634766,
      "activations/layer4_attention_weight_min": -98.31986999511719,
      "activations/layer5_attention_weight_max": 132.82725524902344,
      "activations/layer5_attention_weight_min": -109.83427429199219,
      "activations/layer6_attention_weight_max": 67.21593475341797,
      "activations/layer6_attention_weight_min": -54.1633186340332,
      "activations/layer7_attention_weight_max": 76.2595443725586,
      "activations/layer7_attention_weight_min": -63.17769241333008,
      "activations/layer8_attention_weight_max": 48.636085510253906,
      "activations/layer8_attention_weight_min": -37.91321563720703,
      "activations/layer9_attention_weight_max": 61.86225509643555,
      "activations/layer9_attention_weight_min": -50.62971496582031,
      "epoch": 3.24,
      "learning_rate": 0.00013044090909090908,
      "loss": 2.8845,
      "step": 55700
    },
    {
      "activations/layer0_attention_weight_max": 15.04668140411377,
      "activations/layer0_attention_weight_min": -14.825335502624512,
      "activations/layer10_attention_weight_max": 44.46952438354492,
      "activations/layer10_attention_weight_min": -35.415992736816406,
      "activations/layer11_attention_weight_max": 33.562461853027344,
      "activations/layer11_attention_weight_min": -26.422080993652344,
      "activations/layer12_attention_weight_max": 22.817493438720703,
      "activations/layer12_attention_weight_min": -26.8636417388916,
      "activations/layer13_attention_weight_max": 49.786075592041016,
      "activations/layer13_attention_weight_min": -37.0443229675293,
      "activations/layer14_attention_weight_max": 50.45511245727539,
      "activations/layer14_attention_weight_min": -36.70991516113281,
      "activations/layer15_attention_weight_max": 43.97336959838867,
      "activations/layer15_attention_weight_min": -31.632816314697266,
      "activations/layer16_attention_weight_max": 42.90169143676758,
      "activations/layer16_attention_weight_min": -25.156545639038086,
      "activations/layer17_attention_weight_max": 57.58591842651367,
      "activations/layer17_attention_weight_min": -35.027469635009766,
      "activations/layer18_attention_weight_max": 51.140403747558594,
      "activations/layer18_attention_weight_min": -31.548112869262695,
      "activations/layer19_attention_weight_max": 21.1478328704834,
      "activations/layer19_attention_weight_min": -15.900908470153809,
      "activations/layer1_attention_weight_max": 17.451122283935547,
      "activations/layer1_attention_weight_min": -14.900684356689453,
      "activations/layer20_attention_weight_max": 18.781646728515625,
      "activations/layer20_attention_weight_min": -21.331180572509766,
      "activations/layer21_attention_weight_max": 39.99336624145508,
      "activations/layer21_attention_weight_min": -28.637845993041992,
      "activations/layer22_attention_weight_max": 30.48060417175293,
      "activations/layer22_attention_weight_min": -24.07874870300293,
      "activations/layer23_attention_weight_max": 24.38007926940918,
      "activations/layer23_attention_weight_min": -20.03972625732422,
      "activations/layer2_attention_weight_max": 28.865873336791992,
      "activations/layer2_attention_weight_min": -27.568456649780273,
      "activations/layer3_attention_weight_max": 70.00403594970703,
      "activations/layer3_attention_weight_min": -74.03196716308594,
      "activations/layer4_attention_weight_max": 95.91951751708984,
      "activations/layer4_attention_weight_min": -89.2975082397461,
      "activations/layer5_attention_weight_max": 117.52609252929688,
      "activations/layer5_attention_weight_min": -100.3177719116211,
      "activations/layer6_attention_weight_max": 59.71674346923828,
      "activations/layer6_attention_weight_min": -51.263187408447266,
      "activations/layer7_attention_weight_max": 56.82392883300781,
      "activations/layer7_attention_weight_min": -57.01418685913086,
      "activations/layer8_attention_weight_max": 35.988616943359375,
      "activations/layer8_attention_weight_min": -36.1572265625,
      "activations/layer9_attention_weight_max": 49.7666130065918,
      "activations/layer9_attention_weight_min": -48.11363983154297,
      "epoch": 3.24,
      "learning_rate": 0.0001304219696969697,
      "loss": 2.879,
      "step": 55750
    },
    {
      "activations/layer0_attention_weight_max": 15.071569442749023,
      "activations/layer0_attention_weight_min": -14.910381317138672,
      "activations/layer10_attention_weight_max": 39.044677734375,
      "activations/layer10_attention_weight_min": -38.78453063964844,
      "activations/layer11_attention_weight_max": 34.25221252441406,
      "activations/layer11_attention_weight_min": -29.222715377807617,
      "activations/layer12_attention_weight_max": 28.350034713745117,
      "activations/layer12_attention_weight_min": -25.632246017456055,
      "activations/layer13_attention_weight_max": 44.48112106323242,
      "activations/layer13_attention_weight_min": -33.027591705322266,
      "activations/layer14_attention_weight_max": 49.3090934753418,
      "activations/layer14_attention_weight_min": -33.2126350402832,
      "activations/layer15_attention_weight_max": 44.13365936279297,
      "activations/layer15_attention_weight_min": -31.93142318725586,
      "activations/layer16_attention_weight_max": 41.39921188354492,
      "activations/layer16_attention_weight_min": -27.13875389099121,
      "activations/layer17_attention_weight_max": 57.02119827270508,
      "activations/layer17_attention_weight_min": -34.95094680786133,
      "activations/layer18_attention_weight_max": 51.25682067871094,
      "activations/layer18_attention_weight_min": -31.963741302490234,
      "activations/layer19_attention_weight_max": 22.412172317504883,
      "activations/layer19_attention_weight_min": -16.929880142211914,
      "activations/layer1_attention_weight_max": 17.70380973815918,
      "activations/layer1_attention_weight_min": -14.14445686340332,
      "activations/layer20_attention_weight_max": 18.184621810913086,
      "activations/layer20_attention_weight_min": -21.17003631591797,
      "activations/layer21_attention_weight_max": 42.57441329956055,
      "activations/layer21_attention_weight_min": -26.856048583984375,
      "activations/layer22_attention_weight_max": 40.407073974609375,
      "activations/layer22_attention_weight_min": -22.926347732543945,
      "activations/layer23_attention_weight_max": 23.500255584716797,
      "activations/layer23_attention_weight_min": -15.791584014892578,
      "activations/layer2_attention_weight_max": 29.61652183532715,
      "activations/layer2_attention_weight_min": -28.29814910888672,
      "activations/layer3_attention_weight_max": 75.15445709228516,
      "activations/layer3_attention_weight_min": -79.7703628540039,
      "activations/layer4_attention_weight_max": 103.76177978515625,
      "activations/layer4_attention_weight_min": -96.51702880859375,
      "activations/layer5_attention_weight_max": 121.33091735839844,
      "activations/layer5_attention_weight_min": -109.05271911621094,
      "activations/layer6_attention_weight_max": 61.21173858642578,
      "activations/layer6_attention_weight_min": -52.812538146972656,
      "activations/layer7_attention_weight_max": 69.28089904785156,
      "activations/layer7_attention_weight_min": -58.4688720703125,
      "activations/layer8_attention_weight_max": 43.309505462646484,
      "activations/layer8_attention_weight_min": -39.74789810180664,
      "activations/layer9_attention_weight_max": 61.011993408203125,
      "activations/layer9_attention_weight_min": -47.1186637878418,
      "epoch": 3.24,
      "learning_rate": 0.00013040303030303029,
      "loss": 2.8784,
      "step": 55800
    },
    {
      "activations/layer0_attention_weight_max": 15.278563499450684,
      "activations/layer0_attention_weight_min": -14.593711853027344,
      "activations/layer10_attention_weight_max": 43.72810363769531,
      "activations/layer10_attention_weight_min": -34.49630355834961,
      "activations/layer11_attention_weight_max": 35.964385986328125,
      "activations/layer11_attention_weight_min": -28.94654083251953,
      "activations/layer12_attention_weight_max": 25.653467178344727,
      "activations/layer12_attention_weight_min": -24.56386947631836,
      "activations/layer13_attention_weight_max": 49.201778411865234,
      "activations/layer13_attention_weight_min": -38.66029739379883,
      "activations/layer14_attention_weight_max": 55.115692138671875,
      "activations/layer14_attention_weight_min": -38.0473747253418,
      "activations/layer15_attention_weight_max": 41.02494430541992,
      "activations/layer15_attention_weight_min": -31.645065307617188,
      "activations/layer16_attention_weight_max": 38.74094009399414,
      "activations/layer16_attention_weight_min": -25.62718391418457,
      "activations/layer17_attention_weight_max": 60.70718002319336,
      "activations/layer17_attention_weight_min": -40.4072380065918,
      "activations/layer18_attention_weight_max": 50.09535598754883,
      "activations/layer18_attention_weight_min": -32.298641204833984,
      "activations/layer19_attention_weight_max": 20.051137924194336,
      "activations/layer19_attention_weight_min": -18.83806800842285,
      "activations/layer1_attention_weight_max": 17.030963897705078,
      "activations/layer1_attention_weight_min": -14.430990219116211,
      "activations/layer20_attention_weight_max": 22.592931747436523,
      "activations/layer20_attention_weight_min": -19.279041290283203,
      "activations/layer21_attention_weight_max": 42.17938232421875,
      "activations/layer21_attention_weight_min": -26.69793701171875,
      "activations/layer22_attention_weight_max": 37.973018646240234,
      "activations/layer22_attention_weight_min": -23.96120262145996,
      "activations/layer23_attention_weight_max": 27.801136016845703,
      "activations/layer23_attention_weight_min": -15.845858573913574,
      "activations/layer2_attention_weight_max": 31.16881561279297,
      "activations/layer2_attention_weight_min": -30.923690795898438,
      "activations/layer3_attention_weight_max": 71.22615051269531,
      "activations/layer3_attention_weight_min": -78.72258758544922,
      "activations/layer4_attention_weight_max": 101.42033386230469,
      "activations/layer4_attention_weight_min": -92.64302825927734,
      "activations/layer5_attention_weight_max": 121.65562438964844,
      "activations/layer5_attention_weight_min": -96.96825408935547,
      "activations/layer6_attention_weight_max": 60.7672004699707,
      "activations/layer6_attention_weight_min": -48.908138275146484,
      "activations/layer7_attention_weight_max": 57.72276306152344,
      "activations/layer7_attention_weight_min": -58.700096130371094,
      "activations/layer8_attention_weight_max": 37.311805725097656,
      "activations/layer8_attention_weight_min": -33.251644134521484,
      "activations/layer9_attention_weight_max": 46.8512077331543,
      "activations/layer9_attention_weight_min": -49.16980743408203,
      "epoch": 3.25,
      "learning_rate": 0.0001303840909090909,
      "loss": 2.8786,
      "step": 55850
    },
    {
      "activations/layer0_attention_weight_max": 14.056107521057129,
      "activations/layer0_attention_weight_min": -14.627189636230469,
      "activations/layer10_attention_weight_max": 50.02707290649414,
      "activations/layer10_attention_weight_min": -36.13869857788086,
      "activations/layer11_attention_weight_max": 35.37400817871094,
      "activations/layer11_attention_weight_min": -27.43209457397461,
      "activations/layer12_attention_weight_max": 34.494773864746094,
      "activations/layer12_attention_weight_min": -27.53592300415039,
      "activations/layer13_attention_weight_max": 53.72853088378906,
      "activations/layer13_attention_weight_min": -37.852474212646484,
      "activations/layer14_attention_weight_max": 55.65299606323242,
      "activations/layer14_attention_weight_min": -38.78220748901367,
      "activations/layer15_attention_weight_max": 50.89781188964844,
      "activations/layer15_attention_weight_min": -35.98815155029297,
      "activations/layer16_attention_weight_max": 37.48530578613281,
      "activations/layer16_attention_weight_min": -25.89522361755371,
      "activations/layer17_attention_weight_max": 58.72898483276367,
      "activations/layer17_attention_weight_min": -36.042747497558594,
      "activations/layer18_attention_weight_max": 51.957427978515625,
      "activations/layer18_attention_weight_min": -31.54020881652832,
      "activations/layer19_attention_weight_max": 28.411544799804688,
      "activations/layer19_attention_weight_min": -18.1004695892334,
      "activations/layer1_attention_weight_max": 16.35727310180664,
      "activations/layer1_attention_weight_min": -15.165460586547852,
      "activations/layer20_attention_weight_max": 20.643020629882812,
      "activations/layer20_attention_weight_min": -22.101512908935547,
      "activations/layer21_attention_weight_max": 52.481834411621094,
      "activations/layer21_attention_weight_min": -27.678722381591797,
      "activations/layer22_attention_weight_max": 34.38078689575195,
      "activations/layer22_attention_weight_min": -20.73468017578125,
      "activations/layer23_attention_weight_max": 25.52350616455078,
      "activations/layer23_attention_weight_min": -15.358573913574219,
      "activations/layer2_attention_weight_max": 30.565597534179688,
      "activations/layer2_attention_weight_min": -28.44702911376953,
      "activations/layer3_attention_weight_max": 73.43741607666016,
      "activations/layer3_attention_weight_min": -78.24889373779297,
      "activations/layer4_attention_weight_max": 99.99232482910156,
      "activations/layer4_attention_weight_min": -95.96025848388672,
      "activations/layer5_attention_weight_max": 119.50834655761719,
      "activations/layer5_attention_weight_min": -101.87853240966797,
      "activations/layer6_attention_weight_max": 60.020965576171875,
      "activations/layer6_attention_weight_min": -53.61044692993164,
      "activations/layer7_attention_weight_max": 65.39656066894531,
      "activations/layer7_attention_weight_min": -59.387210845947266,
      "activations/layer8_attention_weight_max": 41.2066650390625,
      "activations/layer8_attention_weight_min": -37.6384162902832,
      "activations/layer9_attention_weight_max": 55.926361083984375,
      "activations/layer9_attention_weight_min": -45.77158737182617,
      "epoch": 3.25,
      "learning_rate": 0.00013036515151515152,
      "loss": 2.8914,
      "step": 55900
    },
    {
      "activations/layer0_attention_weight_max": 14.535479545593262,
      "activations/layer0_attention_weight_min": -14.84953498840332,
      "activations/layer10_attention_weight_max": 43.2509651184082,
      "activations/layer10_attention_weight_min": -36.27497100830078,
      "activations/layer11_attention_weight_max": 34.96150588989258,
      "activations/layer11_attention_weight_min": -26.421140670776367,
      "activations/layer12_attention_weight_max": 26.553287506103516,
      "activations/layer12_attention_weight_min": -25.533018112182617,
      "activations/layer13_attention_weight_max": 59.046630859375,
      "activations/layer13_attention_weight_min": -37.69742965698242,
      "activations/layer14_attention_weight_max": 53.250389099121094,
      "activations/layer14_attention_weight_min": -31.813770294189453,
      "activations/layer15_attention_weight_max": 47.9700813293457,
      "activations/layer15_attention_weight_min": -33.156494140625,
      "activations/layer16_attention_weight_max": 41.098026275634766,
      "activations/layer16_attention_weight_min": -26.94914436340332,
      "activations/layer17_attention_weight_max": 64.59632110595703,
      "activations/layer17_attention_weight_min": -37.6741943359375,
      "activations/layer18_attention_weight_max": 56.84312057495117,
      "activations/layer18_attention_weight_min": -31.954402923583984,
      "activations/layer19_attention_weight_max": 22.682043075561523,
      "activations/layer19_attention_weight_min": -18.73210334777832,
      "activations/layer1_attention_weight_max": 18.311187744140625,
      "activations/layer1_attention_weight_min": -17.161476135253906,
      "activations/layer20_attention_weight_max": 22.446470260620117,
      "activations/layer20_attention_weight_min": -18.88190460205078,
      "activations/layer21_attention_weight_max": 43.112388610839844,
      "activations/layer21_attention_weight_min": -25.986967086791992,
      "activations/layer22_attention_weight_max": 36.97923278808594,
      "activations/layer22_attention_weight_min": -21.567365646362305,
      "activations/layer23_attention_weight_max": 25.29840660095215,
      "activations/layer23_attention_weight_min": -20.769010543823242,
      "activations/layer2_attention_weight_max": 29.78464126586914,
      "activations/layer2_attention_weight_min": -27.927701950073242,
      "activations/layer3_attention_weight_max": 71.04360961914062,
      "activations/layer3_attention_weight_min": -75.12287139892578,
      "activations/layer4_attention_weight_max": 99.90554809570312,
      "activations/layer4_attention_weight_min": -91.81023406982422,
      "activations/layer5_attention_weight_max": 118.51478576660156,
      "activations/layer5_attention_weight_min": -104.52311706542969,
      "activations/layer6_attention_weight_max": 60.58629608154297,
      "activations/layer6_attention_weight_min": -51.598548889160156,
      "activations/layer7_attention_weight_max": 63.04348373413086,
      "activations/layer7_attention_weight_min": -58.174354553222656,
      "activations/layer8_attention_weight_max": 38.54940414428711,
      "activations/layer8_attention_weight_min": -37.30879592895508,
      "activations/layer9_attention_weight_max": 49.29988479614258,
      "activations/layer9_attention_weight_min": -47.39303970336914,
      "epoch": 3.25,
      "learning_rate": 0.0001303462121212121,
      "loss": 2.8883,
      "step": 55950
    },
    {
      "activations/layer0_attention_weight_max": 13.27419662475586,
      "activations/layer0_attention_weight_min": -14.953483581542969,
      "activations/layer10_attention_weight_max": 45.10676956176758,
      "activations/layer10_attention_weight_min": -35.854095458984375,
      "activations/layer11_attention_weight_max": 35.642478942871094,
      "activations/layer11_attention_weight_min": -28.186132431030273,
      "activations/layer12_attention_weight_max": 29.598163604736328,
      "activations/layer12_attention_weight_min": -27.713550567626953,
      "activations/layer13_attention_weight_max": 50.96467971801758,
      "activations/layer13_attention_weight_min": -35.76601028442383,
      "activations/layer14_attention_weight_max": 43.26669692993164,
      "activations/layer14_attention_weight_min": -30.93284034729004,
      "activations/layer15_attention_weight_max": 42.689735412597656,
      "activations/layer15_attention_weight_min": -32.16976547241211,
      "activations/layer16_attention_weight_max": 37.303138732910156,
      "activations/layer16_attention_weight_min": -27.56894302368164,
      "activations/layer17_attention_weight_max": 59.63141632080078,
      "activations/layer17_attention_weight_min": -38.0731201171875,
      "activations/layer18_attention_weight_max": 54.89578628540039,
      "activations/layer18_attention_weight_min": -32.91922378540039,
      "activations/layer19_attention_weight_max": 24.391157150268555,
      "activations/layer19_attention_weight_min": -17.28693962097168,
      "activations/layer1_attention_weight_max": 16.90305519104004,
      "activations/layer1_attention_weight_min": -14.35992431640625,
      "activations/layer20_attention_weight_max": 21.72058868408203,
      "activations/layer20_attention_weight_min": -25.333457946777344,
      "activations/layer21_attention_weight_max": 42.939491271972656,
      "activations/layer21_attention_weight_min": -29.553213119506836,
      "activations/layer22_attention_weight_max": 37.89537048339844,
      "activations/layer22_attention_weight_min": -23.194828033447266,
      "activations/layer23_attention_weight_max": 27.881376266479492,
      "activations/layer23_attention_weight_min": -17.280689239501953,
      "activations/layer2_attention_weight_max": 30.829803466796875,
      "activations/layer2_attention_weight_min": -26.98676300048828,
      "activations/layer3_attention_weight_max": 78.76778411865234,
      "activations/layer3_attention_weight_min": -81.4612045288086,
      "activations/layer4_attention_weight_max": 107.35662078857422,
      "activations/layer4_attention_weight_min": -98.58900451660156,
      "activations/layer5_attention_weight_max": 129.8148193359375,
      "activations/layer5_attention_weight_min": -108.51580810546875,
      "activations/layer6_attention_weight_max": 66.40386962890625,
      "activations/layer6_attention_weight_min": -53.91685485839844,
      "activations/layer7_attention_weight_max": 63.335914611816406,
      "activations/layer7_attention_weight_min": -58.046730041503906,
      "activations/layer8_attention_weight_max": 38.717803955078125,
      "activations/layer8_attention_weight_min": -39.2110481262207,
      "activations/layer9_attention_weight_max": 50.416316986083984,
      "activations/layer9_attention_weight_min": -45.84849166870117,
      "epoch": 3.25,
      "learning_rate": 0.00013032727272727272,
      "loss": 2.8921,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_loss": 2.83984375,
      "eval_runtime": 8.504,
      "eval_samples_per_second": 504.941,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_openwebtext_loss": 2.83984375,
      "eval_openwebtext_ppl": 17.113091407702502,
      "eval_openwebtext_runtime": 8.504,
      "eval_openwebtext_samples_per_second": 504.941,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_wikitext_loss": 3.09765625,
      "eval_wikitext_ppl": 22.145985754016134,
      "eval_wikitext_runtime": 1.9267,
      "eval_wikitext_samples_per_second": 236.669,
      "step": 56000
    },
    {
      "epoch": 3.25,
      "eval_lambada_loss": 2.87109375,
      "eval_lambada_ppl": 17.656319241826594,
      "eval_lambada_runtime": 9.5308,
      "eval_lambada_samples_per_second": 510.871,
      "step": 56000
    },
    {
      "activations/layer0_attention_weight_max": 15.011907577514648,
      "activations/layer0_attention_weight_min": -14.330944061279297,
      "activations/layer10_attention_weight_max": 48.20738220214844,
      "activations/layer10_attention_weight_min": -36.2897834777832,
      "activations/layer11_attention_weight_max": 37.640602111816406,
      "activations/layer11_attention_weight_min": -29.788593292236328,
      "activations/layer12_attention_weight_max": 27.98033332824707,
      "activations/layer12_attention_weight_min": -27.724533081054688,
      "activations/layer13_attention_weight_max": 49.38704299926758,
      "activations/layer13_attention_weight_min": -34.812110900878906,
      "activations/layer14_attention_weight_max": 54.34274673461914,
      "activations/layer14_attention_weight_min": -34.756622314453125,
      "activations/layer15_attention_weight_max": 46.63154220581055,
      "activations/layer15_attention_weight_min": -33.1349983215332,
      "activations/layer16_attention_weight_max": 38.761653900146484,
      "activations/layer16_attention_weight_min": -27.79002571105957,
      "activations/layer17_attention_weight_max": 62.56905746459961,
      "activations/layer17_attention_weight_min": -38.08616256713867,
      "activations/layer18_attention_weight_max": 52.55400466918945,
      "activations/layer18_attention_weight_min": -32.8708610534668,
      "activations/layer19_attention_weight_max": 20.909252166748047,
      "activations/layer19_attention_weight_min": -17.897565841674805,
      "activations/layer1_attention_weight_max": 17.163352966308594,
      "activations/layer1_attention_weight_min": -15.146452903747559,
      "activations/layer20_attention_weight_max": 19.3391170501709,
      "activations/layer20_attention_weight_min": -17.59011459350586,
      "activations/layer21_attention_weight_max": 45.55625915527344,
      "activations/layer21_attention_weight_min": -27.501779556274414,
      "activations/layer22_attention_weight_max": 31.300251007080078,
      "activations/layer22_attention_weight_min": -23.6375789642334,
      "activations/layer23_attention_weight_max": 25.253559112548828,
      "activations/layer23_attention_weight_min": -17.63186264038086,
      "activations/layer2_attention_weight_max": 31.58773422241211,
      "activations/layer2_attention_weight_min": -27.94332504272461,
      "activations/layer3_attention_weight_max": 76.56671905517578,
      "activations/layer3_attention_weight_min": -75.42233276367188,
      "activations/layer4_attention_weight_max": 101.92305755615234,
      "activations/layer4_attention_weight_min": -90.31873321533203,
      "activations/layer5_attention_weight_max": 130.89491271972656,
      "activations/layer5_attention_weight_min": -106.5399398803711,
      "activations/layer6_attention_weight_max": 66.04762268066406,
      "activations/layer6_attention_weight_min": -52.170711517333984,
      "activations/layer7_attention_weight_max": 64.89382934570312,
      "activations/layer7_attention_weight_min": -62.42819595336914,
      "activations/layer8_attention_weight_max": 44.73970413208008,
      "activations/layer8_attention_weight_min": -37.47576141357422,
      "activations/layer9_attention_weight_max": 62.38370895385742,
      "activations/layer9_attention_weight_min": -58.556915283203125,
      "epoch": 3.26,
      "learning_rate": 0.0001303083333333333,
      "loss": 2.8793,
      "step": 56050
    },
    {
      "activations/layer0_attention_weight_max": 14.2191743850708,
      "activations/layer0_attention_weight_min": -14.460564613342285,
      "activations/layer10_attention_weight_max": 39.04908752441406,
      "activations/layer10_attention_weight_min": -33.953582763671875,
      "activations/layer11_attention_weight_max": 35.34404754638672,
      "activations/layer11_attention_weight_min": -27.163061141967773,
      "activations/layer12_attention_weight_max": 30.993328094482422,
      "activations/layer12_attention_weight_min": -24.28284454345703,
      "activations/layer13_attention_weight_max": 45.02368927001953,
      "activations/layer13_attention_weight_min": -37.202789306640625,
      "activations/layer14_attention_weight_max": 41.672203063964844,
      "activations/layer14_attention_weight_min": -30.4853458404541,
      "activations/layer15_attention_weight_max": 42.77326583862305,
      "activations/layer15_attention_weight_min": -33.59162902832031,
      "activations/layer16_attention_weight_max": 35.6417350769043,
      "activations/layer16_attention_weight_min": -25.937711715698242,
      "activations/layer17_attention_weight_max": 58.01509094238281,
      "activations/layer17_attention_weight_min": -33.40840530395508,
      "activations/layer18_attention_weight_max": 49.9427375793457,
      "activations/layer18_attention_weight_min": -32.15013122558594,
      "activations/layer19_attention_weight_max": 20.358232498168945,
      "activations/layer19_attention_weight_min": -19.145626068115234,
      "activations/layer1_attention_weight_max": 16.57602310180664,
      "activations/layer1_attention_weight_min": -13.357429504394531,
      "activations/layer20_attention_weight_max": 20.71908187866211,
      "activations/layer20_attention_weight_min": -18.62071418762207,
      "activations/layer21_attention_weight_max": 36.46632766723633,
      "activations/layer21_attention_weight_min": -27.202035903930664,
      "activations/layer22_attention_weight_max": 34.03709030151367,
      "activations/layer22_attention_weight_min": -24.80975914001465,
      "activations/layer23_attention_weight_max": 23.022647857666016,
      "activations/layer23_attention_weight_min": -17.471500396728516,
      "activations/layer2_attention_weight_max": 31.008840560913086,
      "activations/layer2_attention_weight_min": -27.935691833496094,
      "activations/layer3_attention_weight_max": 74.26715850830078,
      "activations/layer3_attention_weight_min": -79.39362335205078,
      "activations/layer4_attention_weight_max": 104.74491119384766,
      "activations/layer4_attention_weight_min": -97.35965728759766,
      "activations/layer5_attention_weight_max": 121.33468627929688,
      "activations/layer5_attention_weight_min": -100.28611755371094,
      "activations/layer6_attention_weight_max": 61.93423843383789,
      "activations/layer6_attention_weight_min": -50.91648483276367,
      "activations/layer7_attention_weight_max": 60.372215270996094,
      "activations/layer7_attention_weight_min": -55.26661682128906,
      "activations/layer8_attention_weight_max": 40.765174865722656,
      "activations/layer8_attention_weight_min": -34.853240966796875,
      "activations/layer9_attention_weight_max": 52.637123107910156,
      "activations/layer9_attention_weight_min": -46.556732177734375,
      "epoch": 3.26,
      "learning_rate": 0.00013028939393939392,
      "loss": 2.8975,
      "step": 56100
    },
    {
      "activations/layer0_attention_weight_max": 15.005375862121582,
      "activations/layer0_attention_weight_min": -15.032767295837402,
      "activations/layer10_attention_weight_max": 45.786903381347656,
      "activations/layer10_attention_weight_min": -36.15989303588867,
      "activations/layer11_attention_weight_max": 41.52370834350586,
      "activations/layer11_attention_weight_min": -26.997472763061523,
      "activations/layer12_attention_weight_max": 26.317243576049805,
      "activations/layer12_attention_weight_min": -23.661407470703125,
      "activations/layer13_attention_weight_max": 49.45399856567383,
      "activations/layer13_attention_weight_min": -33.65890121459961,
      "activations/layer14_attention_weight_max": 46.597564697265625,
      "activations/layer14_attention_weight_min": -29.845020294189453,
      "activations/layer15_attention_weight_max": 49.70767593383789,
      "activations/layer15_attention_weight_min": -31.727758407592773,
      "activations/layer16_attention_weight_max": 42.703582763671875,
      "activations/layer16_attention_weight_min": -25.28756332397461,
      "activations/layer17_attention_weight_max": 66.12142944335938,
      "activations/layer17_attention_weight_min": -35.45353317260742,
      "activations/layer18_attention_weight_max": 57.02878189086914,
      "activations/layer18_attention_weight_min": -32.208984375,
      "activations/layer19_attention_weight_max": 24.960683822631836,
      "activations/layer19_attention_weight_min": -17.652257919311523,
      "activations/layer1_attention_weight_max": 17.615930557250977,
      "activations/layer1_attention_weight_min": -13.35765266418457,
      "activations/layer20_attention_weight_max": 22.94584846496582,
      "activations/layer20_attention_weight_min": -20.54753875732422,
      "activations/layer21_attention_weight_max": 47.98469161987305,
      "activations/layer21_attention_weight_min": -27.15056037902832,
      "activations/layer22_attention_weight_max": 45.6195068359375,
      "activations/layer22_attention_weight_min": -25.12331199645996,
      "activations/layer23_attention_weight_max": 28.755542755126953,
      "activations/layer23_attention_weight_min": -17.420000076293945,
      "activations/layer2_attention_weight_max": 30.558332443237305,
      "activations/layer2_attention_weight_min": -27.818571090698242,
      "activations/layer3_attention_weight_max": 75.82283782958984,
      "activations/layer3_attention_weight_min": -80.26362609863281,
      "activations/layer4_attention_weight_max": 102.76383209228516,
      "activations/layer4_attention_weight_min": -95.90623474121094,
      "activations/layer5_attention_weight_max": 122.650634765625,
      "activations/layer5_attention_weight_min": -109.74462890625,
      "activations/layer6_attention_weight_max": 61.964195251464844,
      "activations/layer6_attention_weight_min": -54.29902267456055,
      "activations/layer7_attention_weight_max": 62.46687698364258,
      "activations/layer7_attention_weight_min": -57.09259796142578,
      "activations/layer8_attention_weight_max": 39.03742599487305,
      "activations/layer8_attention_weight_min": -36.67982864379883,
      "activations/layer9_attention_weight_max": 49.9830436706543,
      "activations/layer9_attention_weight_min": -47.64120101928711,
      "epoch": 3.26,
      "learning_rate": 0.00013027045454545454,
      "loss": 2.8911,
      "step": 56150
    },
    {
      "activations/layer0_attention_weight_max": 14.709396362304688,
      "activations/layer0_attention_weight_min": -14.852224349975586,
      "activations/layer10_attention_weight_max": 41.435302734375,
      "activations/layer10_attention_weight_min": -34.306705474853516,
      "activations/layer11_attention_weight_max": 34.69744873046875,
      "activations/layer11_attention_weight_min": -25.40782928466797,
      "activations/layer12_attention_weight_max": 25.31617546081543,
      "activations/layer12_attention_weight_min": -25.90450668334961,
      "activations/layer13_attention_weight_max": 57.523963928222656,
      "activations/layer13_attention_weight_min": -40.15859603881836,
      "activations/layer14_attention_weight_max": 53.06834411621094,
      "activations/layer14_attention_weight_min": -36.96327209472656,
      "activations/layer15_attention_weight_max": 49.2502326965332,
      "activations/layer15_attention_weight_min": -31.387290954589844,
      "activations/layer16_attention_weight_max": 39.73171615600586,
      "activations/layer16_attention_weight_min": -28.886075973510742,
      "activations/layer17_attention_weight_max": 64.61064147949219,
      "activations/layer17_attention_weight_min": -35.52695083618164,
      "activations/layer18_attention_weight_max": 57.10793685913086,
      "activations/layer18_attention_weight_min": -34.22274398803711,
      "activations/layer19_attention_weight_max": 23.817108154296875,
      "activations/layer19_attention_weight_min": -19.45937156677246,
      "activations/layer1_attention_weight_max": 17.926239013671875,
      "activations/layer1_attention_weight_min": -16.663055419921875,
      "activations/layer20_attention_weight_max": 22.480758666992188,
      "activations/layer20_attention_weight_min": -20.1168155670166,
      "activations/layer21_attention_weight_max": 46.398712158203125,
      "activations/layer21_attention_weight_min": -29.202518463134766,
      "activations/layer22_attention_weight_max": 41.50849151611328,
      "activations/layer22_attention_weight_min": -23.391355514526367,
      "activations/layer23_attention_weight_max": 25.011985778808594,
      "activations/layer23_attention_weight_min": -17.507108688354492,
      "activations/layer2_attention_weight_max": 30.745193481445312,
      "activations/layer2_attention_weight_min": -28.72357749938965,
      "activations/layer3_attention_weight_max": 74.28263092041016,
      "activations/layer3_attention_weight_min": -78.72486877441406,
      "activations/layer4_attention_weight_max": 100.48091888427734,
      "activations/layer4_attention_weight_min": -97.37480926513672,
      "activations/layer5_attention_weight_max": 116.26265716552734,
      "activations/layer5_attention_weight_min": -105.50257873535156,
      "activations/layer6_attention_weight_max": 57.903053283691406,
      "activations/layer6_attention_weight_min": -51.81885528564453,
      "activations/layer7_attention_weight_max": 61.722599029541016,
      "activations/layer7_attention_weight_min": -58.029945373535156,
      "activations/layer8_attention_weight_max": 39.31499099731445,
      "activations/layer8_attention_weight_min": -36.41244888305664,
      "activations/layer9_attention_weight_max": 51.10316848754883,
      "activations/layer9_attention_weight_min": -46.65639114379883,
      "epoch": 3.27,
      "learning_rate": 0.00013025151515151512,
      "loss": 2.8946,
      "step": 56200
    },
    {
      "activations/layer0_attention_weight_max": 13.919310569763184,
      "activations/layer0_attention_weight_min": -14.769991874694824,
      "activations/layer10_attention_weight_max": 42.70890808105469,
      "activations/layer10_attention_weight_min": -34.242374420166016,
      "activations/layer11_attention_weight_max": 33.63279724121094,
      "activations/layer11_attention_weight_min": -25.857967376708984,
      "activations/layer12_attention_weight_max": 26.322654724121094,
      "activations/layer12_attention_weight_min": -23.22150993347168,
      "activations/layer13_attention_weight_max": 58.125572204589844,
      "activations/layer13_attention_weight_min": -35.932762145996094,
      "activations/layer14_attention_weight_max": 54.06680679321289,
      "activations/layer14_attention_weight_min": -32.92347717285156,
      "activations/layer15_attention_weight_max": 48.06684112548828,
      "activations/layer15_attention_weight_min": -30.94521141052246,
      "activations/layer16_attention_weight_max": 38.70098876953125,
      "activations/layer16_attention_weight_min": -26.149492263793945,
      "activations/layer17_attention_weight_max": 70.43290710449219,
      "activations/layer17_attention_weight_min": -34.717262268066406,
      "activations/layer18_attention_weight_max": 53.31119155883789,
      "activations/layer18_attention_weight_min": -32.68800354003906,
      "activations/layer19_attention_weight_max": 23.84480857849121,
      "activations/layer19_attention_weight_min": -18.087478637695312,
      "activations/layer1_attention_weight_max": 17.73903465270996,
      "activations/layer1_attention_weight_min": -14.016310691833496,
      "activations/layer20_attention_weight_max": 21.83550262451172,
      "activations/layer20_attention_weight_min": -20.46330451965332,
      "activations/layer21_attention_weight_max": 44.53629684448242,
      "activations/layer21_attention_weight_min": -27.995920181274414,
      "activations/layer22_attention_weight_max": 34.508296966552734,
      "activations/layer22_attention_weight_min": -21.370574951171875,
      "activations/layer23_attention_weight_max": 30.189708709716797,
      "activations/layer23_attention_weight_min": -21.21194839477539,
      "activations/layer2_attention_weight_max": 31.641836166381836,
      "activations/layer2_attention_weight_min": -27.364856719970703,
      "activations/layer3_attention_weight_max": 71.13819122314453,
      "activations/layer3_attention_weight_min": -75.53247833251953,
      "activations/layer4_attention_weight_max": 98.25511169433594,
      "activations/layer4_attention_weight_min": -88.44853973388672,
      "activations/layer5_attention_weight_max": 111.96560668945312,
      "activations/layer5_attention_weight_min": -100.05131530761719,
      "activations/layer6_attention_weight_max": 57.985923767089844,
      "activations/layer6_attention_weight_min": -51.42213439941406,
      "activations/layer7_attention_weight_max": 58.61758041381836,
      "activations/layer7_attention_weight_min": -54.26446533203125,
      "activations/layer8_attention_weight_max": 37.27324295043945,
      "activations/layer8_attention_weight_min": -34.76854705810547,
      "activations/layer9_attention_weight_max": 50.76156997680664,
      "activations/layer9_attention_weight_min": -42.59579849243164,
      "epoch": 3.27,
      "learning_rate": 0.00013023257575757574,
      "loss": 2.8899,
      "step": 56250
    },
    {
      "activations/layer0_attention_weight_max": 14.16960620880127,
      "activations/layer0_attention_weight_min": -14.232264518737793,
      "activations/layer10_attention_weight_max": 41.754417419433594,
      "activations/layer10_attention_weight_min": -34.407039642333984,
      "activations/layer11_attention_weight_max": 34.30961608886719,
      "activations/layer11_attention_weight_min": -26.675365447998047,
      "activations/layer12_attention_weight_max": 23.88302230834961,
      "activations/layer12_attention_weight_min": -25.185026168823242,
      "activations/layer13_attention_weight_max": 51.324031829833984,
      "activations/layer13_attention_weight_min": -38.486968994140625,
      "activations/layer14_attention_weight_max": 48.614994049072266,
      "activations/layer14_attention_weight_min": -35.38837814331055,
      "activations/layer15_attention_weight_max": 41.901912689208984,
      "activations/layer15_attention_weight_min": -32.14573287963867,
      "activations/layer16_attention_weight_max": 34.149505615234375,
      "activations/layer16_attention_weight_min": -26.575016021728516,
      "activations/layer17_attention_weight_max": 61.1533088684082,
      "activations/layer17_attention_weight_min": -35.270263671875,
      "activations/layer18_attention_weight_max": 49.37617874145508,
      "activations/layer18_attention_weight_min": -34.149505615234375,
      "activations/layer19_attention_weight_max": 23.325742721557617,
      "activations/layer19_attention_weight_min": -19.679235458374023,
      "activations/layer1_attention_weight_max": 18.134132385253906,
      "activations/layer1_attention_weight_min": -14.344338417053223,
      "activations/layer20_attention_weight_max": 21.58433723449707,
      "activations/layer20_attention_weight_min": -22.393964767456055,
      "activations/layer21_attention_weight_max": 44.732421875,
      "activations/layer21_attention_weight_min": -32.78009796142578,
      "activations/layer22_attention_weight_max": 37.34571838378906,
      "activations/layer22_attention_weight_min": -22.565298080444336,
      "activations/layer23_attention_weight_max": 28.490163803100586,
      "activations/layer23_attention_weight_min": -17.041736602783203,
      "activations/layer2_attention_weight_max": 30.546756744384766,
      "activations/layer2_attention_weight_min": -28.35346221923828,
      "activations/layer3_attention_weight_max": 73.21573638916016,
      "activations/layer3_attention_weight_min": -77.76671600341797,
      "activations/layer4_attention_weight_max": 103.19947814941406,
      "activations/layer4_attention_weight_min": -89.60850524902344,
      "activations/layer5_attention_weight_max": 118.49456787109375,
      "activations/layer5_attention_weight_min": -96.94530487060547,
      "activations/layer6_attention_weight_max": 60.82797622680664,
      "activations/layer6_attention_weight_min": -52.24806213378906,
      "activations/layer7_attention_weight_max": 58.76102828979492,
      "activations/layer7_attention_weight_min": -59.54909896850586,
      "activations/layer8_attention_weight_max": 39.65541076660156,
      "activations/layer8_attention_weight_min": -35.371097564697266,
      "activations/layer9_attention_weight_max": 49.33608627319336,
      "activations/layer9_attention_weight_min": -47.19584274291992,
      "epoch": 3.27,
      "learning_rate": 0.00013021363636363635,
      "loss": 2.8899,
      "step": 56300
    },
    {
      "activations/layer0_attention_weight_max": 14.576953887939453,
      "activations/layer0_attention_weight_min": -14.752758026123047,
      "activations/layer10_attention_weight_max": 40.45594787597656,
      "activations/layer10_attention_weight_min": -34.616085052490234,
      "activations/layer11_attention_weight_max": 37.78262710571289,
      "activations/layer11_attention_weight_min": -27.911212921142578,
      "activations/layer12_attention_weight_max": 23.454254150390625,
      "activations/layer12_attention_weight_min": -33.00508499145508,
      "activations/layer13_attention_weight_max": 45.560733795166016,
      "activations/layer13_attention_weight_min": -34.74958801269531,
      "activations/layer14_attention_weight_max": 46.754512786865234,
      "activations/layer14_attention_weight_min": -33.305572509765625,
      "activations/layer15_attention_weight_max": 43.40630340576172,
      "activations/layer15_attention_weight_min": -32.160301208496094,
      "activations/layer16_attention_weight_max": 34.92295837402344,
      "activations/layer16_attention_weight_min": -26.7380313873291,
      "activations/layer17_attention_weight_max": 58.49629592895508,
      "activations/layer17_attention_weight_min": -35.20286178588867,
      "activations/layer18_attention_weight_max": 47.05276107788086,
      "activations/layer18_attention_weight_min": -31.118698120117188,
      "activations/layer19_attention_weight_max": 21.755109786987305,
      "activations/layer19_attention_weight_min": -17.179655075073242,
      "activations/layer1_attention_weight_max": 17.46831703186035,
      "activations/layer1_attention_weight_min": -14.887965202331543,
      "activations/layer20_attention_weight_max": 19.601722717285156,
      "activations/layer20_attention_weight_min": -19.237491607666016,
      "activations/layer21_attention_weight_max": 36.837921142578125,
      "activations/layer21_attention_weight_min": -26.326255798339844,
      "activations/layer22_attention_weight_max": 37.78132629394531,
      "activations/layer22_attention_weight_min": -23.44322967529297,
      "activations/layer23_attention_weight_max": 25.433752059936523,
      "activations/layer23_attention_weight_min": -16.83092498779297,
      "activations/layer2_attention_weight_max": 30.933307647705078,
      "activations/layer2_attention_weight_min": -28.015003204345703,
      "activations/layer3_attention_weight_max": 78.03964233398438,
      "activations/layer3_attention_weight_min": -79.90491485595703,
      "activations/layer4_attention_weight_max": 107.40180969238281,
      "activations/layer4_attention_weight_min": -96.89803314208984,
      "activations/layer5_attention_weight_max": 119.78077697753906,
      "activations/layer5_attention_weight_min": -105.58788299560547,
      "activations/layer6_attention_weight_max": 59.670372009277344,
      "activations/layer6_attention_weight_min": -52.208309173583984,
      "activations/layer7_attention_weight_max": 61.93178176879883,
      "activations/layer7_attention_weight_min": -59.567726135253906,
      "activations/layer8_attention_weight_max": 40.63722610473633,
      "activations/layer8_attention_weight_min": -37.047664642333984,
      "activations/layer9_attention_weight_max": 54.973114013671875,
      "activations/layer9_attention_weight_min": -52.32638168334961,
      "epoch": 3.27,
      "learning_rate": 0.00013019469696969694,
      "loss": 2.9033,
      "step": 56350
    },
    {
      "activations/layer0_attention_weight_max": 13.976234436035156,
      "activations/layer0_attention_weight_min": -15.401362419128418,
      "activations/layer10_attention_weight_max": 49.8845100402832,
      "activations/layer10_attention_weight_min": -37.34166717529297,
      "activations/layer11_attention_weight_max": 40.440765380859375,
      "activations/layer11_attention_weight_min": -27.8638916015625,
      "activations/layer12_attention_weight_max": 26.49095916748047,
      "activations/layer12_attention_weight_min": -24.934982299804688,
      "activations/layer13_attention_weight_max": 50.85470199584961,
      "activations/layer13_attention_weight_min": -36.97272872924805,
      "activations/layer14_attention_weight_max": 48.23455810546875,
      "activations/layer14_attention_weight_min": -34.1264762878418,
      "activations/layer15_attention_weight_max": 50.18416976928711,
      "activations/layer15_attention_weight_min": -35.84004211425781,
      "activations/layer16_attention_weight_max": 49.099273681640625,
      "activations/layer16_attention_weight_min": -28.06426239013672,
      "activations/layer17_attention_weight_max": 61.22393035888672,
      "activations/layer17_attention_weight_min": -36.994503021240234,
      "activations/layer18_attention_weight_max": 55.24138641357422,
      "activations/layer18_attention_weight_min": -32.72480392456055,
      "activations/layer19_attention_weight_max": 26.50494384765625,
      "activations/layer19_attention_weight_min": -20.615291595458984,
      "activations/layer1_attention_weight_max": 17.13449478149414,
      "activations/layer1_attention_weight_min": -15.480039596557617,
      "activations/layer20_attention_weight_max": 23.037538528442383,
      "activations/layer20_attention_weight_min": -20.598594665527344,
      "activations/layer21_attention_weight_max": 46.6431999206543,
      "activations/layer21_attention_weight_min": -27.783920288085938,
      "activations/layer22_attention_weight_max": 45.049922943115234,
      "activations/layer22_attention_weight_min": -23.38736343383789,
      "activations/layer23_attention_weight_max": 27.630706787109375,
      "activations/layer23_attention_weight_min": -19.00777816772461,
      "activations/layer2_attention_weight_max": 30.08938217163086,
      "activations/layer2_attention_weight_min": -27.324771881103516,
      "activations/layer3_attention_weight_max": 73.16875457763672,
      "activations/layer3_attention_weight_min": -78.57953643798828,
      "activations/layer4_attention_weight_max": 98.87156677246094,
      "activations/layer4_attention_weight_min": -96.24755096435547,
      "activations/layer5_attention_weight_max": 117.29011535644531,
      "activations/layer5_attention_weight_min": -99.03366088867188,
      "activations/layer6_attention_weight_max": 58.97898483276367,
      "activations/layer6_attention_weight_min": -53.67403793334961,
      "activations/layer7_attention_weight_max": 61.94852828979492,
      "activations/layer7_attention_weight_min": -54.972206115722656,
      "activations/layer8_attention_weight_max": 38.455657958984375,
      "activations/layer8_attention_weight_min": -35.11863327026367,
      "activations/layer9_attention_weight_max": 50.1596565246582,
      "activations/layer9_attention_weight_min": -45.488704681396484,
      "epoch": 3.28,
      "learning_rate": 0.00013017575757575756,
      "loss": 2.889,
      "step": 56400
    },
    {
      "activations/layer0_attention_weight_max": 14.089311599731445,
      "activations/layer0_attention_weight_min": -14.684921264648438,
      "activations/layer10_attention_weight_max": 41.29964065551758,
      "activations/layer10_attention_weight_min": -33.91944885253906,
      "activations/layer11_attention_weight_max": 36.78852844238281,
      "activations/layer11_attention_weight_min": -26.094593048095703,
      "activations/layer12_attention_weight_max": 23.92764663696289,
      "activations/layer12_attention_weight_min": -27.907974243164062,
      "activations/layer13_attention_weight_max": 53.33462905883789,
      "activations/layer13_attention_weight_min": -35.997283935546875,
      "activations/layer14_attention_weight_max": 48.289710998535156,
      "activations/layer14_attention_weight_min": -31.199140548706055,
      "activations/layer15_attention_weight_max": 46.24579620361328,
      "activations/layer15_attention_weight_min": -31.338573455810547,
      "activations/layer16_attention_weight_max": 33.434974670410156,
      "activations/layer16_attention_weight_min": -24.39189338684082,
      "activations/layer17_attention_weight_max": 57.40629959106445,
      "activations/layer17_attention_weight_min": -36.25672149658203,
      "activations/layer18_attention_weight_max": 62.37109375,
      "activations/layer18_attention_weight_min": -32.110137939453125,
      "activations/layer19_attention_weight_max": 23.016618728637695,
      "activations/layer19_attention_weight_min": -18.343734741210938,
      "activations/layer1_attention_weight_max": 18.096323013305664,
      "activations/layer1_attention_weight_min": -13.648713111877441,
      "activations/layer20_attention_weight_max": 22.8621883392334,
      "activations/layer20_attention_weight_min": -20.11632537841797,
      "activations/layer21_attention_weight_max": 39.38539123535156,
      "activations/layer21_attention_weight_min": -30.30336570739746,
      "activations/layer22_attention_weight_max": 36.02548599243164,
      "activations/layer22_attention_weight_min": -22.86874771118164,
      "activations/layer23_attention_weight_max": 30.757360458374023,
      "activations/layer23_attention_weight_min": -16.298892974853516,
      "activations/layer2_attention_weight_max": 30.774078369140625,
      "activations/layer2_attention_weight_min": -27.556011199951172,
      "activations/layer3_attention_weight_max": 74.86952209472656,
      "activations/layer3_attention_weight_min": -75.81482696533203,
      "activations/layer4_attention_weight_max": 100.34601593017578,
      "activations/layer4_attention_weight_min": -93.07764434814453,
      "activations/layer5_attention_weight_max": 121.88985443115234,
      "activations/layer5_attention_weight_min": -96.55123138427734,
      "activations/layer6_attention_weight_max": 62.083412170410156,
      "activations/layer6_attention_weight_min": -50.40267562866211,
      "activations/layer7_attention_weight_max": 57.0780029296875,
      "activations/layer7_attention_weight_min": -54.719444274902344,
      "activations/layer8_attention_weight_max": 39.09071350097656,
      "activations/layer8_attention_weight_min": -36.239810943603516,
      "activations/layer9_attention_weight_max": 47.42866134643555,
      "activations/layer9_attention_weight_min": -44.62815856933594,
      "epoch": 3.28,
      "learning_rate": 0.00013015681818181817,
      "loss": 2.8956,
      "step": 56450
    },
    {
      "activations/layer0_attention_weight_max": 13.920636177062988,
      "activations/layer0_attention_weight_min": -14.607564926147461,
      "activations/layer10_attention_weight_max": 47.16636657714844,
      "activations/layer10_attention_weight_min": -33.78541946411133,
      "activations/layer11_attention_weight_max": 34.65304183959961,
      "activations/layer11_attention_weight_min": -27.698698043823242,
      "activations/layer12_attention_weight_max": 28.248783111572266,
      "activations/layer12_attention_weight_min": -23.602933883666992,
      "activations/layer13_attention_weight_max": 53.300682067871094,
      "activations/layer13_attention_weight_min": -36.35274124145508,
      "activations/layer14_attention_weight_max": 51.595176696777344,
      "activations/layer14_attention_weight_min": -34.34124755859375,
      "activations/layer15_attention_weight_max": 46.93007278442383,
      "activations/layer15_attention_weight_min": -31.761899948120117,
      "activations/layer16_attention_weight_max": 38.99819564819336,
      "activations/layer16_attention_weight_min": -27.213754653930664,
      "activations/layer17_attention_weight_max": 61.76598358154297,
      "activations/layer17_attention_weight_min": -36.00438690185547,
      "activations/layer18_attention_weight_max": 54.7720947265625,
      "activations/layer18_attention_weight_min": -30.353172302246094,
      "activations/layer19_attention_weight_max": 24.578210830688477,
      "activations/layer19_attention_weight_min": -18.80391502380371,
      "activations/layer1_attention_weight_max": 17.20520782470703,
      "activations/layer1_attention_weight_min": -14.71802806854248,
      "activations/layer20_attention_weight_max": 21.010398864746094,
      "activations/layer20_attention_weight_min": -20.081937789916992,
      "activations/layer21_attention_weight_max": 40.70823287963867,
      "activations/layer21_attention_weight_min": -27.375225067138672,
      "activations/layer22_attention_weight_max": 34.4883918762207,
      "activations/layer22_attention_weight_min": -23.031951904296875,
      "activations/layer23_attention_weight_max": 24.81003189086914,
      "activations/layer23_attention_weight_min": -17.487762451171875,
      "activations/layer2_attention_weight_max": 30.82323455810547,
      "activations/layer2_attention_weight_min": -27.21704864501953,
      "activations/layer3_attention_weight_max": 70.033203125,
      "activations/layer3_attention_weight_min": -74.92924499511719,
      "activations/layer4_attention_weight_max": 97.87740325927734,
      "activations/layer4_attention_weight_min": -88.49357604980469,
      "activations/layer5_attention_weight_max": 122.93395233154297,
      "activations/layer5_attention_weight_min": -98.47282409667969,
      "activations/layer6_attention_weight_max": 62.582061767578125,
      "activations/layer6_attention_weight_min": -51.18006896972656,
      "activations/layer7_attention_weight_max": 60.07587814331055,
      "activations/layer7_attention_weight_min": -52.90104675292969,
      "activations/layer8_attention_weight_max": 41.853782653808594,
      "activations/layer8_attention_weight_min": -35.522342681884766,
      "activations/layer9_attention_weight_max": 52.1144905090332,
      "activations/layer9_attention_weight_min": -45.783260345458984,
      "epoch": 3.28,
      "learning_rate": 0.00013013787878787876,
      "loss": 2.8887,
      "step": 56500
    },
    {
      "activations/layer0_attention_weight_max": 14.065823554992676,
      "activations/layer0_attention_weight_min": -14.767634391784668,
      "activations/layer10_attention_weight_max": 45.70673370361328,
      "activations/layer10_attention_weight_min": -35.753883361816406,
      "activations/layer11_attention_weight_max": 35.48258972167969,
      "activations/layer11_attention_weight_min": -27.572269439697266,
      "activations/layer12_attention_weight_max": 26.805612564086914,
      "activations/layer12_attention_weight_min": -25.15662956237793,
      "activations/layer13_attention_weight_max": 70.16302490234375,
      "activations/layer13_attention_weight_min": -47.300777435302734,
      "activations/layer14_attention_weight_max": 46.84291458129883,
      "activations/layer14_attention_weight_min": -36.144447326660156,
      "activations/layer15_attention_weight_max": 61.45740509033203,
      "activations/layer15_attention_weight_min": -36.56867599487305,
      "activations/layer16_attention_weight_max": 38.08540725708008,
      "activations/layer16_attention_weight_min": -26.82891845703125,
      "activations/layer17_attention_weight_max": 57.56742477416992,
      "activations/layer17_attention_weight_min": -36.296714782714844,
      "activations/layer18_attention_weight_max": 56.77880096435547,
      "activations/layer18_attention_weight_min": -35.89202117919922,
      "activations/layer19_attention_weight_max": 24.87032699584961,
      "activations/layer19_attention_weight_min": -18.593290328979492,
      "activations/layer1_attention_weight_max": 17.02401351928711,
      "activations/layer1_attention_weight_min": -16.054872512817383,
      "activations/layer20_attention_weight_max": 21.452938079833984,
      "activations/layer20_attention_weight_min": -19.092998504638672,
      "activations/layer21_attention_weight_max": 41.34164810180664,
      "activations/layer21_attention_weight_min": -30.993303298950195,
      "activations/layer22_attention_weight_max": 33.91173553466797,
      "activations/layer22_attention_weight_min": -22.066667556762695,
      "activations/layer23_attention_weight_max": 26.598175048828125,
      "activations/layer23_attention_weight_min": -20.024913787841797,
      "activations/layer2_attention_weight_max": 29.36971664428711,
      "activations/layer2_attention_weight_min": -27.85074234008789,
      "activations/layer3_attention_weight_max": 72.22369384765625,
      "activations/layer3_attention_weight_min": -77.35102844238281,
      "activations/layer4_attention_weight_max": 102.10130310058594,
      "activations/layer4_attention_weight_min": -91.37220001220703,
      "activations/layer5_attention_weight_max": 120.06742858886719,
      "activations/layer5_attention_weight_min": -101.67953491210938,
      "activations/layer6_attention_weight_max": 60.56096649169922,
      "activations/layer6_attention_weight_min": -54.13162612915039,
      "activations/layer7_attention_weight_max": 62.79905319213867,
      "activations/layer7_attention_weight_min": -58.2292594909668,
      "activations/layer8_attention_weight_max": 40.300506591796875,
      "activations/layer8_attention_weight_min": -35.23246765136719,
      "activations/layer9_attention_weight_max": 47.764522552490234,
      "activations/layer9_attention_weight_min": -46.37001419067383,
      "epoch": 3.29,
      "learning_rate": 0.00013011893939393938,
      "loss": 2.9057,
      "step": 56550
    },
    {
      "activations/layer0_attention_weight_max": 14.832355499267578,
      "activations/layer0_attention_weight_min": -14.397250175476074,
      "activations/layer10_attention_weight_max": 49.778221130371094,
      "activations/layer10_attention_weight_min": -36.03156280517578,
      "activations/layer11_attention_weight_max": 43.342552185058594,
      "activations/layer11_attention_weight_min": -29.160564422607422,
      "activations/layer12_attention_weight_max": 26.67814064025879,
      "activations/layer12_attention_weight_min": -24.250120162963867,
      "activations/layer13_attention_weight_max": 70.56165313720703,
      "activations/layer13_attention_weight_min": -47.505741119384766,
      "activations/layer14_attention_weight_max": 56.098854064941406,
      "activations/layer14_attention_weight_min": -32.98711395263672,
      "activations/layer15_attention_weight_max": 41.18752670288086,
      "activations/layer15_attention_weight_min": -30.355493545532227,
      "activations/layer16_attention_weight_max": 39.948177337646484,
      "activations/layer16_attention_weight_min": -24.433792114257812,
      "activations/layer17_attention_weight_max": 59.93971633911133,
      "activations/layer17_attention_weight_min": -36.667354583740234,
      "activations/layer18_attention_weight_max": 49.539363861083984,
      "activations/layer18_attention_weight_min": -30.637845993041992,
      "activations/layer19_attention_weight_max": 20.02876091003418,
      "activations/layer19_attention_weight_min": -17.230365753173828,
      "activations/layer1_attention_weight_max": 17.35704231262207,
      "activations/layer1_attention_weight_min": -16.278528213500977,
      "activations/layer20_attention_weight_max": 19.764904022216797,
      "activations/layer20_attention_weight_min": -21.53012466430664,
      "activations/layer21_attention_weight_max": 42.33004379272461,
      "activations/layer21_attention_weight_min": -27.419414520263672,
      "activations/layer22_attention_weight_max": 30.784400939941406,
      "activations/layer22_attention_weight_min": -21.375015258789062,
      "activations/layer23_attention_weight_max": 24.83582878112793,
      "activations/layer23_attention_weight_min": -16.221820831298828,
      "activations/layer2_attention_weight_max": 30.202369689941406,
      "activations/layer2_attention_weight_min": -28.292224884033203,
      "activations/layer3_attention_weight_max": 77.45736694335938,
      "activations/layer3_attention_weight_min": -77.98971557617188,
      "activations/layer4_attention_weight_max": 104.34320068359375,
      "activations/layer4_attention_weight_min": -93.49674987792969,
      "activations/layer5_attention_weight_max": 128.74288940429688,
      "activations/layer5_attention_weight_min": -101.83204650878906,
      "activations/layer6_attention_weight_max": 64.26835632324219,
      "activations/layer6_attention_weight_min": -54.645999908447266,
      "activations/layer7_attention_weight_max": 75.74154663085938,
      "activations/layer7_attention_weight_min": -58.41183090209961,
      "activations/layer8_attention_weight_max": 47.56269454956055,
      "activations/layer8_attention_weight_min": -35.64237594604492,
      "activations/layer9_attention_weight_max": 61.38874435424805,
      "activations/layer9_attention_weight_min": -47.12033462524414,
      "epoch": 3.29,
      "learning_rate": 0.0001301,
      "loss": 2.8968,
      "step": 56600
    },
    {
      "activations/layer0_attention_weight_max": 14.315677642822266,
      "activations/layer0_attention_weight_min": -14.362944602966309,
      "activations/layer10_attention_weight_max": 41.55841064453125,
      "activations/layer10_attention_weight_min": -32.861045837402344,
      "activations/layer11_attention_weight_max": 33.104000091552734,
      "activations/layer11_attention_weight_min": -25.165693283081055,
      "activations/layer12_attention_weight_max": 27.410749435424805,
      "activations/layer12_attention_weight_min": -25.842863082885742,
      "activations/layer13_attention_weight_max": 49.05551528930664,
      "activations/layer13_attention_weight_min": -36.693084716796875,
      "activations/layer14_attention_weight_max": 44.12739944458008,
      "activations/layer14_attention_weight_min": -28.91777992248535,
      "activations/layer15_attention_weight_max": 46.283565521240234,
      "activations/layer15_attention_weight_min": -29.729541778564453,
      "activations/layer16_attention_weight_max": 40.37680435180664,
      "activations/layer16_attention_weight_min": -24.511199951171875,
      "activations/layer17_attention_weight_max": 54.97185134887695,
      "activations/layer17_attention_weight_min": -35.81603240966797,
      "activations/layer18_attention_weight_max": 49.75019073486328,
      "activations/layer18_attention_weight_min": -30.9588623046875,
      "activations/layer19_attention_weight_max": 19.911479949951172,
      "activations/layer19_attention_weight_min": -18.167814254760742,
      "activations/layer1_attention_weight_max": 17.22020721435547,
      "activations/layer1_attention_weight_min": -13.618393898010254,
      "activations/layer20_attention_weight_max": 19.631528854370117,
      "activations/layer20_attention_weight_min": -18.45585060119629,
      "activations/layer21_attention_weight_max": 35.73799514770508,
      "activations/layer21_attention_weight_min": -29.176176071166992,
      "activations/layer22_attention_weight_max": 32.46810531616211,
      "activations/layer22_attention_weight_min": -22.301681518554688,
      "activations/layer23_attention_weight_max": 23.28600311279297,
      "activations/layer23_attention_weight_min": -18.83578872680664,
      "activations/layer2_attention_weight_max": 28.531814575195312,
      "activations/layer2_attention_weight_min": -27.24826431274414,
      "activations/layer3_attention_weight_max": 73.2424087524414,
      "activations/layer3_attention_weight_min": -78.31362915039062,
      "activations/layer4_attention_weight_max": 100.5704345703125,
      "activations/layer4_attention_weight_min": -94.01634216308594,
      "activations/layer5_attention_weight_max": 115.71795654296875,
      "activations/layer5_attention_weight_min": -106.65319061279297,
      "activations/layer6_attention_weight_max": 56.65813446044922,
      "activations/layer6_attention_weight_min": -55.07515335083008,
      "activations/layer7_attention_weight_max": 61.547611236572266,
      "activations/layer7_attention_weight_min": -56.56032943725586,
      "activations/layer8_attention_weight_max": 38.86137771606445,
      "activations/layer8_attention_weight_min": -38.11140823364258,
      "activations/layer9_attention_weight_max": 49.81916427612305,
      "activations/layer9_attention_weight_min": -44.271385192871094,
      "epoch": 3.29,
      "learning_rate": 0.0001300810606060606,
      "loss": 2.8799,
      "step": 56650
    },
    {
      "activations/layer0_attention_weight_max": 13.450424194335938,
      "activations/layer0_attention_weight_min": -15.61062240600586,
      "activations/layer10_attention_weight_max": 42.424537658691406,
      "activations/layer10_attention_weight_min": -35.91773223876953,
      "activations/layer11_attention_weight_max": 34.592159271240234,
      "activations/layer11_attention_weight_min": -27.31460189819336,
      "activations/layer12_attention_weight_max": 24.941686630249023,
      "activations/layer12_attention_weight_min": -24.43478775024414,
      "activations/layer13_attention_weight_max": 63.0732307434082,
      "activations/layer13_attention_weight_min": -43.77369689941406,
      "activations/layer14_attention_weight_max": 58.460411071777344,
      "activations/layer14_attention_weight_min": -36.481719970703125,
      "activations/layer15_attention_weight_max": 42.35312271118164,
      "activations/layer15_attention_weight_min": -31.729305267333984,
      "activations/layer16_attention_weight_max": 34.76782989501953,
      "activations/layer16_attention_weight_min": -27.474000930786133,
      "activations/layer17_attention_weight_max": 59.04638671875,
      "activations/layer17_attention_weight_min": -35.6961555480957,
      "activations/layer18_attention_weight_max": 49.117698669433594,
      "activations/layer18_attention_weight_min": -32.5303840637207,
      "activations/layer19_attention_weight_max": 19.695093154907227,
      "activations/layer19_attention_weight_min": -17.673555374145508,
      "activations/layer1_attention_weight_max": 16.960399627685547,
      "activations/layer1_attention_weight_min": -15.569711685180664,
      "activations/layer20_attention_weight_max": 21.49822235107422,
      "activations/layer20_attention_weight_min": -19.33384895324707,
      "activations/layer21_attention_weight_max": 39.413822174072266,
      "activations/layer21_attention_weight_min": -26.271099090576172,
      "activations/layer22_attention_weight_max": 36.06990432739258,
      "activations/layer22_attention_weight_min": -24.90826416015625,
      "activations/layer23_attention_weight_max": 25.332258224487305,
      "activations/layer23_attention_weight_min": -16.938003540039062,
      "activations/layer2_attention_weight_max": 28.91574478149414,
      "activations/layer2_attention_weight_min": -26.936199188232422,
      "activations/layer3_attention_weight_max": 73.6092529296875,
      "activations/layer3_attention_weight_min": -77.84813690185547,
      "activations/layer4_attention_weight_max": 102.35100555419922,
      "activations/layer4_attention_weight_min": -93.1711654663086,
      "activations/layer5_attention_weight_max": 120.23818969726562,
      "activations/layer5_attention_weight_min": -101.81622314453125,
      "activations/layer6_attention_weight_max": 62.35369873046875,
      "activations/layer6_attention_weight_min": -50.14781188964844,
      "activations/layer7_attention_weight_max": 61.832542419433594,
      "activations/layer7_attention_weight_min": -56.55998611450195,
      "activations/layer8_attention_weight_max": 39.6721076965332,
      "activations/layer8_attention_weight_min": -36.20289993286133,
      "activations/layer9_attention_weight_max": 50.423709869384766,
      "activations/layer9_attention_weight_min": -55.9271354675293,
      "epoch": 3.29,
      "learning_rate": 0.0001300621212121212,
      "loss": 2.8907,
      "step": 56700
    },
    {
      "activations/layer0_attention_weight_max": 15.531241416931152,
      "activations/layer0_attention_weight_min": -14.65758228302002,
      "activations/layer10_attention_weight_max": 44.34423828125,
      "activations/layer10_attention_weight_min": -39.64885330200195,
      "activations/layer11_attention_weight_max": 37.01526641845703,
      "activations/layer11_attention_weight_min": -30.202123641967773,
      "activations/layer12_attention_weight_max": 22.96143913269043,
      "activations/layer12_attention_weight_min": -25.32035255432129,
      "activations/layer13_attention_weight_max": 62.20945358276367,
      "activations/layer13_attention_weight_min": -43.64896011352539,
      "activations/layer14_attention_weight_max": 74.36316680908203,
      "activations/layer14_attention_weight_min": -54.26571273803711,
      "activations/layer15_attention_weight_max": 52.18988800048828,
      "activations/layer15_attention_weight_min": -41.63544464111328,
      "activations/layer16_attention_weight_max": 35.46268844604492,
      "activations/layer16_attention_weight_min": -27.241479873657227,
      "activations/layer17_attention_weight_max": 56.42634963989258,
      "activations/layer17_attention_weight_min": -35.82145690917969,
      "activations/layer18_attention_weight_max": 56.68935775756836,
      "activations/layer18_attention_weight_min": -31.689058303833008,
      "activations/layer19_attention_weight_max": 23.467016220092773,
      "activations/layer19_attention_weight_min": -18.683670043945312,
      "activations/layer1_attention_weight_max": 17.40604019165039,
      "activations/layer1_attention_weight_min": -16.32208251953125,
      "activations/layer20_attention_weight_max": 19.592601776123047,
      "activations/layer20_attention_weight_min": -20.586570739746094,
      "activations/layer21_attention_weight_max": 44.00529861450195,
      "activations/layer21_attention_weight_min": -26.12108039855957,
      "activations/layer22_attention_weight_max": 36.61414337158203,
      "activations/layer22_attention_weight_min": -23.220531463623047,
      "activations/layer23_attention_weight_max": 25.273365020751953,
      "activations/layer23_attention_weight_min": -16.309146881103516,
      "activations/layer2_attention_weight_max": 30.06574821472168,
      "activations/layer2_attention_weight_min": -27.853389739990234,
      "activations/layer3_attention_weight_max": 72.91675567626953,
      "activations/layer3_attention_weight_min": -76.31298828125,
      "activations/layer4_attention_weight_max": 103.73149871826172,
      "activations/layer4_attention_weight_min": -91.02608489990234,
      "activations/layer5_attention_weight_max": 121.70894622802734,
      "activations/layer5_attention_weight_min": -100.81685638427734,
      "activations/layer6_attention_weight_max": 61.09857177734375,
      "activations/layer6_attention_weight_min": -51.90574264526367,
      "activations/layer7_attention_weight_max": 61.58186340332031,
      "activations/layer7_attention_weight_min": -58.863319396972656,
      "activations/layer8_attention_weight_max": 43.69752502441406,
      "activations/layer8_attention_weight_min": -38.03234100341797,
      "activations/layer9_attention_weight_max": 55.49201583862305,
      "activations/layer9_attention_weight_min": -49.61066818237305,
      "epoch": 3.3,
      "learning_rate": 0.0001300431818181818,
      "loss": 2.9,
      "step": 56750
    },
    {
      "activations/layer0_attention_weight_max": 14.418831825256348,
      "activations/layer0_attention_weight_min": -14.95191478729248,
      "activations/layer10_attention_weight_max": 51.184356689453125,
      "activations/layer10_attention_weight_min": -35.70280838012695,
      "activations/layer11_attention_weight_max": 39.81804656982422,
      "activations/layer11_attention_weight_min": -27.7247314453125,
      "activations/layer12_attention_weight_max": 34.05410385131836,
      "activations/layer12_attention_weight_min": -24.79928970336914,
      "activations/layer13_attention_weight_max": 86.43730926513672,
      "activations/layer13_attention_weight_min": -47.45145034790039,
      "activations/layer14_attention_weight_max": 89.93000793457031,
      "activations/layer14_attention_weight_min": -46.008209228515625,
      "activations/layer15_attention_weight_max": 55.62359619140625,
      "activations/layer15_attention_weight_min": -32.62237548828125,
      "activations/layer16_attention_weight_max": 50.636077880859375,
      "activations/layer16_attention_weight_min": -26.6560115814209,
      "activations/layer17_attention_weight_max": 66.55583190917969,
      "activations/layer17_attention_weight_min": -35.81303024291992,
      "activations/layer18_attention_weight_max": 62.796630859375,
      "activations/layer18_attention_weight_min": -31.716594696044922,
      "activations/layer19_attention_weight_max": 24.216161727905273,
      "activations/layer19_attention_weight_min": -19.29960060119629,
      "activations/layer1_attention_weight_max": 17.42519187927246,
      "activations/layer1_attention_weight_min": -16.208370208740234,
      "activations/layer20_attention_weight_max": 23.426292419433594,
      "activations/layer20_attention_weight_min": -17.779773712158203,
      "activations/layer21_attention_weight_max": 56.57157897949219,
      "activations/layer21_attention_weight_min": -29.384994506835938,
      "activations/layer22_attention_weight_max": 42.21708297729492,
      "activations/layer22_attention_weight_min": -22.38890838623047,
      "activations/layer23_attention_weight_max": 31.325340270996094,
      "activations/layer23_attention_weight_min": -16.849788665771484,
      "activations/layer2_attention_weight_max": 30.352758407592773,
      "activations/layer2_attention_weight_min": -27.865713119506836,
      "activations/layer3_attention_weight_max": 77.12036895751953,
      "activations/layer3_attention_weight_min": -79.3751220703125,
      "activations/layer4_attention_weight_max": 105.66924285888672,
      "activations/layer4_attention_weight_min": -100.64531707763672,
      "activations/layer5_attention_weight_max": 124.32066345214844,
      "activations/layer5_attention_weight_min": -107.59648132324219,
      "activations/layer6_attention_weight_max": 62.928367614746094,
      "activations/layer6_attention_weight_min": -55.36659622192383,
      "activations/layer7_attention_weight_max": 71.8796615600586,
      "activations/layer7_attention_weight_min": -63.66727066040039,
      "activations/layer8_attention_weight_max": 46.50813293457031,
      "activations/layer8_attention_weight_min": -42.766685485839844,
      "activations/layer9_attention_weight_max": 66.90338897705078,
      "activations/layer9_attention_weight_min": -54.1786003112793,
      "epoch": 3.3,
      "learning_rate": 0.00013002424242424242,
      "loss": 2.8911,
      "step": 56800
    },
    {
      "activations/layer0_attention_weight_max": 14.975984573364258,
      "activations/layer0_attention_weight_min": -14.799858093261719,
      "activations/layer10_attention_weight_max": 47.425010681152344,
      "activations/layer10_attention_weight_min": -34.68599319458008,
      "activations/layer11_attention_weight_max": 39.822296142578125,
      "activations/layer11_attention_weight_min": -26.26091766357422,
      "activations/layer12_attention_weight_max": 27.461149215698242,
      "activations/layer12_attention_weight_min": -27.44597053527832,
      "activations/layer13_attention_weight_max": 58.661163330078125,
      "activations/layer13_attention_weight_min": -40.8531494140625,
      "activations/layer14_attention_weight_max": 56.75944900512695,
      "activations/layer14_attention_weight_min": -37.31023025512695,
      "activations/layer15_attention_weight_max": 47.671321868896484,
      "activations/layer15_attention_weight_min": -33.12828063964844,
      "activations/layer16_attention_weight_max": 41.607887268066406,
      "activations/layer16_attention_weight_min": -25.88546371459961,
      "activations/layer17_attention_weight_max": 69.7356185913086,
      "activations/layer17_attention_weight_min": -38.295448303222656,
      "activations/layer18_attention_weight_max": 53.663307189941406,
      "activations/layer18_attention_weight_min": -32.98554611206055,
      "activations/layer19_attention_weight_max": 22.452672958374023,
      "activations/layer19_attention_weight_min": -17.495878219604492,
      "activations/layer1_attention_weight_max": 17.480880737304688,
      "activations/layer1_attention_weight_min": -15.870770454406738,
      "activations/layer20_attention_weight_max": 23.020666122436523,
      "activations/layer20_attention_weight_min": -20.111909866333008,
      "activations/layer21_attention_weight_max": 47.97230911254883,
      "activations/layer21_attention_weight_min": -26.435203552246094,
      "activations/layer22_attention_weight_max": 36.05681228637695,
      "activations/layer22_attention_weight_min": -27.20354461669922,
      "activations/layer23_attention_weight_max": 23.20796775817871,
      "activations/layer23_attention_weight_min": -15.506827354431152,
      "activations/layer2_attention_weight_max": 31.803993225097656,
      "activations/layer2_attention_weight_min": -27.459104537963867,
      "activations/layer3_attention_weight_max": 76.00218963623047,
      "activations/layer3_attention_weight_min": -77.7948989868164,
      "activations/layer4_attention_weight_max": 100.65470123291016,
      "activations/layer4_attention_weight_min": -95.77754974365234,
      "activations/layer5_attention_weight_max": 120.69105529785156,
      "activations/layer5_attention_weight_min": -104.42994689941406,
      "activations/layer6_attention_weight_max": 64.72331237792969,
      "activations/layer6_attention_weight_min": -53.62639236450195,
      "activations/layer7_attention_weight_max": 64.56344604492188,
      "activations/layer7_attention_weight_min": -56.668880462646484,
      "activations/layer8_attention_weight_max": 40.39939498901367,
      "activations/layer8_attention_weight_min": -36.50113296508789,
      "activations/layer9_attention_weight_max": 58.58991241455078,
      "activations/layer9_attention_weight_min": -48.95777893066406,
      "epoch": 3.3,
      "learning_rate": 0.00013000530303030304,
      "loss": 2.8661,
      "step": 56850
    },
    {
      "activations/layer0_attention_weight_max": 15.467334747314453,
      "activations/layer0_attention_weight_min": -14.531373977661133,
      "activations/layer10_attention_weight_max": 42.21027755737305,
      "activations/layer10_attention_weight_min": -37.00419998168945,
      "activations/layer11_attention_weight_max": 33.30003356933594,
      "activations/layer11_attention_weight_min": -30.86652183532715,
      "activations/layer12_attention_weight_max": 30.462373733520508,
      "activations/layer12_attention_weight_min": -26.567951202392578,
      "activations/layer13_attention_weight_max": 51.387821197509766,
      "activations/layer13_attention_weight_min": -37.49340057373047,
      "activations/layer14_attention_weight_max": 48.55617904663086,
      "activations/layer14_attention_weight_min": -35.653438568115234,
      "activations/layer15_attention_weight_max": 48.38657760620117,
      "activations/layer15_attention_weight_min": -33.812232971191406,
      "activations/layer16_attention_weight_max": 36.90134811401367,
      "activations/layer16_attention_weight_min": -27.01818084716797,
      "activations/layer17_attention_weight_max": 56.60213088989258,
      "activations/layer17_attention_weight_min": -35.753238677978516,
      "activations/layer18_attention_weight_max": 53.13776779174805,
      "activations/layer18_attention_weight_min": -32.082359313964844,
      "activations/layer19_attention_weight_max": 23.705062866210938,
      "activations/layer19_attention_weight_min": -16.166868209838867,
      "activations/layer1_attention_weight_max": 17.548236846923828,
      "activations/layer1_attention_weight_min": -13.07862377166748,
      "activations/layer20_attention_weight_max": 20.341533660888672,
      "activations/layer20_attention_weight_min": -18.36076545715332,
      "activations/layer21_attention_weight_max": 40.49264144897461,
      "activations/layer21_attention_weight_min": -28.460966110229492,
      "activations/layer22_attention_weight_max": 34.57301330566406,
      "activations/layer22_attention_weight_min": -23.65199851989746,
      "activations/layer23_attention_weight_max": 26.89252471923828,
      "activations/layer23_attention_weight_min": -16.173385620117188,
      "activations/layer2_attention_weight_max": 29.085891723632812,
      "activations/layer2_attention_weight_min": -27.942108154296875,
      "activations/layer3_attention_weight_max": 71.12713623046875,
      "activations/layer3_attention_weight_min": -73.847412109375,
      "activations/layer4_attention_weight_max": 97.26371765136719,
      "activations/layer4_attention_weight_min": -92.63166046142578,
      "activations/layer5_attention_weight_max": 115.27374267578125,
      "activations/layer5_attention_weight_min": -100.1364974975586,
      "activations/layer6_attention_weight_max": 59.323890686035156,
      "activations/layer6_attention_weight_min": -53.623558044433594,
      "activations/layer7_attention_weight_max": 61.47734832763672,
      "activations/layer7_attention_weight_min": -58.147216796875,
      "activations/layer8_attention_weight_max": 40.53037643432617,
      "activations/layer8_attention_weight_min": -38.387054443359375,
      "activations/layer9_attention_weight_max": 58.433837890625,
      "activations/layer9_attention_weight_min": -51.87129592895508,
      "epoch": 3.31,
      "learning_rate": 0.00012998636363636363,
      "loss": 2.8957,
      "step": 56900
    },
    {
      "activations/layer0_attention_weight_max": 13.596109390258789,
      "activations/layer0_attention_weight_min": -14.550341606140137,
      "activations/layer10_attention_weight_max": 48.234710693359375,
      "activations/layer10_attention_weight_min": -34.7618293762207,
      "activations/layer11_attention_weight_max": 35.34736633300781,
      "activations/layer11_attention_weight_min": -26.362058639526367,
      "activations/layer12_attention_weight_max": 26.574525833129883,
      "activations/layer12_attention_weight_min": -28.534835815429688,
      "activations/layer13_attention_weight_max": 63.56269836425781,
      "activations/layer13_attention_weight_min": -41.8625373840332,
      "activations/layer14_attention_weight_max": 64.2991714477539,
      "activations/layer14_attention_weight_min": -36.06365966796875,
      "activations/layer15_attention_weight_max": 51.81076431274414,
      "activations/layer15_attention_weight_min": -31.640470504760742,
      "activations/layer16_attention_weight_max": 37.84874725341797,
      "activations/layer16_attention_weight_min": -27.844831466674805,
      "activations/layer17_attention_weight_max": 61.68451690673828,
      "activations/layer17_attention_weight_min": -35.76543045043945,
      "activations/layer18_attention_weight_max": 53.76549530029297,
      "activations/layer18_attention_weight_min": -31.286502838134766,
      "activations/layer19_attention_weight_max": 26.96546745300293,
      "activations/layer19_attention_weight_min": -17.868995666503906,
      "activations/layer1_attention_weight_max": 17.164066314697266,
      "activations/layer1_attention_weight_min": -15.276737213134766,
      "activations/layer20_attention_weight_max": 23.82176971435547,
      "activations/layer20_attention_weight_min": -19.31453514099121,
      "activations/layer21_attention_weight_max": 41.80501174926758,
      "activations/layer21_attention_weight_min": -24.51157569885254,
      "activations/layer22_attention_weight_max": 37.261558532714844,
      "activations/layer22_attention_weight_min": -21.940738677978516,
      "activations/layer23_attention_weight_max": 29.96685791015625,
      "activations/layer23_attention_weight_min": -15.623554229736328,
      "activations/layer2_attention_weight_max": 29.262861251831055,
      "activations/layer2_attention_weight_min": -26.145307540893555,
      "activations/layer3_attention_weight_max": 75.41407012939453,
      "activations/layer3_attention_weight_min": -75.7483901977539,
      "activations/layer4_attention_weight_max": 99.33035278320312,
      "activations/layer4_attention_weight_min": -90.10768127441406,
      "activations/layer5_attention_weight_max": 120.01260375976562,
      "activations/layer5_attention_weight_min": -102.03184509277344,
      "activations/layer6_attention_weight_max": 60.87080383300781,
      "activations/layer6_attention_weight_min": -55.60782241821289,
      "activations/layer7_attention_weight_max": 65.88459777832031,
      "activations/layer7_attention_weight_min": -57.54573059082031,
      "activations/layer8_attention_weight_max": 40.47189712524414,
      "activations/layer8_attention_weight_min": -38.69309997558594,
      "activations/layer9_attention_weight_max": 56.7384147644043,
      "activations/layer9_attention_weight_min": -46.35139083862305,
      "epoch": 3.31,
      "learning_rate": 0.00012996742424242424,
      "loss": 2.8898,
      "step": 56950
    },
    {
      "activations/layer0_attention_weight_max": 15.396976470947266,
      "activations/layer0_attention_weight_min": -14.705178260803223,
      "activations/layer10_attention_weight_max": 51.38309860229492,
      "activations/layer10_attention_weight_min": -37.96769332885742,
      "activations/layer11_attention_weight_max": 45.94667053222656,
      "activations/layer11_attention_weight_min": -27.606067657470703,
      "activations/layer12_attention_weight_max": 25.97840118408203,
      "activations/layer12_attention_weight_min": -25.09621810913086,
      "activations/layer13_attention_weight_max": 79.31057739257812,
      "activations/layer13_attention_weight_min": -40.85357666015625,
      "activations/layer14_attention_weight_max": 105.87760925292969,
      "activations/layer14_attention_weight_min": -51.06654357910156,
      "activations/layer15_attention_weight_max": 65.82211303710938,
      "activations/layer15_attention_weight_min": -31.819791793823242,
      "activations/layer16_attention_weight_max": 41.817874908447266,
      "activations/layer16_attention_weight_min": -25.594621658325195,
      "activations/layer17_attention_weight_max": 59.61386489868164,
      "activations/layer17_attention_weight_min": -33.76946258544922,
      "activations/layer18_attention_weight_max": 51.115543365478516,
      "activations/layer18_attention_weight_min": -29.8107967376709,
      "activations/layer19_attention_weight_max": 24.47062110900879,
      "activations/layer19_attention_weight_min": -19.057050704956055,
      "activations/layer1_attention_weight_max": 18.89891242980957,
      "activations/layer1_attention_weight_min": -15.110394477844238,
      "activations/layer20_attention_weight_max": 20.88661003112793,
      "activations/layer20_attention_weight_min": -19.796709060668945,
      "activations/layer21_attention_weight_max": 38.395050048828125,
      "activations/layer21_attention_weight_min": -26.881872177124023,
      "activations/layer22_attention_weight_max": 32.571693420410156,
      "activations/layer22_attention_weight_min": -20.698753356933594,
      "activations/layer23_attention_weight_max": 24.21047019958496,
      "activations/layer23_attention_weight_min": -15.009576797485352,
      "activations/layer2_attention_weight_max": 28.239553451538086,
      "activations/layer2_attention_weight_min": -26.903522491455078,
      "activations/layer3_attention_weight_max": 70.62751770019531,
      "activations/layer3_attention_weight_min": -72.30709838867188,
      "activations/layer4_attention_weight_max": 103.30742645263672,
      "activations/layer4_attention_weight_min": -90.79830169677734,
      "activations/layer5_attention_weight_max": 122.65984344482422,
      "activations/layer5_attention_weight_min": -102.07857513427734,
      "activations/layer6_attention_weight_max": 62.379364013671875,
      "activations/layer6_attention_weight_min": -50.923526763916016,
      "activations/layer7_attention_weight_max": 76.66624450683594,
      "activations/layer7_attention_weight_min": -55.5294303894043,
      "activations/layer8_attention_weight_max": 51.275115966796875,
      "activations/layer8_attention_weight_min": -37.84364318847656,
      "activations/layer9_attention_weight_max": 75.45765686035156,
      "activations/layer9_attention_weight_min": -58.85681915283203,
      "epoch": 3.31,
      "learning_rate": 0.00012994848484848483,
      "loss": 2.8961,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_loss": 2.84375,
      "eval_runtime": 8.406,
      "eval_samples_per_second": 510.823,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_openwebtext_loss": 2.84375,
      "eval_openwebtext_ppl": 17.180070153709277,
      "eval_openwebtext_runtime": 8.406,
      "eval_openwebtext_samples_per_second": 510.823,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_wikitext_loss": 3.087890625,
      "eval_wikitext_ppl": 21.930768935644906,
      "eval_wikitext_runtime": 1.927,
      "eval_wikitext_samples_per_second": 236.636,
      "step": 57000
    },
    {
      "epoch": 3.31,
      "eval_lambada_loss": 2.947265625,
      "eval_lambada_ppl": 19.053782246709744,
      "eval_lambada_runtime": 9.5315,
      "eval_lambada_samples_per_second": 510.831,
      "step": 57000
    },
    {
      "activations/layer0_attention_weight_max": 14.724576950073242,
      "activations/layer0_attention_weight_min": -14.646306991577148,
      "activations/layer10_attention_weight_max": 42.59203338623047,
      "activations/layer10_attention_weight_min": -33.50257873535156,
      "activations/layer11_attention_weight_max": 38.49482727050781,
      "activations/layer11_attention_weight_min": -24.90285873413086,
      "activations/layer12_attention_weight_max": 25.788509368896484,
      "activations/layer12_attention_weight_min": -25.806718826293945,
      "activations/layer13_attention_weight_max": 58.88407516479492,
      "activations/layer13_attention_weight_min": -38.51001739501953,
      "activations/layer14_attention_weight_max": 53.106143951416016,
      "activations/layer14_attention_weight_min": -33.964683532714844,
      "activations/layer15_attention_weight_max": 45.23395919799805,
      "activations/layer15_attention_weight_min": -33.11198043823242,
      "activations/layer16_attention_weight_max": 38.81170654296875,
      "activations/layer16_attention_weight_min": -25.95339584350586,
      "activations/layer17_attention_weight_max": 62.46410369873047,
      "activations/layer17_attention_weight_min": -37.98106384277344,
      "activations/layer18_attention_weight_max": 54.42454147338867,
      "activations/layer18_attention_weight_min": -32.78464126586914,
      "activations/layer19_attention_weight_max": 24.8339900970459,
      "activations/layer19_attention_weight_min": -18.3349552154541,
      "activations/layer1_attention_weight_max": 17.665964126586914,
      "activations/layer1_attention_weight_min": -13.169593811035156,
      "activations/layer20_attention_weight_max": 24.05117416381836,
      "activations/layer20_attention_weight_min": -18.081968307495117,
      "activations/layer21_attention_weight_max": 42.980281829833984,
      "activations/layer21_attention_weight_min": -28.13237762451172,
      "activations/layer22_attention_weight_max": 36.571712493896484,
      "activations/layer22_attention_weight_min": -24.024229049682617,
      "activations/layer23_attention_weight_max": 30.36410140991211,
      "activations/layer23_attention_weight_min": -15.743136405944824,
      "activations/layer2_attention_weight_max": 31.388975143432617,
      "activations/layer2_attention_weight_min": -28.342130661010742,
      "activations/layer3_attention_weight_max": 78.20988464355469,
      "activations/layer3_attention_weight_min": -75.69367218017578,
      "activations/layer4_attention_weight_max": 101.81690216064453,
      "activations/layer4_attention_weight_min": -94.92498779296875,
      "activations/layer5_attention_weight_max": 119.78207397460938,
      "activations/layer5_attention_weight_min": -99.56298828125,
      "activations/layer6_attention_weight_max": 61.01437759399414,
      "activations/layer6_attention_weight_min": -50.51352310180664,
      "activations/layer7_attention_weight_max": 67.61949920654297,
      "activations/layer7_attention_weight_min": -55.58882522583008,
      "activations/layer8_attention_weight_max": 40.50454330444336,
      "activations/layer8_attention_weight_min": -36.080833435058594,
      "activations/layer9_attention_weight_max": 62.486446380615234,
      "activations/layer9_attention_weight_min": -48.94065475463867,
      "epoch": 3.31,
      "learning_rate": 0.00012992954545454545,
      "loss": 2.8908,
      "step": 57050
    },
    {
      "activations/layer0_attention_weight_max": 13.532442092895508,
      "activations/layer0_attention_weight_min": -14.496549606323242,
      "activations/layer10_attention_weight_max": 44.59060287475586,
      "activations/layer10_attention_weight_min": -35.662261962890625,
      "activations/layer11_attention_weight_max": 34.91089630126953,
      "activations/layer11_attention_weight_min": -27.860061645507812,
      "activations/layer12_attention_weight_max": 25.919456481933594,
      "activations/layer12_attention_weight_min": -25.84701156616211,
      "activations/layer13_attention_weight_max": 56.46086120605469,
      "activations/layer13_attention_weight_min": -39.145301818847656,
      "activations/layer14_attention_weight_max": 51.75486755371094,
      "activations/layer14_attention_weight_min": -33.710514068603516,
      "activations/layer15_attention_weight_max": 42.52827835083008,
      "activations/layer15_attention_weight_min": -33.159488677978516,
      "activations/layer16_attention_weight_max": 35.26018524169922,
      "activations/layer16_attention_weight_min": -24.847017288208008,
      "activations/layer17_attention_weight_max": 52.22490310668945,
      "activations/layer17_attention_weight_min": -34.67921829223633,
      "activations/layer18_attention_weight_max": 50.67396545410156,
      "activations/layer18_attention_weight_min": -32.86682891845703,
      "activations/layer19_attention_weight_max": 25.019039154052734,
      "activations/layer19_attention_weight_min": -17.165918350219727,
      "activations/layer1_attention_weight_max": 17.755075454711914,
      "activations/layer1_attention_weight_min": -14.135115623474121,
      "activations/layer20_attention_weight_max": 24.105566024780273,
      "activations/layer20_attention_weight_min": -21.159570693969727,
      "activations/layer21_attention_weight_max": 35.33479690551758,
      "activations/layer21_attention_weight_min": -29.0328426361084,
      "activations/layer22_attention_weight_max": 36.23847579956055,
      "activations/layer22_attention_weight_min": -23.83603858947754,
      "activations/layer23_attention_weight_max": 23.706802368164062,
      "activations/layer23_attention_weight_min": -15.8854398727417,
      "activations/layer2_attention_weight_max": 29.127241134643555,
      "activations/layer2_attention_weight_min": -28.1883602142334,
      "activations/layer3_attention_weight_max": 71.83155059814453,
      "activations/layer3_attention_weight_min": -72.73344421386719,
      "activations/layer4_attention_weight_max": 96.01812744140625,
      "activations/layer4_attention_weight_min": -88.89693450927734,
      "activations/layer5_attention_weight_max": 109.62092590332031,
      "activations/layer5_attention_weight_min": -94.4215316772461,
      "activations/layer6_attention_weight_max": 56.15216064453125,
      "activations/layer6_attention_weight_min": -49.19524383544922,
      "activations/layer7_attention_weight_max": 60.6191291809082,
      "activations/layer7_attention_weight_min": -57.079280853271484,
      "activations/layer8_attention_weight_max": 37.685302734375,
      "activations/layer8_attention_weight_min": -35.88175964355469,
      "activations/layer9_attention_weight_max": 55.250850677490234,
      "activations/layer9_attention_weight_min": -49.09340286254883,
      "epoch": 3.32,
      "learning_rate": 0.00012991060606060606,
      "loss": 2.8705,
      "step": 57100
    },
    {
      "activations/layer0_attention_weight_max": 14.529664993286133,
      "activations/layer0_attention_weight_min": -14.648679733276367,
      "activations/layer10_attention_weight_max": 49.64631271362305,
      "activations/layer10_attention_weight_min": -36.95988464355469,
      "activations/layer11_attention_weight_max": 41.28371810913086,
      "activations/layer11_attention_weight_min": -28.558364868164062,
      "activations/layer12_attention_weight_max": 24.427852630615234,
      "activations/layer12_attention_weight_min": -27.00834083557129,
      "activations/layer13_attention_weight_max": 71.3418197631836,
      "activations/layer13_attention_weight_min": -42.705848693847656,
      "activations/layer14_attention_weight_max": 86.82222747802734,
      "activations/layer14_attention_weight_min": -45.841392517089844,
      "activations/layer15_attention_weight_max": 50.49815368652344,
      "activations/layer15_attention_weight_min": -31.69415283203125,
      "activations/layer16_attention_weight_max": 39.02762985229492,
      "activations/layer16_attention_weight_min": -26.828022003173828,
      "activations/layer17_attention_weight_max": 59.78245162963867,
      "activations/layer17_attention_weight_min": -35.00923156738281,
      "activations/layer18_attention_weight_max": 52.55845642089844,
      "activations/layer18_attention_weight_min": -32.6786994934082,
      "activations/layer19_attention_weight_max": 23.192638397216797,
      "activations/layer19_attention_weight_min": -16.42034149169922,
      "activations/layer1_attention_weight_max": 18.56056022644043,
      "activations/layer1_attention_weight_min": -14.925405502319336,
      "activations/layer20_attention_weight_max": 24.563642501831055,
      "activations/layer20_attention_weight_min": -17.623394012451172,
      "activations/layer21_attention_weight_max": 44.30076217651367,
      "activations/layer21_attention_weight_min": -29.378347396850586,
      "activations/layer22_attention_weight_max": 38.3620491027832,
      "activations/layer22_attention_weight_min": -23.40943145751953,
      "activations/layer23_attention_weight_max": 27.03193473815918,
      "activations/layer23_attention_weight_min": -14.874990463256836,
      "activations/layer2_attention_weight_max": 31.341398239135742,
      "activations/layer2_attention_weight_min": -30.226299285888672,
      "activations/layer3_attention_weight_max": 84.09916687011719,
      "activations/layer3_attention_weight_min": -88.91032409667969,
      "activations/layer4_attention_weight_max": 124.55108642578125,
      "activations/layer4_attention_weight_min": -106.11297607421875,
      "activations/layer5_attention_weight_max": 146.996337890625,
      "activations/layer5_attention_weight_min": -119.05564880371094,
      "activations/layer6_attention_weight_max": 74.75145721435547,
      "activations/layer6_attention_weight_min": -59.92626953125,
      "activations/layer7_attention_weight_max": 75.87874603271484,
      "activations/layer7_attention_weight_min": -68.14312744140625,
      "activations/layer8_attention_weight_max": 48.65175247192383,
      "activations/layer8_attention_weight_min": -41.56669998168945,
      "activations/layer9_attention_weight_max": 67.66609191894531,
      "activations/layer9_attention_weight_min": -53.25696563720703,
      "epoch": 3.32,
      "learning_rate": 0.00012989166666666665,
      "loss": 2.8842,
      "step": 57150
    },
    {
      "activations/layer0_attention_weight_max": 13.97612190246582,
      "activations/layer0_attention_weight_min": -14.942727088928223,
      "activations/layer10_attention_weight_max": 40.893943786621094,
      "activations/layer10_attention_weight_min": -32.96705627441406,
      "activations/layer11_attention_weight_max": 31.790815353393555,
      "activations/layer11_attention_weight_min": -28.319889068603516,
      "activations/layer12_attention_weight_max": 24.50619125366211,
      "activations/layer12_attention_weight_min": -26.277189254760742,
      "activations/layer13_attention_weight_max": 51.52745056152344,
      "activations/layer13_attention_weight_min": -38.52260208129883,
      "activations/layer14_attention_weight_max": 45.42905044555664,
      "activations/layer14_attention_weight_min": -33.54213333129883,
      "activations/layer15_attention_weight_max": 53.59334945678711,
      "activations/layer15_attention_weight_min": -30.977741241455078,
      "activations/layer16_attention_weight_max": 36.26856994628906,
      "activations/layer16_attention_weight_min": -26.07866096496582,
      "activations/layer17_attention_weight_max": 55.059242248535156,
      "activations/layer17_attention_weight_min": -34.046180725097656,
      "activations/layer18_attention_weight_max": 50.619140625,
      "activations/layer18_attention_weight_min": -31.4990177154541,
      "activations/layer19_attention_weight_max": 21.650470733642578,
      "activations/layer19_attention_weight_min": -17.667631149291992,
      "activations/layer1_attention_weight_max": 18.655654907226562,
      "activations/layer1_attention_weight_min": -15.04821491241455,
      "activations/layer20_attention_weight_max": 20.452293395996094,
      "activations/layer20_attention_weight_min": -18.232402801513672,
      "activations/layer21_attention_weight_max": 41.80430603027344,
      "activations/layer21_attention_weight_min": -26.910417556762695,
      "activations/layer22_attention_weight_max": 33.354705810546875,
      "activations/layer22_attention_weight_min": -21.504531860351562,
      "activations/layer23_attention_weight_max": 24.22812271118164,
      "activations/layer23_attention_weight_min": -15.24513053894043,
      "activations/layer2_attention_weight_max": 29.99946403503418,
      "activations/layer2_attention_weight_min": -28.393070220947266,
      "activations/layer3_attention_weight_max": 74.32392120361328,
      "activations/layer3_attention_weight_min": -75.9413070678711,
      "activations/layer4_attention_weight_max": 100.60752868652344,
      "activations/layer4_attention_weight_min": -91.16634368896484,
      "activations/layer5_attention_weight_max": 116.84176635742188,
      "activations/layer5_attention_weight_min": -97.15911865234375,
      "activations/layer6_attention_weight_max": 59.92210388183594,
      "activations/layer6_attention_weight_min": -50.29262161254883,
      "activations/layer7_attention_weight_max": 61.89818572998047,
      "activations/layer7_attention_weight_min": -56.9952392578125,
      "activations/layer8_attention_weight_max": 38.9008674621582,
      "activations/layer8_attention_weight_min": -34.68907165527344,
      "activations/layer9_attention_weight_max": 52.39321517944336,
      "activations/layer9_attention_weight_min": -44.17942428588867,
      "epoch": 3.32,
      "learning_rate": 0.00012987272727272726,
      "loss": 2.9002,
      "step": 57200
    },
    {
      "activations/layer0_attention_weight_max": 15.101862907409668,
      "activations/layer0_attention_weight_min": -15.372091293334961,
      "activations/layer10_attention_weight_max": 43.95600509643555,
      "activations/layer10_attention_weight_min": -37.2322883605957,
      "activations/layer11_attention_weight_max": 38.46097946166992,
      "activations/layer11_attention_weight_min": -29.037673950195312,
      "activations/layer12_attention_weight_max": 26.090431213378906,
      "activations/layer12_attention_weight_min": -25.77348518371582,
      "activations/layer13_attention_weight_max": 60.135623931884766,
      "activations/layer13_attention_weight_min": -46.23740005493164,
      "activations/layer14_attention_weight_max": 53.54570007324219,
      "activations/layer14_attention_weight_min": -34.27812957763672,
      "activations/layer15_attention_weight_max": 48.89331817626953,
      "activations/layer15_attention_weight_min": -34.692039489746094,
      "activations/layer16_attention_weight_max": 41.751522064208984,
      "activations/layer16_attention_weight_min": -26.409122467041016,
      "activations/layer17_attention_weight_max": 59.11072540283203,
      "activations/layer17_attention_weight_min": -37.66128921508789,
      "activations/layer18_attention_weight_max": 51.20098114013672,
      "activations/layer18_attention_weight_min": -34.11838912963867,
      "activations/layer19_attention_weight_max": 24.905475616455078,
      "activations/layer19_attention_weight_min": -17.497909545898438,
      "activations/layer1_attention_weight_max": 18.801342010498047,
      "activations/layer1_attention_weight_min": -15.060298919677734,
      "activations/layer20_attention_weight_max": 21.184261322021484,
      "activations/layer20_attention_weight_min": -16.510425567626953,
      "activations/layer21_attention_weight_max": 51.96908950805664,
      "activations/layer21_attention_weight_min": -30.11886978149414,
      "activations/layer22_attention_weight_max": 35.85814666748047,
      "activations/layer22_attention_weight_min": -22.677734375,
      "activations/layer23_attention_weight_max": 24.380897521972656,
      "activations/layer23_attention_weight_min": -15.790264129638672,
      "activations/layer2_attention_weight_max": 30.913253784179688,
      "activations/layer2_attention_weight_min": -28.785125732421875,
      "activations/layer3_attention_weight_max": 74.79696655273438,
      "activations/layer3_attention_weight_min": -79.54486083984375,
      "activations/layer4_attention_weight_max": 104.33455657958984,
      "activations/layer4_attention_weight_min": -95.74286651611328,
      "activations/layer5_attention_weight_max": 120.48846435546875,
      "activations/layer5_attention_weight_min": -98.10208129882812,
      "activations/layer6_attention_weight_max": 60.55282974243164,
      "activations/layer6_attention_weight_min": -50.97639846801758,
      "activations/layer7_attention_weight_max": 60.301185607910156,
      "activations/layer7_attention_weight_min": -57.75300598144531,
      "activations/layer8_attention_weight_max": 39.51137161254883,
      "activations/layer8_attention_weight_min": -35.97157287597656,
      "activations/layer9_attention_weight_max": 50.34603500366211,
      "activations/layer9_attention_weight_min": -45.79962921142578,
      "epoch": 3.33,
      "learning_rate": 0.00012985378787878788,
      "loss": 2.8964,
      "step": 57250
    },
    {
      "activations/layer0_attention_weight_max": 14.103996276855469,
      "activations/layer0_attention_weight_min": -14.970674514770508,
      "activations/layer10_attention_weight_max": 47.9494743347168,
      "activations/layer10_attention_weight_min": -35.293270111083984,
      "activations/layer11_attention_weight_max": 38.92253875732422,
      "activations/layer11_attention_weight_min": -27.149038314819336,
      "activations/layer12_attention_weight_max": 40.632606506347656,
      "activations/layer12_attention_weight_min": -23.15991973876953,
      "activations/layer13_attention_weight_max": 67.7354507446289,
      "activations/layer13_attention_weight_min": -43.152381896972656,
      "activations/layer14_attention_weight_max": 47.191650390625,
      "activations/layer14_attention_weight_min": -31.366289138793945,
      "activations/layer15_attention_weight_max": 49.37800598144531,
      "activations/layer15_attention_weight_min": -32.63617706298828,
      "activations/layer16_attention_weight_max": 42.25456619262695,
      "activations/layer16_attention_weight_min": -26.076305389404297,
      "activations/layer17_attention_weight_max": 65.18619537353516,
      "activations/layer17_attention_weight_min": -36.07780838012695,
      "activations/layer18_attention_weight_max": 51.7521858215332,
      "activations/layer18_attention_weight_min": -31.50043296813965,
      "activations/layer19_attention_weight_max": 25.065044403076172,
      "activations/layer19_attention_weight_min": -17.523733139038086,
      "activations/layer1_attention_weight_max": 17.613615036010742,
      "activations/layer1_attention_weight_min": -13.713873863220215,
      "activations/layer20_attention_weight_max": 23.358898162841797,
      "activations/layer20_attention_weight_min": -17.222898483276367,
      "activations/layer21_attention_weight_max": 45.56984329223633,
      "activations/layer21_attention_weight_min": -29.406082153320312,
      "activations/layer22_attention_weight_max": 37.56327438354492,
      "activations/layer22_attention_weight_min": -21.782289505004883,
      "activations/layer23_attention_weight_max": 26.20192527770996,
      "activations/layer23_attention_weight_min": -16.297138214111328,
      "activations/layer2_attention_weight_max": 28.929889678955078,
      "activations/layer2_attention_weight_min": -28.08300018310547,
      "activations/layer3_attention_weight_max": 72.64747619628906,
      "activations/layer3_attention_weight_min": -77.62483978271484,
      "activations/layer4_attention_weight_max": 97.7162857055664,
      "activations/layer4_attention_weight_min": -89.10710906982422,
      "activations/layer5_attention_weight_max": 114.39601135253906,
      "activations/layer5_attention_weight_min": -99.31553649902344,
      "activations/layer6_attention_weight_max": 61.54336929321289,
      "activations/layer6_attention_weight_min": -51.93602752685547,
      "activations/layer7_attention_weight_max": 62.752342224121094,
      "activations/layer7_attention_weight_min": -56.906768798828125,
      "activations/layer8_attention_weight_max": 40.19425582885742,
      "activations/layer8_attention_weight_min": -36.37129592895508,
      "activations/layer9_attention_weight_max": 50.97011947631836,
      "activations/layer9_attention_weight_min": -47.56587600708008,
      "epoch": 3.33,
      "learning_rate": 0.00012983484848484847,
      "loss": 2.9025,
      "step": 57300
    },
    {
      "activations/layer0_attention_weight_max": 14.523279190063477,
      "activations/layer0_attention_weight_min": -14.857315063476562,
      "activations/layer10_attention_weight_max": 45.72772979736328,
      "activations/layer10_attention_weight_min": -35.93423843383789,
      "activations/layer11_attention_weight_max": 36.64845275878906,
      "activations/layer11_attention_weight_min": -29.568525314331055,
      "activations/layer12_attention_weight_max": 26.29989242553711,
      "activations/layer12_attention_weight_min": -26.506919860839844,
      "activations/layer13_attention_weight_max": 52.91383361816406,
      "activations/layer13_attention_weight_min": -35.122779846191406,
      "activations/layer14_attention_weight_max": 49.82852554321289,
      "activations/layer14_attention_weight_min": -31.619922637939453,
      "activations/layer15_attention_weight_max": 48.03463363647461,
      "activations/layer15_attention_weight_min": -33.79205322265625,
      "activations/layer16_attention_weight_max": 39.10380935668945,
      "activations/layer16_attention_weight_min": -27.200199127197266,
      "activations/layer17_attention_weight_max": 63.93425369262695,
      "activations/layer17_attention_weight_min": -36.75751876831055,
      "activations/layer18_attention_weight_max": 52.90317153930664,
      "activations/layer18_attention_weight_min": -33.6361083984375,
      "activations/layer19_attention_weight_max": 27.352127075195312,
      "activations/layer19_attention_weight_min": -18.066104888916016,
      "activations/layer1_attention_weight_max": 19.25447654724121,
      "activations/layer1_attention_weight_min": -14.946401596069336,
      "activations/layer20_attention_weight_max": 21.992570877075195,
      "activations/layer20_attention_weight_min": -20.252092361450195,
      "activations/layer21_attention_weight_max": 42.41064453125,
      "activations/layer21_attention_weight_min": -26.38126564025879,
      "activations/layer22_attention_weight_max": 36.826412200927734,
      "activations/layer22_attention_weight_min": -22.222036361694336,
      "activations/layer23_attention_weight_max": 27.71249008178711,
      "activations/layer23_attention_weight_min": -16.94797134399414,
      "activations/layer2_attention_weight_max": 30.65566635131836,
      "activations/layer2_attention_weight_min": -28.24982261657715,
      "activations/layer3_attention_weight_max": 76.30736541748047,
      "activations/layer3_attention_weight_min": -74.43666076660156,
      "activations/layer4_attention_weight_max": 109.75044250488281,
      "activations/layer4_attention_weight_min": -95.81318664550781,
      "activations/layer5_attention_weight_max": 126.41985321044922,
      "activations/layer5_attention_weight_min": -106.24849700927734,
      "activations/layer6_attention_weight_max": 61.82822036743164,
      "activations/layer6_attention_weight_min": -51.49909591674805,
      "activations/layer7_attention_weight_max": 61.57511901855469,
      "activations/layer7_attention_weight_min": -60.2103385925293,
      "activations/layer8_attention_weight_max": 39.39990997314453,
      "activations/layer8_attention_weight_min": -36.24372863769531,
      "activations/layer9_attention_weight_max": 57.78886032104492,
      "activations/layer9_attention_weight_min": -52.615516662597656,
      "epoch": 3.33,
      "learning_rate": 0.00012981590909090908,
      "loss": 2.8932,
      "step": 57350
    },
    {
      "activations/layer0_attention_weight_max": 13.982283592224121,
      "activations/layer0_attention_weight_min": -14.790081977844238,
      "activations/layer10_attention_weight_max": 44.81113815307617,
      "activations/layer10_attention_weight_min": -34.38029098510742,
      "activations/layer11_attention_weight_max": 34.44884490966797,
      "activations/layer11_attention_weight_min": -26.75189781188965,
      "activations/layer12_attention_weight_max": 28.53376007080078,
      "activations/layer12_attention_weight_min": -24.55240249633789,
      "activations/layer13_attention_weight_max": 60.651763916015625,
      "activations/layer13_attention_weight_min": -38.20695495605469,
      "activations/layer14_attention_weight_max": 63.15302276611328,
      "activations/layer14_attention_weight_min": -40.67627716064453,
      "activations/layer15_attention_weight_max": 64.00818634033203,
      "activations/layer15_attention_weight_min": -34.66999816894531,
      "activations/layer16_attention_weight_max": 42.45505905151367,
      "activations/layer16_attention_weight_min": -27.4122371673584,
      "activations/layer17_attention_weight_max": 61.01863098144531,
      "activations/layer17_attention_weight_min": -39.208518981933594,
      "activations/layer18_attention_weight_max": 55.52537155151367,
      "activations/layer18_attention_weight_min": -33.5842399597168,
      "activations/layer19_attention_weight_max": 24.886457443237305,
      "activations/layer19_attention_weight_min": -18.91934585571289,
      "activations/layer1_attention_weight_max": 17.481922149658203,
      "activations/layer1_attention_weight_min": -15.087014198303223,
      "activations/layer20_attention_weight_max": 22.8487606048584,
      "activations/layer20_attention_weight_min": -20.43959617614746,
      "activations/layer21_attention_weight_max": 44.259422302246094,
      "activations/layer21_attention_weight_min": -29.938940048217773,
      "activations/layer22_attention_weight_max": 38.132049560546875,
      "activations/layer22_attention_weight_min": -23.719146728515625,
      "activations/layer23_attention_weight_max": 26.781173706054688,
      "activations/layer23_attention_weight_min": -18.38951873779297,
      "activations/layer2_attention_weight_max": 32.96179962158203,
      "activations/layer2_attention_weight_min": -29.129316329956055,
      "activations/layer3_attention_weight_max": 76.73489379882812,
      "activations/layer3_attention_weight_min": -78.37751770019531,
      "activations/layer4_attention_weight_max": 103.96639251708984,
      "activations/layer4_attention_weight_min": -95.21910858154297,
      "activations/layer5_attention_weight_max": 120.97904968261719,
      "activations/layer5_attention_weight_min": -101.769775390625,
      "activations/layer6_attention_weight_max": 65.10298919677734,
      "activations/layer6_attention_weight_min": -52.72976303100586,
      "activations/layer7_attention_weight_max": 63.671321868896484,
      "activations/layer7_attention_weight_min": -57.73946762084961,
      "activations/layer8_attention_weight_max": 40.89823913574219,
      "activations/layer8_attention_weight_min": -35.538944244384766,
      "activations/layer9_attention_weight_max": 54.81089401245117,
      "activations/layer9_attention_weight_min": -46.16205596923828,
      "epoch": 3.34,
      "learning_rate": 0.0001297969696969697,
      "loss": 2.8885,
      "step": 57400
    },
    {
      "activations/layer0_attention_weight_max": 13.808984756469727,
      "activations/layer0_attention_weight_min": -15.211185455322266,
      "activations/layer10_attention_weight_max": 44.7467041015625,
      "activations/layer10_attention_weight_min": -33.439208984375,
      "activations/layer11_attention_weight_max": 36.09093475341797,
      "activations/layer11_attention_weight_min": -28.22359275817871,
      "activations/layer12_attention_weight_max": 30.97018814086914,
      "activations/layer12_attention_weight_min": -25.28681755065918,
      "activations/layer13_attention_weight_max": 60.5626335144043,
      "activations/layer13_attention_weight_min": -39.294002532958984,
      "activations/layer14_attention_weight_max": 56.27699661254883,
      "activations/layer14_attention_weight_min": -37.50181579589844,
      "activations/layer15_attention_weight_max": 54.44707107543945,
      "activations/layer15_attention_weight_min": -30.249971389770508,
      "activations/layer16_attention_weight_max": 43.812522888183594,
      "activations/layer16_attention_weight_min": -28.63914680480957,
      "activations/layer17_attention_weight_max": 59.81025695800781,
      "activations/layer17_attention_weight_min": -35.873252868652344,
      "activations/layer18_attention_weight_max": 59.77599334716797,
      "activations/layer18_attention_weight_min": -32.999271392822266,
      "activations/layer19_attention_weight_max": 24.600072860717773,
      "activations/layer19_attention_weight_min": -19.6293888092041,
      "activations/layer1_attention_weight_max": 18.881576538085938,
      "activations/layer1_attention_weight_min": -15.479925155639648,
      "activations/layer20_attention_weight_max": 21.62105941772461,
      "activations/layer20_attention_weight_min": -18.375953674316406,
      "activations/layer21_attention_weight_max": 44.0085334777832,
      "activations/layer21_attention_weight_min": -29.220245361328125,
      "activations/layer22_attention_weight_max": 41.47238540649414,
      "activations/layer22_attention_weight_min": -22.02838897705078,
      "activations/layer23_attention_weight_max": 25.521595001220703,
      "activations/layer23_attention_weight_min": -15.645197868347168,
      "activations/layer2_attention_weight_max": 29.9149169921875,
      "activations/layer2_attention_weight_min": -27.33833122253418,
      "activations/layer3_attention_weight_max": 70.21246337890625,
      "activations/layer3_attention_weight_min": -75.01616668701172,
      "activations/layer4_attention_weight_max": 101.72704315185547,
      "activations/layer4_attention_weight_min": -86.55429077148438,
      "activations/layer5_attention_weight_max": 116.76048278808594,
      "activations/layer5_attention_weight_min": -95.13713836669922,
      "activations/layer6_attention_weight_max": 61.509124755859375,
      "activations/layer6_attention_weight_min": -50.60865020751953,
      "activations/layer7_attention_weight_max": 61.769569396972656,
      "activations/layer7_attention_weight_min": -55.40850067138672,
      "activations/layer8_attention_weight_max": 38.70124816894531,
      "activations/layer8_attention_weight_min": -35.8048095703125,
      "activations/layer9_attention_weight_max": 49.43368148803711,
      "activations/layer9_attention_weight_min": -49.42515182495117,
      "epoch": 3.34,
      "learning_rate": 0.00012977803030303028,
      "loss": 2.8886,
      "step": 57450
    },
    {
      "activations/layer0_attention_weight_max": 15.388522148132324,
      "activations/layer0_attention_weight_min": -15.1337890625,
      "activations/layer10_attention_weight_max": 42.263519287109375,
      "activations/layer10_attention_weight_min": -34.42795181274414,
      "activations/layer11_attention_weight_max": 31.43202781677246,
      "activations/layer11_attention_weight_min": -27.505619049072266,
      "activations/layer12_attention_weight_max": 26.802831649780273,
      "activations/layer12_attention_weight_min": -24.524234771728516,
      "activations/layer13_attention_weight_max": 63.92961502075195,
      "activations/layer13_attention_weight_min": -39.01008224487305,
      "activations/layer14_attention_weight_max": 48.597755432128906,
      "activations/layer14_attention_weight_min": -29.6431941986084,
      "activations/layer15_attention_weight_max": 57.05356979370117,
      "activations/layer15_attention_weight_min": -32.96711349487305,
      "activations/layer16_attention_weight_max": 38.07889175415039,
      "activations/layer16_attention_weight_min": -25.470306396484375,
      "activations/layer17_attention_weight_max": 59.30897521972656,
      "activations/layer17_attention_weight_min": -33.79523468017578,
      "activations/layer18_attention_weight_max": 47.725765228271484,
      "activations/layer18_attention_weight_min": -30.600337982177734,
      "activations/layer19_attention_weight_max": 22.248502731323242,
      "activations/layer19_attention_weight_min": -17.85333824157715,
      "activations/layer1_attention_weight_max": 18.083951950073242,
      "activations/layer1_attention_weight_min": -13.88936710357666,
      "activations/layer20_attention_weight_max": 23.90007781982422,
      "activations/layer20_attention_weight_min": -20.526487350463867,
      "activations/layer21_attention_weight_max": 49.42975997924805,
      "activations/layer21_attention_weight_min": -28.715078353881836,
      "activations/layer22_attention_weight_max": 35.095760345458984,
      "activations/layer22_attention_weight_min": -22.34075164794922,
      "activations/layer23_attention_weight_max": 24.853132247924805,
      "activations/layer23_attention_weight_min": -14.598963737487793,
      "activations/layer2_attention_weight_max": 28.632606506347656,
      "activations/layer2_attention_weight_min": -26.740461349487305,
      "activations/layer3_attention_weight_max": 72.87605285644531,
      "activations/layer3_attention_weight_min": -77.84513854980469,
      "activations/layer4_attention_weight_max": 99.43629455566406,
      "activations/layer4_attention_weight_min": -92.86767578125,
      "activations/layer5_attention_weight_max": 122.55197143554688,
      "activations/layer5_attention_weight_min": -103.57335662841797,
      "activations/layer6_attention_weight_max": 63.568756103515625,
      "activations/layer6_attention_weight_min": -52.488670349121094,
      "activations/layer7_attention_weight_max": 62.247501373291016,
      "activations/layer7_attention_weight_min": -57.798492431640625,
      "activations/layer8_attention_weight_max": 41.4711799621582,
      "activations/layer8_attention_weight_min": -39.18461990356445,
      "activations/layer9_attention_weight_max": 52.597450256347656,
      "activations/layer9_attention_weight_min": -49.372459411621094,
      "epoch": 3.34,
      "learning_rate": 0.0001297590909090909,
      "loss": 2.9049,
      "step": 57500
    },
    {
      "activations/layer0_attention_weight_max": 15.781484603881836,
      "activations/layer0_attention_weight_min": -14.492891311645508,
      "activations/layer10_attention_weight_max": 47.7492790222168,
      "activations/layer10_attention_weight_min": -35.5504264831543,
      "activations/layer11_attention_weight_max": 40.36376190185547,
      "activations/layer11_attention_weight_min": -29.81196403503418,
      "activations/layer12_attention_weight_max": 23.430892944335938,
      "activations/layer12_attention_weight_min": -26.033727645874023,
      "activations/layer13_attention_weight_max": 52.35721969604492,
      "activations/layer13_attention_weight_min": -37.27190017700195,
      "activations/layer14_attention_weight_max": 58.56501007080078,
      "activations/layer14_attention_weight_min": -34.55600357055664,
      "activations/layer15_attention_weight_max": 43.68142318725586,
      "activations/layer15_attention_weight_min": -30.731639862060547,
      "activations/layer16_attention_weight_max": 38.83185577392578,
      "activations/layer16_attention_weight_min": -24.301870346069336,
      "activations/layer17_attention_weight_max": 58.0812873840332,
      "activations/layer17_attention_weight_min": -33.799888610839844,
      "activations/layer18_attention_weight_max": 45.49674606323242,
      "activations/layer18_attention_weight_min": -31.876140594482422,
      "activations/layer19_attention_weight_max": 20.581098556518555,
      "activations/layer19_attention_weight_min": -17.8707332611084,
      "activations/layer1_attention_weight_max": 17.757259368896484,
      "activations/layer1_attention_weight_min": -18.73164176940918,
      "activations/layer20_attention_weight_max": 17.781572341918945,
      "activations/layer20_attention_weight_min": -17.885982513427734,
      "activations/layer21_attention_weight_max": 38.245784759521484,
      "activations/layer21_attention_weight_min": -28.412595748901367,
      "activations/layer22_attention_weight_max": 32.489768981933594,
      "activations/layer22_attention_weight_min": -21.631175994873047,
      "activations/layer23_attention_weight_max": 22.106380462646484,
      "activations/layer23_attention_weight_min": -14.675025939941406,
      "activations/layer2_attention_weight_max": 29.25653648376465,
      "activations/layer2_attention_weight_min": -27.660442352294922,
      "activations/layer3_attention_weight_max": 70.90876007080078,
      "activations/layer3_attention_weight_min": -73.59932708740234,
      "activations/layer4_attention_weight_max": 99.16023254394531,
      "activations/layer4_attention_weight_min": -89.33993530273438,
      "activations/layer5_attention_weight_max": 118.95721435546875,
      "activations/layer5_attention_weight_min": -98.1638412475586,
      "activations/layer6_attention_weight_max": 59.699241638183594,
      "activations/layer6_attention_weight_min": -51.40862274169922,
      "activations/layer7_attention_weight_max": 74.45404815673828,
      "activations/layer7_attention_weight_min": -61.49106979370117,
      "activations/layer8_attention_weight_max": 49.637752532958984,
      "activations/layer8_attention_weight_min": -37.76316833496094,
      "activations/layer9_attention_weight_max": 72.8196029663086,
      "activations/layer9_attention_weight_min": -51.68143844604492,
      "epoch": 3.34,
      "learning_rate": 0.0001297401515151515,
      "loss": 2.9074,
      "step": 57550
    },
    {
      "activations/layer0_attention_weight_max": 13.573394775390625,
      "activations/layer0_attention_weight_min": -15.02497673034668,
      "activations/layer10_attention_weight_max": 46.85847091674805,
      "activations/layer10_attention_weight_min": -35.093868255615234,
      "activations/layer11_attention_weight_max": 41.745094299316406,
      "activations/layer11_attention_weight_min": -28.532445907592773,
      "activations/layer12_attention_weight_max": 25.8797664642334,
      "activations/layer12_attention_weight_min": -25.928146362304688,
      "activations/layer13_attention_weight_max": 59.05189895629883,
      "activations/layer13_attention_weight_min": -38.16379928588867,
      "activations/layer14_attention_weight_max": 56.355899810791016,
      "activations/layer14_attention_weight_min": -35.74345779418945,
      "activations/layer15_attention_weight_max": 44.49613571166992,
      "activations/layer15_attention_weight_min": -30.832775115966797,
      "activations/layer16_attention_weight_max": 36.83957290649414,
      "activations/layer16_attention_weight_min": -25.888248443603516,
      "activations/layer17_attention_weight_max": 51.72153091430664,
      "activations/layer17_attention_weight_min": -37.05366897583008,
      "activations/layer18_attention_weight_max": 45.055728912353516,
      "activations/layer18_attention_weight_min": -32.535396575927734,
      "activations/layer19_attention_weight_max": 18.622289657592773,
      "activations/layer19_attention_weight_min": -17.66261863708496,
      "activations/layer1_attention_weight_max": 18.98440170288086,
      "activations/layer1_attention_weight_min": -14.2244291305542,
      "activations/layer20_attention_weight_max": 18.555200576782227,
      "activations/layer20_attention_weight_min": -19.47538948059082,
      "activations/layer21_attention_weight_max": 37.459083557128906,
      "activations/layer21_attention_weight_min": -29.211034774780273,
      "activations/layer22_attention_weight_max": 32.41653060913086,
      "activations/layer22_attention_weight_min": -22.262287139892578,
      "activations/layer23_attention_weight_max": 22.046781539916992,
      "activations/layer23_attention_weight_min": -17.192058563232422,
      "activations/layer2_attention_weight_max": 27.770111083984375,
      "activations/layer2_attention_weight_min": -27.22878074645996,
      "activations/layer3_attention_weight_max": 75.11251831054688,
      "activations/layer3_attention_weight_min": -78.87091827392578,
      "activations/layer4_attention_weight_max": 105.1463394165039,
      "activations/layer4_attention_weight_min": -95.07131958007812,
      "activations/layer5_attention_weight_max": 126.60458374023438,
      "activations/layer5_attention_weight_min": -107.43357849121094,
      "activations/layer6_attention_weight_max": 62.522911071777344,
      "activations/layer6_attention_weight_min": -55.741455078125,
      "activations/layer7_attention_weight_max": 72.75121307373047,
      "activations/layer7_attention_weight_min": -64.09771728515625,
      "activations/layer8_attention_weight_max": 49.32786178588867,
      "activations/layer8_attention_weight_min": -37.341182708740234,
      "activations/layer9_attention_weight_max": 65.23909759521484,
      "activations/layer9_attention_weight_min": -54.21242141723633,
      "epoch": 3.35,
      "learning_rate": 0.0001297212121212121,
      "loss": 2.8712,
      "step": 57600
    },
    {
      "activations/layer0_attention_weight_max": 14.402902603149414,
      "activations/layer0_attention_weight_min": -14.881254196166992,
      "activations/layer10_attention_weight_max": 43.100040435791016,
      "activations/layer10_attention_weight_min": -35.573116302490234,
      "activations/layer11_attention_weight_max": 29.940853118896484,
      "activations/layer11_attention_weight_min": -26.68587875366211,
      "activations/layer12_attention_weight_max": 24.137298583984375,
      "activations/layer12_attention_weight_min": -22.727460861206055,
      "activations/layer13_attention_weight_max": 68.95988464355469,
      "activations/layer13_attention_weight_min": -48.73491287231445,
      "activations/layer14_attention_weight_max": 46.92380142211914,
      "activations/layer14_attention_weight_min": -32.94889831542969,
      "activations/layer15_attention_weight_max": 47.53242874145508,
      "activations/layer15_attention_weight_min": -33.80290222167969,
      "activations/layer16_attention_weight_max": 34.583309173583984,
      "activations/layer16_attention_weight_min": -26.326114654541016,
      "activations/layer17_attention_weight_max": 58.80185317993164,
      "activations/layer17_attention_weight_min": -35.572998046875,
      "activations/layer18_attention_weight_max": 53.00749969482422,
      "activations/layer18_attention_weight_min": -33.8765983581543,
      "activations/layer19_attention_weight_max": 20.081613540649414,
      "activations/layer19_attention_weight_min": -19.7434139251709,
      "activations/layer1_attention_weight_max": 16.843494415283203,
      "activations/layer1_attention_weight_min": -14.507526397705078,
      "activations/layer20_attention_weight_max": 23.56678581237793,
      "activations/layer20_attention_weight_min": -17.522497177124023,
      "activations/layer21_attention_weight_max": 45.89448165893555,
      "activations/layer21_attention_weight_min": -27.127328872680664,
      "activations/layer22_attention_weight_max": 36.37070846557617,
      "activations/layer22_attention_weight_min": -24.37117576599121,
      "activations/layer23_attention_weight_max": 25.574466705322266,
      "activations/layer23_attention_weight_min": -16.11058807373047,
      "activations/layer2_attention_weight_max": 28.841060638427734,
      "activations/layer2_attention_weight_min": -27.773967742919922,
      "activations/layer3_attention_weight_max": 71.096923828125,
      "activations/layer3_attention_weight_min": -74.52518463134766,
      "activations/layer4_attention_weight_max": 100.69487762451172,
      "activations/layer4_attention_weight_min": -92.11461639404297,
      "activations/layer5_attention_weight_max": 123.90000915527344,
      "activations/layer5_attention_weight_min": -97.02821350097656,
      "activations/layer6_attention_weight_max": 64.63392639160156,
      "activations/layer6_attention_weight_min": -52.105308532714844,
      "activations/layer7_attention_weight_max": 62.30363845825195,
      "activations/layer7_attention_weight_min": -55.684268951416016,
      "activations/layer8_attention_weight_max": 40.65608596801758,
      "activations/layer8_attention_weight_min": -40.12366485595703,
      "activations/layer9_attention_weight_max": 52.63739013671875,
      "activations/layer9_attention_weight_min": -46.92530822753906,
      "epoch": 3.35,
      "learning_rate": 0.00012970227272727272,
      "loss": 2.8859,
      "step": 57650
    },
    {
      "activations/layer0_attention_weight_max": 14.716443061828613,
      "activations/layer0_attention_weight_min": -15.20539665222168,
      "activations/layer10_attention_weight_max": 41.44593811035156,
      "activations/layer10_attention_weight_min": -32.94955825805664,
      "activations/layer11_attention_weight_max": 40.95960998535156,
      "activations/layer11_attention_weight_min": -27.104101181030273,
      "activations/layer12_attention_weight_max": 22.15902328491211,
      "activations/layer12_attention_weight_min": -24.15420913696289,
      "activations/layer13_attention_weight_max": 48.62776184082031,
      "activations/layer13_attention_weight_min": -38.0338249206543,
      "activations/layer14_attention_weight_max": 48.348472595214844,
      "activations/layer14_attention_weight_min": -35.62062072753906,
      "activations/layer15_attention_weight_max": 43.59122085571289,
      "activations/layer15_attention_weight_min": -33.726905822753906,
      "activations/layer16_attention_weight_max": 35.17686462402344,
      "activations/layer16_attention_weight_min": -29.096487045288086,
      "activations/layer17_attention_weight_max": 58.58307647705078,
      "activations/layer17_attention_weight_min": -36.76588821411133,
      "activations/layer18_attention_weight_max": 48.97203063964844,
      "activations/layer18_attention_weight_min": -34.874794006347656,
      "activations/layer19_attention_weight_max": 21.38957405090332,
      "activations/layer19_attention_weight_min": -19.221158981323242,
      "activations/layer1_attention_weight_max": 18.34895896911621,
      "activations/layer1_attention_weight_min": -14.016748428344727,
      "activations/layer20_attention_weight_max": 21.249624252319336,
      "activations/layer20_attention_weight_min": -19.16278076171875,
      "activations/layer21_attention_weight_max": 46.81439208984375,
      "activations/layer21_attention_weight_min": -28.49818992614746,
      "activations/layer22_attention_weight_max": 35.32192611694336,
      "activations/layer22_attention_weight_min": -21.187400817871094,
      "activations/layer23_attention_weight_max": 24.661436080932617,
      "activations/layer23_attention_weight_min": -16.48333168029785,
      "activations/layer2_attention_weight_max": 29.089900970458984,
      "activations/layer2_attention_weight_min": -27.82404899597168,
      "activations/layer3_attention_weight_max": 71.2718505859375,
      "activations/layer3_attention_weight_min": -81.14555358886719,
      "activations/layer4_attention_weight_max": 95.95103454589844,
      "activations/layer4_attention_weight_min": -96.18181610107422,
      "activations/layer5_attention_weight_max": 111.24935150146484,
      "activations/layer5_attention_weight_min": -99.98051452636719,
      "activations/layer6_attention_weight_max": 57.27874755859375,
      "activations/layer6_attention_weight_min": -49.04878616333008,
      "activations/layer7_attention_weight_max": 57.004024505615234,
      "activations/layer7_attention_weight_min": -56.139190673828125,
      "activations/layer8_attention_weight_max": 36.65254592895508,
      "activations/layer8_attention_weight_min": -36.46609878540039,
      "activations/layer9_attention_weight_max": 55.6589241027832,
      "activations/layer9_attention_weight_min": -43.589935302734375,
      "epoch": 3.35,
      "learning_rate": 0.0001296833333333333,
      "loss": 2.8786,
      "step": 57700
    },
    {
      "activations/layer0_attention_weight_max": 14.615117073059082,
      "activations/layer0_attention_weight_min": -15.163370132446289,
      "activations/layer10_attention_weight_max": 42.87620162963867,
      "activations/layer10_attention_weight_min": -35.09993362426758,
      "activations/layer11_attention_weight_max": 33.461849212646484,
      "activations/layer11_attention_weight_min": -27.10470199584961,
      "activations/layer12_attention_weight_max": 26.465347290039062,
      "activations/layer12_attention_weight_min": -23.498329162597656,
      "activations/layer13_attention_weight_max": 61.247955322265625,
      "activations/layer13_attention_weight_min": -44.14213943481445,
      "activations/layer14_attention_weight_max": 56.56509017944336,
      "activations/layer14_attention_weight_min": -35.375404357910156,
      "activations/layer15_attention_weight_max": 47.20465087890625,
      "activations/layer15_attention_weight_min": -33.04542541503906,
      "activations/layer16_attention_weight_max": 37.60250473022461,
      "activations/layer16_attention_weight_min": -24.909351348876953,
      "activations/layer17_attention_weight_max": 59.96921920776367,
      "activations/layer17_attention_weight_min": -34.707672119140625,
      "activations/layer18_attention_weight_max": 54.18650436401367,
      "activations/layer18_attention_weight_min": -30.934885025024414,
      "activations/layer19_attention_weight_max": 23.71449089050293,
      "activations/layer19_attention_weight_min": -17.769672393798828,
      "activations/layer1_attention_weight_max": 17.14115333557129,
      "activations/layer1_attention_weight_min": -16.483112335205078,
      "activations/layer20_attention_weight_max": 19.799345016479492,
      "activations/layer20_attention_weight_min": -20.62212371826172,
      "activations/layer21_attention_weight_max": 43.19685745239258,
      "activations/layer21_attention_weight_min": -27.70982551574707,
      "activations/layer22_attention_weight_max": 35.92084884643555,
      "activations/layer22_attention_weight_min": -22.800033569335938,
      "activations/layer23_attention_weight_max": 26.96895980834961,
      "activations/layer23_attention_weight_min": -18.002300262451172,
      "activations/layer2_attention_weight_max": 31.32061767578125,
      "activations/layer2_attention_weight_min": -28.348234176635742,
      "activations/layer3_attention_weight_max": 74.02889251708984,
      "activations/layer3_attention_weight_min": -77.07022857666016,
      "activations/layer4_attention_weight_max": 100.80064392089844,
      "activations/layer4_attention_weight_min": -91.2502670288086,
      "activations/layer5_attention_weight_max": 121.52269744873047,
      "activations/layer5_attention_weight_min": -104.77555847167969,
      "activations/layer6_attention_weight_max": 65.02069091796875,
      "activations/layer6_attention_weight_min": -53.762454986572266,
      "activations/layer7_attention_weight_max": 62.145381927490234,
      "activations/layer7_attention_weight_min": -60.670101165771484,
      "activations/layer8_attention_weight_max": 39.64733123779297,
      "activations/layer8_attention_weight_min": -36.35968780517578,
      "activations/layer9_attention_weight_max": 50.860755920410156,
      "activations/layer9_attention_weight_min": -49.63436508178711,
      "epoch": 3.36,
      "learning_rate": 0.00012966439393939392,
      "loss": 2.8894,
      "step": 57750
    },
    {
      "activations/layer0_attention_weight_max": 15.571342468261719,
      "activations/layer0_attention_weight_min": -15.168177604675293,
      "activations/layer10_attention_weight_max": 48.63515090942383,
      "activations/layer10_attention_weight_min": -36.101261138916016,
      "activations/layer11_attention_weight_max": 43.25034713745117,
      "activations/layer11_attention_weight_min": -28.569929122924805,
      "activations/layer12_attention_weight_max": 31.7536678314209,
      "activations/layer12_attention_weight_min": -24.704669952392578,
      "activations/layer13_attention_weight_max": 50.017189025878906,
      "activations/layer13_attention_weight_min": -35.14482116699219,
      "activations/layer14_attention_weight_max": 47.51593017578125,
      "activations/layer14_attention_weight_min": -38.580135345458984,
      "activations/layer15_attention_weight_max": 56.593441009521484,
      "activations/layer15_attention_weight_min": -36.49363708496094,
      "activations/layer16_attention_weight_max": 42.41721725463867,
      "activations/layer16_attention_weight_min": -27.214189529418945,
      "activations/layer17_attention_weight_max": 65.75608825683594,
      "activations/layer17_attention_weight_min": -37.74312210083008,
      "activations/layer18_attention_weight_max": 57.76465606689453,
      "activations/layer18_attention_weight_min": -36.0024528503418,
      "activations/layer19_attention_weight_max": 26.091144561767578,
      "activations/layer19_attention_weight_min": -18.870176315307617,
      "activations/layer1_attention_weight_max": 18.688047409057617,
      "activations/layer1_attention_weight_min": -14.142555236816406,
      "activations/layer20_attention_weight_max": 22.176250457763672,
      "activations/layer20_attention_weight_min": -17.434425354003906,
      "activations/layer21_attention_weight_max": 43.92747116088867,
      "activations/layer21_attention_weight_min": -25.965560913085938,
      "activations/layer22_attention_weight_max": 37.24024200439453,
      "activations/layer22_attention_weight_min": -23.30773162841797,
      "activations/layer23_attention_weight_max": 29.342008590698242,
      "activations/layer23_attention_weight_min": -16.601404190063477,
      "activations/layer2_attention_weight_max": 30.53293228149414,
      "activations/layer2_attention_weight_min": -28.796676635742188,
      "activations/layer3_attention_weight_max": 73.42011260986328,
      "activations/layer3_attention_weight_min": -78.14676666259766,
      "activations/layer4_attention_weight_max": 99.66629791259766,
      "activations/layer4_attention_weight_min": -96.62541198730469,
      "activations/layer5_attention_weight_max": 121.70311737060547,
      "activations/layer5_attention_weight_min": -111.21116638183594,
      "activations/layer6_attention_weight_max": 61.21833801269531,
      "activations/layer6_attention_weight_min": -52.450984954833984,
      "activations/layer7_attention_weight_max": 75.02434539794922,
      "activations/layer7_attention_weight_min": -62.273128509521484,
      "activations/layer8_attention_weight_max": 48.659420013427734,
      "activations/layer8_attention_weight_min": -38.5711669921875,
      "activations/layer9_attention_weight_max": 68.13228607177734,
      "activations/layer9_attention_weight_min": -51.656612396240234,
      "epoch": 3.36,
      "learning_rate": 0.00012964545454545454,
      "loss": 2.8872,
      "step": 57800
    },
    {
      "activations/layer0_attention_weight_max": 14.926918029785156,
      "activations/layer0_attention_weight_min": -15.23807144165039,
      "activations/layer10_attention_weight_max": 55.31013870239258,
      "activations/layer10_attention_weight_min": -35.26095199584961,
      "activations/layer11_attention_weight_max": 40.19096374511719,
      "activations/layer11_attention_weight_min": -26.88654899597168,
      "activations/layer12_attention_weight_max": 28.827058792114258,
      "activations/layer12_attention_weight_min": -24.553062438964844,
      "activations/layer13_attention_weight_max": 76.82777404785156,
      "activations/layer13_attention_weight_min": -44.64210891723633,
      "activations/layer14_attention_weight_max": 64.21485900878906,
      "activations/layer14_attention_weight_min": -38.15302276611328,
      "activations/layer15_attention_weight_max": 59.153263092041016,
      "activations/layer15_attention_weight_min": -36.601898193359375,
      "activations/layer16_attention_weight_max": 50.16233825683594,
      "activations/layer16_attention_weight_min": -25.23556137084961,
      "activations/layer17_attention_weight_max": 68.28182983398438,
      "activations/layer17_attention_weight_min": -37.36433029174805,
      "activations/layer18_attention_weight_max": 65.02843475341797,
      "activations/layer18_attention_weight_min": -31.406957626342773,
      "activations/layer19_attention_weight_max": 23.870769500732422,
      "activations/layer19_attention_weight_min": -17.020614624023438,
      "activations/layer1_attention_weight_max": 16.719863891601562,
      "activations/layer1_attention_weight_min": -14.443849563598633,
      "activations/layer20_attention_weight_max": 24.331724166870117,
      "activations/layer20_attention_weight_min": -18.421030044555664,
      "activations/layer21_attention_weight_max": 54.97345733642578,
      "activations/layer21_attention_weight_min": -26.24376678466797,
      "activations/layer22_attention_weight_max": 40.602561950683594,
      "activations/layer22_attention_weight_min": -23.455442428588867,
      "activations/layer23_attention_weight_max": 35.31451416015625,
      "activations/layer23_attention_weight_min": -17.9257869720459,
      "activations/layer2_attention_weight_max": 31.03620719909668,
      "activations/layer2_attention_weight_min": -28.596738815307617,
      "activations/layer3_attention_weight_max": 74.9722671508789,
      "activations/layer3_attention_weight_min": -80.67178344726562,
      "activations/layer4_attention_weight_max": 101.7607650756836,
      "activations/layer4_attention_weight_min": -97.19359588623047,
      "activations/layer5_attention_weight_max": 121.27668762207031,
      "activations/layer5_attention_weight_min": -109.13999938964844,
      "activations/layer6_attention_weight_max": 61.620487213134766,
      "activations/layer6_attention_weight_min": -54.94361877441406,
      "activations/layer7_attention_weight_max": 64.86199951171875,
      "activations/layer7_attention_weight_min": -60.929134368896484,
      "activations/layer8_attention_weight_max": 44.66167068481445,
      "activations/layer8_attention_weight_min": -38.59064865112305,
      "activations/layer9_attention_weight_max": 51.78523635864258,
      "activations/layer9_attention_weight_min": -43.923362731933594,
      "epoch": 3.36,
      "learning_rate": 0.00012962651515151512,
      "loss": 2.9016,
      "step": 57850
    },
    {
      "activations/layer0_attention_weight_max": 13.352285385131836,
      "activations/layer0_attention_weight_min": -14.962260246276855,
      "activations/layer10_attention_weight_max": 44.882568359375,
      "activations/layer10_attention_weight_min": -34.350074768066406,
      "activations/layer11_attention_weight_max": 34.78614044189453,
      "activations/layer11_attention_weight_min": -26.00227165222168,
      "activations/layer12_attention_weight_max": 30.704730987548828,
      "activations/layer12_attention_weight_min": -23.55206298828125,
      "activations/layer13_attention_weight_max": 54.126258850097656,
      "activations/layer13_attention_weight_min": -34.53532028198242,
      "activations/layer14_attention_weight_max": 46.942142486572266,
      "activations/layer14_attention_weight_min": -27.166414260864258,
      "activations/layer15_attention_weight_max": 45.545616149902344,
      "activations/layer15_attention_weight_min": -30.3099422454834,
      "activations/layer16_attention_weight_max": 40.02585220336914,
      "activations/layer16_attention_weight_min": -28.07463264465332,
      "activations/layer17_attention_weight_max": 65.00466918945312,
      "activations/layer17_attention_weight_min": -36.81418991088867,
      "activations/layer18_attention_weight_max": 59.050601959228516,
      "activations/layer18_attention_weight_min": -33.887451171875,
      "activations/layer19_attention_weight_max": 21.985755920410156,
      "activations/layer19_attention_weight_min": -19.7666072845459,
      "activations/layer1_attention_weight_max": 19.163679122924805,
      "activations/layer1_attention_weight_min": -15.352981567382812,
      "activations/layer20_attention_weight_max": 23.90098762512207,
      "activations/layer20_attention_weight_min": -21.103408813476562,
      "activations/layer21_attention_weight_max": 42.0202751159668,
      "activations/layer21_attention_weight_min": -29.877960205078125,
      "activations/layer22_attention_weight_max": 35.820621490478516,
      "activations/layer22_attention_weight_min": -23.41202163696289,
      "activations/layer23_attention_weight_max": 29.45004653930664,
      "activations/layer23_attention_weight_min": -15.968446731567383,
      "activations/layer2_attention_weight_max": 29.987346649169922,
      "activations/layer2_attention_weight_min": -28.973114013671875,
      "activations/layer3_attention_weight_max": 73.33914184570312,
      "activations/layer3_attention_weight_min": -80.05023193359375,
      "activations/layer4_attention_weight_max": 99.3289566040039,
      "activations/layer4_attention_weight_min": -93.6047134399414,
      "activations/layer5_attention_weight_max": 114.4249496459961,
      "activations/layer5_attention_weight_min": -103.03295135498047,
      "activations/layer6_attention_weight_max": 57.98657989501953,
      "activations/layer6_attention_weight_min": -51.06379699707031,
      "activations/layer7_attention_weight_max": 66.67399597167969,
      "activations/layer7_attention_weight_min": -59.45475769042969,
      "activations/layer8_attention_weight_max": 42.176780700683594,
      "activations/layer8_attention_weight_min": -37.94755935668945,
      "activations/layer9_attention_weight_max": 56.72471237182617,
      "activations/layer9_attention_weight_min": -47.26310348510742,
      "epoch": 3.36,
      "learning_rate": 0.00012960795454545454,
      "loss": 2.884,
      "step": 57900
    },
    {
      "activations/layer0_attention_weight_max": 14.771183967590332,
      "activations/layer0_attention_weight_min": -14.575728416442871,
      "activations/layer10_attention_weight_max": 46.36902618408203,
      "activations/layer10_attention_weight_min": -33.54871368408203,
      "activations/layer11_attention_weight_max": 40.137855529785156,
      "activations/layer11_attention_weight_min": -27.935775756835938,
      "activations/layer12_attention_weight_max": 25.188261032104492,
      "activations/layer12_attention_weight_min": -28.9368953704834,
      "activations/layer13_attention_weight_max": 47.93552780151367,
      "activations/layer13_attention_weight_min": -35.436485290527344,
      "activations/layer14_attention_weight_max": 52.50563430786133,
      "activations/layer14_attention_weight_min": -30.291826248168945,
      "activations/layer15_attention_weight_max": 40.62623977661133,
      "activations/layer15_attention_weight_min": -28.99441146850586,
      "activations/layer16_attention_weight_max": 33.0256462097168,
      "activations/layer16_attention_weight_min": -24.616960525512695,
      "activations/layer17_attention_weight_max": 54.70396423339844,
      "activations/layer17_attention_weight_min": -34.49302291870117,
      "activations/layer18_attention_weight_max": 46.982566833496094,
      "activations/layer18_attention_weight_min": -32.54389953613281,
      "activations/layer19_attention_weight_max": 19.88874626159668,
      "activations/layer19_attention_weight_min": -16.935400009155273,
      "activations/layer1_attention_weight_max": 18.071731567382812,
      "activations/layer1_attention_weight_min": -18.32707977294922,
      "activations/layer20_attention_weight_max": 18.796293258666992,
      "activations/layer20_attention_weight_min": -20.37887954711914,
      "activations/layer21_attention_weight_max": 35.75865936279297,
      "activations/layer21_attention_weight_min": -26.596038818359375,
      "activations/layer22_attention_weight_max": 30.885631561279297,
      "activations/layer22_attention_weight_min": -21.750621795654297,
      "activations/layer23_attention_weight_max": 23.113027572631836,
      "activations/layer23_attention_weight_min": -16.22241973876953,
      "activations/layer2_attention_weight_max": 32.49287414550781,
      "activations/layer2_attention_weight_min": -31.036537170410156,
      "activations/layer3_attention_weight_max": 75.26850891113281,
      "activations/layer3_attention_weight_min": -78.1002426147461,
      "activations/layer4_attention_weight_max": 99.56788635253906,
      "activations/layer4_attention_weight_min": -90.49667358398438,
      "activations/layer5_attention_weight_max": 112.89263916015625,
      "activations/layer5_attention_weight_min": -94.93637084960938,
      "activations/layer6_attention_weight_max": 58.85544967651367,
      "activations/layer6_attention_weight_min": -48.859413146972656,
      "activations/layer7_attention_weight_max": 71.70616912841797,
      "activations/layer7_attention_weight_min": -58.02835464477539,
      "activations/layer8_attention_weight_max": 47.750362396240234,
      "activations/layer8_attention_weight_min": -34.04930877685547,
      "activations/layer9_attention_weight_max": 72.65491485595703,
      "activations/layer9_attention_weight_min": -49.31858444213867,
      "epoch": 3.37,
      "learning_rate": 0.00012958901515151513,
      "loss": 2.8866,
      "step": 57950
    },
    {
      "activations/layer0_attention_weight_max": 14.002595901489258,
      "activations/layer0_attention_weight_min": -14.903131484985352,
      "activations/layer10_attention_weight_max": 44.090965270996094,
      "activations/layer10_attention_weight_min": -33.49609375,
      "activations/layer11_attention_weight_max": 34.59087371826172,
      "activations/layer11_attention_weight_min": -25.517860412597656,
      "activations/layer12_attention_weight_max": 27.386878967285156,
      "activations/layer12_attention_weight_min": -31.670839309692383,
      "activations/layer13_attention_weight_max": 49.71534729003906,
      "activations/layer13_attention_weight_min": -38.47960662841797,
      "activations/layer14_attention_weight_max": 38.444740295410156,
      "activations/layer14_attention_weight_min": -30.415115356445312,
      "activations/layer15_attention_weight_max": 45.230140686035156,
      "activations/layer15_attention_weight_min": -31.9261417388916,
      "activations/layer16_attention_weight_max": 37.213016510009766,
      "activations/layer16_attention_weight_min": -25.20376968383789,
      "activations/layer17_attention_weight_max": 59.89934158325195,
      "activations/layer17_attention_weight_min": -37.75068283081055,
      "activations/layer18_attention_weight_max": 49.630531311035156,
      "activations/layer18_attention_weight_min": -33.87405014038086,
      "activations/layer19_attention_weight_max": 20.382604598999023,
      "activations/layer19_attention_weight_min": -17.571874618530273,
      "activations/layer1_attention_weight_max": 16.90749168395996,
      "activations/layer1_attention_weight_min": -13.232073783874512,
      "activations/layer20_attention_weight_max": 21.179363250732422,
      "activations/layer20_attention_weight_min": -21.074480056762695,
      "activations/layer21_attention_weight_max": 46.81529235839844,
      "activations/layer21_attention_weight_min": -26.56093978881836,
      "activations/layer22_attention_weight_max": 33.45524215698242,
      "activations/layer22_attention_weight_min": -22.05548095703125,
      "activations/layer23_attention_weight_max": 26.239646911621094,
      "activations/layer23_attention_weight_min": -15.597728729248047,
      "activations/layer2_attention_weight_max": 31.657642364501953,
      "activations/layer2_attention_weight_min": -29.625743865966797,
      "activations/layer3_attention_weight_max": 73.26122283935547,
      "activations/layer3_attention_weight_min": -81.78277587890625,
      "activations/layer4_attention_weight_max": 100.53234100341797,
      "activations/layer4_attention_weight_min": -99.22107696533203,
      "activations/layer5_attention_weight_max": 118.623779296875,
      "activations/layer5_attention_weight_min": -102.45545959472656,
      "activations/layer6_attention_weight_max": 60.50648880004883,
      "activations/layer6_attention_weight_min": -51.91971206665039,
      "activations/layer7_attention_weight_max": 60.267486572265625,
      "activations/layer7_attention_weight_min": -55.56695556640625,
      "activations/layer8_attention_weight_max": 39.71852111816406,
      "activations/layer8_attention_weight_min": -36.44643020629883,
      "activations/layer9_attention_weight_max": 48.134239196777344,
      "activations/layer9_attention_weight_min": -46.87830352783203,
      "epoch": 3.37,
      "learning_rate": 0.00012957007575757574,
      "loss": 2.9025,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_loss": 2.84375,
      "eval_runtime": 8.4164,
      "eval_samples_per_second": 510.195,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_openwebtext_loss": 2.84375,
      "eval_openwebtext_ppl": 17.180070153709277,
      "eval_openwebtext_runtime": 8.4164,
      "eval_openwebtext_samples_per_second": 510.195,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_wikitext_loss": 3.109375,
      "eval_wikitext_ppl": 22.40703562619394,
      "eval_wikitext_runtime": 1.9427,
      "eval_wikitext_samples_per_second": 234.721,
      "step": 58000
    },
    {
      "epoch": 3.37,
      "eval_lambada_loss": 2.95703125,
      "eval_lambada_ppl": 19.24076585887165,
      "eval_lambada_runtime": 9.5296,
      "eval_lambada_samples_per_second": 510.937,
      "step": 58000
    },
    {
      "activations/layer0_attention_weight_max": 14.263813018798828,
      "activations/layer0_attention_weight_min": -14.670930862426758,
      "activations/layer10_attention_weight_max": 42.53313064575195,
      "activations/layer10_attention_weight_min": -34.33259963989258,
      "activations/layer11_attention_weight_max": 35.19501495361328,
      "activations/layer11_attention_weight_min": -29.739208221435547,
      "activations/layer12_attention_weight_max": 28.18485450744629,
      "activations/layer12_attention_weight_min": -25.748247146606445,
      "activations/layer13_attention_weight_max": 55.40943145751953,
      "activations/layer13_attention_weight_min": -40.6785774230957,
      "activations/layer14_attention_weight_max": 71.07272338867188,
      "activations/layer14_attention_weight_min": -38.59157943725586,
      "activations/layer15_attention_weight_max": 45.97814178466797,
      "activations/layer15_attention_weight_min": -33.193355560302734,
      "activations/layer16_attention_weight_max": 39.29533767700195,
      "activations/layer16_attention_weight_min": -25.98743438720703,
      "activations/layer17_attention_weight_max": 63.85332489013672,
      "activations/layer17_attention_weight_min": -35.00468826293945,
      "activations/layer18_attention_weight_max": 54.22037124633789,
      "activations/layer18_attention_weight_min": -31.982662200927734,
      "activations/layer19_attention_weight_max": 23.89269256591797,
      "activations/layer19_attention_weight_min": -17.189085006713867,
      "activations/layer1_attention_weight_max": 17.636751174926758,
      "activations/layer1_attention_weight_min": -13.790475845336914,
      "activations/layer20_attention_weight_max": 22.10135269165039,
      "activations/layer20_attention_weight_min": -20.041406631469727,
      "activations/layer21_attention_weight_max": 50.526729583740234,
      "activations/layer21_attention_weight_min": -27.497642517089844,
      "activations/layer22_attention_weight_max": 35.796749114990234,
      "activations/layer22_attention_weight_min": -26.59348487854004,
      "activations/layer23_attention_weight_max": 28.63729476928711,
      "activations/layer23_attention_weight_min": -19.023252487182617,
      "activations/layer2_attention_weight_max": 28.881559371948242,
      "activations/layer2_attention_weight_min": -27.77138900756836,
      "activations/layer3_attention_weight_max": 77.55789184570312,
      "activations/layer3_attention_weight_min": -77.69114685058594,
      "activations/layer4_attention_weight_max": 98.90064239501953,
      "activations/layer4_attention_weight_min": -93.39781951904297,
      "activations/layer5_attention_weight_max": 119.74713134765625,
      "activations/layer5_attention_weight_min": -100.10650634765625,
      "activations/layer6_attention_weight_max": 60.68487548828125,
      "activations/layer6_attention_weight_min": -50.99431610107422,
      "activations/layer7_attention_weight_max": 67.28156280517578,
      "activations/layer7_attention_weight_min": -59.15237045288086,
      "activations/layer8_attention_weight_max": 41.032039642333984,
      "activations/layer8_attention_weight_min": -37.08942413330078,
      "activations/layer9_attention_weight_max": 55.913787841796875,
      "activations/layer9_attention_weight_min": -48.0960693359375,
      "epoch": 3.37,
      "learning_rate": 0.00012955113636363636,
      "loss": 2.8957,
      "step": 58050
    },
    {
      "activations/layer0_attention_weight_max": 14.055319786071777,
      "activations/layer0_attention_weight_min": -14.616792678833008,
      "activations/layer10_attention_weight_max": 46.1015510559082,
      "activations/layer10_attention_weight_min": -33.793060302734375,
      "activations/layer11_attention_weight_max": 38.5872802734375,
      "activations/layer11_attention_weight_min": -28.11687469482422,
      "activations/layer12_attention_weight_max": 28.162174224853516,
      "activations/layer12_attention_weight_min": -27.29630470275879,
      "activations/layer13_attention_weight_max": 61.084903717041016,
      "activations/layer13_attention_weight_min": -40.30497741699219,
      "activations/layer14_attention_weight_max": 82.55608367919922,
      "activations/layer14_attention_weight_min": -50.075721740722656,
      "activations/layer15_attention_weight_max": 59.73933410644531,
      "activations/layer15_attention_weight_min": -36.097103118896484,
      "activations/layer16_attention_weight_max": 35.51865005493164,
      "activations/layer16_attention_weight_min": -26.50455665588379,
      "activations/layer17_attention_weight_max": 57.56656265258789,
      "activations/layer17_attention_weight_min": -35.92181396484375,
      "activations/layer18_attention_weight_max": 51.488853454589844,
      "activations/layer18_attention_weight_min": -30.83726692199707,
      "activations/layer19_attention_weight_max": 23.416065216064453,
      "activations/layer19_attention_weight_min": -18.399389266967773,
      "activations/layer1_attention_weight_max": 18.360801696777344,
      "activations/layer1_attention_weight_min": -15.114212036132812,
      "activations/layer20_attention_weight_max": 21.372684478759766,
      "activations/layer20_attention_weight_min": -20.70692253112793,
      "activations/layer21_attention_weight_max": 41.499813079833984,
      "activations/layer21_attention_weight_min": -28.30888557434082,
      "activations/layer22_attention_weight_max": 34.98786163330078,
      "activations/layer22_attention_weight_min": -23.73235321044922,
      "activations/layer23_attention_weight_max": 24.681203842163086,
      "activations/layer23_attention_weight_min": -16.484661102294922,
      "activations/layer2_attention_weight_max": 29.953670501708984,
      "activations/layer2_attention_weight_min": -29.013547897338867,
      "activations/layer3_attention_weight_max": 73.81983184814453,
      "activations/layer3_attention_weight_min": -78.08094787597656,
      "activations/layer4_attention_weight_max": 99.74720764160156,
      "activations/layer4_attention_weight_min": -93.09235382080078,
      "activations/layer5_attention_weight_max": 123.04971313476562,
      "activations/layer5_attention_weight_min": -101.46673583984375,
      "activations/layer6_attention_weight_max": 57.50430679321289,
      "activations/layer6_attention_weight_min": -48.883853912353516,
      "activations/layer7_attention_weight_max": 70.09642028808594,
      "activations/layer7_attention_weight_min": -69.49868774414062,
      "activations/layer8_attention_weight_max": 45.05287551879883,
      "activations/layer8_attention_weight_min": -37.90779495239258,
      "activations/layer9_attention_weight_max": 63.35457992553711,
      "activations/layer9_attention_weight_min": -51.17323303222656,
      "epoch": 3.38,
      "learning_rate": 0.00012953257575757575,
      "loss": 2.8972,
      "step": 58100
    },
    {
      "activations/layer0_attention_weight_max": 14.606776237487793,
      "activations/layer0_attention_weight_min": -14.757177352905273,
      "activations/layer10_attention_weight_max": 40.02753448486328,
      "activations/layer10_attention_weight_min": -34.11412811279297,
      "activations/layer11_attention_weight_max": 33.46610641479492,
      "activations/layer11_attention_weight_min": -24.763227462768555,
      "activations/layer12_attention_weight_max": 24.010330200195312,
      "activations/layer12_attention_weight_min": -23.62067985534668,
      "activations/layer13_attention_weight_max": 54.1651496887207,
      "activations/layer13_attention_weight_min": -41.33671951293945,
      "activations/layer14_attention_weight_max": 55.78034591674805,
      "activations/layer14_attention_weight_min": -32.62626647949219,
      "activations/layer15_attention_weight_max": 45.436527252197266,
      "activations/layer15_attention_weight_min": -31.40876579284668,
      "activations/layer16_attention_weight_max": 41.628658294677734,
      "activations/layer16_attention_weight_min": -27.269779205322266,
      "activations/layer17_attention_weight_max": 65.04243469238281,
      "activations/layer17_attention_weight_min": -34.79946517944336,
      "activations/layer18_attention_weight_max": 59.2390251159668,
      "activations/layer18_attention_weight_min": -30.956504821777344,
      "activations/layer19_attention_weight_max": 24.138282775878906,
      "activations/layer19_attention_weight_min": -20.8757381439209,
      "activations/layer1_attention_weight_max": 17.22467803955078,
      "activations/layer1_attention_weight_min": -13.677116394042969,
      "activations/layer20_attention_weight_max": 23.94273567199707,
      "activations/layer20_attention_weight_min": -19.66621208190918,
      "activations/layer21_attention_weight_max": 49.01428985595703,
      "activations/layer21_attention_weight_min": -26.57699203491211,
      "activations/layer22_attention_weight_max": 42.46965026855469,
      "activations/layer22_attention_weight_min": -24.65948486328125,
      "activations/layer23_attention_weight_max": 29.79414176940918,
      "activations/layer23_attention_weight_min": -17.04330062866211,
      "activations/layer2_attention_weight_max": 30.352079391479492,
      "activations/layer2_attention_weight_min": -28.357223510742188,
      "activations/layer3_attention_weight_max": 74.74800872802734,
      "activations/layer3_attention_weight_min": -79.93217468261719,
      "activations/layer4_attention_weight_max": 100.4289779663086,
      "activations/layer4_attention_weight_min": -94.3742904663086,
      "activations/layer5_attention_weight_max": 114.659423828125,
      "activations/layer5_attention_weight_min": -101.69999694824219,
      "activations/layer6_attention_weight_max": 60.115631103515625,
      "activations/layer6_attention_weight_min": -53.24246597290039,
      "activations/layer7_attention_weight_max": 61.144126892089844,
      "activations/layer7_attention_weight_min": -59.322288513183594,
      "activations/layer8_attention_weight_max": 38.239994049072266,
      "activations/layer8_attention_weight_min": -39.12992858886719,
      "activations/layer9_attention_weight_max": 47.58130645751953,
      "activations/layer9_attention_weight_min": -45.704063415527344,
      "epoch": 3.38,
      "learning_rate": 0.00012951363636363637,
      "loss": 2.898,
      "step": 58150
    },
    {
      "activations/layer0_attention_weight_max": 13.863849639892578,
      "activations/layer0_attention_weight_min": -15.076123237609863,
      "activations/layer10_attention_weight_max": 43.45340347290039,
      "activations/layer10_attention_weight_min": -33.82959747314453,
      "activations/layer11_attention_weight_max": 34.80278015136719,
      "activations/layer11_attention_weight_min": -27.21136474609375,
      "activations/layer12_attention_weight_max": 26.53061294555664,
      "activations/layer12_attention_weight_min": -24.175922393798828,
      "activations/layer13_attention_weight_max": 58.39841079711914,
      "activations/layer13_attention_weight_min": -36.14168930053711,
      "activations/layer14_attention_weight_max": 48.803470611572266,
      "activations/layer14_attention_weight_min": -27.248750686645508,
      "activations/layer15_attention_weight_max": 46.672142028808594,
      "activations/layer15_attention_weight_min": -30.71122169494629,
      "activations/layer16_attention_weight_max": 38.34599304199219,
      "activations/layer16_attention_weight_min": -25.103796005249023,
      "activations/layer17_attention_weight_max": 56.612770080566406,
      "activations/layer17_attention_weight_min": -36.60468292236328,
      "activations/layer18_attention_weight_max": 54.45539474487305,
      "activations/layer18_attention_weight_min": -30.885276794433594,
      "activations/layer19_attention_weight_max": 22.382081985473633,
      "activations/layer19_attention_weight_min": -18.895872116088867,
      "activations/layer1_attention_weight_max": 16.879486083984375,
      "activations/layer1_attention_weight_min": -15.106496810913086,
      "activations/layer20_attention_weight_max": 22.02220916748047,
      "activations/layer20_attention_weight_min": -23.93614959716797,
      "activations/layer21_attention_weight_max": 45.34556198120117,
      "activations/layer21_attention_weight_min": -28.591703414916992,
      "activations/layer22_attention_weight_max": 35.005767822265625,
      "activations/layer22_attention_weight_min": -22.8807315826416,
      "activations/layer23_attention_weight_max": 24.366092681884766,
      "activations/layer23_attention_weight_min": -18.76609992980957,
      "activations/layer2_attention_weight_max": 28.688522338867188,
      "activations/layer2_attention_weight_min": -27.57269859313965,
      "activations/layer3_attention_weight_max": 70.72433471679688,
      "activations/layer3_attention_weight_min": -76.78022003173828,
      "activations/layer4_attention_weight_max": 96.17626190185547,
      "activations/layer4_attention_weight_min": -90.30260467529297,
      "activations/layer5_attention_weight_max": 116.4373779296875,
      "activations/layer5_attention_weight_min": -98.93273162841797,
      "activations/layer6_attention_weight_max": 59.20634078979492,
      "activations/layer6_attention_weight_min": -51.174842834472656,
      "activations/layer7_attention_weight_max": 66.35736083984375,
      "activations/layer7_attention_weight_min": -55.739349365234375,
      "activations/layer8_attention_weight_max": 39.905426025390625,
      "activations/layer8_attention_weight_min": -37.0986213684082,
      "activations/layer9_attention_weight_max": 51.66796875,
      "activations/layer9_attention_weight_min": -46.08432388305664,
      "epoch": 3.38,
      "learning_rate": 0.00012949469696969695,
      "loss": 2.8956,
      "step": 58200
    },
    {
      "activations/layer0_attention_weight_max": 14.22763729095459,
      "activations/layer0_attention_weight_min": -14.361112594604492,
      "activations/layer10_attention_weight_max": 38.986114501953125,
      "activations/layer10_attention_weight_min": -32.96643829345703,
      "activations/layer11_attention_weight_max": 34.55562973022461,
      "activations/layer11_attention_weight_min": -25.133825302124023,
      "activations/layer12_attention_weight_max": 28.31913948059082,
      "activations/layer12_attention_weight_min": -27.47586441040039,
      "activations/layer13_attention_weight_max": 44.194332122802734,
      "activations/layer13_attention_weight_min": -32.382171630859375,
      "activations/layer14_attention_weight_max": 36.81879806518555,
      "activations/layer14_attention_weight_min": -29.881324768066406,
      "activations/layer15_attention_weight_max": 41.28692626953125,
      "activations/layer15_attention_weight_min": -33.53701400756836,
      "activations/layer16_attention_weight_max": 35.183563232421875,
      "activations/layer16_attention_weight_min": -26.03947639465332,
      "activations/layer17_attention_weight_max": 60.10689163208008,
      "activations/layer17_attention_weight_min": -34.87282180786133,
      "activations/layer18_attention_weight_max": 51.50840377807617,
      "activations/layer18_attention_weight_min": -32.879634857177734,
      "activations/layer19_attention_weight_max": 22.98048210144043,
      "activations/layer19_attention_weight_min": -18.553058624267578,
      "activations/layer1_attention_weight_max": 17.2496337890625,
      "activations/layer1_attention_weight_min": -14.526302337646484,
      "activations/layer20_attention_weight_max": 23.043169021606445,
      "activations/layer20_attention_weight_min": -20.77965545654297,
      "activations/layer21_attention_weight_max": 40.66000747680664,
      "activations/layer21_attention_weight_min": -27.82900619506836,
      "activations/layer22_attention_weight_max": 33.150569915771484,
      "activations/layer22_attention_weight_min": -22.39008331298828,
      "activations/layer23_attention_weight_max": 26.178434371948242,
      "activations/layer23_attention_weight_min": -19.82219696044922,
      "activations/layer2_attention_weight_max": 28.855701446533203,
      "activations/layer2_attention_weight_min": -27.69831085205078,
      "activations/layer3_attention_weight_max": 71.2726821899414,
      "activations/layer3_attention_weight_min": -74.7104721069336,
      "activations/layer4_attention_weight_max": 96.22207641601562,
      "activations/layer4_attention_weight_min": -89.11573791503906,
      "activations/layer5_attention_weight_max": 116.12069702148438,
      "activations/layer5_attention_weight_min": -108.22481536865234,
      "activations/layer6_attention_weight_max": 57.078460693359375,
      "activations/layer6_attention_weight_min": -53.17121124267578,
      "activations/layer7_attention_weight_max": 58.85896301269531,
      "activations/layer7_attention_weight_min": -61.810569763183594,
      "activations/layer8_attention_weight_max": 38.956172943115234,
      "activations/layer8_attention_weight_min": -37.25123596191406,
      "activations/layer9_attention_weight_max": 46.31740951538086,
      "activations/layer9_attention_weight_min": -42.718299865722656,
      "epoch": 3.38,
      "learning_rate": 0.00012947575757575757,
      "loss": 2.8983,
      "step": 58250
    },
    {
      "activations/layer0_attention_weight_max": 18.323623657226562,
      "activations/layer0_attention_weight_min": -14.715066909790039,
      "activations/layer10_attention_weight_max": 45.50804138183594,
      "activations/layer10_attention_weight_min": -33.47353744506836,
      "activations/layer11_attention_weight_max": 35.133426666259766,
      "activations/layer11_attention_weight_min": -26.955833435058594,
      "activations/layer12_attention_weight_max": 34.22804641723633,
      "activations/layer12_attention_weight_min": -24.379953384399414,
      "activations/layer13_attention_weight_max": 56.291534423828125,
      "activations/layer13_attention_weight_min": -42.30332946777344,
      "activations/layer14_attention_weight_max": 49.553009033203125,
      "activations/layer14_attention_weight_min": -37.4669303894043,
      "activations/layer15_attention_weight_max": 50.238033294677734,
      "activations/layer15_attention_weight_min": -32.15018081665039,
      "activations/layer16_attention_weight_max": 38.37905502319336,
      "activations/layer16_attention_weight_min": -27.267967224121094,
      "activations/layer17_attention_weight_max": 62.85696792602539,
      "activations/layer17_attention_weight_min": -37.27279281616211,
      "activations/layer18_attention_weight_max": 55.99148941040039,
      "activations/layer18_attention_weight_min": -34.4064826965332,
      "activations/layer19_attention_weight_max": 25.312692642211914,
      "activations/layer19_attention_weight_min": -18.926097869873047,
      "activations/layer1_attention_weight_max": 17.596088409423828,
      "activations/layer1_attention_weight_min": -16.614017486572266,
      "activations/layer20_attention_weight_max": 22.80377960205078,
      "activations/layer20_attention_weight_min": -18.973896026611328,
      "activations/layer21_attention_weight_max": 47.183990478515625,
      "activations/layer21_attention_weight_min": -26.543338775634766,
      "activations/layer22_attention_weight_max": 39.45199203491211,
      "activations/layer22_attention_weight_min": -22.82334327697754,
      "activations/layer23_attention_weight_max": 29.65536117553711,
      "activations/layer23_attention_weight_min": -19.1302547454834,
      "activations/layer2_attention_weight_max": 32.201534271240234,
      "activations/layer2_attention_weight_min": -28.837657928466797,
      "activations/layer3_attention_weight_max": 74.45231628417969,
      "activations/layer3_attention_weight_min": -77.58972930908203,
      "activations/layer4_attention_weight_max": 102.12596893310547,
      "activations/layer4_attention_weight_min": -94.29403686523438,
      "activations/layer5_attention_weight_max": 116.57832336425781,
      "activations/layer5_attention_weight_min": -98.4992904663086,
      "activations/layer6_attention_weight_max": 61.40412521362305,
      "activations/layer6_attention_weight_min": -50.28004837036133,
      "activations/layer7_attention_weight_max": 67.57288360595703,
      "activations/layer7_attention_weight_min": -56.9384651184082,
      "activations/layer8_attention_weight_max": 41.47087097167969,
      "activations/layer8_attention_weight_min": -37.197479248046875,
      "activations/layer9_attention_weight_max": 53.06287384033203,
      "activations/layer9_attention_weight_min": -49.40719223022461,
      "epoch": 3.39,
      "learning_rate": 0.00012945681818181818,
      "loss": 2.901,
      "step": 58300
    },
    {
      "activations/layer0_attention_weight_max": 13.820343017578125,
      "activations/layer0_attention_weight_min": -15.029806137084961,
      "activations/layer10_attention_weight_max": 40.41531753540039,
      "activations/layer10_attention_weight_min": -32.61219024658203,
      "activations/layer11_attention_weight_max": 34.0013542175293,
      "activations/layer11_attention_weight_min": -25.147756576538086,
      "activations/layer12_attention_weight_max": 25.083921432495117,
      "activations/layer12_attention_weight_min": -28.407203674316406,
      "activations/layer13_attention_weight_max": 53.472129821777344,
      "activations/layer13_attention_weight_min": -39.90873718261719,
      "activations/layer14_attention_weight_max": 57.69982147216797,
      "activations/layer14_attention_weight_min": -40.76277542114258,
      "activations/layer15_attention_weight_max": 45.028038024902344,
      "activations/layer15_attention_weight_min": -31.01921844482422,
      "activations/layer16_attention_weight_max": 38.813724517822266,
      "activations/layer16_attention_weight_min": -26.55968475341797,
      "activations/layer17_attention_weight_max": 59.19426345825195,
      "activations/layer17_attention_weight_min": -38.84904861450195,
      "activations/layer18_attention_weight_max": 59.83131790161133,
      "activations/layer18_attention_weight_min": -32.3886833190918,
      "activations/layer19_attention_weight_max": 19.973358154296875,
      "activations/layer19_attention_weight_min": -17.58539581298828,
      "activations/layer1_attention_weight_max": 17.379106521606445,
      "activations/layer1_attention_weight_min": -16.58100700378418,
      "activations/layer20_attention_weight_max": 23.00006675720215,
      "activations/layer20_attention_weight_min": -20.163381576538086,
      "activations/layer21_attention_weight_max": 45.73091506958008,
      "activations/layer21_attention_weight_min": -29.19425392150879,
      "activations/layer22_attention_weight_max": 32.70469284057617,
      "activations/layer22_attention_weight_min": -25.30112075805664,
      "activations/layer23_attention_weight_max": 25.07453727722168,
      "activations/layer23_attention_weight_min": -16.70149803161621,
      "activations/layer2_attention_weight_max": 32.75572967529297,
      "activations/layer2_attention_weight_min": -29.61071014404297,
      "activations/layer3_attention_weight_max": 76.39913177490234,
      "activations/layer3_attention_weight_min": -78.33358001708984,
      "activations/layer4_attention_weight_max": 103.63703918457031,
      "activations/layer4_attention_weight_min": -96.82671356201172,
      "activations/layer5_attention_weight_max": 120.29428100585938,
      "activations/layer5_attention_weight_min": -101.83467102050781,
      "activations/layer6_attention_weight_max": 63.230506896972656,
      "activations/layer6_attention_weight_min": -55.045326232910156,
      "activations/layer7_attention_weight_max": 63.86357879638672,
      "activations/layer7_attention_weight_min": -59.91698455810547,
      "activations/layer8_attention_weight_max": 40.65166091918945,
      "activations/layer8_attention_weight_min": -38.08616638183594,
      "activations/layer9_attention_weight_max": 50.848079681396484,
      "activations/layer9_attention_weight_min": -48.60902786254883,
      "epoch": 3.39,
      "learning_rate": 0.0001294378787878788,
      "loss": 2.8769,
      "step": 58350
    },
    {
      "activations/layer0_attention_weight_max": 14.503622055053711,
      "activations/layer0_attention_weight_min": -14.717089653015137,
      "activations/layer10_attention_weight_max": 48.782588958740234,
      "activations/layer10_attention_weight_min": -38.20905303955078,
      "activations/layer11_attention_weight_max": 43.46600341796875,
      "activations/layer11_attention_weight_min": -30.46820831298828,
      "activations/layer12_attention_weight_max": 31.563108444213867,
      "activations/layer12_attention_weight_min": -25.813188552856445,
      "activations/layer13_attention_weight_max": 70.68092346191406,
      "activations/layer13_attention_weight_min": -46.15873718261719,
      "activations/layer14_attention_weight_max": 59.16643142700195,
      "activations/layer14_attention_weight_min": -39.12688446044922,
      "activations/layer15_attention_weight_max": 52.0286979675293,
      "activations/layer15_attention_weight_min": -34.53847885131836,
      "activations/layer16_attention_weight_max": 36.580020904541016,
      "activations/layer16_attention_weight_min": -26.371274948120117,
      "activations/layer17_attention_weight_max": 57.10456848144531,
      "activations/layer17_attention_weight_min": -36.005550384521484,
      "activations/layer18_attention_weight_max": 53.56517791748047,
      "activations/layer18_attention_weight_min": -32.801151275634766,
      "activations/layer19_attention_weight_max": 23.08686065673828,
      "activations/layer19_attention_weight_min": -18.532011032104492,
      "activations/layer1_attention_weight_max": 18.921432495117188,
      "activations/layer1_attention_weight_min": -15.329564094543457,
      "activations/layer20_attention_weight_max": 25.367109298706055,
      "activations/layer20_attention_weight_min": -18.632511138916016,
      "activations/layer21_attention_weight_max": 47.215938568115234,
      "activations/layer21_attention_weight_min": -28.780914306640625,
      "activations/layer22_attention_weight_max": 34.64701843261719,
      "activations/layer22_attention_weight_min": -22.09632110595703,
      "activations/layer23_attention_weight_max": 27.733352661132812,
      "activations/layer23_attention_weight_min": -16.69825553894043,
      "activations/layer2_attention_weight_max": 31.772811889648438,
      "activations/layer2_attention_weight_min": -28.134048461914062,
      "activations/layer3_attention_weight_max": 76.03536987304688,
      "activations/layer3_attention_weight_min": -80.40534210205078,
      "activations/layer4_attention_weight_max": 109.54647064208984,
      "activations/layer4_attention_weight_min": -96.816650390625,
      "activations/layer5_attention_weight_max": 128.63079833984375,
      "activations/layer5_attention_weight_min": -96.88677215576172,
      "activations/layer6_attention_weight_max": 67.30195617675781,
      "activations/layer6_attention_weight_min": -51.428565979003906,
      "activations/layer7_attention_weight_max": 75.41657257080078,
      "activations/layer7_attention_weight_min": -64.83694458007812,
      "activations/layer8_attention_weight_max": 47.77046203613281,
      "activations/layer8_attention_weight_min": -39.73374938964844,
      "activations/layer9_attention_weight_max": 65.28987884521484,
      "activations/layer9_attention_weight_min": -51.47343826293945,
      "epoch": 3.39,
      "learning_rate": 0.00012941893939393939,
      "loss": 2.8986,
      "step": 58400
    },
    {
      "activations/layer0_attention_weight_max": 15.281033515930176,
      "activations/layer0_attention_weight_min": -14.657414436340332,
      "activations/layer10_attention_weight_max": 53.12577819824219,
      "activations/layer10_attention_weight_min": -39.892250061035156,
      "activations/layer11_attention_weight_max": 45.05640411376953,
      "activations/layer11_attention_weight_min": -31.40846061706543,
      "activations/layer12_attention_weight_max": 25.321815490722656,
      "activations/layer12_attention_weight_min": -26.400644302368164,
      "activations/layer13_attention_weight_max": 72.69845581054688,
      "activations/layer13_attention_weight_min": -41.98413848876953,
      "activations/layer14_attention_weight_max": 100.00939178466797,
      "activations/layer14_attention_weight_min": -49.26963806152344,
      "activations/layer15_attention_weight_max": 68.97969818115234,
      "activations/layer15_attention_weight_min": -34.68610382080078,
      "activations/layer16_attention_weight_max": 32.721683502197266,
      "activations/layer16_attention_weight_min": -24.695974349975586,
      "activations/layer17_attention_weight_max": 53.04610061645508,
      "activations/layer17_attention_weight_min": -35.82722854614258,
      "activations/layer18_attention_weight_max": 46.903785705566406,
      "activations/layer18_attention_weight_min": -31.360937118530273,
      "activations/layer19_attention_weight_max": 21.352628707885742,
      "activations/layer19_attention_weight_min": -18.663524627685547,
      "activations/layer1_attention_weight_max": 17.51372528076172,
      "activations/layer1_attention_weight_min": -17.907718658447266,
      "activations/layer20_attention_weight_max": 18.738767623901367,
      "activations/layer20_attention_weight_min": -21.09575653076172,
      "activations/layer21_attention_weight_max": 39.17137145996094,
      "activations/layer21_attention_weight_min": -28.611370086669922,
      "activations/layer22_attention_weight_max": 32.24998092651367,
      "activations/layer22_attention_weight_min": -21.208831787109375,
      "activations/layer23_attention_weight_max": 22.499494552612305,
      "activations/layer23_attention_weight_min": -15.398688316345215,
      "activations/layer2_attention_weight_max": 28.309066772460938,
      "activations/layer2_attention_weight_min": -28.117713928222656,
      "activations/layer3_attention_weight_max": 76.39773559570312,
      "activations/layer3_attention_weight_min": -79.97886657714844,
      "activations/layer4_attention_weight_max": 103.90691375732422,
      "activations/layer4_attention_weight_min": -97.3102798461914,
      "activations/layer5_attention_weight_max": 120.33580780029297,
      "activations/layer5_attention_weight_min": -103.52969360351562,
      "activations/layer6_attention_weight_max": 61.67268371582031,
      "activations/layer6_attention_weight_min": -52.975608825683594,
      "activations/layer7_attention_weight_max": 72.11028289794922,
      "activations/layer7_attention_weight_min": -59.37785339355469,
      "activations/layer8_attention_weight_max": 47.622928619384766,
      "activations/layer8_attention_weight_min": -40.776798248291016,
      "activations/layer9_attention_weight_max": 65.89214324951172,
      "activations/layer9_attention_weight_min": -61.03961181640625,
      "epoch": 3.4,
      "learning_rate": 0.00012940037878787878,
      "loss": 2.8929,
      "step": 58450
    },
    {
      "activations/layer0_attention_weight_max": 15.152796745300293,
      "activations/layer0_attention_weight_min": -14.619531631469727,
      "activations/layer10_attention_weight_max": 51.82276153564453,
      "activations/layer10_attention_weight_min": -34.42247009277344,
      "activations/layer11_attention_weight_max": 46.163414001464844,
      "activations/layer11_attention_weight_min": -26.957778930664062,
      "activations/layer12_attention_weight_max": 29.66824722290039,
      "activations/layer12_attention_weight_min": -24.99131965637207,
      "activations/layer13_attention_weight_max": 70.65640258789062,
      "activations/layer13_attention_weight_min": -44.33124923706055,
      "activations/layer14_attention_weight_max": 60.13751220703125,
      "activations/layer14_attention_weight_min": -32.15645980834961,
      "activations/layer15_attention_weight_max": 48.7779541015625,
      "activations/layer15_attention_weight_min": -33.74933624267578,
      "activations/layer16_attention_weight_max": 42.505008697509766,
      "activations/layer16_attention_weight_min": -28.355304718017578,
      "activations/layer17_attention_weight_max": 71.4566650390625,
      "activations/layer17_attention_weight_min": -35.27187728881836,
      "activations/layer18_attention_weight_max": 55.95970916748047,
      "activations/layer18_attention_weight_min": -31.940458297729492,
      "activations/layer19_attention_weight_max": 25.115856170654297,
      "activations/layer19_attention_weight_min": -18.324851989746094,
      "activations/layer1_attention_weight_max": 19.183061599731445,
      "activations/layer1_attention_weight_min": -13.521536827087402,
      "activations/layer20_attention_weight_max": 20.748165130615234,
      "activations/layer20_attention_weight_min": -17.892866134643555,
      "activations/layer21_attention_weight_max": 50.104637145996094,
      "activations/layer21_attention_weight_min": -26.953500747680664,
      "activations/layer22_attention_weight_max": 36.02003479003906,
      "activations/layer22_attention_weight_min": -24.977725982666016,
      "activations/layer23_attention_weight_max": 27.040990829467773,
      "activations/layer23_attention_weight_min": -16.631677627563477,
      "activations/layer2_attention_weight_max": 31.341964721679688,
      "activations/layer2_attention_weight_min": -27.838369369506836,
      "activations/layer3_attention_weight_max": 76.64131164550781,
      "activations/layer3_attention_weight_min": -79.4708023071289,
      "activations/layer4_attention_weight_max": 106.40262603759766,
      "activations/layer4_attention_weight_min": -95.76850128173828,
      "activations/layer5_attention_weight_max": 127.16869354248047,
      "activations/layer5_attention_weight_min": -100.30867004394531,
      "activations/layer6_attention_weight_max": 69.08733367919922,
      "activations/layer6_attention_weight_min": -54.71866989135742,
      "activations/layer7_attention_weight_max": 77.50611877441406,
      "activations/layer7_attention_weight_min": -64.21627807617188,
      "activations/layer8_attention_weight_max": 54.329833984375,
      "activations/layer8_attention_weight_min": -42.573360443115234,
      "activations/layer9_attention_weight_max": 66.22525787353516,
      "activations/layer9_attention_weight_min": -54.13528060913086,
      "epoch": 3.4,
      "learning_rate": 0.0001293814393939394,
      "loss": 2.9081,
      "step": 58500
    },
    {
      "activations/layer0_attention_weight_max": 13.113595008850098,
      "activations/layer0_attention_weight_min": -14.646185874938965,
      "activations/layer10_attention_weight_max": 44.305076599121094,
      "activations/layer10_attention_weight_min": -34.02698516845703,
      "activations/layer11_attention_weight_max": 38.05788803100586,
      "activations/layer11_attention_weight_min": -27.49263572692871,
      "activations/layer12_attention_weight_max": 25.352218627929688,
      "activations/layer12_attention_weight_min": -23.95072364807129,
      "activations/layer13_attention_weight_max": 54.25680923461914,
      "activations/layer13_attention_weight_min": -37.16837692260742,
      "activations/layer14_attention_weight_max": 51.71079635620117,
      "activations/layer14_attention_weight_min": -32.50932312011719,
      "activations/layer15_attention_weight_max": 49.08901596069336,
      "activations/layer15_attention_weight_min": -30.17021942138672,
      "activations/layer16_attention_weight_max": 36.89751434326172,
      "activations/layer16_attention_weight_min": -27.009807586669922,
      "activations/layer17_attention_weight_max": 67.16317749023438,
      "activations/layer17_attention_weight_min": -34.648681640625,
      "activations/layer18_attention_weight_max": 54.18623352050781,
      "activations/layer18_attention_weight_min": -34.91652297973633,
      "activations/layer19_attention_weight_max": 24.300710678100586,
      "activations/layer19_attention_weight_min": -17.27744483947754,
      "activations/layer1_attention_weight_max": 18.347143173217773,
      "activations/layer1_attention_weight_min": -14.021550178527832,
      "activations/layer20_attention_weight_max": 23.363506317138672,
      "activations/layer20_attention_weight_min": -20.42473602294922,
      "activations/layer21_attention_weight_max": 45.012290954589844,
      "activations/layer21_attention_weight_min": -26.80338478088379,
      "activations/layer22_attention_weight_max": 37.35420227050781,
      "activations/layer22_attention_weight_min": -22.319150924682617,
      "activations/layer23_attention_weight_max": 26.148649215698242,
      "activations/layer23_attention_weight_min": -15.271464347839355,
      "activations/layer2_attention_weight_max": 29.27899932861328,
      "activations/layer2_attention_weight_min": -27.538177490234375,
      "activations/layer3_attention_weight_max": 75.15218353271484,
      "activations/layer3_attention_weight_min": -78.4582290649414,
      "activations/layer4_attention_weight_max": 98.39096069335938,
      "activations/layer4_attention_weight_min": -93.2629623413086,
      "activations/layer5_attention_weight_max": 114.4239730834961,
      "activations/layer5_attention_weight_min": -97.94985961914062,
      "activations/layer6_attention_weight_max": 59.441162109375,
      "activations/layer6_attention_weight_min": -52.210411071777344,
      "activations/layer7_attention_weight_max": 61.900760650634766,
      "activations/layer7_attention_weight_min": -58.53062057495117,
      "activations/layer8_attention_weight_max": 42.13396072387695,
      "activations/layer8_attention_weight_min": -37.3324089050293,
      "activations/layer9_attention_weight_max": 50.40743637084961,
      "activations/layer9_attention_weight_min": -44.586830139160156,
      "epoch": 3.4,
      "learning_rate": 0.0001293625,
      "loss": 2.8882,
      "step": 58550
    },
    {
      "activations/layer0_attention_weight_max": 14.15414810180664,
      "activations/layer0_attention_weight_min": -14.82216739654541,
      "activations/layer10_attention_weight_max": 43.86153030395508,
      "activations/layer10_attention_weight_min": -34.138431549072266,
      "activations/layer11_attention_weight_max": 37.56697082519531,
      "activations/layer11_attention_weight_min": -27.545185089111328,
      "activations/layer12_attention_weight_max": 28.809907913208008,
      "activations/layer12_attention_weight_min": -24.902441024780273,
      "activations/layer13_attention_weight_max": 57.74811935424805,
      "activations/layer13_attention_weight_min": -39.34746170043945,
      "activations/layer14_attention_weight_max": 48.84346008300781,
      "activations/layer14_attention_weight_min": -32.98296356201172,
      "activations/layer15_attention_weight_max": 49.15861892700195,
      "activations/layer15_attention_weight_min": -33.40267562866211,
      "activations/layer16_attention_weight_max": 34.824851989746094,
      "activations/layer16_attention_weight_min": -27.959232330322266,
      "activations/layer17_attention_weight_max": 60.14883804321289,
      "activations/layer17_attention_weight_min": -36.8031120300293,
      "activations/layer18_attention_weight_max": 51.50185775756836,
      "activations/layer18_attention_weight_min": -32.0151481628418,
      "activations/layer19_attention_weight_max": 20.167219161987305,
      "activations/layer19_attention_weight_min": -17.594289779663086,
      "activations/layer1_attention_weight_max": 17.2552490234375,
      "activations/layer1_attention_weight_min": -13.734297752380371,
      "activations/layer20_attention_weight_max": 21.663440704345703,
      "activations/layer20_attention_weight_min": -18.31534194946289,
      "activations/layer21_attention_weight_max": 39.3857307434082,
      "activations/layer21_attention_weight_min": -25.689231872558594,
      "activations/layer22_attention_weight_max": 34.32624816894531,
      "activations/layer22_attention_weight_min": -22.782386779785156,
      "activations/layer23_attention_weight_max": 25.76645278930664,
      "activations/layer23_attention_weight_min": -19.424785614013672,
      "activations/layer2_attention_weight_max": 29.571544647216797,
      "activations/layer2_attention_weight_min": -28.322376251220703,
      "activations/layer3_attention_weight_max": 72.5478515625,
      "activations/layer3_attention_weight_min": -74.38859558105469,
      "activations/layer4_attention_weight_max": 98.3041000366211,
      "activations/layer4_attention_weight_min": -96.61605072021484,
      "activations/layer5_attention_weight_max": 122.52546691894531,
      "activations/layer5_attention_weight_min": -100.32147216796875,
      "activations/layer6_attention_weight_max": 66.72256469726562,
      "activations/layer6_attention_weight_min": -54.563072204589844,
      "activations/layer7_attention_weight_max": 76.18236541748047,
      "activations/layer7_attention_weight_min": -60.45525360107422,
      "activations/layer8_attention_weight_max": 49.762840270996094,
      "activations/layer8_attention_weight_min": -35.97534942626953,
      "activations/layer9_attention_weight_max": 58.939422607421875,
      "activations/layer9_attention_weight_min": -48.16599655151367,
      "epoch": 3.4,
      "learning_rate": 0.0001293435606060606,
      "loss": 2.8899,
      "step": 58600
    },
    {
      "activations/layer0_attention_weight_max": 14.386076927185059,
      "activations/layer0_attention_weight_min": -14.46336841583252,
      "activations/layer10_attention_weight_max": 43.71612548828125,
      "activations/layer10_attention_weight_min": -36.61967849731445,
      "activations/layer11_attention_weight_max": 35.570594787597656,
      "activations/layer11_attention_weight_min": -29.49178123474121,
      "activations/layer12_attention_weight_max": 36.43549346923828,
      "activations/layer12_attention_weight_min": -25.39832305908203,
      "activations/layer13_attention_weight_max": 58.386680603027344,
      "activations/layer13_attention_weight_min": -38.71709060668945,
      "activations/layer14_attention_weight_max": 62.11642074584961,
      "activations/layer14_attention_weight_min": -37.62892150878906,
      "activations/layer15_attention_weight_max": 47.131507873535156,
      "activations/layer15_attention_weight_min": -31.855844497680664,
      "activations/layer16_attention_weight_max": 41.385948181152344,
      "activations/layer16_attention_weight_min": -27.833454132080078,
      "activations/layer17_attention_weight_max": 57.93030548095703,
      "activations/layer17_attention_weight_min": -35.52313232421875,
      "activations/layer18_attention_weight_max": 54.44363784790039,
      "activations/layer18_attention_weight_min": -32.76335906982422,
      "activations/layer19_attention_weight_max": 21.68314552307129,
      "activations/layer19_attention_weight_min": -17.951467514038086,
      "activations/layer1_attention_weight_max": 17.9196720123291,
      "activations/layer1_attention_weight_min": -15.285825729370117,
      "activations/layer20_attention_weight_max": 18.937728881835938,
      "activations/layer20_attention_weight_min": -16.625717163085938,
      "activations/layer21_attention_weight_max": 42.71884536743164,
      "activations/layer21_attention_weight_min": -26.48635482788086,
      "activations/layer22_attention_weight_max": 37.04364776611328,
      "activations/layer22_attention_weight_min": -21.72365379333496,
      "activations/layer23_attention_weight_max": 24.381834030151367,
      "activations/layer23_attention_weight_min": -14.186983108520508,
      "activations/layer2_attention_weight_max": 29.7460994720459,
      "activations/layer2_attention_weight_min": -29.567846298217773,
      "activations/layer3_attention_weight_max": 72.06468200683594,
      "activations/layer3_attention_weight_min": -79.05467224121094,
      "activations/layer4_attention_weight_max": 99.2825698852539,
      "activations/layer4_attention_weight_min": -91.80243682861328,
      "activations/layer5_attention_weight_max": 108.46809387207031,
      "activations/layer5_attention_weight_min": -101.40351867675781,
      "activations/layer6_attention_weight_max": 56.18332290649414,
      "activations/layer6_attention_weight_min": -52.75869369506836,
      "activations/layer7_attention_weight_max": 62.21707534790039,
      "activations/layer7_attention_weight_min": -58.0966682434082,
      "activations/layer8_attention_weight_max": 42.23289108276367,
      "activations/layer8_attention_weight_min": -36.574737548828125,
      "activations/layer9_attention_weight_max": 56.269439697265625,
      "activations/layer9_attention_weight_min": -49.46657943725586,
      "epoch": 3.41,
      "learning_rate": 0.0001293246212121212,
      "loss": 2.8812,
      "step": 58650
    },
    {
      "activations/layer0_attention_weight_max": 13.385318756103516,
      "activations/layer0_attention_weight_min": -14.758584976196289,
      "activations/layer10_attention_weight_max": 39.24264144897461,
      "activations/layer10_attention_weight_min": -33.37192153930664,
      "activations/layer11_attention_weight_max": 33.03913879394531,
      "activations/layer11_attention_weight_min": -25.47791290283203,
      "activations/layer12_attention_weight_max": 21.522733688354492,
      "activations/layer12_attention_weight_min": -24.521757125854492,
      "activations/layer13_attention_weight_max": 52.725894927978516,
      "activations/layer13_attention_weight_min": -40.49406814575195,
      "activations/layer14_attention_weight_max": 43.83509826660156,
      "activations/layer14_attention_weight_min": -28.24362564086914,
      "activations/layer15_attention_weight_max": 44.23080062866211,
      "activations/layer15_attention_weight_min": -33.98736572265625,
      "activations/layer16_attention_weight_max": 32.7567138671875,
      "activations/layer16_attention_weight_min": -27.008081436157227,
      "activations/layer17_attention_weight_max": 62.04108428955078,
      "activations/layer17_attention_weight_min": -37.68540573120117,
      "activations/layer18_attention_weight_max": 49.15888977050781,
      "activations/layer18_attention_weight_min": -34.4974479675293,
      "activations/layer19_attention_weight_max": 22.46024513244629,
      "activations/layer19_attention_weight_min": -18.447500228881836,
      "activations/layer1_attention_weight_max": 17.17999267578125,
      "activations/layer1_attention_weight_min": -13.602155685424805,
      "activations/layer20_attention_weight_max": 21.3900203704834,
      "activations/layer20_attention_weight_min": -18.515987396240234,
      "activations/layer21_attention_weight_max": 38.617191314697266,
      "activations/layer21_attention_weight_min": -30.249710083007812,
      "activations/layer22_attention_weight_max": 38.10980224609375,
      "activations/layer22_attention_weight_min": -23.351848602294922,
      "activations/layer23_attention_weight_max": 26.53289031982422,
      "activations/layer23_attention_weight_min": -17.410470962524414,
      "activations/layer2_attention_weight_max": 29.78388023376465,
      "activations/layer2_attention_weight_min": -28.037479400634766,
      "activations/layer3_attention_weight_max": 73.1012954711914,
      "activations/layer3_attention_weight_min": -77.32038879394531,
      "activations/layer4_attention_weight_max": 98.2049789428711,
      "activations/layer4_attention_weight_min": -90.62227630615234,
      "activations/layer5_attention_weight_max": 109.39644622802734,
      "activations/layer5_attention_weight_min": -102.27973937988281,
      "activations/layer6_attention_weight_max": 57.892234802246094,
      "activations/layer6_attention_weight_min": -51.0468635559082,
      "activations/layer7_attention_weight_max": 60.26908874511719,
      "activations/layer7_attention_weight_min": -58.180030822753906,
      "activations/layer8_attention_weight_max": 37.127471923828125,
      "activations/layer8_attention_weight_min": -38.661048889160156,
      "activations/layer9_attention_weight_max": 51.02994918823242,
      "activations/layer9_attention_weight_min": -43.130916595458984,
      "epoch": 3.41,
      "learning_rate": 0.0001293056818181818,
      "loss": 2.8966,
      "step": 58700
    },
    {
      "activations/layer0_attention_weight_max": 15.053638458251953,
      "activations/layer0_attention_weight_min": -14.475232124328613,
      "activations/layer10_attention_weight_max": 43.19658660888672,
      "activations/layer10_attention_weight_min": -34.94868850708008,
      "activations/layer11_attention_weight_max": 37.163429260253906,
      "activations/layer11_attention_weight_min": -27.49486541748047,
      "activations/layer12_attention_weight_max": 23.68588638305664,
      "activations/layer12_attention_weight_min": -27.625137329101562,
      "activations/layer13_attention_weight_max": 51.298160552978516,
      "activations/layer13_attention_weight_min": -37.87957763671875,
      "activations/layer14_attention_weight_max": 46.68606948852539,
      "activations/layer14_attention_weight_min": -35.77684783935547,
      "activations/layer15_attention_weight_max": 40.417816162109375,
      "activations/layer15_attention_weight_min": -31.914587020874023,
      "activations/layer16_attention_weight_max": 35.14915466308594,
      "activations/layer16_attention_weight_min": -26.65949821472168,
      "activations/layer17_attention_weight_max": 62.86934280395508,
      "activations/layer17_attention_weight_min": -36.64106750488281,
      "activations/layer18_attention_weight_max": 54.182796478271484,
      "activations/layer18_attention_weight_min": -33.14344024658203,
      "activations/layer19_attention_weight_max": 24.62264060974121,
      "activations/layer19_attention_weight_min": -16.6225643157959,
      "activations/layer1_attention_weight_max": 17.67363739013672,
      "activations/layer1_attention_weight_min": -13.769752502441406,
      "activations/layer20_attention_weight_max": 19.474882125854492,
      "activations/layer20_attention_weight_min": -20.544269561767578,
      "activations/layer21_attention_weight_max": 38.657569885253906,
      "activations/layer21_attention_weight_min": -29.007164001464844,
      "activations/layer22_attention_weight_max": 33.249271392822266,
      "activations/layer22_attention_weight_min": -24.4296817779541,
      "activations/layer23_attention_weight_max": 24.84805679321289,
      "activations/layer23_attention_weight_min": -17.137399673461914,
      "activations/layer2_attention_weight_max": 28.99871063232422,
      "activations/layer2_attention_weight_min": -27.849445343017578,
      "activations/layer3_attention_weight_max": 73.45025634765625,
      "activations/layer3_attention_weight_min": -75.92366027832031,
      "activations/layer4_attention_weight_max": 98.44766998291016,
      "activations/layer4_attention_weight_min": -97.55007934570312,
      "activations/layer5_attention_weight_max": 116.17974853515625,
      "activations/layer5_attention_weight_min": -105.7122802734375,
      "activations/layer6_attention_weight_max": 61.16340255737305,
      "activations/layer6_attention_weight_min": -54.467872619628906,
      "activations/layer7_attention_weight_max": 72.73368072509766,
      "activations/layer7_attention_weight_min": -59.183006286621094,
      "activations/layer8_attention_weight_max": 42.61085891723633,
      "activations/layer8_attention_weight_min": -37.967918395996094,
      "activations/layer9_attention_weight_max": 60.54499053955078,
      "activations/layer9_attention_weight_min": -47.311912536621094,
      "epoch": 3.41,
      "learning_rate": 0.0001292867424242424,
      "loss": 2.9015,
      "step": 58750
    },
    {
      "activations/layer0_attention_weight_max": 14.873394012451172,
      "activations/layer0_attention_weight_min": -14.87767219543457,
      "activations/layer10_attention_weight_max": 46.781734466552734,
      "activations/layer10_attention_weight_min": -36.0751838684082,
      "activations/layer11_attention_weight_max": 35.739715576171875,
      "activations/layer11_attention_weight_min": -27.281089782714844,
      "activations/layer12_attention_weight_max": 30.102279663085938,
      "activations/layer12_attention_weight_min": -26.802085876464844,
      "activations/layer13_attention_weight_max": 58.65202331542969,
      "activations/layer13_attention_weight_min": -42.351280212402344,
      "activations/layer14_attention_weight_max": 59.93836975097656,
      "activations/layer14_attention_weight_min": -42.93191909790039,
      "activations/layer15_attention_weight_max": 49.63821792602539,
      "activations/layer15_attention_weight_min": -38.432682037353516,
      "activations/layer16_attention_weight_max": 35.563446044921875,
      "activations/layer16_attention_weight_min": -27.674497604370117,
      "activations/layer17_attention_weight_max": 63.5569953918457,
      "activations/layer17_attention_weight_min": -38.4141731262207,
      "activations/layer18_attention_weight_max": 55.494937896728516,
      "activations/layer18_attention_weight_min": -30.468313217163086,
      "activations/layer19_attention_weight_max": 22.019317626953125,
      "activations/layer19_attention_weight_min": -18.206871032714844,
      "activations/layer1_attention_weight_max": 17.988006591796875,
      "activations/layer1_attention_weight_min": -16.182512283325195,
      "activations/layer20_attention_weight_max": 22.150835037231445,
      "activations/layer20_attention_weight_min": -18.268861770629883,
      "activations/layer21_attention_weight_max": 42.54414367675781,
      "activations/layer21_attention_weight_min": -28.148027420043945,
      "activations/layer22_attention_weight_max": 34.72706604003906,
      "activations/layer22_attention_weight_min": -20.62436294555664,
      "activations/layer23_attention_weight_max": 24.92767333984375,
      "activations/layer23_attention_weight_min": -14.973333358764648,
      "activations/layer2_attention_weight_max": 29.422523498535156,
      "activations/layer2_attention_weight_min": -28.804149627685547,
      "activations/layer3_attention_weight_max": 77.85282897949219,
      "activations/layer3_attention_weight_min": -79.5804672241211,
      "activations/layer4_attention_weight_max": 107.22293853759766,
      "activations/layer4_attention_weight_min": -95.9002456665039,
      "activations/layer5_attention_weight_max": 125.85891723632812,
      "activations/layer5_attention_weight_min": -104.93758392333984,
      "activations/layer6_attention_weight_max": 63.2000732421875,
      "activations/layer6_attention_weight_min": -54.485294342041016,
      "activations/layer7_attention_weight_max": 64.28277587890625,
      "activations/layer7_attention_weight_min": -64.80577850341797,
      "activations/layer8_attention_weight_max": 43.51792526245117,
      "activations/layer8_attention_weight_min": -37.44713592529297,
      "activations/layer9_attention_weight_max": 49.39707565307617,
      "activations/layer9_attention_weight_min": -47.985599517822266,
      "epoch": 3.42,
      "learning_rate": 0.00012926780303030303,
      "loss": 2.9045,
      "step": 58800
    },
    {
      "activations/layer0_attention_weight_max": 14.584429740905762,
      "activations/layer0_attention_weight_min": -14.718628883361816,
      "activations/layer10_attention_weight_max": 49.72049331665039,
      "activations/layer10_attention_weight_min": -34.45697784423828,
      "activations/layer11_attention_weight_max": 40.75443649291992,
      "activations/layer11_attention_weight_min": -27.765335083007812,
      "activations/layer12_attention_weight_max": 23.706418991088867,
      "activations/layer12_attention_weight_min": -23.77043914794922,
      "activations/layer13_attention_weight_max": 71.44951629638672,
      "activations/layer13_attention_weight_min": -38.42351150512695,
      "activations/layer14_attention_weight_max": 67.0366439819336,
      "activations/layer14_attention_weight_min": -37.89200973510742,
      "activations/layer15_attention_weight_max": 44.893131256103516,
      "activations/layer15_attention_weight_min": -33.78481674194336,
      "activations/layer16_attention_weight_max": 33.884037017822266,
      "activations/layer16_attention_weight_min": -25.512786865234375,
      "activations/layer17_attention_weight_max": 67.1928939819336,
      "activations/layer17_attention_weight_min": -37.61262512207031,
      "activations/layer18_attention_weight_max": 48.52558898925781,
      "activations/layer18_attention_weight_min": -31.91047477722168,
      "activations/layer19_attention_weight_max": 21.46173667907715,
      "activations/layer19_attention_weight_min": -18.10775375366211,
      "activations/layer1_attention_weight_max": 16.883708953857422,
      "activations/layer1_attention_weight_min": -15.639716148376465,
      "activations/layer20_attention_weight_max": 22.293073654174805,
      "activations/layer20_attention_weight_min": -17.843767166137695,
      "activations/layer21_attention_weight_max": 37.95207977294922,
      "activations/layer21_attention_weight_min": -28.18907356262207,
      "activations/layer22_attention_weight_max": 30.97018814086914,
      "activations/layer22_attention_weight_min": -21.61989402770996,
      "activations/layer23_attention_weight_max": 25.761289596557617,
      "activations/layer23_attention_weight_min": -16.153648376464844,
      "activations/layer2_attention_weight_max": 30.53464126586914,
      "activations/layer2_attention_weight_min": -29.08773422241211,
      "activations/layer3_attention_weight_max": 76.90525817871094,
      "activations/layer3_attention_weight_min": -79.85609436035156,
      "activations/layer4_attention_weight_max": 103.893798828125,
      "activations/layer4_attention_weight_min": -96.94713592529297,
      "activations/layer5_attention_weight_max": 133.03570556640625,
      "activations/layer5_attention_weight_min": -105.27532958984375,
      "activations/layer6_attention_weight_max": 70.11663818359375,
      "activations/layer6_attention_weight_min": -55.07681655883789,
      "activations/layer7_attention_weight_max": 68.50917053222656,
      "activations/layer7_attention_weight_min": -68.43930053710938,
      "activations/layer8_attention_weight_max": 49.56974792480469,
      "activations/layer8_attention_weight_min": -37.26197814941406,
      "activations/layer9_attention_weight_max": 58.4531135559082,
      "activations/layer9_attention_weight_min": -46.348487854003906,
      "epoch": 3.42,
      "learning_rate": 0.00012924886363636362,
      "loss": 2.8949,
      "step": 58850
    },
    {
      "activations/layer0_attention_weight_max": 14.614194869995117,
      "activations/layer0_attention_weight_min": -14.515451431274414,
      "activations/layer10_attention_weight_max": 53.24064636230469,
      "activations/layer10_attention_weight_min": -33.9640998840332,
      "activations/layer11_attention_weight_max": 47.167747497558594,
      "activations/layer11_attention_weight_min": -29.02252769470215,
      "activations/layer12_attention_weight_max": 28.617963790893555,
      "activations/layer12_attention_weight_min": -27.186887741088867,
      "activations/layer13_attention_weight_max": 90.38829803466797,
      "activations/layer13_attention_weight_min": -46.059017181396484,
      "activations/layer14_attention_weight_max": 91.5865707397461,
      "activations/layer14_attention_weight_min": -41.90854263305664,
      "activations/layer15_attention_weight_max": 55.83122253417969,
      "activations/layer15_attention_weight_min": -32.059967041015625,
      "activations/layer16_attention_weight_max": 37.33088684082031,
      "activations/layer16_attention_weight_min": -25.83701515197754,
      "activations/layer17_attention_weight_max": 67.91529846191406,
      "activations/layer17_attention_weight_min": -36.47798538208008,
      "activations/layer18_attention_weight_max": 58.19184875488281,
      "activations/layer18_attention_weight_min": -31.85258674621582,
      "activations/layer19_attention_weight_max": 23.552265167236328,
      "activations/layer19_attention_weight_min": -17.25636100769043,
      "activations/layer1_attention_weight_max": 17.6743221282959,
      "activations/layer1_attention_weight_min": -14.187309265136719,
      "activations/layer20_attention_weight_max": 23.50499153137207,
      "activations/layer20_attention_weight_min": -17.997854232788086,
      "activations/layer21_attention_weight_max": 39.49727249145508,
      "activations/layer21_attention_weight_min": -27.236812591552734,
      "activations/layer22_attention_weight_max": 34.125511169433594,
      "activations/layer22_attention_weight_min": -21.33589744567871,
      "activations/layer23_attention_weight_max": 28.86703872680664,
      "activations/layer23_attention_weight_min": -15.582447052001953,
      "activations/layer2_attention_weight_max": 31.595718383789062,
      "activations/layer2_attention_weight_min": -29.156471252441406,
      "activations/layer3_attention_weight_max": 77.51081085205078,
      "activations/layer3_attention_weight_min": -79.23210144042969,
      "activations/layer4_attention_weight_max": 111.17264556884766,
      "activations/layer4_attention_weight_min": -100.38365936279297,
      "activations/layer5_attention_weight_max": 133.29458618164062,
      "activations/layer5_attention_weight_min": -110.24556732177734,
      "activations/layer6_attention_weight_max": 65.13233947753906,
      "activations/layer6_attention_weight_min": -55.91532897949219,
      "activations/layer7_attention_weight_max": 80.427490234375,
      "activations/layer7_attention_weight_min": -64.76914978027344,
      "activations/layer8_attention_weight_max": 53.60744094848633,
      "activations/layer8_attention_weight_min": -38.008792877197266,
      "activations/layer9_attention_weight_max": 73.13493347167969,
      "activations/layer9_attention_weight_min": -53.42094039916992,
      "epoch": 3.42,
      "learning_rate": 0.00012922992424242423,
      "loss": 2.8844,
      "step": 58900
    },
    {
      "activations/layer0_attention_weight_max": 14.219937324523926,
      "activations/layer0_attention_weight_min": -14.890892028808594,
      "activations/layer10_attention_weight_max": 51.238155364990234,
      "activations/layer10_attention_weight_min": -35.32331085205078,
      "activations/layer11_attention_weight_max": 44.668983459472656,
      "activations/layer11_attention_weight_min": -30.51349449157715,
      "activations/layer12_attention_weight_max": 26.80208969116211,
      "activations/layer12_attention_weight_min": -27.658954620361328,
      "activations/layer13_attention_weight_max": 61.0380859375,
      "activations/layer13_attention_weight_min": -43.53144454956055,
      "activations/layer14_attention_weight_max": 82.06898498535156,
      "activations/layer14_attention_weight_min": -41.96207809448242,
      "activations/layer15_attention_weight_max": 62.474365234375,
      "activations/layer15_attention_weight_min": -35.8251953125,
      "activations/layer16_attention_weight_max": 39.79744338989258,
      "activations/layer16_attention_weight_min": -26.88104820251465,
      "activations/layer17_attention_weight_max": 65.74007415771484,
      "activations/layer17_attention_weight_min": -37.529727935791016,
      "activations/layer18_attention_weight_max": 58.81265640258789,
      "activations/layer18_attention_weight_min": -32.90351486206055,
      "activations/layer19_attention_weight_max": 28.863605499267578,
      "activations/layer19_attention_weight_min": -19.098440170288086,
      "activations/layer1_attention_weight_max": 17.581220626831055,
      "activations/layer1_attention_weight_min": -16.666181564331055,
      "activations/layer20_attention_weight_max": 27.577640533447266,
      "activations/layer20_attention_weight_min": -24.037006378173828,
      "activations/layer21_attention_weight_max": 48.55546569824219,
      "activations/layer21_attention_weight_min": -26.2181453704834,
      "activations/layer22_attention_weight_max": 35.25177764892578,
      "activations/layer22_attention_weight_min": -22.541353225708008,
      "activations/layer23_attention_weight_max": 25.090803146362305,
      "activations/layer23_attention_weight_min": -16.25794219970703,
      "activations/layer2_attention_weight_max": 29.920940399169922,
      "activations/layer2_attention_weight_min": -27.78509521484375,
      "activations/layer3_attention_weight_max": 77.78690338134766,
      "activations/layer3_attention_weight_min": -77.4646987915039,
      "activations/layer4_attention_weight_max": 102.76903533935547,
      "activations/layer4_attention_weight_min": -92.81336975097656,
      "activations/layer5_attention_weight_max": 126.43798828125,
      "activations/layer5_attention_weight_min": -105.12034606933594,
      "activations/layer6_attention_weight_max": 61.31365966796875,
      "activations/layer6_attention_weight_min": -52.83790588378906,
      "activations/layer7_attention_weight_max": 65.80432891845703,
      "activations/layer7_attention_weight_min": -59.134769439697266,
      "activations/layer8_attention_weight_max": 46.56427764892578,
      "activations/layer8_attention_weight_min": -37.831825256347656,
      "activations/layer9_attention_weight_max": 61.500850677490234,
      "activations/layer9_attention_weight_min": -53.355079650878906,
      "epoch": 3.43,
      "learning_rate": 0.00012921098484848485,
      "loss": 2.8946,
      "step": 58950
    },
    {
      "activations/layer0_attention_weight_max": 13.806239128112793,
      "activations/layer0_attention_weight_min": -15.227296829223633,
      "activations/layer10_attention_weight_max": 42.204959869384766,
      "activations/layer10_attention_weight_min": -34.65876007080078,
      "activations/layer11_attention_weight_max": 36.10467529296875,
      "activations/layer11_attention_weight_min": -28.253826141357422,
      "activations/layer12_attention_weight_max": 25.718408584594727,
      "activations/layer12_attention_weight_min": -24.024658203125,
      "activations/layer13_attention_weight_max": 66.08311462402344,
      "activations/layer13_attention_weight_min": -48.92082977294922,
      "activations/layer14_attention_weight_max": 71.3993148803711,
      "activations/layer14_attention_weight_min": -48.54074478149414,
      "activations/layer15_attention_weight_max": 51.312862396240234,
      "activations/layer15_attention_weight_min": -32.973854064941406,
      "activations/layer16_attention_weight_max": 47.404178619384766,
      "activations/layer16_attention_weight_min": -27.777185440063477,
      "activations/layer17_attention_weight_max": 60.484375,
      "activations/layer17_attention_weight_min": -37.872676849365234,
      "activations/layer18_attention_weight_max": 50.010650634765625,
      "activations/layer18_attention_weight_min": -31.292448043823242,
      "activations/layer19_attention_weight_max": 24.2039852142334,
      "activations/layer19_attention_weight_min": -18.425973892211914,
      "activations/layer1_attention_weight_max": 17.03529930114746,
      "activations/layer1_attention_weight_min": -14.709211349487305,
      "activations/layer20_attention_weight_max": 23.84056282043457,
      "activations/layer20_attention_weight_min": -20.901634216308594,
      "activations/layer21_attention_weight_max": 56.371761322021484,
      "activations/layer21_attention_weight_min": -29.18953514099121,
      "activations/layer22_attention_weight_max": 38.47077941894531,
      "activations/layer22_attention_weight_min": -26.6595516204834,
      "activations/layer23_attention_weight_max": 26.037841796875,
      "activations/layer23_attention_weight_min": -19.600143432617188,
      "activations/layer2_attention_weight_max": 32.03615188598633,
      "activations/layer2_attention_weight_min": -29.489234924316406,
      "activations/layer3_attention_weight_max": 77.12397003173828,
      "activations/layer3_attention_weight_min": -75.44318389892578,
      "activations/layer4_attention_weight_max": 105.71605682373047,
      "activations/layer4_attention_weight_min": -94.12659454345703,
      "activations/layer5_attention_weight_max": 121.27880096435547,
      "activations/layer5_attention_weight_min": -98.53271484375,
      "activations/layer6_attention_weight_max": 62.02305221557617,
      "activations/layer6_attention_weight_min": -52.02141571044922,
      "activations/layer7_attention_weight_max": 63.74708938598633,
      "activations/layer7_attention_weight_min": -63.0806999206543,
      "activations/layer8_attention_weight_max": 38.58751678466797,
      "activations/layer8_attention_weight_min": -36.35381317138672,
      "activations/layer9_attention_weight_max": 51.041141510009766,
      "activations/layer9_attention_weight_min": -47.43501281738281,
      "epoch": 3.43,
      "learning_rate": 0.00012919204545454543,
      "loss": 2.8967,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_loss": 2.841796875,
      "eval_runtime": 8.4793,
      "eval_samples_per_second": 506.412,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_openwebtext_loss": 2.841796875,
      "eval_openwebtext_ppl": 17.14654807625046,
      "eval_openwebtext_runtime": 8.4793,
      "eval_openwebtext_samples_per_second": 506.412,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_wikitext_loss": 3.0859375,
      "eval_wikitext_ppl": 21.887977204970085,
      "eval_wikitext_runtime": 1.9276,
      "eval_wikitext_samples_per_second": 236.561,
      "step": 59000
    },
    {
      "epoch": 3.43,
      "eval_lambada_loss": 3.0078125,
      "eval_lambada_ppl": 20.24306974171993,
      "eval_lambada_runtime": 9.535,
      "eval_lambada_samples_per_second": 510.645,
      "step": 59000
    },
    {
      "activations/layer0_attention_weight_max": 14.46236801147461,
      "activations/layer0_attention_weight_min": -15.017122268676758,
      "activations/layer10_attention_weight_max": 46.073394775390625,
      "activations/layer10_attention_weight_min": -33.56660079956055,
      "activations/layer11_attention_weight_max": 37.55324935913086,
      "activations/layer11_attention_weight_min": -26.506559371948242,
      "activations/layer12_attention_weight_max": 26.883516311645508,
      "activations/layer12_attention_weight_min": -26.99266815185547,
      "activations/layer13_attention_weight_max": 90.24931335449219,
      "activations/layer13_attention_weight_min": -52.09904861450195,
      "activations/layer14_attention_weight_max": 82.69190979003906,
      "activations/layer14_attention_weight_min": -41.5718879699707,
      "activations/layer15_attention_weight_max": 54.75339889526367,
      "activations/layer15_attention_weight_min": -30.784500122070312,
      "activations/layer16_attention_weight_max": 39.630619049072266,
      "activations/layer16_attention_weight_min": -25.189680099487305,
      "activations/layer17_attention_weight_max": 64.84549713134766,
      "activations/layer17_attention_weight_min": -35.40983581542969,
      "activations/layer18_attention_weight_max": 62.236019134521484,
      "activations/layer18_attention_weight_min": -33.23990249633789,
      "activations/layer19_attention_weight_max": 21.93955421447754,
      "activations/layer19_attention_weight_min": -18.851318359375,
      "activations/layer1_attention_weight_max": 16.639522552490234,
      "activations/layer1_attention_weight_min": -13.75091552734375,
      "activations/layer20_attention_weight_max": 26.60057258605957,
      "activations/layer20_attention_weight_min": -20.745723724365234,
      "activations/layer21_attention_weight_max": 42.288047790527344,
      "activations/layer21_attention_weight_min": -29.973857879638672,
      "activations/layer22_attention_weight_max": 37.59608840942383,
      "activations/layer22_attention_weight_min": -24.189390182495117,
      "activations/layer23_attention_weight_max": 29.18790054321289,
      "activations/layer23_attention_weight_min": -17.925045013427734,
      "activations/layer2_attention_weight_max": 30.505695343017578,
      "activations/layer2_attention_weight_min": -31.45863914489746,
      "activations/layer3_attention_weight_max": 82.76229095458984,
      "activations/layer3_attention_weight_min": -82.73687744140625,
      "activations/layer4_attention_weight_max": 110.7137222290039,
      "activations/layer4_attention_weight_min": -100.45633697509766,
      "activations/layer5_attention_weight_max": 131.44082641601562,
      "activations/layer5_attention_weight_min": -108.89696502685547,
      "activations/layer6_attention_weight_max": 68.7056884765625,
      "activations/layer6_attention_weight_min": -55.51222229003906,
      "activations/layer7_attention_weight_max": 65.81615447998047,
      "activations/layer7_attention_weight_min": -61.473384857177734,
      "activations/layer8_attention_weight_max": 42.32960510253906,
      "activations/layer8_attention_weight_min": -39.204105377197266,
      "activations/layer9_attention_weight_max": 58.8998908996582,
      "activations/layer9_attention_weight_min": -48.987876892089844,
      "epoch": 3.43,
      "learning_rate": 0.00012917310606060605,
      "loss": 2.9015,
      "step": 59050
    },
    {
      "activations/layer0_attention_weight_max": 15.227011680603027,
      "activations/layer0_attention_weight_min": -14.502388954162598,
      "activations/layer10_attention_weight_max": 41.619808197021484,
      "activations/layer10_attention_weight_min": -33.850677490234375,
      "activations/layer11_attention_weight_max": 31.274169921875,
      "activations/layer11_attention_weight_min": -25.83521842956543,
      "activations/layer12_attention_weight_max": 24.27796173095703,
      "activations/layer12_attention_weight_min": -22.860065460205078,
      "activations/layer13_attention_weight_max": 55.43975830078125,
      "activations/layer13_attention_weight_min": -35.69308090209961,
      "activations/layer14_attention_weight_max": 41.43633270263672,
      "activations/layer14_attention_weight_min": -28.310546875,
      "activations/layer15_attention_weight_max": 46.93048095703125,
      "activations/layer15_attention_weight_min": -33.277259826660156,
      "activations/layer16_attention_weight_max": 37.310001373291016,
      "activations/layer16_attention_weight_min": -25.890342712402344,
      "activations/layer17_attention_weight_max": 61.61910629272461,
      "activations/layer17_attention_weight_min": -34.557777404785156,
      "activations/layer18_attention_weight_max": 55.5367546081543,
      "activations/layer18_attention_weight_min": -32.132781982421875,
      "activations/layer19_attention_weight_max": 21.86385726928711,
      "activations/layer19_attention_weight_min": -17.873138427734375,
      "activations/layer1_attention_weight_max": 17.525554656982422,
      "activations/layer1_attention_weight_min": -14.815698623657227,
      "activations/layer20_attention_weight_max": 22.425474166870117,
      "activations/layer20_attention_weight_min": -18.251619338989258,
      "activations/layer21_attention_weight_max": 42.18136978149414,
      "activations/layer21_attention_weight_min": -26.197729110717773,
      "activations/layer22_attention_weight_max": 37.899898529052734,
      "activations/layer22_attention_weight_min": -24.21333122253418,
      "activations/layer23_attention_weight_max": 28.642696380615234,
      "activations/layer23_attention_weight_min": -16.016281127929688,
      "activations/layer2_attention_weight_max": 31.312475204467773,
      "activations/layer2_attention_weight_min": -29.307926177978516,
      "activations/layer3_attention_weight_max": 80.02521514892578,
      "activations/layer3_attention_weight_min": -86.46222686767578,
      "activations/layer4_attention_weight_max": 104.57318115234375,
      "activations/layer4_attention_weight_min": -99.67668151855469,
      "activations/layer5_attention_weight_max": 119.09883117675781,
      "activations/layer5_attention_weight_min": -105.28126525878906,
      "activations/layer6_attention_weight_max": 60.08344650268555,
      "activations/layer6_attention_weight_min": -53.94306945800781,
      "activations/layer7_attention_weight_max": 59.89350128173828,
      "activations/layer7_attention_weight_min": -56.790992736816406,
      "activations/layer8_attention_weight_max": 41.29175567626953,
      "activations/layer8_attention_weight_min": -35.59370803833008,
      "activations/layer9_attention_weight_max": 48.60383987426758,
      "activations/layer9_attention_weight_min": -42.75966262817383,
      "epoch": 3.43,
      "learning_rate": 0.00012915416666666664,
      "loss": 2.909,
      "step": 59100
    },
    {
      "activations/layer0_attention_weight_max": 13.487147331237793,
      "activations/layer0_attention_weight_min": -14.735289573669434,
      "activations/layer10_attention_weight_max": 40.45894241333008,
      "activations/layer10_attention_weight_min": -34.6061897277832,
      "activations/layer11_attention_weight_max": 31.985509872436523,
      "activations/layer11_attention_weight_min": -27.938045501708984,
      "activations/layer12_attention_weight_max": 27.389867782592773,
      "activations/layer12_attention_weight_min": -25.333106994628906,
      "activations/layer13_attention_weight_max": 57.25069046020508,
      "activations/layer13_attention_weight_min": -42.54226303100586,
      "activations/layer14_attention_weight_max": 49.263973236083984,
      "activations/layer14_attention_weight_min": -33.877845764160156,
      "activations/layer15_attention_weight_max": 44.344078063964844,
      "activations/layer15_attention_weight_min": -31.8565731048584,
      "activations/layer16_attention_weight_max": 38.00390625,
      "activations/layer16_attention_weight_min": -26.22270965576172,
      "activations/layer17_attention_weight_max": 61.57074737548828,
      "activations/layer17_attention_weight_min": -38.75898361206055,
      "activations/layer18_attention_weight_max": 49.339439392089844,
      "activations/layer18_attention_weight_min": -32.47400665283203,
      "activations/layer19_attention_weight_max": 25.340099334716797,
      "activations/layer19_attention_weight_min": -18.066879272460938,
      "activations/layer1_attention_weight_max": 18.18789291381836,
      "activations/layer1_attention_weight_min": -12.170852661132812,
      "activations/layer20_attention_weight_max": 28.578378677368164,
      "activations/layer20_attention_weight_min": -18.061119079589844,
      "activations/layer21_attention_weight_max": 37.10992431640625,
      "activations/layer21_attention_weight_min": -27.655075073242188,
      "activations/layer22_attention_weight_max": 36.64536666870117,
      "activations/layer22_attention_weight_min": -22.14483070373535,
      "activations/layer23_attention_weight_max": 26.604583740234375,
      "activations/layer23_attention_weight_min": -16.564481735229492,
      "activations/layer2_attention_weight_max": 29.312511444091797,
      "activations/layer2_attention_weight_min": -26.773422241210938,
      "activations/layer3_attention_weight_max": 71.36785125732422,
      "activations/layer3_attention_weight_min": -74.97271728515625,
      "activations/layer4_attention_weight_max": 96.86907958984375,
      "activations/layer4_attention_weight_min": -91.45066833496094,
      "activations/layer5_attention_weight_max": 115.70586395263672,
      "activations/layer5_attention_weight_min": -102.17144012451172,
      "activations/layer6_attention_weight_max": 58.35533142089844,
      "activations/layer6_attention_weight_min": -52.60411834716797,
      "activations/layer7_attention_weight_max": 62.9997673034668,
      "activations/layer7_attention_weight_min": -58.869773864746094,
      "activations/layer8_attention_weight_max": 38.57646179199219,
      "activations/layer8_attention_weight_min": -36.614891052246094,
      "activations/layer9_attention_weight_max": 49.0629768371582,
      "activations/layer9_attention_weight_min": -45.66789627075195,
      "epoch": 3.44,
      "learning_rate": 0.00012913522727272725,
      "loss": 2.9097,
      "step": 59150
    },
    {
      "activations/layer0_attention_weight_max": 14.760156631469727,
      "activations/layer0_attention_weight_min": -14.448410987854004,
      "activations/layer10_attention_weight_max": 38.064231872558594,
      "activations/layer10_attention_weight_min": -32.74005889892578,
      "activations/layer11_attention_weight_max": 33.03923797607422,
      "activations/layer11_attention_weight_min": -25.670475006103516,
      "activations/layer12_attention_weight_max": 25.18712615966797,
      "activations/layer12_attention_weight_min": -23.868879318237305,
      "activations/layer13_attention_weight_max": 59.10456848144531,
      "activations/layer13_attention_weight_min": -39.31046676635742,
      "activations/layer14_attention_weight_max": 48.76913833618164,
      "activations/layer14_attention_weight_min": -33.862369537353516,
      "activations/layer15_attention_weight_max": 43.956661224365234,
      "activations/layer15_attention_weight_min": -31.692581176757812,
      "activations/layer16_attention_weight_max": 33.98371505737305,
      "activations/layer16_attention_weight_min": -25.883079528808594,
      "activations/layer17_attention_weight_max": 59.47228240966797,
      "activations/layer17_attention_weight_min": -35.37580871582031,
      "activations/layer18_attention_weight_max": 49.93436813354492,
      "activations/layer18_attention_weight_min": -31.42735481262207,
      "activations/layer19_attention_weight_max": 19.165021896362305,
      "activations/layer19_attention_weight_min": -17.270545959472656,
      "activations/layer1_attention_weight_max": 18.86065101623535,
      "activations/layer1_attention_weight_min": -12.810587882995605,
      "activations/layer20_attention_weight_max": 20.49345588684082,
      "activations/layer20_attention_weight_min": -20.54014015197754,
      "activations/layer21_attention_weight_max": 41.217369079589844,
      "activations/layer21_attention_weight_min": -28.484569549560547,
      "activations/layer22_attention_weight_max": 31.519330978393555,
      "activations/layer22_attention_weight_min": -22.93917465209961,
      "activations/layer23_attention_weight_max": 25.765851974487305,
      "activations/layer23_attention_weight_min": -16.426692962646484,
      "activations/layer2_attention_weight_max": 28.61248016357422,
      "activations/layer2_attention_weight_min": -27.08614158630371,
      "activations/layer3_attention_weight_max": 69.73055267333984,
      "activations/layer3_attention_weight_min": -72.05982971191406,
      "activations/layer4_attention_weight_max": 100.58712005615234,
      "activations/layer4_attention_weight_min": -90.69467163085938,
      "activations/layer5_attention_weight_max": 124.08785247802734,
      "activations/layer5_attention_weight_min": -98.24028778076172,
      "activations/layer6_attention_weight_max": 63.74060821533203,
      "activations/layer6_attention_weight_min": -51.79861068725586,
      "activations/layer7_attention_weight_max": 60.80690383911133,
      "activations/layer7_attention_weight_min": -60.53295135498047,
      "activations/layer8_attention_weight_max": 40.70279312133789,
      "activations/layer8_attention_weight_min": -36.21832275390625,
      "activations/layer9_attention_weight_max": 47.42999267578125,
      "activations/layer9_attention_weight_min": -45.35148239135742,
      "epoch": 3.44,
      "learning_rate": 0.00012911628787878787,
      "loss": 2.8766,
      "step": 59200
    },
    {
      "activations/layer0_attention_weight_max": 15.654261589050293,
      "activations/layer0_attention_weight_min": -14.50721263885498,
      "activations/layer10_attention_weight_max": 46.45207214355469,
      "activations/layer10_attention_weight_min": -32.856868743896484,
      "activations/layer11_attention_weight_max": 37.90843963623047,
      "activations/layer11_attention_weight_min": -25.795307159423828,
      "activations/layer12_attention_weight_max": 30.587284088134766,
      "activations/layer12_attention_weight_min": -31.739290237426758,
      "activations/layer13_attention_weight_max": 56.66578674316406,
      "activations/layer13_attention_weight_min": -35.672977447509766,
      "activations/layer14_attention_weight_max": 52.424537658691406,
      "activations/layer14_attention_weight_min": -33.20966339111328,
      "activations/layer15_attention_weight_max": 65.82051849365234,
      "activations/layer15_attention_weight_min": -37.7722282409668,
      "activations/layer16_attention_weight_max": 35.42957305908203,
      "activations/layer16_attention_weight_min": -26.171375274658203,
      "activations/layer17_attention_weight_max": 58.60890197753906,
      "activations/layer17_attention_weight_min": -36.49679183959961,
      "activations/layer18_attention_weight_max": 54.30903244018555,
      "activations/layer18_attention_weight_min": -34.48732376098633,
      "activations/layer19_attention_weight_max": 21.36333656311035,
      "activations/layer19_attention_weight_min": -21.558090209960938,
      "activations/layer1_attention_weight_max": 17.892797470092773,
      "activations/layer1_attention_weight_min": -13.633484840393066,
      "activations/layer20_attention_weight_max": 22.594438552856445,
      "activations/layer20_attention_weight_min": -19.347654342651367,
      "activations/layer21_attention_weight_max": 48.8712158203125,
      "activations/layer21_attention_weight_min": -29.153362274169922,
      "activations/layer22_attention_weight_max": 35.620723724365234,
      "activations/layer22_attention_weight_min": -23.21993064880371,
      "activations/layer23_attention_weight_max": 27.656875610351562,
      "activations/layer23_attention_weight_min": -16.283878326416016,
      "activations/layer2_attention_weight_max": 28.244789123535156,
      "activations/layer2_attention_weight_min": -27.163169860839844,
      "activations/layer3_attention_weight_max": 71.23644256591797,
      "activations/layer3_attention_weight_min": -75.13211059570312,
      "activations/layer4_attention_weight_max": 96.03955841064453,
      "activations/layer4_attention_weight_min": -90.46772003173828,
      "activations/layer5_attention_weight_max": 114.59803771972656,
      "activations/layer5_attention_weight_min": -99.15972900390625,
      "activations/layer6_attention_weight_max": 59.58588790893555,
      "activations/layer6_attention_weight_min": -48.74075698852539,
      "activations/layer7_attention_weight_max": 68.80098724365234,
      "activations/layer7_attention_weight_min": -63.990562438964844,
      "activations/layer8_attention_weight_max": 42.30656433105469,
      "activations/layer8_attention_weight_min": -38.203468322753906,
      "activations/layer9_attention_weight_max": 64.05750274658203,
      "activations/layer9_attention_weight_min": -49.46600341796875,
      "epoch": 3.44,
      "learning_rate": 0.00012909734848484846,
      "loss": 2.8781,
      "step": 59250
    },
    {
      "activations/layer0_attention_weight_max": 15.160789489746094,
      "activations/layer0_attention_weight_min": -14.518545150756836,
      "activations/layer10_attention_weight_max": 45.14794921875,
      "activations/layer10_attention_weight_min": -36.40326690673828,
      "activations/layer11_attention_weight_max": 36.906124114990234,
      "activations/layer11_attention_weight_min": -27.42599868774414,
      "activations/layer12_attention_weight_max": 35.072086334228516,
      "activations/layer12_attention_weight_min": -25.624670028686523,
      "activations/layer13_attention_weight_max": 53.170814514160156,
      "activations/layer13_attention_weight_min": -37.51310348510742,
      "activations/layer14_attention_weight_max": 52.59745788574219,
      "activations/layer14_attention_weight_min": -36.87165069580078,
      "activations/layer15_attention_weight_max": 48.241031646728516,
      "activations/layer15_attention_weight_min": -33.49049377441406,
      "activations/layer16_attention_weight_max": 40.11295700073242,
      "activations/layer16_attention_weight_min": -28.301759719848633,
      "activations/layer17_attention_weight_max": 62.76930236816406,
      "activations/layer17_attention_weight_min": -35.623634338378906,
      "activations/layer18_attention_weight_max": 62.32053756713867,
      "activations/layer18_attention_weight_min": -32.603416442871094,
      "activations/layer19_attention_weight_max": 25.84522819519043,
      "activations/layer19_attention_weight_min": -18.71120262145996,
      "activations/layer1_attention_weight_max": 17.141843795776367,
      "activations/layer1_attention_weight_min": -14.03631591796875,
      "activations/layer20_attention_weight_max": 21.475902557373047,
      "activations/layer20_attention_weight_min": -19.566438674926758,
      "activations/layer21_attention_weight_max": 47.865474700927734,
      "activations/layer21_attention_weight_min": -27.683353424072266,
      "activations/layer22_attention_weight_max": 34.817813873291016,
      "activations/layer22_attention_weight_min": -21.716293334960938,
      "activations/layer23_attention_weight_max": 28.68344497680664,
      "activations/layer23_attention_weight_min": -19.524635314941406,
      "activations/layer2_attention_weight_max": 30.199512481689453,
      "activations/layer2_attention_weight_min": -28.690710067749023,
      "activations/layer3_attention_weight_max": 74.91224670410156,
      "activations/layer3_attention_weight_min": -74.4004135131836,
      "activations/layer4_attention_weight_max": 101.34066772460938,
      "activations/layer4_attention_weight_min": -92.05728912353516,
      "activations/layer5_attention_weight_max": 125.34646606445312,
      "activations/layer5_attention_weight_min": -104.1719970703125,
      "activations/layer6_attention_weight_max": 63.811546325683594,
      "activations/layer6_attention_weight_min": -52.45018768310547,
      "activations/layer7_attention_weight_max": 62.295928955078125,
      "activations/layer7_attention_weight_min": -59.32522964477539,
      "activations/layer8_attention_weight_max": 44.86336135864258,
      "activations/layer8_attention_weight_min": -38.718360900878906,
      "activations/layer9_attention_weight_max": 58.524749755859375,
      "activations/layer9_attention_weight_min": -50.85990524291992,
      "epoch": 3.45,
      "learning_rate": 0.00012907840909090907,
      "loss": 2.8858,
      "step": 59300
    },
    {
      "activations/layer0_attention_weight_max": 13.412759780883789,
      "activations/layer0_attention_weight_min": -14.13193416595459,
      "activations/layer10_attention_weight_max": 40.22254180908203,
      "activations/layer10_attention_weight_min": -32.65673828125,
      "activations/layer11_attention_weight_max": 29.685453414916992,
      "activations/layer11_attention_weight_min": -25.270843505859375,
      "activations/layer12_attention_weight_max": 25.797910690307617,
      "activations/layer12_attention_weight_min": -24.957109451293945,
      "activations/layer13_attention_weight_max": 43.82897186279297,
      "activations/layer13_attention_weight_min": -37.242523193359375,
      "activations/layer14_attention_weight_max": 43.4866943359375,
      "activations/layer14_attention_weight_min": -34.797698974609375,
      "activations/layer15_attention_weight_max": 42.642616271972656,
      "activations/layer15_attention_weight_min": -32.19059753417969,
      "activations/layer16_attention_weight_max": 39.89841079711914,
      "activations/layer16_attention_weight_min": -27.076343536376953,
      "activations/layer17_attention_weight_max": 54.919921875,
      "activations/layer17_attention_weight_min": -34.908668518066406,
      "activations/layer18_attention_weight_max": 49.85976028442383,
      "activations/layer18_attention_weight_min": -31.651325225830078,
      "activations/layer19_attention_weight_max": 22.92589569091797,
      "activations/layer19_attention_weight_min": -19.152864456176758,
      "activations/layer1_attention_weight_max": 17.619873046875,
      "activations/layer1_attention_weight_min": -12.433371543884277,
      "activations/layer20_attention_weight_max": 20.815996170043945,
      "activations/layer20_attention_weight_min": -19.752262115478516,
      "activations/layer21_attention_weight_max": 40.8168830871582,
      "activations/layer21_attention_weight_min": -25.163986206054688,
      "activations/layer22_attention_weight_max": 37.0073127746582,
      "activations/layer22_attention_weight_min": -21.561439514160156,
      "activations/layer23_attention_weight_max": 27.15218162536621,
      "activations/layer23_attention_weight_min": -16.402912139892578,
      "activations/layer2_attention_weight_max": 30.44007682800293,
      "activations/layer2_attention_weight_min": -27.486217498779297,
      "activations/layer3_attention_weight_max": 71.53717803955078,
      "activations/layer3_attention_weight_min": -73.60076904296875,
      "activations/layer4_attention_weight_max": 99.41123962402344,
      "activations/layer4_attention_weight_min": -94.95553588867188,
      "activations/layer5_attention_weight_max": 113.88768005371094,
      "activations/layer5_attention_weight_min": -99.67535400390625,
      "activations/layer6_attention_weight_max": 59.35514831542969,
      "activations/layer6_attention_weight_min": -52.357749938964844,
      "activations/layer7_attention_weight_max": 60.458168029785156,
      "activations/layer7_attention_weight_min": -54.924217224121094,
      "activations/layer8_attention_weight_max": 40.7103157043457,
      "activations/layer8_attention_weight_min": -36.30039978027344,
      "activations/layer9_attention_weight_max": 52.30202102661133,
      "activations/layer9_attention_weight_min": -41.99839782714844,
      "epoch": 3.45,
      "learning_rate": 0.00012905946969696969,
      "loss": 2.8796,
      "step": 59350
    },
    {
      "activations/layer0_attention_weight_max": 13.715083122253418,
      "activations/layer0_attention_weight_min": -14.454935073852539,
      "activations/layer10_attention_weight_max": 45.43515396118164,
      "activations/layer10_attention_weight_min": -34.989036560058594,
      "activations/layer11_attention_weight_max": 33.91346740722656,
      "activations/layer11_attention_weight_min": -28.466705322265625,
      "activations/layer12_attention_weight_max": 27.68616485595703,
      "activations/layer12_attention_weight_min": -27.85446548461914,
      "activations/layer13_attention_weight_max": 51.64592361450195,
      "activations/layer13_attention_weight_min": -35.07359313964844,
      "activations/layer14_attention_weight_max": 50.34754943847656,
      "activations/layer14_attention_weight_min": -30.971269607543945,
      "activations/layer15_attention_weight_max": 42.29320526123047,
      "activations/layer15_attention_weight_min": -32.089759826660156,
      "activations/layer16_attention_weight_max": 43.86827850341797,
      "activations/layer16_attention_weight_min": -28.76118278503418,
      "activations/layer17_attention_weight_max": 60.08702850341797,
      "activations/layer17_attention_weight_min": -35.421138763427734,
      "activations/layer18_attention_weight_max": 52.42509460449219,
      "activations/layer18_attention_weight_min": -30.838741302490234,
      "activations/layer19_attention_weight_max": 22.444467544555664,
      "activations/layer19_attention_weight_min": -17.91832160949707,
      "activations/layer1_attention_weight_max": 18.049142837524414,
      "activations/layer1_attention_weight_min": -13.846096992492676,
      "activations/layer20_attention_weight_max": 21.537412643432617,
      "activations/layer20_attention_weight_min": -17.81768798828125,
      "activations/layer21_attention_weight_max": 46.02593994140625,
      "activations/layer21_attention_weight_min": -29.594993591308594,
      "activations/layer22_attention_weight_max": 36.021114349365234,
      "activations/layer22_attention_weight_min": -25.818811416625977,
      "activations/layer23_attention_weight_max": 25.025428771972656,
      "activations/layer23_attention_weight_min": -18.304821014404297,
      "activations/layer2_attention_weight_max": 29.208280563354492,
      "activations/layer2_attention_weight_min": -27.520366668701172,
      "activations/layer3_attention_weight_max": 73.81891632080078,
      "activations/layer3_attention_weight_min": -73.89804077148438,
      "activations/layer4_attention_weight_max": 99.1524429321289,
      "activations/layer4_attention_weight_min": -91.4836654663086,
      "activations/layer5_attention_weight_max": 121.32476043701172,
      "activations/layer5_attention_weight_min": -99.06767272949219,
      "activations/layer6_attention_weight_max": 61.9262809753418,
      "activations/layer6_attention_weight_min": -51.31618118286133,
      "activations/layer7_attention_weight_max": 63.25471496582031,
      "activations/layer7_attention_weight_min": -56.21827697753906,
      "activations/layer8_attention_weight_max": 40.08732986450195,
      "activations/layer8_attention_weight_min": -36.18956756591797,
      "activations/layer9_attention_weight_max": 50.54082107543945,
      "activations/layer9_attention_weight_min": -46.58530044555664,
      "epoch": 3.45,
      "learning_rate": 0.0001290405303030303,
      "loss": 2.8979,
      "step": 59400
    },
    {
      "activations/layer0_attention_weight_max": 14.042698860168457,
      "activations/layer0_attention_weight_min": -14.510446548461914,
      "activations/layer10_attention_weight_max": 48.42038345336914,
      "activations/layer10_attention_weight_min": -36.934173583984375,
      "activations/layer11_attention_weight_max": 40.404170989990234,
      "activations/layer11_attention_weight_min": -32.26930236816406,
      "activations/layer12_attention_weight_max": 30.60629653930664,
      "activations/layer12_attention_weight_min": -26.40932846069336,
      "activations/layer13_attention_weight_max": 75.89828491210938,
      "activations/layer13_attention_weight_min": -45.609519958496094,
      "activations/layer14_attention_weight_max": 65.45427703857422,
      "activations/layer14_attention_weight_min": -38.89540100097656,
      "activations/layer15_attention_weight_max": 56.21781539916992,
      "activations/layer15_attention_weight_min": -33.822872161865234,
      "activations/layer16_attention_weight_max": 44.20928192138672,
      "activations/layer16_attention_weight_min": -28.480356216430664,
      "activations/layer17_attention_weight_max": 73.77405548095703,
      "activations/layer17_attention_weight_min": -36.677818298339844,
      "activations/layer18_attention_weight_max": 60.146915435791016,
      "activations/layer18_attention_weight_min": -32.20001220703125,
      "activations/layer19_attention_weight_max": 29.26169204711914,
      "activations/layer19_attention_weight_min": -18.98904800415039,
      "activations/layer1_attention_weight_max": 18.111827850341797,
      "activations/layer1_attention_weight_min": -15.259033203125,
      "activations/layer20_attention_weight_max": 30.411853790283203,
      "activations/layer20_attention_weight_min": -18.038928985595703,
      "activations/layer21_attention_weight_max": 57.4359130859375,
      "activations/layer21_attention_weight_min": -29.72739601135254,
      "activations/layer22_attention_weight_max": 44.97442626953125,
      "activations/layer22_attention_weight_min": -23.279071807861328,
      "activations/layer23_attention_weight_max": 30.518688201904297,
      "activations/layer23_attention_weight_min": -16.289947509765625,
      "activations/layer2_attention_weight_max": 29.579734802246094,
      "activations/layer2_attention_weight_min": -27.484249114990234,
      "activations/layer3_attention_weight_max": 72.93415832519531,
      "activations/layer3_attention_weight_min": -77.07521057128906,
      "activations/layer4_attention_weight_max": 102.65801239013672,
      "activations/layer4_attention_weight_min": -91.76468658447266,
      "activations/layer5_attention_weight_max": 124.76292419433594,
      "activations/layer5_attention_weight_min": -102.15084838867188,
      "activations/layer6_attention_weight_max": 62.99513626098633,
      "activations/layer6_attention_weight_min": -52.416927337646484,
      "activations/layer7_attention_weight_max": 74.21013641357422,
      "activations/layer7_attention_weight_min": -64.05610656738281,
      "activations/layer8_attention_weight_max": 53.49087142944336,
      "activations/layer8_attention_weight_min": -38.06084060668945,
      "activations/layer9_attention_weight_max": 76.37122344970703,
      "activations/layer9_attention_weight_min": -61.64142608642578,
      "epoch": 3.45,
      "learning_rate": 0.0001290215909090909,
      "loss": 2.898,
      "step": 59450
    },
    {
      "activations/layer0_attention_weight_max": 15.05795955657959,
      "activations/layer0_attention_weight_min": -14.956681251525879,
      "activations/layer10_attention_weight_max": 43.39704132080078,
      "activations/layer10_attention_weight_min": -35.75676345825195,
      "activations/layer11_attention_weight_max": 35.13692855834961,
      "activations/layer11_attention_weight_min": -28.038978576660156,
      "activations/layer12_attention_weight_max": 33.328086853027344,
      "activations/layer12_attention_weight_min": -26.099594116210938,
      "activations/layer13_attention_weight_max": 68.80266571044922,
      "activations/layer13_attention_weight_min": -49.465755462646484,
      "activations/layer14_attention_weight_max": 76.72215270996094,
      "activations/layer14_attention_weight_min": -48.313716888427734,
      "activations/layer15_attention_weight_max": 48.50418472290039,
      "activations/layer15_attention_weight_min": -33.79405212402344,
      "activations/layer16_attention_weight_max": 39.50608444213867,
      "activations/layer16_attention_weight_min": -27.97227668762207,
      "activations/layer17_attention_weight_max": 69.14534759521484,
      "activations/layer17_attention_weight_min": -35.74382019042969,
      "activations/layer18_attention_weight_max": 51.16781997680664,
      "activations/layer18_attention_weight_min": -34.23031997680664,
      "activations/layer19_attention_weight_max": 26.671030044555664,
      "activations/layer19_attention_weight_min": -17.502866744995117,
      "activations/layer1_attention_weight_max": 17.67815589904785,
      "activations/layer1_attention_weight_min": -15.465812683105469,
      "activations/layer20_attention_weight_max": 21.125024795532227,
      "activations/layer20_attention_weight_min": -19.542404174804688,
      "activations/layer21_attention_weight_max": 41.95235824584961,
      "activations/layer21_attention_weight_min": -29.716617584228516,
      "activations/layer22_attention_weight_max": 36.827354431152344,
      "activations/layer22_attention_weight_min": -24.56560516357422,
      "activations/layer23_attention_weight_max": 28.949687957763672,
      "activations/layer23_attention_weight_min": -15.900318145751953,
      "activations/layer2_attention_weight_max": 31.437335968017578,
      "activations/layer2_attention_weight_min": -28.997562408447266,
      "activations/layer3_attention_weight_max": 77.95905303955078,
      "activations/layer3_attention_weight_min": -80.3858871459961,
      "activations/layer4_attention_weight_max": 105.0982666015625,
      "activations/layer4_attention_weight_min": -101.933349609375,
      "activations/layer5_attention_weight_max": 129.95114135742188,
      "activations/layer5_attention_weight_min": -104.49342346191406,
      "activations/layer6_attention_weight_max": 67.19243621826172,
      "activations/layer6_attention_weight_min": -51.88086700439453,
      "activations/layer7_attention_weight_max": 66.82229614257812,
      "activations/layer7_attention_weight_min": -61.064842224121094,
      "activations/layer8_attention_weight_max": 42.89906692504883,
      "activations/layer8_attention_weight_min": -41.125732421875,
      "activations/layer9_attention_weight_max": 64.7275161743164,
      "activations/layer9_attention_weight_min": -49.75139236450195,
      "epoch": 3.46,
      "learning_rate": 0.0001290026515151515,
      "loss": 2.8867,
      "step": 59500
    },
    {
      "activations/layer0_attention_weight_max": 15.168889045715332,
      "activations/layer0_attention_weight_min": -14.616036415100098,
      "activations/layer10_attention_weight_max": 40.11546325683594,
      "activations/layer10_attention_weight_min": -36.525360107421875,
      "activations/layer11_attention_weight_max": 34.11106491088867,
      "activations/layer11_attention_weight_min": -26.400188446044922,
      "activations/layer12_attention_weight_max": 25.031564712524414,
      "activations/layer12_attention_weight_min": -22.664987564086914,
      "activations/layer13_attention_weight_max": 52.471290588378906,
      "activations/layer13_attention_weight_min": -35.26362609863281,
      "activations/layer14_attention_weight_max": 44.42280578613281,
      "activations/layer14_attention_weight_min": -34.258602142333984,
      "activations/layer15_attention_weight_max": 42.12598419189453,
      "activations/layer15_attention_weight_min": -32.375301361083984,
      "activations/layer16_attention_weight_max": 35.07044982910156,
      "activations/layer16_attention_weight_min": -27.107311248779297,
      "activations/layer17_attention_weight_max": 53.239566802978516,
      "activations/layer17_attention_weight_min": -36.0095100402832,
      "activations/layer18_attention_weight_max": 47.87703323364258,
      "activations/layer18_attention_weight_min": -32.227210998535156,
      "activations/layer19_attention_weight_max": 21.307126998901367,
      "activations/layer19_attention_weight_min": -17.585386276245117,
      "activations/layer1_attention_weight_max": 19.03717613220215,
      "activations/layer1_attention_weight_min": -14.076394081115723,
      "activations/layer20_attention_weight_max": 20.054019927978516,
      "activations/layer20_attention_weight_min": -18.80636978149414,
      "activations/layer21_attention_weight_max": 41.52825927734375,
      "activations/layer21_attention_weight_min": -26.706998825073242,
      "activations/layer22_attention_weight_max": 31.62487030029297,
      "activations/layer22_attention_weight_min": -23.257213592529297,
      "activations/layer23_attention_weight_max": 25.314186096191406,
      "activations/layer23_attention_weight_min": -17.52773094177246,
      "activations/layer2_attention_weight_max": 31.036327362060547,
      "activations/layer2_attention_weight_min": -28.19631576538086,
      "activations/layer3_attention_weight_max": 74.08482360839844,
      "activations/layer3_attention_weight_min": -77.74063110351562,
      "activations/layer4_attention_weight_max": 98.95945739746094,
      "activations/layer4_attention_weight_min": -90.97563934326172,
      "activations/layer5_attention_weight_max": 113.05403900146484,
      "activations/layer5_attention_weight_min": -97.52607727050781,
      "activations/layer6_attention_weight_max": 56.86796188354492,
      "activations/layer6_attention_weight_min": -52.086978912353516,
      "activations/layer7_attention_weight_max": 63.16246795654297,
      "activations/layer7_attention_weight_min": -57.164573669433594,
      "activations/layer8_attention_weight_max": 38.58785629272461,
      "activations/layer8_attention_weight_min": -37.76963806152344,
      "activations/layer9_attention_weight_max": 53.60467529296875,
      "activations/layer9_attention_weight_min": -48.63733673095703,
      "epoch": 3.46,
      "learning_rate": 0.00012898371212121212,
      "loss": 2.901,
      "step": 59550
    },
    {
      "activations/layer0_attention_weight_max": 14.33944034576416,
      "activations/layer0_attention_weight_min": -14.708062171936035,
      "activations/layer10_attention_weight_max": 47.23838806152344,
      "activations/layer10_attention_weight_min": -35.43450927734375,
      "activations/layer11_attention_weight_max": 37.64849090576172,
      "activations/layer11_attention_weight_min": -28.021940231323242,
      "activations/layer12_attention_weight_max": 24.700477600097656,
      "activations/layer12_attention_weight_min": -25.365955352783203,
      "activations/layer13_attention_weight_max": 59.15324783325195,
      "activations/layer13_attention_weight_min": -41.543800354003906,
      "activations/layer14_attention_weight_max": 62.8864631652832,
      "activations/layer14_attention_weight_min": -41.100154876708984,
      "activations/layer15_attention_weight_max": 43.3028450012207,
      "activations/layer15_attention_weight_min": -30.98020362854004,
      "activations/layer16_attention_weight_max": 36.36224365234375,
      "activations/layer16_attention_weight_min": -26.168289184570312,
      "activations/layer17_attention_weight_max": 59.78280258178711,
      "activations/layer17_attention_weight_min": -35.78886795043945,
      "activations/layer18_attention_weight_max": 57.596153259277344,
      "activations/layer18_attention_weight_min": -31.482967376708984,
      "activations/layer19_attention_weight_max": 22.83382797241211,
      "activations/layer19_attention_weight_min": -17.522024154663086,
      "activations/layer1_attention_weight_max": 18.540618896484375,
      "activations/layer1_attention_weight_min": -12.865229606628418,
      "activations/layer20_attention_weight_max": 22.208219528198242,
      "activations/layer20_attention_weight_min": -19.81260871887207,
      "activations/layer21_attention_weight_max": 42.58391571044922,
      "activations/layer21_attention_weight_min": -28.178056716918945,
      "activations/layer22_attention_weight_max": 39.64285659790039,
      "activations/layer22_attention_weight_min": -23.436901092529297,
      "activations/layer23_attention_weight_max": 24.203731536865234,
      "activations/layer23_attention_weight_min": -15.930740356445312,
      "activations/layer2_attention_weight_max": 32.46852111816406,
      "activations/layer2_attention_weight_min": -28.736303329467773,
      "activations/layer3_attention_weight_max": 79.108154296875,
      "activations/layer3_attention_weight_min": -80.4439926147461,
      "activations/layer4_attention_weight_max": 109.79622650146484,
      "activations/layer4_attention_weight_min": -99.48277282714844,
      "activations/layer5_attention_weight_max": 134.36044311523438,
      "activations/layer5_attention_weight_min": -110.41903686523438,
      "activations/layer6_attention_weight_max": 66.333984375,
      "activations/layer6_attention_weight_min": -59.646278381347656,
      "activations/layer7_attention_weight_max": 82.18553161621094,
      "activations/layer7_attention_weight_min": -67.24363708496094,
      "activations/layer8_attention_weight_max": 53.48430252075195,
      "activations/layer8_attention_weight_min": -39.99907302856445,
      "activations/layer9_attention_weight_max": 69.71770477294922,
      "activations/layer9_attention_weight_min": -59.13857650756836,
      "epoch": 3.46,
      "learning_rate": 0.0001289647727272727,
      "loss": 2.8901,
      "step": 59600
    },
    {
      "activations/layer0_attention_weight_max": 15.412951469421387,
      "activations/layer0_attention_weight_min": -14.709744453430176,
      "activations/layer10_attention_weight_max": 40.27421569824219,
      "activations/layer10_attention_weight_min": -35.2162971496582,
      "activations/layer11_attention_weight_max": 33.193546295166016,
      "activations/layer11_attention_weight_min": -26.422481536865234,
      "activations/layer12_attention_weight_max": 27.939668655395508,
      "activations/layer12_attention_weight_min": -23.910730361938477,
      "activations/layer13_attention_weight_max": 60.67264175415039,
      "activations/layer13_attention_weight_min": -47.90781021118164,
      "activations/layer14_attention_weight_max": 45.513816833496094,
      "activations/layer14_attention_weight_min": -36.28364181518555,
      "activations/layer15_attention_weight_max": 47.738426208496094,
      "activations/layer15_attention_weight_min": -34.61400604248047,
      "activations/layer16_attention_weight_max": 36.66712951660156,
      "activations/layer16_attention_weight_min": -26.301475524902344,
      "activations/layer17_attention_weight_max": 59.38374710083008,
      "activations/layer17_attention_weight_min": -35.352046966552734,
      "activations/layer18_attention_weight_max": 54.15407943725586,
      "activations/layer18_attention_weight_min": -30.984683990478516,
      "activations/layer19_attention_weight_max": 21.68195343017578,
      "activations/layer19_attention_weight_min": -18.469436645507812,
      "activations/layer1_attention_weight_max": 17.046308517456055,
      "activations/layer1_attention_weight_min": -13.430871963500977,
      "activations/layer20_attention_weight_max": 21.29547691345215,
      "activations/layer20_attention_weight_min": -21.18154525756836,
      "activations/layer21_attention_weight_max": 38.14906311035156,
      "activations/layer21_attention_weight_min": -27.002647399902344,
      "activations/layer22_attention_weight_max": 33.291805267333984,
      "activations/layer22_attention_weight_min": -23.841360092163086,
      "activations/layer23_attention_weight_max": 28.282623291015625,
      "activations/layer23_attention_weight_min": -17.00054931640625,
      "activations/layer2_attention_weight_max": 29.924671173095703,
      "activations/layer2_attention_weight_min": -28.056102752685547,
      "activations/layer3_attention_weight_max": 74.99234008789062,
      "activations/layer3_attention_weight_min": -79.28173065185547,
      "activations/layer4_attention_weight_max": 96.76824188232422,
      "activations/layer4_attention_weight_min": -92.74633026123047,
      "activations/layer5_attention_weight_max": 110.30406951904297,
      "activations/layer5_attention_weight_min": -97.41261291503906,
      "activations/layer6_attention_weight_max": 58.25843811035156,
      "activations/layer6_attention_weight_min": -49.98170852661133,
      "activations/layer7_attention_weight_max": 60.57421875,
      "activations/layer7_attention_weight_min": -56.919097900390625,
      "activations/layer8_attention_weight_max": 38.42765808105469,
      "activations/layer8_attention_weight_min": -37.86577224731445,
      "activations/layer9_attention_weight_max": 46.84830856323242,
      "activations/layer9_attention_weight_min": -48.03957748413086,
      "epoch": 3.47,
      "learning_rate": 0.00012894583333333332,
      "loss": 2.8821,
      "step": 59650
    },
    {
      "activations/layer0_attention_weight_max": 14.21642780303955,
      "activations/layer0_attention_weight_min": -14.638866424560547,
      "activations/layer10_attention_weight_max": 46.11388397216797,
      "activations/layer10_attention_weight_min": -35.30620193481445,
      "activations/layer11_attention_weight_max": 40.063228607177734,
      "activations/layer11_attention_weight_min": -27.475170135498047,
      "activations/layer12_attention_weight_max": 24.723175048828125,
      "activations/layer12_attention_weight_min": -24.921539306640625,
      "activations/layer13_attention_weight_max": 65.85914611816406,
      "activations/layer13_attention_weight_min": -44.10020446777344,
      "activations/layer14_attention_weight_max": 61.18507385253906,
      "activations/layer14_attention_weight_min": -37.10087585449219,
      "activations/layer15_attention_weight_max": 49.15076446533203,
      "activations/layer15_attention_weight_min": -31.508403778076172,
      "activations/layer16_attention_weight_max": 41.93606948852539,
      "activations/layer16_attention_weight_min": -28.767961502075195,
      "activations/layer17_attention_weight_max": 61.92741775512695,
      "activations/layer17_attention_weight_min": -35.950477600097656,
      "activations/layer18_attention_weight_max": 53.759422302246094,
      "activations/layer18_attention_weight_min": -30.01055908203125,
      "activations/layer19_attention_weight_max": 24.81305694580078,
      "activations/layer19_attention_weight_min": -17.53056526184082,
      "activations/layer1_attention_weight_max": 17.60921287536621,
      "activations/layer1_attention_weight_min": -15.60159969329834,
      "activations/layer20_attention_weight_max": 22.65635108947754,
      "activations/layer20_attention_weight_min": -19.33860206604004,
      "activations/layer21_attention_weight_max": 46.755126953125,
      "activations/layer21_attention_weight_min": -27.053537368774414,
      "activations/layer22_attention_weight_max": 37.359676361083984,
      "activations/layer22_attention_weight_min": -24.06647491455078,
      "activations/layer23_attention_weight_max": 28.427188873291016,
      "activations/layer23_attention_weight_min": -17.021251678466797,
      "activations/layer2_attention_weight_max": 29.62319564819336,
      "activations/layer2_attention_weight_min": -28.146339416503906,
      "activations/layer3_attention_weight_max": 75.41899871826172,
      "activations/layer3_attention_weight_min": -79.72499084472656,
      "activations/layer4_attention_weight_max": 104.393798828125,
      "activations/layer4_attention_weight_min": -95.01905822753906,
      "activations/layer5_attention_weight_max": 126.587158203125,
      "activations/layer5_attention_weight_min": -103.771240234375,
      "activations/layer6_attention_weight_max": 66.23307037353516,
      "activations/layer6_attention_weight_min": -52.32527160644531,
      "activations/layer7_attention_weight_max": 62.88521957397461,
      "activations/layer7_attention_weight_min": -60.81987380981445,
      "activations/layer8_attention_weight_max": 41.4012336730957,
      "activations/layer8_attention_weight_min": -37.48138427734375,
      "activations/layer9_attention_weight_max": 53.61588668823242,
      "activations/layer9_attention_weight_min": -48.75832748413086,
      "epoch": 3.47,
      "learning_rate": 0.00012892689393939394,
      "loss": 2.8999,
      "step": 59700
    },
    {
      "activations/layer0_attention_weight_max": 14.349898338317871,
      "activations/layer0_attention_weight_min": -14.291679382324219,
      "activations/layer10_attention_weight_max": 39.18904495239258,
      "activations/layer10_attention_weight_min": -34.6456298828125,
      "activations/layer11_attention_weight_max": 35.06451416015625,
      "activations/layer11_attention_weight_min": -27.02998161315918,
      "activations/layer12_attention_weight_max": 27.345869064331055,
      "activations/layer12_attention_weight_min": -24.584304809570312,
      "activations/layer13_attention_weight_max": 61.14655303955078,
      "activations/layer13_attention_weight_min": -47.469093322753906,
      "activations/layer14_attention_weight_max": 57.66096496582031,
      "activations/layer14_attention_weight_min": -41.56882858276367,
      "activations/layer15_attention_weight_max": 44.43779754638672,
      "activations/layer15_attention_weight_min": -31.271106719970703,
      "activations/layer16_attention_weight_max": 39.862850189208984,
      "activations/layer16_attention_weight_min": -26.627973556518555,
      "activations/layer17_attention_weight_max": 60.22505187988281,
      "activations/layer17_attention_weight_min": -36.118534088134766,
      "activations/layer18_attention_weight_max": 51.78122329711914,
      "activations/layer18_attention_weight_min": -33.01648712158203,
      "activations/layer19_attention_weight_max": 21.30738639831543,
      "activations/layer19_attention_weight_min": -17.46055030822754,
      "activations/layer1_attention_weight_max": 17.224185943603516,
      "activations/layer1_attention_weight_min": -14.897552490234375,
      "activations/layer20_attention_weight_max": 20.597352981567383,
      "activations/layer20_attention_weight_min": -18.6386775970459,
      "activations/layer21_attention_weight_max": 39.301300048828125,
      "activations/layer21_attention_weight_min": -26.663753509521484,
      "activations/layer22_attention_weight_max": 33.42964553833008,
      "activations/layer22_attention_weight_min": -24.657936096191406,
      "activations/layer23_attention_weight_max": 23.21499252319336,
      "activations/layer23_attention_weight_min": -17.210224151611328,
      "activations/layer2_attention_weight_max": 27.92797088623047,
      "activations/layer2_attention_weight_min": -25.38711929321289,
      "activations/layer3_attention_weight_max": 72.09536743164062,
      "activations/layer3_attention_weight_min": -72.02108001708984,
      "activations/layer4_attention_weight_max": 97.7999496459961,
      "activations/layer4_attention_weight_min": -92.15959930419922,
      "activations/layer5_attention_weight_max": 114.99739074707031,
      "activations/layer5_attention_weight_min": -98.84600067138672,
      "activations/layer6_attention_weight_max": 59.620399475097656,
      "activations/layer6_attention_weight_min": -51.938255310058594,
      "activations/layer7_attention_weight_max": 66.42620849609375,
      "activations/layer7_attention_weight_min": -61.025978088378906,
      "activations/layer8_attention_weight_max": 40.16007614135742,
      "activations/layer8_attention_weight_min": -35.3912239074707,
      "activations/layer9_attention_weight_max": 52.93477249145508,
      "activations/layer9_attention_weight_min": -44.8786735534668,
      "epoch": 3.47,
      "learning_rate": 0.00012890795454545455,
      "loss": 2.8931,
      "step": 59750
    },
    {
      "activations/layer0_attention_weight_max": 14.383713722229004,
      "activations/layer0_attention_weight_min": -14.516719818115234,
      "activations/layer10_attention_weight_max": 42.98961639404297,
      "activations/layer10_attention_weight_min": -33.66358947753906,
      "activations/layer11_attention_weight_max": 32.951351165771484,
      "activations/layer11_attention_weight_min": -25.116214752197266,
      "activations/layer12_attention_weight_max": 23.372032165527344,
      "activations/layer12_attention_weight_min": -23.60141372680664,
      "activations/layer13_attention_weight_max": 50.28122329711914,
      "activations/layer13_attention_weight_min": -38.740257263183594,
      "activations/layer14_attention_weight_max": 50.4947395324707,
      "activations/layer14_attention_weight_min": -36.178627014160156,
      "activations/layer15_attention_weight_max": 44.81696701049805,
      "activations/layer15_attention_weight_min": -31.221933364868164,
      "activations/layer16_attention_weight_max": 36.440513610839844,
      "activations/layer16_attention_weight_min": -28.922536849975586,
      "activations/layer17_attention_weight_max": 64.1489028930664,
      "activations/layer17_attention_weight_min": -36.527774810791016,
      "activations/layer18_attention_weight_max": 53.74254608154297,
      "activations/layer18_attention_weight_min": -33.11151123046875,
      "activations/layer19_attention_weight_max": 22.704593658447266,
      "activations/layer19_attention_weight_min": -18.136503219604492,
      "activations/layer1_attention_weight_max": 18.840051651000977,
      "activations/layer1_attention_weight_min": -14.741911888122559,
      "activations/layer20_attention_weight_max": 20.29381561279297,
      "activations/layer20_attention_weight_min": -19.71765899658203,
      "activations/layer21_attention_weight_max": 39.7879524230957,
      "activations/layer21_attention_weight_min": -27.016307830810547,
      "activations/layer22_attention_weight_max": 32.33893585205078,
      "activations/layer22_attention_weight_min": -25.729707717895508,
      "activations/layer23_attention_weight_max": 24.995452880859375,
      "activations/layer23_attention_weight_min": -17.899370193481445,
      "activations/layer2_attention_weight_max": 29.033004760742188,
      "activations/layer2_attention_weight_min": -26.313228607177734,
      "activations/layer3_attention_weight_max": 71.39090728759766,
      "activations/layer3_attention_weight_min": -72.7009048461914,
      "activations/layer4_attention_weight_max": 102.8324203491211,
      "activations/layer4_attention_weight_min": -90.90727233886719,
      "activations/layer5_attention_weight_max": 119.29696655273438,
      "activations/layer5_attention_weight_min": -101.91183471679688,
      "activations/layer6_attention_weight_max": 60.55381393432617,
      "activations/layer6_attention_weight_min": -51.28424835205078,
      "activations/layer7_attention_weight_max": 64.16339874267578,
      "activations/layer7_attention_weight_min": -59.45256042480469,
      "activations/layer8_attention_weight_max": 39.326934814453125,
      "activations/layer8_attention_weight_min": -37.360206604003906,
      "activations/layer9_attention_weight_max": 53.00556564331055,
      "activations/layer9_attention_weight_min": -43.524166107177734,
      "epoch": 3.47,
      "learning_rate": 0.00012888901515151514,
      "loss": 2.8931,
      "step": 59800
    },
    {
      "activations/layer0_attention_weight_max": 14.376484870910645,
      "activations/layer0_attention_weight_min": -14.53596305847168,
      "activations/layer10_attention_weight_max": 39.498130798339844,
      "activations/layer10_attention_weight_min": -32.44602584838867,
      "activations/layer11_attention_weight_max": 31.745361328125,
      "activations/layer11_attention_weight_min": -26.02802848815918,
      "activations/layer12_attention_weight_max": 23.694257736206055,
      "activations/layer12_attention_weight_min": -23.05451011657715,
      "activations/layer13_attention_weight_max": 55.447471618652344,
      "activations/layer13_attention_weight_min": -42.47378158569336,
      "activations/layer14_attention_weight_max": 46.734580993652344,
      "activations/layer14_attention_weight_min": -36.16263198852539,
      "activations/layer15_attention_weight_max": 44.726356506347656,
      "activations/layer15_attention_weight_min": -31.39914894104004,
      "activations/layer16_attention_weight_max": 35.875308990478516,
      "activations/layer16_attention_weight_min": -27.533632278442383,
      "activations/layer17_attention_weight_max": 54.785274505615234,
      "activations/layer17_attention_weight_min": -35.80764389038086,
      "activations/layer18_attention_weight_max": 46.924503326416016,
      "activations/layer18_attention_weight_min": -31.863224029541016,
      "activations/layer19_attention_weight_max": 21.924701690673828,
      "activations/layer19_attention_weight_min": -17.063894271850586,
      "activations/layer1_attention_weight_max": 17.793407440185547,
      "activations/layer1_attention_weight_min": -15.680707931518555,
      "activations/layer20_attention_weight_max": 20.568845748901367,
      "activations/layer20_attention_weight_min": -20.2683048248291,
      "activations/layer21_attention_weight_max": 38.72718811035156,
      "activations/layer21_attention_weight_min": -29.620113372802734,
      "activations/layer22_attention_weight_max": 38.9844856262207,
      "activations/layer22_attention_weight_min": -22.91989517211914,
      "activations/layer23_attention_weight_max": 23.823698043823242,
      "activations/layer23_attention_weight_min": -16.529815673828125,
      "activations/layer2_attention_weight_max": 27.86754608154297,
      "activations/layer2_attention_weight_min": -26.774396896362305,
      "activations/layer3_attention_weight_max": 75.80976104736328,
      "activations/layer3_attention_weight_min": -77.81566619873047,
      "activations/layer4_attention_weight_max": 100.94149780273438,
      "activations/layer4_attention_weight_min": -92.32140350341797,
      "activations/layer5_attention_weight_max": 114.773193359375,
      "activations/layer5_attention_weight_min": -98.65145111083984,
      "activations/layer6_attention_weight_max": 60.84104919433594,
      "activations/layer6_attention_weight_min": -49.05215835571289,
      "activations/layer7_attention_weight_max": 61.10608673095703,
      "activations/layer7_attention_weight_min": -58.28180694580078,
      "activations/layer8_attention_weight_max": 37.20014572143555,
      "activations/layer8_attention_weight_min": -34.43598175048828,
      "activations/layer9_attention_weight_max": 53.63212203979492,
      "activations/layer9_attention_weight_min": -43.84601974487305,
      "epoch": 3.48,
      "learning_rate": 0.00012887007575757575,
      "loss": 2.8672,
      "step": 59850
    },
    {
      "activations/layer0_attention_weight_max": 14.390067100524902,
      "activations/layer0_attention_weight_min": -14.692177772521973,
      "activations/layer10_attention_weight_max": 42.42485427856445,
      "activations/layer10_attention_weight_min": -34.924659729003906,
      "activations/layer11_attention_weight_max": 35.26431655883789,
      "activations/layer11_attention_weight_min": -27.12990379333496,
      "activations/layer12_attention_weight_max": 38.35496139526367,
      "activations/layer12_attention_weight_min": -24.222034454345703,
      "activations/layer13_attention_weight_max": 60.10219192504883,
      "activations/layer13_attention_weight_min": -37.4648551940918,
      "activations/layer14_attention_weight_max": 48.35408020019531,
      "activations/layer14_attention_weight_min": -29.145566940307617,
      "activations/layer15_attention_weight_max": 57.185035705566406,
      "activations/layer15_attention_weight_min": -31.996416091918945,
      "activations/layer16_attention_weight_max": 37.36640167236328,
      "activations/layer16_attention_weight_min": -26.898014068603516,
      "activations/layer17_attention_weight_max": 58.6025276184082,
      "activations/layer17_attention_weight_min": -35.83156204223633,
      "activations/layer18_attention_weight_max": 58.67173385620117,
      "activations/layer18_attention_weight_min": -30.974748611450195,
      "activations/layer19_attention_weight_max": 22.947526931762695,
      "activations/layer19_attention_weight_min": -18.833698272705078,
      "activations/layer1_attention_weight_max": 17.190322875976562,
      "activations/layer1_attention_weight_min": -14.37802791595459,
      "activations/layer20_attention_weight_max": 22.32305908203125,
      "activations/layer20_attention_weight_min": -18.660690307617188,
      "activations/layer21_attention_weight_max": 43.727088928222656,
      "activations/layer21_attention_weight_min": -27.530981063842773,
      "activations/layer22_attention_weight_max": 34.971092224121094,
      "activations/layer22_attention_weight_min": -22.681665420532227,
      "activations/layer23_attention_weight_max": 24.843040466308594,
      "activations/layer23_attention_weight_min": -15.184508323669434,
      "activations/layer2_attention_weight_max": 30.01110076904297,
      "activations/layer2_attention_weight_min": -28.502391815185547,
      "activations/layer3_attention_weight_max": 74.40679931640625,
      "activations/layer3_attention_weight_min": -78.76861572265625,
      "activations/layer4_attention_weight_max": 103.88484954833984,
      "activations/layer4_attention_weight_min": -93.3289794921875,
      "activations/layer5_attention_weight_max": 129.251708984375,
      "activations/layer5_attention_weight_min": -102.84951782226562,
      "activations/layer6_attention_weight_max": 63.21729278564453,
      "activations/layer6_attention_weight_min": -53.12283706665039,
      "activations/layer7_attention_weight_max": 66.52388000488281,
      "activations/layer7_attention_weight_min": -59.762176513671875,
      "activations/layer8_attention_weight_max": 41.850486755371094,
      "activations/layer8_attention_weight_min": -40.53165054321289,
      "activations/layer9_attention_weight_max": 49.52799987792969,
      "activations/layer9_attention_weight_min": -49.66451644897461,
      "epoch": 3.48,
      "learning_rate": 0.00012885113636363637,
      "loss": 2.8739,
      "step": 59900
    },
    {
      "activations/layer0_attention_weight_max": 13.940720558166504,
      "activations/layer0_attention_weight_min": -14.716207504272461,
      "activations/layer10_attention_weight_max": 38.112483978271484,
      "activations/layer10_attention_weight_min": -33.05860137939453,
      "activations/layer11_attention_weight_max": 32.553382873535156,
      "activations/layer11_attention_weight_min": -26.357351303100586,
      "activations/layer12_attention_weight_max": 25.21937370300293,
      "activations/layer12_attention_weight_min": -27.894065856933594,
      "activations/layer13_attention_weight_max": 47.86601257324219,
      "activations/layer13_attention_weight_min": -36.6383171081543,
      "activations/layer14_attention_weight_max": 42.008453369140625,
      "activations/layer14_attention_weight_min": -33.04546356201172,
      "activations/layer15_attention_weight_max": 48.5363883972168,
      "activations/layer15_attention_weight_min": -31.38374900817871,
      "activations/layer16_attention_weight_max": 34.16290283203125,
      "activations/layer16_attention_weight_min": -27.875581741333008,
      "activations/layer17_attention_weight_max": 66.24768829345703,
      "activations/layer17_attention_weight_min": -35.393619537353516,
      "activations/layer18_attention_weight_max": 53.42284393310547,
      "activations/layer18_attention_weight_min": -31.17668342590332,
      "activations/layer19_attention_weight_max": 25.720258712768555,
      "activations/layer19_attention_weight_min": -16.75082778930664,
      "activations/layer1_attention_weight_max": 17.06211280822754,
      "activations/layer1_attention_weight_min": -14.104008674621582,
      "activations/layer20_attention_weight_max": 20.42881965637207,
      "activations/layer20_attention_weight_min": -21.048625946044922,
      "activations/layer21_attention_weight_max": 43.0800666809082,
      "activations/layer21_attention_weight_min": -27.449504852294922,
      "activations/layer22_attention_weight_max": 35.721736907958984,
      "activations/layer22_attention_weight_min": -22.707012176513672,
      "activations/layer23_attention_weight_max": 22.93378257751465,
      "activations/layer23_attention_weight_min": -15.887327194213867,
      "activations/layer2_attention_weight_max": 28.23996353149414,
      "activations/layer2_attention_weight_min": -27.379858016967773,
      "activations/layer3_attention_weight_max": 70.1620101928711,
      "activations/layer3_attention_weight_min": -77.24898529052734,
      "activations/layer4_attention_weight_max": 95.0899887084961,
      "activations/layer4_attention_weight_min": -90.68965148925781,
      "activations/layer5_attention_weight_max": 112.5423583984375,
      "activations/layer5_attention_weight_min": -95.5264892578125,
      "activations/layer6_attention_weight_max": 57.55143356323242,
      "activations/layer6_attention_weight_min": -50.34615707397461,
      "activations/layer7_attention_weight_max": 59.94387435913086,
      "activations/layer7_attention_weight_min": -56.91273880004883,
      "activations/layer8_attention_weight_max": 36.99417495727539,
      "activations/layer8_attention_weight_min": -36.78624725341797,
      "activations/layer9_attention_weight_max": 46.16431427001953,
      "activations/layer9_attention_weight_min": -43.041534423828125,
      "epoch": 3.48,
      "learning_rate": 0.00012883219696969696,
      "loss": 2.8908,
      "step": 59950
    },
    {
      "activations/layer0_attention_weight_max": 13.248831748962402,
      "activations/layer0_attention_weight_min": -14.513606071472168,
      "activations/layer10_attention_weight_max": 41.38539505004883,
      "activations/layer10_attention_weight_min": -34.018489837646484,
      "activations/layer11_attention_weight_max": 34.27784729003906,
      "activations/layer11_attention_weight_min": -24.848430633544922,
      "activations/layer12_attention_weight_max": 27.88715934753418,
      "activations/layer12_attention_weight_min": -26.86307716369629,
      "activations/layer13_attention_weight_max": 49.384605407714844,
      "activations/layer13_attention_weight_min": -36.950035095214844,
      "activations/layer14_attention_weight_max": 43.9412727355957,
      "activations/layer14_attention_weight_min": -33.63233184814453,
      "activations/layer15_attention_weight_max": 41.2257194519043,
      "activations/layer15_attention_weight_min": -30.849550247192383,
      "activations/layer16_attention_weight_max": 35.951168060302734,
      "activations/layer16_attention_weight_min": -25.687135696411133,
      "activations/layer17_attention_weight_max": 66.71615600585938,
      "activations/layer17_attention_weight_min": -35.05463409423828,
      "activations/layer18_attention_weight_max": 48.35352325439453,
      "activations/layer18_attention_weight_min": -31.688505172729492,
      "activations/layer19_attention_weight_max": 20.497661590576172,
      "activations/layer19_attention_weight_min": -19.891599655151367,
      "activations/layer1_attention_weight_max": 17.885013580322266,
      "activations/layer1_attention_weight_min": -16.14158058166504,
      "activations/layer20_attention_weight_max": 20.300378799438477,
      "activations/layer20_attention_weight_min": -20.104143142700195,
      "activations/layer21_attention_weight_max": 45.6699104309082,
      "activations/layer21_attention_weight_min": -27.556875228881836,
      "activations/layer22_attention_weight_max": 33.30966567993164,
      "activations/layer22_attention_weight_min": -21.960718154907227,
      "activations/layer23_attention_weight_max": 23.755252838134766,
      "activations/layer23_attention_weight_min": -16.11831283569336,
      "activations/layer2_attention_weight_max": 28.848339080810547,
      "activations/layer2_attention_weight_min": -27.494888305664062,
      "activations/layer3_attention_weight_max": 71.36289978027344,
      "activations/layer3_attention_weight_min": -74.18445587158203,
      "activations/layer4_attention_weight_max": 100.91018676757812,
      "activations/layer4_attention_weight_min": -89.51338195800781,
      "activations/layer5_attention_weight_max": 113.13835906982422,
      "activations/layer5_attention_weight_min": -97.1057357788086,
      "activations/layer6_attention_weight_max": 59.68509292602539,
      "activations/layer6_attention_weight_min": -50.46575164794922,
      "activations/layer7_attention_weight_max": 59.52996063232422,
      "activations/layer7_attention_weight_min": -56.36573028564453,
      "activations/layer8_attention_weight_max": 37.626258850097656,
      "activations/layer8_attention_weight_min": -37.4525146484375,
      "activations/layer9_attention_weight_max": 52.05263900756836,
      "activations/layer9_attention_weight_min": -45.989105224609375,
      "epoch": 3.49,
      "learning_rate": 0.00012881325757575757,
      "loss": 2.9044,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_loss": 2.83984375,
      "eval_runtime": 8.3934,
      "eval_samples_per_second": 511.593,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_openwebtext_loss": 2.83984375,
      "eval_openwebtext_ppl": 17.113091407702502,
      "eval_openwebtext_runtime": 8.3934,
      "eval_openwebtext_samples_per_second": 511.593,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_wikitext_loss": 3.0859375,
      "eval_wikitext_ppl": 21.887977204970085,
      "eval_wikitext_runtime": 1.9151,
      "eval_wikitext_samples_per_second": 238.105,
      "step": 60000
    },
    {
      "epoch": 3.49,
      "eval_lambada_loss": 2.984375,
      "eval_lambada_ppl": 19.774139536218833,
      "eval_lambada_runtime": 9.5022,
      "eval_lambada_samples_per_second": 512.408,
      "step": 60000
    },
    {
      "activations/layer0_attention_weight_max": 14.035957336425781,
      "activations/layer0_attention_weight_min": -15.686970710754395,
      "activations/layer10_attention_weight_max": 41.477691650390625,
      "activations/layer10_attention_weight_min": -33.27963638305664,
      "activations/layer11_attention_weight_max": 34.172672271728516,
      "activations/layer11_attention_weight_min": -25.818288803100586,
      "activations/layer12_attention_weight_max": 27.89218521118164,
      "activations/layer12_attention_weight_min": -25.00011444091797,
      "activations/layer13_attention_weight_max": 53.753623962402344,
      "activations/layer13_attention_weight_min": -39.522430419921875,
      "activations/layer14_attention_weight_max": 59.358089447021484,
      "activations/layer14_attention_weight_min": -41.82728958129883,
      "activations/layer15_attention_weight_max": 49.91119384765625,
      "activations/layer15_attention_weight_min": -32.150169372558594,
      "activations/layer16_attention_weight_max": 43.799102783203125,
      "activations/layer16_attention_weight_min": -26.657421112060547,
      "activations/layer17_attention_weight_max": 65.47935485839844,
      "activations/layer17_attention_weight_min": -38.62672805786133,
      "activations/layer18_attention_weight_max": 59.6727180480957,
      "activations/layer18_attention_weight_min": -33.31647872924805,
      "activations/layer19_attention_weight_max": 26.485254287719727,
      "activations/layer19_attention_weight_min": -18.86458396911621,
      "activations/layer1_attention_weight_max": 18.072811126708984,
      "activations/layer1_attention_weight_min": -13.998973846435547,
      "activations/layer20_attention_weight_max": 27.721418380737305,
      "activations/layer20_attention_weight_min": -19.655139923095703,
      "activations/layer21_attention_weight_max": 51.918357849121094,
      "activations/layer21_attention_weight_min": -28.792156219482422,
      "activations/layer22_attention_weight_max": 43.36919021606445,
      "activations/layer22_attention_weight_min": -23.453266143798828,
      "activations/layer23_attention_weight_max": 29.603408813476562,
      "activations/layer23_attention_weight_min": -16.430482864379883,
      "activations/layer2_attention_weight_max": 29.558025360107422,
      "activations/layer2_attention_weight_min": -26.734901428222656,
      "activations/layer3_attention_weight_max": 73.90626525878906,
      "activations/layer3_attention_weight_min": -77.46368408203125,
      "activations/layer4_attention_weight_max": 100.36799621582031,
      "activations/layer4_attention_weight_min": -88.7738265991211,
      "activations/layer5_attention_weight_max": 111.70333099365234,
      "activations/layer5_attention_weight_min": -92.09810638427734,
      "activations/layer6_attention_weight_max": 57.80803298950195,
      "activations/layer6_attention_weight_min": -48.126243591308594,
      "activations/layer7_attention_weight_max": 62.00301742553711,
      "activations/layer7_attention_weight_min": -56.13622283935547,
      "activations/layer8_attention_weight_max": 38.129554748535156,
      "activations/layer8_attention_weight_min": -37.34605026245117,
      "activations/layer9_attention_weight_max": 46.74909591674805,
      "activations/layer9_attention_weight_min": -46.01133346557617,
      "epoch": 3.49,
      "learning_rate": 0.00012879431818181816,
      "loss": 2.8956,
      "step": 60050
    },
    {
      "activations/layer0_attention_weight_max": 14.564138412475586,
      "activations/layer0_attention_weight_min": -14.416888236999512,
      "activations/layer10_attention_weight_max": 43.90625762939453,
      "activations/layer10_attention_weight_min": -34.816890716552734,
      "activations/layer11_attention_weight_max": 33.52432632446289,
      "activations/layer11_attention_weight_min": -28.585708618164062,
      "activations/layer12_attention_weight_max": 25.429582595825195,
      "activations/layer12_attention_weight_min": -23.986743927001953,
      "activations/layer13_attention_weight_max": 58.78535079956055,
      "activations/layer13_attention_weight_min": -38.89158248901367,
      "activations/layer14_attention_weight_max": 63.84461975097656,
      "activations/layer14_attention_weight_min": -32.033329010009766,
      "activations/layer15_attention_weight_max": 49.0399284362793,
      "activations/layer15_attention_weight_min": -32.31267166137695,
      "activations/layer16_attention_weight_max": 46.42750549316406,
      "activations/layer16_attention_weight_min": -27.17120933532715,
      "activations/layer17_attention_weight_max": 59.91999053955078,
      "activations/layer17_attention_weight_min": -35.64445877075195,
      "activations/layer18_attention_weight_max": 57.0889892578125,
      "activations/layer18_attention_weight_min": -31.46980094909668,
      "activations/layer19_attention_weight_max": 25.384498596191406,
      "activations/layer19_attention_weight_min": -18.814313888549805,
      "activations/layer1_attention_weight_max": 17.726831436157227,
      "activations/layer1_attention_weight_min": -16.51168441772461,
      "activations/layer20_attention_weight_max": 22.918771743774414,
      "activations/layer20_attention_weight_min": -17.594465255737305,
      "activations/layer21_attention_weight_max": 43.48223876953125,
      "activations/layer21_attention_weight_min": -26.457794189453125,
      "activations/layer22_attention_weight_max": 37.06262969970703,
      "activations/layer22_attention_weight_min": -25.07574462890625,
      "activations/layer23_attention_weight_max": 28.200572967529297,
      "activations/layer23_attention_weight_min": -15.315340995788574,
      "activations/layer2_attention_weight_max": 29.535968780517578,
      "activations/layer2_attention_weight_min": -28.104278564453125,
      "activations/layer3_attention_weight_max": 71.81990814208984,
      "activations/layer3_attention_weight_min": -76.22969055175781,
      "activations/layer4_attention_weight_max": 101.20628356933594,
      "activations/layer4_attention_weight_min": -90.24202728271484,
      "activations/layer5_attention_weight_max": 115.11998748779297,
      "activations/layer5_attention_weight_min": -93.92305755615234,
      "activations/layer6_attention_weight_max": 59.50291061401367,
      "activations/layer6_attention_weight_min": -51.946292877197266,
      "activations/layer7_attention_weight_max": 63.63998031616211,
      "activations/layer7_attention_weight_min": -61.033912658691406,
      "activations/layer8_attention_weight_max": 41.94306945800781,
      "activations/layer8_attention_weight_min": -38.410491943359375,
      "activations/layer9_attention_weight_max": 50.22932815551758,
      "activations/layer9_attention_weight_min": -50.66219711303711,
      "epoch": 3.49,
      "learning_rate": 0.00012877537878787878,
      "loss": 2.8951,
      "step": 60100
    },
    {
      "activations/layer0_attention_weight_max": 15.121766090393066,
      "activations/layer0_attention_weight_min": -14.808809280395508,
      "activations/layer10_attention_weight_max": 43.57769775390625,
      "activations/layer10_attention_weight_min": -35.84684371948242,
      "activations/layer11_attention_weight_max": 33.70464324951172,
      "activations/layer11_attention_weight_min": -26.352331161499023,
      "activations/layer12_attention_weight_max": 29.273405075073242,
      "activations/layer12_attention_weight_min": -23.890764236450195,
      "activations/layer13_attention_weight_max": 66.5846176147461,
      "activations/layer13_attention_weight_min": -40.36189270019531,
      "activations/layer14_attention_weight_max": 57.81620407104492,
      "activations/layer14_attention_weight_min": -32.71781921386719,
      "activations/layer15_attention_weight_max": 51.59734344482422,
      "activations/layer15_attention_weight_min": -34.30339813232422,
      "activations/layer16_attention_weight_max": 44.58507537841797,
      "activations/layer16_attention_weight_min": -27.63701057434082,
      "activations/layer17_attention_weight_max": 64.23226928710938,
      "activations/layer17_attention_weight_min": -36.07810974121094,
      "activations/layer18_attention_weight_max": 54.304237365722656,
      "activations/layer18_attention_weight_min": -32.36050033569336,
      "activations/layer19_attention_weight_max": 23.14516830444336,
      "activations/layer19_attention_weight_min": -16.8328857421875,
      "activations/layer1_attention_weight_max": 19.173309326171875,
      "activations/layer1_attention_weight_min": -13.927371978759766,
      "activations/layer20_attention_weight_max": 21.071697235107422,
      "activations/layer20_attention_weight_min": -19.525503158569336,
      "activations/layer21_attention_weight_max": 43.773651123046875,
      "activations/layer21_attention_weight_min": -27.022186279296875,
      "activations/layer22_attention_weight_max": 39.76811599731445,
      "activations/layer22_attention_weight_min": -23.81475830078125,
      "activations/layer23_attention_weight_max": 29.39679527282715,
      "activations/layer23_attention_weight_min": -16.825279235839844,
      "activations/layer2_attention_weight_max": 28.725902557373047,
      "activations/layer2_attention_weight_min": -26.823623657226562,
      "activations/layer3_attention_weight_max": 72.1031723022461,
      "activations/layer3_attention_weight_min": -77.797607421875,
      "activations/layer4_attention_weight_max": 97.80987548828125,
      "activations/layer4_attention_weight_min": -90.76252746582031,
      "activations/layer5_attention_weight_max": 111.95114135742188,
      "activations/layer5_attention_weight_min": -97.83684539794922,
      "activations/layer6_attention_weight_max": 57.61637496948242,
      "activations/layer6_attention_weight_min": -49.51332473754883,
      "activations/layer7_attention_weight_max": 60.64170455932617,
      "activations/layer7_attention_weight_min": -56.36534118652344,
      "activations/layer8_attention_weight_max": 39.9100456237793,
      "activations/layer8_attention_weight_min": -35.83098220825195,
      "activations/layer9_attention_weight_max": 55.328216552734375,
      "activations/layer9_attention_weight_min": -46.06269073486328,
      "epoch": 3.5,
      "learning_rate": 0.0001287564393939394,
      "loss": 2.8901,
      "step": 60150
    },
    {
      "activations/layer0_attention_weight_max": 13.883675575256348,
      "activations/layer0_attention_weight_min": -14.175519943237305,
      "activations/layer10_attention_weight_max": 44.79393005371094,
      "activations/layer10_attention_weight_min": -38.89815139770508,
      "activations/layer11_attention_weight_max": 36.652137756347656,
      "activations/layer11_attention_weight_min": -29.126890182495117,
      "activations/layer12_attention_weight_max": 27.27916145324707,
      "activations/layer12_attention_weight_min": -25.227218627929688,
      "activations/layer13_attention_weight_max": 52.973793029785156,
      "activations/layer13_attention_weight_min": -38.31611633300781,
      "activations/layer14_attention_weight_max": 46.867916107177734,
      "activations/layer14_attention_weight_min": -35.71094512939453,
      "activations/layer15_attention_weight_max": 47.24797821044922,
      "activations/layer15_attention_weight_min": -32.606563568115234,
      "activations/layer16_attention_weight_max": 35.852516174316406,
      "activations/layer16_attention_weight_min": -26.678789138793945,
      "activations/layer17_attention_weight_max": 60.46949768066406,
      "activations/layer17_attention_weight_min": -35.74095916748047,
      "activations/layer18_attention_weight_max": 50.46833801269531,
      "activations/layer18_attention_weight_min": -31.200960159301758,
      "activations/layer19_attention_weight_max": 21.25873374938965,
      "activations/layer19_attention_weight_min": -19.656105041503906,
      "activations/layer1_attention_weight_max": 17.72047996520996,
      "activations/layer1_attention_weight_min": -16.360004425048828,
      "activations/layer20_attention_weight_max": 21.244714736938477,
      "activations/layer20_attention_weight_min": -19.511173248291016,
      "activations/layer21_attention_weight_max": 39.69953155517578,
      "activations/layer21_attention_weight_min": -26.91094970703125,
      "activations/layer22_attention_weight_max": 31.02910614013672,
      "activations/layer22_attention_weight_min": -21.994056701660156,
      "activations/layer23_attention_weight_max": 24.796165466308594,
      "activations/layer23_attention_weight_min": -16.318950653076172,
      "activations/layer2_attention_weight_max": 32.07792282104492,
      "activations/layer2_attention_weight_min": -28.521976470947266,
      "activations/layer3_attention_weight_max": 76.26126098632812,
      "activations/layer3_attention_weight_min": -78.53894805908203,
      "activations/layer4_attention_weight_max": 105.1216049194336,
      "activations/layer4_attention_weight_min": -97.41950988769531,
      "activations/layer5_attention_weight_max": 121.42349243164062,
      "activations/layer5_attention_weight_min": -104.96285247802734,
      "activations/layer6_attention_weight_max": 62.39631652832031,
      "activations/layer6_attention_weight_min": -53.94265365600586,
      "activations/layer7_attention_weight_max": 68.54981231689453,
      "activations/layer7_attention_weight_min": -60.31648254394531,
      "activations/layer8_attention_weight_max": 44.31840133666992,
      "activations/layer8_attention_weight_min": -39.90869903564453,
      "activations/layer9_attention_weight_max": 62.6778678894043,
      "activations/layer9_attention_weight_min": -52.77938461303711,
      "epoch": 3.5,
      "learning_rate": 0.00012873749999999998,
      "loss": 2.8872,
      "step": 60200
    },
    {
      "activations/layer0_attention_weight_max": 16.6796875,
      "activations/layer0_attention_weight_min": -14.583991050720215,
      "activations/layer10_attention_weight_max": 45.05489730834961,
      "activations/layer10_attention_weight_min": -37.78682327270508,
      "activations/layer11_attention_weight_max": 38.47935485839844,
      "activations/layer11_attention_weight_min": -29.10657501220703,
      "activations/layer12_attention_weight_max": 22.99717903137207,
      "activations/layer12_attention_weight_min": -24.760543823242188,
      "activations/layer13_attention_weight_max": 52.13705825805664,
      "activations/layer13_attention_weight_min": -43.57331466674805,
      "activations/layer14_attention_weight_max": 40.8370475769043,
      "activations/layer14_attention_weight_min": -29.458229064941406,
      "activations/layer15_attention_weight_max": 44.327762603759766,
      "activations/layer15_attention_weight_min": -31.49359703063965,
      "activations/layer16_attention_weight_max": 34.19210433959961,
      "activations/layer16_attention_weight_min": -25.80689811706543,
      "activations/layer17_attention_weight_max": 56.27146911621094,
      "activations/layer17_attention_weight_min": -34.330833435058594,
      "activations/layer18_attention_weight_max": 48.10513687133789,
      "activations/layer18_attention_weight_min": -31.10091781616211,
      "activations/layer19_attention_weight_max": 23.84706687927246,
      "activations/layer19_attention_weight_min": -17.178483963012695,
      "activations/layer1_attention_weight_max": 19.380321502685547,
      "activations/layer1_attention_weight_min": -16.450153350830078,
      "activations/layer20_attention_weight_max": 20.317922592163086,
      "activations/layer20_attention_weight_min": -19.078413009643555,
      "activations/layer21_attention_weight_max": 36.06326675415039,
      "activations/layer21_attention_weight_min": -25.352922439575195,
      "activations/layer22_attention_weight_max": 34.6771125793457,
      "activations/layer22_attention_weight_min": -23.68311309814453,
      "activations/layer23_attention_weight_max": 24.931434631347656,
      "activations/layer23_attention_weight_min": -15.88221549987793,
      "activations/layer2_attention_weight_max": 29.295612335205078,
      "activations/layer2_attention_weight_min": -27.517261505126953,
      "activations/layer3_attention_weight_max": 73.31168365478516,
      "activations/layer3_attention_weight_min": -76.12451171875,
      "activations/layer4_attention_weight_max": 98.60601043701172,
      "activations/layer4_attention_weight_min": -90.47792053222656,
      "activations/layer5_attention_weight_max": 118.43651580810547,
      "activations/layer5_attention_weight_min": -98.51342010498047,
      "activations/layer6_attention_weight_max": 62.791465759277344,
      "activations/layer6_attention_weight_min": -51.242496490478516,
      "activations/layer7_attention_weight_max": 61.81235885620117,
      "activations/layer7_attention_weight_min": -58.935516357421875,
      "activations/layer8_attention_weight_max": 40.4338264465332,
      "activations/layer8_attention_weight_min": -38.45315933227539,
      "activations/layer9_attention_weight_max": 49.88859558105469,
      "activations/layer9_attention_weight_min": -49.15802001953125,
      "epoch": 3.5,
      "learning_rate": 0.0001287185606060606,
      "loss": 2.9029,
      "step": 60250
    },
    {
      "activations/layer0_attention_weight_max": 13.913582801818848,
      "activations/layer0_attention_weight_min": -14.626075744628906,
      "activations/layer10_attention_weight_max": 43.42320251464844,
      "activations/layer10_attention_weight_min": -35.48279571533203,
      "activations/layer11_attention_weight_max": 37.88359832763672,
      "activations/layer11_attention_weight_min": -27.83788299560547,
      "activations/layer12_attention_weight_max": 28.224504470825195,
      "activations/layer12_attention_weight_min": -25.215248107910156,
      "activations/layer13_attention_weight_max": 64.09434509277344,
      "activations/layer13_attention_weight_min": -41.970603942871094,
      "activations/layer14_attention_weight_max": 66.98062896728516,
      "activations/layer14_attention_weight_min": -36.89899826049805,
      "activations/layer15_attention_weight_max": 49.80620574951172,
      "activations/layer15_attention_weight_min": -30.67862892150879,
      "activations/layer16_attention_weight_max": 46.08717346191406,
      "activations/layer16_attention_weight_min": -28.072410583496094,
      "activations/layer17_attention_weight_max": 61.57910919189453,
      "activations/layer17_attention_weight_min": -37.68531036376953,
      "activations/layer18_attention_weight_max": 58.837806701660156,
      "activations/layer18_attention_weight_min": -34.3482780456543,
      "activations/layer19_attention_weight_max": 23.32489585876465,
      "activations/layer19_attention_weight_min": -17.936555862426758,
      "activations/layer1_attention_weight_max": 17.18927001953125,
      "activations/layer1_attention_weight_min": -13.630206108093262,
      "activations/layer20_attention_weight_max": 26.44548797607422,
      "activations/layer20_attention_weight_min": -20.147031784057617,
      "activations/layer21_attention_weight_max": 46.85210037231445,
      "activations/layer21_attention_weight_min": -27.58087158203125,
      "activations/layer22_attention_weight_max": 44.934654235839844,
      "activations/layer22_attention_weight_min": -23.53351402282715,
      "activations/layer23_attention_weight_max": 31.37580108642578,
      "activations/layer23_attention_weight_min": -18.691617965698242,
      "activations/layer2_attention_weight_max": 29.2015380859375,
      "activations/layer2_attention_weight_min": -27.461463928222656,
      "activations/layer3_attention_weight_max": 74.45709228515625,
      "activations/layer3_attention_weight_min": -75.90718841552734,
      "activations/layer4_attention_weight_max": 102.60345458984375,
      "activations/layer4_attention_weight_min": -95.40174102783203,
      "activations/layer5_attention_weight_max": 117.6632308959961,
      "activations/layer5_attention_weight_min": -98.52371215820312,
      "activations/layer6_attention_weight_max": 60.80579376220703,
      "activations/layer6_attention_weight_min": -52.84410095214844,
      "activations/layer7_attention_weight_max": 64.5991439819336,
      "activations/layer7_attention_weight_min": -57.79582977294922,
      "activations/layer8_attention_weight_max": 41.552005767822266,
      "activations/layer8_attention_weight_min": -36.700862884521484,
      "activations/layer9_attention_weight_max": 48.70363235473633,
      "activations/layer9_attention_weight_min": -45.525978088378906,
      "epoch": 3.5,
      "learning_rate": 0.0001286996212121212,
      "loss": 2.8936,
      "step": 60300
    },
    {
      "activations/layer0_attention_weight_max": 14.166626930236816,
      "activations/layer0_attention_weight_min": -14.882972717285156,
      "activations/layer10_attention_weight_max": 44.42753601074219,
      "activations/layer10_attention_weight_min": -34.985652923583984,
      "activations/layer11_attention_weight_max": 34.938026428222656,
      "activations/layer11_attention_weight_min": -26.384796142578125,
      "activations/layer12_attention_weight_max": 28.68767547607422,
      "activations/layer12_attention_weight_min": -27.8138484954834,
      "activations/layer13_attention_weight_max": 51.94990158081055,
      "activations/layer13_attention_weight_min": -42.70439529418945,
      "activations/layer14_attention_weight_max": 45.91134262084961,
      "activations/layer14_attention_weight_min": -34.07597732543945,
      "activations/layer15_attention_weight_max": 46.492977142333984,
      "activations/layer15_attention_weight_min": -36.03006362915039,
      "activations/layer16_attention_weight_max": 34.83741760253906,
      "activations/layer16_attention_weight_min": -27.163990020751953,
      "activations/layer17_attention_weight_max": 54.79499816894531,
      "activations/layer17_attention_weight_min": -38.54991912841797,
      "activations/layer18_attention_weight_max": 52.79698944091797,
      "activations/layer18_attention_weight_min": -34.200775146484375,
      "activations/layer19_attention_weight_max": 20.263198852539062,
      "activations/layer19_attention_weight_min": -17.418790817260742,
      "activations/layer1_attention_weight_max": 18.044471740722656,
      "activations/layer1_attention_weight_min": -13.55452823638916,
      "activations/layer20_attention_weight_max": 19.63752555847168,
      "activations/layer20_attention_weight_min": -18.682451248168945,
      "activations/layer21_attention_weight_max": 39.125274658203125,
      "activations/layer21_attention_weight_min": -26.264183044433594,
      "activations/layer22_attention_weight_max": 31.22837257385254,
      "activations/layer22_attention_weight_min": -24.054555892944336,
      "activations/layer23_attention_weight_max": 23.748451232910156,
      "activations/layer23_attention_weight_min": -16.803457260131836,
      "activations/layer2_attention_weight_max": 30.227252960205078,
      "activations/layer2_attention_weight_min": -28.456466674804688,
      "activations/layer3_attention_weight_max": 76.68682861328125,
      "activations/layer3_attention_weight_min": -78.86199188232422,
      "activations/layer4_attention_weight_max": 103.6728515625,
      "activations/layer4_attention_weight_min": -97.87345123291016,
      "activations/layer5_attention_weight_max": 121.92704772949219,
      "activations/layer5_attention_weight_min": -105.01446533203125,
      "activations/layer6_attention_weight_max": 62.45008850097656,
      "activations/layer6_attention_weight_min": -53.352413177490234,
      "activations/layer7_attention_weight_max": 62.5963020324707,
      "activations/layer7_attention_weight_min": -59.17926025390625,
      "activations/layer8_attention_weight_max": 41.7530632019043,
      "activations/layer8_attention_weight_min": -40.14554214477539,
      "activations/layer9_attention_weight_max": 53.06464385986328,
      "activations/layer9_attention_weight_min": -48.01630401611328,
      "epoch": 3.51,
      "learning_rate": 0.0001286806818181818,
      "loss": 2.9024,
      "step": 60350
    },
    {
      "activations/layer0_attention_weight_max": 14.492308616638184,
      "activations/layer0_attention_weight_min": -15.064871788024902,
      "activations/layer10_attention_weight_max": 40.155052185058594,
      "activations/layer10_attention_weight_min": -32.24146270751953,
      "activations/layer11_attention_weight_max": 33.34461975097656,
      "activations/layer11_attention_weight_min": -25.73150634765625,
      "activations/layer12_attention_weight_max": 25.505170822143555,
      "activations/layer12_attention_weight_min": -26.423110961914062,
      "activations/layer13_attention_weight_max": 61.72361755371094,
      "activations/layer13_attention_weight_min": -42.526451110839844,
      "activations/layer14_attention_weight_max": 52.875465393066406,
      "activations/layer14_attention_weight_min": -35.67292022705078,
      "activations/layer15_attention_weight_max": 51.594337463378906,
      "activations/layer15_attention_weight_min": -32.77266311645508,
      "activations/layer16_attention_weight_max": 35.53152847290039,
      "activations/layer16_attention_weight_min": -24.56555938720703,
      "activations/layer17_attention_weight_max": 70.9615707397461,
      "activations/layer17_attention_weight_min": -38.33314895629883,
      "activations/layer18_attention_weight_max": 52.646583557128906,
      "activations/layer18_attention_weight_min": -34.02983474731445,
      "activations/layer19_attention_weight_max": 26.350271224975586,
      "activations/layer19_attention_weight_min": -18.60334587097168,
      "activations/layer1_attention_weight_max": 17.23678970336914,
      "activations/layer1_attention_weight_min": -17.62844467163086,
      "activations/layer20_attention_weight_max": 21.076255798339844,
      "activations/layer20_attention_weight_min": -17.260318756103516,
      "activations/layer21_attention_weight_max": 39.482391357421875,
      "activations/layer21_attention_weight_min": -28.439138412475586,
      "activations/layer22_attention_weight_max": 34.461971282958984,
      "activations/layer22_attention_weight_min": -23.305622100830078,
      "activations/layer23_attention_weight_max": 25.173574447631836,
      "activations/layer23_attention_weight_min": -19.853160858154297,
      "activations/layer2_attention_weight_max": 29.881927490234375,
      "activations/layer2_attention_weight_min": -28.49575424194336,
      "activations/layer3_attention_weight_max": 77.7409896850586,
      "activations/layer3_attention_weight_min": -79.2645263671875,
      "activations/layer4_attention_weight_max": 99.99117279052734,
      "activations/layer4_attention_weight_min": -95.2172622680664,
      "activations/layer5_attention_weight_max": 113.16365051269531,
      "activations/layer5_attention_weight_min": -99.97200775146484,
      "activations/layer6_attention_weight_max": 58.965965270996094,
      "activations/layer6_attention_weight_min": -49.91252136230469,
      "activations/layer7_attention_weight_max": 64.5125732421875,
      "activations/layer7_attention_weight_min": -58.685611724853516,
      "activations/layer8_attention_weight_max": 37.17869567871094,
      "activations/layer8_attention_weight_min": -33.65365982055664,
      "activations/layer9_attention_weight_max": 51.77741241455078,
      "activations/layer9_attention_weight_min": -46.92472839355469,
      "epoch": 3.51,
      "learning_rate": 0.0001286617424242424,
      "loss": 2.886,
      "step": 60400
    },
    {
      "activations/layer0_attention_weight_max": 17.13741683959961,
      "activations/layer0_attention_weight_min": -15.170011520385742,
      "activations/layer10_attention_weight_max": 49.38975524902344,
      "activations/layer10_attention_weight_min": -34.024497985839844,
      "activations/layer11_attention_weight_max": 38.08552551269531,
      "activations/layer11_attention_weight_min": -25.63852310180664,
      "activations/layer12_attention_weight_max": 30.887556076049805,
      "activations/layer12_attention_weight_min": -29.03470230102539,
      "activations/layer13_attention_weight_max": 68.56676483154297,
      "activations/layer13_attention_weight_min": -48.53176498413086,
      "activations/layer14_attention_weight_max": 57.04865264892578,
      "activations/layer14_attention_weight_min": -32.905059814453125,
      "activations/layer15_attention_weight_max": 57.84203338623047,
      "activations/layer15_attention_weight_min": -34.19941329956055,
      "activations/layer16_attention_weight_max": 49.51742172241211,
      "activations/layer16_attention_weight_min": -25.860214233398438,
      "activations/layer17_attention_weight_max": 62.03137969970703,
      "activations/layer17_attention_weight_min": -36.80699157714844,
      "activations/layer18_attention_weight_max": 53.271385192871094,
      "activations/layer18_attention_weight_min": -32.786956787109375,
      "activations/layer19_attention_weight_max": 21.32535171508789,
      "activations/layer19_attention_weight_min": -16.877229690551758,
      "activations/layer1_attention_weight_max": 18.741138458251953,
      "activations/layer1_attention_weight_min": -13.784418106079102,
      "activations/layer20_attention_weight_max": 19.639114379882812,
      "activations/layer20_attention_weight_min": -24.689212799072266,
      "activations/layer21_attention_weight_max": 40.28078842163086,
      "activations/layer21_attention_weight_min": -27.218807220458984,
      "activations/layer22_attention_weight_max": 33.482967376708984,
      "activations/layer22_attention_weight_min": -22.209514617919922,
      "activations/layer23_attention_weight_max": 23.84316635131836,
      "activations/layer23_attention_weight_min": -15.497625350952148,
      "activations/layer2_attention_weight_max": 29.312744140625,
      "activations/layer2_attention_weight_min": -27.376399993896484,
      "activations/layer3_attention_weight_max": 73.5807876586914,
      "activations/layer3_attention_weight_min": -76.50493621826172,
      "activations/layer4_attention_weight_max": 103.91413879394531,
      "activations/layer4_attention_weight_min": -90.63607025146484,
      "activations/layer5_attention_weight_max": 123.87628173828125,
      "activations/layer5_attention_weight_min": -95.97005462646484,
      "activations/layer6_attention_weight_max": 65.38804626464844,
      "activations/layer6_attention_weight_min": -51.36711883544922,
      "activations/layer7_attention_weight_max": 67.762939453125,
      "activations/layer7_attention_weight_min": -60.65365219116211,
      "activations/layer8_attention_weight_max": 50.72798538208008,
      "activations/layer8_attention_weight_min": -37.08969497680664,
      "activations/layer9_attention_weight_max": 63.54038619995117,
      "activations/layer9_attention_weight_min": -48.00062942504883,
      "epoch": 3.51,
      "learning_rate": 0.00012864280303030303,
      "loss": 2.8906,
      "step": 60450
    },
    {
      "activations/layer0_attention_weight_max": 14.139984130859375,
      "activations/layer0_attention_weight_min": -14.654727935791016,
      "activations/layer10_attention_weight_max": 39.46710968017578,
      "activations/layer10_attention_weight_min": -33.81834030151367,
      "activations/layer11_attention_weight_max": 36.03699493408203,
      "activations/layer11_attention_weight_min": -28.18977165222168,
      "activations/layer12_attention_weight_max": 21.498132705688477,
      "activations/layer12_attention_weight_min": -22.97716522216797,
      "activations/layer13_attention_weight_max": 61.67049026489258,
      "activations/layer13_attention_weight_min": -46.59947967529297,
      "activations/layer14_attention_weight_max": 59.207454681396484,
      "activations/layer14_attention_weight_min": -40.795021057128906,
      "activations/layer15_attention_weight_max": 41.162357330322266,
      "activations/layer15_attention_weight_min": -30.462718963623047,
      "activations/layer16_attention_weight_max": 38.2880973815918,
      "activations/layer16_attention_weight_min": -26.222684860229492,
      "activations/layer17_attention_weight_max": 56.32600402832031,
      "activations/layer17_attention_weight_min": -36.47211837768555,
      "activations/layer18_attention_weight_max": 43.63108825683594,
      "activations/layer18_attention_weight_min": -30.148168563842773,
      "activations/layer19_attention_weight_max": 18.42774772644043,
      "activations/layer19_attention_weight_min": -17.026235580444336,
      "activations/layer1_attention_weight_max": 17.045875549316406,
      "activations/layer1_attention_weight_min": -14.21265983581543,
      "activations/layer20_attention_weight_max": 19.614700317382812,
      "activations/layer20_attention_weight_min": -18.5323543548584,
      "activations/layer21_attention_weight_max": 36.53599166870117,
      "activations/layer21_attention_weight_min": -27.777034759521484,
      "activations/layer22_attention_weight_max": 35.460453033447266,
      "activations/layer22_attention_weight_min": -22.76776695251465,
      "activations/layer23_attention_weight_max": 21.80811309814453,
      "activations/layer23_attention_weight_min": -17.75798225402832,
      "activations/layer2_attention_weight_max": 29.848844528198242,
      "activations/layer2_attention_weight_min": -27.906888961791992,
      "activations/layer3_attention_weight_max": 72.64588165283203,
      "activations/layer3_attention_weight_min": -78.02566528320312,
      "activations/layer4_attention_weight_max": 102.86531829833984,
      "activations/layer4_attention_weight_min": -95.45221710205078,
      "activations/layer5_attention_weight_max": 114.83126831054688,
      "activations/layer5_attention_weight_min": -99.52790832519531,
      "activations/layer6_attention_weight_max": 60.262413024902344,
      "activations/layer6_attention_weight_min": -51.60415267944336,
      "activations/layer7_attention_weight_max": 68.45372772216797,
      "activations/layer7_attention_weight_min": -54.5091552734375,
      "activations/layer8_attention_weight_max": 43.11678695678711,
      "activations/layer8_attention_weight_min": -35.08335876464844,
      "activations/layer9_attention_weight_max": 62.89801788330078,
      "activations/layer9_attention_weight_min": -48.21827697753906,
      "epoch": 3.52,
      "learning_rate": 0.00012862386363636361,
      "loss": 2.8766,
      "step": 60500
    },
    {
      "activations/layer0_attention_weight_max": 13.519871711730957,
      "activations/layer0_attention_weight_min": -14.936990737915039,
      "activations/layer10_attention_weight_max": 45.11323547363281,
      "activations/layer10_attention_weight_min": -35.22685623168945,
      "activations/layer11_attention_weight_max": 38.022979736328125,
      "activations/layer11_attention_weight_min": -26.941211700439453,
      "activations/layer12_attention_weight_max": 23.30768394470215,
      "activations/layer12_attention_weight_min": -27.629060745239258,
      "activations/layer13_attention_weight_max": 49.95021438598633,
      "activations/layer13_attention_weight_min": -35.796844482421875,
      "activations/layer14_attention_weight_max": 45.537418365478516,
      "activations/layer14_attention_weight_min": -31.536853790283203,
      "activations/layer15_attention_weight_max": 48.18307113647461,
      "activations/layer15_attention_weight_min": -30.76361656188965,
      "activations/layer16_attention_weight_max": 37.42910385131836,
      "activations/layer16_attention_weight_min": -25.922452926635742,
      "activations/layer17_attention_weight_max": 62.03855514526367,
      "activations/layer17_attention_weight_min": -36.2491569519043,
      "activations/layer18_attention_weight_max": 55.968074798583984,
      "activations/layer18_attention_weight_min": -30.81507682800293,
      "activations/layer19_attention_weight_max": 25.69563865661621,
      "activations/layer19_attention_weight_min": -17.895132064819336,
      "activations/layer1_attention_weight_max": 18.07176399230957,
      "activations/layer1_attention_weight_min": -13.21347713470459,
      "activations/layer20_attention_weight_max": 23.432931900024414,
      "activations/layer20_attention_weight_min": -18.381839752197266,
      "activations/layer21_attention_weight_max": 47.45579528808594,
      "activations/layer21_attention_weight_min": -28.098783493041992,
      "activations/layer22_attention_weight_max": 42.23206329345703,
      "activations/layer22_attention_weight_min": -22.066598892211914,
      "activations/layer23_attention_weight_max": 29.146024703979492,
      "activations/layer23_attention_weight_min": -17.365577697753906,
      "activations/layer2_attention_weight_max": 31.650959014892578,
      "activations/layer2_attention_weight_min": -28.383298873901367,
      "activations/layer3_attention_weight_max": 78.92665100097656,
      "activations/layer3_attention_weight_min": -78.75608825683594,
      "activations/layer4_attention_weight_max": 109.23177337646484,
      "activations/layer4_attention_weight_min": -97.76280975341797,
      "activations/layer5_attention_weight_max": 121.97240447998047,
      "activations/layer5_attention_weight_min": -106.25625610351562,
      "activations/layer6_attention_weight_max": 61.97038269042969,
      "activations/layer6_attention_weight_min": -55.31302261352539,
      "activations/layer7_attention_weight_max": 62.444427490234375,
      "activations/layer7_attention_weight_min": -57.201148986816406,
      "activations/layer8_attention_weight_max": 44.0367317199707,
      "activations/layer8_attention_weight_min": -38.5528678894043,
      "activations/layer9_attention_weight_max": 51.66398239135742,
      "activations/layer9_attention_weight_min": -45.07245635986328,
      "epoch": 3.52,
      "learning_rate": 0.00012860492424242423,
      "loss": 2.8849,
      "step": 60550
    },
    {
      "activations/layer0_attention_weight_max": 15.535633087158203,
      "activations/layer0_attention_weight_min": -14.79049301147461,
      "activations/layer10_attention_weight_max": 39.86418151855469,
      "activations/layer10_attention_weight_min": -33.15752029418945,
      "activations/layer11_attention_weight_max": 37.66664505004883,
      "activations/layer11_attention_weight_min": -30.96975326538086,
      "activations/layer12_attention_weight_max": 28.223127365112305,
      "activations/layer12_attention_weight_min": -27.057788848876953,
      "activations/layer13_attention_weight_max": 52.51285171508789,
      "activations/layer13_attention_weight_min": -37.39195251464844,
      "activations/layer14_attention_weight_max": 52.18610382080078,
      "activations/layer14_attention_weight_min": -33.01412582397461,
      "activations/layer15_attention_weight_max": 45.18552780151367,
      "activations/layer15_attention_weight_min": -31.097980499267578,
      "activations/layer16_attention_weight_max": 36.204673767089844,
      "activations/layer16_attention_weight_min": -25.429372787475586,
      "activations/layer17_attention_weight_max": 61.31315994262695,
      "activations/layer17_attention_weight_min": -35.02782440185547,
      "activations/layer18_attention_weight_max": 47.03071212768555,
      "activations/layer18_attention_weight_min": -32.864742279052734,
      "activations/layer19_attention_weight_max": 21.758153915405273,
      "activations/layer19_attention_weight_min": -20.75751304626465,
      "activations/layer1_attention_weight_max": 17.52015495300293,
      "activations/layer1_attention_weight_min": -14.115686416625977,
      "activations/layer20_attention_weight_max": 22.2393741607666,
      "activations/layer20_attention_weight_min": -19.501371383666992,
      "activations/layer21_attention_weight_max": 42.54928207397461,
      "activations/layer21_attention_weight_min": -25.609447479248047,
      "activations/layer22_attention_weight_max": 36.369346618652344,
      "activations/layer22_attention_weight_min": -23.091829299926758,
      "activations/layer23_attention_weight_max": 24.759044647216797,
      "activations/layer23_attention_weight_min": -19.357025146484375,
      "activations/layer2_attention_weight_max": 29.392553329467773,
      "activations/layer2_attention_weight_min": -27.426605224609375,
      "activations/layer3_attention_weight_max": 78.06822967529297,
      "activations/layer3_attention_weight_min": -80.3595199584961,
      "activations/layer4_attention_weight_max": 105.50495910644531,
      "activations/layer4_attention_weight_min": -92.4688949584961,
      "activations/layer5_attention_weight_max": 126.74667358398438,
      "activations/layer5_attention_weight_min": -107.50313568115234,
      "activations/layer6_attention_weight_max": 64.0263671875,
      "activations/layer6_attention_weight_min": -54.41631317138672,
      "activations/layer7_attention_weight_max": 80.04460144042969,
      "activations/layer7_attention_weight_min": -67.97698974609375,
      "activations/layer8_attention_weight_max": 46.31611633300781,
      "activations/layer8_attention_weight_min": -38.468692779541016,
      "activations/layer9_attention_weight_max": 68.6213607788086,
      "activations/layer9_attention_weight_min": -50.80425262451172,
      "epoch": 3.52,
      "learning_rate": 0.00012858598484848482,
      "loss": 2.8805,
      "step": 60600
    },
    {
      "activations/layer0_attention_weight_max": 14.614697456359863,
      "activations/layer0_attention_weight_min": -14.564663887023926,
      "activations/layer10_attention_weight_max": 40.04676055908203,
      "activations/layer10_attention_weight_min": -35.851688385009766,
      "activations/layer11_attention_weight_max": 31.22503662109375,
      "activations/layer11_attention_weight_min": -26.557754516601562,
      "activations/layer12_attention_weight_max": 24.586750030517578,
      "activations/layer12_attention_weight_min": -27.653385162353516,
      "activations/layer13_attention_weight_max": 51.9653205871582,
      "activations/layer13_attention_weight_min": -41.5952262878418,
      "activations/layer14_attention_weight_max": 44.04600143432617,
      "activations/layer14_attention_weight_min": -30.799829483032227,
      "activations/layer15_attention_weight_max": 44.61747360229492,
      "activations/layer15_attention_weight_min": -31.026676177978516,
      "activations/layer16_attention_weight_max": 38.37006378173828,
      "activations/layer16_attention_weight_min": -27.841089248657227,
      "activations/layer17_attention_weight_max": 54.22661209106445,
      "activations/layer17_attention_weight_min": -36.27322006225586,
      "activations/layer18_attention_weight_max": 49.7442741394043,
      "activations/layer18_attention_weight_min": -31.342933654785156,
      "activations/layer19_attention_weight_max": 22.066240310668945,
      "activations/layer19_attention_weight_min": -18.34381103515625,
      "activations/layer1_attention_weight_max": 18.60597038269043,
      "activations/layer1_attention_weight_min": -12.779312133789062,
      "activations/layer20_attention_weight_max": 20.37938690185547,
      "activations/layer20_attention_weight_min": -22.926607131958008,
      "activations/layer21_attention_weight_max": 40.07878875732422,
      "activations/layer21_attention_weight_min": -27.931591033935547,
      "activations/layer22_attention_weight_max": 32.18121337890625,
      "activations/layer22_attention_weight_min": -24.46287727355957,
      "activations/layer23_attention_weight_max": 26.96283531188965,
      "activations/layer23_attention_weight_min": -16.337329864501953,
      "activations/layer2_attention_weight_max": 27.141576766967773,
      "activations/layer2_attention_weight_min": -25.054908752441406,
      "activations/layer3_attention_weight_max": 69.98899841308594,
      "activations/layer3_attention_weight_min": -72.26837921142578,
      "activations/layer4_attention_weight_max": 98.20603942871094,
      "activations/layer4_attention_weight_min": -87.34242248535156,
      "activations/layer5_attention_weight_max": 118.39118957519531,
      "activations/layer5_attention_weight_min": -94.52001190185547,
      "activations/layer6_attention_weight_max": 58.50920867919922,
      "activations/layer6_attention_weight_min": -50.82780838012695,
      "activations/layer7_attention_weight_max": 64.88404846191406,
      "activations/layer7_attention_weight_min": -57.0352783203125,
      "activations/layer8_attention_weight_max": 36.965816497802734,
      "activations/layer8_attention_weight_min": -34.83631896972656,
      "activations/layer9_attention_weight_max": 53.74561309814453,
      "activations/layer9_attention_weight_min": -46.37212371826172,
      "epoch": 3.52,
      "learning_rate": 0.00012856704545454543,
      "loss": 2.9002,
      "step": 60650
    },
    {
      "activations/layer0_attention_weight_max": 15.064014434814453,
      "activations/layer0_attention_weight_min": -14.785309791564941,
      "activations/layer10_attention_weight_max": 45.69171905517578,
      "activations/layer10_attention_weight_min": -33.39634323120117,
      "activations/layer11_attention_weight_max": 37.33782958984375,
      "activations/layer11_attention_weight_min": -24.877159118652344,
      "activations/layer12_attention_weight_max": 25.235923767089844,
      "activations/layer12_attention_weight_min": -25.97565460205078,
      "activations/layer13_attention_weight_max": 50.337467193603516,
      "activations/layer13_attention_weight_min": -34.2093505859375,
      "activations/layer14_attention_weight_max": 57.17292785644531,
      "activations/layer14_attention_weight_min": -35.06493377685547,
      "activations/layer15_attention_weight_max": 46.647579193115234,
      "activations/layer15_attention_weight_min": -31.09345245361328,
      "activations/layer16_attention_weight_max": 40.611297607421875,
      "activations/layer16_attention_weight_min": -24.83514404296875,
      "activations/layer17_attention_weight_max": 64.3177719116211,
      "activations/layer17_attention_weight_min": -34.898502349853516,
      "activations/layer18_attention_weight_max": 55.5900993347168,
      "activations/layer18_attention_weight_min": -32.01554489135742,
      "activations/layer19_attention_weight_max": 24.035968780517578,
      "activations/layer19_attention_weight_min": -17.807666778564453,
      "activations/layer1_attention_weight_max": 19.34908103942871,
      "activations/layer1_attention_weight_min": -16.712642669677734,
      "activations/layer20_attention_weight_max": 21.03217124938965,
      "activations/layer20_attention_weight_min": -18.853946685791016,
      "activations/layer21_attention_weight_max": 41.96531295776367,
      "activations/layer21_attention_weight_min": -24.84581756591797,
      "activations/layer22_attention_weight_max": 40.357913970947266,
      "activations/layer22_attention_weight_min": -21.97200584411621,
      "activations/layer23_attention_weight_max": 23.497753143310547,
      "activations/layer23_attention_weight_min": -15.26683235168457,
      "activations/layer2_attention_weight_max": 29.75653648376465,
      "activations/layer2_attention_weight_min": -27.733760833740234,
      "activations/layer3_attention_weight_max": 72.5209732055664,
      "activations/layer3_attention_weight_min": -77.91622924804688,
      "activations/layer4_attention_weight_max": 104.93071746826172,
      "activations/layer4_attention_weight_min": -93.12657928466797,
      "activations/layer5_attention_weight_max": 118.29817962646484,
      "activations/layer5_attention_weight_min": -97.89370727539062,
      "activations/layer6_attention_weight_max": 62.597415924072266,
      "activations/layer6_attention_weight_min": -52.23359680175781,
      "activations/layer7_attention_weight_max": 68.59236907958984,
      "activations/layer7_attention_weight_min": -61.97342300415039,
      "activations/layer8_attention_weight_max": 41.776859283447266,
      "activations/layer8_attention_weight_min": -37.5616340637207,
      "activations/layer9_attention_weight_max": 57.610321044921875,
      "activations/layer9_attention_weight_min": -50.46103286743164,
      "epoch": 3.53,
      "learning_rate": 0.00012854810606060605,
      "loss": 2.8968,
      "step": 60700
    },
    {
      "activations/layer0_attention_weight_max": 15.395061492919922,
      "activations/layer0_attention_weight_min": -14.808984756469727,
      "activations/layer10_attention_weight_max": 46.78605651855469,
      "activations/layer10_attention_weight_min": -36.34168243408203,
      "activations/layer11_attention_weight_max": 37.75680160522461,
      "activations/layer11_attention_weight_min": -29.660568237304688,
      "activations/layer12_attention_weight_max": 25.565692901611328,
      "activations/layer12_attention_weight_min": -32.05337142944336,
      "activations/layer13_attention_weight_max": 69.88917541503906,
      "activations/layer13_attention_weight_min": -37.79171371459961,
      "activations/layer14_attention_weight_max": 75.13854217529297,
      "activations/layer14_attention_weight_min": -37.496402740478516,
      "activations/layer15_attention_weight_max": 51.35579299926758,
      "activations/layer15_attention_weight_min": -30.58131217956543,
      "activations/layer16_attention_weight_max": 48.978023529052734,
      "activations/layer16_attention_weight_min": -26.834196090698242,
      "activations/layer17_attention_weight_max": 64.42472076416016,
      "activations/layer17_attention_weight_min": -35.52582931518555,
      "activations/layer18_attention_weight_max": 52.04010009765625,
      "activations/layer18_attention_weight_min": -32.85237503051758,
      "activations/layer19_attention_weight_max": 23.76930046081543,
      "activations/layer19_attention_weight_min": -17.875043869018555,
      "activations/layer1_attention_weight_max": 17.891530990600586,
      "activations/layer1_attention_weight_min": -13.121206283569336,
      "activations/layer20_attention_weight_max": 24.08086395263672,
      "activations/layer20_attention_weight_min": -20.25165557861328,
      "activations/layer21_attention_weight_max": 59.516510009765625,
      "activations/layer21_attention_weight_min": -27.353954315185547,
      "activations/layer22_attention_weight_max": 34.244537353515625,
      "activations/layer22_attention_weight_min": -23.928178787231445,
      "activations/layer23_attention_weight_max": 23.10637664794922,
      "activations/layer23_attention_weight_min": -19.67259407043457,
      "activations/layer2_attention_weight_max": 29.49649429321289,
      "activations/layer2_attention_weight_min": -28.11507797241211,
      "activations/layer3_attention_weight_max": 75.53730010986328,
      "activations/layer3_attention_weight_min": -79.00737762451172,
      "activations/layer4_attention_weight_max": 102.83910369873047,
      "activations/layer4_attention_weight_min": -92.47063446044922,
      "activations/layer5_attention_weight_max": 122.78572082519531,
      "activations/layer5_attention_weight_min": -102.88865661621094,
      "activations/layer6_attention_weight_max": 62.52439498901367,
      "activations/layer6_attention_weight_min": -55.31317901611328,
      "activations/layer7_attention_weight_max": 66.93081665039062,
      "activations/layer7_attention_weight_min": -61.814971923828125,
      "activations/layer8_attention_weight_max": 41.07816696166992,
      "activations/layer8_attention_weight_min": -38.92072677612305,
      "activations/layer9_attention_weight_max": 57.25431442260742,
      "activations/layer9_attention_weight_min": -50.11640930175781,
      "epoch": 3.53,
      "learning_rate": 0.00012852916666666666,
      "loss": 2.8826,
      "step": 60750
    },
    {
      "activations/layer0_attention_weight_max": 14.191190719604492,
      "activations/layer0_attention_weight_min": -14.830403327941895,
      "activations/layer10_attention_weight_max": 39.840545654296875,
      "activations/layer10_attention_weight_min": -34.34538650512695,
      "activations/layer11_attention_weight_max": 32.556827545166016,
      "activations/layer11_attention_weight_min": -25.96377944946289,
      "activations/layer12_attention_weight_max": 26.3934383392334,
      "activations/layer12_attention_weight_min": -26.19161605834961,
      "activations/layer13_attention_weight_max": 51.33687210083008,
      "activations/layer13_attention_weight_min": -38.50128936767578,
      "activations/layer14_attention_weight_max": 44.66336441040039,
      "activations/layer14_attention_weight_min": -30.43851089477539,
      "activations/layer15_attention_weight_max": 41.5602912902832,
      "activations/layer15_attention_weight_min": -30.845335006713867,
      "activations/layer16_attention_weight_max": 32.59547805786133,
      "activations/layer16_attention_weight_min": -26.93547821044922,
      "activations/layer17_attention_weight_max": 59.59482955932617,
      "activations/layer17_attention_weight_min": -34.51137161254883,
      "activations/layer18_attention_weight_max": 42.58918762207031,
      "activations/layer18_attention_weight_min": -31.5805606842041,
      "activations/layer19_attention_weight_max": 21.610240936279297,
      "activations/layer19_attention_weight_min": -20.180728912353516,
      "activations/layer1_attention_weight_max": 16.933074951171875,
      "activations/layer1_attention_weight_min": -15.997949600219727,
      "activations/layer20_attention_weight_max": 20.557035446166992,
      "activations/layer20_attention_weight_min": -19.358760833740234,
      "activations/layer21_attention_weight_max": 42.33693313598633,
      "activations/layer21_attention_weight_min": -28.047666549682617,
      "activations/layer22_attention_weight_max": 32.320674896240234,
      "activations/layer22_attention_weight_min": -22.779008865356445,
      "activations/layer23_attention_weight_max": 26.54592514038086,
      "activations/layer23_attention_weight_min": -16.62251853942871,
      "activations/layer2_attention_weight_max": 30.1768798828125,
      "activations/layer2_attention_weight_min": -28.488880157470703,
      "activations/layer3_attention_weight_max": 74.68944549560547,
      "activations/layer3_attention_weight_min": -77.63091278076172,
      "activations/layer4_attention_weight_max": 99.12711334228516,
      "activations/layer4_attention_weight_min": -93.40616607666016,
      "activations/layer5_attention_weight_max": 119.59014892578125,
      "activations/layer5_attention_weight_min": -98.21588134765625,
      "activations/layer6_attention_weight_max": 62.85639190673828,
      "activations/layer6_attention_weight_min": -50.410911560058594,
      "activations/layer7_attention_weight_max": 62.30751419067383,
      "activations/layer7_attention_weight_min": -57.80463790893555,
      "activations/layer8_attention_weight_max": 40.151668548583984,
      "activations/layer8_attention_weight_min": -39.6552734375,
      "activations/layer9_attention_weight_max": 51.71868896484375,
      "activations/layer9_attention_weight_min": -45.16172409057617,
      "epoch": 3.53,
      "learning_rate": 0.00012851022727272725,
      "loss": 2.8899,
      "step": 60800
    },
    {
      "activations/layer0_attention_weight_max": 14.188806533813477,
      "activations/layer0_attention_weight_min": -14.849406242370605,
      "activations/layer10_attention_weight_max": 38.813514709472656,
      "activations/layer10_attention_weight_min": -33.670021057128906,
      "activations/layer11_attention_weight_max": 30.31386947631836,
      "activations/layer11_attention_weight_min": -25.600095748901367,
      "activations/layer12_attention_weight_max": 26.17992401123047,
      "activations/layer12_attention_weight_min": -23.3148136138916,
      "activations/layer13_attention_weight_max": 42.34819030761719,
      "activations/layer13_attention_weight_min": -33.23853302001953,
      "activations/layer14_attention_weight_max": 45.10513687133789,
      "activations/layer14_attention_weight_min": -31.258756637573242,
      "activations/layer15_attention_weight_max": 41.43174362182617,
      "activations/layer15_attention_weight_min": -32.39065170288086,
      "activations/layer16_attention_weight_max": 33.26860427856445,
      "activations/layer16_attention_weight_min": -27.214805603027344,
      "activations/layer17_attention_weight_max": 57.14425277709961,
      "activations/layer17_attention_weight_min": -36.64748001098633,
      "activations/layer18_attention_weight_max": 45.72184371948242,
      "activations/layer18_attention_weight_min": -30.78780746459961,
      "activations/layer19_attention_weight_max": 20.490266799926758,
      "activations/layer19_attention_weight_min": -19.080686569213867,
      "activations/layer1_attention_weight_max": 17.363540649414062,
      "activations/layer1_attention_weight_min": -13.822635650634766,
      "activations/layer20_attention_weight_max": 17.788665771484375,
      "activations/layer20_attention_weight_min": -18.2772159576416,
      "activations/layer21_attention_weight_max": 36.8328857421875,
      "activations/layer21_attention_weight_min": -30.349716186523438,
      "activations/layer22_attention_weight_max": 32.616973876953125,
      "activations/layer22_attention_weight_min": -23.821929931640625,
      "activations/layer23_attention_weight_max": 21.840553283691406,
      "activations/layer23_attention_weight_min": -19.136247634887695,
      "activations/layer2_attention_weight_max": 28.37177276611328,
      "activations/layer2_attention_weight_min": -27.080013275146484,
      "activations/layer3_attention_weight_max": 73.64136505126953,
      "activations/layer3_attention_weight_min": -73.18510437011719,
      "activations/layer4_attention_weight_max": 95.18765258789062,
      "activations/layer4_attention_weight_min": -85.0189437866211,
      "activations/layer5_attention_weight_max": 112.1574935913086,
      "activations/layer5_attention_weight_min": -89.94400024414062,
      "activations/layer6_attention_weight_max": 56.171634674072266,
      "activations/layer6_attention_weight_min": -45.567466735839844,
      "activations/layer7_attention_weight_max": 57.77043151855469,
      "activations/layer7_attention_weight_min": -55.134254455566406,
      "activations/layer8_attention_weight_max": 35.546913146972656,
      "activations/layer8_attention_weight_min": -33.20357131958008,
      "activations/layer9_attention_weight_max": 45.54401779174805,
      "activations/layer9_attention_weight_min": -43.29536056518555,
      "epoch": 3.54,
      "learning_rate": 0.00012849128787878787,
      "loss": 2.8926,
      "step": 60850
    },
    {
      "activations/layer0_attention_weight_max": 14.875296592712402,
      "activations/layer0_attention_weight_min": -14.914957046508789,
      "activations/layer10_attention_weight_max": 42.075233459472656,
      "activations/layer10_attention_weight_min": -34.402645111083984,
      "activations/layer11_attention_weight_max": 32.38031005859375,
      "activations/layer11_attention_weight_min": -24.990802764892578,
      "activations/layer12_attention_weight_max": 26.68024444580078,
      "activations/layer12_attention_weight_min": -26.627395629882812,
      "activations/layer13_attention_weight_max": 62.18223190307617,
      "activations/layer13_attention_weight_min": -37.24359893798828,
      "activations/layer14_attention_weight_max": 45.12325668334961,
      "activations/layer14_attention_weight_min": -27.06578254699707,
      "activations/layer15_attention_weight_max": 43.21760559082031,
      "activations/layer15_attention_weight_min": -31.535001754760742,
      "activations/layer16_attention_weight_max": 37.11203384399414,
      "activations/layer16_attention_weight_min": -27.035676956176758,
      "activations/layer17_attention_weight_max": 57.89021301269531,
      "activations/layer17_attention_weight_min": -38.16060256958008,
      "activations/layer18_attention_weight_max": 48.99718475341797,
      "activations/layer18_attention_weight_min": -30.823223114013672,
      "activations/layer19_attention_weight_max": 19.870681762695312,
      "activations/layer19_attention_weight_min": -17.576520919799805,
      "activations/layer1_attention_weight_max": 17.817394256591797,
      "activations/layer1_attention_weight_min": -15.223857879638672,
      "activations/layer20_attention_weight_max": 21.02492904663086,
      "activations/layer20_attention_weight_min": -19.65964126586914,
      "activations/layer21_attention_weight_max": 39.09727478027344,
      "activations/layer21_attention_weight_min": -27.417552947998047,
      "activations/layer22_attention_weight_max": 38.80290985107422,
      "activations/layer22_attention_weight_min": -23.94472312927246,
      "activations/layer23_attention_weight_max": 24.71828269958496,
      "activations/layer23_attention_weight_min": -17.927270889282227,
      "activations/layer2_attention_weight_max": 30.25790023803711,
      "activations/layer2_attention_weight_min": -29.451566696166992,
      "activations/layer3_attention_weight_max": 74.0455093383789,
      "activations/layer3_attention_weight_min": -77.61898803710938,
      "activations/layer4_attention_weight_max": 101.59955596923828,
      "activations/layer4_attention_weight_min": -92.00358581542969,
      "activations/layer5_attention_weight_max": 118.69117736816406,
      "activations/layer5_attention_weight_min": -95.28441619873047,
      "activations/layer6_attention_weight_max": 59.419281005859375,
      "activations/layer6_attention_weight_min": -49.76044464111328,
      "activations/layer7_attention_weight_max": 65.37802124023438,
      "activations/layer7_attention_weight_min": -56.94562530517578,
      "activations/layer8_attention_weight_max": 37.37092971801758,
      "activations/layer8_attention_weight_min": -34.649723052978516,
      "activations/layer9_attention_weight_max": 52.72748947143555,
      "activations/layer9_attention_weight_min": -45.05841827392578,
      "epoch": 3.54,
      "learning_rate": 0.00012847234848484848,
      "loss": 2.8789,
      "step": 60900
    },
    {
      "activations/layer0_attention_weight_max": 15.673059463500977,
      "activations/layer0_attention_weight_min": -15.136611938476562,
      "activations/layer10_attention_weight_max": 43.20499801635742,
      "activations/layer10_attention_weight_min": -34.464542388916016,
      "activations/layer11_attention_weight_max": 34.15311813354492,
      "activations/layer11_attention_weight_min": -25.815872192382812,
      "activations/layer12_attention_weight_max": 29.058815002441406,
      "activations/layer12_attention_weight_min": -24.22150230407715,
      "activations/layer13_attention_weight_max": 70.27565002441406,
      "activations/layer13_attention_weight_min": -42.440208435058594,
      "activations/layer14_attention_weight_max": 58.31622314453125,
      "activations/layer14_attention_weight_min": -35.71879959106445,
      "activations/layer15_attention_weight_max": 43.59085464477539,
      "activations/layer15_attention_weight_min": -31.479686737060547,
      "activations/layer16_attention_weight_max": 38.2237434387207,
      "activations/layer16_attention_weight_min": -25.841630935668945,
      "activations/layer17_attention_weight_max": 58.98542022705078,
      "activations/layer17_attention_weight_min": -36.813594818115234,
      "activations/layer18_attention_weight_max": 55.33364486694336,
      "activations/layer18_attention_weight_min": -34.437889099121094,
      "activations/layer19_attention_weight_max": 24.26881217956543,
      "activations/layer19_attention_weight_min": -17.705480575561523,
      "activations/layer1_attention_weight_max": 17.10167694091797,
      "activations/layer1_attention_weight_min": -13.645732879638672,
      "activations/layer20_attention_weight_max": 21.392606735229492,
      "activations/layer20_attention_weight_min": -19.90188980102539,
      "activations/layer21_attention_weight_max": 47.890647888183594,
      "activations/layer21_attention_weight_min": -32.2411994934082,
      "activations/layer22_attention_weight_max": 35.41659164428711,
      "activations/layer22_attention_weight_min": -25.20179557800293,
      "activations/layer23_attention_weight_max": 29.60106658935547,
      "activations/layer23_attention_weight_min": -16.75711441040039,
      "activations/layer2_attention_weight_max": 28.617591857910156,
      "activations/layer2_attention_weight_min": -26.66449546813965,
      "activations/layer3_attention_weight_max": 71.38339233398438,
      "activations/layer3_attention_weight_min": -72.25326538085938,
      "activations/layer4_attention_weight_max": 96.50789642333984,
      "activations/layer4_attention_weight_min": -88.169921875,
      "activations/layer5_attention_weight_max": 113.85073852539062,
      "activations/layer5_attention_weight_min": -97.427978515625,
      "activations/layer6_attention_weight_max": 57.32743835449219,
      "activations/layer6_attention_weight_min": -48.55372619628906,
      "activations/layer7_attention_weight_max": 61.111454010009766,
      "activations/layer7_attention_weight_min": -57.80039978027344,
      "activations/layer8_attention_weight_max": 39.28937911987305,
      "activations/layer8_attention_weight_min": -35.49447250366211,
      "activations/layer9_attention_weight_max": 50.054718017578125,
      "activations/layer9_attention_weight_min": -45.65509796142578,
      "epoch": 3.54,
      "learning_rate": 0.00012845340909090907,
      "loss": 2.8881,
      "step": 60950
    },
    {
      "activations/layer0_attention_weight_max": 14.038488388061523,
      "activations/layer0_attention_weight_min": -15.388142585754395,
      "activations/layer10_attention_weight_max": 43.31166458129883,
      "activations/layer10_attention_weight_min": -34.818241119384766,
      "activations/layer11_attention_weight_max": 32.421478271484375,
      "activations/layer11_attention_weight_min": -27.116058349609375,
      "activations/layer12_attention_weight_max": 36.18830108642578,
      "activations/layer12_attention_weight_min": -25.883716583251953,
      "activations/layer13_attention_weight_max": 50.44245147705078,
      "activations/layer13_attention_weight_min": -37.41359329223633,
      "activations/layer14_attention_weight_max": 53.712432861328125,
      "activations/layer14_attention_weight_min": -36.511199951171875,
      "activations/layer15_attention_weight_max": 50.63758850097656,
      "activations/layer15_attention_weight_min": -31.58318519592285,
      "activations/layer16_attention_weight_max": 35.32538986206055,
      "activations/layer16_attention_weight_min": -28.54362678527832,
      "activations/layer17_attention_weight_max": 58.19418716430664,
      "activations/layer17_attention_weight_min": -39.48799133300781,
      "activations/layer18_attention_weight_max": 48.65500259399414,
      "activations/layer18_attention_weight_min": -31.305217742919922,
      "activations/layer19_attention_weight_max": 22.867586135864258,
      "activations/layer19_attention_weight_min": -16.871768951416016,
      "activations/layer1_attention_weight_max": 17.410114288330078,
      "activations/layer1_attention_weight_min": -15.441580772399902,
      "activations/layer20_attention_weight_max": 21.2940673828125,
      "activations/layer20_attention_weight_min": -18.908262252807617,
      "activations/layer21_attention_weight_max": 40.95631790161133,
      "activations/layer21_attention_weight_min": -27.97050666809082,
      "activations/layer22_attention_weight_max": 33.71797561645508,
      "activations/layer22_attention_weight_min": -23.198368072509766,
      "activations/layer23_attention_weight_max": 27.432754516601562,
      "activations/layer23_attention_weight_min": -16.559326171875,
      "activations/layer2_attention_weight_max": 30.30109214782715,
      "activations/layer2_attention_weight_min": -27.30977439880371,
      "activations/layer3_attention_weight_max": 77.46533966064453,
      "activations/layer3_attention_weight_min": -79.14842987060547,
      "activations/layer4_attention_weight_max": 102.34561157226562,
      "activations/layer4_attention_weight_min": -96.17471313476562,
      "activations/layer5_attention_weight_max": 117.8746337890625,
      "activations/layer5_attention_weight_min": -96.964599609375,
      "activations/layer6_attention_weight_max": 57.96013641357422,
      "activations/layer6_attention_weight_min": -50.6742057800293,
      "activations/layer7_attention_weight_max": 63.1043586730957,
      "activations/layer7_attention_weight_min": -61.47272872924805,
      "activations/layer8_attention_weight_max": 44.375064849853516,
      "activations/layer8_attention_weight_min": -36.4632453918457,
      "activations/layer9_attention_weight_max": 56.138973236083984,
      "activations/layer9_attention_weight_min": -49.722381591796875,
      "epoch": 3.54,
      "learning_rate": 0.00012843446969696968,
      "loss": 2.8994,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_loss": 2.83984375,
      "eval_runtime": 8.4174,
      "eval_samples_per_second": 510.133,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_openwebtext_loss": 2.83984375,
      "eval_openwebtext_ppl": 17.113091407702502,
      "eval_openwebtext_runtime": 8.4174,
      "eval_openwebtext_samples_per_second": 510.133,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_wikitext_loss": 3.08984375,
      "eval_wikitext_ppl": 21.973644325590612,
      "eval_wikitext_runtime": 1.9348,
      "eval_wikitext_samples_per_second": 235.682,
      "step": 61000
    },
    {
      "epoch": 3.54,
      "eval_lambada_loss": 2.8828125,
      "eval_lambada_ppl": 17.864446345871784,
      "eval_lambada_runtime": 9.5348,
      "eval_lambada_samples_per_second": 510.654,
      "step": 61000
    },
    {
      "activations/layer0_attention_weight_max": 14.282266616821289,
      "activations/layer0_attention_weight_min": -14.25161361694336,
      "activations/layer10_attention_weight_max": 49.69551467895508,
      "activations/layer10_attention_weight_min": -39.59497833251953,
      "activations/layer11_attention_weight_max": 44.55914306640625,
      "activations/layer11_attention_weight_min": -29.498533248901367,
      "activations/layer12_attention_weight_max": 32.78711700439453,
      "activations/layer12_attention_weight_min": -22.222429275512695,
      "activations/layer13_attention_weight_max": 55.198333740234375,
      "activations/layer13_attention_weight_min": -37.545310974121094,
      "activations/layer14_attention_weight_max": 59.122779846191406,
      "activations/layer14_attention_weight_min": -33.86707305908203,
      "activations/layer15_attention_weight_max": 50.45419692993164,
      "activations/layer15_attention_weight_min": -30.304569244384766,
      "activations/layer16_attention_weight_max": 41.67203903198242,
      "activations/layer16_attention_weight_min": -26.75285530090332,
      "activations/layer17_attention_weight_max": 57.64932632446289,
      "activations/layer17_attention_weight_min": -35.595947265625,
      "activations/layer18_attention_weight_max": 57.02902603149414,
      "activations/layer18_attention_weight_min": -30.448604583740234,
      "activations/layer19_attention_weight_max": 21.754924774169922,
      "activations/layer19_attention_weight_min": -18.472270965576172,
      "activations/layer1_attention_weight_max": 18.238771438598633,
      "activations/layer1_attention_weight_min": -13.57945442199707,
      "activations/layer20_attention_weight_max": 19.186513900756836,
      "activations/layer20_attention_weight_min": -21.143142700195312,
      "activations/layer21_attention_weight_max": 43.87092971801758,
      "activations/layer21_attention_weight_min": -25.13641357421875,
      "activations/layer22_attention_weight_max": 42.19720458984375,
      "activations/layer22_attention_weight_min": -21.98430824279785,
      "activations/layer23_attention_weight_max": 24.75787353515625,
      "activations/layer23_attention_weight_min": -15.938613891601562,
      "activations/layer2_attention_weight_max": 30.153318405151367,
      "activations/layer2_attention_weight_min": -29.98310089111328,
      "activations/layer3_attention_weight_max": 79.0748519897461,
      "activations/layer3_attention_weight_min": -78.72815704345703,
      "activations/layer4_attention_weight_max": 107.86878967285156,
      "activations/layer4_attention_weight_min": -102.26472473144531,
      "activations/layer5_attention_weight_max": 135.939208984375,
      "activations/layer5_attention_weight_min": -110.73104095458984,
      "activations/layer6_attention_weight_max": 68.84373474121094,
      "activations/layer6_attention_weight_min": -57.96860122680664,
      "activations/layer7_attention_weight_max": 79.99862670898438,
      "activations/layer7_attention_weight_min": -64.43811798095703,
      "activations/layer8_attention_weight_max": 53.24320602416992,
      "activations/layer8_attention_weight_min": -39.66670227050781,
      "activations/layer9_attention_weight_max": 74.18344116210938,
      "activations/layer9_attention_weight_min": -48.8088264465332,
      "epoch": 3.55,
      "learning_rate": 0.0001284155303030303,
      "loss": 2.8833,
      "step": 61050
    },
    {
      "activations/layer0_attention_weight_max": 14.168248176574707,
      "activations/layer0_attention_weight_min": -15.026496887207031,
      "activations/layer10_attention_weight_max": 43.09659957885742,
      "activations/layer10_attention_weight_min": -33.69677734375,
      "activations/layer11_attention_weight_max": 35.35830307006836,
      "activations/layer11_attention_weight_min": -26.847654342651367,
      "activations/layer12_attention_weight_max": 29.348169326782227,
      "activations/layer12_attention_weight_min": -24.979896545410156,
      "activations/layer13_attention_weight_max": 57.08195877075195,
      "activations/layer13_attention_weight_min": -39.13542175292969,
      "activations/layer14_attention_weight_max": 45.58574295043945,
      "activations/layer14_attention_weight_min": -30.271669387817383,
      "activations/layer15_attention_weight_max": 47.157588958740234,
      "activations/layer15_attention_weight_min": -32.267913818359375,
      "activations/layer16_attention_weight_max": 38.59259033203125,
      "activations/layer16_attention_weight_min": -26.285724639892578,
      "activations/layer17_attention_weight_max": 59.18665313720703,
      "activations/layer17_attention_weight_min": -36.07861328125,
      "activations/layer18_attention_weight_max": 51.81589126586914,
      "activations/layer18_attention_weight_min": -32.26358413696289,
      "activations/layer19_attention_weight_max": 21.030961990356445,
      "activations/layer19_attention_weight_min": -18.57917594909668,
      "activations/layer1_attention_weight_max": 18.326309204101562,
      "activations/layer1_attention_weight_min": -15.20727252960205,
      "activations/layer20_attention_weight_max": 19.6673641204834,
      "activations/layer20_attention_weight_min": -20.366575241088867,
      "activations/layer21_attention_weight_max": 41.7306022644043,
      "activations/layer21_attention_weight_min": -28.662370681762695,
      "activations/layer22_attention_weight_max": 38.93373107910156,
      "activations/layer22_attention_weight_min": -23.65009880065918,
      "activations/layer23_attention_weight_max": 27.358428955078125,
      "activations/layer23_attention_weight_min": -16.2805118560791,
      "activations/layer2_attention_weight_max": 28.485395431518555,
      "activations/layer2_attention_weight_min": -27.526290893554688,
      "activations/layer3_attention_weight_max": 72.4815673828125,
      "activations/layer3_attention_weight_min": -77.962158203125,
      "activations/layer4_attention_weight_max": 97.89608001708984,
      "activations/layer4_attention_weight_min": -91.34639739990234,
      "activations/layer5_attention_weight_max": 112.34283447265625,
      "activations/layer5_attention_weight_min": -100.13111877441406,
      "activations/layer6_attention_weight_max": 58.72894287109375,
      "activations/layer6_attention_weight_min": -53.19245147705078,
      "activations/layer7_attention_weight_max": 63.173343658447266,
      "activations/layer7_attention_weight_min": -58.37699890136719,
      "activations/layer8_attention_weight_max": 38.764225006103516,
      "activations/layer8_attention_weight_min": -36.54648971557617,
      "activations/layer9_attention_weight_max": 50.142860412597656,
      "activations/layer9_attention_weight_min": -43.820247650146484,
      "epoch": 3.55,
      "learning_rate": 0.00012839659090909091,
      "loss": 2.8843,
      "step": 61100
    },
    {
      "activations/layer0_attention_weight_max": 13.804886817932129,
      "activations/layer0_attention_weight_min": -14.71741008758545,
      "activations/layer10_attention_weight_max": 51.24170684814453,
      "activations/layer10_attention_weight_min": -40.172813415527344,
      "activations/layer11_attention_weight_max": 45.68865966796875,
      "activations/layer11_attention_weight_min": -30.866300582885742,
      "activations/layer12_attention_weight_max": 24.970884323120117,
      "activations/layer12_attention_weight_min": -23.55160903930664,
      "activations/layer13_attention_weight_max": 85.04327392578125,
      "activations/layer13_attention_weight_min": -43.83683776855469,
      "activations/layer14_attention_weight_max": 103.06531524658203,
      "activations/layer14_attention_weight_min": -49.137516021728516,
      "activations/layer15_attention_weight_max": 70.9985122680664,
      "activations/layer15_attention_weight_min": -38.31732177734375,
      "activations/layer16_attention_weight_max": 40.892120361328125,
      "activations/layer16_attention_weight_min": -26.221546173095703,
      "activations/layer17_attention_weight_max": 61.373046875,
      "activations/layer17_attention_weight_min": -33.624610900878906,
      "activations/layer18_attention_weight_max": 56.376953125,
      "activations/layer18_attention_weight_min": -30.080114364624023,
      "activations/layer19_attention_weight_max": 22.58235740661621,
      "activations/layer19_attention_weight_min": -16.431467056274414,
      "activations/layer1_attention_weight_max": 17.11546516418457,
      "activations/layer1_attention_weight_min": -13.462693214416504,
      "activations/layer20_attention_weight_max": 22.30278205871582,
      "activations/layer20_attention_weight_min": -20.03649139404297,
      "activations/layer21_attention_weight_max": 40.19332504272461,
      "activations/layer21_attention_weight_min": -26.6907958984375,
      "activations/layer22_attention_weight_max": 32.564449310302734,
      "activations/layer22_attention_weight_min": -22.89276885986328,
      "activations/layer23_attention_weight_max": 25.091064453125,
      "activations/layer23_attention_weight_min": -14.577807426452637,
      "activations/layer2_attention_weight_max": 30.792404174804688,
      "activations/layer2_attention_weight_min": -27.5872802734375,
      "activations/layer3_attention_weight_max": 71.58942413330078,
      "activations/layer3_attention_weight_min": -75.66996002197266,
      "activations/layer4_attention_weight_max": 98.64225006103516,
      "activations/layer4_attention_weight_min": -89.31651306152344,
      "activations/layer5_attention_weight_max": 120.83287048339844,
      "activations/layer5_attention_weight_min": -104.28538513183594,
      "activations/layer6_attention_weight_max": 62.81543731689453,
      "activations/layer6_attention_weight_min": -56.045188903808594,
      "activations/layer7_attention_weight_max": 91.33399200439453,
      "activations/layer7_attention_weight_min": -58.938232421875,
      "activations/layer8_attention_weight_max": 58.24853515625,
      "activations/layer8_attention_weight_min": -40.61003875732422,
      "activations/layer9_attention_weight_max": 80.49004364013672,
      "activations/layer9_attention_weight_min": -55.849159240722656,
      "epoch": 3.55,
      "learning_rate": 0.0001283776515151515,
      "loss": 2.888,
      "step": 61150
    },
    {
      "activations/layer0_attention_weight_max": 14.394871711730957,
      "activations/layer0_attention_weight_min": -15.28409194946289,
      "activations/layer10_attention_weight_max": 37.0140380859375,
      "activations/layer10_attention_weight_min": -33.398223876953125,
      "activations/layer11_attention_weight_max": 32.922237396240234,
      "activations/layer11_attention_weight_min": -26.591291427612305,
      "activations/layer12_attention_weight_max": 21.038482666015625,
      "activations/layer12_attention_weight_min": -28.90610694885254,
      "activations/layer13_attention_weight_max": 49.837791442871094,
      "activations/layer13_attention_weight_min": -38.3134880065918,
      "activations/layer14_attention_weight_max": 48.918216705322266,
      "activations/layer14_attention_weight_min": -33.158851623535156,
      "activations/layer15_attention_weight_max": 39.438812255859375,
      "activations/layer15_attention_weight_min": -30.27212142944336,
      "activations/layer16_attention_weight_max": 33.72528076171875,
      "activations/layer16_attention_weight_min": -24.080127716064453,
      "activations/layer17_attention_weight_max": 59.8013916015625,
      "activations/layer17_attention_weight_min": -34.64723587036133,
      "activations/layer18_attention_weight_max": 44.806739807128906,
      "activations/layer18_attention_weight_min": -30.274080276489258,
      "activations/layer19_attention_weight_max": 20.472761154174805,
      "activations/layer19_attention_weight_min": -16.539724349975586,
      "activations/layer1_attention_weight_max": 19.251094818115234,
      "activations/layer1_attention_weight_min": -14.882928848266602,
      "activations/layer20_attention_weight_max": 19.12877655029297,
      "activations/layer20_attention_weight_min": -23.177757263183594,
      "activations/layer21_attention_weight_max": 34.878021240234375,
      "activations/layer21_attention_weight_min": -26.675939559936523,
      "activations/layer22_attention_weight_max": 31.341630935668945,
      "activations/layer22_attention_weight_min": -20.83405303955078,
      "activations/layer23_attention_weight_max": 21.756681442260742,
      "activations/layer23_attention_weight_min": -14.76401138305664,
      "activations/layer2_attention_weight_max": 27.75677490234375,
      "activations/layer2_attention_weight_min": -26.42757797241211,
      "activations/layer3_attention_weight_max": 76.24579620361328,
      "activations/layer3_attention_weight_min": -76.95411682128906,
      "activations/layer4_attention_weight_max": 111.31279754638672,
      "activations/layer4_attention_weight_min": -96.92829132080078,
      "activations/layer5_attention_weight_max": 116.45295715332031,
      "activations/layer5_attention_weight_min": -104.00570678710938,
      "activations/layer6_attention_weight_max": 55.45478820800781,
      "activations/layer6_attention_weight_min": -50.03474044799805,
      "activations/layer7_attention_weight_max": 64.60712432861328,
      "activations/layer7_attention_weight_min": -59.87200164794922,
      "activations/layer8_attention_weight_max": 38.27859115600586,
      "activations/layer8_attention_weight_min": -34.28407669067383,
      "activations/layer9_attention_weight_max": 54.612152099609375,
      "activations/layer9_attention_weight_min": -50.17961120605469,
      "epoch": 3.56,
      "learning_rate": 0.00012835871212121212,
      "loss": 2.8918,
      "step": 61200
    },
    {
      "activations/layer0_attention_weight_max": 14.46033763885498,
      "activations/layer0_attention_weight_min": -15.615564346313477,
      "activations/layer10_attention_weight_max": 38.047428131103516,
      "activations/layer10_attention_weight_min": -32.34757614135742,
      "activations/layer11_attention_weight_max": 32.4301643371582,
      "activations/layer11_attention_weight_min": -27.636005401611328,
      "activations/layer12_attention_weight_max": 22.21273422241211,
      "activations/layer12_attention_weight_min": -26.248659133911133,
      "activations/layer13_attention_weight_max": 50.170692443847656,
      "activations/layer13_attention_weight_min": -38.60597229003906,
      "activations/layer14_attention_weight_max": 45.31255340576172,
      "activations/layer14_attention_weight_min": -31.397138595581055,
      "activations/layer15_attention_weight_max": 46.33893585205078,
      "activations/layer15_attention_weight_min": -32.02176284790039,
      "activations/layer16_attention_weight_max": 37.67890167236328,
      "activations/layer16_attention_weight_min": -26.677927017211914,
      "activations/layer17_attention_weight_max": 56.40178298950195,
      "activations/layer17_attention_weight_min": -34.48722839355469,
      "activations/layer18_attention_weight_max": 46.96684646606445,
      "activations/layer18_attention_weight_min": -31.919282913208008,
      "activations/layer19_attention_weight_max": 20.354787826538086,
      "activations/layer19_attention_weight_min": -17.413074493408203,
      "activations/layer1_attention_weight_max": 16.846004486083984,
      "activations/layer1_attention_weight_min": -17.090730667114258,
      "activations/layer20_attention_weight_max": 21.589576721191406,
      "activations/layer20_attention_weight_min": -19.193357467651367,
      "activations/layer21_attention_weight_max": 40.00935363769531,
      "activations/layer21_attention_weight_min": -24.496227264404297,
      "activations/layer22_attention_weight_max": 36.31874465942383,
      "activations/layer22_attention_weight_min": -21.820816040039062,
      "activations/layer23_attention_weight_max": 22.542739868164062,
      "activations/layer23_attention_weight_min": -16.81108856201172,
      "activations/layer2_attention_weight_max": 29.517730712890625,
      "activations/layer2_attention_weight_min": -27.85626220703125,
      "activations/layer3_attention_weight_max": 72.16230773925781,
      "activations/layer3_attention_weight_min": -78.1369400024414,
      "activations/layer4_attention_weight_max": 93.7047348022461,
      "activations/layer4_attention_weight_min": -88.45166778564453,
      "activations/layer5_attention_weight_max": 111.8970947265625,
      "activations/layer5_attention_weight_min": -99.03892517089844,
      "activations/layer6_attention_weight_max": 55.9464225769043,
      "activations/layer6_attention_weight_min": -50.76591873168945,
      "activations/layer7_attention_weight_max": 62.07890701293945,
      "activations/layer7_attention_weight_min": -56.22343826293945,
      "activations/layer8_attention_weight_max": 39.461544036865234,
      "activations/layer8_attention_weight_min": -36.90012741088867,
      "activations/layer9_attention_weight_max": 50.79649353027344,
      "activations/layer9_attention_weight_min": -48.902713775634766,
      "epoch": 3.56,
      "learning_rate": 0.00012833977272727273,
      "loss": 2.8891,
      "step": 61250
    },
    {
      "activations/layer0_attention_weight_max": 14.142667770385742,
      "activations/layer0_attention_weight_min": -15.240899085998535,
      "activations/layer10_attention_weight_max": 45.0648193359375,
      "activations/layer10_attention_weight_min": -32.974605560302734,
      "activations/layer11_attention_weight_max": 38.047767639160156,
      "activations/layer11_attention_weight_min": -26.14450454711914,
      "activations/layer12_attention_weight_max": 24.880502700805664,
      "activations/layer12_attention_weight_min": -24.237533569335938,
      "activations/layer13_attention_weight_max": 69.13196563720703,
      "activations/layer13_attention_weight_min": -45.270999908447266,
      "activations/layer14_attention_weight_max": 46.599788665771484,
      "activations/layer14_attention_weight_min": -36.89948272705078,
      "activations/layer15_attention_weight_max": 51.85638427734375,
      "activations/layer15_attention_weight_min": -32.35785675048828,
      "activations/layer16_attention_weight_max": 41.13431167602539,
      "activations/layer16_attention_weight_min": -27.044363021850586,
      "activations/layer17_attention_weight_max": 61.53053665161133,
      "activations/layer17_attention_weight_min": -36.65828323364258,
      "activations/layer18_attention_weight_max": 55.80231857299805,
      "activations/layer18_attention_weight_min": -31.19536018371582,
      "activations/layer19_attention_weight_max": 26.68326187133789,
      "activations/layer19_attention_weight_min": -17.292306900024414,
      "activations/layer1_attention_weight_max": 18.41996955871582,
      "activations/layer1_attention_weight_min": -15.457592010498047,
      "activations/layer20_attention_weight_max": 23.66132926940918,
      "activations/layer20_attention_weight_min": -19.518766403198242,
      "activations/layer21_attention_weight_max": 40.44367218017578,
      "activations/layer21_attention_weight_min": -25.841533660888672,
      "activations/layer22_attention_weight_max": 34.6889762878418,
      "activations/layer22_attention_weight_min": -22.64510154724121,
      "activations/layer23_attention_weight_max": 26.926467895507812,
      "activations/layer23_attention_weight_min": -16.012428283691406,
      "activations/layer2_attention_weight_max": 29.461830139160156,
      "activations/layer2_attention_weight_min": -26.550827026367188,
      "activations/layer3_attention_weight_max": 70.9095230102539,
      "activations/layer3_attention_weight_min": -76.07687377929688,
      "activations/layer4_attention_weight_max": 99.10214233398438,
      "activations/layer4_attention_weight_min": -88.64738464355469,
      "activations/layer5_attention_weight_max": 114.68742370605469,
      "activations/layer5_attention_weight_min": -94.78893280029297,
      "activations/layer6_attention_weight_max": 58.0649528503418,
      "activations/layer6_attention_weight_min": -49.41713333129883,
      "activations/layer7_attention_weight_max": 59.3892707824707,
      "activations/layer7_attention_weight_min": -60.68611526489258,
      "activations/layer8_attention_weight_max": 38.901126861572266,
      "activations/layer8_attention_weight_min": -36.43025588989258,
      "activations/layer9_attention_weight_max": 50.654457092285156,
      "activations/layer9_attention_weight_min": -48.24435043334961,
      "epoch": 3.56,
      "learning_rate": 0.00012832083333333332,
      "loss": 2.8869,
      "step": 61300
    },
    {
      "activations/layer0_attention_weight_max": 14.337627410888672,
      "activations/layer0_attention_weight_min": -14.292195320129395,
      "activations/layer10_attention_weight_max": 43.616455078125,
      "activations/layer10_attention_weight_min": -34.128028869628906,
      "activations/layer11_attention_weight_max": 34.75077819824219,
      "activations/layer11_attention_weight_min": -28.316221237182617,
      "activations/layer12_attention_weight_max": 28.40444564819336,
      "activations/layer12_attention_weight_min": -23.845890045166016,
      "activations/layer13_attention_weight_max": 60.70096206665039,
      "activations/layer13_attention_weight_min": -36.600242614746094,
      "activations/layer14_attention_weight_max": 57.04899978637695,
      "activations/layer14_attention_weight_min": -32.52028274536133,
      "activations/layer15_attention_weight_max": 54.81230545043945,
      "activations/layer15_attention_weight_min": -31.754192352294922,
      "activations/layer16_attention_weight_max": 39.30551528930664,
      "activations/layer16_attention_weight_min": -26.8459529876709,
      "activations/layer17_attention_weight_max": 68.6893081665039,
      "activations/layer17_attention_weight_min": -37.32686233520508,
      "activations/layer18_attention_weight_max": 55.34077072143555,
      "activations/layer18_attention_weight_min": -33.255924224853516,
      "activations/layer19_attention_weight_max": 25.81357765197754,
      "activations/layer19_attention_weight_min": -17.189138412475586,
      "activations/layer1_attention_weight_max": 17.809280395507812,
      "activations/layer1_attention_weight_min": -17.75031089782715,
      "activations/layer20_attention_weight_max": 24.033796310424805,
      "activations/layer20_attention_weight_min": -18.65150260925293,
      "activations/layer21_attention_weight_max": 44.34309768676758,
      "activations/layer21_attention_weight_min": -28.883708953857422,
      "activations/layer22_attention_weight_max": 38.2003059387207,
      "activations/layer22_attention_weight_min": -25.409027099609375,
      "activations/layer23_attention_weight_max": 27.101722717285156,
      "activations/layer23_attention_weight_min": -15.722061157226562,
      "activations/layer2_attention_weight_max": 30.776416778564453,
      "activations/layer2_attention_weight_min": -28.92770767211914,
      "activations/layer3_attention_weight_max": 74.69879150390625,
      "activations/layer3_attention_weight_min": -78.3895034790039,
      "activations/layer4_attention_weight_max": 99.70952606201172,
      "activations/layer4_attention_weight_min": -90.28337860107422,
      "activations/layer5_attention_weight_max": 115.46664428710938,
      "activations/layer5_attention_weight_min": -104.59446716308594,
      "activations/layer6_attention_weight_max": 61.342864990234375,
      "activations/layer6_attention_weight_min": -55.34230422973633,
      "activations/layer7_attention_weight_max": 62.963680267333984,
      "activations/layer7_attention_weight_min": -56.77873611450195,
      "activations/layer8_attention_weight_max": 41.78140640258789,
      "activations/layer8_attention_weight_min": -37.04648971557617,
      "activations/layer9_attention_weight_max": 51.13260269165039,
      "activations/layer9_attention_weight_min": -46.44906234741211,
      "epoch": 3.56,
      "learning_rate": 0.00012830189393939393,
      "loss": 2.8862,
      "step": 61350
    },
    {
      "activations/layer0_attention_weight_max": 15.176993370056152,
      "activations/layer0_attention_weight_min": -15.122793197631836,
      "activations/layer10_attention_weight_max": 43.8415412902832,
      "activations/layer10_attention_weight_min": -33.73977279663086,
      "activations/layer11_attention_weight_max": 36.75616455078125,
      "activations/layer11_attention_weight_min": -25.466419219970703,
      "activations/layer12_attention_weight_max": 32.81224060058594,
      "activations/layer12_attention_weight_min": -24.599546432495117,
      "activations/layer13_attention_weight_max": 51.775901794433594,
      "activations/layer13_attention_weight_min": -36.939353942871094,
      "activations/layer14_attention_weight_max": 51.709320068359375,
      "activations/layer14_attention_weight_min": -29.009708404541016,
      "activations/layer15_attention_weight_max": 51.35390090942383,
      "activations/layer15_attention_weight_min": -33.8334846496582,
      "activations/layer16_attention_weight_max": 38.00114440917969,
      "activations/layer16_attention_weight_min": -26.912668228149414,
      "activations/layer17_attention_weight_max": 68.31648254394531,
      "activations/layer17_attention_weight_min": -34.97025680541992,
      "activations/layer18_attention_weight_max": 53.321266174316406,
      "activations/layer18_attention_weight_min": -32.89219284057617,
      "activations/layer19_attention_weight_max": 24.703899383544922,
      "activations/layer19_attention_weight_min": -18.90373992919922,
      "activations/layer1_attention_weight_max": 18.741409301757812,
      "activations/layer1_attention_weight_min": -15.067849159240723,
      "activations/layer20_attention_weight_max": 22.452960968017578,
      "activations/layer20_attention_weight_min": -20.413658142089844,
      "activations/layer21_attention_weight_max": 43.972530364990234,
      "activations/layer21_attention_weight_min": -30.033910751342773,
      "activations/layer22_attention_weight_max": 34.3786735534668,
      "activations/layer22_attention_weight_min": -23.264068603515625,
      "activations/layer23_attention_weight_max": 26.20070457458496,
      "activations/layer23_attention_weight_min": -16.294124603271484,
      "activations/layer2_attention_weight_max": 29.557662963867188,
      "activations/layer2_attention_weight_min": -28.360248565673828,
      "activations/layer3_attention_weight_max": 70.88212585449219,
      "activations/layer3_attention_weight_min": -75.75790405273438,
      "activations/layer4_attention_weight_max": 99.50899505615234,
      "activations/layer4_attention_weight_min": -93.17517852783203,
      "activations/layer5_attention_weight_max": 116.87093353271484,
      "activations/layer5_attention_weight_min": -97.29598236083984,
      "activations/layer6_attention_weight_max": 61.06084060668945,
      "activations/layer6_attention_weight_min": -53.52034378051758,
      "activations/layer7_attention_weight_max": 60.80419921875,
      "activations/layer7_attention_weight_min": -59.25712585449219,
      "activations/layer8_attention_weight_max": 40.5212287902832,
      "activations/layer8_attention_weight_min": -37.24961853027344,
      "activations/layer9_attention_weight_max": 52.71778106689453,
      "activations/layer9_attention_weight_min": -46.79241943359375,
      "epoch": 3.57,
      "learning_rate": 0.00012828295454545455,
      "loss": 2.8794,
      "step": 61400
    },
    {
      "activations/layer0_attention_weight_max": 13.613814353942871,
      "activations/layer0_attention_weight_min": -14.742694854736328,
      "activations/layer10_attention_weight_max": 41.7683219909668,
      "activations/layer10_attention_weight_min": -32.703765869140625,
      "activations/layer11_attention_weight_max": 31.969593048095703,
      "activations/layer11_attention_weight_min": -25.109609603881836,
      "activations/layer12_attention_weight_max": 23.716020584106445,
      "activations/layer12_attention_weight_min": -28.236509323120117,
      "activations/layer13_attention_weight_max": 55.34385681152344,
      "activations/layer13_attention_weight_min": -43.41522979736328,
      "activations/layer14_attention_weight_max": 46.46092224121094,
      "activations/layer14_attention_weight_min": -35.81870651245117,
      "activations/layer15_attention_weight_max": 46.52595901489258,
      "activations/layer15_attention_weight_min": -32.06169891357422,
      "activations/layer16_attention_weight_max": 43.997806549072266,
      "activations/layer16_attention_weight_min": -26.217241287231445,
      "activations/layer17_attention_weight_max": 66.71733093261719,
      "activations/layer17_attention_weight_min": -35.158939361572266,
      "activations/layer18_attention_weight_max": 48.0308837890625,
      "activations/layer18_attention_weight_min": -30.27235221862793,
      "activations/layer19_attention_weight_max": 23.405059814453125,
      "activations/layer19_attention_weight_min": -18.129087448120117,
      "activations/layer1_attention_weight_max": 18.881608963012695,
      "activations/layer1_attention_weight_min": -14.725160598754883,
      "activations/layer20_attention_weight_max": 19.692798614501953,
      "activations/layer20_attention_weight_min": -18.705501556396484,
      "activations/layer21_attention_weight_max": 39.12031173706055,
      "activations/layer21_attention_weight_min": -29.599212646484375,
      "activations/layer22_attention_weight_max": 35.653255462646484,
      "activations/layer22_attention_weight_min": -24.138139724731445,
      "activations/layer23_attention_weight_max": 22.515140533447266,
      "activations/layer23_attention_weight_min": -19.15915870666504,
      "activations/layer2_attention_weight_max": 29.254240036010742,
      "activations/layer2_attention_weight_min": -28.006750106811523,
      "activations/layer3_attention_weight_max": 76.55328369140625,
      "activations/layer3_attention_weight_min": -80.33643341064453,
      "activations/layer4_attention_weight_max": 101.8613510131836,
      "activations/layer4_attention_weight_min": -89.97118377685547,
      "activations/layer5_attention_weight_max": 110.80584716796875,
      "activations/layer5_attention_weight_min": -102.25213623046875,
      "activations/layer6_attention_weight_max": 56.720245361328125,
      "activations/layer6_attention_weight_min": -51.795902252197266,
      "activations/layer7_attention_weight_max": 67.40365600585938,
      "activations/layer7_attention_weight_min": -58.73391342163086,
      "activations/layer8_attention_weight_max": 41.78327178955078,
      "activations/layer8_attention_weight_min": -35.875675201416016,
      "activations/layer9_attention_weight_max": 62.11467361450195,
      "activations/layer9_attention_weight_min": -50.09767532348633,
      "epoch": 3.57,
      "learning_rate": 0.00012826439393939394,
      "loss": 2.8909,
      "step": 61450
    },
    {
      "activations/layer0_attention_weight_max": 14.03331184387207,
      "activations/layer0_attention_weight_min": -14.888071060180664,
      "activations/layer10_attention_weight_max": 39.35182571411133,
      "activations/layer10_attention_weight_min": -35.62287902832031,
      "activations/layer11_attention_weight_max": 34.18629837036133,
      "activations/layer11_attention_weight_min": -27.559864044189453,
      "activations/layer12_attention_weight_max": 35.202232360839844,
      "activations/layer12_attention_weight_min": -29.162229537963867,
      "activations/layer13_attention_weight_max": 64.86107635498047,
      "activations/layer13_attention_weight_min": -39.07447814941406,
      "activations/layer14_attention_weight_max": 61.103519439697266,
      "activations/layer14_attention_weight_min": -35.22304153442383,
      "activations/layer15_attention_weight_max": 45.12541198730469,
      "activations/layer15_attention_weight_min": -32.764671325683594,
      "activations/layer16_attention_weight_max": 35.43837356567383,
      "activations/layer16_attention_weight_min": -26.886707305908203,
      "activations/layer17_attention_weight_max": 55.571800231933594,
      "activations/layer17_attention_weight_min": -37.09856033325195,
      "activations/layer18_attention_weight_max": 49.241336822509766,
      "activations/layer18_attention_weight_min": -32.829559326171875,
      "activations/layer19_attention_weight_max": 23.956369400024414,
      "activations/layer19_attention_weight_min": -16.901674270629883,
      "activations/layer1_attention_weight_max": 17.547643661499023,
      "activations/layer1_attention_weight_min": -15.232648849487305,
      "activations/layer20_attention_weight_max": 20.082321166992188,
      "activations/layer20_attention_weight_min": -17.654739379882812,
      "activations/layer21_attention_weight_max": 36.37590026855469,
      "activations/layer21_attention_weight_min": -28.655712127685547,
      "activations/layer22_attention_weight_max": 33.7164421081543,
      "activations/layer22_attention_weight_min": -23.46485710144043,
      "activations/layer23_attention_weight_max": 26.99588394165039,
      "activations/layer23_attention_weight_min": -19.018596649169922,
      "activations/layer2_attention_weight_max": 28.46735191345215,
      "activations/layer2_attention_weight_min": -27.35151481628418,
      "activations/layer3_attention_weight_max": 75.81742858886719,
      "activations/layer3_attention_weight_min": -78.05181121826172,
      "activations/layer4_attention_weight_max": 99.60084533691406,
      "activations/layer4_attention_weight_min": -94.35396575927734,
      "activations/layer5_attention_weight_max": 123.91729736328125,
      "activations/layer5_attention_weight_min": -104.45208740234375,
      "activations/layer6_attention_weight_max": 60.54936599731445,
      "activations/layer6_attention_weight_min": -51.573890686035156,
      "activations/layer7_attention_weight_max": 65.35590362548828,
      "activations/layer7_attention_weight_min": -62.26667022705078,
      "activations/layer8_attention_weight_max": 41.55312728881836,
      "activations/layer8_attention_weight_min": -36.773155212402344,
      "activations/layer9_attention_weight_max": 54.75616455078125,
      "activations/layer9_attention_weight_min": -52.139892578125,
      "epoch": 3.57,
      "learning_rate": 0.00012824545454545453,
      "loss": 2.8965,
      "step": 61500
    },
    {
      "activations/layer0_attention_weight_max": 13.777302742004395,
      "activations/layer0_attention_weight_min": -15.004040718078613,
      "activations/layer10_attention_weight_max": 37.652191162109375,
      "activations/layer10_attention_weight_min": -31.86662483215332,
      "activations/layer11_attention_weight_max": 31.45759391784668,
      "activations/layer11_attention_weight_min": -25.641963958740234,
      "activations/layer12_attention_weight_max": 26.27895736694336,
      "activations/layer12_attention_weight_min": -25.00387954711914,
      "activations/layer13_attention_weight_max": 47.13471603393555,
      "activations/layer13_attention_weight_min": -37.35466003417969,
      "activations/layer14_attention_weight_max": 40.27644348144531,
      "activations/layer14_attention_weight_min": -30.613924026489258,
      "activations/layer15_attention_weight_max": 44.326324462890625,
      "activations/layer15_attention_weight_min": -31.28159523010254,
      "activations/layer16_attention_weight_max": 39.94953536987305,
      "activations/layer16_attention_weight_min": -26.936113357543945,
      "activations/layer17_attention_weight_max": 61.45455551147461,
      "activations/layer17_attention_weight_min": -37.858551025390625,
      "activations/layer18_attention_weight_max": 55.11101531982422,
      "activations/layer18_attention_weight_min": -30.34282112121582,
      "activations/layer19_attention_weight_max": 19.62982177734375,
      "activations/layer19_attention_weight_min": -18.609121322631836,
      "activations/layer1_attention_weight_max": 18.393491744995117,
      "activations/layer1_attention_weight_min": -14.259784698486328,
      "activations/layer20_attention_weight_max": 21.09243392944336,
      "activations/layer20_attention_weight_min": -20.733915328979492,
      "activations/layer21_attention_weight_max": 35.451698303222656,
      "activations/layer21_attention_weight_min": -26.59987449645996,
      "activations/layer22_attention_weight_max": 31.70899772644043,
      "activations/layer22_attention_weight_min": -22.320039749145508,
      "activations/layer23_attention_weight_max": 26.003170013427734,
      "activations/layer23_attention_weight_min": -16.99654769897461,
      "activations/layer2_attention_weight_max": 29.565502166748047,
      "activations/layer2_attention_weight_min": -26.44571304321289,
      "activations/layer3_attention_weight_max": 71.33421325683594,
      "activations/layer3_attention_weight_min": -72.33956909179688,
      "activations/layer4_attention_weight_max": 97.75733184814453,
      "activations/layer4_attention_weight_min": -88.00469207763672,
      "activations/layer5_attention_weight_max": 113.19670104980469,
      "activations/layer5_attention_weight_min": -92.22417449951172,
      "activations/layer6_attention_weight_max": 58.01116943359375,
      "activations/layer6_attention_weight_min": -48.370540618896484,
      "activations/layer7_attention_weight_max": 64.44281005859375,
      "activations/layer7_attention_weight_min": -58.30686950683594,
      "activations/layer8_attention_weight_max": 38.455970764160156,
      "activations/layer8_attention_weight_min": -33.50029373168945,
      "activations/layer9_attention_weight_max": 47.4207649230957,
      "activations/layer9_attention_weight_min": -42.825172424316406,
      "epoch": 3.58,
      "learning_rate": 0.00012822651515151514,
      "loss": 2.8783,
      "step": 61550
    },
    {
      "activations/layer0_attention_weight_max": 14.123208045959473,
      "activations/layer0_attention_weight_min": -15.003539085388184,
      "activations/layer10_attention_weight_max": 40.966552734375,
      "activations/layer10_attention_weight_min": -32.87421798706055,
      "activations/layer11_attention_weight_max": 35.30534744262695,
      "activations/layer11_attention_weight_min": -27.955955505371094,
      "activations/layer12_attention_weight_max": 26.653520584106445,
      "activations/layer12_attention_weight_min": -24.434228897094727,
      "activations/layer13_attention_weight_max": 51.85626983642578,
      "activations/layer13_attention_weight_min": -39.1257438659668,
      "activations/layer14_attention_weight_max": 53.26544189453125,
      "activations/layer14_attention_weight_min": -38.2410774230957,
      "activations/layer15_attention_weight_max": 45.0412712097168,
      "activations/layer15_attention_weight_min": -34.04344177246094,
      "activations/layer16_attention_weight_max": 39.63151931762695,
      "activations/layer16_attention_weight_min": -25.25701332092285,
      "activations/layer17_attention_weight_max": 56.12046432495117,
      "activations/layer17_attention_weight_min": -36.26045608520508,
      "activations/layer18_attention_weight_max": 52.22210693359375,
      "activations/layer18_attention_weight_min": -31.60012435913086,
      "activations/layer19_attention_weight_max": 21.286026000976562,
      "activations/layer19_attention_weight_min": -22.143178939819336,
      "activations/layer1_attention_weight_max": 17.383718490600586,
      "activations/layer1_attention_weight_min": -16.49966812133789,
      "activations/layer20_attention_weight_max": 20.245603561401367,
      "activations/layer20_attention_weight_min": -18.660533905029297,
      "activations/layer21_attention_weight_max": 41.45442199707031,
      "activations/layer21_attention_weight_min": -25.442583084106445,
      "activations/layer22_attention_weight_max": 33.809288024902344,
      "activations/layer22_attention_weight_min": -22.2725887298584,
      "activations/layer23_attention_weight_max": 24.985456466674805,
      "activations/layer23_attention_weight_min": -17.55065155029297,
      "activations/layer2_attention_weight_max": 28.975536346435547,
      "activations/layer2_attention_weight_min": -27.9278564453125,
      "activations/layer3_attention_weight_max": 70.62200927734375,
      "activations/layer3_attention_weight_min": -76.73889923095703,
      "activations/layer4_attention_weight_max": 96.07921600341797,
      "activations/layer4_attention_weight_min": -88.18568420410156,
      "activations/layer5_attention_weight_max": 115.40216064453125,
      "activations/layer5_attention_weight_min": -100.8714828491211,
      "activations/layer6_attention_weight_max": 57.95635223388672,
      "activations/layer6_attention_weight_min": -51.66746139526367,
      "activations/layer7_attention_weight_max": 58.95598602294922,
      "activations/layer7_attention_weight_min": -55.99626541137695,
      "activations/layer8_attention_weight_max": 38.6011848449707,
      "activations/layer8_attention_weight_min": -38.57575225830078,
      "activations/layer9_attention_weight_max": 49.692657470703125,
      "activations/layer9_attention_weight_min": -43.99594497680664,
      "epoch": 3.58,
      "learning_rate": 0.00012820757575757576,
      "loss": 2.8926,
      "step": 61600
    },
    {
      "activations/layer0_attention_weight_max": 14.947369575500488,
      "activations/layer0_attention_weight_min": -14.542511940002441,
      "activations/layer10_attention_weight_max": 43.08223342895508,
      "activations/layer10_attention_weight_min": -34.06907653808594,
      "activations/layer11_attention_weight_max": 29.67624282836914,
      "activations/layer11_attention_weight_min": -25.333885192871094,
      "activations/layer12_attention_weight_max": 25.413761138916016,
      "activations/layer12_attention_weight_min": -24.230119705200195,
      "activations/layer13_attention_weight_max": 50.21721649169922,
      "activations/layer13_attention_weight_min": -35.79801559448242,
      "activations/layer14_attention_weight_max": 48.83985137939453,
      "activations/layer14_attention_weight_min": -33.88779830932617,
      "activations/layer15_attention_weight_max": 40.70690155029297,
      "activations/layer15_attention_weight_min": -29.733043670654297,
      "activations/layer16_attention_weight_max": 36.37937927246094,
      "activations/layer16_attention_weight_min": -26.67523765563965,
      "activations/layer17_attention_weight_max": 54.5928840637207,
      "activations/layer17_attention_weight_min": -34.293270111083984,
      "activations/layer18_attention_weight_max": 50.6550407409668,
      "activations/layer18_attention_weight_min": -32.02326965332031,
      "activations/layer19_attention_weight_max": 23.396991729736328,
      "activations/layer19_attention_weight_min": -18.441844940185547,
      "activations/layer1_attention_weight_max": 18.1513671875,
      "activations/layer1_attention_weight_min": -16.14735984802246,
      "activations/layer20_attention_weight_max": 21.348251342773438,
      "activations/layer20_attention_weight_min": -19.32495880126953,
      "activations/layer21_attention_weight_max": 39.289222717285156,
      "activations/layer21_attention_weight_min": -28.258769989013672,
      "activations/layer22_attention_weight_max": 32.661861419677734,
      "activations/layer22_attention_weight_min": -23.452579498291016,
      "activations/layer23_attention_weight_max": 26.395614624023438,
      "activations/layer23_attention_weight_min": -18.70145606994629,
      "activations/layer2_attention_weight_max": 28.16737937927246,
      "activations/layer2_attention_weight_min": -28.079547882080078,
      "activations/layer3_attention_weight_max": 74.27265930175781,
      "activations/layer3_attention_weight_min": -81.82894897460938,
      "activations/layer4_attention_weight_max": 103.0667495727539,
      "activations/layer4_attention_weight_min": -96.41827392578125,
      "activations/layer5_attention_weight_max": 120.26152038574219,
      "activations/layer5_attention_weight_min": -105.96994018554688,
      "activations/layer6_attention_weight_max": 59.09406280517578,
      "activations/layer6_attention_weight_min": -54.50908279418945,
      "activations/layer7_attention_weight_max": 60.9460334777832,
      "activations/layer7_attention_weight_min": -60.83039093017578,
      "activations/layer8_attention_weight_max": 39.124717712402344,
      "activations/layer8_attention_weight_min": -36.82194137573242,
      "activations/layer9_attention_weight_max": 46.99148178100586,
      "activations/layer9_attention_weight_min": -48.67558670043945,
      "epoch": 3.58,
      "learning_rate": 0.00012818863636363635,
      "loss": 2.8966,
      "step": 61650
    },
    {
      "activations/layer0_attention_weight_max": 15.793304443359375,
      "activations/layer0_attention_weight_min": -14.68076229095459,
      "activations/layer10_attention_weight_max": 44.675758361816406,
      "activations/layer10_attention_weight_min": -35.21163558959961,
      "activations/layer11_attention_weight_max": 38.284847259521484,
      "activations/layer11_attention_weight_min": -27.783658981323242,
      "activations/layer12_attention_weight_max": 25.97092056274414,
      "activations/layer12_attention_weight_min": -29.724824905395508,
      "activations/layer13_attention_weight_max": 51.81769943237305,
      "activations/layer13_attention_weight_min": -34.024898529052734,
      "activations/layer14_attention_weight_max": 49.87297439575195,
      "activations/layer14_attention_weight_min": -33.76323318481445,
      "activations/layer15_attention_weight_max": 47.9970703125,
      "activations/layer15_attention_weight_min": -32.73680877685547,
      "activations/layer16_attention_weight_max": 50.640525817871094,
      "activations/layer16_attention_weight_min": -27.13309669494629,
      "activations/layer17_attention_weight_max": 60.75338363647461,
      "activations/layer17_attention_weight_min": -38.581504821777344,
      "activations/layer18_attention_weight_max": 54.132164001464844,
      "activations/layer18_attention_weight_min": -32.733150482177734,
      "activations/layer19_attention_weight_max": 29.802549362182617,
      "activations/layer19_attention_weight_min": -18.57593536376953,
      "activations/layer1_attention_weight_max": 18.81337547302246,
      "activations/layer1_attention_weight_min": -14.16401195526123,
      "activations/layer20_attention_weight_max": 23.104093551635742,
      "activations/layer20_attention_weight_min": -20.856508255004883,
      "activations/layer21_attention_weight_max": 51.60618591308594,
      "activations/layer21_attention_weight_min": -29.377561569213867,
      "activations/layer22_attention_weight_max": 44.1967887878418,
      "activations/layer22_attention_weight_min": -25.717323303222656,
      "activations/layer23_attention_weight_max": 28.69936180114746,
      "activations/layer23_attention_weight_min": -18.716110229492188,
      "activations/layer2_attention_weight_max": 28.40254020690918,
      "activations/layer2_attention_weight_min": -26.549205780029297,
      "activations/layer3_attention_weight_max": 72.89922332763672,
      "activations/layer3_attention_weight_min": -76.0887451171875,
      "activations/layer4_attention_weight_max": 100.11358642578125,
      "activations/layer4_attention_weight_min": -92.20415496826172,
      "activations/layer5_attention_weight_max": 117.49113464355469,
      "activations/layer5_attention_weight_min": -101.36419677734375,
      "activations/layer6_attention_weight_max": 61.36418533325195,
      "activations/layer6_attention_weight_min": -53.473758697509766,
      "activations/layer7_attention_weight_max": 69.7999267578125,
      "activations/layer7_attention_weight_min": -60.08156204223633,
      "activations/layer8_attention_weight_max": 43.130332946777344,
      "activations/layer8_attention_weight_min": -39.9005012512207,
      "activations/layer9_attention_weight_max": 55.290130615234375,
      "activations/layer9_attention_weight_min": -47.97687530517578,
      "epoch": 3.59,
      "learning_rate": 0.00012816969696969696,
      "loss": 2.8958,
      "step": 61700
    },
    {
      "activations/layer0_attention_weight_max": 13.999911308288574,
      "activations/layer0_attention_weight_min": -14.920022010803223,
      "activations/layer10_attention_weight_max": 38.79704284667969,
      "activations/layer10_attention_weight_min": -34.18489456176758,
      "activations/layer11_attention_weight_max": 32.96440887451172,
      "activations/layer11_attention_weight_min": -27.826513290405273,
      "activations/layer12_attention_weight_max": 28.39606475830078,
      "activations/layer12_attention_weight_min": -24.8631534576416,
      "activations/layer13_attention_weight_max": 52.83620834350586,
      "activations/layer13_attention_weight_min": -37.95957946777344,
      "activations/layer14_attention_weight_max": 40.70490264892578,
      "activations/layer14_attention_weight_min": -29.717214584350586,
      "activations/layer15_attention_weight_max": 46.40753173828125,
      "activations/layer15_attention_weight_min": -30.996204376220703,
      "activations/layer16_attention_weight_max": 37.98422622680664,
      "activations/layer16_attention_weight_min": -26.955930709838867,
      "activations/layer17_attention_weight_max": 55.40754699707031,
      "activations/layer17_attention_weight_min": -37.38334274291992,
      "activations/layer18_attention_weight_max": 49.548797607421875,
      "activations/layer18_attention_weight_min": -32.12746810913086,
      "activations/layer19_attention_weight_max": 21.159622192382812,
      "activations/layer19_attention_weight_min": -17.81977081298828,
      "activations/layer1_attention_weight_max": 16.842641830444336,
      "activations/layer1_attention_weight_min": -14.001152992248535,
      "activations/layer20_attention_weight_max": 21.56224822998047,
      "activations/layer20_attention_weight_min": -20.49665069580078,
      "activations/layer21_attention_weight_max": 44.382450103759766,
      "activations/layer21_attention_weight_min": -28.851030349731445,
      "activations/layer22_attention_weight_max": 33.69572830200195,
      "activations/layer22_attention_weight_min": -21.902416229248047,
      "activations/layer23_attention_weight_max": 28.543790817260742,
      "activations/layer23_attention_weight_min": -15.261472702026367,
      "activations/layer2_attention_weight_max": 28.414051055908203,
      "activations/layer2_attention_weight_min": -27.8107967376709,
      "activations/layer3_attention_weight_max": 75.68785095214844,
      "activations/layer3_attention_weight_min": -79.94859313964844,
      "activations/layer4_attention_weight_max": 102.70336151123047,
      "activations/layer4_attention_weight_min": -91.83618927001953,
      "activations/layer5_attention_weight_max": 115.39573669433594,
      "activations/layer5_attention_weight_min": -103.68461608886719,
      "activations/layer6_attention_weight_max": 58.832183837890625,
      "activations/layer6_attention_weight_min": -52.42116928100586,
      "activations/layer7_attention_weight_max": 67.00348663330078,
      "activations/layer7_attention_weight_min": -60.73978042602539,
      "activations/layer8_attention_weight_max": 40.334259033203125,
      "activations/layer8_attention_weight_min": -40.7287712097168,
      "activations/layer9_attention_weight_max": 53.42964553833008,
      "activations/layer9_attention_weight_min": -47.436092376708984,
      "epoch": 3.59,
      "learning_rate": 0.00012815075757575755,
      "loss": 2.8768,
      "step": 61750
    },
    {
      "activations/layer0_attention_weight_max": 15.020543098449707,
      "activations/layer0_attention_weight_min": -15.089086532592773,
      "activations/layer10_attention_weight_max": 45.427642822265625,
      "activations/layer10_attention_weight_min": -34.81145477294922,
      "activations/layer11_attention_weight_max": 34.6479377746582,
      "activations/layer11_attention_weight_min": -26.710447311401367,
      "activations/layer12_attention_weight_max": 24.386764526367188,
      "activations/layer12_attention_weight_min": -24.778507232666016,
      "activations/layer13_attention_weight_max": 58.98021697998047,
      "activations/layer13_attention_weight_min": -40.89800262451172,
      "activations/layer14_attention_weight_max": 50.51801300048828,
      "activations/layer14_attention_weight_min": -32.335384368896484,
      "activations/layer15_attention_weight_max": 52.27892303466797,
      "activations/layer15_attention_weight_min": -33.29072570800781,
      "activations/layer16_attention_weight_max": 36.8017692565918,
      "activations/layer16_attention_weight_min": -27.247631072998047,
      "activations/layer17_attention_weight_max": 58.94057083129883,
      "activations/layer17_attention_weight_min": -35.72548294067383,
      "activations/layer18_attention_weight_max": 49.020843505859375,
      "activations/layer18_attention_weight_min": -31.27912712097168,
      "activations/layer19_attention_weight_max": 20.74915885925293,
      "activations/layer19_attention_weight_min": -19.725149154663086,
      "activations/layer1_attention_weight_max": 17.728071212768555,
      "activations/layer1_attention_weight_min": -15.733758926391602,
      "activations/layer20_attention_weight_max": 21.44624900817871,
      "activations/layer20_attention_weight_min": -18.833532333374023,
      "activations/layer21_attention_weight_max": 40.251625061035156,
      "activations/layer21_attention_weight_min": -29.788114547729492,
      "activations/layer22_attention_weight_max": 34.73897933959961,
      "activations/layer22_attention_weight_min": -22.036724090576172,
      "activations/layer23_attention_weight_max": 28.884559631347656,
      "activations/layer23_attention_weight_min": -16.55561637878418,
      "activations/layer2_attention_weight_max": 30.927709579467773,
      "activations/layer2_attention_weight_min": -29.234886169433594,
      "activations/layer3_attention_weight_max": 77.75255584716797,
      "activations/layer3_attention_weight_min": -80.13005065917969,
      "activations/layer4_attention_weight_max": 105.98418426513672,
      "activations/layer4_attention_weight_min": -97.49987030029297,
      "activations/layer5_attention_weight_max": 123.19615936279297,
      "activations/layer5_attention_weight_min": -102.01454162597656,
      "activations/layer6_attention_weight_max": 64.80111694335938,
      "activations/layer6_attention_weight_min": -51.734230041503906,
      "activations/layer7_attention_weight_max": 65.96957397460938,
      "activations/layer7_attention_weight_min": -64.41088104248047,
      "activations/layer8_attention_weight_max": 44.207157135009766,
      "activations/layer8_attention_weight_min": -39.43840408325195,
      "activations/layer9_attention_weight_max": 56.102596282958984,
      "activations/layer9_attention_weight_min": -50.38410186767578,
      "epoch": 3.59,
      "learning_rate": 0.00012813181818181816,
      "loss": 2.8977,
      "step": 61800
    },
    {
      "activations/layer0_attention_weight_max": 14.368804931640625,
      "activations/layer0_attention_weight_min": -15.011747360229492,
      "activations/layer10_attention_weight_max": 44.81943130493164,
      "activations/layer10_attention_weight_min": -37.42146682739258,
      "activations/layer11_attention_weight_max": 38.64884948730469,
      "activations/layer11_attention_weight_min": -27.6939697265625,
      "activations/layer12_attention_weight_max": 34.43404006958008,
      "activations/layer12_attention_weight_min": -26.413888931274414,
      "activations/layer13_attention_weight_max": 57.849517822265625,
      "activations/layer13_attention_weight_min": -39.31200408935547,
      "activations/layer14_attention_weight_max": 53.967464447021484,
      "activations/layer14_attention_weight_min": -36.51569366455078,
      "activations/layer15_attention_weight_max": 49.192970275878906,
      "activations/layer15_attention_weight_min": -34.11581802368164,
      "activations/layer16_attention_weight_max": 42.69500732421875,
      "activations/layer16_attention_weight_min": -26.177331924438477,
      "activations/layer17_attention_weight_max": 65.61019134521484,
      "activations/layer17_attention_weight_min": -37.08622741699219,
      "activations/layer18_attention_weight_max": 60.47635269165039,
      "activations/layer18_attention_weight_min": -34.720245361328125,
      "activations/layer19_attention_weight_max": 23.582115173339844,
      "activations/layer19_attention_weight_min": -18.678300857543945,
      "activations/layer1_attention_weight_max": 17.45201873779297,
      "activations/layer1_attention_weight_min": -17.37405014038086,
      "activations/layer20_attention_weight_max": 23.06101417541504,
      "activations/layer20_attention_weight_min": -20.22893524169922,
      "activations/layer21_attention_weight_max": 48.738521575927734,
      "activations/layer21_attention_weight_min": -27.96581268310547,
      "activations/layer22_attention_weight_max": 44.3719367980957,
      "activations/layer22_attention_weight_min": -23.71810531616211,
      "activations/layer23_attention_weight_max": 32.107139587402344,
      "activations/layer23_attention_weight_min": -17.280052185058594,
      "activations/layer2_attention_weight_max": 28.82813835144043,
      "activations/layer2_attention_weight_min": -29.214874267578125,
      "activations/layer3_attention_weight_max": 74.24673461914062,
      "activations/layer3_attention_weight_min": -77.18275451660156,
      "activations/layer4_attention_weight_max": 102.89681243896484,
      "activations/layer4_attention_weight_min": -93.35533905029297,
      "activations/layer5_attention_weight_max": 121.67108154296875,
      "activations/layer5_attention_weight_min": -104.32618713378906,
      "activations/layer6_attention_weight_max": 59.9517936706543,
      "activations/layer6_attention_weight_min": -52.37753677368164,
      "activations/layer7_attention_weight_max": 65.8639144897461,
      "activations/layer7_attention_weight_min": -58.409996032714844,
      "activations/layer8_attention_weight_max": 39.895896911621094,
      "activations/layer8_attention_weight_min": -38.54161071777344,
      "activations/layer9_attention_weight_max": 57.21148681640625,
      "activations/layer9_attention_weight_min": -47.54187774658203,
      "epoch": 3.59,
      "learning_rate": 0.00012811287878787878,
      "loss": 2.8641,
      "step": 61850
    },
    {
      "activations/layer0_attention_weight_max": 16.035280227661133,
      "activations/layer0_attention_weight_min": -14.625214576721191,
      "activations/layer10_attention_weight_max": 48.894283294677734,
      "activations/layer10_attention_weight_min": -33.882320404052734,
      "activations/layer11_attention_weight_max": 34.46537780761719,
      "activations/layer11_attention_weight_min": -26.298015594482422,
      "activations/layer12_attention_weight_max": 27.385507583618164,
      "activations/layer12_attention_weight_min": -22.399641036987305,
      "activations/layer13_attention_weight_max": 58.99209976196289,
      "activations/layer13_attention_weight_min": -39.916595458984375,
      "activations/layer14_attention_weight_max": 51.263031005859375,
      "activations/layer14_attention_weight_min": -33.59552764892578,
      "activations/layer15_attention_weight_max": 50.67771530151367,
      "activations/layer15_attention_weight_min": -32.081138610839844,
      "activations/layer16_attention_weight_max": 39.2296257019043,
      "activations/layer16_attention_weight_min": -25.912017822265625,
      "activations/layer17_attention_weight_max": 60.42962646484375,
      "activations/layer17_attention_weight_min": -35.97319030761719,
      "activations/layer18_attention_weight_max": 52.4388427734375,
      "activations/layer18_attention_weight_min": -32.8565788269043,
      "activations/layer19_attention_weight_max": 26.562286376953125,
      "activations/layer19_attention_weight_min": -18.769306182861328,
      "activations/layer1_attention_weight_max": 18.106876373291016,
      "activations/layer1_attention_weight_min": -16.06312370300293,
      "activations/layer20_attention_weight_max": 22.686260223388672,
      "activations/layer20_attention_weight_min": -18.245811462402344,
      "activations/layer21_attention_weight_max": 38.27178955078125,
      "activations/layer21_attention_weight_min": -27.339900970458984,
      "activations/layer22_attention_weight_max": 36.5255126953125,
      "activations/layer22_attention_weight_min": -21.858545303344727,
      "activations/layer23_attention_weight_max": 27.763874053955078,
      "activations/layer23_attention_weight_min": -15.489779472351074,
      "activations/layer2_attention_weight_max": 31.1981258392334,
      "activations/layer2_attention_weight_min": -28.100719451904297,
      "activations/layer3_attention_weight_max": 73.23391723632812,
      "activations/layer3_attention_weight_min": -78.32249450683594,
      "activations/layer4_attention_weight_max": 100.85018157958984,
      "activations/layer4_attention_weight_min": -92.39801025390625,
      "activations/layer5_attention_weight_max": 116.58318328857422,
      "activations/layer5_attention_weight_min": -98.24774169921875,
      "activations/layer6_attention_weight_max": 61.444847106933594,
      "activations/layer6_attention_weight_min": -51.855918884277344,
      "activations/layer7_attention_weight_max": 62.5334358215332,
      "activations/layer7_attention_weight_min": -61.65830993652344,
      "activations/layer8_attention_weight_max": 40.828914642333984,
      "activations/layer8_attention_weight_min": -37.78648376464844,
      "activations/layer9_attention_weight_max": 55.51030349731445,
      "activations/layer9_attention_weight_min": -46.911285400390625,
      "epoch": 3.6,
      "learning_rate": 0.00012809393939393937,
      "loss": 2.8803,
      "step": 61900
    },
    {
      "activations/layer0_attention_weight_max": 13.93746566772461,
      "activations/layer0_attention_weight_min": -15.059600830078125,
      "activations/layer10_attention_weight_max": 44.903011322021484,
      "activations/layer10_attention_weight_min": -34.618717193603516,
      "activations/layer11_attention_weight_max": 39.050071716308594,
      "activations/layer11_attention_weight_min": -25.558517456054688,
      "activations/layer12_attention_weight_max": 26.857990264892578,
      "activations/layer12_attention_weight_min": -24.31271743774414,
      "activations/layer13_attention_weight_max": 59.9022102355957,
      "activations/layer13_attention_weight_min": -38.32646942138672,
      "activations/layer14_attention_weight_max": 62.88141632080078,
      "activations/layer14_attention_weight_min": -35.18410873413086,
      "activations/layer15_attention_weight_max": 51.48036575317383,
      "activations/layer15_attention_weight_min": -32.3785514831543,
      "activations/layer16_attention_weight_max": 38.880889892578125,
      "activations/layer16_attention_weight_min": -25.435741424560547,
      "activations/layer17_attention_weight_max": 64.19673919677734,
      "activations/layer17_attention_weight_min": -40.45289611816406,
      "activations/layer18_attention_weight_max": 50.099266052246094,
      "activations/layer18_attention_weight_min": -34.0941276550293,
      "activations/layer19_attention_weight_max": 25.68759536743164,
      "activations/layer19_attention_weight_min": -17.648805618286133,
      "activations/layer1_attention_weight_max": 18.56717872619629,
      "activations/layer1_attention_weight_min": -13.016205787658691,
      "activations/layer20_attention_weight_max": 19.94231605529785,
      "activations/layer20_attention_weight_min": -21.234155654907227,
      "activations/layer21_attention_weight_max": 41.06943130493164,
      "activations/layer21_attention_weight_min": -27.05026626586914,
      "activations/layer22_attention_weight_max": 35.88809585571289,
      "activations/layer22_attention_weight_min": -21.43086814880371,
      "activations/layer23_attention_weight_max": 27.128087997436523,
      "activations/layer23_attention_weight_min": -14.941405296325684,
      "activations/layer2_attention_weight_max": 28.64844512939453,
      "activations/layer2_attention_weight_min": -28.20612335205078,
      "activations/layer3_attention_weight_max": 71.56744384765625,
      "activations/layer3_attention_weight_min": -77.63526916503906,
      "activations/layer4_attention_weight_max": 102.94438934326172,
      "activations/layer4_attention_weight_min": -94.64710235595703,
      "activations/layer5_attention_weight_max": 121.5628662109375,
      "activations/layer5_attention_weight_min": -105.12289428710938,
      "activations/layer6_attention_weight_max": 60.844459533691406,
      "activations/layer6_attention_weight_min": -55.881019592285156,
      "activations/layer7_attention_weight_max": 65.50798034667969,
      "activations/layer7_attention_weight_min": -59.78644943237305,
      "activations/layer8_attention_weight_max": 42.430030822753906,
      "activations/layer8_attention_weight_min": -37.0217399597168,
      "activations/layer9_attention_weight_max": 52.5052604675293,
      "activations/layer9_attention_weight_min": -48.76694869995117,
      "epoch": 3.6,
      "learning_rate": 0.00012807499999999998,
      "loss": 2.8793,
      "step": 61950
    },
    {
      "activations/layer0_attention_weight_max": 14.935847282409668,
      "activations/layer0_attention_weight_min": -15.289084434509277,
      "activations/layer10_attention_weight_max": 40.367149353027344,
      "activations/layer10_attention_weight_min": -33.777626037597656,
      "activations/layer11_attention_weight_max": 33.97835922241211,
      "activations/layer11_attention_weight_min": -27.128467559814453,
      "activations/layer12_attention_weight_max": 23.16022300720215,
      "activations/layer12_attention_weight_min": -26.558183670043945,
      "activations/layer13_attention_weight_max": 62.73472213745117,
      "activations/layer13_attention_weight_min": -42.676700592041016,
      "activations/layer14_attention_weight_max": 58.043975830078125,
      "activations/layer14_attention_weight_min": -36.545833587646484,
      "activations/layer15_attention_weight_max": 53.32108688354492,
      "activations/layer15_attention_weight_min": -37.169647216796875,
      "activations/layer16_attention_weight_max": 44.0275993347168,
      "activations/layer16_attention_weight_min": -25.92427635192871,
      "activations/layer17_attention_weight_max": 60.62869644165039,
      "activations/layer17_attention_weight_min": -37.41306686401367,
      "activations/layer18_attention_weight_max": 52.2321891784668,
      "activations/layer18_attention_weight_min": -33.238922119140625,
      "activations/layer19_attention_weight_max": 19.759239196777344,
      "activations/layer19_attention_weight_min": -19.060503005981445,
      "activations/layer1_attention_weight_max": 19.603113174438477,
      "activations/layer1_attention_weight_min": -15.97594928741455,
      "activations/layer20_attention_weight_max": 19.78653907775879,
      "activations/layer20_attention_weight_min": -17.39932632446289,
      "activations/layer21_attention_weight_max": 40.278770446777344,
      "activations/layer21_attention_weight_min": -27.60134506225586,
      "activations/layer22_attention_weight_max": 33.38165283203125,
      "activations/layer22_attention_weight_min": -25.74868392944336,
      "activations/layer23_attention_weight_max": 23.08003807067871,
      "activations/layer23_attention_weight_min": -17.298803329467773,
      "activations/layer2_attention_weight_max": 30.266508102416992,
      "activations/layer2_attention_weight_min": -27.546125411987305,
      "activations/layer3_attention_weight_max": 72.5624771118164,
      "activations/layer3_attention_weight_min": -77.97715759277344,
      "activations/layer4_attention_weight_max": 94.89739227294922,
      "activations/layer4_attention_weight_min": -86.66307830810547,
      "activations/layer5_attention_weight_max": 110.0504379272461,
      "activations/layer5_attention_weight_min": -95.49844360351562,
      "activations/layer6_attention_weight_max": 57.475730895996094,
      "activations/layer6_attention_weight_min": -49.41696548461914,
      "activations/layer7_attention_weight_max": 61.37535858154297,
      "activations/layer7_attention_weight_min": -57.031585693359375,
      "activations/layer8_attention_weight_max": 37.80683898925781,
      "activations/layer8_attention_weight_min": -35.86288833618164,
      "activations/layer9_attention_weight_max": 49.9697151184082,
      "activations/layer9_attention_weight_min": -49.59893035888672,
      "epoch": 3.6,
      "learning_rate": 0.0001280560606060606,
      "loss": 2.891,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_loss": 2.837890625,
      "eval_runtime": 8.4353,
      "eval_samples_per_second": 509.053,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_openwebtext_loss": 2.837890625,
      "eval_openwebtext_ppl": 17.079700020438295,
      "eval_openwebtext_runtime": 8.4353,
      "eval_openwebtext_samples_per_second": 509.053,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_wikitext_loss": 3.080078125,
      "eval_wikitext_ppl": 21.760102337787384,
      "eval_wikitext_runtime": 1.9342,
      "eval_wikitext_samples_per_second": 235.757,
      "step": 62000
    },
    {
      "epoch": 3.6,
      "eval_lambada_loss": 2.880859375,
      "eval_lambada_ppl": 17.829588900657228,
      "eval_lambada_runtime": 9.5095,
      "eval_lambada_samples_per_second": 512.012,
      "step": 62000
    },
    {
      "activations/layer0_attention_weight_max": 15.561015129089355,
      "activations/layer0_attention_weight_min": -15.045548439025879,
      "activations/layer10_attention_weight_max": 42.96187973022461,
      "activations/layer10_attention_weight_min": -34.083831787109375,
      "activations/layer11_attention_weight_max": 32.0909423828125,
      "activations/layer11_attention_weight_min": -25.782562255859375,
      "activations/layer12_attention_weight_max": 25.598609924316406,
      "activations/layer12_attention_weight_min": -26.930936813354492,
      "activations/layer13_attention_weight_max": 52.505611419677734,
      "activations/layer13_attention_weight_min": -41.12468719482422,
      "activations/layer14_attention_weight_max": 48.56045913696289,
      "activations/layer14_attention_weight_min": -32.73998260498047,
      "activations/layer15_attention_weight_max": 44.21275329589844,
      "activations/layer15_attention_weight_min": -31.153202056884766,
      "activations/layer16_attention_weight_max": 38.56269073486328,
      "activations/layer16_attention_weight_min": -26.410900115966797,
      "activations/layer17_attention_weight_max": 61.89757537841797,
      "activations/layer17_attention_weight_min": -36.099395751953125,
      "activations/layer18_attention_weight_max": 60.61982727050781,
      "activations/layer18_attention_weight_min": -32.52062225341797,
      "activations/layer19_attention_weight_max": 22.11441993713379,
      "activations/layer19_attention_weight_min": -18.693143844604492,
      "activations/layer1_attention_weight_max": 20.029117584228516,
      "activations/layer1_attention_weight_min": -13.959707260131836,
      "activations/layer20_attention_weight_max": 23.546045303344727,
      "activations/layer20_attention_weight_min": -17.329784393310547,
      "activations/layer21_attention_weight_max": 47.2353630065918,
      "activations/layer21_attention_weight_min": -28.869205474853516,
      "activations/layer22_attention_weight_max": 39.542659759521484,
      "activations/layer22_attention_weight_min": -22.36103630065918,
      "activations/layer23_attention_weight_max": 27.594970703125,
      "activations/layer23_attention_weight_min": -17.860029220581055,
      "activations/layer2_attention_weight_max": 30.48468017578125,
      "activations/layer2_attention_weight_min": -28.13732147216797,
      "activations/layer3_attention_weight_max": 73.8414535522461,
      "activations/layer3_attention_weight_min": -82.31974029541016,
      "activations/layer4_attention_weight_max": 96.05724334716797,
      "activations/layer4_attention_weight_min": -88.47744750976562,
      "activations/layer5_attention_weight_max": 114.35713195800781,
      "activations/layer5_attention_weight_min": -93.45878601074219,
      "activations/layer6_attention_weight_max": 60.24889373779297,
      "activations/layer6_attention_weight_min": -50.69211959838867,
      "activations/layer7_attention_weight_max": 61.45514678955078,
      "activations/layer7_attention_weight_min": -57.2198371887207,
      "activations/layer8_attention_weight_max": 40.73541259765625,
      "activations/layer8_attention_weight_min": -39.36030578613281,
      "activations/layer9_attention_weight_max": 45.40042495727539,
      "activations/layer9_attention_weight_min": -42.199729919433594,
      "epoch": 3.61,
      "learning_rate": 0.00012803712121212119,
      "loss": 2.8872,
      "step": 62050
    },
    {
      "activations/layer0_attention_weight_max": 14.287898063659668,
      "activations/layer0_attention_weight_min": -15.211581230163574,
      "activations/layer10_attention_weight_max": 43.2826042175293,
      "activations/layer10_attention_weight_min": -35.547821044921875,
      "activations/layer11_attention_weight_max": 34.656673431396484,
      "activations/layer11_attention_weight_min": -26.81275177001953,
      "activations/layer12_attention_weight_max": 24.078739166259766,
      "activations/layer12_attention_weight_min": -24.803882598876953,
      "activations/layer13_attention_weight_max": 58.030982971191406,
      "activations/layer13_attention_weight_min": -41.10453796386719,
      "activations/layer14_attention_weight_max": 52.4956169128418,
      "activations/layer14_attention_weight_min": -32.57405090332031,
      "activations/layer15_attention_weight_max": 53.30730438232422,
      "activations/layer15_attention_weight_min": -32.113224029541016,
      "activations/layer16_attention_weight_max": 47.14554214477539,
      "activations/layer16_attention_weight_min": -25.58092498779297,
      "activations/layer17_attention_weight_max": 63.70989227294922,
      "activations/layer17_attention_weight_min": -36.552024841308594,
      "activations/layer18_attention_weight_max": 52.9525032043457,
      "activations/layer18_attention_weight_min": -30.81440544128418,
      "activations/layer19_attention_weight_max": 28.342884063720703,
      "activations/layer19_attention_weight_min": -18.52796745300293,
      "activations/layer1_attention_weight_max": 17.55845832824707,
      "activations/layer1_attention_weight_min": -14.236958503723145,
      "activations/layer20_attention_weight_max": 21.07637596130371,
      "activations/layer20_attention_weight_min": -19.502437591552734,
      "activations/layer21_attention_weight_max": 44.87246322631836,
      "activations/layer21_attention_weight_min": -24.883508682250977,
      "activations/layer22_attention_weight_max": 38.333831787109375,
      "activations/layer22_attention_weight_min": -23.15932273864746,
      "activations/layer23_attention_weight_max": 25.66192626953125,
      "activations/layer23_attention_weight_min": -16.109413146972656,
      "activations/layer2_attention_weight_max": 28.310409545898438,
      "activations/layer2_attention_weight_min": -26.446578979492188,
      "activations/layer3_attention_weight_max": 72.34331512451172,
      "activations/layer3_attention_weight_min": -76.31237030029297,
      "activations/layer4_attention_weight_max": 96.30781555175781,
      "activations/layer4_attention_weight_min": -91.65179443359375,
      "activations/layer5_attention_weight_max": 117.64805603027344,
      "activations/layer5_attention_weight_min": -98.56936645507812,
      "activations/layer6_attention_weight_max": 60.662086486816406,
      "activations/layer6_attention_weight_min": -50.16365051269531,
      "activations/layer7_attention_weight_max": 69.30046081542969,
      "activations/layer7_attention_weight_min": -60.15454864501953,
      "activations/layer8_attention_weight_max": 37.303871154785156,
      "activations/layer8_attention_weight_min": -36.635684967041016,
      "activations/layer9_attention_weight_max": 56.18718338012695,
      "activations/layer9_attention_weight_min": -47.780738830566406,
      "epoch": 3.61,
      "learning_rate": 0.0001280181818181818,
      "loss": 2.869,
      "step": 62100
    },
    {
      "activations/layer0_attention_weight_max": 15.457948684692383,
      "activations/layer0_attention_weight_min": -15.080794334411621,
      "activations/layer10_attention_weight_max": 40.08067321777344,
      "activations/layer10_attention_weight_min": -34.91126251220703,
      "activations/layer11_attention_weight_max": 35.02833557128906,
      "activations/layer11_attention_weight_min": -27.24567413330078,
      "activations/layer12_attention_weight_max": 25.658309936523438,
      "activations/layer12_attention_weight_min": -25.32520294189453,
      "activations/layer13_attention_weight_max": 54.84933090209961,
      "activations/layer13_attention_weight_min": -38.0633659362793,
      "activations/layer14_attention_weight_max": 59.965293884277344,
      "activations/layer14_attention_weight_min": -37.81993103027344,
      "activations/layer15_attention_weight_max": 44.80836486816406,
      "activations/layer15_attention_weight_min": -32.184173583984375,
      "activations/layer16_attention_weight_max": 39.89984893798828,
      "activations/layer16_attention_weight_min": -27.31780242919922,
      "activations/layer17_attention_weight_max": 60.86347198486328,
      "activations/layer17_attention_weight_min": -39.25809097290039,
      "activations/layer18_attention_weight_max": 53.46628952026367,
      "activations/layer18_attention_weight_min": -39.16393280029297,
      "activations/layer19_attention_weight_max": 21.732648849487305,
      "activations/layer19_attention_weight_min": -19.060073852539062,
      "activations/layer1_attention_weight_max": 17.40793228149414,
      "activations/layer1_attention_weight_min": -14.296770095825195,
      "activations/layer20_attention_weight_max": 21.271705627441406,
      "activations/layer20_attention_weight_min": -20.153369903564453,
      "activations/layer21_attention_weight_max": 46.02960968017578,
      "activations/layer21_attention_weight_min": -32.19227600097656,
      "activations/layer22_attention_weight_max": 32.89116287231445,
      "activations/layer22_attention_weight_min": -25.294328689575195,
      "activations/layer23_attention_weight_max": 26.601865768432617,
      "activations/layer23_attention_weight_min": -19.372005462646484,
      "activations/layer2_attention_weight_max": 30.505918502807617,
      "activations/layer2_attention_weight_min": -29.409549713134766,
      "activations/layer3_attention_weight_max": 75.00377655029297,
      "activations/layer3_attention_weight_min": -82.91168212890625,
      "activations/layer4_attention_weight_max": 100.9992446899414,
      "activations/layer4_attention_weight_min": -99.06481170654297,
      "activations/layer5_attention_weight_max": 114.7615966796875,
      "activations/layer5_attention_weight_min": -99.52513122558594,
      "activations/layer6_attention_weight_max": 57.050933837890625,
      "activations/layer6_attention_weight_min": -50.64521026611328,
      "activations/layer7_attention_weight_max": 68.50595092773438,
      "activations/layer7_attention_weight_min": -58.759246826171875,
      "activations/layer8_attention_weight_max": 39.962806701660156,
      "activations/layer8_attention_weight_min": -39.027374267578125,
      "activations/layer9_attention_weight_max": 57.10026168823242,
      "activations/layer9_attention_weight_min": -58.420196533203125,
      "epoch": 3.61,
      "learning_rate": 0.00012799924242424242,
      "loss": 2.8751,
      "step": 62150
    },
    {
      "activations/layer0_attention_weight_max": 13.473702430725098,
      "activations/layer0_attention_weight_min": -15.407602310180664,
      "activations/layer10_attention_weight_max": 39.530982971191406,
      "activations/layer10_attention_weight_min": -32.11187744140625,
      "activations/layer11_attention_weight_max": 32.943695068359375,
      "activations/layer11_attention_weight_min": -26.780986785888672,
      "activations/layer12_attention_weight_max": 35.321205139160156,
      "activations/layer12_attention_weight_min": -24.253707885742188,
      "activations/layer13_attention_weight_max": 54.22575378417969,
      "activations/layer13_attention_weight_min": -37.15542221069336,
      "activations/layer14_attention_weight_max": 44.2322998046875,
      "activations/layer14_attention_weight_min": -29.987430572509766,
      "activations/layer15_attention_weight_max": 48.12731170654297,
      "activations/layer15_attention_weight_min": -30.991777420043945,
      "activations/layer16_attention_weight_max": 37.24588394165039,
      "activations/layer16_attention_weight_min": -25.414295196533203,
      "activations/layer17_attention_weight_max": 58.64939498901367,
      "activations/layer17_attention_weight_min": -36.181724548339844,
      "activations/layer18_attention_weight_max": 48.7622184753418,
      "activations/layer18_attention_weight_min": -31.9875545501709,
      "activations/layer19_attention_weight_max": 22.725595474243164,
      "activations/layer19_attention_weight_min": -17.175527572631836,
      "activations/layer1_attention_weight_max": 18.198259353637695,
      "activations/layer1_attention_weight_min": -16.165544509887695,
      "activations/layer20_attention_weight_max": 19.739229202270508,
      "activations/layer20_attention_weight_min": -21.202112197875977,
      "activations/layer21_attention_weight_max": 38.75278854370117,
      "activations/layer21_attention_weight_min": -27.822038650512695,
      "activations/layer22_attention_weight_max": 32.54575729370117,
      "activations/layer22_attention_weight_min": -23.658916473388672,
      "activations/layer23_attention_weight_max": 22.831764221191406,
      "activations/layer23_attention_weight_min": -16.72019386291504,
      "activations/layer2_attention_weight_max": 29.99352264404297,
      "activations/layer2_attention_weight_min": -27.47540855407715,
      "activations/layer3_attention_weight_max": 75.26808166503906,
      "activations/layer3_attention_weight_min": -81.55782318115234,
      "activations/layer4_attention_weight_max": 101.51439666748047,
      "activations/layer4_attention_weight_min": -92.4910888671875,
      "activations/layer5_attention_weight_max": 116.10987854003906,
      "activations/layer5_attention_weight_min": -97.72305297851562,
      "activations/layer6_attention_weight_max": 59.374366760253906,
      "activations/layer6_attention_weight_min": -50.77373123168945,
      "activations/layer7_attention_weight_max": 62.5572624206543,
      "activations/layer7_attention_weight_min": -60.33668518066406,
      "activations/layer8_attention_weight_max": 40.776390075683594,
      "activations/layer8_attention_weight_min": -36.470211029052734,
      "activations/layer9_attention_weight_max": 48.7668571472168,
      "activations/layer9_attention_weight_min": -52.622802734375,
      "epoch": 3.61,
      "learning_rate": 0.000127980303030303,
      "loss": 2.9017,
      "step": 62200
    },
    {
      "activations/layer0_attention_weight_max": 13.680211067199707,
      "activations/layer0_attention_weight_min": -15.148996353149414,
      "activations/layer10_attention_weight_max": 46.42449188232422,
      "activations/layer10_attention_weight_min": -37.0494499206543,
      "activations/layer11_attention_weight_max": 35.25034713745117,
      "activations/layer11_attention_weight_min": -27.672008514404297,
      "activations/layer12_attention_weight_max": 25.14279556274414,
      "activations/layer12_attention_weight_min": -27.38950538635254,
      "activations/layer13_attention_weight_max": 59.539520263671875,
      "activations/layer13_attention_weight_min": -39.21257781982422,
      "activations/layer14_attention_weight_max": 51.346981048583984,
      "activations/layer14_attention_weight_min": -29.125411987304688,
      "activations/layer15_attention_weight_max": 45.11003494262695,
      "activations/layer15_attention_weight_min": -35.602081298828125,
      "activations/layer16_attention_weight_max": 38.31636428833008,
      "activations/layer16_attention_weight_min": -27.195951461791992,
      "activations/layer17_attention_weight_max": 59.43096923828125,
      "activations/layer17_attention_weight_min": -36.27021789550781,
      "activations/layer18_attention_weight_max": 50.041996002197266,
      "activations/layer18_attention_weight_min": -30.491975784301758,
      "activations/layer19_attention_weight_max": 21.054147720336914,
      "activations/layer19_attention_weight_min": -18.81563377380371,
      "activations/layer1_attention_weight_max": 17.933107376098633,
      "activations/layer1_attention_weight_min": -14.444604873657227,
      "activations/layer20_attention_weight_max": 21.68101692199707,
      "activations/layer20_attention_weight_min": -19.399568557739258,
      "activations/layer21_attention_weight_max": 46.55945587158203,
      "activations/layer21_attention_weight_min": -25.66415786743164,
      "activations/layer22_attention_weight_max": 37.43439483642578,
      "activations/layer22_attention_weight_min": -21.918603897094727,
      "activations/layer23_attention_weight_max": 28.166690826416016,
      "activations/layer23_attention_weight_min": -15.371020317077637,
      "activations/layer2_attention_weight_max": 29.59455680847168,
      "activations/layer2_attention_weight_min": -27.99746322631836,
      "activations/layer3_attention_weight_max": 76.01078796386719,
      "activations/layer3_attention_weight_min": -79.9847412109375,
      "activations/layer4_attention_weight_max": 109.60526275634766,
      "activations/layer4_attention_weight_min": -99.18994903564453,
      "activations/layer5_attention_weight_max": 126.80846405029297,
      "activations/layer5_attention_weight_min": -111.41863250732422,
      "activations/layer6_attention_weight_max": 64.84585571289062,
      "activations/layer6_attention_weight_min": -59.283164978027344,
      "activations/layer7_attention_weight_max": 73.77613830566406,
      "activations/layer7_attention_weight_min": -60.89701461791992,
      "activations/layer8_attention_weight_max": 43.110477447509766,
      "activations/layer8_attention_weight_min": -41.71987533569336,
      "activations/layer9_attention_weight_max": 55.89794921875,
      "activations/layer9_attention_weight_min": -50.444149017333984,
      "epoch": 3.62,
      "learning_rate": 0.00012796136363636362,
      "loss": 2.8775,
      "step": 62250
    },
    {
      "activations/layer0_attention_weight_max": 14.209648132324219,
      "activations/layer0_attention_weight_min": -14.9430513381958,
      "activations/layer10_attention_weight_max": 39.362979888916016,
      "activations/layer10_attention_weight_min": -31.999752044677734,
      "activations/layer11_attention_weight_max": 32.499820709228516,
      "activations/layer11_attention_weight_min": -27.251859664916992,
      "activations/layer12_attention_weight_max": 26.824874877929688,
      "activations/layer12_attention_weight_min": -26.826261520385742,
      "activations/layer13_attention_weight_max": 52.378658294677734,
      "activations/layer13_attention_weight_min": -37.948699951171875,
      "activations/layer14_attention_weight_max": 48.06455993652344,
      "activations/layer14_attention_weight_min": -32.07564926147461,
      "activations/layer15_attention_weight_max": 43.56094741821289,
      "activations/layer15_attention_weight_min": -31.272132873535156,
      "activations/layer16_attention_weight_max": 36.59164047241211,
      "activations/layer16_attention_weight_min": -27.29701042175293,
      "activations/layer17_attention_weight_max": 56.420249938964844,
      "activations/layer17_attention_weight_min": -35.54800033569336,
      "activations/layer18_attention_weight_max": 53.06755065917969,
      "activations/layer18_attention_weight_min": -34.602203369140625,
      "activations/layer19_attention_weight_max": 21.565040588378906,
      "activations/layer19_attention_weight_min": -17.709741592407227,
      "activations/layer1_attention_weight_max": 18.22450828552246,
      "activations/layer1_attention_weight_min": -12.27079963684082,
      "activations/layer20_attention_weight_max": 20.320945739746094,
      "activations/layer20_attention_weight_min": -18.86701011657715,
      "activations/layer21_attention_weight_max": 38.917816162109375,
      "activations/layer21_attention_weight_min": -30.599857330322266,
      "activations/layer22_attention_weight_max": 36.99325942993164,
      "activations/layer22_attention_weight_min": -23.757444381713867,
      "activations/layer23_attention_weight_max": 27.300823211669922,
      "activations/layer23_attention_weight_min": -18.069721221923828,
      "activations/layer2_attention_weight_max": 29.19549560546875,
      "activations/layer2_attention_weight_min": -28.804288864135742,
      "activations/layer3_attention_weight_max": 82.56775665283203,
      "activations/layer3_attention_weight_min": -83.77790069580078,
      "activations/layer4_attention_weight_max": 99.86830139160156,
      "activations/layer4_attention_weight_min": -92.23287200927734,
      "activations/layer5_attention_weight_max": 120.47154998779297,
      "activations/layer5_attention_weight_min": -103.81697845458984,
      "activations/layer6_attention_weight_max": 62.289127349853516,
      "activations/layer6_attention_weight_min": -52.08674240112305,
      "activations/layer7_attention_weight_max": 69.51676940917969,
      "activations/layer7_attention_weight_min": -58.49284744262695,
      "activations/layer8_attention_weight_max": 39.88921356201172,
      "activations/layer8_attention_weight_min": -35.1737060546875,
      "activations/layer9_attention_weight_max": 60.210609436035156,
      "activations/layer9_attention_weight_min": -52.5107536315918,
      "epoch": 3.62,
      "learning_rate": 0.00012794242424242423,
      "loss": 2.8908,
      "step": 62300
    },
    {
      "activations/layer0_attention_weight_max": 15.545790672302246,
      "activations/layer0_attention_weight_min": -15.05387020111084,
      "activations/layer10_attention_weight_max": 39.42973327636719,
      "activations/layer10_attention_weight_min": -33.03837585449219,
      "activations/layer11_attention_weight_max": 32.20733642578125,
      "activations/layer11_attention_weight_min": -25.03411865234375,
      "activations/layer12_attention_weight_max": 23.912927627563477,
      "activations/layer12_attention_weight_min": -25.57534408569336,
      "activations/layer13_attention_weight_max": 54.9012451171875,
      "activations/layer13_attention_weight_min": -42.10343551635742,
      "activations/layer14_attention_weight_max": 50.586265563964844,
      "activations/layer14_attention_weight_min": -39.44926071166992,
      "activations/layer15_attention_weight_max": 44.834564208984375,
      "activations/layer15_attention_weight_min": -38.26768112182617,
      "activations/layer16_attention_weight_max": 36.78669738769531,
      "activations/layer16_attention_weight_min": -26.26177406311035,
      "activations/layer17_attention_weight_max": 56.61134338378906,
      "activations/layer17_attention_weight_min": -34.788997650146484,
      "activations/layer18_attention_weight_max": 52.418540954589844,
      "activations/layer18_attention_weight_min": -30.557424545288086,
      "activations/layer19_attention_weight_max": 26.360044479370117,
      "activations/layer19_attention_weight_min": -18.794897079467773,
      "activations/layer1_attention_weight_max": 17.81266212463379,
      "activations/layer1_attention_weight_min": -16.844350814819336,
      "activations/layer20_attention_weight_max": 21.045089721679688,
      "activations/layer20_attention_weight_min": -17.80124282836914,
      "activations/layer21_attention_weight_max": 46.35356140136719,
      "activations/layer21_attention_weight_min": -27.870744705200195,
      "activations/layer22_attention_weight_max": 34.797889709472656,
      "activations/layer22_attention_weight_min": -23.97203254699707,
      "activations/layer23_attention_weight_max": 24.357837677001953,
      "activations/layer23_attention_weight_min": -17.958959579467773,
      "activations/layer2_attention_weight_max": 30.43927001953125,
      "activations/layer2_attention_weight_min": -29.01685905456543,
      "activations/layer3_attention_weight_max": 76.55133056640625,
      "activations/layer3_attention_weight_min": -77.67015838623047,
      "activations/layer4_attention_weight_max": 102.06998443603516,
      "activations/layer4_attention_weight_min": -93.88776397705078,
      "activations/layer5_attention_weight_max": 123.79330444335938,
      "activations/layer5_attention_weight_min": -98.16841125488281,
      "activations/layer6_attention_weight_max": 62.79685592651367,
      "activations/layer6_attention_weight_min": -51.91793441772461,
      "activations/layer7_attention_weight_max": 68.57706451416016,
      "activations/layer7_attention_weight_min": -56.6482048034668,
      "activations/layer8_attention_weight_max": 38.976688385009766,
      "activations/layer8_attention_weight_min": -34.65384292602539,
      "activations/layer9_attention_weight_max": 60.52728271484375,
      "activations/layer9_attention_weight_min": -46.8272819519043,
      "epoch": 3.62,
      "learning_rate": 0.00012792348484848485,
      "loss": 2.8816,
      "step": 62350
    },
    {
      "activations/layer0_attention_weight_max": 13.586209297180176,
      "activations/layer0_attention_weight_min": -14.585498809814453,
      "activations/layer10_attention_weight_max": 42.15105056762695,
      "activations/layer10_attention_weight_min": -34.03151321411133,
      "activations/layer11_attention_weight_max": 34.946292877197266,
      "activations/layer11_attention_weight_min": -26.085384368896484,
      "activations/layer12_attention_weight_max": 21.528348922729492,
      "activations/layer12_attention_weight_min": -27.473920822143555,
      "activations/layer13_attention_weight_max": 60.33049392700195,
      "activations/layer13_attention_weight_min": -39.669281005859375,
      "activations/layer14_attention_weight_max": 46.58991622924805,
      "activations/layer14_attention_weight_min": -35.23005294799805,
      "activations/layer15_attention_weight_max": 57.1034049987793,
      "activations/layer15_attention_weight_min": -36.68141555786133,
      "activations/layer16_attention_weight_max": 39.13802719116211,
      "activations/layer16_attention_weight_min": -25.57362174987793,
      "activations/layer17_attention_weight_max": 57.44729995727539,
      "activations/layer17_attention_weight_min": -34.13184356689453,
      "activations/layer18_attention_weight_max": 51.592933654785156,
      "activations/layer18_attention_weight_min": -31.143001556396484,
      "activations/layer19_attention_weight_max": 25.73256492614746,
      "activations/layer19_attention_weight_min": -17.719362258911133,
      "activations/layer1_attention_weight_max": 17.543224334716797,
      "activations/layer1_attention_weight_min": -13.719916343688965,
      "activations/layer20_attention_weight_max": 23.96017837524414,
      "activations/layer20_attention_weight_min": -19.02899742126465,
      "activations/layer21_attention_weight_max": 38.04811477661133,
      "activations/layer21_attention_weight_min": -26.32920265197754,
      "activations/layer22_attention_weight_max": 36.42354965209961,
      "activations/layer22_attention_weight_min": -24.173568725585938,
      "activations/layer23_attention_weight_max": 25.085302352905273,
      "activations/layer23_attention_weight_min": -16.61119270324707,
      "activations/layer2_attention_weight_max": 30.093990325927734,
      "activations/layer2_attention_weight_min": -26.737112045288086,
      "activations/layer3_attention_weight_max": 74.11237335205078,
      "activations/layer3_attention_weight_min": -82.19731140136719,
      "activations/layer4_attention_weight_max": 101.52764129638672,
      "activations/layer4_attention_weight_min": -90.46851348876953,
      "activations/layer5_attention_weight_max": 113.06997680664062,
      "activations/layer5_attention_weight_min": -96.30147552490234,
      "activations/layer6_attention_weight_max": 60.45769119262695,
      "activations/layer6_attention_weight_min": -51.39790344238281,
      "activations/layer7_attention_weight_max": 62.62773895263672,
      "activations/layer7_attention_weight_min": -59.56504821777344,
      "activations/layer8_attention_weight_max": 38.479591369628906,
      "activations/layer8_attention_weight_min": -36.03315734863281,
      "activations/layer9_attention_weight_max": 47.635501861572266,
      "activations/layer9_attention_weight_min": -46.45859146118164,
      "epoch": 3.63,
      "learning_rate": 0.00012790454545454544,
      "loss": 2.8842,
      "step": 62400
    },
    {
      "activations/layer0_attention_weight_max": 14.656261444091797,
      "activations/layer0_attention_weight_min": -14.656248092651367,
      "activations/layer10_attention_weight_max": 42.612823486328125,
      "activations/layer10_attention_weight_min": -32.6839714050293,
      "activations/layer11_attention_weight_max": 32.45779800415039,
      "activations/layer11_attention_weight_min": -26.040035247802734,
      "activations/layer12_attention_weight_max": 23.595731735229492,
      "activations/layer12_attention_weight_min": -23.082630157470703,
      "activations/layer13_attention_weight_max": 62.541324615478516,
      "activations/layer13_attention_weight_min": -37.982994079589844,
      "activations/layer14_attention_weight_max": 49.83285140991211,
      "activations/layer14_attention_weight_min": -35.299583435058594,
      "activations/layer15_attention_weight_max": 48.49946594238281,
      "activations/layer15_attention_weight_min": -31.478954315185547,
      "activations/layer16_attention_weight_max": 40.062042236328125,
      "activations/layer16_attention_weight_min": -26.246828079223633,
      "activations/layer17_attention_weight_max": 60.399658203125,
      "activations/layer17_attention_weight_min": -36.7889404296875,
      "activations/layer18_attention_weight_max": 51.937564849853516,
      "activations/layer18_attention_weight_min": -31.14154624938965,
      "activations/layer19_attention_weight_max": 24.9758358001709,
      "activations/layer19_attention_weight_min": -19.561080932617188,
      "activations/layer1_attention_weight_max": 17.705461502075195,
      "activations/layer1_attention_weight_min": -13.832538604736328,
      "activations/layer20_attention_weight_max": 21.029844284057617,
      "activations/layer20_attention_weight_min": -20.7391357421875,
      "activations/layer21_attention_weight_max": 43.75596618652344,
      "activations/layer21_attention_weight_min": -28.77831268310547,
      "activations/layer22_attention_weight_max": 38.3715934753418,
      "activations/layer22_attention_weight_min": -22.09056854248047,
      "activations/layer23_attention_weight_max": 24.632083892822266,
      "activations/layer23_attention_weight_min": -21.500226974487305,
      "activations/layer2_attention_weight_max": 29.427303314208984,
      "activations/layer2_attention_weight_min": -28.370410919189453,
      "activations/layer3_attention_weight_max": 75.40888977050781,
      "activations/layer3_attention_weight_min": -79.51077270507812,
      "activations/layer4_attention_weight_max": 95.91051483154297,
      "activations/layer4_attention_weight_min": -97.33086395263672,
      "activations/layer5_attention_weight_max": 115.58184051513672,
      "activations/layer5_attention_weight_min": -100.46359252929688,
      "activations/layer6_attention_weight_max": 60.930580139160156,
      "activations/layer6_attention_weight_min": -53.07963562011719,
      "activations/layer7_attention_weight_max": 64.94976806640625,
      "activations/layer7_attention_weight_min": -63.54650115966797,
      "activations/layer8_attention_weight_max": 40.12053298950195,
      "activations/layer8_attention_weight_min": -37.898746490478516,
      "activations/layer9_attention_weight_max": 51.327674865722656,
      "activations/layer9_attention_weight_min": -48.27154541015625,
      "epoch": 3.63,
      "learning_rate": 0.00012788598484848483,
      "loss": 2.8761,
      "step": 62450
    },
    {
      "activations/layer0_attention_weight_max": 14.251405715942383,
      "activations/layer0_attention_weight_min": -14.797154426574707,
      "activations/layer10_attention_weight_max": 41.53932189941406,
      "activations/layer10_attention_weight_min": -33.83618927001953,
      "activations/layer11_attention_weight_max": 34.983455657958984,
      "activations/layer11_attention_weight_min": -28.15087890625,
      "activations/layer12_attention_weight_max": 26.46013832092285,
      "activations/layer12_attention_weight_min": -25.559112548828125,
      "activations/layer13_attention_weight_max": 57.678955078125,
      "activations/layer13_attention_weight_min": -36.65758514404297,
      "activations/layer14_attention_weight_max": 56.553585052490234,
      "activations/layer14_attention_weight_min": -35.51228332519531,
      "activations/layer15_attention_weight_max": 48.44392013549805,
      "activations/layer15_attention_weight_min": -29.01835823059082,
      "activations/layer16_attention_weight_max": 39.841495513916016,
      "activations/layer16_attention_weight_min": -28.710792541503906,
      "activations/layer17_attention_weight_max": 64.25850677490234,
      "activations/layer17_attention_weight_min": -35.522850036621094,
      "activations/layer18_attention_weight_max": 52.70172119140625,
      "activations/layer18_attention_weight_min": -30.892902374267578,
      "activations/layer19_attention_weight_max": 27.032733917236328,
      "activations/layer19_attention_weight_min": -18.757112503051758,
      "activations/layer1_attention_weight_max": 18.087940216064453,
      "activations/layer1_attention_weight_min": -16.41707992553711,
      "activations/layer20_attention_weight_max": 27.036664962768555,
      "activations/layer20_attention_weight_min": -19.848600387573242,
      "activations/layer21_attention_weight_max": 46.13717269897461,
      "activations/layer21_attention_weight_min": -30.3582763671875,
      "activations/layer22_attention_weight_max": 37.52521514892578,
      "activations/layer22_attention_weight_min": -24.06901741027832,
      "activations/layer23_attention_weight_max": 25.886924743652344,
      "activations/layer23_attention_weight_min": -17.06681251525879,
      "activations/layer2_attention_weight_max": 29.398488998413086,
      "activations/layer2_attention_weight_min": -28.49505615234375,
      "activations/layer3_attention_weight_max": 75.9820556640625,
      "activations/layer3_attention_weight_min": -78.40254974365234,
      "activations/layer4_attention_weight_max": 101.9577407836914,
      "activations/layer4_attention_weight_min": -92.3358383178711,
      "activations/layer5_attention_weight_max": 119.02012634277344,
      "activations/layer5_attention_weight_min": -100.14701843261719,
      "activations/layer6_attention_weight_max": 59.062225341796875,
      "activations/layer6_attention_weight_min": -54.60947036743164,
      "activations/layer7_attention_weight_max": 66.75638580322266,
      "activations/layer7_attention_weight_min": -59.998653411865234,
      "activations/layer8_attention_weight_max": 39.76995086669922,
      "activations/layer8_attention_weight_min": -36.25947570800781,
      "activations/layer9_attention_weight_max": 55.198974609375,
      "activations/layer9_attention_weight_min": -47.87253952026367,
      "epoch": 3.63,
      "learning_rate": 0.00012786742424242425,
      "loss": 2.885,
      "step": 62500
    },
    {
      "activations/layer0_attention_weight_max": 14.072331428527832,
      "activations/layer0_attention_weight_min": -15.22608470916748,
      "activations/layer10_attention_weight_max": 42.11275863647461,
      "activations/layer10_attention_weight_min": -34.7437629699707,
      "activations/layer11_attention_weight_max": 35.849151611328125,
      "activations/layer11_attention_weight_min": -26.147985458374023,
      "activations/layer12_attention_weight_max": 25.34033203125,
      "activations/layer12_attention_weight_min": -25.252500534057617,
      "activations/layer13_attention_weight_max": 64.31665802001953,
      "activations/layer13_attention_weight_min": -43.47604751586914,
      "activations/layer14_attention_weight_max": 57.63309860229492,
      "activations/layer14_attention_weight_min": -35.83361053466797,
      "activations/layer15_attention_weight_max": 44.69361114501953,
      "activations/layer15_attention_weight_min": -30.070537567138672,
      "activations/layer16_attention_weight_max": 37.96229553222656,
      "activations/layer16_attention_weight_min": -26.86112403869629,
      "activations/layer17_attention_weight_max": 61.12948226928711,
      "activations/layer17_attention_weight_min": -36.093299865722656,
      "activations/layer18_attention_weight_max": 57.12995910644531,
      "activations/layer18_attention_weight_min": -30.42558479309082,
      "activations/layer19_attention_weight_max": 24.403701782226562,
      "activations/layer19_attention_weight_min": -18.29152488708496,
      "activations/layer1_attention_weight_max": 17.167076110839844,
      "activations/layer1_attention_weight_min": -14.903494834899902,
      "activations/layer20_attention_weight_max": 24.765775680541992,
      "activations/layer20_attention_weight_min": -20.32868003845215,
      "activations/layer21_attention_weight_max": 46.226234436035156,
      "activations/layer21_attention_weight_min": -26.241361618041992,
      "activations/layer22_attention_weight_max": 37.94209289550781,
      "activations/layer22_attention_weight_min": -23.784313201904297,
      "activations/layer23_attention_weight_max": 30.77021026611328,
      "activations/layer23_attention_weight_min": -17.580184936523438,
      "activations/layer2_attention_weight_max": 30.243682861328125,
      "activations/layer2_attention_weight_min": -28.673355102539062,
      "activations/layer3_attention_weight_max": 78.79248809814453,
      "activations/layer3_attention_weight_min": -81.68229675292969,
      "activations/layer4_attention_weight_max": 99.6729965209961,
      "activations/layer4_attention_weight_min": -94.00154876708984,
      "activations/layer5_attention_weight_max": 116.17182922363281,
      "activations/layer5_attention_weight_min": -98.95983123779297,
      "activations/layer6_attention_weight_max": 63.59716796875,
      "activations/layer6_attention_weight_min": -57.30536651611328,
      "activations/layer7_attention_weight_max": 64.2264633178711,
      "activations/layer7_attention_weight_min": -61.17349624633789,
      "activations/layer8_attention_weight_max": 39.413047790527344,
      "activations/layer8_attention_weight_min": -38.978694915771484,
      "activations/layer9_attention_weight_max": 49.89352035522461,
      "activations/layer9_attention_weight_min": -51.346351623535156,
      "epoch": 3.63,
      "learning_rate": 0.00012784848484848483,
      "loss": 2.9038,
      "step": 62550
    },
    {
      "activations/layer0_attention_weight_max": 13.950811386108398,
      "activations/layer0_attention_weight_min": -14.760278701782227,
      "activations/layer10_attention_weight_max": 41.971927642822266,
      "activations/layer10_attention_weight_min": -36.09318161010742,
      "activations/layer11_attention_weight_max": 33.76560974121094,
      "activations/layer11_attention_weight_min": -25.962392807006836,
      "activations/layer12_attention_weight_max": 24.295570373535156,
      "activations/layer12_attention_weight_min": -23.6429443359375,
      "activations/layer13_attention_weight_max": 59.74442672729492,
      "activations/layer13_attention_weight_min": -40.06599807739258,
      "activations/layer14_attention_weight_max": 61.999671936035156,
      "activations/layer14_attention_weight_min": -39.30461883544922,
      "activations/layer15_attention_weight_max": 48.059967041015625,
      "activations/layer15_attention_weight_min": -31.68236541748047,
      "activations/layer16_attention_weight_max": 35.7743034362793,
      "activations/layer16_attention_weight_min": -25.354867935180664,
      "activations/layer17_attention_weight_max": 57.24785232543945,
      "activations/layer17_attention_weight_min": -36.317203521728516,
      "activations/layer18_attention_weight_max": 47.66426086425781,
      "activations/layer18_attention_weight_min": -32.16230392456055,
      "activations/layer19_attention_weight_max": 23.88506507873535,
      "activations/layer19_attention_weight_min": -16.457258224487305,
      "activations/layer1_attention_weight_max": 18.17842674255371,
      "activations/layer1_attention_weight_min": -15.733407020568848,
      "activations/layer20_attention_weight_max": 23.00452995300293,
      "activations/layer20_attention_weight_min": -22.154897689819336,
      "activations/layer21_attention_weight_max": 39.025970458984375,
      "activations/layer21_attention_weight_min": -25.17454719543457,
      "activations/layer22_attention_weight_max": 35.18342208862305,
      "activations/layer22_attention_weight_min": -24.28072166442871,
      "activations/layer23_attention_weight_max": 27.20508575439453,
      "activations/layer23_attention_weight_min": -16.192827224731445,
      "activations/layer2_attention_weight_max": 28.51703453063965,
      "activations/layer2_attention_weight_min": -26.959091186523438,
      "activations/layer3_attention_weight_max": 71.9375991821289,
      "activations/layer3_attention_weight_min": -75.62498474121094,
      "activations/layer4_attention_weight_max": 104.1834487915039,
      "activations/layer4_attention_weight_min": -96.18817138671875,
      "activations/layer5_attention_weight_max": 123.80374145507812,
      "activations/layer5_attention_weight_min": -109.23478698730469,
      "activations/layer6_attention_weight_max": 61.1251106262207,
      "activations/layer6_attention_weight_min": -53.66374969482422,
      "activations/layer7_attention_weight_max": 66.46421813964844,
      "activations/layer7_attention_weight_min": -57.93750762939453,
      "activations/layer8_attention_weight_max": 40.28947830200195,
      "activations/layer8_attention_weight_min": -40.164817810058594,
      "activations/layer9_attention_weight_max": 48.59437561035156,
      "activations/layer9_attention_weight_min": -50.41947555541992,
      "epoch": 3.64,
      "learning_rate": 0.00012782954545454545,
      "loss": 2.8934,
      "step": 62600
    },
    {
      "activations/layer0_attention_weight_max": 14.062262535095215,
      "activations/layer0_attention_weight_min": -14.971050262451172,
      "activations/layer10_attention_weight_max": 43.07196044921875,
      "activations/layer10_attention_weight_min": -34.898284912109375,
      "activations/layer11_attention_weight_max": 34.37999725341797,
      "activations/layer11_attention_weight_min": -25.40380859375,
      "activations/layer12_attention_weight_max": 35.62792205810547,
      "activations/layer12_attention_weight_min": -25.48466682434082,
      "activations/layer13_attention_weight_max": 58.89470672607422,
      "activations/layer13_attention_weight_min": -38.86293411254883,
      "activations/layer14_attention_weight_max": 53.62242889404297,
      "activations/layer14_attention_weight_min": -34.88130569458008,
      "activations/layer15_attention_weight_max": 46.69542694091797,
      "activations/layer15_attention_weight_min": -33.297183990478516,
      "activations/layer16_attention_weight_max": 38.16828918457031,
      "activations/layer16_attention_weight_min": -26.7730655670166,
      "activations/layer17_attention_weight_max": 58.3314208984375,
      "activations/layer17_attention_weight_min": -37.8568000793457,
      "activations/layer18_attention_weight_max": 47.8320198059082,
      "activations/layer18_attention_weight_min": -32.22218322753906,
      "activations/layer19_attention_weight_max": 22.28420639038086,
      "activations/layer19_attention_weight_min": -18.482280731201172,
      "activations/layer1_attention_weight_max": 16.904132843017578,
      "activations/layer1_attention_weight_min": -13.248739242553711,
      "activations/layer20_attention_weight_max": 23.452299118041992,
      "activations/layer20_attention_weight_min": -18.827882766723633,
      "activations/layer21_attention_weight_max": 36.67812728881836,
      "activations/layer21_attention_weight_min": -29.305461883544922,
      "activations/layer22_attention_weight_max": 35.38612365722656,
      "activations/layer22_attention_weight_min": -23.5704288482666,
      "activations/layer23_attention_weight_max": 27.904935836791992,
      "activations/layer23_attention_weight_min": -17.429595947265625,
      "activations/layer2_attention_weight_max": 30.02241325378418,
      "activations/layer2_attention_weight_min": -27.882461547851562,
      "activations/layer3_attention_weight_max": 77.6221694946289,
      "activations/layer3_attention_weight_min": -84.83545684814453,
      "activations/layer4_attention_weight_max": 101.22661590576172,
      "activations/layer4_attention_weight_min": -96.1853256225586,
      "activations/layer5_attention_weight_max": 115.19878387451172,
      "activations/layer5_attention_weight_min": -97.84931945800781,
      "activations/layer6_attention_weight_max": 59.44541931152344,
      "activations/layer6_attention_weight_min": -49.949954986572266,
      "activations/layer7_attention_weight_max": 61.35213851928711,
      "activations/layer7_attention_weight_min": -57.99482727050781,
      "activations/layer8_attention_weight_max": 39.46696853637695,
      "activations/layer8_attention_weight_min": -38.8029670715332,
      "activations/layer9_attention_weight_max": 51.914794921875,
      "activations/layer9_attention_weight_min": -43.18010330200195,
      "epoch": 3.64,
      "learning_rate": 0.00012781060606060606,
      "loss": 2.8788,
      "step": 62650
    },
    {
      "activations/layer0_attention_weight_max": 13.827343940734863,
      "activations/layer0_attention_weight_min": -14.629737854003906,
      "activations/layer10_attention_weight_max": 41.45475387573242,
      "activations/layer10_attention_weight_min": -33.37966537475586,
      "activations/layer11_attention_weight_max": 32.366050720214844,
      "activations/layer11_attention_weight_min": -27.161441802978516,
      "activations/layer12_attention_weight_max": 25.05253028869629,
      "activations/layer12_attention_weight_min": -24.569868087768555,
      "activations/layer13_attention_weight_max": 51.07620620727539,
      "activations/layer13_attention_weight_min": -42.645896911621094,
      "activations/layer14_attention_weight_max": 41.193031311035156,
      "activations/layer14_attention_weight_min": -29.67963981628418,
      "activations/layer15_attention_weight_max": 48.78898620605469,
      "activations/layer15_attention_weight_min": -35.78892517089844,
      "activations/layer16_attention_weight_max": 38.518707275390625,
      "activations/layer16_attention_weight_min": -27.941553115844727,
      "activations/layer17_attention_weight_max": 57.77320098876953,
      "activations/layer17_attention_weight_min": -36.48677444458008,
      "activations/layer18_attention_weight_max": 48.18050003051758,
      "activations/layer18_attention_weight_min": -32.30739974975586,
      "activations/layer19_attention_weight_max": 21.096519470214844,
      "activations/layer19_attention_weight_min": -16.811052322387695,
      "activations/layer1_attention_weight_max": 17.161157608032227,
      "activations/layer1_attention_weight_min": -14.57827377319336,
      "activations/layer20_attention_weight_max": 21.818157196044922,
      "activations/layer20_attention_weight_min": -19.908781051635742,
      "activations/layer21_attention_weight_max": 40.067352294921875,
      "activations/layer21_attention_weight_min": -27.52589988708496,
      "activations/layer22_attention_weight_max": 35.57737350463867,
      "activations/layer22_attention_weight_min": -27.62081527709961,
      "activations/layer23_attention_weight_max": 27.294384002685547,
      "activations/layer23_attention_weight_min": -18.944671630859375,
      "activations/layer2_attention_weight_max": 30.609596252441406,
      "activations/layer2_attention_weight_min": -29.167327880859375,
      "activations/layer3_attention_weight_max": 79.1065902709961,
      "activations/layer3_attention_weight_min": -84.26542663574219,
      "activations/layer4_attention_weight_max": 99.66292572021484,
      "activations/layer4_attention_weight_min": -93.20918273925781,
      "activations/layer5_attention_weight_max": 115.95767211914062,
      "activations/layer5_attention_weight_min": -99.99516296386719,
      "activations/layer6_attention_weight_max": 59.166168212890625,
      "activations/layer6_attention_weight_min": -52.5173225402832,
      "activations/layer7_attention_weight_max": 61.80757141113281,
      "activations/layer7_attention_weight_min": -58.47472381591797,
      "activations/layer8_attention_weight_max": 38.0549430847168,
      "activations/layer8_attention_weight_min": -38.651004791259766,
      "activations/layer9_attention_weight_max": 48.1229362487793,
      "activations/layer9_attention_weight_min": -44.356536865234375,
      "epoch": 3.64,
      "learning_rate": 0.00012779166666666665,
      "loss": 2.9019,
      "step": 62700
    },
    {
      "activations/layer0_attention_weight_max": 14.013569831848145,
      "activations/layer0_attention_weight_min": -14.955936431884766,
      "activations/layer10_attention_weight_max": 42.5811653137207,
      "activations/layer10_attention_weight_min": -34.73919677734375,
      "activations/layer11_attention_weight_max": 32.95162582397461,
      "activations/layer11_attention_weight_min": -26.20724868774414,
      "activations/layer12_attention_weight_max": 24.906940460205078,
      "activations/layer12_attention_weight_min": -29.419755935668945,
      "activations/layer13_attention_weight_max": 50.304080963134766,
      "activations/layer13_attention_weight_min": -37.87087631225586,
      "activations/layer14_attention_weight_max": 47.55183792114258,
      "activations/layer14_attention_weight_min": -33.50574493408203,
      "activations/layer15_attention_weight_max": 43.0456428527832,
      "activations/layer15_attention_weight_min": -30.811010360717773,
      "activations/layer16_attention_weight_max": 36.7070198059082,
      "activations/layer16_attention_weight_min": -25.09493637084961,
      "activations/layer17_attention_weight_max": 59.21628952026367,
      "activations/layer17_attention_weight_min": -34.36137390136719,
      "activations/layer18_attention_weight_max": 48.07280349731445,
      "activations/layer18_attention_weight_min": -32.630340576171875,
      "activations/layer19_attention_weight_max": 24.82114028930664,
      "activations/layer19_attention_weight_min": -18.378746032714844,
      "activations/layer1_attention_weight_max": 17.62371826171875,
      "activations/layer1_attention_weight_min": -15.006009101867676,
      "activations/layer20_attention_weight_max": 19.515920639038086,
      "activations/layer20_attention_weight_min": -20.503686904907227,
      "activations/layer21_attention_weight_max": 37.002159118652344,
      "activations/layer21_attention_weight_min": -25.592557907104492,
      "activations/layer22_attention_weight_max": 35.52983474731445,
      "activations/layer22_attention_weight_min": -21.783235549926758,
      "activations/layer23_attention_weight_max": 25.924747467041016,
      "activations/layer23_attention_weight_min": -16.798250198364258,
      "activations/layer2_attention_weight_max": 32.89344787597656,
      "activations/layer2_attention_weight_min": -30.104049682617188,
      "activations/layer3_attention_weight_max": 80.84170532226562,
      "activations/layer3_attention_weight_min": -80.10594940185547,
      "activations/layer4_attention_weight_max": 106.24310302734375,
      "activations/layer4_attention_weight_min": -99.6765365600586,
      "activations/layer5_attention_weight_max": 117.72045135498047,
      "activations/layer5_attention_weight_min": -112.39281463623047,
      "activations/layer6_attention_weight_max": 61.35997772216797,
      "activations/layer6_attention_weight_min": -57.43593215942383,
      "activations/layer7_attention_weight_max": 66.97200775146484,
      "activations/layer7_attention_weight_min": -62.39244842529297,
      "activations/layer8_attention_weight_max": 40.54958724975586,
      "activations/layer8_attention_weight_min": -41.49370574951172,
      "activations/layer9_attention_weight_max": 51.899391174316406,
      "activations/layer9_attention_weight_min": -52.44439697265625,
      "epoch": 3.65,
      "learning_rate": 0.00012777272727272727,
      "loss": 2.882,
      "step": 62750
    },
    {
      "activations/layer0_attention_weight_max": 16.13897132873535,
      "activations/layer0_attention_weight_min": -12.509149551391602,
      "activations/layer10_attention_weight_max": 43.80691146850586,
      "activations/layer10_attention_weight_min": -34.00838088989258,
      "activations/layer11_attention_weight_max": 32.66099548339844,
      "activations/layer11_attention_weight_min": -26.365957260131836,
      "activations/layer12_attention_weight_max": 29.721158981323242,
      "activations/layer12_attention_weight_min": -27.08777618408203,
      "activations/layer13_attention_weight_max": 49.021602630615234,
      "activations/layer13_attention_weight_min": -38.15765380859375,
      "activations/layer14_attention_weight_max": 45.27690124511719,
      "activations/layer14_attention_weight_min": -30.503826141357422,
      "activations/layer15_attention_weight_max": 43.744808197021484,
      "activations/layer15_attention_weight_min": -29.57355499267578,
      "activations/layer16_attention_weight_max": 36.58595275878906,
      "activations/layer16_attention_weight_min": -25.012348175048828,
      "activations/layer17_attention_weight_max": 59.61159133911133,
      "activations/layer17_attention_weight_min": -34.696083068847656,
      "activations/layer18_attention_weight_max": 54.747379302978516,
      "activations/layer18_attention_weight_min": -31.568729400634766,
      "activations/layer19_attention_weight_max": 27.103595733642578,
      "activations/layer19_attention_weight_min": -19.11307144165039,
      "activations/layer1_attention_weight_max": 16.920934677124023,
      "activations/layer1_attention_weight_min": -15.735088348388672,
      "activations/layer20_attention_weight_max": 22.4580078125,
      "activations/layer20_attention_weight_min": -16.543846130371094,
      "activations/layer21_attention_weight_max": 45.57819366455078,
      "activations/layer21_attention_weight_min": -28.323293685913086,
      "activations/layer22_attention_weight_max": 35.12154006958008,
      "activations/layer22_attention_weight_min": -22.940887451171875,
      "activations/layer23_attention_weight_max": 27.37646484375,
      "activations/layer23_attention_weight_min": -16.11933135986328,
      "activations/layer2_attention_weight_max": 30.05293846130371,
      "activations/layer2_attention_weight_min": -27.102703094482422,
      "activations/layer3_attention_weight_max": 71.682861328125,
      "activations/layer3_attention_weight_min": -71.21404266357422,
      "activations/layer4_attention_weight_max": 101.33893585205078,
      "activations/layer4_attention_weight_min": -89.39631652832031,
      "activations/layer5_attention_weight_max": 112.82674407958984,
      "activations/layer5_attention_weight_min": -98.672607421875,
      "activations/layer6_attention_weight_max": 60.81956481933594,
      "activations/layer6_attention_weight_min": -53.08159637451172,
      "activations/layer7_attention_weight_max": 65.51696014404297,
      "activations/layer7_attention_weight_min": -61.59870147705078,
      "activations/layer8_attention_weight_max": 40.47956848144531,
      "activations/layer8_attention_weight_min": -37.84684753417969,
      "activations/layer9_attention_weight_max": 49.461822509765625,
      "activations/layer9_attention_weight_min": -46.30957794189453,
      "epoch": 3.65,
      "learning_rate": 0.00012775378787878788,
      "loss": 2.9189,
      "step": 62800
    },
    {
      "activations/layer0_attention_weight_max": 14.393035888671875,
      "activations/layer0_attention_weight_min": -13.327829360961914,
      "activations/layer10_attention_weight_max": 42.274471282958984,
      "activations/layer10_attention_weight_min": -36.56755447387695,
      "activations/layer11_attention_weight_max": 33.66054153442383,
      "activations/layer11_attention_weight_min": -27.367046356201172,
      "activations/layer12_attention_weight_max": 22.33720588684082,
      "activations/layer12_attention_weight_min": -22.388208389282227,
      "activations/layer13_attention_weight_max": 54.0399055480957,
      "activations/layer13_attention_weight_min": -41.15562057495117,
      "activations/layer14_attention_weight_max": 58.637847900390625,
      "activations/layer14_attention_weight_min": -35.40841293334961,
      "activations/layer15_attention_weight_max": 43.07815933227539,
      "activations/layer15_attention_weight_min": -37.20354080200195,
      "activations/layer16_attention_weight_max": 34.937503814697266,
      "activations/layer16_attention_weight_min": -26.878082275390625,
      "activations/layer17_attention_weight_max": 54.52296829223633,
      "activations/layer17_attention_weight_min": -39.07478713989258,
      "activations/layer18_attention_weight_max": 44.65620422363281,
      "activations/layer18_attention_weight_min": -35.351097106933594,
      "activations/layer19_attention_weight_max": 19.89824104309082,
      "activations/layer19_attention_weight_min": -19.581729888916016,
      "activations/layer1_attention_weight_max": 16.898653030395508,
      "activations/layer1_attention_weight_min": -15.603435516357422,
      "activations/layer20_attention_weight_max": 20.816587448120117,
      "activations/layer20_attention_weight_min": -18.59694480895996,
      "activations/layer21_attention_weight_max": 45.80714416503906,
      "activations/layer21_attention_weight_min": -27.516632080078125,
      "activations/layer22_attention_weight_max": 34.01341247558594,
      "activations/layer22_attention_weight_min": -22.14251136779785,
      "activations/layer23_attention_weight_max": 26.432903289794922,
      "activations/layer23_attention_weight_min": -15.740497589111328,
      "activations/layer2_attention_weight_max": 29.05264663696289,
      "activations/layer2_attention_weight_min": -27.540618896484375,
      "activations/layer3_attention_weight_max": 71.82427978515625,
      "activations/layer3_attention_weight_min": -79.22620391845703,
      "activations/layer4_attention_weight_max": 99.1285629272461,
      "activations/layer4_attention_weight_min": -96.79435729980469,
      "activations/layer5_attention_weight_max": 111.02902221679688,
      "activations/layer5_attention_weight_min": -101.29119110107422,
      "activations/layer6_attention_weight_max": 57.87037658691406,
      "activations/layer6_attention_weight_min": -52.640010833740234,
      "activations/layer7_attention_weight_max": 61.83330154418945,
      "activations/layer7_attention_weight_min": -60.519989013671875,
      "activations/layer8_attention_weight_max": 38.59182357788086,
      "activations/layer8_attention_weight_min": -37.60213851928711,
      "activations/layer9_attention_weight_max": 52.802486419677734,
      "activations/layer9_attention_weight_min": -48.976470947265625,
      "epoch": 3.65,
      "learning_rate": 0.00012773484848484847,
      "loss": 2.9003,
      "step": 62850
    },
    {
      "activations/layer0_attention_weight_max": 14.873270034790039,
      "activations/layer0_attention_weight_min": -13.833904266357422,
      "activations/layer10_attention_weight_max": 50.649627685546875,
      "activations/layer10_attention_weight_min": -36.50409698486328,
      "activations/layer11_attention_weight_max": 38.960670471191406,
      "activations/layer11_attention_weight_min": -26.931005477905273,
      "activations/layer12_attention_weight_max": 31.555570602416992,
      "activations/layer12_attention_weight_min": -25.528783798217773,
      "activations/layer13_attention_weight_max": 63.93029022216797,
      "activations/layer13_attention_weight_min": -38.824214935302734,
      "activations/layer14_attention_weight_max": 48.6082878112793,
      "activations/layer14_attention_weight_min": -32.0709114074707,
      "activations/layer15_attention_weight_max": 52.070030212402344,
      "activations/layer15_attention_weight_min": -32.2474479675293,
      "activations/layer16_attention_weight_max": 42.35892105102539,
      "activations/layer16_attention_weight_min": -27.099336624145508,
      "activations/layer17_attention_weight_max": 66.74871826171875,
      "activations/layer17_attention_weight_min": -38.14977264404297,
      "activations/layer18_attention_weight_max": 61.887020111083984,
      "activations/layer18_attention_weight_min": -34.7579345703125,
      "activations/layer19_attention_weight_max": 23.79389762878418,
      "activations/layer19_attention_weight_min": -18.131778717041016,
      "activations/layer1_attention_weight_max": 17.541181564331055,
      "activations/layer1_attention_weight_min": -15.585367202758789,
      "activations/layer20_attention_weight_max": 24.559213638305664,
      "activations/layer20_attention_weight_min": -18.984519958496094,
      "activations/layer21_attention_weight_max": 54.071556091308594,
      "activations/layer21_attention_weight_min": -28.819061279296875,
      "activations/layer22_attention_weight_max": 38.92106246948242,
      "activations/layer22_attention_weight_min": -24.40867042541504,
      "activations/layer23_attention_weight_max": 28.044605255126953,
      "activations/layer23_attention_weight_min": -17.670547485351562,
      "activations/layer2_attention_weight_max": 28.880887985229492,
      "activations/layer2_attention_weight_min": -25.977821350097656,
      "activations/layer3_attention_weight_max": 72.42404174804688,
      "activations/layer3_attention_weight_min": -77.01438903808594,
      "activations/layer4_attention_weight_max": 104.80438232421875,
      "activations/layer4_attention_weight_min": -94.26958465576172,
      "activations/layer5_attention_weight_max": 120.25035858154297,
      "activations/layer5_attention_weight_min": -102.45650482177734,
      "activations/layer6_attention_weight_max": 63.71448516845703,
      "activations/layer6_attention_weight_min": -55.56187057495117,
      "activations/layer7_attention_weight_max": 66.82997131347656,
      "activations/layer7_attention_weight_min": -59.45130920410156,
      "activations/layer8_attention_weight_max": 43.0844841003418,
      "activations/layer8_attention_weight_min": -37.89884567260742,
      "activations/layer9_attention_weight_max": 51.88676834106445,
      "activations/layer9_attention_weight_min": -47.94843673706055,
      "epoch": 3.65,
      "learning_rate": 0.00012771590909090908,
      "loss": 2.8822,
      "step": 62900
    },
    {
      "activations/layer0_attention_weight_max": 13.651545524597168,
      "activations/layer0_attention_weight_min": -14.478360176086426,
      "activations/layer10_attention_weight_max": 42.279354095458984,
      "activations/layer10_attention_weight_min": -32.765743255615234,
      "activations/layer11_attention_weight_max": 33.72850036621094,
      "activations/layer11_attention_weight_min": -27.810562133789062,
      "activations/layer12_attention_weight_max": 30.041854858398438,
      "activations/layer12_attention_weight_min": -26.96873664855957,
      "activations/layer13_attention_weight_max": 42.81781768798828,
      "activations/layer13_attention_weight_min": -36.583351135253906,
      "activations/layer14_attention_weight_max": 35.44658660888672,
      "activations/layer14_attention_weight_min": -29.19093132019043,
      "activations/layer15_attention_weight_max": 45.53880310058594,
      "activations/layer15_attention_weight_min": -32.7769889831543,
      "activations/layer16_attention_weight_max": 36.59811782836914,
      "activations/layer16_attention_weight_min": -25.55440330505371,
      "activations/layer17_attention_weight_max": 59.57907485961914,
      "activations/layer17_attention_weight_min": -37.694801330566406,
      "activations/layer18_attention_weight_max": 53.54206466674805,
      "activations/layer18_attention_weight_min": -35.556854248046875,
      "activations/layer19_attention_weight_max": 20.929359436035156,
      "activations/layer19_attention_weight_min": -18.748945236206055,
      "activations/layer1_attention_weight_max": 16.565750122070312,
      "activations/layer1_attention_weight_min": -15.892428398132324,
      "activations/layer20_attention_weight_max": 20.829912185668945,
      "activations/layer20_attention_weight_min": -19.664770126342773,
      "activations/layer21_attention_weight_max": 50.56681823730469,
      "activations/layer21_attention_weight_min": -32.915348052978516,
      "activations/layer22_attention_weight_max": 34.6886100769043,
      "activations/layer22_attention_weight_min": -22.38672637939453,
      "activations/layer23_attention_weight_max": 24.913555145263672,
      "activations/layer23_attention_weight_min": -16.130788803100586,
      "activations/layer2_attention_weight_max": 29.602018356323242,
      "activations/layer2_attention_weight_min": -25.684253692626953,
      "activations/layer3_attention_weight_max": 75.21278381347656,
      "activations/layer3_attention_weight_min": -77.86209106445312,
      "activations/layer4_attention_weight_max": 104.52408599853516,
      "activations/layer4_attention_weight_min": -91.58419036865234,
      "activations/layer5_attention_weight_max": 116.51180267333984,
      "activations/layer5_attention_weight_min": -96.68714904785156,
      "activations/layer6_attention_weight_max": 61.011844635009766,
      "activations/layer6_attention_weight_min": -53.27986526489258,
      "activations/layer7_attention_weight_max": 65.11822509765625,
      "activations/layer7_attention_weight_min": -60.095741271972656,
      "activations/layer8_attention_weight_max": 39.17512130737305,
      "activations/layer8_attention_weight_min": -36.7747688293457,
      "activations/layer9_attention_weight_max": 55.36579132080078,
      "activations/layer9_attention_weight_min": -54.303192138671875,
      "epoch": 3.66,
      "learning_rate": 0.0001276969696969697,
      "loss": 2.9013,
      "step": 62950
    },
    {
      "activations/layer0_attention_weight_max": 16.23764991760254,
      "activations/layer0_attention_weight_min": -14.084616661071777,
      "activations/layer10_attention_weight_max": 40.2229118347168,
      "activations/layer10_attention_weight_min": -34.114967346191406,
      "activations/layer11_attention_weight_max": 33.33625793457031,
      "activations/layer11_attention_weight_min": -26.81317901611328,
      "activations/layer12_attention_weight_max": 25.82719612121582,
      "activations/layer12_attention_weight_min": -24.16053009033203,
      "activations/layer13_attention_weight_max": 53.77644348144531,
      "activations/layer13_attention_weight_min": -39.24025344848633,
      "activations/layer14_attention_weight_max": 51.803775787353516,
      "activations/layer14_attention_weight_min": -35.26763153076172,
      "activations/layer15_attention_weight_max": 48.0582275390625,
      "activations/layer15_attention_weight_min": -30.437244415283203,
      "activations/layer16_attention_weight_max": 38.43804931640625,
      "activations/layer16_attention_weight_min": -26.528587341308594,
      "activations/layer17_attention_weight_max": 63.20537567138672,
      "activations/layer17_attention_weight_min": -36.93178176879883,
      "activations/layer18_attention_weight_max": 56.09375,
      "activations/layer18_attention_weight_min": -31.88081169128418,
      "activations/layer19_attention_weight_max": 29.6713924407959,
      "activations/layer19_attention_weight_min": -19.0921688079834,
      "activations/layer1_attention_weight_max": 18.10134506225586,
      "activations/layer1_attention_weight_min": -15.483052253723145,
      "activations/layer20_attention_weight_max": 26.807945251464844,
      "activations/layer20_attention_weight_min": -21.21928596496582,
      "activations/layer21_attention_weight_max": 44.00381851196289,
      "activations/layer21_attention_weight_min": -26.081602096557617,
      "activations/layer22_attention_weight_max": 35.4155158996582,
      "activations/layer22_attention_weight_min": -21.90058708190918,
      "activations/layer23_attention_weight_max": 26.963661193847656,
      "activations/layer23_attention_weight_min": -15.723623275756836,
      "activations/layer2_attention_weight_max": 30.249156951904297,
      "activations/layer2_attention_weight_min": -26.635221481323242,
      "activations/layer3_attention_weight_max": 70.353759765625,
      "activations/layer3_attention_weight_min": -72.92508697509766,
      "activations/layer4_attention_weight_max": 93.11204528808594,
      "activations/layer4_attention_weight_min": -86.6771240234375,
      "activations/layer5_attention_weight_max": 107.5892333984375,
      "activations/layer5_attention_weight_min": -97.51429748535156,
      "activations/layer6_attention_weight_max": 55.8294677734375,
      "activations/layer6_attention_weight_min": -50.38106918334961,
      "activations/layer7_attention_weight_max": 62.99835968017578,
      "activations/layer7_attention_weight_min": -55.47662353515625,
      "activations/layer8_attention_weight_max": 40.29963302612305,
      "activations/layer8_attention_weight_min": -34.833736419677734,
      "activations/layer9_attention_weight_max": 59.46696090698242,
      "activations/layer9_attention_weight_min": -44.008365631103516,
      "epoch": 3.66,
      "learning_rate": 0.0001276780303030303,
      "loss": 2.887,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_loss": 2.8359375,
      "eval_runtime": 8.4853,
      "eval_samples_per_second": 506.05,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_openwebtext_loss": 2.8359375,
      "eval_openwebtext_ppl": 17.046373787079762,
      "eval_openwebtext_runtime": 8.4853,
      "eval_openwebtext_samples_per_second": 506.05,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_wikitext_loss": 3.072265625,
      "eval_wikitext_ppl": 21.590763877931302,
      "eval_wikitext_runtime": 1.9196,
      "eval_wikitext_samples_per_second": 237.548,
      "step": 63000
    },
    {
      "epoch": 3.66,
      "eval_lambada_loss": 2.921875,
      "eval_lambada_ppl": 18.576084986220376,
      "eval_lambada_runtime": 9.5528,
      "eval_lambada_samples_per_second": 509.696,
      "step": 63000
    },
    {
      "activations/layer0_attention_weight_max": 16.02180290222168,
      "activations/layer0_attention_weight_min": -14.344846725463867,
      "activations/layer10_attention_weight_max": 42.791717529296875,
      "activations/layer10_attention_weight_min": -33.99461364746094,
      "activations/layer11_attention_weight_max": 35.837120056152344,
      "activations/layer11_attention_weight_min": -26.288650512695312,
      "activations/layer12_attention_weight_max": 25.603797912597656,
      "activations/layer12_attention_weight_min": -23.98606300354004,
      "activations/layer13_attention_weight_max": 55.905643463134766,
      "activations/layer13_attention_weight_min": -39.41752243041992,
      "activations/layer14_attention_weight_max": 57.96059036254883,
      "activations/layer14_attention_weight_min": -34.69835662841797,
      "activations/layer15_attention_weight_max": 45.91933822631836,
      "activations/layer15_attention_weight_min": -32.65122985839844,
      "activations/layer16_attention_weight_max": 37.95518493652344,
      "activations/layer16_attention_weight_min": -27.232797622680664,
      "activations/layer17_attention_weight_max": 62.09019088745117,
      "activations/layer17_attention_weight_min": -36.882164001464844,
      "activations/layer18_attention_weight_max": 56.172119140625,
      "activations/layer18_attention_weight_min": -32.393287658691406,
      "activations/layer19_attention_weight_max": 24.032386779785156,
      "activations/layer19_attention_weight_min": -20.90932273864746,
      "activations/layer1_attention_weight_max": 17.69601821899414,
      "activations/layer1_attention_weight_min": -14.346168518066406,
      "activations/layer20_attention_weight_max": 22.87184715270996,
      "activations/layer20_attention_weight_min": -19.34615707397461,
      "activations/layer21_attention_weight_max": 41.754364013671875,
      "activations/layer21_attention_weight_min": -32.71555709838867,
      "activations/layer22_attention_weight_max": 38.5670166015625,
      "activations/layer22_attention_weight_min": -25.4005184173584,
      "activations/layer23_attention_weight_max": 27.859655380249023,
      "activations/layer23_attention_weight_min": -17.897859573364258,
      "activations/layer2_attention_weight_max": 28.935501098632812,
      "activations/layer2_attention_weight_min": -25.619131088256836,
      "activations/layer3_attention_weight_max": 72.76617431640625,
      "activations/layer3_attention_weight_min": -77.56427764892578,
      "activations/layer4_attention_weight_max": 98.56339263916016,
      "activations/layer4_attention_weight_min": -96.38445281982422,
      "activations/layer5_attention_weight_max": 115.04460906982422,
      "activations/layer5_attention_weight_min": -97.7391357421875,
      "activations/layer6_attention_weight_max": 58.75483322143555,
      "activations/layer6_attention_weight_min": -49.86559295654297,
      "activations/layer7_attention_weight_max": 60.50009536743164,
      "activations/layer7_attention_weight_min": -59.018978118896484,
      "activations/layer8_attention_weight_max": 38.98366928100586,
      "activations/layer8_attention_weight_min": -37.461368560791016,
      "activations/layer9_attention_weight_max": 50.01078414916992,
      "activations/layer9_attention_weight_min": -47.80051803588867,
      "epoch": 3.66,
      "learning_rate": 0.0001276590909090909,
      "loss": 2.8734,
      "step": 63050
    },
    {
      "activations/layer0_attention_weight_max": 14.190837860107422,
      "activations/layer0_attention_weight_min": -14.027459144592285,
      "activations/layer10_attention_weight_max": 37.882896423339844,
      "activations/layer10_attention_weight_min": -36.52238845825195,
      "activations/layer11_attention_weight_max": 29.665983200073242,
      "activations/layer11_attention_weight_min": -27.36379051208496,
      "activations/layer12_attention_weight_max": 22.732969284057617,
      "activations/layer12_attention_weight_min": -28.71059799194336,
      "activations/layer13_attention_weight_max": 45.41600036621094,
      "activations/layer13_attention_weight_min": -33.88029098510742,
      "activations/layer14_attention_weight_max": 42.698570251464844,
      "activations/layer14_attention_weight_min": -30.083650588989258,
      "activations/layer15_attention_weight_max": 43.37977600097656,
      "activations/layer15_attention_weight_min": -32.29695129394531,
      "activations/layer16_attention_weight_max": 35.745445251464844,
      "activations/layer16_attention_weight_min": -24.527788162231445,
      "activations/layer17_attention_weight_max": 57.04121398925781,
      "activations/layer17_attention_weight_min": -35.25627517700195,
      "activations/layer18_attention_weight_max": 46.64533615112305,
      "activations/layer18_attention_weight_min": -31.165678024291992,
      "activations/layer19_attention_weight_max": 19.18893051147461,
      "activations/layer19_attention_weight_min": -18.969308853149414,
      "activations/layer1_attention_weight_max": 19.23649024963379,
      "activations/layer1_attention_weight_min": -14.852546691894531,
      "activations/layer20_attention_weight_max": 24.302061080932617,
      "activations/layer20_attention_weight_min": -22.360706329345703,
      "activations/layer21_attention_weight_max": 43.47330856323242,
      "activations/layer21_attention_weight_min": -27.82375717163086,
      "activations/layer22_attention_weight_max": 34.689937591552734,
      "activations/layer22_attention_weight_min": -23.752506256103516,
      "activations/layer23_attention_weight_max": 31.321128845214844,
      "activations/layer23_attention_weight_min": -17.2215576171875,
      "activations/layer2_attention_weight_max": 28.82331657409668,
      "activations/layer2_attention_weight_min": -25.922496795654297,
      "activations/layer3_attention_weight_max": 72.75666809082031,
      "activations/layer3_attention_weight_min": -78.16271209716797,
      "activations/layer4_attention_weight_max": 96.69376373291016,
      "activations/layer4_attention_weight_min": -89.76122283935547,
      "activations/layer5_attention_weight_max": 108.410400390625,
      "activations/layer5_attention_weight_min": -95.34474182128906,
      "activations/layer6_attention_weight_max": 56.49612808227539,
      "activations/layer6_attention_weight_min": -49.36442565917969,
      "activations/layer7_attention_weight_max": 60.39215087890625,
      "activations/layer7_attention_weight_min": -58.54447937011719,
      "activations/layer8_attention_weight_max": 37.30620193481445,
      "activations/layer8_attention_weight_min": -35.90298843383789,
      "activations/layer9_attention_weight_max": 45.953468322753906,
      "activations/layer9_attention_weight_min": -46.30538558959961,
      "epoch": 3.67,
      "learning_rate": 0.00012764015151515152,
      "loss": 2.8914,
      "step": 63100
    },
    {
      "activations/layer0_attention_weight_max": 13.991265296936035,
      "activations/layer0_attention_weight_min": -13.915502548217773,
      "activations/layer10_attention_weight_max": 42.52070617675781,
      "activations/layer10_attention_weight_min": -35.35063552856445,
      "activations/layer11_attention_weight_max": 36.62956237792969,
      "activations/layer11_attention_weight_min": -26.974943161010742,
      "activations/layer12_attention_weight_max": 25.88451385498047,
      "activations/layer12_attention_weight_min": -29.58698272705078,
      "activations/layer13_attention_weight_max": 80.0159912109375,
      "activations/layer13_attention_weight_min": -46.36484909057617,
      "activations/layer14_attention_weight_max": 66.5060806274414,
      "activations/layer14_attention_weight_min": -35.90974807739258,
      "activations/layer15_attention_weight_max": 46.42185592651367,
      "activations/layer15_attention_weight_min": -30.47944450378418,
      "activations/layer16_attention_weight_max": 39.02553176879883,
      "activations/layer16_attention_weight_min": -25.71988296508789,
      "activations/layer17_attention_weight_max": 69.02347564697266,
      "activations/layer17_attention_weight_min": -38.7781867980957,
      "activations/layer18_attention_weight_max": 55.3692512512207,
      "activations/layer18_attention_weight_min": -32.22252655029297,
      "activations/layer19_attention_weight_max": 22.0792293548584,
      "activations/layer19_attention_weight_min": -18.606544494628906,
      "activations/layer1_attention_weight_max": 18.007219314575195,
      "activations/layer1_attention_weight_min": -14.105295181274414,
      "activations/layer20_attention_weight_max": 21.939208984375,
      "activations/layer20_attention_weight_min": -21.696542739868164,
      "activations/layer21_attention_weight_max": 49.534873962402344,
      "activations/layer21_attention_weight_min": -29.3431453704834,
      "activations/layer22_attention_weight_max": 33.04156494140625,
      "activations/layer22_attention_weight_min": -24.58976936340332,
      "activations/layer23_attention_weight_max": 27.71961212158203,
      "activations/layer23_attention_weight_min": -16.43844985961914,
      "activations/layer2_attention_weight_max": 30.868450164794922,
      "activations/layer2_attention_weight_min": -27.77703857421875,
      "activations/layer3_attention_weight_max": 76.278076171875,
      "activations/layer3_attention_weight_min": -82.96607971191406,
      "activations/layer4_attention_weight_max": 100.141845703125,
      "activations/layer4_attention_weight_min": -93.65388488769531,
      "activations/layer5_attention_weight_max": 115.48985290527344,
      "activations/layer5_attention_weight_min": -100.65005493164062,
      "activations/layer6_attention_weight_max": 59.294700622558594,
      "activations/layer6_attention_weight_min": -52.95362091064453,
      "activations/layer7_attention_weight_max": 64.775146484375,
      "activations/layer7_attention_weight_min": -62.90193176269531,
      "activations/layer8_attention_weight_max": 43.623809814453125,
      "activations/layer8_attention_weight_min": -38.6225700378418,
      "activations/layer9_attention_weight_max": 54.57926559448242,
      "activations/layer9_attention_weight_min": -48.273468017578125,
      "epoch": 3.67,
      "learning_rate": 0.0001276212121212121,
      "loss": 2.8897,
      "step": 63150
    },
    {
      "activations/layer0_attention_weight_max": 15.232197761535645,
      "activations/layer0_attention_weight_min": -14.02585220336914,
      "activations/layer10_attention_weight_max": 43.000694274902344,
      "activations/layer10_attention_weight_min": -34.70611572265625,
      "activations/layer11_attention_weight_max": 35.80531311035156,
      "activations/layer11_attention_weight_min": -26.21685791015625,
      "activations/layer12_attention_weight_max": 26.84853744506836,
      "activations/layer12_attention_weight_min": -25.817487716674805,
      "activations/layer13_attention_weight_max": 63.854248046875,
      "activations/layer13_attention_weight_min": -48.53712844848633,
      "activations/layer14_attention_weight_max": 62.930206298828125,
      "activations/layer14_attention_weight_min": -46.792625427246094,
      "activations/layer15_attention_weight_max": 47.7340087890625,
      "activations/layer15_attention_weight_min": -33.59353256225586,
      "activations/layer16_attention_weight_max": 40.57537841796875,
      "activations/layer16_attention_weight_min": -28.13536262512207,
      "activations/layer17_attention_weight_max": 57.31920623779297,
      "activations/layer17_attention_weight_min": -37.75690841674805,
      "activations/layer18_attention_weight_max": 50.26449966430664,
      "activations/layer18_attention_weight_min": -34.12714767456055,
      "activations/layer19_attention_weight_max": 23.24005126953125,
      "activations/layer19_attention_weight_min": -18.73668670654297,
      "activations/layer1_attention_weight_max": 17.441251754760742,
      "activations/layer1_attention_weight_min": -14.191683769226074,
      "activations/layer20_attention_weight_max": 23.439668655395508,
      "activations/layer20_attention_weight_min": -21.34575653076172,
      "activations/layer21_attention_weight_max": 45.43978500366211,
      "activations/layer21_attention_weight_min": -33.613807678222656,
      "activations/layer22_attention_weight_max": 35.93580627441406,
      "activations/layer22_attention_weight_min": -24.619810104370117,
      "activations/layer23_attention_weight_max": 27.31412124633789,
      "activations/layer23_attention_weight_min": -17.516008377075195,
      "activations/layer2_attention_weight_max": 29.25617027282715,
      "activations/layer2_attention_weight_min": -26.188762664794922,
      "activations/layer3_attention_weight_max": 73.82637023925781,
      "activations/layer3_attention_weight_min": -79.64415740966797,
      "activations/layer4_attention_weight_max": 99.58145141601562,
      "activations/layer4_attention_weight_min": -95.8013687133789,
      "activations/layer5_attention_weight_max": 115.98976135253906,
      "activations/layer5_attention_weight_min": -99.462646484375,
      "activations/layer6_attention_weight_max": 61.84468078613281,
      "activations/layer6_attention_weight_min": -52.77419662475586,
      "activations/layer7_attention_weight_max": 65.40916442871094,
      "activations/layer7_attention_weight_min": -62.3634033203125,
      "activations/layer8_attention_weight_max": 40.45397186279297,
      "activations/layer8_attention_weight_min": -38.853553771972656,
      "activations/layer9_attention_weight_max": 52.551937103271484,
      "activations/layer9_attention_weight_min": -52.49612045288086,
      "epoch": 3.67,
      "learning_rate": 0.00012760227272727272,
      "loss": 2.8888,
      "step": 63200
    },
    {
      "activations/layer0_attention_weight_max": 13.384027481079102,
      "activations/layer0_attention_weight_min": -14.305232048034668,
      "activations/layer10_attention_weight_max": 40.438961029052734,
      "activations/layer10_attention_weight_min": -35.076995849609375,
      "activations/layer11_attention_weight_max": 33.274940490722656,
      "activations/layer11_attention_weight_min": -26.223243713378906,
      "activations/layer12_attention_weight_max": 26.464988708496094,
      "activations/layer12_attention_weight_min": -28.490127563476562,
      "activations/layer13_attention_weight_max": 51.083187103271484,
      "activations/layer13_attention_weight_min": -35.975006103515625,
      "activations/layer14_attention_weight_max": 47.78367233276367,
      "activations/layer14_attention_weight_min": -30.39603614807129,
      "activations/layer15_attention_weight_max": 47.98564147949219,
      "activations/layer15_attention_weight_min": -31.77165412902832,
      "activations/layer16_attention_weight_max": 36.3317756652832,
      "activations/layer16_attention_weight_min": -25.361616134643555,
      "activations/layer17_attention_weight_max": 63.47739791870117,
      "activations/layer17_attention_weight_min": -36.34150314331055,
      "activations/layer18_attention_weight_max": 50.877464294433594,
      "activations/layer18_attention_weight_min": -33.57984924316406,
      "activations/layer19_attention_weight_max": 25.52508544921875,
      "activations/layer19_attention_weight_min": -18.350278854370117,
      "activations/layer1_attention_weight_max": 17.700138092041016,
      "activations/layer1_attention_weight_min": -15.962055206298828,
      "activations/layer20_attention_weight_max": 23.745107650756836,
      "activations/layer20_attention_weight_min": -18.343015670776367,
      "activations/layer21_attention_weight_max": 41.147552490234375,
      "activations/layer21_attention_weight_min": -27.36771583557129,
      "activations/layer22_attention_weight_max": 34.8315544128418,
      "activations/layer22_attention_weight_min": -21.751203536987305,
      "activations/layer23_attention_weight_max": 28.31294822692871,
      "activations/layer23_attention_weight_min": -17.19040870666504,
      "activations/layer2_attention_weight_max": 28.532955169677734,
      "activations/layer2_attention_weight_min": -26.2620849609375,
      "activations/layer3_attention_weight_max": 74.12310028076172,
      "activations/layer3_attention_weight_min": -78.07007598876953,
      "activations/layer4_attention_weight_max": 101.82686614990234,
      "activations/layer4_attention_weight_min": -94.55245208740234,
      "activations/layer5_attention_weight_max": 114.60221862792969,
      "activations/layer5_attention_weight_min": -104.51287078857422,
      "activations/layer6_attention_weight_max": 61.201881408691406,
      "activations/layer6_attention_weight_min": -56.97690963745117,
      "activations/layer7_attention_weight_max": 64.59806060791016,
      "activations/layer7_attention_weight_min": -63.21525955200195,
      "activations/layer8_attention_weight_max": 42.673316955566406,
      "activations/layer8_attention_weight_min": -40.40019989013672,
      "activations/layer9_attention_weight_max": 54.87356185913086,
      "activations/layer9_attention_weight_min": -46.606380462646484,
      "epoch": 3.68,
      "learning_rate": 0.0001275833333333333,
      "loss": 2.8835,
      "step": 63250
    },
    {
      "activations/layer0_attention_weight_max": 14.835548400878906,
      "activations/layer0_attention_weight_min": -14.995810508728027,
      "activations/layer10_attention_weight_max": 38.26918411254883,
      "activations/layer10_attention_weight_min": -32.96681213378906,
      "activations/layer11_attention_weight_max": 30.703582763671875,
      "activations/layer11_attention_weight_min": -25.673643112182617,
      "activations/layer12_attention_weight_max": 28.583280563354492,
      "activations/layer12_attention_weight_min": -25.938709259033203,
      "activations/layer13_attention_weight_max": 45.63083267211914,
      "activations/layer13_attention_weight_min": -38.73919677734375,
      "activations/layer14_attention_weight_max": 40.77525329589844,
      "activations/layer14_attention_weight_min": -33.01487350463867,
      "activations/layer15_attention_weight_max": 48.86698913574219,
      "activations/layer15_attention_weight_min": -31.5011043548584,
      "activations/layer16_attention_weight_max": 35.65332794189453,
      "activations/layer16_attention_weight_min": -25.38687515258789,
      "activations/layer17_attention_weight_max": 54.095619201660156,
      "activations/layer17_attention_weight_min": -36.47489929199219,
      "activations/layer18_attention_weight_max": 50.824466705322266,
      "activations/layer18_attention_weight_min": -32.907657623291016,
      "activations/layer19_attention_weight_max": 21.14029312133789,
      "activations/layer19_attention_weight_min": -19.571443557739258,
      "activations/layer1_attention_weight_max": 16.568166732788086,
      "activations/layer1_attention_weight_min": -13.9243803024292,
      "activations/layer20_attention_weight_max": 22.042470932006836,
      "activations/layer20_attention_weight_min": -20.128122329711914,
      "activations/layer21_attention_weight_max": 40.29584884643555,
      "activations/layer21_attention_weight_min": -28.459104537963867,
      "activations/layer22_attention_weight_max": 40.60823440551758,
      "activations/layer22_attention_weight_min": -23.695737838745117,
      "activations/layer23_attention_weight_max": 28.00320816040039,
      "activations/layer23_attention_weight_min": -17.035381317138672,
      "activations/layer2_attention_weight_max": 29.362552642822266,
      "activations/layer2_attention_weight_min": -26.592405319213867,
      "activations/layer3_attention_weight_max": 73.98750305175781,
      "activations/layer3_attention_weight_min": -82.0507583618164,
      "activations/layer4_attention_weight_max": 98.46308135986328,
      "activations/layer4_attention_weight_min": -95.20182037353516,
      "activations/layer5_attention_weight_max": 114.4725341796875,
      "activations/layer5_attention_weight_min": -103.78334045410156,
      "activations/layer6_attention_weight_max": 60.811283111572266,
      "activations/layer6_attention_weight_min": -52.53312683105469,
      "activations/layer7_attention_weight_max": 62.656944274902344,
      "activations/layer7_attention_weight_min": -60.542999267578125,
      "activations/layer8_attention_weight_max": 40.53879165649414,
      "activations/layer8_attention_weight_min": -38.10850524902344,
      "activations/layer9_attention_weight_max": 52.48771286010742,
      "activations/layer9_attention_weight_min": -46.082706451416016,
      "epoch": 3.68,
      "learning_rate": 0.00012756439393939392,
      "loss": 2.8804,
      "step": 63300
    },
    {
      "activations/layer0_attention_weight_max": 16.423274993896484,
      "activations/layer0_attention_weight_min": -14.386911392211914,
      "activations/layer10_attention_weight_max": 37.250244140625,
      "activations/layer10_attention_weight_min": -33.99875259399414,
      "activations/layer11_attention_weight_max": 29.566364288330078,
      "activations/layer11_attention_weight_min": -27.355724334716797,
      "activations/layer12_attention_weight_max": 22.148706436157227,
      "activations/layer12_attention_weight_min": -23.093074798583984,
      "activations/layer13_attention_weight_max": 44.72792434692383,
      "activations/layer13_attention_weight_min": -32.85601806640625,
      "activations/layer14_attention_weight_max": 45.95082092285156,
      "activations/layer14_attention_weight_min": -34.61560821533203,
      "activations/layer15_attention_weight_max": 43.78196716308594,
      "activations/layer15_attention_weight_min": -30.640535354614258,
      "activations/layer16_attention_weight_max": 34.63858413696289,
      "activations/layer16_attention_weight_min": -27.694454193115234,
      "activations/layer17_attention_weight_max": 60.71717834472656,
      "activations/layer17_attention_weight_min": -37.19351577758789,
      "activations/layer18_attention_weight_max": 51.87593078613281,
      "activations/layer18_attention_weight_min": -32.26056671142578,
      "activations/layer19_attention_weight_max": 22.30056381225586,
      "activations/layer19_attention_weight_min": -19.340993881225586,
      "activations/layer1_attention_weight_max": 17.11924171447754,
      "activations/layer1_attention_weight_min": -14.912768363952637,
      "activations/layer20_attention_weight_max": 18.363384246826172,
      "activations/layer20_attention_weight_min": -20.251407623291016,
      "activations/layer21_attention_weight_max": 43.59165954589844,
      "activations/layer21_attention_weight_min": -31.521785736083984,
      "activations/layer22_attention_weight_max": 34.6707763671875,
      "activations/layer22_attention_weight_min": -22.33483123779297,
      "activations/layer23_attention_weight_max": 22.586624145507812,
      "activations/layer23_attention_weight_min": -18.56056785583496,
      "activations/layer2_attention_weight_max": 31.081134796142578,
      "activations/layer2_attention_weight_min": -27.894622802734375,
      "activations/layer3_attention_weight_max": 75.22982025146484,
      "activations/layer3_attention_weight_min": -80.93605041503906,
      "activations/layer4_attention_weight_max": 101.59754180908203,
      "activations/layer4_attention_weight_min": -95.74445343017578,
      "activations/layer5_attention_weight_max": 108.51904296875,
      "activations/layer5_attention_weight_min": -100.37864685058594,
      "activations/layer6_attention_weight_max": 58.05027389526367,
      "activations/layer6_attention_weight_min": -54.289302825927734,
      "activations/layer7_attention_weight_max": 62.789337158203125,
      "activations/layer7_attention_weight_min": -65.64706420898438,
      "activations/layer8_attention_weight_max": 39.7497673034668,
      "activations/layer8_attention_weight_min": -39.12543869018555,
      "activations/layer9_attention_weight_max": 50.17493438720703,
      "activations/layer9_attention_weight_min": -50.55693435668945,
      "epoch": 3.68,
      "learning_rate": 0.00012754545454545454,
      "loss": 2.8895,
      "step": 63350
    },
    {
      "activations/layer0_attention_weight_max": 13.536361694335938,
      "activations/layer0_attention_weight_min": -14.79991626739502,
      "activations/layer10_attention_weight_max": 36.648590087890625,
      "activations/layer10_attention_weight_min": -31.491533279418945,
      "activations/layer11_attention_weight_max": 32.31737518310547,
      "activations/layer11_attention_weight_min": -26.941526412963867,
      "activations/layer12_attention_weight_max": 26.083194732666016,
      "activations/layer12_attention_weight_min": -26.274328231811523,
      "activations/layer13_attention_weight_max": 47.10401153564453,
      "activations/layer13_attention_weight_min": -36.12307357788086,
      "activations/layer14_attention_weight_max": 38.82938003540039,
      "activations/layer14_attention_weight_min": -30.251201629638672,
      "activations/layer15_attention_weight_max": 47.252567291259766,
      "activations/layer15_attention_weight_min": -33.28664779663086,
      "activations/layer16_attention_weight_max": 40.39682388305664,
      "activations/layer16_attention_weight_min": -27.775863647460938,
      "activations/layer17_attention_weight_max": 57.11289596557617,
      "activations/layer17_attention_weight_min": -36.18003845214844,
      "activations/layer18_attention_weight_max": 51.6041259765625,
      "activations/layer18_attention_weight_min": -31.79958724975586,
      "activations/layer19_attention_weight_max": 21.826942443847656,
      "activations/layer19_attention_weight_min": -18.91983413696289,
      "activations/layer1_attention_weight_max": 18.038219451904297,
      "activations/layer1_attention_weight_min": -14.124370574951172,
      "activations/layer20_attention_weight_max": 20.1608943939209,
      "activations/layer20_attention_weight_min": -21.079723358154297,
      "activations/layer21_attention_weight_max": 36.76875686645508,
      "activations/layer21_attention_weight_min": -30.22667694091797,
      "activations/layer22_attention_weight_max": 37.869014739990234,
      "activations/layer22_attention_weight_min": -22.655956268310547,
      "activations/layer23_attention_weight_max": 21.810203552246094,
      "activations/layer23_attention_weight_min": -19.442346572875977,
      "activations/layer2_attention_weight_max": 29.353370666503906,
      "activations/layer2_attention_weight_min": -26.03689956665039,
      "activations/layer3_attention_weight_max": 74.51799774169922,
      "activations/layer3_attention_weight_min": -77.72504425048828,
      "activations/layer4_attention_weight_max": 102.39995574951172,
      "activations/layer4_attention_weight_min": -94.35836029052734,
      "activations/layer5_attention_weight_max": 105.51156616210938,
      "activations/layer5_attention_weight_min": -99.18276977539062,
      "activations/layer6_attention_weight_max": 56.28465270996094,
      "activations/layer6_attention_weight_min": -48.25762176513672,
      "activations/layer7_attention_weight_max": 67.22553253173828,
      "activations/layer7_attention_weight_min": -66.3426742553711,
      "activations/layer8_attention_weight_max": 38.93357467651367,
      "activations/layer8_attention_weight_min": -36.78075408935547,
      "activations/layer9_attention_weight_max": 55.38888168334961,
      "activations/layer9_attention_weight_min": -52.06977081298828,
      "epoch": 3.68,
      "learning_rate": 0.00012752651515151513,
      "loss": 2.9069,
      "step": 63400
    },
    {
      "activations/layer0_attention_weight_max": 14.407967567443848,
      "activations/layer0_attention_weight_min": -14.854744911193848,
      "activations/layer10_attention_weight_max": 42.194725036621094,
      "activations/layer10_attention_weight_min": -35.39285659790039,
      "activations/layer11_attention_weight_max": 32.44891357421875,
      "activations/layer11_attention_weight_min": -28.266281127929688,
      "activations/layer12_attention_weight_max": 27.3048152923584,
      "activations/layer12_attention_weight_min": -28.124195098876953,
      "activations/layer13_attention_weight_max": 52.42214584350586,
      "activations/layer13_attention_weight_min": -42.785133361816406,
      "activations/layer14_attention_weight_max": 44.278350830078125,
      "activations/layer14_attention_weight_min": -36.37868881225586,
      "activations/layer15_attention_weight_max": 46.54594039916992,
      "activations/layer15_attention_weight_min": -30.74074363708496,
      "activations/layer16_attention_weight_max": 38.369773864746094,
      "activations/layer16_attention_weight_min": -28.084064483642578,
      "activations/layer17_attention_weight_max": 62.13070297241211,
      "activations/layer17_attention_weight_min": -38.741363525390625,
      "activations/layer18_attention_weight_max": 50.94947814941406,
      "activations/layer18_attention_weight_min": -30.70445442199707,
      "activations/layer19_attention_weight_max": 22.546581268310547,
      "activations/layer19_attention_weight_min": -19.59956169128418,
      "activations/layer1_attention_weight_max": 17.456228256225586,
      "activations/layer1_attention_weight_min": -14.814024925231934,
      "activations/layer20_attention_weight_max": 25.792224884033203,
      "activations/layer20_attention_weight_min": -18.728967666625977,
      "activations/layer21_attention_weight_max": 38.83696365356445,
      "activations/layer21_attention_weight_min": -27.08349609375,
      "activations/layer22_attention_weight_max": 36.18373107910156,
      "activations/layer22_attention_weight_min": -23.339508056640625,
      "activations/layer23_attention_weight_max": 24.26320457458496,
      "activations/layer23_attention_weight_min": -15.89407730102539,
      "activations/layer2_attention_weight_max": 28.662214279174805,
      "activations/layer2_attention_weight_min": -27.51223373413086,
      "activations/layer3_attention_weight_max": 74.08148193359375,
      "activations/layer3_attention_weight_min": -79.89392852783203,
      "activations/layer4_attention_weight_max": 104.08614349365234,
      "activations/layer4_attention_weight_min": -93.93083190917969,
      "activations/layer5_attention_weight_max": 114.45793151855469,
      "activations/layer5_attention_weight_min": -104.32949829101562,
      "activations/layer6_attention_weight_max": 62.635223388671875,
      "activations/layer6_attention_weight_min": -54.68942642211914,
      "activations/layer7_attention_weight_max": 67.06047821044922,
      "activations/layer7_attention_weight_min": -59.85371017456055,
      "activations/layer8_attention_weight_max": 42.42848205566406,
      "activations/layer8_attention_weight_min": -38.26443862915039,
      "activations/layer9_attention_weight_max": 50.079341888427734,
      "activations/layer9_attention_weight_min": -46.4995002746582,
      "epoch": 3.69,
      "learning_rate": 0.00012750757575757574,
      "loss": 2.8727,
      "step": 63450
    },
    {
      "activations/layer0_attention_weight_max": 14.060457229614258,
      "activations/layer0_attention_weight_min": -14.450068473815918,
      "activations/layer10_attention_weight_max": 44.94997024536133,
      "activations/layer10_attention_weight_min": -35.346824645996094,
      "activations/layer11_attention_weight_max": 35.55129623413086,
      "activations/layer11_attention_weight_min": -26.49569320678711,
      "activations/layer12_attention_weight_max": 24.662715911865234,
      "activations/layer12_attention_weight_min": -26.83945083618164,
      "activations/layer13_attention_weight_max": 44.966590881347656,
      "activations/layer13_attention_weight_min": -34.64140319824219,
      "activations/layer14_attention_weight_max": 46.90005874633789,
      "activations/layer14_attention_weight_min": -32.21845245361328,
      "activations/layer15_attention_weight_max": 45.9271125793457,
      "activations/layer15_attention_weight_min": -31.978836059570312,
      "activations/layer16_attention_weight_max": 35.8943977355957,
      "activations/layer16_attention_weight_min": -26.74679946899414,
      "activations/layer17_attention_weight_max": 63.07304763793945,
      "activations/layer17_attention_weight_min": -34.977603912353516,
      "activations/layer18_attention_weight_max": 56.966224670410156,
      "activations/layer18_attention_weight_min": -33.89719009399414,
      "activations/layer19_attention_weight_max": 22.251333236694336,
      "activations/layer19_attention_weight_min": -18.011899948120117,
      "activations/layer1_attention_weight_max": 17.09929847717285,
      "activations/layer1_attention_weight_min": -13.696067810058594,
      "activations/layer20_attention_weight_max": 20.119678497314453,
      "activations/layer20_attention_weight_min": -21.416730880737305,
      "activations/layer21_attention_weight_max": 38.54405212402344,
      "activations/layer21_attention_weight_min": -30.828277587890625,
      "activations/layer22_attention_weight_max": 34.71183395385742,
      "activations/layer22_attention_weight_min": -23.2972469329834,
      "activations/layer23_attention_weight_max": 27.746749877929688,
      "activations/layer23_attention_weight_min": -16.732324600219727,
      "activations/layer2_attention_weight_max": 29.318988800048828,
      "activations/layer2_attention_weight_min": -26.66119384765625,
      "activations/layer3_attention_weight_max": 77.75724029541016,
      "activations/layer3_attention_weight_min": -77.92481994628906,
      "activations/layer4_attention_weight_max": 101.54287719726562,
      "activations/layer4_attention_weight_min": -95.0150146484375,
      "activations/layer5_attention_weight_max": 117.51537322998047,
      "activations/layer5_attention_weight_min": -100.33589172363281,
      "activations/layer6_attention_weight_max": 62.109466552734375,
      "activations/layer6_attention_weight_min": -52.09796905517578,
      "activations/layer7_attention_weight_max": 63.749576568603516,
      "activations/layer7_attention_weight_min": -61.103294372558594,
      "activations/layer8_attention_weight_max": 41.6086311340332,
      "activations/layer8_attention_weight_min": -39.640804290771484,
      "activations/layer9_attention_weight_max": 48.03666305541992,
      "activations/layer9_attention_weight_min": -46.601226806640625,
      "epoch": 3.69,
      "learning_rate": 0.00012748863636363636,
      "loss": 2.907,
      "step": 63500
    },
    {
      "activations/layer0_attention_weight_max": 15.498112678527832,
      "activations/layer0_attention_weight_min": -15.059638977050781,
      "activations/layer10_attention_weight_max": 48.165802001953125,
      "activations/layer10_attention_weight_min": -38.494903564453125,
      "activations/layer11_attention_weight_max": 36.63661193847656,
      "activations/layer11_attention_weight_min": -27.0556640625,
      "activations/layer12_attention_weight_max": 25.576200485229492,
      "activations/layer12_attention_weight_min": -23.819812774658203,
      "activations/layer13_attention_weight_max": 53.720497131347656,
      "activations/layer13_attention_weight_min": -33.39104080200195,
      "activations/layer14_attention_weight_max": 49.10910415649414,
      "activations/layer14_attention_weight_min": -31.06551742553711,
      "activations/layer15_attention_weight_max": 44.71922302246094,
      "activations/layer15_attention_weight_min": -29.651905059814453,
      "activations/layer16_attention_weight_max": 44.313045501708984,
      "activations/layer16_attention_weight_min": -25.655025482177734,
      "activations/layer17_attention_weight_max": 65.77933502197266,
      "activations/layer17_attention_weight_min": -37.98432159423828,
      "activations/layer18_attention_weight_max": 61.25090026855469,
      "activations/layer18_attention_weight_min": -32.3764762878418,
      "activations/layer19_attention_weight_max": 25.571401596069336,
      "activations/layer19_attention_weight_min": -21.210214614868164,
      "activations/layer1_attention_weight_max": 17.099102020263672,
      "activations/layer1_attention_weight_min": -14.039803504943848,
      "activations/layer20_attention_weight_max": 27.076255798339844,
      "activations/layer20_attention_weight_min": -20.941991806030273,
      "activations/layer21_attention_weight_max": 43.96589279174805,
      "activations/layer21_attention_weight_min": -28.376480102539062,
      "activations/layer22_attention_weight_max": 39.986572265625,
      "activations/layer22_attention_weight_min": -25.09844970703125,
      "activations/layer23_attention_weight_max": 34.38264846801758,
      "activations/layer23_attention_weight_min": -17.32126808166504,
      "activations/layer2_attention_weight_max": 31.027172088623047,
      "activations/layer2_attention_weight_min": -27.435672760009766,
      "activations/layer3_attention_weight_max": 77.59071350097656,
      "activations/layer3_attention_weight_min": -80.75882720947266,
      "activations/layer4_attention_weight_max": 107.5123519897461,
      "activations/layer4_attention_weight_min": -96.15227508544922,
      "activations/layer5_attention_weight_max": 125.22563934326172,
      "activations/layer5_attention_weight_min": -104.04055786132812,
      "activations/layer6_attention_weight_max": 64.07007598876953,
      "activations/layer6_attention_weight_min": -57.76072311401367,
      "activations/layer7_attention_weight_max": 65.30304718017578,
      "activations/layer7_attention_weight_min": -60.375091552734375,
      "activations/layer8_attention_weight_max": 39.87320327758789,
      "activations/layer8_attention_weight_min": -39.64783477783203,
      "activations/layer9_attention_weight_max": 48.86997604370117,
      "activations/layer9_attention_weight_min": -46.4672966003418,
      "epoch": 3.69,
      "learning_rate": 0.00012746969696969694,
      "loss": 2.8822,
      "step": 63550
    },
    {
      "activations/layer0_attention_weight_max": 14.35246467590332,
      "activations/layer0_attention_weight_min": -14.738527297973633,
      "activations/layer10_attention_weight_max": 41.713043212890625,
      "activations/layer10_attention_weight_min": -32.89000701904297,
      "activations/layer11_attention_weight_max": 37.908905029296875,
      "activations/layer11_attention_weight_min": -27.462087631225586,
      "activations/layer12_attention_weight_max": 44.28781509399414,
      "activations/layer12_attention_weight_min": -26.363540649414062,
      "activations/layer13_attention_weight_max": 62.015037536621094,
      "activations/layer13_attention_weight_min": -41.77169418334961,
      "activations/layer14_attention_weight_max": 50.913639068603516,
      "activations/layer14_attention_weight_min": -33.06429672241211,
      "activations/layer15_attention_weight_max": 50.031490325927734,
      "activations/layer15_attention_weight_min": -33.35695266723633,
      "activations/layer16_attention_weight_max": 37.279537200927734,
      "activations/layer16_attention_weight_min": -24.222909927368164,
      "activations/layer17_attention_weight_max": 56.0474739074707,
      "activations/layer17_attention_weight_min": -35.61140441894531,
      "activations/layer18_attention_weight_max": 47.34113693237305,
      "activations/layer18_attention_weight_min": -31.530380249023438,
      "activations/layer19_attention_weight_max": 22.458965301513672,
      "activations/layer19_attention_weight_min": -21.068456649780273,
      "activations/layer1_attention_weight_max": 18.513870239257812,
      "activations/layer1_attention_weight_min": -14.147199630737305,
      "activations/layer20_attention_weight_max": 20.075803756713867,
      "activations/layer20_attention_weight_min": -18.665233612060547,
      "activations/layer21_attention_weight_max": 39.18632125854492,
      "activations/layer21_attention_weight_min": -27.842998504638672,
      "activations/layer22_attention_weight_max": 34.72272491455078,
      "activations/layer22_attention_weight_min": -23.157760620117188,
      "activations/layer23_attention_weight_max": 25.956663131713867,
      "activations/layer23_attention_weight_min": -17.08607292175293,
      "activations/layer2_attention_weight_max": 29.899526596069336,
      "activations/layer2_attention_weight_min": -27.015575408935547,
      "activations/layer3_attention_weight_max": 74.58849334716797,
      "activations/layer3_attention_weight_min": -80.76704406738281,
      "activations/layer4_attention_weight_max": 99.49498748779297,
      "activations/layer4_attention_weight_min": -92.03084564208984,
      "activations/layer5_attention_weight_max": 112.79634857177734,
      "activations/layer5_attention_weight_min": -98.24752044677734,
      "activations/layer6_attention_weight_max": 59.013370513916016,
      "activations/layer6_attention_weight_min": -51.40747833251953,
      "activations/layer7_attention_weight_max": 62.20037078857422,
      "activations/layer7_attention_weight_min": -57.78152084350586,
      "activations/layer8_attention_weight_max": 40.69721984863281,
      "activations/layer8_attention_weight_min": -37.28853225708008,
      "activations/layer9_attention_weight_max": 50.14979553222656,
      "activations/layer9_attention_weight_min": -47.099449157714844,
      "epoch": 3.7,
      "learning_rate": 0.00012745075757575756,
      "loss": 2.8739,
      "step": 63600
    },
    {
      "activations/layer0_attention_weight_max": 15.145258903503418,
      "activations/layer0_attention_weight_min": -15.47546100616455,
      "activations/layer10_attention_weight_max": 41.558258056640625,
      "activations/layer10_attention_weight_min": -33.591373443603516,
      "activations/layer11_attention_weight_max": 41.235130310058594,
      "activations/layer11_attention_weight_min": -28.8967342376709,
      "activations/layer12_attention_weight_max": 25.15550422668457,
      "activations/layer12_attention_weight_min": -24.253293991088867,
      "activations/layer13_attention_weight_max": 57.464908599853516,
      "activations/layer13_attention_weight_min": -37.3804931640625,
      "activations/layer14_attention_weight_max": 56.535675048828125,
      "activations/layer14_attention_weight_min": -35.33623123168945,
      "activations/layer15_attention_weight_max": 54.280662536621094,
      "activations/layer15_attention_weight_min": -32.462196350097656,
      "activations/layer16_attention_weight_max": 38.16158676147461,
      "activations/layer16_attention_weight_min": -26.593692779541016,
      "activations/layer17_attention_weight_max": 60.28254699707031,
      "activations/layer17_attention_weight_min": -37.181854248046875,
      "activations/layer18_attention_weight_max": 54.5993537902832,
      "activations/layer18_attention_weight_min": -34.09920120239258,
      "activations/layer19_attention_weight_max": 26.26837158203125,
      "activations/layer19_attention_weight_min": -17.8167781829834,
      "activations/layer1_attention_weight_max": 16.828529357910156,
      "activations/layer1_attention_weight_min": -15.580255508422852,
      "activations/layer20_attention_weight_max": 20.02495002746582,
      "activations/layer20_attention_weight_min": -19.29154396057129,
      "activations/layer21_attention_weight_max": 43.31452941894531,
      "activations/layer21_attention_weight_min": -28.19930076599121,
      "activations/layer22_attention_weight_max": 38.34053039550781,
      "activations/layer22_attention_weight_min": -25.37928581237793,
      "activations/layer23_attention_weight_max": 26.265949249267578,
      "activations/layer23_attention_weight_min": -18.44080352783203,
      "activations/layer2_attention_weight_max": 30.78826141357422,
      "activations/layer2_attention_weight_min": -28.42353057861328,
      "activations/layer3_attention_weight_max": 74.60514831542969,
      "activations/layer3_attention_weight_min": -83.97660827636719,
      "activations/layer4_attention_weight_max": 106.47755432128906,
      "activations/layer4_attention_weight_min": -101.74417114257812,
      "activations/layer5_attention_weight_max": 124.67915344238281,
      "activations/layer5_attention_weight_min": -110.12801361083984,
      "activations/layer6_attention_weight_max": 64.66561126708984,
      "activations/layer6_attention_weight_min": -56.10687255859375,
      "activations/layer7_attention_weight_max": 75.32096862792969,
      "activations/layer7_attention_weight_min": -67.33416748046875,
      "activations/layer8_attention_weight_max": 46.91874313354492,
      "activations/layer8_attention_weight_min": -37.35249328613281,
      "activations/layer9_attention_weight_max": 69.36051177978516,
      "activations/layer9_attention_weight_min": -49.30575180053711,
      "epoch": 3.7,
      "learning_rate": 0.00012743181818181817,
      "loss": 2.8875,
      "step": 63650
    },
    {
      "activations/layer0_attention_weight_max": 14.240889549255371,
      "activations/layer0_attention_weight_min": -15.099503517150879,
      "activations/layer10_attention_weight_max": 38.577877044677734,
      "activations/layer10_attention_weight_min": -33.278072357177734,
      "activations/layer11_attention_weight_max": 34.772247314453125,
      "activations/layer11_attention_weight_min": -25.644317626953125,
      "activations/layer12_attention_weight_max": 26.736007690429688,
      "activations/layer12_attention_weight_min": -27.04690170288086,
      "activations/layer13_attention_weight_max": 54.43539047241211,
      "activations/layer13_attention_weight_min": -37.22068405151367,
      "activations/layer14_attention_weight_max": 57.96711349487305,
      "activations/layer14_attention_weight_min": -32.43716049194336,
      "activations/layer15_attention_weight_max": 45.13031005859375,
      "activations/layer15_attention_weight_min": -32.993797302246094,
      "activations/layer16_attention_weight_max": 34.931365966796875,
      "activations/layer16_attention_weight_min": -26.90892219543457,
      "activations/layer17_attention_weight_max": 57.44792938232422,
      "activations/layer17_attention_weight_min": -38.03419494628906,
      "activations/layer18_attention_weight_max": 47.957237243652344,
      "activations/layer18_attention_weight_min": -32.35649490356445,
      "activations/layer19_attention_weight_max": 24.86086082458496,
      "activations/layer19_attention_weight_min": -18.900638580322266,
      "activations/layer1_attention_weight_max": 19.240476608276367,
      "activations/layer1_attention_weight_min": -13.735149383544922,
      "activations/layer20_attention_weight_max": 21.988584518432617,
      "activations/layer20_attention_weight_min": -18.763029098510742,
      "activations/layer21_attention_weight_max": 41.12458801269531,
      "activations/layer21_attention_weight_min": -30.587120056152344,
      "activations/layer22_attention_weight_max": 36.565120697021484,
      "activations/layer22_attention_weight_min": -25.495494842529297,
      "activations/layer23_attention_weight_max": 26.23712730407715,
      "activations/layer23_attention_weight_min": -17.905702590942383,
      "activations/layer2_attention_weight_max": 28.613414764404297,
      "activations/layer2_attention_weight_min": -27.083778381347656,
      "activations/layer3_attention_weight_max": 77.13958740234375,
      "activations/layer3_attention_weight_min": -79.2071762084961,
      "activations/layer4_attention_weight_max": 106.18047332763672,
      "activations/layer4_attention_weight_min": -92.95628356933594,
      "activations/layer5_attention_weight_max": 113.31672668457031,
      "activations/layer5_attention_weight_min": -102.589111328125,
      "activations/layer6_attention_weight_max": 59.078529357910156,
      "activations/layer6_attention_weight_min": -52.77287673950195,
      "activations/layer7_attention_weight_max": 63.83145523071289,
      "activations/layer7_attention_weight_min": -61.648887634277344,
      "activations/layer8_attention_weight_max": 42.68955993652344,
      "activations/layer8_attention_weight_min": -37.30110549926758,
      "activations/layer9_attention_weight_max": 57.83150863647461,
      "activations/layer9_attention_weight_min": -47.93964767456055,
      "epoch": 3.7,
      "learning_rate": 0.00012741287878787876,
      "loss": 2.8801,
      "step": 63700
    },
    {
      "activations/layer0_attention_weight_max": 14.743453025817871,
      "activations/layer0_attention_weight_min": -15.573200225830078,
      "activations/layer10_attention_weight_max": 40.752960205078125,
      "activations/layer10_attention_weight_min": -36.0,
      "activations/layer11_attention_weight_max": 35.445682525634766,
      "activations/layer11_attention_weight_min": -26.81810760498047,
      "activations/layer12_attention_weight_max": 27.270164489746094,
      "activations/layer12_attention_weight_min": -25.799848556518555,
      "activations/layer13_attention_weight_max": 52.06709671020508,
      "activations/layer13_attention_weight_min": -36.29859161376953,
      "activations/layer14_attention_weight_max": 45.61697769165039,
      "activations/layer14_attention_weight_min": -32.63666534423828,
      "activations/layer15_attention_weight_max": 45.94771194458008,
      "activations/layer15_attention_weight_min": -31.809688568115234,
      "activations/layer16_attention_weight_max": 38.96560287475586,
      "activations/layer16_attention_weight_min": -26.35748863220215,
      "activations/layer17_attention_weight_max": 65.61882019042969,
      "activations/layer17_attention_weight_min": -36.69970703125,
      "activations/layer18_attention_weight_max": 53.08076095581055,
      "activations/layer18_attention_weight_min": -34.13114547729492,
      "activations/layer19_attention_weight_max": 21.820823669433594,
      "activations/layer19_attention_weight_min": -18.249135971069336,
      "activations/layer1_attention_weight_max": 18.202960968017578,
      "activations/layer1_attention_weight_min": -17.056589126586914,
      "activations/layer20_attention_weight_max": 22.95404815673828,
      "activations/layer20_attention_weight_min": -20.752859115600586,
      "activations/layer21_attention_weight_max": 39.718204498291016,
      "activations/layer21_attention_weight_min": -28.98043441772461,
      "activations/layer22_attention_weight_max": 40.30957794189453,
      "activations/layer22_attention_weight_min": -27.891904830932617,
      "activations/layer23_attention_weight_max": 30.313993453979492,
      "activations/layer23_attention_weight_min": -20.079116821289062,
      "activations/layer2_attention_weight_max": 29.818897247314453,
      "activations/layer2_attention_weight_min": -28.369911193847656,
      "activations/layer3_attention_weight_max": 78.93064880371094,
      "activations/layer3_attention_weight_min": -79.48005676269531,
      "activations/layer4_attention_weight_max": 103.83257293701172,
      "activations/layer4_attention_weight_min": -91.21121215820312,
      "activations/layer5_attention_weight_max": 115.61943054199219,
      "activations/layer5_attention_weight_min": -97.67636108398438,
      "activations/layer6_attention_weight_max": 62.492191314697266,
      "activations/layer6_attention_weight_min": -51.40291976928711,
      "activations/layer7_attention_weight_max": 67.36319732666016,
      "activations/layer7_attention_weight_min": -57.79412078857422,
      "activations/layer8_attention_weight_max": 40.02873611450195,
      "activations/layer8_attention_weight_min": -38.22675323486328,
      "activations/layer9_attention_weight_max": 54.30538558959961,
      "activations/layer9_attention_weight_min": -44.46241760253906,
      "epoch": 3.7,
      "learning_rate": 0.00012739393939393938,
      "loss": 2.9045,
      "step": 63750
    },
    {
      "activations/layer0_attention_weight_max": 14.7637939453125,
      "activations/layer0_attention_weight_min": -15.31914234161377,
      "activations/layer10_attention_weight_max": 38.24470901489258,
      "activations/layer10_attention_weight_min": -33.34511947631836,
      "activations/layer11_attention_weight_max": 31.346389770507812,
      "activations/layer11_attention_weight_min": -25.216773986816406,
      "activations/layer12_attention_weight_max": 25.018875122070312,
      "activations/layer12_attention_weight_min": -28.223899841308594,
      "activations/layer13_attention_weight_max": 48.11643981933594,
      "activations/layer13_attention_weight_min": -39.28900146484375,
      "activations/layer14_attention_weight_max": 50.329925537109375,
      "activations/layer14_attention_weight_min": -43.02153778076172,
      "activations/layer15_attention_weight_max": 46.37296676635742,
      "activations/layer15_attention_weight_min": -33.336238861083984,
      "activations/layer16_attention_weight_max": 38.97828674316406,
      "activations/layer16_attention_weight_min": -26.700395584106445,
      "activations/layer17_attention_weight_max": 61.54391098022461,
      "activations/layer17_attention_weight_min": -36.6993293762207,
      "activations/layer18_attention_weight_max": 51.403072357177734,
      "activations/layer18_attention_weight_min": -33.99346923828125,
      "activations/layer19_attention_weight_max": 22.525882720947266,
      "activations/layer19_attention_weight_min": -20.960657119750977,
      "activations/layer1_attention_weight_max": 17.497419357299805,
      "activations/layer1_attention_weight_min": -14.363000869750977,
      "activations/layer20_attention_weight_max": 23.317420959472656,
      "activations/layer20_attention_weight_min": -20.078838348388672,
      "activations/layer21_attention_weight_max": 41.72224807739258,
      "activations/layer21_attention_weight_min": -27.6693172454834,
      "activations/layer22_attention_weight_max": 34.906490325927734,
      "activations/layer22_attention_weight_min": -24.26411247253418,
      "activations/layer23_attention_weight_max": 24.925090789794922,
      "activations/layer23_attention_weight_min": -19.943954467773438,
      "activations/layer2_attention_weight_max": 30.327077865600586,
      "activations/layer2_attention_weight_min": -27.893409729003906,
      "activations/layer3_attention_weight_max": 76.91695404052734,
      "activations/layer3_attention_weight_min": -80.1816177368164,
      "activations/layer4_attention_weight_max": 102.89989471435547,
      "activations/layer4_attention_weight_min": -93.68582916259766,
      "activations/layer5_attention_weight_max": 111.99066162109375,
      "activations/layer5_attention_weight_min": -100.76603698730469,
      "activations/layer6_attention_weight_max": 59.6860237121582,
      "activations/layer6_attention_weight_min": -51.99937057495117,
      "activations/layer7_attention_weight_max": 60.058658599853516,
      "activations/layer7_attention_weight_min": -56.40715026855469,
      "activations/layer8_attention_weight_max": 38.2478141784668,
      "activations/layer8_attention_weight_min": -39.07023620605469,
      "activations/layer9_attention_weight_max": 48.24132537841797,
      "activations/layer9_attention_weight_min": -44.879459381103516,
      "epoch": 3.71,
      "learning_rate": 0.000127375,
      "loss": 2.8844,
      "step": 63800
    },
    {
      "activations/layer0_attention_weight_max": 14.479969024658203,
      "activations/layer0_attention_weight_min": -15.214869499206543,
      "activations/layer10_attention_weight_max": 38.73268508911133,
      "activations/layer10_attention_weight_min": -36.064697265625,
      "activations/layer11_attention_weight_max": 34.76415252685547,
      "activations/layer11_attention_weight_min": -26.92292594909668,
      "activations/layer12_attention_weight_max": 26.138385772705078,
      "activations/layer12_attention_weight_min": -26.053781509399414,
      "activations/layer13_attention_weight_max": 59.4280891418457,
      "activations/layer13_attention_weight_min": -42.35099411010742,
      "activations/layer14_attention_weight_max": 52.39731216430664,
      "activations/layer14_attention_weight_min": -36.99131774902344,
      "activations/layer15_attention_weight_max": 41.7376594543457,
      "activations/layer15_attention_weight_min": -32.12428283691406,
      "activations/layer16_attention_weight_max": 35.6568489074707,
      "activations/layer16_attention_weight_min": -26.239469528198242,
      "activations/layer17_attention_weight_max": 57.09054183959961,
      "activations/layer17_attention_weight_min": -39.2476692199707,
      "activations/layer18_attention_weight_max": 48.83465576171875,
      "activations/layer18_attention_weight_min": -33.576568603515625,
      "activations/layer19_attention_weight_max": 21.206096649169922,
      "activations/layer19_attention_weight_min": -18.20689582824707,
      "activations/layer1_attention_weight_max": 17.48163604736328,
      "activations/layer1_attention_weight_min": -16.135236740112305,
      "activations/layer20_attention_weight_max": 20.033201217651367,
      "activations/layer20_attention_weight_min": -19.78154945373535,
      "activations/layer21_attention_weight_max": 39.3388557434082,
      "activations/layer21_attention_weight_min": -30.614572525024414,
      "activations/layer22_attention_weight_max": 34.016746520996094,
      "activations/layer22_attention_weight_min": -22.18193817138672,
      "activations/layer23_attention_weight_max": 24.13780975341797,
      "activations/layer23_attention_weight_min": -15.91580867767334,
      "activations/layer2_attention_weight_max": 30.53424835205078,
      "activations/layer2_attention_weight_min": -28.81087875366211,
      "activations/layer3_attention_weight_max": 80.82386016845703,
      "activations/layer3_attention_weight_min": -80.43428802490234,
      "activations/layer4_attention_weight_max": 103.0721435546875,
      "activations/layer4_attention_weight_min": -96.2646255493164,
      "activations/layer5_attention_weight_max": 112.670166015625,
      "activations/layer5_attention_weight_min": -99.19148254394531,
      "activations/layer6_attention_weight_max": 57.83683776855469,
      "activations/layer6_attention_weight_min": -53.073360443115234,
      "activations/layer7_attention_weight_max": 64.18233489990234,
      "activations/layer7_attention_weight_min": -57.204620361328125,
      "activations/layer8_attention_weight_max": 40.42277908325195,
      "activations/layer8_attention_weight_min": -38.630008697509766,
      "activations/layer9_attention_weight_max": 52.0210075378418,
      "activations/layer9_attention_weight_min": -46.81924057006836,
      "epoch": 3.71,
      "learning_rate": 0.0001273560606060606,
      "loss": 2.8725,
      "step": 63850
    },
    {
      "activations/layer0_attention_weight_max": 16.138303756713867,
      "activations/layer0_attention_weight_min": -14.365479469299316,
      "activations/layer10_attention_weight_max": 47.069637298583984,
      "activations/layer10_attention_weight_min": -35.849822998046875,
      "activations/layer11_attention_weight_max": 39.25260925292969,
      "activations/layer11_attention_weight_min": -27.429502487182617,
      "activations/layer12_attention_weight_max": 34.75726318359375,
      "activations/layer12_attention_weight_min": -27.68069839477539,
      "activations/layer13_attention_weight_max": 76.64076232910156,
      "activations/layer13_attention_weight_min": -40.16901779174805,
      "activations/layer14_attention_weight_max": 75.93299865722656,
      "activations/layer14_attention_weight_min": -36.36695098876953,
      "activations/layer15_attention_weight_max": 47.78690719604492,
      "activations/layer15_attention_weight_min": -31.773155212402344,
      "activations/layer16_attention_weight_max": 36.3111572265625,
      "activations/layer16_attention_weight_min": -26.579862594604492,
      "activations/layer17_attention_weight_max": 59.50177001953125,
      "activations/layer17_attention_weight_min": -38.46858596801758,
      "activations/layer18_attention_weight_max": 50.647308349609375,
      "activations/layer18_attention_weight_min": -34.96418380737305,
      "activations/layer19_attention_weight_max": 30.60616111755371,
      "activations/layer19_attention_weight_min": -20.552297592163086,
      "activations/layer1_attention_weight_max": 18.45335578918457,
      "activations/layer1_attention_weight_min": -14.239721298217773,
      "activations/layer20_attention_weight_max": 23.995704650878906,
      "activations/layer20_attention_weight_min": -21.67463493347168,
      "activations/layer21_attention_weight_max": 39.44385528564453,
      "activations/layer21_attention_weight_min": -28.792211532592773,
      "activations/layer22_attention_weight_max": 35.32076644897461,
      "activations/layer22_attention_weight_min": -26.61612892150879,
      "activations/layer23_attention_weight_max": 29.20351219177246,
      "activations/layer23_attention_weight_min": -19.783523559570312,
      "activations/layer2_attention_weight_max": 31.016050338745117,
      "activations/layer2_attention_weight_min": -28.546916961669922,
      "activations/layer3_attention_weight_max": 76.46429443359375,
      "activations/layer3_attention_weight_min": -79.45703125,
      "activations/layer4_attention_weight_max": 101.35719299316406,
      "activations/layer4_attention_weight_min": -95.20305633544922,
      "activations/layer5_attention_weight_max": 114.4776840209961,
      "activations/layer5_attention_weight_min": -98.7369613647461,
      "activations/layer6_attention_weight_max": 57.44197082519531,
      "activations/layer6_attention_weight_min": -52.75949478149414,
      "activations/layer7_attention_weight_max": 73.64640808105469,
      "activations/layer7_attention_weight_min": -59.54157638549805,
      "activations/layer8_attention_weight_max": 48.20463562011719,
      "activations/layer8_attention_weight_min": -38.0058708190918,
      "activations/layer9_attention_weight_max": 67.6179428100586,
      "activations/layer9_attention_weight_min": -53.189727783203125,
      "epoch": 3.71,
      "learning_rate": 0.0001273371212121212,
      "loss": 2.8827,
      "step": 63900
    },
    {
      "activations/layer0_attention_weight_max": 14.656688690185547,
      "activations/layer0_attention_weight_min": -14.458436965942383,
      "activations/layer10_attention_weight_max": 38.9063606262207,
      "activations/layer10_attention_weight_min": -33.23772048950195,
      "activations/layer11_attention_weight_max": 30.48550033569336,
      "activations/layer11_attention_weight_min": -26.98488998413086,
      "activations/layer12_attention_weight_max": 23.395339965820312,
      "activations/layer12_attention_weight_min": -24.9202938079834,
      "activations/layer13_attention_weight_max": 51.748172760009766,
      "activations/layer13_attention_weight_min": -39.058128356933594,
      "activations/layer14_attention_weight_max": 50.47836685180664,
      "activations/layer14_attention_weight_min": -34.94133377075195,
      "activations/layer15_attention_weight_max": 46.42543411254883,
      "activations/layer15_attention_weight_min": -31.256000518798828,
      "activations/layer16_attention_weight_max": 34.49787139892578,
      "activations/layer16_attention_weight_min": -26.204063415527344,
      "activations/layer17_attention_weight_max": 55.33492660522461,
      "activations/layer17_attention_weight_min": -37.572105407714844,
      "activations/layer18_attention_weight_max": 47.73002243041992,
      "activations/layer18_attention_weight_min": -31.5764217376709,
      "activations/layer19_attention_weight_max": 24.31519889831543,
      "activations/layer19_attention_weight_min": -18.09942626953125,
      "activations/layer1_attention_weight_max": 16.832754135131836,
      "activations/layer1_attention_weight_min": -13.779285430908203,
      "activations/layer20_attention_weight_max": 21.709171295166016,
      "activations/layer20_attention_weight_min": -21.02895164489746,
      "activations/layer21_attention_weight_max": 42.4960823059082,
      "activations/layer21_attention_weight_min": -27.415706634521484,
      "activations/layer22_attention_weight_max": 39.45981979370117,
      "activations/layer22_attention_weight_min": -24.430694580078125,
      "activations/layer23_attention_weight_max": 27.866657257080078,
      "activations/layer23_attention_weight_min": -17.602336883544922,
      "activations/layer2_attention_weight_max": 29.45185661315918,
      "activations/layer2_attention_weight_min": -27.27615737915039,
      "activations/layer3_attention_weight_max": 73.73843383789062,
      "activations/layer3_attention_weight_min": -77.76981353759766,
      "activations/layer4_attention_weight_max": 98.18191528320312,
      "activations/layer4_attention_weight_min": -90.80033111572266,
      "activations/layer5_attention_weight_max": 110.89766693115234,
      "activations/layer5_attention_weight_min": -94.09378051757812,
      "activations/layer6_attention_weight_max": 58.70552444458008,
      "activations/layer6_attention_weight_min": -49.35655975341797,
      "activations/layer7_attention_weight_max": 61.297908782958984,
      "activations/layer7_attention_weight_min": -55.161922454833984,
      "activations/layer8_attention_weight_max": 37.97316360473633,
      "activations/layer8_attention_weight_min": -36.900203704833984,
      "activations/layer9_attention_weight_max": 51.134090423583984,
      "activations/layer9_attention_weight_min": -43.94373321533203,
      "epoch": 3.72,
      "learning_rate": 0.0001273181818181818,
      "loss": 2.8757,
      "step": 63950
    },
    {
      "activations/layer0_attention_weight_max": 16.088834762573242,
      "activations/layer0_attention_weight_min": -14.657586097717285,
      "activations/layer10_attention_weight_max": 43.55458450317383,
      "activations/layer10_attention_weight_min": -37.67052459716797,
      "activations/layer11_attention_weight_max": 34.00995635986328,
      "activations/layer11_attention_weight_min": -27.25886344909668,
      "activations/layer12_attention_weight_max": 22.823829650878906,
      "activations/layer12_attention_weight_min": -24.788732528686523,
      "activations/layer13_attention_weight_max": 58.45985412597656,
      "activations/layer13_attention_weight_min": -39.93133544921875,
      "activations/layer14_attention_weight_max": 55.67290496826172,
      "activations/layer14_attention_weight_min": -33.635337829589844,
      "activations/layer15_attention_weight_max": 50.4990234375,
      "activations/layer15_attention_weight_min": -31.323429107666016,
      "activations/layer16_attention_weight_max": 34.32867431640625,
      "activations/layer16_attention_weight_min": -26.793535232543945,
      "activations/layer17_attention_weight_max": 65.95309448242188,
      "activations/layer17_attention_weight_min": -35.60749053955078,
      "activations/layer18_attention_weight_max": 55.82846450805664,
      "activations/layer18_attention_weight_min": -31.338171005249023,
      "activations/layer19_attention_weight_max": 23.288728713989258,
      "activations/layer19_attention_weight_min": -17.863435745239258,
      "activations/layer1_attention_weight_max": 19.999895095825195,
      "activations/layer1_attention_weight_min": -16.054750442504883,
      "activations/layer20_attention_weight_max": 23.687393188476562,
      "activations/layer20_attention_weight_min": -20.856712341308594,
      "activations/layer21_attention_weight_max": 47.061092376708984,
      "activations/layer21_attention_weight_min": -26.55734634399414,
      "activations/layer22_attention_weight_max": 34.85664367675781,
      "activations/layer22_attention_weight_min": -22.66106414794922,
      "activations/layer23_attention_weight_max": 28.637828826904297,
      "activations/layer23_attention_weight_min": -19.461177825927734,
      "activations/layer2_attention_weight_max": 30.134193420410156,
      "activations/layer2_attention_weight_min": -28.35384750366211,
      "activations/layer3_attention_weight_max": 76.82027435302734,
      "activations/layer3_attention_weight_min": -82.4717025756836,
      "activations/layer4_attention_weight_max": 105.30531311035156,
      "activations/layer4_attention_weight_min": -96.34966278076172,
      "activations/layer5_attention_weight_max": 115.78244018554688,
      "activations/layer5_attention_weight_min": -102.59300231933594,
      "activations/layer6_attention_weight_max": 59.937591552734375,
      "activations/layer6_attention_weight_min": -51.41233444213867,
      "activations/layer7_attention_weight_max": 61.67587661743164,
      "activations/layer7_attention_weight_min": -59.80221939086914,
      "activations/layer8_attention_weight_max": 39.362483978271484,
      "activations/layer8_attention_weight_min": -37.68410110473633,
      "activations/layer9_attention_weight_max": 52.791664123535156,
      "activations/layer9_attention_weight_min": -44.53953552246094,
      "epoch": 3.72,
      "learning_rate": 0.00012729924242424243,
      "loss": 2.8801,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_loss": 2.833984375,
      "eval_runtime": 8.4272,
      "eval_samples_per_second": 509.542,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_openwebtext_loss": 2.833984375,
      "eval_openwebtext_ppl": 17.013112580497378,
      "eval_openwebtext_runtime": 8.4272,
      "eval_openwebtext_samples_per_second": 509.542,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_wikitext_loss": 3.072265625,
      "eval_wikitext_ppl": 21.590763877931302,
      "eval_wikitext_runtime": 1.9485,
      "eval_wikitext_samples_per_second": 234.023,
      "step": 64000
    },
    {
      "epoch": 3.72,
      "eval_lambada_loss": 2.955078125,
      "eval_lambada_ppl": 19.20322291302118,
      "eval_lambada_runtime": 9.5266,
      "eval_lambada_samples_per_second": 511.096,
      "step": 64000
    },
    {
      "activations/layer0_attention_weight_max": 15.888319969177246,
      "activations/layer0_attention_weight_min": -14.461562156677246,
      "activations/layer10_attention_weight_max": 41.04715347290039,
      "activations/layer10_attention_weight_min": -32.883087158203125,
      "activations/layer11_attention_weight_max": 33.66386413574219,
      "activations/layer11_attention_weight_min": -27.67943572998047,
      "activations/layer12_attention_weight_max": 23.52808952331543,
      "activations/layer12_attention_weight_min": -24.965307235717773,
      "activations/layer13_attention_weight_max": 52.14384460449219,
      "activations/layer13_attention_weight_min": -40.282447814941406,
      "activations/layer14_attention_weight_max": 52.24613952636719,
      "activations/layer14_attention_weight_min": -37.7519416809082,
      "activations/layer15_attention_weight_max": 42.35663986206055,
      "activations/layer15_attention_weight_min": -30.706491470336914,
      "activations/layer16_attention_weight_max": 36.59601974487305,
      "activations/layer16_attention_weight_min": -26.964040756225586,
      "activations/layer17_attention_weight_max": 53.171470642089844,
      "activations/layer17_attention_weight_min": -36.43589782714844,
      "activations/layer18_attention_weight_max": 51.0745964050293,
      "activations/layer18_attention_weight_min": -29.19793701171875,
      "activations/layer19_attention_weight_max": 23.52161979675293,
      "activations/layer19_attention_weight_min": -17.811241149902344,
      "activations/layer1_attention_weight_max": 18.112346649169922,
      "activations/layer1_attention_weight_min": -17.375595092773438,
      "activations/layer20_attention_weight_max": 20.124427795410156,
      "activations/layer20_attention_weight_min": -19.986530303955078,
      "activations/layer21_attention_weight_max": 39.715415954589844,
      "activations/layer21_attention_weight_min": -26.321325302124023,
      "activations/layer22_attention_weight_max": 35.568695068359375,
      "activations/layer22_attention_weight_min": -21.339153289794922,
      "activations/layer23_attention_weight_max": 26.18347930908203,
      "activations/layer23_attention_weight_min": -14.627238273620605,
      "activations/layer2_attention_weight_max": 30.84676170349121,
      "activations/layer2_attention_weight_min": -28.956188201904297,
      "activations/layer3_attention_weight_max": 77.16522979736328,
      "activations/layer3_attention_weight_min": -80.33942413330078,
      "activations/layer4_attention_weight_max": 102.97750091552734,
      "activations/layer4_attention_weight_min": -93.54927825927734,
      "activations/layer5_attention_weight_max": 115.48503112792969,
      "activations/layer5_attention_weight_min": -99.76206970214844,
      "activations/layer6_attention_weight_max": 62.241764068603516,
      "activations/layer6_attention_weight_min": -53.901329040527344,
      "activations/layer7_attention_weight_max": 64.31381225585938,
      "activations/layer7_attention_weight_min": -63.75316619873047,
      "activations/layer8_attention_weight_max": 39.523658752441406,
      "activations/layer8_attention_weight_min": -38.31465530395508,
      "activations/layer9_attention_weight_max": 56.90359115600586,
      "activations/layer9_attention_weight_min": -49.82216262817383,
      "epoch": 3.72,
      "learning_rate": 0.00012728030303030301,
      "loss": 2.8826,
      "step": 64050
    },
    {
      "activations/layer0_attention_weight_max": 14.621260643005371,
      "activations/layer0_attention_weight_min": -15.009547233581543,
      "activations/layer10_attention_weight_max": 40.31719207763672,
      "activations/layer10_attention_weight_min": -35.45460510253906,
      "activations/layer11_attention_weight_max": 33.75394058227539,
      "activations/layer11_attention_weight_min": -26.767004013061523,
      "activations/layer12_attention_weight_max": 29.42352867126465,
      "activations/layer12_attention_weight_min": -23.70303726196289,
      "activations/layer13_attention_weight_max": 51.61777877807617,
      "activations/layer13_attention_weight_min": -42.29875946044922,
      "activations/layer14_attention_weight_max": 51.22100067138672,
      "activations/layer14_attention_weight_min": -34.4501838684082,
      "activations/layer15_attention_weight_max": 45.74380111694336,
      "activations/layer15_attention_weight_min": -37.40193557739258,
      "activations/layer16_attention_weight_max": 34.778297424316406,
      "activations/layer16_attention_weight_min": -27.858068466186523,
      "activations/layer17_attention_weight_max": 56.77725601196289,
      "activations/layer17_attention_weight_min": -36.791236877441406,
      "activations/layer18_attention_weight_max": 53.2036247253418,
      "activations/layer18_attention_weight_min": -33.7840690612793,
      "activations/layer19_attention_weight_max": 22.2451171875,
      "activations/layer19_attention_weight_min": -18.757450103759766,
      "activations/layer1_attention_weight_max": 18.61604881286621,
      "activations/layer1_attention_weight_min": -16.100135803222656,
      "activations/layer20_attention_weight_max": 24.521595001220703,
      "activations/layer20_attention_weight_min": -20.31828498840332,
      "activations/layer21_attention_weight_max": 42.28133010864258,
      "activations/layer21_attention_weight_min": -32.70857620239258,
      "activations/layer22_attention_weight_max": 36.44809341430664,
      "activations/layer22_attention_weight_min": -24.17144012451172,
      "activations/layer23_attention_weight_max": 27.025625228881836,
      "activations/layer23_attention_weight_min": -16.09302520751953,
      "activations/layer2_attention_weight_max": 30.786399841308594,
      "activations/layer2_attention_weight_min": -26.90923309326172,
      "activations/layer3_attention_weight_max": 75.8567886352539,
      "activations/layer3_attention_weight_min": -76.39093780517578,
      "activations/layer4_attention_weight_max": 101.3790054321289,
      "activations/layer4_attention_weight_min": -91.9561996459961,
      "activations/layer5_attention_weight_max": 111.05804443359375,
      "activations/layer5_attention_weight_min": -98.04507446289062,
      "activations/layer6_attention_weight_max": 59.163352966308594,
      "activations/layer6_attention_weight_min": -51.55248260498047,
      "activations/layer7_attention_weight_max": 62.34342956542969,
      "activations/layer7_attention_weight_min": -56.76388931274414,
      "activations/layer8_attention_weight_max": 41.06716537475586,
      "activations/layer8_attention_weight_min": -38.32635498046875,
      "activations/layer9_attention_weight_max": 47.113948822021484,
      "activations/layer9_attention_weight_min": -44.947425842285156,
      "epoch": 3.72,
      "learning_rate": 0.00012726136363636363,
      "loss": 2.8841,
      "step": 64100
    },
    {
      "activations/layer0_attention_weight_max": 15.334501266479492,
      "activations/layer0_attention_weight_min": -14.443480491638184,
      "activations/layer10_attention_weight_max": 46.3502311706543,
      "activations/layer10_attention_weight_min": -34.38300323486328,
      "activations/layer11_attention_weight_max": 37.535152435302734,
      "activations/layer11_attention_weight_min": -27.75275230407715,
      "activations/layer12_attention_weight_max": 28.77411460876465,
      "activations/layer12_attention_weight_min": -23.47174835205078,
      "activations/layer13_attention_weight_max": 71.81255340576172,
      "activations/layer13_attention_weight_min": -44.26700973510742,
      "activations/layer14_attention_weight_max": 79.95437622070312,
      "activations/layer14_attention_weight_min": -41.72371292114258,
      "activations/layer15_attention_weight_max": 53.713619232177734,
      "activations/layer15_attention_weight_min": -32.93327331542969,
      "activations/layer16_attention_weight_max": 44.34660720825195,
      "activations/layer16_attention_weight_min": -27.67891502380371,
      "activations/layer17_attention_weight_max": 62.7465705871582,
      "activations/layer17_attention_weight_min": -38.480857849121094,
      "activations/layer18_attention_weight_max": 51.75582504272461,
      "activations/layer18_attention_weight_min": -35.40222930908203,
      "activations/layer19_attention_weight_max": 22.264179229736328,
      "activations/layer19_attention_weight_min": -20.244672775268555,
      "activations/layer1_attention_weight_max": 17.766149520874023,
      "activations/layer1_attention_weight_min": -13.250822067260742,
      "activations/layer20_attention_weight_max": 22.268768310546875,
      "activations/layer20_attention_weight_min": -21.022705078125,
      "activations/layer21_attention_weight_max": 49.26930618286133,
      "activations/layer21_attention_weight_min": -28.753019332885742,
      "activations/layer22_attention_weight_max": 40.83024597167969,
      "activations/layer22_attention_weight_min": -24.85956573486328,
      "activations/layer23_attention_weight_max": 25.89655113220215,
      "activations/layer23_attention_weight_min": -19.11417579650879,
      "activations/layer2_attention_weight_max": 29.358694076538086,
      "activations/layer2_attention_weight_min": -27.89839744567871,
      "activations/layer3_attention_weight_max": 78.42897033691406,
      "activations/layer3_attention_weight_min": -80.92703247070312,
      "activations/layer4_attention_weight_max": 106.2383804321289,
      "activations/layer4_attention_weight_min": -93.99485778808594,
      "activations/layer5_attention_weight_max": 128.79034423828125,
      "activations/layer5_attention_weight_min": -103.7020263671875,
      "activations/layer6_attention_weight_max": 67.65121459960938,
      "activations/layer6_attention_weight_min": -54.96990966796875,
      "activations/layer7_attention_weight_max": 67.69007110595703,
      "activations/layer7_attention_weight_min": -62.99863815307617,
      "activations/layer8_attention_weight_max": 43.26158142089844,
      "activations/layer8_attention_weight_min": -40.79517364501953,
      "activations/layer9_attention_weight_max": 54.7362174987793,
      "activations/layer9_attention_weight_min": -45.75056457519531,
      "epoch": 3.73,
      "learning_rate": 0.00012724242424242424,
      "loss": 2.9,
      "step": 64150
    },
    {
      "activations/layer0_attention_weight_max": 13.28872299194336,
      "activations/layer0_attention_weight_min": -14.409412384033203,
      "activations/layer10_attention_weight_max": 40.960533142089844,
      "activations/layer10_attention_weight_min": -33.955501556396484,
      "activations/layer11_attention_weight_max": 32.21943664550781,
      "activations/layer11_attention_weight_min": -27.22840118408203,
      "activations/layer12_attention_weight_max": 24.814380645751953,
      "activations/layer12_attention_weight_min": -22.893226623535156,
      "activations/layer13_attention_weight_max": 61.9195442199707,
      "activations/layer13_attention_weight_min": -46.20204162597656,
      "activations/layer14_attention_weight_max": 52.431060791015625,
      "activations/layer14_attention_weight_min": -38.73709487915039,
      "activations/layer15_attention_weight_max": 49.9913444519043,
      "activations/layer15_attention_weight_min": -33.32879638671875,
      "activations/layer16_attention_weight_max": 41.29625701904297,
      "activations/layer16_attention_weight_min": -25.795063018798828,
      "activations/layer17_attention_weight_max": 63.86035919189453,
      "activations/layer17_attention_weight_min": -36.074954986572266,
      "activations/layer18_attention_weight_max": 62.51247024536133,
      "activations/layer18_attention_weight_min": -32.17814254760742,
      "activations/layer19_attention_weight_max": 23.088178634643555,
      "activations/layer19_attention_weight_min": -18.396183013916016,
      "activations/layer1_attention_weight_max": 18.109237670898438,
      "activations/layer1_attention_weight_min": -14.304913520812988,
      "activations/layer20_attention_weight_max": 25.662757873535156,
      "activations/layer20_attention_weight_min": -19.50082778930664,
      "activations/layer21_attention_weight_max": 43.22745132446289,
      "activations/layer21_attention_weight_min": -29.2104549407959,
      "activations/layer22_attention_weight_max": 41.93601608276367,
      "activations/layer22_attention_weight_min": -25.853290557861328,
      "activations/layer23_attention_weight_max": 27.361778259277344,
      "activations/layer23_attention_weight_min": -16.904685974121094,
      "activations/layer2_attention_weight_max": 28.80358123779297,
      "activations/layer2_attention_weight_min": -27.2009334564209,
      "activations/layer3_attention_weight_max": 73.44601440429688,
      "activations/layer3_attention_weight_min": -77.925537109375,
      "activations/layer4_attention_weight_max": 99.6654281616211,
      "activations/layer4_attention_weight_min": -93.58252716064453,
      "activations/layer5_attention_weight_max": 114.27790832519531,
      "activations/layer5_attention_weight_min": -104.52220153808594,
      "activations/layer6_attention_weight_max": 62.34263229370117,
      "activations/layer6_attention_weight_min": -53.3392448425293,
      "activations/layer7_attention_weight_max": 63.19124221801758,
      "activations/layer7_attention_weight_min": -60.71257019042969,
      "activations/layer8_attention_weight_max": 39.68686294555664,
      "activations/layer8_attention_weight_min": -38.28387451171875,
      "activations/layer9_attention_weight_max": 53.83893966674805,
      "activations/layer9_attention_weight_min": -46.77114486694336,
      "epoch": 3.73,
      "learning_rate": 0.00012722348484848483,
      "loss": 2.8906,
      "step": 64200
    },
    {
      "activations/layer0_attention_weight_max": 15.203763008117676,
      "activations/layer0_attention_weight_min": -14.676438331604004,
      "activations/layer10_attention_weight_max": 43.58587646484375,
      "activations/layer10_attention_weight_min": -34.07274627685547,
      "activations/layer11_attention_weight_max": 36.00705337524414,
      "activations/layer11_attention_weight_min": -28.152074813842773,
      "activations/layer12_attention_weight_max": 29.842479705810547,
      "activations/layer12_attention_weight_min": -27.206295013427734,
      "activations/layer13_attention_weight_max": 54.193992614746094,
      "activations/layer13_attention_weight_min": -35.32249069213867,
      "activations/layer14_attention_weight_max": 51.598419189453125,
      "activations/layer14_attention_weight_min": -31.96003532409668,
      "activations/layer15_attention_weight_max": 46.79642105102539,
      "activations/layer15_attention_weight_min": -31.1011962890625,
      "activations/layer16_attention_weight_max": 44.38656234741211,
      "activations/layer16_attention_weight_min": -27.818777084350586,
      "activations/layer17_attention_weight_max": 64.24825286865234,
      "activations/layer17_attention_weight_min": -38.70506286621094,
      "activations/layer18_attention_weight_max": 61.542083740234375,
      "activations/layer18_attention_weight_min": -32.684173583984375,
      "activations/layer19_attention_weight_max": 25.534093856811523,
      "activations/layer19_attention_weight_min": -20.397201538085938,
      "activations/layer1_attention_weight_max": 17.241273880004883,
      "activations/layer1_attention_weight_min": -13.881292343139648,
      "activations/layer20_attention_weight_max": 22.135589599609375,
      "activations/layer20_attention_weight_min": -19.43838119506836,
      "activations/layer21_attention_weight_max": 43.71312713623047,
      "activations/layer21_attention_weight_min": -27.02789306640625,
      "activations/layer22_attention_weight_max": 41.155967712402344,
      "activations/layer22_attention_weight_min": -24.462310791015625,
      "activations/layer23_attention_weight_max": 28.587963104248047,
      "activations/layer23_attention_weight_min": -17.956493377685547,
      "activations/layer2_attention_weight_max": 29.720355987548828,
      "activations/layer2_attention_weight_min": -28.34630584716797,
      "activations/layer3_attention_weight_max": 78.59381866455078,
      "activations/layer3_attention_weight_min": -79.37925720214844,
      "activations/layer4_attention_weight_max": 97.20191955566406,
      "activations/layer4_attention_weight_min": -92.17393493652344,
      "activations/layer5_attention_weight_max": 113.54800415039062,
      "activations/layer5_attention_weight_min": -98.14034271240234,
      "activations/layer6_attention_weight_max": 61.788169860839844,
      "activations/layer6_attention_weight_min": -50.244056701660156,
      "activations/layer7_attention_weight_max": 65.39053344726562,
      "activations/layer7_attention_weight_min": -59.660526275634766,
      "activations/layer8_attention_weight_max": 39.12712097167969,
      "activations/layer8_attention_weight_min": -40.879878997802734,
      "activations/layer9_attention_weight_max": 55.8812255859375,
      "activations/layer9_attention_weight_min": -48.01048278808594,
      "epoch": 3.73,
      "learning_rate": 0.00012720454545454545,
      "loss": 2.8847,
      "step": 64250
    },
    {
      "activations/layer0_attention_weight_max": 13.703228950500488,
      "activations/layer0_attention_weight_min": -14.182451248168945,
      "activations/layer10_attention_weight_max": 38.10655975341797,
      "activations/layer10_attention_weight_min": -34.18536376953125,
      "activations/layer11_attention_weight_max": 32.46038818359375,
      "activations/layer11_attention_weight_min": -25.616682052612305,
      "activations/layer12_attention_weight_max": 32.74189376831055,
      "activations/layer12_attention_weight_min": -24.79737663269043,
      "activations/layer13_attention_weight_max": 58.33081817626953,
      "activations/layer13_attention_weight_min": -41.299110412597656,
      "activations/layer14_attention_weight_max": 54.326534271240234,
      "activations/layer14_attention_weight_min": -37.46889114379883,
      "activations/layer15_attention_weight_max": 43.98875427246094,
      "activations/layer15_attention_weight_min": -32.77189636230469,
      "activations/layer16_attention_weight_max": 37.21578598022461,
      "activations/layer16_attention_weight_min": -26.223398208618164,
      "activations/layer17_attention_weight_max": 57.39913558959961,
      "activations/layer17_attention_weight_min": -37.819942474365234,
      "activations/layer18_attention_weight_max": 48.225887298583984,
      "activations/layer18_attention_weight_min": -31.16942024230957,
      "activations/layer19_attention_weight_max": 24.334157943725586,
      "activations/layer19_attention_weight_min": -18.351797103881836,
      "activations/layer1_attention_weight_max": 17.7357234954834,
      "activations/layer1_attention_weight_min": -14.939807891845703,
      "activations/layer20_attention_weight_max": 20.57681655883789,
      "activations/layer20_attention_weight_min": -19.419496536254883,
      "activations/layer21_attention_weight_max": 35.41685104370117,
      "activations/layer21_attention_weight_min": -24.16519546508789,
      "activations/layer22_attention_weight_max": 33.67765808105469,
      "activations/layer22_attention_weight_min": -23.783254623413086,
      "activations/layer23_attention_weight_max": 25.417499542236328,
      "activations/layer23_attention_weight_min": -15.695049285888672,
      "activations/layer2_attention_weight_max": 28.350017547607422,
      "activations/layer2_attention_weight_min": -25.399765014648438,
      "activations/layer3_attention_weight_max": 79.90299224853516,
      "activations/layer3_attention_weight_min": -78.97429656982422,
      "activations/layer4_attention_weight_max": 100.290771484375,
      "activations/layer4_attention_weight_min": -96.77725982666016,
      "activations/layer5_attention_weight_max": 116.98284149169922,
      "activations/layer5_attention_weight_min": -101.8044662475586,
      "activations/layer6_attention_weight_max": 59.43678665161133,
      "activations/layer6_attention_weight_min": -53.15635299682617,
      "activations/layer7_attention_weight_max": 65.64105224609375,
      "activations/layer7_attention_weight_min": -62.10721969604492,
      "activations/layer8_attention_weight_max": 38.780364990234375,
      "activations/layer8_attention_weight_min": -37.06776428222656,
      "activations/layer9_attention_weight_max": 53.35085678100586,
      "activations/layer9_attention_weight_min": -46.120018005371094,
      "epoch": 3.74,
      "learning_rate": 0.00012718560606060606,
      "loss": 2.8758,
      "step": 64300
    },
    {
      "activations/layer0_attention_weight_max": 14.756394386291504,
      "activations/layer0_attention_weight_min": -13.849865913391113,
      "activations/layer10_attention_weight_max": 40.469478607177734,
      "activations/layer10_attention_weight_min": -34.206024169921875,
      "activations/layer11_attention_weight_max": 36.49652862548828,
      "activations/layer11_attention_weight_min": -26.380329132080078,
      "activations/layer12_attention_weight_max": 24.509666442871094,
      "activations/layer12_attention_weight_min": -27.045076370239258,
      "activations/layer13_attention_weight_max": 63.22949981689453,
      "activations/layer13_attention_weight_min": -45.25102996826172,
      "activations/layer14_attention_weight_max": 73.96931457519531,
      "activations/layer14_attention_weight_min": -44.736595153808594,
      "activations/layer15_attention_weight_max": 48.36102294921875,
      "activations/layer15_attention_weight_min": -30.22952651977539,
      "activations/layer16_attention_weight_max": 35.21509552001953,
      "activations/layer16_attention_weight_min": -25.872400283813477,
      "activations/layer17_attention_weight_max": 55.250404357910156,
      "activations/layer17_attention_weight_min": -36.521949768066406,
      "activations/layer18_attention_weight_max": 50.52145004272461,
      "activations/layer18_attention_weight_min": -30.758769989013672,
      "activations/layer19_attention_weight_max": 25.2412109375,
      "activations/layer19_attention_weight_min": -18.081968307495117,
      "activations/layer1_attention_weight_max": 17.879899978637695,
      "activations/layer1_attention_weight_min": -14.431119918823242,
      "activations/layer20_attention_weight_max": 20.28456687927246,
      "activations/layer20_attention_weight_min": -20.501476287841797,
      "activations/layer21_attention_weight_max": 41.21000289916992,
      "activations/layer21_attention_weight_min": -28.73722267150879,
      "activations/layer22_attention_weight_max": 37.065277099609375,
      "activations/layer22_attention_weight_min": -21.906938552856445,
      "activations/layer23_attention_weight_max": 28.193822860717773,
      "activations/layer23_attention_weight_min": -16.530437469482422,
      "activations/layer2_attention_weight_max": 29.60382843017578,
      "activations/layer2_attention_weight_min": -28.316402435302734,
      "activations/layer3_attention_weight_max": 77.33561706542969,
      "activations/layer3_attention_weight_min": -78.20227813720703,
      "activations/layer4_attention_weight_max": 100.04886627197266,
      "activations/layer4_attention_weight_min": -90.17526245117188,
      "activations/layer5_attention_weight_max": 108.56334686279297,
      "activations/layer5_attention_weight_min": -95.07041931152344,
      "activations/layer6_attention_weight_max": 59.28129196166992,
      "activations/layer6_attention_weight_min": -50.3846549987793,
      "activations/layer7_attention_weight_max": 62.16105270385742,
      "activations/layer7_attention_weight_min": -59.48167037963867,
      "activations/layer8_attention_weight_max": 39.708282470703125,
      "activations/layer8_attention_weight_min": -39.85948944091797,
      "activations/layer9_attention_weight_max": 52.10646438598633,
      "activations/layer9_attention_weight_min": -44.720516204833984,
      "epoch": 3.74,
      "learning_rate": 0.00012716666666666665,
      "loss": 2.886,
      "step": 64350
    },
    {
      "activations/layer0_attention_weight_max": 13.998546600341797,
      "activations/layer0_attention_weight_min": -14.60373592376709,
      "activations/layer10_attention_weight_max": 37.4958610534668,
      "activations/layer10_attention_weight_min": -31.626327514648438,
      "activations/layer11_attention_weight_max": 28.475149154663086,
      "activations/layer11_attention_weight_min": -27.472305297851562,
      "activations/layer12_attention_weight_max": 23.52284049987793,
      "activations/layer12_attention_weight_min": -25.966501235961914,
      "activations/layer13_attention_weight_max": 51.921634674072266,
      "activations/layer13_attention_weight_min": -44.58730697631836,
      "activations/layer14_attention_weight_max": 45.67744064331055,
      "activations/layer14_attention_weight_min": -37.827911376953125,
      "activations/layer15_attention_weight_max": 42.946678161621094,
      "activations/layer15_attention_weight_min": -31.973480224609375,
      "activations/layer16_attention_weight_max": 37.59071350097656,
      "activations/layer16_attention_weight_min": -28.12952995300293,
      "activations/layer17_attention_weight_max": 61.02079391479492,
      "activations/layer17_attention_weight_min": -35.996238708496094,
      "activations/layer18_attention_weight_max": 57.093318939208984,
      "activations/layer18_attention_weight_min": -31.309484481811523,
      "activations/layer19_attention_weight_max": 21.541337966918945,
      "activations/layer19_attention_weight_min": -18.66439437866211,
      "activations/layer1_attention_weight_max": 18.623109817504883,
      "activations/layer1_attention_weight_min": -12.820777893066406,
      "activations/layer20_attention_weight_max": 21.671960830688477,
      "activations/layer20_attention_weight_min": -23.235902786254883,
      "activations/layer21_attention_weight_max": 41.100860595703125,
      "activations/layer21_attention_weight_min": -27.434844970703125,
      "activations/layer22_attention_weight_max": 32.902103424072266,
      "activations/layer22_attention_weight_min": -22.807811737060547,
      "activations/layer23_attention_weight_max": 26.372982025146484,
      "activations/layer23_attention_weight_min": -16.279577255249023,
      "activations/layer2_attention_weight_max": 28.98818588256836,
      "activations/layer2_attention_weight_min": -27.817771911621094,
      "activations/layer3_attention_weight_max": 75.79487609863281,
      "activations/layer3_attention_weight_min": -76.1763916015625,
      "activations/layer4_attention_weight_max": 96.81631469726562,
      "activations/layer4_attention_weight_min": -91.6412582397461,
      "activations/layer5_attention_weight_max": 108.40090942382812,
      "activations/layer5_attention_weight_min": -96.18746948242188,
      "activations/layer6_attention_weight_max": 59.66214370727539,
      "activations/layer6_attention_weight_min": -51.587135314941406,
      "activations/layer7_attention_weight_max": 66.65214538574219,
      "activations/layer7_attention_weight_min": -61.69578170776367,
      "activations/layer8_attention_weight_max": 41.37562942504883,
      "activations/layer8_attention_weight_min": -35.89530563354492,
      "activations/layer9_attention_weight_max": 57.47639083862305,
      "activations/layer9_attention_weight_min": -55.283973693847656,
      "epoch": 3.74,
      "learning_rate": 0.00012714772727272726,
      "loss": 2.8746,
      "step": 64400
    },
    {
      "activations/layer0_attention_weight_max": 15.088894844055176,
      "activations/layer0_attention_weight_min": -14.951239585876465,
      "activations/layer10_attention_weight_max": 40.1829948425293,
      "activations/layer10_attention_weight_min": -32.69785690307617,
      "activations/layer11_attention_weight_max": 33.82582092285156,
      "activations/layer11_attention_weight_min": -26.16969108581543,
      "activations/layer12_attention_weight_max": 27.8461856842041,
      "activations/layer12_attention_weight_min": -28.769990921020508,
      "activations/layer13_attention_weight_max": 57.6947135925293,
      "activations/layer13_attention_weight_min": -41.3270263671875,
      "activations/layer14_attention_weight_max": 60.7175407409668,
      "activations/layer14_attention_weight_min": -38.503868103027344,
      "activations/layer15_attention_weight_max": 46.46917724609375,
      "activations/layer15_attention_weight_min": -30.384246826171875,
      "activations/layer16_attention_weight_max": 35.7714958190918,
      "activations/layer16_attention_weight_min": -25.76242446899414,
      "activations/layer17_attention_weight_max": 61.96142578125,
      "activations/layer17_attention_weight_min": -36.2386589050293,
      "activations/layer18_attention_weight_max": 55.10331726074219,
      "activations/layer18_attention_weight_min": -32.66811752319336,
      "activations/layer19_attention_weight_max": 24.4349308013916,
      "activations/layer19_attention_weight_min": -18.37957191467285,
      "activations/layer1_attention_weight_max": 18.087434768676758,
      "activations/layer1_attention_weight_min": -15.18980598449707,
      "activations/layer20_attention_weight_max": 21.884822845458984,
      "activations/layer20_attention_weight_min": -19.601179122924805,
      "activations/layer21_attention_weight_max": 51.152828216552734,
      "activations/layer21_attention_weight_min": -29.35161018371582,
      "activations/layer22_attention_weight_max": 39.5004997253418,
      "activations/layer22_attention_weight_min": -23.29900550842285,
      "activations/layer23_attention_weight_max": 27.85712242126465,
      "activations/layer23_attention_weight_min": -19.981454849243164,
      "activations/layer2_attention_weight_max": 29.905376434326172,
      "activations/layer2_attention_weight_min": -28.434110641479492,
      "activations/layer3_attention_weight_max": 76.71865844726562,
      "activations/layer3_attention_weight_min": -78.3119125366211,
      "activations/layer4_attention_weight_max": 101.06890106201172,
      "activations/layer4_attention_weight_min": -99.5289535522461,
      "activations/layer5_attention_weight_max": 119.64764404296875,
      "activations/layer5_attention_weight_min": -106.47308349609375,
      "activations/layer6_attention_weight_max": 61.47430419921875,
      "activations/layer6_attention_weight_min": -54.950462341308594,
      "activations/layer7_attention_weight_max": 68.81644439697266,
      "activations/layer7_attention_weight_min": -59.49040222167969,
      "activations/layer8_attention_weight_max": 41.07419967651367,
      "activations/layer8_attention_weight_min": -37.9255256652832,
      "activations/layer9_attention_weight_max": 56.34479904174805,
      "activations/layer9_attention_weight_min": -46.2573356628418,
      "epoch": 3.74,
      "learning_rate": 0.00012712878787878788,
      "loss": 2.8786,
      "step": 64450
    },
    {
      "activations/layer0_attention_weight_max": 14.152570724487305,
      "activations/layer0_attention_weight_min": -14.395051956176758,
      "activations/layer10_attention_weight_max": 38.2187614440918,
      "activations/layer10_attention_weight_min": -31.239871978759766,
      "activations/layer11_attention_weight_max": 32.61528396606445,
      "activations/layer11_attention_weight_min": -24.710094451904297,
      "activations/layer12_attention_weight_max": 28.236881256103516,
      "activations/layer12_attention_weight_min": -26.26261329650879,
      "activations/layer13_attention_weight_max": 51.6939582824707,
      "activations/layer13_attention_weight_min": -40.763755798339844,
      "activations/layer14_attention_weight_max": 49.905643463134766,
      "activations/layer14_attention_weight_min": -36.36680221557617,
      "activations/layer15_attention_weight_max": 42.40061569213867,
      "activations/layer15_attention_weight_min": -33.86385726928711,
      "activations/layer16_attention_weight_max": 37.842647552490234,
      "activations/layer16_attention_weight_min": -26.0778751373291,
      "activations/layer17_attention_weight_max": 57.18843460083008,
      "activations/layer17_attention_weight_min": -37.29983139038086,
      "activations/layer18_attention_weight_max": 51.48966598510742,
      "activations/layer18_attention_weight_min": -30.792766571044922,
      "activations/layer19_attention_weight_max": 23.3846492767334,
      "activations/layer19_attention_weight_min": -19.23002815246582,
      "activations/layer1_attention_weight_max": 17.560714721679688,
      "activations/layer1_attention_weight_min": -14.380541801452637,
      "activations/layer20_attention_weight_max": 20.149202346801758,
      "activations/layer20_attention_weight_min": -18.622650146484375,
      "activations/layer21_attention_weight_max": 42.0260124206543,
      "activations/layer21_attention_weight_min": -25.89595603942871,
      "activations/layer22_attention_weight_max": 36.5145149230957,
      "activations/layer22_attention_weight_min": -21.992883682250977,
      "activations/layer23_attention_weight_max": 25.857601165771484,
      "activations/layer23_attention_weight_min": -17.690874099731445,
      "activations/layer2_attention_weight_max": 28.616313934326172,
      "activations/layer2_attention_weight_min": -26.58669662475586,
      "activations/layer3_attention_weight_max": 75.06372833251953,
      "activations/layer3_attention_weight_min": -78.43025207519531,
      "activations/layer4_attention_weight_max": 96.10267639160156,
      "activations/layer4_attention_weight_min": -90.37267303466797,
      "activations/layer5_attention_weight_max": 114.93327331542969,
      "activations/layer5_attention_weight_min": -99.91433715820312,
      "activations/layer6_attention_weight_max": 60.2093620300293,
      "activations/layer6_attention_weight_min": -52.21737289428711,
      "activations/layer7_attention_weight_max": 59.834537506103516,
      "activations/layer7_attention_weight_min": -58.18673324584961,
      "activations/layer8_attention_weight_max": 39.15192413330078,
      "activations/layer8_attention_weight_min": -37.06243133544922,
      "activations/layer9_attention_weight_max": 48.125423431396484,
      "activations/layer9_attention_weight_min": -45.83494186401367,
      "epoch": 3.75,
      "learning_rate": 0.00012711022727272727,
      "loss": 2.8873,
      "step": 64500
    },
    {
      "activations/layer0_attention_weight_max": 13.0570068359375,
      "activations/layer0_attention_weight_min": -14.629220962524414,
      "activations/layer10_attention_weight_max": 42.51594543457031,
      "activations/layer10_attention_weight_min": -36.661895751953125,
      "activations/layer11_attention_weight_max": 33.70683288574219,
      "activations/layer11_attention_weight_min": -25.956233978271484,
      "activations/layer12_attention_weight_max": 32.52956771850586,
      "activations/layer12_attention_weight_min": -23.1790714263916,
      "activations/layer13_attention_weight_max": 69.83386993408203,
      "activations/layer13_attention_weight_min": -48.61699676513672,
      "activations/layer14_attention_weight_max": 61.4318733215332,
      "activations/layer14_attention_weight_min": -36.223594665527344,
      "activations/layer15_attention_weight_max": 48.75458908081055,
      "activations/layer15_attention_weight_min": -37.71693420410156,
      "activations/layer16_attention_weight_max": 44.988285064697266,
      "activations/layer16_attention_weight_min": -27.148406982421875,
      "activations/layer17_attention_weight_max": 59.64601516723633,
      "activations/layer17_attention_weight_min": -39.267704010009766,
      "activations/layer18_attention_weight_max": 58.453922271728516,
      "activations/layer18_attention_weight_min": -35.68769454956055,
      "activations/layer19_attention_weight_max": 22.695192337036133,
      "activations/layer19_attention_weight_min": -18.448318481445312,
      "activations/layer1_attention_weight_max": 17.241764068603516,
      "activations/layer1_attention_weight_min": -14.144189834594727,
      "activations/layer20_attention_weight_max": 22.177146911621094,
      "activations/layer20_attention_weight_min": -22.51235580444336,
      "activations/layer21_attention_weight_max": 46.83853530883789,
      "activations/layer21_attention_weight_min": -28.131479263305664,
      "activations/layer22_attention_weight_max": 38.45661926269531,
      "activations/layer22_attention_weight_min": -24.282329559326172,
      "activations/layer23_attention_weight_max": 26.853626251220703,
      "activations/layer23_attention_weight_min": -17.026079177856445,
      "activations/layer2_attention_weight_max": 29.472599029541016,
      "activations/layer2_attention_weight_min": -28.45592498779297,
      "activations/layer3_attention_weight_max": 78.0876693725586,
      "activations/layer3_attention_weight_min": -81.99007415771484,
      "activations/layer4_attention_weight_max": 101.04564666748047,
      "activations/layer4_attention_weight_min": -94.77220916748047,
      "activations/layer5_attention_weight_max": 116.54570770263672,
      "activations/layer5_attention_weight_min": -101.29736328125,
      "activations/layer6_attention_weight_max": 59.6911735534668,
      "activations/layer6_attention_weight_min": -54.490596771240234,
      "activations/layer7_attention_weight_max": 64.14655303955078,
      "activations/layer7_attention_weight_min": -58.063865661621094,
      "activations/layer8_attention_weight_max": 39.63544464111328,
      "activations/layer8_attention_weight_min": -40.28013229370117,
      "activations/layer9_attention_weight_max": 56.843544006347656,
      "activations/layer9_attention_weight_min": -47.67768096923828,
      "epoch": 3.75,
      "learning_rate": 0.00012709128787878786,
      "loss": 2.887,
      "step": 64550
    },
    {
      "activations/layer0_attention_weight_max": 13.802196502685547,
      "activations/layer0_attention_weight_min": -14.334633827209473,
      "activations/layer10_attention_weight_max": 42.48738098144531,
      "activations/layer10_attention_weight_min": -33.65797805786133,
      "activations/layer11_attention_weight_max": 33.989444732666016,
      "activations/layer11_attention_weight_min": -26.847883224487305,
      "activations/layer12_attention_weight_max": 29.215991973876953,
      "activations/layer12_attention_weight_min": -26.507707595825195,
      "activations/layer13_attention_weight_max": 60.92111587524414,
      "activations/layer13_attention_weight_min": -40.87429428100586,
      "activations/layer14_attention_weight_max": 43.25128173828125,
      "activations/layer14_attention_weight_min": -32.082401275634766,
      "activations/layer15_attention_weight_max": 46.97408676147461,
      "activations/layer15_attention_weight_min": -32.9042854309082,
      "activations/layer16_attention_weight_max": 39.75141525268555,
      "activations/layer16_attention_weight_min": -26.768022537231445,
      "activations/layer17_attention_weight_max": 60.00935745239258,
      "activations/layer17_attention_weight_min": -36.97429656982422,
      "activations/layer18_attention_weight_max": 52.4429817199707,
      "activations/layer18_attention_weight_min": -32.793216705322266,
      "activations/layer19_attention_weight_max": 20.91371726989746,
      "activations/layer19_attention_weight_min": -17.297990798950195,
      "activations/layer1_attention_weight_max": 17.09168243408203,
      "activations/layer1_attention_weight_min": -15.907377243041992,
      "activations/layer20_attention_weight_max": 19.865928649902344,
      "activations/layer20_attention_weight_min": -20.46237564086914,
      "activations/layer21_attention_weight_max": 41.59635925292969,
      "activations/layer21_attention_weight_min": -26.38045310974121,
      "activations/layer22_attention_weight_max": 41.51593780517578,
      "activations/layer22_attention_weight_min": -25.28569221496582,
      "activations/layer23_attention_weight_max": 26.649669647216797,
      "activations/layer23_attention_weight_min": -15.866183280944824,
      "activations/layer2_attention_weight_max": 28.826892852783203,
      "activations/layer2_attention_weight_min": -27.127994537353516,
      "activations/layer3_attention_weight_max": 76.0203857421875,
      "activations/layer3_attention_weight_min": -79.50731658935547,
      "activations/layer4_attention_weight_max": 105.24250793457031,
      "activations/layer4_attention_weight_min": -96.66107940673828,
      "activations/layer5_attention_weight_max": 118.49322509765625,
      "activations/layer5_attention_weight_min": -105.02680969238281,
      "activations/layer6_attention_weight_max": 62.111846923828125,
      "activations/layer6_attention_weight_min": -53.68711471557617,
      "activations/layer7_attention_weight_max": 64.12525939941406,
      "activations/layer7_attention_weight_min": -59.98187255859375,
      "activations/layer8_attention_weight_max": 41.5737190246582,
      "activations/layer8_attention_weight_min": -39.115081787109375,
      "activations/layer9_attention_weight_max": 48.896453857421875,
      "activations/layer9_attention_weight_min": -46.61465072631836,
      "epoch": 3.75,
      "learning_rate": 0.00012707234848484847,
      "loss": 2.8801,
      "step": 64600
    },
    {
      "activations/layer0_attention_weight_max": 16.196186065673828,
      "activations/layer0_attention_weight_min": -14.324408531188965,
      "activations/layer10_attention_weight_max": 45.83274841308594,
      "activations/layer10_attention_weight_min": -33.35626983642578,
      "activations/layer11_attention_weight_max": 35.11961364746094,
      "activations/layer11_attention_weight_min": -25.408039093017578,
      "activations/layer12_attention_weight_max": 28.249975204467773,
      "activations/layer12_attention_weight_min": -33.800758361816406,
      "activations/layer13_attention_weight_max": 51.99671173095703,
      "activations/layer13_attention_weight_min": -36.78363800048828,
      "activations/layer14_attention_weight_max": 57.74302291870117,
      "activations/layer14_attention_weight_min": -34.90496063232422,
      "activations/layer15_attention_weight_max": 50.259254455566406,
      "activations/layer15_attention_weight_min": -30.52884292602539,
      "activations/layer16_attention_weight_max": 45.36143112182617,
      "activations/layer16_attention_weight_min": -26.891433715820312,
      "activations/layer17_attention_weight_max": 68.94488525390625,
      "activations/layer17_attention_weight_min": -35.90462112426758,
      "activations/layer18_attention_weight_max": 51.22649383544922,
      "activations/layer18_attention_weight_min": -33.325042724609375,
      "activations/layer19_attention_weight_max": 25.889814376831055,
      "activations/layer19_attention_weight_min": -18.83734130859375,
      "activations/layer1_attention_weight_max": 17.881641387939453,
      "activations/layer1_attention_weight_min": -13.817255973815918,
      "activations/layer20_attention_weight_max": 26.393102645874023,
      "activations/layer20_attention_weight_min": -20.01152801513672,
      "activations/layer21_attention_weight_max": 47.80243682861328,
      "activations/layer21_attention_weight_min": -28.311302185058594,
      "activations/layer22_attention_weight_max": 39.87552261352539,
      "activations/layer22_attention_weight_min": -24.33264923095703,
      "activations/layer23_attention_weight_max": 29.269378662109375,
      "activations/layer23_attention_weight_min": -17.613079071044922,
      "activations/layer2_attention_weight_max": 29.624998092651367,
      "activations/layer2_attention_weight_min": -27.571813583374023,
      "activations/layer3_attention_weight_max": 76.15960693359375,
      "activations/layer3_attention_weight_min": -79.10033416748047,
      "activations/layer4_attention_weight_max": 102.9546890258789,
      "activations/layer4_attention_weight_min": -100.0455093383789,
      "activations/layer5_attention_weight_max": 121.73419189453125,
      "activations/layer5_attention_weight_min": -102.50869750976562,
      "activations/layer6_attention_weight_max": 60.36620330810547,
      "activations/layer6_attention_weight_min": -54.8409538269043,
      "activations/layer7_attention_weight_max": 66.71284484863281,
      "activations/layer7_attention_weight_min": -60.171443939208984,
      "activations/layer8_attention_weight_max": 40.860809326171875,
      "activations/layer8_attention_weight_min": -37.908226013183594,
      "activations/layer9_attention_weight_max": 55.786128997802734,
      "activations/layer9_attention_weight_min": -49.14059066772461,
      "epoch": 3.76,
      "learning_rate": 0.0001270534090909091,
      "loss": 2.8958,
      "step": 64650
    },
    {
      "activations/layer0_attention_weight_max": 13.815627098083496,
      "activations/layer0_attention_weight_min": -14.153724670410156,
      "activations/layer10_attention_weight_max": 38.15595626831055,
      "activations/layer10_attention_weight_min": -34.001033782958984,
      "activations/layer11_attention_weight_max": 30.062740325927734,
      "activations/layer11_attention_weight_min": -25.20612335205078,
      "activations/layer12_attention_weight_max": 24.2648868560791,
      "activations/layer12_attention_weight_min": -24.08222007751465,
      "activations/layer13_attention_weight_max": 60.10489273071289,
      "activations/layer13_attention_weight_min": -36.69841003417969,
      "activations/layer14_attention_weight_max": 41.95256423950195,
      "activations/layer14_attention_weight_min": -30.463640213012695,
      "activations/layer15_attention_weight_max": 40.95294952392578,
      "activations/layer15_attention_weight_min": -29.353906631469727,
      "activations/layer16_attention_weight_max": 37.445030212402344,
      "activations/layer16_attention_weight_min": -27.006418228149414,
      "activations/layer17_attention_weight_max": 57.62518310546875,
      "activations/layer17_attention_weight_min": -35.1418571472168,
      "activations/layer18_attention_weight_max": 44.87693405151367,
      "activations/layer18_attention_weight_min": -30.407766342163086,
      "activations/layer19_attention_weight_max": 21.757179260253906,
      "activations/layer19_attention_weight_min": -21.60443687438965,
      "activations/layer1_attention_weight_max": 17.647411346435547,
      "activations/layer1_attention_weight_min": -17.575952529907227,
      "activations/layer20_attention_weight_max": 20.3979434967041,
      "activations/layer20_attention_weight_min": -19.994300842285156,
      "activations/layer21_attention_weight_max": 44.94913864135742,
      "activations/layer21_attention_weight_min": -28.717784881591797,
      "activations/layer22_attention_weight_max": 30.983266830444336,
      "activations/layer22_attention_weight_min": -23.97452735900879,
      "activations/layer23_attention_weight_max": 25.04458236694336,
      "activations/layer23_attention_weight_min": -17.903884887695312,
      "activations/layer2_attention_weight_max": 31.322708129882812,
      "activations/layer2_attention_weight_min": -29.443279266357422,
      "activations/layer3_attention_weight_max": 75.73583984375,
      "activations/layer3_attention_weight_min": -83.34130096435547,
      "activations/layer4_attention_weight_max": 97.85423278808594,
      "activations/layer4_attention_weight_min": -91.47923278808594,
      "activations/layer5_attention_weight_max": 112.78335571289062,
      "activations/layer5_attention_weight_min": -97.83479309082031,
      "activations/layer6_attention_weight_max": 59.0907096862793,
      "activations/layer6_attention_weight_min": -50.36996841430664,
      "activations/layer7_attention_weight_max": 59.27181625366211,
      "activations/layer7_attention_weight_min": -59.63856506347656,
      "activations/layer8_attention_weight_max": 38.366859436035156,
      "activations/layer8_attention_weight_min": -37.09385681152344,
      "activations/layer9_attention_weight_max": 52.71211624145508,
      "activations/layer9_attention_weight_min": -45.31944274902344,
      "epoch": 3.76,
      "learning_rate": 0.00012703446969696968,
      "loss": 2.8849,
      "step": 64700
    },
    {
      "activations/layer0_attention_weight_max": 15.6851167678833,
      "activations/layer0_attention_weight_min": -14.670409202575684,
      "activations/layer10_attention_weight_max": 43.35841369628906,
      "activations/layer10_attention_weight_min": -31.794174194335938,
      "activations/layer11_attention_weight_max": 35.99208450317383,
      "activations/layer11_attention_weight_min": -26.047298431396484,
      "activations/layer12_attention_weight_max": 23.28213119506836,
      "activations/layer12_attention_weight_min": -26.646915435791016,
      "activations/layer13_attention_weight_max": 75.74639892578125,
      "activations/layer13_attention_weight_min": -48.36475372314453,
      "activations/layer14_attention_weight_max": 91.06072998046875,
      "activations/layer14_attention_weight_min": -50.81569290161133,
      "activations/layer15_attention_weight_max": 63.00647735595703,
      "activations/layer15_attention_weight_min": -39.19710922241211,
      "activations/layer16_attention_weight_max": 37.07699966430664,
      "activations/layer16_attention_weight_min": -27.45993423461914,
      "activations/layer17_attention_weight_max": 65.76685333251953,
      "activations/layer17_attention_weight_min": -39.4268684387207,
      "activations/layer18_attention_weight_max": 51.69930648803711,
      "activations/layer18_attention_weight_min": -33.43158721923828,
      "activations/layer19_attention_weight_max": 23.493310928344727,
      "activations/layer19_attention_weight_min": -20.45752716064453,
      "activations/layer1_attention_weight_max": 17.986156463623047,
      "activations/layer1_attention_weight_min": -13.573015213012695,
      "activations/layer20_attention_weight_max": 23.55228614807129,
      "activations/layer20_attention_weight_min": -19.40721893310547,
      "activations/layer21_attention_weight_max": 41.7064094543457,
      "activations/layer21_attention_weight_min": -26.115631103515625,
      "activations/layer22_attention_weight_max": 38.497432708740234,
      "activations/layer22_attention_weight_min": -23.078136444091797,
      "activations/layer23_attention_weight_max": 25.919105529785156,
      "activations/layer23_attention_weight_min": -16.392032623291016,
      "activations/layer2_attention_weight_max": 28.733543395996094,
      "activations/layer2_attention_weight_min": -28.781970977783203,
      "activations/layer3_attention_weight_max": 78.81389617919922,
      "activations/layer3_attention_weight_min": -82.96858215332031,
      "activations/layer4_attention_weight_max": 111.25040435791016,
      "activations/layer4_attention_weight_min": -97.49845123291016,
      "activations/layer5_attention_weight_max": 124.44689178466797,
      "activations/layer5_attention_weight_min": -103.5517807006836,
      "activations/layer6_attention_weight_max": 60.44557571411133,
      "activations/layer6_attention_weight_min": -51.826820373535156,
      "activations/layer7_attention_weight_max": 71.28617095947266,
      "activations/layer7_attention_weight_min": -61.89788055419922,
      "activations/layer8_attention_weight_max": 42.13669967651367,
      "activations/layer8_attention_weight_min": -39.836952209472656,
      "activations/layer9_attention_weight_max": 64.88289642333984,
      "activations/layer9_attention_weight_min": -49.48686981201172,
      "epoch": 3.76,
      "learning_rate": 0.0001270155303030303,
      "loss": 2.894,
      "step": 64750
    },
    {
      "activations/layer0_attention_weight_max": 14.667045593261719,
      "activations/layer0_attention_weight_min": -14.743152618408203,
      "activations/layer10_attention_weight_max": 39.32877731323242,
      "activations/layer10_attention_weight_min": -32.769798278808594,
      "activations/layer11_attention_weight_max": 32.679283142089844,
      "activations/layer11_attention_weight_min": -25.69099998474121,
      "activations/layer12_attention_weight_max": 24.13581657409668,
      "activations/layer12_attention_weight_min": -22.64931869506836,
      "activations/layer13_attention_weight_max": 44.67085266113281,
      "activations/layer13_attention_weight_min": -35.23044967651367,
      "activations/layer14_attention_weight_max": 41.51835632324219,
      "activations/layer14_attention_weight_min": -28.276073455810547,
      "activations/layer15_attention_weight_max": 38.786590576171875,
      "activations/layer15_attention_weight_min": -30.444578170776367,
      "activations/layer16_attention_weight_max": 36.494773864746094,
      "activations/layer16_attention_weight_min": -24.145259857177734,
      "activations/layer17_attention_weight_max": 56.98731231689453,
      "activations/layer17_attention_weight_min": -36.12685012817383,
      "activations/layer18_attention_weight_max": 50.02457046508789,
      "activations/layer18_attention_weight_min": -30.83552360534668,
      "activations/layer19_attention_weight_max": 21.926191329956055,
      "activations/layer19_attention_weight_min": -18.347410202026367,
      "activations/layer1_attention_weight_max": 17.474403381347656,
      "activations/layer1_attention_weight_min": -14.498038291931152,
      "activations/layer20_attention_weight_max": 20.0,
      "activations/layer20_attention_weight_min": -17.361114501953125,
      "activations/layer21_attention_weight_max": 38.63180160522461,
      "activations/layer21_attention_weight_min": -28.77521324157715,
      "activations/layer22_attention_weight_max": 32.36048889160156,
      "activations/layer22_attention_weight_min": -24.73115348815918,
      "activations/layer23_attention_weight_max": 29.84152603149414,
      "activations/layer23_attention_weight_min": -14.405858993530273,
      "activations/layer2_attention_weight_max": 30.330793380737305,
      "activations/layer2_attention_weight_min": -29.723478317260742,
      "activations/layer3_attention_weight_max": 76.37894439697266,
      "activations/layer3_attention_weight_min": -76.88325500488281,
      "activations/layer4_attention_weight_max": 103.39379119873047,
      "activations/layer4_attention_weight_min": -91.657470703125,
      "activations/layer5_attention_weight_max": 115.00929260253906,
      "activations/layer5_attention_weight_min": -100.57264709472656,
      "activations/layer6_attention_weight_max": 59.24542236328125,
      "activations/layer6_attention_weight_min": -54.73238754272461,
      "activations/layer7_attention_weight_max": 64.94296264648438,
      "activations/layer7_attention_weight_min": -57.51982498168945,
      "activations/layer8_attention_weight_max": 42.38473892211914,
      "activations/layer8_attention_weight_min": -39.15448760986328,
      "activations/layer9_attention_weight_max": 49.1959114074707,
      "activations/layer9_attention_weight_min": -44.99095916748047,
      "epoch": 3.77,
      "learning_rate": 0.0001269965909090909,
      "loss": 2.8671,
      "step": 64800
    },
    {
      "activations/layer0_attention_weight_max": 14.35327434539795,
      "activations/layer0_attention_weight_min": -14.892348289489746,
      "activations/layer10_attention_weight_max": 42.01375198364258,
      "activations/layer10_attention_weight_min": -34.95773696899414,
      "activations/layer11_attention_weight_max": 32.67671203613281,
      "activations/layer11_attention_weight_min": -27.135534286499023,
      "activations/layer12_attention_weight_max": 27.172168731689453,
      "activations/layer12_attention_weight_min": -26.974584579467773,
      "activations/layer13_attention_weight_max": 67.25161743164062,
      "activations/layer13_attention_weight_min": -40.13798904418945,
      "activations/layer14_attention_weight_max": 65.17437744140625,
      "activations/layer14_attention_weight_min": -34.737056732177734,
      "activations/layer15_attention_weight_max": 52.40654373168945,
      "activations/layer15_attention_weight_min": -33.994239807128906,
      "activations/layer16_attention_weight_max": 43.39839553833008,
      "activations/layer16_attention_weight_min": -25.77585792541504,
      "activations/layer17_attention_weight_max": 64.56925201416016,
      "activations/layer17_attention_weight_min": -39.96339797973633,
      "activations/layer18_attention_weight_max": 61.21116256713867,
      "activations/layer18_attention_weight_min": -34.05465316772461,
      "activations/layer19_attention_weight_max": 27.004379272460938,
      "activations/layer19_attention_weight_min": -19.927915573120117,
      "activations/layer1_attention_weight_max": 18.013151168823242,
      "activations/layer1_attention_weight_min": -15.773961067199707,
      "activations/layer20_attention_weight_max": 25.676424026489258,
      "activations/layer20_attention_weight_min": -18.744840621948242,
      "activations/layer21_attention_weight_max": 44.38542556762695,
      "activations/layer21_attention_weight_min": -26.693035125732422,
      "activations/layer22_attention_weight_max": 40.412254333496094,
      "activations/layer22_attention_weight_min": -22.147552490234375,
      "activations/layer23_attention_weight_max": 29.91210174560547,
      "activations/layer23_attention_weight_min": -16.880945205688477,
      "activations/layer2_attention_weight_max": 28.207143783569336,
      "activations/layer2_attention_weight_min": -25.168033599853516,
      "activations/layer3_attention_weight_max": 74.8188247680664,
      "activations/layer3_attention_weight_min": -79.80625915527344,
      "activations/layer4_attention_weight_max": 97.1748275756836,
      "activations/layer4_attention_weight_min": -91.6124496459961,
      "activations/layer5_attention_weight_max": 112.54611206054688,
      "activations/layer5_attention_weight_min": -94.64244079589844,
      "activations/layer6_attention_weight_max": 57.50620651245117,
      "activations/layer6_attention_weight_min": -50.46272277832031,
      "activations/layer7_attention_weight_max": 61.18905258178711,
      "activations/layer7_attention_weight_min": -57.91191482543945,
      "activations/layer8_attention_weight_max": 40.72927474975586,
      "activations/layer8_attention_weight_min": -38.86216354370117,
      "activations/layer9_attention_weight_max": 53.79110336303711,
      "activations/layer9_attention_weight_min": -50.00925827026367,
      "epoch": 3.77,
      "learning_rate": 0.0001269776515151515,
      "loss": 2.9096,
      "step": 64850
    },
    {
      "activations/layer0_attention_weight_max": 15.140466690063477,
      "activations/layer0_attention_weight_min": -15.176043510437012,
      "activations/layer10_attention_weight_max": 37.87909698486328,
      "activations/layer10_attention_weight_min": -32.05342483520508,
      "activations/layer11_attention_weight_max": 36.2879753112793,
      "activations/layer11_attention_weight_min": -27.664901733398438,
      "activations/layer12_attention_weight_max": 23.582717895507812,
      "activations/layer12_attention_weight_min": -26.3155460357666,
      "activations/layer13_attention_weight_max": 64.9957275390625,
      "activations/layer13_attention_weight_min": -44.581932067871094,
      "activations/layer14_attention_weight_max": 57.08139419555664,
      "activations/layer14_attention_weight_min": -36.12213897705078,
      "activations/layer15_attention_weight_max": 47.20466995239258,
      "activations/layer15_attention_weight_min": -29.10994529724121,
      "activations/layer16_attention_weight_max": 38.0435905456543,
      "activations/layer16_attention_weight_min": -25.5065860748291,
      "activations/layer17_attention_weight_max": 59.52444839477539,
      "activations/layer17_attention_weight_min": -35.43900680541992,
      "activations/layer18_attention_weight_max": 55.07537078857422,
      "activations/layer18_attention_weight_min": -33.97065734863281,
      "activations/layer19_attention_weight_max": 22.98886489868164,
      "activations/layer19_attention_weight_min": -18.14628028869629,
      "activations/layer1_attention_weight_max": 18.032310485839844,
      "activations/layer1_attention_weight_min": -15.386236190795898,
      "activations/layer20_attention_weight_max": 22.68230628967285,
      "activations/layer20_attention_weight_min": -18.157794952392578,
      "activations/layer21_attention_weight_max": 42.216949462890625,
      "activations/layer21_attention_weight_min": -25.039762496948242,
      "activations/layer22_attention_weight_max": 39.36520767211914,
      "activations/layer22_attention_weight_min": -22.125173568725586,
      "activations/layer23_attention_weight_max": 26.013511657714844,
      "activations/layer23_attention_weight_min": -16.861188888549805,
      "activations/layer2_attention_weight_max": 30.135665893554688,
      "activations/layer2_attention_weight_min": -27.590808868408203,
      "activations/layer3_attention_weight_max": 76.16863250732422,
      "activations/layer3_attention_weight_min": -78.22200012207031,
      "activations/layer4_attention_weight_max": 101.46270751953125,
      "activations/layer4_attention_weight_min": -93.06978607177734,
      "activations/layer5_attention_weight_max": 115.00057220458984,
      "activations/layer5_attention_weight_min": -99.64297485351562,
      "activations/layer6_attention_weight_max": 58.85763931274414,
      "activations/layer6_attention_weight_min": -52.244407653808594,
      "activations/layer7_attention_weight_max": 62.89254379272461,
      "activations/layer7_attention_weight_min": -59.31010055541992,
      "activations/layer8_attention_weight_max": 39.6271858215332,
      "activations/layer8_attention_weight_min": -36.20180130004883,
      "activations/layer9_attention_weight_max": 54.50508499145508,
      "activations/layer9_attention_weight_min": -50.8942985534668,
      "epoch": 3.77,
      "learning_rate": 0.0001269587121212121,
      "loss": 2.8633,
      "step": 64900
    },
    {
      "activations/layer0_attention_weight_max": 14.651485443115234,
      "activations/layer0_attention_weight_min": -15.089200973510742,
      "activations/layer10_attention_weight_max": 42.97848892211914,
      "activations/layer10_attention_weight_min": -34.824424743652344,
      "activations/layer11_attention_weight_max": 32.343482971191406,
      "activations/layer11_attention_weight_min": -26.8519344329834,
      "activations/layer12_attention_weight_max": 25.029800415039062,
      "activations/layer12_attention_weight_min": -24.12957191467285,
      "activations/layer13_attention_weight_max": 59.2799072265625,
      "activations/layer13_attention_weight_min": -35.5440673828125,
      "activations/layer14_attention_weight_max": 52.22526168823242,
      "activations/layer14_attention_weight_min": -29.89437484741211,
      "activations/layer15_attention_weight_max": 48.069854736328125,
      "activations/layer15_attention_weight_min": -32.563175201416016,
      "activations/layer16_attention_weight_max": 40.35084533691406,
      "activations/layer16_attention_weight_min": -25.824064254760742,
      "activations/layer17_attention_weight_max": 64.82996368408203,
      "activations/layer17_attention_weight_min": -35.871131896972656,
      "activations/layer18_attention_weight_max": 51.2262077331543,
      "activations/layer18_attention_weight_min": -31.959102630615234,
      "activations/layer19_attention_weight_max": 27.928726196289062,
      "activations/layer19_attention_weight_min": -18.47840690612793,
      "activations/layer1_attention_weight_max": 18.507095336914062,
      "activations/layer1_attention_weight_min": -15.949982643127441,
      "activations/layer20_attention_weight_max": 21.665660858154297,
      "activations/layer20_attention_weight_min": -21.737199783325195,
      "activations/layer21_attention_weight_max": 42.387786865234375,
      "activations/layer21_attention_weight_min": -29.651548385620117,
      "activations/layer22_attention_weight_max": 41.02922058105469,
      "activations/layer22_attention_weight_min": -22.097061157226562,
      "activations/layer23_attention_weight_max": 27.27788543701172,
      "activations/layer23_attention_weight_min": -20.096084594726562,
      "activations/layer2_attention_weight_max": 29.879064559936523,
      "activations/layer2_attention_weight_min": -27.84011459350586,
      "activations/layer3_attention_weight_max": 73.15142822265625,
      "activations/layer3_attention_weight_min": -78.41282653808594,
      "activations/layer4_attention_weight_max": 103.3363265991211,
      "activations/layer4_attention_weight_min": -95.07382202148438,
      "activations/layer5_attention_weight_max": 121.88180541992188,
      "activations/layer5_attention_weight_min": -103.07539367675781,
      "activations/layer6_attention_weight_max": 63.03889083862305,
      "activations/layer6_attention_weight_min": -54.58456802368164,
      "activations/layer7_attention_weight_max": 67.70854187011719,
      "activations/layer7_attention_weight_min": -59.60834503173828,
      "activations/layer8_attention_weight_max": 44.881649017333984,
      "activations/layer8_attention_weight_min": -40.87839126586914,
      "activations/layer9_attention_weight_max": 55.00844192504883,
      "activations/layer9_attention_weight_min": -48.03901290893555,
      "epoch": 3.77,
      "learning_rate": 0.0001269397727272727,
      "loss": 2.8797,
      "step": 64950
    },
    {
      "activations/layer0_attention_weight_max": 14.058923721313477,
      "activations/layer0_attention_weight_min": -15.72436809539795,
      "activations/layer10_attention_weight_max": 39.09066390991211,
      "activations/layer10_attention_weight_min": -34.111244201660156,
      "activations/layer11_attention_weight_max": 32.06906509399414,
      "activations/layer11_attention_weight_min": -25.539676666259766,
      "activations/layer12_attention_weight_max": 40.55348587036133,
      "activations/layer12_attention_weight_min": -23.69622039794922,
      "activations/layer13_attention_weight_max": 52.52416229248047,
      "activations/layer13_attention_weight_min": -37.70494079589844,
      "activations/layer14_attention_weight_max": 40.85783767700195,
      "activations/layer14_attention_weight_min": -31.78861427307129,
      "activations/layer15_attention_weight_max": 45.64449691772461,
      "activations/layer15_attention_weight_min": -32.41747283935547,
      "activations/layer16_attention_weight_max": 32.64246368408203,
      "activations/layer16_attention_weight_min": -27.968774795532227,
      "activations/layer17_attention_weight_max": 59.66078186035156,
      "activations/layer17_attention_weight_min": -41.10078430175781,
      "activations/layer18_attention_weight_max": 59.9091682434082,
      "activations/layer18_attention_weight_min": -35.333587646484375,
      "activations/layer19_attention_weight_max": 27.840625762939453,
      "activations/layer19_attention_weight_min": -17.578432083129883,
      "activations/layer1_attention_weight_max": 17.07346534729004,
      "activations/layer1_attention_weight_min": -13.922904968261719,
      "activations/layer20_attention_weight_max": 21.142864227294922,
      "activations/layer20_attention_weight_min": -22.391054153442383,
      "activations/layer21_attention_weight_max": 40.464263916015625,
      "activations/layer21_attention_weight_min": -28.77743148803711,
      "activations/layer22_attention_weight_max": 37.47766876220703,
      "activations/layer22_attention_weight_min": -24.005958557128906,
      "activations/layer23_attention_weight_max": 28.330617904663086,
      "activations/layer23_attention_weight_min": -16.4825439453125,
      "activations/layer2_attention_weight_max": 30.3087158203125,
      "activations/layer2_attention_weight_min": -27.26453399658203,
      "activations/layer3_attention_weight_max": 75.30126953125,
      "activations/layer3_attention_weight_min": -79.04727172851562,
      "activations/layer4_attention_weight_max": 102.0177001953125,
      "activations/layer4_attention_weight_min": -92.90013885498047,
      "activations/layer5_attention_weight_max": 120.54193115234375,
      "activations/layer5_attention_weight_min": -100.3640365600586,
      "activations/layer6_attention_weight_max": 61.1893196105957,
      "activations/layer6_attention_weight_min": -52.865047454833984,
      "activations/layer7_attention_weight_max": 61.704193115234375,
      "activations/layer7_attention_weight_min": -57.49424362182617,
      "activations/layer8_attention_weight_max": 40.71681594848633,
      "activations/layer8_attention_weight_min": -39.361724853515625,
      "activations/layer9_attention_weight_max": 48.40263748168945,
      "activations/layer9_attention_weight_min": -46.12985610961914,
      "epoch": 3.78,
      "learning_rate": 0.0001269208333333333,
      "loss": 2.8883,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_loss": 2.833984375,
      "eval_runtime": 8.4249,
      "eval_samples_per_second": 509.679,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_openwebtext_loss": 2.833984375,
      "eval_openwebtext_ppl": 17.013112580497378,
      "eval_openwebtext_runtime": 8.4249,
      "eval_openwebtext_samples_per_second": 509.679,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_wikitext_loss": 3.064453125,
      "eval_wikitext_ppl": 21.42274322042474,
      "eval_wikitext_runtime": 1.9324,
      "eval_wikitext_samples_per_second": 235.971,
      "step": 65000
    },
    {
      "epoch": 3.78,
      "eval_lambada_loss": 2.923828125,
      "eval_lambada_ppl": 18.61240185635776,
      "eval_lambada_runtime": 9.5194,
      "eval_lambada_samples_per_second": 511.484,
      "step": 65000
    },
    {
      "activations/layer0_attention_weight_max": 16.325044631958008,
      "activations/layer0_attention_weight_min": -15.23422908782959,
      "activations/layer10_attention_weight_max": 39.842674255371094,
      "activations/layer10_attention_weight_min": -32.6484375,
      "activations/layer11_attention_weight_max": 30.504165649414062,
      "activations/layer11_attention_weight_min": -25.672605514526367,
      "activations/layer12_attention_weight_max": 25.78451156616211,
      "activations/layer12_attention_weight_min": -24.466426849365234,
      "activations/layer13_attention_weight_max": 47.929649353027344,
      "activations/layer13_attention_weight_min": -37.053077697753906,
      "activations/layer14_attention_weight_max": 39.49257278442383,
      "activations/layer14_attention_weight_min": -28.37029457092285,
      "activations/layer15_attention_weight_max": 40.79252624511719,
      "activations/layer15_attention_weight_min": -31.159610748291016,
      "activations/layer16_attention_weight_max": 37.00096130371094,
      "activations/layer16_attention_weight_min": -25.449657440185547,
      "activations/layer17_attention_weight_max": 58.19148635864258,
      "activations/layer17_attention_weight_min": -35.29411315917969,
      "activations/layer18_attention_weight_max": 52.51155471801758,
      "activations/layer18_attention_weight_min": -33.90171813964844,
      "activations/layer19_attention_weight_max": 21.877220153808594,
      "activations/layer19_attention_weight_min": -18.15861701965332,
      "activations/layer1_attention_weight_max": 17.706449508666992,
      "activations/layer1_attention_weight_min": -15.576765060424805,
      "activations/layer20_attention_weight_max": 19.85456085205078,
      "activations/layer20_attention_weight_min": -22.057571411132812,
      "activations/layer21_attention_weight_max": 36.751434326171875,
      "activations/layer21_attention_weight_min": -24.538066864013672,
      "activations/layer22_attention_weight_max": 37.49205780029297,
      "activations/layer22_attention_weight_min": -23.177183151245117,
      "activations/layer23_attention_weight_max": 26.439931869506836,
      "activations/layer23_attention_weight_min": -15.836627006530762,
      "activations/layer2_attention_weight_max": 29.34246826171875,
      "activations/layer2_attention_weight_min": -26.576602935791016,
      "activations/layer3_attention_weight_max": 72.41779327392578,
      "activations/layer3_attention_weight_min": -78.48619842529297,
      "activations/layer4_attention_weight_max": 94.96312713623047,
      "activations/layer4_attention_weight_min": -89.88634490966797,
      "activations/layer5_attention_weight_max": 109.0765609741211,
      "activations/layer5_attention_weight_min": -102.90379333496094,
      "activations/layer6_attention_weight_max": 60.29589080810547,
      "activations/layer6_attention_weight_min": -55.410011291503906,
      "activations/layer7_attention_weight_max": 60.5356330871582,
      "activations/layer7_attention_weight_min": -56.56218338012695,
      "activations/layer8_attention_weight_max": 40.03329086303711,
      "activations/layer8_attention_weight_min": -37.85321044921875,
      "activations/layer9_attention_weight_max": 48.02932357788086,
      "activations/layer9_attention_weight_min": -43.970054626464844,
      "epoch": 3.78,
      "learning_rate": 0.00012690189393939393,
      "loss": 2.8837,
      "step": 65050
    },
    {
      "activations/layer0_attention_weight_max": 14.096479415893555,
      "activations/layer0_attention_weight_min": -15.607635498046875,
      "activations/layer10_attention_weight_max": 46.27314376831055,
      "activations/layer10_attention_weight_min": -35.906532287597656,
      "activations/layer11_attention_weight_max": 33.70154571533203,
      "activations/layer11_attention_weight_min": -29.178762435913086,
      "activations/layer12_attention_weight_max": 27.287870407104492,
      "activations/layer12_attention_weight_min": -26.540607452392578,
      "activations/layer13_attention_weight_max": 45.87779235839844,
      "activations/layer13_attention_weight_min": -36.39188766479492,
      "activations/layer14_attention_weight_max": 38.72248077392578,
      "activations/layer14_attention_weight_min": -28.524532318115234,
      "activations/layer15_attention_weight_max": 50.07024002075195,
      "activations/layer15_attention_weight_min": -32.36006164550781,
      "activations/layer16_attention_weight_max": 35.60541534423828,
      "activations/layer16_attention_weight_min": -28.454395294189453,
      "activations/layer17_attention_weight_max": 57.74786376953125,
      "activations/layer17_attention_weight_min": -38.0687370300293,
      "activations/layer18_attention_weight_max": 48.58204650878906,
      "activations/layer18_attention_weight_min": -32.800018310546875,
      "activations/layer19_attention_weight_max": 20.26624870300293,
      "activations/layer19_attention_weight_min": -19.79616355895996,
      "activations/layer1_attention_weight_max": 18.83167266845703,
      "activations/layer1_attention_weight_min": -16.364953994750977,
      "activations/layer20_attention_weight_max": 20.65160369873047,
      "activations/layer20_attention_weight_min": -19.78890037536621,
      "activations/layer21_attention_weight_max": 39.45276641845703,
      "activations/layer21_attention_weight_min": -28.262590408325195,
      "activations/layer22_attention_weight_max": 34.01333999633789,
      "activations/layer22_attention_weight_min": -22.82434844970703,
      "activations/layer23_attention_weight_max": 26.715702056884766,
      "activations/layer23_attention_weight_min": -16.87959861755371,
      "activations/layer2_attention_weight_max": 29.79286766052246,
      "activations/layer2_attention_weight_min": -27.56391716003418,
      "activations/layer3_attention_weight_max": 78.67680358886719,
      "activations/layer3_attention_weight_min": -79.98192596435547,
      "activations/layer4_attention_weight_max": 108.70096588134766,
      "activations/layer4_attention_weight_min": -99.3600845336914,
      "activations/layer5_attention_weight_max": 117.65782165527344,
      "activations/layer5_attention_weight_min": -106.38384246826172,
      "activations/layer6_attention_weight_max": 63.40125274658203,
      "activations/layer6_attention_weight_min": -55.18769073486328,
      "activations/layer7_attention_weight_max": 72.23824310302734,
      "activations/layer7_attention_weight_min": -59.08355712890625,
      "activations/layer8_attention_weight_max": 42.216575622558594,
      "activations/layer8_attention_weight_min": -38.77949905395508,
      "activations/layer9_attention_weight_max": 50.04421615600586,
      "activations/layer9_attention_weight_min": -48.3421516418457,
      "epoch": 3.78,
      "learning_rate": 0.00012688295454545454,
      "loss": 2.8967,
      "step": 65100
    },
    {
      "activations/layer0_attention_weight_max": 14.111002922058105,
      "activations/layer0_attention_weight_min": -14.926695823669434,
      "activations/layer10_attention_weight_max": 44.05839920043945,
      "activations/layer10_attention_weight_min": -35.10075759887695,
      "activations/layer11_attention_weight_max": 36.07408905029297,
      "activations/layer11_attention_weight_min": -28.494815826416016,
      "activations/layer12_attention_weight_max": 26.7657413482666,
      "activations/layer12_attention_weight_min": -27.204822540283203,
      "activations/layer13_attention_weight_max": 55.48662185668945,
      "activations/layer13_attention_weight_min": -34.2171516418457,
      "activations/layer14_attention_weight_max": 49.889522552490234,
      "activations/layer14_attention_weight_min": -32.469825744628906,
      "activations/layer15_attention_weight_max": 45.38678741455078,
      "activations/layer15_attention_weight_min": -30.399696350097656,
      "activations/layer16_attention_weight_max": 37.197017669677734,
      "activations/layer16_attention_weight_min": -25.87314224243164,
      "activations/layer17_attention_weight_max": 59.55421447753906,
      "activations/layer17_attention_weight_min": -37.09462356567383,
      "activations/layer18_attention_weight_max": 54.4790153503418,
      "activations/layer18_attention_weight_min": -30.355064392089844,
      "activations/layer19_attention_weight_max": 24.47765350341797,
      "activations/layer19_attention_weight_min": -20.011075973510742,
      "activations/layer1_attention_weight_max": 17.350282669067383,
      "activations/layer1_attention_weight_min": -16.557941436767578,
      "activations/layer20_attention_weight_max": 20.922725677490234,
      "activations/layer20_attention_weight_min": -19.809833526611328,
      "activations/layer21_attention_weight_max": 41.420021057128906,
      "activations/layer21_attention_weight_min": -27.716772079467773,
      "activations/layer22_attention_weight_max": 39.293601989746094,
      "activations/layer22_attention_weight_min": -23.85103988647461,
      "activations/layer23_attention_weight_max": 27.49671745300293,
      "activations/layer23_attention_weight_min": -18.022058486938477,
      "activations/layer2_attention_weight_max": 28.510093688964844,
      "activations/layer2_attention_weight_min": -26.704763412475586,
      "activations/layer3_attention_weight_max": 72.90897369384766,
      "activations/layer3_attention_weight_min": -74.49868774414062,
      "activations/layer4_attention_weight_max": 98.9136734008789,
      "activations/layer4_attention_weight_min": -91.98198699951172,
      "activations/layer5_attention_weight_max": 115.25222778320312,
      "activations/layer5_attention_weight_min": -99.91598510742188,
      "activations/layer6_attention_weight_max": 58.95838928222656,
      "activations/layer6_attention_weight_min": -51.61332321166992,
      "activations/layer7_attention_weight_max": 69.50804138183594,
      "activations/layer7_attention_weight_min": -56.66232681274414,
      "activations/layer8_attention_weight_max": 41.76253890991211,
      "activations/layer8_attention_weight_min": -36.34646987915039,
      "activations/layer9_attention_weight_max": 59.49873733520508,
      "activations/layer9_attention_weight_min": -45.956443786621094,
      "epoch": 3.79,
      "learning_rate": 0.00012686401515151513,
      "loss": 2.8732,
      "step": 65150
    },
    {
      "activations/layer0_attention_weight_max": 14.288817405700684,
      "activations/layer0_attention_weight_min": -14.774662971496582,
      "activations/layer10_attention_weight_max": 36.82349395751953,
      "activations/layer10_attention_weight_min": -32.388763427734375,
      "activations/layer11_attention_weight_max": 30.46335220336914,
      "activations/layer11_attention_weight_min": -26.740562438964844,
      "activations/layer12_attention_weight_max": 23.451194763183594,
      "activations/layer12_attention_weight_min": -22.82688331604004,
      "activations/layer13_attention_weight_max": 44.140602111816406,
      "activations/layer13_attention_weight_min": -34.030723571777344,
      "activations/layer14_attention_weight_max": 57.3631706237793,
      "activations/layer14_attention_weight_min": -38.888458251953125,
      "activations/layer15_attention_weight_max": 42.3768424987793,
      "activations/layer15_attention_weight_min": -31.221118927001953,
      "activations/layer16_attention_weight_max": 36.65921401977539,
      "activations/layer16_attention_weight_min": -30.73351287841797,
      "activations/layer17_attention_weight_max": 59.23120880126953,
      "activations/layer17_attention_weight_min": -38.27046203613281,
      "activations/layer18_attention_weight_max": 49.05545425415039,
      "activations/layer18_attention_weight_min": -32.88898849487305,
      "activations/layer19_attention_weight_max": 24.592058181762695,
      "activations/layer19_attention_weight_min": -18.20949935913086,
      "activations/layer1_attention_weight_max": 16.83426856994629,
      "activations/layer1_attention_weight_min": -16.761444091796875,
      "activations/layer20_attention_weight_max": 20.78215789794922,
      "activations/layer20_attention_weight_min": -21.432390213012695,
      "activations/layer21_attention_weight_max": 40.821067810058594,
      "activations/layer21_attention_weight_min": -28.65202522277832,
      "activations/layer22_attention_weight_max": 35.275081634521484,
      "activations/layer22_attention_weight_min": -23.39781951904297,
      "activations/layer23_attention_weight_max": 26.622486114501953,
      "activations/layer23_attention_weight_min": -16.38446044921875,
      "activations/layer2_attention_weight_max": 29.827238082885742,
      "activations/layer2_attention_weight_min": -27.95083999633789,
      "activations/layer3_attention_weight_max": 73.56201934814453,
      "activations/layer3_attention_weight_min": -81.56209564208984,
      "activations/layer4_attention_weight_max": 101.55152893066406,
      "activations/layer4_attention_weight_min": -93.0009994506836,
      "activations/layer5_attention_weight_max": 113.2197494506836,
      "activations/layer5_attention_weight_min": -95.30009460449219,
      "activations/layer6_attention_weight_max": 58.833778381347656,
      "activations/layer6_attention_weight_min": -53.240577697753906,
      "activations/layer7_attention_weight_max": 59.27241516113281,
      "activations/layer7_attention_weight_min": -57.073760986328125,
      "activations/layer8_attention_weight_max": 39.97866439819336,
      "activations/layer8_attention_weight_min": -36.322532653808594,
      "activations/layer9_attention_weight_max": 47.25568389892578,
      "activations/layer9_attention_weight_min": -45.0502815246582,
      "epoch": 3.79,
      "learning_rate": 0.00012684507575757575,
      "loss": 2.8811,
      "step": 65200
    },
    {
      "activations/layer0_attention_weight_max": 14.456181526184082,
      "activations/layer0_attention_weight_min": -14.684077262878418,
      "activations/layer10_attention_weight_max": 37.32997512817383,
      "activations/layer10_attention_weight_min": -32.23792266845703,
      "activations/layer11_attention_weight_max": 31.251453399658203,
      "activations/layer11_attention_weight_min": -26.8209228515625,
      "activations/layer12_attention_weight_max": 24.08207893371582,
      "activations/layer12_attention_weight_min": -27.40603256225586,
      "activations/layer13_attention_weight_max": 50.62080001831055,
      "activations/layer13_attention_weight_min": -37.49113082885742,
      "activations/layer14_attention_weight_max": 42.999000549316406,
      "activations/layer14_attention_weight_min": -30.465911865234375,
      "activations/layer15_attention_weight_max": 44.49347686767578,
      "activations/layer15_attention_weight_min": -31.412410736083984,
      "activations/layer16_attention_weight_max": 35.28238296508789,
      "activations/layer16_attention_weight_min": -27.011035919189453,
      "activations/layer17_attention_weight_max": 58.04975128173828,
      "activations/layer17_attention_weight_min": -35.147850036621094,
      "activations/layer18_attention_weight_max": 51.331111907958984,
      "activations/layer18_attention_weight_min": -35.15230178833008,
      "activations/layer19_attention_weight_max": 22.768388748168945,
      "activations/layer19_attention_weight_min": -17.86507797241211,
      "activations/layer1_attention_weight_max": 17.45855140686035,
      "activations/layer1_attention_weight_min": -13.116436958312988,
      "activations/layer20_attention_weight_max": 20.802011489868164,
      "activations/layer20_attention_weight_min": -19.358631134033203,
      "activations/layer21_attention_weight_max": 39.21503829956055,
      "activations/layer21_attention_weight_min": -27.580780029296875,
      "activations/layer22_attention_weight_max": 37.607688903808594,
      "activations/layer22_attention_weight_min": -24.319644927978516,
      "activations/layer23_attention_weight_max": 24.131010055541992,
      "activations/layer23_attention_weight_min": -17.799728393554688,
      "activations/layer2_attention_weight_max": 29.240554809570312,
      "activations/layer2_attention_weight_min": -27.67910385131836,
      "activations/layer3_attention_weight_max": 77.8597183227539,
      "activations/layer3_attention_weight_min": -85.55657958984375,
      "activations/layer4_attention_weight_max": 105.11547088623047,
      "activations/layer4_attention_weight_min": -99.57527923583984,
      "activations/layer5_attention_weight_max": 113.30221557617188,
      "activations/layer5_attention_weight_min": -106.2841796875,
      "activations/layer6_attention_weight_max": 59.42770767211914,
      "activations/layer6_attention_weight_min": -53.414371490478516,
      "activations/layer7_attention_weight_max": 65.94182586669922,
      "activations/layer7_attention_weight_min": -63.15843963623047,
      "activations/layer8_attention_weight_max": 39.90993881225586,
      "activations/layer8_attention_weight_min": -37.45378494262695,
      "activations/layer9_attention_weight_max": 53.40305709838867,
      "activations/layer9_attention_weight_min": -45.308170318603516,
      "epoch": 3.79,
      "learning_rate": 0.00012682613636363636,
      "loss": 2.8844,
      "step": 65250
    },
    {
      "activations/layer0_attention_weight_max": 14.308423042297363,
      "activations/layer0_attention_weight_min": -14.751240730285645,
      "activations/layer10_attention_weight_max": 38.01669692993164,
      "activations/layer10_attention_weight_min": -32.39832305908203,
      "activations/layer11_attention_weight_max": 31.637670516967773,
      "activations/layer11_attention_weight_min": -24.279767990112305,
      "activations/layer12_attention_weight_max": 23.480613708496094,
      "activations/layer12_attention_weight_min": -28.200963973999023,
      "activations/layer13_attention_weight_max": 45.33086395263672,
      "activations/layer13_attention_weight_min": -37.929840087890625,
      "activations/layer14_attention_weight_max": 37.06111145019531,
      "activations/layer14_attention_weight_min": -28.422998428344727,
      "activations/layer15_attention_weight_max": 46.42293167114258,
      "activations/layer15_attention_weight_min": -29.634321212768555,
      "activations/layer16_attention_weight_max": 33.243465423583984,
      "activations/layer16_attention_weight_min": -25.165695190429688,
      "activations/layer17_attention_weight_max": 51.919166564941406,
      "activations/layer17_attention_weight_min": -34.09779739379883,
      "activations/layer18_attention_weight_max": 46.72599792480469,
      "activations/layer18_attention_weight_min": -31.241628646850586,
      "activations/layer19_attention_weight_max": 20.361387252807617,
      "activations/layer19_attention_weight_min": -19.357709884643555,
      "activations/layer1_attention_weight_max": 17.464418411254883,
      "activations/layer1_attention_weight_min": -15.399412155151367,
      "activations/layer20_attention_weight_max": 21.84943389892578,
      "activations/layer20_attention_weight_min": -21.97107696533203,
      "activations/layer21_attention_weight_max": 38.55806350708008,
      "activations/layer21_attention_weight_min": -25.276243209838867,
      "activations/layer22_attention_weight_max": 34.467552185058594,
      "activations/layer22_attention_weight_min": -22.508901596069336,
      "activations/layer23_attention_weight_max": 25.92026138305664,
      "activations/layer23_attention_weight_min": -18.285533905029297,
      "activations/layer2_attention_weight_max": 30.14216423034668,
      "activations/layer2_attention_weight_min": -27.02576446533203,
      "activations/layer3_attention_weight_max": 75.51099395751953,
      "activations/layer3_attention_weight_min": -79.11654663085938,
      "activations/layer4_attention_weight_max": 95.75284576416016,
      "activations/layer4_attention_weight_min": -93.8892593383789,
      "activations/layer5_attention_weight_max": 113.81806945800781,
      "activations/layer5_attention_weight_min": -100.8379135131836,
      "activations/layer6_attention_weight_max": 61.22751998901367,
      "activations/layer6_attention_weight_min": -51.73518753051758,
      "activations/layer7_attention_weight_max": 61.490142822265625,
      "activations/layer7_attention_weight_min": -57.87261962890625,
      "activations/layer8_attention_weight_max": 40.68846130371094,
      "activations/layer8_attention_weight_min": -37.096580505371094,
      "activations/layer9_attention_weight_max": 45.04729461669922,
      "activations/layer9_attention_weight_min": -44.786659240722656,
      "epoch": 3.79,
      "learning_rate": 0.00012680719696969695,
      "loss": 2.8961,
      "step": 65300
    },
    {
      "activations/layer0_attention_weight_max": 15.165872573852539,
      "activations/layer0_attention_weight_min": -14.890292167663574,
      "activations/layer10_attention_weight_max": 43.59394073486328,
      "activations/layer10_attention_weight_min": -33.968692779541016,
      "activations/layer11_attention_weight_max": 34.236610412597656,
      "activations/layer11_attention_weight_min": -26.404380798339844,
      "activations/layer12_attention_weight_max": 26.5610294342041,
      "activations/layer12_attention_weight_min": -23.924259185791016,
      "activations/layer13_attention_weight_max": 50.3476448059082,
      "activations/layer13_attention_weight_min": -35.3616943359375,
      "activations/layer14_attention_weight_max": 44.76893615722656,
      "activations/layer14_attention_weight_min": -30.2481632232666,
      "activations/layer15_attention_weight_max": 49.08051300048828,
      "activations/layer15_attention_weight_min": -32.53468704223633,
      "activations/layer16_attention_weight_max": 45.394222259521484,
      "activations/layer16_attention_weight_min": -26.511085510253906,
      "activations/layer17_attention_weight_max": 59.97152328491211,
      "activations/layer17_attention_weight_min": -35.3871955871582,
      "activations/layer18_attention_weight_max": 52.66703414916992,
      "activations/layer18_attention_weight_min": -31.333309173583984,
      "activations/layer19_attention_weight_max": 26.283193588256836,
      "activations/layer19_attention_weight_min": -17.41779899597168,
      "activations/layer1_attention_weight_max": 17.051774978637695,
      "activations/layer1_attention_weight_min": -14.783178329467773,
      "activations/layer20_attention_weight_max": 25.744117736816406,
      "activations/layer20_attention_weight_min": -18.422298431396484,
      "activations/layer21_attention_weight_max": 42.53130340576172,
      "activations/layer21_attention_weight_min": -29.966196060180664,
      "activations/layer22_attention_weight_max": 36.91273498535156,
      "activations/layer22_attention_weight_min": -24.542999267578125,
      "activations/layer23_attention_weight_max": 29.343177795410156,
      "activations/layer23_attention_weight_min": -19.454347610473633,
      "activations/layer2_attention_weight_max": 29.71392059326172,
      "activations/layer2_attention_weight_min": -26.686864852905273,
      "activations/layer3_attention_weight_max": 76.1927261352539,
      "activations/layer3_attention_weight_min": -80.03736877441406,
      "activations/layer4_attention_weight_max": 100.57725524902344,
      "activations/layer4_attention_weight_min": -98.2931137084961,
      "activations/layer5_attention_weight_max": 112.92843627929688,
      "activations/layer5_attention_weight_min": -95.23262023925781,
      "activations/layer6_attention_weight_max": 60.017093658447266,
      "activations/layer6_attention_weight_min": -50.44579315185547,
      "activations/layer7_attention_weight_max": 64.4019775390625,
      "activations/layer7_attention_weight_min": -55.76316452026367,
      "activations/layer8_attention_weight_max": 42.3260498046875,
      "activations/layer8_attention_weight_min": -38.30267333984375,
      "activations/layer9_attention_weight_max": 52.6270866394043,
      "activations/layer9_attention_weight_min": -45.61262512207031,
      "epoch": 3.8,
      "learning_rate": 0.00012678825757575756,
      "loss": 2.8875,
      "step": 65350
    },
    {
      "activations/layer0_attention_weight_max": 15.265427589416504,
      "activations/layer0_attention_weight_min": -14.895042419433594,
      "activations/layer10_attention_weight_max": 40.48549270629883,
      "activations/layer10_attention_weight_min": -35.28689956665039,
      "activations/layer11_attention_weight_max": 32.147491455078125,
      "activations/layer11_attention_weight_min": -25.66719627380371,
      "activations/layer12_attention_weight_max": 24.90561866760254,
      "activations/layer12_attention_weight_min": -24.47108268737793,
      "activations/layer13_attention_weight_max": 53.87983322143555,
      "activations/layer13_attention_weight_min": -39.3293342590332,
      "activations/layer14_attention_weight_max": 43.761314392089844,
      "activations/layer14_attention_weight_min": -31.356245040893555,
      "activations/layer15_attention_weight_max": 42.1494255065918,
      "activations/layer15_attention_weight_min": -32.3160514831543,
      "activations/layer16_attention_weight_max": 35.0252799987793,
      "activations/layer16_attention_weight_min": -25.76495361328125,
      "activations/layer17_attention_weight_max": 56.02353286743164,
      "activations/layer17_attention_weight_min": -38.30531692504883,
      "activations/layer18_attention_weight_max": 46.50281524658203,
      "activations/layer18_attention_weight_min": -33.05548095703125,
      "activations/layer19_attention_weight_max": 18.989845275878906,
      "activations/layer19_attention_weight_min": -17.575138092041016,
      "activations/layer1_attention_weight_max": 18.289318084716797,
      "activations/layer1_attention_weight_min": -13.694978713989258,
      "activations/layer20_attention_weight_max": 18.010713577270508,
      "activations/layer20_attention_weight_min": -19.81791114807129,
      "activations/layer21_attention_weight_max": 39.154884338378906,
      "activations/layer21_attention_weight_min": -30.43285369873047,
      "activations/layer22_attention_weight_max": 31.921310424804688,
      "activations/layer22_attention_weight_min": -23.068851470947266,
      "activations/layer23_attention_weight_max": 22.183067321777344,
      "activations/layer23_attention_weight_min": -16.23770523071289,
      "activations/layer2_attention_weight_max": 30.228260040283203,
      "activations/layer2_attention_weight_min": -28.523643493652344,
      "activations/layer3_attention_weight_max": 76.9639892578125,
      "activations/layer3_attention_weight_min": -84.73442077636719,
      "activations/layer4_attention_weight_max": 100.13359832763672,
      "activations/layer4_attention_weight_min": -97.14834594726562,
      "activations/layer5_attention_weight_max": 108.69580078125,
      "activations/layer5_attention_weight_min": -103.66371154785156,
      "activations/layer6_attention_weight_max": 55.948936462402344,
      "activations/layer6_attention_weight_min": -52.93767166137695,
      "activations/layer7_attention_weight_max": 61.22772979736328,
      "activations/layer7_attention_weight_min": -56.90502166748047,
      "activations/layer8_attention_weight_max": 39.04819869995117,
      "activations/layer8_attention_weight_min": -36.87510681152344,
      "activations/layer9_attention_weight_max": 51.70168685913086,
      "activations/layer9_attention_weight_min": -45.63697814941406,
      "epoch": 3.8,
      "learning_rate": 0.00012676931818181818,
      "loss": 2.8666,
      "step": 65400
    },
    {
      "activations/layer0_attention_weight_max": 15.447636604309082,
      "activations/layer0_attention_weight_min": -15.306449890136719,
      "activations/layer10_attention_weight_max": 42.21686553955078,
      "activations/layer10_attention_weight_min": -37.529632568359375,
      "activations/layer11_attention_weight_max": 36.421749114990234,
      "activations/layer11_attention_weight_min": -30.921913146972656,
      "activations/layer12_attention_weight_max": 23.63701629638672,
      "activations/layer12_attention_weight_min": -25.351165771484375,
      "activations/layer13_attention_weight_max": 57.85858154296875,
      "activations/layer13_attention_weight_min": -49.87434768676758,
      "activations/layer14_attention_weight_max": 55.311710357666016,
      "activations/layer14_attention_weight_min": -44.86355209350586,
      "activations/layer15_attention_weight_max": 44.89448165893555,
      "activations/layer15_attention_weight_min": -37.30908203125,
      "activations/layer16_attention_weight_max": 32.75746154785156,
      "activations/layer16_attention_weight_min": -29.3577823638916,
      "activations/layer17_attention_weight_max": 58.46033477783203,
      "activations/layer17_attention_weight_min": -41.04248046875,
      "activations/layer18_attention_weight_max": 52.23746109008789,
      "activations/layer18_attention_weight_min": -36.70290756225586,
      "activations/layer19_attention_weight_max": 18.962080001831055,
      "activations/layer19_attention_weight_min": -17.30695152282715,
      "activations/layer1_attention_weight_max": 17.763050079345703,
      "activations/layer1_attention_weight_min": -13.13209342956543,
      "activations/layer20_attention_weight_max": 21.58745002746582,
      "activations/layer20_attention_weight_min": -18.965715408325195,
      "activations/layer21_attention_weight_max": 37.32760238647461,
      "activations/layer21_attention_weight_min": -25.759300231933594,
      "activations/layer22_attention_weight_max": 38.987709045410156,
      "activations/layer22_attention_weight_min": -26.70101547241211,
      "activations/layer23_attention_weight_max": 25.761505126953125,
      "activations/layer23_attention_weight_min": -16.80154037475586,
      "activations/layer2_attention_weight_max": 26.479772567749023,
      "activations/layer2_attention_weight_min": -27.50330352783203,
      "activations/layer3_attention_weight_max": 75.9270248413086,
      "activations/layer3_attention_weight_min": -80.82284545898438,
      "activations/layer4_attention_weight_max": 103.95870208740234,
      "activations/layer4_attention_weight_min": -98.98238372802734,
      "activations/layer5_attention_weight_max": 120.3236312866211,
      "activations/layer5_attention_weight_min": -110.49107360839844,
      "activations/layer6_attention_weight_max": 61.9229621887207,
      "activations/layer6_attention_weight_min": -58.49781036376953,
      "activations/layer7_attention_weight_max": 67.2360610961914,
      "activations/layer7_attention_weight_min": -66.97655487060547,
      "activations/layer8_attention_weight_max": 45.72576904296875,
      "activations/layer8_attention_weight_min": -40.53643035888672,
      "activations/layer9_attention_weight_max": 54.57416915893555,
      "activations/layer9_attention_weight_min": -52.08181381225586,
      "epoch": 3.8,
      "learning_rate": 0.00012675151515151515,
      "loss": 2.8829,
      "step": 65450
    },
    {
      "activations/layer0_attention_weight_max": 14.18130874633789,
      "activations/layer0_attention_weight_min": -14.43906021118164,
      "activations/layer10_attention_weight_max": 39.440879821777344,
      "activations/layer10_attention_weight_min": -34.82590866088867,
      "activations/layer11_attention_weight_max": 31.11695098876953,
      "activations/layer11_attention_weight_min": -28.442487716674805,
      "activations/layer12_attention_weight_max": 21.142667770385742,
      "activations/layer12_attention_weight_min": -25.023656845092773,
      "activations/layer13_attention_weight_max": 47.72663879394531,
      "activations/layer13_attention_weight_min": -35.37828063964844,
      "activations/layer14_attention_weight_max": 49.887184143066406,
      "activations/layer14_attention_weight_min": -31.992637634277344,
      "activations/layer15_attention_weight_max": 41.264102935791016,
      "activations/layer15_attention_weight_min": -32.552734375,
      "activations/layer16_attention_weight_max": 37.31802749633789,
      "activations/layer16_attention_weight_min": -26.78178596496582,
      "activations/layer17_attention_weight_max": 54.10993576049805,
      "activations/layer17_attention_weight_min": -34.95130157470703,
      "activations/layer18_attention_weight_max": 45.701377868652344,
      "activations/layer18_attention_weight_min": -31.55426597595215,
      "activations/layer19_attention_weight_max": 20.23172378540039,
      "activations/layer19_attention_weight_min": -22.84124755859375,
      "activations/layer1_attention_weight_max": 18.737958908081055,
      "activations/layer1_attention_weight_min": -17.784236907958984,
      "activations/layer20_attention_weight_max": 19.678770065307617,
      "activations/layer20_attention_weight_min": -19.896686553955078,
      "activations/layer21_attention_weight_max": 39.186710357666016,
      "activations/layer21_attention_weight_min": -27.454198837280273,
      "activations/layer22_attention_weight_max": 31.58340835571289,
      "activations/layer22_attention_weight_min": -23.12970542907715,
      "activations/layer23_attention_weight_max": 25.04581069946289,
      "activations/layer23_attention_weight_min": -18.09476089477539,
      "activations/layer2_attention_weight_max": 34.990577697753906,
      "activations/layer2_attention_weight_min": -31.155155181884766,
      "activations/layer3_attention_weight_max": 79.7081527709961,
      "activations/layer3_attention_weight_min": -75.08319854736328,
      "activations/layer4_attention_weight_max": 100.6353530883789,
      "activations/layer4_attention_weight_min": -90.51656341552734,
      "activations/layer5_attention_weight_max": 117.9485092163086,
      "activations/layer5_attention_weight_min": -101.4171142578125,
      "activations/layer6_attention_weight_max": 62.38972473144531,
      "activations/layer6_attention_weight_min": -53.66554641723633,
      "activations/layer7_attention_weight_max": 65.17308807373047,
      "activations/layer7_attention_weight_min": -58.590511322021484,
      "activations/layer8_attention_weight_max": 41.6965446472168,
      "activations/layer8_attention_weight_min": -37.35028076171875,
      "activations/layer9_attention_weight_max": 49.33884811401367,
      "activations/layer9_attention_weight_min": -44.68852615356445,
      "epoch": 3.81,
      "learning_rate": 0.00012673257575757574,
      "loss": 2.9264,
      "step": 65500
    },
    {
      "activations/layer0_attention_weight_max": 14.09447956085205,
      "activations/layer0_attention_weight_min": -14.7044677734375,
      "activations/layer10_attention_weight_max": 47.011680603027344,
      "activations/layer10_attention_weight_min": -33.641273498535156,
      "activations/layer11_attention_weight_max": 34.685020446777344,
      "activations/layer11_attention_weight_min": -25.93987274169922,
      "activations/layer12_attention_weight_max": 21.73479652404785,
      "activations/layer12_attention_weight_min": -24.2656307220459,
      "activations/layer13_attention_weight_max": 60.482791900634766,
      "activations/layer13_attention_weight_min": -39.71013259887695,
      "activations/layer14_attention_weight_max": 52.332984924316406,
      "activations/layer14_attention_weight_min": -32.65993118286133,
      "activations/layer15_attention_weight_max": 47.04785919189453,
      "activations/layer15_attention_weight_min": -32.222145080566406,
      "activations/layer16_attention_weight_max": 40.84720230102539,
      "activations/layer16_attention_weight_min": -26.368867874145508,
      "activations/layer17_attention_weight_max": 62.5711555480957,
      "activations/layer17_attention_weight_min": -37.36543273925781,
      "activations/layer18_attention_weight_max": 55.25929641723633,
      "activations/layer18_attention_weight_min": -32.711090087890625,
      "activations/layer19_attention_weight_max": 26.076536178588867,
      "activations/layer19_attention_weight_min": -18.377965927124023,
      "activations/layer1_attention_weight_max": 18.090837478637695,
      "activations/layer1_attention_weight_min": -15.767071723937988,
      "activations/layer20_attention_weight_max": 20.923343658447266,
      "activations/layer20_attention_weight_min": -19.9080753326416,
      "activations/layer21_attention_weight_max": 44.48502731323242,
      "activations/layer21_attention_weight_min": -28.972484588623047,
      "activations/layer22_attention_weight_max": 36.03691482543945,
      "activations/layer22_attention_weight_min": -22.83591651916504,
      "activations/layer23_attention_weight_max": 27.173009872436523,
      "activations/layer23_attention_weight_min": -20.08699607849121,
      "activations/layer2_attention_weight_max": 29.803699493408203,
      "activations/layer2_attention_weight_min": -26.169240951538086,
      "activations/layer3_attention_weight_max": 74.84852600097656,
      "activations/layer3_attention_weight_min": -76.038330078125,
      "activations/layer4_attention_weight_max": 99.71786499023438,
      "activations/layer4_attention_weight_min": -96.10462951660156,
      "activations/layer5_attention_weight_max": 121.13246154785156,
      "activations/layer5_attention_weight_min": -105.78590393066406,
      "activations/layer6_attention_weight_max": 63.47077178955078,
      "activations/layer6_attention_weight_min": -55.085487365722656,
      "activations/layer7_attention_weight_max": 72.21379089355469,
      "activations/layer7_attention_weight_min": -59.382164001464844,
      "activations/layer8_attention_weight_max": 44.77260208129883,
      "activations/layer8_attention_weight_min": -38.437660217285156,
      "activations/layer9_attention_weight_max": 53.229408264160156,
      "activations/layer9_attention_weight_min": -46.74026107788086,
      "epoch": 3.81,
      "learning_rate": 0.00012671363636363635,
      "loss": 2.9011,
      "step": 65550
    },
    {
      "activations/layer0_attention_weight_max": 13.908576965332031,
      "activations/layer0_attention_weight_min": -14.400800704956055,
      "activations/layer10_attention_weight_max": 41.46626281738281,
      "activations/layer10_attention_weight_min": -33.93769073486328,
      "activations/layer11_attention_weight_max": 31.331287384033203,
      "activations/layer11_attention_weight_min": -25.917041778564453,
      "activations/layer12_attention_weight_max": 54.659767150878906,
      "activations/layer12_attention_weight_min": -25.95050811767578,
      "activations/layer13_attention_weight_max": 50.722042083740234,
      "activations/layer13_attention_weight_min": -34.84364700317383,
      "activations/layer14_attention_weight_max": 40.15540313720703,
      "activations/layer14_attention_weight_min": -30.249950408935547,
      "activations/layer15_attention_weight_max": 52.036685943603516,
      "activations/layer15_attention_weight_min": -32.78837966918945,
      "activations/layer16_attention_weight_max": 41.0855598449707,
      "activations/layer16_attention_weight_min": -25.974618911743164,
      "activations/layer17_attention_weight_max": 59.32143783569336,
      "activations/layer17_attention_weight_min": -35.5662841796875,
      "activations/layer18_attention_weight_max": 52.25129318237305,
      "activations/layer18_attention_weight_min": -31.77189826965332,
      "activations/layer19_attention_weight_max": 21.09126853942871,
      "activations/layer19_attention_weight_min": -17.89789581298828,
      "activations/layer1_attention_weight_max": 18.494726181030273,
      "activations/layer1_attention_weight_min": -14.41690444946289,
      "activations/layer20_attention_weight_max": 21.73553466796875,
      "activations/layer20_attention_weight_min": -21.81480598449707,
      "activations/layer21_attention_weight_max": 38.63299560546875,
      "activations/layer21_attention_weight_min": -24.251184463500977,
      "activations/layer22_attention_weight_max": 35.614986419677734,
      "activations/layer22_attention_weight_min": -21.66981315612793,
      "activations/layer23_attention_weight_max": 23.825294494628906,
      "activations/layer23_attention_weight_min": -15.943648338317871,
      "activations/layer2_attention_weight_max": 30.486417770385742,
      "activations/layer2_attention_weight_min": -26.33917808532715,
      "activations/layer3_attention_weight_max": 70.01241302490234,
      "activations/layer3_attention_weight_min": -76.89058685302734,
      "activations/layer4_attention_weight_max": 94.965087890625,
      "activations/layer4_attention_weight_min": -87.98351287841797,
      "activations/layer5_attention_weight_max": 115.54853820800781,
      "activations/layer5_attention_weight_min": -98.2060546875,
      "activations/layer6_attention_weight_max": 57.89934539794922,
      "activations/layer6_attention_weight_min": -48.8432502746582,
      "activations/layer7_attention_weight_max": 63.013084411621094,
      "activations/layer7_attention_weight_min": -57.35934829711914,
      "activations/layer8_attention_weight_max": 43.31111145019531,
      "activations/layer8_attention_weight_min": -36.20650100708008,
      "activations/layer9_attention_weight_max": 51.17399597167969,
      "activations/layer9_attention_weight_min": -44.2417106628418,
      "epoch": 3.81,
      "learning_rate": 0.00012669469696969697,
      "loss": 2.8696,
      "step": 65600
    },
    {
      "activations/layer0_attention_weight_max": 14.32368278503418,
      "activations/layer0_attention_weight_min": -14.244665145874023,
      "activations/layer10_attention_weight_max": 38.75392532348633,
      "activations/layer10_attention_weight_min": -34.41692352294922,
      "activations/layer11_attention_weight_max": 30.088111877441406,
      "activations/layer11_attention_weight_min": -25.50599479675293,
      "activations/layer12_attention_weight_max": 25.45877456665039,
      "activations/layer12_attention_weight_min": -23.717182159423828,
      "activations/layer13_attention_weight_max": 51.33097839355469,
      "activations/layer13_attention_weight_min": -36.437103271484375,
      "activations/layer14_attention_weight_max": 43.36162567138672,
      "activations/layer14_attention_weight_min": -30.639257431030273,
      "activations/layer15_attention_weight_max": 42.10975646972656,
      "activations/layer15_attention_weight_min": -32.40729904174805,
      "activations/layer16_attention_weight_max": 36.01187515258789,
      "activations/layer16_attention_weight_min": -27.18022346496582,
      "activations/layer17_attention_weight_max": 58.02193069458008,
      "activations/layer17_attention_weight_min": -36.13032913208008,
      "activations/layer18_attention_weight_max": 51.19345474243164,
      "activations/layer18_attention_weight_min": -33.994651794433594,
      "activations/layer19_attention_weight_max": 21.50154685974121,
      "activations/layer19_attention_weight_min": -18.378618240356445,
      "activations/layer1_attention_weight_max": 17.3927001953125,
      "activations/layer1_attention_weight_min": -16.672964096069336,
      "activations/layer20_attention_weight_max": 19.576536178588867,
      "activations/layer20_attention_weight_min": -18.478628158569336,
      "activations/layer21_attention_weight_max": 39.154624938964844,
      "activations/layer21_attention_weight_min": -30.21709442138672,
      "activations/layer22_attention_weight_max": 35.42393493652344,
      "activations/layer22_attention_weight_min": -24.0242977142334,
      "activations/layer23_attention_weight_max": 24.78307342529297,
      "activations/layer23_attention_weight_min": -16.521930694580078,
      "activations/layer2_attention_weight_max": 28.789596557617188,
      "activations/layer2_attention_weight_min": -26.895660400390625,
      "activations/layer3_attention_weight_max": 70.87572479248047,
      "activations/layer3_attention_weight_min": -75.19073486328125,
      "activations/layer4_attention_weight_max": 93.57937622070312,
      "activations/layer4_attention_weight_min": -88.65838623046875,
      "activations/layer5_attention_weight_max": 106.93331146240234,
      "activations/layer5_attention_weight_min": -95.05245971679688,
      "activations/layer6_attention_weight_max": 55.960411071777344,
      "activations/layer6_attention_weight_min": -50.69942092895508,
      "activations/layer7_attention_weight_max": 61.17253494262695,
      "activations/layer7_attention_weight_min": -55.76248550415039,
      "activations/layer8_attention_weight_max": 37.63859176635742,
      "activations/layer8_attention_weight_min": -36.53541564941406,
      "activations/layer9_attention_weight_max": 44.605979919433594,
      "activations/layer9_attention_weight_min": -44.87104797363281,
      "epoch": 3.81,
      "learning_rate": 0.00012667575757575758,
      "loss": 2.8748,
      "step": 65650
    },
    {
      "activations/layer0_attention_weight_max": 15.98499584197998,
      "activations/layer0_attention_weight_min": -14.74279499053955,
      "activations/layer10_attention_weight_max": 48.66093826293945,
      "activations/layer10_attention_weight_min": -37.73614501953125,
      "activations/layer11_attention_weight_max": 35.33307647705078,
      "activations/layer11_attention_weight_min": -27.919769287109375,
      "activations/layer12_attention_weight_max": 28.41617774963379,
      "activations/layer12_attention_weight_min": -27.6973934173584,
      "activations/layer13_attention_weight_max": 54.60146713256836,
      "activations/layer13_attention_weight_min": -38.67837142944336,
      "activations/layer14_attention_weight_max": 52.58327102661133,
      "activations/layer14_attention_weight_min": -32.57957077026367,
      "activations/layer15_attention_weight_max": 42.33979415893555,
      "activations/layer15_attention_weight_min": -31.29034996032715,
      "activations/layer16_attention_weight_max": 35.67342758178711,
      "activations/layer16_attention_weight_min": -26.061220169067383,
      "activations/layer17_attention_weight_max": 51.504234313964844,
      "activations/layer17_attention_weight_min": -36.10606002807617,
      "activations/layer18_attention_weight_max": 47.49811553955078,
      "activations/layer18_attention_weight_min": -31.197031021118164,
      "activations/layer19_attention_weight_max": 17.953554153442383,
      "activations/layer19_attention_weight_min": -17.87172508239746,
      "activations/layer1_attention_weight_max": 17.799924850463867,
      "activations/layer1_attention_weight_min": -14.969894409179688,
      "activations/layer20_attention_weight_max": 17.613481521606445,
      "activations/layer20_attention_weight_min": -18.27238655090332,
      "activations/layer21_attention_weight_max": 40.174705505371094,
      "activations/layer21_attention_weight_min": -28.89792823791504,
      "activations/layer22_attention_weight_max": 33.6507568359375,
      "activations/layer22_attention_weight_min": -21.775239944458008,
      "activations/layer23_attention_weight_max": 23.76247787475586,
      "activations/layer23_attention_weight_min": -16.544580459594727,
      "activations/layer2_attention_weight_max": 30.951950073242188,
      "activations/layer2_attention_weight_min": -30.901412963867188,
      "activations/layer3_attention_weight_max": 75.46916961669922,
      "activations/layer3_attention_weight_min": -74.47441864013672,
      "activations/layer4_attention_weight_max": 102.4369888305664,
      "activations/layer4_attention_weight_min": -91.788330078125,
      "activations/layer5_attention_weight_max": 119.26527404785156,
      "activations/layer5_attention_weight_min": -100.89876556396484,
      "activations/layer6_attention_weight_max": 61.612640380859375,
      "activations/layer6_attention_weight_min": -52.484561920166016,
      "activations/layer7_attention_weight_max": 64.46820068359375,
      "activations/layer7_attention_weight_min": -59.71609878540039,
      "activations/layer8_attention_weight_max": 45.42607116699219,
      "activations/layer8_attention_weight_min": -39.658939361572266,
      "activations/layer9_attention_weight_max": 53.364501953125,
      "activations/layer9_attention_weight_min": -51.53878402709961,
      "epoch": 3.82,
      "learning_rate": 0.00012665681818181817,
      "loss": 2.8826,
      "step": 65700
    },
    {
      "activations/layer0_attention_weight_max": 13.753754615783691,
      "activations/layer0_attention_weight_min": -14.75728702545166,
      "activations/layer10_attention_weight_max": 38.085636138916016,
      "activations/layer10_attention_weight_min": -32.57875061035156,
      "activations/layer11_attention_weight_max": 31.230512619018555,
      "activations/layer11_attention_weight_min": -25.303958892822266,
      "activations/layer12_attention_weight_max": 25.657320022583008,
      "activations/layer12_attention_weight_min": -26.972225189208984,
      "activations/layer13_attention_weight_max": 50.29094696044922,
      "activations/layer13_attention_weight_min": -36.76298904418945,
      "activations/layer14_attention_weight_max": 46.65773391723633,
      "activations/layer14_attention_weight_min": -31.867036819458008,
      "activations/layer15_attention_weight_max": 43.360511779785156,
      "activations/layer15_attention_weight_min": -30.55291748046875,
      "activations/layer16_attention_weight_max": 37.71726989746094,
      "activations/layer16_attention_weight_min": -28.029443740844727,
      "activations/layer17_attention_weight_max": 61.1956672668457,
      "activations/layer17_attention_weight_min": -37.68339920043945,
      "activations/layer18_attention_weight_max": 48.6826286315918,
      "activations/layer18_attention_weight_min": -34.58671951293945,
      "activations/layer19_attention_weight_max": 24.51581573486328,
      "activations/layer19_attention_weight_min": -18.743932723999023,
      "activations/layer1_attention_weight_max": 16.604583740234375,
      "activations/layer1_attention_weight_min": -15.828640937805176,
      "activations/layer20_attention_weight_max": 20.87017822265625,
      "activations/layer20_attention_weight_min": -21.033870697021484,
      "activations/layer21_attention_weight_max": 38.41886520385742,
      "activations/layer21_attention_weight_min": -28.140348434448242,
      "activations/layer22_attention_weight_max": 34.88446807861328,
      "activations/layer22_attention_weight_min": -24.876131057739258,
      "activations/layer23_attention_weight_max": 29.536861419677734,
      "activations/layer23_attention_weight_min": -18.35501480102539,
      "activations/layer2_attention_weight_max": 29.612850189208984,
      "activations/layer2_attention_weight_min": -27.42729949951172,
      "activations/layer3_attention_weight_max": 72.98917388916016,
      "activations/layer3_attention_weight_min": -77.11382293701172,
      "activations/layer4_attention_weight_max": 97.54048919677734,
      "activations/layer4_attention_weight_min": -90.5848388671875,
      "activations/layer5_attention_weight_max": 108.77198791503906,
      "activations/layer5_attention_weight_min": -96.73361206054688,
      "activations/layer6_attention_weight_max": 55.56122970581055,
      "activations/layer6_attention_weight_min": -49.909461975097656,
      "activations/layer7_attention_weight_max": 61.89218521118164,
      "activations/layer7_attention_weight_min": -58.19184875488281,
      "activations/layer8_attention_weight_max": 39.4223747253418,
      "activations/layer8_attention_weight_min": -38.13113784790039,
      "activations/layer9_attention_weight_max": 52.1362190246582,
      "activations/layer9_attention_weight_min": -53.29508590698242,
      "epoch": 3.82,
      "learning_rate": 0.00012663787878787878,
      "loss": 2.879,
      "step": 65750
    },
    {
      "activations/layer0_attention_weight_max": 16.497722625732422,
      "activations/layer0_attention_weight_min": -14.909107208251953,
      "activations/layer10_attention_weight_max": 43.271907806396484,
      "activations/layer10_attention_weight_min": -33.21894836425781,
      "activations/layer11_attention_weight_max": 33.40187072753906,
      "activations/layer11_attention_weight_min": -27.56452178955078,
      "activations/layer12_attention_weight_max": 24.76104736328125,
      "activations/layer12_attention_weight_min": -24.591571807861328,
      "activations/layer13_attention_weight_max": 69.26656341552734,
      "activations/layer13_attention_weight_min": -44.59950256347656,
      "activations/layer14_attention_weight_max": 57.8405876159668,
      "activations/layer14_attention_weight_min": -36.95750045776367,
      "activations/layer15_attention_weight_max": 48.2895622253418,
      "activations/layer15_attention_weight_min": -32.476436614990234,
      "activations/layer16_attention_weight_max": 37.190433502197266,
      "activations/layer16_attention_weight_min": -26.254180908203125,
      "activations/layer17_attention_weight_max": 61.53810501098633,
      "activations/layer17_attention_weight_min": -38.36941146850586,
      "activations/layer18_attention_weight_max": 52.17661666870117,
      "activations/layer18_attention_weight_min": -37.00344467163086,
      "activations/layer19_attention_weight_max": 21.21554183959961,
      "activations/layer19_attention_weight_min": -19.595632553100586,
      "activations/layer1_attention_weight_max": 16.972984313964844,
      "activations/layer1_attention_weight_min": -13.630560874938965,
      "activations/layer20_attention_weight_max": 23.18375015258789,
      "activations/layer20_attention_weight_min": -19.127565383911133,
      "activations/layer21_attention_weight_max": 44.00471115112305,
      "activations/layer21_attention_weight_min": -31.028783798217773,
      "activations/layer22_attention_weight_max": 39.08992385864258,
      "activations/layer22_attention_weight_min": -24.9451961517334,
      "activations/layer23_attention_weight_max": 27.033018112182617,
      "activations/layer23_attention_weight_min": -17.431819915771484,
      "activations/layer2_attention_weight_max": 28.287979125976562,
      "activations/layer2_attention_weight_min": -29.20162010192871,
      "activations/layer3_attention_weight_max": 73.66030883789062,
      "activations/layer3_attention_weight_min": -76.84806060791016,
      "activations/layer4_attention_weight_max": 98.02392578125,
      "activations/layer4_attention_weight_min": -90.41165924072266,
      "activations/layer5_attention_weight_max": 112.28700256347656,
      "activations/layer5_attention_weight_min": -96.22393035888672,
      "activations/layer6_attention_weight_max": 59.82218933105469,
      "activations/layer6_attention_weight_min": -51.596519470214844,
      "activations/layer7_attention_weight_max": 62.56004333496094,
      "activations/layer7_attention_weight_min": -56.471466064453125,
      "activations/layer8_attention_weight_max": 44.74840545654297,
      "activations/layer8_attention_weight_min": -38.2275276184082,
      "activations/layer9_attention_weight_max": 57.915950775146484,
      "activations/layer9_attention_weight_min": -46.23634338378906,
      "epoch": 3.82,
      "learning_rate": 0.0001266189393939394,
      "loss": 2.8841,
      "step": 65800
    },
    {
      "activations/layer0_attention_weight_max": 13.813862800598145,
      "activations/layer0_attention_weight_min": -14.914934158325195,
      "activations/layer10_attention_weight_max": 40.79002380371094,
      "activations/layer10_attention_weight_min": -32.33418655395508,
      "activations/layer11_attention_weight_max": 32.56641387939453,
      "activations/layer11_attention_weight_min": -25.36697006225586,
      "activations/layer12_attention_weight_max": 34.16077423095703,
      "activations/layer12_attention_weight_min": -26.37078094482422,
      "activations/layer13_attention_weight_max": 55.18424987792969,
      "activations/layer13_attention_weight_min": -38.36090087890625,
      "activations/layer14_attention_weight_max": 44.424259185791016,
      "activations/layer14_attention_weight_min": -32.0859375,
      "activations/layer15_attention_weight_max": 50.78570556640625,
      "activations/layer15_attention_weight_min": -33.88645553588867,
      "activations/layer16_attention_weight_max": 35.005157470703125,
      "activations/layer16_attention_weight_min": -26.14126205444336,
      "activations/layer17_attention_weight_max": 60.63360595703125,
      "activations/layer17_attention_weight_min": -38.43727111816406,
      "activations/layer18_attention_weight_max": 52.59341812133789,
      "activations/layer18_attention_weight_min": -34.23084259033203,
      "activations/layer19_attention_weight_max": 24.429950714111328,
      "activations/layer19_attention_weight_min": -21.374845504760742,
      "activations/layer1_attention_weight_max": 16.813758850097656,
      "activations/layer1_attention_weight_min": -16.535842895507812,
      "activations/layer20_attention_weight_max": 23.994352340698242,
      "activations/layer20_attention_weight_min": -20.530548095703125,
      "activations/layer21_attention_weight_max": 42.07901382446289,
      "activations/layer21_attention_weight_min": -28.270811080932617,
      "activations/layer22_attention_weight_max": 34.72294235229492,
      "activations/layer22_attention_weight_min": -25.434114456176758,
      "activations/layer23_attention_weight_max": 30.01870346069336,
      "activations/layer23_attention_weight_min": -18.20490264892578,
      "activations/layer2_attention_weight_max": 30.458484649658203,
      "activations/layer2_attention_weight_min": -28.60955810546875,
      "activations/layer3_attention_weight_max": 73.9906234741211,
      "activations/layer3_attention_weight_min": -79.3986587524414,
      "activations/layer4_attention_weight_max": 99.3347396850586,
      "activations/layer4_attention_weight_min": -99.2132568359375,
      "activations/layer5_attention_weight_max": 112.27314758300781,
      "activations/layer5_attention_weight_min": -98.71987915039062,
      "activations/layer6_attention_weight_max": 58.910560607910156,
      "activations/layer6_attention_weight_min": -52.8591194152832,
      "activations/layer7_attention_weight_max": 61.2433967590332,
      "activations/layer7_attention_weight_min": -56.750762939453125,
      "activations/layer8_attention_weight_max": 40.25629425048828,
      "activations/layer8_attention_weight_min": -40.664764404296875,
      "activations/layer9_attention_weight_max": 47.46894836425781,
      "activations/layer9_attention_weight_min": -46.102909088134766,
      "epoch": 3.83,
      "learning_rate": 0.0001266,
      "loss": 2.8988,
      "step": 65850
    },
    {
      "activations/layer0_attention_weight_max": 14.414993286132812,
      "activations/layer0_attention_weight_min": -14.856672286987305,
      "activations/layer10_attention_weight_max": 42.05818557739258,
      "activations/layer10_attention_weight_min": -33.967247009277344,
      "activations/layer11_attention_weight_max": 33.80128479003906,
      "activations/layer11_attention_weight_min": -27.254650115966797,
      "activations/layer12_attention_weight_max": 31.683061599731445,
      "activations/layer12_attention_weight_min": -24.65190887451172,
      "activations/layer13_attention_weight_max": 52.16823959350586,
      "activations/layer13_attention_weight_min": -35.37470626831055,
      "activations/layer14_attention_weight_max": 46.52125930786133,
      "activations/layer14_attention_weight_min": -34.41866683959961,
      "activations/layer15_attention_weight_max": 45.35791778564453,
      "activations/layer15_attention_weight_min": -30.929292678833008,
      "activations/layer16_attention_weight_max": 35.90376281738281,
      "activations/layer16_attention_weight_min": -25.35558319091797,
      "activations/layer17_attention_weight_max": 59.66713333129883,
      "activations/layer17_attention_weight_min": -36.23394775390625,
      "activations/layer18_attention_weight_max": 49.77438735961914,
      "activations/layer18_attention_weight_min": -32.56059646606445,
      "activations/layer19_attention_weight_max": 22.39314079284668,
      "activations/layer19_attention_weight_min": -16.766754150390625,
      "activations/layer1_attention_weight_max": 16.896228790283203,
      "activations/layer1_attention_weight_min": -13.720915794372559,
      "activations/layer20_attention_weight_max": 23.7352352142334,
      "activations/layer20_attention_weight_min": -18.928863525390625,
      "activations/layer21_attention_weight_max": 38.37318420410156,
      "activations/layer21_attention_weight_min": -27.15131187438965,
      "activations/layer22_attention_weight_max": 39.1815299987793,
      "activations/layer22_attention_weight_min": -23.05864715576172,
      "activations/layer23_attention_weight_max": 27.51161766052246,
      "activations/layer23_attention_weight_min": -17.17953872680664,
      "activations/layer2_attention_weight_max": 29.177825927734375,
      "activations/layer2_attention_weight_min": -27.537982940673828,
      "activations/layer3_attention_weight_max": 72.26496124267578,
      "activations/layer3_attention_weight_min": -79.2305908203125,
      "activations/layer4_attention_weight_max": 94.23040771484375,
      "activations/layer4_attention_weight_min": -92.74351501464844,
      "activations/layer5_attention_weight_max": 108.13264465332031,
      "activations/layer5_attention_weight_min": -94.93252563476562,
      "activations/layer6_attention_weight_max": 56.62054443359375,
      "activations/layer6_attention_weight_min": -51.4730339050293,
      "activations/layer7_attention_weight_max": 60.941341400146484,
      "activations/layer7_attention_weight_min": -58.846126556396484,
      "activations/layer8_attention_weight_max": 39.6028938293457,
      "activations/layer8_attention_weight_min": -37.8936882019043,
      "activations/layer9_attention_weight_max": 51.5218620300293,
      "activations/layer9_attention_weight_min": -45.485530853271484,
      "epoch": 3.83,
      "learning_rate": 0.0001265810606060606,
      "loss": 2.8705,
      "step": 65900
    },
    {
      "activations/layer0_attention_weight_max": 14.181395530700684,
      "activations/layer0_attention_weight_min": -14.752477645874023,
      "activations/layer10_attention_weight_max": 39.45532989501953,
      "activations/layer10_attention_weight_min": -34.05550003051758,
      "activations/layer11_attention_weight_max": 35.92414093017578,
      "activations/layer11_attention_weight_min": -24.991580963134766,
      "activations/layer12_attention_weight_max": 27.199155807495117,
      "activations/layer12_attention_weight_min": -26.19622230529785,
      "activations/layer13_attention_weight_max": 60.75711441040039,
      "activations/layer13_attention_weight_min": -38.59974670410156,
      "activations/layer14_attention_weight_max": 54.004737854003906,
      "activations/layer14_attention_weight_min": -33.57889938354492,
      "activations/layer15_attention_weight_max": 46.77863693237305,
      "activations/layer15_attention_weight_min": -31.158920288085938,
      "activations/layer16_attention_weight_max": 42.41623306274414,
      "activations/layer16_attention_weight_min": -26.367856979370117,
      "activations/layer17_attention_weight_max": 66.5495376586914,
      "activations/layer17_attention_weight_min": -37.366844177246094,
      "activations/layer18_attention_weight_max": 59.5290641784668,
      "activations/layer18_attention_weight_min": -32.50510787963867,
      "activations/layer19_attention_weight_max": 24.79078483581543,
      "activations/layer19_attention_weight_min": -21.218505859375,
      "activations/layer1_attention_weight_max": 19.03810691833496,
      "activations/layer1_attention_weight_min": -15.069718360900879,
      "activations/layer20_attention_weight_max": 29.401905059814453,
      "activations/layer20_attention_weight_min": -20.08395767211914,
      "activations/layer21_attention_weight_max": 53.193626403808594,
      "activations/layer21_attention_weight_min": -25.707609176635742,
      "activations/layer22_attention_weight_max": 40.14334487915039,
      "activations/layer22_attention_weight_min": -25.46012306213379,
      "activations/layer23_attention_weight_max": 28.832382202148438,
      "activations/layer23_attention_weight_min": -17.734163284301758,
      "activations/layer2_attention_weight_max": 32.489322662353516,
      "activations/layer2_attention_weight_min": -28.201772689819336,
      "activations/layer3_attention_weight_max": 81.69747161865234,
      "activations/layer3_attention_weight_min": -82.14073181152344,
      "activations/layer4_attention_weight_max": 105.30474090576172,
      "activations/layer4_attention_weight_min": -93.99663543701172,
      "activations/layer5_attention_weight_max": 118.19941711425781,
      "activations/layer5_attention_weight_min": -97.13048553466797,
      "activations/layer6_attention_weight_max": 59.4260368347168,
      "activations/layer6_attention_weight_min": -54.42948913574219,
      "activations/layer7_attention_weight_max": 62.832916259765625,
      "activations/layer7_attention_weight_min": -60.483001708984375,
      "activations/layer8_attention_weight_max": 42.679962158203125,
      "activations/layer8_attention_weight_min": -39.10627746582031,
      "activations/layer9_attention_weight_max": 57.92041015625,
      "activations/layer9_attention_weight_min": -48.71345138549805,
      "epoch": 3.83,
      "learning_rate": 0.00012656212121212122,
      "loss": 2.8849,
      "step": 65950
    },
    {
      "activations/layer0_attention_weight_max": 13.8297700881958,
      "activations/layer0_attention_weight_min": -14.648294448852539,
      "activations/layer10_attention_weight_max": 38.700504302978516,
      "activations/layer10_attention_weight_min": -32.39638137817383,
      "activations/layer11_attention_weight_max": 30.953054428100586,
      "activations/layer11_attention_weight_min": -25.303340911865234,
      "activations/layer12_attention_weight_max": 21.352819442749023,
      "activations/layer12_attention_weight_min": -23.319095611572266,
      "activations/layer13_attention_weight_max": 53.53446960449219,
      "activations/layer13_attention_weight_min": -42.584800720214844,
      "activations/layer14_attention_weight_max": 56.136104583740234,
      "activations/layer14_attention_weight_min": -38.87065887451172,
      "activations/layer15_attention_weight_max": 42.947715759277344,
      "activations/layer15_attention_weight_min": -32.59675979614258,
      "activations/layer16_attention_weight_max": 37.878360748291016,
      "activations/layer16_attention_weight_min": -25.319698333740234,
      "activations/layer17_attention_weight_max": 54.88896560668945,
      "activations/layer17_attention_weight_min": -35.641910552978516,
      "activations/layer18_attention_weight_max": 48.26033020019531,
      "activations/layer18_attention_weight_min": -31.05239486694336,
      "activations/layer19_attention_weight_max": 22.44612693786621,
      "activations/layer19_attention_weight_min": -18.039304733276367,
      "activations/layer1_attention_weight_max": 17.027177810668945,
      "activations/layer1_attention_weight_min": -13.803255081176758,
      "activations/layer20_attention_weight_max": 21.35765838623047,
      "activations/layer20_attention_weight_min": -21.230361938476562,
      "activations/layer21_attention_weight_max": 36.38504409790039,
      "activations/layer21_attention_weight_min": -27.320892333984375,
      "activations/layer22_attention_weight_max": 39.01234817504883,
      "activations/layer22_attention_weight_min": -22.37969398498535,
      "activations/layer23_attention_weight_max": 30.39754867553711,
      "activations/layer23_attention_weight_min": -16.739410400390625,
      "activations/layer2_attention_weight_max": 28.716442108154297,
      "activations/layer2_attention_weight_min": -26.907867431640625,
      "activations/layer3_attention_weight_max": 68.5805892944336,
      "activations/layer3_attention_weight_min": -75.63971710205078,
      "activations/layer4_attention_weight_max": 93.12974548339844,
      "activations/layer4_attention_weight_min": -87.94646453857422,
      "activations/layer5_attention_weight_max": 109.98863983154297,
      "activations/layer5_attention_weight_min": -98.19701385498047,
      "activations/layer6_attention_weight_max": 58.11273193359375,
      "activations/layer6_attention_weight_min": -50.214115142822266,
      "activations/layer7_attention_weight_max": 57.42945861816406,
      "activations/layer7_attention_weight_min": -56.2745361328125,
      "activations/layer8_attention_weight_max": 39.988338470458984,
      "activations/layer8_attention_weight_min": -36.7453498840332,
      "activations/layer9_attention_weight_max": 47.4991340637207,
      "activations/layer9_attention_weight_min": -45.180267333984375,
      "epoch": 3.83,
      "learning_rate": 0.0001265431818181818,
      "loss": 2.8627,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_loss": 2.83203125,
      "eval_runtime": 8.4268,
      "eval_samples_per_second": 509.566,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_openwebtext_loss": 2.83203125,
      "eval_openwebtext_ppl": 16.979916273809664,
      "eval_openwebtext_runtime": 8.4268,
      "eval_openwebtext_samples_per_second": 509.566,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_wikitext_loss": 3.064453125,
      "eval_wikitext_ppl": 21.42274322042474,
      "eval_wikitext_runtime": 1.9426,
      "eval_wikitext_samples_per_second": 234.738,
      "step": 66000
    },
    {
      "epoch": 3.83,
      "eval_lambada_loss": 2.95703125,
      "eval_lambada_ppl": 19.24076585887165,
      "eval_lambada_runtime": 9.5205,
      "eval_lambada_samples_per_second": 511.422,
      "step": 66000
    },
    {
      "activations/layer0_attention_weight_max": 14.430381774902344,
      "activations/layer0_attention_weight_min": -15.025156021118164,
      "activations/layer10_attention_weight_max": 40.60840606689453,
      "activations/layer10_attention_weight_min": -33.50794219970703,
      "activations/layer11_attention_weight_max": 31.155637741088867,
      "activations/layer11_attention_weight_min": -25.71077537536621,
      "activations/layer12_attention_weight_max": 24.118066787719727,
      "activations/layer12_attention_weight_min": -24.771671295166016,
      "activations/layer13_attention_weight_max": 50.015724182128906,
      "activations/layer13_attention_weight_min": -34.69257354736328,
      "activations/layer14_attention_weight_max": 49.26880645751953,
      "activations/layer14_attention_weight_min": -34.12859344482422,
      "activations/layer15_attention_weight_max": 45.623687744140625,
      "activations/layer15_attention_weight_min": -30.843708038330078,
      "activations/layer16_attention_weight_max": 38.400733947753906,
      "activations/layer16_attention_weight_min": -26.248157501220703,
      "activations/layer17_attention_weight_max": 59.505126953125,
      "activations/layer17_attention_weight_min": -37.337886810302734,
      "activations/layer18_attention_weight_max": 54.085758209228516,
      "activations/layer18_attention_weight_min": -32.91276550292969,
      "activations/layer19_attention_weight_max": 22.556991577148438,
      "activations/layer19_attention_weight_min": -21.72572135925293,
      "activations/layer1_attention_weight_max": 17.000154495239258,
      "activations/layer1_attention_weight_min": -16.239212036132812,
      "activations/layer20_attention_weight_max": 24.452503204345703,
      "activations/layer20_attention_weight_min": -20.484987258911133,
      "activations/layer21_attention_weight_max": 42.73237991333008,
      "activations/layer21_attention_weight_min": -29.795940399169922,
      "activations/layer22_attention_weight_max": 39.629661560058594,
      "activations/layer22_attention_weight_min": -24.175657272338867,
      "activations/layer23_attention_weight_max": 30.425445556640625,
      "activations/layer23_attention_weight_min": -17.79027557373047,
      "activations/layer2_attention_weight_max": 30.56399917602539,
      "activations/layer2_attention_weight_min": -28.04005241394043,
      "activations/layer3_attention_weight_max": 73.29691314697266,
      "activations/layer3_attention_weight_min": -80.4433364868164,
      "activations/layer4_attention_weight_max": 95.69580078125,
      "activations/layer4_attention_weight_min": -90.95328521728516,
      "activations/layer5_attention_weight_max": 112.75569915771484,
      "activations/layer5_attention_weight_min": -97.22581481933594,
      "activations/layer6_attention_weight_max": 61.96867752075195,
      "activations/layer6_attention_weight_min": -52.54841232299805,
      "activations/layer7_attention_weight_max": 63.342491149902344,
      "activations/layer7_attention_weight_min": -60.306358337402344,
      "activations/layer8_attention_weight_max": 39.98854064941406,
      "activations/layer8_attention_weight_min": -37.994110107421875,
      "activations/layer9_attention_weight_max": 45.75979232788086,
      "activations/layer9_attention_weight_min": -43.88908004760742,
      "epoch": 3.84,
      "learning_rate": 0.00012652424242424242,
      "loss": 2.8901,
      "step": 66050
    },
    {
      "activations/layer0_attention_weight_max": 14.660076141357422,
      "activations/layer0_attention_weight_min": -15.164151191711426,
      "activations/layer10_attention_weight_max": 39.77037048339844,
      "activations/layer10_attention_weight_min": -34.18212890625,
      "activations/layer11_attention_weight_max": 29.761398315429688,
      "activations/layer11_attention_weight_min": -26.02557945251465,
      "activations/layer12_attention_weight_max": 30.027189254760742,
      "activations/layer12_attention_weight_min": -24.492212295532227,
      "activations/layer13_attention_weight_max": 51.9642448425293,
      "activations/layer13_attention_weight_min": -39.617584228515625,
      "activations/layer14_attention_weight_max": 43.260887145996094,
      "activations/layer14_attention_weight_min": -30.068117141723633,
      "activations/layer15_attention_weight_max": 45.4237174987793,
      "activations/layer15_attention_weight_min": -32.438331604003906,
      "activations/layer16_attention_weight_max": 37.64285659790039,
      "activations/layer16_attention_weight_min": -26.80971336364746,
      "activations/layer17_attention_weight_max": 63.56652069091797,
      "activations/layer17_attention_weight_min": -39.73311233520508,
      "activations/layer18_attention_weight_max": 51.786949157714844,
      "activations/layer18_attention_weight_min": -34.16201400756836,
      "activations/layer19_attention_weight_max": 28.579055786132812,
      "activations/layer19_attention_weight_min": -18.57285499572754,
      "activations/layer1_attention_weight_max": 17.57924461364746,
      "activations/layer1_attention_weight_min": -13.346190452575684,
      "activations/layer20_attention_weight_max": 22.237083435058594,
      "activations/layer20_attention_weight_min": -21.885087966918945,
      "activations/layer21_attention_weight_max": 41.095211029052734,
      "activations/layer21_attention_weight_min": -25.908855438232422,
      "activations/layer22_attention_weight_max": 37.17139434814453,
      "activations/layer22_attention_weight_min": -26.07649040222168,
      "activations/layer23_attention_weight_max": 26.687408447265625,
      "activations/layer23_attention_weight_min": -16.767730712890625,
      "activations/layer2_attention_weight_max": 29.84168815612793,
      "activations/layer2_attention_weight_min": -29.911048889160156,
      "activations/layer3_attention_weight_max": 77.77977752685547,
      "activations/layer3_attention_weight_min": -82.42571258544922,
      "activations/layer4_attention_weight_max": 104.0751953125,
      "activations/layer4_attention_weight_min": -95.4354476928711,
      "activations/layer5_attention_weight_max": 116.19204711914062,
      "activations/layer5_attention_weight_min": -103.10955810546875,
      "activations/layer6_attention_weight_max": 64.22001647949219,
      "activations/layer6_attention_weight_min": -55.108673095703125,
      "activations/layer7_attention_weight_max": 61.13618469238281,
      "activations/layer7_attention_weight_min": -58.9289436340332,
      "activations/layer8_attention_weight_max": 40.82037353515625,
      "activations/layer8_attention_weight_min": -38.86145782470703,
      "activations/layer9_attention_weight_max": 51.454750061035156,
      "activations/layer9_attention_weight_min": -50.91218185424805,
      "epoch": 3.84,
      "learning_rate": 0.000126505303030303,
      "loss": 2.8669,
      "step": 66100
    },
    {
      "activations/layer0_attention_weight_max": 13.875114440917969,
      "activations/layer0_attention_weight_min": -15.089056015014648,
      "activations/layer10_attention_weight_max": 40.72150802612305,
      "activations/layer10_attention_weight_min": -33.84265899658203,
      "activations/layer11_attention_weight_max": 31.851119995117188,
      "activations/layer11_attention_weight_min": -27.71137046813965,
      "activations/layer12_attention_weight_max": 25.34827423095703,
      "activations/layer12_attention_weight_min": -26.117433547973633,
      "activations/layer13_attention_weight_max": 48.48054122924805,
      "activations/layer13_attention_weight_min": -35.42728042602539,
      "activations/layer14_attention_weight_max": 42.98828125,
      "activations/layer14_attention_weight_min": -29.888322830200195,
      "activations/layer15_attention_weight_max": 40.0474853515625,
      "activations/layer15_attention_weight_min": -30.616220474243164,
      "activations/layer16_attention_weight_max": 33.72964096069336,
      "activations/layer16_attention_weight_min": -27.19642448425293,
      "activations/layer17_attention_weight_max": 55.60892105102539,
      "activations/layer17_attention_weight_min": -37.47720718383789,
      "activations/layer18_attention_weight_max": 48.426551818847656,
      "activations/layer18_attention_weight_min": -32.842002868652344,
      "activations/layer19_attention_weight_max": 20.91815185546875,
      "activations/layer19_attention_weight_min": -19.108806610107422,
      "activations/layer1_attention_weight_max": 16.9130916595459,
      "activations/layer1_attention_weight_min": -16.33669662475586,
      "activations/layer20_attention_weight_max": 20.418901443481445,
      "activations/layer20_attention_weight_min": -18.120073318481445,
      "activations/layer21_attention_weight_max": 37.7973518371582,
      "activations/layer21_attention_weight_min": -27.996801376342773,
      "activations/layer22_attention_weight_max": 37.023780822753906,
      "activations/layer22_attention_weight_min": -25.615461349487305,
      "activations/layer23_attention_weight_max": 26.519195556640625,
      "activations/layer23_attention_weight_min": -17.744836807250977,
      "activations/layer2_attention_weight_max": 30.237712860107422,
      "activations/layer2_attention_weight_min": -30.30030059814453,
      "activations/layer3_attention_weight_max": 80.10140228271484,
      "activations/layer3_attention_weight_min": -82.61238861083984,
      "activations/layer4_attention_weight_max": 103.9493637084961,
      "activations/layer4_attention_weight_min": -94.11174774169922,
      "activations/layer5_attention_weight_max": 117.35385131835938,
      "activations/layer5_attention_weight_min": -103.54222869873047,
      "activations/layer6_attention_weight_max": 63.38738250732422,
      "activations/layer6_attention_weight_min": -56.63050842285156,
      "activations/layer7_attention_weight_max": 60.48016357421875,
      "activations/layer7_attention_weight_min": -57.894893646240234,
      "activations/layer8_attention_weight_max": 43.52145004272461,
      "activations/layer8_attention_weight_min": -38.60577392578125,
      "activations/layer9_attention_weight_max": 50.535423278808594,
      "activations/layer9_attention_weight_min": -45.1176872253418,
      "epoch": 3.84,
      "learning_rate": 0.00012648636363636362,
      "loss": 2.885,
      "step": 66150
    },
    {
      "activations/layer0_attention_weight_max": 14.386407852172852,
      "activations/layer0_attention_weight_min": -15.166783332824707,
      "activations/layer10_attention_weight_max": 42.01340866088867,
      "activations/layer10_attention_weight_min": -33.927215576171875,
      "activations/layer11_attention_weight_max": 32.957557678222656,
      "activations/layer11_attention_weight_min": -26.448421478271484,
      "activations/layer12_attention_weight_max": 25.459590911865234,
      "activations/layer12_attention_weight_min": -26.16779327392578,
      "activations/layer13_attention_weight_max": 52.69319534301758,
      "activations/layer13_attention_weight_min": -39.2133674621582,
      "activations/layer14_attention_weight_max": 46.22663116455078,
      "activations/layer14_attention_weight_min": -32.54088592529297,
      "activations/layer15_attention_weight_max": 41.92436599731445,
      "activations/layer15_attention_weight_min": -30.388708114624023,
      "activations/layer16_attention_weight_max": 36.29248046875,
      "activations/layer16_attention_weight_min": -24.751026153564453,
      "activations/layer17_attention_weight_max": 62.38162612915039,
      "activations/layer17_attention_weight_min": -36.435302734375,
      "activations/layer18_attention_weight_max": 54.205875396728516,
      "activations/layer18_attention_weight_min": -32.655799865722656,
      "activations/layer19_attention_weight_max": 24.216100692749023,
      "activations/layer19_attention_weight_min": -18.155118942260742,
      "activations/layer1_attention_weight_max": 18.608501434326172,
      "activations/layer1_attention_weight_min": -15.737833023071289,
      "activations/layer20_attention_weight_max": 21.674894332885742,
      "activations/layer20_attention_weight_min": -19.319509506225586,
      "activations/layer21_attention_weight_max": 42.0765266418457,
      "activations/layer21_attention_weight_min": -26.674497604370117,
      "activations/layer22_attention_weight_max": 39.19131851196289,
      "activations/layer22_attention_weight_min": -23.47348976135254,
      "activations/layer23_attention_weight_max": 23.28799819946289,
      "activations/layer23_attention_weight_min": -15.730998039245605,
      "activations/layer2_attention_weight_max": 29.177152633666992,
      "activations/layer2_attention_weight_min": -26.20577621459961,
      "activations/layer3_attention_weight_max": 71.56208038330078,
      "activations/layer3_attention_weight_min": -77.24102783203125,
      "activations/layer4_attention_weight_max": 96.69673919677734,
      "activations/layer4_attention_weight_min": -93.65514373779297,
      "activations/layer5_attention_weight_max": 113.7411117553711,
      "activations/layer5_attention_weight_min": -99.09209442138672,
      "activations/layer6_attention_weight_max": 61.87489700317383,
      "activations/layer6_attention_weight_min": -53.58389663696289,
      "activations/layer7_attention_weight_max": 63.6253547668457,
      "activations/layer7_attention_weight_min": -56.799720764160156,
      "activations/layer8_attention_weight_max": 40.11049270629883,
      "activations/layer8_attention_weight_min": -37.22978973388672,
      "activations/layer9_attention_weight_max": 50.774715423583984,
      "activations/layer9_attention_weight_min": -45.22768020629883,
      "epoch": 3.85,
      "learning_rate": 0.00012646742424242424,
      "loss": 2.8751,
      "step": 66200
    },
    {
      "activations/layer0_attention_weight_max": 13.685397148132324,
      "activations/layer0_attention_weight_min": -14.815688133239746,
      "activations/layer10_attention_weight_max": 38.55470657348633,
      "activations/layer10_attention_weight_min": -34.332481384277344,
      "activations/layer11_attention_weight_max": 33.28950500488281,
      "activations/layer11_attention_weight_min": -27.20077133178711,
      "activations/layer12_attention_weight_max": 23.844894409179688,
      "activations/layer12_attention_weight_min": -38.291778564453125,
      "activations/layer13_attention_weight_max": 66.78176879882812,
      "activations/layer13_attention_weight_min": -42.07081604003906,
      "activations/layer14_attention_weight_max": 59.04706954956055,
      "activations/layer14_attention_weight_min": -35.33903884887695,
      "activations/layer15_attention_weight_max": 52.065608978271484,
      "activations/layer15_attention_weight_min": -33.081085205078125,
      "activations/layer16_attention_weight_max": 37.69065856933594,
      "activations/layer16_attention_weight_min": -24.77003288269043,
      "activations/layer17_attention_weight_max": 56.26226043701172,
      "activations/layer17_attention_weight_min": -34.96879959106445,
      "activations/layer18_attention_weight_max": 52.00663757324219,
      "activations/layer18_attention_weight_min": -32.884002685546875,
      "activations/layer19_attention_weight_max": 20.942411422729492,
      "activations/layer19_attention_weight_min": -16.19951820373535,
      "activations/layer1_attention_weight_max": 17.417020797729492,
      "activations/layer1_attention_weight_min": -14.356729507446289,
      "activations/layer20_attention_weight_max": 21.009075164794922,
      "activations/layer20_attention_weight_min": -17.017684936523438,
      "activations/layer21_attention_weight_max": 43.966609954833984,
      "activations/layer21_attention_weight_min": -27.475095748901367,
      "activations/layer22_attention_weight_max": 36.09503173828125,
      "activations/layer22_attention_weight_min": -21.838552474975586,
      "activations/layer23_attention_weight_max": 25.262657165527344,
      "activations/layer23_attention_weight_min": -16.885854721069336,
      "activations/layer2_attention_weight_max": 29.70686912536621,
      "activations/layer2_attention_weight_min": -29.821622848510742,
      "activations/layer3_attention_weight_max": 75.81214141845703,
      "activations/layer3_attention_weight_min": -79.60867309570312,
      "activations/layer4_attention_weight_max": 101.35081481933594,
      "activations/layer4_attention_weight_min": -90.20514678955078,
      "activations/layer5_attention_weight_max": 113.94078826904297,
      "activations/layer5_attention_weight_min": -97.5151596069336,
      "activations/layer6_attention_weight_max": 56.96897888183594,
      "activations/layer6_attention_weight_min": -49.338706970214844,
      "activations/layer7_attention_weight_max": 68.10037231445312,
      "activations/layer7_attention_weight_min": -63.67163848876953,
      "activations/layer8_attention_weight_max": 46.476593017578125,
      "activations/layer8_attention_weight_min": -39.76765441894531,
      "activations/layer9_attention_weight_max": 58.19146728515625,
      "activations/layer9_attention_weight_min": -58.7087516784668,
      "epoch": 3.85,
      "learning_rate": 0.00012644848484848483,
      "loss": 2.8845,
      "step": 66250
    },
    {
      "activations/layer0_attention_weight_max": 14.807689666748047,
      "activations/layer0_attention_weight_min": -15.085161209106445,
      "activations/layer10_attention_weight_max": 40.4317512512207,
      "activations/layer10_attention_weight_min": -35.0943603515625,
      "activations/layer11_attention_weight_max": 36.08270263671875,
      "activations/layer11_attention_weight_min": -30.166187286376953,
      "activations/layer12_attention_weight_max": 21.68767547607422,
      "activations/layer12_attention_weight_min": -22.1822452545166,
      "activations/layer13_attention_weight_max": 50.3858757019043,
      "activations/layer13_attention_weight_min": -34.741634368896484,
      "activations/layer14_attention_weight_max": 49.31288528442383,
      "activations/layer14_attention_weight_min": -35.46875,
      "activations/layer15_attention_weight_max": 42.9474983215332,
      "activations/layer15_attention_weight_min": -29.12180519104004,
      "activations/layer16_attention_weight_max": 33.463470458984375,
      "activations/layer16_attention_weight_min": -26.075639724731445,
      "activations/layer17_attention_weight_max": 67.5085678100586,
      "activations/layer17_attention_weight_min": -36.44723129272461,
      "activations/layer18_attention_weight_max": 48.90913009643555,
      "activations/layer18_attention_weight_min": -32.745635986328125,
      "activations/layer19_attention_weight_max": 23.096418380737305,
      "activations/layer19_attention_weight_min": -17.119993209838867,
      "activations/layer1_attention_weight_max": 18.666475296020508,
      "activations/layer1_attention_weight_min": -16.318967819213867,
      "activations/layer20_attention_weight_max": 20.60120964050293,
      "activations/layer20_attention_weight_min": -21.142210006713867,
      "activations/layer21_attention_weight_max": 38.44243240356445,
      "activations/layer21_attention_weight_min": -24.94728660583496,
      "activations/layer22_attention_weight_max": 34.277652740478516,
      "activations/layer22_attention_weight_min": -21.84988784790039,
      "activations/layer23_attention_weight_max": 24.278091430664062,
      "activations/layer23_attention_weight_min": -16.94631576538086,
      "activations/layer2_attention_weight_max": 29.929767608642578,
      "activations/layer2_attention_weight_min": -29.07735824584961,
      "activations/layer3_attention_weight_max": 76.5429458618164,
      "activations/layer3_attention_weight_min": -81.18213653564453,
      "activations/layer4_attention_weight_max": 97.82691955566406,
      "activations/layer4_attention_weight_min": -95.73189544677734,
      "activations/layer5_attention_weight_max": 113.11235046386719,
      "activations/layer5_attention_weight_min": -107.28976440429688,
      "activations/layer6_attention_weight_max": 60.71770477294922,
      "activations/layer6_attention_weight_min": -54.74888610839844,
      "activations/layer7_attention_weight_max": 68.67623901367188,
      "activations/layer7_attention_weight_min": -63.32764434814453,
      "activations/layer8_attention_weight_max": 42.63174819946289,
      "activations/layer8_attention_weight_min": -41.61629104614258,
      "activations/layer9_attention_weight_max": 59.839168548583984,
      "activations/layer9_attention_weight_min": -48.355926513671875,
      "epoch": 3.85,
      "learning_rate": 0.00012642954545454544,
      "loss": 2.8569,
      "step": 66300
    },
    {
      "activations/layer0_attention_weight_max": 14.582058906555176,
      "activations/layer0_attention_weight_min": -15.428707122802734,
      "activations/layer10_attention_weight_max": 41.38901901245117,
      "activations/layer10_attention_weight_min": -33.625099182128906,
      "activations/layer11_attention_weight_max": 30.829879760742188,
      "activations/layer11_attention_weight_min": -26.841590881347656,
      "activations/layer12_attention_weight_max": 31.238014221191406,
      "activations/layer12_attention_weight_min": -26.0258846282959,
      "activations/layer13_attention_weight_max": 45.910804748535156,
      "activations/layer13_attention_weight_min": -33.49856185913086,
      "activations/layer14_attention_weight_max": 44.28060531616211,
      "activations/layer14_attention_weight_min": -30.894554138183594,
      "activations/layer15_attention_weight_max": 45.414493560791016,
      "activations/layer15_attention_weight_min": -31.788684844970703,
      "activations/layer16_attention_weight_max": 36.85895538330078,
      "activations/layer16_attention_weight_min": -26.821908950805664,
      "activations/layer17_attention_weight_max": 59.09977340698242,
      "activations/layer17_attention_weight_min": -36.895591735839844,
      "activations/layer18_attention_weight_max": 53.37413787841797,
      "activations/layer18_attention_weight_min": -32.19351577758789,
      "activations/layer19_attention_weight_max": 24.500154495239258,
      "activations/layer19_attention_weight_min": -19.39316177368164,
      "activations/layer1_attention_weight_max": 18.544189453125,
      "activations/layer1_attention_weight_min": -16.36048698425293,
      "activations/layer20_attention_weight_max": 21.560335159301758,
      "activations/layer20_attention_weight_min": -24.01020622253418,
      "activations/layer21_attention_weight_max": 43.1804313659668,
      "activations/layer21_attention_weight_min": -29.32760238647461,
      "activations/layer22_attention_weight_max": 38.90776824951172,
      "activations/layer22_attention_weight_min": -25.61486053466797,
      "activations/layer23_attention_weight_max": 27.385257720947266,
      "activations/layer23_attention_weight_min": -17.0998592376709,
      "activations/layer2_attention_weight_max": 29.941741943359375,
      "activations/layer2_attention_weight_min": -28.85463523864746,
      "activations/layer3_attention_weight_max": 76.14580535888672,
      "activations/layer3_attention_weight_min": -85.17626190185547,
      "activations/layer4_attention_weight_max": 101.60192108154297,
      "activations/layer4_attention_weight_min": -98.91030883789062,
      "activations/layer5_attention_weight_max": 115.49237060546875,
      "activations/layer5_attention_weight_min": -102.6766586303711,
      "activations/layer6_attention_weight_max": 60.1305046081543,
      "activations/layer6_attention_weight_min": -54.811485290527344,
      "activations/layer7_attention_weight_max": 66.62437438964844,
      "activations/layer7_attention_weight_min": -57.67668533325195,
      "activations/layer8_attention_weight_max": 40.05392837524414,
      "activations/layer8_attention_weight_min": -38.02960968017578,
      "activations/layer9_attention_weight_max": 50.88604736328125,
      "activations/layer9_attention_weight_min": -45.527793884277344,
      "epoch": 3.86,
      "learning_rate": 0.00012641060606060606,
      "loss": 2.8929,
      "step": 66350
    },
    {
      "activations/layer0_attention_weight_max": 14.614651679992676,
      "activations/layer0_attention_weight_min": -15.233745574951172,
      "activations/layer10_attention_weight_max": 42.58225631713867,
      "activations/layer10_attention_weight_min": -34.39201736450195,
      "activations/layer11_attention_weight_max": 36.817176818847656,
      "activations/layer11_attention_weight_min": -27.007919311523438,
      "activations/layer12_attention_weight_max": 24.079605102539062,
      "activations/layer12_attention_weight_min": -27.570308685302734,
      "activations/layer13_attention_weight_max": 73.03655242919922,
      "activations/layer13_attention_weight_min": -46.39775466918945,
      "activations/layer14_attention_weight_max": 56.01301193237305,
      "activations/layer14_attention_weight_min": -32.20661544799805,
      "activations/layer15_attention_weight_max": 55.19511413574219,
      "activations/layer15_attention_weight_min": -33.65593719482422,
      "activations/layer16_attention_weight_max": 44.24600601196289,
      "activations/layer16_attention_weight_min": -28.759477615356445,
      "activations/layer17_attention_weight_max": 67.2294692993164,
      "activations/layer17_attention_weight_min": -37.1000862121582,
      "activations/layer18_attention_weight_max": 54.967586517333984,
      "activations/layer18_attention_weight_min": -32.484275817871094,
      "activations/layer19_attention_weight_max": 23.71956443786621,
      "activations/layer19_attention_weight_min": -18.005420684814453,
      "activations/layer1_attention_weight_max": 18.509754180908203,
      "activations/layer1_attention_weight_min": -15.131118774414062,
      "activations/layer20_attention_weight_max": 23.126224517822266,
      "activations/layer20_attention_weight_min": -22.9254207611084,
      "activations/layer21_attention_weight_max": 40.417686462402344,
      "activations/layer21_attention_weight_min": -29.43914031982422,
      "activations/layer22_attention_weight_max": 36.65900802612305,
      "activations/layer22_attention_weight_min": -22.93349838256836,
      "activations/layer23_attention_weight_max": 28.549131393432617,
      "activations/layer23_attention_weight_min": -17.741037368774414,
      "activations/layer2_attention_weight_max": 29.055442810058594,
      "activations/layer2_attention_weight_min": -28.010910034179688,
      "activations/layer3_attention_weight_max": 75.7781753540039,
      "activations/layer3_attention_weight_min": -81.72965240478516,
      "activations/layer4_attention_weight_max": 100.33698272705078,
      "activations/layer4_attention_weight_min": -97.51178741455078,
      "activations/layer5_attention_weight_max": 116.66051483154297,
      "activations/layer5_attention_weight_min": -105.97399139404297,
      "activations/layer6_attention_weight_max": 58.96171188354492,
      "activations/layer6_attention_weight_min": -55.048004150390625,
      "activations/layer7_attention_weight_max": 64.68077087402344,
      "activations/layer7_attention_weight_min": -59.24003601074219,
      "activations/layer8_attention_weight_max": 40.01142883300781,
      "activations/layer8_attention_weight_min": -38.9539680480957,
      "activations/layer9_attention_weight_max": 53.80548858642578,
      "activations/layer9_attention_weight_min": -47.37587356567383,
      "epoch": 3.86,
      "learning_rate": 0.00012639166666666664,
      "loss": 2.8937,
      "step": 66400
    },
    {
      "activations/layer0_attention_weight_max": 16.7443790435791,
      "activations/layer0_attention_weight_min": -15.172770500183105,
      "activations/layer10_attention_weight_max": 41.555267333984375,
      "activations/layer10_attention_weight_min": -33.92445755004883,
      "activations/layer11_attention_weight_max": 34.81420135498047,
      "activations/layer11_attention_weight_min": -27.20582389831543,
      "activations/layer12_attention_weight_max": 24.1643123626709,
      "activations/layer12_attention_weight_min": -26.190841674804688,
      "activations/layer13_attention_weight_max": 57.16081619262695,
      "activations/layer13_attention_weight_min": -35.24125671386719,
      "activations/layer14_attention_weight_max": 51.665775299072266,
      "activations/layer14_attention_weight_min": -29.221832275390625,
      "activations/layer15_attention_weight_max": 39.24058151245117,
      "activations/layer15_attention_weight_min": -30.37338638305664,
      "activations/layer16_attention_weight_max": 34.424072265625,
      "activations/layer16_attention_weight_min": -28.29229736328125,
      "activations/layer17_attention_weight_max": 62.37833786010742,
      "activations/layer17_attention_weight_min": -37.327999114990234,
      "activations/layer18_attention_weight_max": 47.38847351074219,
      "activations/layer18_attention_weight_min": -33.80849075317383,
      "activations/layer19_attention_weight_max": 21.72174644470215,
      "activations/layer19_attention_weight_min": -19.265270233154297,
      "activations/layer1_attention_weight_max": 17.691957473754883,
      "activations/layer1_attention_weight_min": -15.699045181274414,
      "activations/layer20_attention_weight_max": 20.205730438232422,
      "activations/layer20_attention_weight_min": -22.558834075927734,
      "activations/layer21_attention_weight_max": 39.92636489868164,
      "activations/layer21_attention_weight_min": -29.68180274963379,
      "activations/layer22_attention_weight_max": 34.69965362548828,
      "activations/layer22_attention_weight_min": -23.99205207824707,
      "activations/layer23_attention_weight_max": 24.74311065673828,
      "activations/layer23_attention_weight_min": -18.772140502929688,
      "activations/layer2_attention_weight_max": 29.640132904052734,
      "activations/layer2_attention_weight_min": -29.404926300048828,
      "activations/layer3_attention_weight_max": 74.41008758544922,
      "activations/layer3_attention_weight_min": -81.37378692626953,
      "activations/layer4_attention_weight_max": 98.89417266845703,
      "activations/layer4_attention_weight_min": -93.94525909423828,
      "activations/layer5_attention_weight_max": 116.76122283935547,
      "activations/layer5_attention_weight_min": -100.09093475341797,
      "activations/layer6_attention_weight_max": 63.29557418823242,
      "activations/layer6_attention_weight_min": -54.952030181884766,
      "activations/layer7_attention_weight_max": 65.90424346923828,
      "activations/layer7_attention_weight_min": -59.39838409423828,
      "activations/layer8_attention_weight_max": 43.334815979003906,
      "activations/layer8_attention_weight_min": -42.7405891418457,
      "activations/layer9_attention_weight_max": 58.523406982421875,
      "activations/layer9_attention_weight_min": -50.19480895996094,
      "epoch": 3.86,
      "learning_rate": 0.00012637272727272726,
      "loss": 2.8645,
      "step": 66450
    },
    {
      "activations/layer0_attention_weight_max": 15.179670333862305,
      "activations/layer0_attention_weight_min": -14.464922904968262,
      "activations/layer10_attention_weight_max": 40.263587951660156,
      "activations/layer10_attention_weight_min": -36.09954833984375,
      "activations/layer11_attention_weight_max": 35.225833892822266,
      "activations/layer11_attention_weight_min": -29.81578826904297,
      "activations/layer12_attention_weight_max": 25.277076721191406,
      "activations/layer12_attention_weight_min": -26.24561309814453,
      "activations/layer13_attention_weight_max": 51.092430114746094,
      "activations/layer13_attention_weight_min": -38.91349792480469,
      "activations/layer14_attention_weight_max": 41.65288162231445,
      "activations/layer14_attention_weight_min": -31.600786209106445,
      "activations/layer15_attention_weight_max": 45.903297424316406,
      "activations/layer15_attention_weight_min": -34.12964630126953,
      "activations/layer16_attention_weight_max": 47.17911148071289,
      "activations/layer16_attention_weight_min": -28.360000610351562,
      "activations/layer17_attention_weight_max": 63.11991500854492,
      "activations/layer17_attention_weight_min": -39.20573806762695,
      "activations/layer18_attention_weight_max": 55.223697662353516,
      "activations/layer18_attention_weight_min": -34.44365692138672,
      "activations/layer19_attention_weight_max": 23.948965072631836,
      "activations/layer19_attention_weight_min": -18.505943298339844,
      "activations/layer1_attention_weight_max": 18.280630111694336,
      "activations/layer1_attention_weight_min": -15.980871200561523,
      "activations/layer20_attention_weight_max": 22.238542556762695,
      "activations/layer20_attention_weight_min": -21.05640983581543,
      "activations/layer21_attention_weight_max": 45.79071044921875,
      "activations/layer21_attention_weight_min": -27.364072799682617,
      "activations/layer22_attention_weight_max": 34.60852813720703,
      "activations/layer22_attention_weight_min": -23.406024932861328,
      "activations/layer23_attention_weight_max": 25.829978942871094,
      "activations/layer23_attention_weight_min": -16.386661529541016,
      "activations/layer2_attention_weight_max": 31.42211151123047,
      "activations/layer2_attention_weight_min": -28.957761764526367,
      "activations/layer3_attention_weight_max": 84.30722045898438,
      "activations/layer3_attention_weight_min": -81.2914047241211,
      "activations/layer4_attention_weight_max": 110.78047180175781,
      "activations/layer4_attention_weight_min": -92.66018676757812,
      "activations/layer5_attention_weight_max": 119.6905746459961,
      "activations/layer5_attention_weight_min": -100.70421600341797,
      "activations/layer6_attention_weight_max": 61.368019104003906,
      "activations/layer6_attention_weight_min": -55.09662628173828,
      "activations/layer7_attention_weight_max": 70.16056823730469,
      "activations/layer7_attention_weight_min": -64.89937591552734,
      "activations/layer8_attention_weight_max": 43.59330368041992,
      "activations/layer8_attention_weight_min": -39.54120635986328,
      "activations/layer9_attention_weight_max": 59.359771728515625,
      "activations/layer9_attention_weight_min": -50.18281936645508,
      "epoch": 3.86,
      "learning_rate": 0.00012635378787878785,
      "loss": 2.8892,
      "step": 66500
    },
    {
      "activations/layer0_attention_weight_max": 14.531403541564941,
      "activations/layer0_attention_weight_min": -14.446497917175293,
      "activations/layer10_attention_weight_max": 39.506431579589844,
      "activations/layer10_attention_weight_min": -34.26036834716797,
      "activations/layer11_attention_weight_max": 29.216815948486328,
      "activations/layer11_attention_weight_min": -25.969894409179688,
      "activations/layer12_attention_weight_max": 28.057260513305664,
      "activations/layer12_attention_weight_min": -28.041955947875977,
      "activations/layer13_attention_weight_max": 49.02342987060547,
      "activations/layer13_attention_weight_min": -37.9770622253418,
      "activations/layer14_attention_weight_max": 45.36915588378906,
      "activations/layer14_attention_weight_min": -32.621768951416016,
      "activations/layer15_attention_weight_max": 41.09232711791992,
      "activations/layer15_attention_weight_min": -32.46503829956055,
      "activations/layer16_attention_weight_max": 38.023956298828125,
      "activations/layer16_attention_weight_min": -29.320905685424805,
      "activations/layer17_attention_weight_max": 59.31179428100586,
      "activations/layer17_attention_weight_min": -37.66944885253906,
      "activations/layer18_attention_weight_max": 56.5304069519043,
      "activations/layer18_attention_weight_min": -33.133243560791016,
      "activations/layer19_attention_weight_max": 23.070947647094727,
      "activations/layer19_attention_weight_min": -18.608999252319336,
      "activations/layer1_attention_weight_max": 18.15753746032715,
      "activations/layer1_attention_weight_min": -15.255437850952148,
      "activations/layer20_attention_weight_max": 23.340192794799805,
      "activations/layer20_attention_weight_min": -22.45799446105957,
      "activations/layer21_attention_weight_max": 38.43637466430664,
      "activations/layer21_attention_weight_min": -26.73625946044922,
      "activations/layer22_attention_weight_max": 35.22136688232422,
      "activations/layer22_attention_weight_min": -25.58246421813965,
      "activations/layer23_attention_weight_max": 29.781055450439453,
      "activations/layer23_attention_weight_min": -19.012353897094727,
      "activations/layer2_attention_weight_max": 32.0911979675293,
      "activations/layer2_attention_weight_min": -29.053937911987305,
      "activations/layer3_attention_weight_max": 78.63056945800781,
      "activations/layer3_attention_weight_min": -80.83119201660156,
      "activations/layer4_attention_weight_max": 98.6506576538086,
      "activations/layer4_attention_weight_min": -91.5940170288086,
      "activations/layer5_attention_weight_max": 111.83154296875,
      "activations/layer5_attention_weight_min": -94.65184020996094,
      "activations/layer6_attention_weight_max": 59.21635437011719,
      "activations/layer6_attention_weight_min": -52.07207489013672,
      "activations/layer7_attention_weight_max": 63.481353759765625,
      "activations/layer7_attention_weight_min": -58.99628448486328,
      "activations/layer8_attention_weight_max": 40.57278823852539,
      "activations/layer8_attention_weight_min": -38.1506233215332,
      "activations/layer9_attention_weight_max": 56.5591926574707,
      "activations/layer9_attention_weight_min": -48.05482482910156,
      "epoch": 3.87,
      "learning_rate": 0.00012633484848484846,
      "loss": 2.8845,
      "step": 66550
    },
    {
      "activations/layer0_attention_weight_max": 13.860627174377441,
      "activations/layer0_attention_weight_min": -14.69051456451416,
      "activations/layer10_attention_weight_max": 41.854496002197266,
      "activations/layer10_attention_weight_min": -36.6227912902832,
      "activations/layer11_attention_weight_max": 33.58013153076172,
      "activations/layer11_attention_weight_min": -26.670944213867188,
      "activations/layer12_attention_weight_max": 35.2913703918457,
      "activations/layer12_attention_weight_min": -29.23359489440918,
      "activations/layer13_attention_weight_max": 52.74111557006836,
      "activations/layer13_attention_weight_min": -41.50859451293945,
      "activations/layer14_attention_weight_max": 55.37178421020508,
      "activations/layer14_attention_weight_min": -37.162498474121094,
      "activations/layer15_attention_weight_max": 46.2412223815918,
      "activations/layer15_attention_weight_min": -32.623390197753906,
      "activations/layer16_attention_weight_max": 37.00653076171875,
      "activations/layer16_attention_weight_min": -24.571748733520508,
      "activations/layer17_attention_weight_max": 59.62433624267578,
      "activations/layer17_attention_weight_min": -39.18881607055664,
      "activations/layer18_attention_weight_max": 52.681522369384766,
      "activations/layer18_attention_weight_min": -34.15514373779297,
      "activations/layer19_attention_weight_max": 24.644620895385742,
      "activations/layer19_attention_weight_min": -19.52581214904785,
      "activations/layer1_attention_weight_max": 16.611574172973633,
      "activations/layer1_attention_weight_min": -14.700821876525879,
      "activations/layer20_attention_weight_max": 22.161508560180664,
      "activations/layer20_attention_weight_min": -23.553983688354492,
      "activations/layer21_attention_weight_max": 39.67668533325195,
      "activations/layer21_attention_weight_min": -26.03422737121582,
      "activations/layer22_attention_weight_max": 35.166587829589844,
      "activations/layer22_attention_weight_min": -24.334266662597656,
      "activations/layer23_attention_weight_max": 26.05691909790039,
      "activations/layer23_attention_weight_min": -16.372835159301758,
      "activations/layer2_attention_weight_max": 31.101417541503906,
      "activations/layer2_attention_weight_min": -28.43018913269043,
      "activations/layer3_attention_weight_max": 82.95992279052734,
      "activations/layer3_attention_weight_min": -79.7088394165039,
      "activations/layer4_attention_weight_max": 107.79536437988281,
      "activations/layer4_attention_weight_min": -98.4605712890625,
      "activations/layer5_attention_weight_max": 116.77714538574219,
      "activations/layer5_attention_weight_min": -103.76225280761719,
      "activations/layer6_attention_weight_max": 64.11925506591797,
      "activations/layer6_attention_weight_min": -55.65345001220703,
      "activations/layer7_attention_weight_max": 68.57162475585938,
      "activations/layer7_attention_weight_min": -60.3385009765625,
      "activations/layer8_attention_weight_max": 42.31119155883789,
      "activations/layer8_attention_weight_min": -42.507484436035156,
      "activations/layer9_attention_weight_max": 59.149940490722656,
      "activations/layer9_attention_weight_min": -48.53069305419922,
      "epoch": 3.87,
      "learning_rate": 0.00012631590909090908,
      "loss": 2.8818,
      "step": 66600
    },
    {
      "activations/layer0_attention_weight_max": 14.596733093261719,
      "activations/layer0_attention_weight_min": -14.526748657226562,
      "activations/layer10_attention_weight_max": 39.192726135253906,
      "activations/layer10_attention_weight_min": -33.00400924682617,
      "activations/layer11_attention_weight_max": 32.890785217285156,
      "activations/layer11_attention_weight_min": -26.302547454833984,
      "activations/layer12_attention_weight_max": 26.641952514648438,
      "activations/layer12_attention_weight_min": -24.616928100585938,
      "activations/layer13_attention_weight_max": 52.22769546508789,
      "activations/layer13_attention_weight_min": -36.61286544799805,
      "activations/layer14_attention_weight_max": 41.697052001953125,
      "activations/layer14_attention_weight_min": -30.26865577697754,
      "activations/layer15_attention_weight_max": 43.11811447143555,
      "activations/layer15_attention_weight_min": -31.201251983642578,
      "activations/layer16_attention_weight_max": 38.88644790649414,
      "activations/layer16_attention_weight_min": -26.39157485961914,
      "activations/layer17_attention_weight_max": 54.447086334228516,
      "activations/layer17_attention_weight_min": -35.76572036743164,
      "activations/layer18_attention_weight_max": 50.16568374633789,
      "activations/layer18_attention_weight_min": -31.390010833740234,
      "activations/layer19_attention_weight_max": 23.8453426361084,
      "activations/layer19_attention_weight_min": -18.06196403503418,
      "activations/layer1_attention_weight_max": 17.774160385131836,
      "activations/layer1_attention_weight_min": -16.33255386352539,
      "activations/layer20_attention_weight_max": 21.51622772216797,
      "activations/layer20_attention_weight_min": -20.42872428894043,
      "activations/layer21_attention_weight_max": 40.14822769165039,
      "activations/layer21_attention_weight_min": -26.1427001953125,
      "activations/layer22_attention_weight_max": 33.82489013671875,
      "activations/layer22_attention_weight_min": -22.949615478515625,
      "activations/layer23_attention_weight_max": 25.061386108398438,
      "activations/layer23_attention_weight_min": -18.253095626831055,
      "activations/layer2_attention_weight_max": 30.20922088623047,
      "activations/layer2_attention_weight_min": -27.79546356201172,
      "activations/layer3_attention_weight_max": 76.9655990600586,
      "activations/layer3_attention_weight_min": -80.6044692993164,
      "activations/layer4_attention_weight_max": 102.71977996826172,
      "activations/layer4_attention_weight_min": -95.72339630126953,
      "activations/layer5_attention_weight_max": 114.57830810546875,
      "activations/layer5_attention_weight_min": -102.13059997558594,
      "activations/layer6_attention_weight_max": 63.05815887451172,
      "activations/layer6_attention_weight_min": -53.422061920166016,
      "activations/layer7_attention_weight_max": 61.44585037231445,
      "activations/layer7_attention_weight_min": -60.26155471801758,
      "activations/layer8_attention_weight_max": 41.95804214477539,
      "activations/layer8_attention_weight_min": -40.65522003173828,
      "activations/layer9_attention_weight_max": 50.71186065673828,
      "activations/layer9_attention_weight_min": -45.482643127441406,
      "epoch": 3.87,
      "learning_rate": 0.0001262969696969697,
      "loss": 2.8817,
      "step": 66650
    },
    {
      "activations/layer0_attention_weight_max": 16.723522186279297,
      "activations/layer0_attention_weight_min": -14.561588287353516,
      "activations/layer10_attention_weight_max": 42.6749153137207,
      "activations/layer10_attention_weight_min": -34.419212341308594,
      "activations/layer11_attention_weight_max": 32.91142272949219,
      "activations/layer11_attention_weight_min": -25.18313980102539,
      "activations/layer12_attention_weight_max": 27.229677200317383,
      "activations/layer12_attention_weight_min": -27.025053024291992,
      "activations/layer13_attention_weight_max": 51.70930099487305,
      "activations/layer13_attention_weight_min": -36.91033172607422,
      "activations/layer14_attention_weight_max": 42.0489501953125,
      "activations/layer14_attention_weight_min": -32.3561897277832,
      "activations/layer15_attention_weight_max": 43.954830169677734,
      "activations/layer15_attention_weight_min": -29.84107780456543,
      "activations/layer16_attention_weight_max": 37.49619674682617,
      "activations/layer16_attention_weight_min": -25.672714233398438,
      "activations/layer17_attention_weight_max": 63.19180679321289,
      "activations/layer17_attention_weight_min": -38.395938873291016,
      "activations/layer18_attention_weight_max": 54.921443939208984,
      "activations/layer18_attention_weight_min": -36.50514602661133,
      "activations/layer19_attention_weight_max": 24.769147872924805,
      "activations/layer19_attention_weight_min": -19.622560501098633,
      "activations/layer1_attention_weight_max": 17.579570770263672,
      "activations/layer1_attention_weight_min": -14.723555564880371,
      "activations/layer20_attention_weight_max": 21.84459114074707,
      "activations/layer20_attention_weight_min": -20.40892219543457,
      "activations/layer21_attention_weight_max": 49.615478515625,
      "activations/layer21_attention_weight_min": -29.21445083618164,
      "activations/layer22_attention_weight_max": 36.054012298583984,
      "activations/layer22_attention_weight_min": -23.290063858032227,
      "activations/layer23_attention_weight_max": 28.701560974121094,
      "activations/layer23_attention_weight_min": -18.446136474609375,
      "activations/layer2_attention_weight_max": 29.71323013305664,
      "activations/layer2_attention_weight_min": -28.44662094116211,
      "activations/layer3_attention_weight_max": 76.6675033569336,
      "activations/layer3_attention_weight_min": -79.64705657958984,
      "activations/layer4_attention_weight_max": 97.14067077636719,
      "activations/layer4_attention_weight_min": -93.60365295410156,
      "activations/layer5_attention_weight_max": 112.94375610351562,
      "activations/layer5_attention_weight_min": -98.01710510253906,
      "activations/layer6_attention_weight_max": 61.70787811279297,
      "activations/layer6_attention_weight_min": -53.69297409057617,
      "activations/layer7_attention_weight_max": 62.22666549682617,
      "activations/layer7_attention_weight_min": -58.81172561645508,
      "activations/layer8_attention_weight_max": 42.42391586303711,
      "activations/layer8_attention_weight_min": -39.08574295043945,
      "activations/layer9_attention_weight_max": 53.23622512817383,
      "activations/layer9_attention_weight_min": -48.12260055541992,
      "epoch": 3.88,
      "learning_rate": 0.00012627803030303028,
      "loss": 2.8905,
      "step": 66700
    },
    {
      "activations/layer0_attention_weight_max": 15.23518180847168,
      "activations/layer0_attention_weight_min": -14.683329582214355,
      "activations/layer10_attention_weight_max": 43.67122268676758,
      "activations/layer10_attention_weight_min": -35.4282112121582,
      "activations/layer11_attention_weight_max": 34.002464294433594,
      "activations/layer11_attention_weight_min": -27.871009826660156,
      "activations/layer12_attention_weight_max": 26.24613380432129,
      "activations/layer12_attention_weight_min": -26.221633911132812,
      "activations/layer13_attention_weight_max": 66.74346923828125,
      "activations/layer13_attention_weight_min": -40.53916931152344,
      "activations/layer14_attention_weight_max": 64.58140563964844,
      "activations/layer14_attention_weight_min": -37.83076477050781,
      "activations/layer15_attention_weight_max": 49.99481201171875,
      "activations/layer15_attention_weight_min": -31.61067008972168,
      "activations/layer16_attention_weight_max": 47.23754119873047,
      "activations/layer16_attention_weight_min": -27.542049407958984,
      "activations/layer17_attention_weight_max": 61.64176559448242,
      "activations/layer17_attention_weight_min": -35.478816986083984,
      "activations/layer18_attention_weight_max": 52.789852142333984,
      "activations/layer18_attention_weight_min": -33.18584442138672,
      "activations/layer19_attention_weight_max": 23.391578674316406,
      "activations/layer19_attention_weight_min": -18.94288444519043,
      "activations/layer1_attention_weight_max": 17.12542724609375,
      "activations/layer1_attention_weight_min": -13.646271705627441,
      "activations/layer20_attention_weight_max": 23.42506217956543,
      "activations/layer20_attention_weight_min": -21.24735450744629,
      "activations/layer21_attention_weight_max": 53.553070068359375,
      "activations/layer21_attention_weight_min": -31.469139099121094,
      "activations/layer22_attention_weight_max": 36.78981018066406,
      "activations/layer22_attention_weight_min": -25.153564453125,
      "activations/layer23_attention_weight_max": 27.478397369384766,
      "activations/layer23_attention_weight_min": -16.474763870239258,
      "activations/layer2_attention_weight_max": 30.992923736572266,
      "activations/layer2_attention_weight_min": -29.665729522705078,
      "activations/layer3_attention_weight_max": 80.40119934082031,
      "activations/layer3_attention_weight_min": -82.64164733886719,
      "activations/layer4_attention_weight_max": 100.2166976928711,
      "activations/layer4_attention_weight_min": -96.30760955810547,
      "activations/layer5_attention_weight_max": 113.51176452636719,
      "activations/layer5_attention_weight_min": -98.5530014038086,
      "activations/layer6_attention_weight_max": 59.02745819091797,
      "activations/layer6_attention_weight_min": -53.251888275146484,
      "activations/layer7_attention_weight_max": 60.83242416381836,
      "activations/layer7_attention_weight_min": -59.72477722167969,
      "activations/layer8_attention_weight_max": 40.043190002441406,
      "activations/layer8_attention_weight_min": -41.17280960083008,
      "activations/layer9_attention_weight_max": 50.10257339477539,
      "activations/layer9_attention_weight_min": -49.908199310302734,
      "epoch": 3.88,
      "learning_rate": 0.0001262590909090909,
      "loss": 2.8767,
      "step": 66750
    },
    {
      "activations/layer0_attention_weight_max": 15.118878364562988,
      "activations/layer0_attention_weight_min": -14.624946594238281,
      "activations/layer10_attention_weight_max": 37.3931884765625,
      "activations/layer10_attention_weight_min": -31.48490333557129,
      "activations/layer11_attention_weight_max": 32.66215133666992,
      "activations/layer11_attention_weight_min": -25.779823303222656,
      "activations/layer12_attention_weight_max": 23.25536346435547,
      "activations/layer12_attention_weight_min": -23.77458953857422,
      "activations/layer13_attention_weight_max": 47.38880920410156,
      "activations/layer13_attention_weight_min": -40.939796447753906,
      "activations/layer14_attention_weight_max": 47.3026237487793,
      "activations/layer14_attention_weight_min": -36.22435760498047,
      "activations/layer15_attention_weight_max": 45.00129699707031,
      "activations/layer15_attention_weight_min": -30.450105667114258,
      "activations/layer16_attention_weight_max": 34.63759231567383,
      "activations/layer16_attention_weight_min": -25.915138244628906,
      "activations/layer17_attention_weight_max": 59.65166091918945,
      "activations/layer17_attention_weight_min": -38.25144577026367,
      "activations/layer18_attention_weight_max": 57.1613655090332,
      "activations/layer18_attention_weight_min": -33.24586486816406,
      "activations/layer19_attention_weight_max": 20.74333953857422,
      "activations/layer19_attention_weight_min": -19.901575088500977,
      "activations/layer1_attention_weight_max": 17.337465286254883,
      "activations/layer1_attention_weight_min": -15.241708755493164,
      "activations/layer20_attention_weight_max": 20.21335220336914,
      "activations/layer20_attention_weight_min": -21.32652473449707,
      "activations/layer21_attention_weight_max": 39.10730743408203,
      "activations/layer21_attention_weight_min": -28.627981185913086,
      "activations/layer22_attention_weight_max": 34.298709869384766,
      "activations/layer22_attention_weight_min": -23.778690338134766,
      "activations/layer23_attention_weight_max": 27.142759323120117,
      "activations/layer23_attention_weight_min": -19.689136505126953,
      "activations/layer2_attention_weight_max": 29.07802391052246,
      "activations/layer2_attention_weight_min": -28.422882080078125,
      "activations/layer3_attention_weight_max": 73.22551727294922,
      "activations/layer3_attention_weight_min": -78.44624328613281,
      "activations/layer4_attention_weight_max": 92.84384155273438,
      "activations/layer4_attention_weight_min": -88.41212463378906,
      "activations/layer5_attention_weight_max": 107.44114685058594,
      "activations/layer5_attention_weight_min": -95.71111297607422,
      "activations/layer6_attention_weight_max": 56.281803131103516,
      "activations/layer6_attention_weight_min": -50.36909103393555,
      "activations/layer7_attention_weight_max": 59.23505783081055,
      "activations/layer7_attention_weight_min": -54.304874420166016,
      "activations/layer8_attention_weight_max": 37.999969482421875,
      "activations/layer8_attention_weight_min": -35.28982162475586,
      "activations/layer9_attention_weight_max": 46.02016830444336,
      "activations/layer9_attention_weight_min": -41.13067626953125,
      "epoch": 3.88,
      "learning_rate": 0.0001262401515151515,
      "loss": 2.8638,
      "step": 66800
    },
    {
      "activations/layer0_attention_weight_max": 14.659628868103027,
      "activations/layer0_attention_weight_min": -14.32490348815918,
      "activations/layer10_attention_weight_max": 43.255096435546875,
      "activations/layer10_attention_weight_min": -33.37885284423828,
      "activations/layer11_attention_weight_max": 35.97538757324219,
      "activations/layer11_attention_weight_min": -27.73024559020996,
      "activations/layer12_attention_weight_max": 25.311288833618164,
      "activations/layer12_attention_weight_min": -24.015743255615234,
      "activations/layer13_attention_weight_max": 58.50090789794922,
      "activations/layer13_attention_weight_min": -42.409034729003906,
      "activations/layer14_attention_weight_max": 55.473201751708984,
      "activations/layer14_attention_weight_min": -35.41752624511719,
      "activations/layer15_attention_weight_max": 50.904457092285156,
      "activations/layer15_attention_weight_min": -31.268672943115234,
      "activations/layer16_attention_weight_max": 43.48674774169922,
      "activations/layer16_attention_weight_min": -26.86625862121582,
      "activations/layer17_attention_weight_max": 62.71613311767578,
      "activations/layer17_attention_weight_min": -36.36166000366211,
      "activations/layer18_attention_weight_max": 49.48822021484375,
      "activations/layer18_attention_weight_min": -33.87894058227539,
      "activations/layer19_attention_weight_max": 25.376340866088867,
      "activations/layer19_attention_weight_min": -20.050384521484375,
      "activations/layer1_attention_weight_max": 17.793365478515625,
      "activations/layer1_attention_weight_min": -14.17612075805664,
      "activations/layer20_attention_weight_max": 20.49767303466797,
      "activations/layer20_attention_weight_min": -22.227176666259766,
      "activations/layer21_attention_weight_max": 38.781097412109375,
      "activations/layer21_attention_weight_min": -28.104515075683594,
      "activations/layer22_attention_weight_max": 34.826786041259766,
      "activations/layer22_attention_weight_min": -23.540058135986328,
      "activations/layer23_attention_weight_max": 25.22530174255371,
      "activations/layer23_attention_weight_min": -19.946020126342773,
      "activations/layer2_attention_weight_max": 31.148155212402344,
      "activations/layer2_attention_weight_min": -29.318561553955078,
      "activations/layer3_attention_weight_max": 76.33352661132812,
      "activations/layer3_attention_weight_min": -80.79015350341797,
      "activations/layer4_attention_weight_max": 99.39476013183594,
      "activations/layer4_attention_weight_min": -92.11730194091797,
      "activations/layer5_attention_weight_max": 111.24407196044922,
      "activations/layer5_attention_weight_min": -94.78793334960938,
      "activations/layer6_attention_weight_max": 56.81019973754883,
      "activations/layer6_attention_weight_min": -52.87779998779297,
      "activations/layer7_attention_weight_max": 68.0157241821289,
      "activations/layer7_attention_weight_min": -61.531986236572266,
      "activations/layer8_attention_weight_max": 45.57451248168945,
      "activations/layer8_attention_weight_min": -41.01803207397461,
      "activations/layer9_attention_weight_max": 61.3350830078125,
      "activations/layer9_attention_weight_min": -54.34436798095703,
      "epoch": 3.88,
      "learning_rate": 0.0001262212121212121,
      "loss": 2.8929,
      "step": 66850
    },
    {
      "activations/layer0_attention_weight_max": 16.96670150756836,
      "activations/layer0_attention_weight_min": -14.785623550415039,
      "activations/layer10_attention_weight_max": 40.457313537597656,
      "activations/layer10_attention_weight_min": -33.27671432495117,
      "activations/layer11_attention_weight_max": 29.644927978515625,
      "activations/layer11_attention_weight_min": -25.978233337402344,
      "activations/layer12_attention_weight_max": 35.164451599121094,
      "activations/layer12_attention_weight_min": -26.193477630615234,
      "activations/layer13_attention_weight_max": 62.42148971557617,
      "activations/layer13_attention_weight_min": -46.941368103027344,
      "activations/layer14_attention_weight_max": 44.60739517211914,
      "activations/layer14_attention_weight_min": -32.33148193359375,
      "activations/layer15_attention_weight_max": 54.8697395324707,
      "activations/layer15_attention_weight_min": -40.282840728759766,
      "activations/layer16_attention_weight_max": 41.56636047363281,
      "activations/layer16_attention_weight_min": -26.433334350585938,
      "activations/layer17_attention_weight_max": 59.32381820678711,
      "activations/layer17_attention_weight_min": -34.85761260986328,
      "activations/layer18_attention_weight_max": 50.717140197753906,
      "activations/layer18_attention_weight_min": -31.726558685302734,
      "activations/layer19_attention_weight_max": 27.942541122436523,
      "activations/layer19_attention_weight_min": -16.763465881347656,
      "activations/layer1_attention_weight_max": 17.883319854736328,
      "activations/layer1_attention_weight_min": -14.188862800598145,
      "activations/layer20_attention_weight_max": 23.465686798095703,
      "activations/layer20_attention_weight_min": -22.20176887512207,
      "activations/layer21_attention_weight_max": 40.4371337890625,
      "activations/layer21_attention_weight_min": -27.1490478515625,
      "activations/layer22_attention_weight_max": 39.360687255859375,
      "activations/layer22_attention_weight_min": -23.35162353515625,
      "activations/layer23_attention_weight_max": 26.546232223510742,
      "activations/layer23_attention_weight_min": -15.944079399108887,
      "activations/layer2_attention_weight_max": 30.275699615478516,
      "activations/layer2_attention_weight_min": -28.063602447509766,
      "activations/layer3_attention_weight_max": 76.83990478515625,
      "activations/layer3_attention_weight_min": -78.21695709228516,
      "activations/layer4_attention_weight_max": 103.560302734375,
      "activations/layer4_attention_weight_min": -92.57698822021484,
      "activations/layer5_attention_weight_max": 118.61540222167969,
      "activations/layer5_attention_weight_min": -98.74540710449219,
      "activations/layer6_attention_weight_max": 62.71361541748047,
      "activations/layer6_attention_weight_min": -51.53636932373047,
      "activations/layer7_attention_weight_max": 64.53327941894531,
      "activations/layer7_attention_weight_min": -59.95439529418945,
      "activations/layer8_attention_weight_max": 43.073463439941406,
      "activations/layer8_attention_weight_min": -38.09257888793945,
      "activations/layer9_attention_weight_max": 52.307830810546875,
      "activations/layer9_attention_weight_min": -47.32266616821289,
      "epoch": 3.89,
      "learning_rate": 0.0001262022727272727,
      "loss": 2.885,
      "step": 66900
    },
    {
      "activations/layer0_attention_weight_max": 14.851339340209961,
      "activations/layer0_attention_weight_min": -14.748981475830078,
      "activations/layer10_attention_weight_max": 42.4727897644043,
      "activations/layer10_attention_weight_min": -33.807472229003906,
      "activations/layer11_attention_weight_max": 34.03316116333008,
      "activations/layer11_attention_weight_min": -27.475372314453125,
      "activations/layer12_attention_weight_max": 29.018381118774414,
      "activations/layer12_attention_weight_min": -23.865936279296875,
      "activations/layer13_attention_weight_max": 50.580909729003906,
      "activations/layer13_attention_weight_min": -42.659690856933594,
      "activations/layer14_attention_weight_max": 40.53392028808594,
      "activations/layer14_attention_weight_min": -29.44171714782715,
      "activations/layer15_attention_weight_max": 42.28444290161133,
      "activations/layer15_attention_weight_min": -31.096878051757812,
      "activations/layer16_attention_weight_max": 41.81631088256836,
      "activations/layer16_attention_weight_min": -27.803218841552734,
      "activations/layer17_attention_weight_max": 60.98430252075195,
      "activations/layer17_attention_weight_min": -37.22465133666992,
      "activations/layer18_attention_weight_max": 52.42119598388672,
      "activations/layer18_attention_weight_min": -30.438119888305664,
      "activations/layer19_attention_weight_max": 24.360864639282227,
      "activations/layer19_attention_weight_min": -17.939165115356445,
      "activations/layer1_attention_weight_max": 17.894390106201172,
      "activations/layer1_attention_weight_min": -14.636480331420898,
      "activations/layer20_attention_weight_max": 20.161312103271484,
      "activations/layer20_attention_weight_min": -17.851293563842773,
      "activations/layer21_attention_weight_max": 41.02647018432617,
      "activations/layer21_attention_weight_min": -26.764699935913086,
      "activations/layer22_attention_weight_max": 38.7640380859375,
      "activations/layer22_attention_weight_min": -22.373275756835938,
      "activations/layer23_attention_weight_max": 26.76805877685547,
      "activations/layer23_attention_weight_min": -16.378089904785156,
      "activations/layer2_attention_weight_max": 29.28504180908203,
      "activations/layer2_attention_weight_min": -30.01099395751953,
      "activations/layer3_attention_weight_max": 77.037353515625,
      "activations/layer3_attention_weight_min": -82.21539306640625,
      "activations/layer4_attention_weight_max": 103.00318908691406,
      "activations/layer4_attention_weight_min": -97.8673095703125,
      "activations/layer5_attention_weight_max": 124.72933197021484,
      "activations/layer5_attention_weight_min": -109.36752319335938,
      "activations/layer6_attention_weight_max": 63.42766571044922,
      "activations/layer6_attention_weight_min": -54.26457977294922,
      "activations/layer7_attention_weight_max": 67.28491973876953,
      "activations/layer7_attention_weight_min": -60.15778350830078,
      "activations/layer8_attention_weight_max": 43.596031188964844,
      "activations/layer8_attention_weight_min": -40.20542907714844,
      "activations/layer9_attention_weight_max": 56.52861404418945,
      "activations/layer9_attention_weight_min": -46.6120491027832,
      "epoch": 3.89,
      "learning_rate": 0.00012618333333333333,
      "loss": 2.8661,
      "step": 66950
    },
    {
      "activations/layer0_attention_weight_max": 15.634251594543457,
      "activations/layer0_attention_weight_min": -15.01280403137207,
      "activations/layer10_attention_weight_max": 41.612796783447266,
      "activations/layer10_attention_weight_min": -33.80278396606445,
      "activations/layer11_attention_weight_max": 31.53852081298828,
      "activations/layer11_attention_weight_min": -25.950437545776367,
      "activations/layer12_attention_weight_max": 22.029687881469727,
      "activations/layer12_attention_weight_min": -24.892087936401367,
      "activations/layer13_attention_weight_max": 52.60907745361328,
      "activations/layer13_attention_weight_min": -37.74786376953125,
      "activations/layer14_attention_weight_max": 52.700599670410156,
      "activations/layer14_attention_weight_min": -32.05080795288086,
      "activations/layer15_attention_weight_max": 41.90207290649414,
      "activations/layer15_attention_weight_min": -31.793004989624023,
      "activations/layer16_attention_weight_max": 42.041831970214844,
      "activations/layer16_attention_weight_min": -26.13028907775879,
      "activations/layer17_attention_weight_max": 55.7955436706543,
      "activations/layer17_attention_weight_min": -38.86549377441406,
      "activations/layer18_attention_weight_max": 49.68259048461914,
      "activations/layer18_attention_weight_min": -33.71482849121094,
      "activations/layer19_attention_weight_max": 22.975366592407227,
      "activations/layer19_attention_weight_min": -17.554174423217773,
      "activations/layer1_attention_weight_max": 17.722402572631836,
      "activations/layer1_attention_weight_min": -16.74727439880371,
      "activations/layer20_attention_weight_max": 19.63339614868164,
      "activations/layer20_attention_weight_min": -19.313583374023438,
      "activations/layer21_attention_weight_max": 41.06050491333008,
      "activations/layer21_attention_weight_min": -28.381200790405273,
      "activations/layer22_attention_weight_max": 33.52000045776367,
      "activations/layer22_attention_weight_min": -23.010417938232422,
      "activations/layer23_attention_weight_max": 25.247325897216797,
      "activations/layer23_attention_weight_min": -16.499832153320312,
      "activations/layer2_attention_weight_max": 30.067441940307617,
      "activations/layer2_attention_weight_min": -28.63911247253418,
      "activations/layer3_attention_weight_max": 76.56341552734375,
      "activations/layer3_attention_weight_min": -76.34458923339844,
      "activations/layer4_attention_weight_max": 96.97162628173828,
      "activations/layer4_attention_weight_min": -90.80966186523438,
      "activations/layer5_attention_weight_max": 113.1166000366211,
      "activations/layer5_attention_weight_min": -100.55569458007812,
      "activations/layer6_attention_weight_max": 60.59422302246094,
      "activations/layer6_attention_weight_min": -54.6227912902832,
      "activations/layer7_attention_weight_max": 62.9105339050293,
      "activations/layer7_attention_weight_min": -58.91544723510742,
      "activations/layer8_attention_weight_max": 41.61909866333008,
      "activations/layer8_attention_weight_min": -37.48802185058594,
      "activations/layer9_attention_weight_max": 55.39374923706055,
      "activations/layer9_attention_weight_min": -48.63985824584961,
      "epoch": 3.89,
      "learning_rate": 0.00012616439393939394,
      "loss": 2.8815,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_loss": 2.828125,
      "eval_runtime": 8.4503,
      "eval_samples_per_second": 508.15,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_openwebtext_loss": 2.828125,
      "eval_openwebtext_ppl": 16.91371785382974,
      "eval_openwebtext_runtime": 8.4503,
      "eval_openwebtext_samples_per_second": 508.15,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_wikitext_loss": 3.068359375,
      "eval_wikitext_ppl": 21.506589466713315,
      "eval_wikitext_runtime": 1.946,
      "eval_wikitext_samples_per_second": 234.33,
      "step": 67000
    },
    {
      "epoch": 3.89,
      "eval_lambada_loss": 2.849609375,
      "eval_lambada_ppl": 17.28103011956753,
      "eval_lambada_runtime": 9.5506,
      "eval_lambada_samples_per_second": 509.809,
      "step": 67000
    },
    {
      "activations/layer0_attention_weight_max": 15.101480484008789,
      "activations/layer0_attention_weight_min": -14.691061019897461,
      "activations/layer10_attention_weight_max": 38.8590087890625,
      "activations/layer10_attention_weight_min": -31.63815689086914,
      "activations/layer11_attention_weight_max": 29.926185607910156,
      "activations/layer11_attention_weight_min": -25.66134262084961,
      "activations/layer12_attention_weight_max": 23.951311111450195,
      "activations/layer12_attention_weight_min": -28.117265701293945,
      "activations/layer13_attention_weight_max": 49.80860137939453,
      "activations/layer13_attention_weight_min": -41.60063552856445,
      "activations/layer14_attention_weight_max": 52.67388916015625,
      "activations/layer14_attention_weight_min": -34.432796478271484,
      "activations/layer15_attention_weight_max": 44.23897171020508,
      "activations/layer15_attention_weight_min": -30.180696487426758,
      "activations/layer16_attention_weight_max": 38.54574966430664,
      "activations/layer16_attention_weight_min": -26.897403717041016,
      "activations/layer17_attention_weight_max": 60.79815673828125,
      "activations/layer17_attention_weight_min": -35.545833587646484,
      "activations/layer18_attention_weight_max": 58.200843811035156,
      "activations/layer18_attention_weight_min": -33.1588134765625,
      "activations/layer19_attention_weight_max": 25.853652954101562,
      "activations/layer19_attention_weight_min": -21.164173126220703,
      "activations/layer1_attention_weight_max": 17.81561279296875,
      "activations/layer1_attention_weight_min": -14.739814758300781,
      "activations/layer20_attention_weight_max": 27.789472579956055,
      "activations/layer20_attention_weight_min": -19.97880744934082,
      "activations/layer21_attention_weight_max": 52.017967224121094,
      "activations/layer21_attention_weight_min": -31.314451217651367,
      "activations/layer22_attention_weight_max": 36.5569953918457,
      "activations/layer22_attention_weight_min": -24.69729232788086,
      "activations/layer23_attention_weight_max": 31.650999069213867,
      "activations/layer23_attention_weight_min": -19.68732261657715,
      "activations/layer2_attention_weight_max": 30.678220748901367,
      "activations/layer2_attention_weight_min": -29.385112762451172,
      "activations/layer3_attention_weight_max": 77.71067810058594,
      "activations/layer3_attention_weight_min": -80.11744689941406,
      "activations/layer4_attention_weight_max": 100.60847473144531,
      "activations/layer4_attention_weight_min": -90.16883087158203,
      "activations/layer5_attention_weight_max": 115.76814270019531,
      "activations/layer5_attention_weight_min": -96.14039611816406,
      "activations/layer6_attention_weight_max": 59.40278244018555,
      "activations/layer6_attention_weight_min": -51.14054489135742,
      "activations/layer7_attention_weight_max": 61.0025520324707,
      "activations/layer7_attention_weight_min": -58.8331184387207,
      "activations/layer8_attention_weight_max": 40.05156326293945,
      "activations/layer8_attention_weight_min": -37.14350891113281,
      "activations/layer9_attention_weight_max": 47.96332931518555,
      "activations/layer9_attention_weight_min": -45.62443542480469,
      "epoch": 3.9,
      "learning_rate": 0.00012614545454545453,
      "loss": 2.8792,
      "step": 67050
    },
    {
      "activations/layer0_attention_weight_max": 15.075989723205566,
      "activations/layer0_attention_weight_min": -14.496879577636719,
      "activations/layer10_attention_weight_max": 38.733943939208984,
      "activations/layer10_attention_weight_min": -33.593292236328125,
      "activations/layer11_attention_weight_max": 31.40380859375,
      "activations/layer11_attention_weight_min": -25.97047233581543,
      "activations/layer12_attention_weight_max": 24.097253799438477,
      "activations/layer12_attention_weight_min": -25.075275421142578,
      "activations/layer13_attention_weight_max": 55.591400146484375,
      "activations/layer13_attention_weight_min": -41.14614486694336,
      "activations/layer14_attention_weight_max": 56.64788818359375,
      "activations/layer14_attention_weight_min": -30.388072967529297,
      "activations/layer15_attention_weight_max": 47.176387786865234,
      "activations/layer15_attention_weight_min": -32.032562255859375,
      "activations/layer16_attention_weight_max": 43.46613311767578,
      "activations/layer16_attention_weight_min": -28.29292869567871,
      "activations/layer17_attention_weight_max": 62.7280158996582,
      "activations/layer17_attention_weight_min": -38.29064178466797,
      "activations/layer18_attention_weight_max": 53.98721694946289,
      "activations/layer18_attention_weight_min": -34.49903869628906,
      "activations/layer19_attention_weight_max": 23.299407958984375,
      "activations/layer19_attention_weight_min": -17.27819061279297,
      "activations/layer1_attention_weight_max": 19.31287384033203,
      "activations/layer1_attention_weight_min": -15.573238372802734,
      "activations/layer20_attention_weight_max": 20.87379264831543,
      "activations/layer20_attention_weight_min": -20.66061782836914,
      "activations/layer21_attention_weight_max": 38.87842559814453,
      "activations/layer21_attention_weight_min": -26.911277770996094,
      "activations/layer22_attention_weight_max": 37.73164367675781,
      "activations/layer22_attention_weight_min": -24.22014617919922,
      "activations/layer23_attention_weight_max": 24.63100814819336,
      "activations/layer23_attention_weight_min": -16.512958526611328,
      "activations/layer2_attention_weight_max": 32.457054138183594,
      "activations/layer2_attention_weight_min": -30.600749969482422,
      "activations/layer3_attention_weight_max": 79.79219818115234,
      "activations/layer3_attention_weight_min": -81.89024353027344,
      "activations/layer4_attention_weight_max": 106.1661605834961,
      "activations/layer4_attention_weight_min": -102.07979583740234,
      "activations/layer5_attention_weight_max": 119.79493713378906,
      "activations/layer5_attention_weight_min": -107.1098861694336,
      "activations/layer6_attention_weight_max": 64.77847290039062,
      "activations/layer6_attention_weight_min": -56.039371490478516,
      "activations/layer7_attention_weight_max": 64.26200866699219,
      "activations/layer7_attention_weight_min": -59.39593505859375,
      "activations/layer8_attention_weight_max": 40.118736267089844,
      "activations/layer8_attention_weight_min": -38.58940887451172,
      "activations/layer9_attention_weight_max": 57.73102569580078,
      "activations/layer9_attention_weight_min": -50.0375862121582,
      "epoch": 3.9,
      "learning_rate": 0.00012612651515151515,
      "loss": 2.8807,
      "step": 67100
    },
    {
      "activations/layer0_attention_weight_max": 14.002324104309082,
      "activations/layer0_attention_weight_min": -14.994224548339844,
      "activations/layer10_attention_weight_max": 40.48662185668945,
      "activations/layer10_attention_weight_min": -31.92058563232422,
      "activations/layer11_attention_weight_max": 32.67830276489258,
      "activations/layer11_attention_weight_min": -26.208606719970703,
      "activations/layer12_attention_weight_max": 26.460603713989258,
      "activations/layer12_attention_weight_min": -22.587238311767578,
      "activations/layer13_attention_weight_max": 49.81248092651367,
      "activations/layer13_attention_weight_min": -35.03569793701172,
      "activations/layer14_attention_weight_max": 42.68502426147461,
      "activations/layer14_attention_weight_min": -32.455284118652344,
      "activations/layer15_attention_weight_max": 50.47940444946289,
      "activations/layer15_attention_weight_min": -31.8855037689209,
      "activations/layer16_attention_weight_max": 36.098331451416016,
      "activations/layer16_attention_weight_min": -25.939619064331055,
      "activations/layer17_attention_weight_max": 59.87117385864258,
      "activations/layer17_attention_weight_min": -35.495548248291016,
      "activations/layer18_attention_weight_max": 46.10063934326172,
      "activations/layer18_attention_weight_min": -30.153728485107422,
      "activations/layer19_attention_weight_max": 25.369138717651367,
      "activations/layer19_attention_weight_min": -16.888092041015625,
      "activations/layer1_attention_weight_max": 17.443540573120117,
      "activations/layer1_attention_weight_min": -13.945764541625977,
      "activations/layer20_attention_weight_max": 20.043779373168945,
      "activations/layer20_attention_weight_min": -19.675352096557617,
      "activations/layer21_attention_weight_max": 39.900936126708984,
      "activations/layer21_attention_weight_min": -26.963735580444336,
      "activations/layer22_attention_weight_max": 38.29361343383789,
      "activations/layer22_attention_weight_min": -22.86972999572754,
      "activations/layer23_attention_weight_max": 28.844776153564453,
      "activations/layer23_attention_weight_min": -17.711101531982422,
      "activations/layer2_attention_weight_max": 30.68112564086914,
      "activations/layer2_attention_weight_min": -28.267553329467773,
      "activations/layer3_attention_weight_max": 76.10565948486328,
      "activations/layer3_attention_weight_min": -79.81212615966797,
      "activations/layer4_attention_weight_max": 100.89302825927734,
      "activations/layer4_attention_weight_min": -95.5202865600586,
      "activations/layer5_attention_weight_max": 117.79851531982422,
      "activations/layer5_attention_weight_min": -100.86763000488281,
      "activations/layer6_attention_weight_max": 58.814945220947266,
      "activations/layer6_attention_weight_min": -55.68931579589844,
      "activations/layer7_attention_weight_max": 61.55017852783203,
      "activations/layer7_attention_weight_min": -58.55886459350586,
      "activations/layer8_attention_weight_max": 40.02862548828125,
      "activations/layer8_attention_weight_min": -38.72502136230469,
      "activations/layer9_attention_weight_max": 50.6298942565918,
      "activations/layer9_attention_weight_min": -44.7321891784668,
      "epoch": 3.9,
      "learning_rate": 0.00012610757575757576,
      "loss": 2.8909,
      "step": 67150
    },
    {
      "activations/layer0_attention_weight_max": 14.25598430633545,
      "activations/layer0_attention_weight_min": -14.960205078125,
      "activations/layer10_attention_weight_max": 37.457401275634766,
      "activations/layer10_attention_weight_min": -32.37030792236328,
      "activations/layer11_attention_weight_max": 30.026214599609375,
      "activations/layer11_attention_weight_min": -26.09272003173828,
      "activations/layer12_attention_weight_max": 22.432859420776367,
      "activations/layer12_attention_weight_min": -23.874723434448242,
      "activations/layer13_attention_weight_max": 48.011573791503906,
      "activations/layer13_attention_weight_min": -35.6005859375,
      "activations/layer14_attention_weight_max": 37.764095306396484,
      "activations/layer14_attention_weight_min": -29.3636531829834,
      "activations/layer15_attention_weight_max": 39.30888748168945,
      "activations/layer15_attention_weight_min": -31.279129028320312,
      "activations/layer16_attention_weight_max": 33.445945739746094,
      "activations/layer16_attention_weight_min": -24.58608055114746,
      "activations/layer17_attention_weight_max": 53.359920501708984,
      "activations/layer17_attention_weight_min": -35.84824752807617,
      "activations/layer18_attention_weight_max": 45.61185073852539,
      "activations/layer18_attention_weight_min": -32.72285079956055,
      "activations/layer19_attention_weight_max": 21.393226623535156,
      "activations/layer19_attention_weight_min": -18.64480209350586,
      "activations/layer1_attention_weight_max": 17.634807586669922,
      "activations/layer1_attention_weight_min": -14.083656311035156,
      "activations/layer20_attention_weight_max": 19.379539489746094,
      "activations/layer20_attention_weight_min": -18.992338180541992,
      "activations/layer21_attention_weight_max": 37.1900634765625,
      "activations/layer21_attention_weight_min": -25.837575912475586,
      "activations/layer22_attention_weight_max": 31.192394256591797,
      "activations/layer22_attention_weight_min": -22.76596450805664,
      "activations/layer23_attention_weight_max": 22.422826766967773,
      "activations/layer23_attention_weight_min": -17.08810043334961,
      "activations/layer2_attention_weight_max": 32.74559020996094,
      "activations/layer2_attention_weight_min": -29.07289695739746,
      "activations/layer3_attention_weight_max": 74.37995147705078,
      "activations/layer3_attention_weight_min": -77.38069915771484,
      "activations/layer4_attention_weight_max": 96.39994049072266,
      "activations/layer4_attention_weight_min": -89.3606948852539,
      "activations/layer5_attention_weight_max": 105.24263000488281,
      "activations/layer5_attention_weight_min": -94.41975402832031,
      "activations/layer6_attention_weight_max": 53.94969940185547,
      "activations/layer6_attention_weight_min": -50.05538558959961,
      "activations/layer7_attention_weight_max": 58.36857604980469,
      "activations/layer7_attention_weight_min": -58.06105422973633,
      "activations/layer8_attention_weight_max": 36.75908279418945,
      "activations/layer8_attention_weight_min": -38.848876953125,
      "activations/layer9_attention_weight_max": 44.52126693725586,
      "activations/layer9_attention_weight_min": -45.99155807495117,
      "epoch": 3.9,
      "learning_rate": 0.00012608863636363635,
      "loss": 2.875,
      "step": 67200
    },
    {
      "activations/layer0_attention_weight_max": 14.929022789001465,
      "activations/layer0_attention_weight_min": -15.015074729919434,
      "activations/layer10_attention_weight_max": 42.0150032043457,
      "activations/layer10_attention_weight_min": -36.71015167236328,
      "activations/layer11_attention_weight_max": 34.50298309326172,
      "activations/layer11_attention_weight_min": -26.833965301513672,
      "activations/layer12_attention_weight_max": 27.377567291259766,
      "activations/layer12_attention_weight_min": -24.85894203186035,
      "activations/layer13_attention_weight_max": 57.994956970214844,
      "activations/layer13_attention_weight_min": -36.93689727783203,
      "activations/layer14_attention_weight_max": 52.383460998535156,
      "activations/layer14_attention_weight_min": -38.41320037841797,
      "activations/layer15_attention_weight_max": 49.69400405883789,
      "activations/layer15_attention_weight_min": -30.810977935791016,
      "activations/layer16_attention_weight_max": 34.78535079956055,
      "activations/layer16_attention_weight_min": -25.733259201049805,
      "activations/layer17_attention_weight_max": 59.44813537597656,
      "activations/layer17_attention_weight_min": -34.67307662963867,
      "activations/layer18_attention_weight_max": 51.61901092529297,
      "activations/layer18_attention_weight_min": -31.62969398498535,
      "activations/layer19_attention_weight_max": 23.02227210998535,
      "activations/layer19_attention_weight_min": -17.649763107299805,
      "activations/layer1_attention_weight_max": 17.08415412902832,
      "activations/layer1_attention_weight_min": -14.550908088684082,
      "activations/layer20_attention_weight_max": 21.335947036743164,
      "activations/layer20_attention_weight_min": -19.53931427001953,
      "activations/layer21_attention_weight_max": 39.371253967285156,
      "activations/layer21_attention_weight_min": -27.037776947021484,
      "activations/layer22_attention_weight_max": 32.68667221069336,
      "activations/layer22_attention_weight_min": -22.21336555480957,
      "activations/layer23_attention_weight_max": 27.777822494506836,
      "activations/layer23_attention_weight_min": -15.725042343139648,
      "activations/layer2_attention_weight_max": 31.295307159423828,
      "activations/layer2_attention_weight_min": -28.795391082763672,
      "activations/layer3_attention_weight_max": 81.42340087890625,
      "activations/layer3_attention_weight_min": -82.17440032958984,
      "activations/layer4_attention_weight_max": 101.25397491455078,
      "activations/layer4_attention_weight_min": -95.54312133789062,
      "activations/layer5_attention_weight_max": 114.9641342163086,
      "activations/layer5_attention_weight_min": -102.28587341308594,
      "activations/layer6_attention_weight_max": 60.27893829345703,
      "activations/layer6_attention_weight_min": -54.33952713012695,
      "activations/layer7_attention_weight_max": 62.81392288208008,
      "activations/layer7_attention_weight_min": -58.958621978759766,
      "activations/layer8_attention_weight_max": 40.786190032958984,
      "activations/layer8_attention_weight_min": -40.56364822387695,
      "activations/layer9_attention_weight_max": 51.962432861328125,
      "activations/layer9_attention_weight_min": -44.66513442993164,
      "epoch": 3.91,
      "learning_rate": 0.00012606969696969696,
      "loss": 2.8894,
      "step": 67250
    },
    {
      "activations/layer0_attention_weight_max": 14.147549629211426,
      "activations/layer0_attention_weight_min": -14.984265327453613,
      "activations/layer10_attention_weight_max": 40.00634765625,
      "activations/layer10_attention_weight_min": -33.02858352661133,
      "activations/layer11_attention_weight_max": 31.16646385192871,
      "activations/layer11_attention_weight_min": -25.65595054626465,
      "activations/layer12_attention_weight_max": 25.209672927856445,
      "activations/layer12_attention_weight_min": -25.443742752075195,
      "activations/layer13_attention_weight_max": 44.90464401245117,
      "activations/layer13_attention_weight_min": -36.44524383544922,
      "activations/layer14_attention_weight_max": 45.1861457824707,
      "activations/layer14_attention_weight_min": -33.9784049987793,
      "activations/layer15_attention_weight_max": 39.669700622558594,
      "activations/layer15_attention_weight_min": -31.448257446289062,
      "activations/layer16_attention_weight_max": 40.79810333251953,
      "activations/layer16_attention_weight_min": -25.933801651000977,
      "activations/layer17_attention_weight_max": 54.60429763793945,
      "activations/layer17_attention_weight_min": -35.3770637512207,
      "activations/layer18_attention_weight_max": 50.21840286254883,
      "activations/layer18_attention_weight_min": -33.0883674621582,
      "activations/layer19_attention_weight_max": 19.732175827026367,
      "activations/layer19_attention_weight_min": -19.11334228515625,
      "activations/layer1_attention_weight_max": 17.19284439086914,
      "activations/layer1_attention_weight_min": -17.121234893798828,
      "activations/layer20_attention_weight_max": 23.218101501464844,
      "activations/layer20_attention_weight_min": -18.904985427856445,
      "activations/layer21_attention_weight_max": 41.5018310546875,
      "activations/layer21_attention_weight_min": -29.577945709228516,
      "activations/layer22_attention_weight_max": 35.52595138549805,
      "activations/layer22_attention_weight_min": -25.48543930053711,
      "activations/layer23_attention_weight_max": 27.39942169189453,
      "activations/layer23_attention_weight_min": -16.744304656982422,
      "activations/layer2_attention_weight_max": 31.50666046142578,
      "activations/layer2_attention_weight_min": -29.401409149169922,
      "activations/layer3_attention_weight_max": 77.2442626953125,
      "activations/layer3_attention_weight_min": -82.84742736816406,
      "activations/layer4_attention_weight_max": 95.1222152709961,
      "activations/layer4_attention_weight_min": -92.61473846435547,
      "activations/layer5_attention_weight_max": 105.79036712646484,
      "activations/layer5_attention_weight_min": -97.12173461914062,
      "activations/layer6_attention_weight_max": 57.485355377197266,
      "activations/layer6_attention_weight_min": -49.76911163330078,
      "activations/layer7_attention_weight_max": 64.0897445678711,
      "activations/layer7_attention_weight_min": -59.52454376220703,
      "activations/layer8_attention_weight_max": 39.02437973022461,
      "activations/layer8_attention_weight_min": -36.47748565673828,
      "activations/layer9_attention_weight_max": 48.246212005615234,
      "activations/layer9_attention_weight_min": -46.509578704833984,
      "epoch": 3.91,
      "learning_rate": 0.00012605075757575758,
      "loss": 2.8723,
      "step": 67300
    },
    {
      "activations/layer0_attention_weight_max": 15.047430992126465,
      "activations/layer0_attention_weight_min": -14.398666381835938,
      "activations/layer10_attention_weight_max": 45.05573272705078,
      "activations/layer10_attention_weight_min": -37.05865478515625,
      "activations/layer11_attention_weight_max": 33.757179260253906,
      "activations/layer11_attention_weight_min": -27.27001953125,
      "activations/layer12_attention_weight_max": 43.335994720458984,
      "activations/layer12_attention_weight_min": -26.685993194580078,
      "activations/layer13_attention_weight_max": 60.235191345214844,
      "activations/layer13_attention_weight_min": -37.31201171875,
      "activations/layer14_attention_weight_max": 37.46835708618164,
      "activations/layer14_attention_weight_min": -28.015357971191406,
      "activations/layer15_attention_weight_max": 50.06052017211914,
      "activations/layer15_attention_weight_min": -30.614002227783203,
      "activations/layer16_attention_weight_max": 38.27035140991211,
      "activations/layer16_attention_weight_min": -25.04108238220215,
      "activations/layer17_attention_weight_max": 65.0085678100586,
      "activations/layer17_attention_weight_min": -37.14280319213867,
      "activations/layer18_attention_weight_max": 53.562767028808594,
      "activations/layer18_attention_weight_min": -32.14220428466797,
      "activations/layer19_attention_weight_max": 22.181568145751953,
      "activations/layer19_attention_weight_min": -16.431644439697266,
      "activations/layer1_attention_weight_max": 18.539283752441406,
      "activations/layer1_attention_weight_min": -14.528271675109863,
      "activations/layer20_attention_weight_max": 20.083343505859375,
      "activations/layer20_attention_weight_min": -18.54202651977539,
      "activations/layer21_attention_weight_max": 49.27363967895508,
      "activations/layer21_attention_weight_min": -33.112457275390625,
      "activations/layer22_attention_weight_max": 37.247947692871094,
      "activations/layer22_attention_weight_min": -24.487655639648438,
      "activations/layer23_attention_weight_max": 27.425703048706055,
      "activations/layer23_attention_weight_min": -18.390777587890625,
      "activations/layer2_attention_weight_max": 31.542442321777344,
      "activations/layer2_attention_weight_min": -29.849987030029297,
      "activations/layer3_attention_weight_max": 80.06433868408203,
      "activations/layer3_attention_weight_min": -84.01097106933594,
      "activations/layer4_attention_weight_max": 105.1590576171875,
      "activations/layer4_attention_weight_min": -96.87252807617188,
      "activations/layer5_attention_weight_max": 122.6068115234375,
      "activations/layer5_attention_weight_min": -108.4016342163086,
      "activations/layer6_attention_weight_max": 61.941471099853516,
      "activations/layer6_attention_weight_min": -56.47804260253906,
      "activations/layer7_attention_weight_max": 64.7398452758789,
      "activations/layer7_attention_weight_min": -58.9822883605957,
      "activations/layer8_attention_weight_max": 42.07887268066406,
      "activations/layer8_attention_weight_min": -41.33950424194336,
      "activations/layer9_attention_weight_max": 46.89162826538086,
      "activations/layer9_attention_weight_min": -42.73002624511719,
      "epoch": 3.91,
      "learning_rate": 0.00012603181818181817,
      "loss": 2.882,
      "step": 67350
    },
    {
      "activations/layer0_attention_weight_max": 14.993176460266113,
      "activations/layer0_attention_weight_min": -14.619609832763672,
      "activations/layer10_attention_weight_max": 39.83902359008789,
      "activations/layer10_attention_weight_min": -33.21000671386719,
      "activations/layer11_attention_weight_max": 35.37168884277344,
      "activations/layer11_attention_weight_min": -28.135345458984375,
      "activations/layer12_attention_weight_max": 30.95136833190918,
      "activations/layer12_attention_weight_min": -25.802827835083008,
      "activations/layer13_attention_weight_max": 57.35880661010742,
      "activations/layer13_attention_weight_min": -36.930267333984375,
      "activations/layer14_attention_weight_max": 42.379215240478516,
      "activations/layer14_attention_weight_min": -31.419574737548828,
      "activations/layer15_attention_weight_max": 46.20803451538086,
      "activations/layer15_attention_weight_min": -29.45492172241211,
      "activations/layer16_attention_weight_max": 43.02101135253906,
      "activations/layer16_attention_weight_min": -26.239585876464844,
      "activations/layer17_attention_weight_max": 61.38274002075195,
      "activations/layer17_attention_weight_min": -36.24393844604492,
      "activations/layer18_attention_weight_max": 57.37990951538086,
      "activations/layer18_attention_weight_min": -32.570709228515625,
      "activations/layer19_attention_weight_max": 22.840967178344727,
      "activations/layer19_attention_weight_min": -17.78224754333496,
      "activations/layer1_attention_weight_max": 18.569957733154297,
      "activations/layer1_attention_weight_min": -15.022540092468262,
      "activations/layer20_attention_weight_max": 20.682209014892578,
      "activations/layer20_attention_weight_min": -21.004337310791016,
      "activations/layer21_attention_weight_max": 46.79444122314453,
      "activations/layer21_attention_weight_min": -29.67498779296875,
      "activations/layer22_attention_weight_max": 40.55896759033203,
      "activations/layer22_attention_weight_min": -22.809507369995117,
      "activations/layer23_attention_weight_max": 31.405410766601562,
      "activations/layer23_attention_weight_min": -15.174357414245605,
      "activations/layer2_attention_weight_max": 31.03963851928711,
      "activations/layer2_attention_weight_min": -27.446439743041992,
      "activations/layer3_attention_weight_max": 77.81714630126953,
      "activations/layer3_attention_weight_min": -79.9343032836914,
      "activations/layer4_attention_weight_max": 101.04315185546875,
      "activations/layer4_attention_weight_min": -93.01477813720703,
      "activations/layer5_attention_weight_max": 113.44688415527344,
      "activations/layer5_attention_weight_min": -99.30504608154297,
      "activations/layer6_attention_weight_max": 58.99562454223633,
      "activations/layer6_attention_weight_min": -52.601707458496094,
      "activations/layer7_attention_weight_max": 63.35820007324219,
      "activations/layer7_attention_weight_min": -59.33765411376953,
      "activations/layer8_attention_weight_max": 41.344547271728516,
      "activations/layer8_attention_weight_min": -39.00606918334961,
      "activations/layer9_attention_weight_max": 51.091773986816406,
      "activations/layer9_attention_weight_min": -51.13798522949219,
      "epoch": 3.92,
      "learning_rate": 0.00012601287878787878,
      "loss": 2.8811,
      "step": 67400
    },
    {
      "activations/layer0_attention_weight_max": 15.010455131530762,
      "activations/layer0_attention_weight_min": -14.638679504394531,
      "activations/layer10_attention_weight_max": 41.08706283569336,
      "activations/layer10_attention_weight_min": -35.86121368408203,
      "activations/layer11_attention_weight_max": 35.66145324707031,
      "activations/layer11_attention_weight_min": -28.138916015625,
      "activations/layer12_attention_weight_max": 29.03935432434082,
      "activations/layer12_attention_weight_min": -27.887067794799805,
      "activations/layer13_attention_weight_max": 58.00621795654297,
      "activations/layer13_attention_weight_min": -41.249351501464844,
      "activations/layer14_attention_weight_max": 76.41687774658203,
      "activations/layer14_attention_weight_min": -47.274261474609375,
      "activations/layer15_attention_weight_max": 54.634727478027344,
      "activations/layer15_attention_weight_min": -34.98317337036133,
      "activations/layer16_attention_weight_max": 39.71296310424805,
      "activations/layer16_attention_weight_min": -25.70668601989746,
      "activations/layer17_attention_weight_max": 62.39990234375,
      "activations/layer17_attention_weight_min": -37.02312469482422,
      "activations/layer18_attention_weight_max": 55.05839538574219,
      "activations/layer18_attention_weight_min": -32.761653900146484,
      "activations/layer19_attention_weight_max": 22.050884246826172,
      "activations/layer19_attention_weight_min": -17.203237533569336,
      "activations/layer1_attention_weight_max": 17.790258407592773,
      "activations/layer1_attention_weight_min": -13.52531909942627,
      "activations/layer20_attention_weight_max": 19.17418670654297,
      "activations/layer20_attention_weight_min": -21.068265914916992,
      "activations/layer21_attention_weight_max": 37.96138000488281,
      "activations/layer21_attention_weight_min": -33.09136199951172,
      "activations/layer22_attention_weight_max": 35.88311004638672,
      "activations/layer22_attention_weight_min": -24.96689796447754,
      "activations/layer23_attention_weight_max": 22.803394317626953,
      "activations/layer23_attention_weight_min": -17.98190689086914,
      "activations/layer2_attention_weight_max": 30.85597038269043,
      "activations/layer2_attention_weight_min": -29.57427215576172,
      "activations/layer3_attention_weight_max": 82.8990478515625,
      "activations/layer3_attention_weight_min": -87.55891418457031,
      "activations/layer4_attention_weight_max": 110.96559143066406,
      "activations/layer4_attention_weight_min": -102.54548645019531,
      "activations/layer5_attention_weight_max": 126.7967758178711,
      "activations/layer5_attention_weight_min": -109.0596923828125,
      "activations/layer6_attention_weight_max": 62.386600494384766,
      "activations/layer6_attention_weight_min": -58.834327697753906,
      "activations/layer7_attention_weight_max": 79.34880065917969,
      "activations/layer7_attention_weight_min": -65.93892669677734,
      "activations/layer8_attention_weight_max": 53.435420989990234,
      "activations/layer8_attention_weight_min": -41.82027816772461,
      "activations/layer9_attention_weight_max": 70.48001098632812,
      "activations/layer9_attention_weight_min": -56.05963134765625,
      "epoch": 3.92,
      "learning_rate": 0.00012599393939393937,
      "loss": 2.888,
      "step": 67450
    },
    {
      "activations/layer0_attention_weight_max": 13.420707702636719,
      "activations/layer0_attention_weight_min": -15.080626487731934,
      "activations/layer10_attention_weight_max": 45.865238189697266,
      "activations/layer10_attention_weight_min": -35.25668716430664,
      "activations/layer11_attention_weight_max": 36.86341094970703,
      "activations/layer11_attention_weight_min": -30.32297134399414,
      "activations/layer12_attention_weight_max": 26.64128303527832,
      "activations/layer12_attention_weight_min": -24.749439239501953,
      "activations/layer13_attention_weight_max": 68.89329528808594,
      "activations/layer13_attention_weight_min": -40.261104583740234,
      "activations/layer14_attention_weight_max": 70.5947494506836,
      "activations/layer14_attention_weight_min": -32.77714920043945,
      "activations/layer15_attention_weight_max": 47.0155029296875,
      "activations/layer15_attention_weight_min": -30.882125854492188,
      "activations/layer16_attention_weight_max": 42.65454864501953,
      "activations/layer16_attention_weight_min": -26.49156379699707,
      "activations/layer17_attention_weight_max": 65.03721618652344,
      "activations/layer17_attention_weight_min": -39.999542236328125,
      "activations/layer18_attention_weight_max": 55.75209426879883,
      "activations/layer18_attention_weight_min": -32.4901008605957,
      "activations/layer19_attention_weight_max": 25.5589656829834,
      "activations/layer19_attention_weight_min": -22.089879989624023,
      "activations/layer1_attention_weight_max": 18.24532699584961,
      "activations/layer1_attention_weight_min": -17.611507415771484,
      "activations/layer20_attention_weight_max": 24.30885124206543,
      "activations/layer20_attention_weight_min": -18.426746368408203,
      "activations/layer21_attention_weight_max": 45.13482666015625,
      "activations/layer21_attention_weight_min": -29.646848678588867,
      "activations/layer22_attention_weight_max": 40.82393264770508,
      "activations/layer22_attention_weight_min": -23.91539192199707,
      "activations/layer23_attention_weight_max": 32.706336975097656,
      "activations/layer23_attention_weight_min": -17.781532287597656,
      "activations/layer2_attention_weight_max": 34.45219421386719,
      "activations/layer2_attention_weight_min": -29.593780517578125,
      "activations/layer3_attention_weight_max": 89.6387710571289,
      "activations/layer3_attention_weight_min": -80.4567642211914,
      "activations/layer4_attention_weight_max": 119.71337890625,
      "activations/layer4_attention_weight_min": -104.92980194091797,
      "activations/layer5_attention_weight_max": 139.32791137695312,
      "activations/layer5_attention_weight_min": -114.86326599121094,
      "activations/layer6_attention_weight_max": 69.73743438720703,
      "activations/layer6_attention_weight_min": -57.52361297607422,
      "activations/layer7_attention_weight_max": 72.78434753417969,
      "activations/layer7_attention_weight_min": -68.3604736328125,
      "activations/layer8_attention_weight_max": 44.52405548095703,
      "activations/layer8_attention_weight_min": -41.68793869018555,
      "activations/layer9_attention_weight_max": 60.43683624267578,
      "activations/layer9_attention_weight_min": -51.08899688720703,
      "epoch": 3.92,
      "learning_rate": 0.00012597499999999999,
      "loss": 2.8649,
      "step": 67500
    },
    {
      "activations/layer0_attention_weight_max": 15.125840187072754,
      "activations/layer0_attention_weight_min": -14.985928535461426,
      "activations/layer10_attention_weight_max": 38.47314453125,
      "activations/layer10_attention_weight_min": -34.43659973144531,
      "activations/layer11_attention_weight_max": 32.184898376464844,
      "activations/layer11_attention_weight_min": -27.963069915771484,
      "activations/layer12_attention_weight_max": 25.50984001159668,
      "activations/layer12_attention_weight_min": -25.885726928710938,
      "activations/layer13_attention_weight_max": 59.919219970703125,
      "activations/layer13_attention_weight_min": -43.91901779174805,
      "activations/layer14_attention_weight_max": 58.33934783935547,
      "activations/layer14_attention_weight_min": -40.06045150756836,
      "activations/layer15_attention_weight_max": 47.705692291259766,
      "activations/layer15_attention_weight_min": -34.029388427734375,
      "activations/layer16_attention_weight_max": 37.53175735473633,
      "activations/layer16_attention_weight_min": -25.425722122192383,
      "activations/layer17_attention_weight_max": 54.83613586425781,
      "activations/layer17_attention_weight_min": -37.660640716552734,
      "activations/layer18_attention_weight_max": 48.86076354980469,
      "activations/layer18_attention_weight_min": -32.39377975463867,
      "activations/layer19_attention_weight_max": 23.810115814208984,
      "activations/layer19_attention_weight_min": -18.0904541015625,
      "activations/layer1_attention_weight_max": 18.581932067871094,
      "activations/layer1_attention_weight_min": -18.53835105895996,
      "activations/layer20_attention_weight_max": 21.17444610595703,
      "activations/layer20_attention_weight_min": -21.00886344909668,
      "activations/layer21_attention_weight_max": 43.28467559814453,
      "activations/layer21_attention_weight_min": -24.87743377685547,
      "activations/layer22_attention_weight_max": 32.23868179321289,
      "activations/layer22_attention_weight_min": -22.98443031311035,
      "activations/layer23_attention_weight_max": 25.787954330444336,
      "activations/layer23_attention_weight_min": -18.855548858642578,
      "activations/layer2_attention_weight_max": 32.38878631591797,
      "activations/layer2_attention_weight_min": -29.565467834472656,
      "activations/layer3_attention_weight_max": 80.0541000366211,
      "activations/layer3_attention_weight_min": -80.03535461425781,
      "activations/layer4_attention_weight_max": 98.47716522216797,
      "activations/layer4_attention_weight_min": -91.95030975341797,
      "activations/layer5_attention_weight_max": 112.98825073242188,
      "activations/layer5_attention_weight_min": -97.15299987792969,
      "activations/layer6_attention_weight_max": 58.56909942626953,
      "activations/layer6_attention_weight_min": -53.028587341308594,
      "activations/layer7_attention_weight_max": 61.08921432495117,
      "activations/layer7_attention_weight_min": -60.84260177612305,
      "activations/layer8_attention_weight_max": 40.24480056762695,
      "activations/layer8_attention_weight_min": -38.74602127075195,
      "activations/layer9_attention_weight_max": 48.38673400878906,
      "activations/layer9_attention_weight_min": -49.33231735229492,
      "epoch": 3.92,
      "learning_rate": 0.0001259560606060606,
      "loss": 2.8817,
      "step": 67550
    },
    {
      "activations/layer0_attention_weight_max": 15.384056091308594,
      "activations/layer0_attention_weight_min": -14.915958404541016,
      "activations/layer10_attention_weight_max": 44.40869140625,
      "activations/layer10_attention_weight_min": -37.58189392089844,
      "activations/layer11_attention_weight_max": 31.66315460205078,
      "activations/layer11_attention_weight_min": -26.378082275390625,
      "activations/layer12_attention_weight_max": 22.316720962524414,
      "activations/layer12_attention_weight_min": -23.983688354492188,
      "activations/layer13_attention_weight_max": 56.886619567871094,
      "activations/layer13_attention_weight_min": -45.72191619873047,
      "activations/layer14_attention_weight_max": 66.26869201660156,
      "activations/layer14_attention_weight_min": -42.11504364013672,
      "activations/layer15_attention_weight_max": 47.733646392822266,
      "activations/layer15_attention_weight_min": -36.84331512451172,
      "activations/layer16_attention_weight_max": 39.36932373046875,
      "activations/layer16_attention_weight_min": -28.312463760375977,
      "activations/layer17_attention_weight_max": 59.2039680480957,
      "activations/layer17_attention_weight_min": -38.25206756591797,
      "activations/layer18_attention_weight_max": 52.02919006347656,
      "activations/layer18_attention_weight_min": -36.71992874145508,
      "activations/layer19_attention_weight_max": 24.780561447143555,
      "activations/layer19_attention_weight_min": -19.19281578063965,
      "activations/layer1_attention_weight_max": 18.21284294128418,
      "activations/layer1_attention_weight_min": -16.382957458496094,
      "activations/layer20_attention_weight_max": 24.06599998474121,
      "activations/layer20_attention_weight_min": -20.921655654907227,
      "activations/layer21_attention_weight_max": 44.40850830078125,
      "activations/layer21_attention_weight_min": -28.747148513793945,
      "activations/layer22_attention_weight_max": 34.92949676513672,
      "activations/layer22_attention_weight_min": -23.222808837890625,
      "activations/layer23_attention_weight_max": 28.32147979736328,
      "activations/layer23_attention_weight_min": -17.827434539794922,
      "activations/layer2_attention_weight_max": 30.754621505737305,
      "activations/layer2_attention_weight_min": -29.866260528564453,
      "activations/layer3_attention_weight_max": 77.92815399169922,
      "activations/layer3_attention_weight_min": -79.0566635131836,
      "activations/layer4_attention_weight_max": 103.31900787353516,
      "activations/layer4_attention_weight_min": -95.16321563720703,
      "activations/layer5_attention_weight_max": 114.5897216796875,
      "activations/layer5_attention_weight_min": -98.23306274414062,
      "activations/layer6_attention_weight_max": 59.21928787231445,
      "activations/layer6_attention_weight_min": -52.014549255371094,
      "activations/layer7_attention_weight_max": 67.79720306396484,
      "activations/layer7_attention_weight_min": -62.805908203125,
      "activations/layer8_attention_weight_max": 45.41508102416992,
      "activations/layer8_attention_weight_min": -41.40251922607422,
      "activations/layer9_attention_weight_max": 61.566261291503906,
      "activations/layer9_attention_weight_min": -51.05941390991211,
      "epoch": 3.93,
      "learning_rate": 0.0001259371212121212,
      "loss": 2.8701,
      "step": 67600
    },
    {
      "activations/layer0_attention_weight_max": 14.733261108398438,
      "activations/layer0_attention_weight_min": -14.896330833435059,
      "activations/layer10_attention_weight_max": 43.81550216674805,
      "activations/layer10_attention_weight_min": -33.27425003051758,
      "activations/layer11_attention_weight_max": 33.61663818359375,
      "activations/layer11_attention_weight_min": -25.968975067138672,
      "activations/layer12_attention_weight_max": 24.437522888183594,
      "activations/layer12_attention_weight_min": -24.78806495666504,
      "activations/layer13_attention_weight_max": 57.4805908203125,
      "activations/layer13_attention_weight_min": -38.807518005371094,
      "activations/layer14_attention_weight_max": 51.60654067993164,
      "activations/layer14_attention_weight_min": -31.115257263183594,
      "activations/layer15_attention_weight_max": 48.20546340942383,
      "activations/layer15_attention_weight_min": -32.62671661376953,
      "activations/layer16_attention_weight_max": 35.10624313354492,
      "activations/layer16_attention_weight_min": -26.48610496520996,
      "activations/layer17_attention_weight_max": 66.9416275024414,
      "activations/layer17_attention_weight_min": -36.83447265625,
      "activations/layer18_attention_weight_max": 48.86961364746094,
      "activations/layer18_attention_weight_min": -33.47638702392578,
      "activations/layer19_attention_weight_max": 23.983718872070312,
      "activations/layer19_attention_weight_min": -18.669940948486328,
      "activations/layer1_attention_weight_max": 17.366626739501953,
      "activations/layer1_attention_weight_min": -18.4377384185791,
      "activations/layer20_attention_weight_max": 20.585338592529297,
      "activations/layer20_attention_weight_min": -23.988924026489258,
      "activations/layer21_attention_weight_max": 41.446441650390625,
      "activations/layer21_attention_weight_min": -29.25237274169922,
      "activations/layer22_attention_weight_max": 38.288246154785156,
      "activations/layer22_attention_weight_min": -26.02226448059082,
      "activations/layer23_attention_weight_max": 26.595182418823242,
      "activations/layer23_attention_weight_min": -18.384910583496094,
      "activations/layer2_attention_weight_max": 31.458641052246094,
      "activations/layer2_attention_weight_min": -27.588214874267578,
      "activations/layer3_attention_weight_max": 77.28744506835938,
      "activations/layer3_attention_weight_min": -80.23291778564453,
      "activations/layer4_attention_weight_max": 98.51959991455078,
      "activations/layer4_attention_weight_min": -94.52091979980469,
      "activations/layer5_attention_weight_max": 112.88442993164062,
      "activations/layer5_attention_weight_min": -97.94935607910156,
      "activations/layer6_attention_weight_max": 59.946170806884766,
      "activations/layer6_attention_weight_min": -52.509437561035156,
      "activations/layer7_attention_weight_max": 64.74585723876953,
      "activations/layer7_attention_weight_min": -60.22331237792969,
      "activations/layer8_attention_weight_max": 40.04831314086914,
      "activations/layer8_attention_weight_min": -38.34612274169922,
      "activations/layer9_attention_weight_max": 52.989898681640625,
      "activations/layer9_attention_weight_min": -53.0705451965332,
      "epoch": 3.93,
      "learning_rate": 0.0001259181818181818,
      "loss": 2.8662,
      "step": 67650
    },
    {
      "activations/layer0_attention_weight_max": 14.333602905273438,
      "activations/layer0_attention_weight_min": -14.248839378356934,
      "activations/layer10_attention_weight_max": 40.973777770996094,
      "activations/layer10_attention_weight_min": -34.764122009277344,
      "activations/layer11_attention_weight_max": 34.761444091796875,
      "activations/layer11_attention_weight_min": -27.226627349853516,
      "activations/layer12_attention_weight_max": 24.267271041870117,
      "activations/layer12_attention_weight_min": -25.26052474975586,
      "activations/layer13_attention_weight_max": 59.69491958618164,
      "activations/layer13_attention_weight_min": -37.38438415527344,
      "activations/layer14_attention_weight_max": 47.37625503540039,
      "activations/layer14_attention_weight_min": -27.430484771728516,
      "activations/layer15_attention_weight_max": 48.45893478393555,
      "activations/layer15_attention_weight_min": -31.75827980041504,
      "activations/layer16_attention_weight_max": 37.49380874633789,
      "activations/layer16_attention_weight_min": -25.243486404418945,
      "activations/layer17_attention_weight_max": 65.87944793701172,
      "activations/layer17_attention_weight_min": -37.35670852661133,
      "activations/layer18_attention_weight_max": 53.87651062011719,
      "activations/layer18_attention_weight_min": -32.32000732421875,
      "activations/layer19_attention_weight_max": 28.699024200439453,
      "activations/layer19_attention_weight_min": -19.302717208862305,
      "activations/layer1_attention_weight_max": 17.423572540283203,
      "activations/layer1_attention_weight_min": -14.077823638916016,
      "activations/layer20_attention_weight_max": 22.597164154052734,
      "activations/layer20_attention_weight_min": -23.506752014160156,
      "activations/layer21_attention_weight_max": 48.18143844604492,
      "activations/layer21_attention_weight_min": -26.809326171875,
      "activations/layer22_attention_weight_max": 41.35812759399414,
      "activations/layer22_attention_weight_min": -21.834243774414062,
      "activations/layer23_attention_weight_max": 29.55651092529297,
      "activations/layer23_attention_weight_min": -21.679555892944336,
      "activations/layer2_attention_weight_max": 29.480979919433594,
      "activations/layer2_attention_weight_min": -27.469200134277344,
      "activations/layer3_attention_weight_max": 79.5013656616211,
      "activations/layer3_attention_weight_min": -83.96402740478516,
      "activations/layer4_attention_weight_max": 107.92528533935547,
      "activations/layer4_attention_weight_min": -97.94725799560547,
      "activations/layer5_attention_weight_max": 120.17572021484375,
      "activations/layer5_attention_weight_min": -108.13664245605469,
      "activations/layer6_attention_weight_max": 63.11717224121094,
      "activations/layer6_attention_weight_min": -56.05486297607422,
      "activations/layer7_attention_weight_max": 64.37545013427734,
      "activations/layer7_attention_weight_min": -60.02919006347656,
      "activations/layer8_attention_weight_max": 44.14187240600586,
      "activations/layer8_attention_weight_min": -41.09034729003906,
      "activations/layer9_attention_weight_max": 53.034698486328125,
      "activations/layer9_attention_weight_min": -43.82780075073242,
      "epoch": 3.93,
      "learning_rate": 0.00012589924242424242,
      "loss": 2.874,
      "step": 67700
    },
    {
      "activations/layer0_attention_weight_max": 14.138509750366211,
      "activations/layer0_attention_weight_min": -14.861730575561523,
      "activations/layer10_attention_weight_max": 40.40159225463867,
      "activations/layer10_attention_weight_min": -34.74348449707031,
      "activations/layer11_attention_weight_max": 33.88208770751953,
      "activations/layer11_attention_weight_min": -26.882352828979492,
      "activations/layer12_attention_weight_max": 33.58369827270508,
      "activations/layer12_attention_weight_min": -23.62253761291504,
      "activations/layer13_attention_weight_max": 53.216575622558594,
      "activations/layer13_attention_weight_min": -37.22986602783203,
      "activations/layer14_attention_weight_max": 51.253028869628906,
      "activations/layer14_attention_weight_min": -30.302249908447266,
      "activations/layer15_attention_weight_max": 43.42436218261719,
      "activations/layer15_attention_weight_min": -31.1580753326416,
      "activations/layer16_attention_weight_max": 37.18637466430664,
      "activations/layer16_attention_weight_min": -26.1774959564209,
      "activations/layer17_attention_weight_max": 55.23038864135742,
      "activations/layer17_attention_weight_min": -40.32407760620117,
      "activations/layer18_attention_weight_max": 50.34899139404297,
      "activations/layer18_attention_weight_min": -33.452606201171875,
      "activations/layer19_attention_weight_max": 23.99824333190918,
      "activations/layer19_attention_weight_min": -17.52406883239746,
      "activations/layer1_attention_weight_max": 17.647531509399414,
      "activations/layer1_attention_weight_min": -15.31920337677002,
      "activations/layer20_attention_weight_max": 23.542930603027344,
      "activations/layer20_attention_weight_min": -20.219511032104492,
      "activations/layer21_attention_weight_max": 38.48439407348633,
      "activations/layer21_attention_weight_min": -24.76702308654785,
      "activations/layer22_attention_weight_max": 37.82086944580078,
      "activations/layer22_attention_weight_min": -23.57080078125,
      "activations/layer23_attention_weight_max": 27.3309383392334,
      "activations/layer23_attention_weight_min": -15.415983200073242,
      "activations/layer2_attention_weight_max": 30.67434310913086,
      "activations/layer2_attention_weight_min": -28.076459884643555,
      "activations/layer3_attention_weight_max": 78.0599594116211,
      "activations/layer3_attention_weight_min": -78.04894256591797,
      "activations/layer4_attention_weight_max": 107.6032943725586,
      "activations/layer4_attention_weight_min": -92.45425415039062,
      "activations/layer5_attention_weight_max": 124.46965026855469,
      "activations/layer5_attention_weight_min": -103.30489349365234,
      "activations/layer6_attention_weight_max": 64.91034698486328,
      "activations/layer6_attention_weight_min": -55.06916046142578,
      "activations/layer7_attention_weight_max": 62.39866638183594,
      "activations/layer7_attention_weight_min": -61.42574691772461,
      "activations/layer8_attention_weight_max": 41.792030334472656,
      "activations/layer8_attention_weight_min": -38.024471282958984,
      "activations/layer9_attention_weight_max": 49.72115707397461,
      "activations/layer9_attention_weight_min": -55.431251525878906,
      "epoch": 3.94,
      "learning_rate": 0.000125880303030303,
      "loss": 2.8715,
      "step": 67750
    },
    {
      "activations/layer0_attention_weight_max": 15.730244636535645,
      "activations/layer0_attention_weight_min": -14.64146614074707,
      "activations/layer10_attention_weight_max": 39.95187759399414,
      "activations/layer10_attention_weight_min": -32.132415771484375,
      "activations/layer11_attention_weight_max": 29.751815795898438,
      "activations/layer11_attention_weight_min": -24.908161163330078,
      "activations/layer12_attention_weight_max": 23.4066162109375,
      "activations/layer12_attention_weight_min": -24.436792373657227,
      "activations/layer13_attention_weight_max": 49.10849380493164,
      "activations/layer13_attention_weight_min": -37.45741653442383,
      "activations/layer14_attention_weight_max": 48.700439453125,
      "activations/layer14_attention_weight_min": -32.654014587402344,
      "activations/layer15_attention_weight_max": 44.94371032714844,
      "activations/layer15_attention_weight_min": -29.895843505859375,
      "activations/layer16_attention_weight_max": 36.23206329345703,
      "activations/layer16_attention_weight_min": -28.038192749023438,
      "activations/layer17_attention_weight_max": 63.30327606201172,
      "activations/layer17_attention_weight_min": -34.014095306396484,
      "activations/layer18_attention_weight_max": 51.03695297241211,
      "activations/layer18_attention_weight_min": -31.173015594482422,
      "activations/layer19_attention_weight_max": 26.248022079467773,
      "activations/layer19_attention_weight_min": -17.762104034423828,
      "activations/layer1_attention_weight_max": 17.510482788085938,
      "activations/layer1_attention_weight_min": -13.292121887207031,
      "activations/layer20_attention_weight_max": 21.38228416442871,
      "activations/layer20_attention_weight_min": -20.673112869262695,
      "activations/layer21_attention_weight_max": 37.162200927734375,
      "activations/layer21_attention_weight_min": -25.11113739013672,
      "activations/layer22_attention_weight_max": 32.604061126708984,
      "activations/layer22_attention_weight_min": -25.289703369140625,
      "activations/layer23_attention_weight_max": 26.46870231628418,
      "activations/layer23_attention_weight_min": -16.118824005126953,
      "activations/layer2_attention_weight_max": 29.646352767944336,
      "activations/layer2_attention_weight_min": -26.70226287841797,
      "activations/layer3_attention_weight_max": 75.0494613647461,
      "activations/layer3_attention_weight_min": -76.10769653320312,
      "activations/layer4_attention_weight_max": 94.79589080810547,
      "activations/layer4_attention_weight_min": -93.28072357177734,
      "activations/layer5_attention_weight_max": 110.11007690429688,
      "activations/layer5_attention_weight_min": -96.14791870117188,
      "activations/layer6_attention_weight_max": 59.83603286743164,
      "activations/layer6_attention_weight_min": -51.71896743774414,
      "activations/layer7_attention_weight_max": 62.21833801269531,
      "activations/layer7_attention_weight_min": -58.94440841674805,
      "activations/layer8_attention_weight_max": 39.955379486083984,
      "activations/layer8_attention_weight_min": -38.70760726928711,
      "activations/layer9_attention_weight_max": 46.50163269042969,
      "activations/layer9_attention_weight_min": -46.9713134765625,
      "epoch": 3.94,
      "learning_rate": 0.00012586136363636362,
      "loss": 2.8799,
      "step": 67800
    },
    {
      "activations/layer0_attention_weight_max": 15.97170352935791,
      "activations/layer0_attention_weight_min": -14.781617164611816,
      "activations/layer10_attention_weight_max": 38.771453857421875,
      "activations/layer10_attention_weight_min": -33.31489181518555,
      "activations/layer11_attention_weight_max": 30.338415145874023,
      "activations/layer11_attention_weight_min": -26.92976188659668,
      "activations/layer12_attention_weight_max": 25.423419952392578,
      "activations/layer12_attention_weight_min": -27.624826431274414,
      "activations/layer13_attention_weight_max": 49.72612762451172,
      "activations/layer13_attention_weight_min": -36.33240509033203,
      "activations/layer14_attention_weight_max": 52.86703872680664,
      "activations/layer14_attention_weight_min": -32.343013763427734,
      "activations/layer15_attention_weight_max": 42.28483963012695,
      "activations/layer15_attention_weight_min": -31.95330047607422,
      "activations/layer16_attention_weight_max": 36.50031280517578,
      "activations/layer16_attention_weight_min": -30.694339752197266,
      "activations/layer17_attention_weight_max": 59.54988479614258,
      "activations/layer17_attention_weight_min": -39.37733459472656,
      "activations/layer18_attention_weight_max": 53.98950958251953,
      "activations/layer18_attention_weight_min": -35.03945541381836,
      "activations/layer19_attention_weight_max": 23.299020767211914,
      "activations/layer19_attention_weight_min": -17.993968963623047,
      "activations/layer1_attention_weight_max": 17.806243896484375,
      "activations/layer1_attention_weight_min": -14.778521537780762,
      "activations/layer20_attention_weight_max": 20.620059967041016,
      "activations/layer20_attention_weight_min": -24.054540634155273,
      "activations/layer21_attention_weight_max": 41.92850112915039,
      "activations/layer21_attention_weight_min": -29.421781539916992,
      "activations/layer22_attention_weight_max": 37.677364349365234,
      "activations/layer22_attention_weight_min": -24.297836303710938,
      "activations/layer23_attention_weight_max": 31.15111541748047,
      "activations/layer23_attention_weight_min": -18.225208282470703,
      "activations/layer2_attention_weight_max": 29.363319396972656,
      "activations/layer2_attention_weight_min": -26.847110748291016,
      "activations/layer3_attention_weight_max": 77.38016510009766,
      "activations/layer3_attention_weight_min": -80.30175018310547,
      "activations/layer4_attention_weight_max": 100.81295013427734,
      "activations/layer4_attention_weight_min": -95.40399169921875,
      "activations/layer5_attention_weight_max": 112.97964477539062,
      "activations/layer5_attention_weight_min": -105.12196350097656,
      "activations/layer6_attention_weight_max": 59.68898391723633,
      "activations/layer6_attention_weight_min": -52.532997131347656,
      "activations/layer7_attention_weight_max": 61.79611587524414,
      "activations/layer7_attention_weight_min": -59.969425201416016,
      "activations/layer8_attention_weight_max": 41.5976448059082,
      "activations/layer8_attention_weight_min": -38.747314453125,
      "activations/layer9_attention_weight_max": 50.78479766845703,
      "activations/layer9_attention_weight_min": -50.359127044677734,
      "epoch": 3.94,
      "learning_rate": 0.00012584242424242424,
      "loss": 2.8792,
      "step": 67850
    },
    {
      "activations/layer0_attention_weight_max": 16.19781494140625,
      "activations/layer0_attention_weight_min": -15.077077865600586,
      "activations/layer10_attention_weight_max": 37.567588806152344,
      "activations/layer10_attention_weight_min": -33.459228515625,
      "activations/layer11_attention_weight_max": 33.17467498779297,
      "activations/layer11_attention_weight_min": -25.458377838134766,
      "activations/layer12_attention_weight_max": 25.305904388427734,
      "activations/layer12_attention_weight_min": -26.247934341430664,
      "activations/layer13_attention_weight_max": 53.81819534301758,
      "activations/layer13_attention_weight_min": -42.439266204833984,
      "activations/layer14_attention_weight_max": 52.347862243652344,
      "activations/layer14_attention_weight_min": -32.813655853271484,
      "activations/layer15_attention_weight_max": 50.881839752197266,
      "activations/layer15_attention_weight_min": -34.532142639160156,
      "activations/layer16_attention_weight_max": 38.903846740722656,
      "activations/layer16_attention_weight_min": -25.639894485473633,
      "activations/layer17_attention_weight_max": 61.85152053833008,
      "activations/layer17_attention_weight_min": -36.2322883605957,
      "activations/layer18_attention_weight_max": 56.140174865722656,
      "activations/layer18_attention_weight_min": -33.29865646362305,
      "activations/layer19_attention_weight_max": 24.438495635986328,
      "activations/layer19_attention_weight_min": -18.369531631469727,
      "activations/layer1_attention_weight_max": 17.665075302124023,
      "activations/layer1_attention_weight_min": -13.991291999816895,
      "activations/layer20_attention_weight_max": 24.31393814086914,
      "activations/layer20_attention_weight_min": -21.292600631713867,
      "activations/layer21_attention_weight_max": 42.298824310302734,
      "activations/layer21_attention_weight_min": -27.136608123779297,
      "activations/layer22_attention_weight_max": 42.20234680175781,
      "activations/layer22_attention_weight_min": -24.26800537109375,
      "activations/layer23_attention_weight_max": 31.98133087158203,
      "activations/layer23_attention_weight_min": -16.999174118041992,
      "activations/layer2_attention_weight_max": 30.029172897338867,
      "activations/layer2_attention_weight_min": -28.284709930419922,
      "activations/layer3_attention_weight_max": 78.8045654296875,
      "activations/layer3_attention_weight_min": -83.26181030273438,
      "activations/layer4_attention_weight_max": 102.99908447265625,
      "activations/layer4_attention_weight_min": -95.1057357788086,
      "activations/layer5_attention_weight_max": 115.38792419433594,
      "activations/layer5_attention_weight_min": -102.14996337890625,
      "activations/layer6_attention_weight_max": 60.3691291809082,
      "activations/layer6_attention_weight_min": -52.74083709716797,
      "activations/layer7_attention_weight_max": 65.21633911132812,
      "activations/layer7_attention_weight_min": -60.9824104309082,
      "activations/layer8_attention_weight_max": 42.1865234375,
      "activations/layer8_attention_weight_min": -38.14480972290039,
      "activations/layer9_attention_weight_max": 55.67403793334961,
      "activations/layer9_attention_weight_min": -49.40909957885742,
      "epoch": 3.95,
      "learning_rate": 0.00012582348484848482,
      "loss": 2.8755,
      "step": 67900
    },
    {
      "activations/layer0_attention_weight_max": 14.127420425415039,
      "activations/layer0_attention_weight_min": -14.547933578491211,
      "activations/layer10_attention_weight_max": 39.71710968017578,
      "activations/layer10_attention_weight_min": -35.88238525390625,
      "activations/layer11_attention_weight_max": 29.72378921508789,
      "activations/layer11_attention_weight_min": -26.539554595947266,
      "activations/layer12_attention_weight_max": 24.28192901611328,
      "activations/layer12_attention_weight_min": -26.578096389770508,
      "activations/layer13_attention_weight_max": 47.42815399169922,
      "activations/layer13_attention_weight_min": -36.49777603149414,
      "activations/layer14_attention_weight_max": 37.949119567871094,
      "activations/layer14_attention_weight_min": -31.2018985748291,
      "activations/layer15_attention_weight_max": 42.59941482543945,
      "activations/layer15_attention_weight_min": -34.78009796142578,
      "activations/layer16_attention_weight_max": 36.60529327392578,
      "activations/layer16_attention_weight_min": -30.50347900390625,
      "activations/layer17_attention_weight_max": 58.60355758666992,
      "activations/layer17_attention_weight_min": -38.90766906738281,
      "activations/layer18_attention_weight_max": 50.28150939941406,
      "activations/layer18_attention_weight_min": -34.42964553833008,
      "activations/layer19_attention_weight_max": 19.745237350463867,
      "activations/layer19_attention_weight_min": -20.22621726989746,
      "activations/layer1_attention_weight_max": 17.585081100463867,
      "activations/layer1_attention_weight_min": -15.401351928710938,
      "activations/layer20_attention_weight_max": 22.036895751953125,
      "activations/layer20_attention_weight_min": -20.42882537841797,
      "activations/layer21_attention_weight_max": 41.41965103149414,
      "activations/layer21_attention_weight_min": -30.69045066833496,
      "activations/layer22_attention_weight_max": 32.78094482421875,
      "activations/layer22_attention_weight_min": -23.807344436645508,
      "activations/layer23_attention_weight_max": 24.045835494995117,
      "activations/layer23_attention_weight_min": -17.728891372680664,
      "activations/layer2_attention_weight_max": 30.17281150817871,
      "activations/layer2_attention_weight_min": -29.616268157958984,
      "activations/layer3_attention_weight_max": 78.58781433105469,
      "activations/layer3_attention_weight_min": -79.98734283447266,
      "activations/layer4_attention_weight_max": 104.3644027709961,
      "activations/layer4_attention_weight_min": -92.22277069091797,
      "activations/layer5_attention_weight_max": 109.0382080078125,
      "activations/layer5_attention_weight_min": -96.58922576904297,
      "activations/layer6_attention_weight_max": 58.56560516357422,
      "activations/layer6_attention_weight_min": -50.568424224853516,
      "activations/layer7_attention_weight_max": 59.4072380065918,
      "activations/layer7_attention_weight_min": -60.06814193725586,
      "activations/layer8_attention_weight_max": 39.031890869140625,
      "activations/layer8_attention_weight_min": -38.9798469543457,
      "activations/layer9_attention_weight_max": 48.3624382019043,
      "activations/layer9_attention_weight_min": -46.30897521972656,
      "epoch": 3.95,
      "learning_rate": 0.00012580454545454544,
      "loss": 2.8927,
      "step": 67950
    },
    {
      "activations/layer0_attention_weight_max": 14.673013687133789,
      "activations/layer0_attention_weight_min": -14.799676895141602,
      "activations/layer10_attention_weight_max": 38.95038986206055,
      "activations/layer10_attention_weight_min": -32.273590087890625,
      "activations/layer11_attention_weight_max": 37.301841735839844,
      "activations/layer11_attention_weight_min": -24.08291244506836,
      "activations/layer12_attention_weight_max": 25.824329376220703,
      "activations/layer12_attention_weight_min": -27.027217864990234,
      "activations/layer13_attention_weight_max": 51.4663200378418,
      "activations/layer13_attention_weight_min": -41.37732696533203,
      "activations/layer14_attention_weight_max": 61.276607513427734,
      "activations/layer14_attention_weight_min": -34.92796325683594,
      "activations/layer15_attention_weight_max": 46.53125762939453,
      "activations/layer15_attention_weight_min": -32.70814514160156,
      "activations/layer16_attention_weight_max": 37.746768951416016,
      "activations/layer16_attention_weight_min": -26.0925350189209,
      "activations/layer17_attention_weight_max": 61.01169967651367,
      "activations/layer17_attention_weight_min": -35.12935256958008,
      "activations/layer18_attention_weight_max": 47.43117904663086,
      "activations/layer18_attention_weight_min": -32.805416107177734,
      "activations/layer19_attention_weight_max": 24.399402618408203,
      "activations/layer19_attention_weight_min": -18.175642013549805,
      "activations/layer1_attention_weight_max": 17.485492706298828,
      "activations/layer1_attention_weight_min": -16.53299331665039,
      "activations/layer20_attention_weight_max": 22.609220504760742,
      "activations/layer20_attention_weight_min": -20.138782501220703,
      "activations/layer21_attention_weight_max": 42.470455169677734,
      "activations/layer21_attention_weight_min": -29.111608505249023,
      "activations/layer22_attention_weight_max": 33.56001281738281,
      "activations/layer22_attention_weight_min": -22.96074104309082,
      "activations/layer23_attention_weight_max": 29.29949188232422,
      "activations/layer23_attention_weight_min": -16.220579147338867,
      "activations/layer2_attention_weight_max": 32.05970001220703,
      "activations/layer2_attention_weight_min": -29.184856414794922,
      "activations/layer3_attention_weight_max": 81.38053894042969,
      "activations/layer3_attention_weight_min": -79.68557739257812,
      "activations/layer4_attention_weight_max": 109.86328125,
      "activations/layer4_attention_weight_min": -92.32593536376953,
      "activations/layer5_attention_weight_max": 116.78865051269531,
      "activations/layer5_attention_weight_min": -98.19570922851562,
      "activations/layer6_attention_weight_max": 61.33456802368164,
      "activations/layer6_attention_weight_min": -52.313865661621094,
      "activations/layer7_attention_weight_max": 74.46756744384766,
      "activations/layer7_attention_weight_min": -65.85855865478516,
      "activations/layer8_attention_weight_max": 44.649452209472656,
      "activations/layer8_attention_weight_min": -38.739845275878906,
      "activations/layer9_attention_weight_max": 68.53544616699219,
      "activations/layer9_attention_weight_min": -50.98388671875,
      "epoch": 3.95,
      "learning_rate": 0.00012578560606060605,
      "loss": 2.89,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_loss": 2.826171875,
      "eval_runtime": 8.4321,
      "eval_samples_per_second": 509.243,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_openwebtext_loss": 2.826171875,
      "eval_openwebtext_ppl": 16.88071548801052,
      "eval_openwebtext_runtime": 8.4321,
      "eval_openwebtext_samples_per_second": 509.243,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_wikitext_loss": 3.064453125,
      "eval_wikitext_ppl": 21.42274322042474,
      "eval_wikitext_runtime": 1.931,
      "eval_wikitext_samples_per_second": 236.148,
      "step": 68000
    },
    {
      "epoch": 3.95,
      "eval_lambada_loss": 2.86328125,
      "eval_lambada_ppl": 17.51891667538985,
      "eval_lambada_runtime": 9.528,
      "eval_lambada_samples_per_second": 511.019,
      "step": 68000
    },
    {
      "activations/layer0_attention_weight_max": 13.853462219238281,
      "activations/layer0_attention_weight_min": -14.900250434875488,
      "activations/layer10_attention_weight_max": 40.428741455078125,
      "activations/layer10_attention_weight_min": -34.7120246887207,
      "activations/layer11_attention_weight_max": 31.632034301757812,
      "activations/layer11_attention_weight_min": -25.158123016357422,
      "activations/layer12_attention_weight_max": 26.309694290161133,
      "activations/layer12_attention_weight_min": -25.37805938720703,
      "activations/layer13_attention_weight_max": 44.091182708740234,
      "activations/layer13_attention_weight_min": -35.06150436401367,
      "activations/layer14_attention_weight_max": 43.6539306640625,
      "activations/layer14_attention_weight_min": -31.71677589416504,
      "activations/layer15_attention_weight_max": 43.4095458984375,
      "activations/layer15_attention_weight_min": -31.163999557495117,
      "activations/layer16_attention_weight_max": 36.95425796508789,
      "activations/layer16_attention_weight_min": -25.190288543701172,
      "activations/layer17_attention_weight_max": 61.5941047668457,
      "activations/layer17_attention_weight_min": -36.5511474609375,
      "activations/layer18_attention_weight_max": 51.58010482788086,
      "activations/layer18_attention_weight_min": -31.721805572509766,
      "activations/layer19_attention_weight_max": 24.019062042236328,
      "activations/layer19_attention_weight_min": -19.09336280822754,
      "activations/layer1_attention_weight_max": 17.50939178466797,
      "activations/layer1_attention_weight_min": -14.811139106750488,
      "activations/layer20_attention_weight_max": 22.472034454345703,
      "activations/layer20_attention_weight_min": -20.643835067749023,
      "activations/layer21_attention_weight_max": 44.624488830566406,
      "activations/layer21_attention_weight_min": -25.98177146911621,
      "activations/layer22_attention_weight_max": 32.91656494140625,
      "activations/layer22_attention_weight_min": -21.84319496154785,
      "activations/layer23_attention_weight_max": 26.24085807800293,
      "activations/layer23_attention_weight_min": -17.386701583862305,
      "activations/layer2_attention_weight_max": 31.62141227722168,
      "activations/layer2_attention_weight_min": -30.48291015625,
      "activations/layer3_attention_weight_max": 73.97564697265625,
      "activations/layer3_attention_weight_min": -81.21836853027344,
      "activations/layer4_attention_weight_max": 101.03215026855469,
      "activations/layer4_attention_weight_min": -93.41681671142578,
      "activations/layer5_attention_weight_max": 118.36296081542969,
      "activations/layer5_attention_weight_min": -101.6259994506836,
      "activations/layer6_attention_weight_max": 64.371826171875,
      "activations/layer6_attention_weight_min": -52.910587310791016,
      "activations/layer7_attention_weight_max": 61.79962921142578,
      "activations/layer7_attention_weight_min": -61.73666000366211,
      "activations/layer8_attention_weight_max": 40.73299026489258,
      "activations/layer8_attention_weight_min": -37.784671783447266,
      "activations/layer9_attention_weight_max": 46.77120590209961,
      "activations/layer9_attention_weight_min": -46.11158752441406,
      "epoch": 3.95,
      "learning_rate": 0.00012576666666666664,
      "loss": 2.8923,
      "step": 68050
    },
    {
      "activations/layer0_attention_weight_max": 14.332959175109863,
      "activations/layer0_attention_weight_min": -15.258569717407227,
      "activations/layer10_attention_weight_max": 38.7099609375,
      "activations/layer10_attention_weight_min": -31.444351196289062,
      "activations/layer11_attention_weight_max": 34.040977478027344,
      "activations/layer11_attention_weight_min": -25.638090133666992,
      "activations/layer12_attention_weight_max": 21.856164932250977,
      "activations/layer12_attention_weight_min": -25.807008743286133,
      "activations/layer13_attention_weight_max": 52.16007614135742,
      "activations/layer13_attention_weight_min": -41.67406463623047,
      "activations/layer14_attention_weight_max": 48.180999755859375,
      "activations/layer14_attention_weight_min": -31.73628044128418,
      "activations/layer15_attention_weight_max": 49.505680084228516,
      "activations/layer15_attention_weight_min": -31.167863845825195,
      "activations/layer16_attention_weight_max": 39.46272659301758,
      "activations/layer16_attention_weight_min": -27.223281860351562,
      "activations/layer17_attention_weight_max": 63.11570358276367,
      "activations/layer17_attention_weight_min": -36.7089729309082,
      "activations/layer18_attention_weight_max": 52.57811737060547,
      "activations/layer18_attention_weight_min": -34.639888763427734,
      "activations/layer19_attention_weight_max": 26.523283004760742,
      "activations/layer19_attention_weight_min": -19.510746002197266,
      "activations/layer1_attention_weight_max": 20.640674591064453,
      "activations/layer1_attention_weight_min": -14.261575698852539,
      "activations/layer20_attention_weight_max": 22.33995246887207,
      "activations/layer20_attention_weight_min": -20.404895782470703,
      "activations/layer21_attention_weight_max": 42.28151321411133,
      "activations/layer21_attention_weight_min": -27.696197509765625,
      "activations/layer22_attention_weight_max": 39.504093170166016,
      "activations/layer22_attention_weight_min": -25.424654006958008,
      "activations/layer23_attention_weight_max": 28.805370330810547,
      "activations/layer23_attention_weight_min": -18.636962890625,
      "activations/layer2_attention_weight_max": 30.632558822631836,
      "activations/layer2_attention_weight_min": -28.3951473236084,
      "activations/layer3_attention_weight_max": 77.82650756835938,
      "activations/layer3_attention_weight_min": -83.5902099609375,
      "activations/layer4_attention_weight_max": 100.40026092529297,
      "activations/layer4_attention_weight_min": -93.1190414428711,
      "activations/layer5_attention_weight_max": 112.61309814453125,
      "activations/layer5_attention_weight_min": -100.26317596435547,
      "activations/layer6_attention_weight_max": 61.10533142089844,
      "activations/layer6_attention_weight_min": -50.630130767822266,
      "activations/layer7_attention_weight_max": 64.36131286621094,
      "activations/layer7_attention_weight_min": -63.04510498046875,
      "activations/layer8_attention_weight_max": 39.011016845703125,
      "activations/layer8_attention_weight_min": -37.685176849365234,
      "activations/layer9_attention_weight_max": 50.99876022338867,
      "activations/layer9_attention_weight_min": -45.09302520751953,
      "epoch": 3.96,
      "learning_rate": 0.00012574772727272726,
      "loss": 2.8973,
      "step": 68100
    },
    {
      "activations/layer0_attention_weight_max": 13.717519760131836,
      "activations/layer0_attention_weight_min": -15.007715225219727,
      "activations/layer10_attention_weight_max": 42.38405990600586,
      "activations/layer10_attention_weight_min": -36.76293182373047,
      "activations/layer11_attention_weight_max": 33.79962158203125,
      "activations/layer11_attention_weight_min": -28.056774139404297,
      "activations/layer12_attention_weight_max": 27.550870895385742,
      "activations/layer12_attention_weight_min": -26.452598571777344,
      "activations/layer13_attention_weight_max": 62.02655792236328,
      "activations/layer13_attention_weight_min": -40.35380935668945,
      "activations/layer14_attention_weight_max": 57.003082275390625,
      "activations/layer14_attention_weight_min": -34.012752532958984,
      "activations/layer15_attention_weight_max": 47.61185073852539,
      "activations/layer15_attention_weight_min": -29.64861297607422,
      "activations/layer16_attention_weight_max": 39.49536895751953,
      "activations/layer16_attention_weight_min": -26.24147605895996,
      "activations/layer17_attention_weight_max": 62.22303771972656,
      "activations/layer17_attention_weight_min": -37.1589241027832,
      "activations/layer18_attention_weight_max": 57.8905029296875,
      "activations/layer18_attention_weight_min": -32.384552001953125,
      "activations/layer19_attention_weight_max": 24.453962326049805,
      "activations/layer19_attention_weight_min": -19.016128540039062,
      "activations/layer1_attention_weight_max": 18.335128784179688,
      "activations/layer1_attention_weight_min": -13.593172073364258,
      "activations/layer20_attention_weight_max": 22.086196899414062,
      "activations/layer20_attention_weight_min": -19.19304084777832,
      "activations/layer21_attention_weight_max": 42.764102935791016,
      "activations/layer21_attention_weight_min": -23.63896942138672,
      "activations/layer22_attention_weight_max": 39.7324104309082,
      "activations/layer22_attention_weight_min": -22.232202529907227,
      "activations/layer23_attention_weight_max": 30.822124481201172,
      "activations/layer23_attention_weight_min": -16.679594039916992,
      "activations/layer2_attention_weight_max": 29.28266716003418,
      "activations/layer2_attention_weight_min": -27.394527435302734,
      "activations/layer3_attention_weight_max": 82.17413330078125,
      "activations/layer3_attention_weight_min": -84.35070037841797,
      "activations/layer4_attention_weight_max": 107.0050277709961,
      "activations/layer4_attention_weight_min": -99.09545135498047,
      "activations/layer5_attention_weight_max": 124.95891571044922,
      "activations/layer5_attention_weight_min": -104.08248901367188,
      "activations/layer6_attention_weight_max": 62.44293212890625,
      "activations/layer6_attention_weight_min": -56.10135269165039,
      "activations/layer7_attention_weight_max": 64.70243072509766,
      "activations/layer7_attention_weight_min": -64.01264953613281,
      "activations/layer8_attention_weight_max": 46.020111083984375,
      "activations/layer8_attention_weight_min": -42.52675247192383,
      "activations/layer9_attention_weight_max": 63.474971771240234,
      "activations/layer9_attention_weight_min": -52.16423416137695,
      "epoch": 3.96,
      "learning_rate": 0.00012572878787878787,
      "loss": 2.889,
      "step": 68150
    },
    {
      "activations/layer0_attention_weight_max": 14.154732704162598,
      "activations/layer0_attention_weight_min": -15.214919090270996,
      "activations/layer10_attention_weight_max": 38.86548614501953,
      "activations/layer10_attention_weight_min": -33.875057220458984,
      "activations/layer11_attention_weight_max": 32.17033004760742,
      "activations/layer11_attention_weight_min": -25.76430892944336,
      "activations/layer12_attention_weight_max": 27.35914421081543,
      "activations/layer12_attention_weight_min": -25.294265747070312,
      "activations/layer13_attention_weight_max": 58.9493522644043,
      "activations/layer13_attention_weight_min": -44.0199089050293,
      "activations/layer14_attention_weight_max": 50.54132080078125,
      "activations/layer14_attention_weight_min": -34.97901153564453,
      "activations/layer15_attention_weight_max": 42.520668029785156,
      "activations/layer15_attention_weight_min": -31.1042423248291,
      "activations/layer16_attention_weight_max": 35.62813949584961,
      "activations/layer16_attention_weight_min": -27.26136016845703,
      "activations/layer17_attention_weight_max": 55.71766662597656,
      "activations/layer17_attention_weight_min": -38.82584762573242,
      "activations/layer18_attention_weight_max": 49.423583984375,
      "activations/layer18_attention_weight_min": -35.60849380493164,
      "activations/layer19_attention_weight_max": 20.91093635559082,
      "activations/layer19_attention_weight_min": -19.128822326660156,
      "activations/layer1_attention_weight_max": 17.700292587280273,
      "activations/layer1_attention_weight_min": -16.720134735107422,
      "activations/layer20_attention_weight_max": 20.704551696777344,
      "activations/layer20_attention_weight_min": -20.935874938964844,
      "activations/layer21_attention_weight_max": 40.00288772583008,
      "activations/layer21_attention_weight_min": -27.66594123840332,
      "activations/layer22_attention_weight_max": 32.762657165527344,
      "activations/layer22_attention_weight_min": -23.3471622467041,
      "activations/layer23_attention_weight_max": 23.49671745300293,
      "activations/layer23_attention_weight_min": -18.978187561035156,
      "activations/layer2_attention_weight_max": 30.089920043945312,
      "activations/layer2_attention_weight_min": -28.55634307861328,
      "activations/layer3_attention_weight_max": 73.4648208618164,
      "activations/layer3_attention_weight_min": -77.57456970214844,
      "activations/layer4_attention_weight_max": 100.31390380859375,
      "activations/layer4_attention_weight_min": -92.96319580078125,
      "activations/layer5_attention_weight_max": 111.15261840820312,
      "activations/layer5_attention_weight_min": -97.89817810058594,
      "activations/layer6_attention_weight_max": 57.624267578125,
      "activations/layer6_attention_weight_min": -52.599998474121094,
      "activations/layer7_attention_weight_max": 60.070926666259766,
      "activations/layer7_attention_weight_min": -60.820343017578125,
      "activations/layer8_attention_weight_max": 38.57754135131836,
      "activations/layer8_attention_weight_min": -39.78696823120117,
      "activations/layer9_attention_weight_max": 48.31699752807617,
      "activations/layer9_attention_weight_min": -45.98333740234375,
      "epoch": 3.96,
      "learning_rate": 0.0001257098484848485,
      "loss": 2.8784,
      "step": 68200
    },
    {
      "activations/layer0_attention_weight_max": 14.284671783447266,
      "activations/layer0_attention_weight_min": -15.28089427947998,
      "activations/layer10_attention_weight_max": 39.21196365356445,
      "activations/layer10_attention_weight_min": -34.464420318603516,
      "activations/layer11_attention_weight_max": 33.476646423339844,
      "activations/layer11_attention_weight_min": -26.621891021728516,
      "activations/layer12_attention_weight_max": 26.246442794799805,
      "activations/layer12_attention_weight_min": -28.25992774963379,
      "activations/layer13_attention_weight_max": 49.47800064086914,
      "activations/layer13_attention_weight_min": -37.7348747253418,
      "activations/layer14_attention_weight_max": 46.78701400756836,
      "activations/layer14_attention_weight_min": -29.262897491455078,
      "activations/layer15_attention_weight_max": 42.18986892700195,
      "activations/layer15_attention_weight_min": -30.54458236694336,
      "activations/layer16_attention_weight_max": 32.18846130371094,
      "activations/layer16_attention_weight_min": -27.26274299621582,
      "activations/layer17_attention_weight_max": 62.78851318359375,
      "activations/layer17_attention_weight_min": -36.275413513183594,
      "activations/layer18_attention_weight_max": 49.4261589050293,
      "activations/layer18_attention_weight_min": -32.39307403564453,
      "activations/layer19_attention_weight_max": 24.305648803710938,
      "activations/layer19_attention_weight_min": -18.968061447143555,
      "activations/layer1_attention_weight_max": 17.329940795898438,
      "activations/layer1_attention_weight_min": -15.325252532958984,
      "activations/layer20_attention_weight_max": 19.50223159790039,
      "activations/layer20_attention_weight_min": -19.929765701293945,
      "activations/layer21_attention_weight_max": 39.93123245239258,
      "activations/layer21_attention_weight_min": -26.974050521850586,
      "activations/layer22_attention_weight_max": 32.69943618774414,
      "activations/layer22_attention_weight_min": -22.176883697509766,
      "activations/layer23_attention_weight_max": 25.375125885009766,
      "activations/layer23_attention_weight_min": -18.496723175048828,
      "activations/layer2_attention_weight_max": 30.553434371948242,
      "activations/layer2_attention_weight_min": -28.973190307617188,
      "activations/layer3_attention_weight_max": 77.1673355102539,
      "activations/layer3_attention_weight_min": -83.45144653320312,
      "activations/layer4_attention_weight_max": 101.4499740600586,
      "activations/layer4_attention_weight_min": -97.03853607177734,
      "activations/layer5_attention_weight_max": 114.784423828125,
      "activations/layer5_attention_weight_min": -104.13042449951172,
      "activations/layer6_attention_weight_max": 60.12343215942383,
      "activations/layer6_attention_weight_min": -54.62667465209961,
      "activations/layer7_attention_weight_max": 63.287078857421875,
      "activations/layer7_attention_weight_min": -61.20395278930664,
      "activations/layer8_attention_weight_max": 42.50783920288086,
      "activations/layer8_attention_weight_min": -39.60810470581055,
      "activations/layer9_attention_weight_max": 47.7950439453125,
      "activations/layer9_attention_weight_min": -50.98226547241211,
      "epoch": 3.97,
      "learning_rate": 0.00012569090909090908,
      "loss": 2.8712,
      "step": 68250
    },
    {
      "activations/layer0_attention_weight_max": 13.942212104797363,
      "activations/layer0_attention_weight_min": -15.499555587768555,
      "activations/layer10_attention_weight_max": 41.2296028137207,
      "activations/layer10_attention_weight_min": -33.364044189453125,
      "activations/layer11_attention_weight_max": 36.157432556152344,
      "activations/layer11_attention_weight_min": -27.34534454345703,
      "activations/layer12_attention_weight_max": 27.291553497314453,
      "activations/layer12_attention_weight_min": -25.422164916992188,
      "activations/layer13_attention_weight_max": 49.852169036865234,
      "activations/layer13_attention_weight_min": -38.13307571411133,
      "activations/layer14_attention_weight_max": 41.8246955871582,
      "activations/layer14_attention_weight_min": -28.316059112548828,
      "activations/layer15_attention_weight_max": 41.66664505004883,
      "activations/layer15_attention_weight_min": -31.791479110717773,
      "activations/layer16_attention_weight_max": 36.85542297363281,
      "activations/layer16_attention_weight_min": -27.7514591217041,
      "activations/layer17_attention_weight_max": 57.18893814086914,
      "activations/layer17_attention_weight_min": -37.63127136230469,
      "activations/layer18_attention_weight_max": 51.42699432373047,
      "activations/layer18_attention_weight_min": -31.71473503112793,
      "activations/layer19_attention_weight_max": 21.393625259399414,
      "activations/layer19_attention_weight_min": -17.72218132019043,
      "activations/layer1_attention_weight_max": 17.81973648071289,
      "activations/layer1_attention_weight_min": -15.001587867736816,
      "activations/layer20_attention_weight_max": 22.721084594726562,
      "activations/layer20_attention_weight_min": -17.628568649291992,
      "activations/layer21_attention_weight_max": 40.7151985168457,
      "activations/layer21_attention_weight_min": -26.375150680541992,
      "activations/layer22_attention_weight_max": 36.99773025512695,
      "activations/layer22_attention_weight_min": -22.87299919128418,
      "activations/layer23_attention_weight_max": 28.02539825439453,
      "activations/layer23_attention_weight_min": -16.799495697021484,
      "activations/layer2_attention_weight_max": 31.879196166992188,
      "activations/layer2_attention_weight_min": -28.498579025268555,
      "activations/layer3_attention_weight_max": 78.59784698486328,
      "activations/layer3_attention_weight_min": -81.82709503173828,
      "activations/layer4_attention_weight_max": 106.6788330078125,
      "activations/layer4_attention_weight_min": -99.71349334716797,
      "activations/layer5_attention_weight_max": 116.4807357788086,
      "activations/layer5_attention_weight_min": -102.6247787475586,
      "activations/layer6_attention_weight_max": 61.340763092041016,
      "activations/layer6_attention_weight_min": -54.190799713134766,
      "activations/layer7_attention_weight_max": 71.58448791503906,
      "activations/layer7_attention_weight_min": -61.37240219116211,
      "activations/layer8_attention_weight_max": 45.96891403198242,
      "activations/layer8_attention_weight_min": -41.88896942138672,
      "activations/layer9_attention_weight_max": 64.93767547607422,
      "activations/layer9_attention_weight_min": -51.313690185546875,
      "epoch": 3.97,
      "learning_rate": 0.0001256719696969697,
      "loss": 2.8819,
      "step": 68300
    },
    {
      "activations/layer0_attention_weight_max": 13.983428001403809,
      "activations/layer0_attention_weight_min": -14.970718383789062,
      "activations/layer10_attention_weight_max": 38.30445098876953,
      "activations/layer10_attention_weight_min": -32.619747161865234,
      "activations/layer11_attention_weight_max": 30.452285766601562,
      "activations/layer11_attention_weight_min": -26.002525329589844,
      "activations/layer12_attention_weight_max": 23.927814483642578,
      "activations/layer12_attention_weight_min": -28.227039337158203,
      "activations/layer13_attention_weight_max": 48.94353103637695,
      "activations/layer13_attention_weight_min": -35.380062103271484,
      "activations/layer14_attention_weight_max": 43.258914947509766,
      "activations/layer14_attention_weight_min": -31.398035049438477,
      "activations/layer15_attention_weight_max": 46.9669189453125,
      "activations/layer15_attention_weight_min": -31.380470275878906,
      "activations/layer16_attention_weight_max": 33.40628433227539,
      "activations/layer16_attention_weight_min": -27.143388748168945,
      "activations/layer17_attention_weight_max": 57.88666534423828,
      "activations/layer17_attention_weight_min": -36.03512191772461,
      "activations/layer18_attention_weight_max": 49.08662796020508,
      "activations/layer18_attention_weight_min": -34.176780700683594,
      "activations/layer19_attention_weight_max": 20.43248176574707,
      "activations/layer19_attention_weight_min": -17.15450096130371,
      "activations/layer1_attention_weight_max": 17.923866271972656,
      "activations/layer1_attention_weight_min": -15.102529525756836,
      "activations/layer20_attention_weight_max": 21.167179107666016,
      "activations/layer20_attention_weight_min": -19.59828758239746,
      "activations/layer21_attention_weight_max": 38.8764762878418,
      "activations/layer21_attention_weight_min": -28.235275268554688,
      "activations/layer22_attention_weight_max": 32.5302619934082,
      "activations/layer22_attention_weight_min": -22.633481979370117,
      "activations/layer23_attention_weight_max": 26.18592643737793,
      "activations/layer23_attention_weight_min": -16.629009246826172,
      "activations/layer2_attention_weight_max": 29.661083221435547,
      "activations/layer2_attention_weight_min": -27.7493953704834,
      "activations/layer3_attention_weight_max": 82.83152770996094,
      "activations/layer3_attention_weight_min": -82.62146759033203,
      "activations/layer4_attention_weight_max": 103.86871337890625,
      "activations/layer4_attention_weight_min": -95.73509979248047,
      "activations/layer5_attention_weight_max": 112.37963104248047,
      "activations/layer5_attention_weight_min": -96.54812622070312,
      "activations/layer6_attention_weight_max": 57.36250686645508,
      "activations/layer6_attention_weight_min": -53.11455154418945,
      "activations/layer7_attention_weight_max": 60.86003494262695,
      "activations/layer7_attention_weight_min": -59.553829193115234,
      "activations/layer8_attention_weight_max": 38.26639175415039,
      "activations/layer8_attention_weight_min": -36.92448425292969,
      "activations/layer9_attention_weight_max": 44.94587326049805,
      "activations/layer9_attention_weight_min": -46.551368713378906,
      "epoch": 3.97,
      "learning_rate": 0.0001256530303030303,
      "loss": 2.8859,
      "step": 68350
    },
    {
      "activations/layer0_attention_weight_max": 14.818552017211914,
      "activations/layer0_attention_weight_min": -15.332086563110352,
      "activations/layer10_attention_weight_max": 44.38117599487305,
      "activations/layer10_attention_weight_min": -33.58063888549805,
      "activations/layer11_attention_weight_max": 32.67448806762695,
      "activations/layer11_attention_weight_min": -25.535419464111328,
      "activations/layer12_attention_weight_max": 28.54975128173828,
      "activations/layer12_attention_weight_min": -24.17087173461914,
      "activations/layer13_attention_weight_max": 90.62001037597656,
      "activations/layer13_attention_weight_min": -41.22399139404297,
      "activations/layer14_attention_weight_max": 82.96245574951172,
      "activations/layer14_attention_weight_min": -37.783077239990234,
      "activations/layer15_attention_weight_max": 47.891258239746094,
      "activations/layer15_attention_weight_min": -31.469409942626953,
      "activations/layer16_attention_weight_max": 45.473426818847656,
      "activations/layer16_attention_weight_min": -26.122356414794922,
      "activations/layer17_attention_weight_max": 71.0334701538086,
      "activations/layer17_attention_weight_min": -36.43228530883789,
      "activations/layer18_attention_weight_max": 54.12116622924805,
      "activations/layer18_attention_weight_min": -31.617881774902344,
      "activations/layer19_attention_weight_max": 27.108047485351562,
      "activations/layer19_attention_weight_min": -18.109107971191406,
      "activations/layer1_attention_weight_max": 17.656641006469727,
      "activations/layer1_attention_weight_min": -14.94024658203125,
      "activations/layer20_attention_weight_max": 22.318513870239258,
      "activations/layer20_attention_weight_min": -22.39019203186035,
      "activations/layer21_attention_weight_max": 49.09909439086914,
      "activations/layer21_attention_weight_min": -25.88777732849121,
      "activations/layer22_attention_weight_max": 37.81288146972656,
      "activations/layer22_attention_weight_min": -23.445987701416016,
      "activations/layer23_attention_weight_max": 27.309988021850586,
      "activations/layer23_attention_weight_min": -17.72132682800293,
      "activations/layer2_attention_weight_max": 30.201284408569336,
      "activations/layer2_attention_weight_min": -28.380126953125,
      "activations/layer3_attention_weight_max": 76.00367736816406,
      "activations/layer3_attention_weight_min": -82.75537109375,
      "activations/layer4_attention_weight_max": 99.93629455566406,
      "activations/layer4_attention_weight_min": -92.20841979980469,
      "activations/layer5_attention_weight_max": 112.30776977539062,
      "activations/layer5_attention_weight_min": -96.69368743896484,
      "activations/layer6_attention_weight_max": 58.204505920410156,
      "activations/layer6_attention_weight_min": -51.29271697998047,
      "activations/layer7_attention_weight_max": 67.62853240966797,
      "activations/layer7_attention_weight_min": -61.847808837890625,
      "activations/layer8_attention_weight_max": 42.1303825378418,
      "activations/layer8_attention_weight_min": -37.77909851074219,
      "activations/layer9_attention_weight_max": 53.27964401245117,
      "activations/layer9_attention_weight_min": -44.90214157104492,
      "epoch": 3.97,
      "learning_rate": 0.0001256340909090909,
      "loss": 2.8816,
      "step": 68400
    },
    {
      "activations/layer0_attention_weight_max": 14.86742877960205,
      "activations/layer0_attention_weight_min": -14.630919456481934,
      "activations/layer10_attention_weight_max": 41.568687438964844,
      "activations/layer10_attention_weight_min": -31.727489471435547,
      "activations/layer11_attention_weight_max": 34.47822570800781,
      "activations/layer11_attention_weight_min": -24.24441146850586,
      "activations/layer12_attention_weight_max": 24.89999771118164,
      "activations/layer12_attention_weight_min": -24.62275505065918,
      "activations/layer13_attention_weight_max": 56.4283447265625,
      "activations/layer13_attention_weight_min": -36.64103317260742,
      "activations/layer14_attention_weight_max": 49.50321578979492,
      "activations/layer14_attention_weight_min": -34.97050857543945,
      "activations/layer15_attention_weight_max": 47.81752395629883,
      "activations/layer15_attention_weight_min": -31.425806045532227,
      "activations/layer16_attention_weight_max": 36.95489501953125,
      "activations/layer16_attention_weight_min": -26.704866409301758,
      "activations/layer17_attention_weight_max": 63.01333236694336,
      "activations/layer17_attention_weight_min": -35.74361801147461,
      "activations/layer18_attention_weight_max": 54.51706314086914,
      "activations/layer18_attention_weight_min": -32.00596237182617,
      "activations/layer19_attention_weight_max": 25.07124137878418,
      "activations/layer19_attention_weight_min": -19.7455997467041,
      "activations/layer1_attention_weight_max": 18.349838256835938,
      "activations/layer1_attention_weight_min": -17.59577178955078,
      "activations/layer20_attention_weight_max": 21.843006134033203,
      "activations/layer20_attention_weight_min": -24.616626739501953,
      "activations/layer21_attention_weight_max": 41.70873260498047,
      "activations/layer21_attention_weight_min": -29.089675903320312,
      "activations/layer22_attention_weight_max": 36.23672866821289,
      "activations/layer22_attention_weight_min": -23.65794563293457,
      "activations/layer23_attention_weight_max": 26.828990936279297,
      "activations/layer23_attention_weight_min": -18.52338409423828,
      "activations/layer2_attention_weight_max": 30.13534927368164,
      "activations/layer2_attention_weight_min": -27.018857955932617,
      "activations/layer3_attention_weight_max": 79.02149200439453,
      "activations/layer3_attention_weight_min": -80.70518493652344,
      "activations/layer4_attention_weight_max": 105.5522689819336,
      "activations/layer4_attention_weight_min": -93.52623748779297,
      "activations/layer5_attention_weight_max": 116.8391342163086,
      "activations/layer5_attention_weight_min": -103.15169525146484,
      "activations/layer6_attention_weight_max": 59.55637741088867,
      "activations/layer6_attention_weight_min": -53.26008605957031,
      "activations/layer7_attention_weight_max": 63.825443267822266,
      "activations/layer7_attention_weight_min": -59.192405700683594,
      "activations/layer8_attention_weight_max": 41.30023956298828,
      "activations/layer8_attention_weight_min": -38.607845306396484,
      "activations/layer9_attention_weight_max": 48.45590591430664,
      "activations/layer9_attention_weight_min": -43.51890563964844,
      "epoch": 3.98,
      "learning_rate": 0.0001256151515151515,
      "loss": 2.8762,
      "step": 68450
    },
    {
      "activations/layer0_attention_weight_max": 14.313342094421387,
      "activations/layer0_attention_weight_min": -15.401413917541504,
      "activations/layer10_attention_weight_max": 43.51227951049805,
      "activations/layer10_attention_weight_min": -35.28163528442383,
      "activations/layer11_attention_weight_max": 30.93416404724121,
      "activations/layer11_attention_weight_min": -27.359357833862305,
      "activations/layer12_attention_weight_max": 30.565135955810547,
      "activations/layer12_attention_weight_min": -22.912214279174805,
      "activations/layer13_attention_weight_max": 61.584571838378906,
      "activations/layer13_attention_weight_min": -43.70665740966797,
      "activations/layer14_attention_weight_max": 53.277320861816406,
      "activations/layer14_attention_weight_min": -35.39053726196289,
      "activations/layer15_attention_weight_max": 55.36284637451172,
      "activations/layer15_attention_weight_min": -33.994972229003906,
      "activations/layer16_attention_weight_max": 34.15009689331055,
      "activations/layer16_attention_weight_min": -25.435565948486328,
      "activations/layer17_attention_weight_max": 60.39780807495117,
      "activations/layer17_attention_weight_min": -37.009578704833984,
      "activations/layer18_attention_weight_max": 47.61037063598633,
      "activations/layer18_attention_weight_min": -30.547664642333984,
      "activations/layer19_attention_weight_max": 25.888586044311523,
      "activations/layer19_attention_weight_min": -17.000335693359375,
      "activations/layer1_attention_weight_max": 17.603923797607422,
      "activations/layer1_attention_weight_min": -16.671859741210938,
      "activations/layer20_attention_weight_max": 25.033151626586914,
      "activations/layer20_attention_weight_min": -20.845396041870117,
      "activations/layer21_attention_weight_max": 41.68499755859375,
      "activations/layer21_attention_weight_min": -27.626298904418945,
      "activations/layer22_attention_weight_max": 34.770668029785156,
      "activations/layer22_attention_weight_min": -22.299070358276367,
      "activations/layer23_attention_weight_max": 26.611038208007812,
      "activations/layer23_attention_weight_min": -17.821168899536133,
      "activations/layer2_attention_weight_max": 28.229053497314453,
      "activations/layer2_attention_weight_min": -26.944974899291992,
      "activations/layer3_attention_weight_max": 74.94070434570312,
      "activations/layer3_attention_weight_min": -82.99684143066406,
      "activations/layer4_attention_weight_max": 96.33015441894531,
      "activations/layer4_attention_weight_min": -95.61885833740234,
      "activations/layer5_attention_weight_max": 112.89886474609375,
      "activations/layer5_attention_weight_min": -98.14067840576172,
      "activations/layer6_attention_weight_max": 59.20591354370117,
      "activations/layer6_attention_weight_min": -52.06718063354492,
      "activations/layer7_attention_weight_max": 61.11436462402344,
      "activations/layer7_attention_weight_min": -58.58115768432617,
      "activations/layer8_attention_weight_max": 40.12578582763672,
      "activations/layer8_attention_weight_min": -40.031253814697266,
      "activations/layer9_attention_weight_max": 47.246212005615234,
      "activations/layer9_attention_weight_min": -46.54371643066406,
      "epoch": 3.98,
      "learning_rate": 0.00012559621212121212,
      "loss": 2.8736,
      "step": 68500
    },
    {
      "activations/layer0_attention_weight_max": 13.596504211425781,
      "activations/layer0_attention_weight_min": -14.882672309875488,
      "activations/layer10_attention_weight_max": 39.10546112060547,
      "activations/layer10_attention_weight_min": -33.439369201660156,
      "activations/layer11_attention_weight_max": 30.39303970336914,
      "activations/layer11_attention_weight_min": -26.141223907470703,
      "activations/layer12_attention_weight_max": 25.238204956054688,
      "activations/layer12_attention_weight_min": -24.83652687072754,
      "activations/layer13_attention_weight_max": 48.52864456176758,
      "activations/layer13_attention_weight_min": -34.28350067138672,
      "activations/layer14_attention_weight_max": 48.64487075805664,
      "activations/layer14_attention_weight_min": -35.01658630371094,
      "activations/layer15_attention_weight_max": 43.447792053222656,
      "activations/layer15_attention_weight_min": -31.486074447631836,
      "activations/layer16_attention_weight_max": 37.20288848876953,
      "activations/layer16_attention_weight_min": -26.52692413330078,
      "activations/layer17_attention_weight_max": 63.63120651245117,
      "activations/layer17_attention_weight_min": -36.596744537353516,
      "activations/layer18_attention_weight_max": 52.55347442626953,
      "activations/layer18_attention_weight_min": -32.01565170288086,
      "activations/layer19_attention_weight_max": 22.877683639526367,
      "activations/layer19_attention_weight_min": -18.105257034301758,
      "activations/layer1_attention_weight_max": 17.7972354888916,
      "activations/layer1_attention_weight_min": -16.67372703552246,
      "activations/layer20_attention_weight_max": 25.670743942260742,
      "activations/layer20_attention_weight_min": -20.979612350463867,
      "activations/layer21_attention_weight_max": 48.225181579589844,
      "activations/layer21_attention_weight_min": -24.90909194946289,
      "activations/layer22_attention_weight_max": 37.785160064697266,
      "activations/layer22_attention_weight_min": -24.64547348022461,
      "activations/layer23_attention_weight_max": 27.667285919189453,
      "activations/layer23_attention_weight_min": -15.640701293945312,
      "activations/layer2_attention_weight_max": 30.11591339111328,
      "activations/layer2_attention_weight_min": -28.492971420288086,
      "activations/layer3_attention_weight_max": 74.97502899169922,
      "activations/layer3_attention_weight_min": -80.64677429199219,
      "activations/layer4_attention_weight_max": 92.57518005371094,
      "activations/layer4_attention_weight_min": -90.56010437011719,
      "activations/layer5_attention_weight_max": 111.53955078125,
      "activations/layer5_attention_weight_min": -98.17362976074219,
      "activations/layer6_attention_weight_max": 60.1292610168457,
      "activations/layer6_attention_weight_min": -51.3297233581543,
      "activations/layer7_attention_weight_max": 68.73615264892578,
      "activations/layer7_attention_weight_min": -59.27765655517578,
      "activations/layer8_attention_weight_max": 37.75557327270508,
      "activations/layer8_attention_weight_min": -38.6870002746582,
      "activations/layer9_attention_weight_max": 54.599647521972656,
      "activations/layer9_attention_weight_min": -50.72939682006836,
      "epoch": 3.98,
      "learning_rate": 0.0001255772727272727,
      "loss": 2.8904,
      "step": 68550
    },
    {
      "activations/layer0_attention_weight_max": 14.1464204788208,
      "activations/layer0_attention_weight_min": -15.301841735839844,
      "activations/layer10_attention_weight_max": 41.956268310546875,
      "activations/layer10_attention_weight_min": -35.1396369934082,
      "activations/layer11_attention_weight_max": 32.18241500854492,
      "activations/layer11_attention_weight_min": -26.227140426635742,
      "activations/layer12_attention_weight_max": 24.72744369506836,
      "activations/layer12_attention_weight_min": -25.057376861572266,
      "activations/layer13_attention_weight_max": 50.95120620727539,
      "activations/layer13_attention_weight_min": -41.21620559692383,
      "activations/layer14_attention_weight_max": 44.71200942993164,
      "activations/layer14_attention_weight_min": -30.834171295166016,
      "activations/layer15_attention_weight_max": 44.83389663696289,
      "activations/layer15_attention_weight_min": -34.23005294799805,
      "activations/layer16_attention_weight_max": 40.52772903442383,
      "activations/layer16_attention_weight_min": -26.638626098632812,
      "activations/layer17_attention_weight_max": 61.82263946533203,
      "activations/layer17_attention_weight_min": -39.42543411254883,
      "activations/layer18_attention_weight_max": 54.144466400146484,
      "activations/layer18_attention_weight_min": -34.77387619018555,
      "activations/layer19_attention_weight_max": 26.8319149017334,
      "activations/layer19_attention_weight_min": -21.280488967895508,
      "activations/layer1_attention_weight_max": 18.122478485107422,
      "activations/layer1_attention_weight_min": -14.157955169677734,
      "activations/layer20_attention_weight_max": 25.606279373168945,
      "activations/layer20_attention_weight_min": -19.541215896606445,
      "activations/layer21_attention_weight_max": 50.71451950073242,
      "activations/layer21_attention_weight_min": -31.23570442199707,
      "activations/layer22_attention_weight_max": 35.62149429321289,
      "activations/layer22_attention_weight_min": -25.784992218017578,
      "activations/layer23_attention_weight_max": 27.908245086669922,
      "activations/layer23_attention_weight_min": -17.299270629882812,
      "activations/layer2_attention_weight_max": 29.090160369873047,
      "activations/layer2_attention_weight_min": -27.1204891204834,
      "activations/layer3_attention_weight_max": 72.91230010986328,
      "activations/layer3_attention_weight_min": -76.56073760986328,
      "activations/layer4_attention_weight_max": 96.14794158935547,
      "activations/layer4_attention_weight_min": -89.5888900756836,
      "activations/layer5_attention_weight_max": 109.99526977539062,
      "activations/layer5_attention_weight_min": -104.12496185302734,
      "activations/layer6_attention_weight_max": 56.563087463378906,
      "activations/layer6_attention_weight_min": -53.047828674316406,
      "activations/layer7_attention_weight_max": 62.24773025512695,
      "activations/layer7_attention_weight_min": -60.45123291015625,
      "activations/layer8_attention_weight_max": 40.29163360595703,
      "activations/layer8_attention_weight_min": -39.2719612121582,
      "activations/layer9_attention_weight_max": 49.781707763671875,
      "activations/layer9_attention_weight_min": -48.66130447387695,
      "epoch": 3.99,
      "learning_rate": 0.00012555833333333333,
      "loss": 2.8839,
      "step": 68600
    },
    {
      "activations/layer0_attention_weight_max": 14.564469337463379,
      "activations/layer0_attention_weight_min": -15.35603141784668,
      "activations/layer10_attention_weight_max": 41.26518249511719,
      "activations/layer10_attention_weight_min": -34.37679672241211,
      "activations/layer11_attention_weight_max": 33.533164978027344,
      "activations/layer11_attention_weight_min": -26.242145538330078,
      "activations/layer12_attention_weight_max": 22.964309692382812,
      "activations/layer12_attention_weight_min": -23.907514572143555,
      "activations/layer13_attention_weight_max": 55.523990631103516,
      "activations/layer13_attention_weight_min": -38.11943435668945,
      "activations/layer14_attention_weight_max": 51.46394729614258,
      "activations/layer14_attention_weight_min": -32.18557357788086,
      "activations/layer15_attention_weight_max": 45.71739959716797,
      "activations/layer15_attention_weight_min": -31.715227127075195,
      "activations/layer16_attention_weight_max": 36.10524368286133,
      "activations/layer16_attention_weight_min": -27.35694694519043,
      "activations/layer17_attention_weight_max": 57.27017593383789,
      "activations/layer17_attention_weight_min": -36.836997985839844,
      "activations/layer18_attention_weight_max": 49.212677001953125,
      "activations/layer18_attention_weight_min": -33.71867752075195,
      "activations/layer19_attention_weight_max": 23.095104217529297,
      "activations/layer19_attention_weight_min": -17.40270233154297,
      "activations/layer1_attention_weight_max": 16.807369232177734,
      "activations/layer1_attention_weight_min": -16.15883445739746,
      "activations/layer20_attention_weight_max": 22.040903091430664,
      "activations/layer20_attention_weight_min": -23.109172821044922,
      "activations/layer21_attention_weight_max": 47.109619140625,
      "activations/layer21_attention_weight_min": -30.541276931762695,
      "activations/layer22_attention_weight_max": 34.267723083496094,
      "activations/layer22_attention_weight_min": -28.4599552154541,
      "activations/layer23_attention_weight_max": 25.959814071655273,
      "activations/layer23_attention_weight_min": -17.287620544433594,
      "activations/layer2_attention_weight_max": 29.994281768798828,
      "activations/layer2_attention_weight_min": -27.039562225341797,
      "activations/layer3_attention_weight_max": 73.45248413085938,
      "activations/layer3_attention_weight_min": -78.67835235595703,
      "activations/layer4_attention_weight_max": 96.28466796875,
      "activations/layer4_attention_weight_min": -92.17677307128906,
      "activations/layer5_attention_weight_max": 107.59880828857422,
      "activations/layer5_attention_weight_min": -98.61125946044922,
      "activations/layer6_attention_weight_max": 58.370521545410156,
      "activations/layer6_attention_weight_min": -51.31757736206055,
      "activations/layer7_attention_weight_max": 62.87739944458008,
      "activations/layer7_attention_weight_min": -58.33817672729492,
      "activations/layer8_attention_weight_max": 39.878334045410156,
      "activations/layer8_attention_weight_min": -37.4095573425293,
      "activations/layer9_attention_weight_max": 53.354156494140625,
      "activations/layer9_attention_weight_min": -46.39348602294922,
      "epoch": 3.99,
      "learning_rate": 0.00012553939393939394,
      "loss": 2.8818,
      "step": 68650
    },
    {
      "activations/layer0_attention_weight_max": 14.980917930603027,
      "activations/layer0_attention_weight_min": -15.238859176635742,
      "activations/layer10_attention_weight_max": 41.2112922668457,
      "activations/layer10_attention_weight_min": -33.398704528808594,
      "activations/layer11_attention_weight_max": 32.72280502319336,
      "activations/layer11_attention_weight_min": -27.3485107421875,
      "activations/layer12_attention_weight_max": 27.16802406311035,
      "activations/layer12_attention_weight_min": -26.071077346801758,
      "activations/layer13_attention_weight_max": 54.04605484008789,
      "activations/layer13_attention_weight_min": -39.56330871582031,
      "activations/layer14_attention_weight_max": 46.61116409301758,
      "activations/layer14_attention_weight_min": -34.68875503540039,
      "activations/layer15_attention_weight_max": 45.04558563232422,
      "activations/layer15_attention_weight_min": -33.62623596191406,
      "activations/layer16_attention_weight_max": 35.602760314941406,
      "activations/layer16_attention_weight_min": -26.167552947998047,
      "activations/layer17_attention_weight_max": 56.30963134765625,
      "activations/layer17_attention_weight_min": -37.69297409057617,
      "activations/layer18_attention_weight_max": 51.317928314208984,
      "activations/layer18_attention_weight_min": -33.14519500732422,
      "activations/layer19_attention_weight_max": 21.436140060424805,
      "activations/layer19_attention_weight_min": -18.252777099609375,
      "activations/layer1_attention_weight_max": 17.721149444580078,
      "activations/layer1_attention_weight_min": -16.121076583862305,
      "activations/layer20_attention_weight_max": 27.049453735351562,
      "activations/layer20_attention_weight_min": -20.131397247314453,
      "activations/layer21_attention_weight_max": 39.50261306762695,
      "activations/layer21_attention_weight_min": -26.462724685668945,
      "activations/layer22_attention_weight_max": 34.69110870361328,
      "activations/layer22_attention_weight_min": -24.239604949951172,
      "activations/layer23_attention_weight_max": 27.20187759399414,
      "activations/layer23_attention_weight_min": -16.674488067626953,
      "activations/layer2_attention_weight_max": 28.484586715698242,
      "activations/layer2_attention_weight_min": -27.73072052001953,
      "activations/layer3_attention_weight_max": 75.32678985595703,
      "activations/layer3_attention_weight_min": -78.3415756225586,
      "activations/layer4_attention_weight_max": 98.44725799560547,
      "activations/layer4_attention_weight_min": -94.80123901367188,
      "activations/layer5_attention_weight_max": 109.52803039550781,
      "activations/layer5_attention_weight_min": -103.8623046875,
      "activations/layer6_attention_weight_max": 57.610328674316406,
      "activations/layer6_attention_weight_min": -54.528499603271484,
      "activations/layer7_attention_weight_max": 63.564544677734375,
      "activations/layer7_attention_weight_min": -63.78224182128906,
      "activations/layer8_attention_weight_max": 40.33072280883789,
      "activations/layer8_attention_weight_min": -39.70726776123047,
      "activations/layer9_attention_weight_max": 53.21369552612305,
      "activations/layer9_attention_weight_min": -45.35138702392578,
      "epoch": 3.99,
      "learning_rate": 0.00012552045454545453,
      "loss": 2.881,
      "step": 68700
    },
    {
      "activations/layer0_attention_weight_max": 14.780991554260254,
      "activations/layer0_attention_weight_min": -15.460410118103027,
      "activations/layer10_attention_weight_max": 39.40083694458008,
      "activations/layer10_attention_weight_min": -36.57395553588867,
      "activations/layer11_attention_weight_max": 33.557464599609375,
      "activations/layer11_attention_weight_min": -27.844295501708984,
      "activations/layer12_attention_weight_max": 27.43924903869629,
      "activations/layer12_attention_weight_min": -36.126197814941406,
      "activations/layer13_attention_weight_max": 59.21168899536133,
      "activations/layer13_attention_weight_min": -37.52105712890625,
      "activations/layer14_attention_weight_max": 62.43305587768555,
      "activations/layer14_attention_weight_min": -34.46854019165039,
      "activations/layer15_attention_weight_max": 49.04160690307617,
      "activations/layer15_attention_weight_min": -33.35615921020508,
      "activations/layer16_attention_weight_max": 39.42030334472656,
      "activations/layer16_attention_weight_min": -26.44023323059082,
      "activations/layer17_attention_weight_max": 65.61490631103516,
      "activations/layer17_attention_weight_min": -39.46417236328125,
      "activations/layer18_attention_weight_max": 60.20766067504883,
      "activations/layer18_attention_weight_min": -36.777000427246094,
      "activations/layer19_attention_weight_max": 26.340497970581055,
      "activations/layer19_attention_weight_min": -20.223445892333984,
      "activations/layer1_attention_weight_max": 17.958982467651367,
      "activations/layer1_attention_weight_min": -18.12896728515625,
      "activations/layer20_attention_weight_max": 25.042734146118164,
      "activations/layer20_attention_weight_min": -19.92082405090332,
      "activations/layer21_attention_weight_max": 55.956878662109375,
      "activations/layer21_attention_weight_min": -30.146635055541992,
      "activations/layer22_attention_weight_max": 41.73038864135742,
      "activations/layer22_attention_weight_min": -24.985689163208008,
      "activations/layer23_attention_weight_max": 33.25624465942383,
      "activations/layer23_attention_weight_min": -19.06881332397461,
      "activations/layer2_attention_weight_max": 32.92937469482422,
      "activations/layer2_attention_weight_min": -29.517478942871094,
      "activations/layer3_attention_weight_max": 80.30001831054688,
      "activations/layer3_attention_weight_min": -82.69523620605469,
      "activations/layer4_attention_weight_max": 105.40107727050781,
      "activations/layer4_attention_weight_min": -101.24645233154297,
      "activations/layer5_attention_weight_max": 118.92031860351562,
      "activations/layer5_attention_weight_min": -99.46875,
      "activations/layer6_attention_weight_max": 60.379234313964844,
      "activations/layer6_attention_weight_min": -51.73657989501953,
      "activations/layer7_attention_weight_max": 73.84761047363281,
      "activations/layer7_attention_weight_min": -64.32301330566406,
      "activations/layer8_attention_weight_max": 44.9815559387207,
      "activations/layer8_attention_weight_min": -40.230777740478516,
      "activations/layer9_attention_weight_max": 69.97078704833984,
      "activations/layer9_attention_weight_min": -58.54301834106445,
      "epoch": 3.99,
      "learning_rate": 0.00012550151515151514,
      "loss": 2.8844,
      "step": 68750
    },
    {
      "activations/layer0_attention_weight_max": 14.03906536102295,
      "activations/layer0_attention_weight_min": -15.512681007385254,
      "activations/layer10_attention_weight_max": 37.29860305786133,
      "activations/layer10_attention_weight_min": -33.09046173095703,
      "activations/layer11_attention_weight_max": 30.87981414794922,
      "activations/layer11_attention_weight_min": -26.258962631225586,
      "activations/layer12_attention_weight_max": 23.058425903320312,
      "activations/layer12_attention_weight_min": -24.325946807861328,
      "activations/layer13_attention_weight_max": 51.51173400878906,
      "activations/layer13_attention_weight_min": -37.35099411010742,
      "activations/layer14_attention_weight_max": 51.122520446777344,
      "activations/layer14_attention_weight_min": -32.52162551879883,
      "activations/layer15_attention_weight_max": 44.4216194152832,
      "activations/layer15_attention_weight_min": -32.73431396484375,
      "activations/layer16_attention_weight_max": 37.41898727416992,
      "activations/layer16_attention_weight_min": -26.942222595214844,
      "activations/layer17_attention_weight_max": 60.75890350341797,
      "activations/layer17_attention_weight_min": -36.0551643371582,
      "activations/layer18_attention_weight_max": 50.172813415527344,
      "activations/layer18_attention_weight_min": -31.064828872680664,
      "activations/layer19_attention_weight_max": 24.365367889404297,
      "activations/layer19_attention_weight_min": -18.053363800048828,
      "activations/layer1_attention_weight_max": 17.497604370117188,
      "activations/layer1_attention_weight_min": -14.74767780303955,
      "activations/layer20_attention_weight_max": 22.89228630065918,
      "activations/layer20_attention_weight_min": -23.915407180786133,
      "activations/layer21_attention_weight_max": 40.5866584777832,
      "activations/layer21_attention_weight_min": -26.691368103027344,
      "activations/layer22_attention_weight_max": 37.497032165527344,
      "activations/layer22_attention_weight_min": -23.663450241088867,
      "activations/layer23_attention_weight_max": 27.530487060546875,
      "activations/layer23_attention_weight_min": -17.89788818359375,
      "activations/layer2_attention_weight_max": 28.161155700683594,
      "activations/layer2_attention_weight_min": -26.346153259277344,
      "activations/layer3_attention_weight_max": 75.3464126586914,
      "activations/layer3_attention_weight_min": -80.25605010986328,
      "activations/layer4_attention_weight_max": 102.3777084350586,
      "activations/layer4_attention_weight_min": -95.00611114501953,
      "activations/layer5_attention_weight_max": 114.96648406982422,
      "activations/layer5_attention_weight_min": -98.02853393554688,
      "activations/layer6_attention_weight_max": 58.77399826049805,
      "activations/layer6_attention_weight_min": -53.98135757446289,
      "activations/layer7_attention_weight_max": 63.149024963378906,
      "activations/layer7_attention_weight_min": -58.231903076171875,
      "activations/layer8_attention_weight_max": 38.541969299316406,
      "activations/layer8_attention_weight_min": -35.43215560913086,
      "activations/layer9_attention_weight_max": 51.596004486083984,
      "activations/layer9_attention_weight_min": -46.47796630859375,
      "epoch": 4.0,
      "learning_rate": 0.00012548257575757576,
      "loss": 2.9027,
      "step": 68800
    },
    {
      "activations/layer0_attention_weight_max": 15.91305923461914,
      "activations/layer0_attention_weight_min": -15.090670585632324,
      "activations/layer10_attention_weight_max": 43.34226989746094,
      "activations/layer10_attention_weight_min": -36.20589828491211,
      "activations/layer11_attention_weight_max": 37.4800910949707,
      "activations/layer11_attention_weight_min": -27.229061126708984,
      "activations/layer12_attention_weight_max": 33.58237838745117,
      "activations/layer12_attention_weight_min": -26.10419464111328,
      "activations/layer13_attention_weight_max": 55.01923370361328,
      "activations/layer13_attention_weight_min": -41.221126556396484,
      "activations/layer14_attention_weight_max": 60.119693756103516,
      "activations/layer14_attention_weight_min": -36.10198211669922,
      "activations/layer15_attention_weight_max": 46.08151626586914,
      "activations/layer15_attention_weight_min": -33.276611328125,
      "activations/layer16_attention_weight_max": 39.02032470703125,
      "activations/layer16_attention_weight_min": -27.079654693603516,
      "activations/layer17_attention_weight_max": 68.35400390625,
      "activations/layer17_attention_weight_min": -38.90180969238281,
      "activations/layer18_attention_weight_max": 56.17101287841797,
      "activations/layer18_attention_weight_min": -33.16969680786133,
      "activations/layer19_attention_weight_max": 25.485103607177734,
      "activations/layer19_attention_weight_min": -18.67206382751465,
      "activations/layer1_attention_weight_max": 17.509057998657227,
      "activations/layer1_attention_weight_min": -15.441500663757324,
      "activations/layer20_attention_weight_max": 23.06838607788086,
      "activations/layer20_attention_weight_min": -22.781766891479492,
      "activations/layer21_attention_weight_max": 51.781394958496094,
      "activations/layer21_attention_weight_min": -34.621246337890625,
      "activations/layer22_attention_weight_max": 40.12384033203125,
      "activations/layer22_attention_weight_min": -22.816125869750977,
      "activations/layer23_attention_weight_max": 30.559345245361328,
      "activations/layer23_attention_weight_min": -16.453269958496094,
      "activations/layer2_attention_weight_max": 28.769458770751953,
      "activations/layer2_attention_weight_min": -27.845359802246094,
      "activations/layer3_attention_weight_max": 77.174072265625,
      "activations/layer3_attention_weight_min": -77.37909698486328,
      "activations/layer4_attention_weight_max": 99.90674591064453,
      "activations/layer4_attention_weight_min": -92.36346435546875,
      "activations/layer5_attention_weight_max": 111.43714904785156,
      "activations/layer5_attention_weight_min": -95.8956527709961,
      "activations/layer6_attention_weight_max": 57.73728561401367,
      "activations/layer6_attention_weight_min": -50.807701110839844,
      "activations/layer7_attention_weight_max": 63.41401290893555,
      "activations/layer7_attention_weight_min": -58.71162796020508,
      "activations/layer8_attention_weight_max": 41.514381408691406,
      "activations/layer8_attention_weight_min": -37.425392150878906,
      "activations/layer9_attention_weight_max": 52.04246139526367,
      "activations/layer9_attention_weight_min": -45.252193450927734,
      "epoch": 4.0,
      "learning_rate": 0.00012546363636363635,
      "loss": 2.9355,
      "step": 68850
    },
    {
      "activations/layer0_attention_weight_max": 15.492258071899414,
      "activations/layer0_attention_weight_min": -15.044400215148926,
      "activations/layer10_attention_weight_max": 36.91931915283203,
      "activations/layer10_attention_weight_min": -33.5761833190918,
      "activations/layer11_attention_weight_max": 32.00989532470703,
      "activations/layer11_attention_weight_min": -24.875530242919922,
      "activations/layer12_attention_weight_max": 21.086811065673828,
      "activations/layer12_attention_weight_min": -27.636859893798828,
      "activations/layer13_attention_weight_max": 48.431461334228516,
      "activations/layer13_attention_weight_min": -34.55486297607422,
      "activations/layer14_attention_weight_max": 42.122283935546875,
      "activations/layer14_attention_weight_min": -29.30423927307129,
      "activations/layer15_attention_weight_max": 36.50030517578125,
      "activations/layer15_attention_weight_min": -30.172775268554688,
      "activations/layer16_attention_weight_max": 33.45737838745117,
      "activations/layer16_attention_weight_min": -26.14798355102539,
      "activations/layer17_attention_weight_max": 55.51652908325195,
      "activations/layer17_attention_weight_min": -36.446590423583984,
      "activations/layer18_attention_weight_max": 50.7629508972168,
      "activations/layer18_attention_weight_min": -31.048051834106445,
      "activations/layer19_attention_weight_max": 24.130369186401367,
      "activations/layer19_attention_weight_min": -18.796751022338867,
      "activations/layer1_attention_weight_max": 17.327939987182617,
      "activations/layer1_attention_weight_min": -15.565558433532715,
      "activations/layer20_attention_weight_max": 23.752897262573242,
      "activations/layer20_attention_weight_min": -23.870899200439453,
      "activations/layer21_attention_weight_max": 37.13669204711914,
      "activations/layer21_attention_weight_min": -24.68120574951172,
      "activations/layer22_attention_weight_max": 31.48020362854004,
      "activations/layer22_attention_weight_min": -23.39359474182129,
      "activations/layer23_attention_weight_max": 25.77349090576172,
      "activations/layer23_attention_weight_min": -17.667808532714844,
      "activations/layer2_attention_weight_max": 30.54253387451172,
      "activations/layer2_attention_weight_min": -27.79908561706543,
      "activations/layer3_attention_weight_max": 77.3701171875,
      "activations/layer3_attention_weight_min": -80.34052276611328,
      "activations/layer4_attention_weight_max": 99.59974670410156,
      "activations/layer4_attention_weight_min": -91.57170867919922,
      "activations/layer5_attention_weight_max": 107.21487426757812,
      "activations/layer5_attention_weight_min": -96.09957122802734,
      "activations/layer6_attention_weight_max": 54.702022552490234,
      "activations/layer6_attention_weight_min": -52.46123123168945,
      "activations/layer7_attention_weight_max": 63.90727615356445,
      "activations/layer7_attention_weight_min": -60.02067184448242,
      "activations/layer8_attention_weight_max": 42.0867919921875,
      "activations/layer8_attention_weight_min": -39.35676956176758,
      "activations/layer9_attention_weight_max": 54.18024444580078,
      "activations/layer9_attention_weight_min": -48.81389236450195,
      "epoch": 4.0,
      "learning_rate": 0.00012544469696969696,
      "loss": 2.8753,
      "step": 68900
    },
    {
      "activations/layer0_attention_weight_max": 14.06151008605957,
      "activations/layer0_attention_weight_min": -14.960577011108398,
      "activations/layer10_attention_weight_max": 45.45830154418945,
      "activations/layer10_attention_weight_min": -36.48334503173828,
      "activations/layer11_attention_weight_max": 40.83245849609375,
      "activations/layer11_attention_weight_min": -27.55681610107422,
      "activations/layer12_attention_weight_max": 26.489835739135742,
      "activations/layer12_attention_weight_min": -22.28107261657715,
      "activations/layer13_attention_weight_max": 64.38374328613281,
      "activations/layer13_attention_weight_min": -39.42534637451172,
      "activations/layer14_attention_weight_max": 44.53404235839844,
      "activations/layer14_attention_weight_min": -29.754671096801758,
      "activations/layer15_attention_weight_max": 44.14951705932617,
      "activations/layer15_attention_weight_min": -30.77629852294922,
      "activations/layer16_attention_weight_max": 37.94902420043945,
      "activations/layer16_attention_weight_min": -26.29250144958496,
      "activations/layer17_attention_weight_max": 69.4472427368164,
      "activations/layer17_attention_weight_min": -39.26013946533203,
      "activations/layer18_attention_weight_max": 58.500038146972656,
      "activations/layer18_attention_weight_min": -38.886653900146484,
      "activations/layer19_attention_weight_max": 22.592191696166992,
      "activations/layer19_attention_weight_min": -20.493825912475586,
      "activations/layer1_attention_weight_max": 17.268280029296875,
      "activations/layer1_attention_weight_min": -14.557279586791992,
      "activations/layer20_attention_weight_max": 25.46251678466797,
      "activations/layer20_attention_weight_min": -19.55416488647461,
      "activations/layer21_attention_weight_max": 48.80530548095703,
      "activations/layer21_attention_weight_min": -28.839500427246094,
      "activations/layer22_attention_weight_max": 37.39999771118164,
      "activations/layer22_attention_weight_min": -22.615299224853516,
      "activations/layer23_attention_weight_max": 32.681251525878906,
      "activations/layer23_attention_weight_min": -18.300731658935547,
      "activations/layer2_attention_weight_max": 29.677249908447266,
      "activations/layer2_attention_weight_min": -29.481834411621094,
      "activations/layer3_attention_weight_max": 75.70769500732422,
      "activations/layer3_attention_weight_min": -78.26100158691406,
      "activations/layer4_attention_weight_max": 99.6843032836914,
      "activations/layer4_attention_weight_min": -95.55500030517578,
      "activations/layer5_attention_weight_max": 114.34813690185547,
      "activations/layer5_attention_weight_min": -102.1075439453125,
      "activations/layer6_attention_weight_max": 59.83087921142578,
      "activations/layer6_attention_weight_min": -53.66809844970703,
      "activations/layer7_attention_weight_max": 66.77854919433594,
      "activations/layer7_attention_weight_min": -64.8676528930664,
      "activations/layer8_attention_weight_max": 42.304664611816406,
      "activations/layer8_attention_weight_min": -43.19240951538086,
      "activations/layer9_attention_weight_max": 54.34511184692383,
      "activations/layer9_attention_weight_min": -46.050315856933594,
      "epoch": 4.01,
      "learning_rate": 0.00012542575757575755,
      "loss": 2.8676,
      "step": 68950
    },
    {
      "activations/layer0_attention_weight_max": 14.717070579528809,
      "activations/layer0_attention_weight_min": -14.866945266723633,
      "activations/layer10_attention_weight_max": 39.2966194152832,
      "activations/layer10_attention_weight_min": -33.1650505065918,
      "activations/layer11_attention_weight_max": 35.490509033203125,
      "activations/layer11_attention_weight_min": -25.997695922851562,
      "activations/layer12_attention_weight_max": 24.014320373535156,
      "activations/layer12_attention_weight_min": -25.778162002563477,
      "activations/layer13_attention_weight_max": 58.311866760253906,
      "activations/layer13_attention_weight_min": -41.18778991699219,
      "activations/layer14_attention_weight_max": 47.874305725097656,
      "activations/layer14_attention_weight_min": -34.0417594909668,
      "activations/layer15_attention_weight_max": 51.71928024291992,
      "activations/layer15_attention_weight_min": -31.838834762573242,
      "activations/layer16_attention_weight_max": 36.31279373168945,
      "activations/layer16_attention_weight_min": -28.98318099975586,
      "activations/layer17_attention_weight_max": 66.57096862792969,
      "activations/layer17_attention_weight_min": -36.790611267089844,
      "activations/layer18_attention_weight_max": 50.106475830078125,
      "activations/layer18_attention_weight_min": -34.36571502685547,
      "activations/layer19_attention_weight_max": 25.450855255126953,
      "activations/layer19_attention_weight_min": -18.094364166259766,
      "activations/layer1_attention_weight_max": 19.551958084106445,
      "activations/layer1_attention_weight_min": -16.760025024414062,
      "activations/layer20_attention_weight_max": 24.211454391479492,
      "activations/layer20_attention_weight_min": -20.933452606201172,
      "activations/layer21_attention_weight_max": 43.7942008972168,
      "activations/layer21_attention_weight_min": -31.504873275756836,
      "activations/layer22_attention_weight_max": 35.70072555541992,
      "activations/layer22_attention_weight_min": -27.63146209716797,
      "activations/layer23_attention_weight_max": 28.40399742126465,
      "activations/layer23_attention_weight_min": -20.080860137939453,
      "activations/layer2_attention_weight_max": 30.964847564697266,
      "activations/layer2_attention_weight_min": -29.81763458251953,
      "activations/layer3_attention_weight_max": 78.09765625,
      "activations/layer3_attention_weight_min": -81.3775405883789,
      "activations/layer4_attention_weight_max": 99.81983947753906,
      "activations/layer4_attention_weight_min": -96.36788940429688,
      "activations/layer5_attention_weight_max": 113.47430419921875,
      "activations/layer5_attention_weight_min": -99.82662200927734,
      "activations/layer6_attention_weight_max": 59.46357727050781,
      "activations/layer6_attention_weight_min": -57.473121643066406,
      "activations/layer7_attention_weight_max": 63.530948638916016,
      "activations/layer7_attention_weight_min": -58.69619369506836,
      "activations/layer8_attention_weight_max": 39.43364715576172,
      "activations/layer8_attention_weight_min": -39.12080001831055,
      "activations/layer9_attention_weight_max": 49.239044189453125,
      "activations/layer9_attention_weight_min": -43.70273971557617,
      "epoch": 4.01,
      "learning_rate": 0.00012540681818181817,
      "loss": 2.8687,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_loss": 2.826171875,
      "eval_runtime": 8.5423,
      "eval_samples_per_second": 502.677,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_openwebtext_loss": 2.826171875,
      "eval_openwebtext_ppl": 16.88071548801052,
      "eval_openwebtext_runtime": 8.5423,
      "eval_openwebtext_samples_per_second": 502.677,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_wikitext_loss": 3.068359375,
      "eval_wikitext_ppl": 21.506589466713315,
      "eval_wikitext_runtime": 1.9221,
      "eval_wikitext_samples_per_second": 237.242,
      "step": 69000
    },
    {
      "epoch": 4.01,
      "eval_lambada_loss": 2.81640625,
      "eval_lambada_ppl": 16.716667072215767,
      "eval_lambada_runtime": 9.5355,
      "eval_lambada_samples_per_second": 510.621,
      "step": 69000
    },
    {
      "activations/layer0_attention_weight_max": 14.05532455444336,
      "activations/layer0_attention_weight_min": -15.146042823791504,
      "activations/layer10_attention_weight_max": 37.48057174682617,
      "activations/layer10_attention_weight_min": -34.15861892700195,
      "activations/layer11_attention_weight_max": 31.98371124267578,
      "activations/layer11_attention_weight_min": -26.011991500854492,
      "activations/layer12_attention_weight_max": 24.72056770324707,
      "activations/layer12_attention_weight_min": -24.93716812133789,
      "activations/layer13_attention_weight_max": 48.424129486083984,
      "activations/layer13_attention_weight_min": -39.61949157714844,
      "activations/layer14_attention_weight_max": 40.46289825439453,
      "activations/layer14_attention_weight_min": -27.89557647705078,
      "activations/layer15_attention_weight_max": 41.98306655883789,
      "activations/layer15_attention_weight_min": -30.705810546875,
      "activations/layer16_attention_weight_max": 37.46767044067383,
      "activations/layer16_attention_weight_min": -26.22862434387207,
      "activations/layer17_attention_weight_max": 59.50657272338867,
      "activations/layer17_attention_weight_min": -35.8664665222168,
      "activations/layer18_attention_weight_max": 53.897422790527344,
      "activations/layer18_attention_weight_min": -28.767173767089844,
      "activations/layer19_attention_weight_max": 22.878225326538086,
      "activations/layer19_attention_weight_min": -18.73297691345215,
      "activations/layer1_attention_weight_max": 17.77471923828125,
      "activations/layer1_attention_weight_min": -14.129256248474121,
      "activations/layer20_attention_weight_max": 20.152992248535156,
      "activations/layer20_attention_weight_min": -22.577125549316406,
      "activations/layer21_attention_weight_max": 39.73318099975586,
      "activations/layer21_attention_weight_min": -29.69027328491211,
      "activations/layer22_attention_weight_max": 33.24907684326172,
      "activations/layer22_attention_weight_min": -24.072433471679688,
      "activations/layer23_attention_weight_max": 24.32659149169922,
      "activations/layer23_attention_weight_min": -17.63753318786621,
      "activations/layer2_attention_weight_max": 30.14920425415039,
      "activations/layer2_attention_weight_min": -28.898494720458984,
      "activations/layer3_attention_weight_max": 76.90887451171875,
      "activations/layer3_attention_weight_min": -79.9482421875,
      "activations/layer4_attention_weight_max": 102.3239517211914,
      "activations/layer4_attention_weight_min": -95.05945587158203,
      "activations/layer5_attention_weight_max": 121.08367919921875,
      "activations/layer5_attention_weight_min": -100.53666687011719,
      "activations/layer6_attention_weight_max": 62.28106689453125,
      "activations/layer6_attention_weight_min": -51.603736877441406,
      "activations/layer7_attention_weight_max": 67.81194305419922,
      "activations/layer7_attention_weight_min": -73.37359619140625,
      "activations/layer8_attention_weight_max": 44.29655456542969,
      "activations/layer8_attention_weight_min": -39.98550033569336,
      "activations/layer9_attention_weight_max": 57.51517868041992,
      "activations/layer9_attention_weight_min": -53.95622634887695,
      "epoch": 4.01,
      "learning_rate": 0.00012538787878787878,
      "loss": 2.8677,
      "step": 69050
    },
    {
      "activations/layer0_attention_weight_max": 15.221136093139648,
      "activations/layer0_attention_weight_min": -15.059449195861816,
      "activations/layer10_attention_weight_max": 40.808929443359375,
      "activations/layer10_attention_weight_min": -33.25627899169922,
      "activations/layer11_attention_weight_max": 34.594032287597656,
      "activations/layer11_attention_weight_min": -26.879478454589844,
      "activations/layer12_attention_weight_max": 31.50615119934082,
      "activations/layer12_attention_weight_min": -23.434690475463867,
      "activations/layer13_attention_weight_max": 52.24037551879883,
      "activations/layer13_attention_weight_min": -35.408302307128906,
      "activations/layer14_attention_weight_max": 46.254966735839844,
      "activations/layer14_attention_weight_min": -34.16108703613281,
      "activations/layer15_attention_weight_max": 44.64216995239258,
      "activations/layer15_attention_weight_min": -29.55463409423828,
      "activations/layer16_attention_weight_max": 36.95401382446289,
      "activations/layer16_attention_weight_min": -27.370901107788086,
      "activations/layer17_attention_weight_max": 61.49626541137695,
      "activations/layer17_attention_weight_min": -39.361385345458984,
      "activations/layer18_attention_weight_max": 56.184814453125,
      "activations/layer18_attention_weight_min": -34.87052917480469,
      "activations/layer19_attention_weight_max": 28.231307983398438,
      "activations/layer19_attention_weight_min": -19.6790828704834,
      "activations/layer1_attention_weight_max": 18.019929885864258,
      "activations/layer1_attention_weight_min": -15.901117324829102,
      "activations/layer20_attention_weight_max": 22.559274673461914,
      "activations/layer20_attention_weight_min": -20.13934326171875,
      "activations/layer21_attention_weight_max": 44.127716064453125,
      "activations/layer21_attention_weight_min": -28.37581443786621,
      "activations/layer22_attention_weight_max": 35.632606506347656,
      "activations/layer22_attention_weight_min": -25.419105529785156,
      "activations/layer23_attention_weight_max": 29.475738525390625,
      "activations/layer23_attention_weight_min": -18.109153747558594,
      "activations/layer2_attention_weight_max": 30.228878021240234,
      "activations/layer2_attention_weight_min": -28.05952262878418,
      "activations/layer3_attention_weight_max": 75.55708312988281,
      "activations/layer3_attention_weight_min": -79.81759643554688,
      "activations/layer4_attention_weight_max": 102.55476379394531,
      "activations/layer4_attention_weight_min": -96.64872741699219,
      "activations/layer5_attention_weight_max": 116.36080932617188,
      "activations/layer5_attention_weight_min": -103.7705307006836,
      "activations/layer6_attention_weight_max": 60.50675964355469,
      "activations/layer6_attention_weight_min": -52.48454666137695,
      "activations/layer7_attention_weight_max": 65.60438537597656,
      "activations/layer7_attention_weight_min": -65.14031982421875,
      "activations/layer8_attention_weight_max": 41.705814361572266,
      "activations/layer8_attention_weight_min": -39.136863708496094,
      "activations/layer9_attention_weight_max": 52.665443420410156,
      "activations/layer9_attention_weight_min": -49.24280548095703,
      "epoch": 4.02,
      "learning_rate": 0.00012536893939393937,
      "loss": 2.8842,
      "step": 69100
    },
    {
      "activations/layer0_attention_weight_max": 16.441932678222656,
      "activations/layer0_attention_weight_min": -15.394082069396973,
      "activations/layer10_attention_weight_max": 38.074256896972656,
      "activations/layer10_attention_weight_min": -33.55453872680664,
      "activations/layer11_attention_weight_max": 31.473073959350586,
      "activations/layer11_attention_weight_min": -27.399127960205078,
      "activations/layer12_attention_weight_max": 23.03590202331543,
      "activations/layer12_attention_weight_min": -26.695066452026367,
      "activations/layer13_attention_weight_max": 58.055763244628906,
      "activations/layer13_attention_weight_min": -38.003318786621094,
      "activations/layer14_attention_weight_max": 56.082969665527344,
      "activations/layer14_attention_weight_min": -36.73371505737305,
      "activations/layer15_attention_weight_max": 43.103515625,
      "activations/layer15_attention_weight_min": -30.788930892944336,
      "activations/layer16_attention_weight_max": 36.30659866333008,
      "activations/layer16_attention_weight_min": -25.60850715637207,
      "activations/layer17_attention_weight_max": 57.21696090698242,
      "activations/layer17_attention_weight_min": -37.669551849365234,
      "activations/layer18_attention_weight_max": 54.466495513916016,
      "activations/layer18_attention_weight_min": -31.965185165405273,
      "activations/layer19_attention_weight_max": 22.385408401489258,
      "activations/layer19_attention_weight_min": -18.228317260742188,
      "activations/layer1_attention_weight_max": 17.978124618530273,
      "activations/layer1_attention_weight_min": -16.044509887695312,
      "activations/layer20_attention_weight_max": 22.23421287536621,
      "activations/layer20_attention_weight_min": -20.81914710998535,
      "activations/layer21_attention_weight_max": 41.823753356933594,
      "activations/layer21_attention_weight_min": -24.4249267578125,
      "activations/layer22_attention_weight_max": 34.747528076171875,
      "activations/layer22_attention_weight_min": -22.06105613708496,
      "activations/layer23_attention_weight_max": 27.076019287109375,
      "activations/layer23_attention_weight_min": -16.982641220092773,
      "activations/layer2_attention_weight_max": 32.632144927978516,
      "activations/layer2_attention_weight_min": -32.077903747558594,
      "activations/layer3_attention_weight_max": 79.75829315185547,
      "activations/layer3_attention_weight_min": -83.67135620117188,
      "activations/layer4_attention_weight_max": 104.42256927490234,
      "activations/layer4_attention_weight_min": -99.04418182373047,
      "activations/layer5_attention_weight_max": 115.53082275390625,
      "activations/layer5_attention_weight_min": -101.13216400146484,
      "activations/layer6_attention_weight_max": 61.134559631347656,
      "activations/layer6_attention_weight_min": -52.04216384887695,
      "activations/layer7_attention_weight_max": 64.76905822753906,
      "activations/layer7_attention_weight_min": -61.59809112548828,
      "activations/layer8_attention_weight_max": 40.95088577270508,
      "activations/layer8_attention_weight_min": -40.28523254394531,
      "activations/layer9_attention_weight_max": 54.554622650146484,
      "activations/layer9_attention_weight_min": -52.5439338684082,
      "epoch": 4.02,
      "learning_rate": 0.00012534999999999998,
      "loss": 2.8771,
      "step": 69150
    },
    {
      "activations/layer0_attention_weight_max": 14.478011131286621,
      "activations/layer0_attention_weight_min": -15.725050926208496,
      "activations/layer10_attention_weight_max": 36.91071319580078,
      "activations/layer10_attention_weight_min": -31.95993423461914,
      "activations/layer11_attention_weight_max": 33.36289978027344,
      "activations/layer11_attention_weight_min": -28.25885581970215,
      "activations/layer12_attention_weight_max": 26.377368927001953,
      "activations/layer12_attention_weight_min": -23.933032989501953,
      "activations/layer13_attention_weight_max": 54.111534118652344,
      "activations/layer13_attention_weight_min": -38.941200256347656,
      "activations/layer14_attention_weight_max": 43.669376373291016,
      "activations/layer14_attention_weight_min": -31.139314651489258,
      "activations/layer15_attention_weight_max": 44.14863586425781,
      "activations/layer15_attention_weight_min": -31.210599899291992,
      "activations/layer16_attention_weight_max": 36.92966079711914,
      "activations/layer16_attention_weight_min": -25.912538528442383,
      "activations/layer17_attention_weight_max": 60.14376449584961,
      "activations/layer17_attention_weight_min": -39.835845947265625,
      "activations/layer18_attention_weight_max": 51.166160583496094,
      "activations/layer18_attention_weight_min": -32.555694580078125,
      "activations/layer19_attention_weight_max": 24.79422950744629,
      "activations/layer19_attention_weight_min": -18.631460189819336,
      "activations/layer1_attention_weight_max": 19.007314682006836,
      "activations/layer1_attention_weight_min": -14.063459396362305,
      "activations/layer20_attention_weight_max": 23.09937286376953,
      "activations/layer20_attention_weight_min": -17.96546173095703,
      "activations/layer21_attention_weight_max": 49.10437774658203,
      "activations/layer21_attention_weight_min": -28.065622329711914,
      "activations/layer22_attention_weight_max": 36.380916595458984,
      "activations/layer22_attention_weight_min": -22.04271697998047,
      "activations/layer23_attention_weight_max": 29.08712387084961,
      "activations/layer23_attention_weight_min": -16.337722778320312,
      "activations/layer2_attention_weight_max": 30.348770141601562,
      "activations/layer2_attention_weight_min": -27.9482479095459,
      "activations/layer3_attention_weight_max": 78.04495239257812,
      "activations/layer3_attention_weight_min": -82.60671997070312,
      "activations/layer4_attention_weight_max": 97.0797348022461,
      "activations/layer4_attention_weight_min": -93.60491943359375,
      "activations/layer5_attention_weight_max": 112.000244140625,
      "activations/layer5_attention_weight_min": -97.50179290771484,
      "activations/layer6_attention_weight_max": 58.198829650878906,
      "activations/layer6_attention_weight_min": -52.7715950012207,
      "activations/layer7_attention_weight_max": 64.08377075195312,
      "activations/layer7_attention_weight_min": -62.877254486083984,
      "activations/layer8_attention_weight_max": 42.0376091003418,
      "activations/layer8_attention_weight_min": -40.32604217529297,
      "activations/layer9_attention_weight_max": 51.28647994995117,
      "activations/layer9_attention_weight_min": -46.8294792175293,
      "epoch": 4.02,
      "learning_rate": 0.0001253310606060606,
      "loss": 2.8783,
      "step": 69200
    },
    {
      "activations/layer0_attention_weight_max": 15.20881462097168,
      "activations/layer0_attention_weight_min": -15.761852264404297,
      "activations/layer10_attention_weight_max": 37.1898078918457,
      "activations/layer10_attention_weight_min": -33.86544418334961,
      "activations/layer11_attention_weight_max": 30.35999870300293,
      "activations/layer11_attention_weight_min": -26.041122436523438,
      "activations/layer12_attention_weight_max": 23.84657096862793,
      "activations/layer12_attention_weight_min": -24.663129806518555,
      "activations/layer13_attention_weight_max": 53.30164337158203,
      "activations/layer13_attention_weight_min": -35.43195343017578,
      "activations/layer14_attention_weight_max": 49.63032150268555,
      "activations/layer14_attention_weight_min": -30.80055046081543,
      "activations/layer15_attention_weight_max": 42.65903091430664,
      "activations/layer15_attention_weight_min": -30.367685317993164,
      "activations/layer16_attention_weight_max": 33.947669982910156,
      "activations/layer16_attention_weight_min": -25.771591186523438,
      "activations/layer17_attention_weight_max": 60.38203430175781,
      "activations/layer17_attention_weight_min": -36.48396682739258,
      "activations/layer18_attention_weight_max": 54.621421813964844,
      "activations/layer18_attention_weight_min": -32.50540542602539,
      "activations/layer19_attention_weight_max": 23.602527618408203,
      "activations/layer19_attention_weight_min": -20.12845230102539,
      "activations/layer1_attention_weight_max": 17.673423767089844,
      "activations/layer1_attention_weight_min": -14.971487045288086,
      "activations/layer20_attention_weight_max": 23.4097900390625,
      "activations/layer20_attention_weight_min": -20.253002166748047,
      "activations/layer21_attention_weight_max": 43.19462585449219,
      "activations/layer21_attention_weight_min": -28.98676109313965,
      "activations/layer22_attention_weight_max": 37.259376525878906,
      "activations/layer22_attention_weight_min": -24.883752822875977,
      "activations/layer23_attention_weight_max": 32.88674545288086,
      "activations/layer23_attention_weight_min": -17.661876678466797,
      "activations/layer2_attention_weight_max": 30.100988388061523,
      "activations/layer2_attention_weight_min": -27.72427749633789,
      "activations/layer3_attention_weight_max": 79.91350555419922,
      "activations/layer3_attention_weight_min": -83.53199005126953,
      "activations/layer4_attention_weight_max": 103.60070037841797,
      "activations/layer4_attention_weight_min": -97.08676147460938,
      "activations/layer5_attention_weight_max": 118.19461059570312,
      "activations/layer5_attention_weight_min": -105.41290283203125,
      "activations/layer6_attention_weight_max": 61.864105224609375,
      "activations/layer6_attention_weight_min": -53.24242401123047,
      "activations/layer7_attention_weight_max": 63.084632873535156,
      "activations/layer7_attention_weight_min": -64.01506805419922,
      "activations/layer8_attention_weight_max": 40.90799331665039,
      "activations/layer8_attention_weight_min": -39.94617462158203,
      "activations/layer9_attention_weight_max": 49.56309127807617,
      "activations/layer9_attention_weight_min": -45.19603729248047,
      "epoch": 4.02,
      "learning_rate": 0.0001253121212121212,
      "loss": 2.8748,
      "step": 69250
    },
    {
      "activations/layer0_attention_weight_max": 14.74966812133789,
      "activations/layer0_attention_weight_min": -15.686281204223633,
      "activations/layer10_attention_weight_max": 36.452552795410156,
      "activations/layer10_attention_weight_min": -31.753976821899414,
      "activations/layer11_attention_weight_max": 29.93818473815918,
      "activations/layer11_attention_weight_min": -25.662694931030273,
      "activations/layer12_attention_weight_max": 28.222654342651367,
      "activations/layer12_attention_weight_min": -33.129783630371094,
      "activations/layer13_attention_weight_max": 40.7712287902832,
      "activations/layer13_attention_weight_min": -33.584922790527344,
      "activations/layer14_attention_weight_max": 38.84579849243164,
      "activations/layer14_attention_weight_min": -29.443464279174805,
      "activations/layer15_attention_weight_max": 39.98027420043945,
      "activations/layer15_attention_weight_min": -30.485952377319336,
      "activations/layer16_attention_weight_max": 36.49433135986328,
      "activations/layer16_attention_weight_min": -27.026264190673828,
      "activations/layer17_attention_weight_max": 56.69157028198242,
      "activations/layer17_attention_weight_min": -36.84315872192383,
      "activations/layer18_attention_weight_max": 47.32026290893555,
      "activations/layer18_attention_weight_min": -32.06896209716797,
      "activations/layer19_attention_weight_max": 19.800935745239258,
      "activations/layer19_attention_weight_min": -21.627689361572266,
      "activations/layer1_attention_weight_max": 17.235424041748047,
      "activations/layer1_attention_weight_min": -15.144049644470215,
      "activations/layer20_attention_weight_max": 21.546588897705078,
      "activations/layer20_attention_weight_min": -21.17648696899414,
      "activations/layer21_attention_weight_max": 41.422969818115234,
      "activations/layer21_attention_weight_min": -30.064603805541992,
      "activations/layer22_attention_weight_max": 32.56956100463867,
      "activations/layer22_attention_weight_min": -24.769306182861328,
      "activations/layer23_attention_weight_max": 28.296527862548828,
      "activations/layer23_attention_weight_min": -19.197307586669922,
      "activations/layer2_attention_weight_max": 29.64191436767578,
      "activations/layer2_attention_weight_min": -27.296802520751953,
      "activations/layer3_attention_weight_max": 78.82637023925781,
      "activations/layer3_attention_weight_min": -80.70057678222656,
      "activations/layer4_attention_weight_max": 107.9007568359375,
      "activations/layer4_attention_weight_min": -101.00662994384766,
      "activations/layer5_attention_weight_max": 115.5589599609375,
      "activations/layer5_attention_weight_min": -105.80996704101562,
      "activations/layer6_attention_weight_max": 58.21149826049805,
      "activations/layer6_attention_weight_min": -52.96290969848633,
      "activations/layer7_attention_weight_max": 63.428375244140625,
      "activations/layer7_attention_weight_min": -64.86051940917969,
      "activations/layer8_attention_weight_max": 38.17215347290039,
      "activations/layer8_attention_weight_min": -37.29030990600586,
      "activations/layer9_attention_weight_max": 52.80824661254883,
      "activations/layer9_attention_weight_min": -48.499263763427734,
      "epoch": 4.03,
      "learning_rate": 0.0001252931818181818,
      "loss": 2.8474,
      "step": 69300
    },
    {
      "activations/layer0_attention_weight_max": 14.545736312866211,
      "activations/layer0_attention_weight_min": -15.88127326965332,
      "activations/layer10_attention_weight_max": 39.2173957824707,
      "activations/layer10_attention_weight_min": -32.78392028808594,
      "activations/layer11_attention_weight_max": 35.47947692871094,
      "activations/layer11_attention_weight_min": -26.158340454101562,
      "activations/layer12_attention_weight_max": 25.602846145629883,
      "activations/layer12_attention_weight_min": -24.452468872070312,
      "activations/layer13_attention_weight_max": 49.70823669433594,
      "activations/layer13_attention_weight_min": -35.29271697998047,
      "activations/layer14_attention_weight_max": 54.91463088989258,
      "activations/layer14_attention_weight_min": -31.430923461914062,
      "activations/layer15_attention_weight_max": 47.1311149597168,
      "activations/layer15_attention_weight_min": -30.220006942749023,
      "activations/layer16_attention_weight_max": 40.49122619628906,
      "activations/layer16_attention_weight_min": -25.77043342590332,
      "activations/layer17_attention_weight_max": 60.91755676269531,
      "activations/layer17_attention_weight_min": -35.93312072753906,
      "activations/layer18_attention_weight_max": 61.87085723876953,
      "activations/layer18_attention_weight_min": -32.55756759643555,
      "activations/layer19_attention_weight_max": 24.830368041992188,
      "activations/layer19_attention_weight_min": -21.749465942382812,
      "activations/layer1_attention_weight_max": 18.138296127319336,
      "activations/layer1_attention_weight_min": -14.161172866821289,
      "activations/layer20_attention_weight_max": 23.378801345825195,
      "activations/layer20_attention_weight_min": -18.06976890563965,
      "activations/layer21_attention_weight_max": 47.52152633666992,
      "activations/layer21_attention_weight_min": -24.434886932373047,
      "activations/layer22_attention_weight_max": 38.323753356933594,
      "activations/layer22_attention_weight_min": -22.843048095703125,
      "activations/layer23_attention_weight_max": 29.694616317749023,
      "activations/layer23_attention_weight_min": -17.44013214111328,
      "activations/layer2_attention_weight_max": 32.219520568847656,
      "activations/layer2_attention_weight_min": -31.255081176757812,
      "activations/layer3_attention_weight_max": 76.08209228515625,
      "activations/layer3_attention_weight_min": -83.54835510253906,
      "activations/layer4_attention_weight_max": 99.11990356445312,
      "activations/layer4_attention_weight_min": -92.26648712158203,
      "activations/layer5_attention_weight_max": 117.107177734375,
      "activations/layer5_attention_weight_min": -95.7025146484375,
      "activations/layer6_attention_weight_max": 60.17951965332031,
      "activations/layer6_attention_weight_min": -53.670204162597656,
      "activations/layer7_attention_weight_max": 63.49824142456055,
      "activations/layer7_attention_weight_min": -61.08967590332031,
      "activations/layer8_attention_weight_max": 41.67068862915039,
      "activations/layer8_attention_weight_min": -38.53828811645508,
      "activations/layer9_attention_weight_max": 49.1390266418457,
      "activations/layer9_attention_weight_min": -50.297218322753906,
      "epoch": 4.03,
      "learning_rate": 0.00012527424242424242,
      "loss": 2.8522,
      "step": 69350
    },
    {
      "activations/layer0_attention_weight_max": 15.549028396606445,
      "activations/layer0_attention_weight_min": -15.520577430725098,
      "activations/layer10_attention_weight_max": 39.24549865722656,
      "activations/layer10_attention_weight_min": -33.80469512939453,
      "activations/layer11_attention_weight_max": 31.109169006347656,
      "activations/layer11_attention_weight_min": -26.117313385009766,
      "activations/layer12_attention_weight_max": 23.044294357299805,
      "activations/layer12_attention_weight_min": -24.27646827697754,
      "activations/layer13_attention_weight_max": 55.17793655395508,
      "activations/layer13_attention_weight_min": -37.87588119506836,
      "activations/layer14_attention_weight_max": 44.75328826904297,
      "activations/layer14_attention_weight_min": -29.44840431213379,
      "activations/layer15_attention_weight_max": 50.685302734375,
      "activations/layer15_attention_weight_min": -32.27814483642578,
      "activations/layer16_attention_weight_max": 36.015594482421875,
      "activations/layer16_attention_weight_min": -25.884050369262695,
      "activations/layer17_attention_weight_max": 57.53730773925781,
      "activations/layer17_attention_weight_min": -36.720943450927734,
      "activations/layer18_attention_weight_max": 48.78437423706055,
      "activations/layer18_attention_weight_min": -32.65329360961914,
      "activations/layer19_attention_weight_max": 23.73512840270996,
      "activations/layer19_attention_weight_min": -19.2227840423584,
      "activations/layer1_attention_weight_max": 17.30097198486328,
      "activations/layer1_attention_weight_min": -15.159928321838379,
      "activations/layer20_attention_weight_max": 21.7960262298584,
      "activations/layer20_attention_weight_min": -22.603229522705078,
      "activations/layer21_attention_weight_max": 41.60703659057617,
      "activations/layer21_attention_weight_min": -31.51687240600586,
      "activations/layer22_attention_weight_max": 34.603538513183594,
      "activations/layer22_attention_weight_min": -23.967388153076172,
      "activations/layer23_attention_weight_max": 29.964834213256836,
      "activations/layer23_attention_weight_min": -18.31928253173828,
      "activations/layer2_attention_weight_max": 31.365333557128906,
      "activations/layer2_attention_weight_min": -28.88846206665039,
      "activations/layer3_attention_weight_max": 77.45376586914062,
      "activations/layer3_attention_weight_min": -85.47663879394531,
      "activations/layer4_attention_weight_max": 100.32978057861328,
      "activations/layer4_attention_weight_min": -95.01527404785156,
      "activations/layer5_attention_weight_max": 115.7782211303711,
      "activations/layer5_attention_weight_min": -101.18477630615234,
      "activations/layer6_attention_weight_max": 61.42762756347656,
      "activations/layer6_attention_weight_min": -53.511287689208984,
      "activations/layer7_attention_weight_max": 70.46271514892578,
      "activations/layer7_attention_weight_min": -62.340309143066406,
      "activations/layer8_attention_weight_max": 42.54827880859375,
      "activations/layer8_attention_weight_min": -38.316226959228516,
      "activations/layer9_attention_weight_max": 55.907955169677734,
      "activations/layer9_attention_weight_min": -45.36577224731445,
      "epoch": 4.03,
      "learning_rate": 0.000125255303030303,
      "loss": 2.8692,
      "step": 69400
    },
    {
      "activations/layer0_attention_weight_max": 15.32382583618164,
      "activations/layer0_attention_weight_min": -15.990242958068848,
      "activations/layer10_attention_weight_max": 34.17201614379883,
      "activations/layer10_attention_weight_min": -33.705570220947266,
      "activations/layer11_attention_weight_max": 29.575376510620117,
      "activations/layer11_attention_weight_min": -24.987524032592773,
      "activations/layer12_attention_weight_max": 24.075658798217773,
      "activations/layer12_attention_weight_min": -24.419071197509766,
      "activations/layer13_attention_weight_max": 46.279136657714844,
      "activations/layer13_attention_weight_min": -36.097774505615234,
      "activations/layer14_attention_weight_max": 45.49418640136719,
      "activations/layer14_attention_weight_min": -34.450340270996094,
      "activations/layer15_attention_weight_max": 39.141883850097656,
      "activations/layer15_attention_weight_min": -31.15559196472168,
      "activations/layer16_attention_weight_max": 32.75080490112305,
      "activations/layer16_attention_weight_min": -25.889923095703125,
      "activations/layer17_attention_weight_max": 60.16068649291992,
      "activations/layer17_attention_weight_min": -34.55960464477539,
      "activations/layer18_attention_weight_max": 48.802734375,
      "activations/layer18_attention_weight_min": -33.99287796020508,
      "activations/layer19_attention_weight_max": 21.05265998840332,
      "activations/layer19_attention_weight_min": -19.7486629486084,
      "activations/layer1_attention_weight_max": 18.475954055786133,
      "activations/layer1_attention_weight_min": -13.97513198852539,
      "activations/layer20_attention_weight_max": 21.44914436340332,
      "activations/layer20_attention_weight_min": -21.984621047973633,
      "activations/layer21_attention_weight_max": 39.088409423828125,
      "activations/layer21_attention_weight_min": -28.562368392944336,
      "activations/layer22_attention_weight_max": 36.3878059387207,
      "activations/layer22_attention_weight_min": -21.636308670043945,
      "activations/layer23_attention_weight_max": 27.775556564331055,
      "activations/layer23_attention_weight_min": -17.310352325439453,
      "activations/layer2_attention_weight_max": 29.572416305541992,
      "activations/layer2_attention_weight_min": -27.464996337890625,
      "activations/layer3_attention_weight_max": 75.16970825195312,
      "activations/layer3_attention_weight_min": -77.21295928955078,
      "activations/layer4_attention_weight_max": 96.7103500366211,
      "activations/layer4_attention_weight_min": -87.41020965576172,
      "activations/layer5_attention_weight_max": 106.46554565429688,
      "activations/layer5_attention_weight_min": -92.19673156738281,
      "activations/layer6_attention_weight_max": 55.67764663696289,
      "activations/layer6_attention_weight_min": -50.853309631347656,
      "activations/layer7_attention_weight_max": 62.17517852783203,
      "activations/layer7_attention_weight_min": -58.272605895996094,
      "activations/layer8_attention_weight_max": 36.678436279296875,
      "activations/layer8_attention_weight_min": -34.724456787109375,
      "activations/layer9_attention_weight_max": 47.63737106323242,
      "activations/layer9_attention_weight_min": -44.63322448730469,
      "epoch": 4.04,
      "learning_rate": 0.00012523636363636362,
      "loss": 2.869,
      "step": 69450
    },
    {
      "activations/layer0_attention_weight_max": 14.75537109375,
      "activations/layer0_attention_weight_min": -16.049535751342773,
      "activations/layer10_attention_weight_max": 46.292510986328125,
      "activations/layer10_attention_weight_min": -34.37178421020508,
      "activations/layer11_attention_weight_max": 38.59466552734375,
      "activations/layer11_attention_weight_min": -27.493993759155273,
      "activations/layer12_attention_weight_max": 32.56657028198242,
      "activations/layer12_attention_weight_min": -24.505191802978516,
      "activations/layer13_attention_weight_max": 71.15965270996094,
      "activations/layer13_attention_weight_min": -45.428043365478516,
      "activations/layer14_attention_weight_max": 57.033599853515625,
      "activations/layer14_attention_weight_min": -34.13611602783203,
      "activations/layer15_attention_weight_max": 54.15129852294922,
      "activations/layer15_attention_weight_min": -31.735546112060547,
      "activations/layer16_attention_weight_max": 42.63547134399414,
      "activations/layer16_attention_weight_min": -26.29502296447754,
      "activations/layer17_attention_weight_max": 68.3442611694336,
      "activations/layer17_attention_weight_min": -37.98693084716797,
      "activations/layer18_attention_weight_max": 61.403358459472656,
      "activations/layer18_attention_weight_min": -33.17656326293945,
      "activations/layer19_attention_weight_max": 26.903732299804688,
      "activations/layer19_attention_weight_min": -20.195363998413086,
      "activations/layer1_attention_weight_max": 17.24822998046875,
      "activations/layer1_attention_weight_min": -16.160118103027344,
      "activations/layer20_attention_weight_max": 24.60790252685547,
      "activations/layer20_attention_weight_min": -22.717174530029297,
      "activations/layer21_attention_weight_max": 48.094200134277344,
      "activations/layer21_attention_weight_min": -26.7619686126709,
      "activations/layer22_attention_weight_max": 40.76812744140625,
      "activations/layer22_attention_weight_min": -23.483736038208008,
      "activations/layer23_attention_weight_max": 35.79788589477539,
      "activations/layer23_attention_weight_min": -18.459209442138672,
      "activations/layer2_attention_weight_max": 30.03927230834961,
      "activations/layer2_attention_weight_min": -29.06169891357422,
      "activations/layer3_attention_weight_max": 79.51692199707031,
      "activations/layer3_attention_weight_min": -79.73902130126953,
      "activations/layer4_attention_weight_max": 107.53125,
      "activations/layer4_attention_weight_min": -98.50627136230469,
      "activations/layer5_attention_weight_max": 122.43449401855469,
      "activations/layer5_attention_weight_min": -101.72380828857422,
      "activations/layer6_attention_weight_max": 62.840728759765625,
      "activations/layer6_attention_weight_min": -56.88261795043945,
      "activations/layer7_attention_weight_max": 70.3841781616211,
      "activations/layer7_attention_weight_min": -63.469112396240234,
      "activations/layer8_attention_weight_max": 41.58416748046875,
      "activations/layer8_attention_weight_min": -42.06975173950195,
      "activations/layer9_attention_weight_max": 53.0351448059082,
      "activations/layer9_attention_weight_min": -44.74115753173828,
      "epoch": 4.04,
      "learning_rate": 0.00012521742424242424,
      "loss": 2.8609,
      "step": 69500
    },
    {
      "activations/layer0_attention_weight_max": 15.104204177856445,
      "activations/layer0_attention_weight_min": -15.26887321472168,
      "activations/layer10_attention_weight_max": 41.40279006958008,
      "activations/layer10_attention_weight_min": -35.464202880859375,
      "activations/layer11_attention_weight_max": 32.65280532836914,
      "activations/layer11_attention_weight_min": -27.559432983398438,
      "activations/layer12_attention_weight_max": 25.903335571289062,
      "activations/layer12_attention_weight_min": -26.493478775024414,
      "activations/layer13_attention_weight_max": 50.12160110473633,
      "activations/layer13_attention_weight_min": -35.17987060546875,
      "activations/layer14_attention_weight_max": 53.160423278808594,
      "activations/layer14_attention_weight_min": -32.44160842895508,
      "activations/layer15_attention_weight_max": 45.345340728759766,
      "activations/layer15_attention_weight_min": -32.331966400146484,
      "activations/layer16_attention_weight_max": 39.0091552734375,
      "activations/layer16_attention_weight_min": -26.97314453125,
      "activations/layer17_attention_weight_max": 58.4560432434082,
      "activations/layer17_attention_weight_min": -37.819766998291016,
      "activations/layer18_attention_weight_max": 57.72050476074219,
      "activations/layer18_attention_weight_min": -33.654747009277344,
      "activations/layer19_attention_weight_max": 20.93134880065918,
      "activations/layer19_attention_weight_min": -18.112607955932617,
      "activations/layer1_attention_weight_max": 17.625621795654297,
      "activations/layer1_attention_weight_min": -15.618308067321777,
      "activations/layer20_attention_weight_max": 22.17157745361328,
      "activations/layer20_attention_weight_min": -19.888164520263672,
      "activations/layer21_attention_weight_max": 44.6767578125,
      "activations/layer21_attention_weight_min": -27.457693099975586,
      "activations/layer22_attention_weight_max": 39.31645202636719,
      "activations/layer22_attention_weight_min": -23.40044593811035,
      "activations/layer23_attention_weight_max": 31.257095336914062,
      "activations/layer23_attention_weight_min": -17.391305923461914,
      "activations/layer2_attention_weight_max": 31.43709945678711,
      "activations/layer2_attention_weight_min": -28.461349487304688,
      "activations/layer3_attention_weight_max": 79.81535339355469,
      "activations/layer3_attention_weight_min": -80.89273071289062,
      "activations/layer4_attention_weight_max": 104.31007385253906,
      "activations/layer4_attention_weight_min": -96.88801574707031,
      "activations/layer5_attention_weight_max": 114.32856750488281,
      "activations/layer5_attention_weight_min": -108.10647583007812,
      "activations/layer6_attention_weight_max": 63.41143035888672,
      "activations/layer6_attention_weight_min": -55.76719284057617,
      "activations/layer7_attention_weight_max": 64.6565170288086,
      "activations/layer7_attention_weight_min": -65.97035217285156,
      "activations/layer8_attention_weight_max": 43.557090759277344,
      "activations/layer8_attention_weight_min": -41.44004821777344,
      "activations/layer9_attention_weight_max": 55.94185256958008,
      "activations/layer9_attention_weight_min": -48.60248947143555,
      "epoch": 4.04,
      "learning_rate": 0.00012519848484848485,
      "loss": 2.8618,
      "step": 69550
    },
    {
      "activations/layer0_attention_weight_max": 16.21323585510254,
      "activations/layer0_attention_weight_min": -15.144203186035156,
      "activations/layer10_attention_weight_max": 44.344390869140625,
      "activations/layer10_attention_weight_min": -37.128700256347656,
      "activations/layer11_attention_weight_max": 39.006996154785156,
      "activations/layer11_attention_weight_min": -27.419193267822266,
      "activations/layer12_attention_weight_max": 28.366168975830078,
      "activations/layer12_attention_weight_min": -29.51020050048828,
      "activations/layer13_attention_weight_max": 66.83930969238281,
      "activations/layer13_attention_weight_min": -40.1953125,
      "activations/layer14_attention_weight_max": 85.65591430664062,
      "activations/layer14_attention_weight_min": -47.98904037475586,
      "activations/layer15_attention_weight_max": 60.577999114990234,
      "activations/layer15_attention_weight_min": -37.32811737060547,
      "activations/layer16_attention_weight_max": 34.16606903076172,
      "activations/layer16_attention_weight_min": -26.531909942626953,
      "activations/layer17_attention_weight_max": 61.23796463012695,
      "activations/layer17_attention_weight_min": -37.73141860961914,
      "activations/layer18_attention_weight_max": 52.0685920715332,
      "activations/layer18_attention_weight_min": -32.54865646362305,
      "activations/layer19_attention_weight_max": 23.705421447753906,
      "activations/layer19_attention_weight_min": -20.04175567626953,
      "activations/layer1_attention_weight_max": 18.324548721313477,
      "activations/layer1_attention_weight_min": -15.452919006347656,
      "activations/layer20_attention_weight_max": 22.281991958618164,
      "activations/layer20_attention_weight_min": -17.868886947631836,
      "activations/layer21_attention_weight_max": 41.88224411010742,
      "activations/layer21_attention_weight_min": -26.04913902282715,
      "activations/layer22_attention_weight_max": 33.52632141113281,
      "activations/layer22_attention_weight_min": -23.770599365234375,
      "activations/layer23_attention_weight_max": 27.677310943603516,
      "activations/layer23_attention_weight_min": -18.780471801757812,
      "activations/layer2_attention_weight_max": 34.580406188964844,
      "activations/layer2_attention_weight_min": -31.310028076171875,
      "activations/layer3_attention_weight_max": 94.65052795410156,
      "activations/layer3_attention_weight_min": -93.51793670654297,
      "activations/layer4_attention_weight_max": 126.92762756347656,
      "activations/layer4_attention_weight_min": -110.55998992919922,
      "activations/layer5_attention_weight_max": 143.5476837158203,
      "activations/layer5_attention_weight_min": -116.6407699584961,
      "activations/layer6_attention_weight_max": 72.69998931884766,
      "activations/layer6_attention_weight_min": -61.86134338378906,
      "activations/layer7_attention_weight_max": 82.1319580078125,
      "activations/layer7_attention_weight_min": -69.00638580322266,
      "activations/layer8_attention_weight_max": 54.21575164794922,
      "activations/layer8_attention_weight_min": -44.505638122558594,
      "activations/layer9_attention_weight_max": 68.83601379394531,
      "activations/layer9_attention_weight_min": -58.8384895324707,
      "epoch": 4.04,
      "learning_rate": 0.00012517954545454544,
      "loss": 2.8591,
      "step": 69600
    },
    {
      "activations/layer0_attention_weight_max": 14.164064407348633,
      "activations/layer0_attention_weight_min": -16.005504608154297,
      "activations/layer10_attention_weight_max": 38.23583984375,
      "activations/layer10_attention_weight_min": -33.711700439453125,
      "activations/layer11_attention_weight_max": 33.167335510253906,
      "activations/layer11_attention_weight_min": -27.136852264404297,
      "activations/layer12_attention_weight_max": 30.986661911010742,
      "activations/layer12_attention_weight_min": -25.60100746154785,
      "activations/layer13_attention_weight_max": 56.26620864868164,
      "activations/layer13_attention_weight_min": -36.39582061767578,
      "activations/layer14_attention_weight_max": 51.305152893066406,
      "activations/layer14_attention_weight_min": -38.00321960449219,
      "activations/layer15_attention_weight_max": 48.719383239746094,
      "activations/layer15_attention_weight_min": -32.419002532958984,
      "activations/layer16_attention_weight_max": 36.34634780883789,
      "activations/layer16_attention_weight_min": -27.029741287231445,
      "activations/layer17_attention_weight_max": 64.78072357177734,
      "activations/layer17_attention_weight_min": -36.291358947753906,
      "activations/layer18_attention_weight_max": 53.2632942199707,
      "activations/layer18_attention_weight_min": -34.15795135498047,
      "activations/layer19_attention_weight_max": 23.794496536254883,
      "activations/layer19_attention_weight_min": -17.340883255004883,
      "activations/layer1_attention_weight_max": 17.51662254333496,
      "activations/layer1_attention_weight_min": -14.879684448242188,
      "activations/layer20_attention_weight_max": 22.24186897277832,
      "activations/layer20_attention_weight_min": -20.978408813476562,
      "activations/layer21_attention_weight_max": 45.377037048339844,
      "activations/layer21_attention_weight_min": -25.190786361694336,
      "activations/layer22_attention_weight_max": 38.524837493896484,
      "activations/layer22_attention_weight_min": -21.865978240966797,
      "activations/layer23_attention_weight_max": 29.040958404541016,
      "activations/layer23_attention_weight_min": -18.933109283447266,
      "activations/layer2_attention_weight_max": 30.165096282958984,
      "activations/layer2_attention_weight_min": -27.69039535522461,
      "activations/layer3_attention_weight_max": 76.40410614013672,
      "activations/layer3_attention_weight_min": -81.10578155517578,
      "activations/layer4_attention_weight_max": 97.04280853271484,
      "activations/layer4_attention_weight_min": -91.92327117919922,
      "activations/layer5_attention_weight_max": 109.23644256591797,
      "activations/layer5_attention_weight_min": -97.77001953125,
      "activations/layer6_attention_weight_max": 59.52931213378906,
      "activations/layer6_attention_weight_min": -50.35239791870117,
      "activations/layer7_attention_weight_max": 65.77249145507812,
      "activations/layer7_attention_weight_min": -64.98393249511719,
      "activations/layer8_attention_weight_max": 41.22760009765625,
      "activations/layer8_attention_weight_min": -39.05604934692383,
      "activations/layer9_attention_weight_max": 51.18659210205078,
      "activations/layer9_attention_weight_min": -43.64265060424805,
      "epoch": 4.05,
      "learning_rate": 0.00012516060606060605,
      "loss": 2.86,
      "step": 69650
    },
    {
      "activations/layer0_attention_weight_max": 14.296385765075684,
      "activations/layer0_attention_weight_min": -15.45965576171875,
      "activations/layer10_attention_weight_max": 46.58356857299805,
      "activations/layer10_attention_weight_min": -38.07902908325195,
      "activations/layer11_attention_weight_max": 34.259788513183594,
      "activations/layer11_attention_weight_min": -28.850078582763672,
      "activations/layer12_attention_weight_max": 30.446622848510742,
      "activations/layer12_attention_weight_min": -22.422119140625,
      "activations/layer13_attention_weight_max": 60.39374923706055,
      "activations/layer13_attention_weight_min": -38.20631408691406,
      "activations/layer14_attention_weight_max": 48.19761276245117,
      "activations/layer14_attention_weight_min": -30.053098678588867,
      "activations/layer15_attention_weight_max": 52.79070281982422,
      "activations/layer15_attention_weight_min": -32.29460906982422,
      "activations/layer16_attention_weight_max": 36.284793853759766,
      "activations/layer16_attention_weight_min": -27.6171875,
      "activations/layer17_attention_weight_max": 61.75343704223633,
      "activations/layer17_attention_weight_min": -35.903099060058594,
      "activations/layer18_attention_weight_max": 52.11436080932617,
      "activations/layer18_attention_weight_min": -33.424983978271484,
      "activations/layer19_attention_weight_max": 26.570377349853516,
      "activations/layer19_attention_weight_min": -19.705947875976562,
      "activations/layer1_attention_weight_max": 18.02804946899414,
      "activations/layer1_attention_weight_min": -16.46653175354004,
      "activations/layer20_attention_weight_max": 23.245864868164062,
      "activations/layer20_attention_weight_min": -20.711505889892578,
      "activations/layer21_attention_weight_max": 48.88208770751953,
      "activations/layer21_attention_weight_min": -28.966819763183594,
      "activations/layer22_attention_weight_max": 34.53752899169922,
      "activations/layer22_attention_weight_min": -24.021774291992188,
      "activations/layer23_attention_weight_max": 26.44393539428711,
      "activations/layer23_attention_weight_min": -15.823087692260742,
      "activations/layer2_attention_weight_max": 31.244884490966797,
      "activations/layer2_attention_weight_min": -28.643796920776367,
      "activations/layer3_attention_weight_max": 81.67442321777344,
      "activations/layer3_attention_weight_min": -90.36016082763672,
      "activations/layer4_attention_weight_max": 106.06434631347656,
      "activations/layer4_attention_weight_min": -103.25572967529297,
      "activations/layer5_attention_weight_max": 119.22179412841797,
      "activations/layer5_attention_weight_min": -111.21837615966797,
      "activations/layer6_attention_weight_max": 64.06340026855469,
      "activations/layer6_attention_weight_min": -57.797607421875,
      "activations/layer7_attention_weight_max": 69.31790924072266,
      "activations/layer7_attention_weight_min": -65.23526763916016,
      "activations/layer8_attention_weight_max": 44.434696197509766,
      "activations/layer8_attention_weight_min": -44.21086120605469,
      "activations/layer9_attention_weight_max": 51.90507888793945,
      "activations/layer9_attention_weight_min": -48.43415069580078,
      "epoch": 4.05,
      "learning_rate": 0.00012514166666666667,
      "loss": 2.8626,
      "step": 69700
    },
    {
      "activations/layer0_attention_weight_max": 15.07716178894043,
      "activations/layer0_attention_weight_min": -15.74223518371582,
      "activations/layer10_attention_weight_max": 37.84075927734375,
      "activations/layer10_attention_weight_min": -33.0594367980957,
      "activations/layer11_attention_weight_max": 29.599576950073242,
      "activations/layer11_attention_weight_min": -26.07185173034668,
      "activations/layer12_attention_weight_max": 24.768346786499023,
      "activations/layer12_attention_weight_min": -27.299882888793945,
      "activations/layer13_attention_weight_max": 53.82645034790039,
      "activations/layer13_attention_weight_min": -42.86182403564453,
      "activations/layer14_attention_weight_max": 40.252037048339844,
      "activations/layer14_attention_weight_min": -31.294458389282227,
      "activations/layer15_attention_weight_max": 44.43891143798828,
      "activations/layer15_attention_weight_min": -31.935821533203125,
      "activations/layer16_attention_weight_max": 35.05205154418945,
      "activations/layer16_attention_weight_min": -27.135034561157227,
      "activations/layer17_attention_weight_max": 56.78620529174805,
      "activations/layer17_attention_weight_min": -38.3735466003418,
      "activations/layer18_attention_weight_max": 46.26485061645508,
      "activations/layer18_attention_weight_min": -33.31590270996094,
      "activations/layer19_attention_weight_max": 22.3393611907959,
      "activations/layer19_attention_weight_min": -19.885440826416016,
      "activations/layer1_attention_weight_max": 17.0234432220459,
      "activations/layer1_attention_weight_min": -14.277138710021973,
      "activations/layer20_attention_weight_max": 24.261083602905273,
      "activations/layer20_attention_weight_min": -21.965713500976562,
      "activations/layer21_attention_weight_max": 40.75506591796875,
      "activations/layer21_attention_weight_min": -28.966552734375,
      "activations/layer22_attention_weight_max": 32.13434600830078,
      "activations/layer22_attention_weight_min": -22.8901424407959,
      "activations/layer23_attention_weight_max": 24.812419891357422,
      "activations/layer23_attention_weight_min": -19.126808166503906,
      "activations/layer2_attention_weight_max": 29.899654388427734,
      "activations/layer2_attention_weight_min": -27.973827362060547,
      "activations/layer3_attention_weight_max": 77.25553131103516,
      "activations/layer3_attention_weight_min": -85.62053680419922,
      "activations/layer4_attention_weight_max": 104.50676727294922,
      "activations/layer4_attention_weight_min": -97.96033477783203,
      "activations/layer5_attention_weight_max": 107.471435546875,
      "activations/layer5_attention_weight_min": -95.23890686035156,
      "activations/layer6_attention_weight_max": 55.65687561035156,
      "activations/layer6_attention_weight_min": -52.032901763916016,
      "activations/layer7_attention_weight_max": 61.48038864135742,
      "activations/layer7_attention_weight_min": -60.89609909057617,
      "activations/layer8_attention_weight_max": 40.14083480834961,
      "activations/layer8_attention_weight_min": -38.014408111572266,
      "activations/layer9_attention_weight_max": 48.76191329956055,
      "activations/layer9_attention_weight_min": -48.33931350708008,
      "epoch": 4.05,
      "learning_rate": 0.00012512272727272726,
      "loss": 2.8721,
      "step": 69750
    },
    {
      "activations/layer0_attention_weight_max": 14.84449291229248,
      "activations/layer0_attention_weight_min": -15.614513397216797,
      "activations/layer10_attention_weight_max": 39.8203125,
      "activations/layer10_attention_weight_min": -33.503028869628906,
      "activations/layer11_attention_weight_max": 34.8394889831543,
      "activations/layer11_attention_weight_min": -27.083698272705078,
      "activations/layer12_attention_weight_max": 25.733030319213867,
      "activations/layer12_attention_weight_min": -25.9237003326416,
      "activations/layer13_attention_weight_max": 45.27225875854492,
      "activations/layer13_attention_weight_min": -36.524803161621094,
      "activations/layer14_attention_weight_max": 51.96683120727539,
      "activations/layer14_attention_weight_min": -34.6793212890625,
      "activations/layer15_attention_weight_max": 41.19765090942383,
      "activations/layer15_attention_weight_min": -30.470659255981445,
      "activations/layer16_attention_weight_max": 34.1484260559082,
      "activations/layer16_attention_weight_min": -26.27000617980957,
      "activations/layer17_attention_weight_max": 60.4964485168457,
      "activations/layer17_attention_weight_min": -36.068546295166016,
      "activations/layer18_attention_weight_max": 49.29990768432617,
      "activations/layer18_attention_weight_min": -31.480504989624023,
      "activations/layer19_attention_weight_max": 25.921051025390625,
      "activations/layer19_attention_weight_min": -18.169893264770508,
      "activations/layer1_attention_weight_max": 18.326648712158203,
      "activations/layer1_attention_weight_min": -13.716911315917969,
      "activations/layer20_attention_weight_max": 22.595298767089844,
      "activations/layer20_attention_weight_min": -23.735515594482422,
      "activations/layer21_attention_weight_max": 40.08190155029297,
      "activations/layer21_attention_weight_min": -24.10259437561035,
      "activations/layer22_attention_weight_max": 35.447418212890625,
      "activations/layer22_attention_weight_min": -22.1987247467041,
      "activations/layer23_attention_weight_max": 31.25984001159668,
      "activations/layer23_attention_weight_min": -18.14382553100586,
      "activations/layer2_attention_weight_max": 29.553640365600586,
      "activations/layer2_attention_weight_min": -26.64788055419922,
      "activations/layer3_attention_weight_max": 76.66206359863281,
      "activations/layer3_attention_weight_min": -80.1758041381836,
      "activations/layer4_attention_weight_max": 96.9836196899414,
      "activations/layer4_attention_weight_min": -93.38555145263672,
      "activations/layer5_attention_weight_max": 111.30120849609375,
      "activations/layer5_attention_weight_min": -102.5872802734375,
      "activations/layer6_attention_weight_max": 57.07463455200195,
      "activations/layer6_attention_weight_min": -53.608848571777344,
      "activations/layer7_attention_weight_max": 61.252281188964844,
      "activations/layer7_attention_weight_min": -58.18479537963867,
      "activations/layer8_attention_weight_max": 40.874088287353516,
      "activations/layer8_attention_weight_min": -38.07416915893555,
      "activations/layer9_attention_weight_max": 48.32938766479492,
      "activations/layer9_attention_weight_min": -44.34505844116211,
      "epoch": 4.06,
      "learning_rate": 0.00012510378787878787,
      "loss": 2.8674,
      "step": 69800
    },
    {
      "activations/layer0_attention_weight_max": 14.347382545471191,
      "activations/layer0_attention_weight_min": -15.599149703979492,
      "activations/layer10_attention_weight_max": 40.403160095214844,
      "activations/layer10_attention_weight_min": -32.305328369140625,
      "activations/layer11_attention_weight_max": 31.82758140563965,
      "activations/layer11_attention_weight_min": -25.705005645751953,
      "activations/layer12_attention_weight_max": 26.507787704467773,
      "activations/layer12_attention_weight_min": -26.152267456054688,
      "activations/layer13_attention_weight_max": 63.16263198852539,
      "activations/layer13_attention_weight_min": -43.69034957885742,
      "activations/layer14_attention_weight_max": 54.00382614135742,
      "activations/layer14_attention_weight_min": -43.029876708984375,
      "activations/layer15_attention_weight_max": 41.407466888427734,
      "activations/layer15_attention_weight_min": -33.917720794677734,
      "activations/layer16_attention_weight_max": 36.56420135498047,
      "activations/layer16_attention_weight_min": -26.928512573242188,
      "activations/layer17_attention_weight_max": 59.192626953125,
      "activations/layer17_attention_weight_min": -37.11723709106445,
      "activations/layer18_attention_weight_max": 49.72452163696289,
      "activations/layer18_attention_weight_min": -31.540218353271484,
      "activations/layer19_attention_weight_max": 20.26496124267578,
      "activations/layer19_attention_weight_min": -19.47355842590332,
      "activations/layer1_attention_weight_max": 17.47331428527832,
      "activations/layer1_attention_weight_min": -16.806385040283203,
      "activations/layer20_attention_weight_max": 21.514785766601562,
      "activations/layer20_attention_weight_min": -17.14836883544922,
      "activations/layer21_attention_weight_max": 39.131988525390625,
      "activations/layer21_attention_weight_min": -25.848167419433594,
      "activations/layer22_attention_weight_max": 41.35279846191406,
      "activations/layer22_attention_weight_min": -22.919654846191406,
      "activations/layer23_attention_weight_max": 27.005170822143555,
      "activations/layer23_attention_weight_min": -15.083678245544434,
      "activations/layer2_attention_weight_max": 32.533748626708984,
      "activations/layer2_attention_weight_min": -30.00692367553711,
      "activations/layer3_attention_weight_max": 80.74745178222656,
      "activations/layer3_attention_weight_min": -83.23123931884766,
      "activations/layer4_attention_weight_max": 107.02408599853516,
      "activations/layer4_attention_weight_min": -100.80563354492188,
      "activations/layer5_attention_weight_max": 129.99008178710938,
      "activations/layer5_attention_weight_min": -107.52009582519531,
      "activations/layer6_attention_weight_max": 67.15478515625,
      "activations/layer6_attention_weight_min": -55.61705017089844,
      "activations/layer7_attention_weight_max": 67.0904312133789,
      "activations/layer7_attention_weight_min": -70.1803970336914,
      "activations/layer8_attention_weight_max": 42.53763961791992,
      "activations/layer8_attention_weight_min": -41.219181060791016,
      "activations/layer9_attention_weight_max": 56.0340690612793,
      "activations/layer9_attention_weight_min": -47.49540328979492,
      "epoch": 4.06,
      "learning_rate": 0.00012508484848484849,
      "loss": 2.8632,
      "step": 69850
    },
    {
      "activations/layer0_attention_weight_max": 14.906228065490723,
      "activations/layer0_attention_weight_min": -14.695999145507812,
      "activations/layer10_attention_weight_max": 40.8303108215332,
      "activations/layer10_attention_weight_min": -34.34267044067383,
      "activations/layer11_attention_weight_max": 29.518718719482422,
      "activations/layer11_attention_weight_min": -25.65620994567871,
      "activations/layer12_attention_weight_max": 22.765214920043945,
      "activations/layer12_attention_weight_min": -25.969932556152344,
      "activations/layer13_attention_weight_max": 63.862064361572266,
      "activations/layer13_attention_weight_min": -40.87568664550781,
      "activations/layer14_attention_weight_max": 67.40560913085938,
      "activations/layer14_attention_weight_min": -36.77678298950195,
      "activations/layer15_attention_weight_max": 44.53133010864258,
      "activations/layer15_attention_weight_min": -31.543987274169922,
      "activations/layer16_attention_weight_max": 36.713897705078125,
      "activations/layer16_attention_weight_min": -26.732135772705078,
      "activations/layer17_attention_weight_max": 57.68381118774414,
      "activations/layer17_attention_weight_min": -36.178985595703125,
      "activations/layer18_attention_weight_max": 50.02731704711914,
      "activations/layer18_attention_weight_min": -32.12950897216797,
      "activations/layer19_attention_weight_max": 21.876283645629883,
      "activations/layer19_attention_weight_min": -19.565519332885742,
      "activations/layer1_attention_weight_max": 18.547700881958008,
      "activations/layer1_attention_weight_min": -16.451208114624023,
      "activations/layer20_attention_weight_max": 19.671342849731445,
      "activations/layer20_attention_weight_min": -25.209949493408203,
      "activations/layer21_attention_weight_max": 40.7693977355957,
      "activations/layer21_attention_weight_min": -26.984420776367188,
      "activations/layer22_attention_weight_max": 32.597965240478516,
      "activations/layer22_attention_weight_min": -22.695642471313477,
      "activations/layer23_attention_weight_max": 24.338773727416992,
      "activations/layer23_attention_weight_min": -17.564401626586914,
      "activations/layer2_attention_weight_max": 29.8934383392334,
      "activations/layer2_attention_weight_min": -29.67928123474121,
      "activations/layer3_attention_weight_max": 76.55118560791016,
      "activations/layer3_attention_weight_min": -79.28689575195312,
      "activations/layer4_attention_weight_max": 98.03297424316406,
      "activations/layer4_attention_weight_min": -90.94317626953125,
      "activations/layer5_attention_weight_max": 112.59652709960938,
      "activations/layer5_attention_weight_min": -98.62751770019531,
      "activations/layer6_attention_weight_max": 62.634002685546875,
      "activations/layer6_attention_weight_min": -54.15776824951172,
      "activations/layer7_attention_weight_max": 64.77123260498047,
      "activations/layer7_attention_weight_min": -64.10530853271484,
      "activations/layer8_attention_weight_max": 42.10420608520508,
      "activations/layer8_attention_weight_min": -42.239463806152344,
      "activations/layer9_attention_weight_max": 49.90802001953125,
      "activations/layer9_attention_weight_min": -45.97671127319336,
      "epoch": 4.06,
      "learning_rate": 0.00012506590909090907,
      "loss": 2.8696,
      "step": 69900
    },
    {
      "activations/layer0_attention_weight_max": 15.237199783325195,
      "activations/layer0_attention_weight_min": -15.203054428100586,
      "activations/layer10_attention_weight_max": 38.19672393798828,
      "activations/layer10_attention_weight_min": -33.78788757324219,
      "activations/layer11_attention_weight_max": 31.474544525146484,
      "activations/layer11_attention_weight_min": -25.69355583190918,
      "activations/layer12_attention_weight_max": 32.80681610107422,
      "activations/layer12_attention_weight_min": -24.328947067260742,
      "activations/layer13_attention_weight_max": 58.284141540527344,
      "activations/layer13_attention_weight_min": -42.65232849121094,
      "activations/layer14_attention_weight_max": 56.965301513671875,
      "activations/layer14_attention_weight_min": -39.01258087158203,
      "activations/layer15_attention_weight_max": 42.306034088134766,
      "activations/layer15_attention_weight_min": -31.794666290283203,
      "activations/layer16_attention_weight_max": 33.30013656616211,
      "activations/layer16_attention_weight_min": -26.8756160736084,
      "activations/layer17_attention_weight_max": 55.77567672729492,
      "activations/layer17_attention_weight_min": -36.565162658691406,
      "activations/layer18_attention_weight_max": 47.329105377197266,
      "activations/layer18_attention_weight_min": -31.994728088378906,
      "activations/layer19_attention_weight_max": 22.780487060546875,
      "activations/layer19_attention_weight_min": -18.413246154785156,
      "activations/layer1_attention_weight_max": 18.44280242919922,
      "activations/layer1_attention_weight_min": -15.587737083435059,
      "activations/layer20_attention_weight_max": 20.965679168701172,
      "activations/layer20_attention_weight_min": -20.041454315185547,
      "activations/layer21_attention_weight_max": 37.06766891479492,
      "activations/layer21_attention_weight_min": -29.68782615661621,
      "activations/layer22_attention_weight_max": 33.2005615234375,
      "activations/layer22_attention_weight_min": -24.001956939697266,
      "activations/layer23_attention_weight_max": 23.44482421875,
      "activations/layer23_attention_weight_min": -16.31873321533203,
      "activations/layer2_attention_weight_max": 33.57013702392578,
      "activations/layer2_attention_weight_min": -29.138906478881836,
      "activations/layer3_attention_weight_max": 77.08277893066406,
      "activations/layer3_attention_weight_min": -81.80961608886719,
      "activations/layer4_attention_weight_max": 98.56217193603516,
      "activations/layer4_attention_weight_min": -93.23191833496094,
      "activations/layer5_attention_weight_max": 116.25910949707031,
      "activations/layer5_attention_weight_min": -99.1487045288086,
      "activations/layer6_attention_weight_max": 60.30706787109375,
      "activations/layer6_attention_weight_min": -52.03012466430664,
      "activations/layer7_attention_weight_max": 63.7002067565918,
      "activations/layer7_attention_weight_min": -61.308319091796875,
      "activations/layer8_attention_weight_max": 43.432315826416016,
      "activations/layer8_attention_weight_min": -42.76105499267578,
      "activations/layer9_attention_weight_max": 50.175907135009766,
      "activations/layer9_attention_weight_min": -49.82040023803711,
      "epoch": 4.06,
      "learning_rate": 0.0001250469696969697,
      "loss": 2.8852,
      "step": 69950
    },
    {
      "activations/layer0_attention_weight_max": 16.11783218383789,
      "activations/layer0_attention_weight_min": -15.264945983886719,
      "activations/layer10_attention_weight_max": 39.06134033203125,
      "activations/layer10_attention_weight_min": -33.97431945800781,
      "activations/layer11_attention_weight_max": 31.21915054321289,
      "activations/layer11_attention_weight_min": -26.38469696044922,
      "activations/layer12_attention_weight_max": 22.548709869384766,
      "activations/layer12_attention_weight_min": -26.784093856811523,
      "activations/layer13_attention_weight_max": 48.614295959472656,
      "activations/layer13_attention_weight_min": -36.87571334838867,
      "activations/layer14_attention_weight_max": 43.31626510620117,
      "activations/layer14_attention_weight_min": -30.541967391967773,
      "activations/layer15_attention_weight_max": 39.210960388183594,
      "activations/layer15_attention_weight_min": -30.23953628540039,
      "activations/layer16_attention_weight_max": 32.812164306640625,
      "activations/layer16_attention_weight_min": -30.60401153564453,
      "activations/layer17_attention_weight_max": 58.3321418762207,
      "activations/layer17_attention_weight_min": -37.926612854003906,
      "activations/layer18_attention_weight_max": 49.17301940917969,
      "activations/layer18_attention_weight_min": -33.466712951660156,
      "activations/layer19_attention_weight_max": 21.381851196289062,
      "activations/layer19_attention_weight_min": -19.530900955200195,
      "activations/layer1_attention_weight_max": 17.285917282104492,
      "activations/layer1_attention_weight_min": -18.519947052001953,
      "activations/layer20_attention_weight_max": 21.569141387939453,
      "activations/layer20_attention_weight_min": -23.851037979125977,
      "activations/layer21_attention_weight_max": 42.12208557128906,
      "activations/layer21_attention_weight_min": -29.886465072631836,
      "activations/layer22_attention_weight_max": 34.54676818847656,
      "activations/layer22_attention_weight_min": -25.56687355041504,
      "activations/layer23_attention_weight_max": 29.19880485534668,
      "activations/layer23_attention_weight_min": -17.933197021484375,
      "activations/layer2_attention_weight_max": 33.288028717041016,
      "activations/layer2_attention_weight_min": -28.663366317749023,
      "activations/layer3_attention_weight_max": 76.89008331298828,
      "activations/layer3_attention_weight_min": -83.02584838867188,
      "activations/layer4_attention_weight_max": 98.2446517944336,
      "activations/layer4_attention_weight_min": -90.2379379272461,
      "activations/layer5_attention_weight_max": 108.93345642089844,
      "activations/layer5_attention_weight_min": -94.11358642578125,
      "activations/layer6_attention_weight_max": 58.37043380737305,
      "activations/layer6_attention_weight_min": -50.19841003417969,
      "activations/layer7_attention_weight_max": 59.44347381591797,
      "activations/layer7_attention_weight_min": -59.12747573852539,
      "activations/layer8_attention_weight_max": 40.313743591308594,
      "activations/layer8_attention_weight_min": -35.853431701660156,
      "activations/layer9_attention_weight_max": 45.28069305419922,
      "activations/layer9_attention_weight_min": -44.78813934326172,
      "epoch": 4.07,
      "learning_rate": 0.0001250280303030303,
      "loss": 2.8565,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4726,
      "eval_samples_per_second": 506.811,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4726,
      "eval_openwebtext_samples_per_second": 506.811,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_wikitext_loss": 3.06640625,
      "eval_wikitext_ppl": 21.464625403032095,
      "eval_wikitext_runtime": 1.9732,
      "eval_wikitext_samples_per_second": 231.099,
      "step": 70000
    },
    {
      "epoch": 4.07,
      "eval_lambada_loss": 2.86328125,
      "eval_lambada_ppl": 17.51891667538985,
      "eval_lambada_runtime": 9.5321,
      "eval_lambada_samples_per_second": 510.799,
      "step": 70000
    },
    {
      "activations/layer0_attention_weight_max": 15.22487735748291,
      "activations/layer0_attention_weight_min": -15.40920639038086,
      "activations/layer10_attention_weight_max": 37.60860824584961,
      "activations/layer10_attention_weight_min": -32.19960403442383,
      "activations/layer11_attention_weight_max": 28.29446792602539,
      "activations/layer11_attention_weight_min": -25.47296905517578,
      "activations/layer12_attention_weight_max": 22.199710845947266,
      "activations/layer12_attention_weight_min": -26.119245529174805,
      "activations/layer13_attention_weight_max": 48.926231384277344,
      "activations/layer13_attention_weight_min": -37.90615463256836,
      "activations/layer14_attention_weight_max": 43.839385986328125,
      "activations/layer14_attention_weight_min": -28.8317813873291,
      "activations/layer15_attention_weight_max": 42.369808197021484,
      "activations/layer15_attention_weight_min": -29.410951614379883,
      "activations/layer16_attention_weight_max": 33.89021301269531,
      "activations/layer16_attention_weight_min": -25.685810089111328,
      "activations/layer17_attention_weight_max": 54.70025634765625,
      "activations/layer17_attention_weight_min": -35.7964973449707,
      "activations/layer18_attention_weight_max": 44.08552551269531,
      "activations/layer18_attention_weight_min": -29.580406188964844,
      "activations/layer19_attention_weight_max": 21.489158630371094,
      "activations/layer19_attention_weight_min": -18.317184448242188,
      "activations/layer1_attention_weight_max": 17.23030662536621,
      "activations/layer1_attention_weight_min": -14.637693405151367,
      "activations/layer20_attention_weight_max": 19.507402420043945,
      "activations/layer20_attention_weight_min": -23.03463363647461,
      "activations/layer21_attention_weight_max": 37.088565826416016,
      "activations/layer21_attention_weight_min": -25.1822509765625,
      "activations/layer22_attention_weight_max": 31.15203857421875,
      "activations/layer22_attention_weight_min": -21.96809959411621,
      "activations/layer23_attention_weight_max": 24.045150756835938,
      "activations/layer23_attention_weight_min": -16.924457550048828,
      "activations/layer2_attention_weight_max": 30.68697738647461,
      "activations/layer2_attention_weight_min": -29.631145477294922,
      "activations/layer3_attention_weight_max": 76.95909881591797,
      "activations/layer3_attention_weight_min": -86.61749267578125,
      "activations/layer4_attention_weight_max": 99.4976806640625,
      "activations/layer4_attention_weight_min": -97.2249984741211,
      "activations/layer5_attention_weight_max": 107.63507080078125,
      "activations/layer5_attention_weight_min": -98.42352294921875,
      "activations/layer6_attention_weight_max": 56.25880432128906,
      "activations/layer6_attention_weight_min": -52.35503387451172,
      "activations/layer7_attention_weight_max": 64.25035858154297,
      "activations/layer7_attention_weight_min": -60.16957092285156,
      "activations/layer8_attention_weight_max": 39.4130744934082,
      "activations/layer8_attention_weight_min": -36.53072738647461,
      "activations/layer9_attention_weight_max": 50.22037124633789,
      "activations/layer9_attention_weight_min": -48.17670440673828,
      "epoch": 4.07,
      "learning_rate": 0.0001250090909090909,
      "loss": 2.8681,
      "step": 70050
    },
    {
      "activations/layer0_attention_weight_max": 15.28345775604248,
      "activations/layer0_attention_weight_min": -15.2582426071167,
      "activations/layer10_attention_weight_max": 40.450294494628906,
      "activations/layer10_attention_weight_min": -35.082489013671875,
      "activations/layer11_attention_weight_max": 36.38175964355469,
      "activations/layer11_attention_weight_min": -27.448501586914062,
      "activations/layer12_attention_weight_max": 27.243030548095703,
      "activations/layer12_attention_weight_min": -23.458024978637695,
      "activations/layer13_attention_weight_max": 55.19913101196289,
      "activations/layer13_attention_weight_min": -39.33943176269531,
      "activations/layer14_attention_weight_max": 66.27542114257812,
      "activations/layer14_attention_weight_min": -44.479522705078125,
      "activations/layer15_attention_weight_max": 53.82240295410156,
      "activations/layer15_attention_weight_min": -37.28167724609375,
      "activations/layer16_attention_weight_max": 34.17726516723633,
      "activations/layer16_attention_weight_min": -25.179561614990234,
      "activations/layer17_attention_weight_max": 56.655033111572266,
      "activations/layer17_attention_weight_min": -36.21370315551758,
      "activations/layer18_attention_weight_max": 50.63790512084961,
      "activations/layer18_attention_weight_min": -30.861228942871094,
      "activations/layer19_attention_weight_max": 21.830312728881836,
      "activations/layer19_attention_weight_min": -18.106998443603516,
      "activations/layer1_attention_weight_max": 17.720077514648438,
      "activations/layer1_attention_weight_min": -14.611923217773438,
      "activations/layer20_attention_weight_max": 21.779071807861328,
      "activations/layer20_attention_weight_min": -18.002023696899414,
      "activations/layer21_attention_weight_max": 36.45207595825195,
      "activations/layer21_attention_weight_min": -25.953947067260742,
      "activations/layer22_attention_weight_max": 37.6311149597168,
      "activations/layer22_attention_weight_min": -21.895233154296875,
      "activations/layer23_attention_weight_max": 28.573379516601562,
      "activations/layer23_attention_weight_min": -15.858667373657227,
      "activations/layer2_attention_weight_max": 29.96276092529297,
      "activations/layer2_attention_weight_min": -28.764179229736328,
      "activations/layer3_attention_weight_max": 79.9267807006836,
      "activations/layer3_attention_weight_min": -81.62674713134766,
      "activations/layer4_attention_weight_max": 104.1142807006836,
      "activations/layer4_attention_weight_min": -95.3243637084961,
      "activations/layer5_attention_weight_max": 122.64281463623047,
      "activations/layer5_attention_weight_min": -102.92434692382812,
      "activations/layer6_attention_weight_max": 61.8519172668457,
      "activations/layer6_attention_weight_min": -53.11082077026367,
      "activations/layer7_attention_weight_max": 74.69039916992188,
      "activations/layer7_attention_weight_min": -63.611106872558594,
      "activations/layer8_attention_weight_max": 48.3453254699707,
      "activations/layer8_attention_weight_min": -39.88228988647461,
      "activations/layer9_attention_weight_max": 61.906978607177734,
      "activations/layer9_attention_weight_min": -56.81791305541992,
      "epoch": 4.07,
      "learning_rate": 0.0001249901515151515,
      "loss": 2.8742,
      "step": 70100
    },
    {
      "activations/layer0_attention_weight_max": 14.223118782043457,
      "activations/layer0_attention_weight_min": -15.530784606933594,
      "activations/layer10_attention_weight_max": 39.94599151611328,
      "activations/layer10_attention_weight_min": -34.145748138427734,
      "activations/layer11_attention_weight_max": 32.344329833984375,
      "activations/layer11_attention_weight_min": -29.034011840820312,
      "activations/layer12_attention_weight_max": 29.162960052490234,
      "activations/layer12_attention_weight_min": -21.886980056762695,
      "activations/layer13_attention_weight_max": 60.206581115722656,
      "activations/layer13_attention_weight_min": -39.72040939331055,
      "activations/layer14_attention_weight_max": 54.321468353271484,
      "activations/layer14_attention_weight_min": -31.845243453979492,
      "activations/layer15_attention_weight_max": 49.117916107177734,
      "activations/layer15_attention_weight_min": -32.8177604675293,
      "activations/layer16_attention_weight_max": 39.07264709472656,
      "activations/layer16_attention_weight_min": -27.877429962158203,
      "activations/layer17_attention_weight_max": 62.35026168823242,
      "activations/layer17_attention_weight_min": -38.44011688232422,
      "activations/layer18_attention_weight_max": 53.96304702758789,
      "activations/layer18_attention_weight_min": -33.421417236328125,
      "activations/layer19_attention_weight_max": 25.69988441467285,
      "activations/layer19_attention_weight_min": -20.502033233642578,
      "activations/layer1_attention_weight_max": 17.885690689086914,
      "activations/layer1_attention_weight_min": -15.124726295471191,
      "activations/layer20_attention_weight_max": 24.905921936035156,
      "activations/layer20_attention_weight_min": -17.85173225402832,
      "activations/layer21_attention_weight_max": 47.5152587890625,
      "activations/layer21_attention_weight_min": -29.813232421875,
      "activations/layer22_attention_weight_max": 39.554405212402344,
      "activations/layer22_attention_weight_min": -24.22089385986328,
      "activations/layer23_attention_weight_max": 26.046567916870117,
      "activations/layer23_attention_weight_min": -16.603343963623047,
      "activations/layer2_attention_weight_max": 30.857662200927734,
      "activations/layer2_attention_weight_min": -30.00191879272461,
      "activations/layer3_attention_weight_max": 80.10700225830078,
      "activations/layer3_attention_weight_min": -84.64966583251953,
      "activations/layer4_attention_weight_max": 105.08251190185547,
      "activations/layer4_attention_weight_min": -101.02488708496094,
      "activations/layer5_attention_weight_max": 111.81197357177734,
      "activations/layer5_attention_weight_min": -103.44185638427734,
      "activations/layer6_attention_weight_max": 59.59846878051758,
      "activations/layer6_attention_weight_min": -57.00859069824219,
      "activations/layer7_attention_weight_max": 71.19869995117188,
      "activations/layer7_attention_weight_min": -65.0113525390625,
      "activations/layer8_attention_weight_max": 42.26884460449219,
      "activations/layer8_attention_weight_min": -38.21849060058594,
      "activations/layer9_attention_weight_max": 63.4132194519043,
      "activations/layer9_attention_weight_min": -46.5562858581543,
      "epoch": 4.08,
      "learning_rate": 0.00012497121212121212,
      "loss": 2.8652,
      "step": 70150
    },
    {
      "activations/layer0_attention_weight_max": 14.433682441711426,
      "activations/layer0_attention_weight_min": -15.404889106750488,
      "activations/layer10_attention_weight_max": 39.07976531982422,
      "activations/layer10_attention_weight_min": -32.579078674316406,
      "activations/layer11_attention_weight_max": 30.553991317749023,
      "activations/layer11_attention_weight_min": -25.930675506591797,
      "activations/layer12_attention_weight_max": 26.35804557800293,
      "activations/layer12_attention_weight_min": -24.903852462768555,
      "activations/layer13_attention_weight_max": 50.688533782958984,
      "activations/layer13_attention_weight_min": -33.322811126708984,
      "activations/layer14_attention_weight_max": 41.437042236328125,
      "activations/layer14_attention_weight_min": -29.943843841552734,
      "activations/layer15_attention_weight_max": 44.61448287963867,
      "activations/layer15_attention_weight_min": -30.59882926940918,
      "activations/layer16_attention_weight_max": 36.360145568847656,
      "activations/layer16_attention_weight_min": -28.466297149658203,
      "activations/layer17_attention_weight_max": 59.7391357421875,
      "activations/layer17_attention_weight_min": -35.7208137512207,
      "activations/layer18_attention_weight_max": 54.5901985168457,
      "activations/layer18_attention_weight_min": -32.564109802246094,
      "activations/layer19_attention_weight_max": 23.145429611206055,
      "activations/layer19_attention_weight_min": -18.19944953918457,
      "activations/layer1_attention_weight_max": 17.169204711914062,
      "activations/layer1_attention_weight_min": -14.89939022064209,
      "activations/layer20_attention_weight_max": 22.644390106201172,
      "activations/layer20_attention_weight_min": -22.169702529907227,
      "activations/layer21_attention_weight_max": 39.80801773071289,
      "activations/layer21_attention_weight_min": -27.238454818725586,
      "activations/layer22_attention_weight_max": 37.244163513183594,
      "activations/layer22_attention_weight_min": -24.195354461669922,
      "activations/layer23_attention_weight_max": 29.2344970703125,
      "activations/layer23_attention_weight_min": -19.83411979675293,
      "activations/layer2_attention_weight_max": 31.490392684936523,
      "activations/layer2_attention_weight_min": -28.851177215576172,
      "activations/layer3_attention_weight_max": 73.44403839111328,
      "activations/layer3_attention_weight_min": -80.27852630615234,
      "activations/layer4_attention_weight_max": 95.99108123779297,
      "activations/layer4_attention_weight_min": -91.11811828613281,
      "activations/layer5_attention_weight_max": 106.66167449951172,
      "activations/layer5_attention_weight_min": -93.61946868896484,
      "activations/layer6_attention_weight_max": 57.93561935424805,
      "activations/layer6_attention_weight_min": -50.94648361206055,
      "activations/layer7_attention_weight_max": 62.704673767089844,
      "activations/layer7_attention_weight_min": -59.29067611694336,
      "activations/layer8_attention_weight_max": 38.05769729614258,
      "activations/layer8_attention_weight_min": -38.50907516479492,
      "activations/layer9_attention_weight_max": 49.07161331176758,
      "activations/layer9_attention_weight_min": -46.581024169921875,
      "epoch": 4.08,
      "learning_rate": 0.0001249522727272727,
      "loss": 2.8682,
      "step": 70200
    },
    {
      "activations/layer0_attention_weight_max": 14.684503555297852,
      "activations/layer0_attention_weight_min": -15.538631439208984,
      "activations/layer10_attention_weight_max": 39.551979064941406,
      "activations/layer10_attention_weight_min": -33.208412170410156,
      "activations/layer11_attention_weight_max": 31.059965133666992,
      "activations/layer11_attention_weight_min": -25.55146026611328,
      "activations/layer12_attention_weight_max": 21.60730743408203,
      "activations/layer12_attention_weight_min": -26.395795822143555,
      "activations/layer13_attention_weight_max": 50.0496711730957,
      "activations/layer13_attention_weight_min": -36.248809814453125,
      "activations/layer14_attention_weight_max": 46.355262756347656,
      "activations/layer14_attention_weight_min": -32.709747314453125,
      "activations/layer15_attention_weight_max": 43.736053466796875,
      "activations/layer15_attention_weight_min": -29.918725967407227,
      "activations/layer16_attention_weight_max": 36.683414459228516,
      "activations/layer16_attention_weight_min": -26.608861923217773,
      "activations/layer17_attention_weight_max": 59.36139678955078,
      "activations/layer17_attention_weight_min": -35.08082962036133,
      "activations/layer18_attention_weight_max": 50.30540084838867,
      "activations/layer18_attention_weight_min": -31.43756103515625,
      "activations/layer19_attention_weight_max": 24.99835205078125,
      "activations/layer19_attention_weight_min": -21.182966232299805,
      "activations/layer1_attention_weight_max": 18.878511428833008,
      "activations/layer1_attention_weight_min": -16.005462646484375,
      "activations/layer20_attention_weight_max": 22.5721492767334,
      "activations/layer20_attention_weight_min": -19.270748138427734,
      "activations/layer21_attention_weight_max": 43.1999626159668,
      "activations/layer21_attention_weight_min": -28.230709075927734,
      "activations/layer22_attention_weight_max": 35.88102340698242,
      "activations/layer22_attention_weight_min": -24.066247940063477,
      "activations/layer23_attention_weight_max": 26.400455474853516,
      "activations/layer23_attention_weight_min": -18.571483612060547,
      "activations/layer2_attention_weight_max": 31.331279754638672,
      "activations/layer2_attention_weight_min": -27.230459213256836,
      "activations/layer3_attention_weight_max": 75.38507843017578,
      "activations/layer3_attention_weight_min": -80.94750213623047,
      "activations/layer4_attention_weight_max": 97.9478530883789,
      "activations/layer4_attention_weight_min": -92.25806427001953,
      "activations/layer5_attention_weight_max": 111.18917083740234,
      "activations/layer5_attention_weight_min": -97.49664306640625,
      "activations/layer6_attention_weight_max": 60.50517272949219,
      "activations/layer6_attention_weight_min": -50.142826080322266,
      "activations/layer7_attention_weight_max": 62.99946975708008,
      "activations/layer7_attention_weight_min": -59.56509017944336,
      "activations/layer8_attention_weight_max": 43.88414001464844,
      "activations/layer8_attention_weight_min": -36.72222137451172,
      "activations/layer9_attention_weight_max": 45.830936431884766,
      "activations/layer9_attention_weight_min": -45.48698806762695,
      "epoch": 4.08,
      "learning_rate": 0.00012493333333333333,
      "loss": 2.8761,
      "step": 70250
    },
    {
      "activations/layer0_attention_weight_max": 14.972892761230469,
      "activations/layer0_attention_weight_min": -15.420109748840332,
      "activations/layer10_attention_weight_max": 37.550437927246094,
      "activations/layer10_attention_weight_min": -34.598480224609375,
      "activations/layer11_attention_weight_max": 30.195396423339844,
      "activations/layer11_attention_weight_min": -27.055034637451172,
      "activations/layer12_attention_weight_max": 26.284257888793945,
      "activations/layer12_attention_weight_min": -25.564048767089844,
      "activations/layer13_attention_weight_max": 50.58732604980469,
      "activations/layer13_attention_weight_min": -35.38668441772461,
      "activations/layer14_attention_weight_max": 53.74976348876953,
      "activations/layer14_attention_weight_min": -35.519874572753906,
      "activations/layer15_attention_weight_max": 44.689945220947266,
      "activations/layer15_attention_weight_min": -30.766765594482422,
      "activations/layer16_attention_weight_max": 30.956693649291992,
      "activations/layer16_attention_weight_min": -25.47337532043457,
      "activations/layer17_attention_weight_max": 55.04795837402344,
      "activations/layer17_attention_weight_min": -37.054649353027344,
      "activations/layer18_attention_weight_max": 42.17197799682617,
      "activations/layer18_attention_weight_min": -31.200336456298828,
      "activations/layer19_attention_weight_max": 20.10226058959961,
      "activations/layer19_attention_weight_min": -18.567630767822266,
      "activations/layer1_attention_weight_max": 19.941219329833984,
      "activations/layer1_attention_weight_min": -18.893444061279297,
      "activations/layer20_attention_weight_max": 19.665931701660156,
      "activations/layer20_attention_weight_min": -22.820276260375977,
      "activations/layer21_attention_weight_max": 37.16152572631836,
      "activations/layer21_attention_weight_min": -27.094810485839844,
      "activations/layer22_attention_weight_max": 31.85037612915039,
      "activations/layer22_attention_weight_min": -23.42238426208496,
      "activations/layer23_attention_weight_max": 24.123334884643555,
      "activations/layer23_attention_weight_min": -17.697370529174805,
      "activations/layer2_attention_weight_max": 33.055908203125,
      "activations/layer2_attention_weight_min": -29.28687286376953,
      "activations/layer3_attention_weight_max": 79.03685760498047,
      "activations/layer3_attention_weight_min": -82.82464599609375,
      "activations/layer4_attention_weight_max": 104.66241455078125,
      "activations/layer4_attention_weight_min": -99.0239028930664,
      "activations/layer5_attention_weight_max": 118.17327880859375,
      "activations/layer5_attention_weight_min": -106.62321472167969,
      "activations/layer6_attention_weight_max": 62.867393493652344,
      "activations/layer6_attention_weight_min": -55.133880615234375,
      "activations/layer7_attention_weight_max": 67.90462493896484,
      "activations/layer7_attention_weight_min": -61.20675277709961,
      "activations/layer8_attention_weight_max": 39.78419876098633,
      "activations/layer8_attention_weight_min": -38.4734992980957,
      "activations/layer9_attention_weight_max": 54.49686813354492,
      "activations/layer9_attention_weight_min": -46.80133819580078,
      "epoch": 4.08,
      "learning_rate": 0.00012491439393939394,
      "loss": 2.8605,
      "step": 70300
    },
    {
      "activations/layer0_attention_weight_max": 15.009977340698242,
      "activations/layer0_attention_weight_min": -15.155680656433105,
      "activations/layer10_attention_weight_max": 38.04986572265625,
      "activations/layer10_attention_weight_min": -34.33265686035156,
      "activations/layer11_attention_weight_max": 28.38751792907715,
      "activations/layer11_attention_weight_min": -26.406692504882812,
      "activations/layer12_attention_weight_max": 24.125391006469727,
      "activations/layer12_attention_weight_min": -24.18287467956543,
      "activations/layer13_attention_weight_max": 47.900291442871094,
      "activations/layer13_attention_weight_min": -36.8621826171875,
      "activations/layer14_attention_weight_max": 40.25960159301758,
      "activations/layer14_attention_weight_min": -29.931007385253906,
      "activations/layer15_attention_weight_max": 41.09050750732422,
      "activations/layer15_attention_weight_min": -31.7553653717041,
      "activations/layer16_attention_weight_max": 32.26396942138672,
      "activations/layer16_attention_weight_min": -30.720232009887695,
      "activations/layer17_attention_weight_max": 54.41463851928711,
      "activations/layer17_attention_weight_min": -38.85280990600586,
      "activations/layer18_attention_weight_max": 47.20060729980469,
      "activations/layer18_attention_weight_min": -32.436737060546875,
      "activations/layer19_attention_weight_max": 21.66132164001465,
      "activations/layer19_attention_weight_min": -17.281757354736328,
      "activations/layer1_attention_weight_max": 18.82769203186035,
      "activations/layer1_attention_weight_min": -15.595504760742188,
      "activations/layer20_attention_weight_max": 19.256988525390625,
      "activations/layer20_attention_weight_min": -23.99384880065918,
      "activations/layer21_attention_weight_max": 36.58466339111328,
      "activations/layer21_attention_weight_min": -27.894811630249023,
      "activations/layer22_attention_weight_max": 32.422733306884766,
      "activations/layer22_attention_weight_min": -25.92449951171875,
      "activations/layer23_attention_weight_max": 24.862274169921875,
      "activations/layer23_attention_weight_min": -16.76249885559082,
      "activations/layer2_attention_weight_max": 30.980871200561523,
      "activations/layer2_attention_weight_min": -28.911148071289062,
      "activations/layer3_attention_weight_max": 80.55717468261719,
      "activations/layer3_attention_weight_min": -81.34178161621094,
      "activations/layer4_attention_weight_max": 99.97911834716797,
      "activations/layer4_attention_weight_min": -95.23006439208984,
      "activations/layer5_attention_weight_max": 109.62715911865234,
      "activations/layer5_attention_weight_min": -96.53131103515625,
      "activations/layer6_attention_weight_max": 58.076255798339844,
      "activations/layer6_attention_weight_min": -54.143497467041016,
      "activations/layer7_attention_weight_max": 62.685081481933594,
      "activations/layer7_attention_weight_min": -62.32671356201172,
      "activations/layer8_attention_weight_max": 38.515419006347656,
      "activations/layer8_attention_weight_min": -38.67363357543945,
      "activations/layer9_attention_weight_max": 48.84487533569336,
      "activations/layer9_attention_weight_min": -44.33683395385742,
      "epoch": 4.09,
      "learning_rate": 0.00012489583333333333,
      "loss": 2.8634,
      "step": 70350
    },
    {
      "activations/layer0_attention_weight_max": 13.744625091552734,
      "activations/layer0_attention_weight_min": -15.219362258911133,
      "activations/layer10_attention_weight_max": 41.94721984863281,
      "activations/layer10_attention_weight_min": -33.76207733154297,
      "activations/layer11_attention_weight_max": 38.561912536621094,
      "activations/layer11_attention_weight_min": -26.962623596191406,
      "activations/layer12_attention_weight_max": 25.534259796142578,
      "activations/layer12_attention_weight_min": -23.635086059570312,
      "activations/layer13_attention_weight_max": 80.28726959228516,
      "activations/layer13_attention_weight_min": -48.33763122558594,
      "activations/layer14_attention_weight_max": 73.58122253417969,
      "activations/layer14_attention_weight_min": -39.5186653137207,
      "activations/layer15_attention_weight_max": 60.771278381347656,
      "activations/layer15_attention_weight_min": -32.78074264526367,
      "activations/layer16_attention_weight_max": 38.67593002319336,
      "activations/layer16_attention_weight_min": -25.401416778564453,
      "activations/layer17_attention_weight_max": 57.51919174194336,
      "activations/layer17_attention_weight_min": -35.25563430786133,
      "activations/layer18_attention_weight_max": 49.36489486694336,
      "activations/layer18_attention_weight_min": -31.08245277404785,
      "activations/layer19_attention_weight_max": 28.16763687133789,
      "activations/layer19_attention_weight_min": -19.891891479492188,
      "activations/layer1_attention_weight_max": 18.232343673706055,
      "activations/layer1_attention_weight_min": -14.76742172241211,
      "activations/layer20_attention_weight_max": 24.37112808227539,
      "activations/layer20_attention_weight_min": -20.940011978149414,
      "activations/layer21_attention_weight_max": 44.75483703613281,
      "activations/layer21_attention_weight_min": -26.768144607543945,
      "activations/layer22_attention_weight_max": 34.33241271972656,
      "activations/layer22_attention_weight_min": -21.6249942779541,
      "activations/layer23_attention_weight_max": 29.8020076751709,
      "activations/layer23_attention_weight_min": -19.160655975341797,
      "activations/layer2_attention_weight_max": 30.504905700683594,
      "activations/layer2_attention_weight_min": -27.680585861206055,
      "activations/layer3_attention_weight_max": 79.11688232421875,
      "activations/layer3_attention_weight_min": -81.52909088134766,
      "activations/layer4_attention_weight_max": 101.0759506225586,
      "activations/layer4_attention_weight_min": -91.65962982177734,
      "activations/layer5_attention_weight_max": 114.54647827148438,
      "activations/layer5_attention_weight_min": -100.17086029052734,
      "activations/layer6_attention_weight_max": 63.79479217529297,
      "activations/layer6_attention_weight_min": -57.94630813598633,
      "activations/layer7_attention_weight_max": 76.74759674072266,
      "activations/layer7_attention_weight_min": -66.8696517944336,
      "activations/layer8_attention_weight_max": 47.54861068725586,
      "activations/layer8_attention_weight_min": -40.17926788330078,
      "activations/layer9_attention_weight_max": 68.63375091552734,
      "activations/layer9_attention_weight_min": -52.8095703125,
      "epoch": 4.09,
      "learning_rate": 0.00012487689393939392,
      "loss": 2.8692,
      "step": 70400
    },
    {
      "activations/layer0_attention_weight_max": 14.842144012451172,
      "activations/layer0_attention_weight_min": -14.702354431152344,
      "activations/layer10_attention_weight_max": 42.18687438964844,
      "activations/layer10_attention_weight_min": -31.731101989746094,
      "activations/layer11_attention_weight_max": 32.008506774902344,
      "activations/layer11_attention_weight_min": -24.90106964111328,
      "activations/layer12_attention_weight_max": 31.769397735595703,
      "activations/layer12_attention_weight_min": -26.98784637451172,
      "activations/layer13_attention_weight_max": 54.82185363769531,
      "activations/layer13_attention_weight_min": -36.99821090698242,
      "activations/layer14_attention_weight_max": 53.61859893798828,
      "activations/layer14_attention_weight_min": -29.177446365356445,
      "activations/layer15_attention_weight_max": 48.33322525024414,
      "activations/layer15_attention_weight_min": -29.57914161682129,
      "activations/layer16_attention_weight_max": 47.466392517089844,
      "activations/layer16_attention_weight_min": -28.31614875793457,
      "activations/layer17_attention_weight_max": 61.00384521484375,
      "activations/layer17_attention_weight_min": -36.56837463378906,
      "activations/layer18_attention_weight_max": 59.7967529296875,
      "activations/layer18_attention_weight_min": -34.367408752441406,
      "activations/layer19_attention_weight_max": 22.26051139831543,
      "activations/layer19_attention_weight_min": -18.19547462463379,
      "activations/layer1_attention_weight_max": 17.799976348876953,
      "activations/layer1_attention_weight_min": -14.316325187683105,
      "activations/layer20_attention_weight_max": 22.039365768432617,
      "activations/layer20_attention_weight_min": -18.672740936279297,
      "activations/layer21_attention_weight_max": 50.84825134277344,
      "activations/layer21_attention_weight_min": -27.856964111328125,
      "activations/layer22_attention_weight_max": 38.51294708251953,
      "activations/layer22_attention_weight_min": -21.915508270263672,
      "activations/layer23_attention_weight_max": 28.9752197265625,
      "activations/layer23_attention_weight_min": -17.10405731201172,
      "activations/layer2_attention_weight_max": 28.738143920898438,
      "activations/layer2_attention_weight_min": -26.49135971069336,
      "activations/layer3_attention_weight_max": 80.36970520019531,
      "activations/layer3_attention_weight_min": -79.34049224853516,
      "activations/layer4_attention_weight_max": 104.0328369140625,
      "activations/layer4_attention_weight_min": -98.50453186035156,
      "activations/layer5_attention_weight_max": 111.46914672851562,
      "activations/layer5_attention_weight_min": -98.81852722167969,
      "activations/layer6_attention_weight_max": 59.18112564086914,
      "activations/layer6_attention_weight_min": -52.89209747314453,
      "activations/layer7_attention_weight_max": 67.158447265625,
      "activations/layer7_attention_weight_min": -58.516483306884766,
      "activations/layer8_attention_weight_max": 39.424774169921875,
      "activations/layer8_attention_weight_min": -37.518592834472656,
      "activations/layer9_attention_weight_max": 50.015708923339844,
      "activations/layer9_attention_weight_min": -42.30417251586914,
      "epoch": 4.09,
      "learning_rate": 0.00012485795454545453,
      "loss": 2.8776,
      "step": 70450
    },
    {
      "activations/layer0_attention_weight_max": 15.340249061584473,
      "activations/layer0_attention_weight_min": -14.65552043914795,
      "activations/layer10_attention_weight_max": 39.2474365234375,
      "activations/layer10_attention_weight_min": -31.96014404296875,
      "activations/layer11_attention_weight_max": 32.233489990234375,
      "activations/layer11_attention_weight_min": -25.36659812927246,
      "activations/layer12_attention_weight_max": 22.492916107177734,
      "activations/layer12_attention_weight_min": -25.21564292907715,
      "activations/layer13_attention_weight_max": 51.89442825317383,
      "activations/layer13_attention_weight_min": -36.451290130615234,
      "activations/layer14_attention_weight_max": 40.6294059753418,
      "activations/layer14_attention_weight_min": -29.552623748779297,
      "activations/layer15_attention_weight_max": 50.75313949584961,
      "activations/layer15_attention_weight_min": -28.905107498168945,
      "activations/layer16_attention_weight_max": 36.94866943359375,
      "activations/layer16_attention_weight_min": -25.77484130859375,
      "activations/layer17_attention_weight_max": 61.17326736450195,
      "activations/layer17_attention_weight_min": -37.08081817626953,
      "activations/layer18_attention_weight_max": 50.114288330078125,
      "activations/layer18_attention_weight_min": -31.072837829589844,
      "activations/layer19_attention_weight_max": 25.817169189453125,
      "activations/layer19_attention_weight_min": -16.98139762878418,
      "activations/layer1_attention_weight_max": 18.33658790588379,
      "activations/layer1_attention_weight_min": -16.798786163330078,
      "activations/layer20_attention_weight_max": 20.25977325439453,
      "activations/layer20_attention_weight_min": -21.346092224121094,
      "activations/layer21_attention_weight_max": 36.324440002441406,
      "activations/layer21_attention_weight_min": -23.537145614624023,
      "activations/layer22_attention_weight_max": 32.00095748901367,
      "activations/layer22_attention_weight_min": -23.447874069213867,
      "activations/layer23_attention_weight_max": 25.67822265625,
      "activations/layer23_attention_weight_min": -17.046466827392578,
      "activations/layer2_attention_weight_max": 29.664688110351562,
      "activations/layer2_attention_weight_min": -26.162214279174805,
      "activations/layer3_attention_weight_max": 75.60212707519531,
      "activations/layer3_attention_weight_min": -75.2650375366211,
      "activations/layer4_attention_weight_max": 97.1649169921875,
      "activations/layer4_attention_weight_min": -90.55782318115234,
      "activations/layer5_attention_weight_max": 108.90339660644531,
      "activations/layer5_attention_weight_min": -99.57209777832031,
      "activations/layer6_attention_weight_max": 57.54188537597656,
      "activations/layer6_attention_weight_min": -50.46092224121094,
      "activations/layer7_attention_weight_max": 66.74935150146484,
      "activations/layer7_attention_weight_min": -58.172996520996094,
      "activations/layer8_attention_weight_max": 40.07515335083008,
      "activations/layer8_attention_weight_min": -37.27278137207031,
      "activations/layer9_attention_weight_max": 47.82106399536133,
      "activations/layer9_attention_weight_min": -43.28704071044922,
      "epoch": 4.1,
      "learning_rate": 0.00012483901515151515,
      "loss": 2.8785,
      "step": 70500
    },
    {
      "activations/layer0_attention_weight_max": 13.419530868530273,
      "activations/layer0_attention_weight_min": -14.911659240722656,
      "activations/layer10_attention_weight_max": 39.929073333740234,
      "activations/layer10_attention_weight_min": -33.628692626953125,
      "activations/layer11_attention_weight_max": 30.456039428710938,
      "activations/layer11_attention_weight_min": -26.792190551757812,
      "activations/layer12_attention_weight_max": 23.87021255493164,
      "activations/layer12_attention_weight_min": -26.145130157470703,
      "activations/layer13_attention_weight_max": 48.683101654052734,
      "activations/layer13_attention_weight_min": -35.541709899902344,
      "activations/layer14_attention_weight_max": 37.156612396240234,
      "activations/layer14_attention_weight_min": -27.109607696533203,
      "activations/layer15_attention_weight_max": 48.40693664550781,
      "activations/layer15_attention_weight_min": -31.775985717773438,
      "activations/layer16_attention_weight_max": 37.06608963012695,
      "activations/layer16_attention_weight_min": -25.035486221313477,
      "activations/layer17_attention_weight_max": 56.84214401245117,
      "activations/layer17_attention_weight_min": -37.0450325012207,
      "activations/layer18_attention_weight_max": 46.91688537597656,
      "activations/layer18_attention_weight_min": -29.778873443603516,
      "activations/layer19_attention_weight_max": 26.1726131439209,
      "activations/layer19_attention_weight_min": -17.574514389038086,
      "activations/layer1_attention_weight_max": 17.51533317565918,
      "activations/layer1_attention_weight_min": -18.49413299560547,
      "activations/layer20_attention_weight_max": 22.191707611083984,
      "activations/layer20_attention_weight_min": -20.02444839477539,
      "activations/layer21_attention_weight_max": 37.956939697265625,
      "activations/layer21_attention_weight_min": -30.511816024780273,
      "activations/layer22_attention_weight_max": 31.169296264648438,
      "activations/layer22_attention_weight_min": -23.866676330566406,
      "activations/layer23_attention_weight_max": 23.509151458740234,
      "activations/layer23_attention_weight_min": -17.05099105834961,
      "activations/layer2_attention_weight_max": 30.36878776550293,
      "activations/layer2_attention_weight_min": -29.060379028320312,
      "activations/layer3_attention_weight_max": 74.9764404296875,
      "activations/layer3_attention_weight_min": -82.65394592285156,
      "activations/layer4_attention_weight_max": 98.29601287841797,
      "activations/layer4_attention_weight_min": -95.54473114013672,
      "activations/layer5_attention_weight_max": 108.56631469726562,
      "activations/layer5_attention_weight_min": -96.60044860839844,
      "activations/layer6_attention_weight_max": 59.30307388305664,
      "activations/layer6_attention_weight_min": -53.5893669128418,
      "activations/layer7_attention_weight_max": 63.357322692871094,
      "activations/layer7_attention_weight_min": -60.239532470703125,
      "activations/layer8_attention_weight_max": 40.52691650390625,
      "activations/layer8_attention_weight_min": -39.743614196777344,
      "activations/layer9_attention_weight_max": 50.64328384399414,
      "activations/layer9_attention_weight_min": -47.848262786865234,
      "epoch": 4.1,
      "learning_rate": 0.00012482007575757574,
      "loss": 2.875,
      "step": 70550
    },
    {
      "activations/layer0_attention_weight_max": 15.172661781311035,
      "activations/layer0_attention_weight_min": -15.187026977539062,
      "activations/layer10_attention_weight_max": 38.364715576171875,
      "activations/layer10_attention_weight_min": -32.23847198486328,
      "activations/layer11_attention_weight_max": 34.767669677734375,
      "activations/layer11_attention_weight_min": -26.835384368896484,
      "activations/layer12_attention_weight_max": 24.436702728271484,
      "activations/layer12_attention_weight_min": -26.278614044189453,
      "activations/layer13_attention_weight_max": 46.52974319458008,
      "activations/layer13_attention_weight_min": -34.68514633178711,
      "activations/layer14_attention_weight_max": 42.68229293823242,
      "activations/layer14_attention_weight_min": -31.09961700439453,
      "activations/layer15_attention_weight_max": 44.038490295410156,
      "activations/layer15_attention_weight_min": -30.42666244506836,
      "activations/layer16_attention_weight_max": 35.535892486572266,
      "activations/layer16_attention_weight_min": -28.005887985229492,
      "activations/layer17_attention_weight_max": 62.96181106567383,
      "activations/layer17_attention_weight_min": -36.05315017700195,
      "activations/layer18_attention_weight_max": 52.906898498535156,
      "activations/layer18_attention_weight_min": -32.60383224487305,
      "activations/layer19_attention_weight_max": 27.24848747253418,
      "activations/layer19_attention_weight_min": -18.079662322998047,
      "activations/layer1_attention_weight_max": 17.158935546875,
      "activations/layer1_attention_weight_min": -16.388286590576172,
      "activations/layer20_attention_weight_max": 24.852622985839844,
      "activations/layer20_attention_weight_min": -20.757551193237305,
      "activations/layer21_attention_weight_max": 42.1577033996582,
      "activations/layer21_attention_weight_min": -25.029953002929688,
      "activations/layer22_attention_weight_max": 35.544944763183594,
      "activations/layer22_attention_weight_min": -25.14176368713379,
      "activations/layer23_attention_weight_max": 31.361392974853516,
      "activations/layer23_attention_weight_min": -17.776281356811523,
      "activations/layer2_attention_weight_max": 29.366729736328125,
      "activations/layer2_attention_weight_min": -26.484603881835938,
      "activations/layer3_attention_weight_max": 73.1235122680664,
      "activations/layer3_attention_weight_min": -79.40660858154297,
      "activations/layer4_attention_weight_max": 96.88407135009766,
      "activations/layer4_attention_weight_min": -96.70350646972656,
      "activations/layer5_attention_weight_max": 108.40111541748047,
      "activations/layer5_attention_weight_min": -98.06053924560547,
      "activations/layer6_attention_weight_max": 56.7343864440918,
      "activations/layer6_attention_weight_min": -51.963661193847656,
      "activations/layer7_attention_weight_max": 69.9384536743164,
      "activations/layer7_attention_weight_min": -58.7805290222168,
      "activations/layer8_attention_weight_max": 40.38222122192383,
      "activations/layer8_attention_weight_min": -37.54357147216797,
      "activations/layer9_attention_weight_max": 66.399658203125,
      "activations/layer9_attention_weight_min": -46.72564697265625,
      "epoch": 4.1,
      "learning_rate": 0.00012480113636363635,
      "loss": 2.8602,
      "step": 70600
    },
    {
      "activations/layer0_attention_weight_max": 14.71777629852295,
      "activations/layer0_attention_weight_min": -15.240144729614258,
      "activations/layer10_attention_weight_max": 41.42724609375,
      "activations/layer10_attention_weight_min": -33.243003845214844,
      "activations/layer11_attention_weight_max": 32.92494583129883,
      "activations/layer11_attention_weight_min": -26.177120208740234,
      "activations/layer12_attention_weight_max": 23.989824295043945,
      "activations/layer12_attention_weight_min": -25.780485153198242,
      "activations/layer13_attention_weight_max": 49.8747444152832,
      "activations/layer13_attention_weight_min": -38.26555633544922,
      "activations/layer14_attention_weight_max": 40.840267181396484,
      "activations/layer14_attention_weight_min": -31.87938690185547,
      "activations/layer15_attention_weight_max": 43.36630630493164,
      "activations/layer15_attention_weight_min": -31.887914657592773,
      "activations/layer16_attention_weight_max": 37.922428131103516,
      "activations/layer16_attention_weight_min": -25.05126190185547,
      "activations/layer17_attention_weight_max": 57.04608917236328,
      "activations/layer17_attention_weight_min": -35.41044998168945,
      "activations/layer18_attention_weight_max": 47.77740478515625,
      "activations/layer18_attention_weight_min": -31.393070220947266,
      "activations/layer19_attention_weight_max": 22.9928035736084,
      "activations/layer19_attention_weight_min": -17.0225887298584,
      "activations/layer1_attention_weight_max": 17.856334686279297,
      "activations/layer1_attention_weight_min": -15.50236988067627,
      "activations/layer20_attention_weight_max": 20.834291458129883,
      "activations/layer20_attention_weight_min": -19.672840118408203,
      "activations/layer21_attention_weight_max": 38.83625030517578,
      "activations/layer21_attention_weight_min": -23.84086799621582,
      "activations/layer22_attention_weight_max": 37.17977523803711,
      "activations/layer22_attention_weight_min": -25.21316146850586,
      "activations/layer23_attention_weight_max": 25.827085494995117,
      "activations/layer23_attention_weight_min": -17.469083786010742,
      "activations/layer2_attention_weight_max": 30.283550262451172,
      "activations/layer2_attention_weight_min": -27.97119140625,
      "activations/layer3_attention_weight_max": 75.72908782958984,
      "activations/layer3_attention_weight_min": -82.9840316772461,
      "activations/layer4_attention_weight_max": 97.66596221923828,
      "activations/layer4_attention_weight_min": -90.21296691894531,
      "activations/layer5_attention_weight_max": 109.17431640625,
      "activations/layer5_attention_weight_min": -97.29637908935547,
      "activations/layer6_attention_weight_max": 57.032352447509766,
      "activations/layer6_attention_weight_min": -52.35536575317383,
      "activations/layer7_attention_weight_max": 61.37930679321289,
      "activations/layer7_attention_weight_min": -58.73222351074219,
      "activations/layer8_attention_weight_max": 39.75154495239258,
      "activations/layer8_attention_weight_min": -39.965545654296875,
      "activations/layer9_attention_weight_max": 47.52840805053711,
      "activations/layer9_attention_weight_min": -43.81108093261719,
      "epoch": 4.11,
      "learning_rate": 0.00012478219696969694,
      "loss": 2.868,
      "step": 70650
    },
    {
      "activations/layer0_attention_weight_max": 15.106878280639648,
      "activations/layer0_attention_weight_min": -14.997355461120605,
      "activations/layer10_attention_weight_max": 41.05475616455078,
      "activations/layer10_attention_weight_min": -32.625450134277344,
      "activations/layer11_attention_weight_max": 34.72100830078125,
      "activations/layer11_attention_weight_min": -26.342041015625,
      "activations/layer12_attention_weight_max": 27.683509826660156,
      "activations/layer12_attention_weight_min": -26.058712005615234,
      "activations/layer13_attention_weight_max": 52.9260368347168,
      "activations/layer13_attention_weight_min": -37.242431640625,
      "activations/layer14_attention_weight_max": 66.71803283691406,
      "activations/layer14_attention_weight_min": -38.36558151245117,
      "activations/layer15_attention_weight_max": 44.054443359375,
      "activations/layer15_attention_weight_min": -29.012718200683594,
      "activations/layer16_attention_weight_max": 38.89566421508789,
      "activations/layer16_attention_weight_min": -25.064542770385742,
      "activations/layer17_attention_weight_max": 58.91303253173828,
      "activations/layer17_attention_weight_min": -35.17350769042969,
      "activations/layer18_attention_weight_max": 48.912601470947266,
      "activations/layer18_attention_weight_min": -30.602073669433594,
      "activations/layer19_attention_weight_max": 23.431474685668945,
      "activations/layer19_attention_weight_min": -21.956140518188477,
      "activations/layer1_attention_weight_max": 17.887300491333008,
      "activations/layer1_attention_weight_min": -15.543268203735352,
      "activations/layer20_attention_weight_max": 24.3662109375,
      "activations/layer20_attention_weight_min": -18.801095962524414,
      "activations/layer21_attention_weight_max": 43.18715286254883,
      "activations/layer21_attention_weight_min": -26.190465927124023,
      "activations/layer22_attention_weight_max": 37.149658203125,
      "activations/layer22_attention_weight_min": -22.917022705078125,
      "activations/layer23_attention_weight_max": 27.56311798095703,
      "activations/layer23_attention_weight_min": -17.555320739746094,
      "activations/layer2_attention_weight_max": 30.414777755737305,
      "activations/layer2_attention_weight_min": -28.343488693237305,
      "activations/layer3_attention_weight_max": 77.61029815673828,
      "activations/layer3_attention_weight_min": -80.15967559814453,
      "activations/layer4_attention_weight_max": 104.53973388671875,
      "activations/layer4_attention_weight_min": -95.73268127441406,
      "activations/layer5_attention_weight_max": 116.58427429199219,
      "activations/layer5_attention_weight_min": -97.37065124511719,
      "activations/layer6_attention_weight_max": 60.52643585205078,
      "activations/layer6_attention_weight_min": -53.77776336669922,
      "activations/layer7_attention_weight_max": 63.94837951660156,
      "activations/layer7_attention_weight_min": -62.56396484375,
      "activations/layer8_attention_weight_max": 42.1930046081543,
      "activations/layer8_attention_weight_min": -38.5207633972168,
      "activations/layer9_attention_weight_max": 52.581146240234375,
      "activations/layer9_attention_weight_min": -44.76072692871094,
      "epoch": 4.11,
      "learning_rate": 0.00012476325757575756,
      "loss": 2.8708,
      "step": 70700
    },
    {
      "activations/layer0_attention_weight_max": 17.468276977539062,
      "activations/layer0_attention_weight_min": -15.125393867492676,
      "activations/layer10_attention_weight_max": 39.18764877319336,
      "activations/layer10_attention_weight_min": -34.27496337890625,
      "activations/layer11_attention_weight_max": 31.630117416381836,
      "activations/layer11_attention_weight_min": -26.299428939819336,
      "activations/layer12_attention_weight_max": 23.4052734375,
      "activations/layer12_attention_weight_min": -25.184955596923828,
      "activations/layer13_attention_weight_max": 52.51089859008789,
      "activations/layer13_attention_weight_min": -36.53445816040039,
      "activations/layer14_attention_weight_max": 51.29263687133789,
      "activations/layer14_attention_weight_min": -34.75381851196289,
      "activations/layer15_attention_weight_max": 41.91880798339844,
      "activations/layer15_attention_weight_min": -30.261995315551758,
      "activations/layer16_attention_weight_max": 40.992759704589844,
      "activations/layer16_attention_weight_min": -26.69338035583496,
      "activations/layer17_attention_weight_max": 58.4415283203125,
      "activations/layer17_attention_weight_min": -36.362403869628906,
      "activations/layer18_attention_weight_max": 49.483726501464844,
      "activations/layer18_attention_weight_min": -31.06391716003418,
      "activations/layer19_attention_weight_max": 25.664209365844727,
      "activations/layer19_attention_weight_min": -17.527944564819336,
      "activations/layer1_attention_weight_max": 17.532941818237305,
      "activations/layer1_attention_weight_min": -15.945758819580078,
      "activations/layer20_attention_weight_max": 22.704553604125977,
      "activations/layer20_attention_weight_min": -21.081010818481445,
      "activations/layer21_attention_weight_max": 41.668060302734375,
      "activations/layer21_attention_weight_min": -24.326400756835938,
      "activations/layer22_attention_weight_max": 34.45907974243164,
      "activations/layer22_attention_weight_min": -23.43674087524414,
      "activations/layer23_attention_weight_max": 27.973255157470703,
      "activations/layer23_attention_weight_min": -18.90226173400879,
      "activations/layer2_attention_weight_max": 31.125568389892578,
      "activations/layer2_attention_weight_min": -28.969839096069336,
      "activations/layer3_attention_weight_max": 77.88602447509766,
      "activations/layer3_attention_weight_min": -80.83927154541016,
      "activations/layer4_attention_weight_max": 102.02513885498047,
      "activations/layer4_attention_weight_min": -97.11983489990234,
      "activations/layer5_attention_weight_max": 111.23104095458984,
      "activations/layer5_attention_weight_min": -99.30962371826172,
      "activations/layer6_attention_weight_max": 58.139930725097656,
      "activations/layer6_attention_weight_min": -53.653358459472656,
      "activations/layer7_attention_weight_max": 65.89212799072266,
      "activations/layer7_attention_weight_min": -60.17263412475586,
      "activations/layer8_attention_weight_max": 40.13667297363281,
      "activations/layer8_attention_weight_min": -39.398170471191406,
      "activations/layer9_attention_weight_max": 52.2025032043457,
      "activations/layer9_attention_weight_min": -49.01492691040039,
      "epoch": 4.11,
      "learning_rate": 0.00012474431818181817,
      "loss": 2.8566,
      "step": 70750
    },
    {
      "activations/layer0_attention_weight_max": 14.395308494567871,
      "activations/layer0_attention_weight_min": -14.951838493347168,
      "activations/layer10_attention_weight_max": 41.560150146484375,
      "activations/layer10_attention_weight_min": -33.653350830078125,
      "activations/layer11_attention_weight_max": 30.90616226196289,
      "activations/layer11_attention_weight_min": -26.75857925415039,
      "activations/layer12_attention_weight_max": 25.58062171936035,
      "activations/layer12_attention_weight_min": -24.986366271972656,
      "activations/layer13_attention_weight_max": 48.2830924987793,
      "activations/layer13_attention_weight_min": -40.86745834350586,
      "activations/layer14_attention_weight_max": 52.08863830566406,
      "activations/layer14_attention_weight_min": -40.741600036621094,
      "activations/layer15_attention_weight_max": 40.737220764160156,
      "activations/layer15_attention_weight_min": -29.901458740234375,
      "activations/layer16_attention_weight_max": 36.77853012084961,
      "activations/layer16_attention_weight_min": -26.838184356689453,
      "activations/layer17_attention_weight_max": 58.01940155029297,
      "activations/layer17_attention_weight_min": -35.435001373291016,
      "activations/layer18_attention_weight_max": 47.02300262451172,
      "activations/layer18_attention_weight_min": -33.584877014160156,
      "activations/layer19_attention_weight_max": 24.764448165893555,
      "activations/layer19_attention_weight_min": -18.77263069152832,
      "activations/layer1_attention_weight_max": 17.15046501159668,
      "activations/layer1_attention_weight_min": -15.574902534484863,
      "activations/layer20_attention_weight_max": 22.627107620239258,
      "activations/layer20_attention_weight_min": -20.701343536376953,
      "activations/layer21_attention_weight_max": 40.69374465942383,
      "activations/layer21_attention_weight_min": -29.988922119140625,
      "activations/layer22_attention_weight_max": 38.16580581665039,
      "activations/layer22_attention_weight_min": -23.35972023010254,
      "activations/layer23_attention_weight_max": 27.110244750976562,
      "activations/layer23_attention_weight_min": -18.228328704833984,
      "activations/layer2_attention_weight_max": 29.555707931518555,
      "activations/layer2_attention_weight_min": -26.014877319335938,
      "activations/layer3_attention_weight_max": 74.80968475341797,
      "activations/layer3_attention_weight_min": -79.99396514892578,
      "activations/layer4_attention_weight_max": 99.60707092285156,
      "activations/layer4_attention_weight_min": -91.6879653930664,
      "activations/layer5_attention_weight_max": 113.60577392578125,
      "activations/layer5_attention_weight_min": -95.54779052734375,
      "activations/layer6_attention_weight_max": 61.87835693359375,
      "activations/layer6_attention_weight_min": -51.50844192504883,
      "activations/layer7_attention_weight_max": 66.08183288574219,
      "activations/layer7_attention_weight_min": -65.3034896850586,
      "activations/layer8_attention_weight_max": 41.70144271850586,
      "activations/layer8_attention_weight_min": -41.32359313964844,
      "activations/layer9_attention_weight_max": 54.96552276611328,
      "activations/layer9_attention_weight_min": -45.47892379760742,
      "epoch": 4.11,
      "learning_rate": 0.00012472537878787876,
      "loss": 2.8671,
      "step": 70800
    },
    {
      "activations/layer0_attention_weight_max": 14.114505767822266,
      "activations/layer0_attention_weight_min": -15.667551040649414,
      "activations/layer10_attention_weight_max": 39.63377380371094,
      "activations/layer10_attention_weight_min": -33.09086227416992,
      "activations/layer11_attention_weight_max": 30.960119247436523,
      "activations/layer11_attention_weight_min": -26.7648983001709,
      "activations/layer12_attention_weight_max": 25.577280044555664,
      "activations/layer12_attention_weight_min": -28.062599182128906,
      "activations/layer13_attention_weight_max": 56.29399490356445,
      "activations/layer13_attention_weight_min": -37.94297790527344,
      "activations/layer14_attention_weight_max": 40.072322845458984,
      "activations/layer14_attention_weight_min": -28.50274658203125,
      "activations/layer15_attention_weight_max": 46.27645492553711,
      "activations/layer15_attention_weight_min": -30.705862045288086,
      "activations/layer16_attention_weight_max": 37.42728042602539,
      "activations/layer16_attention_weight_min": -25.97327423095703,
      "activations/layer17_attention_weight_max": 59.32276153564453,
      "activations/layer17_attention_weight_min": -37.267189025878906,
      "activations/layer18_attention_weight_max": 50.001441955566406,
      "activations/layer18_attention_weight_min": -31.2058048248291,
      "activations/layer19_attention_weight_max": 25.494394302368164,
      "activations/layer19_attention_weight_min": -18.098031997680664,
      "activations/layer1_attention_weight_max": 18.74917221069336,
      "activations/layer1_attention_weight_min": -15.52879810333252,
      "activations/layer20_attention_weight_max": 23.675647735595703,
      "activations/layer20_attention_weight_min": -23.84969711303711,
      "activations/layer21_attention_weight_max": 38.00148010253906,
      "activations/layer21_attention_weight_min": -25.635662078857422,
      "activations/layer22_attention_weight_max": 33.436397552490234,
      "activations/layer22_attention_weight_min": -24.716388702392578,
      "activations/layer23_attention_weight_max": 25.98798179626465,
      "activations/layer23_attention_weight_min": -18.641983032226562,
      "activations/layer2_attention_weight_max": 29.532142639160156,
      "activations/layer2_attention_weight_min": -26.56838607788086,
      "activations/layer3_attention_weight_max": 76.40264892578125,
      "activations/layer3_attention_weight_min": -84.51568603515625,
      "activations/layer4_attention_weight_max": 105.55852508544922,
      "activations/layer4_attention_weight_min": -98.43480682373047,
      "activations/layer5_attention_weight_max": 117.45755004882812,
      "activations/layer5_attention_weight_min": -98.5057601928711,
      "activations/layer6_attention_weight_max": 58.08794403076172,
      "activations/layer6_attention_weight_min": -53.51343536376953,
      "activations/layer7_attention_weight_max": 66.01251983642578,
      "activations/layer7_attention_weight_min": -62.4719352722168,
      "activations/layer8_attention_weight_max": 41.50203323364258,
      "activations/layer8_attention_weight_min": -37.906349182128906,
      "activations/layer9_attention_weight_max": 48.13033676147461,
      "activations/layer9_attention_weight_min": -49.48944854736328,
      "epoch": 4.12,
      "learning_rate": 0.00012470643939393937,
      "loss": 2.8733,
      "step": 70850
    },
    {
      "activations/layer0_attention_weight_max": 14.727683067321777,
      "activations/layer0_attention_weight_min": -15.726885795593262,
      "activations/layer10_attention_weight_max": 38.891273498535156,
      "activations/layer10_attention_weight_min": -34.1111946105957,
      "activations/layer11_attention_weight_max": 29.506635665893555,
      "activations/layer11_attention_weight_min": -26.507640838623047,
      "activations/layer12_attention_weight_max": 23.789894104003906,
      "activations/layer12_attention_weight_min": -25.43918800354004,
      "activations/layer13_attention_weight_max": 45.95912551879883,
      "activations/layer13_attention_weight_min": -36.29972839355469,
      "activations/layer14_attention_weight_max": 47.79452896118164,
      "activations/layer14_attention_weight_min": -32.30272674560547,
      "activations/layer15_attention_weight_max": 42.48564147949219,
      "activations/layer15_attention_weight_min": -32.43696975708008,
      "activations/layer16_attention_weight_max": 37.71909713745117,
      "activations/layer16_attention_weight_min": -28.530115127563477,
      "activations/layer17_attention_weight_max": 58.05228805541992,
      "activations/layer17_attention_weight_min": -38.84483337402344,
      "activations/layer18_attention_weight_max": 53.047203063964844,
      "activations/layer18_attention_weight_min": -33.736053466796875,
      "activations/layer19_attention_weight_max": 23.22550392150879,
      "activations/layer19_attention_weight_min": -19.302021026611328,
      "activations/layer1_attention_weight_max": 18.55422592163086,
      "activations/layer1_attention_weight_min": -15.074417114257812,
      "activations/layer20_attention_weight_max": 22.62136459350586,
      "activations/layer20_attention_weight_min": -19.27225685119629,
      "activations/layer21_attention_weight_max": 45.55455017089844,
      "activations/layer21_attention_weight_min": -27.79527473449707,
      "activations/layer22_attention_weight_max": 36.76750564575195,
      "activations/layer22_attention_weight_min": -23.964773178100586,
      "activations/layer23_attention_weight_max": 26.870771408081055,
      "activations/layer23_attention_weight_min": -18.037248611450195,
      "activations/layer2_attention_weight_max": 31.102561950683594,
      "activations/layer2_attention_weight_min": -29.289016723632812,
      "activations/layer3_attention_weight_max": 79.85905456542969,
      "activations/layer3_attention_weight_min": -81.08145141601562,
      "activations/layer4_attention_weight_max": 99.38304901123047,
      "activations/layer4_attention_weight_min": -96.0563735961914,
      "activations/layer5_attention_weight_max": 109.246337890625,
      "activations/layer5_attention_weight_min": -96.44246673583984,
      "activations/layer6_attention_weight_max": 59.687095642089844,
      "activations/layer6_attention_weight_min": -52.25676345825195,
      "activations/layer7_attention_weight_max": 64.35560607910156,
      "activations/layer7_attention_weight_min": -61.246673583984375,
      "activations/layer8_attention_weight_max": 40.62013626098633,
      "activations/layer8_attention_weight_min": -40.7477912902832,
      "activations/layer9_attention_weight_max": 43.999752044677734,
      "activations/layer9_attention_weight_min": -44.1372184753418,
      "epoch": 4.12,
      "learning_rate": 0.0001246875,
      "loss": 2.8706,
      "step": 70900
    },
    {
      "activations/layer0_attention_weight_max": 14.665634155273438,
      "activations/layer0_attention_weight_min": -15.28211498260498,
      "activations/layer10_attention_weight_max": 39.69300842285156,
      "activations/layer10_attention_weight_min": -35.64043426513672,
      "activations/layer11_attention_weight_max": 30.39808464050293,
      "activations/layer11_attention_weight_min": -26.714506149291992,
      "activations/layer12_attention_weight_max": 22.158979415893555,
      "activations/layer12_attention_weight_min": -24.857563018798828,
      "activations/layer13_attention_weight_max": 55.71031951904297,
      "activations/layer13_attention_weight_min": -43.900596618652344,
      "activations/layer14_attention_weight_max": 53.38465881347656,
      "activations/layer14_attention_weight_min": -36.86317443847656,
      "activations/layer15_attention_weight_max": 46.6357421875,
      "activations/layer15_attention_weight_min": -32.80796813964844,
      "activations/layer16_attention_weight_max": 39.71775436401367,
      "activations/layer16_attention_weight_min": -27.35608673095703,
      "activations/layer17_attention_weight_max": 62.48907470703125,
      "activations/layer17_attention_weight_min": -37.88045120239258,
      "activations/layer18_attention_weight_max": 53.98342514038086,
      "activations/layer18_attention_weight_min": -35.26617431640625,
      "activations/layer19_attention_weight_max": 23.846166610717773,
      "activations/layer19_attention_weight_min": -20.28346824645996,
      "activations/layer1_attention_weight_max": 18.12114715576172,
      "activations/layer1_attention_weight_min": -15.415480613708496,
      "activations/layer20_attention_weight_max": 24.117321014404297,
      "activations/layer20_attention_weight_min": -21.361539840698242,
      "activations/layer21_attention_weight_max": 45.14849853515625,
      "activations/layer21_attention_weight_min": -27.880096435546875,
      "activations/layer22_attention_weight_max": 38.73918151855469,
      "activations/layer22_attention_weight_min": -24.586166381835938,
      "activations/layer23_attention_weight_max": 30.826988220214844,
      "activations/layer23_attention_weight_min": -18.85649871826172,
      "activations/layer2_attention_weight_max": 29.814306259155273,
      "activations/layer2_attention_weight_min": -28.552303314208984,
      "activations/layer3_attention_weight_max": 78.53971099853516,
      "activations/layer3_attention_weight_min": -85.91264343261719,
      "activations/layer4_attention_weight_max": 99.22903442382812,
      "activations/layer4_attention_weight_min": -94.7916030883789,
      "activations/layer5_attention_weight_max": 108.489501953125,
      "activations/layer5_attention_weight_min": -99.96843719482422,
      "activations/layer6_attention_weight_max": 57.62382125854492,
      "activations/layer6_attention_weight_min": -53.144691467285156,
      "activations/layer7_attention_weight_max": 62.92211151123047,
      "activations/layer7_attention_weight_min": -62.03477096557617,
      "activations/layer8_attention_weight_max": 40.5914306640625,
      "activations/layer8_attention_weight_min": -38.187496185302734,
      "activations/layer9_attention_weight_max": 47.51689910888672,
      "activations/layer9_attention_weight_min": -48.26524353027344,
      "epoch": 4.12,
      "learning_rate": 0.0001246685606060606,
      "loss": 2.8628,
      "step": 70950
    },
    {
      "activations/layer0_attention_weight_max": 15.030326843261719,
      "activations/layer0_attention_weight_min": -15.254958152770996,
      "activations/layer10_attention_weight_max": 40.076576232910156,
      "activations/layer10_attention_weight_min": -32.63056945800781,
      "activations/layer11_attention_weight_max": 32.22694396972656,
      "activations/layer11_attention_weight_min": -25.183441162109375,
      "activations/layer12_attention_weight_max": 23.93307113647461,
      "activations/layer12_attention_weight_min": -25.762670516967773,
      "activations/layer13_attention_weight_max": 55.13111877441406,
      "activations/layer13_attention_weight_min": -38.17562484741211,
      "activations/layer14_attention_weight_max": 45.81454086303711,
      "activations/layer14_attention_weight_min": -28.629344940185547,
      "activations/layer15_attention_weight_max": 42.237396240234375,
      "activations/layer15_attention_weight_min": -30.116823196411133,
      "activations/layer16_attention_weight_max": 36.24987030029297,
      "activations/layer16_attention_weight_min": -26.147485733032227,
      "activations/layer17_attention_weight_max": 57.79337692260742,
      "activations/layer17_attention_weight_min": -37.8112678527832,
      "activations/layer18_attention_weight_max": 49.61427688598633,
      "activations/layer18_attention_weight_min": -30.348966598510742,
      "activations/layer19_attention_weight_max": 25.60198974609375,
      "activations/layer19_attention_weight_min": -18.517629623413086,
      "activations/layer1_attention_weight_max": 17.52028465270996,
      "activations/layer1_attention_weight_min": -15.266709327697754,
      "activations/layer20_attention_weight_max": 21.086641311645508,
      "activations/layer20_attention_weight_min": -20.42410659790039,
      "activations/layer21_attention_weight_max": 38.4183235168457,
      "activations/layer21_attention_weight_min": -25.00164794921875,
      "activations/layer22_attention_weight_max": 33.004425048828125,
      "activations/layer22_attention_weight_min": -24.480737686157227,
      "activations/layer23_attention_weight_max": 25.98200225830078,
      "activations/layer23_attention_weight_min": -17.073455810546875,
      "activations/layer2_attention_weight_max": 29.773406982421875,
      "activations/layer2_attention_weight_min": -28.816099166870117,
      "activations/layer3_attention_weight_max": 76.92813873291016,
      "activations/layer3_attention_weight_min": -81.6501235961914,
      "activations/layer4_attention_weight_max": 102.85013580322266,
      "activations/layer4_attention_weight_min": -95.81067657470703,
      "activations/layer5_attention_weight_max": 113.02436828613281,
      "activations/layer5_attention_weight_min": -96.08921813964844,
      "activations/layer6_attention_weight_max": 57.57001876831055,
      "activations/layer6_attention_weight_min": -52.65853500366211,
      "activations/layer7_attention_weight_max": 66.30223083496094,
      "activations/layer7_attention_weight_min": -63.675601959228516,
      "activations/layer8_attention_weight_max": 44.23506546020508,
      "activations/layer8_attention_weight_min": -39.93077850341797,
      "activations/layer9_attention_weight_max": 55.70012283325195,
      "activations/layer9_attention_weight_min": -45.2644157409668,
      "epoch": 4.13,
      "learning_rate": 0.0001246496212121212,
      "loss": 2.8757,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_loss": 2.82421875,
      "eval_runtime": 8.4126,
      "eval_samples_per_second": 510.425,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_openwebtext_loss": 2.82421875,
      "eval_openwebtext_ppl": 16.84777751703098,
      "eval_openwebtext_runtime": 8.4126,
      "eval_openwebtext_samples_per_second": 510.425,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_wikitext_loss": 3.0625,
      "eval_wikitext_ppl": 21.380942759123343,
      "eval_wikitext_runtime": 1.9518,
      "eval_wikitext_samples_per_second": 233.631,
      "step": 71000
    },
    {
      "epoch": 4.13,
      "eval_lambada_loss": 3.02734375,
      "eval_lambada_ppl": 20.64232851707703,
      "eval_lambada_runtime": 9.5542,
      "eval_lambada_samples_per_second": 509.619,
      "step": 71000
    },
    {
      "activations/layer0_attention_weight_max": 14.446457862854004,
      "activations/layer0_attention_weight_min": -15.074261665344238,
      "activations/layer10_attention_weight_max": 36.464508056640625,
      "activations/layer10_attention_weight_min": -35.46956253051758,
      "activations/layer11_attention_weight_max": 30.977323532104492,
      "activations/layer11_attention_weight_min": -26.145729064941406,
      "activations/layer12_attention_weight_max": 21.53740882873535,
      "activations/layer12_attention_weight_min": -28.193490982055664,
      "activations/layer13_attention_weight_max": 48.86337661743164,
      "activations/layer13_attention_weight_min": -33.63816833496094,
      "activations/layer14_attention_weight_max": 44.33442306518555,
      "activations/layer14_attention_weight_min": -33.284942626953125,
      "activations/layer15_attention_weight_max": 38.48786163330078,
      "activations/layer15_attention_weight_min": -28.35306739807129,
      "activations/layer16_attention_weight_max": 35.00168991088867,
      "activations/layer16_attention_weight_min": -25.480939865112305,
      "activations/layer17_attention_weight_max": 55.53865432739258,
      "activations/layer17_attention_weight_min": -34.8875732421875,
      "activations/layer18_attention_weight_max": 44.71337127685547,
      "activations/layer18_attention_weight_min": -30.796920776367188,
      "activations/layer19_attention_weight_max": 21.171537399291992,
      "activations/layer19_attention_weight_min": -19.145479202270508,
      "activations/layer1_attention_weight_max": 17.99810791015625,
      "activations/layer1_attention_weight_min": -14.963183403015137,
      "activations/layer20_attention_weight_max": 21.227048873901367,
      "activations/layer20_attention_weight_min": -20.29617691040039,
      "activations/layer21_attention_weight_max": 39.1808967590332,
      "activations/layer21_attention_weight_min": -26.847875595092773,
      "activations/layer22_attention_weight_max": 32.11370086669922,
      "activations/layer22_attention_weight_min": -22.580102920532227,
      "activations/layer23_attention_weight_max": 25.538406372070312,
      "activations/layer23_attention_weight_min": -17.94510269165039,
      "activations/layer2_attention_weight_max": 31.390499114990234,
      "activations/layer2_attention_weight_min": -29.56928062438965,
      "activations/layer3_attention_weight_max": 76.84864044189453,
      "activations/layer3_attention_weight_min": -85.03375244140625,
      "activations/layer4_attention_weight_max": 102.38484954833984,
      "activations/layer4_attention_weight_min": -96.2889404296875,
      "activations/layer5_attention_weight_max": 109.3580551147461,
      "activations/layer5_attention_weight_min": -98.51446533203125,
      "activations/layer6_attention_weight_max": 59.29025650024414,
      "activations/layer6_attention_weight_min": -50.274024963378906,
      "activations/layer7_attention_weight_max": 62.93085479736328,
      "activations/layer7_attention_weight_min": -60.05950164794922,
      "activations/layer8_attention_weight_max": 39.89915084838867,
      "activations/layer8_attention_weight_min": -37.6298713684082,
      "activations/layer9_attention_weight_max": 48.300899505615234,
      "activations/layer9_attention_weight_min": -46.41802215576172,
      "epoch": 4.13,
      "learning_rate": 0.0001246306818181818,
      "loss": 2.8724,
      "step": 71050
    },
    {
      "activations/layer0_attention_weight_max": 14.971418380737305,
      "activations/layer0_attention_weight_min": -14.923881530761719,
      "activations/layer10_attention_weight_max": 47.18477249145508,
      "activations/layer10_attention_weight_min": -37.27286148071289,
      "activations/layer11_attention_weight_max": 40.94797897338867,
      "activations/layer11_attention_weight_min": -27.324180603027344,
      "activations/layer12_attention_weight_max": 26.406448364257812,
      "activations/layer12_attention_weight_min": -25.51181983947754,
      "activations/layer13_attention_weight_max": 51.98603057861328,
      "activations/layer13_attention_weight_min": -34.73884582519531,
      "activations/layer14_attention_weight_max": 53.81293869018555,
      "activations/layer14_attention_weight_min": -31.896343231201172,
      "activations/layer15_attention_weight_max": 44.094879150390625,
      "activations/layer15_attention_weight_min": -28.531625747680664,
      "activations/layer16_attention_weight_max": 35.170902252197266,
      "activations/layer16_attention_weight_min": -25.63268280029297,
      "activations/layer17_attention_weight_max": 67.50086975097656,
      "activations/layer17_attention_weight_min": -34.903564453125,
      "activations/layer18_attention_weight_max": 53.919395446777344,
      "activations/layer18_attention_weight_min": -30.957088470458984,
      "activations/layer19_attention_weight_max": 23.10883140563965,
      "activations/layer19_attention_weight_min": -23.703006744384766,
      "activations/layer1_attention_weight_max": 18.659053802490234,
      "activations/layer1_attention_weight_min": -14.75374984741211,
      "activations/layer20_attention_weight_max": 22.283065795898438,
      "activations/layer20_attention_weight_min": -19.906919479370117,
      "activations/layer21_attention_weight_max": 44.848934173583984,
      "activations/layer21_attention_weight_min": -29.02826499938965,
      "activations/layer22_attention_weight_max": 36.61520767211914,
      "activations/layer22_attention_weight_min": -24.80069351196289,
      "activations/layer23_attention_weight_max": 34.15747833251953,
      "activations/layer23_attention_weight_min": -17.81582260131836,
      "activations/layer2_attention_weight_max": 31.1845703125,
      "activations/layer2_attention_weight_min": -29.025983810424805,
      "activations/layer3_attention_weight_max": 86.02906036376953,
      "activations/layer3_attention_weight_min": -84.71559143066406,
      "activations/layer4_attention_weight_max": 115.40290832519531,
      "activations/layer4_attention_weight_min": -105.27764129638672,
      "activations/layer5_attention_weight_max": 125.47529602050781,
      "activations/layer5_attention_weight_min": -110.49894714355469,
      "activations/layer6_attention_weight_max": 60.61116409301758,
      "activations/layer6_attention_weight_min": -56.70056915283203,
      "activations/layer7_attention_weight_max": 79.4085693359375,
      "activations/layer7_attention_weight_min": -72.43231964111328,
      "activations/layer8_attention_weight_max": 51.80426788330078,
      "activations/layer8_attention_weight_min": -41.034908294677734,
      "activations/layer9_attention_weight_max": 71.00753784179688,
      "activations/layer9_attention_weight_min": -56.494117736816406,
      "epoch": 4.13,
      "learning_rate": 0.00012461174242424242,
      "loss": 2.8644,
      "step": 71100
    },
    {
      "activations/layer0_attention_weight_max": 16.506681442260742,
      "activations/layer0_attention_weight_min": -15.173437118530273,
      "activations/layer10_attention_weight_max": 40.017433166503906,
      "activations/layer10_attention_weight_min": -35.55292892456055,
      "activations/layer11_attention_weight_max": 32.1136474609375,
      "activations/layer11_attention_weight_min": -26.385196685791016,
      "activations/layer12_attention_weight_max": 29.08534049987793,
      "activations/layer12_attention_weight_min": -24.96129608154297,
      "activations/layer13_attention_weight_max": 58.021305084228516,
      "activations/layer13_attention_weight_min": -42.864566802978516,
      "activations/layer14_attention_weight_max": 42.92201614379883,
      "activations/layer14_attention_weight_min": -29.1243839263916,
      "activations/layer15_attention_weight_max": 49.12398147583008,
      "activations/layer15_attention_weight_min": -32.47916793823242,
      "activations/layer16_attention_weight_max": 34.46319580078125,
      "activations/layer16_attention_weight_min": -25.497528076171875,
      "activations/layer17_attention_weight_max": 57.1990852355957,
      "activations/layer17_attention_weight_min": -36.45781326293945,
      "activations/layer18_attention_weight_max": 49.91241455078125,
      "activations/layer18_attention_weight_min": -32.3401985168457,
      "activations/layer19_attention_weight_max": 24.393287658691406,
      "activations/layer19_attention_weight_min": -17.926570892333984,
      "activations/layer1_attention_weight_max": 17.745193481445312,
      "activations/layer1_attention_weight_min": -15.670095443725586,
      "activations/layer20_attention_weight_max": 24.446460723876953,
      "activations/layer20_attention_weight_min": -21.727737426757812,
      "activations/layer21_attention_weight_max": 50.09315872192383,
      "activations/layer21_attention_weight_min": -26.211912155151367,
      "activations/layer22_attention_weight_max": 37.89543533325195,
      "activations/layer22_attention_weight_min": -23.452613830566406,
      "activations/layer23_attention_weight_max": 30.219112396240234,
      "activations/layer23_attention_weight_min": -15.774843215942383,
      "activations/layer2_attention_weight_max": 28.978086471557617,
      "activations/layer2_attention_weight_min": -28.30156707763672,
      "activations/layer3_attention_weight_max": 80.67369842529297,
      "activations/layer3_attention_weight_min": -82.3106918334961,
      "activations/layer4_attention_weight_max": 104.2177963256836,
      "activations/layer4_attention_weight_min": -96.5328598022461,
      "activations/layer5_attention_weight_max": 111.15725708007812,
      "activations/layer5_attention_weight_min": -96.26414489746094,
      "activations/layer6_attention_weight_max": 58.92884063720703,
      "activations/layer6_attention_weight_min": -52.43021774291992,
      "activations/layer7_attention_weight_max": 65.55826568603516,
      "activations/layer7_attention_weight_min": -58.685508728027344,
      "activations/layer8_attention_weight_max": 42.0274658203125,
      "activations/layer8_attention_weight_min": -40.10581970214844,
      "activations/layer9_attention_weight_max": 49.08324432373047,
      "activations/layer9_attention_weight_min": -44.33248519897461,
      "epoch": 4.13,
      "learning_rate": 0.00012459280303030304,
      "loss": 2.8718,
      "step": 71150
    },
    {
      "activations/layer0_attention_weight_max": 14.5025634765625,
      "activations/layer0_attention_weight_min": -15.865837097167969,
      "activations/layer10_attention_weight_max": 46.12593460083008,
      "activations/layer10_attention_weight_min": -37.39735794067383,
      "activations/layer11_attention_weight_max": 40.30741500854492,
      "activations/layer11_attention_weight_min": -28.548892974853516,
      "activations/layer12_attention_weight_max": 25.09617805480957,
      "activations/layer12_attention_weight_min": -23.56985092163086,
      "activations/layer13_attention_weight_max": 56.44856262207031,
      "activations/layer13_attention_weight_min": -37.01239013671875,
      "activations/layer14_attention_weight_max": 52.59463119506836,
      "activations/layer14_attention_weight_min": -29.34813117980957,
      "activations/layer15_attention_weight_max": 45.4299430847168,
      "activations/layer15_attention_weight_min": -29.229705810546875,
      "activations/layer16_attention_weight_max": 37.91105651855469,
      "activations/layer16_attention_weight_min": -25.421710968017578,
      "activations/layer17_attention_weight_max": 60.36702346801758,
      "activations/layer17_attention_weight_min": -35.14689254760742,
      "activations/layer18_attention_weight_max": 53.02735900878906,
      "activations/layer18_attention_weight_min": -30.4259090423584,
      "activations/layer19_attention_weight_max": 28.10210609436035,
      "activations/layer19_attention_weight_min": -18.374704360961914,
      "activations/layer1_attention_weight_max": 17.50676155090332,
      "activations/layer1_attention_weight_min": -14.413008689880371,
      "activations/layer20_attention_weight_max": 20.66490364074707,
      "activations/layer20_attention_weight_min": -20.112144470214844,
      "activations/layer21_attention_weight_max": 40.705299377441406,
      "activations/layer21_attention_weight_min": -27.04289436340332,
      "activations/layer22_attention_weight_max": 38.09538650512695,
      "activations/layer22_attention_weight_min": -21.638925552368164,
      "activations/layer23_attention_weight_max": 25.33282470703125,
      "activations/layer23_attention_weight_min": -15.70492935180664,
      "activations/layer2_attention_weight_max": 33.026939392089844,
      "activations/layer2_attention_weight_min": -29.397476196289062,
      "activations/layer3_attention_weight_max": 83.55429077148438,
      "activations/layer3_attention_weight_min": -84.99005126953125,
      "activations/layer4_attention_weight_max": 114.4101791381836,
      "activations/layer4_attention_weight_min": -107.30616760253906,
      "activations/layer5_attention_weight_max": 132.2579345703125,
      "activations/layer5_attention_weight_min": -114.15994262695312,
      "activations/layer6_attention_weight_max": 67.18809509277344,
      "activations/layer6_attention_weight_min": -63.16914749145508,
      "activations/layer7_attention_weight_max": 81.71886444091797,
      "activations/layer7_attention_weight_min": -72.02318572998047,
      "activations/layer8_attention_weight_max": 52.57600402832031,
      "activations/layer8_attention_weight_min": -42.47316360473633,
      "activations/layer9_attention_weight_max": 65.46499633789062,
      "activations/layer9_attention_weight_min": -57.600799560546875,
      "epoch": 4.14,
      "learning_rate": 0.00012457386363636362,
      "loss": 2.8624,
      "step": 71200
    },
    {
      "activations/layer0_attention_weight_max": 15.056175231933594,
      "activations/layer0_attention_weight_min": -15.420854568481445,
      "activations/layer10_attention_weight_max": 41.9116325378418,
      "activations/layer10_attention_weight_min": -33.10321807861328,
      "activations/layer11_attention_weight_max": 32.78662109375,
      "activations/layer11_attention_weight_min": -27.03376007080078,
      "activations/layer12_attention_weight_max": 27.75702476501465,
      "activations/layer12_attention_weight_min": -24.455429077148438,
      "activations/layer13_attention_weight_max": 56.398948669433594,
      "activations/layer13_attention_weight_min": -39.26780700683594,
      "activations/layer14_attention_weight_max": 61.77808380126953,
      "activations/layer14_attention_weight_min": -32.28539276123047,
      "activations/layer15_attention_weight_max": 50.803836822509766,
      "activations/layer15_attention_weight_min": -29.100353240966797,
      "activations/layer16_attention_weight_max": 41.799381256103516,
      "activations/layer16_attention_weight_min": -26.63238525390625,
      "activations/layer17_attention_weight_max": 61.84400177001953,
      "activations/layer17_attention_weight_min": -35.68464660644531,
      "activations/layer18_attention_weight_max": 54.934505462646484,
      "activations/layer18_attention_weight_min": -32.2791633605957,
      "activations/layer19_attention_weight_max": 27.289005279541016,
      "activations/layer19_attention_weight_min": -17.329587936401367,
      "activations/layer1_attention_weight_max": 17.801307678222656,
      "activations/layer1_attention_weight_min": -14.007761001586914,
      "activations/layer20_attention_weight_max": 27.334672927856445,
      "activations/layer20_attention_weight_min": -17.605112075805664,
      "activations/layer21_attention_weight_max": 45.90911865234375,
      "activations/layer21_attention_weight_min": -26.553869247436523,
      "activations/layer22_attention_weight_max": 42.005672454833984,
      "activations/layer22_attention_weight_min": -22.996234893798828,
      "activations/layer23_attention_weight_max": 29.569568634033203,
      "activations/layer23_attention_weight_min": -17.551162719726562,
      "activations/layer2_attention_weight_max": 30.338794708251953,
      "activations/layer2_attention_weight_min": -28.7689208984375,
      "activations/layer3_attention_weight_max": 78.82251739501953,
      "activations/layer3_attention_weight_min": -84.64742279052734,
      "activations/layer4_attention_weight_max": 105.47833251953125,
      "activations/layer4_attention_weight_min": -98.48538208007812,
      "activations/layer5_attention_weight_max": 114.72227478027344,
      "activations/layer5_attention_weight_min": -102.67593383789062,
      "activations/layer6_attention_weight_max": 60.03133010864258,
      "activations/layer6_attention_weight_min": -53.41963195800781,
      "activations/layer7_attention_weight_max": 71.39176177978516,
      "activations/layer7_attention_weight_min": -66.84870147705078,
      "activations/layer8_attention_weight_max": 41.780113220214844,
      "activations/layer8_attention_weight_min": -37.9517936706543,
      "activations/layer9_attention_weight_max": 53.738311767578125,
      "activations/layer9_attention_weight_min": -45.926116943359375,
      "epoch": 4.14,
      "learning_rate": 0.00012455492424242424,
      "loss": 2.8682,
      "step": 71250
    },
    {
      "activations/layer0_attention_weight_max": 15.261457443237305,
      "activations/layer0_attention_weight_min": -14.857911109924316,
      "activations/layer10_attention_weight_max": 41.74262619018555,
      "activations/layer10_attention_weight_min": -35.289066314697266,
      "activations/layer11_attention_weight_max": 36.92029571533203,
      "activations/layer11_attention_weight_min": -26.513317108154297,
      "activations/layer12_attention_weight_max": 24.43541717529297,
      "activations/layer12_attention_weight_min": -29.470924377441406,
      "activations/layer13_attention_weight_max": 66.57574462890625,
      "activations/layer13_attention_weight_min": -47.832542419433594,
      "activations/layer14_attention_weight_max": 89.20706939697266,
      "activations/layer14_attention_weight_min": -53.947853088378906,
      "activations/layer15_attention_weight_max": 59.37936019897461,
      "activations/layer15_attention_weight_min": -38.97858810424805,
      "activations/layer16_attention_weight_max": 38.147438049316406,
      "activations/layer16_attention_weight_min": -26.78192901611328,
      "activations/layer17_attention_weight_max": 55.042457580566406,
      "activations/layer17_attention_weight_min": -37.849998474121094,
      "activations/layer18_attention_weight_max": 51.971153259277344,
      "activations/layer18_attention_weight_min": -31.550174713134766,
      "activations/layer19_attention_weight_max": 22.176050186157227,
      "activations/layer19_attention_weight_min": -19.24481201171875,
      "activations/layer1_attention_weight_max": 18.47409439086914,
      "activations/layer1_attention_weight_min": -14.215161323547363,
      "activations/layer20_attention_weight_max": 19.76974868774414,
      "activations/layer20_attention_weight_min": -19.950700759887695,
      "activations/layer21_attention_weight_max": 48.60686111450195,
      "activations/layer21_attention_weight_min": -25.809703826904297,
      "activations/layer22_attention_weight_max": 38.69855499267578,
      "activations/layer22_attention_weight_min": -24.34391212463379,
      "activations/layer23_attention_weight_max": 25.53775405883789,
      "activations/layer23_attention_weight_min": -21.31256866455078,
      "activations/layer2_attention_weight_max": 30.175174713134766,
      "activations/layer2_attention_weight_min": -28.53516960144043,
      "activations/layer3_attention_weight_max": 77.12596130371094,
      "activations/layer3_attention_weight_min": -83.25682067871094,
      "activations/layer4_attention_weight_max": 99.59365844726562,
      "activations/layer4_attention_weight_min": -97.44538116455078,
      "activations/layer5_attention_weight_max": 115.19027709960938,
      "activations/layer5_attention_weight_min": -99.569580078125,
      "activations/layer6_attention_weight_max": 63.445316314697266,
      "activations/layer6_attention_weight_min": -55.34461975097656,
      "activations/layer7_attention_weight_max": 76.47312927246094,
      "activations/layer7_attention_weight_min": -69.55074310302734,
      "activations/layer8_attention_weight_max": 47.04312515258789,
      "activations/layer8_attention_weight_min": -39.54865646362305,
      "activations/layer9_attention_weight_max": 61.852783203125,
      "activations/layer9_attention_weight_min": -59.613746643066406,
      "epoch": 4.14,
      "learning_rate": 0.00012453598484848485,
      "loss": 2.8603,
      "step": 71300
    },
    {
      "activations/layer0_attention_weight_max": 14.723771095275879,
      "activations/layer0_attention_weight_min": -14.94461441040039,
      "activations/layer10_attention_weight_max": 43.710514068603516,
      "activations/layer10_attention_weight_min": -36.86688995361328,
      "activations/layer11_attention_weight_max": 33.23966598510742,
      "activations/layer11_attention_weight_min": -27.174047470092773,
      "activations/layer12_attention_weight_max": 28.194747924804688,
      "activations/layer12_attention_weight_min": -25.46941375732422,
      "activations/layer13_attention_weight_max": 57.33167266845703,
      "activations/layer13_attention_weight_min": -37.42867660522461,
      "activations/layer14_attention_weight_max": 64.23704528808594,
      "activations/layer14_attention_weight_min": -36.289913177490234,
      "activations/layer15_attention_weight_max": 46.96187210083008,
      "activations/layer15_attention_weight_min": -31.309612274169922,
      "activations/layer16_attention_weight_max": 40.686614990234375,
      "activations/layer16_attention_weight_min": -26.240753173828125,
      "activations/layer17_attention_weight_max": 56.7563362121582,
      "activations/layer17_attention_weight_min": -38.19673156738281,
      "activations/layer18_attention_weight_max": 57.137786865234375,
      "activations/layer18_attention_weight_min": -34.935089111328125,
      "activations/layer19_attention_weight_max": 25.239168167114258,
      "activations/layer19_attention_weight_min": -19.12995719909668,
      "activations/layer1_attention_weight_max": 18.474531173706055,
      "activations/layer1_attention_weight_min": -14.44342041015625,
      "activations/layer20_attention_weight_max": 26.984899520874023,
      "activations/layer20_attention_weight_min": -18.128005981445312,
      "activations/layer21_attention_weight_max": 48.84952163696289,
      "activations/layer21_attention_weight_min": -28.72457504272461,
      "activations/layer22_attention_weight_max": 41.4031982421875,
      "activations/layer22_attention_weight_min": -22.98247528076172,
      "activations/layer23_attention_weight_max": 29.04436492919922,
      "activations/layer23_attention_weight_min": -18.690223693847656,
      "activations/layer2_attention_weight_max": 30.482257843017578,
      "activations/layer2_attention_weight_min": -28.465423583984375,
      "activations/layer3_attention_weight_max": 82.19180297851562,
      "activations/layer3_attention_weight_min": -89.01818084716797,
      "activations/layer4_attention_weight_max": 107.8316421508789,
      "activations/layer4_attention_weight_min": -106.53704833984375,
      "activations/layer5_attention_weight_max": 117.24571228027344,
      "activations/layer5_attention_weight_min": -109.76675415039062,
      "activations/layer6_attention_weight_max": 64.44823455810547,
      "activations/layer6_attention_weight_min": -55.81595993041992,
      "activations/layer7_attention_weight_max": 72.39694213867188,
      "activations/layer7_attention_weight_min": -68.47199249267578,
      "activations/layer8_attention_weight_max": 43.91493606567383,
      "activations/layer8_attention_weight_min": -42.08559036254883,
      "activations/layer9_attention_weight_max": 55.01361083984375,
      "activations/layer9_attention_weight_min": -49.93731689453125,
      "epoch": 4.15,
      "learning_rate": 0.00012451704545454544,
      "loss": 2.8753,
      "step": 71350
    },
    {
      "activations/layer0_attention_weight_max": 15.112001419067383,
      "activations/layer0_attention_weight_min": -14.079020500183105,
      "activations/layer10_attention_weight_max": 37.17720413208008,
      "activations/layer10_attention_weight_min": -29.423709869384766,
      "activations/layer11_attention_weight_max": 31.038000106811523,
      "activations/layer11_attention_weight_min": -24.586483001708984,
      "activations/layer12_attention_weight_max": 23.835233688354492,
      "activations/layer12_attention_weight_min": -24.54627227783203,
      "activations/layer13_attention_weight_max": 44.36743927001953,
      "activations/layer13_attention_weight_min": -32.450042724609375,
      "activations/layer14_attention_weight_max": 39.377559661865234,
      "activations/layer14_attention_weight_min": -27.759492874145508,
      "activations/layer15_attention_weight_max": 40.3050537109375,
      "activations/layer15_attention_weight_min": -28.161325454711914,
      "activations/layer16_attention_weight_max": 34.71099090576172,
      "activations/layer16_attention_weight_min": -24.905691146850586,
      "activations/layer17_attention_weight_max": 60.04920196533203,
      "activations/layer17_attention_weight_min": -35.427494049072266,
      "activations/layer18_attention_weight_max": 49.383365631103516,
      "activations/layer18_attention_weight_min": -30.765817642211914,
      "activations/layer19_attention_weight_max": 24.193744659423828,
      "activations/layer19_attention_weight_min": -20.07038116455078,
      "activations/layer1_attention_weight_max": 18.709280014038086,
      "activations/layer1_attention_weight_min": -16.463123321533203,
      "activations/layer20_attention_weight_max": 19.44023895263672,
      "activations/layer20_attention_weight_min": -21.15117073059082,
      "activations/layer21_attention_weight_max": 40.113162994384766,
      "activations/layer21_attention_weight_min": -26.84111785888672,
      "activations/layer22_attention_weight_max": 34.521236419677734,
      "activations/layer22_attention_weight_min": -22.727035522460938,
      "activations/layer23_attention_weight_max": 26.47901153564453,
      "activations/layer23_attention_weight_min": -17.858383178710938,
      "activations/layer2_attention_weight_max": 37.760536193847656,
      "activations/layer2_attention_weight_min": -26.68707847595215,
      "activations/layer3_attention_weight_max": 70.15701293945312,
      "activations/layer3_attention_weight_min": -52.99428176879883,
      "activations/layer4_attention_weight_max": 90.6190185546875,
      "activations/layer4_attention_weight_min": -81.26030731201172,
      "activations/layer5_attention_weight_max": 101.21540832519531,
      "activations/layer5_attention_weight_min": -87.27285766601562,
      "activations/layer6_attention_weight_max": 57.95610046386719,
      "activations/layer6_attention_weight_min": -44.080020904541016,
      "activations/layer7_attention_weight_max": 58.77217102050781,
      "activations/layer7_attention_weight_min": -58.047607421875,
      "activations/layer8_attention_weight_max": 38.78715896606445,
      "activations/layer8_attention_weight_min": -33.456668853759766,
      "activations/layer9_attention_weight_max": 47.404640197753906,
      "activations/layer9_attention_weight_min": -42.980316162109375,
      "epoch": 4.15,
      "learning_rate": 0.0001244992424242424,
      "loss": 2.8947,
      "step": 71400
    },
    {
      "activations/layer0_attention_weight_max": 14.020962715148926,
      "activations/layer0_attention_weight_min": -17.11992645263672,
      "activations/layer10_attention_weight_max": 40.22832489013672,
      "activations/layer10_attention_weight_min": -33.616355895996094,
      "activations/layer11_attention_weight_max": 28.92452049255371,
      "activations/layer11_attention_weight_min": -26.947811126708984,
      "activations/layer12_attention_weight_max": 21.154800415039062,
      "activations/layer12_attention_weight_min": -25.090072631835938,
      "activations/layer13_attention_weight_max": 50.706504821777344,
      "activations/layer13_attention_weight_min": -44.685630798339844,
      "activations/layer14_attention_weight_max": 42.88115310668945,
      "activations/layer14_attention_weight_min": -31.509170532226562,
      "activations/layer15_attention_weight_max": 42.701698303222656,
      "activations/layer15_attention_weight_min": -30.12565803527832,
      "activations/layer16_attention_weight_max": 35.58573532104492,
      "activations/layer16_attention_weight_min": -25.36405372619629,
      "activations/layer17_attention_weight_max": 53.65520095825195,
      "activations/layer17_attention_weight_min": -38.22834396362305,
      "activations/layer18_attention_weight_max": 44.92527770996094,
      "activations/layer18_attention_weight_min": -33.745479583740234,
      "activations/layer19_attention_weight_max": 24.71839141845703,
      "activations/layer19_attention_weight_min": -19.900197982788086,
      "activations/layer1_attention_weight_max": 18.495826721191406,
      "activations/layer1_attention_weight_min": -15.812878608703613,
      "activations/layer20_attention_weight_max": 22.855127334594727,
      "activations/layer20_attention_weight_min": -24.486282348632812,
      "activations/layer21_attention_weight_max": 37.81578826904297,
      "activations/layer21_attention_weight_min": -27.771011352539062,
      "activations/layer22_attention_weight_max": 34.639129638671875,
      "activations/layer22_attention_weight_min": -23.367387771606445,
      "activations/layer23_attention_weight_max": 25.81005096435547,
      "activations/layer23_attention_weight_min": -18.295146942138672,
      "activations/layer2_attention_weight_max": 28.326290130615234,
      "activations/layer2_attention_weight_min": -27.42129135131836,
      "activations/layer3_attention_weight_max": 65.59860229492188,
      "activations/layer3_attention_weight_min": -68.5265884399414,
      "activations/layer4_attention_weight_max": 88.8955307006836,
      "activations/layer4_attention_weight_min": -81.47412872314453,
      "activations/layer5_attention_weight_max": 97.66100311279297,
      "activations/layer5_attention_weight_min": -83.93523406982422,
      "activations/layer6_attention_weight_max": 52.048362731933594,
      "activations/layer6_attention_weight_min": -46.94790267944336,
      "activations/layer7_attention_weight_max": 63.398189544677734,
      "activations/layer7_attention_weight_min": -58.53490447998047,
      "activations/layer8_attention_weight_max": 39.57600402832031,
      "activations/layer8_attention_weight_min": -40.914066314697266,
      "activations/layer9_attention_weight_max": 46.784156799316406,
      "activations/layer9_attention_weight_min": -46.34297180175781,
      "epoch": 4.15,
      "learning_rate": 0.00012448030303030303,
      "loss": 2.9155,
      "step": 71450
    },
    {
      "activations/layer0_attention_weight_max": 14.228147506713867,
      "activations/layer0_attention_weight_min": -16.2716064453125,
      "activations/layer10_attention_weight_max": 37.90638732910156,
      "activations/layer10_attention_weight_min": -33.9791259765625,
      "activations/layer11_attention_weight_max": 27.3055419921875,
      "activations/layer11_attention_weight_min": -25.51732635498047,
      "activations/layer12_attention_weight_max": 25.766990661621094,
      "activations/layer12_attention_weight_min": -24.431684494018555,
      "activations/layer13_attention_weight_max": 47.35554885864258,
      "activations/layer13_attention_weight_min": -39.12617492675781,
      "activations/layer14_attention_weight_max": 34.902687072753906,
      "activations/layer14_attention_weight_min": -27.432601928710938,
      "activations/layer15_attention_weight_max": 40.2514533996582,
      "activations/layer15_attention_weight_min": -30.300621032714844,
      "activations/layer16_attention_weight_max": 36.24167251586914,
      "activations/layer16_attention_weight_min": -27.786346435546875,
      "activations/layer17_attention_weight_max": 54.04068374633789,
      "activations/layer17_attention_weight_min": -37.437625885009766,
      "activations/layer18_attention_weight_max": 48.09952926635742,
      "activations/layer18_attention_weight_min": -31.906513214111328,
      "activations/layer19_attention_weight_max": 24.42015266418457,
      "activations/layer19_attention_weight_min": -18.841875076293945,
      "activations/layer1_attention_weight_max": 18.31422233581543,
      "activations/layer1_attention_weight_min": -16.217370986938477,
      "activations/layer20_attention_weight_max": 19.754276275634766,
      "activations/layer20_attention_weight_min": -19.178138732910156,
      "activations/layer21_attention_weight_max": 34.54096221923828,
      "activations/layer21_attention_weight_min": -26.100412368774414,
      "activations/layer22_attention_weight_max": 32.237789154052734,
      "activations/layer22_attention_weight_min": -22.925457000732422,
      "activations/layer23_attention_weight_max": 23.837493896484375,
      "activations/layer23_attention_weight_min": -18.100770950317383,
      "activations/layer2_attention_weight_max": 28.117931365966797,
      "activations/layer2_attention_weight_min": -27.199817657470703,
      "activations/layer3_attention_weight_max": 67.37248229980469,
      "activations/layer3_attention_weight_min": -72.27810668945312,
      "activations/layer4_attention_weight_max": 86.7983627319336,
      "activations/layer4_attention_weight_min": -83.47509765625,
      "activations/layer5_attention_weight_max": 98.73595428466797,
      "activations/layer5_attention_weight_min": -88.61040496826172,
      "activations/layer6_attention_weight_max": 53.382972717285156,
      "activations/layer6_attention_weight_min": -48.43659210205078,
      "activations/layer7_attention_weight_max": 61.15034866333008,
      "activations/layer7_attention_weight_min": -60.5379524230957,
      "activations/layer8_attention_weight_max": 40.2738151550293,
      "activations/layer8_attention_weight_min": -40.39717483520508,
      "activations/layer9_attention_weight_max": 47.817440032958984,
      "activations/layer9_attention_weight_min": -43.9678840637207,
      "epoch": 4.15,
      "learning_rate": 0.00012446136363636364,
      "loss": 2.8857,
      "step": 71500
    },
    {
      "activations/layer0_attention_weight_max": 14.305109024047852,
      "activations/layer0_attention_weight_min": -16.207035064697266,
      "activations/layer10_attention_weight_max": 45.01100158691406,
      "activations/layer10_attention_weight_min": -35.60519027709961,
      "activations/layer11_attention_weight_max": 39.641929626464844,
      "activations/layer11_attention_weight_min": -27.507549285888672,
      "activations/layer12_attention_weight_max": 25.36271858215332,
      "activations/layer12_attention_weight_min": -23.864013671875,
      "activations/layer13_attention_weight_max": 58.2641487121582,
      "activations/layer13_attention_weight_min": -35.800540924072266,
      "activations/layer14_attention_weight_max": 53.78402328491211,
      "activations/layer14_attention_weight_min": -31.111665725708008,
      "activations/layer15_attention_weight_max": 50.970829010009766,
      "activations/layer15_attention_weight_min": -30.951820373535156,
      "activations/layer16_attention_weight_max": 42.9271125793457,
      "activations/layer16_attention_weight_min": -23.90825843811035,
      "activations/layer17_attention_weight_max": 67.31436157226562,
      "activations/layer17_attention_weight_min": -36.49922561645508,
      "activations/layer18_attention_weight_max": 54.98596954345703,
      "activations/layer18_attention_weight_min": -31.82701301574707,
      "activations/layer19_attention_weight_max": 24.013553619384766,
      "activations/layer19_attention_weight_min": -18.219728469848633,
      "activations/layer1_attention_weight_max": 18.426488876342773,
      "activations/layer1_attention_weight_min": -19.035146713256836,
      "activations/layer20_attention_weight_max": 21.273759841918945,
      "activations/layer20_attention_weight_min": -17.272920608520508,
      "activations/layer21_attention_weight_max": 43.940521240234375,
      "activations/layer21_attention_weight_min": -28.50899887084961,
      "activations/layer22_attention_weight_max": 36.289730072021484,
      "activations/layer22_attention_weight_min": -21.663761138916016,
      "activations/layer23_attention_weight_max": 27.141860961914062,
      "activations/layer23_attention_weight_min": -17.343505859375,
      "activations/layer2_attention_weight_max": 31.576772689819336,
      "activations/layer2_attention_weight_min": -28.680830001831055,
      "activations/layer3_attention_weight_max": 79.38272094726562,
      "activations/layer3_attention_weight_min": -77.0870361328125,
      "activations/layer4_attention_weight_max": 98.64539337158203,
      "activations/layer4_attention_weight_min": -95.0093765258789,
      "activations/layer5_attention_weight_max": 112.851806640625,
      "activations/layer5_attention_weight_min": -98.05370330810547,
      "activations/layer6_attention_weight_max": 60.107505798339844,
      "activations/layer6_attention_weight_min": -52.65572738647461,
      "activations/layer7_attention_weight_max": 71.35427856445312,
      "activations/layer7_attention_weight_min": -61.269989013671875,
      "activations/layer8_attention_weight_max": 45.20811080932617,
      "activations/layer8_attention_weight_min": -42.40833282470703,
      "activations/layer9_attention_weight_max": 60.40261459350586,
      "activations/layer9_attention_weight_min": -49.413753509521484,
      "epoch": 4.16,
      "learning_rate": 0.00012444242424242423,
      "loss": 2.8704,
      "step": 71550
    },
    {
      "activations/layer0_attention_weight_max": 14.486727714538574,
      "activations/layer0_attention_weight_min": -15.943771362304688,
      "activations/layer10_attention_weight_max": 40.712581634521484,
      "activations/layer10_attention_weight_min": -35.65822219848633,
      "activations/layer11_attention_weight_max": 32.87314987182617,
      "activations/layer11_attention_weight_min": -26.30572509765625,
      "activations/layer12_attention_weight_max": 27.6595401763916,
      "activations/layer12_attention_weight_min": -22.491165161132812,
      "activations/layer13_attention_weight_max": 54.25862503051758,
      "activations/layer13_attention_weight_min": -39.96566390991211,
      "activations/layer14_attention_weight_max": 40.375179290771484,
      "activations/layer14_attention_weight_min": -29.250568389892578,
      "activations/layer15_attention_weight_max": 50.36953353881836,
      "activations/layer15_attention_weight_min": -32.500343322753906,
      "activations/layer16_attention_weight_max": 34.519432067871094,
      "activations/layer16_attention_weight_min": -26.379003524780273,
      "activations/layer17_attention_weight_max": 58.65093231201172,
      "activations/layer17_attention_weight_min": -37.55767822265625,
      "activations/layer18_attention_weight_max": 48.53343963623047,
      "activations/layer18_attention_weight_min": -35.61944580078125,
      "activations/layer19_attention_weight_max": 22.55984115600586,
      "activations/layer19_attention_weight_min": -18.552671432495117,
      "activations/layer1_attention_weight_max": 17.42586898803711,
      "activations/layer1_attention_weight_min": -14.976094245910645,
      "activations/layer20_attention_weight_max": 20.80220603942871,
      "activations/layer20_attention_weight_min": -20.873882293701172,
      "activations/layer21_attention_weight_max": 38.413726806640625,
      "activations/layer21_attention_weight_min": -26.661588668823242,
      "activations/layer22_attention_weight_max": 33.33576583862305,
      "activations/layer22_attention_weight_min": -22.828250885009766,
      "activations/layer23_attention_weight_max": 26.65692901611328,
      "activations/layer23_attention_weight_min": -17.29275131225586,
      "activations/layer2_attention_weight_max": 28.72321128845215,
      "activations/layer2_attention_weight_min": -27.905406951904297,
      "activations/layer3_attention_weight_max": 70.7022476196289,
      "activations/layer3_attention_weight_min": -74.56781768798828,
      "activations/layer4_attention_weight_max": 96.59050750732422,
      "activations/layer4_attention_weight_min": -86.2254638671875,
      "activations/layer5_attention_weight_max": 104.25801086425781,
      "activations/layer5_attention_weight_min": -93.72900390625,
      "activations/layer6_attention_weight_max": 56.63706970214844,
      "activations/layer6_attention_weight_min": -51.40016174316406,
      "activations/layer7_attention_weight_max": 61.7346076965332,
      "activations/layer7_attention_weight_min": -59.98896789550781,
      "activations/layer8_attention_weight_max": 40.651981353759766,
      "activations/layer8_attention_weight_min": -39.794700622558594,
      "activations/layer9_attention_weight_max": 49.76688766479492,
      "activations/layer9_attention_weight_min": -50.299560546875,
      "epoch": 4.16,
      "learning_rate": 0.00012442348484848484,
      "loss": 2.8658,
      "step": 71600
    },
    {
      "activations/layer0_attention_weight_max": 14.885175704956055,
      "activations/layer0_attention_weight_min": -14.850539207458496,
      "activations/layer10_attention_weight_max": 38.890933990478516,
      "activations/layer10_attention_weight_min": -34.04246139526367,
      "activations/layer11_attention_weight_max": 30.869300842285156,
      "activations/layer11_attention_weight_min": -26.403913497924805,
      "activations/layer12_attention_weight_max": 26.5455379486084,
      "activations/layer12_attention_weight_min": -26.615190505981445,
      "activations/layer13_attention_weight_max": 51.94466781616211,
      "activations/layer13_attention_weight_min": -37.05095672607422,
      "activations/layer14_attention_weight_max": 37.85769271850586,
      "activations/layer14_attention_weight_min": -27.576168060302734,
      "activations/layer15_attention_weight_max": 41.157711029052734,
      "activations/layer15_attention_weight_min": -30.663841247558594,
      "activations/layer16_attention_weight_max": 36.387420654296875,
      "activations/layer16_attention_weight_min": -25.52557945251465,
      "activations/layer17_attention_weight_max": 57.81207275390625,
      "activations/layer17_attention_weight_min": -39.69072723388672,
      "activations/layer18_attention_weight_max": 46.83845138549805,
      "activations/layer18_attention_weight_min": -33.19663619995117,
      "activations/layer19_attention_weight_max": 24.220012664794922,
      "activations/layer19_attention_weight_min": -16.43528175354004,
      "activations/layer1_attention_weight_max": 17.08278465270996,
      "activations/layer1_attention_weight_min": -14.24388599395752,
      "activations/layer20_attention_weight_max": 20.01621437072754,
      "activations/layer20_attention_weight_min": -20.69042205810547,
      "activations/layer21_attention_weight_max": 36.82017135620117,
      "activations/layer21_attention_weight_min": -23.914390563964844,
      "activations/layer22_attention_weight_max": 36.99530792236328,
      "activations/layer22_attention_weight_min": -20.841527938842773,
      "activations/layer23_attention_weight_max": 25.53749656677246,
      "activations/layer23_attention_weight_min": -15.232836723327637,
      "activations/layer2_attention_weight_max": 27.807720184326172,
      "activations/layer2_attention_weight_min": -27.65386962890625,
      "activations/layer3_attention_weight_max": 72.75614929199219,
      "activations/layer3_attention_weight_min": -79.10929870605469,
      "activations/layer4_attention_weight_max": 95.79483795166016,
      "activations/layer4_attention_weight_min": -88.43357849121094,
      "activations/layer5_attention_weight_max": 104.09837341308594,
      "activations/layer5_attention_weight_min": -97.33436584472656,
      "activations/layer6_attention_weight_max": 57.9402961730957,
      "activations/layer6_attention_weight_min": -50.82518005371094,
      "activations/layer7_attention_weight_max": 62.7418098449707,
      "activations/layer7_attention_weight_min": -62.4704704284668,
      "activations/layer8_attention_weight_max": 41.51273727416992,
      "activations/layer8_attention_weight_min": -38.990718841552734,
      "activations/layer9_attention_weight_max": 52.383033752441406,
      "activations/layer9_attention_weight_min": -51.1942024230957,
      "epoch": 4.16,
      "learning_rate": 0.00012440454545454546,
      "loss": 2.8642,
      "step": 71650
    },
    {
      "activations/layer0_attention_weight_max": 14.725844383239746,
      "activations/layer0_attention_weight_min": -15.464927673339844,
      "activations/layer10_attention_weight_max": 39.22810745239258,
      "activations/layer10_attention_weight_min": -34.43390655517578,
      "activations/layer11_attention_weight_max": 30.560352325439453,
      "activations/layer11_attention_weight_min": -27.443906784057617,
      "activations/layer12_attention_weight_max": 23.119218826293945,
      "activations/layer12_attention_weight_min": -26.704910278320312,
      "activations/layer13_attention_weight_max": 49.087772369384766,
      "activations/layer13_attention_weight_min": -36.8652229309082,
      "activations/layer14_attention_weight_max": 46.18788146972656,
      "activations/layer14_attention_weight_min": -27.527385711669922,
      "activations/layer15_attention_weight_max": 39.57427978515625,
      "activations/layer15_attention_weight_min": -29.695369720458984,
      "activations/layer16_attention_weight_max": 32.03158950805664,
      "activations/layer16_attention_weight_min": -28.46342658996582,
      "activations/layer17_attention_weight_max": 60.3363037109375,
      "activations/layer17_attention_weight_min": -36.23650360107422,
      "activations/layer18_attention_weight_max": 53.267723083496094,
      "activations/layer18_attention_weight_min": -31.809444427490234,
      "activations/layer19_attention_weight_max": 21.074790954589844,
      "activations/layer19_attention_weight_min": -17.842302322387695,
      "activations/layer1_attention_weight_max": 17.23416519165039,
      "activations/layer1_attention_weight_min": -15.491781234741211,
      "activations/layer20_attention_weight_max": 19.164722442626953,
      "activations/layer20_attention_weight_min": -18.88339614868164,
      "activations/layer21_attention_weight_max": 37.58076858520508,
      "activations/layer21_attention_weight_min": -26.4773006439209,
      "activations/layer22_attention_weight_max": 37.83600616455078,
      "activations/layer22_attention_weight_min": -22.354598999023438,
      "activations/layer23_attention_weight_max": 27.155738830566406,
      "activations/layer23_attention_weight_min": -15.672704696655273,
      "activations/layer2_attention_weight_max": 29.251445770263672,
      "activations/layer2_attention_weight_min": -27.233755111694336,
      "activations/layer3_attention_weight_max": 75.69038391113281,
      "activations/layer3_attention_weight_min": -76.14404296875,
      "activations/layer4_attention_weight_max": 94.72772216796875,
      "activations/layer4_attention_weight_min": -87.48858642578125,
      "activations/layer5_attention_weight_max": 102.71945190429688,
      "activations/layer5_attention_weight_min": -96.35287475585938,
      "activations/layer6_attention_weight_max": 57.2432861328125,
      "activations/layer6_attention_weight_min": -52.43938064575195,
      "activations/layer7_attention_weight_max": 65.16474151611328,
      "activations/layer7_attention_weight_min": -62.3201904296875,
      "activations/layer8_attention_weight_max": 42.15791702270508,
      "activations/layer8_attention_weight_min": -42.06460952758789,
      "activations/layer9_attention_weight_max": 54.351558685302734,
      "activations/layer9_attention_weight_min": -48.75599670410156,
      "epoch": 4.17,
      "learning_rate": 0.00012438560606060605,
      "loss": 2.8459,
      "step": 71700
    },
    {
      "activations/layer0_attention_weight_max": 15.44412612915039,
      "activations/layer0_attention_weight_min": -15.88538646697998,
      "activations/layer10_attention_weight_max": 40.42747497558594,
      "activations/layer10_attention_weight_min": -32.936641693115234,
      "activations/layer11_attention_weight_max": 33.811370849609375,
      "activations/layer11_attention_weight_min": -26.510046005249023,
      "activations/layer12_attention_weight_max": 24.518085479736328,
      "activations/layer12_attention_weight_min": -22.467166900634766,
      "activations/layer13_attention_weight_max": 53.608177185058594,
      "activations/layer13_attention_weight_min": -37.07197570800781,
      "activations/layer14_attention_weight_max": 47.125877380371094,
      "activations/layer14_attention_weight_min": -31.002321243286133,
      "activations/layer15_attention_weight_max": 47.3608283996582,
      "activations/layer15_attention_weight_min": -32.07392883300781,
      "activations/layer16_attention_weight_max": 42.68844223022461,
      "activations/layer16_attention_weight_min": -25.69782257080078,
      "activations/layer17_attention_weight_max": 65.50757598876953,
      "activations/layer17_attention_weight_min": -37.685081481933594,
      "activations/layer18_attention_weight_max": 53.67903518676758,
      "activations/layer18_attention_weight_min": -34.4168586730957,
      "activations/layer19_attention_weight_max": 27.489459991455078,
      "activations/layer19_attention_weight_min": -18.258764266967773,
      "activations/layer1_attention_weight_max": 16.5341854095459,
      "activations/layer1_attention_weight_min": -14.870227813720703,
      "activations/layer20_attention_weight_max": 23.24256706237793,
      "activations/layer20_attention_weight_min": -23.165729522705078,
      "activations/layer21_attention_weight_max": 42.25335693359375,
      "activations/layer21_attention_weight_min": -26.943309783935547,
      "activations/layer22_attention_weight_max": 37.937618255615234,
      "activations/layer22_attention_weight_min": -25.817813873291016,
      "activations/layer23_attention_weight_max": 28.40622329711914,
      "activations/layer23_attention_weight_min": -16.452821731567383,
      "activations/layer2_attention_weight_max": 29.103885650634766,
      "activations/layer2_attention_weight_min": -28.446439743041992,
      "activations/layer3_attention_weight_max": 72.54296112060547,
      "activations/layer3_attention_weight_min": -75.7377700805664,
      "activations/layer4_attention_weight_max": 95.70497131347656,
      "activations/layer4_attention_weight_min": -88.05745697021484,
      "activations/layer5_attention_weight_max": 106.7458724975586,
      "activations/layer5_attention_weight_min": -93.94781494140625,
      "activations/layer6_attention_weight_max": 57.501773834228516,
      "activations/layer6_attention_weight_min": -50.70746612548828,
      "activations/layer7_attention_weight_max": 61.270301818847656,
      "activations/layer7_attention_weight_min": -59.655277252197266,
      "activations/layer8_attention_weight_max": 40.6345329284668,
      "activations/layer8_attention_weight_min": -39.83686828613281,
      "activations/layer9_attention_weight_max": 46.724822998046875,
      "activations/layer9_attention_weight_min": -47.64235305786133,
      "epoch": 4.17,
      "learning_rate": 0.00012436666666666666,
      "loss": 2.8783,
      "step": 71750
    },
    {
      "activations/layer0_attention_weight_max": 13.410974502563477,
      "activations/layer0_attention_weight_min": -14.875889778137207,
      "activations/layer10_attention_weight_max": 38.60697555541992,
      "activations/layer10_attention_weight_min": -33.839290618896484,
      "activations/layer11_attention_weight_max": 33.059654235839844,
      "activations/layer11_attention_weight_min": -27.38587760925293,
      "activations/layer12_attention_weight_max": 23.10822296142578,
      "activations/layer12_attention_weight_min": -27.32919692993164,
      "activations/layer13_attention_weight_max": 49.34489440917969,
      "activations/layer13_attention_weight_min": -34.751686096191406,
      "activations/layer14_attention_weight_max": 43.59376907348633,
      "activations/layer14_attention_weight_min": -28.165292739868164,
      "activations/layer15_attention_weight_max": 42.775394439697266,
      "activations/layer15_attention_weight_min": -32.80552291870117,
      "activations/layer16_attention_weight_max": 34.63002014160156,
      "activations/layer16_attention_weight_min": -25.356786727905273,
      "activations/layer17_attention_weight_max": 58.98751449584961,
      "activations/layer17_attention_weight_min": -41.50242233276367,
      "activations/layer18_attention_weight_max": 47.24164962768555,
      "activations/layer18_attention_weight_min": -32.5255012512207,
      "activations/layer19_attention_weight_max": 22.086284637451172,
      "activations/layer19_attention_weight_min": -19.198854446411133,
      "activations/layer1_attention_weight_max": 17.50930404663086,
      "activations/layer1_attention_weight_min": -14.28660774230957,
      "activations/layer20_attention_weight_max": 20.824005126953125,
      "activations/layer20_attention_weight_min": -21.167200088500977,
      "activations/layer21_attention_weight_max": 40.2757568359375,
      "activations/layer21_attention_weight_min": -25.61822509765625,
      "activations/layer22_attention_weight_max": 32.4005012512207,
      "activations/layer22_attention_weight_min": -26.365917205810547,
      "activations/layer23_attention_weight_max": 26.072851181030273,
      "activations/layer23_attention_weight_min": -18.58135223388672,
      "activations/layer2_attention_weight_max": 29.93398666381836,
      "activations/layer2_attention_weight_min": -27.516029357910156,
      "activations/layer3_attention_weight_max": 76.1525650024414,
      "activations/layer3_attention_weight_min": -76.4477767944336,
      "activations/layer4_attention_weight_max": 97.037109375,
      "activations/layer4_attention_weight_min": -91.32644653320312,
      "activations/layer5_attention_weight_max": 106.91783905029297,
      "activations/layer5_attention_weight_min": -94.04206848144531,
      "activations/layer6_attention_weight_max": 60.971038818359375,
      "activations/layer6_attention_weight_min": -52.60062789916992,
      "activations/layer7_attention_weight_max": 62.514442443847656,
      "activations/layer7_attention_weight_min": -59.29631423950195,
      "activations/layer8_attention_weight_max": 42.604496002197266,
      "activations/layer8_attention_weight_min": -39.446739196777344,
      "activations/layer9_attention_weight_max": 52.40983200073242,
      "activations/layer9_attention_weight_min": -47.173892974853516,
      "epoch": 4.17,
      "learning_rate": 0.00012434772727272725,
      "loss": 2.8785,
      "step": 71800
    },
    {
      "activations/layer0_attention_weight_max": 13.561857223510742,
      "activations/layer0_attention_weight_min": -15.083447456359863,
      "activations/layer10_attention_weight_max": 37.96126937866211,
      "activations/layer10_attention_weight_min": -32.028297424316406,
      "activations/layer11_attention_weight_max": 28.88277244567871,
      "activations/layer11_attention_weight_min": -26.24795150756836,
      "activations/layer12_attention_weight_max": 23.651695251464844,
      "activations/layer12_attention_weight_min": -24.384180068969727,
      "activations/layer13_attention_weight_max": 44.78221130371094,
      "activations/layer13_attention_weight_min": -37.3133659362793,
      "activations/layer14_attention_weight_max": 44.9027214050293,
      "activations/layer14_attention_weight_min": -28.61778450012207,
      "activations/layer15_attention_weight_max": 39.408599853515625,
      "activations/layer15_attention_weight_min": -29.826698303222656,
      "activations/layer16_attention_weight_max": 43.50998306274414,
      "activations/layer16_attention_weight_min": -26.55564308166504,
      "activations/layer17_attention_weight_max": 53.333492279052734,
      "activations/layer17_attention_weight_min": -37.334922790527344,
      "activations/layer18_attention_weight_max": 50.21917724609375,
      "activations/layer18_attention_weight_min": -31.577241897583008,
      "activations/layer19_attention_weight_max": 23.398889541625977,
      "activations/layer19_attention_weight_min": -20.27776527404785,
      "activations/layer1_attention_weight_max": 17.55160903930664,
      "activations/layer1_attention_weight_min": -15.99915599822998,
      "activations/layer20_attention_weight_max": 21.649843215942383,
      "activations/layer20_attention_weight_min": -19.777530670166016,
      "activations/layer21_attention_weight_max": 36.295265197753906,
      "activations/layer21_attention_weight_min": -29.580642700195312,
      "activations/layer22_attention_weight_max": 36.03589630126953,
      "activations/layer22_attention_weight_min": -24.58637237548828,
      "activations/layer23_attention_weight_max": 26.207733154296875,
      "activations/layer23_attention_weight_min": -18.865589141845703,
      "activations/layer2_attention_weight_max": 28.825233459472656,
      "activations/layer2_attention_weight_min": -28.332416534423828,
      "activations/layer3_attention_weight_max": 73.90121459960938,
      "activations/layer3_attention_weight_min": -77.37313842773438,
      "activations/layer4_attention_weight_max": 92.93688201904297,
      "activations/layer4_attention_weight_min": -89.07848358154297,
      "activations/layer5_attention_weight_max": 100.65531921386719,
      "activations/layer5_attention_weight_min": -90.11540222167969,
      "activations/layer6_attention_weight_max": 55.077293395996094,
      "activations/layer6_attention_weight_min": -50.36771011352539,
      "activations/layer7_attention_weight_max": 63.664894104003906,
      "activations/layer7_attention_weight_min": -57.20456314086914,
      "activations/layer8_attention_weight_max": 37.98065948486328,
      "activations/layer8_attention_weight_min": -38.0838737487793,
      "activations/layer9_attention_weight_max": 46.609066009521484,
      "activations/layer9_attention_weight_min": -52.064857482910156,
      "epoch": 4.17,
      "learning_rate": 0.00012432878787878787,
      "loss": 2.8702,
      "step": 71850
    },
    {
      "activations/layer0_attention_weight_max": 14.145570755004883,
      "activations/layer0_attention_weight_min": -14.740814208984375,
      "activations/layer10_attention_weight_max": 38.14341735839844,
      "activations/layer10_attention_weight_min": -34.537322998046875,
      "activations/layer11_attention_weight_max": 29.561939239501953,
      "activations/layer11_attention_weight_min": -27.779842376708984,
      "activations/layer12_attention_weight_max": 21.914621353149414,
      "activations/layer12_attention_weight_min": -24.703216552734375,
      "activations/layer13_attention_weight_max": 50.34922790527344,
      "activations/layer13_attention_weight_min": -40.253623962402344,
      "activations/layer14_attention_weight_max": 39.69804000854492,
      "activations/layer14_attention_weight_min": -28.400087356567383,
      "activations/layer15_attention_weight_max": 49.294647216796875,
      "activations/layer15_attention_weight_min": -31.697559356689453,
      "activations/layer16_attention_weight_max": 32.47045135498047,
      "activations/layer16_attention_weight_min": -27.006196975708008,
      "activations/layer17_attention_weight_max": 58.12160110473633,
      "activations/layer17_attention_weight_min": -37.01035690307617,
      "activations/layer18_attention_weight_max": 50.641075134277344,
      "activations/layer18_attention_weight_min": -32.610694885253906,
      "activations/layer19_attention_weight_max": 21.6754150390625,
      "activations/layer19_attention_weight_min": -21.52705192565918,
      "activations/layer1_attention_weight_max": 18.104564666748047,
      "activations/layer1_attention_weight_min": -17.64043426513672,
      "activations/layer20_attention_weight_max": 22.8109073638916,
      "activations/layer20_attention_weight_min": -21.404823303222656,
      "activations/layer21_attention_weight_max": 42.80836868286133,
      "activations/layer21_attention_weight_min": -32.682403564453125,
      "activations/layer22_attention_weight_max": 34.747615814208984,
      "activations/layer22_attention_weight_min": -26.24811363220215,
      "activations/layer23_attention_weight_max": 31.04291534423828,
      "activations/layer23_attention_weight_min": -17.929344177246094,
      "activations/layer2_attention_weight_max": 30.507795333862305,
      "activations/layer2_attention_weight_min": -32.47596740722656,
      "activations/layer3_attention_weight_max": 78.85020446777344,
      "activations/layer3_attention_weight_min": -84.87162017822266,
      "activations/layer4_attention_weight_max": 100.17677307128906,
      "activations/layer4_attention_weight_min": -94.77682495117188,
      "activations/layer5_attention_weight_max": 107.95189666748047,
      "activations/layer5_attention_weight_min": -97.22359466552734,
      "activations/layer6_attention_weight_max": 56.63168716430664,
      "activations/layer6_attention_weight_min": -52.06062316894531,
      "activations/layer7_attention_weight_max": 64.75498962402344,
      "activations/layer7_attention_weight_min": -59.7526741027832,
      "activations/layer8_attention_weight_max": 40.71464920043945,
      "activations/layer8_attention_weight_min": -39.17893600463867,
      "activations/layer9_attention_weight_max": 45.15248107910156,
      "activations/layer9_attention_weight_min": -45.103057861328125,
      "epoch": 4.18,
      "learning_rate": 0.00012430984848484848,
      "loss": 2.8543,
      "step": 71900
    },
    {
      "activations/layer0_attention_weight_max": 13.615913391113281,
      "activations/layer0_attention_weight_min": -14.446101188659668,
      "activations/layer10_attention_weight_max": 42.413047790527344,
      "activations/layer10_attention_weight_min": -33.80472946166992,
      "activations/layer11_attention_weight_max": 35.07328414916992,
      "activations/layer11_attention_weight_min": -27.073715209960938,
      "activations/layer12_attention_weight_max": 26.08279800415039,
      "activations/layer12_attention_weight_min": -28.876157760620117,
      "activations/layer13_attention_weight_max": 66.27408599853516,
      "activations/layer13_attention_weight_min": -42.5863037109375,
      "activations/layer14_attention_weight_max": 89.63629150390625,
      "activations/layer14_attention_weight_min": -50.41107177734375,
      "activations/layer15_attention_weight_max": 63.5468635559082,
      "activations/layer15_attention_weight_min": -38.278560638427734,
      "activations/layer16_attention_weight_max": 40.12615203857422,
      "activations/layer16_attention_weight_min": -26.291866302490234,
      "activations/layer17_attention_weight_max": 64.07076263427734,
      "activations/layer17_attention_weight_min": -40.056724548339844,
      "activations/layer18_attention_weight_max": 50.49364471435547,
      "activations/layer18_attention_weight_min": -33.00212097167969,
      "activations/layer19_attention_weight_max": 26.364965438842773,
      "activations/layer19_attention_weight_min": -20.793149948120117,
      "activations/layer1_attention_weight_max": 17.02738380432129,
      "activations/layer1_attention_weight_min": -14.057171821594238,
      "activations/layer20_attention_weight_max": 24.250530242919922,
      "activations/layer20_attention_weight_min": -21.473556518554688,
      "activations/layer21_attention_weight_max": 43.00503921508789,
      "activations/layer21_attention_weight_min": -28.82357406616211,
      "activations/layer22_attention_weight_max": 36.43010330200195,
      "activations/layer22_attention_weight_min": -24.959747314453125,
      "activations/layer23_attention_weight_max": 29.557796478271484,
      "activations/layer23_attention_weight_min": -17.44971466064453,
      "activations/layer2_attention_weight_max": 30.207426071166992,
      "activations/layer2_attention_weight_min": -27.490781784057617,
      "activations/layer3_attention_weight_max": 79.84202575683594,
      "activations/layer3_attention_weight_min": -85.03182220458984,
      "activations/layer4_attention_weight_max": 100.9717788696289,
      "activations/layer4_attention_weight_min": -92.30237579345703,
      "activations/layer5_attention_weight_max": 109.50788879394531,
      "activations/layer5_attention_weight_min": -96.24006652832031,
      "activations/layer6_attention_weight_max": 57.83240509033203,
      "activations/layer6_attention_weight_min": -50.54463577270508,
      "activations/layer7_attention_weight_max": 68.55535888671875,
      "activations/layer7_attention_weight_min": -62.101688385009766,
      "activations/layer8_attention_weight_max": 43.1899528503418,
      "activations/layer8_attention_weight_min": -37.28535079956055,
      "activations/layer9_attention_weight_max": 63.52385330200195,
      "activations/layer9_attention_weight_min": -50.29914474487305,
      "epoch": 4.18,
      "learning_rate": 0.00012429090909090907,
      "loss": 2.8658,
      "step": 71950
    },
    {
      "activations/layer0_attention_weight_max": 14.679777145385742,
      "activations/layer0_attention_weight_min": -14.958456039428711,
      "activations/layer10_attention_weight_max": 47.92625045776367,
      "activations/layer10_attention_weight_min": -35.117530822753906,
      "activations/layer11_attention_weight_max": 35.338050842285156,
      "activations/layer11_attention_weight_min": -26.842634201049805,
      "activations/layer12_attention_weight_max": 24.491683959960938,
      "activations/layer12_attention_weight_min": -30.888547897338867,
      "activations/layer13_attention_weight_max": 55.69301223754883,
      "activations/layer13_attention_weight_min": -42.14623260498047,
      "activations/layer14_attention_weight_max": 46.29887008666992,
      "activations/layer14_attention_weight_min": -32.29574203491211,
      "activations/layer15_attention_weight_max": 46.65721130371094,
      "activations/layer15_attention_weight_min": -32.81837844848633,
      "activations/layer16_attention_weight_max": 38.79265213012695,
      "activations/layer16_attention_weight_min": -27.439573287963867,
      "activations/layer17_attention_weight_max": 64.73834228515625,
      "activations/layer17_attention_weight_min": -39.9699592590332,
      "activations/layer18_attention_weight_max": 54.19313430786133,
      "activations/layer18_attention_weight_min": -33.0025749206543,
      "activations/layer19_attention_weight_max": 21.816423416137695,
      "activations/layer19_attention_weight_min": -20.939355850219727,
      "activations/layer1_attention_weight_max": 17.397382736206055,
      "activations/layer1_attention_weight_min": -14.860394477844238,
      "activations/layer20_attention_weight_max": 22.908727645874023,
      "activations/layer20_attention_weight_min": -23.23792839050293,
      "activations/layer21_attention_weight_max": 45.12214660644531,
      "activations/layer21_attention_weight_min": -30.42742919921875,
      "activations/layer22_attention_weight_max": 37.267051696777344,
      "activations/layer22_attention_weight_min": -23.92550277709961,
      "activations/layer23_attention_weight_max": 30.19180679321289,
      "activations/layer23_attention_weight_min": -17.744815826416016,
      "activations/layer2_attention_weight_max": 30.014875411987305,
      "activations/layer2_attention_weight_min": -28.493789672851562,
      "activations/layer3_attention_weight_max": 76.1947250366211,
      "activations/layer3_attention_weight_min": -78.98578643798828,
      "activations/layer4_attention_weight_max": 98.65656280517578,
      "activations/layer4_attention_weight_min": -89.4646224975586,
      "activations/layer5_attention_weight_max": 106.91439819335938,
      "activations/layer5_attention_weight_min": -92.51366424560547,
      "activations/layer6_attention_weight_max": 57.59702682495117,
      "activations/layer6_attention_weight_min": -50.06174087524414,
      "activations/layer7_attention_weight_max": 62.552547454833984,
      "activations/layer7_attention_weight_min": -65.05496978759766,
      "activations/layer8_attention_weight_max": 40.68259048461914,
      "activations/layer8_attention_weight_min": -38.60520553588867,
      "activations/layer9_attention_weight_max": 49.90817642211914,
      "activations/layer9_attention_weight_min": -45.18174362182617,
      "epoch": 4.18,
      "learning_rate": 0.00012427196969696968,
      "loss": 2.8575,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_loss": 2.822265625,
      "eval_runtime": 8.4339,
      "eval_samples_per_second": 509.138,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_openwebtext_loss": 2.822265625,
      "eval_openwebtext_ppl": 16.8149038152427,
      "eval_openwebtext_runtime": 8.4339,
      "eval_openwebtext_samples_per_second": 509.138,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_wikitext_loss": 3.0546875,
      "eval_wikitext_ppl": 21.214554942517648,
      "eval_wikitext_runtime": 1.9447,
      "eval_wikitext_samples_per_second": 234.489,
      "step": 72000
    },
    {
      "epoch": 4.18,
      "eval_lambada_loss": 2.861328125,
      "eval_lambada_ppl": 17.484733434196265,
      "eval_lambada_runtime": 9.4939,
      "eval_lambada_samples_per_second": 512.858,
      "step": 72000
    },
    {
      "activations/layer0_attention_weight_max": 14.936624526977539,
      "activations/layer0_attention_weight_min": -14.904111862182617,
      "activations/layer10_attention_weight_max": 46.47243881225586,
      "activations/layer10_attention_weight_min": -38.510196685791016,
      "activations/layer11_attention_weight_max": 41.053340911865234,
      "activations/layer11_attention_weight_min": -28.114538192749023,
      "activations/layer12_attention_weight_max": 24.440269470214844,
      "activations/layer12_attention_weight_min": -24.082677841186523,
      "activations/layer13_attention_weight_max": 89.95355987548828,
      "activations/layer13_attention_weight_min": -45.034507751464844,
      "activations/layer14_attention_weight_max": 98.64159393310547,
      "activations/layer14_attention_weight_min": -51.7850341796875,
      "activations/layer15_attention_weight_max": 69.11528015136719,
      "activations/layer15_attention_weight_min": -40.56087112426758,
      "activations/layer16_attention_weight_max": 49.644866943359375,
      "activations/layer16_attention_weight_min": -26.3485164642334,
      "activations/layer17_attention_weight_max": 63.73412322998047,
      "activations/layer17_attention_weight_min": -42.9366340637207,
      "activations/layer18_attention_weight_max": 61.127723693847656,
      "activations/layer18_attention_weight_min": -35.62571716308594,
      "activations/layer19_attention_weight_max": 32.24001693725586,
      "activations/layer19_attention_weight_min": -20.45473861694336,
      "activations/layer1_attention_weight_max": 17.330041885375977,
      "activations/layer1_attention_weight_min": -14.11388874053955,
      "activations/layer20_attention_weight_max": 30.342607498168945,
      "activations/layer20_attention_weight_min": -19.850954055786133,
      "activations/layer21_attention_weight_max": 59.18937683105469,
      "activations/layer21_attention_weight_min": -30.651290893554688,
      "activations/layer22_attention_weight_max": 42.62434005737305,
      "activations/layer22_attention_weight_min": -24.940013885498047,
      "activations/layer23_attention_weight_max": 30.885007858276367,
      "activations/layer23_attention_weight_min": -17.6201114654541,
      "activations/layer2_attention_weight_max": 31.389684677124023,
      "activations/layer2_attention_weight_min": -30.860816955566406,
      "activations/layer3_attention_weight_max": 79.35742950439453,
      "activations/layer3_attention_weight_min": -86.64105987548828,
      "activations/layer4_attention_weight_max": 105.1030044555664,
      "activations/layer4_attention_weight_min": -101.0080795288086,
      "activations/layer5_attention_weight_max": 122.05673217773438,
      "activations/layer5_attention_weight_min": -106.51712036132812,
      "activations/layer6_attention_weight_max": 63.074073791503906,
      "activations/layer6_attention_weight_min": -57.244171142578125,
      "activations/layer7_attention_weight_max": 78.33557891845703,
      "activations/layer7_attention_weight_min": -69.66699981689453,
      "activations/layer8_attention_weight_max": 51.255916595458984,
      "activations/layer8_attention_weight_min": -44.26111602783203,
      "activations/layer9_attention_weight_max": 68.76439666748047,
      "activations/layer9_attention_weight_min": -54.846282958984375,
      "epoch": 4.19,
      "learning_rate": 0.0001242530303030303,
      "loss": 2.8599,
      "step": 72050
    },
    {
      "activations/layer0_attention_weight_max": 13.818586349487305,
      "activations/layer0_attention_weight_min": -15.176067352294922,
      "activations/layer10_attention_weight_max": 52.3577995300293,
      "activations/layer10_attention_weight_min": -40.093196868896484,
      "activations/layer11_attention_weight_max": 41.26753234863281,
      "activations/layer11_attention_weight_min": -29.480358123779297,
      "activations/layer12_attention_weight_max": 40.903053283691406,
      "activations/layer12_attention_weight_min": -26.863584518432617,
      "activations/layer13_attention_weight_max": 85.61200714111328,
      "activations/layer13_attention_weight_min": -36.50961685180664,
      "activations/layer14_attention_weight_max": 108.33645629882812,
      "activations/layer14_attention_weight_min": -44.53489303588867,
      "activations/layer15_attention_weight_max": 72.71430969238281,
      "activations/layer15_attention_weight_min": -31.078054428100586,
      "activations/layer16_attention_weight_max": 43.449378967285156,
      "activations/layer16_attention_weight_min": -27.6092586517334,
      "activations/layer17_attention_weight_max": 61.66619110107422,
      "activations/layer17_attention_weight_min": -38.12797164916992,
      "activations/layer18_attention_weight_max": 55.20973587036133,
      "activations/layer18_attention_weight_min": -34.40192794799805,
      "activations/layer19_attention_weight_max": 28.262388229370117,
      "activations/layer19_attention_weight_min": -22.512250900268555,
      "activations/layer1_attention_weight_max": 17.45059585571289,
      "activations/layer1_attention_weight_min": -14.832088470458984,
      "activations/layer20_attention_weight_max": 23.274721145629883,
      "activations/layer20_attention_weight_min": -23.665239334106445,
      "activations/layer21_attention_weight_max": 43.721736907958984,
      "activations/layer21_attention_weight_min": -28.321300506591797,
      "activations/layer22_attention_weight_max": 35.12580108642578,
      "activations/layer22_attention_weight_min": -23.52289581298828,
      "activations/layer23_attention_weight_max": 25.138015747070312,
      "activations/layer23_attention_weight_min": -15.893717765808105,
      "activations/layer2_attention_weight_max": 31.81674575805664,
      "activations/layer2_attention_weight_min": -30.486007690429688,
      "activations/layer3_attention_weight_max": 83.3260269165039,
      "activations/layer3_attention_weight_min": -84.15868377685547,
      "activations/layer4_attention_weight_max": 103.4796371459961,
      "activations/layer4_attention_weight_min": -95.48621368408203,
      "activations/layer5_attention_weight_max": 112.91036224365234,
      "activations/layer5_attention_weight_min": -100.57064056396484,
      "activations/layer6_attention_weight_max": 60.88698196411133,
      "activations/layer6_attention_weight_min": -56.644805908203125,
      "activations/layer7_attention_weight_max": 79.15328979492188,
      "activations/layer7_attention_weight_min": -71.21150207519531,
      "activations/layer8_attention_weight_max": 54.44032669067383,
      "activations/layer8_attention_weight_min": -48.109561920166016,
      "activations/layer9_attention_weight_max": 72.4761734008789,
      "activations/layer9_attention_weight_min": -62.70280838012695,
      "epoch": 4.19,
      "learning_rate": 0.0001242340909090909,
      "loss": 2.8661,
      "step": 72100
    },
    {
      "activations/layer0_attention_weight_max": 14.730487823486328,
      "activations/layer0_attention_weight_min": -14.829343795776367,
      "activations/layer10_attention_weight_max": 44.35880661010742,
      "activations/layer10_attention_weight_min": -33.86798095703125,
      "activations/layer11_attention_weight_max": 35.03181457519531,
      "activations/layer11_attention_weight_min": -26.205196380615234,
      "activations/layer12_attention_weight_max": 27.067325592041016,
      "activations/layer12_attention_weight_min": -27.69687271118164,
      "activations/layer13_attention_weight_max": 63.426177978515625,
      "activations/layer13_attention_weight_min": -41.331275939941406,
      "activations/layer14_attention_weight_max": 58.8630256652832,
      "activations/layer14_attention_weight_min": -31.86896514892578,
      "activations/layer15_attention_weight_max": 55.31572341918945,
      "activations/layer15_attention_weight_min": -35.13839340209961,
      "activations/layer16_attention_weight_max": 54.91242980957031,
      "activations/layer16_attention_weight_min": -27.157730102539062,
      "activations/layer17_attention_weight_max": 78.1850357055664,
      "activations/layer17_attention_weight_min": -40.38533401489258,
      "activations/layer18_attention_weight_max": 63.85239791870117,
      "activations/layer18_attention_weight_min": -34.21734619140625,
      "activations/layer19_attention_weight_max": 30.27154541015625,
      "activations/layer19_attention_weight_min": -21.08473014831543,
      "activations/layer1_attention_weight_max": 17.301063537597656,
      "activations/layer1_attention_weight_min": -17.30559730529785,
      "activations/layer20_attention_weight_max": 34.38102722167969,
      "activations/layer20_attention_weight_min": -21.379589080810547,
      "activations/layer21_attention_weight_max": 55.24399185180664,
      "activations/layer21_attention_weight_min": -25.387876510620117,
      "activations/layer22_attention_weight_max": 42.68527603149414,
      "activations/layer22_attention_weight_min": -25.750329971313477,
      "activations/layer23_attention_weight_max": 32.1787223815918,
      "activations/layer23_attention_weight_min": -16.112443923950195,
      "activations/layer2_attention_weight_max": 29.27663230895996,
      "activations/layer2_attention_weight_min": -28.564695358276367,
      "activations/layer3_attention_weight_max": 75.84814453125,
      "activations/layer3_attention_weight_min": -79.48271179199219,
      "activations/layer4_attention_weight_max": 97.2588119506836,
      "activations/layer4_attention_weight_min": -89.56741333007812,
      "activations/layer5_attention_weight_max": 103.88707733154297,
      "activations/layer5_attention_weight_min": -93.0758056640625,
      "activations/layer6_attention_weight_max": 55.77490234375,
      "activations/layer6_attention_weight_min": -50.35206604003906,
      "activations/layer7_attention_weight_max": 61.98576736450195,
      "activations/layer7_attention_weight_min": -57.774959564208984,
      "activations/layer8_attention_weight_max": 40.284175872802734,
      "activations/layer8_attention_weight_min": -38.092018127441406,
      "activations/layer9_attention_weight_max": 47.661346435546875,
      "activations/layer9_attention_weight_min": -44.54652786254883,
      "epoch": 4.19,
      "learning_rate": 0.0001242151515151515,
      "loss": 2.871,
      "step": 72150
    },
    {
      "activations/layer0_attention_weight_max": 14.331997871398926,
      "activations/layer0_attention_weight_min": -15.177867889404297,
      "activations/layer10_attention_weight_max": 42.262638092041016,
      "activations/layer10_attention_weight_min": -36.850337982177734,
      "activations/layer11_attention_weight_max": 30.291217803955078,
      "activations/layer11_attention_weight_min": -27.23352813720703,
      "activations/layer12_attention_weight_max": 21.711389541625977,
      "activations/layer12_attention_weight_min": -23.874305725097656,
      "activations/layer13_attention_weight_max": 47.38756561279297,
      "activations/layer13_attention_weight_min": -38.8649787902832,
      "activations/layer14_attention_weight_max": 46.50240707397461,
      "activations/layer14_attention_weight_min": -34.325531005859375,
      "activations/layer15_attention_weight_max": 41.5257568359375,
      "activations/layer15_attention_weight_min": -29.661264419555664,
      "activations/layer16_attention_weight_max": 35.9798469543457,
      "activations/layer16_attention_weight_min": -28.757646560668945,
      "activations/layer17_attention_weight_max": 57.13053512573242,
      "activations/layer17_attention_weight_min": -37.48610305786133,
      "activations/layer18_attention_weight_max": 48.80845642089844,
      "activations/layer18_attention_weight_min": -33.47452926635742,
      "activations/layer19_attention_weight_max": 21.88465118408203,
      "activations/layer19_attention_weight_min": -18.181509017944336,
      "activations/layer1_attention_weight_max": 17.354530334472656,
      "activations/layer1_attention_weight_min": -13.223864555358887,
      "activations/layer20_attention_weight_max": 23.464431762695312,
      "activations/layer20_attention_weight_min": -21.752910614013672,
      "activations/layer21_attention_weight_max": 48.09776306152344,
      "activations/layer21_attention_weight_min": -29.90082359313965,
      "activations/layer22_attention_weight_max": 37.03568649291992,
      "activations/layer22_attention_weight_min": -22.665742874145508,
      "activations/layer23_attention_weight_max": 27.468414306640625,
      "activations/layer23_attention_weight_min": -17.329496383666992,
      "activations/layer2_attention_weight_max": 29.121524810791016,
      "activations/layer2_attention_weight_min": -27.439281463623047,
      "activations/layer3_attention_weight_max": 78.98246765136719,
      "activations/layer3_attention_weight_min": -80.58990478515625,
      "activations/layer4_attention_weight_max": 104.29786682128906,
      "activations/layer4_attention_weight_min": -102.13579559326172,
      "activations/layer5_attention_weight_max": 109.96482849121094,
      "activations/layer5_attention_weight_min": -102.67555236816406,
      "activations/layer6_attention_weight_max": 61.204856872558594,
      "activations/layer6_attention_weight_min": -54.235862731933594,
      "activations/layer7_attention_weight_max": 70.61001586914062,
      "activations/layer7_attention_weight_min": -64.33468627929688,
      "activations/layer8_attention_weight_max": 44.575843811035156,
      "activations/layer8_attention_weight_min": -40.993629455566406,
      "activations/layer9_attention_weight_max": 48.882808685302734,
      "activations/layer9_attention_weight_min": -46.988128662109375,
      "epoch": 4.2,
      "learning_rate": 0.00012419621212121212,
      "loss": 2.8652,
      "step": 72200
    },
    {
      "activations/layer0_attention_weight_max": 15.49757194519043,
      "activations/layer0_attention_weight_min": -15.020747184753418,
      "activations/layer10_attention_weight_max": 41.177791595458984,
      "activations/layer10_attention_weight_min": -33.62957763671875,
      "activations/layer11_attention_weight_max": 31.863088607788086,
      "activations/layer11_attention_weight_min": -28.27363395690918,
      "activations/layer12_attention_weight_max": 26.187528610229492,
      "activations/layer12_attention_weight_min": -26.379465103149414,
      "activations/layer13_attention_weight_max": 51.27505874633789,
      "activations/layer13_attention_weight_min": -37.98788070678711,
      "activations/layer14_attention_weight_max": 45.57334899902344,
      "activations/layer14_attention_weight_min": -31.06049346923828,
      "activations/layer15_attention_weight_max": 41.76157760620117,
      "activations/layer15_attention_weight_min": -32.3971061706543,
      "activations/layer16_attention_weight_max": 36.71485900878906,
      "activations/layer16_attention_weight_min": -27.48606300354004,
      "activations/layer17_attention_weight_max": 59.50715637207031,
      "activations/layer17_attention_weight_min": -37.26499557495117,
      "activations/layer18_attention_weight_max": 51.015968322753906,
      "activations/layer18_attention_weight_min": -30.361623764038086,
      "activations/layer19_attention_weight_max": 21.71192169189453,
      "activations/layer19_attention_weight_min": -18.783422470092773,
      "activations/layer1_attention_weight_max": 17.07610511779785,
      "activations/layer1_attention_weight_min": -16.721105575561523,
      "activations/layer20_attention_weight_max": 19.575695037841797,
      "activations/layer20_attention_weight_min": -19.59565544128418,
      "activations/layer21_attention_weight_max": 38.83976745605469,
      "activations/layer21_attention_weight_min": -25.634191513061523,
      "activations/layer22_attention_weight_max": 36.101749420166016,
      "activations/layer22_attention_weight_min": -25.26854133605957,
      "activations/layer23_attention_weight_max": 27.94701385498047,
      "activations/layer23_attention_weight_min": -18.565940856933594,
      "activations/layer2_attention_weight_max": 30.510669708251953,
      "activations/layer2_attention_weight_min": -28.625938415527344,
      "activations/layer3_attention_weight_max": 81.77059936523438,
      "activations/layer3_attention_weight_min": -82.0632095336914,
      "activations/layer4_attention_weight_max": 106.4987564086914,
      "activations/layer4_attention_weight_min": -93.47537231445312,
      "activations/layer5_attention_weight_max": 114.99044036865234,
      "activations/layer5_attention_weight_min": -98.2080078125,
      "activations/layer6_attention_weight_max": 62.04779052734375,
      "activations/layer6_attention_weight_min": -55.11224365234375,
      "activations/layer7_attention_weight_max": 65.88214874267578,
      "activations/layer7_attention_weight_min": -64.07553100585938,
      "activations/layer8_attention_weight_max": 43.27798080444336,
      "activations/layer8_attention_weight_min": -41.342742919921875,
      "activations/layer9_attention_weight_max": 53.753841400146484,
      "activations/layer9_attention_weight_min": -47.782737731933594,
      "epoch": 4.2,
      "learning_rate": 0.0001241772727272727,
      "loss": 2.8588,
      "step": 72250
    },
    {
      "activations/layer0_attention_weight_max": 14.6924467086792,
      "activations/layer0_attention_weight_min": -15.05295467376709,
      "activations/layer10_attention_weight_max": 47.19179153442383,
      "activations/layer10_attention_weight_min": -35.3502197265625,
      "activations/layer11_attention_weight_max": 45.56231689453125,
      "activations/layer11_attention_weight_min": -28.429845809936523,
      "activations/layer12_attention_weight_max": 28.33043670654297,
      "activations/layer12_attention_weight_min": -27.06828498840332,
      "activations/layer13_attention_weight_max": 82.7088623046875,
      "activations/layer13_attention_weight_min": -42.70835494995117,
      "activations/layer14_attention_weight_max": 71.09149169921875,
      "activations/layer14_attention_weight_min": -33.61198043823242,
      "activations/layer15_attention_weight_max": 56.86571502685547,
      "activations/layer15_attention_weight_min": -30.983030319213867,
      "activations/layer16_attention_weight_max": 45.04275894165039,
      "activations/layer16_attention_weight_min": -25.051593780517578,
      "activations/layer17_attention_weight_max": 67.035400390625,
      "activations/layer17_attention_weight_min": -38.26115798950195,
      "activations/layer18_attention_weight_max": 58.12525177001953,
      "activations/layer18_attention_weight_min": -32.01351547241211,
      "activations/layer19_attention_weight_max": 32.21352767944336,
      "activations/layer19_attention_weight_min": -20.22069549560547,
      "activations/layer1_attention_weight_max": 20.852821350097656,
      "activations/layer1_attention_weight_min": -16.858449935913086,
      "activations/layer20_attention_weight_max": 29.038591384887695,
      "activations/layer20_attention_weight_min": -19.013267517089844,
      "activations/layer21_attention_weight_max": 51.99795913696289,
      "activations/layer21_attention_weight_min": -26.496322631835938,
      "activations/layer22_attention_weight_max": 42.279178619384766,
      "activations/layer22_attention_weight_min": -22.52186393737793,
      "activations/layer23_attention_weight_max": 31.573270797729492,
      "activations/layer23_attention_weight_min": -16.093585968017578,
      "activations/layer2_attention_weight_max": 30.587291717529297,
      "activations/layer2_attention_weight_min": -27.962127685546875,
      "activations/layer3_attention_weight_max": 83.32191467285156,
      "activations/layer3_attention_weight_min": -83.50082397460938,
      "activations/layer4_attention_weight_max": 104.3996353149414,
      "activations/layer4_attention_weight_min": -99.33357238769531,
      "activations/layer5_attention_weight_max": 112.99848937988281,
      "activations/layer5_attention_weight_min": -98.10273742675781,
      "activations/layer6_attention_weight_max": 61.22401809692383,
      "activations/layer6_attention_weight_min": -55.83495330810547,
      "activations/layer7_attention_weight_max": 74.10832977294922,
      "activations/layer7_attention_weight_min": -66.96692657470703,
      "activations/layer8_attention_weight_max": 46.177730560302734,
      "activations/layer8_attention_weight_min": -41.51865005493164,
      "activations/layer9_attention_weight_max": 69.43201446533203,
      "activations/layer9_attention_weight_min": -51.039817810058594,
      "epoch": 4.2,
      "learning_rate": 0.00012415833333333332,
      "loss": 2.8719,
      "step": 72300
    },
    {
      "activations/layer0_attention_weight_max": 14.479352951049805,
      "activations/layer0_attention_weight_min": -15.195165634155273,
      "activations/layer10_attention_weight_max": 39.38494873046875,
      "activations/layer10_attention_weight_min": -34.354461669921875,
      "activations/layer11_attention_weight_max": 30.1180477142334,
      "activations/layer11_attention_weight_min": -27.063339233398438,
      "activations/layer12_attention_weight_max": 23.301475524902344,
      "activations/layer12_attention_weight_min": -24.834381103515625,
      "activations/layer13_attention_weight_max": 52.444557189941406,
      "activations/layer13_attention_weight_min": -39.74913787841797,
      "activations/layer14_attention_weight_max": 47.03969192504883,
      "activations/layer14_attention_weight_min": -31.406646728515625,
      "activations/layer15_attention_weight_max": 43.3428955078125,
      "activations/layer15_attention_weight_min": -31.02931785583496,
      "activations/layer16_attention_weight_max": 35.22193145751953,
      "activations/layer16_attention_weight_min": -26.507707595825195,
      "activations/layer17_attention_weight_max": 60.32338333129883,
      "activations/layer17_attention_weight_min": -36.073997497558594,
      "activations/layer18_attention_weight_max": 54.98404312133789,
      "activations/layer18_attention_weight_min": -31.760860443115234,
      "activations/layer19_attention_weight_max": 21.672016143798828,
      "activations/layer19_attention_weight_min": -20.431110382080078,
      "activations/layer1_attention_weight_max": 19.783597946166992,
      "activations/layer1_attention_weight_min": -15.57396411895752,
      "activations/layer20_attention_weight_max": 21.043989181518555,
      "activations/layer20_attention_weight_min": -21.059843063354492,
      "activations/layer21_attention_weight_max": 39.296875,
      "activations/layer21_attention_weight_min": -28.624711990356445,
      "activations/layer22_attention_weight_max": 34.587623596191406,
      "activations/layer22_attention_weight_min": -25.184202194213867,
      "activations/layer23_attention_weight_max": 26.483640670776367,
      "activations/layer23_attention_weight_min": -17.87430763244629,
      "activations/layer2_attention_weight_max": 31.036481857299805,
      "activations/layer2_attention_weight_min": -27.334829330444336,
      "activations/layer3_attention_weight_max": 81.6609115600586,
      "activations/layer3_attention_weight_min": -78.58146667480469,
      "activations/layer4_attention_weight_max": 99.66801452636719,
      "activations/layer4_attention_weight_min": -91.97721862792969,
      "activations/layer5_attention_weight_max": 107.16044616699219,
      "activations/layer5_attention_weight_min": -96.27247619628906,
      "activations/layer6_attention_weight_max": 58.716941833496094,
      "activations/layer6_attention_weight_min": -53.96388626098633,
      "activations/layer7_attention_weight_max": 66.24710845947266,
      "activations/layer7_attention_weight_min": -62.08613586425781,
      "activations/layer8_attention_weight_max": 40.17108154296875,
      "activations/layer8_attention_weight_min": -39.60687255859375,
      "activations/layer9_attention_weight_max": 47.3873405456543,
      "activations/layer9_attention_weight_min": -46.18898010253906,
      "epoch": 4.2,
      "learning_rate": 0.00012413939393939394,
      "loss": 2.8688,
      "step": 72350
    },
    {
      "activations/layer0_attention_weight_max": 13.564645767211914,
      "activations/layer0_attention_weight_min": -15.116299629211426,
      "activations/layer10_attention_weight_max": 37.17647933959961,
      "activations/layer10_attention_weight_min": -33.817623138427734,
      "activations/layer11_attention_weight_max": 31.11223602294922,
      "activations/layer11_attention_weight_min": -26.523345947265625,
      "activations/layer12_attention_weight_max": 19.626049041748047,
      "activations/layer12_attention_weight_min": -23.060422897338867,
      "activations/layer13_attention_weight_max": 45.67615509033203,
      "activations/layer13_attention_weight_min": -41.571834564208984,
      "activations/layer14_attention_weight_max": 46.8719482421875,
      "activations/layer14_attention_weight_min": -35.84494400024414,
      "activations/layer15_attention_weight_max": 37.35820388793945,
      "activations/layer15_attention_weight_min": -29.962697982788086,
      "activations/layer16_attention_weight_max": 31.342809677124023,
      "activations/layer16_attention_weight_min": -26.839614868164062,
      "activations/layer17_attention_weight_max": 54.0157470703125,
      "activations/layer17_attention_weight_min": -35.5394401550293,
      "activations/layer18_attention_weight_max": 49.8401985168457,
      "activations/layer18_attention_weight_min": -32.513301849365234,
      "activations/layer19_attention_weight_max": 21.361623764038086,
      "activations/layer19_attention_weight_min": -18.639623641967773,
      "activations/layer1_attention_weight_max": 18.02074432373047,
      "activations/layer1_attention_weight_min": -14.298195838928223,
      "activations/layer20_attention_weight_max": 19.95720100402832,
      "activations/layer20_attention_weight_min": -20.337810516357422,
      "activations/layer21_attention_weight_max": 41.49775695800781,
      "activations/layer21_attention_weight_min": -31.851652145385742,
      "activations/layer22_attention_weight_max": 30.93965721130371,
      "activations/layer22_attention_weight_min": -23.480026245117188,
      "activations/layer23_attention_weight_max": 25.872913360595703,
      "activations/layer23_attention_weight_min": -16.541893005371094,
      "activations/layer2_attention_weight_max": 30.127859115600586,
      "activations/layer2_attention_weight_min": -27.065048217773438,
      "activations/layer3_attention_weight_max": 81.60881042480469,
      "activations/layer3_attention_weight_min": -82.46778106689453,
      "activations/layer4_attention_weight_max": 103.0108642578125,
      "activations/layer4_attention_weight_min": -93.1573486328125,
      "activations/layer5_attention_weight_max": 112.2605209350586,
      "activations/layer5_attention_weight_min": -96.68114471435547,
      "activations/layer6_attention_weight_max": 56.960487365722656,
      "activations/layer6_attention_weight_min": -50.301597595214844,
      "activations/layer7_attention_weight_max": 62.329463958740234,
      "activations/layer7_attention_weight_min": -59.619712829589844,
      "activations/layer8_attention_weight_max": 41.161170959472656,
      "activations/layer8_attention_weight_min": -38.57929992675781,
      "activations/layer9_attention_weight_max": 46.48586654663086,
      "activations/layer9_attention_weight_min": -49.60295486450195,
      "epoch": 4.21,
      "learning_rate": 0.00012412045454545452,
      "loss": 2.8593,
      "step": 72400
    },
    {
      "activations/layer0_attention_weight_max": 16.031360626220703,
      "activations/layer0_attention_weight_min": -14.792706489562988,
      "activations/layer10_attention_weight_max": 38.77033233642578,
      "activations/layer10_attention_weight_min": -34.898529052734375,
      "activations/layer11_attention_weight_max": 35.61655044555664,
      "activations/layer11_attention_weight_min": -26.6197509765625,
      "activations/layer12_attention_weight_max": 25.857614517211914,
      "activations/layer12_attention_weight_min": -25.360492706298828,
      "activations/layer13_attention_weight_max": 43.433780670166016,
      "activations/layer13_attention_weight_min": -38.4195671081543,
      "activations/layer14_attention_weight_max": 52.274559020996094,
      "activations/layer14_attention_weight_min": -36.09095764160156,
      "activations/layer15_attention_weight_max": 43.67986297607422,
      "activations/layer15_attention_weight_min": -30.936893463134766,
      "activations/layer16_attention_weight_max": 37.1728515625,
      "activations/layer16_attention_weight_min": -27.323204040527344,
      "activations/layer17_attention_weight_max": 58.9510498046875,
      "activations/layer17_attention_weight_min": -40.02246856689453,
      "activations/layer18_attention_weight_max": 46.142127990722656,
      "activations/layer18_attention_weight_min": -35.250797271728516,
      "activations/layer19_attention_weight_max": 20.707374572753906,
      "activations/layer19_attention_weight_min": -20.088159561157227,
      "activations/layer1_attention_weight_max": 17.499313354492188,
      "activations/layer1_attention_weight_min": -15.600717544555664,
      "activations/layer20_attention_weight_max": 21.300626754760742,
      "activations/layer20_attention_weight_min": -21.49692153930664,
      "activations/layer21_attention_weight_max": 37.0496826171875,
      "activations/layer21_attention_weight_min": -26.82084846496582,
      "activations/layer22_attention_weight_max": 33.655906677246094,
      "activations/layer22_attention_weight_min": -24.36742401123047,
      "activations/layer23_attention_weight_max": 25.966777801513672,
      "activations/layer23_attention_weight_min": -17.35763931274414,
      "activations/layer2_attention_weight_max": 29.37384796142578,
      "activations/layer2_attention_weight_min": -27.907665252685547,
      "activations/layer3_attention_weight_max": 79.74160766601562,
      "activations/layer3_attention_weight_min": -85.98460388183594,
      "activations/layer4_attention_weight_max": 105.39627838134766,
      "activations/layer4_attention_weight_min": -97.59441375732422,
      "activations/layer5_attention_weight_max": 115.57186889648438,
      "activations/layer5_attention_weight_min": -98.92411804199219,
      "activations/layer6_attention_weight_max": 61.94737243652344,
      "activations/layer6_attention_weight_min": -53.36404037475586,
      "activations/layer7_attention_weight_max": 71.97959899902344,
      "activations/layer7_attention_weight_min": -61.90019226074219,
      "activations/layer8_attention_weight_max": 42.10696792602539,
      "activations/layer8_attention_weight_min": -39.20658493041992,
      "activations/layer9_attention_weight_max": 63.288970947265625,
      "activations/layer9_attention_weight_min": -47.15313720703125,
      "epoch": 4.21,
      "learning_rate": 0.00012410151515151514,
      "loss": 2.8703,
      "step": 72450
    },
    {
      "activations/layer0_attention_weight_max": 15.633024215698242,
      "activations/layer0_attention_weight_min": -14.886762619018555,
      "activations/layer10_attention_weight_max": 36.807193756103516,
      "activations/layer10_attention_weight_min": -32.92384338378906,
      "activations/layer11_attention_weight_max": 29.315475463867188,
      "activations/layer11_attention_weight_min": -26.330989837646484,
      "activations/layer12_attention_weight_max": 27.354944229125977,
      "activations/layer12_attention_weight_min": -24.27776336669922,
      "activations/layer13_attention_weight_max": 47.857017517089844,
      "activations/layer13_attention_weight_min": -38.44449996948242,
      "activations/layer14_attention_weight_max": 49.45371627807617,
      "activations/layer14_attention_weight_min": -35.279945373535156,
      "activations/layer15_attention_weight_max": 50.47169494628906,
      "activations/layer15_attention_weight_min": -32.79718017578125,
      "activations/layer16_attention_weight_max": 35.548763275146484,
      "activations/layer16_attention_weight_min": -26.232511520385742,
      "activations/layer17_attention_weight_max": 56.533729553222656,
      "activations/layer17_attention_weight_min": -38.92255783081055,
      "activations/layer18_attention_weight_max": 48.25782012939453,
      "activations/layer18_attention_weight_min": -33.03419494628906,
      "activations/layer19_attention_weight_max": 25.329774856567383,
      "activations/layer19_attention_weight_min": -17.850261688232422,
      "activations/layer1_attention_weight_max": 17.839052200317383,
      "activations/layer1_attention_weight_min": -14.52889347076416,
      "activations/layer20_attention_weight_max": 20.874013900756836,
      "activations/layer20_attention_weight_min": -19.960763931274414,
      "activations/layer21_attention_weight_max": 37.2623176574707,
      "activations/layer21_attention_weight_min": -26.80526351928711,
      "activations/layer22_attention_weight_max": 35.35086441040039,
      "activations/layer22_attention_weight_min": -22.346054077148438,
      "activations/layer23_attention_weight_max": 26.76154899597168,
      "activations/layer23_attention_weight_min": -16.896656036376953,
      "activations/layer2_attention_weight_max": 31.012792587280273,
      "activations/layer2_attention_weight_min": -29.713926315307617,
      "activations/layer3_attention_weight_max": 84.39439392089844,
      "activations/layer3_attention_weight_min": -87.6871566772461,
      "activations/layer4_attention_weight_max": 99.92376708984375,
      "activations/layer4_attention_weight_min": -94.6077880859375,
      "activations/layer5_attention_weight_max": 109.95655822753906,
      "activations/layer5_attention_weight_min": -91.84569549560547,
      "activations/layer6_attention_weight_max": 56.81206130981445,
      "activations/layer6_attention_weight_min": -51.505558013916016,
      "activations/layer7_attention_weight_max": 62.21026611328125,
      "activations/layer7_attention_weight_min": -59.860076904296875,
      "activations/layer8_attention_weight_max": 39.416255950927734,
      "activations/layer8_attention_weight_min": -40.51310729980469,
      "activations/layer9_attention_weight_max": 54.78202438354492,
      "activations/layer9_attention_weight_min": -43.90203857421875,
      "epoch": 4.21,
      "learning_rate": 0.00012408257575757575,
      "loss": 2.8666,
      "step": 72500
    },
    {
      "activations/layer0_attention_weight_max": 13.812871932983398,
      "activations/layer0_attention_weight_min": -14.853118896484375,
      "activations/layer10_attention_weight_max": 34.872230529785156,
      "activations/layer10_attention_weight_min": -31.310758590698242,
      "activations/layer11_attention_weight_max": 28.234357833862305,
      "activations/layer11_attention_weight_min": -26.315608978271484,
      "activations/layer12_attention_weight_max": 23.57720375061035,
      "activations/layer12_attention_weight_min": -25.035947799682617,
      "activations/layer13_attention_weight_max": 43.871009826660156,
      "activations/layer13_attention_weight_min": -35.352237701416016,
      "activations/layer14_attention_weight_max": 39.87998962402344,
      "activations/layer14_attention_weight_min": -28.71110725402832,
      "activations/layer15_attention_weight_max": 40.87519454956055,
      "activations/layer15_attention_weight_min": -32.69352722167969,
      "activations/layer16_attention_weight_max": 34.9971809387207,
      "activations/layer16_attention_weight_min": -25.77545738220215,
      "activations/layer17_attention_weight_max": 58.60721206665039,
      "activations/layer17_attention_weight_min": -36.47208786010742,
      "activations/layer18_attention_weight_max": 54.13552474975586,
      "activations/layer18_attention_weight_min": -34.073062896728516,
      "activations/layer19_attention_weight_max": 20.954763412475586,
      "activations/layer19_attention_weight_min": -20.173709869384766,
      "activations/layer1_attention_weight_max": 17.314023971557617,
      "activations/layer1_attention_weight_min": -14.390010833740234,
      "activations/layer20_attention_weight_max": 21.4660701751709,
      "activations/layer20_attention_weight_min": -23.74844741821289,
      "activations/layer21_attention_weight_max": 37.1771240234375,
      "activations/layer21_attention_weight_min": -30.780818939208984,
      "activations/layer22_attention_weight_max": 31.563642501831055,
      "activations/layer22_attention_weight_min": -22.926973342895508,
      "activations/layer23_attention_weight_max": 25.42951774597168,
      "activations/layer23_attention_weight_min": -17.133026123046875,
      "activations/layer2_attention_weight_max": 30.2218017578125,
      "activations/layer2_attention_weight_min": -28.52321434020996,
      "activations/layer3_attention_weight_max": 81.98208618164062,
      "activations/layer3_attention_weight_min": -85.91824340820312,
      "activations/layer4_attention_weight_max": 95.68977355957031,
      "activations/layer4_attention_weight_min": -95.52381896972656,
      "activations/layer5_attention_weight_max": 103.5339584350586,
      "activations/layer5_attention_weight_min": -94.67425537109375,
      "activations/layer6_attention_weight_max": 59.775150299072266,
      "activations/layer6_attention_weight_min": -51.722434997558594,
      "activations/layer7_attention_weight_max": 64.92192077636719,
      "activations/layer7_attention_weight_min": -59.14863204956055,
      "activations/layer8_attention_weight_max": 39.581024169921875,
      "activations/layer8_attention_weight_min": -37.50658416748047,
      "activations/layer9_attention_weight_max": 49.494510650634766,
      "activations/layer9_attention_weight_min": -43.432247161865234,
      "epoch": 4.22,
      "learning_rate": 0.00012406363636363634,
      "loss": 2.8816,
      "step": 72550
    },
    {
      "activations/layer0_attention_weight_max": 14.507000923156738,
      "activations/layer0_attention_weight_min": -15.601304054260254,
      "activations/layer10_attention_weight_max": 35.693946838378906,
      "activations/layer10_attention_weight_min": -32.66602325439453,
      "activations/layer11_attention_weight_max": 28.448562622070312,
      "activations/layer11_attention_weight_min": -28.0985164642334,
      "activations/layer12_attention_weight_max": 25.339752197265625,
      "activations/layer12_attention_weight_min": -30.734472274780273,
      "activations/layer13_attention_weight_max": 46.01089096069336,
      "activations/layer13_attention_weight_min": -35.67020797729492,
      "activations/layer14_attention_weight_max": 44.86397171020508,
      "activations/layer14_attention_weight_min": -33.52191162109375,
      "activations/layer15_attention_weight_max": 42.839508056640625,
      "activations/layer15_attention_weight_min": -30.682188034057617,
      "activations/layer16_attention_weight_max": 35.55200958251953,
      "activations/layer16_attention_weight_min": -26.5103816986084,
      "activations/layer17_attention_weight_max": 54.640872955322266,
      "activations/layer17_attention_weight_min": -36.7367057800293,
      "activations/layer18_attention_weight_max": 47.029117584228516,
      "activations/layer18_attention_weight_min": -31.334877014160156,
      "activations/layer19_attention_weight_max": 24.914396286010742,
      "activations/layer19_attention_weight_min": -20.81636619567871,
      "activations/layer1_attention_weight_max": 16.780569076538086,
      "activations/layer1_attention_weight_min": -16.25229835510254,
      "activations/layer20_attention_weight_max": 20.73016357421875,
      "activations/layer20_attention_weight_min": -23.818750381469727,
      "activations/layer21_attention_weight_max": 38.40614318847656,
      "activations/layer21_attention_weight_min": -26.351165771484375,
      "activations/layer22_attention_weight_max": 33.089996337890625,
      "activations/layer22_attention_weight_min": -23.772186279296875,
      "activations/layer23_attention_weight_max": 25.74312400817871,
      "activations/layer23_attention_weight_min": -18.806175231933594,
      "activations/layer2_attention_weight_max": 30.025732040405273,
      "activations/layer2_attention_weight_min": -28.128055572509766,
      "activations/layer3_attention_weight_max": 77.65760040283203,
      "activations/layer3_attention_weight_min": -83.6816177368164,
      "activations/layer4_attention_weight_max": 96.28939819335938,
      "activations/layer4_attention_weight_min": -95.23280334472656,
      "activations/layer5_attention_weight_max": 104.60780334472656,
      "activations/layer5_attention_weight_min": -90.67398071289062,
      "activations/layer6_attention_weight_max": 56.35747528076172,
      "activations/layer6_attention_weight_min": -51.36317825317383,
      "activations/layer7_attention_weight_max": 66.93441009521484,
      "activations/layer7_attention_weight_min": -58.95603942871094,
      "activations/layer8_attention_weight_max": 37.816837310791016,
      "activations/layer8_attention_weight_min": -37.36793899536133,
      "activations/layer9_attention_weight_max": 51.473079681396484,
      "activations/layer9_attention_weight_min": -42.889522552490234,
      "epoch": 4.22,
      "learning_rate": 0.00012404469696969696,
      "loss": 2.8682,
      "step": 72600
    },
    {
      "activations/layer0_attention_weight_max": 15.49348258972168,
      "activations/layer0_attention_weight_min": -14.780559539794922,
      "activations/layer10_attention_weight_max": 53.04549026489258,
      "activations/layer10_attention_weight_min": -44.45624542236328,
      "activations/layer11_attention_weight_max": 40.949378967285156,
      "activations/layer11_attention_weight_min": -31.971715927124023,
      "activations/layer12_attention_weight_max": 22.652997970581055,
      "activations/layer12_attention_weight_min": -27.674121856689453,
      "activations/layer13_attention_weight_max": 96.5155258178711,
      "activations/layer13_attention_weight_min": -45.97276306152344,
      "activations/layer14_attention_weight_max": 111.39228057861328,
      "activations/layer14_attention_weight_min": -47.170406341552734,
      "activations/layer15_attention_weight_max": 70.97940063476562,
      "activations/layer15_attention_weight_min": -31.89097785949707,
      "activations/layer16_attention_weight_max": 36.61573791503906,
      "activations/layer16_attention_weight_min": -28.540023803710938,
      "activations/layer17_attention_weight_max": 66.45957946777344,
      "activations/layer17_attention_weight_min": -39.87639617919922,
      "activations/layer18_attention_weight_max": 51.98062515258789,
      "activations/layer18_attention_weight_min": -32.29700469970703,
      "activations/layer19_attention_weight_max": 23.442508697509766,
      "activations/layer19_attention_weight_min": -20.23661231994629,
      "activations/layer1_attention_weight_max": 19.103160858154297,
      "activations/layer1_attention_weight_min": -16.094457626342773,
      "activations/layer20_attention_weight_max": 24.058996200561523,
      "activations/layer20_attention_weight_min": -24.38909339904785,
      "activations/layer21_attention_weight_max": 43.13813400268555,
      "activations/layer21_attention_weight_min": -30.744115829467773,
      "activations/layer22_attention_weight_max": 37.30910110473633,
      "activations/layer22_attention_weight_min": -26.425235748291016,
      "activations/layer23_attention_weight_max": 29.203208923339844,
      "activations/layer23_attention_weight_min": -18.499767303466797,
      "activations/layer2_attention_weight_max": 35.16632080078125,
      "activations/layer2_attention_weight_min": -31.75057601928711,
      "activations/layer3_attention_weight_max": 89.93435668945312,
      "activations/layer3_attention_weight_min": -89.04983520507812,
      "activations/layer4_attention_weight_max": 115.88004302978516,
      "activations/layer4_attention_weight_min": -106.5537338256836,
      "activations/layer5_attention_weight_max": 128.88107299804688,
      "activations/layer5_attention_weight_min": -113.44610595703125,
      "activations/layer6_attention_weight_max": 69.2994384765625,
      "activations/layer6_attention_weight_min": -64.39392852783203,
      "activations/layer7_attention_weight_max": 84.38288879394531,
      "activations/layer7_attention_weight_min": -73.82506561279297,
      "activations/layer8_attention_weight_max": 58.74099349975586,
      "activations/layer8_attention_weight_min": -54.29002380371094,
      "activations/layer9_attention_weight_max": 78.1849365234375,
      "activations/layer9_attention_weight_min": -62.245094299316406,
      "epoch": 4.22,
      "learning_rate": 0.00012402575757575757,
      "loss": 2.8733,
      "step": 72650
    },
    {
      "activations/layer0_attention_weight_max": 15.583816528320312,
      "activations/layer0_attention_weight_min": -14.960187911987305,
      "activations/layer10_attention_weight_max": 39.43886184692383,
      "activations/layer10_attention_weight_min": -33.420082092285156,
      "activations/layer11_attention_weight_max": 30.648128509521484,
      "activations/layer11_attention_weight_min": -26.831764221191406,
      "activations/layer12_attention_weight_max": 24.144269943237305,
      "activations/layer12_attention_weight_min": -24.64946746826172,
      "activations/layer13_attention_weight_max": 58.908512115478516,
      "activations/layer13_attention_weight_min": -42.269248962402344,
      "activations/layer14_attention_weight_max": 49.96376419067383,
      "activations/layer14_attention_weight_min": -32.6558837890625,
      "activations/layer15_attention_weight_max": 42.66608810424805,
      "activations/layer15_attention_weight_min": -32.16031265258789,
      "activations/layer16_attention_weight_max": 45.65073013305664,
      "activations/layer16_attention_weight_min": -27.352054595947266,
      "activations/layer17_attention_weight_max": 57.40131378173828,
      "activations/layer17_attention_weight_min": -38.9033088684082,
      "activations/layer18_attention_weight_max": 49.69930648803711,
      "activations/layer18_attention_weight_min": -34.40068817138672,
      "activations/layer19_attention_weight_max": 22.47471046447754,
      "activations/layer19_attention_weight_min": -19.719215393066406,
      "activations/layer1_attention_weight_max": 18.052602767944336,
      "activations/layer1_attention_weight_min": -14.654105186462402,
      "activations/layer20_attention_weight_max": 25.3359432220459,
      "activations/layer20_attention_weight_min": -22.58930015563965,
      "activations/layer21_attention_weight_max": 39.159217834472656,
      "activations/layer21_attention_weight_min": -27.55583953857422,
      "activations/layer22_attention_weight_max": 38.92121887207031,
      "activations/layer22_attention_weight_min": -26.04351234436035,
      "activations/layer23_attention_weight_max": 29.33304214477539,
      "activations/layer23_attention_weight_min": -20.0579833984375,
      "activations/layer2_attention_weight_max": 29.767478942871094,
      "activations/layer2_attention_weight_min": -27.87598419189453,
      "activations/layer3_attention_weight_max": 78.74440002441406,
      "activations/layer3_attention_weight_min": -84.09796905517578,
      "activations/layer4_attention_weight_max": 96.1253433227539,
      "activations/layer4_attention_weight_min": -94.28434753417969,
      "activations/layer5_attention_weight_max": 108.22958374023438,
      "activations/layer5_attention_weight_min": -95.8264389038086,
      "activations/layer6_attention_weight_max": 57.68561553955078,
      "activations/layer6_attention_weight_min": -51.563724517822266,
      "activations/layer7_attention_weight_max": 65.3557357788086,
      "activations/layer7_attention_weight_min": -61.054420471191406,
      "activations/layer8_attention_weight_max": 40.7224006652832,
      "activations/layer8_attention_weight_min": -39.82990646362305,
      "activations/layer9_attention_weight_max": 48.90885925292969,
      "activations/layer9_attention_weight_min": -42.8420295715332,
      "epoch": 4.22,
      "learning_rate": 0.00012400681818181819,
      "loss": 2.8685,
      "step": 72700
    },
    {
      "activations/layer0_attention_weight_max": 14.45705795288086,
      "activations/layer0_attention_weight_min": -14.6650972366333,
      "activations/layer10_attention_weight_max": 38.05455780029297,
      "activations/layer10_attention_weight_min": -33.88486099243164,
      "activations/layer11_attention_weight_max": 29.313222885131836,
      "activations/layer11_attention_weight_min": -25.816450119018555,
      "activations/layer12_attention_weight_max": 37.020294189453125,
      "activations/layer12_attention_weight_min": -24.01233673095703,
      "activations/layer13_attention_weight_max": 50.59355545043945,
      "activations/layer13_attention_weight_min": -36.82337951660156,
      "activations/layer14_attention_weight_max": 43.62828063964844,
      "activations/layer14_attention_weight_min": -29.566152572631836,
      "activations/layer15_attention_weight_max": 46.71895980834961,
      "activations/layer15_attention_weight_min": -29.978084564208984,
      "activations/layer16_attention_weight_max": 36.02163314819336,
      "activations/layer16_attention_weight_min": -27.61721420288086,
      "activations/layer17_attention_weight_max": 66.92664337158203,
      "activations/layer17_attention_weight_min": -38.951416015625,
      "activations/layer18_attention_weight_max": 50.06512451171875,
      "activations/layer18_attention_weight_min": -34.224029541015625,
      "activations/layer19_attention_weight_max": 24.783119201660156,
      "activations/layer19_attention_weight_min": -22.15819549560547,
      "activations/layer1_attention_weight_max": 18.451448440551758,
      "activations/layer1_attention_weight_min": -15.397675514221191,
      "activations/layer20_attention_weight_max": 21.507780075073242,
      "activations/layer20_attention_weight_min": -19.428192138671875,
      "activations/layer21_attention_weight_max": 42.72969436645508,
      "activations/layer21_attention_weight_min": -29.37765884399414,
      "activations/layer22_attention_weight_max": 33.70961380004883,
      "activations/layer22_attention_weight_min": -23.156837463378906,
      "activations/layer23_attention_weight_max": 26.367019653320312,
      "activations/layer23_attention_weight_min": -16.92999267578125,
      "activations/layer2_attention_weight_max": 28.758956909179688,
      "activations/layer2_attention_weight_min": -27.44000244140625,
      "activations/layer3_attention_weight_max": 78.201171875,
      "activations/layer3_attention_weight_min": -83.83094024658203,
      "activations/layer4_attention_weight_max": 100.67643737792969,
      "activations/layer4_attention_weight_min": -94.03976440429688,
      "activations/layer5_attention_weight_max": 109.89427947998047,
      "activations/layer5_attention_weight_min": -95.39570617675781,
      "activations/layer6_attention_weight_max": 62.57032012939453,
      "activations/layer6_attention_weight_min": -52.82526779174805,
      "activations/layer7_attention_weight_max": 63.65821838378906,
      "activations/layer7_attention_weight_min": -60.23500061035156,
      "activations/layer8_attention_weight_max": 40.173072814941406,
      "activations/layer8_attention_weight_min": -40.93686294555664,
      "activations/layer9_attention_weight_max": 48.13468933105469,
      "activations/layer9_attention_weight_min": -42.75858688354492,
      "epoch": 4.23,
      "learning_rate": 0.00012398787878787877,
      "loss": 2.8725,
      "step": 72750
    },
    {
      "activations/layer0_attention_weight_max": 14.785723686218262,
      "activations/layer0_attention_weight_min": -15.281647682189941,
      "activations/layer10_attention_weight_max": 52.588523864746094,
      "activations/layer10_attention_weight_min": -43.28606414794922,
      "activations/layer11_attention_weight_max": 39.7222900390625,
      "activations/layer11_attention_weight_min": -29.54981803894043,
      "activations/layer12_attention_weight_max": 22.337146759033203,
      "activations/layer12_attention_weight_min": -34.64224624633789,
      "activations/layer13_attention_weight_max": 63.37971496582031,
      "activations/layer13_attention_weight_min": -38.22098922729492,
      "activations/layer14_attention_weight_max": 58.67789077758789,
      "activations/layer14_attention_weight_min": -34.32967758178711,
      "activations/layer15_attention_weight_max": 50.49748229980469,
      "activations/layer15_attention_weight_min": -30.012292861938477,
      "activations/layer16_attention_weight_max": 35.23982620239258,
      "activations/layer16_attention_weight_min": -25.475893020629883,
      "activations/layer17_attention_weight_max": 59.52427291870117,
      "activations/layer17_attention_weight_min": -36.33283996582031,
      "activations/layer18_attention_weight_max": 53.60017013549805,
      "activations/layer18_attention_weight_min": -33.42794418334961,
      "activations/layer19_attention_weight_max": 29.720060348510742,
      "activations/layer19_attention_weight_min": -19.950336456298828,
      "activations/layer1_attention_weight_max": 18.891178131103516,
      "activations/layer1_attention_weight_min": -15.113865852355957,
      "activations/layer20_attention_weight_max": 22.795135498046875,
      "activations/layer20_attention_weight_min": -19.149906158447266,
      "activations/layer21_attention_weight_max": 40.34797668457031,
      "activations/layer21_attention_weight_min": -28.31008529663086,
      "activations/layer22_attention_weight_max": 36.35483169555664,
      "activations/layer22_attention_weight_min": -22.45608901977539,
      "activations/layer23_attention_weight_max": 29.909832000732422,
      "activations/layer23_attention_weight_min": -18.36358070373535,
      "activations/layer2_attention_weight_max": 32.04020309448242,
      "activations/layer2_attention_weight_min": -28.221891403198242,
      "activations/layer3_attention_weight_max": 85.869384765625,
      "activations/layer3_attention_weight_min": -88.0837173461914,
      "activations/layer4_attention_weight_max": 111.74169158935547,
      "activations/layer4_attention_weight_min": -100.32759094238281,
      "activations/layer5_attention_weight_max": 123.96817016601562,
      "activations/layer5_attention_weight_min": -104.1504135131836,
      "activations/layer6_attention_weight_max": 67.4711685180664,
      "activations/layer6_attention_weight_min": -58.84516906738281,
      "activations/layer7_attention_weight_max": 77.85564422607422,
      "activations/layer7_attention_weight_min": -71.41387939453125,
      "activations/layer8_attention_weight_max": 54.477821350097656,
      "activations/layer8_attention_weight_min": -50.4923210144043,
      "activations/layer9_attention_weight_max": 66.71485137939453,
      "activations/layer9_attention_weight_min": -58.154945373535156,
      "epoch": 4.23,
      "learning_rate": 0.0001239689393939394,
      "loss": 2.8706,
      "step": 72800
    },
    {
      "activations/layer0_attention_weight_max": 13.942459106445312,
      "activations/layer0_attention_weight_min": -14.701481819152832,
      "activations/layer10_attention_weight_max": 41.48765182495117,
      "activations/layer10_attention_weight_min": -35.49165725708008,
      "activations/layer11_attention_weight_max": 34.29639434814453,
      "activations/layer11_attention_weight_min": -27.502042770385742,
      "activations/layer12_attention_weight_max": 24.29701042175293,
      "activations/layer12_attention_weight_min": -23.06000328063965,
      "activations/layer13_attention_weight_max": 53.17531967163086,
      "activations/layer13_attention_weight_min": -36.28843688964844,
      "activations/layer14_attention_weight_max": 47.65712356567383,
      "activations/layer14_attention_weight_min": -29.038007736206055,
      "activations/layer15_attention_weight_max": 48.11821746826172,
      "activations/layer15_attention_weight_min": -31.482376098632812,
      "activations/layer16_attention_weight_max": 39.52096176147461,
      "activations/layer16_attention_weight_min": -28.10776710510254,
      "activations/layer17_attention_weight_max": 60.93422317504883,
      "activations/layer17_attention_weight_min": -39.03810501098633,
      "activations/layer18_attention_weight_max": 51.08907699584961,
      "activations/layer18_attention_weight_min": -31.729671478271484,
      "activations/layer19_attention_weight_max": 27.102018356323242,
      "activations/layer19_attention_weight_min": -18.836265563964844,
      "activations/layer1_attention_weight_max": 18.204755783081055,
      "activations/layer1_attention_weight_min": -15.580012321472168,
      "activations/layer20_attention_weight_max": 22.80847930908203,
      "activations/layer20_attention_weight_min": -22.944103240966797,
      "activations/layer21_attention_weight_max": 41.86221694946289,
      "activations/layer21_attention_weight_min": -24.283390045166016,
      "activations/layer22_attention_weight_max": 32.560646057128906,
      "activations/layer22_attention_weight_min": -22.33049774169922,
      "activations/layer23_attention_weight_max": 29.56891632080078,
      "activations/layer23_attention_weight_min": -17.673866271972656,
      "activations/layer2_attention_weight_max": 30.882892608642578,
      "activations/layer2_attention_weight_min": -28.61037254333496,
      "activations/layer3_attention_weight_max": 77.62043762207031,
      "activations/layer3_attention_weight_min": -83.92665100097656,
      "activations/layer4_attention_weight_max": 99.33865356445312,
      "activations/layer4_attention_weight_min": -91.88117980957031,
      "activations/layer5_attention_weight_max": 116.39967346191406,
      "activations/layer5_attention_weight_min": -93.88502502441406,
      "activations/layer6_attention_weight_max": 61.34885787963867,
      "activations/layer6_attention_weight_min": -51.833431243896484,
      "activations/layer7_attention_weight_max": 64.67132568359375,
      "activations/layer7_attention_weight_min": -61.9133415222168,
      "activations/layer8_attention_weight_max": 43.6807861328125,
      "activations/layer8_attention_weight_min": -41.144203186035156,
      "activations/layer9_attention_weight_max": 52.21732711791992,
      "activations/layer9_attention_weight_min": -49.04719161987305,
      "epoch": 4.23,
      "learning_rate": 0.00012395,
      "loss": 2.8826,
      "step": 72850
    },
    {
      "activations/layer0_attention_weight_max": 14.870050430297852,
      "activations/layer0_attention_weight_min": -14.736634254455566,
      "activations/layer10_attention_weight_max": 34.580047607421875,
      "activations/layer10_attention_weight_min": -31.368892669677734,
      "activations/layer11_attention_weight_max": 29.106891632080078,
      "activations/layer11_attention_weight_min": -25.653690338134766,
      "activations/layer12_attention_weight_max": 22.511381149291992,
      "activations/layer12_attention_weight_min": -24.94506072998047,
      "activations/layer13_attention_weight_max": 51.067222595214844,
      "activations/layer13_attention_weight_min": -37.071250915527344,
      "activations/layer14_attention_weight_max": 40.075103759765625,
      "activations/layer14_attention_weight_min": -29.258514404296875,
      "activations/layer15_attention_weight_max": 40.581668853759766,
      "activations/layer15_attention_weight_min": -29.93747329711914,
      "activations/layer16_attention_weight_max": 35.38179397583008,
      "activations/layer16_attention_weight_min": -26.13690757751465,
      "activations/layer17_attention_weight_max": 60.312744140625,
      "activations/layer17_attention_weight_min": -36.90625,
      "activations/layer18_attention_weight_max": 46.15687561035156,
      "activations/layer18_attention_weight_min": -34.127532958984375,
      "activations/layer19_attention_weight_max": 20.917200088500977,
      "activations/layer19_attention_weight_min": -18.768749237060547,
      "activations/layer1_attention_weight_max": 17.34918212890625,
      "activations/layer1_attention_weight_min": -15.324872970581055,
      "activations/layer20_attention_weight_max": 24.488819122314453,
      "activations/layer20_attention_weight_min": -21.081928253173828,
      "activations/layer21_attention_weight_max": 36.505943298339844,
      "activations/layer21_attention_weight_min": -25.575681686401367,
      "activations/layer22_attention_weight_max": 32.81813049316406,
      "activations/layer22_attention_weight_min": -24.704973220825195,
      "activations/layer23_attention_weight_max": 24.90005111694336,
      "activations/layer23_attention_weight_min": -17.02802276611328,
      "activations/layer2_attention_weight_max": 29.361522674560547,
      "activations/layer2_attention_weight_min": -28.537946701049805,
      "activations/layer3_attention_weight_max": 81.00902557373047,
      "activations/layer3_attention_weight_min": -87.54263305664062,
      "activations/layer4_attention_weight_max": 99.13957977294922,
      "activations/layer4_attention_weight_min": -93.39644622802734,
      "activations/layer5_attention_weight_max": 106.13998413085938,
      "activations/layer5_attention_weight_min": -93.45744323730469,
      "activations/layer6_attention_weight_max": 55.93048095703125,
      "activations/layer6_attention_weight_min": -51.91082000732422,
      "activations/layer7_attention_weight_max": 61.801361083984375,
      "activations/layer7_attention_weight_min": -58.754051208496094,
      "activations/layer8_attention_weight_max": 40.3961296081543,
      "activations/layer8_attention_weight_min": -37.84162139892578,
      "activations/layer9_attention_weight_max": 50.703712463378906,
      "activations/layer9_attention_weight_min": -44.75802230834961,
      "epoch": 4.24,
      "learning_rate": 0.0001239310606060606,
      "loss": 2.864,
      "step": 72900
    },
    {
      "activations/layer0_attention_weight_max": 15.73292064666748,
      "activations/layer0_attention_weight_min": -14.744288444519043,
      "activations/layer10_attention_weight_max": 41.22291946411133,
      "activations/layer10_attention_weight_min": -32.37501525878906,
      "activations/layer11_attention_weight_max": 29.429943084716797,
      "activations/layer11_attention_weight_min": -26.29149627685547,
      "activations/layer12_attention_weight_max": 20.190237045288086,
      "activations/layer12_attention_weight_min": -26.709348678588867,
      "activations/layer13_attention_weight_max": 60.18375015258789,
      "activations/layer13_attention_weight_min": -40.4317512512207,
      "activations/layer14_attention_weight_max": 42.256317138671875,
      "activations/layer14_attention_weight_min": -29.803680419921875,
      "activations/layer15_attention_weight_max": 39.72735595703125,
      "activations/layer15_attention_weight_min": -29.669496536254883,
      "activations/layer16_attention_weight_max": 38.087486267089844,
      "activations/layer16_attention_weight_min": -27.115474700927734,
      "activations/layer17_attention_weight_max": 56.6319580078125,
      "activations/layer17_attention_weight_min": -36.72733688354492,
      "activations/layer18_attention_weight_max": 51.06021499633789,
      "activations/layer18_attention_weight_min": -33.103553771972656,
      "activations/layer19_attention_weight_max": 22.72810173034668,
      "activations/layer19_attention_weight_min": -18.58283805847168,
      "activations/layer1_attention_weight_max": 17.76165771484375,
      "activations/layer1_attention_weight_min": -16.95128631591797,
      "activations/layer20_attention_weight_max": 23.86400032043457,
      "activations/layer20_attention_weight_min": -22.436355590820312,
      "activations/layer21_attention_weight_max": 43.886924743652344,
      "activations/layer21_attention_weight_min": -28.415374755859375,
      "activations/layer22_attention_weight_max": 38.048316955566406,
      "activations/layer22_attention_weight_min": -23.9090633392334,
      "activations/layer23_attention_weight_max": 24.409549713134766,
      "activations/layer23_attention_weight_min": -18.558244705200195,
      "activations/layer2_attention_weight_max": 29.947093963623047,
      "activations/layer2_attention_weight_min": -29.11782455444336,
      "activations/layer3_attention_weight_max": 80.95356750488281,
      "activations/layer3_attention_weight_min": -85.05718231201172,
      "activations/layer4_attention_weight_max": 97.32115173339844,
      "activations/layer4_attention_weight_min": -96.7244873046875,
      "activations/layer5_attention_weight_max": 108.22566986083984,
      "activations/layer5_attention_weight_min": -94.42998504638672,
      "activations/layer6_attention_weight_max": 60.46811294555664,
      "activations/layer6_attention_weight_min": -52.10734558105469,
      "activations/layer7_attention_weight_max": 62.0892219543457,
      "activations/layer7_attention_weight_min": -60.6272087097168,
      "activations/layer8_attention_weight_max": 39.290557861328125,
      "activations/layer8_attention_weight_min": -37.429290771484375,
      "activations/layer9_attention_weight_max": 45.14543533325195,
      "activations/layer9_attention_weight_min": -45.476661682128906,
      "epoch": 4.24,
      "learning_rate": 0.0001239121212121212,
      "loss": 2.8609,
      "step": 72950
    },
    {
      "activations/layer0_attention_weight_max": 15.932835578918457,
      "activations/layer0_attention_weight_min": -14.384603500366211,
      "activations/layer10_attention_weight_max": 39.26247024536133,
      "activations/layer10_attention_weight_min": -33.271095275878906,
      "activations/layer11_attention_weight_max": 33.30803298950195,
      "activations/layer11_attention_weight_min": -28.129819869995117,
      "activations/layer12_attention_weight_max": 26.880414962768555,
      "activations/layer12_attention_weight_min": -24.542985916137695,
      "activations/layer13_attention_weight_max": 66.74580383300781,
      "activations/layer13_attention_weight_min": -41.629486083984375,
      "activations/layer14_attention_weight_max": 56.28951644897461,
      "activations/layer14_attention_weight_min": -32.20668411254883,
      "activations/layer15_attention_weight_max": 47.62593460083008,
      "activations/layer15_attention_weight_min": -29.840423583984375,
      "activations/layer16_attention_weight_max": 41.545101165771484,
      "activations/layer16_attention_weight_min": -26.768081665039062,
      "activations/layer17_attention_weight_max": 65.00019836425781,
      "activations/layer17_attention_weight_min": -36.4958381652832,
      "activations/layer18_attention_weight_max": 62.21332931518555,
      "activations/layer18_attention_weight_min": -33.46288299560547,
      "activations/layer19_attention_weight_max": 23.995939254760742,
      "activations/layer19_attention_weight_min": -18.46540069580078,
      "activations/layer1_attention_weight_max": 19.43072509765625,
      "activations/layer1_attention_weight_min": -15.494053840637207,
      "activations/layer20_attention_weight_max": 25.448551177978516,
      "activations/layer20_attention_weight_min": -17.518964767456055,
      "activations/layer21_attention_weight_max": 43.602439880371094,
      "activations/layer21_attention_weight_min": -25.614227294921875,
      "activations/layer22_attention_weight_max": 36.8252067565918,
      "activations/layer22_attention_weight_min": -26.589630126953125,
      "activations/layer23_attention_weight_max": 26.887924194335938,
      "activations/layer23_attention_weight_min": -15.548681259155273,
      "activations/layer2_attention_weight_max": 30.695907592773438,
      "activations/layer2_attention_weight_min": -28.398536682128906,
      "activations/layer3_attention_weight_max": 80.15069580078125,
      "activations/layer3_attention_weight_min": -85.97508239746094,
      "activations/layer4_attention_weight_max": 101.2707748413086,
      "activations/layer4_attention_weight_min": -97.2635269165039,
      "activations/layer5_attention_weight_max": 105.77458953857422,
      "activations/layer5_attention_weight_min": -97.81633758544922,
      "activations/layer6_attention_weight_max": 57.68922424316406,
      "activations/layer6_attention_weight_min": -51.90070724487305,
      "activations/layer7_attention_weight_max": 66.19520568847656,
      "activations/layer7_attention_weight_min": -60.23724365234375,
      "activations/layer8_attention_weight_max": 41.71249771118164,
      "activations/layer8_attention_weight_min": -39.94989013671875,
      "activations/layer9_attention_weight_max": 54.85163116455078,
      "activations/layer9_attention_weight_min": -45.137569427490234,
      "epoch": 4.24,
      "learning_rate": 0.00012389318181818182,
      "loss": 2.8798,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_loss": 2.8203125,
      "eval_runtime": 8.479,
      "eval_samples_per_second": 506.428,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_openwebtext_loss": 2.8203125,
      "eval_openwebtext_ppl": 16.782094257242417,
      "eval_openwebtext_runtime": 8.479,
      "eval_openwebtext_samples_per_second": 506.428,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_wikitext_loss": 3.048828125,
      "eval_wikitext_ppl": 21.090614371389922,
      "eval_wikitext_runtime": 1.9301,
      "eval_wikitext_samples_per_second": 236.26,
      "step": 73000
    },
    {
      "epoch": 4.24,
      "eval_lambada_loss": 2.865234375,
      "eval_lambada_ppl": 17.553166745968216,
      "eval_lambada_runtime": 9.5621,
      "eval_lambada_samples_per_second": 509.197,
      "step": 73000
    },
    {
      "activations/layer0_attention_weight_max": 13.500475883483887,
      "activations/layer0_attention_weight_min": -14.693075180053711,
      "activations/layer10_attention_weight_max": 40.51966094970703,
      "activations/layer10_attention_weight_min": -33.87290573120117,
      "activations/layer11_attention_weight_max": 34.13707733154297,
      "activations/layer11_attention_weight_min": -28.51318359375,
      "activations/layer12_attention_weight_max": 32.230064392089844,
      "activations/layer12_attention_weight_min": -25.047481536865234,
      "activations/layer13_attention_weight_max": 46.27870559692383,
      "activations/layer13_attention_weight_min": -34.730743408203125,
      "activations/layer14_attention_weight_max": 43.24361038208008,
      "activations/layer14_attention_weight_min": -31.28097915649414,
      "activations/layer15_attention_weight_max": 43.99308776855469,
      "activations/layer15_attention_weight_min": -29.593299865722656,
      "activations/layer16_attention_weight_max": 36.53572463989258,
      "activations/layer16_attention_weight_min": -26.163721084594727,
      "activations/layer17_attention_weight_max": 58.93335723876953,
      "activations/layer17_attention_weight_min": -35.807579040527344,
      "activations/layer18_attention_weight_max": 47.39730453491211,
      "activations/layer18_attention_weight_min": -30.090961456298828,
      "activations/layer19_attention_weight_max": 21.110483169555664,
      "activations/layer19_attention_weight_min": -18.48062515258789,
      "activations/layer1_attention_weight_max": 18.75571060180664,
      "activations/layer1_attention_weight_min": -16.558900833129883,
      "activations/layer20_attention_weight_max": 20.73175621032715,
      "activations/layer20_attention_weight_min": -22.010099411010742,
      "activations/layer21_attention_weight_max": 37.58986282348633,
      "activations/layer21_attention_weight_min": -25.531829833984375,
      "activations/layer22_attention_weight_max": 33.77491760253906,
      "activations/layer22_attention_weight_min": -23.07842445373535,
      "activations/layer23_attention_weight_max": 23.53940200805664,
      "activations/layer23_attention_weight_min": -17.097745895385742,
      "activations/layer2_attention_weight_max": 30.529193878173828,
      "activations/layer2_attention_weight_min": -28.437673568725586,
      "activations/layer3_attention_weight_max": 80.84302520751953,
      "activations/layer3_attention_weight_min": -82.047607421875,
      "activations/layer4_attention_weight_max": 99.75753784179688,
      "activations/layer4_attention_weight_min": -93.27558898925781,
      "activations/layer5_attention_weight_max": 112.99910736083984,
      "activations/layer5_attention_weight_min": -100.34239196777344,
      "activations/layer6_attention_weight_max": 59.09531021118164,
      "activations/layer6_attention_weight_min": -52.72935485839844,
      "activations/layer7_attention_weight_max": 68.99644470214844,
      "activations/layer7_attention_weight_min": -63.183799743652344,
      "activations/layer8_attention_weight_max": 46.84279251098633,
      "activations/layer8_attention_weight_min": -41.158958435058594,
      "activations/layer9_attention_weight_max": 64.6590805053711,
      "activations/layer9_attention_weight_min": -47.50507736206055,
      "epoch": 4.24,
      "learning_rate": 0.0001238742424242424,
      "loss": 2.8478,
      "step": 73050
    },
    {
      "activations/layer0_attention_weight_max": 15.808866500854492,
      "activations/layer0_attention_weight_min": -14.65396499633789,
      "activations/layer10_attention_weight_max": 41.117576599121094,
      "activations/layer10_attention_weight_min": -34.525657653808594,
      "activations/layer11_attention_weight_max": 30.90272331237793,
      "activations/layer11_attention_weight_min": -26.506744384765625,
      "activations/layer12_attention_weight_max": 24.711328506469727,
      "activations/layer12_attention_weight_min": -27.014856338500977,
      "activations/layer13_attention_weight_max": 54.639976501464844,
      "activations/layer13_attention_weight_min": -38.25629425048828,
      "activations/layer14_attention_weight_max": 43.557281494140625,
      "activations/layer14_attention_weight_min": -29.0784969329834,
      "activations/layer15_attention_weight_max": 49.06463623046875,
      "activations/layer15_attention_weight_min": -32.32109069824219,
      "activations/layer16_attention_weight_max": 40.58238220214844,
      "activations/layer16_attention_weight_min": -27.833574295043945,
      "activations/layer17_attention_weight_max": 60.93151092529297,
      "activations/layer17_attention_weight_min": -37.630184173583984,
      "activations/layer18_attention_weight_max": 48.89191818237305,
      "activations/layer18_attention_weight_min": -32.01363754272461,
      "activations/layer19_attention_weight_max": 21.420394897460938,
      "activations/layer19_attention_weight_min": -18.651527404785156,
      "activations/layer1_attention_weight_max": 17.722761154174805,
      "activations/layer1_attention_weight_min": -17.09392547607422,
      "activations/layer20_attention_weight_max": 23.286855697631836,
      "activations/layer20_attention_weight_min": -18.410625457763672,
      "activations/layer21_attention_weight_max": 38.80384063720703,
      "activations/layer21_attention_weight_min": -26.009654998779297,
      "activations/layer22_attention_weight_max": 34.253257751464844,
      "activations/layer22_attention_weight_min": -25.09177589416504,
      "activations/layer23_attention_weight_max": 28.411582946777344,
      "activations/layer23_attention_weight_min": -17.895809173583984,
      "activations/layer2_attention_weight_max": 30.810882568359375,
      "activations/layer2_attention_weight_min": -28.690223693847656,
      "activations/layer3_attention_weight_max": 76.53675079345703,
      "activations/layer3_attention_weight_min": -81.64663696289062,
      "activations/layer4_attention_weight_max": 98.91311645507812,
      "activations/layer4_attention_weight_min": -96.74581146240234,
      "activations/layer5_attention_weight_max": 104.04719543457031,
      "activations/layer5_attention_weight_min": -97.05157470703125,
      "activations/layer6_attention_weight_max": 56.67859649658203,
      "activations/layer6_attention_weight_min": -54.53714370727539,
      "activations/layer7_attention_weight_max": 64.2307357788086,
      "activations/layer7_attention_weight_min": -62.841915130615234,
      "activations/layer8_attention_weight_max": 41.02374267578125,
      "activations/layer8_attention_weight_min": -40.843292236328125,
      "activations/layer9_attention_weight_max": 48.40958786010742,
      "activations/layer9_attention_weight_min": -46.04581832885742,
      "epoch": 4.25,
      "learning_rate": 0.00012385530303030303,
      "loss": 2.8678,
      "step": 73100
    },
    {
      "activations/layer0_attention_weight_max": 14.439862251281738,
      "activations/layer0_attention_weight_min": -14.881731986999512,
      "activations/layer10_attention_weight_max": 35.21108627319336,
      "activations/layer10_attention_weight_min": -33.4603157043457,
      "activations/layer11_attention_weight_max": 28.550064086914062,
      "activations/layer11_attention_weight_min": -26.213146209716797,
      "activations/layer12_attention_weight_max": 21.48293685913086,
      "activations/layer12_attention_weight_min": -28.109621047973633,
      "activations/layer13_attention_weight_max": 47.26041793823242,
      "activations/layer13_attention_weight_min": -36.865394592285156,
      "activations/layer14_attention_weight_max": 42.758262634277344,
      "activations/layer14_attention_weight_min": -32.61769104003906,
      "activations/layer15_attention_weight_max": 40.52471160888672,
      "activations/layer15_attention_weight_min": -31.141921997070312,
      "activations/layer16_attention_weight_max": 33.442420959472656,
      "activations/layer16_attention_weight_min": -26.800769805908203,
      "activations/layer17_attention_weight_max": 56.21485137939453,
      "activations/layer17_attention_weight_min": -35.920448303222656,
      "activations/layer18_attention_weight_max": 46.13223648071289,
      "activations/layer18_attention_weight_min": -32.15127944946289,
      "activations/layer19_attention_weight_max": 21.325666427612305,
      "activations/layer19_attention_weight_min": -19.22060203552246,
      "activations/layer1_attention_weight_max": 16.756092071533203,
      "activations/layer1_attention_weight_min": -15.777997970581055,
      "activations/layer20_attention_weight_max": 19.47447967529297,
      "activations/layer20_attention_weight_min": -20.055076599121094,
      "activations/layer21_attention_weight_max": 35.44709014892578,
      "activations/layer21_attention_weight_min": -26.139562606811523,
      "activations/layer22_attention_weight_max": 32.98908996582031,
      "activations/layer22_attention_weight_min": -25.50221824645996,
      "activations/layer23_attention_weight_max": 26.69722557067871,
      "activations/layer23_attention_weight_min": -15.873411178588867,
      "activations/layer2_attention_weight_max": 30.90753746032715,
      "activations/layer2_attention_weight_min": -28.337697982788086,
      "activations/layer3_attention_weight_max": 78.11111450195312,
      "activations/layer3_attention_weight_min": -83.11699676513672,
      "activations/layer4_attention_weight_max": 97.58309173583984,
      "activations/layer4_attention_weight_min": -93.69719696044922,
      "activations/layer5_attention_weight_max": 105.6171875,
      "activations/layer5_attention_weight_min": -93.07321166992188,
      "activations/layer6_attention_weight_max": 54.784263610839844,
      "activations/layer6_attention_weight_min": -50.6724967956543,
      "activations/layer7_attention_weight_max": 62.359901428222656,
      "activations/layer7_attention_weight_min": -58.975135803222656,
      "activations/layer8_attention_weight_max": 40.355342864990234,
      "activations/layer8_attention_weight_min": -37.019813537597656,
      "activations/layer9_attention_weight_max": 47.58041000366211,
      "activations/layer9_attention_weight_min": -43.38814926147461,
      "epoch": 4.25,
      "learning_rate": 0.00012383636363636364,
      "loss": 2.8811,
      "step": 73150
    },
    {
      "activations/layer0_attention_weight_max": 15.726704597473145,
      "activations/layer0_attention_weight_min": -13.945029258728027,
      "activations/layer10_attention_weight_max": 36.82245635986328,
      "activations/layer10_attention_weight_min": -31.69290542602539,
      "activations/layer11_attention_weight_max": 28.087270736694336,
      "activations/layer11_attention_weight_min": -27.301578521728516,
      "activations/layer12_attention_weight_max": 20.508031845092773,
      "activations/layer12_attention_weight_min": -25.280317306518555,
      "activations/layer13_attention_weight_max": 59.97075271606445,
      "activations/layer13_attention_weight_min": -45.00455093383789,
      "activations/layer14_attention_weight_max": 32.393367767333984,
      "activations/layer14_attention_weight_min": -28.31825828552246,
      "activations/layer15_attention_weight_max": 39.52735900878906,
      "activations/layer15_attention_weight_min": -29.76792335510254,
      "activations/layer16_attention_weight_max": 28.52252960205078,
      "activations/layer16_attention_weight_min": -26.44011688232422,
      "activations/layer17_attention_weight_max": 54.47473907470703,
      "activations/layer17_attention_weight_min": -37.23420715332031,
      "activations/layer18_attention_weight_max": 45.2567024230957,
      "activations/layer18_attention_weight_min": -32.847206115722656,
      "activations/layer19_attention_weight_max": 20.544130325317383,
      "activations/layer19_attention_weight_min": -18.396881103515625,
      "activations/layer1_attention_weight_max": 20.20240592956543,
      "activations/layer1_attention_weight_min": -18.854202270507812,
      "activations/layer20_attention_weight_max": 18.224700927734375,
      "activations/layer20_attention_weight_min": -22.7387638092041,
      "activations/layer21_attention_weight_max": 32.81991958618164,
      "activations/layer21_attention_weight_min": -25.29840087890625,
      "activations/layer22_attention_weight_max": 31.480148315429688,
      "activations/layer22_attention_weight_min": -23.84136962890625,
      "activations/layer23_attention_weight_max": 23.363706588745117,
      "activations/layer23_attention_weight_min": -18.980186462402344,
      "activations/layer2_attention_weight_max": 30.303382873535156,
      "activations/layer2_attention_weight_min": -28.308834075927734,
      "activations/layer3_attention_weight_max": 81.07904815673828,
      "activations/layer3_attention_weight_min": -85.24264526367188,
      "activations/layer4_attention_weight_max": 101.45623779296875,
      "activations/layer4_attention_weight_min": -93.10157775878906,
      "activations/layer5_attention_weight_max": 110.64436340332031,
      "activations/layer5_attention_weight_min": -95.32069396972656,
      "activations/layer6_attention_weight_max": 57.84113693237305,
      "activations/layer6_attention_weight_min": -51.41292953491211,
      "activations/layer7_attention_weight_max": 67.87789154052734,
      "activations/layer7_attention_weight_min": -61.32923126220703,
      "activations/layer8_attention_weight_max": 39.696617126464844,
      "activations/layer8_attention_weight_min": -39.59235763549805,
      "activations/layer9_attention_weight_max": 45.886627197265625,
      "activations/layer9_attention_weight_min": -42.35846710205078,
      "epoch": 4.25,
      "learning_rate": 0.00012381742424242423,
      "loss": 2.8625,
      "step": 73200
    },
    {
      "activations/layer0_attention_weight_max": 15.352007865905762,
      "activations/layer0_attention_weight_min": -14.043145179748535,
      "activations/layer10_attention_weight_max": 48.70096969604492,
      "activations/layer10_attention_weight_min": -36.62984848022461,
      "activations/layer11_attention_weight_max": 39.00931167602539,
      "activations/layer11_attention_weight_min": -26.756391525268555,
      "activations/layer12_attention_weight_max": 28.798124313354492,
      "activations/layer12_attention_weight_min": -26.39777946472168,
      "activations/layer13_attention_weight_max": 89.5556411743164,
      "activations/layer13_attention_weight_min": -48.56098937988281,
      "activations/layer14_attention_weight_max": 76.79718017578125,
      "activations/layer14_attention_weight_min": -39.274349212646484,
      "activations/layer15_attention_weight_max": 63.60234832763672,
      "activations/layer15_attention_weight_min": -32.715850830078125,
      "activations/layer16_attention_weight_max": 46.99877166748047,
      "activations/layer16_attention_weight_min": -26.2112979888916,
      "activations/layer17_attention_weight_max": 74.154541015625,
      "activations/layer17_attention_weight_min": -39.135650634765625,
      "activations/layer18_attention_weight_max": 63.56987762451172,
      "activations/layer18_attention_weight_min": -35.827728271484375,
      "activations/layer19_attention_weight_max": 27.389551162719727,
      "activations/layer19_attention_weight_min": -18.616924285888672,
      "activations/layer1_attention_weight_max": 18.023820877075195,
      "activations/layer1_attention_weight_min": -14.189019203186035,
      "activations/layer20_attention_weight_max": 29.159639358520508,
      "activations/layer20_attention_weight_min": -20.898042678833008,
      "activations/layer21_attention_weight_max": 53.921958923339844,
      "activations/layer21_attention_weight_min": -27.770952224731445,
      "activations/layer22_attention_weight_max": 45.72173309326172,
      "activations/layer22_attention_weight_min": -25.22722625732422,
      "activations/layer23_attention_weight_max": 35.14318084716797,
      "activations/layer23_attention_weight_min": -17.758779525756836,
      "activations/layer2_attention_weight_max": 30.84081268310547,
      "activations/layer2_attention_weight_min": -30.10567283630371,
      "activations/layer3_attention_weight_max": 82.05658721923828,
      "activations/layer3_attention_weight_min": -87.05892944335938,
      "activations/layer4_attention_weight_max": 101.85063934326172,
      "activations/layer4_attention_weight_min": -99.10548400878906,
      "activations/layer5_attention_weight_max": 114.20272827148438,
      "activations/layer5_attention_weight_min": -101.95661926269531,
      "activations/layer6_attention_weight_max": 65.84696197509766,
      "activations/layer6_attention_weight_min": -58.501068115234375,
      "activations/layer7_attention_weight_max": 68.48688507080078,
      "activations/layer7_attention_weight_min": -69.71588897705078,
      "activations/layer8_attention_weight_max": 43.409515380859375,
      "activations/layer8_attention_weight_min": -43.7588996887207,
      "activations/layer9_attention_weight_max": 55.19095993041992,
      "activations/layer9_attention_weight_min": -52.2907829284668,
      "epoch": 4.26,
      "learning_rate": 0.00012379848484848484,
      "loss": 2.8641,
      "step": 73250
    },
    {
      "activations/layer0_attention_weight_max": 14.003081321716309,
      "activations/layer0_attention_weight_min": -14.165960311889648,
      "activations/layer10_attention_weight_max": 39.58259963989258,
      "activations/layer10_attention_weight_min": -37.211334228515625,
      "activations/layer11_attention_weight_max": 32.77207946777344,
      "activations/layer11_attention_weight_min": -30.680912017822266,
      "activations/layer12_attention_weight_max": 23.733016967773438,
      "activations/layer12_attention_weight_min": -22.753509521484375,
      "activations/layer13_attention_weight_max": 59.798179626464844,
      "activations/layer13_attention_weight_min": -37.70425033569336,
      "activations/layer14_attention_weight_max": 55.44019317626953,
      "activations/layer14_attention_weight_min": -34.586116790771484,
      "activations/layer15_attention_weight_max": 47.49740219116211,
      "activations/layer15_attention_weight_min": -31.308757781982422,
      "activations/layer16_attention_weight_max": 38.75074005126953,
      "activations/layer16_attention_weight_min": -26.491043090820312,
      "activations/layer17_attention_weight_max": 60.998046875,
      "activations/layer17_attention_weight_min": -36.7502555847168,
      "activations/layer18_attention_weight_max": 57.718055725097656,
      "activations/layer18_attention_weight_min": -32.83469772338867,
      "activations/layer19_attention_weight_max": 23.277578353881836,
      "activations/layer19_attention_weight_min": -18.645475387573242,
      "activations/layer1_attention_weight_max": 17.113208770751953,
      "activations/layer1_attention_weight_min": -12.577071189880371,
      "activations/layer20_attention_weight_max": 27.354137420654297,
      "activations/layer20_attention_weight_min": -18.304506301879883,
      "activations/layer21_attention_weight_max": 45.69339370727539,
      "activations/layer21_attention_weight_min": -25.29595375061035,
      "activations/layer22_attention_weight_max": 41.349876403808594,
      "activations/layer22_attention_weight_min": -22.9831600189209,
      "activations/layer23_attention_weight_max": 28.812870025634766,
      "activations/layer23_attention_weight_min": -14.577072143554688,
      "activations/layer2_attention_weight_max": 28.340253829956055,
      "activations/layer2_attention_weight_min": -27.347251892089844,
      "activations/layer3_attention_weight_max": 80.52545928955078,
      "activations/layer3_attention_weight_min": -82.18963623046875,
      "activations/layer4_attention_weight_max": 98.84840393066406,
      "activations/layer4_attention_weight_min": -94.74763488769531,
      "activations/layer5_attention_weight_max": 104.28985595703125,
      "activations/layer5_attention_weight_min": -95.19845581054688,
      "activations/layer6_attention_weight_max": 55.038978576660156,
      "activations/layer6_attention_weight_min": -53.14823913574219,
      "activations/layer7_attention_weight_max": 63.69292449951172,
      "activations/layer7_attention_weight_min": -60.32595443725586,
      "activations/layer8_attention_weight_max": 38.48282241821289,
      "activations/layer8_attention_weight_min": -39.81544876098633,
      "activations/layer9_attention_weight_max": 48.214664459228516,
      "activations/layer9_attention_weight_min": -48.96171569824219,
      "epoch": 4.26,
      "learning_rate": 0.00012377954545454543,
      "loss": 2.8793,
      "step": 73300
    },
    {
      "activations/layer0_attention_weight_max": 14.503555297851562,
      "activations/layer0_attention_weight_min": -14.196304321289062,
      "activations/layer10_attention_weight_max": 39.443458557128906,
      "activations/layer10_attention_weight_min": -32.783016204833984,
      "activations/layer11_attention_weight_max": 30.13695526123047,
      "activations/layer11_attention_weight_min": -25.677457809448242,
      "activations/layer12_attention_weight_max": 31.30067253112793,
      "activations/layer12_attention_weight_min": -25.57709503173828,
      "activations/layer13_attention_weight_max": 53.189334869384766,
      "activations/layer13_attention_weight_min": -42.12445068359375,
      "activations/layer14_attention_weight_max": 50.47737503051758,
      "activations/layer14_attention_weight_min": -39.030086517333984,
      "activations/layer15_attention_weight_max": 41.675167083740234,
      "activations/layer15_attention_weight_min": -32.31614303588867,
      "activations/layer16_attention_weight_max": 36.778099060058594,
      "activations/layer16_attention_weight_min": -26.69709014892578,
      "activations/layer17_attention_weight_max": 59.09479522705078,
      "activations/layer17_attention_weight_min": -36.299678802490234,
      "activations/layer18_attention_weight_max": 48.70112991333008,
      "activations/layer18_attention_weight_min": -31.852378845214844,
      "activations/layer19_attention_weight_max": 20.98520278930664,
      "activations/layer19_attention_weight_min": -19.94084358215332,
      "activations/layer1_attention_weight_max": 18.62843132019043,
      "activations/layer1_attention_weight_min": -13.87720012664795,
      "activations/layer20_attention_weight_max": 21.113615036010742,
      "activations/layer20_attention_weight_min": -19.705839157104492,
      "activations/layer21_attention_weight_max": 39.563663482666016,
      "activations/layer21_attention_weight_min": -28.435935974121094,
      "activations/layer22_attention_weight_max": 34.90513229370117,
      "activations/layer22_attention_weight_min": -22.654613494873047,
      "activations/layer23_attention_weight_max": 25.75079345703125,
      "activations/layer23_attention_weight_min": -16.95391273498535,
      "activations/layer2_attention_weight_max": 30.06643295288086,
      "activations/layer2_attention_weight_min": -27.87520980834961,
      "activations/layer3_attention_weight_max": 78.49478912353516,
      "activations/layer3_attention_weight_min": -84.20072937011719,
      "activations/layer4_attention_weight_max": 99.83501434326172,
      "activations/layer4_attention_weight_min": -97.2269515991211,
      "activations/layer5_attention_weight_max": 106.18498229980469,
      "activations/layer5_attention_weight_min": -96.37344360351562,
      "activations/layer6_attention_weight_max": 59.96958541870117,
      "activations/layer6_attention_weight_min": -53.791419982910156,
      "activations/layer7_attention_weight_max": 63.14734649658203,
      "activations/layer7_attention_weight_min": -60.3148307800293,
      "activations/layer8_attention_weight_max": 39.88496017456055,
      "activations/layer8_attention_weight_min": -38.92353439331055,
      "activations/layer9_attention_weight_max": 43.83159637451172,
      "activations/layer9_attention_weight_min": -45.04224395751953,
      "epoch": 4.26,
      "learning_rate": 0.00012376060606060605,
      "loss": 2.8702,
      "step": 73350
    },
    {
      "activations/layer0_attention_weight_max": 14.409510612487793,
      "activations/layer0_attention_weight_min": -14.530752182006836,
      "activations/layer10_attention_weight_max": 41.81394958496094,
      "activations/layer10_attention_weight_min": -34.97730255126953,
      "activations/layer11_attention_weight_max": 31.893308639526367,
      "activations/layer11_attention_weight_min": -27.440147399902344,
      "activations/layer12_attention_weight_max": 22.56341552734375,
      "activations/layer12_attention_weight_min": -26.223434448242188,
      "activations/layer13_attention_weight_max": 52.92096710205078,
      "activations/layer13_attention_weight_min": -40.7563362121582,
      "activations/layer14_attention_weight_max": 55.558631896972656,
      "activations/layer14_attention_weight_min": -31.261138916015625,
      "activations/layer15_attention_weight_max": 45.89628982543945,
      "activations/layer15_attention_weight_min": -29.372356414794922,
      "activations/layer16_attention_weight_max": 39.58617401123047,
      "activations/layer16_attention_weight_min": -25.639902114868164,
      "activations/layer17_attention_weight_max": 54.71870803833008,
      "activations/layer17_attention_weight_min": -36.228233337402344,
      "activations/layer18_attention_weight_max": 50.10692596435547,
      "activations/layer18_attention_weight_min": -31.19420051574707,
      "activations/layer19_attention_weight_max": 20.69415855407715,
      "activations/layer19_attention_weight_min": -17.71281623840332,
      "activations/layer1_attention_weight_max": 17.94890785217285,
      "activations/layer1_attention_weight_min": -14.150489807128906,
      "activations/layer20_attention_weight_max": 19.946422576904297,
      "activations/layer20_attention_weight_min": -18.993389129638672,
      "activations/layer21_attention_weight_max": 37.9020881652832,
      "activations/layer21_attention_weight_min": -27.052738189697266,
      "activations/layer22_attention_weight_max": 39.22970199584961,
      "activations/layer22_attention_weight_min": -22.376434326171875,
      "activations/layer23_attention_weight_max": 25.040756225585938,
      "activations/layer23_attention_weight_min": -15.67910385131836,
      "activations/layer2_attention_weight_max": 30.121862411499023,
      "activations/layer2_attention_weight_min": -28.752059936523438,
      "activations/layer3_attention_weight_max": 78.00711059570312,
      "activations/layer3_attention_weight_min": -87.20714569091797,
      "activations/layer4_attention_weight_max": 96.11907196044922,
      "activations/layer4_attention_weight_min": -97.54277801513672,
      "activations/layer5_attention_weight_max": 108.4029541015625,
      "activations/layer5_attention_weight_min": -94.26568603515625,
      "activations/layer6_attention_weight_max": 56.70653533935547,
      "activations/layer6_attention_weight_min": -52.13990783691406,
      "activations/layer7_attention_weight_max": 64.60293579101562,
      "activations/layer7_attention_weight_min": -67.61952209472656,
      "activations/layer8_attention_weight_max": 40.26034927368164,
      "activations/layer8_attention_weight_min": -41.5854377746582,
      "activations/layer9_attention_weight_max": 54.132667541503906,
      "activations/layer9_attention_weight_min": -45.908424377441406,
      "epoch": 4.26,
      "learning_rate": 0.00012374166666666666,
      "loss": 2.8686,
      "step": 73400
    },
    {
      "activations/layer0_attention_weight_max": 13.760111808776855,
      "activations/layer0_attention_weight_min": -15.056666374206543,
      "activations/layer10_attention_weight_max": 40.07035827636719,
      "activations/layer10_attention_weight_min": -34.30154800415039,
      "activations/layer11_attention_weight_max": 30.556493759155273,
      "activations/layer11_attention_weight_min": -28.61639404296875,
      "activations/layer12_attention_weight_max": 22.732526779174805,
      "activations/layer12_attention_weight_min": -24.68628692626953,
      "activations/layer13_attention_weight_max": 45.60122299194336,
      "activations/layer13_attention_weight_min": -35.85280990600586,
      "activations/layer14_attention_weight_max": 43.476600646972656,
      "activations/layer14_attention_weight_min": -29.164995193481445,
      "activations/layer15_attention_weight_max": 41.297515869140625,
      "activations/layer15_attention_weight_min": -31.089580535888672,
      "activations/layer16_attention_weight_max": 39.607421875,
      "activations/layer16_attention_weight_min": -25.80409812927246,
      "activations/layer17_attention_weight_max": 55.941165924072266,
      "activations/layer17_attention_weight_min": -38.613304138183594,
      "activations/layer18_attention_weight_max": 44.313350677490234,
      "activations/layer18_attention_weight_min": -34.34980392456055,
      "activations/layer19_attention_weight_max": 22.484092712402344,
      "activations/layer19_attention_weight_min": -18.25630760192871,
      "activations/layer1_attention_weight_max": 17.48905372619629,
      "activations/layer1_attention_weight_min": -13.478808403015137,
      "activations/layer20_attention_weight_max": 21.305614471435547,
      "activations/layer20_attention_weight_min": -20.859622955322266,
      "activations/layer21_attention_weight_max": 35.809974670410156,
      "activations/layer21_attention_weight_min": -28.55099105834961,
      "activations/layer22_attention_weight_max": 29.52869987487793,
      "activations/layer22_attention_weight_min": -21.529966354370117,
      "activations/layer23_attention_weight_max": 25.820310592651367,
      "activations/layer23_attention_weight_min": -16.800168991088867,
      "activations/layer2_attention_weight_max": 29.53297996520996,
      "activations/layer2_attention_weight_min": -27.99187469482422,
      "activations/layer3_attention_weight_max": 80.7437515258789,
      "activations/layer3_attention_weight_min": -87.50491333007812,
      "activations/layer4_attention_weight_max": 102.90340423583984,
      "activations/layer4_attention_weight_min": -97.23884582519531,
      "activations/layer5_attention_weight_max": 109.13689422607422,
      "activations/layer5_attention_weight_min": -97.31617736816406,
      "activations/layer6_attention_weight_max": 62.00257873535156,
      "activations/layer6_attention_weight_min": -53.20146179199219,
      "activations/layer7_attention_weight_max": 65.60671997070312,
      "activations/layer7_attention_weight_min": -64.07749938964844,
      "activations/layer8_attention_weight_max": 43.95660400390625,
      "activations/layer8_attention_weight_min": -41.97107696533203,
      "activations/layer9_attention_weight_max": 50.91173553466797,
      "activations/layer9_attention_weight_min": -47.51651382446289,
      "epoch": 4.27,
      "learning_rate": 0.00012372272727272725,
      "loss": 2.8821,
      "step": 73450
    },
    {
      "activations/layer0_attention_weight_max": 15.954658508300781,
      "activations/layer0_attention_weight_min": -14.74163818359375,
      "activations/layer10_attention_weight_max": 36.76637268066406,
      "activations/layer10_attention_weight_min": -32.612613677978516,
      "activations/layer11_attention_weight_max": 31.564834594726562,
      "activations/layer11_attention_weight_min": -27.673391342163086,
      "activations/layer12_attention_weight_max": 28.8061466217041,
      "activations/layer12_attention_weight_min": -23.780637741088867,
      "activations/layer13_attention_weight_max": 58.19347381591797,
      "activations/layer13_attention_weight_min": -41.9974365234375,
      "activations/layer14_attention_weight_max": 52.69059753417969,
      "activations/layer14_attention_weight_min": -34.16181945800781,
      "activations/layer15_attention_weight_max": 46.71897506713867,
      "activations/layer15_attention_weight_min": -31.030519485473633,
      "activations/layer16_attention_weight_max": 39.27523422241211,
      "activations/layer16_attention_weight_min": -25.8048095703125,
      "activations/layer17_attention_weight_max": 59.67945861816406,
      "activations/layer17_attention_weight_min": -37.601463317871094,
      "activations/layer18_attention_weight_max": 54.38368606567383,
      "activations/layer18_attention_weight_min": -33.9676628112793,
      "activations/layer19_attention_weight_max": 29.225446701049805,
      "activations/layer19_attention_weight_min": -22.043180465698242,
      "activations/layer1_attention_weight_max": 17.974729537963867,
      "activations/layer1_attention_weight_min": -15.286152839660645,
      "activations/layer20_attention_weight_max": 22.988222122192383,
      "activations/layer20_attention_weight_min": -19.82760238647461,
      "activations/layer21_attention_weight_max": 41.4490852355957,
      "activations/layer21_attention_weight_min": -25.511760711669922,
      "activations/layer22_attention_weight_max": 35.21580505371094,
      "activations/layer22_attention_weight_min": -22.558671951293945,
      "activations/layer23_attention_weight_max": 27.218639373779297,
      "activations/layer23_attention_weight_min": -17.485530853271484,
      "activations/layer2_attention_weight_max": 30.043659210205078,
      "activations/layer2_attention_weight_min": -27.82891082763672,
      "activations/layer3_attention_weight_max": 79.64739227294922,
      "activations/layer3_attention_weight_min": -79.76526641845703,
      "activations/layer4_attention_weight_max": 100.87578582763672,
      "activations/layer4_attention_weight_min": -94.29666137695312,
      "activations/layer5_attention_weight_max": 109.39238739013672,
      "activations/layer5_attention_weight_min": -92.83934020996094,
      "activations/layer6_attention_weight_max": 56.32917785644531,
      "activations/layer6_attention_weight_min": -52.01543426513672,
      "activations/layer7_attention_weight_max": 65.32732391357422,
      "activations/layer7_attention_weight_min": -60.58436965942383,
      "activations/layer8_attention_weight_max": 41.35776138305664,
      "activations/layer8_attention_weight_min": -39.43904113769531,
      "activations/layer9_attention_weight_max": 48.13688278198242,
      "activations/layer9_attention_weight_min": -45.057247161865234,
      "epoch": 4.27,
      "learning_rate": 0.00012370378787878786,
      "loss": 2.8706,
      "step": 73500
    },
    {
      "activations/layer0_attention_weight_max": 14.932609558105469,
      "activations/layer0_attention_weight_min": -15.079490661621094,
      "activations/layer10_attention_weight_max": 41.0617561340332,
      "activations/layer10_attention_weight_min": -34.607086181640625,
      "activations/layer11_attention_weight_max": 32.33812713623047,
      "activations/layer11_attention_weight_min": -26.136985778808594,
      "activations/layer12_attention_weight_max": 26.057056427001953,
      "activations/layer12_attention_weight_min": -24.91385269165039,
      "activations/layer13_attention_weight_max": 65.23389434814453,
      "activations/layer13_attention_weight_min": -42.92949676513672,
      "activations/layer14_attention_weight_max": 52.66262435913086,
      "activations/layer14_attention_weight_min": -32.63317108154297,
      "activations/layer15_attention_weight_max": 52.92748260498047,
      "activations/layer15_attention_weight_min": -31.005489349365234,
      "activations/layer16_attention_weight_max": 51.61673355102539,
      "activations/layer16_attention_weight_min": -26.466957092285156,
      "activations/layer17_attention_weight_max": 62.52252960205078,
      "activations/layer17_attention_weight_min": -36.91519546508789,
      "activations/layer18_attention_weight_max": 55.52275085449219,
      "activations/layer18_attention_weight_min": -31.0070858001709,
      "activations/layer19_attention_weight_max": 21.93846321105957,
      "activations/layer19_attention_weight_min": -17.123750686645508,
      "activations/layer1_attention_weight_max": 17.436372756958008,
      "activations/layer1_attention_weight_min": -14.349647521972656,
      "activations/layer20_attention_weight_max": 23.6911678314209,
      "activations/layer20_attention_weight_min": -17.769969940185547,
      "activations/layer21_attention_weight_max": 43.33399963378906,
      "activations/layer21_attention_weight_min": -29.77567481994629,
      "activations/layer22_attention_weight_max": 36.44646072387695,
      "activations/layer22_attention_weight_min": -21.778175354003906,
      "activations/layer23_attention_weight_max": 31.687664031982422,
      "activations/layer23_attention_weight_min": -15.708890914916992,
      "activations/layer2_attention_weight_max": 28.73055648803711,
      "activations/layer2_attention_weight_min": -28.174488067626953,
      "activations/layer3_attention_weight_max": 78.38668060302734,
      "activations/layer3_attention_weight_min": -82.4279556274414,
      "activations/layer4_attention_weight_max": 97.25096893310547,
      "activations/layer4_attention_weight_min": -95.04841613769531,
      "activations/layer5_attention_weight_max": 104.2986068725586,
      "activations/layer5_attention_weight_min": -92.00984191894531,
      "activations/layer6_attention_weight_max": 57.904624938964844,
      "activations/layer6_attention_weight_min": -50.931217193603516,
      "activations/layer7_attention_weight_max": 65.11665344238281,
      "activations/layer7_attention_weight_min": -60.417781829833984,
      "activations/layer8_attention_weight_max": 39.99992752075195,
      "activations/layer8_attention_weight_min": -38.21903991699219,
      "activations/layer9_attention_weight_max": 47.57124328613281,
      "activations/layer9_attention_weight_min": -46.096473693847656,
      "epoch": 4.27,
      "learning_rate": 0.00012368484848484848,
      "loss": 2.877,
      "step": 73550
    },
    {
      "activations/layer0_attention_weight_max": 14.964115142822266,
      "activations/layer0_attention_weight_min": -15.152336120605469,
      "activations/layer10_attention_weight_max": 38.849830627441406,
      "activations/layer10_attention_weight_min": -34.42268371582031,
      "activations/layer11_attention_weight_max": 32.017555236816406,
      "activations/layer11_attention_weight_min": -27.716644287109375,
      "activations/layer12_attention_weight_max": 22.6234130859375,
      "activations/layer12_attention_weight_min": -23.87594223022461,
      "activations/layer13_attention_weight_max": 42.74880599975586,
      "activations/layer13_attention_weight_min": -32.20945358276367,
      "activations/layer14_attention_weight_max": 45.87548065185547,
      "activations/layer14_attention_weight_min": -38.14522171020508,
      "activations/layer15_attention_weight_max": 41.19490051269531,
      "activations/layer15_attention_weight_min": -29.1317195892334,
      "activations/layer16_attention_weight_max": 43.312862396240234,
      "activations/layer16_attention_weight_min": -26.319576263427734,
      "activations/layer17_attention_weight_max": 59.70388412475586,
      "activations/layer17_attention_weight_min": -37.6755485534668,
      "activations/layer18_attention_weight_max": 49.82817840576172,
      "activations/layer18_attention_weight_min": -31.46102523803711,
      "activations/layer19_attention_weight_max": 23.77875328063965,
      "activations/layer19_attention_weight_min": -19.470922470092773,
      "activations/layer1_attention_weight_max": 18.670488357543945,
      "activations/layer1_attention_weight_min": -15.022663116455078,
      "activations/layer20_attention_weight_max": 20.255748748779297,
      "activations/layer20_attention_weight_min": -23.6648006439209,
      "activations/layer21_attention_weight_max": 40.43960952758789,
      "activations/layer21_attention_weight_min": -26.143213272094727,
      "activations/layer22_attention_weight_max": 33.27235794067383,
      "activations/layer22_attention_weight_min": -22.57427978515625,
      "activations/layer23_attention_weight_max": 30.666364669799805,
      "activations/layer23_attention_weight_min": -18.108245849609375,
      "activations/layer2_attention_weight_max": 30.827064514160156,
      "activations/layer2_attention_weight_min": -28.906761169433594,
      "activations/layer3_attention_weight_max": 76.60061645507812,
      "activations/layer3_attention_weight_min": -84.62051391601562,
      "activations/layer4_attention_weight_max": 93.76981353759766,
      "activations/layer4_attention_weight_min": -96.01509094238281,
      "activations/layer5_attention_weight_max": 101.8813705444336,
      "activations/layer5_attention_weight_min": -97.43182373046875,
      "activations/layer6_attention_weight_max": 55.75537109375,
      "activations/layer6_attention_weight_min": -52.534908294677734,
      "activations/layer7_attention_weight_max": 62.72799301147461,
      "activations/layer7_attention_weight_min": -65.36018371582031,
      "activations/layer8_attention_weight_max": 42.545379638671875,
      "activations/layer8_attention_weight_min": -40.87076950073242,
      "activations/layer9_attention_weight_max": 53.6254997253418,
      "activations/layer9_attention_weight_min": -49.45505142211914,
      "epoch": 4.28,
      "learning_rate": 0.00012366590909090907,
      "loss": 2.856,
      "step": 73600
    },
    {
      "activations/layer0_attention_weight_max": 14.343928337097168,
      "activations/layer0_attention_weight_min": -15.307312965393066,
      "activations/layer10_attention_weight_max": 37.26552200317383,
      "activations/layer10_attention_weight_min": -35.22858428955078,
      "activations/layer11_attention_weight_max": 28.61049461364746,
      "activations/layer11_attention_weight_min": -27.78133773803711,
      "activations/layer12_attention_weight_max": 22.416175842285156,
      "activations/layer12_attention_weight_min": -28.86027717590332,
      "activations/layer13_attention_weight_max": 50.38152313232422,
      "activations/layer13_attention_weight_min": -43.77473449707031,
      "activations/layer14_attention_weight_max": 39.67207336425781,
      "activations/layer14_attention_weight_min": -30.33135414123535,
      "activations/layer15_attention_weight_max": 47.408267974853516,
      "activations/layer15_attention_weight_min": -32.391361236572266,
      "activations/layer16_attention_weight_max": 35.75828170776367,
      "activations/layer16_attention_weight_min": -26.207571029663086,
      "activations/layer17_attention_weight_max": 57.008056640625,
      "activations/layer17_attention_weight_min": -38.211395263671875,
      "activations/layer18_attention_weight_max": 46.22414779663086,
      "activations/layer18_attention_weight_min": -33.64655685424805,
      "activations/layer19_attention_weight_max": 23.449251174926758,
      "activations/layer19_attention_weight_min": -16.73414421081543,
      "activations/layer1_attention_weight_max": 18.242353439331055,
      "activations/layer1_attention_weight_min": -16.689258575439453,
      "activations/layer20_attention_weight_max": 19.614543914794922,
      "activations/layer20_attention_weight_min": -18.390188217163086,
      "activations/layer21_attention_weight_max": 43.06011962890625,
      "activations/layer21_attention_weight_min": -25.694908142089844,
      "activations/layer22_attention_weight_max": 35.789363861083984,
      "activations/layer22_attention_weight_min": -22.390933990478516,
      "activations/layer23_attention_weight_max": 25.724090576171875,
      "activations/layer23_attention_weight_min": -16.370702743530273,
      "activations/layer2_attention_weight_max": 32.42982482910156,
      "activations/layer2_attention_weight_min": -30.814971923828125,
      "activations/layer3_attention_weight_max": 78.97146606445312,
      "activations/layer3_attention_weight_min": -83.76870727539062,
      "activations/layer4_attention_weight_max": 96.94834899902344,
      "activations/layer4_attention_weight_min": -96.39031982421875,
      "activations/layer5_attention_weight_max": 108.60395812988281,
      "activations/layer5_attention_weight_min": -92.9585189819336,
      "activations/layer6_attention_weight_max": 58.512367248535156,
      "activations/layer6_attention_weight_min": -52.21352767944336,
      "activations/layer7_attention_weight_max": 65.44957733154297,
      "activations/layer7_attention_weight_min": -64.37796020507812,
      "activations/layer8_attention_weight_max": 41.07387161254883,
      "activations/layer8_attention_weight_min": -39.28868103027344,
      "activations/layer9_attention_weight_max": 50.300575256347656,
      "activations/layer9_attention_weight_min": -46.659332275390625,
      "epoch": 4.28,
      "learning_rate": 0.00012364696969696968,
      "loss": 2.8563,
      "step": 73650
    },
    {
      "activations/layer0_attention_weight_max": 13.678620338439941,
      "activations/layer0_attention_weight_min": -15.109169960021973,
      "activations/layer10_attention_weight_max": 39.23036193847656,
      "activations/layer10_attention_weight_min": -31.509033203125,
      "activations/layer11_attention_weight_max": 29.80292510986328,
      "activations/layer11_attention_weight_min": -26.087215423583984,
      "activations/layer12_attention_weight_max": 22.93046760559082,
      "activations/layer12_attention_weight_min": -24.470035552978516,
      "activations/layer13_attention_weight_max": 52.9265251159668,
      "activations/layer13_attention_weight_min": -37.401817321777344,
      "activations/layer14_attention_weight_max": 47.20720291137695,
      "activations/layer14_attention_weight_min": -31.430648803710938,
      "activations/layer15_attention_weight_max": 43.91984176635742,
      "activations/layer15_attention_weight_min": -31.979398727416992,
      "activations/layer16_attention_weight_max": 42.210479736328125,
      "activations/layer16_attention_weight_min": -26.940628051757812,
      "activations/layer17_attention_weight_max": 62.51921463012695,
      "activations/layer17_attention_weight_min": -38.11861038208008,
      "activations/layer18_attention_weight_max": 55.67200469970703,
      "activations/layer18_attention_weight_min": -32.02848434448242,
      "activations/layer19_attention_weight_max": 19.74024772644043,
      "activations/layer19_attention_weight_min": -18.65399932861328,
      "activations/layer1_attention_weight_max": 17.4940128326416,
      "activations/layer1_attention_weight_min": -15.297983169555664,
      "activations/layer20_attention_weight_max": 21.139995574951172,
      "activations/layer20_attention_weight_min": -21.73576545715332,
      "activations/layer21_attention_weight_max": 37.7318115234375,
      "activations/layer21_attention_weight_min": -24.600662231445312,
      "activations/layer22_attention_weight_max": 39.869041442871094,
      "activations/layer22_attention_weight_min": -22.41530990600586,
      "activations/layer23_attention_weight_max": 27.086437225341797,
      "activations/layer23_attention_weight_min": -18.462913513183594,
      "activations/layer2_attention_weight_max": 29.542430877685547,
      "activations/layer2_attention_weight_min": -29.076026916503906,
      "activations/layer3_attention_weight_max": 77.2933120727539,
      "activations/layer3_attention_weight_min": -80.00160217285156,
      "activations/layer4_attention_weight_max": 95.56332397460938,
      "activations/layer4_attention_weight_min": -91.17583465576172,
      "activations/layer5_attention_weight_max": 103.4952392578125,
      "activations/layer5_attention_weight_min": -88.70294189453125,
      "activations/layer6_attention_weight_max": 56.48310470581055,
      "activations/layer6_attention_weight_min": -48.540523529052734,
      "activations/layer7_attention_weight_max": 59.21308898925781,
      "activations/layer7_attention_weight_min": -60.08702087402344,
      "activations/layer8_attention_weight_max": 39.25904083251953,
      "activations/layer8_attention_weight_min": -36.57445526123047,
      "activations/layer9_attention_weight_max": 47.15224838256836,
      "activations/layer9_attention_weight_min": -42.67782211303711,
      "epoch": 4.28,
      "learning_rate": 0.0001236280303030303,
      "loss": 2.8623,
      "step": 73700
    },
    {
      "activations/layer0_attention_weight_max": 15.344066619873047,
      "activations/layer0_attention_weight_min": -15.21420669555664,
      "activations/layer10_attention_weight_max": 38.899192810058594,
      "activations/layer10_attention_weight_min": -33.243595123291016,
      "activations/layer11_attention_weight_max": 30.5835018157959,
      "activations/layer11_attention_weight_min": -25.98392105102539,
      "activations/layer12_attention_weight_max": 25.596290588378906,
      "activations/layer12_attention_weight_min": -27.726057052612305,
      "activations/layer13_attention_weight_max": 47.88123321533203,
      "activations/layer13_attention_weight_min": -38.65156173706055,
      "activations/layer14_attention_weight_max": 41.549766540527344,
      "activations/layer14_attention_weight_min": -31.562807083129883,
      "activations/layer15_attention_weight_max": 47.03464126586914,
      "activations/layer15_attention_weight_min": -29.898181915283203,
      "activations/layer16_attention_weight_max": 32.6700439453125,
      "activations/layer16_attention_weight_min": -26.5134220123291,
      "activations/layer17_attention_weight_max": 55.16456985473633,
      "activations/layer17_attention_weight_min": -35.962249755859375,
      "activations/layer18_attention_weight_max": 48.58755874633789,
      "activations/layer18_attention_weight_min": -32.470890045166016,
      "activations/layer19_attention_weight_max": 20.50438117980957,
      "activations/layer19_attention_weight_min": -19.741453170776367,
      "activations/layer1_attention_weight_max": 19.32087516784668,
      "activations/layer1_attention_weight_min": -15.655653953552246,
      "activations/layer20_attention_weight_max": 22.899492263793945,
      "activations/layer20_attention_weight_min": -23.642641067504883,
      "activations/layer21_attention_weight_max": 42.31254196166992,
      "activations/layer21_attention_weight_min": -26.320884704589844,
      "activations/layer22_attention_weight_max": 35.35707473754883,
      "activations/layer22_attention_weight_min": -23.188732147216797,
      "activations/layer23_attention_weight_max": 24.755918502807617,
      "activations/layer23_attention_weight_min": -17.966283798217773,
      "activations/layer2_attention_weight_max": 32.052101135253906,
      "activations/layer2_attention_weight_min": -31.525070190429688,
      "activations/layer3_attention_weight_max": 80.46711730957031,
      "activations/layer3_attention_weight_min": -82.920166015625,
      "activations/layer4_attention_weight_max": 101.4135971069336,
      "activations/layer4_attention_weight_min": -97.30927276611328,
      "activations/layer5_attention_weight_max": 108.70896911621094,
      "activations/layer5_attention_weight_min": -97.76942443847656,
      "activations/layer6_attention_weight_max": 60.219764709472656,
      "activations/layer6_attention_weight_min": -54.133785247802734,
      "activations/layer7_attention_weight_max": 65.38406372070312,
      "activations/layer7_attention_weight_min": -64.95977783203125,
      "activations/layer8_attention_weight_max": 42.49503707885742,
      "activations/layer8_attention_weight_min": -41.15230178833008,
      "activations/layer9_attention_weight_max": 50.676902770996094,
      "activations/layer9_attention_weight_min": -48.95701217651367,
      "epoch": 4.29,
      "learning_rate": 0.00012360909090909089,
      "loss": 2.8624,
      "step": 73750
    },
    {
      "activations/layer0_attention_weight_max": 13.844551086425781,
      "activations/layer0_attention_weight_min": -15.283353805541992,
      "activations/layer10_attention_weight_max": 40.28190612792969,
      "activations/layer10_attention_weight_min": -33.016334533691406,
      "activations/layer11_attention_weight_max": 35.56937026977539,
      "activations/layer11_attention_weight_min": -26.271804809570312,
      "activations/layer12_attention_weight_max": 25.3570556640625,
      "activations/layer12_attention_weight_min": -26.3353328704834,
      "activations/layer13_attention_weight_max": 65.54827880859375,
      "activations/layer13_attention_weight_min": -39.64018249511719,
      "activations/layer14_attention_weight_max": 64.4539794921875,
      "activations/layer14_attention_weight_min": -37.643775939941406,
      "activations/layer15_attention_weight_max": 47.19284439086914,
      "activations/layer15_attention_weight_min": -30.12173080444336,
      "activations/layer16_attention_weight_max": 38.246402740478516,
      "activations/layer16_attention_weight_min": -25.966567993164062,
      "activations/layer17_attention_weight_max": 59.02796936035156,
      "activations/layer17_attention_weight_min": -38.026611328125,
      "activations/layer18_attention_weight_max": 49.7206916809082,
      "activations/layer18_attention_weight_min": -33.598594665527344,
      "activations/layer19_attention_weight_max": 23.105867385864258,
      "activations/layer19_attention_weight_min": -19.467626571655273,
      "activations/layer1_attention_weight_max": 17.22035026550293,
      "activations/layer1_attention_weight_min": -17.725324630737305,
      "activations/layer20_attention_weight_max": 21.932519912719727,
      "activations/layer20_attention_weight_min": -18.872520446777344,
      "activations/layer21_attention_weight_max": 43.27521896362305,
      "activations/layer21_attention_weight_min": -27.05856704711914,
      "activations/layer22_attention_weight_max": 36.53715133666992,
      "activations/layer22_attention_weight_min": -22.44754409790039,
      "activations/layer23_attention_weight_max": 29.788188934326172,
      "activations/layer23_attention_weight_min": -17.32477378845215,
      "activations/layer2_attention_weight_max": 29.814756393432617,
      "activations/layer2_attention_weight_min": -28.441471099853516,
      "activations/layer3_attention_weight_max": 83.0671615600586,
      "activations/layer3_attention_weight_min": -80.73274993896484,
      "activations/layer4_attention_weight_max": 103.62218475341797,
      "activations/layer4_attention_weight_min": -95.1751937866211,
      "activations/layer5_attention_weight_max": 113.93753051757812,
      "activations/layer5_attention_weight_min": -101.02593231201172,
      "activations/layer6_attention_weight_max": 60.362388610839844,
      "activations/layer6_attention_weight_min": -53.896995544433594,
      "activations/layer7_attention_weight_max": 74.0111083984375,
      "activations/layer7_attention_weight_min": -70.81056213378906,
      "activations/layer8_attention_weight_max": 41.9145622253418,
      "activations/layer8_attention_weight_min": -39.72602081298828,
      "activations/layer9_attention_weight_max": 60.2751579284668,
      "activations/layer9_attention_weight_min": -52.93669509887695,
      "epoch": 4.29,
      "learning_rate": 0.0001235901515151515,
      "loss": 2.8774,
      "step": 73800
    },
    {
      "activations/layer0_attention_weight_max": 14.760041236877441,
      "activations/layer0_attention_weight_min": -15.16022777557373,
      "activations/layer10_attention_weight_max": 41.829490661621094,
      "activations/layer10_attention_weight_min": -35.094207763671875,
      "activations/layer11_attention_weight_max": 31.84395980834961,
      "activations/layer11_attention_weight_min": -26.716163635253906,
      "activations/layer12_attention_weight_max": 24.10064125061035,
      "activations/layer12_attention_weight_min": -26.143098831176758,
      "activations/layer13_attention_weight_max": 55.24462127685547,
      "activations/layer13_attention_weight_min": -35.07956314086914,
      "activations/layer14_attention_weight_max": 46.61792755126953,
      "activations/layer14_attention_weight_min": -29.247119903564453,
      "activations/layer15_attention_weight_max": 44.48596954345703,
      "activations/layer15_attention_weight_min": -32.489742279052734,
      "activations/layer16_attention_weight_max": 38.78987503051758,
      "activations/layer16_attention_weight_min": -25.839189529418945,
      "activations/layer17_attention_weight_max": 59.3835334777832,
      "activations/layer17_attention_weight_min": -38.915523529052734,
      "activations/layer18_attention_weight_max": 52.374080657958984,
      "activations/layer18_attention_weight_min": -34.44636535644531,
      "activations/layer19_attention_weight_max": 25.38964080810547,
      "activations/layer19_attention_weight_min": -18.148550033569336,
      "activations/layer1_attention_weight_max": 17.911209106445312,
      "activations/layer1_attention_weight_min": -14.744953155517578,
      "activations/layer20_attention_weight_max": 22.567529678344727,
      "activations/layer20_attention_weight_min": -19.62551498413086,
      "activations/layer21_attention_weight_max": 43.35527038574219,
      "activations/layer21_attention_weight_min": -24.79436683654785,
      "activations/layer22_attention_weight_max": 42.48912048339844,
      "activations/layer22_attention_weight_min": -22.82245635986328,
      "activations/layer23_attention_weight_max": 30.613506317138672,
      "activations/layer23_attention_weight_min": -17.36223030090332,
      "activations/layer2_attention_weight_max": 31.366437911987305,
      "activations/layer2_attention_weight_min": -29.309646606445312,
      "activations/layer3_attention_weight_max": 83.28905487060547,
      "activations/layer3_attention_weight_min": -83.1102523803711,
      "activations/layer4_attention_weight_max": 101.35247039794922,
      "activations/layer4_attention_weight_min": -96.62303924560547,
      "activations/layer5_attention_weight_max": 109.73049926757812,
      "activations/layer5_attention_weight_min": -98.14764404296875,
      "activations/layer6_attention_weight_max": 59.1145133972168,
      "activations/layer6_attention_weight_min": -54.67082214355469,
      "activations/layer7_attention_weight_max": 66.62650299072266,
      "activations/layer7_attention_weight_min": -64.3682861328125,
      "activations/layer8_attention_weight_max": 43.34955978393555,
      "activations/layer8_attention_weight_min": -40.97721862792969,
      "activations/layer9_attention_weight_max": 55.53437423706055,
      "activations/layer9_attention_weight_min": -48.23410415649414,
      "epoch": 4.29,
      "learning_rate": 0.00012357121212121212,
      "loss": 2.8473,
      "step": 73850
    },
    {
      "activations/layer0_attention_weight_max": 13.927949905395508,
      "activations/layer0_attention_weight_min": -15.096802711486816,
      "activations/layer10_attention_weight_max": 37.94490051269531,
      "activations/layer10_attention_weight_min": -34.45607376098633,
      "activations/layer11_attention_weight_max": 27.9332332611084,
      "activations/layer11_attention_weight_min": -26.840717315673828,
      "activations/layer12_attention_weight_max": 27.81305694580078,
      "activations/layer12_attention_weight_min": -22.71665382385254,
      "activations/layer13_attention_weight_max": 49.0584831237793,
      "activations/layer13_attention_weight_min": -35.3712272644043,
      "activations/layer14_attention_weight_max": 41.415321350097656,
      "activations/layer14_attention_weight_min": -29.94642448425293,
      "activations/layer15_attention_weight_max": 42.74660873413086,
      "activations/layer15_attention_weight_min": -30.002981185913086,
      "activations/layer16_attention_weight_max": 33.18574905395508,
      "activations/layer16_attention_weight_min": -28.221046447753906,
      "activations/layer17_attention_weight_max": 51.9936408996582,
      "activations/layer17_attention_weight_min": -36.90475082397461,
      "activations/layer18_attention_weight_max": 46.67711639404297,
      "activations/layer18_attention_weight_min": -30.640573501586914,
      "activations/layer19_attention_weight_max": 22.028841018676758,
      "activations/layer19_attention_weight_min": -18.69141960144043,
      "activations/layer1_attention_weight_max": 17.781654357910156,
      "activations/layer1_attention_weight_min": -17.02935218811035,
      "activations/layer20_attention_weight_max": 19.836313247680664,
      "activations/layer20_attention_weight_min": -20.31865882873535,
      "activations/layer21_attention_weight_max": 39.51185607910156,
      "activations/layer21_attention_weight_min": -26.489879608154297,
      "activations/layer22_attention_weight_max": 31.911632537841797,
      "activations/layer22_attention_weight_min": -22.75939178466797,
      "activations/layer23_attention_weight_max": 25.039596557617188,
      "activations/layer23_attention_weight_min": -17.598724365234375,
      "activations/layer2_attention_weight_max": 30.650535583496094,
      "activations/layer2_attention_weight_min": -28.55289077758789,
      "activations/layer3_attention_weight_max": 79.89464569091797,
      "activations/layer3_attention_weight_min": -81.0152587890625,
      "activations/layer4_attention_weight_max": 96.59426879882812,
      "activations/layer4_attention_weight_min": -92.21419525146484,
      "activations/layer5_attention_weight_max": 102.02080535888672,
      "activations/layer5_attention_weight_min": -94.54375457763672,
      "activations/layer6_attention_weight_max": 56.681640625,
      "activations/layer6_attention_weight_min": -52.04759216308594,
      "activations/layer7_attention_weight_max": 63.508853912353516,
      "activations/layer7_attention_weight_min": -59.34075927734375,
      "activations/layer8_attention_weight_max": 39.572296142578125,
      "activations/layer8_attention_weight_min": -38.42949295043945,
      "activations/layer9_attention_weight_max": 49.20132064819336,
      "activations/layer9_attention_weight_min": -46.16655349731445,
      "epoch": 4.29,
      "learning_rate": 0.0001235522727272727,
      "loss": 2.8702,
      "step": 73900
    },
    {
      "activations/layer0_attention_weight_max": 15.650553703308105,
      "activations/layer0_attention_weight_min": -15.351461410522461,
      "activations/layer10_attention_weight_max": 39.70577621459961,
      "activations/layer10_attention_weight_min": -34.52495574951172,
      "activations/layer11_attention_weight_max": 30.845539093017578,
      "activations/layer11_attention_weight_min": -28.944814682006836,
      "activations/layer12_attention_weight_max": 26.522926330566406,
      "activations/layer12_attention_weight_min": -22.756330490112305,
      "activations/layer13_attention_weight_max": 54.24049758911133,
      "activations/layer13_attention_weight_min": -38.242061614990234,
      "activations/layer14_attention_weight_max": 43.92530822753906,
      "activations/layer14_attention_weight_min": -30.12172508239746,
      "activations/layer15_attention_weight_max": 40.32868194580078,
      "activations/layer15_attention_weight_min": -29.6436767578125,
      "activations/layer16_attention_weight_max": 36.890201568603516,
      "activations/layer16_attention_weight_min": -27.41444969177246,
      "activations/layer17_attention_weight_max": 58.81395721435547,
      "activations/layer17_attention_weight_min": -37.4138298034668,
      "activations/layer18_attention_weight_max": 54.6542854309082,
      "activations/layer18_attention_weight_min": -32.8476448059082,
      "activations/layer19_attention_weight_max": 20.861360549926758,
      "activations/layer19_attention_weight_min": -19.677209854125977,
      "activations/layer1_attention_weight_max": 17.737199783325195,
      "activations/layer1_attention_weight_min": -14.742596626281738,
      "activations/layer20_attention_weight_max": 21.199968338012695,
      "activations/layer20_attention_weight_min": -18.228172302246094,
      "activations/layer21_attention_weight_max": 41.31924057006836,
      "activations/layer21_attention_weight_min": -23.95287322998047,
      "activations/layer22_attention_weight_max": 35.10812759399414,
      "activations/layer22_attention_weight_min": -23.17763328552246,
      "activations/layer23_attention_weight_max": 29.944005966186523,
      "activations/layer23_attention_weight_min": -16.920774459838867,
      "activations/layer2_attention_weight_max": 30.352577209472656,
      "activations/layer2_attention_weight_min": -29.166196823120117,
      "activations/layer3_attention_weight_max": 77.36563873291016,
      "activations/layer3_attention_weight_min": -80.92787170410156,
      "activations/layer4_attention_weight_max": 101.20967102050781,
      "activations/layer4_attention_weight_min": -92.12388610839844,
      "activations/layer5_attention_weight_max": 109.98731994628906,
      "activations/layer5_attention_weight_min": -96.53305053710938,
      "activations/layer6_attention_weight_max": 61.5552978515625,
      "activations/layer6_attention_weight_min": -56.29843521118164,
      "activations/layer7_attention_weight_max": 66.26730346679688,
      "activations/layer7_attention_weight_min": -60.98176574707031,
      "activations/layer8_attention_weight_max": 43.497928619384766,
      "activations/layer8_attention_weight_min": -38.587860107421875,
      "activations/layer9_attention_weight_max": 55.12495040893555,
      "activations/layer9_attention_weight_min": -44.779136657714844,
      "epoch": 4.3,
      "learning_rate": 0.00012353333333333332,
      "loss": 2.8759,
      "step": 73950
    },
    {
      "activations/layer0_attention_weight_max": 14.612480163574219,
      "activations/layer0_attention_weight_min": -14.994621276855469,
      "activations/layer10_attention_weight_max": 37.82664489746094,
      "activations/layer10_attention_weight_min": -34.980857849121094,
      "activations/layer11_attention_weight_max": 31.65207290649414,
      "activations/layer11_attention_weight_min": -26.271038055419922,
      "activations/layer12_attention_weight_max": 20.169876098632812,
      "activations/layer12_attention_weight_min": -23.78199577331543,
      "activations/layer13_attention_weight_max": 67.53011322021484,
      "activations/layer13_attention_weight_min": -51.61880111694336,
      "activations/layer14_attention_weight_max": 55.75656509399414,
      "activations/layer14_attention_weight_min": -38.620906829833984,
      "activations/layer15_attention_weight_max": 47.368438720703125,
      "activations/layer15_attention_weight_min": -32.21568298339844,
      "activations/layer16_attention_weight_max": 44.57282638549805,
      "activations/layer16_attention_weight_min": -25.587566375732422,
      "activations/layer17_attention_weight_max": 61.99348449707031,
      "activations/layer17_attention_weight_min": -41.941646575927734,
      "activations/layer18_attention_weight_max": 53.19737243652344,
      "activations/layer18_attention_weight_min": -32.881919860839844,
      "activations/layer19_attention_weight_max": 26.894514083862305,
      "activations/layer19_attention_weight_min": -20.544843673706055,
      "activations/layer1_attention_weight_max": 17.76978302001953,
      "activations/layer1_attention_weight_min": -17.151453018188477,
      "activations/layer20_attention_weight_max": 23.240846633911133,
      "activations/layer20_attention_weight_min": -21.337060928344727,
      "activations/layer21_attention_weight_max": 40.92463302612305,
      "activations/layer21_attention_weight_min": -26.726762771606445,
      "activations/layer22_attention_weight_max": 41.64594650268555,
      "activations/layer22_attention_weight_min": -22.63166046142578,
      "activations/layer23_attention_weight_max": 27.799579620361328,
      "activations/layer23_attention_weight_min": -18.010263442993164,
      "activations/layer2_attention_weight_max": 28.64833641052246,
      "activations/layer2_attention_weight_min": -26.307369232177734,
      "activations/layer3_attention_weight_max": 72.05823516845703,
      "activations/layer3_attention_weight_min": -76.84745025634766,
      "activations/layer4_attention_weight_max": 91.51560974121094,
      "activations/layer4_attention_weight_min": -83.4227523803711,
      "activations/layer5_attention_weight_max": 101.45245361328125,
      "activations/layer5_attention_weight_min": -89.18194580078125,
      "activations/layer6_attention_weight_max": 54.88899230957031,
      "activations/layer6_attention_weight_min": -48.5770149230957,
      "activations/layer7_attention_weight_max": 62.075721740722656,
      "activations/layer7_attention_weight_min": -57.56300354003906,
      "activations/layer8_attention_weight_max": 39.50889587402344,
      "activations/layer8_attention_weight_min": -36.401424407958984,
      "activations/layer9_attention_weight_max": 49.08296203613281,
      "activations/layer9_attention_weight_min": -47.95317077636719,
      "epoch": 4.3,
      "learning_rate": 0.00012351439393939393,
      "loss": 2.8566,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_loss": 2.818359375,
      "eval_runtime": 8.4484,
      "eval_samples_per_second": 508.261,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_openwebtext_loss": 2.818359375,
      "eval_openwebtext_ppl": 16.74934871787156,
      "eval_openwebtext_runtime": 8.4484,
      "eval_openwebtext_samples_per_second": 508.261,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_wikitext_loss": 3.0546875,
      "eval_wikitext_ppl": 21.214554942517648,
      "eval_wikitext_runtime": 1.926,
      "eval_wikitext_samples_per_second": 236.76,
      "step": 74000
    },
    {
      "epoch": 4.3,
      "eval_lambada_loss": 2.873046875,
      "eval_lambada_ppl": 17.690837939037834,
      "eval_lambada_runtime": 9.5725,
      "eval_lambada_samples_per_second": 508.644,
      "step": 74000
    },
    {
      "activations/layer0_attention_weight_max": 13.943780899047852,
      "activations/layer0_attention_weight_min": -15.47354793548584,
      "activations/layer10_attention_weight_max": 39.135589599609375,
      "activations/layer10_attention_weight_min": -34.60712432861328,
      "activations/layer11_attention_weight_max": 32.00962829589844,
      "activations/layer11_attention_weight_min": -27.006893157958984,
      "activations/layer12_attention_weight_max": 24.672012329101562,
      "activations/layer12_attention_weight_min": -25.21686363220215,
      "activations/layer13_attention_weight_max": 44.04126739501953,
      "activations/layer13_attention_weight_min": -31.512462615966797,
      "activations/layer14_attention_weight_max": 38.832420349121094,
      "activations/layer14_attention_weight_min": -30.423357009887695,
      "activations/layer15_attention_weight_max": 40.85082244873047,
      "activations/layer15_attention_weight_min": -28.7365665435791,
      "activations/layer16_attention_weight_max": 37.047237396240234,
      "activations/layer16_attention_weight_min": -26.4406795501709,
      "activations/layer17_attention_weight_max": 57.0406494140625,
      "activations/layer17_attention_weight_min": -39.91436767578125,
      "activations/layer18_attention_weight_max": 56.43187713623047,
      "activations/layer18_attention_weight_min": -30.705486297607422,
      "activations/layer19_attention_weight_max": 23.759653091430664,
      "activations/layer19_attention_weight_min": -18.549776077270508,
      "activations/layer1_attention_weight_max": 17.39995765686035,
      "activations/layer1_attention_weight_min": -16.2515926361084,
      "activations/layer20_attention_weight_max": 21.66981315612793,
      "activations/layer20_attention_weight_min": -18.400962829589844,
      "activations/layer21_attention_weight_max": 39.73838424682617,
      "activations/layer21_attention_weight_min": -28.629825592041016,
      "activations/layer22_attention_weight_max": 38.51163864135742,
      "activations/layer22_attention_weight_min": -24.07408905029297,
      "activations/layer23_attention_weight_max": 25.064476013183594,
      "activations/layer23_attention_weight_min": -17.858171463012695,
      "activations/layer2_attention_weight_max": 28.802730560302734,
      "activations/layer2_attention_weight_min": -28.306461334228516,
      "activations/layer3_attention_weight_max": 80.3541030883789,
      "activations/layer3_attention_weight_min": -81.7331771850586,
      "activations/layer4_attention_weight_max": 99.48917388916016,
      "activations/layer4_attention_weight_min": -91.78633880615234,
      "activations/layer5_attention_weight_max": 110.22725677490234,
      "activations/layer5_attention_weight_min": -98.88700866699219,
      "activations/layer6_attention_weight_max": 61.51401138305664,
      "activations/layer6_attention_weight_min": -53.531925201416016,
      "activations/layer7_attention_weight_max": 66.25921630859375,
      "activations/layer7_attention_weight_min": -63.234153747558594,
      "activations/layer8_attention_weight_max": 42.15932083129883,
      "activations/layer8_attention_weight_min": -40.74972915649414,
      "activations/layer9_attention_weight_max": 46.24270248413086,
      "activations/layer9_attention_weight_min": -45.99151611328125,
      "epoch": 4.3,
      "learning_rate": 0.00012349545454545455,
      "loss": 2.8539,
      "step": 74050
    },
    {
      "activations/layer0_attention_weight_max": 13.881978034973145,
      "activations/layer0_attention_weight_min": -14.910367965698242,
      "activations/layer10_attention_weight_max": 39.88274383544922,
      "activations/layer10_attention_weight_min": -39.48557662963867,
      "activations/layer11_attention_weight_max": 31.807302474975586,
      "activations/layer11_attention_weight_min": -29.78603172302246,
      "activations/layer12_attention_weight_max": 24.53362464904785,
      "activations/layer12_attention_weight_min": -24.1418514251709,
      "activations/layer13_attention_weight_max": 55.93353271484375,
      "activations/layer13_attention_weight_min": -41.86833953857422,
      "activations/layer14_attention_weight_max": 42.54087448120117,
      "activations/layer14_attention_weight_min": -32.98893737792969,
      "activations/layer15_attention_weight_max": 46.3495979309082,
      "activations/layer15_attention_weight_min": -32.712615966796875,
      "activations/layer16_attention_weight_max": 36.07744216918945,
      "activations/layer16_attention_weight_min": -27.611352920532227,
      "activations/layer17_attention_weight_max": 60.84602355957031,
      "activations/layer17_attention_weight_min": -39.58931350708008,
      "activations/layer18_attention_weight_max": 51.34324264526367,
      "activations/layer18_attention_weight_min": -33.50689697265625,
      "activations/layer19_attention_weight_max": 22.851360321044922,
      "activations/layer19_attention_weight_min": -19.845918655395508,
      "activations/layer1_attention_weight_max": 18.474098205566406,
      "activations/layer1_attention_weight_min": -13.894720077514648,
      "activations/layer20_attention_weight_max": 20.425992965698242,
      "activations/layer20_attention_weight_min": -20.902050018310547,
      "activations/layer21_attention_weight_max": 37.32392120361328,
      "activations/layer21_attention_weight_min": -29.190208435058594,
      "activations/layer22_attention_weight_max": 33.03287887573242,
      "activations/layer22_attention_weight_min": -24.855859756469727,
      "activations/layer23_attention_weight_max": 28.477218627929688,
      "activations/layer23_attention_weight_min": -18.80695915222168,
      "activations/layer2_attention_weight_max": 28.42457389831543,
      "activations/layer2_attention_weight_min": -27.857030868530273,
      "activations/layer3_attention_weight_max": 79.46479034423828,
      "activations/layer3_attention_weight_min": -81.45095825195312,
      "activations/layer4_attention_weight_max": 106.09169006347656,
      "activations/layer4_attention_weight_min": -101.05465698242188,
      "activations/layer5_attention_weight_max": 112.95161437988281,
      "activations/layer5_attention_weight_min": -105.33718872070312,
      "activations/layer6_attention_weight_max": 61.838417053222656,
      "activations/layer6_attention_weight_min": -57.97734451293945,
      "activations/layer7_attention_weight_max": 67.71241760253906,
      "activations/layer7_attention_weight_min": -66.41861724853516,
      "activations/layer8_attention_weight_max": 45.4472541809082,
      "activations/layer8_attention_weight_min": -42.26483154296875,
      "activations/layer9_attention_weight_max": 51.91129684448242,
      "activations/layer9_attention_weight_min": -48.38570022583008,
      "epoch": 4.31,
      "learning_rate": 0.00012347689393939394,
      "loss": 2.8688,
      "step": 74100
    },
    {
      "activations/layer0_attention_weight_max": 14.562383651733398,
      "activations/layer0_attention_weight_min": -14.788836479187012,
      "activations/layer10_attention_weight_max": 40.90779113769531,
      "activations/layer10_attention_weight_min": -33.46343231201172,
      "activations/layer11_attention_weight_max": 29.820301055908203,
      "activations/layer11_attention_weight_min": -26.535261154174805,
      "activations/layer12_attention_weight_max": 22.364715576171875,
      "activations/layer12_attention_weight_min": -24.778244018554688,
      "activations/layer13_attention_weight_max": 53.064693450927734,
      "activations/layer13_attention_weight_min": -38.22132110595703,
      "activations/layer14_attention_weight_max": 43.5927619934082,
      "activations/layer14_attention_weight_min": -32.841949462890625,
      "activations/layer15_attention_weight_max": 41.92121505737305,
      "activations/layer15_attention_weight_min": -28.653013229370117,
      "activations/layer16_attention_weight_max": 35.13250732421875,
      "activations/layer16_attention_weight_min": -26.100343704223633,
      "activations/layer17_attention_weight_max": 66.366455078125,
      "activations/layer17_attention_weight_min": -37.66450119018555,
      "activations/layer18_attention_weight_max": 55.89162826538086,
      "activations/layer18_attention_weight_min": -32.73522186279297,
      "activations/layer19_attention_weight_max": 21.86370086669922,
      "activations/layer19_attention_weight_min": -19.456092834472656,
      "activations/layer1_attention_weight_max": 17.530258178710938,
      "activations/layer1_attention_weight_min": -14.932231903076172,
      "activations/layer20_attention_weight_max": 22.016918182373047,
      "activations/layer20_attention_weight_min": -21.33051109313965,
      "activations/layer21_attention_weight_max": 39.29341125488281,
      "activations/layer21_attention_weight_min": -28.93754768371582,
      "activations/layer22_attention_weight_max": 34.49885940551758,
      "activations/layer22_attention_weight_min": -23.515644073486328,
      "activations/layer23_attention_weight_max": 24.6014404296875,
      "activations/layer23_attention_weight_min": -16.054126739501953,
      "activations/layer2_attention_weight_max": 31.801433563232422,
      "activations/layer2_attention_weight_min": -28.77339744567871,
      "activations/layer3_attention_weight_max": 79.28121948242188,
      "activations/layer3_attention_weight_min": -82.69873046875,
      "activations/layer4_attention_weight_max": 97.29415893554688,
      "activations/layer4_attention_weight_min": -92.09086608886719,
      "activations/layer5_attention_weight_max": 106.32464599609375,
      "activations/layer5_attention_weight_min": -96.08080291748047,
      "activations/layer6_attention_weight_max": 56.828880310058594,
      "activations/layer6_attention_weight_min": -52.080875396728516,
      "activations/layer7_attention_weight_max": 63.014488220214844,
      "activations/layer7_attention_weight_min": -59.53810501098633,
      "activations/layer8_attention_weight_max": 40.527835845947266,
      "activations/layer8_attention_weight_min": -39.623191833496094,
      "activations/layer9_attention_weight_max": 47.86028289794922,
      "activations/layer9_attention_weight_min": -44.07600021362305,
      "epoch": 4.31,
      "learning_rate": 0.00012345795454545453,
      "loss": 2.8707,
      "step": 74150
    },
    {
      "activations/layer0_attention_weight_max": 14.580960273742676,
      "activations/layer0_attention_weight_min": -15.120706558227539,
      "activations/layer10_attention_weight_max": 43.245643615722656,
      "activations/layer10_attention_weight_min": -33.346435546875,
      "activations/layer11_attention_weight_max": 35.80892562866211,
      "activations/layer11_attention_weight_min": -26.519725799560547,
      "activations/layer12_attention_weight_max": 24.099382400512695,
      "activations/layer12_attention_weight_min": -24.37908172607422,
      "activations/layer13_attention_weight_max": 63.75324249267578,
      "activations/layer13_attention_weight_min": -37.54942321777344,
      "activations/layer14_attention_weight_max": 64.62113189697266,
      "activations/layer14_attention_weight_min": -33.17234802246094,
      "activations/layer15_attention_weight_max": 57.621337890625,
      "activations/layer15_attention_weight_min": -31.222824096679688,
      "activations/layer16_attention_weight_max": 37.45832443237305,
      "activations/layer16_attention_weight_min": -27.588577270507812,
      "activations/layer17_attention_weight_max": 59.62856674194336,
      "activations/layer17_attention_weight_min": -39.00658416748047,
      "activations/layer18_attention_weight_max": 46.52894973754883,
      "activations/layer18_attention_weight_min": -32.987335205078125,
      "activations/layer19_attention_weight_max": 30.118270874023438,
      "activations/layer19_attention_weight_min": -19.008359909057617,
      "activations/layer1_attention_weight_max": 18.46018409729004,
      "activations/layer1_attention_weight_min": -17.489837646484375,
      "activations/layer20_attention_weight_max": 22.947832107543945,
      "activations/layer20_attention_weight_min": -21.563230514526367,
      "activations/layer21_attention_weight_max": 39.992835998535156,
      "activations/layer21_attention_weight_min": -26.52901268005371,
      "activations/layer22_attention_weight_max": 35.407588958740234,
      "activations/layer22_attention_weight_min": -23.918210983276367,
      "activations/layer23_attention_weight_max": 24.669498443603516,
      "activations/layer23_attention_weight_min": -17.875839233398438,
      "activations/layer2_attention_weight_max": 31.174968719482422,
      "activations/layer2_attention_weight_min": -29.334829330444336,
      "activations/layer3_attention_weight_max": 80.42157745361328,
      "activations/layer3_attention_weight_min": -86.68769836425781,
      "activations/layer4_attention_weight_max": 101.32329559326172,
      "activations/layer4_attention_weight_min": -96.69707489013672,
      "activations/layer5_attention_weight_max": 111.98998260498047,
      "activations/layer5_attention_weight_min": -102.3890380859375,
      "activations/layer6_attention_weight_max": 60.79374694824219,
      "activations/layer6_attention_weight_min": -53.347862243652344,
      "activations/layer7_attention_weight_max": 66.18316650390625,
      "activations/layer7_attention_weight_min": -61.83224868774414,
      "activations/layer8_attention_weight_max": 45.71101379394531,
      "activations/layer8_attention_weight_min": -39.024173736572266,
      "activations/layer9_attention_weight_max": 55.95587921142578,
      "activations/layer9_attention_weight_min": -49.73664093017578,
      "epoch": 4.31,
      "learning_rate": 0.00012343901515151514,
      "loss": 2.8636,
      "step": 74200
    },
    {
      "activations/layer0_attention_weight_max": 14.441246032714844,
      "activations/layer0_attention_weight_min": -15.108661651611328,
      "activations/layer10_attention_weight_max": 41.2922477722168,
      "activations/layer10_attention_weight_min": -35.93576431274414,
      "activations/layer11_attention_weight_max": 34.89784240722656,
      "activations/layer11_attention_weight_min": -26.240583419799805,
      "activations/layer12_attention_weight_max": 23.10476303100586,
      "activations/layer12_attention_weight_min": -27.728395462036133,
      "activations/layer13_attention_weight_max": 51.34703826904297,
      "activations/layer13_attention_weight_min": -40.13475036621094,
      "activations/layer14_attention_weight_max": 40.52096939086914,
      "activations/layer14_attention_weight_min": -32.093505859375,
      "activations/layer15_attention_weight_max": 42.294464111328125,
      "activations/layer15_attention_weight_min": -30.08284568786621,
      "activations/layer16_attention_weight_max": 40.05767822265625,
      "activations/layer16_attention_weight_min": -26.378774642944336,
      "activations/layer17_attention_weight_max": 63.55553436279297,
      "activations/layer17_attention_weight_min": -38.631195068359375,
      "activations/layer18_attention_weight_max": 55.357025146484375,
      "activations/layer18_attention_weight_min": -33.379364013671875,
      "activations/layer19_attention_weight_max": 23.959829330444336,
      "activations/layer19_attention_weight_min": -19.36935043334961,
      "activations/layer1_attention_weight_max": 17.028013229370117,
      "activations/layer1_attention_weight_min": -14.532567024230957,
      "activations/layer20_attention_weight_max": 22.01659393310547,
      "activations/layer20_attention_weight_min": -18.627052307128906,
      "activations/layer21_attention_weight_max": 42.82684326171875,
      "activations/layer21_attention_weight_min": -30.270977020263672,
      "activations/layer22_attention_weight_max": 38.98247528076172,
      "activations/layer22_attention_weight_min": -23.23769187927246,
      "activations/layer23_attention_weight_max": 30.407028198242188,
      "activations/layer23_attention_weight_min": -19.0146427154541,
      "activations/layer2_attention_weight_max": 28.70235824584961,
      "activations/layer2_attention_weight_min": -26.475303649902344,
      "activations/layer3_attention_weight_max": 76.33673858642578,
      "activations/layer3_attention_weight_min": -76.38475036621094,
      "activations/layer4_attention_weight_max": 94.55084228515625,
      "activations/layer4_attention_weight_min": -86.32183074951172,
      "activations/layer5_attention_weight_max": 104.59984588623047,
      "activations/layer5_attention_weight_min": -91.7401123046875,
      "activations/layer6_attention_weight_max": 57.23088073730469,
      "activations/layer6_attention_weight_min": -48.88658142089844,
      "activations/layer7_attention_weight_max": 63.80158996582031,
      "activations/layer7_attention_weight_min": -62.20825958251953,
      "activations/layer8_attention_weight_max": 41.0792236328125,
      "activations/layer8_attention_weight_min": -37.976829528808594,
      "activations/layer9_attention_weight_max": 48.82503890991211,
      "activations/layer9_attention_weight_min": -46.139060974121094,
      "epoch": 4.31,
      "learning_rate": 0.00012342007575757576,
      "loss": 2.8624,
      "step": 74250
    },
    {
      "activations/layer0_attention_weight_max": 15.270600318908691,
      "activations/layer0_attention_weight_min": -15.105169296264648,
      "activations/layer10_attention_weight_max": 41.584190368652344,
      "activations/layer10_attention_weight_min": -36.412940979003906,
      "activations/layer11_attention_weight_max": 34.42718505859375,
      "activations/layer11_attention_weight_min": -29.25295066833496,
      "activations/layer12_attention_weight_max": 26.235811233520508,
      "activations/layer12_attention_weight_min": -26.272602081298828,
      "activations/layer13_attention_weight_max": 77.58949279785156,
      "activations/layer13_attention_weight_min": -46.92782211303711,
      "activations/layer14_attention_weight_max": 63.31296157836914,
      "activations/layer14_attention_weight_min": -36.896400451660156,
      "activations/layer15_attention_weight_max": 50.117637634277344,
      "activations/layer15_attention_weight_min": -30.47586441040039,
      "activations/layer16_attention_weight_max": 39.68154525756836,
      "activations/layer16_attention_weight_min": -25.776473999023438,
      "activations/layer17_attention_weight_max": 63.10211181640625,
      "activations/layer17_attention_weight_min": -38.297698974609375,
      "activations/layer18_attention_weight_max": 55.44783020019531,
      "activations/layer18_attention_weight_min": -33.77050018310547,
      "activations/layer19_attention_weight_max": 24.337732315063477,
      "activations/layer19_attention_weight_min": -19.652082443237305,
      "activations/layer1_attention_weight_max": 18.388948440551758,
      "activations/layer1_attention_weight_min": -15.5679349899292,
      "activations/layer20_attention_weight_max": 23.482030868530273,
      "activations/layer20_attention_weight_min": -18.512126922607422,
      "activations/layer21_attention_weight_max": 47.35795974731445,
      "activations/layer21_attention_weight_min": -27.742746353149414,
      "activations/layer22_attention_weight_max": 35.39084243774414,
      "activations/layer22_attention_weight_min": -23.688785552978516,
      "activations/layer23_attention_weight_max": 29.18220329284668,
      "activations/layer23_attention_weight_min": -18.03167152404785,
      "activations/layer2_attention_weight_max": 30.34625244140625,
      "activations/layer2_attention_weight_min": -27.778079986572266,
      "activations/layer3_attention_weight_max": 79.03025817871094,
      "activations/layer3_attention_weight_min": -80.66572570800781,
      "activations/layer4_attention_weight_max": 102.18965911865234,
      "activations/layer4_attention_weight_min": -97.8973388671875,
      "activations/layer5_attention_weight_max": 108.71949768066406,
      "activations/layer5_attention_weight_min": -94.94229125976562,
      "activations/layer6_attention_weight_max": 60.06111145019531,
      "activations/layer6_attention_weight_min": -54.29719161987305,
      "activations/layer7_attention_weight_max": 66.67015838623047,
      "activations/layer7_attention_weight_min": -64.22817993164062,
      "activations/layer8_attention_weight_max": 43.57148742675781,
      "activations/layer8_attention_weight_min": -42.61593246459961,
      "activations/layer9_attention_weight_max": 52.16095733642578,
      "activations/layer9_attention_weight_min": -48.899436950683594,
      "epoch": 4.32,
      "learning_rate": 0.00012340113636363637,
      "loss": 2.8561,
      "step": 74300
    },
    {
      "activations/layer0_attention_weight_max": 14.60495376586914,
      "activations/layer0_attention_weight_min": -14.996467590332031,
      "activations/layer10_attention_weight_max": 41.506317138671875,
      "activations/layer10_attention_weight_min": -33.73790740966797,
      "activations/layer11_attention_weight_max": 33.63713073730469,
      "activations/layer11_attention_weight_min": -25.514076232910156,
      "activations/layer12_attention_weight_max": 25.07994270324707,
      "activations/layer12_attention_weight_min": -25.6710262298584,
      "activations/layer13_attention_weight_max": 70.3081283569336,
      "activations/layer13_attention_weight_min": -42.95420455932617,
      "activations/layer14_attention_weight_max": 54.0714225769043,
      "activations/layer14_attention_weight_min": -34.66680908203125,
      "activations/layer15_attention_weight_max": 49.9077033996582,
      "activations/layer15_attention_weight_min": -31.021682739257812,
      "activations/layer16_attention_weight_max": 37.950050354003906,
      "activations/layer16_attention_weight_min": -25.791797637939453,
      "activations/layer17_attention_weight_max": 57.07472610473633,
      "activations/layer17_attention_weight_min": -36.806739807128906,
      "activations/layer18_attention_weight_max": 48.49055862426758,
      "activations/layer18_attention_weight_min": -32.54738998413086,
      "activations/layer19_attention_weight_max": 21.271873474121094,
      "activations/layer19_attention_weight_min": -18.868345260620117,
      "activations/layer1_attention_weight_max": 17.706205368041992,
      "activations/layer1_attention_weight_min": -14.620067596435547,
      "activations/layer20_attention_weight_max": 24.98030662536621,
      "activations/layer20_attention_weight_min": -23.359569549560547,
      "activations/layer21_attention_weight_max": 46.59445571899414,
      "activations/layer21_attention_weight_min": -25.782981872558594,
      "activations/layer22_attention_weight_max": 37.52698516845703,
      "activations/layer22_attention_weight_min": -24.15474510192871,
      "activations/layer23_attention_weight_max": 29.271879196166992,
      "activations/layer23_attention_weight_min": -18.421558380126953,
      "activations/layer2_attention_weight_max": 27.852828979492188,
      "activations/layer2_attention_weight_min": -26.064067840576172,
      "activations/layer3_attention_weight_max": 76.29539489746094,
      "activations/layer3_attention_weight_min": -81.00969696044922,
      "activations/layer4_attention_weight_max": 98.22583770751953,
      "activations/layer4_attention_weight_min": -90.75865173339844,
      "activations/layer5_attention_weight_max": 107.90990447998047,
      "activations/layer5_attention_weight_min": -95.5496826171875,
      "activations/layer6_attention_weight_max": 58.5645866394043,
      "activations/layer6_attention_weight_min": -52.68177795410156,
      "activations/layer7_attention_weight_max": 75.17098236083984,
      "activations/layer7_attention_weight_min": -65.66967010498047,
      "activations/layer8_attention_weight_max": 43.32106018066406,
      "activations/layer8_attention_weight_min": -41.692832946777344,
      "activations/layer9_attention_weight_max": 55.3181266784668,
      "activations/layer9_attention_weight_min": -49.440582275390625,
      "epoch": 4.32,
      "learning_rate": 0.00012338219696969696,
      "loss": 2.8569,
      "step": 74350
    },
    {
      "activations/layer0_attention_weight_max": 14.452351570129395,
      "activations/layer0_attention_weight_min": -14.614740371704102,
      "activations/layer10_attention_weight_max": 39.24609375,
      "activations/layer10_attention_weight_min": -35.406105041503906,
      "activations/layer11_attention_weight_max": 30.678813934326172,
      "activations/layer11_attention_weight_min": -28.076271057128906,
      "activations/layer12_attention_weight_max": 21.263385772705078,
      "activations/layer12_attention_weight_min": -25.37566566467285,
      "activations/layer13_attention_weight_max": 54.67741394042969,
      "activations/layer13_attention_weight_min": -38.16172409057617,
      "activations/layer14_attention_weight_max": 47.473602294921875,
      "activations/layer14_attention_weight_min": -30.265363693237305,
      "activations/layer15_attention_weight_max": 39.33080291748047,
      "activations/layer15_attention_weight_min": -30.137237548828125,
      "activations/layer16_attention_weight_max": 35.23099899291992,
      "activations/layer16_attention_weight_min": -26.740163803100586,
      "activations/layer17_attention_weight_max": 60.93219757080078,
      "activations/layer17_attention_weight_min": -38.50889587402344,
      "activations/layer18_attention_weight_max": 46.24943923950195,
      "activations/layer18_attention_weight_min": -33.780799865722656,
      "activations/layer19_attention_weight_max": 23.588064193725586,
      "activations/layer19_attention_weight_min": -20.134544372558594,
      "activations/layer1_attention_weight_max": 17.836809158325195,
      "activations/layer1_attention_weight_min": -18.058042526245117,
      "activations/layer20_attention_weight_max": 22.445493698120117,
      "activations/layer20_attention_weight_min": -19.576622009277344,
      "activations/layer21_attention_weight_max": 35.82316207885742,
      "activations/layer21_attention_weight_min": -25.755815505981445,
      "activations/layer22_attention_weight_max": 31.692441940307617,
      "activations/layer22_attention_weight_min": -24.23056983947754,
      "activations/layer23_attention_weight_max": 26.572479248046875,
      "activations/layer23_attention_weight_min": -18.681758880615234,
      "activations/layer2_attention_weight_max": 29.738462448120117,
      "activations/layer2_attention_weight_min": -27.676908493041992,
      "activations/layer3_attention_weight_max": 81.50389862060547,
      "activations/layer3_attention_weight_min": -81.50080108642578,
      "activations/layer4_attention_weight_max": 107.2660140991211,
      "activations/layer4_attention_weight_min": -95.44139099121094,
      "activations/layer5_attention_weight_max": 115.6131820678711,
      "activations/layer5_attention_weight_min": -98.38945007324219,
      "activations/layer6_attention_weight_max": 61.495731353759766,
      "activations/layer6_attention_weight_min": -54.918724060058594,
      "activations/layer7_attention_weight_max": 74.07780456542969,
      "activations/layer7_attention_weight_min": -66.64739227294922,
      "activations/layer8_attention_weight_max": 46.33269119262695,
      "activations/layer8_attention_weight_min": -42.4513053894043,
      "activations/layer9_attention_weight_max": 57.014984130859375,
      "activations/layer9_attention_weight_min": -48.3675651550293,
      "epoch": 4.32,
      "learning_rate": 0.00012336325757575758,
      "loss": 2.8614,
      "step": 74400
    },
    {
      "activations/layer0_attention_weight_max": 14.269176483154297,
      "activations/layer0_attention_weight_min": -14.838902473449707,
      "activations/layer10_attention_weight_max": 41.73540115356445,
      "activations/layer10_attention_weight_min": -33.66426467895508,
      "activations/layer11_attention_weight_max": 29.52858543395996,
      "activations/layer11_attention_weight_min": -27.020811080932617,
      "activations/layer12_attention_weight_max": 25.26498031616211,
      "activations/layer12_attention_weight_min": -26.782154083251953,
      "activations/layer13_attention_weight_max": 56.3675537109375,
      "activations/layer13_attention_weight_min": -41.708740234375,
      "activations/layer14_attention_weight_max": 66.52134704589844,
      "activations/layer14_attention_weight_min": -49.89505386352539,
      "activations/layer15_attention_weight_max": 46.287296295166016,
      "activations/layer15_attention_weight_min": -36.344844818115234,
      "activations/layer16_attention_weight_max": 35.48583984375,
      "activations/layer16_attention_weight_min": -25.261032104492188,
      "activations/layer17_attention_weight_max": 58.52139663696289,
      "activations/layer17_attention_weight_min": -36.38822937011719,
      "activations/layer18_attention_weight_max": 52.40526580810547,
      "activations/layer18_attention_weight_min": -31.929237365722656,
      "activations/layer19_attention_weight_max": 23.471731185913086,
      "activations/layer19_attention_weight_min": -18.223188400268555,
      "activations/layer1_attention_weight_max": 17.33429527282715,
      "activations/layer1_attention_weight_min": -15.233253479003906,
      "activations/layer20_attention_weight_max": 22.706348419189453,
      "activations/layer20_attention_weight_min": -22.644489288330078,
      "activations/layer21_attention_weight_max": 39.17787551879883,
      "activations/layer21_attention_weight_min": -29.0367431640625,
      "activations/layer22_attention_weight_max": 34.51344299316406,
      "activations/layer22_attention_weight_min": -24.170305252075195,
      "activations/layer23_attention_weight_max": 24.626676559448242,
      "activations/layer23_attention_weight_min": -19.322471618652344,
      "activations/layer2_attention_weight_max": 28.71602439880371,
      "activations/layer2_attention_weight_min": -27.187206268310547,
      "activations/layer3_attention_weight_max": 78.00546264648438,
      "activations/layer3_attention_weight_min": -80.1554946899414,
      "activations/layer4_attention_weight_max": 102.4969711303711,
      "activations/layer4_attention_weight_min": -93.65379333496094,
      "activations/layer5_attention_weight_max": 108.7110595703125,
      "activations/layer5_attention_weight_min": -96.636962890625,
      "activations/layer6_attention_weight_max": 58.552677154541016,
      "activations/layer6_attention_weight_min": -54.47056198120117,
      "activations/layer7_attention_weight_max": 65.79678344726562,
      "activations/layer7_attention_weight_min": -63.014461517333984,
      "activations/layer8_attention_weight_max": 45.0568962097168,
      "activations/layer8_attention_weight_min": -40.88716506958008,
      "activations/layer9_attention_weight_max": 53.65896224975586,
      "activations/layer9_attention_weight_min": -49.160579681396484,
      "epoch": 4.33,
      "learning_rate": 0.00012334431818181816,
      "loss": 2.8707,
      "step": 74450
    },
    {
      "activations/layer0_attention_weight_max": 15.967726707458496,
      "activations/layer0_attention_weight_min": -14.645944595336914,
      "activations/layer10_attention_weight_max": 42.173885345458984,
      "activations/layer10_attention_weight_min": -37.074581146240234,
      "activations/layer11_attention_weight_max": 30.35460662841797,
      "activations/layer11_attention_weight_min": -28.491735458374023,
      "activations/layer12_attention_weight_max": 26.01455307006836,
      "activations/layer12_attention_weight_min": -25.47229766845703,
      "activations/layer13_attention_weight_max": 49.0256233215332,
      "activations/layer13_attention_weight_min": -35.660011291503906,
      "activations/layer14_attention_weight_max": 65.72528076171875,
      "activations/layer14_attention_weight_min": -45.98808670043945,
      "activations/layer15_attention_weight_max": 49.68968200683594,
      "activations/layer15_attention_weight_min": -33.303585052490234,
      "activations/layer16_attention_weight_max": 38.606231689453125,
      "activations/layer16_attention_weight_min": -26.399335861206055,
      "activations/layer17_attention_weight_max": 58.95165252685547,
      "activations/layer17_attention_weight_min": -38.97650146484375,
      "activations/layer18_attention_weight_max": 54.57427215576172,
      "activations/layer18_attention_weight_min": -33.67669677734375,
      "activations/layer19_attention_weight_max": 21.945600509643555,
      "activations/layer19_attention_weight_min": -18.315507888793945,
      "activations/layer1_attention_weight_max": 17.678495407104492,
      "activations/layer1_attention_weight_min": -15.1656494140625,
      "activations/layer20_attention_weight_max": 24.4532527923584,
      "activations/layer20_attention_weight_min": -19.42918586730957,
      "activations/layer21_attention_weight_max": 42.4957275390625,
      "activations/layer21_attention_weight_min": -29.241214752197266,
      "activations/layer22_attention_weight_max": 36.654197692871094,
      "activations/layer22_attention_weight_min": -24.751279830932617,
      "activations/layer23_attention_weight_max": 29.077787399291992,
      "activations/layer23_attention_weight_min": -19.312231063842773,
      "activations/layer2_attention_weight_max": 30.702491760253906,
      "activations/layer2_attention_weight_min": -28.58991813659668,
      "activations/layer3_attention_weight_max": 81.8233642578125,
      "activations/layer3_attention_weight_min": -81.50116729736328,
      "activations/layer4_attention_weight_max": 103.24552154541016,
      "activations/layer4_attention_weight_min": -89.10556030273438,
      "activations/layer5_attention_weight_max": 109.34205627441406,
      "activations/layer5_attention_weight_min": -95.10060119628906,
      "activations/layer6_attention_weight_max": 57.62580871582031,
      "activations/layer6_attention_weight_min": -53.13534164428711,
      "activations/layer7_attention_weight_max": 64.2081527709961,
      "activations/layer7_attention_weight_min": -59.52574920654297,
      "activations/layer8_attention_weight_max": 40.272220611572266,
      "activations/layer8_attention_weight_min": -38.30104446411133,
      "activations/layer9_attention_weight_max": 45.952152252197266,
      "activations/layer9_attention_weight_min": -45.24363327026367,
      "epoch": 4.33,
      "learning_rate": 0.00012332537878787878,
      "loss": 2.8607,
      "step": 74500
    },
    {
      "activations/layer0_attention_weight_max": 13.580032348632812,
      "activations/layer0_attention_weight_min": -14.668386459350586,
      "activations/layer10_attention_weight_max": 43.188682556152344,
      "activations/layer10_attention_weight_min": -34.92396926879883,
      "activations/layer11_attention_weight_max": 31.83273696899414,
      "activations/layer11_attention_weight_min": -27.03132438659668,
      "activations/layer12_attention_weight_max": 28.641380310058594,
      "activations/layer12_attention_weight_min": -25.65554428100586,
      "activations/layer13_attention_weight_max": 61.812828063964844,
      "activations/layer13_attention_weight_min": -42.61590576171875,
      "activations/layer14_attention_weight_max": 68.36760711669922,
      "activations/layer14_attention_weight_min": -46.99336624145508,
      "activations/layer15_attention_weight_max": 47.937686920166016,
      "activations/layer15_attention_weight_min": -35.81569290161133,
      "activations/layer16_attention_weight_max": 34.76390838623047,
      "activations/layer16_attention_weight_min": -25.947582244873047,
      "activations/layer17_attention_weight_max": 56.39373779296875,
      "activations/layer17_attention_weight_min": -37.99666976928711,
      "activations/layer18_attention_weight_max": 47.949127197265625,
      "activations/layer18_attention_weight_min": -33.26689529418945,
      "activations/layer19_attention_weight_max": 20.55760383605957,
      "activations/layer19_attention_weight_min": -19.557039260864258,
      "activations/layer1_attention_weight_max": 19.07394790649414,
      "activations/layer1_attention_weight_min": -14.628171920776367,
      "activations/layer20_attention_weight_max": 19.994171142578125,
      "activations/layer20_attention_weight_min": -22.609848022460938,
      "activations/layer21_attention_weight_max": 43.2411003112793,
      "activations/layer21_attention_weight_min": -27.761545181274414,
      "activations/layer22_attention_weight_max": 32.50117874145508,
      "activations/layer22_attention_weight_min": -22.463077545166016,
      "activations/layer23_attention_weight_max": 23.797988891601562,
      "activations/layer23_attention_weight_min": -17.488048553466797,
      "activations/layer2_attention_weight_max": 29.228790283203125,
      "activations/layer2_attention_weight_min": -28.132369995117188,
      "activations/layer3_attention_weight_max": 75.48675537109375,
      "activations/layer3_attention_weight_min": -80.41413116455078,
      "activations/layer4_attention_weight_max": 96.48384857177734,
      "activations/layer4_attention_weight_min": -94.75880432128906,
      "activations/layer5_attention_weight_max": 104.73126220703125,
      "activations/layer5_attention_weight_min": -97.05938720703125,
      "activations/layer6_attention_weight_max": 59.09307098388672,
      "activations/layer6_attention_weight_min": -52.260414123535156,
      "activations/layer7_attention_weight_max": 62.11207580566406,
      "activations/layer7_attention_weight_min": -62.26677322387695,
      "activations/layer8_attention_weight_max": 43.420860290527344,
      "activations/layer8_attention_weight_min": -39.489742279052734,
      "activations/layer9_attention_weight_max": 49.594810485839844,
      "activations/layer9_attention_weight_min": -48.359928131103516,
      "epoch": 4.33,
      "learning_rate": 0.0001233064393939394,
      "loss": 2.857,
      "step": 74550
    },
    {
      "activations/layer0_attention_weight_max": 16.02284049987793,
      "activations/layer0_attention_weight_min": -15.006324768066406,
      "activations/layer10_attention_weight_max": 42.15324401855469,
      "activations/layer10_attention_weight_min": -32.5755615234375,
      "activations/layer11_attention_weight_max": 36.75366973876953,
      "activations/layer11_attention_weight_min": -25.634374618530273,
      "activations/layer12_attention_weight_max": 28.72989273071289,
      "activations/layer12_attention_weight_min": -28.198286056518555,
      "activations/layer13_attention_weight_max": 66.59867858886719,
      "activations/layer13_attention_weight_min": -42.320091247558594,
      "activations/layer14_attention_weight_max": 55.57786560058594,
      "activations/layer14_attention_weight_min": -38.68446731567383,
      "activations/layer15_attention_weight_max": 39.22528076171875,
      "activations/layer15_attention_weight_min": -29.74405288696289,
      "activations/layer16_attention_weight_max": 39.146324157714844,
      "activations/layer16_attention_weight_min": -26.31618309020996,
      "activations/layer17_attention_weight_max": 60.99542236328125,
      "activations/layer17_attention_weight_min": -34.40045928955078,
      "activations/layer18_attention_weight_max": 48.48826217651367,
      "activations/layer18_attention_weight_min": -31.898534774780273,
      "activations/layer19_attention_weight_max": 23.238828659057617,
      "activations/layer19_attention_weight_min": -21.203086853027344,
      "activations/layer1_attention_weight_max": 17.81089210510254,
      "activations/layer1_attention_weight_min": -13.95559310913086,
      "activations/layer20_attention_weight_max": 22.81461524963379,
      "activations/layer20_attention_weight_min": -24.398635864257812,
      "activations/layer21_attention_weight_max": 40.41258239746094,
      "activations/layer21_attention_weight_min": -25.601186752319336,
      "activations/layer22_attention_weight_max": 34.15293502807617,
      "activations/layer22_attention_weight_min": -22.215248107910156,
      "activations/layer23_attention_weight_max": 29.308536529541016,
      "activations/layer23_attention_weight_min": -17.284656524658203,
      "activations/layer2_attention_weight_max": 28.89569091796875,
      "activations/layer2_attention_weight_min": -26.62569236755371,
      "activations/layer3_attention_weight_max": 79.38451385498047,
      "activations/layer3_attention_weight_min": -83.01187133789062,
      "activations/layer4_attention_weight_max": 102.5739517211914,
      "activations/layer4_attention_weight_min": -93.53257751464844,
      "activations/layer5_attention_weight_max": 104.58132934570312,
      "activations/layer5_attention_weight_min": -94.37020111083984,
      "activations/layer6_attention_weight_max": 58.66218566894531,
      "activations/layer6_attention_weight_min": -53.03281021118164,
      "activations/layer7_attention_weight_max": 73.98918151855469,
      "activations/layer7_attention_weight_min": -61.16849136352539,
      "activations/layer8_attention_weight_max": 41.97856521606445,
      "activations/layer8_attention_weight_min": -38.53317642211914,
      "activations/layer9_attention_weight_max": 63.616451263427734,
      "activations/layer9_attention_weight_min": -47.903018951416016,
      "epoch": 4.33,
      "learning_rate": 0.00012328749999999998,
      "loss": 2.8583,
      "step": 74600
    },
    {
      "activations/layer0_attention_weight_max": 15.133270263671875,
      "activations/layer0_attention_weight_min": -15.233905792236328,
      "activations/layer10_attention_weight_max": 37.18293762207031,
      "activations/layer10_attention_weight_min": -32.62577819824219,
      "activations/layer11_attention_weight_max": 28.770999908447266,
      "activations/layer11_attention_weight_min": -25.303314208984375,
      "activations/layer12_attention_weight_max": 28.709699630737305,
      "activations/layer12_attention_weight_min": -25.058698654174805,
      "activations/layer13_attention_weight_max": 44.05149841308594,
      "activations/layer13_attention_weight_min": -31.97238540649414,
      "activations/layer14_attention_weight_max": 36.75066375732422,
      "activations/layer14_attention_weight_min": -29.6299991607666,
      "activations/layer15_attention_weight_max": 41.652488708496094,
      "activations/layer15_attention_weight_min": -29.26602554321289,
      "activations/layer16_attention_weight_max": 35.3389778137207,
      "activations/layer16_attention_weight_min": -25.734542846679688,
      "activations/layer17_attention_weight_max": 53.04667663574219,
      "activations/layer17_attention_weight_min": -34.7686653137207,
      "activations/layer18_attention_weight_max": 46.83183288574219,
      "activations/layer18_attention_weight_min": -31.181682586669922,
      "activations/layer19_attention_weight_max": 19.088899612426758,
      "activations/layer19_attention_weight_min": -19.153196334838867,
      "activations/layer1_attention_weight_max": 18.11199378967285,
      "activations/layer1_attention_weight_min": -13.821720123291016,
      "activations/layer20_attention_weight_max": 26.452747344970703,
      "activations/layer20_attention_weight_min": -20.545225143432617,
      "activations/layer21_attention_weight_max": 42.02166748046875,
      "activations/layer21_attention_weight_min": -26.009172439575195,
      "activations/layer22_attention_weight_max": 34.9066047668457,
      "activations/layer22_attention_weight_min": -23.475799560546875,
      "activations/layer23_attention_weight_max": 25.07711410522461,
      "activations/layer23_attention_weight_min": -17.199447631835938,
      "activations/layer2_attention_weight_max": 28.419666290283203,
      "activations/layer2_attention_weight_min": -26.791748046875,
      "activations/layer3_attention_weight_max": 79.27481079101562,
      "activations/layer3_attention_weight_min": -87.03474426269531,
      "activations/layer4_attention_weight_max": 99.51912689208984,
      "activations/layer4_attention_weight_min": -100.12178802490234,
      "activations/layer5_attention_weight_max": 105.67646789550781,
      "activations/layer5_attention_weight_min": -101.25953674316406,
      "activations/layer6_attention_weight_max": 57.67453384399414,
      "activations/layer6_attention_weight_min": -51.56279373168945,
      "activations/layer7_attention_weight_max": 72.19178771972656,
      "activations/layer7_attention_weight_min": -62.690879821777344,
      "activations/layer8_attention_weight_max": 41.24932098388672,
      "activations/layer8_attention_weight_min": -38.6157341003418,
      "activations/layer9_attention_weight_max": 52.18587112426758,
      "activations/layer9_attention_weight_min": -48.72846603393555,
      "epoch": 4.34,
      "learning_rate": 0.0001232685606060606,
      "loss": 2.8516,
      "step": 74650
    },
    {
      "activations/layer0_attention_weight_max": 14.630321502685547,
      "activations/layer0_attention_weight_min": -15.11172866821289,
      "activations/layer10_attention_weight_max": 37.54566192626953,
      "activations/layer10_attention_weight_min": -35.17367935180664,
      "activations/layer11_attention_weight_max": 31.140501022338867,
      "activations/layer11_attention_weight_min": -26.7459716796875,
      "activations/layer12_attention_weight_max": 23.986236572265625,
      "activations/layer12_attention_weight_min": -24.766061782836914,
      "activations/layer13_attention_weight_max": 55.90553283691406,
      "activations/layer13_attention_weight_min": -39.48627853393555,
      "activations/layer14_attention_weight_max": 40.11648941040039,
      "activations/layer14_attention_weight_min": -28.981203079223633,
      "activations/layer15_attention_weight_max": 39.12981033325195,
      "activations/layer15_attention_weight_min": -31.78431510925293,
      "activations/layer16_attention_weight_max": 34.54827880859375,
      "activations/layer16_attention_weight_min": -25.061752319335938,
      "activations/layer17_attention_weight_max": 61.9590950012207,
      "activations/layer17_attention_weight_min": -36.69865798950195,
      "activations/layer18_attention_weight_max": 53.50389099121094,
      "activations/layer18_attention_weight_min": -31.98430824279785,
      "activations/layer19_attention_weight_max": 24.161766052246094,
      "activations/layer19_attention_weight_min": -18.32417106628418,
      "activations/layer1_attention_weight_max": 17.748760223388672,
      "activations/layer1_attention_weight_min": -14.02310848236084,
      "activations/layer20_attention_weight_max": 21.679019927978516,
      "activations/layer20_attention_weight_min": -22.190139770507812,
      "activations/layer21_attention_weight_max": 40.76848602294922,
      "activations/layer21_attention_weight_min": -27.399883270263672,
      "activations/layer22_attention_weight_max": 35.30027389526367,
      "activations/layer22_attention_weight_min": -23.142576217651367,
      "activations/layer23_attention_weight_max": 28.365713119506836,
      "activations/layer23_attention_weight_min": -15.986421585083008,
      "activations/layer2_attention_weight_max": 31.229248046875,
      "activations/layer2_attention_weight_min": -29.212247848510742,
      "activations/layer3_attention_weight_max": 80.29776763916016,
      "activations/layer3_attention_weight_min": -86.7909927368164,
      "activations/layer4_attention_weight_max": 100.69709777832031,
      "activations/layer4_attention_weight_min": -94.47278594970703,
      "activations/layer5_attention_weight_max": 110.14773559570312,
      "activations/layer5_attention_weight_min": -98.9411849975586,
      "activations/layer6_attention_weight_max": 60.048988342285156,
      "activations/layer6_attention_weight_min": -56.403785705566406,
      "activations/layer7_attention_weight_max": 64.80729675292969,
      "activations/layer7_attention_weight_min": -65.44336700439453,
      "activations/layer8_attention_weight_max": 41.31904220581055,
      "activations/layer8_attention_weight_min": -38.752044677734375,
      "activations/layer9_attention_weight_max": 47.247676849365234,
      "activations/layer9_attention_weight_min": -47.366085052490234,
      "epoch": 4.34,
      "learning_rate": 0.0001232496212121212,
      "loss": 2.8753,
      "step": 74700
    },
    {
      "activations/layer0_attention_weight_max": 13.982192993164062,
      "activations/layer0_attention_weight_min": -15.0226469039917,
      "activations/layer10_attention_weight_max": 39.30371856689453,
      "activations/layer10_attention_weight_min": -32.733917236328125,
      "activations/layer11_attention_weight_max": 29.348217010498047,
      "activations/layer11_attention_weight_min": -26.365589141845703,
      "activations/layer12_attention_weight_max": 24.506885528564453,
      "activations/layer12_attention_weight_min": -26.443330764770508,
      "activations/layer13_attention_weight_max": 46.23154830932617,
      "activations/layer13_attention_weight_min": -34.15790939331055,
      "activations/layer14_attention_weight_max": 54.100948333740234,
      "activations/layer14_attention_weight_min": -36.660953521728516,
      "activations/layer15_attention_weight_max": 43.466575622558594,
      "activations/layer15_attention_weight_min": -29.743118286132812,
      "activations/layer16_attention_weight_max": 32.72835922241211,
      "activations/layer16_attention_weight_min": -26.724313735961914,
      "activations/layer17_attention_weight_max": 54.04158020019531,
      "activations/layer17_attention_weight_min": -37.06464385986328,
      "activations/layer18_attention_weight_max": 45.91281509399414,
      "activations/layer18_attention_weight_min": -30.993648529052734,
      "activations/layer19_attention_weight_max": 20.96104621887207,
      "activations/layer19_attention_weight_min": -18.582002639770508,
      "activations/layer1_attention_weight_max": 17.08231544494629,
      "activations/layer1_attention_weight_min": -13.386672019958496,
      "activations/layer20_attention_weight_max": 19.73937225341797,
      "activations/layer20_attention_weight_min": -20.1240177154541,
      "activations/layer21_attention_weight_max": 40.24592590332031,
      "activations/layer21_attention_weight_min": -26.42763900756836,
      "activations/layer22_attention_weight_max": 33.26240539550781,
      "activations/layer22_attention_weight_min": -23.11965560913086,
      "activations/layer23_attention_weight_max": 26.590612411499023,
      "activations/layer23_attention_weight_min": -16.321699142456055,
      "activations/layer2_attention_weight_max": 29.108455657958984,
      "activations/layer2_attention_weight_min": -27.22425651550293,
      "activations/layer3_attention_weight_max": 76.37650299072266,
      "activations/layer3_attention_weight_min": -77.37721252441406,
      "activations/layer4_attention_weight_max": 98.62482452392578,
      "activations/layer4_attention_weight_min": -88.96966552734375,
      "activations/layer5_attention_weight_max": 106.26838684082031,
      "activations/layer5_attention_weight_min": -94.67876434326172,
      "activations/layer6_attention_weight_max": 58.077857971191406,
      "activations/layer6_attention_weight_min": -52.1629524230957,
      "activations/layer7_attention_weight_max": 62.874080657958984,
      "activations/layer7_attention_weight_min": -60.31402587890625,
      "activations/layer8_attention_weight_max": 38.69019317626953,
      "activations/layer8_attention_weight_min": -40.452552795410156,
      "activations/layer9_attention_weight_max": 51.791072845458984,
      "activations/layer9_attention_weight_min": -51.91339874267578,
      "epoch": 4.34,
      "learning_rate": 0.0001232306818181818,
      "loss": 2.8636,
      "step": 74750
    },
    {
      "activations/layer0_attention_weight_max": 15.389914512634277,
      "activations/layer0_attention_weight_min": -15.293444633483887,
      "activations/layer10_attention_weight_max": 37.889366149902344,
      "activations/layer10_attention_weight_min": -32.69389343261719,
      "activations/layer11_attention_weight_max": 31.860530853271484,
      "activations/layer11_attention_weight_min": -26.28475570678711,
      "activations/layer12_attention_weight_max": 21.963422775268555,
      "activations/layer12_attention_weight_min": -31.596128463745117,
      "activations/layer13_attention_weight_max": 69.6673583984375,
      "activations/layer13_attention_weight_min": -44.83780288696289,
      "activations/layer14_attention_weight_max": 68.81864929199219,
      "activations/layer14_attention_weight_min": -40.24992370605469,
      "activations/layer15_attention_weight_max": 55.429344177246094,
      "activations/layer15_attention_weight_min": -30.184614181518555,
      "activations/layer16_attention_weight_max": 39.34364318847656,
      "activations/layer16_attention_weight_min": -27.45464324951172,
      "activations/layer17_attention_weight_max": 61.34404754638672,
      "activations/layer17_attention_weight_min": -40.854183197021484,
      "activations/layer18_attention_weight_max": 56.63224792480469,
      "activations/layer18_attention_weight_min": -36.04111862182617,
      "activations/layer19_attention_weight_max": 26.13814353942871,
      "activations/layer19_attention_weight_min": -19.692968368530273,
      "activations/layer1_attention_weight_max": 17.890033721923828,
      "activations/layer1_attention_weight_min": -13.902274131774902,
      "activations/layer20_attention_weight_max": 23.530784606933594,
      "activations/layer20_attention_weight_min": -20.145402908325195,
      "activations/layer21_attention_weight_max": 46.323184967041016,
      "activations/layer21_attention_weight_min": -27.556833267211914,
      "activations/layer22_attention_weight_max": 40.6596794128418,
      "activations/layer22_attention_weight_min": -24.758535385131836,
      "activations/layer23_attention_weight_max": 31.33062744140625,
      "activations/layer23_attention_weight_min": -16.63259506225586,
      "activations/layer2_attention_weight_max": 30.975133895874023,
      "activations/layer2_attention_weight_min": -28.867053985595703,
      "activations/layer3_attention_weight_max": 80.63549041748047,
      "activations/layer3_attention_weight_min": -82.65821075439453,
      "activations/layer4_attention_weight_max": 100.30842590332031,
      "activations/layer4_attention_weight_min": -93.64434051513672,
      "activations/layer5_attention_weight_max": 104.28318786621094,
      "activations/layer5_attention_weight_min": -94.0361328125,
      "activations/layer6_attention_weight_max": 58.6668815612793,
      "activations/layer6_attention_weight_min": -50.865726470947266,
      "activations/layer7_attention_weight_max": 69.9896240234375,
      "activations/layer7_attention_weight_min": -64.6421890258789,
      "activations/layer8_attention_weight_max": 38.2326774597168,
      "activations/layer8_attention_weight_min": -37.44611740112305,
      "activations/layer9_attention_weight_max": 54.061214447021484,
      "activations/layer9_attention_weight_min": -42.93550109863281,
      "epoch": 4.35,
      "learning_rate": 0.00012321174242424241,
      "loss": 2.8651,
      "step": 74800
    },
    {
      "activations/layer0_attention_weight_max": 14.094804763793945,
      "activations/layer0_attention_weight_min": -15.139657974243164,
      "activations/layer10_attention_weight_max": 37.588775634765625,
      "activations/layer10_attention_weight_min": -34.10562515258789,
      "activations/layer11_attention_weight_max": 28.008506774902344,
      "activations/layer11_attention_weight_min": -26.626989364624023,
      "activations/layer12_attention_weight_max": 20.3934383392334,
      "activations/layer12_attention_weight_min": -27.90896224975586,
      "activations/layer13_attention_weight_max": 44.51657485961914,
      "activations/layer13_attention_weight_min": -36.62592315673828,
      "activations/layer14_attention_weight_max": 41.6412239074707,
      "activations/layer14_attention_weight_min": -31.292301177978516,
      "activations/layer15_attention_weight_max": 40.18174743652344,
      "activations/layer15_attention_weight_min": -31.260534286499023,
      "activations/layer16_attention_weight_max": 38.25162124633789,
      "activations/layer16_attention_weight_min": -24.52165412902832,
      "activations/layer17_attention_weight_max": 51.56551742553711,
      "activations/layer17_attention_weight_min": -35.924068450927734,
      "activations/layer18_attention_weight_max": 47.71682357788086,
      "activations/layer18_attention_weight_min": -30.760469436645508,
      "activations/layer19_attention_weight_max": 21.56592559814453,
      "activations/layer19_attention_weight_min": -18.02552604675293,
      "activations/layer1_attention_weight_max": 18.497159957885742,
      "activations/layer1_attention_weight_min": -15.50114631652832,
      "activations/layer20_attention_weight_max": 19.913084030151367,
      "activations/layer20_attention_weight_min": -20.124656677246094,
      "activations/layer21_attention_weight_max": 36.35886001586914,
      "activations/layer21_attention_weight_min": -27.477909088134766,
      "activations/layer22_attention_weight_max": 35.331485748291016,
      "activations/layer22_attention_weight_min": -23.26200294494629,
      "activations/layer23_attention_weight_max": 27.045392990112305,
      "activations/layer23_attention_weight_min": -16.183101654052734,
      "activations/layer2_attention_weight_max": 28.848859786987305,
      "activations/layer2_attention_weight_min": -26.977924346923828,
      "activations/layer3_attention_weight_max": 79.38033294677734,
      "activations/layer3_attention_weight_min": -79.61673736572266,
      "activations/layer4_attention_weight_max": 98.57620239257812,
      "activations/layer4_attention_weight_min": -90.82811737060547,
      "activations/layer5_attention_weight_max": 106.56986999511719,
      "activations/layer5_attention_weight_min": -94.63752746582031,
      "activations/layer6_attention_weight_max": 56.77281951904297,
      "activations/layer6_attention_weight_min": -51.80756378173828,
      "activations/layer7_attention_weight_max": 63.09144973754883,
      "activations/layer7_attention_weight_min": -63.189231872558594,
      "activations/layer8_attention_weight_max": 39.7713508605957,
      "activations/layer8_attention_weight_min": -41.47373962402344,
      "activations/layer9_attention_weight_max": 47.539085388183594,
      "activations/layer9_attention_weight_min": -47.347740173339844,
      "epoch": 4.35,
      "learning_rate": 0.00012319280303030303,
      "loss": 2.8547,
      "step": 74850
    },
    {
      "activations/layer0_attention_weight_max": 14.590110778808594,
      "activations/layer0_attention_weight_min": -14.321640014648438,
      "activations/layer10_attention_weight_max": 45.56675338745117,
      "activations/layer10_attention_weight_min": -36.32505416870117,
      "activations/layer11_attention_weight_max": 39.57820510864258,
      "activations/layer11_attention_weight_min": -27.069257736206055,
      "activations/layer12_attention_weight_max": 33.4644775390625,
      "activations/layer12_attention_weight_min": -29.329429626464844,
      "activations/layer13_attention_weight_max": 61.45273971557617,
      "activations/layer13_attention_weight_min": -35.42570495605469,
      "activations/layer14_attention_weight_max": 60.12674331665039,
      "activations/layer14_attention_weight_min": -30.900291442871094,
      "activations/layer15_attention_weight_max": 50.01701354980469,
      "activations/layer15_attention_weight_min": -33.292606353759766,
      "activations/layer16_attention_weight_max": 40.640289306640625,
      "activations/layer16_attention_weight_min": -26.389156341552734,
      "activations/layer17_attention_weight_max": 55.54616165161133,
      "activations/layer17_attention_weight_min": -37.94041442871094,
      "activations/layer18_attention_weight_max": 51.45125198364258,
      "activations/layer18_attention_weight_min": -32.90821838378906,
      "activations/layer19_attention_weight_max": 19.62437629699707,
      "activations/layer19_attention_weight_min": -19.6944580078125,
      "activations/layer1_attention_weight_max": 19.146135330200195,
      "activations/layer1_attention_weight_min": -17.625398635864258,
      "activations/layer20_attention_weight_max": 20.764158248901367,
      "activations/layer20_attention_weight_min": -19.711448669433594,
      "activations/layer21_attention_weight_max": 39.51702880859375,
      "activations/layer21_attention_weight_min": -26.044931411743164,
      "activations/layer22_attention_weight_max": 35.996734619140625,
      "activations/layer22_attention_weight_min": -22.755596160888672,
      "activations/layer23_attention_weight_max": 26.41725730895996,
      "activations/layer23_attention_weight_min": -19.975025177001953,
      "activations/layer2_attention_weight_max": 27.93970489501953,
      "activations/layer2_attention_weight_min": -27.19434928894043,
      "activations/layer3_attention_weight_max": 77.25485229492188,
      "activations/layer3_attention_weight_min": -78.82408905029297,
      "activations/layer4_attention_weight_max": 99.2259750366211,
      "activations/layer4_attention_weight_min": -94.49986267089844,
      "activations/layer5_attention_weight_max": 103.64260864257812,
      "activations/layer5_attention_weight_min": -103.68978881835938,
      "activations/layer6_attention_weight_max": 57.18815994262695,
      "activations/layer6_attention_weight_min": -56.36238098144531,
      "activations/layer7_attention_weight_max": 74.89442443847656,
      "activations/layer7_attention_weight_min": -63.342716217041016,
      "activations/layer8_attention_weight_max": 45.15584182739258,
      "activations/layer8_attention_weight_min": -42.018184661865234,
      "activations/layer9_attention_weight_max": 64.10726165771484,
      "activations/layer9_attention_weight_min": -49.61013412475586,
      "epoch": 4.35,
      "learning_rate": 0.00012317386363636362,
      "loss": 2.8868,
      "step": 74900
    },
    {
      "activations/layer0_attention_weight_max": 15.054993629455566,
      "activations/layer0_attention_weight_min": -14.470588684082031,
      "activations/layer10_attention_weight_max": 40.0609130859375,
      "activations/layer10_attention_weight_min": -33.660797119140625,
      "activations/layer11_attention_weight_max": 33.87706756591797,
      "activations/layer11_attention_weight_min": -27.438703536987305,
      "activations/layer12_attention_weight_max": 28.576906204223633,
      "activations/layer12_attention_weight_min": -25.136600494384766,
      "activations/layer13_attention_weight_max": 55.396461486816406,
      "activations/layer13_attention_weight_min": -39.154144287109375,
      "activations/layer14_attention_weight_max": 66.5754165649414,
      "activations/layer14_attention_weight_min": -38.953433990478516,
      "activations/layer15_attention_weight_max": 51.29237747192383,
      "activations/layer15_attention_weight_min": -32.12276077270508,
      "activations/layer16_attention_weight_max": 40.57046890258789,
      "activations/layer16_attention_weight_min": -25.806140899658203,
      "activations/layer17_attention_weight_max": 59.51377487182617,
      "activations/layer17_attention_weight_min": -39.64195251464844,
      "activations/layer18_attention_weight_max": 54.980560302734375,
      "activations/layer18_attention_weight_min": -32.166748046875,
      "activations/layer19_attention_weight_max": 22.76732063293457,
      "activations/layer19_attention_weight_min": -18.71388816833496,
      "activations/layer1_attention_weight_max": 17.75636863708496,
      "activations/layer1_attention_weight_min": -14.151382446289062,
      "activations/layer20_attention_weight_max": 22.6558895111084,
      "activations/layer20_attention_weight_min": -19.487485885620117,
      "activations/layer21_attention_weight_max": 43.967918395996094,
      "activations/layer21_attention_weight_min": -27.816930770874023,
      "activations/layer22_attention_weight_max": 41.024497985839844,
      "activations/layer22_attention_weight_min": -24.8475341796875,
      "activations/layer23_attention_weight_max": 29.154136657714844,
      "activations/layer23_attention_weight_min": -18.666397094726562,
      "activations/layer2_attention_weight_max": 30.163986206054688,
      "activations/layer2_attention_weight_min": -27.605314254760742,
      "activations/layer3_attention_weight_max": 83.38707733154297,
      "activations/layer3_attention_weight_min": -78.93930053710938,
      "activations/layer4_attention_weight_max": 99.7448959350586,
      "activations/layer4_attention_weight_min": -90.9262924194336,
      "activations/layer5_attention_weight_max": 105.14599609375,
      "activations/layer5_attention_weight_min": -95.46602630615234,
      "activations/layer6_attention_weight_max": 56.49098587036133,
      "activations/layer6_attention_weight_min": -50.96344757080078,
      "activations/layer7_attention_weight_max": 63.506649017333984,
      "activations/layer7_attention_weight_min": -66.37448120117188,
      "activations/layer8_attention_weight_max": 41.846710205078125,
      "activations/layer8_attention_weight_min": -39.13713455200195,
      "activations/layer9_attention_weight_max": 61.980079650878906,
      "activations/layer9_attention_weight_min": -51.20870590209961,
      "epoch": 4.36,
      "learning_rate": 0.00012315492424242423,
      "loss": 2.8757,
      "step": 74950
    },
    {
      "activations/layer0_attention_weight_max": 14.93138313293457,
      "activations/layer0_attention_weight_min": -14.66835880279541,
      "activations/layer10_attention_weight_max": 42.41916275024414,
      "activations/layer10_attention_weight_min": -32.9301643371582,
      "activations/layer11_attention_weight_max": 35.139556884765625,
      "activations/layer11_attention_weight_min": -25.777122497558594,
      "activations/layer12_attention_weight_max": 23.463436126708984,
      "activations/layer12_attention_weight_min": -26.83116340637207,
      "activations/layer13_attention_weight_max": 71.01628112792969,
      "activations/layer13_attention_weight_min": -38.843902587890625,
      "activations/layer14_attention_weight_max": 60.57436752319336,
      "activations/layer14_attention_weight_min": -30.98371696472168,
      "activations/layer15_attention_weight_max": 46.6408576965332,
      "activations/layer15_attention_weight_min": -32.289146423339844,
      "activations/layer16_attention_weight_max": 43.118980407714844,
      "activations/layer16_attention_weight_min": -26.47134780883789,
      "activations/layer17_attention_weight_max": 68.66544342041016,
      "activations/layer17_attention_weight_min": -39.22677993774414,
      "activations/layer18_attention_weight_max": 55.63002395629883,
      "activations/layer18_attention_weight_min": -33.372039794921875,
      "activations/layer19_attention_weight_max": 26.785993576049805,
      "activations/layer19_attention_weight_min": -19.607891082763672,
      "activations/layer1_attention_weight_max": 18.145370483398438,
      "activations/layer1_attention_weight_min": -17.024993896484375,
      "activations/layer20_attention_weight_max": 26.832962036132812,
      "activations/layer20_attention_weight_min": -18.251371383666992,
      "activations/layer21_attention_weight_max": 46.620361328125,
      "activations/layer21_attention_weight_min": -27.346328735351562,
      "activations/layer22_attention_weight_max": 36.75310516357422,
      "activations/layer22_attention_weight_min": -25.45223045349121,
      "activations/layer23_attention_weight_max": 29.654541015625,
      "activations/layer23_attention_weight_min": -16.99101448059082,
      "activations/layer2_attention_weight_max": 28.42806625366211,
      "activations/layer2_attention_weight_min": -26.87472915649414,
      "activations/layer3_attention_weight_max": 74.95775604248047,
      "activations/layer3_attention_weight_min": -78.44188690185547,
      "activations/layer4_attention_weight_max": 95.26555633544922,
      "activations/layer4_attention_weight_min": -93.6081771850586,
      "activations/layer5_attention_weight_max": 106.75340270996094,
      "activations/layer5_attention_weight_min": -93.09147644042969,
      "activations/layer6_attention_weight_max": 59.09217834472656,
      "activations/layer6_attention_weight_min": -51.48250961303711,
      "activations/layer7_attention_weight_max": 63.12110137939453,
      "activations/layer7_attention_weight_min": -61.13811492919922,
      "activations/layer8_attention_weight_max": 44.37855529785156,
      "activations/layer8_attention_weight_min": -41.34794616699219,
      "activations/layer9_attention_weight_max": 56.547523498535156,
      "activations/layer9_attention_weight_min": -46.543540954589844,
      "epoch": 4.36,
      "learning_rate": 0.00012313598484848482,
      "loss": 2.8556,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_loss": 2.818359375,
      "eval_runtime": 8.4817,
      "eval_samples_per_second": 506.265,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_openwebtext_loss": 2.818359375,
      "eval_openwebtext_ppl": 16.74934871787156,
      "eval_openwebtext_runtime": 8.4817,
      "eval_openwebtext_samples_per_second": 506.265,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_wikitext_loss": 3.052734375,
      "eval_wikitext_ppl": 21.173160702117308,
      "eval_wikitext_runtime": 1.9873,
      "eval_wikitext_samples_per_second": 229.458,
      "step": 75000
    },
    {
      "epoch": 4.36,
      "eval_lambada_loss": 2.97265625,
      "eval_lambada_ppl": 19.543763832601815,
      "eval_lambada_runtime": 9.5706,
      "eval_lambada_samples_per_second": 508.744,
      "step": 75000
    },
    {
      "activations/layer0_attention_weight_max": 15.10385513305664,
      "activations/layer0_attention_weight_min": -14.29433822631836,
      "activations/layer10_attention_weight_max": 44.45186996459961,
      "activations/layer10_attention_weight_min": -34.352779388427734,
      "activations/layer11_attention_weight_max": 36.19438934326172,
      "activations/layer11_attention_weight_min": -27.34364891052246,
      "activations/layer12_attention_weight_max": 22.966236114501953,
      "activations/layer12_attention_weight_min": -26.21964454650879,
      "activations/layer13_attention_weight_max": 59.56957244873047,
      "activations/layer13_attention_weight_min": -36.60645294189453,
      "activations/layer14_attention_weight_max": 63.282127380371094,
      "activations/layer14_attention_weight_min": -36.94575500488281,
      "activations/layer15_attention_weight_max": 48.43730163574219,
      "activations/layer15_attention_weight_min": -31.267669677734375,
      "activations/layer16_attention_weight_max": 37.505271911621094,
      "activations/layer16_attention_weight_min": -25.961997985839844,
      "activations/layer17_attention_weight_max": 57.09783172607422,
      "activations/layer17_attention_weight_min": -36.73353958129883,
      "activations/layer18_attention_weight_max": 48.72458267211914,
      "activations/layer18_attention_weight_min": -30.806631088256836,
      "activations/layer19_attention_weight_max": 22.33700942993164,
      "activations/layer19_attention_weight_min": -19.252500534057617,
      "activations/layer1_attention_weight_max": 17.515806198120117,
      "activations/layer1_attention_weight_min": -16.75396156311035,
      "activations/layer20_attention_weight_max": 22.296459197998047,
      "activations/layer20_attention_weight_min": -20.354591369628906,
      "activations/layer21_attention_weight_max": 43.250823974609375,
      "activations/layer21_attention_weight_min": -28.847002029418945,
      "activations/layer22_attention_weight_max": 33.643035888671875,
      "activations/layer22_attention_weight_min": -23.284015655517578,
      "activations/layer23_attention_weight_max": 24.66732406616211,
      "activations/layer23_attention_weight_min": -16.143945693969727,
      "activations/layer2_attention_weight_max": 32.460914611816406,
      "activations/layer2_attention_weight_min": -29.40069580078125,
      "activations/layer3_attention_weight_max": 85.03106689453125,
      "activations/layer3_attention_weight_min": -81.33995056152344,
      "activations/layer4_attention_weight_max": 101.55279541015625,
      "activations/layer4_attention_weight_min": -94.55037689208984,
      "activations/layer5_attention_weight_max": 108.68492126464844,
      "activations/layer5_attention_weight_min": -93.07213592529297,
      "activations/layer6_attention_weight_max": 59.78623962402344,
      "activations/layer6_attention_weight_min": -54.837196350097656,
      "activations/layer7_attention_weight_max": 72.58647918701172,
      "activations/layer7_attention_weight_min": -60.22049331665039,
      "activations/layer8_attention_weight_max": 49.528018951416016,
      "activations/layer8_attention_weight_min": -39.78205871582031,
      "activations/layer9_attention_weight_max": 62.88071823120117,
      "activations/layer9_attention_weight_min": -47.412017822265625,
      "epoch": 4.36,
      "learning_rate": 0.00012311704545454544,
      "loss": 2.8603,
      "step": 75050
    },
    {
      "activations/layer0_attention_weight_max": 13.887948036193848,
      "activations/layer0_attention_weight_min": -14.812617301940918,
      "activations/layer10_attention_weight_max": 36.94657897949219,
      "activations/layer10_attention_weight_min": -35.95607376098633,
      "activations/layer11_attention_weight_max": 30.214900970458984,
      "activations/layer11_attention_weight_min": -26.369564056396484,
      "activations/layer12_attention_weight_max": 22.17460823059082,
      "activations/layer12_attention_weight_min": -24.396347045898438,
      "activations/layer13_attention_weight_max": 48.12255096435547,
      "activations/layer13_attention_weight_min": -36.750221252441406,
      "activations/layer14_attention_weight_max": 42.155765533447266,
      "activations/layer14_attention_weight_min": -29.275638580322266,
      "activations/layer15_attention_weight_max": 44.09086227416992,
      "activations/layer15_attention_weight_min": -30.66026496887207,
      "activations/layer16_attention_weight_max": 35.52512741088867,
      "activations/layer16_attention_weight_min": -25.815637588500977,
      "activations/layer17_attention_weight_max": 55.19765853881836,
      "activations/layer17_attention_weight_min": -38.1331787109375,
      "activations/layer18_attention_weight_max": 50.05665969848633,
      "activations/layer18_attention_weight_min": -32.13804626464844,
      "activations/layer19_attention_weight_max": 22.784568786621094,
      "activations/layer19_attention_weight_min": -19.083045959472656,
      "activations/layer1_attention_weight_max": 17.97303009033203,
      "activations/layer1_attention_weight_min": -15.108036994934082,
      "activations/layer20_attention_weight_max": 23.986032485961914,
      "activations/layer20_attention_weight_min": -20.321250915527344,
      "activations/layer21_attention_weight_max": 39.23244094848633,
      "activations/layer21_attention_weight_min": -24.80357551574707,
      "activations/layer22_attention_weight_max": 33.41862869262695,
      "activations/layer22_attention_weight_min": -23.749603271484375,
      "activations/layer23_attention_weight_max": 25.140310287475586,
      "activations/layer23_attention_weight_min": -17.220685958862305,
      "activations/layer2_attention_weight_max": 29.22602081298828,
      "activations/layer2_attention_weight_min": -26.391536712646484,
      "activations/layer3_attention_weight_max": 78.55669403076172,
      "activations/layer3_attention_weight_min": -77.3919906616211,
      "activations/layer4_attention_weight_max": 102.36424255371094,
      "activations/layer4_attention_weight_min": -89.96556854248047,
      "activations/layer5_attention_weight_max": 106.67124938964844,
      "activations/layer5_attention_weight_min": -97.12062072753906,
      "activations/layer6_attention_weight_max": 61.7074089050293,
      "activations/layer6_attention_weight_min": -52.40318298339844,
      "activations/layer7_attention_weight_max": 70.0485610961914,
      "activations/layer7_attention_weight_min": -62.668235778808594,
      "activations/layer8_attention_weight_max": 42.1148567199707,
      "activations/layer8_attention_weight_min": -40.87239456176758,
      "activations/layer9_attention_weight_max": 51.79533767700195,
      "activations/layer9_attention_weight_min": -44.895816802978516,
      "epoch": 4.36,
      "learning_rate": 0.00012309810606060605,
      "loss": 2.8933,
      "step": 75100
    },
    {
      "activations/layer0_attention_weight_max": 14.168136596679688,
      "activations/layer0_attention_weight_min": -14.602983474731445,
      "activations/layer10_attention_weight_max": 40.129432678222656,
      "activations/layer10_attention_weight_min": -32.391178131103516,
      "activations/layer11_attention_weight_max": 32.244293212890625,
      "activations/layer11_attention_weight_min": -26.455501556396484,
      "activations/layer12_attention_weight_max": 25.757753372192383,
      "activations/layer12_attention_weight_min": -26.16085433959961,
      "activations/layer13_attention_weight_max": 58.00801086425781,
      "activations/layer13_attention_weight_min": -40.795082092285156,
      "activations/layer14_attention_weight_max": 64.60658264160156,
      "activations/layer14_attention_weight_min": -45.9693717956543,
      "activations/layer15_attention_weight_max": 53.6282958984375,
      "activations/layer15_attention_weight_min": -38.755096435546875,
      "activations/layer16_attention_weight_max": 35.734405517578125,
      "activations/layer16_attention_weight_min": -27.92012596130371,
      "activations/layer17_attention_weight_max": 55.956607818603516,
      "activations/layer17_attention_weight_min": -38.4694938659668,
      "activations/layer18_attention_weight_max": 50.43275833129883,
      "activations/layer18_attention_weight_min": -31.807331085205078,
      "activations/layer19_attention_weight_max": 21.114763259887695,
      "activations/layer19_attention_weight_min": -17.815858840942383,
      "activations/layer1_attention_weight_max": 19.20668601989746,
      "activations/layer1_attention_weight_min": -16.676706314086914,
      "activations/layer20_attention_weight_max": 22.751060485839844,
      "activations/layer20_attention_weight_min": -19.64646339416504,
      "activations/layer21_attention_weight_max": 42.64390563964844,
      "activations/layer21_attention_weight_min": -26.811111450195312,
      "activations/layer22_attention_weight_max": 39.65153121948242,
      "activations/layer22_attention_weight_min": -27.31869888305664,
      "activations/layer23_attention_weight_max": 29.922658920288086,
      "activations/layer23_attention_weight_min": -18.100698471069336,
      "activations/layer2_attention_weight_max": 31.244863510131836,
      "activations/layer2_attention_weight_min": -28.241985321044922,
      "activations/layer3_attention_weight_max": 77.86722564697266,
      "activations/layer3_attention_weight_min": -77.38571166992188,
      "activations/layer4_attention_weight_max": 99.73482513427734,
      "activations/layer4_attention_weight_min": -92.80636596679688,
      "activations/layer5_attention_weight_max": 105.8953857421875,
      "activations/layer5_attention_weight_min": -93.62745666503906,
      "activations/layer6_attention_weight_max": 60.8123664855957,
      "activations/layer6_attention_weight_min": -52.91954040527344,
      "activations/layer7_attention_weight_max": 66.06661224365234,
      "activations/layer7_attention_weight_min": -62.62419891357422,
      "activations/layer8_attention_weight_max": 41.07608413696289,
      "activations/layer8_attention_weight_min": -41.2917594909668,
      "activations/layer9_attention_weight_max": 52.717689514160156,
      "activations/layer9_attention_weight_min": -46.687461853027344,
      "epoch": 4.37,
      "learning_rate": 0.00012307954545454544,
      "loss": 2.8648,
      "step": 75150
    },
    {
      "activations/layer0_attention_weight_max": 15.890345573425293,
      "activations/layer0_attention_weight_min": -15.03483772277832,
      "activations/layer10_attention_weight_max": 37.775245666503906,
      "activations/layer10_attention_weight_min": -35.301658630371094,
      "activations/layer11_attention_weight_max": 30.08867645263672,
      "activations/layer11_attention_weight_min": -25.984241485595703,
      "activations/layer12_attention_weight_max": 24.007537841796875,
      "activations/layer12_attention_weight_min": -32.88361740112305,
      "activations/layer13_attention_weight_max": 53.2170524597168,
      "activations/layer13_attention_weight_min": -41.99334716796875,
      "activations/layer14_attention_weight_max": 39.55263900756836,
      "activations/layer14_attention_weight_min": -30.40296173095703,
      "activations/layer15_attention_weight_max": 38.467525482177734,
      "activations/layer15_attention_weight_min": -30.93709945678711,
      "activations/layer16_attention_weight_max": 36.24777603149414,
      "activations/layer16_attention_weight_min": -27.43744659423828,
      "activations/layer17_attention_weight_max": 52.884803771972656,
      "activations/layer17_attention_weight_min": -37.60567855834961,
      "activations/layer18_attention_weight_max": 49.14264678955078,
      "activations/layer18_attention_weight_min": -32.73359680175781,
      "activations/layer19_attention_weight_max": 19.8209285736084,
      "activations/layer19_attention_weight_min": -19.19970703125,
      "activations/layer1_attention_weight_max": 18.150470733642578,
      "activations/layer1_attention_weight_min": -16.319473266601562,
      "activations/layer20_attention_weight_max": 20.218547821044922,
      "activations/layer20_attention_weight_min": -22.990800857543945,
      "activations/layer21_attention_weight_max": 41.61912155151367,
      "activations/layer21_attention_weight_min": -28.21584701538086,
      "activations/layer22_attention_weight_max": 31.43327522277832,
      "activations/layer22_attention_weight_min": -23.672029495239258,
      "activations/layer23_attention_weight_max": 24.085826873779297,
      "activations/layer23_attention_weight_min": -16.047338485717773,
      "activations/layer2_attention_weight_max": 28.867794036865234,
      "activations/layer2_attention_weight_min": -26.77522087097168,
      "activations/layer3_attention_weight_max": 81.43233489990234,
      "activations/layer3_attention_weight_min": -83.08830261230469,
      "activations/layer4_attention_weight_max": 100.74491882324219,
      "activations/layer4_attention_weight_min": -93.06473541259766,
      "activations/layer5_attention_weight_max": 104.02601623535156,
      "activations/layer5_attention_weight_min": -91.72232055664062,
      "activations/layer6_attention_weight_max": 54.645694732666016,
      "activations/layer6_attention_weight_min": -49.832332611083984,
      "activations/layer7_attention_weight_max": 65.20397186279297,
      "activations/layer7_attention_weight_min": -61.41447830200195,
      "activations/layer8_attention_weight_max": 38.16671371459961,
      "activations/layer8_attention_weight_min": -39.03236389160156,
      "activations/layer9_attention_weight_max": 52.705482482910156,
      "activations/layer9_attention_weight_min": -44.89797592163086,
      "epoch": 4.37,
      "learning_rate": 0.00012306060606060606,
      "loss": 2.8589,
      "step": 75200
    },
    {
      "activations/layer0_attention_weight_max": 14.893417358398438,
      "activations/layer0_attention_weight_min": -15.017207145690918,
      "activations/layer10_attention_weight_max": 36.452362060546875,
      "activations/layer10_attention_weight_min": -32.171714782714844,
      "activations/layer11_attention_weight_max": 28.643489837646484,
      "activations/layer11_attention_weight_min": -25.523887634277344,
      "activations/layer12_attention_weight_max": 20.929176330566406,
      "activations/layer12_attention_weight_min": -26.284162521362305,
      "activations/layer13_attention_weight_max": 46.21944808959961,
      "activations/layer13_attention_weight_min": -41.16291046142578,
      "activations/layer14_attention_weight_max": 36.71675491333008,
      "activations/layer14_attention_weight_min": -34.2048225402832,
      "activations/layer15_attention_weight_max": 37.63796615600586,
      "activations/layer15_attention_weight_min": -30.036632537841797,
      "activations/layer16_attention_weight_max": 35.00973892211914,
      "activations/layer16_attention_weight_min": -31.318714141845703,
      "activations/layer17_attention_weight_max": 53.56852722167969,
      "activations/layer17_attention_weight_min": -37.82999038696289,
      "activations/layer18_attention_weight_max": 44.69522476196289,
      "activations/layer18_attention_weight_min": -31.184831619262695,
      "activations/layer19_attention_weight_max": 18.94256019592285,
      "activations/layer19_attention_weight_min": -17.795583724975586,
      "activations/layer1_attention_weight_max": 16.951335906982422,
      "activations/layer1_attention_weight_min": -15.17917251586914,
      "activations/layer20_attention_weight_max": 19.783851623535156,
      "activations/layer20_attention_weight_min": -18.5410099029541,
      "activations/layer21_attention_weight_max": 38.29058837890625,
      "activations/layer21_attention_weight_min": -28.640090942382812,
      "activations/layer22_attention_weight_max": 33.1131591796875,
      "activations/layer22_attention_weight_min": -23.002582550048828,
      "activations/layer23_attention_weight_max": 24.501880645751953,
      "activations/layer23_attention_weight_min": -17.68524169921875,
      "activations/layer2_attention_weight_max": 32.48485565185547,
      "activations/layer2_attention_weight_min": -29.777210235595703,
      "activations/layer3_attention_weight_max": 83.81982421875,
      "activations/layer3_attention_weight_min": -86.06851196289062,
      "activations/layer4_attention_weight_max": 100.60955047607422,
      "activations/layer4_attention_weight_min": -92.48014068603516,
      "activations/layer5_attention_weight_max": 100.54052734375,
      "activations/layer5_attention_weight_min": -95.24079895019531,
      "activations/layer6_attention_weight_max": 56.62561798095703,
      "activations/layer6_attention_weight_min": -54.55051040649414,
      "activations/layer7_attention_weight_max": 66.98787689208984,
      "activations/layer7_attention_weight_min": -63.0536994934082,
      "activations/layer8_attention_weight_max": 40.565208435058594,
      "activations/layer8_attention_weight_min": -40.17793655395508,
      "activations/layer9_attention_weight_max": 51.18647384643555,
      "activations/layer9_attention_weight_min": -51.24540328979492,
      "epoch": 4.37,
      "learning_rate": 0.00012304166666666664,
      "loss": 2.8725,
      "step": 75250
    },
    {
      "activations/layer0_attention_weight_max": 14.854694366455078,
      "activations/layer0_attention_weight_min": -14.746086120605469,
      "activations/layer10_attention_weight_max": 38.606300354003906,
      "activations/layer10_attention_weight_min": -34.7379264831543,
      "activations/layer11_attention_weight_max": 31.449689865112305,
      "activations/layer11_attention_weight_min": -28.25612449645996,
      "activations/layer12_attention_weight_max": 20.934167861938477,
      "activations/layer12_attention_weight_min": -24.195911407470703,
      "activations/layer13_attention_weight_max": 47.17066192626953,
      "activations/layer13_attention_weight_min": -37.8624153137207,
      "activations/layer14_attention_weight_max": 45.420387268066406,
      "activations/layer14_attention_weight_min": -33.142696380615234,
      "activations/layer15_attention_weight_max": 53.9768180847168,
      "activations/layer15_attention_weight_min": -37.759944915771484,
      "activations/layer16_attention_weight_max": 32.99720764160156,
      "activations/layer16_attention_weight_min": -27.97142219543457,
      "activations/layer17_attention_weight_max": 60.99604034423828,
      "activations/layer17_attention_weight_min": -39.528541564941406,
      "activations/layer18_attention_weight_max": 50.808895111083984,
      "activations/layer18_attention_weight_min": -34.97212600708008,
      "activations/layer19_attention_weight_max": 24.773662567138672,
      "activations/layer19_attention_weight_min": -17.77688217163086,
      "activations/layer1_attention_weight_max": 16.947553634643555,
      "activations/layer1_attention_weight_min": -15.497100830078125,
      "activations/layer20_attention_weight_max": 20.471235275268555,
      "activations/layer20_attention_weight_min": -20.853084564208984,
      "activations/layer21_attention_weight_max": 36.65003967285156,
      "activations/layer21_attention_weight_min": -26.99652671813965,
      "activations/layer22_attention_weight_max": 35.96443176269531,
      "activations/layer22_attention_weight_min": -22.191579818725586,
      "activations/layer23_attention_weight_max": 28.332820892333984,
      "activations/layer23_attention_weight_min": -17.313251495361328,
      "activations/layer2_attention_weight_max": 29.431636810302734,
      "activations/layer2_attention_weight_min": -28.871646881103516,
      "activations/layer3_attention_weight_max": 75.82160186767578,
      "activations/layer3_attention_weight_min": -79.53748321533203,
      "activations/layer4_attention_weight_max": 99.01173400878906,
      "activations/layer4_attention_weight_min": -90.6972427368164,
      "activations/layer5_attention_weight_max": 105.04263305664062,
      "activations/layer5_attention_weight_min": -93.27288818359375,
      "activations/layer6_attention_weight_max": 58.14493942260742,
      "activations/layer6_attention_weight_min": -51.20268630981445,
      "activations/layer7_attention_weight_max": 67.55164337158203,
      "activations/layer7_attention_weight_min": -61.20934295654297,
      "activations/layer8_attention_weight_max": 39.13389587402344,
      "activations/layer8_attention_weight_min": -38.37220001220703,
      "activations/layer9_attention_weight_max": 49.10697937011719,
      "activations/layer9_attention_weight_min": -45.638587951660156,
      "epoch": 4.38,
      "learning_rate": 0.00012302272727272726,
      "loss": 2.8799,
      "step": 75300
    },
    {
      "activations/layer0_attention_weight_max": 14.772594451904297,
      "activations/layer0_attention_weight_min": -14.7849760055542,
      "activations/layer10_attention_weight_max": 40.9589729309082,
      "activations/layer10_attention_weight_min": -35.85017013549805,
      "activations/layer11_attention_weight_max": 32.288963317871094,
      "activations/layer11_attention_weight_min": -26.76728057861328,
      "activations/layer12_attention_weight_max": 26.127649307250977,
      "activations/layer12_attention_weight_min": -24.894243240356445,
      "activations/layer13_attention_weight_max": 50.13364791870117,
      "activations/layer13_attention_weight_min": -35.48598861694336,
      "activations/layer14_attention_weight_max": 39.12643051147461,
      "activations/layer14_attention_weight_min": -28.625606536865234,
      "activations/layer15_attention_weight_max": 45.346458435058594,
      "activations/layer15_attention_weight_min": -32.63307571411133,
      "activations/layer16_attention_weight_max": 34.744083404541016,
      "activations/layer16_attention_weight_min": -27.041900634765625,
      "activations/layer17_attention_weight_max": 55.794456481933594,
      "activations/layer17_attention_weight_min": -36.72511672973633,
      "activations/layer18_attention_weight_max": 51.44534683227539,
      "activations/layer18_attention_weight_min": -32.241886138916016,
      "activations/layer19_attention_weight_max": 21.504011154174805,
      "activations/layer19_attention_weight_min": -16.473325729370117,
      "activations/layer1_attention_weight_max": 18.054872512817383,
      "activations/layer1_attention_weight_min": -14.78502082824707,
      "activations/layer20_attention_weight_max": 20.939443588256836,
      "activations/layer20_attention_weight_min": -17.345500946044922,
      "activations/layer21_attention_weight_max": 36.89167404174805,
      "activations/layer21_attention_weight_min": -24.30272674560547,
      "activations/layer22_attention_weight_max": 32.893951416015625,
      "activations/layer22_attention_weight_min": -23.4859561920166,
      "activations/layer23_attention_weight_max": 28.68743896484375,
      "activations/layer23_attention_weight_min": -18.66366958618164,
      "activations/layer2_attention_weight_max": 29.79964256286621,
      "activations/layer2_attention_weight_min": -27.518367767333984,
      "activations/layer3_attention_weight_max": 79.09002685546875,
      "activations/layer3_attention_weight_min": -82.7745361328125,
      "activations/layer4_attention_weight_max": 99.83492279052734,
      "activations/layer4_attention_weight_min": -96.52178955078125,
      "activations/layer5_attention_weight_max": 107.71646118164062,
      "activations/layer5_attention_weight_min": -94.6753158569336,
      "activations/layer6_attention_weight_max": 58.820396423339844,
      "activations/layer6_attention_weight_min": -52.9285774230957,
      "activations/layer7_attention_weight_max": 64.0958480834961,
      "activations/layer7_attention_weight_min": -61.61007308959961,
      "activations/layer8_attention_weight_max": 39.847381591796875,
      "activations/layer8_attention_weight_min": -38.95330810546875,
      "activations/layer9_attention_weight_max": 51.445648193359375,
      "activations/layer9_attention_weight_min": -47.745975494384766,
      "epoch": 4.38,
      "learning_rate": 0.00012300378787878787,
      "loss": 2.8692,
      "step": 75350
    },
    {
      "activations/layer0_attention_weight_max": 13.78510570526123,
      "activations/layer0_attention_weight_min": -14.702325820922852,
      "activations/layer10_attention_weight_max": 40.209686279296875,
      "activations/layer10_attention_weight_min": -32.9033088684082,
      "activations/layer11_attention_weight_max": 31.447917938232422,
      "activations/layer11_attention_weight_min": -24.972888946533203,
      "activations/layer12_attention_weight_max": 27.76999282836914,
      "activations/layer12_attention_weight_min": -25.57657241821289,
      "activations/layer13_attention_weight_max": 57.4129524230957,
      "activations/layer13_attention_weight_min": -37.32014083862305,
      "activations/layer14_attention_weight_max": 57.06387710571289,
      "activations/layer14_attention_weight_min": -35.69667053222656,
      "activations/layer15_attention_weight_max": 46.91844177246094,
      "activations/layer15_attention_weight_min": -30.124835968017578,
      "activations/layer16_attention_weight_max": 44.878665924072266,
      "activations/layer16_attention_weight_min": -27.08213996887207,
      "activations/layer17_attention_weight_max": 66.98094940185547,
      "activations/layer17_attention_weight_min": -37.988487243652344,
      "activations/layer18_attention_weight_max": 52.24289321899414,
      "activations/layer18_attention_weight_min": -33.315093994140625,
      "activations/layer19_attention_weight_max": 25.633691787719727,
      "activations/layer19_attention_weight_min": -19.431791305541992,
      "activations/layer1_attention_weight_max": 18.307971954345703,
      "activations/layer1_attention_weight_min": -15.492470741271973,
      "activations/layer20_attention_weight_max": 24.721479415893555,
      "activations/layer20_attention_weight_min": -22.355981826782227,
      "activations/layer21_attention_weight_max": 47.54921340942383,
      "activations/layer21_attention_weight_min": -28.98531150817871,
      "activations/layer22_attention_weight_max": 36.5464973449707,
      "activations/layer22_attention_weight_min": -24.215648651123047,
      "activations/layer23_attention_weight_max": 26.50201988220215,
      "activations/layer23_attention_weight_min": -18.43889045715332,
      "activations/layer2_attention_weight_max": 29.67562484741211,
      "activations/layer2_attention_weight_min": -28.295143127441406,
      "activations/layer3_attention_weight_max": 79.82954406738281,
      "activations/layer3_attention_weight_min": -84.51353454589844,
      "activations/layer4_attention_weight_max": 102.22775268554688,
      "activations/layer4_attention_weight_min": -92.84660339355469,
      "activations/layer5_attention_weight_max": 109.03612518310547,
      "activations/layer5_attention_weight_min": -93.60112762451172,
      "activations/layer6_attention_weight_max": 59.671661376953125,
      "activations/layer6_attention_weight_min": -52.7428092956543,
      "activations/layer7_attention_weight_max": 68.82820129394531,
      "activations/layer7_attention_weight_min": -62.59721755981445,
      "activations/layer8_attention_weight_max": 42.346927642822266,
      "activations/layer8_attention_weight_min": -39.74277877807617,
      "activations/layer9_attention_weight_max": 55.5313720703125,
      "activations/layer9_attention_weight_min": -43.249366760253906,
      "epoch": 4.38,
      "learning_rate": 0.00012298484848484846,
      "loss": 2.8762,
      "step": 75400
    },
    {
      "activations/layer0_attention_weight_max": 14.581233024597168,
      "activations/layer0_attention_weight_min": -14.90564250946045,
      "activations/layer10_attention_weight_max": 33.8662223815918,
      "activations/layer10_attention_weight_min": -33.54860305786133,
      "activations/layer11_attention_weight_max": 28.067142486572266,
      "activations/layer11_attention_weight_min": -24.652145385742188,
      "activations/layer12_attention_weight_max": 23.423898696899414,
      "activations/layer12_attention_weight_min": -25.798145294189453,
      "activations/layer13_attention_weight_max": 43.31727600097656,
      "activations/layer13_attention_weight_min": -36.44569778442383,
      "activations/layer14_attention_weight_max": 42.47001647949219,
      "activations/layer14_attention_weight_min": -31.8747501373291,
      "activations/layer15_attention_weight_max": 40.27687454223633,
      "activations/layer15_attention_weight_min": -32.44844055175781,
      "activations/layer16_attention_weight_max": 34.78801727294922,
      "activations/layer16_attention_weight_min": -28.46770477294922,
      "activations/layer17_attention_weight_max": 60.21343994140625,
      "activations/layer17_attention_weight_min": -36.54426574707031,
      "activations/layer18_attention_weight_max": 51.443382263183594,
      "activations/layer18_attention_weight_min": -32.92660140991211,
      "activations/layer19_attention_weight_max": 23.513826370239258,
      "activations/layer19_attention_weight_min": -16.729076385498047,
      "activations/layer1_attention_weight_max": 17.965116500854492,
      "activations/layer1_attention_weight_min": -14.12181282043457,
      "activations/layer20_attention_weight_max": 20.246362686157227,
      "activations/layer20_attention_weight_min": -21.206886291503906,
      "activations/layer21_attention_weight_max": 39.147125244140625,
      "activations/layer21_attention_weight_min": -26.579668045043945,
      "activations/layer22_attention_weight_max": 32.6035041809082,
      "activations/layer22_attention_weight_min": -24.70413589477539,
      "activations/layer23_attention_weight_max": 25.556074142456055,
      "activations/layer23_attention_weight_min": -17.539085388183594,
      "activations/layer2_attention_weight_max": 28.924293518066406,
      "activations/layer2_attention_weight_min": -26.845426559448242,
      "activations/layer3_attention_weight_max": 77.37696075439453,
      "activations/layer3_attention_weight_min": -78.4750747680664,
      "activations/layer4_attention_weight_max": 95.47799682617188,
      "activations/layer4_attention_weight_min": -90.86221313476562,
      "activations/layer5_attention_weight_max": 98.00846862792969,
      "activations/layer5_attention_weight_min": -90.88113403320312,
      "activations/layer6_attention_weight_max": 55.31244659423828,
      "activations/layer6_attention_weight_min": -50.85036087036133,
      "activations/layer7_attention_weight_max": 60.598899841308594,
      "activations/layer7_attention_weight_min": -60.91874313354492,
      "activations/layer8_attention_weight_max": 42.03825378417969,
      "activations/layer8_attention_weight_min": -36.242332458496094,
      "activations/layer9_attention_weight_max": 47.98446273803711,
      "activations/layer9_attention_weight_min": -47.941917419433594,
      "epoch": 4.38,
      "learning_rate": 0.00012296590909090908,
      "loss": 2.8769,
      "step": 75450
    },
    {
      "activations/layer0_attention_weight_max": 15.479626655578613,
      "activations/layer0_attention_weight_min": -15.223873138427734,
      "activations/layer10_attention_weight_max": 37.8929557800293,
      "activations/layer10_attention_weight_min": -32.674781799316406,
      "activations/layer11_attention_weight_max": 31.30067253112793,
      "activations/layer11_attention_weight_min": -26.020111083984375,
      "activations/layer12_attention_weight_max": 22.539331436157227,
      "activations/layer12_attention_weight_min": -25.23459815979004,
      "activations/layer13_attention_weight_max": 53.271183013916016,
      "activations/layer13_attention_weight_min": -35.985897064208984,
      "activations/layer14_attention_weight_max": 50.19367599487305,
      "activations/layer14_attention_weight_min": -30.509004592895508,
      "activations/layer15_attention_weight_max": 45.138526916503906,
      "activations/layer15_attention_weight_min": -32.3727912902832,
      "activations/layer16_attention_weight_max": 38.90607452392578,
      "activations/layer16_attention_weight_min": -27.430883407592773,
      "activations/layer17_attention_weight_max": 63.92366409301758,
      "activations/layer17_attention_weight_min": -40.32014083862305,
      "activations/layer18_attention_weight_max": 54.198822021484375,
      "activations/layer18_attention_weight_min": -34.283592224121094,
      "activations/layer19_attention_weight_max": 21.577198028564453,
      "activations/layer19_attention_weight_min": -19.528057098388672,
      "activations/layer1_attention_weight_max": 19.06293296813965,
      "activations/layer1_attention_weight_min": -16.04327964782715,
      "activations/layer20_attention_weight_max": 27.88605308532715,
      "activations/layer20_attention_weight_min": -18.46470832824707,
      "activations/layer21_attention_weight_max": 47.30502700805664,
      "activations/layer21_attention_weight_min": -31.07945442199707,
      "activations/layer22_attention_weight_max": 40.31219482421875,
      "activations/layer22_attention_weight_min": -23.909440994262695,
      "activations/layer23_attention_weight_max": 33.0692253112793,
      "activations/layer23_attention_weight_min": -17.44316864013672,
      "activations/layer2_attention_weight_max": 29.638118743896484,
      "activations/layer2_attention_weight_min": -27.80390167236328,
      "activations/layer3_attention_weight_max": 79.58879089355469,
      "activations/layer3_attention_weight_min": -83.34752655029297,
      "activations/layer4_attention_weight_max": 97.05223846435547,
      "activations/layer4_attention_weight_min": -94.97311401367188,
      "activations/layer5_attention_weight_max": 106.69847106933594,
      "activations/layer5_attention_weight_min": -92.56065368652344,
      "activations/layer6_attention_weight_max": 58.9692497253418,
      "activations/layer6_attention_weight_min": -50.960731506347656,
      "activations/layer7_attention_weight_max": 63.66459274291992,
      "activations/layer7_attention_weight_min": -61.74382400512695,
      "activations/layer8_attention_weight_max": 40.62321472167969,
      "activations/layer8_attention_weight_min": -38.37130355834961,
      "activations/layer9_attention_weight_max": 52.08244705200195,
      "activations/layer9_attention_weight_min": -47.86295700073242,
      "epoch": 4.39,
      "learning_rate": 0.0001229469696969697,
      "loss": 2.871,
      "step": 75500
    },
    {
      "activations/layer0_attention_weight_max": 14.30376148223877,
      "activations/layer0_attention_weight_min": -15.226715087890625,
      "activations/layer10_attention_weight_max": 40.1158447265625,
      "activations/layer10_attention_weight_min": -34.98046875,
      "activations/layer11_attention_weight_max": 35.161014556884766,
      "activations/layer11_attention_weight_min": -26.32331657409668,
      "activations/layer12_attention_weight_max": 38.848663330078125,
      "activations/layer12_attention_weight_min": -24.184309005737305,
      "activations/layer13_attention_weight_max": 50.212581634521484,
      "activations/layer13_attention_weight_min": -33.9057731628418,
      "activations/layer14_attention_weight_max": 43.99671173095703,
      "activations/layer14_attention_weight_min": -28.839984893798828,
      "activations/layer15_attention_weight_max": 47.558067321777344,
      "activations/layer15_attention_weight_min": -31.932804107666016,
      "activations/layer16_attention_weight_max": 35.844947814941406,
      "activations/layer16_attention_weight_min": -27.279979705810547,
      "activations/layer17_attention_weight_max": 60.92095184326172,
      "activations/layer17_attention_weight_min": -37.911109924316406,
      "activations/layer18_attention_weight_max": 52.45818328857422,
      "activations/layer18_attention_weight_min": -33.21811294555664,
      "activations/layer19_attention_weight_max": 28.565820693969727,
      "activations/layer19_attention_weight_min": -19.379467010498047,
      "activations/layer1_attention_weight_max": 17.430553436279297,
      "activations/layer1_attention_weight_min": -15.256355285644531,
      "activations/layer20_attention_weight_max": 21.439329147338867,
      "activations/layer20_attention_weight_min": -21.092449188232422,
      "activations/layer21_attention_weight_max": 42.99875259399414,
      "activations/layer21_attention_weight_min": -25.7619686126709,
      "activations/layer22_attention_weight_max": 41.38284683227539,
      "activations/layer22_attention_weight_min": -25.717405319213867,
      "activations/layer23_attention_weight_max": 32.514259338378906,
      "activations/layer23_attention_weight_min": -17.852102279663086,
      "activations/layer2_attention_weight_max": 31.927108764648438,
      "activations/layer2_attention_weight_min": -29.26600456237793,
      "activations/layer3_attention_weight_max": 80.62735748291016,
      "activations/layer3_attention_weight_min": -83.08849334716797,
      "activations/layer4_attention_weight_max": 98.36473846435547,
      "activations/layer4_attention_weight_min": -95.81118774414062,
      "activations/layer5_attention_weight_max": 106.96759033203125,
      "activations/layer5_attention_weight_min": -98.73341369628906,
      "activations/layer6_attention_weight_max": 59.39998245239258,
      "activations/layer6_attention_weight_min": -53.64896011352539,
      "activations/layer7_attention_weight_max": 63.25225067138672,
      "activations/layer7_attention_weight_min": -63.847042083740234,
      "activations/layer8_attention_weight_max": 43.376121520996094,
      "activations/layer8_attention_weight_min": -43.93031692504883,
      "activations/layer9_attention_weight_max": 49.07101821899414,
      "activations/layer9_attention_weight_min": -47.1170768737793,
      "epoch": 4.39,
      "learning_rate": 0.0001229280303030303,
      "loss": 2.8473,
      "step": 75550
    },
    {
      "activations/layer0_attention_weight_max": 13.34830379486084,
      "activations/layer0_attention_weight_min": -15.026650428771973,
      "activations/layer10_attention_weight_max": 39.179840087890625,
      "activations/layer10_attention_weight_min": -32.62077713012695,
      "activations/layer11_attention_weight_max": 32.201847076416016,
      "activations/layer11_attention_weight_min": -25.802505493164062,
      "activations/layer12_attention_weight_max": 22.584556579589844,
      "activations/layer12_attention_weight_min": -27.251916885375977,
      "activations/layer13_attention_weight_max": 54.14707946777344,
      "activations/layer13_attention_weight_min": -37.447811126708984,
      "activations/layer14_attention_weight_max": 43.598636627197266,
      "activations/layer14_attention_weight_min": -27.16534996032715,
      "activations/layer15_attention_weight_max": 43.57364273071289,
      "activations/layer15_attention_weight_min": -29.814559936523438,
      "activations/layer16_attention_weight_max": 36.2449836730957,
      "activations/layer16_attention_weight_min": -25.84446144104004,
      "activations/layer17_attention_weight_max": 68.9373550415039,
      "activations/layer17_attention_weight_min": -37.42866897583008,
      "activations/layer18_attention_weight_max": 57.22807312011719,
      "activations/layer18_attention_weight_min": -33.611358642578125,
      "activations/layer19_attention_weight_max": 25.7639102935791,
      "activations/layer19_attention_weight_min": -19.171253204345703,
      "activations/layer1_attention_weight_max": 18.0939998626709,
      "activations/layer1_attention_weight_min": -17.451135635375977,
      "activations/layer20_attention_weight_max": 26.4549503326416,
      "activations/layer20_attention_weight_min": -21.081422805786133,
      "activations/layer21_attention_weight_max": 41.519962310791016,
      "activations/layer21_attention_weight_min": -24.422616958618164,
      "activations/layer22_attention_weight_max": 41.188758850097656,
      "activations/layer22_attention_weight_min": -25.195722579956055,
      "activations/layer23_attention_weight_max": 34.05099868774414,
      "activations/layer23_attention_weight_min": -19.62565803527832,
      "activations/layer2_attention_weight_max": 32.66106414794922,
      "activations/layer2_attention_weight_min": -28.34368896484375,
      "activations/layer3_attention_weight_max": 78.08708190917969,
      "activations/layer3_attention_weight_min": -82.34869384765625,
      "activations/layer4_attention_weight_max": 99.06844329833984,
      "activations/layer4_attention_weight_min": -93.95610046386719,
      "activations/layer5_attention_weight_max": 104.409912109375,
      "activations/layer5_attention_weight_min": -96.0350341796875,
      "activations/layer6_attention_weight_max": 60.52558898925781,
      "activations/layer6_attention_weight_min": -54.95875930786133,
      "activations/layer7_attention_weight_max": 64.63116455078125,
      "activations/layer7_attention_weight_min": -68.05448913574219,
      "activations/layer8_attention_weight_max": 43.55302429199219,
      "activations/layer8_attention_weight_min": -44.45878601074219,
      "activations/layer9_attention_weight_max": 50.08576202392578,
      "activations/layer9_attention_weight_min": -47.6405029296875,
      "epoch": 4.39,
      "learning_rate": 0.0001229090909090909,
      "loss": 2.8866,
      "step": 75600
    },
    {
      "activations/layer0_attention_weight_max": 16.74688148498535,
      "activations/layer0_attention_weight_min": -14.949677467346191,
      "activations/layer10_attention_weight_max": 42.92346954345703,
      "activations/layer10_attention_weight_min": -38.472068786621094,
      "activations/layer11_attention_weight_max": 33.75654602050781,
      "activations/layer11_attention_weight_min": -28.66988754272461,
      "activations/layer12_attention_weight_max": 24.35063362121582,
      "activations/layer12_attention_weight_min": -23.870277404785156,
      "activations/layer13_attention_weight_max": 53.51382827758789,
      "activations/layer13_attention_weight_min": -37.84684371948242,
      "activations/layer14_attention_weight_max": 47.74869918823242,
      "activations/layer14_attention_weight_min": -33.11941146850586,
      "activations/layer15_attention_weight_max": 42.314273834228516,
      "activations/layer15_attention_weight_min": -31.58050537109375,
      "activations/layer16_attention_weight_max": 32.41438293457031,
      "activations/layer16_attention_weight_min": -27.357393264770508,
      "activations/layer17_attention_weight_max": 56.45228576660156,
      "activations/layer17_attention_weight_min": -36.1678466796875,
      "activations/layer18_attention_weight_max": 48.61381912231445,
      "activations/layer18_attention_weight_min": -31.927101135253906,
      "activations/layer19_attention_weight_max": 21.365684509277344,
      "activations/layer19_attention_weight_min": -21.247915267944336,
      "activations/layer1_attention_weight_max": 17.502315521240234,
      "activations/layer1_attention_weight_min": -16.088171005249023,
      "activations/layer20_attention_weight_max": 21.276151657104492,
      "activations/layer20_attention_weight_min": -21.85269546508789,
      "activations/layer21_attention_weight_max": 34.53481674194336,
      "activations/layer21_attention_weight_min": -25.34232521057129,
      "activations/layer22_attention_weight_max": 33.1711540222168,
      "activations/layer22_attention_weight_min": -22.56287956237793,
      "activations/layer23_attention_weight_max": 27.782257080078125,
      "activations/layer23_attention_weight_min": -18.0771427154541,
      "activations/layer2_attention_weight_max": 32.11318588256836,
      "activations/layer2_attention_weight_min": -29.455821990966797,
      "activations/layer3_attention_weight_max": 77.42879486083984,
      "activations/layer3_attention_weight_min": -77.93655395507812,
      "activations/layer4_attention_weight_max": 95.40467071533203,
      "activations/layer4_attention_weight_min": -89.3657455444336,
      "activations/layer5_attention_weight_max": 107.55733489990234,
      "activations/layer5_attention_weight_min": -97.45637512207031,
      "activations/layer6_attention_weight_max": 60.01372146606445,
      "activations/layer6_attention_weight_min": -54.817466735839844,
      "activations/layer7_attention_weight_max": 65.62390899658203,
      "activations/layer7_attention_weight_min": -63.49278259277344,
      "activations/layer8_attention_weight_max": 42.95205307006836,
      "activations/layer8_attention_weight_min": -40.01803207397461,
      "activations/layer9_attention_weight_max": 51.575042724609375,
      "activations/layer9_attention_weight_min": -46.658695220947266,
      "epoch": 4.4,
      "learning_rate": 0.0001228901515151515,
      "loss": 2.8708,
      "step": 75650
    },
    {
      "activations/layer0_attention_weight_max": 14.79260540008545,
      "activations/layer0_attention_weight_min": -14.897482872009277,
      "activations/layer10_attention_weight_max": 37.77632141113281,
      "activations/layer10_attention_weight_min": -32.74036407470703,
      "activations/layer11_attention_weight_max": 29.465368270874023,
      "activations/layer11_attention_weight_min": -25.789501190185547,
      "activations/layer12_attention_weight_max": 33.68119430541992,
      "activations/layer12_attention_weight_min": -22.96831512451172,
      "activations/layer13_attention_weight_max": 52.180580139160156,
      "activations/layer13_attention_weight_min": -35.9764404296875,
      "activations/layer14_attention_weight_max": 42.05483627319336,
      "activations/layer14_attention_weight_min": -30.965496063232422,
      "activations/layer15_attention_weight_max": 43.033695220947266,
      "activations/layer15_attention_weight_min": -29.548715591430664,
      "activations/layer16_attention_weight_max": 33.2444953918457,
      "activations/layer16_attention_weight_min": -26.014257431030273,
      "activations/layer17_attention_weight_max": 57.82984161376953,
      "activations/layer17_attention_weight_min": -37.643924713134766,
      "activations/layer18_attention_weight_max": 50.11752700805664,
      "activations/layer18_attention_weight_min": -32.217411041259766,
      "activations/layer19_attention_weight_max": 20.7903995513916,
      "activations/layer19_attention_weight_min": -19.618961334228516,
      "activations/layer1_attention_weight_max": 18.771591186523438,
      "activations/layer1_attention_weight_min": -17.219053268432617,
      "activations/layer20_attention_weight_max": 23.66510581970215,
      "activations/layer20_attention_weight_min": -20.966392517089844,
      "activations/layer21_attention_weight_max": 41.00275802612305,
      "activations/layer21_attention_weight_min": -27.923763275146484,
      "activations/layer22_attention_weight_max": 36.31519317626953,
      "activations/layer22_attention_weight_min": -24.635618209838867,
      "activations/layer23_attention_weight_max": 26.15748405456543,
      "activations/layer23_attention_weight_min": -17.493247985839844,
      "activations/layer2_attention_weight_max": 30.988670349121094,
      "activations/layer2_attention_weight_min": -29.027647018432617,
      "activations/layer3_attention_weight_max": 79.10050964355469,
      "activations/layer3_attention_weight_min": -84.65159606933594,
      "activations/layer4_attention_weight_max": 105.07286834716797,
      "activations/layer4_attention_weight_min": -95.75239562988281,
      "activations/layer5_attention_weight_max": 109.10838317871094,
      "activations/layer5_attention_weight_min": -98.0654296875,
      "activations/layer6_attention_weight_max": 57.86979675292969,
      "activations/layer6_attention_weight_min": -51.95473861694336,
      "activations/layer7_attention_weight_max": 67.48641967773438,
      "activations/layer7_attention_weight_min": -62.24278259277344,
      "activations/layer8_attention_weight_max": 41.843772888183594,
      "activations/layer8_attention_weight_min": -39.7025146484375,
      "activations/layer9_attention_weight_max": 47.6490478515625,
      "activations/layer9_attention_weight_min": -45.048824310302734,
      "epoch": 4.4,
      "learning_rate": 0.00012287121212121213,
      "loss": 2.8592,
      "step": 75700
    },
    {
      "activations/layer0_attention_weight_max": 14.445014953613281,
      "activations/layer0_attention_weight_min": -15.050657272338867,
      "activations/layer10_attention_weight_max": 40.94669723510742,
      "activations/layer10_attention_weight_min": -33.5072021484375,
      "activations/layer11_attention_weight_max": 32.87678146362305,
      "activations/layer11_attention_weight_min": -28.019607543945312,
      "activations/layer12_attention_weight_max": 32.75511932373047,
      "activations/layer12_attention_weight_min": -30.680706024169922,
      "activations/layer13_attention_weight_max": 51.71012878417969,
      "activations/layer13_attention_weight_min": -38.60457229614258,
      "activations/layer14_attention_weight_max": 53.48972702026367,
      "activations/layer14_attention_weight_min": -33.58415985107422,
      "activations/layer15_attention_weight_max": 45.896240234375,
      "activations/layer15_attention_weight_min": -30.937231063842773,
      "activations/layer16_attention_weight_max": 43.430152893066406,
      "activations/layer16_attention_weight_min": -28.701576232910156,
      "activations/layer17_attention_weight_max": 62.00368118286133,
      "activations/layer17_attention_weight_min": -37.43082046508789,
      "activations/layer18_attention_weight_max": 51.28605651855469,
      "activations/layer18_attention_weight_min": -33.43449020385742,
      "activations/layer19_attention_weight_max": 27.103063583374023,
      "activations/layer19_attention_weight_min": -17.533676147460938,
      "activations/layer1_attention_weight_max": 18.0941162109375,
      "activations/layer1_attention_weight_min": -15.335469245910645,
      "activations/layer20_attention_weight_max": 26.35723114013672,
      "activations/layer20_attention_weight_min": -20.123252868652344,
      "activations/layer21_attention_weight_max": 45.348445892333984,
      "activations/layer21_attention_weight_min": -26.14720344543457,
      "activations/layer22_attention_weight_max": 34.94361877441406,
      "activations/layer22_attention_weight_min": -24.799144744873047,
      "activations/layer23_attention_weight_max": 27.072509765625,
      "activations/layer23_attention_weight_min": -17.666149139404297,
      "activations/layer2_attention_weight_max": 31.058490753173828,
      "activations/layer2_attention_weight_min": -29.806739807128906,
      "activations/layer3_attention_weight_max": 81.14019012451172,
      "activations/layer3_attention_weight_min": -81.8884506225586,
      "activations/layer4_attention_weight_max": 101.01850891113281,
      "activations/layer4_attention_weight_min": -96.35550689697266,
      "activations/layer5_attention_weight_max": 104.82658386230469,
      "activations/layer5_attention_weight_min": -99.7607650756836,
      "activations/layer6_attention_weight_max": 57.4514274597168,
      "activations/layer6_attention_weight_min": -53.32545471191406,
      "activations/layer7_attention_weight_max": 64.4537582397461,
      "activations/layer7_attention_weight_min": -63.09733200073242,
      "activations/layer8_attention_weight_max": 42.325103759765625,
      "activations/layer8_attention_weight_min": -39.002647399902344,
      "activations/layer9_attention_weight_max": 52.40167236328125,
      "activations/layer9_attention_weight_min": -49.41344451904297,
      "epoch": 4.4,
      "learning_rate": 0.0001228522727272727,
      "loss": 2.864,
      "step": 75750
    },
    {
      "activations/layer0_attention_weight_max": 16.408172607421875,
      "activations/layer0_attention_weight_min": -14.641324996948242,
      "activations/layer10_attention_weight_max": 38.17132568359375,
      "activations/layer10_attention_weight_min": -35.87150192260742,
      "activations/layer11_attention_weight_max": 31.581809997558594,
      "activations/layer11_attention_weight_min": -25.571622848510742,
      "activations/layer12_attention_weight_max": 25.40687370300293,
      "activations/layer12_attention_weight_min": -29.362247467041016,
      "activations/layer13_attention_weight_max": 64.8457260131836,
      "activations/layer13_attention_weight_min": -35.528934478759766,
      "activations/layer14_attention_weight_max": 73.58820343017578,
      "activations/layer14_attention_weight_min": -40.89760971069336,
      "activations/layer15_attention_weight_max": 41.38801193237305,
      "activations/layer15_attention_weight_min": -29.777557373046875,
      "activations/layer16_attention_weight_max": 39.225830078125,
      "activations/layer16_attention_weight_min": -26.668352127075195,
      "activations/layer17_attention_weight_max": 59.04391860961914,
      "activations/layer17_attention_weight_min": -35.879188537597656,
      "activations/layer18_attention_weight_max": 50.33534240722656,
      "activations/layer18_attention_weight_min": -34.276451110839844,
      "activations/layer19_attention_weight_max": 20.402639389038086,
      "activations/layer19_attention_weight_min": -19.46478843688965,
      "activations/layer1_attention_weight_max": 19.902647018432617,
      "activations/layer1_attention_weight_min": -15.774206161499023,
      "activations/layer20_attention_weight_max": 21.09528923034668,
      "activations/layer20_attention_weight_min": -18.2654972076416,
      "activations/layer21_attention_weight_max": 40.95267868041992,
      "activations/layer21_attention_weight_min": -24.674503326416016,
      "activations/layer22_attention_weight_max": 37.93746566772461,
      "activations/layer22_attention_weight_min": -22.56049919128418,
      "activations/layer23_attention_weight_max": 26.476720809936523,
      "activations/layer23_attention_weight_min": -16.55031967163086,
      "activations/layer2_attention_weight_max": 30.163177490234375,
      "activations/layer2_attention_weight_min": -29.696060180664062,
      "activations/layer3_attention_weight_max": 79.03199768066406,
      "activations/layer3_attention_weight_min": -85.64127349853516,
      "activations/layer4_attention_weight_max": 97.9078140258789,
      "activations/layer4_attention_weight_min": -94.05045318603516,
      "activations/layer5_attention_weight_max": 106.90286254882812,
      "activations/layer5_attention_weight_min": -94.51104736328125,
      "activations/layer6_attention_weight_max": 57.864501953125,
      "activations/layer6_attention_weight_min": -54.389286041259766,
      "activations/layer7_attention_weight_max": 70.23007202148438,
      "activations/layer7_attention_weight_min": -62.56241226196289,
      "activations/layer8_attention_weight_max": 41.938236236572266,
      "activations/layer8_attention_weight_min": -41.930809020996094,
      "activations/layer9_attention_weight_max": 55.07809066772461,
      "activations/layer9_attention_weight_min": -48.239315032958984,
      "epoch": 4.4,
      "learning_rate": 0.00012283333333333333,
      "loss": 2.8522,
      "step": 75800
    },
    {
      "activations/layer0_attention_weight_max": 13.857094764709473,
      "activations/layer0_attention_weight_min": -14.932642936706543,
      "activations/layer10_attention_weight_max": 47.19406509399414,
      "activations/layer10_attention_weight_min": -38.67579650878906,
      "activations/layer11_attention_weight_max": 37.570587158203125,
      "activations/layer11_attention_weight_min": -29.073200225830078,
      "activations/layer12_attention_weight_max": 35.78068923950195,
      "activations/layer12_attention_weight_min": -24.5399112701416,
      "activations/layer13_attention_weight_max": 57.43428421020508,
      "activations/layer13_attention_weight_min": -37.49619674682617,
      "activations/layer14_attention_weight_max": 59.068904876708984,
      "activations/layer14_attention_weight_min": -39.86627960205078,
      "activations/layer15_attention_weight_max": 49.266788482666016,
      "activations/layer15_attention_weight_min": -29.50141716003418,
      "activations/layer16_attention_weight_max": 39.58869552612305,
      "activations/layer16_attention_weight_min": -25.34722328186035,
      "activations/layer17_attention_weight_max": 58.67045211791992,
      "activations/layer17_attention_weight_min": -38.16048049926758,
      "activations/layer18_attention_weight_max": 55.05976867675781,
      "activations/layer18_attention_weight_min": -31.907201766967773,
      "activations/layer19_attention_weight_max": 22.539403915405273,
      "activations/layer19_attention_weight_min": -18.756322860717773,
      "activations/layer1_attention_weight_max": 18.130611419677734,
      "activations/layer1_attention_weight_min": -17.94978904724121,
      "activations/layer20_attention_weight_max": 20.64655303955078,
      "activations/layer20_attention_weight_min": -17.03057861328125,
      "activations/layer21_attention_weight_max": 56.01205825805664,
      "activations/layer21_attention_weight_min": -26.81009864807129,
      "activations/layer22_attention_weight_max": 37.210079193115234,
      "activations/layer22_attention_weight_min": -22.01247787475586,
      "activations/layer23_attention_weight_max": 26.25054931640625,
      "activations/layer23_attention_weight_min": -16.604963302612305,
      "activations/layer2_attention_weight_max": 29.532270431518555,
      "activations/layer2_attention_weight_min": -28.263654708862305,
      "activations/layer3_attention_weight_max": 81.48302459716797,
      "activations/layer3_attention_weight_min": -82.3335952758789,
      "activations/layer4_attention_weight_max": 100.27405548095703,
      "activations/layer4_attention_weight_min": -97.34292602539062,
      "activations/layer5_attention_weight_max": 111.21772766113281,
      "activations/layer5_attention_weight_min": -99.18212890625,
      "activations/layer6_attention_weight_max": 60.551361083984375,
      "activations/layer6_attention_weight_min": -53.66187286376953,
      "activations/layer7_attention_weight_max": 74.06580352783203,
      "activations/layer7_attention_weight_min": -65.0400619506836,
      "activations/layer8_attention_weight_max": 47.777400970458984,
      "activations/layer8_attention_weight_min": -44.41633605957031,
      "activations/layer9_attention_weight_max": 62.849586486816406,
      "activations/layer9_attention_weight_min": -51.84562301635742,
      "epoch": 4.41,
      "learning_rate": 0.00012281439393939394,
      "loss": 2.8556,
      "step": 75850
    },
    {
      "activations/layer0_attention_weight_max": 16.26589584350586,
      "activations/layer0_attention_weight_min": -14.266895294189453,
      "activations/layer10_attention_weight_max": 36.93618392944336,
      "activations/layer10_attention_weight_min": -33.25944900512695,
      "activations/layer11_attention_weight_max": 30.025495529174805,
      "activations/layer11_attention_weight_min": -25.91727066040039,
      "activations/layer12_attention_weight_max": 22.062477111816406,
      "activations/layer12_attention_weight_min": -26.503992080688477,
      "activations/layer13_attention_weight_max": 49.37580871582031,
      "activations/layer13_attention_weight_min": -34.53981018066406,
      "activations/layer14_attention_weight_max": 43.9777946472168,
      "activations/layer14_attention_weight_min": -28.257097244262695,
      "activations/layer15_attention_weight_max": 40.0540771484375,
      "activations/layer15_attention_weight_min": -29.613683700561523,
      "activations/layer16_attention_weight_max": 31.191499710083008,
      "activations/layer16_attention_weight_min": -27.25785255432129,
      "activations/layer17_attention_weight_max": 61.30768585205078,
      "activations/layer17_attention_weight_min": -37.265071868896484,
      "activations/layer18_attention_weight_max": 48.50175857543945,
      "activations/layer18_attention_weight_min": -33.54854202270508,
      "activations/layer19_attention_weight_max": 22.469369888305664,
      "activations/layer19_attention_weight_min": -17.334108352661133,
      "activations/layer1_attention_weight_max": 19.35822868347168,
      "activations/layer1_attention_weight_min": -16.406354904174805,
      "activations/layer20_attention_weight_max": 21.917213439941406,
      "activations/layer20_attention_weight_min": -18.816133499145508,
      "activations/layer21_attention_weight_max": 44.256038665771484,
      "activations/layer21_attention_weight_min": -27.052892684936523,
      "activations/layer22_attention_weight_max": 33.86640548706055,
      "activations/layer22_attention_weight_min": -23.727815628051758,
      "activations/layer23_attention_weight_max": 29.34668731689453,
      "activations/layer23_attention_weight_min": -15.103323936462402,
      "activations/layer2_attention_weight_max": 31.719463348388672,
      "activations/layer2_attention_weight_min": -29.286558151245117,
      "activations/layer3_attention_weight_max": 86.44207763671875,
      "activations/layer3_attention_weight_min": -89.69417572021484,
      "activations/layer4_attention_weight_max": 103.14749908447266,
      "activations/layer4_attention_weight_min": -96.6962890625,
      "activations/layer5_attention_weight_max": 106.6488037109375,
      "activations/layer5_attention_weight_min": -96.41561889648438,
      "activations/layer6_attention_weight_max": 59.582557678222656,
      "activations/layer6_attention_weight_min": -55.294315338134766,
      "activations/layer7_attention_weight_max": 67.471923828125,
      "activations/layer7_attention_weight_min": -68.64226531982422,
      "activations/layer8_attention_weight_max": 39.73140335083008,
      "activations/layer8_attention_weight_min": -40.935550689697266,
      "activations/layer9_attention_weight_max": 57.64173126220703,
      "activations/layer9_attention_weight_min": -52.20692825317383,
      "epoch": 4.41,
      "learning_rate": 0.00012279545454545453,
      "loss": 2.8713,
      "step": 75900
    },
    {
      "activations/layer0_attention_weight_max": 14.011876106262207,
      "activations/layer0_attention_weight_min": -14.45142650604248,
      "activations/layer10_attention_weight_max": 39.44811248779297,
      "activations/layer10_attention_weight_min": -33.771549224853516,
      "activations/layer11_attention_weight_max": 31.774066925048828,
      "activations/layer11_attention_weight_min": -26.101104736328125,
      "activations/layer12_attention_weight_max": 27.048982620239258,
      "activations/layer12_attention_weight_min": -26.353757858276367,
      "activations/layer13_attention_weight_max": 55.6269416809082,
      "activations/layer13_attention_weight_min": -41.73575973510742,
      "activations/layer14_attention_weight_max": 52.23362350463867,
      "activations/layer14_attention_weight_min": -32.3623161315918,
      "activations/layer15_attention_weight_max": 46.53654098510742,
      "activations/layer15_attention_weight_min": -30.34756088256836,
      "activations/layer16_attention_weight_max": 35.06058883666992,
      "activations/layer16_attention_weight_min": -25.76768684387207,
      "activations/layer17_attention_weight_max": 59.15323257446289,
      "activations/layer17_attention_weight_min": -37.44307327270508,
      "activations/layer18_attention_weight_max": 51.290157318115234,
      "activations/layer18_attention_weight_min": -32.08924102783203,
      "activations/layer19_attention_weight_max": 21.96357536315918,
      "activations/layer19_attention_weight_min": -19.46950912475586,
      "activations/layer1_attention_weight_max": 18.79297637939453,
      "activations/layer1_attention_weight_min": -15.327651977539062,
      "activations/layer20_attention_weight_max": 22.82932472229004,
      "activations/layer20_attention_weight_min": -22.300983428955078,
      "activations/layer21_attention_weight_max": 39.73145294189453,
      "activations/layer21_attention_weight_min": -24.01517105102539,
      "activations/layer22_attention_weight_max": 35.246620178222656,
      "activations/layer22_attention_weight_min": -23.6038818359375,
      "activations/layer23_attention_weight_max": 27.542699813842773,
      "activations/layer23_attention_weight_min": -16.607181549072266,
      "activations/layer2_attention_weight_max": 30.41830825805664,
      "activations/layer2_attention_weight_min": -29.677677154541016,
      "activations/layer3_attention_weight_max": 78.84221649169922,
      "activations/layer3_attention_weight_min": -83.62838745117188,
      "activations/layer4_attention_weight_max": 99.33028411865234,
      "activations/layer4_attention_weight_min": -96.27659606933594,
      "activations/layer5_attention_weight_max": 106.92237854003906,
      "activations/layer5_attention_weight_min": -91.3614730834961,
      "activations/layer6_attention_weight_max": 57.35205078125,
      "activations/layer6_attention_weight_min": -52.55024337768555,
      "activations/layer7_attention_weight_max": 69.34056091308594,
      "activations/layer7_attention_weight_min": -63.28086853027344,
      "activations/layer8_attention_weight_max": 47.083309173583984,
      "activations/layer8_attention_weight_min": -38.8004264831543,
      "activations/layer9_attention_weight_max": 56.17402267456055,
      "activations/layer9_attention_weight_min": -45.1472053527832,
      "epoch": 4.41,
      "learning_rate": 0.00012277651515151515,
      "loss": 2.8631,
      "step": 75950
    },
    {
      "activations/layer0_attention_weight_max": 15.005066871643066,
      "activations/layer0_attention_weight_min": -15.07477855682373,
      "activations/layer10_attention_weight_max": 40.400516510009766,
      "activations/layer10_attention_weight_min": -33.31245803833008,
      "activations/layer11_attention_weight_max": 34.420989990234375,
      "activations/layer11_attention_weight_min": -26.674346923828125,
      "activations/layer12_attention_weight_max": 27.29886817932129,
      "activations/layer12_attention_weight_min": -28.484758377075195,
      "activations/layer13_attention_weight_max": 50.2111701965332,
      "activations/layer13_attention_weight_min": -36.09407043457031,
      "activations/layer14_attention_weight_max": 42.79436492919922,
      "activations/layer14_attention_weight_min": -33.300804138183594,
      "activations/layer15_attention_weight_max": 40.59625244140625,
      "activations/layer15_attention_weight_min": -32.65048599243164,
      "activations/layer16_attention_weight_max": 36.26685333251953,
      "activations/layer16_attention_weight_min": -27.916112899780273,
      "activations/layer17_attention_weight_max": 52.81104278564453,
      "activations/layer17_attention_weight_min": -36.786643981933594,
      "activations/layer18_attention_weight_max": 47.01863098144531,
      "activations/layer18_attention_weight_min": -30.880901336669922,
      "activations/layer19_attention_weight_max": 21.206613540649414,
      "activations/layer19_attention_weight_min": -17.451826095581055,
      "activations/layer1_attention_weight_max": 18.88722038269043,
      "activations/layer1_attention_weight_min": -15.544730186462402,
      "activations/layer20_attention_weight_max": 20.145103454589844,
      "activations/layer20_attention_weight_min": -21.36427879333496,
      "activations/layer21_attention_weight_max": 38.002220153808594,
      "activations/layer21_attention_weight_min": -25.844640731811523,
      "activations/layer22_attention_weight_max": 34.38115310668945,
      "activations/layer22_attention_weight_min": -24.87325096130371,
      "activations/layer23_attention_weight_max": 25.57921600341797,
      "activations/layer23_attention_weight_min": -16.623672485351562,
      "activations/layer2_attention_weight_max": 30.8872127532959,
      "activations/layer2_attention_weight_min": -27.1683349609375,
      "activations/layer3_attention_weight_max": 80.47769927978516,
      "activations/layer3_attention_weight_min": -78.68118286132812,
      "activations/layer4_attention_weight_max": 99.12660217285156,
      "activations/layer4_attention_weight_min": -91.89520263671875,
      "activations/layer5_attention_weight_max": 109.54113006591797,
      "activations/layer5_attention_weight_min": -91.154296875,
      "activations/layer6_attention_weight_max": 58.82950973510742,
      "activations/layer6_attention_weight_min": -51.89320755004883,
      "activations/layer7_attention_weight_max": 70.1614761352539,
      "activations/layer7_attention_weight_min": -63.16292953491211,
      "activations/layer8_attention_weight_max": 44.29843521118164,
      "activations/layer8_attention_weight_min": -41.52219772338867,
      "activations/layer9_attention_weight_max": 54.55609130859375,
      "activations/layer9_attention_weight_min": -50.312225341796875,
      "epoch": 4.42,
      "learning_rate": 0.00012275757575757576,
      "loss": 2.8784,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_loss": 2.814453125,
      "eval_runtime": 8.468,
      "eval_samples_per_second": 507.085,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_openwebtext_loss": 2.814453125,
      "eval_openwebtext_ppl": 16.684049195604416,
      "eval_openwebtext_runtime": 8.468,
      "eval_openwebtext_samples_per_second": 507.085,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_wikitext_loss": 3.060546875,
      "eval_wikitext_ppl": 21.33922385967176,
      "eval_wikitext_runtime": 1.9389,
      "eval_wikitext_samples_per_second": 235.189,
      "step": 76000
    },
    {
      "epoch": 4.42,
      "eval_lambada_loss": 2.8828125,
      "eval_lambada_ppl": 17.864446345871784,
      "eval_lambada_runtime": 9.5453,
      "eval_lambada_samples_per_second": 510.095,
      "step": 76000
    },
    {
      "activations/layer0_attention_weight_max": 14.501704216003418,
      "activations/layer0_attention_weight_min": -14.477757453918457,
      "activations/layer10_attention_weight_max": 40.30451965332031,
      "activations/layer10_attention_weight_min": -33.9686279296875,
      "activations/layer11_attention_weight_max": 29.726688385009766,
      "activations/layer11_attention_weight_min": -26.031986236572266,
      "activations/layer12_attention_weight_max": 29.150562286376953,
      "activations/layer12_attention_weight_min": -28.343257904052734,
      "activations/layer13_attention_weight_max": 56.03032684326172,
      "activations/layer13_attention_weight_min": -36.18126678466797,
      "activations/layer14_attention_weight_max": 49.55606460571289,
      "activations/layer14_attention_weight_min": -32.80085372924805,
      "activations/layer15_attention_weight_max": 40.6501350402832,
      "activations/layer15_attention_weight_min": -29.572351455688477,
      "activations/layer16_attention_weight_max": 37.93278503417969,
      "activations/layer16_attention_weight_min": -26.192874908447266,
      "activations/layer17_attention_weight_max": 59.88744354248047,
      "activations/layer17_attention_weight_min": -39.25592803955078,
      "activations/layer18_attention_weight_max": 51.52204132080078,
      "activations/layer18_attention_weight_min": -35.73786544799805,
      "activations/layer19_attention_weight_max": 22.119123458862305,
      "activations/layer19_attention_weight_min": -18.5496826171875,
      "activations/layer1_attention_weight_max": 18.13888168334961,
      "activations/layer1_attention_weight_min": -16.57755470275879,
      "activations/layer20_attention_weight_max": 20.731502532958984,
      "activations/layer20_attention_weight_min": -17.32598114013672,
      "activations/layer21_attention_weight_max": 37.569496154785156,
      "activations/layer21_attention_weight_min": -29.402517318725586,
      "activations/layer22_attention_weight_max": 37.526947021484375,
      "activations/layer22_attention_weight_min": -23.98971176147461,
      "activations/layer23_attention_weight_max": 26.052967071533203,
      "activations/layer23_attention_weight_min": -17.846132278442383,
      "activations/layer2_attention_weight_max": 33.2271728515625,
      "activations/layer2_attention_weight_min": -29.935447692871094,
      "activations/layer3_attention_weight_max": 89.13614654541016,
      "activations/layer3_attention_weight_min": -90.71772766113281,
      "activations/layer4_attention_weight_max": 115.45745086669922,
      "activations/layer4_attention_weight_min": -106.26252746582031,
      "activations/layer5_attention_weight_max": 119.29207611083984,
      "activations/layer5_attention_weight_min": -107.88291931152344,
      "activations/layer6_attention_weight_max": 61.61787033081055,
      "activations/layer6_attention_weight_min": -56.992679595947266,
      "activations/layer7_attention_weight_max": 69.86589050292969,
      "activations/layer7_attention_weight_min": -64.32901763916016,
      "activations/layer8_attention_weight_max": 41.69843673706055,
      "activations/layer8_attention_weight_min": -40.92180252075195,
      "activations/layer9_attention_weight_max": 50.933406829833984,
      "activations/layer9_attention_weight_min": -47.070274353027344,
      "epoch": 4.42,
      "learning_rate": 0.00012273863636363635,
      "loss": 2.8633,
      "step": 76050
    },
    {
      "activations/layer0_attention_weight_max": 14.591912269592285,
      "activations/layer0_attention_weight_min": -14.695110321044922,
      "activations/layer10_attention_weight_max": 37.782257080078125,
      "activations/layer10_attention_weight_min": -33.574378967285156,
      "activations/layer11_attention_weight_max": 28.08672332763672,
      "activations/layer11_attention_weight_min": -26.916479110717773,
      "activations/layer12_attention_weight_max": 23.520553588867188,
      "activations/layer12_attention_weight_min": -25.752071380615234,
      "activations/layer13_attention_weight_max": 51.208839416503906,
      "activations/layer13_attention_weight_min": -38.2139778137207,
      "activations/layer14_attention_weight_max": 42.62675476074219,
      "activations/layer14_attention_weight_min": -29.662609100341797,
      "activations/layer15_attention_weight_max": 42.8625602722168,
      "activations/layer15_attention_weight_min": -32.828121185302734,
      "activations/layer16_attention_weight_max": 40.055057525634766,
      "activations/layer16_attention_weight_min": -25.831314086914062,
      "activations/layer17_attention_weight_max": 57.50537109375,
      "activations/layer17_attention_weight_min": -35.57460403442383,
      "activations/layer18_attention_weight_max": 45.33627700805664,
      "activations/layer18_attention_weight_min": -32.82721710205078,
      "activations/layer19_attention_weight_max": 22.8834285736084,
      "activations/layer19_attention_weight_min": -18.585527420043945,
      "activations/layer1_attention_weight_max": 17.43167495727539,
      "activations/layer1_attention_weight_min": -15.755433082580566,
      "activations/layer20_attention_weight_max": 20.466894149780273,
      "activations/layer20_attention_weight_min": -22.320953369140625,
      "activations/layer21_attention_weight_max": 36.15662384033203,
      "activations/layer21_attention_weight_min": -26.57591438293457,
      "activations/layer22_attention_weight_max": 34.491512298583984,
      "activations/layer22_attention_weight_min": -23.563114166259766,
      "activations/layer23_attention_weight_max": 24.500255584716797,
      "activations/layer23_attention_weight_min": -19.06096839904785,
      "activations/layer2_attention_weight_max": 30.56692123413086,
      "activations/layer2_attention_weight_min": -28.043373107910156,
      "activations/layer3_attention_weight_max": 79.67306518554688,
      "activations/layer3_attention_weight_min": -81.21404266357422,
      "activations/layer4_attention_weight_max": 99.34336853027344,
      "activations/layer4_attention_weight_min": -88.6152572631836,
      "activations/layer5_attention_weight_max": 109.97512817382812,
      "activations/layer5_attention_weight_min": -94.87725830078125,
      "activations/layer6_attention_weight_max": 58.817161560058594,
      "activations/layer6_attention_weight_min": -53.3155632019043,
      "activations/layer7_attention_weight_max": 64.63070678710938,
      "activations/layer7_attention_weight_min": -61.7966423034668,
      "activations/layer8_attention_weight_max": 40.525447845458984,
      "activations/layer8_attention_weight_min": -39.41648483276367,
      "activations/layer9_attention_weight_max": 47.358245849609375,
      "activations/layer9_attention_weight_min": -47.022865295410156,
      "epoch": 4.42,
      "learning_rate": 0.00012271969696969696,
      "loss": 2.8486,
      "step": 76100
    },
    {
      "activations/layer0_attention_weight_max": 14.144392967224121,
      "activations/layer0_attention_weight_min": -15.041767120361328,
      "activations/layer10_attention_weight_max": 42.04762649536133,
      "activations/layer10_attention_weight_min": -34.95137023925781,
      "activations/layer11_attention_weight_max": 31.89889907836914,
      "activations/layer11_attention_weight_min": -26.582351684570312,
      "activations/layer12_attention_weight_max": 24.601890563964844,
      "activations/layer12_attention_weight_min": -26.754899978637695,
      "activations/layer13_attention_weight_max": 57.255332946777344,
      "activations/layer13_attention_weight_min": -35.435386657714844,
      "activations/layer14_attention_weight_max": 56.470767974853516,
      "activations/layer14_attention_weight_min": -34.5455436706543,
      "activations/layer15_attention_weight_max": 46.54336166381836,
      "activations/layer15_attention_weight_min": -31.502304077148438,
      "activations/layer16_attention_weight_max": 41.72858428955078,
      "activations/layer16_attention_weight_min": -25.34855842590332,
      "activations/layer17_attention_weight_max": 62.03004837036133,
      "activations/layer17_attention_weight_min": -36.830711364746094,
      "activations/layer18_attention_weight_max": 55.09926223754883,
      "activations/layer18_attention_weight_min": -31.618425369262695,
      "activations/layer19_attention_weight_max": 23.393678665161133,
      "activations/layer19_attention_weight_min": -19.166492462158203,
      "activations/layer1_attention_weight_max": 18.295820236206055,
      "activations/layer1_attention_weight_min": -15.224498748779297,
      "activations/layer20_attention_weight_max": 23.74738121032715,
      "activations/layer20_attention_weight_min": -20.955833435058594,
      "activations/layer21_attention_weight_max": 42.48347854614258,
      "activations/layer21_attention_weight_min": -25.563091278076172,
      "activations/layer22_attention_weight_max": 40.34575271606445,
      "activations/layer22_attention_weight_min": -23.0736141204834,
      "activations/layer23_attention_weight_max": 25.94373893737793,
      "activations/layer23_attention_weight_min": -17.77037811279297,
      "activations/layer2_attention_weight_max": 29.21771240234375,
      "activations/layer2_attention_weight_min": -27.54537010192871,
      "activations/layer3_attention_weight_max": 78.85932159423828,
      "activations/layer3_attention_weight_min": -81.8626480102539,
      "activations/layer4_attention_weight_max": 101.2577133178711,
      "activations/layer4_attention_weight_min": -104.3053970336914,
      "activations/layer5_attention_weight_max": 112.41996765136719,
      "activations/layer5_attention_weight_min": -98.33196258544922,
      "activations/layer6_attention_weight_max": 61.65496063232422,
      "activations/layer6_attention_weight_min": -52.34053039550781,
      "activations/layer7_attention_weight_max": 73.58087158203125,
      "activations/layer7_attention_weight_min": -64.00530242919922,
      "activations/layer8_attention_weight_max": 46.218971252441406,
      "activations/layer8_attention_weight_min": -41.072052001953125,
      "activations/layer9_attention_weight_max": 59.7658805847168,
      "activations/layer9_attention_weight_min": -43.80596160888672,
      "epoch": 4.42,
      "learning_rate": 0.00012270075757575755,
      "loss": 2.863,
      "step": 76150
    },
    {
      "activations/layer0_attention_weight_max": 16.115812301635742,
      "activations/layer0_attention_weight_min": -15.411246299743652,
      "activations/layer10_attention_weight_max": 42.84498596191406,
      "activations/layer10_attention_weight_min": -35.55519485473633,
      "activations/layer11_attention_weight_max": 38.37577819824219,
      "activations/layer11_attention_weight_min": -28.971939086914062,
      "activations/layer12_attention_weight_max": 25.632949829101562,
      "activations/layer12_attention_weight_min": -27.454360961914062,
      "activations/layer13_attention_weight_max": 52.91958236694336,
      "activations/layer13_attention_weight_min": -39.294593811035156,
      "activations/layer14_attention_weight_max": 64.35740661621094,
      "activations/layer14_attention_weight_min": -33.803680419921875,
      "activations/layer15_attention_weight_max": 46.06855773925781,
      "activations/layer15_attention_weight_min": -32.10607147216797,
      "activations/layer16_attention_weight_max": 37.974769592285156,
      "activations/layer16_attention_weight_min": -24.862350463867188,
      "activations/layer17_attention_weight_max": 61.13163375854492,
      "activations/layer17_attention_weight_min": -37.4237174987793,
      "activations/layer18_attention_weight_max": 55.2383918762207,
      "activations/layer18_attention_weight_min": -32.6993522644043,
      "activations/layer19_attention_weight_max": 23.468036651611328,
      "activations/layer19_attention_weight_min": -21.01392364501953,
      "activations/layer1_attention_weight_max": 18.072132110595703,
      "activations/layer1_attention_weight_min": -16.158884048461914,
      "activations/layer20_attention_weight_max": 27.964916229248047,
      "activations/layer20_attention_weight_min": -20.78373146057129,
      "activations/layer21_attention_weight_max": 48.45343017578125,
      "activations/layer21_attention_weight_min": -25.04425048828125,
      "activations/layer22_attention_weight_max": 39.278907775878906,
      "activations/layer22_attention_weight_min": -24.52436065673828,
      "activations/layer23_attention_weight_max": 30.810203552246094,
      "activations/layer23_attention_weight_min": -17.73012924194336,
      "activations/layer2_attention_weight_max": 34.73556900024414,
      "activations/layer2_attention_weight_min": -31.395124435424805,
      "activations/layer3_attention_weight_max": 83.86083221435547,
      "activations/layer3_attention_weight_min": -88.56531524658203,
      "activations/layer4_attention_weight_max": 103.67049407958984,
      "activations/layer4_attention_weight_min": -97.97830200195312,
      "activations/layer5_attention_weight_max": 109.86714172363281,
      "activations/layer5_attention_weight_min": -96.60679626464844,
      "activations/layer6_attention_weight_max": 58.990665435791016,
      "activations/layer6_attention_weight_min": -53.41646957397461,
      "activations/layer7_attention_weight_max": 87.60618591308594,
      "activations/layer7_attention_weight_min": -75.89716339111328,
      "activations/layer8_attention_weight_max": 51.4607048034668,
      "activations/layer8_attention_weight_min": -40.49216842651367,
      "activations/layer9_attention_weight_max": 72.6958999633789,
      "activations/layer9_attention_weight_min": -61.3501091003418,
      "epoch": 4.43,
      "learning_rate": 0.00012268181818181817,
      "loss": 2.8597,
      "step": 76200
    },
    {
      "activations/layer0_attention_weight_max": 15.851037979125977,
      "activations/layer0_attention_weight_min": -14.7407808303833,
      "activations/layer10_attention_weight_max": 39.579132080078125,
      "activations/layer10_attention_weight_min": -32.76978302001953,
      "activations/layer11_attention_weight_max": 31.573448181152344,
      "activations/layer11_attention_weight_min": -27.003751754760742,
      "activations/layer12_attention_weight_max": 25.54774284362793,
      "activations/layer12_attention_weight_min": -25.324247360229492,
      "activations/layer13_attention_weight_max": 45.343135833740234,
      "activations/layer13_attention_weight_min": -34.390682220458984,
      "activations/layer14_attention_weight_max": 40.25151443481445,
      "activations/layer14_attention_weight_min": -28.625547409057617,
      "activations/layer15_attention_weight_max": 42.71430587768555,
      "activations/layer15_attention_weight_min": -29.06950569152832,
      "activations/layer16_attention_weight_max": 35.06805419921875,
      "activations/layer16_attention_weight_min": -24.648103713989258,
      "activations/layer17_attention_weight_max": 58.69977569580078,
      "activations/layer17_attention_weight_min": -35.95067596435547,
      "activations/layer18_attention_weight_max": 47.79463577270508,
      "activations/layer18_attention_weight_min": -31.954992294311523,
      "activations/layer19_attention_weight_max": 21.254173278808594,
      "activations/layer19_attention_weight_min": -18.864704132080078,
      "activations/layer1_attention_weight_max": 17.52411460876465,
      "activations/layer1_attention_weight_min": -16.488479614257812,
      "activations/layer20_attention_weight_max": 22.813261032104492,
      "activations/layer20_attention_weight_min": -21.879785537719727,
      "activations/layer21_attention_weight_max": 46.680633544921875,
      "activations/layer21_attention_weight_min": -26.457670211791992,
      "activations/layer22_attention_weight_max": 33.87434387207031,
      "activations/layer22_attention_weight_min": -23.942691802978516,
      "activations/layer23_attention_weight_max": 28.15591049194336,
      "activations/layer23_attention_weight_min": -18.88658905029297,
      "activations/layer2_attention_weight_max": 30.166677474975586,
      "activations/layer2_attention_weight_min": -27.41301727294922,
      "activations/layer3_attention_weight_max": 75.69340515136719,
      "activations/layer3_attention_weight_min": -76.69344329833984,
      "activations/layer4_attention_weight_max": 100.87577056884766,
      "activations/layer4_attention_weight_min": -90.65094757080078,
      "activations/layer5_attention_weight_max": 103.2996826171875,
      "activations/layer5_attention_weight_min": -97.24120330810547,
      "activations/layer6_attention_weight_max": 58.1996955871582,
      "activations/layer6_attention_weight_min": -52.91062927246094,
      "activations/layer7_attention_weight_max": 65.8355484008789,
      "activations/layer7_attention_weight_min": -61.03864288330078,
      "activations/layer8_attention_weight_max": 42.24863815307617,
      "activations/layer8_attention_weight_min": -41.49752426147461,
      "activations/layer9_attention_weight_max": 49.43223190307617,
      "activations/layer9_attention_weight_min": -45.25539016723633,
      "epoch": 4.43,
      "learning_rate": 0.00012266287878787878,
      "loss": 2.8564,
      "step": 76250
    },
    {
      "activations/layer0_attention_weight_max": 14.430826187133789,
      "activations/layer0_attention_weight_min": -14.97092342376709,
      "activations/layer10_attention_weight_max": 37.24150085449219,
      "activations/layer10_attention_weight_min": -35.47126770019531,
      "activations/layer11_attention_weight_max": 27.91683578491211,
      "activations/layer11_attention_weight_min": -26.22423553466797,
      "activations/layer12_attention_weight_max": 21.7088565826416,
      "activations/layer12_attention_weight_min": -23.27873420715332,
      "activations/layer13_attention_weight_max": 44.66042709350586,
      "activations/layer13_attention_weight_min": -34.80967330932617,
      "activations/layer14_attention_weight_max": 42.5534782409668,
      "activations/layer14_attention_weight_min": -30.672271728515625,
      "activations/layer15_attention_weight_max": 40.83820724487305,
      "activations/layer15_attention_weight_min": -29.923961639404297,
      "activations/layer16_attention_weight_max": 37.405635833740234,
      "activations/layer16_attention_weight_min": -27.165569305419922,
      "activations/layer17_attention_weight_max": 58.32540512084961,
      "activations/layer17_attention_weight_min": -38.7016716003418,
      "activations/layer18_attention_weight_max": 50.34943389892578,
      "activations/layer18_attention_weight_min": -34.49872589111328,
      "activations/layer19_attention_weight_max": 23.057523727416992,
      "activations/layer19_attention_weight_min": -19.74708366394043,
      "activations/layer1_attention_weight_max": 19.061391830444336,
      "activations/layer1_attention_weight_min": -14.812590599060059,
      "activations/layer20_attention_weight_max": 20.517547607421875,
      "activations/layer20_attention_weight_min": -21.538896560668945,
      "activations/layer21_attention_weight_max": 39.18412780761719,
      "activations/layer21_attention_weight_min": -27.986669540405273,
      "activations/layer22_attention_weight_max": 35.4243049621582,
      "activations/layer22_attention_weight_min": -24.365428924560547,
      "activations/layer23_attention_weight_max": 26.674388885498047,
      "activations/layer23_attention_weight_min": -17.550683975219727,
      "activations/layer2_attention_weight_max": 31.291244506835938,
      "activations/layer2_attention_weight_min": -29.374034881591797,
      "activations/layer3_attention_weight_max": 78.19966125488281,
      "activations/layer3_attention_weight_min": -79.00630187988281,
      "activations/layer4_attention_weight_max": 94.4593734741211,
      "activations/layer4_attention_weight_min": -89.79254150390625,
      "activations/layer5_attention_weight_max": 102.8335189819336,
      "activations/layer5_attention_weight_min": -89.43672943115234,
      "activations/layer6_attention_weight_max": 54.81785202026367,
      "activations/layer6_attention_weight_min": -51.11554718017578,
      "activations/layer7_attention_weight_max": 66.07246398925781,
      "activations/layer7_attention_weight_min": -64.38910675048828,
      "activations/layer8_attention_weight_max": 40.3289909362793,
      "activations/layer8_attention_weight_min": -37.94989776611328,
      "activations/layer9_attention_weight_max": 48.56365966796875,
      "activations/layer9_attention_weight_min": -45.01890563964844,
      "epoch": 4.43,
      "learning_rate": 0.00012264393939393937,
      "loss": 2.8728,
      "step": 76300
    },
    {
      "activations/layer0_attention_weight_max": 14.101197242736816,
      "activations/layer0_attention_weight_min": -14.971007347106934,
      "activations/layer10_attention_weight_max": 38.10817337036133,
      "activations/layer10_attention_weight_min": -33.0833625793457,
      "activations/layer11_attention_weight_max": 29.138694763183594,
      "activations/layer11_attention_weight_min": -26.566329956054688,
      "activations/layer12_attention_weight_max": 30.104427337646484,
      "activations/layer12_attention_weight_min": -25.53860855102539,
      "activations/layer13_attention_weight_max": 49.912113189697266,
      "activations/layer13_attention_weight_min": -38.67637252807617,
      "activations/layer14_attention_weight_max": 54.10143280029297,
      "activations/layer14_attention_weight_min": -37.18683624267578,
      "activations/layer15_attention_weight_max": 49.12263870239258,
      "activations/layer15_attention_weight_min": -31.495012283325195,
      "activations/layer16_attention_weight_max": 37.98862075805664,
      "activations/layer16_attention_weight_min": -25.779834747314453,
      "activations/layer17_attention_weight_max": 56.54765701293945,
      "activations/layer17_attention_weight_min": -36.95600509643555,
      "activations/layer18_attention_weight_max": 51.2583122253418,
      "activations/layer18_attention_weight_min": -31.551054000854492,
      "activations/layer19_attention_weight_max": 23.768741607666016,
      "activations/layer19_attention_weight_min": -19.607725143432617,
      "activations/layer1_attention_weight_max": 18.422941207885742,
      "activations/layer1_attention_weight_min": -14.506275177001953,
      "activations/layer20_attention_weight_max": 22.228071212768555,
      "activations/layer20_attention_weight_min": -18.158775329589844,
      "activations/layer21_attention_weight_max": 39.87580871582031,
      "activations/layer21_attention_weight_min": -26.47074317932129,
      "activations/layer22_attention_weight_max": 36.65498352050781,
      "activations/layer22_attention_weight_min": -22.46581268310547,
      "activations/layer23_attention_weight_max": 25.089468002319336,
      "activations/layer23_attention_weight_min": -17.252166748046875,
      "activations/layer2_attention_weight_max": 29.959768295288086,
      "activations/layer2_attention_weight_min": -29.21119499206543,
      "activations/layer3_attention_weight_max": 78.87545776367188,
      "activations/layer3_attention_weight_min": -83.2341537475586,
      "activations/layer4_attention_weight_max": 103.10462951660156,
      "activations/layer4_attention_weight_min": -92.33528900146484,
      "activations/layer5_attention_weight_max": 106.58651733398438,
      "activations/layer5_attention_weight_min": -95.02202606201172,
      "activations/layer6_attention_weight_max": 59.93216323852539,
      "activations/layer6_attention_weight_min": -54.851539611816406,
      "activations/layer7_attention_weight_max": 68.06856536865234,
      "activations/layer7_attention_weight_min": -63.72195053100586,
      "activations/layer8_attention_weight_max": 41.969512939453125,
      "activations/layer8_attention_weight_min": -41.8666877746582,
      "activations/layer9_attention_weight_max": 47.466556549072266,
      "activations/layer9_attention_weight_min": -43.962425231933594,
      "epoch": 4.44,
      "learning_rate": 0.00012262499999999999,
      "loss": 2.8737,
      "step": 76350
    },
    {
      "activations/layer0_attention_weight_max": 15.40885066986084,
      "activations/layer0_attention_weight_min": -14.88357162475586,
      "activations/layer10_attention_weight_max": 39.544681549072266,
      "activations/layer10_attention_weight_min": -34.522647857666016,
      "activations/layer11_attention_weight_max": 30.97784423828125,
      "activations/layer11_attention_weight_min": -26.97049331665039,
      "activations/layer12_attention_weight_max": 25.453285217285156,
      "activations/layer12_attention_weight_min": -23.146263122558594,
      "activations/layer13_attention_weight_max": 74.94741821289062,
      "activations/layer13_attention_weight_min": -53.31095886230469,
      "activations/layer14_attention_weight_max": 78.25418090820312,
      "activations/layer14_attention_weight_min": -46.09931182861328,
      "activations/layer15_attention_weight_max": 53.114051818847656,
      "activations/layer15_attention_weight_min": -32.400699615478516,
      "activations/layer16_attention_weight_max": 44.024192810058594,
      "activations/layer16_attention_weight_min": -26.479703903198242,
      "activations/layer17_attention_weight_max": 58.77558898925781,
      "activations/layer17_attention_weight_min": -36.5911750793457,
      "activations/layer18_attention_weight_max": 52.12509536743164,
      "activations/layer18_attention_weight_min": -33.41582489013672,
      "activations/layer19_attention_weight_max": 30.178207397460938,
      "activations/layer19_attention_weight_min": -18.228988647460938,
      "activations/layer1_attention_weight_max": 17.791080474853516,
      "activations/layer1_attention_weight_min": -15.949917793273926,
      "activations/layer20_attention_weight_max": 24.386524200439453,
      "activations/layer20_attention_weight_min": -19.902393341064453,
      "activations/layer21_attention_weight_max": 44.636295318603516,
      "activations/layer21_attention_weight_min": -26.20491600036621,
      "activations/layer22_attention_weight_max": 40.59870529174805,
      "activations/layer22_attention_weight_min": -22.49835205078125,
      "activations/layer23_attention_weight_max": 31.958051681518555,
      "activations/layer23_attention_weight_min": -17.8176212310791,
      "activations/layer2_attention_weight_max": 31.497058868408203,
      "activations/layer2_attention_weight_min": -29.086715698242188,
      "activations/layer3_attention_weight_max": 80.12879943847656,
      "activations/layer3_attention_weight_min": -81.42918395996094,
      "activations/layer4_attention_weight_max": 98.58428955078125,
      "activations/layer4_attention_weight_min": -93.43505096435547,
      "activations/layer5_attention_weight_max": 106.13127899169922,
      "activations/layer5_attention_weight_min": -98.46977996826172,
      "activations/layer6_attention_weight_max": 58.28278350830078,
      "activations/layer6_attention_weight_min": -53.92502212524414,
      "activations/layer7_attention_weight_max": 70.58599853515625,
      "activations/layer7_attention_weight_min": -68.25981140136719,
      "activations/layer8_attention_weight_max": 43.33384704589844,
      "activations/layer8_attention_weight_min": -42.11586380004883,
      "activations/layer9_attention_weight_max": 62.74775314331055,
      "activations/layer9_attention_weight_min": -52.27641677856445,
      "epoch": 4.44,
      "learning_rate": 0.0001226060606060606,
      "loss": 2.8727,
      "step": 76400
    },
    {
      "activations/layer0_attention_weight_max": 16.093976974487305,
      "activations/layer0_attention_weight_min": -15.036681175231934,
      "activations/layer10_attention_weight_max": 39.173213958740234,
      "activations/layer10_attention_weight_min": -34.148345947265625,
      "activations/layer11_attention_weight_max": 35.38117599487305,
      "activations/layer11_attention_weight_min": -26.230695724487305,
      "activations/layer12_attention_weight_max": 28.847753524780273,
      "activations/layer12_attention_weight_min": -24.86809539794922,
      "activations/layer13_attention_weight_max": 62.24112319946289,
      "activations/layer13_attention_weight_min": -34.97706985473633,
      "activations/layer14_attention_weight_max": 63.87218475341797,
      "activations/layer14_attention_weight_min": -30.673017501831055,
      "activations/layer15_attention_weight_max": 56.28638458251953,
      "activations/layer15_attention_weight_min": -31.76898956298828,
      "activations/layer16_attention_weight_max": 42.679466247558594,
      "activations/layer16_attention_weight_min": -26.0838565826416,
      "activations/layer17_attention_weight_max": 62.11988067626953,
      "activations/layer17_attention_weight_min": -37.221397399902344,
      "activations/layer18_attention_weight_max": 54.26274871826172,
      "activations/layer18_attention_weight_min": -32.413291931152344,
      "activations/layer19_attention_weight_max": 26.11480712890625,
      "activations/layer19_attention_weight_min": -19.126585006713867,
      "activations/layer1_attention_weight_max": 18.860050201416016,
      "activations/layer1_attention_weight_min": -17.03287696838379,
      "activations/layer20_attention_weight_max": 23.894275665283203,
      "activations/layer20_attention_weight_min": -22.259674072265625,
      "activations/layer21_attention_weight_max": 48.35041046142578,
      "activations/layer21_attention_weight_min": -26.247589111328125,
      "activations/layer22_attention_weight_max": 36.401004791259766,
      "activations/layer22_attention_weight_min": -23.39037322998047,
      "activations/layer23_attention_weight_max": 30.338241577148438,
      "activations/layer23_attention_weight_min": -18.151147842407227,
      "activations/layer2_attention_weight_max": 30.333738327026367,
      "activations/layer2_attention_weight_min": -29.667455673217773,
      "activations/layer3_attention_weight_max": 79.31854248046875,
      "activations/layer3_attention_weight_min": -82.1937484741211,
      "activations/layer4_attention_weight_max": 99.2604751586914,
      "activations/layer4_attention_weight_min": -96.15060424804688,
      "activations/layer5_attention_weight_max": 109.30152893066406,
      "activations/layer5_attention_weight_min": -94.55767059326172,
      "activations/layer6_attention_weight_max": 59.43644332885742,
      "activations/layer6_attention_weight_min": -53.40892791748047,
      "activations/layer7_attention_weight_max": 65.10039520263672,
      "activations/layer7_attention_weight_min": -65.67437744140625,
      "activations/layer8_attention_weight_max": 44.13297653198242,
      "activations/layer8_attention_weight_min": -40.07004928588867,
      "activations/layer9_attention_weight_max": 55.66765213012695,
      "activations/layer9_attention_weight_min": -47.76345443725586,
      "epoch": 4.44,
      "learning_rate": 0.0001225871212121212,
      "loss": 2.858,
      "step": 76450
    },
    {
      "activations/layer0_attention_weight_max": 14.788304328918457,
      "activations/layer0_attention_weight_min": -14.733386039733887,
      "activations/layer10_attention_weight_max": 40.23191833496094,
      "activations/layer10_attention_weight_min": -33.572547912597656,
      "activations/layer11_attention_weight_max": 31.738353729248047,
      "activations/layer11_attention_weight_min": -25.952892303466797,
      "activations/layer12_attention_weight_max": 30.417097091674805,
      "activations/layer12_attention_weight_min": -25.664676666259766,
      "activations/layer13_attention_weight_max": 52.78462600708008,
      "activations/layer13_attention_weight_min": -37.12385940551758,
      "activations/layer14_attention_weight_max": 49.359981536865234,
      "activations/layer14_attention_weight_min": -30.839954376220703,
      "activations/layer15_attention_weight_max": 41.734466552734375,
      "activations/layer15_attention_weight_min": -29.53199005126953,
      "activations/layer16_attention_weight_max": 36.603946685791016,
      "activations/layer16_attention_weight_min": -26.710281372070312,
      "activations/layer17_attention_weight_max": 61.03025436401367,
      "activations/layer17_attention_weight_min": -39.3378791809082,
      "activations/layer18_attention_weight_max": 50.94377136230469,
      "activations/layer18_attention_weight_min": -33.481353759765625,
      "activations/layer19_attention_weight_max": 21.127185821533203,
      "activations/layer19_attention_weight_min": -20.21546745300293,
      "activations/layer1_attention_weight_max": 18.25282859802246,
      "activations/layer1_attention_weight_min": -16.550888061523438,
      "activations/layer20_attention_weight_max": 23.344396591186523,
      "activations/layer20_attention_weight_min": -19.227603912353516,
      "activations/layer21_attention_weight_max": 44.33530044555664,
      "activations/layer21_attention_weight_min": -24.555362701416016,
      "activations/layer22_attention_weight_max": 36.77016067504883,
      "activations/layer22_attention_weight_min": -25.58933448791504,
      "activations/layer23_attention_weight_max": 30.471885681152344,
      "activations/layer23_attention_weight_min": -18.324312210083008,
      "activations/layer2_attention_weight_max": 31.00623321533203,
      "activations/layer2_attention_weight_min": -28.95718765258789,
      "activations/layer3_attention_weight_max": 79.47830963134766,
      "activations/layer3_attention_weight_min": -82.7786636352539,
      "activations/layer4_attention_weight_max": 101.90039825439453,
      "activations/layer4_attention_weight_min": -94.28598022460938,
      "activations/layer5_attention_weight_max": 106.22411346435547,
      "activations/layer5_attention_weight_min": -92.31425476074219,
      "activations/layer6_attention_weight_max": 57.522979736328125,
      "activations/layer6_attention_weight_min": -50.560585021972656,
      "activations/layer7_attention_weight_max": 67.10514831542969,
      "activations/layer7_attention_weight_min": -64.07778930664062,
      "activations/layer8_attention_weight_max": 43.73710632324219,
      "activations/layer8_attention_weight_min": -39.713565826416016,
      "activations/layer9_attention_weight_max": 48.94181442260742,
      "activations/layer9_attention_weight_min": -45.58689498901367,
      "epoch": 4.45,
      "learning_rate": 0.0001225681818181818,
      "loss": 2.8622,
      "step": 76500
    },
    {
      "activations/layer0_attention_weight_max": 14.073644638061523,
      "activations/layer0_attention_weight_min": -14.360846519470215,
      "activations/layer10_attention_weight_max": 41.557861328125,
      "activations/layer10_attention_weight_min": -39.26945114135742,
      "activations/layer11_attention_weight_max": 30.73996353149414,
      "activations/layer11_attention_weight_min": -30.568058013916016,
      "activations/layer12_attention_weight_max": 21.61647605895996,
      "activations/layer12_attention_weight_min": -30.685779571533203,
      "activations/layer13_attention_weight_max": 50.02681350708008,
      "activations/layer13_attention_weight_min": -36.226409912109375,
      "activations/layer14_attention_weight_max": 50.68891525268555,
      "activations/layer14_attention_weight_min": -33.433189392089844,
      "activations/layer15_attention_weight_max": 38.16603469848633,
      "activations/layer15_attention_weight_min": -30.963699340820312,
      "activations/layer16_attention_weight_max": 37.96482467651367,
      "activations/layer16_attention_weight_min": -25.806543350219727,
      "activations/layer17_attention_weight_max": 56.67668533325195,
      "activations/layer17_attention_weight_min": -37.82575225830078,
      "activations/layer18_attention_weight_max": 47.81707763671875,
      "activations/layer18_attention_weight_min": -34.41469192504883,
      "activations/layer19_attention_weight_max": 20.716501235961914,
      "activations/layer19_attention_weight_min": -20.187442779541016,
      "activations/layer1_attention_weight_max": 18.70570945739746,
      "activations/layer1_attention_weight_min": -16.245891571044922,
      "activations/layer20_attention_weight_max": 21.991397857666016,
      "activations/layer20_attention_weight_min": -19.708009719848633,
      "activations/layer21_attention_weight_max": 41.200382232666016,
      "activations/layer21_attention_weight_min": -27.72916603088379,
      "activations/layer22_attention_weight_max": 38.08384323120117,
      "activations/layer22_attention_weight_min": -23.422685623168945,
      "activations/layer23_attention_weight_max": 28.589557647705078,
      "activations/layer23_attention_weight_min": -19.03854751586914,
      "activations/layer2_attention_weight_max": 31.15541648864746,
      "activations/layer2_attention_weight_min": -29.13979148864746,
      "activations/layer3_attention_weight_max": 78.5013656616211,
      "activations/layer3_attention_weight_min": -82.02568054199219,
      "activations/layer4_attention_weight_max": 97.80488586425781,
      "activations/layer4_attention_weight_min": -96.91259002685547,
      "activations/layer5_attention_weight_max": 104.74724578857422,
      "activations/layer5_attention_weight_min": -99.77330017089844,
      "activations/layer6_attention_weight_max": 56.996131896972656,
      "activations/layer6_attention_weight_min": -56.0366096496582,
      "activations/layer7_attention_weight_max": 72.78791809082031,
      "activations/layer7_attention_weight_min": -68.73883056640625,
      "activations/layer8_attention_weight_max": 49.86280822753906,
      "activations/layer8_attention_weight_min": -39.63985061645508,
      "activations/layer9_attention_weight_max": 70.96703338623047,
      "activations/layer9_attention_weight_min": -56.581451416015625,
      "epoch": 4.45,
      "learning_rate": 0.00012254924242424242,
      "loss": 2.8676,
      "step": 76550
    },
    {
      "activations/layer0_attention_weight_max": 13.792766571044922,
      "activations/layer0_attention_weight_min": -14.815590858459473,
      "activations/layer10_attention_weight_max": 36.71295928955078,
      "activations/layer10_attention_weight_min": -32.91227722167969,
      "activations/layer11_attention_weight_max": 28.12618064880371,
      "activations/layer11_attention_weight_min": -26.232030868530273,
      "activations/layer12_attention_weight_max": 19.89154815673828,
      "activations/layer12_attention_weight_min": -25.06620979309082,
      "activations/layer13_attention_weight_max": 54.60429763793945,
      "activations/layer13_attention_weight_min": -38.00630187988281,
      "activations/layer14_attention_weight_max": 42.912132263183594,
      "activations/layer14_attention_weight_min": -32.995826721191406,
      "activations/layer15_attention_weight_max": 45.00125503540039,
      "activations/layer15_attention_weight_min": -29.745304107666016,
      "activations/layer16_attention_weight_max": 32.86676788330078,
      "activations/layer16_attention_weight_min": -26.438697814941406,
      "activations/layer17_attention_weight_max": 53.79801940917969,
      "activations/layer17_attention_weight_min": -35.35293197631836,
      "activations/layer18_attention_weight_max": 47.563987731933594,
      "activations/layer18_attention_weight_min": -30.32794761657715,
      "activations/layer19_attention_weight_max": 24.115713119506836,
      "activations/layer19_attention_weight_min": -19.015167236328125,
      "activations/layer1_attention_weight_max": 17.652896881103516,
      "activations/layer1_attention_weight_min": -16.21698570251465,
      "activations/layer20_attention_weight_max": 20.536455154418945,
      "activations/layer20_attention_weight_min": -19.233402252197266,
      "activations/layer21_attention_weight_max": 36.49201202392578,
      "activations/layer21_attention_weight_min": -26.534164428710938,
      "activations/layer22_attention_weight_max": 33.118167877197266,
      "activations/layer22_attention_weight_min": -22.64838218688965,
      "activations/layer23_attention_weight_max": 24.150211334228516,
      "activations/layer23_attention_weight_min": -17.714637756347656,
      "activations/layer2_attention_weight_max": 30.869583129882812,
      "activations/layer2_attention_weight_min": -27.672893524169922,
      "activations/layer3_attention_weight_max": 77.6469497680664,
      "activations/layer3_attention_weight_min": -81.03263092041016,
      "activations/layer4_attention_weight_max": 98.11356353759766,
      "activations/layer4_attention_weight_min": -90.60077667236328,
      "activations/layer5_attention_weight_max": 104.34608459472656,
      "activations/layer5_attention_weight_min": -89.06367492675781,
      "activations/layer6_attention_weight_max": 56.52386474609375,
      "activations/layer6_attention_weight_min": -48.083030700683594,
      "activations/layer7_attention_weight_max": 64.70413208007812,
      "activations/layer7_attention_weight_min": -60.557579040527344,
      "activations/layer8_attention_weight_max": 39.15696716308594,
      "activations/layer8_attention_weight_min": -37.61173629760742,
      "activations/layer9_attention_weight_max": 45.29401779174805,
      "activations/layer9_attention_weight_min": -48.090938568115234,
      "epoch": 4.45,
      "learning_rate": 0.000122530303030303,
      "loss": 2.8613,
      "step": 76600
    },
    {
      "activations/layer0_attention_weight_max": 13.983741760253906,
      "activations/layer0_attention_weight_min": -14.841324806213379,
      "activations/layer10_attention_weight_max": 38.042049407958984,
      "activations/layer10_attention_weight_min": -34.967872619628906,
      "activations/layer11_attention_weight_max": 29.413318634033203,
      "activations/layer11_attention_weight_min": -27.384227752685547,
      "activations/layer12_attention_weight_max": 21.433456420898438,
      "activations/layer12_attention_weight_min": -23.653032302856445,
      "activations/layer13_attention_weight_max": 55.88716125488281,
      "activations/layer13_attention_weight_min": -41.878170013427734,
      "activations/layer14_attention_weight_max": 66.15019226074219,
      "activations/layer14_attention_weight_min": -44.451812744140625,
      "activations/layer15_attention_weight_max": 42.836936950683594,
      "activations/layer15_attention_weight_min": -31.489391326904297,
      "activations/layer16_attention_weight_max": 30.3673095703125,
      "activations/layer16_attention_weight_min": -25.35207748413086,
      "activations/layer17_attention_weight_max": 57.59162139892578,
      "activations/layer17_attention_weight_min": -39.55866622924805,
      "activations/layer18_attention_weight_max": 48.40194320678711,
      "activations/layer18_attention_weight_min": -33.42910385131836,
      "activations/layer19_attention_weight_max": 22.231225967407227,
      "activations/layer19_attention_weight_min": -19.361413955688477,
      "activations/layer1_attention_weight_max": 17.146547317504883,
      "activations/layer1_attention_weight_min": -15.524643898010254,
      "activations/layer20_attention_weight_max": 21.420066833496094,
      "activations/layer20_attention_weight_min": -21.37516975402832,
      "activations/layer21_attention_weight_max": 35.91785430908203,
      "activations/layer21_attention_weight_min": -27.330862045288086,
      "activations/layer22_attention_weight_max": 32.95983123779297,
      "activations/layer22_attention_weight_min": -22.49135398864746,
      "activations/layer23_attention_weight_max": 26.498817443847656,
      "activations/layer23_attention_weight_min": -18.21156120300293,
      "activations/layer2_attention_weight_max": 30.1431884765625,
      "activations/layer2_attention_weight_min": -28.79845428466797,
      "activations/layer3_attention_weight_max": 79.12833404541016,
      "activations/layer3_attention_weight_min": -87.32408142089844,
      "activations/layer4_attention_weight_max": 98.0255355834961,
      "activations/layer4_attention_weight_min": -92.91497802734375,
      "activations/layer5_attention_weight_max": 108.29975891113281,
      "activations/layer5_attention_weight_min": -93.96604919433594,
      "activations/layer6_attention_weight_max": 58.8157958984375,
      "activations/layer6_attention_weight_min": -52.771785736083984,
      "activations/layer7_attention_weight_max": 64.39595031738281,
      "activations/layer7_attention_weight_min": -63.176124572753906,
      "activations/layer8_attention_weight_max": 42.626197814941406,
      "activations/layer8_attention_weight_min": -38.76546096801758,
      "activations/layer9_attention_weight_max": 48.0615348815918,
      "activations/layer9_attention_weight_min": -47.59481430053711,
      "epoch": 4.45,
      "learning_rate": 0.00012251136363636362,
      "loss": 2.8666,
      "step": 76650
    },
    {
      "activations/layer0_attention_weight_max": 15.600661277770996,
      "activations/layer0_attention_weight_min": -14.999272346496582,
      "activations/layer10_attention_weight_max": 39.828163146972656,
      "activations/layer10_attention_weight_min": -36.32405090332031,
      "activations/layer11_attention_weight_max": 31.128211975097656,
      "activations/layer11_attention_weight_min": -27.110715866088867,
      "activations/layer12_attention_weight_max": 25.163999557495117,
      "activations/layer12_attention_weight_min": -33.32703399658203,
      "activations/layer13_attention_weight_max": 60.81418228149414,
      "activations/layer13_attention_weight_min": -41.7243537902832,
      "activations/layer14_attention_weight_max": 48.74620056152344,
      "activations/layer14_attention_weight_min": -33.16994094848633,
      "activations/layer15_attention_weight_max": 61.11344909667969,
      "activations/layer15_attention_weight_min": -37.94997787475586,
      "activations/layer16_attention_weight_max": 40.44187927246094,
      "activations/layer16_attention_weight_min": -25.342945098876953,
      "activations/layer17_attention_weight_max": 61.58256912231445,
      "activations/layer17_attention_weight_min": -37.51132583618164,
      "activations/layer18_attention_weight_max": 56.1549186706543,
      "activations/layer18_attention_weight_min": -34.05522918701172,
      "activations/layer19_attention_weight_max": 28.917831420898438,
      "activations/layer19_attention_weight_min": -19.865211486816406,
      "activations/layer1_attention_weight_max": 18.576496124267578,
      "activations/layer1_attention_weight_min": -13.533432006835938,
      "activations/layer20_attention_weight_max": 20.498680114746094,
      "activations/layer20_attention_weight_min": -22.279293060302734,
      "activations/layer21_attention_weight_max": 42.244293212890625,
      "activations/layer21_attention_weight_min": -26.30908203125,
      "activations/layer22_attention_weight_max": 35.967933654785156,
      "activations/layer22_attention_weight_min": -24.09571647644043,
      "activations/layer23_attention_weight_max": 27.289260864257812,
      "activations/layer23_attention_weight_min": -17.574108123779297,
      "activations/layer2_attention_weight_max": 29.053096771240234,
      "activations/layer2_attention_weight_min": -28.571521759033203,
      "activations/layer3_attention_weight_max": 79.26565551757812,
      "activations/layer3_attention_weight_min": -86.57064819335938,
      "activations/layer4_attention_weight_max": 102.1611099243164,
      "activations/layer4_attention_weight_min": -99.70816802978516,
      "activations/layer5_attention_weight_max": 104.58455657958984,
      "activations/layer5_attention_weight_min": -96.68441772460938,
      "activations/layer6_attention_weight_max": 57.739662170410156,
      "activations/layer6_attention_weight_min": -52.51353454589844,
      "activations/layer7_attention_weight_max": 69.8357162475586,
      "activations/layer7_attention_weight_min": -63.270729064941406,
      "activations/layer8_attention_weight_max": 41.3249626159668,
      "activations/layer8_attention_weight_min": -39.9337158203125,
      "activations/layer9_attention_weight_max": 57.194915771484375,
      "activations/layer9_attention_weight_min": -51.12385177612305,
      "epoch": 4.46,
      "learning_rate": 0.00012249242424242424,
      "loss": 2.8571,
      "step": 76700
    },
    {
      "activations/layer0_attention_weight_max": 13.80906867980957,
      "activations/layer0_attention_weight_min": -14.927329063415527,
      "activations/layer10_attention_weight_max": 40.208770751953125,
      "activations/layer10_attention_weight_min": -35.4888916015625,
      "activations/layer11_attention_weight_max": 32.94471740722656,
      "activations/layer11_attention_weight_min": -27.924543380737305,
      "activations/layer12_attention_weight_max": 31.301284790039062,
      "activations/layer12_attention_weight_min": -27.0900936126709,
      "activations/layer13_attention_weight_max": 54.53461837768555,
      "activations/layer13_attention_weight_min": -35.199012756347656,
      "activations/layer14_attention_weight_max": 52.59709930419922,
      "activations/layer14_attention_weight_min": -35.60449981689453,
      "activations/layer15_attention_weight_max": 43.508052825927734,
      "activations/layer15_attention_weight_min": -30.99994468688965,
      "activations/layer16_attention_weight_max": 41.702049255371094,
      "activations/layer16_attention_weight_min": -25.52634620666504,
      "activations/layer17_attention_weight_max": 59.26588821411133,
      "activations/layer17_attention_weight_min": -37.427711486816406,
      "activations/layer18_attention_weight_max": 50.23617935180664,
      "activations/layer18_attention_weight_min": -31.681358337402344,
      "activations/layer19_attention_weight_max": 20.562477111816406,
      "activations/layer19_attention_weight_min": -16.865676879882812,
      "activations/layer1_attention_weight_max": 18.32964515686035,
      "activations/layer1_attention_weight_min": -15.208294868469238,
      "activations/layer20_attention_weight_max": 19.20248031616211,
      "activations/layer20_attention_weight_min": -18.70853614807129,
      "activations/layer21_attention_weight_max": 40.958152770996094,
      "activations/layer21_attention_weight_min": -27.375925064086914,
      "activations/layer22_attention_weight_max": 34.978912353515625,
      "activations/layer22_attention_weight_min": -22.865007400512695,
      "activations/layer23_attention_weight_max": 26.774160385131836,
      "activations/layer23_attention_weight_min": -15.586889266967773,
      "activations/layer2_attention_weight_max": 31.927587509155273,
      "activations/layer2_attention_weight_min": -30.31916046142578,
      "activations/layer3_attention_weight_max": 85.5307388305664,
      "activations/layer3_attention_weight_min": -86.09556579589844,
      "activations/layer4_attention_weight_max": 107.76710510253906,
      "activations/layer4_attention_weight_min": -99.89000701904297,
      "activations/layer5_attention_weight_max": 114.0918197631836,
      "activations/layer5_attention_weight_min": -101.04266357421875,
      "activations/layer6_attention_weight_max": 60.5131950378418,
      "activations/layer6_attention_weight_min": -56.23264694213867,
      "activations/layer7_attention_weight_max": 78.75153350830078,
      "activations/layer7_attention_weight_min": -71.73837280273438,
      "activations/layer8_attention_weight_max": 48.92474365234375,
      "activations/layer8_attention_weight_min": -42.80147933959961,
      "activations/layer9_attention_weight_max": 66.4264144897461,
      "activations/layer9_attention_weight_min": -54.059120178222656,
      "epoch": 4.46,
      "learning_rate": 0.00012247348484848482,
      "loss": 2.8608,
      "step": 76750
    },
    {
      "activations/layer0_attention_weight_max": 14.715167045593262,
      "activations/layer0_attention_weight_min": -15.032365798950195,
      "activations/layer10_attention_weight_max": 36.14594268798828,
      "activations/layer10_attention_weight_min": -32.0816764831543,
      "activations/layer11_attention_weight_max": 29.554250717163086,
      "activations/layer11_attention_weight_min": -26.65943717956543,
      "activations/layer12_attention_weight_max": 20.715967178344727,
      "activations/layer12_attention_weight_min": -24.136722564697266,
      "activations/layer13_attention_weight_max": 39.137847900390625,
      "activations/layer13_attention_weight_min": -35.23800277709961,
      "activations/layer14_attention_weight_max": 37.88220977783203,
      "activations/layer14_attention_weight_min": -30.147235870361328,
      "activations/layer15_attention_weight_max": 41.38016891479492,
      "activations/layer15_attention_weight_min": -30.95439338684082,
      "activations/layer16_attention_weight_max": 33.98339080810547,
      "activations/layer16_attention_weight_min": -25.547880172729492,
      "activations/layer17_attention_weight_max": 52.85774612426758,
      "activations/layer17_attention_weight_min": -38.329471588134766,
      "activations/layer18_attention_weight_max": 50.15443801879883,
      "activations/layer18_attention_weight_min": -33.200252532958984,
      "activations/layer19_attention_weight_max": 21.92279624938965,
      "activations/layer19_attention_weight_min": -17.727256774902344,
      "activations/layer1_attention_weight_max": 17.61947250366211,
      "activations/layer1_attention_weight_min": -13.991488456726074,
      "activations/layer20_attention_weight_max": 19.106101989746094,
      "activations/layer20_attention_weight_min": -19.009294509887695,
      "activations/layer21_attention_weight_max": 35.03241729736328,
      "activations/layer21_attention_weight_min": -26.44428062438965,
      "activations/layer22_attention_weight_max": 34.066707611083984,
      "activations/layer22_attention_weight_min": -21.90695571899414,
      "activations/layer23_attention_weight_max": 25.343273162841797,
      "activations/layer23_attention_weight_min": -18.245410919189453,
      "activations/layer2_attention_weight_max": 28.39562225341797,
      "activations/layer2_attention_weight_min": -28.774642944335938,
      "activations/layer3_attention_weight_max": 73.08829498291016,
      "activations/layer3_attention_weight_min": -79.7576904296875,
      "activations/layer4_attention_weight_max": 94.08683776855469,
      "activations/layer4_attention_weight_min": -86.65152740478516,
      "activations/layer5_attention_weight_max": 102.63978576660156,
      "activations/layer5_attention_weight_min": -87.89336395263672,
      "activations/layer6_attention_weight_max": 58.4675178527832,
      "activations/layer6_attention_weight_min": -49.11469650268555,
      "activations/layer7_attention_weight_max": 66.56913757324219,
      "activations/layer7_attention_weight_min": -60.024993896484375,
      "activations/layer8_attention_weight_max": 41.31633377075195,
      "activations/layer8_attention_weight_min": -37.8092041015625,
      "activations/layer9_attention_weight_max": 49.32194519042969,
      "activations/layer9_attention_weight_min": -47.49982833862305,
      "epoch": 4.46,
      "learning_rate": 0.00012245454545454544,
      "loss": 2.848,
      "step": 76800
    },
    {
      "activations/layer0_attention_weight_max": 14.53318977355957,
      "activations/layer0_attention_weight_min": -14.882502555847168,
      "activations/layer10_attention_weight_max": 37.41868209838867,
      "activations/layer10_attention_weight_min": -32.78340530395508,
      "activations/layer11_attention_weight_max": 28.4755916595459,
      "activations/layer11_attention_weight_min": -25.09430503845215,
      "activations/layer12_attention_weight_max": 21.30939292907715,
      "activations/layer12_attention_weight_min": -24.8640079498291,
      "activations/layer13_attention_weight_max": 59.3123779296875,
      "activations/layer13_attention_weight_min": -40.82530212402344,
      "activations/layer14_attention_weight_max": 51.3726806640625,
      "activations/layer14_attention_weight_min": -33.36669921875,
      "activations/layer15_attention_weight_max": 42.296443939208984,
      "activations/layer15_attention_weight_min": -33.06743240356445,
      "activations/layer16_attention_weight_max": 35.58018493652344,
      "activations/layer16_attention_weight_min": -28.84408950805664,
      "activations/layer17_attention_weight_max": 58.38978576660156,
      "activations/layer17_attention_weight_min": -40.619789123535156,
      "activations/layer18_attention_weight_max": 51.463218688964844,
      "activations/layer18_attention_weight_min": -32.022003173828125,
      "activations/layer19_attention_weight_max": 24.83437156677246,
      "activations/layer19_attention_weight_min": -18.874601364135742,
      "activations/layer1_attention_weight_max": 19.41344451904297,
      "activations/layer1_attention_weight_min": -14.205409049987793,
      "activations/layer20_attention_weight_max": 23.189311981201172,
      "activations/layer20_attention_weight_min": -22.812999725341797,
      "activations/layer21_attention_weight_max": 37.86362075805664,
      "activations/layer21_attention_weight_min": -27.240507125854492,
      "activations/layer22_attention_weight_max": 35.94580841064453,
      "activations/layer22_attention_weight_min": -24.034543991088867,
      "activations/layer23_attention_weight_max": 27.394920349121094,
      "activations/layer23_attention_weight_min": -20.657936096191406,
      "activations/layer2_attention_weight_max": 29.46976089477539,
      "activations/layer2_attention_weight_min": -27.850431442260742,
      "activations/layer3_attention_weight_max": 76.3463134765625,
      "activations/layer3_attention_weight_min": -81.53956604003906,
      "activations/layer4_attention_weight_max": 97.59505462646484,
      "activations/layer4_attention_weight_min": -97.73282623291016,
      "activations/layer5_attention_weight_max": 107.38145446777344,
      "activations/layer5_attention_weight_min": -97.88832092285156,
      "activations/layer6_attention_weight_max": 60.54888916015625,
      "activations/layer6_attention_weight_min": -50.152244567871094,
      "activations/layer7_attention_weight_max": 67.61241912841797,
      "activations/layer7_attention_weight_min": -62.67867660522461,
      "activations/layer8_attention_weight_max": 40.30638885498047,
      "activations/layer8_attention_weight_min": -37.38300704956055,
      "activations/layer9_attention_weight_max": 51.46735763549805,
      "activations/layer9_attention_weight_min": -48.94794845581055,
      "epoch": 4.47,
      "learning_rate": 0.00012243560606060605,
      "loss": 2.8626,
      "step": 76850
    },
    {
      "activations/layer0_attention_weight_max": 15.999885559082031,
      "activations/layer0_attention_weight_min": -14.509416580200195,
      "activations/layer10_attention_weight_max": 38.8992805480957,
      "activations/layer10_attention_weight_min": -38.12554931640625,
      "activations/layer11_attention_weight_max": 32.685386657714844,
      "activations/layer11_attention_weight_min": -28.738853454589844,
      "activations/layer12_attention_weight_max": 22.248807907104492,
      "activations/layer12_attention_weight_min": -31.372760772705078,
      "activations/layer13_attention_weight_max": 59.439552307128906,
      "activations/layer13_attention_weight_min": -32.69858932495117,
      "activations/layer14_attention_weight_max": 66.13407897949219,
      "activations/layer14_attention_weight_min": -32.60451889038086,
      "activations/layer15_attention_weight_max": 46.49687194824219,
      "activations/layer15_attention_weight_min": -28.5921630859375,
      "activations/layer16_attention_weight_max": 36.412288665771484,
      "activations/layer16_attention_weight_min": -26.438356399536133,
      "activations/layer17_attention_weight_max": 63.72260284423828,
      "activations/layer17_attention_weight_min": -38.009300231933594,
      "activations/layer18_attention_weight_max": 49.7158088684082,
      "activations/layer18_attention_weight_min": -31.88215446472168,
      "activations/layer19_attention_weight_max": 24.1708927154541,
      "activations/layer19_attention_weight_min": -19.190095901489258,
      "activations/layer1_attention_weight_max": 19.25956153869629,
      "activations/layer1_attention_weight_min": -13.240570068359375,
      "activations/layer20_attention_weight_max": 21.160619735717773,
      "activations/layer20_attention_weight_min": -17.321428298950195,
      "activations/layer21_attention_weight_max": 44.288818359375,
      "activations/layer21_attention_weight_min": -25.004497528076172,
      "activations/layer22_attention_weight_max": 39.584434509277344,
      "activations/layer22_attention_weight_min": -23.4301700592041,
      "activations/layer23_attention_weight_max": 30.641080856323242,
      "activations/layer23_attention_weight_min": -17.87073516845703,
      "activations/layer2_attention_weight_max": 30.24185562133789,
      "activations/layer2_attention_weight_min": -28.99867057800293,
      "activations/layer3_attention_weight_max": 82.466796875,
      "activations/layer3_attention_weight_min": -83.66482543945312,
      "activations/layer4_attention_weight_max": 103.43011474609375,
      "activations/layer4_attention_weight_min": -99.33748626708984,
      "activations/layer5_attention_weight_max": 111.69374084472656,
      "activations/layer5_attention_weight_min": -100.77440643310547,
      "activations/layer6_attention_weight_max": 61.596107482910156,
      "activations/layer6_attention_weight_min": -53.78929138183594,
      "activations/layer7_attention_weight_max": 76.57740020751953,
      "activations/layer7_attention_weight_min": -67.42169189453125,
      "activations/layer8_attention_weight_max": 45.64838790893555,
      "activations/layer8_attention_weight_min": -44.007137298583984,
      "activations/layer9_attention_weight_max": 54.0499153137207,
      "activations/layer9_attention_weight_min": -48.921695709228516,
      "epoch": 4.47,
      "learning_rate": 0.00012241666666666667,
      "loss": 2.869,
      "step": 76900
    },
    {
      "activations/layer0_attention_weight_max": 14.78306770324707,
      "activations/layer0_attention_weight_min": -15.121920585632324,
      "activations/layer10_attention_weight_max": 37.867069244384766,
      "activations/layer10_attention_weight_min": -32.69545364379883,
      "activations/layer11_attention_weight_max": 30.232688903808594,
      "activations/layer11_attention_weight_min": -26.874876022338867,
      "activations/layer12_attention_weight_max": 25.790454864501953,
      "activations/layer12_attention_weight_min": -23.917619705200195,
      "activations/layer13_attention_weight_max": 45.23654556274414,
      "activations/layer13_attention_weight_min": -35.48197555541992,
      "activations/layer14_attention_weight_max": 41.40791320800781,
      "activations/layer14_attention_weight_min": -28.542665481567383,
      "activations/layer15_attention_weight_max": 37.05255126953125,
      "activations/layer15_attention_weight_min": -30.630096435546875,
      "activations/layer16_attention_weight_max": 35.69388198852539,
      "activations/layer16_attention_weight_min": -29.60185432434082,
      "activations/layer17_attention_weight_max": 57.96026611328125,
      "activations/layer17_attention_weight_min": -38.24112319946289,
      "activations/layer18_attention_weight_max": 46.444454193115234,
      "activations/layer18_attention_weight_min": -32.57097625732422,
      "activations/layer19_attention_weight_max": 24.609073638916016,
      "activations/layer19_attention_weight_min": -18.690780639648438,
      "activations/layer1_attention_weight_max": 17.149917602539062,
      "activations/layer1_attention_weight_min": -14.893660545349121,
      "activations/layer20_attention_weight_max": 21.593656539916992,
      "activations/layer20_attention_weight_min": -18.606853485107422,
      "activations/layer21_attention_weight_max": 37.97706985473633,
      "activations/layer21_attention_weight_min": -28.971250534057617,
      "activations/layer22_attention_weight_max": 34.23274230957031,
      "activations/layer22_attention_weight_min": -22.129241943359375,
      "activations/layer23_attention_weight_max": 26.353422164916992,
      "activations/layer23_attention_weight_min": -17.68326187133789,
      "activations/layer2_attention_weight_max": 30.417694091796875,
      "activations/layer2_attention_weight_min": -28.28261947631836,
      "activations/layer3_attention_weight_max": 84.08482360839844,
      "activations/layer3_attention_weight_min": -86.49671173095703,
      "activations/layer4_attention_weight_max": 106.18574523925781,
      "activations/layer4_attention_weight_min": -98.0959701538086,
      "activations/layer5_attention_weight_max": 108.68888854980469,
      "activations/layer5_attention_weight_min": -94.8864974975586,
      "activations/layer6_attention_weight_max": 57.2489013671875,
      "activations/layer6_attention_weight_min": -51.66340637207031,
      "activations/layer7_attention_weight_max": 65.60814666748047,
      "activations/layer7_attention_weight_min": -61.009376525878906,
      "activations/layer8_attention_weight_max": 39.54203414916992,
      "activations/layer8_attention_weight_min": -38.99815368652344,
      "activations/layer9_attention_weight_max": 51.62150955200195,
      "activations/layer9_attention_weight_min": -48.5034065246582,
      "epoch": 4.47,
      "learning_rate": 0.00012239772727272726,
      "loss": 2.854,
      "step": 76950
    },
    {
      "activations/layer0_attention_weight_max": 13.547823905944824,
      "activations/layer0_attention_weight_min": -14.993191719055176,
      "activations/layer10_attention_weight_max": 43.265567779541016,
      "activations/layer10_attention_weight_min": -33.69572830200195,
      "activations/layer11_attention_weight_max": 32.47223663330078,
      "activations/layer11_attention_weight_min": -27.885250091552734,
      "activations/layer12_attention_weight_max": 30.89270782470703,
      "activations/layer12_attention_weight_min": -24.34223175048828,
      "activations/layer13_attention_weight_max": 56.18193817138672,
      "activations/layer13_attention_weight_min": -36.329559326171875,
      "activations/layer14_attention_weight_max": 42.94854736328125,
      "activations/layer14_attention_weight_min": -29.36625099182129,
      "activations/layer15_attention_weight_max": 45.155242919921875,
      "activations/layer15_attention_weight_min": -30.80746841430664,
      "activations/layer16_attention_weight_max": 42.63875961303711,
      "activations/layer16_attention_weight_min": -28.51418113708496,
      "activations/layer17_attention_weight_max": 66.8290786743164,
      "activations/layer17_attention_weight_min": -38.7927131652832,
      "activations/layer18_attention_weight_max": 54.459293365478516,
      "activations/layer18_attention_weight_min": -33.1247673034668,
      "activations/layer19_attention_weight_max": 24.630817413330078,
      "activations/layer19_attention_weight_min": -19.518239974975586,
      "activations/layer1_attention_weight_max": 18.805076599121094,
      "activations/layer1_attention_weight_min": -17.696821212768555,
      "activations/layer20_attention_weight_max": 23.432727813720703,
      "activations/layer20_attention_weight_min": -20.589561462402344,
      "activations/layer21_attention_weight_max": 41.534202575683594,
      "activations/layer21_attention_weight_min": -27.838558197021484,
      "activations/layer22_attention_weight_max": 40.50970458984375,
      "activations/layer22_attention_weight_min": -21.84670066833496,
      "activations/layer23_attention_weight_max": 33.777992248535156,
      "activations/layer23_attention_weight_min": -16.413192749023438,
      "activations/layer2_attention_weight_max": 32.27578353881836,
      "activations/layer2_attention_weight_min": -28.876020431518555,
      "activations/layer3_attention_weight_max": 80.01358032226562,
      "activations/layer3_attention_weight_min": -80.88349151611328,
      "activations/layer4_attention_weight_max": 100.60478210449219,
      "activations/layer4_attention_weight_min": -90.62495422363281,
      "activations/layer5_attention_weight_max": 109.04182434082031,
      "activations/layer5_attention_weight_min": -94.09690856933594,
      "activations/layer6_attention_weight_max": 62.6837272644043,
      "activations/layer6_attention_weight_min": -52.44377899169922,
      "activations/layer7_attention_weight_max": 65.18853759765625,
      "activations/layer7_attention_weight_min": -63.595001220703125,
      "activations/layer8_attention_weight_max": 42.05085754394531,
      "activations/layer8_attention_weight_min": -40.35175704956055,
      "activations/layer9_attention_weight_max": 48.79336929321289,
      "activations/layer9_attention_weight_min": -50.16606903076172,
      "epoch": 4.47,
      "learning_rate": 0.00012237878787878787,
      "loss": 2.8566,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_loss": 2.814453125,
      "eval_runtime": 8.471,
      "eval_samples_per_second": 506.903,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_openwebtext_loss": 2.814453125,
      "eval_openwebtext_ppl": 16.684049195604416,
      "eval_openwebtext_runtime": 8.471,
      "eval_openwebtext_samples_per_second": 506.903,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_wikitext_loss": 3.05078125,
      "eval_wikitext_ppl": 21.131847230940878,
      "eval_wikitext_runtime": 1.9493,
      "eval_wikitext_samples_per_second": 233.929,
      "step": 77000
    },
    {
      "epoch": 4.47,
      "eval_lambada_loss": 2.828125,
      "eval_lambada_ppl": 16.91371785382974,
      "eval_lambada_runtime": 9.5563,
      "eval_lambada_samples_per_second": 509.508,
      "step": 77000
    },
    {
      "activations/layer0_attention_weight_max": 13.781139373779297,
      "activations/layer0_attention_weight_min": -14.532952308654785,
      "activations/layer10_attention_weight_max": 38.855587005615234,
      "activations/layer10_attention_weight_min": -33.14162063598633,
      "activations/layer11_attention_weight_max": 30.83351707458496,
      "activations/layer11_attention_weight_min": -25.929115295410156,
      "activations/layer12_attention_weight_max": 24.996498107910156,
      "activations/layer12_attention_weight_min": -25.91931915283203,
      "activations/layer13_attention_weight_max": 67.60025024414062,
      "activations/layer13_attention_weight_min": -41.76087188720703,
      "activations/layer14_attention_weight_max": 59.854530334472656,
      "activations/layer14_attention_weight_min": -32.73786926269531,
      "activations/layer15_attention_weight_max": 48.553688049316406,
      "activations/layer15_attention_weight_min": -30.017431259155273,
      "activations/layer16_attention_weight_max": 39.478031158447266,
      "activations/layer16_attention_weight_min": -25.637468338012695,
      "activations/layer17_attention_weight_max": 59.38117980957031,
      "activations/layer17_attention_weight_min": -38.04827117919922,
      "activations/layer18_attention_weight_max": 55.86548614501953,
      "activations/layer18_attention_weight_min": -33.4946403503418,
      "activations/layer19_attention_weight_max": 27.190555572509766,
      "activations/layer19_attention_weight_min": -20.100324630737305,
      "activations/layer1_attention_weight_max": 17.35474967956543,
      "activations/layer1_attention_weight_min": -15.758867263793945,
      "activations/layer20_attention_weight_max": 27.039077758789062,
      "activations/layer20_attention_weight_min": -20.306955337524414,
      "activations/layer21_attention_weight_max": 44.1173210144043,
      "activations/layer21_attention_weight_min": -24.93480110168457,
      "activations/layer22_attention_weight_max": 37.45298385620117,
      "activations/layer22_attention_weight_min": -24.409130096435547,
      "activations/layer23_attention_weight_max": 27.018611907958984,
      "activations/layer23_attention_weight_min": -19.451114654541016,
      "activations/layer2_attention_weight_max": 34.01707458496094,
      "activations/layer2_attention_weight_min": -31.026100158691406,
      "activations/layer3_attention_weight_max": 79.77142333984375,
      "activations/layer3_attention_weight_min": -85.25752258300781,
      "activations/layer4_attention_weight_max": 99.34461975097656,
      "activations/layer4_attention_weight_min": -90.47478485107422,
      "activations/layer5_attention_weight_max": 105.21488952636719,
      "activations/layer5_attention_weight_min": -93.31526947021484,
      "activations/layer6_attention_weight_max": 58.60198974609375,
      "activations/layer6_attention_weight_min": -51.62236022949219,
      "activations/layer7_attention_weight_max": 68.45769500732422,
      "activations/layer7_attention_weight_min": -65.73409271240234,
      "activations/layer8_attention_weight_max": 42.507057189941406,
      "activations/layer8_attention_weight_min": -39.15462875366211,
      "activations/layer9_attention_weight_max": 52.739845275878906,
      "activations/layer9_attention_weight_min": -46.730464935302734,
      "epoch": 4.48,
      "learning_rate": 0.0001223598484848485,
      "loss": 2.8682,
      "step": 77050
    },
    {
      "activations/layer0_attention_weight_max": 16.975872039794922,
      "activations/layer0_attention_weight_min": -15.06005859375,
      "activations/layer10_attention_weight_max": 41.0263786315918,
      "activations/layer10_attention_weight_min": -35.93098831176758,
      "activations/layer11_attention_weight_max": 32.96581268310547,
      "activations/layer11_attention_weight_min": -27.23067283630371,
      "activations/layer12_attention_weight_max": 24.959617614746094,
      "activations/layer12_attention_weight_min": -28.18463706970215,
      "activations/layer13_attention_weight_max": 59.18682098388672,
      "activations/layer13_attention_weight_min": -47.33943176269531,
      "activations/layer14_attention_weight_max": 56.44001007080078,
      "activations/layer14_attention_weight_min": -41.44237518310547,
      "activations/layer15_attention_weight_max": 49.43746566772461,
      "activations/layer15_attention_weight_min": -32.218196868896484,
      "activations/layer16_attention_weight_max": 43.715362548828125,
      "activations/layer16_attention_weight_min": -27.350767135620117,
      "activations/layer17_attention_weight_max": 70.42020416259766,
      "activations/layer17_attention_weight_min": -44.52169418334961,
      "activations/layer18_attention_weight_max": 58.20269012451172,
      "activations/layer18_attention_weight_min": -38.126487731933594,
      "activations/layer19_attention_weight_max": 24.1988582611084,
      "activations/layer19_attention_weight_min": -21.018102645874023,
      "activations/layer1_attention_weight_max": 17.42820167541504,
      "activations/layer1_attention_weight_min": -15.451866149902344,
      "activations/layer20_attention_weight_max": 24.452157974243164,
      "activations/layer20_attention_weight_min": -21.685937881469727,
      "activations/layer21_attention_weight_max": 46.8791389465332,
      "activations/layer21_attention_weight_min": -27.99363136291504,
      "activations/layer22_attention_weight_max": 41.092979431152344,
      "activations/layer22_attention_weight_min": -26.194705963134766,
      "activations/layer23_attention_weight_max": 35.542030334472656,
      "activations/layer23_attention_weight_min": -17.598459243774414,
      "activations/layer2_attention_weight_max": 31.27589988708496,
      "activations/layer2_attention_weight_min": -27.589599609375,
      "activations/layer3_attention_weight_max": 79.46786499023438,
      "activations/layer3_attention_weight_min": -82.85366821289062,
      "activations/layer4_attention_weight_max": 102.55419158935547,
      "activations/layer4_attention_weight_min": -95.52005767822266,
      "activations/layer5_attention_weight_max": 107.0501708984375,
      "activations/layer5_attention_weight_min": -97.62468719482422,
      "activations/layer6_attention_weight_max": 61.368629455566406,
      "activations/layer6_attention_weight_min": -55.854007720947266,
      "activations/layer7_attention_weight_max": 67.55957794189453,
      "activations/layer7_attention_weight_min": -64.78766632080078,
      "activations/layer8_attention_weight_max": 42.308448791503906,
      "activations/layer8_attention_weight_min": -42.326133728027344,
      "activations/layer9_attention_weight_max": 53.34878158569336,
      "activations/layer9_attention_weight_min": -50.10328674316406,
      "epoch": 4.48,
      "learning_rate": 0.00012234090909090908,
      "loss": 2.8912,
      "step": 77100
    },
    {
      "activations/layer0_attention_weight_max": 14.816837310791016,
      "activations/layer0_attention_weight_min": -14.332231521606445,
      "activations/layer10_attention_weight_max": 38.3953742980957,
      "activations/layer10_attention_weight_min": -35.491973876953125,
      "activations/layer11_attention_weight_max": 34.74694061279297,
      "activations/layer11_attention_weight_min": -28.93129539489746,
      "activations/layer12_attention_weight_max": 21.26380729675293,
      "activations/layer12_attention_weight_min": -22.44652557373047,
      "activations/layer13_attention_weight_max": 49.73707580566406,
      "activations/layer13_attention_weight_min": -35.09507369995117,
      "activations/layer14_attention_weight_max": 41.00782775878906,
      "activations/layer14_attention_weight_min": -29.19332504272461,
      "activations/layer15_attention_weight_max": 43.39059066772461,
      "activations/layer15_attention_weight_min": -28.896188735961914,
      "activations/layer16_attention_weight_max": 34.0572624206543,
      "activations/layer16_attention_weight_min": -25.857053756713867,
      "activations/layer17_attention_weight_max": 55.67634963989258,
      "activations/layer17_attention_weight_min": -37.81205368041992,
      "activations/layer18_attention_weight_max": 51.81694412231445,
      "activations/layer18_attention_weight_min": -31.986703872680664,
      "activations/layer19_attention_weight_max": 25.04679298400879,
      "activations/layer19_attention_weight_min": -18.038463592529297,
      "activations/layer1_attention_weight_max": 17.892131805419922,
      "activations/layer1_attention_weight_min": -13.992084503173828,
      "activations/layer20_attention_weight_max": 21.650522232055664,
      "activations/layer20_attention_weight_min": -22.726661682128906,
      "activations/layer21_attention_weight_max": 38.7324104309082,
      "activations/layer21_attention_weight_min": -25.332212448120117,
      "activations/layer22_attention_weight_max": 34.330650329589844,
      "activations/layer22_attention_weight_min": -22.92947769165039,
      "activations/layer23_attention_weight_max": 27.97821807861328,
      "activations/layer23_attention_weight_min": -19.230045318603516,
      "activations/layer2_attention_weight_max": 31.225467681884766,
      "activations/layer2_attention_weight_min": -29.243045806884766,
      "activations/layer3_attention_weight_max": 80.50346374511719,
      "activations/layer3_attention_weight_min": -87.1056900024414,
      "activations/layer4_attention_weight_max": 100.17436981201172,
      "activations/layer4_attention_weight_min": -93.52178192138672,
      "activations/layer5_attention_weight_max": 102.37560272216797,
      "activations/layer5_attention_weight_min": -90.49978637695312,
      "activations/layer6_attention_weight_max": 56.25697708129883,
      "activations/layer6_attention_weight_min": -50.71213150024414,
      "activations/layer7_attention_weight_max": 66.029052734375,
      "activations/layer7_attention_weight_min": -64.92741394042969,
      "activations/layer8_attention_weight_max": 39.68212127685547,
      "activations/layer8_attention_weight_min": -38.2823486328125,
      "activations/layer9_attention_weight_max": 49.574405670166016,
      "activations/layer9_attention_weight_min": -43.69521713256836,
      "epoch": 4.48,
      "learning_rate": 0.0001223219696969697,
      "loss": 2.8774,
      "step": 77150
    },
    {
      "activations/layer0_attention_weight_max": 15.73129653930664,
      "activations/layer0_attention_weight_min": -14.574698448181152,
      "activations/layer10_attention_weight_max": 38.976497650146484,
      "activations/layer10_attention_weight_min": -33.379783630371094,
      "activations/layer11_attention_weight_max": 30.565536499023438,
      "activations/layer11_attention_weight_min": -26.778339385986328,
      "activations/layer12_attention_weight_max": 28.5439510345459,
      "activations/layer12_attention_weight_min": -25.695308685302734,
      "activations/layer13_attention_weight_max": 62.444374084472656,
      "activations/layer13_attention_weight_min": -41.58223342895508,
      "activations/layer14_attention_weight_max": 46.59408950805664,
      "activations/layer14_attention_weight_min": -29.971059799194336,
      "activations/layer15_attention_weight_max": 42.876522064208984,
      "activations/layer15_attention_weight_min": -32.03438949584961,
      "activations/layer16_attention_weight_max": 36.14533996582031,
      "activations/layer16_attention_weight_min": -28.94599723815918,
      "activations/layer17_attention_weight_max": 64.15642547607422,
      "activations/layer17_attention_weight_min": -38.662010192871094,
      "activations/layer18_attention_weight_max": 51.19976806640625,
      "activations/layer18_attention_weight_min": -34.84695816040039,
      "activations/layer19_attention_weight_max": 26.945180892944336,
      "activations/layer19_attention_weight_min": -19.9736328125,
      "activations/layer1_attention_weight_max": 18.193056106567383,
      "activations/layer1_attention_weight_min": -16.284786224365234,
      "activations/layer20_attention_weight_max": 23.460277557373047,
      "activations/layer20_attention_weight_min": -19.64018440246582,
      "activations/layer21_attention_weight_max": 46.48054122924805,
      "activations/layer21_attention_weight_min": -30.32921600341797,
      "activations/layer22_attention_weight_max": 37.87667465209961,
      "activations/layer22_attention_weight_min": -24.341720581054688,
      "activations/layer23_attention_weight_max": 31.59510612487793,
      "activations/layer23_attention_weight_min": -19.716873168945312,
      "activations/layer2_attention_weight_max": 31.773252487182617,
      "activations/layer2_attention_weight_min": -28.236738204956055,
      "activations/layer3_attention_weight_max": 80.09400939941406,
      "activations/layer3_attention_weight_min": -83.2716064453125,
      "activations/layer4_attention_weight_max": 99.48958587646484,
      "activations/layer4_attention_weight_min": -94.57492065429688,
      "activations/layer5_attention_weight_max": 105.85804748535156,
      "activations/layer5_attention_weight_min": -96.14903259277344,
      "activations/layer6_attention_weight_max": 57.7021369934082,
      "activations/layer6_attention_weight_min": -51.528358459472656,
      "activations/layer7_attention_weight_max": 66.06359100341797,
      "activations/layer7_attention_weight_min": -64.18875122070312,
      "activations/layer8_attention_weight_max": 42.39304733276367,
      "activations/layer8_attention_weight_min": -41.68351745605469,
      "activations/layer9_attention_weight_max": 49.79233932495117,
      "activations/layer9_attention_weight_min": -47.6181640625,
      "epoch": 4.49,
      "learning_rate": 0.0001223030303030303,
      "loss": 2.8596,
      "step": 77200
    },
    {
      "activations/layer0_attention_weight_max": 15.509827613830566,
      "activations/layer0_attention_weight_min": -14.660150527954102,
      "activations/layer10_attention_weight_max": 35.00816345214844,
      "activations/layer10_attention_weight_min": -32.62937545776367,
      "activations/layer11_attention_weight_max": 26.94784164428711,
      "activations/layer11_attention_weight_min": -26.127960205078125,
      "activations/layer12_attention_weight_max": 22.334461212158203,
      "activations/layer12_attention_weight_min": -25.009428024291992,
      "activations/layer13_attention_weight_max": 43.25672149658203,
      "activations/layer13_attention_weight_min": -36.45643615722656,
      "activations/layer14_attention_weight_max": 41.31724548339844,
      "activations/layer14_attention_weight_min": -29.508352279663086,
      "activations/layer15_attention_weight_max": 39.06126022338867,
      "activations/layer15_attention_weight_min": -29.091886520385742,
      "activations/layer16_attention_weight_max": 37.7166862487793,
      "activations/layer16_attention_weight_min": -25.779958724975586,
      "activations/layer17_attention_weight_max": 56.3879508972168,
      "activations/layer17_attention_weight_min": -35.27745819091797,
      "activations/layer18_attention_weight_max": 46.85725021362305,
      "activations/layer18_attention_weight_min": -31.154098510742188,
      "activations/layer19_attention_weight_max": 21.055723190307617,
      "activations/layer19_attention_weight_min": -19.122390747070312,
      "activations/layer1_attention_weight_max": 17.289657592773438,
      "activations/layer1_attention_weight_min": -13.90974235534668,
      "activations/layer20_attention_weight_max": 20.378080368041992,
      "activations/layer20_attention_weight_min": -22.17410659790039,
      "activations/layer21_attention_weight_max": 38.14246368408203,
      "activations/layer21_attention_weight_min": -25.024051666259766,
      "activations/layer22_attention_weight_max": 34.736331939697266,
      "activations/layer22_attention_weight_min": -26.535932540893555,
      "activations/layer23_attention_weight_max": 25.048572540283203,
      "activations/layer23_attention_weight_min": -17.827253341674805,
      "activations/layer2_attention_weight_max": 29.00495719909668,
      "activations/layer2_attention_weight_min": -26.369770050048828,
      "activations/layer3_attention_weight_max": 81.01624298095703,
      "activations/layer3_attention_weight_min": -79.4595718383789,
      "activations/layer4_attention_weight_max": 103.00604248046875,
      "activations/layer4_attention_weight_min": -91.82951354980469,
      "activations/layer5_attention_weight_max": 108.11741638183594,
      "activations/layer5_attention_weight_min": -90.51296997070312,
      "activations/layer6_attention_weight_max": 59.268768310546875,
      "activations/layer6_attention_weight_min": -50.41913986206055,
      "activations/layer7_attention_weight_max": 64.34282684326172,
      "activations/layer7_attention_weight_min": -61.88511276245117,
      "activations/layer8_attention_weight_max": 39.20256042480469,
      "activations/layer8_attention_weight_min": -39.1084098815918,
      "activations/layer9_attention_weight_max": 47.60658264160156,
      "activations/layer9_attention_weight_min": -44.297359466552734,
      "epoch": 4.49,
      "learning_rate": 0.0001222840909090909,
      "loss": 2.8432,
      "step": 77250
    },
    {
      "activations/layer0_attention_weight_max": 13.560909271240234,
      "activations/layer0_attention_weight_min": -14.220114707946777,
      "activations/layer10_attention_weight_max": 39.91324996948242,
      "activations/layer10_attention_weight_min": -33.10163116455078,
      "activations/layer11_attention_weight_max": 30.50597381591797,
      "activations/layer11_attention_weight_min": -25.708080291748047,
      "activations/layer12_attention_weight_max": 26.800132751464844,
      "activations/layer12_attention_weight_min": -22.67646026611328,
      "activations/layer13_attention_weight_max": 50.650840759277344,
      "activations/layer13_attention_weight_min": -36.45458221435547,
      "activations/layer14_attention_weight_max": 48.351966857910156,
      "activations/layer14_attention_weight_min": -29.319686889648438,
      "activations/layer15_attention_weight_max": 41.350040435791016,
      "activations/layer15_attention_weight_min": -30.526721954345703,
      "activations/layer16_attention_weight_max": 33.910606384277344,
      "activations/layer16_attention_weight_min": -26.412494659423828,
      "activations/layer17_attention_weight_max": 55.176063537597656,
      "activations/layer17_attention_weight_min": -37.05984115600586,
      "activations/layer18_attention_weight_max": 47.47824478149414,
      "activations/layer18_attention_weight_min": -34.324337005615234,
      "activations/layer19_attention_weight_max": 19.231931686401367,
      "activations/layer19_attention_weight_min": -18.657899856567383,
      "activations/layer1_attention_weight_max": 17.31766700744629,
      "activations/layer1_attention_weight_min": -15.676905632019043,
      "activations/layer20_attention_weight_max": 22.250730514526367,
      "activations/layer20_attention_weight_min": -23.762474060058594,
      "activations/layer21_attention_weight_max": 38.413108825683594,
      "activations/layer21_attention_weight_min": -24.275726318359375,
      "activations/layer22_attention_weight_max": 34.7601203918457,
      "activations/layer22_attention_weight_min": -22.64510154724121,
      "activations/layer23_attention_weight_max": 25.410762786865234,
      "activations/layer23_attention_weight_min": -17.609622955322266,
      "activations/layer2_attention_weight_max": 29.87307357788086,
      "activations/layer2_attention_weight_min": -27.85162925720215,
      "activations/layer3_attention_weight_max": 77.64790344238281,
      "activations/layer3_attention_weight_min": -81.57176208496094,
      "activations/layer4_attention_weight_max": 104.1215591430664,
      "activations/layer4_attention_weight_min": -90.19841766357422,
      "activations/layer5_attention_weight_max": 107.52426147460938,
      "activations/layer5_attention_weight_min": -92.63753509521484,
      "activations/layer6_attention_weight_max": 57.30638885498047,
      "activations/layer6_attention_weight_min": -52.327571868896484,
      "activations/layer7_attention_weight_max": 64.82029724121094,
      "activations/layer7_attention_weight_min": -59.78022384643555,
      "activations/layer8_attention_weight_max": 41.72678756713867,
      "activations/layer8_attention_weight_min": -39.7117805480957,
      "activations/layer9_attention_weight_max": 47.23765563964844,
      "activations/layer9_attention_weight_min": -44.00769805908203,
      "epoch": 4.49,
      "learning_rate": 0.0001222651515151515,
      "loss": 2.8579,
      "step": 77300
    },
    {
      "activations/layer0_attention_weight_max": 15.626887321472168,
      "activations/layer0_attention_weight_min": -14.544747352600098,
      "activations/layer10_attention_weight_max": 37.83745193481445,
      "activations/layer10_attention_weight_min": -33.197872161865234,
      "activations/layer11_attention_weight_max": 29.082046508789062,
      "activations/layer11_attention_weight_min": -25.79450225830078,
      "activations/layer12_attention_weight_max": 22.307918548583984,
      "activations/layer12_attention_weight_min": -26.021909713745117,
      "activations/layer13_attention_weight_max": 54.90718078613281,
      "activations/layer13_attention_weight_min": -38.14826202392578,
      "activations/layer14_attention_weight_max": 41.117835998535156,
      "activations/layer14_attention_weight_min": -29.98833656311035,
      "activations/layer15_attention_weight_max": 38.91933822631836,
      "activations/layer15_attention_weight_min": -30.86048698425293,
      "activations/layer16_attention_weight_max": 35.1278190612793,
      "activations/layer16_attention_weight_min": -27.04465675354004,
      "activations/layer17_attention_weight_max": 54.307029724121094,
      "activations/layer17_attention_weight_min": -38.42277526855469,
      "activations/layer18_attention_weight_max": 45.396942138671875,
      "activations/layer18_attention_weight_min": -31.297842025756836,
      "activations/layer19_attention_weight_max": 21.977493286132812,
      "activations/layer19_attention_weight_min": -24.70697593688965,
      "activations/layer1_attention_weight_max": 17.763914108276367,
      "activations/layer1_attention_weight_min": -16.421838760375977,
      "activations/layer20_attention_weight_max": 23.270849227905273,
      "activations/layer20_attention_weight_min": -23.825035095214844,
      "activations/layer21_attention_weight_max": 38.275474548339844,
      "activations/layer21_attention_weight_min": -31.06624984741211,
      "activations/layer22_attention_weight_max": 40.91273498535156,
      "activations/layer22_attention_weight_min": -25.226776123046875,
      "activations/layer23_attention_weight_max": 27.09876823425293,
      "activations/layer23_attention_weight_min": -17.317411422729492,
      "activations/layer2_attention_weight_max": 30.171918869018555,
      "activations/layer2_attention_weight_min": -27.852537155151367,
      "activations/layer3_attention_weight_max": 79.14091491699219,
      "activations/layer3_attention_weight_min": -81.5361099243164,
      "activations/layer4_attention_weight_max": 100.75728607177734,
      "activations/layer4_attention_weight_min": -95.3053970336914,
      "activations/layer5_attention_weight_max": 109.07708740234375,
      "activations/layer5_attention_weight_min": -93.06890106201172,
      "activations/layer6_attention_weight_max": 59.827796936035156,
      "activations/layer6_attention_weight_min": -50.74848937988281,
      "activations/layer7_attention_weight_max": 67.50724792480469,
      "activations/layer7_attention_weight_min": -62.16804885864258,
      "activations/layer8_attention_weight_max": 41.306861877441406,
      "activations/layer8_attention_weight_min": -39.685909271240234,
      "activations/layer9_attention_weight_max": 48.129878997802734,
      "activations/layer9_attention_weight_min": -46.49897003173828,
      "epoch": 4.49,
      "learning_rate": 0.00012224621212121212,
      "loss": 2.8627,
      "step": 77350
    },
    {
      "activations/layer0_attention_weight_max": 14.97692584991455,
      "activations/layer0_attention_weight_min": -14.705557823181152,
      "activations/layer10_attention_weight_max": 40.18170166015625,
      "activations/layer10_attention_weight_min": -35.725341796875,
      "activations/layer11_attention_weight_max": 30.62708854675293,
      "activations/layer11_attention_weight_min": -27.904216766357422,
      "activations/layer12_attention_weight_max": 20.588666915893555,
      "activations/layer12_attention_weight_min": -24.346460342407227,
      "activations/layer13_attention_weight_max": 47.41801071166992,
      "activations/layer13_attention_weight_min": -37.12196350097656,
      "activations/layer14_attention_weight_max": 43.94532775878906,
      "activations/layer14_attention_weight_min": -31.871261596679688,
      "activations/layer15_attention_weight_max": 40.03660202026367,
      "activations/layer15_attention_weight_min": -31.399110794067383,
      "activations/layer16_attention_weight_max": 32.96965408325195,
      "activations/layer16_attention_weight_min": -27.48395538330078,
      "activations/layer17_attention_weight_max": 61.16809844970703,
      "activations/layer17_attention_weight_min": -39.06000900268555,
      "activations/layer18_attention_weight_max": 46.5760612487793,
      "activations/layer18_attention_weight_min": -34.497535705566406,
      "activations/layer19_attention_weight_max": 21.519771575927734,
      "activations/layer19_attention_weight_min": -19.430498123168945,
      "activations/layer1_attention_weight_max": 17.602121353149414,
      "activations/layer1_attention_weight_min": -15.218133926391602,
      "activations/layer20_attention_weight_max": 21.76692771911621,
      "activations/layer20_attention_weight_min": -24.49007797241211,
      "activations/layer21_attention_weight_max": 38.36342239379883,
      "activations/layer21_attention_weight_min": -30.707672119140625,
      "activations/layer22_attention_weight_max": 31.64504051208496,
      "activations/layer22_attention_weight_min": -23.044025421142578,
      "activations/layer23_attention_weight_max": 25.678220748901367,
      "activations/layer23_attention_weight_min": -16.933761596679688,
      "activations/layer2_attention_weight_max": 30.173505783081055,
      "activations/layer2_attention_weight_min": -28.217235565185547,
      "activations/layer3_attention_weight_max": 75.33724975585938,
      "activations/layer3_attention_weight_min": -80.32249450683594,
      "activations/layer4_attention_weight_max": 96.14049530029297,
      "activations/layer4_attention_weight_min": -91.25865936279297,
      "activations/layer5_attention_weight_max": 106.71597290039062,
      "activations/layer5_attention_weight_min": -90.5024642944336,
      "activations/layer6_attention_weight_max": 55.94312286376953,
      "activations/layer6_attention_weight_min": -50.0777473449707,
      "activations/layer7_attention_weight_max": 70.97806549072266,
      "activations/layer7_attention_weight_min": -63.034568786621094,
      "activations/layer8_attention_weight_max": 42.28498458862305,
      "activations/layer8_attention_weight_min": -38.5316047668457,
      "activations/layer9_attention_weight_max": 68.97773742675781,
      "activations/layer9_attention_weight_min": -45.3218879699707,
      "epoch": 4.5,
      "learning_rate": 0.0001222272727272727,
      "loss": 2.8495,
      "step": 77400
    },
    {
      "activations/layer0_attention_weight_max": 14.463887214660645,
      "activations/layer0_attention_weight_min": -14.569149017333984,
      "activations/layer10_attention_weight_max": 40.65583038330078,
      "activations/layer10_attention_weight_min": -37.68467712402344,
      "activations/layer11_attention_weight_max": 31.80413818359375,
      "activations/layer11_attention_weight_min": -27.025508880615234,
      "activations/layer12_attention_weight_max": 21.85371208190918,
      "activations/layer12_attention_weight_min": -29.0836238861084,
      "activations/layer13_attention_weight_max": 61.697025299072266,
      "activations/layer13_attention_weight_min": -42.943138122558594,
      "activations/layer14_attention_weight_max": 59.656654357910156,
      "activations/layer14_attention_weight_min": -39.87671661376953,
      "activations/layer15_attention_weight_max": 41.81779861450195,
      "activations/layer15_attention_weight_min": -31.421377182006836,
      "activations/layer16_attention_weight_max": 33.70183181762695,
      "activations/layer16_attention_weight_min": -26.79996681213379,
      "activations/layer17_attention_weight_max": 55.8427619934082,
      "activations/layer17_attention_weight_min": -36.03804397583008,
      "activations/layer18_attention_weight_max": 48.138946533203125,
      "activations/layer18_attention_weight_min": -33.151981353759766,
      "activations/layer19_attention_weight_max": 22.567922592163086,
      "activations/layer19_attention_weight_min": -19.583105087280273,
      "activations/layer1_attention_weight_max": 17.64995574951172,
      "activations/layer1_attention_weight_min": -15.214256286621094,
      "activations/layer20_attention_weight_max": 19.23297119140625,
      "activations/layer20_attention_weight_min": -21.827556610107422,
      "activations/layer21_attention_weight_max": 39.10795974731445,
      "activations/layer21_attention_weight_min": -31.071630477905273,
      "activations/layer22_attention_weight_max": 33.475746154785156,
      "activations/layer22_attention_weight_min": -24.519210815429688,
      "activations/layer23_attention_weight_max": 27.358444213867188,
      "activations/layer23_attention_weight_min": -18.96868324279785,
      "activations/layer2_attention_weight_max": 31.152997970581055,
      "activations/layer2_attention_weight_min": -27.986242294311523,
      "activations/layer3_attention_weight_max": 77.46847534179688,
      "activations/layer3_attention_weight_min": -82.11219024658203,
      "activations/layer4_attention_weight_max": 99.54320526123047,
      "activations/layer4_attention_weight_min": -94.6134262084961,
      "activations/layer5_attention_weight_max": 103.87960052490234,
      "activations/layer5_attention_weight_min": -94.07510375976562,
      "activations/layer6_attention_weight_max": 57.80591583251953,
      "activations/layer6_attention_weight_min": -52.99163055419922,
      "activations/layer7_attention_weight_max": 73.15125274658203,
      "activations/layer7_attention_weight_min": -66.73945617675781,
      "activations/layer8_attention_weight_max": 46.910831451416016,
      "activations/layer8_attention_weight_min": -43.62788009643555,
      "activations/layer9_attention_weight_max": 63.18213653564453,
      "activations/layer9_attention_weight_min": -55.59394454956055,
      "epoch": 4.5,
      "learning_rate": 0.0001222087121212121,
      "loss": 2.8623,
      "step": 77450
    },
    {
      "activations/layer0_attention_weight_max": 16.630037307739258,
      "activations/layer0_attention_weight_min": -14.94827651977539,
      "activations/layer10_attention_weight_max": 38.15579605102539,
      "activations/layer10_attention_weight_min": -32.45927810668945,
      "activations/layer11_attention_weight_max": 31.475217819213867,
      "activations/layer11_attention_weight_min": -26.257747650146484,
      "activations/layer12_attention_weight_max": 23.065200805664062,
      "activations/layer12_attention_weight_min": -25.774723052978516,
      "activations/layer13_attention_weight_max": 53.336856842041016,
      "activations/layer13_attention_weight_min": -45.023738861083984,
      "activations/layer14_attention_weight_max": 44.60270690917969,
      "activations/layer14_attention_weight_min": -33.284759521484375,
      "activations/layer15_attention_weight_max": 44.84051513671875,
      "activations/layer15_attention_weight_min": -31.064517974853516,
      "activations/layer16_attention_weight_max": 40.28860855102539,
      "activations/layer16_attention_weight_min": -25.294635772705078,
      "activations/layer17_attention_weight_max": 57.33538818359375,
      "activations/layer17_attention_weight_min": -36.698280334472656,
      "activations/layer18_attention_weight_max": 50.10689163208008,
      "activations/layer18_attention_weight_min": -30.073421478271484,
      "activations/layer19_attention_weight_max": 29.306814193725586,
      "activations/layer19_attention_weight_min": -18.263994216918945,
      "activations/layer1_attention_weight_max": 17.158859252929688,
      "activations/layer1_attention_weight_min": -13.76999568939209,
      "activations/layer20_attention_weight_max": 24.076156616210938,
      "activations/layer20_attention_weight_min": -18.76701545715332,
      "activations/layer21_attention_weight_max": 41.00323486328125,
      "activations/layer21_attention_weight_min": -23.25766944885254,
      "activations/layer22_attention_weight_max": 39.023189544677734,
      "activations/layer22_attention_weight_min": -23.10089111328125,
      "activations/layer23_attention_weight_max": 26.558950424194336,
      "activations/layer23_attention_weight_min": -16.946659088134766,
      "activations/layer2_attention_weight_max": 29.74312973022461,
      "activations/layer2_attention_weight_min": -28.816097259521484,
      "activations/layer3_attention_weight_max": 78.22956848144531,
      "activations/layer3_attention_weight_min": -84.83495330810547,
      "activations/layer4_attention_weight_max": 99.89360809326172,
      "activations/layer4_attention_weight_min": -98.84798431396484,
      "activations/layer5_attention_weight_max": 107.96504211425781,
      "activations/layer5_attention_weight_min": -98.7280502319336,
      "activations/layer6_attention_weight_max": 57.732181549072266,
      "activations/layer6_attention_weight_min": -53.533573150634766,
      "activations/layer7_attention_weight_max": 65.09751892089844,
      "activations/layer7_attention_weight_min": -63.1165657043457,
      "activations/layer8_attention_weight_max": 40.61977005004883,
      "activations/layer8_attention_weight_min": -39.75862503051758,
      "activations/layer9_attention_weight_max": 53.314144134521484,
      "activations/layer9_attention_weight_min": -46.883609771728516,
      "epoch": 4.5,
      "learning_rate": 0.00012218977272727272,
      "loss": 2.8799,
      "step": 77500
    },
    {
      "activations/layer0_attention_weight_max": 14.520657539367676,
      "activations/layer0_attention_weight_min": -14.658312797546387,
      "activations/layer10_attention_weight_max": 36.912166595458984,
      "activations/layer10_attention_weight_min": -32.82707595825195,
      "activations/layer11_attention_weight_max": 31.651687622070312,
      "activations/layer11_attention_weight_min": -27.153945922851562,
      "activations/layer12_attention_weight_max": 24.065996170043945,
      "activations/layer12_attention_weight_min": -28.29936981201172,
      "activations/layer13_attention_weight_max": 43.6738395690918,
      "activations/layer13_attention_weight_min": -31.88589096069336,
      "activations/layer14_attention_weight_max": 51.38082504272461,
      "activations/layer14_attention_weight_min": -30.436565399169922,
      "activations/layer15_attention_weight_max": 45.36944580078125,
      "activations/layer15_attention_weight_min": -29.566640853881836,
      "activations/layer16_attention_weight_max": 33.74828338623047,
      "activations/layer16_attention_weight_min": -26.521989822387695,
      "activations/layer17_attention_weight_max": 56.378875732421875,
      "activations/layer17_attention_weight_min": -34.83877182006836,
      "activations/layer18_attention_weight_max": 46.40778732299805,
      "activations/layer18_attention_weight_min": -30.422189712524414,
      "activations/layer19_attention_weight_max": 23.683393478393555,
      "activations/layer19_attention_weight_min": -19.49091911315918,
      "activations/layer1_attention_weight_max": 16.8817081451416,
      "activations/layer1_attention_weight_min": -15.306190490722656,
      "activations/layer20_attention_weight_max": 21.07560920715332,
      "activations/layer20_attention_weight_min": -21.551830291748047,
      "activations/layer21_attention_weight_max": 40.79554748535156,
      "activations/layer21_attention_weight_min": -23.957571029663086,
      "activations/layer22_attention_weight_max": 34.682762145996094,
      "activations/layer22_attention_weight_min": -22.156301498413086,
      "activations/layer23_attention_weight_max": 26.98009490966797,
      "activations/layer23_attention_weight_min": -16.004791259765625,
      "activations/layer2_attention_weight_max": 29.960521697998047,
      "activations/layer2_attention_weight_min": -27.67865562438965,
      "activations/layer3_attention_weight_max": 79.31718444824219,
      "activations/layer3_attention_weight_min": -79.01176452636719,
      "activations/layer4_attention_weight_max": 94.00875091552734,
      "activations/layer4_attention_weight_min": -90.98311614990234,
      "activations/layer5_attention_weight_max": 100.60213470458984,
      "activations/layer5_attention_weight_min": -94.20637512207031,
      "activations/layer6_attention_weight_max": 58.42947769165039,
      "activations/layer6_attention_weight_min": -54.10569763183594,
      "activations/layer7_attention_weight_max": 66.68878936767578,
      "activations/layer7_attention_weight_min": -62.91925048828125,
      "activations/layer8_attention_weight_max": 37.87305450439453,
      "activations/layer8_attention_weight_min": -38.662296295166016,
      "activations/layer9_attention_weight_max": 50.1370964050293,
      "activations/layer9_attention_weight_min": -45.07483673095703,
      "epoch": 4.51,
      "learning_rate": 0.00012217083333333333,
      "loss": 2.8657,
      "step": 77550
    },
    {
      "activations/layer0_attention_weight_max": 15.346407890319824,
      "activations/layer0_attention_weight_min": -14.85706615447998,
      "activations/layer10_attention_weight_max": 41.3814582824707,
      "activations/layer10_attention_weight_min": -34.5030632019043,
      "activations/layer11_attention_weight_max": 30.758543014526367,
      "activations/layer11_attention_weight_min": -28.055051803588867,
      "activations/layer12_attention_weight_max": 30.250511169433594,
      "activations/layer12_attention_weight_min": -25.83079719543457,
      "activations/layer13_attention_weight_max": 49.79716873168945,
      "activations/layer13_attention_weight_min": -34.798187255859375,
      "activations/layer14_attention_weight_max": 44.42228698730469,
      "activations/layer14_attention_weight_min": -30.279926300048828,
      "activations/layer15_attention_weight_max": 42.19446563720703,
      "activations/layer15_attention_weight_min": -29.514419555664062,
      "activations/layer16_attention_weight_max": 34.140933990478516,
      "activations/layer16_attention_weight_min": -25.345294952392578,
      "activations/layer17_attention_weight_max": 57.6314582824707,
      "activations/layer17_attention_weight_min": -37.81502151489258,
      "activations/layer18_attention_weight_max": 51.72554397583008,
      "activations/layer18_attention_weight_min": -32.9955940246582,
      "activations/layer19_attention_weight_max": 21.848485946655273,
      "activations/layer19_attention_weight_min": -17.201553344726562,
      "activations/layer1_attention_weight_max": 17.312219619750977,
      "activations/layer1_attention_weight_min": -15.443050384521484,
      "activations/layer20_attention_weight_max": 22.84777069091797,
      "activations/layer20_attention_weight_min": -21.539836883544922,
      "activations/layer21_attention_weight_max": 45.80075454711914,
      "activations/layer21_attention_weight_min": -33.49468994140625,
      "activations/layer22_attention_weight_max": 39.334144592285156,
      "activations/layer22_attention_weight_min": -24.088518142700195,
      "activations/layer23_attention_weight_max": 30.401142120361328,
      "activations/layer23_attention_weight_min": -20.135316848754883,
      "activations/layer2_attention_weight_max": 31.646574020385742,
      "activations/layer2_attention_weight_min": -29.278873443603516,
      "activations/layer3_attention_weight_max": 84.93047332763672,
      "activations/layer3_attention_weight_min": -91.88302612304688,
      "activations/layer4_attention_weight_max": 104.226806640625,
      "activations/layer4_attention_weight_min": -96.59664154052734,
      "activations/layer5_attention_weight_max": 109.34718322753906,
      "activations/layer5_attention_weight_min": -97.60711669921875,
      "activations/layer6_attention_weight_max": 61.866424560546875,
      "activations/layer6_attention_weight_min": -53.74980163574219,
      "activations/layer7_attention_weight_max": 71.94710540771484,
      "activations/layer7_attention_weight_min": -70.15621948242188,
      "activations/layer8_attention_weight_max": 50.35005569458008,
      "activations/layer8_attention_weight_min": -43.98383712768555,
      "activations/layer9_attention_weight_max": 57.9456787109375,
      "activations/layer9_attention_weight_min": -50.277244567871094,
      "epoch": 4.51,
      "learning_rate": 0.00012215189393939392,
      "loss": 2.8634,
      "step": 77600
    },
    {
      "activations/layer0_attention_weight_max": 15.078081130981445,
      "activations/layer0_attention_weight_min": -14.73765754699707,
      "activations/layer10_attention_weight_max": 42.06600570678711,
      "activations/layer10_attention_weight_min": -33.95860290527344,
      "activations/layer11_attention_weight_max": 28.53030014038086,
      "activations/layer11_attention_weight_min": -26.22688865661621,
      "activations/layer12_attention_weight_max": 31.710668563842773,
      "activations/layer12_attention_weight_min": -26.530452728271484,
      "activations/layer13_attention_weight_max": 49.38319778442383,
      "activations/layer13_attention_weight_min": -34.826629638671875,
      "activations/layer14_attention_weight_max": 49.18576431274414,
      "activations/layer14_attention_weight_min": -31.46170425415039,
      "activations/layer15_attention_weight_max": 50.00639343261719,
      "activations/layer15_attention_weight_min": -32.32773971557617,
      "activations/layer16_attention_weight_max": 38.236236572265625,
      "activations/layer16_attention_weight_min": -25.364410400390625,
      "activations/layer17_attention_weight_max": 57.17473220825195,
      "activations/layer17_attention_weight_min": -37.40543746948242,
      "activations/layer18_attention_weight_max": 49.35900115966797,
      "activations/layer18_attention_weight_min": -31.497655868530273,
      "activations/layer19_attention_weight_max": 23.659818649291992,
      "activations/layer19_attention_weight_min": -19.530941009521484,
      "activations/layer1_attention_weight_max": 18.827651977539062,
      "activations/layer1_attention_weight_min": -16.9676570892334,
      "activations/layer20_attention_weight_max": 20.66351318359375,
      "activations/layer20_attention_weight_min": -21.146024703979492,
      "activations/layer21_attention_weight_max": 45.23173141479492,
      "activations/layer21_attention_weight_min": -26.022422790527344,
      "activations/layer22_attention_weight_max": 34.86935806274414,
      "activations/layer22_attention_weight_min": -23.74159812927246,
      "activations/layer23_attention_weight_max": 25.911550521850586,
      "activations/layer23_attention_weight_min": -17.837142944335938,
      "activations/layer2_attention_weight_max": 31.261302947998047,
      "activations/layer2_attention_weight_min": -30.233226776123047,
      "activations/layer3_attention_weight_max": 80.37504577636719,
      "activations/layer3_attention_weight_min": -82.06500244140625,
      "activations/layer4_attention_weight_max": 103.40388488769531,
      "activations/layer4_attention_weight_min": -95.59803009033203,
      "activations/layer5_attention_weight_max": 110.58543395996094,
      "activations/layer5_attention_weight_min": -97.70326232910156,
      "activations/layer6_attention_weight_max": 60.75778579711914,
      "activations/layer6_attention_weight_min": -53.32521438598633,
      "activations/layer7_attention_weight_max": 69.7037582397461,
      "activations/layer7_attention_weight_min": -64.159912109375,
      "activations/layer8_attention_weight_max": 45.63859558105469,
      "activations/layer8_attention_weight_min": -43.23944091796875,
      "activations/layer9_attention_weight_max": 48.14714813232422,
      "activations/layer9_attention_weight_min": -46.42129898071289,
      "epoch": 4.51,
      "learning_rate": 0.00012213295454545454,
      "loss": 2.8908,
      "step": 77650
    },
    {
      "activations/layer0_attention_weight_max": 14.879010200500488,
      "activations/layer0_attention_weight_min": -15.236114501953125,
      "activations/layer10_attention_weight_max": 38.71648025512695,
      "activations/layer10_attention_weight_min": -32.714447021484375,
      "activations/layer11_attention_weight_max": 32.16217803955078,
      "activations/layer11_attention_weight_min": -25.687210083007812,
      "activations/layer12_attention_weight_max": 24.45965576171875,
      "activations/layer12_attention_weight_min": -23.02246856689453,
      "activations/layer13_attention_weight_max": 47.64011001586914,
      "activations/layer13_attention_weight_min": -36.01045227050781,
      "activations/layer14_attention_weight_max": 41.596561431884766,
      "activations/layer14_attention_weight_min": -28.124483108520508,
      "activations/layer15_attention_weight_max": 40.78312683105469,
      "activations/layer15_attention_weight_min": -30.73938751220703,
      "activations/layer16_attention_weight_max": 37.796382904052734,
      "activations/layer16_attention_weight_min": -25.681644439697266,
      "activations/layer17_attention_weight_max": 54.81999588012695,
      "activations/layer17_attention_weight_min": -38.12953567504883,
      "activations/layer18_attention_weight_max": 49.13580322265625,
      "activations/layer18_attention_weight_min": -32.82279968261719,
      "activations/layer19_attention_weight_max": 22.740333557128906,
      "activations/layer19_attention_weight_min": -22.9144229888916,
      "activations/layer1_attention_weight_max": 17.939189910888672,
      "activations/layer1_attention_weight_min": -14.387086868286133,
      "activations/layer20_attention_weight_max": 23.858844757080078,
      "activations/layer20_attention_weight_min": -20.17135238647461,
      "activations/layer21_attention_weight_max": 38.4472541809082,
      "activations/layer21_attention_weight_min": -28.5793514251709,
      "activations/layer22_attention_weight_max": 38.52574920654297,
      "activations/layer22_attention_weight_min": -23.694408416748047,
      "activations/layer23_attention_weight_max": 26.297042846679688,
      "activations/layer23_attention_weight_min": -16.664216995239258,
      "activations/layer2_attention_weight_max": 32.829158782958984,
      "activations/layer2_attention_weight_min": -31.254810333251953,
      "activations/layer3_attention_weight_max": 81.1419448852539,
      "activations/layer3_attention_weight_min": -85.65438842773438,
      "activations/layer4_attention_weight_max": 103.90093231201172,
      "activations/layer4_attention_weight_min": -96.74063873291016,
      "activations/layer5_attention_weight_max": 109.59419250488281,
      "activations/layer5_attention_weight_min": -98.19157409667969,
      "activations/layer6_attention_weight_max": 60.935211181640625,
      "activations/layer6_attention_weight_min": -55.84626388549805,
      "activations/layer7_attention_weight_max": 75.44584655761719,
      "activations/layer7_attention_weight_min": -66.29292297363281,
      "activations/layer8_attention_weight_max": 46.86128616333008,
      "activations/layer8_attention_weight_min": -39.47374725341797,
      "activations/layer9_attention_weight_max": 60.043575286865234,
      "activations/layer9_attention_weight_min": -49.001922607421875,
      "epoch": 4.51,
      "learning_rate": 0.00012211401515151515,
      "loss": 2.8747,
      "step": 77700
    },
    {
      "activations/layer0_attention_weight_max": 15.843809127807617,
      "activations/layer0_attention_weight_min": -15.516922950744629,
      "activations/layer10_attention_weight_max": 38.33011245727539,
      "activations/layer10_attention_weight_min": -33.719573974609375,
      "activations/layer11_attention_weight_max": 29.431562423706055,
      "activations/layer11_attention_weight_min": -25.76474380493164,
      "activations/layer12_attention_weight_max": 25.848154067993164,
      "activations/layer12_attention_weight_min": -27.03504753112793,
      "activations/layer13_attention_weight_max": 47.393009185791016,
      "activations/layer13_attention_weight_min": -33.059112548828125,
      "activations/layer14_attention_weight_max": 49.81861877441406,
      "activations/layer14_attention_weight_min": -30.352188110351562,
      "activations/layer15_attention_weight_max": 40.78106689453125,
      "activations/layer15_attention_weight_min": -30.191116333007812,
      "activations/layer16_attention_weight_max": 37.54324722290039,
      "activations/layer16_attention_weight_min": -26.152368545532227,
      "activations/layer17_attention_weight_max": 59.39912414550781,
      "activations/layer17_attention_weight_min": -38.09095764160156,
      "activations/layer18_attention_weight_max": 51.43333435058594,
      "activations/layer18_attention_weight_min": -34.05826187133789,
      "activations/layer19_attention_weight_max": 23.336097717285156,
      "activations/layer19_attention_weight_min": -18.901229858398438,
      "activations/layer1_attention_weight_max": 18.103788375854492,
      "activations/layer1_attention_weight_min": -16.397193908691406,
      "activations/layer20_attention_weight_max": 22.330766677856445,
      "activations/layer20_attention_weight_min": -23.567140579223633,
      "activations/layer21_attention_weight_max": 41.53203201293945,
      "activations/layer21_attention_weight_min": -26.194005966186523,
      "activations/layer22_attention_weight_max": 39.02743911743164,
      "activations/layer22_attention_weight_min": -25.418970108032227,
      "activations/layer23_attention_weight_max": 27.234594345092773,
      "activations/layer23_attention_weight_min": -17.4669246673584,
      "activations/layer2_attention_weight_max": 32.53249740600586,
      "activations/layer2_attention_weight_min": -27.851215362548828,
      "activations/layer3_attention_weight_max": 75.48413848876953,
      "activations/layer3_attention_weight_min": -78.6031265258789,
      "activations/layer4_attention_weight_max": 94.14855194091797,
      "activations/layer4_attention_weight_min": -88.0715103149414,
      "activations/layer5_attention_weight_max": 99.61456298828125,
      "activations/layer5_attention_weight_min": -89.78659057617188,
      "activations/layer6_attention_weight_max": 57.12104415893555,
      "activations/layer6_attention_weight_min": -50.21198272705078,
      "activations/layer7_attention_weight_max": 66.54528045654297,
      "activations/layer7_attention_weight_min": -63.50199890136719,
      "activations/layer8_attention_weight_max": 39.25653839111328,
      "activations/layer8_attention_weight_min": -37.095069885253906,
      "activations/layer9_attention_weight_max": 52.031837463378906,
      "activations/layer9_attention_weight_min": -44.95895004272461,
      "epoch": 4.52,
      "learning_rate": 0.00012209507575757574,
      "loss": 2.8726,
      "step": 77750
    },
    {
      "activations/layer0_attention_weight_max": 16.76462173461914,
      "activations/layer0_attention_weight_min": -15.253059387207031,
      "activations/layer10_attention_weight_max": 39.434783935546875,
      "activations/layer10_attention_weight_min": -34.623802185058594,
      "activations/layer11_attention_weight_max": 29.452022552490234,
      "activations/layer11_attention_weight_min": -28.20326042175293,
      "activations/layer12_attention_weight_max": 26.672046661376953,
      "activations/layer12_attention_weight_min": -30.026447296142578,
      "activations/layer13_attention_weight_max": 56.94440841674805,
      "activations/layer13_attention_weight_min": -38.530609130859375,
      "activations/layer14_attention_weight_max": 45.2512092590332,
      "activations/layer14_attention_weight_min": -29.203935623168945,
      "activations/layer15_attention_weight_max": 50.984840393066406,
      "activations/layer15_attention_weight_min": -29.87579917907715,
      "activations/layer16_attention_weight_max": 42.27347946166992,
      "activations/layer16_attention_weight_min": -28.07318878173828,
      "activations/layer17_attention_weight_max": 61.15776824951172,
      "activations/layer17_attention_weight_min": -37.83567428588867,
      "activations/layer18_attention_weight_max": 53.377647399902344,
      "activations/layer18_attention_weight_min": -32.541900634765625,
      "activations/layer19_attention_weight_max": 23.913599014282227,
      "activations/layer19_attention_weight_min": -17.495922088623047,
      "activations/layer1_attention_weight_max": 18.744731903076172,
      "activations/layer1_attention_weight_min": -15.7598237991333,
      "activations/layer20_attention_weight_max": 20.67955780029297,
      "activations/layer20_attention_weight_min": -19.645357131958008,
      "activations/layer21_attention_weight_max": 36.864383697509766,
      "activations/layer21_attention_weight_min": -26.693626403808594,
      "activations/layer22_attention_weight_max": 32.18716049194336,
      "activations/layer22_attention_weight_min": -23.40097427368164,
      "activations/layer23_attention_weight_max": 27.709815979003906,
      "activations/layer23_attention_weight_min": -17.909954071044922,
      "activations/layer2_attention_weight_max": 31.880475997924805,
      "activations/layer2_attention_weight_min": -29.325483322143555,
      "activations/layer3_attention_weight_max": 81.66993713378906,
      "activations/layer3_attention_weight_min": -80.43953704833984,
      "activations/layer4_attention_weight_max": 97.40657806396484,
      "activations/layer4_attention_weight_min": -94.45877838134766,
      "activations/layer5_attention_weight_max": 109.90584564208984,
      "activations/layer5_attention_weight_min": -91.81686401367188,
      "activations/layer6_attention_weight_max": 57.38823318481445,
      "activations/layer6_attention_weight_min": -52.53681945800781,
      "activations/layer7_attention_weight_max": 66.51594543457031,
      "activations/layer7_attention_weight_min": -62.56978225708008,
      "activations/layer8_attention_weight_max": 44.145328521728516,
      "activations/layer8_attention_weight_min": -38.664794921875,
      "activations/layer9_attention_weight_max": 50.43827438354492,
      "activations/layer9_attention_weight_min": -46.41444396972656,
      "epoch": 4.52,
      "learning_rate": 0.00012207613636363635,
      "loss": 2.8613,
      "step": 77800
    },
    {
      "activations/layer0_attention_weight_max": 13.977407455444336,
      "activations/layer0_attention_weight_min": -15.253835678100586,
      "activations/layer10_attention_weight_max": 42.07114791870117,
      "activations/layer10_attention_weight_min": -33.234291076660156,
      "activations/layer11_attention_weight_max": 32.40624237060547,
      "activations/layer11_attention_weight_min": -26.133405685424805,
      "activations/layer12_attention_weight_max": 25.424341201782227,
      "activations/layer12_attention_weight_min": -24.075292587280273,
      "activations/layer13_attention_weight_max": 55.28053665161133,
      "activations/layer13_attention_weight_min": -35.865386962890625,
      "activations/layer14_attention_weight_max": 56.735965728759766,
      "activations/layer14_attention_weight_min": -32.46322250366211,
      "activations/layer15_attention_weight_max": 47.998477935791016,
      "activations/layer15_attention_weight_min": -33.633140563964844,
      "activations/layer16_attention_weight_max": 38.12098693847656,
      "activations/layer16_attention_weight_min": -27.41631507873535,
      "activations/layer17_attention_weight_max": 62.38136672973633,
      "activations/layer17_attention_weight_min": -38.842960357666016,
      "activations/layer18_attention_weight_max": 54.27614212036133,
      "activations/layer18_attention_weight_min": -35.03364562988281,
      "activations/layer19_attention_weight_max": 26.35944175720215,
      "activations/layer19_attention_weight_min": -18.93537712097168,
      "activations/layer1_attention_weight_max": 17.814620971679688,
      "activations/layer1_attention_weight_min": -17.261924743652344,
      "activations/layer20_attention_weight_max": 27.37487030029297,
      "activations/layer20_attention_weight_min": -21.003522872924805,
      "activations/layer21_attention_weight_max": 46.8780517578125,
      "activations/layer21_attention_weight_min": -25.88466453552246,
      "activations/layer22_attention_weight_max": 37.212223052978516,
      "activations/layer22_attention_weight_min": -22.574968338012695,
      "activations/layer23_attention_weight_max": 30.85376739501953,
      "activations/layer23_attention_weight_min": -17.20162582397461,
      "activations/layer2_attention_weight_max": 30.264568328857422,
      "activations/layer2_attention_weight_min": -28.019317626953125,
      "activations/layer3_attention_weight_max": 79.60333251953125,
      "activations/layer3_attention_weight_min": -88.17765808105469,
      "activations/layer4_attention_weight_max": 100.49544525146484,
      "activations/layer4_attention_weight_min": -100.13435363769531,
      "activations/layer5_attention_weight_max": 109.32237243652344,
      "activations/layer5_attention_weight_min": -100.9415283203125,
      "activations/layer6_attention_weight_max": 60.751983642578125,
      "activations/layer6_attention_weight_min": -58.013919830322266,
      "activations/layer7_attention_weight_max": 70.70440673828125,
      "activations/layer7_attention_weight_min": -69.9911880493164,
      "activations/layer8_attention_weight_max": 43.06525802612305,
      "activations/layer8_attention_weight_min": -42.4405632019043,
      "activations/layer9_attention_weight_max": 49.65238571166992,
      "activations/layer9_attention_weight_min": -47.20018005371094,
      "epoch": 4.52,
      "learning_rate": 0.00012205757575757574,
      "loss": 2.867,
      "step": 77850
    },
    {
      "activations/layer0_attention_weight_max": 15.82093620300293,
      "activations/layer0_attention_weight_min": -15.311418533325195,
      "activations/layer10_attention_weight_max": 36.90089797973633,
      "activations/layer10_attention_weight_min": -33.90718460083008,
      "activations/layer11_attention_weight_max": 27.613460540771484,
      "activations/layer11_attention_weight_min": -26.307615280151367,
      "activations/layer12_attention_weight_max": 21.46241569519043,
      "activations/layer12_attention_weight_min": -24.11368179321289,
      "activations/layer13_attention_weight_max": 44.76591110229492,
      "activations/layer13_attention_weight_min": -35.408443450927734,
      "activations/layer14_attention_weight_max": 45.371337890625,
      "activations/layer14_attention_weight_min": -30.39580726623535,
      "activations/layer15_attention_weight_max": 36.80923080444336,
      "activations/layer15_attention_weight_min": -30.752391815185547,
      "activations/layer16_attention_weight_max": 33.61191177368164,
      "activations/layer16_attention_weight_min": -26.867799758911133,
      "activations/layer17_attention_weight_max": 52.61204528808594,
      "activations/layer17_attention_weight_min": -37.342166900634766,
      "activations/layer18_attention_weight_max": 44.74709701538086,
      "activations/layer18_attention_weight_min": -33.1898307800293,
      "activations/layer19_attention_weight_max": 19.110177993774414,
      "activations/layer19_attention_weight_min": -21.128894805908203,
      "activations/layer1_attention_weight_max": 18.224958419799805,
      "activations/layer1_attention_weight_min": -15.291522026062012,
      "activations/layer20_attention_weight_max": 19.657882690429688,
      "activations/layer20_attention_weight_min": -22.906848907470703,
      "activations/layer21_attention_weight_max": 34.18000030517578,
      "activations/layer21_attention_weight_min": -31.36615562438965,
      "activations/layer22_attention_weight_max": 31.021087646484375,
      "activations/layer22_attention_weight_min": -24.67784881591797,
      "activations/layer23_attention_weight_max": 22.28573989868164,
      "activations/layer23_attention_weight_min": -18.94390869140625,
      "activations/layer2_attention_weight_max": 31.516935348510742,
      "activations/layer2_attention_weight_min": -29.02715301513672,
      "activations/layer3_attention_weight_max": 75.69593048095703,
      "activations/layer3_attention_weight_min": -78.3724594116211,
      "activations/layer4_attention_weight_max": 97.00562286376953,
      "activations/layer4_attention_weight_min": -87.42001342773438,
      "activations/layer5_attention_weight_max": 104.9271469116211,
      "activations/layer5_attention_weight_min": -93.35301971435547,
      "activations/layer6_attention_weight_max": 58.102996826171875,
      "activations/layer6_attention_weight_min": -52.1345100402832,
      "activations/layer7_attention_weight_max": 69.3749008178711,
      "activations/layer7_attention_weight_min": -65.07374572753906,
      "activations/layer8_attention_weight_max": 40.968360900878906,
      "activations/layer8_attention_weight_min": -41.91318130493164,
      "activations/layer9_attention_weight_max": 44.825565338134766,
      "activations/layer9_attention_weight_min": -43.99314880371094,
      "epoch": 4.53,
      "learning_rate": 0.00012203863636363636,
      "loss": 2.8643,
      "step": 77900
    },
    {
      "activations/layer0_attention_weight_max": 14.180305480957031,
      "activations/layer0_attention_weight_min": -14.491423606872559,
      "activations/layer10_attention_weight_max": 37.515602111816406,
      "activations/layer10_attention_weight_min": -33.16862106323242,
      "activations/layer11_attention_weight_max": 28.68854331970215,
      "activations/layer11_attention_weight_min": -27.695323944091797,
      "activations/layer12_attention_weight_max": 21.962345123291016,
      "activations/layer12_attention_weight_min": -25.877050399780273,
      "activations/layer13_attention_weight_max": 49.22658157348633,
      "activations/layer13_attention_weight_min": -34.23955154418945,
      "activations/layer14_attention_weight_max": 51.39981460571289,
      "activations/layer14_attention_weight_min": -34.32073211669922,
      "activations/layer15_attention_weight_max": 39.88163757324219,
      "activations/layer15_attention_weight_min": -29.88521385192871,
      "activations/layer16_attention_weight_max": 34.81304931640625,
      "activations/layer16_attention_weight_min": -25.46030044555664,
      "activations/layer17_attention_weight_max": 59.38631820678711,
      "activations/layer17_attention_weight_min": -36.91295623779297,
      "activations/layer18_attention_weight_max": 49.54713821411133,
      "activations/layer18_attention_weight_min": -32.74871063232422,
      "activations/layer19_attention_weight_max": 23.404590606689453,
      "activations/layer19_attention_weight_min": -19.572420120239258,
      "activations/layer1_attention_weight_max": 18.059953689575195,
      "activations/layer1_attention_weight_min": -15.348015785217285,
      "activations/layer20_attention_weight_max": 21.42443084716797,
      "activations/layer20_attention_weight_min": -19.929594039916992,
      "activations/layer21_attention_weight_max": 40.721336364746094,
      "activations/layer21_attention_weight_min": -24.075822830200195,
      "activations/layer22_attention_weight_max": 36.69575500488281,
      "activations/layer22_attention_weight_min": -23.334964752197266,
      "activations/layer23_attention_weight_max": 26.852420806884766,
      "activations/layer23_attention_weight_min": -18.515705108642578,
      "activations/layer2_attention_weight_max": 31.689680099487305,
      "activations/layer2_attention_weight_min": -29.428991317749023,
      "activations/layer3_attention_weight_max": 78.72855377197266,
      "activations/layer3_attention_weight_min": -77.7670669555664,
      "activations/layer4_attention_weight_max": 99.49763488769531,
      "activations/layer4_attention_weight_min": -88.87920379638672,
      "activations/layer5_attention_weight_max": 105.09170532226562,
      "activations/layer5_attention_weight_min": -94.6319580078125,
      "activations/layer6_attention_weight_max": 57.19670486450195,
      "activations/layer6_attention_weight_min": -51.03093338012695,
      "activations/layer7_attention_weight_max": 65.45631408691406,
      "activations/layer7_attention_weight_min": -62.390045166015625,
      "activations/layer8_attention_weight_max": 39.849178314208984,
      "activations/layer8_attention_weight_min": -40.728790283203125,
      "activations/layer9_attention_weight_max": 49.17698669433594,
      "activations/layer9_attention_weight_min": -47.51569747924805,
      "epoch": 4.53,
      "learning_rate": 0.00012202007575757575,
      "loss": 2.8661,
      "step": 77950
    },
    {
      "activations/layer0_attention_weight_max": 13.680458068847656,
      "activations/layer0_attention_weight_min": -15.100770950317383,
      "activations/layer10_attention_weight_max": 41.91106033325195,
      "activations/layer10_attention_weight_min": -34.825439453125,
      "activations/layer11_attention_weight_max": 31.785289764404297,
      "activations/layer11_attention_weight_min": -27.229000091552734,
      "activations/layer12_attention_weight_max": 35.09840393066406,
      "activations/layer12_attention_weight_min": -25.072423934936523,
      "activations/layer13_attention_weight_max": 64.4408950805664,
      "activations/layer13_attention_weight_min": -44.197689056396484,
      "activations/layer14_attention_weight_max": 48.37664794921875,
      "activations/layer14_attention_weight_min": -30.25544548034668,
      "activations/layer15_attention_weight_max": 43.73621368408203,
      "activations/layer15_attention_weight_min": -32.04422378540039,
      "activations/layer16_attention_weight_max": 38.53752517700195,
      "activations/layer16_attention_weight_min": -27.02205467224121,
      "activations/layer17_attention_weight_max": 61.06964111328125,
      "activations/layer17_attention_weight_min": -39.1324577331543,
      "activations/layer18_attention_weight_max": 53.19013214111328,
      "activations/layer18_attention_weight_min": -34.208133697509766,
      "activations/layer19_attention_weight_max": 23.532541275024414,
      "activations/layer19_attention_weight_min": -17.780263900756836,
      "activations/layer1_attention_weight_max": 18.58843231201172,
      "activations/layer1_attention_weight_min": -13.785890579223633,
      "activations/layer20_attention_weight_max": 21.76822280883789,
      "activations/layer20_attention_weight_min": -21.20946502685547,
      "activations/layer21_attention_weight_max": 47.703548431396484,
      "activations/layer21_attention_weight_min": -28.427471160888672,
      "activations/layer22_attention_weight_max": 39.528236389160156,
      "activations/layer22_attention_weight_min": -21.41931915283203,
      "activations/layer23_attention_weight_max": 28.861591339111328,
      "activations/layer23_attention_weight_min": -16.695579528808594,
      "activations/layer2_attention_weight_max": 29.899259567260742,
      "activations/layer2_attention_weight_min": -27.36921501159668,
      "activations/layer3_attention_weight_max": 77.61774444580078,
      "activations/layer3_attention_weight_min": -81.57456970214844,
      "activations/layer4_attention_weight_max": 96.14901733398438,
      "activations/layer4_attention_weight_min": -95.72731018066406,
      "activations/layer5_attention_weight_max": 101.10787200927734,
      "activations/layer5_attention_weight_min": -91.09908294677734,
      "activations/layer6_attention_weight_max": 57.363037109375,
      "activations/layer6_attention_weight_min": -51.0307502746582,
      "activations/layer7_attention_weight_max": 64.29188537597656,
      "activations/layer7_attention_weight_min": -66.64753723144531,
      "activations/layer8_attention_weight_max": 43.01530838012695,
      "activations/layer8_attention_weight_min": -39.89906311035156,
      "activations/layer9_attention_weight_max": 47.992488861083984,
      "activations/layer9_attention_weight_min": -46.517005920410156,
      "epoch": 4.53,
      "learning_rate": 0.00012200113636363635,
      "loss": 2.8691,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_loss": 2.8125,
      "eval_runtime": 8.51,
      "eval_samples_per_second": 504.584,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_openwebtext_loss": 2.8125,
      "eval_openwebtext_ppl": 16.651494963610144,
      "eval_openwebtext_runtime": 8.51,
      "eval_openwebtext_samples_per_second": 504.584,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_wikitext_loss": 3.05078125,
      "eval_wikitext_ppl": 21.131847230940878,
      "eval_wikitext_runtime": 1.949,
      "eval_wikitext_samples_per_second": 233.962,
      "step": 78000
    },
    {
      "epoch": 4.53,
      "eval_lambada_loss": 2.79296875,
      "eval_lambada_ppl": 16.329425896569767,
      "eval_lambada_runtime": 9.5474,
      "eval_lambada_samples_per_second": 509.98,
      "step": 78000
    },
    {
      "activations/layer0_attention_weight_max": 14.495226860046387,
      "activations/layer0_attention_weight_min": -14.843090057373047,
      "activations/layer10_attention_weight_max": 39.4945068359375,
      "activations/layer10_attention_weight_min": -34.78253173828125,
      "activations/layer11_attention_weight_max": 34.0342903137207,
      "activations/layer11_attention_weight_min": -26.711048126220703,
      "activations/layer12_attention_weight_max": 26.81666374206543,
      "activations/layer12_attention_weight_min": -25.755311965942383,
      "activations/layer13_attention_weight_max": 57.104408264160156,
      "activations/layer13_attention_weight_min": -34.33498764038086,
      "activations/layer14_attention_weight_max": 50.81874465942383,
      "activations/layer14_attention_weight_min": -33.61328887939453,
      "activations/layer15_attention_weight_max": 51.014434814453125,
      "activations/layer15_attention_weight_min": -30.4855899810791,
      "activations/layer16_attention_weight_max": 39.530296325683594,
      "activations/layer16_attention_weight_min": -24.91642951965332,
      "activations/layer17_attention_weight_max": 65.19532012939453,
      "activations/layer17_attention_weight_min": -38.430763244628906,
      "activations/layer18_attention_weight_max": 55.088592529296875,
      "activations/layer18_attention_weight_min": -32.97892761230469,
      "activations/layer19_attention_weight_max": 26.143646240234375,
      "activations/layer19_attention_weight_min": -20.469505310058594,
      "activations/layer1_attention_weight_max": 17.372451782226562,
      "activations/layer1_attention_weight_min": -13.789475440979004,
      "activations/layer20_attention_weight_max": 26.279102325439453,
      "activations/layer20_attention_weight_min": -20.507572174072266,
      "activations/layer21_attention_weight_max": 42.78831100463867,
      "activations/layer21_attention_weight_min": -24.433855056762695,
      "activations/layer22_attention_weight_max": 37.1402473449707,
      "activations/layer22_attention_weight_min": -22.47493553161621,
      "activations/layer23_attention_weight_max": 32.209136962890625,
      "activations/layer23_attention_weight_min": -16.391328811645508,
      "activations/layer2_attention_weight_max": 30.512908935546875,
      "activations/layer2_attention_weight_min": -27.428234100341797,
      "activations/layer3_attention_weight_max": 82.43820190429688,
      "activations/layer3_attention_weight_min": -82.22151947021484,
      "activations/layer4_attention_weight_max": 100.38577270507812,
      "activations/layer4_attention_weight_min": -91.39720916748047,
      "activations/layer5_attention_weight_max": 106.29835510253906,
      "activations/layer5_attention_weight_min": -92.31861877441406,
      "activations/layer6_attention_weight_max": 58.417728424072266,
      "activations/layer6_attention_weight_min": -51.65924835205078,
      "activations/layer7_attention_weight_max": 68.95740509033203,
      "activations/layer7_attention_weight_min": -67.87612915039062,
      "activations/layer8_attention_weight_max": 43.04472732543945,
      "activations/layer8_attention_weight_min": -42.17210388183594,
      "activations/layer9_attention_weight_max": 50.40147018432617,
      "activations/layer9_attention_weight_min": -49.570167541503906,
      "epoch": 4.54,
      "learning_rate": 0.00012198219696969697,
      "loss": 2.8449,
      "step": 78050
    },
    {
      "activations/layer0_attention_weight_max": 15.830665588378906,
      "activations/layer0_attention_weight_min": -14.799466133117676,
      "activations/layer10_attention_weight_max": 38.69287872314453,
      "activations/layer10_attention_weight_min": -35.343318939208984,
      "activations/layer11_attention_weight_max": 33.34221649169922,
      "activations/layer11_attention_weight_min": -29.101919174194336,
      "activations/layer12_attention_weight_max": 24.34801483154297,
      "activations/layer12_attention_weight_min": -28.064407348632812,
      "activations/layer13_attention_weight_max": 60.08688735961914,
      "activations/layer13_attention_weight_min": -47.34911346435547,
      "activations/layer14_attention_weight_max": 55.93720245361328,
      "activations/layer14_attention_weight_min": -37.94944381713867,
      "activations/layer15_attention_weight_max": 40.75105285644531,
      "activations/layer15_attention_weight_min": -30.842260360717773,
      "activations/layer16_attention_weight_max": 35.268009185791016,
      "activations/layer16_attention_weight_min": -27.156187057495117,
      "activations/layer17_attention_weight_max": 55.0257568359375,
      "activations/layer17_attention_weight_min": -38.33421325683594,
      "activations/layer18_attention_weight_max": 52.655662536621094,
      "activations/layer18_attention_weight_min": -34.34516143798828,
      "activations/layer19_attention_weight_max": 23.47566032409668,
      "activations/layer19_attention_weight_min": -20.497583389282227,
      "activations/layer1_attention_weight_max": 17.694326400756836,
      "activations/layer1_attention_weight_min": -16.42426872253418,
      "activations/layer20_attention_weight_max": 25.259252548217773,
      "activations/layer20_attention_weight_min": -21.2723331451416,
      "activations/layer21_attention_weight_max": 44.06415557861328,
      "activations/layer21_attention_weight_min": -27.23978042602539,
      "activations/layer22_attention_weight_max": 35.64681625366211,
      "activations/layer22_attention_weight_min": -24.296175003051758,
      "activations/layer23_attention_weight_max": 30.429731369018555,
      "activations/layer23_attention_weight_min": -17.567859649658203,
      "activations/layer2_attention_weight_max": 30.780975341796875,
      "activations/layer2_attention_weight_min": -27.07721710205078,
      "activations/layer3_attention_weight_max": 80.63356018066406,
      "activations/layer3_attention_weight_min": -82.29296875,
      "activations/layer4_attention_weight_max": 105.367431640625,
      "activations/layer4_attention_weight_min": -97.59578704833984,
      "activations/layer5_attention_weight_max": 113.14340209960938,
      "activations/layer5_attention_weight_min": -95.98832702636719,
      "activations/layer6_attention_weight_max": 59.758766174316406,
      "activations/layer6_attention_weight_min": -55.64356231689453,
      "activations/layer7_attention_weight_max": 69.84854888916016,
      "activations/layer7_attention_weight_min": -66.24724578857422,
      "activations/layer8_attention_weight_max": 41.53049850463867,
      "activations/layer8_attention_weight_min": -40.67707443237305,
      "activations/layer9_attention_weight_max": 52.156341552734375,
      "activations/layer9_attention_weight_min": -47.005043029785156,
      "epoch": 4.54,
      "learning_rate": 0.00012196325757575757,
      "loss": 2.8564,
      "step": 78100
    },
    {
      "activations/layer0_attention_weight_max": 14.337789535522461,
      "activations/layer0_attention_weight_min": -14.973432540893555,
      "activations/layer10_attention_weight_max": 41.113983154296875,
      "activations/layer10_attention_weight_min": -32.995513916015625,
      "activations/layer11_attention_weight_max": 31.82893180847168,
      "activations/layer11_attention_weight_min": -24.917272567749023,
      "activations/layer12_attention_weight_max": 22.505035400390625,
      "activations/layer12_attention_weight_min": -26.06238555908203,
      "activations/layer13_attention_weight_max": 59.4287223815918,
      "activations/layer13_attention_weight_min": -38.14531707763672,
      "activations/layer14_attention_weight_max": 68.02452087402344,
      "activations/layer14_attention_weight_min": -37.14219665527344,
      "activations/layer15_attention_weight_max": 44.20320510864258,
      "activations/layer15_attention_weight_min": -30.07428550720215,
      "activations/layer16_attention_weight_max": 35.13752365112305,
      "activations/layer16_attention_weight_min": -25.901098251342773,
      "activations/layer17_attention_weight_max": 60.82715606689453,
      "activations/layer17_attention_weight_min": -40.24260330200195,
      "activations/layer18_attention_weight_max": 57.47386932373047,
      "activations/layer18_attention_weight_min": -35.02009201049805,
      "activations/layer19_attention_weight_max": 26.48110008239746,
      "activations/layer19_attention_weight_min": -19.94799041748047,
      "activations/layer1_attention_weight_max": 17.978666305541992,
      "activations/layer1_attention_weight_min": -14.845198631286621,
      "activations/layer20_attention_weight_max": 24.174015045166016,
      "activations/layer20_attention_weight_min": -23.314067840576172,
      "activations/layer21_attention_weight_max": 47.458648681640625,
      "activations/layer21_attention_weight_min": -25.881877899169922,
      "activations/layer22_attention_weight_max": 35.69865036010742,
      "activations/layer22_attention_weight_min": -22.719276428222656,
      "activations/layer23_attention_weight_max": 27.411556243896484,
      "activations/layer23_attention_weight_min": -18.771595001220703,
      "activations/layer2_attention_weight_max": 29.94580841064453,
      "activations/layer2_attention_weight_min": -26.988365173339844,
      "activations/layer3_attention_weight_max": 77.4400634765625,
      "activations/layer3_attention_weight_min": -81.43057250976562,
      "activations/layer4_attention_weight_max": 95.30207824707031,
      "activations/layer4_attention_weight_min": -90.15957641601562,
      "activations/layer5_attention_weight_max": 100.26537322998047,
      "activations/layer5_attention_weight_min": -90.19902038574219,
      "activations/layer6_attention_weight_max": 55.36094284057617,
      "activations/layer6_attention_weight_min": -52.27878952026367,
      "activations/layer7_attention_weight_max": 64.27045440673828,
      "activations/layer7_attention_weight_min": -62.826297760009766,
      "activations/layer8_attention_weight_max": 38.701377868652344,
      "activations/layer8_attention_weight_min": -38.74000930786133,
      "activations/layer9_attention_weight_max": 47.80157470703125,
      "activations/layer9_attention_weight_min": -48.42373275756836,
      "epoch": 4.54,
      "learning_rate": 0.00012194431818181817,
      "loss": 2.8735,
      "step": 78150
    },
    {
      "activations/layer0_attention_weight_max": 13.674140930175781,
      "activations/layer0_attention_weight_min": -14.606300354003906,
      "activations/layer10_attention_weight_max": 38.376258850097656,
      "activations/layer10_attention_weight_min": -31.657621383666992,
      "activations/layer11_attention_weight_max": 30.035888671875,
      "activations/layer11_attention_weight_min": -25.396011352539062,
      "activations/layer12_attention_weight_max": 23.715246200561523,
      "activations/layer12_attention_weight_min": -25.889009475708008,
      "activations/layer13_attention_weight_max": 54.69355010986328,
      "activations/layer13_attention_weight_min": -39.68751525878906,
      "activations/layer14_attention_weight_max": 52.97651672363281,
      "activations/layer14_attention_weight_min": -35.25904083251953,
      "activations/layer15_attention_weight_max": 43.07792663574219,
      "activations/layer15_attention_weight_min": -30.914215087890625,
      "activations/layer16_attention_weight_max": 36.452823638916016,
      "activations/layer16_attention_weight_min": -25.05534553527832,
      "activations/layer17_attention_weight_max": 58.93098831176758,
      "activations/layer17_attention_weight_min": -38.69453811645508,
      "activations/layer18_attention_weight_max": 48.74533462524414,
      "activations/layer18_attention_weight_min": -31.873186111450195,
      "activations/layer19_attention_weight_max": 21.453187942504883,
      "activations/layer19_attention_weight_min": -18.95424461364746,
      "activations/layer1_attention_weight_max": 18.042736053466797,
      "activations/layer1_attention_weight_min": -15.754927635192871,
      "activations/layer20_attention_weight_max": 21.908143997192383,
      "activations/layer20_attention_weight_min": -19.53470230102539,
      "activations/layer21_attention_weight_max": 41.58384323120117,
      "activations/layer21_attention_weight_min": -28.996734619140625,
      "activations/layer22_attention_weight_max": 31.334383010864258,
      "activations/layer22_attention_weight_min": -24.43143081665039,
      "activations/layer23_attention_weight_max": 23.92845916748047,
      "activations/layer23_attention_weight_min": -19.040908813476562,
      "activations/layer2_attention_weight_max": 28.79054069519043,
      "activations/layer2_attention_weight_min": -26.266868591308594,
      "activations/layer3_attention_weight_max": 74.58218383789062,
      "activations/layer3_attention_weight_min": -77.35533142089844,
      "activations/layer4_attention_weight_max": 93.69398498535156,
      "activations/layer4_attention_weight_min": -90.8359603881836,
      "activations/layer5_attention_weight_max": 98.39497375488281,
      "activations/layer5_attention_weight_min": -87.7535400390625,
      "activations/layer6_attention_weight_max": 55.470489501953125,
      "activations/layer6_attention_weight_min": -51.35822677612305,
      "activations/layer7_attention_weight_max": 64.45124816894531,
      "activations/layer7_attention_weight_min": -58.982723236083984,
      "activations/layer8_attention_weight_max": 39.59988784790039,
      "activations/layer8_attention_weight_min": -37.286556243896484,
      "activations/layer9_attention_weight_max": 44.99848556518555,
      "activations/layer9_attention_weight_min": -44.6868782043457,
      "epoch": 4.54,
      "learning_rate": 0.00012192537878787878,
      "loss": 2.8655,
      "step": 78200
    },
    {
      "activations/layer0_attention_weight_max": 15.586790084838867,
      "activations/layer0_attention_weight_min": -14.625853538513184,
      "activations/layer10_attention_weight_max": 44.93455123901367,
      "activations/layer10_attention_weight_min": -37.60673141479492,
      "activations/layer11_attention_weight_max": 38.91917419433594,
      "activations/layer11_attention_weight_min": -29.49066734313965,
      "activations/layer12_attention_weight_max": 27.115148544311523,
      "activations/layer12_attention_weight_min": -29.665817260742188,
      "activations/layer13_attention_weight_max": 49.70594787597656,
      "activations/layer13_attention_weight_min": -30.434701919555664,
      "activations/layer14_attention_weight_max": 54.20481872558594,
      "activations/layer14_attention_weight_min": -30.044919967651367,
      "activations/layer15_attention_weight_max": 43.3549919128418,
      "activations/layer15_attention_weight_min": -30.54673957824707,
      "activations/layer16_attention_weight_max": 40.01327133178711,
      "activations/layer16_attention_weight_min": -24.59511375427246,
      "activations/layer17_attention_weight_max": 58.48976135253906,
      "activations/layer17_attention_weight_min": -43.67197799682617,
      "activations/layer18_attention_weight_max": 49.22993087768555,
      "activations/layer18_attention_weight_min": -34.52464294433594,
      "activations/layer19_attention_weight_max": 24.444433212280273,
      "activations/layer19_attention_weight_min": -21.3737850189209,
      "activations/layer1_attention_weight_max": 16.786579132080078,
      "activations/layer1_attention_weight_min": -13.533782005310059,
      "activations/layer20_attention_weight_max": 21.607275009155273,
      "activations/layer20_attention_weight_min": -20.08519744873047,
      "activations/layer21_attention_weight_max": 42.08995056152344,
      "activations/layer21_attention_weight_min": -25.14023780822754,
      "activations/layer22_attention_weight_max": 37.11637878417969,
      "activations/layer22_attention_weight_min": -22.341054916381836,
      "activations/layer23_attention_weight_max": 27.529155731201172,
      "activations/layer23_attention_weight_min": -16.466350555419922,
      "activations/layer2_attention_weight_max": 30.548765182495117,
      "activations/layer2_attention_weight_min": -29.196712493896484,
      "activations/layer3_attention_weight_max": 78.69862365722656,
      "activations/layer3_attention_weight_min": -82.53814697265625,
      "activations/layer4_attention_weight_max": 108.3728256225586,
      "activations/layer4_attention_weight_min": -95.17005157470703,
      "activations/layer5_attention_weight_max": 121.46551513671875,
      "activations/layer5_attention_weight_min": -97.52694702148438,
      "activations/layer6_attention_weight_max": 61.000572204589844,
      "activations/layer6_attention_weight_min": -54.07926559448242,
      "activations/layer7_attention_weight_max": 81.36259460449219,
      "activations/layer7_attention_weight_min": -69.03997039794922,
      "activations/layer8_attention_weight_max": 54.45528030395508,
      "activations/layer8_attention_weight_min": -39.3092155456543,
      "activations/layer9_attention_weight_max": 72.23279571533203,
      "activations/layer9_attention_weight_min": -57.621612548828125,
      "epoch": 4.55,
      "learning_rate": 0.00012190643939393937,
      "loss": 2.8568,
      "step": 78250
    },
    {
      "activations/layer0_attention_weight_max": 13.97510051727295,
      "activations/layer0_attention_weight_min": -15.466843605041504,
      "activations/layer10_attention_weight_max": 41.09632873535156,
      "activations/layer10_attention_weight_min": -33.027740478515625,
      "activations/layer11_attention_weight_max": 33.20543670654297,
      "activations/layer11_attention_weight_min": -27.59390640258789,
      "activations/layer12_attention_weight_max": 21.91647720336914,
      "activations/layer12_attention_weight_min": -24.158403396606445,
      "activations/layer13_attention_weight_max": 61.781402587890625,
      "activations/layer13_attention_weight_min": -43.38340377807617,
      "activations/layer14_attention_weight_max": 62.90719985961914,
      "activations/layer14_attention_weight_min": -32.795162200927734,
      "activations/layer15_attention_weight_max": 47.672874450683594,
      "activations/layer15_attention_weight_min": -29.718568801879883,
      "activations/layer16_attention_weight_max": 34.1158561706543,
      "activations/layer16_attention_weight_min": -26.300687789916992,
      "activations/layer17_attention_weight_max": 55.7469596862793,
      "activations/layer17_attention_weight_min": -38.5693244934082,
      "activations/layer18_attention_weight_max": 49.009761810302734,
      "activations/layer18_attention_weight_min": -31.44597816467285,
      "activations/layer19_attention_weight_max": 22.30744171142578,
      "activations/layer19_attention_weight_min": -18.392065048217773,
      "activations/layer1_attention_weight_max": 17.756086349487305,
      "activations/layer1_attention_weight_min": -15.229164123535156,
      "activations/layer20_attention_weight_max": 21.843923568725586,
      "activations/layer20_attention_weight_min": -21.388153076171875,
      "activations/layer21_attention_weight_max": 40.89653396606445,
      "activations/layer21_attention_weight_min": -26.575448989868164,
      "activations/layer22_attention_weight_max": 31.92242431640625,
      "activations/layer22_attention_weight_min": -22.217266082763672,
      "activations/layer23_attention_weight_max": 25.42798614501953,
      "activations/layer23_attention_weight_min": -17.75731086730957,
      "activations/layer2_attention_weight_max": 32.48737335205078,
      "activations/layer2_attention_weight_min": -29.5188045501709,
      "activations/layer3_attention_weight_max": 81.56441497802734,
      "activations/layer3_attention_weight_min": -84.62364196777344,
      "activations/layer4_attention_weight_max": 102.65770721435547,
      "activations/layer4_attention_weight_min": -96.6720962524414,
      "activations/layer5_attention_weight_max": 104.5301742553711,
      "activations/layer5_attention_weight_min": -95.96419525146484,
      "activations/layer6_attention_weight_max": 57.4679069519043,
      "activations/layer6_attention_weight_min": -53.722450256347656,
      "activations/layer7_attention_weight_max": 65.03594207763672,
      "activations/layer7_attention_weight_min": -61.702796936035156,
      "activations/layer8_attention_weight_max": 41.405338287353516,
      "activations/layer8_attention_weight_min": -40.83937454223633,
      "activations/layer9_attention_weight_max": 51.21087646484375,
      "activations/layer9_attention_weight_min": -45.89512252807617,
      "epoch": 4.55,
      "learning_rate": 0.00012188749999999999,
      "loss": 2.861,
      "step": 78300
    },
    {
      "activations/layer0_attention_weight_max": 15.601948738098145,
      "activations/layer0_attention_weight_min": -14.82140064239502,
      "activations/layer10_attention_weight_max": 34.72445297241211,
      "activations/layer10_attention_weight_min": -32.638946533203125,
      "activations/layer11_attention_weight_max": 28.309246063232422,
      "activations/layer11_attention_weight_min": -25.8032283782959,
      "activations/layer12_attention_weight_max": 23.05235481262207,
      "activations/layer12_attention_weight_min": -29.236881256103516,
      "activations/layer13_attention_weight_max": 43.78318786621094,
      "activations/layer13_attention_weight_min": -32.542945861816406,
      "activations/layer14_attention_weight_max": 38.25407791137695,
      "activations/layer14_attention_weight_min": -28.506511688232422,
      "activations/layer15_attention_weight_max": 37.70067596435547,
      "activations/layer15_attention_weight_min": -30.4267635345459,
      "activations/layer16_attention_weight_max": 34.880706787109375,
      "activations/layer16_attention_weight_min": -26.978740692138672,
      "activations/layer17_attention_weight_max": 58.38486862182617,
      "activations/layer17_attention_weight_min": -41.16867446899414,
      "activations/layer18_attention_weight_max": 48.352046966552734,
      "activations/layer18_attention_weight_min": -35.59476852416992,
      "activations/layer19_attention_weight_max": 23.838987350463867,
      "activations/layer19_attention_weight_min": -19.037242889404297,
      "activations/layer1_attention_weight_max": 17.480571746826172,
      "activations/layer1_attention_weight_min": -14.5097074508667,
      "activations/layer20_attention_weight_max": 21.89053726196289,
      "activations/layer20_attention_weight_min": -20.255212783813477,
      "activations/layer21_attention_weight_max": 38.721282958984375,
      "activations/layer21_attention_weight_min": -25.53020668029785,
      "activations/layer22_attention_weight_max": 34.15425491333008,
      "activations/layer22_attention_weight_min": -25.10296058654785,
      "activations/layer23_attention_weight_max": 27.81233787536621,
      "activations/layer23_attention_weight_min": -20.565940856933594,
      "activations/layer2_attention_weight_max": 28.399829864501953,
      "activations/layer2_attention_weight_min": -26.79578971862793,
      "activations/layer3_attention_weight_max": 75.2294692993164,
      "activations/layer3_attention_weight_min": -80.07263946533203,
      "activations/layer4_attention_weight_max": 97.80455780029297,
      "activations/layer4_attention_weight_min": -88.95527648925781,
      "activations/layer5_attention_weight_max": 101.58802795410156,
      "activations/layer5_attention_weight_min": -91.8496322631836,
      "activations/layer6_attention_weight_max": 56.74480056762695,
      "activations/layer6_attention_weight_min": -51.40855026245117,
      "activations/layer7_attention_weight_max": 63.373321533203125,
      "activations/layer7_attention_weight_min": -62.15790939331055,
      "activations/layer8_attention_weight_max": 41.084739685058594,
      "activations/layer8_attention_weight_min": -39.71186065673828,
      "activations/layer9_attention_weight_max": 52.900543212890625,
      "activations/layer9_attention_weight_min": -47.63603210449219,
      "epoch": 4.55,
      "learning_rate": 0.0001218685606060606,
      "loss": 2.8621,
      "step": 78350
    },
    {
      "activations/layer0_attention_weight_max": 13.780341148376465,
      "activations/layer0_attention_weight_min": -15.04943561553955,
      "activations/layer10_attention_weight_max": 39.87467575073242,
      "activations/layer10_attention_weight_min": -35.42549514770508,
      "activations/layer11_attention_weight_max": 31.194196701049805,
      "activations/layer11_attention_weight_min": -26.89484214782715,
      "activations/layer12_attention_weight_max": 26.168397903442383,
      "activations/layer12_attention_weight_min": -24.973783493041992,
      "activations/layer13_attention_weight_max": 47.83780288696289,
      "activations/layer13_attention_weight_min": -34.328834533691406,
      "activations/layer14_attention_weight_max": 47.15017318725586,
      "activations/layer14_attention_weight_min": -33.91613006591797,
      "activations/layer15_attention_weight_max": 44.82705307006836,
      "activations/layer15_attention_weight_min": -32.233856201171875,
      "activations/layer16_attention_weight_max": 42.30516815185547,
      "activations/layer16_attention_weight_min": -27.739973068237305,
      "activations/layer17_attention_weight_max": 58.83662796020508,
      "activations/layer17_attention_weight_min": -40.05556869506836,
      "activations/layer18_attention_weight_max": 52.8798713684082,
      "activations/layer18_attention_weight_min": -33.326515197753906,
      "activations/layer19_attention_weight_max": 22.251035690307617,
      "activations/layer19_attention_weight_min": -20.60154151916504,
      "activations/layer1_attention_weight_max": 17.32708168029785,
      "activations/layer1_attention_weight_min": -15.59676742553711,
      "activations/layer20_attention_weight_max": 21.999109268188477,
      "activations/layer20_attention_weight_min": -20.925823211669922,
      "activations/layer21_attention_weight_max": 46.323524475097656,
      "activations/layer21_attention_weight_min": -26.694347381591797,
      "activations/layer22_attention_weight_max": 40.55795669555664,
      "activations/layer22_attention_weight_min": -24.181316375732422,
      "activations/layer23_attention_weight_max": 26.62598419189453,
      "activations/layer23_attention_weight_min": -16.8289852142334,
      "activations/layer2_attention_weight_max": 29.16835594177246,
      "activations/layer2_attention_weight_min": -27.83395767211914,
      "activations/layer3_attention_weight_max": 79.03336334228516,
      "activations/layer3_attention_weight_min": -85.7797622680664,
      "activations/layer4_attention_weight_max": 99.21038818359375,
      "activations/layer4_attention_weight_min": -94.13721466064453,
      "activations/layer5_attention_weight_max": 105.88350677490234,
      "activations/layer5_attention_weight_min": -93.72174072265625,
      "activations/layer6_attention_weight_max": 56.4103889465332,
      "activations/layer6_attention_weight_min": -53.156246185302734,
      "activations/layer7_attention_weight_max": 70.59662628173828,
      "activations/layer7_attention_weight_min": -62.95208740234375,
      "activations/layer8_attention_weight_max": 41.097877502441406,
      "activations/layer8_attention_weight_min": -40.89043045043945,
      "activations/layer9_attention_weight_max": 50.41337966918945,
      "activations/layer9_attention_weight_min": -50.35829544067383,
      "epoch": 4.56,
      "learning_rate": 0.00012184962121212119,
      "loss": 2.8594,
      "step": 78400
    },
    {
      "activations/layer0_attention_weight_max": 14.268677711486816,
      "activations/layer0_attention_weight_min": -14.517318725585938,
      "activations/layer10_attention_weight_max": 36.98272705078125,
      "activations/layer10_attention_weight_min": -37.44777297973633,
      "activations/layer11_attention_weight_max": 28.876949310302734,
      "activations/layer11_attention_weight_min": -27.127058029174805,
      "activations/layer12_attention_weight_max": 23.787532806396484,
      "activations/layer12_attention_weight_min": -23.302579879760742,
      "activations/layer13_attention_weight_max": 46.768375396728516,
      "activations/layer13_attention_weight_min": -33.47938537597656,
      "activations/layer14_attention_weight_max": 55.11249923706055,
      "activations/layer14_attention_weight_min": -35.17170333862305,
      "activations/layer15_attention_weight_max": 49.44953918457031,
      "activations/layer15_attention_weight_min": -32.082332611083984,
      "activations/layer16_attention_weight_max": 34.27992248535156,
      "activations/layer16_attention_weight_min": -29.37449836730957,
      "activations/layer17_attention_weight_max": 58.53987503051758,
      "activations/layer17_attention_weight_min": -40.140262603759766,
      "activations/layer18_attention_weight_max": 49.26886749267578,
      "activations/layer18_attention_weight_min": -37.11357879638672,
      "activations/layer19_attention_weight_max": 26.81294059753418,
      "activations/layer19_attention_weight_min": -19.67483139038086,
      "activations/layer1_attention_weight_max": 18.456663131713867,
      "activations/layer1_attention_weight_min": -14.408160209655762,
      "activations/layer20_attention_weight_max": 23.23204231262207,
      "activations/layer20_attention_weight_min": -20.949756622314453,
      "activations/layer21_attention_weight_max": 42.38420486450195,
      "activations/layer21_attention_weight_min": -27.360820770263672,
      "activations/layer22_attention_weight_max": 33.94453430175781,
      "activations/layer22_attention_weight_min": -23.84385871887207,
      "activations/layer23_attention_weight_max": 28.113048553466797,
      "activations/layer23_attention_weight_min": -18.55664825439453,
      "activations/layer2_attention_weight_max": 31.04854393005371,
      "activations/layer2_attention_weight_min": -29.801929473876953,
      "activations/layer3_attention_weight_max": 77.62213134765625,
      "activations/layer3_attention_weight_min": -81.91604614257812,
      "activations/layer4_attention_weight_max": 100.70823669433594,
      "activations/layer4_attention_weight_min": -100.1557388305664,
      "activations/layer5_attention_weight_max": 108.46820068359375,
      "activations/layer5_attention_weight_min": -94.80162811279297,
      "activations/layer6_attention_weight_max": 60.01835250854492,
      "activations/layer6_attention_weight_min": -53.92206573486328,
      "activations/layer7_attention_weight_max": 65.1463851928711,
      "activations/layer7_attention_weight_min": -63.980525970458984,
      "activations/layer8_attention_weight_max": 42.42509841918945,
      "activations/layer8_attention_weight_min": -41.23659133911133,
      "activations/layer9_attention_weight_max": 49.19700241088867,
      "activations/layer9_attention_weight_min": -49.40255355834961,
      "epoch": 4.56,
      "learning_rate": 0.0001218306818181818,
      "loss": 2.8475,
      "step": 78450
    },
    {
      "activations/layer0_attention_weight_max": 14.10013484954834,
      "activations/layer0_attention_weight_min": -14.492066383361816,
      "activations/layer10_attention_weight_max": 42.269935607910156,
      "activations/layer10_attention_weight_min": -34.60465621948242,
      "activations/layer11_attention_weight_max": 33.364566802978516,
      "activations/layer11_attention_weight_min": -25.718650817871094,
      "activations/layer12_attention_weight_max": 32.0564079284668,
      "activations/layer12_attention_weight_min": -26.01959991455078,
      "activations/layer13_attention_weight_max": 50.48058319091797,
      "activations/layer13_attention_weight_min": -33.12096405029297,
      "activations/layer14_attention_weight_max": 44.09910583496094,
      "activations/layer14_attention_weight_min": -32.155433654785156,
      "activations/layer15_attention_weight_max": 39.94450378417969,
      "activations/layer15_attention_weight_min": -28.480548858642578,
      "activations/layer16_attention_weight_max": 37.17763900756836,
      "activations/layer16_attention_weight_min": -25.503108978271484,
      "activations/layer17_attention_weight_max": 68.14932250976562,
      "activations/layer17_attention_weight_min": -39.08639907836914,
      "activations/layer18_attention_weight_max": 47.890106201171875,
      "activations/layer18_attention_weight_min": -31.34886360168457,
      "activations/layer19_attention_weight_max": 20.680328369140625,
      "activations/layer19_attention_weight_min": -20.797611236572266,
      "activations/layer1_attention_weight_max": 20.069225311279297,
      "activations/layer1_attention_weight_min": -15.765488624572754,
      "activations/layer20_attention_weight_max": 20.236169815063477,
      "activations/layer20_attention_weight_min": -21.333656311035156,
      "activations/layer21_attention_weight_max": 37.27976608276367,
      "activations/layer21_attention_weight_min": -23.52933692932129,
      "activations/layer22_attention_weight_max": 32.56936264038086,
      "activations/layer22_attention_weight_min": -22.492883682250977,
      "activations/layer23_attention_weight_max": 27.709980010986328,
      "activations/layer23_attention_weight_min": -17.636152267456055,
      "activations/layer2_attention_weight_max": 30.734630584716797,
      "activations/layer2_attention_weight_min": -28.175857543945312,
      "activations/layer3_attention_weight_max": 79.88128662109375,
      "activations/layer3_attention_weight_min": -82.8801040649414,
      "activations/layer4_attention_weight_max": 98.74314880371094,
      "activations/layer4_attention_weight_min": -97.85206604003906,
      "activations/layer5_attention_weight_max": 111.21089935302734,
      "activations/layer5_attention_weight_min": -97.33583068847656,
      "activations/layer6_attention_weight_max": 60.33713150024414,
      "activations/layer6_attention_weight_min": -54.542911529541016,
      "activations/layer7_attention_weight_max": 75.21519470214844,
      "activations/layer7_attention_weight_min": -66.07135009765625,
      "activations/layer8_attention_weight_max": 49.90074157714844,
      "activations/layer8_attention_weight_min": -43.9310188293457,
      "activations/layer9_attention_weight_max": 61.38230514526367,
      "activations/layer9_attention_weight_min": -45.86836242675781,
      "epoch": 4.56,
      "learning_rate": 0.00012181174242424242,
      "loss": 2.8687,
      "step": 78500
    },
    {
      "activations/layer0_attention_weight_max": 14.655991554260254,
      "activations/layer0_attention_weight_min": -14.868122100830078,
      "activations/layer10_attention_weight_max": 40.749053955078125,
      "activations/layer10_attention_weight_min": -37.76080322265625,
      "activations/layer11_attention_weight_max": 31.422536849975586,
      "activations/layer11_attention_weight_min": -29.541770935058594,
      "activations/layer12_attention_weight_max": 24.111656188964844,
      "activations/layer12_attention_weight_min": -24.511682510375977,
      "activations/layer13_attention_weight_max": 53.2188835144043,
      "activations/layer13_attention_weight_min": -39.820648193359375,
      "activations/layer14_attention_weight_max": 58.759422302246094,
      "activations/layer14_attention_weight_min": -36.67239761352539,
      "activations/layer15_attention_weight_max": 45.588958740234375,
      "activations/layer15_attention_weight_min": -32.65694808959961,
      "activations/layer16_attention_weight_max": 48.64814758300781,
      "activations/layer16_attention_weight_min": -28.05553436279297,
      "activations/layer17_attention_weight_max": 61.30596923828125,
      "activations/layer17_attention_weight_min": -38.9821662902832,
      "activations/layer18_attention_weight_max": 53.62729263305664,
      "activations/layer18_attention_weight_min": -33.699676513671875,
      "activations/layer19_attention_weight_max": 25.630191802978516,
      "activations/layer19_attention_weight_min": -19.19955062866211,
      "activations/layer1_attention_weight_max": 17.823400497436523,
      "activations/layer1_attention_weight_min": -15.648796081542969,
      "activations/layer20_attention_weight_max": 27.083389282226562,
      "activations/layer20_attention_weight_min": -21.275623321533203,
      "activations/layer21_attention_weight_max": 44.36945724487305,
      "activations/layer21_attention_weight_min": -26.493576049804688,
      "activations/layer22_attention_weight_max": 41.01999282836914,
      "activations/layer22_attention_weight_min": -24.340961456298828,
      "activations/layer23_attention_weight_max": 31.86244773864746,
      "activations/layer23_attention_weight_min": -17.26842498779297,
      "activations/layer2_attention_weight_max": 32.30181884765625,
      "activations/layer2_attention_weight_min": -29.28240203857422,
      "activations/layer3_attention_weight_max": 85.5262680053711,
      "activations/layer3_attention_weight_min": -90.26374053955078,
      "activations/layer4_attention_weight_max": 107.70875549316406,
      "activations/layer4_attention_weight_min": -98.64994812011719,
      "activations/layer5_attention_weight_max": 112.67852783203125,
      "activations/layer5_attention_weight_min": -99.00825500488281,
      "activations/layer6_attention_weight_max": 62.623844146728516,
      "activations/layer6_attention_weight_min": -60.35149383544922,
      "activations/layer7_attention_weight_max": 70.91512298583984,
      "activations/layer7_attention_weight_min": -67.13324737548828,
      "activations/layer8_attention_weight_max": 41.99496078491211,
      "activations/layer8_attention_weight_min": -42.50376510620117,
      "activations/layer9_attention_weight_max": 49.87236404418945,
      "activations/layer9_attention_weight_min": -48.82301330566406,
      "epoch": 4.56,
      "learning_rate": 0.00012179280303030301,
      "loss": 2.8647,
      "step": 78550
    },
    {
      "activations/layer0_attention_weight_max": 14.206125259399414,
      "activations/layer0_attention_weight_min": -15.059990882873535,
      "activations/layer10_attention_weight_max": 36.83200454711914,
      "activations/layer10_attention_weight_min": -34.107486724853516,
      "activations/layer11_attention_weight_max": 29.855087280273438,
      "activations/layer11_attention_weight_min": -26.69756317138672,
      "activations/layer12_attention_weight_max": 23.701696395874023,
      "activations/layer12_attention_weight_min": -27.918781280517578,
      "activations/layer13_attention_weight_max": 41.835227966308594,
      "activations/layer13_attention_weight_min": -32.35365676879883,
      "activations/layer14_attention_weight_max": 42.00752258300781,
      "activations/layer14_attention_weight_min": -29.475109100341797,
      "activations/layer15_attention_weight_max": 41.41716766357422,
      "activations/layer15_attention_weight_min": -29.2280216217041,
      "activations/layer16_attention_weight_max": 32.59323501586914,
      "activations/layer16_attention_weight_min": -26.225688934326172,
      "activations/layer17_attention_weight_max": 55.31855392456055,
      "activations/layer17_attention_weight_min": -37.92583084106445,
      "activations/layer18_attention_weight_max": 46.16551971435547,
      "activations/layer18_attention_weight_min": -33.01779556274414,
      "activations/layer19_attention_weight_max": 23.357322692871094,
      "activations/layer19_attention_weight_min": -19.464818954467773,
      "activations/layer1_attention_weight_max": 18.660531997680664,
      "activations/layer1_attention_weight_min": -15.60111141204834,
      "activations/layer20_attention_weight_max": 21.467025756835938,
      "activations/layer20_attention_weight_min": -20.16749382019043,
      "activations/layer21_attention_weight_max": 37.468509674072266,
      "activations/layer21_attention_weight_min": -28.42180061340332,
      "activations/layer22_attention_weight_max": 32.7534065246582,
      "activations/layer22_attention_weight_min": -24.67957305908203,
      "activations/layer23_attention_weight_max": 26.075666427612305,
      "activations/layer23_attention_weight_min": -18.80649185180664,
      "activations/layer2_attention_weight_max": 30.38630485534668,
      "activations/layer2_attention_weight_min": -26.655797958374023,
      "activations/layer3_attention_weight_max": 78.44538879394531,
      "activations/layer3_attention_weight_min": -82.91161346435547,
      "activations/layer4_attention_weight_max": 95.78775787353516,
      "activations/layer4_attention_weight_min": -91.13922119140625,
      "activations/layer5_attention_weight_max": 99.34437561035156,
      "activations/layer5_attention_weight_min": -92.4388427734375,
      "activations/layer6_attention_weight_max": 56.09965133666992,
      "activations/layer6_attention_weight_min": -49.89372253417969,
      "activations/layer7_attention_weight_max": 65.24755859375,
      "activations/layer7_attention_weight_min": -62.5821647644043,
      "activations/layer8_attention_weight_max": 38.475345611572266,
      "activations/layer8_attention_weight_min": -38.7021484375,
      "activations/layer9_attention_weight_max": 49.26022720336914,
      "activations/layer9_attention_weight_min": -48.91973114013672,
      "epoch": 4.57,
      "learning_rate": 0.00012177386363636362,
      "loss": 2.872,
      "step": 78600
    },
    {
      "activations/layer0_attention_weight_max": 14.583941459655762,
      "activations/layer0_attention_weight_min": -15.057721138000488,
      "activations/layer10_attention_weight_max": 41.8880729675293,
      "activations/layer10_attention_weight_min": -35.6455078125,
      "activations/layer11_attention_weight_max": 32.568904876708984,
      "activations/layer11_attention_weight_min": -27.25542449951172,
      "activations/layer12_attention_weight_max": 22.93724822998047,
      "activations/layer12_attention_weight_min": -26.6591739654541,
      "activations/layer13_attention_weight_max": 54.11134719848633,
      "activations/layer13_attention_weight_min": -39.95069122314453,
      "activations/layer14_attention_weight_max": 55.351993560791016,
      "activations/layer14_attention_weight_min": -39.85136795043945,
      "activations/layer15_attention_weight_max": 45.789794921875,
      "activations/layer15_attention_weight_min": -30.196937561035156,
      "activations/layer16_attention_weight_max": 37.33681106567383,
      "activations/layer16_attention_weight_min": -28.512582778930664,
      "activations/layer17_attention_weight_max": 56.03649139404297,
      "activations/layer17_attention_weight_min": -37.5749397277832,
      "activations/layer18_attention_weight_max": 48.56715774536133,
      "activations/layer18_attention_weight_min": -33.89246368408203,
      "activations/layer19_attention_weight_max": 23.848587036132812,
      "activations/layer19_attention_weight_min": -19.760568618774414,
      "activations/layer1_attention_weight_max": 16.64957618713379,
      "activations/layer1_attention_weight_min": -14.900995254516602,
      "activations/layer20_attention_weight_max": 22.003198623657227,
      "activations/layer20_attention_weight_min": -19.526443481445312,
      "activations/layer21_attention_weight_max": 40.0519905090332,
      "activations/layer21_attention_weight_min": -27.686498641967773,
      "activations/layer22_attention_weight_max": 34.0984992980957,
      "activations/layer22_attention_weight_min": -23.514389038085938,
      "activations/layer23_attention_weight_max": 29.0184383392334,
      "activations/layer23_attention_weight_min": -17.474822998046875,
      "activations/layer2_attention_weight_max": 30.028982162475586,
      "activations/layer2_attention_weight_min": -28.106794357299805,
      "activations/layer3_attention_weight_max": 82.02947235107422,
      "activations/layer3_attention_weight_min": -84.19093322753906,
      "activations/layer4_attention_weight_max": 104.867919921875,
      "activations/layer4_attention_weight_min": -94.83969116210938,
      "activations/layer5_attention_weight_max": 114.17314910888672,
      "activations/layer5_attention_weight_min": -98.44126892089844,
      "activations/layer6_attention_weight_max": 61.50676727294922,
      "activations/layer6_attention_weight_min": -52.82583999633789,
      "activations/layer7_attention_weight_max": 69.7331314086914,
      "activations/layer7_attention_weight_min": -66.05220794677734,
      "activations/layer8_attention_weight_max": 44.79906463623047,
      "activations/layer8_attention_weight_min": -41.4619140625,
      "activations/layer9_attention_weight_max": 59.40621566772461,
      "activations/layer9_attention_weight_min": -49.31019973754883,
      "epoch": 4.57,
      "learning_rate": 0.00012175492424242424,
      "loss": 2.8541,
      "step": 78650
    },
    {
      "activations/layer0_attention_weight_max": 14.255781173706055,
      "activations/layer0_attention_weight_min": -15.023245811462402,
      "activations/layer10_attention_weight_max": 43.2411003112793,
      "activations/layer10_attention_weight_min": -36.085548400878906,
      "activations/layer11_attention_weight_max": 33.377296447753906,
      "activations/layer11_attention_weight_min": -27.88909912109375,
      "activations/layer12_attention_weight_max": 24.23833465576172,
      "activations/layer12_attention_weight_min": -34.017120361328125,
      "activations/layer13_attention_weight_max": 64.30059814453125,
      "activations/layer13_attention_weight_min": -38.67012023925781,
      "activations/layer14_attention_weight_max": 65.37101745605469,
      "activations/layer14_attention_weight_min": -33.99372482299805,
      "activations/layer15_attention_weight_max": 51.695194244384766,
      "activations/layer15_attention_weight_min": -32.366329193115234,
      "activations/layer16_attention_weight_max": 40.54833984375,
      "activations/layer16_attention_weight_min": -25.76372528076172,
      "activations/layer17_attention_weight_max": 68.37626647949219,
      "activations/layer17_attention_weight_min": -38.9351921081543,
      "activations/layer18_attention_weight_max": 55.89870071411133,
      "activations/layer18_attention_weight_min": -34.01557540893555,
      "activations/layer19_attention_weight_max": 26.80928611755371,
      "activations/layer19_attention_weight_min": -19.449729919433594,
      "activations/layer1_attention_weight_max": 18.055482864379883,
      "activations/layer1_attention_weight_min": -16.97693634033203,
      "activations/layer20_attention_weight_max": 24.791826248168945,
      "activations/layer20_attention_weight_min": -21.613401412963867,
      "activations/layer21_attention_weight_max": 46.11761474609375,
      "activations/layer21_attention_weight_min": -26.26985740661621,
      "activations/layer22_attention_weight_max": 39.84904098510742,
      "activations/layer22_attention_weight_min": -25.01313018798828,
      "activations/layer23_attention_weight_max": 30.64789581298828,
      "activations/layer23_attention_weight_min": -17.91448974609375,
      "activations/layer2_attention_weight_max": 32.111331939697266,
      "activations/layer2_attention_weight_min": -29.07538414001465,
      "activations/layer3_attention_weight_max": 82.111328125,
      "activations/layer3_attention_weight_min": -86.67056274414062,
      "activations/layer4_attention_weight_max": 103.8831787109375,
      "activations/layer4_attention_weight_min": -99.14373016357422,
      "activations/layer5_attention_weight_max": 107.5186996459961,
      "activations/layer5_attention_weight_min": -97.26252746582031,
      "activations/layer6_attention_weight_max": 59.306800842285156,
      "activations/layer6_attention_weight_min": -55.096832275390625,
      "activations/layer7_attention_weight_max": 70.35408020019531,
      "activations/layer7_attention_weight_min": -63.64617156982422,
      "activations/layer8_attention_weight_max": 44.8953857421875,
      "activations/layer8_attention_weight_min": -41.172340393066406,
      "activations/layer9_attention_weight_max": 58.39148712158203,
      "activations/layer9_attention_weight_min": -49.469608306884766,
      "epoch": 4.57,
      "learning_rate": 0.00012173598484848484,
      "loss": 2.8593,
      "step": 78700
    },
    {
      "activations/layer0_attention_weight_max": 14.231505393981934,
      "activations/layer0_attention_weight_min": -15.05123519897461,
      "activations/layer10_attention_weight_max": 40.34843826293945,
      "activations/layer10_attention_weight_min": -35.86116409301758,
      "activations/layer11_attention_weight_max": 34.02287292480469,
      "activations/layer11_attention_weight_min": -27.30571746826172,
      "activations/layer12_attention_weight_max": 27.386716842651367,
      "activations/layer12_attention_weight_min": -24.278121948242188,
      "activations/layer13_attention_weight_max": 45.3675651550293,
      "activations/layer13_attention_weight_min": -37.16856384277344,
      "activations/layer14_attention_weight_max": 52.27016830444336,
      "activations/layer14_attention_weight_min": -33.82670211791992,
      "activations/layer15_attention_weight_max": 41.819793701171875,
      "activations/layer15_attention_weight_min": -31.683759689331055,
      "activations/layer16_attention_weight_max": 38.82709884643555,
      "activations/layer16_attention_weight_min": -25.647396087646484,
      "activations/layer17_attention_weight_max": 55.332332611083984,
      "activations/layer17_attention_weight_min": -39.096500396728516,
      "activations/layer18_attention_weight_max": 47.48860549926758,
      "activations/layer18_attention_weight_min": -33.030670166015625,
      "activations/layer19_attention_weight_max": 24.210783004760742,
      "activations/layer19_attention_weight_min": -18.815336227416992,
      "activations/layer1_attention_weight_max": 18.84780502319336,
      "activations/layer1_attention_weight_min": -13.428319931030273,
      "activations/layer20_attention_weight_max": 20.691661834716797,
      "activations/layer20_attention_weight_min": -20.335538864135742,
      "activations/layer21_attention_weight_max": 36.88068389892578,
      "activations/layer21_attention_weight_min": -25.749814987182617,
      "activations/layer22_attention_weight_max": 32.152565002441406,
      "activations/layer22_attention_weight_min": -21.790456771850586,
      "activations/layer23_attention_weight_max": 25.015281677246094,
      "activations/layer23_attention_weight_min": -17.1533203125,
      "activations/layer2_attention_weight_max": 31.648780822753906,
      "activations/layer2_attention_weight_min": -28.659046173095703,
      "activations/layer3_attention_weight_max": 83.10747528076172,
      "activations/layer3_attention_weight_min": -83.98350524902344,
      "activations/layer4_attention_weight_max": 104.58135223388672,
      "activations/layer4_attention_weight_min": -101.84709930419922,
      "activations/layer5_attention_weight_max": 109.27256774902344,
      "activations/layer5_attention_weight_min": -100.4229965209961,
      "activations/layer6_attention_weight_max": 59.86469268798828,
      "activations/layer6_attention_weight_min": -55.25564956665039,
      "activations/layer7_attention_weight_max": 73.58988952636719,
      "activations/layer7_attention_weight_min": -75.44534301757812,
      "activations/layer8_attention_weight_max": 47.27095031738281,
      "activations/layer8_attention_weight_min": -43.29123306274414,
      "activations/layer9_attention_weight_max": 66.28396606445312,
      "activations/layer9_attention_weight_min": -53.384517669677734,
      "epoch": 4.58,
      "learning_rate": 0.00012171704545454544,
      "loss": 2.8807,
      "step": 78750
    },
    {
      "activations/layer0_attention_weight_max": 13.79667854309082,
      "activations/layer0_attention_weight_min": -15.201020240783691,
      "activations/layer10_attention_weight_max": 36.83552551269531,
      "activations/layer10_attention_weight_min": -35.941741943359375,
      "activations/layer11_attention_weight_max": 30.25934600830078,
      "activations/layer11_attention_weight_min": -27.991836547851562,
      "activations/layer12_attention_weight_max": 24.46733856201172,
      "activations/layer12_attention_weight_min": -30.104352951049805,
      "activations/layer13_attention_weight_max": 53.15863800048828,
      "activations/layer13_attention_weight_min": -37.06659698486328,
      "activations/layer14_attention_weight_max": 49.12223815917969,
      "activations/layer14_attention_weight_min": -31.977676391601562,
      "activations/layer15_attention_weight_max": 38.52259063720703,
      "activations/layer15_attention_weight_min": -30.732635498046875,
      "activations/layer16_attention_weight_max": 34.28913497924805,
      "activations/layer16_attention_weight_min": -27.206466674804688,
      "activations/layer17_attention_weight_max": 55.3504524230957,
      "activations/layer17_attention_weight_min": -40.331180572509766,
      "activations/layer18_attention_weight_max": 47.9120979309082,
      "activations/layer18_attention_weight_min": -34.37297439575195,
      "activations/layer19_attention_weight_max": 19.63068389892578,
      "activations/layer19_attention_weight_min": -20.794355392456055,
      "activations/layer1_attention_weight_max": 18.053966522216797,
      "activations/layer1_attention_weight_min": -14.727476119995117,
      "activations/layer20_attention_weight_max": 22.27632713317871,
      "activations/layer20_attention_weight_min": -21.919282913208008,
      "activations/layer21_attention_weight_max": 39.67926788330078,
      "activations/layer21_attention_weight_min": -28.836503982543945,
      "activations/layer22_attention_weight_max": 33.267356872558594,
      "activations/layer22_attention_weight_min": -24.387042999267578,
      "activations/layer23_attention_weight_max": 25.99740982055664,
      "activations/layer23_attention_weight_min": -17.986713409423828,
      "activations/layer2_attention_weight_max": 30.0894775390625,
      "activations/layer2_attention_weight_min": -27.157407760620117,
      "activations/layer3_attention_weight_max": 75.89132690429688,
      "activations/layer3_attention_weight_min": -83.74362182617188,
      "activations/layer4_attention_weight_max": 95.9609146118164,
      "activations/layer4_attention_weight_min": -90.54930114746094,
      "activations/layer5_attention_weight_max": 111.44868469238281,
      "activations/layer5_attention_weight_min": -92.84040832519531,
      "activations/layer6_attention_weight_max": 58.84101486206055,
      "activations/layer6_attention_weight_min": -53.38591766357422,
      "activations/layer7_attention_weight_max": 66.03327941894531,
      "activations/layer7_attention_weight_min": -67.4775161743164,
      "activations/layer8_attention_weight_max": 41.11097717285156,
      "activations/layer8_attention_weight_min": -39.57735061645508,
      "activations/layer9_attention_weight_max": 48.92734909057617,
      "activations/layer9_attention_weight_min": -53.53400421142578,
      "epoch": 4.58,
      "learning_rate": 0.00012169810606060604,
      "loss": 2.8532,
      "step": 78800
    },
    {
      "activations/layer0_attention_weight_max": 13.64345645904541,
      "activations/layer0_attention_weight_min": -15.1947660446167,
      "activations/layer10_attention_weight_max": 43.39385223388672,
      "activations/layer10_attention_weight_min": -36.092201232910156,
      "activations/layer11_attention_weight_max": 32.42420196533203,
      "activations/layer11_attention_weight_min": -27.59259605407715,
      "activations/layer12_attention_weight_max": 25.56592559814453,
      "activations/layer12_attention_weight_min": -26.643150329589844,
      "activations/layer13_attention_weight_max": 67.72735595703125,
      "activations/layer13_attention_weight_min": -42.16027069091797,
      "activations/layer14_attention_weight_max": 86.49039459228516,
      "activations/layer14_attention_weight_min": -48.56211471557617,
      "activations/layer15_attention_weight_max": 64.59380340576172,
      "activations/layer15_attention_weight_min": -38.248992919921875,
      "activations/layer16_attention_weight_max": 37.9864387512207,
      "activations/layer16_attention_weight_min": -25.84695816040039,
      "activations/layer17_attention_weight_max": 66.29251861572266,
      "activations/layer17_attention_weight_min": -39.22575759887695,
      "activations/layer18_attention_weight_max": 63.09189987182617,
      "activations/layer18_attention_weight_min": -31.772342681884766,
      "activations/layer19_attention_weight_max": 23.583166122436523,
      "activations/layer19_attention_weight_min": -18.907594680786133,
      "activations/layer1_attention_weight_max": 18.611173629760742,
      "activations/layer1_attention_weight_min": -16.598997116088867,
      "activations/layer20_attention_weight_max": 22.810497283935547,
      "activations/layer20_attention_weight_min": -20.686840057373047,
      "activations/layer21_attention_weight_max": 47.79715347290039,
      "activations/layer21_attention_weight_min": -26.469799041748047,
      "activations/layer22_attention_weight_max": 37.92831802368164,
      "activations/layer22_attention_weight_min": -23.005083084106445,
      "activations/layer23_attention_weight_max": 25.531797409057617,
      "activations/layer23_attention_weight_min": -17.15294075012207,
      "activations/layer2_attention_weight_max": 29.65212631225586,
      "activations/layer2_attention_weight_min": -26.89289093017578,
      "activations/layer3_attention_weight_max": 79.3344955444336,
      "activations/layer3_attention_weight_min": -81.6436767578125,
      "activations/layer4_attention_weight_max": 95.33052825927734,
      "activations/layer4_attention_weight_min": -92.92008972167969,
      "activations/layer5_attention_weight_max": 100.6651611328125,
      "activations/layer5_attention_weight_min": -93.23190307617188,
      "activations/layer6_attention_weight_max": 56.067996978759766,
      "activations/layer6_attention_weight_min": -51.98354721069336,
      "activations/layer7_attention_weight_max": 67.92790222167969,
      "activations/layer7_attention_weight_min": -67.46986389160156,
      "activations/layer8_attention_weight_max": 42.590023040771484,
      "activations/layer8_attention_weight_min": -41.24481964111328,
      "activations/layer9_attention_weight_max": 54.29766845703125,
      "activations/layer9_attention_weight_min": -49.181270599365234,
      "epoch": 4.58,
      "learning_rate": 0.00012167916666666666,
      "loss": 2.8398,
      "step": 78850
    },
    {
      "activations/layer0_attention_weight_max": 14.429201126098633,
      "activations/layer0_attention_weight_min": -14.679242134094238,
      "activations/layer10_attention_weight_max": 37.404685974121094,
      "activations/layer10_attention_weight_min": -32.2049674987793,
      "activations/layer11_attention_weight_max": 30.219928741455078,
      "activations/layer11_attention_weight_min": -25.342243194580078,
      "activations/layer12_attention_weight_max": 26.417842864990234,
      "activations/layer12_attention_weight_min": -29.251522064208984,
      "activations/layer13_attention_weight_max": 43.33207321166992,
      "activations/layer13_attention_weight_min": -36.99635314941406,
      "activations/layer14_attention_weight_max": 46.339412689208984,
      "activations/layer14_attention_weight_min": -36.71095275878906,
      "activations/layer15_attention_weight_max": 37.84550857543945,
      "activations/layer15_attention_weight_min": -30.77581787109375,
      "activations/layer16_attention_weight_max": 32.690914154052734,
      "activations/layer16_attention_weight_min": -24.73348617553711,
      "activations/layer17_attention_weight_max": 49.4251708984375,
      "activations/layer17_attention_weight_min": -35.40195083618164,
      "activations/layer18_attention_weight_max": 42.10136413574219,
      "activations/layer18_attention_weight_min": -32.03175354003906,
      "activations/layer19_attention_weight_max": 20.69117546081543,
      "activations/layer19_attention_weight_min": -19.33455467224121,
      "activations/layer1_attention_weight_max": 17.285741806030273,
      "activations/layer1_attention_weight_min": -16.49981117248535,
      "activations/layer20_attention_weight_max": 18.71636199951172,
      "activations/layer20_attention_weight_min": -20.610898971557617,
      "activations/layer21_attention_weight_max": 37.279518127441406,
      "activations/layer21_attention_weight_min": -28.754003524780273,
      "activations/layer22_attention_weight_max": 31.438764572143555,
      "activations/layer22_attention_weight_min": -22.12805938720703,
      "activations/layer23_attention_weight_max": 24.27895164489746,
      "activations/layer23_attention_weight_min": -17.22532844543457,
      "activations/layer2_attention_weight_max": 31.474454879760742,
      "activations/layer2_attention_weight_min": -28.726987838745117,
      "activations/layer3_attention_weight_max": 85.47792053222656,
      "activations/layer3_attention_weight_min": -83.70101928710938,
      "activations/layer4_attention_weight_max": 106.6102066040039,
      "activations/layer4_attention_weight_min": -97.18029022216797,
      "activations/layer5_attention_weight_max": 103.75056457519531,
      "activations/layer5_attention_weight_min": -89.83193969726562,
      "activations/layer6_attention_weight_max": 54.97538757324219,
      "activations/layer6_attention_weight_min": -49.40005111694336,
      "activations/layer7_attention_weight_max": 68.00743865966797,
      "activations/layer7_attention_weight_min": -65.42164611816406,
      "activations/layer8_attention_weight_max": 40.16828155517578,
      "activations/layer8_attention_weight_min": -38.15742874145508,
      "activations/layer9_attention_weight_max": 54.08259201049805,
      "activations/layer9_attention_weight_min": -52.60740280151367,
      "epoch": 4.58,
      "learning_rate": 0.00012166022727272727,
      "loss": 2.8557,
      "step": 78900
    },
    {
      "activations/layer0_attention_weight_max": 14.386125564575195,
      "activations/layer0_attention_weight_min": -15.053625106811523,
      "activations/layer10_attention_weight_max": 39.00682830810547,
      "activations/layer10_attention_weight_min": -35.7291259765625,
      "activations/layer11_attention_weight_max": 29.283645629882812,
      "activations/layer11_attention_weight_min": -27.023929595947266,
      "activations/layer12_attention_weight_max": 26.570154190063477,
      "activations/layer12_attention_weight_min": -26.01293182373047,
      "activations/layer13_attention_weight_max": 50.19021224975586,
      "activations/layer13_attention_weight_min": -37.57811737060547,
      "activations/layer14_attention_weight_max": 37.05502700805664,
      "activations/layer14_attention_weight_min": -28.409053802490234,
      "activations/layer15_attention_weight_max": 38.5836181640625,
      "activations/layer15_attention_weight_min": -28.948339462280273,
      "activations/layer16_attention_weight_max": 32.867950439453125,
      "activations/layer16_attention_weight_min": -24.35445213317871,
      "activations/layer17_attention_weight_max": 56.07997131347656,
      "activations/layer17_attention_weight_min": -37.77167892456055,
      "activations/layer18_attention_weight_max": 47.05182647705078,
      "activations/layer18_attention_weight_min": -35.87764358520508,
      "activations/layer19_attention_weight_max": 25.816696166992188,
      "activations/layer19_attention_weight_min": -18.82282257080078,
      "activations/layer1_attention_weight_max": 18.585758209228516,
      "activations/layer1_attention_weight_min": -16.031627655029297,
      "activations/layer20_attention_weight_max": 22.783748626708984,
      "activations/layer20_attention_weight_min": -21.634227752685547,
      "activations/layer21_attention_weight_max": 36.207313537597656,
      "activations/layer21_attention_weight_min": -27.09360694885254,
      "activations/layer22_attention_weight_max": 33.18912124633789,
      "activations/layer22_attention_weight_min": -21.801441192626953,
      "activations/layer23_attention_weight_max": 26.406225204467773,
      "activations/layer23_attention_weight_min": -17.87936019897461,
      "activations/layer2_attention_weight_max": 29.477209091186523,
      "activations/layer2_attention_weight_min": -27.681594848632812,
      "activations/layer3_attention_weight_max": 80.22035217285156,
      "activations/layer3_attention_weight_min": -81.46919250488281,
      "activations/layer4_attention_weight_max": 100.66277313232422,
      "activations/layer4_attention_weight_min": -95.68705749511719,
      "activations/layer5_attention_weight_max": 109.41366577148438,
      "activations/layer5_attention_weight_min": -98.7766342163086,
      "activations/layer6_attention_weight_max": 62.20151138305664,
      "activations/layer6_attention_weight_min": -54.34669494628906,
      "activations/layer7_attention_weight_max": 71.47614288330078,
      "activations/layer7_attention_weight_min": -65.53352355957031,
      "activations/layer8_attention_weight_max": 41.14377975463867,
      "activations/layer8_attention_weight_min": -38.93418502807617,
      "activations/layer9_attention_weight_max": 51.35333251953125,
      "activations/layer9_attention_weight_min": -45.811561584472656,
      "epoch": 4.59,
      "learning_rate": 0.00012164128787878786,
      "loss": 2.8591,
      "step": 78950
    },
    {
      "activations/layer0_attention_weight_max": 15.23581600189209,
      "activations/layer0_attention_weight_min": -15.641794204711914,
      "activations/layer10_attention_weight_max": 41.153900146484375,
      "activations/layer10_attention_weight_min": -33.45454025268555,
      "activations/layer11_attention_weight_max": 33.402469635009766,
      "activations/layer11_attention_weight_min": -27.345462799072266,
      "activations/layer12_attention_weight_max": 26.147750854492188,
      "activations/layer12_attention_weight_min": -26.42011070251465,
      "activations/layer13_attention_weight_max": 67.76790618896484,
      "activations/layer13_attention_weight_min": -40.93281936645508,
      "activations/layer14_attention_weight_max": 75.23350524902344,
      "activations/layer14_attention_weight_min": -41.3127555847168,
      "activations/layer15_attention_weight_max": 57.835914611816406,
      "activations/layer15_attention_weight_min": -31.67626953125,
      "activations/layer16_attention_weight_max": 42.97092056274414,
      "activations/layer16_attention_weight_min": -27.19400405883789,
      "activations/layer17_attention_weight_max": 65.75501251220703,
      "activations/layer17_attention_weight_min": -36.76370620727539,
      "activations/layer18_attention_weight_max": 56.672119140625,
      "activations/layer18_attention_weight_min": -33.873512268066406,
      "activations/layer19_attention_weight_max": 24.761484146118164,
      "activations/layer19_attention_weight_min": -18.201780319213867,
      "activations/layer1_attention_weight_max": 18.02237892150879,
      "activations/layer1_attention_weight_min": -14.968284606933594,
      "activations/layer20_attention_weight_max": 26.357975006103516,
      "activations/layer20_attention_weight_min": -21.312803268432617,
      "activations/layer21_attention_weight_max": 47.35797882080078,
      "activations/layer21_attention_weight_min": -29.151639938354492,
      "activations/layer22_attention_weight_max": 42.43912124633789,
      "activations/layer22_attention_weight_min": -23.083110809326172,
      "activations/layer23_attention_weight_max": 32.80928039550781,
      "activations/layer23_attention_weight_min": -18.540645599365234,
      "activations/layer2_attention_weight_max": 31.815753936767578,
      "activations/layer2_attention_weight_min": -28.80461883544922,
      "activations/layer3_attention_weight_max": 79.43087768554688,
      "activations/layer3_attention_weight_min": -84.19918823242188,
      "activations/layer4_attention_weight_max": 98.6828842163086,
      "activations/layer4_attention_weight_min": -93.64039611816406,
      "activations/layer5_attention_weight_max": 98.84284973144531,
      "activations/layer5_attention_weight_min": -89.65779113769531,
      "activations/layer6_attention_weight_max": 56.066715240478516,
      "activations/layer6_attention_weight_min": -48.015968322753906,
      "activations/layer7_attention_weight_max": 68.61451721191406,
      "activations/layer7_attention_weight_min": -68.6566162109375,
      "activations/layer8_attention_weight_max": 41.51871109008789,
      "activations/layer8_attention_weight_min": -40.41667175292969,
      "activations/layer9_attention_weight_max": 55.502471923828125,
      "activations/layer9_attention_weight_min": -49.003292083740234,
      "epoch": 4.59,
      "learning_rate": 0.00012162234848484848,
      "loss": 2.87,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_loss": 2.810546875,
      "eval_runtime": 8.5222,
      "eval_samples_per_second": 503.862,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_openwebtext_loss": 2.810546875,
      "eval_openwebtext_ppl": 16.61900425204837,
      "eval_openwebtext_runtime": 8.5222,
      "eval_openwebtext_samples_per_second": 503.862,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_wikitext_loss": 3.046875,
      "eval_wikitext_ppl": 21.049461966173517,
      "eval_wikitext_runtime": 1.9374,
      "eval_wikitext_samples_per_second": 235.362,
      "step": 79000
    },
    {
      "epoch": 4.59,
      "eval_lambada_loss": 2.876953125,
      "eval_lambada_ppl": 17.76007792103387,
      "eval_lambada_runtime": 9.6079,
      "eval_lambada_samples_per_second": 506.768,
      "step": 79000
    },
    {
      "activations/layer0_attention_weight_max": 14.97104549407959,
      "activations/layer0_attention_weight_min": -14.75424861907959,
      "activations/layer10_attention_weight_max": 38.325130462646484,
      "activations/layer10_attention_weight_min": -33.5721321105957,
      "activations/layer11_attention_weight_max": 30.39881706237793,
      "activations/layer11_attention_weight_min": -26.530132293701172,
      "activations/layer12_attention_weight_max": 29.340885162353516,
      "activations/layer12_attention_weight_min": -23.75055503845215,
      "activations/layer13_attention_weight_max": 56.365028381347656,
      "activations/layer13_attention_weight_min": -47.05716323852539,
      "activations/layer14_attention_weight_max": 54.72679901123047,
      "activations/layer14_attention_weight_min": -41.7135124206543,
      "activations/layer15_attention_weight_max": 41.989627838134766,
      "activations/layer15_attention_weight_min": -34.263465881347656,
      "activations/layer16_attention_weight_max": 36.76201629638672,
      "activations/layer16_attention_weight_min": -27.08259391784668,
      "activations/layer17_attention_weight_max": 57.9477424621582,
      "activations/layer17_attention_weight_min": -37.83660125732422,
      "activations/layer18_attention_weight_max": 49.05767059326172,
      "activations/layer18_attention_weight_min": -33.83737564086914,
      "activations/layer19_attention_weight_max": 24.02669906616211,
      "activations/layer19_attention_weight_min": -20.4234676361084,
      "activations/layer1_attention_weight_max": 17.63472557067871,
      "activations/layer1_attention_weight_min": -13.985925674438477,
      "activations/layer20_attention_weight_max": 20.437217712402344,
      "activations/layer20_attention_weight_min": -21.808406829833984,
      "activations/layer21_attention_weight_max": 36.309303283691406,
      "activations/layer21_attention_weight_min": -29.041593551635742,
      "activations/layer22_attention_weight_max": 32.2609977722168,
      "activations/layer22_attention_weight_min": -22.626934051513672,
      "activations/layer23_attention_weight_max": 27.473575592041016,
      "activations/layer23_attention_weight_min": -16.436445236206055,
      "activations/layer2_attention_weight_max": 32.832305908203125,
      "activations/layer2_attention_weight_min": -29.42208480834961,
      "activations/layer3_attention_weight_max": 79.96778869628906,
      "activations/layer3_attention_weight_min": -82.69678497314453,
      "activations/layer4_attention_weight_max": 97.90452575683594,
      "activations/layer4_attention_weight_min": -96.16578674316406,
      "activations/layer5_attention_weight_max": 103.27696228027344,
      "activations/layer5_attention_weight_min": -99.57931518554688,
      "activations/layer6_attention_weight_max": 58.391944885253906,
      "activations/layer6_attention_weight_min": -52.224666595458984,
      "activations/layer7_attention_weight_max": 68.82806396484375,
      "activations/layer7_attention_weight_min": -64.26802062988281,
      "activations/layer8_attention_weight_max": 42.18527603149414,
      "activations/layer8_attention_weight_min": -41.95187759399414,
      "activations/layer9_attention_weight_max": 49.73581314086914,
      "activations/layer9_attention_weight_min": -46.87554168701172,
      "epoch": 4.59,
      "learning_rate": 0.00012160340909090909,
      "loss": 2.8577,
      "step": 79050
    },
    {
      "activations/layer0_attention_weight_max": 14.531798362731934,
      "activations/layer0_attention_weight_min": -15.109871864318848,
      "activations/layer10_attention_weight_max": 36.92771530151367,
      "activations/layer10_attention_weight_min": -33.2296028137207,
      "activations/layer11_attention_weight_max": 31.183433532714844,
      "activations/layer11_attention_weight_min": -29.107730865478516,
      "activations/layer12_attention_weight_max": 21.748197555541992,
      "activations/layer12_attention_weight_min": -24.86248016357422,
      "activations/layer13_attention_weight_max": 47.92445373535156,
      "activations/layer13_attention_weight_min": -33.31532669067383,
      "activations/layer14_attention_weight_max": 41.59080123901367,
      "activations/layer14_attention_weight_min": -29.169511795043945,
      "activations/layer15_attention_weight_max": 41.778255462646484,
      "activations/layer15_attention_weight_min": -30.313352584838867,
      "activations/layer16_attention_weight_max": 34.31717300415039,
      "activations/layer16_attention_weight_min": -27.663223266601562,
      "activations/layer17_attention_weight_max": 55.62945556640625,
      "activations/layer17_attention_weight_min": -37.57704544067383,
      "activations/layer18_attention_weight_max": 51.604156494140625,
      "activations/layer18_attention_weight_min": -33.978492736816406,
      "activations/layer19_attention_weight_max": 19.583293914794922,
      "activations/layer19_attention_weight_min": -19.715259552001953,
      "activations/layer1_attention_weight_max": 17.09231185913086,
      "activations/layer1_attention_weight_min": -13.728046417236328,
      "activations/layer20_attention_weight_max": 21.456573486328125,
      "activations/layer20_attention_weight_min": -20.37651252746582,
      "activations/layer21_attention_weight_max": 41.16613006591797,
      "activations/layer21_attention_weight_min": -27.96671485900879,
      "activations/layer22_attention_weight_max": 32.787498474121094,
      "activations/layer22_attention_weight_min": -24.44563102722168,
      "activations/layer23_attention_weight_max": 27.150314331054688,
      "activations/layer23_attention_weight_min": -17.87759017944336,
      "activations/layer2_attention_weight_max": 30.589574813842773,
      "activations/layer2_attention_weight_min": -29.85970115661621,
      "activations/layer3_attention_weight_max": 80.80931091308594,
      "activations/layer3_attention_weight_min": -85.8821792602539,
      "activations/layer4_attention_weight_max": 98.9967269897461,
      "activations/layer4_attention_weight_min": -90.40489959716797,
      "activations/layer5_attention_weight_max": 101.68220520019531,
      "activations/layer5_attention_weight_min": -95.6701889038086,
      "activations/layer6_attention_weight_max": 55.6587028503418,
      "activations/layer6_attention_weight_min": -52.025535583496094,
      "activations/layer7_attention_weight_max": 63.83034896850586,
      "activations/layer7_attention_weight_min": -63.10588073730469,
      "activations/layer8_attention_weight_max": 40.27119445800781,
      "activations/layer8_attention_weight_min": -39.4622688293457,
      "activations/layer9_attention_weight_max": 45.02657699584961,
      "activations/layer9_attention_weight_min": -45.345699310302734,
      "epoch": 4.6,
      "learning_rate": 0.00012158446969696968,
      "loss": 2.8615,
      "step": 79100
    },
    {
      "activations/layer0_attention_weight_max": 14.599024772644043,
      "activations/layer0_attention_weight_min": -14.901692390441895,
      "activations/layer10_attention_weight_max": 38.780677795410156,
      "activations/layer10_attention_weight_min": -32.2276496887207,
      "activations/layer11_attention_weight_max": 29.443260192871094,
      "activations/layer11_attention_weight_min": -26.435543060302734,
      "activations/layer12_attention_weight_max": 24.577054977416992,
      "activations/layer12_attention_weight_min": -26.433679580688477,
      "activations/layer13_attention_weight_max": 55.16802215576172,
      "activations/layer13_attention_weight_min": -36.74607467651367,
      "activations/layer14_attention_weight_max": 43.96993637084961,
      "activations/layer14_attention_weight_min": -29.151403427124023,
      "activations/layer15_attention_weight_max": 45.4632568359375,
      "activations/layer15_attention_weight_min": -33.677974700927734,
      "activations/layer16_attention_weight_max": 38.862030029296875,
      "activations/layer16_attention_weight_min": -27.032716751098633,
      "activations/layer17_attention_weight_max": 64.96027374267578,
      "activations/layer17_attention_weight_min": -39.872920989990234,
      "activations/layer18_attention_weight_max": 54.33799362182617,
      "activations/layer18_attention_weight_min": -33.78843688964844,
      "activations/layer19_attention_weight_max": 25.20931625366211,
      "activations/layer19_attention_weight_min": -18.191556930541992,
      "activations/layer1_attention_weight_max": 19.362873077392578,
      "activations/layer1_attention_weight_min": -15.390122413635254,
      "activations/layer20_attention_weight_max": 26.687944412231445,
      "activations/layer20_attention_weight_min": -20.802858352661133,
      "activations/layer21_attention_weight_max": 39.549007415771484,
      "activations/layer21_attention_weight_min": -27.098478317260742,
      "activations/layer22_attention_weight_max": 36.19588851928711,
      "activations/layer22_attention_weight_min": -22.456512451171875,
      "activations/layer23_attention_weight_max": 26.89409637451172,
      "activations/layer23_attention_weight_min": -19.983036041259766,
      "activations/layer2_attention_weight_max": 31.574920654296875,
      "activations/layer2_attention_weight_min": -28.865680694580078,
      "activations/layer3_attention_weight_max": 80.6929702758789,
      "activations/layer3_attention_weight_min": -81.06690216064453,
      "activations/layer4_attention_weight_max": 100.62742614746094,
      "activations/layer4_attention_weight_min": -92.25614929199219,
      "activations/layer5_attention_weight_max": 106.26580047607422,
      "activations/layer5_attention_weight_min": -93.45317840576172,
      "activations/layer6_attention_weight_max": 59.87969970703125,
      "activations/layer6_attention_weight_min": -53.739986419677734,
      "activations/layer7_attention_weight_max": 75.06842041015625,
      "activations/layer7_attention_weight_min": -61.83549499511719,
      "activations/layer8_attention_weight_max": 41.77464294433594,
      "activations/layer8_attention_weight_min": -40.147193908691406,
      "activations/layer9_attention_weight_max": 48.261959075927734,
      "activations/layer9_attention_weight_min": -45.399654388427734,
      "epoch": 4.6,
      "learning_rate": 0.0001215655303030303,
      "loss": 2.8621,
      "step": 79150
    },
    {
      "activations/layer0_attention_weight_max": 14.000807762145996,
      "activations/layer0_attention_weight_min": -15.121508598327637,
      "activations/layer10_attention_weight_max": 43.9804801940918,
      "activations/layer10_attention_weight_min": -36.74077224731445,
      "activations/layer11_attention_weight_max": 35.0296630859375,
      "activations/layer11_attention_weight_min": -29.973655700683594,
      "activations/layer12_attention_weight_max": 27.397008895874023,
      "activations/layer12_attention_weight_min": -24.622766494750977,
      "activations/layer13_attention_weight_max": 69.30457305908203,
      "activations/layer13_attention_weight_min": -42.75657272338867,
      "activations/layer14_attention_weight_max": 73.63082885742188,
      "activations/layer14_attention_weight_min": -40.2460823059082,
      "activations/layer15_attention_weight_max": 50.40801239013672,
      "activations/layer15_attention_weight_min": -30.799047470092773,
      "activations/layer16_attention_weight_max": 42.1373291015625,
      "activations/layer16_attention_weight_min": -26.347049713134766,
      "activations/layer17_attention_weight_max": 59.99496078491211,
      "activations/layer17_attention_weight_min": -37.605262756347656,
      "activations/layer18_attention_weight_max": 52.10527420043945,
      "activations/layer18_attention_weight_min": -34.77397918701172,
      "activations/layer19_attention_weight_max": 24.55974769592285,
      "activations/layer19_attention_weight_min": -20.069393157958984,
      "activations/layer1_attention_weight_max": 18.550674438476562,
      "activations/layer1_attention_weight_min": -16.99541664123535,
      "activations/layer20_attention_weight_max": 26.249544143676758,
      "activations/layer20_attention_weight_min": -23.019716262817383,
      "activations/layer21_attention_weight_max": 47.55231857299805,
      "activations/layer21_attention_weight_min": -29.294952392578125,
      "activations/layer22_attention_weight_max": 40.867244720458984,
      "activations/layer22_attention_weight_min": -23.90165138244629,
      "activations/layer23_attention_weight_max": 30.90387725830078,
      "activations/layer23_attention_weight_min": -16.79951286315918,
      "activations/layer2_attention_weight_max": 31.81563377380371,
      "activations/layer2_attention_weight_min": -29.833250045776367,
      "activations/layer3_attention_weight_max": 86.67969512939453,
      "activations/layer3_attention_weight_min": -90.36848449707031,
      "activations/layer4_attention_weight_max": 107.37532043457031,
      "activations/layer4_attention_weight_min": -103.25264739990234,
      "activations/layer5_attention_weight_max": 110.84799194335938,
      "activations/layer5_attention_weight_min": -101.97740173339844,
      "activations/layer6_attention_weight_max": 60.13297653198242,
      "activations/layer6_attention_weight_min": -52.18812942504883,
      "activations/layer7_attention_weight_max": 70.4752426147461,
      "activations/layer7_attention_weight_min": -70.29281616210938,
      "activations/layer8_attention_weight_max": 46.72511672973633,
      "activations/layer8_attention_weight_min": -44.901756286621094,
      "activations/layer9_attention_weight_max": 65.0577163696289,
      "activations/layer9_attention_weight_min": -54.76746368408203,
      "epoch": 4.6,
      "learning_rate": 0.0001215465909090909,
      "loss": 2.8536,
      "step": 79200
    },
    {
      "activations/layer0_attention_weight_max": 15.238142967224121,
      "activations/layer0_attention_weight_min": -14.693124771118164,
      "activations/layer10_attention_weight_max": 42.01395797729492,
      "activations/layer10_attention_weight_min": -34.52822494506836,
      "activations/layer11_attention_weight_max": 33.04579544067383,
      "activations/layer11_attention_weight_min": -25.91352081298828,
      "activations/layer12_attention_weight_max": 23.507532119750977,
      "activations/layer12_attention_weight_min": -27.323129653930664,
      "activations/layer13_attention_weight_max": 59.66874694824219,
      "activations/layer13_attention_weight_min": -39.58806610107422,
      "activations/layer14_attention_weight_max": 65.75109100341797,
      "activations/layer14_attention_weight_min": -42.238197326660156,
      "activations/layer15_attention_weight_max": 45.35490036010742,
      "activations/layer15_attention_weight_min": -33.38401794433594,
      "activations/layer16_attention_weight_max": 40.13182830810547,
      "activations/layer16_attention_weight_min": -26.20163345336914,
      "activations/layer17_attention_weight_max": 50.911659240722656,
      "activations/layer17_attention_weight_min": -37.42507553100586,
      "activations/layer18_attention_weight_max": 47.42327117919922,
      "activations/layer18_attention_weight_min": -32.67655944824219,
      "activations/layer19_attention_weight_max": 20.503950119018555,
      "activations/layer19_attention_weight_min": -21.4337100982666,
      "activations/layer1_attention_weight_max": 18.625085830688477,
      "activations/layer1_attention_weight_min": -15.484885215759277,
      "activations/layer20_attention_weight_max": 20.453563690185547,
      "activations/layer20_attention_weight_min": -21.76201057434082,
      "activations/layer21_attention_weight_max": 44.23642349243164,
      "activations/layer21_attention_weight_min": -26.43337631225586,
      "activations/layer22_attention_weight_max": 33.71489334106445,
      "activations/layer22_attention_weight_min": -24.60096549987793,
      "activations/layer23_attention_weight_max": 26.10126495361328,
      "activations/layer23_attention_weight_min": -18.76447296142578,
      "activations/layer2_attention_weight_max": 30.76021957397461,
      "activations/layer2_attention_weight_min": -29.10195541381836,
      "activations/layer3_attention_weight_max": 82.15292358398438,
      "activations/layer3_attention_weight_min": -88.36210632324219,
      "activations/layer4_attention_weight_max": 102.97552490234375,
      "activations/layer4_attention_weight_min": -96.03362274169922,
      "activations/layer5_attention_weight_max": 103.15148162841797,
      "activations/layer5_attention_weight_min": -94.01153564453125,
      "activations/layer6_attention_weight_max": 61.5067253112793,
      "activations/layer6_attention_weight_min": -50.91895294189453,
      "activations/layer7_attention_weight_max": 77.09153747558594,
      "activations/layer7_attention_weight_min": -69.5862808227539,
      "activations/layer8_attention_weight_max": 48.43739318847656,
      "activations/layer8_attention_weight_min": -42.874080657958984,
      "activations/layer9_attention_weight_max": 63.12376022338867,
      "activations/layer9_attention_weight_min": -51.2820930480957,
      "epoch": 4.6,
      "learning_rate": 0.0001215276515151515,
      "loss": 2.8726,
      "step": 79250
    },
    {
      "activations/layer0_attention_weight_max": 14.458391189575195,
      "activations/layer0_attention_weight_min": -14.844954490661621,
      "activations/layer10_attention_weight_max": 37.87436294555664,
      "activations/layer10_attention_weight_min": -35.085697174072266,
      "activations/layer11_attention_weight_max": 30.16537857055664,
      "activations/layer11_attention_weight_min": -28.036466598510742,
      "activations/layer12_attention_weight_max": 25.05624008178711,
      "activations/layer12_attention_weight_min": -34.73078918457031,
      "activations/layer13_attention_weight_max": 48.76449203491211,
      "activations/layer13_attention_weight_min": -35.726985931396484,
      "activations/layer14_attention_weight_max": 46.25101089477539,
      "activations/layer14_attention_weight_min": -32.09577941894531,
      "activations/layer15_attention_weight_max": 39.01637268066406,
      "activations/layer15_attention_weight_min": -28.481765747070312,
      "activations/layer16_attention_weight_max": 33.12708282470703,
      "activations/layer16_attention_weight_min": -26.912261962890625,
      "activations/layer17_attention_weight_max": 52.86528778076172,
      "activations/layer17_attention_weight_min": -36.516571044921875,
      "activations/layer18_attention_weight_max": 44.58686828613281,
      "activations/layer18_attention_weight_min": -32.50530242919922,
      "activations/layer19_attention_weight_max": 19.97430419921875,
      "activations/layer19_attention_weight_min": -18.860506057739258,
      "activations/layer1_attention_weight_max": 18.495698928833008,
      "activations/layer1_attention_weight_min": -14.594633102416992,
      "activations/layer20_attention_weight_max": 20.339824676513672,
      "activations/layer20_attention_weight_min": -17.77936363220215,
      "activations/layer21_attention_weight_max": 36.19927978515625,
      "activations/layer21_attention_weight_min": -28.25499153137207,
      "activations/layer22_attention_weight_max": 32.476768493652344,
      "activations/layer22_attention_weight_min": -23.552108764648438,
      "activations/layer23_attention_weight_max": 25.401166915893555,
      "activations/layer23_attention_weight_min": -17.536714553833008,
      "activations/layer2_attention_weight_max": 32.7026481628418,
      "activations/layer2_attention_weight_min": -28.569618225097656,
      "activations/layer3_attention_weight_max": 87.29669189453125,
      "activations/layer3_attention_weight_min": -86.36813354492188,
      "activations/layer4_attention_weight_max": 109.11087799072266,
      "activations/layer4_attention_weight_min": -101.26887512207031,
      "activations/layer5_attention_weight_max": 117.91241455078125,
      "activations/layer5_attention_weight_min": -101.76878356933594,
      "activations/layer6_attention_weight_max": 60.304542541503906,
      "activations/layer6_attention_weight_min": -54.05789566040039,
      "activations/layer7_attention_weight_max": 77.038818359375,
      "activations/layer7_attention_weight_min": -74.55168151855469,
      "activations/layer8_attention_weight_max": 43.849491119384766,
      "activations/layer8_attention_weight_min": -40.34040832519531,
      "activations/layer9_attention_weight_max": 58.48983383178711,
      "activations/layer9_attention_weight_min": -52.11849594116211,
      "epoch": 4.61,
      "learning_rate": 0.00012150871212121211,
      "loss": 2.8763,
      "step": 79300
    },
    {
      "activations/layer0_attention_weight_max": 14.703991889953613,
      "activations/layer0_attention_weight_min": -14.389291763305664,
      "activations/layer10_attention_weight_max": 37.93054962158203,
      "activations/layer10_attention_weight_min": -32.71049499511719,
      "activations/layer11_attention_weight_max": 28.448055267333984,
      "activations/layer11_attention_weight_min": -25.932384490966797,
      "activations/layer12_attention_weight_max": 24.5194091796875,
      "activations/layer12_attention_weight_min": -24.578636169433594,
      "activations/layer13_attention_weight_max": 49.72581100463867,
      "activations/layer13_attention_weight_min": -34.00431442260742,
      "activations/layer14_attention_weight_max": 47.31997299194336,
      "activations/layer14_attention_weight_min": -32.77571105957031,
      "activations/layer15_attention_weight_max": 40.83771896362305,
      "activations/layer15_attention_weight_min": -29.732772827148438,
      "activations/layer16_attention_weight_max": 35.87826156616211,
      "activations/layer16_attention_weight_min": -27.065433502197266,
      "activations/layer17_attention_weight_max": 55.339683532714844,
      "activations/layer17_attention_weight_min": -40.54862594604492,
      "activations/layer18_attention_weight_max": 46.84332275390625,
      "activations/layer18_attention_weight_min": -34.09611511230469,
      "activations/layer19_attention_weight_max": 23.140941619873047,
      "activations/layer19_attention_weight_min": -19.543434143066406,
      "activations/layer1_attention_weight_max": 17.629844665527344,
      "activations/layer1_attention_weight_min": -14.877086639404297,
      "activations/layer20_attention_weight_max": 22.90093994140625,
      "activations/layer20_attention_weight_min": -22.069074630737305,
      "activations/layer21_attention_weight_max": 40.4399528503418,
      "activations/layer21_attention_weight_min": -27.05223274230957,
      "activations/layer22_attention_weight_max": 32.88949966430664,
      "activations/layer22_attention_weight_min": -23.250675201416016,
      "activations/layer23_attention_weight_max": 28.0789794921875,
      "activations/layer23_attention_weight_min": -18.43147850036621,
      "activations/layer2_attention_weight_max": 30.565937042236328,
      "activations/layer2_attention_weight_min": -27.13949203491211,
      "activations/layer3_attention_weight_max": 79.89840698242188,
      "activations/layer3_attention_weight_min": -80.94319915771484,
      "activations/layer4_attention_weight_max": 99.97216033935547,
      "activations/layer4_attention_weight_min": -95.45584106445312,
      "activations/layer5_attention_weight_max": 103.492431640625,
      "activations/layer5_attention_weight_min": -92.65516662597656,
      "activations/layer6_attention_weight_max": 56.11418151855469,
      "activations/layer6_attention_weight_min": -51.78950881958008,
      "activations/layer7_attention_weight_max": 69.6868667602539,
      "activations/layer7_attention_weight_min": -64.5793685913086,
      "activations/layer8_attention_weight_max": 38.002864837646484,
      "activations/layer8_attention_weight_min": -37.193580627441406,
      "activations/layer9_attention_weight_max": 47.13248825073242,
      "activations/layer9_attention_weight_min": -44.16868591308594,
      "epoch": 4.61,
      "learning_rate": 0.00012148977272727271,
      "loss": 2.8587,
      "step": 79350
    },
    {
      "activations/layer0_attention_weight_max": 14.687536239624023,
      "activations/layer0_attention_weight_min": -14.77613639831543,
      "activations/layer10_attention_weight_max": 36.858402252197266,
      "activations/layer10_attention_weight_min": -33.77576446533203,
      "activations/layer11_attention_weight_max": 29.67837142944336,
      "activations/layer11_attention_weight_min": -26.30667495727539,
      "activations/layer12_attention_weight_max": 20.441314697265625,
      "activations/layer12_attention_weight_min": -29.09271812438965,
      "activations/layer13_attention_weight_max": 46.0417594909668,
      "activations/layer13_attention_weight_min": -34.08176803588867,
      "activations/layer14_attention_weight_max": 40.186187744140625,
      "activations/layer14_attention_weight_min": -30.198463439941406,
      "activations/layer15_attention_weight_max": 41.07798767089844,
      "activations/layer15_attention_weight_min": -33.87250900268555,
      "activations/layer16_attention_weight_max": 32.31414031982422,
      "activations/layer16_attention_weight_min": -27.693086624145508,
      "activations/layer17_attention_weight_max": 57.1513557434082,
      "activations/layer17_attention_weight_min": -39.43928909301758,
      "activations/layer18_attention_weight_max": 48.69438552856445,
      "activations/layer18_attention_weight_min": -35.847999572753906,
      "activations/layer19_attention_weight_max": 24.557174682617188,
      "activations/layer19_attention_weight_min": -18.102174758911133,
      "activations/layer1_attention_weight_max": 17.7877197265625,
      "activations/layer1_attention_weight_min": -15.013880729675293,
      "activations/layer20_attention_weight_max": 22.379192352294922,
      "activations/layer20_attention_weight_min": -21.715845108032227,
      "activations/layer21_attention_weight_max": 35.5107307434082,
      "activations/layer21_attention_weight_min": -27.418773651123047,
      "activations/layer22_attention_weight_max": 32.50849151611328,
      "activations/layer22_attention_weight_min": -22.92747688293457,
      "activations/layer23_attention_weight_max": 27.912588119506836,
      "activations/layer23_attention_weight_min": -17.109943389892578,
      "activations/layer2_attention_weight_max": 29.82784080505371,
      "activations/layer2_attention_weight_min": -26.757688522338867,
      "activations/layer3_attention_weight_max": 79.04446411132812,
      "activations/layer3_attention_weight_min": -78.61874389648438,
      "activations/layer4_attention_weight_max": 95.93981170654297,
      "activations/layer4_attention_weight_min": -86.9885482788086,
      "activations/layer5_attention_weight_max": 100.29387664794922,
      "activations/layer5_attention_weight_min": -87.98876190185547,
      "activations/layer6_attention_weight_max": 56.532962799072266,
      "activations/layer6_attention_weight_min": -50.31641387939453,
      "activations/layer7_attention_weight_max": 68.15485382080078,
      "activations/layer7_attention_weight_min": -60.68657684326172,
      "activations/layer8_attention_weight_max": 38.84563446044922,
      "activations/layer8_attention_weight_min": -36.9964485168457,
      "activations/layer9_attention_weight_max": 57.029823303222656,
      "activations/layer9_attention_weight_min": -49.024559020996094,
      "epoch": 4.61,
      "learning_rate": 0.00012147083333333333,
      "loss": 2.8484,
      "step": 79400
    },
    {
      "activations/layer0_attention_weight_max": 14.09442138671875,
      "activations/layer0_attention_weight_min": -14.677571296691895,
      "activations/layer10_attention_weight_max": 36.68708419799805,
      "activations/layer10_attention_weight_min": -33.72450637817383,
      "activations/layer11_attention_weight_max": 28.206716537475586,
      "activations/layer11_attention_weight_min": -25.422813415527344,
      "activations/layer12_attention_weight_max": 28.988859176635742,
      "activations/layer12_attention_weight_min": -22.705820083618164,
      "activations/layer13_attention_weight_max": 43.78254318237305,
      "activations/layer13_attention_weight_min": -34.21977996826172,
      "activations/layer14_attention_weight_max": 40.377342224121094,
      "activations/layer14_attention_weight_min": -30.525257110595703,
      "activations/layer15_attention_weight_max": 41.403079986572266,
      "activations/layer15_attention_weight_min": -31.583925247192383,
      "activations/layer16_attention_weight_max": 35.36631393432617,
      "activations/layer16_attention_weight_min": -27.128236770629883,
      "activations/layer17_attention_weight_max": 58.75335693359375,
      "activations/layer17_attention_weight_min": -39.718727111816406,
      "activations/layer18_attention_weight_max": 49.208091735839844,
      "activations/layer18_attention_weight_min": -34.26263427734375,
      "activations/layer19_attention_weight_max": 22.53489112854004,
      "activations/layer19_attention_weight_min": -19.154769897460938,
      "activations/layer1_attention_weight_max": 18.50534439086914,
      "activations/layer1_attention_weight_min": -15.310872077941895,
      "activations/layer20_attention_weight_max": 22.35491180419922,
      "activations/layer20_attention_weight_min": -20.204933166503906,
      "activations/layer21_attention_weight_max": 38.54542922973633,
      "activations/layer21_attention_weight_min": -25.829082489013672,
      "activations/layer22_attention_weight_max": 32.162349700927734,
      "activations/layer22_attention_weight_min": -22.788482666015625,
      "activations/layer23_attention_weight_max": 26.590770721435547,
      "activations/layer23_attention_weight_min": -18.762035369873047,
      "activations/layer2_attention_weight_max": 32.490760803222656,
      "activations/layer2_attention_weight_min": -30.322265625,
      "activations/layer3_attention_weight_max": 81.81852722167969,
      "activations/layer3_attention_weight_min": -84.9723129272461,
      "activations/layer4_attention_weight_max": 101.24551391601562,
      "activations/layer4_attention_weight_min": -100.40825653076172,
      "activations/layer5_attention_weight_max": 105.6326904296875,
      "activations/layer5_attention_weight_min": -95.24856567382812,
      "activations/layer6_attention_weight_max": 57.42042541503906,
      "activations/layer6_attention_weight_min": -53.35585403442383,
      "activations/layer7_attention_weight_max": 70.58551788330078,
      "activations/layer7_attention_weight_min": -63.5146369934082,
      "activations/layer8_attention_weight_max": 46.13593292236328,
      "activations/layer8_attention_weight_min": -39.09602737426758,
      "activations/layer9_attention_weight_max": 52.87223434448242,
      "activations/layer9_attention_weight_min": -48.52164840698242,
      "epoch": 4.62,
      "learning_rate": 0.00012145189393939393,
      "loss": 2.8708,
      "step": 79450
    },
    {
      "activations/layer0_attention_weight_max": 14.641438484191895,
      "activations/layer0_attention_weight_min": -15.167370796203613,
      "activations/layer10_attention_weight_max": 34.04771423339844,
      "activations/layer10_attention_weight_min": -30.809261322021484,
      "activations/layer11_attention_weight_max": 29.69686508178711,
      "activations/layer11_attention_weight_min": -25.382415771484375,
      "activations/layer12_attention_weight_max": 21.311725616455078,
      "activations/layer12_attention_weight_min": -30.57297134399414,
      "activations/layer13_attention_weight_max": 44.38304901123047,
      "activations/layer13_attention_weight_min": -33.59298324584961,
      "activations/layer14_attention_weight_max": 50.05347442626953,
      "activations/layer14_attention_weight_min": -34.77865219116211,
      "activations/layer15_attention_weight_max": 39.74223327636719,
      "activations/layer15_attention_weight_min": -31.391902923583984,
      "activations/layer16_attention_weight_max": 31.029224395751953,
      "activations/layer16_attention_weight_min": -25.143463134765625,
      "activations/layer17_attention_weight_max": 53.62498092651367,
      "activations/layer17_attention_weight_min": -39.062835693359375,
      "activations/layer18_attention_weight_max": 45.28075408935547,
      "activations/layer18_attention_weight_min": -32.693321228027344,
      "activations/layer19_attention_weight_max": 21.40936851501465,
      "activations/layer19_attention_weight_min": -18.934415817260742,
      "activations/layer1_attention_weight_max": 17.014841079711914,
      "activations/layer1_attention_weight_min": -15.542859077453613,
      "activations/layer20_attention_weight_max": 21.35634422302246,
      "activations/layer20_attention_weight_min": -20.331859588623047,
      "activations/layer21_attention_weight_max": 35.828094482421875,
      "activations/layer21_attention_weight_min": -24.373010635375977,
      "activations/layer22_attention_weight_max": 30.05777931213379,
      "activations/layer22_attention_weight_min": -24.338050842285156,
      "activations/layer23_attention_weight_max": 24.184810638427734,
      "activations/layer23_attention_weight_min": -19.757537841796875,
      "activations/layer2_attention_weight_max": 31.639240264892578,
      "activations/layer2_attention_weight_min": -28.946577072143555,
      "activations/layer3_attention_weight_max": 82.50379180908203,
      "activations/layer3_attention_weight_min": -82.06927490234375,
      "activations/layer4_attention_weight_max": 96.19297790527344,
      "activations/layer4_attention_weight_min": -87.53736877441406,
      "activations/layer5_attention_weight_max": 100.4537353515625,
      "activations/layer5_attention_weight_min": -86.06217956542969,
      "activations/layer6_attention_weight_max": 54.300071716308594,
      "activations/layer6_attention_weight_min": -48.38417434692383,
      "activations/layer7_attention_weight_max": 69.5377426147461,
      "activations/layer7_attention_weight_min": -60.89833450317383,
      "activations/layer8_attention_weight_max": 38.38362503051758,
      "activations/layer8_attention_weight_min": -36.84873580932617,
      "activations/layer9_attention_weight_max": 50.07249450683594,
      "activations/layer9_attention_weight_min": -46.15560531616211,
      "epoch": 4.62,
      "learning_rate": 0.00012143295454545453,
      "loss": 2.8617,
      "step": 79500
    },
    {
      "activations/layer0_attention_weight_max": 15.058478355407715,
      "activations/layer0_attention_weight_min": -15.375001907348633,
      "activations/layer10_attention_weight_max": 38.831268310546875,
      "activations/layer10_attention_weight_min": -33.50831985473633,
      "activations/layer11_attention_weight_max": 30.054168701171875,
      "activations/layer11_attention_weight_min": -26.547462463378906,
      "activations/layer12_attention_weight_max": 27.70365333557129,
      "activations/layer12_attention_weight_min": -26.33524513244629,
      "activations/layer13_attention_weight_max": 47.58625411987305,
      "activations/layer13_attention_weight_min": -35.03999710083008,
      "activations/layer14_attention_weight_max": 50.473976135253906,
      "activations/layer14_attention_weight_min": -34.26465606689453,
      "activations/layer15_attention_weight_max": 42.16754913330078,
      "activations/layer15_attention_weight_min": -28.888193130493164,
      "activations/layer16_attention_weight_max": 40.642147064208984,
      "activations/layer16_attention_weight_min": -26.67082405090332,
      "activations/layer17_attention_weight_max": 60.82175064086914,
      "activations/layer17_attention_weight_min": -36.878482818603516,
      "activations/layer18_attention_weight_max": 56.17131042480469,
      "activations/layer18_attention_weight_min": -32.65180206298828,
      "activations/layer19_attention_weight_max": 23.071802139282227,
      "activations/layer19_attention_weight_min": -19.146242141723633,
      "activations/layer1_attention_weight_max": 18.551048278808594,
      "activations/layer1_attention_weight_min": -17.432849884033203,
      "activations/layer20_attention_weight_max": 23.69829559326172,
      "activations/layer20_attention_weight_min": -20.43956184387207,
      "activations/layer21_attention_weight_max": 39.095306396484375,
      "activations/layer21_attention_weight_min": -24.55906867980957,
      "activations/layer22_attention_weight_max": 36.49720001220703,
      "activations/layer22_attention_weight_min": -23.32275390625,
      "activations/layer23_attention_weight_max": 27.120662689208984,
      "activations/layer23_attention_weight_min": -17.67328643798828,
      "activations/layer2_attention_weight_max": 33.79155349731445,
      "activations/layer2_attention_weight_min": -30.56856346130371,
      "activations/layer3_attention_weight_max": 79.1523208618164,
      "activations/layer3_attention_weight_min": -80.11689758300781,
      "activations/layer4_attention_weight_max": 100.50898742675781,
      "activations/layer4_attention_weight_min": -91.44835662841797,
      "activations/layer5_attention_weight_max": 104.46852111816406,
      "activations/layer5_attention_weight_min": -90.77275085449219,
      "activations/layer6_attention_weight_max": 58.79335403442383,
      "activations/layer6_attention_weight_min": -49.732486724853516,
      "activations/layer7_attention_weight_max": 67.69207000732422,
      "activations/layer7_attention_weight_min": -63.93864059448242,
      "activations/layer8_attention_weight_max": 40.57349395751953,
      "activations/layer8_attention_weight_min": -40.42699432373047,
      "activations/layer9_attention_weight_max": 49.915897369384766,
      "activations/layer9_attention_weight_min": -46.01585006713867,
      "epoch": 4.62,
      "learning_rate": 0.00012141401515151515,
      "loss": 2.8739,
      "step": 79550
    },
    {
      "activations/layer0_attention_weight_max": 14.115898132324219,
      "activations/layer0_attention_weight_min": -14.947209358215332,
      "activations/layer10_attention_weight_max": 37.25582504272461,
      "activations/layer10_attention_weight_min": -32.87617492675781,
      "activations/layer11_attention_weight_max": 31.12989616394043,
      "activations/layer11_attention_weight_min": -26.769359588623047,
      "activations/layer12_attention_weight_max": 21.151594161987305,
      "activations/layer12_attention_weight_min": -26.237577438354492,
      "activations/layer13_attention_weight_max": 46.61320114135742,
      "activations/layer13_attention_weight_min": -33.588260650634766,
      "activations/layer14_attention_weight_max": 46.734092712402344,
      "activations/layer14_attention_weight_min": -30.049880981445312,
      "activations/layer15_attention_weight_max": 40.098594665527344,
      "activations/layer15_attention_weight_min": -30.90764617919922,
      "activations/layer16_attention_weight_max": 32.871219635009766,
      "activations/layer16_attention_weight_min": -27.443077087402344,
      "activations/layer17_attention_weight_max": 55.287925720214844,
      "activations/layer17_attention_weight_min": -37.80323028564453,
      "activations/layer18_attention_weight_max": 46.35466766357422,
      "activations/layer18_attention_weight_min": -33.21440887451172,
      "activations/layer19_attention_weight_max": 19.303556442260742,
      "activations/layer19_attention_weight_min": -19.658044815063477,
      "activations/layer1_attention_weight_max": 18.388778686523438,
      "activations/layer1_attention_weight_min": -15.398728370666504,
      "activations/layer20_attention_weight_max": 19.479393005371094,
      "activations/layer20_attention_weight_min": -21.27962875366211,
      "activations/layer21_attention_weight_max": 34.973060607910156,
      "activations/layer21_attention_weight_min": -29.624717712402344,
      "activations/layer22_attention_weight_max": 36.670467376708984,
      "activations/layer22_attention_weight_min": -24.16044807434082,
      "activations/layer23_attention_weight_max": 23.73391342163086,
      "activations/layer23_attention_weight_min": -17.04246711730957,
      "activations/layer2_attention_weight_max": 31.210582733154297,
      "activations/layer2_attention_weight_min": -29.508689880371094,
      "activations/layer3_attention_weight_max": 83.20277404785156,
      "activations/layer3_attention_weight_min": -84.4710922241211,
      "activations/layer4_attention_weight_max": 103.69022369384766,
      "activations/layer4_attention_weight_min": -98.12688446044922,
      "activations/layer5_attention_weight_max": 111.26687622070312,
      "activations/layer5_attention_weight_min": -96.703125,
      "activations/layer6_attention_weight_max": 59.7271614074707,
      "activations/layer6_attention_weight_min": -54.099266052246094,
      "activations/layer7_attention_weight_max": 69.93949890136719,
      "activations/layer7_attention_weight_min": -62.72308349609375,
      "activations/layer8_attention_weight_max": 42.51729202270508,
      "activations/layer8_attention_weight_min": -40.061363220214844,
      "activations/layer9_attention_weight_max": 49.98279571533203,
      "activations/layer9_attention_weight_min": -44.02898025512695,
      "epoch": 4.63,
      "learning_rate": 0.00012139507575757575,
      "loss": 2.866,
      "step": 79600
    },
    {
      "activations/layer0_attention_weight_max": 14.476776123046875,
      "activations/layer0_attention_weight_min": -14.686786651611328,
      "activations/layer10_attention_weight_max": 44.516666412353516,
      "activations/layer10_attention_weight_min": -37.1048583984375,
      "activations/layer11_attention_weight_max": 34.382164001464844,
      "activations/layer11_attention_weight_min": -27.13913345336914,
      "activations/layer12_attention_weight_max": 22.824520111083984,
      "activations/layer12_attention_weight_min": -25.839527130126953,
      "activations/layer13_attention_weight_max": 58.442291259765625,
      "activations/layer13_attention_weight_min": -43.05223083496094,
      "activations/layer14_attention_weight_max": 56.95968246459961,
      "activations/layer14_attention_weight_min": -39.2038459777832,
      "activations/layer15_attention_weight_max": 48.27687072753906,
      "activations/layer15_attention_weight_min": -31.589920043945312,
      "activations/layer16_attention_weight_max": 37.194278717041016,
      "activations/layer16_attention_weight_min": -28.45755386352539,
      "activations/layer17_attention_weight_max": 66.41585540771484,
      "activations/layer17_attention_weight_min": -39.07111740112305,
      "activations/layer18_attention_weight_max": 54.958343505859375,
      "activations/layer18_attention_weight_min": -34.77003479003906,
      "activations/layer19_attention_weight_max": 28.390134811401367,
      "activations/layer19_attention_weight_min": -18.962156295776367,
      "activations/layer1_attention_weight_max": 17.9383487701416,
      "activations/layer1_attention_weight_min": -15.978060722351074,
      "activations/layer20_attention_weight_max": 25.75530242919922,
      "activations/layer20_attention_weight_min": -23.500534057617188,
      "activations/layer21_attention_weight_max": 49.5787239074707,
      "activations/layer21_attention_weight_min": -24.839996337890625,
      "activations/layer22_attention_weight_max": 34.54207992553711,
      "activations/layer22_attention_weight_min": -22.751781463623047,
      "activations/layer23_attention_weight_max": 28.510520935058594,
      "activations/layer23_attention_weight_min": -17.701824188232422,
      "activations/layer2_attention_weight_max": 30.615612030029297,
      "activations/layer2_attention_weight_min": -28.90699577331543,
      "activations/layer3_attention_weight_max": 82.78126525878906,
      "activations/layer3_attention_weight_min": -84.5784912109375,
      "activations/layer4_attention_weight_max": 102.52019500732422,
      "activations/layer4_attention_weight_min": -99.33696746826172,
      "activations/layer5_attention_weight_max": 107.90522766113281,
      "activations/layer5_attention_weight_min": -103.70359802246094,
      "activations/layer6_attention_weight_max": 59.995941162109375,
      "activations/layer6_attention_weight_min": -55.4876823425293,
      "activations/layer7_attention_weight_max": 68.86478424072266,
      "activations/layer7_attention_weight_min": -70.66047668457031,
      "activations/layer8_attention_weight_max": 46.72792053222656,
      "activations/layer8_attention_weight_min": -42.67931365966797,
      "activations/layer9_attention_weight_max": 57.736385345458984,
      "activations/layer9_attention_weight_min": -53.35383224487305,
      "epoch": 4.63,
      "learning_rate": 0.00012137613636363635,
      "loss": 2.8793,
      "step": 79650
    },
    {
      "activations/layer0_attention_weight_max": 14.71833610534668,
      "activations/layer0_attention_weight_min": -15.436655044555664,
      "activations/layer10_attention_weight_max": 38.247684478759766,
      "activations/layer10_attention_weight_min": -33.8148078918457,
      "activations/layer11_attention_weight_max": 31.669706344604492,
      "activations/layer11_attention_weight_min": -28.19074249267578,
      "activations/layer12_attention_weight_max": 24.43980598449707,
      "activations/layer12_attention_weight_min": -25.17896842956543,
      "activations/layer13_attention_weight_max": 49.45930862426758,
      "activations/layer13_attention_weight_min": -37.355674743652344,
      "activations/layer14_attention_weight_max": 65.05191802978516,
      "activations/layer14_attention_weight_min": -36.704612731933594,
      "activations/layer15_attention_weight_max": 39.182552337646484,
      "activations/layer15_attention_weight_min": -31.567392349243164,
      "activations/layer16_attention_weight_max": 36.74466323852539,
      "activations/layer16_attention_weight_min": -27.1003475189209,
      "activations/layer17_attention_weight_max": 60.3764533996582,
      "activations/layer17_attention_weight_min": -38.64702224731445,
      "activations/layer18_attention_weight_max": 49.88235855102539,
      "activations/layer18_attention_weight_min": -33.07685852050781,
      "activations/layer19_attention_weight_max": 25.2902889251709,
      "activations/layer19_attention_weight_min": -19.61856460571289,
      "activations/layer1_attention_weight_max": 17.934913635253906,
      "activations/layer1_attention_weight_min": -16.384546279907227,
      "activations/layer20_attention_weight_max": 24.100542068481445,
      "activations/layer20_attention_weight_min": -22.280460357666016,
      "activations/layer21_attention_weight_max": 42.573646545410156,
      "activations/layer21_attention_weight_min": -26.522136688232422,
      "activations/layer22_attention_weight_max": 36.07479476928711,
      "activations/layer22_attention_weight_min": -24.944169998168945,
      "activations/layer23_attention_weight_max": 30.87776756286621,
      "activations/layer23_attention_weight_min": -17.74734115600586,
      "activations/layer2_attention_weight_max": 30.64546775817871,
      "activations/layer2_attention_weight_min": -28.377912521362305,
      "activations/layer3_attention_weight_max": 85.00326538085938,
      "activations/layer3_attention_weight_min": -84.77171325683594,
      "activations/layer4_attention_weight_max": 101.61550903320312,
      "activations/layer4_attention_weight_min": -89.76235961914062,
      "activations/layer5_attention_weight_max": 105.87739562988281,
      "activations/layer5_attention_weight_min": -95.60633850097656,
      "activations/layer6_attention_weight_max": 58.5118522644043,
      "activations/layer6_attention_weight_min": -53.82526779174805,
      "activations/layer7_attention_weight_max": 73.95047760009766,
      "activations/layer7_attention_weight_min": -66.71631622314453,
      "activations/layer8_attention_weight_max": 40.076332092285156,
      "activations/layer8_attention_weight_min": -38.96799087524414,
      "activations/layer9_attention_weight_max": 55.52594757080078,
      "activations/layer9_attention_weight_min": -49.325904846191406,
      "epoch": 4.63,
      "learning_rate": 0.00012135719696969697,
      "loss": 2.8622,
      "step": 79700
    },
    {
      "activations/layer0_attention_weight_max": 13.914569854736328,
      "activations/layer0_attention_weight_min": -15.101256370544434,
      "activations/layer10_attention_weight_max": 42.730995178222656,
      "activations/layer10_attention_weight_min": -36.73419189453125,
      "activations/layer11_attention_weight_max": 33.203346252441406,
      "activations/layer11_attention_weight_min": -28.02535629272461,
      "activations/layer12_attention_weight_max": 23.345935821533203,
      "activations/layer12_attention_weight_min": -23.181848526000977,
      "activations/layer13_attention_weight_max": 57.90284729003906,
      "activations/layer13_attention_weight_min": -39.318115234375,
      "activations/layer14_attention_weight_max": 52.58194351196289,
      "activations/layer14_attention_weight_min": -33.62587356567383,
      "activations/layer15_attention_weight_max": 52.137577056884766,
      "activations/layer15_attention_weight_min": -31.93337631225586,
      "activations/layer16_attention_weight_max": 39.478118896484375,
      "activations/layer16_attention_weight_min": -26.90349578857422,
      "activations/layer17_attention_weight_max": 73.31326293945312,
      "activations/layer17_attention_weight_min": -42.34727096557617,
      "activations/layer18_attention_weight_max": 60.85951614379883,
      "activations/layer18_attention_weight_min": -37.08942794799805,
      "activations/layer19_attention_weight_max": 26.36690330505371,
      "activations/layer19_attention_weight_min": -21.250783920288086,
      "activations/layer1_attention_weight_max": 17.817392349243164,
      "activations/layer1_attention_weight_min": -16.210506439208984,
      "activations/layer20_attention_weight_max": 28.962705612182617,
      "activations/layer20_attention_weight_min": -21.08253288269043,
      "activations/layer21_attention_weight_max": 53.251399993896484,
      "activations/layer21_attention_weight_min": -26.060754776000977,
      "activations/layer22_attention_weight_max": 44.58228302001953,
      "activations/layer22_attention_weight_min": -26.086408615112305,
      "activations/layer23_attention_weight_max": 32.44498825073242,
      "activations/layer23_attention_weight_min": -20.556236267089844,
      "activations/layer2_attention_weight_max": 31.098966598510742,
      "activations/layer2_attention_weight_min": -28.530284881591797,
      "activations/layer3_attention_weight_max": 81.40815734863281,
      "activations/layer3_attention_weight_min": -82.63697814941406,
      "activations/layer4_attention_weight_max": 103.05721282958984,
      "activations/layer4_attention_weight_min": -99.8980941772461,
      "activations/layer5_attention_weight_max": 109.4618148803711,
      "activations/layer5_attention_weight_min": -98.42811584472656,
      "activations/layer6_attention_weight_max": 58.65393829345703,
      "activations/layer6_attention_weight_min": -54.097755432128906,
      "activations/layer7_attention_weight_max": 69.79497528076172,
      "activations/layer7_attention_weight_min": -66.03205871582031,
      "activations/layer8_attention_weight_max": 42.61452102661133,
      "activations/layer8_attention_weight_min": -42.05162048339844,
      "activations/layer9_attention_weight_max": 55.8718376159668,
      "activations/layer9_attention_weight_min": -51.777008056640625,
      "epoch": 4.63,
      "learning_rate": 0.00012133825757575755,
      "loss": 2.8658,
      "step": 79750
    },
    {
      "activations/layer0_attention_weight_max": 13.905107498168945,
      "activations/layer0_attention_weight_min": -15.024173736572266,
      "activations/layer10_attention_weight_max": 37.86532211303711,
      "activations/layer10_attention_weight_min": -32.97087478637695,
      "activations/layer11_attention_weight_max": 29.61815643310547,
      "activations/layer11_attention_weight_min": -27.063987731933594,
      "activations/layer12_attention_weight_max": 23.970050811767578,
      "activations/layer12_attention_weight_min": -24.397436141967773,
      "activations/layer13_attention_weight_max": 54.61524200439453,
      "activations/layer13_attention_weight_min": -38.115234375,
      "activations/layer14_attention_weight_max": 49.45059585571289,
      "activations/layer14_attention_weight_min": -36.1567497253418,
      "activations/layer15_attention_weight_max": 41.35417556762695,
      "activations/layer15_attention_weight_min": -30.905601501464844,
      "activations/layer16_attention_weight_max": 34.424407958984375,
      "activations/layer16_attention_weight_min": -28.19156837463379,
      "activations/layer17_attention_weight_max": 52.96485900878906,
      "activations/layer17_attention_weight_min": -37.285037994384766,
      "activations/layer18_attention_weight_max": 47.613807678222656,
      "activations/layer18_attention_weight_min": -36.61540222167969,
      "activations/layer19_attention_weight_max": 19.415973663330078,
      "activations/layer19_attention_weight_min": -17.996614456176758,
      "activations/layer1_attention_weight_max": 17.833356857299805,
      "activations/layer1_attention_weight_min": -15.546504974365234,
      "activations/layer20_attention_weight_max": 21.72646141052246,
      "activations/layer20_attention_weight_min": -21.709148406982422,
      "activations/layer21_attention_weight_max": 39.227210998535156,
      "activations/layer21_attention_weight_min": -26.472057342529297,
      "activations/layer22_attention_weight_max": 32.49359893798828,
      "activations/layer22_attention_weight_min": -25.58852195739746,
      "activations/layer23_attention_weight_max": 27.17611312866211,
      "activations/layer23_attention_weight_min": -19.264108657836914,
      "activations/layer2_attention_weight_max": 29.776355743408203,
      "activations/layer2_attention_weight_min": -27.89777946472168,
      "activations/layer3_attention_weight_max": 78.70588684082031,
      "activations/layer3_attention_weight_min": -80.86842346191406,
      "activations/layer4_attention_weight_max": 98.66190338134766,
      "activations/layer4_attention_weight_min": -93.82799530029297,
      "activations/layer5_attention_weight_max": 104.94103240966797,
      "activations/layer5_attention_weight_min": -94.9384765625,
      "activations/layer6_attention_weight_max": 56.77927017211914,
      "activations/layer6_attention_weight_min": -54.34463119506836,
      "activations/layer7_attention_weight_max": 69.97488403320312,
      "activations/layer7_attention_weight_min": -63.96086883544922,
      "activations/layer8_attention_weight_max": 41.97813034057617,
      "activations/layer8_attention_weight_min": -40.52528762817383,
      "activations/layer9_attention_weight_max": 48.631649017333984,
      "activations/layer9_attention_weight_min": -44.96683883666992,
      "epoch": 4.64,
      "learning_rate": 0.00012131931818181817,
      "loss": 2.8592,
      "step": 79800
    },
    {
      "activations/layer0_attention_weight_max": 14.838676452636719,
      "activations/layer0_attention_weight_min": -14.340729713439941,
      "activations/layer10_attention_weight_max": 40.280311584472656,
      "activations/layer10_attention_weight_min": -34.81031036376953,
      "activations/layer11_attention_weight_max": 30.09150505065918,
      "activations/layer11_attention_weight_min": -27.556476593017578,
      "activations/layer12_attention_weight_max": 21.698318481445312,
      "activations/layer12_attention_weight_min": -24.661048889160156,
      "activations/layer13_attention_weight_max": 46.66642761230469,
      "activations/layer13_attention_weight_min": -34.979251861572266,
      "activations/layer14_attention_weight_max": 49.42462158203125,
      "activations/layer14_attention_weight_min": -33.026145935058594,
      "activations/layer15_attention_weight_max": 41.46563720703125,
      "activations/layer15_attention_weight_min": -29.451745986938477,
      "activations/layer16_attention_weight_max": 33.91211700439453,
      "activations/layer16_attention_weight_min": -29.506053924560547,
      "activations/layer17_attention_weight_max": 63.14484405517578,
      "activations/layer17_attention_weight_min": -38.76433563232422,
      "activations/layer18_attention_weight_max": 47.79190444946289,
      "activations/layer18_attention_weight_min": -32.4054069519043,
      "activations/layer19_attention_weight_max": 21.166364669799805,
      "activations/layer19_attention_weight_min": -18.882099151611328,
      "activations/layer1_attention_weight_max": 18.647907257080078,
      "activations/layer1_attention_weight_min": -16.236085891723633,
      "activations/layer20_attention_weight_max": 21.636682510375977,
      "activations/layer20_attention_weight_min": -18.844057083129883,
      "activations/layer21_attention_weight_max": 42.32777786254883,
      "activations/layer21_attention_weight_min": -25.33257484436035,
      "activations/layer22_attention_weight_max": 35.022090911865234,
      "activations/layer22_attention_weight_min": -23.171710968017578,
      "activations/layer23_attention_weight_max": 27.639049530029297,
      "activations/layer23_attention_weight_min": -17.130046844482422,
      "activations/layer2_attention_weight_max": 31.838788986206055,
      "activations/layer2_attention_weight_min": -28.10519790649414,
      "activations/layer3_attention_weight_max": 84.55931091308594,
      "activations/layer3_attention_weight_min": -82.90703582763672,
      "activations/layer4_attention_weight_max": 104.94847869873047,
      "activations/layer4_attention_weight_min": -96.51883697509766,
      "activations/layer5_attention_weight_max": 106.97518157958984,
      "activations/layer5_attention_weight_min": -97.61251831054688,
      "activations/layer6_attention_weight_max": 58.78620529174805,
      "activations/layer6_attention_weight_min": -55.594947814941406,
      "activations/layer7_attention_weight_max": 71.39655303955078,
      "activations/layer7_attention_weight_min": -67.49298858642578,
      "activations/layer8_attention_weight_max": 44.88808059692383,
      "activations/layer8_attention_weight_min": -40.221160888671875,
      "activations/layer9_attention_weight_max": 54.85512924194336,
      "activations/layer9_attention_weight_min": -44.495479583740234,
      "epoch": 4.64,
      "learning_rate": 0.00012130037878787878,
      "loss": 2.8458,
      "step": 79850
    },
    {
      "activations/layer0_attention_weight_max": 14.00103759765625,
      "activations/layer0_attention_weight_min": -14.624929428100586,
      "activations/layer10_attention_weight_max": 40.39854049682617,
      "activations/layer10_attention_weight_min": -36.62091064453125,
      "activations/layer11_attention_weight_max": 34.2138786315918,
      "activations/layer11_attention_weight_min": -29.876562118530273,
      "activations/layer12_attention_weight_max": 22.02429962158203,
      "activations/layer12_attention_weight_min": -31.03632164001465,
      "activations/layer13_attention_weight_max": 41.35213088989258,
      "activations/layer13_attention_weight_min": -33.87950897216797,
      "activations/layer14_attention_weight_max": 35.73163986206055,
      "activations/layer14_attention_weight_min": -30.984777450561523,
      "activations/layer15_attention_weight_max": 36.95122146606445,
      "activations/layer15_attention_weight_min": -28.78316307067871,
      "activations/layer16_attention_weight_max": 43.06166458129883,
      "activations/layer16_attention_weight_min": -27.467830657958984,
      "activations/layer17_attention_weight_max": 54.46644973754883,
      "activations/layer17_attention_weight_min": -38.81492233276367,
      "activations/layer18_attention_weight_max": 48.046504974365234,
      "activations/layer18_attention_weight_min": -33.75292205810547,
      "activations/layer19_attention_weight_max": 20.36482810974121,
      "activations/layer19_attention_weight_min": -21.157394409179688,
      "activations/layer1_attention_weight_max": 19.848562240600586,
      "activations/layer1_attention_weight_min": -14.7875394821167,
      "activations/layer20_attention_weight_max": 20.612316131591797,
      "activations/layer20_attention_weight_min": -19.475732803344727,
      "activations/layer21_attention_weight_max": 44.97174835205078,
      "activations/layer21_attention_weight_min": -25.260557174682617,
      "activations/layer22_attention_weight_max": 33.6368293762207,
      "activations/layer22_attention_weight_min": -24.716230392456055,
      "activations/layer23_attention_weight_max": 27.548011779785156,
      "activations/layer23_attention_weight_min": -17.845556259155273,
      "activations/layer2_attention_weight_max": 30.346359252929688,
      "activations/layer2_attention_weight_min": -28.047094345092773,
      "activations/layer3_attention_weight_max": 83.92576599121094,
      "activations/layer3_attention_weight_min": -87.4008560180664,
      "activations/layer4_attention_weight_max": 107.66279602050781,
      "activations/layer4_attention_weight_min": -101.93415832519531,
      "activations/layer5_attention_weight_max": 112.78517150878906,
      "activations/layer5_attention_weight_min": -103.76235961914062,
      "activations/layer6_attention_weight_max": 60.58475112915039,
      "activations/layer6_attention_weight_min": -56.919090270996094,
      "activations/layer7_attention_weight_max": 85.20887756347656,
      "activations/layer7_attention_weight_min": -71.2076644897461,
      "activations/layer8_attention_weight_max": 47.32550048828125,
      "activations/layer8_attention_weight_min": -40.99907302856445,
      "activations/layer9_attention_weight_max": 66.90238952636719,
      "activations/layer9_attention_weight_min": -52.07170486450195,
      "epoch": 4.64,
      "learning_rate": 0.00012128143939393938,
      "loss": 2.8568,
      "step": 79900
    },
    {
      "activations/layer0_attention_weight_max": 14.585474967956543,
      "activations/layer0_attention_weight_min": -14.91503620147705,
      "activations/layer10_attention_weight_max": 40.30467224121094,
      "activations/layer10_attention_weight_min": -34.341949462890625,
      "activations/layer11_attention_weight_max": 30.903152465820312,
      "activations/layer11_attention_weight_min": -26.744367599487305,
      "activations/layer12_attention_weight_max": 23.785541534423828,
      "activations/layer12_attention_weight_min": -25.084653854370117,
      "activations/layer13_attention_weight_max": 59.006919860839844,
      "activations/layer13_attention_weight_min": -42.074989318847656,
      "activations/layer14_attention_weight_max": 48.031429290771484,
      "activations/layer14_attention_weight_min": -32.743553161621094,
      "activations/layer15_attention_weight_max": 43.68690490722656,
      "activations/layer15_attention_weight_min": -32.78362274169922,
      "activations/layer16_attention_weight_max": 35.544124603271484,
      "activations/layer16_attention_weight_min": -26.310806274414062,
      "activations/layer17_attention_weight_max": 59.18898391723633,
      "activations/layer17_attention_weight_min": -38.396461486816406,
      "activations/layer18_attention_weight_max": 53.39458465576172,
      "activations/layer18_attention_weight_min": -32.30443572998047,
      "activations/layer19_attention_weight_max": 28.44130516052246,
      "activations/layer19_attention_weight_min": -19.408512115478516,
      "activations/layer1_attention_weight_max": 17.309621810913086,
      "activations/layer1_attention_weight_min": -15.600292205810547,
      "activations/layer20_attention_weight_max": 25.896055221557617,
      "activations/layer20_attention_weight_min": -20.17697525024414,
      "activations/layer21_attention_weight_max": 39.33909606933594,
      "activations/layer21_attention_weight_min": -27.747207641601562,
      "activations/layer22_attention_weight_max": 34.48937225341797,
      "activations/layer22_attention_weight_min": -22.788660049438477,
      "activations/layer23_attention_weight_max": 26.43198013305664,
      "activations/layer23_attention_weight_min": -18.869728088378906,
      "activations/layer2_attention_weight_max": 27.983348846435547,
      "activations/layer2_attention_weight_min": -26.765274047851562,
      "activations/layer3_attention_weight_max": 80.6325454711914,
      "activations/layer3_attention_weight_min": -89.82659149169922,
      "activations/layer4_attention_weight_max": 102.54068756103516,
      "activations/layer4_attention_weight_min": -102.67911529541016,
      "activations/layer5_attention_weight_max": 107.00819396972656,
      "activations/layer5_attention_weight_min": -99.63658905029297,
      "activations/layer6_attention_weight_max": 59.1794319152832,
      "activations/layer6_attention_weight_min": -57.004276275634766,
      "activations/layer7_attention_weight_max": 67.54496765136719,
      "activations/layer7_attention_weight_min": -68.81272888183594,
      "activations/layer8_attention_weight_max": 43.165035247802734,
      "activations/layer8_attention_weight_min": -44.8978271484375,
      "activations/layer9_attention_weight_max": 55.60587692260742,
      "activations/layer9_attention_weight_min": -52.30184555053711,
      "epoch": 4.65,
      "learning_rate": 0.00012126249999999999,
      "loss": 2.8554,
      "step": 79950
    },
    {
      "activations/layer0_attention_weight_max": 14.11728572845459,
      "activations/layer0_attention_weight_min": -14.238171577453613,
      "activations/layer10_attention_weight_max": 38.449676513671875,
      "activations/layer10_attention_weight_min": -34.856727600097656,
      "activations/layer11_attention_weight_max": 29.691776275634766,
      "activations/layer11_attention_weight_min": -28.95560073852539,
      "activations/layer12_attention_weight_max": 22.255151748657227,
      "activations/layer12_attention_weight_min": -24.21735954284668,
      "activations/layer13_attention_weight_max": 51.33151626586914,
      "activations/layer13_attention_weight_min": -36.8631591796875,
      "activations/layer14_attention_weight_max": 46.655975341796875,
      "activations/layer14_attention_weight_min": -30.382753372192383,
      "activations/layer15_attention_weight_max": 47.59303283691406,
      "activations/layer15_attention_weight_min": -31.87575340270996,
      "activations/layer16_attention_weight_max": 34.328521728515625,
      "activations/layer16_attention_weight_min": -27.13032341003418,
      "activations/layer17_attention_weight_max": 60.35028839111328,
      "activations/layer17_attention_weight_min": -38.847740173339844,
      "activations/layer18_attention_weight_max": 49.643463134765625,
      "activations/layer18_attention_weight_min": -34.618465423583984,
      "activations/layer19_attention_weight_max": 22.88690757751465,
      "activations/layer19_attention_weight_min": -18.040870666503906,
      "activations/layer1_attention_weight_max": 17.26873016357422,
      "activations/layer1_attention_weight_min": -16.543840408325195,
      "activations/layer20_attention_weight_max": 21.686126708984375,
      "activations/layer20_attention_weight_min": -18.912519454956055,
      "activations/layer21_attention_weight_max": 37.732276916503906,
      "activations/layer21_attention_weight_min": -29.624181747436523,
      "activations/layer22_attention_weight_max": 36.296512603759766,
      "activations/layer22_attention_weight_min": -25.972415924072266,
      "activations/layer23_attention_weight_max": 27.437088012695312,
      "activations/layer23_attention_weight_min": -19.5664119720459,
      "activations/layer2_attention_weight_max": 32.392250061035156,
      "activations/layer2_attention_weight_min": -31.20532989501953,
      "activations/layer3_attention_weight_max": 80.9184799194336,
      "activations/layer3_attention_weight_min": -83.97811126708984,
      "activations/layer4_attention_weight_max": 103.18597412109375,
      "activations/layer4_attention_weight_min": -96.81297302246094,
      "activations/layer5_attention_weight_max": 105.82432556152344,
      "activations/layer5_attention_weight_min": -99.3878173828125,
      "activations/layer6_attention_weight_max": 55.7665901184082,
      "activations/layer6_attention_weight_min": -56.87824630737305,
      "activations/layer7_attention_weight_max": 66.77835845947266,
      "activations/layer7_attention_weight_min": -68.66657257080078,
      "activations/layer8_attention_weight_max": 40.85417938232422,
      "activations/layer8_attention_weight_min": -39.50069808959961,
      "activations/layer9_attention_weight_max": 47.45539855957031,
      "activations/layer9_attention_weight_min": -46.41445541381836,
      "epoch": 4.65,
      "learning_rate": 0.0001212435606060606,
      "loss": 2.8762,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_loss": 2.810546875,
      "eval_runtime": 8.5977,
      "eval_samples_per_second": 499.436,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_openwebtext_loss": 2.810546875,
      "eval_openwebtext_ppl": 16.61900425204837,
      "eval_openwebtext_runtime": 8.5977,
      "eval_openwebtext_samples_per_second": 499.436,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_wikitext_loss": 3.0390625,
      "eval_wikitext_ppl": 20.8856537535641,
      "eval_wikitext_runtime": 1.9122,
      "eval_wikitext_samples_per_second": 238.469,
      "step": 80000
    },
    {
      "epoch": 4.65,
      "eval_lambada_loss": 2.86328125,
      "eval_lambada_ppl": 17.51891667538985,
      "eval_lambada_runtime": 9.5841,
      "eval_lambada_samples_per_second": 508.029,
      "step": 80000
    },
    {
      "activations/layer0_attention_weight_max": 14.27016544342041,
      "activations/layer0_attention_weight_min": -14.958701133728027,
      "activations/layer10_attention_weight_max": 35.54706954956055,
      "activations/layer10_attention_weight_min": -33.84987258911133,
      "activations/layer11_attention_weight_max": 28.474346160888672,
      "activations/layer11_attention_weight_min": -27.562477111816406,
      "activations/layer12_attention_weight_max": 23.86992073059082,
      "activations/layer12_attention_weight_min": -26.44767951965332,
      "activations/layer13_attention_weight_max": 46.03712844848633,
      "activations/layer13_attention_weight_min": -34.27304458618164,
      "activations/layer14_attention_weight_max": 44.10919189453125,
      "activations/layer14_attention_weight_min": -31.137210845947266,
      "activations/layer15_attention_weight_max": 41.265899658203125,
      "activations/layer15_attention_weight_min": -30.797643661499023,
      "activations/layer16_attention_weight_max": 35.10177993774414,
      "activations/layer16_attention_weight_min": -25.57466697692871,
      "activations/layer17_attention_weight_max": 60.99479293823242,
      "activations/layer17_attention_weight_min": -38.4186897277832,
      "activations/layer18_attention_weight_max": 50.6978645324707,
      "activations/layer18_attention_weight_min": -33.0452995300293,
      "activations/layer19_attention_weight_max": 21.76193618774414,
      "activations/layer19_attention_weight_min": -19.06102180480957,
      "activations/layer1_attention_weight_max": 17.399179458618164,
      "activations/layer1_attention_weight_min": -16.046262741088867,
      "activations/layer20_attention_weight_max": 19.62407875061035,
      "activations/layer20_attention_weight_min": -20.419445037841797,
      "activations/layer21_attention_weight_max": 38.19527053833008,
      "activations/layer21_attention_weight_min": -30.191795349121094,
      "activations/layer22_attention_weight_max": 33.70465850830078,
      "activations/layer22_attention_weight_min": -23.90017318725586,
      "activations/layer23_attention_weight_max": 26.2514705657959,
      "activations/layer23_attention_weight_min": -19.724708557128906,
      "activations/layer2_attention_weight_max": 31.081785202026367,
      "activations/layer2_attention_weight_min": -28.456436157226562,
      "activations/layer3_attention_weight_max": 82.3758316040039,
      "activations/layer3_attention_weight_min": -86.95379638671875,
      "activations/layer4_attention_weight_max": 96.72830963134766,
      "activations/layer4_attention_weight_min": -92.75505065917969,
      "activations/layer5_attention_weight_max": 99.93916320800781,
      "activations/layer5_attention_weight_min": -91.72823333740234,
      "activations/layer6_attention_weight_max": 55.552001953125,
      "activations/layer6_attention_weight_min": -51.439781188964844,
      "activations/layer7_attention_weight_max": 69.24112701416016,
      "activations/layer7_attention_weight_min": -66.01624298095703,
      "activations/layer8_attention_weight_max": 37.835693359375,
      "activations/layer8_attention_weight_min": -38.91127395629883,
      "activations/layer9_attention_weight_max": 48.90742111206055,
      "activations/layer9_attention_weight_min": -45.48977279663086,
      "epoch": 4.65,
      "learning_rate": 0.0001212246212121212,
      "loss": 2.8529,
      "step": 80050
    },
    {
      "activations/layer0_attention_weight_max": 15.313323020935059,
      "activations/layer0_attention_weight_min": -14.408679962158203,
      "activations/layer10_attention_weight_max": 38.45135498046875,
      "activations/layer10_attention_weight_min": -34.78030014038086,
      "activations/layer11_attention_weight_max": 31.419815063476562,
      "activations/layer11_attention_weight_min": -27.017072677612305,
      "activations/layer12_attention_weight_max": 22.02880096435547,
      "activations/layer12_attention_weight_min": -27.012022018432617,
      "activations/layer13_attention_weight_max": 48.24394607543945,
      "activations/layer13_attention_weight_min": -32.581565856933594,
      "activations/layer14_attention_weight_max": 46.263275146484375,
      "activations/layer14_attention_weight_min": -30.803537368774414,
      "activations/layer15_attention_weight_max": 39.09758758544922,
      "activations/layer15_attention_weight_min": -32.121402740478516,
      "activations/layer16_attention_weight_max": 36.345516204833984,
      "activations/layer16_attention_weight_min": -25.74533462524414,
      "activations/layer17_attention_weight_max": 54.86625671386719,
      "activations/layer17_attention_weight_min": -38.4798469543457,
      "activations/layer18_attention_weight_max": 53.509403228759766,
      "activations/layer18_attention_weight_min": -33.35817337036133,
      "activations/layer19_attention_weight_max": 22.0161075592041,
      "activations/layer19_attention_weight_min": -18.21989631652832,
      "activations/layer1_attention_weight_max": 18.17979621887207,
      "activations/layer1_attention_weight_min": -13.892382621765137,
      "activations/layer20_attention_weight_max": 20.769439697265625,
      "activations/layer20_attention_weight_min": -20.912748336791992,
      "activations/layer21_attention_weight_max": 39.792633056640625,
      "activations/layer21_attention_weight_min": -28.62230682373047,
      "activations/layer22_attention_weight_max": 32.52482604980469,
      "activations/layer22_attention_weight_min": -22.82499885559082,
      "activations/layer23_attention_weight_max": 25.459075927734375,
      "activations/layer23_attention_weight_min": -17.342269897460938,
      "activations/layer2_attention_weight_max": 29.668415069580078,
      "activations/layer2_attention_weight_min": -28.36605453491211,
      "activations/layer3_attention_weight_max": 76.7696533203125,
      "activations/layer3_attention_weight_min": -81.15834045410156,
      "activations/layer4_attention_weight_max": 97.22724151611328,
      "activations/layer4_attention_weight_min": -89.51870727539062,
      "activations/layer5_attention_weight_max": 103.92708587646484,
      "activations/layer5_attention_weight_min": -95.11529541015625,
      "activations/layer6_attention_weight_max": 56.91637420654297,
      "activations/layer6_attention_weight_min": -52.820411682128906,
      "activations/layer7_attention_weight_max": 70.10871124267578,
      "activations/layer7_attention_weight_min": -63.642333984375,
      "activations/layer8_attention_weight_max": 41.497737884521484,
      "activations/layer8_attention_weight_min": -40.35378646850586,
      "activations/layer9_attention_weight_max": 48.12240219116211,
      "activations/layer9_attention_weight_min": -51.65838623046875,
      "epoch": 4.65,
      "learning_rate": 0.0001212056818181818,
      "loss": 2.8691,
      "step": 80100
    },
    {
      "activations/layer0_attention_weight_max": 15.025307655334473,
      "activations/layer0_attention_weight_min": -14.776932716369629,
      "activations/layer10_attention_weight_max": 36.94034957885742,
      "activations/layer10_attention_weight_min": -34.03497314453125,
      "activations/layer11_attention_weight_max": 29.03955078125,
      "activations/layer11_attention_weight_min": -27.4559383392334,
      "activations/layer12_attention_weight_max": 21.38228988647461,
      "activations/layer12_attention_weight_min": -25.804580688476562,
      "activations/layer13_attention_weight_max": 47.38872146606445,
      "activations/layer13_attention_weight_min": -37.09470748901367,
      "activations/layer14_attention_weight_max": 53.61192321777344,
      "activations/layer14_attention_weight_min": -35.40703582763672,
      "activations/layer15_attention_weight_max": 42.95580291748047,
      "activations/layer15_attention_weight_min": -31.64918327331543,
      "activations/layer16_attention_weight_max": 37.5807991027832,
      "activations/layer16_attention_weight_min": -25.89816665649414,
      "activations/layer17_attention_weight_max": 55.341209411621094,
      "activations/layer17_attention_weight_min": -37.42549514770508,
      "activations/layer18_attention_weight_max": 47.71751022338867,
      "activations/layer18_attention_weight_min": -31.8426570892334,
      "activations/layer19_attention_weight_max": 21.268878936767578,
      "activations/layer19_attention_weight_min": -19.76457977294922,
      "activations/layer1_attention_weight_max": 18.388742446899414,
      "activations/layer1_attention_weight_min": -14.656774520874023,
      "activations/layer20_attention_weight_max": 21.66843605041504,
      "activations/layer20_attention_weight_min": -19.224803924560547,
      "activations/layer21_attention_weight_max": 40.331180572509766,
      "activations/layer21_attention_weight_min": -25.88412857055664,
      "activations/layer22_attention_weight_max": 35.807247161865234,
      "activations/layer22_attention_weight_min": -23.325532913208008,
      "activations/layer23_attention_weight_max": 25.386245727539062,
      "activations/layer23_attention_weight_min": -18.465225219726562,
      "activations/layer2_attention_weight_max": 29.631263732910156,
      "activations/layer2_attention_weight_min": -27.640254974365234,
      "activations/layer3_attention_weight_max": 79.4207763671875,
      "activations/layer3_attention_weight_min": -80.70777893066406,
      "activations/layer4_attention_weight_max": 95.77228546142578,
      "activations/layer4_attention_weight_min": -92.99691009521484,
      "activations/layer5_attention_weight_max": 98.5457763671875,
      "activations/layer5_attention_weight_min": -86.33397674560547,
      "activations/layer6_attention_weight_max": 55.18692398071289,
      "activations/layer6_attention_weight_min": -49.716732025146484,
      "activations/layer7_attention_weight_max": 67.42478942871094,
      "activations/layer7_attention_weight_min": -63.15031051635742,
      "activations/layer8_attention_weight_max": 39.63466262817383,
      "activations/layer8_attention_weight_min": -38.02016830444336,
      "activations/layer9_attention_weight_max": 48.92266082763672,
      "activations/layer9_attention_weight_min": -45.306312561035156,
      "epoch": 4.66,
      "learning_rate": 0.00012118712121212121,
      "loss": 2.872,
      "step": 80150
    },
    {
      "activations/layer0_attention_weight_max": 15.090981483459473,
      "activations/layer0_attention_weight_min": -15.286422729492188,
      "activations/layer10_attention_weight_max": 44.22390365600586,
      "activations/layer10_attention_weight_min": -34.647212982177734,
      "activations/layer11_attention_weight_max": 34.03938293457031,
      "activations/layer11_attention_weight_min": -26.706541061401367,
      "activations/layer12_attention_weight_max": 26.51119613647461,
      "activations/layer12_attention_weight_min": -25.5450496673584,
      "activations/layer13_attention_weight_max": 68.13018798828125,
      "activations/layer13_attention_weight_min": -44.00444030761719,
      "activations/layer14_attention_weight_max": 62.935386657714844,
      "activations/layer14_attention_weight_min": -32.740020751953125,
      "activations/layer15_attention_weight_max": 47.671573638916016,
      "activations/layer15_attention_weight_min": -29.60720443725586,
      "activations/layer16_attention_weight_max": 37.960716247558594,
      "activations/layer16_attention_weight_min": -26.595531463623047,
      "activations/layer17_attention_weight_max": 67.4886245727539,
      "activations/layer17_attention_weight_min": -37.29597091674805,
      "activations/layer18_attention_weight_max": 53.90869140625,
      "activations/layer18_attention_weight_min": -31.329574584960938,
      "activations/layer19_attention_weight_max": 26.20644187927246,
      "activations/layer19_attention_weight_min": -20.153139114379883,
      "activations/layer1_attention_weight_max": 16.61690330505371,
      "activations/layer1_attention_weight_min": -14.943602561950684,
      "activations/layer20_attention_weight_max": 29.903697967529297,
      "activations/layer20_attention_weight_min": -19.505664825439453,
      "activations/layer21_attention_weight_max": 40.895206451416016,
      "activations/layer21_attention_weight_min": -27.13597869873047,
      "activations/layer22_attention_weight_max": 39.599876403808594,
      "activations/layer22_attention_weight_min": -23.346782684326172,
      "activations/layer23_attention_weight_max": 31.95236587524414,
      "activations/layer23_attention_weight_min": -18.90054702758789,
      "activations/layer2_attention_weight_max": 29.82448959350586,
      "activations/layer2_attention_weight_min": -28.872047424316406,
      "activations/layer3_attention_weight_max": 80.15660858154297,
      "activations/layer3_attention_weight_min": -83.27420806884766,
      "activations/layer4_attention_weight_max": 103.49005126953125,
      "activations/layer4_attention_weight_min": -100.79022979736328,
      "activations/layer5_attention_weight_max": 108.27206420898438,
      "activations/layer5_attention_weight_min": -95.97737884521484,
      "activations/layer6_attention_weight_max": 59.2563591003418,
      "activations/layer6_attention_weight_min": -53.60249710083008,
      "activations/layer7_attention_weight_max": 67.32080078125,
      "activations/layer7_attention_weight_min": -65.52169036865234,
      "activations/layer8_attention_weight_max": 46.853271484375,
      "activations/layer8_attention_weight_min": -41.25624084472656,
      "activations/layer9_attention_weight_max": 56.70402145385742,
      "activations/layer9_attention_weight_min": -51.63096237182617,
      "epoch": 4.66,
      "learning_rate": 0.00012116818181818181,
      "loss": 2.8785,
      "step": 80200
    },
    {
      "activations/layer0_attention_weight_max": 13.929282188415527,
      "activations/layer0_attention_weight_min": -14.345221519470215,
      "activations/layer10_attention_weight_max": 39.403995513916016,
      "activations/layer10_attention_weight_min": -34.56154251098633,
      "activations/layer11_attention_weight_max": 31.52979278564453,
      "activations/layer11_attention_weight_min": -25.258644104003906,
      "activations/layer12_attention_weight_max": 22.952768325805664,
      "activations/layer12_attention_weight_min": -24.947742462158203,
      "activations/layer13_attention_weight_max": 54.44165802001953,
      "activations/layer13_attention_weight_min": -40.0064697265625,
      "activations/layer14_attention_weight_max": 46.73088073730469,
      "activations/layer14_attention_weight_min": -32.44392776489258,
      "activations/layer15_attention_weight_max": 41.992496490478516,
      "activations/layer15_attention_weight_min": -32.22983932495117,
      "activations/layer16_attention_weight_max": 33.93704605102539,
      "activations/layer16_attention_weight_min": -27.306713104248047,
      "activations/layer17_attention_weight_max": 57.76175308227539,
      "activations/layer17_attention_weight_min": -39.04364013671875,
      "activations/layer18_attention_weight_max": 47.41069793701172,
      "activations/layer18_attention_weight_min": -33.6633415222168,
      "activations/layer19_attention_weight_max": 21.746801376342773,
      "activations/layer19_attention_weight_min": -18.951026916503906,
      "activations/layer1_attention_weight_max": 16.787811279296875,
      "activations/layer1_attention_weight_min": -15.05246639251709,
      "activations/layer20_attention_weight_max": 20.77806282043457,
      "activations/layer20_attention_weight_min": -20.354825973510742,
      "activations/layer21_attention_weight_max": 41.05946731567383,
      "activations/layer21_attention_weight_min": -26.768136978149414,
      "activations/layer22_attention_weight_max": 35.654361724853516,
      "activations/layer22_attention_weight_min": -22.8017635345459,
      "activations/layer23_attention_weight_max": 26.69244384765625,
      "activations/layer23_attention_weight_min": -17.729415893554688,
      "activations/layer2_attention_weight_max": 29.767555236816406,
      "activations/layer2_attention_weight_min": -27.961166381835938,
      "activations/layer3_attention_weight_max": 78.78824615478516,
      "activations/layer3_attention_weight_min": -84.01005554199219,
      "activations/layer4_attention_weight_max": 101.59058380126953,
      "activations/layer4_attention_weight_min": -91.44966888427734,
      "activations/layer5_attention_weight_max": 106.2415771484375,
      "activations/layer5_attention_weight_min": -94.68041229248047,
      "activations/layer6_attention_weight_max": 55.21534729003906,
      "activations/layer6_attention_weight_min": -52.1833610534668,
      "activations/layer7_attention_weight_max": 65.56439971923828,
      "activations/layer7_attention_weight_min": -62.90591812133789,
      "activations/layer8_attention_weight_max": 39.81346130371094,
      "activations/layer8_attention_weight_min": -39.01683044433594,
      "activations/layer9_attention_weight_max": 45.71644592285156,
      "activations/layer9_attention_weight_min": -45.36991500854492,
      "epoch": 4.66,
      "learning_rate": 0.00012114924242424241,
      "loss": 2.8617,
      "step": 80250
    },
    {
      "activations/layer0_attention_weight_max": 13.967215538024902,
      "activations/layer0_attention_weight_min": -14.931427001953125,
      "activations/layer10_attention_weight_max": 48.631927490234375,
      "activations/layer10_attention_weight_min": -35.43951416015625,
      "activations/layer11_attention_weight_max": 44.10797119140625,
      "activations/layer11_attention_weight_min": -27.57647705078125,
      "activations/layer12_attention_weight_max": 24.565025329589844,
      "activations/layer12_attention_weight_min": -27.00629997253418,
      "activations/layer13_attention_weight_max": 52.06092834472656,
      "activations/layer13_attention_weight_min": -34.83773422241211,
      "activations/layer14_attention_weight_max": 56.51044845581055,
      "activations/layer14_attention_weight_min": -30.80265235900879,
      "activations/layer15_attention_weight_max": 44.611541748046875,
      "activations/layer15_attention_weight_min": -32.521705627441406,
      "activations/layer16_attention_weight_max": 37.98585510253906,
      "activations/layer16_attention_weight_min": -25.886747360229492,
      "activations/layer17_attention_weight_max": 57.53105926513672,
      "activations/layer17_attention_weight_min": -39.762351989746094,
      "activations/layer18_attention_weight_max": 52.582763671875,
      "activations/layer18_attention_weight_min": -32.792293548583984,
      "activations/layer19_attention_weight_max": 22.866802215576172,
      "activations/layer19_attention_weight_min": -18.23065185546875,
      "activations/layer1_attention_weight_max": 18.6966609954834,
      "activations/layer1_attention_weight_min": -15.513570785522461,
      "activations/layer20_attention_weight_max": 27.662132263183594,
      "activations/layer20_attention_weight_min": -23.687898635864258,
      "activations/layer21_attention_weight_max": 45.419830322265625,
      "activations/layer21_attention_weight_min": -27.34016990661621,
      "activations/layer22_attention_weight_max": 42.77608108520508,
      "activations/layer22_attention_weight_min": -23.568771362304688,
      "activations/layer23_attention_weight_max": 30.96947479248047,
      "activations/layer23_attention_weight_min": -16.4197998046875,
      "activations/layer2_attention_weight_max": 31.68368911743164,
      "activations/layer2_attention_weight_min": -27.394046783447266,
      "activations/layer3_attention_weight_max": 89.8603744506836,
      "activations/layer3_attention_weight_min": -84.72623443603516,
      "activations/layer4_attention_weight_max": 112.61083984375,
      "activations/layer4_attention_weight_min": -105.28752899169922,
      "activations/layer5_attention_weight_max": 128.73190307617188,
      "activations/layer5_attention_weight_min": -106.20234680175781,
      "activations/layer6_attention_weight_max": 66.57978820800781,
      "activations/layer6_attention_weight_min": -55.808921813964844,
      "activations/layer7_attention_weight_max": 88.74344635009766,
      "activations/layer7_attention_weight_min": -65.2978515625,
      "activations/layer8_attention_weight_max": 57.08574676513672,
      "activations/layer8_attention_weight_min": -42.68931198120117,
      "activations/layer9_attention_weight_max": 83.4098892211914,
      "activations/layer9_attention_weight_min": -49.43057632446289,
      "epoch": 4.67,
      "learning_rate": 0.00012113030303030303,
      "loss": 2.8655,
      "step": 80300
    },
    {
      "activations/layer0_attention_weight_max": 15.389962196350098,
      "activations/layer0_attention_weight_min": -14.593918800354004,
      "activations/layer10_attention_weight_max": 37.7592658996582,
      "activations/layer10_attention_weight_min": -36.240142822265625,
      "activations/layer11_attention_weight_max": 27.80170440673828,
      "activations/layer11_attention_weight_min": -26.090225219726562,
      "activations/layer12_attention_weight_max": 24.880970001220703,
      "activations/layer12_attention_weight_min": -23.628385543823242,
      "activations/layer13_attention_weight_max": 59.1827278137207,
      "activations/layer13_attention_weight_min": -37.56911849975586,
      "activations/layer14_attention_weight_max": 59.77327346801758,
      "activations/layer14_attention_weight_min": -37.3482780456543,
      "activations/layer15_attention_weight_max": 38.83401107788086,
      "activations/layer15_attention_weight_min": -30.809776306152344,
      "activations/layer16_attention_weight_max": 39.07851028442383,
      "activations/layer16_attention_weight_min": -27.843124389648438,
      "activations/layer17_attention_weight_max": 60.01615524291992,
      "activations/layer17_attention_weight_min": -38.97805404663086,
      "activations/layer18_attention_weight_max": 48.551753997802734,
      "activations/layer18_attention_weight_min": -32.90068817138672,
      "activations/layer19_attention_weight_max": 25.779226303100586,
      "activations/layer19_attention_weight_min": -18.566150665283203,
      "activations/layer1_attention_weight_max": 17.338485717773438,
      "activations/layer1_attention_weight_min": -13.577563285827637,
      "activations/layer20_attention_weight_max": 19.99484634399414,
      "activations/layer20_attention_weight_min": -22.018028259277344,
      "activations/layer21_attention_weight_max": 39.999427795410156,
      "activations/layer21_attention_weight_min": -24.397764205932617,
      "activations/layer22_attention_weight_max": 41.17807388305664,
      "activations/layer22_attention_weight_min": -24.402517318725586,
      "activations/layer23_attention_weight_max": 28.703144073486328,
      "activations/layer23_attention_weight_min": -16.569454193115234,
      "activations/layer2_attention_weight_max": 31.386512756347656,
      "activations/layer2_attention_weight_min": -28.858123779296875,
      "activations/layer3_attention_weight_max": 80.77700805664062,
      "activations/layer3_attention_weight_min": -84.02775573730469,
      "activations/layer4_attention_weight_max": 97.2203369140625,
      "activations/layer4_attention_weight_min": -95.82774353027344,
      "activations/layer5_attention_weight_max": 107.54562377929688,
      "activations/layer5_attention_weight_min": -96.32689666748047,
      "activations/layer6_attention_weight_max": 56.50440216064453,
      "activations/layer6_attention_weight_min": -52.78089141845703,
      "activations/layer7_attention_weight_max": 66.22540283203125,
      "activations/layer7_attention_weight_min": -64.55445861816406,
      "activations/layer8_attention_weight_max": 40.45128631591797,
      "activations/layer8_attention_weight_min": -40.659603118896484,
      "activations/layer9_attention_weight_max": 47.0777587890625,
      "activations/layer9_attention_weight_min": -45.58724594116211,
      "epoch": 4.67,
      "learning_rate": 0.00012111136363636361,
      "loss": 2.8636,
      "step": 80350
    },
    {
      "activations/layer0_attention_weight_max": 14.26038646697998,
      "activations/layer0_attention_weight_min": -14.634450912475586,
      "activations/layer10_attention_weight_max": 41.802650451660156,
      "activations/layer10_attention_weight_min": -35.49853515625,
      "activations/layer11_attention_weight_max": 32.86315155029297,
      "activations/layer11_attention_weight_min": -28.155487060546875,
      "activations/layer12_attention_weight_max": 32.50904846191406,
      "activations/layer12_attention_weight_min": -26.834074020385742,
      "activations/layer13_attention_weight_max": 57.888587951660156,
      "activations/layer13_attention_weight_min": -38.40435791015625,
      "activations/layer14_attention_weight_max": 55.00331497192383,
      "activations/layer14_attention_weight_min": -33.2952880859375,
      "activations/layer15_attention_weight_max": 47.859798431396484,
      "activations/layer15_attention_weight_min": -30.593969345092773,
      "activations/layer16_attention_weight_max": 45.82754135131836,
      "activations/layer16_attention_weight_min": -28.357866287231445,
      "activations/layer17_attention_weight_max": 65.87938690185547,
      "activations/layer17_attention_weight_min": -46.06908416748047,
      "activations/layer18_attention_weight_max": 53.06696701049805,
      "activations/layer18_attention_weight_min": -36.20994186401367,
      "activations/layer19_attention_weight_max": 22.95462989807129,
      "activations/layer19_attention_weight_min": -18.59425163269043,
      "activations/layer1_attention_weight_max": 17.207645416259766,
      "activations/layer1_attention_weight_min": -16.331642150878906,
      "activations/layer20_attention_weight_max": 22.31981658935547,
      "activations/layer20_attention_weight_min": -21.25341033935547,
      "activations/layer21_attention_weight_max": 42.91765213012695,
      "activations/layer21_attention_weight_min": -24.96811294555664,
      "activations/layer22_attention_weight_max": 41.36798095703125,
      "activations/layer22_attention_weight_min": -24.54404640197754,
      "activations/layer23_attention_weight_max": 28.37282943725586,
      "activations/layer23_attention_weight_min": -19.152257919311523,
      "activations/layer2_attention_weight_max": 30.825138092041016,
      "activations/layer2_attention_weight_min": -28.89039421081543,
      "activations/layer3_attention_weight_max": 79.22647857666016,
      "activations/layer3_attention_weight_min": -80.21875762939453,
      "activations/layer4_attention_weight_max": 102.81153869628906,
      "activations/layer4_attention_weight_min": -93.23255920410156,
      "activations/layer5_attention_weight_max": 111.88285827636719,
      "activations/layer5_attention_weight_min": -97.92594909667969,
      "activations/layer6_attention_weight_max": 58.52915954589844,
      "activations/layer6_attention_weight_min": -50.87525939941406,
      "activations/layer7_attention_weight_max": 76.53800201416016,
      "activations/layer7_attention_weight_min": -68.49011993408203,
      "activations/layer8_attention_weight_max": 48.71428680419922,
      "activations/layer8_attention_weight_min": -43.430755615234375,
      "activations/layer9_attention_weight_max": 59.97735595703125,
      "activations/layer9_attention_weight_min": -51.59855270385742,
      "epoch": 4.67,
      "learning_rate": 0.00012109242424242423,
      "loss": 2.878,
      "step": 80400
    },
    {
      "activations/layer0_attention_weight_max": 14.609100341796875,
      "activations/layer0_attention_weight_min": -15.584492683410645,
      "activations/layer10_attention_weight_max": 38.668373107910156,
      "activations/layer10_attention_weight_min": -34.35105514526367,
      "activations/layer11_attention_weight_max": 28.259565353393555,
      "activations/layer11_attention_weight_min": -28.3653564453125,
      "activations/layer12_attention_weight_max": 21.60701560974121,
      "activations/layer12_attention_weight_min": -29.73371124267578,
      "activations/layer13_attention_weight_max": 49.459835052490234,
      "activations/layer13_attention_weight_min": -41.225372314453125,
      "activations/layer14_attention_weight_max": 42.39676284790039,
      "activations/layer14_attention_weight_min": -34.526573181152344,
      "activations/layer15_attention_weight_max": 38.27448272705078,
      "activations/layer15_attention_weight_min": -29.740589141845703,
      "activations/layer16_attention_weight_max": 38.25533676147461,
      "activations/layer16_attention_weight_min": -28.52629280090332,
      "activations/layer17_attention_weight_max": 62.26521682739258,
      "activations/layer17_attention_weight_min": -40.43837356567383,
      "activations/layer18_attention_weight_max": 48.534542083740234,
      "activations/layer18_attention_weight_min": -33.090423583984375,
      "activations/layer19_attention_weight_max": 19.993423461914062,
      "activations/layer19_attention_weight_min": -19.478496551513672,
      "activations/layer1_attention_weight_max": 18.89144515991211,
      "activations/layer1_attention_weight_min": -14.03140640258789,
      "activations/layer20_attention_weight_max": 24.953777313232422,
      "activations/layer20_attention_weight_min": -21.244325637817383,
      "activations/layer21_attention_weight_max": 37.99364471435547,
      "activations/layer21_attention_weight_min": -27.934429168701172,
      "activations/layer22_attention_weight_max": 35.3205451965332,
      "activations/layer22_attention_weight_min": -21.7559757232666,
      "activations/layer23_attention_weight_max": 26.89696502685547,
      "activations/layer23_attention_weight_min": -17.28555679321289,
      "activations/layer2_attention_weight_max": 31.802955627441406,
      "activations/layer2_attention_weight_min": -27.646209716796875,
      "activations/layer3_attention_weight_max": 77.85356140136719,
      "activations/layer3_attention_weight_min": -80.89247131347656,
      "activations/layer4_attention_weight_max": 101.45828247070312,
      "activations/layer4_attention_weight_min": -94.7827377319336,
      "activations/layer5_attention_weight_max": 103.27700805664062,
      "activations/layer5_attention_weight_min": -96.989990234375,
      "activations/layer6_attention_weight_max": 58.474220275878906,
      "activations/layer6_attention_weight_min": -54.31113815307617,
      "activations/layer7_attention_weight_max": 74.41336059570312,
      "activations/layer7_attention_weight_min": -66.12439727783203,
      "activations/layer8_attention_weight_max": 42.068153381347656,
      "activations/layer8_attention_weight_min": -40.46841812133789,
      "activations/layer9_attention_weight_max": 50.27248764038086,
      "activations/layer9_attention_weight_min": -45.783164978027344,
      "epoch": 4.67,
      "learning_rate": 0.00012107348484848484,
      "loss": 2.8631,
      "step": 80450
    },
    {
      "activations/layer0_attention_weight_max": 15.759552001953125,
      "activations/layer0_attention_weight_min": -14.616464614868164,
      "activations/layer10_attention_weight_max": 41.6688117980957,
      "activations/layer10_attention_weight_min": -36.739051818847656,
      "activations/layer11_attention_weight_max": 30.8017635345459,
      "activations/layer11_attention_weight_min": -26.42617416381836,
      "activations/layer12_attention_weight_max": 33.448577880859375,
      "activations/layer12_attention_weight_min": -23.27761459350586,
      "activations/layer13_attention_weight_max": 51.36819076538086,
      "activations/layer13_attention_weight_min": -34.57445526123047,
      "activations/layer14_attention_weight_max": 56.0509147644043,
      "activations/layer14_attention_weight_min": -37.5040397644043,
      "activations/layer15_attention_weight_max": 41.36301040649414,
      "activations/layer15_attention_weight_min": -29.31768798828125,
      "activations/layer16_attention_weight_max": 34.44619369506836,
      "activations/layer16_attention_weight_min": -30.499340057373047,
      "activations/layer17_attention_weight_max": 56.40257263183594,
      "activations/layer17_attention_weight_min": -39.55933380126953,
      "activations/layer18_attention_weight_max": 44.31449890136719,
      "activations/layer18_attention_weight_min": -33.100921630859375,
      "activations/layer19_attention_weight_max": 23.625707626342773,
      "activations/layer19_attention_weight_min": -20.004249572753906,
      "activations/layer1_attention_weight_max": 19.261463165283203,
      "activations/layer1_attention_weight_min": -15.773869514465332,
      "activations/layer20_attention_weight_max": 21.617313385009766,
      "activations/layer20_attention_weight_min": -21.42778968811035,
      "activations/layer21_attention_weight_max": 35.69192886352539,
      "activations/layer21_attention_weight_min": -26.233322143554688,
      "activations/layer22_attention_weight_max": 34.117191314697266,
      "activations/layer22_attention_weight_min": -24.577539443969727,
      "activations/layer23_attention_weight_max": 26.046825408935547,
      "activations/layer23_attention_weight_min": -20.31599235534668,
      "activations/layer2_attention_weight_max": 29.35487937927246,
      "activations/layer2_attention_weight_min": -28.276342391967773,
      "activations/layer3_attention_weight_max": 76.36091613769531,
      "activations/layer3_attention_weight_min": -79.69075012207031,
      "activations/layer4_attention_weight_max": 95.76233673095703,
      "activations/layer4_attention_weight_min": -91.4771957397461,
      "activations/layer5_attention_weight_max": 101.72689819335938,
      "activations/layer5_attention_weight_min": -92.3013916015625,
      "activations/layer6_attention_weight_max": 55.883548736572266,
      "activations/layer6_attention_weight_min": -50.57770919799805,
      "activations/layer7_attention_weight_max": 66.42068481445312,
      "activations/layer7_attention_weight_min": -65.54715728759766,
      "activations/layer8_attention_weight_max": 46.31588363647461,
      "activations/layer8_attention_weight_min": -42.05258560180664,
      "activations/layer9_attention_weight_max": 57.7567253112793,
      "activations/layer9_attention_weight_min": -46.91024398803711,
      "epoch": 4.68,
      "learning_rate": 0.00012105454545454543,
      "loss": 2.8573,
      "step": 80500
    },
    {
      "activations/layer0_attention_weight_max": 14.20262622833252,
      "activations/layer0_attention_weight_min": -14.98653793334961,
      "activations/layer10_attention_weight_max": 39.122840881347656,
      "activations/layer10_attention_weight_min": -32.914981842041016,
      "activations/layer11_attention_weight_max": 28.375699996948242,
      "activations/layer11_attention_weight_min": -25.337217330932617,
      "activations/layer12_attention_weight_max": 39.19133377075195,
      "activations/layer12_attention_weight_min": -26.562427520751953,
      "activations/layer13_attention_weight_max": 52.53533172607422,
      "activations/layer13_attention_weight_min": -35.09343719482422,
      "activations/layer14_attention_weight_max": 51.02033233642578,
      "activations/layer14_attention_weight_min": -31.65656089782715,
      "activations/layer15_attention_weight_max": 45.932586669921875,
      "activations/layer15_attention_weight_min": -28.828521728515625,
      "activations/layer16_attention_weight_max": 38.67917251586914,
      "activations/layer16_attention_weight_min": -26.246118545532227,
      "activations/layer17_attention_weight_max": 60.84415054321289,
      "activations/layer17_attention_weight_min": -39.63154983520508,
      "activations/layer18_attention_weight_max": 57.7496223449707,
      "activations/layer18_attention_weight_min": -34.96302795410156,
      "activations/layer19_attention_weight_max": 28.731552124023438,
      "activations/layer19_attention_weight_min": -20.54576301574707,
      "activations/layer1_attention_weight_max": 16.919004440307617,
      "activations/layer1_attention_weight_min": -13.864947319030762,
      "activations/layer20_attention_weight_max": 27.5537166595459,
      "activations/layer20_attention_weight_min": -20.813732147216797,
      "activations/layer21_attention_weight_max": 50.97651290893555,
      "activations/layer21_attention_weight_min": -28.648910522460938,
      "activations/layer22_attention_weight_max": 37.74327087402344,
      "activations/layer22_attention_weight_min": -24.88739776611328,
      "activations/layer23_attention_weight_max": 35.43145751953125,
      "activations/layer23_attention_weight_min": -18.12506866455078,
      "activations/layer2_attention_weight_max": 28.673595428466797,
      "activations/layer2_attention_weight_min": -27.870014190673828,
      "activations/layer3_attention_weight_max": 83.70841217041016,
      "activations/layer3_attention_weight_min": -86.03536987304688,
      "activations/layer4_attention_weight_max": 106.32482147216797,
      "activations/layer4_attention_weight_min": -99.84671783447266,
      "activations/layer5_attention_weight_max": 108.08666229248047,
      "activations/layer5_attention_weight_min": -98.60015869140625,
      "activations/layer6_attention_weight_max": 59.20857238769531,
      "activations/layer6_attention_weight_min": -52.867061614990234,
      "activations/layer7_attention_weight_max": 68.16291046142578,
      "activations/layer7_attention_weight_min": -69.60038757324219,
      "activations/layer8_attention_weight_max": 42.747825622558594,
      "activations/layer8_attention_weight_min": -41.41827392578125,
      "activations/layer9_attention_weight_max": 46.65275955200195,
      "activations/layer9_attention_weight_min": -46.769981384277344,
      "epoch": 4.68,
      "learning_rate": 0.00012103560606060605,
      "loss": 2.8657,
      "step": 80550
    },
    {
      "activations/layer0_attention_weight_max": 13.778858184814453,
      "activations/layer0_attention_weight_min": -14.456775665283203,
      "activations/layer10_attention_weight_max": 35.694557189941406,
      "activations/layer10_attention_weight_min": -33.14492416381836,
      "activations/layer11_attention_weight_max": 30.400917053222656,
      "activations/layer11_attention_weight_min": -24.983741760253906,
      "activations/layer12_attention_weight_max": 22.89139175415039,
      "activations/layer12_attention_weight_min": -24.31574058532715,
      "activations/layer13_attention_weight_max": 54.31621551513672,
      "activations/layer13_attention_weight_min": -38.89124298095703,
      "activations/layer14_attention_weight_max": 62.55992889404297,
      "activations/layer14_attention_weight_min": -33.4619255065918,
      "activations/layer15_attention_weight_max": 48.663848876953125,
      "activations/layer15_attention_weight_min": -33.89884567260742,
      "activations/layer16_attention_weight_max": 41.10440444946289,
      "activations/layer16_attention_weight_min": -26.990745544433594,
      "activations/layer17_attention_weight_max": 58.86410903930664,
      "activations/layer17_attention_weight_min": -38.60820388793945,
      "activations/layer18_attention_weight_max": 49.933265686035156,
      "activations/layer18_attention_weight_min": -32.29785919189453,
      "activations/layer19_attention_weight_max": 21.676313400268555,
      "activations/layer19_attention_weight_min": -19.810043334960938,
      "activations/layer1_attention_weight_max": 17.110109329223633,
      "activations/layer1_attention_weight_min": -13.865737915039062,
      "activations/layer20_attention_weight_max": 23.506406784057617,
      "activations/layer20_attention_weight_min": -24.501380920410156,
      "activations/layer21_attention_weight_max": 39.954978942871094,
      "activations/layer21_attention_weight_min": -25.62755584716797,
      "activations/layer22_attention_weight_max": 35.283512115478516,
      "activations/layer22_attention_weight_min": -23.77057647705078,
      "activations/layer23_attention_weight_max": 28.899717330932617,
      "activations/layer23_attention_weight_min": -20.238567352294922,
      "activations/layer2_attention_weight_max": 29.489418029785156,
      "activations/layer2_attention_weight_min": -27.891738891601562,
      "activations/layer3_attention_weight_max": 80.37989807128906,
      "activations/layer3_attention_weight_min": -87.38084411621094,
      "activations/layer4_attention_weight_max": 99.68820190429688,
      "activations/layer4_attention_weight_min": -101.64067840576172,
      "activations/layer5_attention_weight_max": 101.95333862304688,
      "activations/layer5_attention_weight_min": -92.87459564208984,
      "activations/layer6_attention_weight_max": 54.6440544128418,
      "activations/layer6_attention_weight_min": -53.23497009277344,
      "activations/layer7_attention_weight_max": 66.83280181884766,
      "activations/layer7_attention_weight_min": -61.429222106933594,
      "activations/layer8_attention_weight_max": 38.351348876953125,
      "activations/layer8_attention_weight_min": -37.87233352661133,
      "activations/layer9_attention_weight_max": 50.83517074584961,
      "activations/layer9_attention_weight_min": -46.76865768432617,
      "epoch": 4.68,
      "learning_rate": 0.00012101666666666666,
      "loss": 2.862,
      "step": 80600
    },
    {
      "activations/layer0_attention_weight_max": 15.788650512695312,
      "activations/layer0_attention_weight_min": -15.082282066345215,
      "activations/layer10_attention_weight_max": 37.84946060180664,
      "activations/layer10_attention_weight_min": -32.92677307128906,
      "activations/layer11_attention_weight_max": 30.153715133666992,
      "activations/layer11_attention_weight_min": -27.318599700927734,
      "activations/layer12_attention_weight_max": 27.4599666595459,
      "activations/layer12_attention_weight_min": -25.088809967041016,
      "activations/layer13_attention_weight_max": 50.87397766113281,
      "activations/layer13_attention_weight_min": -40.64141082763672,
      "activations/layer14_attention_weight_max": 44.619693756103516,
      "activations/layer14_attention_weight_min": -32.84832763671875,
      "activations/layer15_attention_weight_max": 42.94146728515625,
      "activations/layer15_attention_weight_min": -31.504779815673828,
      "activations/layer16_attention_weight_max": 37.833213806152344,
      "activations/layer16_attention_weight_min": -28.27232551574707,
      "activations/layer17_attention_weight_max": 61.69013214111328,
      "activations/layer17_attention_weight_min": -38.28435134887695,
      "activations/layer18_attention_weight_max": 48.367191314697266,
      "activations/layer18_attention_weight_min": -33.826171875,
      "activations/layer19_attention_weight_max": 22.241361618041992,
      "activations/layer19_attention_weight_min": -20.2445125579834,
      "activations/layer1_attention_weight_max": 17.619670867919922,
      "activations/layer1_attention_weight_min": -15.50924301147461,
      "activations/layer20_attention_weight_max": 24.666967391967773,
      "activations/layer20_attention_weight_min": -23.409635543823242,
      "activations/layer21_attention_weight_max": 41.790157318115234,
      "activations/layer21_attention_weight_min": -25.07470703125,
      "activations/layer22_attention_weight_max": 35.565860748291016,
      "activations/layer22_attention_weight_min": -25.274412155151367,
      "activations/layer23_attention_weight_max": 29.82807731628418,
      "activations/layer23_attention_weight_min": -19.394014358520508,
      "activations/layer2_attention_weight_max": 30.067825317382812,
      "activations/layer2_attention_weight_min": -28.490957260131836,
      "activations/layer3_attention_weight_max": 77.11428833007812,
      "activations/layer3_attention_weight_min": -82.6581039428711,
      "activations/layer4_attention_weight_max": 97.46809387207031,
      "activations/layer4_attention_weight_min": -94.81218719482422,
      "activations/layer5_attention_weight_max": 104.026611328125,
      "activations/layer5_attention_weight_min": -90.5042953491211,
      "activations/layer6_attention_weight_max": 58.86000442504883,
      "activations/layer6_attention_weight_min": -53.5207405090332,
      "activations/layer7_attention_weight_max": 72.48371887207031,
      "activations/layer7_attention_weight_min": -64.50731658935547,
      "activations/layer8_attention_weight_max": 39.37526321411133,
      "activations/layer8_attention_weight_min": -39.23233413696289,
      "activations/layer9_attention_weight_max": 62.09492111206055,
      "activations/layer9_attention_weight_min": -51.848915100097656,
      "epoch": 4.69,
      "learning_rate": 0.00012099772727272725,
      "loss": 2.8646,
      "step": 80650
    },
    {
      "activations/layer0_attention_weight_max": 14.852106094360352,
      "activations/layer0_attention_weight_min": -14.698094367980957,
      "activations/layer10_attention_weight_max": 37.209632873535156,
      "activations/layer10_attention_weight_min": -34.305118560791016,
      "activations/layer11_attention_weight_max": 28.20368003845215,
      "activations/layer11_attention_weight_min": -26.958341598510742,
      "activations/layer12_attention_weight_max": 29.498722076416016,
      "activations/layer12_attention_weight_min": -26.4337100982666,
      "activations/layer13_attention_weight_max": 51.907203674316406,
      "activations/layer13_attention_weight_min": -41.3396110534668,
      "activations/layer14_attention_weight_max": 42.40642547607422,
      "activations/layer14_attention_weight_min": -30.761564254760742,
      "activations/layer15_attention_weight_max": 45.8057975769043,
      "activations/layer15_attention_weight_min": -29.33076286315918,
      "activations/layer16_attention_weight_max": 35.82304763793945,
      "activations/layer16_attention_weight_min": -25.833696365356445,
      "activations/layer17_attention_weight_max": 57.60408401489258,
      "activations/layer17_attention_weight_min": -37.44060134887695,
      "activations/layer18_attention_weight_max": 50.083396911621094,
      "activations/layer18_attention_weight_min": -36.35698318481445,
      "activations/layer19_attention_weight_max": 18.907140731811523,
      "activations/layer19_attention_weight_min": -18.61457633972168,
      "activations/layer1_attention_weight_max": 17.66372299194336,
      "activations/layer1_attention_weight_min": -15.3458833694458,
      "activations/layer20_attention_weight_max": 24.314979553222656,
      "activations/layer20_attention_weight_min": -21.339632034301758,
      "activations/layer21_attention_weight_max": 41.374935150146484,
      "activations/layer21_attention_weight_min": -26.851940155029297,
      "activations/layer22_attention_weight_max": 36.38712692260742,
      "activations/layer22_attention_weight_min": -25.19355583190918,
      "activations/layer23_attention_weight_max": 27.632526397705078,
      "activations/layer23_attention_weight_min": -17.21884536743164,
      "activations/layer2_attention_weight_max": 31.348560333251953,
      "activations/layer2_attention_weight_min": -29.769275665283203,
      "activations/layer3_attention_weight_max": 83.41441345214844,
      "activations/layer3_attention_weight_min": -88.73075103759766,
      "activations/layer4_attention_weight_max": 107.9513931274414,
      "activations/layer4_attention_weight_min": -101.52739715576172,
      "activations/layer5_attention_weight_max": 112.59514617919922,
      "activations/layer5_attention_weight_min": -96.91694641113281,
      "activations/layer6_attention_weight_max": 61.26103591918945,
      "activations/layer6_attention_weight_min": -52.647464752197266,
      "activations/layer7_attention_weight_max": 73.10819244384766,
      "activations/layer7_attention_weight_min": -67.35050964355469,
      "activations/layer8_attention_weight_max": 41.118751525878906,
      "activations/layer8_attention_weight_min": -40.43626022338867,
      "activations/layer9_attention_weight_max": 48.53487777709961,
      "activations/layer9_attention_weight_min": -48.668697357177734,
      "epoch": 4.69,
      "learning_rate": 0.00012097878787878787,
      "loss": 2.867,
      "step": 80700
    },
    {
      "activations/layer0_attention_weight_max": 14.510143280029297,
      "activations/layer0_attention_weight_min": -14.593205451965332,
      "activations/layer10_attention_weight_max": 38.621009826660156,
      "activations/layer10_attention_weight_min": -36.92219161987305,
      "activations/layer11_attention_weight_max": 32.08816146850586,
      "activations/layer11_attention_weight_min": -27.656238555908203,
      "activations/layer12_attention_weight_max": 23.45903968811035,
      "activations/layer12_attention_weight_min": -25.760831832885742,
      "activations/layer13_attention_weight_max": 52.694454193115234,
      "activations/layer13_attention_weight_min": -37.077796936035156,
      "activations/layer14_attention_weight_max": 48.337364196777344,
      "activations/layer14_attention_weight_min": -32.20540237426758,
      "activations/layer15_attention_weight_max": 40.02299118041992,
      "activations/layer15_attention_weight_min": -30.578306198120117,
      "activations/layer16_attention_weight_max": 37.2028923034668,
      "activations/layer16_attention_weight_min": -27.16728401184082,
      "activations/layer17_attention_weight_max": 58.10326385498047,
      "activations/layer17_attention_weight_min": -40.00489807128906,
      "activations/layer18_attention_weight_max": 51.73126220703125,
      "activations/layer18_attention_weight_min": -35.67518997192383,
      "activations/layer19_attention_weight_max": 25.544750213623047,
      "activations/layer19_attention_weight_min": -20.390453338623047,
      "activations/layer1_attention_weight_max": 17.08131980895996,
      "activations/layer1_attention_weight_min": -13.782196998596191,
      "activations/layer20_attention_weight_max": 25.256040573120117,
      "activations/layer20_attention_weight_min": -22.69707489013672,
      "activations/layer21_attention_weight_max": 42.861358642578125,
      "activations/layer21_attention_weight_min": -27.33724594116211,
      "activations/layer22_attention_weight_max": 42.85603332519531,
      "activations/layer22_attention_weight_min": -24.53622055053711,
      "activations/layer23_attention_weight_max": 30.495620727539062,
      "activations/layer23_attention_weight_min": -17.78569221496582,
      "activations/layer2_attention_weight_max": 28.664897918701172,
      "activations/layer2_attention_weight_min": -29.04104232788086,
      "activations/layer3_attention_weight_max": 83.17679595947266,
      "activations/layer3_attention_weight_min": -81.63093566894531,
      "activations/layer4_attention_weight_max": 101.4977035522461,
      "activations/layer4_attention_weight_min": -92.39399719238281,
      "activations/layer5_attention_weight_max": 103.54534149169922,
      "activations/layer5_attention_weight_min": -92.32211303710938,
      "activations/layer6_attention_weight_max": 57.49967956542969,
      "activations/layer6_attention_weight_min": -54.42455291748047,
      "activations/layer7_attention_weight_max": 68.53702545166016,
      "activations/layer7_attention_weight_min": -67.11666107177734,
      "activations/layer8_attention_weight_max": 44.1481819152832,
      "activations/layer8_attention_weight_min": -41.60071563720703,
      "activations/layer9_attention_weight_max": 53.838661193847656,
      "activations/layer9_attention_weight_min": -47.68036651611328,
      "epoch": 4.69,
      "learning_rate": 0.00012095984848484848,
      "loss": 2.8592,
      "step": 80750
    },
    {
      "activations/layer0_attention_weight_max": 15.06362533569336,
      "activations/layer0_attention_weight_min": -14.887744903564453,
      "activations/layer10_attention_weight_max": 37.90327835083008,
      "activations/layer10_attention_weight_min": -32.23698425292969,
      "activations/layer11_attention_weight_max": 28.811073303222656,
      "activations/layer11_attention_weight_min": -26.480670928955078,
      "activations/layer12_attention_weight_max": 20.077136993408203,
      "activations/layer12_attention_weight_min": -27.111055374145508,
      "activations/layer13_attention_weight_max": 45.54739761352539,
      "activations/layer13_attention_weight_min": -36.57143783569336,
      "activations/layer14_attention_weight_max": 35.54231262207031,
      "activations/layer14_attention_weight_min": -29.530271530151367,
      "activations/layer15_attention_weight_max": 38.50864028930664,
      "activations/layer15_attention_weight_min": -30.67557716369629,
      "activations/layer16_attention_weight_max": 37.81202697753906,
      "activations/layer16_attention_weight_min": -25.004249572753906,
      "activations/layer17_attention_weight_max": 51.33348846435547,
      "activations/layer17_attention_weight_min": -38.3908576965332,
      "activations/layer18_attention_weight_max": 45.79742431640625,
      "activations/layer18_attention_weight_min": -33.85094451904297,
      "activations/layer19_attention_weight_max": 20.743438720703125,
      "activations/layer19_attention_weight_min": -21.52029037475586,
      "activations/layer1_attention_weight_max": 18.634641647338867,
      "activations/layer1_attention_weight_min": -15.968526840209961,
      "activations/layer20_attention_weight_max": 20.028013229370117,
      "activations/layer20_attention_weight_min": -20.457645416259766,
      "activations/layer21_attention_weight_max": 33.952484130859375,
      "activations/layer21_attention_weight_min": -24.09196662902832,
      "activations/layer22_attention_weight_max": 34.40915298461914,
      "activations/layer22_attention_weight_min": -24.08360481262207,
      "activations/layer23_attention_weight_max": 25.493986129760742,
      "activations/layer23_attention_weight_min": -20.31353187561035,
      "activations/layer2_attention_weight_max": 31.125289916992188,
      "activations/layer2_attention_weight_min": -29.8570556640625,
      "activations/layer3_attention_weight_max": 81.96977233886719,
      "activations/layer3_attention_weight_min": -89.24974060058594,
      "activations/layer4_attention_weight_max": 97.9395523071289,
      "activations/layer4_attention_weight_min": -96.50218963623047,
      "activations/layer5_attention_weight_max": 101.72816467285156,
      "activations/layer5_attention_weight_min": -98.48369598388672,
      "activations/layer6_attention_weight_max": 56.847686767578125,
      "activations/layer6_attention_weight_min": -52.37712097167969,
      "activations/layer7_attention_weight_max": 69.1248550415039,
      "activations/layer7_attention_weight_min": -65.18555450439453,
      "activations/layer8_attention_weight_max": 40.6420783996582,
      "activations/layer8_attention_weight_min": -40.05360794067383,
      "activations/layer9_attention_weight_max": 51.34946823120117,
      "activations/layer9_attention_weight_min": -47.72359848022461,
      "epoch": 4.69,
      "learning_rate": 0.00012094128787878787,
      "loss": 2.8616,
      "step": 80800
    },
    {
      "activations/layer0_attention_weight_max": 14.735457420349121,
      "activations/layer0_attention_weight_min": -15.147518157958984,
      "activations/layer10_attention_weight_max": 38.786346435546875,
      "activations/layer10_attention_weight_min": -34.491905212402344,
      "activations/layer11_attention_weight_max": 28.701244354248047,
      "activations/layer11_attention_weight_min": -27.514305114746094,
      "activations/layer12_attention_weight_max": 21.494543075561523,
      "activations/layer12_attention_weight_min": -24.820960998535156,
      "activations/layer13_attention_weight_max": 48.81966781616211,
      "activations/layer13_attention_weight_min": -36.5866813659668,
      "activations/layer14_attention_weight_max": 47.23410415649414,
      "activations/layer14_attention_weight_min": -34.14741134643555,
      "activations/layer15_attention_weight_max": 36.85616683959961,
      "activations/layer15_attention_weight_min": -31.382293701171875,
      "activations/layer16_attention_weight_max": 35.720394134521484,
      "activations/layer16_attention_weight_min": -25.91046714782715,
      "activations/layer17_attention_weight_max": 51.81288528442383,
      "activations/layer17_attention_weight_min": -36.77592849731445,
      "activations/layer18_attention_weight_max": 45.887596130371094,
      "activations/layer18_attention_weight_min": -30.642536163330078,
      "activations/layer19_attention_weight_max": 19.791624069213867,
      "activations/layer19_attention_weight_min": -19.0499210357666,
      "activations/layer1_attention_weight_max": 18.949668884277344,
      "activations/layer1_attention_weight_min": -15.938209533691406,
      "activations/layer20_attention_weight_max": 19.684526443481445,
      "activations/layer20_attention_weight_min": -20.98122787475586,
      "activations/layer21_attention_weight_max": 38.00249099731445,
      "activations/layer21_attention_weight_min": -28.096670150756836,
      "activations/layer22_attention_weight_max": 35.972232818603516,
      "activations/layer22_attention_weight_min": -25.11878776550293,
      "activations/layer23_attention_weight_max": 26.752914428710938,
      "activations/layer23_attention_weight_min": -17.39678955078125,
      "activations/layer2_attention_weight_max": 30.03744888305664,
      "activations/layer2_attention_weight_min": -27.412200927734375,
      "activations/layer3_attention_weight_max": 78.38471984863281,
      "activations/layer3_attention_weight_min": -80.52304077148438,
      "activations/layer4_attention_weight_max": 97.3789291381836,
      "activations/layer4_attention_weight_min": -92.93585968017578,
      "activations/layer5_attention_weight_max": 103.83387756347656,
      "activations/layer5_attention_weight_min": -95.66503143310547,
      "activations/layer6_attention_weight_max": 60.07094192504883,
      "activations/layer6_attention_weight_min": -52.823020935058594,
      "activations/layer7_attention_weight_max": 70.04463958740234,
      "activations/layer7_attention_weight_min": -63.85686492919922,
      "activations/layer8_attention_weight_max": 40.3859748840332,
      "activations/layer8_attention_weight_min": -40.630977630615234,
      "activations/layer9_attention_weight_max": 57.86513137817383,
      "activations/layer9_attention_weight_min": -48.3428840637207,
      "epoch": 4.7,
      "learning_rate": 0.00012092234848484847,
      "loss": 2.8647,
      "step": 80850
    },
    {
      "activations/layer0_attention_weight_max": 13.916352272033691,
      "activations/layer0_attention_weight_min": -14.944557189941406,
      "activations/layer10_attention_weight_max": 40.98756790161133,
      "activations/layer10_attention_weight_min": -33.164161682128906,
      "activations/layer11_attention_weight_max": 33.818641662597656,
      "activations/layer11_attention_weight_min": -27.480091094970703,
      "activations/layer12_attention_weight_max": 28.679162979125977,
      "activations/layer12_attention_weight_min": -23.1950740814209,
      "activations/layer13_attention_weight_max": 60.67293167114258,
      "activations/layer13_attention_weight_min": -37.14828109741211,
      "activations/layer14_attention_weight_max": 63.48971939086914,
      "activations/layer14_attention_weight_min": -32.89186096191406,
      "activations/layer15_attention_weight_max": 44.75499725341797,
      "activations/layer15_attention_weight_min": -30.947107315063477,
      "activations/layer16_attention_weight_max": 36.91340637207031,
      "activations/layer16_attention_weight_min": -26.327428817749023,
      "activations/layer17_attention_weight_max": 62.00404739379883,
      "activations/layer17_attention_weight_min": -39.34804916381836,
      "activations/layer18_attention_weight_max": 51.29243087768555,
      "activations/layer18_attention_weight_min": -33.45740509033203,
      "activations/layer19_attention_weight_max": 26.897724151611328,
      "activations/layer19_attention_weight_min": -20.28416633605957,
      "activations/layer1_attention_weight_max": 19.109111785888672,
      "activations/layer1_attention_weight_min": -14.861129760742188,
      "activations/layer20_attention_weight_max": 24.645938873291016,
      "activations/layer20_attention_weight_min": -19.655284881591797,
      "activations/layer21_attention_weight_max": 46.42163848876953,
      "activations/layer21_attention_weight_min": -26.93297576904297,
      "activations/layer22_attention_weight_max": 38.27390670776367,
      "activations/layer22_attention_weight_min": -23.63027000427246,
      "activations/layer23_attention_weight_max": 29.97878646850586,
      "activations/layer23_attention_weight_min": -19.148662567138672,
      "activations/layer2_attention_weight_max": 30.740697860717773,
      "activations/layer2_attention_weight_min": -28.76272201538086,
      "activations/layer3_attention_weight_max": 82.81056213378906,
      "activations/layer3_attention_weight_min": -87.6845932006836,
      "activations/layer4_attention_weight_max": 101.56876373291016,
      "activations/layer4_attention_weight_min": -93.94868469238281,
      "activations/layer5_attention_weight_max": 103.89132690429688,
      "activations/layer5_attention_weight_min": -89.4775390625,
      "activations/layer6_attention_weight_max": 57.42524337768555,
      "activations/layer6_attention_weight_min": -50.804420471191406,
      "activations/layer7_attention_weight_max": 70.69129943847656,
      "activations/layer7_attention_weight_min": -70.3812255859375,
      "activations/layer8_attention_weight_max": 41.788795471191406,
      "activations/layer8_attention_weight_min": -39.80058288574219,
      "activations/layer9_attention_weight_max": 60.25432205200195,
      "activations/layer9_attention_weight_min": -48.99628829956055,
      "epoch": 4.7,
      "learning_rate": 0.00012090340909090907,
      "loss": 2.8574,
      "step": 80900
    },
    {
      "activations/layer0_attention_weight_max": 15.29353141784668,
      "activations/layer0_attention_weight_min": -15.037736892700195,
      "activations/layer10_attention_weight_max": 40.80179977416992,
      "activations/layer10_attention_weight_min": -35.08523941040039,
      "activations/layer11_attention_weight_max": 31.64455795288086,
      "activations/layer11_attention_weight_min": -27.591327667236328,
      "activations/layer12_attention_weight_max": 23.5926513671875,
      "activations/layer12_attention_weight_min": -26.38153839111328,
      "activations/layer13_attention_weight_max": 61.3256721496582,
      "activations/layer13_attention_weight_min": -44.52610778808594,
      "activations/layer14_attention_weight_max": 62.04938507080078,
      "activations/layer14_attention_weight_min": -31.48588752746582,
      "activations/layer15_attention_weight_max": 51.192604064941406,
      "activations/layer15_attention_weight_min": -31.650115966796875,
      "activations/layer16_attention_weight_max": 44.833641052246094,
      "activations/layer16_attention_weight_min": -26.126176834106445,
      "activations/layer17_attention_weight_max": 62.21675109863281,
      "activations/layer17_attention_weight_min": -40.09850311279297,
      "activations/layer18_attention_weight_max": 56.78718185424805,
      "activations/layer18_attention_weight_min": -35.94467544555664,
      "activations/layer19_attention_weight_max": 21.65057945251465,
      "activations/layer19_attention_weight_min": -20.453371047973633,
      "activations/layer1_attention_weight_max": 17.25851058959961,
      "activations/layer1_attention_weight_min": -15.054481506347656,
      "activations/layer20_attention_weight_max": 24.548006057739258,
      "activations/layer20_attention_weight_min": -19.91094398498535,
      "activations/layer21_attention_weight_max": 48.31272506713867,
      "activations/layer21_attention_weight_min": -27.98693084716797,
      "activations/layer22_attention_weight_max": 36.89761734008789,
      "activations/layer22_attention_weight_min": -24.507524490356445,
      "activations/layer23_attention_weight_max": 28.138578414916992,
      "activations/layer23_attention_weight_min": -19.735532760620117,
      "activations/layer2_attention_weight_max": 30.091487884521484,
      "activations/layer2_attention_weight_min": -28.16082000732422,
      "activations/layer3_attention_weight_max": 79.56160736083984,
      "activations/layer3_attention_weight_min": -79.9001693725586,
      "activations/layer4_attention_weight_max": 95.1583480834961,
      "activations/layer4_attention_weight_min": -88.74402618408203,
      "activations/layer5_attention_weight_max": 103.39743041992188,
      "activations/layer5_attention_weight_min": -88.49370574951172,
      "activations/layer6_attention_weight_max": 58.06172180175781,
      "activations/layer6_attention_weight_min": -52.73469161987305,
      "activations/layer7_attention_weight_max": 70.18831634521484,
      "activations/layer7_attention_weight_min": -64.22975158691406,
      "activations/layer8_attention_weight_max": 43.432960510253906,
      "activations/layer8_attention_weight_min": -43.07145690917969,
      "activations/layer9_attention_weight_max": 59.23228073120117,
      "activations/layer9_attention_weight_min": -47.675025939941406,
      "epoch": 4.7,
      "learning_rate": 0.00012088446969696969,
      "loss": 2.8594,
      "step": 80950
    },
    {
      "activations/layer0_attention_weight_max": 15.597972869873047,
      "activations/layer0_attention_weight_min": -15.224390029907227,
      "activations/layer10_attention_weight_max": 36.5633430480957,
      "activations/layer10_attention_weight_min": -33.334754943847656,
      "activations/layer11_attention_weight_max": 29.257413864135742,
      "activations/layer11_attention_weight_min": -27.939910888671875,
      "activations/layer12_attention_weight_max": 21.36890983581543,
      "activations/layer12_attention_weight_min": -28.287382125854492,
      "activations/layer13_attention_weight_max": 50.384742736816406,
      "activations/layer13_attention_weight_min": -36.72560119628906,
      "activations/layer14_attention_weight_max": 45.73617935180664,
      "activations/layer14_attention_weight_min": -31.591703414916992,
      "activations/layer15_attention_weight_max": 44.33237838745117,
      "activations/layer15_attention_weight_min": -29.200416564941406,
      "activations/layer16_attention_weight_max": 35.65030288696289,
      "activations/layer16_attention_weight_min": -25.652578353881836,
      "activations/layer17_attention_weight_max": 61.41361618041992,
      "activations/layer17_attention_weight_min": -37.43412780761719,
      "activations/layer18_attention_weight_max": 49.570987701416016,
      "activations/layer18_attention_weight_min": -30.97266960144043,
      "activations/layer19_attention_weight_max": 21.65403938293457,
      "activations/layer19_attention_weight_min": -18.028348922729492,
      "activations/layer1_attention_weight_max": 18.24428367614746,
      "activations/layer1_attention_weight_min": -16.00259017944336,
      "activations/layer20_attention_weight_max": 22.67168426513672,
      "activations/layer20_attention_weight_min": -17.989364624023438,
      "activations/layer21_attention_weight_max": 38.50717544555664,
      "activations/layer21_attention_weight_min": -25.934160232543945,
      "activations/layer22_attention_weight_max": 37.211490631103516,
      "activations/layer22_attention_weight_min": -21.355443954467773,
      "activations/layer23_attention_weight_max": 29.419330596923828,
      "activations/layer23_attention_weight_min": -16.66193962097168,
      "activations/layer2_attention_weight_max": 29.39032554626465,
      "activations/layer2_attention_weight_min": -29.11477279663086,
      "activations/layer3_attention_weight_max": 81.3421859741211,
      "activations/layer3_attention_weight_min": -80.49640655517578,
      "activations/layer4_attention_weight_max": 94.139892578125,
      "activations/layer4_attention_weight_min": -90.89115905761719,
      "activations/layer5_attention_weight_max": 99.464111328125,
      "activations/layer5_attention_weight_min": -93.23516845703125,
      "activations/layer6_attention_weight_max": 57.54472732543945,
      "activations/layer6_attention_weight_min": -54.68132019042969,
      "activations/layer7_attention_weight_max": 69.65734100341797,
      "activations/layer7_attention_weight_min": -63.144004821777344,
      "activations/layer8_attention_weight_max": 42.35376739501953,
      "activations/layer8_attention_weight_min": -39.044925689697266,
      "activations/layer9_attention_weight_max": 56.578041076660156,
      "activations/layer9_attention_weight_min": -45.807979583740234,
      "epoch": 4.71,
      "learning_rate": 0.00012086553030303029,
      "loss": 2.8677,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_loss": 2.810546875,
      "eval_runtime": 8.4692,
      "eval_samples_per_second": 507.012,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_openwebtext_loss": 2.810546875,
      "eval_openwebtext_ppl": 16.61900425204837,
      "eval_openwebtext_runtime": 8.4692,
      "eval_openwebtext_samples_per_second": 507.012,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_wikitext_loss": 3.044921875,
      "eval_wikitext_ppl": 21.008389858307645,
      "eval_wikitext_runtime": 1.9452,
      "eval_wikitext_samples_per_second": 234.419,
      "step": 81000
    },
    {
      "epoch": 4.71,
      "eval_lambada_loss": 2.875,
      "eval_lambada_ppl": 17.725424121461643,
      "eval_lambada_runtime": 9.5764,
      "eval_lambada_samples_per_second": 508.437,
      "step": 81000
    },
    {
      "activations/layer0_attention_weight_max": 14.084280014038086,
      "activations/layer0_attention_weight_min": -15.013851165771484,
      "activations/layer10_attention_weight_max": 38.10548782348633,
      "activations/layer10_attention_weight_min": -35.01320266723633,
      "activations/layer11_attention_weight_max": 30.739139556884766,
      "activations/layer11_attention_weight_min": -28.637083053588867,
      "activations/layer12_attention_weight_max": 27.65585708618164,
      "activations/layer12_attention_weight_min": -26.716196060180664,
      "activations/layer13_attention_weight_max": 52.40872573852539,
      "activations/layer13_attention_weight_min": -37.31757354736328,
      "activations/layer14_attention_weight_max": 44.909934997558594,
      "activations/layer14_attention_weight_min": -30.409109115600586,
      "activations/layer15_attention_weight_max": 42.629730224609375,
      "activations/layer15_attention_weight_min": -30.28856658935547,
      "activations/layer16_attention_weight_max": 37.31414031982422,
      "activations/layer16_attention_weight_min": -26.421358108520508,
      "activations/layer17_attention_weight_max": 60.46337127685547,
      "activations/layer17_attention_weight_min": -39.31507873535156,
      "activations/layer18_attention_weight_max": 54.48413848876953,
      "activations/layer18_attention_weight_min": -35.015838623046875,
      "activations/layer19_attention_weight_max": 20.870412826538086,
      "activations/layer19_attention_weight_min": -20.731794357299805,
      "activations/layer1_attention_weight_max": 17.45013427734375,
      "activations/layer1_attention_weight_min": -16.267057418823242,
      "activations/layer20_attention_weight_max": 22.097469329833984,
      "activations/layer20_attention_weight_min": -21.886411666870117,
      "activations/layer21_attention_weight_max": 39.0190544128418,
      "activations/layer21_attention_weight_min": -31.176237106323242,
      "activations/layer22_attention_weight_max": 32.317115783691406,
      "activations/layer22_attention_weight_min": -22.765548706054688,
      "activations/layer23_attention_weight_max": 28.880287170410156,
      "activations/layer23_attention_weight_min": -20.36503791809082,
      "activations/layer2_attention_weight_max": 28.337745666503906,
      "activations/layer2_attention_weight_min": -27.112987518310547,
      "activations/layer3_attention_weight_max": 77.27800750732422,
      "activations/layer3_attention_weight_min": -83.9855728149414,
      "activations/layer4_attention_weight_max": 98.91166687011719,
      "activations/layer4_attention_weight_min": -102.04818725585938,
      "activations/layer5_attention_weight_max": 101.98252868652344,
      "activations/layer5_attention_weight_min": -99.6607894897461,
      "activations/layer6_attention_weight_max": 57.21164321899414,
      "activations/layer6_attention_weight_min": -51.996849060058594,
      "activations/layer7_attention_weight_max": 71.38456726074219,
      "activations/layer7_attention_weight_min": -66.33519744873047,
      "activations/layer8_attention_weight_max": 44.38613510131836,
      "activations/layer8_attention_weight_min": -43.17639923095703,
      "activations/layer9_attention_weight_max": 56.833065032958984,
      "activations/layer9_attention_weight_min": -46.656551361083984,
      "epoch": 4.71,
      "learning_rate": 0.0001208465909090909,
      "loss": 2.856,
      "step": 81050
    },
    {
      "activations/layer0_attention_weight_max": 14.92136287689209,
      "activations/layer0_attention_weight_min": -14.749415397644043,
      "activations/layer10_attention_weight_max": 37.121826171875,
      "activations/layer10_attention_weight_min": -34.42631912231445,
      "activations/layer11_attention_weight_max": 29.797439575195312,
      "activations/layer11_attention_weight_min": -27.78468132019043,
      "activations/layer12_attention_weight_max": 22.79766845703125,
      "activations/layer12_attention_weight_min": -24.305252075195312,
      "activations/layer13_attention_weight_max": 46.41214370727539,
      "activations/layer13_attention_weight_min": -34.17894744873047,
      "activations/layer14_attention_weight_max": 45.050621032714844,
      "activations/layer14_attention_weight_min": -31.34292221069336,
      "activations/layer15_attention_weight_max": 43.66138458251953,
      "activations/layer15_attention_weight_min": -30.09033203125,
      "activations/layer16_attention_weight_max": 34.62540054321289,
      "activations/layer16_attention_weight_min": -26.270660400390625,
      "activations/layer17_attention_weight_max": 58.69198989868164,
      "activations/layer17_attention_weight_min": -39.52483367919922,
      "activations/layer18_attention_weight_max": 53.79969024658203,
      "activations/layer18_attention_weight_min": -33.265777587890625,
      "activations/layer19_attention_weight_max": 23.954885482788086,
      "activations/layer19_attention_weight_min": -21.353124618530273,
      "activations/layer1_attention_weight_max": 16.700313568115234,
      "activations/layer1_attention_weight_min": -13.725881576538086,
      "activations/layer20_attention_weight_max": 22.698305130004883,
      "activations/layer20_attention_weight_min": -23.943944931030273,
      "activations/layer21_attention_weight_max": 41.402862548828125,
      "activations/layer21_attention_weight_min": -27.050443649291992,
      "activations/layer22_attention_weight_max": 32.064212799072266,
      "activations/layer22_attention_weight_min": -23.85341453552246,
      "activations/layer23_attention_weight_max": 27.945388793945312,
      "activations/layer23_attention_weight_min": -16.181062698364258,
      "activations/layer2_attention_weight_max": 28.457523345947266,
      "activations/layer2_attention_weight_min": -28.623485565185547,
      "activations/layer3_attention_weight_max": 73.98530578613281,
      "activations/layer3_attention_weight_min": -81.93084716796875,
      "activations/layer4_attention_weight_max": 93.40782928466797,
      "activations/layer4_attention_weight_min": -96.21837615966797,
      "activations/layer5_attention_weight_max": 99.77084350585938,
      "activations/layer5_attention_weight_min": -93.21993255615234,
      "activations/layer6_attention_weight_max": 55.07159423828125,
      "activations/layer6_attention_weight_min": -52.269447326660156,
      "activations/layer7_attention_weight_max": 65.87732696533203,
      "activations/layer7_attention_weight_min": -63.10770797729492,
      "activations/layer8_attention_weight_max": 40.92487335205078,
      "activations/layer8_attention_weight_min": -40.39856719970703,
      "activations/layer9_attention_weight_max": 47.109130859375,
      "activations/layer9_attention_weight_min": -45.47146987915039,
      "epoch": 4.71,
      "learning_rate": 0.0001208276515151515,
      "loss": 2.8469,
      "step": 81100
    },
    {
      "activations/layer0_attention_weight_max": 14.858710289001465,
      "activations/layer0_attention_weight_min": -14.465307235717773,
      "activations/layer10_attention_weight_max": 39.544708251953125,
      "activations/layer10_attention_weight_min": -32.74555206298828,
      "activations/layer11_attention_weight_max": 30.161001205444336,
      "activations/layer11_attention_weight_min": -27.837913513183594,
      "activations/layer12_attention_weight_max": 25.756677627563477,
      "activations/layer12_attention_weight_min": -27.94273567199707,
      "activations/layer13_attention_weight_max": 60.00315475463867,
      "activations/layer13_attention_weight_min": -37.57674026489258,
      "activations/layer14_attention_weight_max": 46.324745178222656,
      "activations/layer14_attention_weight_min": -28.40039825439453,
      "activations/layer15_attention_weight_max": 41.35844802856445,
      "activations/layer15_attention_weight_min": -27.969558715820312,
      "activations/layer16_attention_weight_max": 37.23174285888672,
      "activations/layer16_attention_weight_min": -26.645042419433594,
      "activations/layer17_attention_weight_max": 62.15559387207031,
      "activations/layer17_attention_weight_min": -38.17050552368164,
      "activations/layer18_attention_weight_max": 48.203514099121094,
      "activations/layer18_attention_weight_min": -33.33842468261719,
      "activations/layer19_attention_weight_max": 23.18117904663086,
      "activations/layer19_attention_weight_min": -17.85757064819336,
      "activations/layer1_attention_weight_max": 17.67812156677246,
      "activations/layer1_attention_weight_min": -15.383220672607422,
      "activations/layer20_attention_weight_max": 21.524497985839844,
      "activations/layer20_attention_weight_min": -19.04566192626953,
      "activations/layer21_attention_weight_max": 41.57036209106445,
      "activations/layer21_attention_weight_min": -28.343608856201172,
      "activations/layer22_attention_weight_max": 34.44269561767578,
      "activations/layer22_attention_weight_min": -23.939701080322266,
      "activations/layer23_attention_weight_max": 27.692466735839844,
      "activations/layer23_attention_weight_min": -17.578083038330078,
      "activations/layer2_attention_weight_max": 29.787574768066406,
      "activations/layer2_attention_weight_min": -28.16644859313965,
      "activations/layer3_attention_weight_max": 78.0140609741211,
      "activations/layer3_attention_weight_min": -83.68350982666016,
      "activations/layer4_attention_weight_max": 99.17009735107422,
      "activations/layer4_attention_weight_min": -98.92657470703125,
      "activations/layer5_attention_weight_max": 103.55937194824219,
      "activations/layer5_attention_weight_min": -94.88192749023438,
      "activations/layer6_attention_weight_max": 56.606971740722656,
      "activations/layer6_attention_weight_min": -52.012386322021484,
      "activations/layer7_attention_weight_max": 72.12577056884766,
      "activations/layer7_attention_weight_min": -66.6130142211914,
      "activations/layer8_attention_weight_max": 43.99776077270508,
      "activations/layer8_attention_weight_min": -41.88520050048828,
      "activations/layer9_attention_weight_max": 48.488311767578125,
      "activations/layer9_attention_weight_min": -45.324745178222656,
      "epoch": 4.72,
      "learning_rate": 0.00012080871212121211,
      "loss": 2.8669,
      "step": 81150
    },
    {
      "activations/layer0_attention_weight_max": 15.212714195251465,
      "activations/layer0_attention_weight_min": -14.629590034484863,
      "activations/layer10_attention_weight_max": 38.37928771972656,
      "activations/layer10_attention_weight_min": -33.483795166015625,
      "activations/layer11_attention_weight_max": 29.603199005126953,
      "activations/layer11_attention_weight_min": -26.697202682495117,
      "activations/layer12_attention_weight_max": 19.925458908081055,
      "activations/layer12_attention_weight_min": -25.660825729370117,
      "activations/layer13_attention_weight_max": 45.63208770751953,
      "activations/layer13_attention_weight_min": -35.28347396850586,
      "activations/layer14_attention_weight_max": 41.0918083190918,
      "activations/layer14_attention_weight_min": -29.116348266601562,
      "activations/layer15_attention_weight_max": 47.56925964355469,
      "activations/layer15_attention_weight_min": -31.595355987548828,
      "activations/layer16_attention_weight_max": 38.5496940612793,
      "activations/layer16_attention_weight_min": -25.229564666748047,
      "activations/layer17_attention_weight_max": 57.42000198364258,
      "activations/layer17_attention_weight_min": -39.35835266113281,
      "activations/layer18_attention_weight_max": 51.1764030456543,
      "activations/layer18_attention_weight_min": -33.881874084472656,
      "activations/layer19_attention_weight_max": 22.815664291381836,
      "activations/layer19_attention_weight_min": -20.6250057220459,
      "activations/layer1_attention_weight_max": 18.656967163085938,
      "activations/layer1_attention_weight_min": -14.385279655456543,
      "activations/layer20_attention_weight_max": 22.809188842773438,
      "activations/layer20_attention_weight_min": -22.97608757019043,
      "activations/layer21_attention_weight_max": 36.53224182128906,
      "activations/layer21_attention_weight_min": -27.07517433166504,
      "activations/layer22_attention_weight_max": 34.500694274902344,
      "activations/layer22_attention_weight_min": -24.656375885009766,
      "activations/layer23_attention_weight_max": 28.372268676757812,
      "activations/layer23_attention_weight_min": -18.608901977539062,
      "activations/layer2_attention_weight_max": 29.688541412353516,
      "activations/layer2_attention_weight_min": -27.901472091674805,
      "activations/layer3_attention_weight_max": 81.13163757324219,
      "activations/layer3_attention_weight_min": -81.13248443603516,
      "activations/layer4_attention_weight_max": 100.1594467163086,
      "activations/layer4_attention_weight_min": -94.6556167602539,
      "activations/layer5_attention_weight_max": 105.12649536132812,
      "activations/layer5_attention_weight_min": -91.2288818359375,
      "activations/layer6_attention_weight_max": 58.87813186645508,
      "activations/layer6_attention_weight_min": -51.735591888427734,
      "activations/layer7_attention_weight_max": 65.97938537597656,
      "activations/layer7_attention_weight_min": -65.36321258544922,
      "activations/layer8_attention_weight_max": 42.755332946777344,
      "activations/layer8_attention_weight_min": -41.87492370605469,
      "activations/layer9_attention_weight_max": 49.67050552368164,
      "activations/layer9_attention_weight_min": -43.47954177856445,
      "epoch": 4.72,
      "learning_rate": 0.00012078977272727272,
      "loss": 2.8546,
      "step": 81200
    },
    {
      "activations/layer0_attention_weight_max": 14.136241912841797,
      "activations/layer0_attention_weight_min": -14.75168514251709,
      "activations/layer10_attention_weight_max": 44.562469482421875,
      "activations/layer10_attention_weight_min": -37.368770599365234,
      "activations/layer11_attention_weight_max": 34.81446075439453,
      "activations/layer11_attention_weight_min": -27.400897979736328,
      "activations/layer12_attention_weight_max": 40.92694854736328,
      "activations/layer12_attention_weight_min": -26.556001663208008,
      "activations/layer13_attention_weight_max": 54.737632751464844,
      "activations/layer13_attention_weight_min": -36.16151428222656,
      "activations/layer14_attention_weight_max": 75.27169799804688,
      "activations/layer14_attention_weight_min": -36.30800247192383,
      "activations/layer15_attention_weight_max": 54.624507904052734,
      "activations/layer15_attention_weight_min": -28.582387924194336,
      "activations/layer16_attention_weight_max": 43.41693115234375,
      "activations/layer16_attention_weight_min": -27.144039154052734,
      "activations/layer17_attention_weight_max": 67.6810531616211,
      "activations/layer17_attention_weight_min": -37.7806282043457,
      "activations/layer18_attention_weight_max": 55.66206359863281,
      "activations/layer18_attention_weight_min": -31.255056381225586,
      "activations/layer19_attention_weight_max": 26.208524703979492,
      "activations/layer19_attention_weight_min": -19.663429260253906,
      "activations/layer1_attention_weight_max": 16.360944747924805,
      "activations/layer1_attention_weight_min": -16.78035545349121,
      "activations/layer20_attention_weight_max": 27.410194396972656,
      "activations/layer20_attention_weight_min": -20.957752227783203,
      "activations/layer21_attention_weight_max": 52.92133331298828,
      "activations/layer21_attention_weight_min": -27.771324157714844,
      "activations/layer22_attention_weight_max": 44.81626510620117,
      "activations/layer22_attention_weight_min": -21.815683364868164,
      "activations/layer23_attention_weight_max": 28.16895294189453,
      "activations/layer23_attention_weight_min": -16.011367797851562,
      "activations/layer2_attention_weight_max": 30.83700180053711,
      "activations/layer2_attention_weight_min": -28.97667694091797,
      "activations/layer3_attention_weight_max": 82.56544494628906,
      "activations/layer3_attention_weight_min": -87.1881103515625,
      "activations/layer4_attention_weight_max": 110.69486236572266,
      "activations/layer4_attention_weight_min": -97.7813949584961,
      "activations/layer5_attention_weight_max": 115.68316650390625,
      "activations/layer5_attention_weight_min": -95.80827331542969,
      "activations/layer6_attention_weight_max": 60.5938720703125,
      "activations/layer6_attention_weight_min": -52.12010192871094,
      "activations/layer7_attention_weight_max": 72.83757019042969,
      "activations/layer7_attention_weight_min": -66.9710464477539,
      "activations/layer8_attention_weight_max": 45.49831008911133,
      "activations/layer8_attention_weight_min": -44.21890640258789,
      "activations/layer9_attention_weight_max": 57.25504684448242,
      "activations/layer9_attention_weight_min": -56.847259521484375,
      "epoch": 4.72,
      "learning_rate": 0.00012077083333333331,
      "loss": 2.8789,
      "step": 81250
    },
    {
      "activations/layer0_attention_weight_max": 14.411046028137207,
      "activations/layer0_attention_weight_min": -14.839134216308594,
      "activations/layer10_attention_weight_max": 38.29668045043945,
      "activations/layer10_attention_weight_min": -33.32733154296875,
      "activations/layer11_attention_weight_max": 31.703184127807617,
      "activations/layer11_attention_weight_min": -26.68933868408203,
      "activations/layer12_attention_weight_max": 25.153989791870117,
      "activations/layer12_attention_weight_min": -22.408157348632812,
      "activations/layer13_attention_weight_max": 54.503719329833984,
      "activations/layer13_attention_weight_min": -44.0464973449707,
      "activations/layer14_attention_weight_max": 44.43365478515625,
      "activations/layer14_attention_weight_min": -31.315608978271484,
      "activations/layer15_attention_weight_max": 41.52399826049805,
      "activations/layer15_attention_weight_min": -29.809476852416992,
      "activations/layer16_attention_weight_max": 36.46573257446289,
      "activations/layer16_attention_weight_min": -28.83031463623047,
      "activations/layer17_attention_weight_max": 54.50928497314453,
      "activations/layer17_attention_weight_min": -40.322349548339844,
      "activations/layer18_attention_weight_max": 51.42831039428711,
      "activations/layer18_attention_weight_min": -33.44442367553711,
      "activations/layer19_attention_weight_max": 20.227144241333008,
      "activations/layer19_attention_weight_min": -18.75635528564453,
      "activations/layer1_attention_weight_max": 17.423690795898438,
      "activations/layer1_attention_weight_min": -15.965703964233398,
      "activations/layer20_attention_weight_max": 21.842838287353516,
      "activations/layer20_attention_weight_min": -24.433208465576172,
      "activations/layer21_attention_weight_max": 44.519248962402344,
      "activations/layer21_attention_weight_min": -25.960453033447266,
      "activations/layer22_attention_weight_max": 32.94630432128906,
      "activations/layer22_attention_weight_min": -24.948286056518555,
      "activations/layer23_attention_weight_max": 26.116270065307617,
      "activations/layer23_attention_weight_min": -19.778156280517578,
      "activations/layer2_attention_weight_max": 28.459484100341797,
      "activations/layer2_attention_weight_min": -27.721435546875,
      "activations/layer3_attention_weight_max": 75.999755859375,
      "activations/layer3_attention_weight_min": -80.27589416503906,
      "activations/layer4_attention_weight_max": 97.70181274414062,
      "activations/layer4_attention_weight_min": -93.87810516357422,
      "activations/layer5_attention_weight_max": 99.64201354980469,
      "activations/layer5_attention_weight_min": -92.68184661865234,
      "activations/layer6_attention_weight_max": 55.00125503540039,
      "activations/layer6_attention_weight_min": -56.983070373535156,
      "activations/layer7_attention_weight_max": 70.13784790039062,
      "activations/layer7_attention_weight_min": -67.42294311523438,
      "activations/layer8_attention_weight_max": 41.12592315673828,
      "activations/layer8_attention_weight_min": -39.194305419921875,
      "activations/layer9_attention_weight_max": 49.36061477661133,
      "activations/layer9_attention_weight_min": -48.82882308959961,
      "epoch": 4.72,
      "learning_rate": 0.00012075189393939393,
      "loss": 2.8516,
      "step": 81300
    },
    {
      "activations/layer0_attention_weight_max": 15.155963897705078,
      "activations/layer0_attention_weight_min": -14.709656715393066,
      "activations/layer10_attention_weight_max": 40.4713249206543,
      "activations/layer10_attention_weight_min": -35.76831817626953,
      "activations/layer11_attention_weight_max": 33.65874099731445,
      "activations/layer11_attention_weight_min": -26.46759033203125,
      "activations/layer12_attention_weight_max": 22.77783966064453,
      "activations/layer12_attention_weight_min": -28.559768676757812,
      "activations/layer13_attention_weight_max": 48.06922912597656,
      "activations/layer13_attention_weight_min": -33.892635345458984,
      "activations/layer14_attention_weight_max": 57.79396057128906,
      "activations/layer14_attention_weight_min": -27.951862335205078,
      "activations/layer15_attention_weight_max": 42.93731689453125,
      "activations/layer15_attention_weight_min": -29.882761001586914,
      "activations/layer16_attention_weight_max": 39.009029388427734,
      "activations/layer16_attention_weight_min": -24.891714096069336,
      "activations/layer17_attention_weight_max": 60.53193664550781,
      "activations/layer17_attention_weight_min": -36.739837646484375,
      "activations/layer18_attention_weight_max": 48.2906379699707,
      "activations/layer18_attention_weight_min": -31.817012786865234,
      "activations/layer19_attention_weight_max": 20.657644271850586,
      "activations/layer19_attention_weight_min": -16.48878288269043,
      "activations/layer1_attention_weight_max": 18.711095809936523,
      "activations/layer1_attention_weight_min": -14.830204010009766,
      "activations/layer20_attention_weight_max": 21.159574508666992,
      "activations/layer20_attention_weight_min": -20.425079345703125,
      "activations/layer21_attention_weight_max": 40.86469268798828,
      "activations/layer21_attention_weight_min": -28.992355346679688,
      "activations/layer22_attention_weight_max": 33.79645538330078,
      "activations/layer22_attention_weight_min": -20.734588623046875,
      "activations/layer23_attention_weight_max": 26.74953269958496,
      "activations/layer23_attention_weight_min": -16.187358856201172,
      "activations/layer2_attention_weight_max": 34.46147918701172,
      "activations/layer2_attention_weight_min": -30.53143310546875,
      "activations/layer3_attention_weight_max": 93.55775451660156,
      "activations/layer3_attention_weight_min": -90.87101745605469,
      "activations/layer4_attention_weight_max": 106.35694885253906,
      "activations/layer4_attention_weight_min": -103.25060272216797,
      "activations/layer5_attention_weight_max": 106.30398559570312,
      "activations/layer5_attention_weight_min": -92.35189819335938,
      "activations/layer6_attention_weight_max": 59.367042541503906,
      "activations/layer6_attention_weight_min": -50.64439392089844,
      "activations/layer7_attention_weight_max": 73.19835662841797,
      "activations/layer7_attention_weight_min": -70.91055297851562,
      "activations/layer8_attention_weight_max": 45.234291076660156,
      "activations/layer8_attention_weight_min": -38.728946685791016,
      "activations/layer9_attention_weight_max": 62.42626953125,
      "activations/layer9_attention_weight_min": -54.75563430786133,
      "epoch": 4.73,
      "learning_rate": 0.00012073295454545454,
      "loss": 2.8644,
      "step": 81350
    },
    {
      "activations/layer0_attention_weight_max": 14.098159790039062,
      "activations/layer0_attention_weight_min": -13.955822944641113,
      "activations/layer10_attention_weight_max": 38.67538833618164,
      "activations/layer10_attention_weight_min": -34.77350616455078,
      "activations/layer11_attention_weight_max": 29.50123405456543,
      "activations/layer11_attention_weight_min": -26.323657989501953,
      "activations/layer12_attention_weight_max": 28.991785049438477,
      "activations/layer12_attention_weight_min": -27.265682220458984,
      "activations/layer13_attention_weight_max": 48.72990417480469,
      "activations/layer13_attention_weight_min": -36.01067352294922,
      "activations/layer14_attention_weight_max": 40.53398513793945,
      "activations/layer14_attention_weight_min": -29.95083236694336,
      "activations/layer15_attention_weight_max": 42.696537017822266,
      "activations/layer15_attention_weight_min": -30.770050048828125,
      "activations/layer16_attention_weight_max": 36.64814376831055,
      "activations/layer16_attention_weight_min": -27.011943817138672,
      "activations/layer17_attention_weight_max": 52.567535400390625,
      "activations/layer17_attention_weight_min": -38.1153450012207,
      "activations/layer18_attention_weight_max": 49.337608337402344,
      "activations/layer18_attention_weight_min": -31.542823791503906,
      "activations/layer19_attention_weight_max": 21.765186309814453,
      "activations/layer19_attention_weight_min": -17.87388801574707,
      "activations/layer1_attention_weight_max": 18.347742080688477,
      "activations/layer1_attention_weight_min": -14.603203773498535,
      "activations/layer20_attention_weight_max": 21.38607406616211,
      "activations/layer20_attention_weight_min": -19.80574607849121,
      "activations/layer21_attention_weight_max": 41.476104736328125,
      "activations/layer21_attention_weight_min": -24.861404418945312,
      "activations/layer22_attention_weight_max": 32.6280403137207,
      "activations/layer22_attention_weight_min": -23.47644805908203,
      "activations/layer23_attention_weight_max": 30.919912338256836,
      "activations/layer23_attention_weight_min": -17.769506454467773,
      "activations/layer2_attention_weight_max": 29.53644561767578,
      "activations/layer2_attention_weight_min": -27.107322692871094,
      "activations/layer3_attention_weight_max": 80.44696807861328,
      "activations/layer3_attention_weight_min": -79.51972961425781,
      "activations/layer4_attention_weight_max": 104.13323974609375,
      "activations/layer4_attention_weight_min": -92.89801788330078,
      "activations/layer5_attention_weight_max": 105.94618225097656,
      "activations/layer5_attention_weight_min": -97.49671936035156,
      "activations/layer6_attention_weight_max": 60.43604278564453,
      "activations/layer6_attention_weight_min": -55.900169372558594,
      "activations/layer7_attention_weight_max": 70.19683837890625,
      "activations/layer7_attention_weight_min": -65.89653778076172,
      "activations/layer8_attention_weight_max": 42.766292572021484,
      "activations/layer8_attention_weight_min": -39.588348388671875,
      "activations/layer9_attention_weight_max": 49.07819366455078,
      "activations/layer9_attention_weight_min": -46.487457275390625,
      "epoch": 4.73,
      "learning_rate": 0.00012071401515151513,
      "loss": 2.8666,
      "step": 81400
    },
    {
      "activations/layer0_attention_weight_max": 13.916916847229004,
      "activations/layer0_attention_weight_min": -14.7449312210083,
      "activations/layer10_attention_weight_max": 36.40336608886719,
      "activations/layer10_attention_weight_min": -33.25713348388672,
      "activations/layer11_attention_weight_max": 29.591915130615234,
      "activations/layer11_attention_weight_min": -25.60240936279297,
      "activations/layer12_attention_weight_max": 22.438074111938477,
      "activations/layer12_attention_weight_min": -25.721521377563477,
      "activations/layer13_attention_weight_max": 45.65528106689453,
      "activations/layer13_attention_weight_min": -36.79829025268555,
      "activations/layer14_attention_weight_max": 43.5703010559082,
      "activations/layer14_attention_weight_min": -32.90671920776367,
      "activations/layer15_attention_weight_max": 39.38764572143555,
      "activations/layer15_attention_weight_min": -32.382938385009766,
      "activations/layer16_attention_weight_max": 32.042945861816406,
      "activations/layer16_attention_weight_min": -31.462324142456055,
      "activations/layer17_attention_weight_max": 52.29203796386719,
      "activations/layer17_attention_weight_min": -39.15624237060547,
      "activations/layer18_attention_weight_max": 44.891780853271484,
      "activations/layer18_attention_weight_min": -32.32659149169922,
      "activations/layer19_attention_weight_max": 25.74466896057129,
      "activations/layer19_attention_weight_min": -19.93177032470703,
      "activations/layer1_attention_weight_max": 18.276792526245117,
      "activations/layer1_attention_weight_min": -14.953542709350586,
      "activations/layer20_attention_weight_max": 19.17567253112793,
      "activations/layer20_attention_weight_min": -23.455095291137695,
      "activations/layer21_attention_weight_max": 36.77579116821289,
      "activations/layer21_attention_weight_min": -26.25077247619629,
      "activations/layer22_attention_weight_max": 32.33808135986328,
      "activations/layer22_attention_weight_min": -25.23342514038086,
      "activations/layer23_attention_weight_max": 25.098785400390625,
      "activations/layer23_attention_weight_min": -18.41680335998535,
      "activations/layer2_attention_weight_max": 31.391891479492188,
      "activations/layer2_attention_weight_min": -28.254606246948242,
      "activations/layer3_attention_weight_max": 77.55614471435547,
      "activations/layer3_attention_weight_min": -80.77120971679688,
      "activations/layer4_attention_weight_max": 99.05022430419922,
      "activations/layer4_attention_weight_min": -91.00780487060547,
      "activations/layer5_attention_weight_max": 106.3650131225586,
      "activations/layer5_attention_weight_min": -97.74609375,
      "activations/layer6_attention_weight_max": 59.34303283691406,
      "activations/layer6_attention_weight_min": -56.313480377197266,
      "activations/layer7_attention_weight_max": 69.53199768066406,
      "activations/layer7_attention_weight_min": -67.71952056884766,
      "activations/layer8_attention_weight_max": 43.061283111572266,
      "activations/layer8_attention_weight_min": -43.806400299072266,
      "activations/layer9_attention_weight_max": 47.05741882324219,
      "activations/layer9_attention_weight_min": -48.93699264526367,
      "epoch": 4.73,
      "learning_rate": 0.00012069507575757575,
      "loss": 2.8911,
      "step": 81450
    },
    {
      "activations/layer0_attention_weight_max": 14.178206443786621,
      "activations/layer0_attention_weight_min": -15.190999984741211,
      "activations/layer10_attention_weight_max": 37.555782318115234,
      "activations/layer10_attention_weight_min": -32.87711715698242,
      "activations/layer11_attention_weight_max": 30.037731170654297,
      "activations/layer11_attention_weight_min": -26.564390182495117,
      "activations/layer12_attention_weight_max": 24.925756454467773,
      "activations/layer12_attention_weight_min": -23.55746078491211,
      "activations/layer13_attention_weight_max": 43.49832534790039,
      "activations/layer13_attention_weight_min": -32.65928649902344,
      "activations/layer14_attention_weight_max": 40.215274810791016,
      "activations/layer14_attention_weight_min": -29.278339385986328,
      "activations/layer15_attention_weight_max": 40.44351577758789,
      "activations/layer15_attention_weight_min": -30.189250946044922,
      "activations/layer16_attention_weight_max": 36.515052795410156,
      "activations/layer16_attention_weight_min": -25.597145080566406,
      "activations/layer17_attention_weight_max": 54.46971893310547,
      "activations/layer17_attention_weight_min": -39.29173278808594,
      "activations/layer18_attention_weight_max": 44.783782958984375,
      "activations/layer18_attention_weight_min": -32.810218811035156,
      "activations/layer19_attention_weight_max": 19.570953369140625,
      "activations/layer19_attention_weight_min": -18.581727981567383,
      "activations/layer1_attention_weight_max": 18.092063903808594,
      "activations/layer1_attention_weight_min": -16.926855087280273,
      "activations/layer20_attention_weight_max": 20.924236297607422,
      "activations/layer20_attention_weight_min": -24.361101150512695,
      "activations/layer21_attention_weight_max": 40.74504089355469,
      "activations/layer21_attention_weight_min": -27.232574462890625,
      "activations/layer22_attention_weight_max": 32.39391326904297,
      "activations/layer22_attention_weight_min": -23.34916877746582,
      "activations/layer23_attention_weight_max": 25.536956787109375,
      "activations/layer23_attention_weight_min": -16.827449798583984,
      "activations/layer2_attention_weight_max": 31.017919540405273,
      "activations/layer2_attention_weight_min": -28.874975204467773,
      "activations/layer3_attention_weight_max": 81.42267608642578,
      "activations/layer3_attention_weight_min": -84.1123275756836,
      "activations/layer4_attention_weight_max": 97.71411895751953,
      "activations/layer4_attention_weight_min": -90.08746337890625,
      "activations/layer5_attention_weight_max": 99.71530151367188,
      "activations/layer5_attention_weight_min": -89.01066589355469,
      "activations/layer6_attention_weight_max": 56.734413146972656,
      "activations/layer6_attention_weight_min": -52.68396759033203,
      "activations/layer7_attention_weight_max": 67.5343246459961,
      "activations/layer7_attention_weight_min": -62.602725982666016,
      "activations/layer8_attention_weight_max": 40.29588317871094,
      "activations/layer8_attention_weight_min": -38.26975631713867,
      "activations/layer9_attention_weight_max": 50.13947677612305,
      "activations/layer9_attention_weight_min": -43.9373893737793,
      "epoch": 4.74,
      "learning_rate": 0.00012067613636363636,
      "loss": 2.8679,
      "step": 81500
    },
    {
      "activations/layer0_attention_weight_max": 13.9441499710083,
      "activations/layer0_attention_weight_min": -15.20157241821289,
      "activations/layer10_attention_weight_max": 36.73369216918945,
      "activations/layer10_attention_weight_min": -33.258155822753906,
      "activations/layer11_attention_weight_max": 28.56621551513672,
      "activations/layer11_attention_weight_min": -27.670400619506836,
      "activations/layer12_attention_weight_max": 21.97129249572754,
      "activations/layer12_attention_weight_min": -25.00872802734375,
      "activations/layer13_attention_weight_max": 44.121158599853516,
      "activations/layer13_attention_weight_min": -35.735565185546875,
      "activations/layer14_attention_weight_max": 39.96780776977539,
      "activations/layer14_attention_weight_min": -30.360301971435547,
      "activations/layer15_attention_weight_max": 39.71324157714844,
      "activations/layer15_attention_weight_min": -30.66659927368164,
      "activations/layer16_attention_weight_max": 36.44671630859375,
      "activations/layer16_attention_weight_min": -26.338369369506836,
      "activations/layer17_attention_weight_max": 58.06938552856445,
      "activations/layer17_attention_weight_min": -39.470821380615234,
      "activations/layer18_attention_weight_max": 47.71320724487305,
      "activations/layer18_attention_weight_min": -32.101497650146484,
      "activations/layer19_attention_weight_max": 24.48280143737793,
      "activations/layer19_attention_weight_min": -20.50050926208496,
      "activations/layer1_attention_weight_max": 18.40740203857422,
      "activations/layer1_attention_weight_min": -13.48996639251709,
      "activations/layer20_attention_weight_max": 23.29401206970215,
      "activations/layer20_attention_weight_min": -21.881744384765625,
      "activations/layer21_attention_weight_max": 38.19719696044922,
      "activations/layer21_attention_weight_min": -26.25151824951172,
      "activations/layer22_attention_weight_max": 34.8896484375,
      "activations/layer22_attention_weight_min": -23.3963680267334,
      "activations/layer23_attention_weight_max": 27.93901824951172,
      "activations/layer23_attention_weight_min": -17.5892276763916,
      "activations/layer2_attention_weight_max": 31.33614158630371,
      "activations/layer2_attention_weight_min": -28.80420684814453,
      "activations/layer3_attention_weight_max": 77.62581634521484,
      "activations/layer3_attention_weight_min": -88.3768539428711,
      "activations/layer4_attention_weight_max": 97.38445281982422,
      "activations/layer4_attention_weight_min": -96.74111938476562,
      "activations/layer5_attention_weight_max": 99.86117553710938,
      "activations/layer5_attention_weight_min": -91.17111206054688,
      "activations/layer6_attention_weight_max": 57.206817626953125,
      "activations/layer6_attention_weight_min": -50.92644119262695,
      "activations/layer7_attention_weight_max": 66.42288970947266,
      "activations/layer7_attention_weight_min": -65.72478485107422,
      "activations/layer8_attention_weight_max": 39.18252944946289,
      "activations/layer8_attention_weight_min": -38.328609466552734,
      "activations/layer9_attention_weight_max": 47.19419479370117,
      "activations/layer9_attention_weight_min": -43.9598388671875,
      "epoch": 4.74,
      "learning_rate": 0.00012065719696969696,
      "loss": 2.8716,
      "step": 81550
    },
    {
      "activations/layer0_attention_weight_max": 14.147006034851074,
      "activations/layer0_attention_weight_min": -15.174811363220215,
      "activations/layer10_attention_weight_max": 35.8970947265625,
      "activations/layer10_attention_weight_min": -32.67120361328125,
      "activations/layer11_attention_weight_max": 27.79698944091797,
      "activations/layer11_attention_weight_min": -25.164215087890625,
      "activations/layer12_attention_weight_max": 24.592792510986328,
      "activations/layer12_attention_weight_min": -25.937458038330078,
      "activations/layer13_attention_weight_max": 51.582096099853516,
      "activations/layer13_attention_weight_min": -36.12849044799805,
      "activations/layer14_attention_weight_max": 36.16783142089844,
      "activations/layer14_attention_weight_min": -29.822345733642578,
      "activations/layer15_attention_weight_max": 44.41142272949219,
      "activations/layer15_attention_weight_min": -31.127256393432617,
      "activations/layer16_attention_weight_max": 35.290592193603516,
      "activations/layer16_attention_weight_min": -26.992815017700195,
      "activations/layer17_attention_weight_max": 50.65706253051758,
      "activations/layer17_attention_weight_min": -37.61539840698242,
      "activations/layer18_attention_weight_max": 47.13398742675781,
      "activations/layer18_attention_weight_min": -34.197364807128906,
      "activations/layer19_attention_weight_max": 21.448448181152344,
      "activations/layer19_attention_weight_min": -21.427785873413086,
      "activations/layer1_attention_weight_max": 17.50530242919922,
      "activations/layer1_attention_weight_min": -14.721397399902344,
      "activations/layer20_attention_weight_max": 22.88145637512207,
      "activations/layer20_attention_weight_min": -22.918127059936523,
      "activations/layer21_attention_weight_max": 37.29178237915039,
      "activations/layer21_attention_weight_min": -25.922807693481445,
      "activations/layer22_attention_weight_max": 29.61366844177246,
      "activations/layer22_attention_weight_min": -23.51197624206543,
      "activations/layer23_attention_weight_max": 29.95110321044922,
      "activations/layer23_attention_weight_min": -21.09088897705078,
      "activations/layer2_attention_weight_max": 29.71833610534668,
      "activations/layer2_attention_weight_min": -28.450613021850586,
      "activations/layer3_attention_weight_max": 79.4610366821289,
      "activations/layer3_attention_weight_min": -83.18586730957031,
      "activations/layer4_attention_weight_max": 98.96012878417969,
      "activations/layer4_attention_weight_min": -91.03385162353516,
      "activations/layer5_attention_weight_max": 100.10446166992188,
      "activations/layer5_attention_weight_min": -88.8659896850586,
      "activations/layer6_attention_weight_max": 55.60957336425781,
      "activations/layer6_attention_weight_min": -50.04016876220703,
      "activations/layer7_attention_weight_max": 66.84024047851562,
      "activations/layer7_attention_weight_min": -62.432586669921875,
      "activations/layer8_attention_weight_max": 40.05948257446289,
      "activations/layer8_attention_weight_min": -41.30159378051758,
      "activations/layer9_attention_weight_max": 46.52629852294922,
      "activations/layer9_attention_weight_min": -42.87674331665039,
      "epoch": 4.74,
      "learning_rate": 0.00012063825757575756,
      "loss": 2.8599,
      "step": 81600
    },
    {
      "activations/layer0_attention_weight_max": 14.162212371826172,
      "activations/layer0_attention_weight_min": -15.05974292755127,
      "activations/layer10_attention_weight_max": 37.646366119384766,
      "activations/layer10_attention_weight_min": -33.144805908203125,
      "activations/layer11_attention_weight_max": 30.443199157714844,
      "activations/layer11_attention_weight_min": -26.774778366088867,
      "activations/layer12_attention_weight_max": 21.327957153320312,
      "activations/layer12_attention_weight_min": -26.14984893798828,
      "activations/layer13_attention_weight_max": 47.0400276184082,
      "activations/layer13_attention_weight_min": -37.085243225097656,
      "activations/layer14_attention_weight_max": 38.45183563232422,
      "activations/layer14_attention_weight_min": -29.85020637512207,
      "activations/layer15_attention_weight_max": 42.73106384277344,
      "activations/layer15_attention_weight_min": -29.747650146484375,
      "activations/layer16_attention_weight_max": 34.269840240478516,
      "activations/layer16_attention_weight_min": -27.063241958618164,
      "activations/layer17_attention_weight_max": 56.46188735961914,
      "activations/layer17_attention_weight_min": -38.37815475463867,
      "activations/layer18_attention_weight_max": 47.76554489135742,
      "activations/layer18_attention_weight_min": -32.22581100463867,
      "activations/layer19_attention_weight_max": 22.457345962524414,
      "activations/layer19_attention_weight_min": -21.453847885131836,
      "activations/layer1_attention_weight_max": 18.447860717773438,
      "activations/layer1_attention_weight_min": -16.000925064086914,
      "activations/layer20_attention_weight_max": 21.588462829589844,
      "activations/layer20_attention_weight_min": -22.60053062438965,
      "activations/layer21_attention_weight_max": 39.070037841796875,
      "activations/layer21_attention_weight_min": -27.487211227416992,
      "activations/layer22_attention_weight_max": 34.318668365478516,
      "activations/layer22_attention_weight_min": -25.516979217529297,
      "activations/layer23_attention_weight_max": 26.031719207763672,
      "activations/layer23_attention_weight_min": -18.913869857788086,
      "activations/layer2_attention_weight_max": 31.438434600830078,
      "activations/layer2_attention_weight_min": -29.05402946472168,
      "activations/layer3_attention_weight_max": 82.58216094970703,
      "activations/layer3_attention_weight_min": -83.4276123046875,
      "activations/layer4_attention_weight_max": 101.84734344482422,
      "activations/layer4_attention_weight_min": -90.39800262451172,
      "activations/layer5_attention_weight_max": 103.88719940185547,
      "activations/layer5_attention_weight_min": -90.7701644897461,
      "activations/layer6_attention_weight_max": 61.909759521484375,
      "activations/layer6_attention_weight_min": -55.19346618652344,
      "activations/layer7_attention_weight_max": 71.78292083740234,
      "activations/layer7_attention_weight_min": -64.3244857788086,
      "activations/layer8_attention_weight_max": 41.4129524230957,
      "activations/layer8_attention_weight_min": -41.978179931640625,
      "activations/layer9_attention_weight_max": 45.3314208984375,
      "activations/layer9_attention_weight_min": -43.3861198425293,
      "epoch": 4.74,
      "learning_rate": 0.00012061931818181816,
      "loss": 2.86,
      "step": 81650
    },
    {
      "activations/layer0_attention_weight_max": 15.229903221130371,
      "activations/layer0_attention_weight_min": -15.541319847106934,
      "activations/layer10_attention_weight_max": 37.0797233581543,
      "activations/layer10_attention_weight_min": -33.86635971069336,
      "activations/layer11_attention_weight_max": 29.67403221130371,
      "activations/layer11_attention_weight_min": -28.5986385345459,
      "activations/layer12_attention_weight_max": 20.853647232055664,
      "activations/layer12_attention_weight_min": -27.482162475585938,
      "activations/layer13_attention_weight_max": 56.098411560058594,
      "activations/layer13_attention_weight_min": -40.8121223449707,
      "activations/layer14_attention_weight_max": 44.58574676513672,
      "activations/layer14_attention_weight_min": -33.25208282470703,
      "activations/layer15_attention_weight_max": 39.36000061035156,
      "activations/layer15_attention_weight_min": -29.911834716796875,
      "activations/layer16_attention_weight_max": 33.691375732421875,
      "activations/layer16_attention_weight_min": -28.260570526123047,
      "activations/layer17_attention_weight_max": 54.72633743286133,
      "activations/layer17_attention_weight_min": -39.19765090942383,
      "activations/layer18_attention_weight_max": 45.01276779174805,
      "activations/layer18_attention_weight_min": -34.11910629272461,
      "activations/layer19_attention_weight_max": 20.050783157348633,
      "activations/layer19_attention_weight_min": -19.793317794799805,
      "activations/layer1_attention_weight_max": 18.198299407958984,
      "activations/layer1_attention_weight_min": -15.926788330078125,
      "activations/layer20_attention_weight_max": 21.69613265991211,
      "activations/layer20_attention_weight_min": -18.562469482421875,
      "activations/layer21_attention_weight_max": 36.2977180480957,
      "activations/layer21_attention_weight_min": -26.246261596679688,
      "activations/layer22_attention_weight_max": 34.362979888916016,
      "activations/layer22_attention_weight_min": -24.565351486206055,
      "activations/layer23_attention_weight_max": 27.4312744140625,
      "activations/layer23_attention_weight_min": -19.734756469726562,
      "activations/layer2_attention_weight_max": 31.01203155517578,
      "activations/layer2_attention_weight_min": -27.45754623413086,
      "activations/layer3_attention_weight_max": 78.9421157836914,
      "activations/layer3_attention_weight_min": -81.67780303955078,
      "activations/layer4_attention_weight_max": 99.29772186279297,
      "activations/layer4_attention_weight_min": -93.64916229248047,
      "activations/layer5_attention_weight_max": 104.59603881835938,
      "activations/layer5_attention_weight_min": -93.86907958984375,
      "activations/layer6_attention_weight_max": 59.68383026123047,
      "activations/layer6_attention_weight_min": -54.27228927612305,
      "activations/layer7_attention_weight_max": 69.3641128540039,
      "activations/layer7_attention_weight_min": -68.50770568847656,
      "activations/layer8_attention_weight_max": 43.05458450317383,
      "activations/layer8_attention_weight_min": -41.64303970336914,
      "activations/layer9_attention_weight_max": 49.67831039428711,
      "activations/layer9_attention_weight_min": -47.274356842041016,
      "epoch": 4.75,
      "learning_rate": 0.00012060037878787878,
      "loss": 2.8712,
      "step": 81700
    },
    {
      "activations/layer0_attention_weight_max": 16.680683135986328,
      "activations/layer0_attention_weight_min": -15.384571075439453,
      "activations/layer10_attention_weight_max": 37.18603515625,
      "activations/layer10_attention_weight_min": -33.91146469116211,
      "activations/layer11_attention_weight_max": 31.007606506347656,
      "activations/layer11_attention_weight_min": -27.284774780273438,
      "activations/layer12_attention_weight_max": 19.32133674621582,
      "activations/layer12_attention_weight_min": -24.86117935180664,
      "activations/layer13_attention_weight_max": 42.546905517578125,
      "activations/layer13_attention_weight_min": -34.693485260009766,
      "activations/layer14_attention_weight_max": 37.95305633544922,
      "activations/layer14_attention_weight_min": -28.928163528442383,
      "activations/layer15_attention_weight_max": 42.58034133911133,
      "activations/layer15_attention_weight_min": -30.787628173828125,
      "activations/layer16_attention_weight_max": 32.34541702270508,
      "activations/layer16_attention_weight_min": -28.237157821655273,
      "activations/layer17_attention_weight_max": 58.02930450439453,
      "activations/layer17_attention_weight_min": -38.54644012451172,
      "activations/layer18_attention_weight_max": 49.22989273071289,
      "activations/layer18_attention_weight_min": -32.36665344238281,
      "activations/layer19_attention_weight_max": 22.914527893066406,
      "activations/layer19_attention_weight_min": -19.637922286987305,
      "activations/layer1_attention_weight_max": 17.33013916015625,
      "activations/layer1_attention_weight_min": -15.321553230285645,
      "activations/layer20_attention_weight_max": 23.301918029785156,
      "activations/layer20_attention_weight_min": -22.484342575073242,
      "activations/layer21_attention_weight_max": 39.86536407470703,
      "activations/layer21_attention_weight_min": -27.808897018432617,
      "activations/layer22_attention_weight_max": 34.13222122192383,
      "activations/layer22_attention_weight_min": -24.030563354492188,
      "activations/layer23_attention_weight_max": 28.25393295288086,
      "activations/layer23_attention_weight_min": -21.83265495300293,
      "activations/layer2_attention_weight_max": 29.589658737182617,
      "activations/layer2_attention_weight_min": -27.728206634521484,
      "activations/layer3_attention_weight_max": 75.74225616455078,
      "activations/layer3_attention_weight_min": -80.39317321777344,
      "activations/layer4_attention_weight_max": 97.48384094238281,
      "activations/layer4_attention_weight_min": -88.78673553466797,
      "activations/layer5_attention_weight_max": 100.24089813232422,
      "activations/layer5_attention_weight_min": -92.78316497802734,
      "activations/layer6_attention_weight_max": 55.49912643432617,
      "activations/layer6_attention_weight_min": -54.48847579956055,
      "activations/layer7_attention_weight_max": 65.5746841430664,
      "activations/layer7_attention_weight_min": -69.046875,
      "activations/layer8_attention_weight_max": 41.81828689575195,
      "activations/layer8_attention_weight_min": -41.47645950317383,
      "activations/layer9_attention_weight_max": 47.69062423706055,
      "activations/layer9_attention_weight_min": -46.64595031738281,
      "epoch": 4.75,
      "learning_rate": 0.0001205814393939394,
      "loss": 2.8657,
      "step": 81750
    },
    {
      "activations/layer0_attention_weight_max": 13.379327774047852,
      "activations/layer0_attention_weight_min": -15.109509468078613,
      "activations/layer10_attention_weight_max": 40.23661804199219,
      "activations/layer10_attention_weight_min": -34.09925079345703,
      "activations/layer11_attention_weight_max": 30.238037109375,
      "activations/layer11_attention_weight_min": -28.122838973999023,
      "activations/layer12_attention_weight_max": 22.611000061035156,
      "activations/layer12_attention_weight_min": -26.172914505004883,
      "activations/layer13_attention_weight_max": 54.56254196166992,
      "activations/layer13_attention_weight_min": -38.8485221862793,
      "activations/layer14_attention_weight_max": 59.35727310180664,
      "activations/layer14_attention_weight_min": -34.29521179199219,
      "activations/layer15_attention_weight_max": 46.8253059387207,
      "activations/layer15_attention_weight_min": -29.3037109375,
      "activations/layer16_attention_weight_max": 40.9510383605957,
      "activations/layer16_attention_weight_min": -25.483013153076172,
      "activations/layer17_attention_weight_max": 58.40974807739258,
      "activations/layer17_attention_weight_min": -37.67777633666992,
      "activations/layer18_attention_weight_max": 58.53767395019531,
      "activations/layer18_attention_weight_min": -31.605670928955078,
      "activations/layer19_attention_weight_max": 21.491352081298828,
      "activations/layer19_attention_weight_min": -19.60951042175293,
      "activations/layer1_attention_weight_max": 17.766719818115234,
      "activations/layer1_attention_weight_min": -16.783681869506836,
      "activations/layer20_attention_weight_max": 24.244789123535156,
      "activations/layer20_attention_weight_min": -18.747222900390625,
      "activations/layer21_attention_weight_max": 46.043453216552734,
      "activations/layer21_attention_weight_min": -26.491270065307617,
      "activations/layer22_attention_weight_max": 37.96046829223633,
      "activations/layer22_attention_weight_min": -25.181028366088867,
      "activations/layer23_attention_weight_max": 28.255680084228516,
      "activations/layer23_attention_weight_min": -21.73410415649414,
      "activations/layer2_attention_weight_max": 32.40195083618164,
      "activations/layer2_attention_weight_min": -29.632949829101562,
      "activations/layer3_attention_weight_max": 84.71868896484375,
      "activations/layer3_attention_weight_min": -87.87955474853516,
      "activations/layer4_attention_weight_max": 103.1040267944336,
      "activations/layer4_attention_weight_min": -95.74073791503906,
      "activations/layer5_attention_weight_max": 107.6256103515625,
      "activations/layer5_attention_weight_min": -98.50080871582031,
      "activations/layer6_attention_weight_max": 59.84025573730469,
      "activations/layer6_attention_weight_min": -54.721229553222656,
      "activations/layer7_attention_weight_max": 70.22026062011719,
      "activations/layer7_attention_weight_min": -69.94535827636719,
      "activations/layer8_attention_weight_max": 44.09055709838867,
      "activations/layer8_attention_weight_min": -41.7082405090332,
      "activations/layer9_attention_weight_max": 50.009769439697266,
      "activations/layer9_attention_weight_min": -50.28448486328125,
      "epoch": 4.75,
      "learning_rate": 0.00012056249999999998,
      "loss": 2.8619,
      "step": 81800
    },
    {
      "activations/layer0_attention_weight_max": 15.34029769897461,
      "activations/layer0_attention_weight_min": -15.275675773620605,
      "activations/layer10_attention_weight_max": 41.067256927490234,
      "activations/layer10_attention_weight_min": -35.148353576660156,
      "activations/layer11_attention_weight_max": 36.116512298583984,
      "activations/layer11_attention_weight_min": -28.460552215576172,
      "activations/layer12_attention_weight_max": 27.501976013183594,
      "activations/layer12_attention_weight_min": -25.5133056640625,
      "activations/layer13_attention_weight_max": 52.37337875366211,
      "activations/layer13_attention_weight_min": -36.21500778198242,
      "activations/layer14_attention_weight_max": 76.69129943847656,
      "activations/layer14_attention_weight_min": -42.73521041870117,
      "activations/layer15_attention_weight_max": 57.71785354614258,
      "activations/layer15_attention_weight_min": -33.2072868347168,
      "activations/layer16_attention_weight_max": 38.42570495605469,
      "activations/layer16_attention_weight_min": -27.457693099975586,
      "activations/layer17_attention_weight_max": 60.60345458984375,
      "activations/layer17_attention_weight_min": -38.0882453918457,
      "activations/layer18_attention_weight_max": 48.60892868041992,
      "activations/layer18_attention_weight_min": -32.257476806640625,
      "activations/layer19_attention_weight_max": 27.223920822143555,
      "activations/layer19_attention_weight_min": -19.54127311706543,
      "activations/layer1_attention_weight_max": 19.298538208007812,
      "activations/layer1_attention_weight_min": -17.05963706970215,
      "activations/layer20_attention_weight_max": 23.994747161865234,
      "activations/layer20_attention_weight_min": -23.660703659057617,
      "activations/layer21_attention_weight_max": 42.4427490234375,
      "activations/layer21_attention_weight_min": -23.551254272460938,
      "activations/layer22_attention_weight_max": 35.066734313964844,
      "activations/layer22_attention_weight_min": -23.025266647338867,
      "activations/layer23_attention_weight_max": 28.036184310913086,
      "activations/layer23_attention_weight_min": -18.828277587890625,
      "activations/layer2_attention_weight_max": 32.647422790527344,
      "activations/layer2_attention_weight_min": -29.598033905029297,
      "activations/layer3_attention_weight_max": 85.56517791748047,
      "activations/layer3_attention_weight_min": -86.55180358886719,
      "activations/layer4_attention_weight_max": 109.7457275390625,
      "activations/layer4_attention_weight_min": -101.85481262207031,
      "activations/layer5_attention_weight_max": 115.39356994628906,
      "activations/layer5_attention_weight_min": -101.31965637207031,
      "activations/layer6_attention_weight_max": 60.4632568359375,
      "activations/layer6_attention_weight_min": -55.42607498168945,
      "activations/layer7_attention_weight_max": 74.3927001953125,
      "activations/layer7_attention_weight_min": -69.16474914550781,
      "activations/layer8_attention_weight_max": 45.3912467956543,
      "activations/layer8_attention_weight_min": -41.7074089050293,
      "activations/layer9_attention_weight_max": 59.50642013549805,
      "activations/layer9_attention_weight_min": -54.93610763549805,
      "epoch": 4.76,
      "learning_rate": 0.0001205435606060606,
      "loss": 2.8528,
      "step": 81850
    },
    {
      "activations/layer0_attention_weight_max": 15.122535705566406,
      "activations/layer0_attention_weight_min": -15.271026611328125,
      "activations/layer10_attention_weight_max": 41.12448501586914,
      "activations/layer10_attention_weight_min": -37.93802261352539,
      "activations/layer11_attention_weight_max": 32.42510223388672,
      "activations/layer11_attention_weight_min": -28.11026382446289,
      "activations/layer12_attention_weight_max": 23.224830627441406,
      "activations/layer12_attention_weight_min": -25.870698928833008,
      "activations/layer13_attention_weight_max": 48.6957893371582,
      "activations/layer13_attention_weight_min": -34.83934783935547,
      "activations/layer14_attention_weight_max": 37.99878692626953,
      "activations/layer14_attention_weight_min": -30.362133026123047,
      "activations/layer15_attention_weight_max": 37.34907531738281,
      "activations/layer15_attention_weight_min": -29.322338104248047,
      "activations/layer16_attention_weight_max": 33.368167877197266,
      "activations/layer16_attention_weight_min": -26.717529296875,
      "activations/layer17_attention_weight_max": 54.55712127685547,
      "activations/layer17_attention_weight_min": -37.71123504638672,
      "activations/layer18_attention_weight_max": 47.81224822998047,
      "activations/layer18_attention_weight_min": -32.74552536010742,
      "activations/layer19_attention_weight_max": 20.042491912841797,
      "activations/layer19_attention_weight_min": -18.73495101928711,
      "activations/layer1_attention_weight_max": 16.885116577148438,
      "activations/layer1_attention_weight_min": -16.249235153198242,
      "activations/layer20_attention_weight_max": 22.105627059936523,
      "activations/layer20_attention_weight_min": -20.68754768371582,
      "activations/layer21_attention_weight_max": 38.914039611816406,
      "activations/layer21_attention_weight_min": -24.74774169921875,
      "activations/layer22_attention_weight_max": 31.74821662902832,
      "activations/layer22_attention_weight_min": -22.792282104492188,
      "activations/layer23_attention_weight_max": 29.117843627929688,
      "activations/layer23_attention_weight_min": -16.699073791503906,
      "activations/layer2_attention_weight_max": 31.543006896972656,
      "activations/layer2_attention_weight_min": -28.84848403930664,
      "activations/layer3_attention_weight_max": 81.03242492675781,
      "activations/layer3_attention_weight_min": -81.90629577636719,
      "activations/layer4_attention_weight_max": 103.2194595336914,
      "activations/layer4_attention_weight_min": -97.63111877441406,
      "activations/layer5_attention_weight_max": 103.507568359375,
      "activations/layer5_attention_weight_min": -92.3853759765625,
      "activations/layer6_attention_weight_max": 57.69794464111328,
      "activations/layer6_attention_weight_min": -52.890254974365234,
      "activations/layer7_attention_weight_max": 69.4209213256836,
      "activations/layer7_attention_weight_min": -67.44561004638672,
      "activations/layer8_attention_weight_max": 42.274044036865234,
      "activations/layer8_attention_weight_min": -42.85235595703125,
      "activations/layer9_attention_weight_max": 49.183006286621094,
      "activations/layer9_attention_weight_min": -47.7845458984375,
      "epoch": 4.76,
      "learning_rate": 0.00012052462121212121,
      "loss": 2.8806,
      "step": 81900
    },
    {
      "activations/layer0_attention_weight_max": 14.334327697753906,
      "activations/layer0_attention_weight_min": -15.326804161071777,
      "activations/layer10_attention_weight_max": 36.23039627075195,
      "activations/layer10_attention_weight_min": -34.77197265625,
      "activations/layer11_attention_weight_max": 29.833295822143555,
      "activations/layer11_attention_weight_min": -28.218978881835938,
      "activations/layer12_attention_weight_max": 23.184200286865234,
      "activations/layer12_attention_weight_min": -25.24201774597168,
      "activations/layer13_attention_weight_max": 47.4328727722168,
      "activations/layer13_attention_weight_min": -39.23456954956055,
      "activations/layer14_attention_weight_max": 41.61463928222656,
      "activations/layer14_attention_weight_min": -30.343997955322266,
      "activations/layer15_attention_weight_max": 41.79515075683594,
      "activations/layer15_attention_weight_min": -30.034914016723633,
      "activations/layer16_attention_weight_max": 34.28373718261719,
      "activations/layer16_attention_weight_min": -25.558734893798828,
      "activations/layer17_attention_weight_max": 59.11105728149414,
      "activations/layer17_attention_weight_min": -37.02492141723633,
      "activations/layer18_attention_weight_max": 49.88063049316406,
      "activations/layer18_attention_weight_min": -35.8492546081543,
      "activations/layer19_attention_weight_max": 22.458127975463867,
      "activations/layer19_attention_weight_min": -20.657880783081055,
      "activations/layer1_attention_weight_max": 17.521766662597656,
      "activations/layer1_attention_weight_min": -16.50684356689453,
      "activations/layer20_attention_weight_max": 23.714582443237305,
      "activations/layer20_attention_weight_min": -22.298818588256836,
      "activations/layer21_attention_weight_max": 36.77372360229492,
      "activations/layer21_attention_weight_min": -24.85495948791504,
      "activations/layer22_attention_weight_max": 37.85281753540039,
      "activations/layer22_attention_weight_min": -23.08435821533203,
      "activations/layer23_attention_weight_max": 29.035133361816406,
      "activations/layer23_attention_weight_min": -19.42422866821289,
      "activations/layer2_attention_weight_max": 29.608213424682617,
      "activations/layer2_attention_weight_min": -28.454343795776367,
      "activations/layer3_attention_weight_max": 80.14244842529297,
      "activations/layer3_attention_weight_min": -81.2061538696289,
      "activations/layer4_attention_weight_max": 101.07917022705078,
      "activations/layer4_attention_weight_min": -94.42096710205078,
      "activations/layer5_attention_weight_max": 102.70677947998047,
      "activations/layer5_attention_weight_min": -94.13450622558594,
      "activations/layer6_attention_weight_max": 60.49081802368164,
      "activations/layer6_attention_weight_min": -52.75850296020508,
      "activations/layer7_attention_weight_max": 72.41464233398438,
      "activations/layer7_attention_weight_min": -65.39268493652344,
      "activations/layer8_attention_weight_max": 44.58878707885742,
      "activations/layer8_attention_weight_min": -42.25880432128906,
      "activations/layer9_attention_weight_max": 47.643150329589844,
      "activations/layer9_attention_weight_min": -46.536746978759766,
      "epoch": 4.76,
      "learning_rate": 0.0001205056818181818,
      "loss": 2.8641,
      "step": 81950
    },
    {
      "activations/layer0_attention_weight_max": 17.337352752685547,
      "activations/layer0_attention_weight_min": -14.933287620544434,
      "activations/layer10_attention_weight_max": 37.78703689575195,
      "activations/layer10_attention_weight_min": -34.3460807800293,
      "activations/layer11_attention_weight_max": 27.588600158691406,
      "activations/layer11_attention_weight_min": -25.460752487182617,
      "activations/layer12_attention_weight_max": 19.7924861907959,
      "activations/layer12_attention_weight_min": -30.587526321411133,
      "activations/layer13_attention_weight_max": 47.47317123413086,
      "activations/layer13_attention_weight_min": -33.797027587890625,
      "activations/layer14_attention_weight_max": 43.109893798828125,
      "activations/layer14_attention_weight_min": -29.526342391967773,
      "activations/layer15_attention_weight_max": 38.387908935546875,
      "activations/layer15_attention_weight_min": -28.82053565979004,
      "activations/layer16_attention_weight_max": 31.75693702697754,
      "activations/layer16_attention_weight_min": -24.59113311767578,
      "activations/layer17_attention_weight_max": 52.78337478637695,
      "activations/layer17_attention_weight_min": -37.628021240234375,
      "activations/layer18_attention_weight_max": 45.628536224365234,
      "activations/layer18_attention_weight_min": -31.015522003173828,
      "activations/layer19_attention_weight_max": 23.001365661621094,
      "activations/layer19_attention_weight_min": -20.682382583618164,
      "activations/layer1_attention_weight_max": 18.519886016845703,
      "activations/layer1_attention_weight_min": -15.162144660949707,
      "activations/layer20_attention_weight_max": 20.345457077026367,
      "activations/layer20_attention_weight_min": -20.710668563842773,
      "activations/layer21_attention_weight_max": 34.7779541015625,
      "activations/layer21_attention_weight_min": -22.483877182006836,
      "activations/layer22_attention_weight_max": 31.18174934387207,
      "activations/layer22_attention_weight_min": -21.742128372192383,
      "activations/layer23_attention_weight_max": 27.936264038085938,
      "activations/layer23_attention_weight_min": -16.588205337524414,
      "activations/layer2_attention_weight_max": 32.63602828979492,
      "activations/layer2_attention_weight_min": -30.746360778808594,
      "activations/layer3_attention_weight_max": 80.62178802490234,
      "activations/layer3_attention_weight_min": -89.17375946044922,
      "activations/layer4_attention_weight_max": 103.69408416748047,
      "activations/layer4_attention_weight_min": -99.57633209228516,
      "activations/layer5_attention_weight_max": 104.72420501708984,
      "activations/layer5_attention_weight_min": -97.7307357788086,
      "activations/layer6_attention_weight_max": 57.86125183105469,
      "activations/layer6_attention_weight_min": -55.94874954223633,
      "activations/layer7_attention_weight_max": 68.29969024658203,
      "activations/layer7_attention_weight_min": -66.48973083496094,
      "activations/layer8_attention_weight_max": 40.75108337402344,
      "activations/layer8_attention_weight_min": -39.420082092285156,
      "activations/layer9_attention_weight_max": 52.34804153442383,
      "activations/layer9_attention_weight_min": -48.39081954956055,
      "epoch": 4.76,
      "learning_rate": 0.00012048674242424242,
      "loss": 2.8548,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_loss": 2.80859375,
      "eval_runtime": 8.4973,
      "eval_samples_per_second": 505.335,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_openwebtext_loss": 2.80859375,
      "eval_openwebtext_ppl": 16.58657693697683,
      "eval_openwebtext_runtime": 8.4973,
      "eval_openwebtext_samples_per_second": 505.335,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_wikitext_loss": 3.041015625,
      "eval_wikitext_ppl": 20.92648590822237,
      "eval_wikitext_runtime": 1.9582,
      "eval_wikitext_samples_per_second": 232.867,
      "step": 82000
    },
    {
      "epoch": 4.76,
      "eval_lambada_loss": 2.798828125,
      "eval_lambada_ppl": 16.425386987895497,
      "eval_lambada_runtime": 9.5393,
      "eval_lambada_samples_per_second": 510.413,
      "step": 82000
    },
    {
      "activations/layer0_attention_weight_max": 14.724974632263184,
      "activations/layer0_attention_weight_min": -15.14132022857666,
      "activations/layer10_attention_weight_max": 36.94902420043945,
      "activations/layer10_attention_weight_min": -33.094730377197266,
      "activations/layer11_attention_weight_max": 30.211400985717773,
      "activations/layer11_attention_weight_min": -24.190763473510742,
      "activations/layer12_attention_weight_max": 21.636213302612305,
      "activations/layer12_attention_weight_min": -24.535287857055664,
      "activations/layer13_attention_weight_max": 53.29866027832031,
      "activations/layer13_attention_weight_min": -39.96019744873047,
      "activations/layer14_attention_weight_max": 48.74100112915039,
      "activations/layer14_attention_weight_min": -32.16705322265625,
      "activations/layer15_attention_weight_max": 38.78750228881836,
      "activations/layer15_attention_weight_min": -29.25929832458496,
      "activations/layer16_attention_weight_max": 36.93312454223633,
      "activations/layer16_attention_weight_min": -26.757450103759766,
      "activations/layer17_attention_weight_max": 55.0501708984375,
      "activations/layer17_attention_weight_min": -35.49906921386719,
      "activations/layer18_attention_weight_max": 48.67837905883789,
      "activations/layer18_attention_weight_min": -31.42095947265625,
      "activations/layer19_attention_weight_max": 18.778911590576172,
      "activations/layer19_attention_weight_min": -18.147262573242188,
      "activations/layer1_attention_weight_max": 17.8258056640625,
      "activations/layer1_attention_weight_min": -16.133514404296875,
      "activations/layer20_attention_weight_max": 21.53809356689453,
      "activations/layer20_attention_weight_min": -19.440183639526367,
      "activations/layer21_attention_weight_max": 36.87525177001953,
      "activations/layer21_attention_weight_min": -25.528127670288086,
      "activations/layer22_attention_weight_max": 31.811059951782227,
      "activations/layer22_attention_weight_min": -21.517290115356445,
      "activations/layer23_attention_weight_max": 24.499351501464844,
      "activations/layer23_attention_weight_min": -17.09805679321289,
      "activations/layer2_attention_weight_max": 30.99081802368164,
      "activations/layer2_attention_weight_min": -29.4917049407959,
      "activations/layer3_attention_weight_max": 78.9463882446289,
      "activations/layer3_attention_weight_min": -84.32384490966797,
      "activations/layer4_attention_weight_max": 99.08516693115234,
      "activations/layer4_attention_weight_min": -97.00463104248047,
      "activations/layer5_attention_weight_max": 103.16714477539062,
      "activations/layer5_attention_weight_min": -90.45564270019531,
      "activations/layer6_attention_weight_max": 57.33271026611328,
      "activations/layer6_attention_weight_min": -53.12063980102539,
      "activations/layer7_attention_weight_max": 70.99113464355469,
      "activations/layer7_attention_weight_min": -65.0752182006836,
      "activations/layer8_attention_weight_max": 40.83536148071289,
      "activations/layer8_attention_weight_min": -39.567203521728516,
      "activations/layer9_attention_weight_max": 50.393943786621094,
      "activations/layer9_attention_weight_min": -46.034934997558594,
      "epoch": 4.77,
      "learning_rate": 0.00012046780303030302,
      "loss": 2.8583,
      "step": 82050
    },
    {
      "activations/layer0_attention_weight_max": 15.057196617126465,
      "activations/layer0_attention_weight_min": -14.925135612487793,
      "activations/layer10_attention_weight_max": 41.518863677978516,
      "activations/layer10_attention_weight_min": -35.30128860473633,
      "activations/layer11_attention_weight_max": 30.54604148864746,
      "activations/layer11_attention_weight_min": -26.782817840576172,
      "activations/layer12_attention_weight_max": 24.739429473876953,
      "activations/layer12_attention_weight_min": -31.423219680786133,
      "activations/layer13_attention_weight_max": 55.178611755371094,
      "activations/layer13_attention_weight_min": -36.179466247558594,
      "activations/layer14_attention_weight_max": 46.67269515991211,
      "activations/layer14_attention_weight_min": -28.77779197692871,
      "activations/layer15_attention_weight_max": 40.45553970336914,
      "activations/layer15_attention_weight_min": -29.607234954833984,
      "activations/layer16_attention_weight_max": 36.2633056640625,
      "activations/layer16_attention_weight_min": -26.60844612121582,
      "activations/layer17_attention_weight_max": 58.45052719116211,
      "activations/layer17_attention_weight_min": -38.25066375732422,
      "activations/layer18_attention_weight_max": 48.3369140625,
      "activations/layer18_attention_weight_min": -32.85833740234375,
      "activations/layer19_attention_weight_max": 22.473115921020508,
      "activations/layer19_attention_weight_min": -19.98710823059082,
      "activations/layer1_attention_weight_max": 18.034299850463867,
      "activations/layer1_attention_weight_min": -15.137120246887207,
      "activations/layer20_attention_weight_max": 22.599590301513672,
      "activations/layer20_attention_weight_min": -19.432140350341797,
      "activations/layer21_attention_weight_max": 38.309593200683594,
      "activations/layer21_attention_weight_min": -26.765493392944336,
      "activations/layer22_attention_weight_max": 34.799400329589844,
      "activations/layer22_attention_weight_min": -23.90965461730957,
      "activations/layer23_attention_weight_max": 31.12140655517578,
      "activations/layer23_attention_weight_min": -17.475923538208008,
      "activations/layer2_attention_weight_max": 31.80514144897461,
      "activations/layer2_attention_weight_min": -31.03636932373047,
      "activations/layer3_attention_weight_max": 83.15357971191406,
      "activations/layer3_attention_weight_min": -87.13699340820312,
      "activations/layer4_attention_weight_max": 104.58104705810547,
      "activations/layer4_attention_weight_min": -97.2686996459961,
      "activations/layer5_attention_weight_max": 108.89586639404297,
      "activations/layer5_attention_weight_min": -94.26020812988281,
      "activations/layer6_attention_weight_max": 59.60950469970703,
      "activations/layer6_attention_weight_min": -52.995243072509766,
      "activations/layer7_attention_weight_max": 77.3567886352539,
      "activations/layer7_attention_weight_min": -69.85079956054688,
      "activations/layer8_attention_weight_max": 46.65867233276367,
      "activations/layer8_attention_weight_min": -42.388671875,
      "activations/layer9_attention_weight_max": 61.78730392456055,
      "activations/layer9_attention_weight_min": -51.609405517578125,
      "epoch": 4.77,
      "learning_rate": 0.00012044886363636362,
      "loss": 2.8479,
      "step": 82100
    },
    {
      "activations/layer0_attention_weight_max": 14.044662475585938,
      "activations/layer0_attention_weight_min": -15.365412712097168,
      "activations/layer10_attention_weight_max": 39.47514724731445,
      "activations/layer10_attention_weight_min": -33.0673828125,
      "activations/layer11_attention_weight_max": 30.993297576904297,
      "activations/layer11_attention_weight_min": -27.298187255859375,
      "activations/layer12_attention_weight_max": 23.309165954589844,
      "activations/layer12_attention_weight_min": -23.14971923828125,
      "activations/layer13_attention_weight_max": 49.734046936035156,
      "activations/layer13_attention_weight_min": -37.45177459716797,
      "activations/layer14_attention_weight_max": 44.52973937988281,
      "activations/layer14_attention_weight_min": -31.49425506591797,
      "activations/layer15_attention_weight_max": 43.633705139160156,
      "activations/layer15_attention_weight_min": -34.097625732421875,
      "activations/layer16_attention_weight_max": 37.70819854736328,
      "activations/layer16_attention_weight_min": -27.96894645690918,
      "activations/layer17_attention_weight_max": 65.2112808227539,
      "activations/layer17_attention_weight_min": -42.03277587890625,
      "activations/layer18_attention_weight_max": 54.19013977050781,
      "activations/layer18_attention_weight_min": -38.456233978271484,
      "activations/layer19_attention_weight_max": 25.334619522094727,
      "activations/layer19_attention_weight_min": -20.32685089111328,
      "activations/layer1_attention_weight_max": 17.582448959350586,
      "activations/layer1_attention_weight_min": -15.06199836730957,
      "activations/layer20_attention_weight_max": 22.327348709106445,
      "activations/layer20_attention_weight_min": -21.587749481201172,
      "activations/layer21_attention_weight_max": 41.956390380859375,
      "activations/layer21_attention_weight_min": -27.04436492919922,
      "activations/layer22_attention_weight_max": 38.56551742553711,
      "activations/layer22_attention_weight_min": -27.730846405029297,
      "activations/layer23_attention_weight_max": 30.70117950439453,
      "activations/layer23_attention_weight_min": -19.0177001953125,
      "activations/layer2_attention_weight_max": 29.553672790527344,
      "activations/layer2_attention_weight_min": -27.373905181884766,
      "activations/layer3_attention_weight_max": 79.9364242553711,
      "activations/layer3_attention_weight_min": -82.72592163085938,
      "activations/layer4_attention_weight_max": 101.53238677978516,
      "activations/layer4_attention_weight_min": -93.9864730834961,
      "activations/layer5_attention_weight_max": 102.35894775390625,
      "activations/layer5_attention_weight_min": -95.64229583740234,
      "activations/layer6_attention_weight_max": 58.01897048950195,
      "activations/layer6_attention_weight_min": -52.80208969116211,
      "activations/layer7_attention_weight_max": 67.40316772460938,
      "activations/layer7_attention_weight_min": -67.54843139648438,
      "activations/layer8_attention_weight_max": 42.57472610473633,
      "activations/layer8_attention_weight_min": -43.48188781738281,
      "activations/layer9_attention_weight_max": 50.439613342285156,
      "activations/layer9_attention_weight_min": -45.82248306274414,
      "epoch": 4.77,
      "learning_rate": 0.00012042992424242423,
      "loss": 2.8678,
      "step": 82150
    },
    {
      "activations/layer0_attention_weight_max": 14.899473190307617,
      "activations/layer0_attention_weight_min": -14.585678100585938,
      "activations/layer10_attention_weight_max": 38.03511428833008,
      "activations/layer10_attention_weight_min": -37.96507263183594,
      "activations/layer11_attention_weight_max": 31.487560272216797,
      "activations/layer11_attention_weight_min": -29.626583099365234,
      "activations/layer12_attention_weight_max": 22.774734497070312,
      "activations/layer12_attention_weight_min": -27.793079376220703,
      "activations/layer13_attention_weight_max": 54.71262741088867,
      "activations/layer13_attention_weight_min": -39.815284729003906,
      "activations/layer14_attention_weight_max": 49.313751220703125,
      "activations/layer14_attention_weight_min": -32.00431823730469,
      "activations/layer15_attention_weight_max": 42.8162956237793,
      "activations/layer15_attention_weight_min": -32.45399475097656,
      "activations/layer16_attention_weight_max": 37.56242752075195,
      "activations/layer16_attention_weight_min": -26.98175048828125,
      "activations/layer17_attention_weight_max": 54.354713439941406,
      "activations/layer17_attention_weight_min": -43.81845474243164,
      "activations/layer18_attention_weight_max": 53.08149719238281,
      "activations/layer18_attention_weight_min": -35.2112922668457,
      "activations/layer19_attention_weight_max": 23.480924606323242,
      "activations/layer19_attention_weight_min": -19.68098258972168,
      "activations/layer1_attention_weight_max": 17.52223777770996,
      "activations/layer1_attention_weight_min": -16.238895416259766,
      "activations/layer20_attention_weight_max": 26.36528205871582,
      "activations/layer20_attention_weight_min": -18.458438873291016,
      "activations/layer21_attention_weight_max": 40.33173751831055,
      "activations/layer21_attention_weight_min": -26.894603729248047,
      "activations/layer22_attention_weight_max": 36.621971130371094,
      "activations/layer22_attention_weight_min": -25.20231819152832,
      "activations/layer23_attention_weight_max": 28.00068473815918,
      "activations/layer23_attention_weight_min": -17.116470336914062,
      "activations/layer2_attention_weight_max": 30.84206199645996,
      "activations/layer2_attention_weight_min": -27.88397789001465,
      "activations/layer3_attention_weight_max": 82.17870330810547,
      "activations/layer3_attention_weight_min": -84.55923461914062,
      "activations/layer4_attention_weight_max": 101.67411041259766,
      "activations/layer4_attention_weight_min": -94.05772399902344,
      "activations/layer5_attention_weight_max": 108.42644500732422,
      "activations/layer5_attention_weight_min": -93.78679656982422,
      "activations/layer6_attention_weight_max": 60.004615783691406,
      "activations/layer6_attention_weight_min": -55.41135787963867,
      "activations/layer7_attention_weight_max": 68.2026596069336,
      "activations/layer7_attention_weight_min": -66.2511978149414,
      "activations/layer8_attention_weight_max": 40.74098587036133,
      "activations/layer8_attention_weight_min": -40.375675201416016,
      "activations/layer9_attention_weight_max": 53.27985763549805,
      "activations/layer9_attention_weight_min": -47.90453338623047,
      "epoch": 4.78,
      "learning_rate": 0.00012041098484848484,
      "loss": 2.8574,
      "step": 82200
    },
    {
      "activations/layer0_attention_weight_max": 14.310790061950684,
      "activations/layer0_attention_weight_min": -14.864009857177734,
      "activations/layer10_attention_weight_max": 41.39270782470703,
      "activations/layer10_attention_weight_min": -36.19819641113281,
      "activations/layer11_attention_weight_max": 32.43073272705078,
      "activations/layer11_attention_weight_min": -28.49671173095703,
      "activations/layer12_attention_weight_max": 26.661819458007812,
      "activations/layer12_attention_weight_min": -27.87027359008789,
      "activations/layer13_attention_weight_max": 53.761478424072266,
      "activations/layer13_attention_weight_min": -37.95565414428711,
      "activations/layer14_attention_weight_max": 53.255592346191406,
      "activations/layer14_attention_weight_min": -32.92019271850586,
      "activations/layer15_attention_weight_max": 42.17796325683594,
      "activations/layer15_attention_weight_min": -28.83596420288086,
      "activations/layer16_attention_weight_max": 37.3962287902832,
      "activations/layer16_attention_weight_min": -27.235597610473633,
      "activations/layer17_attention_weight_max": 55.157466888427734,
      "activations/layer17_attention_weight_min": -38.37671661376953,
      "activations/layer18_attention_weight_max": 44.824241638183594,
      "activations/layer18_attention_weight_min": -31.877521514892578,
      "activations/layer19_attention_weight_max": 22.72144889831543,
      "activations/layer19_attention_weight_min": -19.580198287963867,
      "activations/layer1_attention_weight_max": 18.06671714782715,
      "activations/layer1_attention_weight_min": -17.384620666503906,
      "activations/layer20_attention_weight_max": 22.07457160949707,
      "activations/layer20_attention_weight_min": -20.634654998779297,
      "activations/layer21_attention_weight_max": 47.067535400390625,
      "activations/layer21_attention_weight_min": -27.324533462524414,
      "activations/layer22_attention_weight_max": 32.09373092651367,
      "activations/layer22_attention_weight_min": -22.713804244995117,
      "activations/layer23_attention_weight_max": 32.245849609375,
      "activations/layer23_attention_weight_min": -19.679553985595703,
      "activations/layer2_attention_weight_max": 30.81946563720703,
      "activations/layer2_attention_weight_min": -29.388296127319336,
      "activations/layer3_attention_weight_max": 81.03772735595703,
      "activations/layer3_attention_weight_min": -82.71326446533203,
      "activations/layer4_attention_weight_max": 105.0285415649414,
      "activations/layer4_attention_weight_min": -97.18196868896484,
      "activations/layer5_attention_weight_max": 108.26043701171875,
      "activations/layer5_attention_weight_min": -95.41890716552734,
      "activations/layer6_attention_weight_max": 62.397281646728516,
      "activations/layer6_attention_weight_min": -55.55751037597656,
      "activations/layer7_attention_weight_max": 77.5128402709961,
      "activations/layer7_attention_weight_min": -64.89543151855469,
      "activations/layer8_attention_weight_max": 46.14259719848633,
      "activations/layer8_attention_weight_min": -39.98347473144531,
      "activations/layer9_attention_weight_max": 55.937477111816406,
      "activations/layer9_attention_weight_min": -49.08243942260742,
      "epoch": 4.78,
      "learning_rate": 0.00012039204545454545,
      "loss": 2.8444,
      "step": 82250
    },
    {
      "activations/layer0_attention_weight_max": 14.230588912963867,
      "activations/layer0_attention_weight_min": -14.743337631225586,
      "activations/layer10_attention_weight_max": 39.67345428466797,
      "activations/layer10_attention_weight_min": -34.731746673583984,
      "activations/layer11_attention_weight_max": 30.493366241455078,
      "activations/layer11_attention_weight_min": -26.963455200195312,
      "activations/layer12_attention_weight_max": 22.92392921447754,
      "activations/layer12_attention_weight_min": -24.570289611816406,
      "activations/layer13_attention_weight_max": 57.75788116455078,
      "activations/layer13_attention_weight_min": -46.26442337036133,
      "activations/layer14_attention_weight_max": 67.84788513183594,
      "activations/layer14_attention_weight_min": -52.74684143066406,
      "activations/layer15_attention_weight_max": 43.06106185913086,
      "activations/layer15_attention_weight_min": -36.21986389160156,
      "activations/layer16_attention_weight_max": 41.71794891357422,
      "activations/layer16_attention_weight_min": -26.983882904052734,
      "activations/layer17_attention_weight_max": 63.17354202270508,
      "activations/layer17_attention_weight_min": -39.78622817993164,
      "activations/layer18_attention_weight_max": 52.759178161621094,
      "activations/layer18_attention_weight_min": -32.84870529174805,
      "activations/layer19_attention_weight_max": 23.882644653320312,
      "activations/layer19_attention_weight_min": -18.354305267333984,
      "activations/layer1_attention_weight_max": 18.57809066772461,
      "activations/layer1_attention_weight_min": -17.56699562072754,
      "activations/layer20_attention_weight_max": 25.470365524291992,
      "activations/layer20_attention_weight_min": -20.599266052246094,
      "activations/layer21_attention_weight_max": 42.193023681640625,
      "activations/layer21_attention_weight_min": -27.910375595092773,
      "activations/layer22_attention_weight_max": 37.18310546875,
      "activations/layer22_attention_weight_min": -23.498458862304688,
      "activations/layer23_attention_weight_max": 27.339101791381836,
      "activations/layer23_attention_weight_min": -18.35888671875,
      "activations/layer2_attention_weight_max": 30.278793334960938,
      "activations/layer2_attention_weight_min": -28.37902069091797,
      "activations/layer3_attention_weight_max": 78.06892395019531,
      "activations/layer3_attention_weight_min": -80.77433013916016,
      "activations/layer4_attention_weight_max": 103.96233367919922,
      "activations/layer4_attention_weight_min": -99.53430938720703,
      "activations/layer5_attention_weight_max": 104.98416137695312,
      "activations/layer5_attention_weight_min": -92.97215270996094,
      "activations/layer6_attention_weight_max": 59.405364990234375,
      "activations/layer6_attention_weight_min": -52.627803802490234,
      "activations/layer7_attention_weight_max": 71.58565521240234,
      "activations/layer7_attention_weight_min": -67.19071960449219,
      "activations/layer8_attention_weight_max": 42.73296356201172,
      "activations/layer8_attention_weight_min": -40.93758010864258,
      "activations/layer9_attention_weight_max": 59.973388671875,
      "activations/layer9_attention_weight_min": -56.45791244506836,
      "epoch": 4.78,
      "learning_rate": 0.00012037310606060605,
      "loss": 2.8685,
      "step": 82300
    },
    {
      "activations/layer0_attention_weight_max": 14.244691848754883,
      "activations/layer0_attention_weight_min": -15.144563674926758,
      "activations/layer10_attention_weight_max": 39.8314094543457,
      "activations/layer10_attention_weight_min": -34.69548034667969,
      "activations/layer11_attention_weight_max": 32.95262908935547,
      "activations/layer11_attention_weight_min": -28.210861206054688,
      "activations/layer12_attention_weight_max": 24.496381759643555,
      "activations/layer12_attention_weight_min": -28.072982788085938,
      "activations/layer13_attention_weight_max": 54.22364807128906,
      "activations/layer13_attention_weight_min": -36.83747100830078,
      "activations/layer14_attention_weight_max": 51.15446853637695,
      "activations/layer14_attention_weight_min": -34.455169677734375,
      "activations/layer15_attention_weight_max": 45.654441833496094,
      "activations/layer15_attention_weight_min": -30.353790283203125,
      "activations/layer16_attention_weight_max": 37.488609313964844,
      "activations/layer16_attention_weight_min": -28.876117706298828,
      "activations/layer17_attention_weight_max": 62.04711151123047,
      "activations/layer17_attention_weight_min": -40.490081787109375,
      "activations/layer18_attention_weight_max": 59.32190704345703,
      "activations/layer18_attention_weight_min": -36.97443389892578,
      "activations/layer19_attention_weight_max": 25.06134796142578,
      "activations/layer19_attention_weight_min": -21.228736877441406,
      "activations/layer1_attention_weight_max": 18.15117073059082,
      "activations/layer1_attention_weight_min": -15.846837997436523,
      "activations/layer20_attention_weight_max": 25.027347564697266,
      "activations/layer20_attention_weight_min": -21.356170654296875,
      "activations/layer21_attention_weight_max": 47.343135833740234,
      "activations/layer21_attention_weight_min": -25.414981842041016,
      "activations/layer22_attention_weight_max": 35.2455940246582,
      "activations/layer22_attention_weight_min": -24.38343048095703,
      "activations/layer23_attention_weight_max": 30.040390014648438,
      "activations/layer23_attention_weight_min": -18.142990112304688,
      "activations/layer2_attention_weight_max": 30.81844139099121,
      "activations/layer2_attention_weight_min": -30.11996841430664,
      "activations/layer3_attention_weight_max": 79.67732238769531,
      "activations/layer3_attention_weight_min": -83.6329574584961,
      "activations/layer4_attention_weight_max": 100.22320556640625,
      "activations/layer4_attention_weight_min": -94.61778259277344,
      "activations/layer5_attention_weight_max": 108.32479858398438,
      "activations/layer5_attention_weight_min": -95.7560043334961,
      "activations/layer6_attention_weight_max": 61.13054275512695,
      "activations/layer6_attention_weight_min": -53.13807678222656,
      "activations/layer7_attention_weight_max": 76.80655670166016,
      "activations/layer7_attention_weight_min": -65.10848999023438,
      "activations/layer8_attention_weight_max": 47.37214279174805,
      "activations/layer8_attention_weight_min": -39.4221076965332,
      "activations/layer9_attention_weight_max": 65.25032806396484,
      "activations/layer9_attention_weight_min": -46.25248718261719,
      "epoch": 4.78,
      "learning_rate": 0.00012035416666666665,
      "loss": 2.8413,
      "step": 82350
    },
    {
      "activations/layer0_attention_weight_max": 14.97422981262207,
      "activations/layer0_attention_weight_min": -15.331820487976074,
      "activations/layer10_attention_weight_max": 38.751461029052734,
      "activations/layer10_attention_weight_min": -35.78822326660156,
      "activations/layer11_attention_weight_max": 31.095306396484375,
      "activations/layer11_attention_weight_min": -27.37775993347168,
      "activations/layer12_attention_weight_max": 26.66538429260254,
      "activations/layer12_attention_weight_min": -22.946609497070312,
      "activations/layer13_attention_weight_max": 57.9636344909668,
      "activations/layer13_attention_weight_min": -38.636756896972656,
      "activations/layer14_attention_weight_max": 44.43498611450195,
      "activations/layer14_attention_weight_min": -30.453895568847656,
      "activations/layer15_attention_weight_max": 42.596317291259766,
      "activations/layer15_attention_weight_min": -29.821802139282227,
      "activations/layer16_attention_weight_max": 35.32311248779297,
      "activations/layer16_attention_weight_min": -26.020078659057617,
      "activations/layer17_attention_weight_max": 60.35481643676758,
      "activations/layer17_attention_weight_min": -39.25557327270508,
      "activations/layer18_attention_weight_max": 52.840118408203125,
      "activations/layer18_attention_weight_min": -33.71965408325195,
      "activations/layer19_attention_weight_max": 22.027013778686523,
      "activations/layer19_attention_weight_min": -18.746280670166016,
      "activations/layer1_attention_weight_max": 19.768203735351562,
      "activations/layer1_attention_weight_min": -15.517143249511719,
      "activations/layer20_attention_weight_max": 20.105112075805664,
      "activations/layer20_attention_weight_min": -23.409841537475586,
      "activations/layer21_attention_weight_max": 42.62852478027344,
      "activations/layer21_attention_weight_min": -27.594144821166992,
      "activations/layer22_attention_weight_max": 34.483734130859375,
      "activations/layer22_attention_weight_min": -26.77540397644043,
      "activations/layer23_attention_weight_max": 28.780088424682617,
      "activations/layer23_attention_weight_min": -17.10500717163086,
      "activations/layer2_attention_weight_max": 31.687223434448242,
      "activations/layer2_attention_weight_min": -28.530868530273438,
      "activations/layer3_attention_weight_max": 86.04792022705078,
      "activations/layer3_attention_weight_min": -87.26158905029297,
      "activations/layer4_attention_weight_max": 109.20539855957031,
      "activations/layer4_attention_weight_min": -100.29637145996094,
      "activations/layer5_attention_weight_max": 114.72758483886719,
      "activations/layer5_attention_weight_min": -96.36678314208984,
      "activations/layer6_attention_weight_max": 64.68487548828125,
      "activations/layer6_attention_weight_min": -58.094974517822266,
      "activations/layer7_attention_weight_max": 73.35911560058594,
      "activations/layer7_attention_weight_min": -66.89688110351562,
      "activations/layer8_attention_weight_max": 43.642765045166016,
      "activations/layer8_attention_weight_min": -42.572933197021484,
      "activations/layer9_attention_weight_max": 51.70936965942383,
      "activations/layer9_attention_weight_min": -49.11238098144531,
      "epoch": 4.79,
      "learning_rate": 0.00012033522727272727,
      "loss": 2.8645,
      "step": 82400
    },
    {
      "activations/layer0_attention_weight_max": 13.95752239227295,
      "activations/layer0_attention_weight_min": -14.65963363647461,
      "activations/layer10_attention_weight_max": 39.99603271484375,
      "activations/layer10_attention_weight_min": -35.50004959106445,
      "activations/layer11_attention_weight_max": 31.74365234375,
      "activations/layer11_attention_weight_min": -27.10072135925293,
      "activations/layer12_attention_weight_max": 25.525455474853516,
      "activations/layer12_attention_weight_min": -25.979175567626953,
      "activations/layer13_attention_weight_max": 51.53323745727539,
      "activations/layer13_attention_weight_min": -38.762481689453125,
      "activations/layer14_attention_weight_max": 59.11798858642578,
      "activations/layer14_attention_weight_min": -39.68120574951172,
      "activations/layer15_attention_weight_max": 47.75676345825195,
      "activations/layer15_attention_weight_min": -32.12050247192383,
      "activations/layer16_attention_weight_max": 41.97195816040039,
      "activations/layer16_attention_weight_min": -27.210969924926758,
      "activations/layer17_attention_weight_max": 62.80708694458008,
      "activations/layer17_attention_weight_min": -41.43233108520508,
      "activations/layer18_attention_weight_max": 55.51889419555664,
      "activations/layer18_attention_weight_min": -35.007606506347656,
      "activations/layer19_attention_weight_max": 27.03263282775879,
      "activations/layer19_attention_weight_min": -19.576749801635742,
      "activations/layer1_attention_weight_max": 18.502504348754883,
      "activations/layer1_attention_weight_min": -17.373985290527344,
      "activations/layer20_attention_weight_max": 23.240703582763672,
      "activations/layer20_attention_weight_min": -21.425514221191406,
      "activations/layer21_attention_weight_max": 42.11848831176758,
      "activations/layer21_attention_weight_min": -27.598388671875,
      "activations/layer22_attention_weight_max": 39.35581970214844,
      "activations/layer22_attention_weight_min": -23.179441452026367,
      "activations/layer23_attention_weight_max": 32.85357666015625,
      "activations/layer23_attention_weight_min": -21.035667419433594,
      "activations/layer2_attention_weight_max": 34.46173095703125,
      "activations/layer2_attention_weight_min": -31.334381103515625,
      "activations/layer3_attention_weight_max": 89.76426696777344,
      "activations/layer3_attention_weight_min": -89.11249542236328,
      "activations/layer4_attention_weight_max": 112.1069107055664,
      "activations/layer4_attention_weight_min": -104.74140930175781,
      "activations/layer5_attention_weight_max": 117.00264739990234,
      "activations/layer5_attention_weight_min": -100.78829193115234,
      "activations/layer6_attention_weight_max": 62.710025787353516,
      "activations/layer6_attention_weight_min": -56.47787857055664,
      "activations/layer7_attention_weight_max": 79.55704498291016,
      "activations/layer7_attention_weight_min": -76.8805923461914,
      "activations/layer8_attention_weight_max": 47.39669418334961,
      "activations/layer8_attention_weight_min": -45.23018264770508,
      "activations/layer9_attention_weight_max": 68.24974060058594,
      "activations/layer9_attention_weight_min": -58.0816535949707,
      "epoch": 4.79,
      "learning_rate": 0.00012031628787878787,
      "loss": 2.8634,
      "step": 82450
    },
    {
      "activations/layer0_attention_weight_max": 14.534488677978516,
      "activations/layer0_attention_weight_min": -14.864295959472656,
      "activations/layer10_attention_weight_max": 39.8721923828125,
      "activations/layer10_attention_weight_min": -33.47418975830078,
      "activations/layer11_attention_weight_max": 34.12469482421875,
      "activations/layer11_attention_weight_min": -27.817909240722656,
      "activations/layer12_attention_weight_max": 26.6987247467041,
      "activations/layer12_attention_weight_min": -25.51250457763672,
      "activations/layer13_attention_weight_max": 56.33368682861328,
      "activations/layer13_attention_weight_min": -34.54872512817383,
      "activations/layer14_attention_weight_max": 44.34917068481445,
      "activations/layer14_attention_weight_min": -30.17683219909668,
      "activations/layer15_attention_weight_max": 47.75959777832031,
      "activations/layer15_attention_weight_min": -30.634296417236328,
      "activations/layer16_attention_weight_max": 43.79452133178711,
      "activations/layer16_attention_weight_min": -26.795425415039062,
      "activations/layer17_attention_weight_max": 65.28811645507812,
      "activations/layer17_attention_weight_min": -40.32072830200195,
      "activations/layer18_attention_weight_max": 59.17866516113281,
      "activations/layer18_attention_weight_min": -33.17620849609375,
      "activations/layer19_attention_weight_max": 25.314407348632812,
      "activations/layer19_attention_weight_min": -18.44635581970215,
      "activations/layer1_attention_weight_max": 19.089706420898438,
      "activations/layer1_attention_weight_min": -16.30992889404297,
      "activations/layer20_attention_weight_max": 24.064083099365234,
      "activations/layer20_attention_weight_min": -20.51114845275879,
      "activations/layer21_attention_weight_max": 46.229190826416016,
      "activations/layer21_attention_weight_min": -26.800399780273438,
      "activations/layer22_attention_weight_max": 37.244327545166016,
      "activations/layer22_attention_weight_min": -22.505083084106445,
      "activations/layer23_attention_weight_max": 29.534223556518555,
      "activations/layer23_attention_weight_min": -17.32992172241211,
      "activations/layer2_attention_weight_max": 31.736724853515625,
      "activations/layer2_attention_weight_min": -29.081235885620117,
      "activations/layer3_attention_weight_max": 83.927734375,
      "activations/layer3_attention_weight_min": -85.49286651611328,
      "activations/layer4_attention_weight_max": 105.8182144165039,
      "activations/layer4_attention_weight_min": -97.2220458984375,
      "activations/layer5_attention_weight_max": 114.07575988769531,
      "activations/layer5_attention_weight_min": -97.08634948730469,
      "activations/layer6_attention_weight_max": 61.83571243286133,
      "activations/layer6_attention_weight_min": -55.024879455566406,
      "activations/layer7_attention_weight_max": 74.75188446044922,
      "activations/layer7_attention_weight_min": -68.09900665283203,
      "activations/layer8_attention_weight_max": 43.31632995605469,
      "activations/layer8_attention_weight_min": -41.014949798583984,
      "activations/layer9_attention_weight_max": 53.7442512512207,
      "activations/layer9_attention_weight_min": -45.69094467163086,
      "epoch": 4.79,
      "learning_rate": 0.00012029734848484847,
      "loss": 2.8683,
      "step": 82500
    },
    {
      "activations/layer0_attention_weight_max": 14.10657024383545,
      "activations/layer0_attention_weight_min": -14.383381843566895,
      "activations/layer10_attention_weight_max": 39.11140823364258,
      "activations/layer10_attention_weight_min": -34.338809967041016,
      "activations/layer11_attention_weight_max": 30.920917510986328,
      "activations/layer11_attention_weight_min": -26.688438415527344,
      "activations/layer12_attention_weight_max": 26.456581115722656,
      "activations/layer12_attention_weight_min": -25.025114059448242,
      "activations/layer13_attention_weight_max": 48.855098724365234,
      "activations/layer13_attention_weight_min": -35.76921081542969,
      "activations/layer14_attention_weight_max": 51.610294342041016,
      "activations/layer14_attention_weight_min": -36.960731506347656,
      "activations/layer15_attention_weight_max": 49.10844039916992,
      "activations/layer15_attention_weight_min": -32.707645416259766,
      "activations/layer16_attention_weight_max": 40.23761749267578,
      "activations/layer16_attention_weight_min": -27.688365936279297,
      "activations/layer17_attention_weight_max": 55.898033142089844,
      "activations/layer17_attention_weight_min": -37.43312072753906,
      "activations/layer18_attention_weight_max": 55.13654708862305,
      "activations/layer18_attention_weight_min": -31.841983795166016,
      "activations/layer19_attention_weight_max": 22.748994827270508,
      "activations/layer19_attention_weight_min": -18.46208381652832,
      "activations/layer1_attention_weight_max": 17.114770889282227,
      "activations/layer1_attention_weight_min": -16.096698760986328,
      "activations/layer20_attention_weight_max": 27.33116912841797,
      "activations/layer20_attention_weight_min": -22.198266983032227,
      "activations/layer21_attention_weight_max": 43.4967155456543,
      "activations/layer21_attention_weight_min": -26.68196678161621,
      "activations/layer22_attention_weight_max": 37.3797492980957,
      "activations/layer22_attention_weight_min": -22.085233688354492,
      "activations/layer23_attention_weight_max": 32.241825103759766,
      "activations/layer23_attention_weight_min": -20.774738311767578,
      "activations/layer2_attention_weight_max": 29.047618865966797,
      "activations/layer2_attention_weight_min": -26.416805267333984,
      "activations/layer3_attention_weight_max": 80.62458801269531,
      "activations/layer3_attention_weight_min": -80.07514953613281,
      "activations/layer4_attention_weight_max": 100.87496185302734,
      "activations/layer4_attention_weight_min": -92.84529876708984,
      "activations/layer5_attention_weight_max": 103.43208312988281,
      "activations/layer5_attention_weight_min": -94.71345520019531,
      "activations/layer6_attention_weight_max": 60.49842834472656,
      "activations/layer6_attention_weight_min": -52.48975372314453,
      "activations/layer7_attention_weight_max": 68.53083038330078,
      "activations/layer7_attention_weight_min": -65.66364288330078,
      "activations/layer8_attention_weight_max": 43.80418395996094,
      "activations/layer8_attention_weight_min": -42.55583572387695,
      "activations/layer9_attention_weight_max": 53.01472854614258,
      "activations/layer9_attention_weight_min": -45.27048873901367,
      "epoch": 4.8,
      "learning_rate": 0.00012027840909090909,
      "loss": 2.8637,
      "step": 82550
    },
    {
      "activations/layer0_attention_weight_max": 14.464401245117188,
      "activations/layer0_attention_weight_min": -14.635751724243164,
      "activations/layer10_attention_weight_max": 38.54011154174805,
      "activations/layer10_attention_weight_min": -33.77588653564453,
      "activations/layer11_attention_weight_max": 29.266345977783203,
      "activations/layer11_attention_weight_min": -25.793514251708984,
      "activations/layer12_attention_weight_max": 23.394731521606445,
      "activations/layer12_attention_weight_min": -27.45969009399414,
      "activations/layer13_attention_weight_max": 48.16435241699219,
      "activations/layer13_attention_weight_min": -35.819061279296875,
      "activations/layer14_attention_weight_max": 41.301109313964844,
      "activations/layer14_attention_weight_min": -30.656719207763672,
      "activations/layer15_attention_weight_max": 42.242774963378906,
      "activations/layer15_attention_weight_min": -30.292713165283203,
      "activations/layer16_attention_weight_max": 41.86083221435547,
      "activations/layer16_attention_weight_min": -24.99728012084961,
      "activations/layer17_attention_weight_max": 52.59926986694336,
      "activations/layer17_attention_weight_min": -36.440589904785156,
      "activations/layer18_attention_weight_max": 47.169857025146484,
      "activations/layer18_attention_weight_min": -32.37120819091797,
      "activations/layer19_attention_weight_max": 21.96591567993164,
      "activations/layer19_attention_weight_min": -18.487422943115234,
      "activations/layer1_attention_weight_max": 17.354631423950195,
      "activations/layer1_attention_weight_min": -16.02521514892578,
      "activations/layer20_attention_weight_max": 19.45566177368164,
      "activations/layer20_attention_weight_min": -21.58268165588379,
      "activations/layer21_attention_weight_max": 37.629268646240234,
      "activations/layer21_attention_weight_min": -25.746353149414062,
      "activations/layer22_attention_weight_max": 33.6434211730957,
      "activations/layer22_attention_weight_min": -22.84886932373047,
      "activations/layer23_attention_weight_max": 27.03417205810547,
      "activations/layer23_attention_weight_min": -19.33655548095703,
      "activations/layer2_attention_weight_max": 29.21760368347168,
      "activations/layer2_attention_weight_min": -28.247678756713867,
      "activations/layer3_attention_weight_max": 76.66558074951172,
      "activations/layer3_attention_weight_min": -80.99677276611328,
      "activations/layer4_attention_weight_max": 96.28157806396484,
      "activations/layer4_attention_weight_min": -92.62830352783203,
      "activations/layer5_attention_weight_max": 101.14186096191406,
      "activations/layer5_attention_weight_min": -90.1239013671875,
      "activations/layer6_attention_weight_max": 55.22456359863281,
      "activations/layer6_attention_weight_min": -50.6938362121582,
      "activations/layer7_attention_weight_max": 70.56546020507812,
      "activations/layer7_attention_weight_min": -63.67385482788086,
      "activations/layer8_attention_weight_max": 40.22071838378906,
      "activations/layer8_attention_weight_min": -38.871726989746094,
      "activations/layer9_attention_weight_max": 48.43642807006836,
      "activations/layer9_attention_weight_min": -46.26224899291992,
      "epoch": 4.8,
      "learning_rate": 0.00012025946969696967,
      "loss": 2.8476,
      "step": 82600
    },
    {
      "activations/layer0_attention_weight_max": 14.957121849060059,
      "activations/layer0_attention_weight_min": -14.658957481384277,
      "activations/layer10_attention_weight_max": 37.605777740478516,
      "activations/layer10_attention_weight_min": -33.046142578125,
      "activations/layer11_attention_weight_max": 28.92715835571289,
      "activations/layer11_attention_weight_min": -26.275989532470703,
      "activations/layer12_attention_weight_max": 22.0208740234375,
      "activations/layer12_attention_weight_min": -25.43802833557129,
      "activations/layer13_attention_weight_max": 59.57321548461914,
      "activations/layer13_attention_weight_min": -37.713253021240234,
      "activations/layer14_attention_weight_max": 64.8294906616211,
      "activations/layer14_attention_weight_min": -39.21474075317383,
      "activations/layer15_attention_weight_max": 45.23942565917969,
      "activations/layer15_attention_weight_min": -29.779340744018555,
      "activations/layer16_attention_weight_max": 40.05830764770508,
      "activations/layer16_attention_weight_min": -27.363685607910156,
      "activations/layer17_attention_weight_max": 61.89564514160156,
      "activations/layer17_attention_weight_min": -40.31209945678711,
      "activations/layer18_attention_weight_max": 52.14228439331055,
      "activations/layer18_attention_weight_min": -33.59245681762695,
      "activations/layer19_attention_weight_max": 24.917131423950195,
      "activations/layer19_attention_weight_min": -18.79518699645996,
      "activations/layer1_attention_weight_max": 18.569721221923828,
      "activations/layer1_attention_weight_min": -15.21130084991455,
      "activations/layer20_attention_weight_max": 23.847871780395508,
      "activations/layer20_attention_weight_min": -19.354473114013672,
      "activations/layer21_attention_weight_max": 48.29182815551758,
      "activations/layer21_attention_weight_min": -28.554006576538086,
      "activations/layer22_attention_weight_max": 33.9260139465332,
      "activations/layer22_attention_weight_min": -26.71796417236328,
      "activations/layer23_attention_weight_max": 30.440765380859375,
      "activations/layer23_attention_weight_min": -18.884794235229492,
      "activations/layer2_attention_weight_max": 32.16962814331055,
      "activations/layer2_attention_weight_min": -31.243711471557617,
      "activations/layer3_attention_weight_max": 82.40477752685547,
      "activations/layer3_attention_weight_min": -87.2838363647461,
      "activations/layer4_attention_weight_max": 104.24333953857422,
      "activations/layer4_attention_weight_min": -98.02855682373047,
      "activations/layer5_attention_weight_max": 109.08380126953125,
      "activations/layer5_attention_weight_min": -95.92843627929688,
      "activations/layer6_attention_weight_max": 59.88008499145508,
      "activations/layer6_attention_weight_min": -53.583858489990234,
      "activations/layer7_attention_weight_max": 69.9625015258789,
      "activations/layer7_attention_weight_min": -65.94483184814453,
      "activations/layer8_attention_weight_max": 41.704586029052734,
      "activations/layer8_attention_weight_min": -42.92722702026367,
      "activations/layer9_attention_weight_max": 52.531856536865234,
      "activations/layer9_attention_weight_min": -46.712745666503906,
      "epoch": 4.8,
      "learning_rate": 0.00012024053030303029,
      "loss": 2.8596,
      "step": 82650
    },
    {
      "activations/layer0_attention_weight_max": 15.76198673248291,
      "activations/layer0_attention_weight_min": -15.338449478149414,
      "activations/layer10_attention_weight_max": 39.35783004760742,
      "activations/layer10_attention_weight_min": -31.70174217224121,
      "activations/layer11_attention_weight_max": 33.91368103027344,
      "activations/layer11_attention_weight_min": -27.111602783203125,
      "activations/layer12_attention_weight_max": 29.986783981323242,
      "activations/layer12_attention_weight_min": -25.105331420898438,
      "activations/layer13_attention_weight_max": 53.28623580932617,
      "activations/layer13_attention_weight_min": -36.050716400146484,
      "activations/layer14_attention_weight_max": 45.952491760253906,
      "activations/layer14_attention_weight_min": -32.90574645996094,
      "activations/layer15_attention_weight_max": 43.79375457763672,
      "activations/layer15_attention_weight_min": -30.469127655029297,
      "activations/layer16_attention_weight_max": 36.94629669189453,
      "activations/layer16_attention_weight_min": -24.85148811340332,
      "activations/layer17_attention_weight_max": 66.7448501586914,
      "activations/layer17_attention_weight_min": -41.29549026489258,
      "activations/layer18_attention_weight_max": 51.60540771484375,
      "activations/layer18_attention_weight_min": -36.592323303222656,
      "activations/layer19_attention_weight_max": 26.341733932495117,
      "activations/layer19_attention_weight_min": -20.99873161315918,
      "activations/layer1_attention_weight_max": 20.271652221679688,
      "activations/layer1_attention_weight_min": -18.445634841918945,
      "activations/layer20_attention_weight_max": 24.16111946105957,
      "activations/layer20_attention_weight_min": -20.621549606323242,
      "activations/layer21_attention_weight_max": 44.01041793823242,
      "activations/layer21_attention_weight_min": -29.15047264099121,
      "activations/layer22_attention_weight_max": 37.07364273071289,
      "activations/layer22_attention_weight_min": -22.1655216217041,
      "activations/layer23_attention_weight_max": 28.686767578125,
      "activations/layer23_attention_weight_min": -16.899765014648438,
      "activations/layer2_attention_weight_max": 31.950252532958984,
      "activations/layer2_attention_weight_min": -30.392438888549805,
      "activations/layer3_attention_weight_max": 80.5907211303711,
      "activations/layer3_attention_weight_min": -89.49430847167969,
      "activations/layer4_attention_weight_max": 103.1848373413086,
      "activations/layer4_attention_weight_min": -97.8891830444336,
      "activations/layer5_attention_weight_max": 111.04844665527344,
      "activations/layer5_attention_weight_min": -97.67530822753906,
      "activations/layer6_attention_weight_max": 62.587345123291016,
      "activations/layer6_attention_weight_min": -54.1937370300293,
      "activations/layer7_attention_weight_max": 76.69469451904297,
      "activations/layer7_attention_weight_min": -68.5045394897461,
      "activations/layer8_attention_weight_max": 43.84404754638672,
      "activations/layer8_attention_weight_min": -40.152339935302734,
      "activations/layer9_attention_weight_max": 68.71916961669922,
      "activations/layer9_attention_weight_min": -46.89848709106445,
      "epoch": 4.81,
      "learning_rate": 0.0001202215909090909,
      "loss": 2.8694,
      "step": 82700
    },
    {
      "activations/layer0_attention_weight_max": 14.517285346984863,
      "activations/layer0_attention_weight_min": -15.684289932250977,
      "activations/layer10_attention_weight_max": 37.177276611328125,
      "activations/layer10_attention_weight_min": -37.47467041015625,
      "activations/layer11_attention_weight_max": 28.608661651611328,
      "activations/layer11_attention_weight_min": -26.462749481201172,
      "activations/layer12_attention_weight_max": 20.511499404907227,
      "activations/layer12_attention_weight_min": -26.23221206665039,
      "activations/layer13_attention_weight_max": 49.0473747253418,
      "activations/layer13_attention_weight_min": -35.619972229003906,
      "activations/layer14_attention_weight_max": 57.184303283691406,
      "activations/layer14_attention_weight_min": -38.052574157714844,
      "activations/layer15_attention_weight_max": 42.93953323364258,
      "activations/layer15_attention_weight_min": -29.683753967285156,
      "activations/layer16_attention_weight_max": 36.69380569458008,
      "activations/layer16_attention_weight_min": -25.74355697631836,
      "activations/layer17_attention_weight_max": 54.987457275390625,
      "activations/layer17_attention_weight_min": -37.12394332885742,
      "activations/layer18_attention_weight_max": 46.79138946533203,
      "activations/layer18_attention_weight_min": -33.20600509643555,
      "activations/layer19_attention_weight_max": 21.971616744995117,
      "activations/layer19_attention_weight_min": -18.146678924560547,
      "activations/layer1_attention_weight_max": 18.24471092224121,
      "activations/layer1_attention_weight_min": -17.14784049987793,
      "activations/layer20_attention_weight_max": 23.79021644592285,
      "activations/layer20_attention_weight_min": -18.723112106323242,
      "activations/layer21_attention_weight_max": 46.419036865234375,
      "activations/layer21_attention_weight_min": -29.297842025756836,
      "activations/layer22_attention_weight_max": 31.30777359008789,
      "activations/layer22_attention_weight_min": -23.851438522338867,
      "activations/layer23_attention_weight_max": 29.73617935180664,
      "activations/layer23_attention_weight_min": -17.882221221923828,
      "activations/layer2_attention_weight_max": 31.003524780273438,
      "activations/layer2_attention_weight_min": -28.804580688476562,
      "activations/layer3_attention_weight_max": 82.26302337646484,
      "activations/layer3_attention_weight_min": -86.86676788330078,
      "activations/layer4_attention_weight_max": 101.0027084350586,
      "activations/layer4_attention_weight_min": -97.52278900146484,
      "activations/layer5_attention_weight_max": 105.68988037109375,
      "activations/layer5_attention_weight_min": -91.5025634765625,
      "activations/layer6_attention_weight_max": 60.338401794433594,
      "activations/layer6_attention_weight_min": -51.81221389770508,
      "activations/layer7_attention_weight_max": 71.17455291748047,
      "activations/layer7_attention_weight_min": -70.89533233642578,
      "activations/layer8_attention_weight_max": 45.8034782409668,
      "activations/layer8_attention_weight_min": -43.584712982177734,
      "activations/layer9_attention_weight_max": 54.45766067504883,
      "activations/layer9_attention_weight_min": -47.82804870605469,
      "epoch": 4.81,
      "learning_rate": 0.00012020265151515149,
      "loss": 2.8593,
      "step": 82750
    },
    {
      "activations/layer0_attention_weight_max": 15.080081939697266,
      "activations/layer0_attention_weight_min": -15.056940078735352,
      "activations/layer10_attention_weight_max": 42.30732727050781,
      "activations/layer10_attention_weight_min": -38.012516021728516,
      "activations/layer11_attention_weight_max": 37.92859649658203,
      "activations/layer11_attention_weight_min": -30.239301681518555,
      "activations/layer12_attention_weight_max": 32.73591232299805,
      "activations/layer12_attention_weight_min": -24.997116088867188,
      "activations/layer13_attention_weight_max": 65.0755615234375,
      "activations/layer13_attention_weight_min": -38.34740447998047,
      "activations/layer14_attention_weight_max": 102.11650085449219,
      "activations/layer14_attention_weight_min": -45.24760818481445,
      "activations/layer15_attention_weight_max": 70.2839126586914,
      "activations/layer15_attention_weight_min": -31.071727752685547,
      "activations/layer16_attention_weight_max": 47.52596664428711,
      "activations/layer16_attention_weight_min": -27.913471221923828,
      "activations/layer17_attention_weight_max": 64.92578887939453,
      "activations/layer17_attention_weight_min": -40.0886344909668,
      "activations/layer18_attention_weight_max": 52.695980072021484,
      "activations/layer18_attention_weight_min": -34.76775360107422,
      "activations/layer19_attention_weight_max": 26.70975685119629,
      "activations/layer19_attention_weight_min": -19.545284271240234,
      "activations/layer1_attention_weight_max": 17.815032958984375,
      "activations/layer1_attention_weight_min": -13.538314819335938,
      "activations/layer20_attention_weight_max": 26.267099380493164,
      "activations/layer20_attention_weight_min": -22.568880081176758,
      "activations/layer21_attention_weight_max": 45.59269332885742,
      "activations/layer21_attention_weight_min": -28.19401741027832,
      "activations/layer22_attention_weight_max": 35.9965934753418,
      "activations/layer22_attention_weight_min": -26.092845916748047,
      "activations/layer23_attention_weight_max": 26.056434631347656,
      "activations/layer23_attention_weight_min": -18.640106201171875,
      "activations/layer2_attention_weight_max": 30.59955406188965,
      "activations/layer2_attention_weight_min": -28.836387634277344,
      "activations/layer3_attention_weight_max": 83.5439682006836,
      "activations/layer3_attention_weight_min": -86.26236724853516,
      "activations/layer4_attention_weight_max": 103.7404556274414,
      "activations/layer4_attention_weight_min": -99.992919921875,
      "activations/layer5_attention_weight_max": 107.54296112060547,
      "activations/layer5_attention_weight_min": -97.93307495117188,
      "activations/layer6_attention_weight_max": 59.148277282714844,
      "activations/layer6_attention_weight_min": -52.20551681518555,
      "activations/layer7_attention_weight_max": 73.63697052001953,
      "activations/layer7_attention_weight_min": -66.6340560913086,
      "activations/layer8_attention_weight_max": 47.891212463378906,
      "activations/layer8_attention_weight_min": -39.96782302856445,
      "activations/layer9_attention_weight_max": 64.32763671875,
      "activations/layer9_attention_weight_min": -56.260467529296875,
      "epoch": 4.81,
      "learning_rate": 0.00012018371212121211,
      "loss": 2.8728,
      "step": 82800
    },
    {
      "activations/layer0_attention_weight_max": 14.842336654663086,
      "activations/layer0_attention_weight_min": -15.08965015411377,
      "activations/layer10_attention_weight_max": 36.88695526123047,
      "activations/layer10_attention_weight_min": -33.470428466796875,
      "activations/layer11_attention_weight_max": 28.43486785888672,
      "activations/layer11_attention_weight_min": -26.0755615234375,
      "activations/layer12_attention_weight_max": 20.978940963745117,
      "activations/layer12_attention_weight_min": -25.871265411376953,
      "activations/layer13_attention_weight_max": 53.45669174194336,
      "activations/layer13_attention_weight_min": -34.904197692871094,
      "activations/layer14_attention_weight_max": 40.92272186279297,
      "activations/layer14_attention_weight_min": -28.753080368041992,
      "activations/layer15_attention_weight_max": 40.180442810058594,
      "activations/layer15_attention_weight_min": -30.2086238861084,
      "activations/layer16_attention_weight_max": 32.43306350708008,
      "activations/layer16_attention_weight_min": -24.934471130371094,
      "activations/layer17_attention_weight_max": 54.56968688964844,
      "activations/layer17_attention_weight_min": -36.67243957519531,
      "activations/layer18_attention_weight_max": 47.08864212036133,
      "activations/layer18_attention_weight_min": -32.617610931396484,
      "activations/layer19_attention_weight_max": 20.955034255981445,
      "activations/layer19_attention_weight_min": -19.4084529876709,
      "activations/layer1_attention_weight_max": 19.52263832092285,
      "activations/layer1_attention_weight_min": -16.911319732666016,
      "activations/layer20_attention_weight_max": 21.443025588989258,
      "activations/layer20_attention_weight_min": -23.1735897064209,
      "activations/layer21_attention_weight_max": 34.58778762817383,
      "activations/layer21_attention_weight_min": -27.558406829833984,
      "activations/layer22_attention_weight_max": 30.626251220703125,
      "activations/layer22_attention_weight_min": -23.351783752441406,
      "activations/layer23_attention_weight_max": 23.68575096130371,
      "activations/layer23_attention_weight_min": -17.85889434814453,
      "activations/layer2_attention_weight_max": 29.796634674072266,
      "activations/layer2_attention_weight_min": -27.107704162597656,
      "activations/layer3_attention_weight_max": 81.66397857666016,
      "activations/layer3_attention_weight_min": -84.48797607421875,
      "activations/layer4_attention_weight_max": 98.50267791748047,
      "activations/layer4_attention_weight_min": -95.6943359375,
      "activations/layer5_attention_weight_max": 103.21337890625,
      "activations/layer5_attention_weight_min": -95.01516723632812,
      "activations/layer6_attention_weight_max": 58.52302169799805,
      "activations/layer6_attention_weight_min": -53.724674224853516,
      "activations/layer7_attention_weight_max": 68.57474517822266,
      "activations/layer7_attention_weight_min": -68.34851837158203,
      "activations/layer8_attention_weight_max": 39.78230285644531,
      "activations/layer8_attention_weight_min": -42.657535552978516,
      "activations/layer9_attention_weight_max": 48.93603515625,
      "activations/layer9_attention_weight_min": -43.70981979370117,
      "epoch": 4.81,
      "learning_rate": 0.00012016477272727272,
      "loss": 2.8715,
      "step": 82850
    },
    {
      "activations/layer0_attention_weight_max": 14.855295181274414,
      "activations/layer0_attention_weight_min": -14.996343612670898,
      "activations/layer10_attention_weight_max": 36.5352783203125,
      "activations/layer10_attention_weight_min": -32.699222564697266,
      "activations/layer11_attention_weight_max": 29.92778778076172,
      "activations/layer11_attention_weight_min": -26.924713134765625,
      "activations/layer12_attention_weight_max": 21.145505905151367,
      "activations/layer12_attention_weight_min": -25.21519660949707,
      "activations/layer13_attention_weight_max": 47.478633880615234,
      "activations/layer13_attention_weight_min": -37.189796447753906,
      "activations/layer14_attention_weight_max": 47.664737701416016,
      "activations/layer14_attention_weight_min": -34.197547912597656,
      "activations/layer15_attention_weight_max": 40.24502182006836,
      "activations/layer15_attention_weight_min": -29.01563262939453,
      "activations/layer16_attention_weight_max": 31.801708221435547,
      "activations/layer16_attention_weight_min": -25.564542770385742,
      "activations/layer17_attention_weight_max": 57.60879135131836,
      "activations/layer17_attention_weight_min": -36.54248046875,
      "activations/layer18_attention_weight_max": 46.443687438964844,
      "activations/layer18_attention_weight_min": -32.432281494140625,
      "activations/layer19_attention_weight_max": 19.550188064575195,
      "activations/layer19_attention_weight_min": -20.322546005249023,
      "activations/layer1_attention_weight_max": 19.301061630249023,
      "activations/layer1_attention_weight_min": -16.481447219848633,
      "activations/layer20_attention_weight_max": 24.326309204101562,
      "activations/layer20_attention_weight_min": -19.748302459716797,
      "activations/layer21_attention_weight_max": 39.71199417114258,
      "activations/layer21_attention_weight_min": -26.833629608154297,
      "activations/layer22_attention_weight_max": 33.124900817871094,
      "activations/layer22_attention_weight_min": -23.646217346191406,
      "activations/layer23_attention_weight_max": 26.836042404174805,
      "activations/layer23_attention_weight_min": -18.454179763793945,
      "activations/layer2_attention_weight_max": 33.64130783081055,
      "activations/layer2_attention_weight_min": -29.974506378173828,
      "activations/layer3_attention_weight_max": 82.37313842773438,
      "activations/layer3_attention_weight_min": -85.33570861816406,
      "activations/layer4_attention_weight_max": 109.0982666015625,
      "activations/layer4_attention_weight_min": -97.83403778076172,
      "activations/layer5_attention_weight_max": 110.76028442382812,
      "activations/layer5_attention_weight_min": -96.48558044433594,
      "activations/layer6_attention_weight_max": 60.548118591308594,
      "activations/layer6_attention_weight_min": -51.70830154418945,
      "activations/layer7_attention_weight_max": 70.84748077392578,
      "activations/layer7_attention_weight_min": -64.9388198852539,
      "activations/layer8_attention_weight_max": 40.28242874145508,
      "activations/layer8_attention_weight_min": -44.75571060180664,
      "activations/layer9_attention_weight_max": 49.522377014160156,
      "activations/layer9_attention_weight_min": -45.702171325683594,
      "epoch": 4.82,
      "learning_rate": 0.00012014621212121211,
      "loss": 2.8621,
      "step": 82900
    },
    {
      "activations/layer0_attention_weight_max": 14.224494934082031,
      "activations/layer0_attention_weight_min": -15.639016151428223,
      "activations/layer10_attention_weight_max": 45.31465148925781,
      "activations/layer10_attention_weight_min": -36.27387237548828,
      "activations/layer11_attention_weight_max": 34.26217269897461,
      "activations/layer11_attention_weight_min": -27.886859893798828,
      "activations/layer12_attention_weight_max": 24.551715850830078,
      "activations/layer12_attention_weight_min": -22.67613410949707,
      "activations/layer13_attention_weight_max": 62.80781173706055,
      "activations/layer13_attention_weight_min": -38.291656494140625,
      "activations/layer14_attention_weight_max": 53.407684326171875,
      "activations/layer14_attention_weight_min": -33.35488510131836,
      "activations/layer15_attention_weight_max": 50.24697494506836,
      "activations/layer15_attention_weight_min": -31.47930145263672,
      "activations/layer16_attention_weight_max": 40.43044662475586,
      "activations/layer16_attention_weight_min": -27.83084487915039,
      "activations/layer17_attention_weight_max": 61.95205307006836,
      "activations/layer17_attention_weight_min": -41.54359817504883,
      "activations/layer18_attention_weight_max": 55.38267135620117,
      "activations/layer18_attention_weight_min": -35.089149475097656,
      "activations/layer19_attention_weight_max": 23.975360870361328,
      "activations/layer19_attention_weight_min": -19.700246810913086,
      "activations/layer1_attention_weight_max": 17.239593505859375,
      "activations/layer1_attention_weight_min": -17.439777374267578,
      "activations/layer20_attention_weight_max": 25.10502815246582,
      "activations/layer20_attention_weight_min": -21.63707160949707,
      "activations/layer21_attention_weight_max": 48.60916519165039,
      "activations/layer21_attention_weight_min": -26.075368881225586,
      "activations/layer22_attention_weight_max": 35.94081115722656,
      "activations/layer22_attention_weight_min": -24.290729522705078,
      "activations/layer23_attention_weight_max": 30.615337371826172,
      "activations/layer23_attention_weight_min": -18.28437614440918,
      "activations/layer2_attention_weight_max": 31.534473419189453,
      "activations/layer2_attention_weight_min": -28.21515655517578,
      "activations/layer3_attention_weight_max": 79.51175689697266,
      "activations/layer3_attention_weight_min": -83.44659423828125,
      "activations/layer4_attention_weight_max": 103.5145263671875,
      "activations/layer4_attention_weight_min": -98.98251342773438,
      "activations/layer5_attention_weight_max": 109.7318344116211,
      "activations/layer5_attention_weight_min": -100.7239990234375,
      "activations/layer6_attention_weight_max": 60.5871467590332,
      "activations/layer6_attention_weight_min": -58.50330352783203,
      "activations/layer7_attention_weight_max": 73.13823699951172,
      "activations/layer7_attention_weight_min": -72.38129425048828,
      "activations/layer8_attention_weight_max": 47.64714050292969,
      "activations/layer8_attention_weight_min": -44.0709114074707,
      "activations/layer9_attention_weight_max": 59.51139450073242,
      "activations/layer9_attention_weight_min": -50.20588302612305,
      "epoch": 4.82,
      "learning_rate": 0.00012012727272727271,
      "loss": 2.8634,
      "step": 82950
    },
    {
      "activations/layer0_attention_weight_max": 14.286052703857422,
      "activations/layer0_attention_weight_min": -14.63774299621582,
      "activations/layer10_attention_weight_max": 39.32395935058594,
      "activations/layer10_attention_weight_min": -37.22148895263672,
      "activations/layer11_attention_weight_max": 31.31088638305664,
      "activations/layer11_attention_weight_min": -28.411598205566406,
      "activations/layer12_attention_weight_max": 24.011077880859375,
      "activations/layer12_attention_weight_min": -24.00499153137207,
      "activations/layer13_attention_weight_max": 49.6174430847168,
      "activations/layer13_attention_weight_min": -37.508235931396484,
      "activations/layer14_attention_weight_max": 40.59058380126953,
      "activations/layer14_attention_weight_min": -30.217498779296875,
      "activations/layer15_attention_weight_max": 41.14443588256836,
      "activations/layer15_attention_weight_min": -31.64079475402832,
      "activations/layer16_attention_weight_max": 38.37559127807617,
      "activations/layer16_attention_weight_min": -30.24919319152832,
      "activations/layer17_attention_weight_max": 56.0914306640625,
      "activations/layer17_attention_weight_min": -42.07316207885742,
      "activations/layer18_attention_weight_max": 52.6020622253418,
      "activations/layer18_attention_weight_min": -33.65947341918945,
      "activations/layer19_attention_weight_max": 22.2042293548584,
      "activations/layer19_attention_weight_min": -19.102890014648438,
      "activations/layer1_attention_weight_max": 18.663410186767578,
      "activations/layer1_attention_weight_min": -14.505769729614258,
      "activations/layer20_attention_weight_max": 20.334245681762695,
      "activations/layer20_attention_weight_min": -21.741857528686523,
      "activations/layer21_attention_weight_max": 40.91319274902344,
      "activations/layer21_attention_weight_min": -29.7222957611084,
      "activations/layer22_attention_weight_max": 32.53832244873047,
      "activations/layer22_attention_weight_min": -24.786062240600586,
      "activations/layer23_attention_weight_max": 25.26641082763672,
      "activations/layer23_attention_weight_min": -19.507686614990234,
      "activations/layer2_attention_weight_max": 30.25103759765625,
      "activations/layer2_attention_weight_min": -28.384986877441406,
      "activations/layer3_attention_weight_max": 81.7728271484375,
      "activations/layer3_attention_weight_min": -81.07166290283203,
      "activations/layer4_attention_weight_max": 98.695556640625,
      "activations/layer4_attention_weight_min": -94.38558959960938,
      "activations/layer5_attention_weight_max": 100.1393051147461,
      "activations/layer5_attention_weight_min": -100.45238494873047,
      "activations/layer6_attention_weight_max": 54.27284240722656,
      "activations/layer6_attention_weight_min": -56.99998474121094,
      "activations/layer7_attention_weight_max": 67.52886962890625,
      "activations/layer7_attention_weight_min": -66.12561798095703,
      "activations/layer8_attention_weight_max": 40.90634536743164,
      "activations/layer8_attention_weight_min": -42.75679397583008,
      "activations/layer9_attention_weight_max": 50.39972686767578,
      "activations/layer9_attention_weight_min": -46.972572326660156,
      "epoch": 4.82,
      "learning_rate": 0.00012010833333333332,
      "loss": 2.8671,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_loss": 2.806640625,
      "eval_runtime": 8.4643,
      "eval_samples_per_second": 507.307,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_openwebtext_loss": 2.806640625,
      "eval_openwebtext_ppl": 16.554212894695087,
      "eval_openwebtext_runtime": 8.4643,
      "eval_openwebtext_samples_per_second": 507.307,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_wikitext_loss": 3.041015625,
      "eval_wikitext_ppl": 20.92648590822237,
      "eval_wikitext_runtime": 1.9391,
      "eval_wikitext_samples_per_second": 235.162,
      "step": 83000
    },
    {
      "epoch": 4.82,
      "eval_lambada_loss": 2.951171875,
      "eval_lambada_ppl": 19.128356641900336,
      "eval_lambada_runtime": 9.5611,
      "eval_lambada_samples_per_second": 509.249,
      "step": 83000
    },
    {
      "activations/layer0_attention_weight_max": 15.036827087402344,
      "activations/layer0_attention_weight_min": -15.115739822387695,
      "activations/layer10_attention_weight_max": 37.19000244140625,
      "activations/layer10_attention_weight_min": -34.8775634765625,
      "activations/layer11_attention_weight_max": 31.67403221130371,
      "activations/layer11_attention_weight_min": -26.824871063232422,
      "activations/layer12_attention_weight_max": 22.36266326904297,
      "activations/layer12_attention_weight_min": -25.981142044067383,
      "activations/layer13_attention_weight_max": 49.29978561401367,
      "activations/layer13_attention_weight_min": -35.44493103027344,
      "activations/layer14_attention_weight_max": 46.352874755859375,
      "activations/layer14_attention_weight_min": -31.2236328125,
      "activations/layer15_attention_weight_max": 42.15406036376953,
      "activations/layer15_attention_weight_min": -30.658693313598633,
      "activations/layer16_attention_weight_max": 34.98831558227539,
      "activations/layer16_attention_weight_min": -27.729854583740234,
      "activations/layer17_attention_weight_max": 68.40499114990234,
      "activations/layer17_attention_weight_min": -41.12115478515625,
      "activations/layer18_attention_weight_max": 56.689666748046875,
      "activations/layer18_attention_weight_min": -36.60506057739258,
      "activations/layer19_attention_weight_max": 29.37801170349121,
      "activations/layer19_attention_weight_min": -19.56538200378418,
      "activations/layer1_attention_weight_max": 17.093421936035156,
      "activations/layer1_attention_weight_min": -14.397250175476074,
      "activations/layer20_attention_weight_max": 26.25104331970215,
      "activations/layer20_attention_weight_min": -23.56906509399414,
      "activations/layer21_attention_weight_max": 47.93534851074219,
      "activations/layer21_attention_weight_min": -25.918081283569336,
      "activations/layer22_attention_weight_max": 36.67411422729492,
      "activations/layer22_attention_weight_min": -25.08351707458496,
      "activations/layer23_attention_weight_max": 33.411834716796875,
      "activations/layer23_attention_weight_min": -17.776071548461914,
      "activations/layer2_attention_weight_max": 31.17993927001953,
      "activations/layer2_attention_weight_min": -27.87293243408203,
      "activations/layer3_attention_weight_max": 80.23462677001953,
      "activations/layer3_attention_weight_min": -85.97251892089844,
      "activations/layer4_attention_weight_max": 101.26197052001953,
      "activations/layer4_attention_weight_min": -100.7623291015625,
      "activations/layer5_attention_weight_max": 110.04985046386719,
      "activations/layer5_attention_weight_min": -98.39772033691406,
      "activations/layer6_attention_weight_max": 58.863956451416016,
      "activations/layer6_attention_weight_min": -53.959590911865234,
      "activations/layer7_attention_weight_max": 67.91693115234375,
      "activations/layer7_attention_weight_min": -71.96160125732422,
      "activations/layer8_attention_weight_max": 40.51280975341797,
      "activations/layer8_attention_weight_min": -41.96080780029297,
      "activations/layer9_attention_weight_max": 50.18937301635742,
      "activations/layer9_attention_weight_min": -49.48291778564453,
      "epoch": 4.83,
      "learning_rate": 0.00012008939393939393,
      "loss": 2.8611,
      "step": 83050
    },
    {
      "activations/layer0_attention_weight_max": 14.592296600341797,
      "activations/layer0_attention_weight_min": -14.335661888122559,
      "activations/layer10_attention_weight_max": 37.51546096801758,
      "activations/layer10_attention_weight_min": -33.22774124145508,
      "activations/layer11_attention_weight_max": 29.59865951538086,
      "activations/layer11_attention_weight_min": -26.480125427246094,
      "activations/layer12_attention_weight_max": 21.723770141601562,
      "activations/layer12_attention_weight_min": -26.9930477142334,
      "activations/layer13_attention_weight_max": 46.41201400756836,
      "activations/layer13_attention_weight_min": -34.27759552001953,
      "activations/layer14_attention_weight_max": 40.07746505737305,
      "activations/layer14_attention_weight_min": -29.08221435546875,
      "activations/layer15_attention_weight_max": 43.88541030883789,
      "activations/layer15_attention_weight_min": -29.44419288635254,
      "activations/layer16_attention_weight_max": 44.233341217041016,
      "activations/layer16_attention_weight_min": -26.916765213012695,
      "activations/layer17_attention_weight_max": 64.07945251464844,
      "activations/layer17_attention_weight_min": -40.2659797668457,
      "activations/layer18_attention_weight_max": 53.090415954589844,
      "activations/layer18_attention_weight_min": -35.34357833862305,
      "activations/layer19_attention_weight_max": 28.002798080444336,
      "activations/layer19_attention_weight_min": -18.672752380371094,
      "activations/layer1_attention_weight_max": 17.90200424194336,
      "activations/layer1_attention_weight_min": -14.000577926635742,
      "activations/layer20_attention_weight_max": 25.720115661621094,
      "activations/layer20_attention_weight_min": -20.80171775817871,
      "activations/layer21_attention_weight_max": 37.631072998046875,
      "activations/layer21_attention_weight_min": -25.461456298828125,
      "activations/layer22_attention_weight_max": 40.448699951171875,
      "activations/layer22_attention_weight_min": -26.875513076782227,
      "activations/layer23_attention_weight_max": 33.577152252197266,
      "activations/layer23_attention_weight_min": -19.083702087402344,
      "activations/layer2_attention_weight_max": 30.038009643554688,
      "activations/layer2_attention_weight_min": -27.69265365600586,
      "activations/layer3_attention_weight_max": 83.06622314453125,
      "activations/layer3_attention_weight_min": -86.63623046875,
      "activations/layer4_attention_weight_max": 102.1992416381836,
      "activations/layer4_attention_weight_min": -96.71575164794922,
      "activations/layer5_attention_weight_max": 105.01165771484375,
      "activations/layer5_attention_weight_min": -96.72039794921875,
      "activations/layer6_attention_weight_max": 61.31318283081055,
      "activations/layer6_attention_weight_min": -53.778873443603516,
      "activations/layer7_attention_weight_max": 69.48614501953125,
      "activations/layer7_attention_weight_min": -64.93531036376953,
      "activations/layer8_attention_weight_max": 41.854618072509766,
      "activations/layer8_attention_weight_min": -40.9111328125,
      "activations/layer9_attention_weight_max": 47.50520324707031,
      "activations/layer9_attention_weight_min": -45.56985092163086,
      "epoch": 4.83,
      "learning_rate": 0.00012007045454545453,
      "loss": 2.8669,
      "step": 83100
    },
    {
      "activations/layer0_attention_weight_max": 13.507254600524902,
      "activations/layer0_attention_weight_min": -14.960074424743652,
      "activations/layer10_attention_weight_max": 41.5167121887207,
      "activations/layer10_attention_weight_min": -34.479347229003906,
      "activations/layer11_attention_weight_max": 31.835773468017578,
      "activations/layer11_attention_weight_min": -29.2354736328125,
      "activations/layer12_attention_weight_max": 32.038726806640625,
      "activations/layer12_attention_weight_min": -25.93670082092285,
      "activations/layer13_attention_weight_max": 51.4024772644043,
      "activations/layer13_attention_weight_min": -33.77493667602539,
      "activations/layer14_attention_weight_max": 51.98544692993164,
      "activations/layer14_attention_weight_min": -36.5471305847168,
      "activations/layer15_attention_weight_max": 52.92601013183594,
      "activations/layer15_attention_weight_min": -32.814022064208984,
      "activations/layer16_attention_weight_max": 36.85698318481445,
      "activations/layer16_attention_weight_min": -26.664892196655273,
      "activations/layer17_attention_weight_max": 56.89632797241211,
      "activations/layer17_attention_weight_min": -40.489253997802734,
      "activations/layer18_attention_weight_max": 52.526126861572266,
      "activations/layer18_attention_weight_min": -34.24430465698242,
      "activations/layer19_attention_weight_max": 24.516361236572266,
      "activations/layer19_attention_weight_min": -19.44281005859375,
      "activations/layer1_attention_weight_max": 17.560850143432617,
      "activations/layer1_attention_weight_min": -14.560233116149902,
      "activations/layer20_attention_weight_max": 24.197221755981445,
      "activations/layer20_attention_weight_min": -22.072959899902344,
      "activations/layer21_attention_weight_max": 41.668949127197266,
      "activations/layer21_attention_weight_min": -25.567834854125977,
      "activations/layer22_attention_weight_max": 35.8521728515625,
      "activations/layer22_attention_weight_min": -25.1657772064209,
      "activations/layer23_attention_weight_max": 28.79317283630371,
      "activations/layer23_attention_weight_min": -19.215024948120117,
      "activations/layer2_attention_weight_max": 32.753196716308594,
      "activations/layer2_attention_weight_min": -29.868576049804688,
      "activations/layer3_attention_weight_max": 82.970703125,
      "activations/layer3_attention_weight_min": -84.27471160888672,
      "activations/layer4_attention_weight_max": 104.04144287109375,
      "activations/layer4_attention_weight_min": -95.73013305664062,
      "activations/layer5_attention_weight_max": 109.7696762084961,
      "activations/layer5_attention_weight_min": -95.51173400878906,
      "activations/layer6_attention_weight_max": 61.377601623535156,
      "activations/layer6_attention_weight_min": -57.06898880004883,
      "activations/layer7_attention_weight_max": 70.954345703125,
      "activations/layer7_attention_weight_min": -69.60883331298828,
      "activations/layer8_attention_weight_max": 47.73781204223633,
      "activations/layer8_attention_weight_min": -49.28770446777344,
      "activations/layer9_attention_weight_max": 54.74103546142578,
      "activations/layer9_attention_weight_min": -46.882347106933594,
      "epoch": 4.83,
      "learning_rate": 0.00012005151515151515,
      "loss": 2.8495,
      "step": 83150
    },
    {
      "activations/layer0_attention_weight_max": 14.772247314453125,
      "activations/layer0_attention_weight_min": -14.653481483459473,
      "activations/layer10_attention_weight_max": 49.55548095703125,
      "activations/layer10_attention_weight_min": -43.89204788208008,
      "activations/layer11_attention_weight_max": 39.994300842285156,
      "activations/layer11_attention_weight_min": -30.551841735839844,
      "activations/layer12_attention_weight_max": 25.162492752075195,
      "activations/layer12_attention_weight_min": -32.70585632324219,
      "activations/layer13_attention_weight_max": 54.41094207763672,
      "activations/layer13_attention_weight_min": -34.7579345703125,
      "activations/layer14_attention_weight_max": 72.61669158935547,
      "activations/layer14_attention_weight_min": -33.976497650146484,
      "activations/layer15_attention_weight_max": 50.27079772949219,
      "activations/layer15_attention_weight_min": -30.937225341796875,
      "activations/layer16_attention_weight_max": 39.471466064453125,
      "activations/layer16_attention_weight_min": -27.589765548706055,
      "activations/layer17_attention_weight_max": 63.2377815246582,
      "activations/layer17_attention_weight_min": -41.72825622558594,
      "activations/layer18_attention_weight_max": 54.105430603027344,
      "activations/layer18_attention_weight_min": -35.51295852661133,
      "activations/layer19_attention_weight_max": 22.52005958557129,
      "activations/layer19_attention_weight_min": -22.951154708862305,
      "activations/layer1_attention_weight_max": 17.979351043701172,
      "activations/layer1_attention_weight_min": -15.36219310760498,
      "activations/layer20_attention_weight_max": 27.140037536621094,
      "activations/layer20_attention_weight_min": -20.113998413085938,
      "activations/layer21_attention_weight_max": 46.37451934814453,
      "activations/layer21_attention_weight_min": -29.237436294555664,
      "activations/layer22_attention_weight_max": 35.752288818359375,
      "activations/layer22_attention_weight_min": -22.223861694335938,
      "activations/layer23_attention_weight_max": 29.215011596679688,
      "activations/layer23_attention_weight_min": -16.477352142333984,
      "activations/layer2_attention_weight_max": 33.40412139892578,
      "activations/layer2_attention_weight_min": -31.17786979675293,
      "activations/layer3_attention_weight_max": 89.06832885742188,
      "activations/layer3_attention_weight_min": -89.41960144042969,
      "activations/layer4_attention_weight_max": 106.85029602050781,
      "activations/layer4_attention_weight_min": -101.90631866455078,
      "activations/layer5_attention_weight_max": 108.26040649414062,
      "activations/layer5_attention_weight_min": -100.42970275878906,
      "activations/layer6_attention_weight_max": 61.86511993408203,
      "activations/layer6_attention_weight_min": -55.94174575805664,
      "activations/layer7_attention_weight_max": 87.05502319335938,
      "activations/layer7_attention_weight_min": -77.01737213134766,
      "activations/layer8_attention_weight_max": 55.51631164550781,
      "activations/layer8_attention_weight_min": -50.75541687011719,
      "activations/layer9_attention_weight_max": 76.62200164794922,
      "activations/layer9_attention_weight_min": -59.2952880859375,
      "epoch": 4.83,
      "learning_rate": 0.00012003257575757575,
      "loss": 2.8617,
      "step": 83200
    },
    {
      "activations/layer0_attention_weight_max": 15.445440292358398,
      "activations/layer0_attention_weight_min": -14.842564582824707,
      "activations/layer10_attention_weight_max": 40.23796081542969,
      "activations/layer10_attention_weight_min": -33.037139892578125,
      "activations/layer11_attention_weight_max": 35.99677276611328,
      "activations/layer11_attention_weight_min": -28.11994743347168,
      "activations/layer12_attention_weight_max": 27.94945526123047,
      "activations/layer12_attention_weight_min": -27.529130935668945,
      "activations/layer13_attention_weight_max": 51.18864059448242,
      "activations/layer13_attention_weight_min": -35.53857421875,
      "activations/layer14_attention_weight_max": 49.287479400634766,
      "activations/layer14_attention_weight_min": -31.358829498291016,
      "activations/layer15_attention_weight_max": 44.69239044189453,
      "activations/layer15_attention_weight_min": -28.98595428466797,
      "activations/layer16_attention_weight_max": 43.2852897644043,
      "activations/layer16_attention_weight_min": -26.705575942993164,
      "activations/layer17_attention_weight_max": 62.43961715698242,
      "activations/layer17_attention_weight_min": -39.39530944824219,
      "activations/layer18_attention_weight_max": 53.09856414794922,
      "activations/layer18_attention_weight_min": -34.41907501220703,
      "activations/layer19_attention_weight_max": 31.45939064025879,
      "activations/layer19_attention_weight_min": -19.58882713317871,
      "activations/layer1_attention_weight_max": 17.688526153564453,
      "activations/layer1_attention_weight_min": -14.694947242736816,
      "activations/layer20_attention_weight_max": 24.37389373779297,
      "activations/layer20_attention_weight_min": -23.249914169311523,
      "activations/layer21_attention_weight_max": 39.76781463623047,
      "activations/layer21_attention_weight_min": -28.897262573242188,
      "activations/layer22_attention_weight_max": 38.56187057495117,
      "activations/layer22_attention_weight_min": -27.57712745666504,
      "activations/layer23_attention_weight_max": 32.31008529663086,
      "activations/layer23_attention_weight_min": -18.59552001953125,
      "activations/layer2_attention_weight_max": 29.462446212768555,
      "activations/layer2_attention_weight_min": -27.859830856323242,
      "activations/layer3_attention_weight_max": 81.50975036621094,
      "activations/layer3_attention_weight_min": -82.62812805175781,
      "activations/layer4_attention_weight_max": 97.16618347167969,
      "activations/layer4_attention_weight_min": -91.91195678710938,
      "activations/layer5_attention_weight_max": 100.11473846435547,
      "activations/layer5_attention_weight_min": -92.6353988647461,
      "activations/layer6_attention_weight_max": 57.75031661987305,
      "activations/layer6_attention_weight_min": -50.79214859008789,
      "activations/layer7_attention_weight_max": 69.76087188720703,
      "activations/layer7_attention_weight_min": -65.36870574951172,
      "activations/layer8_attention_weight_max": 44.358516693115234,
      "activations/layer8_attention_weight_min": -43.21837615966797,
      "activations/layer9_attention_weight_max": 53.9407844543457,
      "activations/layer9_attention_weight_min": -47.644935607910156,
      "epoch": 4.84,
      "learning_rate": 0.00012001363636363635,
      "loss": 2.8659,
      "step": 83250
    },
    {
      "activations/layer0_attention_weight_max": 14.923412322998047,
      "activations/layer0_attention_weight_min": -14.92153549194336,
      "activations/layer10_attention_weight_max": 38.89079284667969,
      "activations/layer10_attention_weight_min": -34.23695373535156,
      "activations/layer11_attention_weight_max": 28.485591888427734,
      "activations/layer11_attention_weight_min": -27.700969696044922,
      "activations/layer12_attention_weight_max": 33.713130950927734,
      "activations/layer12_attention_weight_min": -23.53984260559082,
      "activations/layer13_attention_weight_max": 54.5603141784668,
      "activations/layer13_attention_weight_min": -38.76078414916992,
      "activations/layer14_attention_weight_max": 39.859500885009766,
      "activations/layer14_attention_weight_min": -30.005268096923828,
      "activations/layer15_attention_weight_max": 40.512027740478516,
      "activations/layer15_attention_weight_min": -29.37409782409668,
      "activations/layer16_attention_weight_max": 32.99613952636719,
      "activations/layer16_attention_weight_min": -26.69917106628418,
      "activations/layer17_attention_weight_max": 53.96724319458008,
      "activations/layer17_attention_weight_min": -41.39284896850586,
      "activations/layer18_attention_weight_max": 49.6036262512207,
      "activations/layer18_attention_weight_min": -33.537776947021484,
      "activations/layer19_attention_weight_max": 24.93279266357422,
      "activations/layer19_attention_weight_min": -18.057222366333008,
      "activations/layer1_attention_weight_max": 17.383750915527344,
      "activations/layer1_attention_weight_min": -13.3678617477417,
      "activations/layer20_attention_weight_max": 19.862730026245117,
      "activations/layer20_attention_weight_min": -21.107973098754883,
      "activations/layer21_attention_weight_max": 42.03535842895508,
      "activations/layer21_attention_weight_min": -27.454971313476562,
      "activations/layer22_attention_weight_max": 34.68941879272461,
      "activations/layer22_attention_weight_min": -21.873693466186523,
      "activations/layer23_attention_weight_max": 26.005691528320312,
      "activations/layer23_attention_weight_min": -16.194665908813477,
      "activations/layer2_attention_weight_max": 29.361892700195312,
      "activations/layer2_attention_weight_min": -28.004344940185547,
      "activations/layer3_attention_weight_max": 77.07131958007812,
      "activations/layer3_attention_weight_min": -81.59771728515625,
      "activations/layer4_attention_weight_max": 95.96559143066406,
      "activations/layer4_attention_weight_min": -93.13237762451172,
      "activations/layer5_attention_weight_max": 99.69981384277344,
      "activations/layer5_attention_weight_min": -90.71583557128906,
      "activations/layer6_attention_weight_max": 55.849342346191406,
      "activations/layer6_attention_weight_min": -51.618778228759766,
      "activations/layer7_attention_weight_max": 70.84420013427734,
      "activations/layer7_attention_weight_min": -63.7996826171875,
      "activations/layer8_attention_weight_max": 40.04184341430664,
      "activations/layer8_attention_weight_min": -41.06119918823242,
      "activations/layer9_attention_weight_max": 47.543033599853516,
      "activations/layer9_attention_weight_min": -45.32648468017578,
      "epoch": 4.84,
      "learning_rate": 0.00011999469696969697,
      "loss": 2.8318,
      "step": 83300
    },
    {
      "activations/layer0_attention_weight_max": 14.348461151123047,
      "activations/layer0_attention_weight_min": -14.856671333312988,
      "activations/layer10_attention_weight_max": 38.10649871826172,
      "activations/layer10_attention_weight_min": -34.062171936035156,
      "activations/layer11_attention_weight_max": 31.757488250732422,
      "activations/layer11_attention_weight_min": -25.75739860534668,
      "activations/layer12_attention_weight_max": 23.953367233276367,
      "activations/layer12_attention_weight_min": -24.153852462768555,
      "activations/layer13_attention_weight_max": 51.25316619873047,
      "activations/layer13_attention_weight_min": -34.12678527832031,
      "activations/layer14_attention_weight_max": 42.87560272216797,
      "activations/layer14_attention_weight_min": -30.850950241088867,
      "activations/layer15_attention_weight_max": 41.9736328125,
      "activations/layer15_attention_weight_min": -28.955657958984375,
      "activations/layer16_attention_weight_max": 33.25019073486328,
      "activations/layer16_attention_weight_min": -26.49799156188965,
      "activations/layer17_attention_weight_max": 59.99896240234375,
      "activations/layer17_attention_weight_min": -39.3194465637207,
      "activations/layer18_attention_weight_max": 52.87548828125,
      "activations/layer18_attention_weight_min": -33.90483856201172,
      "activations/layer19_attention_weight_max": 23.75499153137207,
      "activations/layer19_attention_weight_min": -18.24600601196289,
      "activations/layer1_attention_weight_max": 17.24456787109375,
      "activations/layer1_attention_weight_min": -15.69151496887207,
      "activations/layer20_attention_weight_max": 22.36823272705078,
      "activations/layer20_attention_weight_min": -19.965253829956055,
      "activations/layer21_attention_weight_max": 37.07692337036133,
      "activations/layer21_attention_weight_min": -26.396160125732422,
      "activations/layer22_attention_weight_max": 34.46570587158203,
      "activations/layer22_attention_weight_min": -23.15615463256836,
      "activations/layer23_attention_weight_max": 26.592838287353516,
      "activations/layer23_attention_weight_min": -18.781234741210938,
      "activations/layer2_attention_weight_max": 30.376821517944336,
      "activations/layer2_attention_weight_min": -29.577938079833984,
      "activations/layer3_attention_weight_max": 79.91522216796875,
      "activations/layer3_attention_weight_min": -78.73216247558594,
      "activations/layer4_attention_weight_max": 101.82042694091797,
      "activations/layer4_attention_weight_min": -93.53030395507812,
      "activations/layer5_attention_weight_max": 105.8304443359375,
      "activations/layer5_attention_weight_min": -89.08135223388672,
      "activations/layer6_attention_weight_max": 58.37314987182617,
      "activations/layer6_attention_weight_min": -50.91549301147461,
      "activations/layer7_attention_weight_max": 68.47588348388672,
      "activations/layer7_attention_weight_min": -65.83466339111328,
      "activations/layer8_attention_weight_max": 41.50185012817383,
      "activations/layer8_attention_weight_min": -40.360374450683594,
      "activations/layer9_attention_weight_max": 49.12208938598633,
      "activations/layer9_attention_weight_min": -46.931575775146484,
      "epoch": 4.84,
      "learning_rate": 0.00011997575757575755,
      "loss": 2.857,
      "step": 83350
    },
    {
      "activations/layer0_attention_weight_max": 14.99451732635498,
      "activations/layer0_attention_weight_min": -15.17432975769043,
      "activations/layer10_attention_weight_max": 39.82976150512695,
      "activations/layer10_attention_weight_min": -33.023406982421875,
      "activations/layer11_attention_weight_max": 32.079254150390625,
      "activations/layer11_attention_weight_min": -28.218420028686523,
      "activations/layer12_attention_weight_max": 25.85157012939453,
      "activations/layer12_attention_weight_min": -23.57096290588379,
      "activations/layer13_attention_weight_max": 52.789302825927734,
      "activations/layer13_attention_weight_min": -35.89384078979492,
      "activations/layer14_attention_weight_max": 55.38486862182617,
      "activations/layer14_attention_weight_min": -30.907711029052734,
      "activations/layer15_attention_weight_max": 46.58512878417969,
      "activations/layer15_attention_weight_min": -29.734262466430664,
      "activations/layer16_attention_weight_max": 38.6076545715332,
      "activations/layer16_attention_weight_min": -27.51451301574707,
      "activations/layer17_attention_weight_max": 57.03850555419922,
      "activations/layer17_attention_weight_min": -38.242523193359375,
      "activations/layer18_attention_weight_max": 48.81426239013672,
      "activations/layer18_attention_weight_min": -34.14924240112305,
      "activations/layer19_attention_weight_max": 23.966577529907227,
      "activations/layer19_attention_weight_min": -19.538747787475586,
      "activations/layer1_attention_weight_max": 16.827716827392578,
      "activations/layer1_attention_weight_min": -14.66251277923584,
      "activations/layer20_attention_weight_max": 19.777624130249023,
      "activations/layer20_attention_weight_min": -21.307580947875977,
      "activations/layer21_attention_weight_max": 38.889896392822266,
      "activations/layer21_attention_weight_min": -26.568584442138672,
      "activations/layer22_attention_weight_max": 32.39860534667969,
      "activations/layer22_attention_weight_min": -23.079580307006836,
      "activations/layer23_attention_weight_max": 27.07709503173828,
      "activations/layer23_attention_weight_min": -22.461164474487305,
      "activations/layer2_attention_weight_max": 31.016157150268555,
      "activations/layer2_attention_weight_min": -27.146846771240234,
      "activations/layer3_attention_weight_max": 79.91400909423828,
      "activations/layer3_attention_weight_min": -79.785400390625,
      "activations/layer4_attention_weight_max": 100.19387817382812,
      "activations/layer4_attention_weight_min": -91.25415802001953,
      "activations/layer5_attention_weight_max": 102.92369079589844,
      "activations/layer5_attention_weight_min": -88.5262680053711,
      "activations/layer6_attention_weight_max": 55.26530838012695,
      "activations/layer6_attention_weight_min": -51.22488021850586,
      "activations/layer7_attention_weight_max": 70.9508285522461,
      "activations/layer7_attention_weight_min": -70.06411743164062,
      "activations/layer8_attention_weight_max": 42.95856857299805,
      "activations/layer8_attention_weight_min": -40.38533401489258,
      "activations/layer9_attention_weight_max": 54.805763244628906,
      "activations/layer9_attention_weight_min": -46.02162170410156,
      "epoch": 4.85,
      "learning_rate": 0.00011995681818181817,
      "loss": 2.8473,
      "step": 83400
    },
    {
      "activations/layer0_attention_weight_max": 14.764128684997559,
      "activations/layer0_attention_weight_min": -14.959710121154785,
      "activations/layer10_attention_weight_max": 35.04939270019531,
      "activations/layer10_attention_weight_min": -34.506813049316406,
      "activations/layer11_attention_weight_max": 27.950054168701172,
      "activations/layer11_attention_weight_min": -27.224224090576172,
      "activations/layer12_attention_weight_max": 19.938302993774414,
      "activations/layer12_attention_weight_min": -23.825103759765625,
      "activations/layer13_attention_weight_max": 46.79072189331055,
      "activations/layer13_attention_weight_min": -37.5661735534668,
      "activations/layer14_attention_weight_max": 38.46493148803711,
      "activations/layer14_attention_weight_min": -27.50639533996582,
      "activations/layer15_attention_weight_max": 40.81971740722656,
      "activations/layer15_attention_weight_min": -29.591617584228516,
      "activations/layer16_attention_weight_max": 37.31100082397461,
      "activations/layer16_attention_weight_min": -25.402795791625977,
      "activations/layer17_attention_weight_max": 53.200279235839844,
      "activations/layer17_attention_weight_min": -37.60497283935547,
      "activations/layer18_attention_weight_max": 48.48074722290039,
      "activations/layer18_attention_weight_min": -34.34114074707031,
      "activations/layer19_attention_weight_max": 23.12529754638672,
      "activations/layer19_attention_weight_min": -18.560325622558594,
      "activations/layer1_attention_weight_max": 17.178796768188477,
      "activations/layer1_attention_weight_min": -16.0203800201416,
      "activations/layer20_attention_weight_max": 23.843387603759766,
      "activations/layer20_attention_weight_min": -21.21198844909668,
      "activations/layer21_attention_weight_max": 36.750732421875,
      "activations/layer21_attention_weight_min": -26.33608055114746,
      "activations/layer22_attention_weight_max": 33.572628021240234,
      "activations/layer22_attention_weight_min": -23.34685707092285,
      "activations/layer23_attention_weight_max": 26.960968017578125,
      "activations/layer23_attention_weight_min": -17.752765655517578,
      "activations/layer2_attention_weight_max": 32.9178352355957,
      "activations/layer2_attention_weight_min": -30.086036682128906,
      "activations/layer3_attention_weight_max": 86.4020767211914,
      "activations/layer3_attention_weight_min": -90.58819580078125,
      "activations/layer4_attention_weight_max": 105.5804672241211,
      "activations/layer4_attention_weight_min": -101.1108627319336,
      "activations/layer5_attention_weight_max": 101.86091613769531,
      "activations/layer5_attention_weight_min": -95.70088195800781,
      "activations/layer6_attention_weight_max": 56.72425842285156,
      "activations/layer6_attention_weight_min": -52.6392707824707,
      "activations/layer7_attention_weight_max": 71.27408599853516,
      "activations/layer7_attention_weight_min": -62.2573127746582,
      "activations/layer8_attention_weight_max": 39.562522888183594,
      "activations/layer8_attention_weight_min": -39.04066467285156,
      "activations/layer9_attention_weight_max": 44.573490142822266,
      "activations/layer9_attention_weight_min": -47.12308883666992,
      "epoch": 4.85,
      "learning_rate": 0.00011993787878787878,
      "loss": 2.8469,
      "step": 83450
    },
    {
      "activations/layer0_attention_weight_max": 13.724010467529297,
      "activations/layer0_attention_weight_min": -14.89068603515625,
      "activations/layer10_attention_weight_max": 48.42424011230469,
      "activations/layer10_attention_weight_min": -40.385284423828125,
      "activations/layer11_attention_weight_max": 40.889739990234375,
      "activations/layer11_attention_weight_min": -30.097923278808594,
      "activations/layer12_attention_weight_max": 24.77880859375,
      "activations/layer12_attention_weight_min": -26.640050888061523,
      "activations/layer13_attention_weight_max": 71.1381607055664,
      "activations/layer13_attention_weight_min": -39.06971740722656,
      "activations/layer14_attention_weight_max": 105.17901611328125,
      "activations/layer14_attention_weight_min": -49.99427032470703,
      "activations/layer15_attention_weight_max": 72.79029083251953,
      "activations/layer15_attention_weight_min": -35.94166946411133,
      "activations/layer16_attention_weight_max": 39.543617248535156,
      "activations/layer16_attention_weight_min": -25.507539749145508,
      "activations/layer17_attention_weight_max": 61.31351089477539,
      "activations/layer17_attention_weight_min": -40.702327728271484,
      "activations/layer18_attention_weight_max": 56.57875442504883,
      "activations/layer18_attention_weight_min": -34.98073959350586,
      "activations/layer19_attention_weight_max": 25.072622299194336,
      "activations/layer19_attention_weight_min": -22.847522735595703,
      "activations/layer1_attention_weight_max": 17.037158966064453,
      "activations/layer1_attention_weight_min": -13.844281196594238,
      "activations/layer20_attention_weight_max": 29.50456428527832,
      "activations/layer20_attention_weight_min": -21.609966278076172,
      "activations/layer21_attention_weight_max": 52.86396026611328,
      "activations/layer21_attention_weight_min": -25.62471580505371,
      "activations/layer22_attention_weight_max": 38.27959060668945,
      "activations/layer22_attention_weight_min": -22.30794334411621,
      "activations/layer23_attention_weight_max": 28.692426681518555,
      "activations/layer23_attention_weight_min": -15.891664505004883,
      "activations/layer2_attention_weight_max": 33.56686019897461,
      "activations/layer2_attention_weight_min": -30.78053855895996,
      "activations/layer3_attention_weight_max": 92.00418853759766,
      "activations/layer3_attention_weight_min": -92.80435180664062,
      "activations/layer4_attention_weight_max": 120.087158203125,
      "activations/layer4_attention_weight_min": -108.94876861572266,
      "activations/layer5_attention_weight_max": 124.60160827636719,
      "activations/layer5_attention_weight_min": -105.02223205566406,
      "activations/layer6_attention_weight_max": 67.25302124023438,
      "activations/layer6_attention_weight_min": -58.9244499206543,
      "activations/layer7_attention_weight_max": 86.58792877197266,
      "activations/layer7_attention_weight_min": -69.29500579833984,
      "activations/layer8_attention_weight_max": 53.59949493408203,
      "activations/layer8_attention_weight_min": -44.74767303466797,
      "activations/layer9_attention_weight_max": 70.65690612792969,
      "activations/layer9_attention_weight_min": -56.28424072265625,
      "epoch": 4.85,
      "learning_rate": 0.00011991893939393937,
      "loss": 2.8587,
      "step": 83500
    },
    {
      "activations/layer0_attention_weight_max": 15.368782997131348,
      "activations/layer0_attention_weight_min": -14.568464279174805,
      "activations/layer10_attention_weight_max": 40.177772521972656,
      "activations/layer10_attention_weight_min": -34.53818893432617,
      "activations/layer11_attention_weight_max": 31.045658111572266,
      "activations/layer11_attention_weight_min": -28.145294189453125,
      "activations/layer12_attention_weight_max": 30.840930938720703,
      "activations/layer12_attention_weight_min": -25.06078338623047,
      "activations/layer13_attention_weight_max": 49.713321685791016,
      "activations/layer13_attention_weight_min": -36.58985137939453,
      "activations/layer14_attention_weight_max": 45.05643844604492,
      "activations/layer14_attention_weight_min": -32.65742874145508,
      "activations/layer15_attention_weight_max": 40.57600784301758,
      "activations/layer15_attention_weight_min": -31.201345443725586,
      "activations/layer16_attention_weight_max": 36.33961868286133,
      "activations/layer16_attention_weight_min": -26.604005813598633,
      "activations/layer17_attention_weight_max": 55.63943862915039,
      "activations/layer17_attention_weight_min": -39.630611419677734,
      "activations/layer18_attention_weight_max": 48.50897979736328,
      "activations/layer18_attention_weight_min": -33.31880187988281,
      "activations/layer19_attention_weight_max": 21.714662551879883,
      "activations/layer19_attention_weight_min": -18.588607788085938,
      "activations/layer1_attention_weight_max": 17.64081382751465,
      "activations/layer1_attention_weight_min": -14.747130393981934,
      "activations/layer20_attention_weight_max": 21.469486236572266,
      "activations/layer20_attention_weight_min": -21.453231811523438,
      "activations/layer21_attention_weight_max": 37.28731155395508,
      "activations/layer21_attention_weight_min": -26.538406372070312,
      "activations/layer22_attention_weight_max": 36.4315185546875,
      "activations/layer22_attention_weight_min": -24.76907730102539,
      "activations/layer23_attention_weight_max": 26.89242172241211,
      "activations/layer23_attention_weight_min": -16.777292251586914,
      "activations/layer2_attention_weight_max": 30.050825119018555,
      "activations/layer2_attention_weight_min": -28.459972381591797,
      "activations/layer3_attention_weight_max": 82.70854187011719,
      "activations/layer3_attention_weight_min": -83.9589614868164,
      "activations/layer4_attention_weight_max": 102.17623138427734,
      "activations/layer4_attention_weight_min": -96.11235809326172,
      "activations/layer5_attention_weight_max": 107.64956665039062,
      "activations/layer5_attention_weight_min": -93.64202880859375,
      "activations/layer6_attention_weight_max": 59.42482376098633,
      "activations/layer6_attention_weight_min": -51.44148635864258,
      "activations/layer7_attention_weight_max": 66.48143768310547,
      "activations/layer7_attention_weight_min": -64.31856536865234,
      "activations/layer8_attention_weight_max": 43.170066833496094,
      "activations/layer8_attention_weight_min": -39.140708923339844,
      "activations/layer9_attention_weight_max": 49.155391693115234,
      "activations/layer9_attention_weight_min": -45.590328216552734,
      "epoch": 4.85,
      "learning_rate": 0.00011989999999999999,
      "loss": 2.8669,
      "step": 83550
    },
    {
      "activations/layer0_attention_weight_max": 13.616941452026367,
      "activations/layer0_attention_weight_min": -15.05084228515625,
      "activations/layer10_attention_weight_max": 44.47405242919922,
      "activations/layer10_attention_weight_min": -37.3294563293457,
      "activations/layer11_attention_weight_max": 35.337371826171875,
      "activations/layer11_attention_weight_min": -27.173147201538086,
      "activations/layer12_attention_weight_max": 20.649328231811523,
      "activations/layer12_attention_weight_min": -28.417896270751953,
      "activations/layer13_attention_weight_max": 45.30332946777344,
      "activations/layer13_attention_weight_min": -33.113006591796875,
      "activations/layer14_attention_weight_max": 54.29369354248047,
      "activations/layer14_attention_weight_min": -34.97610855102539,
      "activations/layer15_attention_weight_max": 39.88905715942383,
      "activations/layer15_attention_weight_min": -33.50941848754883,
      "activations/layer16_attention_weight_max": 30.945234298706055,
      "activations/layer16_attention_weight_min": -25.848098754882812,
      "activations/layer17_attention_weight_max": 53.88301467895508,
      "activations/layer17_attention_weight_min": -37.22298049926758,
      "activations/layer18_attention_weight_max": 43.273162841796875,
      "activations/layer18_attention_weight_min": -32.778194427490234,
      "activations/layer19_attention_weight_max": 20.898792266845703,
      "activations/layer19_attention_weight_min": -22.9521427154541,
      "activations/layer1_attention_weight_max": 18.696693420410156,
      "activations/layer1_attention_weight_min": -14.540196418762207,
      "activations/layer20_attention_weight_max": 20.512731552124023,
      "activations/layer20_attention_weight_min": -20.827383041381836,
      "activations/layer21_attention_weight_max": 34.92584991455078,
      "activations/layer21_attention_weight_min": -28.697813034057617,
      "activations/layer22_attention_weight_max": 30.260562896728516,
      "activations/layer22_attention_weight_min": -23.72372055053711,
      "activations/layer23_attention_weight_max": 27.00543975830078,
      "activations/layer23_attention_weight_min": -22.58658218383789,
      "activations/layer2_attention_weight_max": 31.718242645263672,
      "activations/layer2_attention_weight_min": -30.171188354492188,
      "activations/layer3_attention_weight_max": 85.25791931152344,
      "activations/layer3_attention_weight_min": -86.75930786132812,
      "activations/layer4_attention_weight_max": 108.91676330566406,
      "activations/layer4_attention_weight_min": -100.43402862548828,
      "activations/layer5_attention_weight_max": 111.44149780273438,
      "activations/layer5_attention_weight_min": -101.99647521972656,
      "activations/layer6_attention_weight_max": 61.041263580322266,
      "activations/layer6_attention_weight_min": -58.15446090698242,
      "activations/layer7_attention_weight_max": 87.91716766357422,
      "activations/layer7_attention_weight_min": -67.7577896118164,
      "activations/layer8_attention_weight_max": 49.947818756103516,
      "activations/layer8_attention_weight_min": -41.81636428833008,
      "activations/layer9_attention_weight_max": 72.04109191894531,
      "activations/layer9_attention_weight_min": -52.319923400878906,
      "epoch": 4.86,
      "learning_rate": 0.0001198810606060606,
      "loss": 2.8504,
      "step": 83600
    },
    {
      "activations/layer0_attention_weight_max": 13.962152481079102,
      "activations/layer0_attention_weight_min": -14.793081283569336,
      "activations/layer10_attention_weight_max": 36.392024993896484,
      "activations/layer10_attention_weight_min": -33.65720748901367,
      "activations/layer11_attention_weight_max": 29.38860321044922,
      "activations/layer11_attention_weight_min": -27.09352684020996,
      "activations/layer12_attention_weight_max": 21.949222564697266,
      "activations/layer12_attention_weight_min": -25.285837173461914,
      "activations/layer13_attention_weight_max": 50.06929397583008,
      "activations/layer13_attention_weight_min": -38.450531005859375,
      "activations/layer14_attention_weight_max": 55.426273345947266,
      "activations/layer14_attention_weight_min": -36.21236038208008,
      "activations/layer15_attention_weight_max": 43.60496139526367,
      "activations/layer15_attention_weight_min": -30.21222496032715,
      "activations/layer16_attention_weight_max": 35.19142150878906,
      "activations/layer16_attention_weight_min": -27.715837478637695,
      "activations/layer17_attention_weight_max": 60.02201461791992,
      "activations/layer17_attention_weight_min": -38.862281799316406,
      "activations/layer18_attention_weight_max": 52.348487854003906,
      "activations/layer18_attention_weight_min": -34.8015251159668,
      "activations/layer19_attention_weight_max": 27.038650512695312,
      "activations/layer19_attention_weight_min": -19.617284774780273,
      "activations/layer1_attention_weight_max": 17.920120239257812,
      "activations/layer1_attention_weight_min": -16.642356872558594,
      "activations/layer20_attention_weight_max": 23.601634979248047,
      "activations/layer20_attention_weight_min": -21.593935012817383,
      "activations/layer21_attention_weight_max": 44.91436767578125,
      "activations/layer21_attention_weight_min": -27.061702728271484,
      "activations/layer22_attention_weight_max": 32.714107513427734,
      "activations/layer22_attention_weight_min": -25.37173080444336,
      "activations/layer23_attention_weight_max": 33.06829071044922,
      "activations/layer23_attention_weight_min": -19.16809844970703,
      "activations/layer2_attention_weight_max": 30.766010284423828,
      "activations/layer2_attention_weight_min": -28.42641258239746,
      "activations/layer3_attention_weight_max": 79.81271362304688,
      "activations/layer3_attention_weight_min": -80.66698455810547,
      "activations/layer4_attention_weight_max": 99.29203033447266,
      "activations/layer4_attention_weight_min": -91.81321716308594,
      "activations/layer5_attention_weight_max": 101.21334075927734,
      "activations/layer5_attention_weight_min": -91.71874237060547,
      "activations/layer6_attention_weight_max": 56.81901931762695,
      "activations/layer6_attention_weight_min": -51.227882385253906,
      "activations/layer7_attention_weight_max": 67.1530990600586,
      "activations/layer7_attention_weight_min": -64.85770416259766,
      "activations/layer8_attention_weight_max": 42.59558868408203,
      "activations/layer8_attention_weight_min": -39.6137809753418,
      "activations/layer9_attention_weight_max": 49.033138275146484,
      "activations/layer9_attention_weight_min": -43.75666046142578,
      "epoch": 4.86,
      "learning_rate": 0.0001198621212121212,
      "loss": 2.8485,
      "step": 83650
    },
    {
      "activations/layer0_attention_weight_max": 14.774969100952148,
      "activations/layer0_attention_weight_min": -14.328878402709961,
      "activations/layer10_attention_weight_max": 37.8339729309082,
      "activations/layer10_attention_weight_min": -33.75801086425781,
      "activations/layer11_attention_weight_max": 29.354312896728516,
      "activations/layer11_attention_weight_min": -26.895553588867188,
      "activations/layer12_attention_weight_max": 24.898780822753906,
      "activations/layer12_attention_weight_min": -23.25493049621582,
      "activations/layer13_attention_weight_max": 47.75405502319336,
      "activations/layer13_attention_weight_min": -34.25181579589844,
      "activations/layer14_attention_weight_max": 52.063934326171875,
      "activations/layer14_attention_weight_min": -29.656600952148438,
      "activations/layer15_attention_weight_max": 40.82029342651367,
      "activations/layer15_attention_weight_min": -29.9442195892334,
      "activations/layer16_attention_weight_max": 35.23954391479492,
      "activations/layer16_attention_weight_min": -26.474903106689453,
      "activations/layer17_attention_weight_max": 62.32242965698242,
      "activations/layer17_attention_weight_min": -39.18425750732422,
      "activations/layer18_attention_weight_max": 50.00408172607422,
      "activations/layer18_attention_weight_min": -33.614891052246094,
      "activations/layer19_attention_weight_max": 21.55848503112793,
      "activations/layer19_attention_weight_min": -19.904260635375977,
      "activations/layer1_attention_weight_max": 16.919357299804688,
      "activations/layer1_attention_weight_min": -15.468216896057129,
      "activations/layer20_attention_weight_max": 21.703935623168945,
      "activations/layer20_attention_weight_min": -20.69684600830078,
      "activations/layer21_attention_weight_max": 45.364234924316406,
      "activations/layer21_attention_weight_min": -25.518056869506836,
      "activations/layer22_attention_weight_max": 35.0726318359375,
      "activations/layer22_attention_weight_min": -25.35378074645996,
      "activations/layer23_attention_weight_max": 27.92843246459961,
      "activations/layer23_attention_weight_min": -18.416887283325195,
      "activations/layer2_attention_weight_max": 29.469738006591797,
      "activations/layer2_attention_weight_min": -28.830516815185547,
      "activations/layer3_attention_weight_max": 83.44749450683594,
      "activations/layer3_attention_weight_min": -84.97366333007812,
      "activations/layer4_attention_weight_max": 105.16773986816406,
      "activations/layer4_attention_weight_min": -92.31038665771484,
      "activations/layer5_attention_weight_max": 107.97605895996094,
      "activations/layer5_attention_weight_min": -94.37556457519531,
      "activations/layer6_attention_weight_max": 61.31315612792969,
      "activations/layer6_attention_weight_min": -53.0788688659668,
      "activations/layer7_attention_weight_max": 70.43968200683594,
      "activations/layer7_attention_weight_min": -64.31723022460938,
      "activations/layer8_attention_weight_max": 40.96510314941406,
      "activations/layer8_attention_weight_min": -42.74456024169922,
      "activations/layer9_attention_weight_max": 45.94916915893555,
      "activations/layer9_attention_weight_min": -44.98762130737305,
      "epoch": 4.86,
      "learning_rate": 0.0001198431818181818,
      "loss": 2.8763,
      "step": 83700
    },
    {
      "activations/layer0_attention_weight_max": 14.768447875976562,
      "activations/layer0_attention_weight_min": -14.815760612487793,
      "activations/layer10_attention_weight_max": 38.57102966308594,
      "activations/layer10_attention_weight_min": -35.94046401977539,
      "activations/layer11_attention_weight_max": 33.05180358886719,
      "activations/layer11_attention_weight_min": -26.54482078552246,
      "activations/layer12_attention_weight_max": 23.705976486206055,
      "activations/layer12_attention_weight_min": -26.3597354888916,
      "activations/layer13_attention_weight_max": 59.429283142089844,
      "activations/layer13_attention_weight_min": -38.961082458496094,
      "activations/layer14_attention_weight_max": 60.37196350097656,
      "activations/layer14_attention_weight_min": -32.08464431762695,
      "activations/layer15_attention_weight_max": 46.93818664550781,
      "activations/layer15_attention_weight_min": -30.014060974121094,
      "activations/layer16_attention_weight_max": 38.584781646728516,
      "activations/layer16_attention_weight_min": -26.305463790893555,
      "activations/layer17_attention_weight_max": 67.9598159790039,
      "activations/layer17_attention_weight_min": -39.3480339050293,
      "activations/layer18_attention_weight_max": 50.12297439575195,
      "activations/layer18_attention_weight_min": -32.45253372192383,
      "activations/layer19_attention_weight_max": 25.82928466796875,
      "activations/layer19_attention_weight_min": -19.42861557006836,
      "activations/layer1_attention_weight_max": 17.282726287841797,
      "activations/layer1_attention_weight_min": -15.41387939453125,
      "activations/layer20_attention_weight_max": 26.14366912841797,
      "activations/layer20_attention_weight_min": -20.242910385131836,
      "activations/layer21_attention_weight_max": 46.22908401489258,
      "activations/layer21_attention_weight_min": -28.956741333007812,
      "activations/layer22_attention_weight_max": 39.69685363769531,
      "activations/layer22_attention_weight_min": -24.919973373413086,
      "activations/layer23_attention_weight_max": 31.414215087890625,
      "activations/layer23_attention_weight_min": -18.451210021972656,
      "activations/layer2_attention_weight_max": 30.888906478881836,
      "activations/layer2_attention_weight_min": -28.02635955810547,
      "activations/layer3_attention_weight_max": 89.867919921875,
      "activations/layer3_attention_weight_min": -86.45227813720703,
      "activations/layer4_attention_weight_max": 107.8031234741211,
      "activations/layer4_attention_weight_min": -107.235595703125,
      "activations/layer5_attention_weight_max": 105.55029296875,
      "activations/layer5_attention_weight_min": -99.98278045654297,
      "activations/layer6_attention_weight_max": 57.840370178222656,
      "activations/layer6_attention_weight_min": -54.279396057128906,
      "activations/layer7_attention_weight_max": 74.29149627685547,
      "activations/layer7_attention_weight_min": -72.25782012939453,
      "activations/layer8_attention_weight_max": 49.48531723022461,
      "activations/layer8_attention_weight_min": -44.794921875,
      "activations/layer9_attention_weight_max": 62.815467834472656,
      "activations/layer9_attention_weight_min": -53.183197021484375,
      "epoch": 4.87,
      "learning_rate": 0.0001198242424242424,
      "loss": 2.8503,
      "step": 83750
    },
    {
      "activations/layer0_attention_weight_max": 15.875514030456543,
      "activations/layer0_attention_weight_min": -14.325606346130371,
      "activations/layer10_attention_weight_max": 39.47588348388672,
      "activations/layer10_attention_weight_min": -34.09718322753906,
      "activations/layer11_attention_weight_max": 31.47115135192871,
      "activations/layer11_attention_weight_min": -28.88707733154297,
      "activations/layer12_attention_weight_max": 24.339115142822266,
      "activations/layer12_attention_weight_min": -25.43275260925293,
      "activations/layer13_attention_weight_max": 70.98986053466797,
      "activations/layer13_attention_weight_min": -44.41392135620117,
      "activations/layer14_attention_weight_max": 68.41832733154297,
      "activations/layer14_attention_weight_min": -42.25031661987305,
      "activations/layer15_attention_weight_max": 48.260833740234375,
      "activations/layer15_attention_weight_min": -32.14521789550781,
      "activations/layer16_attention_weight_max": 37.48640060424805,
      "activations/layer16_attention_weight_min": -26.55358123779297,
      "activations/layer17_attention_weight_max": 58.81949996948242,
      "activations/layer17_attention_weight_min": -39.44194030761719,
      "activations/layer18_attention_weight_max": 50.666072845458984,
      "activations/layer18_attention_weight_min": -34.74462890625,
      "activations/layer19_attention_weight_max": 24.55658531188965,
      "activations/layer19_attention_weight_min": -18.190670013427734,
      "activations/layer1_attention_weight_max": 18.466081619262695,
      "activations/layer1_attention_weight_min": -15.149479866027832,
      "activations/layer20_attention_weight_max": 24.041589736938477,
      "activations/layer20_attention_weight_min": -20.83477020263672,
      "activations/layer21_attention_weight_max": 43.408721923828125,
      "activations/layer21_attention_weight_min": -26.121959686279297,
      "activations/layer22_attention_weight_max": 34.77021789550781,
      "activations/layer22_attention_weight_min": -23.730419158935547,
      "activations/layer23_attention_weight_max": 30.478803634643555,
      "activations/layer23_attention_weight_min": -18.34537124633789,
      "activations/layer2_attention_weight_max": 30.32022476196289,
      "activations/layer2_attention_weight_min": -28.037967681884766,
      "activations/layer3_attention_weight_max": 81.74974822998047,
      "activations/layer3_attention_weight_min": -82.44158172607422,
      "activations/layer4_attention_weight_max": 107.20111083984375,
      "activations/layer4_attention_weight_min": -99.23064422607422,
      "activations/layer5_attention_weight_max": 111.89388275146484,
      "activations/layer5_attention_weight_min": -99.20068359375,
      "activations/layer6_attention_weight_max": 59.78443145751953,
      "activations/layer6_attention_weight_min": -56.791343688964844,
      "activations/layer7_attention_weight_max": 86.4482421875,
      "activations/layer7_attention_weight_min": -68.1546859741211,
      "activations/layer8_attention_weight_max": 48.96458435058594,
      "activations/layer8_attention_weight_min": -42.17580032348633,
      "activations/layer9_attention_weight_max": 66.7072982788086,
      "activations/layer9_attention_weight_min": -48.317745208740234,
      "epoch": 4.87,
      "learning_rate": 0.00011980530303030302,
      "loss": 2.8557,
      "step": 83800
    },
    {
      "activations/layer0_attention_weight_max": 13.842555046081543,
      "activations/layer0_attention_weight_min": -15.40119743347168,
      "activations/layer10_attention_weight_max": 38.625614166259766,
      "activations/layer10_attention_weight_min": -36.665321350097656,
      "activations/layer11_attention_weight_max": 29.05451774597168,
      "activations/layer11_attention_weight_min": -27.748706817626953,
      "activations/layer12_attention_weight_max": 19.716867446899414,
      "activations/layer12_attention_weight_min": -24.24131202697754,
      "activations/layer13_attention_weight_max": 49.19693374633789,
      "activations/layer13_attention_weight_min": -34.868873596191406,
      "activations/layer14_attention_weight_max": 42.56227493286133,
      "activations/layer14_attention_weight_min": -31.262361526489258,
      "activations/layer15_attention_weight_max": 37.01291275024414,
      "activations/layer15_attention_weight_min": -28.85625648498535,
      "activations/layer16_attention_weight_max": 34.63808059692383,
      "activations/layer16_attention_weight_min": -25.24768829345703,
      "activations/layer17_attention_weight_max": 61.86360549926758,
      "activations/layer17_attention_weight_min": -38.08879470825195,
      "activations/layer18_attention_weight_max": 50.152381896972656,
      "activations/layer18_attention_weight_min": -33.24667739868164,
      "activations/layer19_attention_weight_max": 21.515541076660156,
      "activations/layer19_attention_weight_min": -19.313541412353516,
      "activations/layer1_attention_weight_max": 17.68074607849121,
      "activations/layer1_attention_weight_min": -15.592058181762695,
      "activations/layer20_attention_weight_max": 20.294605255126953,
      "activations/layer20_attention_weight_min": -20.04115104675293,
      "activations/layer21_attention_weight_max": 39.218955993652344,
      "activations/layer21_attention_weight_min": -25.295377731323242,
      "activations/layer22_attention_weight_max": 33.75045394897461,
      "activations/layer22_attention_weight_min": -23.116493225097656,
      "activations/layer23_attention_weight_max": 27.265918731689453,
      "activations/layer23_attention_weight_min": -19.453105926513672,
      "activations/layer2_attention_weight_max": 31.293991088867188,
      "activations/layer2_attention_weight_min": -28.968215942382812,
      "activations/layer3_attention_weight_max": 81.54959869384766,
      "activations/layer3_attention_weight_min": -82.06511688232422,
      "activations/layer4_attention_weight_max": 104.0515365600586,
      "activations/layer4_attention_weight_min": -96.78312683105469,
      "activations/layer5_attention_weight_max": 106.24140930175781,
      "activations/layer5_attention_weight_min": -99.13874816894531,
      "activations/layer6_attention_weight_max": 59.5211181640625,
      "activations/layer6_attention_weight_min": -53.98853302001953,
      "activations/layer7_attention_weight_max": 69.31596374511719,
      "activations/layer7_attention_weight_min": -66.58950805664062,
      "activations/layer8_attention_weight_max": 46.291080474853516,
      "activations/layer8_attention_weight_min": -44.83198165893555,
      "activations/layer9_attention_weight_max": 51.72160720825195,
      "activations/layer9_attention_weight_min": -46.69623565673828,
      "epoch": 4.87,
      "learning_rate": 0.00011978636363636364,
      "loss": 2.8641,
      "step": 83850
    },
    {
      "activations/layer0_attention_weight_max": 14.160633087158203,
      "activations/layer0_attention_weight_min": -14.94344425201416,
      "activations/layer10_attention_weight_max": 42.16591262817383,
      "activations/layer10_attention_weight_min": -35.53900146484375,
      "activations/layer11_attention_weight_max": 33.464088439941406,
      "activations/layer11_attention_weight_min": -28.456928253173828,
      "activations/layer12_attention_weight_max": 24.771804809570312,
      "activations/layer12_attention_weight_min": -25.330900192260742,
      "activations/layer13_attention_weight_max": 51.07805633544922,
      "activations/layer13_attention_weight_min": -34.406707763671875,
      "activations/layer14_attention_weight_max": 48.814476013183594,
      "activations/layer14_attention_weight_min": -29.325756072998047,
      "activations/layer15_attention_weight_max": 45.56587219238281,
      "activations/layer15_attention_weight_min": -30.365276336669922,
      "activations/layer16_attention_weight_max": 37.20785903930664,
      "activations/layer16_attention_weight_min": -29.148527145385742,
      "activations/layer17_attention_weight_max": 60.46088409423828,
      "activations/layer17_attention_weight_min": -39.65251922607422,
      "activations/layer18_attention_weight_max": 54.8810920715332,
      "activations/layer18_attention_weight_min": -31.87251091003418,
      "activations/layer19_attention_weight_max": 24.638771057128906,
      "activations/layer19_attention_weight_min": -19.161046981811523,
      "activations/layer1_attention_weight_max": 18.37934684753418,
      "activations/layer1_attention_weight_min": -15.77220344543457,
      "activations/layer20_attention_weight_max": 21.240337371826172,
      "activations/layer20_attention_weight_min": -19.15056800842285,
      "activations/layer21_attention_weight_max": 39.72623062133789,
      "activations/layer21_attention_weight_min": -28.567073822021484,
      "activations/layer22_attention_weight_max": 40.83148956298828,
      "activations/layer22_attention_weight_min": -24.34636688232422,
      "activations/layer23_attention_weight_max": 31.448095321655273,
      "activations/layer23_attention_weight_min": -17.51815414428711,
      "activations/layer2_attention_weight_max": 31.620954513549805,
      "activations/layer2_attention_weight_min": -30.17766571044922,
      "activations/layer3_attention_weight_max": 87.22734069824219,
      "activations/layer3_attention_weight_min": -90.2694091796875,
      "activations/layer4_attention_weight_max": 107.62017822265625,
      "activations/layer4_attention_weight_min": -101.0295639038086,
      "activations/layer5_attention_weight_max": 111.63909912109375,
      "activations/layer5_attention_weight_min": -97.42349243164062,
      "activations/layer6_attention_weight_max": 64.16929626464844,
      "activations/layer6_attention_weight_min": -53.59347152709961,
      "activations/layer7_attention_weight_max": 73.74951171875,
      "activations/layer7_attention_weight_min": -66.98906707763672,
      "activations/layer8_attention_weight_max": 48.08454132080078,
      "activations/layer8_attention_weight_min": -44.11672592163086,
      "activations/layer9_attention_weight_max": 55.06936264038086,
      "activations/layer9_attention_weight_min": -48.45231628417969,
      "epoch": 4.88,
      "learning_rate": 0.00011976742424242422,
      "loss": 2.8582,
      "step": 83900
    },
    {
      "activations/layer0_attention_weight_max": 13.86259651184082,
      "activations/layer0_attention_weight_min": -14.371594429016113,
      "activations/layer10_attention_weight_max": 35.796051025390625,
      "activations/layer10_attention_weight_min": -33.89683151245117,
      "activations/layer11_attention_weight_max": 32.937965393066406,
      "activations/layer11_attention_weight_min": -26.732528686523438,
      "activations/layer12_attention_weight_max": 22.02772331237793,
      "activations/layer12_attention_weight_min": -25.14228630065918,
      "activations/layer13_attention_weight_max": 50.20376205444336,
      "activations/layer13_attention_weight_min": -39.62614059448242,
      "activations/layer14_attention_weight_max": 53.02659225463867,
      "activations/layer14_attention_weight_min": -35.38885498046875,
      "activations/layer15_attention_weight_max": 42.458168029785156,
      "activations/layer15_attention_weight_min": -31.18536376953125,
      "activations/layer16_attention_weight_max": 38.99106216430664,
      "activations/layer16_attention_weight_min": -27.317020416259766,
      "activations/layer17_attention_weight_max": 56.8160285949707,
      "activations/layer17_attention_weight_min": -44.204837799072266,
      "activations/layer18_attention_weight_max": 50.46536636352539,
      "activations/layer18_attention_weight_min": -36.497955322265625,
      "activations/layer19_attention_weight_max": 23.333209991455078,
      "activations/layer19_attention_weight_min": -20.452392578125,
      "activations/layer1_attention_weight_max": 17.439550399780273,
      "activations/layer1_attention_weight_min": -15.928135871887207,
      "activations/layer20_attention_weight_max": 26.08983039855957,
      "activations/layer20_attention_weight_min": -19.77358627319336,
      "activations/layer21_attention_weight_max": 52.79133987426758,
      "activations/layer21_attention_weight_min": -26.646127700805664,
      "activations/layer22_attention_weight_max": 39.08552169799805,
      "activations/layer22_attention_weight_min": -23.347978591918945,
      "activations/layer23_attention_weight_max": 27.97994041442871,
      "activations/layer23_attention_weight_min": -17.09081268310547,
      "activations/layer2_attention_weight_max": 32.775550842285156,
      "activations/layer2_attention_weight_min": -30.88943099975586,
      "activations/layer3_attention_weight_max": 91.02735900878906,
      "activations/layer3_attention_weight_min": -87.20826721191406,
      "activations/layer4_attention_weight_max": 104.13818359375,
      "activations/layer4_attention_weight_min": -94.65119171142578,
      "activations/layer5_attention_weight_max": 101.1048583984375,
      "activations/layer5_attention_weight_min": -90.2715072631836,
      "activations/layer6_attention_weight_max": 57.51445770263672,
      "activations/layer6_attention_weight_min": -51.62444305419922,
      "activations/layer7_attention_weight_max": 73.33152770996094,
      "activations/layer7_attention_weight_min": -65.39180755615234,
      "activations/layer8_attention_weight_max": 41.231101989746094,
      "activations/layer8_attention_weight_min": -43.11836242675781,
      "activations/layer9_attention_weight_max": 51.906917572021484,
      "activations/layer9_attention_weight_min": -47.3330192565918,
      "epoch": 4.88,
      "learning_rate": 0.00011974848484848484,
      "loss": 2.8471,
      "step": 83950
    },
    {
      "activations/layer0_attention_weight_max": 14.392377853393555,
      "activations/layer0_attention_weight_min": -15.084059715270996,
      "activations/layer10_attention_weight_max": 42.36153030395508,
      "activations/layer10_attention_weight_min": -35.16361999511719,
      "activations/layer11_attention_weight_max": 35.98628234863281,
      "activations/layer11_attention_weight_min": -26.307132720947266,
      "activations/layer12_attention_weight_max": 24.82196617126465,
      "activations/layer12_attention_weight_min": -22.060325622558594,
      "activations/layer13_attention_weight_max": 47.307254791259766,
      "activations/layer13_attention_weight_min": -34.33838653564453,
      "activations/layer14_attention_weight_max": 45.92189407348633,
      "activations/layer14_attention_weight_min": -29.131349563598633,
      "activations/layer15_attention_weight_max": 47.46907043457031,
      "activations/layer15_attention_weight_min": -31.134693145751953,
      "activations/layer16_attention_weight_max": 40.068302154541016,
      "activations/layer16_attention_weight_min": -27.167987823486328,
      "activations/layer17_attention_weight_max": 60.63252258300781,
      "activations/layer17_attention_weight_min": -41.92957305908203,
      "activations/layer18_attention_weight_max": 57.5434455871582,
      "activations/layer18_attention_weight_min": -34.555137634277344,
      "activations/layer19_attention_weight_max": 29.638952255249023,
      "activations/layer19_attention_weight_min": -19.473161697387695,
      "activations/layer1_attention_weight_max": 18.047138214111328,
      "activations/layer1_attention_weight_min": -16.841394424438477,
      "activations/layer20_attention_weight_max": 24.875349044799805,
      "activations/layer20_attention_weight_min": -18.930862426757812,
      "activations/layer21_attention_weight_max": 47.33761978149414,
      "activations/layer21_attention_weight_min": -28.523611068725586,
      "activations/layer22_attention_weight_max": 39.77519607543945,
      "activations/layer22_attention_weight_min": -25.795883178710938,
      "activations/layer23_attention_weight_max": 31.871328353881836,
      "activations/layer23_attention_weight_min": -19.030385971069336,
      "activations/layer2_attention_weight_max": 32.912315368652344,
      "activations/layer2_attention_weight_min": -30.50741958618164,
      "activations/layer3_attention_weight_max": 87.26498413085938,
      "activations/layer3_attention_weight_min": -88.12395477294922,
      "activations/layer4_attention_weight_max": 104.6593017578125,
      "activations/layer4_attention_weight_min": -100.85437774658203,
      "activations/layer5_attention_weight_max": 107.20651245117188,
      "activations/layer5_attention_weight_min": -99.40872192382812,
      "activations/layer6_attention_weight_max": 60.2679557800293,
      "activations/layer6_attention_weight_min": -53.91822814941406,
      "activations/layer7_attention_weight_max": 76.78861999511719,
      "activations/layer7_attention_weight_min": -70.17230987548828,
      "activations/layer8_attention_weight_max": 48.93759536743164,
      "activations/layer8_attention_weight_min": -42.92734909057617,
      "activations/layer9_attention_weight_max": 59.12968826293945,
      "activations/layer9_attention_weight_min": -49.74140167236328,
      "epoch": 4.88,
      "learning_rate": 0.00011972954545454545,
      "loss": 2.8817,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_loss": 2.806640625,
      "eval_runtime": 8.473,
      "eval_samples_per_second": 506.784,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_openwebtext_loss": 2.806640625,
      "eval_openwebtext_ppl": 16.554212894695087,
      "eval_openwebtext_runtime": 8.473,
      "eval_openwebtext_samples_per_second": 506.784,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_wikitext_loss": 3.05078125,
      "eval_wikitext_ppl": 21.131847230940878,
      "eval_wikitext_runtime": 1.9683,
      "eval_wikitext_samples_per_second": 231.669,
      "step": 84000
    },
    {
      "epoch": 4.88,
      "eval_lambada_loss": 2.8515625,
      "eval_lambada_ppl": 17.314815113938415,
      "eval_lambada_runtime": 9.6034,
      "eval_lambada_samples_per_second": 507.008,
      "step": 84000
    },
    {
      "activations/layer0_attention_weight_max": 14.527832984924316,
      "activations/layer0_attention_weight_min": -14.785622596740723,
      "activations/layer10_attention_weight_max": 42.07304382324219,
      "activations/layer10_attention_weight_min": -36.59664535522461,
      "activations/layer11_attention_weight_max": 31.59515953063965,
      "activations/layer11_attention_weight_min": -27.810218811035156,
      "activations/layer12_attention_weight_max": 41.55222702026367,
      "activations/layer12_attention_weight_min": -25.27145767211914,
      "activations/layer13_attention_weight_max": 60.59497833251953,
      "activations/layer13_attention_weight_min": -37.39986038208008,
      "activations/layer14_attention_weight_max": 51.76839065551758,
      "activations/layer14_attention_weight_min": -31.903348922729492,
      "activations/layer15_attention_weight_max": 52.0652961730957,
      "activations/layer15_attention_weight_min": -30.64253044128418,
      "activations/layer16_attention_weight_max": 40.827056884765625,
      "activations/layer16_attention_weight_min": -28.138498306274414,
      "activations/layer17_attention_weight_max": 67.64884185791016,
      "activations/layer17_attention_weight_min": -39.04793930053711,
      "activations/layer18_attention_weight_max": 53.63340759277344,
      "activations/layer18_attention_weight_min": -34.3922004699707,
      "activations/layer19_attention_weight_max": 24.548614501953125,
      "activations/layer19_attention_weight_min": -18.996807098388672,
      "activations/layer1_attention_weight_max": 17.505794525146484,
      "activations/layer1_attention_weight_min": -19.932153701782227,
      "activations/layer20_attention_weight_max": 23.752708435058594,
      "activations/layer20_attention_weight_min": -22.511377334594727,
      "activations/layer21_attention_weight_max": 44.43634796142578,
      "activations/layer21_attention_weight_min": -29.42265510559082,
      "activations/layer22_attention_weight_max": 42.98683547973633,
      "activations/layer22_attention_weight_min": -27.9749698638916,
      "activations/layer23_attention_weight_max": 36.58483123779297,
      "activations/layer23_attention_weight_min": -17.61931800842285,
      "activations/layer2_attention_weight_max": 32.25673294067383,
      "activations/layer2_attention_weight_min": -28.745471954345703,
      "activations/layer3_attention_weight_max": 81.69232940673828,
      "activations/layer3_attention_weight_min": -83.67738342285156,
      "activations/layer4_attention_weight_max": 103.63077545166016,
      "activations/layer4_attention_weight_min": -101.97417449951172,
      "activations/layer5_attention_weight_max": 108.7887191772461,
      "activations/layer5_attention_weight_min": -99.45793151855469,
      "activations/layer6_attention_weight_max": 57.33623123168945,
      "activations/layer6_attention_weight_min": -56.1920051574707,
      "activations/layer7_attention_weight_max": 75.13335418701172,
      "activations/layer7_attention_weight_min": -72.1082534790039,
      "activations/layer8_attention_weight_max": 47.8201789855957,
      "activations/layer8_attention_weight_min": -42.116729736328125,
      "activations/layer9_attention_weight_max": 51.50576400756836,
      "activations/layer9_attention_weight_min": -48.622188568115234,
      "epoch": 4.88,
      "learning_rate": 0.00011971060606060604,
      "loss": 2.8499,
      "step": 84050
    },
    {
      "activations/layer0_attention_weight_max": 15.572989463806152,
      "activations/layer0_attention_weight_min": -14.722195625305176,
      "activations/layer10_attention_weight_max": 42.451080322265625,
      "activations/layer10_attention_weight_min": -34.59495544433594,
      "activations/layer11_attention_weight_max": 34.51364517211914,
      "activations/layer11_attention_weight_min": -27.44585418701172,
      "activations/layer12_attention_weight_max": 25.88981819152832,
      "activations/layer12_attention_weight_min": -27.85857391357422,
      "activations/layer13_attention_weight_max": 54.43793869018555,
      "activations/layer13_attention_weight_min": -37.187644958496094,
      "activations/layer14_attention_weight_max": 57.83489990234375,
      "activations/layer14_attention_weight_min": -34.52299118041992,
      "activations/layer15_attention_weight_max": 45.840396881103516,
      "activations/layer15_attention_weight_min": -30.2569522857666,
      "activations/layer16_attention_weight_max": 40.22916030883789,
      "activations/layer16_attention_weight_min": -28.026334762573242,
      "activations/layer17_attention_weight_max": 68.19500732421875,
      "activations/layer17_attention_weight_min": -40.66925811767578,
      "activations/layer18_attention_weight_max": 58.70531463623047,
      "activations/layer18_attention_weight_min": -36.84626007080078,
      "activations/layer19_attention_weight_max": 27.32480239868164,
      "activations/layer19_attention_weight_min": -19.95796775817871,
      "activations/layer1_attention_weight_max": 18.278600692749023,
      "activations/layer1_attention_weight_min": -15.45477294921875,
      "activations/layer20_attention_weight_max": 32.40492248535156,
      "activations/layer20_attention_weight_min": -19.445072174072266,
      "activations/layer21_attention_weight_max": 58.55516052246094,
      "activations/layer21_attention_weight_min": -28.92626190185547,
      "activations/layer22_attention_weight_max": 41.118370056152344,
      "activations/layer22_attention_weight_min": -27.088825225830078,
      "activations/layer23_attention_weight_max": 32.07916259765625,
      "activations/layer23_attention_weight_min": -16.622634887695312,
      "activations/layer2_attention_weight_max": 32.71406555175781,
      "activations/layer2_attention_weight_min": -30.232162475585938,
      "activations/layer3_attention_weight_max": 89.14068603515625,
      "activations/layer3_attention_weight_min": -89.40633392333984,
      "activations/layer4_attention_weight_max": 108.20128631591797,
      "activations/layer4_attention_weight_min": -104.59105682373047,
      "activations/layer5_attention_weight_max": 106.1544189453125,
      "activations/layer5_attention_weight_min": -101.65414428710938,
      "activations/layer6_attention_weight_max": 58.7563362121582,
      "activations/layer6_attention_weight_min": -54.554176330566406,
      "activations/layer7_attention_weight_max": 74.44309997558594,
      "activations/layer7_attention_weight_min": -67.08928680419922,
      "activations/layer8_attention_weight_max": 47.12702941894531,
      "activations/layer8_attention_weight_min": -42.82612991333008,
      "activations/layer9_attention_weight_max": 55.508094787597656,
      "activations/layer9_attention_weight_min": -49.2260627746582,
      "epoch": 4.89,
      "learning_rate": 0.00011969166666666666,
      "loss": 2.8499,
      "step": 84100
    },
    {
      "activations/layer0_attention_weight_max": 14.474540710449219,
      "activations/layer0_attention_weight_min": -14.340657234191895,
      "activations/layer10_attention_weight_max": 39.223236083984375,
      "activations/layer10_attention_weight_min": -36.49169921875,
      "activations/layer11_attention_weight_max": 29.35968780517578,
      "activations/layer11_attention_weight_min": -27.669479370117188,
      "activations/layer12_attention_weight_max": 22.74477767944336,
      "activations/layer12_attention_weight_min": -25.298444747924805,
      "activations/layer13_attention_weight_max": 45.47870635986328,
      "activations/layer13_attention_weight_min": -37.91903305053711,
      "activations/layer14_attention_weight_max": 41.3903923034668,
      "activations/layer14_attention_weight_min": -28.99526023864746,
      "activations/layer15_attention_weight_max": 36.888999938964844,
      "activations/layer15_attention_weight_min": -30.19691276550293,
      "activations/layer16_attention_weight_max": 37.14421081542969,
      "activations/layer16_attention_weight_min": -25.556642532348633,
      "activations/layer17_attention_weight_max": 54.54642868041992,
      "activations/layer17_attention_weight_min": -39.11835479736328,
      "activations/layer18_attention_weight_max": 46.212379455566406,
      "activations/layer18_attention_weight_min": -35.23435974121094,
      "activations/layer19_attention_weight_max": 23.983030319213867,
      "activations/layer19_attention_weight_min": -21.697336196899414,
      "activations/layer1_attention_weight_max": 17.877376556396484,
      "activations/layer1_attention_weight_min": -15.123961448669434,
      "activations/layer20_attention_weight_max": 22.69939422607422,
      "activations/layer20_attention_weight_min": -20.525245666503906,
      "activations/layer21_attention_weight_max": 38.47955322265625,
      "activations/layer21_attention_weight_min": -27.738590240478516,
      "activations/layer22_attention_weight_max": 34.09098434448242,
      "activations/layer22_attention_weight_min": -23.75540542602539,
      "activations/layer23_attention_weight_max": 25.25775718688965,
      "activations/layer23_attention_weight_min": -19.570127487182617,
      "activations/layer2_attention_weight_max": 31.99394989013672,
      "activations/layer2_attention_weight_min": -30.187389373779297,
      "activations/layer3_attention_weight_max": 84.1163558959961,
      "activations/layer3_attention_weight_min": -87.69023132324219,
      "activations/layer4_attention_weight_max": 107.2200698852539,
      "activations/layer4_attention_weight_min": -101.16970825195312,
      "activations/layer5_attention_weight_max": 108.8712387084961,
      "activations/layer5_attention_weight_min": -101.69056701660156,
      "activations/layer6_attention_weight_max": 61.06486511230469,
      "activations/layer6_attention_weight_min": -57.40135192871094,
      "activations/layer7_attention_weight_max": 68.06234741210938,
      "activations/layer7_attention_weight_min": -68.05313110351562,
      "activations/layer8_attention_weight_max": 40.367523193359375,
      "activations/layer8_attention_weight_min": -42.47101974487305,
      "activations/layer9_attention_weight_max": 48.57686996459961,
      "activations/layer9_attention_weight_min": -47.04985046386719,
      "epoch": 4.89,
      "learning_rate": 0.00011967272727272727,
      "loss": 2.8375,
      "step": 84150
    },
    {
      "activations/layer0_attention_weight_max": 14.182822227478027,
      "activations/layer0_attention_weight_min": -15.260709762573242,
      "activations/layer10_attention_weight_max": 38.190181732177734,
      "activations/layer10_attention_weight_min": -32.78327178955078,
      "activations/layer11_attention_weight_max": 30.086013793945312,
      "activations/layer11_attention_weight_min": -26.139240264892578,
      "activations/layer12_attention_weight_max": 21.79386329650879,
      "activations/layer12_attention_weight_min": -28.317523956298828,
      "activations/layer13_attention_weight_max": 51.313438415527344,
      "activations/layer13_attention_weight_min": -35.24091339111328,
      "activations/layer14_attention_weight_max": 58.61168670654297,
      "activations/layer14_attention_weight_min": -37.39612579345703,
      "activations/layer15_attention_weight_max": 40.773681640625,
      "activations/layer15_attention_weight_min": -30.76447296142578,
      "activations/layer16_attention_weight_max": 32.988624572753906,
      "activations/layer16_attention_weight_min": -26.569719314575195,
      "activations/layer17_attention_weight_max": 54.398685455322266,
      "activations/layer17_attention_weight_min": -39.83047103881836,
      "activations/layer18_attention_weight_max": 48.81841278076172,
      "activations/layer18_attention_weight_min": -34.06269454956055,
      "activations/layer19_attention_weight_max": 21.982728958129883,
      "activations/layer19_attention_weight_min": -18.644330978393555,
      "activations/layer1_attention_weight_max": 17.503267288208008,
      "activations/layer1_attention_weight_min": -14.39728832244873,
      "activations/layer20_attention_weight_max": 18.552719116210938,
      "activations/layer20_attention_weight_min": -19.012128829956055,
      "activations/layer21_attention_weight_max": 35.948570251464844,
      "activations/layer21_attention_weight_min": -23.927213668823242,
      "activations/layer22_attention_weight_max": 35.43396759033203,
      "activations/layer22_attention_weight_min": -22.98562240600586,
      "activations/layer23_attention_weight_max": 25.46236801147461,
      "activations/layer23_attention_weight_min": -16.976991653442383,
      "activations/layer2_attention_weight_max": 32.77188491821289,
      "activations/layer2_attention_weight_min": -30.166515350341797,
      "activations/layer3_attention_weight_max": 89.9612045288086,
      "activations/layer3_attention_weight_min": -90.68988800048828,
      "activations/layer4_attention_weight_max": 107.24808502197266,
      "activations/layer4_attention_weight_min": -103.37269592285156,
      "activations/layer5_attention_weight_max": 105.186279296875,
      "activations/layer5_attention_weight_min": -99.66616821289062,
      "activations/layer6_attention_weight_max": 57.6569938659668,
      "activations/layer6_attention_weight_min": -53.683467864990234,
      "activations/layer7_attention_weight_max": 73.59280395507812,
      "activations/layer7_attention_weight_min": -69.54729461669922,
      "activations/layer8_attention_weight_max": 42.23634719848633,
      "activations/layer8_attention_weight_min": -41.087947845458984,
      "activations/layer9_attention_weight_max": 60.41290283203125,
      "activations/layer9_attention_weight_min": -52.12781524658203,
      "epoch": 4.89,
      "learning_rate": 0.00011965378787878786,
      "loss": 2.8363,
      "step": 84200
    },
    {
      "activations/layer0_attention_weight_max": 13.717845916748047,
      "activations/layer0_attention_weight_min": -14.626067161560059,
      "activations/layer10_attention_weight_max": 38.17652130126953,
      "activations/layer10_attention_weight_min": -34.362571716308594,
      "activations/layer11_attention_weight_max": 30.603595733642578,
      "activations/layer11_attention_weight_min": -28.248104095458984,
      "activations/layer12_attention_weight_max": 26.18794059753418,
      "activations/layer12_attention_weight_min": -25.5109806060791,
      "activations/layer13_attention_weight_max": 53.30759048461914,
      "activations/layer13_attention_weight_min": -39.69398498535156,
      "activations/layer14_attention_weight_max": 38.93552017211914,
      "activations/layer14_attention_weight_min": -31.730607986450195,
      "activations/layer15_attention_weight_max": 43.57676315307617,
      "activations/layer15_attention_weight_min": -30.586563110351562,
      "activations/layer16_attention_weight_max": 38.278839111328125,
      "activations/layer16_attention_weight_min": -26.866565704345703,
      "activations/layer17_attention_weight_max": 54.216087341308594,
      "activations/layer17_attention_weight_min": -37.695430755615234,
      "activations/layer18_attention_weight_max": 47.47063446044922,
      "activations/layer18_attention_weight_min": -33.446598052978516,
      "activations/layer19_attention_weight_max": 20.119770050048828,
      "activations/layer19_attention_weight_min": -19.189268112182617,
      "activations/layer1_attention_weight_max": 16.849416732788086,
      "activations/layer1_attention_weight_min": -15.826870918273926,
      "activations/layer20_attention_weight_max": 22.644378662109375,
      "activations/layer20_attention_weight_min": -20.104469299316406,
      "activations/layer21_attention_weight_max": 38.52510452270508,
      "activations/layer21_attention_weight_min": -24.544763565063477,
      "activations/layer22_attention_weight_max": 32.53202819824219,
      "activations/layer22_attention_weight_min": -24.239336013793945,
      "activations/layer23_attention_weight_max": 29.576154708862305,
      "activations/layer23_attention_weight_min": -18.168306350708008,
      "activations/layer2_attention_weight_max": 31.309085845947266,
      "activations/layer2_attention_weight_min": -29.50472640991211,
      "activations/layer3_attention_weight_max": 85.9903793334961,
      "activations/layer3_attention_weight_min": -90.37846374511719,
      "activations/layer4_attention_weight_max": 104.35428619384766,
      "activations/layer4_attention_weight_min": -99.81592559814453,
      "activations/layer5_attention_weight_max": 110.2410888671875,
      "activations/layer5_attention_weight_min": -95.52903747558594,
      "activations/layer6_attention_weight_max": 60.273353576660156,
      "activations/layer6_attention_weight_min": -55.663978576660156,
      "activations/layer7_attention_weight_max": 69.77506256103516,
      "activations/layer7_attention_weight_min": -68.33926391601562,
      "activations/layer8_attention_weight_max": 43.807926177978516,
      "activations/layer8_attention_weight_min": -41.11788558959961,
      "activations/layer9_attention_weight_max": 49.37297821044922,
      "activations/layer9_attention_weight_min": -47.81193923950195,
      "epoch": 4.9,
      "learning_rate": 0.00011963484848484848,
      "loss": 2.8286,
      "step": 84250
    },
    {
      "activations/layer0_attention_weight_max": 16.451129913330078,
      "activations/layer0_attention_weight_min": -14.408979415893555,
      "activations/layer10_attention_weight_max": 37.784690856933594,
      "activations/layer10_attention_weight_min": -32.63820266723633,
      "activations/layer11_attention_weight_max": 29.511993408203125,
      "activations/layer11_attention_weight_min": -26.905242919921875,
      "activations/layer12_attention_weight_max": 24.81438636779785,
      "activations/layer12_attention_weight_min": -24.063785552978516,
      "activations/layer13_attention_weight_max": 43.60293960571289,
      "activations/layer13_attention_weight_min": -35.04869842529297,
      "activations/layer14_attention_weight_max": 45.34712219238281,
      "activations/layer14_attention_weight_min": -32.17000198364258,
      "activations/layer15_attention_weight_max": 39.03062057495117,
      "activations/layer15_attention_weight_min": -30.729934692382812,
      "activations/layer16_attention_weight_max": 41.36641311645508,
      "activations/layer16_attention_weight_min": -27.819412231445312,
      "activations/layer17_attention_weight_max": 53.94237518310547,
      "activations/layer17_attention_weight_min": -39.15555191040039,
      "activations/layer18_attention_weight_max": 47.4552001953125,
      "activations/layer18_attention_weight_min": -34.530399322509766,
      "activations/layer19_attention_weight_max": 23.191553115844727,
      "activations/layer19_attention_weight_min": -21.124723434448242,
      "activations/layer1_attention_weight_max": 17.268917083740234,
      "activations/layer1_attention_weight_min": -17.90985107421875,
      "activations/layer20_attention_weight_max": 22.712499618530273,
      "activations/layer20_attention_weight_min": -21.77509117126465,
      "activations/layer21_attention_weight_max": 39.51002883911133,
      "activations/layer21_attention_weight_min": -26.52842903137207,
      "activations/layer22_attention_weight_max": 35.12152862548828,
      "activations/layer22_attention_weight_min": -23.035276412963867,
      "activations/layer23_attention_weight_max": 28.102378845214844,
      "activations/layer23_attention_weight_min": -19.15287208557129,
      "activations/layer2_attention_weight_max": 31.125751495361328,
      "activations/layer2_attention_weight_min": -27.788406372070312,
      "activations/layer3_attention_weight_max": 84.36707305908203,
      "activations/layer3_attention_weight_min": -84.37591552734375,
      "activations/layer4_attention_weight_max": 101.61412811279297,
      "activations/layer4_attention_weight_min": -94.5346908569336,
      "activations/layer5_attention_weight_max": 102.93218994140625,
      "activations/layer5_attention_weight_min": -92.79798889160156,
      "activations/layer6_attention_weight_max": 60.35990905761719,
      "activations/layer6_attention_weight_min": -54.22128677368164,
      "activations/layer7_attention_weight_max": 73.82355499267578,
      "activations/layer7_attention_weight_min": -72.08601379394531,
      "activations/layer8_attention_weight_max": 44.74906921386719,
      "activations/layer8_attention_weight_min": -42.94441604614258,
      "activations/layer9_attention_weight_max": 53.63656234741211,
      "activations/layer9_attention_weight_min": -46.84809112548828,
      "epoch": 4.9,
      "learning_rate": 0.00011961590909090908,
      "loss": 2.8287,
      "step": 84300
    },
    {
      "activations/layer0_attention_weight_max": 13.844467163085938,
      "activations/layer0_attention_weight_min": -15.093774795532227,
      "activations/layer10_attention_weight_max": 39.280094146728516,
      "activations/layer10_attention_weight_min": -39.13713836669922,
      "activations/layer11_attention_weight_max": 31.01787567138672,
      "activations/layer11_attention_weight_min": -27.895580291748047,
      "activations/layer12_attention_weight_max": 36.22959899902344,
      "activations/layer12_attention_weight_min": -27.798629760742188,
      "activations/layer13_attention_weight_max": 63.58479309082031,
      "activations/layer13_attention_weight_min": -41.990047454833984,
      "activations/layer14_attention_weight_max": 52.202518463134766,
      "activations/layer14_attention_weight_min": -38.250003814697266,
      "activations/layer15_attention_weight_max": 42.0678596496582,
      "activations/layer15_attention_weight_min": -29.158668518066406,
      "activations/layer16_attention_weight_max": 40.73695373535156,
      "activations/layer16_attention_weight_min": -26.466081619262695,
      "activations/layer17_attention_weight_max": 63.19744873046875,
      "activations/layer17_attention_weight_min": -38.943973541259766,
      "activations/layer18_attention_weight_max": 54.268768310546875,
      "activations/layer18_attention_weight_min": -34.50117492675781,
      "activations/layer19_attention_weight_max": 22.627403259277344,
      "activations/layer19_attention_weight_min": -21.740652084350586,
      "activations/layer1_attention_weight_max": 16.68752670288086,
      "activations/layer1_attention_weight_min": -14.257357597351074,
      "activations/layer20_attention_weight_max": 22.3287410736084,
      "activations/layer20_attention_weight_min": -18.971508026123047,
      "activations/layer21_attention_weight_max": 44.00178146362305,
      "activations/layer21_attention_weight_min": -27.545934677124023,
      "activations/layer22_attention_weight_max": 31.507144927978516,
      "activations/layer22_attention_weight_min": -22.5133113861084,
      "activations/layer23_attention_weight_max": 32.00857925415039,
      "activations/layer23_attention_weight_min": -23.534748077392578,
      "activations/layer2_attention_weight_max": 32.64241027832031,
      "activations/layer2_attention_weight_min": -30.17300796508789,
      "activations/layer3_attention_weight_max": 85.3473129272461,
      "activations/layer3_attention_weight_min": -90.62694549560547,
      "activations/layer4_attention_weight_max": 109.49918365478516,
      "activations/layer4_attention_weight_min": -105.15089416503906,
      "activations/layer5_attention_weight_max": 112.09224700927734,
      "activations/layer5_attention_weight_min": -102.26016235351562,
      "activations/layer6_attention_weight_max": 62.54292297363281,
      "activations/layer6_attention_weight_min": -56.6229248046875,
      "activations/layer7_attention_weight_max": 74.44578552246094,
      "activations/layer7_attention_weight_min": -68.06837463378906,
      "activations/layer8_attention_weight_max": 47.768253326416016,
      "activations/layer8_attention_weight_min": -46.220672607421875,
      "activations/layer9_attention_weight_max": 54.57950973510742,
      "activations/layer9_attention_weight_min": -50.26519012451172,
      "epoch": 4.9,
      "learning_rate": 0.00011959696969696969,
      "loss": 2.8428,
      "step": 84350
    },
    {
      "activations/layer0_attention_weight_max": 16.567813873291016,
      "activations/layer0_attention_weight_min": -13.764848709106445,
      "activations/layer10_attention_weight_max": 49.529693603515625,
      "activations/layer10_attention_weight_min": -37.75593185424805,
      "activations/layer11_attention_weight_max": 34.294029235839844,
      "activations/layer11_attention_weight_min": -27.748823165893555,
      "activations/layer12_attention_weight_max": 19.470951080322266,
      "activations/layer12_attention_weight_min": -31.04376983642578,
      "activations/layer13_attention_weight_max": 57.498802185058594,
      "activations/layer13_attention_weight_min": -35.67140579223633,
      "activations/layer14_attention_weight_max": 66.55582427978516,
      "activations/layer14_attention_weight_min": -39.331424713134766,
      "activations/layer15_attention_weight_max": 51.635257720947266,
      "activations/layer15_attention_weight_min": -30.05074119567871,
      "activations/layer16_attention_weight_max": 35.98389434814453,
      "activations/layer16_attention_weight_min": -28.03811264038086,
      "activations/layer17_attention_weight_max": 57.40474319458008,
      "activations/layer17_attention_weight_min": -37.79692840576172,
      "activations/layer18_attention_weight_max": 55.47660827636719,
      "activations/layer18_attention_weight_min": -35.23537063598633,
      "activations/layer19_attention_weight_max": 21.05078125,
      "activations/layer19_attention_weight_min": -21.47499656677246,
      "activations/layer1_attention_weight_max": 17.529190063476562,
      "activations/layer1_attention_weight_min": -14.77798843383789,
      "activations/layer20_attention_weight_max": 23.854103088378906,
      "activations/layer20_attention_weight_min": -19.698266983032227,
      "activations/layer21_attention_weight_max": 39.1041259765625,
      "activations/layer21_attention_weight_min": -25.65465545654297,
      "activations/layer22_attention_weight_max": 33.778072357177734,
      "activations/layer22_attention_weight_min": -25.13319969177246,
      "activations/layer23_attention_weight_max": 28.23097801208496,
      "activations/layer23_attention_weight_min": -17.3629207611084,
      "activations/layer2_attention_weight_max": 35.17282485961914,
      "activations/layer2_attention_weight_min": -31.057828903198242,
      "activations/layer3_attention_weight_max": 91.04887390136719,
      "activations/layer3_attention_weight_min": -91.28538513183594,
      "activations/layer4_attention_weight_max": 109.28218841552734,
      "activations/layer4_attention_weight_min": -102.801025390625,
      "activations/layer5_attention_weight_max": 109.43504333496094,
      "activations/layer5_attention_weight_min": -96.12987518310547,
      "activations/layer6_attention_weight_max": 58.78074645996094,
      "activations/layer6_attention_weight_min": -53.152400970458984,
      "activations/layer7_attention_weight_max": 79.11927032470703,
      "activations/layer7_attention_weight_min": -69.7082748413086,
      "activations/layer8_attention_weight_max": 49.6036491394043,
      "activations/layer8_attention_weight_min": -45.995628356933594,
      "activations/layer9_attention_weight_max": 64.66728973388672,
      "activations/layer9_attention_weight_min": -52.13029098510742,
      "epoch": 4.9,
      "learning_rate": 0.0001195780303030303,
      "loss": 2.804,
      "step": 84400
    },
    {
      "activations/layer0_attention_weight_max": 13.989362716674805,
      "activations/layer0_attention_weight_min": -14.352715492248535,
      "activations/layer10_attention_weight_max": 35.30115509033203,
      "activations/layer10_attention_weight_min": -35.09638214111328,
      "activations/layer11_attention_weight_max": 29.166250228881836,
      "activations/layer11_attention_weight_min": -26.275243759155273,
      "activations/layer12_attention_weight_max": 21.27877426147461,
      "activations/layer12_attention_weight_min": -25.90799331665039,
      "activations/layer13_attention_weight_max": 45.16386413574219,
      "activations/layer13_attention_weight_min": -34.20962905883789,
      "activations/layer14_attention_weight_max": 42.46787643432617,
      "activations/layer14_attention_weight_min": -28.685745239257812,
      "activations/layer15_attention_weight_max": 41.83418273925781,
      "activations/layer15_attention_weight_min": -30.728652954101562,
      "activations/layer16_attention_weight_max": 37.06488800048828,
      "activations/layer16_attention_weight_min": -28.001001358032227,
      "activations/layer17_attention_weight_max": 53.41116714477539,
      "activations/layer17_attention_weight_min": -39.519798278808594,
      "activations/layer18_attention_weight_max": 51.74541473388672,
      "activations/layer18_attention_weight_min": -32.752037048339844,
      "activations/layer19_attention_weight_max": 24.979835510253906,
      "activations/layer19_attention_weight_min": -17.1228084564209,
      "activations/layer1_attention_weight_max": 17.2447452545166,
      "activations/layer1_attention_weight_min": -16.055742263793945,
      "activations/layer20_attention_weight_max": 20.815893173217773,
      "activations/layer20_attention_weight_min": -18.357574462890625,
      "activations/layer21_attention_weight_max": 36.64710235595703,
      "activations/layer21_attention_weight_min": -25.658422470092773,
      "activations/layer22_attention_weight_max": 35.5422477722168,
      "activations/layer22_attention_weight_min": -23.261674880981445,
      "activations/layer23_attention_weight_max": 27.65337371826172,
      "activations/layer23_attention_weight_min": -17.784992218017578,
      "activations/layer2_attention_weight_max": 30.93484115600586,
      "activations/layer2_attention_weight_min": -27.816770553588867,
      "activations/layer3_attention_weight_max": 83.7228012084961,
      "activations/layer3_attention_weight_min": -85.96884155273438,
      "activations/layer4_attention_weight_max": 104.3666763305664,
      "activations/layer4_attention_weight_min": -99.79796600341797,
      "activations/layer5_attention_weight_max": 102.14678955078125,
      "activations/layer5_attention_weight_min": -97.49508666992188,
      "activations/layer6_attention_weight_max": 60.92402267456055,
      "activations/layer6_attention_weight_min": -53.98503494262695,
      "activations/layer7_attention_weight_max": 71.023681640625,
      "activations/layer7_attention_weight_min": -66.88333129882812,
      "activations/layer8_attention_weight_max": 39.84929275512695,
      "activations/layer8_attention_weight_min": -39.91848373413086,
      "activations/layer9_attention_weight_max": 47.32861328125,
      "activations/layer9_attention_weight_min": -48.32032012939453,
      "epoch": 4.91,
      "learning_rate": 0.0001195590909090909,
      "loss": 2.8372,
      "step": 84450
    },
    {
      "activations/layer0_attention_weight_max": 14.70989990234375,
      "activations/layer0_attention_weight_min": -14.604888916015625,
      "activations/layer10_attention_weight_max": 38.66049575805664,
      "activations/layer10_attention_weight_min": -36.30713653564453,
      "activations/layer11_attention_weight_max": 31.29488754272461,
      "activations/layer11_attention_weight_min": -28.255117416381836,
      "activations/layer12_attention_weight_max": 22.25347900390625,
      "activations/layer12_attention_weight_min": -24.616416931152344,
      "activations/layer13_attention_weight_max": 64.63595581054688,
      "activations/layer13_attention_weight_min": -40.040706634521484,
      "activations/layer14_attention_weight_max": 63.49953079223633,
      "activations/layer14_attention_weight_min": -34.61433029174805,
      "activations/layer15_attention_weight_max": 47.94218444824219,
      "activations/layer15_attention_weight_min": -30.287261962890625,
      "activations/layer16_attention_weight_max": 36.3074951171875,
      "activations/layer16_attention_weight_min": -27.69681167602539,
      "activations/layer17_attention_weight_max": 63.74229049682617,
      "activations/layer17_attention_weight_min": -40.77040481567383,
      "activations/layer18_attention_weight_max": 54.32412338256836,
      "activations/layer18_attention_weight_min": -35.5080680847168,
      "activations/layer19_attention_weight_max": 25.171480178833008,
      "activations/layer19_attention_weight_min": -19.27326011657715,
      "activations/layer1_attention_weight_max": 16.735912322998047,
      "activations/layer1_attention_weight_min": -15.741655349731445,
      "activations/layer20_attention_weight_max": 24.272911071777344,
      "activations/layer20_attention_weight_min": -21.802335739135742,
      "activations/layer21_attention_weight_max": 52.14203643798828,
      "activations/layer21_attention_weight_min": -26.39727783203125,
      "activations/layer22_attention_weight_max": 37.43385696411133,
      "activations/layer22_attention_weight_min": -26.08159637451172,
      "activations/layer23_attention_weight_max": 30.42105484008789,
      "activations/layer23_attention_weight_min": -17.531185150146484,
      "activations/layer2_attention_weight_max": 30.201499938964844,
      "activations/layer2_attention_weight_min": -29.088422775268555,
      "activations/layer3_attention_weight_max": 82.813232421875,
      "activations/layer3_attention_weight_min": -87.9295883178711,
      "activations/layer4_attention_weight_max": 102.56511688232422,
      "activations/layer4_attention_weight_min": -96.33972930908203,
      "activations/layer5_attention_weight_max": 103.4222412109375,
      "activations/layer5_attention_weight_min": -96.08821105957031,
      "activations/layer6_attention_weight_max": 56.89476013183594,
      "activations/layer6_attention_weight_min": -51.66682815551758,
      "activations/layer7_attention_weight_max": 69.63446807861328,
      "activations/layer7_attention_weight_min": -67.27323150634766,
      "activations/layer8_attention_weight_max": 42.85359191894531,
      "activations/layer8_attention_weight_min": -42.47810363769531,
      "activations/layer9_attention_weight_max": 51.20626449584961,
      "activations/layer9_attention_weight_min": -47.077735900878906,
      "epoch": 4.91,
      "learning_rate": 0.00011954015151515151,
      "loss": 2.8257,
      "step": 84500
    },
    {
      "activations/layer0_attention_weight_max": 14.973756790161133,
      "activations/layer0_attention_weight_min": -14.5877685546875,
      "activations/layer10_attention_weight_max": 35.114959716796875,
      "activations/layer10_attention_weight_min": -33.77174758911133,
      "activations/layer11_attention_weight_max": 30.047168731689453,
      "activations/layer11_attention_weight_min": -27.332199096679688,
      "activations/layer12_attention_weight_max": 30.465761184692383,
      "activations/layer12_attention_weight_min": -29.570632934570312,
      "activations/layer13_attention_weight_max": 51.867977142333984,
      "activations/layer13_attention_weight_min": -30.627145767211914,
      "activations/layer14_attention_weight_max": 55.65312957763672,
      "activations/layer14_attention_weight_min": -30.549388885498047,
      "activations/layer15_attention_weight_max": 37.943450927734375,
      "activations/layer15_attention_weight_min": -30.31982421875,
      "activations/layer16_attention_weight_max": 33.557533264160156,
      "activations/layer16_attention_weight_min": -26.141788482666016,
      "activations/layer17_attention_weight_max": 61.49146270751953,
      "activations/layer17_attention_weight_min": -36.58236312866211,
      "activations/layer18_attention_weight_max": 45.04564666748047,
      "activations/layer18_attention_weight_min": -32.41279983520508,
      "activations/layer19_attention_weight_max": 24.50718116760254,
      "activations/layer19_attention_weight_min": -18.789121627807617,
      "activations/layer1_attention_weight_max": 17.952669143676758,
      "activations/layer1_attention_weight_min": -15.829019546508789,
      "activations/layer20_attention_weight_max": 19.724260330200195,
      "activations/layer20_attention_weight_min": -19.069887161254883,
      "activations/layer21_attention_weight_max": 40.397579193115234,
      "activations/layer21_attention_weight_min": -25.30841064453125,
      "activations/layer22_attention_weight_max": 32.66057205200195,
      "activations/layer22_attention_weight_min": -22.69359588623047,
      "activations/layer23_attention_weight_max": 30.752117156982422,
      "activations/layer23_attention_weight_min": -19.290260314941406,
      "activations/layer2_attention_weight_max": 32.83234405517578,
      "activations/layer2_attention_weight_min": -29.66165542602539,
      "activations/layer3_attention_weight_max": 85.84583282470703,
      "activations/layer3_attention_weight_min": -84.33797454833984,
      "activations/layer4_attention_weight_max": 106.05138397216797,
      "activations/layer4_attention_weight_min": -100.2882308959961,
      "activations/layer5_attention_weight_max": 108.95384979248047,
      "activations/layer5_attention_weight_min": -94.71853637695312,
      "activations/layer6_attention_weight_max": 59.071441650390625,
      "activations/layer6_attention_weight_min": -55.83519744873047,
      "activations/layer7_attention_weight_max": 74.07890319824219,
      "activations/layer7_attention_weight_min": -68.13185119628906,
      "activations/layer8_attention_weight_max": 41.90007400512695,
      "activations/layer8_attention_weight_min": -40.813270568847656,
      "activations/layer9_attention_weight_max": 58.94532012939453,
      "activations/layer9_attention_weight_min": -48.6241569519043,
      "epoch": 4.91,
      "learning_rate": 0.00011952121212121211,
      "loss": 2.8365,
      "step": 84550
    },
    {
      "activations/layer0_attention_weight_max": 15.028144836425781,
      "activations/layer0_attention_weight_min": -13.865845680236816,
      "activations/layer10_attention_weight_max": 38.939292907714844,
      "activations/layer10_attention_weight_min": -33.921714782714844,
      "activations/layer11_attention_weight_max": 30.096649169921875,
      "activations/layer11_attention_weight_min": -27.418455123901367,
      "activations/layer12_attention_weight_max": 27.179794311523438,
      "activations/layer12_attention_weight_min": -24.47617530822754,
      "activations/layer13_attention_weight_max": 44.64396667480469,
      "activations/layer13_attention_weight_min": -36.793331146240234,
      "activations/layer14_attention_weight_max": 40.055240631103516,
      "activations/layer14_attention_weight_min": -29.237085342407227,
      "activations/layer15_attention_weight_max": 39.61771011352539,
      "activations/layer15_attention_weight_min": -31.469919204711914,
      "activations/layer16_attention_weight_max": 35.278987884521484,
      "activations/layer16_attention_weight_min": -26.83697509765625,
      "activations/layer17_attention_weight_max": 61.32917022705078,
      "activations/layer17_attention_weight_min": -41.1717414855957,
      "activations/layer18_attention_weight_max": 50.329673767089844,
      "activations/layer18_attention_weight_min": -34.69210433959961,
      "activations/layer19_attention_weight_max": 20.559587478637695,
      "activations/layer19_attention_weight_min": -19.028919219970703,
      "activations/layer1_attention_weight_max": 16.887826919555664,
      "activations/layer1_attention_weight_min": -16.124528884887695,
      "activations/layer20_attention_weight_max": 22.47625732421875,
      "activations/layer20_attention_weight_min": -21.119165420532227,
      "activations/layer21_attention_weight_max": 37.14772033691406,
      "activations/layer21_attention_weight_min": -25.547069549560547,
      "activations/layer22_attention_weight_max": 32.21580123901367,
      "activations/layer22_attention_weight_min": -21.808259963989258,
      "activations/layer23_attention_weight_max": 25.622093200683594,
      "activations/layer23_attention_weight_min": -17.026126861572266,
      "activations/layer2_attention_weight_max": 31.062217712402344,
      "activations/layer2_attention_weight_min": -28.377952575683594,
      "activations/layer3_attention_weight_max": 82.28229522705078,
      "activations/layer3_attention_weight_min": -85.05987548828125,
      "activations/layer4_attention_weight_max": 102.79776763916016,
      "activations/layer4_attention_weight_min": -99.82051849365234,
      "activations/layer5_attention_weight_max": 111.60310363769531,
      "activations/layer5_attention_weight_min": -96.12109375,
      "activations/layer6_attention_weight_max": 59.96292495727539,
      "activations/layer6_attention_weight_min": -54.33405303955078,
      "activations/layer7_attention_weight_max": 69.17794036865234,
      "activations/layer7_attention_weight_min": -65.99556732177734,
      "activations/layer8_attention_weight_max": 39.637901306152344,
      "activations/layer8_attention_weight_min": -38.66438674926758,
      "activations/layer9_attention_weight_max": 49.88100814819336,
      "activations/layer9_attention_weight_min": -48.99431228637695,
      "epoch": 4.92,
      "learning_rate": 0.00011950227272727271,
      "loss": 2.8464,
      "step": 84600
    },
    {
      "activations/layer0_attention_weight_max": 18.948715209960938,
      "activations/layer0_attention_weight_min": -14.73233699798584,
      "activations/layer10_attention_weight_max": 38.390968322753906,
      "activations/layer10_attention_weight_min": -33.74237823486328,
      "activations/layer11_attention_weight_max": 31.24639129638672,
      "activations/layer11_attention_weight_min": -27.07659149169922,
      "activations/layer12_attention_weight_max": 24.313812255859375,
      "activations/layer12_attention_weight_min": -24.995723724365234,
      "activations/layer13_attention_weight_max": 61.334293365478516,
      "activations/layer13_attention_weight_min": -43.98802185058594,
      "activations/layer14_attention_weight_max": 43.7803955078125,
      "activations/layer14_attention_weight_min": -32.57928466796875,
      "activations/layer15_attention_weight_max": 42.53434371948242,
      "activations/layer15_attention_weight_min": -31.806243896484375,
      "activations/layer16_attention_weight_max": 36.5467643737793,
      "activations/layer16_attention_weight_min": -29.076791763305664,
      "activations/layer17_attention_weight_max": 62.63703918457031,
      "activations/layer17_attention_weight_min": -40.797607421875,
      "activations/layer18_attention_weight_max": 53.48166275024414,
      "activations/layer18_attention_weight_min": -35.1513557434082,
      "activations/layer19_attention_weight_max": 25.599103927612305,
      "activations/layer19_attention_weight_min": -19.718727111816406,
      "activations/layer1_attention_weight_max": 17.203351974487305,
      "activations/layer1_attention_weight_min": -13.68278980255127,
      "activations/layer20_attention_weight_max": 20.987857818603516,
      "activations/layer20_attention_weight_min": -19.27897071838379,
      "activations/layer21_attention_weight_max": 44.712852478027344,
      "activations/layer21_attention_weight_min": -26.74881935119629,
      "activations/layer22_attention_weight_max": 35.70643997192383,
      "activations/layer22_attention_weight_min": -25.73255729675293,
      "activations/layer23_attention_weight_max": 28.945114135742188,
      "activations/layer23_attention_weight_min": -19.28059959411621,
      "activations/layer2_attention_weight_max": 31.221656799316406,
      "activations/layer2_attention_weight_min": -29.41913414001465,
      "activations/layer3_attention_weight_max": 81.97412109375,
      "activations/layer3_attention_weight_min": -88.25640869140625,
      "activations/layer4_attention_weight_max": 105.28045654296875,
      "activations/layer4_attention_weight_min": -101.68395233154297,
      "activations/layer5_attention_weight_max": 106.11629486083984,
      "activations/layer5_attention_weight_min": -104.29039764404297,
      "activations/layer6_attention_weight_max": 62.47350311279297,
      "activations/layer6_attention_weight_min": -55.13404083251953,
      "activations/layer7_attention_weight_max": 70.27131652832031,
      "activations/layer7_attention_weight_min": -73.9708480834961,
      "activations/layer8_attention_weight_max": 44.83596420288086,
      "activations/layer8_attention_weight_min": -43.91682815551758,
      "activations/layer9_attention_weight_max": 53.15285110473633,
      "activations/layer9_attention_weight_min": -51.15139389038086,
      "epoch": 4.92,
      "learning_rate": 0.00011948333333333333,
      "loss": 2.82,
      "step": 84650
    },
    {
      "activations/layer0_attention_weight_max": 13.880887985229492,
      "activations/layer0_attention_weight_min": -14.487395286560059,
      "activations/layer10_attention_weight_max": 42.30978775024414,
      "activations/layer10_attention_weight_min": -34.54647445678711,
      "activations/layer11_attention_weight_max": 31.04349136352539,
      "activations/layer11_attention_weight_min": -27.078876495361328,
      "activations/layer12_attention_weight_max": 22.816606521606445,
      "activations/layer12_attention_weight_min": -30.079328536987305,
      "activations/layer13_attention_weight_max": 66.40059661865234,
      "activations/layer13_attention_weight_min": -40.60152816772461,
      "activations/layer14_attention_weight_max": 61.20622634887695,
      "activations/layer14_attention_weight_min": -36.0014762878418,
      "activations/layer15_attention_weight_max": 47.29048538208008,
      "activations/layer15_attention_weight_min": -31.21470832824707,
      "activations/layer16_attention_weight_max": 39.61763000488281,
      "activations/layer16_attention_weight_min": -26.445106506347656,
      "activations/layer17_attention_weight_max": 63.83601379394531,
      "activations/layer17_attention_weight_min": -40.81639862060547,
      "activations/layer18_attention_weight_max": 56.040496826171875,
      "activations/layer18_attention_weight_min": -37.09849166870117,
      "activations/layer19_attention_weight_max": 27.373838424682617,
      "activations/layer19_attention_weight_min": -20.0002498626709,
      "activations/layer1_attention_weight_max": 17.804697036743164,
      "activations/layer1_attention_weight_min": -14.124414443969727,
      "activations/layer20_attention_weight_max": 28.47234535217285,
      "activations/layer20_attention_weight_min": -20.418560028076172,
      "activations/layer21_attention_weight_max": 45.097198486328125,
      "activations/layer21_attention_weight_min": -25.21659278869629,
      "activations/layer22_attention_weight_max": 38.52293395996094,
      "activations/layer22_attention_weight_min": -23.657718658447266,
      "activations/layer23_attention_weight_max": 34.217803955078125,
      "activations/layer23_attention_weight_min": -17.96881866455078,
      "activations/layer2_attention_weight_max": 32.73197555541992,
      "activations/layer2_attention_weight_min": -29.60573387145996,
      "activations/layer3_attention_weight_max": 81.1834716796875,
      "activations/layer3_attention_weight_min": -82.32174682617188,
      "activations/layer4_attention_weight_max": 105.30952453613281,
      "activations/layer4_attention_weight_min": -96.2314682006836,
      "activations/layer5_attention_weight_max": 109.63176727294922,
      "activations/layer5_attention_weight_min": -100.5030517578125,
      "activations/layer6_attention_weight_max": 58.31879806518555,
      "activations/layer6_attention_weight_min": -57.33185577392578,
      "activations/layer7_attention_weight_max": 70.72007751464844,
      "activations/layer7_attention_weight_min": -69.22611236572266,
      "activations/layer8_attention_weight_max": 42.46724319458008,
      "activations/layer8_attention_weight_min": -41.747779846191406,
      "activations/layer9_attention_weight_max": 59.453487396240234,
      "activations/layer9_attention_weight_min": -46.27927780151367,
      "epoch": 4.92,
      "learning_rate": 0.00011946439393939392,
      "loss": 2.8505,
      "step": 84700
    },
    {
      "activations/layer0_attention_weight_max": 14.829681396484375,
      "activations/layer0_attention_weight_min": -15.157628059387207,
      "activations/layer10_attention_weight_max": 36.48921585083008,
      "activations/layer10_attention_weight_min": -34.04777908325195,
      "activations/layer11_attention_weight_max": 29.026792526245117,
      "activations/layer11_attention_weight_min": -26.98965835571289,
      "activations/layer12_attention_weight_max": 21.524972915649414,
      "activations/layer12_attention_weight_min": -24.80571174621582,
      "activations/layer13_attention_weight_max": 45.17802047729492,
      "activations/layer13_attention_weight_min": -32.63140869140625,
      "activations/layer14_attention_weight_max": 45.93526840209961,
      "activations/layer14_attention_weight_min": -32.793243408203125,
      "activations/layer15_attention_weight_max": 37.85109329223633,
      "activations/layer15_attention_weight_min": -30.31248664855957,
      "activations/layer16_attention_weight_max": 36.83782196044922,
      "activations/layer16_attention_weight_min": -26.163949966430664,
      "activations/layer17_attention_weight_max": 52.72724533081055,
      "activations/layer17_attention_weight_min": -38.78998947143555,
      "activations/layer18_attention_weight_max": 43.128578186035156,
      "activations/layer18_attention_weight_min": -33.15480422973633,
      "activations/layer19_attention_weight_max": 22.807973861694336,
      "activations/layer19_attention_weight_min": -18.095294952392578,
      "activations/layer1_attention_weight_max": 17.294328689575195,
      "activations/layer1_attention_weight_min": -15.563157081604004,
      "activations/layer20_attention_weight_max": 19.515247344970703,
      "activations/layer20_attention_weight_min": -18.922882080078125,
      "activations/layer21_attention_weight_max": 40.56612014770508,
      "activations/layer21_attention_weight_min": -25.99595832824707,
      "activations/layer22_attention_weight_max": 31.40875244140625,
      "activations/layer22_attention_weight_min": -23.07270050048828,
      "activations/layer23_attention_weight_max": 24.518226623535156,
      "activations/layer23_attention_weight_min": -18.39633560180664,
      "activations/layer2_attention_weight_max": 30.489791870117188,
      "activations/layer2_attention_weight_min": -29.57947540283203,
      "activations/layer3_attention_weight_max": 78.6747817993164,
      "activations/layer3_attention_weight_min": -85.84968566894531,
      "activations/layer4_attention_weight_max": 98.53013610839844,
      "activations/layer4_attention_weight_min": -92.56005096435547,
      "activations/layer5_attention_weight_max": 107.47088623046875,
      "activations/layer5_attention_weight_min": -91.38406372070312,
      "activations/layer6_attention_weight_max": 61.429080963134766,
      "activations/layer6_attention_weight_min": -53.031089782714844,
      "activations/layer7_attention_weight_max": 70.725830078125,
      "activations/layer7_attention_weight_min": -64.81649780273438,
      "activations/layer8_attention_weight_max": 43.8208122253418,
      "activations/layer8_attention_weight_min": -38.06903839111328,
      "activations/layer9_attention_weight_max": 49.019351959228516,
      "activations/layer9_attention_weight_min": -47.55343246459961,
      "epoch": 4.92,
      "learning_rate": 0.00011944545454545453,
      "loss": 2.8535,
      "step": 84750
    },
    {
      "activations/layer0_attention_weight_max": 15.771950721740723,
      "activations/layer0_attention_weight_min": -14.407415390014648,
      "activations/layer10_attention_weight_max": 35.50023651123047,
      "activations/layer10_attention_weight_min": -33.40772247314453,
      "activations/layer11_attention_weight_max": 28.733259201049805,
      "activations/layer11_attention_weight_min": -28.29018783569336,
      "activations/layer12_attention_weight_max": 22.47730255126953,
      "activations/layer12_attention_weight_min": -25.29360580444336,
      "activations/layer13_attention_weight_max": 46.533138275146484,
      "activations/layer13_attention_weight_min": -38.267181396484375,
      "activations/layer14_attention_weight_max": 40.74936294555664,
      "activations/layer14_attention_weight_min": -31.37933921813965,
      "activations/layer15_attention_weight_max": 41.21247863769531,
      "activations/layer15_attention_weight_min": -28.666181564331055,
      "activations/layer16_attention_weight_max": 39.675113677978516,
      "activations/layer16_attention_weight_min": -25.97695541381836,
      "activations/layer17_attention_weight_max": 54.22356033325195,
      "activations/layer17_attention_weight_min": -36.960723876953125,
      "activations/layer18_attention_weight_max": 47.06258773803711,
      "activations/layer18_attention_weight_min": -32.93458557128906,
      "activations/layer19_attention_weight_max": 25.495656967163086,
      "activations/layer19_attention_weight_min": -20.30322265625,
      "activations/layer1_attention_weight_max": 18.359054565429688,
      "activations/layer1_attention_weight_min": -17.555824279785156,
      "activations/layer20_attention_weight_max": 21.905597686767578,
      "activations/layer20_attention_weight_min": -24.08745574951172,
      "activations/layer21_attention_weight_max": 41.34959411621094,
      "activations/layer21_attention_weight_min": -27.94508934020996,
      "activations/layer22_attention_weight_max": 33.91043472290039,
      "activations/layer22_attention_weight_min": -23.964298248291016,
      "activations/layer23_attention_weight_max": 26.892627716064453,
      "activations/layer23_attention_weight_min": -18.685684204101562,
      "activations/layer2_attention_weight_max": 31.698041915893555,
      "activations/layer2_attention_weight_min": -30.080947875976562,
      "activations/layer3_attention_weight_max": 80.58863830566406,
      "activations/layer3_attention_weight_min": -81.25714111328125,
      "activations/layer4_attention_weight_max": 105.94857025146484,
      "activations/layer4_attention_weight_min": -92.64727020263672,
      "activations/layer5_attention_weight_max": 110.46422576904297,
      "activations/layer5_attention_weight_min": -93.61644744873047,
      "activations/layer6_attention_weight_max": 58.266754150390625,
      "activations/layer6_attention_weight_min": -52.612735748291016,
      "activations/layer7_attention_weight_max": 68.02022552490234,
      "activations/layer7_attention_weight_min": -69.40477752685547,
      "activations/layer8_attention_weight_max": 41.96940994262695,
      "activations/layer8_attention_weight_min": -40.62818908691406,
      "activations/layer9_attention_weight_max": 46.93584442138672,
      "activations/layer9_attention_weight_min": -45.25656509399414,
      "epoch": 4.93,
      "learning_rate": 0.00011942651515151515,
      "loss": 2.8515,
      "step": 84800
    },
    {
      "activations/layer0_attention_weight_max": 13.9866304397583,
      "activations/layer0_attention_weight_min": -14.639981269836426,
      "activations/layer10_attention_weight_max": 39.39532470703125,
      "activations/layer10_attention_weight_min": -37.839874267578125,
      "activations/layer11_attention_weight_max": 31.741308212280273,
      "activations/layer11_attention_weight_min": -27.928081512451172,
      "activations/layer12_attention_weight_max": 29.6320743560791,
      "activations/layer12_attention_weight_min": -26.23210334777832,
      "activations/layer13_attention_weight_max": 52.87342834472656,
      "activations/layer13_attention_weight_min": -37.982666015625,
      "activations/layer14_attention_weight_max": 40.66640853881836,
      "activations/layer14_attention_weight_min": -29.185651779174805,
      "activations/layer15_attention_weight_max": 42.52757263183594,
      "activations/layer15_attention_weight_min": -29.894929885864258,
      "activations/layer16_attention_weight_max": 37.4540901184082,
      "activations/layer16_attention_weight_min": -26.53571891784668,
      "activations/layer17_attention_weight_max": 55.72465896606445,
      "activations/layer17_attention_weight_min": -38.6601448059082,
      "activations/layer18_attention_weight_max": 52.84065628051758,
      "activations/layer18_attention_weight_min": -35.53480911254883,
      "activations/layer19_attention_weight_max": 21.267305374145508,
      "activations/layer19_attention_weight_min": -20.749929428100586,
      "activations/layer1_attention_weight_max": 17.483562469482422,
      "activations/layer1_attention_weight_min": -15.737910270690918,
      "activations/layer20_attention_weight_max": 21.306888580322266,
      "activations/layer20_attention_weight_min": -20.862468719482422,
      "activations/layer21_attention_weight_max": 38.67981719970703,
      "activations/layer21_attention_weight_min": -25.65367317199707,
      "activations/layer22_attention_weight_max": 34.685848236083984,
      "activations/layer22_attention_weight_min": -23.861236572265625,
      "activations/layer23_attention_weight_max": 30.79505729675293,
      "activations/layer23_attention_weight_min": -19.453311920166016,
      "activations/layer2_attention_weight_max": 31.67461395263672,
      "activations/layer2_attention_weight_min": -29.708763122558594,
      "activations/layer3_attention_weight_max": 84.71049499511719,
      "activations/layer3_attention_weight_min": -89.02252960205078,
      "activations/layer4_attention_weight_max": 106.92317199707031,
      "activations/layer4_attention_weight_min": -100.87779235839844,
      "activations/layer5_attention_weight_max": 109.86572265625,
      "activations/layer5_attention_weight_min": -100.96282958984375,
      "activations/layer6_attention_weight_max": 60.028690338134766,
      "activations/layer6_attention_weight_min": -54.73419952392578,
      "activations/layer7_attention_weight_max": 70.43531799316406,
      "activations/layer7_attention_weight_min": -68.99982452392578,
      "activations/layer8_attention_weight_max": 48.47298049926758,
      "activations/layer8_attention_weight_min": -45.816104888916016,
      "activations/layer9_attention_weight_max": 56.35932540893555,
      "activations/layer9_attention_weight_min": -47.495094299316406,
      "epoch": 4.93,
      "learning_rate": 0.00011940757575757573,
      "loss": 2.8649,
      "step": 84850
    },
    {
      "activations/layer0_attention_weight_max": 13.773365020751953,
      "activations/layer0_attention_weight_min": -15.097651481628418,
      "activations/layer10_attention_weight_max": 36.13100814819336,
      "activations/layer10_attention_weight_min": -34.61348342895508,
      "activations/layer11_attention_weight_max": 29.25149154663086,
      "activations/layer11_attention_weight_min": -26.7258358001709,
      "activations/layer12_attention_weight_max": 27.436973571777344,
      "activations/layer12_attention_weight_min": -21.832473754882812,
      "activations/layer13_attention_weight_max": 56.33879470825195,
      "activations/layer13_attention_weight_min": -37.65562057495117,
      "activations/layer14_attention_weight_max": 58.7280387878418,
      "activations/layer14_attention_weight_min": -35.74974060058594,
      "activations/layer15_attention_weight_max": 43.164939880371094,
      "activations/layer15_attention_weight_min": -30.62471580505371,
      "activations/layer16_attention_weight_max": 34.84000015258789,
      "activations/layer16_attention_weight_min": -28.27736473083496,
      "activations/layer17_attention_weight_max": 59.90192413330078,
      "activations/layer17_attention_weight_min": -38.66471862792969,
      "activations/layer18_attention_weight_max": 47.83418655395508,
      "activations/layer18_attention_weight_min": -33.35029602050781,
      "activations/layer19_attention_weight_max": 24.171016693115234,
      "activations/layer19_attention_weight_min": -19.515527725219727,
      "activations/layer1_attention_weight_max": 17.76063346862793,
      "activations/layer1_attention_weight_min": -17.15306854248047,
      "activations/layer20_attention_weight_max": 21.984155654907227,
      "activations/layer20_attention_weight_min": -22.39368438720703,
      "activations/layer21_attention_weight_max": 43.38439178466797,
      "activations/layer21_attention_weight_min": -24.944808959960938,
      "activations/layer22_attention_weight_max": 32.80265426635742,
      "activations/layer22_attention_weight_min": -25.225374221801758,
      "activations/layer23_attention_weight_max": 28.727338790893555,
      "activations/layer23_attention_weight_min": -18.429729461669922,
      "activations/layer2_attention_weight_max": 31.882755279541016,
      "activations/layer2_attention_weight_min": -29.50123405456543,
      "activations/layer3_attention_weight_max": 84.40290832519531,
      "activations/layer3_attention_weight_min": -90.38269805908203,
      "activations/layer4_attention_weight_max": 106.20491790771484,
      "activations/layer4_attention_weight_min": -96.07318115234375,
      "activations/layer5_attention_weight_max": 105.4117431640625,
      "activations/layer5_attention_weight_min": -94.77389526367188,
      "activations/layer6_attention_weight_max": 57.12532424926758,
      "activations/layer6_attention_weight_min": -52.208656311035156,
      "activations/layer7_attention_weight_max": 67.03853607177734,
      "activations/layer7_attention_weight_min": -70.7226791381836,
      "activations/layer8_attention_weight_max": 41.277095794677734,
      "activations/layer8_attention_weight_min": -40.34768295288086,
      "activations/layer9_attention_weight_max": 51.5407829284668,
      "activations/layer9_attention_weight_min": -47.17170333862305,
      "epoch": 4.93,
      "learning_rate": 0.00011938863636363635,
      "loss": 2.8475,
      "step": 84900
    },
    {
      "activations/layer0_attention_weight_max": 15.779912948608398,
      "activations/layer0_attention_weight_min": -15.274099349975586,
      "activations/layer10_attention_weight_max": 41.906951904296875,
      "activations/layer10_attention_weight_min": -37.04515075683594,
      "activations/layer11_attention_weight_max": 30.660114288330078,
      "activations/layer11_attention_weight_min": -29.068801879882812,
      "activations/layer12_attention_weight_max": 23.406883239746094,
      "activations/layer12_attention_weight_min": -23.15659523010254,
      "activations/layer13_attention_weight_max": 73.15995025634766,
      "activations/layer13_attention_weight_min": -42.42771911621094,
      "activations/layer14_attention_weight_max": 66.64627838134766,
      "activations/layer14_attention_weight_min": -42.51829528808594,
      "activations/layer15_attention_weight_max": 46.97489547729492,
      "activations/layer15_attention_weight_min": -31.086761474609375,
      "activations/layer16_attention_weight_max": 40.153350830078125,
      "activations/layer16_attention_weight_min": -26.708698272705078,
      "activations/layer17_attention_weight_max": 64.04576873779297,
      "activations/layer17_attention_weight_min": -38.85905838012695,
      "activations/layer18_attention_weight_max": 52.79649353027344,
      "activations/layer18_attention_weight_min": -34.28866958618164,
      "activations/layer19_attention_weight_max": 22.367319107055664,
      "activations/layer19_attention_weight_min": -20.955001831054688,
      "activations/layer1_attention_weight_max": 18.25318717956543,
      "activations/layer1_attention_weight_min": -14.234748840332031,
      "activations/layer20_attention_weight_max": 24.726451873779297,
      "activations/layer20_attention_weight_min": -21.89200210571289,
      "activations/layer21_attention_weight_max": 40.92694091796875,
      "activations/layer21_attention_weight_min": -26.02361488342285,
      "activations/layer22_attention_weight_max": 35.96458053588867,
      "activations/layer22_attention_weight_min": -24.696611404418945,
      "activations/layer23_attention_weight_max": 31.389081954956055,
      "activations/layer23_attention_weight_min": -16.721424102783203,
      "activations/layer2_attention_weight_max": 29.82301139831543,
      "activations/layer2_attention_weight_min": -29.660179138183594,
      "activations/layer3_attention_weight_max": 86.61321258544922,
      "activations/layer3_attention_weight_min": -88.56238555908203,
      "activations/layer4_attention_weight_max": 107.2966537475586,
      "activations/layer4_attention_weight_min": -104.7745132446289,
      "activations/layer5_attention_weight_max": 104.96734619140625,
      "activations/layer5_attention_weight_min": -99.21080780029297,
      "activations/layer6_attention_weight_max": 58.42899703979492,
      "activations/layer6_attention_weight_min": -54.242733001708984,
      "activations/layer7_attention_weight_max": 73.24500274658203,
      "activations/layer7_attention_weight_min": -69.0835952758789,
      "activations/layer8_attention_weight_max": 45.8781852722168,
      "activations/layer8_attention_weight_min": -41.471248626708984,
      "activations/layer9_attention_weight_max": 56.82954788208008,
      "activations/layer9_attention_weight_min": -48.53744125366211,
      "epoch": 4.94,
      "learning_rate": 0.00011936969696969696,
      "loss": 2.8338,
      "step": 84950
    },
    {
      "activations/layer0_attention_weight_max": 14.52562141418457,
      "activations/layer0_attention_weight_min": -15.27950668334961,
      "activations/layer10_attention_weight_max": 40.98247146606445,
      "activations/layer10_attention_weight_min": -37.279571533203125,
      "activations/layer11_attention_weight_max": 32.196388244628906,
      "activations/layer11_attention_weight_min": -28.607337951660156,
      "activations/layer12_attention_weight_max": 23.7144832611084,
      "activations/layer12_attention_weight_min": -25.092416763305664,
      "activations/layer13_attention_weight_max": 60.62894821166992,
      "activations/layer13_attention_weight_min": -39.801509857177734,
      "activations/layer14_attention_weight_max": 73.68914031982422,
      "activations/layer14_attention_weight_min": -41.321495056152344,
      "activations/layer15_attention_weight_max": 48.27267074584961,
      "activations/layer15_attention_weight_min": -31.68450164794922,
      "activations/layer16_attention_weight_max": 33.473670959472656,
      "activations/layer16_attention_weight_min": -28.101404190063477,
      "activations/layer17_attention_weight_max": 55.31352615356445,
      "activations/layer17_attention_weight_min": -39.81111526489258,
      "activations/layer18_attention_weight_max": 47.09563446044922,
      "activations/layer18_attention_weight_min": -33.6270866394043,
      "activations/layer19_attention_weight_max": 23.317655563354492,
      "activations/layer19_attention_weight_min": -20.604894638061523,
      "activations/layer1_attention_weight_max": 17.489898681640625,
      "activations/layer1_attention_weight_min": -15.084343910217285,
      "activations/layer20_attention_weight_max": 21.513975143432617,
      "activations/layer20_attention_weight_min": -21.028772354125977,
      "activations/layer21_attention_weight_max": 38.83312225341797,
      "activations/layer21_attention_weight_min": -26.58424949645996,
      "activations/layer22_attention_weight_max": 32.22001266479492,
      "activations/layer22_attention_weight_min": -21.86336326599121,
      "activations/layer23_attention_weight_max": 25.959632873535156,
      "activations/layer23_attention_weight_min": -20.494285583496094,
      "activations/layer2_attention_weight_max": 31.856428146362305,
      "activations/layer2_attention_weight_min": -28.99791717529297,
      "activations/layer3_attention_weight_max": 82.78978729248047,
      "activations/layer3_attention_weight_min": -83.56040954589844,
      "activations/layer4_attention_weight_max": 102.8721694946289,
      "activations/layer4_attention_weight_min": -96.99431610107422,
      "activations/layer5_attention_weight_max": 111.39183807373047,
      "activations/layer5_attention_weight_min": -95.60204315185547,
      "activations/layer6_attention_weight_max": 63.0292854309082,
      "activations/layer6_attention_weight_min": -56.14713668823242,
      "activations/layer7_attention_weight_max": 76.54055786132812,
      "activations/layer7_attention_weight_min": -71.397216796875,
      "activations/layer8_attention_weight_max": 48.56929397583008,
      "activations/layer8_attention_weight_min": -44.06339645385742,
      "activations/layer9_attention_weight_max": 58.90861892700195,
      "activations/layer9_attention_weight_min": -52.9454345703125,
      "epoch": 4.94,
      "learning_rate": 0.00011935075757575757,
      "loss": 2.8526,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_loss": 2.794921875,
      "eval_runtime": 8.5,
      "eval_samples_per_second": 505.175,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_openwebtext_loss": 2.794921875,
      "eval_openwebtext_ppl": 16.36135047271935,
      "eval_openwebtext_runtime": 8.5,
      "eval_openwebtext_samples_per_second": 505.175,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_wikitext_loss": 3.025390625,
      "eval_wikitext_ppl": 20.602050815688813,
      "eval_wikitext_runtime": 1.9156,
      "eval_wikitext_samples_per_second": 238.044,
      "step": 85000
    },
    {
      "epoch": 4.94,
      "eval_lambada_loss": 2.7890625,
      "eval_lambada_ppl": 16.265763498233948,
      "eval_lambada_runtime": 9.534,
      "eval_lambada_samples_per_second": 510.7,
      "step": 85000
    },
    {
      "activations/layer0_attention_weight_max": 14.539346694946289,
      "activations/layer0_attention_weight_min": -15.219564437866211,
      "activations/layer10_attention_weight_max": 44.89950180053711,
      "activations/layer10_attention_weight_min": -36.24187088012695,
      "activations/layer11_attention_weight_max": 37.22248458862305,
      "activations/layer11_attention_weight_min": -27.391094207763672,
      "activations/layer12_attention_weight_max": 24.570505142211914,
      "activations/layer12_attention_weight_min": -27.49651527404785,
      "activations/layer13_attention_weight_max": 65.09554290771484,
      "activations/layer13_attention_weight_min": -39.7684211730957,
      "activations/layer14_attention_weight_max": 71.22716522216797,
      "activations/layer14_attention_weight_min": -41.174224853515625,
      "activations/layer15_attention_weight_max": 57.97419738769531,
      "activations/layer15_attention_weight_min": -34.61223602294922,
      "activations/layer16_attention_weight_max": 42.28135299682617,
      "activations/layer16_attention_weight_min": -25.36760902404785,
      "activations/layer17_attention_weight_max": 55.41679763793945,
      "activations/layer17_attention_weight_min": -39.896095275878906,
      "activations/layer18_attention_weight_max": 47.23792266845703,
      "activations/layer18_attention_weight_min": -35.20802688598633,
      "activations/layer19_attention_weight_max": 25.63263511657715,
      "activations/layer19_attention_weight_min": -19.70871353149414,
      "activations/layer1_attention_weight_max": 18.603137969970703,
      "activations/layer1_attention_weight_min": -18.05353546142578,
      "activations/layer20_attention_weight_max": 25.90028953552246,
      "activations/layer20_attention_weight_min": -23.309486389160156,
      "activations/layer21_attention_weight_max": 44.027923583984375,
      "activations/layer21_attention_weight_min": -23.07394027709961,
      "activations/layer22_attention_weight_max": 34.05963134765625,
      "activations/layer22_attention_weight_min": -22.240516662597656,
      "activations/layer23_attention_weight_max": 28.702423095703125,
      "activations/layer23_attention_weight_min": -17.19179916381836,
      "activations/layer2_attention_weight_max": 31.262676239013672,
      "activations/layer2_attention_weight_min": -30.907997131347656,
      "activations/layer3_attention_weight_max": 82.49710845947266,
      "activations/layer3_attention_weight_min": -87.30751037597656,
      "activations/layer4_attention_weight_max": 108.69168853759766,
      "activations/layer4_attention_weight_min": -100.9778060913086,
      "activations/layer5_attention_weight_max": 112.35994720458984,
      "activations/layer5_attention_weight_min": -98.72834777832031,
      "activations/layer6_attention_weight_max": 62.70566177368164,
      "activations/layer6_attention_weight_min": -54.93745803833008,
      "activations/layer7_attention_weight_max": 84.09496307373047,
      "activations/layer7_attention_weight_min": -71.54715728759766,
      "activations/layer8_attention_weight_max": 52.830501556396484,
      "activations/layer8_attention_weight_min": -43.647064208984375,
      "activations/layer9_attention_weight_max": 68.37407684326172,
      "activations/layer9_attention_weight_min": -55.24782943725586,
      "epoch": 4.94,
      "learning_rate": 0.00011933181818181817,
      "loss": 2.817,
      "step": 85050
    },
    {
      "activations/layer0_attention_weight_max": 15.08285140991211,
      "activations/layer0_attention_weight_min": -14.978119850158691,
      "activations/layer10_attention_weight_max": 40.69477081298828,
      "activations/layer10_attention_weight_min": -36.26149368286133,
      "activations/layer11_attention_weight_max": 30.52638816833496,
      "activations/layer11_attention_weight_min": -28.518239974975586,
      "activations/layer12_attention_weight_max": 21.75737190246582,
      "activations/layer12_attention_weight_min": -26.534547805786133,
      "activations/layer13_attention_weight_max": 46.43899917602539,
      "activations/layer13_attention_weight_min": -33.89153289794922,
      "activations/layer14_attention_weight_max": 45.24113464355469,
      "activations/layer14_attention_weight_min": -30.72744369506836,
      "activations/layer15_attention_weight_max": 38.61246109008789,
      "activations/layer15_attention_weight_min": -29.66519546508789,
      "activations/layer16_attention_weight_max": 40.02696228027344,
      "activations/layer16_attention_weight_min": -25.576330184936523,
      "activations/layer17_attention_weight_max": 56.63820266723633,
      "activations/layer17_attention_weight_min": -39.50801467895508,
      "activations/layer18_attention_weight_max": 50.23107147216797,
      "activations/layer18_attention_weight_min": -33.61727523803711,
      "activations/layer19_attention_weight_max": 21.723392486572266,
      "activations/layer19_attention_weight_min": -19.550655364990234,
      "activations/layer1_attention_weight_max": 17.895936965942383,
      "activations/layer1_attention_weight_min": -16.558238983154297,
      "activations/layer20_attention_weight_max": 22.28952980041504,
      "activations/layer20_attention_weight_min": -22.995107650756836,
      "activations/layer21_attention_weight_max": 44.66371536254883,
      "activations/layer21_attention_weight_min": -24.750757217407227,
      "activations/layer22_attention_weight_max": 37.42254638671875,
      "activations/layer22_attention_weight_min": -25.09758186340332,
      "activations/layer23_attention_weight_max": 28.810361862182617,
      "activations/layer23_attention_weight_min": -17.244903564453125,
      "activations/layer2_attention_weight_max": 30.036436080932617,
      "activations/layer2_attention_weight_min": -28.770483016967773,
      "activations/layer3_attention_weight_max": 84.08180236816406,
      "activations/layer3_attention_weight_min": -84.02439880371094,
      "activations/layer4_attention_weight_max": 109.79668426513672,
      "activations/layer4_attention_weight_min": -96.33193969726562,
      "activations/layer5_attention_weight_max": 113.41780853271484,
      "activations/layer5_attention_weight_min": -94.28462982177734,
      "activations/layer6_attention_weight_max": 60.41675567626953,
      "activations/layer6_attention_weight_min": -52.323631286621094,
      "activations/layer7_attention_weight_max": 77.34532928466797,
      "activations/layer7_attention_weight_min": -67.5213623046875,
      "activations/layer8_attention_weight_max": 47.01760482788086,
      "activations/layer8_attention_weight_min": -39.947357177734375,
      "activations/layer9_attention_weight_max": 63.84197998046875,
      "activations/layer9_attention_weight_min": -57.26583480834961,
      "epoch": 4.94,
      "learning_rate": 0.00011931287878787878,
      "loss": 2.8303,
      "step": 85100
    },
    {
      "activations/layer0_attention_weight_max": 14.643952369689941,
      "activations/layer0_attention_weight_min": -14.925859451293945,
      "activations/layer10_attention_weight_max": 39.391571044921875,
      "activations/layer10_attention_weight_min": -35.37372589111328,
      "activations/layer11_attention_weight_max": 32.89793395996094,
      "activations/layer11_attention_weight_min": -28.53382682800293,
      "activations/layer12_attention_weight_max": 23.3259220123291,
      "activations/layer12_attention_weight_min": -25.259550094604492,
      "activations/layer13_attention_weight_max": 56.722904205322266,
      "activations/layer13_attention_weight_min": -45.25779342651367,
      "activations/layer14_attention_weight_max": 49.20753860473633,
      "activations/layer14_attention_weight_min": -32.360748291015625,
      "activations/layer15_attention_weight_max": 43.09047317504883,
      "activations/layer15_attention_weight_min": -31.791852951049805,
      "activations/layer16_attention_weight_max": 39.71214294433594,
      "activations/layer16_attention_weight_min": -27.678972244262695,
      "activations/layer17_attention_weight_max": 61.64003372192383,
      "activations/layer17_attention_weight_min": -39.3736572265625,
      "activations/layer18_attention_weight_max": 60.04335403442383,
      "activations/layer18_attention_weight_min": -36.35599899291992,
      "activations/layer19_attention_weight_max": 24.232158660888672,
      "activations/layer19_attention_weight_min": -21.1501522064209,
      "activations/layer1_attention_weight_max": 16.963863372802734,
      "activations/layer1_attention_weight_min": -15.163847923278809,
      "activations/layer20_attention_weight_max": 23.009977340698242,
      "activations/layer20_attention_weight_min": -24.945995330810547,
      "activations/layer21_attention_weight_max": 45.63508987426758,
      "activations/layer21_attention_weight_min": -29.284543991088867,
      "activations/layer22_attention_weight_max": 36.82798385620117,
      "activations/layer22_attention_weight_min": -23.35017204284668,
      "activations/layer23_attention_weight_max": 29.577741622924805,
      "activations/layer23_attention_weight_min": -17.170028686523438,
      "activations/layer2_attention_weight_max": 29.926502227783203,
      "activations/layer2_attention_weight_min": -28.944812774658203,
      "activations/layer3_attention_weight_max": 82.61893463134766,
      "activations/layer3_attention_weight_min": -85.99066925048828,
      "activations/layer4_attention_weight_max": 103.72835540771484,
      "activations/layer4_attention_weight_min": -99.10253143310547,
      "activations/layer5_attention_weight_max": 111.57038879394531,
      "activations/layer5_attention_weight_min": -96.77205657958984,
      "activations/layer6_attention_weight_max": 59.45734405517578,
      "activations/layer6_attention_weight_min": -54.633113861083984,
      "activations/layer7_attention_weight_max": 79.8164291381836,
      "activations/layer7_attention_weight_min": -69.020263671875,
      "activations/layer8_attention_weight_max": 44.01331329345703,
      "activations/layer8_attention_weight_min": -44.545066833496094,
      "activations/layer9_attention_weight_max": 51.12665557861328,
      "activations/layer9_attention_weight_min": -47.18043518066406,
      "epoch": 4.95,
      "learning_rate": 0.00011929393939393938,
      "loss": 2.8264,
      "step": 85150
    },
    {
      "activations/layer0_attention_weight_max": 13.948989868164062,
      "activations/layer0_attention_weight_min": -15.00444507598877,
      "activations/layer10_attention_weight_max": 42.59029006958008,
      "activations/layer10_attention_weight_min": -37.98529815673828,
      "activations/layer11_attention_weight_max": 34.692237854003906,
      "activations/layer11_attention_weight_min": -29.97870635986328,
      "activations/layer12_attention_weight_max": 24.016748428344727,
      "activations/layer12_attention_weight_min": -25.380817413330078,
      "activations/layer13_attention_weight_max": 49.71870040893555,
      "activations/layer13_attention_weight_min": -36.72748565673828,
      "activations/layer14_attention_weight_max": 50.12693405151367,
      "activations/layer14_attention_weight_min": -30.8853816986084,
      "activations/layer15_attention_weight_max": 47.590335845947266,
      "activations/layer15_attention_weight_min": -32.772483825683594,
      "activations/layer16_attention_weight_max": 35.86542510986328,
      "activations/layer16_attention_weight_min": -27.16442108154297,
      "activations/layer17_attention_weight_max": 55.90085983276367,
      "activations/layer17_attention_weight_min": -39.379356384277344,
      "activations/layer18_attention_weight_max": 52.41204833984375,
      "activations/layer18_attention_weight_min": -36.570804595947266,
      "activations/layer19_attention_weight_max": 23.458520889282227,
      "activations/layer19_attention_weight_min": -20.06220245361328,
      "activations/layer1_attention_weight_max": 16.211009979248047,
      "activations/layer1_attention_weight_min": -15.526924133300781,
      "activations/layer20_attention_weight_max": 23.291114807128906,
      "activations/layer20_attention_weight_min": -22.057775497436523,
      "activations/layer21_attention_weight_max": 39.933441162109375,
      "activations/layer21_attention_weight_min": -24.7612361907959,
      "activations/layer22_attention_weight_max": 36.514366149902344,
      "activations/layer22_attention_weight_min": -23.292593002319336,
      "activations/layer23_attention_weight_max": 29.935842514038086,
      "activations/layer23_attention_weight_min": -19.482500076293945,
      "activations/layer2_attention_weight_max": 30.546262741088867,
      "activations/layer2_attention_weight_min": -29.25897789001465,
      "activations/layer3_attention_weight_max": 82.68213653564453,
      "activations/layer3_attention_weight_min": -89.5700912475586,
      "activations/layer4_attention_weight_max": 105.2750473022461,
      "activations/layer4_attention_weight_min": -102.12273406982422,
      "activations/layer5_attention_weight_max": 110.34799194335938,
      "activations/layer5_attention_weight_min": -103.43801879882812,
      "activations/layer6_attention_weight_max": 60.226417541503906,
      "activations/layer6_attention_weight_min": -58.969451904296875,
      "activations/layer7_attention_weight_max": 77.69491577148438,
      "activations/layer7_attention_weight_min": -70.80693054199219,
      "activations/layer8_attention_weight_max": 46.707088470458984,
      "activations/layer8_attention_weight_min": -43.58457565307617,
      "activations/layer9_attention_weight_max": 54.45085525512695,
      "activations/layer9_attention_weight_min": -49.606197357177734,
      "epoch": 4.95,
      "learning_rate": 0.000119275,
      "loss": 2.8352,
      "step": 85200
    },
    {
      "activations/layer0_attention_weight_max": 14.364429473876953,
      "activations/layer0_attention_weight_min": -15.309296607971191,
      "activations/layer10_attention_weight_max": 38.93037033081055,
      "activations/layer10_attention_weight_min": -36.732547760009766,
      "activations/layer11_attention_weight_max": 29.22179412841797,
      "activations/layer11_attention_weight_min": -27.00984764099121,
      "activations/layer12_attention_weight_max": 27.873212814331055,
      "activations/layer12_attention_weight_min": -25.866418838500977,
      "activations/layer13_attention_weight_max": 50.909549713134766,
      "activations/layer13_attention_weight_min": -34.757286071777344,
      "activations/layer14_attention_weight_max": 37.190101623535156,
      "activations/layer14_attention_weight_min": -29.826763153076172,
      "activations/layer15_attention_weight_max": 46.08327865600586,
      "activations/layer15_attention_weight_min": -29.914152145385742,
      "activations/layer16_attention_weight_max": 49.4119758605957,
      "activations/layer16_attention_weight_min": -28.455097198486328,
      "activations/layer17_attention_weight_max": 62.13411331176758,
      "activations/layer17_attention_weight_min": -41.31699752807617,
      "activations/layer18_attention_weight_max": 54.26138687133789,
      "activations/layer18_attention_weight_min": -36.001312255859375,
      "activations/layer19_attention_weight_max": 23.134347915649414,
      "activations/layer19_attention_weight_min": -21.150753021240234,
      "activations/layer1_attention_weight_max": 17.635046005249023,
      "activations/layer1_attention_weight_min": -14.307430267333984,
      "activations/layer20_attention_weight_max": 20.828432083129883,
      "activations/layer20_attention_weight_min": -19.574289321899414,
      "activations/layer21_attention_weight_max": 46.50865173339844,
      "activations/layer21_attention_weight_min": -25.52080535888672,
      "activations/layer22_attention_weight_max": 36.399139404296875,
      "activations/layer22_attention_weight_min": -22.96419906616211,
      "activations/layer23_attention_weight_max": 30.35629653930664,
      "activations/layer23_attention_weight_min": -17.889772415161133,
      "activations/layer2_attention_weight_max": 30.20431137084961,
      "activations/layer2_attention_weight_min": -30.62820816040039,
      "activations/layer3_attention_weight_max": 82.27861785888672,
      "activations/layer3_attention_weight_min": -90.43291473388672,
      "activations/layer4_attention_weight_max": 105.8475570678711,
      "activations/layer4_attention_weight_min": -98.97032165527344,
      "activations/layer5_attention_weight_max": 109.76799011230469,
      "activations/layer5_attention_weight_min": -97.50177001953125,
      "activations/layer6_attention_weight_max": 59.86893844604492,
      "activations/layer6_attention_weight_min": -54.264869689941406,
      "activations/layer7_attention_weight_max": 71.514892578125,
      "activations/layer7_attention_weight_min": -70.54631042480469,
      "activations/layer8_attention_weight_max": 44.23404312133789,
      "activations/layer8_attention_weight_min": -42.94314956665039,
      "activations/layer9_attention_weight_max": 59.27279281616211,
      "activations/layer9_attention_weight_min": -47.88802719116211,
      "epoch": 4.95,
      "learning_rate": 0.00011925606060606059,
      "loss": 2.8241,
      "step": 85250
    },
    {
      "activations/layer0_attention_weight_max": 15.849398612976074,
      "activations/layer0_attention_weight_min": -14.598209381103516,
      "activations/layer10_attention_weight_max": 39.693939208984375,
      "activations/layer10_attention_weight_min": -34.022621154785156,
      "activations/layer11_attention_weight_max": 30.779720306396484,
      "activations/layer11_attention_weight_min": -28.06056785583496,
      "activations/layer12_attention_weight_max": 23.702789306640625,
      "activations/layer12_attention_weight_min": -24.022613525390625,
      "activations/layer13_attention_weight_max": 60.58567428588867,
      "activations/layer13_attention_weight_min": -38.760231018066406,
      "activations/layer14_attention_weight_max": 51.5572624206543,
      "activations/layer14_attention_weight_min": -34.82610321044922,
      "activations/layer15_attention_weight_max": 41.734230041503906,
      "activations/layer15_attention_weight_min": -29.280345916748047,
      "activations/layer16_attention_weight_max": 38.80763244628906,
      "activations/layer16_attention_weight_min": -25.540912628173828,
      "activations/layer17_attention_weight_max": 61.23908233642578,
      "activations/layer17_attention_weight_min": -36.73381423950195,
      "activations/layer18_attention_weight_max": 51.320655822753906,
      "activations/layer18_attention_weight_min": -32.38644027709961,
      "activations/layer19_attention_weight_max": 21.91280746459961,
      "activations/layer19_attention_weight_min": -20.423322677612305,
      "activations/layer1_attention_weight_max": 17.190126419067383,
      "activations/layer1_attention_weight_min": -14.789161682128906,
      "activations/layer20_attention_weight_max": 23.072452545166016,
      "activations/layer20_attention_weight_min": -21.641254425048828,
      "activations/layer21_attention_weight_max": 46.59730911254883,
      "activations/layer21_attention_weight_min": -27.51528549194336,
      "activations/layer22_attention_weight_max": 35.80049133300781,
      "activations/layer22_attention_weight_min": -23.26389503479004,
      "activations/layer23_attention_weight_max": 31.209943771362305,
      "activations/layer23_attention_weight_min": -20.475351333618164,
      "activations/layer2_attention_weight_max": 31.063901901245117,
      "activations/layer2_attention_weight_min": -28.547500610351562,
      "activations/layer3_attention_weight_max": 84.77520751953125,
      "activations/layer3_attention_weight_min": -87.5227279663086,
      "activations/layer4_attention_weight_max": 105.34968566894531,
      "activations/layer4_attention_weight_min": -97.0644760131836,
      "activations/layer5_attention_weight_max": 107.85577392578125,
      "activations/layer5_attention_weight_min": -93.48818969726562,
      "activations/layer6_attention_weight_max": 63.018211364746094,
      "activations/layer6_attention_weight_min": -56.90480041503906,
      "activations/layer7_attention_weight_max": 73.47500610351562,
      "activations/layer7_attention_weight_min": -67.2742919921875,
      "activations/layer8_attention_weight_max": 45.38330078125,
      "activations/layer8_attention_weight_min": -41.29486846923828,
      "activations/layer9_attention_weight_max": 53.907257080078125,
      "activations/layer9_attention_weight_min": -47.0617790222168,
      "epoch": 4.96,
      "learning_rate": 0.0001192371212121212,
      "loss": 2.8239,
      "step": 85300
    },
    {
      "activations/layer0_attention_weight_max": 14.751775741577148,
      "activations/layer0_attention_weight_min": -14.614258766174316,
      "activations/layer10_attention_weight_max": 38.749027252197266,
      "activations/layer10_attention_weight_min": -35.555057525634766,
      "activations/layer11_attention_weight_max": 34.35185241699219,
      "activations/layer11_attention_weight_min": -26.924728393554688,
      "activations/layer12_attention_weight_max": 22.484073638916016,
      "activations/layer12_attention_weight_min": -23.734859466552734,
      "activations/layer13_attention_weight_max": 53.857994079589844,
      "activations/layer13_attention_weight_min": -37.78501892089844,
      "activations/layer14_attention_weight_max": 50.7567253112793,
      "activations/layer14_attention_weight_min": -30.483842849731445,
      "activations/layer15_attention_weight_max": 50.97499465942383,
      "activations/layer15_attention_weight_min": -30.99198341369629,
      "activations/layer16_attention_weight_max": 35.024505615234375,
      "activations/layer16_attention_weight_min": -25.776607513427734,
      "activations/layer17_attention_weight_max": 62.89719772338867,
      "activations/layer17_attention_weight_min": -40.221710205078125,
      "activations/layer18_attention_weight_max": 52.59650802612305,
      "activations/layer18_attention_weight_min": -35.08222961425781,
      "activations/layer19_attention_weight_max": 23.64119529724121,
      "activations/layer19_attention_weight_min": -19.50503921508789,
      "activations/layer1_attention_weight_max": 18.135995864868164,
      "activations/layer1_attention_weight_min": -13.831759452819824,
      "activations/layer20_attention_weight_max": 24.064725875854492,
      "activations/layer20_attention_weight_min": -20.095611572265625,
      "activations/layer21_attention_weight_max": 40.712162017822266,
      "activations/layer21_attention_weight_min": -23.889995574951172,
      "activations/layer22_attention_weight_max": 39.158390045166016,
      "activations/layer22_attention_weight_min": -23.56698226928711,
      "activations/layer23_attention_weight_max": 29.014408111572266,
      "activations/layer23_attention_weight_min": -19.029611587524414,
      "activations/layer2_attention_weight_max": 30.23805046081543,
      "activations/layer2_attention_weight_min": -27.755931854248047,
      "activations/layer3_attention_weight_max": 80.56578063964844,
      "activations/layer3_attention_weight_min": -83.9801025390625,
      "activations/layer4_attention_weight_max": 105.6551284790039,
      "activations/layer4_attention_weight_min": -97.21649169921875,
      "activations/layer5_attention_weight_max": 106.8695068359375,
      "activations/layer5_attention_weight_min": -95.31319427490234,
      "activations/layer6_attention_weight_max": 59.255802154541016,
      "activations/layer6_attention_weight_min": -54.71669006347656,
      "activations/layer7_attention_weight_max": 70.90602111816406,
      "activations/layer7_attention_weight_min": -73.5910415649414,
      "activations/layer8_attention_weight_max": 42.76788330078125,
      "activations/layer8_attention_weight_min": -41.27878952026367,
      "activations/layer9_attention_weight_max": 50.43549346923828,
      "activations/layer9_attention_weight_min": -43.85295867919922,
      "epoch": 4.96,
      "learning_rate": 0.00011921818181818182,
      "loss": 2.8407,
      "step": 85350
    },
    {
      "activations/layer0_attention_weight_max": 15.389669418334961,
      "activations/layer0_attention_weight_min": -15.154496192932129,
      "activations/layer10_attention_weight_max": 37.485591888427734,
      "activations/layer10_attention_weight_min": -33.72542190551758,
      "activations/layer11_attention_weight_max": 30.24202537536621,
      "activations/layer11_attention_weight_min": -26.797786712646484,
      "activations/layer12_attention_weight_max": 23.486038208007812,
      "activations/layer12_attention_weight_min": -23.83582878112793,
      "activations/layer13_attention_weight_max": 48.14297866821289,
      "activations/layer13_attention_weight_min": -33.40314865112305,
      "activations/layer14_attention_weight_max": 45.35982894897461,
      "activations/layer14_attention_weight_min": -29.893178939819336,
      "activations/layer15_attention_weight_max": 48.425933837890625,
      "activations/layer15_attention_weight_min": -30.450511932373047,
      "activations/layer16_attention_weight_max": 35.98002243041992,
      "activations/layer16_attention_weight_min": -27.080034255981445,
      "activations/layer17_attention_weight_max": 55.137027740478516,
      "activations/layer17_attention_weight_min": -38.40385055541992,
      "activations/layer18_attention_weight_max": 54.133487701416016,
      "activations/layer18_attention_weight_min": -35.16741180419922,
      "activations/layer19_attention_weight_max": 20.89921760559082,
      "activations/layer19_attention_weight_min": -20.076539993286133,
      "activations/layer1_attention_weight_max": 16.511667251586914,
      "activations/layer1_attention_weight_min": -16.126174926757812,
      "activations/layer20_attention_weight_max": 23.533227920532227,
      "activations/layer20_attention_weight_min": -19.839263916015625,
      "activations/layer21_attention_weight_max": 40.32834243774414,
      "activations/layer21_attention_weight_min": -26.919784545898438,
      "activations/layer22_attention_weight_max": 35.72194290161133,
      "activations/layer22_attention_weight_min": -23.225170135498047,
      "activations/layer23_attention_weight_max": 25.96630859375,
      "activations/layer23_attention_weight_min": -18.128376007080078,
      "activations/layer2_attention_weight_max": 32.06852722167969,
      "activations/layer2_attention_weight_min": -29.26228904724121,
      "activations/layer3_attention_weight_max": 82.19161987304688,
      "activations/layer3_attention_weight_min": -84.76750183105469,
      "activations/layer4_attention_weight_max": 103.88641357421875,
      "activations/layer4_attention_weight_min": -97.56356048583984,
      "activations/layer5_attention_weight_max": 107.4345932006836,
      "activations/layer5_attention_weight_min": -97.5554428100586,
      "activations/layer6_attention_weight_max": 59.81917190551758,
      "activations/layer6_attention_weight_min": -52.768394470214844,
      "activations/layer7_attention_weight_max": 75.53328704833984,
      "activations/layer7_attention_weight_min": -69.75482177734375,
      "activations/layer8_attention_weight_max": 43.1530647277832,
      "activations/layer8_attention_weight_min": -42.128787994384766,
      "activations/layer9_attention_weight_max": 55.119850158691406,
      "activations/layer9_attention_weight_min": -47.29282760620117,
      "epoch": 4.96,
      "learning_rate": 0.0001191992424242424,
      "loss": 2.8373,
      "step": 85400
    },
    {
      "activations/layer0_attention_weight_max": 14.780001640319824,
      "activations/layer0_attention_weight_min": -15.001851081848145,
      "activations/layer10_attention_weight_max": 43.84706115722656,
      "activations/layer10_attention_weight_min": -38.153648376464844,
      "activations/layer11_attention_weight_max": 35.859195709228516,
      "activations/layer11_attention_weight_min": -28.915424346923828,
      "activations/layer12_attention_weight_max": 23.33582878112793,
      "activations/layer12_attention_weight_min": -26.712614059448242,
      "activations/layer13_attention_weight_max": 68.60397338867188,
      "activations/layer13_attention_weight_min": -45.371498107910156,
      "activations/layer14_attention_weight_max": 70.59024810791016,
      "activations/layer14_attention_weight_min": -35.314754486083984,
      "activations/layer15_attention_weight_max": 52.612979888916016,
      "activations/layer15_attention_weight_min": -29.873485565185547,
      "activations/layer16_attention_weight_max": 48.55008316040039,
      "activations/layer16_attention_weight_min": -27.270002365112305,
      "activations/layer17_attention_weight_max": 69.52490997314453,
      "activations/layer17_attention_weight_min": -43.683162689208984,
      "activations/layer18_attention_weight_max": 59.799617767333984,
      "activations/layer18_attention_weight_min": -34.48312759399414,
      "activations/layer19_attention_weight_max": 29.630996704101562,
      "activations/layer19_attention_weight_min": -21.59033966064453,
      "activations/layer1_attention_weight_max": 16.79945945739746,
      "activations/layer1_attention_weight_min": -14.102081298828125,
      "activations/layer20_attention_weight_max": 29.994333267211914,
      "activations/layer20_attention_weight_min": -22.3656063079834,
      "activations/layer21_attention_weight_max": 56.366790771484375,
      "activations/layer21_attention_weight_min": -24.297618865966797,
      "activations/layer22_attention_weight_max": 48.77729797363281,
      "activations/layer22_attention_weight_min": -24.10997200012207,
      "activations/layer23_attention_weight_max": 37.35546875,
      "activations/layer23_attention_weight_min": -18.176116943359375,
      "activations/layer2_attention_weight_max": 30.841087341308594,
      "activations/layer2_attention_weight_min": -28.38669204711914,
      "activations/layer3_attention_weight_max": 86.72405242919922,
      "activations/layer3_attention_weight_min": -87.4307861328125,
      "activations/layer4_attention_weight_max": 107.34856414794922,
      "activations/layer4_attention_weight_min": -101.31857299804688,
      "activations/layer5_attention_weight_max": 105.14561462402344,
      "activations/layer5_attention_weight_min": -99.91435241699219,
      "activations/layer6_attention_weight_max": 60.238590240478516,
      "activations/layer6_attention_weight_min": -53.61001205444336,
      "activations/layer7_attention_weight_max": 77.65123748779297,
      "activations/layer7_attention_weight_min": -79.35799407958984,
      "activations/layer8_attention_weight_max": 45.92921829223633,
      "activations/layer8_attention_weight_min": -44.95100402832031,
      "activations/layer9_attention_weight_max": 59.07114791870117,
      "activations/layer9_attention_weight_min": -54.00392532348633,
      "epoch": 4.97,
      "learning_rate": 0.0001191806818181818,
      "loss": 2.8428,
      "step": 85450
    },
    {
      "activations/layer0_attention_weight_max": 14.410778999328613,
      "activations/layer0_attention_weight_min": -15.171978950500488,
      "activations/layer10_attention_weight_max": 39.95172882080078,
      "activations/layer10_attention_weight_min": -37.704349517822266,
      "activations/layer11_attention_weight_max": 29.943105697631836,
      "activations/layer11_attention_weight_min": -29.95931625366211,
      "activations/layer12_attention_weight_max": 20.083349227905273,
      "activations/layer12_attention_weight_min": -24.955350875854492,
      "activations/layer13_attention_weight_max": 50.76304244995117,
      "activations/layer13_attention_weight_min": -36.170249938964844,
      "activations/layer14_attention_weight_max": 40.66347122192383,
      "activations/layer14_attention_weight_min": -30.166725158691406,
      "activations/layer15_attention_weight_max": 46.261077880859375,
      "activations/layer15_attention_weight_min": -32.31113052368164,
      "activations/layer16_attention_weight_max": 37.35615539550781,
      "activations/layer16_attention_weight_min": -26.51337242126465,
      "activations/layer17_attention_weight_max": 55.65291976928711,
      "activations/layer17_attention_weight_min": -38.85348129272461,
      "activations/layer18_attention_weight_max": 48.9013786315918,
      "activations/layer18_attention_weight_min": -33.22623062133789,
      "activations/layer19_attention_weight_max": 21.52120018005371,
      "activations/layer19_attention_weight_min": -19.82988166809082,
      "activations/layer1_attention_weight_max": 17.18282699584961,
      "activations/layer1_attention_weight_min": -14.695655822753906,
      "activations/layer20_attention_weight_max": 20.41671371459961,
      "activations/layer20_attention_weight_min": -20.986862182617188,
      "activations/layer21_attention_weight_max": 36.634620666503906,
      "activations/layer21_attention_weight_min": -26.397628784179688,
      "activations/layer22_attention_weight_max": 32.34248733520508,
      "activations/layer22_attention_weight_min": -24.54483985900879,
      "activations/layer23_attention_weight_max": 27.896223068237305,
      "activations/layer23_attention_weight_min": -18.557025909423828,
      "activations/layer2_attention_weight_max": 29.43347930908203,
      "activations/layer2_attention_weight_min": -27.8315486907959,
      "activations/layer3_attention_weight_max": 80.14446258544922,
      "activations/layer3_attention_weight_min": -84.58416748046875,
      "activations/layer4_attention_weight_max": 99.6534652709961,
      "activations/layer4_attention_weight_min": -96.9392318725586,
      "activations/layer5_attention_weight_max": 103.07696533203125,
      "activations/layer5_attention_weight_min": -98.2163314819336,
      "activations/layer6_attention_weight_max": 58.72428512573242,
      "activations/layer6_attention_weight_min": -54.654335021972656,
      "activations/layer7_attention_weight_max": 72.02938842773438,
      "activations/layer7_attention_weight_min": -71.9096450805664,
      "activations/layer8_attention_weight_max": 43.88025665283203,
      "activations/layer8_attention_weight_min": -45.998600006103516,
      "activations/layer9_attention_weight_max": 50.19744110107422,
      "activations/layer9_attention_weight_min": -47.950958251953125,
      "epoch": 4.97,
      "learning_rate": 0.00011916174242424241,
      "loss": 2.8327,
      "step": 85500
    },
    {
      "activations/layer0_attention_weight_max": 15.140771865844727,
      "activations/layer0_attention_weight_min": -15.086874961853027,
      "activations/layer10_attention_weight_max": 37.871124267578125,
      "activations/layer10_attention_weight_min": -32.620361328125,
      "activations/layer11_attention_weight_max": 28.410865783691406,
      "activations/layer11_attention_weight_min": -26.372276306152344,
      "activations/layer12_attention_weight_max": 19.8714599609375,
      "activations/layer12_attention_weight_min": -23.62911605834961,
      "activations/layer13_attention_weight_max": 44.25314712524414,
      "activations/layer13_attention_weight_min": -36.356204986572266,
      "activations/layer14_attention_weight_max": 40.98662185668945,
      "activations/layer14_attention_weight_min": -29.629514694213867,
      "activations/layer15_attention_weight_max": 38.084354400634766,
      "activations/layer15_attention_weight_min": -30.39906120300293,
      "activations/layer16_attention_weight_max": 35.13863754272461,
      "activations/layer16_attention_weight_min": -30.51123809814453,
      "activations/layer17_attention_weight_max": 52.72737503051758,
      "activations/layer17_attention_weight_min": -38.3889045715332,
      "activations/layer18_attention_weight_max": 49.37889862060547,
      "activations/layer18_attention_weight_min": -34.41865158081055,
      "activations/layer19_attention_weight_max": 19.620243072509766,
      "activations/layer19_attention_weight_min": -20.555715560913086,
      "activations/layer1_attention_weight_max": 17.118989944458008,
      "activations/layer1_attention_weight_min": -18.923416137695312,
      "activations/layer20_attention_weight_max": 21.053308486938477,
      "activations/layer20_attention_weight_min": -21.51568603515625,
      "activations/layer21_attention_weight_max": 40.1570930480957,
      "activations/layer21_attention_weight_min": -25.94562339782715,
      "activations/layer22_attention_weight_max": 32.466243743896484,
      "activations/layer22_attention_weight_min": -25.07081413269043,
      "activations/layer23_attention_weight_max": 26.046606063842773,
      "activations/layer23_attention_weight_min": -17.8306827545166,
      "activations/layer2_attention_weight_max": 31.19074821472168,
      "activations/layer2_attention_weight_min": -29.254003524780273,
      "activations/layer3_attention_weight_max": 83.3661880493164,
      "activations/layer3_attention_weight_min": -86.85130310058594,
      "activations/layer4_attention_weight_max": 103.5728988647461,
      "activations/layer4_attention_weight_min": -100.78215026855469,
      "activations/layer5_attention_weight_max": 110.41466522216797,
      "activations/layer5_attention_weight_min": -97.96380615234375,
      "activations/layer6_attention_weight_max": 61.08135223388672,
      "activations/layer6_attention_weight_min": -55.85271072387695,
      "activations/layer7_attention_weight_max": 71.6215591430664,
      "activations/layer7_attention_weight_min": -70.72725677490234,
      "activations/layer8_attention_weight_max": 42.757015228271484,
      "activations/layer8_attention_weight_min": -40.00399398803711,
      "activations/layer9_attention_weight_max": 48.76161193847656,
      "activations/layer9_attention_weight_min": -46.5037956237793,
      "epoch": 4.97,
      "learning_rate": 0.00011914280303030303,
      "loss": 2.8364,
      "step": 85550
    },
    {
      "activations/layer0_attention_weight_max": 14.133041381835938,
      "activations/layer0_attention_weight_min": -14.769635200500488,
      "activations/layer10_attention_weight_max": 38.89495849609375,
      "activations/layer10_attention_weight_min": -36.8088493347168,
      "activations/layer11_attention_weight_max": 32.35091781616211,
      "activations/layer11_attention_weight_min": -28.394023895263672,
      "activations/layer12_attention_weight_max": 24.36146354675293,
      "activations/layer12_attention_weight_min": -25.393985748291016,
      "activations/layer13_attention_weight_max": 45.165409088134766,
      "activations/layer13_attention_weight_min": -32.62063217163086,
      "activations/layer14_attention_weight_max": 43.292091369628906,
      "activations/layer14_attention_weight_min": -28.683849334716797,
      "activations/layer15_attention_weight_max": 42.83568572998047,
      "activations/layer15_attention_weight_min": -29.35750389099121,
      "activations/layer16_attention_weight_max": 37.38978958129883,
      "activations/layer16_attention_weight_min": -24.90991973876953,
      "activations/layer17_attention_weight_max": 56.230613708496094,
      "activations/layer17_attention_weight_min": -40.93197250366211,
      "activations/layer18_attention_weight_max": 48.4833869934082,
      "activations/layer18_attention_weight_min": -31.562355041503906,
      "activations/layer19_attention_weight_max": 22.88973045349121,
      "activations/layer19_attention_weight_min": -17.904638290405273,
      "activations/layer1_attention_weight_max": 17.437471389770508,
      "activations/layer1_attention_weight_min": -16.47182273864746,
      "activations/layer20_attention_weight_max": 19.22936248779297,
      "activations/layer20_attention_weight_min": -19.606246948242188,
      "activations/layer21_attention_weight_max": 34.71413040161133,
      "activations/layer21_attention_weight_min": -23.64122200012207,
      "activations/layer22_attention_weight_max": 33.87295913696289,
      "activations/layer22_attention_weight_min": -22.85915184020996,
      "activations/layer23_attention_weight_max": 27.004119873046875,
      "activations/layer23_attention_weight_min": -15.72087287902832,
      "activations/layer2_attention_weight_max": 33.0284309387207,
      "activations/layer2_attention_weight_min": -31.158491134643555,
      "activations/layer3_attention_weight_max": 84.60799407958984,
      "activations/layer3_attention_weight_min": -90.50430297851562,
      "activations/layer4_attention_weight_max": 103.8256607055664,
      "activations/layer4_attention_weight_min": -100.74408721923828,
      "activations/layer5_attention_weight_max": 110.22756958007812,
      "activations/layer5_attention_weight_min": -94.83119201660156,
      "activations/layer6_attention_weight_max": 62.65205001831055,
      "activations/layer6_attention_weight_min": -55.63645935058594,
      "activations/layer7_attention_weight_max": 75.08100891113281,
      "activations/layer7_attention_weight_min": -71.50418853759766,
      "activations/layer8_attention_weight_max": 45.46588897705078,
      "activations/layer8_attention_weight_min": -42.33125686645508,
      "activations/layer9_attention_weight_max": 60.21814727783203,
      "activations/layer9_attention_weight_min": -53.20405197143555,
      "epoch": 4.97,
      "learning_rate": 0.00011912386363636361,
      "loss": 2.8388,
      "step": 85600
    },
    {
      "activations/layer0_attention_weight_max": 13.250398635864258,
      "activations/layer0_attention_weight_min": -15.258040428161621,
      "activations/layer10_attention_weight_max": 38.42431640625,
      "activations/layer10_attention_weight_min": -36.684146881103516,
      "activations/layer11_attention_weight_max": 33.5328369140625,
      "activations/layer11_attention_weight_min": -27.527210235595703,
      "activations/layer12_attention_weight_max": 21.288814544677734,
      "activations/layer12_attention_weight_min": -24.623863220214844,
      "activations/layer13_attention_weight_max": 44.68588638305664,
      "activations/layer13_attention_weight_min": -36.897972106933594,
      "activations/layer14_attention_weight_max": 40.544960021972656,
      "activations/layer14_attention_weight_min": -29.149038314819336,
      "activations/layer15_attention_weight_max": 45.21100616455078,
      "activations/layer15_attention_weight_min": -29.024526596069336,
      "activations/layer16_attention_weight_max": 36.5550651550293,
      "activations/layer16_attention_weight_min": -26.594932556152344,
      "activations/layer17_attention_weight_max": 61.50893020629883,
      "activations/layer17_attention_weight_min": -38.800071716308594,
      "activations/layer18_attention_weight_max": 50.07918930053711,
      "activations/layer18_attention_weight_min": -32.12003707885742,
      "activations/layer19_attention_weight_max": 22.65877342224121,
      "activations/layer19_attention_weight_min": -19.218547821044922,
      "activations/layer1_attention_weight_max": 17.894737243652344,
      "activations/layer1_attention_weight_min": -20.268442153930664,
      "activations/layer20_attention_weight_max": 22.348569869995117,
      "activations/layer20_attention_weight_min": -21.149333953857422,
      "activations/layer21_attention_weight_max": 40.533084869384766,
      "activations/layer21_attention_weight_min": -24.72957420349121,
      "activations/layer22_attention_weight_max": 35.23300552368164,
      "activations/layer22_attention_weight_min": -21.700674057006836,
      "activations/layer23_attention_weight_max": 29.221616744995117,
      "activations/layer23_attention_weight_min": -18.656980514526367,
      "activations/layer2_attention_weight_max": 30.08908462524414,
      "activations/layer2_attention_weight_min": -27.979690551757812,
      "activations/layer3_attention_weight_max": 83.85285186767578,
      "activations/layer3_attention_weight_min": -89.92881774902344,
      "activations/layer4_attention_weight_max": 104.78639221191406,
      "activations/layer4_attention_weight_min": -96.6299819946289,
      "activations/layer5_attention_weight_max": 109.11314392089844,
      "activations/layer5_attention_weight_min": -94.20680236816406,
      "activations/layer6_attention_weight_max": 59.414405822753906,
      "activations/layer6_attention_weight_min": -53.21687316894531,
      "activations/layer7_attention_weight_max": 73.96865844726562,
      "activations/layer7_attention_weight_min": -68.44783782958984,
      "activations/layer8_attention_weight_max": 42.24913787841797,
      "activations/layer8_attention_weight_min": -41.8880500793457,
      "activations/layer9_attention_weight_max": 51.30497360229492,
      "activations/layer9_attention_weight_min": -54.115169525146484,
      "epoch": 4.98,
      "learning_rate": 0.00011910492424242423,
      "loss": 2.8454,
      "step": 85650
    },
    {
      "activations/layer0_attention_weight_max": 14.68554973602295,
      "activations/layer0_attention_weight_min": -14.62287425994873,
      "activations/layer10_attention_weight_max": 37.49805450439453,
      "activations/layer10_attention_weight_min": -33.63493728637695,
      "activations/layer11_attention_weight_max": 28.828763961791992,
      "activations/layer11_attention_weight_min": -27.477529525756836,
      "activations/layer12_attention_weight_max": 25.874393463134766,
      "activations/layer12_attention_weight_min": -25.638729095458984,
      "activations/layer13_attention_weight_max": 50.11606979370117,
      "activations/layer13_attention_weight_min": -40.45223617553711,
      "activations/layer14_attention_weight_max": 54.23396301269531,
      "activations/layer14_attention_weight_min": -40.87104034423828,
      "activations/layer15_attention_weight_max": 39.156654357910156,
      "activations/layer15_attention_weight_min": -29.70273780822754,
      "activations/layer16_attention_weight_max": 36.3403205871582,
      "activations/layer16_attention_weight_min": -25.652070999145508,
      "activations/layer17_attention_weight_max": 59.39741516113281,
      "activations/layer17_attention_weight_min": -37.11809158325195,
      "activations/layer18_attention_weight_max": 49.69277572631836,
      "activations/layer18_attention_weight_min": -32.36750411987305,
      "activations/layer19_attention_weight_max": 24.675228118896484,
      "activations/layer19_attention_weight_min": -19.172151565551758,
      "activations/layer1_attention_weight_max": 18.638954162597656,
      "activations/layer1_attention_weight_min": -18.62359619140625,
      "activations/layer20_attention_weight_max": 23.096172332763672,
      "activations/layer20_attention_weight_min": -22.456552505493164,
      "activations/layer21_attention_weight_max": 41.51352310180664,
      "activations/layer21_attention_weight_min": -24.406803131103516,
      "activations/layer22_attention_weight_max": 37.827789306640625,
      "activations/layer22_attention_weight_min": -23.8323974609375,
      "activations/layer23_attention_weight_max": 26.21729850769043,
      "activations/layer23_attention_weight_min": -16.54491424560547,
      "activations/layer2_attention_weight_max": 31.11708641052246,
      "activations/layer2_attention_weight_min": -29.55608367919922,
      "activations/layer3_attention_weight_max": 82.04425048828125,
      "activations/layer3_attention_weight_min": -85.04914093017578,
      "activations/layer4_attention_weight_max": 105.41033935546875,
      "activations/layer4_attention_weight_min": -94.2978286743164,
      "activations/layer5_attention_weight_max": 108.13587951660156,
      "activations/layer5_attention_weight_min": -91.99388885498047,
      "activations/layer6_attention_weight_max": 56.6804084777832,
      "activations/layer6_attention_weight_min": -53.31393051147461,
      "activations/layer7_attention_weight_max": 71.97159576416016,
      "activations/layer7_attention_weight_min": -66.04647064208984,
      "activations/layer8_attention_weight_max": 41.97905349731445,
      "activations/layer8_attention_weight_min": -39.63938522338867,
      "activations/layer9_attention_weight_max": 57.762725830078125,
      "activations/layer9_attention_weight_min": -46.13153839111328,
      "epoch": 4.98,
      "learning_rate": 0.00011908598484848484,
      "loss": 2.8444,
      "step": 85700
    },
    {
      "activations/layer0_attention_weight_max": 15.744295120239258,
      "activations/layer0_attention_weight_min": -14.90092945098877,
      "activations/layer10_attention_weight_max": 44.640296936035156,
      "activations/layer10_attention_weight_min": -38.646156311035156,
      "activations/layer11_attention_weight_max": 35.51740646362305,
      "activations/layer11_attention_weight_min": -29.981887817382812,
      "activations/layer12_attention_weight_max": 22.342926025390625,
      "activations/layer12_attention_weight_min": -26.95664405822754,
      "activations/layer13_attention_weight_max": 77.2942123413086,
      "activations/layer13_attention_weight_min": -45.06069564819336,
      "activations/layer14_attention_weight_max": 78.50591278076172,
      "activations/layer14_attention_weight_min": -44.39421081542969,
      "activations/layer15_attention_weight_max": 54.28759002685547,
      "activations/layer15_attention_weight_min": -36.92560958862305,
      "activations/layer16_attention_weight_max": 34.24618148803711,
      "activations/layer16_attention_weight_min": -26.448413848876953,
      "activations/layer17_attention_weight_max": 58.2606201171875,
      "activations/layer17_attention_weight_min": -37.8655891418457,
      "activations/layer18_attention_weight_max": 50.80910873413086,
      "activations/layer18_attention_weight_min": -33.41004943847656,
      "activations/layer19_attention_weight_max": 22.0289363861084,
      "activations/layer19_attention_weight_min": -19.56507682800293,
      "activations/layer1_attention_weight_max": 16.85405921936035,
      "activations/layer1_attention_weight_min": -16.827896118164062,
      "activations/layer20_attention_weight_max": 21.69406509399414,
      "activations/layer20_attention_weight_min": -20.54979705810547,
      "activations/layer21_attention_weight_max": 41.19416427612305,
      "activations/layer21_attention_weight_min": -24.618772506713867,
      "activations/layer22_attention_weight_max": 31.94588851928711,
      "activations/layer22_attention_weight_min": -22.562707901000977,
      "activations/layer23_attention_weight_max": 29.71634292602539,
      "activations/layer23_attention_weight_min": -19.602218627929688,
      "activations/layer2_attention_weight_max": 33.12826919555664,
      "activations/layer2_attention_weight_min": -31.17841339111328,
      "activations/layer3_attention_weight_max": 86.68733215332031,
      "activations/layer3_attention_weight_min": -87.3824691772461,
      "activations/layer4_attention_weight_max": 112.60411834716797,
      "activations/layer4_attention_weight_min": -106.1350326538086,
      "activations/layer5_attention_weight_max": 117.1605224609375,
      "activations/layer5_attention_weight_min": -108.63688659667969,
      "activations/layer6_attention_weight_max": 65.2287826538086,
      "activations/layer6_attention_weight_min": -59.54840087890625,
      "activations/layer7_attention_weight_max": 79.27296447753906,
      "activations/layer7_attention_weight_min": -71.28307342529297,
      "activations/layer8_attention_weight_max": 51.38570022583008,
      "activations/layer8_attention_weight_min": -44.687782287597656,
      "activations/layer9_attention_weight_max": 60.15899658203125,
      "activations/layer9_attention_weight_min": -52.297515869140625,
      "epoch": 4.98,
      "learning_rate": 0.00011906704545454545,
      "loss": 2.842,
      "step": 85750
    },
    {
      "activations/layer0_attention_weight_max": 13.87852668762207,
      "activations/layer0_attention_weight_min": -14.524598121643066,
      "activations/layer10_attention_weight_max": 41.43790817260742,
      "activations/layer10_attention_weight_min": -35.91843032836914,
      "activations/layer11_attention_weight_max": 33.43217086791992,
      "activations/layer11_attention_weight_min": -27.54674530029297,
      "activations/layer12_attention_weight_max": 40.646392822265625,
      "activations/layer12_attention_weight_min": -25.889549255371094,
      "activations/layer13_attention_weight_max": 56.809486389160156,
      "activations/layer13_attention_weight_min": -36.03720474243164,
      "activations/layer14_attention_weight_max": 39.02971649169922,
      "activations/layer14_attention_weight_min": -28.743053436279297,
      "activations/layer15_attention_weight_max": 41.32888412475586,
      "activations/layer15_attention_weight_min": -31.148500442504883,
      "activations/layer16_attention_weight_max": 36.07923889160156,
      "activations/layer16_attention_weight_min": -24.797988891601562,
      "activations/layer17_attention_weight_max": 55.302730560302734,
      "activations/layer17_attention_weight_min": -39.33993148803711,
      "activations/layer18_attention_weight_max": 46.27197265625,
      "activations/layer18_attention_weight_min": -31.8780460357666,
      "activations/layer19_attention_weight_max": 19.803991317749023,
      "activations/layer19_attention_weight_min": -18.403095245361328,
      "activations/layer1_attention_weight_max": 17.546588897705078,
      "activations/layer1_attention_weight_min": -14.601068496704102,
      "activations/layer20_attention_weight_max": 20.365928649902344,
      "activations/layer20_attention_weight_min": -21.072528839111328,
      "activations/layer21_attention_weight_max": 39.43657302856445,
      "activations/layer21_attention_weight_min": -25.92632484436035,
      "activations/layer22_attention_weight_max": 32.069488525390625,
      "activations/layer22_attention_weight_min": -23.78940773010254,
      "activations/layer23_attention_weight_max": 25.849063873291016,
      "activations/layer23_attention_weight_min": -16.54410171508789,
      "activations/layer2_attention_weight_max": 31.35723876953125,
      "activations/layer2_attention_weight_min": -29.2688045501709,
      "activations/layer3_attention_weight_max": 84.84076690673828,
      "activations/layer3_attention_weight_min": -89.56404113769531,
      "activations/layer4_attention_weight_max": 102.95719146728516,
      "activations/layer4_attention_weight_min": -100.16742706298828,
      "activations/layer5_attention_weight_max": 105.77178955078125,
      "activations/layer5_attention_weight_min": -94.58440399169922,
      "activations/layer6_attention_weight_max": 58.120914459228516,
      "activations/layer6_attention_weight_min": -53.319801330566406,
      "activations/layer7_attention_weight_max": 68.4881362915039,
      "activations/layer7_attention_weight_min": -66.15476989746094,
      "activations/layer8_attention_weight_max": 41.71157455444336,
      "activations/layer8_attention_weight_min": -40.26773452758789,
      "activations/layer9_attention_weight_max": 50.53593444824219,
      "activations/layer9_attention_weight_min": -47.208438873291016,
      "epoch": 4.99,
      "learning_rate": 0.00011904848484848484,
      "loss": 2.8441,
      "step": 85800
    },
    {
      "activations/layer0_attention_weight_max": 14.68568229675293,
      "activations/layer0_attention_weight_min": -14.801189422607422,
      "activations/layer10_attention_weight_max": 37.87125778198242,
      "activations/layer10_attention_weight_min": -34.653221130371094,
      "activations/layer11_attention_weight_max": 29.64570426940918,
      "activations/layer11_attention_weight_min": -26.719215393066406,
      "activations/layer12_attention_weight_max": 22.431127548217773,
      "activations/layer12_attention_weight_min": -27.906957626342773,
      "activations/layer13_attention_weight_max": 48.25840377807617,
      "activations/layer13_attention_weight_min": -34.31397247314453,
      "activations/layer14_attention_weight_max": 53.351863861083984,
      "activations/layer14_attention_weight_min": -33.893978118896484,
      "activations/layer15_attention_weight_max": 42.11636734008789,
      "activations/layer15_attention_weight_min": -31.450542449951172,
      "activations/layer16_attention_weight_max": 36.57779312133789,
      "activations/layer16_attention_weight_min": -26.003969192504883,
      "activations/layer17_attention_weight_max": 58.0660400390625,
      "activations/layer17_attention_weight_min": -39.8697395324707,
      "activations/layer18_attention_weight_max": 49.3143196105957,
      "activations/layer18_attention_weight_min": -34.920684814453125,
      "activations/layer19_attention_weight_max": 21.51861000061035,
      "activations/layer19_attention_weight_min": -20.138513565063477,
      "activations/layer1_attention_weight_max": 17.707366943359375,
      "activations/layer1_attention_weight_min": -17.380779266357422,
      "activations/layer20_attention_weight_max": 22.117847442626953,
      "activations/layer20_attention_weight_min": -19.674152374267578,
      "activations/layer21_attention_weight_max": 42.905269622802734,
      "activations/layer21_attention_weight_min": -25.68170738220215,
      "activations/layer22_attention_weight_max": 36.86899185180664,
      "activations/layer22_attention_weight_min": -24.80801010131836,
      "activations/layer23_attention_weight_max": 28.079544067382812,
      "activations/layer23_attention_weight_min": -16.563554763793945,
      "activations/layer2_attention_weight_max": 30.562114715576172,
      "activations/layer2_attention_weight_min": -28.056066513061523,
      "activations/layer3_attention_weight_max": 84.04457092285156,
      "activations/layer3_attention_weight_min": -84.31105041503906,
      "activations/layer4_attention_weight_max": 99.36913299560547,
      "activations/layer4_attention_weight_min": -93.9461441040039,
      "activations/layer5_attention_weight_max": 100.9959716796875,
      "activations/layer5_attention_weight_min": -90.85528564453125,
      "activations/layer6_attention_weight_max": 58.64646530151367,
      "activations/layer6_attention_weight_min": -51.622962951660156,
      "activations/layer7_attention_weight_max": 70.49005126953125,
      "activations/layer7_attention_weight_min": -65.86233520507812,
      "activations/layer8_attention_weight_max": 41.62657928466797,
      "activations/layer8_attention_weight_min": -39.807579040527344,
      "activations/layer9_attention_weight_max": 50.43947982788086,
      "activations/layer9_attention_weight_min": -46.88724136352539,
      "epoch": 4.99,
      "learning_rate": 0.00011902954545454544,
      "loss": 2.8462,
      "step": 85850
    },
    {
      "activations/layer0_attention_weight_max": 14.966198921203613,
      "activations/layer0_attention_weight_min": -15.489728927612305,
      "activations/layer10_attention_weight_max": 37.18623733520508,
      "activations/layer10_attention_weight_min": -32.35251998901367,
      "activations/layer11_attention_weight_max": 28.11947250366211,
      "activations/layer11_attention_weight_min": -27.529403686523438,
      "activations/layer12_attention_weight_max": 26.828031539916992,
      "activations/layer12_attention_weight_min": -26.24094009399414,
      "activations/layer13_attention_weight_max": 44.9060173034668,
      "activations/layer13_attention_weight_min": -38.671321868896484,
      "activations/layer14_attention_weight_max": 36.44196319580078,
      "activations/layer14_attention_weight_min": -27.857284545898438,
      "activations/layer15_attention_weight_max": 39.42186737060547,
      "activations/layer15_attention_weight_min": -29.117389678955078,
      "activations/layer16_attention_weight_max": 34.07956314086914,
      "activations/layer16_attention_weight_min": -25.673818588256836,
      "activations/layer17_attention_weight_max": 57.64951705932617,
      "activations/layer17_attention_weight_min": -36.82575988769531,
      "activations/layer18_attention_weight_max": 48.514862060546875,
      "activations/layer18_attention_weight_min": -32.278045654296875,
      "activations/layer19_attention_weight_max": 23.43398094177246,
      "activations/layer19_attention_weight_min": -22.587955474853516,
      "activations/layer1_attention_weight_max": 17.992149353027344,
      "activations/layer1_attention_weight_min": -16.276182174682617,
      "activations/layer20_attention_weight_max": 21.50086212158203,
      "activations/layer20_attention_weight_min": -25.203018188476562,
      "activations/layer21_attention_weight_max": 42.24831008911133,
      "activations/layer21_attention_weight_min": -24.396081924438477,
      "activations/layer22_attention_weight_max": 39.89213180541992,
      "activations/layer22_attention_weight_min": -22.21489906311035,
      "activations/layer23_attention_weight_max": 28.449552536010742,
      "activations/layer23_attention_weight_min": -18.06572723388672,
      "activations/layer2_attention_weight_max": 33.803218841552734,
      "activations/layer2_attention_weight_min": -31.40521812438965,
      "activations/layer3_attention_weight_max": 84.77373504638672,
      "activations/layer3_attention_weight_min": -87.16896057128906,
      "activations/layer4_attention_weight_max": 108.67286682128906,
      "activations/layer4_attention_weight_min": -99.61752319335938,
      "activations/layer5_attention_weight_max": 109.37115478515625,
      "activations/layer5_attention_weight_min": -99.52202606201172,
      "activations/layer6_attention_weight_max": 60.618194580078125,
      "activations/layer6_attention_weight_min": -54.69200897216797,
      "activations/layer7_attention_weight_max": 70.5451889038086,
      "activations/layer7_attention_weight_min": -66.716552734375,
      "activations/layer8_attention_weight_max": 42.77825927734375,
      "activations/layer8_attention_weight_min": -44.92786407470703,
      "activations/layer9_attention_weight_max": 46.73527908325195,
      "activations/layer9_attention_weight_min": -49.02398681640625,
      "epoch": 4.99,
      "learning_rate": 0.00011901060606060605,
      "loss": 2.8347,
      "step": 85900
    },
    {
      "activations/layer0_attention_weight_max": 13.928240776062012,
      "activations/layer0_attention_weight_min": -14.95266342163086,
      "activations/layer10_attention_weight_max": 38.2442626953125,
      "activations/layer10_attention_weight_min": -33.78341293334961,
      "activations/layer11_attention_weight_max": 29.240018844604492,
      "activations/layer11_attention_weight_min": -27.489368438720703,
      "activations/layer12_attention_weight_max": 23.274152755737305,
      "activations/layer12_attention_weight_min": -23.089847564697266,
      "activations/layer13_attention_weight_max": 45.5480842590332,
      "activations/layer13_attention_weight_min": -33.41667175292969,
      "activations/layer14_attention_weight_max": 41.3826789855957,
      "activations/layer14_attention_weight_min": -30.59317398071289,
      "activations/layer15_attention_weight_max": 39.868507385253906,
      "activations/layer15_attention_weight_min": -28.463468551635742,
      "activations/layer16_attention_weight_max": 32.71321487426758,
      "activations/layer16_attention_weight_min": -26.21405029296875,
      "activations/layer17_attention_weight_max": 61.76387405395508,
      "activations/layer17_attention_weight_min": -37.41255187988281,
      "activations/layer18_attention_weight_max": 52.068363189697266,
      "activations/layer18_attention_weight_min": -32.494083404541016,
      "activations/layer19_attention_weight_max": 22.91919708251953,
      "activations/layer19_attention_weight_min": -19.52794075012207,
      "activations/layer1_attention_weight_max": 17.3249454498291,
      "activations/layer1_attention_weight_min": -14.81776237487793,
      "activations/layer20_attention_weight_max": 21.296287536621094,
      "activations/layer20_attention_weight_min": -21.74119758605957,
      "activations/layer21_attention_weight_max": 40.27288055419922,
      "activations/layer21_attention_weight_min": -24.233407974243164,
      "activations/layer22_attention_weight_max": 31.561141967773438,
      "activations/layer22_attention_weight_min": -24.01140785217285,
      "activations/layer23_attention_weight_max": 25.95357322692871,
      "activations/layer23_attention_weight_min": -16.799455642700195,
      "activations/layer2_attention_weight_max": 30.20191192626953,
      "activations/layer2_attention_weight_min": -28.70620346069336,
      "activations/layer3_attention_weight_max": 81.14634704589844,
      "activations/layer3_attention_weight_min": -85.81278991699219,
      "activations/layer4_attention_weight_max": 101.2958984375,
      "activations/layer4_attention_weight_min": -100.42942810058594,
      "activations/layer5_attention_weight_max": 106.36449432373047,
      "activations/layer5_attention_weight_min": -96.17244720458984,
      "activations/layer6_attention_weight_max": 57.907413482666016,
      "activations/layer6_attention_weight_min": -53.32760238647461,
      "activations/layer7_attention_weight_max": 69.04608154296875,
      "activations/layer7_attention_weight_min": -66.76167297363281,
      "activations/layer8_attention_weight_max": 41.92760467529297,
      "activations/layer8_attention_weight_min": -42.346221923828125,
      "activations/layer9_attention_weight_max": 49.96294021606445,
      "activations/layer9_attention_weight_min": -45.08462142944336,
      "epoch": 4.99,
      "learning_rate": 0.00011899166666666665,
      "loss": 2.8448,
      "step": 85950
    },
    {
      "activations/layer0_attention_weight_max": 13.951497077941895,
      "activations/layer0_attention_weight_min": -15.716703414916992,
      "activations/layer10_attention_weight_max": 37.79896926879883,
      "activations/layer10_attention_weight_min": -33.33063507080078,
      "activations/layer11_attention_weight_max": 28.652860641479492,
      "activations/layer11_attention_weight_min": -26.81586456298828,
      "activations/layer12_attention_weight_max": 23.05463218688965,
      "activations/layer12_attention_weight_min": -25.5936279296875,
      "activations/layer13_attention_weight_max": 49.3101692199707,
      "activations/layer13_attention_weight_min": -35.993587493896484,
      "activations/layer14_attention_weight_max": 47.074134826660156,
      "activations/layer14_attention_weight_min": -30.378023147583008,
      "activations/layer15_attention_weight_max": 37.69606399536133,
      "activations/layer15_attention_weight_min": -29.72842025756836,
      "activations/layer16_attention_weight_max": 36.733558654785156,
      "activations/layer16_attention_weight_min": -25.862802505493164,
      "activations/layer17_attention_weight_max": 54.44456100463867,
      "activations/layer17_attention_weight_min": -38.52687072753906,
      "activations/layer18_attention_weight_max": 50.665897369384766,
      "activations/layer18_attention_weight_min": -33.2703742980957,
      "activations/layer19_attention_weight_max": 21.28896713256836,
      "activations/layer19_attention_weight_min": -20.371482849121094,
      "activations/layer1_attention_weight_max": 17.298507690429688,
      "activations/layer1_attention_weight_min": -14.958045959472656,
      "activations/layer20_attention_weight_max": 21.19798469543457,
      "activations/layer20_attention_weight_min": -20.429492950439453,
      "activations/layer21_attention_weight_max": 41.58232879638672,
      "activations/layer21_attention_weight_min": -30.829463958740234,
      "activations/layer22_attention_weight_max": 34.13145065307617,
      "activations/layer22_attention_weight_min": -23.65627098083496,
      "activations/layer23_attention_weight_max": 28.218679428100586,
      "activations/layer23_attention_weight_min": -17.85651397705078,
      "activations/layer2_attention_weight_max": 31.284496307373047,
      "activations/layer2_attention_weight_min": -30.20082664489746,
      "activations/layer3_attention_weight_max": 82.17881774902344,
      "activations/layer3_attention_weight_min": -87.427978515625,
      "activations/layer4_attention_weight_max": 104.60408782958984,
      "activations/layer4_attention_weight_min": -103.86177062988281,
      "activations/layer5_attention_weight_max": 104.70303344726562,
      "activations/layer5_attention_weight_min": -97.90081787109375,
      "activations/layer6_attention_weight_max": 58.84883499145508,
      "activations/layer6_attention_weight_min": -52.960670471191406,
      "activations/layer7_attention_weight_max": 73.30477142333984,
      "activations/layer7_attention_weight_min": -66.9339828491211,
      "activations/layer8_attention_weight_max": 42.13585662841797,
      "activations/layer8_attention_weight_min": -40.74071502685547,
      "activations/layer9_attention_weight_max": 49.373008728027344,
      "activations/layer9_attention_weight_min": -46.556461334228516,
      "epoch": 5.0,
      "learning_rate": 0.00011897272727272727,
      "loss": 2.8349,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_loss": 2.79296875,
      "eval_runtime": 8.5685,
      "eval_samples_per_second": 501.135,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_openwebtext_loss": 2.79296875,
      "eval_openwebtext_ppl": 16.329425896569767,
      "eval_openwebtext_runtime": 8.5685,
      "eval_openwebtext_samples_per_second": 501.135,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_wikitext_loss": 3.0234375,
      "eval_wikitext_ppl": 20.56185170491249,
      "eval_wikitext_runtime": 2.0331,
      "eval_wikitext_samples_per_second": 224.287,
      "step": 86000
    },
    {
      "epoch": 5.0,
      "eval_lambada_loss": 2.86328125,
      "eval_lambada_ppl": 17.51891667538985,
      "eval_lambada_runtime": 9.623,
      "eval_lambada_samples_per_second": 505.977,
      "step": 86000
    },
    {
      "activations/layer0_attention_weight_max": 14.518725395202637,
      "activations/layer0_attention_weight_min": -15.263821601867676,
      "activations/layer10_attention_weight_max": 40.04465866088867,
      "activations/layer10_attention_weight_min": -34.322593688964844,
      "activations/layer11_attention_weight_max": 30.792356491088867,
      "activations/layer11_attention_weight_min": -28.526840209960938,
      "activations/layer12_attention_weight_max": 25.358234405517578,
      "activations/layer12_attention_weight_min": -26.170286178588867,
      "activations/layer13_attention_weight_max": 54.16039276123047,
      "activations/layer13_attention_weight_min": -34.880210876464844,
      "activations/layer14_attention_weight_max": 46.0290641784668,
      "activations/layer14_attention_weight_min": -31.229639053344727,
      "activations/layer15_attention_weight_max": 40.32848358154297,
      "activations/layer15_attention_weight_min": -31.315372467041016,
      "activations/layer16_attention_weight_max": 37.86502456665039,
      "activations/layer16_attention_weight_min": -27.996591567993164,
      "activations/layer17_attention_weight_max": 55.22159957885742,
      "activations/layer17_attention_weight_min": -44.220638275146484,
      "activations/layer18_attention_weight_max": 53.40692138671875,
      "activations/layer18_attention_weight_min": -36.759796142578125,
      "activations/layer19_attention_weight_max": 21.522037506103516,
      "activations/layer19_attention_weight_min": -18.5483455657959,
      "activations/layer1_attention_weight_max": 16.90108299255371,
      "activations/layer1_attention_weight_min": -15.391522407531738,
      "activations/layer20_attention_weight_max": 24.52235984802246,
      "activations/layer20_attention_weight_min": -20.450735092163086,
      "activations/layer21_attention_weight_max": 41.47438049316406,
      "activations/layer21_attention_weight_min": -30.95970916748047,
      "activations/layer22_attention_weight_max": 36.20988082885742,
      "activations/layer22_attention_weight_min": -25.719127655029297,
      "activations/layer23_attention_weight_max": 26.07192039489746,
      "activations/layer23_attention_weight_min": -16.842594146728516,
      "activations/layer2_attention_weight_max": 32.279869079589844,
      "activations/layer2_attention_weight_min": -30.34210205078125,
      "activations/layer3_attention_weight_max": 80.90767669677734,
      "activations/layer3_attention_weight_min": -84.43787384033203,
      "activations/layer4_attention_weight_max": 104.12872314453125,
      "activations/layer4_attention_weight_min": -97.22161102294922,
      "activations/layer5_attention_weight_max": 107.85694885253906,
      "activations/layer5_attention_weight_min": -98.08358764648438,
      "activations/layer6_attention_weight_max": 59.81370162963867,
      "activations/layer6_attention_weight_min": -55.737953186035156,
      "activations/layer7_attention_weight_max": 71.13026428222656,
      "activations/layer7_attention_weight_min": -70.10936737060547,
      "activations/layer8_attention_weight_max": 41.72591781616211,
      "activations/layer8_attention_weight_min": -40.06379699707031,
      "activations/layer9_attention_weight_max": 50.28623580932617,
      "activations/layer9_attention_weight_min": -48.654884338378906,
      "epoch": 5.0,
      "learning_rate": 0.00011895378787878787,
      "loss": 2.8392,
      "step": 86050
    },
    {
      "activations/layer0_attention_weight_max": 15.773804664611816,
      "activations/layer0_attention_weight_min": -15.477814674377441,
      "activations/layer10_attention_weight_max": 38.79249954223633,
      "activations/layer10_attention_weight_min": -36.330352783203125,
      "activations/layer11_attention_weight_max": 30.831655502319336,
      "activations/layer11_attention_weight_min": -29.530338287353516,
      "activations/layer12_attention_weight_max": 29.505374908447266,
      "activations/layer12_attention_weight_min": -25.014541625976562,
      "activations/layer13_attention_weight_max": 48.05387878417969,
      "activations/layer13_attention_weight_min": -32.83230209350586,
      "activations/layer14_attention_weight_max": 44.36566925048828,
      "activations/layer14_attention_weight_min": -29.7887020111084,
      "activations/layer15_attention_weight_max": 46.89944839477539,
      "activations/layer15_attention_weight_min": -29.432905197143555,
      "activations/layer16_attention_weight_max": 40.15110778808594,
      "activations/layer16_attention_weight_min": -25.657703399658203,
      "activations/layer17_attention_weight_max": 58.51649856567383,
      "activations/layer17_attention_weight_min": -38.39130783081055,
      "activations/layer18_attention_weight_max": 55.49208450317383,
      "activations/layer18_attention_weight_min": -33.52392578125,
      "activations/layer19_attention_weight_max": 25.67445945739746,
      "activations/layer19_attention_weight_min": -20.119482040405273,
      "activations/layer1_attention_weight_max": 16.831825256347656,
      "activations/layer1_attention_weight_min": -14.232656478881836,
      "activations/layer20_attention_weight_max": 23.88902473449707,
      "activations/layer20_attention_weight_min": -18.406696319580078,
      "activations/layer21_attention_weight_max": 42.03620529174805,
      "activations/layer21_attention_weight_min": -28.388229370117188,
      "activations/layer22_attention_weight_max": 38.42666244506836,
      "activations/layer22_attention_weight_min": -25.94525718688965,
      "activations/layer23_attention_weight_max": 27.792869567871094,
      "activations/layer23_attention_weight_min": -19.88302230834961,
      "activations/layer2_attention_weight_max": 29.310523986816406,
      "activations/layer2_attention_weight_min": -27.086212158203125,
      "activations/layer3_attention_weight_max": 80.97921752929688,
      "activations/layer3_attention_weight_min": -82.76805114746094,
      "activations/layer4_attention_weight_max": 104.12711334228516,
      "activations/layer4_attention_weight_min": -96.4468002319336,
      "activations/layer5_attention_weight_max": 106.2241439819336,
      "activations/layer5_attention_weight_min": -95.55159759521484,
      "activations/layer6_attention_weight_max": 59.34714889526367,
      "activations/layer6_attention_weight_min": -54.03981399536133,
      "activations/layer7_attention_weight_max": 69.74129486083984,
      "activations/layer7_attention_weight_min": -69.73675537109375,
      "activations/layer8_attention_weight_max": 42.580467224121094,
      "activations/layer8_attention_weight_min": -40.17253875732422,
      "activations/layer9_attention_weight_max": 49.14066696166992,
      "activations/layer9_attention_weight_min": -49.52647399902344,
      "epoch": 5.0,
      "learning_rate": 0.00011893484848484847,
      "loss": 2.8943,
      "step": 86100
    },
    {
      "activations/layer0_attention_weight_max": 14.293473243713379,
      "activations/layer0_attention_weight_min": -14.906886100769043,
      "activations/layer10_attention_weight_max": 36.0764045715332,
      "activations/layer10_attention_weight_min": -33.35054397583008,
      "activations/layer11_attention_weight_max": 27.626707077026367,
      "activations/layer11_attention_weight_min": -27.54220199584961,
      "activations/layer12_attention_weight_max": 22.512121200561523,
      "activations/layer12_attention_weight_min": -24.305011749267578,
      "activations/layer13_attention_weight_max": 47.60096740722656,
      "activations/layer13_attention_weight_min": -35.446041107177734,
      "activations/layer14_attention_weight_max": 49.67091369628906,
      "activations/layer14_attention_weight_min": -29.896930694580078,
      "activations/layer15_attention_weight_max": 45.82923126220703,
      "activations/layer15_attention_weight_min": -28.064373016357422,
      "activations/layer16_attention_weight_max": 38.82893371582031,
      "activations/layer16_attention_weight_min": -26.900981903076172,
      "activations/layer17_attention_weight_max": 60.32876968383789,
      "activations/layer17_attention_weight_min": -37.663848876953125,
      "activations/layer18_attention_weight_max": 55.12432098388672,
      "activations/layer18_attention_weight_min": -34.55463790893555,
      "activations/layer19_attention_weight_max": 24.995927810668945,
      "activations/layer19_attention_weight_min": -20.446971893310547,
      "activations/layer1_attention_weight_max": 17.736042022705078,
      "activations/layer1_attention_weight_min": -17.91232681274414,
      "activations/layer20_attention_weight_max": 29.88942527770996,
      "activations/layer20_attention_weight_min": -19.905244827270508,
      "activations/layer21_attention_weight_max": 46.471229553222656,
      "activations/layer21_attention_weight_min": -29.01201629638672,
      "activations/layer22_attention_weight_max": 38.222591400146484,
      "activations/layer22_attention_weight_min": -23.13459014892578,
      "activations/layer23_attention_weight_max": 29.37904167175293,
      "activations/layer23_attention_weight_min": -17.741817474365234,
      "activations/layer2_attention_weight_max": 30.84819793701172,
      "activations/layer2_attention_weight_min": -28.490459442138672,
      "activations/layer3_attention_weight_max": 82.41165161132812,
      "activations/layer3_attention_weight_min": -82.61955261230469,
      "activations/layer4_attention_weight_max": 101.53157043457031,
      "activations/layer4_attention_weight_min": -93.55318450927734,
      "activations/layer5_attention_weight_max": 102.21943664550781,
      "activations/layer5_attention_weight_min": -92.18949890136719,
      "activations/layer6_attention_weight_max": 60.749778747558594,
      "activations/layer6_attention_weight_min": -53.227928161621094,
      "activations/layer7_attention_weight_max": 77.26266479492188,
      "activations/layer7_attention_weight_min": -66.38760375976562,
      "activations/layer8_attention_weight_max": 41.46892547607422,
      "activations/layer8_attention_weight_min": -41.5037727355957,
      "activations/layer9_attention_weight_max": 48.55794143676758,
      "activations/layer9_attention_weight_min": -45.89675521850586,
      "epoch": 5.01,
      "learning_rate": 0.00011891590909090909,
      "loss": 2.8404,
      "step": 86150
    },
    {
      "activations/layer0_attention_weight_max": 15.584063529968262,
      "activations/layer0_attention_weight_min": -15.217997550964355,
      "activations/layer10_attention_weight_max": 40.24170684814453,
      "activations/layer10_attention_weight_min": -34.689308166503906,
      "activations/layer11_attention_weight_max": 30.79317283630371,
      "activations/layer11_attention_weight_min": -26.773855209350586,
      "activations/layer12_attention_weight_max": 24.0365047454834,
      "activations/layer12_attention_weight_min": -26.524978637695312,
      "activations/layer13_attention_weight_max": 49.43626403808594,
      "activations/layer13_attention_weight_min": -38.2241096496582,
      "activations/layer14_attention_weight_max": 40.8881721496582,
      "activations/layer14_attention_weight_min": -28.744136810302734,
      "activations/layer15_attention_weight_max": 39.44105911254883,
      "activations/layer15_attention_weight_min": -29.41275405883789,
      "activations/layer16_attention_weight_max": 37.70955276489258,
      "activations/layer16_attention_weight_min": -27.609516143798828,
      "activations/layer17_attention_weight_max": 59.28146743774414,
      "activations/layer17_attention_weight_min": -38.365779876708984,
      "activations/layer18_attention_weight_max": 53.298702239990234,
      "activations/layer18_attention_weight_min": -32.906803131103516,
      "activations/layer19_attention_weight_max": 23.517906188964844,
      "activations/layer19_attention_weight_min": -20.736581802368164,
      "activations/layer1_attention_weight_max": 16.900108337402344,
      "activations/layer1_attention_weight_min": -15.859846115112305,
      "activations/layer20_attention_weight_max": 21.47913360595703,
      "activations/layer20_attention_weight_min": -24.384580612182617,
      "activations/layer21_attention_weight_max": 39.49351501464844,
      "activations/layer21_attention_weight_min": -30.67531394958496,
      "activations/layer22_attention_weight_max": 34.635093688964844,
      "activations/layer22_attention_weight_min": -23.11372947692871,
      "activations/layer23_attention_weight_max": 28.484939575195312,
      "activations/layer23_attention_weight_min": -19.080718994140625,
      "activations/layer2_attention_weight_max": 31.002525329589844,
      "activations/layer2_attention_weight_min": -29.051593780517578,
      "activations/layer3_attention_weight_max": 84.30984497070312,
      "activations/layer3_attention_weight_min": -87.64652252197266,
      "activations/layer4_attention_weight_max": 105.23384857177734,
      "activations/layer4_attention_weight_min": -99.54666137695312,
      "activations/layer5_attention_weight_max": 107.25492858886719,
      "activations/layer5_attention_weight_min": -97.47383117675781,
      "activations/layer6_attention_weight_max": 61.261871337890625,
      "activations/layer6_attention_weight_min": -54.28810501098633,
      "activations/layer7_attention_weight_max": 76.1554946899414,
      "activations/layer7_attention_weight_min": -65.05611419677734,
      "activations/layer8_attention_weight_max": 43.80389404296875,
      "activations/layer8_attention_weight_min": -41.40808868408203,
      "activations/layer9_attention_weight_max": 55.176918029785156,
      "activations/layer9_attention_weight_min": -46.402767181396484,
      "epoch": 5.01,
      "learning_rate": 0.00011889696969696968,
      "loss": 2.8477,
      "step": 86200
    },
    {
      "activations/layer0_attention_weight_max": 14.320486068725586,
      "activations/layer0_attention_weight_min": -15.131905555725098,
      "activations/layer10_attention_weight_max": 35.20649337768555,
      "activations/layer10_attention_weight_min": -31.81746482849121,
      "activations/layer11_attention_weight_max": 28.730178833007812,
      "activations/layer11_attention_weight_min": -25.377357482910156,
      "activations/layer12_attention_weight_max": 22.317771911621094,
      "activations/layer12_attention_weight_min": -28.022775650024414,
      "activations/layer13_attention_weight_max": 49.07693099975586,
      "activations/layer13_attention_weight_min": -38.24952697753906,
      "activations/layer14_attention_weight_max": 46.68251419067383,
      "activations/layer14_attention_weight_min": -34.980220794677734,
      "activations/layer15_attention_weight_max": 39.77821731567383,
      "activations/layer15_attention_weight_min": -28.28192138671875,
      "activations/layer16_attention_weight_max": 34.17977523803711,
      "activations/layer16_attention_weight_min": -25.73691749572754,
      "activations/layer17_attention_weight_max": 57.77257537841797,
      "activations/layer17_attention_weight_min": -38.941593170166016,
      "activations/layer18_attention_weight_max": 46.563392639160156,
      "activations/layer18_attention_weight_min": -34.09608840942383,
      "activations/layer19_attention_weight_max": 24.3297176361084,
      "activations/layer19_attention_weight_min": -21.477663040161133,
      "activations/layer1_attention_weight_max": 16.654821395874023,
      "activations/layer1_attention_weight_min": -17.241497039794922,
      "activations/layer20_attention_weight_max": 23.0522518157959,
      "activations/layer20_attention_weight_min": -23.254497528076172,
      "activations/layer21_attention_weight_max": 42.37086868286133,
      "activations/layer21_attention_weight_min": -26.11429214477539,
      "activations/layer22_attention_weight_max": 35.686405181884766,
      "activations/layer22_attention_weight_min": -26.3166446685791,
      "activations/layer23_attention_weight_max": 29.044525146484375,
      "activations/layer23_attention_weight_min": -20.06055450439453,
      "activations/layer2_attention_weight_max": 31.887222290039062,
      "activations/layer2_attention_weight_min": -28.806591033935547,
      "activations/layer3_attention_weight_max": 81.6736068725586,
      "activations/layer3_attention_weight_min": -84.02413940429688,
      "activations/layer4_attention_weight_max": 102.70938873291016,
      "activations/layer4_attention_weight_min": -97.0390625,
      "activations/layer5_attention_weight_max": 101.06718444824219,
      "activations/layer5_attention_weight_min": -91.40573120117188,
      "activations/layer6_attention_weight_max": 56.65435791015625,
      "activations/layer6_attention_weight_min": -52.20187759399414,
      "activations/layer7_attention_weight_max": 67.15641021728516,
      "activations/layer7_attention_weight_min": -69.45645141601562,
      "activations/layer8_attention_weight_max": 42.84159851074219,
      "activations/layer8_attention_weight_min": -41.042633056640625,
      "activations/layer9_attention_weight_max": 45.4710807800293,
      "activations/layer9_attention_weight_min": -45.64698028564453,
      "epoch": 5.01,
      "learning_rate": 0.00011887803030303029,
      "loss": 2.8365,
      "step": 86250
    },
    {
      "activations/layer0_attention_weight_max": 14.575409889221191,
      "activations/layer0_attention_weight_min": -15.65090560913086,
      "activations/layer10_attention_weight_max": 37.07991027832031,
      "activations/layer10_attention_weight_min": -33.019287109375,
      "activations/layer11_attention_weight_max": 28.447265625,
      "activations/layer11_attention_weight_min": -27.46771240234375,
      "activations/layer12_attention_weight_max": 21.834228515625,
      "activations/layer12_attention_weight_min": -27.433256149291992,
      "activations/layer13_attention_weight_max": 48.21195983886719,
      "activations/layer13_attention_weight_min": -37.08562469482422,
      "activations/layer14_attention_weight_max": 42.30955505371094,
      "activations/layer14_attention_weight_min": -30.696603775024414,
      "activations/layer15_attention_weight_max": 39.75794219970703,
      "activations/layer15_attention_weight_min": -30.01923942565918,
      "activations/layer16_attention_weight_max": 33.64291763305664,
      "activations/layer16_attention_weight_min": -25.27507781982422,
      "activations/layer17_attention_weight_max": 52.47858428955078,
      "activations/layer17_attention_weight_min": -36.30763626098633,
      "activations/layer18_attention_weight_max": 45.64454650878906,
      "activations/layer18_attention_weight_min": -31.949142456054688,
      "activations/layer19_attention_weight_max": 20.399669647216797,
      "activations/layer19_attention_weight_min": -20.347003936767578,
      "activations/layer1_attention_weight_max": 16.566923141479492,
      "activations/layer1_attention_weight_min": -16.89983558654785,
      "activations/layer20_attention_weight_max": 22.76811408996582,
      "activations/layer20_attention_weight_min": -19.372655868530273,
      "activations/layer21_attention_weight_max": 38.997676849365234,
      "activations/layer21_attention_weight_min": -26.200077056884766,
      "activations/layer22_attention_weight_max": 30.874454498291016,
      "activations/layer22_attention_weight_min": -25.13458251953125,
      "activations/layer23_attention_weight_max": 25.33985710144043,
      "activations/layer23_attention_weight_min": -17.113306045532227,
      "activations/layer2_attention_weight_max": 32.04344940185547,
      "activations/layer2_attention_weight_min": -28.872825622558594,
      "activations/layer3_attention_weight_max": 86.26432800292969,
      "activations/layer3_attention_weight_min": -89.23551177978516,
      "activations/layer4_attention_weight_max": 105.24275970458984,
      "activations/layer4_attention_weight_min": -98.13291931152344,
      "activations/layer5_attention_weight_max": 102.32005310058594,
      "activations/layer5_attention_weight_min": -90.70962524414062,
      "activations/layer6_attention_weight_max": 57.75086212158203,
      "activations/layer6_attention_weight_min": -50.50934982299805,
      "activations/layer7_attention_weight_max": 69.10646057128906,
      "activations/layer7_attention_weight_min": -65.96808624267578,
      "activations/layer8_attention_weight_max": 40.048099517822266,
      "activations/layer8_attention_weight_min": -39.12543487548828,
      "activations/layer9_attention_weight_max": 52.15861892700195,
      "activations/layer9_attention_weight_min": -47.742462158203125,
      "epoch": 5.01,
      "learning_rate": 0.0001188590909090909,
      "loss": 2.8383,
      "step": 86300
    },
    {
      "activations/layer0_attention_weight_max": 16.140233993530273,
      "activations/layer0_attention_weight_min": -15.026572227478027,
      "activations/layer10_attention_weight_max": 37.03956985473633,
      "activations/layer10_attention_weight_min": -34.54875183105469,
      "activations/layer11_attention_weight_max": 34.169559478759766,
      "activations/layer11_attention_weight_min": -27.62799072265625,
      "activations/layer12_attention_weight_max": 20.973325729370117,
      "activations/layer12_attention_weight_min": -25.12921142578125,
      "activations/layer13_attention_weight_max": 58.703346252441406,
      "activations/layer13_attention_weight_min": -40.62984085083008,
      "activations/layer14_attention_weight_max": 61.24872970581055,
      "activations/layer14_attention_weight_min": -37.88642120361328,
      "activations/layer15_attention_weight_max": 49.66117858886719,
      "activations/layer15_attention_weight_min": -30.983381271362305,
      "activations/layer16_attention_weight_max": 36.74940490722656,
      "activations/layer16_attention_weight_min": -24.69139289855957,
      "activations/layer17_attention_weight_max": 63.25191116333008,
      "activations/layer17_attention_weight_min": -39.0152587890625,
      "activations/layer18_attention_weight_max": 48.8453483581543,
      "activations/layer18_attention_weight_min": -32.48786544799805,
      "activations/layer19_attention_weight_max": 21.2793025970459,
      "activations/layer19_attention_weight_min": -18.35188102722168,
      "activations/layer1_attention_weight_max": 17.36403465270996,
      "activations/layer1_attention_weight_min": -16.12373161315918,
      "activations/layer20_attention_weight_max": 20.535715103149414,
      "activations/layer20_attention_weight_min": -20.09109878540039,
      "activations/layer21_attention_weight_max": 39.79952621459961,
      "activations/layer21_attention_weight_min": -23.5994930267334,
      "activations/layer22_attention_weight_max": 33.80546951293945,
      "activations/layer22_attention_weight_min": -21.6004638671875,
      "activations/layer23_attention_weight_max": 27.940387725830078,
      "activations/layer23_attention_weight_min": -18.20410919189453,
      "activations/layer2_attention_weight_max": 30.883525848388672,
      "activations/layer2_attention_weight_min": -28.84040641784668,
      "activations/layer3_attention_weight_max": 81.22537994384766,
      "activations/layer3_attention_weight_min": -81.9463882446289,
      "activations/layer4_attention_weight_max": 99.60313415527344,
      "activations/layer4_attention_weight_min": -99.3370132446289,
      "activations/layer5_attention_weight_max": 103.42333221435547,
      "activations/layer5_attention_weight_min": -95.16612243652344,
      "activations/layer6_attention_weight_max": 59.310218811035156,
      "activations/layer6_attention_weight_min": -51.81925964355469,
      "activations/layer7_attention_weight_max": 81.29254913330078,
      "activations/layer7_attention_weight_min": -68.13806915283203,
      "activations/layer8_attention_weight_max": 47.46818161010742,
      "activations/layer8_attention_weight_min": -42.99005126953125,
      "activations/layer9_attention_weight_max": 61.377410888671875,
      "activations/layer9_attention_weight_min": -48.37388229370117,
      "epoch": 5.02,
      "learning_rate": 0.0001188401515151515,
      "loss": 2.837,
      "step": 86350
    },
    {
      "activations/layer0_attention_weight_max": 14.210932731628418,
      "activations/layer0_attention_weight_min": -15.417722702026367,
      "activations/layer10_attention_weight_max": 39.12082290649414,
      "activations/layer10_attention_weight_min": -34.71133804321289,
      "activations/layer11_attention_weight_max": 31.222034454345703,
      "activations/layer11_attention_weight_min": -27.928455352783203,
      "activations/layer12_attention_weight_max": 22.122114181518555,
      "activations/layer12_attention_weight_min": -28.6108341217041,
      "activations/layer13_attention_weight_max": 49.28540802001953,
      "activations/layer13_attention_weight_min": -36.96673583984375,
      "activations/layer14_attention_weight_max": 61.129268646240234,
      "activations/layer14_attention_weight_min": -39.59060287475586,
      "activations/layer15_attention_weight_max": 43.03131866455078,
      "activations/layer15_attention_weight_min": -30.00600814819336,
      "activations/layer16_attention_weight_max": 35.73280334472656,
      "activations/layer16_attention_weight_min": -28.956342697143555,
      "activations/layer17_attention_weight_max": 53.85700225830078,
      "activations/layer17_attention_weight_min": -37.450721740722656,
      "activations/layer18_attention_weight_max": 46.92344665527344,
      "activations/layer18_attention_weight_min": -33.85564422607422,
      "activations/layer19_attention_weight_max": 22.730274200439453,
      "activations/layer19_attention_weight_min": -21.974199295043945,
      "activations/layer1_attention_weight_max": 18.155561447143555,
      "activations/layer1_attention_weight_min": -15.671013832092285,
      "activations/layer20_attention_weight_max": 20.983074188232422,
      "activations/layer20_attention_weight_min": -24.529455184936523,
      "activations/layer21_attention_weight_max": 43.51424789428711,
      "activations/layer21_attention_weight_min": -25.541812896728516,
      "activations/layer22_attention_weight_max": 31.259323120117188,
      "activations/layer22_attention_weight_min": -24.071441650390625,
      "activations/layer23_attention_weight_max": 26.260799407958984,
      "activations/layer23_attention_weight_min": -19.178478240966797,
      "activations/layer2_attention_weight_max": 30.341766357421875,
      "activations/layer2_attention_weight_min": -28.74651336669922,
      "activations/layer3_attention_weight_max": 83.9439468383789,
      "activations/layer3_attention_weight_min": -87.95036315917969,
      "activations/layer4_attention_weight_max": 104.10587310791016,
      "activations/layer4_attention_weight_min": -96.90996551513672,
      "activations/layer5_attention_weight_max": 107.70185852050781,
      "activations/layer5_attention_weight_min": -92.26722717285156,
      "activations/layer6_attention_weight_max": 57.712928771972656,
      "activations/layer6_attention_weight_min": -51.0803108215332,
      "activations/layer7_attention_weight_max": 74.49752044677734,
      "activations/layer7_attention_weight_min": -69.78594970703125,
      "activations/layer8_attention_weight_max": 43.1195182800293,
      "activations/layer8_attention_weight_min": -43.07704544067383,
      "activations/layer9_attention_weight_max": 61.08756637573242,
      "activations/layer9_attention_weight_min": -49.6291618347168,
      "epoch": 5.02,
      "learning_rate": 0.00011882121212121211,
      "loss": 2.8391,
      "step": 86400
    },
    {
      "activations/layer0_attention_weight_max": 13.502920150756836,
      "activations/layer0_attention_weight_min": -15.140970230102539,
      "activations/layer10_attention_weight_max": 40.130523681640625,
      "activations/layer10_attention_weight_min": -34.69317626953125,
      "activations/layer11_attention_weight_max": 31.991409301757812,
      "activations/layer11_attention_weight_min": -27.941070556640625,
      "activations/layer12_attention_weight_max": 37.52076721191406,
      "activations/layer12_attention_weight_min": -26.489723205566406,
      "activations/layer13_attention_weight_max": 52.64268493652344,
      "activations/layer13_attention_weight_min": -33.455875396728516,
      "activations/layer14_attention_weight_max": 41.24211502075195,
      "activations/layer14_attention_weight_min": -27.946556091308594,
      "activations/layer15_attention_weight_max": 41.58573532104492,
      "activations/layer15_attention_weight_min": -29.020240783691406,
      "activations/layer16_attention_weight_max": 36.79381561279297,
      "activations/layer16_attention_weight_min": -26.78009605407715,
      "activations/layer17_attention_weight_max": 58.42105484008789,
      "activations/layer17_attention_weight_min": -37.73114013671875,
      "activations/layer18_attention_weight_max": 46.68291091918945,
      "activations/layer18_attention_weight_min": -33.59794616699219,
      "activations/layer19_attention_weight_max": 21.927717208862305,
      "activations/layer19_attention_weight_min": -17.071613311767578,
      "activations/layer1_attention_weight_max": 16.471399307250977,
      "activations/layer1_attention_weight_min": -16.287425994873047,
      "activations/layer20_attention_weight_max": 20.83913230895996,
      "activations/layer20_attention_weight_min": -22.650100708007812,
      "activations/layer21_attention_weight_max": 40.6724739074707,
      "activations/layer21_attention_weight_min": -24.442270278930664,
      "activations/layer22_attention_weight_max": 35.17634582519531,
      "activations/layer22_attention_weight_min": -21.092390060424805,
      "activations/layer23_attention_weight_max": 28.385208129882812,
      "activations/layer23_attention_weight_min": -18.10137176513672,
      "activations/layer2_attention_weight_max": 30.024486541748047,
      "activations/layer2_attention_weight_min": -28.585050582885742,
      "activations/layer3_attention_weight_max": 82.92330169677734,
      "activations/layer3_attention_weight_min": -86.02327728271484,
      "activations/layer4_attention_weight_max": 100.17583465576172,
      "activations/layer4_attention_weight_min": -97.4198226928711,
      "activations/layer5_attention_weight_max": 101.06871032714844,
      "activations/layer5_attention_weight_min": -91.61962127685547,
      "activations/layer6_attention_weight_max": 57.41434860229492,
      "activations/layer6_attention_weight_min": -52.40492248535156,
      "activations/layer7_attention_weight_max": 68.36601257324219,
      "activations/layer7_attention_weight_min": -66.26597595214844,
      "activations/layer8_attention_weight_max": 42.02845001220703,
      "activations/layer8_attention_weight_min": -40.96238327026367,
      "activations/layer9_attention_weight_max": 55.54397201538086,
      "activations/layer9_attention_weight_min": -44.452213287353516,
      "epoch": 5.02,
      "learning_rate": 0.00011880227272727272,
      "loss": 2.838,
      "step": 86450
    },
    {
      "activations/layer0_attention_weight_max": 16.359960556030273,
      "activations/layer0_attention_weight_min": -14.833489418029785,
      "activations/layer10_attention_weight_max": 38.4292106628418,
      "activations/layer10_attention_weight_min": -34.11751937866211,
      "activations/layer11_attention_weight_max": 30.445159912109375,
      "activations/layer11_attention_weight_min": -28.93991470336914,
      "activations/layer12_attention_weight_max": 20.838003158569336,
      "activations/layer12_attention_weight_min": -25.53394889831543,
      "activations/layer13_attention_weight_max": 46.99032974243164,
      "activations/layer13_attention_weight_min": -34.99479675292969,
      "activations/layer14_attention_weight_max": 53.746212005615234,
      "activations/layer14_attention_weight_min": -37.33476638793945,
      "activations/layer15_attention_weight_max": 38.42756652832031,
      "activations/layer15_attention_weight_min": -33.89494323730469,
      "activations/layer16_attention_weight_max": 34.812557220458984,
      "activations/layer16_attention_weight_min": -27.73228645324707,
      "activations/layer17_attention_weight_max": 58.24641799926758,
      "activations/layer17_attention_weight_min": -43.197669982910156,
      "activations/layer18_attention_weight_max": 46.684627532958984,
      "activations/layer18_attention_weight_min": -35.29655456542969,
      "activations/layer19_attention_weight_max": 21.501602172851562,
      "activations/layer19_attention_weight_min": -19.85944366455078,
      "activations/layer1_attention_weight_max": 17.723955154418945,
      "activations/layer1_attention_weight_min": -15.469745635986328,
      "activations/layer20_attention_weight_max": 25.090669631958008,
      "activations/layer20_attention_weight_min": -20.51956558227539,
      "activations/layer21_attention_weight_max": 35.72748947143555,
      "activations/layer21_attention_weight_min": -26.592979431152344,
      "activations/layer22_attention_weight_max": 34.03340148925781,
      "activations/layer22_attention_weight_min": -26.32271385192871,
      "activations/layer23_attention_weight_max": 25.576757431030273,
      "activations/layer23_attention_weight_min": -19.38991928100586,
      "activations/layer2_attention_weight_max": 31.31998062133789,
      "activations/layer2_attention_weight_min": -30.446971893310547,
      "activations/layer3_attention_weight_max": 81.2020263671875,
      "activations/layer3_attention_weight_min": -83.74641418457031,
      "activations/layer4_attention_weight_max": 102.0811767578125,
      "activations/layer4_attention_weight_min": -94.24357604980469,
      "activations/layer5_attention_weight_max": 102.53856658935547,
      "activations/layer5_attention_weight_min": -93.59269714355469,
      "activations/layer6_attention_weight_max": 56.88603973388672,
      "activations/layer6_attention_weight_min": -53.85684585571289,
      "activations/layer7_attention_weight_max": 68.90837097167969,
      "activations/layer7_attention_weight_min": -66.8880615234375,
      "activations/layer8_attention_weight_max": 43.39969253540039,
      "activations/layer8_attention_weight_min": -42.6497802734375,
      "activations/layer9_attention_weight_max": 48.982364654541016,
      "activations/layer9_attention_weight_min": -45.92415237426758,
      "epoch": 5.03,
      "learning_rate": 0.00011878333333333332,
      "loss": 2.8447,
      "step": 86500
    },
    {
      "activations/layer0_attention_weight_max": 14.58939266204834,
      "activations/layer0_attention_weight_min": -15.23127555847168,
      "activations/layer10_attention_weight_max": 37.519142150878906,
      "activations/layer10_attention_weight_min": -35.93743896484375,
      "activations/layer11_attention_weight_max": 29.57618522644043,
      "activations/layer11_attention_weight_min": -30.46986198425293,
      "activations/layer12_attention_weight_max": 25.156124114990234,
      "activations/layer12_attention_weight_min": -23.857078552246094,
      "activations/layer13_attention_weight_max": 54.902862548828125,
      "activations/layer13_attention_weight_min": -44.89302062988281,
      "activations/layer14_attention_weight_max": 53.07221603393555,
      "activations/layer14_attention_weight_min": -37.253902435302734,
      "activations/layer15_attention_weight_max": 41.940391540527344,
      "activations/layer15_attention_weight_min": -31.86224365234375,
      "activations/layer16_attention_weight_max": 40.81547927856445,
      "activations/layer16_attention_weight_min": -27.32611846923828,
      "activations/layer17_attention_weight_max": 55.53650665283203,
      "activations/layer17_attention_weight_min": -39.46793746948242,
      "activations/layer18_attention_weight_max": 48.83928680419922,
      "activations/layer18_attention_weight_min": -33.998931884765625,
      "activations/layer19_attention_weight_max": 22.63446617126465,
      "activations/layer19_attention_weight_min": -20.273164749145508,
      "activations/layer1_attention_weight_max": 17.457265853881836,
      "activations/layer1_attention_weight_min": -14.5828218460083,
      "activations/layer20_attention_weight_max": 22.9443416595459,
      "activations/layer20_attention_weight_min": -23.025142669677734,
      "activations/layer21_attention_weight_max": 39.96467971801758,
      "activations/layer21_attention_weight_min": -28.14413070678711,
      "activations/layer22_attention_weight_max": 34.660362243652344,
      "activations/layer22_attention_weight_min": -24.73609733581543,
      "activations/layer23_attention_weight_max": 28.017719268798828,
      "activations/layer23_attention_weight_min": -18.29627227783203,
      "activations/layer2_attention_weight_max": 29.516094207763672,
      "activations/layer2_attention_weight_min": -27.477378845214844,
      "activations/layer3_attention_weight_max": 81.94125366210938,
      "activations/layer3_attention_weight_min": -81.82202911376953,
      "activations/layer4_attention_weight_max": 105.59539794921875,
      "activations/layer4_attention_weight_min": -97.5483627319336,
      "activations/layer5_attention_weight_max": 106.62451171875,
      "activations/layer5_attention_weight_min": -94.50422668457031,
      "activations/layer6_attention_weight_max": 60.06858825683594,
      "activations/layer6_attention_weight_min": -54.307498931884766,
      "activations/layer7_attention_weight_max": 73.33934783935547,
      "activations/layer7_attention_weight_min": -66.33391571044922,
      "activations/layer8_attention_weight_max": 44.0428466796875,
      "activations/layer8_attention_weight_min": -42.200538635253906,
      "activations/layer9_attention_weight_max": 50.84199905395508,
      "activations/layer9_attention_weight_min": -52.362056732177734,
      "epoch": 5.03,
      "learning_rate": 0.00011876439393939393,
      "loss": 2.8438,
      "step": 86550
    },
    {
      "activations/layer0_attention_weight_max": 14.271194458007812,
      "activations/layer0_attention_weight_min": -15.073225021362305,
      "activations/layer10_attention_weight_max": 39.570091247558594,
      "activations/layer10_attention_weight_min": -37.171913146972656,
      "activations/layer11_attention_weight_max": 29.861698150634766,
      "activations/layer11_attention_weight_min": -27.68490219116211,
      "activations/layer12_attention_weight_max": 31.001976013183594,
      "activations/layer12_attention_weight_min": -24.13204574584961,
      "activations/layer13_attention_weight_max": 41.7506217956543,
      "activations/layer13_attention_weight_min": -37.94488525390625,
      "activations/layer14_attention_weight_max": 42.29000473022461,
      "activations/layer14_attention_weight_min": -32.86079406738281,
      "activations/layer15_attention_weight_max": 42.21815490722656,
      "activations/layer15_attention_weight_min": -30.63608741760254,
      "activations/layer16_attention_weight_max": 33.20981216430664,
      "activations/layer16_attention_weight_min": -26.964344024658203,
      "activations/layer17_attention_weight_max": 55.727603912353516,
      "activations/layer17_attention_weight_min": -40.983238220214844,
      "activations/layer18_attention_weight_max": 47.778385162353516,
      "activations/layer18_attention_weight_min": -32.45482635498047,
      "activations/layer19_attention_weight_max": 21.389257431030273,
      "activations/layer19_attention_weight_min": -20.14907455444336,
      "activations/layer1_attention_weight_max": 17.707670211791992,
      "activations/layer1_attention_weight_min": -15.684432029724121,
      "activations/layer20_attention_weight_max": 19.80575180053711,
      "activations/layer20_attention_weight_min": -20.55516242980957,
      "activations/layer21_attention_weight_max": 38.81547927856445,
      "activations/layer21_attention_weight_min": -27.286968231201172,
      "activations/layer22_attention_weight_max": 35.04425048828125,
      "activations/layer22_attention_weight_min": -22.594482421875,
      "activations/layer23_attention_weight_max": 29.018661499023438,
      "activations/layer23_attention_weight_min": -19.95681381225586,
      "activations/layer2_attention_weight_max": 29.285390853881836,
      "activations/layer2_attention_weight_min": -27.68060874938965,
      "activations/layer3_attention_weight_max": 81.5849380493164,
      "activations/layer3_attention_weight_min": -81.83822631835938,
      "activations/layer4_attention_weight_max": 100.78679656982422,
      "activations/layer4_attention_weight_min": -95.46692657470703,
      "activations/layer5_attention_weight_max": 100.51225280761719,
      "activations/layer5_attention_weight_min": -96.10545349121094,
      "activations/layer6_attention_weight_max": 60.48786544799805,
      "activations/layer6_attention_weight_min": -55.26962661743164,
      "activations/layer7_attention_weight_max": 71.85002899169922,
      "activations/layer7_attention_weight_min": -69.6892318725586,
      "activations/layer8_attention_weight_max": 43.23554229736328,
      "activations/layer8_attention_weight_min": -46.924564361572266,
      "activations/layer9_attention_weight_max": 48.975440979003906,
      "activations/layer9_attention_weight_min": -47.43928909301758,
      "epoch": 5.03,
      "learning_rate": 0.00011874545454545453,
      "loss": 2.8376,
      "step": 86600
    },
    {
      "activations/layer0_attention_weight_max": 13.884446144104004,
      "activations/layer0_attention_weight_min": -15.0615816116333,
      "activations/layer10_attention_weight_max": 38.82157516479492,
      "activations/layer10_attention_weight_min": -33.32339859008789,
      "activations/layer11_attention_weight_max": 28.15828514099121,
      "activations/layer11_attention_weight_min": -26.56568717956543,
      "activations/layer12_attention_weight_max": 28.05133819580078,
      "activations/layer12_attention_weight_min": -24.725278854370117,
      "activations/layer13_attention_weight_max": 45.191139221191406,
      "activations/layer13_attention_weight_min": -31.500137329101562,
      "activations/layer14_attention_weight_max": 56.24401092529297,
      "activations/layer14_attention_weight_min": -35.78529739379883,
      "activations/layer15_attention_weight_max": 38.771244049072266,
      "activations/layer15_attention_weight_min": -31.046865463256836,
      "activations/layer16_attention_weight_max": 37.71149444580078,
      "activations/layer16_attention_weight_min": -25.0800838470459,
      "activations/layer17_attention_weight_max": 52.723575592041016,
      "activations/layer17_attention_weight_min": -38.5090446472168,
      "activations/layer18_attention_weight_max": 50.050540924072266,
      "activations/layer18_attention_weight_min": -32.861549377441406,
      "activations/layer19_attention_weight_max": 22.414405822753906,
      "activations/layer19_attention_weight_min": -18.511241912841797,
      "activations/layer1_attention_weight_max": 16.466279983520508,
      "activations/layer1_attention_weight_min": -16.408470153808594,
      "activations/layer20_attention_weight_max": 24.097232818603516,
      "activations/layer20_attention_weight_min": -18.315750122070312,
      "activations/layer21_attention_weight_max": 40.87543869018555,
      "activations/layer21_attention_weight_min": -25.43303871154785,
      "activations/layer22_attention_weight_max": 32.848506927490234,
      "activations/layer22_attention_weight_min": -21.881940841674805,
      "activations/layer23_attention_weight_max": 27.08967399597168,
      "activations/layer23_attention_weight_min": -17.146230697631836,
      "activations/layer2_attention_weight_max": 31.422643661499023,
      "activations/layer2_attention_weight_min": -29.907758712768555,
      "activations/layer3_attention_weight_max": 85.26192474365234,
      "activations/layer3_attention_weight_min": -86.00824737548828,
      "activations/layer4_attention_weight_max": 105.0627670288086,
      "activations/layer4_attention_weight_min": -98.98696899414062,
      "activations/layer5_attention_weight_max": 106.68728637695312,
      "activations/layer5_attention_weight_min": -96.44307708740234,
      "activations/layer6_attention_weight_max": 60.01703643798828,
      "activations/layer6_attention_weight_min": -55.530181884765625,
      "activations/layer7_attention_weight_max": 69.06694030761719,
      "activations/layer7_attention_weight_min": -67.22772979736328,
      "activations/layer8_attention_weight_max": 44.68369674682617,
      "activations/layer8_attention_weight_min": -41.17946243286133,
      "activations/layer9_attention_weight_max": 53.634124755859375,
      "activations/layer9_attention_weight_min": -51.708709716796875,
      "epoch": 5.03,
      "learning_rate": 0.00011872651515151514,
      "loss": 2.8594,
      "step": 86650
    },
    {
      "activations/layer0_attention_weight_max": 13.41775131225586,
      "activations/layer0_attention_weight_min": -15.048518180847168,
      "activations/layer10_attention_weight_max": 35.809661865234375,
      "activations/layer10_attention_weight_min": -32.997894287109375,
      "activations/layer11_attention_weight_max": 27.9353084564209,
      "activations/layer11_attention_weight_min": -28.238866806030273,
      "activations/layer12_attention_weight_max": 22.889137268066406,
      "activations/layer12_attention_weight_min": -28.906389236450195,
      "activations/layer13_attention_weight_max": 47.30015563964844,
      "activations/layer13_attention_weight_min": -34.05381774902344,
      "activations/layer14_attention_weight_max": 45.43491744995117,
      "activations/layer14_attention_weight_min": -28.601049423217773,
      "activations/layer15_attention_weight_max": 42.74605941772461,
      "activations/layer15_attention_weight_min": -27.318248748779297,
      "activations/layer16_attention_weight_max": 34.07620620727539,
      "activations/layer16_attention_weight_min": -24.715871810913086,
      "activations/layer17_attention_weight_max": 57.7991943359375,
      "activations/layer17_attention_weight_min": -36.97670364379883,
      "activations/layer18_attention_weight_max": 45.45905685424805,
      "activations/layer18_attention_weight_min": -32.11171340942383,
      "activations/layer19_attention_weight_max": 19.498573303222656,
      "activations/layer19_attention_weight_min": -21.092981338500977,
      "activations/layer1_attention_weight_max": 17.611722946166992,
      "activations/layer1_attention_weight_min": -15.147225379943848,
      "activations/layer20_attention_weight_max": 20.564687728881836,
      "activations/layer20_attention_weight_min": -19.96837043762207,
      "activations/layer21_attention_weight_max": 37.629127502441406,
      "activations/layer21_attention_weight_min": -25.85370445251465,
      "activations/layer22_attention_weight_max": 33.726619720458984,
      "activations/layer22_attention_weight_min": -22.961933135986328,
      "activations/layer23_attention_weight_max": 25.637487411499023,
      "activations/layer23_attention_weight_min": -17.896007537841797,
      "activations/layer2_attention_weight_max": 29.894428253173828,
      "activations/layer2_attention_weight_min": -28.871597290039062,
      "activations/layer3_attention_weight_max": 81.91353607177734,
      "activations/layer3_attention_weight_min": -82.61323547363281,
      "activations/layer4_attention_weight_max": 102.5338363647461,
      "activations/layer4_attention_weight_min": -90.59264373779297,
      "activations/layer5_attention_weight_max": 105.99156951904297,
      "activations/layer5_attention_weight_min": -92.07223510742188,
      "activations/layer6_attention_weight_max": 59.05158615112305,
      "activations/layer6_attention_weight_min": -52.382164001464844,
      "activations/layer7_attention_weight_max": 74.24703216552734,
      "activations/layer7_attention_weight_min": -69.04469299316406,
      "activations/layer8_attention_weight_max": 40.44144821166992,
      "activations/layer8_attention_weight_min": -40.52538299560547,
      "activations/layer9_attention_weight_max": 45.80320739746094,
      "activations/layer9_attention_weight_min": -46.30010986328125,
      "epoch": 5.04,
      "learning_rate": 0.00011870757575757576,
      "loss": 2.8283,
      "step": 86700
    },
    {
      "activations/layer0_attention_weight_max": 15.2095365524292,
      "activations/layer0_attention_weight_min": -15.600545883178711,
      "activations/layer10_attention_weight_max": 40.92950439453125,
      "activations/layer10_attention_weight_min": -34.19033432006836,
      "activations/layer11_attention_weight_max": 29.543033599853516,
      "activations/layer11_attention_weight_min": -27.332656860351562,
      "activations/layer12_attention_weight_max": 21.211774826049805,
      "activations/layer12_attention_weight_min": -35.35786056518555,
      "activations/layer13_attention_weight_max": 46.95995330810547,
      "activations/layer13_attention_weight_min": -33.5168571472168,
      "activations/layer14_attention_weight_max": 51.03680419921875,
      "activations/layer14_attention_weight_min": -33.71228790283203,
      "activations/layer15_attention_weight_max": 39.612422943115234,
      "activations/layer15_attention_weight_min": -29.98250961303711,
      "activations/layer16_attention_weight_max": 32.432498931884766,
      "activations/layer16_attention_weight_min": -25.488439559936523,
      "activations/layer17_attention_weight_max": 59.32190704345703,
      "activations/layer17_attention_weight_min": -38.48271179199219,
      "activations/layer18_attention_weight_max": 46.68149185180664,
      "activations/layer18_attention_weight_min": -33.04076385498047,
      "activations/layer19_attention_weight_max": 23.234272003173828,
      "activations/layer19_attention_weight_min": -20.018295288085938,
      "activations/layer1_attention_weight_max": 17.638269424438477,
      "activations/layer1_attention_weight_min": -16.714195251464844,
      "activations/layer20_attention_weight_max": 21.881975173950195,
      "activations/layer20_attention_weight_min": -20.596548080444336,
      "activations/layer21_attention_weight_max": 38.22557830810547,
      "activations/layer21_attention_weight_min": -26.33120346069336,
      "activations/layer22_attention_weight_max": 33.990352630615234,
      "activations/layer22_attention_weight_min": -25.5682315826416,
      "activations/layer23_attention_weight_max": 26.447227478027344,
      "activations/layer23_attention_weight_min": -19.075965881347656,
      "activations/layer2_attention_weight_max": 30.950902938842773,
      "activations/layer2_attention_weight_min": -29.366775512695312,
      "activations/layer3_attention_weight_max": 80.90689849853516,
      "activations/layer3_attention_weight_min": -84.58851623535156,
      "activations/layer4_attention_weight_max": 102.76399993896484,
      "activations/layer4_attention_weight_min": -98.63114166259766,
      "activations/layer5_attention_weight_max": 106.25222778320312,
      "activations/layer5_attention_weight_min": -93.85247039794922,
      "activations/layer6_attention_weight_max": 60.919132232666016,
      "activations/layer6_attention_weight_min": -55.27116394042969,
      "activations/layer7_attention_weight_max": 76.49813842773438,
      "activations/layer7_attention_weight_min": -71.04977416992188,
      "activations/layer8_attention_weight_max": 44.390018463134766,
      "activations/layer8_attention_weight_min": -42.971519470214844,
      "activations/layer9_attention_weight_max": 62.885799407958984,
      "activations/layer9_attention_weight_min": -54.771175384521484,
      "epoch": 5.04,
      "learning_rate": 0.00011868863636363635,
      "loss": 2.8308,
      "step": 86750
    },
    {
      "activations/layer0_attention_weight_max": 15.166304588317871,
      "activations/layer0_attention_weight_min": -15.351508140563965,
      "activations/layer10_attention_weight_max": 35.43532180786133,
      "activations/layer10_attention_weight_min": -33.27312088012695,
      "activations/layer11_attention_weight_max": 29.83043670654297,
      "activations/layer11_attention_weight_min": -25.848173141479492,
      "activations/layer12_attention_weight_max": 26.23149299621582,
      "activations/layer12_attention_weight_min": -25.672807693481445,
      "activations/layer13_attention_weight_max": 47.18226623535156,
      "activations/layer13_attention_weight_min": -32.55746078491211,
      "activations/layer14_attention_weight_max": 48.62484359741211,
      "activations/layer14_attention_weight_min": -30.982580184936523,
      "activations/layer15_attention_weight_max": 43.07624816894531,
      "activations/layer15_attention_weight_min": -28.271240234375,
      "activations/layer16_attention_weight_max": 33.654537200927734,
      "activations/layer16_attention_weight_min": -25.439821243286133,
      "activations/layer17_attention_weight_max": 58.34466552734375,
      "activations/layer17_attention_weight_min": -37.24284744262695,
      "activations/layer18_attention_weight_max": 45.99936294555664,
      "activations/layer18_attention_weight_min": -33.4056282043457,
      "activations/layer19_attention_weight_max": 22.4140682220459,
      "activations/layer19_attention_weight_min": -19.4202823638916,
      "activations/layer1_attention_weight_max": 16.8236083984375,
      "activations/layer1_attention_weight_min": -14.849932670593262,
      "activations/layer20_attention_weight_max": 22.26689338684082,
      "activations/layer20_attention_weight_min": -22.81340980529785,
      "activations/layer21_attention_weight_max": 37.7135124206543,
      "activations/layer21_attention_weight_min": -24.71324920654297,
      "activations/layer22_attention_weight_max": 30.347232818603516,
      "activations/layer22_attention_weight_min": -21.682579040527344,
      "activations/layer23_attention_weight_max": 26.076831817626953,
      "activations/layer23_attention_weight_min": -18.45749282836914,
      "activations/layer2_attention_weight_max": 29.877042770385742,
      "activations/layer2_attention_weight_min": -28.32474136352539,
      "activations/layer3_attention_weight_max": 80.05378723144531,
      "activations/layer3_attention_weight_min": -82.35718536376953,
      "activations/layer4_attention_weight_max": 98.38318634033203,
      "activations/layer4_attention_weight_min": -93.21343231201172,
      "activations/layer5_attention_weight_max": 100.01945495605469,
      "activations/layer5_attention_weight_min": -91.7281723022461,
      "activations/layer6_attention_weight_max": 57.81061935424805,
      "activations/layer6_attention_weight_min": -53.026004791259766,
      "activations/layer7_attention_weight_max": 69.88909912109375,
      "activations/layer7_attention_weight_min": -65.95613861083984,
      "activations/layer8_attention_weight_max": 41.59402847290039,
      "activations/layer8_attention_weight_min": -39.429813385009766,
      "activations/layer9_attention_weight_max": 56.76237106323242,
      "activations/layer9_attention_weight_min": -46.23491287231445,
      "epoch": 5.04,
      "learning_rate": 0.00011866969696969696,
      "loss": 2.8443,
      "step": 86800
    },
    {
      "activations/layer0_attention_weight_max": 15.104169845581055,
      "activations/layer0_attention_weight_min": -14.877496719360352,
      "activations/layer10_attention_weight_max": 46.44456481933594,
      "activations/layer10_attention_weight_min": -35.845367431640625,
      "activations/layer11_attention_weight_max": 41.291297912597656,
      "activations/layer11_attention_weight_min": -28.032779693603516,
      "activations/layer12_attention_weight_max": 25.660524368286133,
      "activations/layer12_attention_weight_min": -26.349706649780273,
      "activations/layer13_attention_weight_max": 53.10695266723633,
      "activations/layer13_attention_weight_min": -38.84064865112305,
      "activations/layer14_attention_weight_max": 69.89544677734375,
      "activations/layer14_attention_weight_min": -35.911277770996094,
      "activations/layer15_attention_weight_max": 55.89142608642578,
      "activations/layer15_attention_weight_min": -30.788896560668945,
      "activations/layer16_attention_weight_max": 44.98918533325195,
      "activations/layer16_attention_weight_min": -26.712495803833008,
      "activations/layer17_attention_weight_max": 58.52977752685547,
      "activations/layer17_attention_weight_min": -38.10231399536133,
      "activations/layer18_attention_weight_max": 45.297950744628906,
      "activations/layer18_attention_weight_min": -32.89414596557617,
      "activations/layer19_attention_weight_max": 22.483144760131836,
      "activations/layer19_attention_weight_min": -18.918415069580078,
      "activations/layer1_attention_weight_max": 18.622766494750977,
      "activations/layer1_attention_weight_min": -16.952669143676758,
      "activations/layer20_attention_weight_max": 23.962804794311523,
      "activations/layer20_attention_weight_min": -22.700368881225586,
      "activations/layer21_attention_weight_max": 50.64448165893555,
      "activations/layer21_attention_weight_min": -25.558334350585938,
      "activations/layer22_attention_weight_max": 35.11552047729492,
      "activations/layer22_attention_weight_min": -22.168359756469727,
      "activations/layer23_attention_weight_max": 28.164344787597656,
      "activations/layer23_attention_weight_min": -18.886981964111328,
      "activations/layer2_attention_weight_max": 31.975284576416016,
      "activations/layer2_attention_weight_min": -29.413379669189453,
      "activations/layer3_attention_weight_max": 82.7005386352539,
      "activations/layer3_attention_weight_min": -83.22767639160156,
      "activations/layer4_attention_weight_max": 100.7468490600586,
      "activations/layer4_attention_weight_min": -95.14775085449219,
      "activations/layer5_attention_weight_max": 101.1069564819336,
      "activations/layer5_attention_weight_min": -89.38936614990234,
      "activations/layer6_attention_weight_max": 58.09560012817383,
      "activations/layer6_attention_weight_min": -56.562774658203125,
      "activations/layer7_attention_weight_max": 76.66192626953125,
      "activations/layer7_attention_weight_min": -70.46849822998047,
      "activations/layer8_attention_weight_max": 45.92960739135742,
      "activations/layer8_attention_weight_min": -43.36543655395508,
      "activations/layer9_attention_weight_max": 63.888343811035156,
      "activations/layer9_attention_weight_min": -50.869686126708984,
      "epoch": 5.05,
      "learning_rate": 0.00011865075757575758,
      "loss": 2.8436,
      "step": 86850
    },
    {
      "activations/layer0_attention_weight_max": 14.341306686401367,
      "activations/layer0_attention_weight_min": -15.332316398620605,
      "activations/layer10_attention_weight_max": 40.146766662597656,
      "activations/layer10_attention_weight_min": -34.42969512939453,
      "activations/layer11_attention_weight_max": 31.740032196044922,
      "activations/layer11_attention_weight_min": -27.23253631591797,
      "activations/layer12_attention_weight_max": 22.59855842590332,
      "activations/layer12_attention_weight_min": -26.563295364379883,
      "activations/layer13_attention_weight_max": 57.83405685424805,
      "activations/layer13_attention_weight_min": -36.46165084838867,
      "activations/layer14_attention_weight_max": 44.052734375,
      "activations/layer14_attention_weight_min": -28.732620239257812,
      "activations/layer15_attention_weight_max": 40.37969207763672,
      "activations/layer15_attention_weight_min": -29.590513229370117,
      "activations/layer16_attention_weight_max": 38.62329864501953,
      "activations/layer16_attention_weight_min": -25.688671112060547,
      "activations/layer17_attention_weight_max": 57.92237854003906,
      "activations/layer17_attention_weight_min": -37.71370315551758,
      "activations/layer18_attention_weight_max": 48.86972427368164,
      "activations/layer18_attention_weight_min": -32.48125076293945,
      "activations/layer19_attention_weight_max": 21.86205291748047,
      "activations/layer19_attention_weight_min": -19.416614532470703,
      "activations/layer1_attention_weight_max": 16.494773864746094,
      "activations/layer1_attention_weight_min": -15.657210350036621,
      "activations/layer20_attention_weight_max": 23.117103576660156,
      "activations/layer20_attention_weight_min": -22.10838508605957,
      "activations/layer21_attention_weight_max": 39.399513244628906,
      "activations/layer21_attention_weight_min": -28.06558609008789,
      "activations/layer22_attention_weight_max": 36.88130187988281,
      "activations/layer22_attention_weight_min": -25.36785125732422,
      "activations/layer23_attention_weight_max": 31.55563735961914,
      "activations/layer23_attention_weight_min": -17.8828067779541,
      "activations/layer2_attention_weight_max": 29.976598739624023,
      "activations/layer2_attention_weight_min": -30.166990280151367,
      "activations/layer3_attention_weight_max": 81.01653289794922,
      "activations/layer3_attention_weight_min": -85.2389907836914,
      "activations/layer4_attention_weight_max": 103.06603240966797,
      "activations/layer4_attention_weight_min": -93.86084747314453,
      "activations/layer5_attention_weight_max": 107.20738983154297,
      "activations/layer5_attention_weight_min": -94.4515609741211,
      "activations/layer6_attention_weight_max": 60.676795959472656,
      "activations/layer6_attention_weight_min": -55.40813064575195,
      "activations/layer7_attention_weight_max": 71.93351745605469,
      "activations/layer7_attention_weight_min": -67.5704574584961,
      "activations/layer8_attention_weight_max": 42.54132843017578,
      "activations/layer8_attention_weight_min": -40.463134765625,
      "activations/layer9_attention_weight_max": 52.96132278442383,
      "activations/layer9_attention_weight_min": -45.628173828125,
      "epoch": 5.05,
      "learning_rate": 0.00011863181818181816,
      "loss": 2.8398,
      "step": 86900
    },
    {
      "activations/layer0_attention_weight_max": 14.199252128601074,
      "activations/layer0_attention_weight_min": -15.06856918334961,
      "activations/layer10_attention_weight_max": 40.531394958496094,
      "activations/layer10_attention_weight_min": -36.09440994262695,
      "activations/layer11_attention_weight_max": 30.409465789794922,
      "activations/layer11_attention_weight_min": -26.908985137939453,
      "activations/layer12_attention_weight_max": 21.54413414001465,
      "activations/layer12_attention_weight_min": -29.227270126342773,
      "activations/layer13_attention_weight_max": 46.2093620300293,
      "activations/layer13_attention_weight_min": -33.432151794433594,
      "activations/layer14_attention_weight_max": 44.628990173339844,
      "activations/layer14_attention_weight_min": -28.006624221801758,
      "activations/layer15_attention_weight_max": 42.07450866699219,
      "activations/layer15_attention_weight_min": -28.778457641601562,
      "activations/layer16_attention_weight_max": 35.99638748168945,
      "activations/layer16_attention_weight_min": -26.623016357421875,
      "activations/layer17_attention_weight_max": 58.60824966430664,
      "activations/layer17_attention_weight_min": -37.487857818603516,
      "activations/layer18_attention_weight_max": 51.38217544555664,
      "activations/layer18_attention_weight_min": -33.67684555053711,
      "activations/layer19_attention_weight_max": 26.483169555664062,
      "activations/layer19_attention_weight_min": -19.61263084411621,
      "activations/layer1_attention_weight_max": 16.725122451782227,
      "activations/layer1_attention_weight_min": -14.681206703186035,
      "activations/layer20_attention_weight_max": 23.142396926879883,
      "activations/layer20_attention_weight_min": -21.094181060791016,
      "activations/layer21_attention_weight_max": 36.71286392211914,
      "activations/layer21_attention_weight_min": -26.055856704711914,
      "activations/layer22_attention_weight_max": 31.056612014770508,
      "activations/layer22_attention_weight_min": -23.554380416870117,
      "activations/layer23_attention_weight_max": 27.584426879882812,
      "activations/layer23_attention_weight_min": -20.063180923461914,
      "activations/layer2_attention_weight_max": 31.356037139892578,
      "activations/layer2_attention_weight_min": -28.815277099609375,
      "activations/layer3_attention_weight_max": 85.16513061523438,
      "activations/layer3_attention_weight_min": -87.12049865722656,
      "activations/layer4_attention_weight_max": 102.88893127441406,
      "activations/layer4_attention_weight_min": -96.95938873291016,
      "activations/layer5_attention_weight_max": 105.58943939208984,
      "activations/layer5_attention_weight_min": -92.75988006591797,
      "activations/layer6_attention_weight_max": 56.42831802368164,
      "activations/layer6_attention_weight_min": -52.475399017333984,
      "activations/layer7_attention_weight_max": 69.48133850097656,
      "activations/layer7_attention_weight_min": -67.24794006347656,
      "activations/layer8_attention_weight_max": 42.88916778564453,
      "activations/layer8_attention_weight_min": -40.447303771972656,
      "activations/layer9_attention_weight_max": 48.661895751953125,
      "activations/layer9_attention_weight_min": -45.6192626953125,
      "epoch": 5.05,
      "learning_rate": 0.00011861287878787878,
      "loss": 2.842,
      "step": 86950
    },
    {
      "activations/layer0_attention_weight_max": 14.068366050720215,
      "activations/layer0_attention_weight_min": -15.423066139221191,
      "activations/layer10_attention_weight_max": 39.15349197387695,
      "activations/layer10_attention_weight_min": -34.77711868286133,
      "activations/layer11_attention_weight_max": 27.409950256347656,
      "activations/layer11_attention_weight_min": -27.023780822753906,
      "activations/layer12_attention_weight_max": 23.37700080871582,
      "activations/layer12_attention_weight_min": -24.073211669921875,
      "activations/layer13_attention_weight_max": 44.02994918823242,
      "activations/layer13_attention_weight_min": -32.91178512573242,
      "activations/layer14_attention_weight_max": 45.681312561035156,
      "activations/layer14_attention_weight_min": -32.19297409057617,
      "activations/layer15_attention_weight_max": 39.72716522216797,
      "activations/layer15_attention_weight_min": -30.607067108154297,
      "activations/layer16_attention_weight_max": 35.072330474853516,
      "activations/layer16_attention_weight_min": -24.68300437927246,
      "activations/layer17_attention_weight_max": 51.70987319946289,
      "activations/layer17_attention_weight_min": -38.89958572387695,
      "activations/layer18_attention_weight_max": 49.19236755371094,
      "activations/layer18_attention_weight_min": -33.908634185791016,
      "activations/layer19_attention_weight_max": 22.69219207763672,
      "activations/layer19_attention_weight_min": -20.146560668945312,
      "activations/layer1_attention_weight_max": 16.74030113220215,
      "activations/layer1_attention_weight_min": -14.830926895141602,
      "activations/layer20_attention_weight_max": 24.189125061035156,
      "activations/layer20_attention_weight_min": -19.811994552612305,
      "activations/layer21_attention_weight_max": 43.44230651855469,
      "activations/layer21_attention_weight_min": -26.23025131225586,
      "activations/layer22_attention_weight_max": 31.893051147460938,
      "activations/layer22_attention_weight_min": -23.53861427307129,
      "activations/layer23_attention_weight_max": 27.417007446289062,
      "activations/layer23_attention_weight_min": -19.950668334960938,
      "activations/layer2_attention_weight_max": 33.12057876586914,
      "activations/layer2_attention_weight_min": -30.188947677612305,
      "activations/layer3_attention_weight_max": 81.2165298461914,
      "activations/layer3_attention_weight_min": -82.94200897216797,
      "activations/layer4_attention_weight_max": 100.83164978027344,
      "activations/layer4_attention_weight_min": -94.15959930419922,
      "activations/layer5_attention_weight_max": 98.91252899169922,
      "activations/layer5_attention_weight_min": -94.23825073242188,
      "activations/layer6_attention_weight_max": 57.40430450439453,
      "activations/layer6_attention_weight_min": -54.713050842285156,
      "activations/layer7_attention_weight_max": 68.92548370361328,
      "activations/layer7_attention_weight_min": -65.8750228881836,
      "activations/layer8_attention_weight_max": 43.94527816772461,
      "activations/layer8_attention_weight_min": -40.97844314575195,
      "activations/layer9_attention_weight_max": 49.0416145324707,
      "activations/layer9_attention_weight_min": -45.73728561401367,
      "epoch": 5.06,
      "learning_rate": 0.0001185939393939394,
      "loss": 2.8496,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_loss": 2.796875,
      "eval_runtime": 8.5309,
      "eval_samples_per_second": 503.347,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_openwebtext_loss": 2.796875,
      "eval_openwebtext_ppl": 16.39333746248769,
      "eval_openwebtext_runtime": 8.5309,
      "eval_openwebtext_samples_per_second": 503.347,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_wikitext_loss": 3.029296875,
      "eval_wikitext_ppl": 20.68268496272443,
      "eval_wikitext_runtime": 2.0019,
      "eval_wikitext_samples_per_second": 227.782,
      "step": 87000
    },
    {
      "epoch": 5.06,
      "eval_lambada_loss": 2.828125,
      "eval_lambada_ppl": 16.91371785382974,
      "eval_lambada_runtime": 9.5702,
      "eval_lambada_samples_per_second": 508.768,
      "step": 87000
    },
    {
      "activations/layer0_attention_weight_max": 14.6556396484375,
      "activations/layer0_attention_weight_min": -14.980267524719238,
      "activations/layer10_attention_weight_max": 37.346717834472656,
      "activations/layer10_attention_weight_min": -31.333660125732422,
      "activations/layer11_attention_weight_max": 29.796783447265625,
      "activations/layer11_attention_weight_min": -26.54254150390625,
      "activations/layer12_attention_weight_max": 24.260568618774414,
      "activations/layer12_attention_weight_min": -26.7197322845459,
      "activations/layer13_attention_weight_max": 42.769168853759766,
      "activations/layer13_attention_weight_min": -29.55413246154785,
      "activations/layer14_attention_weight_max": 38.46609115600586,
      "activations/layer14_attention_weight_min": -28.041202545166016,
      "activations/layer15_attention_weight_max": 39.51750564575195,
      "activations/layer15_attention_weight_min": -30.31279754638672,
      "activations/layer16_attention_weight_max": 34.24480056762695,
      "activations/layer16_attention_weight_min": -26.248319625854492,
      "activations/layer17_attention_weight_max": 55.89350891113281,
      "activations/layer17_attention_weight_min": -39.02528381347656,
      "activations/layer18_attention_weight_max": 46.221275329589844,
      "activations/layer18_attention_weight_min": -35.268760681152344,
      "activations/layer19_attention_weight_max": 21.56801414489746,
      "activations/layer19_attention_weight_min": -20.68867301940918,
      "activations/layer1_attention_weight_max": 17.52931022644043,
      "activations/layer1_attention_weight_min": -16.69441795349121,
      "activations/layer20_attention_weight_max": 20.879669189453125,
      "activations/layer20_attention_weight_min": -19.420743942260742,
      "activations/layer21_attention_weight_max": 41.295196533203125,
      "activations/layer21_attention_weight_min": -26.778104782104492,
      "activations/layer22_attention_weight_max": 35.27450942993164,
      "activations/layer22_attention_weight_min": -24.45321273803711,
      "activations/layer23_attention_weight_max": 30.05956268310547,
      "activations/layer23_attention_weight_min": -19.749319076538086,
      "activations/layer2_attention_weight_max": 30.65924644470215,
      "activations/layer2_attention_weight_min": -28.783836364746094,
      "activations/layer3_attention_weight_max": 84.44081115722656,
      "activations/layer3_attention_weight_min": -80.80786895751953,
      "activations/layer4_attention_weight_max": 103.63197326660156,
      "activations/layer4_attention_weight_min": -93.71028900146484,
      "activations/layer5_attention_weight_max": 103.21398162841797,
      "activations/layer5_attention_weight_min": -90.66252136230469,
      "activations/layer6_attention_weight_max": 59.52714538574219,
      "activations/layer6_attention_weight_min": -53.62044906616211,
      "activations/layer7_attention_weight_max": 73.11784362792969,
      "activations/layer7_attention_weight_min": -62.72090148925781,
      "activations/layer8_attention_weight_max": 41.726192474365234,
      "activations/layer8_attention_weight_min": -40.65806198120117,
      "activations/layer9_attention_weight_max": 48.94904327392578,
      "activations/layer9_attention_weight_min": -44.420082092285156,
      "epoch": 5.06,
      "learning_rate": 0.00011857499999999998,
      "loss": 2.8464,
      "step": 87050
    },
    {
      "activations/layer0_attention_weight_max": 14.495980262756348,
      "activations/layer0_attention_weight_min": -14.907281875610352,
      "activations/layer10_attention_weight_max": 36.90851593017578,
      "activations/layer10_attention_weight_min": -32.096519470214844,
      "activations/layer11_attention_weight_max": 27.916088104248047,
      "activations/layer11_attention_weight_min": -26.431190490722656,
      "activations/layer12_attention_weight_max": 20.667861938476562,
      "activations/layer12_attention_weight_min": -29.083572387695312,
      "activations/layer13_attention_weight_max": 48.4828987121582,
      "activations/layer13_attention_weight_min": -36.421791076660156,
      "activations/layer14_attention_weight_max": 43.16804504394531,
      "activations/layer14_attention_weight_min": -29.42209815979004,
      "activations/layer15_attention_weight_max": 36.1611442565918,
      "activations/layer15_attention_weight_min": -30.419811248779297,
      "activations/layer16_attention_weight_max": 33.58224105834961,
      "activations/layer16_attention_weight_min": -26.261489868164062,
      "activations/layer17_attention_weight_max": 53.27276611328125,
      "activations/layer17_attention_weight_min": -39.942466735839844,
      "activations/layer18_attention_weight_max": 49.560455322265625,
      "activations/layer18_attention_weight_min": -35.83845138549805,
      "activations/layer19_attention_weight_max": 21.4605770111084,
      "activations/layer19_attention_weight_min": -19.99959373474121,
      "activations/layer1_attention_weight_max": 17.45230484008789,
      "activations/layer1_attention_weight_min": -15.957141876220703,
      "activations/layer20_attention_weight_max": 21.566469192504883,
      "activations/layer20_attention_weight_min": -21.45908546447754,
      "activations/layer21_attention_weight_max": 39.295738220214844,
      "activations/layer21_attention_weight_min": -28.8236083984375,
      "activations/layer22_attention_weight_max": 33.9638786315918,
      "activations/layer22_attention_weight_min": -24.38628387451172,
      "activations/layer23_attention_weight_max": 24.920909881591797,
      "activations/layer23_attention_weight_min": -16.847562789916992,
      "activations/layer2_attention_weight_max": 29.987552642822266,
      "activations/layer2_attention_weight_min": -27.983016967773438,
      "activations/layer3_attention_weight_max": 82.4962158203125,
      "activations/layer3_attention_weight_min": -79.88562774658203,
      "activations/layer4_attention_weight_max": 103.13958740234375,
      "activations/layer4_attention_weight_min": -92.86284637451172,
      "activations/layer5_attention_weight_max": 101.33858489990234,
      "activations/layer5_attention_weight_min": -89.78153991699219,
      "activations/layer6_attention_weight_max": 59.83515167236328,
      "activations/layer6_attention_weight_min": -52.59437561035156,
      "activations/layer7_attention_weight_max": 67.53897857666016,
      "activations/layer7_attention_weight_min": -63.87480545043945,
      "activations/layer8_attention_weight_max": 39.58827209472656,
      "activations/layer8_attention_weight_min": -38.79655075073242,
      "activations/layer9_attention_weight_max": 44.52651596069336,
      "activations/layer9_attention_weight_min": -47.04193878173828,
      "epoch": 5.06,
      "learning_rate": 0.0001185560606060606,
      "loss": 2.8512,
      "step": 87100
    },
    {
      "activations/layer0_attention_weight_max": 14.83922004699707,
      "activations/layer0_attention_weight_min": -14.912117958068848,
      "activations/layer10_attention_weight_max": 38.385807037353516,
      "activations/layer10_attention_weight_min": -33.99835205078125,
      "activations/layer11_attention_weight_max": 29.768917083740234,
      "activations/layer11_attention_weight_min": -26.12062644958496,
      "activations/layer12_attention_weight_max": 22.718908309936523,
      "activations/layer12_attention_weight_min": -22.289453506469727,
      "activations/layer13_attention_weight_max": 47.05229187011719,
      "activations/layer13_attention_weight_min": -32.82607650756836,
      "activations/layer14_attention_weight_max": 49.65176773071289,
      "activations/layer14_attention_weight_min": -28.027469635009766,
      "activations/layer15_attention_weight_max": 41.122806549072266,
      "activations/layer15_attention_weight_min": -30.27783966064453,
      "activations/layer16_attention_weight_max": 35.272220611572266,
      "activations/layer16_attention_weight_min": -27.07484245300293,
      "activations/layer17_attention_weight_max": 59.98440933227539,
      "activations/layer17_attention_weight_min": -38.341121673583984,
      "activations/layer18_attention_weight_max": 51.65449142456055,
      "activations/layer18_attention_weight_min": -33.881500244140625,
      "activations/layer19_attention_weight_max": 22.721120834350586,
      "activations/layer19_attention_weight_min": -22.04103660583496,
      "activations/layer1_attention_weight_max": 17.112943649291992,
      "activations/layer1_attention_weight_min": -14.82656192779541,
      "activations/layer20_attention_weight_max": 23.228792190551758,
      "activations/layer20_attention_weight_min": -22.40378761291504,
      "activations/layer21_attention_weight_max": 40.592254638671875,
      "activations/layer21_attention_weight_min": -24.77773094177246,
      "activations/layer22_attention_weight_max": 36.449562072753906,
      "activations/layer22_attention_weight_min": -23.92780113220215,
      "activations/layer23_attention_weight_max": 28.373851776123047,
      "activations/layer23_attention_weight_min": -19.99115562438965,
      "activations/layer2_attention_weight_max": 30.62281608581543,
      "activations/layer2_attention_weight_min": -28.53606414794922,
      "activations/layer3_attention_weight_max": 75.53131103515625,
      "activations/layer3_attention_weight_min": -80.21923828125,
      "activations/layer4_attention_weight_max": 99.19243621826172,
      "activations/layer4_attention_weight_min": -91.78264617919922,
      "activations/layer5_attention_weight_max": 104.31979370117188,
      "activations/layer5_attention_weight_min": -95.53607177734375,
      "activations/layer6_attention_weight_max": 56.17780303955078,
      "activations/layer6_attention_weight_min": -56.181434631347656,
      "activations/layer7_attention_weight_max": 68.57539367675781,
      "activations/layer7_attention_weight_min": -66.93256378173828,
      "activations/layer8_attention_weight_max": 40.24861526489258,
      "activations/layer8_attention_weight_min": -40.24668502807617,
      "activations/layer9_attention_weight_max": 49.012115478515625,
      "activations/layer9_attention_weight_min": -45.8394889831543,
      "epoch": 5.06,
      "learning_rate": 0.0001185371212121212,
      "loss": 2.8488,
      "step": 87150
    },
    {
      "activations/layer0_attention_weight_max": 14.835792541503906,
      "activations/layer0_attention_weight_min": -15.392590522766113,
      "activations/layer10_attention_weight_max": 35.57110595703125,
      "activations/layer10_attention_weight_min": -33.80680465698242,
      "activations/layer11_attention_weight_max": 29.000713348388672,
      "activations/layer11_attention_weight_min": -26.876142501831055,
      "activations/layer12_attention_weight_max": 22.53849220275879,
      "activations/layer12_attention_weight_min": -25.151451110839844,
      "activations/layer13_attention_weight_max": 43.69431686401367,
      "activations/layer13_attention_weight_min": -37.05779266357422,
      "activations/layer14_attention_weight_max": 41.746795654296875,
      "activations/layer14_attention_weight_min": -29.99910545349121,
      "activations/layer15_attention_weight_max": 36.19847106933594,
      "activations/layer15_attention_weight_min": -28.757713317871094,
      "activations/layer16_attention_weight_max": 33.96710205078125,
      "activations/layer16_attention_weight_min": -27.28767204284668,
      "activations/layer17_attention_weight_max": 54.61383819580078,
      "activations/layer17_attention_weight_min": -40.94524383544922,
      "activations/layer18_attention_weight_max": 44.77089309692383,
      "activations/layer18_attention_weight_min": -36.03063201904297,
      "activations/layer19_attention_weight_max": 19.793874740600586,
      "activations/layer19_attention_weight_min": -20.1862850189209,
      "activations/layer1_attention_weight_max": 18.645952224731445,
      "activations/layer1_attention_weight_min": -16.499149322509766,
      "activations/layer20_attention_weight_max": 22.427276611328125,
      "activations/layer20_attention_weight_min": -25.845787048339844,
      "activations/layer21_attention_weight_max": 38.32622146606445,
      "activations/layer21_attention_weight_min": -25.125730514526367,
      "activations/layer22_attention_weight_max": 36.18592071533203,
      "activations/layer22_attention_weight_min": -23.83511734008789,
      "activations/layer23_attention_weight_max": 28.68074607849121,
      "activations/layer23_attention_weight_min": -17.612579345703125,
      "activations/layer2_attention_weight_max": 29.186918258666992,
      "activations/layer2_attention_weight_min": -27.88282012939453,
      "activations/layer3_attention_weight_max": 81.41826629638672,
      "activations/layer3_attention_weight_min": -83.46969604492188,
      "activations/layer4_attention_weight_max": 102.15349578857422,
      "activations/layer4_attention_weight_min": -93.6534423828125,
      "activations/layer5_attention_weight_max": 99.4275131225586,
      "activations/layer5_attention_weight_min": -93.28439331054688,
      "activations/layer6_attention_weight_max": 58.4208984375,
      "activations/layer6_attention_weight_min": -52.047828674316406,
      "activations/layer7_attention_weight_max": 69.92977905273438,
      "activations/layer7_attention_weight_min": -67.96739959716797,
      "activations/layer8_attention_weight_max": 40.44649887084961,
      "activations/layer8_attention_weight_min": -39.55643081665039,
      "activations/layer9_attention_weight_max": 49.07198715209961,
      "activations/layer9_attention_weight_min": -42.48651123046875,
      "epoch": 5.07,
      "learning_rate": 0.0001185181818181818,
      "loss": 2.8365,
      "step": 87200
    },
    {
      "activations/layer0_attention_weight_max": 13.815274238586426,
      "activations/layer0_attention_weight_min": -15.0811185836792,
      "activations/layer10_attention_weight_max": 37.53264236450195,
      "activations/layer10_attention_weight_min": -34.83869171142578,
      "activations/layer11_attention_weight_max": 29.709548950195312,
      "activations/layer11_attention_weight_min": -28.006275177001953,
      "activations/layer12_attention_weight_max": 23.848716735839844,
      "activations/layer12_attention_weight_min": -31.855554580688477,
      "activations/layer13_attention_weight_max": 50.24770736694336,
      "activations/layer13_attention_weight_min": -35.811180114746094,
      "activations/layer14_attention_weight_max": 52.74065399169922,
      "activations/layer14_attention_weight_min": -36.56205368041992,
      "activations/layer15_attention_weight_max": 40.75087356567383,
      "activations/layer15_attention_weight_min": -30.150115966796875,
      "activations/layer16_attention_weight_max": 33.84535598754883,
      "activations/layer16_attention_weight_min": -26.886703491210938,
      "activations/layer17_attention_weight_max": 55.830543518066406,
      "activations/layer17_attention_weight_min": -37.97847366333008,
      "activations/layer18_attention_weight_max": 47.795631408691406,
      "activations/layer18_attention_weight_min": -35.077362060546875,
      "activations/layer19_attention_weight_max": 20.949390411376953,
      "activations/layer19_attention_weight_min": -18.089651107788086,
      "activations/layer1_attention_weight_max": 18.67171287536621,
      "activations/layer1_attention_weight_min": -16.515277862548828,
      "activations/layer20_attention_weight_max": 23.066194534301758,
      "activations/layer20_attention_weight_min": -17.805320739746094,
      "activations/layer21_attention_weight_max": 37.84607696533203,
      "activations/layer21_attention_weight_min": -27.046506881713867,
      "activations/layer22_attention_weight_max": 33.704681396484375,
      "activations/layer22_attention_weight_min": -23.746511459350586,
      "activations/layer23_attention_weight_max": 27.788681030273438,
      "activations/layer23_attention_weight_min": -16.971715927124023,
      "activations/layer2_attention_weight_max": 31.968536376953125,
      "activations/layer2_attention_weight_min": -30.598438262939453,
      "activations/layer3_attention_weight_max": 80.32942962646484,
      "activations/layer3_attention_weight_min": -86.09275817871094,
      "activations/layer4_attention_weight_max": 100.99625396728516,
      "activations/layer4_attention_weight_min": -93.55425262451172,
      "activations/layer5_attention_weight_max": 97.98959350585938,
      "activations/layer5_attention_weight_min": -91.22402954101562,
      "activations/layer6_attention_weight_max": 55.47565841674805,
      "activations/layer6_attention_weight_min": -50.21052551269531,
      "activations/layer7_attention_weight_max": 71.58489227294922,
      "activations/layer7_attention_weight_min": -65.62481689453125,
      "activations/layer8_attention_weight_max": 42.27998352050781,
      "activations/layer8_attention_weight_min": -40.165958404541016,
      "activations/layer9_attention_weight_max": 54.94222640991211,
      "activations/layer9_attention_weight_min": -45.86259078979492,
      "epoch": 5.07,
      "learning_rate": 0.0001184996212121212,
      "loss": 2.8433,
      "step": 87250
    },
    {
      "activations/layer0_attention_weight_max": 14.603768348693848,
      "activations/layer0_attention_weight_min": -15.193244934082031,
      "activations/layer10_attention_weight_max": 38.62417221069336,
      "activations/layer10_attention_weight_min": -36.63907241821289,
      "activations/layer11_attention_weight_max": 32.01469802856445,
      "activations/layer11_attention_weight_min": -28.105884552001953,
      "activations/layer12_attention_weight_max": 22.31022834777832,
      "activations/layer12_attention_weight_min": -25.16240119934082,
      "activations/layer13_attention_weight_max": 52.45772171020508,
      "activations/layer13_attention_weight_min": -37.6873664855957,
      "activations/layer14_attention_weight_max": 40.52778244018555,
      "activations/layer14_attention_weight_min": -31.753267288208008,
      "activations/layer15_attention_weight_max": 49.650657653808594,
      "activations/layer15_attention_weight_min": -30.597869873046875,
      "activations/layer16_attention_weight_max": 31.543045043945312,
      "activations/layer16_attention_weight_min": -26.583606719970703,
      "activations/layer17_attention_weight_max": 58.33538055419922,
      "activations/layer17_attention_weight_min": -38.99045181274414,
      "activations/layer18_attention_weight_max": 49.39737319946289,
      "activations/layer18_attention_weight_min": -33.0848274230957,
      "activations/layer19_attention_weight_max": 18.72022819519043,
      "activations/layer19_attention_weight_min": -17.545812606811523,
      "activations/layer1_attention_weight_max": 17.111299514770508,
      "activations/layer1_attention_weight_min": -15.01164436340332,
      "activations/layer20_attention_weight_max": 19.81602668762207,
      "activations/layer20_attention_weight_min": -19.572113037109375,
      "activations/layer21_attention_weight_max": 38.43415832519531,
      "activations/layer21_attention_weight_min": -26.005558013916016,
      "activations/layer22_attention_weight_max": 35.21577835083008,
      "activations/layer22_attention_weight_min": -23.76158905029297,
      "activations/layer23_attention_weight_max": 26.914051055908203,
      "activations/layer23_attention_weight_min": -17.54767417907715,
      "activations/layer2_attention_weight_max": 29.82648468017578,
      "activations/layer2_attention_weight_min": -26.894794464111328,
      "activations/layer3_attention_weight_max": 80.59300994873047,
      "activations/layer3_attention_weight_min": -84.65245819091797,
      "activations/layer4_attention_weight_max": 98.76998138427734,
      "activations/layer4_attention_weight_min": -97.96478271484375,
      "activations/layer5_attention_weight_max": 97.7868881225586,
      "activations/layer5_attention_weight_min": -93.79632568359375,
      "activations/layer6_attention_weight_max": 56.01349639892578,
      "activations/layer6_attention_weight_min": -52.444087982177734,
      "activations/layer7_attention_weight_max": 70.96450805664062,
      "activations/layer7_attention_weight_min": -71.86490631103516,
      "activations/layer8_attention_weight_max": 41.635372161865234,
      "activations/layer8_attention_weight_min": -42.80096435546875,
      "activations/layer9_attention_weight_max": 47.241031646728516,
      "activations/layer9_attention_weight_min": -47.14346694946289,
      "epoch": 5.07,
      "learning_rate": 0.0001184806818181818,
      "loss": 2.8493,
      "step": 87300
    },
    {
      "activations/layer0_attention_weight_max": 15.041730880737305,
      "activations/layer0_attention_weight_min": -15.616728782653809,
      "activations/layer10_attention_weight_max": 37.4764518737793,
      "activations/layer10_attention_weight_min": -31.156402587890625,
      "activations/layer11_attention_weight_max": 28.263547897338867,
      "activations/layer11_attention_weight_min": -26.715152740478516,
      "activations/layer12_attention_weight_max": 20.96517562866211,
      "activations/layer12_attention_weight_min": -23.13534927368164,
      "activations/layer13_attention_weight_max": 44.50352478027344,
      "activations/layer13_attention_weight_min": -32.019405364990234,
      "activations/layer14_attention_weight_max": 38.959808349609375,
      "activations/layer14_attention_weight_min": -29.290498733520508,
      "activations/layer15_attention_weight_max": 39.39414978027344,
      "activations/layer15_attention_weight_min": -31.5051212310791,
      "activations/layer16_attention_weight_max": 35.35139846801758,
      "activations/layer16_attention_weight_min": -25.199729919433594,
      "activations/layer17_attention_weight_max": 58.32655715942383,
      "activations/layer17_attention_weight_min": -38.65217971801758,
      "activations/layer18_attention_weight_max": 47.009376525878906,
      "activations/layer18_attention_weight_min": -34.086509704589844,
      "activations/layer19_attention_weight_max": 20.821876525878906,
      "activations/layer19_attention_weight_min": -18.823461532592773,
      "activations/layer1_attention_weight_max": 18.14371109008789,
      "activations/layer1_attention_weight_min": -15.126341819763184,
      "activations/layer20_attention_weight_max": 21.566057205200195,
      "activations/layer20_attention_weight_min": -19.430578231811523,
      "activations/layer21_attention_weight_max": 36.030948638916016,
      "activations/layer21_attention_weight_min": -27.19704818725586,
      "activations/layer22_attention_weight_max": 33.431331634521484,
      "activations/layer22_attention_weight_min": -25.796541213989258,
      "activations/layer23_attention_weight_max": 27.176929473876953,
      "activations/layer23_attention_weight_min": -18.16492462158203,
      "activations/layer2_attention_weight_max": 30.250425338745117,
      "activations/layer2_attention_weight_min": -28.632930755615234,
      "activations/layer3_attention_weight_max": 80.29104614257812,
      "activations/layer3_attention_weight_min": -85.9231185913086,
      "activations/layer4_attention_weight_max": 99.83522033691406,
      "activations/layer4_attention_weight_min": -91.35770416259766,
      "activations/layer5_attention_weight_max": 101.1004867553711,
      "activations/layer5_attention_weight_min": -89.88236999511719,
      "activations/layer6_attention_weight_max": 61.49988555908203,
      "activations/layer6_attention_weight_min": -53.202308654785156,
      "activations/layer7_attention_weight_max": 69.91358947753906,
      "activations/layer7_attention_weight_min": -70.12036895751953,
      "activations/layer8_attention_weight_max": 40.46412658691406,
      "activations/layer8_attention_weight_min": -40.889305114746094,
      "activations/layer9_attention_weight_max": 49.37855911254883,
      "activations/layer9_attention_weight_min": -44.269283294677734,
      "epoch": 5.08,
      "learning_rate": 0.00011846174242424241,
      "loss": 2.8433,
      "step": 87350
    },
    {
      "activations/layer0_attention_weight_max": 14.662302017211914,
      "activations/layer0_attention_weight_min": -15.42036247253418,
      "activations/layer10_attention_weight_max": 37.577632904052734,
      "activations/layer10_attention_weight_min": -34.616607666015625,
      "activations/layer11_attention_weight_max": 32.35171127319336,
      "activations/layer11_attention_weight_min": -28.683759689331055,
      "activations/layer12_attention_weight_max": 22.048152923583984,
      "activations/layer12_attention_weight_min": -22.40829849243164,
      "activations/layer13_attention_weight_max": 48.84993362426758,
      "activations/layer13_attention_weight_min": -35.080718994140625,
      "activations/layer14_attention_weight_max": 46.26346969604492,
      "activations/layer14_attention_weight_min": -31.860445022583008,
      "activations/layer15_attention_weight_max": 41.17825698852539,
      "activations/layer15_attention_weight_min": -29.919919967651367,
      "activations/layer16_attention_weight_max": 34.725425720214844,
      "activations/layer16_attention_weight_min": -25.55318260192871,
      "activations/layer17_attention_weight_max": 65.44998168945312,
      "activations/layer17_attention_weight_min": -39.15702819824219,
      "activations/layer18_attention_weight_max": 50.25604248046875,
      "activations/layer18_attention_weight_min": -33.11049270629883,
      "activations/layer19_attention_weight_max": 24.91388511657715,
      "activations/layer19_attention_weight_min": -20.243133544921875,
      "activations/layer1_attention_weight_max": 17.595918655395508,
      "activations/layer1_attention_weight_min": -13.812652587890625,
      "activations/layer20_attention_weight_max": 23.02820587158203,
      "activations/layer20_attention_weight_min": -22.99677848815918,
      "activations/layer21_attention_weight_max": 44.131072998046875,
      "activations/layer21_attention_weight_min": -24.543643951416016,
      "activations/layer22_attention_weight_max": 31.39572525024414,
      "activations/layer22_attention_weight_min": -24.305370330810547,
      "activations/layer23_attention_weight_max": 27.91468620300293,
      "activations/layer23_attention_weight_min": -18.673858642578125,
      "activations/layer2_attention_weight_max": 30.203847885131836,
      "activations/layer2_attention_weight_min": -27.58041763305664,
      "activations/layer3_attention_weight_max": 78.46969604492188,
      "activations/layer3_attention_weight_min": -81.45321655273438,
      "activations/layer4_attention_weight_max": 100.16450500488281,
      "activations/layer4_attention_weight_min": -93.02932739257812,
      "activations/layer5_attention_weight_max": 103.46453857421875,
      "activations/layer5_attention_weight_min": -89.3141098022461,
      "activations/layer6_attention_weight_max": 58.33769607543945,
      "activations/layer6_attention_weight_min": -52.22565841674805,
      "activations/layer7_attention_weight_max": 70.10977935791016,
      "activations/layer7_attention_weight_min": -65.85778045654297,
      "activations/layer8_attention_weight_max": 41.871543884277344,
      "activations/layer8_attention_weight_min": -39.45676040649414,
      "activations/layer9_attention_weight_max": 47.07379150390625,
      "activations/layer9_attention_weight_min": -44.05531311035156,
      "epoch": 5.08,
      "learning_rate": 0.00011844280303030302,
      "loss": 2.8426,
      "step": 87400
    },
    {
      "activations/layer0_attention_weight_max": 14.335554122924805,
      "activations/layer0_attention_weight_min": -14.98289966583252,
      "activations/layer10_attention_weight_max": 37.90502166748047,
      "activations/layer10_attention_weight_min": -34.658023834228516,
      "activations/layer11_attention_weight_max": 32.76099395751953,
      "activations/layer11_attention_weight_min": -27.385665893554688,
      "activations/layer12_attention_weight_max": 22.9092960357666,
      "activations/layer12_attention_weight_min": -26.03082847595215,
      "activations/layer13_attention_weight_max": 52.15431594848633,
      "activations/layer13_attention_weight_min": -37.06884002685547,
      "activations/layer14_attention_weight_max": 45.29071044921875,
      "activations/layer14_attention_weight_min": -33.31418991088867,
      "activations/layer15_attention_weight_max": 44.49387741088867,
      "activations/layer15_attention_weight_min": -31.323942184448242,
      "activations/layer16_attention_weight_max": 39.68722152709961,
      "activations/layer16_attention_weight_min": -28.026966094970703,
      "activations/layer17_attention_weight_max": 60.03838348388672,
      "activations/layer17_attention_weight_min": -39.64614486694336,
      "activations/layer18_attention_weight_max": 50.736995697021484,
      "activations/layer18_attention_weight_min": -34.79196548461914,
      "activations/layer19_attention_weight_max": 22.214967727661133,
      "activations/layer19_attention_weight_min": -19.851716995239258,
      "activations/layer1_attention_weight_max": 17.807819366455078,
      "activations/layer1_attention_weight_min": -12.60558795928955,
      "activations/layer20_attention_weight_max": 21.644376754760742,
      "activations/layer20_attention_weight_min": -23.22226905822754,
      "activations/layer21_attention_weight_max": 39.85004425048828,
      "activations/layer21_attention_weight_min": -25.179834365844727,
      "activations/layer22_attention_weight_max": 35.075130462646484,
      "activations/layer22_attention_weight_min": -23.82764434814453,
      "activations/layer23_attention_weight_max": 26.71172332763672,
      "activations/layer23_attention_weight_min": -18.41602897644043,
      "activations/layer2_attention_weight_max": 29.47942352294922,
      "activations/layer2_attention_weight_min": -27.213890075683594,
      "activations/layer3_attention_weight_max": 78.4775161743164,
      "activations/layer3_attention_weight_min": -78.78642272949219,
      "activations/layer4_attention_weight_max": 95.9535140991211,
      "activations/layer4_attention_weight_min": -91.87349700927734,
      "activations/layer5_attention_weight_max": 97.14369201660156,
      "activations/layer5_attention_weight_min": -89.03962707519531,
      "activations/layer6_attention_weight_max": 54.696102142333984,
      "activations/layer6_attention_weight_min": -50.81118392944336,
      "activations/layer7_attention_weight_max": 71.72422790527344,
      "activations/layer7_attention_weight_min": -65.78736877441406,
      "activations/layer8_attention_weight_max": 40.84457778930664,
      "activations/layer8_attention_weight_min": -41.03472137451172,
      "activations/layer9_attention_weight_max": 48.103858947753906,
      "activations/layer9_attention_weight_min": -45.15689468383789,
      "epoch": 5.08,
      "learning_rate": 0.00011842386363636362,
      "loss": 2.8467,
      "step": 87450
    },
    {
      "activations/layer0_attention_weight_max": 14.10590648651123,
      "activations/layer0_attention_weight_min": -14.640358924865723,
      "activations/layer10_attention_weight_max": 39.18300247192383,
      "activations/layer10_attention_weight_min": -33.825233459472656,
      "activations/layer11_attention_weight_max": 29.521129608154297,
      "activations/layer11_attention_weight_min": -26.79151725769043,
      "activations/layer12_attention_weight_max": 29.578624725341797,
      "activations/layer12_attention_weight_min": -27.33902931213379,
      "activations/layer13_attention_weight_max": 49.887447357177734,
      "activations/layer13_attention_weight_min": -36.58735656738281,
      "activations/layer14_attention_weight_max": 37.40682601928711,
      "activations/layer14_attention_weight_min": -27.209280014038086,
      "activations/layer15_attention_weight_max": 37.73141098022461,
      "activations/layer15_attention_weight_min": -28.53205680847168,
      "activations/layer16_attention_weight_max": 35.43400955200195,
      "activations/layer16_attention_weight_min": -25.27215003967285,
      "activations/layer17_attention_weight_max": 56.119171142578125,
      "activations/layer17_attention_weight_min": -39.46236801147461,
      "activations/layer18_attention_weight_max": 47.83666229248047,
      "activations/layer18_attention_weight_min": -31.922948837280273,
      "activations/layer19_attention_weight_max": 23.225889205932617,
      "activations/layer19_attention_weight_min": -19.107263565063477,
      "activations/layer1_attention_weight_max": 18.121177673339844,
      "activations/layer1_attention_weight_min": -15.171924591064453,
      "activations/layer20_attention_weight_max": 20.38106918334961,
      "activations/layer20_attention_weight_min": -20.558042526245117,
      "activations/layer21_attention_weight_max": 37.21388244628906,
      "activations/layer21_attention_weight_min": -27.975601196289062,
      "activations/layer22_attention_weight_max": 33.94120407104492,
      "activations/layer22_attention_weight_min": -25.35194206237793,
      "activations/layer23_attention_weight_max": 29.728622436523438,
      "activations/layer23_attention_weight_min": -18.819419860839844,
      "activations/layer2_attention_weight_max": 29.26399040222168,
      "activations/layer2_attention_weight_min": -27.31496810913086,
      "activations/layer3_attention_weight_max": 76.67182159423828,
      "activations/layer3_attention_weight_min": -79.29386901855469,
      "activations/layer4_attention_weight_max": 105.05245208740234,
      "activations/layer4_attention_weight_min": -92.42546844482422,
      "activations/layer5_attention_weight_max": 110.51559448242188,
      "activations/layer5_attention_weight_min": -94.66290283203125,
      "activations/layer6_attention_weight_max": 63.82124328613281,
      "activations/layer6_attention_weight_min": -55.00212860107422,
      "activations/layer7_attention_weight_max": 74.72737884521484,
      "activations/layer7_attention_weight_min": -73.90348052978516,
      "activations/layer8_attention_weight_max": 43.84807205200195,
      "activations/layer8_attention_weight_min": -43.010704040527344,
      "activations/layer9_attention_weight_max": 52.39236068725586,
      "activations/layer9_attention_weight_min": -48.826751708984375,
      "epoch": 5.08,
      "learning_rate": 0.00011840492424242423,
      "loss": 2.8426,
      "step": 87500
    },
    {
      "activations/layer0_attention_weight_max": 15.074078559875488,
      "activations/layer0_attention_weight_min": -14.540282249450684,
      "activations/layer10_attention_weight_max": 38.79729080200195,
      "activations/layer10_attention_weight_min": -34.31488800048828,
      "activations/layer11_attention_weight_max": 29.18391227722168,
      "activations/layer11_attention_weight_min": -26.804790496826172,
      "activations/layer12_attention_weight_max": 23.35181999206543,
      "activations/layer12_attention_weight_min": -22.292333602905273,
      "activations/layer13_attention_weight_max": 48.19896697998047,
      "activations/layer13_attention_weight_min": -38.84058380126953,
      "activations/layer14_attention_weight_max": 37.18879318237305,
      "activations/layer14_attention_weight_min": -29.875286102294922,
      "activations/layer15_attention_weight_max": 40.37152862548828,
      "activations/layer15_attention_weight_min": -28.44978904724121,
      "activations/layer16_attention_weight_max": 34.98419952392578,
      "activations/layer16_attention_weight_min": -25.37466812133789,
      "activations/layer17_attention_weight_max": 54.69397735595703,
      "activations/layer17_attention_weight_min": -37.95293045043945,
      "activations/layer18_attention_weight_max": 47.91600036621094,
      "activations/layer18_attention_weight_min": -34.14308547973633,
      "activations/layer19_attention_weight_max": 23.31328010559082,
      "activations/layer19_attention_weight_min": -18.317974090576172,
      "activations/layer1_attention_weight_max": 16.90666389465332,
      "activations/layer1_attention_weight_min": -12.00112533569336,
      "activations/layer20_attention_weight_max": 20.51113510131836,
      "activations/layer20_attention_weight_min": -20.850082397460938,
      "activations/layer21_attention_weight_max": 39.4619026184082,
      "activations/layer21_attention_weight_min": -27.726402282714844,
      "activations/layer22_attention_weight_max": 33.02196502685547,
      "activations/layer22_attention_weight_min": -26.152076721191406,
      "activations/layer23_attention_weight_max": 25.352313995361328,
      "activations/layer23_attention_weight_min": -18.832971572875977,
      "activations/layer2_attention_weight_max": 28.525550842285156,
      "activations/layer2_attention_weight_min": -26.188308715820312,
      "activations/layer3_attention_weight_max": 75.41512298583984,
      "activations/layer3_attention_weight_min": -81.39097595214844,
      "activations/layer4_attention_weight_max": 94.93217468261719,
      "activations/layer4_attention_weight_min": -89.34994506835938,
      "activations/layer5_attention_weight_max": 100.8856430053711,
      "activations/layer5_attention_weight_min": -93.98800659179688,
      "activations/layer6_attention_weight_max": 58.81310272216797,
      "activations/layer6_attention_weight_min": -52.71842575073242,
      "activations/layer7_attention_weight_max": 72.44539642333984,
      "activations/layer7_attention_weight_min": -67.7396240234375,
      "activations/layer8_attention_weight_max": 45.86293029785156,
      "activations/layer8_attention_weight_min": -44.7284049987793,
      "activations/layer9_attention_weight_max": 50.4206657409668,
      "activations/layer9_attention_weight_min": -44.65549087524414,
      "epoch": 5.09,
      "learning_rate": 0.00011838636363636362,
      "loss": 2.8423,
      "step": 87550
    },
    {
      "activations/layer0_attention_weight_max": 15.85102653503418,
      "activations/layer0_attention_weight_min": -14.780045509338379,
      "activations/layer10_attention_weight_max": 39.85820388793945,
      "activations/layer10_attention_weight_min": -32.59257507324219,
      "activations/layer11_attention_weight_max": 29.98196029663086,
      "activations/layer11_attention_weight_min": -25.738554000854492,
      "activations/layer12_attention_weight_max": 20.409881591796875,
      "activations/layer12_attention_weight_min": -24.06696891784668,
      "activations/layer13_attention_weight_max": 44.722984313964844,
      "activations/layer13_attention_weight_min": -36.428714752197266,
      "activations/layer14_attention_weight_max": 53.200721740722656,
      "activations/layer14_attention_weight_min": -32.017051696777344,
      "activations/layer15_attention_weight_max": 41.2503662109375,
      "activations/layer15_attention_weight_min": -30.013673782348633,
      "activations/layer16_attention_weight_max": 34.99061965942383,
      "activations/layer16_attention_weight_min": -26.974925994873047,
      "activations/layer17_attention_weight_max": 55.66032409667969,
      "activations/layer17_attention_weight_min": -40.709022521972656,
      "activations/layer18_attention_weight_max": 48.320674896240234,
      "activations/layer18_attention_weight_min": -32.452720642089844,
      "activations/layer19_attention_weight_max": 21.817283630371094,
      "activations/layer19_attention_weight_min": -19.803770065307617,
      "activations/layer1_attention_weight_max": 17.11752700805664,
      "activations/layer1_attention_weight_min": -13.189326286315918,
      "activations/layer20_attention_weight_max": 24.302209854125977,
      "activations/layer20_attention_weight_min": -20.09557342529297,
      "activations/layer21_attention_weight_max": 44.36298751831055,
      "activations/layer21_attention_weight_min": -25.391441345214844,
      "activations/layer22_attention_weight_max": 37.57050704956055,
      "activations/layer22_attention_weight_min": -23.563453674316406,
      "activations/layer23_attention_weight_max": 26.827892303466797,
      "activations/layer23_attention_weight_min": -19.0639591217041,
      "activations/layer2_attention_weight_max": 24.936338424682617,
      "activations/layer2_attention_weight_min": -23.703336715698242,
      "activations/layer3_attention_weight_max": 68.70723724365234,
      "activations/layer3_attention_weight_min": -72.23084259033203,
      "activations/layer4_attention_weight_max": 88.54580688476562,
      "activations/layer4_attention_weight_min": -85.26038360595703,
      "activations/layer5_attention_weight_max": 94.35139465332031,
      "activations/layer5_attention_weight_min": -85.89678955078125,
      "activations/layer6_attention_weight_max": 54.098350524902344,
      "activations/layer6_attention_weight_min": -47.75642776489258,
      "activations/layer7_attention_weight_max": 73.07079315185547,
      "activations/layer7_attention_weight_min": -64.52405548095703,
      "activations/layer8_attention_weight_max": 37.28557205200195,
      "activations/layer8_attention_weight_min": -39.702518463134766,
      "activations/layer9_attention_weight_max": 50.837608337402344,
      "activations/layer9_attention_weight_min": -43.997554779052734,
      "epoch": 5.09,
      "learning_rate": 0.00011836742424242423,
      "loss": 2.8518,
      "step": 87600
    },
    {
      "activations/layer0_attention_weight_max": 14.810393333435059,
      "activations/layer0_attention_weight_min": -14.418699264526367,
      "activations/layer10_attention_weight_max": 41.876495361328125,
      "activations/layer10_attention_weight_min": -33.285369873046875,
      "activations/layer11_attention_weight_max": 34.31309509277344,
      "activations/layer11_attention_weight_min": -26.458499908447266,
      "activations/layer12_attention_weight_max": 27.023239135742188,
      "activations/layer12_attention_weight_min": -23.715173721313477,
      "activations/layer13_attention_weight_max": 57.02058410644531,
      "activations/layer13_attention_weight_min": -36.25877380371094,
      "activations/layer14_attention_weight_max": 42.59263229370117,
      "activations/layer14_attention_weight_min": -29.99482536315918,
      "activations/layer15_attention_weight_max": 43.97406005859375,
      "activations/layer15_attention_weight_min": -30.397720336914062,
      "activations/layer16_attention_weight_max": 42.357940673828125,
      "activations/layer16_attention_weight_min": -26.671794891357422,
      "activations/layer17_attention_weight_max": 62.69336700439453,
      "activations/layer17_attention_weight_min": -37.95610427856445,
      "activations/layer18_attention_weight_max": 53.23981475830078,
      "activations/layer18_attention_weight_min": -32.290443420410156,
      "activations/layer19_attention_weight_max": 22.892635345458984,
      "activations/layer19_attention_weight_min": -18.561594009399414,
      "activations/layer1_attention_weight_max": 17.10826301574707,
      "activations/layer1_attention_weight_min": -13.739663124084473,
      "activations/layer20_attention_weight_max": 22.649402618408203,
      "activations/layer20_attention_weight_min": -22.98518180847168,
      "activations/layer21_attention_weight_max": 48.250240325927734,
      "activations/layer21_attention_weight_min": -25.569774627685547,
      "activations/layer22_attention_weight_max": 38.165672302246094,
      "activations/layer22_attention_weight_min": -25.88667106628418,
      "activations/layer23_attention_weight_max": 28.797386169433594,
      "activations/layer23_attention_weight_min": -17.288429260253906,
      "activations/layer2_attention_weight_max": 31.39220428466797,
      "activations/layer2_attention_weight_min": -29.695297241210938,
      "activations/layer3_attention_weight_max": 75.32073974609375,
      "activations/layer3_attention_weight_min": -78.48836517333984,
      "activations/layer4_attention_weight_max": 100.25985717773438,
      "activations/layer4_attention_weight_min": -93.00457000732422,
      "activations/layer5_attention_weight_max": 101.24916076660156,
      "activations/layer5_attention_weight_min": -88.95121765136719,
      "activations/layer6_attention_weight_max": 57.07646560668945,
      "activations/layer6_attention_weight_min": -52.00950622558594,
      "activations/layer7_attention_weight_max": 73.86180114746094,
      "activations/layer7_attention_weight_min": -68.1353530883789,
      "activations/layer8_attention_weight_max": 42.28876876831055,
      "activations/layer8_attention_weight_min": -41.73271560668945,
      "activations/layer9_attention_weight_max": 53.28939437866211,
      "activations/layer9_attention_weight_min": -48.177364349365234,
      "epoch": 5.09,
      "learning_rate": 0.00011834848484848485,
      "loss": 2.8737,
      "step": 87650
    },
    {
      "activations/layer0_attention_weight_max": 14.424666404724121,
      "activations/layer0_attention_weight_min": -14.43708610534668,
      "activations/layer10_attention_weight_max": 39.19684600830078,
      "activations/layer10_attention_weight_min": -33.14454650878906,
      "activations/layer11_attention_weight_max": 28.622772216796875,
      "activations/layer11_attention_weight_min": -26.318159103393555,
      "activations/layer12_attention_weight_max": 22.83250617980957,
      "activations/layer12_attention_weight_min": -23.340328216552734,
      "activations/layer13_attention_weight_max": 47.79929733276367,
      "activations/layer13_attention_weight_min": -31.034038543701172,
      "activations/layer14_attention_weight_max": 57.26589584350586,
      "activations/layer14_attention_weight_min": -30.79304313659668,
      "activations/layer15_attention_weight_max": 39.08491897583008,
      "activations/layer15_attention_weight_min": -28.715869903564453,
      "activations/layer16_attention_weight_max": 37.19701385498047,
      "activations/layer16_attention_weight_min": -26.15861701965332,
      "activations/layer17_attention_weight_max": 59.99617004394531,
      "activations/layer17_attention_weight_min": -38.7619743347168,
      "activations/layer18_attention_weight_max": 52.515167236328125,
      "activations/layer18_attention_weight_min": -33.47236633300781,
      "activations/layer19_attention_weight_max": 23.001323699951172,
      "activations/layer19_attention_weight_min": -20.987092971801758,
      "activations/layer1_attention_weight_max": 16.39375877380371,
      "activations/layer1_attention_weight_min": -13.519847869873047,
      "activations/layer20_attention_weight_max": 24.143390655517578,
      "activations/layer20_attention_weight_min": -21.80070686340332,
      "activations/layer21_attention_weight_max": 51.833396911621094,
      "activations/layer21_attention_weight_min": -24.820648193359375,
      "activations/layer22_attention_weight_max": 33.79091262817383,
      "activations/layer22_attention_weight_min": -23.60350799560547,
      "activations/layer23_attention_weight_max": 29.337017059326172,
      "activations/layer23_attention_weight_min": -18.96250343322754,
      "activations/layer2_attention_weight_max": 30.16128158569336,
      "activations/layer2_attention_weight_min": -28.603422164916992,
      "activations/layer3_attention_weight_max": 77.93067932128906,
      "activations/layer3_attention_weight_min": -84.24214172363281,
      "activations/layer4_attention_weight_max": 98.0107650756836,
      "activations/layer4_attention_weight_min": -96.62373352050781,
      "activations/layer5_attention_weight_max": 102.63162231445312,
      "activations/layer5_attention_weight_min": -93.40911102294922,
      "activations/layer6_attention_weight_max": 59.53702163696289,
      "activations/layer6_attention_weight_min": -51.41931915283203,
      "activations/layer7_attention_weight_max": 68.4815902709961,
      "activations/layer7_attention_weight_min": -68.88459014892578,
      "activations/layer8_attention_weight_max": 42.889739990234375,
      "activations/layer8_attention_weight_min": -39.41033935546875,
      "activations/layer9_attention_weight_max": 49.32366943359375,
      "activations/layer9_attention_weight_min": -47.383140563964844,
      "epoch": 5.1,
      "learning_rate": 0.00011832954545454543,
      "loss": 2.8472,
      "step": 87700
    },
    {
      "activations/layer0_attention_weight_max": 14.878422737121582,
      "activations/layer0_attention_weight_min": -13.352584838867188,
      "activations/layer10_attention_weight_max": 38.76561737060547,
      "activations/layer10_attention_weight_min": -33.006591796875,
      "activations/layer11_attention_weight_max": 32.34300994873047,
      "activations/layer11_attention_weight_min": -28.39698600769043,
      "activations/layer12_attention_weight_max": 23.500511169433594,
      "activations/layer12_attention_weight_min": -23.20452308654785,
      "activations/layer13_attention_weight_max": 51.73760223388672,
      "activations/layer13_attention_weight_min": -35.75739288330078,
      "activations/layer14_attention_weight_max": 50.96029281616211,
      "activations/layer14_attention_weight_min": -30.935874938964844,
      "activations/layer15_attention_weight_max": 44.767486572265625,
      "activations/layer15_attention_weight_min": -31.794034957885742,
      "activations/layer16_attention_weight_max": 41.857418060302734,
      "activations/layer16_attention_weight_min": -24.67693519592285,
      "activations/layer17_attention_weight_max": 62.21745681762695,
      "activations/layer17_attention_weight_min": -38.66495132446289,
      "activations/layer18_attention_weight_max": 52.59623718261719,
      "activations/layer18_attention_weight_min": -31.98604965209961,
      "activations/layer19_attention_weight_max": 26.477218627929688,
      "activations/layer19_attention_weight_min": -19.623537063598633,
      "activations/layer1_attention_weight_max": 16.097196578979492,
      "activations/layer1_attention_weight_min": -13.622152328491211,
      "activations/layer20_attention_weight_max": 24.651426315307617,
      "activations/layer20_attention_weight_min": -23.6535701751709,
      "activations/layer21_attention_weight_max": 45.564537048339844,
      "activations/layer21_attention_weight_min": -27.75302505493164,
      "activations/layer22_attention_weight_max": 37.04608154296875,
      "activations/layer22_attention_weight_min": -24.56173324584961,
      "activations/layer23_attention_weight_max": 32.865943908691406,
      "activations/layer23_attention_weight_min": -18.864927291870117,
      "activations/layer2_attention_weight_max": 31.719480514526367,
      "activations/layer2_attention_weight_min": -28.230281829833984,
      "activations/layer3_attention_weight_max": 81.0029525756836,
      "activations/layer3_attention_weight_min": -84.36222076416016,
      "activations/layer4_attention_weight_max": 97.32462310791016,
      "activations/layer4_attention_weight_min": -91.23542785644531,
      "activations/layer5_attention_weight_max": 103.5548324584961,
      "activations/layer5_attention_weight_min": -92.43346405029297,
      "activations/layer6_attention_weight_max": 57.51692199707031,
      "activations/layer6_attention_weight_min": -51.93431091308594,
      "activations/layer7_attention_weight_max": 66.64253997802734,
      "activations/layer7_attention_weight_min": -67.07670593261719,
      "activations/layer8_attention_weight_max": 39.5173225402832,
      "activations/layer8_attention_weight_min": -39.59121322631836,
      "activations/layer9_attention_weight_max": 49.407840728759766,
      "activations/layer9_attention_weight_min": -47.06037521362305,
      "epoch": 5.1,
      "learning_rate": 0.00011831060606060605,
      "loss": 2.8592,
      "step": 87750
    },
    {
      "activations/layer0_attention_weight_max": 14.404311180114746,
      "activations/layer0_attention_weight_min": -13.584676742553711,
      "activations/layer10_attention_weight_max": 36.64474868774414,
      "activations/layer10_attention_weight_min": -32.60938262939453,
      "activations/layer11_attention_weight_max": 28.406047821044922,
      "activations/layer11_attention_weight_min": -27.051849365234375,
      "activations/layer12_attention_weight_max": 37.48358917236328,
      "activations/layer12_attention_weight_min": -24.71784019470215,
      "activations/layer13_attention_weight_max": 40.64298629760742,
      "activations/layer13_attention_weight_min": -31.531951904296875,
      "activations/layer14_attention_weight_max": 35.56379318237305,
      "activations/layer14_attention_weight_min": -28.59779167175293,
      "activations/layer15_attention_weight_max": 37.026695251464844,
      "activations/layer15_attention_weight_min": -28.569520950317383,
      "activations/layer16_attention_weight_max": 37.169639587402344,
      "activations/layer16_attention_weight_min": -24.89903450012207,
      "activations/layer17_attention_weight_max": 53.68831253051758,
      "activations/layer17_attention_weight_min": -37.925254821777344,
      "activations/layer18_attention_weight_max": 48.976341247558594,
      "activations/layer18_attention_weight_min": -32.77654266357422,
      "activations/layer19_attention_weight_max": 22.54193115234375,
      "activations/layer19_attention_weight_min": -18.92532730102539,
      "activations/layer1_attention_weight_max": 17.46994400024414,
      "activations/layer1_attention_weight_min": -16.447101593017578,
      "activations/layer20_attention_weight_max": 20.57035255432129,
      "activations/layer20_attention_weight_min": -22.518444061279297,
      "activations/layer21_attention_weight_max": 38.4864501953125,
      "activations/layer21_attention_weight_min": -25.147050857543945,
      "activations/layer22_attention_weight_max": 36.25747299194336,
      "activations/layer22_attention_weight_min": -22.348447799682617,
      "activations/layer23_attention_weight_max": 28.406333923339844,
      "activations/layer23_attention_weight_min": -18.307052612304688,
      "activations/layer2_attention_weight_max": 29.335866928100586,
      "activations/layer2_attention_weight_min": -27.75687026977539,
      "activations/layer3_attention_weight_max": 74.51290893554688,
      "activations/layer3_attention_weight_min": -79.70326232910156,
      "activations/layer4_attention_weight_max": 90.35676574707031,
      "activations/layer4_attention_weight_min": -93.32310485839844,
      "activations/layer5_attention_weight_max": 94.849365234375,
      "activations/layer5_attention_weight_min": -89.88334655761719,
      "activations/layer6_attention_weight_max": 57.797080993652344,
      "activations/layer6_attention_weight_min": -52.441322326660156,
      "activations/layer7_attention_weight_max": 66.13416290283203,
      "activations/layer7_attention_weight_min": -65.91891479492188,
      "activations/layer8_attention_weight_max": 41.06556701660156,
      "activations/layer8_attention_weight_min": -40.42202377319336,
      "activations/layer9_attention_weight_max": 44.84637451171875,
      "activations/layer9_attention_weight_min": -44.783592224121094,
      "epoch": 5.1,
      "learning_rate": 0.00011829166666666666,
      "loss": 2.8492,
      "step": 87800
    },
    {
      "activations/layer0_attention_weight_max": 14.590963363647461,
      "activations/layer0_attention_weight_min": -13.771496772766113,
      "activations/layer10_attention_weight_max": 39.555023193359375,
      "activations/layer10_attention_weight_min": -34.27710723876953,
      "activations/layer11_attention_weight_max": 34.8636589050293,
      "activations/layer11_attention_weight_min": -29.113208770751953,
      "activations/layer12_attention_weight_max": 21.723297119140625,
      "activations/layer12_attention_weight_min": -31.708171844482422,
      "activations/layer13_attention_weight_max": 48.251609802246094,
      "activations/layer13_attention_weight_min": -35.03603744506836,
      "activations/layer14_attention_weight_max": 49.87142562866211,
      "activations/layer14_attention_weight_min": -37.63052749633789,
      "activations/layer15_attention_weight_max": 42.48289108276367,
      "activations/layer15_attention_weight_min": -30.983001708984375,
      "activations/layer16_attention_weight_max": 31.111316680908203,
      "activations/layer16_attention_weight_min": -26.332727432250977,
      "activations/layer17_attention_weight_max": 57.07115936279297,
      "activations/layer17_attention_weight_min": -36.6451530456543,
      "activations/layer18_attention_weight_max": 45.34140396118164,
      "activations/layer18_attention_weight_min": -31.58648109436035,
      "activations/layer19_attention_weight_max": 19.254804611206055,
      "activations/layer19_attention_weight_min": -20.020727157592773,
      "activations/layer1_attention_weight_max": 16.824853897094727,
      "activations/layer1_attention_weight_min": -16.071439743041992,
      "activations/layer20_attention_weight_max": 20.879480361938477,
      "activations/layer20_attention_weight_min": -20.934614181518555,
      "activations/layer21_attention_weight_max": 33.67461013793945,
      "activations/layer21_attention_weight_min": -26.82842254638672,
      "activations/layer22_attention_weight_max": 33.46318817138672,
      "activations/layer22_attention_weight_min": -24.60042381286621,
      "activations/layer23_attention_weight_max": 25.948856353759766,
      "activations/layer23_attention_weight_min": -16.90094566345215,
      "activations/layer2_attention_weight_max": 31.148090362548828,
      "activations/layer2_attention_weight_min": -29.960838317871094,
      "activations/layer3_attention_weight_max": 79.07102966308594,
      "activations/layer3_attention_weight_min": -82.37682342529297,
      "activations/layer4_attention_weight_max": 98.85189056396484,
      "activations/layer4_attention_weight_min": -93.85558319091797,
      "activations/layer5_attention_weight_max": 100.31685638427734,
      "activations/layer5_attention_weight_min": -91.60161590576172,
      "activations/layer6_attention_weight_max": 57.50732421875,
      "activations/layer6_attention_weight_min": -51.64133834838867,
      "activations/layer7_attention_weight_max": 83.92436218261719,
      "activations/layer7_attention_weight_min": -68.99503326416016,
      "activations/layer8_attention_weight_max": 50.2446174621582,
      "activations/layer8_attention_weight_min": -40.001548767089844,
      "activations/layer9_attention_weight_max": 73.68145751953125,
      "activations/layer9_attention_weight_min": -59.83526611328125,
      "epoch": 5.1,
      "learning_rate": 0.00011827272727272725,
      "loss": 2.8594,
      "step": 87850
    },
    {
      "activations/layer0_attention_weight_max": 14.215121269226074,
      "activations/layer0_attention_weight_min": -14.231858253479004,
      "activations/layer10_attention_weight_max": 40.301673889160156,
      "activations/layer10_attention_weight_min": -31.3040828704834,
      "activations/layer11_attention_weight_max": 30.4688777923584,
      "activations/layer11_attention_weight_min": -27.036712646484375,
      "activations/layer12_attention_weight_max": 26.733835220336914,
      "activations/layer12_attention_weight_min": -24.27491569519043,
      "activations/layer13_attention_weight_max": 44.90296936035156,
      "activations/layer13_attention_weight_min": -31.25895118713379,
      "activations/layer14_attention_weight_max": 50.60672378540039,
      "activations/layer14_attention_weight_min": -31.188234329223633,
      "activations/layer15_attention_weight_max": 39.77843475341797,
      "activations/layer15_attention_weight_min": -28.569602966308594,
      "activations/layer16_attention_weight_max": 35.49205780029297,
      "activations/layer16_attention_weight_min": -25.598421096801758,
      "activations/layer17_attention_weight_max": 67.25305938720703,
      "activations/layer17_attention_weight_min": -37.40431594848633,
      "activations/layer18_attention_weight_max": 51.13386154174805,
      "activations/layer18_attention_weight_min": -32.08730697631836,
      "activations/layer19_attention_weight_max": 22.49571990966797,
      "activations/layer19_attention_weight_min": -20.696731567382812,
      "activations/layer1_attention_weight_max": 16.90236473083496,
      "activations/layer1_attention_weight_min": -16.184144973754883,
      "activations/layer20_attention_weight_max": 21.915191650390625,
      "activations/layer20_attention_weight_min": -20.418107986450195,
      "activations/layer21_attention_weight_max": 42.0070686340332,
      "activations/layer21_attention_weight_min": -26.465667724609375,
      "activations/layer22_attention_weight_max": 35.98038864135742,
      "activations/layer22_attention_weight_min": -24.449317932128906,
      "activations/layer23_attention_weight_max": 28.305686950683594,
      "activations/layer23_attention_weight_min": -19.3663272857666,
      "activations/layer2_attention_weight_max": 31.92654800415039,
      "activations/layer2_attention_weight_min": -29.290733337402344,
      "activations/layer3_attention_weight_max": 81.1192398071289,
      "activations/layer3_attention_weight_min": -80.94400024414062,
      "activations/layer4_attention_weight_max": 94.30113983154297,
      "activations/layer4_attention_weight_min": -91.2265625,
      "activations/layer5_attention_weight_max": 95.4432373046875,
      "activations/layer5_attention_weight_min": -90.74539184570312,
      "activations/layer6_attention_weight_max": 55.172916412353516,
      "activations/layer6_attention_weight_min": -51.610755920410156,
      "activations/layer7_attention_weight_max": 68.62786865234375,
      "activations/layer7_attention_weight_min": -68.45999908447266,
      "activations/layer8_attention_weight_max": 41.56442642211914,
      "activations/layer8_attention_weight_min": -41.74013137817383,
      "activations/layer9_attention_weight_max": 53.37935256958008,
      "activations/layer9_attention_weight_min": -43.90320587158203,
      "epoch": 5.11,
      "learning_rate": 0.00011825378787878787,
      "loss": 2.8475,
      "step": 87900
    },
    {
      "activations/layer0_attention_weight_max": 13.783167839050293,
      "activations/layer0_attention_weight_min": -14.168877601623535,
      "activations/layer10_attention_weight_max": 38.33008575439453,
      "activations/layer10_attention_weight_min": -35.53923797607422,
      "activations/layer11_attention_weight_max": 28.75826072692871,
      "activations/layer11_attention_weight_min": -27.410856246948242,
      "activations/layer12_attention_weight_max": 20.778982162475586,
      "activations/layer12_attention_weight_min": -28.517004013061523,
      "activations/layer13_attention_weight_max": 50.0003547668457,
      "activations/layer13_attention_weight_min": -35.876869201660156,
      "activations/layer14_attention_weight_max": 41.31033706665039,
      "activations/layer14_attention_weight_min": -31.413265228271484,
      "activations/layer15_attention_weight_max": 36.81327438354492,
      "activations/layer15_attention_weight_min": -29.757028579711914,
      "activations/layer16_attention_weight_max": 35.41769790649414,
      "activations/layer16_attention_weight_min": -25.267620086669922,
      "activations/layer17_attention_weight_max": 55.497005462646484,
      "activations/layer17_attention_weight_min": -37.71744918823242,
      "activations/layer18_attention_weight_max": 47.56529998779297,
      "activations/layer18_attention_weight_min": -32.90336608886719,
      "activations/layer19_attention_weight_max": 25.993488311767578,
      "activations/layer19_attention_weight_min": -19.80385971069336,
      "activations/layer1_attention_weight_max": 17.639142990112305,
      "activations/layer1_attention_weight_min": -14.18612289428711,
      "activations/layer20_attention_weight_max": 21.10129165649414,
      "activations/layer20_attention_weight_min": -19.33864974975586,
      "activations/layer21_attention_weight_max": 37.3751335144043,
      "activations/layer21_attention_weight_min": -26.83498764038086,
      "activations/layer22_attention_weight_max": 35.36204147338867,
      "activations/layer22_attention_weight_min": -25.101531982421875,
      "activations/layer23_attention_weight_max": 29.363746643066406,
      "activations/layer23_attention_weight_min": -21.021257400512695,
      "activations/layer2_attention_weight_max": 30.11254119873047,
      "activations/layer2_attention_weight_min": -28.399755477905273,
      "activations/layer3_attention_weight_max": 75.0323715209961,
      "activations/layer3_attention_weight_min": -79.636962890625,
      "activations/layer4_attention_weight_max": 94.93180847167969,
      "activations/layer4_attention_weight_min": -88.44894409179688,
      "activations/layer5_attention_weight_max": 98.16754150390625,
      "activations/layer5_attention_weight_min": -87.30775451660156,
      "activations/layer6_attention_weight_max": 56.08549499511719,
      "activations/layer6_attention_weight_min": -51.21174621582031,
      "activations/layer7_attention_weight_max": 67.21818542480469,
      "activations/layer7_attention_weight_min": -65.27656555175781,
      "activations/layer8_attention_weight_max": 41.93046188354492,
      "activations/layer8_attention_weight_min": -41.30034255981445,
      "activations/layer9_attention_weight_max": 56.718017578125,
      "activations/layer9_attention_weight_min": -47.40291976928711,
      "epoch": 5.11,
      "learning_rate": 0.00011823484848484848,
      "loss": 2.8527,
      "step": 87950
    },
    {
      "activations/layer0_attention_weight_max": 17.496641159057617,
      "activations/layer0_attention_weight_min": -14.030949592590332,
      "activations/layer10_attention_weight_max": 38.453189849853516,
      "activations/layer10_attention_weight_min": -32.82362747192383,
      "activations/layer11_attention_weight_max": 29.135723114013672,
      "activations/layer11_attention_weight_min": -25.989444732666016,
      "activations/layer12_attention_weight_max": 25.516098022460938,
      "activations/layer12_attention_weight_min": -27.218631744384766,
      "activations/layer13_attention_weight_max": 49.25884246826172,
      "activations/layer13_attention_weight_min": -37.52553939819336,
      "activations/layer14_attention_weight_max": 52.77260208129883,
      "activations/layer14_attention_weight_min": -37.805931091308594,
      "activations/layer15_attention_weight_max": 42.81233596801758,
      "activations/layer15_attention_weight_min": -28.98036003112793,
      "activations/layer16_attention_weight_max": 34.614402770996094,
      "activations/layer16_attention_weight_min": -26.657991409301758,
      "activations/layer17_attention_weight_max": 57.5067138671875,
      "activations/layer17_attention_weight_min": -40.0335578918457,
      "activations/layer18_attention_weight_max": 51.094669342041016,
      "activations/layer18_attention_weight_min": -34.52601623535156,
      "activations/layer19_attention_weight_max": 25.870813369750977,
      "activations/layer19_attention_weight_min": -19.83915138244629,
      "activations/layer1_attention_weight_max": 16.994373321533203,
      "activations/layer1_attention_weight_min": -15.008536338806152,
      "activations/layer20_attention_weight_max": 23.210044860839844,
      "activations/layer20_attention_weight_min": -24.016071319580078,
      "activations/layer21_attention_weight_max": 43.54880905151367,
      "activations/layer21_attention_weight_min": -27.636741638183594,
      "activations/layer22_attention_weight_max": 36.356929779052734,
      "activations/layer22_attention_weight_min": -23.45203971862793,
      "activations/layer23_attention_weight_max": 27.860198974609375,
      "activations/layer23_attention_weight_min": -17.033308029174805,
      "activations/layer2_attention_weight_max": 29.32671356201172,
      "activations/layer2_attention_weight_min": -27.485387802124023,
      "activations/layer3_attention_weight_max": 75.75202178955078,
      "activations/layer3_attention_weight_min": -82.92670440673828,
      "activations/layer4_attention_weight_max": 99.51342010498047,
      "activations/layer4_attention_weight_min": -91.94548797607422,
      "activations/layer5_attention_weight_max": 103.55047607421875,
      "activations/layer5_attention_weight_min": -94.11566162109375,
      "activations/layer6_attention_weight_max": 58.24305725097656,
      "activations/layer6_attention_weight_min": -53.41459274291992,
      "activations/layer7_attention_weight_max": 71.71170806884766,
      "activations/layer7_attention_weight_min": -69.72313690185547,
      "activations/layer8_attention_weight_max": 42.2249755859375,
      "activations/layer8_attention_weight_min": -42.34757614135742,
      "activations/layer9_attention_weight_max": 48.17039108276367,
      "activations/layer9_attention_weight_min": -44.24860382080078,
      "epoch": 5.11,
      "learning_rate": 0.00011821590909090908,
      "loss": 2.8483,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.4909,
      "eval_samples_per_second": 505.717,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.4909,
      "eval_openwebtext_samples_per_second": 505.717,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_wikitext_loss": 3.0234375,
      "eval_wikitext_ppl": 20.56185170491249,
      "eval_wikitext_runtime": 2.038,
      "eval_wikitext_samples_per_second": 223.75,
      "step": 88000
    },
    {
      "epoch": 5.11,
      "eval_lambada_loss": 2.841796875,
      "eval_lambada_ppl": 17.14654807625046,
      "eval_lambada_runtime": 9.6369,
      "eval_lambada_samples_per_second": 505.244,
      "step": 88000
    },
    {
      "activations/layer0_attention_weight_max": 14.56424617767334,
      "activations/layer0_attention_weight_min": -14.149152755737305,
      "activations/layer10_attention_weight_max": 43.278167724609375,
      "activations/layer10_attention_weight_min": -33.12982177734375,
      "activations/layer11_attention_weight_max": 33.702239990234375,
      "activations/layer11_attention_weight_min": -27.92593002319336,
      "activations/layer12_attention_weight_max": 28.630630493164062,
      "activations/layer12_attention_weight_min": -23.738712310791016,
      "activations/layer13_attention_weight_max": 61.557884216308594,
      "activations/layer13_attention_weight_min": -37.10603713989258,
      "activations/layer14_attention_weight_max": 72.64189147949219,
      "activations/layer14_attention_weight_min": -33.21281433105469,
      "activations/layer15_attention_weight_max": 55.18971252441406,
      "activations/layer15_attention_weight_min": -29.143020629882812,
      "activations/layer16_attention_weight_max": 35.996986389160156,
      "activations/layer16_attention_weight_min": -24.468589782714844,
      "activations/layer17_attention_weight_max": 59.0196533203125,
      "activations/layer17_attention_weight_min": -37.83424758911133,
      "activations/layer18_attention_weight_max": 49.00149917602539,
      "activations/layer18_attention_weight_min": -32.69374465942383,
      "activations/layer19_attention_weight_max": 26.235538482666016,
      "activations/layer19_attention_weight_min": -19.731714248657227,
      "activations/layer1_attention_weight_max": 17.200185775756836,
      "activations/layer1_attention_weight_min": -14.978707313537598,
      "activations/layer20_attention_weight_max": 26.03466796875,
      "activations/layer20_attention_weight_min": -18.849925994873047,
      "activations/layer21_attention_weight_max": 53.12709426879883,
      "activations/layer21_attention_weight_min": -24.81999397277832,
      "activations/layer22_attention_weight_max": 37.281455993652344,
      "activations/layer22_attention_weight_min": -22.331302642822266,
      "activations/layer23_attention_weight_max": 33.607208251953125,
      "activations/layer23_attention_weight_min": -18.470077514648438,
      "activations/layer2_attention_weight_max": 30.512462615966797,
      "activations/layer2_attention_weight_min": -29.164030075073242,
      "activations/layer3_attention_weight_max": 78.02987670898438,
      "activations/layer3_attention_weight_min": -84.98291778564453,
      "activations/layer4_attention_weight_max": 98.019775390625,
      "activations/layer4_attention_weight_min": -94.747802734375,
      "activations/layer5_attention_weight_max": 99.3955307006836,
      "activations/layer5_attention_weight_min": -89.07040405273438,
      "activations/layer6_attention_weight_max": 55.969383239746094,
      "activations/layer6_attention_weight_min": -52.12667465209961,
      "activations/layer7_attention_weight_max": 68.51460266113281,
      "activations/layer7_attention_weight_min": -65.7447509765625,
      "activations/layer8_attention_weight_max": 41.75873565673828,
      "activations/layer8_attention_weight_min": -44.557350158691406,
      "activations/layer9_attention_weight_max": 52.41115951538086,
      "activations/layer9_attention_weight_min": -45.25706100463867,
      "epoch": 5.12,
      "learning_rate": 0.00011819696969696969,
      "loss": 2.8614,
      "step": 88050
    },
    {
      "activations/layer0_attention_weight_max": 14.480152130126953,
      "activations/layer0_attention_weight_min": -13.970494270324707,
      "activations/layer10_attention_weight_max": 39.39474868774414,
      "activations/layer10_attention_weight_min": -35.20698928833008,
      "activations/layer11_attention_weight_max": 33.709659576416016,
      "activations/layer11_attention_weight_min": -28.057401657104492,
      "activations/layer12_attention_weight_max": 23.13733673095703,
      "activations/layer12_attention_weight_min": -26.465408325195312,
      "activations/layer13_attention_weight_max": 45.473716735839844,
      "activations/layer13_attention_weight_min": -34.02338790893555,
      "activations/layer14_attention_weight_max": 45.055240631103516,
      "activations/layer14_attention_weight_min": -28.460233688354492,
      "activations/layer15_attention_weight_max": 34.681129455566406,
      "activations/layer15_attention_weight_min": -32.501609802246094,
      "activations/layer16_attention_weight_max": 33.313968658447266,
      "activations/layer16_attention_weight_min": -25.558168411254883,
      "activations/layer17_attention_weight_max": 51.353328704833984,
      "activations/layer17_attention_weight_min": -37.337158203125,
      "activations/layer18_attention_weight_max": 41.50697326660156,
      "activations/layer18_attention_weight_min": -31.348163604736328,
      "activations/layer19_attention_weight_max": 19.64059066772461,
      "activations/layer19_attention_weight_min": -21.55121421813965,
      "activations/layer1_attention_weight_max": 16.855846405029297,
      "activations/layer1_attention_weight_min": -16.388010025024414,
      "activations/layer20_attention_weight_max": 20.59190559387207,
      "activations/layer20_attention_weight_min": -24.030805587768555,
      "activations/layer21_attention_weight_max": 35.80432891845703,
      "activations/layer21_attention_weight_min": -24.914751052856445,
      "activations/layer22_attention_weight_max": 29.860172271728516,
      "activations/layer22_attention_weight_min": -22.54861068725586,
      "activations/layer23_attention_weight_max": 24.32960319519043,
      "activations/layer23_attention_weight_min": -17.368804931640625,
      "activations/layer2_attention_weight_max": 33.54768753051758,
      "activations/layer2_attention_weight_min": -30.936119079589844,
      "activations/layer3_attention_weight_max": 85.47945404052734,
      "activations/layer3_attention_weight_min": -87.48326110839844,
      "activations/layer4_attention_weight_max": 102.28324127197266,
      "activations/layer4_attention_weight_min": -93.61676025390625,
      "activations/layer5_attention_weight_max": 102.14082336425781,
      "activations/layer5_attention_weight_min": -88.24810791015625,
      "activations/layer6_attention_weight_max": 56.2155876159668,
      "activations/layer6_attention_weight_min": -49.78470993041992,
      "activations/layer7_attention_weight_max": 83.32502746582031,
      "activations/layer7_attention_weight_min": -64.44474792480469,
      "activations/layer8_attention_weight_max": 48.37715148925781,
      "activations/layer8_attention_weight_min": -40.46326446533203,
      "activations/layer9_attention_weight_max": 67.2624282836914,
      "activations/layer9_attention_weight_min": -49.15580749511719,
      "epoch": 5.12,
      "learning_rate": 0.00011817803030303029,
      "loss": 2.8238,
      "step": 88100
    },
    {
      "activations/layer0_attention_weight_max": 15.74222469329834,
      "activations/layer0_attention_weight_min": -14.793522834777832,
      "activations/layer10_attention_weight_max": 39.62535858154297,
      "activations/layer10_attention_weight_min": -37.066009521484375,
      "activations/layer11_attention_weight_max": 33.169342041015625,
      "activations/layer11_attention_weight_min": -29.694366455078125,
      "activations/layer12_attention_weight_max": 29.23322296142578,
      "activations/layer12_attention_weight_min": -23.22262191772461,
      "activations/layer13_attention_weight_max": 54.920166015625,
      "activations/layer13_attention_weight_min": -37.06599426269531,
      "activations/layer14_attention_weight_max": 56.783634185791016,
      "activations/layer14_attention_weight_min": -39.97792434692383,
      "activations/layer15_attention_weight_max": 43.75385284423828,
      "activations/layer15_attention_weight_min": -30.784271240234375,
      "activations/layer16_attention_weight_max": 35.96933364868164,
      "activations/layer16_attention_weight_min": -26.96528434753418,
      "activations/layer17_attention_weight_max": 59.83863830566406,
      "activations/layer17_attention_weight_min": -37.6810188293457,
      "activations/layer18_attention_weight_max": 52.33464813232422,
      "activations/layer18_attention_weight_min": -32.02983093261719,
      "activations/layer19_attention_weight_max": 27.64955711364746,
      "activations/layer19_attention_weight_min": -19.137134552001953,
      "activations/layer1_attention_weight_max": 16.060245513916016,
      "activations/layer1_attention_weight_min": -13.447342872619629,
      "activations/layer20_attention_weight_max": 23.07801055908203,
      "activations/layer20_attention_weight_min": -20.36743927001953,
      "activations/layer21_attention_weight_max": 41.061912536621094,
      "activations/layer21_attention_weight_min": -24.911529541015625,
      "activations/layer22_attention_weight_max": 34.81083679199219,
      "activations/layer22_attention_weight_min": -23.136066436767578,
      "activations/layer23_attention_weight_max": 30.74246597290039,
      "activations/layer23_attention_weight_min": -18.968284606933594,
      "activations/layer2_attention_weight_max": 32.559974670410156,
      "activations/layer2_attention_weight_min": -30.244380950927734,
      "activations/layer3_attention_weight_max": 82.3313217163086,
      "activations/layer3_attention_weight_min": -83.74127960205078,
      "activations/layer4_attention_weight_max": 102.23670959472656,
      "activations/layer4_attention_weight_min": -96.16605377197266,
      "activations/layer5_attention_weight_max": 103.29037475585938,
      "activations/layer5_attention_weight_min": -98.75337982177734,
      "activations/layer6_attention_weight_max": 55.234832763671875,
      "activations/layer6_attention_weight_min": -54.49138641357422,
      "activations/layer7_attention_weight_max": 77.82848358154297,
      "activations/layer7_attention_weight_min": -70.74877166748047,
      "activations/layer8_attention_weight_max": 48.22145462036133,
      "activations/layer8_attention_weight_min": -43.14728546142578,
      "activations/layer9_attention_weight_max": 61.80741500854492,
      "activations/layer9_attention_weight_min": -55.679603576660156,
      "epoch": 5.12,
      "learning_rate": 0.0001181590909090909,
      "loss": 2.8485,
      "step": 88150
    },
    {
      "activations/layer0_attention_weight_max": 16.885162353515625,
      "activations/layer0_attention_weight_min": -14.169301986694336,
      "activations/layer10_attention_weight_max": 37.806785583496094,
      "activations/layer10_attention_weight_min": -33.0591926574707,
      "activations/layer11_attention_weight_max": 28.727493286132812,
      "activations/layer11_attention_weight_min": -26.022808074951172,
      "activations/layer12_attention_weight_max": 25.028648376464844,
      "activations/layer12_attention_weight_min": -25.587326049804688,
      "activations/layer13_attention_weight_max": 50.25682067871094,
      "activations/layer13_attention_weight_min": -35.13623809814453,
      "activations/layer14_attention_weight_max": 34.08089828491211,
      "activations/layer14_attention_weight_min": -28.56192398071289,
      "activations/layer15_attention_weight_max": 37.85746383666992,
      "activations/layer15_attention_weight_min": -28.82766342163086,
      "activations/layer16_attention_weight_max": 37.64012908935547,
      "activations/layer16_attention_weight_min": -25.477304458618164,
      "activations/layer17_attention_weight_max": 52.67243194580078,
      "activations/layer17_attention_weight_min": -37.17367172241211,
      "activations/layer18_attention_weight_max": 45.835575103759766,
      "activations/layer18_attention_weight_min": -32.01376724243164,
      "activations/layer19_attention_weight_max": 23.217702865600586,
      "activations/layer19_attention_weight_min": -20.55805206298828,
      "activations/layer1_attention_weight_max": 18.037399291992188,
      "activations/layer1_attention_weight_min": -15.626676559448242,
      "activations/layer20_attention_weight_max": 18.060832977294922,
      "activations/layer20_attention_weight_min": -21.426422119140625,
      "activations/layer21_attention_weight_max": 34.05074691772461,
      "activations/layer21_attention_weight_min": -24.424209594726562,
      "activations/layer22_attention_weight_max": 32.250301361083984,
      "activations/layer22_attention_weight_min": -22.851255416870117,
      "activations/layer23_attention_weight_max": 24.095077514648438,
      "activations/layer23_attention_weight_min": -17.409156799316406,
      "activations/layer2_attention_weight_max": 30.93207359313965,
      "activations/layer2_attention_weight_min": -29.423704147338867,
      "activations/layer3_attention_weight_max": 82.56053161621094,
      "activations/layer3_attention_weight_min": -83.69615173339844,
      "activations/layer4_attention_weight_max": 99.89813232421875,
      "activations/layer4_attention_weight_min": -92.63497161865234,
      "activations/layer5_attention_weight_max": 100.96669006347656,
      "activations/layer5_attention_weight_min": -87.5546875,
      "activations/layer6_attention_weight_max": 56.25099182128906,
      "activations/layer6_attention_weight_min": -54.25746536254883,
      "activations/layer7_attention_weight_max": 69.07032012939453,
      "activations/layer7_attention_weight_min": -68.21044921875,
      "activations/layer8_attention_weight_max": 43.65644454956055,
      "activations/layer8_attention_weight_min": -41.59907913208008,
      "activations/layer9_attention_weight_max": 46.87993240356445,
      "activations/layer9_attention_weight_min": -46.543094635009766,
      "epoch": 5.12,
      "learning_rate": 0.0001181401515151515,
      "loss": 2.8442,
      "step": 88200
    },
    {
      "activations/layer0_attention_weight_max": 14.18563461303711,
      "activations/layer0_attention_weight_min": -14.176292419433594,
      "activations/layer10_attention_weight_max": 38.52785110473633,
      "activations/layer10_attention_weight_min": -35.2685546875,
      "activations/layer11_attention_weight_max": 30.353544235229492,
      "activations/layer11_attention_weight_min": -28.570823669433594,
      "activations/layer12_attention_weight_max": 25.6118221282959,
      "activations/layer12_attention_weight_min": -24.028104782104492,
      "activations/layer13_attention_weight_max": 44.57236099243164,
      "activations/layer13_attention_weight_min": -37.76472854614258,
      "activations/layer14_attention_weight_max": 44.69646072387695,
      "activations/layer14_attention_weight_min": -30.0833683013916,
      "activations/layer15_attention_weight_max": 38.98631286621094,
      "activations/layer15_attention_weight_min": -30.94649314880371,
      "activations/layer16_attention_weight_max": 37.48487091064453,
      "activations/layer16_attention_weight_min": -27.278945922851562,
      "activations/layer17_attention_weight_max": 59.69441223144531,
      "activations/layer17_attention_weight_min": -39.90647888183594,
      "activations/layer18_attention_weight_max": 51.70673370361328,
      "activations/layer18_attention_weight_min": -35.643836975097656,
      "activations/layer19_attention_weight_max": 24.760740280151367,
      "activations/layer19_attention_weight_min": -21.763486862182617,
      "activations/layer1_attention_weight_max": 17.026416778564453,
      "activations/layer1_attention_weight_min": -14.497729301452637,
      "activations/layer20_attention_weight_max": 21.329023361206055,
      "activations/layer20_attention_weight_min": -23.392948150634766,
      "activations/layer21_attention_weight_max": 39.45817184448242,
      "activations/layer21_attention_weight_min": -27.948362350463867,
      "activations/layer22_attention_weight_max": 33.60778045654297,
      "activations/layer22_attention_weight_min": -24.1669921875,
      "activations/layer23_attention_weight_max": 29.046913146972656,
      "activations/layer23_attention_weight_min": -19.83590316772461,
      "activations/layer2_attention_weight_max": 31.56359100341797,
      "activations/layer2_attention_weight_min": -31.479358673095703,
      "activations/layer3_attention_weight_max": 81.36027526855469,
      "activations/layer3_attention_weight_min": -84.32076263427734,
      "activations/layer4_attention_weight_max": 98.75658416748047,
      "activations/layer4_attention_weight_min": -96.68292236328125,
      "activations/layer5_attention_weight_max": 101.14356994628906,
      "activations/layer5_attention_weight_min": -91.61677551269531,
      "activations/layer6_attention_weight_max": 56.437164306640625,
      "activations/layer6_attention_weight_min": -55.2528076171875,
      "activations/layer7_attention_weight_max": 68.7322006225586,
      "activations/layer7_attention_weight_min": -69.00605773925781,
      "activations/layer8_attention_weight_max": 41.8935432434082,
      "activations/layer8_attention_weight_min": -40.91331100463867,
      "activations/layer9_attention_weight_max": 52.21177291870117,
      "activations/layer9_attention_weight_min": -49.057437896728516,
      "epoch": 5.13,
      "learning_rate": 0.0001181212121212121,
      "loss": 2.8342,
      "step": 88250
    },
    {
      "activations/layer0_attention_weight_max": 14.887349128723145,
      "activations/layer0_attention_weight_min": -14.32478141784668,
      "activations/layer10_attention_weight_max": 41.41569900512695,
      "activations/layer10_attention_weight_min": -35.634727478027344,
      "activations/layer11_attention_weight_max": 32.262691497802734,
      "activations/layer11_attention_weight_min": -27.765058517456055,
      "activations/layer12_attention_weight_max": 22.771146774291992,
      "activations/layer12_attention_weight_min": -23.50238037109375,
      "activations/layer13_attention_weight_max": 53.72731399536133,
      "activations/layer13_attention_weight_min": -37.78884506225586,
      "activations/layer14_attention_weight_max": 42.47322463989258,
      "activations/layer14_attention_weight_min": -32.524871826171875,
      "activations/layer15_attention_weight_max": 39.964508056640625,
      "activations/layer15_attention_weight_min": -30.922948837280273,
      "activations/layer16_attention_weight_max": 36.38637161254883,
      "activations/layer16_attention_weight_min": -25.72760772705078,
      "activations/layer17_attention_weight_max": 56.05180740356445,
      "activations/layer17_attention_weight_min": -42.149940490722656,
      "activations/layer18_attention_weight_max": 49.52513885498047,
      "activations/layer18_attention_weight_min": -35.77214813232422,
      "activations/layer19_attention_weight_max": 20.91322898864746,
      "activations/layer19_attention_weight_min": -22.85358238220215,
      "activations/layer1_attention_weight_max": 17.53237533569336,
      "activations/layer1_attention_weight_min": -15.12746524810791,
      "activations/layer20_attention_weight_max": 27.000633239746094,
      "activations/layer20_attention_weight_min": -22.121356964111328,
      "activations/layer21_attention_weight_max": 40.87068557739258,
      "activations/layer21_attention_weight_min": -28.24755096435547,
      "activations/layer22_attention_weight_max": 39.21168518066406,
      "activations/layer22_attention_weight_min": -23.85143280029297,
      "activations/layer23_attention_weight_max": 28.834341049194336,
      "activations/layer23_attention_weight_min": -17.410953521728516,
      "activations/layer2_attention_weight_max": 30.752744674682617,
      "activations/layer2_attention_weight_min": -29.238754272460938,
      "activations/layer3_attention_weight_max": 83.08415222167969,
      "activations/layer3_attention_weight_min": -87.88936614990234,
      "activations/layer4_attention_weight_max": 102.77315521240234,
      "activations/layer4_attention_weight_min": -99.59798431396484,
      "activations/layer5_attention_weight_max": 102.43510437011719,
      "activations/layer5_attention_weight_min": -97.5999984741211,
      "activations/layer6_attention_weight_max": 57.389617919921875,
      "activations/layer6_attention_weight_min": -54.46660232543945,
      "activations/layer7_attention_weight_max": 68.78470611572266,
      "activations/layer7_attention_weight_min": -69.93032836914062,
      "activations/layer8_attention_weight_max": 44.82273483276367,
      "activations/layer8_attention_weight_min": -42.8055305480957,
      "activations/layer9_attention_weight_max": 50.9212532043457,
      "activations/layer9_attention_weight_min": -47.155059814453125,
      "epoch": 5.13,
      "learning_rate": 0.00011810227272727272,
      "loss": 2.8469,
      "step": 88300
    },
    {
      "activations/layer0_attention_weight_max": 15.5414457321167,
      "activations/layer0_attention_weight_min": -14.716642379760742,
      "activations/layer10_attention_weight_max": 48.28092956542969,
      "activations/layer10_attention_weight_min": -42.39632034301758,
      "activations/layer11_attention_weight_max": 38.540977478027344,
      "activations/layer11_attention_weight_min": -30.398075103759766,
      "activations/layer12_attention_weight_max": 25.997169494628906,
      "activations/layer12_attention_weight_min": -26.635290145874023,
      "activations/layer13_attention_weight_max": 59.07791519165039,
      "activations/layer13_attention_weight_min": -37.04133987426758,
      "activations/layer14_attention_weight_max": 98.3662338256836,
      "activations/layer14_attention_weight_min": -45.51167297363281,
      "activations/layer15_attention_weight_max": 72.7861328125,
      "activations/layer15_attention_weight_min": -39.32704544067383,
      "activations/layer16_attention_weight_max": 53.34157943725586,
      "activations/layer16_attention_weight_min": -26.430463790893555,
      "activations/layer17_attention_weight_max": 62.8801155090332,
      "activations/layer17_attention_weight_min": -44.97795486450195,
      "activations/layer18_attention_weight_max": 55.10061264038086,
      "activations/layer18_attention_weight_min": -40.14131164550781,
      "activations/layer19_attention_weight_max": 32.283782958984375,
      "activations/layer19_attention_weight_min": -25.2025203704834,
      "activations/layer1_attention_weight_max": 16.873796463012695,
      "activations/layer1_attention_weight_min": -17.271883010864258,
      "activations/layer20_attention_weight_max": 35.30997085571289,
      "activations/layer20_attention_weight_min": -26.929975509643555,
      "activations/layer21_attention_weight_max": 79.78911590576172,
      "activations/layer21_attention_weight_min": -34.8011589050293,
      "activations/layer22_attention_weight_max": 45.19913101196289,
      "activations/layer22_attention_weight_min": -28.047821044921875,
      "activations/layer23_attention_weight_max": 36.74712371826172,
      "activations/layer23_attention_weight_min": -18.49959373474121,
      "activations/layer2_attention_weight_max": 34.504920959472656,
      "activations/layer2_attention_weight_min": -31.658477783203125,
      "activations/layer3_attention_weight_max": 98.13360595703125,
      "activations/layer3_attention_weight_min": -99.41827392578125,
      "activations/layer4_attention_weight_max": 125.0104751586914,
      "activations/layer4_attention_weight_min": -114.35099029541016,
      "activations/layer5_attention_weight_max": 119.96388244628906,
      "activations/layer5_attention_weight_min": -104.45464324951172,
      "activations/layer6_attention_weight_max": 67.23046112060547,
      "activations/layer6_attention_weight_min": -58.53668212890625,
      "activations/layer7_attention_weight_max": 85.77943420410156,
      "activations/layer7_attention_weight_min": -73.91294860839844,
      "activations/layer8_attention_weight_max": 57.82209014892578,
      "activations/layer8_attention_weight_min": -51.3382682800293,
      "activations/layer9_attention_weight_max": 68.01162719726562,
      "activations/layer9_attention_weight_min": -60.60224151611328,
      "epoch": 5.13,
      "learning_rate": 0.00011808333333333333,
      "loss": 2.8201,
      "step": 88350
    },
    {
      "activations/layer0_attention_weight_max": 15.126112937927246,
      "activations/layer0_attention_weight_min": -14.947800636291504,
      "activations/layer10_attention_weight_max": 43.91392135620117,
      "activations/layer10_attention_weight_min": -37.766632080078125,
      "activations/layer11_attention_weight_max": 35.68135452270508,
      "activations/layer11_attention_weight_min": -28.474576950073242,
      "activations/layer12_attention_weight_max": 22.776655197143555,
      "activations/layer12_attention_weight_min": -27.488744735717773,
      "activations/layer13_attention_weight_max": 56.913238525390625,
      "activations/layer13_attention_weight_min": -41.236289978027344,
      "activations/layer14_attention_weight_max": 54.670597076416016,
      "activations/layer14_attention_weight_min": -37.75170135498047,
      "activations/layer15_attention_weight_max": 43.94679641723633,
      "activations/layer15_attention_weight_min": -31.924022674560547,
      "activations/layer16_attention_weight_max": 34.953102111816406,
      "activations/layer16_attention_weight_min": -27.490983963012695,
      "activations/layer17_attention_weight_max": 56.080039978027344,
      "activations/layer17_attention_weight_min": -42.25117874145508,
      "activations/layer18_attention_weight_max": 48.54951477050781,
      "activations/layer18_attention_weight_min": -34.85381317138672,
      "activations/layer19_attention_weight_max": 27.719079971313477,
      "activations/layer19_attention_weight_min": -18.242671966552734,
      "activations/layer1_attention_weight_max": 17.145469665527344,
      "activations/layer1_attention_weight_min": -17.197397232055664,
      "activations/layer20_attention_weight_max": 20.11928367614746,
      "activations/layer20_attention_weight_min": -22.470375061035156,
      "activations/layer21_attention_weight_max": 41.58748245239258,
      "activations/layer21_attention_weight_min": -25.235788345336914,
      "activations/layer22_attention_weight_max": 35.35824203491211,
      "activations/layer22_attention_weight_min": -23.58767318725586,
      "activations/layer23_attention_weight_max": 26.846708297729492,
      "activations/layer23_attention_weight_min": -16.74687957763672,
      "activations/layer2_attention_weight_max": 31.483022689819336,
      "activations/layer2_attention_weight_min": -29.826068878173828,
      "activations/layer3_attention_weight_max": 84.66161346435547,
      "activations/layer3_attention_weight_min": -87.38150787353516,
      "activations/layer4_attention_weight_max": 105.62371063232422,
      "activations/layer4_attention_weight_min": -97.81206512451172,
      "activations/layer5_attention_weight_max": 108.42741394042969,
      "activations/layer5_attention_weight_min": -96.85400390625,
      "activations/layer6_attention_weight_max": 61.39598083496094,
      "activations/layer6_attention_weight_min": -54.993812561035156,
      "activations/layer7_attention_weight_max": 89.60468292236328,
      "activations/layer7_attention_weight_min": -71.35328674316406,
      "activations/layer8_attention_weight_max": 53.413246154785156,
      "activations/layer8_attention_weight_min": -43.015628814697266,
      "activations/layer9_attention_weight_max": 75.09136199951172,
      "activations/layer9_attention_weight_min": -56.131446838378906,
      "epoch": 5.14,
      "learning_rate": 0.00011806439393939392,
      "loss": 2.8454,
      "step": 88400
    },
    {
      "activations/layer0_attention_weight_max": 13.383635520935059,
      "activations/layer0_attention_weight_min": -14.86846923828125,
      "activations/layer10_attention_weight_max": 37.267520904541016,
      "activations/layer10_attention_weight_min": -33.16041946411133,
      "activations/layer11_attention_weight_max": 29.940542221069336,
      "activations/layer11_attention_weight_min": -26.330612182617188,
      "activations/layer12_attention_weight_max": 30.52562713623047,
      "activations/layer12_attention_weight_min": -24.073347091674805,
      "activations/layer13_attention_weight_max": 56.388404846191406,
      "activations/layer13_attention_weight_min": -34.955814361572266,
      "activations/layer14_attention_weight_max": 54.35831832885742,
      "activations/layer14_attention_weight_min": -29.274953842163086,
      "activations/layer15_attention_weight_max": 38.16749954223633,
      "activations/layer15_attention_weight_min": -29.573087692260742,
      "activations/layer16_attention_weight_max": 34.01308059692383,
      "activations/layer16_attention_weight_min": -26.337352752685547,
      "activations/layer17_attention_weight_max": 57.36321258544922,
      "activations/layer17_attention_weight_min": -38.21135330200195,
      "activations/layer18_attention_weight_max": 45.817649841308594,
      "activations/layer18_attention_weight_min": -32.594200134277344,
      "activations/layer19_attention_weight_max": 24.473329544067383,
      "activations/layer19_attention_weight_min": -17.486120223999023,
      "activations/layer1_attention_weight_max": 17.477096557617188,
      "activations/layer1_attention_weight_min": -18.31493377685547,
      "activations/layer20_attention_weight_max": 20.034015655517578,
      "activations/layer20_attention_weight_min": -19.48917579650879,
      "activations/layer21_attention_weight_max": 35.070892333984375,
      "activations/layer21_attention_weight_min": -25.197677612304688,
      "activations/layer22_attention_weight_max": 32.986083984375,
      "activations/layer22_attention_weight_min": -23.71451759338379,
      "activations/layer23_attention_weight_max": 26.59243392944336,
      "activations/layer23_attention_weight_min": -16.85401725769043,
      "activations/layer2_attention_weight_max": 29.41536521911621,
      "activations/layer2_attention_weight_min": -28.622982025146484,
      "activations/layer3_attention_weight_max": 78.5033950805664,
      "activations/layer3_attention_weight_min": -81.58880615234375,
      "activations/layer4_attention_weight_max": 95.17049407958984,
      "activations/layer4_attention_weight_min": -92.03728485107422,
      "activations/layer5_attention_weight_max": 98.98197937011719,
      "activations/layer5_attention_weight_min": -86.64019775390625,
      "activations/layer6_attention_weight_max": 56.58157730102539,
      "activations/layer6_attention_weight_min": -50.30995559692383,
      "activations/layer7_attention_weight_max": 67.20149993896484,
      "activations/layer7_attention_weight_min": -68.27220153808594,
      "activations/layer8_attention_weight_max": 42.5768928527832,
      "activations/layer8_attention_weight_min": -42.0604133605957,
      "activations/layer9_attention_weight_max": 50.08763885498047,
      "activations/layer9_attention_weight_min": -47.71674728393555,
      "epoch": 5.14,
      "learning_rate": 0.00011804545454545454,
      "loss": 2.8319,
      "step": 88450
    },
    {
      "activations/layer0_attention_weight_max": 14.013757705688477,
      "activations/layer0_attention_weight_min": -14.300423622131348,
      "activations/layer10_attention_weight_max": 37.86228942871094,
      "activations/layer10_attention_weight_min": -34.95353698730469,
      "activations/layer11_attention_weight_max": 31.58971405029297,
      "activations/layer11_attention_weight_min": -29.801177978515625,
      "activations/layer12_attention_weight_max": 23.209680557250977,
      "activations/layer12_attention_weight_min": -25.533700942993164,
      "activations/layer13_attention_weight_max": 55.73344421386719,
      "activations/layer13_attention_weight_min": -37.42790603637695,
      "activations/layer14_attention_weight_max": 54.94882583618164,
      "activations/layer14_attention_weight_min": -35.883934020996094,
      "activations/layer15_attention_weight_max": 42.997398376464844,
      "activations/layer15_attention_weight_min": -29.132492065429688,
      "activations/layer16_attention_weight_max": 36.11638641357422,
      "activations/layer16_attention_weight_min": -30.000164031982422,
      "activations/layer17_attention_weight_max": 58.89295959472656,
      "activations/layer17_attention_weight_min": -40.60105895996094,
      "activations/layer18_attention_weight_max": 54.2197380065918,
      "activations/layer18_attention_weight_min": -34.72946548461914,
      "activations/layer19_attention_weight_max": 24.15218162536621,
      "activations/layer19_attention_weight_min": -18.579336166381836,
      "activations/layer1_attention_weight_max": 16.9113712310791,
      "activations/layer1_attention_weight_min": -14.966349601745605,
      "activations/layer20_attention_weight_max": 24.34444236755371,
      "activations/layer20_attention_weight_min": -21.04705047607422,
      "activations/layer21_attention_weight_max": 45.40864944458008,
      "activations/layer21_attention_weight_min": -26.303747177124023,
      "activations/layer22_attention_weight_max": 38.01691818237305,
      "activations/layer22_attention_weight_min": -25.58356475830078,
      "activations/layer23_attention_weight_max": 30.143390655517578,
      "activations/layer23_attention_weight_min": -17.961231231689453,
      "activations/layer2_attention_weight_max": 30.717872619628906,
      "activations/layer2_attention_weight_min": -29.058536529541016,
      "activations/layer3_attention_weight_max": 83.04105377197266,
      "activations/layer3_attention_weight_min": -87.13226318359375,
      "activations/layer4_attention_weight_max": 105.81465911865234,
      "activations/layer4_attention_weight_min": -97.8061294555664,
      "activations/layer5_attention_weight_max": 107.19441986083984,
      "activations/layer5_attention_weight_min": -94.07601165771484,
      "activations/layer6_attention_weight_max": 63.4300422668457,
      "activations/layer6_attention_weight_min": -54.08891677856445,
      "activations/layer7_attention_weight_max": 72.58645629882812,
      "activations/layer7_attention_weight_min": -71.24539947509766,
      "activations/layer8_attention_weight_max": 43.64045715332031,
      "activations/layer8_attention_weight_min": -43.900394439697266,
      "activations/layer9_attention_weight_max": 51.73366165161133,
      "activations/layer9_attention_weight_min": -46.945945739746094,
      "epoch": 5.14,
      "learning_rate": 0.00011802651515151514,
      "loss": 2.8519,
      "step": 88500
    },
    {
      "activations/layer0_attention_weight_max": 14.567147254943848,
      "activations/layer0_attention_weight_min": -14.867137908935547,
      "activations/layer10_attention_weight_max": 35.892459869384766,
      "activations/layer10_attention_weight_min": -35.20052719116211,
      "activations/layer11_attention_weight_max": 29.10261344909668,
      "activations/layer11_attention_weight_min": -27.664405822753906,
      "activations/layer12_attention_weight_max": 21.752164840698242,
      "activations/layer12_attention_weight_min": -26.885364532470703,
      "activations/layer13_attention_weight_max": 55.01701736450195,
      "activations/layer13_attention_weight_min": -34.4068603515625,
      "activations/layer14_attention_weight_max": 46.44680404663086,
      "activations/layer14_attention_weight_min": -30.700286865234375,
      "activations/layer15_attention_weight_max": 43.0980110168457,
      "activations/layer15_attention_weight_min": -31.528703689575195,
      "activations/layer16_attention_weight_max": 35.83914566040039,
      "activations/layer16_attention_weight_min": -26.76203155517578,
      "activations/layer17_attention_weight_max": 56.0687141418457,
      "activations/layer17_attention_weight_min": -39.25431823730469,
      "activations/layer18_attention_weight_max": 52.79762268066406,
      "activations/layer18_attention_weight_min": -33.0338020324707,
      "activations/layer19_attention_weight_max": 23.88678550720215,
      "activations/layer19_attention_weight_min": -19.490966796875,
      "activations/layer1_attention_weight_max": 16.950668334960938,
      "activations/layer1_attention_weight_min": -14.910157203674316,
      "activations/layer20_attention_weight_max": 24.815214157104492,
      "activations/layer20_attention_weight_min": -25.554075241088867,
      "activations/layer21_attention_weight_max": 39.731971740722656,
      "activations/layer21_attention_weight_min": -25.355632781982422,
      "activations/layer22_attention_weight_max": 33.64091110229492,
      "activations/layer22_attention_weight_min": -26.795190811157227,
      "activations/layer23_attention_weight_max": 29.08705711364746,
      "activations/layer23_attention_weight_min": -18.310771942138672,
      "activations/layer2_attention_weight_max": 29.623512268066406,
      "activations/layer2_attention_weight_min": -27.58937644958496,
      "activations/layer3_attention_weight_max": 78.39183044433594,
      "activations/layer3_attention_weight_min": -81.55345153808594,
      "activations/layer4_attention_weight_max": 97.72059631347656,
      "activations/layer4_attention_weight_min": -97.19086456298828,
      "activations/layer5_attention_weight_max": 99.14268493652344,
      "activations/layer5_attention_weight_min": -96.03543090820312,
      "activations/layer6_attention_weight_max": 55.477806091308594,
      "activations/layer6_attention_weight_min": -52.46945571899414,
      "activations/layer7_attention_weight_max": 68.1556625366211,
      "activations/layer7_attention_weight_min": -69.48558044433594,
      "activations/layer8_attention_weight_max": 40.3707160949707,
      "activations/layer8_attention_weight_min": -40.02486038208008,
      "activations/layer9_attention_weight_max": 46.072383880615234,
      "activations/layer9_attention_weight_min": -43.6464958190918,
      "epoch": 5.15,
      "learning_rate": 0.00011800757575757574,
      "loss": 2.8456,
      "step": 88550
    },
    {
      "activations/layer0_attention_weight_max": 13.894766807556152,
      "activations/layer0_attention_weight_min": -14.438292503356934,
      "activations/layer10_attention_weight_max": 36.066619873046875,
      "activations/layer10_attention_weight_min": -35.007015228271484,
      "activations/layer11_attention_weight_max": 29.989013671875,
      "activations/layer11_attention_weight_min": -27.81269645690918,
      "activations/layer12_attention_weight_max": 24.23387908935547,
      "activations/layer12_attention_weight_min": -23.913583755493164,
      "activations/layer13_attention_weight_max": 45.74761199951172,
      "activations/layer13_attention_weight_min": -36.168914794921875,
      "activations/layer14_attention_weight_max": 38.437252044677734,
      "activations/layer14_attention_weight_min": -31.500530242919922,
      "activations/layer15_attention_weight_max": 36.4752311706543,
      "activations/layer15_attention_weight_min": -29.750486373901367,
      "activations/layer16_attention_weight_max": 34.216243743896484,
      "activations/layer16_attention_weight_min": -26.209503173828125,
      "activations/layer17_attention_weight_max": 63.72760009765625,
      "activations/layer17_attention_weight_min": -39.883323669433594,
      "activations/layer18_attention_weight_max": 49.48277282714844,
      "activations/layer18_attention_weight_min": -34.995506286621094,
      "activations/layer19_attention_weight_max": 27.41293716430664,
      "activations/layer19_attention_weight_min": -20.626569747924805,
      "activations/layer1_attention_weight_max": 16.3863582611084,
      "activations/layer1_attention_weight_min": -15.652709007263184,
      "activations/layer20_attention_weight_max": 24.966753005981445,
      "activations/layer20_attention_weight_min": -25.434850692749023,
      "activations/layer21_attention_weight_max": 42.2590217590332,
      "activations/layer21_attention_weight_min": -24.44316864013672,
      "activations/layer22_attention_weight_max": 34.02446365356445,
      "activations/layer22_attention_weight_min": -23.401731491088867,
      "activations/layer23_attention_weight_max": 28.57366180419922,
      "activations/layer23_attention_weight_min": -16.20071029663086,
      "activations/layer2_attention_weight_max": 30.48464012145996,
      "activations/layer2_attention_weight_min": -29.613216400146484,
      "activations/layer3_attention_weight_max": 82.5515365600586,
      "activations/layer3_attention_weight_min": -82.81165313720703,
      "activations/layer4_attention_weight_max": 103.28267669677734,
      "activations/layer4_attention_weight_min": -96.7969970703125,
      "activations/layer5_attention_weight_max": 105.3076400756836,
      "activations/layer5_attention_weight_min": -94.00457000732422,
      "activations/layer6_attention_weight_max": 58.83281707763672,
      "activations/layer6_attention_weight_min": -56.78104019165039,
      "activations/layer7_attention_weight_max": 69.0019760131836,
      "activations/layer7_attention_weight_min": -66.74589538574219,
      "activations/layer8_attention_weight_max": 42.428138732910156,
      "activations/layer8_attention_weight_min": -42.50141143798828,
      "activations/layer9_attention_weight_max": 47.049076080322266,
      "activations/layer9_attention_weight_min": -44.91075897216797,
      "epoch": 5.15,
      "learning_rate": 0.00011798863636363636,
      "loss": 2.858,
      "step": 88600
    },
    {
      "activations/layer0_attention_weight_max": 15.70683479309082,
      "activations/layer0_attention_weight_min": -14.500930786132812,
      "activations/layer10_attention_weight_max": 38.443424224853516,
      "activations/layer10_attention_weight_min": -33.73148727416992,
      "activations/layer11_attention_weight_max": 30.47283172607422,
      "activations/layer11_attention_weight_min": -26.959915161132812,
      "activations/layer12_attention_weight_max": 27.120943069458008,
      "activations/layer12_attention_weight_min": -26.868000030517578,
      "activations/layer13_attention_weight_max": 55.297454833984375,
      "activations/layer13_attention_weight_min": -34.445716857910156,
      "activations/layer14_attention_weight_max": 50.33027267456055,
      "activations/layer14_attention_weight_min": -31.19878387451172,
      "activations/layer15_attention_weight_max": 43.05938720703125,
      "activations/layer15_attention_weight_min": -28.10346031188965,
      "activations/layer16_attention_weight_max": 36.4357795715332,
      "activations/layer16_attention_weight_min": -25.6318416595459,
      "activations/layer17_attention_weight_max": 60.96640396118164,
      "activations/layer17_attention_weight_min": -38.50651931762695,
      "activations/layer18_attention_weight_max": 51.27252197265625,
      "activations/layer18_attention_weight_min": -32.872825622558594,
      "activations/layer19_attention_weight_max": 20.974275588989258,
      "activations/layer19_attention_weight_min": -20.69539451599121,
      "activations/layer1_attention_weight_max": 18.105045318603516,
      "activations/layer1_attention_weight_min": -16.128826141357422,
      "activations/layer20_attention_weight_max": 23.00641441345215,
      "activations/layer20_attention_weight_min": -21.81197166442871,
      "activations/layer21_attention_weight_max": 42.67761993408203,
      "activations/layer21_attention_weight_min": -25.163475036621094,
      "activations/layer22_attention_weight_max": 35.82072830200195,
      "activations/layer22_attention_weight_min": -24.39798355102539,
      "activations/layer23_attention_weight_max": 28.029422760009766,
      "activations/layer23_attention_weight_min": -19.24481773376465,
      "activations/layer2_attention_weight_max": 31.914655685424805,
      "activations/layer2_attention_weight_min": -29.06250762939453,
      "activations/layer3_attention_weight_max": 83.52469635009766,
      "activations/layer3_attention_weight_min": -83.81446838378906,
      "activations/layer4_attention_weight_max": 98.89949035644531,
      "activations/layer4_attention_weight_min": -97.68354034423828,
      "activations/layer5_attention_weight_max": 101.91740417480469,
      "activations/layer5_attention_weight_min": -93.4714584350586,
      "activations/layer6_attention_weight_max": 57.51829147338867,
      "activations/layer6_attention_weight_min": -51.83705139160156,
      "activations/layer7_attention_weight_max": 70.77759552001953,
      "activations/layer7_attention_weight_min": -68.8514633178711,
      "activations/layer8_attention_weight_max": 43.884498596191406,
      "activations/layer8_attention_weight_min": -43.50807189941406,
      "activations/layer9_attention_weight_max": 56.694122314453125,
      "activations/layer9_attention_weight_min": -47.55519104003906,
      "epoch": 5.15,
      "learning_rate": 0.00011796969696969696,
      "loss": 2.8571,
      "step": 88650
    },
    {
      "activations/layer0_attention_weight_max": 14.332209587097168,
      "activations/layer0_attention_weight_min": -14.327115058898926,
      "activations/layer10_attention_weight_max": 40.234458923339844,
      "activations/layer10_attention_weight_min": -35.025550842285156,
      "activations/layer11_attention_weight_max": 30.4456787109375,
      "activations/layer11_attention_weight_min": -26.346817016601562,
      "activations/layer12_attention_weight_max": 27.637163162231445,
      "activations/layer12_attention_weight_min": -28.338062286376953,
      "activations/layer13_attention_weight_max": 48.436275482177734,
      "activations/layer13_attention_weight_min": -34.313926696777344,
      "activations/layer14_attention_weight_max": 39.39868927001953,
      "activations/layer14_attention_weight_min": -30.11680030822754,
      "activations/layer15_attention_weight_max": 43.49775695800781,
      "activations/layer15_attention_weight_min": -30.04210090637207,
      "activations/layer16_attention_weight_max": 34.779483795166016,
      "activations/layer16_attention_weight_min": -26.469501495361328,
      "activations/layer17_attention_weight_max": 58.16551971435547,
      "activations/layer17_attention_weight_min": -36.88178634643555,
      "activations/layer18_attention_weight_max": 50.362239837646484,
      "activations/layer18_attention_weight_min": -31.858186721801758,
      "activations/layer19_attention_weight_max": 27.6680850982666,
      "activations/layer19_attention_weight_min": -19.98482894897461,
      "activations/layer1_attention_weight_max": 16.752559661865234,
      "activations/layer1_attention_weight_min": -17.678129196166992,
      "activations/layer20_attention_weight_max": 23.382116317749023,
      "activations/layer20_attention_weight_min": -20.24658966064453,
      "activations/layer21_attention_weight_max": 37.336944580078125,
      "activations/layer21_attention_weight_min": -28.071319580078125,
      "activations/layer22_attention_weight_max": 38.04938888549805,
      "activations/layer22_attention_weight_min": -23.84124183654785,
      "activations/layer23_attention_weight_max": 28.52045249938965,
      "activations/layer23_attention_weight_min": -19.198129653930664,
      "activations/layer2_attention_weight_max": 29.25566864013672,
      "activations/layer2_attention_weight_min": -27.885658264160156,
      "activations/layer3_attention_weight_max": 81.95704650878906,
      "activations/layer3_attention_weight_min": -84.5121078491211,
      "activations/layer4_attention_weight_max": 98.99772644042969,
      "activations/layer4_attention_weight_min": -97.38398742675781,
      "activations/layer5_attention_weight_max": 102.21163940429688,
      "activations/layer5_attention_weight_min": -96.65534973144531,
      "activations/layer6_attention_weight_max": 58.07352828979492,
      "activations/layer6_attention_weight_min": -57.21337127685547,
      "activations/layer7_attention_weight_max": 73.75858306884766,
      "activations/layer7_attention_weight_min": -68.89618682861328,
      "activations/layer8_attention_weight_max": 45.10946273803711,
      "activations/layer8_attention_weight_min": -40.825645446777344,
      "activations/layer9_attention_weight_max": 56.14482498168945,
      "activations/layer9_attention_weight_min": -43.46686935424805,
      "epoch": 5.15,
      "learning_rate": 0.00011795075757575756,
      "loss": 2.8454,
      "step": 88700
    },
    {
      "activations/layer0_attention_weight_max": 14.216650009155273,
      "activations/layer0_attention_weight_min": -14.757232666015625,
      "activations/layer10_attention_weight_max": 36.43980026245117,
      "activations/layer10_attention_weight_min": -35.1324462890625,
      "activations/layer11_attention_weight_max": 28.161273956298828,
      "activations/layer11_attention_weight_min": -28.699180603027344,
      "activations/layer12_attention_weight_max": 25.36085319519043,
      "activations/layer12_attention_weight_min": -24.479894638061523,
      "activations/layer13_attention_weight_max": 45.21714782714844,
      "activations/layer13_attention_weight_min": -34.740623474121094,
      "activations/layer14_attention_weight_max": 48.202964782714844,
      "activations/layer14_attention_weight_min": -32.76652908325195,
      "activations/layer15_attention_weight_max": 39.3581657409668,
      "activations/layer15_attention_weight_min": -29.44639778137207,
      "activations/layer16_attention_weight_max": 39.649879455566406,
      "activations/layer16_attention_weight_min": -26.591331481933594,
      "activations/layer17_attention_weight_max": 55.49659729003906,
      "activations/layer17_attention_weight_min": -38.44882583618164,
      "activations/layer18_attention_weight_max": 49.0608024597168,
      "activations/layer18_attention_weight_min": -32.158164978027344,
      "activations/layer19_attention_weight_max": 26.423749923706055,
      "activations/layer19_attention_weight_min": -18.870813369750977,
      "activations/layer1_attention_weight_max": 17.11859130859375,
      "activations/layer1_attention_weight_min": -16.778356552124023,
      "activations/layer20_attention_weight_max": 19.978626251220703,
      "activations/layer20_attention_weight_min": -22.107746124267578,
      "activations/layer21_attention_weight_max": 39.96611022949219,
      "activations/layer21_attention_weight_min": -25.712528228759766,
      "activations/layer22_attention_weight_max": 33.779747009277344,
      "activations/layer22_attention_weight_min": -24.40359878540039,
      "activations/layer23_attention_weight_max": 27.312856674194336,
      "activations/layer23_attention_weight_min": -17.798784255981445,
      "activations/layer2_attention_weight_max": 30.035415649414062,
      "activations/layer2_attention_weight_min": -28.027511596679688,
      "activations/layer3_attention_weight_max": 81.93409729003906,
      "activations/layer3_attention_weight_min": -82.19425964355469,
      "activations/layer4_attention_weight_max": 102.70918273925781,
      "activations/layer4_attention_weight_min": -91.7104721069336,
      "activations/layer5_attention_weight_max": 103.78521728515625,
      "activations/layer5_attention_weight_min": -90.29794311523438,
      "activations/layer6_attention_weight_max": 57.64667510986328,
      "activations/layer6_attention_weight_min": -50.26156997680664,
      "activations/layer7_attention_weight_max": 70.22310638427734,
      "activations/layer7_attention_weight_min": -63.81599426269531,
      "activations/layer8_attention_weight_max": 43.00581741333008,
      "activations/layer8_attention_weight_min": -40.70424270629883,
      "activations/layer9_attention_weight_max": 55.16255569458008,
      "activations/layer9_attention_weight_min": -47.28565979003906,
      "epoch": 5.16,
      "learning_rate": 0.00011793181818181817,
      "loss": 2.8501,
      "step": 88750
    },
    {
      "activations/layer0_attention_weight_max": 14.395307540893555,
      "activations/layer0_attention_weight_min": -14.615567207336426,
      "activations/layer10_attention_weight_max": 37.46391677856445,
      "activations/layer10_attention_weight_min": -33.469207763671875,
      "activations/layer11_attention_weight_max": 27.819637298583984,
      "activations/layer11_attention_weight_min": -25.097522735595703,
      "activations/layer12_attention_weight_max": 28.144433975219727,
      "activations/layer12_attention_weight_min": -24.54648780822754,
      "activations/layer13_attention_weight_max": 52.864845275878906,
      "activations/layer13_attention_weight_min": -39.7756462097168,
      "activations/layer14_attention_weight_max": 54.04926681518555,
      "activations/layer14_attention_weight_min": -28.532072067260742,
      "activations/layer15_attention_weight_max": 49.38285446166992,
      "activations/layer15_attention_weight_min": -29.617507934570312,
      "activations/layer16_attention_weight_max": 40.93240737915039,
      "activations/layer16_attention_weight_min": -25.6608829498291,
      "activations/layer17_attention_weight_max": 60.6400032043457,
      "activations/layer17_attention_weight_min": -38.95778274536133,
      "activations/layer18_attention_weight_max": 57.83828353881836,
      "activations/layer18_attention_weight_min": -32.31480026245117,
      "activations/layer19_attention_weight_max": 24.483285903930664,
      "activations/layer19_attention_weight_min": -19.272729873657227,
      "activations/layer1_attention_weight_max": 16.518264770507812,
      "activations/layer1_attention_weight_min": -15.508904457092285,
      "activations/layer20_attention_weight_max": 24.723987579345703,
      "activations/layer20_attention_weight_min": -24.34934425354004,
      "activations/layer21_attention_weight_max": 50.316612243652344,
      "activations/layer21_attention_weight_min": -26.726383209228516,
      "activations/layer22_attention_weight_max": 34.41444396972656,
      "activations/layer22_attention_weight_min": -22.634538650512695,
      "activations/layer23_attention_weight_max": 31.867420196533203,
      "activations/layer23_attention_weight_min": -17.75495147705078,
      "activations/layer2_attention_weight_max": 30.266632080078125,
      "activations/layer2_attention_weight_min": -28.783103942871094,
      "activations/layer3_attention_weight_max": 80.51626586914062,
      "activations/layer3_attention_weight_min": -82.99659729003906,
      "activations/layer4_attention_weight_max": 101.1957015991211,
      "activations/layer4_attention_weight_min": -93.0262680053711,
      "activations/layer5_attention_weight_max": 104.10548400878906,
      "activations/layer5_attention_weight_min": -93.9435043334961,
      "activations/layer6_attention_weight_max": 62.52273941040039,
      "activations/layer6_attention_weight_min": -54.63018035888672,
      "activations/layer7_attention_weight_max": 70.06920623779297,
      "activations/layer7_attention_weight_min": -67.67615509033203,
      "activations/layer8_attention_weight_max": 43.5610237121582,
      "activations/layer8_attention_weight_min": -42.2154426574707,
      "activations/layer9_attention_weight_max": 50.698177337646484,
      "activations/layer9_attention_weight_min": -45.83427047729492,
      "epoch": 5.16,
      "learning_rate": 0.00011791287878787878,
      "loss": 2.8449,
      "step": 88800
    },
    {
      "activations/layer0_attention_weight_max": 14.966294288635254,
      "activations/layer0_attention_weight_min": -14.77782917022705,
      "activations/layer10_attention_weight_max": 36.91912078857422,
      "activations/layer10_attention_weight_min": -33.95252990722656,
      "activations/layer11_attention_weight_max": 28.118825912475586,
      "activations/layer11_attention_weight_min": -26.84893798828125,
      "activations/layer12_attention_weight_max": 19.49161720275879,
      "activations/layer12_attention_weight_min": -25.624208450317383,
      "activations/layer13_attention_weight_max": 43.8555793762207,
      "activations/layer13_attention_weight_min": -35.002342224121094,
      "activations/layer14_attention_weight_max": 36.84960174560547,
      "activations/layer14_attention_weight_min": -30.33026695251465,
      "activations/layer15_attention_weight_max": 46.77442169189453,
      "activations/layer15_attention_weight_min": -29.494342803955078,
      "activations/layer16_attention_weight_max": 35.555259704589844,
      "activations/layer16_attention_weight_min": -28.089895248413086,
      "activations/layer17_attention_weight_max": 53.5982551574707,
      "activations/layer17_attention_weight_min": -37.8542366027832,
      "activations/layer18_attention_weight_max": 50.6167106628418,
      "activations/layer18_attention_weight_min": -32.76618576049805,
      "activations/layer19_attention_weight_max": 25.070693969726562,
      "activations/layer19_attention_weight_min": -20.089519500732422,
      "activations/layer1_attention_weight_max": 18.144515991210938,
      "activations/layer1_attention_weight_min": -17.32301139831543,
      "activations/layer20_attention_weight_max": 23.64012336730957,
      "activations/layer20_attention_weight_min": -25.329931259155273,
      "activations/layer21_attention_weight_max": 44.236202239990234,
      "activations/layer21_attention_weight_min": -26.150249481201172,
      "activations/layer22_attention_weight_max": 35.185482025146484,
      "activations/layer22_attention_weight_min": -25.102924346923828,
      "activations/layer23_attention_weight_max": 29.635128021240234,
      "activations/layer23_attention_weight_min": -19.530452728271484,
      "activations/layer2_attention_weight_max": 30.977306365966797,
      "activations/layer2_attention_weight_min": -27.1832275390625,
      "activations/layer3_attention_weight_max": 80.97235870361328,
      "activations/layer3_attention_weight_min": -83.33033752441406,
      "activations/layer4_attention_weight_max": 101.96379852294922,
      "activations/layer4_attention_weight_min": -96.28902435302734,
      "activations/layer5_attention_weight_max": 102.71025848388672,
      "activations/layer5_attention_weight_min": -92.34510803222656,
      "activations/layer6_attention_weight_max": 57.33900833129883,
      "activations/layer6_attention_weight_min": -52.30625915527344,
      "activations/layer7_attention_weight_max": 67.06764221191406,
      "activations/layer7_attention_weight_min": -64.97073364257812,
      "activations/layer8_attention_weight_max": 43.460601806640625,
      "activations/layer8_attention_weight_min": -42.920902252197266,
      "activations/layer9_attention_weight_max": 45.94283676147461,
      "activations/layer9_attention_weight_min": -48.561519622802734,
      "epoch": 5.16,
      "learning_rate": 0.00011789393939393939,
      "loss": 2.8343,
      "step": 88850
    },
    {
      "activations/layer0_attention_weight_max": 15.231319427490234,
      "activations/layer0_attention_weight_min": -14.086557388305664,
      "activations/layer10_attention_weight_max": 39.19891357421875,
      "activations/layer10_attention_weight_min": -34.18428421020508,
      "activations/layer11_attention_weight_max": 29.159317016601562,
      "activations/layer11_attention_weight_min": -25.416748046875,
      "activations/layer12_attention_weight_max": 18.69259262084961,
      "activations/layer12_attention_weight_min": -29.651426315307617,
      "activations/layer13_attention_weight_max": 39.52696228027344,
      "activations/layer13_attention_weight_min": -34.3651008605957,
      "activations/layer14_attention_weight_max": 36.08953094482422,
      "activations/layer14_attention_weight_min": -26.917036056518555,
      "activations/layer15_attention_weight_max": 35.869869232177734,
      "activations/layer15_attention_weight_min": -28.27224349975586,
      "activations/layer16_attention_weight_max": 28.83818817138672,
      "activations/layer16_attention_weight_min": -25.1053466796875,
      "activations/layer17_attention_weight_max": 52.31605529785156,
      "activations/layer17_attention_weight_min": -36.863372802734375,
      "activations/layer18_attention_weight_max": 43.487918853759766,
      "activations/layer18_attention_weight_min": -30.514450073242188,
      "activations/layer19_attention_weight_max": 20.61065101623535,
      "activations/layer19_attention_weight_min": -20.647520065307617,
      "activations/layer1_attention_weight_max": 17.82927703857422,
      "activations/layer1_attention_weight_min": -13.924110412597656,
      "activations/layer20_attention_weight_max": 18.922258377075195,
      "activations/layer20_attention_weight_min": -22.48615264892578,
      "activations/layer21_attention_weight_max": 29.443740844726562,
      "activations/layer21_attention_weight_min": -22.51100730895996,
      "activations/layer22_attention_weight_max": 29.531938552856445,
      "activations/layer22_attention_weight_min": -24.32902717590332,
      "activations/layer23_attention_weight_max": 25.53961753845215,
      "activations/layer23_attention_weight_min": -19.821712493896484,
      "activations/layer2_attention_weight_max": 29.317989349365234,
      "activations/layer2_attention_weight_min": -28.761737823486328,
      "activations/layer3_attention_weight_max": 85.66404724121094,
      "activations/layer3_attention_weight_min": -84.96637725830078,
      "activations/layer4_attention_weight_max": 107.2750473022461,
      "activations/layer4_attention_weight_min": -96.63805389404297,
      "activations/layer5_attention_weight_max": 100.1122055053711,
      "activations/layer5_attention_weight_min": -92.18462371826172,
      "activations/layer6_attention_weight_max": 56.011966705322266,
      "activations/layer6_attention_weight_min": -52.60721969604492,
      "activations/layer7_attention_weight_max": 69.14385223388672,
      "activations/layer7_attention_weight_min": -70.32112884521484,
      "activations/layer8_attention_weight_max": 41.139347076416016,
      "activations/layer8_attention_weight_min": -40.23543167114258,
      "activations/layer9_attention_weight_max": 61.06269073486328,
      "activations/layer9_attention_weight_min": -51.78091049194336,
      "epoch": 5.17,
      "learning_rate": 0.00011787499999999999,
      "loss": 2.8355,
      "step": 88900
    },
    {
      "activations/layer0_attention_weight_max": 15.356568336486816,
      "activations/layer0_attention_weight_min": -14.46821403503418,
      "activations/layer10_attention_weight_max": 36.643428802490234,
      "activations/layer10_attention_weight_min": -32.492801666259766,
      "activations/layer11_attention_weight_max": 29.76741600036621,
      "activations/layer11_attention_weight_min": -28.45228385925293,
      "activations/layer12_attention_weight_max": 21.782142639160156,
      "activations/layer12_attention_weight_min": -25.431882858276367,
      "activations/layer13_attention_weight_max": 46.66450881958008,
      "activations/layer13_attention_weight_min": -36.99253463745117,
      "activations/layer14_attention_weight_max": 42.06853485107422,
      "activations/layer14_attention_weight_min": -26.964704513549805,
      "activations/layer15_attention_weight_max": 38.437740325927734,
      "activations/layer15_attention_weight_min": -29.440961837768555,
      "activations/layer16_attention_weight_max": 30.55400848388672,
      "activations/layer16_attention_weight_min": -25.731342315673828,
      "activations/layer17_attention_weight_max": 53.03843307495117,
      "activations/layer17_attention_weight_min": -35.73478317260742,
      "activations/layer18_attention_weight_max": 44.36737060546875,
      "activations/layer18_attention_weight_min": -31.132436752319336,
      "activations/layer19_attention_weight_max": 19.664997100830078,
      "activations/layer19_attention_weight_min": -19.01688003540039,
      "activations/layer1_attention_weight_max": 18.42426109313965,
      "activations/layer1_attention_weight_min": -15.26848030090332,
      "activations/layer20_attention_weight_max": 19.579397201538086,
      "activations/layer20_attention_weight_min": -23.843303680419922,
      "activations/layer21_attention_weight_max": 34.112548828125,
      "activations/layer21_attention_weight_min": -27.404314041137695,
      "activations/layer22_attention_weight_max": 33.69678497314453,
      "activations/layer22_attention_weight_min": -24.11382293701172,
      "activations/layer23_attention_weight_max": 26.614843368530273,
      "activations/layer23_attention_weight_min": -17.667400360107422,
      "activations/layer2_attention_weight_max": 32.60249328613281,
      "activations/layer2_attention_weight_min": -30.3883056640625,
      "activations/layer3_attention_weight_max": 80.3380126953125,
      "activations/layer3_attention_weight_min": -84.61820220947266,
      "activations/layer4_attention_weight_max": 97.10501861572266,
      "activations/layer4_attention_weight_min": -94.54898071289062,
      "activations/layer5_attention_weight_max": 99.8541259765625,
      "activations/layer5_attention_weight_min": -89.82796478271484,
      "activations/layer6_attention_weight_max": 58.81543731689453,
      "activations/layer6_attention_weight_min": -50.4969367980957,
      "activations/layer7_attention_weight_max": 69.40802001953125,
      "activations/layer7_attention_weight_min": -68.39214324951172,
      "activations/layer8_attention_weight_max": 40.21868133544922,
      "activations/layer8_attention_weight_min": -42.3448371887207,
      "activations/layer9_attention_weight_max": 45.998287200927734,
      "activations/layer9_attention_weight_min": -45.50778579711914,
      "epoch": 5.17,
      "learning_rate": 0.0001178560606060606,
      "loss": 2.8378,
      "step": 88950
    },
    {
      "activations/layer0_attention_weight_max": 15.633251190185547,
      "activations/layer0_attention_weight_min": -14.77466106414795,
      "activations/layer10_attention_weight_max": 40.7517204284668,
      "activations/layer10_attention_weight_min": -37.362945556640625,
      "activations/layer11_attention_weight_max": 30.601423263549805,
      "activations/layer11_attention_weight_min": -29.080612182617188,
      "activations/layer12_attention_weight_max": 19.64495277404785,
      "activations/layer12_attention_weight_min": -26.437152862548828,
      "activations/layer13_attention_weight_max": 53.45280075073242,
      "activations/layer13_attention_weight_min": -41.12333297729492,
      "activations/layer14_attention_weight_max": 61.57463073730469,
      "activations/layer14_attention_weight_min": -40.152061462402344,
      "activations/layer15_attention_weight_max": 46.396263122558594,
      "activations/layer15_attention_weight_min": -34.39543151855469,
      "activations/layer16_attention_weight_max": 32.95215606689453,
      "activations/layer16_attention_weight_min": -27.25847053527832,
      "activations/layer17_attention_weight_max": 51.64866638183594,
      "activations/layer17_attention_weight_min": -38.73092269897461,
      "activations/layer18_attention_weight_max": 45.18079376220703,
      "activations/layer18_attention_weight_min": -35.06721115112305,
      "activations/layer19_attention_weight_max": 25.245023727416992,
      "activations/layer19_attention_weight_min": -19.887828826904297,
      "activations/layer1_attention_weight_max": 17.448528289794922,
      "activations/layer1_attention_weight_min": -16.209257125854492,
      "activations/layer20_attention_weight_max": 25.202184677124023,
      "activations/layer20_attention_weight_min": -21.462928771972656,
      "activations/layer21_attention_weight_max": 46.00872039794922,
      "activations/layer21_attention_weight_min": -27.361019134521484,
      "activations/layer22_attention_weight_max": 33.03899002075195,
      "activations/layer22_attention_weight_min": -23.24237060546875,
      "activations/layer23_attention_weight_max": 28.055334091186523,
      "activations/layer23_attention_weight_min": -18.840147018432617,
      "activations/layer2_attention_weight_max": 31.09229850769043,
      "activations/layer2_attention_weight_min": -31.06238555908203,
      "activations/layer3_attention_weight_max": 81.90672302246094,
      "activations/layer3_attention_weight_min": -86.52102661132812,
      "activations/layer4_attention_weight_max": 104.62203216552734,
      "activations/layer4_attention_weight_min": -99.5925064086914,
      "activations/layer5_attention_weight_max": 108.10166931152344,
      "activations/layer5_attention_weight_min": -97.5459213256836,
      "activations/layer6_attention_weight_max": 63.599735260009766,
      "activations/layer6_attention_weight_min": -55.842594146728516,
      "activations/layer7_attention_weight_max": 70.81340026855469,
      "activations/layer7_attention_weight_min": -72.02993774414062,
      "activations/layer8_attention_weight_max": 46.54497146606445,
      "activations/layer8_attention_weight_min": -43.41823196411133,
      "activations/layer9_attention_weight_max": 54.99673080444336,
      "activations/layer9_attention_weight_min": -51.862422943115234,
      "epoch": 5.17,
      "learning_rate": 0.00011783712121212121,
      "loss": 2.8483,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_loss": 2.80078125,
      "eval_runtime": 8.4618,
      "eval_samples_per_second": 507.455,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_openwebtext_loss": 2.80078125,
      "eval_openwebtext_ppl": 16.457499171202052,
      "eval_openwebtext_runtime": 8.4618,
      "eval_openwebtext_samples_per_second": 507.455,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 2.0422,
      "eval_wikitext_samples_per_second": 223.286,
      "step": 89000
    },
    {
      "epoch": 5.17,
      "eval_lambada_loss": 2.873046875,
      "eval_lambada_ppl": 17.690837939037834,
      "eval_lambada_runtime": 9.5994,
      "eval_lambada_samples_per_second": 507.217,
      "step": 89000
    },
    {
      "activations/layer0_attention_weight_max": 15.077385902404785,
      "activations/layer0_attention_weight_min": -14.20360279083252,
      "activations/layer10_attention_weight_max": 38.38227081298828,
      "activations/layer10_attention_weight_min": -35.53669357299805,
      "activations/layer11_attention_weight_max": 29.630308151245117,
      "activations/layer11_attention_weight_min": -27.101417541503906,
      "activations/layer12_attention_weight_max": 21.206588745117188,
      "activations/layer12_attention_weight_min": -25.841724395751953,
      "activations/layer13_attention_weight_max": 44.992347717285156,
      "activations/layer13_attention_weight_min": -38.696590423583984,
      "activations/layer14_attention_weight_max": 47.31449890136719,
      "activations/layer14_attention_weight_min": -31.793899536132812,
      "activations/layer15_attention_weight_max": 37.42950439453125,
      "activations/layer15_attention_weight_min": -30.553466796875,
      "activations/layer16_attention_weight_max": 32.75498580932617,
      "activations/layer16_attention_weight_min": -26.555805206298828,
      "activations/layer17_attention_weight_max": 60.07423400878906,
      "activations/layer17_attention_weight_min": -38.169918060302734,
      "activations/layer18_attention_weight_max": 47.37165451049805,
      "activations/layer18_attention_weight_min": -34.785888671875,
      "activations/layer19_attention_weight_max": 20.283369064331055,
      "activations/layer19_attention_weight_min": -21.45179557800293,
      "activations/layer1_attention_weight_max": 16.6492977142334,
      "activations/layer1_attention_weight_min": -15.150490760803223,
      "activations/layer20_attention_weight_max": 23.50455665588379,
      "activations/layer20_attention_weight_min": -20.988000869750977,
      "activations/layer21_attention_weight_max": 42.93998336791992,
      "activations/layer21_attention_weight_min": -27.210996627807617,
      "activations/layer22_attention_weight_max": 31.94717788696289,
      "activations/layer22_attention_weight_min": -24.905773162841797,
      "activations/layer23_attention_weight_max": 27.6114559173584,
      "activations/layer23_attention_weight_min": -17.575307846069336,
      "activations/layer2_attention_weight_max": 31.174400329589844,
      "activations/layer2_attention_weight_min": -28.995628356933594,
      "activations/layer3_attention_weight_max": 86.1604995727539,
      "activations/layer3_attention_weight_min": -87.35708618164062,
      "activations/layer4_attention_weight_max": 104.2276382446289,
      "activations/layer4_attention_weight_min": -99.95128631591797,
      "activations/layer5_attention_weight_max": 101.55094909667969,
      "activations/layer5_attention_weight_min": -94.54844665527344,
      "activations/layer6_attention_weight_max": 58.824249267578125,
      "activations/layer6_attention_weight_min": -52.523597717285156,
      "activations/layer7_attention_weight_max": 68.31647491455078,
      "activations/layer7_attention_weight_min": -66.0521240234375,
      "activations/layer8_attention_weight_max": 46.476078033447266,
      "activations/layer8_attention_weight_min": -40.98419189453125,
      "activations/layer9_attention_weight_max": 51.76917266845703,
      "activations/layer9_attention_weight_min": -46.26947021484375,
      "epoch": 5.17,
      "learning_rate": 0.0001178181818181818,
      "loss": 2.8425,
      "step": 89050
    },
    {
      "activations/layer0_attention_weight_max": 15.017027854919434,
      "activations/layer0_attention_weight_min": -14.721355438232422,
      "activations/layer10_attention_weight_max": 34.59405517578125,
      "activations/layer10_attention_weight_min": -34.24200439453125,
      "activations/layer11_attention_weight_max": 29.157756805419922,
      "activations/layer11_attention_weight_min": -27.71601676940918,
      "activations/layer12_attention_weight_max": 20.180871963500977,
      "activations/layer12_attention_weight_min": -23.062423706054688,
      "activations/layer13_attention_weight_max": 47.03849411010742,
      "activations/layer13_attention_weight_min": -34.05048370361328,
      "activations/layer14_attention_weight_max": 39.65937042236328,
      "activations/layer14_attention_weight_min": -28.366384506225586,
      "activations/layer15_attention_weight_max": 44.77459716796875,
      "activations/layer15_attention_weight_min": -28.886390686035156,
      "activations/layer16_attention_weight_max": 37.675323486328125,
      "activations/layer16_attention_weight_min": -26.410053253173828,
      "activations/layer17_attention_weight_max": 58.5187873840332,
      "activations/layer17_attention_weight_min": -40.849369049072266,
      "activations/layer18_attention_weight_max": 52.424102783203125,
      "activations/layer18_attention_weight_min": -35.20256042480469,
      "activations/layer19_attention_weight_max": 23.318347930908203,
      "activations/layer19_attention_weight_min": -17.819988250732422,
      "activations/layer1_attention_weight_max": 18.098785400390625,
      "activations/layer1_attention_weight_min": -13.15739917755127,
      "activations/layer20_attention_weight_max": 21.94877815246582,
      "activations/layer20_attention_weight_min": -21.153440475463867,
      "activations/layer21_attention_weight_max": 37.70270538330078,
      "activations/layer21_attention_weight_min": -27.29477310180664,
      "activations/layer22_attention_weight_max": 36.52125930786133,
      "activations/layer22_attention_weight_min": -23.524791717529297,
      "activations/layer23_attention_weight_max": 30.486087799072266,
      "activations/layer23_attention_weight_min": -17.212377548217773,
      "activations/layer2_attention_weight_max": 28.836294174194336,
      "activations/layer2_attention_weight_min": -27.507980346679688,
      "activations/layer3_attention_weight_max": 75.19599914550781,
      "activations/layer3_attention_weight_min": -79.70506286621094,
      "activations/layer4_attention_weight_max": 96.94670867919922,
      "activations/layer4_attention_weight_min": -91.8488998413086,
      "activations/layer5_attention_weight_max": 97.04415130615234,
      "activations/layer5_attention_weight_min": -90.60957336425781,
      "activations/layer6_attention_weight_max": 54.64436340332031,
      "activations/layer6_attention_weight_min": -50.130943298339844,
      "activations/layer7_attention_weight_max": 68.37450408935547,
      "activations/layer7_attention_weight_min": -67.83393859863281,
      "activations/layer8_attention_weight_max": 41.363826751708984,
      "activations/layer8_attention_weight_min": -42.58716583251953,
      "activations/layer9_attention_weight_max": 48.74265670776367,
      "activations/layer9_attention_weight_min": -44.66543197631836,
      "epoch": 5.18,
      "learning_rate": 0.00011779924242424241,
      "loss": 2.8459,
      "step": 89100
    },
    {
      "activations/layer0_attention_weight_max": 14.092936515808105,
      "activations/layer0_attention_weight_min": -14.545942306518555,
      "activations/layer10_attention_weight_max": 35.57489776611328,
      "activations/layer10_attention_weight_min": -31.765487670898438,
      "activations/layer11_attention_weight_max": 28.02467155456543,
      "activations/layer11_attention_weight_min": -25.143009185791016,
      "activations/layer12_attention_weight_max": 19.653614044189453,
      "activations/layer12_attention_weight_min": -24.063980102539062,
      "activations/layer13_attention_weight_max": 47.72648620605469,
      "activations/layer13_attention_weight_min": -37.14287185668945,
      "activations/layer14_attention_weight_max": 46.42533874511719,
      "activations/layer14_attention_weight_min": -36.516563415527344,
      "activations/layer15_attention_weight_max": 36.029052734375,
      "activations/layer15_attention_weight_min": -29.6668643951416,
      "activations/layer16_attention_weight_max": 28.256765365600586,
      "activations/layer16_attention_weight_min": -26.464815139770508,
      "activations/layer17_attention_weight_max": 48.653011322021484,
      "activations/layer17_attention_weight_min": -38.91520309448242,
      "activations/layer18_attention_weight_max": 43.49317932128906,
      "activations/layer18_attention_weight_min": -33.681373596191406,
      "activations/layer19_attention_weight_max": 21.579429626464844,
      "activations/layer19_attention_weight_min": -18.25386619567871,
      "activations/layer1_attention_weight_max": 17.20012664794922,
      "activations/layer1_attention_weight_min": -14.568618774414062,
      "activations/layer20_attention_weight_max": 18.0318546295166,
      "activations/layer20_attention_weight_min": -20.7994327545166,
      "activations/layer21_attention_weight_max": 36.49881362915039,
      "activations/layer21_attention_weight_min": -25.081340789794922,
      "activations/layer22_attention_weight_max": 34.06141662597656,
      "activations/layer22_attention_weight_min": -27.192659378051758,
      "activations/layer23_attention_weight_max": 26.33020782470703,
      "activations/layer23_attention_weight_min": -17.101245880126953,
      "activations/layer2_attention_weight_max": 32.40009307861328,
      "activations/layer2_attention_weight_min": -28.286434173583984,
      "activations/layer3_attention_weight_max": 79.62239837646484,
      "activations/layer3_attention_weight_min": -82.6158676147461,
      "activations/layer4_attention_weight_max": 100.60271453857422,
      "activations/layer4_attention_weight_min": -92.4251937866211,
      "activations/layer5_attention_weight_max": 104.20510864257812,
      "activations/layer5_attention_weight_min": -86.41960144042969,
      "activations/layer6_attention_weight_max": 56.282474517822266,
      "activations/layer6_attention_weight_min": -52.3169059753418,
      "activations/layer7_attention_weight_max": 68.82078552246094,
      "activations/layer7_attention_weight_min": -62.23651123046875,
      "activations/layer8_attention_weight_max": 40.08109664916992,
      "activations/layer8_attention_weight_min": -39.10240173339844,
      "activations/layer9_attention_weight_max": 45.94395446777344,
      "activations/layer9_attention_weight_min": -45.86910629272461,
      "epoch": 5.18,
      "learning_rate": 0.00011778030303030303,
      "loss": 2.8582,
      "step": 89150
    },
    {
      "activations/layer0_attention_weight_max": 15.721383094787598,
      "activations/layer0_attention_weight_min": -14.365368843078613,
      "activations/layer10_attention_weight_max": 37.274566650390625,
      "activations/layer10_attention_weight_min": -35.450103759765625,
      "activations/layer11_attention_weight_max": 29.078969955444336,
      "activations/layer11_attention_weight_min": -26.34069061279297,
      "activations/layer12_attention_weight_max": 22.561017990112305,
      "activations/layer12_attention_weight_min": -26.433435440063477,
      "activations/layer13_attention_weight_max": 43.7158088684082,
      "activations/layer13_attention_weight_min": -34.20444869995117,
      "activations/layer14_attention_weight_max": 44.24592208862305,
      "activations/layer14_attention_weight_min": -29.815988540649414,
      "activations/layer15_attention_weight_max": 37.2221565246582,
      "activations/layer15_attention_weight_min": -28.898103713989258,
      "activations/layer16_attention_weight_max": 36.024349212646484,
      "activations/layer16_attention_weight_min": -26.67009162902832,
      "activations/layer17_attention_weight_max": 51.1428337097168,
      "activations/layer17_attention_weight_min": -39.85734176635742,
      "activations/layer18_attention_weight_max": 42.36655044555664,
      "activations/layer18_attention_weight_min": -33.27485275268555,
      "activations/layer19_attention_weight_max": 18.95719337463379,
      "activations/layer19_attention_weight_min": -19.408100128173828,
      "activations/layer1_attention_weight_max": 17.420909881591797,
      "activations/layer1_attention_weight_min": -15.361333847045898,
      "activations/layer20_attention_weight_max": 19.79250144958496,
      "activations/layer20_attention_weight_min": -22.349109649658203,
      "activations/layer21_attention_weight_max": 35.11091232299805,
      "activations/layer21_attention_weight_min": -23.201807022094727,
      "activations/layer22_attention_weight_max": 35.82731246948242,
      "activations/layer22_attention_weight_min": -26.510217666625977,
      "activations/layer23_attention_weight_max": 28.043609619140625,
      "activations/layer23_attention_weight_min": -21.077489852905273,
      "activations/layer2_attention_weight_max": 30.430877685546875,
      "activations/layer2_attention_weight_min": -27.250263214111328,
      "activations/layer3_attention_weight_max": 86.00230407714844,
      "activations/layer3_attention_weight_min": -84.9177017211914,
      "activations/layer4_attention_weight_max": 100.275634765625,
      "activations/layer4_attention_weight_min": -91.80937957763672,
      "activations/layer5_attention_weight_max": 97.29900360107422,
      "activations/layer5_attention_weight_min": -89.072021484375,
      "activations/layer6_attention_weight_max": 57.303314208984375,
      "activations/layer6_attention_weight_min": -52.99228286743164,
      "activations/layer7_attention_weight_max": 69.93938446044922,
      "activations/layer7_attention_weight_min": -68.0716781616211,
      "activations/layer8_attention_weight_max": 41.40745544433594,
      "activations/layer8_attention_weight_min": -41.872989654541016,
      "activations/layer9_attention_weight_max": 51.92279052734375,
      "activations/layer9_attention_weight_min": -47.5936393737793,
      "epoch": 5.18,
      "learning_rate": 0.00011776136363636361,
      "loss": 2.8459,
      "step": 89200
    },
    {
      "activations/layer0_attention_weight_max": 13.909725189208984,
      "activations/layer0_attention_weight_min": -15.003551483154297,
      "activations/layer10_attention_weight_max": 37.408782958984375,
      "activations/layer10_attention_weight_min": -32.17515182495117,
      "activations/layer11_attention_weight_max": 29.05689239501953,
      "activations/layer11_attention_weight_min": -28.444496154785156,
      "activations/layer12_attention_weight_max": 19.532405853271484,
      "activations/layer12_attention_weight_min": -25.664627075195312,
      "activations/layer13_attention_weight_max": 53.43684768676758,
      "activations/layer13_attention_weight_min": -36.97022247314453,
      "activations/layer14_attention_weight_max": 41.05124282836914,
      "activations/layer14_attention_weight_min": -28.937471389770508,
      "activations/layer15_attention_weight_max": 41.841732025146484,
      "activations/layer15_attention_weight_min": -30.51970863342285,
      "activations/layer16_attention_weight_max": 34.535064697265625,
      "activations/layer16_attention_weight_min": -25.109006881713867,
      "activations/layer17_attention_weight_max": 53.3613166809082,
      "activations/layer17_attention_weight_min": -37.35322189331055,
      "activations/layer18_attention_weight_max": 48.40099334716797,
      "activations/layer18_attention_weight_min": -33.29848098754883,
      "activations/layer19_attention_weight_max": 19.308454513549805,
      "activations/layer19_attention_weight_min": -21.844680786132812,
      "activations/layer1_attention_weight_max": 17.2544002532959,
      "activations/layer1_attention_weight_min": -15.644100189208984,
      "activations/layer20_attention_weight_max": 22.761667251586914,
      "activations/layer20_attention_weight_min": -20.584768295288086,
      "activations/layer21_attention_weight_max": 38.04920959472656,
      "activations/layer21_attention_weight_min": -24.963321685791016,
      "activations/layer22_attention_weight_max": 36.077999114990234,
      "activations/layer22_attention_weight_min": -23.775205612182617,
      "activations/layer23_attention_weight_max": 32.52190017700195,
      "activations/layer23_attention_weight_min": -18.977861404418945,
      "activations/layer2_attention_weight_max": 30.68648338317871,
      "activations/layer2_attention_weight_min": -29.586299896240234,
      "activations/layer3_attention_weight_max": 81.42306518554688,
      "activations/layer3_attention_weight_min": -85.1496353149414,
      "activations/layer4_attention_weight_max": 101.7878646850586,
      "activations/layer4_attention_weight_min": -94.19754791259766,
      "activations/layer5_attention_weight_max": 102.51155090332031,
      "activations/layer5_attention_weight_min": -97.9515380859375,
      "activations/layer6_attention_weight_max": 59.57337188720703,
      "activations/layer6_attention_weight_min": -56.97554016113281,
      "activations/layer7_attention_weight_max": 71.3407211303711,
      "activations/layer7_attention_weight_min": -70.18721008300781,
      "activations/layer8_attention_weight_max": 41.09815216064453,
      "activations/layer8_attention_weight_min": -40.38608169555664,
      "activations/layer9_attention_weight_max": 48.29458236694336,
      "activations/layer9_attention_weight_min": -45.959938049316406,
      "epoch": 5.19,
      "learning_rate": 0.00011774242424242423,
      "loss": 2.8481,
      "step": 89250
    },
    {
      "activations/layer0_attention_weight_max": 15.04027271270752,
      "activations/layer0_attention_weight_min": -14.733684539794922,
      "activations/layer10_attention_weight_max": 37.761878967285156,
      "activations/layer10_attention_weight_min": -34.326358795166016,
      "activations/layer11_attention_weight_max": 31.42217254638672,
      "activations/layer11_attention_weight_min": -26.612224578857422,
      "activations/layer12_attention_weight_max": 23.69363021850586,
      "activations/layer12_attention_weight_min": -26.235862731933594,
      "activations/layer13_attention_weight_max": 47.20919418334961,
      "activations/layer13_attention_weight_min": -34.93535614013672,
      "activations/layer14_attention_weight_max": 38.33024215698242,
      "activations/layer14_attention_weight_min": -30.587257385253906,
      "activations/layer15_attention_weight_max": 38.45797348022461,
      "activations/layer15_attention_weight_min": -29.85421371459961,
      "activations/layer16_attention_weight_max": 31.566015243530273,
      "activations/layer16_attention_weight_min": -27.021400451660156,
      "activations/layer17_attention_weight_max": 54.849884033203125,
      "activations/layer17_attention_weight_min": -37.83625030517578,
      "activations/layer18_attention_weight_max": 46.19129180908203,
      "activations/layer18_attention_weight_min": -32.31781005859375,
      "activations/layer19_attention_weight_max": 22.238250732421875,
      "activations/layer19_attention_weight_min": -19.16145133972168,
      "activations/layer1_attention_weight_max": 18.199424743652344,
      "activations/layer1_attention_weight_min": -14.927452087402344,
      "activations/layer20_attention_weight_max": 21.100040435791016,
      "activations/layer20_attention_weight_min": -23.66200828552246,
      "activations/layer21_attention_weight_max": 35.724159240722656,
      "activations/layer21_attention_weight_min": -24.812164306640625,
      "activations/layer22_attention_weight_max": 32.67876052856445,
      "activations/layer22_attention_weight_min": -23.321502685546875,
      "activations/layer23_attention_weight_max": 27.966938018798828,
      "activations/layer23_attention_weight_min": -19.258081436157227,
      "activations/layer2_attention_weight_max": 29.656253814697266,
      "activations/layer2_attention_weight_min": -28.20303726196289,
      "activations/layer3_attention_weight_max": 82.81401824951172,
      "activations/layer3_attention_weight_min": -88.57978057861328,
      "activations/layer4_attention_weight_max": 103.57368469238281,
      "activations/layer4_attention_weight_min": -97.4451675415039,
      "activations/layer5_attention_weight_max": 100.8902587890625,
      "activations/layer5_attention_weight_min": -89.01432800292969,
      "activations/layer6_attention_weight_max": 59.46807861328125,
      "activations/layer6_attention_weight_min": -50.36579513549805,
      "activations/layer7_attention_weight_max": 72.43701171875,
      "activations/layer7_attention_weight_min": -67.58721160888672,
      "activations/layer8_attention_weight_max": 44.69951248168945,
      "activations/layer8_attention_weight_min": -40.315452575683594,
      "activations/layer9_attention_weight_max": 59.123016357421875,
      "activations/layer9_attention_weight_min": -49.858890533447266,
      "epoch": 5.19,
      "learning_rate": 0.00011772348484848484,
      "loss": 2.8501,
      "step": 89300
    },
    {
      "activations/layer0_attention_weight_max": 14.35327434539795,
      "activations/layer0_attention_weight_min": -14.309059143066406,
      "activations/layer10_attention_weight_max": 36.82085037231445,
      "activations/layer10_attention_weight_min": -33.72552490234375,
      "activations/layer11_attention_weight_max": 27.464384078979492,
      "activations/layer11_attention_weight_min": -27.008323669433594,
      "activations/layer12_attention_weight_max": 24.722854614257812,
      "activations/layer12_attention_weight_min": -25.372323989868164,
      "activations/layer13_attention_weight_max": 48.16967010498047,
      "activations/layer13_attention_weight_min": -38.29350280761719,
      "activations/layer14_attention_weight_max": 42.68056869506836,
      "activations/layer14_attention_weight_min": -30.066865921020508,
      "activations/layer15_attention_weight_max": 41.1100959777832,
      "activations/layer15_attention_weight_min": -31.959253311157227,
      "activations/layer16_attention_weight_max": 34.761287689208984,
      "activations/layer16_attention_weight_min": -25.32735824584961,
      "activations/layer17_attention_weight_max": 56.30316162109375,
      "activations/layer17_attention_weight_min": -38.67991638183594,
      "activations/layer18_attention_weight_max": 49.79912185668945,
      "activations/layer18_attention_weight_min": -32.67467498779297,
      "activations/layer19_attention_weight_max": 22.458229064941406,
      "activations/layer19_attention_weight_min": -17.801973342895508,
      "activations/layer1_attention_weight_max": 17.215608596801758,
      "activations/layer1_attention_weight_min": -13.77923583984375,
      "activations/layer20_attention_weight_max": 21.073406219482422,
      "activations/layer20_attention_weight_min": -19.644060134887695,
      "activations/layer21_attention_weight_max": 40.427337646484375,
      "activations/layer21_attention_weight_min": -25.314224243164062,
      "activations/layer22_attention_weight_max": 33.84765625,
      "activations/layer22_attention_weight_min": -23.188148498535156,
      "activations/layer23_attention_weight_max": 29.001754760742188,
      "activations/layer23_attention_weight_min": -17.795921325683594,
      "activations/layer2_attention_weight_max": 29.194740295410156,
      "activations/layer2_attention_weight_min": -28.445165634155273,
      "activations/layer3_attention_weight_max": 79.53076934814453,
      "activations/layer3_attention_weight_min": -87.97409057617188,
      "activations/layer4_attention_weight_max": 95.37152862548828,
      "activations/layer4_attention_weight_min": -94.19872283935547,
      "activations/layer5_attention_weight_max": 99.49607849121094,
      "activations/layer5_attention_weight_min": -92.78102111816406,
      "activations/layer6_attention_weight_max": 58.50596237182617,
      "activations/layer6_attention_weight_min": -54.54596710205078,
      "activations/layer7_attention_weight_max": 71.85265350341797,
      "activations/layer7_attention_weight_min": -64.98340606689453,
      "activations/layer8_attention_weight_max": 43.234107971191406,
      "activations/layer8_attention_weight_min": -40.315650939941406,
      "activations/layer9_attention_weight_max": 49.24433135986328,
      "activations/layer9_attention_weight_min": -44.92606735229492,
      "epoch": 5.19,
      "learning_rate": 0.00011770454545454545,
      "loss": 2.8504,
      "step": 89350
    },
    {
      "activations/layer0_attention_weight_max": 15.603882789611816,
      "activations/layer0_attention_weight_min": -14.141937255859375,
      "activations/layer10_attention_weight_max": 37.427310943603516,
      "activations/layer10_attention_weight_min": -33.803016662597656,
      "activations/layer11_attention_weight_max": 31.368324279785156,
      "activations/layer11_attention_weight_min": -26.11288833618164,
      "activations/layer12_attention_weight_max": 19.980016708374023,
      "activations/layer12_attention_weight_min": -25.715564727783203,
      "activations/layer13_attention_weight_max": 45.38832092285156,
      "activations/layer13_attention_weight_min": -35.4865837097168,
      "activations/layer14_attention_weight_max": 38.593441009521484,
      "activations/layer14_attention_weight_min": -32.231834411621094,
      "activations/layer15_attention_weight_max": 39.08098602294922,
      "activations/layer15_attention_weight_min": -29.72414779663086,
      "activations/layer16_attention_weight_max": 36.767024993896484,
      "activations/layer16_attention_weight_min": -25.105693817138672,
      "activations/layer17_attention_weight_max": 57.464599609375,
      "activations/layer17_attention_weight_min": -39.55659866333008,
      "activations/layer18_attention_weight_max": 51.9747428894043,
      "activations/layer18_attention_weight_min": -31.675870895385742,
      "activations/layer19_attention_weight_max": 22.458375930786133,
      "activations/layer19_attention_weight_min": -18.381929397583008,
      "activations/layer1_attention_weight_max": 19.52385902404785,
      "activations/layer1_attention_weight_min": -15.872517585754395,
      "activations/layer20_attention_weight_max": 22.089994430541992,
      "activations/layer20_attention_weight_min": -21.30387306213379,
      "activations/layer21_attention_weight_max": 39.02918243408203,
      "activations/layer21_attention_weight_min": -25.45829963684082,
      "activations/layer22_attention_weight_max": 34.07159423828125,
      "activations/layer22_attention_weight_min": -23.896570205688477,
      "activations/layer23_attention_weight_max": 27.93878936767578,
      "activations/layer23_attention_weight_min": -17.112228393554688,
      "activations/layer2_attention_weight_max": 32.13824462890625,
      "activations/layer2_attention_weight_min": -29.96783447265625,
      "activations/layer3_attention_weight_max": 87.6321029663086,
      "activations/layer3_attention_weight_min": -89.01183319091797,
      "activations/layer4_attention_weight_max": 107.53582763671875,
      "activations/layer4_attention_weight_min": -102.06137084960938,
      "activations/layer5_attention_weight_max": 106.16497802734375,
      "activations/layer5_attention_weight_min": -98.42076873779297,
      "activations/layer6_attention_weight_max": 58.56829833984375,
      "activations/layer6_attention_weight_min": -54.50491714477539,
      "activations/layer7_attention_weight_max": 73.45443725585938,
      "activations/layer7_attention_weight_min": -67.4786376953125,
      "activations/layer8_attention_weight_max": 45.97114181518555,
      "activations/layer8_attention_weight_min": -43.30583190917969,
      "activations/layer9_attention_weight_max": 53.16946029663086,
      "activations/layer9_attention_weight_min": -46.81814956665039,
      "epoch": 5.19,
      "learning_rate": 0.00011768560606060605,
      "loss": 2.8239,
      "step": 89400
    },
    {
      "activations/layer0_attention_weight_max": 15.637638092041016,
      "activations/layer0_attention_weight_min": -14.760250091552734,
      "activations/layer10_attention_weight_max": 41.10747146606445,
      "activations/layer10_attention_weight_min": -35.28171920776367,
      "activations/layer11_attention_weight_max": 30.981826782226562,
      "activations/layer11_attention_weight_min": -28.059362411499023,
      "activations/layer12_attention_weight_max": 28.396814346313477,
      "activations/layer12_attention_weight_min": -24.58463478088379,
      "activations/layer13_attention_weight_max": 56.8673210144043,
      "activations/layer13_attention_weight_min": -37.572998046875,
      "activations/layer14_attention_weight_max": 46.960113525390625,
      "activations/layer14_attention_weight_min": -30.398523330688477,
      "activations/layer15_attention_weight_max": 49.02095031738281,
      "activations/layer15_attention_weight_min": -28.839630126953125,
      "activations/layer16_attention_weight_max": 37.353031158447266,
      "activations/layer16_attention_weight_min": -26.407758712768555,
      "activations/layer17_attention_weight_max": 62.03568649291992,
      "activations/layer17_attention_weight_min": -38.388370513916016,
      "activations/layer18_attention_weight_max": 52.22045135498047,
      "activations/layer18_attention_weight_min": -32.228721618652344,
      "activations/layer19_attention_weight_max": 25.52783966064453,
      "activations/layer19_attention_weight_min": -17.736309051513672,
      "activations/layer1_attention_weight_max": 16.2727108001709,
      "activations/layer1_attention_weight_min": -14.75940990447998,
      "activations/layer20_attention_weight_max": 22.882427215576172,
      "activations/layer20_attention_weight_min": -19.08024024963379,
      "activations/layer21_attention_weight_max": 38.91867446899414,
      "activations/layer21_attention_weight_min": -26.02252769470215,
      "activations/layer22_attention_weight_max": 35.57950973510742,
      "activations/layer22_attention_weight_min": -23.491851806640625,
      "activations/layer23_attention_weight_max": 33.22464370727539,
      "activations/layer23_attention_weight_min": -17.235960006713867,
      "activations/layer2_attention_weight_max": 31.17763900756836,
      "activations/layer2_attention_weight_min": -28.88934326171875,
      "activations/layer3_attention_weight_max": 82.31608581542969,
      "activations/layer3_attention_weight_min": -84.9291000366211,
      "activations/layer4_attention_weight_max": 100.99376678466797,
      "activations/layer4_attention_weight_min": -93.4382095336914,
      "activations/layer5_attention_weight_max": 102.97392272949219,
      "activations/layer5_attention_weight_min": -93.7219467163086,
      "activations/layer6_attention_weight_max": 60.05673599243164,
      "activations/layer6_attention_weight_min": -55.7386474609375,
      "activations/layer7_attention_weight_max": 68.949951171875,
      "activations/layer7_attention_weight_min": -67.01416015625,
      "activations/layer8_attention_weight_max": 42.75117874145508,
      "activations/layer8_attention_weight_min": -42.916202545166016,
      "activations/layer9_attention_weight_max": 49.621986389160156,
      "activations/layer9_attention_weight_min": -49.80331802368164,
      "epoch": 5.2,
      "learning_rate": 0.00011766666666666666,
      "loss": 2.8556,
      "step": 89450
    },
    {
      "activations/layer0_attention_weight_max": 14.227537155151367,
      "activations/layer0_attention_weight_min": -14.588981628417969,
      "activations/layer10_attention_weight_max": 36.088111877441406,
      "activations/layer10_attention_weight_min": -35.91640090942383,
      "activations/layer11_attention_weight_max": 29.123035430908203,
      "activations/layer11_attention_weight_min": -26.764013290405273,
      "activations/layer12_attention_weight_max": 24.988439559936523,
      "activations/layer12_attention_weight_min": -23.5269775390625,
      "activations/layer13_attention_weight_max": 46.69269943237305,
      "activations/layer13_attention_weight_min": -34.213226318359375,
      "activations/layer14_attention_weight_max": 38.33964157104492,
      "activations/layer14_attention_weight_min": -28.1610050201416,
      "activations/layer15_attention_weight_max": 45.796302795410156,
      "activations/layer15_attention_weight_min": -28.623008728027344,
      "activations/layer16_attention_weight_max": 31.274646759033203,
      "activations/layer16_attention_weight_min": -26.951026916503906,
      "activations/layer17_attention_weight_max": 55.39546585083008,
      "activations/layer17_attention_weight_min": -39.33121109008789,
      "activations/layer18_attention_weight_max": 47.93710708618164,
      "activations/layer18_attention_weight_min": -32.97822952270508,
      "activations/layer19_attention_weight_max": 22.261619567871094,
      "activations/layer19_attention_weight_min": -17.451295852661133,
      "activations/layer1_attention_weight_max": 17.218263626098633,
      "activations/layer1_attention_weight_min": -12.174187660217285,
      "activations/layer20_attention_weight_max": 19.345794677734375,
      "activations/layer20_attention_weight_min": -21.80698013305664,
      "activations/layer21_attention_weight_max": 34.90144348144531,
      "activations/layer21_attention_weight_min": -24.671310424804688,
      "activations/layer22_attention_weight_max": 32.80949401855469,
      "activations/layer22_attention_weight_min": -23.205841064453125,
      "activations/layer23_attention_weight_max": 28.649768829345703,
      "activations/layer23_attention_weight_min": -18.96358871459961,
      "activations/layer2_attention_weight_max": 26.562414169311523,
      "activations/layer2_attention_weight_min": -26.162460327148438,
      "activations/layer3_attention_weight_max": 73.19743347167969,
      "activations/layer3_attention_weight_min": -83.1364517211914,
      "activations/layer4_attention_weight_max": 93.6758041381836,
      "activations/layer4_attention_weight_min": -90.80349731445312,
      "activations/layer5_attention_weight_max": 98.42625427246094,
      "activations/layer5_attention_weight_min": -86.96385192871094,
      "activations/layer6_attention_weight_max": 54.911598205566406,
      "activations/layer6_attention_weight_min": -51.10673522949219,
      "activations/layer7_attention_weight_max": 67.52316284179688,
      "activations/layer7_attention_weight_min": -63.66571807861328,
      "activations/layer8_attention_weight_max": 41.12165069580078,
      "activations/layer8_attention_weight_min": -38.55811309814453,
      "activations/layer9_attention_weight_max": 47.9232292175293,
      "activations/layer9_attention_weight_min": -52.15863037109375,
      "epoch": 5.2,
      "learning_rate": 0.00011764772727272726,
      "loss": 2.8558,
      "step": 89500
    },
    {
      "activations/layer0_attention_weight_max": 14.346299171447754,
      "activations/layer0_attention_weight_min": -15.079706192016602,
      "activations/layer10_attention_weight_max": 36.754730224609375,
      "activations/layer10_attention_weight_min": -32.60498046875,
      "activations/layer11_attention_weight_max": 29.191932678222656,
      "activations/layer11_attention_weight_min": -26.835805892944336,
      "activations/layer12_attention_weight_max": 24.64096450805664,
      "activations/layer12_attention_weight_min": -21.257856369018555,
      "activations/layer13_attention_weight_max": 51.67585372924805,
      "activations/layer13_attention_weight_min": -37.20589065551758,
      "activations/layer14_attention_weight_max": 45.556461334228516,
      "activations/layer14_attention_weight_min": -29.163209915161133,
      "activations/layer15_attention_weight_max": 44.231964111328125,
      "activations/layer15_attention_weight_min": -27.871583938598633,
      "activations/layer16_attention_weight_max": 37.90796661376953,
      "activations/layer16_attention_weight_min": -26.398889541625977,
      "activations/layer17_attention_weight_max": 55.19296646118164,
      "activations/layer17_attention_weight_min": -38.89247131347656,
      "activations/layer18_attention_weight_max": 49.61886978149414,
      "activations/layer18_attention_weight_min": -31.256858825683594,
      "activations/layer19_attention_weight_max": 21.67254066467285,
      "activations/layer19_attention_weight_min": -18.48732566833496,
      "activations/layer1_attention_weight_max": 16.886459350585938,
      "activations/layer1_attention_weight_min": -14.932157516479492,
      "activations/layer20_attention_weight_max": 21.875791549682617,
      "activations/layer20_attention_weight_min": -17.586807250976562,
      "activations/layer21_attention_weight_max": 38.951416015625,
      "activations/layer21_attention_weight_min": -25.219646453857422,
      "activations/layer22_attention_weight_max": 33.463043212890625,
      "activations/layer22_attention_weight_min": -24.91105079650879,
      "activations/layer23_attention_weight_max": 30.417797088623047,
      "activations/layer23_attention_weight_min": -17.01329231262207,
      "activations/layer2_attention_weight_max": 29.942716598510742,
      "activations/layer2_attention_weight_min": -28.458209991455078,
      "activations/layer3_attention_weight_max": 77.6506118774414,
      "activations/layer3_attention_weight_min": -83.16153717041016,
      "activations/layer4_attention_weight_max": 98.51583099365234,
      "activations/layer4_attention_weight_min": -92.23799896240234,
      "activations/layer5_attention_weight_max": 97.54296875,
      "activations/layer5_attention_weight_min": -90.81510925292969,
      "activations/layer6_attention_weight_max": 55.40211868286133,
      "activations/layer6_attention_weight_min": -53.21147918701172,
      "activations/layer7_attention_weight_max": 70.69270324707031,
      "activations/layer7_attention_weight_min": -68.20259094238281,
      "activations/layer8_attention_weight_max": 42.07963562011719,
      "activations/layer8_attention_weight_min": -41.05359649658203,
      "activations/layer9_attention_weight_max": 46.507080078125,
      "activations/layer9_attention_weight_min": -44.54217529296875,
      "epoch": 5.2,
      "learning_rate": 0.00011762878787878787,
      "loss": 2.8472,
      "step": 89550
    },
    {
      "activations/layer0_attention_weight_max": 14.426522254943848,
      "activations/layer0_attention_weight_min": -15.363404273986816,
      "activations/layer10_attention_weight_max": 42.69502639770508,
      "activations/layer10_attention_weight_min": -36.767364501953125,
      "activations/layer11_attention_weight_max": 31.213085174560547,
      "activations/layer11_attention_weight_min": -28.51866912841797,
      "activations/layer12_attention_weight_max": 26.726469039916992,
      "activations/layer12_attention_weight_min": -25.295480728149414,
      "activations/layer13_attention_weight_max": 44.13508224487305,
      "activations/layer13_attention_weight_min": -33.333778381347656,
      "activations/layer14_attention_weight_max": 45.10130310058594,
      "activations/layer14_attention_weight_min": -29.408863067626953,
      "activations/layer15_attention_weight_max": 43.470977783203125,
      "activations/layer15_attention_weight_min": -29.095455169677734,
      "activations/layer16_attention_weight_max": 35.510318756103516,
      "activations/layer16_attention_weight_min": -26.162124633789062,
      "activations/layer17_attention_weight_max": 63.3033332824707,
      "activations/layer17_attention_weight_min": -40.20528793334961,
      "activations/layer18_attention_weight_max": 53.98605728149414,
      "activations/layer18_attention_weight_min": -34.50337600708008,
      "activations/layer19_attention_weight_max": 21.74066734313965,
      "activations/layer19_attention_weight_min": -19.380979537963867,
      "activations/layer1_attention_weight_max": 17.273603439331055,
      "activations/layer1_attention_weight_min": -15.653541564941406,
      "activations/layer20_attention_weight_max": 23.397554397583008,
      "activations/layer20_attention_weight_min": -20.101606369018555,
      "activations/layer21_attention_weight_max": 46.651451110839844,
      "activations/layer21_attention_weight_min": -29.068735122680664,
      "activations/layer22_attention_weight_max": 34.07609558105469,
      "activations/layer22_attention_weight_min": -22.27534294128418,
      "activations/layer23_attention_weight_max": 32.38824462890625,
      "activations/layer23_attention_weight_min": -17.687660217285156,
      "activations/layer2_attention_weight_max": 29.642562866210938,
      "activations/layer2_attention_weight_min": -28.409984588623047,
      "activations/layer3_attention_weight_max": 82.90272521972656,
      "activations/layer3_attention_weight_min": -89.18751525878906,
      "activations/layer4_attention_weight_max": 105.55690002441406,
      "activations/layer4_attention_weight_min": -103.22029876708984,
      "activations/layer5_attention_weight_max": 104.02734375,
      "activations/layer5_attention_weight_min": -99.3434066772461,
      "activations/layer6_attention_weight_max": 60.198211669921875,
      "activations/layer6_attention_weight_min": -56.17286682128906,
      "activations/layer7_attention_weight_max": 74.26643371582031,
      "activations/layer7_attention_weight_min": -69.12769317626953,
      "activations/layer8_attention_weight_max": 44.55955505371094,
      "activations/layer8_attention_weight_min": -41.753936767578125,
      "activations/layer9_attention_weight_max": 51.088443756103516,
      "activations/layer9_attention_weight_min": -46.81553268432617,
      "epoch": 5.21,
      "learning_rate": 0.00011760984848484847,
      "loss": 2.8439,
      "step": 89600
    },
    {
      "activations/layer0_attention_weight_max": 14.173297882080078,
      "activations/layer0_attention_weight_min": -14.804069519042969,
      "activations/layer10_attention_weight_max": 39.006046295166016,
      "activations/layer10_attention_weight_min": -32.851341247558594,
      "activations/layer11_attention_weight_max": 29.993946075439453,
      "activations/layer11_attention_weight_min": -27.234960556030273,
      "activations/layer12_attention_weight_max": 25.951190948486328,
      "activations/layer12_attention_weight_min": -27.516202926635742,
      "activations/layer13_attention_weight_max": 54.593238830566406,
      "activations/layer13_attention_weight_min": -41.859947204589844,
      "activations/layer14_attention_weight_max": 52.12755584716797,
      "activations/layer14_attention_weight_min": -35.18655776977539,
      "activations/layer15_attention_weight_max": 40.1585807800293,
      "activations/layer15_attention_weight_min": -28.61798858642578,
      "activations/layer16_attention_weight_max": 35.16796112060547,
      "activations/layer16_attention_weight_min": -25.92270851135254,
      "activations/layer17_attention_weight_max": 54.644657135009766,
      "activations/layer17_attention_weight_min": -40.3091926574707,
      "activations/layer18_attention_weight_max": 46.51308059692383,
      "activations/layer18_attention_weight_min": -33.72616958618164,
      "activations/layer19_attention_weight_max": 23.138574600219727,
      "activations/layer19_attention_weight_min": -20.15253257751465,
      "activations/layer1_attention_weight_max": 17.609527587890625,
      "activations/layer1_attention_weight_min": -15.021127700805664,
      "activations/layer20_attention_weight_max": 22.328819274902344,
      "activations/layer20_attention_weight_min": -20.870079040527344,
      "activations/layer21_attention_weight_max": 40.43213653564453,
      "activations/layer21_attention_weight_min": -25.53883934020996,
      "activations/layer22_attention_weight_max": 33.068790435791016,
      "activations/layer22_attention_weight_min": -22.878896713256836,
      "activations/layer23_attention_weight_max": 28.693843841552734,
      "activations/layer23_attention_weight_min": -17.998170852661133,
      "activations/layer2_attention_weight_max": 30.444753646850586,
      "activations/layer2_attention_weight_min": -28.728595733642578,
      "activations/layer3_attention_weight_max": 82.95170593261719,
      "activations/layer3_attention_weight_min": -85.69246673583984,
      "activations/layer4_attention_weight_max": 98.5741958618164,
      "activations/layer4_attention_weight_min": -94.53018951416016,
      "activations/layer5_attention_weight_max": 97.27877044677734,
      "activations/layer5_attention_weight_min": -88.1046371459961,
      "activations/layer6_attention_weight_max": 54.993778228759766,
      "activations/layer6_attention_weight_min": -50.28110885620117,
      "activations/layer7_attention_weight_max": 67.03255462646484,
      "activations/layer7_attention_weight_min": -61.46052932739258,
      "activations/layer8_attention_weight_max": 40.01560974121094,
      "activations/layer8_attention_weight_min": -41.135406494140625,
      "activations/layer9_attention_weight_max": 50.2740592956543,
      "activations/layer9_attention_weight_min": -43.78614807128906,
      "epoch": 5.21,
      "learning_rate": 0.00011759090909090908,
      "loss": 2.8494,
      "step": 89650
    },
    {
      "activations/layer0_attention_weight_max": 15.106338500976562,
      "activations/layer0_attention_weight_min": -15.09696102142334,
      "activations/layer10_attention_weight_max": 38.33989715576172,
      "activations/layer10_attention_weight_min": -34.303287506103516,
      "activations/layer11_attention_weight_max": 30.50519371032715,
      "activations/layer11_attention_weight_min": -30.3046932220459,
      "activations/layer12_attention_weight_max": 20.52328109741211,
      "activations/layer12_attention_weight_min": -23.14261817932129,
      "activations/layer13_attention_weight_max": 41.595516204833984,
      "activations/layer13_attention_weight_min": -32.58607482910156,
      "activations/layer14_attention_weight_max": 36.245323181152344,
      "activations/layer14_attention_weight_min": -29.837535858154297,
      "activations/layer15_attention_weight_max": 46.02115249633789,
      "activations/layer15_attention_weight_min": -31.546371459960938,
      "activations/layer16_attention_weight_max": 33.80546188354492,
      "activations/layer16_attention_weight_min": -28.0450439453125,
      "activations/layer17_attention_weight_max": 53.79384231567383,
      "activations/layer17_attention_weight_min": -41.61069869995117,
      "activations/layer18_attention_weight_max": 42.707637786865234,
      "activations/layer18_attention_weight_min": -33.6181755065918,
      "activations/layer19_attention_weight_max": 28.26926612854004,
      "activations/layer19_attention_weight_min": -22.01360511779785,
      "activations/layer1_attention_weight_max": 17.000999450683594,
      "activations/layer1_attention_weight_min": -14.509712219238281,
      "activations/layer20_attention_weight_max": 23.113014221191406,
      "activations/layer20_attention_weight_min": -23.62250328063965,
      "activations/layer21_attention_weight_max": 42.12665557861328,
      "activations/layer21_attention_weight_min": -27.53855323791504,
      "activations/layer22_attention_weight_max": 32.39186096191406,
      "activations/layer22_attention_weight_min": -27.320005416870117,
      "activations/layer23_attention_weight_max": 28.23876190185547,
      "activations/layer23_attention_weight_min": -19.133188247680664,
      "activations/layer2_attention_weight_max": 29.80047607421875,
      "activations/layer2_attention_weight_min": -29.376399993896484,
      "activations/layer3_attention_weight_max": 83.0135498046875,
      "activations/layer3_attention_weight_min": -81.92352294921875,
      "activations/layer4_attention_weight_max": 101.17022705078125,
      "activations/layer4_attention_weight_min": -92.36048889160156,
      "activations/layer5_attention_weight_max": 99.62545776367188,
      "activations/layer5_attention_weight_min": -95.11610412597656,
      "activations/layer6_attention_weight_max": 58.283756256103516,
      "activations/layer6_attention_weight_min": -52.523170471191406,
      "activations/layer7_attention_weight_max": 72.88813781738281,
      "activations/layer7_attention_weight_min": -67.3538589477539,
      "activations/layer8_attention_weight_max": 43.51649856567383,
      "activations/layer8_attention_weight_min": -41.529930114746094,
      "activations/layer9_attention_weight_max": 52.76325225830078,
      "activations/layer9_attention_weight_min": -50.645233154296875,
      "epoch": 5.21,
      "learning_rate": 0.0001175719696969697,
      "loss": 2.8506,
      "step": 89700
    },
    {
      "activations/layer0_attention_weight_max": 14.222358703613281,
      "activations/layer0_attention_weight_min": -14.812545776367188,
      "activations/layer10_attention_weight_max": 37.62004470825195,
      "activations/layer10_attention_weight_min": -33.37758255004883,
      "activations/layer11_attention_weight_max": 30.426780700683594,
      "activations/layer11_attention_weight_min": -27.62848663330078,
      "activations/layer12_attention_weight_max": 20.198068618774414,
      "activations/layer12_attention_weight_min": -23.795507431030273,
      "activations/layer13_attention_weight_max": 44.408260345458984,
      "activations/layer13_attention_weight_min": -33.23224639892578,
      "activations/layer14_attention_weight_max": 50.237850189208984,
      "activations/layer14_attention_weight_min": -30.988859176635742,
      "activations/layer15_attention_weight_max": 41.180294036865234,
      "activations/layer15_attention_weight_min": -29.821796417236328,
      "activations/layer16_attention_weight_max": 35.66114807128906,
      "activations/layer16_attention_weight_min": -28.81516456604004,
      "activations/layer17_attention_weight_max": 55.55752182006836,
      "activations/layer17_attention_weight_min": -38.16294479370117,
      "activations/layer18_attention_weight_max": 47.09456253051758,
      "activations/layer18_attention_weight_min": -33.88877868652344,
      "activations/layer19_attention_weight_max": 21.366657257080078,
      "activations/layer19_attention_weight_min": -19.64694595336914,
      "activations/layer1_attention_weight_max": 17.459535598754883,
      "activations/layer1_attention_weight_min": -13.467700958251953,
      "activations/layer20_attention_weight_max": 22.02322769165039,
      "activations/layer20_attention_weight_min": -20.318382263183594,
      "activations/layer21_attention_weight_max": 37.67586135864258,
      "activations/layer21_attention_weight_min": -24.501279830932617,
      "activations/layer22_attention_weight_max": 32.84241485595703,
      "activations/layer22_attention_weight_min": -25.138399124145508,
      "activations/layer23_attention_weight_max": 29.379051208496094,
      "activations/layer23_attention_weight_min": -18.466564178466797,
      "activations/layer2_attention_weight_max": 30.53481101989746,
      "activations/layer2_attention_weight_min": -28.376163482666016,
      "activations/layer3_attention_weight_max": 86.53105163574219,
      "activations/layer3_attention_weight_min": -87.82945251464844,
      "activations/layer4_attention_weight_max": 107.45446014404297,
      "activations/layer4_attention_weight_min": -94.8434066772461,
      "activations/layer5_attention_weight_max": 101.19175720214844,
      "activations/layer5_attention_weight_min": -91.2735595703125,
      "activations/layer6_attention_weight_max": 56.85188293457031,
      "activations/layer6_attention_weight_min": -51.15752410888672,
      "activations/layer7_attention_weight_max": 71.44147491455078,
      "activations/layer7_attention_weight_min": -64.71405792236328,
      "activations/layer8_attention_weight_max": 44.7419319152832,
      "activations/layer8_attention_weight_min": -43.29007339477539,
      "activations/layer9_attention_weight_max": 48.34475326538086,
      "activations/layer9_attention_weight_min": -45.93631362915039,
      "epoch": 5.21,
      "learning_rate": 0.00011755303030303029,
      "loss": 2.8583,
      "step": 89750
    },
    {
      "activations/layer0_attention_weight_max": 14.459378242492676,
      "activations/layer0_attention_weight_min": -14.486915588378906,
      "activations/layer10_attention_weight_max": 49.13494873046875,
      "activations/layer10_attention_weight_min": -42.67619705200195,
      "activations/layer11_attention_weight_max": 42.224151611328125,
      "activations/layer11_attention_weight_min": -31.15618133544922,
      "activations/layer12_attention_weight_max": 23.897201538085938,
      "activations/layer12_attention_weight_min": -23.164466857910156,
      "activations/layer13_attention_weight_max": 74.07725524902344,
      "activations/layer13_attention_weight_min": -34.70142364501953,
      "activations/layer14_attention_weight_max": 79.41412353515625,
      "activations/layer14_attention_weight_min": -38.04286575317383,
      "activations/layer15_attention_weight_max": 55.69850158691406,
      "activations/layer15_attention_weight_min": -29.015560150146484,
      "activations/layer16_attention_weight_max": 41.53907012939453,
      "activations/layer16_attention_weight_min": -29.686403274536133,
      "activations/layer17_attention_weight_max": 65.6100845336914,
      "activations/layer17_attention_weight_min": -42.682525634765625,
      "activations/layer18_attention_weight_max": 50.49788284301758,
      "activations/layer18_attention_weight_min": -33.3540153503418,
      "activations/layer19_attention_weight_max": 23.756105422973633,
      "activations/layer19_attention_weight_min": -19.643428802490234,
      "activations/layer1_attention_weight_max": 18.835859298706055,
      "activations/layer1_attention_weight_min": -19.072795867919922,
      "activations/layer20_attention_weight_max": 22.985782623291016,
      "activations/layer20_attention_weight_min": -24.234973907470703,
      "activations/layer21_attention_weight_max": 40.6138916015625,
      "activations/layer21_attention_weight_min": -25.1387939453125,
      "activations/layer22_attention_weight_max": 35.61674118041992,
      "activations/layer22_attention_weight_min": -22.15317153930664,
      "activations/layer23_attention_weight_max": 26.167598724365234,
      "activations/layer23_attention_weight_min": -16.18343734741211,
      "activations/layer2_attention_weight_max": 31.98285675048828,
      "activations/layer2_attention_weight_min": -28.703763961791992,
      "activations/layer3_attention_weight_max": 81.75885772705078,
      "activations/layer3_attention_weight_min": -83.44987487792969,
      "activations/layer4_attention_weight_max": 102.5182876586914,
      "activations/layer4_attention_weight_min": -98.91252899169922,
      "activations/layer5_attention_weight_max": 107.04429626464844,
      "activations/layer5_attention_weight_min": -96.92134094238281,
      "activations/layer6_attention_weight_max": 63.17699432373047,
      "activations/layer6_attention_weight_min": -56.90727615356445,
      "activations/layer7_attention_weight_max": 90.39146423339844,
      "activations/layer7_attention_weight_min": -71.22099304199219,
      "activations/layer8_attention_weight_max": 57.33389663696289,
      "activations/layer8_attention_weight_min": -49.31207275390625,
      "activations/layer9_attention_weight_max": 79.53330993652344,
      "activations/layer9_attention_weight_min": -54.716552734375,
      "epoch": 5.22,
      "learning_rate": 0.0001175340909090909,
      "loss": 2.8373,
      "step": 89800
    },
    {
      "activations/layer0_attention_weight_max": 14.936433792114258,
      "activations/layer0_attention_weight_min": -14.862236976623535,
      "activations/layer10_attention_weight_max": 37.354347229003906,
      "activations/layer10_attention_weight_min": -35.66178512573242,
      "activations/layer11_attention_weight_max": 28.50546646118164,
      "activations/layer11_attention_weight_min": -28.194580078125,
      "activations/layer12_attention_weight_max": 20.05203628540039,
      "activations/layer12_attention_weight_min": -24.999897003173828,
      "activations/layer13_attention_weight_max": 45.93159866333008,
      "activations/layer13_attention_weight_min": -34.369808197021484,
      "activations/layer14_attention_weight_max": 34.58759689331055,
      "activations/layer14_attention_weight_min": -30.165830612182617,
      "activations/layer15_attention_weight_max": 34.928070068359375,
      "activations/layer15_attention_weight_min": -30.725069046020508,
      "activations/layer16_attention_weight_max": 33.553314208984375,
      "activations/layer16_attention_weight_min": -27.55152130126953,
      "activations/layer17_attention_weight_max": 53.8159065246582,
      "activations/layer17_attention_weight_min": -38.09836196899414,
      "activations/layer18_attention_weight_max": 46.583824157714844,
      "activations/layer18_attention_weight_min": -34.03767013549805,
      "activations/layer19_attention_weight_max": 20.636947631835938,
      "activations/layer19_attention_weight_min": -22.576108932495117,
      "activations/layer1_attention_weight_max": 17.40159797668457,
      "activations/layer1_attention_weight_min": -16.77250099182129,
      "activations/layer20_attention_weight_max": 18.967918395996094,
      "activations/layer20_attention_weight_min": -20.871667861938477,
      "activations/layer21_attention_weight_max": 33.92293167114258,
      "activations/layer21_attention_weight_min": -24.095651626586914,
      "activations/layer22_attention_weight_max": 33.88062286376953,
      "activations/layer22_attention_weight_min": -22.654653549194336,
      "activations/layer23_attention_weight_max": 25.568593978881836,
      "activations/layer23_attention_weight_min": -18.826358795166016,
      "activations/layer2_attention_weight_max": 30.21048355102539,
      "activations/layer2_attention_weight_min": -28.668739318847656,
      "activations/layer3_attention_weight_max": 82.72997283935547,
      "activations/layer3_attention_weight_min": -84.79419708251953,
      "activations/layer4_attention_weight_max": 106.45149993896484,
      "activations/layer4_attention_weight_min": -96.27945709228516,
      "activations/layer5_attention_weight_max": 102.83629608154297,
      "activations/layer5_attention_weight_min": -90.59917449951172,
      "activations/layer6_attention_weight_max": 57.00847244262695,
      "activations/layer6_attention_weight_min": -49.88334655761719,
      "activations/layer7_attention_weight_max": 67.57376861572266,
      "activations/layer7_attention_weight_min": -63.32998275756836,
      "activations/layer8_attention_weight_max": 42.76618576049805,
      "activations/layer8_attention_weight_min": -41.14298629760742,
      "activations/layer9_attention_weight_max": 53.488525390625,
      "activations/layer9_attention_weight_min": -52.567684173583984,
      "epoch": 5.22,
      "learning_rate": 0.00011751515151515152,
      "loss": 2.8476,
      "step": 89850
    },
    {
      "activations/layer0_attention_weight_max": 14.408163070678711,
      "activations/layer0_attention_weight_min": -14.762199401855469,
      "activations/layer10_attention_weight_max": 39.12660598754883,
      "activations/layer10_attention_weight_min": -34.864723205566406,
      "activations/layer11_attention_weight_max": 30.40573501586914,
      "activations/layer11_attention_weight_min": -27.560400009155273,
      "activations/layer12_attention_weight_max": 21.61589241027832,
      "activations/layer12_attention_weight_min": -23.38126564025879,
      "activations/layer13_attention_weight_max": 48.8220329284668,
      "activations/layer13_attention_weight_min": -30.42981719970703,
      "activations/layer14_attention_weight_max": 48.58038330078125,
      "activations/layer14_attention_weight_min": -29.644758224487305,
      "activations/layer15_attention_weight_max": 44.36781311035156,
      "activations/layer15_attention_weight_min": -32.417945861816406,
      "activations/layer16_attention_weight_max": 36.65938186645508,
      "activations/layer16_attention_weight_min": -27.15386390686035,
      "activations/layer17_attention_weight_max": 60.50768280029297,
      "activations/layer17_attention_weight_min": -40.742549896240234,
      "activations/layer18_attention_weight_max": 52.10643005371094,
      "activations/layer18_attention_weight_min": -35.04506301879883,
      "activations/layer19_attention_weight_max": 22.24787712097168,
      "activations/layer19_attention_weight_min": -19.096200942993164,
      "activations/layer1_attention_weight_max": 17.128082275390625,
      "activations/layer1_attention_weight_min": -14.787751197814941,
      "activations/layer20_attention_weight_max": 22.996986389160156,
      "activations/layer20_attention_weight_min": -22.582792282104492,
      "activations/layer21_attention_weight_max": 40.39664077758789,
      "activations/layer21_attention_weight_min": -24.985546112060547,
      "activations/layer22_attention_weight_max": 35.407630920410156,
      "activations/layer22_attention_weight_min": -24.208555221557617,
      "activations/layer23_attention_weight_max": 29.647031784057617,
      "activations/layer23_attention_weight_min": -20.588891983032227,
      "activations/layer2_attention_weight_max": 30.02021026611328,
      "activations/layer2_attention_weight_min": -28.345394134521484,
      "activations/layer3_attention_weight_max": 81.22413635253906,
      "activations/layer3_attention_weight_min": -81.25692749023438,
      "activations/layer4_attention_weight_max": 98.87903594970703,
      "activations/layer4_attention_weight_min": -96.7921142578125,
      "activations/layer5_attention_weight_max": 100.52197265625,
      "activations/layer5_attention_weight_min": -97.23900604248047,
      "activations/layer6_attention_weight_max": 55.747657775878906,
      "activations/layer6_attention_weight_min": -52.31230926513672,
      "activations/layer7_attention_weight_max": 69.71419525146484,
      "activations/layer7_attention_weight_min": -64.68798828125,
      "activations/layer8_attention_weight_max": 41.323272705078125,
      "activations/layer8_attention_weight_min": -42.0659294128418,
      "activations/layer9_attention_weight_max": 49.47282028198242,
      "activations/layer9_attention_weight_min": -46.604461669921875,
      "epoch": 5.22,
      "learning_rate": 0.0001174962121212121,
      "loss": 2.8443,
      "step": 89900
    },
    {
      "activations/layer0_attention_weight_max": 14.278238296508789,
      "activations/layer0_attention_weight_min": -14.3855562210083,
      "activations/layer10_attention_weight_max": 38.74668502807617,
      "activations/layer10_attention_weight_min": -35.645347595214844,
      "activations/layer11_attention_weight_max": 31.041250228881836,
      "activations/layer11_attention_weight_min": -27.16438865661621,
      "activations/layer12_attention_weight_max": 23.242572784423828,
      "activations/layer12_attention_weight_min": -22.864906311035156,
      "activations/layer13_attention_weight_max": 43.75840377807617,
      "activations/layer13_attention_weight_min": -35.01264953613281,
      "activations/layer14_attention_weight_max": 48.195465087890625,
      "activations/layer14_attention_weight_min": -30.61849594116211,
      "activations/layer15_attention_weight_max": 43.757476806640625,
      "activations/layer15_attention_weight_min": -30.653060913085938,
      "activations/layer16_attention_weight_max": 35.76956558227539,
      "activations/layer16_attention_weight_min": -25.733858108520508,
      "activations/layer17_attention_weight_max": 62.76247787475586,
      "activations/layer17_attention_weight_min": -39.67354202270508,
      "activations/layer18_attention_weight_max": 49.34625244140625,
      "activations/layer18_attention_weight_min": -35.15996170043945,
      "activations/layer19_attention_weight_max": 25.6068172454834,
      "activations/layer19_attention_weight_min": -19.723365783691406,
      "activations/layer1_attention_weight_max": 16.80339241027832,
      "activations/layer1_attention_weight_min": -14.772896766662598,
      "activations/layer20_attention_weight_max": 24.850940704345703,
      "activations/layer20_attention_weight_min": -18.606935501098633,
      "activations/layer21_attention_weight_max": 42.3223762512207,
      "activations/layer21_attention_weight_min": -26.679128646850586,
      "activations/layer22_attention_weight_max": 33.08060073852539,
      "activations/layer22_attention_weight_min": -24.54935073852539,
      "activations/layer23_attention_weight_max": 31.235036849975586,
      "activations/layer23_attention_weight_min": -20.44110107421875,
      "activations/layer2_attention_weight_max": 29.121219635009766,
      "activations/layer2_attention_weight_min": -28.841609954833984,
      "activations/layer3_attention_weight_max": 79.79417419433594,
      "activations/layer3_attention_weight_min": -86.41680145263672,
      "activations/layer4_attention_weight_max": 102.87086486816406,
      "activations/layer4_attention_weight_min": -96.9215316772461,
      "activations/layer5_attention_weight_max": 101.66615295410156,
      "activations/layer5_attention_weight_min": -91.5403060913086,
      "activations/layer6_attention_weight_max": 59.349334716796875,
      "activations/layer6_attention_weight_min": -54.3367805480957,
      "activations/layer7_attention_weight_max": 68.57709503173828,
      "activations/layer7_attention_weight_min": -64.3294906616211,
      "activations/layer8_attention_weight_max": 45.52688980102539,
      "activations/layer8_attention_weight_min": -43.16865158081055,
      "activations/layer9_attention_weight_max": 53.13697052001953,
      "activations/layer9_attention_weight_min": -46.2869873046875,
      "epoch": 5.23,
      "learning_rate": 0.00011747727272727272,
      "loss": 2.8663,
      "step": 89950
    },
    {
      "activations/layer0_attention_weight_max": 14.694503784179688,
      "activations/layer0_attention_weight_min": -14.998612403869629,
      "activations/layer10_attention_weight_max": 48.9110107421875,
      "activations/layer10_attention_weight_min": -41.819435119628906,
      "activations/layer11_attention_weight_max": 41.25395965576172,
      "activations/layer11_attention_weight_min": -30.8675594329834,
      "activations/layer12_attention_weight_max": 22.40678596496582,
      "activations/layer12_attention_weight_min": -25.550968170166016,
      "activations/layer13_attention_weight_max": 71.97415161132812,
      "activations/layer13_attention_weight_min": -42.962364196777344,
      "activations/layer14_attention_weight_max": 75.39508819580078,
      "activations/layer14_attention_weight_min": -46.6279296875,
      "activations/layer15_attention_weight_max": 49.656272888183594,
      "activations/layer15_attention_weight_min": -31.089820861816406,
      "activations/layer16_attention_weight_max": 35.5118522644043,
      "activations/layer16_attention_weight_min": -25.63185691833496,
      "activations/layer17_attention_weight_max": 60.58584976196289,
      "activations/layer17_attention_weight_min": -37.410919189453125,
      "activations/layer18_attention_weight_max": 48.341896057128906,
      "activations/layer18_attention_weight_min": -33.44281005859375,
      "activations/layer19_attention_weight_max": 22.440969467163086,
      "activations/layer19_attention_weight_min": -19.400968551635742,
      "activations/layer1_attention_weight_max": 17.020278930664062,
      "activations/layer1_attention_weight_min": -16.019052505493164,
      "activations/layer20_attention_weight_max": 20.95013427734375,
      "activations/layer20_attention_weight_min": -17.961898803710938,
      "activations/layer21_attention_weight_max": 34.60149002075195,
      "activations/layer21_attention_weight_min": -25.38223648071289,
      "activations/layer22_attention_weight_max": 32.20646286010742,
      "activations/layer22_attention_weight_min": -25.217018127441406,
      "activations/layer23_attention_weight_max": 28.430707931518555,
      "activations/layer23_attention_weight_min": -16.9505558013916,
      "activations/layer2_attention_weight_max": 31.484554290771484,
      "activations/layer2_attention_weight_min": -28.555660247802734,
      "activations/layer3_attention_weight_max": 83.54743957519531,
      "activations/layer3_attention_weight_min": -87.8460693359375,
      "activations/layer4_attention_weight_max": 112.5910873413086,
      "activations/layer4_attention_weight_min": -110.17559051513672,
      "activations/layer5_attention_weight_max": 115.584228515625,
      "activations/layer5_attention_weight_min": -105.62474822998047,
      "activations/layer6_attention_weight_max": 62.03554153442383,
      "activations/layer6_attention_weight_min": -63.087745666503906,
      "activations/layer7_attention_weight_max": 91.26472473144531,
      "activations/layer7_attention_weight_min": -71.25367736816406,
      "activations/layer8_attention_weight_max": 57.7830810546875,
      "activations/layer8_attention_weight_min": -46.24195098876953,
      "activations/layer9_attention_weight_max": 76.11385345458984,
      "activations/layer9_attention_weight_min": -54.57826614379883,
      "epoch": 5.23,
      "learning_rate": 0.00011745833333333332,
      "loss": 2.8484,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.5456,
      "eval_samples_per_second": 502.482,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.5456,
      "eval_openwebtext_samples_per_second": 502.482,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_wikitext_loss": 3.029296875,
      "eval_wikitext_ppl": 20.68268496272443,
      "eval_wikitext_runtime": 2.0437,
      "eval_wikitext_samples_per_second": 223.125,
      "step": 90000
    },
    {
      "epoch": 5.23,
      "eval_lambada_loss": 2.775390625,
      "eval_lambada_ppl": 16.04489330813532,
      "eval_lambada_runtime": 9.63,
      "eval_lambada_samples_per_second": 505.61,
      "step": 90000
    },
    {
      "activations/layer0_attention_weight_max": 14.706001281738281,
      "activations/layer0_attention_weight_min": -14.965860366821289,
      "activations/layer10_attention_weight_max": 40.15907669067383,
      "activations/layer10_attention_weight_min": -35.79218292236328,
      "activations/layer11_attention_weight_max": 32.39649200439453,
      "activations/layer11_attention_weight_min": -27.716392517089844,
      "activations/layer12_attention_weight_max": 27.462617874145508,
      "activations/layer12_attention_weight_min": -24.542428970336914,
      "activations/layer13_attention_weight_max": 55.70568084716797,
      "activations/layer13_attention_weight_min": -36.41471481323242,
      "activations/layer14_attention_weight_max": 54.33553695678711,
      "activations/layer14_attention_weight_min": -33.71675109863281,
      "activations/layer15_attention_weight_max": 39.830562591552734,
      "activations/layer15_attention_weight_min": -29.592374801635742,
      "activations/layer16_attention_weight_max": 38.8172607421875,
      "activations/layer16_attention_weight_min": -25.970561981201172,
      "activations/layer17_attention_weight_max": 60.35144805908203,
      "activations/layer17_attention_weight_min": -38.42360305786133,
      "activations/layer18_attention_weight_max": 47.37836837768555,
      "activations/layer18_attention_weight_min": -32.27883529663086,
      "activations/layer19_attention_weight_max": 21.872350692749023,
      "activations/layer19_attention_weight_min": -16.863176345825195,
      "activations/layer1_attention_weight_max": 16.703197479248047,
      "activations/layer1_attention_weight_min": -16.024295806884766,
      "activations/layer20_attention_weight_max": 21.2646484375,
      "activations/layer20_attention_weight_min": -22.075260162353516,
      "activations/layer21_attention_weight_max": 39.830623626708984,
      "activations/layer21_attention_weight_min": -25.021244049072266,
      "activations/layer22_attention_weight_max": 39.58794403076172,
      "activations/layer22_attention_weight_min": -23.052698135375977,
      "activations/layer23_attention_weight_max": 29.376386642456055,
      "activations/layer23_attention_weight_min": -16.617000579833984,
      "activations/layer2_attention_weight_max": 31.895322799682617,
      "activations/layer2_attention_weight_min": -30.976903915405273,
      "activations/layer3_attention_weight_max": 85.787109375,
      "activations/layer3_attention_weight_min": -85.1686782836914,
      "activations/layer4_attention_weight_max": 106.80928039550781,
      "activations/layer4_attention_weight_min": -95.5525894165039,
      "activations/layer5_attention_weight_max": 108.68524169921875,
      "activations/layer5_attention_weight_min": -93.56758880615234,
      "activations/layer6_attention_weight_max": 60.59341812133789,
      "activations/layer6_attention_weight_min": -55.99606704711914,
      "activations/layer7_attention_weight_max": 73.07164764404297,
      "activations/layer7_attention_weight_min": -69.55524444580078,
      "activations/layer8_attention_weight_max": 45.60566329956055,
      "activations/layer8_attention_weight_min": -44.67085266113281,
      "activations/layer9_attention_weight_max": 47.68800735473633,
      "activations/layer9_attention_weight_min": -48.99150848388672,
      "epoch": 5.23,
      "learning_rate": 0.00011743939393939392,
      "loss": 2.8519,
      "step": 90050
    },
    {
      "activations/layer0_attention_weight_max": 13.924640655517578,
      "activations/layer0_attention_weight_min": -14.85122013092041,
      "activations/layer10_attention_weight_max": 37.81488037109375,
      "activations/layer10_attention_weight_min": -37.69671630859375,
      "activations/layer11_attention_weight_max": 29.510356903076172,
      "activations/layer11_attention_weight_min": -28.321569442749023,
      "activations/layer12_attention_weight_max": 23.61711311340332,
      "activations/layer12_attention_weight_min": -24.8970947265625,
      "activations/layer13_attention_weight_max": 52.63025665283203,
      "activations/layer13_attention_weight_min": -32.572845458984375,
      "activations/layer14_attention_weight_max": 53.358734130859375,
      "activations/layer14_attention_weight_min": -30.855253219604492,
      "activations/layer15_attention_weight_max": 40.36231994628906,
      "activations/layer15_attention_weight_min": -28.4566707611084,
      "activations/layer16_attention_weight_max": 35.21104431152344,
      "activations/layer16_attention_weight_min": -25.800678253173828,
      "activations/layer17_attention_weight_max": 53.39952087402344,
      "activations/layer17_attention_weight_min": -37.11800003051758,
      "activations/layer18_attention_weight_max": 46.97885513305664,
      "activations/layer18_attention_weight_min": -31.852739334106445,
      "activations/layer19_attention_weight_max": 23.535531997680664,
      "activations/layer19_attention_weight_min": -18.414339065551758,
      "activations/layer1_attention_weight_max": 17.20138931274414,
      "activations/layer1_attention_weight_min": -13.86105728149414,
      "activations/layer20_attention_weight_max": 22.05694007873535,
      "activations/layer20_attention_weight_min": -22.70804214477539,
      "activations/layer21_attention_weight_max": 39.102821350097656,
      "activations/layer21_attention_weight_min": -27.00298309326172,
      "activations/layer22_attention_weight_max": 34.61954879760742,
      "activations/layer22_attention_weight_min": -23.284482955932617,
      "activations/layer23_attention_weight_max": 27.430675506591797,
      "activations/layer23_attention_weight_min": -17.289031982421875,
      "activations/layer2_attention_weight_max": 29.47393798828125,
      "activations/layer2_attention_weight_min": -28.446857452392578,
      "activations/layer3_attention_weight_max": 80.08330535888672,
      "activations/layer3_attention_weight_min": -84.54714965820312,
      "activations/layer4_attention_weight_max": 98.9613265991211,
      "activations/layer4_attention_weight_min": -91.94192504882812,
      "activations/layer5_attention_weight_max": 98.94949340820312,
      "activations/layer5_attention_weight_min": -88.82232666015625,
      "activations/layer6_attention_weight_max": 55.025306701660156,
      "activations/layer6_attention_weight_min": -51.08759307861328,
      "activations/layer7_attention_weight_max": 68.35800170898438,
      "activations/layer7_attention_weight_min": -67.47529602050781,
      "activations/layer8_attention_weight_max": 42.091285705566406,
      "activations/layer8_attention_weight_min": -41.74819564819336,
      "activations/layer9_attention_weight_max": 49.782508850097656,
      "activations/layer9_attention_weight_min": -46.074031829833984,
      "epoch": 5.24,
      "learning_rate": 0.00011742045454545454,
      "loss": 2.8428,
      "step": 90100
    },
    {
      "activations/layer0_attention_weight_max": 14.48436450958252,
      "activations/layer0_attention_weight_min": -14.556051254272461,
      "activations/layer10_attention_weight_max": 35.814483642578125,
      "activations/layer10_attention_weight_min": -33.02812194824219,
      "activations/layer11_attention_weight_max": 28.698972702026367,
      "activations/layer11_attention_weight_min": -28.710142135620117,
      "activations/layer12_attention_weight_max": 21.48356056213379,
      "activations/layer12_attention_weight_min": -23.12466049194336,
      "activations/layer13_attention_weight_max": 47.38151931762695,
      "activations/layer13_attention_weight_min": -35.654685974121094,
      "activations/layer14_attention_weight_max": 40.12593460083008,
      "activations/layer14_attention_weight_min": -29.742069244384766,
      "activations/layer15_attention_weight_max": 36.54523849487305,
      "activations/layer15_attention_weight_min": -30.473678588867188,
      "activations/layer16_attention_weight_max": 32.782466888427734,
      "activations/layer16_attention_weight_min": -27.41115379333496,
      "activations/layer17_attention_weight_max": 55.713050842285156,
      "activations/layer17_attention_weight_min": -40.45814895629883,
      "activations/layer18_attention_weight_max": 52.31929397583008,
      "activations/layer18_attention_weight_min": -34.598045349121094,
      "activations/layer19_attention_weight_max": 24.307476043701172,
      "activations/layer19_attention_weight_min": -21.190906524658203,
      "activations/layer1_attention_weight_max": 19.786331176757812,
      "activations/layer1_attention_weight_min": -15.582473754882812,
      "activations/layer20_attention_weight_max": 21.632816314697266,
      "activations/layer20_attention_weight_min": -22.582996368408203,
      "activations/layer21_attention_weight_max": 35.935943603515625,
      "activations/layer21_attention_weight_min": -28.188976287841797,
      "activations/layer22_attention_weight_max": 38.87870407104492,
      "activations/layer22_attention_weight_min": -23.802515029907227,
      "activations/layer23_attention_weight_max": 31.97113037109375,
      "activations/layer23_attention_weight_min": -19.095264434814453,
      "activations/layer2_attention_weight_max": 29.43879508972168,
      "activations/layer2_attention_weight_min": -28.36464500427246,
      "activations/layer3_attention_weight_max": 82.53179931640625,
      "activations/layer3_attention_weight_min": -88.22552490234375,
      "activations/layer4_attention_weight_max": 96.35733032226562,
      "activations/layer4_attention_weight_min": -91.63578796386719,
      "activations/layer5_attention_weight_max": 95.59600830078125,
      "activations/layer5_attention_weight_min": -87.69903564453125,
      "activations/layer6_attention_weight_max": 54.782405853271484,
      "activations/layer6_attention_weight_min": -52.00044250488281,
      "activations/layer7_attention_weight_max": 67.6943130493164,
      "activations/layer7_attention_weight_min": -63.40498352050781,
      "activations/layer8_attention_weight_max": 40.59185028076172,
      "activations/layer8_attention_weight_min": -40.166873931884766,
      "activations/layer9_attention_weight_max": 45.35390853881836,
      "activations/layer9_attention_weight_min": -44.09230041503906,
      "epoch": 5.24,
      "learning_rate": 0.00011740151515151514,
      "loss": 2.8616,
      "step": 90150
    },
    {
      "activations/layer0_attention_weight_max": 14.659594535827637,
      "activations/layer0_attention_weight_min": -14.786810874938965,
      "activations/layer10_attention_weight_max": 38.983768463134766,
      "activations/layer10_attention_weight_min": -35.21805953979492,
      "activations/layer11_attention_weight_max": 34.019386291503906,
      "activations/layer11_attention_weight_min": -28.04052734375,
      "activations/layer12_attention_weight_max": 23.5054874420166,
      "activations/layer12_attention_weight_min": -27.173851013183594,
      "activations/layer13_attention_weight_max": 53.439029693603516,
      "activations/layer13_attention_weight_min": -35.452613830566406,
      "activations/layer14_attention_weight_max": 58.069942474365234,
      "activations/layer14_attention_weight_min": -39.22511672973633,
      "activations/layer15_attention_weight_max": 38.636985778808594,
      "activations/layer15_attention_weight_min": -30.44222640991211,
      "activations/layer16_attention_weight_max": 33.96916580200195,
      "activations/layer16_attention_weight_min": -26.021671295166016,
      "activations/layer17_attention_weight_max": 54.08705139160156,
      "activations/layer17_attention_weight_min": -37.878448486328125,
      "activations/layer18_attention_weight_max": 50.982948303222656,
      "activations/layer18_attention_weight_min": -31.878576278686523,
      "activations/layer19_attention_weight_max": 23.467641830444336,
      "activations/layer19_attention_weight_min": -19.172727584838867,
      "activations/layer1_attention_weight_max": 18.451242446899414,
      "activations/layer1_attention_weight_min": -16.679380416870117,
      "activations/layer20_attention_weight_max": 23.01256561279297,
      "activations/layer20_attention_weight_min": -19.64206886291504,
      "activations/layer21_attention_weight_max": 41.109554290771484,
      "activations/layer21_attention_weight_min": -29.245403289794922,
      "activations/layer22_attention_weight_max": 31.405027389526367,
      "activations/layer22_attention_weight_min": -23.0277099609375,
      "activations/layer23_attention_weight_max": 28.477127075195312,
      "activations/layer23_attention_weight_min": -18.4755802154541,
      "activations/layer2_attention_weight_max": 34.2027702331543,
      "activations/layer2_attention_weight_min": -32.01481628417969,
      "activations/layer3_attention_weight_max": 88.64979553222656,
      "activations/layer3_attention_weight_min": -86.88967895507812,
      "activations/layer4_attention_weight_max": 109.33251953125,
      "activations/layer4_attention_weight_min": -100.83985900878906,
      "activations/layer5_attention_weight_max": 110.70372009277344,
      "activations/layer5_attention_weight_min": -100.92025756835938,
      "activations/layer6_attention_weight_max": 64.04969024658203,
      "activations/layer6_attention_weight_min": -58.575538635253906,
      "activations/layer7_attention_weight_max": 87.35433197021484,
      "activations/layer7_attention_weight_min": -79.6111068725586,
      "activations/layer8_attention_weight_max": 56.1548957824707,
      "activations/layer8_attention_weight_min": -42.524559020996094,
      "activations/layer9_attention_weight_max": 64.8611831665039,
      "activations/layer9_attention_weight_min": -51.89331817626953,
      "epoch": 5.24,
      "learning_rate": 0.00011738257575757575,
      "loss": 2.8444,
      "step": 90200
    },
    {
      "activations/layer0_attention_weight_max": 14.333749771118164,
      "activations/layer0_attention_weight_min": -14.511124610900879,
      "activations/layer10_attention_weight_max": 36.7426872253418,
      "activations/layer10_attention_weight_min": -35.804725646972656,
      "activations/layer11_attention_weight_max": 30.983335494995117,
      "activations/layer11_attention_weight_min": -26.884628295898438,
      "activations/layer12_attention_weight_max": 26.73796844482422,
      "activations/layer12_attention_weight_min": -26.85735321044922,
      "activations/layer13_attention_weight_max": 45.460391998291016,
      "activations/layer13_attention_weight_min": -33.46797180175781,
      "activations/layer14_attention_weight_max": 44.78215408325195,
      "activations/layer14_attention_weight_min": -31.58654022216797,
      "activations/layer15_attention_weight_max": 43.356056213378906,
      "activations/layer15_attention_weight_min": -29.715665817260742,
      "activations/layer16_attention_weight_max": 32.42055130004883,
      "activations/layer16_attention_weight_min": -27.128633499145508,
      "activations/layer17_attention_weight_max": 56.91195297241211,
      "activations/layer17_attention_weight_min": -41.77530288696289,
      "activations/layer18_attention_weight_max": 52.88976287841797,
      "activations/layer18_attention_weight_min": -37.71654510498047,
      "activations/layer19_attention_weight_max": 23.529216766357422,
      "activations/layer19_attention_weight_min": -21.835485458374023,
      "activations/layer1_attention_weight_max": 17.442890167236328,
      "activations/layer1_attention_weight_min": -14.710346221923828,
      "activations/layer20_attention_weight_max": 22.763996124267578,
      "activations/layer20_attention_weight_min": -24.397964477539062,
      "activations/layer21_attention_weight_max": 46.760948181152344,
      "activations/layer21_attention_weight_min": -27.147701263427734,
      "activations/layer22_attention_weight_max": 38.359737396240234,
      "activations/layer22_attention_weight_min": -26.618560791015625,
      "activations/layer23_attention_weight_max": 30.41305160522461,
      "activations/layer23_attention_weight_min": -20.255765914916992,
      "activations/layer2_attention_weight_max": 30.07443618774414,
      "activations/layer2_attention_weight_min": -28.881511688232422,
      "activations/layer3_attention_weight_max": 78.04513549804688,
      "activations/layer3_attention_weight_min": -79.39884185791016,
      "activations/layer4_attention_weight_max": 101.27137756347656,
      "activations/layer4_attention_weight_min": -96.2876205444336,
      "activations/layer5_attention_weight_max": 98.68341064453125,
      "activations/layer5_attention_weight_min": -99.29206848144531,
      "activations/layer6_attention_weight_max": 58.14643096923828,
      "activations/layer6_attention_weight_min": -53.615230560302734,
      "activations/layer7_attention_weight_max": 69.4604721069336,
      "activations/layer7_attention_weight_min": -65.56863403320312,
      "activations/layer8_attention_weight_max": 41.305870056152344,
      "activations/layer8_attention_weight_min": -41.83283996582031,
      "activations/layer9_attention_weight_max": 46.41324996948242,
      "activations/layer9_attention_weight_min": -47.439388275146484,
      "epoch": 5.24,
      "learning_rate": 0.00011736363636363635,
      "loss": 2.8492,
      "step": 90250
    },
    {
      "activations/layer0_attention_weight_max": 13.670449256896973,
      "activations/layer0_attention_weight_min": -14.915724754333496,
      "activations/layer10_attention_weight_max": 35.46581268310547,
      "activations/layer10_attention_weight_min": -34.501895904541016,
      "activations/layer11_attention_weight_max": 28.06148910522461,
      "activations/layer11_attention_weight_min": -27.060253143310547,
      "activations/layer12_attention_weight_max": 20.381790161132812,
      "activations/layer12_attention_weight_min": -23.680686950683594,
      "activations/layer13_attention_weight_max": 41.51783752441406,
      "activations/layer13_attention_weight_min": -33.98276138305664,
      "activations/layer14_attention_weight_max": 46.72763442993164,
      "activations/layer14_attention_weight_min": -30.965225219726562,
      "activations/layer15_attention_weight_max": 36.224849700927734,
      "activations/layer15_attention_weight_min": -28.75054359436035,
      "activations/layer16_attention_weight_max": 32.38294982910156,
      "activations/layer16_attention_weight_min": -26.455806732177734,
      "activations/layer17_attention_weight_max": 57.40862274169922,
      "activations/layer17_attention_weight_min": -38.58400344848633,
      "activations/layer18_attention_weight_max": 50.0109748840332,
      "activations/layer18_attention_weight_min": -33.69457244873047,
      "activations/layer19_attention_weight_max": 21.85354232788086,
      "activations/layer19_attention_weight_min": -21.197193145751953,
      "activations/layer1_attention_weight_max": 17.065814971923828,
      "activations/layer1_attention_weight_min": -15.905753135681152,
      "activations/layer20_attention_weight_max": 19.777111053466797,
      "activations/layer20_attention_weight_min": -19.511734008789062,
      "activations/layer21_attention_weight_max": 36.363067626953125,
      "activations/layer21_attention_weight_min": -23.918365478515625,
      "activations/layer22_attention_weight_max": 31.52374839782715,
      "activations/layer22_attention_weight_min": -23.57560920715332,
      "activations/layer23_attention_weight_max": 27.892614364624023,
      "activations/layer23_attention_weight_min": -21.20697784423828,
      "activations/layer2_attention_weight_max": 31.329938888549805,
      "activations/layer2_attention_weight_min": -28.61385726928711,
      "activations/layer3_attention_weight_max": 79.60562896728516,
      "activations/layer3_attention_weight_min": -84.49922943115234,
      "activations/layer4_attention_weight_max": 97.78218841552734,
      "activations/layer4_attention_weight_min": -96.2873764038086,
      "activations/layer5_attention_weight_max": 100.04573822021484,
      "activations/layer5_attention_weight_min": -86.436767578125,
      "activations/layer6_attention_weight_max": 53.27307891845703,
      "activations/layer6_attention_weight_min": -51.00856399536133,
      "activations/layer7_attention_weight_max": 66.24584197998047,
      "activations/layer7_attention_weight_min": -64.38166046142578,
      "activations/layer8_attention_weight_max": 42.22834396362305,
      "activations/layer8_attention_weight_min": -39.939247131347656,
      "activations/layer9_attention_weight_max": 46.73430252075195,
      "activations/layer9_attention_weight_min": -46.69577407836914,
      "epoch": 5.25,
      "learning_rate": 0.00011734469696969696,
      "loss": 2.8456,
      "step": 90300
    },
    {
      "activations/layer0_attention_weight_max": 15.145380973815918,
      "activations/layer0_attention_weight_min": -15.255980491638184,
      "activations/layer10_attention_weight_max": 42.42757034301758,
      "activations/layer10_attention_weight_min": -34.82685852050781,
      "activations/layer11_attention_weight_max": 34.683677673339844,
      "activations/layer11_attention_weight_min": -29.412372589111328,
      "activations/layer12_attention_weight_max": 24.606712341308594,
      "activations/layer12_attention_weight_min": -22.253047943115234,
      "activations/layer13_attention_weight_max": 75.43028259277344,
      "activations/layer13_attention_weight_min": -39.84940719604492,
      "activations/layer14_attention_weight_max": 64.6468505859375,
      "activations/layer14_attention_weight_min": -32.66337966918945,
      "activations/layer15_attention_weight_max": 54.88401794433594,
      "activations/layer15_attention_weight_min": -29.585582733154297,
      "activations/layer16_attention_weight_max": 40.44490051269531,
      "activations/layer16_attention_weight_min": -24.994752883911133,
      "activations/layer17_attention_weight_max": 64.48554229736328,
      "activations/layer17_attention_weight_min": -39.56407165527344,
      "activations/layer18_attention_weight_max": 58.35453796386719,
      "activations/layer18_attention_weight_min": -33.80032730102539,
      "activations/layer19_attention_weight_max": 26.951831817626953,
      "activations/layer19_attention_weight_min": -19.813926696777344,
      "activations/layer1_attention_weight_max": 16.733118057250977,
      "activations/layer1_attention_weight_min": -15.4013032913208,
      "activations/layer20_attention_weight_max": 28.550994873046875,
      "activations/layer20_attention_weight_min": -21.46381950378418,
      "activations/layer21_attention_weight_max": 53.97650146484375,
      "activations/layer21_attention_weight_min": -25.185813903808594,
      "activations/layer22_attention_weight_max": 42.906646728515625,
      "activations/layer22_attention_weight_min": -25.02311897277832,
      "activations/layer23_attention_weight_max": 39.29740524291992,
      "activations/layer23_attention_weight_min": -18.081703186035156,
      "activations/layer2_attention_weight_max": 32.621803283691406,
      "activations/layer2_attention_weight_min": -31.46246910095215,
      "activations/layer3_attention_weight_max": 81.04439544677734,
      "activations/layer3_attention_weight_min": -86.3239974975586,
      "activations/layer4_attention_weight_max": 104.2186279296875,
      "activations/layer4_attention_weight_min": -93.91893768310547,
      "activations/layer5_attention_weight_max": 104.5603256225586,
      "activations/layer5_attention_weight_min": -96.10922241210938,
      "activations/layer6_attention_weight_max": 62.216922760009766,
      "activations/layer6_attention_weight_min": -52.80067443847656,
      "activations/layer7_attention_weight_max": 75.15985870361328,
      "activations/layer7_attention_weight_min": -66.68785858154297,
      "activations/layer8_attention_weight_max": 43.86557388305664,
      "activations/layer8_attention_weight_min": -43.47841262817383,
      "activations/layer9_attention_weight_max": 49.19721603393555,
      "activations/layer9_attention_weight_min": -48.48304748535156,
      "epoch": 5.25,
      "learning_rate": 0.00011732575757575757,
      "loss": 2.8486,
      "step": 90350
    },
    {
      "activations/layer0_attention_weight_max": 14.770377159118652,
      "activations/layer0_attention_weight_min": -14.964102745056152,
      "activations/layer10_attention_weight_max": 38.44651794433594,
      "activations/layer10_attention_weight_min": -36.784568786621094,
      "activations/layer11_attention_weight_max": 28.761289596557617,
      "activations/layer11_attention_weight_min": -28.29158592224121,
      "activations/layer12_attention_weight_max": 22.476343154907227,
      "activations/layer12_attention_weight_min": -27.979576110839844,
      "activations/layer13_attention_weight_max": 48.74824523925781,
      "activations/layer13_attention_weight_min": -30.845306396484375,
      "activations/layer14_attention_weight_max": 53.45474624633789,
      "activations/layer14_attention_weight_min": -35.7537841796875,
      "activations/layer15_attention_weight_max": 41.63197326660156,
      "activations/layer15_attention_weight_min": -31.418392181396484,
      "activations/layer16_attention_weight_max": 35.61790466308594,
      "activations/layer16_attention_weight_min": -27.304685592651367,
      "activations/layer17_attention_weight_max": 57.73370361328125,
      "activations/layer17_attention_weight_min": -38.82857894897461,
      "activations/layer18_attention_weight_max": 47.243431091308594,
      "activations/layer18_attention_weight_min": -34.352596282958984,
      "activations/layer19_attention_weight_max": 24.429834365844727,
      "activations/layer19_attention_weight_min": -20.02804946899414,
      "activations/layer1_attention_weight_max": 17.251935958862305,
      "activations/layer1_attention_weight_min": -15.800707817077637,
      "activations/layer20_attention_weight_max": 21.52784538269043,
      "activations/layer20_attention_weight_min": -21.414302825927734,
      "activations/layer21_attention_weight_max": 42.55714797973633,
      "activations/layer21_attention_weight_min": -27.71706771850586,
      "activations/layer22_attention_weight_max": 32.519386291503906,
      "activations/layer22_attention_weight_min": -26.164113998413086,
      "activations/layer23_attention_weight_max": 31.483436584472656,
      "activations/layer23_attention_weight_min": -20.235774993896484,
      "activations/layer2_attention_weight_max": 31.602981567382812,
      "activations/layer2_attention_weight_min": -29.272075653076172,
      "activations/layer3_attention_weight_max": 81.41278076171875,
      "activations/layer3_attention_weight_min": -85.09063720703125,
      "activations/layer4_attention_weight_max": 103.39678955078125,
      "activations/layer4_attention_weight_min": -96.25982666015625,
      "activations/layer5_attention_weight_max": 105.65667724609375,
      "activations/layer5_attention_weight_min": -94.3985595703125,
      "activations/layer6_attention_weight_max": 59.07411575317383,
      "activations/layer6_attention_weight_min": -53.33413314819336,
      "activations/layer7_attention_weight_max": 71.94506072998047,
      "activations/layer7_attention_weight_min": -67.91378784179688,
      "activations/layer8_attention_weight_max": 43.7091064453125,
      "activations/layer8_attention_weight_min": -42.787593841552734,
      "activations/layer9_attention_weight_max": 50.61248779296875,
      "activations/layer9_attention_weight_min": -47.424949645996094,
      "epoch": 5.25,
      "learning_rate": 0.00011730719696969696,
      "loss": 2.8333,
      "step": 90400
    },
    {
      "activations/layer0_attention_weight_max": 14.404618263244629,
      "activations/layer0_attention_weight_min": -14.998586654663086,
      "activations/layer10_attention_weight_max": 37.85845184326172,
      "activations/layer10_attention_weight_min": -35.920135498046875,
      "activations/layer11_attention_weight_max": 29.44685935974121,
      "activations/layer11_attention_weight_min": -28.136600494384766,
      "activations/layer12_attention_weight_max": 21.088871002197266,
      "activations/layer12_attention_weight_min": -24.837003707885742,
      "activations/layer13_attention_weight_max": 42.666786193847656,
      "activations/layer13_attention_weight_min": -33.63108825683594,
      "activations/layer14_attention_weight_max": 38.66972732543945,
      "activations/layer14_attention_weight_min": -30.76263427734375,
      "activations/layer15_attention_weight_max": 38.451881408691406,
      "activations/layer15_attention_weight_min": -30.849193572998047,
      "activations/layer16_attention_weight_max": 30.163713455200195,
      "activations/layer16_attention_weight_min": -25.791677474975586,
      "activations/layer17_attention_weight_max": 51.14738082885742,
      "activations/layer17_attention_weight_min": -39.41154861450195,
      "activations/layer18_attention_weight_max": 44.52507400512695,
      "activations/layer18_attention_weight_min": -32.568199157714844,
      "activations/layer19_attention_weight_max": 22.866544723510742,
      "activations/layer19_attention_weight_min": -20.145248413085938,
      "activations/layer1_attention_weight_max": 17.03998374938965,
      "activations/layer1_attention_weight_min": -14.398653984069824,
      "activations/layer20_attention_weight_max": 20.957605361938477,
      "activations/layer20_attention_weight_min": -20.834726333618164,
      "activations/layer21_attention_weight_max": 38.52068328857422,
      "activations/layer21_attention_weight_min": -27.10549545288086,
      "activations/layer22_attention_weight_max": 28.997297286987305,
      "activations/layer22_attention_weight_min": -24.669208526611328,
      "activations/layer23_attention_weight_max": 26.522560119628906,
      "activations/layer23_attention_weight_min": -17.963134765625,
      "activations/layer2_attention_weight_max": 31.879398345947266,
      "activations/layer2_attention_weight_min": -29.118667602539062,
      "activations/layer3_attention_weight_max": 88.2878189086914,
      "activations/layer3_attention_weight_min": -86.66920471191406,
      "activations/layer4_attention_weight_max": 100.67205047607422,
      "activations/layer4_attention_weight_min": -94.67510986328125,
      "activations/layer5_attention_weight_max": 102.65992736816406,
      "activations/layer5_attention_weight_min": -89.67454528808594,
      "activations/layer6_attention_weight_max": 57.68653106689453,
      "activations/layer6_attention_weight_min": -53.57242202758789,
      "activations/layer7_attention_weight_max": 66.2539291381836,
      "activations/layer7_attention_weight_min": -66.9596939086914,
      "activations/layer8_attention_weight_max": 44.325626373291016,
      "activations/layer8_attention_weight_min": -43.898536682128906,
      "activations/layer9_attention_weight_max": 48.82638168334961,
      "activations/layer9_attention_weight_min": -45.76380920410156,
      "epoch": 5.26,
      "learning_rate": 0.00011728825757575758,
      "loss": 2.8434,
      "step": 90450
    },
    {
      "activations/layer0_attention_weight_max": 14.47046184539795,
      "activations/layer0_attention_weight_min": -15.35721492767334,
      "activations/layer10_attention_weight_max": 36.571205139160156,
      "activations/layer10_attention_weight_min": -33.03805160522461,
      "activations/layer11_attention_weight_max": 30.154258728027344,
      "activations/layer11_attention_weight_min": -25.64368438720703,
      "activations/layer12_attention_weight_max": 21.56498146057129,
      "activations/layer12_attention_weight_min": -22.07597541809082,
      "activations/layer13_attention_weight_max": 53.98426055908203,
      "activations/layer13_attention_weight_min": -36.68894577026367,
      "activations/layer14_attention_weight_max": 48.58543014526367,
      "activations/layer14_attention_weight_min": -33.62977981567383,
      "activations/layer15_attention_weight_max": 39.65097427368164,
      "activations/layer15_attention_weight_min": -27.606782913208008,
      "activations/layer16_attention_weight_max": 36.12087631225586,
      "activations/layer16_attention_weight_min": -25.017065048217773,
      "activations/layer17_attention_weight_max": 59.96146774291992,
      "activations/layer17_attention_weight_min": -39.60283279418945,
      "activations/layer18_attention_weight_max": 54.54269790649414,
      "activations/layer18_attention_weight_min": -34.42311096191406,
      "activations/layer19_attention_weight_max": 27.607900619506836,
      "activations/layer19_attention_weight_min": -18.849166870117188,
      "activations/layer1_attention_weight_max": 16.970012664794922,
      "activations/layer1_attention_weight_min": -13.95330810546875,
      "activations/layer20_attention_weight_max": 24.209829330444336,
      "activations/layer20_attention_weight_min": -21.095609664916992,
      "activations/layer21_attention_weight_max": 39.45134353637695,
      "activations/layer21_attention_weight_min": -22.43412208557129,
      "activations/layer22_attention_weight_max": 37.119422912597656,
      "activations/layer22_attention_weight_min": -28.266090393066406,
      "activations/layer23_attention_weight_max": 29.137174606323242,
      "activations/layer23_attention_weight_min": -16.91330909729004,
      "activations/layer2_attention_weight_max": 28.746206283569336,
      "activations/layer2_attention_weight_min": -27.66598892211914,
      "activations/layer3_attention_weight_max": 79.1481704711914,
      "activations/layer3_attention_weight_min": -83.39180755615234,
      "activations/layer4_attention_weight_max": 98.42420196533203,
      "activations/layer4_attention_weight_min": -93.93697357177734,
      "activations/layer5_attention_weight_max": 96.20366668701172,
      "activations/layer5_attention_weight_min": -90.87596130371094,
      "activations/layer6_attention_weight_max": 55.55238723754883,
      "activations/layer6_attention_weight_min": -51.71750259399414,
      "activations/layer7_attention_weight_max": 67.75116729736328,
      "activations/layer7_attention_weight_min": -62.95787048339844,
      "activations/layer8_attention_weight_max": 40.43026351928711,
      "activations/layer8_attention_weight_min": -41.10957336425781,
      "activations/layer9_attention_weight_max": 51.836891174316406,
      "activations/layer9_attention_weight_min": -47.40111541748047,
      "epoch": 5.26,
      "learning_rate": 0.00011726931818181816,
      "loss": 2.8201,
      "step": 90500
    },
    {
      "activations/layer0_attention_weight_max": 14.479277610778809,
      "activations/layer0_attention_weight_min": -15.21349811553955,
      "activations/layer10_attention_weight_max": 38.84328079223633,
      "activations/layer10_attention_weight_min": -35.19336700439453,
      "activations/layer11_attention_weight_max": 32.915985107421875,
      "activations/layer11_attention_weight_min": -28.322553634643555,
      "activations/layer12_attention_weight_max": 26.44224739074707,
      "activations/layer12_attention_weight_min": -25.06843376159668,
      "activations/layer13_attention_weight_max": 54.84098815917969,
      "activations/layer13_attention_weight_min": -35.09975051879883,
      "activations/layer14_attention_weight_max": 51.52189636230469,
      "activations/layer14_attention_weight_min": -30.3621883392334,
      "activations/layer15_attention_weight_max": 49.832054138183594,
      "activations/layer15_attention_weight_min": -29.902002334594727,
      "activations/layer16_attention_weight_max": 35.370880126953125,
      "activations/layer16_attention_weight_min": -24.848451614379883,
      "activations/layer17_attention_weight_max": 61.54326248168945,
      "activations/layer17_attention_weight_min": -39.9426155090332,
      "activations/layer18_attention_weight_max": 61.69868087768555,
      "activations/layer18_attention_weight_min": -34.59819412231445,
      "activations/layer19_attention_weight_max": 26.722715377807617,
      "activations/layer19_attention_weight_min": -20.868215560913086,
      "activations/layer1_attention_weight_max": 18.34557342529297,
      "activations/layer1_attention_weight_min": -14.666328430175781,
      "activations/layer20_attention_weight_max": 22.325658798217773,
      "activations/layer20_attention_weight_min": -19.81492042541504,
      "activations/layer21_attention_weight_max": 46.06563186645508,
      "activations/layer21_attention_weight_min": -25.041332244873047,
      "activations/layer22_attention_weight_max": 32.375492095947266,
      "activations/layer22_attention_weight_min": -23.22947883605957,
      "activations/layer23_attention_weight_max": 28.59490966796875,
      "activations/layer23_attention_weight_min": -19.419681549072266,
      "activations/layer2_attention_weight_max": 31.797590255737305,
      "activations/layer2_attention_weight_min": -29.67854881286621,
      "activations/layer3_attention_weight_max": 82.89163970947266,
      "activations/layer3_attention_weight_min": -84.32463073730469,
      "activations/layer4_attention_weight_max": 103.05914306640625,
      "activations/layer4_attention_weight_min": -95.96315002441406,
      "activations/layer5_attention_weight_max": 97.91371154785156,
      "activations/layer5_attention_weight_min": -92.35791015625,
      "activations/layer6_attention_weight_max": 57.81785202026367,
      "activations/layer6_attention_weight_min": -53.33552932739258,
      "activations/layer7_attention_weight_max": 68.13098907470703,
      "activations/layer7_attention_weight_min": -64.73287963867188,
      "activations/layer8_attention_weight_max": 44.76710510253906,
      "activations/layer8_attention_weight_min": -43.686134338378906,
      "activations/layer9_attention_weight_max": 51.36475372314453,
      "activations/layer9_attention_weight_min": -44.715675354003906,
      "epoch": 5.26,
      "learning_rate": 0.00011725037878787878,
      "loss": 2.8484,
      "step": 90550
    },
    {
      "activations/layer0_attention_weight_max": 15.966447830200195,
      "activations/layer0_attention_weight_min": -15.08691692352295,
      "activations/layer10_attention_weight_max": 37.33234405517578,
      "activations/layer10_attention_weight_min": -34.22505187988281,
      "activations/layer11_attention_weight_max": 28.666221618652344,
      "activations/layer11_attention_weight_min": -27.624616622924805,
      "activations/layer12_attention_weight_max": 25.08060646057129,
      "activations/layer12_attention_weight_min": -22.8903751373291,
      "activations/layer13_attention_weight_max": 50.468135833740234,
      "activations/layer13_attention_weight_min": -36.00349426269531,
      "activations/layer14_attention_weight_max": 44.71076965332031,
      "activations/layer14_attention_weight_min": -28.49325942993164,
      "activations/layer15_attention_weight_max": 40.178714752197266,
      "activations/layer15_attention_weight_min": -30.136884689331055,
      "activations/layer16_attention_weight_max": 32.884273529052734,
      "activations/layer16_attention_weight_min": -27.26145362854004,
      "activations/layer17_attention_weight_max": 56.3975715637207,
      "activations/layer17_attention_weight_min": -37.77682113647461,
      "activations/layer18_attention_weight_max": 53.35944747924805,
      "activations/layer18_attention_weight_min": -32.16682052612305,
      "activations/layer19_attention_weight_max": 24.830215454101562,
      "activations/layer19_attention_weight_min": -19.48941993713379,
      "activations/layer1_attention_weight_max": 17.835559844970703,
      "activations/layer1_attention_weight_min": -16.79378318786621,
      "activations/layer20_attention_weight_max": 20.57546043395996,
      "activations/layer20_attention_weight_min": -21.784732818603516,
      "activations/layer21_attention_weight_max": 38.35731887817383,
      "activations/layer21_attention_weight_min": -25.449174880981445,
      "activations/layer22_attention_weight_max": 33.209712982177734,
      "activations/layer22_attention_weight_min": -21.083757400512695,
      "activations/layer23_attention_weight_max": 30.455602645874023,
      "activations/layer23_attention_weight_min": -17.276554107666016,
      "activations/layer2_attention_weight_max": 30.690664291381836,
      "activations/layer2_attention_weight_min": -28.034605026245117,
      "activations/layer3_attention_weight_max": 80.36880493164062,
      "activations/layer3_attention_weight_min": -84.06746673583984,
      "activations/layer4_attention_weight_max": 102.08836364746094,
      "activations/layer4_attention_weight_min": -95.10787200927734,
      "activations/layer5_attention_weight_max": 103.2658920288086,
      "activations/layer5_attention_weight_min": -94.57278442382812,
      "activations/layer6_attention_weight_max": 59.224571228027344,
      "activations/layer6_attention_weight_min": -51.77861022949219,
      "activations/layer7_attention_weight_max": 73.0120620727539,
      "activations/layer7_attention_weight_min": -68.71101379394531,
      "activations/layer8_attention_weight_max": 44.029048919677734,
      "activations/layer8_attention_weight_min": -41.97626876831055,
      "activations/layer9_attention_weight_max": 55.5684814453125,
      "activations/layer9_attention_weight_min": -46.40255355834961,
      "epoch": 5.26,
      "learning_rate": 0.0001172314393939394,
      "loss": 2.8462,
      "step": 90600
    },
    {
      "activations/layer0_attention_weight_max": 14.643027305603027,
      "activations/layer0_attention_weight_min": -15.213093757629395,
      "activations/layer10_attention_weight_max": 42.394317626953125,
      "activations/layer10_attention_weight_min": -34.84858322143555,
      "activations/layer11_attention_weight_max": 34.40602111816406,
      "activations/layer11_attention_weight_min": -28.570676803588867,
      "activations/layer12_attention_weight_max": 25.448665618896484,
      "activations/layer12_attention_weight_min": -27.09663200378418,
      "activations/layer13_attention_weight_max": 69.17649841308594,
      "activations/layer13_attention_weight_min": -42.20745849609375,
      "activations/layer14_attention_weight_max": 45.18865203857422,
      "activations/layer14_attention_weight_min": -30.14277458190918,
      "activations/layer15_attention_weight_max": 43.734710693359375,
      "activations/layer15_attention_weight_min": -30.643552780151367,
      "activations/layer16_attention_weight_max": 37.898643493652344,
      "activations/layer16_attention_weight_min": -26.23130989074707,
      "activations/layer17_attention_weight_max": 60.62086868286133,
      "activations/layer17_attention_weight_min": -40.88138961791992,
      "activations/layer18_attention_weight_max": 51.71221160888672,
      "activations/layer18_attention_weight_min": -33.5709342956543,
      "activations/layer19_attention_weight_max": 23.900127410888672,
      "activations/layer19_attention_weight_min": -21.330305099487305,
      "activations/layer1_attention_weight_max": 16.751127243041992,
      "activations/layer1_attention_weight_min": -15.307297706604004,
      "activations/layer20_attention_weight_max": 22.62712287902832,
      "activations/layer20_attention_weight_min": -20.813880920410156,
      "activations/layer21_attention_weight_max": 40.58500671386719,
      "activations/layer21_attention_weight_min": -24.7249755859375,
      "activations/layer22_attention_weight_max": 38.051727294921875,
      "activations/layer22_attention_weight_min": -23.42205047607422,
      "activations/layer23_attention_weight_max": 32.41231155395508,
      "activations/layer23_attention_weight_min": -17.742874145507812,
      "activations/layer2_attention_weight_max": 32.21522521972656,
      "activations/layer2_attention_weight_min": -29.1530818939209,
      "activations/layer3_attention_weight_max": 85.27691650390625,
      "activations/layer3_attention_weight_min": -85.61231231689453,
      "activations/layer4_attention_weight_max": 103.4346923828125,
      "activations/layer4_attention_weight_min": -100.32349395751953,
      "activations/layer5_attention_weight_max": 104.31355285644531,
      "activations/layer5_attention_weight_min": -91.66893005371094,
      "activations/layer6_attention_weight_max": 58.68770217895508,
      "activations/layer6_attention_weight_min": -53.277671813964844,
      "activations/layer7_attention_weight_max": 69.61137390136719,
      "activations/layer7_attention_weight_min": -67.79269409179688,
      "activations/layer8_attention_weight_max": 43.987510681152344,
      "activations/layer8_attention_weight_min": -42.83424377441406,
      "activations/layer9_attention_weight_max": 51.10133743286133,
      "activations/layer9_attention_weight_min": -50.06401824951172,
      "epoch": 5.27,
      "learning_rate": 0.00011721249999999998,
      "loss": 2.8581,
      "step": 90650
    },
    {
      "activations/layer0_attention_weight_max": 14.569993019104004,
      "activations/layer0_attention_weight_min": -15.191701889038086,
      "activations/layer10_attention_weight_max": 35.02348709106445,
      "activations/layer10_attention_weight_min": -33.471099853515625,
      "activations/layer11_attention_weight_max": 28.149688720703125,
      "activations/layer11_attention_weight_min": -27.094181060791016,
      "activations/layer12_attention_weight_max": 20.85955810546875,
      "activations/layer12_attention_weight_min": -26.14257049560547,
      "activations/layer13_attention_weight_max": 50.88215637207031,
      "activations/layer13_attention_weight_min": -38.818172454833984,
      "activations/layer14_attention_weight_max": 37.63010025024414,
      "activations/layer14_attention_weight_min": -28.196115493774414,
      "activations/layer15_attention_weight_max": 36.287071228027344,
      "activations/layer15_attention_weight_min": -30.376022338867188,
      "activations/layer16_attention_weight_max": 31.734636306762695,
      "activations/layer16_attention_weight_min": -25.793052673339844,
      "activations/layer17_attention_weight_max": 62.2079963684082,
      "activations/layer17_attention_weight_min": -37.45431137084961,
      "activations/layer18_attention_weight_max": 47.92523193359375,
      "activations/layer18_attention_weight_min": -33.1693229675293,
      "activations/layer19_attention_weight_max": 21.752378463745117,
      "activations/layer19_attention_weight_min": -19.898208618164062,
      "activations/layer1_attention_weight_max": 18.191850662231445,
      "activations/layer1_attention_weight_min": -15.691243171691895,
      "activations/layer20_attention_weight_max": 21.97593879699707,
      "activations/layer20_attention_weight_min": -19.448020935058594,
      "activations/layer21_attention_weight_max": 39.243255615234375,
      "activations/layer21_attention_weight_min": -25.342254638671875,
      "activations/layer22_attention_weight_max": 35.16458511352539,
      "activations/layer22_attention_weight_min": -24.138774871826172,
      "activations/layer23_attention_weight_max": 31.990501403808594,
      "activations/layer23_attention_weight_min": -18.30256462097168,
      "activations/layer2_attention_weight_max": 29.950803756713867,
      "activations/layer2_attention_weight_min": -27.633594512939453,
      "activations/layer3_attention_weight_max": 82.95372009277344,
      "activations/layer3_attention_weight_min": -90.03305053710938,
      "activations/layer4_attention_weight_max": 102.64708709716797,
      "activations/layer4_attention_weight_min": -99.20995330810547,
      "activations/layer5_attention_weight_max": 100.67778015136719,
      "activations/layer5_attention_weight_min": -88.833251953125,
      "activations/layer6_attention_weight_max": 54.66551971435547,
      "activations/layer6_attention_weight_min": -50.31987762451172,
      "activations/layer7_attention_weight_max": 65.25363159179688,
      "activations/layer7_attention_weight_min": -63.76277160644531,
      "activations/layer8_attention_weight_max": 40.15459060668945,
      "activations/layer8_attention_weight_min": -40.490074157714844,
      "activations/layer9_attention_weight_max": 45.7488899230957,
      "activations/layer9_attention_weight_min": -50.99369430541992,
      "epoch": 5.27,
      "learning_rate": 0.0001171935606060606,
      "loss": 2.8529,
      "step": 90700
    },
    {
      "activations/layer0_attention_weight_max": 13.623175621032715,
      "activations/layer0_attention_weight_min": -15.176692008972168,
      "activations/layer10_attention_weight_max": 33.69165802001953,
      "activations/layer10_attention_weight_min": -32.115142822265625,
      "activations/layer11_attention_weight_max": 28.443870544433594,
      "activations/layer11_attention_weight_min": -28.831462860107422,
      "activations/layer12_attention_weight_max": 20.96995735168457,
      "activations/layer12_attention_weight_min": -24.16442108154297,
      "activations/layer13_attention_weight_max": 44.904048919677734,
      "activations/layer13_attention_weight_min": -32.527313232421875,
      "activations/layer14_attention_weight_max": 39.16194152832031,
      "activations/layer14_attention_weight_min": -29.024076461791992,
      "activations/layer15_attention_weight_max": 37.891334533691406,
      "activations/layer15_attention_weight_min": -30.17461585998535,
      "activations/layer16_attention_weight_max": 35.779747009277344,
      "activations/layer16_attention_weight_min": -25.907146453857422,
      "activations/layer17_attention_weight_max": 56.05137252807617,
      "activations/layer17_attention_weight_min": -38.52845001220703,
      "activations/layer18_attention_weight_max": 47.80617904663086,
      "activations/layer18_attention_weight_min": -31.732046127319336,
      "activations/layer19_attention_weight_max": 21.962736129760742,
      "activations/layer19_attention_weight_min": -20.731395721435547,
      "activations/layer1_attention_weight_max": 16.658145904541016,
      "activations/layer1_attention_weight_min": -14.656298637390137,
      "activations/layer20_attention_weight_max": 19.98416519165039,
      "activations/layer20_attention_weight_min": -22.718454360961914,
      "activations/layer21_attention_weight_max": 39.08685302734375,
      "activations/layer21_attention_weight_min": -25.43149185180664,
      "activations/layer22_attention_weight_max": 35.10539627075195,
      "activations/layer22_attention_weight_min": -25.383922576904297,
      "activations/layer23_attention_weight_max": 29.135513305664062,
      "activations/layer23_attention_weight_min": -18.16047477722168,
      "activations/layer2_attention_weight_max": 29.989734649658203,
      "activations/layer2_attention_weight_min": -27.966129302978516,
      "activations/layer3_attention_weight_max": 77.75806427001953,
      "activations/layer3_attention_weight_min": -83.00581359863281,
      "activations/layer4_attention_weight_max": 96.30534362792969,
      "activations/layer4_attention_weight_min": -91.29828643798828,
      "activations/layer5_attention_weight_max": 97.64414978027344,
      "activations/layer5_attention_weight_min": -87.95210266113281,
      "activations/layer6_attention_weight_max": 53.14744567871094,
      "activations/layer6_attention_weight_min": -49.242042541503906,
      "activations/layer7_attention_weight_max": 65.80328369140625,
      "activations/layer7_attention_weight_min": -64.65605163574219,
      "activations/layer8_attention_weight_max": 38.62715148925781,
      "activations/layer8_attention_weight_min": -38.8342399597168,
      "activations/layer9_attention_weight_max": 50.59358596801758,
      "activations/layer9_attention_weight_min": -45.100128173828125,
      "epoch": 5.27,
      "learning_rate": 0.0001171746212121212,
      "loss": 2.8243,
      "step": 90750
    },
    {
      "activations/layer0_attention_weight_max": 13.605497360229492,
      "activations/layer0_attention_weight_min": -14.6730375289917,
      "activations/layer10_attention_weight_max": 40.56220626831055,
      "activations/layer10_attention_weight_min": -35.16230773925781,
      "activations/layer11_attention_weight_max": 30.742483139038086,
      "activations/layer11_attention_weight_min": -30.34547233581543,
      "activations/layer12_attention_weight_max": 23.73409652709961,
      "activations/layer12_attention_weight_min": -27.639677047729492,
      "activations/layer13_attention_weight_max": 52.13993835449219,
      "activations/layer13_attention_weight_min": -37.43853759765625,
      "activations/layer14_attention_weight_max": 43.8460578918457,
      "activations/layer14_attention_weight_min": -30.928529739379883,
      "activations/layer15_attention_weight_max": 40.18480682373047,
      "activations/layer15_attention_weight_min": -30.08565330505371,
      "activations/layer16_attention_weight_max": 35.97516632080078,
      "activations/layer16_attention_weight_min": -28.644855499267578,
      "activations/layer17_attention_weight_max": 54.450050354003906,
      "activations/layer17_attention_weight_min": -41.34061050415039,
      "activations/layer18_attention_weight_max": 50.767127990722656,
      "activations/layer18_attention_weight_min": -34.44681930541992,
      "activations/layer19_attention_weight_max": 21.000089645385742,
      "activations/layer19_attention_weight_min": -19.565282821655273,
      "activations/layer1_attention_weight_max": 19.972347259521484,
      "activations/layer1_attention_weight_min": -17.35724449157715,
      "activations/layer20_attention_weight_max": 22.498769760131836,
      "activations/layer20_attention_weight_min": -25.91518211364746,
      "activations/layer21_attention_weight_max": 42.3278923034668,
      "activations/layer21_attention_weight_min": -23.51476287841797,
      "activations/layer22_attention_weight_max": 36.79945755004883,
      "activations/layer22_attention_weight_min": -27.0673828125,
      "activations/layer23_attention_weight_max": 30.622783660888672,
      "activations/layer23_attention_weight_min": -19.427162170410156,
      "activations/layer2_attention_weight_max": 30.97290802001953,
      "activations/layer2_attention_weight_min": -28.31593894958496,
      "activations/layer3_attention_weight_max": 83.71360778808594,
      "activations/layer3_attention_weight_min": -85.02122497558594,
      "activations/layer4_attention_weight_max": 103.90083312988281,
      "activations/layer4_attention_weight_min": -96.79911804199219,
      "activations/layer5_attention_weight_max": 106.91151428222656,
      "activations/layer5_attention_weight_min": -93.23031616210938,
      "activations/layer6_attention_weight_max": 61.6061897277832,
      "activations/layer6_attention_weight_min": -55.45659255981445,
      "activations/layer7_attention_weight_max": 72.25791931152344,
      "activations/layer7_attention_weight_min": -69.62918853759766,
      "activations/layer8_attention_weight_max": 45.636199951171875,
      "activations/layer8_attention_weight_min": -43.0974006652832,
      "activations/layer9_attention_weight_max": 51.047882080078125,
      "activations/layer9_attention_weight_min": -47.90388107299805,
      "epoch": 5.28,
      "learning_rate": 0.0001171556818181818,
      "loss": 2.8476,
      "step": 90800
    },
    {
      "activations/layer0_attention_weight_max": 14.296313285827637,
      "activations/layer0_attention_weight_min": -14.731822967529297,
      "activations/layer10_attention_weight_max": 40.510318756103516,
      "activations/layer10_attention_weight_min": -36.10675048828125,
      "activations/layer11_attention_weight_max": 31.343482971191406,
      "activations/layer11_attention_weight_min": -27.000995635986328,
      "activations/layer12_attention_weight_max": 24.34280014038086,
      "activations/layer12_attention_weight_min": -26.51121711730957,
      "activations/layer13_attention_weight_max": 48.123390197753906,
      "activations/layer13_attention_weight_min": -36.160858154296875,
      "activations/layer14_attention_weight_max": 49.850215911865234,
      "activations/layer14_attention_weight_min": -34.87418746948242,
      "activations/layer15_attention_weight_max": 38.25227355957031,
      "activations/layer15_attention_weight_min": -30.38654136657715,
      "activations/layer16_attention_weight_max": 37.920509338378906,
      "activations/layer16_attention_weight_min": -26.547014236450195,
      "activations/layer17_attention_weight_max": 57.52571487426758,
      "activations/layer17_attention_weight_min": -36.60605239868164,
      "activations/layer18_attention_weight_max": 49.92367935180664,
      "activations/layer18_attention_weight_min": -32.30149841308594,
      "activations/layer19_attention_weight_max": 26.01123046875,
      "activations/layer19_attention_weight_min": -19.260242462158203,
      "activations/layer1_attention_weight_max": 18.947786331176758,
      "activations/layer1_attention_weight_min": -17.551616668701172,
      "activations/layer20_attention_weight_max": 21.500185012817383,
      "activations/layer20_attention_weight_min": -21.767108917236328,
      "activations/layer21_attention_weight_max": 40.97157669067383,
      "activations/layer21_attention_weight_min": -27.427579879760742,
      "activations/layer22_attention_weight_max": 34.99618911743164,
      "activations/layer22_attention_weight_min": -23.85747718811035,
      "activations/layer23_attention_weight_max": 30.497772216796875,
      "activations/layer23_attention_weight_min": -19.402462005615234,
      "activations/layer2_attention_weight_max": 30.685649871826172,
      "activations/layer2_attention_weight_min": -27.66872215270996,
      "activations/layer3_attention_weight_max": 80.35076904296875,
      "activations/layer3_attention_weight_min": -79.3515853881836,
      "activations/layer4_attention_weight_max": 99.93817901611328,
      "activations/layer4_attention_weight_min": -91.9317855834961,
      "activations/layer5_attention_weight_max": 105.17192840576172,
      "activations/layer5_attention_weight_min": -90.5886459350586,
      "activations/layer6_attention_weight_max": 58.3505859375,
      "activations/layer6_attention_weight_min": -51.15530776977539,
      "activations/layer7_attention_weight_max": 69.82945251464844,
      "activations/layer7_attention_weight_min": -64.98624420166016,
      "activations/layer8_attention_weight_max": 41.12031936645508,
      "activations/layer8_attention_weight_min": -41.76588821411133,
      "activations/layer9_attention_weight_max": 50.76038360595703,
      "activations/layer9_attention_weight_min": -45.92519760131836,
      "epoch": 5.28,
      "learning_rate": 0.00011713674242424242,
      "loss": 2.8246,
      "step": 90850
    },
    {
      "activations/layer0_attention_weight_max": 14.02536678314209,
      "activations/layer0_attention_weight_min": -14.818489074707031,
      "activations/layer10_attention_weight_max": 37.880821228027344,
      "activations/layer10_attention_weight_min": -35.11577224731445,
      "activations/layer11_attention_weight_max": 29.49111557006836,
      "activations/layer11_attention_weight_min": -28.9473934173584,
      "activations/layer12_attention_weight_max": 23.81028938293457,
      "activations/layer12_attention_weight_min": -23.87265968322754,
      "activations/layer13_attention_weight_max": 49.14265441894531,
      "activations/layer13_attention_weight_min": -36.675926208496094,
      "activations/layer14_attention_weight_max": 41.84185028076172,
      "activations/layer14_attention_weight_min": -30.66904067993164,
      "activations/layer15_attention_weight_max": 43.517024993896484,
      "activations/layer15_attention_weight_min": -29.937170028686523,
      "activations/layer16_attention_weight_max": 34.173030853271484,
      "activations/layer16_attention_weight_min": -27.469650268554688,
      "activations/layer17_attention_weight_max": 60.45670700073242,
      "activations/layer17_attention_weight_min": -40.24180221557617,
      "activations/layer18_attention_weight_max": 57.6279411315918,
      "activations/layer18_attention_weight_min": -33.79180145263672,
      "activations/layer19_attention_weight_max": 26.68832778930664,
      "activations/layer19_attention_weight_min": -19.88877296447754,
      "activations/layer1_attention_weight_max": 19.998764038085938,
      "activations/layer1_attention_weight_min": -19.71292495727539,
      "activations/layer20_attention_weight_max": 23.813966751098633,
      "activations/layer20_attention_weight_min": -22.25705337524414,
      "activations/layer21_attention_weight_max": 43.02669143676758,
      "activations/layer21_attention_weight_min": -26.664175033569336,
      "activations/layer22_attention_weight_max": 33.63801956176758,
      "activations/layer22_attention_weight_min": -23.224702835083008,
      "activations/layer23_attention_weight_max": 30.720624923706055,
      "activations/layer23_attention_weight_min": -18.583192825317383,
      "activations/layer2_attention_weight_max": 28.369281768798828,
      "activations/layer2_attention_weight_min": -26.81282615661621,
      "activations/layer3_attention_weight_max": 80.40776824951172,
      "activations/layer3_attention_weight_min": -85.08889770507812,
      "activations/layer4_attention_weight_max": 103.55449676513672,
      "activations/layer4_attention_weight_min": -97.80073547363281,
      "activations/layer5_attention_weight_max": 98.14265441894531,
      "activations/layer5_attention_weight_min": -92.69247436523438,
      "activations/layer6_attention_weight_max": 57.144081115722656,
      "activations/layer6_attention_weight_min": -54.179420471191406,
      "activations/layer7_attention_weight_max": 73.14036560058594,
      "activations/layer7_attention_weight_min": -67.99433135986328,
      "activations/layer8_attention_weight_max": 45.10475540161133,
      "activations/layer8_attention_weight_min": -46.29888916015625,
      "activations/layer9_attention_weight_max": 49.873443603515625,
      "activations/layer9_attention_weight_min": -47.73514175415039,
      "epoch": 5.28,
      "learning_rate": 0.00011711780303030302,
      "loss": 2.8461,
      "step": 90900
    },
    {
      "activations/layer0_attention_weight_max": 15.834366798400879,
      "activations/layer0_attention_weight_min": -15.383833885192871,
      "activations/layer10_attention_weight_max": 39.128448486328125,
      "activations/layer10_attention_weight_min": -34.40314483642578,
      "activations/layer11_attention_weight_max": 33.722023010253906,
      "activations/layer11_attention_weight_min": -26.778717041015625,
      "activations/layer12_attention_weight_max": 21.38353157043457,
      "activations/layer12_attention_weight_min": -26.5992488861084,
      "activations/layer13_attention_weight_max": 56.09143829345703,
      "activations/layer13_attention_weight_min": -39.21985626220703,
      "activations/layer14_attention_weight_max": 42.708683013916016,
      "activations/layer14_attention_weight_min": -32.612667083740234,
      "activations/layer15_attention_weight_max": 41.34254455566406,
      "activations/layer15_attention_weight_min": -30.337162017822266,
      "activations/layer16_attention_weight_max": 35.60122299194336,
      "activations/layer16_attention_weight_min": -28.243026733398438,
      "activations/layer17_attention_weight_max": 55.33027648925781,
      "activations/layer17_attention_weight_min": -38.96136474609375,
      "activations/layer18_attention_weight_max": 47.122337341308594,
      "activations/layer18_attention_weight_min": -33.41502380371094,
      "activations/layer19_attention_weight_max": 26.14996337890625,
      "activations/layer19_attention_weight_min": -18.455955505371094,
      "activations/layer1_attention_weight_max": 17.723806381225586,
      "activations/layer1_attention_weight_min": -15.326866149902344,
      "activations/layer20_attention_weight_max": 21.497766494750977,
      "activations/layer20_attention_weight_min": -20.91202735900879,
      "activations/layer21_attention_weight_max": 39.20716094970703,
      "activations/layer21_attention_weight_min": -27.174949645996094,
      "activations/layer22_attention_weight_max": 35.70966720581055,
      "activations/layer22_attention_weight_min": -26.47252082824707,
      "activations/layer23_attention_weight_max": 29.370994567871094,
      "activations/layer23_attention_weight_min": -19.371349334716797,
      "activations/layer2_attention_weight_max": 28.88799285888672,
      "activations/layer2_attention_weight_min": -28.520170211791992,
      "activations/layer3_attention_weight_max": 83.78257751464844,
      "activations/layer3_attention_weight_min": -84.82825469970703,
      "activations/layer4_attention_weight_max": 99.6578598022461,
      "activations/layer4_attention_weight_min": -95.83431243896484,
      "activations/layer5_attention_weight_max": 97.565185546875,
      "activations/layer5_attention_weight_min": -88.94851684570312,
      "activations/layer6_attention_weight_max": 56.06101989746094,
      "activations/layer6_attention_weight_min": -51.90037536621094,
      "activations/layer7_attention_weight_max": 73.08145904541016,
      "activations/layer7_attention_weight_min": -67.60400390625,
      "activations/layer8_attention_weight_max": 46.67043685913086,
      "activations/layer8_attention_weight_min": -45.23577880859375,
      "activations/layer9_attention_weight_max": 59.388885498046875,
      "activations/layer9_attention_weight_min": -51.222686767578125,
      "epoch": 5.28,
      "learning_rate": 0.00011709886363636363,
      "loss": 2.8575,
      "step": 90950
    },
    {
      "activations/layer0_attention_weight_max": 14.015030860900879,
      "activations/layer0_attention_weight_min": -15.048443794250488,
      "activations/layer10_attention_weight_max": 37.77173614501953,
      "activations/layer10_attention_weight_min": -31.84568977355957,
      "activations/layer11_attention_weight_max": 29.13282012939453,
      "activations/layer11_attention_weight_min": -26.007375717163086,
      "activations/layer12_attention_weight_max": 23.427770614624023,
      "activations/layer12_attention_weight_min": -24.171579360961914,
      "activations/layer13_attention_weight_max": 48.2829704284668,
      "activations/layer13_attention_weight_min": -34.21855926513672,
      "activations/layer14_attention_weight_max": 50.20606231689453,
      "activations/layer14_attention_weight_min": -36.75840377807617,
      "activations/layer15_attention_weight_max": 42.11556625366211,
      "activations/layer15_attention_weight_min": -29.664268493652344,
      "activations/layer16_attention_weight_max": 33.056785583496094,
      "activations/layer16_attention_weight_min": -27.467958450317383,
      "activations/layer17_attention_weight_max": 55.118919372558594,
      "activations/layer17_attention_weight_min": -38.54343032836914,
      "activations/layer18_attention_weight_max": 49.76494598388672,
      "activations/layer18_attention_weight_min": -32.14636993408203,
      "activations/layer19_attention_weight_max": 22.00523567199707,
      "activations/layer19_attention_weight_min": -21.96526336669922,
      "activations/layer1_attention_weight_max": 16.49350357055664,
      "activations/layer1_attention_weight_min": -16.392908096313477,
      "activations/layer20_attention_weight_max": 21.97004508972168,
      "activations/layer20_attention_weight_min": -22.239036560058594,
      "activations/layer21_attention_weight_max": 38.940338134765625,
      "activations/layer21_attention_weight_min": -26.005260467529297,
      "activations/layer22_attention_weight_max": 33.93605041503906,
      "activations/layer22_attention_weight_min": -24.473896026611328,
      "activations/layer23_attention_weight_max": 28.97164535522461,
      "activations/layer23_attention_weight_min": -18.053049087524414,
      "activations/layer2_attention_weight_max": 29.979129791259766,
      "activations/layer2_attention_weight_min": -29.34932518005371,
      "activations/layer3_attention_weight_max": 79.8440933227539,
      "activations/layer3_attention_weight_min": -85.3099594116211,
      "activations/layer4_attention_weight_max": 101.14339447021484,
      "activations/layer4_attention_weight_min": -94.68416595458984,
      "activations/layer5_attention_weight_max": 103.06609344482422,
      "activations/layer5_attention_weight_min": -89.83336639404297,
      "activations/layer6_attention_weight_max": 60.076744079589844,
      "activations/layer6_attention_weight_min": -51.91121292114258,
      "activations/layer7_attention_weight_max": 68.42825317382812,
      "activations/layer7_attention_weight_min": -64.6830062866211,
      "activations/layer8_attention_weight_max": 43.6515998840332,
      "activations/layer8_attention_weight_min": -41.97045135498047,
      "activations/layer9_attention_weight_max": 46.83126449584961,
      "activations/layer9_attention_weight_min": -46.69816207885742,
      "epoch": 5.29,
      "learning_rate": 0.00011707992424242423,
      "loss": 2.8426,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.5818,
      "eval_samples_per_second": 500.363,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.5818,
      "eval_openwebtext_samples_per_second": 500.363,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_wikitext_loss": 3.033203125,
      "eval_wikitext_ppl": 20.763634702888442,
      "eval_wikitext_runtime": 2.0188,
      "eval_wikitext_samples_per_second": 225.875,
      "step": 91000
    },
    {
      "epoch": 5.29,
      "eval_lambada_loss": 2.79296875,
      "eval_lambada_ppl": 16.329425896569767,
      "eval_lambada_runtime": 9.604,
      "eval_lambada_samples_per_second": 506.976,
      "step": 91000
    },
    {
      "activations/layer0_attention_weight_max": 16.825231552124023,
      "activations/layer0_attention_weight_min": -14.783492088317871,
      "activations/layer10_attention_weight_max": 37.4898681640625,
      "activations/layer10_attention_weight_min": -34.842491149902344,
      "activations/layer11_attention_weight_max": 27.362918853759766,
      "activations/layer11_attention_weight_min": -30.449880599975586,
      "activations/layer12_attention_weight_max": 23.944156646728516,
      "activations/layer12_attention_weight_min": -27.169204711914062,
      "activations/layer13_attention_weight_max": 44.98650360107422,
      "activations/layer13_attention_weight_min": -32.883846282958984,
      "activations/layer14_attention_weight_max": 37.302772521972656,
      "activations/layer14_attention_weight_min": -30.869836807250977,
      "activations/layer15_attention_weight_max": 36.183475494384766,
      "activations/layer15_attention_weight_min": -30.872333526611328,
      "activations/layer16_attention_weight_max": 30.75999641418457,
      "activations/layer16_attention_weight_min": -24.904205322265625,
      "activations/layer17_attention_weight_max": 52.722618103027344,
      "activations/layer17_attention_weight_min": -39.4108772277832,
      "activations/layer18_attention_weight_max": 40.8464241027832,
      "activations/layer18_attention_weight_min": -30.959186553955078,
      "activations/layer19_attention_weight_max": 18.905376434326172,
      "activations/layer19_attention_weight_min": -21.425762176513672,
      "activations/layer1_attention_weight_max": 17.51775550842285,
      "activations/layer1_attention_weight_min": -15.831448554992676,
      "activations/layer20_attention_weight_max": 19.736013412475586,
      "activations/layer20_attention_weight_min": -24.92558479309082,
      "activations/layer21_attention_weight_max": 33.706661224365234,
      "activations/layer21_attention_weight_min": -25.210134506225586,
      "activations/layer22_attention_weight_max": 33.296688079833984,
      "activations/layer22_attention_weight_min": -22.255374908447266,
      "activations/layer23_attention_weight_max": 29.093612670898438,
      "activations/layer23_attention_weight_min": -16.272357940673828,
      "activations/layer2_attention_weight_max": 29.122482299804688,
      "activations/layer2_attention_weight_min": -26.656936645507812,
      "activations/layer3_attention_weight_max": 79.1047592163086,
      "activations/layer3_attention_weight_min": -81.79855346679688,
      "activations/layer4_attention_weight_max": 100.04926300048828,
      "activations/layer4_attention_weight_min": -92.26071166992188,
      "activations/layer5_attention_weight_max": 101.55900573730469,
      "activations/layer5_attention_weight_min": -89.95114135742188,
      "activations/layer6_attention_weight_max": 56.50663757324219,
      "activations/layer6_attention_weight_min": -52.095218658447266,
      "activations/layer7_attention_weight_max": 66.4671630859375,
      "activations/layer7_attention_weight_min": -65.57640075683594,
      "activations/layer8_attention_weight_max": 43.28267288208008,
      "activations/layer8_attention_weight_min": -41.70222854614258,
      "activations/layer9_attention_weight_max": 51.13108444213867,
      "activations/layer9_attention_weight_min": -47.46245193481445,
      "epoch": 5.29,
      "learning_rate": 0.00011706098484848484,
      "loss": 2.8649,
      "step": 91050
    },
    {
      "activations/layer0_attention_weight_max": 14.249224662780762,
      "activations/layer0_attention_weight_min": -14.305065155029297,
      "activations/layer10_attention_weight_max": 37.53178787231445,
      "activations/layer10_attention_weight_min": -32.163482666015625,
      "activations/layer11_attention_weight_max": 30.685317993164062,
      "activations/layer11_attention_weight_min": -27.693950653076172,
      "activations/layer12_attention_weight_max": 20.527921676635742,
      "activations/layer12_attention_weight_min": -24.158119201660156,
      "activations/layer13_attention_weight_max": 42.25586700439453,
      "activations/layer13_attention_weight_min": -36.873260498046875,
      "activations/layer14_attention_weight_max": 43.284122467041016,
      "activations/layer14_attention_weight_min": -30.9000244140625,
      "activations/layer15_attention_weight_max": 38.3966064453125,
      "activations/layer15_attention_weight_min": -28.94597053527832,
      "activations/layer16_attention_weight_max": 36.734004974365234,
      "activations/layer16_attention_weight_min": -24.99994468688965,
      "activations/layer17_attention_weight_max": 54.64762496948242,
      "activations/layer17_attention_weight_min": -41.47102737426758,
      "activations/layer18_attention_weight_max": 44.990333557128906,
      "activations/layer18_attention_weight_min": -33.03770446777344,
      "activations/layer19_attention_weight_max": 19.810855865478516,
      "activations/layer19_attention_weight_min": -19.736835479736328,
      "activations/layer1_attention_weight_max": 18.218660354614258,
      "activations/layer1_attention_weight_min": -14.63471794128418,
      "activations/layer20_attention_weight_max": 20.82661247253418,
      "activations/layer20_attention_weight_min": -21.41659927368164,
      "activations/layer21_attention_weight_max": 38.934810638427734,
      "activations/layer21_attention_weight_min": -26.119726181030273,
      "activations/layer22_attention_weight_max": 34.29966735839844,
      "activations/layer22_attention_weight_min": -22.46125030517578,
      "activations/layer23_attention_weight_max": 26.960247039794922,
      "activations/layer23_attention_weight_min": -20.423629760742188,
      "activations/layer2_attention_weight_max": 30.28936004638672,
      "activations/layer2_attention_weight_min": -28.353078842163086,
      "activations/layer3_attention_weight_max": 80.68878173828125,
      "activations/layer3_attention_weight_min": -84.07854461669922,
      "activations/layer4_attention_weight_max": 103.60639953613281,
      "activations/layer4_attention_weight_min": -93.09899139404297,
      "activations/layer5_attention_weight_max": 97.65831756591797,
      "activations/layer5_attention_weight_min": -90.25570678710938,
      "activations/layer6_attention_weight_max": 55.89820861816406,
      "activations/layer6_attention_weight_min": -50.26597213745117,
      "activations/layer7_attention_weight_max": 66.84422302246094,
      "activations/layer7_attention_weight_min": -66.7486572265625,
      "activations/layer8_attention_weight_max": 42.17173385620117,
      "activations/layer8_attention_weight_min": -41.105899810791016,
      "activations/layer9_attention_weight_max": 49.32575607299805,
      "activations/layer9_attention_weight_min": -45.59925842285156,
      "epoch": 5.29,
      "learning_rate": 0.00011704204545454545,
      "loss": 2.8368,
      "step": 91100
    },
    {
      "activations/layer0_attention_weight_max": 14.665726661682129,
      "activations/layer0_attention_weight_min": -15.3563232421875,
      "activations/layer10_attention_weight_max": 43.4690055847168,
      "activations/layer10_attention_weight_min": -42.19878005981445,
      "activations/layer11_attention_weight_max": 35.768592834472656,
      "activations/layer11_attention_weight_min": -33.79499435424805,
      "activations/layer12_attention_weight_max": 23.364715576171875,
      "activations/layer12_attention_weight_min": -24.284574508666992,
      "activations/layer13_attention_weight_max": 40.40296936035156,
      "activations/layer13_attention_weight_min": -33.5840950012207,
      "activations/layer14_attention_weight_max": 52.152137756347656,
      "activations/layer14_attention_weight_min": -43.07573318481445,
      "activations/layer15_attention_weight_max": 37.68838882446289,
      "activations/layer15_attention_weight_min": -33.00410461425781,
      "activations/layer16_attention_weight_max": 33.28373336791992,
      "activations/layer16_attention_weight_min": -25.752567291259766,
      "activations/layer17_attention_weight_max": 58.05537414550781,
      "activations/layer17_attention_weight_min": -43.80986404418945,
      "activations/layer18_attention_weight_max": 52.530982971191406,
      "activations/layer18_attention_weight_min": -35.59001922607422,
      "activations/layer19_attention_weight_max": 24.172542572021484,
      "activations/layer19_attention_weight_min": -20.89247703552246,
      "activations/layer1_attention_weight_max": 18.660173416137695,
      "activations/layer1_attention_weight_min": -16.459800720214844,
      "activations/layer20_attention_weight_max": 21.0496826171875,
      "activations/layer20_attention_weight_min": -19.70993995666504,
      "activations/layer21_attention_weight_max": 38.266517639160156,
      "activations/layer21_attention_weight_min": -24.985441207885742,
      "activations/layer22_attention_weight_max": 36.68064880371094,
      "activations/layer22_attention_weight_min": -22.48915672302246,
      "activations/layer23_attention_weight_max": 26.931243896484375,
      "activations/layer23_attention_weight_min": -17.934188842773438,
      "activations/layer2_attention_weight_max": 29.8802547454834,
      "activations/layer2_attention_weight_min": -27.791921615600586,
      "activations/layer3_attention_weight_max": 81.58580780029297,
      "activations/layer3_attention_weight_min": -88.29749298095703,
      "activations/layer4_attention_weight_max": 109.01275634765625,
      "activations/layer4_attention_weight_min": -100.10821533203125,
      "activations/layer5_attention_weight_max": 101.0248031616211,
      "activations/layer5_attention_weight_min": -90.22077941894531,
      "activations/layer6_attention_weight_max": 57.24534225463867,
      "activations/layer6_attention_weight_min": -51.90267562866211,
      "activations/layer7_attention_weight_max": 86.59716796875,
      "activations/layer7_attention_weight_min": -72.27843475341797,
      "activations/layer8_attention_weight_max": 55.3656005859375,
      "activations/layer8_attention_weight_min": -45.56646728515625,
      "activations/layer9_attention_weight_max": 74.83085632324219,
      "activations/layer9_attention_weight_min": -60.58170700073242,
      "epoch": 5.3,
      "learning_rate": 0.00011702310606060604,
      "loss": 2.8515,
      "step": 91150
    },
    {
      "activations/layer0_attention_weight_max": 14.023760795593262,
      "activations/layer0_attention_weight_min": -15.027251243591309,
      "activations/layer10_attention_weight_max": 41.96574783325195,
      "activations/layer10_attention_weight_min": -37.182735443115234,
      "activations/layer11_attention_weight_max": 30.39215660095215,
      "activations/layer11_attention_weight_min": -29.779464721679688,
      "activations/layer12_attention_weight_max": 23.238956451416016,
      "activations/layer12_attention_weight_min": -22.33994483947754,
      "activations/layer13_attention_weight_max": 50.36011505126953,
      "activations/layer13_attention_weight_min": -36.44725799560547,
      "activations/layer14_attention_weight_max": 49.73390197753906,
      "activations/layer14_attention_weight_min": -35.596839904785156,
      "activations/layer15_attention_weight_max": 44.509944915771484,
      "activations/layer15_attention_weight_min": -29.605154037475586,
      "activations/layer16_attention_weight_max": 33.17569351196289,
      "activations/layer16_attention_weight_min": -24.46493148803711,
      "activations/layer17_attention_weight_max": 58.2092170715332,
      "activations/layer17_attention_weight_min": -36.82862091064453,
      "activations/layer18_attention_weight_max": 48.82433319091797,
      "activations/layer18_attention_weight_min": -31.49542808532715,
      "activations/layer19_attention_weight_max": 20.47461700439453,
      "activations/layer19_attention_weight_min": -17.69537353515625,
      "activations/layer1_attention_weight_max": 18.135616302490234,
      "activations/layer1_attention_weight_min": -17.99761390686035,
      "activations/layer20_attention_weight_max": 21.978975296020508,
      "activations/layer20_attention_weight_min": -17.99998664855957,
      "activations/layer21_attention_weight_max": 40.88502883911133,
      "activations/layer21_attention_weight_min": -25.004783630371094,
      "activations/layer22_attention_weight_max": 34.42243957519531,
      "activations/layer22_attention_weight_min": -22.95176887512207,
      "activations/layer23_attention_weight_max": 30.213613510131836,
      "activations/layer23_attention_weight_min": -19.22528076171875,
      "activations/layer2_attention_weight_max": 34.8648567199707,
      "activations/layer2_attention_weight_min": -33.79621887207031,
      "activations/layer3_attention_weight_max": 87.10157775878906,
      "activations/layer3_attention_weight_min": -90.85066223144531,
      "activations/layer4_attention_weight_max": 104.25675201416016,
      "activations/layer4_attention_weight_min": -102.02764892578125,
      "activations/layer5_attention_weight_max": 102.49565887451172,
      "activations/layer5_attention_weight_min": -95.92724609375,
      "activations/layer6_attention_weight_max": 60.01472091674805,
      "activations/layer6_attention_weight_min": -55.34138870239258,
      "activations/layer7_attention_weight_max": 71.27391052246094,
      "activations/layer7_attention_weight_min": -72.70337677001953,
      "activations/layer8_attention_weight_max": 46.68115234375,
      "activations/layer8_attention_weight_min": -45.26011276245117,
      "activations/layer9_attention_weight_max": 56.513587951660156,
      "activations/layer9_attention_weight_min": -50.93722915649414,
      "epoch": 5.3,
      "learning_rate": 0.00011700416666666665,
      "loss": 2.8456,
      "step": 91200
    },
    {
      "activations/layer0_attention_weight_max": 14.05721664428711,
      "activations/layer0_attention_weight_min": -14.710892677307129,
      "activations/layer10_attention_weight_max": 37.46196365356445,
      "activations/layer10_attention_weight_min": -33.61776351928711,
      "activations/layer11_attention_weight_max": 30.727100372314453,
      "activations/layer11_attention_weight_min": -27.017684936523438,
      "activations/layer12_attention_weight_max": 28.97947883605957,
      "activations/layer12_attention_weight_min": -24.659820556640625,
      "activations/layer13_attention_weight_max": 40.11769485473633,
      "activations/layer13_attention_weight_min": -31.44808006286621,
      "activations/layer14_attention_weight_max": 35.21672058105469,
      "activations/layer14_attention_weight_min": -28.113950729370117,
      "activations/layer15_attention_weight_max": 40.17863845825195,
      "activations/layer15_attention_weight_min": -30.227519989013672,
      "activations/layer16_attention_weight_max": 33.18598556518555,
      "activations/layer16_attention_weight_min": -26.989330291748047,
      "activations/layer17_attention_weight_max": 55.36000442504883,
      "activations/layer17_attention_weight_min": -38.28728103637695,
      "activations/layer18_attention_weight_max": 45.268245697021484,
      "activations/layer18_attention_weight_min": -31.515235900878906,
      "activations/layer19_attention_weight_max": 20.811086654663086,
      "activations/layer19_attention_weight_min": -18.168182373046875,
      "activations/layer1_attention_weight_max": 16.630531311035156,
      "activations/layer1_attention_weight_min": -14.82995891571045,
      "activations/layer20_attention_weight_max": 21.16598892211914,
      "activations/layer20_attention_weight_min": -20.851194381713867,
      "activations/layer21_attention_weight_max": 31.801273345947266,
      "activations/layer21_attention_weight_min": -26.244644165039062,
      "activations/layer22_attention_weight_max": 31.47270393371582,
      "activations/layer22_attention_weight_min": -25.257287979125977,
      "activations/layer23_attention_weight_max": 27.836544036865234,
      "activations/layer23_attention_weight_min": -20.300521850585938,
      "activations/layer2_attention_weight_max": 29.63625144958496,
      "activations/layer2_attention_weight_min": -26.722373962402344,
      "activations/layer3_attention_weight_max": 76.76731872558594,
      "activations/layer3_attention_weight_min": -79.39727783203125,
      "activations/layer4_attention_weight_max": 93.78575134277344,
      "activations/layer4_attention_weight_min": -85.87296295166016,
      "activations/layer5_attention_weight_max": 100.98591613769531,
      "activations/layer5_attention_weight_min": -85.64166259765625,
      "activations/layer6_attention_weight_max": 57.40119171142578,
      "activations/layer6_attention_weight_min": -49.95328140258789,
      "activations/layer7_attention_weight_max": 73.33662414550781,
      "activations/layer7_attention_weight_min": -62.961830139160156,
      "activations/layer8_attention_weight_max": 42.51173782348633,
      "activations/layer8_attention_weight_min": -41.17820358276367,
      "activations/layer9_attention_weight_max": 52.27839279174805,
      "activations/layer9_attention_weight_min": -45.01829528808594,
      "epoch": 5.3,
      "learning_rate": 0.00011698522727272727,
      "loss": 2.8496,
      "step": 91250
    },
    {
      "activations/layer0_attention_weight_max": 14.347676277160645,
      "activations/layer0_attention_weight_min": -15.150467872619629,
      "activations/layer10_attention_weight_max": 36.3387565612793,
      "activations/layer10_attention_weight_min": -33.29905700683594,
      "activations/layer11_attention_weight_max": 27.510108947753906,
      "activations/layer11_attention_weight_min": -26.87579917907715,
      "activations/layer12_attention_weight_max": 24.137439727783203,
      "activations/layer12_attention_weight_min": -27.08475685119629,
      "activations/layer13_attention_weight_max": 47.2119026184082,
      "activations/layer13_attention_weight_min": -36.434932708740234,
      "activations/layer14_attention_weight_max": 43.57601547241211,
      "activations/layer14_attention_weight_min": -36.17653274536133,
      "activations/layer15_attention_weight_max": 41.29776382446289,
      "activations/layer15_attention_weight_min": -29.89677619934082,
      "activations/layer16_attention_weight_max": 36.044342041015625,
      "activations/layer16_attention_weight_min": -26.661788940429688,
      "activations/layer17_attention_weight_max": 64.7138442993164,
      "activations/layer17_attention_weight_min": -38.471717834472656,
      "activations/layer18_attention_weight_max": 54.0745735168457,
      "activations/layer18_attention_weight_min": -32.5059928894043,
      "activations/layer19_attention_weight_max": 22.893949508666992,
      "activations/layer19_attention_weight_min": -17.938621520996094,
      "activations/layer1_attention_weight_max": 17.9948673248291,
      "activations/layer1_attention_weight_min": -15.95192813873291,
      "activations/layer20_attention_weight_max": 20.71617889404297,
      "activations/layer20_attention_weight_min": -20.196916580200195,
      "activations/layer21_attention_weight_max": 42.31716537475586,
      "activations/layer21_attention_weight_min": -23.430612564086914,
      "activations/layer22_attention_weight_max": 33.432865142822266,
      "activations/layer22_attention_weight_min": -22.007253646850586,
      "activations/layer23_attention_weight_max": 27.151586532592773,
      "activations/layer23_attention_weight_min": -17.669780731201172,
      "activations/layer2_attention_weight_max": 32.1724853515625,
      "activations/layer2_attention_weight_min": -29.239667892456055,
      "activations/layer3_attention_weight_max": 88.69878387451172,
      "activations/layer3_attention_weight_min": -92.7409896850586,
      "activations/layer4_attention_weight_max": 105.7036361694336,
      "activations/layer4_attention_weight_min": -92.72688293457031,
      "activations/layer5_attention_weight_max": 100.50164031982422,
      "activations/layer5_attention_weight_min": -87.8765869140625,
      "activations/layer6_attention_weight_max": 57.778099060058594,
      "activations/layer6_attention_weight_min": -51.360591888427734,
      "activations/layer7_attention_weight_max": 71.46673583984375,
      "activations/layer7_attention_weight_min": -62.818992614746094,
      "activations/layer8_attention_weight_max": 40.85735321044922,
      "activations/layer8_attention_weight_min": -39.74876022338867,
      "activations/layer9_attention_weight_max": 52.149566650390625,
      "activations/layer9_attention_weight_min": -45.146575927734375,
      "epoch": 5.31,
      "learning_rate": 0.00011696628787878786,
      "loss": 2.8391,
      "step": 91300
    },
    {
      "activations/layer0_attention_weight_max": 14.901986122131348,
      "activations/layer0_attention_weight_min": -15.187640190124512,
      "activations/layer10_attention_weight_max": 36.850006103515625,
      "activations/layer10_attention_weight_min": -33.863407135009766,
      "activations/layer11_attention_weight_max": 31.571971893310547,
      "activations/layer11_attention_weight_min": -26.91664695739746,
      "activations/layer12_attention_weight_max": 23.187192916870117,
      "activations/layer12_attention_weight_min": -31.453758239746094,
      "activations/layer13_attention_weight_max": 47.04536437988281,
      "activations/layer13_attention_weight_min": -31.912277221679688,
      "activations/layer14_attention_weight_max": 48.460243225097656,
      "activations/layer14_attention_weight_min": -32.39005661010742,
      "activations/layer15_attention_weight_max": 39.51042938232422,
      "activations/layer15_attention_weight_min": -28.843917846679688,
      "activations/layer16_attention_weight_max": 33.26354217529297,
      "activations/layer16_attention_weight_min": -27.681507110595703,
      "activations/layer17_attention_weight_max": 52.16944122314453,
      "activations/layer17_attention_weight_min": -37.630802154541016,
      "activations/layer18_attention_weight_max": 45.42058563232422,
      "activations/layer18_attention_weight_min": -33.73704147338867,
      "activations/layer19_attention_weight_max": 23.078922271728516,
      "activations/layer19_attention_weight_min": -22.014951705932617,
      "activations/layer1_attention_weight_max": 17.328636169433594,
      "activations/layer1_attention_weight_min": -16.5013484954834,
      "activations/layer20_attention_weight_max": 19.926836013793945,
      "activations/layer20_attention_weight_min": -20.199033737182617,
      "activations/layer21_attention_weight_max": 45.899173736572266,
      "activations/layer21_attention_weight_min": -23.890329360961914,
      "activations/layer22_attention_weight_max": 31.783260345458984,
      "activations/layer22_attention_weight_min": -23.196104049682617,
      "activations/layer23_attention_weight_max": 29.83400535583496,
      "activations/layer23_attention_weight_min": -18.92095184326172,
      "activations/layer2_attention_weight_max": 30.39967918395996,
      "activations/layer2_attention_weight_min": -27.959156036376953,
      "activations/layer3_attention_weight_max": 81.84318542480469,
      "activations/layer3_attention_weight_min": -83.9923095703125,
      "activations/layer4_attention_weight_max": 97.0768051147461,
      "activations/layer4_attention_weight_min": -94.4157943725586,
      "activations/layer5_attention_weight_max": 96.3753890991211,
      "activations/layer5_attention_weight_min": -85.04278564453125,
      "activations/layer6_attention_weight_max": 53.2149543762207,
      "activations/layer6_attention_weight_min": -50.92014694213867,
      "activations/layer7_attention_weight_max": 65.62601470947266,
      "activations/layer7_attention_weight_min": -62.88222885131836,
      "activations/layer8_attention_weight_max": 40.5681266784668,
      "activations/layer8_attention_weight_min": -39.00809097290039,
      "activations/layer9_attention_weight_max": 51.665679931640625,
      "activations/layer9_attention_weight_min": -46.58934020996094,
      "epoch": 5.31,
      "learning_rate": 0.00011694734848484847,
      "loss": 2.8296,
      "step": 91350
    },
    {
      "activations/layer0_attention_weight_max": 16.702775955200195,
      "activations/layer0_attention_weight_min": -15.474250793457031,
      "activations/layer10_attention_weight_max": 35.456024169921875,
      "activations/layer10_attention_weight_min": -33.1634521484375,
      "activations/layer11_attention_weight_max": 28.68246078491211,
      "activations/layer11_attention_weight_min": -26.053749084472656,
      "activations/layer12_attention_weight_max": 28.18025016784668,
      "activations/layer12_attention_weight_min": -24.70194435119629,
      "activations/layer13_attention_weight_max": 51.559242248535156,
      "activations/layer13_attention_weight_min": -37.83687973022461,
      "activations/layer14_attention_weight_max": 41.454193115234375,
      "activations/layer14_attention_weight_min": -31.188188552856445,
      "activations/layer15_attention_weight_max": 38.477420806884766,
      "activations/layer15_attention_weight_min": -30.409128189086914,
      "activations/layer16_attention_weight_max": 34.242774963378906,
      "activations/layer16_attention_weight_min": -26.71986198425293,
      "activations/layer17_attention_weight_max": 55.874786376953125,
      "activations/layer17_attention_weight_min": -39.9720344543457,
      "activations/layer18_attention_weight_max": 51.24708938598633,
      "activations/layer18_attention_weight_min": -36.125267028808594,
      "activations/layer19_attention_weight_max": 20.59410285949707,
      "activations/layer19_attention_weight_min": -18.588960647583008,
      "activations/layer1_attention_weight_max": 16.44584083557129,
      "activations/layer1_attention_weight_min": -14.278708457946777,
      "activations/layer20_attention_weight_max": 20.803796768188477,
      "activations/layer20_attention_weight_min": -21.515159606933594,
      "activations/layer21_attention_weight_max": 40.18566131591797,
      "activations/layer21_attention_weight_min": -25.16381072998047,
      "activations/layer22_attention_weight_max": 36.746490478515625,
      "activations/layer22_attention_weight_min": -25.09518051147461,
      "activations/layer23_attention_weight_max": 28.220550537109375,
      "activations/layer23_attention_weight_min": -18.45094871520996,
      "activations/layer2_attention_weight_max": 31.17371368408203,
      "activations/layer2_attention_weight_min": -28.156301498413086,
      "activations/layer3_attention_weight_max": 80.25773620605469,
      "activations/layer3_attention_weight_min": -81.44343566894531,
      "activations/layer4_attention_weight_max": 99.8143539428711,
      "activations/layer4_attention_weight_min": -92.61038208007812,
      "activations/layer5_attention_weight_max": 100.32302856445312,
      "activations/layer5_attention_weight_min": -88.38445281982422,
      "activations/layer6_attention_weight_max": 56.503265380859375,
      "activations/layer6_attention_weight_min": -51.80274200439453,
      "activations/layer7_attention_weight_max": 70.12388610839844,
      "activations/layer7_attention_weight_min": -65.16301727294922,
      "activations/layer8_attention_weight_max": 43.14112091064453,
      "activations/layer8_attention_weight_min": -42.27242660522461,
      "activations/layer9_attention_weight_max": 48.814720153808594,
      "activations/layer9_attention_weight_min": -46.75423812866211,
      "epoch": 5.31,
      "learning_rate": 0.00011692878787878786,
      "loss": 2.8418,
      "step": 91400
    },
    {
      "activations/layer0_attention_weight_max": 14.216897964477539,
      "activations/layer0_attention_weight_min": -16.13571548461914,
      "activations/layer10_attention_weight_max": 34.954105377197266,
      "activations/layer10_attention_weight_min": -32.07965087890625,
      "activations/layer11_attention_weight_max": 26.83173370361328,
      "activations/layer11_attention_weight_min": -26.655864715576172,
      "activations/layer12_attention_weight_max": 23.722944259643555,
      "activations/layer12_attention_weight_min": -29.425962448120117,
      "activations/layer13_attention_weight_max": 50.92487716674805,
      "activations/layer13_attention_weight_min": -37.29087829589844,
      "activations/layer14_attention_weight_max": 44.13639450073242,
      "activations/layer14_attention_weight_min": -28.274105072021484,
      "activations/layer15_attention_weight_max": 41.16344451904297,
      "activations/layer15_attention_weight_min": -30.21388053894043,
      "activations/layer16_attention_weight_max": 35.638389587402344,
      "activations/layer16_attention_weight_min": -26.32163429260254,
      "activations/layer17_attention_weight_max": 53.95844268798828,
      "activations/layer17_attention_weight_min": -38.90578842163086,
      "activations/layer18_attention_weight_max": 51.83582305908203,
      "activations/layer18_attention_weight_min": -33.62281036376953,
      "activations/layer19_attention_weight_max": 23.98090171813965,
      "activations/layer19_attention_weight_min": -20.647722244262695,
      "activations/layer1_attention_weight_max": 17.862581253051758,
      "activations/layer1_attention_weight_min": -15.893425941467285,
      "activations/layer20_attention_weight_max": 21.731895446777344,
      "activations/layer20_attention_weight_min": -18.8050594329834,
      "activations/layer21_attention_weight_max": 35.6088981628418,
      "activations/layer21_attention_weight_min": -24.94891929626465,
      "activations/layer22_attention_weight_max": 33.44873046875,
      "activations/layer22_attention_weight_min": -25.099451065063477,
      "activations/layer23_attention_weight_max": 27.870725631713867,
      "activations/layer23_attention_weight_min": -17.612680435180664,
      "activations/layer2_attention_weight_max": 28.471012115478516,
      "activations/layer2_attention_weight_min": -27.155738830566406,
      "activations/layer3_attention_weight_max": 81.92160034179688,
      "activations/layer3_attention_weight_min": -86.159423828125,
      "activations/layer4_attention_weight_max": 96.60504150390625,
      "activations/layer4_attention_weight_min": -89.3791732788086,
      "activations/layer5_attention_weight_max": 98.28211975097656,
      "activations/layer5_attention_weight_min": -85.3289794921875,
      "activations/layer6_attention_weight_max": 54.26176834106445,
      "activations/layer6_attention_weight_min": -50.23772048950195,
      "activations/layer7_attention_weight_max": 68.57856750488281,
      "activations/layer7_attention_weight_min": -63.46476745605469,
      "activations/layer8_attention_weight_max": 40.31235122680664,
      "activations/layer8_attention_weight_min": -41.36940002441406,
      "activations/layer9_attention_weight_max": 50.42463684082031,
      "activations/layer9_attention_weight_min": -45.68007278442383,
      "epoch": 5.31,
      "learning_rate": 0.00011690984848484848,
      "loss": 2.8328,
      "step": 91450
    },
    {
      "activations/layer0_attention_weight_max": 14.494836807250977,
      "activations/layer0_attention_weight_min": -15.334010124206543,
      "activations/layer10_attention_weight_max": 37.045379638671875,
      "activations/layer10_attention_weight_min": -32.809783935546875,
      "activations/layer11_attention_weight_max": 27.966110229492188,
      "activations/layer11_attention_weight_min": -28.06557846069336,
      "activations/layer12_attention_weight_max": 20.4836368560791,
      "activations/layer12_attention_weight_min": -27.055177688598633,
      "activations/layer13_attention_weight_max": 50.14900588989258,
      "activations/layer13_attention_weight_min": -41.003997802734375,
      "activations/layer14_attention_weight_max": 40.53384780883789,
      "activations/layer14_attention_weight_min": -32.64975357055664,
      "activations/layer15_attention_weight_max": 39.488441467285156,
      "activations/layer15_attention_weight_min": -31.116504669189453,
      "activations/layer16_attention_weight_max": 32.282047271728516,
      "activations/layer16_attention_weight_min": -28.31273078918457,
      "activations/layer17_attention_weight_max": 57.26477813720703,
      "activations/layer17_attention_weight_min": -37.64514923095703,
      "activations/layer18_attention_weight_max": 47.80284118652344,
      "activations/layer18_attention_weight_min": -33.86351776123047,
      "activations/layer19_attention_weight_max": 23.627649307250977,
      "activations/layer19_attention_weight_min": -21.068771362304688,
      "activations/layer1_attention_weight_max": 17.23252296447754,
      "activations/layer1_attention_weight_min": -16.599668502807617,
      "activations/layer20_attention_weight_max": 22.236860275268555,
      "activations/layer20_attention_weight_min": -22.580259323120117,
      "activations/layer21_attention_weight_max": 36.88765335083008,
      "activations/layer21_attention_weight_min": -27.57891845703125,
      "activations/layer22_attention_weight_max": 34.48686218261719,
      "activations/layer22_attention_weight_min": -26.259756088256836,
      "activations/layer23_attention_weight_max": 29.105688095092773,
      "activations/layer23_attention_weight_min": -19.059104919433594,
      "activations/layer2_attention_weight_max": 30.272573471069336,
      "activations/layer2_attention_weight_min": -27.634727478027344,
      "activations/layer3_attention_weight_max": 80.56083679199219,
      "activations/layer3_attention_weight_min": -80.76089477539062,
      "activations/layer4_attention_weight_max": 102.16473388671875,
      "activations/layer4_attention_weight_min": -93.85462188720703,
      "activations/layer5_attention_weight_max": 102.35955810546875,
      "activations/layer5_attention_weight_min": -93.80205535888672,
      "activations/layer6_attention_weight_max": 57.456138610839844,
      "activations/layer6_attention_weight_min": -54.00166320800781,
      "activations/layer7_attention_weight_max": 71.92139434814453,
      "activations/layer7_attention_weight_min": -67.71871948242188,
      "activations/layer8_attention_weight_max": 40.9635124206543,
      "activations/layer8_attention_weight_min": -40.57124710083008,
      "activations/layer9_attention_weight_max": 53.51639938354492,
      "activations/layer9_attention_weight_min": -46.246498107910156,
      "epoch": 5.32,
      "learning_rate": 0.00011689090909090908,
      "loss": 2.8436,
      "step": 91500
    },
    {
      "activations/layer0_attention_weight_max": 14.316904067993164,
      "activations/layer0_attention_weight_min": -14.039994239807129,
      "activations/layer10_attention_weight_max": 38.09773635864258,
      "activations/layer10_attention_weight_min": -34.48468780517578,
      "activations/layer11_attention_weight_max": 28.955650329589844,
      "activations/layer11_attention_weight_min": -27.66132164001465,
      "activations/layer12_attention_weight_max": 22.27188491821289,
      "activations/layer12_attention_weight_min": -25.967384338378906,
      "activations/layer13_attention_weight_max": 47.6336784362793,
      "activations/layer13_attention_weight_min": -38.24870300292969,
      "activations/layer14_attention_weight_max": 43.78904724121094,
      "activations/layer14_attention_weight_min": -29.269832611083984,
      "activations/layer15_attention_weight_max": 38.95161819458008,
      "activations/layer15_attention_weight_min": -29.22687339782715,
      "activations/layer16_attention_weight_max": 34.09020233154297,
      "activations/layer16_attention_weight_min": -27.239011764526367,
      "activations/layer17_attention_weight_max": 54.965572357177734,
      "activations/layer17_attention_weight_min": -39.46904373168945,
      "activations/layer18_attention_weight_max": 50.3045654296875,
      "activations/layer18_attention_weight_min": -33.254730224609375,
      "activations/layer19_attention_weight_max": 25.82077407836914,
      "activations/layer19_attention_weight_min": -18.794828414916992,
      "activations/layer1_attention_weight_max": 18.876909255981445,
      "activations/layer1_attention_weight_min": -14.452787399291992,
      "activations/layer20_attention_weight_max": 20.180644989013672,
      "activations/layer20_attention_weight_min": -21.071054458618164,
      "activations/layer21_attention_weight_max": 36.2778205871582,
      "activations/layer21_attention_weight_min": -28.447866439819336,
      "activations/layer22_attention_weight_max": 32.37156295776367,
      "activations/layer22_attention_weight_min": -25.96602439880371,
      "activations/layer23_attention_weight_max": 26.34395408630371,
      "activations/layer23_attention_weight_min": -17.69068717956543,
      "activations/layer2_attention_weight_max": 28.500499725341797,
      "activations/layer2_attention_weight_min": -26.641403198242188,
      "activations/layer3_attention_weight_max": 74.52862548828125,
      "activations/layer3_attention_weight_min": -75.41793060302734,
      "activations/layer4_attention_weight_max": 94.36418151855469,
      "activations/layer4_attention_weight_min": -87.37442779541016,
      "activations/layer5_attention_weight_max": 95.84066009521484,
      "activations/layer5_attention_weight_min": -86.7205810546875,
      "activations/layer6_attention_weight_max": 55.304161071777344,
      "activations/layer6_attention_weight_min": -54.49361038208008,
      "activations/layer7_attention_weight_max": 69.39025115966797,
      "activations/layer7_attention_weight_min": -63.39722442626953,
      "activations/layer8_attention_weight_max": 42.00374221801758,
      "activations/layer8_attention_weight_min": -40.318031311035156,
      "activations/layer9_attention_weight_max": 50.013790130615234,
      "activations/layer9_attention_weight_min": -43.36760330200195,
      "epoch": 5.32,
      "learning_rate": 0.00011687196969696968,
      "loss": 2.8412,
      "step": 91550
    },
    {
      "activations/layer0_attention_weight_max": 15.202536582946777,
      "activations/layer0_attention_weight_min": -14.983785629272461,
      "activations/layer10_attention_weight_max": 39.8373908996582,
      "activations/layer10_attention_weight_min": -32.76042556762695,
      "activations/layer11_attention_weight_max": 28.199668884277344,
      "activations/layer11_attention_weight_min": -27.146347045898438,
      "activations/layer12_attention_weight_max": 25.06688690185547,
      "activations/layer12_attention_weight_min": -22.75482749938965,
      "activations/layer13_attention_weight_max": 53.48902893066406,
      "activations/layer13_attention_weight_min": -33.02516555786133,
      "activations/layer14_attention_weight_max": 50.572242736816406,
      "activations/layer14_attention_weight_min": -31.886272430419922,
      "activations/layer15_attention_weight_max": 38.733402252197266,
      "activations/layer15_attention_weight_min": -31.58872413635254,
      "activations/layer16_attention_weight_max": 37.74469757080078,
      "activations/layer16_attention_weight_min": -26.055009841918945,
      "activations/layer17_attention_weight_max": 59.10428237915039,
      "activations/layer17_attention_weight_min": -40.769004821777344,
      "activations/layer18_attention_weight_max": 51.61832809448242,
      "activations/layer18_attention_weight_min": -34.139251708984375,
      "activations/layer19_attention_weight_max": 21.251554489135742,
      "activations/layer19_attention_weight_min": -19.572141647338867,
      "activations/layer1_attention_weight_max": 16.978412628173828,
      "activations/layer1_attention_weight_min": -17.791975021362305,
      "activations/layer20_attention_weight_max": 24.2889461517334,
      "activations/layer20_attention_weight_min": -21.575294494628906,
      "activations/layer21_attention_weight_max": 39.48828125,
      "activations/layer21_attention_weight_min": -26.33694839477539,
      "activations/layer22_attention_weight_max": 37.900936126708984,
      "activations/layer22_attention_weight_min": -24.45256805419922,
      "activations/layer23_attention_weight_max": 30.540220260620117,
      "activations/layer23_attention_weight_min": -16.74578857421875,
      "activations/layer2_attention_weight_max": 29.637969970703125,
      "activations/layer2_attention_weight_min": -27.880573272705078,
      "activations/layer3_attention_weight_max": 78.32876586914062,
      "activations/layer3_attention_weight_min": -84.8794937133789,
      "activations/layer4_attention_weight_max": 96.79363250732422,
      "activations/layer4_attention_weight_min": -92.20039367675781,
      "activations/layer5_attention_weight_max": 99.14485931396484,
      "activations/layer5_attention_weight_min": -88.49171447753906,
      "activations/layer6_attention_weight_max": 56.85986328125,
      "activations/layer6_attention_weight_min": -48.61212921142578,
      "activations/layer7_attention_weight_max": 67.99385070800781,
      "activations/layer7_attention_weight_min": -62.62720489501953,
      "activations/layer8_attention_weight_max": 40.05189514160156,
      "activations/layer8_attention_weight_min": -41.92622756958008,
      "activations/layer9_attention_weight_max": 46.22187042236328,
      "activations/layer9_attention_weight_min": -42.958919525146484,
      "epoch": 5.32,
      "learning_rate": 0.0001168530303030303,
      "loss": 2.8505,
      "step": 91600
    },
    {
      "activations/layer0_attention_weight_max": 14.155208587646484,
      "activations/layer0_attention_weight_min": -15.392701148986816,
      "activations/layer10_attention_weight_max": 39.774688720703125,
      "activations/layer10_attention_weight_min": -34.83229064941406,
      "activations/layer11_attention_weight_max": 31.53795623779297,
      "activations/layer11_attention_weight_min": -28.255573272705078,
      "activations/layer12_attention_weight_max": 22.90224838256836,
      "activations/layer12_attention_weight_min": -26.267642974853516,
      "activations/layer13_attention_weight_max": 52.85557556152344,
      "activations/layer13_attention_weight_min": -34.191123962402344,
      "activations/layer14_attention_weight_max": 53.78281021118164,
      "activations/layer14_attention_weight_min": -35.18478012084961,
      "activations/layer15_attention_weight_max": 42.236297607421875,
      "activations/layer15_attention_weight_min": -30.324710845947266,
      "activations/layer16_attention_weight_max": 37.01856994628906,
      "activations/layer16_attention_weight_min": -28.85069465637207,
      "activations/layer17_attention_weight_max": 61.72294998168945,
      "activations/layer17_attention_weight_min": -41.29335021972656,
      "activations/layer18_attention_weight_max": 54.54895782470703,
      "activations/layer18_attention_weight_min": -36.81106185913086,
      "activations/layer19_attention_weight_max": 25.6120548248291,
      "activations/layer19_attention_weight_min": -19.878864288330078,
      "activations/layer1_attention_weight_max": 17.979633331298828,
      "activations/layer1_attention_weight_min": -14.497163772583008,
      "activations/layer20_attention_weight_max": 22.424028396606445,
      "activations/layer20_attention_weight_min": -21.986665725708008,
      "activations/layer21_attention_weight_max": 40.951412200927734,
      "activations/layer21_attention_weight_min": -24.256614685058594,
      "activations/layer22_attention_weight_max": 36.65860366821289,
      "activations/layer22_attention_weight_min": -24.4766845703125,
      "activations/layer23_attention_weight_max": 28.50223159790039,
      "activations/layer23_attention_weight_min": -18.245628356933594,
      "activations/layer2_attention_weight_max": 29.673070907592773,
      "activations/layer2_attention_weight_min": -25.524215698242188,
      "activations/layer3_attention_weight_max": 78.35704040527344,
      "activations/layer3_attention_weight_min": -78.4758529663086,
      "activations/layer4_attention_weight_max": 102.20277404785156,
      "activations/layer4_attention_weight_min": -92.74932098388672,
      "activations/layer5_attention_weight_max": 108.9112548828125,
      "activations/layer5_attention_weight_min": -91.3989486694336,
      "activations/layer6_attention_weight_max": 61.83506393432617,
      "activations/layer6_attention_weight_min": -53.65427780151367,
      "activations/layer7_attention_weight_max": 74.41932678222656,
      "activations/layer7_attention_weight_min": -70.60224151611328,
      "activations/layer8_attention_weight_max": 44.4019660949707,
      "activations/layer8_attention_weight_min": -43.206424713134766,
      "activations/layer9_attention_weight_max": 53.137935638427734,
      "activations/layer9_attention_weight_min": -47.968109130859375,
      "epoch": 5.33,
      "learning_rate": 0.0001168340909090909,
      "loss": 2.8518,
      "step": 91650
    },
    {
      "activations/layer0_attention_weight_max": 15.490531921386719,
      "activations/layer0_attention_weight_min": -14.825121879577637,
      "activations/layer10_attention_weight_max": 40.18839645385742,
      "activations/layer10_attention_weight_min": -36.29290008544922,
      "activations/layer11_attention_weight_max": 31.75044822692871,
      "activations/layer11_attention_weight_min": -27.665843963623047,
      "activations/layer12_attention_weight_max": 21.949182510375977,
      "activations/layer12_attention_weight_min": -24.69175148010254,
      "activations/layer13_attention_weight_max": 52.166744232177734,
      "activations/layer13_attention_weight_min": -33.71957015991211,
      "activations/layer14_attention_weight_max": 44.3277702331543,
      "activations/layer14_attention_weight_min": -29.922163009643555,
      "activations/layer15_attention_weight_max": 44.339229583740234,
      "activations/layer15_attention_weight_min": -28.42585563659668,
      "activations/layer16_attention_weight_max": 34.83578872680664,
      "activations/layer16_attention_weight_min": -27.47848129272461,
      "activations/layer17_attention_weight_max": 56.94654846191406,
      "activations/layer17_attention_weight_min": -40.08876419067383,
      "activations/layer18_attention_weight_max": 50.41600799560547,
      "activations/layer18_attention_weight_min": -34.10771942138672,
      "activations/layer19_attention_weight_max": 22.978513717651367,
      "activations/layer19_attention_weight_min": -21.11661148071289,
      "activations/layer1_attention_weight_max": 16.71815299987793,
      "activations/layer1_attention_weight_min": -17.78620147705078,
      "activations/layer20_attention_weight_max": 24.193363189697266,
      "activations/layer20_attention_weight_min": -21.310056686401367,
      "activations/layer21_attention_weight_max": 47.36124801635742,
      "activations/layer21_attention_weight_min": -26.22684097290039,
      "activations/layer22_attention_weight_max": 34.54424285888672,
      "activations/layer22_attention_weight_min": -23.44309425354004,
      "activations/layer23_attention_weight_max": 28.721698760986328,
      "activations/layer23_attention_weight_min": -19.57352638244629,
      "activations/layer2_attention_weight_max": 30.58160400390625,
      "activations/layer2_attention_weight_min": -30.34730339050293,
      "activations/layer3_attention_weight_max": 79.42832946777344,
      "activations/layer3_attention_weight_min": -84.49203491210938,
      "activations/layer4_attention_weight_max": 98.09252166748047,
      "activations/layer4_attention_weight_min": -94.14305877685547,
      "activations/layer5_attention_weight_max": 97.61013793945312,
      "activations/layer5_attention_weight_min": -89.5018310546875,
      "activations/layer6_attention_weight_max": 62.8438835144043,
      "activations/layer6_attention_weight_min": -50.31523513793945,
      "activations/layer7_attention_weight_max": 72.37980651855469,
      "activations/layer7_attention_weight_min": -71.55142211914062,
      "activations/layer8_attention_weight_max": 51.3908576965332,
      "activations/layer8_attention_weight_min": -43.30610275268555,
      "activations/layer9_attention_weight_max": 54.069580078125,
      "activations/layer9_attention_weight_min": -46.38145446777344,
      "epoch": 5.33,
      "learning_rate": 0.00011681515151515151,
      "loss": 2.8603,
      "step": 91700
    },
    {
      "activations/layer0_attention_weight_max": 15.023022651672363,
      "activations/layer0_attention_weight_min": -15.787842750549316,
      "activations/layer10_attention_weight_max": 40.328609466552734,
      "activations/layer10_attention_weight_min": -33.92635726928711,
      "activations/layer11_attention_weight_max": 32.171669006347656,
      "activations/layer11_attention_weight_min": -26.96931266784668,
      "activations/layer12_attention_weight_max": 20.768539428710938,
      "activations/layer12_attention_weight_min": -25.603370666503906,
      "activations/layer13_attention_weight_max": 51.778526306152344,
      "activations/layer13_attention_weight_min": -36.20289993286133,
      "activations/layer14_attention_weight_max": 45.011566162109375,
      "activations/layer14_attention_weight_min": -32.18924331665039,
      "activations/layer15_attention_weight_max": 36.306400299072266,
      "activations/layer15_attention_weight_min": -28.4916934967041,
      "activations/layer16_attention_weight_max": 36.37887954711914,
      "activations/layer16_attention_weight_min": -26.877620697021484,
      "activations/layer17_attention_weight_max": 56.114662170410156,
      "activations/layer17_attention_weight_min": -36.81393051147461,
      "activations/layer18_attention_weight_max": 50.39915466308594,
      "activations/layer18_attention_weight_min": -31.7003173828125,
      "activations/layer19_attention_weight_max": 25.779775619506836,
      "activations/layer19_attention_weight_min": -21.616735458374023,
      "activations/layer1_attention_weight_max": 17.816484451293945,
      "activations/layer1_attention_weight_min": -16.120561599731445,
      "activations/layer20_attention_weight_max": 24.065290451049805,
      "activations/layer20_attention_weight_min": -23.08169937133789,
      "activations/layer21_attention_weight_max": 41.3050537109375,
      "activations/layer21_attention_weight_min": -25.710474014282227,
      "activations/layer22_attention_weight_max": 37.18584060668945,
      "activations/layer22_attention_weight_min": -25.876441955566406,
      "activations/layer23_attention_weight_max": 31.62013053894043,
      "activations/layer23_attention_weight_min": -19.61728286743164,
      "activations/layer2_attention_weight_max": 31.160293579101562,
      "activations/layer2_attention_weight_min": -29.489988327026367,
      "activations/layer3_attention_weight_max": 86.7401123046875,
      "activations/layer3_attention_weight_min": -89.81965637207031,
      "activations/layer4_attention_weight_max": 108.9260025024414,
      "activations/layer4_attention_weight_min": -105.39830780029297,
      "activations/layer5_attention_weight_max": 102.88728332519531,
      "activations/layer5_attention_weight_min": -99.83839416503906,
      "activations/layer6_attention_weight_max": 57.178955078125,
      "activations/layer6_attention_weight_min": -52.02963638305664,
      "activations/layer7_attention_weight_max": 78.09765625,
      "activations/layer7_attention_weight_min": -72.08140563964844,
      "activations/layer8_attention_weight_max": 47.84560012817383,
      "activations/layer8_attention_weight_min": -40.3743782043457,
      "activations/layer9_attention_weight_max": 58.535919189453125,
      "activations/layer9_attention_weight_min": -48.36741256713867,
      "epoch": 5.33,
      "learning_rate": 0.0001167965909090909,
      "loss": 2.8477,
      "step": 91750
    },
    {
      "activations/layer0_attention_weight_max": 14.304398536682129,
      "activations/layer0_attention_weight_min": -15.054842948913574,
      "activations/layer10_attention_weight_max": 39.09638214111328,
      "activations/layer10_attention_weight_min": -32.51866149902344,
      "activations/layer11_attention_weight_max": 29.47347640991211,
      "activations/layer11_attention_weight_min": -25.875974655151367,
      "activations/layer12_attention_weight_max": 24.913558959960938,
      "activations/layer12_attention_weight_min": -25.21741485595703,
      "activations/layer13_attention_weight_max": 46.60289001464844,
      "activations/layer13_attention_weight_min": -30.990402221679688,
      "activations/layer14_attention_weight_max": 44.57763671875,
      "activations/layer14_attention_weight_min": -29.204477310180664,
      "activations/layer15_attention_weight_max": 41.12086486816406,
      "activations/layer15_attention_weight_min": -31.619474411010742,
      "activations/layer16_attention_weight_max": 38.369075775146484,
      "activations/layer16_attention_weight_min": -26.432254791259766,
      "activations/layer17_attention_weight_max": 63.60089111328125,
      "activations/layer17_attention_weight_min": -38.818546295166016,
      "activations/layer18_attention_weight_max": 53.076087951660156,
      "activations/layer18_attention_weight_min": -34.07225799560547,
      "activations/layer19_attention_weight_max": 21.93250846862793,
      "activations/layer19_attention_weight_min": -19.865022659301758,
      "activations/layer1_attention_weight_max": 16.71210479736328,
      "activations/layer1_attention_weight_min": -14.450310707092285,
      "activations/layer20_attention_weight_max": 23.005041122436523,
      "activations/layer20_attention_weight_min": -23.441490173339844,
      "activations/layer21_attention_weight_max": 47.29825973510742,
      "activations/layer21_attention_weight_min": -27.719083786010742,
      "activations/layer22_attention_weight_max": 34.380313873291016,
      "activations/layer22_attention_weight_min": -24.88113021850586,
      "activations/layer23_attention_weight_max": 29.491987228393555,
      "activations/layer23_attention_weight_min": -20.088443756103516,
      "activations/layer2_attention_weight_max": 29.336135864257812,
      "activations/layer2_attention_weight_min": -26.867706298828125,
      "activations/layer3_attention_weight_max": 77.1272964477539,
      "activations/layer3_attention_weight_min": -77.5882339477539,
      "activations/layer4_attention_weight_max": 96.35309600830078,
      "activations/layer4_attention_weight_min": -92.39142608642578,
      "activations/layer5_attention_weight_max": 97.79031372070312,
      "activations/layer5_attention_weight_min": -87.74766540527344,
      "activations/layer6_attention_weight_max": 56.057098388671875,
      "activations/layer6_attention_weight_min": -49.54779815673828,
      "activations/layer7_attention_weight_max": 69.77107238769531,
      "activations/layer7_attention_weight_min": -61.48017883300781,
      "activations/layer8_attention_weight_max": 41.968353271484375,
      "activations/layer8_attention_weight_min": -39.79772186279297,
      "activations/layer9_attention_weight_max": 48.663631439208984,
      "activations/layer9_attention_weight_min": -43.713836669921875,
      "epoch": 5.33,
      "learning_rate": 0.0001167776515151515,
      "loss": 2.8563,
      "step": 91800
    },
    {
      "activations/layer0_attention_weight_max": 14.943222045898438,
      "activations/layer0_attention_weight_min": -14.573939323425293,
      "activations/layer10_attention_weight_max": 38.90653991699219,
      "activations/layer10_attention_weight_min": -32.12914276123047,
      "activations/layer11_attention_weight_max": 29.826025009155273,
      "activations/layer11_attention_weight_min": -26.11923599243164,
      "activations/layer12_attention_weight_max": 25.351457595825195,
      "activations/layer12_attention_weight_min": -26.14809799194336,
      "activations/layer13_attention_weight_max": 58.51193618774414,
      "activations/layer13_attention_weight_min": -38.463661193847656,
      "activations/layer14_attention_weight_max": 47.781150817871094,
      "activations/layer14_attention_weight_min": -31.41964340209961,
      "activations/layer15_attention_weight_max": 48.004722595214844,
      "activations/layer15_attention_weight_min": -31.90690040588379,
      "activations/layer16_attention_weight_max": 40.23879623413086,
      "activations/layer16_attention_weight_min": -26.631975173950195,
      "activations/layer17_attention_weight_max": 56.51304626464844,
      "activations/layer17_attention_weight_min": -39.98564529418945,
      "activations/layer18_attention_weight_max": 51.88087463378906,
      "activations/layer18_attention_weight_min": -33.156822204589844,
      "activations/layer19_attention_weight_max": 25.715436935424805,
      "activations/layer19_attention_weight_min": -19.46213150024414,
      "activations/layer1_attention_weight_max": 18.43144416809082,
      "activations/layer1_attention_weight_min": -13.87098217010498,
      "activations/layer20_attention_weight_max": 24.237552642822266,
      "activations/layer20_attention_weight_min": -19.73929786682129,
      "activations/layer21_attention_weight_max": 45.630489349365234,
      "activations/layer21_attention_weight_min": -23.85001564025879,
      "activations/layer22_attention_weight_max": 37.717689514160156,
      "activations/layer22_attention_weight_min": -23.684690475463867,
      "activations/layer23_attention_weight_max": 31.992597579956055,
      "activations/layer23_attention_weight_min": -19.12333106994629,
      "activations/layer2_attention_weight_max": 29.799650192260742,
      "activations/layer2_attention_weight_min": -27.81757926940918,
      "activations/layer3_attention_weight_max": 81.7015609741211,
      "activations/layer3_attention_weight_min": -80.91622161865234,
      "activations/layer4_attention_weight_max": 102.96312713623047,
      "activations/layer4_attention_weight_min": -92.86862182617188,
      "activations/layer5_attention_weight_max": 101.05522155761719,
      "activations/layer5_attention_weight_min": -92.93550109863281,
      "activations/layer6_attention_weight_max": 56.81212615966797,
      "activations/layer6_attention_weight_min": -52.20698547363281,
      "activations/layer7_attention_weight_max": 72.0941162109375,
      "activations/layer7_attention_weight_min": -66.73617553710938,
      "activations/layer8_attention_weight_max": 43.40597152709961,
      "activations/layer8_attention_weight_min": -41.7303352355957,
      "activations/layer9_attention_weight_max": 50.525814056396484,
      "activations/layer9_attention_weight_min": -45.56246566772461,
      "epoch": 5.34,
      "learning_rate": 0.0001167587121212121,
      "loss": 2.8454,
      "step": 91850
    },
    {
      "activations/layer0_attention_weight_max": 14.691040992736816,
      "activations/layer0_attention_weight_min": -14.779508590698242,
      "activations/layer10_attention_weight_max": 36.865379333496094,
      "activations/layer10_attention_weight_min": -32.85394287109375,
      "activations/layer11_attention_weight_max": 28.031042098999023,
      "activations/layer11_attention_weight_min": -25.88565444946289,
      "activations/layer12_attention_weight_max": 21.570938110351562,
      "activations/layer12_attention_weight_min": -26.296815872192383,
      "activations/layer13_attention_weight_max": 56.183658599853516,
      "activations/layer13_attention_weight_min": -36.78187561035156,
      "activations/layer14_attention_weight_max": 55.13251495361328,
      "activations/layer14_attention_weight_min": -32.69916915893555,
      "activations/layer15_attention_weight_max": 52.2918815612793,
      "activations/layer15_attention_weight_min": -29.64385414123535,
      "activations/layer16_attention_weight_max": 42.19466018676758,
      "activations/layer16_attention_weight_min": -29.0801944732666,
      "activations/layer17_attention_weight_max": 66.46928405761719,
      "activations/layer17_attention_weight_min": -42.207763671875,
      "activations/layer18_attention_weight_max": 65.23056030273438,
      "activations/layer18_attention_weight_min": -37.535987854003906,
      "activations/layer19_attention_weight_max": 34.91489028930664,
      "activations/layer19_attention_weight_min": -21.28510093688965,
      "activations/layer1_attention_weight_max": 17.23332405090332,
      "activations/layer1_attention_weight_min": -15.835771560668945,
      "activations/layer20_attention_weight_max": 27.208662033081055,
      "activations/layer20_attention_weight_min": -26.1254825592041,
      "activations/layer21_attention_weight_max": 47.89256286621094,
      "activations/layer21_attention_weight_min": -26.247005462646484,
      "activations/layer22_attention_weight_max": 41.31538009643555,
      "activations/layer22_attention_weight_min": -24.771577835083008,
      "activations/layer23_attention_weight_max": 35.7972412109375,
      "activations/layer23_attention_weight_min": -16.974945068359375,
      "activations/layer2_attention_weight_max": 30.795303344726562,
      "activations/layer2_attention_weight_min": -29.23447036743164,
      "activations/layer3_attention_weight_max": 84.97936248779297,
      "activations/layer3_attention_weight_min": -88.34443664550781,
      "activations/layer4_attention_weight_max": 106.3324203491211,
      "activations/layer4_attention_weight_min": -92.30010223388672,
      "activations/layer5_attention_weight_max": 104.98249816894531,
      "activations/layer5_attention_weight_min": -85.57268524169922,
      "activations/layer6_attention_weight_max": 60.15122985839844,
      "activations/layer6_attention_weight_min": -48.456565856933594,
      "activations/layer7_attention_weight_max": 72.29595947265625,
      "activations/layer7_attention_weight_min": -67.16722869873047,
      "activations/layer8_attention_weight_max": 42.75620651245117,
      "activations/layer8_attention_weight_min": -38.90509033203125,
      "activations/layer9_attention_weight_max": 48.74110794067383,
      "activations/layer9_attention_weight_min": -45.87045669555664,
      "epoch": 5.34,
      "learning_rate": 0.00011673977272727272,
      "loss": 2.8608,
      "step": 91900
    },
    {
      "activations/layer0_attention_weight_max": 15.260607719421387,
      "activations/layer0_attention_weight_min": -15.031400680541992,
      "activations/layer10_attention_weight_max": 37.52825927734375,
      "activations/layer10_attention_weight_min": -35.3209228515625,
      "activations/layer11_attention_weight_max": 31.043807983398438,
      "activations/layer11_attention_weight_min": -28.1712646484375,
      "activations/layer12_attention_weight_max": 29.603134155273438,
      "activations/layer12_attention_weight_min": -24.29255485534668,
      "activations/layer13_attention_weight_max": 44.817970275878906,
      "activations/layer13_attention_weight_min": -32.43367385864258,
      "activations/layer14_attention_weight_max": 40.02907180786133,
      "activations/layer14_attention_weight_min": -28.795368194580078,
      "activations/layer15_attention_weight_max": 43.415470123291016,
      "activations/layer15_attention_weight_min": -31.13919448852539,
      "activations/layer16_attention_weight_max": 36.13703918457031,
      "activations/layer16_attention_weight_min": -26.564191818237305,
      "activations/layer17_attention_weight_max": 57.41450881958008,
      "activations/layer17_attention_weight_min": -41.065067291259766,
      "activations/layer18_attention_weight_max": 54.61098861694336,
      "activations/layer18_attention_weight_min": -33.9522590637207,
      "activations/layer19_attention_weight_max": 24.133644104003906,
      "activations/layer19_attention_weight_min": -22.372085571289062,
      "activations/layer1_attention_weight_max": 17.067106246948242,
      "activations/layer1_attention_weight_min": -14.245207786560059,
      "activations/layer20_attention_weight_max": 25.379531860351562,
      "activations/layer20_attention_weight_min": -21.301414489746094,
      "activations/layer21_attention_weight_max": 48.44009780883789,
      "activations/layer21_attention_weight_min": -27.172115325927734,
      "activations/layer22_attention_weight_max": 37.237361907958984,
      "activations/layer22_attention_weight_min": -24.362384796142578,
      "activations/layer23_attention_weight_max": 32.86050033569336,
      "activations/layer23_attention_weight_min": -17.59450912475586,
      "activations/layer2_attention_weight_max": 27.945653915405273,
      "activations/layer2_attention_weight_min": -26.961345672607422,
      "activations/layer3_attention_weight_max": 78.34647369384766,
      "activations/layer3_attention_weight_min": -80.3307113647461,
      "activations/layer4_attention_weight_max": 96.935302734375,
      "activations/layer4_attention_weight_min": -93.9552001953125,
      "activations/layer5_attention_weight_max": 98.66915130615234,
      "activations/layer5_attention_weight_min": -88.6885986328125,
      "activations/layer6_attention_weight_max": 57.438385009765625,
      "activations/layer6_attention_weight_min": -55.03501510620117,
      "activations/layer7_attention_weight_max": 73.50679779052734,
      "activations/layer7_attention_weight_min": -68.44963073730469,
      "activations/layer8_attention_weight_max": 45.55247497558594,
      "activations/layer8_attention_weight_min": -42.89502716064453,
      "activations/layer9_attention_weight_max": 53.882057189941406,
      "activations/layer9_attention_weight_min": -49.0238151550293,
      "epoch": 5.34,
      "learning_rate": 0.00011672083333333334,
      "loss": 2.8575,
      "step": 91950
    },
    {
      "activations/layer0_attention_weight_max": 15.47375774383545,
      "activations/layer0_attention_weight_min": -14.916706085205078,
      "activations/layer10_attention_weight_max": 42.32973098754883,
      "activations/layer10_attention_weight_min": -35.53582000732422,
      "activations/layer11_attention_weight_max": 34.891754150390625,
      "activations/layer11_attention_weight_min": -27.26616668701172,
      "activations/layer12_attention_weight_max": 26.106449127197266,
      "activations/layer12_attention_weight_min": -23.264663696289062,
      "activations/layer13_attention_weight_max": 48.23209762573242,
      "activations/layer13_attention_weight_min": -33.220027923583984,
      "activations/layer14_attention_weight_max": 39.82477951049805,
      "activations/layer14_attention_weight_min": -29.812503814697266,
      "activations/layer15_attention_weight_max": 43.61665344238281,
      "activations/layer15_attention_weight_min": -30.00908851623535,
      "activations/layer16_attention_weight_max": 34.36320495605469,
      "activations/layer16_attention_weight_min": -28.283506393432617,
      "activations/layer17_attention_weight_max": 57.72816467285156,
      "activations/layer17_attention_weight_min": -38.84928894042969,
      "activations/layer18_attention_weight_max": 49.83082962036133,
      "activations/layer18_attention_weight_min": -32.872528076171875,
      "activations/layer19_attention_weight_max": 20.610122680664062,
      "activations/layer19_attention_weight_min": -22.193830490112305,
      "activations/layer1_attention_weight_max": 17.25560760498047,
      "activations/layer1_attention_weight_min": -16.76317024230957,
      "activations/layer20_attention_weight_max": 25.824199676513672,
      "activations/layer20_attention_weight_min": -22.13899040222168,
      "activations/layer21_attention_weight_max": 42.793460845947266,
      "activations/layer21_attention_weight_min": -24.705942153930664,
      "activations/layer22_attention_weight_max": 34.201255798339844,
      "activations/layer22_attention_weight_min": -23.48211097717285,
      "activations/layer23_attention_weight_max": 30.410743713378906,
      "activations/layer23_attention_weight_min": -20.135555267333984,
      "activations/layer2_attention_weight_max": 29.574092864990234,
      "activations/layer2_attention_weight_min": -29.22256088256836,
      "activations/layer3_attention_weight_max": 82.8082275390625,
      "activations/layer3_attention_weight_min": -89.96321868896484,
      "activations/layer4_attention_weight_max": 103.25182342529297,
      "activations/layer4_attention_weight_min": -95.98007202148438,
      "activations/layer5_attention_weight_max": 103.4758529663086,
      "activations/layer5_attention_weight_min": -88.8472900390625,
      "activations/layer6_attention_weight_max": 58.458351135253906,
      "activations/layer6_attention_weight_min": -53.19194793701172,
      "activations/layer7_attention_weight_max": 68.72174835205078,
      "activations/layer7_attention_weight_min": -64.63752746582031,
      "activations/layer8_attention_weight_max": 43.598724365234375,
      "activations/layer8_attention_weight_min": -42.43475341796875,
      "activations/layer9_attention_weight_max": 48.740516662597656,
      "activations/layer9_attention_weight_min": -45.478416442871094,
      "epoch": 5.35,
      "learning_rate": 0.00011670189393939392,
      "loss": 2.8322,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_loss": 2.798828125,
      "eval_runtime": 8.5533,
      "eval_samples_per_second": 502.028,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_openwebtext_loss": 2.798828125,
      "eval_openwebtext_ppl": 16.425386987895497,
      "eval_openwebtext_runtime": 8.5533,
      "eval_openwebtext_samples_per_second": 502.028,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_wikitext_loss": 3.025390625,
      "eval_wikitext_ppl": 20.602050815688813,
      "eval_wikitext_runtime": 2.0158,
      "eval_wikitext_samples_per_second": 226.216,
      "step": 92000
    },
    {
      "epoch": 5.35,
      "eval_lambada_loss": 2.888671875,
      "eval_lambada_ppl": 17.969428099556087,
      "eval_lambada_runtime": 9.612,
      "eval_lambada_samples_per_second": 506.553,
      "step": 92000
    },
    {
      "activations/layer0_attention_weight_max": 13.857742309570312,
      "activations/layer0_attention_weight_min": -15.236153602600098,
      "activations/layer10_attention_weight_max": 40.138187408447266,
      "activations/layer10_attention_weight_min": -34.09282684326172,
      "activations/layer11_attention_weight_max": 30.600261688232422,
      "activations/layer11_attention_weight_min": -27.888242721557617,
      "activations/layer12_attention_weight_max": 24.540863037109375,
      "activations/layer12_attention_weight_min": -25.114988327026367,
      "activations/layer13_attention_weight_max": 63.115447998046875,
      "activations/layer13_attention_weight_min": -40.72504806518555,
      "activations/layer14_attention_weight_max": 58.54267883300781,
      "activations/layer14_attention_weight_min": -31.248504638671875,
      "activations/layer15_attention_weight_max": 45.8410758972168,
      "activations/layer15_attention_weight_min": -30.190277099609375,
      "activations/layer16_attention_weight_max": 38.00962829589844,
      "activations/layer16_attention_weight_min": -27.5360107421875,
      "activations/layer17_attention_weight_max": 61.2073974609375,
      "activations/layer17_attention_weight_min": -38.050357818603516,
      "activations/layer18_attention_weight_max": 50.688629150390625,
      "activations/layer18_attention_weight_min": -33.04789733886719,
      "activations/layer19_attention_weight_max": 21.293960571289062,
      "activations/layer19_attention_weight_min": -21.68733787536621,
      "activations/layer1_attention_weight_max": 17.637449264526367,
      "activations/layer1_attention_weight_min": -16.851381301879883,
      "activations/layer20_attention_weight_max": 23.723344802856445,
      "activations/layer20_attention_weight_min": -22.312280654907227,
      "activations/layer21_attention_weight_max": 43.46113967895508,
      "activations/layer21_attention_weight_min": -24.55984878540039,
      "activations/layer22_attention_weight_max": 39.19998550415039,
      "activations/layer22_attention_weight_min": -24.44768524169922,
      "activations/layer23_attention_weight_max": 35.1878662109375,
      "activations/layer23_attention_weight_min": -20.09693145751953,
      "activations/layer2_attention_weight_max": 28.414037704467773,
      "activations/layer2_attention_weight_min": -27.810039520263672,
      "activations/layer3_attention_weight_max": 78.05914306640625,
      "activations/layer3_attention_weight_min": -83.75660705566406,
      "activations/layer4_attention_weight_max": 100.83348083496094,
      "activations/layer4_attention_weight_min": -100.6744613647461,
      "activations/layer5_attention_weight_max": 98.36384582519531,
      "activations/layer5_attention_weight_min": -93.19552612304688,
      "activations/layer6_attention_weight_max": 55.2734260559082,
      "activations/layer6_attention_weight_min": -50.54984664916992,
      "activations/layer7_attention_weight_max": 66.84651184082031,
      "activations/layer7_attention_weight_min": -65.04013061523438,
      "activations/layer8_attention_weight_max": 43.121036529541016,
      "activations/layer8_attention_weight_min": -41.783226013183594,
      "activations/layer9_attention_weight_max": 51.29663848876953,
      "activations/layer9_attention_weight_min": -47.56901168823242,
      "epoch": 5.35,
      "learning_rate": 0.00011668295454545454,
      "loss": 2.8482,
      "step": 92050
    },
    {
      "activations/layer0_attention_weight_max": 15.158080101013184,
      "activations/layer0_attention_weight_min": -14.928472518920898,
      "activations/layer10_attention_weight_max": 35.87544250488281,
      "activations/layer10_attention_weight_min": -33.20010757446289,
      "activations/layer11_attention_weight_max": 28.022918701171875,
      "activations/layer11_attention_weight_min": -27.92186737060547,
      "activations/layer12_attention_weight_max": 22.24829864501953,
      "activations/layer12_attention_weight_min": -25.35127067565918,
      "activations/layer13_attention_weight_max": 46.42998504638672,
      "activations/layer13_attention_weight_min": -32.985260009765625,
      "activations/layer14_attention_weight_max": 46.950721740722656,
      "activations/layer14_attention_weight_min": -28.691240310668945,
      "activations/layer15_attention_weight_max": 39.469844818115234,
      "activations/layer15_attention_weight_min": -27.196523666381836,
      "activations/layer16_attention_weight_max": 30.021631240844727,
      "activations/layer16_attention_weight_min": -26.016773223876953,
      "activations/layer17_attention_weight_max": 53.24912643432617,
      "activations/layer17_attention_weight_min": -37.55511474609375,
      "activations/layer18_attention_weight_max": 42.046348571777344,
      "activations/layer18_attention_weight_min": -32.00661087036133,
      "activations/layer19_attention_weight_max": 21.916858673095703,
      "activations/layer19_attention_weight_min": -21.100034713745117,
      "activations/layer1_attention_weight_max": 17.73814582824707,
      "activations/layer1_attention_weight_min": -15.283528327941895,
      "activations/layer20_attention_weight_max": 22.652578353881836,
      "activations/layer20_attention_weight_min": -21.832313537597656,
      "activations/layer21_attention_weight_max": 35.448463439941406,
      "activations/layer21_attention_weight_min": -26.66872787475586,
      "activations/layer22_attention_weight_max": 33.85236358642578,
      "activations/layer22_attention_weight_min": -23.66570281982422,
      "activations/layer23_attention_weight_max": 26.25548553466797,
      "activations/layer23_attention_weight_min": -18.415592193603516,
      "activations/layer2_attention_weight_max": 28.76791763305664,
      "activations/layer2_attention_weight_min": -27.370208740234375,
      "activations/layer3_attention_weight_max": 79.0468978881836,
      "activations/layer3_attention_weight_min": -85.87126922607422,
      "activations/layer4_attention_weight_max": 97.29249572753906,
      "activations/layer4_attention_weight_min": -95.57012176513672,
      "activations/layer5_attention_weight_max": 95.77269744873047,
      "activations/layer5_attention_weight_min": -89.39152526855469,
      "activations/layer6_attention_weight_max": 53.96706771850586,
      "activations/layer6_attention_weight_min": -49.629493713378906,
      "activations/layer7_attention_weight_max": 72.2225570678711,
      "activations/layer7_attention_weight_min": -67.64799499511719,
      "activations/layer8_attention_weight_max": 40.178070068359375,
      "activations/layer8_attention_weight_min": -40.80521774291992,
      "activations/layer9_attention_weight_max": 46.19404220581055,
      "activations/layer9_attention_weight_min": -46.54886245727539,
      "epoch": 5.35,
      "learning_rate": 0.00011666401515151513,
      "loss": 2.8585,
      "step": 92100
    },
    {
      "activations/layer0_attention_weight_max": 15.295069694519043,
      "activations/layer0_attention_weight_min": -15.176596641540527,
      "activations/layer10_attention_weight_max": 38.937381744384766,
      "activations/layer10_attention_weight_min": -36.36603546142578,
      "activations/layer11_attention_weight_max": 30.86847686767578,
      "activations/layer11_attention_weight_min": -28.693998336791992,
      "activations/layer12_attention_weight_max": 33.36747741699219,
      "activations/layer12_attention_weight_min": -23.26609992980957,
      "activations/layer13_attention_weight_max": 47.91421890258789,
      "activations/layer13_attention_weight_min": -32.514503479003906,
      "activations/layer14_attention_weight_max": 40.072898864746094,
      "activations/layer14_attention_weight_min": -31.589031219482422,
      "activations/layer15_attention_weight_max": 41.43464279174805,
      "activations/layer15_attention_weight_min": -31.129199981689453,
      "activations/layer16_attention_weight_max": 37.448768615722656,
      "activations/layer16_attention_weight_min": -26.806474685668945,
      "activations/layer17_attention_weight_max": 67.5641860961914,
      "activations/layer17_attention_weight_min": -41.60796356201172,
      "activations/layer18_attention_weight_max": 52.267822265625,
      "activations/layer18_attention_weight_min": -36.8709831237793,
      "activations/layer19_attention_weight_max": 20.098573684692383,
      "activations/layer19_attention_weight_min": -20.000545501708984,
      "activations/layer1_attention_weight_max": 17.69245147705078,
      "activations/layer1_attention_weight_min": -13.963875770568848,
      "activations/layer20_attention_weight_max": 21.751596450805664,
      "activations/layer20_attention_weight_min": -21.68228530883789,
      "activations/layer21_attention_weight_max": 37.5587272644043,
      "activations/layer21_attention_weight_min": -24.03449058532715,
      "activations/layer22_attention_weight_max": 34.5150032043457,
      "activations/layer22_attention_weight_min": -24.90764808654785,
      "activations/layer23_attention_weight_max": 31.798437118530273,
      "activations/layer23_attention_weight_min": -19.850671768188477,
      "activations/layer2_attention_weight_max": 29.843425750732422,
      "activations/layer2_attention_weight_min": -28.609228134155273,
      "activations/layer3_attention_weight_max": 81.44722747802734,
      "activations/layer3_attention_weight_min": -86.06732177734375,
      "activations/layer4_attention_weight_max": 107.4320297241211,
      "activations/layer4_attention_weight_min": -97.4649887084961,
      "activations/layer5_attention_weight_max": 104.50453186035156,
      "activations/layer5_attention_weight_min": -98.15400695800781,
      "activations/layer6_attention_weight_max": 58.14745330810547,
      "activations/layer6_attention_weight_min": -56.064361572265625,
      "activations/layer7_attention_weight_max": 71.14901733398438,
      "activations/layer7_attention_weight_min": -73.7713623046875,
      "activations/layer8_attention_weight_max": 47.0550422668457,
      "activations/layer8_attention_weight_min": -47.935001373291016,
      "activations/layer9_attention_weight_max": 49.238731384277344,
      "activations/layer9_attention_weight_min": -50.07162094116211,
      "epoch": 5.35,
      "learning_rate": 0.00011664507575757574,
      "loss": 2.8511,
      "step": 92150
    },
    {
      "activations/layer0_attention_weight_max": 16.8829345703125,
      "activations/layer0_attention_weight_min": -14.916061401367188,
      "activations/layer10_attention_weight_max": 36.378456115722656,
      "activations/layer10_attention_weight_min": -33.48545837402344,
      "activations/layer11_attention_weight_max": 30.815359115600586,
      "activations/layer11_attention_weight_min": -26.745281219482422,
      "activations/layer12_attention_weight_max": 22.595548629760742,
      "activations/layer12_attention_weight_min": -36.70644760131836,
      "activations/layer13_attention_weight_max": 47.91011047363281,
      "activations/layer13_attention_weight_min": -35.12484359741211,
      "activations/layer14_attention_weight_max": 42.60945510864258,
      "activations/layer14_attention_weight_min": -29.952255249023438,
      "activations/layer15_attention_weight_max": 40.82460403442383,
      "activations/layer15_attention_weight_min": -28.183443069458008,
      "activations/layer16_attention_weight_max": 41.88130187988281,
      "activations/layer16_attention_weight_min": -26.991607666015625,
      "activations/layer17_attention_weight_max": 54.50305938720703,
      "activations/layer17_attention_weight_min": -39.31214904785156,
      "activations/layer18_attention_weight_max": 44.52700424194336,
      "activations/layer18_attention_weight_min": -32.161346435546875,
      "activations/layer19_attention_weight_max": 21.747167587280273,
      "activations/layer19_attention_weight_min": -18.138559341430664,
      "activations/layer1_attention_weight_max": 17.422332763671875,
      "activations/layer1_attention_weight_min": -16.16385269165039,
      "activations/layer20_attention_weight_max": 20.250810623168945,
      "activations/layer20_attention_weight_min": -20.083547592163086,
      "activations/layer21_attention_weight_max": 39.445472717285156,
      "activations/layer21_attention_weight_min": -26.902502059936523,
      "activations/layer22_attention_weight_max": 32.53207778930664,
      "activations/layer22_attention_weight_min": -24.31256103515625,
      "activations/layer23_attention_weight_max": 26.318645477294922,
      "activations/layer23_attention_weight_min": -18.927322387695312,
      "activations/layer2_attention_weight_max": 29.493085861206055,
      "activations/layer2_attention_weight_min": -27.017414093017578,
      "activations/layer3_attention_weight_max": 80.89696502685547,
      "activations/layer3_attention_weight_min": -83.05272674560547,
      "activations/layer4_attention_weight_max": 101.2148666381836,
      "activations/layer4_attention_weight_min": -96.00601196289062,
      "activations/layer5_attention_weight_max": 101.72139739990234,
      "activations/layer5_attention_weight_min": -88.7550048828125,
      "activations/layer6_attention_weight_max": 57.12565231323242,
      "activations/layer6_attention_weight_min": -51.299354553222656,
      "activations/layer7_attention_weight_max": 82.62123107910156,
      "activations/layer7_attention_weight_min": -64.4398193359375,
      "activations/layer8_attention_weight_max": 48.74827194213867,
      "activations/layer8_attention_weight_min": -41.535823822021484,
      "activations/layer9_attention_weight_max": 72.83883666992188,
      "activations/layer9_attention_weight_min": -54.38070297241211,
      "epoch": 5.36,
      "learning_rate": 0.00011662613636363636,
      "loss": 2.852,
      "step": 92200
    },
    {
      "activations/layer0_attention_weight_max": 14.22409439086914,
      "activations/layer0_attention_weight_min": -15.182186126708984,
      "activations/layer10_attention_weight_max": 38.62355422973633,
      "activations/layer10_attention_weight_min": -34.38051986694336,
      "activations/layer11_attention_weight_max": 31.55215072631836,
      "activations/layer11_attention_weight_min": -27.778289794921875,
      "activations/layer12_attention_weight_max": 21.9884090423584,
      "activations/layer12_attention_weight_min": -23.642850875854492,
      "activations/layer13_attention_weight_max": 61.37196731567383,
      "activations/layer13_attention_weight_min": -38.00324249267578,
      "activations/layer14_attention_weight_max": 51.143470764160156,
      "activations/layer14_attention_weight_min": -30.12147331237793,
      "activations/layer15_attention_weight_max": 52.348350524902344,
      "activations/layer15_attention_weight_min": -31.23667335510254,
      "activations/layer16_attention_weight_max": 49.535926818847656,
      "activations/layer16_attention_weight_min": -27.100055694580078,
      "activations/layer17_attention_weight_max": 59.39156723022461,
      "activations/layer17_attention_weight_min": -37.82942199707031,
      "activations/layer18_attention_weight_max": 51.64118957519531,
      "activations/layer18_attention_weight_min": -33.705902099609375,
      "activations/layer19_attention_weight_max": 27.767114639282227,
      "activations/layer19_attention_weight_min": -19.2312068939209,
      "activations/layer1_attention_weight_max": 16.442729949951172,
      "activations/layer1_attention_weight_min": -16.702585220336914,
      "activations/layer20_attention_weight_max": 27.308759689331055,
      "activations/layer20_attention_weight_min": -22.3778076171875,
      "activations/layer21_attention_weight_max": 47.3492431640625,
      "activations/layer21_attention_weight_min": -26.040380477905273,
      "activations/layer22_attention_weight_max": 34.87740707397461,
      "activations/layer22_attention_weight_min": -24.3790283203125,
      "activations/layer23_attention_weight_max": 31.24462127685547,
      "activations/layer23_attention_weight_min": -20.57440185546875,
      "activations/layer2_attention_weight_max": 31.33763885498047,
      "activations/layer2_attention_weight_min": -30.339893341064453,
      "activations/layer3_attention_weight_max": 83.17313385009766,
      "activations/layer3_attention_weight_min": -91.49809265136719,
      "activations/layer4_attention_weight_max": 103.35343170166016,
      "activations/layer4_attention_weight_min": -99.46949768066406,
      "activations/layer5_attention_weight_max": 104.54692840576172,
      "activations/layer5_attention_weight_min": -97.22777557373047,
      "activations/layer6_attention_weight_max": 62.839088439941406,
      "activations/layer6_attention_weight_min": -55.13699722290039,
      "activations/layer7_attention_weight_max": 71.84703063964844,
      "activations/layer7_attention_weight_min": -67.52972412109375,
      "activations/layer8_attention_weight_max": 45.748321533203125,
      "activations/layer8_attention_weight_min": -43.49321365356445,
      "activations/layer9_attention_weight_max": 50.17300033569336,
      "activations/layer9_attention_weight_min": -47.164024353027344,
      "epoch": 5.36,
      "learning_rate": 0.00011660719696969696,
      "loss": 2.8355,
      "step": 92250
    },
    {
      "activations/layer0_attention_weight_max": 14.426600456237793,
      "activations/layer0_attention_weight_min": -15.17404556274414,
      "activations/layer10_attention_weight_max": 39.17733383178711,
      "activations/layer10_attention_weight_min": -34.49272537231445,
      "activations/layer11_attention_weight_max": 29.67938995361328,
      "activations/layer11_attention_weight_min": -27.563987731933594,
      "activations/layer12_attention_weight_max": 32.91745376586914,
      "activations/layer12_attention_weight_min": -24.655317306518555,
      "activations/layer13_attention_weight_max": 51.33646774291992,
      "activations/layer13_attention_weight_min": -33.776939392089844,
      "activations/layer14_attention_weight_max": 46.23371124267578,
      "activations/layer14_attention_weight_min": -29.62772560119629,
      "activations/layer15_attention_weight_max": 43.957828521728516,
      "activations/layer15_attention_weight_min": -28.744211196899414,
      "activations/layer16_attention_weight_max": 36.757240295410156,
      "activations/layer16_attention_weight_min": -25.64335823059082,
      "activations/layer17_attention_weight_max": 54.606319427490234,
      "activations/layer17_attention_weight_min": -37.50986099243164,
      "activations/layer18_attention_weight_max": 48.46719741821289,
      "activations/layer18_attention_weight_min": -35.11252975463867,
      "activations/layer19_attention_weight_max": 21.61516761779785,
      "activations/layer19_attention_weight_min": -18.953678131103516,
      "activations/layer1_attention_weight_max": 16.8745174407959,
      "activations/layer1_attention_weight_min": -14.732683181762695,
      "activations/layer20_attention_weight_max": 23.079933166503906,
      "activations/layer20_attention_weight_min": -21.847965240478516,
      "activations/layer21_attention_weight_max": 36.27585220336914,
      "activations/layer21_attention_weight_min": -25.510478973388672,
      "activations/layer22_attention_weight_max": 37.59511947631836,
      "activations/layer22_attention_weight_min": -25.76487922668457,
      "activations/layer23_attention_weight_max": 28.132339477539062,
      "activations/layer23_attention_weight_min": -19.127601623535156,
      "activations/layer2_attention_weight_max": 29.947500228881836,
      "activations/layer2_attention_weight_min": -28.442035675048828,
      "activations/layer3_attention_weight_max": 78.15055084228516,
      "activations/layer3_attention_weight_min": -77.70274353027344,
      "activations/layer4_attention_weight_max": 99.82705688476562,
      "activations/layer4_attention_weight_min": -92.34797668457031,
      "activations/layer5_attention_weight_max": 102.17943572998047,
      "activations/layer5_attention_weight_min": -89.29375457763672,
      "activations/layer6_attention_weight_max": 60.77769470214844,
      "activations/layer6_attention_weight_min": -54.28012466430664,
      "activations/layer7_attention_weight_max": 72.9057388305664,
      "activations/layer7_attention_weight_min": -69.06110382080078,
      "activations/layer8_attention_weight_max": 46.32158660888672,
      "activations/layer8_attention_weight_min": -44.17731857299805,
      "activations/layer9_attention_weight_max": 51.90252685546875,
      "activations/layer9_attention_weight_min": -45.76743698120117,
      "epoch": 5.36,
      "learning_rate": 0.00011658825757575756,
      "loss": 2.8372,
      "step": 92300
    },
    {
      "activations/layer0_attention_weight_max": 14.598723411560059,
      "activations/layer0_attention_weight_min": -14.695968627929688,
      "activations/layer10_attention_weight_max": 41.30624008178711,
      "activations/layer10_attention_weight_min": -37.92116165161133,
      "activations/layer11_attention_weight_max": 33.725303649902344,
      "activations/layer11_attention_weight_min": -30.64191436767578,
      "activations/layer12_attention_weight_max": 26.286348342895508,
      "activations/layer12_attention_weight_min": -26.5616512298584,
      "activations/layer13_attention_weight_max": 67.71798706054688,
      "activations/layer13_attention_weight_min": -52.28144836425781,
      "activations/layer14_attention_weight_max": 62.963409423828125,
      "activations/layer14_attention_weight_min": -42.12154006958008,
      "activations/layer15_attention_weight_max": 44.85569381713867,
      "activations/layer15_attention_weight_min": -34.85966873168945,
      "activations/layer16_attention_weight_max": 35.817466735839844,
      "activations/layer16_attention_weight_min": -27.60771369934082,
      "activations/layer17_attention_weight_max": 57.95258331298828,
      "activations/layer17_attention_weight_min": -42.37712097167969,
      "activations/layer18_attention_weight_max": 52.52786636352539,
      "activations/layer18_attention_weight_min": -37.052059173583984,
      "activations/layer19_attention_weight_max": 21.995342254638672,
      "activations/layer19_attention_weight_min": -19.4554443359375,
      "activations/layer1_attention_weight_max": 17.621950149536133,
      "activations/layer1_attention_weight_min": -17.922515869140625,
      "activations/layer20_attention_weight_max": 23.96470832824707,
      "activations/layer20_attention_weight_min": -25.1942195892334,
      "activations/layer21_attention_weight_max": 46.47578048706055,
      "activations/layer21_attention_weight_min": -25.94603157043457,
      "activations/layer22_attention_weight_max": 36.662715911865234,
      "activations/layer22_attention_weight_min": -22.76681137084961,
      "activations/layer23_attention_weight_max": 30.48706817626953,
      "activations/layer23_attention_weight_min": -18.9298152923584,
      "activations/layer2_attention_weight_max": 30.875572204589844,
      "activations/layer2_attention_weight_min": -29.217548370361328,
      "activations/layer3_attention_weight_max": 82.7718276977539,
      "activations/layer3_attention_weight_min": -87.00084686279297,
      "activations/layer4_attention_weight_max": 100.43566131591797,
      "activations/layer4_attention_weight_min": -103.48648834228516,
      "activations/layer5_attention_weight_max": 102.37747192382812,
      "activations/layer5_attention_weight_min": -97.45211791992188,
      "activations/layer6_attention_weight_max": 55.635719299316406,
      "activations/layer6_attention_weight_min": -56.769474029541016,
      "activations/layer7_attention_weight_max": 75.68350982666016,
      "activations/layer7_attention_weight_min": -67.45010375976562,
      "activations/layer8_attention_weight_max": 45.17384719848633,
      "activations/layer8_attention_weight_min": -44.118438720703125,
      "activations/layer9_attention_weight_max": 64.45970153808594,
      "activations/layer9_attention_weight_min": -49.87430953979492,
      "epoch": 5.37,
      "learning_rate": 0.00011656931818181817,
      "loss": 2.8507,
      "step": 92350
    },
    {
      "activations/layer0_attention_weight_max": 14.510722160339355,
      "activations/layer0_attention_weight_min": -15.347458839416504,
      "activations/layer10_attention_weight_max": 38.6953125,
      "activations/layer10_attention_weight_min": -36.28280258178711,
      "activations/layer11_attention_weight_max": 30.853687286376953,
      "activations/layer11_attention_weight_min": -29.25436782836914,
      "activations/layer12_attention_weight_max": 23.716163635253906,
      "activations/layer12_attention_weight_min": -26.196489334106445,
      "activations/layer13_attention_weight_max": 52.436954498291016,
      "activations/layer13_attention_weight_min": -35.102935791015625,
      "activations/layer14_attention_weight_max": 47.416717529296875,
      "activations/layer14_attention_weight_min": -30.007394790649414,
      "activations/layer15_attention_weight_max": 41.644805908203125,
      "activations/layer15_attention_weight_min": -29.84092903137207,
      "activations/layer16_attention_weight_max": 33.87031936645508,
      "activations/layer16_attention_weight_min": -25.115982055664062,
      "activations/layer17_attention_weight_max": 60.22441482543945,
      "activations/layer17_attention_weight_min": -38.13383483886719,
      "activations/layer18_attention_weight_max": 56.3438606262207,
      "activations/layer18_attention_weight_min": -32.722049713134766,
      "activations/layer19_attention_weight_max": 25.16524887084961,
      "activations/layer19_attention_weight_min": -18.562654495239258,
      "activations/layer1_attention_weight_max": 18.26347541809082,
      "activations/layer1_attention_weight_min": -14.355096817016602,
      "activations/layer20_attention_weight_max": 23.726560592651367,
      "activations/layer20_attention_weight_min": -23.234325408935547,
      "activations/layer21_attention_weight_max": 43.45939254760742,
      "activations/layer21_attention_weight_min": -26.74032211303711,
      "activations/layer22_attention_weight_max": 35.47321319580078,
      "activations/layer22_attention_weight_min": -23.36185646057129,
      "activations/layer23_attention_weight_max": 32.3250732421875,
      "activations/layer23_attention_weight_min": -18.570125579833984,
      "activations/layer2_attention_weight_max": 29.694622039794922,
      "activations/layer2_attention_weight_min": -28.58116912841797,
      "activations/layer3_attention_weight_max": 82.7319564819336,
      "activations/layer3_attention_weight_min": -88.86260223388672,
      "activations/layer4_attention_weight_max": 103.4747314453125,
      "activations/layer4_attention_weight_min": -98.81665802001953,
      "activations/layer5_attention_weight_max": 102.27886199951172,
      "activations/layer5_attention_weight_min": -93.36851501464844,
      "activations/layer6_attention_weight_max": 58.970115661621094,
      "activations/layer6_attention_weight_min": -53.2946662902832,
      "activations/layer7_attention_weight_max": 70.91039276123047,
      "activations/layer7_attention_weight_min": -70.19300842285156,
      "activations/layer8_attention_weight_max": 43.10844802856445,
      "activations/layer8_attention_weight_min": -45.09577178955078,
      "activations/layer9_attention_weight_max": 53.79888916015625,
      "activations/layer9_attention_weight_min": -50.91914367675781,
      "epoch": 5.37,
      "learning_rate": 0.00011655037878787878,
      "loss": 2.8527,
      "step": 92400
    },
    {
      "activations/layer0_attention_weight_max": 15.35937786102295,
      "activations/layer0_attention_weight_min": -15.025336265563965,
      "activations/layer10_attention_weight_max": 36.7835693359375,
      "activations/layer10_attention_weight_min": -33.94113540649414,
      "activations/layer11_attention_weight_max": 32.3151969909668,
      "activations/layer11_attention_weight_min": -27.190885543823242,
      "activations/layer12_attention_weight_max": 20.528400421142578,
      "activations/layer12_attention_weight_min": -26.949647903442383,
      "activations/layer13_attention_weight_max": 58.14421844482422,
      "activations/layer13_attention_weight_min": -37.273006439208984,
      "activations/layer14_attention_weight_max": 39.39330291748047,
      "activations/layer14_attention_weight_min": -29.56294822692871,
      "activations/layer15_attention_weight_max": 46.232276916503906,
      "activations/layer15_attention_weight_min": -29.80940818786621,
      "activations/layer16_attention_weight_max": 35.79619216918945,
      "activations/layer16_attention_weight_min": -25.47005844116211,
      "activations/layer17_attention_weight_max": 58.49800109863281,
      "activations/layer17_attention_weight_min": -38.67694091796875,
      "activations/layer18_attention_weight_max": 53.73411178588867,
      "activations/layer18_attention_weight_min": -33.9161262512207,
      "activations/layer19_attention_weight_max": 24.593664169311523,
      "activations/layer19_attention_weight_min": -20.128910064697266,
      "activations/layer1_attention_weight_max": 17.554546356201172,
      "activations/layer1_attention_weight_min": -13.324507713317871,
      "activations/layer20_attention_weight_max": 24.647958755493164,
      "activations/layer20_attention_weight_min": -24.071273803710938,
      "activations/layer21_attention_weight_max": 44.904212951660156,
      "activations/layer21_attention_weight_min": -23.665447235107422,
      "activations/layer22_attention_weight_max": 32.49103927612305,
      "activations/layer22_attention_weight_min": -23.309093475341797,
      "activations/layer23_attention_weight_max": 29.88986587524414,
      "activations/layer23_attention_weight_min": -18.003267288208008,
      "activations/layer2_attention_weight_max": 28.195308685302734,
      "activations/layer2_attention_weight_min": -26.97885513305664,
      "activations/layer3_attention_weight_max": 76.33707427978516,
      "activations/layer3_attention_weight_min": -81.69784545898438,
      "activations/layer4_attention_weight_max": 96.9807357788086,
      "activations/layer4_attention_weight_min": -92.88018035888672,
      "activations/layer5_attention_weight_max": 96.55133056640625,
      "activations/layer5_attention_weight_min": -86.09051513671875,
      "activations/layer6_attention_weight_max": 53.71040344238281,
      "activations/layer6_attention_weight_min": -52.08509826660156,
      "activations/layer7_attention_weight_max": 66.04285430908203,
      "activations/layer7_attention_weight_min": -67.71045684814453,
      "activations/layer8_attention_weight_max": 40.87061309814453,
      "activations/layer8_attention_weight_min": -40.405391693115234,
      "activations/layer9_attention_weight_max": 45.4376335144043,
      "activations/layer9_attention_weight_min": -45.5265007019043,
      "epoch": 5.37,
      "learning_rate": 0.00011653143939393939,
      "loss": 2.857,
      "step": 92450
    },
    {
      "activations/layer0_attention_weight_max": 15.180536270141602,
      "activations/layer0_attention_weight_min": -14.710700035095215,
      "activations/layer10_attention_weight_max": 37.51612091064453,
      "activations/layer10_attention_weight_min": -35.23284149169922,
      "activations/layer11_attention_weight_max": 28.74184226989746,
      "activations/layer11_attention_weight_min": -27.91011619567871,
      "activations/layer12_attention_weight_max": 18.56833839416504,
      "activations/layer12_attention_weight_min": -23.344261169433594,
      "activations/layer13_attention_weight_max": 49.90927505493164,
      "activations/layer13_attention_weight_min": -32.22956466674805,
      "activations/layer14_attention_weight_max": 46.141578674316406,
      "activations/layer14_attention_weight_min": -28.856464385986328,
      "activations/layer15_attention_weight_max": 37.04389190673828,
      "activations/layer15_attention_weight_min": -29.086688995361328,
      "activations/layer16_attention_weight_max": 38.91027069091797,
      "activations/layer16_attention_weight_min": -26.698583602905273,
      "activations/layer17_attention_weight_max": 57.16648483276367,
      "activations/layer17_attention_weight_min": -38.595909118652344,
      "activations/layer18_attention_weight_max": 47.53836441040039,
      "activations/layer18_attention_weight_min": -33.896575927734375,
      "activations/layer19_attention_weight_max": 20.784589767456055,
      "activations/layer19_attention_weight_min": -17.936689376831055,
      "activations/layer1_attention_weight_max": 17.802148818969727,
      "activations/layer1_attention_weight_min": -14.213608741760254,
      "activations/layer20_attention_weight_max": 26.276445388793945,
      "activations/layer20_attention_weight_min": -18.403783798217773,
      "activations/layer21_attention_weight_max": 45.95327377319336,
      "activations/layer21_attention_weight_min": -22.743837356567383,
      "activations/layer22_attention_weight_max": 37.73680877685547,
      "activations/layer22_attention_weight_min": -21.822101593017578,
      "activations/layer23_attention_weight_max": 30.096555709838867,
      "activations/layer23_attention_weight_min": -16.16400909423828,
      "activations/layer2_attention_weight_max": 32.24443054199219,
      "activations/layer2_attention_weight_min": -28.985374450683594,
      "activations/layer3_attention_weight_max": 84.20523071289062,
      "activations/layer3_attention_weight_min": -85.00785827636719,
      "activations/layer4_attention_weight_max": 99.4581527709961,
      "activations/layer4_attention_weight_min": -94.51008605957031,
      "activations/layer5_attention_weight_max": 99.45947265625,
      "activations/layer5_attention_weight_min": -91.37283325195312,
      "activations/layer6_attention_weight_max": 61.258766174316406,
      "activations/layer6_attention_weight_min": -52.99534606933594,
      "activations/layer7_attention_weight_max": 74.07566833496094,
      "activations/layer7_attention_weight_min": -70.21634674072266,
      "activations/layer8_attention_weight_max": 46.88557052612305,
      "activations/layer8_attention_weight_min": -47.21897506713867,
      "activations/layer9_attention_weight_max": 59.90394973754883,
      "activations/layer9_attention_weight_min": -55.379634857177734,
      "epoch": 5.37,
      "learning_rate": 0.00011651249999999999,
      "loss": 2.848,
      "step": 92500
    },
    {
      "activations/layer0_attention_weight_max": 13.733755111694336,
      "activations/layer0_attention_weight_min": -14.755678176879883,
      "activations/layer10_attention_weight_max": 37.029354095458984,
      "activations/layer10_attention_weight_min": -34.247528076171875,
      "activations/layer11_attention_weight_max": 29.18497085571289,
      "activations/layer11_attention_weight_min": -27.278884887695312,
      "activations/layer12_attention_weight_max": 23.974504470825195,
      "activations/layer12_attention_weight_min": -26.94443130493164,
      "activations/layer13_attention_weight_max": 54.76102066040039,
      "activations/layer13_attention_weight_min": -42.19404602050781,
      "activations/layer14_attention_weight_max": 44.31280517578125,
      "activations/layer14_attention_weight_min": -30.37982940673828,
      "activations/layer15_attention_weight_max": 41.65987777709961,
      "activations/layer15_attention_weight_min": -29.42251205444336,
      "activations/layer16_attention_weight_max": 36.365875244140625,
      "activations/layer16_attention_weight_min": -27.49003028869629,
      "activations/layer17_attention_weight_max": 53.13045120239258,
      "activations/layer17_attention_weight_min": -38.94681167602539,
      "activations/layer18_attention_weight_max": 52.202335357666016,
      "activations/layer18_attention_weight_min": -32.77289581298828,
      "activations/layer19_attention_weight_max": 23.793046951293945,
      "activations/layer19_attention_weight_min": -18.78005027770996,
      "activations/layer1_attention_weight_max": 18.281681060791016,
      "activations/layer1_attention_weight_min": -14.055604934692383,
      "activations/layer20_attention_weight_max": 22.635387420654297,
      "activations/layer20_attention_weight_min": -22.565502166748047,
      "activations/layer21_attention_weight_max": 41.23161315917969,
      "activations/layer21_attention_weight_min": -25.662912368774414,
      "activations/layer22_attention_weight_max": 36.816566467285156,
      "activations/layer22_attention_weight_min": -24.130294799804688,
      "activations/layer23_attention_weight_max": 31.202613830566406,
      "activations/layer23_attention_weight_min": -17.109315872192383,
      "activations/layer2_attention_weight_max": 28.98310661315918,
      "activations/layer2_attention_weight_min": -28.090045928955078,
      "activations/layer3_attention_weight_max": 79.14229583740234,
      "activations/layer3_attention_weight_min": -80.3151626586914,
      "activations/layer4_attention_weight_max": 100.8183822631836,
      "activations/layer4_attention_weight_min": -93.4964828491211,
      "activations/layer5_attention_weight_max": 99.03356170654297,
      "activations/layer5_attention_weight_min": -88.7706298828125,
      "activations/layer6_attention_weight_max": 55.564537048339844,
      "activations/layer6_attention_weight_min": -50.18296813964844,
      "activations/layer7_attention_weight_max": 71.02320098876953,
      "activations/layer7_attention_weight_min": -67.08602142333984,
      "activations/layer8_attention_weight_max": 40.863304138183594,
      "activations/layer8_attention_weight_min": -41.37576675415039,
      "activations/layer9_attention_weight_max": 52.614532470703125,
      "activations/layer9_attention_weight_min": -47.013214111328125,
      "epoch": 5.38,
      "learning_rate": 0.0001164935606060606,
      "loss": 2.8585,
      "step": 92550
    },
    {
      "activations/layer0_attention_weight_max": 14.862656593322754,
      "activations/layer0_attention_weight_min": -15.129599571228027,
      "activations/layer10_attention_weight_max": 37.52548599243164,
      "activations/layer10_attention_weight_min": -35.165958404541016,
      "activations/layer11_attention_weight_max": 29.549894332885742,
      "activations/layer11_attention_weight_min": -26.528812408447266,
      "activations/layer12_attention_weight_max": 29.08477210998535,
      "activations/layer12_attention_weight_min": -22.283166885375977,
      "activations/layer13_attention_weight_max": 42.73804473876953,
      "activations/layer13_attention_weight_min": -31.7022705078125,
      "activations/layer14_attention_weight_max": 35.15949630737305,
      "activations/layer14_attention_weight_min": -28.956592559814453,
      "activations/layer15_attention_weight_max": 38.884429931640625,
      "activations/layer15_attention_weight_min": -30.059226989746094,
      "activations/layer16_attention_weight_max": 34.52623748779297,
      "activations/layer16_attention_weight_min": -25.04373550415039,
      "activations/layer17_attention_weight_max": 55.05136489868164,
      "activations/layer17_attention_weight_min": -41.28178787231445,
      "activations/layer18_attention_weight_max": 46.63410568237305,
      "activations/layer18_attention_weight_min": -34.0095329284668,
      "activations/layer19_attention_weight_max": 21.590150833129883,
      "activations/layer19_attention_weight_min": -19.952251434326172,
      "activations/layer1_attention_weight_max": 17.14179801940918,
      "activations/layer1_attention_weight_min": -14.357986450195312,
      "activations/layer20_attention_weight_max": 19.914859771728516,
      "activations/layer20_attention_weight_min": -21.54399299621582,
      "activations/layer21_attention_weight_max": 37.654449462890625,
      "activations/layer21_attention_weight_min": -26.47581672668457,
      "activations/layer22_attention_weight_max": 34.60373306274414,
      "activations/layer22_attention_weight_min": -25.010068893432617,
      "activations/layer23_attention_weight_max": 29.155513763427734,
      "activations/layer23_attention_weight_min": -17.526813507080078,
      "activations/layer2_attention_weight_max": 28.619680404663086,
      "activations/layer2_attention_weight_min": -25.601295471191406,
      "activations/layer3_attention_weight_max": 77.48399353027344,
      "activations/layer3_attention_weight_min": -81.15767669677734,
      "activations/layer4_attention_weight_max": 94.53765106201172,
      "activations/layer4_attention_weight_min": -90.47726440429688,
      "activations/layer5_attention_weight_max": 96.7215576171875,
      "activations/layer5_attention_weight_min": -88.71196746826172,
      "activations/layer6_attention_weight_max": 56.52904510498047,
      "activations/layer6_attention_weight_min": -49.19755935668945,
      "activations/layer7_attention_weight_max": 64.77818298339844,
      "activations/layer7_attention_weight_min": -60.882850646972656,
      "activations/layer8_attention_weight_max": 42.59420394897461,
      "activations/layer8_attention_weight_min": -38.22755813598633,
      "activations/layer9_attention_weight_max": 48.05136489868164,
      "activations/layer9_attention_weight_min": -45.47829818725586,
      "epoch": 5.38,
      "learning_rate": 0.00011647462121212121,
      "loss": 2.8514,
      "step": 92600
    },
    {
      "activations/layer0_attention_weight_max": 15.420595169067383,
      "activations/layer0_attention_weight_min": -15.637019157409668,
      "activations/layer10_attention_weight_max": 38.6390380859375,
      "activations/layer10_attention_weight_min": -36.67269515991211,
      "activations/layer11_attention_weight_max": 29.320133209228516,
      "activations/layer11_attention_weight_min": -30.359371185302734,
      "activations/layer12_attention_weight_max": 24.29129981994629,
      "activations/layer12_attention_weight_min": -27.455228805541992,
      "activations/layer13_attention_weight_max": 43.898921966552734,
      "activations/layer13_attention_weight_min": -34.17628860473633,
      "activations/layer14_attention_weight_max": 54.61651611328125,
      "activations/layer14_attention_weight_min": -31.736705780029297,
      "activations/layer15_attention_weight_max": 40.83533477783203,
      "activations/layer15_attention_weight_min": -30.04806900024414,
      "activations/layer16_attention_weight_max": 39.014564514160156,
      "activations/layer16_attention_weight_min": -26.214698791503906,
      "activations/layer17_attention_weight_max": 59.7691535949707,
      "activations/layer17_attention_weight_min": -41.43513870239258,
      "activations/layer18_attention_weight_max": 50.14775466918945,
      "activations/layer18_attention_weight_min": -34.14398193359375,
      "activations/layer19_attention_weight_max": 23.182710647583008,
      "activations/layer19_attention_weight_min": -20.577939987182617,
      "activations/layer1_attention_weight_max": 17.36986541748047,
      "activations/layer1_attention_weight_min": -16.428138732910156,
      "activations/layer20_attention_weight_max": 23.629192352294922,
      "activations/layer20_attention_weight_min": -19.943880081176758,
      "activations/layer21_attention_weight_max": 43.220947265625,
      "activations/layer21_attention_weight_min": -25.130596160888672,
      "activations/layer22_attention_weight_max": 33.724021911621094,
      "activations/layer22_attention_weight_min": -25.84107208251953,
      "activations/layer23_attention_weight_max": 34.77983474731445,
      "activations/layer23_attention_weight_min": -19.75486183166504,
      "activations/layer2_attention_weight_max": 28.714168548583984,
      "activations/layer2_attention_weight_min": -27.668758392333984,
      "activations/layer3_attention_weight_max": 81.95694732666016,
      "activations/layer3_attention_weight_min": -88.03791809082031,
      "activations/layer4_attention_weight_max": 103.43934631347656,
      "activations/layer4_attention_weight_min": -100.37267303466797,
      "activations/layer5_attention_weight_max": 108.26275634765625,
      "activations/layer5_attention_weight_min": -96.46260070800781,
      "activations/layer6_attention_weight_max": 62.01444625854492,
      "activations/layer6_attention_weight_min": -55.92137908935547,
      "activations/layer7_attention_weight_max": 73.52262115478516,
      "activations/layer7_attention_weight_min": -70.8869400024414,
      "activations/layer8_attention_weight_max": 44.764034271240234,
      "activations/layer8_attention_weight_min": -44.797645568847656,
      "activations/layer9_attention_weight_max": 52.237403869628906,
      "activations/layer9_attention_weight_min": -46.67972946166992,
      "epoch": 5.38,
      "learning_rate": 0.0001164556818181818,
      "loss": 2.8187,
      "step": 92650
    },
    {
      "activations/layer0_attention_weight_max": 14.616416931152344,
      "activations/layer0_attention_weight_min": -15.548696517944336,
      "activations/layer10_attention_weight_max": 40.430198669433594,
      "activations/layer10_attention_weight_min": -35.913909912109375,
      "activations/layer11_attention_weight_max": 32.781612396240234,
      "activations/layer11_attention_weight_min": -29.203201293945312,
      "activations/layer12_attention_weight_max": 23.167722702026367,
      "activations/layer12_attention_weight_min": -22.871051788330078,
      "activations/layer13_attention_weight_max": 55.28071212768555,
      "activations/layer13_attention_weight_min": -37.5963249206543,
      "activations/layer14_attention_weight_max": 43.79994583129883,
      "activations/layer14_attention_weight_min": -30.53253936767578,
      "activations/layer15_attention_weight_max": 44.377193450927734,
      "activations/layer15_attention_weight_min": -29.745372772216797,
      "activations/layer16_attention_weight_max": 37.02196502685547,
      "activations/layer16_attention_weight_min": -27.285850524902344,
      "activations/layer17_attention_weight_max": 58.30286407470703,
      "activations/layer17_attention_weight_min": -39.60576629638672,
      "activations/layer18_attention_weight_max": 49.976688385009766,
      "activations/layer18_attention_weight_min": -34.66022872924805,
      "activations/layer19_attention_weight_max": 22.928964614868164,
      "activations/layer19_attention_weight_min": -19.775630950927734,
      "activations/layer1_attention_weight_max": 17.13692855834961,
      "activations/layer1_attention_weight_min": -15.178441047668457,
      "activations/layer20_attention_weight_max": 21.78099250793457,
      "activations/layer20_attention_weight_min": -19.47227668762207,
      "activations/layer21_attention_weight_max": 41.37580108642578,
      "activations/layer21_attention_weight_min": -24.707908630371094,
      "activations/layer22_attention_weight_max": 33.63948440551758,
      "activations/layer22_attention_weight_min": -25.024738311767578,
      "activations/layer23_attention_weight_max": 31.190486907958984,
      "activations/layer23_attention_weight_min": -18.694007873535156,
      "activations/layer2_attention_weight_max": 27.75520133972168,
      "activations/layer2_attention_weight_min": -27.90234375,
      "activations/layer3_attention_weight_max": 77.23372650146484,
      "activations/layer3_attention_weight_min": -83.20228576660156,
      "activations/layer4_attention_weight_max": 103.20515441894531,
      "activations/layer4_attention_weight_min": -97.0205078125,
      "activations/layer5_attention_weight_max": 101.17509460449219,
      "activations/layer5_attention_weight_min": -95.51054382324219,
      "activations/layer6_attention_weight_max": 59.50837707519531,
      "activations/layer6_attention_weight_min": -55.37052917480469,
      "activations/layer7_attention_weight_max": 72.64045715332031,
      "activations/layer7_attention_weight_min": -75.00543212890625,
      "activations/layer8_attention_weight_max": 43.16893768310547,
      "activations/layer8_attention_weight_min": -45.03462600708008,
      "activations/layer9_attention_weight_max": 55.485042572021484,
      "activations/layer9_attention_weight_min": -49.62532424926758,
      "epoch": 5.39,
      "learning_rate": 0.00011643674242424241,
      "loss": 2.8418,
      "step": 92700
    },
    {
      "activations/layer0_attention_weight_max": 17.416934967041016,
      "activations/layer0_attention_weight_min": -14.902957916259766,
      "activations/layer10_attention_weight_max": 39.121246337890625,
      "activations/layer10_attention_weight_min": -34.74602508544922,
      "activations/layer11_attention_weight_max": 33.225589752197266,
      "activations/layer11_attention_weight_min": -26.580907821655273,
      "activations/layer12_attention_weight_max": 29.295955657958984,
      "activations/layer12_attention_weight_min": -29.021259307861328,
      "activations/layer13_attention_weight_max": 62.7307014465332,
      "activations/layer13_attention_weight_min": -38.66196823120117,
      "activations/layer14_attention_weight_max": 69.71018981933594,
      "activations/layer14_attention_weight_min": -35.037193298339844,
      "activations/layer15_attention_weight_max": 49.12736511230469,
      "activations/layer15_attention_weight_min": -28.78577423095703,
      "activations/layer16_attention_weight_max": 44.69842529296875,
      "activations/layer16_attention_weight_min": -25.467151641845703,
      "activations/layer17_attention_weight_max": 56.330848693847656,
      "activations/layer17_attention_weight_min": -40.01178741455078,
      "activations/layer18_attention_weight_max": 46.12263488769531,
      "activations/layer18_attention_weight_min": -33.7747917175293,
      "activations/layer19_attention_weight_max": 22.192079544067383,
      "activations/layer19_attention_weight_min": -19.81492042541504,
      "activations/layer1_attention_weight_max": 17.00014305114746,
      "activations/layer1_attention_weight_min": -18.544227600097656,
      "activations/layer20_attention_weight_max": 23.07179832458496,
      "activations/layer20_attention_weight_min": -19.496105194091797,
      "activations/layer21_attention_weight_max": 48.192230224609375,
      "activations/layer21_attention_weight_min": -25.147855758666992,
      "activations/layer22_attention_weight_max": 35.64963150024414,
      "activations/layer22_attention_weight_min": -23.494714736938477,
      "activations/layer23_attention_weight_max": 28.722633361816406,
      "activations/layer23_attention_weight_min": -17.983598709106445,
      "activations/layer2_attention_weight_max": 30.752151489257812,
      "activations/layer2_attention_weight_min": -28.77106475830078,
      "activations/layer3_attention_weight_max": 82.95794677734375,
      "activations/layer3_attention_weight_min": -85.7945327758789,
      "activations/layer4_attention_weight_max": 100.04415130615234,
      "activations/layer4_attention_weight_min": -95.06411743164062,
      "activations/layer5_attention_weight_max": 100.90927124023438,
      "activations/layer5_attention_weight_min": -93.08379364013672,
      "activations/layer6_attention_weight_max": 56.31595230102539,
      "activations/layer6_attention_weight_min": -49.42104721069336,
      "activations/layer7_attention_weight_max": 81.1722412109375,
      "activations/layer7_attention_weight_min": -72.9374008178711,
      "activations/layer8_attention_weight_max": 50.39912414550781,
      "activations/layer8_attention_weight_min": -41.6280403137207,
      "activations/layer9_attention_weight_max": 67.14599609375,
      "activations/layer9_attention_weight_min": -52.33563995361328,
      "epoch": 5.39,
      "learning_rate": 0.00011641780303030303,
      "loss": 2.838,
      "step": 92750
    },
    {
      "activations/layer0_attention_weight_max": 14.760855674743652,
      "activations/layer0_attention_weight_min": -15.290864944458008,
      "activations/layer10_attention_weight_max": 39.936405181884766,
      "activations/layer10_attention_weight_min": -34.765113830566406,
      "activations/layer11_attention_weight_max": 29.910003662109375,
      "activations/layer11_attention_weight_min": -27.20290756225586,
      "activations/layer12_attention_weight_max": 24.242095947265625,
      "activations/layer12_attention_weight_min": -23.121932983398438,
      "activations/layer13_attention_weight_max": 48.542320251464844,
      "activations/layer13_attention_weight_min": -36.77228927612305,
      "activations/layer14_attention_weight_max": 45.10371780395508,
      "activations/layer14_attention_weight_min": -28.992691040039062,
      "activations/layer15_attention_weight_max": 38.23323059082031,
      "activations/layer15_attention_weight_min": -29.929046630859375,
      "activations/layer16_attention_weight_max": 32.20372772216797,
      "activations/layer16_attention_weight_min": -27.65812873840332,
      "activations/layer17_attention_weight_max": 54.47858810424805,
      "activations/layer17_attention_weight_min": -38.09722900390625,
      "activations/layer18_attention_weight_max": 50.08603286743164,
      "activations/layer18_attention_weight_min": -33.50589370727539,
      "activations/layer19_attention_weight_max": 20.534988403320312,
      "activations/layer19_attention_weight_min": -20.604188919067383,
      "activations/layer1_attention_weight_max": 17.584745407104492,
      "activations/layer1_attention_weight_min": -15.02563762664795,
      "activations/layer20_attention_weight_max": 20.164051055908203,
      "activations/layer20_attention_weight_min": -24.63347625732422,
      "activations/layer21_attention_weight_max": 41.03843307495117,
      "activations/layer21_attention_weight_min": -26.78554916381836,
      "activations/layer22_attention_weight_max": 35.811973571777344,
      "activations/layer22_attention_weight_min": -24.71259117126465,
      "activations/layer23_attention_weight_max": 27.815237045288086,
      "activations/layer23_attention_weight_min": -17.386802673339844,
      "activations/layer2_attention_weight_max": 30.019712448120117,
      "activations/layer2_attention_weight_min": -28.062847137451172,
      "activations/layer3_attention_weight_max": 83.59757995605469,
      "activations/layer3_attention_weight_min": -84.71241760253906,
      "activations/layer4_attention_weight_max": 100.3167953491211,
      "activations/layer4_attention_weight_min": -95.72710418701172,
      "activations/layer5_attention_weight_max": 100.85792541503906,
      "activations/layer5_attention_weight_min": -92.86846923828125,
      "activations/layer6_attention_weight_max": 57.0009765625,
      "activations/layer6_attention_weight_min": -52.648765563964844,
      "activations/layer7_attention_weight_max": 74.60408020019531,
      "activations/layer7_attention_weight_min": -67.22827911376953,
      "activations/layer8_attention_weight_max": 43.62661361694336,
      "activations/layer8_attention_weight_min": -41.39267349243164,
      "activations/layer9_attention_weight_max": 55.01417922973633,
      "activations/layer9_attention_weight_min": -47.94929885864258,
      "epoch": 5.39,
      "learning_rate": 0.00011639924242424242,
      "loss": 2.848,
      "step": 92800
    },
    {
      "activations/layer0_attention_weight_max": 15.461484909057617,
      "activations/layer0_attention_weight_min": -15.339010238647461,
      "activations/layer10_attention_weight_max": 34.697696685791016,
      "activations/layer10_attention_weight_min": -34.570045471191406,
      "activations/layer11_attention_weight_max": 29.94638442993164,
      "activations/layer11_attention_weight_min": -26.420425415039062,
      "activations/layer12_attention_weight_max": 23.199031829833984,
      "activations/layer12_attention_weight_min": -26.124515533447266,
      "activations/layer13_attention_weight_max": 44.65052795410156,
      "activations/layer13_attention_weight_min": -32.60150146484375,
      "activations/layer14_attention_weight_max": 36.24002456665039,
      "activations/layer14_attention_weight_min": -30.023664474487305,
      "activations/layer15_attention_weight_max": 36.02682876586914,
      "activations/layer15_attention_weight_min": -29.535846710205078,
      "activations/layer16_attention_weight_max": 34.19822692871094,
      "activations/layer16_attention_weight_min": -28.597591400146484,
      "activations/layer17_attention_weight_max": 52.81005859375,
      "activations/layer17_attention_weight_min": -37.780757904052734,
      "activations/layer18_attention_weight_max": 42.5201301574707,
      "activations/layer18_attention_weight_min": -33.87916946411133,
      "activations/layer19_attention_weight_max": 22.145301818847656,
      "activations/layer19_attention_weight_min": -19.318912506103516,
      "activations/layer1_attention_weight_max": 16.83106231689453,
      "activations/layer1_attention_weight_min": -17.96756935119629,
      "activations/layer20_attention_weight_max": 22.054241180419922,
      "activations/layer20_attention_weight_min": -24.275875091552734,
      "activations/layer21_attention_weight_max": 37.5470085144043,
      "activations/layer21_attention_weight_min": -26.102659225463867,
      "activations/layer22_attention_weight_max": 32.150753021240234,
      "activations/layer22_attention_weight_min": -23.95318603515625,
      "activations/layer23_attention_weight_max": 27.70424461364746,
      "activations/layer23_attention_weight_min": -19.61777687072754,
      "activations/layer2_attention_weight_max": 28.728763580322266,
      "activations/layer2_attention_weight_min": -26.761226654052734,
      "activations/layer3_attention_weight_max": 80.03009796142578,
      "activations/layer3_attention_weight_min": -82.0546875,
      "activations/layer4_attention_weight_max": 99.36070251464844,
      "activations/layer4_attention_weight_min": -94.89948272705078,
      "activations/layer5_attention_weight_max": 99.3833236694336,
      "activations/layer5_attention_weight_min": -91.16049194335938,
      "activations/layer6_attention_weight_max": 56.99998474121094,
      "activations/layer6_attention_weight_min": -51.52880096435547,
      "activations/layer7_attention_weight_max": 71.37166595458984,
      "activations/layer7_attention_weight_min": -64.6644058227539,
      "activations/layer8_attention_weight_max": 43.879268646240234,
      "activations/layer8_attention_weight_min": -42.05674362182617,
      "activations/layer9_attention_weight_max": 47.82160568237305,
      "activations/layer9_attention_weight_min": -47.57230758666992,
      "epoch": 5.4,
      "learning_rate": 0.000116380303030303,
      "loss": 2.8614,
      "step": 92850
    },
    {
      "activations/layer0_attention_weight_max": 14.600711822509766,
      "activations/layer0_attention_weight_min": -14.729275703430176,
      "activations/layer10_attention_weight_max": 38.95183181762695,
      "activations/layer10_attention_weight_min": -35.540740966796875,
      "activations/layer11_attention_weight_max": 29.868114471435547,
      "activations/layer11_attention_weight_min": -27.4545955657959,
      "activations/layer12_attention_weight_max": 24.139236450195312,
      "activations/layer12_attention_weight_min": -28.42534065246582,
      "activations/layer13_attention_weight_max": 60.00727081298828,
      "activations/layer13_attention_weight_min": -42.986106872558594,
      "activations/layer14_attention_weight_max": 51.44820022583008,
      "activations/layer14_attention_weight_min": -37.76887893676758,
      "activations/layer15_attention_weight_max": 40.6251335144043,
      "activations/layer15_attention_weight_min": -33.31687545776367,
      "activations/layer16_attention_weight_max": 35.71855926513672,
      "activations/layer16_attention_weight_min": -26.609882354736328,
      "activations/layer17_attention_weight_max": 57.75887680053711,
      "activations/layer17_attention_weight_min": -39.620887756347656,
      "activations/layer18_attention_weight_max": 49.38539123535156,
      "activations/layer18_attention_weight_min": -33.73361587524414,
      "activations/layer19_attention_weight_max": 27.22478675842285,
      "activations/layer19_attention_weight_min": -18.935056686401367,
      "activations/layer1_attention_weight_max": 17.40265464782715,
      "activations/layer1_attention_weight_min": -17.299741744995117,
      "activations/layer20_attention_weight_max": 22.486553192138672,
      "activations/layer20_attention_weight_min": -18.46410369873047,
      "activations/layer21_attention_weight_max": 40.85045623779297,
      "activations/layer21_attention_weight_min": -23.82207679748535,
      "activations/layer22_attention_weight_max": 37.138301849365234,
      "activations/layer22_attention_weight_min": -27.005111694335938,
      "activations/layer23_attention_weight_max": 31.411632537841797,
      "activations/layer23_attention_weight_min": -18.072860717773438,
      "activations/layer2_attention_weight_max": 28.63555145263672,
      "activations/layer2_attention_weight_min": -27.476226806640625,
      "activations/layer3_attention_weight_max": 81.85212707519531,
      "activations/layer3_attention_weight_min": -80.02401733398438,
      "activations/layer4_attention_weight_max": 101.20296478271484,
      "activations/layer4_attention_weight_min": -96.64171600341797,
      "activations/layer5_attention_weight_max": 101.37469482421875,
      "activations/layer5_attention_weight_min": -92.17691040039062,
      "activations/layer6_attention_weight_max": 61.27141189575195,
      "activations/layer6_attention_weight_min": -56.108890533447266,
      "activations/layer7_attention_weight_max": 74.6521224975586,
      "activations/layer7_attention_weight_min": -71.13407897949219,
      "activations/layer8_attention_weight_max": 47.67587661743164,
      "activations/layer8_attention_weight_min": -43.769100189208984,
      "activations/layer9_attention_weight_max": 61.31098556518555,
      "activations/layer9_attention_weight_min": -49.97547912597656,
      "epoch": 5.4,
      "learning_rate": 0.00011636136363636362,
      "loss": 2.8565,
      "step": 92900
    },
    {
      "activations/layer0_attention_weight_max": 15.294918060302734,
      "activations/layer0_attention_weight_min": -14.877077102661133,
      "activations/layer10_attention_weight_max": 35.78871536254883,
      "activations/layer10_attention_weight_min": -32.65022659301758,
      "activations/layer11_attention_weight_max": 28.420928955078125,
      "activations/layer11_attention_weight_min": -27.888553619384766,
      "activations/layer12_attention_weight_max": 22.92649269104004,
      "activations/layer12_attention_weight_min": -26.700281143188477,
      "activations/layer13_attention_weight_max": 47.096778869628906,
      "activations/layer13_attention_weight_min": -35.90298843383789,
      "activations/layer14_attention_weight_max": 43.38554763793945,
      "activations/layer14_attention_weight_min": -31.735301971435547,
      "activations/layer15_attention_weight_max": 37.68191146850586,
      "activations/layer15_attention_weight_min": -29.639842987060547,
      "activations/layer16_attention_weight_max": 37.526397705078125,
      "activations/layer16_attention_weight_min": -26.090299606323242,
      "activations/layer17_attention_weight_max": 54.95903396606445,
      "activations/layer17_attention_weight_min": -37.94337844848633,
      "activations/layer18_attention_weight_max": 43.87086486816406,
      "activations/layer18_attention_weight_min": -31.888437271118164,
      "activations/layer19_attention_weight_max": 20.456995010375977,
      "activations/layer19_attention_weight_min": -18.192983627319336,
      "activations/layer1_attention_weight_max": 16.837108612060547,
      "activations/layer1_attention_weight_min": -14.825510025024414,
      "activations/layer20_attention_weight_max": 20.24570083618164,
      "activations/layer20_attention_weight_min": -22.093730926513672,
      "activations/layer21_attention_weight_max": 46.231056213378906,
      "activations/layer21_attention_weight_min": -27.274436950683594,
      "activations/layer22_attention_weight_max": 33.620513916015625,
      "activations/layer22_attention_weight_min": -23.15367889404297,
      "activations/layer23_attention_weight_max": 27.730451583862305,
      "activations/layer23_attention_weight_min": -18.241653442382812,
      "activations/layer2_attention_weight_max": 29.07367515563965,
      "activations/layer2_attention_weight_min": -27.028236389160156,
      "activations/layer3_attention_weight_max": 77.21958923339844,
      "activations/layer3_attention_weight_min": -81.1335678100586,
      "activations/layer4_attention_weight_max": 96.43802642822266,
      "activations/layer4_attention_weight_min": -93.5461196899414,
      "activations/layer5_attention_weight_max": 97.15757751464844,
      "activations/layer5_attention_weight_min": -87.2708969116211,
      "activations/layer6_attention_weight_max": 56.14716339111328,
      "activations/layer6_attention_weight_min": -52.159568786621094,
      "activations/layer7_attention_weight_max": 75.5743179321289,
      "activations/layer7_attention_weight_min": -66.72599792480469,
      "activations/layer8_attention_weight_max": 40.9870491027832,
      "activations/layer8_attention_weight_min": -39.84998321533203,
      "activations/layer9_attention_weight_max": 50.40700912475586,
      "activations/layer9_attention_weight_min": -46.61017990112305,
      "epoch": 5.4,
      "learning_rate": 0.00011634242424242424,
      "loss": 2.8529,
      "step": 92950
    },
    {
      "activations/layer0_attention_weight_max": 15.933079719543457,
      "activations/layer0_attention_weight_min": -14.911717414855957,
      "activations/layer10_attention_weight_max": 36.44464874267578,
      "activations/layer10_attention_weight_min": -33.92718505859375,
      "activations/layer11_attention_weight_max": 30.52713394165039,
      "activations/layer11_attention_weight_min": -28.368213653564453,
      "activations/layer12_attention_weight_max": 26.459732055664062,
      "activations/layer12_attention_weight_min": -24.728851318359375,
      "activations/layer13_attention_weight_max": 52.51892852783203,
      "activations/layer13_attention_weight_min": -33.76898956298828,
      "activations/layer14_attention_weight_max": 48.33819580078125,
      "activations/layer14_attention_weight_min": -31.855777740478516,
      "activations/layer15_attention_weight_max": 41.89542007446289,
      "activations/layer15_attention_weight_min": -31.38814926147461,
      "activations/layer16_attention_weight_max": 38.585636138916016,
      "activations/layer16_attention_weight_min": -26.477319717407227,
      "activations/layer17_attention_weight_max": 61.18685531616211,
      "activations/layer17_attention_weight_min": -41.00267791748047,
      "activations/layer18_attention_weight_max": 51.3547477722168,
      "activations/layer18_attention_weight_min": -35.21446228027344,
      "activations/layer19_attention_weight_max": 21.984962463378906,
      "activations/layer19_attention_weight_min": -21.240415573120117,
      "activations/layer1_attention_weight_max": 18.544572830200195,
      "activations/layer1_attention_weight_min": -14.552417755126953,
      "activations/layer20_attention_weight_max": 26.016468048095703,
      "activations/layer20_attention_weight_min": -22.367088317871094,
      "activations/layer21_attention_weight_max": 43.77547836303711,
      "activations/layer21_attention_weight_min": -26.99222183227539,
      "activations/layer22_attention_weight_max": 41.15956497192383,
      "activations/layer22_attention_weight_min": -23.82875633239746,
      "activations/layer23_attention_weight_max": 31.687040328979492,
      "activations/layer23_attention_weight_min": -19.398792266845703,
      "activations/layer2_attention_weight_max": 29.0542049407959,
      "activations/layer2_attention_weight_min": -26.286230087280273,
      "activations/layer3_attention_weight_max": 79.62482452392578,
      "activations/layer3_attention_weight_min": -81.13196563720703,
      "activations/layer4_attention_weight_max": 96.58419036865234,
      "activations/layer4_attention_weight_min": -94.85032653808594,
      "activations/layer5_attention_weight_max": 97.36251068115234,
      "activations/layer5_attention_weight_min": -90.5103988647461,
      "activations/layer6_attention_weight_max": 61.28459167480469,
      "activations/layer6_attention_weight_min": -53.21840286254883,
      "activations/layer7_attention_weight_max": 72.45494079589844,
      "activations/layer7_attention_weight_min": -69.45968627929688,
      "activations/layer8_attention_weight_max": 40.77143478393555,
      "activations/layer8_attention_weight_min": -43.25506591796875,
      "activations/layer9_attention_weight_max": 50.46001052856445,
      "activations/layer9_attention_weight_min": -48.48588943481445,
      "epoch": 5.4,
      "learning_rate": 0.00011632348484848484,
      "loss": 2.8649,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_loss": 2.796875,
      "eval_runtime": 8.5042,
      "eval_samples_per_second": 504.924,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_openwebtext_loss": 2.796875,
      "eval_openwebtext_ppl": 16.39333746248769,
      "eval_openwebtext_runtime": 8.5042,
      "eval_openwebtext_samples_per_second": 504.924,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_wikitext_loss": 3.025390625,
      "eval_wikitext_ppl": 20.602050815688813,
      "eval_wikitext_runtime": 2.026,
      "eval_wikitext_samples_per_second": 225.077,
      "step": 93000
    },
    {
      "epoch": 5.4,
      "eval_lambada_loss": 2.734375,
      "eval_lambada_ppl": 15.400115355370453,
      "eval_lambada_runtime": 9.6394,
      "eval_lambada_samples_per_second": 505.114,
      "step": 93000
    },
    {
      "activations/layer0_attention_weight_max": 14.433452606201172,
      "activations/layer0_attention_weight_min": -14.733290672302246,
      "activations/layer10_attention_weight_max": 37.449241638183594,
      "activations/layer10_attention_weight_min": -34.576141357421875,
      "activations/layer11_attention_weight_max": 28.494298934936523,
      "activations/layer11_attention_weight_min": -27.796478271484375,
      "activations/layer12_attention_weight_max": 24.04161262512207,
      "activations/layer12_attention_weight_min": -26.92201042175293,
      "activations/layer13_attention_weight_max": 50.20751190185547,
      "activations/layer13_attention_weight_min": -34.257423400878906,
      "activations/layer14_attention_weight_max": 44.986915588378906,
      "activations/layer14_attention_weight_min": -29.23452377319336,
      "activations/layer15_attention_weight_max": 38.88365173339844,
      "activations/layer15_attention_weight_min": -28.407615661621094,
      "activations/layer16_attention_weight_max": 34.410362243652344,
      "activations/layer16_attention_weight_min": -25.674163818359375,
      "activations/layer17_attention_weight_max": 57.15891647338867,
      "activations/layer17_attention_weight_min": -39.72111892700195,
      "activations/layer18_attention_weight_max": 51.09727096557617,
      "activations/layer18_attention_weight_min": -33.165985107421875,
      "activations/layer19_attention_weight_max": 21.661832809448242,
      "activations/layer19_attention_weight_min": -21.032270431518555,
      "activations/layer1_attention_weight_max": 17.522356033325195,
      "activations/layer1_attention_weight_min": -11.955409049987793,
      "activations/layer20_attention_weight_max": 24.914743423461914,
      "activations/layer20_attention_weight_min": -20.942569732666016,
      "activations/layer21_attention_weight_max": 43.2115592956543,
      "activations/layer21_attention_weight_min": -24.766984939575195,
      "activations/layer22_attention_weight_max": 36.53256607055664,
      "activations/layer22_attention_weight_min": -23.881027221679688,
      "activations/layer23_attention_weight_max": 30.150001525878906,
      "activations/layer23_attention_weight_min": -19.947444915771484,
      "activations/layer2_attention_weight_max": 30.412261962890625,
      "activations/layer2_attention_weight_min": -27.46755599975586,
      "activations/layer3_attention_weight_max": 81.76497650146484,
      "activations/layer3_attention_weight_min": -77.78855895996094,
      "activations/layer4_attention_weight_max": 95.6113510131836,
      "activations/layer4_attention_weight_min": -86.20054626464844,
      "activations/layer5_attention_weight_max": 95.47648620605469,
      "activations/layer5_attention_weight_min": -85.1158447265625,
      "activations/layer6_attention_weight_max": 56.333656311035156,
      "activations/layer6_attention_weight_min": -48.21729278564453,
      "activations/layer7_attention_weight_max": 69.29090881347656,
      "activations/layer7_attention_weight_min": -68.81272888183594,
      "activations/layer8_attention_weight_max": 41.35043716430664,
      "activations/layer8_attention_weight_min": -39.079689025878906,
      "activations/layer9_attention_weight_max": 48.091583251953125,
      "activations/layer9_attention_weight_min": -46.17810821533203,
      "epoch": 5.41,
      "learning_rate": 0.00011630454545454544,
      "loss": 2.8537,
      "step": 93050
    },
    {
      "activations/layer0_attention_weight_max": 14.051868438720703,
      "activations/layer0_attention_weight_min": -14.268718719482422,
      "activations/layer10_attention_weight_max": 35.23851776123047,
      "activations/layer10_attention_weight_min": -34.548583984375,
      "activations/layer11_attention_weight_max": 30.244892120361328,
      "activations/layer11_attention_weight_min": -27.91277313232422,
      "activations/layer12_attention_weight_max": 20.127424240112305,
      "activations/layer12_attention_weight_min": -23.900623321533203,
      "activations/layer13_attention_weight_max": 46.965309143066406,
      "activations/layer13_attention_weight_min": -31.476680755615234,
      "activations/layer14_attention_weight_max": 47.94239807128906,
      "activations/layer14_attention_weight_min": -31.324995040893555,
      "activations/layer15_attention_weight_max": 43.650691986083984,
      "activations/layer15_attention_weight_min": -30.463415145874023,
      "activations/layer16_attention_weight_max": 34.5715446472168,
      "activations/layer16_attention_weight_min": -26.131351470947266,
      "activations/layer17_attention_weight_max": 60.29243850708008,
      "activations/layer17_attention_weight_min": -40.809207916259766,
      "activations/layer18_attention_weight_max": 49.655601501464844,
      "activations/layer18_attention_weight_min": -34.453330993652344,
      "activations/layer19_attention_weight_max": 23.639291763305664,
      "activations/layer19_attention_weight_min": -20.384552001953125,
      "activations/layer1_attention_weight_max": 17.005144119262695,
      "activations/layer1_attention_weight_min": -13.060580253601074,
      "activations/layer20_attention_weight_max": 22.85065269470215,
      "activations/layer20_attention_weight_min": -23.24983024597168,
      "activations/layer21_attention_weight_max": 39.43361282348633,
      "activations/layer21_attention_weight_min": -26.891740798950195,
      "activations/layer22_attention_weight_max": 35.662471771240234,
      "activations/layer22_attention_weight_min": -26.04709243774414,
      "activations/layer23_attention_weight_max": 31.5266170501709,
      "activations/layer23_attention_weight_min": -20.562284469604492,
      "activations/layer2_attention_weight_max": 28.884418487548828,
      "activations/layer2_attention_weight_min": -26.80206298828125,
      "activations/layer3_attention_weight_max": 78.23529815673828,
      "activations/layer3_attention_weight_min": -79.93516540527344,
      "activations/layer4_attention_weight_max": 99.11681365966797,
      "activations/layer4_attention_weight_min": -93.19569396972656,
      "activations/layer5_attention_weight_max": 99.89877319335938,
      "activations/layer5_attention_weight_min": -89.01314544677734,
      "activations/layer6_attention_weight_max": 55.72883605957031,
      "activations/layer6_attention_weight_min": -51.07340621948242,
      "activations/layer7_attention_weight_max": 69.41554260253906,
      "activations/layer7_attention_weight_min": -63.76051330566406,
      "activations/layer8_attention_weight_max": 41.77033233642578,
      "activations/layer8_attention_weight_min": -43.43690490722656,
      "activations/layer9_attention_weight_max": 47.74299621582031,
      "activations/layer9_attention_weight_min": -44.87327575683594,
      "epoch": 5.41,
      "learning_rate": 0.00011628560606060605,
      "loss": 2.8501,
      "step": 93100
    },
    {
      "activations/layer0_attention_weight_max": 15.239892959594727,
      "activations/layer0_attention_weight_min": -14.310848236083984,
      "activations/layer10_attention_weight_max": 39.00586700439453,
      "activations/layer10_attention_weight_min": -34.35612869262695,
      "activations/layer11_attention_weight_max": 28.5767822265625,
      "activations/layer11_attention_weight_min": -26.787031173706055,
      "activations/layer12_attention_weight_max": 21.274106979370117,
      "activations/layer12_attention_weight_min": -24.6147403717041,
      "activations/layer13_attention_weight_max": 51.50395202636719,
      "activations/layer13_attention_weight_min": -39.453548431396484,
      "activations/layer14_attention_weight_max": 42.40473937988281,
      "activations/layer14_attention_weight_min": -29.792505264282227,
      "activations/layer15_attention_weight_max": 39.01451873779297,
      "activations/layer15_attention_weight_min": -30.540620803833008,
      "activations/layer16_attention_weight_max": 38.28667068481445,
      "activations/layer16_attention_weight_min": -26.970783233642578,
      "activations/layer17_attention_weight_max": 57.50240707397461,
      "activations/layer17_attention_weight_min": -40.6793098449707,
      "activations/layer18_attention_weight_max": 48.77077865600586,
      "activations/layer18_attention_weight_min": -35.90019989013672,
      "activations/layer19_attention_weight_max": 20.480144500732422,
      "activations/layer19_attention_weight_min": -24.66586685180664,
      "activations/layer1_attention_weight_max": 17.445966720581055,
      "activations/layer1_attention_weight_min": -15.798502922058105,
      "activations/layer20_attention_weight_max": 20.823904037475586,
      "activations/layer20_attention_weight_min": -21.135005950927734,
      "activations/layer21_attention_weight_max": 42.14105987548828,
      "activations/layer21_attention_weight_min": -26.452518463134766,
      "activations/layer22_attention_weight_max": 32.17937088012695,
      "activations/layer22_attention_weight_min": -22.34530258178711,
      "activations/layer23_attention_weight_max": 24.61298179626465,
      "activations/layer23_attention_weight_min": -22.799697875976562,
      "activations/layer2_attention_weight_max": 30.11817169189453,
      "activations/layer2_attention_weight_min": -27.31966781616211,
      "activations/layer3_attention_weight_max": 82.31979370117188,
      "activations/layer3_attention_weight_min": -84.27227783203125,
      "activations/layer4_attention_weight_max": 103.0451889038086,
      "activations/layer4_attention_weight_min": -96.58455657958984,
      "activations/layer5_attention_weight_max": 101.69039916992188,
      "activations/layer5_attention_weight_min": -97.5132064819336,
      "activations/layer6_attention_weight_max": 62.24034118652344,
      "activations/layer6_attention_weight_min": -57.16397476196289,
      "activations/layer7_attention_weight_max": 74.01660919189453,
      "activations/layer7_attention_weight_min": -68.60803985595703,
      "activations/layer8_attention_weight_max": 41.85213088989258,
      "activations/layer8_attention_weight_min": -41.72568130493164,
      "activations/layer9_attention_weight_max": 50.26517868041992,
      "activations/layer9_attention_weight_min": -46.49319076538086,
      "epoch": 5.41,
      "learning_rate": 0.00011626666666666666,
      "loss": 2.8389,
      "step": 93150
    },
    {
      "activations/layer0_attention_weight_max": 16.64179229736328,
      "activations/layer0_attention_weight_min": -14.265085220336914,
      "activations/layer10_attention_weight_max": 37.87458038330078,
      "activations/layer10_attention_weight_min": -33.00044250488281,
      "activations/layer11_attention_weight_max": 29.191390991210938,
      "activations/layer11_attention_weight_min": -27.170509338378906,
      "activations/layer12_attention_weight_max": 22.853513717651367,
      "activations/layer12_attention_weight_min": -26.900026321411133,
      "activations/layer13_attention_weight_max": 57.261817932128906,
      "activations/layer13_attention_weight_min": -34.71514892578125,
      "activations/layer14_attention_weight_max": 62.003143310546875,
      "activations/layer14_attention_weight_min": -38.74836349487305,
      "activations/layer15_attention_weight_max": 44.620479583740234,
      "activations/layer15_attention_weight_min": -29.526294708251953,
      "activations/layer16_attention_weight_max": 34.48072814941406,
      "activations/layer16_attention_weight_min": -29.028554916381836,
      "activations/layer17_attention_weight_max": 60.62037658691406,
      "activations/layer17_attention_weight_min": -41.47320556640625,
      "activations/layer18_attention_weight_max": 49.452877044677734,
      "activations/layer18_attention_weight_min": -34.7050895690918,
      "activations/layer19_attention_weight_max": 23.24888038635254,
      "activations/layer19_attention_weight_min": -19.753437042236328,
      "activations/layer1_attention_weight_max": 16.778039932250977,
      "activations/layer1_attention_weight_min": -13.404199600219727,
      "activations/layer20_attention_weight_max": 24.168954849243164,
      "activations/layer20_attention_weight_min": -23.789199829101562,
      "activations/layer21_attention_weight_max": 42.22885513305664,
      "activations/layer21_attention_weight_min": -28.736045837402344,
      "activations/layer22_attention_weight_max": 34.40721893310547,
      "activations/layer22_attention_weight_min": -23.263723373413086,
      "activations/layer23_attention_weight_max": 31.187185287475586,
      "activations/layer23_attention_weight_min": -17.966218948364258,
      "activations/layer2_attention_weight_max": 29.757993698120117,
      "activations/layer2_attention_weight_min": -28.70757293701172,
      "activations/layer3_attention_weight_max": 82.89236450195312,
      "activations/layer3_attention_weight_min": -84.3187484741211,
      "activations/layer4_attention_weight_max": 99.63754272460938,
      "activations/layer4_attention_weight_min": -94.37939453125,
      "activations/layer5_attention_weight_max": 102.53939056396484,
      "activations/layer5_attention_weight_min": -88.9365463256836,
      "activations/layer6_attention_weight_max": 60.121795654296875,
      "activations/layer6_attention_weight_min": -54.2678337097168,
      "activations/layer7_attention_weight_max": 68.4883804321289,
      "activations/layer7_attention_weight_min": -69.18831634521484,
      "activations/layer8_attention_weight_max": 44.95573043823242,
      "activations/layer8_attention_weight_min": -46.01621627807617,
      "activations/layer9_attention_weight_max": 54.06473922729492,
      "activations/layer9_attention_weight_min": -47.5728759765625,
      "epoch": 5.42,
      "learning_rate": 0.00011624772727272727,
      "loss": 2.8514,
      "step": 93200
    },
    {
      "activations/layer0_attention_weight_max": 13.998719215393066,
      "activations/layer0_attention_weight_min": -14.465632438659668,
      "activations/layer10_attention_weight_max": 38.96619415283203,
      "activations/layer10_attention_weight_min": -35.915985107421875,
      "activations/layer11_attention_weight_max": 29.683067321777344,
      "activations/layer11_attention_weight_min": -28.534955978393555,
      "activations/layer12_attention_weight_max": 20.364852905273438,
      "activations/layer12_attention_weight_min": -23.800512313842773,
      "activations/layer13_attention_weight_max": 40.33259963989258,
      "activations/layer13_attention_weight_min": -31.663854598999023,
      "activations/layer14_attention_weight_max": 39.848567962646484,
      "activations/layer14_attention_weight_min": -30.029287338256836,
      "activations/layer15_attention_weight_max": 38.711551666259766,
      "activations/layer15_attention_weight_min": -31.14756202697754,
      "activations/layer16_attention_weight_max": 34.49501037597656,
      "activations/layer16_attention_weight_min": -25.229942321777344,
      "activations/layer17_attention_weight_max": 52.01925277709961,
      "activations/layer17_attention_weight_min": -39.0648078918457,
      "activations/layer18_attention_weight_max": 46.2369384765625,
      "activations/layer18_attention_weight_min": -33.7393913269043,
      "activations/layer19_attention_weight_max": 24.823015213012695,
      "activations/layer19_attention_weight_min": -20.778974533081055,
      "activations/layer1_attention_weight_max": 17.327407836914062,
      "activations/layer1_attention_weight_min": -15.153286933898926,
      "activations/layer20_attention_weight_max": 20.91905975341797,
      "activations/layer20_attention_weight_min": -21.219989776611328,
      "activations/layer21_attention_weight_max": 35.52748107910156,
      "activations/layer21_attention_weight_min": -25.066484451293945,
      "activations/layer22_attention_weight_max": 32.58995056152344,
      "activations/layer22_attention_weight_min": -23.64988899230957,
      "activations/layer23_attention_weight_max": 27.035083770751953,
      "activations/layer23_attention_weight_min": -19.767318725585938,
      "activations/layer2_attention_weight_max": 28.487831115722656,
      "activations/layer2_attention_weight_min": -27.39095687866211,
      "activations/layer3_attention_weight_max": 78.4974365234375,
      "activations/layer3_attention_weight_min": -83.3506088256836,
      "activations/layer4_attention_weight_max": 98.0557632446289,
      "activations/layer4_attention_weight_min": -91.63993072509766,
      "activations/layer5_attention_weight_max": 96.6480941772461,
      "activations/layer5_attention_weight_min": -88.14836883544922,
      "activations/layer6_attention_weight_max": 59.70927047729492,
      "activations/layer6_attention_weight_min": -52.97427749633789,
      "activations/layer7_attention_weight_max": 68.39874267578125,
      "activations/layer7_attention_weight_min": -64.45938110351562,
      "activations/layer8_attention_weight_max": 42.876312255859375,
      "activations/layer8_attention_weight_min": -41.54669189453125,
      "activations/layer9_attention_weight_max": 49.2092399597168,
      "activations/layer9_attention_weight_min": -45.10189437866211,
      "epoch": 5.42,
      "learning_rate": 0.00011622878787878787,
      "loss": 2.8425,
      "step": 93250
    },
    {
      "activations/layer0_attention_weight_max": 14.566640853881836,
      "activations/layer0_attention_weight_min": -14.55406379699707,
      "activations/layer10_attention_weight_max": 37.93287658691406,
      "activations/layer10_attention_weight_min": -34.187744140625,
      "activations/layer11_attention_weight_max": 31.529531478881836,
      "activations/layer11_attention_weight_min": -28.464523315429688,
      "activations/layer12_attention_weight_max": 21.359668731689453,
      "activations/layer12_attention_weight_min": -26.245529174804688,
      "activations/layer13_attention_weight_max": 43.04255294799805,
      "activations/layer13_attention_weight_min": -34.866912841796875,
      "activations/layer14_attention_weight_max": 38.87184524536133,
      "activations/layer14_attention_weight_min": -27.74875259399414,
      "activations/layer15_attention_weight_max": 38.073368072509766,
      "activations/layer15_attention_weight_min": -28.994937896728516,
      "activations/layer16_attention_weight_max": 34.120758056640625,
      "activations/layer16_attention_weight_min": -25.129995346069336,
      "activations/layer17_attention_weight_max": 55.61295700073242,
      "activations/layer17_attention_weight_min": -38.72449493408203,
      "activations/layer18_attention_weight_max": 49.48518753051758,
      "activations/layer18_attention_weight_min": -30.71357536315918,
      "activations/layer19_attention_weight_max": 21.99093246459961,
      "activations/layer19_attention_weight_min": -18.979124069213867,
      "activations/layer1_attention_weight_max": 17.05299949645996,
      "activations/layer1_attention_weight_min": -13.871085166931152,
      "activations/layer20_attention_weight_max": 21.033132553100586,
      "activations/layer20_attention_weight_min": -21.60205841064453,
      "activations/layer21_attention_weight_max": 41.4617805480957,
      "activations/layer21_attention_weight_min": -25.828062057495117,
      "activations/layer22_attention_weight_max": 31.386831283569336,
      "activations/layer22_attention_weight_min": -23.611509323120117,
      "activations/layer23_attention_weight_max": 29.245208740234375,
      "activations/layer23_attention_weight_min": -17.86819839477539,
      "activations/layer2_attention_weight_max": 28.606666564941406,
      "activations/layer2_attention_weight_min": -26.87459945678711,
      "activations/layer3_attention_weight_max": 80.51538848876953,
      "activations/layer3_attention_weight_min": -83.53892517089844,
      "activations/layer4_attention_weight_max": 96.22337341308594,
      "activations/layer4_attention_weight_min": -91.3780746459961,
      "activations/layer5_attention_weight_max": 98.70906066894531,
      "activations/layer5_attention_weight_min": -86.43123626708984,
      "activations/layer6_attention_weight_max": 55.38671875,
      "activations/layer6_attention_weight_min": -50.78633117675781,
      "activations/layer7_attention_weight_max": 68.45016479492188,
      "activations/layer7_attention_weight_min": -65.84054565429688,
      "activations/layer8_attention_weight_max": 44.77721405029297,
      "activations/layer8_attention_weight_min": -42.551353454589844,
      "activations/layer9_attention_weight_max": 51.218780517578125,
      "activations/layer9_attention_weight_min": -46.16352844238281,
      "epoch": 5.42,
      "learning_rate": 0.00011620984848484847,
      "loss": 2.8359,
      "step": 93300
    },
    {
      "activations/layer0_attention_weight_max": 17.016284942626953,
      "activations/layer0_attention_weight_min": -14.338662147521973,
      "activations/layer10_attention_weight_max": 51.11869812011719,
      "activations/layer10_attention_weight_min": -45.51817321777344,
      "activations/layer11_attention_weight_max": 38.18281555175781,
      "activations/layer11_attention_weight_min": -33.69181442260742,
      "activations/layer12_attention_weight_max": 23.059406280517578,
      "activations/layer12_attention_weight_min": -32.74403762817383,
      "activations/layer13_attention_weight_max": 55.42711639404297,
      "activations/layer13_attention_weight_min": -39.45539855957031,
      "activations/layer14_attention_weight_max": 70.78023529052734,
      "activations/layer14_attention_weight_min": -40.604305267333984,
      "activations/layer15_attention_weight_max": 54.063392639160156,
      "activations/layer15_attention_weight_min": -31.057022094726562,
      "activations/layer16_attention_weight_max": 34.04869842529297,
      "activations/layer16_attention_weight_min": -24.90906524658203,
      "activations/layer17_attention_weight_max": 51.55443572998047,
      "activations/layer17_attention_weight_min": -38.60171127319336,
      "activations/layer18_attention_weight_max": 45.01512145996094,
      "activations/layer18_attention_weight_min": -35.178680419921875,
      "activations/layer19_attention_weight_max": 25.54789161682129,
      "activations/layer19_attention_weight_min": -19.115392684936523,
      "activations/layer1_attention_weight_max": 17.510618209838867,
      "activations/layer1_attention_weight_min": -13.87985610961914,
      "activations/layer20_attention_weight_max": 27.41497802734375,
      "activations/layer20_attention_weight_min": -20.112211227416992,
      "activations/layer21_attention_weight_max": 44.12914276123047,
      "activations/layer21_attention_weight_min": -24.53844451904297,
      "activations/layer22_attention_weight_max": 37.49057388305664,
      "activations/layer22_attention_weight_min": -23.182872772216797,
      "activations/layer23_attention_weight_max": 31.443904876708984,
      "activations/layer23_attention_weight_min": -19.741016387939453,
      "activations/layer2_attention_weight_max": 30.174072265625,
      "activations/layer2_attention_weight_min": -30.539907455444336,
      "activations/layer3_attention_weight_max": 84.10252380371094,
      "activations/layer3_attention_weight_min": -84.09100341796875,
      "activations/layer4_attention_weight_max": 110.76201629638672,
      "activations/layer4_attention_weight_min": -107.3291244506836,
      "activations/layer5_attention_weight_max": 106.67613983154297,
      "activations/layer5_attention_weight_min": -93.83670043945312,
      "activations/layer6_attention_weight_max": 60.075218200683594,
      "activations/layer6_attention_weight_min": -54.05677032470703,
      "activations/layer7_attention_weight_max": 84.65816497802734,
      "activations/layer7_attention_weight_min": -73.06121063232422,
      "activations/layer8_attention_weight_max": 54.421539306640625,
      "activations/layer8_attention_weight_min": -48.149879455566406,
      "activations/layer9_attention_weight_max": 75.69869232177734,
      "activations/layer9_attention_weight_min": -59.780548095703125,
      "epoch": 5.42,
      "learning_rate": 0.00011619090909090909,
      "loss": 2.8428,
      "step": 93350
    },
    {
      "activations/layer0_attention_weight_max": 15.273391723632812,
      "activations/layer0_attention_weight_min": -14.575765609741211,
      "activations/layer10_attention_weight_max": 36.898780822753906,
      "activations/layer10_attention_weight_min": -31.6712703704834,
      "activations/layer11_attention_weight_max": 28.07617950439453,
      "activations/layer11_attention_weight_min": -27.110158920288086,
      "activations/layer12_attention_weight_max": 24.154220581054688,
      "activations/layer12_attention_weight_min": -24.0954647064209,
      "activations/layer13_attention_weight_max": 43.43666458129883,
      "activations/layer13_attention_weight_min": -34.15154266357422,
      "activations/layer14_attention_weight_max": 40.464393615722656,
      "activations/layer14_attention_weight_min": -28.760482788085938,
      "activations/layer15_attention_weight_max": 37.01613235473633,
      "activations/layer15_attention_weight_min": -28.0447998046875,
      "activations/layer16_attention_weight_max": 31.643871307373047,
      "activations/layer16_attention_weight_min": -26.338655471801758,
      "activations/layer17_attention_weight_max": 50.47900390625,
      "activations/layer17_attention_weight_min": -39.319580078125,
      "activations/layer18_attention_weight_max": 45.87125015258789,
      "activations/layer18_attention_weight_min": -33.303672790527344,
      "activations/layer19_attention_weight_max": 23.251787185668945,
      "activations/layer19_attention_weight_min": -19.617856979370117,
      "activations/layer1_attention_weight_max": 17.168310165405273,
      "activations/layer1_attention_weight_min": -13.859972953796387,
      "activations/layer20_attention_weight_max": 20.28949546813965,
      "activations/layer20_attention_weight_min": -23.725622177124023,
      "activations/layer21_attention_weight_max": 38.30055618286133,
      "activations/layer21_attention_weight_min": -24.046281814575195,
      "activations/layer22_attention_weight_max": 34.339149475097656,
      "activations/layer22_attention_weight_min": -23.48352813720703,
      "activations/layer23_attention_weight_max": 28.625396728515625,
      "activations/layer23_attention_weight_min": -19.29287338256836,
      "activations/layer2_attention_weight_max": 27.937084197998047,
      "activations/layer2_attention_weight_min": -26.916141510009766,
      "activations/layer3_attention_weight_max": 79.0143814086914,
      "activations/layer3_attention_weight_min": -80.1682357788086,
      "activations/layer4_attention_weight_max": 95.61051177978516,
      "activations/layer4_attention_weight_min": -91.2491226196289,
      "activations/layer5_attention_weight_max": 95.68116760253906,
      "activations/layer5_attention_weight_min": -84.23651885986328,
      "activations/layer6_attention_weight_max": 56.98948287963867,
      "activations/layer6_attention_weight_min": -47.53274154663086,
      "activations/layer7_attention_weight_max": 71.64923095703125,
      "activations/layer7_attention_weight_min": -62.58587646484375,
      "activations/layer8_attention_weight_max": 42.01571273803711,
      "activations/layer8_attention_weight_min": -39.35270309448242,
      "activations/layer9_attention_weight_max": 49.28183364868164,
      "activations/layer9_attention_weight_min": -44.16046905517578,
      "epoch": 5.43,
      "learning_rate": 0.00011617196969696968,
      "loss": 2.8591,
      "step": 93400
    },
    {
      "activations/layer0_attention_weight_max": 14.542227745056152,
      "activations/layer0_attention_weight_min": -14.725351333618164,
      "activations/layer10_attention_weight_max": 43.149322509765625,
      "activations/layer10_attention_weight_min": -37.30475616455078,
      "activations/layer11_attention_weight_max": 32.3199348449707,
      "activations/layer11_attention_weight_min": -26.311767578125,
      "activations/layer12_attention_weight_max": 35.039215087890625,
      "activations/layer12_attention_weight_min": -27.52362823486328,
      "activations/layer13_attention_weight_max": 53.12338638305664,
      "activations/layer13_attention_weight_min": -39.00844192504883,
      "activations/layer14_attention_weight_max": 58.19369888305664,
      "activations/layer14_attention_weight_min": -42.18954086303711,
      "activations/layer15_attention_weight_max": 41.38259506225586,
      "activations/layer15_attention_weight_min": -31.76263427734375,
      "activations/layer16_attention_weight_max": 31.126846313476562,
      "activations/layer16_attention_weight_min": -26.562368392944336,
      "activations/layer17_attention_weight_max": 54.798133850097656,
      "activations/layer17_attention_weight_min": -40.38120651245117,
      "activations/layer18_attention_weight_max": 48.21290588378906,
      "activations/layer18_attention_weight_min": -34.973270416259766,
      "activations/layer19_attention_weight_max": 24.13329315185547,
      "activations/layer19_attention_weight_min": -21.256757736206055,
      "activations/layer1_attention_weight_max": 17.120834350585938,
      "activations/layer1_attention_weight_min": -16.907215118408203,
      "activations/layer20_attention_weight_max": 22.55774688720703,
      "activations/layer20_attention_weight_min": -22.491865158081055,
      "activations/layer21_attention_weight_max": 40.47156524658203,
      "activations/layer21_attention_weight_min": -27.710994720458984,
      "activations/layer22_attention_weight_max": 32.646976470947266,
      "activations/layer22_attention_weight_min": -25.51565170288086,
      "activations/layer23_attention_weight_max": 29.361652374267578,
      "activations/layer23_attention_weight_min": -20.123394012451172,
      "activations/layer2_attention_weight_max": 30.50577163696289,
      "activations/layer2_attention_weight_min": -28.980201721191406,
      "activations/layer3_attention_weight_max": 82.80447387695312,
      "activations/layer3_attention_weight_min": -86.0474624633789,
      "activations/layer4_attention_weight_max": 99.60791015625,
      "activations/layer4_attention_weight_min": -96.558837890625,
      "activations/layer5_attention_weight_max": 103.68975067138672,
      "activations/layer5_attention_weight_min": -93.33739471435547,
      "activations/layer6_attention_weight_max": 57.21246337890625,
      "activations/layer6_attention_weight_min": -54.51517868041992,
      "activations/layer7_attention_weight_max": 68.69219970703125,
      "activations/layer7_attention_weight_min": -65.74932861328125,
      "activations/layer8_attention_weight_max": 44.428646087646484,
      "activations/layer8_attention_weight_min": -43.09063720703125,
      "activations/layer9_attention_weight_max": 54.958343505859375,
      "activations/layer9_attention_weight_min": -49.68331527709961,
      "epoch": 5.43,
      "learning_rate": 0.00011615303030303029,
      "loss": 2.8291,
      "step": 93450
    },
    {
      "activations/layer0_attention_weight_max": 16.57414436340332,
      "activations/layer0_attention_weight_min": -14.289521217346191,
      "activations/layer10_attention_weight_max": 38.65758514404297,
      "activations/layer10_attention_weight_min": -33.55608367919922,
      "activations/layer11_attention_weight_max": 28.78704833984375,
      "activations/layer11_attention_weight_min": -27.300077438354492,
      "activations/layer12_attention_weight_max": 21.875776290893555,
      "activations/layer12_attention_weight_min": -25.925800323486328,
      "activations/layer13_attention_weight_max": 68.14363861083984,
      "activations/layer13_attention_weight_min": -42.01852035522461,
      "activations/layer14_attention_weight_max": 61.81135559082031,
      "activations/layer14_attention_weight_min": -32.519195556640625,
      "activations/layer15_attention_weight_max": 47.248538970947266,
      "activations/layer15_attention_weight_min": -30.65077781677246,
      "activations/layer16_attention_weight_max": 45.98252868652344,
      "activations/layer16_attention_weight_min": -26.101242065429688,
      "activations/layer17_attention_weight_max": 69.90123748779297,
      "activations/layer17_attention_weight_min": -39.72521209716797,
      "activations/layer18_attention_weight_max": 56.65034484863281,
      "activations/layer18_attention_weight_min": -36.28426742553711,
      "activations/layer19_attention_weight_max": 28.112802505493164,
      "activations/layer19_attention_weight_min": -19.557586669921875,
      "activations/layer1_attention_weight_max": 17.417808532714844,
      "activations/layer1_attention_weight_min": -14.29434585571289,
      "activations/layer20_attention_weight_max": 29.237621307373047,
      "activations/layer20_attention_weight_min": -21.42479705810547,
      "activations/layer21_attention_weight_max": 58.916534423828125,
      "activations/layer21_attention_weight_min": -25.924169540405273,
      "activations/layer22_attention_weight_max": 42.90597915649414,
      "activations/layer22_attention_weight_min": -25.990957260131836,
      "activations/layer23_attention_weight_max": 40.110374450683594,
      "activations/layer23_attention_weight_min": -20.249996185302734,
      "activations/layer2_attention_weight_max": 31.268447875976562,
      "activations/layer2_attention_weight_min": -29.302696228027344,
      "activations/layer3_attention_weight_max": 82.33065795898438,
      "activations/layer3_attention_weight_min": -87.56188201904297,
      "activations/layer4_attention_weight_max": 96.61151123046875,
      "activations/layer4_attention_weight_min": -94.76532745361328,
      "activations/layer5_attention_weight_max": 96.89720153808594,
      "activations/layer5_attention_weight_min": -87.17977142333984,
      "activations/layer6_attention_weight_max": 55.9163703918457,
      "activations/layer6_attention_weight_min": -49.945884704589844,
      "activations/layer7_attention_weight_max": 69.60760498046875,
      "activations/layer7_attention_weight_min": -64.27706909179688,
      "activations/layer8_attention_weight_max": 40.67535400390625,
      "activations/layer8_attention_weight_min": -39.58658218383789,
      "activations/layer9_attention_weight_max": 47.03506088256836,
      "activations/layer9_attention_weight_min": -48.14424133300781,
      "epoch": 5.43,
      "learning_rate": 0.00011613409090909091,
      "loss": 2.8496,
      "step": 93500
    },
    {
      "activations/layer0_attention_weight_max": 14.693007469177246,
      "activations/layer0_attention_weight_min": -14.630609512329102,
      "activations/layer10_attention_weight_max": 37.64461898803711,
      "activations/layer10_attention_weight_min": -36.27780532836914,
      "activations/layer11_attention_weight_max": 29.227325439453125,
      "activations/layer11_attention_weight_min": -27.549535751342773,
      "activations/layer12_attention_weight_max": 30.333396911621094,
      "activations/layer12_attention_weight_min": -25.502344131469727,
      "activations/layer13_attention_weight_max": 56.232574462890625,
      "activations/layer13_attention_weight_min": -42.88861846923828,
      "activations/layer14_attention_weight_max": 48.056880950927734,
      "activations/layer14_attention_weight_min": -35.66999435424805,
      "activations/layer15_attention_weight_max": 39.95274353027344,
      "activations/layer15_attention_weight_min": -29.91291618347168,
      "activations/layer16_attention_weight_max": 38.68064880371094,
      "activations/layer16_attention_weight_min": -28.650148391723633,
      "activations/layer17_attention_weight_max": 58.75862121582031,
      "activations/layer17_attention_weight_min": -40.57301330566406,
      "activations/layer18_attention_weight_max": 49.67502212524414,
      "activations/layer18_attention_weight_min": -32.669029235839844,
      "activations/layer19_attention_weight_max": 23.459829330444336,
      "activations/layer19_attention_weight_min": -19.36328125,
      "activations/layer1_attention_weight_max": 17.22954750061035,
      "activations/layer1_attention_weight_min": -14.329398155212402,
      "activations/layer20_attention_weight_max": 24.525585174560547,
      "activations/layer20_attention_weight_min": -20.32049560546875,
      "activations/layer21_attention_weight_max": 49.969871520996094,
      "activations/layer21_attention_weight_min": -25.716201782226562,
      "activations/layer22_attention_weight_max": 38.44573974609375,
      "activations/layer22_attention_weight_min": -23.81706428527832,
      "activations/layer23_attention_weight_max": 32.116477966308594,
      "activations/layer23_attention_weight_min": -19.066184997558594,
      "activations/layer2_attention_weight_max": 30.573514938354492,
      "activations/layer2_attention_weight_min": -29.545318603515625,
      "activations/layer3_attention_weight_max": 79.44770812988281,
      "activations/layer3_attention_weight_min": -78.04496765136719,
      "activations/layer4_attention_weight_max": 97.72938537597656,
      "activations/layer4_attention_weight_min": -93.23779296875,
      "activations/layer5_attention_weight_max": 99.94328308105469,
      "activations/layer5_attention_weight_min": -88.0655517578125,
      "activations/layer6_attention_weight_max": 58.93379592895508,
      "activations/layer6_attention_weight_min": -53.5595817565918,
      "activations/layer7_attention_weight_max": 70.9713134765625,
      "activations/layer7_attention_weight_min": -67.0609359741211,
      "activations/layer8_attention_weight_max": 43.966209411621094,
      "activations/layer8_attention_weight_min": -42.1563720703125,
      "activations/layer9_attention_weight_max": 57.55791091918945,
      "activations/layer9_attention_weight_min": -49.26027297973633,
      "epoch": 5.44,
      "learning_rate": 0.0001161151515151515,
      "loss": 2.8518,
      "step": 93550
    },
    {
      "activations/layer0_attention_weight_max": 14.69835376739502,
      "activations/layer0_attention_weight_min": -14.470743179321289,
      "activations/layer10_attention_weight_max": 44.205204010009766,
      "activations/layer10_attention_weight_min": -38.1842041015625,
      "activations/layer11_attention_weight_max": 36.206748962402344,
      "activations/layer11_attention_weight_min": -27.45797348022461,
      "activations/layer12_attention_weight_max": 22.835336685180664,
      "activations/layer12_attention_weight_min": -30.0609073638916,
      "activations/layer13_attention_weight_max": 59.80242156982422,
      "activations/layer13_attention_weight_min": -37.72481918334961,
      "activations/layer14_attention_weight_max": 53.833526611328125,
      "activations/layer14_attention_weight_min": -32.845436096191406,
      "activations/layer15_attention_weight_max": 38.83804702758789,
      "activations/layer15_attention_weight_min": -32.55262756347656,
      "activations/layer16_attention_weight_max": 33.24898147583008,
      "activations/layer16_attention_weight_min": -26.091955184936523,
      "activations/layer17_attention_weight_max": 54.93159866333008,
      "activations/layer17_attention_weight_min": -38.17558670043945,
      "activations/layer18_attention_weight_max": 46.73900604248047,
      "activations/layer18_attention_weight_min": -33.89362716674805,
      "activations/layer19_attention_weight_max": 23.644041061401367,
      "activations/layer19_attention_weight_min": -20.313390731811523,
      "activations/layer1_attention_weight_max": 16.924448013305664,
      "activations/layer1_attention_weight_min": -14.643847465515137,
      "activations/layer20_attention_weight_max": 20.52897834777832,
      "activations/layer20_attention_weight_min": -20.394784927368164,
      "activations/layer21_attention_weight_max": 38.75741958618164,
      "activations/layer21_attention_weight_min": -27.390539169311523,
      "activations/layer22_attention_weight_max": 32.76803207397461,
      "activations/layer22_attention_weight_min": -23.37451171875,
      "activations/layer23_attention_weight_max": 27.150358200073242,
      "activations/layer23_attention_weight_min": -19.81744384765625,
      "activations/layer2_attention_weight_max": 32.75815200805664,
      "activations/layer2_attention_weight_min": -32.06730651855469,
      "activations/layer3_attention_weight_max": 91.54283142089844,
      "activations/layer3_attention_weight_min": -89.29888916015625,
      "activations/layer4_attention_weight_max": 107.84889221191406,
      "activations/layer4_attention_weight_min": -102.99336242675781,
      "activations/layer5_attention_weight_max": 105.83168029785156,
      "activations/layer5_attention_weight_min": -94.53462982177734,
      "activations/layer6_attention_weight_max": 58.20323944091797,
      "activations/layer6_attention_weight_min": -54.04145812988281,
      "activations/layer7_attention_weight_max": 81.19644165039062,
      "activations/layer7_attention_weight_min": -71.53120422363281,
      "activations/layer8_attention_weight_max": 51.09139633178711,
      "activations/layer8_attention_weight_min": -42.22367477416992,
      "activations/layer9_attention_weight_max": 66.10026550292969,
      "activations/layer9_attention_weight_min": -50.35559844970703,
      "epoch": 5.44,
      "learning_rate": 0.00011609621212121211,
      "loss": 2.8366,
      "step": 93600
    },
    {
      "activations/layer0_attention_weight_max": 15.46169376373291,
      "activations/layer0_attention_weight_min": -14.791535377502441,
      "activations/layer10_attention_weight_max": 38.4908447265625,
      "activations/layer10_attention_weight_min": -35.313446044921875,
      "activations/layer11_attention_weight_max": 30.10451316833496,
      "activations/layer11_attention_weight_min": -28.32515525817871,
      "activations/layer12_attention_weight_max": 22.943408966064453,
      "activations/layer12_attention_weight_min": -27.381715774536133,
      "activations/layer13_attention_weight_max": 46.62714767456055,
      "activations/layer13_attention_weight_min": -33.733001708984375,
      "activations/layer14_attention_weight_max": 57.535301208496094,
      "activations/layer14_attention_weight_min": -34.27454376220703,
      "activations/layer15_attention_weight_max": 37.85738754272461,
      "activations/layer15_attention_weight_min": -31.342226028442383,
      "activations/layer16_attention_weight_max": 32.02333068847656,
      "activations/layer16_attention_weight_min": -27.43338394165039,
      "activations/layer17_attention_weight_max": 53.55596160888672,
      "activations/layer17_attention_weight_min": -39.72146224975586,
      "activations/layer18_attention_weight_max": 53.22256088256836,
      "activations/layer18_attention_weight_min": -32.6026611328125,
      "activations/layer19_attention_weight_max": 22.69635581970215,
      "activations/layer19_attention_weight_min": -18.330610275268555,
      "activations/layer1_attention_weight_max": 17.853891372680664,
      "activations/layer1_attention_weight_min": -13.88304615020752,
      "activations/layer20_attention_weight_max": 21.25140380859375,
      "activations/layer20_attention_weight_min": -22.920108795166016,
      "activations/layer21_attention_weight_max": 41.130889892578125,
      "activations/layer21_attention_weight_min": -24.31455421447754,
      "activations/layer22_attention_weight_max": 33.405147552490234,
      "activations/layer22_attention_weight_min": -23.406044006347656,
      "activations/layer23_attention_weight_max": 29.776782989501953,
      "activations/layer23_attention_weight_min": -17.38565444946289,
      "activations/layer2_attention_weight_max": 28.732105255126953,
      "activations/layer2_attention_weight_min": -28.160995483398438,
      "activations/layer3_attention_weight_max": 81.64620208740234,
      "activations/layer3_attention_weight_min": -81.78717041015625,
      "activations/layer4_attention_weight_max": 100.6788101196289,
      "activations/layer4_attention_weight_min": -91.69012451171875,
      "activations/layer5_attention_weight_max": 102.95208740234375,
      "activations/layer5_attention_weight_min": -88.50738525390625,
      "activations/layer6_attention_weight_max": 59.539146423339844,
      "activations/layer6_attention_weight_min": -52.37147903442383,
      "activations/layer7_attention_weight_max": 71.50987243652344,
      "activations/layer7_attention_weight_min": -68.7452163696289,
      "activations/layer8_attention_weight_max": 41.94716262817383,
      "activations/layer8_attention_weight_min": -40.5775146484375,
      "activations/layer9_attention_weight_max": 53.743709564208984,
      "activations/layer9_attention_weight_min": -48.79432678222656,
      "epoch": 5.44,
      "learning_rate": 0.00011607727272727272,
      "loss": 2.8371,
      "step": 93650
    },
    {
      "activations/layer0_attention_weight_max": 14.073186874389648,
      "activations/layer0_attention_weight_min": -14.032678604125977,
      "activations/layer10_attention_weight_max": 38.80964660644531,
      "activations/layer10_attention_weight_min": -34.2670783996582,
      "activations/layer11_attention_weight_max": 31.491016387939453,
      "activations/layer11_attention_weight_min": -27.35671043395996,
      "activations/layer12_attention_weight_max": 23.639860153198242,
      "activations/layer12_attention_weight_min": -25.915563583374023,
      "activations/layer13_attention_weight_max": 49.97301483154297,
      "activations/layer13_attention_weight_min": -38.034934997558594,
      "activations/layer14_attention_weight_max": 52.85524368286133,
      "activations/layer14_attention_weight_min": -37.679893493652344,
      "activations/layer15_attention_weight_max": 41.09709548950195,
      "activations/layer15_attention_weight_min": -28.90816307067871,
      "activations/layer16_attention_weight_max": 33.333072662353516,
      "activations/layer16_attention_weight_min": -26.19353485107422,
      "activations/layer17_attention_weight_max": 52.46182632446289,
      "activations/layer17_attention_weight_min": -38.897342681884766,
      "activations/layer18_attention_weight_max": 47.08135986328125,
      "activations/layer18_attention_weight_min": -33.06676483154297,
      "activations/layer19_attention_weight_max": 23.12541961669922,
      "activations/layer19_attention_weight_min": -19.707082748413086,
      "activations/layer1_attention_weight_max": 17.826261520385742,
      "activations/layer1_attention_weight_min": -16.311819076538086,
      "activations/layer20_attention_weight_max": 24.607938766479492,
      "activations/layer20_attention_weight_min": -21.631853103637695,
      "activations/layer21_attention_weight_max": 46.03554153442383,
      "activations/layer21_attention_weight_min": -26.291038513183594,
      "activations/layer22_attention_weight_max": 35.27350997924805,
      "activations/layer22_attention_weight_min": -27.09269142150879,
      "activations/layer23_attention_weight_max": 31.591917037963867,
      "activations/layer23_attention_weight_min": -18.814685821533203,
      "activations/layer2_attention_weight_max": 30.28004264831543,
      "activations/layer2_attention_weight_min": -28.619413375854492,
      "activations/layer3_attention_weight_max": 82.25927734375,
      "activations/layer3_attention_weight_min": -82.17536163330078,
      "activations/layer4_attention_weight_max": 102.38233947753906,
      "activations/layer4_attention_weight_min": -96.48589324951172,
      "activations/layer5_attention_weight_max": 100.09976196289062,
      "activations/layer5_attention_weight_min": -91.26203918457031,
      "activations/layer6_attention_weight_max": 59.23732376098633,
      "activations/layer6_attention_weight_min": -54.48896408081055,
      "activations/layer7_attention_weight_max": 74.27853393554688,
      "activations/layer7_attention_weight_min": -69.54972839355469,
      "activations/layer8_attention_weight_max": 44.12825012207031,
      "activations/layer8_attention_weight_min": -43.73329544067383,
      "activations/layer9_attention_weight_max": 55.144161224365234,
      "activations/layer9_attention_weight_min": -44.61698532104492,
      "epoch": 5.44,
      "learning_rate": 0.00011605833333333333,
      "loss": 2.8152,
      "step": 93700
    },
    {
      "activations/layer0_attention_weight_max": 14.682007789611816,
      "activations/layer0_attention_weight_min": -14.506484985351562,
      "activations/layer10_attention_weight_max": 38.30170822143555,
      "activations/layer10_attention_weight_min": -36.42721939086914,
      "activations/layer11_attention_weight_max": 29.447019577026367,
      "activations/layer11_attention_weight_min": -27.957565307617188,
      "activations/layer12_attention_weight_max": 24.20257568359375,
      "activations/layer12_attention_weight_min": -31.925397872924805,
      "activations/layer13_attention_weight_max": 56.45207595825195,
      "activations/layer13_attention_weight_min": -40.7468376159668,
      "activations/layer14_attention_weight_max": 59.43220138549805,
      "activations/layer14_attention_weight_min": -45.10417556762695,
      "activations/layer15_attention_weight_max": 42.628780364990234,
      "activations/layer15_attention_weight_min": -31.18619155883789,
      "activations/layer16_attention_weight_max": 32.068180084228516,
      "activations/layer16_attention_weight_min": -27.76532745361328,
      "activations/layer17_attention_weight_max": 53.025306701660156,
      "activations/layer17_attention_weight_min": -38.7138786315918,
      "activations/layer18_attention_weight_max": 45.41565704345703,
      "activations/layer18_attention_weight_min": -34.39752197265625,
      "activations/layer19_attention_weight_max": 19.86858558654785,
      "activations/layer19_attention_weight_min": -18.217037200927734,
      "activations/layer1_attention_weight_max": 17.049100875854492,
      "activations/layer1_attention_weight_min": -15.186080932617188,
      "activations/layer20_attention_weight_max": 20.234493255615234,
      "activations/layer20_attention_weight_min": -21.814830780029297,
      "activations/layer21_attention_weight_max": 34.220130920410156,
      "activations/layer21_attention_weight_min": -23.5572509765625,
      "activations/layer22_attention_weight_max": 32.712093353271484,
      "activations/layer22_attention_weight_min": -27.74526596069336,
      "activations/layer23_attention_weight_max": 27.182371139526367,
      "activations/layer23_attention_weight_min": -18.525312423706055,
      "activations/layer2_attention_weight_max": 29.447919845581055,
      "activations/layer2_attention_weight_min": -28.491437911987305,
      "activations/layer3_attention_weight_max": 78.57994842529297,
      "activations/layer3_attention_weight_min": -80.73158264160156,
      "activations/layer4_attention_weight_max": 96.7602767944336,
      "activations/layer4_attention_weight_min": -94.1165542602539,
      "activations/layer5_attention_weight_max": 99.67182922363281,
      "activations/layer5_attention_weight_min": -87.82502746582031,
      "activations/layer6_attention_weight_max": 58.85697937011719,
      "activations/layer6_attention_weight_min": -50.96684646606445,
      "activations/layer7_attention_weight_max": 70.15132141113281,
      "activations/layer7_attention_weight_min": -71.58000183105469,
      "activations/layer8_attention_weight_max": 44.855289459228516,
      "activations/layer8_attention_weight_min": -42.0968017578125,
      "activations/layer9_attention_weight_max": 63.878173828125,
      "activations/layer9_attention_weight_min": -48.75181579589844,
      "epoch": 5.45,
      "learning_rate": 0.00011603939393939393,
      "loss": 2.8373,
      "step": 93750
    },
    {
      "activations/layer0_attention_weight_max": 14.080317497253418,
      "activations/layer0_attention_weight_min": -14.48021125793457,
      "activations/layer10_attention_weight_max": 36.057769775390625,
      "activations/layer10_attention_weight_min": -34.86066436767578,
      "activations/layer11_attention_weight_max": 28.694515228271484,
      "activations/layer11_attention_weight_min": -26.746000289916992,
      "activations/layer12_attention_weight_max": 25.503189086914062,
      "activations/layer12_attention_weight_min": -24.278762817382812,
      "activations/layer13_attention_weight_max": 48.14926528930664,
      "activations/layer13_attention_weight_min": -36.8863525390625,
      "activations/layer14_attention_weight_max": 43.67979049682617,
      "activations/layer14_attention_weight_min": -30.860898971557617,
      "activations/layer15_attention_weight_max": 36.82111358642578,
      "activations/layer15_attention_weight_min": -29.527605056762695,
      "activations/layer16_attention_weight_max": 38.10617446899414,
      "activations/layer16_attention_weight_min": -26.015554428100586,
      "activations/layer17_attention_weight_max": 57.50708770751953,
      "activations/layer17_attention_weight_min": -41.4101448059082,
      "activations/layer18_attention_weight_max": 49.01913833618164,
      "activations/layer18_attention_weight_min": -33.51473617553711,
      "activations/layer19_attention_weight_max": 22.34185028076172,
      "activations/layer19_attention_weight_min": -19.84511947631836,
      "activations/layer1_attention_weight_max": 16.814313888549805,
      "activations/layer1_attention_weight_min": -16.32248878479004,
      "activations/layer20_attention_weight_max": 23.657421112060547,
      "activations/layer20_attention_weight_min": -23.79673957824707,
      "activations/layer21_attention_weight_max": 42.97923278808594,
      "activations/layer21_attention_weight_min": -30.305709838867188,
      "activations/layer22_attention_weight_max": 33.788902282714844,
      "activations/layer22_attention_weight_min": -23.02134895324707,
      "activations/layer23_attention_weight_max": 28.887054443359375,
      "activations/layer23_attention_weight_min": -18.308979034423828,
      "activations/layer2_attention_weight_max": 29.772457122802734,
      "activations/layer2_attention_weight_min": -28.328144073486328,
      "activations/layer3_attention_weight_max": 82.76703643798828,
      "activations/layer3_attention_weight_min": -86.39190673828125,
      "activations/layer4_attention_weight_max": 103.07930755615234,
      "activations/layer4_attention_weight_min": -96.7506332397461,
      "activations/layer5_attention_weight_max": 107.06246185302734,
      "activations/layer5_attention_weight_min": -90.62914276123047,
      "activations/layer6_attention_weight_max": 61.160186767578125,
      "activations/layer6_attention_weight_min": -53.82194519042969,
      "activations/layer7_attention_weight_max": 74.72198486328125,
      "activations/layer7_attention_weight_min": -65.19659423828125,
      "activations/layer8_attention_weight_max": 44.254940032958984,
      "activations/layer8_attention_weight_min": -42.016849517822266,
      "activations/layer9_attention_weight_max": 53.50061798095703,
      "activations/layer9_attention_weight_min": -45.04349899291992,
      "epoch": 5.45,
      "learning_rate": 0.00011602045454545453,
      "loss": 2.8372,
      "step": 93800
    },
    {
      "activations/layer0_attention_weight_max": 16.940319061279297,
      "activations/layer0_attention_weight_min": -14.597833633422852,
      "activations/layer10_attention_weight_max": 38.21332550048828,
      "activations/layer10_attention_weight_min": -32.50791931152344,
      "activations/layer11_attention_weight_max": 31.20488739013672,
      "activations/layer11_attention_weight_min": -26.7905330657959,
      "activations/layer12_attention_weight_max": 22.467294692993164,
      "activations/layer12_attention_weight_min": -26.70918083190918,
      "activations/layer13_attention_weight_max": 51.413822174072266,
      "activations/layer13_attention_weight_min": -37.20451354980469,
      "activations/layer14_attention_weight_max": 44.37001419067383,
      "activations/layer14_attention_weight_min": -30.218334197998047,
      "activations/layer15_attention_weight_max": 40.46510696411133,
      "activations/layer15_attention_weight_min": -28.854663848876953,
      "activations/layer16_attention_weight_max": 34.736122131347656,
      "activations/layer16_attention_weight_min": -27.21767807006836,
      "activations/layer17_attention_weight_max": 62.35603332519531,
      "activations/layer17_attention_weight_min": -39.6553840637207,
      "activations/layer18_attention_weight_max": 53.720787048339844,
      "activations/layer18_attention_weight_min": -35.36161422729492,
      "activations/layer19_attention_weight_max": 28.303531646728516,
      "activations/layer19_attention_weight_min": -20.430492401123047,
      "activations/layer1_attention_weight_max": 17.606298446655273,
      "activations/layer1_attention_weight_min": -13.624149322509766,
      "activations/layer20_attention_weight_max": 26.346818923950195,
      "activations/layer20_attention_weight_min": -21.81110191345215,
      "activations/layer21_attention_weight_max": 43.568519592285156,
      "activations/layer21_attention_weight_min": -24.502708435058594,
      "activations/layer22_attention_weight_max": 34.58510208129883,
      "activations/layer22_attention_weight_min": -24.29964828491211,
      "activations/layer23_attention_weight_max": 31.481426239013672,
      "activations/layer23_attention_weight_min": -19.61655044555664,
      "activations/layer2_attention_weight_max": 31.81938934326172,
      "activations/layer2_attention_weight_min": -29.010005950927734,
      "activations/layer3_attention_weight_max": 83.10875701904297,
      "activations/layer3_attention_weight_min": -84.82500457763672,
      "activations/layer4_attention_weight_max": 105.2240219116211,
      "activations/layer4_attention_weight_min": -101.63208770751953,
      "activations/layer5_attention_weight_max": 98.76814270019531,
      "activations/layer5_attention_weight_min": -91.42046356201172,
      "activations/layer6_attention_weight_max": 57.187259674072266,
      "activations/layer6_attention_weight_min": -51.710609436035156,
      "activations/layer7_attention_weight_max": 68.1840591430664,
      "activations/layer7_attention_weight_min": -66.29281616210938,
      "activations/layer8_attention_weight_max": 42.975669860839844,
      "activations/layer8_attention_weight_min": -44.3342170715332,
      "activations/layer9_attention_weight_max": 47.32947540283203,
      "activations/layer9_attention_weight_min": -45.92875289916992,
      "epoch": 5.45,
      "learning_rate": 0.00011600151515151514,
      "loss": 2.8355,
      "step": 93850
    },
    {
      "activations/layer0_attention_weight_max": 14.079510688781738,
      "activations/layer0_attention_weight_min": -13.507445335388184,
      "activations/layer10_attention_weight_max": 40.28428649902344,
      "activations/layer10_attention_weight_min": -36.001155853271484,
      "activations/layer11_attention_weight_max": 30.707019805908203,
      "activations/layer11_attention_weight_min": -27.879554748535156,
      "activations/layer12_attention_weight_max": 23.633655548095703,
      "activations/layer12_attention_weight_min": -25.403186798095703,
      "activations/layer13_attention_weight_max": 41.94246292114258,
      "activations/layer13_attention_weight_min": -30.12184715270996,
      "activations/layer14_attention_weight_max": 42.008270263671875,
      "activations/layer14_attention_weight_min": -31.315387725830078,
      "activations/layer15_attention_weight_max": 38.73366928100586,
      "activations/layer15_attention_weight_min": -31.624391555786133,
      "activations/layer16_attention_weight_max": 36.92764663696289,
      "activations/layer16_attention_weight_min": -27.325803756713867,
      "activations/layer17_attention_weight_max": 59.54444122314453,
      "activations/layer17_attention_weight_min": -44.06787109375,
      "activations/layer18_attention_weight_max": 53.59379196166992,
      "activations/layer18_attention_weight_min": -34.87546157836914,
      "activations/layer19_attention_weight_max": 24.936338424682617,
      "activations/layer19_attention_weight_min": -20.398412704467773,
      "activations/layer1_attention_weight_max": 17.196191787719727,
      "activations/layer1_attention_weight_min": -13.781729698181152,
      "activations/layer20_attention_weight_max": 22.900188446044922,
      "activations/layer20_attention_weight_min": -20.13064193725586,
      "activations/layer21_attention_weight_max": 40.736759185791016,
      "activations/layer21_attention_weight_min": -26.909664154052734,
      "activations/layer22_attention_weight_max": 38.149600982666016,
      "activations/layer22_attention_weight_min": -23.835176467895508,
      "activations/layer23_attention_weight_max": 29.960193634033203,
      "activations/layer23_attention_weight_min": -18.120121002197266,
      "activations/layer2_attention_weight_max": 31.465015411376953,
      "activations/layer2_attention_weight_min": -31.23297119140625,
      "activations/layer3_attention_weight_max": 79.04711151123047,
      "activations/layer3_attention_weight_min": -83.24211883544922,
      "activations/layer4_attention_weight_max": 98.90196228027344,
      "activations/layer4_attention_weight_min": -92.46495819091797,
      "activations/layer5_attention_weight_max": 99.8912353515625,
      "activations/layer5_attention_weight_min": -87.10436248779297,
      "activations/layer6_attention_weight_max": 60.37821578979492,
      "activations/layer6_attention_weight_min": -51.628726959228516,
      "activations/layer7_attention_weight_max": 75.3804702758789,
      "activations/layer7_attention_weight_min": -67.58364868164062,
      "activations/layer8_attention_weight_max": 45.738033294677734,
      "activations/layer8_attention_weight_min": -42.642845153808594,
      "activations/layer9_attention_weight_max": 61.29085922241211,
      "activations/layer9_attention_weight_min": -50.929134368896484,
      "epoch": 5.46,
      "learning_rate": 0.00011598257575757575,
      "loss": 2.8334,
      "step": 93900
    },
    {
      "activations/layer0_attention_weight_max": 16.833486557006836,
      "activations/layer0_attention_weight_min": -13.84770679473877,
      "activations/layer10_attention_weight_max": 40.687896728515625,
      "activations/layer10_attention_weight_min": -35.834991455078125,
      "activations/layer11_attention_weight_max": 32.5636100769043,
      "activations/layer11_attention_weight_min": -27.89657211303711,
      "activations/layer12_attention_weight_max": 21.368558883666992,
      "activations/layer12_attention_weight_min": -23.879337310791016,
      "activations/layer13_attention_weight_max": 54.281333923339844,
      "activations/layer13_attention_weight_min": -37.391902923583984,
      "activations/layer14_attention_weight_max": 49.49686050415039,
      "activations/layer14_attention_weight_min": -29.53322410583496,
      "activations/layer15_attention_weight_max": 38.37816619873047,
      "activations/layer15_attention_weight_min": -29.242454528808594,
      "activations/layer16_attention_weight_max": 36.64513397216797,
      "activations/layer16_attention_weight_min": -26.353254318237305,
      "activations/layer17_attention_weight_max": 57.1566047668457,
      "activations/layer17_attention_weight_min": -38.8010139465332,
      "activations/layer18_attention_weight_max": 47.992618560791016,
      "activations/layer18_attention_weight_min": -34.650390625,
      "activations/layer19_attention_weight_max": 21.978927612304688,
      "activations/layer19_attention_weight_min": -19.238094329833984,
      "activations/layer1_attention_weight_max": 17.2000789642334,
      "activations/layer1_attention_weight_min": -15.812886238098145,
      "activations/layer20_attention_weight_max": 26.01786994934082,
      "activations/layer20_attention_weight_min": -23.87870979309082,
      "activations/layer21_attention_weight_max": 43.29326248168945,
      "activations/layer21_attention_weight_min": -27.903499603271484,
      "activations/layer22_attention_weight_max": 34.39109420776367,
      "activations/layer22_attention_weight_min": -25.664705276489258,
      "activations/layer23_attention_weight_max": 30.49086570739746,
      "activations/layer23_attention_weight_min": -18.716819763183594,
      "activations/layer2_attention_weight_max": 30.476491928100586,
      "activations/layer2_attention_weight_min": -29.06927490234375,
      "activations/layer3_attention_weight_max": 81.66577911376953,
      "activations/layer3_attention_weight_min": -81.53459930419922,
      "activations/layer4_attention_weight_max": 98.75383758544922,
      "activations/layer4_attention_weight_min": -92.2894515991211,
      "activations/layer5_attention_weight_max": 104.05970764160156,
      "activations/layer5_attention_weight_min": -92.96044921875,
      "activations/layer6_attention_weight_max": 59.03117752075195,
      "activations/layer6_attention_weight_min": -52.268489837646484,
      "activations/layer7_attention_weight_max": 74.73213195800781,
      "activations/layer7_attention_weight_min": -70.55245208740234,
      "activations/layer8_attention_weight_max": 44.336490631103516,
      "activations/layer8_attention_weight_min": -43.784915924072266,
      "activations/layer9_attention_weight_max": 56.45085525512695,
      "activations/layer9_attention_weight_min": -51.171722412109375,
      "epoch": 5.46,
      "learning_rate": 0.00011596363636363635,
      "loss": 2.8395,
      "step": 93950
    },
    {
      "activations/layer0_attention_weight_max": 14.652588844299316,
      "activations/layer0_attention_weight_min": -14.22486400604248,
      "activations/layer10_attention_weight_max": 36.77222442626953,
      "activations/layer10_attention_weight_min": -34.22303771972656,
      "activations/layer11_attention_weight_max": 29.07463836669922,
      "activations/layer11_attention_weight_min": -28.21274185180664,
      "activations/layer12_attention_weight_max": 21.95718765258789,
      "activations/layer12_attention_weight_min": -23.384395599365234,
      "activations/layer13_attention_weight_max": 52.9410514831543,
      "activations/layer13_attention_weight_min": -37.66506576538086,
      "activations/layer14_attention_weight_max": 39.46217727661133,
      "activations/layer14_attention_weight_min": -29.043697357177734,
      "activations/layer15_attention_weight_max": 40.10091781616211,
      "activations/layer15_attention_weight_min": -28.833322525024414,
      "activations/layer16_attention_weight_max": 37.08628845214844,
      "activations/layer16_attention_weight_min": -25.27000617980957,
      "activations/layer17_attention_weight_max": 52.27890396118164,
      "activations/layer17_attention_weight_min": -39.85057830810547,
      "activations/layer18_attention_weight_max": 44.567535400390625,
      "activations/layer18_attention_weight_min": -33.094940185546875,
      "activations/layer19_attention_weight_max": 22.652742385864258,
      "activations/layer19_attention_weight_min": -18.258922576904297,
      "activations/layer1_attention_weight_max": 17.171815872192383,
      "activations/layer1_attention_weight_min": -14.195556640625,
      "activations/layer20_attention_weight_max": 20.38058853149414,
      "activations/layer20_attention_weight_min": -25.17827796936035,
      "activations/layer21_attention_weight_max": 34.36631393432617,
      "activations/layer21_attention_weight_min": -28.145816802978516,
      "activations/layer22_attention_weight_max": 38.49522018432617,
      "activations/layer22_attention_weight_min": -24.60943603515625,
      "activations/layer23_attention_weight_max": 27.506694793701172,
      "activations/layer23_attention_weight_min": -18.709209442138672,
      "activations/layer2_attention_weight_max": 27.442581176757812,
      "activations/layer2_attention_weight_min": -27.063400268554688,
      "activations/layer3_attention_weight_max": 81.64797973632812,
      "activations/layer3_attention_weight_min": -83.46810150146484,
      "activations/layer4_attention_weight_max": 103.09000396728516,
      "activations/layer4_attention_weight_min": -93.4658432006836,
      "activations/layer5_attention_weight_max": 103.0267562866211,
      "activations/layer5_attention_weight_min": -88.27169799804688,
      "activations/layer6_attention_weight_max": 56.965824127197266,
      "activations/layer6_attention_weight_min": -51.247623443603516,
      "activations/layer7_attention_weight_max": 71.9127426147461,
      "activations/layer7_attention_weight_min": -65.52345275878906,
      "activations/layer8_attention_weight_max": 41.43295669555664,
      "activations/layer8_attention_weight_min": -39.513145446777344,
      "activations/layer9_attention_weight_max": 53.653297424316406,
      "activations/layer9_attention_weight_min": -43.85918426513672,
      "epoch": 5.46,
      "learning_rate": 0.00011594469696969696,
      "loss": 2.8564,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_loss": 2.794921875,
      "eval_runtime": 8.5439,
      "eval_samples_per_second": 502.582,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_openwebtext_loss": 2.794921875,
      "eval_openwebtext_ppl": 16.36135047271935,
      "eval_openwebtext_runtime": 8.5439,
      "eval_openwebtext_samples_per_second": 502.582,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_wikitext_loss": 3.041015625,
      "eval_wikitext_ppl": 20.92648590822237,
      "eval_wikitext_runtime": 2.0315,
      "eval_wikitext_samples_per_second": 224.462,
      "step": 94000
    },
    {
      "epoch": 5.46,
      "eval_lambada_loss": 2.861328125,
      "eval_lambada_ppl": 17.484733434196265,
      "eval_lambada_runtime": 9.5895,
      "eval_lambada_samples_per_second": 507.743,
      "step": 94000
    },
    {
      "activations/layer0_attention_weight_max": 14.335472106933594,
      "activations/layer0_attention_weight_min": -14.7528657913208,
      "activations/layer10_attention_weight_max": 38.88648223876953,
      "activations/layer10_attention_weight_min": -33.906375885009766,
      "activations/layer11_attention_weight_max": 30.475439071655273,
      "activations/layer11_attention_weight_min": -28.732135772705078,
      "activations/layer12_attention_weight_max": 30.248981475830078,
      "activations/layer12_attention_weight_min": -26.268455505371094,
      "activations/layer13_attention_weight_max": 45.78643035888672,
      "activations/layer13_attention_weight_min": -36.39917755126953,
      "activations/layer14_attention_weight_max": 43.7259407043457,
      "activations/layer14_attention_weight_min": -30.30682373046875,
      "activations/layer15_attention_weight_max": 39.83151626586914,
      "activations/layer15_attention_weight_min": -31.26677894592285,
      "activations/layer16_attention_weight_max": 35.790767669677734,
      "activations/layer16_attention_weight_min": -26.472578048706055,
      "activations/layer17_attention_weight_max": 57.6264533996582,
      "activations/layer17_attention_weight_min": -40.941673278808594,
      "activations/layer18_attention_weight_max": 51.02656173706055,
      "activations/layer18_attention_weight_min": -32.001319885253906,
      "activations/layer19_attention_weight_max": 22.194746017456055,
      "activations/layer19_attention_weight_min": -19.848411560058594,
      "activations/layer1_attention_weight_max": 17.31243896484375,
      "activations/layer1_attention_weight_min": -16.146760940551758,
      "activations/layer20_attention_weight_max": 25.945911407470703,
      "activations/layer20_attention_weight_min": -24.08610725402832,
      "activations/layer21_attention_weight_max": 40.34162521362305,
      "activations/layer21_attention_weight_min": -24.627843856811523,
      "activations/layer22_attention_weight_max": 35.66499710083008,
      "activations/layer22_attention_weight_min": -22.7836856842041,
      "activations/layer23_attention_weight_max": 31.648672103881836,
      "activations/layer23_attention_weight_min": -17.801240921020508,
      "activations/layer2_attention_weight_max": 29.190837860107422,
      "activations/layer2_attention_weight_min": -27.91640281677246,
      "activations/layer3_attention_weight_max": 82.17618560791016,
      "activations/layer3_attention_weight_min": -84.82279968261719,
      "activations/layer4_attention_weight_max": 99.53593444824219,
      "activations/layer4_attention_weight_min": -97.91178131103516,
      "activations/layer5_attention_weight_max": 98.16302490234375,
      "activations/layer5_attention_weight_min": -86.90385437011719,
      "activations/layer6_attention_weight_max": 57.514122009277344,
      "activations/layer6_attention_weight_min": -52.434898376464844,
      "activations/layer7_attention_weight_max": 66.39759063720703,
      "activations/layer7_attention_weight_min": -66.6639404296875,
      "activations/layer8_attention_weight_max": 42.47977828979492,
      "activations/layer8_attention_weight_min": -43.04622268676758,
      "activations/layer9_attention_weight_max": 46.93095397949219,
      "activations/layer9_attention_weight_min": -47.35071563720703,
      "epoch": 5.46,
      "learning_rate": 0.00011592575757575758,
      "loss": 2.8483,
      "step": 94050
    },
    {
      "activations/layer0_attention_weight_max": 14.655329704284668,
      "activations/layer0_attention_weight_min": -14.575785636901855,
      "activations/layer10_attention_weight_max": 38.11282730102539,
      "activations/layer10_attention_weight_min": -34.47154998779297,
      "activations/layer11_attention_weight_max": 30.836698532104492,
      "activations/layer11_attention_weight_min": -26.799062728881836,
      "activations/layer12_attention_weight_max": 25.173389434814453,
      "activations/layer12_attention_weight_min": -25.010517120361328,
      "activations/layer13_attention_weight_max": 62.960147857666016,
      "activations/layer13_attention_weight_min": -40.199432373046875,
      "activations/layer14_attention_weight_max": 83.13260650634766,
      "activations/layer14_attention_weight_min": -40.01026153564453,
      "activations/layer15_attention_weight_max": 45.64019775390625,
      "activations/layer15_attention_weight_min": -31.775413513183594,
      "activations/layer16_attention_weight_max": 38.667720794677734,
      "activations/layer16_attention_weight_min": -27.610727310180664,
      "activations/layer17_attention_weight_max": 64.85093688964844,
      "activations/layer17_attention_weight_min": -41.75678634643555,
      "activations/layer18_attention_weight_max": 58.70496368408203,
      "activations/layer18_attention_weight_min": -36.00732421875,
      "activations/layer19_attention_weight_max": 24.290298461914062,
      "activations/layer19_attention_weight_min": -21.65586280822754,
      "activations/layer1_attention_weight_max": 17.300683975219727,
      "activations/layer1_attention_weight_min": -17.436819076538086,
      "activations/layer20_attention_weight_max": 28.836301803588867,
      "activations/layer20_attention_weight_min": -19.8828182220459,
      "activations/layer21_attention_weight_max": 51.07035827636719,
      "activations/layer21_attention_weight_min": -25.030948638916016,
      "activations/layer22_attention_weight_max": 43.93632125854492,
      "activations/layer22_attention_weight_min": -26.18867301940918,
      "activations/layer23_attention_weight_max": 39.80469512939453,
      "activations/layer23_attention_weight_min": -22.53827476501465,
      "activations/layer2_attention_weight_max": 30.094844818115234,
      "activations/layer2_attention_weight_min": -27.85338592529297,
      "activations/layer3_attention_weight_max": 80.21770477294922,
      "activations/layer3_attention_weight_min": -83.14918518066406,
      "activations/layer4_attention_weight_max": 100.47289276123047,
      "activations/layer4_attention_weight_min": -93.05606842041016,
      "activations/layer5_attention_weight_max": 101.54341888427734,
      "activations/layer5_attention_weight_min": -85.9201889038086,
      "activations/layer6_attention_weight_max": 57.542327880859375,
      "activations/layer6_attention_weight_min": -51.96976089477539,
      "activations/layer7_attention_weight_max": 76.21627807617188,
      "activations/layer7_attention_weight_min": -64.96473693847656,
      "activations/layer8_attention_weight_max": 42.24160385131836,
      "activations/layer8_attention_weight_min": -40.84623336791992,
      "activations/layer9_attention_weight_max": 50.9263801574707,
      "activations/layer9_attention_weight_min": -46.31427001953125,
      "epoch": 5.47,
      "learning_rate": 0.00011590681818181817,
      "loss": 2.8385,
      "step": 94100
    },
    {
      "activations/layer0_attention_weight_max": 14.858855247497559,
      "activations/layer0_attention_weight_min": -14.504140853881836,
      "activations/layer10_attention_weight_max": 38.17888259887695,
      "activations/layer10_attention_weight_min": -34.367149353027344,
      "activations/layer11_attention_weight_max": 36.707122802734375,
      "activations/layer11_attention_weight_min": -27.200712203979492,
      "activations/layer12_attention_weight_max": 21.070585250854492,
      "activations/layer12_attention_weight_min": -28.2504940032959,
      "activations/layer13_attention_weight_max": 50.57394027709961,
      "activations/layer13_attention_weight_min": -34.28856658935547,
      "activations/layer14_attention_weight_max": 49.10759353637695,
      "activations/layer14_attention_weight_min": -31.909900665283203,
      "activations/layer15_attention_weight_max": 41.12030792236328,
      "activations/layer15_attention_weight_min": -31.396902084350586,
      "activations/layer16_attention_weight_max": 33.09048843383789,
      "activations/layer16_attention_weight_min": -25.789592742919922,
      "activations/layer17_attention_weight_max": 57.01919937133789,
      "activations/layer17_attention_weight_min": -41.66252899169922,
      "activations/layer18_attention_weight_max": 46.97725296020508,
      "activations/layer18_attention_weight_min": -35.25359344482422,
      "activations/layer19_attention_weight_max": 22.884611129760742,
      "activations/layer19_attention_weight_min": -21.92778968811035,
      "activations/layer1_attention_weight_max": 17.74857521057129,
      "activations/layer1_attention_weight_min": -19.465412139892578,
      "activations/layer20_attention_weight_max": 24.826519012451172,
      "activations/layer20_attention_weight_min": -21.220672607421875,
      "activations/layer21_attention_weight_max": 57.179893493652344,
      "activations/layer21_attention_weight_min": -26.41165542602539,
      "activations/layer22_attention_weight_max": 32.98573303222656,
      "activations/layer22_attention_weight_min": -24.57303237915039,
      "activations/layer23_attention_weight_max": 28.597858428955078,
      "activations/layer23_attention_weight_min": -20.00152587890625,
      "activations/layer2_attention_weight_max": 31.605703353881836,
      "activations/layer2_attention_weight_min": -30.08005142211914,
      "activations/layer3_attention_weight_max": 82.1622314453125,
      "activations/layer3_attention_weight_min": -83.367431640625,
      "activations/layer4_attention_weight_max": 99.08216857910156,
      "activations/layer4_attention_weight_min": -96.6139907836914,
      "activations/layer5_attention_weight_max": 99.70916748046875,
      "activations/layer5_attention_weight_min": -91.27172088623047,
      "activations/layer6_attention_weight_max": 55.392520904541016,
      "activations/layer6_attention_weight_min": -52.22142028808594,
      "activations/layer7_attention_weight_max": 79.91809844970703,
      "activations/layer7_attention_weight_min": -72.5975112915039,
      "activations/layer8_attention_weight_max": 47.987911224365234,
      "activations/layer8_attention_weight_min": -41.683860778808594,
      "activations/layer9_attention_weight_max": 68.65381622314453,
      "activations/layer9_attention_weight_min": -55.85858154296875,
      "epoch": 5.47,
      "learning_rate": 0.00011588787878787878,
      "loss": 2.8186,
      "step": 94150
    },
    {
      "activations/layer0_attention_weight_max": 14.54316234588623,
      "activations/layer0_attention_weight_min": -14.36889934539795,
      "activations/layer10_attention_weight_max": 40.754066467285156,
      "activations/layer10_attention_weight_min": -36.30241012573242,
      "activations/layer11_attention_weight_max": 33.95033264160156,
      "activations/layer11_attention_weight_min": -29.987396240234375,
      "activations/layer12_attention_weight_max": 21.520349502563477,
      "activations/layer12_attention_weight_min": -26.588459014892578,
      "activations/layer13_attention_weight_max": 55.732276916503906,
      "activations/layer13_attention_weight_min": -39.90980529785156,
      "activations/layer14_attention_weight_max": 53.538238525390625,
      "activations/layer14_attention_weight_min": -37.45215606689453,
      "activations/layer15_attention_weight_max": 42.78971481323242,
      "activations/layer15_attention_weight_min": -29.31003189086914,
      "activations/layer16_attention_weight_max": 34.64690399169922,
      "activations/layer16_attention_weight_min": -28.169902801513672,
      "activations/layer17_attention_weight_max": 63.78831100463867,
      "activations/layer17_attention_weight_min": -40.18035125732422,
      "activations/layer18_attention_weight_max": 53.634071350097656,
      "activations/layer18_attention_weight_min": -33.21062088012695,
      "activations/layer19_attention_weight_max": 24.600393295288086,
      "activations/layer19_attention_weight_min": -18.492141723632812,
      "activations/layer1_attention_weight_max": 16.84306526184082,
      "activations/layer1_attention_weight_min": -17.14019012451172,
      "activations/layer20_attention_weight_max": 23.828001022338867,
      "activations/layer20_attention_weight_min": -22.28551483154297,
      "activations/layer21_attention_weight_max": 42.81098175048828,
      "activations/layer21_attention_weight_min": -26.20174217224121,
      "activations/layer22_attention_weight_max": 37.22116470336914,
      "activations/layer22_attention_weight_min": -25.36031723022461,
      "activations/layer23_attention_weight_max": 30.936878204345703,
      "activations/layer23_attention_weight_min": -19.69673728942871,
      "activations/layer2_attention_weight_max": 32.75065612792969,
      "activations/layer2_attention_weight_min": -31.090824127197266,
      "activations/layer3_attention_weight_max": 82.08424377441406,
      "activations/layer3_attention_weight_min": -85.13578033447266,
      "activations/layer4_attention_weight_max": 106.14350891113281,
      "activations/layer4_attention_weight_min": -97.37297821044922,
      "activations/layer5_attention_weight_max": 108.74358367919922,
      "activations/layer5_attention_weight_min": -93.76129150390625,
      "activations/layer6_attention_weight_max": 59.68307876586914,
      "activations/layer6_attention_weight_min": -54.16987228393555,
      "activations/layer7_attention_weight_max": 81.86226654052734,
      "activations/layer7_attention_weight_min": -71.39585876464844,
      "activations/layer8_attention_weight_max": 47.94987106323242,
      "activations/layer8_attention_weight_min": -43.236698150634766,
      "activations/layer9_attention_weight_max": 68.67012023925781,
      "activations/layer9_attention_weight_min": -50.89762496948242,
      "epoch": 5.47,
      "learning_rate": 0.0001158689393939394,
      "loss": 2.8377,
      "step": 94200
    },
    {
      "activations/layer0_attention_weight_max": 14.69593620300293,
      "activations/layer0_attention_weight_min": -14.5577392578125,
      "activations/layer10_attention_weight_max": 39.73781204223633,
      "activations/layer10_attention_weight_min": -35.262901306152344,
      "activations/layer11_attention_weight_max": 30.62618064880371,
      "activations/layer11_attention_weight_min": -26.392065048217773,
      "activations/layer12_attention_weight_max": 22.6264705657959,
      "activations/layer12_attention_weight_min": -24.973630905151367,
      "activations/layer13_attention_weight_max": 60.287078857421875,
      "activations/layer13_attention_weight_min": -37.98305892944336,
      "activations/layer14_attention_weight_max": 60.484954833984375,
      "activations/layer14_attention_weight_min": -37.42005920410156,
      "activations/layer15_attention_weight_max": 42.00138473510742,
      "activations/layer15_attention_weight_min": -30.784591674804688,
      "activations/layer16_attention_weight_max": 31.983474731445312,
      "activations/layer16_attention_weight_min": -27.983285903930664,
      "activations/layer17_attention_weight_max": 53.3450813293457,
      "activations/layer17_attention_weight_min": -39.77656173706055,
      "activations/layer18_attention_weight_max": 44.5257682800293,
      "activations/layer18_attention_weight_min": -32.1709098815918,
      "activations/layer19_attention_weight_max": 19.944074630737305,
      "activations/layer19_attention_weight_min": -20.00925636291504,
      "activations/layer1_attention_weight_max": 17.27501106262207,
      "activations/layer1_attention_weight_min": -14.637134552001953,
      "activations/layer20_attention_weight_max": 19.09435272216797,
      "activations/layer20_attention_weight_min": -21.278881072998047,
      "activations/layer21_attention_weight_max": 35.63796615600586,
      "activations/layer21_attention_weight_min": -25.647092819213867,
      "activations/layer22_attention_weight_max": 27.74339485168457,
      "activations/layer22_attention_weight_min": -22.907827377319336,
      "activations/layer23_attention_weight_max": 26.163944244384766,
      "activations/layer23_attention_weight_min": -16.731407165527344,
      "activations/layer2_attention_weight_max": 29.214374542236328,
      "activations/layer2_attention_weight_min": -27.899953842163086,
      "activations/layer3_attention_weight_max": 79.56497192382812,
      "activations/layer3_attention_weight_min": -83.59574127197266,
      "activations/layer4_attention_weight_max": 94.885986328125,
      "activations/layer4_attention_weight_min": -88.99193572998047,
      "activations/layer5_attention_weight_max": 94.86541748046875,
      "activations/layer5_attention_weight_min": -83.61981201171875,
      "activations/layer6_attention_weight_max": 57.65098190307617,
      "activations/layer6_attention_weight_min": -49.05276107788086,
      "activations/layer7_attention_weight_max": 70.60809326171875,
      "activations/layer7_attention_weight_min": -66.79682922363281,
      "activations/layer8_attention_weight_max": 43.20515441894531,
      "activations/layer8_attention_weight_min": -40.47321701049805,
      "activations/layer9_attention_weight_max": 53.179988861083984,
      "activations/layer9_attention_weight_min": -48.07963943481445,
      "epoch": 5.48,
      "learning_rate": 0.00011584999999999998,
      "loss": 2.8389,
      "step": 94250
    },
    {
      "activations/layer0_attention_weight_max": 14.262633323669434,
      "activations/layer0_attention_weight_min": -14.621613502502441,
      "activations/layer10_attention_weight_max": 35.75019836425781,
      "activations/layer10_attention_weight_min": -34.4059944152832,
      "activations/layer11_attention_weight_max": 27.438026428222656,
      "activations/layer11_attention_weight_min": -28.332565307617188,
      "activations/layer12_attention_weight_max": 31.62907600402832,
      "activations/layer12_attention_weight_min": -23.9838809967041,
      "activations/layer13_attention_weight_max": 50.034812927246094,
      "activations/layer13_attention_weight_min": -37.75014877319336,
      "activations/layer14_attention_weight_max": 40.03318786621094,
      "activations/layer14_attention_weight_min": -29.508466720581055,
      "activations/layer15_attention_weight_max": 37.035423278808594,
      "activations/layer15_attention_weight_min": -30.38131332397461,
      "activations/layer16_attention_weight_max": 34.86553955078125,
      "activations/layer16_attention_weight_min": -26.26892852783203,
      "activations/layer17_attention_weight_max": 53.63930130004883,
      "activations/layer17_attention_weight_min": -38.55146408081055,
      "activations/layer18_attention_weight_max": 48.40813446044922,
      "activations/layer18_attention_weight_min": -32.91545867919922,
      "activations/layer19_attention_weight_max": 20.64760398864746,
      "activations/layer19_attention_weight_min": -17.561260223388672,
      "activations/layer1_attention_weight_max": 17.848485946655273,
      "activations/layer1_attention_weight_min": -15.246371269226074,
      "activations/layer20_attention_weight_max": 21.318269729614258,
      "activations/layer20_attention_weight_min": -18.339433670043945,
      "activations/layer21_attention_weight_max": 35.147789001464844,
      "activations/layer21_attention_weight_min": -25.896169662475586,
      "activations/layer22_attention_weight_max": 36.80570602416992,
      "activations/layer22_attention_weight_min": -26.35570526123047,
      "activations/layer23_attention_weight_max": 30.04000473022461,
      "activations/layer23_attention_weight_min": -17.040714263916016,
      "activations/layer2_attention_weight_max": 29.805282592773438,
      "activations/layer2_attention_weight_min": -27.699615478515625,
      "activations/layer3_attention_weight_max": 82.66238403320312,
      "activations/layer3_attention_weight_min": -81.33490753173828,
      "activations/layer4_attention_weight_max": 99.61444854736328,
      "activations/layer4_attention_weight_min": -90.4714584350586,
      "activations/layer5_attention_weight_max": 98.7734375,
      "activations/layer5_attention_weight_min": -89.65453338623047,
      "activations/layer6_attention_weight_max": 55.842472076416016,
      "activations/layer6_attention_weight_min": -51.530311584472656,
      "activations/layer7_attention_weight_max": 72.34989929199219,
      "activations/layer7_attention_weight_min": -65.88648223876953,
      "activations/layer8_attention_weight_max": 41.95522689819336,
      "activations/layer8_attention_weight_min": -43.36629867553711,
      "activations/layer9_attention_weight_max": 53.977760314941406,
      "activations/layer9_attention_weight_min": -47.60439682006836,
      "epoch": 5.48,
      "learning_rate": 0.0001158310606060606,
      "loss": 2.8544,
      "step": 94300
    },
    {
      "activations/layer0_attention_weight_max": 15.757740020751953,
      "activations/layer0_attention_weight_min": -14.954167366027832,
      "activations/layer10_attention_weight_max": 36.77754592895508,
      "activations/layer10_attention_weight_min": -33.70350646972656,
      "activations/layer11_attention_weight_max": 29.42205810546875,
      "activations/layer11_attention_weight_min": -26.344436645507812,
      "activations/layer12_attention_weight_max": 25.657007217407227,
      "activations/layer12_attention_weight_min": -28.31092643737793,
      "activations/layer13_attention_weight_max": 48.25585174560547,
      "activations/layer13_attention_weight_min": -35.855899810791016,
      "activations/layer14_attention_weight_max": 45.54856872558594,
      "activations/layer14_attention_weight_min": -29.789840698242188,
      "activations/layer15_attention_weight_max": 39.091739654541016,
      "activations/layer15_attention_weight_min": -29.89008903503418,
      "activations/layer16_attention_weight_max": 33.78349685668945,
      "activations/layer16_attention_weight_min": -25.8665828704834,
      "activations/layer17_attention_weight_max": 56.96263885498047,
      "activations/layer17_attention_weight_min": -38.255882263183594,
      "activations/layer18_attention_weight_max": 45.73711013793945,
      "activations/layer18_attention_weight_min": -32.48794174194336,
      "activations/layer19_attention_weight_max": 22.67156982421875,
      "activations/layer19_attention_weight_min": -20.01438331604004,
      "activations/layer1_attention_weight_max": 17.112300872802734,
      "activations/layer1_attention_weight_min": -17.252153396606445,
      "activations/layer20_attention_weight_max": 20.968626022338867,
      "activations/layer20_attention_weight_min": -21.188932418823242,
      "activations/layer21_attention_weight_max": 37.282466888427734,
      "activations/layer21_attention_weight_min": -25.307891845703125,
      "activations/layer22_attention_weight_max": 33.250450134277344,
      "activations/layer22_attention_weight_min": -24.10667610168457,
      "activations/layer23_attention_weight_max": 24.718162536621094,
      "activations/layer23_attention_weight_min": -19.122106552124023,
      "activations/layer2_attention_weight_max": 29.754758834838867,
      "activations/layer2_attention_weight_min": -28.12245750427246,
      "activations/layer3_attention_weight_max": 80.90996551513672,
      "activations/layer3_attention_weight_min": -82.12722778320312,
      "activations/layer4_attention_weight_max": 98.34326171875,
      "activations/layer4_attention_weight_min": -89.42024230957031,
      "activations/layer5_attention_weight_max": 98.373291015625,
      "activations/layer5_attention_weight_min": -89.4183349609375,
      "activations/layer6_attention_weight_max": 55.97370910644531,
      "activations/layer6_attention_weight_min": -51.70741271972656,
      "activations/layer7_attention_weight_max": 70.5500259399414,
      "activations/layer7_attention_weight_min": -65.88087463378906,
      "activations/layer8_attention_weight_max": 42.77069091796875,
      "activations/layer8_attention_weight_min": -43.49003982543945,
      "activations/layer9_attention_weight_max": 49.54817581176758,
      "activations/layer9_attention_weight_min": -48.01439666748047,
      "epoch": 5.48,
      "learning_rate": 0.0001158121212121212,
      "loss": 2.8511,
      "step": 94350
    },
    {
      "activations/layer0_attention_weight_max": 14.384511947631836,
      "activations/layer0_attention_weight_min": -14.940443992614746,
      "activations/layer10_attention_weight_max": 38.07900619506836,
      "activations/layer10_attention_weight_min": -33.39030838012695,
      "activations/layer11_attention_weight_max": 29.142311096191406,
      "activations/layer11_attention_weight_min": -27.722686767578125,
      "activations/layer12_attention_weight_max": 21.80071449279785,
      "activations/layer12_attention_weight_min": -26.258039474487305,
      "activations/layer13_attention_weight_max": 50.65589904785156,
      "activations/layer13_attention_weight_min": -37.32997512817383,
      "activations/layer14_attention_weight_max": 51.54446792602539,
      "activations/layer14_attention_weight_min": -35.85456085205078,
      "activations/layer15_attention_weight_max": 42.478370666503906,
      "activations/layer15_attention_weight_min": -29.06792640686035,
      "activations/layer16_attention_weight_max": 33.44533157348633,
      "activations/layer16_attention_weight_min": -26.418848037719727,
      "activations/layer17_attention_weight_max": 59.97172164916992,
      "activations/layer17_attention_weight_min": -38.96662521362305,
      "activations/layer18_attention_weight_max": 47.13352966308594,
      "activations/layer18_attention_weight_min": -35.05732345581055,
      "activations/layer19_attention_weight_max": 22.028732299804688,
      "activations/layer19_attention_weight_min": -22.34913444519043,
      "activations/layer1_attention_weight_max": 17.14720916748047,
      "activations/layer1_attention_weight_min": -17.24917984008789,
      "activations/layer20_attention_weight_max": 22.295148849487305,
      "activations/layer20_attention_weight_min": -23.090822219848633,
      "activations/layer21_attention_weight_max": 43.678401947021484,
      "activations/layer21_attention_weight_min": -25.315231323242188,
      "activations/layer22_attention_weight_max": 31.141592025756836,
      "activations/layer22_attention_weight_min": -24.80113410949707,
      "activations/layer23_attention_weight_max": 27.947444915771484,
      "activations/layer23_attention_weight_min": -18.07954978942871,
      "activations/layer2_attention_weight_max": 29.379188537597656,
      "activations/layer2_attention_weight_min": -28.137638092041016,
      "activations/layer3_attention_weight_max": 81.51573181152344,
      "activations/layer3_attention_weight_min": -83.99549102783203,
      "activations/layer4_attention_weight_max": 102.26748657226562,
      "activations/layer4_attention_weight_min": -93.1082992553711,
      "activations/layer5_attention_weight_max": 102.9244384765625,
      "activations/layer5_attention_weight_min": -92.27141571044922,
      "activations/layer6_attention_weight_max": 62.807594299316406,
      "activations/layer6_attention_weight_min": -54.843326568603516,
      "activations/layer7_attention_weight_max": 71.2293930053711,
      "activations/layer7_attention_weight_min": -65.14091491699219,
      "activations/layer8_attention_weight_max": 44.96980667114258,
      "activations/layer8_attention_weight_min": -42.380558013916016,
      "activations/layer9_attention_weight_max": 53.3983039855957,
      "activations/layer9_attention_weight_min": -45.880611419677734,
      "epoch": 5.49,
      "learning_rate": 0.0001157931818181818,
      "loss": 2.8461,
      "step": 94400
    },
    {
      "activations/layer0_attention_weight_max": 15.085195541381836,
      "activations/layer0_attention_weight_min": -15.728066444396973,
      "activations/layer10_attention_weight_max": 36.03361129760742,
      "activations/layer10_attention_weight_min": -34.712562561035156,
      "activations/layer11_attention_weight_max": 27.688934326171875,
      "activations/layer11_attention_weight_min": -26.183467864990234,
      "activations/layer12_attention_weight_max": 23.91257095336914,
      "activations/layer12_attention_weight_min": -25.286584854125977,
      "activations/layer13_attention_weight_max": 48.43727111816406,
      "activations/layer13_attention_weight_min": -37.14458465576172,
      "activations/layer14_attention_weight_max": 35.3740234375,
      "activations/layer14_attention_weight_min": -27.605335235595703,
      "activations/layer15_attention_weight_max": 37.87799835205078,
      "activations/layer15_attention_weight_min": -29.390806198120117,
      "activations/layer16_attention_weight_max": 29.455020904541016,
      "activations/layer16_attention_weight_min": -26.04334259033203,
      "activations/layer17_attention_weight_max": 52.34260940551758,
      "activations/layer17_attention_weight_min": -37.76675796508789,
      "activations/layer18_attention_weight_max": 45.44374084472656,
      "activations/layer18_attention_weight_min": -33.327919006347656,
      "activations/layer19_attention_weight_max": 21.842788696289062,
      "activations/layer19_attention_weight_min": -19.54840850830078,
      "activations/layer1_attention_weight_max": 16.701261520385742,
      "activations/layer1_attention_weight_min": -16.443126678466797,
      "activations/layer20_attention_weight_max": 19.341787338256836,
      "activations/layer20_attention_weight_min": -22.777217864990234,
      "activations/layer21_attention_weight_max": 39.33113479614258,
      "activations/layer21_attention_weight_min": -25.3821964263916,
      "activations/layer22_attention_weight_max": 30.272356033325195,
      "activations/layer22_attention_weight_min": -24.185396194458008,
      "activations/layer23_attention_weight_max": 25.252769470214844,
      "activations/layer23_attention_weight_min": -18.735254287719727,
      "activations/layer2_attention_weight_max": 29.333595275878906,
      "activations/layer2_attention_weight_min": -29.008237838745117,
      "activations/layer3_attention_weight_max": 79.58438110351562,
      "activations/layer3_attention_weight_min": -80.98851013183594,
      "activations/layer4_attention_weight_max": 95.90033721923828,
      "activations/layer4_attention_weight_min": -89.29430389404297,
      "activations/layer5_attention_weight_max": 94.93891906738281,
      "activations/layer5_attention_weight_min": -85.50802612304688,
      "activations/layer6_attention_weight_max": 55.55437088012695,
      "activations/layer6_attention_weight_min": -50.39870071411133,
      "activations/layer7_attention_weight_max": 65.20613861083984,
      "activations/layer7_attention_weight_min": -64.2645034790039,
      "activations/layer8_attention_weight_max": 42.47529220581055,
      "activations/layer8_attention_weight_min": -42.111610412597656,
      "activations/layer9_attention_weight_max": 46.621646881103516,
      "activations/layer9_attention_weight_min": -44.6645393371582,
      "epoch": 5.49,
      "learning_rate": 0.00011577424242424242,
      "loss": 2.8344,
      "step": 94450
    },
    {
      "activations/layer0_attention_weight_max": 14.262896537780762,
      "activations/layer0_attention_weight_min": -15.632231712341309,
      "activations/layer10_attention_weight_max": 38.2231330871582,
      "activations/layer10_attention_weight_min": -34.60572052001953,
      "activations/layer11_attention_weight_max": 29.873958587646484,
      "activations/layer11_attention_weight_min": -28.104808807373047,
      "activations/layer12_attention_weight_max": 21.978532791137695,
      "activations/layer12_attention_weight_min": -24.458322525024414,
      "activations/layer13_attention_weight_max": 58.76176452636719,
      "activations/layer13_attention_weight_min": -40.80895233154297,
      "activations/layer14_attention_weight_max": 58.28113555908203,
      "activations/layer14_attention_weight_min": -35.16748046875,
      "activations/layer15_attention_weight_max": 43.0438117980957,
      "activations/layer15_attention_weight_min": -30.04068946838379,
      "activations/layer16_attention_weight_max": 35.5682258605957,
      "activations/layer16_attention_weight_min": -25.39891815185547,
      "activations/layer17_attention_weight_max": 60.56570053100586,
      "activations/layer17_attention_weight_min": -41.945865631103516,
      "activations/layer18_attention_weight_max": 50.75355911254883,
      "activations/layer18_attention_weight_min": -33.61687469482422,
      "activations/layer19_attention_weight_max": 22.961891174316406,
      "activations/layer19_attention_weight_min": -20.763385772705078,
      "activations/layer1_attention_weight_max": 17.635286331176758,
      "activations/layer1_attention_weight_min": -16.126060485839844,
      "activations/layer20_attention_weight_max": 24.05109214782715,
      "activations/layer20_attention_weight_min": -23.045042037963867,
      "activations/layer21_attention_weight_max": 43.24525451660156,
      "activations/layer21_attention_weight_min": -25.544872283935547,
      "activations/layer22_attention_weight_max": 34.23988342285156,
      "activations/layer22_attention_weight_min": -26.744688034057617,
      "activations/layer23_attention_weight_max": 30.450780868530273,
      "activations/layer23_attention_weight_min": -20.306917190551758,
      "activations/layer2_attention_weight_max": 29.15579605102539,
      "activations/layer2_attention_weight_min": -29.39025115966797,
      "activations/layer3_attention_weight_max": 81.61732482910156,
      "activations/layer3_attention_weight_min": -84.72206115722656,
      "activations/layer4_attention_weight_max": 99.20098114013672,
      "activations/layer4_attention_weight_min": -96.125732421875,
      "activations/layer5_attention_weight_max": 97.60823059082031,
      "activations/layer5_attention_weight_min": -89.92195892333984,
      "activations/layer6_attention_weight_max": 58.24028778076172,
      "activations/layer6_attention_weight_min": -53.92914962768555,
      "activations/layer7_attention_weight_max": 69.1725082397461,
      "activations/layer7_attention_weight_min": -69.91857147216797,
      "activations/layer8_attention_weight_max": 41.744285583496094,
      "activations/layer8_attention_weight_min": -45.27909469604492,
      "activations/layer9_attention_weight_max": 47.66643142700195,
      "activations/layer9_attention_weight_min": -50.716407775878906,
      "epoch": 5.49,
      "learning_rate": 0.00011575530303030302,
      "loss": 2.856,
      "step": 94500
    },
    {
      "activations/layer0_attention_weight_max": 15.275333404541016,
      "activations/layer0_attention_weight_min": -14.893242835998535,
      "activations/layer10_attention_weight_max": 39.191123962402344,
      "activations/layer10_attention_weight_min": -32.97251510620117,
      "activations/layer11_attention_weight_max": 32.97603225708008,
      "activations/layer11_attention_weight_min": -27.748332977294922,
      "activations/layer12_attention_weight_max": 22.84842300415039,
      "activations/layer12_attention_weight_min": -27.486539840698242,
      "activations/layer13_attention_weight_max": 42.513465881347656,
      "activations/layer13_attention_weight_min": -30.696857452392578,
      "activations/layer14_attention_weight_max": 48.31836700439453,
      "activations/layer14_attention_weight_min": -33.02873611450195,
      "activations/layer15_attention_weight_max": 41.93592071533203,
      "activations/layer15_attention_weight_min": -29.636993408203125,
      "activations/layer16_attention_weight_max": 33.07583999633789,
      "activations/layer16_attention_weight_min": -28.32367706298828,
      "activations/layer17_attention_weight_max": 55.40163040161133,
      "activations/layer17_attention_weight_min": -41.5175666809082,
      "activations/layer18_attention_weight_max": 46.98667907714844,
      "activations/layer18_attention_weight_min": -33.77399826049805,
      "activations/layer19_attention_weight_max": 24.949369430541992,
      "activations/layer19_attention_weight_min": -20.388586044311523,
      "activations/layer1_attention_weight_max": 17.181453704833984,
      "activations/layer1_attention_weight_min": -15.360267639160156,
      "activations/layer20_attention_weight_max": 20.213117599487305,
      "activations/layer20_attention_weight_min": -21.907146453857422,
      "activations/layer21_attention_weight_max": 38.726219177246094,
      "activations/layer21_attention_weight_min": -25.388778686523438,
      "activations/layer22_attention_weight_max": 36.9257926940918,
      "activations/layer22_attention_weight_min": -23.953781127929688,
      "activations/layer23_attention_weight_max": 29.361778259277344,
      "activations/layer23_attention_weight_min": -18.574281692504883,
      "activations/layer2_attention_weight_max": 30.45050048828125,
      "activations/layer2_attention_weight_min": -30.03525161743164,
      "activations/layer3_attention_weight_max": 80.11236572265625,
      "activations/layer3_attention_weight_min": -83.39302825927734,
      "activations/layer4_attention_weight_max": 100.48355102539062,
      "activations/layer4_attention_weight_min": -89.93480682373047,
      "activations/layer5_attention_weight_max": 100.02818298339844,
      "activations/layer5_attention_weight_min": -89.5133285522461,
      "activations/layer6_attention_weight_max": 57.254798889160156,
      "activations/layer6_attention_weight_min": -51.58163070678711,
      "activations/layer7_attention_weight_max": 68.92875671386719,
      "activations/layer7_attention_weight_min": -65.30228424072266,
      "activations/layer8_attention_weight_max": 42.76802444458008,
      "activations/layer8_attention_weight_min": -42.16830062866211,
      "activations/layer9_attention_weight_max": 49.31589889526367,
      "activations/layer9_attention_weight_min": -44.96055603027344,
      "epoch": 5.49,
      "learning_rate": 0.00011573636363636363,
      "loss": 2.8557,
      "step": 94550
    },
    {
      "activations/layer0_attention_weight_max": 13.842020034790039,
      "activations/layer0_attention_weight_min": -14.763933181762695,
      "activations/layer10_attention_weight_max": 37.49677658081055,
      "activations/layer10_attention_weight_min": -34.33036422729492,
      "activations/layer11_attention_weight_max": 30.037567138671875,
      "activations/layer11_attention_weight_min": -28.08779525756836,
      "activations/layer12_attention_weight_max": 26.09652328491211,
      "activations/layer12_attention_weight_min": -25.72445297241211,
      "activations/layer13_attention_weight_max": 46.306514739990234,
      "activations/layer13_attention_weight_min": -31.85178565979004,
      "activations/layer14_attention_weight_max": 40.37226867675781,
      "activations/layer14_attention_weight_min": -29.27897834777832,
      "activations/layer15_attention_weight_max": 41.73796081542969,
      "activations/layer15_attention_weight_min": -29.71126937866211,
      "activations/layer16_attention_weight_max": 41.18171691894531,
      "activations/layer16_attention_weight_min": -27.119638442993164,
      "activations/layer17_attention_weight_max": 57.94874954223633,
      "activations/layer17_attention_weight_min": -38.33437728881836,
      "activations/layer18_attention_weight_max": 52.59230422973633,
      "activations/layer18_attention_weight_min": -34.494346618652344,
      "activations/layer19_attention_weight_max": 22.586286544799805,
      "activations/layer19_attention_weight_min": -18.0087833404541,
      "activations/layer1_attention_weight_max": 17.055130004882812,
      "activations/layer1_attention_weight_min": -14.384878158569336,
      "activations/layer20_attention_weight_max": 20.13847541809082,
      "activations/layer20_attention_weight_min": -18.953968048095703,
      "activations/layer21_attention_weight_max": 39.60518264770508,
      "activations/layer21_attention_weight_min": -30.231903076171875,
      "activations/layer22_attention_weight_max": 32.497642517089844,
      "activations/layer22_attention_weight_min": -25.880815505981445,
      "activations/layer23_attention_weight_max": 26.722389221191406,
      "activations/layer23_attention_weight_min": -19.443161010742188,
      "activations/layer2_attention_weight_max": 30.404081344604492,
      "activations/layer2_attention_weight_min": -28.6163387298584,
      "activations/layer3_attention_weight_max": 81.86139678955078,
      "activations/layer3_attention_weight_min": -83.41329193115234,
      "activations/layer4_attention_weight_max": 100.99849700927734,
      "activations/layer4_attention_weight_min": -93.27548217773438,
      "activations/layer5_attention_weight_max": 98.45018005371094,
      "activations/layer5_attention_weight_min": -91.79662322998047,
      "activations/layer6_attention_weight_max": 57.949256896972656,
      "activations/layer6_attention_weight_min": -56.081668853759766,
      "activations/layer7_attention_weight_max": 67.26811981201172,
      "activations/layer7_attention_weight_min": -66.72530364990234,
      "activations/layer8_attention_weight_max": 47.072837829589844,
      "activations/layer8_attention_weight_min": -46.06806182861328,
      "activations/layer9_attention_weight_max": 50.592750549316406,
      "activations/layer9_attention_weight_min": -46.326473236083984,
      "epoch": 5.5,
      "learning_rate": 0.00011571742424242423,
      "loss": 2.8492,
      "step": 94600
    },
    {
      "activations/layer0_attention_weight_max": 14.815315246582031,
      "activations/layer0_attention_weight_min": -15.388662338256836,
      "activations/layer10_attention_weight_max": 39.7647590637207,
      "activations/layer10_attention_weight_min": -34.42790985107422,
      "activations/layer11_attention_weight_max": 30.294235229492188,
      "activations/layer11_attention_weight_min": -29.102603912353516,
      "activations/layer12_attention_weight_max": 22.85771942138672,
      "activations/layer12_attention_weight_min": -23.415565490722656,
      "activations/layer13_attention_weight_max": 55.43925476074219,
      "activations/layer13_attention_weight_min": -34.25065612792969,
      "activations/layer14_attention_weight_max": 46.970436096191406,
      "activations/layer14_attention_weight_min": -30.149654388427734,
      "activations/layer15_attention_weight_max": 41.57268142700195,
      "activations/layer15_attention_weight_min": -30.40869903564453,
      "activations/layer16_attention_weight_max": 38.374900817871094,
      "activations/layer16_attention_weight_min": -27.86651039123535,
      "activations/layer17_attention_weight_max": 56.73965835571289,
      "activations/layer17_attention_weight_min": -38.450782775878906,
      "activations/layer18_attention_weight_max": 49.99888229370117,
      "activations/layer18_attention_weight_min": -34.68043518066406,
      "activations/layer19_attention_weight_max": 22.64171028137207,
      "activations/layer19_attention_weight_min": -19.925424575805664,
      "activations/layer1_attention_weight_max": 16.396745681762695,
      "activations/layer1_attention_weight_min": -15.658201217651367,
      "activations/layer20_attention_weight_max": 20.364662170410156,
      "activations/layer20_attention_weight_min": -19.760061264038086,
      "activations/layer21_attention_weight_max": 39.81112289428711,
      "activations/layer21_attention_weight_min": -25.290796279907227,
      "activations/layer22_attention_weight_max": 34.3708381652832,
      "activations/layer22_attention_weight_min": -22.340892791748047,
      "activations/layer23_attention_weight_max": 30.754032135009766,
      "activations/layer23_attention_weight_min": -18.49764633178711,
      "activations/layer2_attention_weight_max": 32.12714385986328,
      "activations/layer2_attention_weight_min": -31.012283325195312,
      "activations/layer3_attention_weight_max": 81.3558349609375,
      "activations/layer3_attention_weight_min": -81.96155548095703,
      "activations/layer4_attention_weight_max": 100.02458190917969,
      "activations/layer4_attention_weight_min": -94.39262390136719,
      "activations/layer5_attention_weight_max": 100.43601989746094,
      "activations/layer5_attention_weight_min": -90.25801849365234,
      "activations/layer6_attention_weight_max": 57.49763107299805,
      "activations/layer6_attention_weight_min": -53.27509307861328,
      "activations/layer7_attention_weight_max": 73.45191192626953,
      "activations/layer7_attention_weight_min": -70.32763671875,
      "activations/layer8_attention_weight_max": 44.59125518798828,
      "activations/layer8_attention_weight_min": -43.589473724365234,
      "activations/layer9_attention_weight_max": 49.59922790527344,
      "activations/layer9_attention_weight_min": -47.49446105957031,
      "epoch": 5.5,
      "learning_rate": 0.00011569848484848484,
      "loss": 2.8592,
      "step": 94650
    },
    {
      "activations/layer0_attention_weight_max": 15.494879722595215,
      "activations/layer0_attention_weight_min": -15.2876615524292,
      "activations/layer10_attention_weight_max": 49.67653274536133,
      "activations/layer10_attention_weight_min": -38.60211181640625,
      "activations/layer11_attention_weight_max": 38.27443313598633,
      "activations/layer11_attention_weight_min": -28.898387908935547,
      "activations/layer12_attention_weight_max": 28.331567764282227,
      "activations/layer12_attention_weight_min": -26.520248413085938,
      "activations/layer13_attention_weight_max": 54.08658981323242,
      "activations/layer13_attention_weight_min": -36.28568649291992,
      "activations/layer14_attention_weight_max": 53.2137451171875,
      "activations/layer14_attention_weight_min": -32.6910400390625,
      "activations/layer15_attention_weight_max": 39.03485107421875,
      "activations/layer15_attention_weight_min": -29.25958824157715,
      "activations/layer16_attention_weight_max": 36.09152603149414,
      "activations/layer16_attention_weight_min": -26.927635192871094,
      "activations/layer17_attention_weight_max": 59.22108840942383,
      "activations/layer17_attention_weight_min": -40.659358978271484,
      "activations/layer18_attention_weight_max": 47.00388717651367,
      "activations/layer18_attention_weight_min": -33.50951385498047,
      "activations/layer19_attention_weight_max": 19.59664535522461,
      "activations/layer19_attention_weight_min": -21.89176368713379,
      "activations/layer1_attention_weight_max": 17.598865509033203,
      "activations/layer1_attention_weight_min": -19.960538864135742,
      "activations/layer20_attention_weight_max": 20.46673011779785,
      "activations/layer20_attention_weight_min": -19.429832458496094,
      "activations/layer21_attention_weight_max": 40.84333038330078,
      "activations/layer21_attention_weight_min": -24.395200729370117,
      "activations/layer22_attention_weight_max": 31.43556785583496,
      "activations/layer22_attention_weight_min": -23.379060745239258,
      "activations/layer23_attention_weight_max": 29.70734405517578,
      "activations/layer23_attention_weight_min": -18.785127639770508,
      "activations/layer2_attention_weight_max": 34.67334747314453,
      "activations/layer2_attention_weight_min": -31.54231071472168,
      "activations/layer3_attention_weight_max": 90.65702819824219,
      "activations/layer3_attention_weight_min": -90.89411926269531,
      "activations/layer4_attention_weight_max": 108.50902557373047,
      "activations/layer4_attention_weight_min": -104.6413345336914,
      "activations/layer5_attention_weight_max": 107.4964599609375,
      "activations/layer5_attention_weight_min": -92.64903259277344,
      "activations/layer6_attention_weight_max": 63.54493713378906,
      "activations/layer6_attention_weight_min": -54.99240493774414,
      "activations/layer7_attention_weight_max": 82.2442626953125,
      "activations/layer7_attention_weight_min": -70.59795379638672,
      "activations/layer8_attention_weight_max": 51.07667922973633,
      "activations/layer8_attention_weight_min": -47.9634895324707,
      "activations/layer9_attention_weight_max": 70.26390838623047,
      "activations/layer9_attention_weight_min": -53.700592041015625,
      "epoch": 5.5,
      "learning_rate": 0.00011567954545454545,
      "loss": 2.8479,
      "step": 94700
    },
    {
      "activations/layer0_attention_weight_max": 14.907448768615723,
      "activations/layer0_attention_weight_min": -15.143061637878418,
      "activations/layer10_attention_weight_max": 43.8580207824707,
      "activations/layer10_attention_weight_min": -36.81926345825195,
      "activations/layer11_attention_weight_max": 31.65937614440918,
      "activations/layer11_attention_weight_min": -29.26643180847168,
      "activations/layer12_attention_weight_max": 20.61617088317871,
      "activations/layer12_attention_weight_min": -24.058752059936523,
      "activations/layer13_attention_weight_max": 52.205345153808594,
      "activations/layer13_attention_weight_min": -37.245662689208984,
      "activations/layer14_attention_weight_max": 71.64657592773438,
      "activations/layer14_attention_weight_min": -45.98063659667969,
      "activations/layer15_attention_weight_max": 53.4383544921875,
      "activations/layer15_attention_weight_min": -36.81816864013672,
      "activations/layer16_attention_weight_max": 36.053436279296875,
      "activations/layer16_attention_weight_min": -27.390913009643555,
      "activations/layer17_attention_weight_max": 57.45660400390625,
      "activations/layer17_attention_weight_min": -39.34754943847656,
      "activations/layer18_attention_weight_max": 48.54990005493164,
      "activations/layer18_attention_weight_min": -34.896270751953125,
      "activations/layer19_attention_weight_max": 24.26640510559082,
      "activations/layer19_attention_weight_min": -21.129268646240234,
      "activations/layer1_attention_weight_max": 18.054466247558594,
      "activations/layer1_attention_weight_min": -15.619868278503418,
      "activations/layer20_attention_weight_max": 24.306989669799805,
      "activations/layer20_attention_weight_min": -21.73093032836914,
      "activations/layer21_attention_weight_max": 44.1847038269043,
      "activations/layer21_attention_weight_min": -26.038314819335938,
      "activations/layer22_attention_weight_max": 36.8880500793457,
      "activations/layer22_attention_weight_min": -24.647003173828125,
      "activations/layer23_attention_weight_max": 29.231224060058594,
      "activations/layer23_attention_weight_min": -19.45807647705078,
      "activations/layer2_attention_weight_max": 33.68376159667969,
      "activations/layer2_attention_weight_min": -30.378599166870117,
      "activations/layer3_attention_weight_max": 83.43103790283203,
      "activations/layer3_attention_weight_min": -86.30935668945312,
      "activations/layer4_attention_weight_max": 103.7638168334961,
      "activations/layer4_attention_weight_min": -98.70635223388672,
      "activations/layer5_attention_weight_max": 101.16524505615234,
      "activations/layer5_attention_weight_min": -90.89695739746094,
      "activations/layer6_attention_weight_max": 56.842288970947266,
      "activations/layer6_attention_weight_min": -51.746681213378906,
      "activations/layer7_attention_weight_max": 69.56858825683594,
      "activations/layer7_attention_weight_min": -71.39019012451172,
      "activations/layer8_attention_weight_max": 44.344730377197266,
      "activations/layer8_attention_weight_min": -44.15701675415039,
      "activations/layer9_attention_weight_max": 53.338775634765625,
      "activations/layer9_attention_weight_min": -52.4360237121582,
      "epoch": 5.51,
      "learning_rate": 0.00011566060606060604,
      "loss": 2.833,
      "step": 94750
    },
    {
      "activations/layer0_attention_weight_max": 15.287736892700195,
      "activations/layer0_attention_weight_min": -15.269575119018555,
      "activations/layer10_attention_weight_max": 40.6389045715332,
      "activations/layer10_attention_weight_min": -39.219024658203125,
      "activations/layer11_attention_weight_max": 33.898529052734375,
      "activations/layer11_attention_weight_min": -27.719398498535156,
      "activations/layer12_attention_weight_max": 21.585208892822266,
      "activations/layer12_attention_weight_min": -25.191892623901367,
      "activations/layer13_attention_weight_max": 64.43937683105469,
      "activations/layer13_attention_weight_min": -43.32365036010742,
      "activations/layer14_attention_weight_max": 71.73654174804688,
      "activations/layer14_attention_weight_min": -39.56375503540039,
      "activations/layer15_attention_weight_max": 42.724666595458984,
      "activations/layer15_attention_weight_min": -31.312183380126953,
      "activations/layer16_attention_weight_max": 44.562232971191406,
      "activations/layer16_attention_weight_min": -28.636632919311523,
      "activations/layer17_attention_weight_max": 65.31393432617188,
      "activations/layer17_attention_weight_min": -40.47210693359375,
      "activations/layer18_attention_weight_max": 53.90420150756836,
      "activations/layer18_attention_weight_min": -37.43227767944336,
      "activations/layer19_attention_weight_max": 24.902551651000977,
      "activations/layer19_attention_weight_min": -19.749799728393555,
      "activations/layer1_attention_weight_max": 17.90508460998535,
      "activations/layer1_attention_weight_min": -15.603501319885254,
      "activations/layer20_attention_weight_max": 29.498462677001953,
      "activations/layer20_attention_weight_min": -23.772706985473633,
      "activations/layer21_attention_weight_max": 58.17362594604492,
      "activations/layer21_attention_weight_min": -29.35563087463379,
      "activations/layer22_attention_weight_max": 39.79831314086914,
      "activations/layer22_attention_weight_min": -24.362688064575195,
      "activations/layer23_attention_weight_max": 32.809356689453125,
      "activations/layer23_attention_weight_min": -18.43447494506836,
      "activations/layer2_attention_weight_max": 31.440351486206055,
      "activations/layer2_attention_weight_min": -28.744802474975586,
      "activations/layer3_attention_weight_max": 84.01065826416016,
      "activations/layer3_attention_weight_min": -87.23622131347656,
      "activations/layer4_attention_weight_max": 107.5892562866211,
      "activations/layer4_attention_weight_min": -101.04737091064453,
      "activations/layer5_attention_weight_max": 105.27099609375,
      "activations/layer5_attention_weight_min": -87.1131591796875,
      "activations/layer6_attention_weight_max": 58.90408706665039,
      "activations/layer6_attention_weight_min": -52.034507751464844,
      "activations/layer7_attention_weight_max": 81.66796112060547,
      "activations/layer7_attention_weight_min": -67.33377838134766,
      "activations/layer8_attention_weight_max": 50.4556770324707,
      "activations/layer8_attention_weight_min": -41.17443084716797,
      "activations/layer9_attention_weight_max": 64.4539794921875,
      "activations/layer9_attention_weight_min": -51.77030563354492,
      "epoch": 5.51,
      "learning_rate": 0.00011564166666666665,
      "loss": 2.8409,
      "step": 94800
    },
    {
      "activations/layer0_attention_weight_max": 14.457230567932129,
      "activations/layer0_attention_weight_min": -15.309311866760254,
      "activations/layer10_attention_weight_max": 44.6118278503418,
      "activations/layer10_attention_weight_min": -40.14567947387695,
      "activations/layer11_attention_weight_max": 35.38987731933594,
      "activations/layer11_attention_weight_min": -28.963825225830078,
      "activations/layer12_attention_weight_max": 18.961095809936523,
      "activations/layer12_attention_weight_min": -27.566415786743164,
      "activations/layer13_attention_weight_max": 47.77653884887695,
      "activations/layer13_attention_weight_min": -35.51789093017578,
      "activations/layer14_attention_weight_max": 52.806297302246094,
      "activations/layer14_attention_weight_min": -37.7579460144043,
      "activations/layer15_attention_weight_max": 39.8023681640625,
      "activations/layer15_attention_weight_min": -28.983423233032227,
      "activations/layer16_attention_weight_max": 32.71279525756836,
      "activations/layer16_attention_weight_min": -25.488964080810547,
      "activations/layer17_attention_weight_max": 50.927242279052734,
      "activations/layer17_attention_weight_min": -38.12071228027344,
      "activations/layer18_attention_weight_max": 45.248252868652344,
      "activations/layer18_attention_weight_min": -32.07952880859375,
      "activations/layer19_attention_weight_max": 19.579143524169922,
      "activations/layer19_attention_weight_min": -18.04630470275879,
      "activations/layer1_attention_weight_max": 16.97863006591797,
      "activations/layer1_attention_weight_min": -16.386844635009766,
      "activations/layer20_attention_weight_max": 18.886369705200195,
      "activations/layer20_attention_weight_min": -20.841909408569336,
      "activations/layer21_attention_weight_max": 33.386287689208984,
      "activations/layer21_attention_weight_min": -22.77586555480957,
      "activations/layer22_attention_weight_max": 30.21403694152832,
      "activations/layer22_attention_weight_min": -23.743074417114258,
      "activations/layer23_attention_weight_max": 24.96015739440918,
      "activations/layer23_attention_weight_min": -16.85173797607422,
      "activations/layer2_attention_weight_max": 31.85309410095215,
      "activations/layer2_attention_weight_min": -29.568754196166992,
      "activations/layer3_attention_weight_max": 84.3735580444336,
      "activations/layer3_attention_weight_min": -85.72709655761719,
      "activations/layer4_attention_weight_max": 106.0060806274414,
      "activations/layer4_attention_weight_min": -100.57147979736328,
      "activations/layer5_attention_weight_max": 101.27926635742188,
      "activations/layer5_attention_weight_min": -93.29795837402344,
      "activations/layer6_attention_weight_max": 56.24794387817383,
      "activations/layer6_attention_weight_min": -54.19334030151367,
      "activations/layer7_attention_weight_max": 88.583251953125,
      "activations/layer7_attention_weight_min": -71.01071166992188,
      "activations/layer8_attention_weight_max": 56.651824951171875,
      "activations/layer8_attention_weight_min": -45.457008361816406,
      "activations/layer9_attention_weight_max": 73.38179779052734,
      "activations/layer9_attention_weight_min": -53.897518157958984,
      "epoch": 5.51,
      "learning_rate": 0.00011562272727272727,
      "loss": 2.8371,
      "step": 94850
    },
    {
      "activations/layer0_attention_weight_max": 14.533048629760742,
      "activations/layer0_attention_weight_min": -14.87654972076416,
      "activations/layer10_attention_weight_max": 45.59489822387695,
      "activations/layer10_attention_weight_min": -35.54460144042969,
      "activations/layer11_attention_weight_max": 40.9316291809082,
      "activations/layer11_attention_weight_min": -28.729747772216797,
      "activations/layer12_attention_weight_max": 26.190214157104492,
      "activations/layer12_attention_weight_min": -26.464832305908203,
      "activations/layer13_attention_weight_max": 90.65193939208984,
      "activations/layer13_attention_weight_min": -43.515377044677734,
      "activations/layer14_attention_weight_max": 93.74063873291016,
      "activations/layer14_attention_weight_min": -42.37800979614258,
      "activations/layer15_attention_weight_max": 63.843772888183594,
      "activations/layer15_attention_weight_min": -30.274070739746094,
      "activations/layer16_attention_weight_max": 47.015472412109375,
      "activations/layer16_attention_weight_min": -27.12013816833496,
      "activations/layer17_attention_weight_max": 65.01669311523438,
      "activations/layer17_attention_weight_min": -43.69245529174805,
      "activations/layer18_attention_weight_max": 62.76157760620117,
      "activations/layer18_attention_weight_min": -36.16239929199219,
      "activations/layer19_attention_weight_max": 25.078580856323242,
      "activations/layer19_attention_weight_min": -21.608173370361328,
      "activations/layer1_attention_weight_max": 16.986499786376953,
      "activations/layer1_attention_weight_min": -14.660662651062012,
      "activations/layer20_attention_weight_max": 29.686370849609375,
      "activations/layer20_attention_weight_min": -20.80408477783203,
      "activations/layer21_attention_weight_max": 61.50971603393555,
      "activations/layer21_attention_weight_min": -26.45958709716797,
      "activations/layer22_attention_weight_max": 38.02313995361328,
      "activations/layer22_attention_weight_min": -22.517990112304688,
      "activations/layer23_attention_weight_max": 35.735984802246094,
      "activations/layer23_attention_weight_min": -17.777856826782227,
      "activations/layer2_attention_weight_max": 33.92401123046875,
      "activations/layer2_attention_weight_min": -31.606342315673828,
      "activations/layer3_attention_weight_max": 84.60389709472656,
      "activations/layer3_attention_weight_min": -83.87677764892578,
      "activations/layer4_attention_weight_max": 109.60196685791016,
      "activations/layer4_attention_weight_min": -98.01168060302734,
      "activations/layer5_attention_weight_max": 104.9782943725586,
      "activations/layer5_attention_weight_min": -93.75611877441406,
      "activations/layer6_attention_weight_max": 55.79903030395508,
      "activations/layer6_attention_weight_min": -51.09579086303711,
      "activations/layer7_attention_weight_max": 78.95111083984375,
      "activations/layer7_attention_weight_min": -65.96217346191406,
      "activations/layer8_attention_weight_max": 47.449222564697266,
      "activations/layer8_attention_weight_min": -44.80955505371094,
      "activations/layer9_attention_weight_max": 65.98140716552734,
      "activations/layer9_attention_weight_min": -49.19761276245117,
      "epoch": 5.51,
      "learning_rate": 0.00011560378787878786,
      "loss": 2.8527,
      "step": 94900
    },
    {
      "activations/layer0_attention_weight_max": 14.253022193908691,
      "activations/layer0_attention_weight_min": -15.264824867248535,
      "activations/layer10_attention_weight_max": 39.6994514465332,
      "activations/layer10_attention_weight_min": -37.20469284057617,
      "activations/layer11_attention_weight_max": 28.406667709350586,
      "activations/layer11_attention_weight_min": -29.533597946166992,
      "activations/layer12_attention_weight_max": 22.688749313354492,
      "activations/layer12_attention_weight_min": -28.135072708129883,
      "activations/layer13_attention_weight_max": 49.50955581665039,
      "activations/layer13_attention_weight_min": -35.480194091796875,
      "activations/layer14_attention_weight_max": 51.656471252441406,
      "activations/layer14_attention_weight_min": -31.792449951171875,
      "activations/layer15_attention_weight_max": 39.315067291259766,
      "activations/layer15_attention_weight_min": -28.995342254638672,
      "activations/layer16_attention_weight_max": 36.39685821533203,
      "activations/layer16_attention_weight_min": -27.573015213012695,
      "activations/layer17_attention_weight_max": 57.45702362060547,
      "activations/layer17_attention_weight_min": -40.65520095825195,
      "activations/layer18_attention_weight_max": 47.34114456176758,
      "activations/layer18_attention_weight_min": -32.06621551513672,
      "activations/layer19_attention_weight_max": 22.565336227416992,
      "activations/layer19_attention_weight_min": -20.53038787841797,
      "activations/layer1_attention_weight_max": 18.11251449584961,
      "activations/layer1_attention_weight_min": -17.15517234802246,
      "activations/layer20_attention_weight_max": 26.373929977416992,
      "activations/layer20_attention_weight_min": -23.606468200683594,
      "activations/layer21_attention_weight_max": 43.609893798828125,
      "activations/layer21_attention_weight_min": -28.448558807373047,
      "activations/layer22_attention_weight_max": 32.728946685791016,
      "activations/layer22_attention_weight_min": -23.60674285888672,
      "activations/layer23_attention_weight_max": 29.143726348876953,
      "activations/layer23_attention_weight_min": -18.4802303314209,
      "activations/layer2_attention_weight_max": 31.14309310913086,
      "activations/layer2_attention_weight_min": -28.6453857421875,
      "activations/layer3_attention_weight_max": 88.35096740722656,
      "activations/layer3_attention_weight_min": -89.31482696533203,
      "activations/layer4_attention_weight_max": 103.16899871826172,
      "activations/layer4_attention_weight_min": -98.66716766357422,
      "activations/layer5_attention_weight_max": 102.98229217529297,
      "activations/layer5_attention_weight_min": -92.70903778076172,
      "activations/layer6_attention_weight_max": 60.873294830322266,
      "activations/layer6_attention_weight_min": -52.94386291503906,
      "activations/layer7_attention_weight_max": 74.69669342041016,
      "activations/layer7_attention_weight_min": -66.19677734375,
      "activations/layer8_attention_weight_max": 46.92348098754883,
      "activations/layer8_attention_weight_min": -44.250144958496094,
      "activations/layer9_attention_weight_max": 51.63115310668945,
      "activations/layer9_attention_weight_min": -48.70326614379883,
      "epoch": 5.52,
      "learning_rate": 0.00011558484848484847,
      "loss": 2.8309,
      "step": 94950
    },
    {
      "activations/layer0_attention_weight_max": 14.960089683532715,
      "activations/layer0_attention_weight_min": -14.814778327941895,
      "activations/layer10_attention_weight_max": 37.77605056762695,
      "activations/layer10_attention_weight_min": -33.14399337768555,
      "activations/layer11_attention_weight_max": 28.882701873779297,
      "activations/layer11_attention_weight_min": -25.804325103759766,
      "activations/layer12_attention_weight_max": 21.334598541259766,
      "activations/layer12_attention_weight_min": -22.72843360900879,
      "activations/layer13_attention_weight_max": 48.87944793701172,
      "activations/layer13_attention_weight_min": -35.21027374267578,
      "activations/layer14_attention_weight_max": 38.34565353393555,
      "activations/layer14_attention_weight_min": -29.231578826904297,
      "activations/layer15_attention_weight_max": 38.07231521606445,
      "activations/layer15_attention_weight_min": -30.3113956451416,
      "activations/layer16_attention_weight_max": 31.46409034729004,
      "activations/layer16_attention_weight_min": -27.296850204467773,
      "activations/layer17_attention_weight_max": 51.224647521972656,
      "activations/layer17_attention_weight_min": -38.268653869628906,
      "activations/layer18_attention_weight_max": 47.74357604980469,
      "activations/layer18_attention_weight_min": -34.24260330200195,
      "activations/layer19_attention_weight_max": 20.876789093017578,
      "activations/layer19_attention_weight_min": -17.248695373535156,
      "activations/layer1_attention_weight_max": 18.256309509277344,
      "activations/layer1_attention_weight_min": -17.089433670043945,
      "activations/layer20_attention_weight_max": 21.110761642456055,
      "activations/layer20_attention_weight_min": -19.698335647583008,
      "activations/layer21_attention_weight_max": 43.17744445800781,
      "activations/layer21_attention_weight_min": -24.32183074951172,
      "activations/layer22_attention_weight_max": 33.225643157958984,
      "activations/layer22_attention_weight_min": -23.418123245239258,
      "activations/layer23_attention_weight_max": 28.72195816040039,
      "activations/layer23_attention_weight_min": -19.250900268554688,
      "activations/layer2_attention_weight_max": 30.122777938842773,
      "activations/layer2_attention_weight_min": -28.34961700439453,
      "activations/layer3_attention_weight_max": 81.1527099609375,
      "activations/layer3_attention_weight_min": -84.70731353759766,
      "activations/layer4_attention_weight_max": 100.28639221191406,
      "activations/layer4_attention_weight_min": -93.87097930908203,
      "activations/layer5_attention_weight_max": 98.71345520019531,
      "activations/layer5_attention_weight_min": -91.09724426269531,
      "activations/layer6_attention_weight_max": 58.67357635498047,
      "activations/layer6_attention_weight_min": -52.3248405456543,
      "activations/layer7_attention_weight_max": 72.97949981689453,
      "activations/layer7_attention_weight_min": -66.02648162841797,
      "activations/layer8_attention_weight_max": 42.576358795166016,
      "activations/layer8_attention_weight_min": -40.344093322753906,
      "activations/layer9_attention_weight_max": 48.89873123168945,
      "activations/layer9_attention_weight_min": -44.83932113647461,
      "epoch": 5.52,
      "learning_rate": 0.00011556590909090909,
      "loss": 2.8433,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_loss": 2.794921875,
      "eval_runtime": 8.5126,
      "eval_samples_per_second": 504.431,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_openwebtext_loss": 2.794921875,
      "eval_openwebtext_ppl": 16.36135047271935,
      "eval_openwebtext_runtime": 8.5126,
      "eval_openwebtext_samples_per_second": 504.431,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_wikitext_loss": 3.03515625,
      "eval_wikitext_ppl": 20.8042283062039,
      "eval_wikitext_runtime": 2.0335,
      "eval_wikitext_samples_per_second": 224.248,
      "step": 95000
    },
    {
      "epoch": 5.52,
      "eval_lambada_loss": 2.76953125,
      "eval_lambada_ppl": 15.951155153095918,
      "eval_lambada_runtime": 9.6306,
      "eval_lambada_samples_per_second": 505.578,
      "step": 95000
    },
    {
      "activations/layer0_attention_weight_max": 14.780059814453125,
      "activations/layer0_attention_weight_min": -15.055240631103516,
      "activations/layer10_attention_weight_max": 36.9489631652832,
      "activations/layer10_attention_weight_min": -31.007755279541016,
      "activations/layer11_attention_weight_max": 27.78754425048828,
      "activations/layer11_attention_weight_min": -27.526859283447266,
      "activations/layer12_attention_weight_max": 18.628101348876953,
      "activations/layer12_attention_weight_min": -27.624897003173828,
      "activations/layer13_attention_weight_max": 47.70785140991211,
      "activations/layer13_attention_weight_min": -35.295135498046875,
      "activations/layer14_attention_weight_max": 40.73905563354492,
      "activations/layer14_attention_weight_min": -29.758037567138672,
      "activations/layer15_attention_weight_max": 37.82512283325195,
      "activations/layer15_attention_weight_min": -28.519573211669922,
      "activations/layer16_attention_weight_max": 29.678422927856445,
      "activations/layer16_attention_weight_min": -25.425161361694336,
      "activations/layer17_attention_weight_max": 50.68186950683594,
      "activations/layer17_attention_weight_min": -39.93740463256836,
      "activations/layer18_attention_weight_max": 45.64297866821289,
      "activations/layer18_attention_weight_min": -31.94859504699707,
      "activations/layer19_attention_weight_max": 20.564722061157227,
      "activations/layer19_attention_weight_min": -18.999059677124023,
      "activations/layer1_attention_weight_max": 17.43163299560547,
      "activations/layer1_attention_weight_min": -15.53488540649414,
      "activations/layer20_attention_weight_max": 21.012699127197266,
      "activations/layer20_attention_weight_min": -19.8941593170166,
      "activations/layer21_attention_weight_max": 33.770809173583984,
      "activations/layer21_attention_weight_min": -24.508665084838867,
      "activations/layer22_attention_weight_max": 28.83976936340332,
      "activations/layer22_attention_weight_min": -22.84200668334961,
      "activations/layer23_attention_weight_max": 25.527114868164062,
      "activations/layer23_attention_weight_min": -21.4593505859375,
      "activations/layer2_attention_weight_max": 31.746610641479492,
      "activations/layer2_attention_weight_min": -29.991283416748047,
      "activations/layer3_attention_weight_max": 79.61892700195312,
      "activations/layer3_attention_weight_min": -83.98008728027344,
      "activations/layer4_attention_weight_max": 100.8824462890625,
      "activations/layer4_attention_weight_min": -96.29243469238281,
      "activations/layer5_attention_weight_max": 101.24568176269531,
      "activations/layer5_attention_weight_min": -93.16637420654297,
      "activations/layer6_attention_weight_max": 59.81879425048828,
      "activations/layer6_attention_weight_min": -55.748374938964844,
      "activations/layer7_attention_weight_max": 73.35502624511719,
      "activations/layer7_attention_weight_min": -69.75337219238281,
      "activations/layer8_attention_weight_max": 43.64387893676758,
      "activations/layer8_attention_weight_min": -40.920074462890625,
      "activations/layer9_attention_weight_max": 47.3575439453125,
      "activations/layer9_attention_weight_min": -46.78364562988281,
      "epoch": 5.52,
      "learning_rate": 0.00011554696969696969,
      "loss": 2.8482,
      "step": 95050
    },
    {
      "activations/layer0_attention_weight_max": 15.190446853637695,
      "activations/layer0_attention_weight_min": -15.180070877075195,
      "activations/layer10_attention_weight_max": 57.32974624633789,
      "activations/layer10_attention_weight_min": -51.09651565551758,
      "activations/layer11_attention_weight_max": 47.76246643066406,
      "activations/layer11_attention_weight_min": -36.70526123046875,
      "activations/layer12_attention_weight_max": 21.636280059814453,
      "activations/layer12_attention_weight_min": -23.577329635620117,
      "activations/layer13_attention_weight_max": 93.76069641113281,
      "activations/layer13_attention_weight_min": -43.75807189941406,
      "activations/layer14_attention_weight_max": 113.14614868164062,
      "activations/layer14_attention_weight_min": -41.552574157714844,
      "activations/layer15_attention_weight_max": 76.15619659423828,
      "activations/layer15_attention_weight_min": -32.780540466308594,
      "activations/layer16_attention_weight_max": 44.2445182800293,
      "activations/layer16_attention_weight_min": -31.42472267150879,
      "activations/layer17_attention_weight_max": 67.2272720336914,
      "activations/layer17_attention_weight_min": -49.640830993652344,
      "activations/layer18_attention_weight_max": 55.7205924987793,
      "activations/layer18_attention_weight_min": -39.263938903808594,
      "activations/layer19_attention_weight_max": 23.03120231628418,
      "activations/layer19_attention_weight_min": -23.544082641601562,
      "activations/layer1_attention_weight_max": 19.23155403137207,
      "activations/layer1_attention_weight_min": -15.845948219299316,
      "activations/layer20_attention_weight_max": 26.368778228759766,
      "activations/layer20_attention_weight_min": -23.752410888671875,
      "activations/layer21_attention_weight_max": 46.710784912109375,
      "activations/layer21_attention_weight_min": -23.969493865966797,
      "activations/layer22_attention_weight_max": 35.2916145324707,
      "activations/layer22_attention_weight_min": -22.3725528717041,
      "activations/layer23_attention_weight_max": 29.968158721923828,
      "activations/layer23_attention_weight_min": -19.37359619140625,
      "activations/layer2_attention_weight_max": 37.092742919921875,
      "activations/layer2_attention_weight_min": -33.041786193847656,
      "activations/layer3_attention_weight_max": 103.41804504394531,
      "activations/layer3_attention_weight_min": -100.5034408569336,
      "activations/layer4_attention_weight_max": 124.38996887207031,
      "activations/layer4_attention_weight_min": -114.49344635009766,
      "activations/layer5_attention_weight_max": 124.94792175292969,
      "activations/layer5_attention_weight_min": -111.3725814819336,
      "activations/layer6_attention_weight_max": 74.82318115234375,
      "activations/layer6_attention_weight_min": -63.922142028808594,
      "activations/layer7_attention_weight_max": 102.74346923828125,
      "activations/layer7_attention_weight_min": -84.0759048461914,
      "activations/layer8_attention_weight_max": 63.23167037963867,
      "activations/layer8_attention_weight_min": -53.877803802490234,
      "activations/layer9_attention_weight_max": 83.9720230102539,
      "activations/layer9_attention_weight_min": -60.1296272277832,
      "epoch": 5.53,
      "learning_rate": 0.00011552803030303029,
      "loss": 2.8322,
      "step": 95100
    },
    {
      "activations/layer0_attention_weight_max": 16.02515983581543,
      "activations/layer0_attention_weight_min": -15.32471752166748,
      "activations/layer10_attention_weight_max": 41.165374755859375,
      "activations/layer10_attention_weight_min": -35.76583480834961,
      "activations/layer11_attention_weight_max": 33.8150634765625,
      "activations/layer11_attention_weight_min": -28.18043327331543,
      "activations/layer12_attention_weight_max": 26.95977210998535,
      "activations/layer12_attention_weight_min": -27.03411293029785,
      "activations/layer13_attention_weight_max": 69.76231384277344,
      "activations/layer13_attention_weight_min": -39.14826965332031,
      "activations/layer14_attention_weight_max": 63.55799865722656,
      "activations/layer14_attention_weight_min": -32.82234573364258,
      "activations/layer15_attention_weight_max": 55.62709045410156,
      "activations/layer15_attention_weight_min": -31.532623291015625,
      "activations/layer16_attention_weight_max": 41.721580505371094,
      "activations/layer16_attention_weight_min": -27.594247817993164,
      "activations/layer17_attention_weight_max": 63.96216583251953,
      "activations/layer17_attention_weight_min": -44.72264099121094,
      "activations/layer18_attention_weight_max": 60.06016159057617,
      "activations/layer18_attention_weight_min": -38.09278106689453,
      "activations/layer19_attention_weight_max": 24.1524715423584,
      "activations/layer19_attention_weight_min": -21.838720321655273,
      "activations/layer1_attention_weight_max": 17.922605514526367,
      "activations/layer1_attention_weight_min": -18.315641403198242,
      "activations/layer20_attention_weight_max": 24.57708740234375,
      "activations/layer20_attention_weight_min": -21.360042572021484,
      "activations/layer21_attention_weight_max": 46.0052604675293,
      "activations/layer21_attention_weight_min": -26.246843338012695,
      "activations/layer22_attention_weight_max": 42.37530517578125,
      "activations/layer22_attention_weight_min": -27.939544677734375,
      "activations/layer23_attention_weight_max": 38.78722381591797,
      "activations/layer23_attention_weight_min": -19.000110626220703,
      "activations/layer2_attention_weight_max": 31.325294494628906,
      "activations/layer2_attention_weight_min": -29.55928611755371,
      "activations/layer3_attention_weight_max": 83.85062408447266,
      "activations/layer3_attention_weight_min": -85.23743438720703,
      "activations/layer4_attention_weight_max": 105.01570892333984,
      "activations/layer4_attention_weight_min": -94.86968231201172,
      "activations/layer5_attention_weight_max": 103.0203857421875,
      "activations/layer5_attention_weight_min": -92.3539047241211,
      "activations/layer6_attention_weight_max": 61.62442398071289,
      "activations/layer6_attention_weight_min": -52.62759780883789,
      "activations/layer7_attention_weight_max": 71.56964111328125,
      "activations/layer7_attention_weight_min": -76.44279479980469,
      "activations/layer8_attention_weight_max": 45.60087203979492,
      "activations/layer8_attention_weight_min": -46.24382400512695,
      "activations/layer9_attention_weight_max": 56.8651123046875,
      "activations/layer9_attention_weight_min": -51.22193145751953,
      "epoch": 5.53,
      "learning_rate": 0.0001155090909090909,
      "loss": 2.8492,
      "step": 95150
    },
    {
      "activations/layer0_attention_weight_max": 14.363831520080566,
      "activations/layer0_attention_weight_min": -14.922087669372559,
      "activations/layer10_attention_weight_max": 36.46752166748047,
      "activations/layer10_attention_weight_min": -34.06072235107422,
      "activations/layer11_attention_weight_max": 29.17392349243164,
      "activations/layer11_attention_weight_min": -27.240219116210938,
      "activations/layer12_attention_weight_max": 25.833463668823242,
      "activations/layer12_attention_weight_min": -26.199167251586914,
      "activations/layer13_attention_weight_max": 44.63869857788086,
      "activations/layer13_attention_weight_min": -32.98342514038086,
      "activations/layer14_attention_weight_max": 42.535247802734375,
      "activations/layer14_attention_weight_min": -30.667770385742188,
      "activations/layer15_attention_weight_max": 40.11433029174805,
      "activations/layer15_attention_weight_min": -32.97865676879883,
      "activations/layer16_attention_weight_max": 33.15614700317383,
      "activations/layer16_attention_weight_min": -28.30127716064453,
      "activations/layer17_attention_weight_max": 58.44586944580078,
      "activations/layer17_attention_weight_min": -40.52850341796875,
      "activations/layer18_attention_weight_max": 50.918270111083984,
      "activations/layer18_attention_weight_min": -37.95926284790039,
      "activations/layer19_attention_weight_max": 24.46251678466797,
      "activations/layer19_attention_weight_min": -21.276426315307617,
      "activations/layer1_attention_weight_max": 17.09406089782715,
      "activations/layer1_attention_weight_min": -16.941513061523438,
      "activations/layer20_attention_weight_max": 23.456722259521484,
      "activations/layer20_attention_weight_min": -22.528615951538086,
      "activations/layer21_attention_weight_max": 39.40715789794922,
      "activations/layer21_attention_weight_min": -26.078622817993164,
      "activations/layer22_attention_weight_max": 36.33509063720703,
      "activations/layer22_attention_weight_min": -23.61111068725586,
      "activations/layer23_attention_weight_max": 28.772245407104492,
      "activations/layer23_attention_weight_min": -19.157039642333984,
      "activations/layer2_attention_weight_max": 29.392288208007812,
      "activations/layer2_attention_weight_min": -27.77657699584961,
      "activations/layer3_attention_weight_max": 76.53685760498047,
      "activations/layer3_attention_weight_min": -81.56449127197266,
      "activations/layer4_attention_weight_max": 96.62934112548828,
      "activations/layer4_attention_weight_min": -91.4399185180664,
      "activations/layer5_attention_weight_max": 95.1380844116211,
      "activations/layer5_attention_weight_min": -89.169189453125,
      "activations/layer6_attention_weight_max": 54.863685607910156,
      "activations/layer6_attention_weight_min": -52.74232482910156,
      "activations/layer7_attention_weight_max": 67.24529266357422,
      "activations/layer7_attention_weight_min": -68.48735809326172,
      "activations/layer8_attention_weight_max": 42.04170608520508,
      "activations/layer8_attention_weight_min": -41.44468688964844,
      "activations/layer9_attention_weight_max": 48.1424674987793,
      "activations/layer9_attention_weight_min": -47.13132858276367,
      "epoch": 5.53,
      "learning_rate": 0.0001154905303030303,
      "loss": 2.8578,
      "step": 95200
    },
    {
      "activations/layer0_attention_weight_max": 14.440686225891113,
      "activations/layer0_attention_weight_min": -15.244050979614258,
      "activations/layer10_attention_weight_max": 39.70399856567383,
      "activations/layer10_attention_weight_min": -40.97730255126953,
      "activations/layer11_attention_weight_max": 34.68379211425781,
      "activations/layer11_attention_weight_min": -30.185619354248047,
      "activations/layer12_attention_weight_max": 27.470951080322266,
      "activations/layer12_attention_weight_min": -23.91448211669922,
      "activations/layer13_attention_weight_max": 45.89850997924805,
      "activations/layer13_attention_weight_min": -38.215362548828125,
      "activations/layer14_attention_weight_max": 70.05242919921875,
      "activations/layer14_attention_weight_min": -38.62264633178711,
      "activations/layer15_attention_weight_max": 51.53675842285156,
      "activations/layer15_attention_weight_min": -31.634687423706055,
      "activations/layer16_attention_weight_max": 37.962589263916016,
      "activations/layer16_attention_weight_min": -29.398563385009766,
      "activations/layer17_attention_weight_max": 64.87979888916016,
      "activations/layer17_attention_weight_min": -47.57054901123047,
      "activations/layer18_attention_weight_max": 48.7070198059082,
      "activations/layer18_attention_weight_min": -38.729496002197266,
      "activations/layer19_attention_weight_max": 24.25043296813965,
      "activations/layer19_attention_weight_min": -19.928329467773438,
      "activations/layer1_attention_weight_max": 16.748994827270508,
      "activations/layer1_attention_weight_min": -15.624541282653809,
      "activations/layer20_attention_weight_max": 24.72137451171875,
      "activations/layer20_attention_weight_min": -21.344911575317383,
      "activations/layer21_attention_weight_max": 41.77573776245117,
      "activations/layer21_attention_weight_min": -26.05141258239746,
      "activations/layer22_attention_weight_max": 35.43208312988281,
      "activations/layer22_attention_weight_min": -24.319286346435547,
      "activations/layer23_attention_weight_max": 27.773815155029297,
      "activations/layer23_attention_weight_min": -19.001285552978516,
      "activations/layer2_attention_weight_max": 29.03919792175293,
      "activations/layer2_attention_weight_min": -28.28443145751953,
      "activations/layer3_attention_weight_max": 81.23942565917969,
      "activations/layer3_attention_weight_min": -86.62603759765625,
      "activations/layer4_attention_weight_max": 105.23365783691406,
      "activations/layer4_attention_weight_min": -99.55047607421875,
      "activations/layer5_attention_weight_max": 96.78102111816406,
      "activations/layer5_attention_weight_min": -90.65863037109375,
      "activations/layer6_attention_weight_max": 58.7927360534668,
      "activations/layer6_attention_weight_min": -53.81081008911133,
      "activations/layer7_attention_weight_max": 81.0549087524414,
      "activations/layer7_attention_weight_min": -68.29859924316406,
      "activations/layer8_attention_weight_max": 50.11143493652344,
      "activations/layer8_attention_weight_min": -41.15045166015625,
      "activations/layer9_attention_weight_max": 70.9571762084961,
      "activations/layer9_attention_weight_min": -54.02900314331055,
      "epoch": 5.53,
      "learning_rate": 0.0001154715909090909,
      "loss": 2.8443,
      "step": 95250
    },
    {
      "activations/layer0_attention_weight_max": 16.182029724121094,
      "activations/layer0_attention_weight_min": -14.685081481933594,
      "activations/layer10_attention_weight_max": 40.314998626708984,
      "activations/layer10_attention_weight_min": -36.131107330322266,
      "activations/layer11_attention_weight_max": 31.056903839111328,
      "activations/layer11_attention_weight_min": -29.143455505371094,
      "activations/layer12_attention_weight_max": 30.140308380126953,
      "activations/layer12_attention_weight_min": -26.104276657104492,
      "activations/layer13_attention_weight_max": 50.53322982788086,
      "activations/layer13_attention_weight_min": -35.4666748046875,
      "activations/layer14_attention_weight_max": 46.09025192260742,
      "activations/layer14_attention_weight_min": -30.960710525512695,
      "activations/layer15_attention_weight_max": 43.97404098510742,
      "activations/layer15_attention_weight_min": -31.69168472290039,
      "activations/layer16_attention_weight_max": 32.82452392578125,
      "activations/layer16_attention_weight_min": -26.536537170410156,
      "activations/layer17_attention_weight_max": 59.78118896484375,
      "activations/layer17_attention_weight_min": -40.34236526489258,
      "activations/layer18_attention_weight_max": 50.08045196533203,
      "activations/layer18_attention_weight_min": -36.167457580566406,
      "activations/layer19_attention_weight_max": 23.99956703186035,
      "activations/layer19_attention_weight_min": -19.868083953857422,
      "activations/layer1_attention_weight_max": 18.030858993530273,
      "activations/layer1_attention_weight_min": -15.400096893310547,
      "activations/layer20_attention_weight_max": 24.185199737548828,
      "activations/layer20_attention_weight_min": -21.45207977294922,
      "activations/layer21_attention_weight_max": 40.94422149658203,
      "activations/layer21_attention_weight_min": -26.719802856445312,
      "activations/layer22_attention_weight_max": 38.13776779174805,
      "activations/layer22_attention_weight_min": -25.47212791442871,
      "activations/layer23_attention_weight_max": 32.07046127319336,
      "activations/layer23_attention_weight_min": -19.43987274169922,
      "activations/layer2_attention_weight_max": 30.877777099609375,
      "activations/layer2_attention_weight_min": -28.769994735717773,
      "activations/layer3_attention_weight_max": 81.89228057861328,
      "activations/layer3_attention_weight_min": -80.15827941894531,
      "activations/layer4_attention_weight_max": 100.7873764038086,
      "activations/layer4_attention_weight_min": -95.0101547241211,
      "activations/layer5_attention_weight_max": 98.95770263671875,
      "activations/layer5_attention_weight_min": -92.59944152832031,
      "activations/layer6_attention_weight_max": 56.38322067260742,
      "activations/layer6_attention_weight_min": -54.301578521728516,
      "activations/layer7_attention_weight_max": 71.54938507080078,
      "activations/layer7_attention_weight_min": -65.61112976074219,
      "activations/layer8_attention_weight_max": 43.59966278076172,
      "activations/layer8_attention_weight_min": -42.9234733581543,
      "activations/layer9_attention_weight_max": 51.321510314941406,
      "activations/layer9_attention_weight_min": -50.555355072021484,
      "epoch": 5.54,
      "learning_rate": 0.00011545265151515151,
      "loss": 2.8407,
      "step": 95300
    },
    {
      "activations/layer0_attention_weight_max": 14.483770370483398,
      "activations/layer0_attention_weight_min": -14.921793937683105,
      "activations/layer10_attention_weight_max": 36.82108688354492,
      "activations/layer10_attention_weight_min": -35.545379638671875,
      "activations/layer11_attention_weight_max": 29.351585388183594,
      "activations/layer11_attention_weight_min": -28.12741470336914,
      "activations/layer12_attention_weight_max": 20.58393096923828,
      "activations/layer12_attention_weight_min": -24.241708755493164,
      "activations/layer13_attention_weight_max": 52.289241790771484,
      "activations/layer13_attention_weight_min": -38.38112258911133,
      "activations/layer14_attention_weight_max": 41.928367614746094,
      "activations/layer14_attention_weight_min": -29.59229278564453,
      "activations/layer15_attention_weight_max": 43.34263229370117,
      "activations/layer15_attention_weight_min": -30.165769577026367,
      "activations/layer16_attention_weight_max": 35.124210357666016,
      "activations/layer16_attention_weight_min": -27.431745529174805,
      "activations/layer17_attention_weight_max": 58.36288070678711,
      "activations/layer17_attention_weight_min": -39.52193832397461,
      "activations/layer18_attention_weight_max": 46.751895904541016,
      "activations/layer18_attention_weight_min": -34.8385124206543,
      "activations/layer19_attention_weight_max": 22.636762619018555,
      "activations/layer19_attention_weight_min": -17.805891036987305,
      "activations/layer1_attention_weight_max": 16.802709579467773,
      "activations/layer1_attention_weight_min": -15.209796905517578,
      "activations/layer20_attention_weight_max": 21.370298385620117,
      "activations/layer20_attention_weight_min": -21.96123504638672,
      "activations/layer21_attention_weight_max": 37.05829620361328,
      "activations/layer21_attention_weight_min": -26.87230110168457,
      "activations/layer22_attention_weight_max": 37.28047180175781,
      "activations/layer22_attention_weight_min": -23.192773818969727,
      "activations/layer23_attention_weight_max": 26.071990966796875,
      "activations/layer23_attention_weight_min": -16.832962036132812,
      "activations/layer2_attention_weight_max": 31.211029052734375,
      "activations/layer2_attention_weight_min": -28.422863006591797,
      "activations/layer3_attention_weight_max": 81.3060073852539,
      "activations/layer3_attention_weight_min": -82.0505142211914,
      "activations/layer4_attention_weight_max": 97.5525894165039,
      "activations/layer4_attention_weight_min": -96.97738647460938,
      "activations/layer5_attention_weight_max": 96.17549133300781,
      "activations/layer5_attention_weight_min": -91.27086639404297,
      "activations/layer6_attention_weight_max": 56.7230224609375,
      "activations/layer6_attention_weight_min": -52.787086486816406,
      "activations/layer7_attention_weight_max": 75.25656127929688,
      "activations/layer7_attention_weight_min": -70.56011199951172,
      "activations/layer8_attention_weight_max": 43.22209167480469,
      "activations/layer8_attention_weight_min": -41.77755355834961,
      "activations/layer9_attention_weight_max": 56.6744384765625,
      "activations/layer9_attention_weight_min": -45.12760925292969,
      "epoch": 5.54,
      "learning_rate": 0.00011543371212121211,
      "loss": 2.8388,
      "step": 95350
    },
    {
      "activations/layer0_attention_weight_max": 14.255826950073242,
      "activations/layer0_attention_weight_min": -14.545989036560059,
      "activations/layer10_attention_weight_max": 37.096588134765625,
      "activations/layer10_attention_weight_min": -34.01375961303711,
      "activations/layer11_attention_weight_max": 27.801368713378906,
      "activations/layer11_attention_weight_min": -27.84769058227539,
      "activations/layer12_attention_weight_max": 20.04716682434082,
      "activations/layer12_attention_weight_min": -23.661264419555664,
      "activations/layer13_attention_weight_max": 59.56333541870117,
      "activations/layer13_attention_weight_min": -37.812530517578125,
      "activations/layer14_attention_weight_max": 53.14352035522461,
      "activations/layer14_attention_weight_min": -31.329544067382812,
      "activations/layer15_attention_weight_max": 42.53812026977539,
      "activations/layer15_attention_weight_min": -30.15013313293457,
      "activations/layer16_attention_weight_max": 35.63797378540039,
      "activations/layer16_attention_weight_min": -29.10567855834961,
      "activations/layer17_attention_weight_max": 57.55883026123047,
      "activations/layer17_attention_weight_min": -39.1759033203125,
      "activations/layer18_attention_weight_max": 49.69005584716797,
      "activations/layer18_attention_weight_min": -34.20918273925781,
      "activations/layer19_attention_weight_max": 25.849014282226562,
      "activations/layer19_attention_weight_min": -19.512561798095703,
      "activations/layer1_attention_weight_max": 16.648935317993164,
      "activations/layer1_attention_weight_min": -15.074507713317871,
      "activations/layer20_attention_weight_max": 21.480064392089844,
      "activations/layer20_attention_weight_min": -20.457157135009766,
      "activations/layer21_attention_weight_max": 45.4997673034668,
      "activations/layer21_attention_weight_min": -26.155418395996094,
      "activations/layer22_attention_weight_max": 34.72681427001953,
      "activations/layer22_attention_weight_min": -25.532625198364258,
      "activations/layer23_attention_weight_max": 31.190505981445312,
      "activations/layer23_attention_weight_min": -17.74118423461914,
      "activations/layer2_attention_weight_max": 29.608402252197266,
      "activations/layer2_attention_weight_min": -28.803075790405273,
      "activations/layer3_attention_weight_max": 81.4775390625,
      "activations/layer3_attention_weight_min": -88.26728057861328,
      "activations/layer4_attention_weight_max": 101.29313659667969,
      "activations/layer4_attention_weight_min": -99.84794616699219,
      "activations/layer5_attention_weight_max": 96.81097412109375,
      "activations/layer5_attention_weight_min": -87.75634765625,
      "activations/layer6_attention_weight_max": 55.79025650024414,
      "activations/layer6_attention_weight_min": -52.85060119628906,
      "activations/layer7_attention_weight_max": 83.27352142333984,
      "activations/layer7_attention_weight_min": -68.15652465820312,
      "activations/layer8_attention_weight_max": 44.7602424621582,
      "activations/layer8_attention_weight_min": -43.31144332885742,
      "activations/layer9_attention_weight_max": 53.10852813720703,
      "activations/layer9_attention_weight_min": -47.94007110595703,
      "epoch": 5.54,
      "learning_rate": 0.00011541477272727272,
      "loss": 2.8381,
      "step": 95400
    },
    {
      "activations/layer0_attention_weight_max": 14.700740814208984,
      "activations/layer0_attention_weight_min": -14.80311107635498,
      "activations/layer10_attention_weight_max": 39.13937759399414,
      "activations/layer10_attention_weight_min": -32.77699661254883,
      "activations/layer11_attention_weight_max": 28.6471004486084,
      "activations/layer11_attention_weight_min": -27.890235900878906,
      "activations/layer12_attention_weight_max": 26.12494468688965,
      "activations/layer12_attention_weight_min": -24.534521102905273,
      "activations/layer13_attention_weight_max": 48.4564094543457,
      "activations/layer13_attention_weight_min": -34.566707611083984,
      "activations/layer14_attention_weight_max": 43.49971389770508,
      "activations/layer14_attention_weight_min": -31.869312286376953,
      "activations/layer15_attention_weight_max": 37.89512252807617,
      "activations/layer15_attention_weight_min": -31.066770553588867,
      "activations/layer16_attention_weight_max": 31.178178787231445,
      "activations/layer16_attention_weight_min": -25.388547897338867,
      "activations/layer17_attention_weight_max": 52.77286148071289,
      "activations/layer17_attention_weight_min": -36.900386810302734,
      "activations/layer18_attention_weight_max": 46.94460678100586,
      "activations/layer18_attention_weight_min": -31.988374710083008,
      "activations/layer19_attention_weight_max": 26.85878562927246,
      "activations/layer19_attention_weight_min": -19.86702537536621,
      "activations/layer1_attention_weight_max": 16.70170021057129,
      "activations/layer1_attention_weight_min": -15.816926002502441,
      "activations/layer20_attention_weight_max": 20.89391326904297,
      "activations/layer20_attention_weight_min": -21.61668586730957,
      "activations/layer21_attention_weight_max": 44.55586624145508,
      "activations/layer21_attention_weight_min": -25.8050479888916,
      "activations/layer22_attention_weight_max": 33.3571662902832,
      "activations/layer22_attention_weight_min": -24.071252822875977,
      "activations/layer23_attention_weight_max": 31.013784408569336,
      "activations/layer23_attention_weight_min": -18.544845581054688,
      "activations/layer2_attention_weight_max": 31.25519371032715,
      "activations/layer2_attention_weight_min": -28.336339950561523,
      "activations/layer3_attention_weight_max": 79.61061096191406,
      "activations/layer3_attention_weight_min": -81.46897888183594,
      "activations/layer4_attention_weight_max": 98.42931365966797,
      "activations/layer4_attention_weight_min": -94.77972412109375,
      "activations/layer5_attention_weight_max": 94.93987274169922,
      "activations/layer5_attention_weight_min": -85.28296661376953,
      "activations/layer6_attention_weight_max": 58.26390075683594,
      "activations/layer6_attention_weight_min": -54.36537551879883,
      "activations/layer7_attention_weight_max": 72.15069580078125,
      "activations/layer7_attention_weight_min": -68.26620483398438,
      "activations/layer8_attention_weight_max": 42.57514572143555,
      "activations/layer8_attention_weight_min": -40.322242736816406,
      "activations/layer9_attention_weight_max": 53.6314697265625,
      "activations/layer9_attention_weight_min": -44.311058044433594,
      "epoch": 5.55,
      "learning_rate": 0.00011539583333333333,
      "loss": 2.8393,
      "step": 95450
    },
    {
      "activations/layer0_attention_weight_max": 15.223556518554688,
      "activations/layer0_attention_weight_min": -14.668980598449707,
      "activations/layer10_attention_weight_max": 44.81528091430664,
      "activations/layer10_attention_weight_min": -35.86061096191406,
      "activations/layer11_attention_weight_max": 35.64052200317383,
      "activations/layer11_attention_weight_min": -28.060529708862305,
      "activations/layer12_attention_weight_max": 24.009056091308594,
      "activations/layer12_attention_weight_min": -25.75816535949707,
      "activations/layer13_attention_weight_max": 53.08506774902344,
      "activations/layer13_attention_weight_min": -34.551204681396484,
      "activations/layer14_attention_weight_max": 61.46607971191406,
      "activations/layer14_attention_weight_min": -35.378868103027344,
      "activations/layer15_attention_weight_max": 43.30889129638672,
      "activations/layer15_attention_weight_min": -29.11908531188965,
      "activations/layer16_attention_weight_max": 32.05928421020508,
      "activations/layer16_attention_weight_min": -24.699600219726562,
      "activations/layer17_attention_weight_max": 57.47686767578125,
      "activations/layer17_attention_weight_min": -41.97597885131836,
      "activations/layer18_attention_weight_max": 46.52045822143555,
      "activations/layer18_attention_weight_min": -34.87810516357422,
      "activations/layer19_attention_weight_max": 22.666908264160156,
      "activations/layer19_attention_weight_min": -20.923168182373047,
      "activations/layer1_attention_weight_max": 18.372661590576172,
      "activations/layer1_attention_weight_min": -20.054014205932617,
      "activations/layer20_attention_weight_max": 21.4093017578125,
      "activations/layer20_attention_weight_min": -21.241497039794922,
      "activations/layer21_attention_weight_max": 38.12749099731445,
      "activations/layer21_attention_weight_min": -24.996843338012695,
      "activations/layer22_attention_weight_max": 32.948387145996094,
      "activations/layer22_attention_weight_min": -23.24041175842285,
      "activations/layer23_attention_weight_max": 25.605491638183594,
      "activations/layer23_attention_weight_min": -19.072219848632812,
      "activations/layer2_attention_weight_max": 30.363479614257812,
      "activations/layer2_attention_weight_min": -28.208189010620117,
      "activations/layer3_attention_weight_max": 80.55630493164062,
      "activations/layer3_attention_weight_min": -81.08163452148438,
      "activations/layer4_attention_weight_max": 99.64247131347656,
      "activations/layer4_attention_weight_min": -95.69417572021484,
      "activations/layer5_attention_weight_max": 99.95455932617188,
      "activations/layer5_attention_weight_min": -88.68453979492188,
      "activations/layer6_attention_weight_max": 60.80488586425781,
      "activations/layer6_attention_weight_min": -53.29029846191406,
      "activations/layer7_attention_weight_max": 80.83572387695312,
      "activations/layer7_attention_weight_min": -72.40604400634766,
      "activations/layer8_attention_weight_max": 52.00682067871094,
      "activations/layer8_attention_weight_min": -46.09367370605469,
      "activations/layer9_attention_weight_max": 66.59928131103516,
      "activations/layer9_attention_weight_min": -53.439266204833984,
      "epoch": 5.55,
      "learning_rate": 0.00011537689393939392,
      "loss": 2.8351,
      "step": 95500
    },
    {
      "activations/layer0_attention_weight_max": 15.358667373657227,
      "activations/layer0_attention_weight_min": -14.82098388671875,
      "activations/layer10_attention_weight_max": 35.924896240234375,
      "activations/layer10_attention_weight_min": -35.385101318359375,
      "activations/layer11_attention_weight_max": 29.644607543945312,
      "activations/layer11_attention_weight_min": -28.30927848815918,
      "activations/layer12_attention_weight_max": 24.253141403198242,
      "activations/layer12_attention_weight_min": -26.687990188598633,
      "activations/layer13_attention_weight_max": 43.47627258300781,
      "activations/layer13_attention_weight_min": -34.07014465332031,
      "activations/layer14_attention_weight_max": 40.042877197265625,
      "activations/layer14_attention_weight_min": -31.065088272094727,
      "activations/layer15_attention_weight_max": 42.00326156616211,
      "activations/layer15_attention_weight_min": -31.349185943603516,
      "activations/layer16_attention_weight_max": 37.43942642211914,
      "activations/layer16_attention_weight_min": -28.528194427490234,
      "activations/layer17_attention_weight_max": 59.38096237182617,
      "activations/layer17_attention_weight_min": -41.326271057128906,
      "activations/layer18_attention_weight_max": 52.71813201904297,
      "activations/layer18_attention_weight_min": -37.246551513671875,
      "activations/layer19_attention_weight_max": 23.099227905273438,
      "activations/layer19_attention_weight_min": -20.32769775390625,
      "activations/layer1_attention_weight_max": 17.24742317199707,
      "activations/layer1_attention_weight_min": -14.741929054260254,
      "activations/layer20_attention_weight_max": 23.90191078186035,
      "activations/layer20_attention_weight_min": -24.088237762451172,
      "activations/layer21_attention_weight_max": 44.4854621887207,
      "activations/layer21_attention_weight_min": -25.103843688964844,
      "activations/layer22_attention_weight_max": 35.56466293334961,
      "activations/layer22_attention_weight_min": -28.9466552734375,
      "activations/layer23_attention_weight_max": 30.85091209411621,
      "activations/layer23_attention_weight_min": -18.652587890625,
      "activations/layer2_attention_weight_max": 31.063175201416016,
      "activations/layer2_attention_weight_min": -28.869108200073242,
      "activations/layer3_attention_weight_max": 80.84233856201172,
      "activations/layer3_attention_weight_min": -82.05641174316406,
      "activations/layer4_attention_weight_max": 102.69219207763672,
      "activations/layer4_attention_weight_min": -93.90657806396484,
      "activations/layer5_attention_weight_max": 97.31660461425781,
      "activations/layer5_attention_weight_min": -86.74130249023438,
      "activations/layer6_attention_weight_max": 57.97746658325195,
      "activations/layer6_attention_weight_min": -53.39628219604492,
      "activations/layer7_attention_weight_max": 74.78730010986328,
      "activations/layer7_attention_weight_min": -70.46076965332031,
      "activations/layer8_attention_weight_max": 43.78036117553711,
      "activations/layer8_attention_weight_min": -40.957481384277344,
      "activations/layer9_attention_weight_max": 46.9471549987793,
      "activations/layer9_attention_weight_min": -48.8742561340332,
      "epoch": 5.55,
      "learning_rate": 0.00011535795454545453,
      "loss": 2.8516,
      "step": 95550
    },
    {
      "activations/layer0_attention_weight_max": 14.97923755645752,
      "activations/layer0_attention_weight_min": -14.833451271057129,
      "activations/layer10_attention_weight_max": 38.35476303100586,
      "activations/layer10_attention_weight_min": -38.76628112792969,
      "activations/layer11_attention_weight_max": 29.84508514404297,
      "activations/layer11_attention_weight_min": -28.301502227783203,
      "activations/layer12_attention_weight_max": 19.995723724365234,
      "activations/layer12_attention_weight_min": -25.5052547454834,
      "activations/layer13_attention_weight_max": 49.146202087402344,
      "activations/layer13_attention_weight_min": -37.89502716064453,
      "activations/layer14_attention_weight_max": 37.378936767578125,
      "activations/layer14_attention_weight_min": -30.004615783691406,
      "activations/layer15_attention_weight_max": 40.851905822753906,
      "activations/layer15_attention_weight_min": -30.837678909301758,
      "activations/layer16_attention_weight_max": 34.58578872680664,
      "activations/layer16_attention_weight_min": -26.303422927856445,
      "activations/layer17_attention_weight_max": 54.13786697387695,
      "activations/layer17_attention_weight_min": -39.31216812133789,
      "activations/layer18_attention_weight_max": 46.160179138183594,
      "activations/layer18_attention_weight_min": -34.552799224853516,
      "activations/layer19_attention_weight_max": 22.259441375732422,
      "activations/layer19_attention_weight_min": -19.427005767822266,
      "activations/layer1_attention_weight_max": 17.392135620117188,
      "activations/layer1_attention_weight_min": -17.201618194580078,
      "activations/layer20_attention_weight_max": 22.132909774780273,
      "activations/layer20_attention_weight_min": -23.988561630249023,
      "activations/layer21_attention_weight_max": 39.79716873168945,
      "activations/layer21_attention_weight_min": -25.446208953857422,
      "activations/layer22_attention_weight_max": 34.75559616088867,
      "activations/layer22_attention_weight_min": -26.732337951660156,
      "activations/layer23_attention_weight_max": 29.39928436279297,
      "activations/layer23_attention_weight_min": -19.812376022338867,
      "activations/layer2_attention_weight_max": 30.199405670166016,
      "activations/layer2_attention_weight_min": -31.220746994018555,
      "activations/layer3_attention_weight_max": 80.40665435791016,
      "activations/layer3_attention_weight_min": -84.40481567382812,
      "activations/layer4_attention_weight_max": 99.17255401611328,
      "activations/layer4_attention_weight_min": -97.3075180053711,
      "activations/layer5_attention_weight_max": 101.96215057373047,
      "activations/layer5_attention_weight_min": -94.13424682617188,
      "activations/layer6_attention_weight_max": 59.69483184814453,
      "activations/layer6_attention_weight_min": -57.77265930175781,
      "activations/layer7_attention_weight_max": 75.73226928710938,
      "activations/layer7_attention_weight_min": -69.50836944580078,
      "activations/layer8_attention_weight_max": 46.51177215576172,
      "activations/layer8_attention_weight_min": -45.55186080932617,
      "activations/layer9_attention_weight_max": 51.61152267456055,
      "activations/layer9_attention_weight_min": -51.14353561401367,
      "epoch": 5.55,
      "learning_rate": 0.00011533901515151515,
      "loss": 2.8351,
      "step": 95600
    },
    {
      "activations/layer0_attention_weight_max": 15.42910385131836,
      "activations/layer0_attention_weight_min": -14.861236572265625,
      "activations/layer10_attention_weight_max": 37.98951721191406,
      "activations/layer10_attention_weight_min": -35.59942626953125,
      "activations/layer11_attention_weight_max": 29.384010314941406,
      "activations/layer11_attention_weight_min": -29.012710571289062,
      "activations/layer12_attention_weight_max": 22.089595794677734,
      "activations/layer12_attention_weight_min": -24.249860763549805,
      "activations/layer13_attention_weight_max": 52.721824645996094,
      "activations/layer13_attention_weight_min": -39.50516891479492,
      "activations/layer14_attention_weight_max": 43.5504035949707,
      "activations/layer14_attention_weight_min": -34.2711296081543,
      "activations/layer15_attention_weight_max": 38.4808349609375,
      "activations/layer15_attention_weight_min": -29.696800231933594,
      "activations/layer16_attention_weight_max": 33.21564865112305,
      "activations/layer16_attention_weight_min": -25.730907440185547,
      "activations/layer17_attention_weight_max": 56.6915168762207,
      "activations/layer17_attention_weight_min": -38.821144104003906,
      "activations/layer18_attention_weight_max": 54.2739143371582,
      "activations/layer18_attention_weight_min": -34.4655876159668,
      "activations/layer19_attention_weight_max": 21.396169662475586,
      "activations/layer19_attention_weight_min": -19.17786979675293,
      "activations/layer1_attention_weight_max": 17.298301696777344,
      "activations/layer1_attention_weight_min": -14.271749496459961,
      "activations/layer20_attention_weight_max": 21.177675247192383,
      "activations/layer20_attention_weight_min": -25.513553619384766,
      "activations/layer21_attention_weight_max": 39.38948440551758,
      "activations/layer21_attention_weight_min": -25.759580612182617,
      "activations/layer22_attention_weight_max": 31.704925537109375,
      "activations/layer22_attention_weight_min": -25.635658264160156,
      "activations/layer23_attention_weight_max": 30.442495346069336,
      "activations/layer23_attention_weight_min": -19.776124954223633,
      "activations/layer2_attention_weight_max": 29.294654846191406,
      "activations/layer2_attention_weight_min": -28.049549102783203,
      "activations/layer3_attention_weight_max": 80.76780700683594,
      "activations/layer3_attention_weight_min": -81.7613296508789,
      "activations/layer4_attention_weight_max": 98.15105438232422,
      "activations/layer4_attention_weight_min": -95.17413330078125,
      "activations/layer5_attention_weight_max": 96.91283416748047,
      "activations/layer5_attention_weight_min": -88.61781311035156,
      "activations/layer6_attention_weight_max": 57.097782135009766,
      "activations/layer6_attention_weight_min": -52.29417419433594,
      "activations/layer7_attention_weight_max": 69.11812591552734,
      "activations/layer7_attention_weight_min": -65.3982162475586,
      "activations/layer8_attention_weight_max": 44.303497314453125,
      "activations/layer8_attention_weight_min": -45.71773147583008,
      "activations/layer9_attention_weight_max": 51.56037139892578,
      "activations/layer9_attention_weight_min": -48.01872253417969,
      "epoch": 5.56,
      "learning_rate": 0.00011532007575757574,
      "loss": 2.8477,
      "step": 95650
    },
    {
      "activations/layer0_attention_weight_max": 14.03613567352295,
      "activations/layer0_attention_weight_min": -14.245499610900879,
      "activations/layer10_attention_weight_max": 47.677093505859375,
      "activations/layer10_attention_weight_min": -42.01216506958008,
      "activations/layer11_attention_weight_max": 35.2685432434082,
      "activations/layer11_attention_weight_min": -31.745288848876953,
      "activations/layer12_attention_weight_max": 20.05357551574707,
      "activations/layer12_attention_weight_min": -26.41423988342285,
      "activations/layer13_attention_weight_max": 51.447792053222656,
      "activations/layer13_attention_weight_min": -41.526485443115234,
      "activations/layer14_attention_weight_max": 56.660804748535156,
      "activations/layer14_attention_weight_min": -40.268760681152344,
      "activations/layer15_attention_weight_max": 41.2060661315918,
      "activations/layer15_attention_weight_min": -29.135459899902344,
      "activations/layer16_attention_weight_max": 34.64763259887695,
      "activations/layer16_attention_weight_min": -27.17656898498535,
      "activations/layer17_attention_weight_max": 56.30766296386719,
      "activations/layer17_attention_weight_min": -37.79787826538086,
      "activations/layer18_attention_weight_max": 45.8621940612793,
      "activations/layer18_attention_weight_min": -34.539703369140625,
      "activations/layer19_attention_weight_max": 19.506925582885742,
      "activations/layer19_attention_weight_min": -22.411754608154297,
      "activations/layer1_attention_weight_max": 18.587717056274414,
      "activations/layer1_attention_weight_min": -16.360023498535156,
      "activations/layer20_attention_weight_max": 22.700597763061523,
      "activations/layer20_attention_weight_min": -21.146202087402344,
      "activations/layer21_attention_weight_max": 39.93092727661133,
      "activations/layer21_attention_weight_min": -26.00706672668457,
      "activations/layer22_attention_weight_max": 32.80216979980469,
      "activations/layer22_attention_weight_min": -23.74432373046875,
      "activations/layer23_attention_weight_max": 29.446186065673828,
      "activations/layer23_attention_weight_min": -20.576629638671875,
      "activations/layer2_attention_weight_max": 29.90822410583496,
      "activations/layer2_attention_weight_min": -26.992103576660156,
      "activations/layer3_attention_weight_max": 85.16349792480469,
      "activations/layer3_attention_weight_min": -83.48590850830078,
      "activations/layer4_attention_weight_max": 107.6132583618164,
      "activations/layer4_attention_weight_min": -96.2611312866211,
      "activations/layer5_attention_weight_max": 101.03379821777344,
      "activations/layer5_attention_weight_min": -91.21621704101562,
      "activations/layer6_attention_weight_max": 58.74484634399414,
      "activations/layer6_attention_weight_min": -54.9701042175293,
      "activations/layer7_attention_weight_max": 85.55889892578125,
      "activations/layer7_attention_weight_min": -73.33604431152344,
      "activations/layer8_attention_weight_max": 53.716644287109375,
      "activations/layer8_attention_weight_min": -45.88296890258789,
      "activations/layer9_attention_weight_max": 71.2387924194336,
      "activations/layer9_attention_weight_min": -55.56528854370117,
      "epoch": 5.56,
      "learning_rate": 0.00011530113636363635,
      "loss": 2.8551,
      "step": 95700
    },
    {
      "activations/layer0_attention_weight_max": 14.939742088317871,
      "activations/layer0_attention_weight_min": -14.946249961853027,
      "activations/layer10_attention_weight_max": 37.3425178527832,
      "activations/layer10_attention_weight_min": -34.18648147583008,
      "activations/layer11_attention_weight_max": 27.705564498901367,
      "activations/layer11_attention_weight_min": -26.575727462768555,
      "activations/layer12_attention_weight_max": 22.54752540588379,
      "activations/layer12_attention_weight_min": -24.049373626708984,
      "activations/layer13_attention_weight_max": 53.85793685913086,
      "activations/layer13_attention_weight_min": -34.499114990234375,
      "activations/layer14_attention_weight_max": 42.22721862792969,
      "activations/layer14_attention_weight_min": -29.195695877075195,
      "activations/layer15_attention_weight_max": 51.95229721069336,
      "activations/layer15_attention_weight_min": -29.325801849365234,
      "activations/layer16_attention_weight_max": 32.2523193359375,
      "activations/layer16_attention_weight_min": -28.529069900512695,
      "activations/layer17_attention_weight_max": 53.33498001098633,
      "activations/layer17_attention_weight_min": -39.500732421875,
      "activations/layer18_attention_weight_max": 47.59061813354492,
      "activations/layer18_attention_weight_min": -32.381797790527344,
      "activations/layer19_attention_weight_max": 22.803258895874023,
      "activations/layer19_attention_weight_min": -19.594953536987305,
      "activations/layer1_attention_weight_max": 17.273406982421875,
      "activations/layer1_attention_weight_min": -16.736568450927734,
      "activations/layer20_attention_weight_max": 20.813596725463867,
      "activations/layer20_attention_weight_min": -21.363216400146484,
      "activations/layer21_attention_weight_max": 38.35625076293945,
      "activations/layer21_attention_weight_min": -24.3801212310791,
      "activations/layer22_attention_weight_max": 33.88794708251953,
      "activations/layer22_attention_weight_min": -24.510597229003906,
      "activations/layer23_attention_weight_max": 28.576337814331055,
      "activations/layer23_attention_weight_min": -17.7852840423584,
      "activations/layer2_attention_weight_max": 32.11148452758789,
      "activations/layer2_attention_weight_min": -29.261302947998047,
      "activations/layer3_attention_weight_max": 83.73837280273438,
      "activations/layer3_attention_weight_min": -84.85990142822266,
      "activations/layer4_attention_weight_max": 102.39675903320312,
      "activations/layer4_attention_weight_min": -94.35090637207031,
      "activations/layer5_attention_weight_max": 102.5531234741211,
      "activations/layer5_attention_weight_min": -90.35032653808594,
      "activations/layer6_attention_weight_max": 57.8382453918457,
      "activations/layer6_attention_weight_min": -53.96357345581055,
      "activations/layer7_attention_weight_max": 70.73816680908203,
      "activations/layer7_attention_weight_min": -64.61177062988281,
      "activations/layer8_attention_weight_max": 41.8634033203125,
      "activations/layer8_attention_weight_min": -43.72169876098633,
      "activations/layer9_attention_weight_max": 48.01981735229492,
      "activations/layer9_attention_weight_min": -42.255367279052734,
      "epoch": 5.56,
      "learning_rate": 0.00011528219696969697,
      "loss": 2.8343,
      "step": 95750
    },
    {
      "activations/layer0_attention_weight_max": 14.049150466918945,
      "activations/layer0_attention_weight_min": -14.973063468933105,
      "activations/layer10_attention_weight_max": 42.861053466796875,
      "activations/layer10_attention_weight_min": -35.586368560791016,
      "activations/layer11_attention_weight_max": 32.13606643676758,
      "activations/layer11_attention_weight_min": -26.384998321533203,
      "activations/layer12_attention_weight_max": 25.40397071838379,
      "activations/layer12_attention_weight_min": -24.90023422241211,
      "activations/layer13_attention_weight_max": 46.63487243652344,
      "activations/layer13_attention_weight_min": -31.57872200012207,
      "activations/layer14_attention_weight_max": 59.18416213989258,
      "activations/layer14_attention_weight_min": -35.378990173339844,
      "activations/layer15_attention_weight_max": 40.0628547668457,
      "activations/layer15_attention_weight_min": -29.463102340698242,
      "activations/layer16_attention_weight_max": 32.169822692871094,
      "activations/layer16_attention_weight_min": -26.23691177368164,
      "activations/layer17_attention_weight_max": 59.87063980102539,
      "activations/layer17_attention_weight_min": -38.80198287963867,
      "activations/layer18_attention_weight_max": 50.66057586669922,
      "activations/layer18_attention_weight_min": -33.475433349609375,
      "activations/layer19_attention_weight_max": 22.71887969970703,
      "activations/layer19_attention_weight_min": -18.773788452148438,
      "activations/layer1_attention_weight_max": 17.425159454345703,
      "activations/layer1_attention_weight_min": -17.061643600463867,
      "activations/layer20_attention_weight_max": 24.9888858795166,
      "activations/layer20_attention_weight_min": -21.630617141723633,
      "activations/layer21_attention_weight_max": 47.53184127807617,
      "activations/layer21_attention_weight_min": -24.975614547729492,
      "activations/layer22_attention_weight_max": 34.55845260620117,
      "activations/layer22_attention_weight_min": -23.35130500793457,
      "activations/layer23_attention_weight_max": 32.646568298339844,
      "activations/layer23_attention_weight_min": -19.1199951171875,
      "activations/layer2_attention_weight_max": 31.5850887298584,
      "activations/layer2_attention_weight_min": -29.065563201904297,
      "activations/layer3_attention_weight_max": 86.54237365722656,
      "activations/layer3_attention_weight_min": -93.6826400756836,
      "activations/layer4_attention_weight_max": 107.8285140991211,
      "activations/layer4_attention_weight_min": -103.21392822265625,
      "activations/layer5_attention_weight_max": 106.130126953125,
      "activations/layer5_attention_weight_min": -97.13673400878906,
      "activations/layer6_attention_weight_max": 61.77753829956055,
      "activations/layer6_attention_weight_min": -55.54503631591797,
      "activations/layer7_attention_weight_max": 70.79067993164062,
      "activations/layer7_attention_weight_min": -69.77985382080078,
      "activations/layer8_attention_weight_max": 43.83106231689453,
      "activations/layer8_attention_weight_min": -43.7339973449707,
      "activations/layer9_attention_weight_max": 56.07255172729492,
      "activations/layer9_attention_weight_min": -47.1770133972168,
      "epoch": 5.57,
      "learning_rate": 0.00011526325757575757,
      "loss": 2.8476,
      "step": 95800
    },
    {
      "activations/layer0_attention_weight_max": 13.936772346496582,
      "activations/layer0_attention_weight_min": -13.939923286437988,
      "activations/layer10_attention_weight_max": 40.5157585144043,
      "activations/layer10_attention_weight_min": -35.21652603149414,
      "activations/layer11_attention_weight_max": 30.41323471069336,
      "activations/layer11_attention_weight_min": -29.984960556030273,
      "activations/layer12_attention_weight_max": 22.314720153808594,
      "activations/layer12_attention_weight_min": -26.05909538269043,
      "activations/layer13_attention_weight_max": 50.89735794067383,
      "activations/layer13_attention_weight_min": -33.114112854003906,
      "activations/layer14_attention_weight_max": 40.015159606933594,
      "activations/layer14_attention_weight_min": -27.848390579223633,
      "activations/layer15_attention_weight_max": 39.264163970947266,
      "activations/layer15_attention_weight_min": -28.620285034179688,
      "activations/layer16_attention_weight_max": 35.32022476196289,
      "activations/layer16_attention_weight_min": -26.037860870361328,
      "activations/layer17_attention_weight_max": 57.1562385559082,
      "activations/layer17_attention_weight_min": -39.29096984863281,
      "activations/layer18_attention_weight_max": 50.67613220214844,
      "activations/layer18_attention_weight_min": -32.695556640625,
      "activations/layer19_attention_weight_max": 22.55237579345703,
      "activations/layer19_attention_weight_min": -19.15618896484375,
      "activations/layer1_attention_weight_max": 16.31024169921875,
      "activations/layer1_attention_weight_min": -17.285079956054688,
      "activations/layer20_attention_weight_max": 23.614051818847656,
      "activations/layer20_attention_weight_min": -20.811918258666992,
      "activations/layer21_attention_weight_max": 41.01667404174805,
      "activations/layer21_attention_weight_min": -24.84404945373535,
      "activations/layer22_attention_weight_max": 34.902000427246094,
      "activations/layer22_attention_weight_min": -25.30146026611328,
      "activations/layer23_attention_weight_max": 29.363391876220703,
      "activations/layer23_attention_weight_min": -20.107749938964844,
      "activations/layer2_attention_weight_max": 29.377437591552734,
      "activations/layer2_attention_weight_min": -29.097660064697266,
      "activations/layer3_attention_weight_max": 78.973388671875,
      "activations/layer3_attention_weight_min": -85.83303833007812,
      "activations/layer4_attention_weight_max": 100.69478607177734,
      "activations/layer4_attention_weight_min": -96.52252197265625,
      "activations/layer5_attention_weight_max": 97.82049560546875,
      "activations/layer5_attention_weight_min": -97.27964782714844,
      "activations/layer6_attention_weight_max": 59.37199020385742,
      "activations/layer6_attention_weight_min": -58.53630447387695,
      "activations/layer7_attention_weight_max": 74.43807983398438,
      "activations/layer7_attention_weight_min": -72.36039733886719,
      "activations/layer8_attention_weight_max": 45.9111442565918,
      "activations/layer8_attention_weight_min": -44.92045974731445,
      "activations/layer9_attention_weight_max": 49.56257629394531,
      "activations/layer9_attention_weight_min": -49.61240005493164,
      "epoch": 5.57,
      "learning_rate": 0.00011524431818181817,
      "loss": 2.8623,
      "step": 95850
    },
    {
      "activations/layer0_attention_weight_max": 14.317110061645508,
      "activations/layer0_attention_weight_min": -14.985369682312012,
      "activations/layer10_attention_weight_max": 38.558509826660156,
      "activations/layer10_attention_weight_min": -33.323944091796875,
      "activations/layer11_attention_weight_max": 29.8321533203125,
      "activations/layer11_attention_weight_min": -28.805862426757812,
      "activations/layer12_attention_weight_max": 22.43913459777832,
      "activations/layer12_attention_weight_min": -24.3989200592041,
      "activations/layer13_attention_weight_max": 45.0188102722168,
      "activations/layer13_attention_weight_min": -36.41524887084961,
      "activations/layer14_attention_weight_max": 35.41203308105469,
      "activations/layer14_attention_weight_min": -30.46941566467285,
      "activations/layer15_attention_weight_max": 35.87419128417969,
      "activations/layer15_attention_weight_min": -29.279605865478516,
      "activations/layer16_attention_weight_max": 31.322742462158203,
      "activations/layer16_attention_weight_min": -25.51591682434082,
      "activations/layer17_attention_weight_max": 54.08727264404297,
      "activations/layer17_attention_weight_min": -39.595359802246094,
      "activations/layer18_attention_weight_max": 47.85871505737305,
      "activations/layer18_attention_weight_min": -35.74528503417969,
      "activations/layer19_attention_weight_max": 23.361631393432617,
      "activations/layer19_attention_weight_min": -20.99018669128418,
      "activations/layer1_attention_weight_max": 17.830045700073242,
      "activations/layer1_attention_weight_min": -17.66077995300293,
      "activations/layer20_attention_weight_max": 21.031448364257812,
      "activations/layer20_attention_weight_min": -22.09696388244629,
      "activations/layer21_attention_weight_max": 36.03964614868164,
      "activations/layer21_attention_weight_min": -26.18440818786621,
      "activations/layer22_attention_weight_max": 34.98245620727539,
      "activations/layer22_attention_weight_min": -24.382720947265625,
      "activations/layer23_attention_weight_max": 29.21028709411621,
      "activations/layer23_attention_weight_min": -18.1012020111084,
      "activations/layer2_attention_weight_max": 30.15283966064453,
      "activations/layer2_attention_weight_min": -26.83465576171875,
      "activations/layer3_attention_weight_max": 75.16529083251953,
      "activations/layer3_attention_weight_min": -82.02302551269531,
      "activations/layer4_attention_weight_max": 94.36931610107422,
      "activations/layer4_attention_weight_min": -88.7639389038086,
      "activations/layer5_attention_weight_max": 96.62596130371094,
      "activations/layer5_attention_weight_min": -84.9176025390625,
      "activations/layer6_attention_weight_max": 55.14817810058594,
      "activations/layer6_attention_weight_min": -51.104793548583984,
      "activations/layer7_attention_weight_max": 72.70816802978516,
      "activations/layer7_attention_weight_min": -70.0035171508789,
      "activations/layer8_attention_weight_max": 40.05455780029297,
      "activations/layer8_attention_weight_min": -42.635807037353516,
      "activations/layer9_attention_weight_max": 47.660552978515625,
      "activations/layer9_attention_weight_min": -47.30021286010742,
      "epoch": 5.57,
      "learning_rate": 0.00011522537878787878,
      "loss": 2.8482,
      "step": 95900
    },
    {
      "activations/layer0_attention_weight_max": 14.02578067779541,
      "activations/layer0_attention_weight_min": -15.743576049804688,
      "activations/layer10_attention_weight_max": 40.67847442626953,
      "activations/layer10_attention_weight_min": -39.59910202026367,
      "activations/layer11_attention_weight_max": 31.614665985107422,
      "activations/layer11_attention_weight_min": -28.910079956054688,
      "activations/layer12_attention_weight_max": 25.12442970275879,
      "activations/layer12_attention_weight_min": -27.811182022094727,
      "activations/layer13_attention_weight_max": 46.428794860839844,
      "activations/layer13_attention_weight_min": -33.013465881347656,
      "activations/layer14_attention_weight_max": 58.56734085083008,
      "activations/layer14_attention_weight_min": -35.56724166870117,
      "activations/layer15_attention_weight_max": 41.736846923828125,
      "activations/layer15_attention_weight_min": -29.739526748657227,
      "activations/layer16_attention_weight_max": 33.11586380004883,
      "activations/layer16_attention_weight_min": -27.730491638183594,
      "activations/layer17_attention_weight_max": 63.7585334777832,
      "activations/layer17_attention_weight_min": -39.389034271240234,
      "activations/layer18_attention_weight_max": 50.09222412109375,
      "activations/layer18_attention_weight_min": -34.93352127075195,
      "activations/layer19_attention_weight_max": 18.873586654663086,
      "activations/layer19_attention_weight_min": -18.405744552612305,
      "activations/layer1_attention_weight_max": 17.64265251159668,
      "activations/layer1_attention_weight_min": -14.30575942993164,
      "activations/layer20_attention_weight_max": 23.732046127319336,
      "activations/layer20_attention_weight_min": -21.008142471313477,
      "activations/layer21_attention_weight_max": 38.68983459472656,
      "activations/layer21_attention_weight_min": -24.544687271118164,
      "activations/layer22_attention_weight_max": 33.10908508300781,
      "activations/layer22_attention_weight_min": -24.779680252075195,
      "activations/layer23_attention_weight_max": 32.99617385864258,
      "activations/layer23_attention_weight_min": -19.968618392944336,
      "activations/layer2_attention_weight_max": 33.53362274169922,
      "activations/layer2_attention_weight_min": -30.050296783447266,
      "activations/layer3_attention_weight_max": 83.25148010253906,
      "activations/layer3_attention_weight_min": -90.4281005859375,
      "activations/layer4_attention_weight_max": 104.63631439208984,
      "activations/layer4_attention_weight_min": -97.66796112060547,
      "activations/layer5_attention_weight_max": 103.51393127441406,
      "activations/layer5_attention_weight_min": -100.45585632324219,
      "activations/layer6_attention_weight_max": 60.47105407714844,
      "activations/layer6_attention_weight_min": -55.59321594238281,
      "activations/layer7_attention_weight_max": 72.91083526611328,
      "activations/layer7_attention_weight_min": -73.09046936035156,
      "activations/layer8_attention_weight_max": 45.34566116333008,
      "activations/layer8_attention_weight_min": -45.966896057128906,
      "activations/layer9_attention_weight_max": 58.88664627075195,
      "activations/layer9_attention_weight_min": -55.51503372192383,
      "epoch": 5.58,
      "learning_rate": 0.00011520643939393939,
      "loss": 2.8322,
      "step": 95950
    },
    {
      "activations/layer0_attention_weight_max": 14.343140602111816,
      "activations/layer0_attention_weight_min": -15.164408683776855,
      "activations/layer10_attention_weight_max": 46.21806716918945,
      "activations/layer10_attention_weight_min": -39.56364822387695,
      "activations/layer11_attention_weight_max": 33.90907669067383,
      "activations/layer11_attention_weight_min": -32.125,
      "activations/layer12_attention_weight_max": 32.02528381347656,
      "activations/layer12_attention_weight_min": -25.99175262451172,
      "activations/layer13_attention_weight_max": 56.86516571044922,
      "activations/layer13_attention_weight_min": -40.1228141784668,
      "activations/layer14_attention_weight_max": 45.516056060791016,
      "activations/layer14_attention_weight_min": -29.514860153198242,
      "activations/layer15_attention_weight_max": 42.486968994140625,
      "activations/layer15_attention_weight_min": -30.07636833190918,
      "activations/layer16_attention_weight_max": 37.696388244628906,
      "activations/layer16_attention_weight_min": -25.920331954956055,
      "activations/layer17_attention_weight_max": 60.81182861328125,
      "activations/layer17_attention_weight_min": -38.965545654296875,
      "activations/layer18_attention_weight_max": 50.56937026977539,
      "activations/layer18_attention_weight_min": -33.71304702758789,
      "activations/layer19_attention_weight_max": 21.699731826782227,
      "activations/layer19_attention_weight_min": -23.203048706054688,
      "activations/layer1_attention_weight_max": 18.115930557250977,
      "activations/layer1_attention_weight_min": -14.584976196289062,
      "activations/layer20_attention_weight_max": 25.507251739501953,
      "activations/layer20_attention_weight_min": -18.890975952148438,
      "activations/layer21_attention_weight_max": 42.43748092651367,
      "activations/layer21_attention_weight_min": -24.96916961669922,
      "activations/layer22_attention_weight_max": 34.86807632446289,
      "activations/layer22_attention_weight_min": -22.123254776000977,
      "activations/layer23_attention_weight_max": 29.393081665039062,
      "activations/layer23_attention_weight_min": -18.84796905517578,
      "activations/layer2_attention_weight_max": 30.61647605895996,
      "activations/layer2_attention_weight_min": -27.48072624206543,
      "activations/layer3_attention_weight_max": 81.47298431396484,
      "activations/layer3_attention_weight_min": -83.85096740722656,
      "activations/layer4_attention_weight_max": 102.03082275390625,
      "activations/layer4_attention_weight_min": -97.2585678100586,
      "activations/layer5_attention_weight_max": 102.20698547363281,
      "activations/layer5_attention_weight_min": -94.91386413574219,
      "activations/layer6_attention_weight_max": 62.1192626953125,
      "activations/layer6_attention_weight_min": -55.92548370361328,
      "activations/layer7_attention_weight_max": 75.46307373046875,
      "activations/layer7_attention_weight_min": -75.62918853759766,
      "activations/layer8_attention_weight_max": 50.00313186645508,
      "activations/layer8_attention_weight_min": -47.74150085449219,
      "activations/layer9_attention_weight_max": 57.77822494506836,
      "activations/layer9_attention_weight_min": -52.61052322387695,
      "epoch": 5.58,
      "learning_rate": 0.00011518749999999999,
      "loss": 2.8392,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_loss": 2.79296875,
      "eval_runtime": 8.472,
      "eval_samples_per_second": 506.844,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_openwebtext_loss": 2.79296875,
      "eval_openwebtext_ppl": 16.329425896569767,
      "eval_openwebtext_runtime": 8.472,
      "eval_openwebtext_samples_per_second": 506.844,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 2.0481,
      "eval_wikitext_samples_per_second": 222.642,
      "step": 96000
    },
    {
      "epoch": 5.58,
      "eval_lambada_loss": 2.826171875,
      "eval_lambada_ppl": 16.88071548801052,
      "eval_lambada_runtime": 9.5962,
      "eval_lambada_samples_per_second": 507.39,
      "step": 96000
    },
    {
      "activations/layer0_attention_weight_max": 13.735389709472656,
      "activations/layer0_attention_weight_min": -15.399398803710938,
      "activations/layer10_attention_weight_max": 40.321327209472656,
      "activations/layer10_attention_weight_min": -36.80180740356445,
      "activations/layer11_attention_weight_max": 30.036083221435547,
      "activations/layer11_attention_weight_min": -27.338125228881836,
      "activations/layer12_attention_weight_max": 21.676359176635742,
      "activations/layer12_attention_weight_min": -25.437171936035156,
      "activations/layer13_attention_weight_max": 52.02477264404297,
      "activations/layer13_attention_weight_min": -37.36939239501953,
      "activations/layer14_attention_weight_max": 44.76115036010742,
      "activations/layer14_attention_weight_min": -28.265174865722656,
      "activations/layer15_attention_weight_max": 35.052398681640625,
      "activations/layer15_attention_weight_min": -29.495668411254883,
      "activations/layer16_attention_weight_max": 35.59526824951172,
      "activations/layer16_attention_weight_min": -27.314804077148438,
      "activations/layer17_attention_weight_max": 56.96610641479492,
      "activations/layer17_attention_weight_min": -40.522274017333984,
      "activations/layer18_attention_weight_max": 47.52154541015625,
      "activations/layer18_attention_weight_min": -34.029727935791016,
      "activations/layer19_attention_weight_max": 21.200740814208984,
      "activations/layer19_attention_weight_min": -21.99449920654297,
      "activations/layer1_attention_weight_max": 18.616153717041016,
      "activations/layer1_attention_weight_min": -14.164543151855469,
      "activations/layer20_attention_weight_max": 20.584688186645508,
      "activations/layer20_attention_weight_min": -21.774995803833008,
      "activations/layer21_attention_weight_max": 38.4998664855957,
      "activations/layer21_attention_weight_min": -24.357208251953125,
      "activations/layer22_attention_weight_max": 37.827945709228516,
      "activations/layer22_attention_weight_min": -23.092195510864258,
      "activations/layer23_attention_weight_max": 34.071048736572266,
      "activations/layer23_attention_weight_min": -17.34233856201172,
      "activations/layer2_attention_weight_max": 31.670536041259766,
      "activations/layer2_attention_weight_min": -27.814041137695312,
      "activations/layer3_attention_weight_max": 80.41506958007812,
      "activations/layer3_attention_weight_min": -88.21955871582031,
      "activations/layer4_attention_weight_max": 98.3326416015625,
      "activations/layer4_attention_weight_min": -93.45204162597656,
      "activations/layer5_attention_weight_max": 99.39421081542969,
      "activations/layer5_attention_weight_min": -88.266845703125,
      "activations/layer6_attention_weight_max": 57.04887008666992,
      "activations/layer6_attention_weight_min": -52.997676849365234,
      "activations/layer7_attention_weight_max": 71.72740173339844,
      "activations/layer7_attention_weight_min": -69.8610610961914,
      "activations/layer8_attention_weight_max": 45.267303466796875,
      "activations/layer8_attention_weight_min": -43.98410415649414,
      "activations/layer9_attention_weight_max": 60.11214065551758,
      "activations/layer9_attention_weight_min": -48.70975112915039,
      "epoch": 5.58,
      "learning_rate": 0.00011516856060606059,
      "loss": 2.8552,
      "step": 96050
    },
    {
      "activations/layer0_attention_weight_max": 14.261974334716797,
      "activations/layer0_attention_weight_min": -14.962430953979492,
      "activations/layer10_attention_weight_max": 38.743507385253906,
      "activations/layer10_attention_weight_min": -36.372535705566406,
      "activations/layer11_attention_weight_max": 30.433332443237305,
      "activations/layer11_attention_weight_min": -29.98767852783203,
      "activations/layer12_attention_weight_max": 23.933399200439453,
      "activations/layer12_attention_weight_min": -25.305084228515625,
      "activations/layer13_attention_weight_max": 50.823482513427734,
      "activations/layer13_attention_weight_min": -36.72681427001953,
      "activations/layer14_attention_weight_max": 41.944984436035156,
      "activations/layer14_attention_weight_min": -30.30828857421875,
      "activations/layer15_attention_weight_max": 41.59464645385742,
      "activations/layer15_attention_weight_min": -29.928205490112305,
      "activations/layer16_attention_weight_max": 40.41048049926758,
      "activations/layer16_attention_weight_min": -27.1873779296875,
      "activations/layer17_attention_weight_max": 57.47819900512695,
      "activations/layer17_attention_weight_min": -40.502845764160156,
      "activations/layer18_attention_weight_max": 47.37751770019531,
      "activations/layer18_attention_weight_min": -33.835601806640625,
      "activations/layer19_attention_weight_max": 21.575822830200195,
      "activations/layer19_attention_weight_min": -19.787899017333984,
      "activations/layer1_attention_weight_max": 16.998079299926758,
      "activations/layer1_attention_weight_min": -14.925152778625488,
      "activations/layer20_attention_weight_max": 19.659940719604492,
      "activations/layer20_attention_weight_min": -24.27391242980957,
      "activations/layer21_attention_weight_max": 38.4149284362793,
      "activations/layer21_attention_weight_min": -23.622983932495117,
      "activations/layer22_attention_weight_max": 31.94184112548828,
      "activations/layer22_attention_weight_min": -23.62497329711914,
      "activations/layer23_attention_weight_max": 32.66456985473633,
      "activations/layer23_attention_weight_min": -17.11115837097168,
      "activations/layer2_attention_weight_max": 30.045507431030273,
      "activations/layer2_attention_weight_min": -27.20943260192871,
      "activations/layer3_attention_weight_max": 85.01647186279297,
      "activations/layer3_attention_weight_min": -84.71212768554688,
      "activations/layer4_attention_weight_max": 104.80098724365234,
      "activations/layer4_attention_weight_min": -96.96536254882812,
      "activations/layer5_attention_weight_max": 101.0882797241211,
      "activations/layer5_attention_weight_min": -90.70819091796875,
      "activations/layer6_attention_weight_max": 58.39844512939453,
      "activations/layer6_attention_weight_min": -53.692420959472656,
      "activations/layer7_attention_weight_max": 70.95345306396484,
      "activations/layer7_attention_weight_min": -72.22606658935547,
      "activations/layer8_attention_weight_max": 42.756160736083984,
      "activations/layer8_attention_weight_min": -42.00209426879883,
      "activations/layer9_attention_weight_max": 51.893165588378906,
      "activations/layer9_attention_weight_min": -46.81439208984375,
      "epoch": 5.58,
      "learning_rate": 0.0001151496212121212,
      "loss": 2.8444,
      "step": 96100
    },
    {
      "activations/layer0_attention_weight_max": 13.67821979522705,
      "activations/layer0_attention_weight_min": -15.251907348632812,
      "activations/layer10_attention_weight_max": 41.27121353149414,
      "activations/layer10_attention_weight_min": -35.97615051269531,
      "activations/layer11_attention_weight_max": 32.97825622558594,
      "activations/layer11_attention_weight_min": -29.1636962890625,
      "activations/layer12_attention_weight_max": 23.730880737304688,
      "activations/layer12_attention_weight_min": -25.751008987426758,
      "activations/layer13_attention_weight_max": 41.65076446533203,
      "activations/layer13_attention_weight_min": -32.244911193847656,
      "activations/layer14_attention_weight_max": 40.68832778930664,
      "activations/layer14_attention_weight_min": -31.022659301757812,
      "activations/layer15_attention_weight_max": 40.526058197021484,
      "activations/layer15_attention_weight_min": -31.535633087158203,
      "activations/layer16_attention_weight_max": 34.193756103515625,
      "activations/layer16_attention_weight_min": -29.01047706604004,
      "activations/layer17_attention_weight_max": 61.36471176147461,
      "activations/layer17_attention_weight_min": -41.0404167175293,
      "activations/layer18_attention_weight_max": 46.99021530151367,
      "activations/layer18_attention_weight_min": -35.147743225097656,
      "activations/layer19_attention_weight_max": 26.255268096923828,
      "activations/layer19_attention_weight_min": -19.331993103027344,
      "activations/layer1_attention_weight_max": 17.867321014404297,
      "activations/layer1_attention_weight_min": -15.451766967773438,
      "activations/layer20_attention_weight_max": 20.86852264404297,
      "activations/layer20_attention_weight_min": -22.612390518188477,
      "activations/layer21_attention_weight_max": 36.6190185546875,
      "activations/layer21_attention_weight_min": -28.45134162902832,
      "activations/layer22_attention_weight_max": 31.70124626159668,
      "activations/layer22_attention_weight_min": -25.826963424682617,
      "activations/layer23_attention_weight_max": 29.198217391967773,
      "activations/layer23_attention_weight_min": -20.47542381286621,
      "activations/layer2_attention_weight_max": 30.700973510742188,
      "activations/layer2_attention_weight_min": -28.387990951538086,
      "activations/layer3_attention_weight_max": 84.57493591308594,
      "activations/layer3_attention_weight_min": -85.34534454345703,
      "activations/layer4_attention_weight_max": 96.74220275878906,
      "activations/layer4_attention_weight_min": -92.78011322021484,
      "activations/layer5_attention_weight_max": 97.36607360839844,
      "activations/layer5_attention_weight_min": -84.4695816040039,
      "activations/layer6_attention_weight_max": 58.086090087890625,
      "activations/layer6_attention_weight_min": -50.25092697143555,
      "activations/layer7_attention_weight_max": 70.69253540039062,
      "activations/layer7_attention_weight_min": -66.86361694335938,
      "activations/layer8_attention_weight_max": 42.265193939208984,
      "activations/layer8_attention_weight_min": -42.98967361450195,
      "activations/layer9_attention_weight_max": 50.653018951416016,
      "activations/layer9_attention_weight_min": -46.60865783691406,
      "epoch": 5.59,
      "learning_rate": 0.00011513068181818182,
      "loss": 2.8466,
      "step": 96150
    },
    {
      "activations/layer0_attention_weight_max": 13.66535472869873,
      "activations/layer0_attention_weight_min": -14.776654243469238,
      "activations/layer10_attention_weight_max": 40.75969314575195,
      "activations/layer10_attention_weight_min": -36.535888671875,
      "activations/layer11_attention_weight_max": 33.964881896972656,
      "activations/layer11_attention_weight_min": -29.41141128540039,
      "activations/layer12_attention_weight_max": 26.367557525634766,
      "activations/layer12_attention_weight_min": -24.11949920654297,
      "activations/layer13_attention_weight_max": 43.519981384277344,
      "activations/layer13_attention_weight_min": -33.209041595458984,
      "activations/layer14_attention_weight_max": 45.588260650634766,
      "activations/layer14_attention_weight_min": -30.268972396850586,
      "activations/layer15_attention_weight_max": 40.604801177978516,
      "activations/layer15_attention_weight_min": -29.46135711669922,
      "activations/layer16_attention_weight_max": 32.63950729370117,
      "activations/layer16_attention_weight_min": -29.0736026763916,
      "activations/layer17_attention_weight_max": 55.68901062011719,
      "activations/layer17_attention_weight_min": -38.44941329956055,
      "activations/layer18_attention_weight_max": 52.52951431274414,
      "activations/layer18_attention_weight_min": -35.61925506591797,
      "activations/layer19_attention_weight_max": 22.491647720336914,
      "activations/layer19_attention_weight_min": -20.031335830688477,
      "activations/layer1_attention_weight_max": 17.068403244018555,
      "activations/layer1_attention_weight_min": -17.174707412719727,
      "activations/layer20_attention_weight_max": 21.280210494995117,
      "activations/layer20_attention_weight_min": -21.07403564453125,
      "activations/layer21_attention_weight_max": 37.443824768066406,
      "activations/layer21_attention_weight_min": -28.604049682617188,
      "activations/layer22_attention_weight_max": 31.845632553100586,
      "activations/layer22_attention_weight_min": -25.738924026489258,
      "activations/layer23_attention_weight_max": 27.413917541503906,
      "activations/layer23_attention_weight_min": -19.012542724609375,
      "activations/layer2_attention_weight_max": 32.09360122680664,
      "activations/layer2_attention_weight_min": -28.679080963134766,
      "activations/layer3_attention_weight_max": 82.60631561279297,
      "activations/layer3_attention_weight_min": -86.59795379638672,
      "activations/layer4_attention_weight_max": 99.47136688232422,
      "activations/layer4_attention_weight_min": -93.63746643066406,
      "activations/layer5_attention_weight_max": 93.64751434326172,
      "activations/layer5_attention_weight_min": -89.85497283935547,
      "activations/layer6_attention_weight_max": 55.19638442993164,
      "activations/layer6_attention_weight_min": -51.8382453918457,
      "activations/layer7_attention_weight_max": 70.67440795898438,
      "activations/layer7_attention_weight_min": -66.10082244873047,
      "activations/layer8_attention_weight_max": 45.26913833618164,
      "activations/layer8_attention_weight_min": -42.317684173583984,
      "activations/layer9_attention_weight_max": 48.426998138427734,
      "activations/layer9_attention_weight_min": -45.2680778503418,
      "epoch": 5.59,
      "learning_rate": 0.00011511174242424241,
      "loss": 2.8428,
      "step": 96200
    },
    {
      "activations/layer0_attention_weight_max": 14.747811317443848,
      "activations/layer0_attention_weight_min": -14.773795127868652,
      "activations/layer10_attention_weight_max": 37.52615737915039,
      "activations/layer10_attention_weight_min": -34.91328430175781,
      "activations/layer11_attention_weight_max": 31.137802124023438,
      "activations/layer11_attention_weight_min": -27.07904815673828,
      "activations/layer12_attention_weight_max": 19.76454734802246,
      "activations/layer12_attention_weight_min": -26.852903366088867,
      "activations/layer13_attention_weight_max": 56.051658630371094,
      "activations/layer13_attention_weight_min": -41.98060989379883,
      "activations/layer14_attention_weight_max": 52.64894104003906,
      "activations/layer14_attention_weight_min": -37.90956115722656,
      "activations/layer15_attention_weight_max": 40.02781295776367,
      "activations/layer15_attention_weight_min": -28.946189880371094,
      "activations/layer16_attention_weight_max": 31.397905349731445,
      "activations/layer16_attention_weight_min": -26.7467098236084,
      "activations/layer17_attention_weight_max": 59.6651611328125,
      "activations/layer17_attention_weight_min": -40.17683029174805,
      "activations/layer18_attention_weight_max": 48.12162780761719,
      "activations/layer18_attention_weight_min": -34.43634033203125,
      "activations/layer19_attention_weight_max": 22.375764846801758,
      "activations/layer19_attention_weight_min": -21.838165283203125,
      "activations/layer1_attention_weight_max": 18.991180419921875,
      "activations/layer1_attention_weight_min": -16.390697479248047,
      "activations/layer20_attention_weight_max": 23.407695770263672,
      "activations/layer20_attention_weight_min": -21.984739303588867,
      "activations/layer21_attention_weight_max": 45.12059020996094,
      "activations/layer21_attention_weight_min": -27.320144653320312,
      "activations/layer22_attention_weight_max": 34.070072174072266,
      "activations/layer22_attention_weight_min": -24.147947311401367,
      "activations/layer23_attention_weight_max": 29.2946834564209,
      "activations/layer23_attention_weight_min": -20.76316261291504,
      "activations/layer2_attention_weight_max": 32.34015655517578,
      "activations/layer2_attention_weight_min": -30.3273983001709,
      "activations/layer3_attention_weight_max": 85.76093292236328,
      "activations/layer3_attention_weight_min": -87.9359130859375,
      "activations/layer4_attention_weight_max": 103.86556243896484,
      "activations/layer4_attention_weight_min": -106.63348388671875,
      "activations/layer5_attention_weight_max": 97.4165267944336,
      "activations/layer5_attention_weight_min": -90.64836120605469,
      "activations/layer6_attention_weight_max": 57.476837158203125,
      "activations/layer6_attention_weight_min": -51.473934173583984,
      "activations/layer7_attention_weight_max": 67.99922180175781,
      "activations/layer7_attention_weight_min": -66.93843078613281,
      "activations/layer8_attention_weight_max": 41.1063346862793,
      "activations/layer8_attention_weight_min": -39.55348587036133,
      "activations/layer9_attention_weight_max": 48.434322357177734,
      "activations/layer9_attention_weight_min": -44.185733795166016,
      "epoch": 5.59,
      "learning_rate": 0.00011509280303030302,
      "loss": 2.8334,
      "step": 96250
    },
    {
      "activations/layer0_attention_weight_max": 13.662303924560547,
      "activations/layer0_attention_weight_min": -14.886584281921387,
      "activations/layer10_attention_weight_max": 44.57977294921875,
      "activations/layer10_attention_weight_min": -36.68119812011719,
      "activations/layer11_attention_weight_max": 32.9278564453125,
      "activations/layer11_attention_weight_min": -27.69611358642578,
      "activations/layer12_attention_weight_max": 21.030622482299805,
      "activations/layer12_attention_weight_min": -25.220394134521484,
      "activations/layer13_attention_weight_max": 46.62541198730469,
      "activations/layer13_attention_weight_min": -31.9090576171875,
      "activations/layer14_attention_weight_max": 50.78512954711914,
      "activations/layer14_attention_weight_min": -33.7364616394043,
      "activations/layer15_attention_weight_max": 42.01177215576172,
      "activations/layer15_attention_weight_min": -32.964866638183594,
      "activations/layer16_attention_weight_max": 35.45824432373047,
      "activations/layer16_attention_weight_min": -27.550579071044922,
      "activations/layer17_attention_weight_max": 55.8193244934082,
      "activations/layer17_attention_weight_min": -39.73835372924805,
      "activations/layer18_attention_weight_max": 47.690589904785156,
      "activations/layer18_attention_weight_min": -35.4220085144043,
      "activations/layer19_attention_weight_max": 26.656469345092773,
      "activations/layer19_attention_weight_min": -21.427780151367188,
      "activations/layer1_attention_weight_max": 17.880752563476562,
      "activations/layer1_attention_weight_min": -16.925031661987305,
      "activations/layer20_attention_weight_max": 23.97738265991211,
      "activations/layer20_attention_weight_min": -21.987462997436523,
      "activations/layer21_attention_weight_max": 44.176143646240234,
      "activations/layer21_attention_weight_min": -27.76068687438965,
      "activations/layer22_attention_weight_max": 39.208587646484375,
      "activations/layer22_attention_weight_min": -25.30377197265625,
      "activations/layer23_attention_weight_max": 29.681028366088867,
      "activations/layer23_attention_weight_min": -18.07337188720703,
      "activations/layer2_attention_weight_max": 33.152549743652344,
      "activations/layer2_attention_weight_min": -28.452346801757812,
      "activations/layer3_attention_weight_max": 87.11188507080078,
      "activations/layer3_attention_weight_min": -92.55440521240234,
      "activations/layer4_attention_weight_max": 108.41719055175781,
      "activations/layer4_attention_weight_min": -103.6834716796875,
      "activations/layer5_attention_weight_max": 101.60520935058594,
      "activations/layer5_attention_weight_min": -93.70143127441406,
      "activations/layer6_attention_weight_max": 60.795616149902344,
      "activations/layer6_attention_weight_min": -54.699424743652344,
      "activations/layer7_attention_weight_max": 72.22532653808594,
      "activations/layer7_attention_weight_min": -71.99657440185547,
      "activations/layer8_attention_weight_max": 46.46026611328125,
      "activations/layer8_attention_weight_min": -44.18936538696289,
      "activations/layer9_attention_weight_max": 52.87060546875,
      "activations/layer9_attention_weight_min": -50.704185485839844,
      "epoch": 5.6,
      "learning_rate": 0.00011507386363636364,
      "loss": 2.8388,
      "step": 96300
    },
    {
      "activations/layer0_attention_weight_max": 14.308335304260254,
      "activations/layer0_attention_weight_min": -14.955253601074219,
      "activations/layer10_attention_weight_max": 38.359378814697266,
      "activations/layer10_attention_weight_min": -35.65871047973633,
      "activations/layer11_attention_weight_max": 29.876522064208984,
      "activations/layer11_attention_weight_min": -30.0983943939209,
      "activations/layer12_attention_weight_max": 21.098434448242188,
      "activations/layer12_attention_weight_min": -26.569719314575195,
      "activations/layer13_attention_weight_max": 51.50633239746094,
      "activations/layer13_attention_weight_min": -36.472877502441406,
      "activations/layer14_attention_weight_max": 41.67441940307617,
      "activations/layer14_attention_weight_min": -30.801212310791016,
      "activations/layer15_attention_weight_max": 45.035377502441406,
      "activations/layer15_attention_weight_min": -29.17701530456543,
      "activations/layer16_attention_weight_max": 36.18647003173828,
      "activations/layer16_attention_weight_min": -26.91123390197754,
      "activations/layer17_attention_weight_max": 59.52109146118164,
      "activations/layer17_attention_weight_min": -40.48330307006836,
      "activations/layer18_attention_weight_max": 50.80440902709961,
      "activations/layer18_attention_weight_min": -34.40574645996094,
      "activations/layer19_attention_weight_max": 27.18793296813965,
      "activations/layer19_attention_weight_min": -20.777008056640625,
      "activations/layer1_attention_weight_max": 16.474782943725586,
      "activations/layer1_attention_weight_min": -14.983823776245117,
      "activations/layer20_attention_weight_max": 25.55162811279297,
      "activations/layer20_attention_weight_min": -21.21941375732422,
      "activations/layer21_attention_weight_max": 42.602691650390625,
      "activations/layer21_attention_weight_min": -25.05990982055664,
      "activations/layer22_attention_weight_max": 33.532657623291016,
      "activations/layer22_attention_weight_min": -22.802907943725586,
      "activations/layer23_attention_weight_max": 33.935115814208984,
      "activations/layer23_attention_weight_min": -19.05328369140625,
      "activations/layer2_attention_weight_max": 31.423267364501953,
      "activations/layer2_attention_weight_min": -29.152542114257812,
      "activations/layer3_attention_weight_max": 86.25595092773438,
      "activations/layer3_attention_weight_min": -88.55439758300781,
      "activations/layer4_attention_weight_max": 101.33333587646484,
      "activations/layer4_attention_weight_min": -96.10529327392578,
      "activations/layer5_attention_weight_max": 97.86656188964844,
      "activations/layer5_attention_weight_min": -87.5525131225586,
      "activations/layer6_attention_weight_max": 55.450687408447266,
      "activations/layer6_attention_weight_min": -51.765201568603516,
      "activations/layer7_attention_weight_max": 70.41116333007812,
      "activations/layer7_attention_weight_min": -73.25240325927734,
      "activations/layer8_attention_weight_max": 42.7443733215332,
      "activations/layer8_attention_weight_min": -44.26483154296875,
      "activations/layer9_attention_weight_max": 48.05540084838867,
      "activations/layer9_attention_weight_min": -48.71891403198242,
      "epoch": 5.6,
      "learning_rate": 0.00011505492424242423,
      "loss": 2.8479,
      "step": 96350
    },
    {
      "activations/layer0_attention_weight_max": 16.58005714416504,
      "activations/layer0_attention_weight_min": -14.80809211730957,
      "activations/layer10_attention_weight_max": 41.18871307373047,
      "activations/layer10_attention_weight_min": -36.34535598754883,
      "activations/layer11_attention_weight_max": 32.51849365234375,
      "activations/layer11_attention_weight_min": -29.40118408203125,
      "activations/layer12_attention_weight_max": 26.787599563598633,
      "activations/layer12_attention_weight_min": -25.339160919189453,
      "activations/layer13_attention_weight_max": 49.959110260009766,
      "activations/layer13_attention_weight_min": -34.63993835449219,
      "activations/layer14_attention_weight_max": 38.32426071166992,
      "activations/layer14_attention_weight_min": -28.648717880249023,
      "activations/layer15_attention_weight_max": 41.16069030761719,
      "activations/layer15_attention_weight_min": -29.849027633666992,
      "activations/layer16_attention_weight_max": 36.47956085205078,
      "activations/layer16_attention_weight_min": -26.150131225585938,
      "activations/layer17_attention_weight_max": 60.18796920776367,
      "activations/layer17_attention_weight_min": -40.4296760559082,
      "activations/layer18_attention_weight_max": 53.96213912963867,
      "activations/layer18_attention_weight_min": -33.589813232421875,
      "activations/layer19_attention_weight_max": 22.794836044311523,
      "activations/layer19_attention_weight_min": -24.3976993560791,
      "activations/layer1_attention_weight_max": 17.385839462280273,
      "activations/layer1_attention_weight_min": -16.637676239013672,
      "activations/layer20_attention_weight_max": 21.917118072509766,
      "activations/layer20_attention_weight_min": -23.050561904907227,
      "activations/layer21_attention_weight_max": 45.91588592529297,
      "activations/layer21_attention_weight_min": -26.35637092590332,
      "activations/layer22_attention_weight_max": 40.147823333740234,
      "activations/layer22_attention_weight_min": -24.633535385131836,
      "activations/layer23_attention_weight_max": 32.71410369873047,
      "activations/layer23_attention_weight_min": -18.62066650390625,
      "activations/layer2_attention_weight_max": 32.963111877441406,
      "activations/layer2_attention_weight_min": -30.170835494995117,
      "activations/layer3_attention_weight_max": 85.16719055175781,
      "activations/layer3_attention_weight_min": -91.08226776123047,
      "activations/layer4_attention_weight_max": 105.32518768310547,
      "activations/layer4_attention_weight_min": -103.91905975341797,
      "activations/layer5_attention_weight_max": 102.95677947998047,
      "activations/layer5_attention_weight_min": -97.26263427734375,
      "activations/layer6_attention_weight_max": 60.41984939575195,
      "activations/layer6_attention_weight_min": -56.797935485839844,
      "activations/layer7_attention_weight_max": 81.22157287597656,
      "activations/layer7_attention_weight_min": -70.88595581054688,
      "activations/layer8_attention_weight_max": 46.869972229003906,
      "activations/layer8_attention_weight_min": -44.96597671508789,
      "activations/layer9_attention_weight_max": 54.08783721923828,
      "activations/layer9_attention_weight_min": -50.73347473144531,
      "epoch": 5.6,
      "learning_rate": 0.00011503598484848484,
      "loss": 2.8403,
      "step": 96400
    },
    {
      "activations/layer0_attention_weight_max": 13.559187889099121,
      "activations/layer0_attention_weight_min": -15.217097282409668,
      "activations/layer10_attention_weight_max": 38.479766845703125,
      "activations/layer10_attention_weight_min": -37.49774932861328,
      "activations/layer11_attention_weight_max": 32.38114929199219,
      "activations/layer11_attention_weight_min": -28.094932556152344,
      "activations/layer12_attention_weight_max": 34.05867385864258,
      "activations/layer12_attention_weight_min": -26.73847007751465,
      "activations/layer13_attention_weight_max": 50.91522979736328,
      "activations/layer13_attention_weight_min": -37.06279754638672,
      "activations/layer14_attention_weight_max": 39.366397857666016,
      "activations/layer14_attention_weight_min": -27.86433219909668,
      "activations/layer15_attention_weight_max": 43.32950210571289,
      "activations/layer15_attention_weight_min": -28.649019241333008,
      "activations/layer16_attention_weight_max": 33.82914352416992,
      "activations/layer16_attention_weight_min": -25.122726440429688,
      "activations/layer17_attention_weight_max": 60.91889190673828,
      "activations/layer17_attention_weight_min": -36.8378791809082,
      "activations/layer18_attention_weight_max": 45.55089569091797,
      "activations/layer18_attention_weight_min": -32.888309478759766,
      "activations/layer19_attention_weight_max": 22.658872604370117,
      "activations/layer19_attention_weight_min": -20.13911247253418,
      "activations/layer1_attention_weight_max": 18.427824020385742,
      "activations/layer1_attention_weight_min": -16.025054931640625,
      "activations/layer20_attention_weight_max": 22.283885955810547,
      "activations/layer20_attention_weight_min": -21.626760482788086,
      "activations/layer21_attention_weight_max": 39.68659973144531,
      "activations/layer21_attention_weight_min": -23.32181167602539,
      "activations/layer22_attention_weight_max": 31.298431396484375,
      "activations/layer22_attention_weight_min": -23.723867416381836,
      "activations/layer23_attention_weight_max": 29.190549850463867,
      "activations/layer23_attention_weight_min": -18.433576583862305,
      "activations/layer2_attention_weight_max": 30.720266342163086,
      "activations/layer2_attention_weight_min": -28.272756576538086,
      "activations/layer3_attention_weight_max": 81.67276000976562,
      "activations/layer3_attention_weight_min": -89.11195373535156,
      "activations/layer4_attention_weight_max": 100.57235717773438,
      "activations/layer4_attention_weight_min": -99.45084381103516,
      "activations/layer5_attention_weight_max": 93.44970703125,
      "activations/layer5_attention_weight_min": -94.20736694335938,
      "activations/layer6_attention_weight_max": 56.22459030151367,
      "activations/layer6_attention_weight_min": -53.3941535949707,
      "activations/layer7_attention_weight_max": 69.95835876464844,
      "activations/layer7_attention_weight_min": -70.4178466796875,
      "activations/layer8_attention_weight_max": 41.95589065551758,
      "activations/layer8_attention_weight_min": -41.807594299316406,
      "activations/layer9_attention_weight_max": 49.93260955810547,
      "activations/layer9_attention_weight_min": -46.43595886230469,
      "epoch": 5.6,
      "learning_rate": 0.00011501704545454544,
      "loss": 2.8406,
      "step": 96450
    },
    {
      "activations/layer0_attention_weight_max": 14.855850219726562,
      "activations/layer0_attention_weight_min": -14.452415466308594,
      "activations/layer10_attention_weight_max": 39.6299934387207,
      "activations/layer10_attention_weight_min": -33.49153518676758,
      "activations/layer11_attention_weight_max": 29.638168334960938,
      "activations/layer11_attention_weight_min": -29.574338912963867,
      "activations/layer12_attention_weight_max": 23.43437957763672,
      "activations/layer12_attention_weight_min": -26.989965438842773,
      "activations/layer13_attention_weight_max": 54.41564178466797,
      "activations/layer13_attention_weight_min": -38.23119354248047,
      "activations/layer14_attention_weight_max": 48.04435729980469,
      "activations/layer14_attention_weight_min": -34.49966812133789,
      "activations/layer15_attention_weight_max": 37.5740852355957,
      "activations/layer15_attention_weight_min": -32.897247314453125,
      "activations/layer16_attention_weight_max": 32.769554138183594,
      "activations/layer16_attention_weight_min": -27.480350494384766,
      "activations/layer17_attention_weight_max": 52.06875991821289,
      "activations/layer17_attention_weight_min": -40.7232666015625,
      "activations/layer18_attention_weight_max": 45.101078033447266,
      "activations/layer18_attention_weight_min": -33.18013381958008,
      "activations/layer19_attention_weight_max": 18.8533992767334,
      "activations/layer19_attention_weight_min": -21.670120239257812,
      "activations/layer1_attention_weight_max": 17.566064834594727,
      "activations/layer1_attention_weight_min": -17.078140258789062,
      "activations/layer20_attention_weight_max": 22.249492645263672,
      "activations/layer20_attention_weight_min": -22.14820098876953,
      "activations/layer21_attention_weight_max": 41.905582427978516,
      "activations/layer21_attention_weight_min": -26.46969985961914,
      "activations/layer22_attention_weight_max": 32.96145248413086,
      "activations/layer22_attention_weight_min": -26.072988510131836,
      "activations/layer23_attention_weight_max": 28.344547271728516,
      "activations/layer23_attention_weight_min": -18.116586685180664,
      "activations/layer2_attention_weight_max": 33.512603759765625,
      "activations/layer2_attention_weight_min": -29.527618408203125,
      "activations/layer3_attention_weight_max": 84.56141662597656,
      "activations/layer3_attention_weight_min": -88.9006576538086,
      "activations/layer4_attention_weight_max": 103.42431640625,
      "activations/layer4_attention_weight_min": -98.1109848022461,
      "activations/layer5_attention_weight_max": 98.80644226074219,
      "activations/layer5_attention_weight_min": -86.47486877441406,
      "activations/layer6_attention_weight_max": 58.67469024658203,
      "activations/layer6_attention_weight_min": -51.43484878540039,
      "activations/layer7_attention_weight_max": 68.8081283569336,
      "activations/layer7_attention_weight_min": -68.70886993408203,
      "activations/layer8_attention_weight_max": 41.514766693115234,
      "activations/layer8_attention_weight_min": -42.27509307861328,
      "activations/layer9_attention_weight_max": 50.00490951538086,
      "activations/layer9_attention_weight_min": -45.90983200073242,
      "epoch": 5.61,
      "learning_rate": 0.00011499810606060604,
      "loss": 2.8463,
      "step": 96500
    },
    {
      "activations/layer0_attention_weight_max": 14.236804008483887,
      "activations/layer0_attention_weight_min": -14.407760620117188,
      "activations/layer10_attention_weight_max": 35.32994079589844,
      "activations/layer10_attention_weight_min": -33.23683166503906,
      "activations/layer11_attention_weight_max": 33.21965026855469,
      "activations/layer11_attention_weight_min": -28.800743103027344,
      "activations/layer12_attention_weight_max": 22.64531898498535,
      "activations/layer12_attention_weight_min": -26.98444938659668,
      "activations/layer13_attention_weight_max": 50.421546936035156,
      "activations/layer13_attention_weight_min": -34.2933464050293,
      "activations/layer14_attention_weight_max": 45.53718566894531,
      "activations/layer14_attention_weight_min": -34.134178161621094,
      "activations/layer15_attention_weight_max": 39.90666580200195,
      "activations/layer15_attention_weight_min": -30.92498016357422,
      "activations/layer16_attention_weight_max": 35.28403091430664,
      "activations/layer16_attention_weight_min": -26.799081802368164,
      "activations/layer17_attention_weight_max": 60.86729049682617,
      "activations/layer17_attention_weight_min": -41.19586944580078,
      "activations/layer18_attention_weight_max": 50.83753204345703,
      "activations/layer18_attention_weight_min": -34.26518249511719,
      "activations/layer19_attention_weight_max": 25.59522247314453,
      "activations/layer19_attention_weight_min": -21.791736602783203,
      "activations/layer1_attention_weight_max": 17.165191650390625,
      "activations/layer1_attention_weight_min": -13.784616470336914,
      "activations/layer20_attention_weight_max": 23.225482940673828,
      "activations/layer20_attention_weight_min": -20.27751922607422,
      "activations/layer21_attention_weight_max": 45.75705337524414,
      "activations/layer21_attention_weight_min": -26.73671531677246,
      "activations/layer22_attention_weight_max": 37.77018737792969,
      "activations/layer22_attention_weight_min": -23.686492919921875,
      "activations/layer23_attention_weight_max": 34.682945251464844,
      "activations/layer23_attention_weight_min": -18.96709632873535,
      "activations/layer2_attention_weight_max": 28.950424194335938,
      "activations/layer2_attention_weight_min": -27.01735496520996,
      "activations/layer3_attention_weight_max": 80.9549789428711,
      "activations/layer3_attention_weight_min": -83.32488250732422,
      "activations/layer4_attention_weight_max": 100.88522338867188,
      "activations/layer4_attention_weight_min": -96.0819320678711,
      "activations/layer5_attention_weight_max": 92.07833099365234,
      "activations/layer5_attention_weight_min": -86.90132141113281,
      "activations/layer6_attention_weight_max": 54.352394104003906,
      "activations/layer6_attention_weight_min": -50.03739929199219,
      "activations/layer7_attention_weight_max": 67.95085906982422,
      "activations/layer7_attention_weight_min": -65.83544158935547,
      "activations/layer8_attention_weight_max": 43.53279495239258,
      "activations/layer8_attention_weight_min": -42.201332092285156,
      "activations/layer9_attention_weight_max": 58.455726623535156,
      "activations/layer9_attention_weight_min": -49.58740234375,
      "epoch": 5.61,
      "learning_rate": 0.00011497916666666666,
      "loss": 2.8316,
      "step": 96550
    },
    {
      "activations/layer0_attention_weight_max": 14.343462944030762,
      "activations/layer0_attention_weight_min": -15.328755378723145,
      "activations/layer10_attention_weight_max": 38.23251724243164,
      "activations/layer10_attention_weight_min": -35.598785400390625,
      "activations/layer11_attention_weight_max": 32.67752456665039,
      "activations/layer11_attention_weight_min": -27.851964950561523,
      "activations/layer12_attention_weight_max": 24.16450309753418,
      "activations/layer12_attention_weight_min": -28.841712951660156,
      "activations/layer13_attention_weight_max": 39.98686981201172,
      "activations/layer13_attention_weight_min": -33.93436813354492,
      "activations/layer14_attention_weight_max": 40.494354248046875,
      "activations/layer14_attention_weight_min": -31.18134880065918,
      "activations/layer15_attention_weight_max": 36.79088592529297,
      "activations/layer15_attention_weight_min": -30.040904998779297,
      "activations/layer16_attention_weight_max": 29.773771286010742,
      "activations/layer16_attention_weight_min": -26.50408363342285,
      "activations/layer17_attention_weight_max": 58.17978286743164,
      "activations/layer17_attention_weight_min": -42.285491943359375,
      "activations/layer18_attention_weight_max": 51.44941329956055,
      "activations/layer18_attention_weight_min": -33.13185119628906,
      "activations/layer19_attention_weight_max": 21.440141677856445,
      "activations/layer19_attention_weight_min": -19.007741928100586,
      "activations/layer1_attention_weight_max": 17.587604522705078,
      "activations/layer1_attention_weight_min": -16.57345962524414,
      "activations/layer20_attention_weight_max": 23.059003829956055,
      "activations/layer20_attention_weight_min": -20.096336364746094,
      "activations/layer21_attention_weight_max": 45.11310577392578,
      "activations/layer21_attention_weight_min": -23.660451889038086,
      "activations/layer22_attention_weight_max": 34.97899627685547,
      "activations/layer22_attention_weight_min": -23.545047760009766,
      "activations/layer23_attention_weight_max": 33.021514892578125,
      "activations/layer23_attention_weight_min": -17.97627830505371,
      "activations/layer2_attention_weight_max": 29.679044723510742,
      "activations/layer2_attention_weight_min": -28.037818908691406,
      "activations/layer3_attention_weight_max": 82.9429931640625,
      "activations/layer3_attention_weight_min": -85.80294036865234,
      "activations/layer4_attention_weight_max": 101.62628936767578,
      "activations/layer4_attention_weight_min": -97.20043182373047,
      "activations/layer5_attention_weight_max": 97.96405029296875,
      "activations/layer5_attention_weight_min": -88.18758392333984,
      "activations/layer6_attention_weight_max": 57.81517791748047,
      "activations/layer6_attention_weight_min": -53.13034439086914,
      "activations/layer7_attention_weight_max": 71.91996765136719,
      "activations/layer7_attention_weight_min": -68.91585540771484,
      "activations/layer8_attention_weight_max": 42.61402130126953,
      "activations/layer8_attention_weight_min": -41.63780975341797,
      "activations/layer9_attention_weight_max": 48.64432907104492,
      "activations/layer9_attention_weight_min": -48.17059326171875,
      "epoch": 5.61,
      "learning_rate": 0.00011496022727272726,
      "loss": 2.8416,
      "step": 96600
    },
    {
      "activations/layer0_attention_weight_max": 14.897293090820312,
      "activations/layer0_attention_weight_min": -15.163320541381836,
      "activations/layer10_attention_weight_max": 38.47425079345703,
      "activations/layer10_attention_weight_min": -33.44887924194336,
      "activations/layer11_attention_weight_max": 29.244510650634766,
      "activations/layer11_attention_weight_min": -26.740638732910156,
      "activations/layer12_attention_weight_max": 21.66165542602539,
      "activations/layer12_attention_weight_min": -25.985620498657227,
      "activations/layer13_attention_weight_max": 53.68851852416992,
      "activations/layer13_attention_weight_min": -37.13528060913086,
      "activations/layer14_attention_weight_max": 38.22473907470703,
      "activations/layer14_attention_weight_min": -28.46509552001953,
      "activations/layer15_attention_weight_max": 38.028045654296875,
      "activations/layer15_attention_weight_min": -29.428726196289062,
      "activations/layer16_attention_weight_max": 35.51751708984375,
      "activations/layer16_attention_weight_min": -26.970809936523438,
      "activations/layer17_attention_weight_max": 57.83576965332031,
      "activations/layer17_attention_weight_min": -39.70901870727539,
      "activations/layer18_attention_weight_max": 50.73530960083008,
      "activations/layer18_attention_weight_min": -35.258880615234375,
      "activations/layer19_attention_weight_max": 29.314014434814453,
      "activations/layer19_attention_weight_min": -18.748371124267578,
      "activations/layer1_attention_weight_max": 17.92284393310547,
      "activations/layer1_attention_weight_min": -18.555349349975586,
      "activations/layer20_attention_weight_max": 25.087474822998047,
      "activations/layer20_attention_weight_min": -20.059049606323242,
      "activations/layer21_attention_weight_max": 38.73692321777344,
      "activations/layer21_attention_weight_min": -25.53447151184082,
      "activations/layer22_attention_weight_max": 35.477237701416016,
      "activations/layer22_attention_weight_min": -24.30197525024414,
      "activations/layer23_attention_weight_max": 29.236278533935547,
      "activations/layer23_attention_weight_min": -20.98924446105957,
      "activations/layer2_attention_weight_max": 28.233200073242188,
      "activations/layer2_attention_weight_min": -27.34463882446289,
      "activations/layer3_attention_weight_max": 79.81632995605469,
      "activations/layer3_attention_weight_min": -84.96853637695312,
      "activations/layer4_attention_weight_max": 98.94044494628906,
      "activations/layer4_attention_weight_min": -91.77046966552734,
      "activations/layer5_attention_weight_max": 95.22148895263672,
      "activations/layer5_attention_weight_min": -87.72885131835938,
      "activations/layer6_attention_weight_max": 55.87782669067383,
      "activations/layer6_attention_weight_min": -52.622474670410156,
      "activations/layer7_attention_weight_max": 69.15815734863281,
      "activations/layer7_attention_weight_min": -64.50858306884766,
      "activations/layer8_attention_weight_max": 42.98365020751953,
      "activations/layer8_attention_weight_min": -42.74144744873047,
      "activations/layer9_attention_weight_max": 51.142574310302734,
      "activations/layer9_attention_weight_min": -47.12880325317383,
      "epoch": 5.62,
      "learning_rate": 0.00011494128787878787,
      "loss": 2.8472,
      "step": 96650
    },
    {
      "activations/layer0_attention_weight_max": 14.253480911254883,
      "activations/layer0_attention_weight_min": -14.943743705749512,
      "activations/layer10_attention_weight_max": 36.036014556884766,
      "activations/layer10_attention_weight_min": -34.806121826171875,
      "activations/layer11_attention_weight_max": 28.828201293945312,
      "activations/layer11_attention_weight_min": -28.963176727294922,
      "activations/layer12_attention_weight_max": 20.506452560424805,
      "activations/layer12_attention_weight_min": -25.228116989135742,
      "activations/layer13_attention_weight_max": 53.06807327270508,
      "activations/layer13_attention_weight_min": -41.55180740356445,
      "activations/layer14_attention_weight_max": 42.451297760009766,
      "activations/layer14_attention_weight_min": -30.750715255737305,
      "activations/layer15_attention_weight_max": 37.64041519165039,
      "activations/layer15_attention_weight_min": -29.85265350341797,
      "activations/layer16_attention_weight_max": 39.31139373779297,
      "activations/layer16_attention_weight_min": -26.79836082458496,
      "activations/layer17_attention_weight_max": 54.983272552490234,
      "activations/layer17_attention_weight_min": -41.01008224487305,
      "activations/layer18_attention_weight_max": 45.47323226928711,
      "activations/layer18_attention_weight_min": -33.68400573730469,
      "activations/layer19_attention_weight_max": 24.088336944580078,
      "activations/layer19_attention_weight_min": -20.100873947143555,
      "activations/layer1_attention_weight_max": 18.311073303222656,
      "activations/layer1_attention_weight_min": -16.82770347595215,
      "activations/layer20_attention_weight_max": 24.390762329101562,
      "activations/layer20_attention_weight_min": -23.626609802246094,
      "activations/layer21_attention_weight_max": 38.75397491455078,
      "activations/layer21_attention_weight_min": -25.371362686157227,
      "activations/layer22_attention_weight_max": 41.5697021484375,
      "activations/layer22_attention_weight_min": -25.169710159301758,
      "activations/layer23_attention_weight_max": 28.871658325195312,
      "activations/layer23_attention_weight_min": -21.93692398071289,
      "activations/layer2_attention_weight_max": 32.73833084106445,
      "activations/layer2_attention_weight_min": -31.849811553955078,
      "activations/layer3_attention_weight_max": 87.85697174072266,
      "activations/layer3_attention_weight_min": -91.78821563720703,
      "activations/layer4_attention_weight_max": 100.73394775390625,
      "activations/layer4_attention_weight_min": -94.59820556640625,
      "activations/layer5_attention_weight_max": 99.56966400146484,
      "activations/layer5_attention_weight_min": -88.86782836914062,
      "activations/layer6_attention_weight_max": 57.72832489013672,
      "activations/layer6_attention_weight_min": -51.25859069824219,
      "activations/layer7_attention_weight_max": 72.44021606445312,
      "activations/layer7_attention_weight_min": -66.52684020996094,
      "activations/layer8_attention_weight_max": 45.64632034301758,
      "activations/layer8_attention_weight_min": -44.33832931518555,
      "activations/layer9_attention_weight_max": 50.72745895385742,
      "activations/layer9_attention_weight_min": -47.6878662109375,
      "epoch": 5.62,
      "learning_rate": 0.00011492234848484848,
      "loss": 2.8476,
      "step": 96700
    },
    {
      "activations/layer0_attention_weight_max": 15.297392845153809,
      "activations/layer0_attention_weight_min": -14.839892387390137,
      "activations/layer10_attention_weight_max": 40.75776290893555,
      "activations/layer10_attention_weight_min": -37.632144927978516,
      "activations/layer11_attention_weight_max": 36.64652633666992,
      "activations/layer11_attention_weight_min": -27.15545654296875,
      "activations/layer12_attention_weight_max": 29.21393585205078,
      "activations/layer12_attention_weight_min": -27.58584976196289,
      "activations/layer13_attention_weight_max": 45.982444763183594,
      "activations/layer13_attention_weight_min": -35.02220916748047,
      "activations/layer14_attention_weight_max": 55.14306640625,
      "activations/layer14_attention_weight_min": -32.72411346435547,
      "activations/layer15_attention_weight_max": 37.71438217163086,
      "activations/layer15_attention_weight_min": -31.019298553466797,
      "activations/layer16_attention_weight_max": 33.866615295410156,
      "activations/layer16_attention_weight_min": -25.802494049072266,
      "activations/layer17_attention_weight_max": 66.35565185546875,
      "activations/layer17_attention_weight_min": -38.94035720825195,
      "activations/layer18_attention_weight_max": 48.23189163208008,
      "activations/layer18_attention_weight_min": -33.96853256225586,
      "activations/layer19_attention_weight_max": 22.282161712646484,
      "activations/layer19_attention_weight_min": -19.249408721923828,
      "activations/layer1_attention_weight_max": 17.760976791381836,
      "activations/layer1_attention_weight_min": -15.946480751037598,
      "activations/layer20_attention_weight_max": 22.905981063842773,
      "activations/layer20_attention_weight_min": -21.051549911499023,
      "activations/layer21_attention_weight_max": 41.57611083984375,
      "activations/layer21_attention_weight_min": -24.89871597290039,
      "activations/layer22_attention_weight_max": 36.16667556762695,
      "activations/layer22_attention_weight_min": -22.79586410522461,
      "activations/layer23_attention_weight_max": 32.87666320800781,
      "activations/layer23_attention_weight_min": -17.92519760131836,
      "activations/layer2_attention_weight_max": 31.885784149169922,
      "activations/layer2_attention_weight_min": -31.19191551208496,
      "activations/layer3_attention_weight_max": 83.14535522460938,
      "activations/layer3_attention_weight_min": -88.54814910888672,
      "activations/layer4_attention_weight_max": 104.62945556640625,
      "activations/layer4_attention_weight_min": -103.86083984375,
      "activations/layer5_attention_weight_max": 101.0201644897461,
      "activations/layer5_attention_weight_min": -95.7542953491211,
      "activations/layer6_attention_weight_max": 57.71339416503906,
      "activations/layer6_attention_weight_min": -56.53092575073242,
      "activations/layer7_attention_weight_max": 71.56584167480469,
      "activations/layer7_attention_weight_min": -76.0970687866211,
      "activations/layer8_attention_weight_max": 45.64169692993164,
      "activations/layer8_attention_weight_min": -44.7538948059082,
      "activations/layer9_attention_weight_max": 50.399314880371094,
      "activations/layer9_attention_weight_min": -49.41036605834961,
      "epoch": 5.62,
      "learning_rate": 0.00011490340909090908,
      "loss": 2.8527,
      "step": 96750
    },
    {
      "activations/layer0_attention_weight_max": 13.894771575927734,
      "activations/layer0_attention_weight_min": -14.441986083984375,
      "activations/layer10_attention_weight_max": 36.59566879272461,
      "activations/layer10_attention_weight_min": -34.076175689697266,
      "activations/layer11_attention_weight_max": 29.972564697265625,
      "activations/layer11_attention_weight_min": -28.260799407958984,
      "activations/layer12_attention_weight_max": 21.11594009399414,
      "activations/layer12_attention_weight_min": -23.557695388793945,
      "activations/layer13_attention_weight_max": 46.26493453979492,
      "activations/layer13_attention_weight_min": -33.27028274536133,
      "activations/layer14_attention_weight_max": 42.03084182739258,
      "activations/layer14_attention_weight_min": -30.117420196533203,
      "activations/layer15_attention_weight_max": 40.41545104980469,
      "activations/layer15_attention_weight_min": -31.35932731628418,
      "activations/layer16_attention_weight_max": 33.69783401489258,
      "activations/layer16_attention_weight_min": -26.243764877319336,
      "activations/layer17_attention_weight_max": 51.84603500366211,
      "activations/layer17_attention_weight_min": -40.25510025024414,
      "activations/layer18_attention_weight_max": 46.6129035949707,
      "activations/layer18_attention_weight_min": -35.980323791503906,
      "activations/layer19_attention_weight_max": 27.95208168029785,
      "activations/layer19_attention_weight_min": -20.84994125366211,
      "activations/layer1_attention_weight_max": 17.847274780273438,
      "activations/layer1_attention_weight_min": -14.031712532043457,
      "activations/layer20_attention_weight_max": 24.852705001831055,
      "activations/layer20_attention_weight_min": -23.183387756347656,
      "activations/layer21_attention_weight_max": 41.88663864135742,
      "activations/layer21_attention_weight_min": -27.595415115356445,
      "activations/layer22_attention_weight_max": 33.85749435424805,
      "activations/layer22_attention_weight_min": -25.657087326049805,
      "activations/layer23_attention_weight_max": 27.9583740234375,
      "activations/layer23_attention_weight_min": -19.665233612060547,
      "activations/layer2_attention_weight_max": 31.646194458007812,
      "activations/layer2_attention_weight_min": -29.388484954833984,
      "activations/layer3_attention_weight_max": 83.83028411865234,
      "activations/layer3_attention_weight_min": -91.20377349853516,
      "activations/layer4_attention_weight_max": 98.49341583251953,
      "activations/layer4_attention_weight_min": -94.26155090332031,
      "activations/layer5_attention_weight_max": 96.71156311035156,
      "activations/layer5_attention_weight_min": -86.48973083496094,
      "activations/layer6_attention_weight_max": 60.85291290283203,
      "activations/layer6_attention_weight_min": -51.079627990722656,
      "activations/layer7_attention_weight_max": 69.46994018554688,
      "activations/layer7_attention_weight_min": -67.0303955078125,
      "activations/layer8_attention_weight_max": 41.58016586303711,
      "activations/layer8_attention_weight_min": -41.42974090576172,
      "activations/layer9_attention_weight_max": 49.262535095214844,
      "activations/layer9_attention_weight_min": -47.654701232910156,
      "epoch": 5.62,
      "learning_rate": 0.00011488446969696969,
      "loss": 2.8411,
      "step": 96800
    },
    {
      "activations/layer0_attention_weight_max": 14.730548858642578,
      "activations/layer0_attention_weight_min": -14.82360553741455,
      "activations/layer10_attention_weight_max": 39.07548141479492,
      "activations/layer10_attention_weight_min": -35.59820556640625,
      "activations/layer11_attention_weight_max": 29.182493209838867,
      "activations/layer11_attention_weight_min": -28.09564781188965,
      "activations/layer12_attention_weight_max": 22.224857330322266,
      "activations/layer12_attention_weight_min": -26.058061599731445,
      "activations/layer13_attention_weight_max": 52.28791046142578,
      "activations/layer13_attention_weight_min": -35.48307800292969,
      "activations/layer14_attention_weight_max": 51.08744812011719,
      "activations/layer14_attention_weight_min": -31.63275146484375,
      "activations/layer15_attention_weight_max": 42.39480972290039,
      "activations/layer15_attention_weight_min": -33.41083526611328,
      "activations/layer16_attention_weight_max": 37.05849075317383,
      "activations/layer16_attention_weight_min": -27.828767776489258,
      "activations/layer17_attention_weight_max": 61.20140838623047,
      "activations/layer17_attention_weight_min": -40.356483459472656,
      "activations/layer18_attention_weight_max": 50.22319793701172,
      "activations/layer18_attention_weight_min": -35.629730224609375,
      "activations/layer19_attention_weight_max": 25.31490135192871,
      "activations/layer19_attention_weight_min": -21.117576599121094,
      "activations/layer1_attention_weight_max": 17.248981475830078,
      "activations/layer1_attention_weight_min": -15.64237117767334,
      "activations/layer20_attention_weight_max": 26.375627517700195,
      "activations/layer20_attention_weight_min": -21.95512580871582,
      "activations/layer21_attention_weight_max": 44.54020309448242,
      "activations/layer21_attention_weight_min": -27.062339782714844,
      "activations/layer22_attention_weight_max": 34.956356048583984,
      "activations/layer22_attention_weight_min": -26.88044548034668,
      "activations/layer23_attention_weight_max": 32.28443908691406,
      "activations/layer23_attention_weight_min": -19.53483772277832,
      "activations/layer2_attention_weight_max": 29.018909454345703,
      "activations/layer2_attention_weight_min": -27.45732879638672,
      "activations/layer3_attention_weight_max": 80.30240631103516,
      "activations/layer3_attention_weight_min": -85.04255676269531,
      "activations/layer4_attention_weight_max": 101.22473907470703,
      "activations/layer4_attention_weight_min": -97.82369995117188,
      "activations/layer5_attention_weight_max": 95.29808044433594,
      "activations/layer5_attention_weight_min": -90.379638671875,
      "activations/layer6_attention_weight_max": 56.55104064941406,
      "activations/layer6_attention_weight_min": -54.73072052001953,
      "activations/layer7_attention_weight_max": 70.67193603515625,
      "activations/layer7_attention_weight_min": -67.58235931396484,
      "activations/layer8_attention_weight_max": 42.91133499145508,
      "activations/layer8_attention_weight_min": -46.081336975097656,
      "activations/layer9_attention_weight_max": 49.55807876586914,
      "activations/layer9_attention_weight_min": -48.86373519897461,
      "epoch": 5.63,
      "learning_rate": 0.0001148655303030303,
      "loss": 2.8505,
      "step": 96850
    },
    {
      "activations/layer0_attention_weight_max": 14.093381881713867,
      "activations/layer0_attention_weight_min": -14.8551607131958,
      "activations/layer10_attention_weight_max": 38.5343017578125,
      "activations/layer10_attention_weight_min": -34.760955810546875,
      "activations/layer11_attention_weight_max": 31.09949493408203,
      "activations/layer11_attention_weight_min": -26.730499267578125,
      "activations/layer12_attention_weight_max": 23.29229164123535,
      "activations/layer12_attention_weight_min": -23.401132583618164,
      "activations/layer13_attention_weight_max": 55.28075408935547,
      "activations/layer13_attention_weight_min": -35.9285888671875,
      "activations/layer14_attention_weight_max": 46.09797668457031,
      "activations/layer14_attention_weight_min": -29.707714080810547,
      "activations/layer15_attention_weight_max": 40.567832946777344,
      "activations/layer15_attention_weight_min": -31.728422164916992,
      "activations/layer16_attention_weight_max": 37.17442321777344,
      "activations/layer16_attention_weight_min": -25.763591766357422,
      "activations/layer17_attention_weight_max": 57.21096420288086,
      "activations/layer17_attention_weight_min": -42.481571197509766,
      "activations/layer18_attention_weight_max": 48.944881439208984,
      "activations/layer18_attention_weight_min": -34.56398391723633,
      "activations/layer19_attention_weight_max": 24.585886001586914,
      "activations/layer19_attention_weight_min": -19.011795043945312,
      "activations/layer1_attention_weight_max": 17.99058723449707,
      "activations/layer1_attention_weight_min": -19.435749053955078,
      "activations/layer20_attention_weight_max": 21.6025447845459,
      "activations/layer20_attention_weight_min": -24.40151596069336,
      "activations/layer21_attention_weight_max": 37.4024658203125,
      "activations/layer21_attention_weight_min": -27.297842025756836,
      "activations/layer22_attention_weight_max": 37.483543395996094,
      "activations/layer22_attention_weight_min": -24.802474975585938,
      "activations/layer23_attention_weight_max": 30.751676559448242,
      "activations/layer23_attention_weight_min": -19.93024444580078,
      "activations/layer2_attention_weight_max": 30.547332763671875,
      "activations/layer2_attention_weight_min": -30.142196655273438,
      "activations/layer3_attention_weight_max": 82.2198486328125,
      "activations/layer3_attention_weight_min": -85.6285629272461,
      "activations/layer4_attention_weight_max": 99.88594818115234,
      "activations/layer4_attention_weight_min": -97.40062713623047,
      "activations/layer5_attention_weight_max": 93.97967529296875,
      "activations/layer5_attention_weight_min": -90.18605041503906,
      "activations/layer6_attention_weight_max": 55.22074508666992,
      "activations/layer6_attention_weight_min": -53.56032943725586,
      "activations/layer7_attention_weight_max": 70.6448745727539,
      "activations/layer7_attention_weight_min": -69.51142120361328,
      "activations/layer8_attention_weight_max": 41.287837982177734,
      "activations/layer8_attention_weight_min": -40.993038177490234,
      "activations/layer9_attention_weight_max": 48.19814682006836,
      "activations/layer9_attention_weight_min": -49.941062927246094,
      "epoch": 5.63,
      "learning_rate": 0.0001148465909090909,
      "loss": 2.8331,
      "step": 96900
    },
    {
      "activations/layer0_attention_weight_max": 14.962875366210938,
      "activations/layer0_attention_weight_min": -14.93062686920166,
      "activations/layer10_attention_weight_max": 35.2611083984375,
      "activations/layer10_attention_weight_min": -33.63527297973633,
      "activations/layer11_attention_weight_max": 27.274749755859375,
      "activations/layer11_attention_weight_min": -27.978429794311523,
      "activations/layer12_attention_weight_max": 19.691198348999023,
      "activations/layer12_attention_weight_min": -25.90880584716797,
      "activations/layer13_attention_weight_max": 39.170589447021484,
      "activations/layer13_attention_weight_min": -33.07960510253906,
      "activations/layer14_attention_weight_max": 35.792877197265625,
      "activations/layer14_attention_weight_min": -29.24408721923828,
      "activations/layer15_attention_weight_max": 32.22463607788086,
      "activations/layer15_attention_weight_min": -29.54903221130371,
      "activations/layer16_attention_weight_max": 30.93849754333496,
      "activations/layer16_attention_weight_min": -26.34984588623047,
      "activations/layer17_attention_weight_max": 46.86664581298828,
      "activations/layer17_attention_weight_min": -41.790313720703125,
      "activations/layer18_attention_weight_max": 42.260135650634766,
      "activations/layer18_attention_weight_min": -31.860088348388672,
      "activations/layer19_attention_weight_max": 18.289230346679688,
      "activations/layer19_attention_weight_min": -19.072072982788086,
      "activations/layer1_attention_weight_max": 18.004667282104492,
      "activations/layer1_attention_weight_min": -14.829733848571777,
      "activations/layer20_attention_weight_max": 18.6950740814209,
      "activations/layer20_attention_weight_min": -19.552553176879883,
      "activations/layer21_attention_weight_max": 30.58237075805664,
      "activations/layer21_attention_weight_min": -23.996612548828125,
      "activations/layer22_attention_weight_max": 32.28258514404297,
      "activations/layer22_attention_weight_min": -23.966211318969727,
      "activations/layer23_attention_weight_max": 25.415760040283203,
      "activations/layer23_attention_weight_min": -19.547752380371094,
      "activations/layer2_attention_weight_max": 29.879413604736328,
      "activations/layer2_attention_weight_min": -26.912107467651367,
      "activations/layer3_attention_weight_max": 80.88549041748047,
      "activations/layer3_attention_weight_min": -81.6250991821289,
      "activations/layer4_attention_weight_max": 101.18412780761719,
      "activations/layer4_attention_weight_min": -91.31352996826172,
      "activations/layer5_attention_weight_max": 99.4007568359375,
      "activations/layer5_attention_weight_min": -86.2140121459961,
      "activations/layer6_attention_weight_max": 54.99409484863281,
      "activations/layer6_attention_weight_min": -52.08548355102539,
      "activations/layer7_attention_weight_max": 68.14248657226562,
      "activations/layer7_attention_weight_min": -67.25086212158203,
      "activations/layer8_attention_weight_max": 40.813011169433594,
      "activations/layer8_attention_weight_min": -43.749149322509766,
      "activations/layer9_attention_weight_max": 49.869300842285156,
      "activations/layer9_attention_weight_min": -47.81428527832031,
      "epoch": 5.63,
      "learning_rate": 0.00011482765151515151,
      "loss": 2.8375,
      "step": 96950
    },
    {
      "activations/layer0_attention_weight_max": 14.965664863586426,
      "activations/layer0_attention_weight_min": -14.975112915039062,
      "activations/layer10_attention_weight_max": 40.32957458496094,
      "activations/layer10_attention_weight_min": -35.37379455566406,
      "activations/layer11_attention_weight_max": 30.118581771850586,
      "activations/layer11_attention_weight_min": -28.042037963867188,
      "activations/layer12_attention_weight_max": 21.251277923583984,
      "activations/layer12_attention_weight_min": -22.683996200561523,
      "activations/layer13_attention_weight_max": 46.140010833740234,
      "activations/layer13_attention_weight_min": -34.65100860595703,
      "activations/layer14_attention_weight_max": 39.013362884521484,
      "activations/layer14_attention_weight_min": -30.951990127563477,
      "activations/layer15_attention_weight_max": 41.236114501953125,
      "activations/layer15_attention_weight_min": -28.395334243774414,
      "activations/layer16_attention_weight_max": 35.554779052734375,
      "activations/layer16_attention_weight_min": -25.885133743286133,
      "activations/layer17_attention_weight_max": 60.57231903076172,
      "activations/layer17_attention_weight_min": -42.499813079833984,
      "activations/layer18_attention_weight_max": 51.73387908935547,
      "activations/layer18_attention_weight_min": -35.40839767456055,
      "activations/layer19_attention_weight_max": 23.590312957763672,
      "activations/layer19_attention_weight_min": -21.15291404724121,
      "activations/layer1_attention_weight_max": 18.834026336669922,
      "activations/layer1_attention_weight_min": -16.377500534057617,
      "activations/layer20_attention_weight_max": 22.92562484741211,
      "activations/layer20_attention_weight_min": -21.838977813720703,
      "activations/layer21_attention_weight_max": 44.64404296875,
      "activations/layer21_attention_weight_min": -25.839487075805664,
      "activations/layer22_attention_weight_max": 37.09171676635742,
      "activations/layer22_attention_weight_min": -25.3619327545166,
      "activations/layer23_attention_weight_max": 32.30207824707031,
      "activations/layer23_attention_weight_min": -21.465744018554688,
      "activations/layer2_attention_weight_max": 30.465656280517578,
      "activations/layer2_attention_weight_min": -30.152454376220703,
      "activations/layer3_attention_weight_max": 78.70999145507812,
      "activations/layer3_attention_weight_min": -90.09132385253906,
      "activations/layer4_attention_weight_max": 100.63890075683594,
      "activations/layer4_attention_weight_min": -104.0075454711914,
      "activations/layer5_attention_weight_max": 97.45303344726562,
      "activations/layer5_attention_weight_min": -89.68446350097656,
      "activations/layer6_attention_weight_max": 58.17793273925781,
      "activations/layer6_attention_weight_min": -52.48165512084961,
      "activations/layer7_attention_weight_max": 69.66344451904297,
      "activations/layer7_attention_weight_min": -70.55628204345703,
      "activations/layer8_attention_weight_max": 45.216068267822266,
      "activations/layer8_attention_weight_min": -43.647071838378906,
      "activations/layer9_attention_weight_max": 50.988922119140625,
      "activations/layer9_attention_weight_min": -47.52632141113281,
      "epoch": 5.64,
      "learning_rate": 0.0001148087121212121,
      "loss": 2.8409,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_loss": 2.79296875,
      "eval_runtime": 8.5231,
      "eval_samples_per_second": 503.806,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_openwebtext_loss": 2.79296875,
      "eval_openwebtext_ppl": 16.329425896569767,
      "eval_openwebtext_runtime": 8.5231,
      "eval_openwebtext_samples_per_second": 503.806,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_wikitext_loss": 3.01953125,
      "eval_wikitext_ppl": 20.4816886421048,
      "eval_wikitext_runtime": 2.0305,
      "eval_wikitext_samples_per_second": 224.575,
      "step": 97000
    },
    {
      "epoch": 5.64,
      "eval_lambada_loss": 2.798828125,
      "eval_lambada_ppl": 16.425386987895497,
      "eval_lambada_runtime": 9.6101,
      "eval_lambada_samples_per_second": 506.652,
      "step": 97000
    },
    {
      "activations/layer0_attention_weight_max": 13.811646461486816,
      "activations/layer0_attention_weight_min": -14.707270622253418,
      "activations/layer10_attention_weight_max": 40.41353225708008,
      "activations/layer10_attention_weight_min": -35.20043182373047,
      "activations/layer11_attention_weight_max": 34.00263214111328,
      "activations/layer11_attention_weight_min": -29.446155548095703,
      "activations/layer12_attention_weight_max": 24.71501350402832,
      "activations/layer12_attention_weight_min": -21.53300666809082,
      "activations/layer13_attention_weight_max": 53.0966911315918,
      "activations/layer13_attention_weight_min": -38.2252311706543,
      "activations/layer14_attention_weight_max": 49.64107894897461,
      "activations/layer14_attention_weight_min": -31.92116928100586,
      "activations/layer15_attention_weight_max": 37.802093505859375,
      "activations/layer15_attention_weight_min": -29.151065826416016,
      "activations/layer16_attention_weight_max": 38.21457290649414,
      "activations/layer16_attention_weight_min": -25.31096076965332,
      "activations/layer17_attention_weight_max": 62.31698226928711,
      "activations/layer17_attention_weight_min": -40.82101821899414,
      "activations/layer18_attention_weight_max": 51.91108703613281,
      "activations/layer18_attention_weight_min": -34.2767448425293,
      "activations/layer19_attention_weight_max": 22.03582763671875,
      "activations/layer19_attention_weight_min": -20.01222801208496,
      "activations/layer1_attention_weight_max": 17.624549865722656,
      "activations/layer1_attention_weight_min": -16.512781143188477,
      "activations/layer20_attention_weight_max": 20.88524627685547,
      "activations/layer20_attention_weight_min": -21.87406349182129,
      "activations/layer21_attention_weight_max": 45.36445999145508,
      "activations/layer21_attention_weight_min": -25.364303588867188,
      "activations/layer22_attention_weight_max": 39.835262298583984,
      "activations/layer22_attention_weight_min": -25.100666046142578,
      "activations/layer23_attention_weight_max": 32.08979797363281,
      "activations/layer23_attention_weight_min": -17.685630798339844,
      "activations/layer2_attention_weight_max": 30.884769439697266,
      "activations/layer2_attention_weight_min": -30.167341232299805,
      "activations/layer3_attention_weight_max": 87.03993225097656,
      "activations/layer3_attention_weight_min": -90.22016143798828,
      "activations/layer4_attention_weight_max": 105.26948547363281,
      "activations/layer4_attention_weight_min": -104.9996566772461,
      "activations/layer5_attention_weight_max": 99.67237854003906,
      "activations/layer5_attention_weight_min": -98.97606658935547,
      "activations/layer6_attention_weight_max": 58.63539505004883,
      "activations/layer6_attention_weight_min": -57.551353454589844,
      "activations/layer7_attention_weight_max": 78.43163299560547,
      "activations/layer7_attention_weight_min": -71.4865951538086,
      "activations/layer8_attention_weight_max": 45.10670852661133,
      "activations/layer8_attention_weight_min": -43.850372314453125,
      "activations/layer9_attention_weight_max": 58.05685043334961,
      "activations/layer9_attention_weight_min": -49.4919548034668,
      "epoch": 5.64,
      "learning_rate": 0.00011478977272727271,
      "loss": 2.8265,
      "step": 97050
    },
    {
      "activations/layer0_attention_weight_max": 14.835301399230957,
      "activations/layer0_attention_weight_min": -15.184609413146973,
      "activations/layer10_attention_weight_max": 37.23445510864258,
      "activations/layer10_attention_weight_min": -32.296634674072266,
      "activations/layer11_attention_weight_max": 29.666175842285156,
      "activations/layer11_attention_weight_min": -27.003881454467773,
      "activations/layer12_attention_weight_max": 19.90177345275879,
      "activations/layer12_attention_weight_min": -22.61894416809082,
      "activations/layer13_attention_weight_max": 42.84480285644531,
      "activations/layer13_attention_weight_min": -31.991724014282227,
      "activations/layer14_attention_weight_max": 37.026641845703125,
      "activations/layer14_attention_weight_min": -29.310312271118164,
      "activations/layer15_attention_weight_max": 36.486854553222656,
      "activations/layer15_attention_weight_min": -28.89891242980957,
      "activations/layer16_attention_weight_max": 32.028167724609375,
      "activations/layer16_attention_weight_min": -27.469472885131836,
      "activations/layer17_attention_weight_max": 53.76436233520508,
      "activations/layer17_attention_weight_min": -38.71709442138672,
      "activations/layer18_attention_weight_max": 46.081085205078125,
      "activations/layer18_attention_weight_min": -33.940093994140625,
      "activations/layer19_attention_weight_max": 21.523544311523438,
      "activations/layer19_attention_weight_min": -19.399131774902344,
      "activations/layer1_attention_weight_max": 17.623231887817383,
      "activations/layer1_attention_weight_min": -15.621818542480469,
      "activations/layer20_attention_weight_max": 20.018125534057617,
      "activations/layer20_attention_weight_min": -22.33146095275879,
      "activations/layer21_attention_weight_max": 35.7272834777832,
      "activations/layer21_attention_weight_min": -24.144041061401367,
      "activations/layer22_attention_weight_max": 34.124046325683594,
      "activations/layer22_attention_weight_min": -24.20631980895996,
      "activations/layer23_attention_weight_max": 31.70266342163086,
      "activations/layer23_attention_weight_min": -19.46392059326172,
      "activations/layer2_attention_weight_max": 30.931987762451172,
      "activations/layer2_attention_weight_min": -30.179378509521484,
      "activations/layer3_attention_weight_max": 80.37944793701172,
      "activations/layer3_attention_weight_min": -84.37382507324219,
      "activations/layer4_attention_weight_max": 99.14315795898438,
      "activations/layer4_attention_weight_min": -97.79388427734375,
      "activations/layer5_attention_weight_max": 94.20449829101562,
      "activations/layer5_attention_weight_min": -89.19223022460938,
      "activations/layer6_attention_weight_max": 57.13661193847656,
      "activations/layer6_attention_weight_min": -50.79255294799805,
      "activations/layer7_attention_weight_max": 66.67891693115234,
      "activations/layer7_attention_weight_min": -63.64200210571289,
      "activations/layer8_attention_weight_max": 41.88801956176758,
      "activations/layer8_attention_weight_min": -41.215763092041016,
      "activations/layer9_attention_weight_max": 44.7417106628418,
      "activations/layer9_attention_weight_min": -43.79990768432617,
      "epoch": 5.64,
      "learning_rate": 0.00011477083333333333,
      "loss": 2.8565,
      "step": 97100
    },
    {
      "activations/layer0_attention_weight_max": 15.549020767211914,
      "activations/layer0_attention_weight_min": -14.99384880065918,
      "activations/layer10_attention_weight_max": 42.53705596923828,
      "activations/layer10_attention_weight_min": -36.67317581176758,
      "activations/layer11_attention_weight_max": 28.905139923095703,
      "activations/layer11_attention_weight_min": -30.341764450073242,
      "activations/layer12_attention_weight_max": 21.5491943359375,
      "activations/layer12_attention_weight_min": -24.969165802001953,
      "activations/layer13_attention_weight_max": 45.522830963134766,
      "activations/layer13_attention_weight_min": -38.51900100708008,
      "activations/layer14_attention_weight_max": 42.032936096191406,
      "activations/layer14_attention_weight_min": -32.893943786621094,
      "activations/layer15_attention_weight_max": 37.75379180908203,
      "activations/layer15_attention_weight_min": -30.085567474365234,
      "activations/layer16_attention_weight_max": 35.16209030151367,
      "activations/layer16_attention_weight_min": -29.270462036132812,
      "activations/layer17_attention_weight_max": 53.563720703125,
      "activations/layer17_attention_weight_min": -38.34697341918945,
      "activations/layer18_attention_weight_max": 49.58942794799805,
      "activations/layer18_attention_weight_min": -32.659698486328125,
      "activations/layer19_attention_weight_max": 24.259748458862305,
      "activations/layer19_attention_weight_min": -18.659589767456055,
      "activations/layer1_attention_weight_max": 17.40885353088379,
      "activations/layer1_attention_weight_min": -14.678641319274902,
      "activations/layer20_attention_weight_max": 20.403913497924805,
      "activations/layer20_attention_weight_min": -22.640718460083008,
      "activations/layer21_attention_weight_max": 38.053131103515625,
      "activations/layer21_attention_weight_min": -24.783409118652344,
      "activations/layer22_attention_weight_max": 35.93154525756836,
      "activations/layer22_attention_weight_min": -25.460527420043945,
      "activations/layer23_attention_weight_max": 28.900413513183594,
      "activations/layer23_attention_weight_min": -18.159332275390625,
      "activations/layer2_attention_weight_max": 30.331031799316406,
      "activations/layer2_attention_weight_min": -29.73957633972168,
      "activations/layer3_attention_weight_max": 81.62637329101562,
      "activations/layer3_attention_weight_min": -86.50839233398438,
      "activations/layer4_attention_weight_max": 100.96215057373047,
      "activations/layer4_attention_weight_min": -94.55303192138672,
      "activations/layer5_attention_weight_max": 95.92926025390625,
      "activations/layer5_attention_weight_min": -84.35211181640625,
      "activations/layer6_attention_weight_max": 55.95724868774414,
      "activations/layer6_attention_weight_min": -51.141090393066406,
      "activations/layer7_attention_weight_max": 70.4779052734375,
      "activations/layer7_attention_weight_min": -70.0649185180664,
      "activations/layer8_attention_weight_max": 42.086971282958984,
      "activations/layer8_attention_weight_min": -41.686317443847656,
      "activations/layer9_attention_weight_max": 50.999515533447266,
      "activations/layer9_attention_weight_min": -50.2657585144043,
      "epoch": 5.64,
      "learning_rate": 0.00011475189393939393,
      "loss": 2.843,
      "step": 97150
    },
    {
      "activations/layer0_attention_weight_max": 13.948206901550293,
      "activations/layer0_attention_weight_min": -15.007311820983887,
      "activations/layer10_attention_weight_max": 38.72713851928711,
      "activations/layer10_attention_weight_min": -35.81471633911133,
      "activations/layer11_attention_weight_max": 29.898189544677734,
      "activations/layer11_attention_weight_min": -28.901016235351562,
      "activations/layer12_attention_weight_max": 24.202367782592773,
      "activations/layer12_attention_weight_min": -22.694229125976562,
      "activations/layer13_attention_weight_max": 53.25924301147461,
      "activations/layer13_attention_weight_min": -36.342342376708984,
      "activations/layer14_attention_weight_max": 57.173423767089844,
      "activations/layer14_attention_weight_min": -35.91987228393555,
      "activations/layer15_attention_weight_max": 41.39338302612305,
      "activations/layer15_attention_weight_min": -29.080602645874023,
      "activations/layer16_attention_weight_max": 38.01515197753906,
      "activations/layer16_attention_weight_min": -27.609296798706055,
      "activations/layer17_attention_weight_max": 57.03529739379883,
      "activations/layer17_attention_weight_min": -43.790489196777344,
      "activations/layer18_attention_weight_max": 53.759185791015625,
      "activations/layer18_attention_weight_min": -36.923397064208984,
      "activations/layer19_attention_weight_max": 23.077037811279297,
      "activations/layer19_attention_weight_min": -20.796985626220703,
      "activations/layer1_attention_weight_max": 17.55595588684082,
      "activations/layer1_attention_weight_min": -16.25608253479004,
      "activations/layer20_attention_weight_max": 22.907217025756836,
      "activations/layer20_attention_weight_min": -19.97210121154785,
      "activations/layer21_attention_weight_max": 43.30143356323242,
      "activations/layer21_attention_weight_min": -24.027976989746094,
      "activations/layer22_attention_weight_max": 37.428462982177734,
      "activations/layer22_attention_weight_min": -26.61826515197754,
      "activations/layer23_attention_weight_max": 30.193159103393555,
      "activations/layer23_attention_weight_min": -19.795604705810547,
      "activations/layer2_attention_weight_max": 30.42206573486328,
      "activations/layer2_attention_weight_min": -29.400760650634766,
      "activations/layer3_attention_weight_max": 81.86127471923828,
      "activations/layer3_attention_weight_min": -88.64068603515625,
      "activations/layer4_attention_weight_max": 101.46101379394531,
      "activations/layer4_attention_weight_min": -97.40547943115234,
      "activations/layer5_attention_weight_max": 96.881103515625,
      "activations/layer5_attention_weight_min": -91.3062973022461,
      "activations/layer6_attention_weight_max": 58.01263427734375,
      "activations/layer6_attention_weight_min": -56.35556411743164,
      "activations/layer7_attention_weight_max": 70.21754455566406,
      "activations/layer7_attention_weight_min": -68.95294189453125,
      "activations/layer8_attention_weight_max": 44.973602294921875,
      "activations/layer8_attention_weight_min": -42.35533142089844,
      "activations/layer9_attention_weight_max": 51.066558837890625,
      "activations/layer9_attention_weight_min": -45.7619743347168,
      "epoch": 5.65,
      "learning_rate": 0.00011473295454545453,
      "loss": 2.839,
      "step": 97200
    },
    {
      "activations/layer0_attention_weight_max": 14.632673263549805,
      "activations/layer0_attention_weight_min": -15.387319564819336,
      "activations/layer10_attention_weight_max": 39.57661819458008,
      "activations/layer10_attention_weight_min": -35.29420852661133,
      "activations/layer11_attention_weight_max": 30.042709350585938,
      "activations/layer11_attention_weight_min": -27.407146453857422,
      "activations/layer12_attention_weight_max": 27.19049835205078,
      "activations/layer12_attention_weight_min": -26.88473129272461,
      "activations/layer13_attention_weight_max": 57.98197555541992,
      "activations/layer13_attention_weight_min": -37.21144104003906,
      "activations/layer14_attention_weight_max": 54.48318862915039,
      "activations/layer14_attention_weight_min": -34.12425231933594,
      "activations/layer15_attention_weight_max": 43.247459411621094,
      "activations/layer15_attention_weight_min": -29.74065589904785,
      "activations/layer16_attention_weight_max": 36.149200439453125,
      "activations/layer16_attention_weight_min": -27.522977828979492,
      "activations/layer17_attention_weight_max": 55.45542526245117,
      "activations/layer17_attention_weight_min": -39.14882278442383,
      "activations/layer18_attention_weight_max": 54.35913848876953,
      "activations/layer18_attention_weight_min": -33.599002838134766,
      "activations/layer19_attention_weight_max": 22.358642578125,
      "activations/layer19_attention_weight_min": -21.972396850585938,
      "activations/layer1_attention_weight_max": 17.110136032104492,
      "activations/layer1_attention_weight_min": -17.576154708862305,
      "activations/layer20_attention_weight_max": 22.829391479492188,
      "activations/layer20_attention_weight_min": -19.84652328491211,
      "activations/layer21_attention_weight_max": 41.61771011352539,
      "activations/layer21_attention_weight_min": -27.56608009338379,
      "activations/layer22_attention_weight_max": 36.363304138183594,
      "activations/layer22_attention_weight_min": -25.623716354370117,
      "activations/layer23_attention_weight_max": 35.39198303222656,
      "activations/layer23_attention_weight_min": -18.438819885253906,
      "activations/layer2_attention_weight_max": 30.494441986083984,
      "activations/layer2_attention_weight_min": -28.30059814453125,
      "activations/layer3_attention_weight_max": 80.05288696289062,
      "activations/layer3_attention_weight_min": -84.33344268798828,
      "activations/layer4_attention_weight_max": 104.15120697021484,
      "activations/layer4_attention_weight_min": -99.3167953491211,
      "activations/layer5_attention_weight_max": 98.13653564453125,
      "activations/layer5_attention_weight_min": -88.35897827148438,
      "activations/layer6_attention_weight_max": 56.45188903808594,
      "activations/layer6_attention_weight_min": -51.72658920288086,
      "activations/layer7_attention_weight_max": 70.58860778808594,
      "activations/layer7_attention_weight_min": -67.80945587158203,
      "activations/layer8_attention_weight_max": 45.35085678100586,
      "activations/layer8_attention_weight_min": -42.23942184448242,
      "activations/layer9_attention_weight_max": 54.55572509765625,
      "activations/layer9_attention_weight_min": -50.59993362426758,
      "epoch": 5.65,
      "learning_rate": 0.00011471401515151515,
      "loss": 2.8332,
      "step": 97250
    },
    {
      "activations/layer0_attention_weight_max": 15.129324913024902,
      "activations/layer0_attention_weight_min": -15.4232177734375,
      "activations/layer10_attention_weight_max": 41.558197021484375,
      "activations/layer10_attention_weight_min": -36.001522064208984,
      "activations/layer11_attention_weight_max": 33.59518051147461,
      "activations/layer11_attention_weight_min": -26.748455047607422,
      "activations/layer12_attention_weight_max": 21.71759033203125,
      "activations/layer12_attention_weight_min": -28.313949584960938,
      "activations/layer13_attention_weight_max": 49.512855529785156,
      "activations/layer13_attention_weight_min": -31.51721954345703,
      "activations/layer14_attention_weight_max": 53.58251190185547,
      "activations/layer14_attention_weight_min": -33.995662689208984,
      "activations/layer15_attention_weight_max": 38.431793212890625,
      "activations/layer15_attention_weight_min": -28.197778701782227,
      "activations/layer16_attention_weight_max": 33.161155700683594,
      "activations/layer16_attention_weight_min": -26.61893653869629,
      "activations/layer17_attention_weight_max": 59.94057846069336,
      "activations/layer17_attention_weight_min": -37.51005172729492,
      "activations/layer18_attention_weight_max": 48.77277374267578,
      "activations/layer18_attention_weight_min": -32.237430572509766,
      "activations/layer19_attention_weight_max": 20.045682907104492,
      "activations/layer19_attention_weight_min": -20.65825653076172,
      "activations/layer1_attention_weight_max": 18.10399627685547,
      "activations/layer1_attention_weight_min": -17.02709197998047,
      "activations/layer20_attention_weight_max": 25.03682518005371,
      "activations/layer20_attention_weight_min": -20.50054168701172,
      "activations/layer21_attention_weight_max": 41.40016555786133,
      "activations/layer21_attention_weight_min": -24.676158905029297,
      "activations/layer22_attention_weight_max": 33.63269805908203,
      "activations/layer22_attention_weight_min": -23.384540557861328,
      "activations/layer23_attention_weight_max": 27.399723052978516,
      "activations/layer23_attention_weight_min": -18.05630874633789,
      "activations/layer2_attention_weight_max": 31.977094650268555,
      "activations/layer2_attention_weight_min": -31.373363494873047,
      "activations/layer3_attention_weight_max": 88.09688568115234,
      "activations/layer3_attention_weight_min": -89.01010131835938,
      "activations/layer4_attention_weight_max": 107.3042221069336,
      "activations/layer4_attention_weight_min": -99.5687484741211,
      "activations/layer5_attention_weight_max": 100.0482406616211,
      "activations/layer5_attention_weight_min": -89.13642883300781,
      "activations/layer6_attention_weight_max": 56.530704498291016,
      "activations/layer6_attention_weight_min": -52.43611145019531,
      "activations/layer7_attention_weight_max": 90.57550811767578,
      "activations/layer7_attention_weight_min": -69.02717590332031,
      "activations/layer8_attention_weight_max": 50.67793273925781,
      "activations/layer8_attention_weight_min": -39.65195083618164,
      "activations/layer9_attention_weight_max": 72.84929656982422,
      "activations/layer9_attention_weight_min": -50.749916076660156,
      "epoch": 5.65,
      "learning_rate": 0.00011469545454545454,
      "loss": 2.8352,
      "step": 97300
    },
    {
      "activations/layer0_attention_weight_max": 15.140881538391113,
      "activations/layer0_attention_weight_min": -14.984833717346191,
      "activations/layer10_attention_weight_max": 35.914154052734375,
      "activations/layer10_attention_weight_min": -34.65695571899414,
      "activations/layer11_attention_weight_max": 28.092832565307617,
      "activations/layer11_attention_weight_min": -28.697463989257812,
      "activations/layer12_attention_weight_max": 21.490585327148438,
      "activations/layer12_attention_weight_min": -26.927486419677734,
      "activations/layer13_attention_weight_max": 45.0262565612793,
      "activations/layer13_attention_weight_min": -36.47216796875,
      "activations/layer14_attention_weight_max": 47.94425582885742,
      "activations/layer14_attention_weight_min": -33.21999740600586,
      "activations/layer15_attention_weight_max": 38.72919464111328,
      "activations/layer15_attention_weight_min": -29.246957778930664,
      "activations/layer16_attention_weight_max": 34.6881217956543,
      "activations/layer16_attention_weight_min": -26.056110382080078,
      "activations/layer17_attention_weight_max": 58.06943130493164,
      "activations/layer17_attention_weight_min": -42.49604797363281,
      "activations/layer18_attention_weight_max": 50.84626770019531,
      "activations/layer18_attention_weight_min": -34.06795883178711,
      "activations/layer19_attention_weight_max": 22.10907554626465,
      "activations/layer19_attention_weight_min": -21.317617416381836,
      "activations/layer1_attention_weight_max": 17.44843864440918,
      "activations/layer1_attention_weight_min": -14.70057487487793,
      "activations/layer20_attention_weight_max": 22.96824073791504,
      "activations/layer20_attention_weight_min": -19.77183723449707,
      "activations/layer21_attention_weight_max": 40.64490509033203,
      "activations/layer21_attention_weight_min": -25.88463592529297,
      "activations/layer22_attention_weight_max": 34.923072814941406,
      "activations/layer22_attention_weight_min": -23.746538162231445,
      "activations/layer23_attention_weight_max": 28.801740646362305,
      "activations/layer23_attention_weight_min": -19.441070556640625,
      "activations/layer2_attention_weight_max": 32.01493835449219,
      "activations/layer2_attention_weight_min": -29.62844467163086,
      "activations/layer3_attention_weight_max": 79.63362121582031,
      "activations/layer3_attention_weight_min": -82.52387237548828,
      "activations/layer4_attention_weight_max": 100.82376098632812,
      "activations/layer4_attention_weight_min": -93.91868591308594,
      "activations/layer5_attention_weight_max": 95.38113403320312,
      "activations/layer5_attention_weight_min": -89.40807342529297,
      "activations/layer6_attention_weight_max": 56.239906311035156,
      "activations/layer6_attention_weight_min": -51.46110153198242,
      "activations/layer7_attention_weight_max": 69.5458755493164,
      "activations/layer7_attention_weight_min": -67.5727767944336,
      "activations/layer8_attention_weight_max": 40.67188262939453,
      "activations/layer8_attention_weight_min": -41.298431396484375,
      "activations/layer9_attention_weight_max": 61.2909049987793,
      "activations/layer9_attention_weight_min": -48.28634262084961,
      "epoch": 5.66,
      "learning_rate": 0.00011467651515151514,
      "loss": 2.8502,
      "step": 97350
    },
    {
      "activations/layer0_attention_weight_max": 15.624980926513672,
      "activations/layer0_attention_weight_min": -15.193696022033691,
      "activations/layer10_attention_weight_max": 37.25834655761719,
      "activations/layer10_attention_weight_min": -36.48332595825195,
      "activations/layer11_attention_weight_max": 28.016651153564453,
      "activations/layer11_attention_weight_min": -28.241901397705078,
      "activations/layer12_attention_weight_max": 20.78714370727539,
      "activations/layer12_attention_weight_min": -26.48749542236328,
      "activations/layer13_attention_weight_max": 46.72911071777344,
      "activations/layer13_attention_weight_min": -32.99001693725586,
      "activations/layer14_attention_weight_max": 51.44325256347656,
      "activations/layer14_attention_weight_min": -31.688907623291016,
      "activations/layer15_attention_weight_max": 42.373191833496094,
      "activations/layer15_attention_weight_min": -28.394882202148438,
      "activations/layer16_attention_weight_max": 39.89996337890625,
      "activations/layer16_attention_weight_min": -25.743223190307617,
      "activations/layer17_attention_weight_max": 57.478515625,
      "activations/layer17_attention_weight_min": -41.60694122314453,
      "activations/layer18_attention_weight_max": 52.78105163574219,
      "activations/layer18_attention_weight_min": -36.2783203125,
      "activations/layer19_attention_weight_max": 21.585325241088867,
      "activations/layer19_attention_weight_min": -18.119388580322266,
      "activations/layer1_attention_weight_max": 19.068790435791016,
      "activations/layer1_attention_weight_min": -14.101056098937988,
      "activations/layer20_attention_weight_max": 21.01467514038086,
      "activations/layer20_attention_weight_min": -19.011838912963867,
      "activations/layer21_attention_weight_max": 36.87336349487305,
      "activations/layer21_attention_weight_min": -25.617881774902344,
      "activations/layer22_attention_weight_max": 35.3167610168457,
      "activations/layer22_attention_weight_min": -22.593429565429688,
      "activations/layer23_attention_weight_max": 31.533674240112305,
      "activations/layer23_attention_weight_min": -19.071252822875977,
      "activations/layer2_attention_weight_max": 30.322612762451172,
      "activations/layer2_attention_weight_min": -29.33209800720215,
      "activations/layer3_attention_weight_max": 81.5672378540039,
      "activations/layer3_attention_weight_min": -83.21842193603516,
      "activations/layer4_attention_weight_max": 103.0643081665039,
      "activations/layer4_attention_weight_min": -95.94103240966797,
      "activations/layer5_attention_weight_max": 94.92223358154297,
      "activations/layer5_attention_weight_min": -89.88249969482422,
      "activations/layer6_attention_weight_max": 55.74464797973633,
      "activations/layer6_attention_weight_min": -55.09790802001953,
      "activations/layer7_attention_weight_max": 69.29631042480469,
      "activations/layer7_attention_weight_min": -65.33158874511719,
      "activations/layer8_attention_weight_max": 43.03516387939453,
      "activations/layer8_attention_weight_min": -43.40156173706055,
      "activations/layer9_attention_weight_max": 52.85775375366211,
      "activations/layer9_attention_weight_min": -47.30411911010742,
      "epoch": 5.66,
      "learning_rate": 0.00011465757575757575,
      "loss": 2.8427,
      "step": 97400
    },
    {
      "activations/layer0_attention_weight_max": 15.090542793273926,
      "activations/layer0_attention_weight_min": -15.179909706115723,
      "activations/layer10_attention_weight_max": 39.21641159057617,
      "activations/layer10_attention_weight_min": -34.75395584106445,
      "activations/layer11_attention_weight_max": 31.371397018432617,
      "activations/layer11_attention_weight_min": -28.126562118530273,
      "activations/layer12_attention_weight_max": 21.46222496032715,
      "activations/layer12_attention_weight_min": -23.809518814086914,
      "activations/layer13_attention_weight_max": 49.06849670410156,
      "activations/layer13_attention_weight_min": -37.47404479980469,
      "activations/layer14_attention_weight_max": 37.4995231628418,
      "activations/layer14_attention_weight_min": -28.212230682373047,
      "activations/layer15_attention_weight_max": 36.39860153198242,
      "activations/layer15_attention_weight_min": -27.867338180541992,
      "activations/layer16_attention_weight_max": 32.67842483520508,
      "activations/layer16_attention_weight_min": -26.42929458618164,
      "activations/layer17_attention_weight_max": 53.96750259399414,
      "activations/layer17_attention_weight_min": -39.8507080078125,
      "activations/layer18_attention_weight_max": 49.40068054199219,
      "activations/layer18_attention_weight_min": -34.011322021484375,
      "activations/layer19_attention_weight_max": 22.92287826538086,
      "activations/layer19_attention_weight_min": -19.26958465576172,
      "activations/layer1_attention_weight_max": 17.406232833862305,
      "activations/layer1_attention_weight_min": -13.920845985412598,
      "activations/layer20_attention_weight_max": 20.078672409057617,
      "activations/layer20_attention_weight_min": -19.53974151611328,
      "activations/layer21_attention_weight_max": 39.705345153808594,
      "activations/layer21_attention_weight_min": -26.220481872558594,
      "activations/layer22_attention_weight_max": 34.07750701904297,
      "activations/layer22_attention_weight_min": -24.010515213012695,
      "activations/layer23_attention_weight_max": 29.375152587890625,
      "activations/layer23_attention_weight_min": -17.417083740234375,
      "activations/layer2_attention_weight_max": 28.927288055419922,
      "activations/layer2_attention_weight_min": -27.970081329345703,
      "activations/layer3_attention_weight_max": 83.24280548095703,
      "activations/layer3_attention_weight_min": -88.38836669921875,
      "activations/layer4_attention_weight_max": 103.72802734375,
      "activations/layer4_attention_weight_min": -96.40425872802734,
      "activations/layer5_attention_weight_max": 99.55726623535156,
      "activations/layer5_attention_weight_min": -92.45067596435547,
      "activations/layer6_attention_weight_max": 57.28565216064453,
      "activations/layer6_attention_weight_min": -53.376564025878906,
      "activations/layer7_attention_weight_max": 71.81201171875,
      "activations/layer7_attention_weight_min": -69.42212677001953,
      "activations/layer8_attention_weight_max": 43.91642379760742,
      "activations/layer8_attention_weight_min": -44.13747787475586,
      "activations/layer9_attention_weight_max": 48.5380859375,
      "activations/layer9_attention_weight_min": -46.88078689575195,
      "epoch": 5.66,
      "learning_rate": 0.00011463863636363636,
      "loss": 2.8192,
      "step": 97450
    },
    {
      "activations/layer0_attention_weight_max": 15.112442970275879,
      "activations/layer0_attention_weight_min": -15.136454582214355,
      "activations/layer10_attention_weight_max": 42.882423400878906,
      "activations/layer10_attention_weight_min": -35.89678192138672,
      "activations/layer11_attention_weight_max": 32.76978302001953,
      "activations/layer11_attention_weight_min": -28.565853118896484,
      "activations/layer12_attention_weight_max": 26.915307998657227,
      "activations/layer12_attention_weight_min": -25.415807723999023,
      "activations/layer13_attention_weight_max": 52.374717712402344,
      "activations/layer13_attention_weight_min": -35.76367950439453,
      "activations/layer14_attention_weight_max": 42.15424728393555,
      "activations/layer14_attention_weight_min": -28.418596267700195,
      "activations/layer15_attention_weight_max": 44.91572952270508,
      "activations/layer15_attention_weight_min": -29.4644775390625,
      "activations/layer16_attention_weight_max": 37.33318328857422,
      "activations/layer16_attention_weight_min": -26.033193588256836,
      "activations/layer17_attention_weight_max": 62.49649429321289,
      "activations/layer17_attention_weight_min": -39.09852981567383,
      "activations/layer18_attention_weight_max": 52.55593490600586,
      "activations/layer18_attention_weight_min": -34.134151458740234,
      "activations/layer19_attention_weight_max": 29.16952896118164,
      "activations/layer19_attention_weight_min": -23.665040969848633,
      "activations/layer1_attention_weight_max": 17.31961441040039,
      "activations/layer1_attention_weight_min": -15.039909362792969,
      "activations/layer20_attention_weight_max": 26.348485946655273,
      "activations/layer20_attention_weight_min": -25.7182559967041,
      "activations/layer21_attention_weight_max": 42.329742431640625,
      "activations/layer21_attention_weight_min": -24.32156753540039,
      "activations/layer22_attention_weight_max": 36.01473617553711,
      "activations/layer22_attention_weight_min": -26.339115142822266,
      "activations/layer23_attention_weight_max": 33.89677047729492,
      "activations/layer23_attention_weight_min": -19.831993103027344,
      "activations/layer2_attention_weight_max": 31.99382781982422,
      "activations/layer2_attention_weight_min": -31.131175994873047,
      "activations/layer3_attention_weight_max": 85.6651840209961,
      "activations/layer3_attention_weight_min": -88.96839141845703,
      "activations/layer4_attention_weight_max": 109.36901092529297,
      "activations/layer4_attention_weight_min": -99.80452728271484,
      "activations/layer5_attention_weight_max": 100.98519134521484,
      "activations/layer5_attention_weight_min": -92.68456268310547,
      "activations/layer6_attention_weight_max": 60.149452209472656,
      "activations/layer6_attention_weight_min": -54.571834564208984,
      "activations/layer7_attention_weight_max": 71.45774841308594,
      "activations/layer7_attention_weight_min": -72.37210083007812,
      "activations/layer8_attention_weight_max": 47.783626556396484,
      "activations/layer8_attention_weight_min": -43.93276596069336,
      "activations/layer9_attention_weight_max": 54.51226806640625,
      "activations/layer9_attention_weight_min": -52.36476516723633,
      "epoch": 5.67,
      "learning_rate": 0.00011461969696969696,
      "loss": 2.8453,
      "step": 97500
    },
    {
      "activations/layer0_attention_weight_max": 13.838659286499023,
      "activations/layer0_attention_weight_min": -15.68143081665039,
      "activations/layer10_attention_weight_max": 38.87090301513672,
      "activations/layer10_attention_weight_min": -36.35504913330078,
      "activations/layer11_attention_weight_max": 29.481277465820312,
      "activations/layer11_attention_weight_min": -28.05035400390625,
      "activations/layer12_attention_weight_max": 24.565509796142578,
      "activations/layer12_attention_weight_min": -26.888019561767578,
      "activations/layer13_attention_weight_max": 51.3483772277832,
      "activations/layer13_attention_weight_min": -36.147705078125,
      "activations/layer14_attention_weight_max": 39.72132110595703,
      "activations/layer14_attention_weight_min": -30.154708862304688,
      "activations/layer15_attention_weight_max": 41.03221893310547,
      "activations/layer15_attention_weight_min": -30.236177444458008,
      "activations/layer16_attention_weight_max": 33.065799713134766,
      "activations/layer16_attention_weight_min": -26.974746704101562,
      "activations/layer17_attention_weight_max": 56.397159576416016,
      "activations/layer17_attention_weight_min": -38.86750030517578,
      "activations/layer18_attention_weight_max": 49.52653121948242,
      "activations/layer18_attention_weight_min": -32.876285552978516,
      "activations/layer19_attention_weight_max": 25.671478271484375,
      "activations/layer19_attention_weight_min": -20.48346519470215,
      "activations/layer1_attention_weight_max": 17.188566207885742,
      "activations/layer1_attention_weight_min": -15.661186218261719,
      "activations/layer20_attention_weight_max": 21.156349182128906,
      "activations/layer20_attention_weight_min": -24.77012825012207,
      "activations/layer21_attention_weight_max": 39.30849075317383,
      "activations/layer21_attention_weight_min": -26.388774871826172,
      "activations/layer22_attention_weight_max": 33.15761184692383,
      "activations/layer22_attention_weight_min": -24.99460792541504,
      "activations/layer23_attention_weight_max": 28.032575607299805,
      "activations/layer23_attention_weight_min": -19.7413330078125,
      "activations/layer2_attention_weight_max": 29.141407012939453,
      "activations/layer2_attention_weight_min": -27.724475860595703,
      "activations/layer3_attention_weight_max": 81.50141906738281,
      "activations/layer3_attention_weight_min": -87.10747528076172,
      "activations/layer4_attention_weight_max": 103.15052795410156,
      "activations/layer4_attention_weight_min": -96.26580810546875,
      "activations/layer5_attention_weight_max": 99.63157653808594,
      "activations/layer5_attention_weight_min": -90.18154907226562,
      "activations/layer6_attention_weight_max": 59.574501037597656,
      "activations/layer6_attention_weight_min": -55.19414138793945,
      "activations/layer7_attention_weight_max": 71.63145446777344,
      "activations/layer7_attention_weight_min": -65.34738159179688,
      "activations/layer8_attention_weight_max": 43.35580062866211,
      "activations/layer8_attention_weight_min": -43.488643646240234,
      "activations/layer9_attention_weight_max": 52.41936111450195,
      "activations/layer9_attention_weight_min": -46.423152923583984,
      "epoch": 5.67,
      "learning_rate": 0.00011460075757575757,
      "loss": 2.8466,
      "step": 97550
    },
    {
      "activations/layer0_attention_weight_max": 13.75977897644043,
      "activations/layer0_attention_weight_min": -15.362505912780762,
      "activations/layer10_attention_weight_max": 37.933990478515625,
      "activations/layer10_attention_weight_min": -33.923763275146484,
      "activations/layer11_attention_weight_max": 30.296287536621094,
      "activations/layer11_attention_weight_min": -27.501556396484375,
      "activations/layer12_attention_weight_max": 18.641202926635742,
      "activations/layer12_attention_weight_min": -22.722755432128906,
      "activations/layer13_attention_weight_max": 46.94945526123047,
      "activations/layer13_attention_weight_min": -36.96150207519531,
      "activations/layer14_attention_weight_max": 51.3212890625,
      "activations/layer14_attention_weight_min": -33.34552764892578,
      "activations/layer15_attention_weight_max": 38.819313049316406,
      "activations/layer15_attention_weight_min": -31.322473526000977,
      "activations/layer16_attention_weight_max": 35.08273696899414,
      "activations/layer16_attention_weight_min": -30.421756744384766,
      "activations/layer17_attention_weight_max": 54.199771881103516,
      "activations/layer17_attention_weight_min": -40.57229232788086,
      "activations/layer18_attention_weight_max": 48.373104095458984,
      "activations/layer18_attention_weight_min": -34.50161361694336,
      "activations/layer19_attention_weight_max": 22.417882919311523,
      "activations/layer19_attention_weight_min": -18.56259536743164,
      "activations/layer1_attention_weight_max": 16.693967819213867,
      "activations/layer1_attention_weight_min": -14.275517463684082,
      "activations/layer20_attention_weight_max": 22.769939422607422,
      "activations/layer20_attention_weight_min": -20.677921295166016,
      "activations/layer21_attention_weight_max": 39.10774230957031,
      "activations/layer21_attention_weight_min": -25.128570556640625,
      "activations/layer22_attention_weight_max": 32.438438415527344,
      "activations/layer22_attention_weight_min": -23.965574264526367,
      "activations/layer23_attention_weight_max": 30.46486473083496,
      "activations/layer23_attention_weight_min": -22.905445098876953,
      "activations/layer2_attention_weight_max": 29.03780746459961,
      "activations/layer2_attention_weight_min": -27.011871337890625,
      "activations/layer3_attention_weight_max": 80.82730102539062,
      "activations/layer3_attention_weight_min": -81.60875701904297,
      "activations/layer4_attention_weight_max": 96.76335906982422,
      "activations/layer4_attention_weight_min": -93.29169464111328,
      "activations/layer5_attention_weight_max": 92.92965698242188,
      "activations/layer5_attention_weight_min": -84.4910659790039,
      "activations/layer6_attention_weight_max": 56.42515563964844,
      "activations/layer6_attention_weight_min": -50.027687072753906,
      "activations/layer7_attention_weight_max": 66.48004913330078,
      "activations/layer7_attention_weight_min": -66.30638885498047,
      "activations/layer8_attention_weight_max": 41.443931579589844,
      "activations/layer8_attention_weight_min": -39.49237823486328,
      "activations/layer9_attention_weight_max": 49.31093215942383,
      "activations/layer9_attention_weight_min": -45.93660354614258,
      "epoch": 5.67,
      "learning_rate": 0.00011458181818181816,
      "loss": 2.8293,
      "step": 97600
    },
    {
      "activations/layer0_attention_weight_max": 14.193175315856934,
      "activations/layer0_attention_weight_min": -15.47197151184082,
      "activations/layer10_attention_weight_max": 41.62812423706055,
      "activations/layer10_attention_weight_min": -37.7011604309082,
      "activations/layer11_attention_weight_max": 35.70799255371094,
      "activations/layer11_attention_weight_min": -28.284500122070312,
      "activations/layer12_attention_weight_max": 22.238574981689453,
      "activations/layer12_attention_weight_min": -26.450790405273438,
      "activations/layer13_attention_weight_max": 64.10095977783203,
      "activations/layer13_attention_weight_min": -37.525306701660156,
      "activations/layer14_attention_weight_max": 64.29324340820312,
      "activations/layer14_attention_weight_min": -35.634639739990234,
      "activations/layer15_attention_weight_max": 48.94937515258789,
      "activations/layer15_attention_weight_min": -28.65359115600586,
      "activations/layer16_attention_weight_max": 39.156394958496094,
      "activations/layer16_attention_weight_min": -26.990583419799805,
      "activations/layer17_attention_weight_max": 66.46371459960938,
      "activations/layer17_attention_weight_min": -41.66408157348633,
      "activations/layer18_attention_weight_max": 49.909385681152344,
      "activations/layer18_attention_weight_min": -36.796600341796875,
      "activations/layer19_attention_weight_max": 24.632272720336914,
      "activations/layer19_attention_weight_min": -19.635042190551758,
      "activations/layer1_attention_weight_max": 17.567567825317383,
      "activations/layer1_attention_weight_min": -15.42829418182373,
      "activations/layer20_attention_weight_max": 22.6178035736084,
      "activations/layer20_attention_weight_min": -21.518693923950195,
      "activations/layer21_attention_weight_max": 40.22022247314453,
      "activations/layer21_attention_weight_min": -25.760215759277344,
      "activations/layer22_attention_weight_max": 33.90634536743164,
      "activations/layer22_attention_weight_min": -25.276386260986328,
      "activations/layer23_attention_weight_max": 32.29122543334961,
      "activations/layer23_attention_weight_min": -18.001995086669922,
      "activations/layer2_attention_weight_max": 32.39282989501953,
      "activations/layer2_attention_weight_min": -30.459897994995117,
      "activations/layer3_attention_weight_max": 84.02365112304688,
      "activations/layer3_attention_weight_min": -88.84293365478516,
      "activations/layer4_attention_weight_max": 104.75357818603516,
      "activations/layer4_attention_weight_min": -101.11978149414062,
      "activations/layer5_attention_weight_max": 100.33775329589844,
      "activations/layer5_attention_weight_min": -91.77183532714844,
      "activations/layer6_attention_weight_max": 58.78842544555664,
      "activations/layer6_attention_weight_min": -54.723392486572266,
      "activations/layer7_attention_weight_max": 79.96554565429688,
      "activations/layer7_attention_weight_min": -72.20179748535156,
      "activations/layer8_attention_weight_max": 48.166133880615234,
      "activations/layer8_attention_weight_min": -45.87704086303711,
      "activations/layer9_attention_weight_max": 65.87737274169922,
      "activations/layer9_attention_weight_min": -49.76162338256836,
      "epoch": 5.67,
      "learning_rate": 0.00011456287878787878,
      "loss": 2.8534,
      "step": 97650
    },
    {
      "activations/layer0_attention_weight_max": 14.123350143432617,
      "activations/layer0_attention_weight_min": -14.823668479919434,
      "activations/layer10_attention_weight_max": 36.462646484375,
      "activations/layer10_attention_weight_min": -31.73468017578125,
      "activations/layer11_attention_weight_max": 28.441709518432617,
      "activations/layer11_attention_weight_min": -26.743606567382812,
      "activations/layer12_attention_weight_max": 20.674869537353516,
      "activations/layer12_attention_weight_min": -24.998823165893555,
      "activations/layer13_attention_weight_max": 40.55851364135742,
      "activations/layer13_attention_weight_min": -37.42304992675781,
      "activations/layer14_attention_weight_max": 41.66522979736328,
      "activations/layer14_attention_weight_min": -30.994091033935547,
      "activations/layer15_attention_weight_max": 35.091888427734375,
      "activations/layer15_attention_weight_min": -29.357419967651367,
      "activations/layer16_attention_weight_max": 33.839107513427734,
      "activations/layer16_attention_weight_min": -26.349584579467773,
      "activations/layer17_attention_weight_max": 51.04228973388672,
      "activations/layer17_attention_weight_min": -41.09012222290039,
      "activations/layer18_attention_weight_max": 45.09267807006836,
      "activations/layer18_attention_weight_min": -33.630699157714844,
      "activations/layer19_attention_weight_max": 21.62928009033203,
      "activations/layer19_attention_weight_min": -19.950658798217773,
      "activations/layer1_attention_weight_max": 17.1932430267334,
      "activations/layer1_attention_weight_min": -15.084033966064453,
      "activations/layer20_attention_weight_max": 20.814462661743164,
      "activations/layer20_attention_weight_min": -21.07828140258789,
      "activations/layer21_attention_weight_max": 39.695152282714844,
      "activations/layer21_attention_weight_min": -24.49907875061035,
      "activations/layer22_attention_weight_max": 36.48571014404297,
      "activations/layer22_attention_weight_min": -24.67608642578125,
      "activations/layer23_attention_weight_max": 29.348501205444336,
      "activations/layer23_attention_weight_min": -20.885631561279297,
      "activations/layer2_attention_weight_max": 31.0317440032959,
      "activations/layer2_attention_weight_min": -28.146926879882812,
      "activations/layer3_attention_weight_max": 79.74199676513672,
      "activations/layer3_attention_weight_min": -82.02548217773438,
      "activations/layer4_attention_weight_max": 98.17595672607422,
      "activations/layer4_attention_weight_min": -89.07463073730469,
      "activations/layer5_attention_weight_max": 94.40731048583984,
      "activations/layer5_attention_weight_min": -86.07087707519531,
      "activations/layer6_attention_weight_max": 55.91716003417969,
      "activations/layer6_attention_weight_min": -50.58373260498047,
      "activations/layer7_attention_weight_max": 73.8581314086914,
      "activations/layer7_attention_weight_min": -68.46644592285156,
      "activations/layer8_attention_weight_max": 39.9725456237793,
      "activations/layer8_attention_weight_min": -41.007720947265625,
      "activations/layer9_attention_weight_max": 50.0015983581543,
      "activations/layer9_attention_weight_min": -42.80929183959961,
      "epoch": 5.68,
      "learning_rate": 0.00011454393939393939,
      "loss": 2.8467,
      "step": 97700
    },
    {
      "activations/layer0_attention_weight_max": 14.421098709106445,
      "activations/layer0_attention_weight_min": -15.290427207946777,
      "activations/layer10_attention_weight_max": 38.51314163208008,
      "activations/layer10_attention_weight_min": -34.75979232788086,
      "activations/layer11_attention_weight_max": 31.254423141479492,
      "activations/layer11_attention_weight_min": -26.634056091308594,
      "activations/layer12_attention_weight_max": 19.584508895874023,
      "activations/layer12_attention_weight_min": -30.096132278442383,
      "activations/layer13_attention_weight_max": 48.411705017089844,
      "activations/layer13_attention_weight_min": -34.27456283569336,
      "activations/layer14_attention_weight_max": 41.64341354370117,
      "activations/layer14_attention_weight_min": -29.265949249267578,
      "activations/layer15_attention_weight_max": 39.618927001953125,
      "activations/layer15_attention_weight_min": -30.065589904785156,
      "activations/layer16_attention_weight_max": 36.16352462768555,
      "activations/layer16_attention_weight_min": -26.019147872924805,
      "activations/layer17_attention_weight_max": 53.53631591796875,
      "activations/layer17_attention_weight_min": -39.08058547973633,
      "activations/layer18_attention_weight_max": 44.72667694091797,
      "activations/layer18_attention_weight_min": -33.00996017456055,
      "activations/layer19_attention_weight_max": 20.272850036621094,
      "activations/layer19_attention_weight_min": -19.146427154541016,
      "activations/layer1_attention_weight_max": 19.209735870361328,
      "activations/layer1_attention_weight_min": -16.609960556030273,
      "activations/layer20_attention_weight_max": 19.98609733581543,
      "activations/layer20_attention_weight_min": -20.81189727783203,
      "activations/layer21_attention_weight_max": 36.417972564697266,
      "activations/layer21_attention_weight_min": -26.68837547302246,
      "activations/layer22_attention_weight_max": 31.63701057434082,
      "activations/layer22_attention_weight_min": -26.328678131103516,
      "activations/layer23_attention_weight_max": 26.49123764038086,
      "activations/layer23_attention_weight_min": -19.08978843688965,
      "activations/layer2_attention_weight_max": 31.91954803466797,
      "activations/layer2_attention_weight_min": -31.385387420654297,
      "activations/layer3_attention_weight_max": 83.69058990478516,
      "activations/layer3_attention_weight_min": -87.19756317138672,
      "activations/layer4_attention_weight_max": 99.80379486083984,
      "activations/layer4_attention_weight_min": -100.27869415283203,
      "activations/layer5_attention_weight_max": 97.5656967163086,
      "activations/layer5_attention_weight_min": -90.11100769042969,
      "activations/layer6_attention_weight_max": 57.98114013671875,
      "activations/layer6_attention_weight_min": -50.90236282348633,
      "activations/layer7_attention_weight_max": 72.4186019897461,
      "activations/layer7_attention_weight_min": -67.99760437011719,
      "activations/layer8_attention_weight_max": 44.01155090332031,
      "activations/layer8_attention_weight_min": -41.54759216308594,
      "activations/layer9_attention_weight_max": 55.34109115600586,
      "activations/layer9_attention_weight_min": -51.82088088989258,
      "epoch": 5.68,
      "learning_rate": 0.00011452499999999998,
      "loss": 2.8518,
      "step": 97750
    },
    {
      "activations/layer0_attention_weight_max": 16.020614624023438,
      "activations/layer0_attention_weight_min": -15.361433029174805,
      "activations/layer10_attention_weight_max": 36.98267364501953,
      "activations/layer10_attention_weight_min": -34.128292083740234,
      "activations/layer11_attention_weight_max": 27.720645904541016,
      "activations/layer11_attention_weight_min": -27.728527069091797,
      "activations/layer12_attention_weight_max": 20.66371726989746,
      "activations/layer12_attention_weight_min": -26.69959831237793,
      "activations/layer13_attention_weight_max": 41.0692253112793,
      "activations/layer13_attention_weight_min": -34.027435302734375,
      "activations/layer14_attention_weight_max": 42.09355926513672,
      "activations/layer14_attention_weight_min": -32.474403381347656,
      "activations/layer15_attention_weight_max": 37.180545806884766,
      "activations/layer15_attention_weight_min": -30.799095153808594,
      "activations/layer16_attention_weight_max": 31.836694717407227,
      "activations/layer16_attention_weight_min": -25.892841339111328,
      "activations/layer17_attention_weight_max": 55.17177200317383,
      "activations/layer17_attention_weight_min": -40.05122756958008,
      "activations/layer18_attention_weight_max": 45.37479782104492,
      "activations/layer18_attention_weight_min": -35.287086486816406,
      "activations/layer19_attention_weight_max": 22.121976852416992,
      "activations/layer19_attention_weight_min": -20.520376205444336,
      "activations/layer1_attention_weight_max": 17.417987823486328,
      "activations/layer1_attention_weight_min": -17.0683536529541,
      "activations/layer20_attention_weight_max": 21.166820526123047,
      "activations/layer20_attention_weight_min": -27.118879318237305,
      "activations/layer21_attention_weight_max": 42.95945358276367,
      "activations/layer21_attention_weight_min": -25.761550903320312,
      "activations/layer22_attention_weight_max": 32.262840270996094,
      "activations/layer22_attention_weight_min": -24.632694244384766,
      "activations/layer23_attention_weight_max": 28.217931747436523,
      "activations/layer23_attention_weight_min": -18.377628326416016,
      "activations/layer2_attention_weight_max": 30.215927124023438,
      "activations/layer2_attention_weight_min": -29.666685104370117,
      "activations/layer3_attention_weight_max": 82.822998046875,
      "activations/layer3_attention_weight_min": -88.26012420654297,
      "activations/layer4_attention_weight_max": 99.28191375732422,
      "activations/layer4_attention_weight_min": -96.39739227294922,
      "activations/layer5_attention_weight_max": 89.44541931152344,
      "activations/layer5_attention_weight_min": -84.61332702636719,
      "activations/layer6_attention_weight_max": 52.778995513916016,
      "activations/layer6_attention_weight_min": -50.79709243774414,
      "activations/layer7_attention_weight_max": 62.647796630859375,
      "activations/layer7_attention_weight_min": -62.18376922607422,
      "activations/layer8_attention_weight_max": 39.99265670776367,
      "activations/layer8_attention_weight_min": -40.6438102722168,
      "activations/layer9_attention_weight_max": 48.049598693847656,
      "activations/layer9_attention_weight_min": -46.35726547241211,
      "epoch": 5.68,
      "learning_rate": 0.0001145060606060606,
      "loss": 2.8435,
      "step": 97800
    },
    {
      "activations/layer0_attention_weight_max": 14.574614524841309,
      "activations/layer0_attention_weight_min": -15.110779762268066,
      "activations/layer10_attention_weight_max": 39.525638580322266,
      "activations/layer10_attention_weight_min": -36.1464958190918,
      "activations/layer11_attention_weight_max": 31.143943786621094,
      "activations/layer11_attention_weight_min": -26.829984664916992,
      "activations/layer12_attention_weight_max": 25.76959991455078,
      "activations/layer12_attention_weight_min": -23.240808486938477,
      "activations/layer13_attention_weight_max": 52.03676223754883,
      "activations/layer13_attention_weight_min": -33.83808135986328,
      "activations/layer14_attention_weight_max": 42.39119338989258,
      "activations/layer14_attention_weight_min": -30.80877113342285,
      "activations/layer15_attention_weight_max": 43.64692306518555,
      "activations/layer15_attention_weight_min": -29.796710968017578,
      "activations/layer16_attention_weight_max": 35.683738708496094,
      "activations/layer16_attention_weight_min": -27.249828338623047,
      "activations/layer17_attention_weight_max": 57.01335906982422,
      "activations/layer17_attention_weight_min": -42.274967193603516,
      "activations/layer18_attention_weight_max": 50.28536605834961,
      "activations/layer18_attention_weight_min": -33.835723876953125,
      "activations/layer19_attention_weight_max": 25.18390464782715,
      "activations/layer19_attention_weight_min": -20.02195167541504,
      "activations/layer1_attention_weight_max": 17.113908767700195,
      "activations/layer1_attention_weight_min": -14.330961227416992,
      "activations/layer20_attention_weight_max": 21.552555084228516,
      "activations/layer20_attention_weight_min": -22.750703811645508,
      "activations/layer21_attention_weight_max": 41.34331512451172,
      "activations/layer21_attention_weight_min": -25.420757293701172,
      "activations/layer22_attention_weight_max": 36.44483184814453,
      "activations/layer22_attention_weight_min": -24.121227264404297,
      "activations/layer23_attention_weight_max": 31.4696102142334,
      "activations/layer23_attention_weight_min": -19.54473114013672,
      "activations/layer2_attention_weight_max": 30.825145721435547,
      "activations/layer2_attention_weight_min": -30.34064483642578,
      "activations/layer3_attention_weight_max": 84.08372497558594,
      "activations/layer3_attention_weight_min": -89.06302642822266,
      "activations/layer4_attention_weight_max": 104.66301727294922,
      "activations/layer4_attention_weight_min": -98.85649871826172,
      "activations/layer5_attention_weight_max": 104.22447204589844,
      "activations/layer5_attention_weight_min": -89.76461791992188,
      "activations/layer6_attention_weight_max": 59.82887268066406,
      "activations/layer6_attention_weight_min": -53.570228576660156,
      "activations/layer7_attention_weight_max": 71.90428161621094,
      "activations/layer7_attention_weight_min": -69.4898681640625,
      "activations/layer8_attention_weight_max": 44.73048782348633,
      "activations/layer8_attention_weight_min": -44.90564727783203,
      "activations/layer9_attention_weight_max": 56.04520797729492,
      "activations/layer9_attention_weight_min": -51.60477828979492,
      "epoch": 5.69,
      "learning_rate": 0.00011448712121212121,
      "loss": 2.8336,
      "step": 97850
    },
    {
      "activations/layer0_attention_weight_max": 13.791651725769043,
      "activations/layer0_attention_weight_min": -15.39431381225586,
      "activations/layer10_attention_weight_max": 36.478065490722656,
      "activations/layer10_attention_weight_min": -32.16204833984375,
      "activations/layer11_attention_weight_max": 29.827573776245117,
      "activations/layer11_attention_weight_min": -26.301597595214844,
      "activations/layer12_attention_weight_max": 19.564064025878906,
      "activations/layer12_attention_weight_min": -25.790266036987305,
      "activations/layer13_attention_weight_max": 41.54570007324219,
      "activations/layer13_attention_weight_min": -32.801063537597656,
      "activations/layer14_attention_weight_max": 42.023956298828125,
      "activations/layer14_attention_weight_min": -28.83693504333496,
      "activations/layer15_attention_weight_max": 36.79334259033203,
      "activations/layer15_attention_weight_min": -29.570314407348633,
      "activations/layer16_attention_weight_max": 36.4316291809082,
      "activations/layer16_attention_weight_min": -25.754819869995117,
      "activations/layer17_attention_weight_max": 50.44588088989258,
      "activations/layer17_attention_weight_min": -38.93098449707031,
      "activations/layer18_attention_weight_max": 44.91069793701172,
      "activations/layer18_attention_weight_min": -34.578433990478516,
      "activations/layer19_attention_weight_max": 22.35716438293457,
      "activations/layer19_attention_weight_min": -19.623001098632812,
      "activations/layer1_attention_weight_max": 18.597883224487305,
      "activations/layer1_attention_weight_min": -15.797438621520996,
      "activations/layer20_attention_weight_max": 20.780517578125,
      "activations/layer20_attention_weight_min": -20.76171112060547,
      "activations/layer21_attention_weight_max": 36.89493942260742,
      "activations/layer21_attention_weight_min": -26.419265747070312,
      "activations/layer22_attention_weight_max": 33.70766067504883,
      "activations/layer22_attention_weight_min": -24.10055160522461,
      "activations/layer23_attention_weight_max": 29.01543426513672,
      "activations/layer23_attention_weight_min": -18.49057388305664,
      "activations/layer2_attention_weight_max": 31.641569137573242,
      "activations/layer2_attention_weight_min": -28.735240936279297,
      "activations/layer3_attention_weight_max": 85.16232299804688,
      "activations/layer3_attention_weight_min": -87.10660552978516,
      "activations/layer4_attention_weight_max": 102.05731201171875,
      "activations/layer4_attention_weight_min": -93.42266082763672,
      "activations/layer5_attention_weight_max": 94.31256103515625,
      "activations/layer5_attention_weight_min": -85.08804321289062,
      "activations/layer6_attention_weight_max": 56.67885208129883,
      "activations/layer6_attention_weight_min": -50.89912033081055,
      "activations/layer7_attention_weight_max": 70.27650451660156,
      "activations/layer7_attention_weight_min": -68.90182495117188,
      "activations/layer8_attention_weight_max": 42.62921905517578,
      "activations/layer8_attention_weight_min": -41.76252746582031,
      "activations/layer9_attention_weight_max": 51.06618118286133,
      "activations/layer9_attention_weight_min": -50.47406768798828,
      "epoch": 5.69,
      "learning_rate": 0.00011446818181818181,
      "loss": 2.8352,
      "step": 97900
    },
    {
      "activations/layer0_attention_weight_max": 14.829817771911621,
      "activations/layer0_attention_weight_min": -15.213444709777832,
      "activations/layer10_attention_weight_max": 36.47321319580078,
      "activations/layer10_attention_weight_min": -33.73765563964844,
      "activations/layer11_attention_weight_max": 28.384933471679688,
      "activations/layer11_attention_weight_min": -27.36496353149414,
      "activations/layer12_attention_weight_max": 20.59054183959961,
      "activations/layer12_attention_weight_min": -25.25358772277832,
      "activations/layer13_attention_weight_max": 43.87084197998047,
      "activations/layer13_attention_weight_min": -32.65816879272461,
      "activations/layer14_attention_weight_max": 39.386043548583984,
      "activations/layer14_attention_weight_min": -29.007089614868164,
      "activations/layer15_attention_weight_max": 42.573123931884766,
      "activations/layer15_attention_weight_min": -31.800220489501953,
      "activations/layer16_attention_weight_max": 30.88545799255371,
      "activations/layer16_attention_weight_min": -25.522232055664062,
      "activations/layer17_attention_weight_max": 51.26091003417969,
      "activations/layer17_attention_weight_min": -39.52716064453125,
      "activations/layer18_attention_weight_max": 45.449954986572266,
      "activations/layer18_attention_weight_min": -35.098175048828125,
      "activations/layer19_attention_weight_max": 20.759727478027344,
      "activations/layer19_attention_weight_min": -22.144908905029297,
      "activations/layer1_attention_weight_max": 16.74184799194336,
      "activations/layer1_attention_weight_min": -16.01065444946289,
      "activations/layer20_attention_weight_max": 19.841672897338867,
      "activations/layer20_attention_weight_min": -23.7925968170166,
      "activations/layer21_attention_weight_max": 41.383235931396484,
      "activations/layer21_attention_weight_min": -24.98979949951172,
      "activations/layer22_attention_weight_max": 33.403564453125,
      "activations/layer22_attention_weight_min": -24.3210391998291,
      "activations/layer23_attention_weight_max": 25.393896102905273,
      "activations/layer23_attention_weight_min": -21.212181091308594,
      "activations/layer2_attention_weight_max": 30.814353942871094,
      "activations/layer2_attention_weight_min": -30.077308654785156,
      "activations/layer3_attention_weight_max": 83.9932861328125,
      "activations/layer3_attention_weight_min": -90.51024627685547,
      "activations/layer4_attention_weight_max": 100.84515380859375,
      "activations/layer4_attention_weight_min": -100.97883605957031,
      "activations/layer5_attention_weight_max": 94.2373275756836,
      "activations/layer5_attention_weight_min": -89.7393798828125,
      "activations/layer6_attention_weight_max": 56.36956024169922,
      "activations/layer6_attention_weight_min": -53.00824737548828,
      "activations/layer7_attention_weight_max": 66.82363891601562,
      "activations/layer7_attention_weight_min": -62.62040328979492,
      "activations/layer8_attention_weight_max": 43.349517822265625,
      "activations/layer8_attention_weight_min": -42.106224060058594,
      "activations/layer9_attention_weight_max": 47.345428466796875,
      "activations/layer9_attention_weight_min": -43.39746856689453,
      "epoch": 5.69,
      "learning_rate": 0.00011444924242424241,
      "loss": 2.833,
      "step": 97950
    },
    {
      "activations/layer0_attention_weight_max": 14.390271186828613,
      "activations/layer0_attention_weight_min": -14.63735580444336,
      "activations/layer10_attention_weight_max": 37.98009490966797,
      "activations/layer10_attention_weight_min": -35.67047119140625,
      "activations/layer11_attention_weight_max": 28.671924591064453,
      "activations/layer11_attention_weight_min": -27.908676147460938,
      "activations/layer12_attention_weight_max": 22.5164852142334,
      "activations/layer12_attention_weight_min": -27.177461624145508,
      "activations/layer13_attention_weight_max": 51.604591369628906,
      "activations/layer13_attention_weight_min": -35.24051284790039,
      "activations/layer14_attention_weight_max": 47.829612731933594,
      "activations/layer14_attention_weight_min": -34.92457962036133,
      "activations/layer15_attention_weight_max": 39.89336395263672,
      "activations/layer15_attention_weight_min": -30.751054763793945,
      "activations/layer16_attention_weight_max": 34.32803726196289,
      "activations/layer16_attention_weight_min": -25.580537796020508,
      "activations/layer17_attention_weight_max": 52.385345458984375,
      "activations/layer17_attention_weight_min": -38.73488235473633,
      "activations/layer18_attention_weight_max": 48.73997116088867,
      "activations/layer18_attention_weight_min": -33.54499053955078,
      "activations/layer19_attention_weight_max": 20.037572860717773,
      "activations/layer19_attention_weight_min": -20.806591033935547,
      "activations/layer1_attention_weight_max": 16.99559783935547,
      "activations/layer1_attention_weight_min": -15.685789108276367,
      "activations/layer20_attention_weight_max": 19.983806610107422,
      "activations/layer20_attention_weight_min": -24.567344665527344,
      "activations/layer21_attention_weight_max": 38.33480453491211,
      "activations/layer21_attention_weight_min": -29.75408935546875,
      "activations/layer22_attention_weight_max": 34.26020050048828,
      "activations/layer22_attention_weight_min": -24.747413635253906,
      "activations/layer23_attention_weight_max": 25.741008758544922,
      "activations/layer23_attention_weight_min": -18.269668579101562,
      "activations/layer2_attention_weight_max": 31.719314575195312,
      "activations/layer2_attention_weight_min": -28.208965301513672,
      "activations/layer3_attention_weight_max": 80.94334411621094,
      "activations/layer3_attention_weight_min": -83.68612670898438,
      "activations/layer4_attention_weight_max": 100.29148864746094,
      "activations/layer4_attention_weight_min": -93.52356719970703,
      "activations/layer5_attention_weight_max": 94.32208251953125,
      "activations/layer5_attention_weight_min": -85.81385803222656,
      "activations/layer6_attention_weight_max": 56.54974365234375,
      "activations/layer6_attention_weight_min": -49.610599517822266,
      "activations/layer7_attention_weight_max": 74.06610107421875,
      "activations/layer7_attention_weight_min": -68.64064025878906,
      "activations/layer8_attention_weight_max": 42.56705856323242,
      "activations/layer8_attention_weight_min": -43.80740737915039,
      "activations/layer9_attention_weight_max": 51.989715576171875,
      "activations/layer9_attention_weight_min": -48.2618522644043,
      "epoch": 5.69,
      "learning_rate": 0.00011443030303030303,
      "loss": 2.8222,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_loss": 2.791015625,
      "eval_runtime": 8.563,
      "eval_samples_per_second": 501.461,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_openwebtext_loss": 2.791015625,
      "eval_openwebtext_ppl": 16.2975636122563,
      "eval_openwebtext_runtime": 8.563,
      "eval_openwebtext_samples_per_second": 501.461,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_wikitext_loss": 3.0234375,
      "eval_wikitext_ppl": 20.56185170491249,
      "eval_wikitext_runtime": 1.9796,
      "eval_wikitext_samples_per_second": 230.351,
      "step": 98000
    },
    {
      "epoch": 5.69,
      "eval_lambada_loss": 2.75390625,
      "eval_lambada_ppl": 15.703855414343392,
      "eval_lambada_runtime": 9.6054,
      "eval_lambada_samples_per_second": 506.904,
      "step": 98000
    },
    {
      "activations/layer0_attention_weight_max": 14.760523796081543,
      "activations/layer0_attention_weight_min": -14.93160629272461,
      "activations/layer10_attention_weight_max": 35.6039924621582,
      "activations/layer10_attention_weight_min": -32.88014221191406,
      "activations/layer11_attention_weight_max": 29.386676788330078,
      "activations/layer11_attention_weight_min": -28.17896270751953,
      "activations/layer12_attention_weight_max": 20.188278198242188,
      "activations/layer12_attention_weight_min": -23.956871032714844,
      "activations/layer13_attention_weight_max": 45.4200553894043,
      "activations/layer13_attention_weight_min": -32.77582550048828,
      "activations/layer14_attention_weight_max": 42.631690979003906,
      "activations/layer14_attention_weight_min": -29.544504165649414,
      "activations/layer15_attention_weight_max": 42.09833526611328,
      "activations/layer15_attention_weight_min": -29.914966583251953,
      "activations/layer16_attention_weight_max": 35.12376403808594,
      "activations/layer16_attention_weight_min": -26.42631721496582,
      "activations/layer17_attention_weight_max": 57.05255126953125,
      "activations/layer17_attention_weight_min": -40.26946258544922,
      "activations/layer18_attention_weight_max": 48.971134185791016,
      "activations/layer18_attention_weight_min": -35.63553237915039,
      "activations/layer19_attention_weight_max": 25.37677001953125,
      "activations/layer19_attention_weight_min": -21.797441482543945,
      "activations/layer1_attention_weight_max": 18.951332092285156,
      "activations/layer1_attention_weight_min": -16.069055557250977,
      "activations/layer20_attention_weight_max": 21.70182991027832,
      "activations/layer20_attention_weight_min": -20.79491424560547,
      "activations/layer21_attention_weight_max": 39.853355407714844,
      "activations/layer21_attention_weight_min": -24.4613094329834,
      "activations/layer22_attention_weight_max": 33.81248092651367,
      "activations/layer22_attention_weight_min": -24.013750076293945,
      "activations/layer23_attention_weight_max": 28.699369430541992,
      "activations/layer23_attention_weight_min": -19.278079986572266,
      "activations/layer2_attention_weight_max": 30.185760498046875,
      "activations/layer2_attention_weight_min": -28.409513473510742,
      "activations/layer3_attention_weight_max": 82.8841781616211,
      "activations/layer3_attention_weight_min": -92.04324340820312,
      "activations/layer4_attention_weight_max": 98.32500457763672,
      "activations/layer4_attention_weight_min": -98.77071380615234,
      "activations/layer5_attention_weight_max": 96.60527801513672,
      "activations/layer5_attention_weight_min": -89.28441619873047,
      "activations/layer6_attention_weight_max": 55.14916229248047,
      "activations/layer6_attention_weight_min": -52.033973693847656,
      "activations/layer7_attention_weight_max": 68.18217468261719,
      "activations/layer7_attention_weight_min": -65.89480590820312,
      "activations/layer8_attention_weight_max": 42.89801788330078,
      "activations/layer8_attention_weight_min": -46.01764678955078,
      "activations/layer9_attention_weight_max": 46.23830795288086,
      "activations/layer9_attention_weight_min": -46.42023468017578,
      "epoch": 5.7,
      "learning_rate": 0.00011441136363636363,
      "loss": 2.8447,
      "step": 98050
    },
    {
      "activations/layer0_attention_weight_max": 15.399112701416016,
      "activations/layer0_attention_weight_min": -14.907033920288086,
      "activations/layer10_attention_weight_max": 38.284210205078125,
      "activations/layer10_attention_weight_min": -34.104251861572266,
      "activations/layer11_attention_weight_max": 29.519775390625,
      "activations/layer11_attention_weight_min": -27.822444915771484,
      "activations/layer12_attention_weight_max": 21.618215560913086,
      "activations/layer12_attention_weight_min": -30.527368545532227,
      "activations/layer13_attention_weight_max": 46.16835021972656,
      "activations/layer13_attention_weight_min": -34.05915069580078,
      "activations/layer14_attention_weight_max": 53.71417236328125,
      "activations/layer14_attention_weight_min": -35.459144592285156,
      "activations/layer15_attention_weight_max": 40.545005798339844,
      "activations/layer15_attention_weight_min": -35.016082763671875,
      "activations/layer16_attention_weight_max": 36.422061920166016,
      "activations/layer16_attention_weight_min": -28.392087936401367,
      "activations/layer17_attention_weight_max": 55.5936279296875,
      "activations/layer17_attention_weight_min": -39.98963165283203,
      "activations/layer18_attention_weight_max": 47.85074234008789,
      "activations/layer18_attention_weight_min": -35.52885818481445,
      "activations/layer19_attention_weight_max": 23.65550422668457,
      "activations/layer19_attention_weight_min": -21.212968826293945,
      "activations/layer1_attention_weight_max": 17.435100555419922,
      "activations/layer1_attention_weight_min": -14.490950584411621,
      "activations/layer20_attention_weight_max": 20.55105209350586,
      "activations/layer20_attention_weight_min": -20.425920486450195,
      "activations/layer21_attention_weight_max": 43.31642150878906,
      "activations/layer21_attention_weight_min": -25.730581283569336,
      "activations/layer22_attention_weight_max": 36.2745475769043,
      "activations/layer22_attention_weight_min": -24.37348175048828,
      "activations/layer23_attention_weight_max": 29.352317810058594,
      "activations/layer23_attention_weight_min": -17.8796443939209,
      "activations/layer2_attention_weight_max": 32.10694885253906,
      "activations/layer2_attention_weight_min": -30.378070831298828,
      "activations/layer3_attention_weight_max": 83.0043716430664,
      "activations/layer3_attention_weight_min": -86.17876434326172,
      "activations/layer4_attention_weight_max": 101.00464630126953,
      "activations/layer4_attention_weight_min": -93.9342041015625,
      "activations/layer5_attention_weight_max": 98.17681884765625,
      "activations/layer5_attention_weight_min": -85.97041320800781,
      "activations/layer6_attention_weight_max": 58.9199104309082,
      "activations/layer6_attention_weight_min": -52.211429595947266,
      "activations/layer7_attention_weight_max": 75.61308288574219,
      "activations/layer7_attention_weight_min": -71.65470886230469,
      "activations/layer8_attention_weight_max": 44.89875793457031,
      "activations/layer8_attention_weight_min": -43.50732421875,
      "activations/layer9_attention_weight_max": 53.157623291015625,
      "activations/layer9_attention_weight_min": -49.985084533691406,
      "epoch": 5.7,
      "learning_rate": 0.00011439242424242423,
      "loss": 2.8502,
      "step": 98100
    },
    {
      "activations/layer0_attention_weight_max": 14.625988006591797,
      "activations/layer0_attention_weight_min": -14.816774368286133,
      "activations/layer10_attention_weight_max": 36.21348190307617,
      "activations/layer10_attention_weight_min": -33.76094436645508,
      "activations/layer11_attention_weight_max": 28.259031295776367,
      "activations/layer11_attention_weight_min": -26.876819610595703,
      "activations/layer12_attention_weight_max": 25.338743209838867,
      "activations/layer12_attention_weight_min": -25.8347225189209,
      "activations/layer13_attention_weight_max": 43.96720886230469,
      "activations/layer13_attention_weight_min": -34.513309478759766,
      "activations/layer14_attention_weight_max": 50.47917175292969,
      "activations/layer14_attention_weight_min": -34.8106575012207,
      "activations/layer15_attention_weight_max": 40.061004638671875,
      "activations/layer15_attention_weight_min": -30.30903434753418,
      "activations/layer16_attention_weight_max": 35.55312728881836,
      "activations/layer16_attention_weight_min": -25.864063262939453,
      "activations/layer17_attention_weight_max": 54.47047805786133,
      "activations/layer17_attention_weight_min": -38.166542053222656,
      "activations/layer18_attention_weight_max": 46.69225311279297,
      "activations/layer18_attention_weight_min": -34.27025604248047,
      "activations/layer19_attention_weight_max": 24.703214645385742,
      "activations/layer19_attention_weight_min": -20.18653106689453,
      "activations/layer1_attention_weight_max": 16.519920349121094,
      "activations/layer1_attention_weight_min": -14.873772621154785,
      "activations/layer20_attention_weight_max": 23.447154998779297,
      "activations/layer20_attention_weight_min": -23.588178634643555,
      "activations/layer21_attention_weight_max": 42.2549934387207,
      "activations/layer21_attention_weight_min": -25.622671127319336,
      "activations/layer22_attention_weight_max": 34.20252227783203,
      "activations/layer22_attention_weight_min": -25.1652774810791,
      "activations/layer23_attention_weight_max": 27.402252197265625,
      "activations/layer23_attention_weight_min": -20.042064666748047,
      "activations/layer2_attention_weight_max": 31.698171615600586,
      "activations/layer2_attention_weight_min": -28.57498550415039,
      "activations/layer3_attention_weight_max": 80.94773864746094,
      "activations/layer3_attention_weight_min": -80.95264434814453,
      "activations/layer4_attention_weight_max": 95.55022430419922,
      "activations/layer4_attention_weight_min": -93.05525970458984,
      "activations/layer5_attention_weight_max": 94.94456481933594,
      "activations/layer5_attention_weight_min": -87.22184753417969,
      "activations/layer6_attention_weight_max": 55.345428466796875,
      "activations/layer6_attention_weight_min": -49.93803024291992,
      "activations/layer7_attention_weight_max": 73.61888885498047,
      "activations/layer7_attention_weight_min": -64.93510437011719,
      "activations/layer8_attention_weight_max": 39.818939208984375,
      "activations/layer8_attention_weight_min": -40.18977737426758,
      "activations/layer9_attention_weight_max": 48.432071685791016,
      "activations/layer9_attention_weight_min": -46.68312454223633,
      "epoch": 5.7,
      "learning_rate": 0.00011437348484848483,
      "loss": 2.8253,
      "step": 98150
    },
    {
      "activations/layer0_attention_weight_max": 14.151055335998535,
      "activations/layer0_attention_weight_min": -14.705937385559082,
      "activations/layer10_attention_weight_max": 42.84965896606445,
      "activations/layer10_attention_weight_min": -38.7398567199707,
      "activations/layer11_attention_weight_max": 30.945158004760742,
      "activations/layer11_attention_weight_min": -30.799407958984375,
      "activations/layer12_attention_weight_max": 21.396493911743164,
      "activations/layer12_attention_weight_min": -26.2513370513916,
      "activations/layer13_attention_weight_max": 55.69948196411133,
      "activations/layer13_attention_weight_min": -40.202449798583984,
      "activations/layer14_attention_weight_max": 41.78469467163086,
      "activations/layer14_attention_weight_min": -31.479915618896484,
      "activations/layer15_attention_weight_max": 43.43022537231445,
      "activations/layer15_attention_weight_min": -32.21842575073242,
      "activations/layer16_attention_weight_max": 34.8672981262207,
      "activations/layer16_attention_weight_min": -26.854860305786133,
      "activations/layer17_attention_weight_max": 62.05605697631836,
      "activations/layer17_attention_weight_min": -44.28557586669922,
      "activations/layer18_attention_weight_max": 51.32340621948242,
      "activations/layer18_attention_weight_min": -36.09885787963867,
      "activations/layer19_attention_weight_max": 27.9807186126709,
      "activations/layer19_attention_weight_min": -19.386564254760742,
      "activations/layer1_attention_weight_max": 17.419322967529297,
      "activations/layer1_attention_weight_min": -16.107179641723633,
      "activations/layer20_attention_weight_max": 23.6579532623291,
      "activations/layer20_attention_weight_min": -20.713165283203125,
      "activations/layer21_attention_weight_max": 40.63684844970703,
      "activations/layer21_attention_weight_min": -25.138811111450195,
      "activations/layer22_attention_weight_max": 36.225154876708984,
      "activations/layer22_attention_weight_min": -23.90340232849121,
      "activations/layer23_attention_weight_max": 30.499649047851562,
      "activations/layer23_attention_weight_min": -21.845523834228516,
      "activations/layer2_attention_weight_max": 31.299169540405273,
      "activations/layer2_attention_weight_min": -28.987926483154297,
      "activations/layer3_attention_weight_max": 82.41071319580078,
      "activations/layer3_attention_weight_min": -84.71700286865234,
      "activations/layer4_attention_weight_max": 101.3849105834961,
      "activations/layer4_attention_weight_min": -99.18635559082031,
      "activations/layer5_attention_weight_max": 95.7384033203125,
      "activations/layer5_attention_weight_min": -91.1771240234375,
      "activations/layer6_attention_weight_max": 57.271976470947266,
      "activations/layer6_attention_weight_min": -54.40095138549805,
      "activations/layer7_attention_weight_max": 70.58352661132812,
      "activations/layer7_attention_weight_min": -66.15541076660156,
      "activations/layer8_attention_weight_max": 44.80609893798828,
      "activations/layer8_attention_weight_min": -44.89069747924805,
      "activations/layer9_attention_weight_max": 51.07572555541992,
      "activations/layer9_attention_weight_min": -54.79561233520508,
      "epoch": 5.71,
      "learning_rate": 0.00011435492424242424,
      "loss": 2.8657,
      "step": 98200
    },
    {
      "activations/layer0_attention_weight_max": 14.856833457946777,
      "activations/layer0_attention_weight_min": -14.818564414978027,
      "activations/layer10_attention_weight_max": 37.12291717529297,
      "activations/layer10_attention_weight_min": -34.06182861328125,
      "activations/layer11_attention_weight_max": 30.023365020751953,
      "activations/layer11_attention_weight_min": -27.76447296142578,
      "activations/layer12_attention_weight_max": 28.16025733947754,
      "activations/layer12_attention_weight_min": -26.344911575317383,
      "activations/layer13_attention_weight_max": 51.10111999511719,
      "activations/layer13_attention_weight_min": -36.40114974975586,
      "activations/layer14_attention_weight_max": 57.716007232666016,
      "activations/layer14_attention_weight_min": -40.38283157348633,
      "activations/layer15_attention_weight_max": 48.33662414550781,
      "activations/layer15_attention_weight_min": -32.19477462768555,
      "activations/layer16_attention_weight_max": 36.5922737121582,
      "activations/layer16_attention_weight_min": -27.497514724731445,
      "activations/layer17_attention_weight_max": 57.75189208984375,
      "activations/layer17_attention_weight_min": -38.805992126464844,
      "activations/layer18_attention_weight_max": 48.29037857055664,
      "activations/layer18_attention_weight_min": -34.197261810302734,
      "activations/layer19_attention_weight_max": 25.524717330932617,
      "activations/layer19_attention_weight_min": -20.13300895690918,
      "activations/layer1_attention_weight_max": 17.65607261657715,
      "activations/layer1_attention_weight_min": -14.384002685546875,
      "activations/layer20_attention_weight_max": 23.357572555541992,
      "activations/layer20_attention_weight_min": -25.264806747436523,
      "activations/layer21_attention_weight_max": 43.9108772277832,
      "activations/layer21_attention_weight_min": -28.991830825805664,
      "activations/layer22_attention_weight_max": 37.66244888305664,
      "activations/layer22_attention_weight_min": -23.81644630432129,
      "activations/layer23_attention_weight_max": 35.092803955078125,
      "activations/layer23_attention_weight_min": -18.230587005615234,
      "activations/layer2_attention_weight_max": 30.144948959350586,
      "activations/layer2_attention_weight_min": -29.480854034423828,
      "activations/layer3_attention_weight_max": 83.92475128173828,
      "activations/layer3_attention_weight_min": -81.78524780273438,
      "activations/layer4_attention_weight_max": 94.60425567626953,
      "activations/layer4_attention_weight_min": -87.17150115966797,
      "activations/layer5_attention_weight_max": 90.12969207763672,
      "activations/layer5_attention_weight_min": -80.97511291503906,
      "activations/layer6_attention_weight_max": 52.638912200927734,
      "activations/layer6_attention_weight_min": -48.1219367980957,
      "activations/layer7_attention_weight_max": 68.37149047851562,
      "activations/layer7_attention_weight_min": -64.70578002929688,
      "activations/layer8_attention_weight_max": 41.92256164550781,
      "activations/layer8_attention_weight_min": -40.04664611816406,
      "activations/layer9_attention_weight_max": 54.70267868041992,
      "activations/layer9_attention_weight_min": -50.3111686706543,
      "epoch": 5.71,
      "learning_rate": 0.00011433598484848484,
      "loss": 2.8481,
      "step": 98250
    },
    {
      "activations/layer0_attention_weight_max": 15.022143363952637,
      "activations/layer0_attention_weight_min": -14.893141746520996,
      "activations/layer10_attention_weight_max": 38.807586669921875,
      "activations/layer10_attention_weight_min": -33.98372268676758,
      "activations/layer11_attention_weight_max": 33.110225677490234,
      "activations/layer11_attention_weight_min": -29.55361557006836,
      "activations/layer12_attention_weight_max": 21.159753799438477,
      "activations/layer12_attention_weight_min": -23.9847469329834,
      "activations/layer13_attention_weight_max": 52.7577018737793,
      "activations/layer13_attention_weight_min": -34.39060592651367,
      "activations/layer14_attention_weight_max": 39.93340301513672,
      "activations/layer14_attention_weight_min": -32.28858184814453,
      "activations/layer15_attention_weight_max": 42.562496185302734,
      "activations/layer15_attention_weight_min": -30.700477600097656,
      "activations/layer16_attention_weight_max": 39.034236907958984,
      "activations/layer16_attention_weight_min": -27.07148551940918,
      "activations/layer17_attention_weight_max": 56.3385009765625,
      "activations/layer17_attention_weight_min": -42.671173095703125,
      "activations/layer18_attention_weight_max": 48.67869186401367,
      "activations/layer18_attention_weight_min": -35.6489372253418,
      "activations/layer19_attention_weight_max": 22.576894760131836,
      "activations/layer19_attention_weight_min": -20.517499923706055,
      "activations/layer1_attention_weight_max": 17.06391143798828,
      "activations/layer1_attention_weight_min": -15.191524505615234,
      "activations/layer20_attention_weight_max": 22.616464614868164,
      "activations/layer20_attention_weight_min": -25.326560974121094,
      "activations/layer21_attention_weight_max": 39.47099685668945,
      "activations/layer21_attention_weight_min": -26.554332733154297,
      "activations/layer22_attention_weight_max": 33.41062545776367,
      "activations/layer22_attention_weight_min": -26.005878448486328,
      "activations/layer23_attention_weight_max": 32.04837417602539,
      "activations/layer23_attention_weight_min": -18.127092361450195,
      "activations/layer2_attention_weight_max": 28.150733947753906,
      "activations/layer2_attention_weight_min": -28.70178985595703,
      "activations/layer3_attention_weight_max": 81.5848159790039,
      "activations/layer3_attention_weight_min": -81.65242767333984,
      "activations/layer4_attention_weight_max": 99.2474136352539,
      "activations/layer4_attention_weight_min": -93.62596893310547,
      "activations/layer5_attention_weight_max": 96.25729370117188,
      "activations/layer5_attention_weight_min": -86.14790344238281,
      "activations/layer6_attention_weight_max": 58.4747428894043,
      "activations/layer6_attention_weight_min": -53.150489807128906,
      "activations/layer7_attention_weight_max": 70.01516723632812,
      "activations/layer7_attention_weight_min": -68.81464385986328,
      "activations/layer8_attention_weight_max": 42.573997497558594,
      "activations/layer8_attention_weight_min": -40.14807891845703,
      "activations/layer9_attention_weight_max": 49.53806686401367,
      "activations/layer9_attention_weight_min": -46.4223518371582,
      "epoch": 5.71,
      "learning_rate": 0.00011431704545454545,
      "loss": 2.8477,
      "step": 98300
    },
    {
      "activations/layer0_attention_weight_max": 14.62547779083252,
      "activations/layer0_attention_weight_min": -15.328591346740723,
      "activations/layer10_attention_weight_max": 46.074371337890625,
      "activations/layer10_attention_weight_min": -37.70121765136719,
      "activations/layer11_attention_weight_max": 32.010284423828125,
      "activations/layer11_attention_weight_min": -30.137372970581055,
      "activations/layer12_attention_weight_max": 18.976465225219727,
      "activations/layer12_attention_weight_min": -24.89240264892578,
      "activations/layer13_attention_weight_max": 44.887428283691406,
      "activations/layer13_attention_weight_min": -37.20710754394531,
      "activations/layer14_attention_weight_max": 35.306678771972656,
      "activations/layer14_attention_weight_min": -28.32477378845215,
      "activations/layer15_attention_weight_max": 41.096092224121094,
      "activations/layer15_attention_weight_min": -29.331308364868164,
      "activations/layer16_attention_weight_max": 33.26557922363281,
      "activations/layer16_attention_weight_min": -25.790037155151367,
      "activations/layer17_attention_weight_max": 54.61238098144531,
      "activations/layer17_attention_weight_min": -38.26430130004883,
      "activations/layer18_attention_weight_max": 44.7630729675293,
      "activations/layer18_attention_weight_min": -33.186248779296875,
      "activations/layer19_attention_weight_max": 23.395034790039062,
      "activations/layer19_attention_weight_min": -20.15018081665039,
      "activations/layer1_attention_weight_max": 17.78532600402832,
      "activations/layer1_attention_weight_min": -14.977187156677246,
      "activations/layer20_attention_weight_max": 20.35578155517578,
      "activations/layer20_attention_weight_min": -20.280532836914062,
      "activations/layer21_attention_weight_max": 40.72505187988281,
      "activations/layer21_attention_weight_min": -27.32016944885254,
      "activations/layer22_attention_weight_max": 33.620750427246094,
      "activations/layer22_attention_weight_min": -24.31406021118164,
      "activations/layer23_attention_weight_max": 30.480751037597656,
      "activations/layer23_attention_weight_min": -20.72504997253418,
      "activations/layer2_attention_weight_max": 30.734230041503906,
      "activations/layer2_attention_weight_min": -28.902179718017578,
      "activations/layer3_attention_weight_max": 84.88810729980469,
      "activations/layer3_attention_weight_min": -85.07072448730469,
      "activations/layer4_attention_weight_max": 105.89617156982422,
      "activations/layer4_attention_weight_min": -101.22307586669922,
      "activations/layer5_attention_weight_max": 104.15727233886719,
      "activations/layer5_attention_weight_min": -93.68343353271484,
      "activations/layer6_attention_weight_max": 61.48978042602539,
      "activations/layer6_attention_weight_min": -56.0462760925293,
      "activations/layer7_attention_weight_max": 76.84088897705078,
      "activations/layer7_attention_weight_min": -70.86006927490234,
      "activations/layer8_attention_weight_max": 48.37600326538086,
      "activations/layer8_attention_weight_min": -46.65858840942383,
      "activations/layer9_attention_weight_max": 53.93738555908203,
      "activations/layer9_attention_weight_min": -48.731990814208984,
      "epoch": 5.71,
      "learning_rate": 0.00011429810606060604,
      "loss": 2.8474,
      "step": 98350
    },
    {
      "activations/layer0_attention_weight_max": 14.205361366271973,
      "activations/layer0_attention_weight_min": -15.215353965759277,
      "activations/layer10_attention_weight_max": 38.44843292236328,
      "activations/layer10_attention_weight_min": -34.49333572387695,
      "activations/layer11_attention_weight_max": 29.41767120361328,
      "activations/layer11_attention_weight_min": -28.243267059326172,
      "activations/layer12_attention_weight_max": 19.528261184692383,
      "activations/layer12_attention_weight_min": -23.383249282836914,
      "activations/layer13_attention_weight_max": 47.132652282714844,
      "activations/layer13_attention_weight_min": -32.91442108154297,
      "activations/layer14_attention_weight_max": 40.0526008605957,
      "activations/layer14_attention_weight_min": -29.57264518737793,
      "activations/layer15_attention_weight_max": 37.0892448425293,
      "activations/layer15_attention_weight_min": -30.0407772064209,
      "activations/layer16_attention_weight_max": 32.146087646484375,
      "activations/layer16_attention_weight_min": -26.158103942871094,
      "activations/layer17_attention_weight_max": 55.116973876953125,
      "activations/layer17_attention_weight_min": -37.79964065551758,
      "activations/layer18_attention_weight_max": 45.32413101196289,
      "activations/layer18_attention_weight_min": -31.14845085144043,
      "activations/layer19_attention_weight_max": 22.18899154663086,
      "activations/layer19_attention_weight_min": -19.54866600036621,
      "activations/layer1_attention_weight_max": 17.158817291259766,
      "activations/layer1_attention_weight_min": -16.366933822631836,
      "activations/layer20_attention_weight_max": 22.727689743041992,
      "activations/layer20_attention_weight_min": -23.791780471801758,
      "activations/layer21_attention_weight_max": 35.84897994995117,
      "activations/layer21_attention_weight_min": -22.668777465820312,
      "activations/layer22_attention_weight_max": 35.99468231201172,
      "activations/layer22_attention_weight_min": -23.664623260498047,
      "activations/layer23_attention_weight_max": 27.97589874267578,
      "activations/layer23_attention_weight_min": -17.32497787475586,
      "activations/layer2_attention_weight_max": 31.581220626831055,
      "activations/layer2_attention_weight_min": -30.48586654663086,
      "activations/layer3_attention_weight_max": 87.23056030273438,
      "activations/layer3_attention_weight_min": -91.33361053466797,
      "activations/layer4_attention_weight_max": 104.5750732421875,
      "activations/layer4_attention_weight_min": -100.3406982421875,
      "activations/layer5_attention_weight_max": 102.04618835449219,
      "activations/layer5_attention_weight_min": -90.5898666381836,
      "activations/layer6_attention_weight_max": 59.058658599853516,
      "activations/layer6_attention_weight_min": -52.806522369384766,
      "activations/layer7_attention_weight_max": 73.23870086669922,
      "activations/layer7_attention_weight_min": -68.9684066772461,
      "activations/layer8_attention_weight_max": 44.436729431152344,
      "activations/layer8_attention_weight_min": -43.21489334106445,
      "activations/layer9_attention_weight_max": 51.028629302978516,
      "activations/layer9_attention_weight_min": -46.324954986572266,
      "epoch": 5.72,
      "learning_rate": 0.00011427916666666665,
      "loss": 2.8364,
      "step": 98400
    },
    {
      "activations/layer0_attention_weight_max": 15.299917221069336,
      "activations/layer0_attention_weight_min": -15.417595863342285,
      "activations/layer10_attention_weight_max": 37.34237289428711,
      "activations/layer10_attention_weight_min": -34.8697395324707,
      "activations/layer11_attention_weight_max": 29.328292846679688,
      "activations/layer11_attention_weight_min": -27.655668258666992,
      "activations/layer12_attention_weight_max": 21.750293731689453,
      "activations/layer12_attention_weight_min": -28.331928253173828,
      "activations/layer13_attention_weight_max": 46.54399108886719,
      "activations/layer13_attention_weight_min": -36.49363327026367,
      "activations/layer14_attention_weight_max": 39.6097526550293,
      "activations/layer14_attention_weight_min": -30.992250442504883,
      "activations/layer15_attention_weight_max": 38.056640625,
      "activations/layer15_attention_weight_min": -29.100374221801758,
      "activations/layer16_attention_weight_max": 33.897491455078125,
      "activations/layer16_attention_weight_min": -26.060115814208984,
      "activations/layer17_attention_weight_max": 59.56635284423828,
      "activations/layer17_attention_weight_min": -41.595741271972656,
      "activations/layer18_attention_weight_max": 48.137603759765625,
      "activations/layer18_attention_weight_min": -34.31189727783203,
      "activations/layer19_attention_weight_max": 22.692142486572266,
      "activations/layer19_attention_weight_min": -19.756017684936523,
      "activations/layer1_attention_weight_max": 19.600664138793945,
      "activations/layer1_attention_weight_min": -15.479351043701172,
      "activations/layer20_attention_weight_max": 24.06157875061035,
      "activations/layer20_attention_weight_min": -19.034629821777344,
      "activations/layer21_attention_weight_max": 43.297821044921875,
      "activations/layer21_attention_weight_min": -29.220169067382812,
      "activations/layer22_attention_weight_max": 33.91919708251953,
      "activations/layer22_attention_weight_min": -25.108362197875977,
      "activations/layer23_attention_weight_max": 33.156349182128906,
      "activations/layer23_attention_weight_min": -21.171506881713867,
      "activations/layer2_attention_weight_max": 30.971792221069336,
      "activations/layer2_attention_weight_min": -29.28046417236328,
      "activations/layer3_attention_weight_max": 85.11741638183594,
      "activations/layer3_attention_weight_min": -85.25078582763672,
      "activations/layer4_attention_weight_max": 106.17048645019531,
      "activations/layer4_attention_weight_min": -100.14518737792969,
      "activations/layer5_attention_weight_max": 98.97100830078125,
      "activations/layer5_attention_weight_min": -90.71114349365234,
      "activations/layer6_attention_weight_max": 56.577396392822266,
      "activations/layer6_attention_weight_min": -52.090431213378906,
      "activations/layer7_attention_weight_max": 71.96640014648438,
      "activations/layer7_attention_weight_min": -69.38601684570312,
      "activations/layer8_attention_weight_max": 42.78738021850586,
      "activations/layer8_attention_weight_min": -44.010772705078125,
      "activations/layer9_attention_weight_max": 50.891239166259766,
      "activations/layer9_attention_weight_min": -48.99943923950195,
      "epoch": 5.72,
      "learning_rate": 0.00011426022727272727,
      "loss": 2.83,
      "step": 98450
    },
    {
      "activations/layer0_attention_weight_max": 14.180137634277344,
      "activations/layer0_attention_weight_min": -15.768157958984375,
      "activations/layer10_attention_weight_max": 37.575408935546875,
      "activations/layer10_attention_weight_min": -33.603614807128906,
      "activations/layer11_attention_weight_max": 28.815961837768555,
      "activations/layer11_attention_weight_min": -29.60916519165039,
      "activations/layer12_attention_weight_max": 23.802263259887695,
      "activations/layer12_attention_weight_min": -26.00969696044922,
      "activations/layer13_attention_weight_max": 49.59040069580078,
      "activations/layer13_attention_weight_min": -35.7540283203125,
      "activations/layer14_attention_weight_max": 39.3148307800293,
      "activations/layer14_attention_weight_min": -28.299945831298828,
      "activations/layer15_attention_weight_max": 40.44292068481445,
      "activations/layer15_attention_weight_min": -30.301849365234375,
      "activations/layer16_attention_weight_max": 32.891258239746094,
      "activations/layer16_attention_weight_min": -27.33273696899414,
      "activations/layer17_attention_weight_max": 54.058982849121094,
      "activations/layer17_attention_weight_min": -41.40564727783203,
      "activations/layer18_attention_weight_max": 44.658084869384766,
      "activations/layer18_attention_weight_min": -33.9588737487793,
      "activations/layer19_attention_weight_max": 22.52619743347168,
      "activations/layer19_attention_weight_min": -21.043399810791016,
      "activations/layer1_attention_weight_max": 17.713529586791992,
      "activations/layer1_attention_weight_min": -14.533247947692871,
      "activations/layer20_attention_weight_max": 21.30539894104004,
      "activations/layer20_attention_weight_min": -19.79718589782715,
      "activations/layer21_attention_weight_max": 36.61593246459961,
      "activations/layer21_attention_weight_min": -25.75871467590332,
      "activations/layer22_attention_weight_max": 31.95278549194336,
      "activations/layer22_attention_weight_min": -23.786109924316406,
      "activations/layer23_attention_weight_max": 27.881427764892578,
      "activations/layer23_attention_weight_min": -20.231117248535156,
      "activations/layer2_attention_weight_max": 30.39224624633789,
      "activations/layer2_attention_weight_min": -27.910423278808594,
      "activations/layer3_attention_weight_max": 82.27361297607422,
      "activations/layer3_attention_weight_min": -88.51702117919922,
      "activations/layer4_attention_weight_max": 96.3475112915039,
      "activations/layer4_attention_weight_min": -96.59197235107422,
      "activations/layer5_attention_weight_max": 94.43991088867188,
      "activations/layer5_attention_weight_min": -85.56410217285156,
      "activations/layer6_attention_weight_max": 55.40916442871094,
      "activations/layer6_attention_weight_min": -50.50405502319336,
      "activations/layer7_attention_weight_max": 68.21528625488281,
      "activations/layer7_attention_weight_min": -66.78868865966797,
      "activations/layer8_attention_weight_max": 43.9966926574707,
      "activations/layer8_attention_weight_min": -41.779605865478516,
      "activations/layer9_attention_weight_max": 51.125457763671875,
      "activations/layer9_attention_weight_min": -48.808143615722656,
      "epoch": 5.72,
      "learning_rate": 0.00011424128787878786,
      "loss": 2.8307,
      "step": 98500
    },
    {
      "activations/layer0_attention_weight_max": 14.577958106994629,
      "activations/layer0_attention_weight_min": -15.07167911529541,
      "activations/layer10_attention_weight_max": 37.94310760498047,
      "activations/layer10_attention_weight_min": -36.372344970703125,
      "activations/layer11_attention_weight_max": 30.37335968017578,
      "activations/layer11_attention_weight_min": -28.20966911315918,
      "activations/layer12_attention_weight_max": 20.849441528320312,
      "activations/layer12_attention_weight_min": -23.85608673095703,
      "activations/layer13_attention_weight_max": 44.67984390258789,
      "activations/layer13_attention_weight_min": -34.030311584472656,
      "activations/layer14_attention_weight_max": 39.86335754394531,
      "activations/layer14_attention_weight_min": -30.51862907409668,
      "activations/layer15_attention_weight_max": 36.87501525878906,
      "activations/layer15_attention_weight_min": -30.849679946899414,
      "activations/layer16_attention_weight_max": 30.612829208374023,
      "activations/layer16_attention_weight_min": -27.036296844482422,
      "activations/layer17_attention_weight_max": 58.55894088745117,
      "activations/layer17_attention_weight_min": -40.77772903442383,
      "activations/layer18_attention_weight_max": 47.39537811279297,
      "activations/layer18_attention_weight_min": -35.462913513183594,
      "activations/layer19_attention_weight_max": 22.472055435180664,
      "activations/layer19_attention_weight_min": -23.5532169342041,
      "activations/layer1_attention_weight_max": 17.146610260009766,
      "activations/layer1_attention_weight_min": -14.707208633422852,
      "activations/layer20_attention_weight_max": 19.50810432434082,
      "activations/layer20_attention_weight_min": -19.693843841552734,
      "activations/layer21_attention_weight_max": 37.093570709228516,
      "activations/layer21_attention_weight_min": -25.05208396911621,
      "activations/layer22_attention_weight_max": 32.58623504638672,
      "activations/layer22_attention_weight_min": -23.354251861572266,
      "activations/layer23_attention_weight_max": 29.505290985107422,
      "activations/layer23_attention_weight_min": -19.380409240722656,
      "activations/layer2_attention_weight_max": 30.494285583496094,
      "activations/layer2_attention_weight_min": -29.542638778686523,
      "activations/layer3_attention_weight_max": 78.45821380615234,
      "activations/layer3_attention_weight_min": -81.1000747680664,
      "activations/layer4_attention_weight_max": 101.20917510986328,
      "activations/layer4_attention_weight_min": -88.1871566772461,
      "activations/layer5_attention_weight_max": 100.51167297363281,
      "activations/layer5_attention_weight_min": -85.36266326904297,
      "activations/layer6_attention_weight_max": 56.27824401855469,
      "activations/layer6_attention_weight_min": -54.03934097290039,
      "activations/layer7_attention_weight_max": 72.59514617919922,
      "activations/layer7_attention_weight_min": -69.16522216796875,
      "activations/layer8_attention_weight_max": 42.58213424682617,
      "activations/layer8_attention_weight_min": -44.65189743041992,
      "activations/layer9_attention_weight_max": 48.48524856567383,
      "activations/layer9_attention_weight_min": -45.74081039428711,
      "epoch": 5.73,
      "learning_rate": 0.00011422234848484847,
      "loss": 2.8361,
      "step": 98550
    },
    {
      "activations/layer0_attention_weight_max": 14.113713264465332,
      "activations/layer0_attention_weight_min": -15.293733596801758,
      "activations/layer10_attention_weight_max": 39.11626052856445,
      "activations/layer10_attention_weight_min": -34.46559524536133,
      "activations/layer11_attention_weight_max": 33.60340118408203,
      "activations/layer11_attention_weight_min": -27.723541259765625,
      "activations/layer12_attention_weight_max": 23.241281509399414,
      "activations/layer12_attention_weight_min": -25.47991943359375,
      "activations/layer13_attention_weight_max": 49.03681945800781,
      "activations/layer13_attention_weight_min": -32.67671203613281,
      "activations/layer14_attention_weight_max": 72.50289154052734,
      "activations/layer14_attention_weight_min": -39.156211853027344,
      "activations/layer15_attention_weight_max": 56.04643249511719,
      "activations/layer15_attention_weight_min": -35.90061950683594,
      "activations/layer16_attention_weight_max": 36.31877517700195,
      "activations/layer16_attention_weight_min": -26.348405838012695,
      "activations/layer17_attention_weight_max": 57.54745101928711,
      "activations/layer17_attention_weight_min": -41.740394592285156,
      "activations/layer18_attention_weight_max": 48.191368103027344,
      "activations/layer18_attention_weight_min": -34.07570266723633,
      "activations/layer19_attention_weight_max": 26.325544357299805,
      "activations/layer19_attention_weight_min": -20.002267837524414,
      "activations/layer1_attention_weight_max": 17.91616439819336,
      "activations/layer1_attention_weight_min": -15.044564247131348,
      "activations/layer20_attention_weight_max": 27.004528045654297,
      "activations/layer20_attention_weight_min": -22.127538681030273,
      "activations/layer21_attention_weight_max": 43.129825592041016,
      "activations/layer21_attention_weight_min": -26.365880966186523,
      "activations/layer22_attention_weight_max": 35.33774948120117,
      "activations/layer22_attention_weight_min": -22.70442008972168,
      "activations/layer23_attention_weight_max": 33.6820182800293,
      "activations/layer23_attention_weight_min": -18.87246322631836,
      "activations/layer2_attention_weight_max": 32.35191345214844,
      "activations/layer2_attention_weight_min": -30.048315048217773,
      "activations/layer3_attention_weight_max": 86.60942077636719,
      "activations/layer3_attention_weight_min": -85.14143371582031,
      "activations/layer4_attention_weight_max": 107.55805969238281,
      "activations/layer4_attention_weight_min": -97.2918472290039,
      "activations/layer5_attention_weight_max": 99.6007308959961,
      "activations/layer5_attention_weight_min": -89.45011901855469,
      "activations/layer6_attention_weight_max": 58.21040725708008,
      "activations/layer6_attention_weight_min": -51.241336822509766,
      "activations/layer7_attention_weight_max": 77.83120727539062,
      "activations/layer7_attention_weight_min": -67.99076080322266,
      "activations/layer8_attention_weight_max": 46.069698333740234,
      "activations/layer8_attention_weight_min": -44.743438720703125,
      "activations/layer9_attention_weight_max": 63.23858642578125,
      "activations/layer9_attention_weight_min": -52.439693450927734,
      "epoch": 5.73,
      "learning_rate": 0.00011420340909090909,
      "loss": 2.8471,
      "step": 98600
    },
    {
      "activations/layer0_attention_weight_max": 14.456889152526855,
      "activations/layer0_attention_weight_min": -14.834515571594238,
      "activations/layer10_attention_weight_max": 41.18699645996094,
      "activations/layer10_attention_weight_min": -35.69965362548828,
      "activations/layer11_attention_weight_max": 32.04326629638672,
      "activations/layer11_attention_weight_min": -29.175716400146484,
      "activations/layer12_attention_weight_max": 25.6269474029541,
      "activations/layer12_attention_weight_min": -22.4566707611084,
      "activations/layer13_attention_weight_max": 53.9763298034668,
      "activations/layer13_attention_weight_min": -41.78107452392578,
      "activations/layer14_attention_weight_max": 45.834739685058594,
      "activations/layer14_attention_weight_min": -33.33607864379883,
      "activations/layer15_attention_weight_max": 39.63621139526367,
      "activations/layer15_attention_weight_min": -30.986976623535156,
      "activations/layer16_attention_weight_max": 31.39324951171875,
      "activations/layer16_attention_weight_min": -28.77169418334961,
      "activations/layer17_attention_weight_max": 55.58500671386719,
      "activations/layer17_attention_weight_min": -41.39628219604492,
      "activations/layer18_attention_weight_max": 50.14229202270508,
      "activations/layer18_attention_weight_min": -37.45500564575195,
      "activations/layer19_attention_weight_max": 22.87345314025879,
      "activations/layer19_attention_weight_min": -18.284391403198242,
      "activations/layer1_attention_weight_max": 16.491313934326172,
      "activations/layer1_attention_weight_min": -14.314413070678711,
      "activations/layer20_attention_weight_max": 21.94225311279297,
      "activations/layer20_attention_weight_min": -24.671003341674805,
      "activations/layer21_attention_weight_max": 40.07467269897461,
      "activations/layer21_attention_weight_min": -28.51388168334961,
      "activations/layer22_attention_weight_max": 34.7126350402832,
      "activations/layer22_attention_weight_min": -25.17431640625,
      "activations/layer23_attention_weight_max": 31.303733825683594,
      "activations/layer23_attention_weight_min": -18.62212562561035,
      "activations/layer2_attention_weight_max": 30.937339782714844,
      "activations/layer2_attention_weight_min": -29.28542137145996,
      "activations/layer3_attention_weight_max": 82.54586029052734,
      "activations/layer3_attention_weight_min": -88.9387435913086,
      "activations/layer4_attention_weight_max": 105.97234344482422,
      "activations/layer4_attention_weight_min": -97.39466094970703,
      "activations/layer5_attention_weight_max": 101.40104675292969,
      "activations/layer5_attention_weight_min": -92.22412109375,
      "activations/layer6_attention_weight_max": 61.55424118041992,
      "activations/layer6_attention_weight_min": -53.62141418457031,
      "activations/layer7_attention_weight_max": 71.5938491821289,
      "activations/layer7_attention_weight_min": -67.5535888671875,
      "activations/layer8_attention_weight_max": 46.137943267822266,
      "activations/layer8_attention_weight_min": -43.878562927246094,
      "activations/layer9_attention_weight_max": 52.486114501953125,
      "activations/layer9_attention_weight_min": -49.34358596801758,
      "epoch": 5.73,
      "learning_rate": 0.00011418446969696968,
      "loss": 2.8391,
      "step": 98650
    },
    {
      "activations/layer0_attention_weight_max": 14.426553726196289,
      "activations/layer0_attention_weight_min": -14.886377334594727,
      "activations/layer10_attention_weight_max": 41.90382385253906,
      "activations/layer10_attention_weight_min": -35.584712982177734,
      "activations/layer11_attention_weight_max": 31.551143646240234,
      "activations/layer11_attention_weight_min": -28.17157745361328,
      "activations/layer12_attention_weight_max": 22.006240844726562,
      "activations/layer12_attention_weight_min": -23.916488647460938,
      "activations/layer13_attention_weight_max": 56.270233154296875,
      "activations/layer13_attention_weight_min": -37.73414611816406,
      "activations/layer14_attention_weight_max": 47.402854919433594,
      "activations/layer14_attention_weight_min": -30.9107666015625,
      "activations/layer15_attention_weight_max": 46.926734924316406,
      "activations/layer15_attention_weight_min": -30.010868072509766,
      "activations/layer16_attention_weight_max": 36.29073715209961,
      "activations/layer16_attention_weight_min": -25.96441078186035,
      "activations/layer17_attention_weight_max": 56.80858612060547,
      "activations/layer17_attention_weight_min": -41.68190002441406,
      "activations/layer18_attention_weight_max": 50.94395446777344,
      "activations/layer18_attention_weight_min": -33.26685333251953,
      "activations/layer19_attention_weight_max": 25.805912017822266,
      "activations/layer19_attention_weight_min": -19.769515991210938,
      "activations/layer1_attention_weight_max": 17.794925689697266,
      "activations/layer1_attention_weight_min": -15.1267728805542,
      "activations/layer20_attention_weight_max": 25.40680503845215,
      "activations/layer20_attention_weight_min": -24.296415328979492,
      "activations/layer21_attention_weight_max": 37.76586151123047,
      "activations/layer21_attention_weight_min": -24.90855598449707,
      "activations/layer22_attention_weight_max": 38.737491607666016,
      "activations/layer22_attention_weight_min": -24.116052627563477,
      "activations/layer23_attention_weight_max": 30.015087127685547,
      "activations/layer23_attention_weight_min": -18.413867950439453,
      "activations/layer2_attention_weight_max": 30.638214111328125,
      "activations/layer2_attention_weight_min": -28.807662963867188,
      "activations/layer3_attention_weight_max": 81.91478729248047,
      "activations/layer3_attention_weight_min": -82.5912857055664,
      "activations/layer4_attention_weight_max": 98.38579559326172,
      "activations/layer4_attention_weight_min": -94.09317779541016,
      "activations/layer5_attention_weight_max": 94.47047424316406,
      "activations/layer5_attention_weight_min": -88.98900604248047,
      "activations/layer6_attention_weight_max": 55.215660095214844,
      "activations/layer6_attention_weight_min": -52.837459564208984,
      "activations/layer7_attention_weight_max": 67.40289306640625,
      "activations/layer7_attention_weight_min": -66.57838439941406,
      "activations/layer8_attention_weight_max": 46.04488754272461,
      "activations/layer8_attention_weight_min": -43.37147903442383,
      "activations/layer9_attention_weight_max": 52.636165618896484,
      "activations/layer9_attention_weight_min": -45.284114837646484,
      "epoch": 5.74,
      "learning_rate": 0.00011416553030303029,
      "loss": 2.8405,
      "step": 98700
    },
    {
      "activations/layer0_attention_weight_max": 15.766355514526367,
      "activations/layer0_attention_weight_min": -14.639100074768066,
      "activations/layer10_attention_weight_max": 40.0268440246582,
      "activations/layer10_attention_weight_min": -36.485595703125,
      "activations/layer11_attention_weight_max": 32.02882766723633,
      "activations/layer11_attention_weight_min": -28.77387046813965,
      "activations/layer12_attention_weight_max": 30.026033401489258,
      "activations/layer12_attention_weight_min": -25.14529037475586,
      "activations/layer13_attention_weight_max": 50.00340270996094,
      "activations/layer13_attention_weight_min": -36.72029495239258,
      "activations/layer14_attention_weight_max": 40.50310516357422,
      "activations/layer14_attention_weight_min": -28.514347076416016,
      "activations/layer15_attention_weight_max": 40.75259780883789,
      "activations/layer15_attention_weight_min": -29.414756774902344,
      "activations/layer16_attention_weight_max": 32.35990905761719,
      "activations/layer16_attention_weight_min": -26.377187728881836,
      "activations/layer17_attention_weight_max": 58.01693344116211,
      "activations/layer17_attention_weight_min": -38.29988479614258,
      "activations/layer18_attention_weight_max": 48.913597106933594,
      "activations/layer18_attention_weight_min": -31.05154037475586,
      "activations/layer19_attention_weight_max": 25.705001831054688,
      "activations/layer19_attention_weight_min": -18.630168914794922,
      "activations/layer1_attention_weight_max": 17.597623825073242,
      "activations/layer1_attention_weight_min": -14.12639331817627,
      "activations/layer20_attention_weight_max": 21.850194931030273,
      "activations/layer20_attention_weight_min": -20.99336051940918,
      "activations/layer21_attention_weight_max": 39.11565017700195,
      "activations/layer21_attention_weight_min": -24.543851852416992,
      "activations/layer22_attention_weight_max": 34.58113479614258,
      "activations/layer22_attention_weight_min": -24.09514808654785,
      "activations/layer23_attention_weight_max": 27.157560348510742,
      "activations/layer23_attention_weight_min": -17.8510799407959,
      "activations/layer2_attention_weight_max": 30.633670806884766,
      "activations/layer2_attention_weight_min": -28.41431427001953,
      "activations/layer3_attention_weight_max": 81.70586395263672,
      "activations/layer3_attention_weight_min": -85.74366760253906,
      "activations/layer4_attention_weight_max": 105.84846496582031,
      "activations/layer4_attention_weight_min": -98.98261260986328,
      "activations/layer5_attention_weight_max": 101.92483520507812,
      "activations/layer5_attention_weight_min": -92.578125,
      "activations/layer6_attention_weight_max": 60.53586959838867,
      "activations/layer6_attention_weight_min": -54.9843635559082,
      "activations/layer7_attention_weight_max": 72.67671203613281,
      "activations/layer7_attention_weight_min": -67.93505096435547,
      "activations/layer8_attention_weight_max": 45.59575653076172,
      "activations/layer8_attention_weight_min": -46.23424530029297,
      "activations/layer9_attention_weight_max": 52.24519729614258,
      "activations/layer9_attention_weight_min": -47.34629440307617,
      "epoch": 5.74,
      "learning_rate": 0.0001141465909090909,
      "loss": 2.8327,
      "step": 98750
    },
    {
      "activations/layer0_attention_weight_max": 16.413101196289062,
      "activations/layer0_attention_weight_min": -14.970754623413086,
      "activations/layer10_attention_weight_max": 41.80940246582031,
      "activations/layer10_attention_weight_min": -36.262245178222656,
      "activations/layer11_attention_weight_max": 37.67497634887695,
      "activations/layer11_attention_weight_min": -27.021583557128906,
      "activations/layer12_attention_weight_max": 21.142080307006836,
      "activations/layer12_attention_weight_min": -24.007747650146484,
      "activations/layer13_attention_weight_max": 42.109004974365234,
      "activations/layer13_attention_weight_min": -32.31686782836914,
      "activations/layer14_attention_weight_max": 59.95337677001953,
      "activations/layer14_attention_weight_min": -35.947364807128906,
      "activations/layer15_attention_weight_max": 42.362022399902344,
      "activations/layer15_attention_weight_min": -29.571115493774414,
      "activations/layer16_attention_weight_max": 30.201488494873047,
      "activations/layer16_attention_weight_min": -27.011219024658203,
      "activations/layer17_attention_weight_max": 55.720645904541016,
      "activations/layer17_attention_weight_min": -37.762386322021484,
      "activations/layer18_attention_weight_max": 45.56064224243164,
      "activations/layer18_attention_weight_min": -33.54338836669922,
      "activations/layer19_attention_weight_max": 20.783527374267578,
      "activations/layer19_attention_weight_min": -21.207632064819336,
      "activations/layer1_attention_weight_max": 16.360097885131836,
      "activations/layer1_attention_weight_min": -15.543135643005371,
      "activations/layer20_attention_weight_max": 21.577373504638672,
      "activations/layer20_attention_weight_min": -20.54073143005371,
      "activations/layer21_attention_weight_max": 34.697288513183594,
      "activations/layer21_attention_weight_min": -24.17198371887207,
      "activations/layer22_attention_weight_max": 29.72908592224121,
      "activations/layer22_attention_weight_min": -23.153013229370117,
      "activations/layer23_attention_weight_max": 26.818714141845703,
      "activations/layer23_attention_weight_min": -19.390560150146484,
      "activations/layer2_attention_weight_max": 32.941307067871094,
      "activations/layer2_attention_weight_min": -30.82889175415039,
      "activations/layer3_attention_weight_max": 84.36676025390625,
      "activations/layer3_attention_weight_min": -85.90532684326172,
      "activations/layer4_attention_weight_max": 106.8203125,
      "activations/layer4_attention_weight_min": -100.70577239990234,
      "activations/layer5_attention_weight_max": 102.8495101928711,
      "activations/layer5_attention_weight_min": -93.586669921875,
      "activations/layer6_attention_weight_max": 59.196903228759766,
      "activations/layer6_attention_weight_min": -53.053565979003906,
      "activations/layer7_attention_weight_max": 79.6198501586914,
      "activations/layer7_attention_weight_min": -67.46598052978516,
      "activations/layer8_attention_weight_max": 48.81600570678711,
      "activations/layer8_attention_weight_min": -42.24256896972656,
      "activations/layer9_attention_weight_max": 68.94210052490234,
      "activations/layer9_attention_weight_min": -51.174354553222656,
      "epoch": 5.74,
      "learning_rate": 0.0001141280303030303,
      "loss": 2.8418,
      "step": 98800
    },
    {
      "activations/layer0_attention_weight_max": 14.882410049438477,
      "activations/layer0_attention_weight_min": -14.781877517700195,
      "activations/layer10_attention_weight_max": 38.648170471191406,
      "activations/layer10_attention_weight_min": -36.26423263549805,
      "activations/layer11_attention_weight_max": 28.716510772705078,
      "activations/layer11_attention_weight_min": -28.225513458251953,
      "activations/layer12_attention_weight_max": 23.608123779296875,
      "activations/layer12_attention_weight_min": -25.04667854309082,
      "activations/layer13_attention_weight_max": 51.00090789794922,
      "activations/layer13_attention_weight_min": -33.9251594543457,
      "activations/layer14_attention_weight_max": 49.06356430053711,
      "activations/layer14_attention_weight_min": -29.287830352783203,
      "activations/layer15_attention_weight_max": 42.37350845336914,
      "activations/layer15_attention_weight_min": -31.38304328918457,
      "activations/layer16_attention_weight_max": 34.954280853271484,
      "activations/layer16_attention_weight_min": -27.493000030517578,
      "activations/layer17_attention_weight_max": 52.780967712402344,
      "activations/layer17_attention_weight_min": -40.12389373779297,
      "activations/layer18_attention_weight_max": 51.37109375,
      "activations/layer18_attention_weight_min": -33.110008239746094,
      "activations/layer19_attention_weight_max": 25.668018341064453,
      "activations/layer19_attention_weight_min": -21.474842071533203,
      "activations/layer1_attention_weight_max": 17.703628540039062,
      "activations/layer1_attention_weight_min": -18.4248104095459,
      "activations/layer20_attention_weight_max": 24.073802947998047,
      "activations/layer20_attention_weight_min": -20.67331314086914,
      "activations/layer21_attention_weight_max": 43.0565299987793,
      "activations/layer21_attention_weight_min": -25.20511817932129,
      "activations/layer22_attention_weight_max": 36.201297760009766,
      "activations/layer22_attention_weight_min": -25.374862670898438,
      "activations/layer23_attention_weight_max": 30.237401962280273,
      "activations/layer23_attention_weight_min": -22.103302001953125,
      "activations/layer2_attention_weight_max": 30.430166244506836,
      "activations/layer2_attention_weight_min": -28.509098052978516,
      "activations/layer3_attention_weight_max": 79.8747787475586,
      "activations/layer3_attention_weight_min": -84.62538146972656,
      "activations/layer4_attention_weight_max": 96.99922180175781,
      "activations/layer4_attention_weight_min": -95.74032592773438,
      "activations/layer5_attention_weight_max": 92.53267669677734,
      "activations/layer5_attention_weight_min": -85.14987182617188,
      "activations/layer6_attention_weight_max": 54.68151092529297,
      "activations/layer6_attention_weight_min": -50.948726654052734,
      "activations/layer7_attention_weight_max": 72.37715148925781,
      "activations/layer7_attention_weight_min": -68.35047912597656,
      "activations/layer8_attention_weight_max": 42.84214401245117,
      "activations/layer8_attention_weight_min": -41.22787857055664,
      "activations/layer9_attention_weight_max": 54.716712951660156,
      "activations/layer9_attention_weight_min": -49.867431640625,
      "epoch": 5.74,
      "learning_rate": 0.0001141090909090909,
      "loss": 2.8489,
      "step": 98850
    },
    {
      "activations/layer0_attention_weight_max": 13.727203369140625,
      "activations/layer0_attention_weight_min": -14.802972793579102,
      "activations/layer10_attention_weight_max": 37.77735900878906,
      "activations/layer10_attention_weight_min": -35.26353073120117,
      "activations/layer11_attention_weight_max": 31.30661392211914,
      "activations/layer11_attention_weight_min": -28.598957061767578,
      "activations/layer12_attention_weight_max": 22.066211700439453,
      "activations/layer12_attention_weight_min": -21.91023063659668,
      "activations/layer13_attention_weight_max": 51.17466354370117,
      "activations/layer13_attention_weight_min": -36.59993362426758,
      "activations/layer14_attention_weight_max": 43.86539840698242,
      "activations/layer14_attention_weight_min": -30.15251350402832,
      "activations/layer15_attention_weight_max": 42.14193344116211,
      "activations/layer15_attention_weight_min": -31.384496688842773,
      "activations/layer16_attention_weight_max": 38.025699615478516,
      "activations/layer16_attention_weight_min": -26.275104522705078,
      "activations/layer17_attention_weight_max": 59.7308235168457,
      "activations/layer17_attention_weight_min": -39.65679168701172,
      "activations/layer18_attention_weight_max": 55.31293869018555,
      "activations/layer18_attention_weight_min": -35.690467834472656,
      "activations/layer19_attention_weight_max": 25.229671478271484,
      "activations/layer19_attention_weight_min": -20.548233032226562,
      "activations/layer1_attention_weight_max": 16.878555297851562,
      "activations/layer1_attention_weight_min": -13.463623046875,
      "activations/layer20_attention_weight_max": 24.36100959777832,
      "activations/layer20_attention_weight_min": -23.77237319946289,
      "activations/layer21_attention_weight_max": 43.954559326171875,
      "activations/layer21_attention_weight_min": -23.537052154541016,
      "activations/layer22_attention_weight_max": 34.961219787597656,
      "activations/layer22_attention_weight_min": -22.848291397094727,
      "activations/layer23_attention_weight_max": 32.241031646728516,
      "activations/layer23_attention_weight_min": -17.776350021362305,
      "activations/layer2_attention_weight_max": 30.140769958496094,
      "activations/layer2_attention_weight_min": -30.610191345214844,
      "activations/layer3_attention_weight_max": 84.81515502929688,
      "activations/layer3_attention_weight_min": -89.00826263427734,
      "activations/layer4_attention_weight_max": 102.69512176513672,
      "activations/layer4_attention_weight_min": -96.8968734741211,
      "activations/layer5_attention_weight_max": 95.11699676513672,
      "activations/layer5_attention_weight_min": -91.0657958984375,
      "activations/layer6_attention_weight_max": 57.34815979003906,
      "activations/layer6_attention_weight_min": -55.83241653442383,
      "activations/layer7_attention_weight_max": 72.005615234375,
      "activations/layer7_attention_weight_min": -69.38977813720703,
      "activations/layer8_attention_weight_max": 44.12822341918945,
      "activations/layer8_attention_weight_min": -43.79639434814453,
      "activations/layer9_attention_weight_max": 55.8975830078125,
      "activations/layer9_attention_weight_min": -51.15519332885742,
      "epoch": 5.75,
      "learning_rate": 0.00011409015151515151,
      "loss": 2.8281,
      "step": 98900
    },
    {
      "activations/layer0_attention_weight_max": 14.29605484008789,
      "activations/layer0_attention_weight_min": -14.180720329284668,
      "activations/layer10_attention_weight_max": 39.66404342651367,
      "activations/layer10_attention_weight_min": -34.79852294921875,
      "activations/layer11_attention_weight_max": 33.006805419921875,
      "activations/layer11_attention_weight_min": -27.608654022216797,
      "activations/layer12_attention_weight_max": 21.778060913085938,
      "activations/layer12_attention_weight_min": -24.781848907470703,
      "activations/layer13_attention_weight_max": 44.1953239440918,
      "activations/layer13_attention_weight_min": -32.32078552246094,
      "activations/layer14_attention_weight_max": 46.02724838256836,
      "activations/layer14_attention_weight_min": -31.069263458251953,
      "activations/layer15_attention_weight_max": 38.0123291015625,
      "activations/layer15_attention_weight_min": -29.73380470275879,
      "activations/layer16_attention_weight_max": 36.1933708190918,
      "activations/layer16_attention_weight_min": -27.26830291748047,
      "activations/layer17_attention_weight_max": 55.70484161376953,
      "activations/layer17_attention_weight_min": -39.42998123168945,
      "activations/layer18_attention_weight_max": 52.51384353637695,
      "activations/layer18_attention_weight_min": -33.77600860595703,
      "activations/layer19_attention_weight_max": 22.703046798706055,
      "activations/layer19_attention_weight_min": -18.77370262145996,
      "activations/layer1_attention_weight_max": 17.43430519104004,
      "activations/layer1_attention_weight_min": -16.664304733276367,
      "activations/layer20_attention_weight_max": 22.772287368774414,
      "activations/layer20_attention_weight_min": -22.63021469116211,
      "activations/layer21_attention_weight_max": 46.07960510253906,
      "activations/layer21_attention_weight_min": -25.003881454467773,
      "activations/layer22_attention_weight_max": 36.89582443237305,
      "activations/layer22_attention_weight_min": -23.351213455200195,
      "activations/layer23_attention_weight_max": 34.48648452758789,
      "activations/layer23_attention_weight_min": -19.26680564880371,
      "activations/layer2_attention_weight_max": 30.680042266845703,
      "activations/layer2_attention_weight_min": -30.17015838623047,
      "activations/layer3_attention_weight_max": 77.56653594970703,
      "activations/layer3_attention_weight_min": -83.00146484375,
      "activations/layer4_attention_weight_max": 96.8894271850586,
      "activations/layer4_attention_weight_min": -94.13966369628906,
      "activations/layer5_attention_weight_max": 95.48843383789062,
      "activations/layer5_attention_weight_min": -88.51058197021484,
      "activations/layer6_attention_weight_max": 55.47455978393555,
      "activations/layer6_attention_weight_min": -49.70624542236328,
      "activations/layer7_attention_weight_max": 74.65355682373047,
      "activations/layer7_attention_weight_min": -70.61087799072266,
      "activations/layer8_attention_weight_max": 45.78940963745117,
      "activations/layer8_attention_weight_min": -43.37672424316406,
      "activations/layer9_attention_weight_max": 63.99941635131836,
      "activations/layer9_attention_weight_min": -58.95463180541992,
      "epoch": 5.75,
      "learning_rate": 0.00011407121212121211,
      "loss": 2.8234,
      "step": 98950
    },
    {
      "activations/layer0_attention_weight_max": 14.781513214111328,
      "activations/layer0_attention_weight_min": -14.744834899902344,
      "activations/layer10_attention_weight_max": 41.18648147583008,
      "activations/layer10_attention_weight_min": -36.27201843261719,
      "activations/layer11_attention_weight_max": 31.558517456054688,
      "activations/layer11_attention_weight_min": -30.152700424194336,
      "activations/layer12_attention_weight_max": 24.4127254486084,
      "activations/layer12_attention_weight_min": -22.012229919433594,
      "activations/layer13_attention_weight_max": 54.99885559082031,
      "activations/layer13_attention_weight_min": -35.34771728515625,
      "activations/layer14_attention_weight_max": 58.583457946777344,
      "activations/layer14_attention_weight_min": -33.15827178955078,
      "activations/layer15_attention_weight_max": 45.4567756652832,
      "activations/layer15_attention_weight_min": -31.302688598632812,
      "activations/layer16_attention_weight_max": 41.84346008300781,
      "activations/layer16_attention_weight_min": -27.983415603637695,
      "activations/layer17_attention_weight_max": 70.6166763305664,
      "activations/layer17_attention_weight_min": -42.670494079589844,
      "activations/layer18_attention_weight_max": 63.824039459228516,
      "activations/layer18_attention_weight_min": -37.70063400268555,
      "activations/layer19_attention_weight_max": 26.328893661499023,
      "activations/layer19_attention_weight_min": -20.6650390625,
      "activations/layer1_attention_weight_max": 17.34619140625,
      "activations/layer1_attention_weight_min": -15.035799980163574,
      "activations/layer20_attention_weight_max": 30.07866668701172,
      "activations/layer20_attention_weight_min": -21.68576431274414,
      "activations/layer21_attention_weight_max": 50.54999923706055,
      "activations/layer21_attention_weight_min": -26.25031089782715,
      "activations/layer22_attention_weight_max": 43.02985382080078,
      "activations/layer22_attention_weight_min": -25.71821403503418,
      "activations/layer23_attention_weight_max": 41.49702835083008,
      "activations/layer23_attention_weight_min": -19.0516300201416,
      "activations/layer2_attention_weight_max": 32.99549102783203,
      "activations/layer2_attention_weight_min": -29.473297119140625,
      "activations/layer3_attention_weight_max": 85.77195739746094,
      "activations/layer3_attention_weight_min": -83.94328308105469,
      "activations/layer4_attention_weight_max": 102.88896942138672,
      "activations/layer4_attention_weight_min": -97.16670227050781,
      "activations/layer5_attention_weight_max": 101.06660461425781,
      "activations/layer5_attention_weight_min": -93.8100814819336,
      "activations/layer6_attention_weight_max": 59.63089370727539,
      "activations/layer6_attention_weight_min": -52.431663513183594,
      "activations/layer7_attention_weight_max": 75.1313247680664,
      "activations/layer7_attention_weight_min": -73.0431137084961,
      "activations/layer8_attention_weight_max": 44.8847541809082,
      "activations/layer8_attention_weight_min": -45.97235870361328,
      "activations/layer9_attention_weight_max": 51.97857666015625,
      "activations/layer9_attention_weight_min": -51.557044982910156,
      "epoch": 5.75,
      "learning_rate": 0.00011405227272727272,
      "loss": 2.8449,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.5948,
      "eval_samples_per_second": 499.603,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.5948,
      "eval_openwebtext_samples_per_second": 499.603,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_wikitext_loss": 3.021484375,
      "eval_wikitext_ppl": 20.521731031400577,
      "eval_wikitext_runtime": 2.0707,
      "eval_wikitext_samples_per_second": 220.218,
      "step": 99000
    },
    {
      "epoch": 5.75,
      "eval_lambada_loss": 2.703125,
      "eval_lambada_ppl": 14.926303617194517,
      "eval_lambada_runtime": 9.597,
      "eval_lambada_samples_per_second": 507.348,
      "step": 99000
    },
    {
      "activations/layer0_attention_weight_max": 15.013949394226074,
      "activations/layer0_attention_weight_min": -15.02755069732666,
      "activations/layer10_attention_weight_max": 38.93825912475586,
      "activations/layer10_attention_weight_min": -34.70644760131836,
      "activations/layer11_attention_weight_max": 29.302221298217773,
      "activations/layer11_attention_weight_min": -27.488149642944336,
      "activations/layer12_attention_weight_max": 20.844764709472656,
      "activations/layer12_attention_weight_min": -24.309995651245117,
      "activations/layer13_attention_weight_max": 52.60292434692383,
      "activations/layer13_attention_weight_min": -38.76537322998047,
      "activations/layer14_attention_weight_max": 52.06080627441406,
      "activations/layer14_attention_weight_min": -33.73493576049805,
      "activations/layer15_attention_weight_max": 39.91488265991211,
      "activations/layer15_attention_weight_min": -31.10236358642578,
      "activations/layer16_attention_weight_max": 32.850337982177734,
      "activations/layer16_attention_weight_min": -27.44405746459961,
      "activations/layer17_attention_weight_max": 56.77675247192383,
      "activations/layer17_attention_weight_min": -39.78672409057617,
      "activations/layer18_attention_weight_max": 49.60294723510742,
      "activations/layer18_attention_weight_min": -33.73781967163086,
      "activations/layer19_attention_weight_max": 20.16754722595215,
      "activations/layer19_attention_weight_min": -18.417940139770508,
      "activations/layer1_attention_weight_max": 16.995420455932617,
      "activations/layer1_attention_weight_min": -15.77675724029541,
      "activations/layer20_attention_weight_max": 20.194496154785156,
      "activations/layer20_attention_weight_min": -22.17620086669922,
      "activations/layer21_attention_weight_max": 34.53416442871094,
      "activations/layer21_attention_weight_min": -26.98630142211914,
      "activations/layer22_attention_weight_max": 31.76705551147461,
      "activations/layer22_attention_weight_min": -23.0076847076416,
      "activations/layer23_attention_weight_max": 29.129737854003906,
      "activations/layer23_attention_weight_min": -18.18134307861328,
      "activations/layer2_attention_weight_max": 30.533889770507812,
      "activations/layer2_attention_weight_min": -29.28726577758789,
      "activations/layer3_attention_weight_max": 82.38301849365234,
      "activations/layer3_attention_weight_min": -83.78334045410156,
      "activations/layer4_attention_weight_max": 102.7961654663086,
      "activations/layer4_attention_weight_min": -94.04097747802734,
      "activations/layer5_attention_weight_max": 98.91387939453125,
      "activations/layer5_attention_weight_min": -87.44642639160156,
      "activations/layer6_attention_weight_max": 59.693824768066406,
      "activations/layer6_attention_weight_min": -51.959293365478516,
      "activations/layer7_attention_weight_max": 71.82890319824219,
      "activations/layer7_attention_weight_min": -70.5033187866211,
      "activations/layer8_attention_weight_max": 46.392311096191406,
      "activations/layer8_attention_weight_min": -42.6612548828125,
      "activations/layer9_attention_weight_max": 52.659034729003906,
      "activations/layer9_attention_weight_min": -47.49597930908203,
      "epoch": 5.76,
      "learning_rate": 0.00011403333333333333,
      "loss": 2.8447,
      "step": 99050
    },
    {
      "activations/layer0_attention_weight_max": 14.293992042541504,
      "activations/layer0_attention_weight_min": -14.55928897857666,
      "activations/layer10_attention_weight_max": 37.94756317138672,
      "activations/layer10_attention_weight_min": -35.41359329223633,
      "activations/layer11_attention_weight_max": 30.82807731628418,
      "activations/layer11_attention_weight_min": -28.649831771850586,
      "activations/layer12_attention_weight_max": 22.17119598388672,
      "activations/layer12_attention_weight_min": -25.030595779418945,
      "activations/layer13_attention_weight_max": 55.194435119628906,
      "activations/layer13_attention_weight_min": -36.486961364746094,
      "activations/layer14_attention_weight_max": 46.45952224731445,
      "activations/layer14_attention_weight_min": -30.247583389282227,
      "activations/layer15_attention_weight_max": 43.58157730102539,
      "activations/layer15_attention_weight_min": -29.37527847290039,
      "activations/layer16_attention_weight_max": 35.12976837158203,
      "activations/layer16_attention_weight_min": -26.612348556518555,
      "activations/layer17_attention_weight_max": 58.52626419067383,
      "activations/layer17_attention_weight_min": -39.38930130004883,
      "activations/layer18_attention_weight_max": 50.67139434814453,
      "activations/layer18_attention_weight_min": -33.62453842163086,
      "activations/layer19_attention_weight_max": 23.107769012451172,
      "activations/layer19_attention_weight_min": -21.476537704467773,
      "activations/layer1_attention_weight_max": 18.378793716430664,
      "activations/layer1_attention_weight_min": -14.350187301635742,
      "activations/layer20_attention_weight_max": 23.899900436401367,
      "activations/layer20_attention_weight_min": -23.11764907836914,
      "activations/layer21_attention_weight_max": 41.80537796020508,
      "activations/layer21_attention_weight_min": -28.07587242126465,
      "activations/layer22_attention_weight_max": 36.525604248046875,
      "activations/layer22_attention_weight_min": -25.63050651550293,
      "activations/layer23_attention_weight_max": 32.39241027832031,
      "activations/layer23_attention_weight_min": -19.910736083984375,
      "activations/layer2_attention_weight_max": 31.549976348876953,
      "activations/layer2_attention_weight_min": -28.21112632751465,
      "activations/layer3_attention_weight_max": 79.32152557373047,
      "activations/layer3_attention_weight_min": -80.39141845703125,
      "activations/layer4_attention_weight_max": 98.36482238769531,
      "activations/layer4_attention_weight_min": -91.53443908691406,
      "activations/layer5_attention_weight_max": 96.60403442382812,
      "activations/layer5_attention_weight_min": -86.5599594116211,
      "activations/layer6_attention_weight_max": 58.50656509399414,
      "activations/layer6_attention_weight_min": -51.57146453857422,
      "activations/layer7_attention_weight_max": 78.37191009521484,
      "activations/layer7_attention_weight_min": -68.67005157470703,
      "activations/layer8_attention_weight_max": 48.20742416381836,
      "activations/layer8_attention_weight_min": -45.90836715698242,
      "activations/layer9_attention_weight_max": 55.69287109375,
      "activations/layer9_attention_weight_min": -51.14306640625,
      "epoch": 5.76,
      "learning_rate": 0.00011401439393939392,
      "loss": 2.8448,
      "step": 99100
    },
    {
      "activations/layer0_attention_weight_max": 15.195393562316895,
      "activations/layer0_attention_weight_min": -15.100912094116211,
      "activations/layer10_attention_weight_max": 37.43167495727539,
      "activations/layer10_attention_weight_min": -34.19661331176758,
      "activations/layer11_attention_weight_max": 30.471519470214844,
      "activations/layer11_attention_weight_min": -28.79629898071289,
      "activations/layer12_attention_weight_max": 20.60201644897461,
      "activations/layer12_attention_weight_min": -24.2679386138916,
      "activations/layer13_attention_weight_max": 47.9996452331543,
      "activations/layer13_attention_weight_min": -37.511383056640625,
      "activations/layer14_attention_weight_max": 45.82658386230469,
      "activations/layer14_attention_weight_min": -31.382007598876953,
      "activations/layer15_attention_weight_max": 40.33156204223633,
      "activations/layer15_attention_weight_min": -28.4685115814209,
      "activations/layer16_attention_weight_max": 34.65814208984375,
      "activations/layer16_attention_weight_min": -26.05607795715332,
      "activations/layer17_attention_weight_max": 53.8753776550293,
      "activations/layer17_attention_weight_min": -39.07950973510742,
      "activations/layer18_attention_weight_max": 46.54862594604492,
      "activations/layer18_attention_weight_min": -34.25752258300781,
      "activations/layer19_attention_weight_max": 23.292827606201172,
      "activations/layer19_attention_weight_min": -25.301237106323242,
      "activations/layer1_attention_weight_max": 18.619365692138672,
      "activations/layer1_attention_weight_min": -16.593563079833984,
      "activations/layer20_attention_weight_max": 20.943788528442383,
      "activations/layer20_attention_weight_min": -20.215190887451172,
      "activations/layer21_attention_weight_max": 40.4180793762207,
      "activations/layer21_attention_weight_min": -24.59646987915039,
      "activations/layer22_attention_weight_max": 33.79047775268555,
      "activations/layer22_attention_weight_min": -25.938093185424805,
      "activations/layer23_attention_weight_max": 28.69483184814453,
      "activations/layer23_attention_weight_min": -20.139060974121094,
      "activations/layer2_attention_weight_max": 30.644622802734375,
      "activations/layer2_attention_weight_min": -29.80203628540039,
      "activations/layer3_attention_weight_max": 82.14491271972656,
      "activations/layer3_attention_weight_min": -86.44986724853516,
      "activations/layer4_attention_weight_max": 103.0176010131836,
      "activations/layer4_attention_weight_min": -95.27574157714844,
      "activations/layer5_attention_weight_max": 93.68025970458984,
      "activations/layer5_attention_weight_min": -88.03591918945312,
      "activations/layer6_attention_weight_max": 57.00154495239258,
      "activations/layer6_attention_weight_min": -52.63846206665039,
      "activations/layer7_attention_weight_max": 68.84552001953125,
      "activations/layer7_attention_weight_min": -65.55916595458984,
      "activations/layer8_attention_weight_max": 43.0140495300293,
      "activations/layer8_attention_weight_min": -44.1107177734375,
      "activations/layer9_attention_weight_max": 46.048343658447266,
      "activations/layer9_attention_weight_min": -48.31447219848633,
      "epoch": 5.76,
      "learning_rate": 0.00011399545454545453,
      "loss": 2.8586,
      "step": 99150
    },
    {
      "activations/layer0_attention_weight_max": 15.269851684570312,
      "activations/layer0_attention_weight_min": -14.748283386230469,
      "activations/layer10_attention_weight_max": 38.505550384521484,
      "activations/layer10_attention_weight_min": -35.031375885009766,
      "activations/layer11_attention_weight_max": 29.81402587890625,
      "activations/layer11_attention_weight_min": -28.137861251831055,
      "activations/layer12_attention_weight_max": 19.625043869018555,
      "activations/layer12_attention_weight_min": -26.453691482543945,
      "activations/layer13_attention_weight_max": 39.46442413330078,
      "activations/layer13_attention_weight_min": -31.532503128051758,
      "activations/layer14_attention_weight_max": 44.01647186279297,
      "activations/layer14_attention_weight_min": -30.9178524017334,
      "activations/layer15_attention_weight_max": 36.756649017333984,
      "activations/layer15_attention_weight_min": -28.342016220092773,
      "activations/layer16_attention_weight_max": 32.37845230102539,
      "activations/layer16_attention_weight_min": -26.388273239135742,
      "activations/layer17_attention_weight_max": 53.828067779541016,
      "activations/layer17_attention_weight_min": -40.38739776611328,
      "activations/layer18_attention_weight_max": 48.574405670166016,
      "activations/layer18_attention_weight_min": -33.71712112426758,
      "activations/layer19_attention_weight_max": 20.988447189331055,
      "activations/layer19_attention_weight_min": -20.634536743164062,
      "activations/layer1_attention_weight_max": 17.327173233032227,
      "activations/layer1_attention_weight_min": -15.45608139038086,
      "activations/layer20_attention_weight_max": 22.81329345703125,
      "activations/layer20_attention_weight_min": -23.199039459228516,
      "activations/layer21_attention_weight_max": 40.693851470947266,
      "activations/layer21_attention_weight_min": -26.41366195678711,
      "activations/layer22_attention_weight_max": 31.49814224243164,
      "activations/layer22_attention_weight_min": -25.152694702148438,
      "activations/layer23_attention_weight_max": 27.512035369873047,
      "activations/layer23_attention_weight_min": -17.855897903442383,
      "activations/layer2_attention_weight_max": 31.951908111572266,
      "activations/layer2_attention_weight_min": -28.034080505371094,
      "activations/layer3_attention_weight_max": 81.86180114746094,
      "activations/layer3_attention_weight_min": -83.93466186523438,
      "activations/layer4_attention_weight_max": 97.4477767944336,
      "activations/layer4_attention_weight_min": -88.27916717529297,
      "activations/layer5_attention_weight_max": 94.70906829833984,
      "activations/layer5_attention_weight_min": -84.59756469726562,
      "activations/layer6_attention_weight_max": 56.35169982910156,
      "activations/layer6_attention_weight_min": -49.22258758544922,
      "activations/layer7_attention_weight_max": 67.93999481201172,
      "activations/layer7_attention_weight_min": -64.80842590332031,
      "activations/layer8_attention_weight_max": 40.60631561279297,
      "activations/layer8_attention_weight_min": -39.10701370239258,
      "activations/layer9_attention_weight_max": 45.39665985107422,
      "activations/layer9_attention_weight_min": -44.75056076049805,
      "epoch": 5.76,
      "learning_rate": 0.00011397651515151515,
      "loss": 2.8329,
      "step": 99200
    },
    {
      "activations/layer0_attention_weight_max": 16.287864685058594,
      "activations/layer0_attention_weight_min": -14.944320678710938,
      "activations/layer10_attention_weight_max": 38.04011535644531,
      "activations/layer10_attention_weight_min": -38.595611572265625,
      "activations/layer11_attention_weight_max": 30.87673568725586,
      "activations/layer11_attention_weight_min": -30.894580841064453,
      "activations/layer12_attention_weight_max": 23.15778160095215,
      "activations/layer12_attention_weight_min": -26.667110443115234,
      "activations/layer13_attention_weight_max": 43.58429718017578,
      "activations/layer13_attention_weight_min": -32.08926773071289,
      "activations/layer14_attention_weight_max": 38.73789596557617,
      "activations/layer14_attention_weight_min": -28.443782806396484,
      "activations/layer15_attention_weight_max": 34.49794387817383,
      "activations/layer15_attention_weight_min": -28.200538635253906,
      "activations/layer16_attention_weight_max": 33.919769287109375,
      "activations/layer16_attention_weight_min": -26.504825592041016,
      "activations/layer17_attention_weight_max": 53.138614654541016,
      "activations/layer17_attention_weight_min": -41.05609893798828,
      "activations/layer18_attention_weight_max": 43.59271240234375,
      "activations/layer18_attention_weight_min": -31.602506637573242,
      "activations/layer19_attention_weight_max": 20.74738311767578,
      "activations/layer19_attention_weight_min": -19.473939895629883,
      "activations/layer1_attention_weight_max": 17.054737091064453,
      "activations/layer1_attention_weight_min": -14.508665084838867,
      "activations/layer20_attention_weight_max": 21.86011505126953,
      "activations/layer20_attention_weight_min": -20.510868072509766,
      "activations/layer21_attention_weight_max": 41.1196403503418,
      "activations/layer21_attention_weight_min": -24.74711036682129,
      "activations/layer22_attention_weight_max": 32.40525436401367,
      "activations/layer22_attention_weight_min": -22.87264633178711,
      "activations/layer23_attention_weight_max": 27.617477416992188,
      "activations/layer23_attention_weight_min": -18.366172790527344,
      "activations/layer2_attention_weight_max": 31.82476806640625,
      "activations/layer2_attention_weight_min": -31.713016510009766,
      "activations/layer3_attention_weight_max": 84.40135955810547,
      "activations/layer3_attention_weight_min": -88.90959167480469,
      "activations/layer4_attention_weight_max": 107.2310791015625,
      "activations/layer4_attention_weight_min": -103.24398040771484,
      "activations/layer5_attention_weight_max": 102.46635437011719,
      "activations/layer5_attention_weight_min": -93.21233367919922,
      "activations/layer6_attention_weight_max": 63.38953399658203,
      "activations/layer6_attention_weight_min": -59.07103729248047,
      "activations/layer7_attention_weight_max": 77.74034118652344,
      "activations/layer7_attention_weight_min": -71.29586029052734,
      "activations/layer8_attention_weight_max": 46.75971984863281,
      "activations/layer8_attention_weight_min": -43.12921905517578,
      "activations/layer9_attention_weight_max": 72.91412353515625,
      "activations/layer9_attention_weight_min": -53.567626953125,
      "epoch": 5.77,
      "learning_rate": 0.00011395757575757574,
      "loss": 2.8404,
      "step": 99250
    },
    {
      "activations/layer0_attention_weight_max": 14.81490421295166,
      "activations/layer0_attention_weight_min": -15.318591117858887,
      "activations/layer10_attention_weight_max": 42.139957427978516,
      "activations/layer10_attention_weight_min": -37.32915115356445,
      "activations/layer11_attention_weight_max": 35.62801742553711,
      "activations/layer11_attention_weight_min": -29.883880615234375,
      "activations/layer12_attention_weight_max": 31.543846130371094,
      "activations/layer12_attention_weight_min": -29.98763656616211,
      "activations/layer13_attention_weight_max": 67.53474426269531,
      "activations/layer13_attention_weight_min": -36.22978591918945,
      "activations/layer14_attention_weight_max": 61.127838134765625,
      "activations/layer14_attention_weight_min": -29.906156539916992,
      "activations/layer15_attention_weight_max": 57.60286331176758,
      "activations/layer15_attention_weight_min": -30.04539680480957,
      "activations/layer16_attention_weight_max": 43.25030517578125,
      "activations/layer16_attention_weight_min": -27.522916793823242,
      "activations/layer17_attention_weight_max": 69.95547485351562,
      "activations/layer17_attention_weight_min": -41.984779357910156,
      "activations/layer18_attention_weight_max": 58.3980712890625,
      "activations/layer18_attention_weight_min": -36.48353958129883,
      "activations/layer19_attention_weight_max": 24.30427360534668,
      "activations/layer19_attention_weight_min": -20.64747428894043,
      "activations/layer1_attention_weight_max": 17.49681854248047,
      "activations/layer1_attention_weight_min": -14.951583862304688,
      "activations/layer20_attention_weight_max": 28.64813804626465,
      "activations/layer20_attention_weight_min": -19.00042724609375,
      "activations/layer21_attention_weight_max": 48.171226501464844,
      "activations/layer21_attention_weight_min": -27.987899780273438,
      "activations/layer22_attention_weight_max": 37.85574722290039,
      "activations/layer22_attention_weight_min": -26.79192352294922,
      "activations/layer23_attention_weight_max": 41.300254821777344,
      "activations/layer23_attention_weight_min": -17.50747299194336,
      "activations/layer2_attention_weight_max": 30.157896041870117,
      "activations/layer2_attention_weight_min": -28.257139205932617,
      "activations/layer3_attention_weight_max": 83.5545425415039,
      "activations/layer3_attention_weight_min": -85.6550064086914,
      "activations/layer4_attention_weight_max": 103.26143646240234,
      "activations/layer4_attention_weight_min": -103.47521209716797,
      "activations/layer5_attention_weight_max": 97.79135131835938,
      "activations/layer5_attention_weight_min": -93.18954467773438,
      "activations/layer6_attention_weight_max": 59.07160949707031,
      "activations/layer6_attention_weight_min": -54.0094108581543,
      "activations/layer7_attention_weight_max": 73.58201599121094,
      "activations/layer7_attention_weight_min": -71.06245422363281,
      "activations/layer8_attention_weight_max": 47.59075927734375,
      "activations/layer8_attention_weight_min": -45.604530334472656,
      "activations/layer9_attention_weight_max": 52.765254974365234,
      "activations/layer9_attention_weight_min": -51.89384841918945,
      "epoch": 5.77,
      "learning_rate": 0.00011393863636363635,
      "loss": 2.8515,
      "step": 99300
    },
    {
      "activations/layer0_attention_weight_max": 14.636922836303711,
      "activations/layer0_attention_weight_min": -15.13313102722168,
      "activations/layer10_attention_weight_max": 40.080108642578125,
      "activations/layer10_attention_weight_min": -33.96849060058594,
      "activations/layer11_attention_weight_max": 31.233610153198242,
      "activations/layer11_attention_weight_min": -26.892127990722656,
      "activations/layer12_attention_weight_max": 24.392248153686523,
      "activations/layer12_attention_weight_min": -24.033594131469727,
      "activations/layer13_attention_weight_max": 53.34410095214844,
      "activations/layer13_attention_weight_min": -36.962215423583984,
      "activations/layer14_attention_weight_max": 50.979305267333984,
      "activations/layer14_attention_weight_min": -30.809688568115234,
      "activations/layer15_attention_weight_max": 42.35127258300781,
      "activations/layer15_attention_weight_min": -29.790292739868164,
      "activations/layer16_attention_weight_max": 37.547515869140625,
      "activations/layer16_attention_weight_min": -27.30316734313965,
      "activations/layer17_attention_weight_max": 58.88320541381836,
      "activations/layer17_attention_weight_min": -39.43283462524414,
      "activations/layer18_attention_weight_max": 48.84694290161133,
      "activations/layer18_attention_weight_min": -33.844486236572266,
      "activations/layer19_attention_weight_max": 22.984525680541992,
      "activations/layer19_attention_weight_min": -20.580078125,
      "activations/layer1_attention_weight_max": 16.88059425354004,
      "activations/layer1_attention_weight_min": -14.881628036499023,
      "activations/layer20_attention_weight_max": 23.313282012939453,
      "activations/layer20_attention_weight_min": -20.85177993774414,
      "activations/layer21_attention_weight_max": 41.992008209228516,
      "activations/layer21_attention_weight_min": -24.644210815429688,
      "activations/layer22_attention_weight_max": 35.14059829711914,
      "activations/layer22_attention_weight_min": -24.62767219543457,
      "activations/layer23_attention_weight_max": 32.67551040649414,
      "activations/layer23_attention_weight_min": -19.760927200317383,
      "activations/layer2_attention_weight_max": 30.889057159423828,
      "activations/layer2_attention_weight_min": -29.87557601928711,
      "activations/layer3_attention_weight_max": 82.63895416259766,
      "activations/layer3_attention_weight_min": -87.4189682006836,
      "activations/layer4_attention_weight_max": 100.39888000488281,
      "activations/layer4_attention_weight_min": -94.00049591064453,
      "activations/layer5_attention_weight_max": 96.27609252929688,
      "activations/layer5_attention_weight_min": -87.851318359375,
      "activations/layer6_attention_weight_max": 56.437992095947266,
      "activations/layer6_attention_weight_min": -56.186100006103516,
      "activations/layer7_attention_weight_max": 70.31621551513672,
      "activations/layer7_attention_weight_min": -72.36954498291016,
      "activations/layer8_attention_weight_max": 44.24944305419922,
      "activations/layer8_attention_weight_min": -42.40087890625,
      "activations/layer9_attention_weight_max": 48.736976623535156,
      "activations/layer9_attention_weight_min": -46.684146881103516,
      "epoch": 5.77,
      "learning_rate": 0.00011391969696969697,
      "loss": 2.8547,
      "step": 99350
    },
    {
      "activations/layer0_attention_weight_max": 14.369983673095703,
      "activations/layer0_attention_weight_min": -14.952200889587402,
      "activations/layer10_attention_weight_max": 35.88429641723633,
      "activations/layer10_attention_weight_min": -32.016700744628906,
      "activations/layer11_attention_weight_max": 28.540145874023438,
      "activations/layer11_attention_weight_min": -26.548843383789062,
      "activations/layer12_attention_weight_max": 27.877058029174805,
      "activations/layer12_attention_weight_min": -28.808090209960938,
      "activations/layer13_attention_weight_max": 40.37925720214844,
      "activations/layer13_attention_weight_min": -33.69029235839844,
      "activations/layer14_attention_weight_max": 35.980751037597656,
      "activations/layer14_attention_weight_min": -29.397619247436523,
      "activations/layer15_attention_weight_max": 38.60829162597656,
      "activations/layer15_attention_weight_min": -29.17682647705078,
      "activations/layer16_attention_weight_max": 34.897605895996094,
      "activations/layer16_attention_weight_min": -26.959430694580078,
      "activations/layer17_attention_weight_max": 53.074275970458984,
      "activations/layer17_attention_weight_min": -40.31833267211914,
      "activations/layer18_attention_weight_max": 46.091827392578125,
      "activations/layer18_attention_weight_min": -34.58765411376953,
      "activations/layer19_attention_weight_max": 22.87795639038086,
      "activations/layer19_attention_weight_min": -19.313199996948242,
      "activations/layer1_attention_weight_max": 17.018815994262695,
      "activations/layer1_attention_weight_min": -14.445178985595703,
      "activations/layer20_attention_weight_max": 20.35072135925293,
      "activations/layer20_attention_weight_min": -24.698057174682617,
      "activations/layer21_attention_weight_max": 36.15873336791992,
      "activations/layer21_attention_weight_min": -26.4373836517334,
      "activations/layer22_attention_weight_max": 30.580013275146484,
      "activations/layer22_attention_weight_min": -24.099546432495117,
      "activations/layer23_attention_weight_max": 29.263568878173828,
      "activations/layer23_attention_weight_min": -18.597375869750977,
      "activations/layer2_attention_weight_max": 29.21982192993164,
      "activations/layer2_attention_weight_min": -29.708629608154297,
      "activations/layer3_attention_weight_max": 82.42574310302734,
      "activations/layer3_attention_weight_min": -85.27672576904297,
      "activations/layer4_attention_weight_max": 99.37805938720703,
      "activations/layer4_attention_weight_min": -95.78067016601562,
      "activations/layer5_attention_weight_max": 92.3838119506836,
      "activations/layer5_attention_weight_min": -89.59447479248047,
      "activations/layer6_attention_weight_max": 54.079673767089844,
      "activations/layer6_attention_weight_min": -50.62390899658203,
      "activations/layer7_attention_weight_max": 71.46066284179688,
      "activations/layer7_attention_weight_min": -66.0093002319336,
      "activations/layer8_attention_weight_max": 41.94400405883789,
      "activations/layer8_attention_weight_min": -41.45624542236328,
      "activations/layer9_attention_weight_max": 47.895259857177734,
      "activations/layer9_attention_weight_min": -45.7247428894043,
      "epoch": 5.78,
      "learning_rate": 0.00011390075757575756,
      "loss": 2.8524,
      "step": 99400
    },
    {
      "activations/layer0_attention_weight_max": 14.202157020568848,
      "activations/layer0_attention_weight_min": -14.566150665283203,
      "activations/layer10_attention_weight_max": 42.5633659362793,
      "activations/layer10_attention_weight_min": -41.19229507446289,
      "activations/layer11_attention_weight_max": 34.24223327636719,
      "activations/layer11_attention_weight_min": -30.694334030151367,
      "activations/layer12_attention_weight_max": 27.076969146728516,
      "activations/layer12_attention_weight_min": -32.21697235107422,
      "activations/layer13_attention_weight_max": 46.95249938964844,
      "activations/layer13_attention_weight_min": -35.5013542175293,
      "activations/layer14_attention_weight_max": 65.84014129638672,
      "activations/layer14_attention_weight_min": -39.532936096191406,
      "activations/layer15_attention_weight_max": 54.72941589355469,
      "activations/layer15_attention_weight_min": -35.7536735534668,
      "activations/layer16_attention_weight_max": 32.73417282104492,
      "activations/layer16_attention_weight_min": -26.476810455322266,
      "activations/layer17_attention_weight_max": 55.2657585144043,
      "activations/layer17_attention_weight_min": -38.78841018676758,
      "activations/layer18_attention_weight_max": 49.041194915771484,
      "activations/layer18_attention_weight_min": -33.174720764160156,
      "activations/layer19_attention_weight_max": 24.27476692199707,
      "activations/layer19_attention_weight_min": -18.89482879638672,
      "activations/layer1_attention_weight_max": 17.886051177978516,
      "activations/layer1_attention_weight_min": -15.924967765808105,
      "activations/layer20_attention_weight_max": 23.41350746154785,
      "activations/layer20_attention_weight_min": -21.54479217529297,
      "activations/layer21_attention_weight_max": 41.52915573120117,
      "activations/layer21_attention_weight_min": -24.832412719726562,
      "activations/layer22_attention_weight_max": 35.471107482910156,
      "activations/layer22_attention_weight_min": -23.791318893432617,
      "activations/layer23_attention_weight_max": 32.29808807373047,
      "activations/layer23_attention_weight_min": -18.67333221435547,
      "activations/layer2_attention_weight_max": 32.74939727783203,
      "activations/layer2_attention_weight_min": -30.476211547851562,
      "activations/layer3_attention_weight_max": 85.71397399902344,
      "activations/layer3_attention_weight_min": -91.28331756591797,
      "activations/layer4_attention_weight_max": 108.38484954833984,
      "activations/layer4_attention_weight_min": -102.51472473144531,
      "activations/layer5_attention_weight_max": 103.1551513671875,
      "activations/layer5_attention_weight_min": -95.00444030761719,
      "activations/layer6_attention_weight_max": 62.78929138183594,
      "activations/layer6_attention_weight_min": -56.27306365966797,
      "activations/layer7_attention_weight_max": 89.2516860961914,
      "activations/layer7_attention_weight_min": -76.72652435302734,
      "activations/layer8_attention_weight_max": 52.92442321777344,
      "activations/layer8_attention_weight_min": -47.6573486328125,
      "activations/layer9_attention_weight_max": 70.87577056884766,
      "activations/layer9_attention_weight_min": -57.921077728271484,
      "epoch": 5.78,
      "learning_rate": 0.00011388181818181817,
      "loss": 2.8539,
      "step": 99450
    },
    {
      "activations/layer0_attention_weight_max": 16.033151626586914,
      "activations/layer0_attention_weight_min": -15.328377723693848,
      "activations/layer10_attention_weight_max": 42.9031982421875,
      "activations/layer10_attention_weight_min": -37.37813186645508,
      "activations/layer11_attention_weight_max": 37.135074615478516,
      "activations/layer11_attention_weight_min": -28.72638511657715,
      "activations/layer12_attention_weight_max": 35.621822357177734,
      "activations/layer12_attention_weight_min": -27.47003746032715,
      "activations/layer13_attention_weight_max": 50.46918487548828,
      "activations/layer13_attention_weight_min": -33.358482360839844,
      "activations/layer14_attention_weight_max": 47.56217575073242,
      "activations/layer14_attention_weight_min": -30.51555061340332,
      "activations/layer15_attention_weight_max": 40.48178482055664,
      "activations/layer15_attention_weight_min": -29.79833221435547,
      "activations/layer16_attention_weight_max": 32.69333267211914,
      "activations/layer16_attention_weight_min": -27.81706428527832,
      "activations/layer17_attention_weight_max": 56.41222381591797,
      "activations/layer17_attention_weight_min": -41.88005447387695,
      "activations/layer18_attention_weight_max": 52.89785385131836,
      "activations/layer18_attention_weight_min": -34.10733413696289,
      "activations/layer19_attention_weight_max": 24.7247371673584,
      "activations/layer19_attention_weight_min": -19.76790428161621,
      "activations/layer1_attention_weight_max": 18.40530776977539,
      "activations/layer1_attention_weight_min": -17.85487937927246,
      "activations/layer20_attention_weight_max": 23.968137741088867,
      "activations/layer20_attention_weight_min": -25.338529586791992,
      "activations/layer21_attention_weight_max": 37.13729476928711,
      "activations/layer21_attention_weight_min": -25.277738571166992,
      "activations/layer22_attention_weight_max": 34.475711822509766,
      "activations/layer22_attention_weight_min": -24.74859619140625,
      "activations/layer23_attention_weight_max": 33.975650787353516,
      "activations/layer23_attention_weight_min": -19.04458999633789,
      "activations/layer2_attention_weight_max": 31.145050048828125,
      "activations/layer2_attention_weight_min": -29.746688842773438,
      "activations/layer3_attention_weight_max": 82.90567016601562,
      "activations/layer3_attention_weight_min": -86.32984161376953,
      "activations/layer4_attention_weight_max": 102.97537994384766,
      "activations/layer4_attention_weight_min": -99.84587860107422,
      "activations/layer5_attention_weight_max": 99.30381774902344,
      "activations/layer5_attention_weight_min": -91.67916870117188,
      "activations/layer6_attention_weight_max": 56.85498809814453,
      "activations/layer6_attention_weight_min": -54.37553405761719,
      "activations/layer7_attention_weight_max": 71.72715759277344,
      "activations/layer7_attention_weight_min": -69.06059265136719,
      "activations/layer8_attention_weight_max": 48.55384826660156,
      "activations/layer8_attention_weight_min": -46.1063232421875,
      "activations/layer9_attention_weight_max": 57.270362854003906,
      "activations/layer9_attention_weight_min": -46.0600700378418,
      "epoch": 5.78,
      "learning_rate": 0.00011386287878787877,
      "loss": 2.85,
      "step": 99500
    },
    {
      "activations/layer0_attention_weight_max": 14.357346534729004,
      "activations/layer0_attention_weight_min": -15.218671798706055,
      "activations/layer10_attention_weight_max": 33.41951370239258,
      "activations/layer10_attention_weight_min": -33.626556396484375,
      "activations/layer11_attention_weight_max": 26.703861236572266,
      "activations/layer11_attention_weight_min": -30.137920379638672,
      "activations/layer12_attention_weight_max": 19.51799774169922,
      "activations/layer12_attention_weight_min": -24.640335083007812,
      "activations/layer13_attention_weight_max": 55.865501403808594,
      "activations/layer13_attention_weight_min": -35.32606506347656,
      "activations/layer14_attention_weight_max": 44.17107391357422,
      "activations/layer14_attention_weight_min": -32.253292083740234,
      "activations/layer15_attention_weight_max": 46.931697845458984,
      "activations/layer15_attention_weight_min": -30.68479347229004,
      "activations/layer16_attention_weight_max": 36.552764892578125,
      "activations/layer16_attention_weight_min": -26.596834182739258,
      "activations/layer17_attention_weight_max": 59.57733917236328,
      "activations/layer17_attention_weight_min": -39.533199310302734,
      "activations/layer18_attention_weight_max": 51.38188171386719,
      "activations/layer18_attention_weight_min": -34.63145065307617,
      "activations/layer19_attention_weight_max": 22.787282943725586,
      "activations/layer19_attention_weight_min": -20.767087936401367,
      "activations/layer1_attention_weight_max": 18.527189254760742,
      "activations/layer1_attention_weight_min": -17.157928466796875,
      "activations/layer20_attention_weight_max": 23.178733825683594,
      "activations/layer20_attention_weight_min": -22.71908950805664,
      "activations/layer21_attention_weight_max": 37.70942306518555,
      "activations/layer21_attention_weight_min": -26.60165023803711,
      "activations/layer22_attention_weight_max": 34.98516845703125,
      "activations/layer22_attention_weight_min": -24.32920265197754,
      "activations/layer23_attention_weight_max": 30.223417282104492,
      "activations/layer23_attention_weight_min": -17.344619750976562,
      "activations/layer2_attention_weight_max": 30.776931762695312,
      "activations/layer2_attention_weight_min": -28.549320220947266,
      "activations/layer3_attention_weight_max": 80.96345520019531,
      "activations/layer3_attention_weight_min": -85.19133758544922,
      "activations/layer4_attention_weight_max": 99.87584686279297,
      "activations/layer4_attention_weight_min": -91.36874389648438,
      "activations/layer5_attention_weight_max": 95.92083740234375,
      "activations/layer5_attention_weight_min": -86.87576293945312,
      "activations/layer6_attention_weight_max": 55.70487976074219,
      "activations/layer6_attention_weight_min": -51.232364654541016,
      "activations/layer7_attention_weight_max": 66.2280044555664,
      "activations/layer7_attention_weight_min": -64.65827941894531,
      "activations/layer8_attention_weight_max": 40.048824310302734,
      "activations/layer8_attention_weight_min": -41.40214538574219,
      "activations/layer9_attention_weight_max": 48.41074752807617,
      "activations/layer9_attention_weight_min": -46.00478744506836,
      "epoch": 5.78,
      "learning_rate": 0.00011384393939393939,
      "loss": 2.8413,
      "step": 99550
    },
    {
      "activations/layer0_attention_weight_max": 14.287819862365723,
      "activations/layer0_attention_weight_min": -14.94058609008789,
      "activations/layer10_attention_weight_max": 38.378173828125,
      "activations/layer10_attention_weight_min": -36.60769271850586,
      "activations/layer11_attention_weight_max": 28.980403900146484,
      "activations/layer11_attention_weight_min": -29.32695770263672,
      "activations/layer12_attention_weight_max": 23.705713272094727,
      "activations/layer12_attention_weight_min": -25.23325538635254,
      "activations/layer13_attention_weight_max": 50.27761459350586,
      "activations/layer13_attention_weight_min": -35.82624053955078,
      "activations/layer14_attention_weight_max": 40.05155944824219,
      "activations/layer14_attention_weight_min": -29.885475158691406,
      "activations/layer15_attention_weight_max": 36.03489303588867,
      "activations/layer15_attention_weight_min": -29.416500091552734,
      "activations/layer16_attention_weight_max": 34.068294525146484,
      "activations/layer16_attention_weight_min": -26.235837936401367,
      "activations/layer17_attention_weight_max": 52.668800354003906,
      "activations/layer17_attention_weight_min": -39.42891311645508,
      "activations/layer18_attention_weight_max": 46.62234878540039,
      "activations/layer18_attention_weight_min": -33.83291244506836,
      "activations/layer19_attention_weight_max": 24.820968627929688,
      "activations/layer19_attention_weight_min": -19.089916229248047,
      "activations/layer1_attention_weight_max": 16.81007194519043,
      "activations/layer1_attention_weight_min": -13.622187614440918,
      "activations/layer20_attention_weight_max": 19.726959228515625,
      "activations/layer20_attention_weight_min": -21.671281814575195,
      "activations/layer21_attention_weight_max": 31.78270149230957,
      "activations/layer21_attention_weight_min": -26.799394607543945,
      "activations/layer22_attention_weight_max": 34.58499526977539,
      "activations/layer22_attention_weight_min": -25.196029663085938,
      "activations/layer23_attention_weight_max": 28.663000106811523,
      "activations/layer23_attention_weight_min": -21.601133346557617,
      "activations/layer2_attention_weight_max": 27.211044311523438,
      "activations/layer2_attention_weight_min": -25.47498321533203,
      "activations/layer3_attention_weight_max": 69.74436950683594,
      "activations/layer3_attention_weight_min": -73.25614929199219,
      "activations/layer4_attention_weight_max": 91.77115631103516,
      "activations/layer4_attention_weight_min": -90.12061309814453,
      "activations/layer5_attention_weight_max": 89.04705810546875,
      "activations/layer5_attention_weight_min": -89.04939270019531,
      "activations/layer6_attention_weight_max": 54.2951774597168,
      "activations/layer6_attention_weight_min": -53.126197814941406,
      "activations/layer7_attention_weight_max": 70.22570037841797,
      "activations/layer7_attention_weight_min": -69.64429473876953,
      "activations/layer8_attention_weight_max": 40.892452239990234,
      "activations/layer8_attention_weight_min": -42.46854782104492,
      "activations/layer9_attention_weight_max": 48.76026153564453,
      "activations/layer9_attention_weight_min": -48.89913558959961,
      "epoch": 5.79,
      "learning_rate": 0.00011382499999999999,
      "loss": 2.8527,
      "step": 99600
    },
    {
      "activations/layer0_attention_weight_max": 15.237516403198242,
      "activations/layer0_attention_weight_min": -14.792669296264648,
      "activations/layer10_attention_weight_max": 41.51836013793945,
      "activations/layer10_attention_weight_min": -33.400001525878906,
      "activations/layer11_attention_weight_max": 30.061452865600586,
      "activations/layer11_attention_weight_min": -28.41666030883789,
      "activations/layer12_attention_weight_max": 24.13885498046875,
      "activations/layer12_attention_weight_min": -26.028871536254883,
      "activations/layer13_attention_weight_max": 53.23303985595703,
      "activations/layer13_attention_weight_min": -34.196022033691406,
      "activations/layer14_attention_weight_max": 47.84767150878906,
      "activations/layer14_attention_weight_min": -30.96383285522461,
      "activations/layer15_attention_weight_max": 44.404396057128906,
      "activations/layer15_attention_weight_min": -30.51410484313965,
      "activations/layer16_attention_weight_max": 34.36231231689453,
      "activations/layer16_attention_weight_min": -28.143470764160156,
      "activations/layer17_attention_weight_max": 55.56129455566406,
      "activations/layer17_attention_weight_min": -40.55857467651367,
      "activations/layer18_attention_weight_max": 50.43496322631836,
      "activations/layer18_attention_weight_min": -34.19612503051758,
      "activations/layer19_attention_weight_max": 24.706838607788086,
      "activations/layer19_attention_weight_min": -19.51838493347168,
      "activations/layer1_attention_weight_max": 18.205564498901367,
      "activations/layer1_attention_weight_min": -13.396946907043457,
      "activations/layer20_attention_weight_max": 23.82015609741211,
      "activations/layer20_attention_weight_min": -23.35441780090332,
      "activations/layer21_attention_weight_max": 45.36505126953125,
      "activations/layer21_attention_weight_min": -26.68143081665039,
      "activations/layer22_attention_weight_max": 35.04038619995117,
      "activations/layer22_attention_weight_min": -25.79996681213379,
      "activations/layer23_attention_weight_max": 36.619564056396484,
      "activations/layer23_attention_weight_min": -19.954025268554688,
      "activations/layer2_attention_weight_max": 26.27859878540039,
      "activations/layer2_attention_weight_min": -24.55652618408203,
      "activations/layer3_attention_weight_max": 72.70576477050781,
      "activations/layer3_attention_weight_min": -74.61090087890625,
      "activations/layer4_attention_weight_max": 97.62527465820312,
      "activations/layer4_attention_weight_min": -87.63835906982422,
      "activations/layer5_attention_weight_max": 92.2973861694336,
      "activations/layer5_attention_weight_min": -83.79299926757812,
      "activations/layer6_attention_weight_max": 55.64171600341797,
      "activations/layer6_attention_weight_min": -49.833499908447266,
      "activations/layer7_attention_weight_max": 66.82591247558594,
      "activations/layer7_attention_weight_min": -69.94721984863281,
      "activations/layer8_attention_weight_max": 44.6619758605957,
      "activations/layer8_attention_weight_min": -41.354854583740234,
      "activations/layer9_attention_weight_max": 48.889305114746094,
      "activations/layer9_attention_weight_min": -51.302276611328125,
      "epoch": 5.79,
      "learning_rate": 0.00011380606060606059,
      "loss": 2.8463,
      "step": 99650
    },
    {
      "activations/layer0_attention_weight_max": 13.876965522766113,
      "activations/layer0_attention_weight_min": -14.277746200561523,
      "activations/layer10_attention_weight_max": 40.87514114379883,
      "activations/layer10_attention_weight_min": -34.54391860961914,
      "activations/layer11_attention_weight_max": 29.239479064941406,
      "activations/layer11_attention_weight_min": -26.641345977783203,
      "activations/layer12_attention_weight_max": 21.632553100585938,
      "activations/layer12_attention_weight_min": -25.276905059814453,
      "activations/layer13_attention_weight_max": 48.04300308227539,
      "activations/layer13_attention_weight_min": -36.72357940673828,
      "activations/layer14_attention_weight_max": 48.08500671386719,
      "activations/layer14_attention_weight_min": -29.84930419921875,
      "activations/layer15_attention_weight_max": 40.148406982421875,
      "activations/layer15_attention_weight_min": -30.960939407348633,
      "activations/layer16_attention_weight_max": 35.333702087402344,
      "activations/layer16_attention_weight_min": -26.552597045898438,
      "activations/layer17_attention_weight_max": 59.48797607421875,
      "activations/layer17_attention_weight_min": -39.888877868652344,
      "activations/layer18_attention_weight_max": 57.61790084838867,
      "activations/layer18_attention_weight_min": -34.04922866821289,
      "activations/layer19_attention_weight_max": 23.36449432373047,
      "activations/layer19_attention_weight_min": -21.83782196044922,
      "activations/layer1_attention_weight_max": 18.024934768676758,
      "activations/layer1_attention_weight_min": -14.818391799926758,
      "activations/layer20_attention_weight_max": 30.695091247558594,
      "activations/layer20_attention_weight_min": -21.329282760620117,
      "activations/layer21_attention_weight_max": 56.82620620727539,
      "activations/layer21_attention_weight_min": -28.943334579467773,
      "activations/layer22_attention_weight_max": 33.73287582397461,
      "activations/layer22_attention_weight_min": -25.242734909057617,
      "activations/layer23_attention_weight_max": 32.13710021972656,
      "activations/layer23_attention_weight_min": -22.704147338867188,
      "activations/layer2_attention_weight_max": 28.627910614013672,
      "activations/layer2_attention_weight_min": -29.320457458496094,
      "activations/layer3_attention_weight_max": 78.27147674560547,
      "activations/layer3_attention_weight_min": -83.22953033447266,
      "activations/layer4_attention_weight_max": 98.28130340576172,
      "activations/layer4_attention_weight_min": -92.05796813964844,
      "activations/layer5_attention_weight_max": 93.31190490722656,
      "activations/layer5_attention_weight_min": -84.61993408203125,
      "activations/layer6_attention_weight_max": 59.30848693847656,
      "activations/layer6_attention_weight_min": -51.435340881347656,
      "activations/layer7_attention_weight_max": 70.08751678466797,
      "activations/layer7_attention_weight_min": -68.84698486328125,
      "activations/layer8_attention_weight_max": 44.535396575927734,
      "activations/layer8_attention_weight_min": -43.440940856933594,
      "activations/layer9_attention_weight_max": 48.82548141479492,
      "activations/layer9_attention_weight_min": -47.51556396484375,
      "epoch": 5.79,
      "learning_rate": 0.0001137871212121212,
      "loss": 2.8503,
      "step": 99700
    },
    {
      "activations/layer0_attention_weight_max": 14.667673110961914,
      "activations/layer0_attention_weight_min": -14.384340286254883,
      "activations/layer10_attention_weight_max": 39.56983947753906,
      "activations/layer10_attention_weight_min": -37.7327880859375,
      "activations/layer11_attention_weight_max": 30.197568893432617,
      "activations/layer11_attention_weight_min": -26.845653533935547,
      "activations/layer12_attention_weight_max": 21.924585342407227,
      "activations/layer12_attention_weight_min": -26.66791534423828,
      "activations/layer13_attention_weight_max": 47.6890754699707,
      "activations/layer13_attention_weight_min": -33.72218322753906,
      "activations/layer14_attention_weight_max": 44.076377868652344,
      "activations/layer14_attention_weight_min": -29.60065460205078,
      "activations/layer15_attention_weight_max": 39.06964874267578,
      "activations/layer15_attention_weight_min": -30.485170364379883,
      "activations/layer16_attention_weight_max": 33.312042236328125,
      "activations/layer16_attention_weight_min": -25.968746185302734,
      "activations/layer17_attention_weight_max": 57.2073974609375,
      "activations/layer17_attention_weight_min": -38.85939407348633,
      "activations/layer18_attention_weight_max": 48.19707489013672,
      "activations/layer18_attention_weight_min": -32.005332946777344,
      "activations/layer19_attention_weight_max": 21.201744079589844,
      "activations/layer19_attention_weight_min": -19.781240463256836,
      "activations/layer1_attention_weight_max": 17.8236141204834,
      "activations/layer1_attention_weight_min": -15.68557071685791,
      "activations/layer20_attention_weight_max": 22.707624435424805,
      "activations/layer20_attention_weight_min": -21.0488338470459,
      "activations/layer21_attention_weight_max": 39.981361389160156,
      "activations/layer21_attention_weight_min": -24.0960693359375,
      "activations/layer22_attention_weight_max": 38.2664794921875,
      "activations/layer22_attention_weight_min": -22.48148536682129,
      "activations/layer23_attention_weight_max": 29.705039978027344,
      "activations/layer23_attention_weight_min": -18.60898780822754,
      "activations/layer2_attention_weight_max": 31.245990753173828,
      "activations/layer2_attention_weight_min": -29.170413970947266,
      "activations/layer3_attention_weight_max": 80.48141479492188,
      "activations/layer3_attention_weight_min": -80.26258850097656,
      "activations/layer4_attention_weight_max": 97.7508773803711,
      "activations/layer4_attention_weight_min": -93.17794036865234,
      "activations/layer5_attention_weight_max": 95.29524993896484,
      "activations/layer5_attention_weight_min": -87.93272399902344,
      "activations/layer6_attention_weight_max": 60.2276725769043,
      "activations/layer6_attention_weight_min": -51.710758209228516,
      "activations/layer7_attention_weight_max": 75.42011260986328,
      "activations/layer7_attention_weight_min": -66.24618530273438,
      "activations/layer8_attention_weight_max": 42.9739875793457,
      "activations/layer8_attention_weight_min": -46.39570617675781,
      "activations/layer9_attention_weight_max": 52.119991302490234,
      "activations/layer9_attention_weight_min": -45.73325729370117,
      "epoch": 5.8,
      "learning_rate": 0.00011376818181818182,
      "loss": 2.8498,
      "step": 99750
    },
    {
      "activations/layer0_attention_weight_max": 14.12923812866211,
      "activations/layer0_attention_weight_min": -14.315286636352539,
      "activations/layer10_attention_weight_max": 39.733516693115234,
      "activations/layer10_attention_weight_min": -34.63888931274414,
      "activations/layer11_attention_weight_max": 29.29132652282715,
      "activations/layer11_attention_weight_min": -28.376041412353516,
      "activations/layer12_attention_weight_max": 24.527587890625,
      "activations/layer12_attention_weight_min": -24.103416442871094,
      "activations/layer13_attention_weight_max": 48.58427810668945,
      "activations/layer13_attention_weight_min": -35.267616271972656,
      "activations/layer14_attention_weight_max": 44.43223190307617,
      "activations/layer14_attention_weight_min": -29.85747718811035,
      "activations/layer15_attention_weight_max": 37.716487884521484,
      "activations/layer15_attention_weight_min": -29.645185470581055,
      "activations/layer16_attention_weight_max": 37.584251403808594,
      "activations/layer16_attention_weight_min": -25.339031219482422,
      "activations/layer17_attention_weight_max": 59.09486770629883,
      "activations/layer17_attention_weight_min": -40.582035064697266,
      "activations/layer18_attention_weight_max": 49.82135772705078,
      "activations/layer18_attention_weight_min": -35.22737503051758,
      "activations/layer19_attention_weight_max": 21.038183212280273,
      "activations/layer19_attention_weight_min": -19.629344940185547,
      "activations/layer1_attention_weight_max": 17.28112030029297,
      "activations/layer1_attention_weight_min": -13.23141860961914,
      "activations/layer20_attention_weight_max": 23.172338485717773,
      "activations/layer20_attention_weight_min": -22.148271560668945,
      "activations/layer21_attention_weight_max": 37.60039138793945,
      "activations/layer21_attention_weight_min": -26.260778427124023,
      "activations/layer22_attention_weight_max": 36.27412414550781,
      "activations/layer22_attention_weight_min": -24.20755386352539,
      "activations/layer23_attention_weight_max": 31.02400016784668,
      "activations/layer23_attention_weight_min": -18.502492904663086,
      "activations/layer2_attention_weight_max": 28.766498565673828,
      "activations/layer2_attention_weight_min": -27.50217056274414,
      "activations/layer3_attention_weight_max": 78.10559844970703,
      "activations/layer3_attention_weight_min": -81.2159652709961,
      "activations/layer4_attention_weight_max": 94.37248992919922,
      "activations/layer4_attention_weight_min": -90.47631072998047,
      "activations/layer5_attention_weight_max": 91.55921936035156,
      "activations/layer5_attention_weight_min": -83.57225799560547,
      "activations/layer6_attention_weight_max": 57.10422897338867,
      "activations/layer6_attention_weight_min": -49.68616485595703,
      "activations/layer7_attention_weight_max": 70.48426055908203,
      "activations/layer7_attention_weight_min": -65.071044921875,
      "activations/layer8_attention_weight_max": 44.73895263671875,
      "activations/layer8_attention_weight_min": -42.12958908081055,
      "activations/layer9_attention_weight_max": 53.2523307800293,
      "activations/layer9_attention_weight_min": -45.27853012084961,
      "epoch": 5.8,
      "learning_rate": 0.00011374924242424241,
      "loss": 2.8294,
      "step": 99800
    },
    {
      "activations/layer0_attention_weight_max": 14.289589881896973,
      "activations/layer0_attention_weight_min": -14.077608108520508,
      "activations/layer10_attention_weight_max": 37.9989013671875,
      "activations/layer10_attention_weight_min": -35.800025939941406,
      "activations/layer11_attention_weight_max": 30.718414306640625,
      "activations/layer11_attention_weight_min": -28.956411361694336,
      "activations/layer12_attention_weight_max": 27.065780639648438,
      "activations/layer12_attention_weight_min": -23.951818466186523,
      "activations/layer13_attention_weight_max": 51.47555923461914,
      "activations/layer13_attention_weight_min": -35.31575012207031,
      "activations/layer14_attention_weight_max": 53.087303161621094,
      "activations/layer14_attention_weight_min": -38.493507385253906,
      "activations/layer15_attention_weight_max": 45.69718551635742,
      "activations/layer15_attention_weight_min": -30.258102416992188,
      "activations/layer16_attention_weight_max": 35.6611213684082,
      "activations/layer16_attention_weight_min": -27.7856388092041,
      "activations/layer17_attention_weight_max": 59.77422332763672,
      "activations/layer17_attention_weight_min": -39.92177200317383,
      "activations/layer18_attention_weight_max": 53.91586685180664,
      "activations/layer18_attention_weight_min": -34.50926971435547,
      "activations/layer19_attention_weight_max": 23.256582260131836,
      "activations/layer19_attention_weight_min": -19.60622787475586,
      "activations/layer1_attention_weight_max": 17.742748260498047,
      "activations/layer1_attention_weight_min": -16.395492553710938,
      "activations/layer20_attention_weight_max": 21.77737808227539,
      "activations/layer20_attention_weight_min": -21.047893524169922,
      "activations/layer21_attention_weight_max": 40.610130310058594,
      "activations/layer21_attention_weight_min": -28.51498031616211,
      "activations/layer22_attention_weight_max": 35.69205093383789,
      "activations/layer22_attention_weight_min": -23.73594856262207,
      "activations/layer23_attention_weight_max": 33.79225158691406,
      "activations/layer23_attention_weight_min": -19.889846801757812,
      "activations/layer2_attention_weight_max": 29.06237030029297,
      "activations/layer2_attention_weight_min": -30.9036865234375,
      "activations/layer3_attention_weight_max": 78.2862777709961,
      "activations/layer3_attention_weight_min": -81.17379760742188,
      "activations/layer4_attention_weight_max": 101.96305847167969,
      "activations/layer4_attention_weight_min": -97.84685516357422,
      "activations/layer5_attention_weight_max": 98.06956481933594,
      "activations/layer5_attention_weight_min": -90.18531799316406,
      "activations/layer6_attention_weight_max": 58.7685432434082,
      "activations/layer6_attention_weight_min": -51.71387481689453,
      "activations/layer7_attention_weight_max": 71.89027404785156,
      "activations/layer7_attention_weight_min": -70.78350830078125,
      "activations/layer8_attention_weight_max": 45.13777160644531,
      "activations/layer8_attention_weight_min": -46.11760330200195,
      "activations/layer9_attention_weight_max": 52.761695861816406,
      "activations/layer9_attention_weight_min": -53.085693359375,
      "epoch": 5.8,
      "learning_rate": 0.00011373030303030302,
      "loss": 2.8435,
      "step": 99850
    },
    {
      "activations/layer0_attention_weight_max": 13.984686851501465,
      "activations/layer0_attention_weight_min": -14.07300090789795,
      "activations/layer10_attention_weight_max": 36.23573684692383,
      "activations/layer10_attention_weight_min": -34.30503463745117,
      "activations/layer11_attention_weight_max": 29.32840347290039,
      "activations/layer11_attention_weight_min": -27.037025451660156,
      "activations/layer12_attention_weight_max": 18.271284103393555,
      "activations/layer12_attention_weight_min": -25.870288848876953,
      "activations/layer13_attention_weight_max": 46.41444778442383,
      "activations/layer13_attention_weight_min": -34.618038177490234,
      "activations/layer14_attention_weight_max": 42.40898132324219,
      "activations/layer14_attention_weight_min": -32.24837875366211,
      "activations/layer15_attention_weight_max": 38.482242584228516,
      "activations/layer15_attention_weight_min": -31.684755325317383,
      "activations/layer16_attention_weight_max": 32.01328659057617,
      "activations/layer16_attention_weight_min": -28.402629852294922,
      "activations/layer17_attention_weight_max": 53.43245315551758,
      "activations/layer17_attention_weight_min": -41.40779113769531,
      "activations/layer18_attention_weight_max": 51.49665832519531,
      "activations/layer18_attention_weight_min": -34.92440414428711,
      "activations/layer19_attention_weight_max": 21.002281188964844,
      "activations/layer19_attention_weight_min": -21.343412399291992,
      "activations/layer1_attention_weight_max": 17.723785400390625,
      "activations/layer1_attention_weight_min": -16.761550903320312,
      "activations/layer20_attention_weight_max": 22.375228881835938,
      "activations/layer20_attention_weight_min": -21.64116668701172,
      "activations/layer21_attention_weight_max": 42.18681716918945,
      "activations/layer21_attention_weight_min": -25.743961334228516,
      "activations/layer22_attention_weight_max": 35.890682220458984,
      "activations/layer22_attention_weight_min": -24.661041259765625,
      "activations/layer23_attention_weight_max": 34.151939392089844,
      "activations/layer23_attention_weight_min": -21.317529678344727,
      "activations/layer2_attention_weight_max": 29.16640853881836,
      "activations/layer2_attention_weight_min": -27.49299430847168,
      "activations/layer3_attention_weight_max": 77.69718933105469,
      "activations/layer3_attention_weight_min": -77.9208984375,
      "activations/layer4_attention_weight_max": 95.9384994506836,
      "activations/layer4_attention_weight_min": -91.17755889892578,
      "activations/layer5_attention_weight_max": 92.1230697631836,
      "activations/layer5_attention_weight_min": -84.1619873046875,
      "activations/layer6_attention_weight_max": 54.411128997802734,
      "activations/layer6_attention_weight_min": -50.76070785522461,
      "activations/layer7_attention_weight_max": 68.8701171875,
      "activations/layer7_attention_weight_min": -64.93136596679688,
      "activations/layer8_attention_weight_max": 40.964351654052734,
      "activations/layer8_attention_weight_min": -41.32025909423828,
      "activations/layer9_attention_weight_max": 45.55495834350586,
      "activations/layer9_attention_weight_min": -46.47863006591797,
      "epoch": 5.8,
      "learning_rate": 0.00011371136363636364,
      "loss": 2.849,
      "step": 99900
    },
    {
      "activations/layer0_attention_weight_max": 14.666412353515625,
      "activations/layer0_attention_weight_min": -13.856454849243164,
      "activations/layer10_attention_weight_max": 39.371700286865234,
      "activations/layer10_attention_weight_min": -36.21802520751953,
      "activations/layer11_attention_weight_max": 29.893362045288086,
      "activations/layer11_attention_weight_min": -28.89606475830078,
      "activations/layer12_attention_weight_max": 23.522174835205078,
      "activations/layer12_attention_weight_min": -26.576303482055664,
      "activations/layer13_attention_weight_max": 54.85942077636719,
      "activations/layer13_attention_weight_min": -33.33573532104492,
      "activations/layer14_attention_weight_max": 59.80112838745117,
      "activations/layer14_attention_weight_min": -33.901607513427734,
      "activations/layer15_attention_weight_max": 40.1489372253418,
      "activations/layer15_attention_weight_min": -30.451139450073242,
      "activations/layer16_attention_weight_max": 33.2838249206543,
      "activations/layer16_attention_weight_min": -26.16613006591797,
      "activations/layer17_attention_weight_max": 53.359527587890625,
      "activations/layer17_attention_weight_min": -39.1802864074707,
      "activations/layer18_attention_weight_max": 46.2158317565918,
      "activations/layer18_attention_weight_min": -31.95745277404785,
      "activations/layer19_attention_weight_max": 20.96954345703125,
      "activations/layer19_attention_weight_min": -20.099159240722656,
      "activations/layer1_attention_weight_max": 17.378664016723633,
      "activations/layer1_attention_weight_min": -15.555030822753906,
      "activations/layer20_attention_weight_max": 20.118528366088867,
      "activations/layer20_attention_weight_min": -19.522239685058594,
      "activations/layer21_attention_weight_max": 40.61891555786133,
      "activations/layer21_attention_weight_min": -22.503862380981445,
      "activations/layer22_attention_weight_max": 35.70186233520508,
      "activations/layer22_attention_weight_min": -25.035364151000977,
      "activations/layer23_attention_weight_max": 33.57093048095703,
      "activations/layer23_attention_weight_min": -19.646686553955078,
      "activations/layer2_attention_weight_max": 29.951597213745117,
      "activations/layer2_attention_weight_min": -28.397159576416016,
      "activations/layer3_attention_weight_max": 85.0951919555664,
      "activations/layer3_attention_weight_min": -89.34842681884766,
      "activations/layer4_attention_weight_max": 104.82706451416016,
      "activations/layer4_attention_weight_min": -98.00241088867188,
      "activations/layer5_attention_weight_max": 97.82249450683594,
      "activations/layer5_attention_weight_min": -92.3873519897461,
      "activations/layer6_attention_weight_max": 60.41572952270508,
      "activations/layer6_attention_weight_min": -56.55333709716797,
      "activations/layer7_attention_weight_max": 70.67424774169922,
      "activations/layer7_attention_weight_min": -72.99159240722656,
      "activations/layer8_attention_weight_max": 44.68171310424805,
      "activations/layer8_attention_weight_min": -45.37642288208008,
      "activations/layer9_attention_weight_max": 49.81178665161133,
      "activations/layer9_attention_weight_min": -49.3336296081543,
      "epoch": 5.81,
      "learning_rate": 0.00011369242424242423,
      "loss": 2.8532,
      "step": 99950
    },
    {
      "activations/layer0_attention_weight_max": 15.119329452514648,
      "activations/layer0_attention_weight_min": -14.210984230041504,
      "activations/layer10_attention_weight_max": 37.36609649658203,
      "activations/layer10_attention_weight_min": -32.083030700683594,
      "activations/layer11_attention_weight_max": 30.35548210144043,
      "activations/layer11_attention_weight_min": -27.664981842041016,
      "activations/layer12_attention_weight_max": 21.215499877929688,
      "activations/layer12_attention_weight_min": -25.823816299438477,
      "activations/layer13_attention_weight_max": 43.78178024291992,
      "activations/layer13_attention_weight_min": -30.050891876220703,
      "activations/layer14_attention_weight_max": 42.87950134277344,
      "activations/layer14_attention_weight_min": -29.58838653564453,
      "activations/layer15_attention_weight_max": 45.43708419799805,
      "activations/layer15_attention_weight_min": -28.664993286132812,
      "activations/layer16_attention_weight_max": 36.634281158447266,
      "activations/layer16_attention_weight_min": -27.11045265197754,
      "activations/layer17_attention_weight_max": 57.0599365234375,
      "activations/layer17_attention_weight_min": -43.11152267456055,
      "activations/layer18_attention_weight_max": 50.79536056518555,
      "activations/layer18_attention_weight_min": -36.08881759643555,
      "activations/layer19_attention_weight_max": 24.31568145751953,
      "activations/layer19_attention_weight_min": -20.218780517578125,
      "activations/layer1_attention_weight_max": 18.641674041748047,
      "activations/layer1_attention_weight_min": -13.973700523376465,
      "activations/layer20_attention_weight_max": 23.319765090942383,
      "activations/layer20_attention_weight_min": -25.390541076660156,
      "activations/layer21_attention_weight_max": 40.12102508544922,
      "activations/layer21_attention_weight_min": -25.691959381103516,
      "activations/layer22_attention_weight_max": 35.83817672729492,
      "activations/layer22_attention_weight_min": -25.34697914123535,
      "activations/layer23_attention_weight_max": 32.364959716796875,
      "activations/layer23_attention_weight_min": -21.75177574157715,
      "activations/layer2_attention_weight_max": 30.979598999023438,
      "activations/layer2_attention_weight_min": -29.332813262939453,
      "activations/layer3_attention_weight_max": 85.4126968383789,
      "activations/layer3_attention_weight_min": -85.86487579345703,
      "activations/layer4_attention_weight_max": 104.62066650390625,
      "activations/layer4_attention_weight_min": -99.1151351928711,
      "activations/layer5_attention_weight_max": 96.7770004272461,
      "activations/layer5_attention_weight_min": -87.6673812866211,
      "activations/layer6_attention_weight_max": 55.72771072387695,
      "activations/layer6_attention_weight_min": -51.56032180786133,
      "activations/layer7_attention_weight_max": 69.8516845703125,
      "activations/layer7_attention_weight_min": -67.1355209350586,
      "activations/layer8_attention_weight_max": 44.66285705566406,
      "activations/layer8_attention_weight_min": -45.32337188720703,
      "activations/layer9_attention_weight_max": 54.20551681518555,
      "activations/layer9_attention_weight_min": -46.649845123291016,
      "epoch": 5.81,
      "learning_rate": 0.00011367348484848484,
      "loss": 2.8508,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_loss": 2.791015625,
      "eval_runtime": 8.5776,
      "eval_samples_per_second": 500.604,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_openwebtext_loss": 2.791015625,
      "eval_openwebtext_ppl": 16.2975636122563,
      "eval_openwebtext_runtime": 8.5776,
      "eval_openwebtext_samples_per_second": 500.604,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_wikitext_loss": 3.0234375,
      "eval_wikitext_ppl": 20.56185170491249,
      "eval_wikitext_runtime": 2.0308,
      "eval_wikitext_samples_per_second": 224.537,
      "step": 100000
    },
    {
      "epoch": 5.81,
      "eval_lambada_loss": 2.763671875,
      "eval_lambada_ppl": 15.85796463907484,
      "eval_lambada_runtime": 9.6301,
      "eval_lambada_samples_per_second": 505.603,
      "step": 100000
    },
    {
      "activations/layer0_attention_weight_max": 15.11760425567627,
      "activations/layer0_attention_weight_min": -14.761791229248047,
      "activations/layer10_attention_weight_max": 36.87655258178711,
      "activations/layer10_attention_weight_min": -33.12633514404297,
      "activations/layer11_attention_weight_max": 28.90265464782715,
      "activations/layer11_attention_weight_min": -28.265392303466797,
      "activations/layer12_attention_weight_max": 24.14234161376953,
      "activations/layer12_attention_weight_min": -24.97580337524414,
      "activations/layer13_attention_weight_max": 45.6077995300293,
      "activations/layer13_attention_weight_min": -37.9183464050293,
      "activations/layer14_attention_weight_max": 41.63054275512695,
      "activations/layer14_attention_weight_min": -31.72132682800293,
      "activations/layer15_attention_weight_max": 35.51446533203125,
      "activations/layer15_attention_weight_min": -29.928380966186523,
      "activations/layer16_attention_weight_max": 36.4637336730957,
      "activations/layer16_attention_weight_min": -25.820669174194336,
      "activations/layer17_attention_weight_max": 54.04133987426758,
      "activations/layer17_attention_weight_min": -40.052154541015625,
      "activations/layer18_attention_weight_max": 42.14802169799805,
      "activations/layer18_attention_weight_min": -34.522640228271484,
      "activations/layer19_attention_weight_max": 20.42283058166504,
      "activations/layer19_attention_weight_min": -20.187376022338867,
      "activations/layer1_attention_weight_max": 16.893442153930664,
      "activations/layer1_attention_weight_min": -15.572629928588867,
      "activations/layer20_attention_weight_max": 21.846189498901367,
      "activations/layer20_attention_weight_min": -21.291109085083008,
      "activations/layer21_attention_weight_max": 34.57353591918945,
      "activations/layer21_attention_weight_min": -27.188167572021484,
      "activations/layer22_attention_weight_max": 33.80726623535156,
      "activations/layer22_attention_weight_min": -24.62185287475586,
      "activations/layer23_attention_weight_max": 27.98147964477539,
      "activations/layer23_attention_weight_min": -22.963069915771484,
      "activations/layer2_attention_weight_max": 31.070980072021484,
      "activations/layer2_attention_weight_min": -27.431427001953125,
      "activations/layer3_attention_weight_max": 86.8996353149414,
      "activations/layer3_attention_weight_min": -83.02971649169922,
      "activations/layer4_attention_weight_max": 97.87622833251953,
      "activations/layer4_attention_weight_min": -93.32759857177734,
      "activations/layer5_attention_weight_max": 92.6826171875,
      "activations/layer5_attention_weight_min": -86.91307067871094,
      "activations/layer6_attention_weight_max": 53.85227966308594,
      "activations/layer6_attention_weight_min": -50.78923416137695,
      "activations/layer7_attention_weight_max": 67.84141540527344,
      "activations/layer7_attention_weight_min": -67.67456817626953,
      "activations/layer8_attention_weight_max": 41.56306457519531,
      "activations/layer8_attention_weight_min": -43.23418045043945,
      "activations/layer9_attention_weight_max": 51.796932220458984,
      "activations/layer9_attention_weight_min": -45.07011413574219,
      "epoch": 5.81,
      "learning_rate": 0.00011365454545454544,
      "loss": 2.8402,
      "step": 100050
    },
    {
      "activations/layer0_attention_weight_max": 14.3712797164917,
      "activations/layer0_attention_weight_min": -14.15997314453125,
      "activations/layer10_attention_weight_max": 40.979366302490234,
      "activations/layer10_attention_weight_min": -34.711063385009766,
      "activations/layer11_attention_weight_max": 33.79474639892578,
      "activations/layer11_attention_weight_min": -27.78797721862793,
      "activations/layer12_attention_weight_max": 22.075241088867188,
      "activations/layer12_attention_weight_min": -26.017248153686523,
      "activations/layer13_attention_weight_max": 45.03150177001953,
      "activations/layer13_attention_weight_min": -34.03117752075195,
      "activations/layer14_attention_weight_max": 62.73286437988281,
      "activations/layer14_attention_weight_min": -36.54403305053711,
      "activations/layer15_attention_weight_max": 51.22257995605469,
      "activations/layer15_attention_weight_min": -29.177011489868164,
      "activations/layer16_attention_weight_max": 37.43488311767578,
      "activations/layer16_attention_weight_min": -26.10167694091797,
      "activations/layer17_attention_weight_max": 59.87857437133789,
      "activations/layer17_attention_weight_min": -41.5650520324707,
      "activations/layer18_attention_weight_max": 54.57361602783203,
      "activations/layer18_attention_weight_min": -35.816795349121094,
      "activations/layer19_attention_weight_max": 21.734386444091797,
      "activations/layer19_attention_weight_min": -20.69049072265625,
      "activations/layer1_attention_weight_max": 19.223798751831055,
      "activations/layer1_attention_weight_min": -15.111146926879883,
      "activations/layer20_attention_weight_max": 22.587085723876953,
      "activations/layer20_attention_weight_min": -25.271379470825195,
      "activations/layer21_attention_weight_max": 43.31214904785156,
      "activations/layer21_attention_weight_min": -26.371213912963867,
      "activations/layer22_attention_weight_max": 37.775211334228516,
      "activations/layer22_attention_weight_min": -26.67087745666504,
      "activations/layer23_attention_weight_max": 26.468748092651367,
      "activations/layer23_attention_weight_min": -20.520666122436523,
      "activations/layer2_attention_weight_max": 32.688697814941406,
      "activations/layer2_attention_weight_min": -29.476713180541992,
      "activations/layer3_attention_weight_max": 82.68550109863281,
      "activations/layer3_attention_weight_min": -80.00850677490234,
      "activations/layer4_attention_weight_max": 99.56026458740234,
      "activations/layer4_attention_weight_min": -93.84600067138672,
      "activations/layer5_attention_weight_max": 97.55308532714844,
      "activations/layer5_attention_weight_min": -87.29037475585938,
      "activations/layer6_attention_weight_max": 58.25758743286133,
      "activations/layer6_attention_weight_min": -52.224464416503906,
      "activations/layer7_attention_weight_max": 71.99858856201172,
      "activations/layer7_attention_weight_min": -70.24320220947266,
      "activations/layer8_attention_weight_max": 44.7984733581543,
      "activations/layer8_attention_weight_min": -41.4338493347168,
      "activations/layer9_attention_weight_max": 58.9270133972168,
      "activations/layer9_attention_weight_min": -53.48415756225586,
      "epoch": 5.82,
      "learning_rate": 0.00011363560606060604,
      "loss": 2.8451,
      "step": 100100
    },
    {
      "activations/layer0_attention_weight_max": 14.491962432861328,
      "activations/layer0_attention_weight_min": -14.407938957214355,
      "activations/layer10_attention_weight_max": 35.84653091430664,
      "activations/layer10_attention_weight_min": -34.888275146484375,
      "activations/layer11_attention_weight_max": 28.29218101501465,
      "activations/layer11_attention_weight_min": -28.42544174194336,
      "activations/layer12_attention_weight_max": 22.720136642456055,
      "activations/layer12_attention_weight_min": -23.329696655273438,
      "activations/layer13_attention_weight_max": 45.14753723144531,
      "activations/layer13_attention_weight_min": -33.09861373901367,
      "activations/layer14_attention_weight_max": 39.787574768066406,
      "activations/layer14_attention_weight_min": -32.88590621948242,
      "activations/layer15_attention_weight_max": 35.59835433959961,
      "activations/layer15_attention_weight_min": -29.659753799438477,
      "activations/layer16_attention_weight_max": 31.24322509765625,
      "activations/layer16_attention_weight_min": -26.835708618164062,
      "activations/layer17_attention_weight_max": 52.563987731933594,
      "activations/layer17_attention_weight_min": -39.21986770629883,
      "activations/layer18_attention_weight_max": 50.369747161865234,
      "activations/layer18_attention_weight_min": -35.10537338256836,
      "activations/layer19_attention_weight_max": 20.905126571655273,
      "activations/layer19_attention_weight_min": -20.40657615661621,
      "activations/layer1_attention_weight_max": 16.84286880493164,
      "activations/layer1_attention_weight_min": -14.201032638549805,
      "activations/layer20_attention_weight_max": 21.143930435180664,
      "activations/layer20_attention_weight_min": -20.53337287902832,
      "activations/layer21_attention_weight_max": 34.80712127685547,
      "activations/layer21_attention_weight_min": -24.11539649963379,
      "activations/layer22_attention_weight_max": 32.02351379394531,
      "activations/layer22_attention_weight_min": -24.439573287963867,
      "activations/layer23_attention_weight_max": 28.715946197509766,
      "activations/layer23_attention_weight_min": -19.24074935913086,
      "activations/layer2_attention_weight_max": 31.681554794311523,
      "activations/layer2_attention_weight_min": -28.71902847290039,
      "activations/layer3_attention_weight_max": 81.05413818359375,
      "activations/layer3_attention_weight_min": -81.11416625976562,
      "activations/layer4_attention_weight_max": 95.71868896484375,
      "activations/layer4_attention_weight_min": -90.99271392822266,
      "activations/layer5_attention_weight_max": 91.3416976928711,
      "activations/layer5_attention_weight_min": -84.94593811035156,
      "activations/layer6_attention_weight_max": 54.86347579956055,
      "activations/layer6_attention_weight_min": -52.575443267822266,
      "activations/layer7_attention_weight_max": 69.25250244140625,
      "activations/layer7_attention_weight_min": -69.67866516113281,
      "activations/layer8_attention_weight_max": 43.251216888427734,
      "activations/layer8_attention_weight_min": -43.29855728149414,
      "activations/layer9_attention_weight_max": 47.28506851196289,
      "activations/layer9_attention_weight_min": -45.478763580322266,
      "epoch": 5.82,
      "learning_rate": 0.00011361666666666666,
      "loss": 2.8374,
      "step": 100150
    },
    {
      "activations/layer0_attention_weight_max": 14.512140274047852,
      "activations/layer0_attention_weight_min": -14.561748504638672,
      "activations/layer10_attention_weight_max": 38.067779541015625,
      "activations/layer10_attention_weight_min": -33.373138427734375,
      "activations/layer11_attention_weight_max": 30.02865982055664,
      "activations/layer11_attention_weight_min": -30.3858699798584,
      "activations/layer12_attention_weight_max": 21.19902992248535,
      "activations/layer12_attention_weight_min": -25.299732208251953,
      "activations/layer13_attention_weight_max": 58.66223907470703,
      "activations/layer13_attention_weight_min": -36.85218048095703,
      "activations/layer14_attention_weight_max": 50.62007141113281,
      "activations/layer14_attention_weight_min": -30.569608688354492,
      "activations/layer15_attention_weight_max": 42.62012481689453,
      "activations/layer15_attention_weight_min": -31.337890625,
      "activations/layer16_attention_weight_max": 34.51598358154297,
      "activations/layer16_attention_weight_min": -27.396881103515625,
      "activations/layer17_attention_weight_max": 58.97346878051758,
      "activations/layer17_attention_weight_min": -40.434322357177734,
      "activations/layer18_attention_weight_max": 50.24673843383789,
      "activations/layer18_attention_weight_min": -34.566307067871094,
      "activations/layer19_attention_weight_max": 26.3640193939209,
      "activations/layer19_attention_weight_min": -24.673173904418945,
      "activations/layer1_attention_weight_max": 16.901226043701172,
      "activations/layer1_attention_weight_min": -14.116873741149902,
      "activations/layer20_attention_weight_max": 23.490285873413086,
      "activations/layer20_attention_weight_min": -22.255037307739258,
      "activations/layer21_attention_weight_max": 45.1882209777832,
      "activations/layer21_attention_weight_min": -27.94032859802246,
      "activations/layer22_attention_weight_max": 35.45918655395508,
      "activations/layer22_attention_weight_min": -25.51709747314453,
      "activations/layer23_attention_weight_max": 33.4736213684082,
      "activations/layer23_attention_weight_min": -20.48361587524414,
      "activations/layer2_attention_weight_max": 30.121246337890625,
      "activations/layer2_attention_weight_min": -29.167510986328125,
      "activations/layer3_attention_weight_max": 84.06912231445312,
      "activations/layer3_attention_weight_min": -84.7152099609375,
      "activations/layer4_attention_weight_max": 98.87805938720703,
      "activations/layer4_attention_weight_min": -93.80155181884766,
      "activations/layer5_attention_weight_max": 93.22856903076172,
      "activations/layer5_attention_weight_min": -89.63369750976562,
      "activations/layer6_attention_weight_max": 57.36426544189453,
      "activations/layer6_attention_weight_min": -52.00181198120117,
      "activations/layer7_attention_weight_max": 68.03440856933594,
      "activations/layer7_attention_weight_min": -67.03790283203125,
      "activations/layer8_attention_weight_max": 42.586570739746094,
      "activations/layer8_attention_weight_min": -42.651702880859375,
      "activations/layer9_attention_weight_max": 49.71834945678711,
      "activations/layer9_attention_weight_min": -49.52108383178711,
      "epoch": 5.82,
      "learning_rate": 0.00011359772727272726,
      "loss": 2.8499,
      "step": 100200
    },
    {
      "activations/layer0_attention_weight_max": 15.234696388244629,
      "activations/layer0_attention_weight_min": -15.11893367767334,
      "activations/layer10_attention_weight_max": 36.614070892333984,
      "activations/layer10_attention_weight_min": -35.39349365234375,
      "activations/layer11_attention_weight_max": 29.827346801757812,
      "activations/layer11_attention_weight_min": -27.038686752319336,
      "activations/layer12_attention_weight_max": 23.957263946533203,
      "activations/layer12_attention_weight_min": -23.104047775268555,
      "activations/layer13_attention_weight_max": 46.41770553588867,
      "activations/layer13_attention_weight_min": -33.97181701660156,
      "activations/layer14_attention_weight_max": 41.8179817199707,
      "activations/layer14_attention_weight_min": -30.05974769592285,
      "activations/layer15_attention_weight_max": 40.17940139770508,
      "activations/layer15_attention_weight_min": -29.739473342895508,
      "activations/layer16_attention_weight_max": 32.43600845336914,
      "activations/layer16_attention_weight_min": -28.333541870117188,
      "activations/layer17_attention_weight_max": 52.3764762878418,
      "activations/layer17_attention_weight_min": -41.498844146728516,
      "activations/layer18_attention_weight_max": 46.03174591064453,
      "activations/layer18_attention_weight_min": -34.81053924560547,
      "activations/layer19_attention_weight_max": 25.06378746032715,
      "activations/layer19_attention_weight_min": -21.047975540161133,
      "activations/layer1_attention_weight_max": 17.34697914123535,
      "activations/layer1_attention_weight_min": -17.556835174560547,
      "activations/layer20_attention_weight_max": 20.953838348388672,
      "activations/layer20_attention_weight_min": -21.499717712402344,
      "activations/layer21_attention_weight_max": 39.50590896606445,
      "activations/layer21_attention_weight_min": -24.9815616607666,
      "activations/layer22_attention_weight_max": 32.427974700927734,
      "activations/layer22_attention_weight_min": -24.754344940185547,
      "activations/layer23_attention_weight_max": 27.31769561767578,
      "activations/layer23_attention_weight_min": -19.145580291748047,
      "activations/layer2_attention_weight_max": 30.339101791381836,
      "activations/layer2_attention_weight_min": -28.738990783691406,
      "activations/layer3_attention_weight_max": 83.9496078491211,
      "activations/layer3_attention_weight_min": -84.41303253173828,
      "activations/layer4_attention_weight_max": 101.09989166259766,
      "activations/layer4_attention_weight_min": -94.4971923828125,
      "activations/layer5_attention_weight_max": 96.36211395263672,
      "activations/layer5_attention_weight_min": -89.64686584472656,
      "activations/layer6_attention_weight_max": 55.803627014160156,
      "activations/layer6_attention_weight_min": -52.22249221801758,
      "activations/layer7_attention_weight_max": 76.60924530029297,
      "activations/layer7_attention_weight_min": -68.9986343383789,
      "activations/layer8_attention_weight_max": 42.02493667602539,
      "activations/layer8_attention_weight_min": -41.2773323059082,
      "activations/layer9_attention_weight_max": 57.543487548828125,
      "activations/layer9_attention_weight_min": -48.65290832519531,
      "epoch": 5.83,
      "learning_rate": 0.00011357878787878788,
      "loss": 2.8535,
      "step": 100250
    },
    {
      "activations/layer0_attention_weight_max": 13.939002990722656,
      "activations/layer0_attention_weight_min": -15.047285079956055,
      "activations/layer10_attention_weight_max": 42.438472747802734,
      "activations/layer10_attention_weight_min": -36.71140670776367,
      "activations/layer11_attention_weight_max": 29.704174041748047,
      "activations/layer11_attention_weight_min": -29.852699279785156,
      "activations/layer12_attention_weight_max": 23.703569412231445,
      "activations/layer12_attention_weight_min": -24.72871208190918,
      "activations/layer13_attention_weight_max": 48.751407623291016,
      "activations/layer13_attention_weight_min": -33.4000358581543,
      "activations/layer14_attention_weight_max": 52.616539001464844,
      "activations/layer14_attention_weight_min": -31.485172271728516,
      "activations/layer15_attention_weight_max": 39.644989013671875,
      "activations/layer15_attention_weight_min": -29.580547332763672,
      "activations/layer16_attention_weight_max": 38.63104248046875,
      "activations/layer16_attention_weight_min": -27.54439926147461,
      "activations/layer17_attention_weight_max": 60.06386947631836,
      "activations/layer17_attention_weight_min": -42.91236114501953,
      "activations/layer18_attention_weight_max": 53.50578308105469,
      "activations/layer18_attention_weight_min": -33.55156707763672,
      "activations/layer19_attention_weight_max": 22.257904052734375,
      "activations/layer19_attention_weight_min": -20.61593246459961,
      "activations/layer1_attention_weight_max": 17.605308532714844,
      "activations/layer1_attention_weight_min": -15.279376029968262,
      "activations/layer20_attention_weight_max": 23.027177810668945,
      "activations/layer20_attention_weight_min": -23.46783447265625,
      "activations/layer21_attention_weight_max": 44.33891296386719,
      "activations/layer21_attention_weight_min": -26.589275360107422,
      "activations/layer22_attention_weight_max": 35.7391357421875,
      "activations/layer22_attention_weight_min": -24.16154670715332,
      "activations/layer23_attention_weight_max": 33.6190299987793,
      "activations/layer23_attention_weight_min": -20.709495544433594,
      "activations/layer2_attention_weight_max": 31.895231246948242,
      "activations/layer2_attention_weight_min": -29.303342819213867,
      "activations/layer3_attention_weight_max": 81.91205596923828,
      "activations/layer3_attention_weight_min": -88.47999572753906,
      "activations/layer4_attention_weight_max": 100.88909912109375,
      "activations/layer4_attention_weight_min": -97.80294036865234,
      "activations/layer5_attention_weight_max": 98.19834899902344,
      "activations/layer5_attention_weight_min": -87.80984497070312,
      "activations/layer6_attention_weight_max": 58.03779220581055,
      "activations/layer6_attention_weight_min": -52.93242263793945,
      "activations/layer7_attention_weight_max": 71.35275268554688,
      "activations/layer7_attention_weight_min": -69.52005004882812,
      "activations/layer8_attention_weight_max": 45.20685958862305,
      "activations/layer8_attention_weight_min": -46.351810455322266,
      "activations/layer9_attention_weight_max": 52.97217559814453,
      "activations/layer9_attention_weight_min": -51.41695785522461,
      "epoch": 5.83,
      "learning_rate": 0.00011355984848484848,
      "loss": 2.8421,
      "step": 100300
    },
    {
      "activations/layer0_attention_weight_max": 14.200884819030762,
      "activations/layer0_attention_weight_min": -14.779966354370117,
      "activations/layer10_attention_weight_max": 37.004425048828125,
      "activations/layer10_attention_weight_min": -34.71525192260742,
      "activations/layer11_attention_weight_max": 30.53310775756836,
      "activations/layer11_attention_weight_min": -28.29500389099121,
      "activations/layer12_attention_weight_max": 22.828489303588867,
      "activations/layer12_attention_weight_min": -25.185699462890625,
      "activations/layer13_attention_weight_max": 47.31538772583008,
      "activations/layer13_attention_weight_min": -34.30583953857422,
      "activations/layer14_attention_weight_max": 43.56562042236328,
      "activations/layer14_attention_weight_min": -30.071657180786133,
      "activations/layer15_attention_weight_max": 37.32503890991211,
      "activations/layer15_attention_weight_min": -30.714435577392578,
      "activations/layer16_attention_weight_max": 37.56524658203125,
      "activations/layer16_attention_weight_min": -25.43703842163086,
      "activations/layer17_attention_weight_max": 61.45583724975586,
      "activations/layer17_attention_weight_min": -42.32585525512695,
      "activations/layer18_attention_weight_max": 53.27033615112305,
      "activations/layer18_attention_weight_min": -34.54754638671875,
      "activations/layer19_attention_weight_max": 24.263450622558594,
      "activations/layer19_attention_weight_min": -20.73274803161621,
      "activations/layer1_attention_weight_max": 18.1660099029541,
      "activations/layer1_attention_weight_min": -14.519742012023926,
      "activations/layer20_attention_weight_max": 22.545530319213867,
      "activations/layer20_attention_weight_min": -18.99279022216797,
      "activations/layer21_attention_weight_max": 38.68278884887695,
      "activations/layer21_attention_weight_min": -24.423486709594727,
      "activations/layer22_attention_weight_max": 33.897037506103516,
      "activations/layer22_attention_weight_min": -24.45311737060547,
      "activations/layer23_attention_weight_max": 32.469242095947266,
      "activations/layer23_attention_weight_min": -18.75960922241211,
      "activations/layer2_attention_weight_max": 31.100780487060547,
      "activations/layer2_attention_weight_min": -29.330101013183594,
      "activations/layer3_attention_weight_max": 79.96452331542969,
      "activations/layer3_attention_weight_min": -83.51607513427734,
      "activations/layer4_attention_weight_max": 96.38313293457031,
      "activations/layer4_attention_weight_min": -91.68944549560547,
      "activations/layer5_attention_weight_max": 92.46238708496094,
      "activations/layer5_attention_weight_min": -86.25706481933594,
      "activations/layer6_attention_weight_max": 57.5576057434082,
      "activations/layer6_attention_weight_min": -50.80979537963867,
      "activations/layer7_attention_weight_max": 73.62357330322266,
      "activations/layer7_attention_weight_min": -72.98416900634766,
      "activations/layer8_attention_weight_max": 42.18003845214844,
      "activations/layer8_attention_weight_min": -43.654083251953125,
      "activations/layer9_attention_weight_max": 52.18209457397461,
      "activations/layer9_attention_weight_min": -47.32413864135742,
      "epoch": 5.83,
      "learning_rate": 0.00011354090909090908,
      "loss": 2.8519,
      "step": 100350
    },
    {
      "activations/layer0_attention_weight_max": 14.463714599609375,
      "activations/layer0_attention_weight_min": -14.342982292175293,
      "activations/layer10_attention_weight_max": 41.986541748046875,
      "activations/layer10_attention_weight_min": -40.9117431640625,
      "activations/layer11_attention_weight_max": 38.76198196411133,
      "activations/layer11_attention_weight_min": -31.003202438354492,
      "activations/layer12_attention_weight_max": 21.69204330444336,
      "activations/layer12_attention_weight_min": -24.319799423217773,
      "activations/layer13_attention_weight_max": 71.6227798461914,
      "activations/layer13_attention_weight_min": -42.62568283081055,
      "activations/layer14_attention_weight_max": 89.22097778320312,
      "activations/layer14_attention_weight_min": -47.81058883666992,
      "activations/layer15_attention_weight_max": 66.62968444824219,
      "activations/layer15_attention_weight_min": -39.02363586425781,
      "activations/layer16_attention_weight_max": 35.13786315917969,
      "activations/layer16_attention_weight_min": -27.26769256591797,
      "activations/layer17_attention_weight_max": 57.46611404418945,
      "activations/layer17_attention_weight_min": -41.893577575683594,
      "activations/layer18_attention_weight_max": 52.76160430908203,
      "activations/layer18_attention_weight_min": -37.35837936401367,
      "activations/layer19_attention_weight_max": 21.675439834594727,
      "activations/layer19_attention_weight_min": -21.1961612701416,
      "activations/layer1_attention_weight_max": 17.96261215209961,
      "activations/layer1_attention_weight_min": -15.444052696228027,
      "activations/layer20_attention_weight_max": 26.31008529663086,
      "activations/layer20_attention_weight_min": -22.05243492126465,
      "activations/layer21_attention_weight_max": 44.98592758178711,
      "activations/layer21_attention_weight_min": -25.01512908935547,
      "activations/layer22_attention_weight_max": 40.504451751708984,
      "activations/layer22_attention_weight_min": -26.762895584106445,
      "activations/layer23_attention_weight_max": 31.920398712158203,
      "activations/layer23_attention_weight_min": -19.666170120239258,
      "activations/layer2_attention_weight_max": 29.786163330078125,
      "activations/layer2_attention_weight_min": -29.696317672729492,
      "activations/layer3_attention_weight_max": 82.47235870361328,
      "activations/layer3_attention_weight_min": -89.39402770996094,
      "activations/layer4_attention_weight_max": 102.3886947631836,
      "activations/layer4_attention_weight_min": -103.24751281738281,
      "activations/layer5_attention_weight_max": 100.79916381835938,
      "activations/layer5_attention_weight_min": -93.43655395507812,
      "activations/layer6_attention_weight_max": 58.21345520019531,
      "activations/layer6_attention_weight_min": -56.0065803527832,
      "activations/layer7_attention_weight_max": 80.4062728881836,
      "activations/layer7_attention_weight_min": -73.0317153930664,
      "activations/layer8_attention_weight_max": 51.209381103515625,
      "activations/layer8_attention_weight_min": -46.130165100097656,
      "activations/layer9_attention_weight_max": 67.74512481689453,
      "activations/layer9_attention_weight_min": -54.10188674926758,
      "epoch": 5.83,
      "learning_rate": 0.0001135219696969697,
      "loss": 2.8392,
      "step": 100400
    },
    {
      "activations/layer0_attention_weight_max": 15.262286186218262,
      "activations/layer0_attention_weight_min": -14.846399307250977,
      "activations/layer10_attention_weight_max": 38.11075973510742,
      "activations/layer10_attention_weight_min": -34.582298278808594,
      "activations/layer11_attention_weight_max": 28.853300094604492,
      "activations/layer11_attention_weight_min": -27.493263244628906,
      "activations/layer12_attention_weight_max": 20.663330078125,
      "activations/layer12_attention_weight_min": -25.21627426147461,
      "activations/layer13_attention_weight_max": 56.309486389160156,
      "activations/layer13_attention_weight_min": -38.76112747192383,
      "activations/layer14_attention_weight_max": 49.380279541015625,
      "activations/layer14_attention_weight_min": -31.955381393432617,
      "activations/layer15_attention_weight_max": 41.72593307495117,
      "activations/layer15_attention_weight_min": -30.595491409301758,
      "activations/layer16_attention_weight_max": 35.630924224853516,
      "activations/layer16_attention_weight_min": -27.960405349731445,
      "activations/layer17_attention_weight_max": 56.253868103027344,
      "activations/layer17_attention_weight_min": -40.91859436035156,
      "activations/layer18_attention_weight_max": 49.37461471557617,
      "activations/layer18_attention_weight_min": -36.4871711730957,
      "activations/layer19_attention_weight_max": 21.16080093383789,
      "activations/layer19_attention_weight_min": -24.483945846557617,
      "activations/layer1_attention_weight_max": 19.23753547668457,
      "activations/layer1_attention_weight_min": -16.25480842590332,
      "activations/layer20_attention_weight_max": 22.267471313476562,
      "activations/layer20_attention_weight_min": -22.756650924682617,
      "activations/layer21_attention_weight_max": 44.90232467651367,
      "activations/layer21_attention_weight_min": -27.57802391052246,
      "activations/layer22_attention_weight_max": 37.15391159057617,
      "activations/layer22_attention_weight_min": -25.62896156311035,
      "activations/layer23_attention_weight_max": 29.113807678222656,
      "activations/layer23_attention_weight_min": -19.48293685913086,
      "activations/layer2_attention_weight_max": 30.148420333862305,
      "activations/layer2_attention_weight_min": -27.519208908081055,
      "activations/layer3_attention_weight_max": 82.82574462890625,
      "activations/layer3_attention_weight_min": -79.4575424194336,
      "activations/layer4_attention_weight_max": 100.4168930053711,
      "activations/layer4_attention_weight_min": -90.32298278808594,
      "activations/layer5_attention_weight_max": 93.64949798583984,
      "activations/layer5_attention_weight_min": -83.42030334472656,
      "activations/layer6_attention_weight_max": 56.83094787597656,
      "activations/layer6_attention_weight_min": -49.239723205566406,
      "activations/layer7_attention_weight_max": 71.18555450439453,
      "activations/layer7_attention_weight_min": -66.6170654296875,
      "activations/layer8_attention_weight_max": 43.232025146484375,
      "activations/layer8_attention_weight_min": -41.1132926940918,
      "activations/layer9_attention_weight_max": 50.966556549072266,
      "activations/layer9_attention_weight_min": -45.429622650146484,
      "epoch": 5.84,
      "learning_rate": 0.0001135030303030303,
      "loss": 2.8416,
      "step": 100450
    },
    {
      "activations/layer0_attention_weight_max": 14.892486572265625,
      "activations/layer0_attention_weight_min": -14.440892219543457,
      "activations/layer10_attention_weight_max": 38.7615852355957,
      "activations/layer10_attention_weight_min": -34.81026840209961,
      "activations/layer11_attention_weight_max": 33.95566177368164,
      "activations/layer11_attention_weight_min": -31.566192626953125,
      "activations/layer12_attention_weight_max": 36.94063949584961,
      "activations/layer12_attention_weight_min": -27.755130767822266,
      "activations/layer13_attention_weight_max": 51.13667678833008,
      "activations/layer13_attention_weight_min": -35.371986389160156,
      "activations/layer14_attention_weight_max": 39.100704193115234,
      "activations/layer14_attention_weight_min": -29.858285903930664,
      "activations/layer15_attention_weight_max": 39.08573532104492,
      "activations/layer15_attention_weight_min": -30.369840621948242,
      "activations/layer16_attention_weight_max": 34.10304641723633,
      "activations/layer16_attention_weight_min": -26.351408004760742,
      "activations/layer17_attention_weight_max": 59.61568832397461,
      "activations/layer17_attention_weight_min": -41.91704177856445,
      "activations/layer18_attention_weight_max": 55.94263458251953,
      "activations/layer18_attention_weight_min": -37.198707580566406,
      "activations/layer19_attention_weight_max": 25.963459014892578,
      "activations/layer19_attention_weight_min": -22.182104110717773,
      "activations/layer1_attention_weight_max": 16.938373565673828,
      "activations/layer1_attention_weight_min": -16.62833595275879,
      "activations/layer20_attention_weight_max": 22.315486907958984,
      "activations/layer20_attention_weight_min": -25.239267349243164,
      "activations/layer21_attention_weight_max": 39.078216552734375,
      "activations/layer21_attention_weight_min": -28.867345809936523,
      "activations/layer22_attention_weight_max": 33.56766891479492,
      "activations/layer22_attention_weight_min": -24.23358917236328,
      "activations/layer23_attention_weight_max": 28.07418441772461,
      "activations/layer23_attention_weight_min": -18.584815979003906,
      "activations/layer2_attention_weight_max": 30.59193229675293,
      "activations/layer2_attention_weight_min": -29.208311080932617,
      "activations/layer3_attention_weight_max": 84.02108001708984,
      "activations/layer3_attention_weight_min": -86.72953033447266,
      "activations/layer4_attention_weight_max": 102.2907485961914,
      "activations/layer4_attention_weight_min": -97.60888671875,
      "activations/layer5_attention_weight_max": 100.44027709960938,
      "activations/layer5_attention_weight_min": -93.90803527832031,
      "activations/layer6_attention_weight_max": 61.5955924987793,
      "activations/layer6_attention_weight_min": -54.10770797729492,
      "activations/layer7_attention_weight_max": 73.15035247802734,
      "activations/layer7_attention_weight_min": -69.4774169921875,
      "activations/layer8_attention_weight_max": 45.38821029663086,
      "activations/layer8_attention_weight_min": -42.98024368286133,
      "activations/layer9_attention_weight_max": 52.08388900756836,
      "activations/layer9_attention_weight_min": -47.00015640258789,
      "epoch": 5.84,
      "learning_rate": 0.0001134840909090909,
      "loss": 2.8492,
      "step": 100500
    },
    {
      "activations/layer0_attention_weight_max": 14.064948081970215,
      "activations/layer0_attention_weight_min": -14.43320369720459,
      "activations/layer10_attention_weight_max": 38.652008056640625,
      "activations/layer10_attention_weight_min": -33.765228271484375,
      "activations/layer11_attention_weight_max": 29.338207244873047,
      "activations/layer11_attention_weight_min": -27.178142547607422,
      "activations/layer12_attention_weight_max": 20.757740020751953,
      "activations/layer12_attention_weight_min": -24.992937088012695,
      "activations/layer13_attention_weight_max": 48.674293518066406,
      "activations/layer13_attention_weight_min": -35.611873626708984,
      "activations/layer14_attention_weight_max": 43.488555908203125,
      "activations/layer14_attention_weight_min": -31.41730499267578,
      "activations/layer15_attention_weight_max": 38.091793060302734,
      "activations/layer15_attention_weight_min": -29.960128784179688,
      "activations/layer16_attention_weight_max": 32.59213638305664,
      "activations/layer16_attention_weight_min": -26.14267349243164,
      "activations/layer17_attention_weight_max": 53.92224884033203,
      "activations/layer17_attention_weight_min": -39.46192932128906,
      "activations/layer18_attention_weight_max": 45.98174285888672,
      "activations/layer18_attention_weight_min": -35.707820892333984,
      "activations/layer19_attention_weight_max": 24.709564208984375,
      "activations/layer19_attention_weight_min": -19.140439987182617,
      "activations/layer1_attention_weight_max": 17.399028778076172,
      "activations/layer1_attention_weight_min": -13.691120147705078,
      "activations/layer20_attention_weight_max": 21.23517417907715,
      "activations/layer20_attention_weight_min": -22.79404640197754,
      "activations/layer21_attention_weight_max": 36.16657257080078,
      "activations/layer21_attention_weight_min": -26.881376266479492,
      "activations/layer22_attention_weight_max": 31.39813804626465,
      "activations/layer22_attention_weight_min": -24.621320724487305,
      "activations/layer23_attention_weight_max": 29.225494384765625,
      "activations/layer23_attention_weight_min": -19.177705764770508,
      "activations/layer2_attention_weight_max": 29.720348358154297,
      "activations/layer2_attention_weight_min": -28.707202911376953,
      "activations/layer3_attention_weight_max": 82.12144470214844,
      "activations/layer3_attention_weight_min": -85.50263214111328,
      "activations/layer4_attention_weight_max": 98.40153503417969,
      "activations/layer4_attention_weight_min": -90.91691589355469,
      "activations/layer5_attention_weight_max": 94.47783660888672,
      "activations/layer5_attention_weight_min": -84.20531463623047,
      "activations/layer6_attention_weight_max": 55.63591384887695,
      "activations/layer6_attention_weight_min": -52.641849517822266,
      "activations/layer7_attention_weight_max": 68.87496948242188,
      "activations/layer7_attention_weight_min": -70.4420394897461,
      "activations/layer8_attention_weight_max": 42.40467071533203,
      "activations/layer8_attention_weight_min": -41.015296936035156,
      "activations/layer9_attention_weight_max": 48.41935348510742,
      "activations/layer9_attention_weight_min": -45.019920349121094,
      "epoch": 5.84,
      "learning_rate": 0.00011346515151515151,
      "loss": 2.8413,
      "step": 100550
    },
    {
      "activations/layer0_attention_weight_max": 14.181390762329102,
      "activations/layer0_attention_weight_min": -14.889195442199707,
      "activations/layer10_attention_weight_max": 41.545196533203125,
      "activations/layer10_attention_weight_min": -35.534603118896484,
      "activations/layer11_attention_weight_max": 32.55598449707031,
      "activations/layer11_attention_weight_min": -28.0850830078125,
      "activations/layer12_attention_weight_max": 23.003007888793945,
      "activations/layer12_attention_weight_min": -25.68496322631836,
      "activations/layer13_attention_weight_max": 47.94449996948242,
      "activations/layer13_attention_weight_min": -35.19091033935547,
      "activations/layer14_attention_weight_max": 43.77076721191406,
      "activations/layer14_attention_weight_min": -31.756221771240234,
      "activations/layer15_attention_weight_max": 42.84828567504883,
      "activations/layer15_attention_weight_min": -28.140037536621094,
      "activations/layer16_attention_weight_max": 38.72590637207031,
      "activations/layer16_attention_weight_min": -26.036962509155273,
      "activations/layer17_attention_weight_max": 61.44600296020508,
      "activations/layer17_attention_weight_min": -41.71010971069336,
      "activations/layer18_attention_weight_max": 56.2077522277832,
      "activations/layer18_attention_weight_min": -34.111480712890625,
      "activations/layer19_attention_weight_max": 24.844215393066406,
      "activations/layer19_attention_weight_min": -20.335302352905273,
      "activations/layer1_attention_weight_max": 17.351587295532227,
      "activations/layer1_attention_weight_min": -15.520212173461914,
      "activations/layer20_attention_weight_max": 25.533056259155273,
      "activations/layer20_attention_weight_min": -22.394346237182617,
      "activations/layer21_attention_weight_max": 46.98151397705078,
      "activations/layer21_attention_weight_min": -25.856884002685547,
      "activations/layer22_attention_weight_max": 37.81296920776367,
      "activations/layer22_attention_weight_min": -24.682147979736328,
      "activations/layer23_attention_weight_max": 29.90079116821289,
      "activations/layer23_attention_weight_min": -19.052478790283203,
      "activations/layer2_attention_weight_max": 31.72690200805664,
      "activations/layer2_attention_weight_min": -30.542160034179688,
      "activations/layer3_attention_weight_max": 82.46378326416016,
      "activations/layer3_attention_weight_min": -82.08343505859375,
      "activations/layer4_attention_weight_max": 101.84615325927734,
      "activations/layer4_attention_weight_min": -94.6392822265625,
      "activations/layer5_attention_weight_max": 103.7457504272461,
      "activations/layer5_attention_weight_min": -90.28041076660156,
      "activations/layer6_attention_weight_max": 63.767005920410156,
      "activations/layer6_attention_weight_min": -52.90833282470703,
      "activations/layer7_attention_weight_max": 76.56037902832031,
      "activations/layer7_attention_weight_min": -72.99447631835938,
      "activations/layer8_attention_weight_max": 51.41016387939453,
      "activations/layer8_attention_weight_min": -45.16565704345703,
      "activations/layer9_attention_weight_max": 59.62714385986328,
      "activations/layer9_attention_weight_min": -54.889076232910156,
      "epoch": 5.85,
      "learning_rate": 0.0001134462121212121,
      "loss": 2.8428,
      "step": 100600
    },
    {
      "activations/layer0_attention_weight_max": 14.839502334594727,
      "activations/layer0_attention_weight_min": -14.722373962402344,
      "activations/layer10_attention_weight_max": 39.874446868896484,
      "activations/layer10_attention_weight_min": -35.81245803833008,
      "activations/layer11_attention_weight_max": 30.078636169433594,
      "activations/layer11_attention_weight_min": -29.242460250854492,
      "activations/layer12_attention_weight_max": 25.626014709472656,
      "activations/layer12_attention_weight_min": -23.162113189697266,
      "activations/layer13_attention_weight_max": 48.22193908691406,
      "activations/layer13_attention_weight_min": -32.29225540161133,
      "activations/layer14_attention_weight_max": 44.75958251953125,
      "activations/layer14_attention_weight_min": -29.8800106048584,
      "activations/layer15_attention_weight_max": 42.54741668701172,
      "activations/layer15_attention_weight_min": -29.468338012695312,
      "activations/layer16_attention_weight_max": 33.646644592285156,
      "activations/layer16_attention_weight_min": -27.593637466430664,
      "activations/layer17_attention_weight_max": 61.28094482421875,
      "activations/layer17_attention_weight_min": -41.08900833129883,
      "activations/layer18_attention_weight_max": 54.83625411987305,
      "activations/layer18_attention_weight_min": -34.736751556396484,
      "activations/layer19_attention_weight_max": 24.746509552001953,
      "activations/layer19_attention_weight_min": -20.5655517578125,
      "activations/layer1_attention_weight_max": 17.282516479492188,
      "activations/layer1_attention_weight_min": -16.10041618347168,
      "activations/layer20_attention_weight_max": 20.874561309814453,
      "activations/layer20_attention_weight_min": -18.991233825683594,
      "activations/layer21_attention_weight_max": 36.343353271484375,
      "activations/layer21_attention_weight_min": -25.316925048828125,
      "activations/layer22_attention_weight_max": 31.180044174194336,
      "activations/layer22_attention_weight_min": -24.758577346801758,
      "activations/layer23_attention_weight_max": 31.47469139099121,
      "activations/layer23_attention_weight_min": -19.305572509765625,
      "activations/layer2_attention_weight_max": 31.168495178222656,
      "activations/layer2_attention_weight_min": -30.35897445678711,
      "activations/layer3_attention_weight_max": 86.6240005493164,
      "activations/layer3_attention_weight_min": -87.4563217163086,
      "activations/layer4_attention_weight_max": 99.33330535888672,
      "activations/layer4_attention_weight_min": -95.52615356445312,
      "activations/layer5_attention_weight_max": 94.8087387084961,
      "activations/layer5_attention_weight_min": -88.2808837890625,
      "activations/layer6_attention_weight_max": 57.97478485107422,
      "activations/layer6_attention_weight_min": -51.683998107910156,
      "activations/layer7_attention_weight_max": 75.6099853515625,
      "activations/layer7_attention_weight_min": -69.51280975341797,
      "activations/layer8_attention_weight_max": 46.46881866455078,
      "activations/layer8_attention_weight_min": -44.309993743896484,
      "activations/layer9_attention_weight_max": 55.08877182006836,
      "activations/layer9_attention_weight_min": -47.47236251831055,
      "epoch": 5.85,
      "learning_rate": 0.00011342727272727271,
      "loss": 2.844,
      "step": 100650
    },
    {
      "activations/layer0_attention_weight_max": 13.572529792785645,
      "activations/layer0_attention_weight_min": -15.041550636291504,
      "activations/layer10_attention_weight_max": 40.81658935546875,
      "activations/layer10_attention_weight_min": -35.43342971801758,
      "activations/layer11_attention_weight_max": 31.502155303955078,
      "activations/layer11_attention_weight_min": -29.038503646850586,
      "activations/layer12_attention_weight_max": 21.48053741455078,
      "activations/layer12_attention_weight_min": -25.1028995513916,
      "activations/layer13_attention_weight_max": 48.1882209777832,
      "activations/layer13_attention_weight_min": -35.35551071166992,
      "activations/layer14_attention_weight_max": 41.6387939453125,
      "activations/layer14_attention_weight_min": -31.649436950683594,
      "activations/layer15_attention_weight_max": 42.959537506103516,
      "activations/layer15_attention_weight_min": -31.33062744140625,
      "activations/layer16_attention_weight_max": 31.616846084594727,
      "activations/layer16_attention_weight_min": -28.40815544128418,
      "activations/layer17_attention_weight_max": 60.25074768066406,
      "activations/layer17_attention_weight_min": -40.952171325683594,
      "activations/layer18_attention_weight_max": 51.49951934814453,
      "activations/layer18_attention_weight_min": -35.779327392578125,
      "activations/layer19_attention_weight_max": 25.00062370300293,
      "activations/layer19_attention_weight_min": -20.354585647583008,
      "activations/layer1_attention_weight_max": 17.33799934387207,
      "activations/layer1_attention_weight_min": -15.170147895812988,
      "activations/layer20_attention_weight_max": 22.6474609375,
      "activations/layer20_attention_weight_min": -22.753429412841797,
      "activations/layer21_attention_weight_max": 41.7176628112793,
      "activations/layer21_attention_weight_min": -26.666976928710938,
      "activations/layer22_attention_weight_max": 34.04059982299805,
      "activations/layer22_attention_weight_min": -25.835474014282227,
      "activations/layer23_attention_weight_max": 32.74611282348633,
      "activations/layer23_attention_weight_min": -18.917190551757812,
      "activations/layer2_attention_weight_max": 31.301010131835938,
      "activations/layer2_attention_weight_min": -29.797666549682617,
      "activations/layer3_attention_weight_max": 84.84407043457031,
      "activations/layer3_attention_weight_min": -88.20599365234375,
      "activations/layer4_attention_weight_max": 101.4313735961914,
      "activations/layer4_attention_weight_min": -99.97505950927734,
      "activations/layer5_attention_weight_max": 97.07553100585938,
      "activations/layer5_attention_weight_min": -92.00849914550781,
      "activations/layer6_attention_weight_max": 59.685150146484375,
      "activations/layer6_attention_weight_min": -54.36708068847656,
      "activations/layer7_attention_weight_max": 73.60856628417969,
      "activations/layer7_attention_weight_min": -72.34911346435547,
      "activations/layer8_attention_weight_max": 47.184879302978516,
      "activations/layer8_attention_weight_min": -48.52030563354492,
      "activations/layer9_attention_weight_max": 49.555877685546875,
      "activations/layer9_attention_weight_min": -52.54960250854492,
      "epoch": 5.85,
      "learning_rate": 0.00011340833333333333,
      "loss": 2.8277,
      "step": 100700
    },
    {
      "activations/layer0_attention_weight_max": 14.7167387008667,
      "activations/layer0_attention_weight_min": -14.799840927124023,
      "activations/layer10_attention_weight_max": 38.34326171875,
      "activations/layer10_attention_weight_min": -33.026878356933594,
      "activations/layer11_attention_weight_max": 30.830669403076172,
      "activations/layer11_attention_weight_min": -27.228071212768555,
      "activations/layer12_attention_weight_max": 22.09457778930664,
      "activations/layer12_attention_weight_min": -27.348678588867188,
      "activations/layer13_attention_weight_max": 48.30436706542969,
      "activations/layer13_attention_weight_min": -34.02700424194336,
      "activations/layer14_attention_weight_max": 66.4464340209961,
      "activations/layer14_attention_weight_min": -38.587520599365234,
      "activations/layer15_attention_weight_max": 47.8177490234375,
      "activations/layer15_attention_weight_min": -31.19850730895996,
      "activations/layer16_attention_weight_max": 34.58518981933594,
      "activations/layer16_attention_weight_min": -28.198829650878906,
      "activations/layer17_attention_weight_max": 55.28838348388672,
      "activations/layer17_attention_weight_min": -38.65940475463867,
      "activations/layer18_attention_weight_max": 52.15188980102539,
      "activations/layer18_attention_weight_min": -34.86735916137695,
      "activations/layer19_attention_weight_max": 24.952741622924805,
      "activations/layer19_attention_weight_min": -19.616788864135742,
      "activations/layer1_attention_weight_max": 17.16604232788086,
      "activations/layer1_attention_weight_min": -15.846112251281738,
      "activations/layer20_attention_weight_max": 25.3198184967041,
      "activations/layer20_attention_weight_min": -18.950838088989258,
      "activations/layer21_attention_weight_max": 47.372161865234375,
      "activations/layer21_attention_weight_min": -25.60219383239746,
      "activations/layer22_attention_weight_max": 36.96592330932617,
      "activations/layer22_attention_weight_min": -23.775110244750977,
      "activations/layer23_attention_weight_max": 33.89372253417969,
      "activations/layer23_attention_weight_min": -20.724308013916016,
      "activations/layer2_attention_weight_max": 31.370328903198242,
      "activations/layer2_attention_weight_min": -29.664554595947266,
      "activations/layer3_attention_weight_max": 85.1609878540039,
      "activations/layer3_attention_weight_min": -90.59891510009766,
      "activations/layer4_attention_weight_max": 99.93551635742188,
      "activations/layer4_attention_weight_min": -94.43765258789062,
      "activations/layer5_attention_weight_max": 95.29884338378906,
      "activations/layer5_attention_weight_min": -87.25926208496094,
      "activations/layer6_attention_weight_max": 57.001529693603516,
      "activations/layer6_attention_weight_min": -52.007137298583984,
      "activations/layer7_attention_weight_max": 70.49470520019531,
      "activations/layer7_attention_weight_min": -67.49070739746094,
      "activations/layer8_attention_weight_max": 43.130001068115234,
      "activations/layer8_attention_weight_min": -41.39008712768555,
      "activations/layer9_attention_weight_max": 48.878028869628906,
      "activations/layer9_attention_weight_min": -46.04280090332031,
      "epoch": 5.85,
      "learning_rate": 0.00011338939393939392,
      "loss": 2.8525,
      "step": 100750
    },
    {
      "activations/layer0_attention_weight_max": 15.849641799926758,
      "activations/layer0_attention_weight_min": -14.846978187561035,
      "activations/layer10_attention_weight_max": 44.175315856933594,
      "activations/layer10_attention_weight_min": -36.33257293701172,
      "activations/layer11_attention_weight_max": 34.63838195800781,
      "activations/layer11_attention_weight_min": -29.972204208374023,
      "activations/layer12_attention_weight_max": 24.12152099609375,
      "activations/layer12_attention_weight_min": -26.680192947387695,
      "activations/layer13_attention_weight_max": 77.69014739990234,
      "activations/layer13_attention_weight_min": -34.67169952392578,
      "activations/layer14_attention_weight_max": 82.99879455566406,
      "activations/layer14_attention_weight_min": -35.45326232910156,
      "activations/layer15_attention_weight_max": 56.57784652709961,
      "activations/layer15_attention_weight_min": -29.031490325927734,
      "activations/layer16_attention_weight_max": 38.35231399536133,
      "activations/layer16_attention_weight_min": -28.359254837036133,
      "activations/layer17_attention_weight_max": 60.53287124633789,
      "activations/layer17_attention_weight_min": -40.1176643371582,
      "activations/layer18_attention_weight_max": 50.179412841796875,
      "activations/layer18_attention_weight_min": -34.438385009765625,
      "activations/layer19_attention_weight_max": 24.172853469848633,
      "activations/layer19_attention_weight_min": -19.647024154663086,
      "activations/layer1_attention_weight_max": 18.136775970458984,
      "activations/layer1_attention_weight_min": -16.377826690673828,
      "activations/layer20_attention_weight_max": 26.371685028076172,
      "activations/layer20_attention_weight_min": -20.062969207763672,
      "activations/layer21_attention_weight_max": 44.334381103515625,
      "activations/layer21_attention_weight_min": -25.833847045898438,
      "activations/layer22_attention_weight_max": 32.98625183105469,
      "activations/layer22_attention_weight_min": -25.77793312072754,
      "activations/layer23_attention_weight_max": 28.417163848876953,
      "activations/layer23_attention_weight_min": -18.286026000976562,
      "activations/layer2_attention_weight_max": 34.753963470458984,
      "activations/layer2_attention_weight_min": -31.800819396972656,
      "activations/layer3_attention_weight_max": 90.64054107666016,
      "activations/layer3_attention_weight_min": -92.40435791015625,
      "activations/layer4_attention_weight_max": 113.23760223388672,
      "activations/layer4_attention_weight_min": -106.719482421875,
      "activations/layer5_attention_weight_max": 103.99199676513672,
      "activations/layer5_attention_weight_min": -92.97431945800781,
      "activations/layer6_attention_weight_max": 57.63645935058594,
      "activations/layer6_attention_weight_min": -53.45098114013672,
      "activations/layer7_attention_weight_max": 87.44405364990234,
      "activations/layer7_attention_weight_min": -73.40158081054688,
      "activations/layer8_attention_weight_max": 51.89136505126953,
      "activations/layer8_attention_weight_min": -43.031898498535156,
      "activations/layer9_attention_weight_max": 69.80386352539062,
      "activations/layer9_attention_weight_min": -53.476356506347656,
      "epoch": 5.86,
      "learning_rate": 0.00011337045454545453,
      "loss": 2.8342,
      "step": 100800
    },
    {
      "activations/layer0_attention_weight_max": 14.309601783752441,
      "activations/layer0_attention_weight_min": -15.072327613830566,
      "activations/layer10_attention_weight_max": 40.79010009765625,
      "activations/layer10_attention_weight_min": -34.74521255493164,
      "activations/layer11_attention_weight_max": 32.06891632080078,
      "activations/layer11_attention_weight_min": -28.673595428466797,
      "activations/layer12_attention_weight_max": 22.73590660095215,
      "activations/layer12_attention_weight_min": -24.76129150390625,
      "activations/layer13_attention_weight_max": 43.90414047241211,
      "activations/layer13_attention_weight_min": -34.07943344116211,
      "activations/layer14_attention_weight_max": 40.65260314941406,
      "activations/layer14_attention_weight_min": -30.026948928833008,
      "activations/layer15_attention_weight_max": 40.31262969970703,
      "activations/layer15_attention_weight_min": -30.961774826049805,
      "activations/layer16_attention_weight_max": 37.180152893066406,
      "activations/layer16_attention_weight_min": -29.218517303466797,
      "activations/layer17_attention_weight_max": 55.431392669677734,
      "activations/layer17_attention_weight_min": -40.411617279052734,
      "activations/layer18_attention_weight_max": 48.41224670410156,
      "activations/layer18_attention_weight_min": -38.1965446472168,
      "activations/layer19_attention_weight_max": 22.47471046447754,
      "activations/layer19_attention_weight_min": -20.53380012512207,
      "activations/layer1_attention_weight_max": 17.209291458129883,
      "activations/layer1_attention_weight_min": -15.192407608032227,
      "activations/layer20_attention_weight_max": 22.837261199951172,
      "activations/layer20_attention_weight_min": -21.004039764404297,
      "activations/layer21_attention_weight_max": 44.588226318359375,
      "activations/layer21_attention_weight_min": -28.899810791015625,
      "activations/layer22_attention_weight_max": 35.078609466552734,
      "activations/layer22_attention_weight_min": -25.662824630737305,
      "activations/layer23_attention_weight_max": 31.996641159057617,
      "activations/layer23_attention_weight_min": -18.82339096069336,
      "activations/layer2_attention_weight_max": 30.009521484375,
      "activations/layer2_attention_weight_min": -28.747913360595703,
      "activations/layer3_attention_weight_max": 80.22108459472656,
      "activations/layer3_attention_weight_min": -84.60551452636719,
      "activations/layer4_attention_weight_max": 100.0954818725586,
      "activations/layer4_attention_weight_min": -95.5711898803711,
      "activations/layer5_attention_weight_max": 97.09944915771484,
      "activations/layer5_attention_weight_min": -89.58657836914062,
      "activations/layer6_attention_weight_max": 59.26902770996094,
      "activations/layer6_attention_weight_min": -53.261234283447266,
      "activations/layer7_attention_weight_max": 77.45800018310547,
      "activations/layer7_attention_weight_min": -66.09736633300781,
      "activations/layer8_attention_weight_max": 45.1685905456543,
      "activations/layer8_attention_weight_min": -43.455020904541016,
      "activations/layer9_attention_weight_max": 50.46780014038086,
      "activations/layer9_attention_weight_min": -48.25074768066406,
      "epoch": 5.86,
      "learning_rate": 0.00011335151515151515,
      "loss": 2.8313,
      "step": 100850
    },
    {
      "activations/layer0_attention_weight_max": 15.371718406677246,
      "activations/layer0_attention_weight_min": -14.855830192565918,
      "activations/layer10_attention_weight_max": 42.8991813659668,
      "activations/layer10_attention_weight_min": -37.38971710205078,
      "activations/layer11_attention_weight_max": 33.69746780395508,
      "activations/layer11_attention_weight_min": -29.68647003173828,
      "activations/layer12_attention_weight_max": 27.557449340820312,
      "activations/layer12_attention_weight_min": -28.726781845092773,
      "activations/layer13_attention_weight_max": 62.996849060058594,
      "activations/layer13_attention_weight_min": -39.79961013793945,
      "activations/layer14_attention_weight_max": 59.33025360107422,
      "activations/layer14_attention_weight_min": -31.592439651489258,
      "activations/layer15_attention_weight_max": 50.7939338684082,
      "activations/layer15_attention_weight_min": -33.449790954589844,
      "activations/layer16_attention_weight_max": 51.74641418457031,
      "activations/layer16_attention_weight_min": -28.833904266357422,
      "activations/layer17_attention_weight_max": 67.08048248291016,
      "activations/layer17_attention_weight_min": -45.83456039428711,
      "activations/layer18_attention_weight_max": 60.1737060546875,
      "activations/layer18_attention_weight_min": -35.91511154174805,
      "activations/layer19_attention_weight_max": 29.602121353149414,
      "activations/layer19_attention_weight_min": -21.597301483154297,
      "activations/layer1_attention_weight_max": 16.74687957763672,
      "activations/layer1_attention_weight_min": -15.165868759155273,
      "activations/layer20_attention_weight_max": 33.893394470214844,
      "activations/layer20_attention_weight_min": -21.621612548828125,
      "activations/layer21_attention_weight_max": 64.29161071777344,
      "activations/layer21_attention_weight_min": -27.58885955810547,
      "activations/layer22_attention_weight_max": 43.81983947753906,
      "activations/layer22_attention_weight_min": -27.252582550048828,
      "activations/layer23_attention_weight_max": 41.68412780761719,
      "activations/layer23_attention_weight_min": -18.96225357055664,
      "activations/layer2_attention_weight_max": 29.82834243774414,
      "activations/layer2_attention_weight_min": -27.951560974121094,
      "activations/layer3_attention_weight_max": 84.29330444335938,
      "activations/layer3_attention_weight_min": -82.12918853759766,
      "activations/layer4_attention_weight_max": 99.6180648803711,
      "activations/layer4_attention_weight_min": -95.26036071777344,
      "activations/layer5_attention_weight_max": 93.5359115600586,
      "activations/layer5_attention_weight_min": -87.99958801269531,
      "activations/layer6_attention_weight_max": 57.29634094238281,
      "activations/layer6_attention_weight_min": -53.874446868896484,
      "activations/layer7_attention_weight_max": 78.92416381835938,
      "activations/layer7_attention_weight_min": -72.80362701416016,
      "activations/layer8_attention_weight_max": 50.180965423583984,
      "activations/layer8_attention_weight_min": -53.0354118347168,
      "activations/layer9_attention_weight_max": 59.56781005859375,
      "activations/layer9_attention_weight_min": -58.091949462890625,
      "epoch": 5.86,
      "learning_rate": 0.00011333257575757575,
      "loss": 2.8241,
      "step": 100900
    },
    {
      "activations/layer0_attention_weight_max": 15.862082481384277,
      "activations/layer0_attention_weight_min": -14.89986515045166,
      "activations/layer10_attention_weight_max": 40.991458892822266,
      "activations/layer10_attention_weight_min": -38.86804962158203,
      "activations/layer11_attention_weight_max": 31.29121971130371,
      "activations/layer11_attention_weight_min": -31.535945892333984,
      "activations/layer12_attention_weight_max": 25.758386611938477,
      "activations/layer12_attention_weight_min": -26.704477310180664,
      "activations/layer13_attention_weight_max": 48.4286003112793,
      "activations/layer13_attention_weight_min": -37.114280700683594,
      "activations/layer14_attention_weight_max": 44.74965286254883,
      "activations/layer14_attention_weight_min": -30.65365982055664,
      "activations/layer15_attention_weight_max": 40.54273986816406,
      "activations/layer15_attention_weight_min": -30.73972511291504,
      "activations/layer16_attention_weight_max": 43.397701263427734,
      "activations/layer16_attention_weight_min": -26.774656295776367,
      "activations/layer17_attention_weight_max": 59.10768508911133,
      "activations/layer17_attention_weight_min": -39.01115417480469,
      "activations/layer18_attention_weight_max": 49.32700729370117,
      "activations/layer18_attention_weight_min": -34.470359802246094,
      "activations/layer19_attention_weight_max": 23.61588478088379,
      "activations/layer19_attention_weight_min": -20.025556564331055,
      "activations/layer1_attention_weight_max": 17.725887298583984,
      "activations/layer1_attention_weight_min": -15.74254322052002,
      "activations/layer20_attention_weight_max": 25.084325790405273,
      "activations/layer20_attention_weight_min": -20.24506950378418,
      "activations/layer21_attention_weight_max": 41.7283935546875,
      "activations/layer21_attention_weight_min": -28.1672420501709,
      "activations/layer22_attention_weight_max": 36.89522171020508,
      "activations/layer22_attention_weight_min": -23.65480613708496,
      "activations/layer23_attention_weight_max": 31.19611930847168,
      "activations/layer23_attention_weight_min": -20.640674591064453,
      "activations/layer2_attention_weight_max": 29.82394790649414,
      "activations/layer2_attention_weight_min": -29.793058395385742,
      "activations/layer3_attention_weight_max": 78.69273376464844,
      "activations/layer3_attention_weight_min": -84.90243530273438,
      "activations/layer4_attention_weight_max": 100.20148468017578,
      "activations/layer4_attention_weight_min": -92.65222930908203,
      "activations/layer5_attention_weight_max": 101.49658203125,
      "activations/layer5_attention_weight_min": -90.04818725585938,
      "activations/layer6_attention_weight_max": 60.556480407714844,
      "activations/layer6_attention_weight_min": -57.8637809753418,
      "activations/layer7_attention_weight_max": 74.44462585449219,
      "activations/layer7_attention_weight_min": -76.14130401611328,
      "activations/layer8_attention_weight_max": 49.70591354370117,
      "activations/layer8_attention_weight_min": -46.21714401245117,
      "activations/layer9_attention_weight_max": 54.76485061645508,
      "activations/layer9_attention_weight_min": -50.42211151123047,
      "epoch": 5.87,
      "learning_rate": 0.00011331363636363635,
      "loss": 2.8315,
      "step": 100950
    },
    {
      "activations/layer0_attention_weight_max": 14.328020095825195,
      "activations/layer0_attention_weight_min": -15.054154396057129,
      "activations/layer10_attention_weight_max": 44.77522659301758,
      "activations/layer10_attention_weight_min": -35.45092010498047,
      "activations/layer11_attention_weight_max": 35.871910095214844,
      "activations/layer11_attention_weight_min": -28.735939025878906,
      "activations/layer12_attention_weight_max": 22.146095275878906,
      "activations/layer12_attention_weight_min": -27.70287322998047,
      "activations/layer13_attention_weight_max": 60.014076232910156,
      "activations/layer13_attention_weight_min": -41.0736198425293,
      "activations/layer14_attention_weight_max": 91.61844635009766,
      "activations/layer14_attention_weight_min": -48.86164093017578,
      "activations/layer15_attention_weight_max": 65.81035614013672,
      "activations/layer15_attention_weight_min": -37.11211395263672,
      "activations/layer16_attention_weight_max": 36.82547378540039,
      "activations/layer16_attention_weight_min": -26.057729721069336,
      "activations/layer17_attention_weight_max": 61.5068244934082,
      "activations/layer17_attention_weight_min": -41.85568618774414,
      "activations/layer18_attention_weight_max": 51.29971694946289,
      "activations/layer18_attention_weight_min": -35.46986389160156,
      "activations/layer19_attention_weight_max": 28.259357452392578,
      "activations/layer19_attention_weight_min": -21.412694931030273,
      "activations/layer1_attention_weight_max": 17.302433013916016,
      "activations/layer1_attention_weight_min": -14.140623092651367,
      "activations/layer20_attention_weight_max": 26.693618774414062,
      "activations/layer20_attention_weight_min": -20.74510955810547,
      "activations/layer21_attention_weight_max": 57.35553741455078,
      "activations/layer21_attention_weight_min": -25.747257232666016,
      "activations/layer22_attention_weight_max": 35.67350769042969,
      "activations/layer22_attention_weight_min": -25.54225730895996,
      "activations/layer23_attention_weight_max": 31.93901824951172,
      "activations/layer23_attention_weight_min": -20.654499053955078,
      "activations/layer2_attention_weight_max": 31.852697372436523,
      "activations/layer2_attention_weight_min": -28.790111541748047,
      "activations/layer3_attention_weight_max": 88.97018432617188,
      "activations/layer3_attention_weight_min": -88.80122375488281,
      "activations/layer4_attention_weight_max": 104.425048828125,
      "activations/layer4_attention_weight_min": -97.6264419555664,
      "activations/layer5_attention_weight_max": 100.11886596679688,
      "activations/layer5_attention_weight_min": -86.79669189453125,
      "activations/layer6_attention_weight_max": 58.362953186035156,
      "activations/layer6_attention_weight_min": -51.71758270263672,
      "activations/layer7_attention_weight_max": 84.38191986083984,
      "activations/layer7_attention_weight_min": -72.99907684326172,
      "activations/layer8_attention_weight_max": 51.94633865356445,
      "activations/layer8_attention_weight_min": -42.46516799926758,
      "activations/layer9_attention_weight_max": 71.1564712524414,
      "activations/layer9_attention_weight_min": -54.85408401489258,
      "epoch": 5.87,
      "learning_rate": 0.00011329469696969695,
      "loss": 2.8433,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.5561,
      "eval_samples_per_second": 501.867,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.5561,
      "eval_openwebtext_samples_per_second": 501.867,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_wikitext_loss": 3.017578125,
      "eval_wikitext_ppl": 20.44172438427552,
      "eval_wikitext_runtime": 2.0336,
      "eval_wikitext_samples_per_second": 224.235,
      "step": 101000
    },
    {
      "epoch": 5.87,
      "eval_lambada_loss": 2.79296875,
      "eval_lambada_ppl": 16.329425896569767,
      "eval_lambada_runtime": 9.5889,
      "eval_lambada_samples_per_second": 507.776,
      "step": 101000
    },
    {
      "activations/layer0_attention_weight_max": 14.690070152282715,
      "activations/layer0_attention_weight_min": -14.373432159423828,
      "activations/layer10_attention_weight_max": 34.23435592651367,
      "activations/layer10_attention_weight_min": -31.722246170043945,
      "activations/layer11_attention_weight_max": 29.84149742126465,
      "activations/layer11_attention_weight_min": -25.96145248413086,
      "activations/layer12_attention_weight_max": 23.997180938720703,
      "activations/layer12_attention_weight_min": -24.606658935546875,
      "activations/layer13_attention_weight_max": 45.0781135559082,
      "activations/layer13_attention_weight_min": -32.047088623046875,
      "activations/layer14_attention_weight_max": 42.14830780029297,
      "activations/layer14_attention_weight_min": -29.3096866607666,
      "activations/layer15_attention_weight_max": 38.3178596496582,
      "activations/layer15_attention_weight_min": -31.263208389282227,
      "activations/layer16_attention_weight_max": 33.01042556762695,
      "activations/layer16_attention_weight_min": -26.129106521606445,
      "activations/layer17_attention_weight_max": 58.78920364379883,
      "activations/layer17_attention_weight_min": -40.30268096923828,
      "activations/layer18_attention_weight_max": 45.0226936340332,
      "activations/layer18_attention_weight_min": -34.93714141845703,
      "activations/layer19_attention_weight_max": 25.067041397094727,
      "activations/layer19_attention_weight_min": -21.256258010864258,
      "activations/layer1_attention_weight_max": 16.793415069580078,
      "activations/layer1_attention_weight_min": -16.48613739013672,
      "activations/layer20_attention_weight_max": 21.72832489013672,
      "activations/layer20_attention_weight_min": -23.17919158935547,
      "activations/layer21_attention_weight_max": 39.20756912231445,
      "activations/layer21_attention_weight_min": -27.424175262451172,
      "activations/layer22_attention_weight_max": 31.891082763671875,
      "activations/layer22_attention_weight_min": -25.17591667175293,
      "activations/layer23_attention_weight_max": 28.86390495300293,
      "activations/layer23_attention_weight_min": -19.933259963989258,
      "activations/layer2_attention_weight_max": 29.331546783447266,
      "activations/layer2_attention_weight_min": -28.42117691040039,
      "activations/layer3_attention_weight_max": 79.83305358886719,
      "activations/layer3_attention_weight_min": -81.94538879394531,
      "activations/layer4_attention_weight_max": 99.25444793701172,
      "activations/layer4_attention_weight_min": -95.90892791748047,
      "activations/layer5_attention_weight_max": 96.77261352539062,
      "activations/layer5_attention_weight_min": -85.02378845214844,
      "activations/layer6_attention_weight_max": 55.61159133911133,
      "activations/layer6_attention_weight_min": -52.238136291503906,
      "activations/layer7_attention_weight_max": 75.81279754638672,
      "activations/layer7_attention_weight_min": -67.87679290771484,
      "activations/layer8_attention_weight_max": 43.12754440307617,
      "activations/layer8_attention_weight_min": -42.44820785522461,
      "activations/layer9_attention_weight_max": 51.61210250854492,
      "activations/layer9_attention_weight_min": -46.89011764526367,
      "epoch": 5.87,
      "learning_rate": 0.00011327575757575757,
      "loss": 2.8269,
      "step": 101050
    },
    {
      "activations/layer0_attention_weight_max": 15.889986038208008,
      "activations/layer0_attention_weight_min": -15.21767807006836,
      "activations/layer10_attention_weight_max": 39.71387481689453,
      "activations/layer10_attention_weight_min": -34.16150665283203,
      "activations/layer11_attention_weight_max": 29.665287017822266,
      "activations/layer11_attention_weight_min": -26.3863582611084,
      "activations/layer12_attention_weight_max": 22.10137939453125,
      "activations/layer12_attention_weight_min": -24.71485137939453,
      "activations/layer13_attention_weight_max": 46.82325744628906,
      "activations/layer13_attention_weight_min": -34.956539154052734,
      "activations/layer14_attention_weight_max": 45.61454391479492,
      "activations/layer14_attention_weight_min": -35.304054260253906,
      "activations/layer15_attention_weight_max": 35.35308837890625,
      "activations/layer15_attention_weight_min": -29.603771209716797,
      "activations/layer16_attention_weight_max": 34.11796569824219,
      "activations/layer16_attention_weight_min": -28.77796173095703,
      "activations/layer17_attention_weight_max": 55.28572082519531,
      "activations/layer17_attention_weight_min": -39.579105377197266,
      "activations/layer18_attention_weight_max": 48.03282928466797,
      "activations/layer18_attention_weight_min": -33.60118103027344,
      "activations/layer19_attention_weight_max": 22.511869430541992,
      "activations/layer19_attention_weight_min": -18.48603630065918,
      "activations/layer1_attention_weight_max": 18.240285873413086,
      "activations/layer1_attention_weight_min": -14.654669761657715,
      "activations/layer20_attention_weight_max": 23.84539794921875,
      "activations/layer20_attention_weight_min": -22.209386825561523,
      "activations/layer21_attention_weight_max": 36.82295227050781,
      "activations/layer21_attention_weight_min": -25.152536392211914,
      "activations/layer22_attention_weight_max": 32.77012252807617,
      "activations/layer22_attention_weight_min": -24.22420310974121,
      "activations/layer23_attention_weight_max": 31.395185470581055,
      "activations/layer23_attention_weight_min": -19.868810653686523,
      "activations/layer2_attention_weight_max": 31.36642837524414,
      "activations/layer2_attention_weight_min": -30.197853088378906,
      "activations/layer3_attention_weight_max": 86.2188949584961,
      "activations/layer3_attention_weight_min": -91.59834289550781,
      "activations/layer4_attention_weight_max": 100.09529876708984,
      "activations/layer4_attention_weight_min": -105.20550537109375,
      "activations/layer5_attention_weight_max": 94.98941040039062,
      "activations/layer5_attention_weight_min": -86.14373779296875,
      "activations/layer6_attention_weight_max": 55.69272232055664,
      "activations/layer6_attention_weight_min": -51.82900619506836,
      "activations/layer7_attention_weight_max": 69.49710083007812,
      "activations/layer7_attention_weight_min": -67.23767852783203,
      "activations/layer8_attention_weight_max": 43.71132278442383,
      "activations/layer8_attention_weight_min": -42.068721771240234,
      "activations/layer9_attention_weight_max": 49.82099533081055,
      "activations/layer9_attention_weight_min": -46.99705123901367,
      "epoch": 5.87,
      "learning_rate": 0.00011325681818181818,
      "loss": 2.8296,
      "step": 101100
    },
    {
      "activations/layer0_attention_weight_max": 14.313010215759277,
      "activations/layer0_attention_weight_min": -14.612207412719727,
      "activations/layer10_attention_weight_max": 39.606292724609375,
      "activations/layer10_attention_weight_min": -33.18788528442383,
      "activations/layer11_attention_weight_max": 30.383237838745117,
      "activations/layer11_attention_weight_min": -27.10805892944336,
      "activations/layer12_attention_weight_max": 20.559778213500977,
      "activations/layer12_attention_weight_min": -24.57245445251465,
      "activations/layer13_attention_weight_max": 46.629188537597656,
      "activations/layer13_attention_weight_min": -37.39118194580078,
      "activations/layer14_attention_weight_max": 43.23725509643555,
      "activations/layer14_attention_weight_min": -33.010498046875,
      "activations/layer15_attention_weight_max": 41.41115188598633,
      "activations/layer15_attention_weight_min": -29.60888671875,
      "activations/layer16_attention_weight_max": 32.392391204833984,
      "activations/layer16_attention_weight_min": -28.02639389038086,
      "activations/layer17_attention_weight_max": 59.09291458129883,
      "activations/layer17_attention_weight_min": -41.58378601074219,
      "activations/layer18_attention_weight_max": 50.12828826904297,
      "activations/layer18_attention_weight_min": -34.93895721435547,
      "activations/layer19_attention_weight_max": 23.171369552612305,
      "activations/layer19_attention_weight_min": -21.605606079101562,
      "activations/layer1_attention_weight_max": 17.637845993041992,
      "activations/layer1_attention_weight_min": -13.847383499145508,
      "activations/layer20_attention_weight_max": 24.801603317260742,
      "activations/layer20_attention_weight_min": -19.925098419189453,
      "activations/layer21_attention_weight_max": 44.236759185791016,
      "activations/layer21_attention_weight_min": -25.438980102539062,
      "activations/layer22_attention_weight_max": 33.7786865234375,
      "activations/layer22_attention_weight_min": -23.62760353088379,
      "activations/layer23_attention_weight_max": 31.08559226989746,
      "activations/layer23_attention_weight_min": -22.085721969604492,
      "activations/layer2_attention_weight_max": 31.299251556396484,
      "activations/layer2_attention_weight_min": -30.061199188232422,
      "activations/layer3_attention_weight_max": 80.94535064697266,
      "activations/layer3_attention_weight_min": -82.80763244628906,
      "activations/layer4_attention_weight_max": 98.91429138183594,
      "activations/layer4_attention_weight_min": -95.63713836669922,
      "activations/layer5_attention_weight_max": 95.011962890625,
      "activations/layer5_attention_weight_min": -84.6907958984375,
      "activations/layer6_attention_weight_max": 56.8419189453125,
      "activations/layer6_attention_weight_min": -50.99314498901367,
      "activations/layer7_attention_weight_max": 71.166748046875,
      "activations/layer7_attention_weight_min": -66.25470733642578,
      "activations/layer8_attention_weight_max": 42.46022415161133,
      "activations/layer8_attention_weight_min": -39.70304489135742,
      "activations/layer9_attention_weight_max": 48.159976959228516,
      "activations/layer9_attention_weight_min": -45.51005554199219,
      "epoch": 5.88,
      "learning_rate": 0.00011323787878787877,
      "loss": 2.8526,
      "step": 101150
    },
    {
      "activations/layer0_attention_weight_max": 15.542616844177246,
      "activations/layer0_attention_weight_min": -15.500815391540527,
      "activations/layer10_attention_weight_max": 38.46828079223633,
      "activations/layer10_attention_weight_min": -35.661582946777344,
      "activations/layer11_attention_weight_max": 27.34796714782715,
      "activations/layer11_attention_weight_min": -29.0826416015625,
      "activations/layer12_attention_weight_max": 19.935672760009766,
      "activations/layer12_attention_weight_min": -24.72138023376465,
      "activations/layer13_attention_weight_max": 38.12052917480469,
      "activations/layer13_attention_weight_min": -32.370704650878906,
      "activations/layer14_attention_weight_max": 54.037662506103516,
      "activations/layer14_attention_weight_min": -34.029876708984375,
      "activations/layer15_attention_weight_max": 40.68317413330078,
      "activations/layer15_attention_weight_min": -28.44256019592285,
      "activations/layer16_attention_weight_max": 38.812599182128906,
      "activations/layer16_attention_weight_min": -25.486976623535156,
      "activations/layer17_attention_weight_max": 56.3972053527832,
      "activations/layer17_attention_weight_min": -40.232486724853516,
      "activations/layer18_attention_weight_max": 47.7958869934082,
      "activations/layer18_attention_weight_min": -34.23305892944336,
      "activations/layer19_attention_weight_max": 22.684778213500977,
      "activations/layer19_attention_weight_min": -19.297094345092773,
      "activations/layer1_attention_weight_max": 17.70830535888672,
      "activations/layer1_attention_weight_min": -15.257479667663574,
      "activations/layer20_attention_weight_max": 23.240928649902344,
      "activations/layer20_attention_weight_min": -24.673477172851562,
      "activations/layer21_attention_weight_max": 37.66738510131836,
      "activations/layer21_attention_weight_min": -25.830381393432617,
      "activations/layer22_attention_weight_max": 34.27302551269531,
      "activations/layer22_attention_weight_min": -22.56570053100586,
      "activations/layer23_attention_weight_max": 27.33002471923828,
      "activations/layer23_attention_weight_min": -19.682931900024414,
      "activations/layer2_attention_weight_max": 31.48920440673828,
      "activations/layer2_attention_weight_min": -30.463943481445312,
      "activations/layer3_attention_weight_max": 84.0521240234375,
      "activations/layer3_attention_weight_min": -88.95354461669922,
      "activations/layer4_attention_weight_max": 100.96770477294922,
      "activations/layer4_attention_weight_min": -94.32324981689453,
      "activations/layer5_attention_weight_max": 97.14183807373047,
      "activations/layer5_attention_weight_min": -87.09103393554688,
      "activations/layer6_attention_weight_max": 56.22270202636719,
      "activations/layer6_attention_weight_min": -52.793277740478516,
      "activations/layer7_attention_weight_max": 73.46240234375,
      "activations/layer7_attention_weight_min": -65.24217224121094,
      "activations/layer8_attention_weight_max": 44.66714859008789,
      "activations/layer8_attention_weight_min": -42.52390670776367,
      "activations/layer9_attention_weight_max": 48.41684341430664,
      "activations/layer9_attention_weight_min": -48.005619049072266,
      "epoch": 5.88,
      "learning_rate": 0.00011321893939393939,
      "loss": 2.8274,
      "step": 101200
    },
    {
      "activations/layer0_attention_weight_max": 16.41031265258789,
      "activations/layer0_attention_weight_min": -14.611129760742188,
      "activations/layer10_attention_weight_max": 37.69216537475586,
      "activations/layer10_attention_weight_min": -32.11899185180664,
      "activations/layer11_attention_weight_max": 30.721832275390625,
      "activations/layer11_attention_weight_min": -27.04482078552246,
      "activations/layer12_attention_weight_max": 22.959556579589844,
      "activations/layer12_attention_weight_min": -25.176475524902344,
      "activations/layer13_attention_weight_max": 42.83523941040039,
      "activations/layer13_attention_weight_min": -34.574432373046875,
      "activations/layer14_attention_weight_max": 39.36531066894531,
      "activations/layer14_attention_weight_min": -29.105220794677734,
      "activations/layer15_attention_weight_max": 38.10325241088867,
      "activations/layer15_attention_weight_min": -29.495689392089844,
      "activations/layer16_attention_weight_max": 33.4912109375,
      "activations/layer16_attention_weight_min": -26.15064811706543,
      "activations/layer17_attention_weight_max": 50.699649810791016,
      "activations/layer17_attention_weight_min": -41.603206634521484,
      "activations/layer18_attention_weight_max": 46.21528244018555,
      "activations/layer18_attention_weight_min": -33.807769775390625,
      "activations/layer19_attention_weight_max": 21.272323608398438,
      "activations/layer19_attention_weight_min": -20.30781364440918,
      "activations/layer1_attention_weight_max": 16.471981048583984,
      "activations/layer1_attention_weight_min": -16.8863525390625,
      "activations/layer20_attention_weight_max": 21.471452713012695,
      "activations/layer20_attention_weight_min": -22.625303268432617,
      "activations/layer21_attention_weight_max": 48.417781829833984,
      "activations/layer21_attention_weight_min": -26.051437377929688,
      "activations/layer22_attention_weight_max": 35.63223648071289,
      "activations/layer22_attention_weight_min": -24.53919219970703,
      "activations/layer23_attention_weight_max": 29.03775405883789,
      "activations/layer23_attention_weight_min": -21.317718505859375,
      "activations/layer2_attention_weight_max": 31.76978302001953,
      "activations/layer2_attention_weight_min": -30.07710075378418,
      "activations/layer3_attention_weight_max": 83.14791870117188,
      "activations/layer3_attention_weight_min": -87.77153778076172,
      "activations/layer4_attention_weight_max": 96.0029525756836,
      "activations/layer4_attention_weight_min": -92.67532348632812,
      "activations/layer5_attention_weight_max": 94.77013397216797,
      "activations/layer5_attention_weight_min": -84.30450439453125,
      "activations/layer6_attention_weight_max": 56.132022857666016,
      "activations/layer6_attention_weight_min": -50.1754035949707,
      "activations/layer7_attention_weight_max": 71.78353881835938,
      "activations/layer7_attention_weight_min": -66.64070129394531,
      "activations/layer8_attention_weight_max": 43.56269073486328,
      "activations/layer8_attention_weight_min": -43.794715881347656,
      "activations/layer9_attention_weight_max": 58.94269561767578,
      "activations/layer9_attention_weight_min": -46.71587371826172,
      "epoch": 5.88,
      "learning_rate": 0.0001132,
      "loss": 2.8486,
      "step": 101250
    },
    {
      "activations/layer0_attention_weight_max": 14.510766983032227,
      "activations/layer0_attention_weight_min": -14.603598594665527,
      "activations/layer10_attention_weight_max": 36.3021354675293,
      "activations/layer10_attention_weight_min": -33.825469970703125,
      "activations/layer11_attention_weight_max": 29.361526489257812,
      "activations/layer11_attention_weight_min": -27.68018913269043,
      "activations/layer12_attention_weight_max": 22.38164520263672,
      "activations/layer12_attention_weight_min": -27.159595489501953,
      "activations/layer13_attention_weight_max": 53.15990447998047,
      "activations/layer13_attention_weight_min": -36.76802062988281,
      "activations/layer14_attention_weight_max": 45.35319519042969,
      "activations/layer14_attention_weight_min": -31.57927894592285,
      "activations/layer15_attention_weight_max": 42.419673919677734,
      "activations/layer15_attention_weight_min": -31.147096633911133,
      "activations/layer16_attention_weight_max": 35.56307601928711,
      "activations/layer16_attention_weight_min": -26.94594955444336,
      "activations/layer17_attention_weight_max": 59.16610336303711,
      "activations/layer17_attention_weight_min": -42.48371124267578,
      "activations/layer18_attention_weight_max": 49.552093505859375,
      "activations/layer18_attention_weight_min": -35.98729705810547,
      "activations/layer19_attention_weight_max": 24.10744285583496,
      "activations/layer19_attention_weight_min": -19.77767562866211,
      "activations/layer1_attention_weight_max": 17.60096549987793,
      "activations/layer1_attention_weight_min": -17.92980194091797,
      "activations/layer20_attention_weight_max": 23.735300064086914,
      "activations/layer20_attention_weight_min": -19.638729095458984,
      "activations/layer21_attention_weight_max": 47.72529983520508,
      "activations/layer21_attention_weight_min": -24.123455047607422,
      "activations/layer22_attention_weight_max": 34.661354064941406,
      "activations/layer22_attention_weight_min": -27.119680404663086,
      "activations/layer23_attention_weight_max": 31.035316467285156,
      "activations/layer23_attention_weight_min": -18.64063835144043,
      "activations/layer2_attention_weight_max": 33.20539855957031,
      "activations/layer2_attention_weight_min": -30.198490142822266,
      "activations/layer3_attention_weight_max": 91.3829116821289,
      "activations/layer3_attention_weight_min": -91.65288543701172,
      "activations/layer4_attention_weight_max": 101.8548812866211,
      "activations/layer4_attention_weight_min": -99.0541763305664,
      "activations/layer5_attention_weight_max": 92.34319305419922,
      "activations/layer5_attention_weight_min": -87.49501037597656,
      "activations/layer6_attention_weight_max": 54.735939025878906,
      "activations/layer6_attention_weight_min": -52.609458923339844,
      "activations/layer7_attention_weight_max": 73.03093719482422,
      "activations/layer7_attention_weight_min": -69.31653594970703,
      "activations/layer8_attention_weight_max": 42.9831428527832,
      "activations/layer8_attention_weight_min": -42.37087631225586,
      "activations/layer9_attention_weight_max": 59.1427001953125,
      "activations/layer9_attention_weight_min": -49.73335266113281,
      "epoch": 5.89,
      "learning_rate": 0.00011318106060606059,
      "loss": 2.8228,
      "step": 101300
    },
    {
      "activations/layer0_attention_weight_max": 15.768098831176758,
      "activations/layer0_attention_weight_min": -14.840764999389648,
      "activations/layer10_attention_weight_max": 46.65869140625,
      "activations/layer10_attention_weight_min": -41.4084358215332,
      "activations/layer11_attention_weight_max": 37.5256233215332,
      "activations/layer11_attention_weight_min": -28.68313980102539,
      "activations/layer12_attention_weight_max": 22.8221435546875,
      "activations/layer12_attention_weight_min": -26.439754486083984,
      "activations/layer13_attention_weight_max": 66.05046844482422,
      "activations/layer13_attention_weight_min": -35.36452865600586,
      "activations/layer14_attention_weight_max": 60.294986724853516,
      "activations/layer14_attention_weight_min": -35.24348449707031,
      "activations/layer15_attention_weight_max": 44.357303619384766,
      "activations/layer15_attention_weight_min": -29.293088912963867,
      "activations/layer16_attention_weight_max": 40.84645080566406,
      "activations/layer16_attention_weight_min": -28.501998901367188,
      "activations/layer17_attention_weight_max": 63.18952941894531,
      "activations/layer17_attention_weight_min": -43.86720657348633,
      "activations/layer18_attention_weight_max": 53.09294128417969,
      "activations/layer18_attention_weight_min": -38.690521240234375,
      "activations/layer19_attention_weight_max": 20.728116989135742,
      "activations/layer19_attention_weight_min": -20.387060165405273,
      "activations/layer1_attention_weight_max": 17.386781692504883,
      "activations/layer1_attention_weight_min": -18.63948631286621,
      "activations/layer20_attention_weight_max": 23.415546417236328,
      "activations/layer20_attention_weight_min": -19.670385360717773,
      "activations/layer21_attention_weight_max": 45.6483268737793,
      "activations/layer21_attention_weight_min": -29.636974334716797,
      "activations/layer22_attention_weight_max": 36.85901641845703,
      "activations/layer22_attention_weight_min": -23.37656021118164,
      "activations/layer23_attention_weight_max": 29.140684127807617,
      "activations/layer23_attention_weight_min": -17.88868522644043,
      "activations/layer2_attention_weight_max": 35.81789779663086,
      "activations/layer2_attention_weight_min": -31.937002182006836,
      "activations/layer3_attention_weight_max": 87.42451477050781,
      "activations/layer3_attention_weight_min": -88.80815887451172,
      "activations/layer4_attention_weight_max": 104.17182922363281,
      "activations/layer4_attention_weight_min": -99.6411361694336,
      "activations/layer5_attention_weight_max": 101.43951416015625,
      "activations/layer5_attention_weight_min": -91.28665161132812,
      "activations/layer6_attention_weight_max": 61.104034423828125,
      "activations/layer6_attention_weight_min": -54.90643310546875,
      "activations/layer7_attention_weight_max": 86.25267791748047,
      "activations/layer7_attention_weight_min": -74.12763214111328,
      "activations/layer8_attention_weight_max": 57.65938949584961,
      "activations/layer8_attention_weight_min": -53.663238525390625,
      "activations/layer9_attention_weight_max": 69.4571304321289,
      "activations/layer9_attention_weight_min": -58.471778869628906,
      "epoch": 5.89,
      "learning_rate": 0.00011316249999999998,
      "loss": 2.8395,
      "step": 101350
    },
    {
      "activations/layer0_attention_weight_max": 14.366785049438477,
      "activations/layer0_attention_weight_min": -14.827231407165527,
      "activations/layer10_attention_weight_max": 40.52484893798828,
      "activations/layer10_attention_weight_min": -38.21747589111328,
      "activations/layer11_attention_weight_max": 35.39326477050781,
      "activations/layer11_attention_weight_min": -31.363136291503906,
      "activations/layer12_attention_weight_max": 19.304927825927734,
      "activations/layer12_attention_weight_min": -27.028732299804688,
      "activations/layer13_attention_weight_max": 61.59152603149414,
      "activations/layer13_attention_weight_min": -44.325687408447266,
      "activations/layer14_attention_weight_max": 51.32160949707031,
      "activations/layer14_attention_weight_min": -35.48284149169922,
      "activations/layer15_attention_weight_max": 41.769779205322266,
      "activations/layer15_attention_weight_min": -33.01055145263672,
      "activations/layer16_attention_weight_max": 35.51520538330078,
      "activations/layer16_attention_weight_min": -28.567916870117188,
      "activations/layer17_attention_weight_max": 55.67261505126953,
      "activations/layer17_attention_weight_min": -43.96371841430664,
      "activations/layer18_attention_weight_max": 52.433746337890625,
      "activations/layer18_attention_weight_min": -39.52043914794922,
      "activations/layer19_attention_weight_max": 22.392013549804688,
      "activations/layer19_attention_weight_min": -21.258420944213867,
      "activations/layer1_attention_weight_max": 18.085052490234375,
      "activations/layer1_attention_weight_min": -13.391572952270508,
      "activations/layer20_attention_weight_max": 23.86329460144043,
      "activations/layer20_attention_weight_min": -21.060256958007812,
      "activations/layer21_attention_weight_max": 50.140289306640625,
      "activations/layer21_attention_weight_min": -27.259422302246094,
      "activations/layer22_attention_weight_max": 40.83464431762695,
      "activations/layer22_attention_weight_min": -28.09501838684082,
      "activations/layer23_attention_weight_max": 37.564510345458984,
      "activations/layer23_attention_weight_min": -21.718107223510742,
      "activations/layer2_attention_weight_max": 29.447683334350586,
      "activations/layer2_attention_weight_min": -27.756263732910156,
      "activations/layer3_attention_weight_max": 80.96910858154297,
      "activations/layer3_attention_weight_min": -84.37734985351562,
      "activations/layer4_attention_weight_max": 103.27911376953125,
      "activations/layer4_attention_weight_min": -99.77861785888672,
      "activations/layer5_attention_weight_max": 105.4574203491211,
      "activations/layer5_attention_weight_min": -92.61238098144531,
      "activations/layer6_attention_weight_max": 60.35575485229492,
      "activations/layer6_attention_weight_min": -56.33711242675781,
      "activations/layer7_attention_weight_max": 80.80661010742188,
      "activations/layer7_attention_weight_min": -76.30725860595703,
      "activations/layer8_attention_weight_max": 45.63191223144531,
      "activations/layer8_attention_weight_min": -42.1540641784668,
      "activations/layer9_attention_weight_max": 59.4892578125,
      "activations/layer9_attention_weight_min": -52.215614318847656,
      "epoch": 5.89,
      "learning_rate": 0.0001131435606060606,
      "loss": 2.8431,
      "step": 101400
    },
    {
      "activations/layer0_attention_weight_max": 15.410017967224121,
      "activations/layer0_attention_weight_min": -14.997901916503906,
      "activations/layer10_attention_weight_max": 40.85371017456055,
      "activations/layer10_attention_weight_min": -38.290645599365234,
      "activations/layer11_attention_weight_max": 34.38383483886719,
      "activations/layer11_attention_weight_min": -29.126235961914062,
      "activations/layer12_attention_weight_max": 23.141300201416016,
      "activations/layer12_attention_weight_min": -24.484806060791016,
      "activations/layer13_attention_weight_max": 53.980289459228516,
      "activations/layer13_attention_weight_min": -34.49769973754883,
      "activations/layer14_attention_weight_max": 66.54479217529297,
      "activations/layer14_attention_weight_min": -40.75459671020508,
      "activations/layer15_attention_weight_max": 48.6817741394043,
      "activations/layer15_attention_weight_min": -33.228515625,
      "activations/layer16_attention_weight_max": 37.21305465698242,
      "activations/layer16_attention_weight_min": -27.28889274597168,
      "activations/layer17_attention_weight_max": 59.4185676574707,
      "activations/layer17_attention_weight_min": -42.34090805053711,
      "activations/layer18_attention_weight_max": 48.698368072509766,
      "activations/layer18_attention_weight_min": -35.6063346862793,
      "activations/layer19_attention_weight_max": 23.98291778564453,
      "activations/layer19_attention_weight_min": -22.283109664916992,
      "activations/layer1_attention_weight_max": 17.27635955810547,
      "activations/layer1_attention_weight_min": -14.259592056274414,
      "activations/layer20_attention_weight_max": 22.90716552734375,
      "activations/layer20_attention_weight_min": -26.179174423217773,
      "activations/layer21_attention_weight_max": 45.345726013183594,
      "activations/layer21_attention_weight_min": -28.00863265991211,
      "activations/layer22_attention_weight_max": 34.69329071044922,
      "activations/layer22_attention_weight_min": -25.530338287353516,
      "activations/layer23_attention_weight_max": 33.74141311645508,
      "activations/layer23_attention_weight_min": -20.570423126220703,
      "activations/layer2_attention_weight_max": 30.318435668945312,
      "activations/layer2_attention_weight_min": -28.435606002807617,
      "activations/layer3_attention_weight_max": 79.85179138183594,
      "activations/layer3_attention_weight_min": -84.2802963256836,
      "activations/layer4_attention_weight_max": 94.9413070678711,
      "activations/layer4_attention_weight_min": -93.29251861572266,
      "activations/layer5_attention_weight_max": 92.8524398803711,
      "activations/layer5_attention_weight_min": -86.79821014404297,
      "activations/layer6_attention_weight_max": 57.682586669921875,
      "activations/layer6_attention_weight_min": -52.83702850341797,
      "activations/layer7_attention_weight_max": 72.97392272949219,
      "activations/layer7_attention_weight_min": -66.83815002441406,
      "activations/layer8_attention_weight_max": 44.29018783569336,
      "activations/layer8_attention_weight_min": -43.16676712036133,
      "activations/layer9_attention_weight_max": 58.624114990234375,
      "activations/layer9_attention_weight_min": -48.09749221801758,
      "epoch": 5.89,
      "learning_rate": 0.00011312462121212121,
      "loss": 2.8482,
      "step": 101450
    },
    {
      "activations/layer0_attention_weight_max": 16.086528778076172,
      "activations/layer0_attention_weight_min": -14.94283390045166,
      "activations/layer10_attention_weight_max": 38.63623809814453,
      "activations/layer10_attention_weight_min": -35.71876907348633,
      "activations/layer11_attention_weight_max": 33.43695068359375,
      "activations/layer11_attention_weight_min": -29.1826229095459,
      "activations/layer12_attention_weight_max": 24.227054595947266,
      "activations/layer12_attention_weight_min": -28.198759078979492,
      "activations/layer13_attention_weight_max": 52.613014221191406,
      "activations/layer13_attention_weight_min": -32.34144592285156,
      "activations/layer14_attention_weight_max": 46.037784576416016,
      "activations/layer14_attention_weight_min": -30.489459991455078,
      "activations/layer15_attention_weight_max": 45.61726760864258,
      "activations/layer15_attention_weight_min": -31.060304641723633,
      "activations/layer16_attention_weight_max": 35.58072280883789,
      "activations/layer16_attention_weight_min": -27.126232147216797,
      "activations/layer17_attention_weight_max": 59.81610870361328,
      "activations/layer17_attention_weight_min": -38.50764083862305,
      "activations/layer18_attention_weight_max": 52.47309875488281,
      "activations/layer18_attention_weight_min": -32.8015251159668,
      "activations/layer19_attention_weight_max": 23.368892669677734,
      "activations/layer19_attention_weight_min": -17.947938919067383,
      "activations/layer1_attention_weight_max": 17.169591903686523,
      "activations/layer1_attention_weight_min": -15.355273246765137,
      "activations/layer20_attention_weight_max": 19.566936492919922,
      "activations/layer20_attention_weight_min": -20.647743225097656,
      "activations/layer21_attention_weight_max": 40.806915283203125,
      "activations/layer21_attention_weight_min": -25.67089080810547,
      "activations/layer22_attention_weight_max": 33.98892593383789,
      "activations/layer22_attention_weight_min": -23.530986785888672,
      "activations/layer23_attention_weight_max": 30.00143051147461,
      "activations/layer23_attention_weight_min": -18.504854202270508,
      "activations/layer2_attention_weight_max": 29.066654205322266,
      "activations/layer2_attention_weight_min": -25.73957061767578,
      "activations/layer3_attention_weight_max": 75.99259185791016,
      "activations/layer3_attention_weight_min": -76.04405212402344,
      "activations/layer4_attention_weight_max": 89.77252197265625,
      "activations/layer4_attention_weight_min": -84.87142944335938,
      "activations/layer5_attention_weight_max": 94.25847625732422,
      "activations/layer5_attention_weight_min": -85.36506652832031,
      "activations/layer6_attention_weight_max": 57.3491096496582,
      "activations/layer6_attention_weight_min": -51.792781829833984,
      "activations/layer7_attention_weight_max": 72.3957748413086,
      "activations/layer7_attention_weight_min": -74.21383666992188,
      "activations/layer8_attention_weight_max": 44.11148452758789,
      "activations/layer8_attention_weight_min": -45.26353073120117,
      "activations/layer9_attention_weight_max": 50.2716178894043,
      "activations/layer9_attention_weight_min": -48.06844711303711,
      "epoch": 5.9,
      "learning_rate": 0.0001131056818181818,
      "loss": 2.8464,
      "step": 101500
    },
    {
      "activations/layer0_attention_weight_max": 14.700404167175293,
      "activations/layer0_attention_weight_min": -14.843352317810059,
      "activations/layer10_attention_weight_max": 39.501319885253906,
      "activations/layer10_attention_weight_min": -35.48453903198242,
      "activations/layer11_attention_weight_max": 32.899566650390625,
      "activations/layer11_attention_weight_min": -29.10009002685547,
      "activations/layer12_attention_weight_max": 37.109375,
      "activations/layer12_attention_weight_min": -22.92627716064453,
      "activations/layer13_attention_weight_max": 49.75532913208008,
      "activations/layer13_attention_weight_min": -33.521156311035156,
      "activations/layer14_attention_weight_max": 41.929508209228516,
      "activations/layer14_attention_weight_min": -30.95970344543457,
      "activations/layer15_attention_weight_max": 42.1739387512207,
      "activations/layer15_attention_weight_min": -30.60338592529297,
      "activations/layer16_attention_weight_max": 37.668697357177734,
      "activations/layer16_attention_weight_min": -28.839242935180664,
      "activations/layer17_attention_weight_max": 58.03590393066406,
      "activations/layer17_attention_weight_min": -42.9080810546875,
      "activations/layer18_attention_weight_max": 53.021724700927734,
      "activations/layer18_attention_weight_min": -36.039737701416016,
      "activations/layer19_attention_weight_max": 23.29177474975586,
      "activations/layer19_attention_weight_min": -19.07265281677246,
      "activations/layer1_attention_weight_max": 17.809492111206055,
      "activations/layer1_attention_weight_min": -14.807079315185547,
      "activations/layer20_attention_weight_max": 22.620573043823242,
      "activations/layer20_attention_weight_min": -19.748929977416992,
      "activations/layer21_attention_weight_max": 36.935611724853516,
      "activations/layer21_attention_weight_min": -24.718505859375,
      "activations/layer22_attention_weight_max": 39.076080322265625,
      "activations/layer22_attention_weight_min": -24.117233276367188,
      "activations/layer23_attention_weight_max": 32.99127960205078,
      "activations/layer23_attention_weight_min": -17.83277130126953,
      "activations/layer2_attention_weight_max": 30.49087142944336,
      "activations/layer2_attention_weight_min": -29.86806297302246,
      "activations/layer3_attention_weight_max": 76.74311065673828,
      "activations/layer3_attention_weight_min": -76.29502868652344,
      "activations/layer4_attention_weight_max": 94.24951171875,
      "activations/layer4_attention_weight_min": -89.25890350341797,
      "activations/layer5_attention_weight_max": 95.11279296875,
      "activations/layer5_attention_weight_min": -87.4326400756836,
      "activations/layer6_attention_weight_max": 57.2931022644043,
      "activations/layer6_attention_weight_min": -52.95576095581055,
      "activations/layer7_attention_weight_max": 71.97109985351562,
      "activations/layer7_attention_weight_min": -72.08211517333984,
      "activations/layer8_attention_weight_max": 43.48395538330078,
      "activations/layer8_attention_weight_min": -42.4246940612793,
      "activations/layer9_attention_weight_max": 54.28219223022461,
      "activations/layer9_attention_weight_min": -46.50265121459961,
      "epoch": 5.9,
      "learning_rate": 0.00011308674242424241,
      "loss": 2.8224,
      "step": 101550
    },
    {
      "activations/layer0_attention_weight_max": 14.13963508605957,
      "activations/layer0_attention_weight_min": -14.980539321899414,
      "activations/layer10_attention_weight_max": 38.10107421875,
      "activations/layer10_attention_weight_min": -34.85578155517578,
      "activations/layer11_attention_weight_max": 31.962261199951172,
      "activations/layer11_attention_weight_min": -29.490835189819336,
      "activations/layer12_attention_weight_max": 20.55990982055664,
      "activations/layer12_attention_weight_min": -24.29364013671875,
      "activations/layer13_attention_weight_max": 51.43958282470703,
      "activations/layer13_attention_weight_min": -35.435054779052734,
      "activations/layer14_attention_weight_max": 46.776771545410156,
      "activations/layer14_attention_weight_min": -29.216995239257812,
      "activations/layer15_attention_weight_max": 41.30425262451172,
      "activations/layer15_attention_weight_min": -29.545921325683594,
      "activations/layer16_attention_weight_max": 37.17801284790039,
      "activations/layer16_attention_weight_min": -26.734346389770508,
      "activations/layer17_attention_weight_max": 66.8907699584961,
      "activations/layer17_attention_weight_min": -39.76442337036133,
      "activations/layer18_attention_weight_max": 53.27812957763672,
      "activations/layer18_attention_weight_min": -35.256324768066406,
      "activations/layer19_attention_weight_max": 22.11969757080078,
      "activations/layer19_attention_weight_min": -20.620819091796875,
      "activations/layer1_attention_weight_max": 17.465105056762695,
      "activations/layer1_attention_weight_min": -14.489562034606934,
      "activations/layer20_attention_weight_max": 25.529130935668945,
      "activations/layer20_attention_weight_min": -19.328868865966797,
      "activations/layer21_attention_weight_max": 44.54740524291992,
      "activations/layer21_attention_weight_min": -26.02455711364746,
      "activations/layer22_attention_weight_max": 32.55423355102539,
      "activations/layer22_attention_weight_min": -26.32555389404297,
      "activations/layer23_attention_weight_max": 34.850337982177734,
      "activations/layer23_attention_weight_min": -19.560325622558594,
      "activations/layer2_attention_weight_max": 28.91706657409668,
      "activations/layer2_attention_weight_min": -27.264089584350586,
      "activations/layer3_attention_weight_max": 83.01335906982422,
      "activations/layer3_attention_weight_min": -84.59136962890625,
      "activations/layer4_attention_weight_max": 98.13538360595703,
      "activations/layer4_attention_weight_min": -89.80572509765625,
      "activations/layer5_attention_weight_max": 94.41057586669922,
      "activations/layer5_attention_weight_min": -82.58697509765625,
      "activations/layer6_attention_weight_max": 57.142356872558594,
      "activations/layer6_attention_weight_min": -49.68208312988281,
      "activations/layer7_attention_weight_max": 70.93928527832031,
      "activations/layer7_attention_weight_min": -69.34978485107422,
      "activations/layer8_attention_weight_max": 43.84903335571289,
      "activations/layer8_attention_weight_min": -41.927425384521484,
      "activations/layer9_attention_weight_max": 55.130916595458984,
      "activations/layer9_attention_weight_min": -48.0766487121582,
      "epoch": 5.9,
      "learning_rate": 0.00011306780303030303,
      "loss": 2.8392,
      "step": 101600
    },
    {
      "activations/layer0_attention_weight_max": 15.259523391723633,
      "activations/layer0_attention_weight_min": -14.712170600891113,
      "activations/layer10_attention_weight_max": 36.99386978149414,
      "activations/layer10_attention_weight_min": -33.198665618896484,
      "activations/layer11_attention_weight_max": 30.131027221679688,
      "activations/layer11_attention_weight_min": -27.422054290771484,
      "activations/layer12_attention_weight_max": 37.3327522277832,
      "activations/layer12_attention_weight_min": -22.466999053955078,
      "activations/layer13_attention_weight_max": 45.30243682861328,
      "activations/layer13_attention_weight_min": -33.47569274902344,
      "activations/layer14_attention_weight_max": 36.587406158447266,
      "activations/layer14_attention_weight_min": -29.71578025817871,
      "activations/layer15_attention_weight_max": 34.91622543334961,
      "activations/layer15_attention_weight_min": -28.703027725219727,
      "activations/layer16_attention_weight_max": 32.33740234375,
      "activations/layer16_attention_weight_min": -25.624467849731445,
      "activations/layer17_attention_weight_max": 52.743202209472656,
      "activations/layer17_attention_weight_min": -39.14460754394531,
      "activations/layer18_attention_weight_max": 48.096946716308594,
      "activations/layer18_attention_weight_min": -35.11042785644531,
      "activations/layer19_attention_weight_max": 22.776376724243164,
      "activations/layer19_attention_weight_min": -18.12740135192871,
      "activations/layer1_attention_weight_max": 17.962915420532227,
      "activations/layer1_attention_weight_min": -15.782502174377441,
      "activations/layer20_attention_weight_max": 19.434734344482422,
      "activations/layer20_attention_weight_min": -22.087675094604492,
      "activations/layer21_attention_weight_max": 35.035518646240234,
      "activations/layer21_attention_weight_min": -24.808866500854492,
      "activations/layer22_attention_weight_max": 31.424184799194336,
      "activations/layer22_attention_weight_min": -23.17413902282715,
      "activations/layer23_attention_weight_max": 27.888595581054688,
      "activations/layer23_attention_weight_min": -20.384933471679688,
      "activations/layer2_attention_weight_max": 29.877296447753906,
      "activations/layer2_attention_weight_min": -29.060461044311523,
      "activations/layer3_attention_weight_max": 77.7592544555664,
      "activations/layer3_attention_weight_min": -84.79280090332031,
      "activations/layer4_attention_weight_max": 97.72916412353516,
      "activations/layer4_attention_weight_min": -95.2247543334961,
      "activations/layer5_attention_weight_max": 93.68841552734375,
      "activations/layer5_attention_weight_min": -91.18157958984375,
      "activations/layer6_attention_weight_max": 57.148319244384766,
      "activations/layer6_attention_weight_min": -52.06190872192383,
      "activations/layer7_attention_weight_max": 76.65132141113281,
      "activations/layer7_attention_weight_min": -72.103515625,
      "activations/layer8_attention_weight_max": 43.93022537231445,
      "activations/layer8_attention_weight_min": -40.81369400024414,
      "activations/layer9_attention_weight_max": 48.11503601074219,
      "activations/layer9_attention_weight_min": -46.201744079589844,
      "epoch": 5.91,
      "learning_rate": 0.00011304886363636363,
      "loss": 2.8311,
      "step": 101650
    },
    {
      "activations/layer0_attention_weight_max": 17.1447811126709,
      "activations/layer0_attention_weight_min": -14.729947090148926,
      "activations/layer10_attention_weight_max": 38.04921340942383,
      "activations/layer10_attention_weight_min": -36.70048904418945,
      "activations/layer11_attention_weight_max": 30.110584259033203,
      "activations/layer11_attention_weight_min": -28.743038177490234,
      "activations/layer12_attention_weight_max": 22.51689910888672,
      "activations/layer12_attention_weight_min": -29.397314071655273,
      "activations/layer13_attention_weight_max": 54.7960090637207,
      "activations/layer13_attention_weight_min": -37.149837493896484,
      "activations/layer14_attention_weight_max": 57.30668258666992,
      "activations/layer14_attention_weight_min": -30.50017738342285,
      "activations/layer15_attention_weight_max": 40.4359245300293,
      "activations/layer15_attention_weight_min": -29.128019332885742,
      "activations/layer16_attention_weight_max": 38.02182388305664,
      "activations/layer16_attention_weight_min": -26.395000457763672,
      "activations/layer17_attention_weight_max": 54.10983657836914,
      "activations/layer17_attention_weight_min": -40.245445251464844,
      "activations/layer18_attention_weight_max": 48.09719467163086,
      "activations/layer18_attention_weight_min": -36.606441497802734,
      "activations/layer19_attention_weight_max": 23.08770179748535,
      "activations/layer19_attention_weight_min": -21.974599838256836,
      "activations/layer1_attention_weight_max": 16.549549102783203,
      "activations/layer1_attention_weight_min": -14.690865516662598,
      "activations/layer20_attention_weight_max": 25.017898559570312,
      "activations/layer20_attention_weight_min": -19.302236557006836,
      "activations/layer21_attention_weight_max": 40.44407272338867,
      "activations/layer21_attention_weight_min": -24.990825653076172,
      "activations/layer22_attention_weight_max": 32.85050964355469,
      "activations/layer22_attention_weight_min": -25.92184829711914,
      "activations/layer23_attention_weight_max": 32.225521087646484,
      "activations/layer23_attention_weight_min": -20.406404495239258,
      "activations/layer2_attention_weight_max": 27.526351928710938,
      "activations/layer2_attention_weight_min": -28.245628356933594,
      "activations/layer3_attention_weight_max": 82.49407958984375,
      "activations/layer3_attention_weight_min": -83.37706756591797,
      "activations/layer4_attention_weight_max": 101.46047973632812,
      "activations/layer4_attention_weight_min": -95.67378997802734,
      "activations/layer5_attention_weight_max": 95.96865844726562,
      "activations/layer5_attention_weight_min": -91.4512939453125,
      "activations/layer6_attention_weight_max": 56.88653564453125,
      "activations/layer6_attention_weight_min": -52.24246597290039,
      "activations/layer7_attention_weight_max": 79.3497314453125,
      "activations/layer7_attention_weight_min": -67.4690170288086,
      "activations/layer8_attention_weight_max": 44.83769989013672,
      "activations/layer8_attention_weight_min": -43.06486892700195,
      "activations/layer9_attention_weight_max": 69.84777069091797,
      "activations/layer9_attention_weight_min": -49.209083557128906,
      "epoch": 5.91,
      "learning_rate": 0.00011302992424242423,
      "loss": 2.8343,
      "step": 101700
    },
    {
      "activations/layer0_attention_weight_max": 14.42908763885498,
      "activations/layer0_attention_weight_min": -14.917219161987305,
      "activations/layer10_attention_weight_max": 37.84663772583008,
      "activations/layer10_attention_weight_min": -33.93195343017578,
      "activations/layer11_attention_weight_max": 28.54096031188965,
      "activations/layer11_attention_weight_min": -26.727344512939453,
      "activations/layer12_attention_weight_max": 22.691736221313477,
      "activations/layer12_attention_weight_min": -27.468900680541992,
      "activations/layer13_attention_weight_max": 48.79923629760742,
      "activations/layer13_attention_weight_min": -33.43384552001953,
      "activations/layer14_attention_weight_max": 60.63936996459961,
      "activations/layer14_attention_weight_min": -31.285343170166016,
      "activations/layer15_attention_weight_max": 41.117069244384766,
      "activations/layer15_attention_weight_min": -30.83236312866211,
      "activations/layer16_attention_weight_max": 34.879638671875,
      "activations/layer16_attention_weight_min": -26.87319564819336,
      "activations/layer17_attention_weight_max": 53.21714782714844,
      "activations/layer17_attention_weight_min": -41.85259246826172,
      "activations/layer18_attention_weight_max": 46.16801452636719,
      "activations/layer18_attention_weight_min": -36.74619674682617,
      "activations/layer19_attention_weight_max": 23.855695724487305,
      "activations/layer19_attention_weight_min": -20.25372886657715,
      "activations/layer1_attention_weight_max": 17.160541534423828,
      "activations/layer1_attention_weight_min": -14.280889511108398,
      "activations/layer20_attention_weight_max": 21.01045036315918,
      "activations/layer20_attention_weight_min": -21.10083770751953,
      "activations/layer21_attention_weight_max": 42.078975677490234,
      "activations/layer21_attention_weight_min": -24.427108764648438,
      "activations/layer22_attention_weight_max": 36.6494026184082,
      "activations/layer22_attention_weight_min": -26.038557052612305,
      "activations/layer23_attention_weight_max": 29.593589782714844,
      "activations/layer23_attention_weight_min": -17.385244369506836,
      "activations/layer2_attention_weight_max": 28.315359115600586,
      "activations/layer2_attention_weight_min": -27.744016647338867,
      "activations/layer3_attention_weight_max": 75.95805358886719,
      "activations/layer3_attention_weight_min": -78.97289276123047,
      "activations/layer4_attention_weight_max": 93.80335998535156,
      "activations/layer4_attention_weight_min": -85.9793472290039,
      "activations/layer5_attention_weight_max": 92.00091552734375,
      "activations/layer5_attention_weight_min": -84.73210906982422,
      "activations/layer6_attention_weight_max": 54.351375579833984,
      "activations/layer6_attention_weight_min": -48.80580139160156,
      "activations/layer7_attention_weight_max": 70.98030853271484,
      "activations/layer7_attention_weight_min": -64.8515396118164,
      "activations/layer8_attention_weight_max": 40.97639083862305,
      "activations/layer8_attention_weight_min": -41.896392822265625,
      "activations/layer9_attention_weight_max": 49.62639617919922,
      "activations/layer9_attention_weight_min": -47.287376403808594,
      "epoch": 5.91,
      "learning_rate": 0.00011301098484848483,
      "loss": 2.842,
      "step": 101750
    },
    {
      "activations/layer0_attention_weight_max": 14.229414939880371,
      "activations/layer0_attention_weight_min": -14.231863021850586,
      "activations/layer10_attention_weight_max": 41.15528869628906,
      "activations/layer10_attention_weight_min": -35.76674270629883,
      "activations/layer11_attention_weight_max": 33.91489028930664,
      "activations/layer11_attention_weight_min": -29.041210174560547,
      "activations/layer12_attention_weight_max": 37.8029670715332,
      "activations/layer12_attention_weight_min": -26.73054313659668,
      "activations/layer13_attention_weight_max": 59.121009826660156,
      "activations/layer13_attention_weight_min": -39.64288330078125,
      "activations/layer14_attention_weight_max": 64.69841003417969,
      "activations/layer14_attention_weight_min": -35.61537170410156,
      "activations/layer15_attention_weight_max": 43.96087646484375,
      "activations/layer15_attention_weight_min": -28.588825225830078,
      "activations/layer16_attention_weight_max": 34.331233978271484,
      "activations/layer16_attention_weight_min": -25.70357322692871,
      "activations/layer17_attention_weight_max": 67.4132308959961,
      "activations/layer17_attention_weight_min": -42.48776626586914,
      "activations/layer18_attention_weight_max": 49.89126968383789,
      "activations/layer18_attention_weight_min": -34.41569519042969,
      "activations/layer19_attention_weight_max": 26.74267578125,
      "activations/layer19_attention_weight_min": -19.932714462280273,
      "activations/layer1_attention_weight_max": 16.81060791015625,
      "activations/layer1_attention_weight_min": -15.2173490524292,
      "activations/layer20_attention_weight_max": 23.095867156982422,
      "activations/layer20_attention_weight_min": -20.03520965576172,
      "activations/layer21_attention_weight_max": 38.71723937988281,
      "activations/layer21_attention_weight_min": -24.22235107421875,
      "activations/layer22_attention_weight_max": 35.18388366699219,
      "activations/layer22_attention_weight_min": -23.192691802978516,
      "activations/layer23_attention_weight_max": 29.8195858001709,
      "activations/layer23_attention_weight_min": -18.626426696777344,
      "activations/layer2_attention_weight_max": 29.07404327392578,
      "activations/layer2_attention_weight_min": -28.18836784362793,
      "activations/layer3_attention_weight_max": 81.81622314453125,
      "activations/layer3_attention_weight_min": -83.80814361572266,
      "activations/layer4_attention_weight_max": 102.49732971191406,
      "activations/layer4_attention_weight_min": -93.43436431884766,
      "activations/layer5_attention_weight_max": 101.05657196044922,
      "activations/layer5_attention_weight_min": -89.87823486328125,
      "activations/layer6_attention_weight_max": 61.30026626586914,
      "activations/layer6_attention_weight_min": -54.73664855957031,
      "activations/layer7_attention_weight_max": 75.09327697753906,
      "activations/layer7_attention_weight_min": -70.03541564941406,
      "activations/layer8_attention_weight_max": 46.05882263183594,
      "activations/layer8_attention_weight_min": -43.02601623535156,
      "activations/layer9_attention_weight_max": 53.525508880615234,
      "activations/layer9_attention_weight_min": -50.86892318725586,
      "epoch": 5.92,
      "learning_rate": 0.00011299204545454545,
      "loss": 2.8428,
      "step": 101800
    },
    {
      "activations/layer0_attention_weight_max": 13.846607208251953,
      "activations/layer0_attention_weight_min": -14.782814979553223,
      "activations/layer10_attention_weight_max": 35.80622482299805,
      "activations/layer10_attention_weight_min": -37.225494384765625,
      "activations/layer11_attention_weight_max": 29.66518211364746,
      "activations/layer11_attention_weight_min": -28.686805725097656,
      "activations/layer12_attention_weight_max": 22.276832580566406,
      "activations/layer12_attention_weight_min": -23.698604583740234,
      "activations/layer13_attention_weight_max": 44.87535095214844,
      "activations/layer13_attention_weight_min": -35.14356994628906,
      "activations/layer14_attention_weight_max": 46.659915924072266,
      "activations/layer14_attention_weight_min": -33.220458984375,
      "activations/layer15_attention_weight_max": 38.173561096191406,
      "activations/layer15_attention_weight_min": -29.23097038269043,
      "activations/layer16_attention_weight_max": 37.183773040771484,
      "activations/layer16_attention_weight_min": -27.107460021972656,
      "activations/layer17_attention_weight_max": 54.7686767578125,
      "activations/layer17_attention_weight_min": -40.721588134765625,
      "activations/layer18_attention_weight_max": 54.81382369995117,
      "activations/layer18_attention_weight_min": -35.33595657348633,
      "activations/layer19_attention_weight_max": 23.22702980041504,
      "activations/layer19_attention_weight_min": -18.845914840698242,
      "activations/layer1_attention_weight_max": 18.051069259643555,
      "activations/layer1_attention_weight_min": -15.03644847869873,
      "activations/layer20_attention_weight_max": 24.080707550048828,
      "activations/layer20_attention_weight_min": -19.153629302978516,
      "activations/layer21_attention_weight_max": 39.766876220703125,
      "activations/layer21_attention_weight_min": -26.16154670715332,
      "activations/layer22_attention_weight_max": 39.64860153198242,
      "activations/layer22_attention_weight_min": -23.29207420349121,
      "activations/layer23_attention_weight_max": 29.900409698486328,
      "activations/layer23_attention_weight_min": -19.29717445373535,
      "activations/layer2_attention_weight_max": 28.748090744018555,
      "activations/layer2_attention_weight_min": -26.911840438842773,
      "activations/layer3_attention_weight_max": 77.63018035888672,
      "activations/layer3_attention_weight_min": -79.17906188964844,
      "activations/layer4_attention_weight_max": 95.15733337402344,
      "activations/layer4_attention_weight_min": -89.0927963256836,
      "activations/layer5_attention_weight_max": 90.1582260131836,
      "activations/layer5_attention_weight_min": -82.46304321289062,
      "activations/layer6_attention_weight_max": 52.87800598144531,
      "activations/layer6_attention_weight_min": -49.72600173950195,
      "activations/layer7_attention_weight_max": 72.60749053955078,
      "activations/layer7_attention_weight_min": -68.72294616699219,
      "activations/layer8_attention_weight_max": 40.92512512207031,
      "activations/layer8_attention_weight_min": -39.822628021240234,
      "activations/layer9_attention_weight_max": 51.84175491333008,
      "activations/layer9_attention_weight_min": -48.953460693359375,
      "epoch": 5.92,
      "learning_rate": 0.00011297310606060606,
      "loss": 2.8391,
      "step": 101850
    },
    {
      "activations/layer0_attention_weight_max": 15.487736701965332,
      "activations/layer0_attention_weight_min": -14.705737113952637,
      "activations/layer10_attention_weight_max": 40.534942626953125,
      "activations/layer10_attention_weight_min": -36.33369827270508,
      "activations/layer11_attention_weight_max": 38.44817352294922,
      "activations/layer11_attention_weight_min": -29.52721405029297,
      "activations/layer12_attention_weight_max": 23.942646026611328,
      "activations/layer12_attention_weight_min": -23.901325225830078,
      "activations/layer13_attention_weight_max": 57.81984329223633,
      "activations/layer13_attention_weight_min": -37.63926315307617,
      "activations/layer14_attention_weight_max": 63.151241302490234,
      "activations/layer14_attention_weight_min": -33.87368392944336,
      "activations/layer15_attention_weight_max": 49.73601531982422,
      "activations/layer15_attention_weight_min": -30.592626571655273,
      "activations/layer16_attention_weight_max": 39.83956527709961,
      "activations/layer16_attention_weight_min": -27.7896728515625,
      "activations/layer17_attention_weight_max": 60.41926574707031,
      "activations/layer17_attention_weight_min": -41.87049865722656,
      "activations/layer18_attention_weight_max": 55.22536087036133,
      "activations/layer18_attention_weight_min": -34.22553634643555,
      "activations/layer19_attention_weight_max": 25.5620174407959,
      "activations/layer19_attention_weight_min": -19.404294967651367,
      "activations/layer1_attention_weight_max": 17.68132972717285,
      "activations/layer1_attention_weight_min": -14.975369453430176,
      "activations/layer20_attention_weight_max": 24.197446823120117,
      "activations/layer20_attention_weight_min": -20.951824188232422,
      "activations/layer21_attention_weight_max": 51.3857536315918,
      "activations/layer21_attention_weight_min": -26.52586555480957,
      "activations/layer22_attention_weight_max": 42.87392807006836,
      "activations/layer22_attention_weight_min": -22.20109748840332,
      "activations/layer23_attention_weight_max": 34.27442932128906,
      "activations/layer23_attention_weight_min": -16.999202728271484,
      "activations/layer2_attention_weight_max": 30.484695434570312,
      "activations/layer2_attention_weight_min": -28.82400894165039,
      "activations/layer3_attention_weight_max": 79.64395141601562,
      "activations/layer3_attention_weight_min": -85.60929107666016,
      "activations/layer4_attention_weight_max": 99.44469451904297,
      "activations/layer4_attention_weight_min": -98.01750183105469,
      "activations/layer5_attention_weight_max": 100.02928924560547,
      "activations/layer5_attention_weight_min": -96.02752685546875,
      "activations/layer6_attention_weight_max": 58.95875549316406,
      "activations/layer6_attention_weight_min": -58.83655548095703,
      "activations/layer7_attention_weight_max": 75.20398712158203,
      "activations/layer7_attention_weight_min": -75.52783203125,
      "activations/layer8_attention_weight_max": 45.917205810546875,
      "activations/layer8_attention_weight_min": -46.547515869140625,
      "activations/layer9_attention_weight_max": 63.738826751708984,
      "activations/layer9_attention_weight_min": -48.76900863647461,
      "epoch": 5.92,
      "learning_rate": 0.00011295416666666665,
      "loss": 2.8452,
      "step": 101900
    },
    {
      "activations/layer0_attention_weight_max": 13.876471519470215,
      "activations/layer0_attention_weight_min": -15.164774894714355,
      "activations/layer10_attention_weight_max": 37.63420104980469,
      "activations/layer10_attention_weight_min": -35.1406135559082,
      "activations/layer11_attention_weight_max": 30.109067916870117,
      "activations/layer11_attention_weight_min": -29.741689682006836,
      "activations/layer12_attention_weight_max": 26.441495895385742,
      "activations/layer12_attention_weight_min": -29.10717010498047,
      "activations/layer13_attention_weight_max": 42.93172073364258,
      "activations/layer13_attention_weight_min": -34.145423889160156,
      "activations/layer14_attention_weight_max": 39.90834045410156,
      "activations/layer14_attention_weight_min": -29.535503387451172,
      "activations/layer15_attention_weight_max": 35.30603790283203,
      "activations/layer15_attention_weight_min": -30.154232025146484,
      "activations/layer16_attention_weight_max": 33.04408645629883,
      "activations/layer16_attention_weight_min": -27.290969848632812,
      "activations/layer17_attention_weight_max": 56.41730499267578,
      "activations/layer17_attention_weight_min": -43.49092102050781,
      "activations/layer18_attention_weight_max": 49.28179168701172,
      "activations/layer18_attention_weight_min": -34.168975830078125,
      "activations/layer19_attention_weight_max": 20.50457000732422,
      "activations/layer19_attention_weight_min": -20.113882064819336,
      "activations/layer1_attention_weight_max": 16.784860610961914,
      "activations/layer1_attention_weight_min": -16.341394424438477,
      "activations/layer20_attention_weight_max": 19.988323211669922,
      "activations/layer20_attention_weight_min": -20.471010208129883,
      "activations/layer21_attention_weight_max": 34.02251052856445,
      "activations/layer21_attention_weight_min": -26.12746238708496,
      "activations/layer22_attention_weight_max": 30.645463943481445,
      "activations/layer22_attention_weight_min": -26.122400283813477,
      "activations/layer23_attention_weight_max": 29.13088607788086,
      "activations/layer23_attention_weight_min": -19.92116355895996,
      "activations/layer2_attention_weight_max": 28.382495880126953,
      "activations/layer2_attention_weight_min": -28.305885314941406,
      "activations/layer3_attention_weight_max": 75.88996124267578,
      "activations/layer3_attention_weight_min": -85.84452056884766,
      "activations/layer4_attention_weight_max": 95.63582611083984,
      "activations/layer4_attention_weight_min": -90.45682525634766,
      "activations/layer5_attention_weight_max": 93.83628845214844,
      "activations/layer5_attention_weight_min": -86.35442352294922,
      "activations/layer6_attention_weight_max": 58.241966247558594,
      "activations/layer6_attention_weight_min": -53.64293670654297,
      "activations/layer7_attention_weight_max": 71.71819305419922,
      "activations/layer7_attention_weight_min": -68.83934020996094,
      "activations/layer8_attention_weight_max": 43.313873291015625,
      "activations/layer8_attention_weight_min": -43.76182174682617,
      "activations/layer9_attention_weight_max": 56.269222259521484,
      "activations/layer9_attention_weight_min": -48.05678176879883,
      "epoch": 5.92,
      "learning_rate": 0.00011293522727272726,
      "loss": 2.8334,
      "step": 101950
    },
    {
      "activations/layer0_attention_weight_max": 14.411521911621094,
      "activations/layer0_attention_weight_min": -14.4593505859375,
      "activations/layer10_attention_weight_max": 37.72492218017578,
      "activations/layer10_attention_weight_min": -34.33064651489258,
      "activations/layer11_attention_weight_max": 30.27322769165039,
      "activations/layer11_attention_weight_min": -30.375038146972656,
      "activations/layer12_attention_weight_max": 22.766679763793945,
      "activations/layer12_attention_weight_min": -26.74106788635254,
      "activations/layer13_attention_weight_max": 44.13478469848633,
      "activations/layer13_attention_weight_min": -31.325233459472656,
      "activations/layer14_attention_weight_max": 39.18027877807617,
      "activations/layer14_attention_weight_min": -30.318300247192383,
      "activations/layer15_attention_weight_max": 43.30887222290039,
      "activations/layer15_attention_weight_min": -30.37700653076172,
      "activations/layer16_attention_weight_max": 42.64720153808594,
      "activations/layer16_attention_weight_min": -26.0616397857666,
      "activations/layer17_attention_weight_max": 60.74812316894531,
      "activations/layer17_attention_weight_min": -41.78082275390625,
      "activations/layer18_attention_weight_max": 55.53276062011719,
      "activations/layer18_attention_weight_min": -38.05402374267578,
      "activations/layer19_attention_weight_max": 28.90159034729004,
      "activations/layer19_attention_weight_min": -21.29442024230957,
      "activations/layer1_attention_weight_max": 17.634244918823242,
      "activations/layer1_attention_weight_min": -16.3690185546875,
      "activations/layer20_attention_weight_max": 24.370487213134766,
      "activations/layer20_attention_weight_min": -24.233200073242188,
      "activations/layer21_attention_weight_max": 42.48666000366211,
      "activations/layer21_attention_weight_min": -25.503591537475586,
      "activations/layer22_attention_weight_max": 41.755584716796875,
      "activations/layer22_attention_weight_min": -24.549482345581055,
      "activations/layer23_attention_weight_max": 38.555171966552734,
      "activations/layer23_attention_weight_min": -17.755504608154297,
      "activations/layer2_attention_weight_max": 30.42732810974121,
      "activations/layer2_attention_weight_min": -28.646873474121094,
      "activations/layer3_attention_weight_max": 83.0028076171875,
      "activations/layer3_attention_weight_min": -82.03237915039062,
      "activations/layer4_attention_weight_max": 98.68711853027344,
      "activations/layer4_attention_weight_min": -96.68150329589844,
      "activations/layer5_attention_weight_max": 99.229736328125,
      "activations/layer5_attention_weight_min": -92.17747497558594,
      "activations/layer6_attention_weight_max": 56.605804443359375,
      "activations/layer6_attention_weight_min": -54.50586700439453,
      "activations/layer7_attention_weight_max": 75.42464447021484,
      "activations/layer7_attention_weight_min": -72.59111022949219,
      "activations/layer8_attention_weight_max": 44.465110778808594,
      "activations/layer8_attention_weight_min": -44.23604202270508,
      "activations/layer9_attention_weight_max": 54.86542892456055,
      "activations/layer9_attention_weight_min": -51.71820831298828,
      "epoch": 5.93,
      "learning_rate": 0.00011291628787878788,
      "loss": 2.8369,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.5595,
      "eval_samples_per_second": 501.668,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.5595,
      "eval_openwebtext_samples_per_second": 501.668,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 2.0279,
      "eval_wikitext_samples_per_second": 224.86,
      "step": 102000
    },
    {
      "epoch": 5.93,
      "eval_lambada_loss": 2.892578125,
      "eval_lambada_ppl": 18.039758452611142,
      "eval_lambada_runtime": 9.5799,
      "eval_lambada_samples_per_second": 508.251,
      "step": 102000
    },
    {
      "activations/layer0_attention_weight_max": 15.072915077209473,
      "activations/layer0_attention_weight_min": -14.932157516479492,
      "activations/layer10_attention_weight_max": 41.756690979003906,
      "activations/layer10_attention_weight_min": -36.153297424316406,
      "activations/layer11_attention_weight_max": 30.725116729736328,
      "activations/layer11_attention_weight_min": -29.816953659057617,
      "activations/layer12_attention_weight_max": 21.17478370666504,
      "activations/layer12_attention_weight_min": -25.091941833496094,
      "activations/layer13_attention_weight_max": 43.92876434326172,
      "activations/layer13_attention_weight_min": -33.67902755737305,
      "activations/layer14_attention_weight_max": 48.22991943359375,
      "activations/layer14_attention_weight_min": -35.82243728637695,
      "activations/layer15_attention_weight_max": 37.993186950683594,
      "activations/layer15_attention_weight_min": -31.37427520751953,
      "activations/layer16_attention_weight_max": 35.00765609741211,
      "activations/layer16_attention_weight_min": -27.817590713500977,
      "activations/layer17_attention_weight_max": 57.95272445678711,
      "activations/layer17_attention_weight_min": -41.23869323730469,
      "activations/layer18_attention_weight_max": 51.55341339111328,
      "activations/layer18_attention_weight_min": -36.80583572387695,
      "activations/layer19_attention_weight_max": 25.099225997924805,
      "activations/layer19_attention_weight_min": -21.340181350708008,
      "activations/layer1_attention_weight_max": 18.08795166015625,
      "activations/layer1_attention_weight_min": -14.192193984985352,
      "activations/layer20_attention_weight_max": 27.810630798339844,
      "activations/layer20_attention_weight_min": -20.703659057617188,
      "activations/layer21_attention_weight_max": 41.84934997558594,
      "activations/layer21_attention_weight_min": -24.367685317993164,
      "activations/layer22_attention_weight_max": 38.840267181396484,
      "activations/layer22_attention_weight_min": -24.931901931762695,
      "activations/layer23_attention_weight_max": 35.026668548583984,
      "activations/layer23_attention_weight_min": -20.065977096557617,
      "activations/layer2_attention_weight_max": 29.09174156188965,
      "activations/layer2_attention_weight_min": -27.058744430541992,
      "activations/layer3_attention_weight_max": 76.52618408203125,
      "activations/layer3_attention_weight_min": -82.26250457763672,
      "activations/layer4_attention_weight_max": 93.9569091796875,
      "activations/layer4_attention_weight_min": -87.04844665527344,
      "activations/layer5_attention_weight_max": 94.74282836914062,
      "activations/layer5_attention_weight_min": -85.59124755859375,
      "activations/layer6_attention_weight_max": 57.29417037963867,
      "activations/layer6_attention_weight_min": -52.36483383178711,
      "activations/layer7_attention_weight_max": 70.32191467285156,
      "activations/layer7_attention_weight_min": -68.23880004882812,
      "activations/layer8_attention_weight_max": 45.92839431762695,
      "activations/layer8_attention_weight_min": -42.8386116027832,
      "activations/layer9_attention_weight_max": 56.45150375366211,
      "activations/layer9_attention_weight_min": -46.34841537475586,
      "epoch": 5.93,
      "learning_rate": 0.00011289734848484847,
      "loss": 2.8388,
      "step": 102050
    },
    {
      "activations/layer0_attention_weight_max": 15.965066909790039,
      "activations/layer0_attention_weight_min": -14.741057395935059,
      "activations/layer10_attention_weight_max": 39.779808044433594,
      "activations/layer10_attention_weight_min": -35.22407913208008,
      "activations/layer11_attention_weight_max": 33.839820861816406,
      "activations/layer11_attention_weight_min": -29.158409118652344,
      "activations/layer12_attention_weight_max": 29.405811309814453,
      "activations/layer12_attention_weight_min": -25.949167251586914,
      "activations/layer13_attention_weight_max": 48.86217498779297,
      "activations/layer13_attention_weight_min": -34.765045166015625,
      "activations/layer14_attention_weight_max": 48.215824127197266,
      "activations/layer14_attention_weight_min": -30.51847267150879,
      "activations/layer15_attention_weight_max": 42.28700637817383,
      "activations/layer15_attention_weight_min": -30.23990821838379,
      "activations/layer16_attention_weight_max": 37.89993667602539,
      "activations/layer16_attention_weight_min": -27.315685272216797,
      "activations/layer17_attention_weight_max": 64.34637451171875,
      "activations/layer17_attention_weight_min": -42.82093048095703,
      "activations/layer18_attention_weight_max": 56.08369445800781,
      "activations/layer18_attention_weight_min": -35.739742279052734,
      "activations/layer19_attention_weight_max": 24.480525970458984,
      "activations/layer19_attention_weight_min": -22.336048126220703,
      "activations/layer1_attention_weight_max": 16.330547332763672,
      "activations/layer1_attention_weight_min": -15.115933418273926,
      "activations/layer20_attention_weight_max": 24.25673484802246,
      "activations/layer20_attention_weight_min": -25.684001922607422,
      "activations/layer21_attention_weight_max": 44.641658782958984,
      "activations/layer21_attention_weight_min": -27.827733993530273,
      "activations/layer22_attention_weight_max": 38.39817810058594,
      "activations/layer22_attention_weight_min": -26.801958084106445,
      "activations/layer23_attention_weight_max": 36.06367874145508,
      "activations/layer23_attention_weight_min": -23.4453125,
      "activations/layer2_attention_weight_max": 30.028684616088867,
      "activations/layer2_attention_weight_min": -29.47064208984375,
      "activations/layer3_attention_weight_max": 79.6978988647461,
      "activations/layer3_attention_weight_min": -84.50497436523438,
      "activations/layer4_attention_weight_max": 97.94658660888672,
      "activations/layer4_attention_weight_min": -92.37908172607422,
      "activations/layer5_attention_weight_max": 94.13435363769531,
      "activations/layer5_attention_weight_min": -85.91944885253906,
      "activations/layer6_attention_weight_max": 57.466957092285156,
      "activations/layer6_attention_weight_min": -52.47457504272461,
      "activations/layer7_attention_weight_max": 75.58698272705078,
      "activations/layer7_attention_weight_min": -71.3573989868164,
      "activations/layer8_attention_weight_max": 45.13042449951172,
      "activations/layer8_attention_weight_min": -43.67265701293945,
      "activations/layer9_attention_weight_max": 62.96559524536133,
      "activations/layer9_attention_weight_min": -48.277095794677734,
      "epoch": 5.93,
      "learning_rate": 0.00011287840909090908,
      "loss": 2.8447,
      "step": 102100
    },
    {
      "activations/layer0_attention_weight_max": 15.022216796875,
      "activations/layer0_attention_weight_min": -15.125712394714355,
      "activations/layer10_attention_weight_max": 43.03267288208008,
      "activations/layer10_attention_weight_min": -35.092464447021484,
      "activations/layer11_attention_weight_max": 32.91515350341797,
      "activations/layer11_attention_weight_min": -29.20437240600586,
      "activations/layer12_attention_weight_max": 23.08643341064453,
      "activations/layer12_attention_weight_min": -22.916351318359375,
      "activations/layer13_attention_weight_max": 52.29802322387695,
      "activations/layer13_attention_weight_min": -33.247127532958984,
      "activations/layer14_attention_weight_max": 45.250946044921875,
      "activations/layer14_attention_weight_min": -29.8073673248291,
      "activations/layer15_attention_weight_max": 46.874168395996094,
      "activations/layer15_attention_weight_min": -29.953981399536133,
      "activations/layer16_attention_weight_max": 37.80836486816406,
      "activations/layer16_attention_weight_min": -27.065351486206055,
      "activations/layer17_attention_weight_max": 65.46160125732422,
      "activations/layer17_attention_weight_min": -38.66693878173828,
      "activations/layer18_attention_weight_max": 54.62125015258789,
      "activations/layer18_attention_weight_min": -33.85234451293945,
      "activations/layer19_attention_weight_max": 26.999805450439453,
      "activations/layer19_attention_weight_min": -19.903900146484375,
      "activations/layer1_attention_weight_max": 17.5237979888916,
      "activations/layer1_attention_weight_min": -16.23065185546875,
      "activations/layer20_attention_weight_max": 28.571250915527344,
      "activations/layer20_attention_weight_min": -22.78998374938965,
      "activations/layer21_attention_weight_max": 49.54132080078125,
      "activations/layer21_attention_weight_min": -26.1353759765625,
      "activations/layer22_attention_weight_max": 38.29277801513672,
      "activations/layer22_attention_weight_min": -26.14521026611328,
      "activations/layer23_attention_weight_max": 38.83186340332031,
      "activations/layer23_attention_weight_min": -19.147811889648438,
      "activations/layer2_attention_weight_max": 29.67428970336914,
      "activations/layer2_attention_weight_min": -27.403209686279297,
      "activations/layer3_attention_weight_max": 83.58547973632812,
      "activations/layer3_attention_weight_min": -85.26866149902344,
      "activations/layer4_attention_weight_max": 101.17826080322266,
      "activations/layer4_attention_weight_min": -93.845458984375,
      "activations/layer5_attention_weight_max": 101.47079467773438,
      "activations/layer5_attention_weight_min": -86.4715576171875,
      "activations/layer6_attention_weight_max": 60.66620635986328,
      "activations/layer6_attention_weight_min": -52.13802719116211,
      "activations/layer7_attention_weight_max": 72.8578109741211,
      "activations/layer7_attention_weight_min": -69.9589614868164,
      "activations/layer8_attention_weight_max": 47.223182678222656,
      "activations/layer8_attention_weight_min": -44.15644454956055,
      "activations/layer9_attention_weight_max": 52.127418518066406,
      "activations/layer9_attention_weight_min": -49.135066986083984,
      "epoch": 5.94,
      "learning_rate": 0.00011285946969696968,
      "loss": 2.843,
      "step": 102150
    },
    {
      "activations/layer0_attention_weight_max": 14.745726585388184,
      "activations/layer0_attention_weight_min": -14.715811729431152,
      "activations/layer10_attention_weight_max": 37.020164489746094,
      "activations/layer10_attention_weight_min": -34.86273193359375,
      "activations/layer11_attention_weight_max": 29.337738037109375,
      "activations/layer11_attention_weight_min": -27.058490753173828,
      "activations/layer12_attention_weight_max": 21.001089096069336,
      "activations/layer12_attention_weight_min": -24.934202194213867,
      "activations/layer13_attention_weight_max": 39.30666732788086,
      "activations/layer13_attention_weight_min": -32.79045104980469,
      "activations/layer14_attention_weight_max": 34.15835189819336,
      "activations/layer14_attention_weight_min": -30.41901206970215,
      "activations/layer15_attention_weight_max": 34.47887420654297,
      "activations/layer15_attention_weight_min": -29.805574417114258,
      "activations/layer16_attention_weight_max": 33.1350212097168,
      "activations/layer16_attention_weight_min": -28.432567596435547,
      "activations/layer17_attention_weight_max": 50.466304779052734,
      "activations/layer17_attention_weight_min": -44.10212326049805,
      "activations/layer18_attention_weight_max": 45.62989044189453,
      "activations/layer18_attention_weight_min": -36.931766510009766,
      "activations/layer19_attention_weight_max": 20.210979461669922,
      "activations/layer19_attention_weight_min": -22.674283981323242,
      "activations/layer1_attention_weight_max": 18.365392684936523,
      "activations/layer1_attention_weight_min": -17.120532989501953,
      "activations/layer20_attention_weight_max": 21.035253524780273,
      "activations/layer20_attention_weight_min": -21.816131591796875,
      "activations/layer21_attention_weight_max": 37.15673828125,
      "activations/layer21_attention_weight_min": -25.421714782714844,
      "activations/layer22_attention_weight_max": 31.19192886352539,
      "activations/layer22_attention_weight_min": -29.27410125732422,
      "activations/layer23_attention_weight_max": 28.506895065307617,
      "activations/layer23_attention_weight_min": -20.143381118774414,
      "activations/layer2_attention_weight_max": 30.153419494628906,
      "activations/layer2_attention_weight_min": -28.236553192138672,
      "activations/layer3_attention_weight_max": 76.69564056396484,
      "activations/layer3_attention_weight_min": -81.66268157958984,
      "activations/layer4_attention_weight_max": 93.60169982910156,
      "activations/layer4_attention_weight_min": -89.61183166503906,
      "activations/layer5_attention_weight_max": 91.03305053710938,
      "activations/layer5_attention_weight_min": -84.30232238769531,
      "activations/layer6_attention_weight_max": 53.53911209106445,
      "activations/layer6_attention_weight_min": -50.532474517822266,
      "activations/layer7_attention_weight_max": 71.03802490234375,
      "activations/layer7_attention_weight_min": -66.06232452392578,
      "activations/layer8_attention_weight_max": 41.17778396606445,
      "activations/layer8_attention_weight_min": -40.31402587890625,
      "activations/layer9_attention_weight_max": 46.70503616333008,
      "activations/layer9_attention_weight_min": -49.78407669067383,
      "epoch": 5.94,
      "learning_rate": 0.00011284053030303029,
      "loss": 2.8456,
      "step": 102200
    },
    {
      "activations/layer0_attention_weight_max": 14.017561912536621,
      "activations/layer0_attention_weight_min": -14.770306587219238,
      "activations/layer10_attention_weight_max": 36.790565490722656,
      "activations/layer10_attention_weight_min": -33.269134521484375,
      "activations/layer11_attention_weight_max": 28.030574798583984,
      "activations/layer11_attention_weight_min": -27.225425720214844,
      "activations/layer12_attention_weight_max": 20.72367286682129,
      "activations/layer12_attention_weight_min": -27.254932403564453,
      "activations/layer13_attention_weight_max": 47.249210357666016,
      "activations/layer13_attention_weight_min": -32.6702766418457,
      "activations/layer14_attention_weight_max": 37.502681732177734,
      "activations/layer14_attention_weight_min": -28.52118682861328,
      "activations/layer15_attention_weight_max": 34.70487594604492,
      "activations/layer15_attention_weight_min": -30.75519561767578,
      "activations/layer16_attention_weight_max": 34.417240142822266,
      "activations/layer16_attention_weight_min": -25.535802841186523,
      "activations/layer17_attention_weight_max": 51.96055603027344,
      "activations/layer17_attention_weight_min": -39.525787353515625,
      "activations/layer18_attention_weight_max": 44.81243133544922,
      "activations/layer18_attention_weight_min": -33.591064453125,
      "activations/layer19_attention_weight_max": 19.403461456298828,
      "activations/layer19_attention_weight_min": -19.127208709716797,
      "activations/layer1_attention_weight_max": 16.643970489501953,
      "activations/layer1_attention_weight_min": -15.620049476623535,
      "activations/layer20_attention_weight_max": 20.656200408935547,
      "activations/layer20_attention_weight_min": -21.165557861328125,
      "activations/layer21_attention_weight_max": 34.66096115112305,
      "activations/layer21_attention_weight_min": -21.64430809020996,
      "activations/layer22_attention_weight_max": 28.41785430908203,
      "activations/layer22_attention_weight_min": -24.07168197631836,
      "activations/layer23_attention_weight_max": 27.587249755859375,
      "activations/layer23_attention_weight_min": -18.802703857421875,
      "activations/layer2_attention_weight_max": 30.96218490600586,
      "activations/layer2_attention_weight_min": -28.642292022705078,
      "activations/layer3_attention_weight_max": 80.83293151855469,
      "activations/layer3_attention_weight_min": -86.68108367919922,
      "activations/layer4_attention_weight_max": 99.76294708251953,
      "activations/layer4_attention_weight_min": -91.65594482421875,
      "activations/layer5_attention_weight_max": 92.87606048583984,
      "activations/layer5_attention_weight_min": -84.14686584472656,
      "activations/layer6_attention_weight_max": 55.23025894165039,
      "activations/layer6_attention_weight_min": -51.43172836303711,
      "activations/layer7_attention_weight_max": 72.41878509521484,
      "activations/layer7_attention_weight_min": -70.3722915649414,
      "activations/layer8_attention_weight_max": 43.943050384521484,
      "activations/layer8_attention_weight_min": -43.90202713012695,
      "activations/layer9_attention_weight_max": 51.8477783203125,
      "activations/layer9_attention_weight_min": -48.73334503173828,
      "epoch": 5.94,
      "learning_rate": 0.0001128215909090909,
      "loss": 2.8328,
      "step": 102250
    },
    {
      "activations/layer0_attention_weight_max": 14.764883041381836,
      "activations/layer0_attention_weight_min": -14.990169525146484,
      "activations/layer10_attention_weight_max": 38.26924514770508,
      "activations/layer10_attention_weight_min": -33.89142990112305,
      "activations/layer11_attention_weight_max": 29.766098022460938,
      "activations/layer11_attention_weight_min": -28.53313446044922,
      "activations/layer12_attention_weight_max": 34.323978424072266,
      "activations/layer12_attention_weight_min": -24.372636795043945,
      "activations/layer13_attention_weight_max": 47.56984329223633,
      "activations/layer13_attention_weight_min": -31.24026870727539,
      "activations/layer14_attention_weight_max": 44.036773681640625,
      "activations/layer14_attention_weight_min": -31.389543533325195,
      "activations/layer15_attention_weight_max": 38.13412094116211,
      "activations/layer15_attention_weight_min": -29.72087860107422,
      "activations/layer16_attention_weight_max": 38.804359436035156,
      "activations/layer16_attention_weight_min": -29.85195541381836,
      "activations/layer17_attention_weight_max": 59.93525695800781,
      "activations/layer17_attention_weight_min": -39.35428237915039,
      "activations/layer18_attention_weight_max": 51.46105194091797,
      "activations/layer18_attention_weight_min": -36.5256233215332,
      "activations/layer19_attention_weight_max": 25.34050941467285,
      "activations/layer19_attention_weight_min": -23.076644897460938,
      "activations/layer1_attention_weight_max": 16.96783447265625,
      "activations/layer1_attention_weight_min": -17.64795684814453,
      "activations/layer20_attention_weight_max": 21.59385871887207,
      "activations/layer20_attention_weight_min": -20.9185791015625,
      "activations/layer21_attention_weight_max": 39.94810104370117,
      "activations/layer21_attention_weight_min": -30.065824508666992,
      "activations/layer22_attention_weight_max": 34.057254791259766,
      "activations/layer22_attention_weight_min": -25.60721206665039,
      "activations/layer23_attention_weight_max": 29.50393295288086,
      "activations/layer23_attention_weight_min": -18.784528732299805,
      "activations/layer2_attention_weight_max": 34.676116943359375,
      "activations/layer2_attention_weight_min": -31.150922775268555,
      "activations/layer3_attention_weight_max": 89.85275268554688,
      "activations/layer3_attention_weight_min": -96.65625,
      "activations/layer4_attention_weight_max": 106.1683120727539,
      "activations/layer4_attention_weight_min": -106.43782043457031,
      "activations/layer5_attention_weight_max": 96.89961242675781,
      "activations/layer5_attention_weight_min": -92.76527404785156,
      "activations/layer6_attention_weight_max": 56.754356384277344,
      "activations/layer6_attention_weight_min": -52.12809753417969,
      "activations/layer7_attention_weight_max": 75.4722900390625,
      "activations/layer7_attention_weight_min": -73.7989273071289,
      "activations/layer8_attention_weight_max": 42.5906867980957,
      "activations/layer8_attention_weight_min": -42.00986099243164,
      "activations/layer9_attention_weight_max": 48.78071212768555,
      "activations/layer9_attention_weight_min": -47.737709045410156,
      "epoch": 5.94,
      "learning_rate": 0.0001128026515151515,
      "loss": 2.835,
      "step": 102300
    },
    {
      "activations/layer0_attention_weight_max": 14.527594566345215,
      "activations/layer0_attention_weight_min": -14.45205020904541,
      "activations/layer10_attention_weight_max": 39.21114730834961,
      "activations/layer10_attention_weight_min": -34.72080993652344,
      "activations/layer11_attention_weight_max": 32.08423614501953,
      "activations/layer11_attention_weight_min": -30.077268600463867,
      "activations/layer12_attention_weight_max": 25.07411003112793,
      "activations/layer12_attention_weight_min": -24.522628784179688,
      "activations/layer13_attention_weight_max": 48.6489143371582,
      "activations/layer13_attention_weight_min": -36.47541046142578,
      "activations/layer14_attention_weight_max": 39.02024459838867,
      "activations/layer14_attention_weight_min": -30.597482681274414,
      "activations/layer15_attention_weight_max": 40.252079010009766,
      "activations/layer15_attention_weight_min": -29.86740493774414,
      "activations/layer16_attention_weight_max": 34.051876068115234,
      "activations/layer16_attention_weight_min": -25.599193572998047,
      "activations/layer17_attention_weight_max": 54.201820373535156,
      "activations/layer17_attention_weight_min": -39.674583435058594,
      "activations/layer18_attention_weight_max": 46.18881607055664,
      "activations/layer18_attention_weight_min": -35.29338073730469,
      "activations/layer19_attention_weight_max": 23.775026321411133,
      "activations/layer19_attention_weight_min": -21.28706169128418,
      "activations/layer1_attention_weight_max": 17.726076126098633,
      "activations/layer1_attention_weight_min": -14.11281681060791,
      "activations/layer20_attention_weight_max": 24.940690994262695,
      "activations/layer20_attention_weight_min": -22.57328987121582,
      "activations/layer21_attention_weight_max": 42.39524841308594,
      "activations/layer21_attention_weight_min": -26.16172981262207,
      "activations/layer22_attention_weight_max": 31.27114486694336,
      "activations/layer22_attention_weight_min": -27.289649963378906,
      "activations/layer23_attention_weight_max": 27.47359275817871,
      "activations/layer23_attention_weight_min": -17.77480697631836,
      "activations/layer2_attention_weight_max": 32.170631408691406,
      "activations/layer2_attention_weight_min": -30.479576110839844,
      "activations/layer3_attention_weight_max": 90.02387237548828,
      "activations/layer3_attention_weight_min": -88.65634155273438,
      "activations/layer4_attention_weight_max": 109.78242492675781,
      "activations/layer4_attention_weight_min": -99.8691635131836,
      "activations/layer5_attention_weight_max": 100.82830047607422,
      "activations/layer5_attention_weight_min": -92.66798400878906,
      "activations/layer6_attention_weight_max": 60.73719024658203,
      "activations/layer6_attention_weight_min": -59.38455581665039,
      "activations/layer7_attention_weight_max": 73.27783966064453,
      "activations/layer7_attention_weight_min": -71.1696548461914,
      "activations/layer8_attention_weight_max": 44.2491455078125,
      "activations/layer8_attention_weight_min": -43.319698333740234,
      "activations/layer9_attention_weight_max": 50.41764831542969,
      "activations/layer9_attention_weight_min": -49.19572067260742,
      "epoch": 5.95,
      "learning_rate": 0.00011278371212121212,
      "loss": 2.8476,
      "step": 102350
    },
    {
      "activations/layer0_attention_weight_max": 14.230154037475586,
      "activations/layer0_attention_weight_min": -14.924748420715332,
      "activations/layer10_attention_weight_max": 39.88078308105469,
      "activations/layer10_attention_weight_min": -35.842384338378906,
      "activations/layer11_attention_weight_max": 29.896648406982422,
      "activations/layer11_attention_weight_min": -28.618030548095703,
      "activations/layer12_attention_weight_max": 26.432086944580078,
      "activations/layer12_attention_weight_min": -26.78668785095215,
      "activations/layer13_attention_weight_max": 48.55634307861328,
      "activations/layer13_attention_weight_min": -32.385459899902344,
      "activations/layer14_attention_weight_max": 54.523380279541016,
      "activations/layer14_attention_weight_min": -33.9128532409668,
      "activations/layer15_attention_weight_max": 43.923397064208984,
      "activations/layer15_attention_weight_min": -29.744693756103516,
      "activations/layer16_attention_weight_max": 40.66157150268555,
      "activations/layer16_attention_weight_min": -28.30019760131836,
      "activations/layer17_attention_weight_max": 57.19089889526367,
      "activations/layer17_attention_weight_min": -41.51023864746094,
      "activations/layer18_attention_weight_max": 55.2978630065918,
      "activations/layer18_attention_weight_min": -35.43436813354492,
      "activations/layer19_attention_weight_max": 25.330772399902344,
      "activations/layer19_attention_weight_min": -20.299163818359375,
      "activations/layer1_attention_weight_max": 17.115455627441406,
      "activations/layer1_attention_weight_min": -15.70444393157959,
      "activations/layer20_attention_weight_max": 23.140268325805664,
      "activations/layer20_attention_weight_min": -24.776437759399414,
      "activations/layer21_attention_weight_max": 40.526268005371094,
      "activations/layer21_attention_weight_min": -24.2637882232666,
      "activations/layer22_attention_weight_max": 35.62798309326172,
      "activations/layer22_attention_weight_min": -27.556659698486328,
      "activations/layer23_attention_weight_max": 31.3883056640625,
      "activations/layer23_attention_weight_min": -19.022411346435547,
      "activations/layer2_attention_weight_max": 30.51122283935547,
      "activations/layer2_attention_weight_min": -28.541751861572266,
      "activations/layer3_attention_weight_max": 82.76119232177734,
      "activations/layer3_attention_weight_min": -88.18743896484375,
      "activations/layer4_attention_weight_max": 100.27294158935547,
      "activations/layer4_attention_weight_min": -93.23567962646484,
      "activations/layer5_attention_weight_max": 95.32213592529297,
      "activations/layer5_attention_weight_min": -87.15880584716797,
      "activations/layer6_attention_weight_max": 56.93406677246094,
      "activations/layer6_attention_weight_min": -55.202491760253906,
      "activations/layer7_attention_weight_max": 72.43345642089844,
      "activations/layer7_attention_weight_min": -69.60128021240234,
      "activations/layer8_attention_weight_max": 43.995033264160156,
      "activations/layer8_attention_weight_min": -44.64434814453125,
      "activations/layer9_attention_weight_max": 52.65143585205078,
      "activations/layer9_attention_weight_min": -48.286956787109375,
      "epoch": 5.95,
      "learning_rate": 0.00011276477272727272,
      "loss": 2.8445,
      "step": 102400
    },
    {
      "activations/layer0_attention_weight_max": 14.81605339050293,
      "activations/layer0_attention_weight_min": -14.914017677307129,
      "activations/layer10_attention_weight_max": 40.249549865722656,
      "activations/layer10_attention_weight_min": -36.74371337890625,
      "activations/layer11_attention_weight_max": 31.17694664001465,
      "activations/layer11_attention_weight_min": -28.369888305664062,
      "activations/layer12_attention_weight_max": 23.574951171875,
      "activations/layer12_attention_weight_min": -24.975950241088867,
      "activations/layer13_attention_weight_max": 49.117855072021484,
      "activations/layer13_attention_weight_min": -36.43850326538086,
      "activations/layer14_attention_weight_max": 50.9991569519043,
      "activations/layer14_attention_weight_min": -32.58757781982422,
      "activations/layer15_attention_weight_max": 38.133087158203125,
      "activations/layer15_attention_weight_min": -31.04275894165039,
      "activations/layer16_attention_weight_max": 33.51970291137695,
      "activations/layer16_attention_weight_min": -26.162874221801758,
      "activations/layer17_attention_weight_max": 54.376487731933594,
      "activations/layer17_attention_weight_min": -36.98862075805664,
      "activations/layer18_attention_weight_max": 47.31782531738281,
      "activations/layer18_attention_weight_min": -35.16379928588867,
      "activations/layer19_attention_weight_max": 23.024568557739258,
      "activations/layer19_attention_weight_min": -21.21303367614746,
      "activations/layer1_attention_weight_max": 17.42546272277832,
      "activations/layer1_attention_weight_min": -14.979239463806152,
      "activations/layer20_attention_weight_max": 21.853899002075195,
      "activations/layer20_attention_weight_min": -23.71038246154785,
      "activations/layer21_attention_weight_max": 44.934783935546875,
      "activations/layer21_attention_weight_min": -25.76849937438965,
      "activations/layer22_attention_weight_max": 35.103023529052734,
      "activations/layer22_attention_weight_min": -26.900842666625977,
      "activations/layer23_attention_weight_max": 31.267789840698242,
      "activations/layer23_attention_weight_min": -22.454999923706055,
      "activations/layer2_attention_weight_max": 32.15648651123047,
      "activations/layer2_attention_weight_min": -29.769699096679688,
      "activations/layer3_attention_weight_max": 83.37220764160156,
      "activations/layer3_attention_weight_min": -83.68585205078125,
      "activations/layer4_attention_weight_max": 97.41548156738281,
      "activations/layer4_attention_weight_min": -90.66693115234375,
      "activations/layer5_attention_weight_max": 94.96117401123047,
      "activations/layer5_attention_weight_min": -84.51651000976562,
      "activations/layer6_attention_weight_max": 55.56331253051758,
      "activations/layer6_attention_weight_min": -54.11699295043945,
      "activations/layer7_attention_weight_max": 70.45037078857422,
      "activations/layer7_attention_weight_min": -69.88591003417969,
      "activations/layer8_attention_weight_max": 44.098819732666016,
      "activations/layer8_attention_weight_min": -43.06077194213867,
      "activations/layer9_attention_weight_max": 53.129669189453125,
      "activations/layer9_attention_weight_min": -50.05753707885742,
      "epoch": 5.95,
      "learning_rate": 0.00011274583333333332,
      "loss": 2.8437,
      "step": 102450
    },
    {
      "activations/layer0_attention_weight_max": 14.961431503295898,
      "activations/layer0_attention_weight_min": -14.529762268066406,
      "activations/layer10_attention_weight_max": 39.066192626953125,
      "activations/layer10_attention_weight_min": -33.53656768798828,
      "activations/layer11_attention_weight_max": 31.95690155029297,
      "activations/layer11_attention_weight_min": -27.752201080322266,
      "activations/layer12_attention_weight_max": 31.873796463012695,
      "activations/layer12_attention_weight_min": -21.300678253173828,
      "activations/layer13_attention_weight_max": 49.36586380004883,
      "activations/layer13_attention_weight_min": -31.549388885498047,
      "activations/layer14_attention_weight_max": 42.233436584472656,
      "activations/layer14_attention_weight_min": -30.863494873046875,
      "activations/layer15_attention_weight_max": 41.52442932128906,
      "activations/layer15_attention_weight_min": -28.512516021728516,
      "activations/layer16_attention_weight_max": 35.0837287902832,
      "activations/layer16_attention_weight_min": -29.54745101928711,
      "activations/layer17_attention_weight_max": 62.33370590209961,
      "activations/layer17_attention_weight_min": -43.375816345214844,
      "activations/layer18_attention_weight_max": 53.49019241333008,
      "activations/layer18_attention_weight_min": -34.50648880004883,
      "activations/layer19_attention_weight_max": 27.53737449645996,
      "activations/layer19_attention_weight_min": -21.650156021118164,
      "activations/layer1_attention_weight_max": 17.24397087097168,
      "activations/layer1_attention_weight_min": -13.783492088317871,
      "activations/layer20_attention_weight_max": 24.239112854003906,
      "activations/layer20_attention_weight_min": -20.85820960998535,
      "activations/layer21_attention_weight_max": 38.151432037353516,
      "activations/layer21_attention_weight_min": -24.49309539794922,
      "activations/layer22_attention_weight_max": 37.0566291809082,
      "activations/layer22_attention_weight_min": -23.051979064941406,
      "activations/layer23_attention_weight_max": 37.016353607177734,
      "activations/layer23_attention_weight_min": -19.579713821411133,
      "activations/layer2_attention_weight_max": 31.337913513183594,
      "activations/layer2_attention_weight_min": -28.07184600830078,
      "activations/layer3_attention_weight_max": 80.19854736328125,
      "activations/layer3_attention_weight_min": -81.19340515136719,
      "activations/layer4_attention_weight_max": 96.59619140625,
      "activations/layer4_attention_weight_min": -91.42610931396484,
      "activations/layer5_attention_weight_max": 92.69756317138672,
      "activations/layer5_attention_weight_min": -86.10025024414062,
      "activations/layer6_attention_weight_max": 55.940452575683594,
      "activations/layer6_attention_weight_min": -50.953346252441406,
      "activations/layer7_attention_weight_max": 77.42279815673828,
      "activations/layer7_attention_weight_min": -68.68997955322266,
      "activations/layer8_attention_weight_max": 46.55805206298828,
      "activations/layer8_attention_weight_min": -41.527671813964844,
      "activations/layer9_attention_weight_max": 62.381282806396484,
      "activations/layer9_attention_weight_min": -47.08845138549805,
      "epoch": 5.96,
      "learning_rate": 0.00011272689393939394,
      "loss": 2.8489,
      "step": 102500
    },
    {
      "activations/layer0_attention_weight_max": 15.897245407104492,
      "activations/layer0_attention_weight_min": -14.812252044677734,
      "activations/layer10_attention_weight_max": 37.681182861328125,
      "activations/layer10_attention_weight_min": -34.80760192871094,
      "activations/layer11_attention_weight_max": 28.42770767211914,
      "activations/layer11_attention_weight_min": -29.44489288330078,
      "activations/layer12_attention_weight_max": 20.433012008666992,
      "activations/layer12_attention_weight_min": -23.6230525970459,
      "activations/layer13_attention_weight_max": 53.89323043823242,
      "activations/layer13_attention_weight_min": -39.61519241333008,
      "activations/layer14_attention_weight_max": 38.30234146118164,
      "activations/layer14_attention_weight_min": -30.470426559448242,
      "activations/layer15_attention_weight_max": 37.94706726074219,
      "activations/layer15_attention_weight_min": -29.951683044433594,
      "activations/layer16_attention_weight_max": 29.350725173950195,
      "activations/layer16_attention_weight_min": -26.502111434936523,
      "activations/layer17_attention_weight_max": 53.43352127075195,
      "activations/layer17_attention_weight_min": -39.42969512939453,
      "activations/layer18_attention_weight_max": 45.51921081542969,
      "activations/layer18_attention_weight_min": -33.89510726928711,
      "activations/layer19_attention_weight_max": 21.9420223236084,
      "activations/layer19_attention_weight_min": -19.19208335876465,
      "activations/layer1_attention_weight_max": 17.18963050842285,
      "activations/layer1_attention_weight_min": -14.160775184631348,
      "activations/layer20_attention_weight_max": 23.356639862060547,
      "activations/layer20_attention_weight_min": -20.759458541870117,
      "activations/layer21_attention_weight_max": 38.18915557861328,
      "activations/layer21_attention_weight_min": -28.15536880493164,
      "activations/layer22_attention_weight_max": 31.587020874023438,
      "activations/layer22_attention_weight_min": -23.485628128051758,
      "activations/layer23_attention_weight_max": 28.890819549560547,
      "activations/layer23_attention_weight_min": -21.095138549804688,
      "activations/layer2_attention_weight_max": 30.24694061279297,
      "activations/layer2_attention_weight_min": -29.36524200439453,
      "activations/layer3_attention_weight_max": 79.97006225585938,
      "activations/layer3_attention_weight_min": -83.1191635131836,
      "activations/layer4_attention_weight_max": 97.46533203125,
      "activations/layer4_attention_weight_min": -90.3915023803711,
      "activations/layer5_attention_weight_max": 93.94981384277344,
      "activations/layer5_attention_weight_min": -86.14723205566406,
      "activations/layer6_attention_weight_max": 56.364524841308594,
      "activations/layer6_attention_weight_min": -52.367774963378906,
      "activations/layer7_attention_weight_max": 71.8148193359375,
      "activations/layer7_attention_weight_min": -68.5547866821289,
      "activations/layer8_attention_weight_max": 44.01568603515625,
      "activations/layer8_attention_weight_min": -44.33647537231445,
      "activations/layer9_attention_weight_max": 51.233089447021484,
      "activations/layer9_attention_weight_min": -45.78135299682617,
      "epoch": 5.96,
      "learning_rate": 0.00011270795454545454,
      "loss": 2.8383,
      "step": 102550
    },
    {
      "activations/layer0_attention_weight_max": 16.032169342041016,
      "activations/layer0_attention_weight_min": -14.462936401367188,
      "activations/layer10_attention_weight_max": 35.44110107421875,
      "activations/layer10_attention_weight_min": -35.39468765258789,
      "activations/layer11_attention_weight_max": 28.023983001708984,
      "activations/layer11_attention_weight_min": -27.99616050720215,
      "activations/layer12_attention_weight_max": 21.028913497924805,
      "activations/layer12_attention_weight_min": -24.820404052734375,
      "activations/layer13_attention_weight_max": 47.39580535888672,
      "activations/layer13_attention_weight_min": -37.27498245239258,
      "activations/layer14_attention_weight_max": 44.66170120239258,
      "activations/layer14_attention_weight_min": -31.53927230834961,
      "activations/layer15_attention_weight_max": 37.13729476928711,
      "activations/layer15_attention_weight_min": -28.524532318115234,
      "activations/layer16_attention_weight_max": 35.32876968383789,
      "activations/layer16_attention_weight_min": -26.898902893066406,
      "activations/layer17_attention_weight_max": 54.0366325378418,
      "activations/layer17_attention_weight_min": -41.8311882019043,
      "activations/layer18_attention_weight_max": 48.623321533203125,
      "activations/layer18_attention_weight_min": -33.389732360839844,
      "activations/layer19_attention_weight_max": 25.774337768554688,
      "activations/layer19_attention_weight_min": -19.859237670898438,
      "activations/layer1_attention_weight_max": 16.82267951965332,
      "activations/layer1_attention_weight_min": -14.674620628356934,
      "activations/layer20_attention_weight_max": 21.55794906616211,
      "activations/layer20_attention_weight_min": -20.06441879272461,
      "activations/layer21_attention_weight_max": 46.13912582397461,
      "activations/layer21_attention_weight_min": -24.898340225219727,
      "activations/layer22_attention_weight_max": 39.713191986083984,
      "activations/layer22_attention_weight_min": -23.772859573364258,
      "activations/layer23_attention_weight_max": 33.36126708984375,
      "activations/layer23_attention_weight_min": -18.969058990478516,
      "activations/layer2_attention_weight_max": 30.622543334960938,
      "activations/layer2_attention_weight_min": -28.09256362915039,
      "activations/layer3_attention_weight_max": 78.2186050415039,
      "activations/layer3_attention_weight_min": -85.33042907714844,
      "activations/layer4_attention_weight_max": 96.45113372802734,
      "activations/layer4_attention_weight_min": -94.68640899658203,
      "activations/layer5_attention_weight_max": 92.19005584716797,
      "activations/layer5_attention_weight_min": -91.79341888427734,
      "activations/layer6_attention_weight_max": 55.486595153808594,
      "activations/layer6_attention_weight_min": -50.798927307128906,
      "activations/layer7_attention_weight_max": 69.28121948242188,
      "activations/layer7_attention_weight_min": -67.93233489990234,
      "activations/layer8_attention_weight_max": 42.12376022338867,
      "activations/layer8_attention_weight_min": -41.6981201171875,
      "activations/layer9_attention_weight_max": 46.849815368652344,
      "activations/layer9_attention_weight_min": -46.48897171020508,
      "epoch": 5.96,
      "learning_rate": 0.00011268901515151514,
      "loss": 2.8361,
      "step": 102600
    },
    {
      "activations/layer0_attention_weight_max": 15.030946731567383,
      "activations/layer0_attention_weight_min": -14.894515037536621,
      "activations/layer10_attention_weight_max": 43.135101318359375,
      "activations/layer10_attention_weight_min": -38.71315383911133,
      "activations/layer11_attention_weight_max": 33.46234893798828,
      "activations/layer11_attention_weight_min": -32.6814079284668,
      "activations/layer12_attention_weight_max": 28.076852798461914,
      "activations/layer12_attention_weight_min": -24.83525848388672,
      "activations/layer13_attention_weight_max": 53.28157424926758,
      "activations/layer13_attention_weight_min": -35.952144622802734,
      "activations/layer14_attention_weight_max": 47.07011413574219,
      "activations/layer14_attention_weight_min": -31.235145568847656,
      "activations/layer15_attention_weight_max": 41.09503173828125,
      "activations/layer15_attention_weight_min": -30.459081649780273,
      "activations/layer16_attention_weight_max": 36.44654083251953,
      "activations/layer16_attention_weight_min": -26.40117073059082,
      "activations/layer17_attention_weight_max": 59.66012191772461,
      "activations/layer17_attention_weight_min": -42.075836181640625,
      "activations/layer18_attention_weight_max": 49.50127410888672,
      "activations/layer18_attention_weight_min": -33.954856872558594,
      "activations/layer19_attention_weight_max": 19.62228012084961,
      "activations/layer19_attention_weight_min": -20.40377426147461,
      "activations/layer1_attention_weight_max": 17.728702545166016,
      "activations/layer1_attention_weight_min": -14.712885856628418,
      "activations/layer20_attention_weight_max": 23.13302230834961,
      "activations/layer20_attention_weight_min": -23.172794342041016,
      "activations/layer21_attention_weight_max": 42.44417190551758,
      "activations/layer21_attention_weight_min": -26.61532974243164,
      "activations/layer22_attention_weight_max": 36.88991928100586,
      "activations/layer22_attention_weight_min": -25.68291473388672,
      "activations/layer23_attention_weight_max": 35.18408203125,
      "activations/layer23_attention_weight_min": -19.5572566986084,
      "activations/layer2_attention_weight_max": 31.75774383544922,
      "activations/layer2_attention_weight_min": -28.915897369384766,
      "activations/layer3_attention_weight_max": 87.57752227783203,
      "activations/layer3_attention_weight_min": -86.71163177490234,
      "activations/layer4_attention_weight_max": 105.92386627197266,
      "activations/layer4_attention_weight_min": -97.85515594482422,
      "activations/layer5_attention_weight_max": 100.93574523925781,
      "activations/layer5_attention_weight_min": -93.77727508544922,
      "activations/layer6_attention_weight_max": 61.756134033203125,
      "activations/layer6_attention_weight_min": -57.867919921875,
      "activations/layer7_attention_weight_max": 80.19440460205078,
      "activations/layer7_attention_weight_min": -75.25144958496094,
      "activations/layer8_attention_weight_max": 47.5268669128418,
      "activations/layer8_attention_weight_min": -46.32472610473633,
      "activations/layer9_attention_weight_max": 56.3892936706543,
      "activations/layer9_attention_weight_min": -52.39193344116211,
      "epoch": 5.96,
      "learning_rate": 0.00011267045454545453,
      "loss": 2.8404,
      "step": 102650
    },
    {
      "activations/layer0_attention_weight_max": 15.096542358398438,
      "activations/layer0_attention_weight_min": -14.941381454467773,
      "activations/layer10_attention_weight_max": 37.92075729370117,
      "activations/layer10_attention_weight_min": -36.14038848876953,
      "activations/layer11_attention_weight_max": 30.561973571777344,
      "activations/layer11_attention_weight_min": -30.608070373535156,
      "activations/layer12_attention_weight_max": 22.2302303314209,
      "activations/layer12_attention_weight_min": -25.94083595275879,
      "activations/layer13_attention_weight_max": 45.82652282714844,
      "activations/layer13_attention_weight_min": -34.75104522705078,
      "activations/layer14_attention_weight_max": 43.713687896728516,
      "activations/layer14_attention_weight_min": -30.199182510375977,
      "activations/layer15_attention_weight_max": 40.37026596069336,
      "activations/layer15_attention_weight_min": -28.81026840209961,
      "activations/layer16_attention_weight_max": 34.440799713134766,
      "activations/layer16_attention_weight_min": -26.626550674438477,
      "activations/layer17_attention_weight_max": 54.866092681884766,
      "activations/layer17_attention_weight_min": -41.05754089355469,
      "activations/layer18_attention_weight_max": 49.79121398925781,
      "activations/layer18_attention_weight_min": -33.82784652709961,
      "activations/layer19_attention_weight_max": 22.64555549621582,
      "activations/layer19_attention_weight_min": -21.742420196533203,
      "activations/layer1_attention_weight_max": 17.612730026245117,
      "activations/layer1_attention_weight_min": -13.531563758850098,
      "activations/layer20_attention_weight_max": 23.11971664428711,
      "activations/layer20_attention_weight_min": -23.39366912841797,
      "activations/layer21_attention_weight_max": 36.524566650390625,
      "activations/layer21_attention_weight_min": -23.048831939697266,
      "activations/layer22_attention_weight_max": 31.65373420715332,
      "activations/layer22_attention_weight_min": -21.825197219848633,
      "activations/layer23_attention_weight_max": 29.626567840576172,
      "activations/layer23_attention_weight_min": -18.28508758544922,
      "activations/layer2_attention_weight_max": 29.95772933959961,
      "activations/layer2_attention_weight_min": -28.711456298828125,
      "activations/layer3_attention_weight_max": 80.10882568359375,
      "activations/layer3_attention_weight_min": -84.52035522460938,
      "activations/layer4_attention_weight_max": 96.38915252685547,
      "activations/layer4_attention_weight_min": -95.6485824584961,
      "activations/layer5_attention_weight_max": 95.10051727294922,
      "activations/layer5_attention_weight_min": -89.40890502929688,
      "activations/layer6_attention_weight_max": 60.22355270385742,
      "activations/layer6_attention_weight_min": -52.02277374267578,
      "activations/layer7_attention_weight_max": 71.98417663574219,
      "activations/layer7_attention_weight_min": -71.04312133789062,
      "activations/layer8_attention_weight_max": 43.1795539855957,
      "activations/layer8_attention_weight_min": -43.461238861083984,
      "activations/layer9_attention_weight_max": 52.19384765625,
      "activations/layer9_attention_weight_min": -50.84162902832031,
      "epoch": 5.97,
      "learning_rate": 0.00011265151515151514,
      "loss": 2.8432,
      "step": 102700
    },
    {
      "activations/layer0_attention_weight_max": 15.955491065979004,
      "activations/layer0_attention_weight_min": -14.556469917297363,
      "activations/layer10_attention_weight_max": 39.52333068847656,
      "activations/layer10_attention_weight_min": -36.57493209838867,
      "activations/layer11_attention_weight_max": 31.426612854003906,
      "activations/layer11_attention_weight_min": -31.32113265991211,
      "activations/layer12_attention_weight_max": 19.643659591674805,
      "activations/layer12_attention_weight_min": -22.453466415405273,
      "activations/layer13_attention_weight_max": 50.22988510131836,
      "activations/layer13_attention_weight_min": -36.517181396484375,
      "activations/layer14_attention_weight_max": 39.14433288574219,
      "activations/layer14_attention_weight_min": -30.47311019897461,
      "activations/layer15_attention_weight_max": 38.35810089111328,
      "activations/layer15_attention_weight_min": -31.549396514892578,
      "activations/layer16_attention_weight_max": 35.177520751953125,
      "activations/layer16_attention_weight_min": -26.761396408081055,
      "activations/layer17_attention_weight_max": 58.30771255493164,
      "activations/layer17_attention_weight_min": -41.9173698425293,
      "activations/layer18_attention_weight_max": 53.982337951660156,
      "activations/layer18_attention_weight_min": -35.080238342285156,
      "activations/layer19_attention_weight_max": 23.498926162719727,
      "activations/layer19_attention_weight_min": -20.566875457763672,
      "activations/layer1_attention_weight_max": 16.1596736907959,
      "activations/layer1_attention_weight_min": -14.557766914367676,
      "activations/layer20_attention_weight_max": 24.177200317382812,
      "activations/layer20_attention_weight_min": -20.230405807495117,
      "activations/layer21_attention_weight_max": 42.300514221191406,
      "activations/layer21_attention_weight_min": -27.47318458557129,
      "activations/layer22_attention_weight_max": 35.12234115600586,
      "activations/layer22_attention_weight_min": -24.620718002319336,
      "activations/layer23_attention_weight_max": 32.296348571777344,
      "activations/layer23_attention_weight_min": -18.21442413330078,
      "activations/layer2_attention_weight_max": 31.37234115600586,
      "activations/layer2_attention_weight_min": -29.32288360595703,
      "activations/layer3_attention_weight_max": 84.1383056640625,
      "activations/layer3_attention_weight_min": -84.98330688476562,
      "activations/layer4_attention_weight_max": 103.9783935546875,
      "activations/layer4_attention_weight_min": -97.66533660888672,
      "activations/layer5_attention_weight_max": 103.45497131347656,
      "activations/layer5_attention_weight_min": -93.89680480957031,
      "activations/layer6_attention_weight_max": 59.43524932861328,
      "activations/layer6_attention_weight_min": -54.24345397949219,
      "activations/layer7_attention_weight_max": 74.8812484741211,
      "activations/layer7_attention_weight_min": -73.484375,
      "activations/layer8_attention_weight_max": 45.38784408569336,
      "activations/layer8_attention_weight_min": -46.92816162109375,
      "activations/layer9_attention_weight_max": 51.65678787231445,
      "activations/layer9_attention_weight_min": -49.710079193115234,
      "epoch": 5.97,
      "learning_rate": 0.00011263257575757576,
      "loss": 2.8283,
      "step": 102750
    },
    {
      "activations/layer0_attention_weight_max": 15.56316089630127,
      "activations/layer0_attention_weight_min": -14.95368480682373,
      "activations/layer10_attention_weight_max": 40.68574142456055,
      "activations/layer10_attention_weight_min": -38.08565902709961,
      "activations/layer11_attention_weight_max": 32.12052917480469,
      "activations/layer11_attention_weight_min": -30.720983505249023,
      "activations/layer12_attention_weight_max": 23.63806915283203,
      "activations/layer12_attention_weight_min": -25.54627799987793,
      "activations/layer13_attention_weight_max": 48.814064025878906,
      "activations/layer13_attention_weight_min": -38.05189514160156,
      "activations/layer14_attention_weight_max": 60.81941223144531,
      "activations/layer14_attention_weight_min": -35.511253356933594,
      "activations/layer15_attention_weight_max": 35.25497055053711,
      "activations/layer15_attention_weight_min": -31.778854370117188,
      "activations/layer16_attention_weight_max": 36.948265075683594,
      "activations/layer16_attention_weight_min": -26.475160598754883,
      "activations/layer17_attention_weight_max": 56.27602005004883,
      "activations/layer17_attention_weight_min": -42.22377395629883,
      "activations/layer18_attention_weight_max": 47.181427001953125,
      "activations/layer18_attention_weight_min": -35.10246658325195,
      "activations/layer19_attention_weight_max": 18.675092697143555,
      "activations/layer19_attention_weight_min": -21.182580947875977,
      "activations/layer1_attention_weight_max": 18.173181533813477,
      "activations/layer1_attention_weight_min": -15.002642631530762,
      "activations/layer20_attention_weight_max": 19.63115692138672,
      "activations/layer20_attention_weight_min": -21.021400451660156,
      "activations/layer21_attention_weight_max": 41.72718048095703,
      "activations/layer21_attention_weight_min": -23.87425994873047,
      "activations/layer22_attention_weight_max": 33.19797897338867,
      "activations/layer22_attention_weight_min": -24.83066749572754,
      "activations/layer23_attention_weight_max": 30.390365600585938,
      "activations/layer23_attention_weight_min": -19.288265228271484,
      "activations/layer2_attention_weight_max": 31.695852279663086,
      "activations/layer2_attention_weight_min": -30.478490829467773,
      "activations/layer3_attention_weight_max": 83.32186126708984,
      "activations/layer3_attention_weight_min": -84.10472106933594,
      "activations/layer4_attention_weight_max": 101.13992309570312,
      "activations/layer4_attention_weight_min": -98.09346771240234,
      "activations/layer5_attention_weight_max": 97.28611755371094,
      "activations/layer5_attention_weight_min": -93.23381042480469,
      "activations/layer6_attention_weight_max": 61.1612663269043,
      "activations/layer6_attention_weight_min": -54.68680191040039,
      "activations/layer7_attention_weight_max": 84.6390380859375,
      "activations/layer7_attention_weight_min": -75.7501220703125,
      "activations/layer8_attention_weight_max": 48.93733596801758,
      "activations/layer8_attention_weight_min": -47.11013412475586,
      "activations/layer9_attention_weight_max": 62.06897735595703,
      "activations/layer9_attention_weight_min": -52.82788848876953,
      "epoch": 5.97,
      "learning_rate": 0.00011261363636363635,
      "loss": 2.8303,
      "step": 102800
    },
    {
      "activations/layer0_attention_weight_max": 13.910113334655762,
      "activations/layer0_attention_weight_min": -15.454336166381836,
      "activations/layer10_attention_weight_max": 36.32024002075195,
      "activations/layer10_attention_weight_min": -37.17031478881836,
      "activations/layer11_attention_weight_max": 30.736785888671875,
      "activations/layer11_attention_weight_min": -29.580411911010742,
      "activations/layer12_attention_weight_max": 26.672792434692383,
      "activations/layer12_attention_weight_min": -26.47789192199707,
      "activations/layer13_attention_weight_max": 44.47490310668945,
      "activations/layer13_attention_weight_min": -38.76015090942383,
      "activations/layer14_attention_weight_max": 47.564910888671875,
      "activations/layer14_attention_weight_min": -34.83675003051758,
      "activations/layer15_attention_weight_max": 40.076805114746094,
      "activations/layer15_attention_weight_min": -30.177099227905273,
      "activations/layer16_attention_weight_max": 41.16701889038086,
      "activations/layer16_attention_weight_min": -29.3802490234375,
      "activations/layer17_attention_weight_max": 54.14685821533203,
      "activations/layer17_attention_weight_min": -41.37333679199219,
      "activations/layer18_attention_weight_max": 45.74807357788086,
      "activations/layer18_attention_weight_min": -37.0453987121582,
      "activations/layer19_attention_weight_max": 23.475919723510742,
      "activations/layer19_attention_weight_min": -21.70290184020996,
      "activations/layer1_attention_weight_max": 17.417699813842773,
      "activations/layer1_attention_weight_min": -14.420075416564941,
      "activations/layer20_attention_weight_max": 23.485301971435547,
      "activations/layer20_attention_weight_min": -23.96103858947754,
      "activations/layer21_attention_weight_max": 38.20316696166992,
      "activations/layer21_attention_weight_min": -28.416296005249023,
      "activations/layer22_attention_weight_max": 34.4232063293457,
      "activations/layer22_attention_weight_min": -24.873273849487305,
      "activations/layer23_attention_weight_max": 33.18021011352539,
      "activations/layer23_attention_weight_min": -19.413557052612305,
      "activations/layer2_attention_weight_max": 32.04033660888672,
      "activations/layer2_attention_weight_min": -30.350975036621094,
      "activations/layer3_attention_weight_max": 82.46454620361328,
      "activations/layer3_attention_weight_min": -86.10159301757812,
      "activations/layer4_attention_weight_max": 100.7094497680664,
      "activations/layer4_attention_weight_min": -93.6634292602539,
      "activations/layer5_attention_weight_max": 97.41814422607422,
      "activations/layer5_attention_weight_min": -86.23885345458984,
      "activations/layer6_attention_weight_max": 55.99269104003906,
      "activations/layer6_attention_weight_min": -54.99992370605469,
      "activations/layer7_attention_weight_max": 76.72747039794922,
      "activations/layer7_attention_weight_min": -70.78638458251953,
      "activations/layer8_attention_weight_max": 44.18708038330078,
      "activations/layer8_attention_weight_min": -46.7353401184082,
      "activations/layer9_attention_weight_max": 64.70491790771484,
      "activations/layer9_attention_weight_min": -50.47212219238281,
      "epoch": 5.98,
      "learning_rate": 0.00011259469696969696,
      "loss": 2.8409,
      "step": 102850
    },
    {
      "activations/layer0_attention_weight_max": 15.142887115478516,
      "activations/layer0_attention_weight_min": -14.80919075012207,
      "activations/layer10_attention_weight_max": 40.17866897583008,
      "activations/layer10_attention_weight_min": -35.53581619262695,
      "activations/layer11_attention_weight_max": 31.50558853149414,
      "activations/layer11_attention_weight_min": -28.838726043701172,
      "activations/layer12_attention_weight_max": 25.433815002441406,
      "activations/layer12_attention_weight_min": -23.25244140625,
      "activations/layer13_attention_weight_max": 46.08792495727539,
      "activations/layer13_attention_weight_min": -32.713417053222656,
      "activations/layer14_attention_weight_max": 44.82645034790039,
      "activations/layer14_attention_weight_min": -29.737329483032227,
      "activations/layer15_attention_weight_max": 41.1251335144043,
      "activations/layer15_attention_weight_min": -30.395957946777344,
      "activations/layer16_attention_weight_max": 37.165672302246094,
      "activations/layer16_attention_weight_min": -26.988988876342773,
      "activations/layer17_attention_weight_max": 62.24020004272461,
      "activations/layer17_attention_weight_min": -42.72340774536133,
      "activations/layer18_attention_weight_max": 56.75859069824219,
      "activations/layer18_attention_weight_min": -38.532676696777344,
      "activations/layer19_attention_weight_max": 22.70198631286621,
      "activations/layer19_attention_weight_min": -20.455175399780273,
      "activations/layer1_attention_weight_max": 17.412349700927734,
      "activations/layer1_attention_weight_min": -14.400829315185547,
      "activations/layer20_attention_weight_max": 24.25411605834961,
      "activations/layer20_attention_weight_min": -20.09373664855957,
      "activations/layer21_attention_weight_max": 42.90934753417969,
      "activations/layer21_attention_weight_min": -26.063385009765625,
      "activations/layer22_attention_weight_max": 35.75327682495117,
      "activations/layer22_attention_weight_min": -23.53525733947754,
      "activations/layer23_attention_weight_max": 36.15022277832031,
      "activations/layer23_attention_weight_min": -20.24567985534668,
      "activations/layer2_attention_weight_max": 31.77215576171875,
      "activations/layer2_attention_weight_min": -28.834325790405273,
      "activations/layer3_attention_weight_max": 80.23809814453125,
      "activations/layer3_attention_weight_min": -81.5965347290039,
      "activations/layer4_attention_weight_max": 101.15057373046875,
      "activations/layer4_attention_weight_min": -91.5154037475586,
      "activations/layer5_attention_weight_max": 94.52949523925781,
      "activations/layer5_attention_weight_min": -83.10687255859375,
      "activations/layer6_attention_weight_max": 54.80485534667969,
      "activations/layer6_attention_weight_min": -53.435123443603516,
      "activations/layer7_attention_weight_max": 71.25114440917969,
      "activations/layer7_attention_weight_min": -67.38849639892578,
      "activations/layer8_attention_weight_max": 42.67748260498047,
      "activations/layer8_attention_weight_min": -40.82078170776367,
      "activations/layer9_attention_weight_max": 50.40571594238281,
      "activations/layer9_attention_weight_min": -47.43455505371094,
      "epoch": 5.98,
      "learning_rate": 0.00011257575757575756,
      "loss": 2.8633,
      "step": 102900
    },
    {
      "activations/layer0_attention_weight_max": 14.704629898071289,
      "activations/layer0_attention_weight_min": -14.825037002563477,
      "activations/layer10_attention_weight_max": 33.5870361328125,
      "activations/layer10_attention_weight_min": -33.799964904785156,
      "activations/layer11_attention_weight_max": 27.482913970947266,
      "activations/layer11_attention_weight_min": -29.447132110595703,
      "activations/layer12_attention_weight_max": 19.4953670501709,
      "activations/layer12_attention_weight_min": -24.25371742248535,
      "activations/layer13_attention_weight_max": 43.7580451965332,
      "activations/layer13_attention_weight_min": -34.389892578125,
      "activations/layer14_attention_weight_max": 44.898841857910156,
      "activations/layer14_attention_weight_min": -32.223052978515625,
      "activations/layer15_attention_weight_max": 39.45398712158203,
      "activations/layer15_attention_weight_min": -30.572038650512695,
      "activations/layer16_attention_weight_max": 29.38578987121582,
      "activations/layer16_attention_weight_min": -29.65703582763672,
      "activations/layer17_attention_weight_max": 51.66611862182617,
      "activations/layer17_attention_weight_min": -39.08782958984375,
      "activations/layer18_attention_weight_max": 43.320743560791016,
      "activations/layer18_attention_weight_min": -32.86307907104492,
      "activations/layer19_attention_weight_max": 19.08501434326172,
      "activations/layer19_attention_weight_min": -18.90009307861328,
      "activations/layer1_attention_weight_max": 17.909700393676758,
      "activations/layer1_attention_weight_min": -14.182615280151367,
      "activations/layer20_attention_weight_max": 17.84360122680664,
      "activations/layer20_attention_weight_min": -22.87276268005371,
      "activations/layer21_attention_weight_max": 30.69852066040039,
      "activations/layer21_attention_weight_min": -23.442276000976562,
      "activations/layer22_attention_weight_max": 29.52899169921875,
      "activations/layer22_attention_weight_min": -26.18171501159668,
      "activations/layer23_attention_weight_max": 27.745376586914062,
      "activations/layer23_attention_weight_min": -21.488189697265625,
      "activations/layer2_attention_weight_max": 30.562572479248047,
      "activations/layer2_attention_weight_min": -28.85504913330078,
      "activations/layer3_attention_weight_max": 79.44799041748047,
      "activations/layer3_attention_weight_min": -79.50668334960938,
      "activations/layer4_attention_weight_max": 95.47527313232422,
      "activations/layer4_attention_weight_min": -87.97429656982422,
      "activations/layer5_attention_weight_max": 88.558837890625,
      "activations/layer5_attention_weight_min": -81.12594604492188,
      "activations/layer6_attention_weight_max": 55.22540283203125,
      "activations/layer6_attention_weight_min": -53.82112503051758,
      "activations/layer7_attention_weight_max": 73.28353881835938,
      "activations/layer7_attention_weight_min": -66.9000015258789,
      "activations/layer8_attention_weight_max": 40.233028411865234,
      "activations/layer8_attention_weight_min": -42.519161224365234,
      "activations/layer9_attention_weight_max": 48.12630844116211,
      "activations/layer9_attention_weight_min": -49.43877029418945,
      "epoch": 5.98,
      "learning_rate": 0.00011255719696969697,
      "loss": 2.845,
      "step": 102950
    },
    {
      "activations/layer0_attention_weight_max": 14.847512245178223,
      "activations/layer0_attention_weight_min": -14.899950981140137,
      "activations/layer10_attention_weight_max": 39.30571365356445,
      "activations/layer10_attention_weight_min": -35.547183990478516,
      "activations/layer11_attention_weight_max": 31.838987350463867,
      "activations/layer11_attention_weight_min": -30.000205993652344,
      "activations/layer12_attention_weight_max": 23.970489501953125,
      "activations/layer12_attention_weight_min": -23.593881607055664,
      "activations/layer13_attention_weight_max": 50.526790618896484,
      "activations/layer13_attention_weight_min": -37.595115661621094,
      "activations/layer14_attention_weight_max": 49.282798767089844,
      "activations/layer14_attention_weight_min": -31.969892501831055,
      "activations/layer15_attention_weight_max": 41.141143798828125,
      "activations/layer15_attention_weight_min": -29.713504791259766,
      "activations/layer16_attention_weight_max": 37.679500579833984,
      "activations/layer16_attention_weight_min": -25.6198673248291,
      "activations/layer17_attention_weight_max": 61.33708953857422,
      "activations/layer17_attention_weight_min": -39.05403518676758,
      "activations/layer18_attention_weight_max": 48.344688415527344,
      "activations/layer18_attention_weight_min": -32.78321838378906,
      "activations/layer19_attention_weight_max": 19.631467819213867,
      "activations/layer19_attention_weight_min": -19.317174911499023,
      "activations/layer1_attention_weight_max": 17.43085479736328,
      "activations/layer1_attention_weight_min": -14.161042213439941,
      "activations/layer20_attention_weight_max": 22.524917602539062,
      "activations/layer20_attention_weight_min": -19.775657653808594,
      "activations/layer21_attention_weight_max": 40.492164611816406,
      "activations/layer21_attention_weight_min": -26.08641815185547,
      "activations/layer22_attention_weight_max": 35.896114349365234,
      "activations/layer22_attention_weight_min": -24.399242401123047,
      "activations/layer23_attention_weight_max": 32.99607849121094,
      "activations/layer23_attention_weight_min": -19.523820877075195,
      "activations/layer2_attention_weight_max": 30.680173873901367,
      "activations/layer2_attention_weight_min": -28.556997299194336,
      "activations/layer3_attention_weight_max": 82.9190444946289,
      "activations/layer3_attention_weight_min": -81.34232330322266,
      "activations/layer4_attention_weight_max": 96.98526000976562,
      "activations/layer4_attention_weight_min": -92.41569519042969,
      "activations/layer5_attention_weight_max": 93.47078704833984,
      "activations/layer5_attention_weight_min": -87.21144104003906,
      "activations/layer6_attention_weight_max": 55.06715393066406,
      "activations/layer6_attention_weight_min": -52.7270393371582,
      "activations/layer7_attention_weight_max": 71.70526885986328,
      "activations/layer7_attention_weight_min": -70.34740447998047,
      "activations/layer8_attention_weight_max": 43.67549514770508,
      "activations/layer8_attention_weight_min": -44.707305908203125,
      "activations/layer9_attention_weight_max": 51.915985107421875,
      "activations/layer9_attention_weight_min": -48.9905891418457,
      "epoch": 5.98,
      "learning_rate": 0.00011253825757575756,
      "loss": 2.8363,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.5173,
      "eval_samples_per_second": 504.148,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.5173,
      "eval_openwebtext_samples_per_second": 504.148,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_wikitext_loss": 3.021484375,
      "eval_wikitext_ppl": 20.521731031400577,
      "eval_wikitext_runtime": 2.0217,
      "eval_wikitext_samples_per_second": 225.554,
      "step": 103000
    },
    {
      "epoch": 5.98,
      "eval_lambada_loss": 2.888671875,
      "eval_lambada_ppl": 17.969428099556087,
      "eval_lambada_runtime": 9.6005,
      "eval_lambada_samples_per_second": 507.159,
      "step": 103000
    },
    {
      "activations/layer0_attention_weight_max": 14.185677528381348,
      "activations/layer0_attention_weight_min": -15.069418907165527,
      "activations/layer10_attention_weight_max": 39.104820251464844,
      "activations/layer10_attention_weight_min": -35.30011749267578,
      "activations/layer11_attention_weight_max": 31.72174644470215,
      "activations/layer11_attention_weight_min": -29.40335464477539,
      "activations/layer12_attention_weight_max": 22.530309677124023,
      "activations/layer12_attention_weight_min": -27.871742248535156,
      "activations/layer13_attention_weight_max": 54.147212982177734,
      "activations/layer13_attention_weight_min": -40.77555847167969,
      "activations/layer14_attention_weight_max": 53.133731842041016,
      "activations/layer14_attention_weight_min": -34.36744689941406,
      "activations/layer15_attention_weight_max": 40.11407470703125,
      "activations/layer15_attention_weight_min": -33.093101501464844,
      "activations/layer16_attention_weight_max": 32.58363342285156,
      "activations/layer16_attention_weight_min": -29.316524505615234,
      "activations/layer17_attention_weight_max": 53.79949951171875,
      "activations/layer17_attention_weight_min": -42.9370002746582,
      "activations/layer18_attention_weight_max": 46.5820198059082,
      "activations/layer18_attention_weight_min": -37.403289794921875,
      "activations/layer19_attention_weight_max": 20.795602798461914,
      "activations/layer19_attention_weight_min": -20.685182571411133,
      "activations/layer1_attention_weight_max": 16.499719619750977,
      "activations/layer1_attention_weight_min": -15.979204177856445,
      "activations/layer20_attention_weight_max": 24.188804626464844,
      "activations/layer20_attention_weight_min": -25.390151977539062,
      "activations/layer21_attention_weight_max": 46.57057571411133,
      "activations/layer21_attention_weight_min": -27.04697036743164,
      "activations/layer22_attention_weight_max": 32.76423645019531,
      "activations/layer22_attention_weight_min": -24.218303680419922,
      "activations/layer23_attention_weight_max": 31.956737518310547,
      "activations/layer23_attention_weight_min": -19.658695220947266,
      "activations/layer2_attention_weight_max": 32.91844940185547,
      "activations/layer2_attention_weight_min": -29.64541244506836,
      "activations/layer3_attention_weight_max": 84.5353012084961,
      "activations/layer3_attention_weight_min": -88.32756805419922,
      "activations/layer4_attention_weight_max": 103.24198150634766,
      "activations/layer4_attention_weight_min": -95.47943878173828,
      "activations/layer5_attention_weight_max": 98.07453918457031,
      "activations/layer5_attention_weight_min": -87.86485290527344,
      "activations/layer6_attention_weight_max": 57.814762115478516,
      "activations/layer6_attention_weight_min": -51.77357864379883,
      "activations/layer7_attention_weight_max": 74.76110076904297,
      "activations/layer7_attention_weight_min": -72.2291488647461,
      "activations/layer8_attention_weight_max": 45.68331527709961,
      "activations/layer8_attention_weight_min": -43.19347381591797,
      "activations/layer9_attention_weight_max": 58.70981979370117,
      "activations/layer9_attention_weight_min": -52.135189056396484,
      "epoch": 5.99,
      "learning_rate": 0.00011251931818181817,
      "loss": 2.8371,
      "step": 103050
    },
    {
      "activations/layer0_attention_weight_max": 14.482207298278809,
      "activations/layer0_attention_weight_min": -15.235772132873535,
      "activations/layer10_attention_weight_max": 38.43864440917969,
      "activations/layer10_attention_weight_min": -36.13078308105469,
      "activations/layer11_attention_weight_max": 29.393138885498047,
      "activations/layer11_attention_weight_min": -28.945362091064453,
      "activations/layer12_attention_weight_max": 23.20018768310547,
      "activations/layer12_attention_weight_min": -24.50831413269043,
      "activations/layer13_attention_weight_max": 45.52651596069336,
      "activations/layer13_attention_weight_min": -35.70246887207031,
      "activations/layer14_attention_weight_max": 38.04190444946289,
      "activations/layer14_attention_weight_min": -32.29624557495117,
      "activations/layer15_attention_weight_max": 34.76087188720703,
      "activations/layer15_attention_weight_min": -31.818008422851562,
      "activations/layer16_attention_weight_max": 36.34929656982422,
      "activations/layer16_attention_weight_min": -28.21953010559082,
      "activations/layer17_attention_weight_max": 54.64052200317383,
      "activations/layer17_attention_weight_min": -41.292198181152344,
      "activations/layer18_attention_weight_max": 48.740089416503906,
      "activations/layer18_attention_weight_min": -35.21760177612305,
      "activations/layer19_attention_weight_max": 25.24481773376465,
      "activations/layer19_attention_weight_min": -20.37626838684082,
      "activations/layer1_attention_weight_max": 18.8626708984375,
      "activations/layer1_attention_weight_min": -16.780282974243164,
      "activations/layer20_attention_weight_max": 21.66289710998535,
      "activations/layer20_attention_weight_min": -21.510547637939453,
      "activations/layer21_attention_weight_max": 41.58597183227539,
      "activations/layer21_attention_weight_min": -25.786893844604492,
      "activations/layer22_attention_weight_max": 37.433902740478516,
      "activations/layer22_attention_weight_min": -25.70865249633789,
      "activations/layer23_attention_weight_max": 33.622802734375,
      "activations/layer23_attention_weight_min": -19.406631469726562,
      "activations/layer2_attention_weight_max": 32.0015983581543,
      "activations/layer2_attention_weight_min": -30.92695426940918,
      "activations/layer3_attention_weight_max": 85.92493438720703,
      "activations/layer3_attention_weight_min": -89.32784271240234,
      "activations/layer4_attention_weight_max": 101.25629425048828,
      "activations/layer4_attention_weight_min": -94.2085189819336,
      "activations/layer5_attention_weight_max": 99.81822967529297,
      "activations/layer5_attention_weight_min": -89.91078186035156,
      "activations/layer6_attention_weight_max": 62.51533508300781,
      "activations/layer6_attention_weight_min": -54.52256393432617,
      "activations/layer7_attention_weight_max": 73.76399230957031,
      "activations/layer7_attention_weight_min": -75.57744598388672,
      "activations/layer8_attention_weight_max": 47.55180740356445,
      "activations/layer8_attention_weight_min": -45.31960678100586,
      "activations/layer9_attention_weight_max": 51.967769622802734,
      "activations/layer9_attention_weight_min": -48.77348327636719,
      "epoch": 5.99,
      "learning_rate": 0.00011250037878787877,
      "loss": 2.8451,
      "step": 103100
    },
    {
      "activations/layer0_attention_weight_max": 14.751867294311523,
      "activations/layer0_attention_weight_min": -15.183300971984863,
      "activations/layer10_attention_weight_max": 35.47164535522461,
      "activations/layer10_attention_weight_min": -38.995521545410156,
      "activations/layer11_attention_weight_max": 29.535057067871094,
      "activations/layer11_attention_weight_min": -27.948623657226562,
      "activations/layer12_attention_weight_max": 21.1377010345459,
      "activations/layer12_attention_weight_min": -22.759021759033203,
      "activations/layer13_attention_weight_max": 47.990901947021484,
      "activations/layer13_attention_weight_min": -35.574951171875,
      "activations/layer14_attention_weight_max": 41.08765411376953,
      "activations/layer14_attention_weight_min": -29.88604736328125,
      "activations/layer15_attention_weight_max": 38.08244323730469,
      "activations/layer15_attention_weight_min": -30.417530059814453,
      "activations/layer16_attention_weight_max": 32.75689697265625,
      "activations/layer16_attention_weight_min": -28.41483497619629,
      "activations/layer17_attention_weight_max": 49.54161834716797,
      "activations/layer17_attention_weight_min": -41.28529357910156,
      "activations/layer18_attention_weight_max": 42.01205062866211,
      "activations/layer18_attention_weight_min": -35.00436782836914,
      "activations/layer19_attention_weight_max": 18.691049575805664,
      "activations/layer19_attention_weight_min": -18.46503448486328,
      "activations/layer1_attention_weight_max": 16.848712921142578,
      "activations/layer1_attention_weight_min": -14.969170570373535,
      "activations/layer20_attention_weight_max": 20.775554656982422,
      "activations/layer20_attention_weight_min": -20.775049209594727,
      "activations/layer21_attention_weight_max": 37.86480712890625,
      "activations/layer21_attention_weight_min": -25.91896629333496,
      "activations/layer22_attention_weight_max": 30.897174835205078,
      "activations/layer22_attention_weight_min": -23.766998291015625,
      "activations/layer23_attention_weight_max": 28.111003875732422,
      "activations/layer23_attention_weight_min": -19.66335678100586,
      "activations/layer2_attention_weight_max": 28.751087188720703,
      "activations/layer2_attention_weight_min": -28.115951538085938,
      "activations/layer3_attention_weight_max": 79.66102600097656,
      "activations/layer3_attention_weight_min": -80.62397766113281,
      "activations/layer4_attention_weight_max": 95.16716003417969,
      "activations/layer4_attention_weight_min": -88.87025451660156,
      "activations/layer5_attention_weight_max": 92.48048400878906,
      "activations/layer5_attention_weight_min": -85.77059173583984,
      "activations/layer6_attention_weight_max": 54.89883804321289,
      "activations/layer6_attention_weight_min": -50.90912628173828,
      "activations/layer7_attention_weight_max": 72.92686462402344,
      "activations/layer7_attention_weight_min": -67.67546081542969,
      "activations/layer8_attention_weight_max": 40.64217758178711,
      "activations/layer8_attention_weight_min": -44.69208908081055,
      "activations/layer9_attention_weight_max": 52.44368362426758,
      "activations/layer9_attention_weight_min": -48.19633865356445,
      "epoch": 5.99,
      "learning_rate": 0.00011248143939393939,
      "loss": 2.8451,
      "step": 103150
    },
    {
      "activations/layer0_attention_weight_max": 14.210578918457031,
      "activations/layer0_attention_weight_min": -14.757613182067871,
      "activations/layer10_attention_weight_max": 42.20542907714844,
      "activations/layer10_attention_weight_min": -36.44776153564453,
      "activations/layer11_attention_weight_max": 32.81562042236328,
      "activations/layer11_attention_weight_min": -28.756149291992188,
      "activations/layer12_attention_weight_max": 21.050872802734375,
      "activations/layer12_attention_weight_min": -26.190799713134766,
      "activations/layer13_attention_weight_max": 53.914669036865234,
      "activations/layer13_attention_weight_min": -34.62458801269531,
      "activations/layer14_attention_weight_max": 54.626190185546875,
      "activations/layer14_attention_weight_min": -33.5688362121582,
      "activations/layer15_attention_weight_max": 49.719669342041016,
      "activations/layer15_attention_weight_min": -30.35597038269043,
      "activations/layer16_attention_weight_max": 40.39668655395508,
      "activations/layer16_attention_weight_min": -29.969322204589844,
      "activations/layer17_attention_weight_max": 69.30574035644531,
      "activations/layer17_attention_weight_min": -49.0701904296875,
      "activations/layer18_attention_weight_max": 59.86454772949219,
      "activations/layer18_attention_weight_min": -40.71394729614258,
      "activations/layer19_attention_weight_max": 25.90472412109375,
      "activations/layer19_attention_weight_min": -22.375274658203125,
      "activations/layer1_attention_weight_max": 16.703956604003906,
      "activations/layer1_attention_weight_min": -16.128145217895508,
      "activations/layer20_attention_weight_max": 29.435033798217773,
      "activations/layer20_attention_weight_min": -22.156225204467773,
      "activations/layer21_attention_weight_max": 49.595550537109375,
      "activations/layer21_attention_weight_min": -27.45257568359375,
      "activations/layer22_attention_weight_max": 40.43782043457031,
      "activations/layer22_attention_weight_min": -27.170055389404297,
      "activations/layer23_attention_weight_max": 36.524261474609375,
      "activations/layer23_attention_weight_min": -21.077951431274414,
      "activations/layer2_attention_weight_max": 29.570775985717773,
      "activations/layer2_attention_weight_min": -26.995502471923828,
      "activations/layer3_attention_weight_max": 82.70304870605469,
      "activations/layer3_attention_weight_min": -89.06625366210938,
      "activations/layer4_attention_weight_max": 104.74260711669922,
      "activations/layer4_attention_weight_min": -97.50555419921875,
      "activations/layer5_attention_weight_max": 100.59307098388672,
      "activations/layer5_attention_weight_min": -87.46992492675781,
      "activations/layer6_attention_weight_max": 57.996742248535156,
      "activations/layer6_attention_weight_min": -53.805335998535156,
      "activations/layer7_attention_weight_max": 81.62792205810547,
      "activations/layer7_attention_weight_min": -68.596923828125,
      "activations/layer8_attention_weight_max": 51.8259391784668,
      "activations/layer8_attention_weight_min": -46.32710647583008,
      "activations/layer9_attention_weight_max": 59.308082580566406,
      "activations/layer9_attention_weight_min": -47.68978500366211,
      "epoch": 6.0,
      "learning_rate": 0.00011246249999999999,
      "loss": 2.8304,
      "step": 103200
    },
    {
      "activations/layer0_attention_weight_max": 14.364346504211426,
      "activations/layer0_attention_weight_min": -15.07841682434082,
      "activations/layer10_attention_weight_max": 35.77401351928711,
      "activations/layer10_attention_weight_min": -34.357547760009766,
      "activations/layer11_attention_weight_max": 29.391204833984375,
      "activations/layer11_attention_weight_min": -27.846118927001953,
      "activations/layer12_attention_weight_max": 20.250810623168945,
      "activations/layer12_attention_weight_min": -26.79607582092285,
      "activations/layer13_attention_weight_max": 47.36396026611328,
      "activations/layer13_attention_weight_min": -38.555816650390625,
      "activations/layer14_attention_weight_max": 44.24688720703125,
      "activations/layer14_attention_weight_min": -29.67463493347168,
      "activations/layer15_attention_weight_max": 33.81868362426758,
      "activations/layer15_attention_weight_min": -28.869064331054688,
      "activations/layer16_attention_weight_max": 28.539339065551758,
      "activations/layer16_attention_weight_min": -27.61236000061035,
      "activations/layer17_attention_weight_max": 52.28683090209961,
      "activations/layer17_attention_weight_min": -38.868656158447266,
      "activations/layer18_attention_weight_max": 43.31619644165039,
      "activations/layer18_attention_weight_min": -33.83015060424805,
      "activations/layer19_attention_weight_max": 19.266752243041992,
      "activations/layer19_attention_weight_min": -18.986896514892578,
      "activations/layer1_attention_weight_max": 18.616592407226562,
      "activations/layer1_attention_weight_min": -14.716147422790527,
      "activations/layer20_attention_weight_max": 20.062053680419922,
      "activations/layer20_attention_weight_min": -22.127958297729492,
      "activations/layer21_attention_weight_max": 37.11517333984375,
      "activations/layer21_attention_weight_min": -23.50376319885254,
      "activations/layer22_attention_weight_max": 30.95292854309082,
      "activations/layer22_attention_weight_min": -24.373493194580078,
      "activations/layer23_attention_weight_max": 28.38107681274414,
      "activations/layer23_attention_weight_min": -18.586774826049805,
      "activations/layer2_attention_weight_max": 31.733694076538086,
      "activations/layer2_attention_weight_min": -27.9992733001709,
      "activations/layer3_attention_weight_max": 87.59394073486328,
      "activations/layer3_attention_weight_min": -90.13279724121094,
      "activations/layer4_attention_weight_max": 102.33920288085938,
      "activations/layer4_attention_weight_min": -96.16722106933594,
      "activations/layer5_attention_weight_max": 95.7990951538086,
      "activations/layer5_attention_weight_min": -87.18462371826172,
      "activations/layer6_attention_weight_max": 55.9852180480957,
      "activations/layer6_attention_weight_min": -52.20405960083008,
      "activations/layer7_attention_weight_max": 75.21400451660156,
      "activations/layer7_attention_weight_min": -67.06413269042969,
      "activations/layer8_attention_weight_max": 43.73969650268555,
      "activations/layer8_attention_weight_min": -42.290523529052734,
      "activations/layer9_attention_weight_max": 61.40376663208008,
      "activations/layer9_attention_weight_min": -48.83558654785156,
      "epoch": 6.0,
      "learning_rate": 0.00011244356060606059,
      "loss": 2.8524,
      "step": 103250
    },
    {
      "activations/layer0_attention_weight_max": 15.447135925292969,
      "activations/layer0_attention_weight_min": -14.904277801513672,
      "activations/layer10_attention_weight_max": 40.25811767578125,
      "activations/layer10_attention_weight_min": -37.738746643066406,
      "activations/layer11_attention_weight_max": 31.192434310913086,
      "activations/layer11_attention_weight_min": -30.0394344329834,
      "activations/layer12_attention_weight_max": 21.369604110717773,
      "activations/layer12_attention_weight_min": -23.718257904052734,
      "activations/layer13_attention_weight_max": 46.34251022338867,
      "activations/layer13_attention_weight_min": -36.35392379760742,
      "activations/layer14_attention_weight_max": 46.731346130371094,
      "activations/layer14_attention_weight_min": -31.607009887695312,
      "activations/layer15_attention_weight_max": 38.703086853027344,
      "activations/layer15_attention_weight_min": -30.93004035949707,
      "activations/layer16_attention_weight_max": 35.299137115478516,
      "activations/layer16_attention_weight_min": -27.115530014038086,
      "activations/layer17_attention_weight_max": 61.70762252807617,
      "activations/layer17_attention_weight_min": -44.0156135559082,
      "activations/layer18_attention_weight_max": 49.36045455932617,
      "activations/layer18_attention_weight_min": -36.4991455078125,
      "activations/layer19_attention_weight_max": 23.814146041870117,
      "activations/layer19_attention_weight_min": -20.470273971557617,
      "activations/layer1_attention_weight_max": 17.03328514099121,
      "activations/layer1_attention_weight_min": -14.46645736694336,
      "activations/layer20_attention_weight_max": 25.343156814575195,
      "activations/layer20_attention_weight_min": -20.22940444946289,
      "activations/layer21_attention_weight_max": 42.57890319824219,
      "activations/layer21_attention_weight_min": -27.3039608001709,
      "activations/layer22_attention_weight_max": 35.53871154785156,
      "activations/layer22_attention_weight_min": -26.38897705078125,
      "activations/layer23_attention_weight_max": 34.646915435791016,
      "activations/layer23_attention_weight_min": -19.218555450439453,
      "activations/layer2_attention_weight_max": 31.909759521484375,
      "activations/layer2_attention_weight_min": -31.020160675048828,
      "activations/layer3_attention_weight_max": 84.91403198242188,
      "activations/layer3_attention_weight_min": -85.66287994384766,
      "activations/layer4_attention_weight_max": 110.51061248779297,
      "activations/layer4_attention_weight_min": -94.51068878173828,
      "activations/layer5_attention_weight_max": 99.02135467529297,
      "activations/layer5_attention_weight_min": -85.83460998535156,
      "activations/layer6_attention_weight_max": 57.78990173339844,
      "activations/layer6_attention_weight_min": -54.19248962402344,
      "activations/layer7_attention_weight_max": 74.40093994140625,
      "activations/layer7_attention_weight_min": -71.37960052490234,
      "activations/layer8_attention_weight_max": 44.92071533203125,
      "activations/layer8_attention_weight_min": -45.56188201904297,
      "activations/layer9_attention_weight_max": 50.8083610534668,
      "activations/layer9_attention_weight_min": -48.1003532409668,
      "epoch": 6.0,
      "learning_rate": 0.00011242424242424242,
      "loss": 2.8823,
      "step": 103300
    },
    {
      "activations/layer0_attention_weight_max": 14.755290031433105,
      "activations/layer0_attention_weight_min": -15.096338272094727,
      "activations/layer10_attention_weight_max": 39.344539642333984,
      "activations/layer10_attention_weight_min": -34.98429870605469,
      "activations/layer11_attention_weight_max": 31.001113891601562,
      "activations/layer11_attention_weight_min": -28.562419891357422,
      "activations/layer12_attention_weight_max": 19.91200828552246,
      "activations/layer12_attention_weight_min": -22.137266159057617,
      "activations/layer13_attention_weight_max": 45.78467559814453,
      "activations/layer13_attention_weight_min": -35.754329681396484,
      "activations/layer14_attention_weight_max": 37.534297943115234,
      "activations/layer14_attention_weight_min": -31.34043312072754,
      "activations/layer15_attention_weight_max": 39.520652770996094,
      "activations/layer15_attention_weight_min": -29.937761306762695,
      "activations/layer16_attention_weight_max": 40.521087646484375,
      "activations/layer16_attention_weight_min": -26.416343688964844,
      "activations/layer17_attention_weight_max": 58.08358383178711,
      "activations/layer17_attention_weight_min": -40.16962432861328,
      "activations/layer18_attention_weight_max": 49.40142059326172,
      "activations/layer18_attention_weight_min": -36.39360809326172,
      "activations/layer19_attention_weight_max": 21.766521453857422,
      "activations/layer19_attention_weight_min": -18.953550338745117,
      "activations/layer1_attention_weight_max": 18.53920555114746,
      "activations/layer1_attention_weight_min": -14.535150527954102,
      "activations/layer20_attention_weight_max": 22.391916275024414,
      "activations/layer20_attention_weight_min": -20.554134368896484,
      "activations/layer21_attention_weight_max": 37.10390090942383,
      "activations/layer21_attention_weight_min": -26.52482795715332,
      "activations/layer22_attention_weight_max": 30.31130599975586,
      "activations/layer22_attention_weight_min": -22.99106788635254,
      "activations/layer23_attention_weight_max": 32.930694580078125,
      "activations/layer23_attention_weight_min": -20.05134391784668,
      "activations/layer2_attention_weight_max": 31.69062614440918,
      "activations/layer2_attention_weight_min": -29.410934448242188,
      "activations/layer3_attention_weight_max": 80.6633071899414,
      "activations/layer3_attention_weight_min": -86.38709259033203,
      "activations/layer4_attention_weight_max": 101.21074676513672,
      "activations/layer4_attention_weight_min": -95.6573486328125,
      "activations/layer5_attention_weight_max": 97.93592834472656,
      "activations/layer5_attention_weight_min": -89.84620666503906,
      "activations/layer6_attention_weight_max": 59.63772964477539,
      "activations/layer6_attention_weight_min": -54.218605041503906,
      "activations/layer7_attention_weight_max": 71.62891387939453,
      "activations/layer7_attention_weight_min": -70.30797576904297,
      "activations/layer8_attention_weight_max": 44.536128997802734,
      "activations/layer8_attention_weight_min": -43.51863479614258,
      "activations/layer9_attention_weight_max": 52.53013229370117,
      "activations/layer9_attention_weight_min": -46.58249282836914,
      "epoch": 6.01,
      "learning_rate": 0.00011240530303030302,
      "loss": 2.8213,
      "step": 103350
    },
    {
      "activations/layer0_attention_weight_max": 13.911640167236328,
      "activations/layer0_attention_weight_min": -14.54193115234375,
      "activations/layer10_attention_weight_max": 38.739601135253906,
      "activations/layer10_attention_weight_min": -35.062355041503906,
      "activations/layer11_attention_weight_max": 31.833059310913086,
      "activations/layer11_attention_weight_min": -28.332263946533203,
      "activations/layer12_attention_weight_max": 22.957626342773438,
      "activations/layer12_attention_weight_min": -24.420080184936523,
      "activations/layer13_attention_weight_max": 49.006431579589844,
      "activations/layer13_attention_weight_min": -31.949359893798828,
      "activations/layer14_attention_weight_max": 47.03493881225586,
      "activations/layer14_attention_weight_min": -33.153743743896484,
      "activations/layer15_attention_weight_max": 41.9234733581543,
      "activations/layer15_attention_weight_min": -30.078845977783203,
      "activations/layer16_attention_weight_max": 33.04468536376953,
      "activations/layer16_attention_weight_min": -25.56304168701172,
      "activations/layer17_attention_weight_max": 58.70911407470703,
      "activations/layer17_attention_weight_min": -43.95314407348633,
      "activations/layer18_attention_weight_max": 51.966609954833984,
      "activations/layer18_attention_weight_min": -35.8574333190918,
      "activations/layer19_attention_weight_max": 22.137924194335938,
      "activations/layer19_attention_weight_min": -19.448270797729492,
      "activations/layer1_attention_weight_max": 17.76032829284668,
      "activations/layer1_attention_weight_min": -15.003904342651367,
      "activations/layer20_attention_weight_max": 23.203824996948242,
      "activations/layer20_attention_weight_min": -21.908756256103516,
      "activations/layer21_attention_weight_max": 40.59611892700195,
      "activations/layer21_attention_weight_min": -26.23396873474121,
      "activations/layer22_attention_weight_max": 33.69655990600586,
      "activations/layer22_attention_weight_min": -23.8142032623291,
      "activations/layer23_attention_weight_max": 33.07917022705078,
      "activations/layer23_attention_weight_min": -17.01980209350586,
      "activations/layer2_attention_weight_max": 31.372180938720703,
      "activations/layer2_attention_weight_min": -29.032697677612305,
      "activations/layer3_attention_weight_max": 84.63021850585938,
      "activations/layer3_attention_weight_min": -88.78411102294922,
      "activations/layer4_attention_weight_max": 100.64730072021484,
      "activations/layer4_attention_weight_min": -101.80094909667969,
      "activations/layer5_attention_weight_max": 93.94828796386719,
      "activations/layer5_attention_weight_min": -88.87264251708984,
      "activations/layer6_attention_weight_max": 57.47208023071289,
      "activations/layer6_attention_weight_min": -51.14291763305664,
      "activations/layer7_attention_weight_max": 72.14732360839844,
      "activations/layer7_attention_weight_min": -67.40766143798828,
      "activations/layer8_attention_weight_max": 45.28337478637695,
      "activations/layer8_attention_weight_min": -46.02272033691406,
      "activations/layer9_attention_weight_max": 53.19440460205078,
      "activations/layer9_attention_weight_min": -47.65776443481445,
      "epoch": 6.01,
      "learning_rate": 0.00011238636363636363,
      "loss": 2.8256,
      "step": 103400
    },
    {
      "activations/layer0_attention_weight_max": 14.648838996887207,
      "activations/layer0_attention_weight_min": -14.676595687866211,
      "activations/layer10_attention_weight_max": 39.73506546020508,
      "activations/layer10_attention_weight_min": -35.17780303955078,
      "activations/layer11_attention_weight_max": 31.182262420654297,
      "activations/layer11_attention_weight_min": -27.504743576049805,
      "activations/layer12_attention_weight_max": 26.90505599975586,
      "activations/layer12_attention_weight_min": -25.111743927001953,
      "activations/layer13_attention_weight_max": 51.30459213256836,
      "activations/layer13_attention_weight_min": -31.97372817993164,
      "activations/layer14_attention_weight_max": 55.27290725708008,
      "activations/layer14_attention_weight_min": -29.851943969726562,
      "activations/layer15_attention_weight_max": 44.13997268676758,
      "activations/layer15_attention_weight_min": -30.2793025970459,
      "activations/layer16_attention_weight_max": 43.83852005004883,
      "activations/layer16_attention_weight_min": -25.74150276184082,
      "activations/layer17_attention_weight_max": 71.2148208618164,
      "activations/layer17_attention_weight_min": -40.898094177246094,
      "activations/layer18_attention_weight_max": 61.39101791381836,
      "activations/layer18_attention_weight_min": -34.48212432861328,
      "activations/layer19_attention_weight_max": 26.832616806030273,
      "activations/layer19_attention_weight_min": -20.48793601989746,
      "activations/layer1_attention_weight_max": 16.822505950927734,
      "activations/layer1_attention_weight_min": -15.693794250488281,
      "activations/layer20_attention_weight_max": 27.9256534576416,
      "activations/layer20_attention_weight_min": -23.544076919555664,
      "activations/layer21_attention_weight_max": 49.284053802490234,
      "activations/layer21_attention_weight_min": -25.31866455078125,
      "activations/layer22_attention_weight_max": 39.232749938964844,
      "activations/layer22_attention_weight_min": -21.921112060546875,
      "activations/layer23_attention_weight_max": 33.664581298828125,
      "activations/layer23_attention_weight_min": -18.51791763305664,
      "activations/layer2_attention_weight_max": 31.320003509521484,
      "activations/layer2_attention_weight_min": -28.167285919189453,
      "activations/layer3_attention_weight_max": 85.90156555175781,
      "activations/layer3_attention_weight_min": -89.05126953125,
      "activations/layer4_attention_weight_max": 102.68865966796875,
      "activations/layer4_attention_weight_min": -94.46217346191406,
      "activations/layer5_attention_weight_max": 95.64495086669922,
      "activations/layer5_attention_weight_min": -87.25624084472656,
      "activations/layer6_attention_weight_max": 61.818519592285156,
      "activations/layer6_attention_weight_min": -49.90892791748047,
      "activations/layer7_attention_weight_max": 83.63322448730469,
      "activations/layer7_attention_weight_min": -70.90156555175781,
      "activations/layer8_attention_weight_max": 42.403778076171875,
      "activations/layer8_attention_weight_min": -43.457576751708984,
      "activations/layer9_attention_weight_max": 52.5143928527832,
      "activations/layer9_attention_weight_min": -47.560176849365234,
      "epoch": 6.01,
      "learning_rate": 0.00011236780303030302,
      "loss": 2.848,
      "step": 103450
    },
    {
      "activations/layer0_attention_weight_max": 16.34639549255371,
      "activations/layer0_attention_weight_min": -15.0835599899292,
      "activations/layer10_attention_weight_max": 39.730865478515625,
      "activations/layer10_attention_weight_min": -36.044647216796875,
      "activations/layer11_attention_weight_max": 31.29963493347168,
      "activations/layer11_attention_weight_min": -29.072547912597656,
      "activations/layer12_attention_weight_max": 25.429529190063477,
      "activations/layer12_attention_weight_min": -29.5140323638916,
      "activations/layer13_attention_weight_max": 55.41257858276367,
      "activations/layer13_attention_weight_min": -34.862770080566406,
      "activations/layer14_attention_weight_max": 62.144779205322266,
      "activations/layer14_attention_weight_min": -36.72629928588867,
      "activations/layer15_attention_weight_max": 38.080528259277344,
      "activations/layer15_attention_weight_min": -29.677743911743164,
      "activations/layer16_attention_weight_max": 35.894203186035156,
      "activations/layer16_attention_weight_min": -27.12379264831543,
      "activations/layer17_attention_weight_max": 56.878910064697266,
      "activations/layer17_attention_weight_min": -41.22768783569336,
      "activations/layer18_attention_weight_max": 51.09708023071289,
      "activations/layer18_attention_weight_min": -35.8156623840332,
      "activations/layer19_attention_weight_max": 22.003965377807617,
      "activations/layer19_attention_weight_min": -20.92234230041504,
      "activations/layer1_attention_weight_max": 17.692813873291016,
      "activations/layer1_attention_weight_min": -15.639010429382324,
      "activations/layer20_attention_weight_max": 22.75218391418457,
      "activations/layer20_attention_weight_min": -24.08856773376465,
      "activations/layer21_attention_weight_max": 44.99641418457031,
      "activations/layer21_attention_weight_min": -24.52556037902832,
      "activations/layer22_attention_weight_max": 35.748382568359375,
      "activations/layer22_attention_weight_min": -26.93327522277832,
      "activations/layer23_attention_weight_max": 31.81843376159668,
      "activations/layer23_attention_weight_min": -20.181970596313477,
      "activations/layer2_attention_weight_max": 29.739185333251953,
      "activations/layer2_attention_weight_min": -28.942306518554688,
      "activations/layer3_attention_weight_max": 81.21817779541016,
      "activations/layer3_attention_weight_min": -83.37775421142578,
      "activations/layer4_attention_weight_max": 101.1024398803711,
      "activations/layer4_attention_weight_min": -95.54499816894531,
      "activations/layer5_attention_weight_max": 96.37741088867188,
      "activations/layer5_attention_weight_min": -86.25621795654297,
      "activations/layer6_attention_weight_max": 60.669830322265625,
      "activations/layer6_attention_weight_min": -53.09281539916992,
      "activations/layer7_attention_weight_max": 74.52307891845703,
      "activations/layer7_attention_weight_min": -66.7220687866211,
      "activations/layer8_attention_weight_max": 46.0303840637207,
      "activations/layer8_attention_weight_min": -44.845516204833984,
      "activations/layer9_attention_weight_max": 57.30316925048828,
      "activations/layer9_attention_weight_min": -50.92863845825195,
      "epoch": 6.01,
      "learning_rate": 0.00011234886363636364,
      "loss": 2.8257,
      "step": 103500
    },
    {
      "activations/layer0_attention_weight_max": 15.28141975402832,
      "activations/layer0_attention_weight_min": -15.45578384399414,
      "activations/layer10_attention_weight_max": 38.334747314453125,
      "activations/layer10_attention_weight_min": -34.735313415527344,
      "activations/layer11_attention_weight_max": 31.109642028808594,
      "activations/layer11_attention_weight_min": -29.762889862060547,
      "activations/layer12_attention_weight_max": 25.65597915649414,
      "activations/layer12_attention_weight_min": -24.01943588256836,
      "activations/layer13_attention_weight_max": 47.393428802490234,
      "activations/layer13_attention_weight_min": -36.02547836303711,
      "activations/layer14_attention_weight_max": 39.645408630371094,
      "activations/layer14_attention_weight_min": -29.010465621948242,
      "activations/layer15_attention_weight_max": 38.31575393676758,
      "activations/layer15_attention_weight_min": -29.294607162475586,
      "activations/layer16_attention_weight_max": 38.7945556640625,
      "activations/layer16_attention_weight_min": -25.976064682006836,
      "activations/layer17_attention_weight_max": 57.03522491455078,
      "activations/layer17_attention_weight_min": -41.2974853515625,
      "activations/layer18_attention_weight_max": 49.02768325805664,
      "activations/layer18_attention_weight_min": -35.66157150268555,
      "activations/layer19_attention_weight_max": 24.472776412963867,
      "activations/layer19_attention_weight_min": -20.436655044555664,
      "activations/layer1_attention_weight_max": 18.435739517211914,
      "activations/layer1_attention_weight_min": -14.251052856445312,
      "activations/layer20_attention_weight_max": 22.022031784057617,
      "activations/layer20_attention_weight_min": -21.496919631958008,
      "activations/layer21_attention_weight_max": 41.76752853393555,
      "activations/layer21_attention_weight_min": -27.517141342163086,
      "activations/layer22_attention_weight_max": 31.89685821533203,
      "activations/layer22_attention_weight_min": -27.13697624206543,
      "activations/layer23_attention_weight_max": 32.03143310546875,
      "activations/layer23_attention_weight_min": -20.399673461914062,
      "activations/layer2_attention_weight_max": 30.80990219116211,
      "activations/layer2_attention_weight_min": -29.192977905273438,
      "activations/layer3_attention_weight_max": 81.779296875,
      "activations/layer3_attention_weight_min": -84.76227569580078,
      "activations/layer4_attention_weight_max": 101.54698944091797,
      "activations/layer4_attention_weight_min": -94.05016326904297,
      "activations/layer5_attention_weight_max": 92.98493957519531,
      "activations/layer5_attention_weight_min": -89.65464782714844,
      "activations/layer6_attention_weight_max": 55.53928756713867,
      "activations/layer6_attention_weight_min": -54.52766799926758,
      "activations/layer7_attention_weight_max": 78.0290756225586,
      "activations/layer7_attention_weight_min": -71.5717544555664,
      "activations/layer8_attention_weight_max": 44.415618896484375,
      "activations/layer8_attention_weight_min": -44.72529602050781,
      "activations/layer9_attention_weight_max": 56.872474670410156,
      "activations/layer9_attention_weight_min": -48.55686569213867,
      "epoch": 6.02,
      "learning_rate": 0.00011232992424242423,
      "loss": 2.8172,
      "step": 103550
    },
    {
      "activations/layer0_attention_weight_max": 14.632689476013184,
      "activations/layer0_attention_weight_min": -14.682929992675781,
      "activations/layer10_attention_weight_max": 42.4220085144043,
      "activations/layer10_attention_weight_min": -37.42156982421875,
      "activations/layer11_attention_weight_max": 30.870338439941406,
      "activations/layer11_attention_weight_min": -30.979869842529297,
      "activations/layer12_attention_weight_max": 23.040964126586914,
      "activations/layer12_attention_weight_min": -27.130741119384766,
      "activations/layer13_attention_weight_max": 65.35201263427734,
      "activations/layer13_attention_weight_min": -40.629547119140625,
      "activations/layer14_attention_weight_max": 94.17571258544922,
      "activations/layer14_attention_weight_min": -46.08863067626953,
      "activations/layer15_attention_weight_max": 65.87383270263672,
      "activations/layer15_attention_weight_min": -36.1772346496582,
      "activations/layer16_attention_weight_max": 41.79733657836914,
      "activations/layer16_attention_weight_min": -27.018373489379883,
      "activations/layer17_attention_weight_max": 52.99976348876953,
      "activations/layer17_attention_weight_min": -41.710792541503906,
      "activations/layer18_attention_weight_max": 47.17974090576172,
      "activations/layer18_attention_weight_min": -35.10643768310547,
      "activations/layer19_attention_weight_max": 23.324583053588867,
      "activations/layer19_attention_weight_min": -20.532773971557617,
      "activations/layer1_attention_weight_max": 17.724742889404297,
      "activations/layer1_attention_weight_min": -16.732229232788086,
      "activations/layer20_attention_weight_max": 25.468358993530273,
      "activations/layer20_attention_weight_min": -22.0245418548584,
      "activations/layer21_attention_weight_max": 49.30712890625,
      "activations/layer21_attention_weight_min": -23.661264419555664,
      "activations/layer22_attention_weight_max": 36.34510803222656,
      "activations/layer22_attention_weight_min": -23.24208641052246,
      "activations/layer23_attention_weight_max": 28.75922393798828,
      "activations/layer23_attention_weight_min": -19.296709060668945,
      "activations/layer2_attention_weight_max": 30.790298461914062,
      "activations/layer2_attention_weight_min": -28.983654022216797,
      "activations/layer3_attention_weight_max": 78.92420959472656,
      "activations/layer3_attention_weight_min": -86.59878540039062,
      "activations/layer4_attention_weight_max": 96.26578521728516,
      "activations/layer4_attention_weight_min": -93.63872528076172,
      "activations/layer5_attention_weight_max": 90.90756225585938,
      "activations/layer5_attention_weight_min": -84.83624267578125,
      "activations/layer6_attention_weight_max": 53.47357940673828,
      "activations/layer6_attention_weight_min": -49.920711517333984,
      "activations/layer7_attention_weight_max": 70.254150390625,
      "activations/layer7_attention_weight_min": -69.62456512451172,
      "activations/layer8_attention_weight_max": 43.11193084716797,
      "activations/layer8_attention_weight_min": -41.561279296875,
      "activations/layer9_attention_weight_max": 52.936279296875,
      "activations/layer9_attention_weight_min": -49.47335433959961,
      "epoch": 6.02,
      "learning_rate": 0.00011231098484848484,
      "loss": 2.8274,
      "step": 103600
    },
    {
      "activations/layer0_attention_weight_max": 14.972652435302734,
      "activations/layer0_attention_weight_min": -14.985804557800293,
      "activations/layer10_attention_weight_max": 37.516056060791016,
      "activations/layer10_attention_weight_min": -35.40141296386719,
      "activations/layer11_attention_weight_max": 30.348674774169922,
      "activations/layer11_attention_weight_min": -27.639076232910156,
      "activations/layer12_attention_weight_max": 24.533849716186523,
      "activations/layer12_attention_weight_min": -27.48845672607422,
      "activations/layer13_attention_weight_max": 48.62649154663086,
      "activations/layer13_attention_weight_min": -33.13461685180664,
      "activations/layer14_attention_weight_max": 41.834381103515625,
      "activations/layer14_attention_weight_min": -29.95276641845703,
      "activations/layer15_attention_weight_max": 39.23143005371094,
      "activations/layer15_attention_weight_min": -30.4912109375,
      "activations/layer16_attention_weight_max": 36.21220397949219,
      "activations/layer16_attention_weight_min": -26.831878662109375,
      "activations/layer17_attention_weight_max": 56.91603469848633,
      "activations/layer17_attention_weight_min": -44.38340377807617,
      "activations/layer18_attention_weight_max": 50.346134185791016,
      "activations/layer18_attention_weight_min": -35.9361457824707,
      "activations/layer19_attention_weight_max": 21.924484252929688,
      "activations/layer19_attention_weight_min": -20.792888641357422,
      "activations/layer1_attention_weight_max": 17.24631690979004,
      "activations/layer1_attention_weight_min": -16.438982009887695,
      "activations/layer20_attention_weight_max": 23.3951473236084,
      "activations/layer20_attention_weight_min": -24.094013214111328,
      "activations/layer21_attention_weight_max": 41.765541076660156,
      "activations/layer21_attention_weight_min": -27.124406814575195,
      "activations/layer22_attention_weight_max": 36.04204177856445,
      "activations/layer22_attention_weight_min": -27.286659240722656,
      "activations/layer23_attention_weight_max": 32.620338439941406,
      "activations/layer23_attention_weight_min": -21.51527214050293,
      "activations/layer2_attention_weight_max": 30.678638458251953,
      "activations/layer2_attention_weight_min": -29.503660202026367,
      "activations/layer3_attention_weight_max": 84.74494171142578,
      "activations/layer3_attention_weight_min": -87.11515808105469,
      "activations/layer4_attention_weight_max": 102.28855895996094,
      "activations/layer4_attention_weight_min": -96.9757080078125,
      "activations/layer5_attention_weight_max": 95.35133361816406,
      "activations/layer5_attention_weight_min": -88.11265563964844,
      "activations/layer6_attention_weight_max": 56.422061920166016,
      "activations/layer6_attention_weight_min": -51.74137496948242,
      "activations/layer7_attention_weight_max": 69.52394104003906,
      "activations/layer7_attention_weight_min": -72.95556640625,
      "activations/layer8_attention_weight_max": 44.020362854003906,
      "activations/layer8_attention_weight_min": -42.24430465698242,
      "activations/layer9_attention_weight_max": 49.467464447021484,
      "activations/layer9_attention_weight_min": -49.21999740600586,
      "epoch": 6.02,
      "learning_rate": 0.00011229204545454544,
      "loss": 2.8281,
      "step": 103650
    },
    {
      "activations/layer0_attention_weight_max": 14.571821212768555,
      "activations/layer0_attention_weight_min": -14.118432998657227,
      "activations/layer10_attention_weight_max": 46.4644775390625,
      "activations/layer10_attention_weight_min": -39.97246170043945,
      "activations/layer11_attention_weight_max": 39.11969757080078,
      "activations/layer11_attention_weight_min": -30.609106063842773,
      "activations/layer12_attention_weight_max": 24.315568923950195,
      "activations/layer12_attention_weight_min": -26.83999252319336,
      "activations/layer13_attention_weight_max": 58.90744400024414,
      "activations/layer13_attention_weight_min": -34.22673797607422,
      "activations/layer14_attention_weight_max": 71.55660247802734,
      "activations/layer14_attention_weight_min": -36.259674072265625,
      "activations/layer15_attention_weight_max": 46.85901641845703,
      "activations/layer15_attention_weight_min": -30.971845626831055,
      "activations/layer16_attention_weight_max": 35.55164337158203,
      "activations/layer16_attention_weight_min": -26.13265609741211,
      "activations/layer17_attention_weight_max": 69.21163177490234,
      "activations/layer17_attention_weight_min": -41.3365478515625,
      "activations/layer18_attention_weight_max": 49.0247802734375,
      "activations/layer18_attention_weight_min": -33.3946647644043,
      "activations/layer19_attention_weight_max": 24.471269607543945,
      "activations/layer19_attention_weight_min": -20.740814208984375,
      "activations/layer1_attention_weight_max": 17.404033660888672,
      "activations/layer1_attention_weight_min": -15.728426933288574,
      "activations/layer20_attention_weight_max": 23.995534896850586,
      "activations/layer20_attention_weight_min": -20.660167694091797,
      "activations/layer21_attention_weight_max": 42.39311981201172,
      "activations/layer21_attention_weight_min": -26.73695945739746,
      "activations/layer22_attention_weight_max": 35.24762725830078,
      "activations/layer22_attention_weight_min": -23.65851593017578,
      "activations/layer23_attention_weight_max": 26.248884201049805,
      "activations/layer23_attention_weight_min": -20.09062957763672,
      "activations/layer2_attention_weight_max": 34.059295654296875,
      "activations/layer2_attention_weight_min": -31.442729949951172,
      "activations/layer3_attention_weight_max": 82.22847747802734,
      "activations/layer3_attention_weight_min": -84.06890869140625,
      "activations/layer4_attention_weight_max": 99.62469482421875,
      "activations/layer4_attention_weight_min": -99.03669738769531,
      "activations/layer5_attention_weight_max": 97.7828369140625,
      "activations/layer5_attention_weight_min": -90.20753479003906,
      "activations/layer6_attention_weight_max": 56.61325454711914,
      "activations/layer6_attention_weight_min": -52.7278938293457,
      "activations/layer7_attention_weight_max": 86.98160552978516,
      "activations/layer7_attention_weight_min": -76.98651123046875,
      "activations/layer8_attention_weight_max": 53.290863037109375,
      "activations/layer8_attention_weight_min": -47.31340408325195,
      "activations/layer9_attention_weight_max": 66.88203430175781,
      "activations/layer9_attention_weight_min": -60.53512191772461,
      "epoch": 6.03,
      "learning_rate": 0.00011227310606060604,
      "loss": 2.8459,
      "step": 103700
    },
    {
      "activations/layer0_attention_weight_max": 14.950119018554688,
      "activations/layer0_attention_weight_min": -13.94556999206543,
      "activations/layer10_attention_weight_max": 35.46379089355469,
      "activations/layer10_attention_weight_min": -35.1061897277832,
      "activations/layer11_attention_weight_max": 29.294090270996094,
      "activations/layer11_attention_weight_min": -27.529666900634766,
      "activations/layer12_attention_weight_max": 23.815265655517578,
      "activations/layer12_attention_weight_min": -26.027002334594727,
      "activations/layer13_attention_weight_max": 49.32796859741211,
      "activations/layer13_attention_weight_min": -37.731353759765625,
      "activations/layer14_attention_weight_max": 39.910919189453125,
      "activations/layer14_attention_weight_min": -30.470760345458984,
      "activations/layer15_attention_weight_max": 38.50922775268555,
      "activations/layer15_attention_weight_min": -30.303974151611328,
      "activations/layer16_attention_weight_max": 31.901758193969727,
      "activations/layer16_attention_weight_min": -25.768346786499023,
      "activations/layer17_attention_weight_max": 56.29741287231445,
      "activations/layer17_attention_weight_min": -42.477760314941406,
      "activations/layer18_attention_weight_max": 51.171775817871094,
      "activations/layer18_attention_weight_min": -34.29658508300781,
      "activations/layer19_attention_weight_max": 21.841920852661133,
      "activations/layer19_attention_weight_min": -22.243310928344727,
      "activations/layer1_attention_weight_max": 18.443889617919922,
      "activations/layer1_attention_weight_min": -13.981383323669434,
      "activations/layer20_attention_weight_max": 20.48448371887207,
      "activations/layer20_attention_weight_min": -25.36504554748535,
      "activations/layer21_attention_weight_max": 38.93038558959961,
      "activations/layer21_attention_weight_min": -26.356185913085938,
      "activations/layer22_attention_weight_max": 32.74125671386719,
      "activations/layer22_attention_weight_min": -26.433305740356445,
      "activations/layer23_attention_weight_max": 29.600990295410156,
      "activations/layer23_attention_weight_min": -18.640464782714844,
      "activations/layer2_attention_weight_max": 30.287832260131836,
      "activations/layer2_attention_weight_min": -27.259113311767578,
      "activations/layer3_attention_weight_max": 80.71340942382812,
      "activations/layer3_attention_weight_min": -80.46456909179688,
      "activations/layer4_attention_weight_max": 95.222900390625,
      "activations/layer4_attention_weight_min": -88.02879333496094,
      "activations/layer5_attention_weight_max": 90.08108520507812,
      "activations/layer5_attention_weight_min": -83.91973876953125,
      "activations/layer6_attention_weight_max": 56.48939514160156,
      "activations/layer6_attention_weight_min": -49.96109390258789,
      "activations/layer7_attention_weight_max": 70.27850341796875,
      "activations/layer7_attention_weight_min": -67.07198333740234,
      "activations/layer8_attention_weight_max": 39.79998779296875,
      "activations/layer8_attention_weight_min": -44.28171157836914,
      "activations/layer9_attention_weight_max": 47.89030456542969,
      "activations/layer9_attention_weight_min": -47.71980667114258,
      "epoch": 6.03,
      "learning_rate": 0.00011225416666666666,
      "loss": 2.8097,
      "step": 103750
    },
    {
      "activations/layer0_attention_weight_max": 14.905651092529297,
      "activations/layer0_attention_weight_min": -15.02511978149414,
      "activations/layer10_attention_weight_max": 43.14346694946289,
      "activations/layer10_attention_weight_min": -35.555938720703125,
      "activations/layer11_attention_weight_max": 32.17030334472656,
      "activations/layer11_attention_weight_min": -29.291900634765625,
      "activations/layer12_attention_weight_max": 24.571989059448242,
      "activations/layer12_attention_weight_min": -28.914472579956055,
      "activations/layer13_attention_weight_max": 55.047080993652344,
      "activations/layer13_attention_weight_min": -34.78561782836914,
      "activations/layer14_attention_weight_max": 60.18086624145508,
      "activations/layer14_attention_weight_min": -31.061492919921875,
      "activations/layer15_attention_weight_max": 53.4671516418457,
      "activations/layer15_attention_weight_min": -31.526315689086914,
      "activations/layer16_attention_weight_max": 45.19559097290039,
      "activations/layer16_attention_weight_min": -28.249547958374023,
      "activations/layer17_attention_weight_max": 66.20169830322266,
      "activations/layer17_attention_weight_min": -42.8372688293457,
      "activations/layer18_attention_weight_max": 66.5876693725586,
      "activations/layer18_attention_weight_min": -36.050262451171875,
      "activations/layer19_attention_weight_max": 30.947818756103516,
      "activations/layer19_attention_weight_min": -22.489057540893555,
      "activations/layer1_attention_weight_max": 17.11421012878418,
      "activations/layer1_attention_weight_min": -16.137035369873047,
      "activations/layer20_attention_weight_max": 30.139434814453125,
      "activations/layer20_attention_weight_min": -19.564882278442383,
      "activations/layer21_attention_weight_max": 52.91703796386719,
      "activations/layer21_attention_weight_min": -24.37242889404297,
      "activations/layer22_attention_weight_max": 37.317771911621094,
      "activations/layer22_attention_weight_min": -24.366310119628906,
      "activations/layer23_attention_weight_max": 37.476783752441406,
      "activations/layer23_attention_weight_min": -18.354948043823242,
      "activations/layer2_attention_weight_max": 30.711212158203125,
      "activations/layer2_attention_weight_min": -27.049787521362305,
      "activations/layer3_attention_weight_max": 81.86300659179688,
      "activations/layer3_attention_weight_min": -82.3480224609375,
      "activations/layer4_attention_weight_max": 99.06861114501953,
      "activations/layer4_attention_weight_min": -92.6800537109375,
      "activations/layer5_attention_weight_max": 96.77511596679688,
      "activations/layer5_attention_weight_min": -86.82923889160156,
      "activations/layer6_attention_weight_max": 56.01908493041992,
      "activations/layer6_attention_weight_min": -53.1618766784668,
      "activations/layer7_attention_weight_max": 70.18189239501953,
      "activations/layer7_attention_weight_min": -70.62987518310547,
      "activations/layer8_attention_weight_max": 45.14130783081055,
      "activations/layer8_attention_weight_min": -45.2376708984375,
      "activations/layer9_attention_weight_max": 50.42507553100586,
      "activations/layer9_attention_weight_min": -48.96064376831055,
      "epoch": 6.03,
      "learning_rate": 0.00011223522727272726,
      "loss": 2.8369,
      "step": 103800
    },
    {
      "activations/layer0_attention_weight_max": 14.200674057006836,
      "activations/layer0_attention_weight_min": -14.097967147827148,
      "activations/layer10_attention_weight_max": 40.27473449707031,
      "activations/layer10_attention_weight_min": -35.86102294921875,
      "activations/layer11_attention_weight_max": 30.314603805541992,
      "activations/layer11_attention_weight_min": -28.977069854736328,
      "activations/layer12_attention_weight_max": 19.185543060302734,
      "activations/layer12_attention_weight_min": -30.502864837646484,
      "activations/layer13_attention_weight_max": 52.77118682861328,
      "activations/layer13_attention_weight_min": -38.2160758972168,
      "activations/layer14_attention_weight_max": 40.93821334838867,
      "activations/layer14_attention_weight_min": -32.222816467285156,
      "activations/layer15_attention_weight_max": 37.457523345947266,
      "activations/layer15_attention_weight_min": -29.61583709716797,
      "activations/layer16_attention_weight_max": 33.52704620361328,
      "activations/layer16_attention_weight_min": -27.75908088684082,
      "activations/layer17_attention_weight_max": 55.6600227355957,
      "activations/layer17_attention_weight_min": -42.40884017944336,
      "activations/layer18_attention_weight_max": 47.59130096435547,
      "activations/layer18_attention_weight_min": -35.568138122558594,
      "activations/layer19_attention_weight_max": 20.749841690063477,
      "activations/layer19_attention_weight_min": -21.440982818603516,
      "activations/layer1_attention_weight_max": 17.154014587402344,
      "activations/layer1_attention_weight_min": -15.31343936920166,
      "activations/layer20_attention_weight_max": 21.951379776000977,
      "activations/layer20_attention_weight_min": -27.517332077026367,
      "activations/layer21_attention_weight_max": 36.34568786621094,
      "activations/layer21_attention_weight_min": -25.74641990661621,
      "activations/layer22_attention_weight_max": 31.802995681762695,
      "activations/layer22_attention_weight_min": -26.628753662109375,
      "activations/layer23_attention_weight_max": 30.491384506225586,
      "activations/layer23_attention_weight_min": -20.13912582397461,
      "activations/layer2_attention_weight_max": 32.00275802612305,
      "activations/layer2_attention_weight_min": -27.631725311279297,
      "activations/layer3_attention_weight_max": 80.81851959228516,
      "activations/layer3_attention_weight_min": -83.87777709960938,
      "activations/layer4_attention_weight_max": 99.85692596435547,
      "activations/layer4_attention_weight_min": -96.76863861083984,
      "activations/layer5_attention_weight_max": 94.45771789550781,
      "activations/layer5_attention_weight_min": -89.27897644042969,
      "activations/layer6_attention_weight_max": 60.38630676269531,
      "activations/layer6_attention_weight_min": -54.96216583251953,
      "activations/layer7_attention_weight_max": 70.89240264892578,
      "activations/layer7_attention_weight_min": -72.01508331298828,
      "activations/layer8_attention_weight_max": 47.053443908691406,
      "activations/layer8_attention_weight_min": -45.24884033203125,
      "activations/layer9_attention_weight_max": 49.813358306884766,
      "activations/layer9_attention_weight_min": -48.96570587158203,
      "epoch": 6.03,
      "learning_rate": 0.00011221628787878786,
      "loss": 2.8387,
      "step": 103850
    },
    {
      "activations/layer0_attention_weight_max": 14.063594818115234,
      "activations/layer0_attention_weight_min": -14.357674598693848,
      "activations/layer10_attention_weight_max": 33.919185638427734,
      "activations/layer10_attention_weight_min": -34.595848083496094,
      "activations/layer11_attention_weight_max": 30.378652572631836,
      "activations/layer11_attention_weight_min": -28.462963104248047,
      "activations/layer12_attention_weight_max": 22.012752532958984,
      "activations/layer12_attention_weight_min": -25.746253967285156,
      "activations/layer13_attention_weight_max": 48.8829460144043,
      "activations/layer13_attention_weight_min": -37.44429397583008,
      "activations/layer14_attention_weight_max": 40.288047790527344,
      "activations/layer14_attention_weight_min": -31.740848541259766,
      "activations/layer15_attention_weight_max": 36.44388961791992,
      "activations/layer15_attention_weight_min": -30.2054500579834,
      "activations/layer16_attention_weight_max": 34.466148376464844,
      "activations/layer16_attention_weight_min": -26.5222110748291,
      "activations/layer17_attention_weight_max": 56.48476791381836,
      "activations/layer17_attention_weight_min": -42.0799674987793,
      "activations/layer18_attention_weight_max": 48.118099212646484,
      "activations/layer18_attention_weight_min": -35.34971618652344,
      "activations/layer19_attention_weight_max": 20.486852645874023,
      "activations/layer19_attention_weight_min": -18.650663375854492,
      "activations/layer1_attention_weight_max": 18.987279891967773,
      "activations/layer1_attention_weight_min": -13.222227096557617,
      "activations/layer20_attention_weight_max": 21.760643005371094,
      "activations/layer20_attention_weight_min": -21.48909568786621,
      "activations/layer21_attention_weight_max": 34.693519592285156,
      "activations/layer21_attention_weight_min": -25.298721313476562,
      "activations/layer22_attention_weight_max": 32.95004653930664,
      "activations/layer22_attention_weight_min": -25.4274845123291,
      "activations/layer23_attention_weight_max": 28.377723693847656,
      "activations/layer23_attention_weight_min": -19.081418991088867,
      "activations/layer2_attention_weight_max": 28.496883392333984,
      "activations/layer2_attention_weight_min": -28.774559020996094,
      "activations/layer3_attention_weight_max": 75.64728546142578,
      "activations/layer3_attention_weight_min": -76.67384338378906,
      "activations/layer4_attention_weight_max": 90.61959075927734,
      "activations/layer4_attention_weight_min": -90.3646011352539,
      "activations/layer5_attention_weight_max": 86.2975082397461,
      "activations/layer5_attention_weight_min": -82.34481811523438,
      "activations/layer6_attention_weight_max": 54.86737060546875,
      "activations/layer6_attention_weight_min": -51.59392166137695,
      "activations/layer7_attention_weight_max": 70.88357543945312,
      "activations/layer7_attention_weight_min": -65.65856170654297,
      "activations/layer8_attention_weight_max": 40.78216552734375,
      "activations/layer8_attention_weight_min": -41.83920669555664,
      "activations/layer9_attention_weight_max": 46.778953552246094,
      "activations/layer9_attention_weight_min": -48.58345413208008,
      "epoch": 6.04,
      "learning_rate": 0.00011219734848484848,
      "loss": 2.8331,
      "step": 103900
    },
    {
      "activations/layer0_attention_weight_max": 14.12956428527832,
      "activations/layer0_attention_weight_min": -14.808393478393555,
      "activations/layer10_attention_weight_max": 36.4815559387207,
      "activations/layer10_attention_weight_min": -34.41429901123047,
      "activations/layer11_attention_weight_max": 28.479568481445312,
      "activations/layer11_attention_weight_min": -28.292888641357422,
      "activations/layer12_attention_weight_max": 22.22736167907715,
      "activations/layer12_attention_weight_min": -23.243865966796875,
      "activations/layer13_attention_weight_max": 47.879695892333984,
      "activations/layer13_attention_weight_min": -34.26628112792969,
      "activations/layer14_attention_weight_max": 35.16481399536133,
      "activations/layer14_attention_weight_min": -29.68079948425293,
      "activations/layer15_attention_weight_max": 40.52821350097656,
      "activations/layer15_attention_weight_min": -29.59475326538086,
      "activations/layer16_attention_weight_max": 33.36236572265625,
      "activations/layer16_attention_weight_min": -28.164222717285156,
      "activations/layer17_attention_weight_max": 56.11310577392578,
      "activations/layer17_attention_weight_min": -41.29939651489258,
      "activations/layer18_attention_weight_max": 46.36015701293945,
      "activations/layer18_attention_weight_min": -35.78822326660156,
      "activations/layer19_attention_weight_max": 22.933122634887695,
      "activations/layer19_attention_weight_min": -20.842512130737305,
      "activations/layer1_attention_weight_max": 17.024742126464844,
      "activations/layer1_attention_weight_min": -15.320454597473145,
      "activations/layer20_attention_weight_max": 21.964256286621094,
      "activations/layer20_attention_weight_min": -20.793413162231445,
      "activations/layer21_attention_weight_max": 43.84928512573242,
      "activations/layer21_attention_weight_min": -26.6060848236084,
      "activations/layer22_attention_weight_max": 31.60186004638672,
      "activations/layer22_attention_weight_min": -27.22887420654297,
      "activations/layer23_attention_weight_max": 30.09784507751465,
      "activations/layer23_attention_weight_min": -20.38092803955078,
      "activations/layer2_attention_weight_max": 28.93732261657715,
      "activations/layer2_attention_weight_min": -28.755842208862305,
      "activations/layer3_attention_weight_max": 73.97702026367188,
      "activations/layer3_attention_weight_min": -79.19999694824219,
      "activations/layer4_attention_weight_max": 92.89366149902344,
      "activations/layer4_attention_weight_min": -92.52556610107422,
      "activations/layer5_attention_weight_max": 85.0060806274414,
      "activations/layer5_attention_weight_min": -85.24140930175781,
      "activations/layer6_attention_weight_max": 50.33766174316406,
      "activations/layer6_attention_weight_min": -50.25254440307617,
      "activations/layer7_attention_weight_max": 68.95550537109375,
      "activations/layer7_attention_weight_min": -64.7682113647461,
      "activations/layer8_attention_weight_max": 42.00828170776367,
      "activations/layer8_attention_weight_min": -40.30730056762695,
      "activations/layer9_attention_weight_max": 47.36592102050781,
      "activations/layer9_attention_weight_min": -45.94151306152344,
      "epoch": 6.04,
      "learning_rate": 0.00011217840909090908,
      "loss": 2.8364,
      "step": 103950
    },
    {
      "activations/layer0_attention_weight_max": 15.998433113098145,
      "activations/layer0_attention_weight_min": -14.880888938903809,
      "activations/layer10_attention_weight_max": 38.29417419433594,
      "activations/layer10_attention_weight_min": -35.46699905395508,
      "activations/layer11_attention_weight_max": 32.328800201416016,
      "activations/layer11_attention_weight_min": -29.248798370361328,
      "activations/layer12_attention_weight_max": 29.572731018066406,
      "activations/layer12_attention_weight_min": -25.200397491455078,
      "activations/layer13_attention_weight_max": 44.978004455566406,
      "activations/layer13_attention_weight_min": -35.00435256958008,
      "activations/layer14_attention_weight_max": 41.78704833984375,
      "activations/layer14_attention_weight_min": -30.93701934814453,
      "activations/layer15_attention_weight_max": 38.50629425048828,
      "activations/layer15_attention_weight_min": -29.18813705444336,
      "activations/layer16_attention_weight_max": 31.125865936279297,
      "activations/layer16_attention_weight_min": -26.663433074951172,
      "activations/layer17_attention_weight_max": 57.76372146606445,
      "activations/layer17_attention_weight_min": -42.7850456237793,
      "activations/layer18_attention_weight_max": 54.78761672973633,
      "activations/layer18_attention_weight_min": -33.98688888549805,
      "activations/layer19_attention_weight_max": 21.64423179626465,
      "activations/layer19_attention_weight_min": -19.365497589111328,
      "activations/layer1_attention_weight_max": 16.633193969726562,
      "activations/layer1_attention_weight_min": -13.899370193481445,
      "activations/layer20_attention_weight_max": 22.96672821044922,
      "activations/layer20_attention_weight_min": -21.452964782714844,
      "activations/layer21_attention_weight_max": 39.387062072753906,
      "activations/layer21_attention_weight_min": -24.884807586669922,
      "activations/layer22_attention_weight_max": 40.662147521972656,
      "activations/layer22_attention_weight_min": -25.24703598022461,
      "activations/layer23_attention_weight_max": 33.45494842529297,
      "activations/layer23_attention_weight_min": -17.465167999267578,
      "activations/layer2_attention_weight_max": 29.67653465270996,
      "activations/layer2_attention_weight_min": -28.50404167175293,
      "activations/layer3_attention_weight_max": 82.60309600830078,
      "activations/layer3_attention_weight_min": -86.95680236816406,
      "activations/layer4_attention_weight_max": 100.89698791503906,
      "activations/layer4_attention_weight_min": -95.63408660888672,
      "activations/layer5_attention_weight_max": 97.53585815429688,
      "activations/layer5_attention_weight_min": -87.17314910888672,
      "activations/layer6_attention_weight_max": 58.61974334716797,
      "activations/layer6_attention_weight_min": -52.31560134887695,
      "activations/layer7_attention_weight_max": 76.925048828125,
      "activations/layer7_attention_weight_min": -72.3563232421875,
      "activations/layer8_attention_weight_max": 45.4127082824707,
      "activations/layer8_attention_weight_min": -43.683712005615234,
      "activations/layer9_attention_weight_max": 58.201873779296875,
      "activations/layer9_attention_weight_min": -50.97901916503906,
      "epoch": 6.04,
      "learning_rate": 0.0001121594696969697,
      "loss": 2.827,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_loss": 2.787109375,
      "eval_runtime": 8.6086,
      "eval_samples_per_second": 498.804,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_openwebtext_loss": 2.787109375,
      "eval_openwebtext_ppl": 16.23402543319486,
      "eval_openwebtext_runtime": 8.6086,
      "eval_openwebtext_samples_per_second": 498.804,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_wikitext_loss": 3.025390625,
      "eval_wikitext_ppl": 20.602050815688813,
      "eval_wikitext_runtime": 2.0593,
      "eval_wikitext_samples_per_second": 221.434,
      "step": 104000
    },
    {
      "epoch": 6.04,
      "eval_lambada_loss": 2.71875,
      "eval_lambada_ppl": 15.16135869683449,
      "eval_lambada_runtime": 9.6676,
      "eval_lambada_samples_per_second": 503.639,
      "step": 104000
    },
    {
      "activations/layer0_attention_weight_max": 15.422661781311035,
      "activations/layer0_attention_weight_min": -15.527034759521484,
      "activations/layer10_attention_weight_max": 41.316673278808594,
      "activations/layer10_attention_weight_min": -36.94698715209961,
      "activations/layer11_attention_weight_max": 30.798961639404297,
      "activations/layer11_attention_weight_min": -31.01270866394043,
      "activations/layer12_attention_weight_max": 29.61492347717285,
      "activations/layer12_attention_weight_min": -27.7648983001709,
      "activations/layer13_attention_weight_max": 48.25968933105469,
      "activations/layer13_attention_weight_min": -33.71692657470703,
      "activations/layer14_attention_weight_max": 41.60865020751953,
      "activations/layer14_attention_weight_min": -29.883546829223633,
      "activations/layer15_attention_weight_max": 38.087825775146484,
      "activations/layer15_attention_weight_min": -30.726425170898438,
      "activations/layer16_attention_weight_max": 29.368507385253906,
      "activations/layer16_attention_weight_min": -26.178735733032227,
      "activations/layer17_attention_weight_max": 56.65025329589844,
      "activations/layer17_attention_weight_min": -42.51600646972656,
      "activations/layer18_attention_weight_max": 47.12010192871094,
      "activations/layer18_attention_weight_min": -35.994693756103516,
      "activations/layer19_attention_weight_max": 21.510662078857422,
      "activations/layer19_attention_weight_min": -19.300817489624023,
      "activations/layer1_attention_weight_max": 18.003341674804688,
      "activations/layer1_attention_weight_min": -15.344042778015137,
      "activations/layer20_attention_weight_max": 21.851573944091797,
      "activations/layer20_attention_weight_min": -23.284690856933594,
      "activations/layer21_attention_weight_max": 38.1550407409668,
      "activations/layer21_attention_weight_min": -24.108652114868164,
      "activations/layer22_attention_weight_max": 30.276622772216797,
      "activations/layer22_attention_weight_min": -22.940271377563477,
      "activations/layer23_attention_weight_max": 28.402061462402344,
      "activations/layer23_attention_weight_min": -20.188228607177734,
      "activations/layer2_attention_weight_max": 31.51593589782715,
      "activations/layer2_attention_weight_min": -29.004600524902344,
      "activations/layer3_attention_weight_max": 82.07552337646484,
      "activations/layer3_attention_weight_min": -86.86394500732422,
      "activations/layer4_attention_weight_max": 101.59627532958984,
      "activations/layer4_attention_weight_min": -95.90522003173828,
      "activations/layer5_attention_weight_max": 95.82366943359375,
      "activations/layer5_attention_weight_min": -91.17525482177734,
      "activations/layer6_attention_weight_max": 61.61940383911133,
      "activations/layer6_attention_weight_min": -55.79780197143555,
      "activations/layer7_attention_weight_max": 75.57179260253906,
      "activations/layer7_attention_weight_min": -71.78282165527344,
      "activations/layer8_attention_weight_max": 47.9992790222168,
      "activations/layer8_attention_weight_min": -46.63893127441406,
      "activations/layer9_attention_weight_max": 54.30731964111328,
      "activations/layer9_attention_weight_min": -52.70595169067383,
      "epoch": 6.05,
      "learning_rate": 0.0001121405303030303,
      "loss": 2.8259,
      "step": 104050
    },
    {
      "activations/layer0_attention_weight_max": 14.272102355957031,
      "activations/layer0_attention_weight_min": -15.224105834960938,
      "activations/layer10_attention_weight_max": 38.415706634521484,
      "activations/layer10_attention_weight_min": -34.27226257324219,
      "activations/layer11_attention_weight_max": 29.26991081237793,
      "activations/layer11_attention_weight_min": -27.978063583374023,
      "activations/layer12_attention_weight_max": 19.83511734008789,
      "activations/layer12_attention_weight_min": -27.231876373291016,
      "activations/layer13_attention_weight_max": 49.04607391357422,
      "activations/layer13_attention_weight_min": -38.869842529296875,
      "activations/layer14_attention_weight_max": 36.859214782714844,
      "activations/layer14_attention_weight_min": -31.15326690673828,
      "activations/layer15_attention_weight_max": 38.55683898925781,
      "activations/layer15_attention_weight_min": -30.799800872802734,
      "activations/layer16_attention_weight_max": 36.25590133666992,
      "activations/layer16_attention_weight_min": -26.868162155151367,
      "activations/layer17_attention_weight_max": 61.66194152832031,
      "activations/layer17_attention_weight_min": -40.16032409667969,
      "activations/layer18_attention_weight_max": 52.06718826293945,
      "activations/layer18_attention_weight_min": -34.73591995239258,
      "activations/layer19_attention_weight_max": 21.165128707885742,
      "activations/layer19_attention_weight_min": -19.026994705200195,
      "activations/layer1_attention_weight_max": 16.60845947265625,
      "activations/layer1_attention_weight_min": -13.283458709716797,
      "activations/layer20_attention_weight_max": 22.105937957763672,
      "activations/layer20_attention_weight_min": -21.34406089782715,
      "activations/layer21_attention_weight_max": 35.9144287109375,
      "activations/layer21_attention_weight_min": -25.2669677734375,
      "activations/layer22_attention_weight_max": 35.34757614135742,
      "activations/layer22_attention_weight_min": -27.049057006835938,
      "activations/layer23_attention_weight_max": 30.976791381835938,
      "activations/layer23_attention_weight_min": -20.471052169799805,
      "activations/layer2_attention_weight_max": 29.518627166748047,
      "activations/layer2_attention_weight_min": -27.086286544799805,
      "activations/layer3_attention_weight_max": 82.06790161132812,
      "activations/layer3_attention_weight_min": -86.66506958007812,
      "activations/layer4_attention_weight_max": 98.42403411865234,
      "activations/layer4_attention_weight_min": -92.45941925048828,
      "activations/layer5_attention_weight_max": 93.16117858886719,
      "activations/layer5_attention_weight_min": -88.12947845458984,
      "activations/layer6_attention_weight_max": 57.40601348876953,
      "activations/layer6_attention_weight_min": -51.94516372680664,
      "activations/layer7_attention_weight_max": 71.5461196899414,
      "activations/layer7_attention_weight_min": -72.41988372802734,
      "activations/layer8_attention_weight_max": 44.43882751464844,
      "activations/layer8_attention_weight_min": -43.821937561035156,
      "activations/layer9_attention_weight_max": 49.9069938659668,
      "activations/layer9_attention_weight_min": -48.1362419128418,
      "epoch": 6.05,
      "learning_rate": 0.0001121215909090909,
      "loss": 2.8271,
      "step": 104100
    },
    {
      "activations/layer0_attention_weight_max": 14.104496955871582,
      "activations/layer0_attention_weight_min": -15.236788749694824,
      "activations/layer10_attention_weight_max": 38.0645637512207,
      "activations/layer10_attention_weight_min": -34.067928314208984,
      "activations/layer11_attention_weight_max": 29.070701599121094,
      "activations/layer11_attention_weight_min": -27.840681076049805,
      "activations/layer12_attention_weight_max": 31.588947296142578,
      "activations/layer12_attention_weight_min": -29.92709732055664,
      "activations/layer13_attention_weight_max": 47.50265884399414,
      "activations/layer13_attention_weight_min": -35.494384765625,
      "activations/layer14_attention_weight_max": 37.41985321044922,
      "activations/layer14_attention_weight_min": -30.148221969604492,
      "activations/layer15_attention_weight_max": 38.525970458984375,
      "activations/layer15_attention_weight_min": -28.56793212890625,
      "activations/layer16_attention_weight_max": 31.574434280395508,
      "activations/layer16_attention_weight_min": -27.3980770111084,
      "activations/layer17_attention_weight_max": 56.38893508911133,
      "activations/layer17_attention_weight_min": -40.8347053527832,
      "activations/layer18_attention_weight_max": 48.234527587890625,
      "activations/layer18_attention_weight_min": -34.30290603637695,
      "activations/layer19_attention_weight_max": 21.966115951538086,
      "activations/layer19_attention_weight_min": -20.59615135192871,
      "activations/layer1_attention_weight_max": 17.23077964782715,
      "activations/layer1_attention_weight_min": -13.395362854003906,
      "activations/layer20_attention_weight_max": 21.985502243041992,
      "activations/layer20_attention_weight_min": -24.52277183532715,
      "activations/layer21_attention_weight_max": 38.28810501098633,
      "activations/layer21_attention_weight_min": -24.670061111450195,
      "activations/layer22_attention_weight_max": 32.80057907104492,
      "activations/layer22_attention_weight_min": -24.546953201293945,
      "activations/layer23_attention_weight_max": 27.76582145690918,
      "activations/layer23_attention_weight_min": -17.266376495361328,
      "activations/layer2_attention_weight_max": 31.05727195739746,
      "activations/layer2_attention_weight_min": -30.61885643005371,
      "activations/layer3_attention_weight_max": 81.23345184326172,
      "activations/layer3_attention_weight_min": -85.39314270019531,
      "activations/layer4_attention_weight_max": 100.84239959716797,
      "activations/layer4_attention_weight_min": -94.34175872802734,
      "activations/layer5_attention_weight_max": 92.21963500976562,
      "activations/layer5_attention_weight_min": -90.63433837890625,
      "activations/layer6_attention_weight_max": 58.07720184326172,
      "activations/layer6_attention_weight_min": -53.399295806884766,
      "activations/layer7_attention_weight_max": 75.18770599365234,
      "activations/layer7_attention_weight_min": -67.82763671875,
      "activations/layer8_attention_weight_max": 44.6941032409668,
      "activations/layer8_attention_weight_min": -43.20631790161133,
      "activations/layer9_attention_weight_max": 47.615291595458984,
      "activations/layer9_attention_weight_min": -47.43033218383789,
      "epoch": 6.05,
      "learning_rate": 0.00011210265151515151,
      "loss": 2.832,
      "step": 104150
    },
    {
      "activations/layer0_attention_weight_max": 14.274468421936035,
      "activations/layer0_attention_weight_min": -15.55055046081543,
      "activations/layer10_attention_weight_max": 41.91294860839844,
      "activations/layer10_attention_weight_min": -34.03508758544922,
      "activations/layer11_attention_weight_max": 33.128665924072266,
      "activations/layer11_attention_weight_min": -28.23766326904297,
      "activations/layer12_attention_weight_max": 25.417198181152344,
      "activations/layer12_attention_weight_min": -30.579849243164062,
      "activations/layer13_attention_weight_max": 46.93181610107422,
      "activations/layer13_attention_weight_min": -33.37722396850586,
      "activations/layer14_attention_weight_max": 43.46623229980469,
      "activations/layer14_attention_weight_min": -30.568832397460938,
      "activations/layer15_attention_weight_max": 44.500732421875,
      "activations/layer15_attention_weight_min": -30.084718704223633,
      "activations/layer16_attention_weight_max": 40.2162971496582,
      "activations/layer16_attention_weight_min": -26.193693161010742,
      "activations/layer17_attention_weight_max": 63.04273223876953,
      "activations/layer17_attention_weight_min": -38.794673919677734,
      "activations/layer18_attention_weight_max": 49.66881561279297,
      "activations/layer18_attention_weight_min": -32.941162109375,
      "activations/layer19_attention_weight_max": 31.735769271850586,
      "activations/layer19_attention_weight_min": -19.361553192138672,
      "activations/layer1_attention_weight_max": 17.783674240112305,
      "activations/layer1_attention_weight_min": -15.764663696289062,
      "activations/layer20_attention_weight_max": 23.046436309814453,
      "activations/layer20_attention_weight_min": -21.696735382080078,
      "activations/layer21_attention_weight_max": 41.63711929321289,
      "activations/layer21_attention_weight_min": -25.281143188476562,
      "activations/layer22_attention_weight_max": 34.65629959106445,
      "activations/layer22_attention_weight_min": -23.03884506225586,
      "activations/layer23_attention_weight_max": 31.85232925415039,
      "activations/layer23_attention_weight_min": -19.684659957885742,
      "activations/layer2_attention_weight_max": 31.17807388305664,
      "activations/layer2_attention_weight_min": -27.0283203125,
      "activations/layer3_attention_weight_max": 84.7607421875,
      "activations/layer3_attention_weight_min": -82.85457611083984,
      "activations/layer4_attention_weight_max": 101.70964050292969,
      "activations/layer4_attention_weight_min": -89.64299774169922,
      "activations/layer5_attention_weight_max": 95.69984436035156,
      "activations/layer5_attention_weight_min": -87.53733825683594,
      "activations/layer6_attention_weight_max": 56.3774299621582,
      "activations/layer6_attention_weight_min": -52.33952331542969,
      "activations/layer7_attention_weight_max": 73.77466583251953,
      "activations/layer7_attention_weight_min": -70.95067596435547,
      "activations/layer8_attention_weight_max": 45.33538055419922,
      "activations/layer8_attention_weight_min": -44.05364990234375,
      "activations/layer9_attention_weight_max": 55.82767868041992,
      "activations/layer9_attention_weight_min": -49.519683837890625,
      "epoch": 6.05,
      "learning_rate": 0.0001120837121212121,
      "loss": 2.8146,
      "step": 104200
    },
    {
      "activations/layer0_attention_weight_max": 14.672298431396484,
      "activations/layer0_attention_weight_min": -14.66009521484375,
      "activations/layer10_attention_weight_max": 41.61445999145508,
      "activations/layer10_attention_weight_min": -36.71363830566406,
      "activations/layer11_attention_weight_max": 33.36286926269531,
      "activations/layer11_attention_weight_min": -29.18165397644043,
      "activations/layer12_attention_weight_max": 34.69518280029297,
      "activations/layer12_attention_weight_min": -30.435951232910156,
      "activations/layer13_attention_weight_max": 49.15068435668945,
      "activations/layer13_attention_weight_min": -33.474647521972656,
      "activations/layer14_attention_weight_max": 54.66421127319336,
      "activations/layer14_attention_weight_min": -32.890438079833984,
      "activations/layer15_attention_weight_max": 42.443031311035156,
      "activations/layer15_attention_weight_min": -29.391834259033203,
      "activations/layer16_attention_weight_max": 34.274349212646484,
      "activations/layer16_attention_weight_min": -25.49170684814453,
      "activations/layer17_attention_weight_max": 57.85056686401367,
      "activations/layer17_attention_weight_min": -42.12240219116211,
      "activations/layer18_attention_weight_max": 49.19320297241211,
      "activations/layer18_attention_weight_min": -35.495704650878906,
      "activations/layer19_attention_weight_max": 26.948835372924805,
      "activations/layer19_attention_weight_min": -19.985483169555664,
      "activations/layer1_attention_weight_max": 17.419118881225586,
      "activations/layer1_attention_weight_min": -14.876185417175293,
      "activations/layer20_attention_weight_max": 22.172998428344727,
      "activations/layer20_attention_weight_min": -20.714496612548828,
      "activations/layer21_attention_weight_max": 44.620792388916016,
      "activations/layer21_attention_weight_min": -25.0845890045166,
      "activations/layer22_attention_weight_max": 33.82341384887695,
      "activations/layer22_attention_weight_min": -25.861753463745117,
      "activations/layer23_attention_weight_max": 36.61524963378906,
      "activations/layer23_attention_weight_min": -20.199050903320312,
      "activations/layer2_attention_weight_max": 32.702091217041016,
      "activations/layer2_attention_weight_min": -30.424419403076172,
      "activations/layer3_attention_weight_max": 84.9654541015625,
      "activations/layer3_attention_weight_min": -91.95989227294922,
      "activations/layer4_attention_weight_max": 105.2237548828125,
      "activations/layer4_attention_weight_min": -101.36563110351562,
      "activations/layer5_attention_weight_max": 97.45024871826172,
      "activations/layer5_attention_weight_min": -93.58111572265625,
      "activations/layer6_attention_weight_max": 54.1116828918457,
      "activations/layer6_attention_weight_min": -54.71080780029297,
      "activations/layer7_attention_weight_max": 92.71759796142578,
      "activations/layer7_attention_weight_min": -76.42530822753906,
      "activations/layer8_attention_weight_max": 47.00835037231445,
      "activations/layer8_attention_weight_min": -43.50125503540039,
      "activations/layer9_attention_weight_max": 69.1231689453125,
      "activations/layer9_attention_weight_min": -56.7724723815918,
      "epoch": 6.06,
      "learning_rate": 0.00011206477272727272,
      "loss": 2.8344,
      "step": 104250
    },
    {
      "activations/layer0_attention_weight_max": 14.808247566223145,
      "activations/layer0_attention_weight_min": -14.915098190307617,
      "activations/layer10_attention_weight_max": 41.88755798339844,
      "activations/layer10_attention_weight_min": -38.73674392700195,
      "activations/layer11_attention_weight_max": 31.096975326538086,
      "activations/layer11_attention_weight_min": -28.474937438964844,
      "activations/layer12_attention_weight_max": 20.247644424438477,
      "activations/layer12_attention_weight_min": -24.915180206298828,
      "activations/layer13_attention_weight_max": 46.94615936279297,
      "activations/layer13_attention_weight_min": -32.681678771972656,
      "activations/layer14_attention_weight_max": 46.70265197753906,
      "activations/layer14_attention_weight_min": -32.834590911865234,
      "activations/layer15_attention_weight_max": 41.84210968017578,
      "activations/layer15_attention_weight_min": -32.098934173583984,
      "activations/layer16_attention_weight_max": 33.63327407836914,
      "activations/layer16_attention_weight_min": -25.43955421447754,
      "activations/layer17_attention_weight_max": 54.3612174987793,
      "activations/layer17_attention_weight_min": -41.34555435180664,
      "activations/layer18_attention_weight_max": 48.22880554199219,
      "activations/layer18_attention_weight_min": -34.5206184387207,
      "activations/layer19_attention_weight_max": 21.585065841674805,
      "activations/layer19_attention_weight_min": -19.528532028198242,
      "activations/layer1_attention_weight_max": 17.61845588684082,
      "activations/layer1_attention_weight_min": -15.284150123596191,
      "activations/layer20_attention_weight_max": 21.622949600219727,
      "activations/layer20_attention_weight_min": -18.467498779296875,
      "activations/layer21_attention_weight_max": 40.899662017822266,
      "activations/layer21_attention_weight_min": -24.959537506103516,
      "activations/layer22_attention_weight_max": 34.656131744384766,
      "activations/layer22_attention_weight_min": -25.073497772216797,
      "activations/layer23_attention_weight_max": 34.00973129272461,
      "activations/layer23_attention_weight_min": -21.558635711669922,
      "activations/layer2_attention_weight_max": 32.41969299316406,
      "activations/layer2_attention_weight_min": -29.528953552246094,
      "activations/layer3_attention_weight_max": 85.16569519042969,
      "activations/layer3_attention_weight_min": -89.2094955444336,
      "activations/layer4_attention_weight_max": 107.6444091796875,
      "activations/layer4_attention_weight_min": -99.5766830444336,
      "activations/layer5_attention_weight_max": 100.62461853027344,
      "activations/layer5_attention_weight_min": -95.57447814941406,
      "activations/layer6_attention_weight_max": 59.829063415527344,
      "activations/layer6_attention_weight_min": -57.893165588378906,
      "activations/layer7_attention_weight_max": 79.37089538574219,
      "activations/layer7_attention_weight_min": -73.3733901977539,
      "activations/layer8_attention_weight_max": 47.84956741333008,
      "activations/layer8_attention_weight_min": -48.70906448364258,
      "activations/layer9_attention_weight_max": 52.76469039916992,
      "activations/layer9_attention_weight_min": -50.716304779052734,
      "epoch": 6.06,
      "learning_rate": 0.00011204583333333333,
      "loss": 2.8299,
      "step": 104300
    },
    {
      "activations/layer0_attention_weight_max": 15.26052474975586,
      "activations/layer0_attention_weight_min": -14.812639236450195,
      "activations/layer10_attention_weight_max": 35.58710479736328,
      "activations/layer10_attention_weight_min": -35.24435043334961,
      "activations/layer11_attention_weight_max": 28.701663970947266,
      "activations/layer11_attention_weight_min": -29.00704574584961,
      "activations/layer12_attention_weight_max": 23.660755157470703,
      "activations/layer12_attention_weight_min": -22.37558364868164,
      "activations/layer13_attention_weight_max": 57.04802322387695,
      "activations/layer13_attention_weight_min": -42.5154914855957,
      "activations/layer14_attention_weight_max": 40.03480911254883,
      "activations/layer14_attention_weight_min": -32.138641357421875,
      "activations/layer15_attention_weight_max": 40.03555679321289,
      "activations/layer15_attention_weight_min": -30.61073112487793,
      "activations/layer16_attention_weight_max": 33.796836853027344,
      "activations/layer16_attention_weight_min": -27.436254501342773,
      "activations/layer17_attention_weight_max": 53.922969818115234,
      "activations/layer17_attention_weight_min": -42.565650939941406,
      "activations/layer18_attention_weight_max": 48.59050369262695,
      "activations/layer18_attention_weight_min": -36.034149169921875,
      "activations/layer19_attention_weight_max": 21.736852645874023,
      "activations/layer19_attention_weight_min": -20.632062911987305,
      "activations/layer1_attention_weight_max": 17.61766242980957,
      "activations/layer1_attention_weight_min": -13.991576194763184,
      "activations/layer20_attention_weight_max": 22.312114715576172,
      "activations/layer20_attention_weight_min": -23.276580810546875,
      "activations/layer21_attention_weight_max": 36.609798431396484,
      "activations/layer21_attention_weight_min": -26.664531707763672,
      "activations/layer22_attention_weight_max": 32.590328216552734,
      "activations/layer22_attention_weight_min": -24.78639030456543,
      "activations/layer23_attention_weight_max": 32.023929595947266,
      "activations/layer23_attention_weight_min": -19.969669342041016,
      "activations/layer2_attention_weight_max": 29.91183853149414,
      "activations/layer2_attention_weight_min": -29.72022247314453,
      "activations/layer3_attention_weight_max": 81.09085845947266,
      "activations/layer3_attention_weight_min": -87.25312042236328,
      "activations/layer4_attention_weight_max": 98.08355712890625,
      "activations/layer4_attention_weight_min": -96.1129150390625,
      "activations/layer5_attention_weight_max": 94.94198608398438,
      "activations/layer5_attention_weight_min": -90.23269653320312,
      "activations/layer6_attention_weight_max": 56.03852844238281,
      "activations/layer6_attention_weight_min": -53.80537033081055,
      "activations/layer7_attention_weight_max": 71.99774932861328,
      "activations/layer7_attention_weight_min": -68.6328353881836,
      "activations/layer8_attention_weight_max": 41.73563766479492,
      "activations/layer8_attention_weight_min": -40.330528259277344,
      "activations/layer9_attention_weight_max": 49.81269454956055,
      "activations/layer9_attention_weight_min": -47.26316452026367,
      "epoch": 6.06,
      "learning_rate": 0.00011202689393939392,
      "loss": 2.8257,
      "step": 104350
    },
    {
      "activations/layer0_attention_weight_max": 14.043447494506836,
      "activations/layer0_attention_weight_min": -14.454524993896484,
      "activations/layer10_attention_weight_max": 37.85554885864258,
      "activations/layer10_attention_weight_min": -34.032623291015625,
      "activations/layer11_attention_weight_max": 30.95001792907715,
      "activations/layer11_attention_weight_min": -29.254013061523438,
      "activations/layer12_attention_weight_max": 19.967227935791016,
      "activations/layer12_attention_weight_min": -26.40296745300293,
      "activations/layer13_attention_weight_max": 44.093650817871094,
      "activations/layer13_attention_weight_min": -35.093502044677734,
      "activations/layer14_attention_weight_max": 40.63508224487305,
      "activations/layer14_attention_weight_min": -31.603561401367188,
      "activations/layer15_attention_weight_max": 41.48681640625,
      "activations/layer15_attention_weight_min": -30.69403839111328,
      "activations/layer16_attention_weight_max": 36.03376388549805,
      "activations/layer16_attention_weight_min": -29.223440170288086,
      "activations/layer17_attention_weight_max": 58.99920654296875,
      "activations/layer17_attention_weight_min": -43.29767608642578,
      "activations/layer18_attention_weight_max": 49.12562942504883,
      "activations/layer18_attention_weight_min": -35.28828048706055,
      "activations/layer19_attention_weight_max": 27.149250030517578,
      "activations/layer19_attention_weight_min": -20.9188289642334,
      "activations/layer1_attention_weight_max": 17.681154251098633,
      "activations/layer1_attention_weight_min": -16.7050838470459,
      "activations/layer20_attention_weight_max": 23.990251541137695,
      "activations/layer20_attention_weight_min": -22.127885818481445,
      "activations/layer21_attention_weight_max": 43.15435791015625,
      "activations/layer21_attention_weight_min": -29.24101448059082,
      "activations/layer22_attention_weight_max": 36.56653594970703,
      "activations/layer22_attention_weight_min": -24.81439971923828,
      "activations/layer23_attention_weight_max": 31.08684539794922,
      "activations/layer23_attention_weight_min": -20.909685134887695,
      "activations/layer2_attention_weight_max": 30.89031219482422,
      "activations/layer2_attention_weight_min": -29.125144958496094,
      "activations/layer3_attention_weight_max": 80.45709228515625,
      "activations/layer3_attention_weight_min": -85.68608856201172,
      "activations/layer4_attention_weight_max": 104.18989562988281,
      "activations/layer4_attention_weight_min": -95.50501251220703,
      "activations/layer5_attention_weight_max": 101.1927490234375,
      "activations/layer5_attention_weight_min": -86.36299896240234,
      "activations/layer6_attention_weight_max": 59.86496353149414,
      "activations/layer6_attention_weight_min": -52.0405387878418,
      "activations/layer7_attention_weight_max": 75.52023315429688,
      "activations/layer7_attention_weight_min": -67.97016906738281,
      "activations/layer8_attention_weight_max": 43.951393127441406,
      "activations/layer8_attention_weight_min": -42.40652084350586,
      "activations/layer9_attention_weight_max": 50.832435607910156,
      "activations/layer9_attention_weight_min": -48.056671142578125,
      "epoch": 6.07,
      "learning_rate": 0.00011200795454545453,
      "loss": 2.8327,
      "step": 104400
    },
    {
      "activations/layer0_attention_weight_max": 14.88547134399414,
      "activations/layer0_attention_weight_min": -14.77055835723877,
      "activations/layer10_attention_weight_max": 38.11234664916992,
      "activations/layer10_attention_weight_min": -35.29132843017578,
      "activations/layer11_attention_weight_max": 30.194900512695312,
      "activations/layer11_attention_weight_min": -27.596397399902344,
      "activations/layer12_attention_weight_max": 22.980417251586914,
      "activations/layer12_attention_weight_min": -28.750219345092773,
      "activations/layer13_attention_weight_max": 51.54540252685547,
      "activations/layer13_attention_weight_min": -35.06846237182617,
      "activations/layer14_attention_weight_max": 38.927398681640625,
      "activations/layer14_attention_weight_min": -28.21754264831543,
      "activations/layer15_attention_weight_max": 40.4892578125,
      "activations/layer15_attention_weight_min": -28.517412185668945,
      "activations/layer16_attention_weight_max": 34.586978912353516,
      "activations/layer16_attention_weight_min": -26.974613189697266,
      "activations/layer17_attention_weight_max": 60.422607421875,
      "activations/layer17_attention_weight_min": -43.12815475463867,
      "activations/layer18_attention_weight_max": 48.24626159667969,
      "activations/layer18_attention_weight_min": -35.91803741455078,
      "activations/layer19_attention_weight_max": 22.004085540771484,
      "activations/layer19_attention_weight_min": -22.110960006713867,
      "activations/layer1_attention_weight_max": 16.652565002441406,
      "activations/layer1_attention_weight_min": -14.930913925170898,
      "activations/layer20_attention_weight_max": 21.39153289794922,
      "activations/layer20_attention_weight_min": -21.159099578857422,
      "activations/layer21_attention_weight_max": 48.57518005371094,
      "activations/layer21_attention_weight_min": -26.82359504699707,
      "activations/layer22_attention_weight_max": 33.12532043457031,
      "activations/layer22_attention_weight_min": -24.46711540222168,
      "activations/layer23_attention_weight_max": 31.78097915649414,
      "activations/layer23_attention_weight_min": -20.707725524902344,
      "activations/layer2_attention_weight_max": 29.64417266845703,
      "activations/layer2_attention_weight_min": -28.950803756713867,
      "activations/layer3_attention_weight_max": 82.54108428955078,
      "activations/layer3_attention_weight_min": -83.66057586669922,
      "activations/layer4_attention_weight_max": 102.965087890625,
      "activations/layer4_attention_weight_min": -95.03226470947266,
      "activations/layer5_attention_weight_max": 95.88048553466797,
      "activations/layer5_attention_weight_min": -87.24136352539062,
      "activations/layer6_attention_weight_max": 60.8436164855957,
      "activations/layer6_attention_weight_min": -54.80803298950195,
      "activations/layer7_attention_weight_max": 74.71405792236328,
      "activations/layer7_attention_weight_min": -71.6578598022461,
      "activations/layer8_attention_weight_max": 41.83354187011719,
      "activations/layer8_attention_weight_min": -41.22101974487305,
      "activations/layer9_attention_weight_max": 49.584590911865234,
      "activations/layer9_attention_weight_min": -48.19698715209961,
      "epoch": 6.07,
      "learning_rate": 0.00011198901515151515,
      "loss": 2.8344,
      "step": 104450
    },
    {
      "activations/layer0_attention_weight_max": 15.658143043518066,
      "activations/layer0_attention_weight_min": -14.241545677185059,
      "activations/layer10_attention_weight_max": 38.684532165527344,
      "activations/layer10_attention_weight_min": -35.946571350097656,
      "activations/layer11_attention_weight_max": 30.361465454101562,
      "activations/layer11_attention_weight_min": -29.48077392578125,
      "activations/layer12_attention_weight_max": 24.927396774291992,
      "activations/layer12_attention_weight_min": -26.56632423400879,
      "activations/layer13_attention_weight_max": 48.49741744995117,
      "activations/layer13_attention_weight_min": -36.4911003112793,
      "activations/layer14_attention_weight_max": 42.46162033081055,
      "activations/layer14_attention_weight_min": -29.62618637084961,
      "activations/layer15_attention_weight_max": 34.8994140625,
      "activations/layer15_attention_weight_min": -28.867740631103516,
      "activations/layer16_attention_weight_max": 34.03622817993164,
      "activations/layer16_attention_weight_min": -27.2748966217041,
      "activations/layer17_attention_weight_max": 56.537986755371094,
      "activations/layer17_attention_weight_min": -40.93662643432617,
      "activations/layer18_attention_weight_max": 47.638736724853516,
      "activations/layer18_attention_weight_min": -33.48308563232422,
      "activations/layer19_attention_weight_max": 19.516569137573242,
      "activations/layer19_attention_weight_min": -19.105579376220703,
      "activations/layer1_attention_weight_max": 17.38871192932129,
      "activations/layer1_attention_weight_min": -16.139244079589844,
      "activations/layer20_attention_weight_max": 22.352760314941406,
      "activations/layer20_attention_weight_min": -21.313640594482422,
      "activations/layer21_attention_weight_max": 42.87561798095703,
      "activations/layer21_attention_weight_min": -23.06414222717285,
      "activations/layer22_attention_weight_max": 35.160240173339844,
      "activations/layer22_attention_weight_min": -23.636268615722656,
      "activations/layer23_attention_weight_max": 34.905208587646484,
      "activations/layer23_attention_weight_min": -19.9044132232666,
      "activations/layer2_attention_weight_max": 32.07500457763672,
      "activations/layer2_attention_weight_min": -28.60767936706543,
      "activations/layer3_attention_weight_max": 82.03235626220703,
      "activations/layer3_attention_weight_min": -86.49176025390625,
      "activations/layer4_attention_weight_max": 103.2989730834961,
      "activations/layer4_attention_weight_min": -98.46818542480469,
      "activations/layer5_attention_weight_max": 97.11088562011719,
      "activations/layer5_attention_weight_min": -91.13066101074219,
      "activations/layer6_attention_weight_max": 58.7055778503418,
      "activations/layer6_attention_weight_min": -52.63947677612305,
      "activations/layer7_attention_weight_max": 74.227783203125,
      "activations/layer7_attention_weight_min": -74.09259796142578,
      "activations/layer8_attention_weight_max": 45.65882873535156,
      "activations/layer8_attention_weight_min": -45.17302703857422,
      "activations/layer9_attention_weight_max": 54.11318588256836,
      "activations/layer9_attention_weight_min": -50.58828353881836,
      "epoch": 6.07,
      "learning_rate": 0.00011197007575757575,
      "loss": 2.8372,
      "step": 104500
    },
    {
      "activations/layer0_attention_weight_max": 14.308599472045898,
      "activations/layer0_attention_weight_min": -14.852035522460938,
      "activations/layer10_attention_weight_max": 35.45793914794922,
      "activations/layer10_attention_weight_min": -33.17033767700195,
      "activations/layer11_attention_weight_max": 28.548301696777344,
      "activations/layer11_attention_weight_min": -28.053924560546875,
      "activations/layer12_attention_weight_max": 22.236743927001953,
      "activations/layer12_attention_weight_min": -24.33477020263672,
      "activations/layer13_attention_weight_max": 46.0772590637207,
      "activations/layer13_attention_weight_min": -34.57597732543945,
      "activations/layer14_attention_weight_max": 38.66929626464844,
      "activations/layer14_attention_weight_min": -30.710044860839844,
      "activations/layer15_attention_weight_max": 37.33489227294922,
      "activations/layer15_attention_weight_min": -30.019445419311523,
      "activations/layer16_attention_weight_max": 31.430408477783203,
      "activations/layer16_attention_weight_min": -28.28185272216797,
      "activations/layer17_attention_weight_max": 52.6171760559082,
      "activations/layer17_attention_weight_min": -42.7513313293457,
      "activations/layer18_attention_weight_max": 46.840087890625,
      "activations/layer18_attention_weight_min": -34.83819580078125,
      "activations/layer19_attention_weight_max": 22.557323455810547,
      "activations/layer19_attention_weight_min": -20.875600814819336,
      "activations/layer1_attention_weight_max": 17.874902725219727,
      "activations/layer1_attention_weight_min": -13.748919486999512,
      "activations/layer20_attention_weight_max": 21.16038703918457,
      "activations/layer20_attention_weight_min": -22.270605087280273,
      "activations/layer21_attention_weight_max": 36.54317092895508,
      "activations/layer21_attention_weight_min": -22.788925170898438,
      "activations/layer22_attention_weight_max": 34.835262298583984,
      "activations/layer22_attention_weight_min": -24.03077507019043,
      "activations/layer23_attention_weight_max": 29.33461570739746,
      "activations/layer23_attention_weight_min": -18.676095962524414,
      "activations/layer2_attention_weight_max": 29.074613571166992,
      "activations/layer2_attention_weight_min": -27.018301010131836,
      "activations/layer3_attention_weight_max": 78.54766845703125,
      "activations/layer3_attention_weight_min": -80.42578125,
      "activations/layer4_attention_weight_max": 96.345458984375,
      "activations/layer4_attention_weight_min": -89.2759780883789,
      "activations/layer5_attention_weight_max": 90.39865112304688,
      "activations/layer5_attention_weight_min": -81.42802429199219,
      "activations/layer6_attention_weight_max": 54.661354064941406,
      "activations/layer6_attention_weight_min": -49.068233489990234,
      "activations/layer7_attention_weight_max": 74.2782974243164,
      "activations/layer7_attention_weight_min": -68.51605987548828,
      "activations/layer8_attention_weight_max": 42.9573974609375,
      "activations/layer8_attention_weight_min": -40.118682861328125,
      "activations/layer9_attention_weight_max": 45.840328216552734,
      "activations/layer9_attention_weight_min": -47.531333923339844,
      "epoch": 6.07,
      "learning_rate": 0.00011195113636363635,
      "loss": 2.8248,
      "step": 104550
    },
    {
      "activations/layer0_attention_weight_max": 14.705891609191895,
      "activations/layer0_attention_weight_min": -14.436104774475098,
      "activations/layer10_attention_weight_max": 36.524452209472656,
      "activations/layer10_attention_weight_min": -34.83462142944336,
      "activations/layer11_attention_weight_max": 30.908432006835938,
      "activations/layer11_attention_weight_min": -28.696895599365234,
      "activations/layer12_attention_weight_max": 23.370506286621094,
      "activations/layer12_attention_weight_min": -27.91986846923828,
      "activations/layer13_attention_weight_max": 39.12809753417969,
      "activations/layer13_attention_weight_min": -30.648754119873047,
      "activations/layer14_attention_weight_max": 35.836700439453125,
      "activations/layer14_attention_weight_min": -29.00994300842285,
      "activations/layer15_attention_weight_max": 33.5680046081543,
      "activations/layer15_attention_weight_min": -28.83307647705078,
      "activations/layer16_attention_weight_max": 31.25042724609375,
      "activations/layer16_attention_weight_min": -26.03014373779297,
      "activations/layer17_attention_weight_max": 54.87702178955078,
      "activations/layer17_attention_weight_min": -38.423709869384766,
      "activations/layer18_attention_weight_max": 46.57913589477539,
      "activations/layer18_attention_weight_min": -33.45695877075195,
      "activations/layer19_attention_weight_max": 23.08611488342285,
      "activations/layer19_attention_weight_min": -21.18965721130371,
      "activations/layer1_attention_weight_max": 16.78851890563965,
      "activations/layer1_attention_weight_min": -12.749059677124023,
      "activations/layer20_attention_weight_max": 19.869321823120117,
      "activations/layer20_attention_weight_min": -20.687101364135742,
      "activations/layer21_attention_weight_max": 32.8294677734375,
      "activations/layer21_attention_weight_min": -25.45212745666504,
      "activations/layer22_attention_weight_max": 32.46121597290039,
      "activations/layer22_attention_weight_min": -24.699737548828125,
      "activations/layer23_attention_weight_max": 28.67371940612793,
      "activations/layer23_attention_weight_min": -19.437816619873047,
      "activations/layer2_attention_weight_max": 30.12911033630371,
      "activations/layer2_attention_weight_min": -29.38838005065918,
      "activations/layer3_attention_weight_max": 79.3185806274414,
      "activations/layer3_attention_weight_min": -81.8275375366211,
      "activations/layer4_attention_weight_max": 93.8181381225586,
      "activations/layer4_attention_weight_min": -91.86216735839844,
      "activations/layer5_attention_weight_max": 90.41326141357422,
      "activations/layer5_attention_weight_min": -83.6671142578125,
      "activations/layer6_attention_weight_max": 55.786170959472656,
      "activations/layer6_attention_weight_min": -49.96778869628906,
      "activations/layer7_attention_weight_max": 69.6024398803711,
      "activations/layer7_attention_weight_min": -66.25801086425781,
      "activations/layer8_attention_weight_max": 43.093990325927734,
      "activations/layer8_attention_weight_min": -42.92262649536133,
      "activations/layer9_attention_weight_max": 49.47780990600586,
      "activations/layer9_attention_weight_min": -49.90340042114258,
      "epoch": 6.08,
      "learning_rate": 0.00011193219696969695,
      "loss": 2.8259,
      "step": 104600
    },
    {
      "activations/layer0_attention_weight_max": 14.392251014709473,
      "activations/layer0_attention_weight_min": -14.794252395629883,
      "activations/layer10_attention_weight_max": 37.10082244873047,
      "activations/layer10_attention_weight_min": -34.84065628051758,
      "activations/layer11_attention_weight_max": 29.015274047851562,
      "activations/layer11_attention_weight_min": -28.411251068115234,
      "activations/layer12_attention_weight_max": 20.441614151000977,
      "activations/layer12_attention_weight_min": -24.59147834777832,
      "activations/layer13_attention_weight_max": 50.89426803588867,
      "activations/layer13_attention_weight_min": -36.11202621459961,
      "activations/layer14_attention_weight_max": 43.822898864746094,
      "activations/layer14_attention_weight_min": -31.21506690979004,
      "activations/layer15_attention_weight_max": 38.07429504394531,
      "activations/layer15_attention_weight_min": -31.40247344970703,
      "activations/layer16_attention_weight_max": 33.41659927368164,
      "activations/layer16_attention_weight_min": -27.15094757080078,
      "activations/layer17_attention_weight_max": 57.61150360107422,
      "activations/layer17_attention_weight_min": -42.61312484741211,
      "activations/layer18_attention_weight_max": 50.179039001464844,
      "activations/layer18_attention_weight_min": -36.90721893310547,
      "activations/layer19_attention_weight_max": 26.23463249206543,
      "activations/layer19_attention_weight_min": -21.78167724609375,
      "activations/layer1_attention_weight_max": 17.107097625732422,
      "activations/layer1_attention_weight_min": -12.76321792602539,
      "activations/layer20_attention_weight_max": 24.316089630126953,
      "activations/layer20_attention_weight_min": -20.652000427246094,
      "activations/layer21_attention_weight_max": 42.1634521484375,
      "activations/layer21_attention_weight_min": -25.292932510375977,
      "activations/layer22_attention_weight_max": 37.33734130859375,
      "activations/layer22_attention_weight_min": -23.5788516998291,
      "activations/layer23_attention_weight_max": 32.79998779296875,
      "activations/layer23_attention_weight_min": -19.44504737854004,
      "activations/layer2_attention_weight_max": 29.927623748779297,
      "activations/layer2_attention_weight_min": -27.945571899414062,
      "activations/layer3_attention_weight_max": 79.8071060180664,
      "activations/layer3_attention_weight_min": -85.36251068115234,
      "activations/layer4_attention_weight_max": 98.15792083740234,
      "activations/layer4_attention_weight_min": -95.18959045410156,
      "activations/layer5_attention_weight_max": 92.95507049560547,
      "activations/layer5_attention_weight_min": -89.4697494506836,
      "activations/layer6_attention_weight_max": 57.30667495727539,
      "activations/layer6_attention_weight_min": -50.225460052490234,
      "activations/layer7_attention_weight_max": 76.44551849365234,
      "activations/layer7_attention_weight_min": -71.93478393554688,
      "activations/layer8_attention_weight_max": 44.35383224487305,
      "activations/layer8_attention_weight_min": -41.78335189819336,
      "activations/layer9_attention_weight_max": 53.263885498046875,
      "activations/layer9_attention_weight_min": -50.537010192871094,
      "epoch": 6.08,
      "learning_rate": 0.00011191325757575757,
      "loss": 2.831,
      "step": 104650
    },
    {
      "activations/layer0_attention_weight_max": 14.396977424621582,
      "activations/layer0_attention_weight_min": -14.788958549499512,
      "activations/layer10_attention_weight_max": 42.591346740722656,
      "activations/layer10_attention_weight_min": -38.86383056640625,
      "activations/layer11_attention_weight_max": 31.59048843383789,
      "activations/layer11_attention_weight_min": -30.478946685791016,
      "activations/layer12_attention_weight_max": 22.485185623168945,
      "activations/layer12_attention_weight_min": -25.419382095336914,
      "activations/layer13_attention_weight_max": 55.04759216308594,
      "activations/layer13_attention_weight_min": -40.68456268310547,
      "activations/layer14_attention_weight_max": 40.50033187866211,
      "activations/layer14_attention_weight_min": -30.914592742919922,
      "activations/layer15_attention_weight_max": 38.15595245361328,
      "activations/layer15_attention_weight_min": -29.961437225341797,
      "activations/layer16_attention_weight_max": 33.876014709472656,
      "activations/layer16_attention_weight_min": -26.270200729370117,
      "activations/layer17_attention_weight_max": 58.83426284790039,
      "activations/layer17_attention_weight_min": -41.67934036254883,
      "activations/layer18_attention_weight_max": 50.161685943603516,
      "activations/layer18_attention_weight_min": -35.83726119995117,
      "activations/layer19_attention_weight_max": 22.827144622802734,
      "activations/layer19_attention_weight_min": -19.634920120239258,
      "activations/layer1_attention_weight_max": 17.979944229125977,
      "activations/layer1_attention_weight_min": -13.352621078491211,
      "activations/layer20_attention_weight_max": 21.853031158447266,
      "activations/layer20_attention_weight_min": -23.685649871826172,
      "activations/layer21_attention_weight_max": 41.94221115112305,
      "activations/layer21_attention_weight_min": -25.285972595214844,
      "activations/layer22_attention_weight_max": 34.77992248535156,
      "activations/layer22_attention_weight_min": -24.3385009765625,
      "activations/layer23_attention_weight_max": 31.237545013427734,
      "activations/layer23_attention_weight_min": -21.15411949157715,
      "activations/layer2_attention_weight_max": 31.642593383789062,
      "activations/layer2_attention_weight_min": -29.273040771484375,
      "activations/layer3_attention_weight_max": 80.75454711914062,
      "activations/layer3_attention_weight_min": -83.63764953613281,
      "activations/layer4_attention_weight_max": 102.43859100341797,
      "activations/layer4_attention_weight_min": -96.53234100341797,
      "activations/layer5_attention_weight_max": 99.88090515136719,
      "activations/layer5_attention_weight_min": -91.13854217529297,
      "activations/layer6_attention_weight_max": 59.073760986328125,
      "activations/layer6_attention_weight_min": -57.67610168457031,
      "activations/layer7_attention_weight_max": 75.06742858886719,
      "activations/layer7_attention_weight_min": -74.75641632080078,
      "activations/layer8_attention_weight_max": 48.80476760864258,
      "activations/layer8_attention_weight_min": -47.5271110534668,
      "activations/layer9_attention_weight_max": 53.90846633911133,
      "activations/layer9_attention_weight_min": -52.94154739379883,
      "epoch": 6.08,
      "learning_rate": 0.00011189431818181818,
      "loss": 2.8186,
      "step": 104700
    },
    {
      "activations/layer0_attention_weight_max": 14.780171394348145,
      "activations/layer0_attention_weight_min": -14.64698600769043,
      "activations/layer10_attention_weight_max": 40.030364990234375,
      "activations/layer10_attention_weight_min": -34.078495025634766,
      "activations/layer11_attention_weight_max": 31.403457641601562,
      "activations/layer11_attention_weight_min": -29.59613800048828,
      "activations/layer12_attention_weight_max": 23.397260665893555,
      "activations/layer12_attention_weight_min": -21.82444190979004,
      "activations/layer13_attention_weight_max": 50.571754455566406,
      "activations/layer13_attention_weight_min": -35.945552825927734,
      "activations/layer14_attention_weight_max": 50.612125396728516,
      "activations/layer14_attention_weight_min": -31.158824920654297,
      "activations/layer15_attention_weight_max": 39.132118225097656,
      "activations/layer15_attention_weight_min": -29.2739315032959,
      "activations/layer16_attention_weight_max": 32.637428283691406,
      "activations/layer16_attention_weight_min": -26.657176971435547,
      "activations/layer17_attention_weight_max": 58.59602355957031,
      "activations/layer17_attention_weight_min": -40.88423538208008,
      "activations/layer18_attention_weight_max": 49.63375473022461,
      "activations/layer18_attention_weight_min": -37.05396270751953,
      "activations/layer19_attention_weight_max": 24.246517181396484,
      "activations/layer19_attention_weight_min": -20.81284523010254,
      "activations/layer1_attention_weight_max": 16.192459106445312,
      "activations/layer1_attention_weight_min": -13.23032283782959,
      "activations/layer20_attention_weight_max": 23.33419418334961,
      "activations/layer20_attention_weight_min": -24.499839782714844,
      "activations/layer21_attention_weight_max": 37.17913818359375,
      "activations/layer21_attention_weight_min": -27.579784393310547,
      "activations/layer22_attention_weight_max": 36.74356460571289,
      "activations/layer22_attention_weight_min": -24.563018798828125,
      "activations/layer23_attention_weight_max": 30.074974060058594,
      "activations/layer23_attention_weight_min": -19.123022079467773,
      "activations/layer2_attention_weight_max": 31.4683837890625,
      "activations/layer2_attention_weight_min": -28.553159713745117,
      "activations/layer3_attention_weight_max": 79.06076049804688,
      "activations/layer3_attention_weight_min": -86.43003845214844,
      "activations/layer4_attention_weight_max": 95.77001953125,
      "activations/layer4_attention_weight_min": -92.60840606689453,
      "activations/layer5_attention_weight_max": 91.19775390625,
      "activations/layer5_attention_weight_min": -82.80581665039062,
      "activations/layer6_attention_weight_max": 54.2990837097168,
      "activations/layer6_attention_weight_min": -50.907569885253906,
      "activations/layer7_attention_weight_max": 68.67359924316406,
      "activations/layer7_attention_weight_min": -69.28241729736328,
      "activations/layer8_attention_weight_max": 44.9223747253418,
      "activations/layer8_attention_weight_min": -42.63117218017578,
      "activations/layer9_attention_weight_max": 48.146602630615234,
      "activations/layer9_attention_weight_min": -45.12785339355469,
      "epoch": 6.09,
      "learning_rate": 0.00011187537878787877,
      "loss": 2.8285,
      "step": 104750
    },
    {
      "activations/layer0_attention_weight_max": 15.799386978149414,
      "activations/layer0_attention_weight_min": -14.889892578125,
      "activations/layer10_attention_weight_max": 41.06972122192383,
      "activations/layer10_attention_weight_min": -37.55434036254883,
      "activations/layer11_attention_weight_max": 32.520896911621094,
      "activations/layer11_attention_weight_min": -29.485698699951172,
      "activations/layer12_attention_weight_max": 23.273054122924805,
      "activations/layer12_attention_weight_min": -26.569477081298828,
      "activations/layer13_attention_weight_max": 53.28246307373047,
      "activations/layer13_attention_weight_min": -35.53440475463867,
      "activations/layer14_attention_weight_max": 40.868492126464844,
      "activations/layer14_attention_weight_min": -29.802595138549805,
      "activations/layer15_attention_weight_max": 43.456138610839844,
      "activations/layer15_attention_weight_min": -28.439298629760742,
      "activations/layer16_attention_weight_max": 35.783939361572266,
      "activations/layer16_attention_weight_min": -28.40351676940918,
      "activations/layer17_attention_weight_max": 58.02377700805664,
      "activations/layer17_attention_weight_min": -40.735233306884766,
      "activations/layer18_attention_weight_max": 49.018428802490234,
      "activations/layer18_attention_weight_min": -33.40638732910156,
      "activations/layer19_attention_weight_max": 24.296579360961914,
      "activations/layer19_attention_weight_min": -18.31550407409668,
      "activations/layer1_attention_weight_max": 16.580364227294922,
      "activations/layer1_attention_weight_min": -12.862332344055176,
      "activations/layer20_attention_weight_max": 21.77857208251953,
      "activations/layer20_attention_weight_min": -20.06856918334961,
      "activations/layer21_attention_weight_max": 35.76387405395508,
      "activations/layer21_attention_weight_min": -23.6392765045166,
      "activations/layer22_attention_weight_max": 32.50569534301758,
      "activations/layer22_attention_weight_min": -22.966909408569336,
      "activations/layer23_attention_weight_max": 34.01074981689453,
      "activations/layer23_attention_weight_min": -20.193601608276367,
      "activations/layer2_attention_weight_max": 31.660125732421875,
      "activations/layer2_attention_weight_min": -27.61335563659668,
      "activations/layer3_attention_weight_max": 81.04603576660156,
      "activations/layer3_attention_weight_min": -80.24386596679688,
      "activations/layer4_attention_weight_max": 98.85820770263672,
      "activations/layer4_attention_weight_min": -96.24223327636719,
      "activations/layer5_attention_weight_max": 98.1162109375,
      "activations/layer5_attention_weight_min": -89.81851196289062,
      "activations/layer6_attention_weight_max": 57.06587600708008,
      "activations/layer6_attention_weight_min": -54.26924514770508,
      "activations/layer7_attention_weight_max": 78.2930908203125,
      "activations/layer7_attention_weight_min": -70.95698547363281,
      "activations/layer8_attention_weight_max": 48.094078063964844,
      "activations/layer8_attention_weight_min": -45.200103759765625,
      "activations/layer9_attention_weight_max": 51.5572509765625,
      "activations/layer9_attention_weight_min": -52.250057220458984,
      "epoch": 6.09,
      "learning_rate": 0.00011185643939393939,
      "loss": 2.8356,
      "step": 104800
    },
    {
      "activations/layer0_attention_weight_max": 14.649036407470703,
      "activations/layer0_attention_weight_min": -15.136068344116211,
      "activations/layer10_attention_weight_max": 37.833274841308594,
      "activations/layer10_attention_weight_min": -33.75391387939453,
      "activations/layer11_attention_weight_max": 28.666873931884766,
      "activations/layer11_attention_weight_min": -28.688800811767578,
      "activations/layer12_attention_weight_max": 23.1043701171875,
      "activations/layer12_attention_weight_min": -25.15204620361328,
      "activations/layer13_attention_weight_max": 59.440391540527344,
      "activations/layer13_attention_weight_min": -35.281898498535156,
      "activations/layer14_attention_weight_max": 36.22657775878906,
      "activations/layer14_attention_weight_min": -30.037031173706055,
      "activations/layer15_attention_weight_max": 40.865665435791016,
      "activations/layer15_attention_weight_min": -28.58642578125,
      "activations/layer16_attention_weight_max": 32.785980224609375,
      "activations/layer16_attention_weight_min": -27.363752365112305,
      "activations/layer17_attention_weight_max": 51.49752426147461,
      "activations/layer17_attention_weight_min": -37.419639587402344,
      "activations/layer18_attention_weight_max": 44.146793365478516,
      "activations/layer18_attention_weight_min": -35.174957275390625,
      "activations/layer19_attention_weight_max": 19.586843490600586,
      "activations/layer19_attention_weight_min": -20.548654556274414,
      "activations/layer1_attention_weight_max": 17.795156478881836,
      "activations/layer1_attention_weight_min": -14.912031173706055,
      "activations/layer20_attention_weight_max": 20.580869674682617,
      "activations/layer20_attention_weight_min": -23.5504093170166,
      "activations/layer21_attention_weight_max": 35.13051986694336,
      "activations/layer21_attention_weight_min": -26.26505470275879,
      "activations/layer22_attention_weight_max": 30.129413604736328,
      "activations/layer22_attention_weight_min": -25.1058406829834,
      "activations/layer23_attention_weight_max": 30.39181900024414,
      "activations/layer23_attention_weight_min": -19.39495277404785,
      "activations/layer2_attention_weight_max": 30.144439697265625,
      "activations/layer2_attention_weight_min": -28.829269409179688,
      "activations/layer3_attention_weight_max": 81.48970794677734,
      "activations/layer3_attention_weight_min": -85.66500854492188,
      "activations/layer4_attention_weight_max": 96.62451171875,
      "activations/layer4_attention_weight_min": -90.16341400146484,
      "activations/layer5_attention_weight_max": 92.5018539428711,
      "activations/layer5_attention_weight_min": -84.35136413574219,
      "activations/layer6_attention_weight_max": 56.56827926635742,
      "activations/layer6_attention_weight_min": -50.959800720214844,
      "activations/layer7_attention_weight_max": 69.010986328125,
      "activations/layer7_attention_weight_min": -70.89678955078125,
      "activations/layer8_attention_weight_max": 42.731510162353516,
      "activations/layer8_attention_weight_min": -44.6558723449707,
      "activations/layer9_attention_weight_max": 47.65227127075195,
      "activations/layer9_attention_weight_min": -48.82197570800781,
      "epoch": 6.09,
      "learning_rate": 0.0001118375,
      "loss": 2.8323,
      "step": 104850
    },
    {
      "activations/layer0_attention_weight_max": 14.895923614501953,
      "activations/layer0_attention_weight_min": -14.820202827453613,
      "activations/layer10_attention_weight_max": 38.57505416870117,
      "activations/layer10_attention_weight_min": -34.5809440612793,
      "activations/layer11_attention_weight_max": 33.26573181152344,
      "activations/layer11_attention_weight_min": -28.566884994506836,
      "activations/layer12_attention_weight_max": 21.090085983276367,
      "activations/layer12_attention_weight_min": -24.227325439453125,
      "activations/layer13_attention_weight_max": 49.09007263183594,
      "activations/layer13_attention_weight_min": -33.399688720703125,
      "activations/layer14_attention_weight_max": 44.04745101928711,
      "activations/layer14_attention_weight_min": -29.635534286499023,
      "activations/layer15_attention_weight_max": 42.71082305908203,
      "activations/layer15_attention_weight_min": -28.069059371948242,
      "activations/layer16_attention_weight_max": 35.883609771728516,
      "activations/layer16_attention_weight_min": -26.574054718017578,
      "activations/layer17_attention_weight_max": 58.43934631347656,
      "activations/layer17_attention_weight_min": -43.79446792602539,
      "activations/layer18_attention_weight_max": 50.19602584838867,
      "activations/layer18_attention_weight_min": -36.18474197387695,
      "activations/layer19_attention_weight_max": 24.709550857543945,
      "activations/layer19_attention_weight_min": -21.6370906829834,
      "activations/layer1_attention_weight_max": 17.303897857666016,
      "activations/layer1_attention_weight_min": -16.25838279724121,
      "activations/layer20_attention_weight_max": 21.31961441040039,
      "activations/layer20_attention_weight_min": -24.490921020507812,
      "activations/layer21_attention_weight_max": 39.21157455444336,
      "activations/layer21_attention_weight_min": -24.375402450561523,
      "activations/layer22_attention_weight_max": 38.0569953918457,
      "activations/layer22_attention_weight_min": -24.177337646484375,
      "activations/layer23_attention_weight_max": 31.779661178588867,
      "activations/layer23_attention_weight_min": -18.76046371459961,
      "activations/layer2_attention_weight_max": 30.064735412597656,
      "activations/layer2_attention_weight_min": -28.280311584472656,
      "activations/layer3_attention_weight_max": 78.35002136230469,
      "activations/layer3_attention_weight_min": -83.0608901977539,
      "activations/layer4_attention_weight_max": 95.39327239990234,
      "activations/layer4_attention_weight_min": -92.39881134033203,
      "activations/layer5_attention_weight_max": 92.98135375976562,
      "activations/layer5_attention_weight_min": -84.40330505371094,
      "activations/layer6_attention_weight_max": 57.09437561035156,
      "activations/layer6_attention_weight_min": -51.58346939086914,
      "activations/layer7_attention_weight_max": 70.27790832519531,
      "activations/layer7_attention_weight_min": -74.942626953125,
      "activations/layer8_attention_weight_max": 46.02375030517578,
      "activations/layer8_attention_weight_min": -44.424888610839844,
      "activations/layer9_attention_weight_max": 52.5323371887207,
      "activations/layer9_attention_weight_min": -47.229488372802734,
      "epoch": 6.1,
      "learning_rate": 0.00011181856060606059,
      "loss": 2.8358,
      "step": 104900
    },
    {
      "activations/layer0_attention_weight_max": 14.474128723144531,
      "activations/layer0_attention_weight_min": -15.113327026367188,
      "activations/layer10_attention_weight_max": 38.74510955810547,
      "activations/layer10_attention_weight_min": -34.78350830078125,
      "activations/layer11_attention_weight_max": 29.652463912963867,
      "activations/layer11_attention_weight_min": -28.11172866821289,
      "activations/layer12_attention_weight_max": 19.95374298095703,
      "activations/layer12_attention_weight_min": -24.250003814697266,
      "activations/layer13_attention_weight_max": 47.34420394897461,
      "activations/layer13_attention_weight_min": -31.325565338134766,
      "activations/layer14_attention_weight_max": 44.007904052734375,
      "activations/layer14_attention_weight_min": -30.650665283203125,
      "activations/layer15_attention_weight_max": 38.05966567993164,
      "activations/layer15_attention_weight_min": -28.286434173583984,
      "activations/layer16_attention_weight_max": 33.51736831665039,
      "activations/layer16_attention_weight_min": -27.729013442993164,
      "activations/layer17_attention_weight_max": 55.9636344909668,
      "activations/layer17_attention_weight_min": -40.53083038330078,
      "activations/layer18_attention_weight_max": 45.31930923461914,
      "activations/layer18_attention_weight_min": -32.915069580078125,
      "activations/layer19_attention_weight_max": 21.058393478393555,
      "activations/layer19_attention_weight_min": -21.04133415222168,
      "activations/layer1_attention_weight_max": 17.311725616455078,
      "activations/layer1_attention_weight_min": -14.516287803649902,
      "activations/layer20_attention_weight_max": 20.697891235351562,
      "activations/layer20_attention_weight_min": -21.862895965576172,
      "activations/layer21_attention_weight_max": 40.755775451660156,
      "activations/layer21_attention_weight_min": -25.394302368164062,
      "activations/layer22_attention_weight_max": 33.55497360229492,
      "activations/layer22_attention_weight_min": -23.7316837310791,
      "activations/layer23_attention_weight_max": 38.04186248779297,
      "activations/layer23_attention_weight_min": -18.78927230834961,
      "activations/layer2_attention_weight_max": 29.065750122070312,
      "activations/layer2_attention_weight_min": -28.47957992553711,
      "activations/layer3_attention_weight_max": 80.4146957397461,
      "activations/layer3_attention_weight_min": -83.20709991455078,
      "activations/layer4_attention_weight_max": 99.22114562988281,
      "activations/layer4_attention_weight_min": -92.89453125,
      "activations/layer5_attention_weight_max": 96.18202209472656,
      "activations/layer5_attention_weight_min": -84.83567810058594,
      "activations/layer6_attention_weight_max": 56.03826141357422,
      "activations/layer6_attention_weight_min": -50.736263275146484,
      "activations/layer7_attention_weight_max": 73.26559448242188,
      "activations/layer7_attention_weight_min": -73.97875213623047,
      "activations/layer8_attention_weight_max": 46.312957763671875,
      "activations/layer8_attention_weight_min": -44.904258728027344,
      "activations/layer9_attention_weight_max": 55.867855072021484,
      "activations/layer9_attention_weight_min": -48.597503662109375,
      "epoch": 6.1,
      "learning_rate": 0.0001117996212121212,
      "loss": 2.8232,
      "step": 104950
    },
    {
      "activations/layer0_attention_weight_max": 14.535478591918945,
      "activations/layer0_attention_weight_min": -15.064800262451172,
      "activations/layer10_attention_weight_max": 36.79167175292969,
      "activations/layer10_attention_weight_min": -32.34474182128906,
      "activations/layer11_attention_weight_max": 27.886863708496094,
      "activations/layer11_attention_weight_min": -27.340389251708984,
      "activations/layer12_attention_weight_max": 22.064023971557617,
      "activations/layer12_attention_weight_min": -26.000032424926758,
      "activations/layer13_attention_weight_max": 58.121246337890625,
      "activations/layer13_attention_weight_min": -35.11286926269531,
      "activations/layer14_attention_weight_max": 69.35905456542969,
      "activations/layer14_attention_weight_min": -32.95354080200195,
      "activations/layer15_attention_weight_max": 39.55255126953125,
      "activations/layer15_attention_weight_min": -29.5842342376709,
      "activations/layer16_attention_weight_max": 35.41483688354492,
      "activations/layer16_attention_weight_min": -26.19140625,
      "activations/layer17_attention_weight_max": 58.49747848510742,
      "activations/layer17_attention_weight_min": -42.681861877441406,
      "activations/layer18_attention_weight_max": 49.63750457763672,
      "activations/layer18_attention_weight_min": -35.97039794921875,
      "activations/layer19_attention_weight_max": 22.32712745666504,
      "activations/layer19_attention_weight_min": -22.126434326171875,
      "activations/layer1_attention_weight_max": 16.771541595458984,
      "activations/layer1_attention_weight_min": -15.336084365844727,
      "activations/layer20_attention_weight_max": 24.426475524902344,
      "activations/layer20_attention_weight_min": -21.192745208740234,
      "activations/layer21_attention_weight_max": 41.81452560424805,
      "activations/layer21_attention_weight_min": -24.033899307250977,
      "activations/layer22_attention_weight_max": 39.06541442871094,
      "activations/layer22_attention_weight_min": -23.5875301361084,
      "activations/layer23_attention_weight_max": 34.084632873535156,
      "activations/layer23_attention_weight_min": -19.496461868286133,
      "activations/layer2_attention_weight_max": 30.747150421142578,
      "activations/layer2_attention_weight_min": -28.387502670288086,
      "activations/layer3_attention_weight_max": 79.37176513671875,
      "activations/layer3_attention_weight_min": -85.10784149169922,
      "activations/layer4_attention_weight_max": 97.89542388916016,
      "activations/layer4_attention_weight_min": -91.01543426513672,
      "activations/layer5_attention_weight_max": 93.98757934570312,
      "activations/layer5_attention_weight_min": -84.89844512939453,
      "activations/layer6_attention_weight_max": 55.35294723510742,
      "activations/layer6_attention_weight_min": -52.01713180541992,
      "activations/layer7_attention_weight_max": 69.69104766845703,
      "activations/layer7_attention_weight_min": -74.581298828125,
      "activations/layer8_attention_weight_max": 42.17905044555664,
      "activations/layer8_attention_weight_min": -40.50354766845703,
      "activations/layer9_attention_weight_max": 47.37166976928711,
      "activations/layer9_attention_weight_min": -47.33224868774414,
      "epoch": 6.1,
      "learning_rate": 0.00011178068181818182,
      "loss": 2.8306,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_loss": 2.78515625,
      "eval_runtime": 8.6035,
      "eval_samples_per_second": 499.098,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_openwebtext_loss": 2.78515625,
      "eval_openwebtext_ppl": 16.202349296067883,
      "eval_openwebtext_runtime": 8.6035,
      "eval_openwebtext_samples_per_second": 499.098,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 2.02,
      "eval_wikitext_samples_per_second": 225.742,
      "step": 105000
    },
    {
      "epoch": 6.1,
      "eval_lambada_loss": 2.677734375,
      "eval_lambada_ppl": 14.552086348683558,
      "eval_lambada_runtime": 9.6775,
      "eval_lambada_samples_per_second": 503.128,
      "step": 105000
    },
    {
      "activations/layer0_attention_weight_max": 14.112452507019043,
      "activations/layer0_attention_weight_min": -15.524479866027832,
      "activations/layer10_attention_weight_max": 38.92470169067383,
      "activations/layer10_attention_weight_min": -34.866981506347656,
      "activations/layer11_attention_weight_max": 30.633102416992188,
      "activations/layer11_attention_weight_min": -28.865415573120117,
      "activations/layer12_attention_weight_max": 18.436634063720703,
      "activations/layer12_attention_weight_min": -28.084810256958008,
      "activations/layer13_attention_weight_max": 54.11103439331055,
      "activations/layer13_attention_weight_min": -40.32181930541992,
      "activations/layer14_attention_weight_max": 39.6893196105957,
      "activations/layer14_attention_weight_min": -30.529857635498047,
      "activations/layer15_attention_weight_max": 38.445858001708984,
      "activations/layer15_attention_weight_min": -30.729957580566406,
      "activations/layer16_attention_weight_max": 33.16217803955078,
      "activations/layer16_attention_weight_min": -26.36919403076172,
      "activations/layer17_attention_weight_max": 54.06428909301758,
      "activations/layer17_attention_weight_min": -39.048213958740234,
      "activations/layer18_attention_weight_max": 47.8547477722168,
      "activations/layer18_attention_weight_min": -34.1696891784668,
      "activations/layer19_attention_weight_max": 24.546789169311523,
      "activations/layer19_attention_weight_min": -20.14571762084961,
      "activations/layer1_attention_weight_max": 18.003612518310547,
      "activations/layer1_attention_weight_min": -14.045992851257324,
      "activations/layer20_attention_weight_max": 22.154232025146484,
      "activations/layer20_attention_weight_min": -21.046342849731445,
      "activations/layer21_attention_weight_max": 39.210487365722656,
      "activations/layer21_attention_weight_min": -26.022899627685547,
      "activations/layer22_attention_weight_max": 35.24752426147461,
      "activations/layer22_attention_weight_min": -23.267711639404297,
      "activations/layer23_attention_weight_max": 27.906410217285156,
      "activations/layer23_attention_weight_min": -18.288734436035156,
      "activations/layer2_attention_weight_max": 30.3409423828125,
      "activations/layer2_attention_weight_min": -28.512577056884766,
      "activations/layer3_attention_weight_max": 80.68812561035156,
      "activations/layer3_attention_weight_min": -84.34062194824219,
      "activations/layer4_attention_weight_max": 98.93846130371094,
      "activations/layer4_attention_weight_min": -95.10033416748047,
      "activations/layer5_attention_weight_max": 97.35286712646484,
      "activations/layer5_attention_weight_min": -88.73866271972656,
      "activations/layer6_attention_weight_max": 57.41037368774414,
      "activations/layer6_attention_weight_min": -53.150962829589844,
      "activations/layer7_attention_weight_max": 76.7971420288086,
      "activations/layer7_attention_weight_min": -73.2311019897461,
      "activations/layer8_attention_weight_max": 43.27361297607422,
      "activations/layer8_attention_weight_min": -44.54644012451172,
      "activations/layer9_attention_weight_max": 55.37873077392578,
      "activations/layer9_attention_weight_min": -48.70378112792969,
      "epoch": 6.1,
      "learning_rate": 0.00011176212121212121,
      "loss": 2.8198,
      "step": 105050
    },
    {
      "activations/layer0_attention_weight_max": 15.151247024536133,
      "activations/layer0_attention_weight_min": -14.799540519714355,
      "activations/layer10_attention_weight_max": 38.433616638183594,
      "activations/layer10_attention_weight_min": -37.156105041503906,
      "activations/layer11_attention_weight_max": 29.84218406677246,
      "activations/layer11_attention_weight_min": -29.43501091003418,
      "activations/layer12_attention_weight_max": 23.714691162109375,
      "activations/layer12_attention_weight_min": -33.1439094543457,
      "activations/layer13_attention_weight_max": 48.91603088378906,
      "activations/layer13_attention_weight_min": -31.92186737060547,
      "activations/layer14_attention_weight_max": 48.11090850830078,
      "activations/layer14_attention_weight_min": -28.854812622070312,
      "activations/layer15_attention_weight_max": 42.997318267822266,
      "activations/layer15_attention_weight_min": -27.877647399902344,
      "activations/layer16_attention_weight_max": 35.96244812011719,
      "activations/layer16_attention_weight_min": -26.71763801574707,
      "activations/layer17_attention_weight_max": 55.71376037597656,
      "activations/layer17_attention_weight_min": -41.199440002441406,
      "activations/layer18_attention_weight_max": 47.580177307128906,
      "activations/layer18_attention_weight_min": -36.312103271484375,
      "activations/layer19_attention_weight_max": 22.82022476196289,
      "activations/layer19_attention_weight_min": -19.73619842529297,
      "activations/layer1_attention_weight_max": 17.28118133544922,
      "activations/layer1_attention_weight_min": -14.751277923583984,
      "activations/layer20_attention_weight_max": 24.018264770507812,
      "activations/layer20_attention_weight_min": -21.50188446044922,
      "activations/layer21_attention_weight_max": 42.68645477294922,
      "activations/layer21_attention_weight_min": -29.104629516601562,
      "activations/layer22_attention_weight_max": 35.79801940917969,
      "activations/layer22_attention_weight_min": -25.673385620117188,
      "activations/layer23_attention_weight_max": 32.66188049316406,
      "activations/layer23_attention_weight_min": -22.01331901550293,
      "activations/layer2_attention_weight_max": 29.49146842956543,
      "activations/layer2_attention_weight_min": -27.746936798095703,
      "activations/layer3_attention_weight_max": 82.84130096435547,
      "activations/layer3_attention_weight_min": -83.35563659667969,
      "activations/layer4_attention_weight_max": 98.00315856933594,
      "activations/layer4_attention_weight_min": -90.14363861083984,
      "activations/layer5_attention_weight_max": 88.89993286132812,
      "activations/layer5_attention_weight_min": -81.01953887939453,
      "activations/layer6_attention_weight_max": 54.19154357910156,
      "activations/layer6_attention_weight_min": -49.034202575683594,
      "activations/layer7_attention_weight_max": 70.5094223022461,
      "activations/layer7_attention_weight_min": -67.64637756347656,
      "activations/layer8_attention_weight_max": 42.26947784423828,
      "activations/layer8_attention_weight_min": -40.12445831298828,
      "activations/layer9_attention_weight_max": 56.646705627441406,
      "activations/layer9_attention_weight_min": -52.333648681640625,
      "epoch": 6.11,
      "learning_rate": 0.0001117431818181818,
      "loss": 2.8278,
      "step": 105100
    },
    {
      "activations/layer0_attention_weight_max": 15.39575481414795,
      "activations/layer0_attention_weight_min": -14.977446556091309,
      "activations/layer10_attention_weight_max": 37.87790298461914,
      "activations/layer10_attention_weight_min": -36.168704986572266,
      "activations/layer11_attention_weight_max": 29.405357360839844,
      "activations/layer11_attention_weight_min": -29.392730712890625,
      "activations/layer12_attention_weight_max": 28.456680297851562,
      "activations/layer12_attention_weight_min": -24.512277603149414,
      "activations/layer13_attention_weight_max": 43.46097183227539,
      "activations/layer13_attention_weight_min": -34.610107421875,
      "activations/layer14_attention_weight_max": 39.22409439086914,
      "activations/layer14_attention_weight_min": -30.396997451782227,
      "activations/layer15_attention_weight_max": 36.341514587402344,
      "activations/layer15_attention_weight_min": -30.5941104888916,
      "activations/layer16_attention_weight_max": 32.58184814453125,
      "activations/layer16_attention_weight_min": -27.36157989501953,
      "activations/layer17_attention_weight_max": 54.50669860839844,
      "activations/layer17_attention_weight_min": -41.987552642822266,
      "activations/layer18_attention_weight_max": 45.65156555175781,
      "activations/layer18_attention_weight_min": -36.059295654296875,
      "activations/layer19_attention_weight_max": 22.849435806274414,
      "activations/layer19_attention_weight_min": -20.03459358215332,
      "activations/layer1_attention_weight_max": 16.928808212280273,
      "activations/layer1_attention_weight_min": -13.967474937438965,
      "activations/layer20_attention_weight_max": 20.723539352416992,
      "activations/layer20_attention_weight_min": -21.2525691986084,
      "activations/layer21_attention_weight_max": 46.50183868408203,
      "activations/layer21_attention_weight_min": -26.709392547607422,
      "activations/layer22_attention_weight_max": 34.86697769165039,
      "activations/layer22_attention_weight_min": -26.894760131835938,
      "activations/layer23_attention_weight_max": 31.158666610717773,
      "activations/layer23_attention_weight_min": -20.19930648803711,
      "activations/layer2_attention_weight_max": 29.096708297729492,
      "activations/layer2_attention_weight_min": -27.910560607910156,
      "activations/layer3_attention_weight_max": 81.93228912353516,
      "activations/layer3_attention_weight_min": -88.29047393798828,
      "activations/layer4_attention_weight_max": 97.26427459716797,
      "activations/layer4_attention_weight_min": -92.18477630615234,
      "activations/layer5_attention_weight_max": 92.47518920898438,
      "activations/layer5_attention_weight_min": -82.49493408203125,
      "activations/layer6_attention_weight_max": 55.76889419555664,
      "activations/layer6_attention_weight_min": -50.289894104003906,
      "activations/layer7_attention_weight_max": 68.32452392578125,
      "activations/layer7_attention_weight_min": -66.80314636230469,
      "activations/layer8_attention_weight_max": 40.98124313354492,
      "activations/layer8_attention_weight_min": -42.907264709472656,
      "activations/layer9_attention_weight_max": 48.09219741821289,
      "activations/layer9_attention_weight_min": -46.473575592041016,
      "epoch": 6.11,
      "learning_rate": 0.00011172424242424241,
      "loss": 2.8259,
      "step": 105150
    },
    {
      "activations/layer0_attention_weight_max": 14.962828636169434,
      "activations/layer0_attention_weight_min": -15.243865013122559,
      "activations/layer10_attention_weight_max": 39.817840576171875,
      "activations/layer10_attention_weight_min": -34.980892181396484,
      "activations/layer11_attention_weight_max": 28.691200256347656,
      "activations/layer11_attention_weight_min": -27.519681930541992,
      "activations/layer12_attention_weight_max": 21.536983489990234,
      "activations/layer12_attention_weight_min": -26.551883697509766,
      "activations/layer13_attention_weight_max": 47.7990608215332,
      "activations/layer13_attention_weight_min": -32.5804443359375,
      "activations/layer14_attention_weight_max": 43.25271987915039,
      "activations/layer14_attention_weight_min": -31.756065368652344,
      "activations/layer15_attention_weight_max": 40.09978485107422,
      "activations/layer15_attention_weight_min": -30.42105484008789,
      "activations/layer16_attention_weight_max": 35.92428207397461,
      "activations/layer16_attention_weight_min": -26.194561004638672,
      "activations/layer17_attention_weight_max": 61.75453567504883,
      "activations/layer17_attention_weight_min": -39.88167953491211,
      "activations/layer18_attention_weight_max": 48.97562789916992,
      "activations/layer18_attention_weight_min": -34.40685272216797,
      "activations/layer19_attention_weight_max": 23.725717544555664,
      "activations/layer19_attention_weight_min": -21.28935432434082,
      "activations/layer1_attention_weight_max": 17.480674743652344,
      "activations/layer1_attention_weight_min": -13.748991966247559,
      "activations/layer20_attention_weight_max": 20.607521057128906,
      "activations/layer20_attention_weight_min": -21.14927864074707,
      "activations/layer21_attention_weight_max": 43.54928970336914,
      "activations/layer21_attention_weight_min": -24.341299057006836,
      "activations/layer22_attention_weight_max": 37.6982307434082,
      "activations/layer22_attention_weight_min": -25.218353271484375,
      "activations/layer23_attention_weight_max": 32.07746887207031,
      "activations/layer23_attention_weight_min": -20.560739517211914,
      "activations/layer2_attention_weight_max": 27.165000915527344,
      "activations/layer2_attention_weight_min": -27.215070724487305,
      "activations/layer3_attention_weight_max": 74.20967102050781,
      "activations/layer3_attention_weight_min": -79.78369903564453,
      "activations/layer4_attention_weight_max": 93.87329864501953,
      "activations/layer4_attention_weight_min": -89.18794250488281,
      "activations/layer5_attention_weight_max": 90.84032440185547,
      "activations/layer5_attention_weight_min": -83.08775329589844,
      "activations/layer6_attention_weight_max": 53.94302749633789,
      "activations/layer6_attention_weight_min": -49.06047821044922,
      "activations/layer7_attention_weight_max": 68.84695434570312,
      "activations/layer7_attention_weight_min": -67.56812286376953,
      "activations/layer8_attention_weight_max": 41.083824157714844,
      "activations/layer8_attention_weight_min": -41.35065460205078,
      "activations/layer9_attention_weight_max": 48.451438903808594,
      "activations/layer9_attention_weight_min": -46.79111862182617,
      "epoch": 6.11,
      "learning_rate": 0.00011170530303030303,
      "loss": 2.8197,
      "step": 105200
    },
    {
      "activations/layer0_attention_weight_max": 15.493328094482422,
      "activations/layer0_attention_weight_min": -15.732316017150879,
      "activations/layer10_attention_weight_max": 40.627593994140625,
      "activations/layer10_attention_weight_min": -38.88360595703125,
      "activations/layer11_attention_weight_max": 29.707311630249023,
      "activations/layer11_attention_weight_min": -28.32307243347168,
      "activations/layer12_attention_weight_max": 21.150732040405273,
      "activations/layer12_attention_weight_min": -26.194772720336914,
      "activations/layer13_attention_weight_max": 45.99031448364258,
      "activations/layer13_attention_weight_min": -36.73724365234375,
      "activations/layer14_attention_weight_max": 48.15349197387695,
      "activations/layer14_attention_weight_min": -38.31138229370117,
      "activations/layer15_attention_weight_max": 39.43330383300781,
      "activations/layer15_attention_weight_min": -33.60069274902344,
      "activations/layer16_attention_weight_max": 37.13168716430664,
      "activations/layer16_attention_weight_min": -24.97715950012207,
      "activations/layer17_attention_weight_max": 54.219261169433594,
      "activations/layer17_attention_weight_min": -38.22706985473633,
      "activations/layer18_attention_weight_max": 50.17790985107422,
      "activations/layer18_attention_weight_min": -33.6822509765625,
      "activations/layer19_attention_weight_max": 24.03023338317871,
      "activations/layer19_attention_weight_min": -18.779014587402344,
      "activations/layer1_attention_weight_max": 17.61406707763672,
      "activations/layer1_attention_weight_min": -14.574037551879883,
      "activations/layer20_attention_weight_max": 20.8706111907959,
      "activations/layer20_attention_weight_min": -21.0888671875,
      "activations/layer21_attention_weight_max": 35.638877868652344,
      "activations/layer21_attention_weight_min": -23.88006019592285,
      "activations/layer22_attention_weight_max": 36.88869857788086,
      "activations/layer22_attention_weight_min": -24.00334930419922,
      "activations/layer23_attention_weight_max": 32.16716003417969,
      "activations/layer23_attention_weight_min": -22.294048309326172,
      "activations/layer2_attention_weight_max": 31.213315963745117,
      "activations/layer2_attention_weight_min": -31.228412628173828,
      "activations/layer3_attention_weight_max": 81.79325103759766,
      "activations/layer3_attention_weight_min": -86.33228302001953,
      "activations/layer4_attention_weight_max": 99.99995422363281,
      "activations/layer4_attention_weight_min": -96.68637084960938,
      "activations/layer5_attention_weight_max": 96.97923278808594,
      "activations/layer5_attention_weight_min": -91.40594482421875,
      "activations/layer6_attention_weight_max": 60.04603958129883,
      "activations/layer6_attention_weight_min": -55.77019119262695,
      "activations/layer7_attention_weight_max": 76.88154602050781,
      "activations/layer7_attention_weight_min": -73.24616241455078,
      "activations/layer8_attention_weight_max": 45.753013610839844,
      "activations/layer8_attention_weight_min": -46.46923828125,
      "activations/layer9_attention_weight_max": 57.5356559753418,
      "activations/layer9_attention_weight_min": -53.270198822021484,
      "epoch": 6.12,
      "learning_rate": 0.00011168636363636363,
      "loss": 2.8372,
      "step": 105250
    },
    {
      "activations/layer0_attention_weight_max": 14.495345115661621,
      "activations/layer0_attention_weight_min": -15.127767562866211,
      "activations/layer10_attention_weight_max": 40.237892150878906,
      "activations/layer10_attention_weight_min": -35.9976806640625,
      "activations/layer11_attention_weight_max": 29.98003387451172,
      "activations/layer11_attention_weight_min": -29.504100799560547,
      "activations/layer12_attention_weight_max": 22.723222732543945,
      "activations/layer12_attention_weight_min": -24.863746643066406,
      "activations/layer13_attention_weight_max": 55.648712158203125,
      "activations/layer13_attention_weight_min": -35.64874267578125,
      "activations/layer14_attention_weight_max": 41.345733642578125,
      "activations/layer14_attention_weight_min": -30.684680938720703,
      "activations/layer15_attention_weight_max": 42.516380310058594,
      "activations/layer15_attention_weight_min": -30.810213088989258,
      "activations/layer16_attention_weight_max": 38.43569564819336,
      "activations/layer16_attention_weight_min": -26.26901626586914,
      "activations/layer17_attention_weight_max": 56.300350189208984,
      "activations/layer17_attention_weight_min": -42.64881134033203,
      "activations/layer18_attention_weight_max": 55.073081970214844,
      "activations/layer18_attention_weight_min": -36.06792449951172,
      "activations/layer19_attention_weight_max": 30.92413330078125,
      "activations/layer19_attention_weight_min": -21.38690185546875,
      "activations/layer1_attention_weight_max": 16.589590072631836,
      "activations/layer1_attention_weight_min": -13.428250312805176,
      "activations/layer20_attention_weight_max": 25.937410354614258,
      "activations/layer20_attention_weight_min": -21.971471786499023,
      "activations/layer21_attention_weight_max": 45.041847229003906,
      "activations/layer21_attention_weight_min": -24.465482711791992,
      "activations/layer22_attention_weight_max": 37.25471496582031,
      "activations/layer22_attention_weight_min": -23.35674285888672,
      "activations/layer23_attention_weight_max": 36.379119873046875,
      "activations/layer23_attention_weight_min": -18.92618179321289,
      "activations/layer2_attention_weight_max": 30.76888656616211,
      "activations/layer2_attention_weight_min": -29.54188346862793,
      "activations/layer3_attention_weight_max": 82.30136108398438,
      "activations/layer3_attention_weight_min": -84.2825698852539,
      "activations/layer4_attention_weight_max": 99.78913116455078,
      "activations/layer4_attention_weight_min": -94.71370697021484,
      "activations/layer5_attention_weight_max": 93.64559173583984,
      "activations/layer5_attention_weight_min": -88.3189697265625,
      "activations/layer6_attention_weight_max": 57.51054382324219,
      "activations/layer6_attention_weight_min": -52.69688415527344,
      "activations/layer7_attention_weight_max": 76.73314666748047,
      "activations/layer7_attention_weight_min": -69.38484191894531,
      "activations/layer8_attention_weight_max": 43.078948974609375,
      "activations/layer8_attention_weight_min": -45.27996826171875,
      "activations/layer9_attention_weight_max": 46.34394454956055,
      "activations/layer9_attention_weight_min": -49.31843566894531,
      "epoch": 6.12,
      "learning_rate": 0.00011166742424242423,
      "loss": 2.8101,
      "step": 105300
    },
    {
      "activations/layer0_attention_weight_max": 14.732390403747559,
      "activations/layer0_attention_weight_min": -15.337807655334473,
      "activations/layer10_attention_weight_max": 37.906036376953125,
      "activations/layer10_attention_weight_min": -37.935523986816406,
      "activations/layer11_attention_weight_max": 29.548919677734375,
      "activations/layer11_attention_weight_min": -30.05840301513672,
      "activations/layer12_attention_weight_max": 18.8228702545166,
      "activations/layer12_attention_weight_min": -23.95792579650879,
      "activations/layer13_attention_weight_max": 39.346282958984375,
      "activations/layer13_attention_weight_min": -37.068634033203125,
      "activations/layer14_attention_weight_max": 37.01503372192383,
      "activations/layer14_attention_weight_min": -31.90888023376465,
      "activations/layer15_attention_weight_max": 35.361392974853516,
      "activations/layer15_attention_weight_min": -30.356307983398438,
      "activations/layer16_attention_weight_max": 32.73564529418945,
      "activations/layer16_attention_weight_min": -26.710031509399414,
      "activations/layer17_attention_weight_max": 60.55372619628906,
      "activations/layer17_attention_weight_min": -42.72468948364258,
      "activations/layer18_attention_weight_max": 46.031192779541016,
      "activations/layer18_attention_weight_min": -34.920501708984375,
      "activations/layer19_attention_weight_max": 23.446239471435547,
      "activations/layer19_attention_weight_min": -20.348676681518555,
      "activations/layer1_attention_weight_max": 17.570615768432617,
      "activations/layer1_attention_weight_min": -14.704337120056152,
      "activations/layer20_attention_weight_max": 20.352018356323242,
      "activations/layer20_attention_weight_min": -20.89810562133789,
      "activations/layer21_attention_weight_max": 40.735965728759766,
      "activations/layer21_attention_weight_min": -25.933135986328125,
      "activations/layer22_attention_weight_max": 32.192508697509766,
      "activations/layer22_attention_weight_min": -26.23114776611328,
      "activations/layer23_attention_weight_max": 29.245901107788086,
      "activations/layer23_attention_weight_min": -21.139768600463867,
      "activations/layer2_attention_weight_max": 29.77521324157715,
      "activations/layer2_attention_weight_min": -27.79220199584961,
      "activations/layer3_attention_weight_max": 80.85335540771484,
      "activations/layer3_attention_weight_min": -82.52558135986328,
      "activations/layer4_attention_weight_max": 96.50697326660156,
      "activations/layer4_attention_weight_min": -90.34004211425781,
      "activations/layer5_attention_weight_max": 97.22312927246094,
      "activations/layer5_attention_weight_min": -83.66334533691406,
      "activations/layer6_attention_weight_max": 54.62338638305664,
      "activations/layer6_attention_weight_min": -52.22681427001953,
      "activations/layer7_attention_weight_max": 70.55049133300781,
      "activations/layer7_attention_weight_min": -70.27297973632812,
      "activations/layer8_attention_weight_max": 43.85091018676758,
      "activations/layer8_attention_weight_min": -42.37152862548828,
      "activations/layer9_attention_weight_max": 47.710670471191406,
      "activations/layer9_attention_weight_min": -46.51780319213867,
      "epoch": 6.12,
      "learning_rate": 0.00011164848484848483,
      "loss": 2.8316,
      "step": 105350
    },
    {
      "activations/layer0_attention_weight_max": 14.9116792678833,
      "activations/layer0_attention_weight_min": -15.355256080627441,
      "activations/layer10_attention_weight_max": 35.935028076171875,
      "activations/layer10_attention_weight_min": -34.71717071533203,
      "activations/layer11_attention_weight_max": 29.341873168945312,
      "activations/layer11_attention_weight_min": -28.786808013916016,
      "activations/layer12_attention_weight_max": 20.599716186523438,
      "activations/layer12_attention_weight_min": -30.464555740356445,
      "activations/layer13_attention_weight_max": 46.372249603271484,
      "activations/layer13_attention_weight_min": -32.1599235534668,
      "activations/layer14_attention_weight_max": 38.82124328613281,
      "activations/layer14_attention_weight_min": -30.744733810424805,
      "activations/layer15_attention_weight_max": 38.96339797973633,
      "activations/layer15_attention_weight_min": -29.66117286682129,
      "activations/layer16_attention_weight_max": 32.122581481933594,
      "activations/layer16_attention_weight_min": -25.933917999267578,
      "activations/layer17_attention_weight_max": 54.59700393676758,
      "activations/layer17_attention_weight_min": -38.99515914916992,
      "activations/layer18_attention_weight_max": 46.62314224243164,
      "activations/layer18_attention_weight_min": -33.479461669921875,
      "activations/layer19_attention_weight_max": 20.721166610717773,
      "activations/layer19_attention_weight_min": -19.333770751953125,
      "activations/layer1_attention_weight_max": 17.139625549316406,
      "activations/layer1_attention_weight_min": -16.014450073242188,
      "activations/layer20_attention_weight_max": 19.844810485839844,
      "activations/layer20_attention_weight_min": -22.127103805541992,
      "activations/layer21_attention_weight_max": 37.95793914794922,
      "activations/layer21_attention_weight_min": -24.152679443359375,
      "activations/layer22_attention_weight_max": 32.380104064941406,
      "activations/layer22_attention_weight_min": -25.584333419799805,
      "activations/layer23_attention_weight_max": 31.924283981323242,
      "activations/layer23_attention_weight_min": -20.308799743652344,
      "activations/layer2_attention_weight_max": 29.580829620361328,
      "activations/layer2_attention_weight_min": -28.424514770507812,
      "activations/layer3_attention_weight_max": 80.81134796142578,
      "activations/layer3_attention_weight_min": -82.84309387207031,
      "activations/layer4_attention_weight_max": 95.89534759521484,
      "activations/layer4_attention_weight_min": -89.32805633544922,
      "activations/layer5_attention_weight_max": 91.26246643066406,
      "activations/layer5_attention_weight_min": -83.386474609375,
      "activations/layer6_attention_weight_max": 53.920021057128906,
      "activations/layer6_attention_weight_min": -50.443275451660156,
      "activations/layer7_attention_weight_max": 66.96369171142578,
      "activations/layer7_attention_weight_min": -69.76054382324219,
      "activations/layer8_attention_weight_max": 41.45219802856445,
      "activations/layer8_attention_weight_min": -42.30657196044922,
      "activations/layer9_attention_weight_max": 49.230064392089844,
      "activations/layer9_attention_weight_min": -45.89915084838867,
      "epoch": 6.12,
      "learning_rate": 0.00011162954545454545,
      "loss": 2.8396,
      "step": 105400
    },
    {
      "activations/layer0_attention_weight_max": 15.111948013305664,
      "activations/layer0_attention_weight_min": -14.869412422180176,
      "activations/layer10_attention_weight_max": 38.96443176269531,
      "activations/layer10_attention_weight_min": -34.25560760498047,
      "activations/layer11_attention_weight_max": 30.816246032714844,
      "activations/layer11_attention_weight_min": -28.402801513671875,
      "activations/layer12_attention_weight_max": 19.04905891418457,
      "activations/layer12_attention_weight_min": -23.43708610534668,
      "activations/layer13_attention_weight_max": 50.237972259521484,
      "activations/layer13_attention_weight_min": -36.357749938964844,
      "activations/layer14_attention_weight_max": 39.859527587890625,
      "activations/layer14_attention_weight_min": -31.139789581298828,
      "activations/layer15_attention_weight_max": 39.108856201171875,
      "activations/layer15_attention_weight_min": -28.798480987548828,
      "activations/layer16_attention_weight_max": 33.2416877746582,
      "activations/layer16_attention_weight_min": -26.40057373046875,
      "activations/layer17_attention_weight_max": 55.59670639038086,
      "activations/layer17_attention_weight_min": -41.8112678527832,
      "activations/layer18_attention_weight_max": 47.48835372924805,
      "activations/layer18_attention_weight_min": -36.0041618347168,
      "activations/layer19_attention_weight_max": 20.708181381225586,
      "activations/layer19_attention_weight_min": -23.405563354492188,
      "activations/layer1_attention_weight_max": 16.5391788482666,
      "activations/layer1_attention_weight_min": -15.478141784667969,
      "activations/layer20_attention_weight_max": 22.10726547241211,
      "activations/layer20_attention_weight_min": -22.01405906677246,
      "activations/layer21_attention_weight_max": 36.33179473876953,
      "activations/layer21_attention_weight_min": -25.310531616210938,
      "activations/layer22_attention_weight_max": 35.20508575439453,
      "activations/layer22_attention_weight_min": -24.720991134643555,
      "activations/layer23_attention_weight_max": 29.475860595703125,
      "activations/layer23_attention_weight_min": -22.342586517333984,
      "activations/layer2_attention_weight_max": 31.0144100189209,
      "activations/layer2_attention_weight_min": -28.710805892944336,
      "activations/layer3_attention_weight_max": 83.21324920654297,
      "activations/layer3_attention_weight_min": -88.0416030883789,
      "activations/layer4_attention_weight_max": 102.16606903076172,
      "activations/layer4_attention_weight_min": -104.35711669921875,
      "activations/layer5_attention_weight_max": 98.13775634765625,
      "activations/layer5_attention_weight_min": -88.81959533691406,
      "activations/layer6_attention_weight_max": 59.14641189575195,
      "activations/layer6_attention_weight_min": -53.39427185058594,
      "activations/layer7_attention_weight_max": 74.37284851074219,
      "activations/layer7_attention_weight_min": -72.69036865234375,
      "activations/layer8_attention_weight_max": 43.38446807861328,
      "activations/layer8_attention_weight_min": -43.63840866088867,
      "activations/layer9_attention_weight_max": 50.5230827331543,
      "activations/layer9_attention_weight_min": -53.86665725708008,
      "epoch": 6.13,
      "learning_rate": 0.00011161060606060606,
      "loss": 2.8205,
      "step": 105450
    },
    {
      "activations/layer0_attention_weight_max": 13.985831260681152,
      "activations/layer0_attention_weight_min": -14.955605506896973,
      "activations/layer10_attention_weight_max": 38.13616943359375,
      "activations/layer10_attention_weight_min": -36.207420349121094,
      "activations/layer11_attention_weight_max": 31.968381881713867,
      "activations/layer11_attention_weight_min": -29.245319366455078,
      "activations/layer12_attention_weight_max": 22.333694458007812,
      "activations/layer12_attention_weight_min": -27.086767196655273,
      "activations/layer13_attention_weight_max": 51.741783142089844,
      "activations/layer13_attention_weight_min": -31.060272216796875,
      "activations/layer14_attention_weight_max": 53.26703643798828,
      "activations/layer14_attention_weight_min": -30.544490814208984,
      "activations/layer15_attention_weight_max": 38.179054260253906,
      "activations/layer15_attention_weight_min": -28.89013671875,
      "activations/layer16_attention_weight_max": 33.563232421875,
      "activations/layer16_attention_weight_min": -25.419145584106445,
      "activations/layer17_attention_weight_max": 57.85493469238281,
      "activations/layer17_attention_weight_min": -42.249149322509766,
      "activations/layer18_attention_weight_max": 52.78313446044922,
      "activations/layer18_attention_weight_min": -35.29387283325195,
      "activations/layer19_attention_weight_max": 21.934202194213867,
      "activations/layer19_attention_weight_min": -20.46516227722168,
      "activations/layer1_attention_weight_max": 16.389793395996094,
      "activations/layer1_attention_weight_min": -13.60854721069336,
      "activations/layer20_attention_weight_max": 23.0428524017334,
      "activations/layer20_attention_weight_min": -20.671255111694336,
      "activations/layer21_attention_weight_max": 44.93792724609375,
      "activations/layer21_attention_weight_min": -23.579980850219727,
      "activations/layer22_attention_weight_max": 32.10154724121094,
      "activations/layer22_attention_weight_min": -24.037628173828125,
      "activations/layer23_attention_weight_max": 31.26911163330078,
      "activations/layer23_attention_weight_min": -18.91228485107422,
      "activations/layer2_attention_weight_max": 31.040945053100586,
      "activations/layer2_attention_weight_min": -29.059797286987305,
      "activations/layer3_attention_weight_max": 84.87721252441406,
      "activations/layer3_attention_weight_min": -85.36475372314453,
      "activations/layer4_attention_weight_max": 99.27142333984375,
      "activations/layer4_attention_weight_min": -96.90433502197266,
      "activations/layer5_attention_weight_max": 98.28606414794922,
      "activations/layer5_attention_weight_min": -88.64971923828125,
      "activations/layer6_attention_weight_max": 57.52191162109375,
      "activations/layer6_attention_weight_min": -51.80783462524414,
      "activations/layer7_attention_weight_max": 73.50077819824219,
      "activations/layer7_attention_weight_min": -73.34224700927734,
      "activations/layer8_attention_weight_max": 43.67805862426758,
      "activations/layer8_attention_weight_min": -44.11745071411133,
      "activations/layer9_attention_weight_max": 50.99189376831055,
      "activations/layer9_attention_weight_min": -48.17513656616211,
      "epoch": 6.13,
      "learning_rate": 0.00011159166666666665,
      "loss": 2.8284,
      "step": 105500
    },
    {
      "activations/layer0_attention_weight_max": 14.813131332397461,
      "activations/layer0_attention_weight_min": -15.158041000366211,
      "activations/layer10_attention_weight_max": 36.75212478637695,
      "activations/layer10_attention_weight_min": -34.54240417480469,
      "activations/layer11_attention_weight_max": 29.280744552612305,
      "activations/layer11_attention_weight_min": -31.34151840209961,
      "activations/layer12_attention_weight_max": 25.54316520690918,
      "activations/layer12_attention_weight_min": -25.382410049438477,
      "activations/layer13_attention_weight_max": 62.761192321777344,
      "activations/layer13_attention_weight_min": -34.742042541503906,
      "activations/layer14_attention_weight_max": 48.583396911621094,
      "activations/layer14_attention_weight_min": -31.82765007019043,
      "activations/layer15_attention_weight_max": 36.055084228515625,
      "activations/layer15_attention_weight_min": -29.038837432861328,
      "activations/layer16_attention_weight_max": 42.14588928222656,
      "activations/layer16_attention_weight_min": -25.23958969116211,
      "activations/layer17_attention_weight_max": 52.94881820678711,
      "activations/layer17_attention_weight_min": -37.61574172973633,
      "activations/layer18_attention_weight_max": 50.84733581542969,
      "activations/layer18_attention_weight_min": -31.69462776184082,
      "activations/layer19_attention_weight_max": 20.21393394470215,
      "activations/layer19_attention_weight_min": -18.248586654663086,
      "activations/layer1_attention_weight_max": 17.3857364654541,
      "activations/layer1_attention_weight_min": -14.706897735595703,
      "activations/layer20_attention_weight_max": 23.853546142578125,
      "activations/layer20_attention_weight_min": -24.051721572875977,
      "activations/layer21_attention_weight_max": 41.70893478393555,
      "activations/layer21_attention_weight_min": -23.34810447692871,
      "activations/layer22_attention_weight_max": 33.68303298950195,
      "activations/layer22_attention_weight_min": -23.74461555480957,
      "activations/layer23_attention_weight_max": 30.630332946777344,
      "activations/layer23_attention_weight_min": -19.591867446899414,
      "activations/layer2_attention_weight_max": 30.497589111328125,
      "activations/layer2_attention_weight_min": -28.773900985717773,
      "activations/layer3_attention_weight_max": 79.18965148925781,
      "activations/layer3_attention_weight_min": -82.61213684082031,
      "activations/layer4_attention_weight_max": 93.57814025878906,
      "activations/layer4_attention_weight_min": -90.32691192626953,
      "activations/layer5_attention_weight_max": 88.65850067138672,
      "activations/layer5_attention_weight_min": -87.70913696289062,
      "activations/layer6_attention_weight_max": 53.36988067626953,
      "activations/layer6_attention_weight_min": -49.903560638427734,
      "activations/layer7_attention_weight_max": 70.35586547851562,
      "activations/layer7_attention_weight_min": -67.36316680908203,
      "activations/layer8_attention_weight_max": 40.405784606933594,
      "activations/layer8_attention_weight_min": -41.96565628051758,
      "activations/layer9_attention_weight_max": 50.351043701171875,
      "activations/layer9_attention_weight_min": -49.43838119506836,
      "epoch": 6.13,
      "learning_rate": 0.00011157272727272727,
      "loss": 2.8448,
      "step": 105550
    },
    {
      "activations/layer0_attention_weight_max": 15.428983688354492,
      "activations/layer0_attention_weight_min": -14.997166633605957,
      "activations/layer10_attention_weight_max": 39.12569046020508,
      "activations/layer10_attention_weight_min": -35.229278564453125,
      "activations/layer11_attention_weight_max": 31.848323822021484,
      "activations/layer11_attention_weight_min": -28.48187828063965,
      "activations/layer12_attention_weight_max": 20.113672256469727,
      "activations/layer12_attention_weight_min": -23.160661697387695,
      "activations/layer13_attention_weight_max": 43.69352340698242,
      "activations/layer13_attention_weight_min": -33.35383605957031,
      "activations/layer14_attention_weight_max": 42.20606231689453,
      "activations/layer14_attention_weight_min": -31.6414852142334,
      "activations/layer15_attention_weight_max": 39.133060455322266,
      "activations/layer15_attention_weight_min": -29.713699340820312,
      "activations/layer16_attention_weight_max": 37.9923095703125,
      "activations/layer16_attention_weight_min": -27.725053787231445,
      "activations/layer17_attention_weight_max": 57.4429817199707,
      "activations/layer17_attention_weight_min": -43.503875732421875,
      "activations/layer18_attention_weight_max": 52.873226165771484,
      "activations/layer18_attention_weight_min": -35.6303825378418,
      "activations/layer19_attention_weight_max": 22.090232849121094,
      "activations/layer19_attention_weight_min": -19.290414810180664,
      "activations/layer1_attention_weight_max": 17.17588996887207,
      "activations/layer1_attention_weight_min": -15.347151756286621,
      "activations/layer20_attention_weight_max": 21.688804626464844,
      "activations/layer20_attention_weight_min": -20.908357620239258,
      "activations/layer21_attention_weight_max": 44.769927978515625,
      "activations/layer21_attention_weight_min": -25.81510353088379,
      "activations/layer22_attention_weight_max": 36.203590393066406,
      "activations/layer22_attention_weight_min": -24.946002960205078,
      "activations/layer23_attention_weight_max": 32.00830078125,
      "activations/layer23_attention_weight_min": -19.221799850463867,
      "activations/layer2_attention_weight_max": 32.276329040527344,
      "activations/layer2_attention_weight_min": -29.33734703063965,
      "activations/layer3_attention_weight_max": 82.35352325439453,
      "activations/layer3_attention_weight_min": -86.75080871582031,
      "activations/layer4_attention_weight_max": 100.00469207763672,
      "activations/layer4_attention_weight_min": -95.70213317871094,
      "activations/layer5_attention_weight_max": 93.34374237060547,
      "activations/layer5_attention_weight_min": -84.67193603515625,
      "activations/layer6_attention_weight_max": 54.4985237121582,
      "activations/layer6_attention_weight_min": -50.662784576416016,
      "activations/layer7_attention_weight_max": 70.79721069335938,
      "activations/layer7_attention_weight_min": -68.63372039794922,
      "activations/layer8_attention_weight_max": 43.0258674621582,
      "activations/layer8_attention_weight_min": -43.18928909301758,
      "activations/layer9_attention_weight_max": 52.45117950439453,
      "activations/layer9_attention_weight_min": -47.3717155456543,
      "epoch": 6.14,
      "learning_rate": 0.00011155378787878788,
      "loss": 2.828,
      "step": 105600
    },
    {
      "activations/layer0_attention_weight_max": 14.293505668640137,
      "activations/layer0_attention_weight_min": -15.118739128112793,
      "activations/layer10_attention_weight_max": 38.23419189453125,
      "activations/layer10_attention_weight_min": -37.21927261352539,
      "activations/layer11_attention_weight_max": 30.81452751159668,
      "activations/layer11_attention_weight_min": -29.932384490966797,
      "activations/layer12_attention_weight_max": 28.640132904052734,
      "activations/layer12_attention_weight_min": -26.033357620239258,
      "activations/layer13_attention_weight_max": 43.56910705566406,
      "activations/layer13_attention_weight_min": -31.523231506347656,
      "activations/layer14_attention_weight_max": 43.81364440917969,
      "activations/layer14_attention_weight_min": -29.599336624145508,
      "activations/layer15_attention_weight_max": 37.84296798706055,
      "activations/layer15_attention_weight_min": -27.65070152282715,
      "activations/layer16_attention_weight_max": 32.90122985839844,
      "activations/layer16_attention_weight_min": -25.558914184570312,
      "activations/layer17_attention_weight_max": 53.757164001464844,
      "activations/layer17_attention_weight_min": -39.52705764770508,
      "activations/layer18_attention_weight_max": 47.25544738769531,
      "activations/layer18_attention_weight_min": -34.980133056640625,
      "activations/layer19_attention_weight_max": 25.385900497436523,
      "activations/layer19_attention_weight_min": -21.835439682006836,
      "activations/layer1_attention_weight_max": 16.72113800048828,
      "activations/layer1_attention_weight_min": -14.632126808166504,
      "activations/layer20_attention_weight_max": 27.9205322265625,
      "activations/layer20_attention_weight_min": -21.57261848449707,
      "activations/layer21_attention_weight_max": 55.7784423828125,
      "activations/layer21_attention_weight_min": -28.304729461669922,
      "activations/layer22_attention_weight_max": 36.55057144165039,
      "activations/layer22_attention_weight_min": -24.380130767822266,
      "activations/layer23_attention_weight_max": 34.560691833496094,
      "activations/layer23_attention_weight_min": -19.896106719970703,
      "activations/layer2_attention_weight_max": 31.89289093017578,
      "activations/layer2_attention_weight_min": -30.80292320251465,
      "activations/layer3_attention_weight_max": 83.91969299316406,
      "activations/layer3_attention_weight_min": -89.88397979736328,
      "activations/layer4_attention_weight_max": 104.85540008544922,
      "activations/layer4_attention_weight_min": -95.03807067871094,
      "activations/layer5_attention_weight_max": 98.51441955566406,
      "activations/layer5_attention_weight_min": -87.53968811035156,
      "activations/layer6_attention_weight_max": 57.84051513671875,
      "activations/layer6_attention_weight_min": -53.966739654541016,
      "activations/layer7_attention_weight_max": 73.44728088378906,
      "activations/layer7_attention_weight_min": -73.133544921875,
      "activations/layer8_attention_weight_max": 45.278446197509766,
      "activations/layer8_attention_weight_min": -45.041629791259766,
      "activations/layer9_attention_weight_max": 53.58036422729492,
      "activations/layer9_attention_weight_min": -48.25649642944336,
      "epoch": 6.14,
      "learning_rate": 0.00011153484848484847,
      "loss": 2.8406,
      "step": 105650
    },
    {
      "activations/layer0_attention_weight_max": 15.310284614562988,
      "activations/layer0_attention_weight_min": -15.479266166687012,
      "activations/layer10_attention_weight_max": 36.33474349975586,
      "activations/layer10_attention_weight_min": -33.369529724121094,
      "activations/layer11_attention_weight_max": 28.2579345703125,
      "activations/layer11_attention_weight_min": -28.54390525817871,
      "activations/layer12_attention_weight_max": 20.991188049316406,
      "activations/layer12_attention_weight_min": -24.737760543823242,
      "activations/layer13_attention_weight_max": 49.499977111816406,
      "activations/layer13_attention_weight_min": -35.74421310424805,
      "activations/layer14_attention_weight_max": 51.21057891845703,
      "activations/layer14_attention_weight_min": -32.356414794921875,
      "activations/layer15_attention_weight_max": 38.140419006347656,
      "activations/layer15_attention_weight_min": -29.371776580810547,
      "activations/layer16_attention_weight_max": 35.852020263671875,
      "activations/layer16_attention_weight_min": -27.66942024230957,
      "activations/layer17_attention_weight_max": 56.8771858215332,
      "activations/layer17_attention_weight_min": -41.22605514526367,
      "activations/layer18_attention_weight_max": 49.31475067138672,
      "activations/layer18_attention_weight_min": -35.86227798461914,
      "activations/layer19_attention_weight_max": 22.090972900390625,
      "activations/layer19_attention_weight_min": -19.114408493041992,
      "activations/layer1_attention_weight_max": 17.07286262512207,
      "activations/layer1_attention_weight_min": -14.61203670501709,
      "activations/layer20_attention_weight_max": 23.07892417907715,
      "activations/layer20_attention_weight_min": -25.191234588623047,
      "activations/layer21_attention_weight_max": 44.442481994628906,
      "activations/layer21_attention_weight_min": -26.866634368896484,
      "activations/layer22_attention_weight_max": 36.41144561767578,
      "activations/layer22_attention_weight_min": -25.888641357421875,
      "activations/layer23_attention_weight_max": 37.735084533691406,
      "activations/layer23_attention_weight_min": -20.227447509765625,
      "activations/layer2_attention_weight_max": 29.665794372558594,
      "activations/layer2_attention_weight_min": -29.891094207763672,
      "activations/layer3_attention_weight_max": 82.91082000732422,
      "activations/layer3_attention_weight_min": -81.25447845458984,
      "activations/layer4_attention_weight_max": 99.22383880615234,
      "activations/layer4_attention_weight_min": -92.4600830078125,
      "activations/layer5_attention_weight_max": 92.48052978515625,
      "activations/layer5_attention_weight_min": -84.87112426757812,
      "activations/layer6_attention_weight_max": 57.93060302734375,
      "activations/layer6_attention_weight_min": -51.400638580322266,
      "activations/layer7_attention_weight_max": 82.2022476196289,
      "activations/layer7_attention_weight_min": -69.54497528076172,
      "activations/layer8_attention_weight_max": 44.3029670715332,
      "activations/layer8_attention_weight_min": -42.1221809387207,
      "activations/layer9_attention_weight_max": 53.100128173828125,
      "activations/layer9_attention_weight_min": -47.01369857788086,
      "epoch": 6.14,
      "learning_rate": 0.00011151590909090908,
      "loss": 2.8304,
      "step": 105700
    },
    {
      "activations/layer0_attention_weight_max": 14.318323135375977,
      "activations/layer0_attention_weight_min": -15.024558067321777,
      "activations/layer10_attention_weight_max": 42.061920166015625,
      "activations/layer10_attention_weight_min": -36.00979995727539,
      "activations/layer11_attention_weight_max": 31.367237091064453,
      "activations/layer11_attention_weight_min": -28.651386260986328,
      "activations/layer12_attention_weight_max": 21.139535903930664,
      "activations/layer12_attention_weight_min": -25.242408752441406,
      "activations/layer13_attention_weight_max": 44.199771881103516,
      "activations/layer13_attention_weight_min": -32.02369689941406,
      "activations/layer14_attention_weight_max": 36.54228973388672,
      "activations/layer14_attention_weight_min": -28.614940643310547,
      "activations/layer15_attention_weight_max": 34.18513107299805,
      "activations/layer15_attention_weight_min": -30.793663024902344,
      "activations/layer16_attention_weight_max": 33.823238372802734,
      "activations/layer16_attention_weight_min": -26.791343688964844,
      "activations/layer17_attention_weight_max": 59.11197280883789,
      "activations/layer17_attention_weight_min": -40.55183410644531,
      "activations/layer18_attention_weight_max": 50.23772430419922,
      "activations/layer18_attention_weight_min": -35.160423278808594,
      "activations/layer19_attention_weight_max": 21.367679595947266,
      "activations/layer19_attention_weight_min": -22.68632698059082,
      "activations/layer1_attention_weight_max": 17.829164505004883,
      "activations/layer1_attention_weight_min": -14.448906898498535,
      "activations/layer20_attention_weight_max": 22.168333053588867,
      "activations/layer20_attention_weight_min": -25.272592544555664,
      "activations/layer21_attention_weight_max": 41.15937423706055,
      "activations/layer21_attention_weight_min": -26.115209579467773,
      "activations/layer22_attention_weight_max": 35.0052375793457,
      "activations/layer22_attention_weight_min": -25.07290267944336,
      "activations/layer23_attention_weight_max": 31.564682006835938,
      "activations/layer23_attention_weight_min": -20.519733428955078,
      "activations/layer2_attention_weight_max": 32.81349182128906,
      "activations/layer2_attention_weight_min": -30.50354766845703,
      "activations/layer3_attention_weight_max": 89.27607727050781,
      "activations/layer3_attention_weight_min": -86.09649658203125,
      "activations/layer4_attention_weight_max": 102.00505828857422,
      "activations/layer4_attention_weight_min": -89.19503021240234,
      "activations/layer5_attention_weight_max": 96.71905517578125,
      "activations/layer5_attention_weight_min": -86.2098388671875,
      "activations/layer6_attention_weight_max": 57.147911071777344,
      "activations/layer6_attention_weight_min": -51.441471099853516,
      "activations/layer7_attention_weight_max": 73.86392211914062,
      "activations/layer7_attention_weight_min": -71.97590637207031,
      "activations/layer8_attention_weight_max": 45.77803039550781,
      "activations/layer8_attention_weight_min": -44.490447998046875,
      "activations/layer9_attention_weight_max": 54.78266143798828,
      "activations/layer9_attention_weight_min": -50.05873489379883,
      "epoch": 6.14,
      "learning_rate": 0.00011149696969696969,
      "loss": 2.8435,
      "step": 105750
    },
    {
      "activations/layer0_attention_weight_max": 14.709400177001953,
      "activations/layer0_attention_weight_min": -15.576077461242676,
      "activations/layer10_attention_weight_max": 42.11826705932617,
      "activations/layer10_attention_weight_min": -36.311466217041016,
      "activations/layer11_attention_weight_max": 35.568756103515625,
      "activations/layer11_attention_weight_min": -31.423337936401367,
      "activations/layer12_attention_weight_max": 19.114805221557617,
      "activations/layer12_attention_weight_min": -38.04822540283203,
      "activations/layer13_attention_weight_max": 51.45878601074219,
      "activations/layer13_attention_weight_min": -32.38629150390625,
      "activations/layer14_attention_weight_max": 53.75508499145508,
      "activations/layer14_attention_weight_min": -31.63685417175293,
      "activations/layer15_attention_weight_max": 49.544593811035156,
      "activations/layer15_attention_weight_min": -28.46965789794922,
      "activations/layer16_attention_weight_max": 34.31645965576172,
      "activations/layer16_attention_weight_min": -25.743152618408203,
      "activations/layer17_attention_weight_max": 55.83177185058594,
      "activations/layer17_attention_weight_min": -39.45709991455078,
      "activations/layer18_attention_weight_max": 43.97331237792969,
      "activations/layer18_attention_weight_min": -33.54805374145508,
      "activations/layer19_attention_weight_max": 21.740205764770508,
      "activations/layer19_attention_weight_min": -18.556278228759766,
      "activations/layer1_attention_weight_max": 19.259403228759766,
      "activations/layer1_attention_weight_min": -16.39681625366211,
      "activations/layer20_attention_weight_max": 19.811199188232422,
      "activations/layer20_attention_weight_min": -21.783788681030273,
      "activations/layer21_attention_weight_max": 34.83232879638672,
      "activations/layer21_attention_weight_min": -23.016576766967773,
      "activations/layer22_attention_weight_max": 32.32088851928711,
      "activations/layer22_attention_weight_min": -26.591232299804688,
      "activations/layer23_attention_weight_max": 29.68927764892578,
      "activations/layer23_attention_weight_min": -20.19106674194336,
      "activations/layer2_attention_weight_max": 35.61750793457031,
      "activations/layer2_attention_weight_min": -32.73637390136719,
      "activations/layer3_attention_weight_max": 89.37533569335938,
      "activations/layer3_attention_weight_min": -86.93743133544922,
      "activations/layer4_attention_weight_max": 108.69062805175781,
      "activations/layer4_attention_weight_min": -100.51374816894531,
      "activations/layer5_attention_weight_max": 104.23817443847656,
      "activations/layer5_attention_weight_min": -91.88916015625,
      "activations/layer6_attention_weight_max": 59.86445999145508,
      "activations/layer6_attention_weight_min": -59.416770935058594,
      "activations/layer7_attention_weight_max": 88.81401824951172,
      "activations/layer7_attention_weight_min": -77.82603454589844,
      "activations/layer8_attention_weight_max": 51.61098098754883,
      "activations/layer8_attention_weight_min": -46.7515754699707,
      "activations/layer9_attention_weight_max": 69.6041488647461,
      "activations/layer9_attention_weight_min": -60.20754623413086,
      "epoch": 6.15,
      "learning_rate": 0.00011147803030303029,
      "loss": 2.839,
      "step": 105800
    },
    {
      "activations/layer0_attention_weight_max": 15.316192626953125,
      "activations/layer0_attention_weight_min": -14.512085914611816,
      "activations/layer10_attention_weight_max": 39.36601257324219,
      "activations/layer10_attention_weight_min": -35.49329376220703,
      "activations/layer11_attention_weight_max": 32.743141174316406,
      "activations/layer11_attention_weight_min": -29.337833404541016,
      "activations/layer12_attention_weight_max": 34.66422653198242,
      "activations/layer12_attention_weight_min": -31.206697463989258,
      "activations/layer13_attention_weight_max": 49.56794738769531,
      "activations/layer13_attention_weight_min": -33.735084533691406,
      "activations/layer14_attention_weight_max": 40.5069694519043,
      "activations/layer14_attention_weight_min": -30.965667724609375,
      "activations/layer15_attention_weight_max": 46.701744079589844,
      "activations/layer15_attention_weight_min": -29.616107940673828,
      "activations/layer16_attention_weight_max": 36.0577278137207,
      "activations/layer16_attention_weight_min": -27.255159378051758,
      "activations/layer17_attention_weight_max": 56.9884033203125,
      "activations/layer17_attention_weight_min": -43.7471923828125,
      "activations/layer18_attention_weight_max": 47.936317443847656,
      "activations/layer18_attention_weight_min": -39.42501449584961,
      "activations/layer19_attention_weight_max": 24.19010353088379,
      "activations/layer19_attention_weight_min": -21.77864646911621,
      "activations/layer1_attention_weight_max": 17.05607032775879,
      "activations/layer1_attention_weight_min": -13.652030944824219,
      "activations/layer20_attention_weight_max": 25.09457015991211,
      "activations/layer20_attention_weight_min": -21.38873863220215,
      "activations/layer21_attention_weight_max": 41.15936279296875,
      "activations/layer21_attention_weight_min": -26.657058715820312,
      "activations/layer22_attention_weight_max": 39.20869445800781,
      "activations/layer22_attention_weight_min": -27.36592674255371,
      "activations/layer23_attention_weight_max": 33.602081298828125,
      "activations/layer23_attention_weight_min": -20.740198135375977,
      "activations/layer2_attention_weight_max": 30.372671127319336,
      "activations/layer2_attention_weight_min": -31.63189697265625,
      "activations/layer3_attention_weight_max": 80.50581359863281,
      "activations/layer3_attention_weight_min": -81.64049530029297,
      "activations/layer4_attention_weight_max": 99.42098236083984,
      "activations/layer4_attention_weight_min": -91.68999481201172,
      "activations/layer5_attention_weight_max": 94.60711669921875,
      "activations/layer5_attention_weight_min": -87.24407196044922,
      "activations/layer6_attention_weight_max": 60.01175308227539,
      "activations/layer6_attention_weight_min": -53.55393600463867,
      "activations/layer7_attention_weight_max": 74.18517303466797,
      "activations/layer7_attention_weight_min": -71.51261901855469,
      "activations/layer8_attention_weight_max": 45.46257400512695,
      "activations/layer8_attention_weight_min": -47.64323425292969,
      "activations/layer9_attention_weight_max": 55.93793869018555,
      "activations/layer9_attention_weight_min": -47.91826629638672,
      "epoch": 6.15,
      "learning_rate": 0.0001114590909090909,
      "loss": 2.8362,
      "step": 105850
    },
    {
      "activations/layer0_attention_weight_max": 14.445426940917969,
      "activations/layer0_attention_weight_min": -14.850110054016113,
      "activations/layer10_attention_weight_max": 39.48060989379883,
      "activations/layer10_attention_weight_min": -37.6047477722168,
      "activations/layer11_attention_weight_max": 31.332151412963867,
      "activations/layer11_attention_weight_min": -29.703319549560547,
      "activations/layer12_attention_weight_max": 18.466482162475586,
      "activations/layer12_attention_weight_min": -25.09415626525879,
      "activations/layer13_attention_weight_max": 42.49523162841797,
      "activations/layer13_attention_weight_min": -35.17041015625,
      "activations/layer14_attention_weight_max": 46.47450637817383,
      "activations/layer14_attention_weight_min": -30.238046646118164,
      "activations/layer15_attention_weight_max": 39.28472900390625,
      "activations/layer15_attention_weight_min": -29.211833953857422,
      "activations/layer16_attention_weight_max": 38.95470428466797,
      "activations/layer16_attention_weight_min": -26.597471237182617,
      "activations/layer17_attention_weight_max": 54.93375015258789,
      "activations/layer17_attention_weight_min": -41.69527053833008,
      "activations/layer18_attention_weight_max": 47.597389221191406,
      "activations/layer18_attention_weight_min": -36.468719482421875,
      "activations/layer19_attention_weight_max": 25.733564376831055,
      "activations/layer19_attention_weight_min": -19.9132137298584,
      "activations/layer1_attention_weight_max": 16.687992095947266,
      "activations/layer1_attention_weight_min": -17.301738739013672,
      "activations/layer20_attention_weight_max": 23.821420669555664,
      "activations/layer20_attention_weight_min": -18.082780838012695,
      "activations/layer21_attention_weight_max": 41.725833892822266,
      "activations/layer21_attention_weight_min": -23.00472640991211,
      "activations/layer22_attention_weight_max": 35.33005142211914,
      "activations/layer22_attention_weight_min": -24.142681121826172,
      "activations/layer23_attention_weight_max": 29.842031478881836,
      "activations/layer23_attention_weight_min": -19.87271499633789,
      "activations/layer2_attention_weight_max": 31.014930725097656,
      "activations/layer2_attention_weight_min": -31.122793197631836,
      "activations/layer3_attention_weight_max": 85.97632598876953,
      "activations/layer3_attention_weight_min": -87.47929382324219,
      "activations/layer4_attention_weight_max": 103.2470474243164,
      "activations/layer4_attention_weight_min": -96.50179290771484,
      "activations/layer5_attention_weight_max": 94.93679809570312,
      "activations/layer5_attention_weight_min": -87.35981750488281,
      "activations/layer6_attention_weight_max": 56.445228576660156,
      "activations/layer6_attention_weight_min": -53.64470672607422,
      "activations/layer7_attention_weight_max": 73.57252502441406,
      "activations/layer7_attention_weight_min": -71.3302230834961,
      "activations/layer8_attention_weight_max": 44.67948532104492,
      "activations/layer8_attention_weight_min": -46.5095329284668,
      "activations/layer9_attention_weight_max": 51.9620361328125,
      "activations/layer9_attention_weight_min": -49.73408508300781,
      "epoch": 6.15,
      "learning_rate": 0.0001114401515151515,
      "loss": 2.8187,
      "step": 105900
    },
    {
      "activations/layer0_attention_weight_max": 14.88630199432373,
      "activations/layer0_attention_weight_min": -15.403071403503418,
      "activations/layer10_attention_weight_max": 48.093955993652344,
      "activations/layer10_attention_weight_min": -43.37503433227539,
      "activations/layer11_attention_weight_max": 36.23045349121094,
      "activations/layer11_attention_weight_min": -33.34608459472656,
      "activations/layer12_attention_weight_max": 21.900121688842773,
      "activations/layer12_attention_weight_min": -30.08606719970703,
      "activations/layer13_attention_weight_max": 50.72771072387695,
      "activations/layer13_attention_weight_min": -33.846309661865234,
      "activations/layer14_attention_weight_max": 47.88432312011719,
      "activations/layer14_attention_weight_min": -32.328643798828125,
      "activations/layer15_attention_weight_max": 42.01588821411133,
      "activations/layer15_attention_weight_min": -29.565311431884766,
      "activations/layer16_attention_weight_max": 33.75857925415039,
      "activations/layer16_attention_weight_min": -25.80413818359375,
      "activations/layer17_attention_weight_max": 59.45414352416992,
      "activations/layer17_attention_weight_min": -42.636619567871094,
      "activations/layer18_attention_weight_max": 47.8016357421875,
      "activations/layer18_attention_weight_min": -35.093048095703125,
      "activations/layer19_attention_weight_max": 22.373991012573242,
      "activations/layer19_attention_weight_min": -21.60986328125,
      "activations/layer1_attention_weight_max": 17.459047317504883,
      "activations/layer1_attention_weight_min": -18.115848541259766,
      "activations/layer20_attention_weight_max": 22.49277114868164,
      "activations/layer20_attention_weight_min": -20.714096069335938,
      "activations/layer21_attention_weight_max": 40.41123580932617,
      "activations/layer21_attention_weight_min": -25.311784744262695,
      "activations/layer22_attention_weight_max": 35.14840316772461,
      "activations/layer22_attention_weight_min": -25.487445831298828,
      "activations/layer23_attention_weight_max": 35.463783264160156,
      "activations/layer23_attention_weight_min": -21.82550048828125,
      "activations/layer2_attention_weight_max": 34.742713928222656,
      "activations/layer2_attention_weight_min": -31.123088836669922,
      "activations/layer3_attention_weight_max": 87.858642578125,
      "activations/layer3_attention_weight_min": -92.53002166748047,
      "activations/layer4_attention_weight_max": 106.93465423583984,
      "activations/layer4_attention_weight_min": -101.92005157470703,
      "activations/layer5_attention_weight_max": 100.234619140625,
      "activations/layer5_attention_weight_min": -94.43660736083984,
      "activations/layer6_attention_weight_max": 63.361289978027344,
      "activations/layer6_attention_weight_min": -58.5667724609375,
      "activations/layer7_attention_weight_max": 83.13671112060547,
      "activations/layer7_attention_weight_min": -79.88058471679688,
      "activations/layer8_attention_weight_max": 53.52067565917969,
      "activations/layer8_attention_weight_min": -51.11223220825195,
      "activations/layer9_attention_weight_max": 71.3466567993164,
      "activations/layer9_attention_weight_min": -59.09942626953125,
      "epoch": 6.16,
      "learning_rate": 0.0001114212121212121,
      "loss": 2.8188,
      "step": 105950
    },
    {
      "activations/layer0_attention_weight_max": 14.834415435791016,
      "activations/layer0_attention_weight_min": -15.215286254882812,
      "activations/layer10_attention_weight_max": 38.537418365478516,
      "activations/layer10_attention_weight_min": -35.3719596862793,
      "activations/layer11_attention_weight_max": 28.557769775390625,
      "activations/layer11_attention_weight_min": -27.847400665283203,
      "activations/layer12_attention_weight_max": 21.66411781311035,
      "activations/layer12_attention_weight_min": -25.721118927001953,
      "activations/layer13_attention_weight_max": 55.96623992919922,
      "activations/layer13_attention_weight_min": -36.31671905517578,
      "activations/layer14_attention_weight_max": 43.35231018066406,
      "activations/layer14_attention_weight_min": -30.405902862548828,
      "activations/layer15_attention_weight_max": 38.747562408447266,
      "activations/layer15_attention_weight_min": -29.442636489868164,
      "activations/layer16_attention_weight_max": 33.96683120727539,
      "activations/layer16_attention_weight_min": -26.90116310119629,
      "activations/layer17_attention_weight_max": 59.71718215942383,
      "activations/layer17_attention_weight_min": -40.591121673583984,
      "activations/layer18_attention_weight_max": 49.440128326416016,
      "activations/layer18_attention_weight_min": -35.086029052734375,
      "activations/layer19_attention_weight_max": 21.368404388427734,
      "activations/layer19_attention_weight_min": -22.414587020874023,
      "activations/layer1_attention_weight_max": 17.52826499938965,
      "activations/layer1_attention_weight_min": -13.226119041442871,
      "activations/layer20_attention_weight_max": 22.606351852416992,
      "activations/layer20_attention_weight_min": -20.352922439575195,
      "activations/layer21_attention_weight_max": 38.32832336425781,
      "activations/layer21_attention_weight_min": -25.50522232055664,
      "activations/layer22_attention_weight_max": 36.034488677978516,
      "activations/layer22_attention_weight_min": -24.3349552154541,
      "activations/layer23_attention_weight_max": 35.88677215576172,
      "activations/layer23_attention_weight_min": -20.147228240966797,
      "activations/layer2_attention_weight_max": 28.91626739501953,
      "activations/layer2_attention_weight_min": -28.934112548828125,
      "activations/layer3_attention_weight_max": 77.5669937133789,
      "activations/layer3_attention_weight_min": -83.0494384765625,
      "activations/layer4_attention_weight_max": 95.64683532714844,
      "activations/layer4_attention_weight_min": -91.57467651367188,
      "activations/layer5_attention_weight_max": 92.70884704589844,
      "activations/layer5_attention_weight_min": -85.57726287841797,
      "activations/layer6_attention_weight_max": 55.16932678222656,
      "activations/layer6_attention_weight_min": -55.316383361816406,
      "activations/layer7_attention_weight_max": 72.59066009521484,
      "activations/layer7_attention_weight_min": -67.2705078125,
      "activations/layer8_attention_weight_max": 40.16633987426758,
      "activations/layer8_attention_weight_min": -43.9008674621582,
      "activations/layer9_attention_weight_max": 48.85427474975586,
      "activations/layer9_attention_weight_min": -46.52568435668945,
      "epoch": 6.16,
      "learning_rate": 0.00011140227272727272,
      "loss": 2.8126,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_loss": 2.7890625,
      "eval_runtime": 8.5896,
      "eval_samples_per_second": 499.906,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_openwebtext_loss": 2.7890625,
      "eval_openwebtext_ppl": 16.265763498233948,
      "eval_openwebtext_runtime": 8.5896,
      "eval_openwebtext_samples_per_second": 499.906,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_wikitext_loss": 3.02734375,
      "eval_wikitext_ppl": 20.64232851707703,
      "eval_wikitext_runtime": 2.0456,
      "eval_wikitext_samples_per_second": 222.913,
      "step": 106000
    },
    {
      "epoch": 6.16,
      "eval_lambada_loss": 2.6796875,
      "eval_lambada_ppl": 14.580536166314408,
      "eval_lambada_runtime": 9.6424,
      "eval_lambada_samples_per_second": 504.957,
      "step": 106000
    },
    {
      "activations/layer0_attention_weight_max": 14.590370178222656,
      "activations/layer0_attention_weight_min": -15.137688636779785,
      "activations/layer10_attention_weight_max": 36.96632766723633,
      "activations/layer10_attention_weight_min": -35.12549591064453,
      "activations/layer11_attention_weight_max": 31.581100463867188,
      "activations/layer11_attention_weight_min": -26.463659286499023,
      "activations/layer12_attention_weight_max": 24.754817962646484,
      "activations/layer12_attention_weight_min": -25.44445037841797,
      "activations/layer13_attention_weight_max": 50.128700256347656,
      "activations/layer13_attention_weight_min": -36.06380081176758,
      "activations/layer14_attention_weight_max": 44.56930923461914,
      "activations/layer14_attention_weight_min": -31.003095626831055,
      "activations/layer15_attention_weight_max": 38.34653854370117,
      "activations/layer15_attention_weight_min": -30.24312400817871,
      "activations/layer16_attention_weight_max": 33.58369445800781,
      "activations/layer16_attention_weight_min": -25.773042678833008,
      "activations/layer17_attention_weight_max": 57.98659133911133,
      "activations/layer17_attention_weight_min": -43.35145950317383,
      "activations/layer18_attention_weight_max": 46.719940185546875,
      "activations/layer18_attention_weight_min": -36.45043182373047,
      "activations/layer19_attention_weight_max": 27.743112564086914,
      "activations/layer19_attention_weight_min": -20.70754051208496,
      "activations/layer1_attention_weight_max": 17.58731460571289,
      "activations/layer1_attention_weight_min": -15.14981460571289,
      "activations/layer20_attention_weight_max": 22.853439331054688,
      "activations/layer20_attention_weight_min": -24.05973243713379,
      "activations/layer21_attention_weight_max": 37.446407318115234,
      "activations/layer21_attention_weight_min": -25.821170806884766,
      "activations/layer22_attention_weight_max": 34.88188552856445,
      "activations/layer22_attention_weight_min": -25.20532989501953,
      "activations/layer23_attention_weight_max": 29.44591522216797,
      "activations/layer23_attention_weight_min": -19.11871337890625,
      "activations/layer2_attention_weight_max": 28.8648681640625,
      "activations/layer2_attention_weight_min": -29.1207332611084,
      "activations/layer3_attention_weight_max": 82.47684478759766,
      "activations/layer3_attention_weight_min": -79.68466186523438,
      "activations/layer4_attention_weight_max": 94.53392791748047,
      "activations/layer4_attention_weight_min": -87.70728302001953,
      "activations/layer5_attention_weight_max": 91.15160369873047,
      "activations/layer5_attention_weight_min": -82.00590515136719,
      "activations/layer6_attention_weight_max": 55.689361572265625,
      "activations/layer6_attention_weight_min": -52.601112365722656,
      "activations/layer7_attention_weight_max": 79.38780212402344,
      "activations/layer7_attention_weight_min": -67.2564468383789,
      "activations/layer8_attention_weight_max": 41.3162727355957,
      "activations/layer8_attention_weight_min": -41.12912368774414,
      "activations/layer9_attention_weight_max": 62.3488655090332,
      "activations/layer9_attention_weight_min": -48.67666244506836,
      "epoch": 6.16,
      "learning_rate": 0.00011138333333333332,
      "loss": 2.8469,
      "step": 106050
    },
    {
      "activations/layer0_attention_weight_max": 15.117284774780273,
      "activations/layer0_attention_weight_min": -14.87197494506836,
      "activations/layer10_attention_weight_max": 37.46033477783203,
      "activations/layer10_attention_weight_min": -35.03581237792969,
      "activations/layer11_attention_weight_max": 28.013538360595703,
      "activations/layer11_attention_weight_min": -26.924697875976562,
      "activations/layer12_attention_weight_max": 19.995426177978516,
      "activations/layer12_attention_weight_min": -23.1192684173584,
      "activations/layer13_attention_weight_max": 53.125728607177734,
      "activations/layer13_attention_weight_min": -36.117523193359375,
      "activations/layer14_attention_weight_max": 43.95077133178711,
      "activations/layer14_attention_weight_min": -31.350833892822266,
      "activations/layer15_attention_weight_max": 38.90705108642578,
      "activations/layer15_attention_weight_min": -29.9744930267334,
      "activations/layer16_attention_weight_max": 30.915857315063477,
      "activations/layer16_attention_weight_min": -28.21857452392578,
      "activations/layer17_attention_weight_max": 57.25247573852539,
      "activations/layer17_attention_weight_min": -39.09253692626953,
      "activations/layer18_attention_weight_max": 48.82966995239258,
      "activations/layer18_attention_weight_min": -34.767181396484375,
      "activations/layer19_attention_weight_max": 21.663204193115234,
      "activations/layer19_attention_weight_min": -20.78297996520996,
      "activations/layer1_attention_weight_max": 17.214506149291992,
      "activations/layer1_attention_weight_min": -16.584856033325195,
      "activations/layer20_attention_weight_max": 21.2799129486084,
      "activations/layer20_attention_weight_min": -20.457014083862305,
      "activations/layer21_attention_weight_max": 47.283687591552734,
      "activations/layer21_attention_weight_min": -26.716079711914062,
      "activations/layer22_attention_weight_max": 34.384151458740234,
      "activations/layer22_attention_weight_min": -24.70003890991211,
      "activations/layer23_attention_weight_max": 29.975460052490234,
      "activations/layer23_attention_weight_min": -20.31260108947754,
      "activations/layer2_attention_weight_max": 30.49542236328125,
      "activations/layer2_attention_weight_min": -29.463897705078125,
      "activations/layer3_attention_weight_max": 86.00418090820312,
      "activations/layer3_attention_weight_min": -90.1324691772461,
      "activations/layer4_attention_weight_max": 100.05399322509766,
      "activations/layer4_attention_weight_min": -91.87044525146484,
      "activations/layer5_attention_weight_max": 93.40428924560547,
      "activations/layer5_attention_weight_min": -81.43130493164062,
      "activations/layer6_attention_weight_max": 55.34410095214844,
      "activations/layer6_attention_weight_min": -52.59225082397461,
      "activations/layer7_attention_weight_max": 70.90245056152344,
      "activations/layer7_attention_weight_min": -72.17359161376953,
      "activations/layer8_attention_weight_max": 43.055641174316406,
      "activations/layer8_attention_weight_min": -41.7197151184082,
      "activations/layer9_attention_weight_max": 48.34640121459961,
      "activations/layer9_attention_weight_min": -48.92363739013672,
      "epoch": 6.17,
      "learning_rate": 0.00011136439393939394,
      "loss": 2.8311,
      "step": 106100
    },
    {
      "activations/layer0_attention_weight_max": 15.026405334472656,
      "activations/layer0_attention_weight_min": -15.138049125671387,
      "activations/layer10_attention_weight_max": 40.174537658691406,
      "activations/layer10_attention_weight_min": -35.600807189941406,
      "activations/layer11_attention_weight_max": 30.796035766601562,
      "activations/layer11_attention_weight_min": -29.84029197692871,
      "activations/layer12_attention_weight_max": 20.618148803710938,
      "activations/layer12_attention_weight_min": -23.18646240234375,
      "activations/layer13_attention_weight_max": 42.04372787475586,
      "activations/layer13_attention_weight_min": -33.469261169433594,
      "activations/layer14_attention_weight_max": 41.650047302246094,
      "activations/layer14_attention_weight_min": -30.383968353271484,
      "activations/layer15_attention_weight_max": 39.70667266845703,
      "activations/layer15_attention_weight_min": -29.79604148864746,
      "activations/layer16_attention_weight_max": 34.05683898925781,
      "activations/layer16_attention_weight_min": -27.887447357177734,
      "activations/layer17_attention_weight_max": 53.28423309326172,
      "activations/layer17_attention_weight_min": -40.78080368041992,
      "activations/layer18_attention_weight_max": 47.75017166137695,
      "activations/layer18_attention_weight_min": -36.064422607421875,
      "activations/layer19_attention_weight_max": 22.087366104125977,
      "activations/layer19_attention_weight_min": -20.76881217956543,
      "activations/layer1_attention_weight_max": 16.433979034423828,
      "activations/layer1_attention_weight_min": -15.622292518615723,
      "activations/layer20_attention_weight_max": 23.677371978759766,
      "activations/layer20_attention_weight_min": -20.932249069213867,
      "activations/layer21_attention_weight_max": 39.342010498046875,
      "activations/layer21_attention_weight_min": -28.61208724975586,
      "activations/layer22_attention_weight_max": 34.80788040161133,
      "activations/layer22_attention_weight_min": -24.235599517822266,
      "activations/layer23_attention_weight_max": 29.419025421142578,
      "activations/layer23_attention_weight_min": -20.707101821899414,
      "activations/layer2_attention_weight_max": 30.225004196166992,
      "activations/layer2_attention_weight_min": -29.721710205078125,
      "activations/layer3_attention_weight_max": 81.50126647949219,
      "activations/layer3_attention_weight_min": -83.96251678466797,
      "activations/layer4_attention_weight_max": 96.9830093383789,
      "activations/layer4_attention_weight_min": -94.07637786865234,
      "activations/layer5_attention_weight_max": 91.75004577636719,
      "activations/layer5_attention_weight_min": -83.43318176269531,
      "activations/layer6_attention_weight_max": 56.6341552734375,
      "activations/layer6_attention_weight_min": -51.14626693725586,
      "activations/layer7_attention_weight_max": 71.79661560058594,
      "activations/layer7_attention_weight_min": -66.83139038085938,
      "activations/layer8_attention_weight_max": 42.94514083862305,
      "activations/layer8_attention_weight_min": -43.33778381347656,
      "activations/layer9_attention_weight_max": 48.84772872924805,
      "activations/layer9_attention_weight_min": -48.197750091552734,
      "epoch": 6.17,
      "learning_rate": 0.00011134545454545454,
      "loss": 2.843,
      "step": 106150
    },
    {
      "activations/layer0_attention_weight_max": 14.675924301147461,
      "activations/layer0_attention_weight_min": -15.476527214050293,
      "activations/layer10_attention_weight_max": 37.990386962890625,
      "activations/layer10_attention_weight_min": -37.38641357421875,
      "activations/layer11_attention_weight_max": 30.836210250854492,
      "activations/layer11_attention_weight_min": -28.679052352905273,
      "activations/layer12_attention_weight_max": 18.254602432250977,
      "activations/layer12_attention_weight_min": -29.829465866088867,
      "activations/layer13_attention_weight_max": 55.09680938720703,
      "activations/layer13_attention_weight_min": -42.9437370300293,
      "activations/layer14_attention_weight_max": 52.72447204589844,
      "activations/layer14_attention_weight_min": -39.41233444213867,
      "activations/layer15_attention_weight_max": 38.87083053588867,
      "activations/layer15_attention_weight_min": -29.95722770690918,
      "activations/layer16_attention_weight_max": 29.484317779541016,
      "activations/layer16_attention_weight_min": -24.399168014526367,
      "activations/layer17_attention_weight_max": 53.12897491455078,
      "activations/layer17_attention_weight_min": -39.79292678833008,
      "activations/layer18_attention_weight_max": 46.462825775146484,
      "activations/layer18_attention_weight_min": -34.630924224853516,
      "activations/layer19_attention_weight_max": 19.185131072998047,
      "activations/layer19_attention_weight_min": -20.61097526550293,
      "activations/layer1_attention_weight_max": 18.5043888092041,
      "activations/layer1_attention_weight_min": -14.979194641113281,
      "activations/layer20_attention_weight_max": 21.75704002380371,
      "activations/layer20_attention_weight_min": -19.379058837890625,
      "activations/layer21_attention_weight_max": 38.94395065307617,
      "activations/layer21_attention_weight_min": -29.279144287109375,
      "activations/layer22_attention_weight_max": 34.07029342651367,
      "activations/layer22_attention_weight_min": -24.12133026123047,
      "activations/layer23_attention_weight_max": 29.084943771362305,
      "activations/layer23_attention_weight_min": -18.654457092285156,
      "activations/layer2_attention_weight_max": 30.091392517089844,
      "activations/layer2_attention_weight_min": -28.643354415893555,
      "activations/layer3_attention_weight_max": 80.89583587646484,
      "activations/layer3_attention_weight_min": -83.27749633789062,
      "activations/layer4_attention_weight_max": 97.5477523803711,
      "activations/layer4_attention_weight_min": -92.97762298583984,
      "activations/layer5_attention_weight_max": 91.69895935058594,
      "activations/layer5_attention_weight_min": -84.44342041015625,
      "activations/layer6_attention_weight_max": 57.05720520019531,
      "activations/layer6_attention_weight_min": -51.667213439941406,
      "activations/layer7_attention_weight_max": 76.84269714355469,
      "activations/layer7_attention_weight_min": -70.77517700195312,
      "activations/layer8_attention_weight_max": 45.560359954833984,
      "activations/layer8_attention_weight_min": -44.926239013671875,
      "activations/layer9_attention_weight_max": 53.48026657104492,
      "activations/layer9_attention_weight_min": -51.168609619140625,
      "epoch": 6.17,
      "learning_rate": 0.00011132651515151514,
      "loss": 2.8302,
      "step": 106200
    },
    {
      "activations/layer0_attention_weight_max": 15.010947227478027,
      "activations/layer0_attention_weight_min": -15.232260704040527,
      "activations/layer10_attention_weight_max": 42.35194778442383,
      "activations/layer10_attention_weight_min": -36.742408752441406,
      "activations/layer11_attention_weight_max": 30.618270874023438,
      "activations/layer11_attention_weight_min": -27.602317810058594,
      "activations/layer12_attention_weight_max": 16.790929794311523,
      "activations/layer12_attention_weight_min": -23.63117790222168,
      "activations/layer13_attention_weight_max": 37.90767288208008,
      "activations/layer13_attention_weight_min": -31.992799758911133,
      "activations/layer14_attention_weight_max": 37.699989318847656,
      "activations/layer14_attention_weight_min": -30.820465087890625,
      "activations/layer15_attention_weight_max": 32.81733703613281,
      "activations/layer15_attention_weight_min": -29.384977340698242,
      "activations/layer16_attention_weight_max": 32.793399810791016,
      "activations/layer16_attention_weight_min": -26.309946060180664,
      "activations/layer17_attention_weight_max": 47.987918853759766,
      "activations/layer17_attention_weight_min": -40.371028900146484,
      "activations/layer18_attention_weight_max": 41.85179901123047,
      "activations/layer18_attention_weight_min": -33.942100524902344,
      "activations/layer19_attention_weight_max": 21.312986373901367,
      "activations/layer19_attention_weight_min": -21.35089683532715,
      "activations/layer1_attention_weight_max": 16.95282554626465,
      "activations/layer1_attention_weight_min": -15.662240028381348,
      "activations/layer20_attention_weight_max": 20.74028968811035,
      "activations/layer20_attention_weight_min": -24.230405807495117,
      "activations/layer21_attention_weight_max": 32.81775665283203,
      "activations/layer21_attention_weight_min": -24.309404373168945,
      "activations/layer22_attention_weight_max": 31.288835525512695,
      "activations/layer22_attention_weight_min": -24.72250747680664,
      "activations/layer23_attention_weight_max": 30.727699279785156,
      "activations/layer23_attention_weight_min": -22.40364646911621,
      "activations/layer2_attention_weight_max": 32.478511810302734,
      "activations/layer2_attention_weight_min": -30.235952377319336,
      "activations/layer3_attention_weight_max": 83.7161636352539,
      "activations/layer3_attention_weight_min": -87.4604263305664,
      "activations/layer4_attention_weight_max": 101.8182373046875,
      "activations/layer4_attention_weight_min": -96.63285827636719,
      "activations/layer5_attention_weight_max": 94.95794677734375,
      "activations/layer5_attention_weight_min": -87.01610565185547,
      "activations/layer6_attention_weight_max": 55.76262664794922,
      "activations/layer6_attention_weight_min": -52.26647186279297,
      "activations/layer7_attention_weight_max": 72.40029907226562,
      "activations/layer7_attention_weight_min": -69.2006607055664,
      "activations/layer8_attention_weight_max": 44.609588623046875,
      "activations/layer8_attention_weight_min": -45.0186767578125,
      "activations/layer9_attention_weight_max": 52.867671966552734,
      "activations/layer9_attention_weight_min": -51.441253662109375,
      "epoch": 6.17,
      "learning_rate": 0.00011130757575757575,
      "loss": 2.8406,
      "step": 106250
    },
    {
      "activations/layer0_attention_weight_max": 14.920823097229004,
      "activations/layer0_attention_weight_min": -15.346323013305664,
      "activations/layer10_attention_weight_max": 37.83878707885742,
      "activations/layer10_attention_weight_min": -36.2480354309082,
      "activations/layer11_attention_weight_max": 29.87163543701172,
      "activations/layer11_attention_weight_min": -29.179412841796875,
      "activations/layer12_attention_weight_max": 27.887771606445312,
      "activations/layer12_attention_weight_min": -29.197792053222656,
      "activations/layer13_attention_weight_max": 47.573570251464844,
      "activations/layer13_attention_weight_min": -32.17857360839844,
      "activations/layer14_attention_weight_max": 43.351932525634766,
      "activations/layer14_attention_weight_min": -28.743566513061523,
      "activations/layer15_attention_weight_max": 36.289127349853516,
      "activations/layer15_attention_weight_min": -28.415340423583984,
      "activations/layer16_attention_weight_max": 35.20738220214844,
      "activations/layer16_attention_weight_min": -25.761247634887695,
      "activations/layer17_attention_weight_max": 55.64185333251953,
      "activations/layer17_attention_weight_min": -43.09668731689453,
      "activations/layer18_attention_weight_max": 51.54963302612305,
      "activations/layer18_attention_weight_min": -35.76996994018555,
      "activations/layer19_attention_weight_max": 22.72357940673828,
      "activations/layer19_attention_weight_min": -20.974470138549805,
      "activations/layer1_attention_weight_max": 16.88630485534668,
      "activations/layer1_attention_weight_min": -15.210028648376465,
      "activations/layer20_attention_weight_max": 22.20024299621582,
      "activations/layer20_attention_weight_min": -25.08734130859375,
      "activations/layer21_attention_weight_max": 37.314552307128906,
      "activations/layer21_attention_weight_min": -27.012365341186523,
      "activations/layer22_attention_weight_max": 38.47412872314453,
      "activations/layer22_attention_weight_min": -26.15594482421875,
      "activations/layer23_attention_weight_max": 29.697416305541992,
      "activations/layer23_attention_weight_min": -20.880252838134766,
      "activations/layer2_attention_weight_max": 32.825653076171875,
      "activations/layer2_attention_weight_min": -30.873607635498047,
      "activations/layer3_attention_weight_max": 83.42255401611328,
      "activations/layer3_attention_weight_min": -87.95592498779297,
      "activations/layer4_attention_weight_max": 96.71736907958984,
      "activations/layer4_attention_weight_min": -93.99900817871094,
      "activations/layer5_attention_weight_max": 94.08894348144531,
      "activations/layer5_attention_weight_min": -88.1607666015625,
      "activations/layer6_attention_weight_max": 55.483909606933594,
      "activations/layer6_attention_weight_min": -54.7147331237793,
      "activations/layer7_attention_weight_max": 77.46885681152344,
      "activations/layer7_attention_weight_min": -71.34716033935547,
      "activations/layer8_attention_weight_max": 44.42676544189453,
      "activations/layer8_attention_weight_min": -45.14982223510742,
      "activations/layer9_attention_weight_max": 56.2177619934082,
      "activations/layer9_attention_weight_min": -49.00892639160156,
      "epoch": 6.18,
      "learning_rate": 0.00011128863636363634,
      "loss": 2.8167,
      "step": 106300
    },
    {
      "activations/layer0_attention_weight_max": 16.73656463623047,
      "activations/layer0_attention_weight_min": -14.857662200927734,
      "activations/layer10_attention_weight_max": 41.5604248046875,
      "activations/layer10_attention_weight_min": -33.99674987792969,
      "activations/layer11_attention_weight_max": 28.979278564453125,
      "activations/layer11_attention_weight_min": -28.301105499267578,
      "activations/layer12_attention_weight_max": 18.60065460205078,
      "activations/layer12_attention_weight_min": -27.195772171020508,
      "activations/layer13_attention_weight_max": 44.98796081542969,
      "activations/layer13_attention_weight_min": -32.416893005371094,
      "activations/layer14_attention_weight_max": 37.7773551940918,
      "activations/layer14_attention_weight_min": -30.76946258544922,
      "activations/layer15_attention_weight_max": 40.6887092590332,
      "activations/layer15_attention_weight_min": -30.815982818603516,
      "activations/layer16_attention_weight_max": 31.323074340820312,
      "activations/layer16_attention_weight_min": -26.94416046142578,
      "activations/layer17_attention_weight_max": 51.386260986328125,
      "activations/layer17_attention_weight_min": -43.933502197265625,
      "activations/layer18_attention_weight_max": 45.600791931152344,
      "activations/layer18_attention_weight_min": -36.5072135925293,
      "activations/layer19_attention_weight_max": 22.07344627380371,
      "activations/layer19_attention_weight_min": -21.183324813842773,
      "activations/layer1_attention_weight_max": 17.561254501342773,
      "activations/layer1_attention_weight_min": -15.970447540283203,
      "activations/layer20_attention_weight_max": 21.207660675048828,
      "activations/layer20_attention_weight_min": -23.883670806884766,
      "activations/layer21_attention_weight_max": 37.826576232910156,
      "activations/layer21_attention_weight_min": -23.194766998291016,
      "activations/layer22_attention_weight_max": 32.884063720703125,
      "activations/layer22_attention_weight_min": -25.62270164489746,
      "activations/layer23_attention_weight_max": 32.083740234375,
      "activations/layer23_attention_weight_min": -21.64717674255371,
      "activations/layer2_attention_weight_max": 30.588924407958984,
      "activations/layer2_attention_weight_min": -29.046905517578125,
      "activations/layer3_attention_weight_max": 90.11243438720703,
      "activations/layer3_attention_weight_min": -86.49535369873047,
      "activations/layer4_attention_weight_max": 107.02007293701172,
      "activations/layer4_attention_weight_min": -94.85822296142578,
      "activations/layer5_attention_weight_max": 98.82051849365234,
      "activations/layer5_attention_weight_min": -86.94528198242188,
      "activations/layer6_attention_weight_max": 59.97312545776367,
      "activations/layer6_attention_weight_min": -54.5327262878418,
      "activations/layer7_attention_weight_max": 74.05237579345703,
      "activations/layer7_attention_weight_min": -68.89120483398438,
      "activations/layer8_attention_weight_max": 48.50639724731445,
      "activations/layer8_attention_weight_min": -42.69032669067383,
      "activations/layer9_attention_weight_max": 50.45101547241211,
      "activations/layer9_attention_weight_min": -47.3864631652832,
      "epoch": 6.18,
      "learning_rate": 0.00011126969696969696,
      "loss": 2.8524,
      "step": 106350
    },
    {
      "activations/layer0_attention_weight_max": 14.449228286743164,
      "activations/layer0_attention_weight_min": -15.243990898132324,
      "activations/layer10_attention_weight_max": 42.17299270629883,
      "activations/layer10_attention_weight_min": -37.1578254699707,
      "activations/layer11_attention_weight_max": 30.89632225036621,
      "activations/layer11_attention_weight_min": -29.14593505859375,
      "activations/layer12_attention_weight_max": 20.08363914489746,
      "activations/layer12_attention_weight_min": -25.723655700683594,
      "activations/layer13_attention_weight_max": 41.99736022949219,
      "activations/layer13_attention_weight_min": -28.839479446411133,
      "activations/layer14_attention_weight_max": 37.81608200073242,
      "activations/layer14_attention_weight_min": -28.80099868774414,
      "activations/layer15_attention_weight_max": 38.236087799072266,
      "activations/layer15_attention_weight_min": -30.939620971679688,
      "activations/layer16_attention_weight_max": 33.498558044433594,
      "activations/layer16_attention_weight_min": -26.80881118774414,
      "activations/layer17_attention_weight_max": 55.176361083984375,
      "activations/layer17_attention_weight_min": -40.674713134765625,
      "activations/layer18_attention_weight_max": 48.208656311035156,
      "activations/layer18_attention_weight_min": -33.81535339355469,
      "activations/layer19_attention_weight_max": 24.118040084838867,
      "activations/layer19_attention_weight_min": -19.798316955566406,
      "activations/layer1_attention_weight_max": 18.71681785583496,
      "activations/layer1_attention_weight_min": -16.402406692504883,
      "activations/layer20_attention_weight_max": 21.11459732055664,
      "activations/layer20_attention_weight_min": -21.72993278503418,
      "activations/layer21_attention_weight_max": 39.23506546020508,
      "activations/layer21_attention_weight_min": -24.983932495117188,
      "activations/layer22_attention_weight_max": 31.566926956176758,
      "activations/layer22_attention_weight_min": -24.8868465423584,
      "activations/layer23_attention_weight_max": 32.243534088134766,
      "activations/layer23_attention_weight_min": -19.073558807373047,
      "activations/layer2_attention_weight_max": 29.386409759521484,
      "activations/layer2_attention_weight_min": -28.304718017578125,
      "activations/layer3_attention_weight_max": 82.66846466064453,
      "activations/layer3_attention_weight_min": -82.86767578125,
      "activations/layer4_attention_weight_max": 102.30040740966797,
      "activations/layer4_attention_weight_min": -92.48313903808594,
      "activations/layer5_attention_weight_max": 94.87120819091797,
      "activations/layer5_attention_weight_min": -84.2472915649414,
      "activations/layer6_attention_weight_max": 55.00087356567383,
      "activations/layer6_attention_weight_min": -51.55050277709961,
      "activations/layer7_attention_weight_max": 72.75772094726562,
      "activations/layer7_attention_weight_min": -68.74014282226562,
      "activations/layer8_attention_weight_max": 44.85227966308594,
      "activations/layer8_attention_weight_min": -44.05940246582031,
      "activations/layer9_attention_weight_max": 60.4312858581543,
      "activations/layer9_attention_weight_min": -47.49303436279297,
      "epoch": 6.18,
      "learning_rate": 0.00011125075757575757,
      "loss": 2.8267,
      "step": 106400
    },
    {
      "activations/layer0_attention_weight_max": 16.024763107299805,
      "activations/layer0_attention_weight_min": -15.249210357666016,
      "activations/layer10_attention_weight_max": 37.25382614135742,
      "activations/layer10_attention_weight_min": -33.88876724243164,
      "activations/layer11_attention_weight_max": 30.02751922607422,
      "activations/layer11_attention_weight_min": -28.45448875427246,
      "activations/layer12_attention_weight_max": 23.01384735107422,
      "activations/layer12_attention_weight_min": -23.96843719482422,
      "activations/layer13_attention_weight_max": 46.22888946533203,
      "activations/layer13_attention_weight_min": -34.87038040161133,
      "activations/layer14_attention_weight_max": 42.9141960144043,
      "activations/layer14_attention_weight_min": -30.577219009399414,
      "activations/layer15_attention_weight_max": 40.368446350097656,
      "activations/layer15_attention_weight_min": -31.172271728515625,
      "activations/layer16_attention_weight_max": 34.66155242919922,
      "activations/layer16_attention_weight_min": -26.482486724853516,
      "activations/layer17_attention_weight_max": 54.022342681884766,
      "activations/layer17_attention_weight_min": -39.90394973754883,
      "activations/layer18_attention_weight_max": 45.45217514038086,
      "activations/layer18_attention_weight_min": -33.62342834472656,
      "activations/layer19_attention_weight_max": 24.178007125854492,
      "activations/layer19_attention_weight_min": -20.98230743408203,
      "activations/layer1_attention_weight_max": 18.48766326904297,
      "activations/layer1_attention_weight_min": -15.173768997192383,
      "activations/layer20_attention_weight_max": 23.228282928466797,
      "activations/layer20_attention_weight_min": -24.10356903076172,
      "activations/layer21_attention_weight_max": 37.341758728027344,
      "activations/layer21_attention_weight_min": -25.99393081665039,
      "activations/layer22_attention_weight_max": 33.761680603027344,
      "activations/layer22_attention_weight_min": -25.502277374267578,
      "activations/layer23_attention_weight_max": 31.263328552246094,
      "activations/layer23_attention_weight_min": -19.37091636657715,
      "activations/layer2_attention_weight_max": 29.656784057617188,
      "activations/layer2_attention_weight_min": -28.670692443847656,
      "activations/layer3_attention_weight_max": 81.75527954101562,
      "activations/layer3_attention_weight_min": -87.04163360595703,
      "activations/layer4_attention_weight_max": 96.44454193115234,
      "activations/layer4_attention_weight_min": -90.62635040283203,
      "activations/layer5_attention_weight_max": 93.58908081054688,
      "activations/layer5_attention_weight_min": -85.59205627441406,
      "activations/layer6_attention_weight_max": 58.07107925415039,
      "activations/layer6_attention_weight_min": -52.77511978149414,
      "activations/layer7_attention_weight_max": 78.96521759033203,
      "activations/layer7_attention_weight_min": -69.0631103515625,
      "activations/layer8_attention_weight_max": 44.27943801879883,
      "activations/layer8_attention_weight_min": -42.30751037597656,
      "activations/layer9_attention_weight_max": 52.7434196472168,
      "activations/layer9_attention_weight_min": -46.50739288330078,
      "epoch": 6.19,
      "learning_rate": 0.00011123181818181816,
      "loss": 2.8335,
      "step": 106450
    },
    {
      "activations/layer0_attention_weight_max": 14.284284591674805,
      "activations/layer0_attention_weight_min": -14.804969787597656,
      "activations/layer10_attention_weight_max": 40.16295623779297,
      "activations/layer10_attention_weight_min": -35.11304473876953,
      "activations/layer11_attention_weight_max": 30.810646057128906,
      "activations/layer11_attention_weight_min": -28.196128845214844,
      "activations/layer12_attention_weight_max": 24.389400482177734,
      "activations/layer12_attention_weight_min": -22.39055633544922,
      "activations/layer13_attention_weight_max": 47.30276870727539,
      "activations/layer13_attention_weight_min": -31.13226890563965,
      "activations/layer14_attention_weight_max": 53.70549011230469,
      "activations/layer14_attention_weight_min": -34.5845947265625,
      "activations/layer15_attention_weight_max": 36.95002365112305,
      "activations/layer15_attention_weight_min": -28.822561264038086,
      "activations/layer16_attention_weight_max": 39.031070709228516,
      "activations/layer16_attention_weight_min": -26.820690155029297,
      "activations/layer17_attention_weight_max": 54.96482467651367,
      "activations/layer17_attention_weight_min": -39.48660659790039,
      "activations/layer18_attention_weight_max": 46.503414154052734,
      "activations/layer18_attention_weight_min": -34.52742385864258,
      "activations/layer19_attention_weight_max": 20.81934928894043,
      "activations/layer19_attention_weight_min": -20.926605224609375,
      "activations/layer1_attention_weight_max": 17.555383682250977,
      "activations/layer1_attention_weight_min": -14.83814811706543,
      "activations/layer20_attention_weight_max": 21.575864791870117,
      "activations/layer20_attention_weight_min": -20.389822006225586,
      "activations/layer21_attention_weight_max": 37.08502960205078,
      "activations/layer21_attention_weight_min": -24.903520584106445,
      "activations/layer22_attention_weight_max": 34.25070571899414,
      "activations/layer22_attention_weight_min": -24.27650260925293,
      "activations/layer23_attention_weight_max": 29.51496124267578,
      "activations/layer23_attention_weight_min": -21.001508712768555,
      "activations/layer2_attention_weight_max": 31.21027946472168,
      "activations/layer2_attention_weight_min": -28.548038482666016,
      "activations/layer3_attention_weight_max": 82.12688446044922,
      "activations/layer3_attention_weight_min": -88.56782531738281,
      "activations/layer4_attention_weight_max": 99.31371307373047,
      "activations/layer4_attention_weight_min": -90.74541473388672,
      "activations/layer5_attention_weight_max": 98.04779052734375,
      "activations/layer5_attention_weight_min": -85.19392395019531,
      "activations/layer6_attention_weight_max": 55.69150161743164,
      "activations/layer6_attention_weight_min": -50.76525115966797,
      "activations/layer7_attention_weight_max": 75.24259185791016,
      "activations/layer7_attention_weight_min": -66.72171783447266,
      "activations/layer8_attention_weight_max": 42.51881408691406,
      "activations/layer8_attention_weight_min": -43.29158401489258,
      "activations/layer9_attention_weight_max": 55.43739700317383,
      "activations/layer9_attention_weight_min": -48.55428695678711,
      "epoch": 6.19,
      "learning_rate": 0.00011121287878787878,
      "loss": 2.8438,
      "step": 106500
    },
    {
      "activations/layer0_attention_weight_max": 13.854071617126465,
      "activations/layer0_attention_weight_min": -14.96380615234375,
      "activations/layer10_attention_weight_max": 37.57733917236328,
      "activations/layer10_attention_weight_min": -38.85117721557617,
      "activations/layer11_attention_weight_max": 28.645294189453125,
      "activations/layer11_attention_weight_min": -28.850631713867188,
      "activations/layer12_attention_weight_max": 24.092985153198242,
      "activations/layer12_attention_weight_min": -27.1896915435791,
      "activations/layer13_attention_weight_max": 48.15464401245117,
      "activations/layer13_attention_weight_min": -34.90605926513672,
      "activations/layer14_attention_weight_max": 40.88047790527344,
      "activations/layer14_attention_weight_min": -30.575162887573242,
      "activations/layer15_attention_weight_max": 37.224918365478516,
      "activations/layer15_attention_weight_min": -29.414770126342773,
      "activations/layer16_attention_weight_max": 33.05009460449219,
      "activations/layer16_attention_weight_min": -28.230058670043945,
      "activations/layer17_attention_weight_max": 57.61672592163086,
      "activations/layer17_attention_weight_min": -41.53455352783203,
      "activations/layer18_attention_weight_max": 50.335758209228516,
      "activations/layer18_attention_weight_min": -34.38817596435547,
      "activations/layer19_attention_weight_max": 27.654916763305664,
      "activations/layer19_attention_weight_min": -19.87789535522461,
      "activations/layer1_attention_weight_max": 17.32747459411621,
      "activations/layer1_attention_weight_min": -16.8215389251709,
      "activations/layer20_attention_weight_max": 20.924808502197266,
      "activations/layer20_attention_weight_min": -21.74664306640625,
      "activations/layer21_attention_weight_max": 39.3550910949707,
      "activations/layer21_attention_weight_min": -27.698001861572266,
      "activations/layer22_attention_weight_max": 32.3912467956543,
      "activations/layer22_attention_weight_min": -25.60365104675293,
      "activations/layer23_attention_weight_max": 27.949832916259766,
      "activations/layer23_attention_weight_min": -19.459978103637695,
      "activations/layer2_attention_weight_max": 29.41798973083496,
      "activations/layer2_attention_weight_min": -29.52869415283203,
      "activations/layer3_attention_weight_max": 81.7076644897461,
      "activations/layer3_attention_weight_min": -90.19599151611328,
      "activations/layer4_attention_weight_max": 102.79182434082031,
      "activations/layer4_attention_weight_min": -99.52672576904297,
      "activations/layer5_attention_weight_max": 94.42588806152344,
      "activations/layer5_attention_weight_min": -90.75312805175781,
      "activations/layer6_attention_weight_max": 56.91766357421875,
      "activations/layer6_attention_weight_min": -56.08411407470703,
      "activations/layer7_attention_weight_max": 75.56893157958984,
      "activations/layer7_attention_weight_min": -73.06413269042969,
      "activations/layer8_attention_weight_max": 45.701416015625,
      "activations/layer8_attention_weight_min": -49.38315200805664,
      "activations/layer9_attention_weight_max": 53.636348724365234,
      "activations/layer9_attention_weight_min": -51.657649993896484,
      "epoch": 6.19,
      "learning_rate": 0.00011119393939393939,
      "loss": 2.825,
      "step": 106550
    },
    {
      "activations/layer0_attention_weight_max": 16.144298553466797,
      "activations/layer0_attention_weight_min": -14.556253433227539,
      "activations/layer10_attention_weight_max": 37.03312683105469,
      "activations/layer10_attention_weight_min": -35.1041259765625,
      "activations/layer11_attention_weight_max": 30.027435302734375,
      "activations/layer11_attention_weight_min": -28.846426010131836,
      "activations/layer12_attention_weight_max": 23.78460121154785,
      "activations/layer12_attention_weight_min": -26.902542114257812,
      "activations/layer13_attention_weight_max": 41.7461051940918,
      "activations/layer13_attention_weight_min": -29.988534927368164,
      "activations/layer14_attention_weight_max": 38.07624816894531,
      "activations/layer14_attention_weight_min": -28.98272705078125,
      "activations/layer15_attention_weight_max": 40.5796012878418,
      "activations/layer15_attention_weight_min": -28.415252685546875,
      "activations/layer16_attention_weight_max": 36.36941146850586,
      "activations/layer16_attention_weight_min": -28.598974227905273,
      "activations/layer17_attention_weight_max": 55.9720458984375,
      "activations/layer17_attention_weight_min": -38.48592758178711,
      "activations/layer18_attention_weight_max": 46.53281021118164,
      "activations/layer18_attention_weight_min": -32.74435806274414,
      "activations/layer19_attention_weight_max": 23.1706600189209,
      "activations/layer19_attention_weight_min": -19.461397171020508,
      "activations/layer1_attention_weight_max": 17.901159286499023,
      "activations/layer1_attention_weight_min": -14.756196975708008,
      "activations/layer20_attention_weight_max": 21.324405670166016,
      "activations/layer20_attention_weight_min": -22.366758346557617,
      "activations/layer21_attention_weight_max": 38.13493347167969,
      "activations/layer21_attention_weight_min": -24.693140029907227,
      "activations/layer22_attention_weight_max": 33.647151947021484,
      "activations/layer22_attention_weight_min": -23.828449249267578,
      "activations/layer23_attention_weight_max": 30.537479400634766,
      "activations/layer23_attention_weight_min": -20.564945220947266,
      "activations/layer2_attention_weight_max": 32.28706741333008,
      "activations/layer2_attention_weight_min": -29.720149993896484,
      "activations/layer3_attention_weight_max": 79.31683349609375,
      "activations/layer3_attention_weight_min": -83.31752014160156,
      "activations/layer4_attention_weight_max": 96.5692367553711,
      "activations/layer4_attention_weight_min": -90.59310150146484,
      "activations/layer5_attention_weight_max": 95.08937072753906,
      "activations/layer5_attention_weight_min": -85.25456237792969,
      "activations/layer6_attention_weight_max": 57.61933135986328,
      "activations/layer6_attention_weight_min": -53.220882415771484,
      "activations/layer7_attention_weight_max": 74.74690246582031,
      "activations/layer7_attention_weight_min": -76.15032196044922,
      "activations/layer8_attention_weight_max": 44.58204650878906,
      "activations/layer8_attention_weight_min": -45.26948928833008,
      "activations/layer9_attention_weight_max": 56.942779541015625,
      "activations/layer9_attention_weight_min": -51.468074798583984,
      "epoch": 6.19,
      "learning_rate": 0.00011117499999999999,
      "loss": 2.831,
      "step": 106600
    },
    {
      "activations/layer0_attention_weight_max": 14.784856796264648,
      "activations/layer0_attention_weight_min": -15.090414047241211,
      "activations/layer10_attention_weight_max": 34.88852310180664,
      "activations/layer10_attention_weight_min": -34.032814025878906,
      "activations/layer11_attention_weight_max": 28.64047622680664,
      "activations/layer11_attention_weight_min": -27.774524688720703,
      "activations/layer12_attention_weight_max": 24.21129035949707,
      "activations/layer12_attention_weight_min": -24.51860237121582,
      "activations/layer13_attention_weight_max": 64.46834564208984,
      "activations/layer13_attention_weight_min": -39.18953323364258,
      "activations/layer14_attention_weight_max": 47.59652328491211,
      "activations/layer14_attention_weight_min": -29.260385513305664,
      "activations/layer15_attention_weight_max": 38.609561920166016,
      "activations/layer15_attention_weight_min": -28.175554275512695,
      "activations/layer16_attention_weight_max": 35.396766662597656,
      "activations/layer16_attention_weight_min": -26.601245880126953,
      "activations/layer17_attention_weight_max": 56.10452651977539,
      "activations/layer17_attention_weight_min": -40.56007385253906,
      "activations/layer18_attention_weight_max": 50.91594696044922,
      "activations/layer18_attention_weight_min": -34.12833023071289,
      "activations/layer19_attention_weight_max": 24.256088256835938,
      "activations/layer19_attention_weight_min": -20.894086837768555,
      "activations/layer1_attention_weight_max": 16.950878143310547,
      "activations/layer1_attention_weight_min": -15.591730117797852,
      "activations/layer20_attention_weight_max": 23.09201431274414,
      "activations/layer20_attention_weight_min": -19.7617244720459,
      "activations/layer21_attention_weight_max": 41.374267578125,
      "activations/layer21_attention_weight_min": -22.986173629760742,
      "activations/layer22_attention_weight_max": 37.194061279296875,
      "activations/layer22_attention_weight_min": -23.73434829711914,
      "activations/layer23_attention_weight_max": 37.74618911743164,
      "activations/layer23_attention_weight_min": -18.061708450317383,
      "activations/layer2_attention_weight_max": 31.342056274414062,
      "activations/layer2_attention_weight_min": -30.910446166992188,
      "activations/layer3_attention_weight_max": 75.43573760986328,
      "activations/layer3_attention_weight_min": -82.63648986816406,
      "activations/layer4_attention_weight_max": 91.58538055419922,
      "activations/layer4_attention_weight_min": -86.05091857910156,
      "activations/layer5_attention_weight_max": 93.56024169921875,
      "activations/layer5_attention_weight_min": -81.98896789550781,
      "activations/layer6_attention_weight_max": 56.79221725463867,
      "activations/layer6_attention_weight_min": -49.65855407714844,
      "activations/layer7_attention_weight_max": 74.14697265625,
      "activations/layer7_attention_weight_min": -70.90879821777344,
      "activations/layer8_attention_weight_max": 41.53411865234375,
      "activations/layer8_attention_weight_min": -43.28377151489258,
      "activations/layer9_attention_weight_max": 52.88578414916992,
      "activations/layer9_attention_weight_min": -49.09370422363281,
      "epoch": 6.2,
      "learning_rate": 0.0001111560606060606,
      "loss": 2.8181,
      "step": 106650
    },
    {
      "activations/layer0_attention_weight_max": 15.161245346069336,
      "activations/layer0_attention_weight_min": -15.31232738494873,
      "activations/layer10_attention_weight_max": 35.579200744628906,
      "activations/layer10_attention_weight_min": -33.85032272338867,
      "activations/layer11_attention_weight_max": 27.971473693847656,
      "activations/layer11_attention_weight_min": -27.720897674560547,
      "activations/layer12_attention_weight_max": 23.171701431274414,
      "activations/layer12_attention_weight_min": -23.627304077148438,
      "activations/layer13_attention_weight_max": 49.610801696777344,
      "activations/layer13_attention_weight_min": -32.90076446533203,
      "activations/layer14_attention_weight_max": 39.92536544799805,
      "activations/layer14_attention_weight_min": -30.65667724609375,
      "activations/layer15_attention_weight_max": 40.44501876831055,
      "activations/layer15_attention_weight_min": -30.402490615844727,
      "activations/layer16_attention_weight_max": 38.33475112915039,
      "activations/layer16_attention_weight_min": -26.687541961669922,
      "activations/layer17_attention_weight_max": 56.06609344482422,
      "activations/layer17_attention_weight_min": -41.675235748291016,
      "activations/layer18_attention_weight_max": 53.590877532958984,
      "activations/layer18_attention_weight_min": -35.8550910949707,
      "activations/layer19_attention_weight_max": 23.442745208740234,
      "activations/layer19_attention_weight_min": -20.742664337158203,
      "activations/layer1_attention_weight_max": 17.065540313720703,
      "activations/layer1_attention_weight_min": -14.00190258026123,
      "activations/layer20_attention_weight_max": 21.31895637512207,
      "activations/layer20_attention_weight_min": -23.553499221801758,
      "activations/layer21_attention_weight_max": 37.51496505737305,
      "activations/layer21_attention_weight_min": -24.778579711914062,
      "activations/layer22_attention_weight_max": 36.174922943115234,
      "activations/layer22_attention_weight_min": -25.385028839111328,
      "activations/layer23_attention_weight_max": 31.535869598388672,
      "activations/layer23_attention_weight_min": -21.16858673095703,
      "activations/layer2_attention_weight_max": 29.789289474487305,
      "activations/layer2_attention_weight_min": -27.647846221923828,
      "activations/layer3_attention_weight_max": 78.62600708007812,
      "activations/layer3_attention_weight_min": -82.20982360839844,
      "activations/layer4_attention_weight_max": 98.06222534179688,
      "activations/layer4_attention_weight_min": -90.09843444824219,
      "activations/layer5_attention_weight_max": 93.2937240600586,
      "activations/layer5_attention_weight_min": -89.3075180053711,
      "activations/layer6_attention_weight_max": 56.898033142089844,
      "activations/layer6_attention_weight_min": -57.827110290527344,
      "activations/layer7_attention_weight_max": 72.49307250976562,
      "activations/layer7_attention_weight_min": -77.99153900146484,
      "activations/layer8_attention_weight_max": 43.30535125732422,
      "activations/layer8_attention_weight_min": -43.764549255371094,
      "activations/layer9_attention_weight_max": 52.539833068847656,
      "activations/layer9_attention_weight_min": -46.20014572143555,
      "epoch": 6.2,
      "learning_rate": 0.00011113712121212121,
      "loss": 2.8219,
      "step": 106700
    },
    {
      "activations/layer0_attention_weight_max": 15.304424285888672,
      "activations/layer0_attention_weight_min": -15.375049591064453,
      "activations/layer10_attention_weight_max": 36.976280212402344,
      "activations/layer10_attention_weight_min": -34.60592269897461,
      "activations/layer11_attention_weight_max": 30.2340087890625,
      "activations/layer11_attention_weight_min": -29.030567169189453,
      "activations/layer12_attention_weight_max": 22.269508361816406,
      "activations/layer12_attention_weight_min": -23.565895080566406,
      "activations/layer13_attention_weight_max": 45.2471809387207,
      "activations/layer13_attention_weight_min": -32.27892303466797,
      "activations/layer14_attention_weight_max": 48.70515441894531,
      "activations/layer14_attention_weight_min": -31.00072479248047,
      "activations/layer15_attention_weight_max": 40.18241882324219,
      "activations/layer15_attention_weight_min": -31.179086685180664,
      "activations/layer16_attention_weight_max": 34.06684112548828,
      "activations/layer16_attention_weight_min": -29.343379974365234,
      "activations/layer17_attention_weight_max": 60.60987854003906,
      "activations/layer17_attention_weight_min": -42.290645599365234,
      "activations/layer18_attention_weight_max": 53.04024887084961,
      "activations/layer18_attention_weight_min": -36.830989837646484,
      "activations/layer19_attention_weight_max": 25.500768661499023,
      "activations/layer19_attention_weight_min": -20.461088180541992,
      "activations/layer1_attention_weight_max": 16.818601608276367,
      "activations/layer1_attention_weight_min": -16.520658493041992,
      "activations/layer20_attention_weight_max": 26.713848114013672,
      "activations/layer20_attention_weight_min": -22.773283004760742,
      "activations/layer21_attention_weight_max": 42.611236572265625,
      "activations/layer21_attention_weight_min": -24.990318298339844,
      "activations/layer22_attention_weight_max": 36.38545227050781,
      "activations/layer22_attention_weight_min": -26.432479858398438,
      "activations/layer23_attention_weight_max": 34.29554748535156,
      "activations/layer23_attention_weight_min": -20.094905853271484,
      "activations/layer2_attention_weight_max": 30.339336395263672,
      "activations/layer2_attention_weight_min": -29.024229049682617,
      "activations/layer3_attention_weight_max": 84.46212005615234,
      "activations/layer3_attention_weight_min": -85.5040054321289,
      "activations/layer4_attention_weight_max": 95.12399291992188,
      "activations/layer4_attention_weight_min": -94.77965545654297,
      "activations/layer5_attention_weight_max": 92.8995590209961,
      "activations/layer5_attention_weight_min": -86.48107147216797,
      "activations/layer6_attention_weight_max": 53.06416320800781,
      "activations/layer6_attention_weight_min": -51.42504119873047,
      "activations/layer7_attention_weight_max": 70.42669677734375,
      "activations/layer7_attention_weight_min": -65.73051452636719,
      "activations/layer8_attention_weight_max": 41.56807327270508,
      "activations/layer8_attention_weight_min": -43.15904235839844,
      "activations/layer9_attention_weight_max": 48.05884552001953,
      "activations/layer9_attention_weight_min": -47.38930130004883,
      "epoch": 6.2,
      "learning_rate": 0.00011111818181818181,
      "loss": 2.8364,
      "step": 106750
    },
    {
      "activations/layer0_attention_weight_max": 15.374442100524902,
      "activations/layer0_attention_weight_min": -15.489131927490234,
      "activations/layer10_attention_weight_max": 40.61061477661133,
      "activations/layer10_attention_weight_min": -36.30812454223633,
      "activations/layer11_attention_weight_max": 30.79279899597168,
      "activations/layer11_attention_weight_min": -30.027639389038086,
      "activations/layer12_attention_weight_max": 18.842870712280273,
      "activations/layer12_attention_weight_min": -25.42981719970703,
      "activations/layer13_attention_weight_max": 45.418251037597656,
      "activations/layer13_attention_weight_min": -38.249977111816406,
      "activations/layer14_attention_weight_max": 44.26906204223633,
      "activations/layer14_attention_weight_min": -30.66776466369629,
      "activations/layer15_attention_weight_max": 38.9561653137207,
      "activations/layer15_attention_weight_min": -30.450210571289062,
      "activations/layer16_attention_weight_max": 35.793426513671875,
      "activations/layer16_attention_weight_min": -25.41623306274414,
      "activations/layer17_attention_weight_max": 57.87684631347656,
      "activations/layer17_attention_weight_min": -40.63896942138672,
      "activations/layer18_attention_weight_max": 49.4976921081543,
      "activations/layer18_attention_weight_min": -35.50719451904297,
      "activations/layer19_attention_weight_max": 26.0433292388916,
      "activations/layer19_attention_weight_min": -21.085384368896484,
      "activations/layer1_attention_weight_max": 17.32406234741211,
      "activations/layer1_attention_weight_min": -13.593755722045898,
      "activations/layer20_attention_weight_max": 23.848915100097656,
      "activations/layer20_attention_weight_min": -22.237762451171875,
      "activations/layer21_attention_weight_max": 45.697696685791016,
      "activations/layer21_attention_weight_min": -24.563919067382812,
      "activations/layer22_attention_weight_max": 36.667564392089844,
      "activations/layer22_attention_weight_min": -27.26185417175293,
      "activations/layer23_attention_weight_max": 31.826461791992188,
      "activations/layer23_attention_weight_min": -17.4957332611084,
      "activations/layer2_attention_weight_max": 29.95208740234375,
      "activations/layer2_attention_weight_min": -29.125389099121094,
      "activations/layer3_attention_weight_max": 79.51422882080078,
      "activations/layer3_attention_weight_min": -85.36302947998047,
      "activations/layer4_attention_weight_max": 96.93142700195312,
      "activations/layer4_attention_weight_min": -95.26351928710938,
      "activations/layer5_attention_weight_max": 92.10792541503906,
      "activations/layer5_attention_weight_min": -83.11697387695312,
      "activations/layer6_attention_weight_max": 56.525474548339844,
      "activations/layer6_attention_weight_min": -51.50598907470703,
      "activations/layer7_attention_weight_max": 74.6174087524414,
      "activations/layer7_attention_weight_min": -72.27420806884766,
      "activations/layer8_attention_weight_max": 45.37287902832031,
      "activations/layer8_attention_weight_min": -41.49607467651367,
      "activations/layer9_attention_weight_max": 62.756134033203125,
      "activations/layer9_attention_weight_min": -51.601905822753906,
      "epoch": 6.21,
      "learning_rate": 0.00011109924242424242,
      "loss": 2.8365,
      "step": 106800
    },
    {
      "activations/layer0_attention_weight_max": 14.511507987976074,
      "activations/layer0_attention_weight_min": -15.15570068359375,
      "activations/layer10_attention_weight_max": 42.08735656738281,
      "activations/layer10_attention_weight_min": -36.126895904541016,
      "activations/layer11_attention_weight_max": 33.462982177734375,
      "activations/layer11_attention_weight_min": -29.424345016479492,
      "activations/layer12_attention_weight_max": 20.11200523376465,
      "activations/layer12_attention_weight_min": -25.082345962524414,
      "activations/layer13_attention_weight_max": 70.79981994628906,
      "activations/layer13_attention_weight_min": -43.97023391723633,
      "activations/layer14_attention_weight_max": 70.4653549194336,
      "activations/layer14_attention_weight_min": -37.259647369384766,
      "activations/layer15_attention_weight_max": 49.004634857177734,
      "activations/layer15_attention_weight_min": -29.892805099487305,
      "activations/layer16_attention_weight_max": 38.54140090942383,
      "activations/layer16_attention_weight_min": -29.068866729736328,
      "activations/layer17_attention_weight_max": 62.20561981201172,
      "activations/layer17_attention_weight_min": -43.75068664550781,
      "activations/layer18_attention_weight_max": 59.005210876464844,
      "activations/layer18_attention_weight_min": -37.655906677246094,
      "activations/layer19_attention_weight_max": 23.86529541015625,
      "activations/layer19_attention_weight_min": -20.015941619873047,
      "activations/layer1_attention_weight_max": 17.192134857177734,
      "activations/layer1_attention_weight_min": -15.607929229736328,
      "activations/layer20_attention_weight_max": 26.89105796813965,
      "activations/layer20_attention_weight_min": -19.895092010498047,
      "activations/layer21_attention_weight_max": 42.30730056762695,
      "activations/layer21_attention_weight_min": -23.816030502319336,
      "activations/layer22_attention_weight_max": 39.80064010620117,
      "activations/layer22_attention_weight_min": -24.795869827270508,
      "activations/layer23_attention_weight_max": 34.59764099121094,
      "activations/layer23_attention_weight_min": -21.506284713745117,
      "activations/layer2_attention_weight_max": 29.759193420410156,
      "activations/layer2_attention_weight_min": -27.766151428222656,
      "activations/layer3_attention_weight_max": 79.95393371582031,
      "activations/layer3_attention_weight_min": -84.88101959228516,
      "activations/layer4_attention_weight_max": 99.98198699951172,
      "activations/layer4_attention_weight_min": -89.61979675292969,
      "activations/layer5_attention_weight_max": 95.56423950195312,
      "activations/layer5_attention_weight_min": -85.13818359375,
      "activations/layer6_attention_weight_max": 58.83626174926758,
      "activations/layer6_attention_weight_min": -50.7558708190918,
      "activations/layer7_attention_weight_max": 78.11698150634766,
      "activations/layer7_attention_weight_min": -71.44691467285156,
      "activations/layer8_attention_weight_max": 44.53107452392578,
      "activations/layer8_attention_weight_min": -42.69908142089844,
      "activations/layer9_attention_weight_max": 60.29163360595703,
      "activations/layer9_attention_weight_min": -50.47898864746094,
      "epoch": 6.21,
      "learning_rate": 0.00011108030303030301,
      "loss": 2.8306,
      "step": 106850
    },
    {
      "activations/layer0_attention_weight_max": 15.221548080444336,
      "activations/layer0_attention_weight_min": -15.156442642211914,
      "activations/layer10_attention_weight_max": 36.55153274536133,
      "activations/layer10_attention_weight_min": -33.10332107543945,
      "activations/layer11_attention_weight_max": 30.045818328857422,
      "activations/layer11_attention_weight_min": -29.778745651245117,
      "activations/layer12_attention_weight_max": 19.996355056762695,
      "activations/layer12_attention_weight_min": -25.650650024414062,
      "activations/layer13_attention_weight_max": 47.61764144897461,
      "activations/layer13_attention_weight_min": -32.822547912597656,
      "activations/layer14_attention_weight_max": 43.624488830566406,
      "activations/layer14_attention_weight_min": -29.52680015563965,
      "activations/layer15_attention_weight_max": 38.454490661621094,
      "activations/layer15_attention_weight_min": -28.581748962402344,
      "activations/layer16_attention_weight_max": 34.36811828613281,
      "activations/layer16_attention_weight_min": -26.845951080322266,
      "activations/layer17_attention_weight_max": 57.40383529663086,
      "activations/layer17_attention_weight_min": -42.84685134887695,
      "activations/layer18_attention_weight_max": 54.51415252685547,
      "activations/layer18_attention_weight_min": -36.224822998046875,
      "activations/layer19_attention_weight_max": 22.8613224029541,
      "activations/layer19_attention_weight_min": -20.280492782592773,
      "activations/layer1_attention_weight_max": 16.602130889892578,
      "activations/layer1_attention_weight_min": -14.761823654174805,
      "activations/layer20_attention_weight_max": 21.193334579467773,
      "activations/layer20_attention_weight_min": -22.348215103149414,
      "activations/layer21_attention_weight_max": 37.02885818481445,
      "activations/layer21_attention_weight_min": -26.58327293395996,
      "activations/layer22_attention_weight_max": 40.49666976928711,
      "activations/layer22_attention_weight_min": -24.9871883392334,
      "activations/layer23_attention_weight_max": 36.274898529052734,
      "activations/layer23_attention_weight_min": -20.18199920654297,
      "activations/layer2_attention_weight_max": 30.477628707885742,
      "activations/layer2_attention_weight_min": -30.278667449951172,
      "activations/layer3_attention_weight_max": 77.3509750366211,
      "activations/layer3_attention_weight_min": -84.18722534179688,
      "activations/layer4_attention_weight_max": 93.27730560302734,
      "activations/layer4_attention_weight_min": -90.85591888427734,
      "activations/layer5_attention_weight_max": 92.78054809570312,
      "activations/layer5_attention_weight_min": -85.80254364013672,
      "activations/layer6_attention_weight_max": 56.43222427368164,
      "activations/layer6_attention_weight_min": -52.279136657714844,
      "activations/layer7_attention_weight_max": 74.28009796142578,
      "activations/layer7_attention_weight_min": -66.12418365478516,
      "activations/layer8_attention_weight_max": 43.64014434814453,
      "activations/layer8_attention_weight_min": -40.77193069458008,
      "activations/layer9_attention_weight_max": 50.28464889526367,
      "activations/layer9_attention_weight_min": -47.7338981628418,
      "epoch": 6.21,
      "learning_rate": 0.00011106136363636363,
      "loss": 2.8374,
      "step": 106900
    },
    {
      "activations/layer0_attention_weight_max": 14.657532691955566,
      "activations/layer0_attention_weight_min": -15.205163955688477,
      "activations/layer10_attention_weight_max": 40.928436279296875,
      "activations/layer10_attention_weight_min": -35.12590789794922,
      "activations/layer11_attention_weight_max": 32.506378173828125,
      "activations/layer11_attention_weight_min": -29.662979125976562,
      "activations/layer12_attention_weight_max": 28.911861419677734,
      "activations/layer12_attention_weight_min": -31.795223236083984,
      "activations/layer13_attention_weight_max": 48.45323944091797,
      "activations/layer13_attention_weight_min": -33.71403121948242,
      "activations/layer14_attention_weight_max": 42.27641296386719,
      "activations/layer14_attention_weight_min": -29.210796356201172,
      "activations/layer15_attention_weight_max": 38.99437713623047,
      "activations/layer15_attention_weight_min": -28.942787170410156,
      "activations/layer16_attention_weight_max": 34.90987777709961,
      "activations/layer16_attention_weight_min": -25.33085823059082,
      "activations/layer17_attention_weight_max": 53.795475006103516,
      "activations/layer17_attention_weight_min": -41.77206802368164,
      "activations/layer18_attention_weight_max": 47.06505584716797,
      "activations/layer18_attention_weight_min": -35.979122161865234,
      "activations/layer19_attention_weight_max": 21.87696075439453,
      "activations/layer19_attention_weight_min": -21.63276481628418,
      "activations/layer1_attention_weight_max": 16.987224578857422,
      "activations/layer1_attention_weight_min": -17.122346878051758,
      "activations/layer20_attention_weight_max": 22.0319881439209,
      "activations/layer20_attention_weight_min": -20.182342529296875,
      "activations/layer21_attention_weight_max": 37.989749908447266,
      "activations/layer21_attention_weight_min": -25.838621139526367,
      "activations/layer22_attention_weight_max": 33.56028747558594,
      "activations/layer22_attention_weight_min": -29.247188568115234,
      "activations/layer23_attention_weight_max": 30.777379989624023,
      "activations/layer23_attention_weight_min": -20.923038482666016,
      "activations/layer2_attention_weight_max": 32.58521270751953,
      "activations/layer2_attention_weight_min": -29.58496856689453,
      "activations/layer3_attention_weight_max": 84.87596130371094,
      "activations/layer3_attention_weight_min": -84.85214233398438,
      "activations/layer4_attention_weight_max": 105.90071105957031,
      "activations/layer4_attention_weight_min": -97.91104125976562,
      "activations/layer5_attention_weight_max": 100.46028137207031,
      "activations/layer5_attention_weight_min": -87.49864196777344,
      "activations/layer6_attention_weight_max": 56.46335220336914,
      "activations/layer6_attention_weight_min": -53.09447479248047,
      "activations/layer7_attention_weight_max": 77.8038101196289,
      "activations/layer7_attention_weight_min": -71.39022827148438,
      "activations/layer8_attention_weight_max": 44.54579544067383,
      "activations/layer8_attention_weight_min": -43.2557373046875,
      "activations/layer9_attention_weight_max": 58.92964553833008,
      "activations/layer9_attention_weight_min": -54.71390914916992,
      "epoch": 6.21,
      "learning_rate": 0.00011104242424242424,
      "loss": 2.8272,
      "step": 106950
    },
    {
      "activations/layer0_attention_weight_max": 15.391545295715332,
      "activations/layer0_attention_weight_min": -15.258575439453125,
      "activations/layer10_attention_weight_max": 38.66220474243164,
      "activations/layer10_attention_weight_min": -36.3941764831543,
      "activations/layer11_attention_weight_max": 31.08363151550293,
      "activations/layer11_attention_weight_min": -29.88205337524414,
      "activations/layer12_attention_weight_max": 25.336196899414062,
      "activations/layer12_attention_weight_min": -24.02548599243164,
      "activations/layer13_attention_weight_max": 43.87727737426758,
      "activations/layer13_attention_weight_min": -32.357765197753906,
      "activations/layer14_attention_weight_max": 46.76807403564453,
      "activations/layer14_attention_weight_min": -32.34284210205078,
      "activations/layer15_attention_weight_max": 38.432411193847656,
      "activations/layer15_attention_weight_min": -30.173736572265625,
      "activations/layer16_attention_weight_max": 37.99720764160156,
      "activations/layer16_attention_weight_min": -29.807239532470703,
      "activations/layer17_attention_weight_max": 58.56380844116211,
      "activations/layer17_attention_weight_min": -44.29221725463867,
      "activations/layer18_attention_weight_max": 51.51078796386719,
      "activations/layer18_attention_weight_min": -40.73705291748047,
      "activations/layer19_attention_weight_max": 24.322341918945312,
      "activations/layer19_attention_weight_min": -21.337942123413086,
      "activations/layer1_attention_weight_max": 17.44536781311035,
      "activations/layer1_attention_weight_min": -16.76042938232422,
      "activations/layer20_attention_weight_max": 23.00406265258789,
      "activations/layer20_attention_weight_min": -21.68979263305664,
      "activations/layer21_attention_weight_max": 40.90556716918945,
      "activations/layer21_attention_weight_min": -27.793527603149414,
      "activations/layer22_attention_weight_max": 33.98704528808594,
      "activations/layer22_attention_weight_min": -26.89579200744629,
      "activations/layer23_attention_weight_max": 31.67789077758789,
      "activations/layer23_attention_weight_min": -20.91835594177246,
      "activations/layer2_attention_weight_max": 30.602956771850586,
      "activations/layer2_attention_weight_min": -31.298765182495117,
      "activations/layer3_attention_weight_max": 84.91804504394531,
      "activations/layer3_attention_weight_min": -90.51197052001953,
      "activations/layer4_attention_weight_max": 102.13347625732422,
      "activations/layer4_attention_weight_min": -97.88536834716797,
      "activations/layer5_attention_weight_max": 94.07743835449219,
      "activations/layer5_attention_weight_min": -89.35066223144531,
      "activations/layer6_attention_weight_max": 57.30866622924805,
      "activations/layer6_attention_weight_min": -54.16674041748047,
      "activations/layer7_attention_weight_max": 77.1480484008789,
      "activations/layer7_attention_weight_min": -72.55673217773438,
      "activations/layer8_attention_weight_max": 45.47054672241211,
      "activations/layer8_attention_weight_min": -43.92107009887695,
      "activations/layer9_attention_weight_max": 55.8919677734375,
      "activations/layer9_attention_weight_min": -50.904823303222656,
      "epoch": 6.22,
      "learning_rate": 0.00011102348484848483,
      "loss": 2.8125,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.5919,
      "eval_samples_per_second": 499.774,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.5919,
      "eval_openwebtext_samples_per_second": 499.774,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 2.0668,
      "eval_wikitext_samples_per_second": 220.626,
      "step": 107000
    },
    {
      "epoch": 6.22,
      "eval_lambada_loss": 2.7890625,
      "eval_lambada_ppl": 16.265763498233948,
      "eval_lambada_runtime": 9.6705,
      "eval_lambada_samples_per_second": 503.492,
      "step": 107000
    },
    {
      "activations/layer0_attention_weight_max": 15.094032287597656,
      "activations/layer0_attention_weight_min": -14.90084457397461,
      "activations/layer10_attention_weight_max": 42.903907775878906,
      "activations/layer10_attention_weight_min": -34.6566162109375,
      "activations/layer11_attention_weight_max": 31.640478134155273,
      "activations/layer11_attention_weight_min": -29.07636070251465,
      "activations/layer12_attention_weight_max": 18.144140243530273,
      "activations/layer12_attention_weight_min": -25.20798110961914,
      "activations/layer13_attention_weight_max": 46.1729850769043,
      "activations/layer13_attention_weight_min": -33.352088928222656,
      "activations/layer14_attention_weight_max": 37.597930908203125,
      "activations/layer14_attention_weight_min": -29.14089012145996,
      "activations/layer15_attention_weight_max": 34.0897331237793,
      "activations/layer15_attention_weight_min": -29.269044876098633,
      "activations/layer16_attention_weight_max": 32.21967697143555,
      "activations/layer16_attention_weight_min": -26.005577087402344,
      "activations/layer17_attention_weight_max": 54.885650634765625,
      "activations/layer17_attention_weight_min": -40.07399368286133,
      "activations/layer18_attention_weight_max": 45.16796875,
      "activations/layer18_attention_weight_min": -34.37385177612305,
      "activations/layer19_attention_weight_max": 21.044187545776367,
      "activations/layer19_attention_weight_min": -21.30348777770996,
      "activations/layer1_attention_weight_max": 17.918601989746094,
      "activations/layer1_attention_weight_min": -16.782878875732422,
      "activations/layer20_attention_weight_max": 19.707773208618164,
      "activations/layer20_attention_weight_min": -24.913217544555664,
      "activations/layer21_attention_weight_max": 39.39982986450195,
      "activations/layer21_attention_weight_min": -24.7082576751709,
      "activations/layer22_attention_weight_max": 29.68088150024414,
      "activations/layer22_attention_weight_min": -24.96834945678711,
      "activations/layer23_attention_weight_max": 29.056434631347656,
      "activations/layer23_attention_weight_min": -19.28826904296875,
      "activations/layer2_attention_weight_max": 34.049766540527344,
      "activations/layer2_attention_weight_min": -31.77562713623047,
      "activations/layer3_attention_weight_max": 86.47244262695312,
      "activations/layer3_attention_weight_min": -86.29459381103516,
      "activations/layer4_attention_weight_max": 102.58441162109375,
      "activations/layer4_attention_weight_min": -93.60572052001953,
      "activations/layer5_attention_weight_max": 100.63502502441406,
      "activations/layer5_attention_weight_min": -84.2144775390625,
      "activations/layer6_attention_weight_max": 62.47010803222656,
      "activations/layer6_attention_weight_min": -51.1823844909668,
      "activations/layer7_attention_weight_max": 74.45892333984375,
      "activations/layer7_attention_weight_min": -69.164794921875,
      "activations/layer8_attention_weight_max": 45.79793930053711,
      "activations/layer8_attention_weight_min": -41.13542175292969,
      "activations/layer9_attention_weight_max": 54.36674118041992,
      "activations/layer9_attention_weight_min": -54.2890625,
      "epoch": 6.22,
      "learning_rate": 0.00011100454545454545,
      "loss": 2.8287,
      "step": 107050
    },
    {
      "activations/layer0_attention_weight_max": 15.408621788024902,
      "activations/layer0_attention_weight_min": -15.11377239227295,
      "activations/layer10_attention_weight_max": 36.671138763427734,
      "activations/layer10_attention_weight_min": -32.25404739379883,
      "activations/layer11_attention_weight_max": 29.489179611206055,
      "activations/layer11_attention_weight_min": -27.056739807128906,
      "activations/layer12_attention_weight_max": 22.654333114624023,
      "activations/layer12_attention_weight_min": -31.6077938079834,
      "activations/layer13_attention_weight_max": 41.670570373535156,
      "activations/layer13_attention_weight_min": -27.734119415283203,
      "activations/layer14_attention_weight_max": 35.01797103881836,
      "activations/layer14_attention_weight_min": -28.20351791381836,
      "activations/layer15_attention_weight_max": 34.921653747558594,
      "activations/layer15_attention_weight_min": -27.121273040771484,
      "activations/layer16_attention_weight_max": 31.60883331298828,
      "activations/layer16_attention_weight_min": -28.882112503051758,
      "activations/layer17_attention_weight_max": 55.990604400634766,
      "activations/layer17_attention_weight_min": -38.83810043334961,
      "activations/layer18_attention_weight_max": 47.809268951416016,
      "activations/layer18_attention_weight_min": -32.77573013305664,
      "activations/layer19_attention_weight_max": 27.071735382080078,
      "activations/layer19_attention_weight_min": -18.648271560668945,
      "activations/layer1_attention_weight_max": 17.28166389465332,
      "activations/layer1_attention_weight_min": -15.987614631652832,
      "activations/layer20_attention_weight_max": 21.843618392944336,
      "activations/layer20_attention_weight_min": -19.58943748474121,
      "activations/layer21_attention_weight_max": 38.20436477661133,
      "activations/layer21_attention_weight_min": -24.2410945892334,
      "activations/layer22_attention_weight_max": 35.6854248046875,
      "activations/layer22_attention_weight_min": -23.866352081298828,
      "activations/layer23_attention_weight_max": 29.677886962890625,
      "activations/layer23_attention_weight_min": -19.954713821411133,
      "activations/layer2_attention_weight_max": 30.496213912963867,
      "activations/layer2_attention_weight_min": -28.444416046142578,
      "activations/layer3_attention_weight_max": 87.49153900146484,
      "activations/layer3_attention_weight_min": -83.67012786865234,
      "activations/layer4_attention_weight_max": 103.9118423461914,
      "activations/layer4_attention_weight_min": -96.8126449584961,
      "activations/layer5_attention_weight_max": 91.04893493652344,
      "activations/layer5_attention_weight_min": -82.48938751220703,
      "activations/layer6_attention_weight_max": 56.92475509643555,
      "activations/layer6_attention_weight_min": -51.5409049987793,
      "activations/layer7_attention_weight_max": 74.3409423828125,
      "activations/layer7_attention_weight_min": -70.31903839111328,
      "activations/layer8_attention_weight_max": 41.5184440612793,
      "activations/layer8_attention_weight_min": -40.93471145629883,
      "activations/layer9_attention_weight_max": 56.757869720458984,
      "activations/layer9_attention_weight_min": -51.70455551147461,
      "epoch": 6.22,
      "learning_rate": 0.00011098560606060606,
      "loss": 2.82,
      "step": 107100
    },
    {
      "activations/layer0_attention_weight_max": 14.451744079589844,
      "activations/layer0_attention_weight_min": -15.520030975341797,
      "activations/layer10_attention_weight_max": 41.66328430175781,
      "activations/layer10_attention_weight_min": -37.955196380615234,
      "activations/layer11_attention_weight_max": 33.18785095214844,
      "activations/layer11_attention_weight_min": -30.37409210205078,
      "activations/layer12_attention_weight_max": 24.059289932250977,
      "activations/layer12_attention_weight_min": -23.59130096435547,
      "activations/layer13_attention_weight_max": 46.706974029541016,
      "activations/layer13_attention_weight_min": -35.55107879638672,
      "activations/layer14_attention_weight_max": 40.92497253417969,
      "activations/layer14_attention_weight_min": -31.187822341918945,
      "activations/layer15_attention_weight_max": 39.20779800415039,
      "activations/layer15_attention_weight_min": -30.98478126525879,
      "activations/layer16_attention_weight_max": 35.74170684814453,
      "activations/layer16_attention_weight_min": -27.22099494934082,
      "activations/layer17_attention_weight_max": 57.01572036743164,
      "activations/layer17_attention_weight_min": -40.85310745239258,
      "activations/layer18_attention_weight_max": 50.17949295043945,
      "activations/layer18_attention_weight_min": -34.879295349121094,
      "activations/layer19_attention_weight_max": 26.43282127380371,
      "activations/layer19_attention_weight_min": -19.449989318847656,
      "activations/layer1_attention_weight_max": 18.08244514465332,
      "activations/layer1_attention_weight_min": -15.402405738830566,
      "activations/layer20_attention_weight_max": 21.745153427124023,
      "activations/layer20_attention_weight_min": -19.86821746826172,
      "activations/layer21_attention_weight_max": 39.388404846191406,
      "activations/layer21_attention_weight_min": -24.902355194091797,
      "activations/layer22_attention_weight_max": 34.33283233642578,
      "activations/layer22_attention_weight_min": -23.850679397583008,
      "activations/layer23_attention_weight_max": 31.8590087890625,
      "activations/layer23_attention_weight_min": -20.11459732055664,
      "activations/layer2_attention_weight_max": 30.378042221069336,
      "activations/layer2_attention_weight_min": -28.1345272064209,
      "activations/layer3_attention_weight_max": 79.12068176269531,
      "activations/layer3_attention_weight_min": -81.94692993164062,
      "activations/layer4_attention_weight_max": 102.79805755615234,
      "activations/layer4_attention_weight_min": -94.33360290527344,
      "activations/layer5_attention_weight_max": 102.148193359375,
      "activations/layer5_attention_weight_min": -88.96194458007812,
      "activations/layer6_attention_weight_max": 61.42279052734375,
      "activations/layer6_attention_weight_min": -53.15717697143555,
      "activations/layer7_attention_weight_max": 78.24177551269531,
      "activations/layer7_attention_weight_min": -70.72587585449219,
      "activations/layer8_attention_weight_max": 48.2552375793457,
      "activations/layer8_attention_weight_min": -49.23677444458008,
      "activations/layer9_attention_weight_max": 55.35356521606445,
      "activations/layer9_attention_weight_min": -50.3633918762207,
      "epoch": 6.23,
      "learning_rate": 0.00011096666666666665,
      "loss": 2.8355,
      "step": 107150
    },
    {
      "activations/layer0_attention_weight_max": 14.743371963500977,
      "activations/layer0_attention_weight_min": -15.047051429748535,
      "activations/layer10_attention_weight_max": 37.510009765625,
      "activations/layer10_attention_weight_min": -33.2311897277832,
      "activations/layer11_attention_weight_max": 29.685443878173828,
      "activations/layer11_attention_weight_min": -28.767925262451172,
      "activations/layer12_attention_weight_max": 24.53507423400879,
      "activations/layer12_attention_weight_min": -23.1364803314209,
      "activations/layer13_attention_weight_max": 47.97831726074219,
      "activations/layer13_attention_weight_min": -33.892086029052734,
      "activations/layer14_attention_weight_max": 35.96792984008789,
      "activations/layer14_attention_weight_min": -29.17707061767578,
      "activations/layer15_attention_weight_max": 40.33272171020508,
      "activations/layer15_attention_weight_min": -31.131702423095703,
      "activations/layer16_attention_weight_max": 31.482303619384766,
      "activations/layer16_attention_weight_min": -26.879980087280273,
      "activations/layer17_attention_weight_max": 49.65321731567383,
      "activations/layer17_attention_weight_min": -40.4047737121582,
      "activations/layer18_attention_weight_max": 52.695068359375,
      "activations/layer18_attention_weight_min": -34.29119110107422,
      "activations/layer19_attention_weight_max": 18.924718856811523,
      "activations/layer19_attention_weight_min": -19.683731079101562,
      "activations/layer1_attention_weight_max": 17.304410934448242,
      "activations/layer1_attention_weight_min": -15.690986633300781,
      "activations/layer20_attention_weight_max": 19.065351486206055,
      "activations/layer20_attention_weight_min": -21.17730712890625,
      "activations/layer21_attention_weight_max": 40.68477249145508,
      "activations/layer21_attention_weight_min": -25.34708595275879,
      "activations/layer22_attention_weight_max": 32.09019088745117,
      "activations/layer22_attention_weight_min": -23.138160705566406,
      "activations/layer23_attention_weight_max": 29.476652145385742,
      "activations/layer23_attention_weight_min": -20.174509048461914,
      "activations/layer2_attention_weight_max": 29.727384567260742,
      "activations/layer2_attention_weight_min": -28.81497573852539,
      "activations/layer3_attention_weight_max": 78.71569061279297,
      "activations/layer3_attention_weight_min": -82.0472183227539,
      "activations/layer4_attention_weight_max": 98.0307846069336,
      "activations/layer4_attention_weight_min": -93.68560028076172,
      "activations/layer5_attention_weight_max": 91.5667953491211,
      "activations/layer5_attention_weight_min": -84.13640594482422,
      "activations/layer6_attention_weight_max": 53.23844909667969,
      "activations/layer6_attention_weight_min": -51.86616516113281,
      "activations/layer7_attention_weight_max": 72.89328002929688,
      "activations/layer7_attention_weight_min": -69.92780303955078,
      "activations/layer8_attention_weight_max": 42.68843078613281,
      "activations/layer8_attention_weight_min": -41.596900939941406,
      "activations/layer9_attention_weight_max": 50.269229888916016,
      "activations/layer9_attention_weight_min": -46.9102897644043,
      "epoch": 6.23,
      "learning_rate": 0.00011094772727272726,
      "loss": 2.8407,
      "step": 107200
    },
    {
      "activations/layer0_attention_weight_max": 15.31829833984375,
      "activations/layer0_attention_weight_min": -14.908578872680664,
      "activations/layer10_attention_weight_max": 40.6351203918457,
      "activations/layer10_attention_weight_min": -36.22144317626953,
      "activations/layer11_attention_weight_max": 31.57269859313965,
      "activations/layer11_attention_weight_min": -29.066120147705078,
      "activations/layer12_attention_weight_max": 27.742464065551758,
      "activations/layer12_attention_weight_min": -25.130537033081055,
      "activations/layer13_attention_weight_max": 46.57069396972656,
      "activations/layer13_attention_weight_min": -31.947986602783203,
      "activations/layer14_attention_weight_max": 64.40470123291016,
      "activations/layer14_attention_weight_min": -33.70673370361328,
      "activations/layer15_attention_weight_max": 48.47432327270508,
      "activations/layer15_attention_weight_min": -29.122102737426758,
      "activations/layer16_attention_weight_max": 42.5915641784668,
      "activations/layer16_attention_weight_min": -26.474687576293945,
      "activations/layer17_attention_weight_max": 58.81060791015625,
      "activations/layer17_attention_weight_min": -41.38740158081055,
      "activations/layer18_attention_weight_max": 53.69152069091797,
      "activations/layer18_attention_weight_min": -34.11077117919922,
      "activations/layer19_attention_weight_max": 24.336177825927734,
      "activations/layer19_attention_weight_min": -21.527820587158203,
      "activations/layer1_attention_weight_max": 17.22443389892578,
      "activations/layer1_attention_weight_min": -16.76581382751465,
      "activations/layer20_attention_weight_max": 30.51434898376465,
      "activations/layer20_attention_weight_min": -21.10624122619629,
      "activations/layer21_attention_weight_max": 52.97700119018555,
      "activations/layer21_attention_weight_min": -26.270797729492188,
      "activations/layer22_attention_weight_max": 43.14787673950195,
      "activations/layer22_attention_weight_min": -23.85094451904297,
      "activations/layer23_attention_weight_max": 38.8958740234375,
      "activations/layer23_attention_weight_min": -19.29975128173828,
      "activations/layer2_attention_weight_max": 30.60651969909668,
      "activations/layer2_attention_weight_min": -28.77528190612793,
      "activations/layer3_attention_weight_max": 83.96514129638672,
      "activations/layer3_attention_weight_min": -80.6501693725586,
      "activations/layer4_attention_weight_max": 98.44392395019531,
      "activations/layer4_attention_weight_min": -95.7457504272461,
      "activations/layer5_attention_weight_max": 94.90972137451172,
      "activations/layer5_attention_weight_min": -86.6389389038086,
      "activations/layer6_attention_weight_max": 58.889408111572266,
      "activations/layer6_attention_weight_min": -53.04758071899414,
      "activations/layer7_attention_weight_max": 77.51347351074219,
      "activations/layer7_attention_weight_min": -68.27162170410156,
      "activations/layer8_attention_weight_max": 45.68284225463867,
      "activations/layer8_attention_weight_min": -46.272518157958984,
      "activations/layer9_attention_weight_max": 54.56110382080078,
      "activations/layer9_attention_weight_min": -51.05813980102539,
      "epoch": 6.23,
      "learning_rate": 0.00011092878787878787,
      "loss": 2.8278,
      "step": 107250
    },
    {
      "activations/layer0_attention_weight_max": 15.002889633178711,
      "activations/layer0_attention_weight_min": -15.039158821105957,
      "activations/layer10_attention_weight_max": 37.05643081665039,
      "activations/layer10_attention_weight_min": -34.25200271606445,
      "activations/layer11_attention_weight_max": 30.895606994628906,
      "activations/layer11_attention_weight_min": -28.260129928588867,
      "activations/layer12_attention_weight_max": 23.845468521118164,
      "activations/layer12_attention_weight_min": -26.347288131713867,
      "activations/layer13_attention_weight_max": 36.887611389160156,
      "activations/layer13_attention_weight_min": -30.958629608154297,
      "activations/layer14_attention_weight_max": 36.620662689208984,
      "activations/layer14_attention_weight_min": -27.91922378540039,
      "activations/layer15_attention_weight_max": 41.33820343017578,
      "activations/layer15_attention_weight_min": -27.912431716918945,
      "activations/layer16_attention_weight_max": 32.140811920166016,
      "activations/layer16_attention_weight_min": -25.10511589050293,
      "activations/layer17_attention_weight_max": 51.389076232910156,
      "activations/layer17_attention_weight_min": -38.11894607543945,
      "activations/layer18_attention_weight_max": 48.670143127441406,
      "activations/layer18_attention_weight_min": -31.188278198242188,
      "activations/layer19_attention_weight_max": 19.480966567993164,
      "activations/layer19_attention_weight_min": -20.36250114440918,
      "activations/layer1_attention_weight_max": 17.1082820892334,
      "activations/layer1_attention_weight_min": -14.597796440124512,
      "activations/layer20_attention_weight_max": 20.49014663696289,
      "activations/layer20_attention_weight_min": -22.52596664428711,
      "activations/layer21_attention_weight_max": 35.35260772705078,
      "activations/layer21_attention_weight_min": -23.848337173461914,
      "activations/layer22_attention_weight_max": 30.833398818969727,
      "activations/layer22_attention_weight_min": -23.24058723449707,
      "activations/layer23_attention_weight_max": 29.09960174560547,
      "activations/layer23_attention_weight_min": -19.99481964111328,
      "activations/layer2_attention_weight_max": 29.324779510498047,
      "activations/layer2_attention_weight_min": -27.646848678588867,
      "activations/layer3_attention_weight_max": 78.98072814941406,
      "activations/layer3_attention_weight_min": -80.93861389160156,
      "activations/layer4_attention_weight_max": 97.49979400634766,
      "activations/layer4_attention_weight_min": -88.13426971435547,
      "activations/layer5_attention_weight_max": 87.14753723144531,
      "activations/layer5_attention_weight_min": -78.81439208984375,
      "activations/layer6_attention_weight_max": 52.88900375366211,
      "activations/layer6_attention_weight_min": -48.22157287597656,
      "activations/layer7_attention_weight_max": 69.20686340332031,
      "activations/layer7_attention_weight_min": -63.54933166503906,
      "activations/layer8_attention_weight_max": 41.55274200439453,
      "activations/layer8_attention_weight_min": -40.48640060424805,
      "activations/layer9_attention_weight_max": 49.190547943115234,
      "activations/layer9_attention_weight_min": -46.60388946533203,
      "epoch": 6.23,
      "learning_rate": 0.00011090984848484847,
      "loss": 2.8299,
      "step": 107300
    },
    {
      "activations/layer0_attention_weight_max": 14.227190971374512,
      "activations/layer0_attention_weight_min": -15.177271842956543,
      "activations/layer10_attention_weight_max": 36.3804817199707,
      "activations/layer10_attention_weight_min": -33.17053985595703,
      "activations/layer11_attention_weight_max": 30.115150451660156,
      "activations/layer11_attention_weight_min": -27.60509490966797,
      "activations/layer12_attention_weight_max": 25.22088623046875,
      "activations/layer12_attention_weight_min": -26.776430130004883,
      "activations/layer13_attention_weight_max": 47.72502136230469,
      "activations/layer13_attention_weight_min": -32.39142990112305,
      "activations/layer14_attention_weight_max": 36.81320571899414,
      "activations/layer14_attention_weight_min": -30.569467544555664,
      "activations/layer15_attention_weight_max": 39.270259857177734,
      "activations/layer15_attention_weight_min": -30.198009490966797,
      "activations/layer16_attention_weight_max": 33.664310455322266,
      "activations/layer16_attention_weight_min": -27.410932540893555,
      "activations/layer17_attention_weight_max": 52.57554626464844,
      "activations/layer17_attention_weight_min": -40.350223541259766,
      "activations/layer18_attention_weight_max": 47.19172668457031,
      "activations/layer18_attention_weight_min": -34.19295120239258,
      "activations/layer19_attention_weight_max": 22.179157257080078,
      "activations/layer19_attention_weight_min": -21.26264190673828,
      "activations/layer1_attention_weight_max": 16.943222045898438,
      "activations/layer1_attention_weight_min": -13.603873252868652,
      "activations/layer20_attention_weight_max": 22.793249130249023,
      "activations/layer20_attention_weight_min": -27.815837860107422,
      "activations/layer21_attention_weight_max": 36.065059661865234,
      "activations/layer21_attention_weight_min": -22.347488403320312,
      "activations/layer22_attention_weight_max": 34.58468246459961,
      "activations/layer22_attention_weight_min": -25.553810119628906,
      "activations/layer23_attention_weight_max": 32.714813232421875,
      "activations/layer23_attention_weight_min": -20.496057510375977,
      "activations/layer2_attention_weight_max": 30.095508575439453,
      "activations/layer2_attention_weight_min": -27.601682662963867,
      "activations/layer3_attention_weight_max": 82.17289733886719,
      "activations/layer3_attention_weight_min": -83.8777084350586,
      "activations/layer4_attention_weight_max": 100.0926513671875,
      "activations/layer4_attention_weight_min": -93.39354705810547,
      "activations/layer5_attention_weight_max": 95.799072265625,
      "activations/layer5_attention_weight_min": -85.07270050048828,
      "activations/layer6_attention_weight_max": 55.720306396484375,
      "activations/layer6_attention_weight_min": -52.37385177612305,
      "activations/layer7_attention_weight_max": 74.28584289550781,
      "activations/layer7_attention_weight_min": -70.67584228515625,
      "activations/layer8_attention_weight_max": 42.77676773071289,
      "activations/layer8_attention_weight_min": -44.82878494262695,
      "activations/layer9_attention_weight_max": 47.695472717285156,
      "activations/layer9_attention_weight_min": -46.635169982910156,
      "epoch": 6.24,
      "learning_rate": 0.00011089090909090908,
      "loss": 2.8383,
      "step": 107350
    },
    {
      "activations/layer0_attention_weight_max": 14.413890838623047,
      "activations/layer0_attention_weight_min": -15.038647651672363,
      "activations/layer10_attention_weight_max": 40.43819046020508,
      "activations/layer10_attention_weight_min": -35.122528076171875,
      "activations/layer11_attention_weight_max": 35.35454559326172,
      "activations/layer11_attention_weight_min": -29.84071922302246,
      "activations/layer12_attention_weight_max": 21.40458869934082,
      "activations/layer12_attention_weight_min": -26.463401794433594,
      "activations/layer13_attention_weight_max": 70.69291687011719,
      "activations/layer13_attention_weight_min": -48.1666374206543,
      "activations/layer14_attention_weight_max": 68.6861343383789,
      "activations/layer14_attention_weight_min": -44.0002555847168,
      "activations/layer15_attention_weight_max": 51.3500862121582,
      "activations/layer15_attention_weight_min": -36.37517166137695,
      "activations/layer16_attention_weight_max": 40.2700309753418,
      "activations/layer16_attention_weight_min": -27.227184295654297,
      "activations/layer17_attention_weight_max": 55.1356315612793,
      "activations/layer17_attention_weight_min": -42.5116081237793,
      "activations/layer18_attention_weight_max": 45.687599182128906,
      "activations/layer18_attention_weight_min": -36.15618133544922,
      "activations/layer19_attention_weight_max": 22.463462829589844,
      "activations/layer19_attention_weight_min": -19.793664932250977,
      "activations/layer1_attention_weight_max": 17.315135955810547,
      "activations/layer1_attention_weight_min": -13.90376091003418,
      "activations/layer20_attention_weight_max": 21.835039138793945,
      "activations/layer20_attention_weight_min": -23.63987922668457,
      "activations/layer21_attention_weight_max": 46.586997985839844,
      "activations/layer21_attention_weight_min": -25.682302474975586,
      "activations/layer22_attention_weight_max": 32.122833251953125,
      "activations/layer22_attention_weight_min": -25.213289260864258,
      "activations/layer23_attention_weight_max": 30.686857223510742,
      "activations/layer23_attention_weight_min": -18.392683029174805,
      "activations/layer2_attention_weight_max": 30.193862915039062,
      "activations/layer2_attention_weight_min": -29.1988582611084,
      "activations/layer3_attention_weight_max": 83.3514633178711,
      "activations/layer3_attention_weight_min": -87.85830688476562,
      "activations/layer4_attention_weight_max": 101.43013763427734,
      "activations/layer4_attention_weight_min": -95.09512329101562,
      "activations/layer5_attention_weight_max": 93.7193832397461,
      "activations/layer5_attention_weight_min": -86.09556579589844,
      "activations/layer6_attention_weight_max": 59.571128845214844,
      "activations/layer6_attention_weight_min": -57.20531463623047,
      "activations/layer7_attention_weight_max": 76.65803527832031,
      "activations/layer7_attention_weight_min": -72.62564086914062,
      "activations/layer8_attention_weight_max": 45.64382553100586,
      "activations/layer8_attention_weight_min": -47.14793014526367,
      "activations/layer9_attention_weight_max": 62.179317474365234,
      "activations/layer9_attention_weight_min": -51.455440521240234,
      "epoch": 6.24,
      "learning_rate": 0.00011087196969696968,
      "loss": 2.8422,
      "step": 107400
    },
    {
      "activations/layer0_attention_weight_max": 14.823687553405762,
      "activations/layer0_attention_weight_min": -14.863168716430664,
      "activations/layer10_attention_weight_max": 37.46509552001953,
      "activations/layer10_attention_weight_min": -36.199520111083984,
      "activations/layer11_attention_weight_max": 30.130765914916992,
      "activations/layer11_attention_weight_min": -30.760337829589844,
      "activations/layer12_attention_weight_max": 22.389127731323242,
      "activations/layer12_attention_weight_min": -23.242984771728516,
      "activations/layer13_attention_weight_max": 50.651126861572266,
      "activations/layer13_attention_weight_min": -32.53037643432617,
      "activations/layer14_attention_weight_max": 56.15919494628906,
      "activations/layer14_attention_weight_min": -32.36527633666992,
      "activations/layer15_attention_weight_max": 42.98711013793945,
      "activations/layer15_attention_weight_min": -29.473407745361328,
      "activations/layer16_attention_weight_max": 36.412940979003906,
      "activations/layer16_attention_weight_min": -26.604904174804688,
      "activations/layer17_attention_weight_max": 56.57073974609375,
      "activations/layer17_attention_weight_min": -39.39263153076172,
      "activations/layer18_attention_weight_max": 48.01030349731445,
      "activations/layer18_attention_weight_min": -32.79020309448242,
      "activations/layer19_attention_weight_max": 21.365922927856445,
      "activations/layer19_attention_weight_min": -19.197904586791992,
      "activations/layer1_attention_weight_max": 17.50697898864746,
      "activations/layer1_attention_weight_min": -14.2409029006958,
      "activations/layer20_attention_weight_max": 22.794443130493164,
      "activations/layer20_attention_weight_min": -21.544189453125,
      "activations/layer21_attention_weight_max": 38.05533218383789,
      "activations/layer21_attention_weight_min": -25.880552291870117,
      "activations/layer22_attention_weight_max": 36.23676300048828,
      "activations/layer22_attention_weight_min": -23.635164260864258,
      "activations/layer23_attention_weight_max": 32.7670783996582,
      "activations/layer23_attention_weight_min": -17.84493637084961,
      "activations/layer2_attention_weight_max": 28.408496856689453,
      "activations/layer2_attention_weight_min": -29.118452072143555,
      "activations/layer3_attention_weight_max": 79.26478576660156,
      "activations/layer3_attention_weight_min": -82.10265350341797,
      "activations/layer4_attention_weight_max": 99.16992950439453,
      "activations/layer4_attention_weight_min": -95.87334442138672,
      "activations/layer5_attention_weight_max": 91.94222259521484,
      "activations/layer5_attention_weight_min": -85.70463562011719,
      "activations/layer6_attention_weight_max": 55.25788879394531,
      "activations/layer6_attention_weight_min": -51.10442352294922,
      "activations/layer7_attention_weight_max": 78.15837860107422,
      "activations/layer7_attention_weight_min": -70.96574401855469,
      "activations/layer8_attention_weight_max": 47.67451477050781,
      "activations/layer8_attention_weight_min": -40.88312530517578,
      "activations/layer9_attention_weight_max": 58.159576416015625,
      "activations/layer9_attention_weight_min": -51.90374755859375,
      "epoch": 6.24,
      "learning_rate": 0.0001108530303030303,
      "loss": 2.8337,
      "step": 107450
    },
    {
      "activations/layer0_attention_weight_max": 15.71414852142334,
      "activations/layer0_attention_weight_min": -14.936532974243164,
      "activations/layer10_attention_weight_max": 38.726444244384766,
      "activations/layer10_attention_weight_min": -39.501739501953125,
      "activations/layer11_attention_weight_max": 32.75108337402344,
      "activations/layer11_attention_weight_min": -30.726703643798828,
      "activations/layer12_attention_weight_max": 21.869064331054688,
      "activations/layer12_attention_weight_min": -29.608362197875977,
      "activations/layer13_attention_weight_max": 54.11520767211914,
      "activations/layer13_attention_weight_min": -34.47123718261719,
      "activations/layer14_attention_weight_max": 49.456607818603516,
      "activations/layer14_attention_weight_min": -36.55305099487305,
      "activations/layer15_attention_weight_max": 40.61798095703125,
      "activations/layer15_attention_weight_min": -31.265708923339844,
      "activations/layer16_attention_weight_max": 35.94342803955078,
      "activations/layer16_attention_weight_min": -28.401180267333984,
      "activations/layer17_attention_weight_max": 61.94997024536133,
      "activations/layer17_attention_weight_min": -42.222110748291016,
      "activations/layer18_attention_weight_max": 54.73563003540039,
      "activations/layer18_attention_weight_min": -36.14402389526367,
      "activations/layer19_attention_weight_max": 21.515926361083984,
      "activations/layer19_attention_weight_min": -22.886573791503906,
      "activations/layer1_attention_weight_max": 18.18851089477539,
      "activations/layer1_attention_weight_min": -16.280031204223633,
      "activations/layer20_attention_weight_max": 23.612031936645508,
      "activations/layer20_attention_weight_min": -24.174190521240234,
      "activations/layer21_attention_weight_max": 41.91062927246094,
      "activations/layer21_attention_weight_min": -26.30145263671875,
      "activations/layer22_attention_weight_max": 38.167579650878906,
      "activations/layer22_attention_weight_min": -24.10831642150879,
      "activations/layer23_attention_weight_max": 32.396095275878906,
      "activations/layer23_attention_weight_min": -21.3198299407959,
      "activations/layer2_attention_weight_max": 30.027729034423828,
      "activations/layer2_attention_weight_min": -28.6324462890625,
      "activations/layer3_attention_weight_max": 84.24811553955078,
      "activations/layer3_attention_weight_min": -82.30221557617188,
      "activations/layer4_attention_weight_max": 100.22219848632812,
      "activations/layer4_attention_weight_min": -95.0341567993164,
      "activations/layer5_attention_weight_max": 91.62633514404297,
      "activations/layer5_attention_weight_min": -84.05366516113281,
      "activations/layer6_attention_weight_max": 55.83002853393555,
      "activations/layer6_attention_weight_min": -53.456974029541016,
      "activations/layer7_attention_weight_max": 72.599365234375,
      "activations/layer7_attention_weight_min": -70.43450927734375,
      "activations/layer8_attention_weight_max": 42.681419372558594,
      "activations/layer8_attention_weight_min": -44.65398406982422,
      "activations/layer9_attention_weight_max": 52.238014221191406,
      "activations/layer9_attention_weight_min": -51.803016662597656,
      "epoch": 6.25,
      "learning_rate": 0.0001108340909090909,
      "loss": 2.8292,
      "step": 107500
    },
    {
      "activations/layer0_attention_weight_max": 14.91663646697998,
      "activations/layer0_attention_weight_min": -14.950217247009277,
      "activations/layer10_attention_weight_max": 43.33815383911133,
      "activations/layer10_attention_weight_min": -36.431312561035156,
      "activations/layer11_attention_weight_max": 28.46666717529297,
      "activations/layer11_attention_weight_min": -27.35683250427246,
      "activations/layer12_attention_weight_max": 19.32452392578125,
      "activations/layer12_attention_weight_min": -23.339385986328125,
      "activations/layer13_attention_weight_max": 51.60334777832031,
      "activations/layer13_attention_weight_min": -36.13041687011719,
      "activations/layer14_attention_weight_max": 67.4900894165039,
      "activations/layer14_attention_weight_min": -42.58005905151367,
      "activations/layer15_attention_weight_max": 51.5424690246582,
      "activations/layer15_attention_weight_min": -32.694183349609375,
      "activations/layer16_attention_weight_max": 32.26874923706055,
      "activations/layer16_attention_weight_min": -26.235198974609375,
      "activations/layer17_attention_weight_max": 57.66118240356445,
      "activations/layer17_attention_weight_min": -41.55154037475586,
      "activations/layer18_attention_weight_max": 48.111602783203125,
      "activations/layer18_attention_weight_min": -34.51962661743164,
      "activations/layer19_attention_weight_max": 25.824817657470703,
      "activations/layer19_attention_weight_min": -19.790334701538086,
      "activations/layer1_attention_weight_max": 16.51918601989746,
      "activations/layer1_attention_weight_min": -15.525323867797852,
      "activations/layer20_attention_weight_max": 25.012834548950195,
      "activations/layer20_attention_weight_min": -23.688217163085938,
      "activations/layer21_attention_weight_max": 39.139896392822266,
      "activations/layer21_attention_weight_min": -29.56871795654297,
      "activations/layer22_attention_weight_max": 31.874652862548828,
      "activations/layer22_attention_weight_min": -24.816308975219727,
      "activations/layer23_attention_weight_max": 29.36249351501465,
      "activations/layer23_attention_weight_min": -18.519519805908203,
      "activations/layer2_attention_weight_max": 31.622661590576172,
      "activations/layer2_attention_weight_min": -29.43399429321289,
      "activations/layer3_attention_weight_max": 81.87797546386719,
      "activations/layer3_attention_weight_min": -87.7509536743164,
      "activations/layer4_attention_weight_max": 103.94081115722656,
      "activations/layer4_attention_weight_min": -96.42064666748047,
      "activations/layer5_attention_weight_max": 99.19405364990234,
      "activations/layer5_attention_weight_min": -89.276611328125,
      "activations/layer6_attention_weight_max": 58.97859573364258,
      "activations/layer6_attention_weight_min": -52.65827178955078,
      "activations/layer7_attention_weight_max": 73.89226531982422,
      "activations/layer7_attention_weight_min": -71.50878143310547,
      "activations/layer8_attention_weight_max": 47.8972053527832,
      "activations/layer8_attention_weight_min": -46.89402770996094,
      "activations/layer9_attention_weight_max": 57.8045654296875,
      "activations/layer9_attention_weight_min": -47.42545700073242,
      "epoch": 6.25,
      "learning_rate": 0.0001108151515151515,
      "loss": 2.8427,
      "step": 107550
    },
    {
      "activations/layer0_attention_weight_max": 14.892797470092773,
      "activations/layer0_attention_weight_min": -14.859166145324707,
      "activations/layer10_attention_weight_max": 39.39224624633789,
      "activations/layer10_attention_weight_min": -35.48359298706055,
      "activations/layer11_attention_weight_max": 28.432065963745117,
      "activations/layer11_attention_weight_min": -28.362857818603516,
      "activations/layer12_attention_weight_max": 23.72638702392578,
      "activations/layer12_attention_weight_min": -24.491071701049805,
      "activations/layer13_attention_weight_max": 56.23500061035156,
      "activations/layer13_attention_weight_min": -37.7334098815918,
      "activations/layer14_attention_weight_max": 43.854888916015625,
      "activations/layer14_attention_weight_min": -32.38887405395508,
      "activations/layer15_attention_weight_max": 40.27092742919922,
      "activations/layer15_attention_weight_min": -31.42987823486328,
      "activations/layer16_attention_weight_max": 36.35116958618164,
      "activations/layer16_attention_weight_min": -27.18988037109375,
      "activations/layer17_attention_weight_max": 62.88447952270508,
      "activations/layer17_attention_weight_min": -43.65036392211914,
      "activations/layer18_attention_weight_max": 54.900360107421875,
      "activations/layer18_attention_weight_min": -36.483612060546875,
      "activations/layer19_attention_weight_max": 23.862884521484375,
      "activations/layer19_attention_weight_min": -20.218547821044922,
      "activations/layer1_attention_weight_max": 16.624670028686523,
      "activations/layer1_attention_weight_min": -15.902817726135254,
      "activations/layer20_attention_weight_max": 26.596275329589844,
      "activations/layer20_attention_weight_min": -21.269466400146484,
      "activations/layer21_attention_weight_max": 43.503997802734375,
      "activations/layer21_attention_weight_min": -24.37864112854004,
      "activations/layer22_attention_weight_max": 37.68598556518555,
      "activations/layer22_attention_weight_min": -24.761920928955078,
      "activations/layer23_attention_weight_max": 34.91981887817383,
      "activations/layer23_attention_weight_min": -21.52709197998047,
      "activations/layer2_attention_weight_max": 29.918874740600586,
      "activations/layer2_attention_weight_min": -28.167631149291992,
      "activations/layer3_attention_weight_max": 82.04303741455078,
      "activations/layer3_attention_weight_min": -84.17550659179688,
      "activations/layer4_attention_weight_max": 99.34136199951172,
      "activations/layer4_attention_weight_min": -90.9310073852539,
      "activations/layer5_attention_weight_max": 94.88134002685547,
      "activations/layer5_attention_weight_min": -92.56005096435547,
      "activations/layer6_attention_weight_max": 55.827232360839844,
      "activations/layer6_attention_weight_min": -55.30409240722656,
      "activations/layer7_attention_weight_max": 69.96728515625,
      "activations/layer7_attention_weight_min": -69.92416381835938,
      "activations/layer8_attention_weight_max": 44.5815315246582,
      "activations/layer8_attention_weight_min": -45.51187515258789,
      "activations/layer9_attention_weight_max": 52.15735626220703,
      "activations/layer9_attention_weight_min": -48.88573455810547,
      "epoch": 6.25,
      "learning_rate": 0.00011079621212121212,
      "loss": 2.8346,
      "step": 107600
    },
    {
      "activations/layer0_attention_weight_max": 15.21483039855957,
      "activations/layer0_attention_weight_min": -15.273608207702637,
      "activations/layer10_attention_weight_max": 44.59748458862305,
      "activations/layer10_attention_weight_min": -38.17790985107422,
      "activations/layer11_attention_weight_max": 34.43403244018555,
      "activations/layer11_attention_weight_min": -29.42365264892578,
      "activations/layer12_attention_weight_max": 45.51467514038086,
      "activations/layer12_attention_weight_min": -25.73847007751465,
      "activations/layer13_attention_weight_max": 66.24674987792969,
      "activations/layer13_attention_weight_min": -42.6119270324707,
      "activations/layer14_attention_weight_max": 71.58294677734375,
      "activations/layer14_attention_weight_min": -41.84597396850586,
      "activations/layer15_attention_weight_max": 51.78108596801758,
      "activations/layer15_attention_weight_min": -34.101646423339844,
      "activations/layer16_attention_weight_max": 41.36105728149414,
      "activations/layer16_attention_weight_min": -24.98945426940918,
      "activations/layer17_attention_weight_max": 57.24844741821289,
      "activations/layer17_attention_weight_min": -43.0237922668457,
      "activations/layer18_attention_weight_max": 50.846214294433594,
      "activations/layer18_attention_weight_min": -34.136505126953125,
      "activations/layer19_attention_weight_max": 28.271081924438477,
      "activations/layer19_attention_weight_min": -21.230167388916016,
      "activations/layer1_attention_weight_max": 17.665983200073242,
      "activations/layer1_attention_weight_min": -15.151399612426758,
      "activations/layer20_attention_weight_max": 25.11812400817871,
      "activations/layer20_attention_weight_min": -22.849346160888672,
      "activations/layer21_attention_weight_max": 43.15260696411133,
      "activations/layer21_attention_weight_min": -26.83507537841797,
      "activations/layer22_attention_weight_max": 35.427024841308594,
      "activations/layer22_attention_weight_min": -23.54778289794922,
      "activations/layer23_attention_weight_max": 33.06842041015625,
      "activations/layer23_attention_weight_min": -17.005586624145508,
      "activations/layer2_attention_weight_max": 32.99808120727539,
      "activations/layer2_attention_weight_min": -30.61089324951172,
      "activations/layer3_attention_weight_max": 84.96051788330078,
      "activations/layer3_attention_weight_min": -84.68507385253906,
      "activations/layer4_attention_weight_max": 98.39459228515625,
      "activations/layer4_attention_weight_min": -96.39553833007812,
      "activations/layer5_attention_weight_max": 93.2901611328125,
      "activations/layer5_attention_weight_min": -87.40528869628906,
      "activations/layer6_attention_weight_max": 54.854610443115234,
      "activations/layer6_attention_weight_min": -50.0885124206543,
      "activations/layer7_attention_weight_max": 73.68650817871094,
      "activations/layer7_attention_weight_min": -73.01557159423828,
      "activations/layer8_attention_weight_max": 48.45348358154297,
      "activations/layer8_attention_weight_min": -44.428016662597656,
      "activations/layer9_attention_weight_max": 57.11548614501953,
      "activations/layer9_attention_weight_min": -48.113807678222656,
      "epoch": 6.26,
      "learning_rate": 0.00011077727272727273,
      "loss": 2.8324,
      "step": 107650
    },
    {
      "activations/layer0_attention_weight_max": 16.16727638244629,
      "activations/layer0_attention_weight_min": -14.853597640991211,
      "activations/layer10_attention_weight_max": 40.081825256347656,
      "activations/layer10_attention_weight_min": -38.057159423828125,
      "activations/layer11_attention_weight_max": 32.87610626220703,
      "activations/layer11_attention_weight_min": -29.21436882019043,
      "activations/layer12_attention_weight_max": 22.04953956604004,
      "activations/layer12_attention_weight_min": -24.547698974609375,
      "activations/layer13_attention_weight_max": 51.2767219543457,
      "activations/layer13_attention_weight_min": -32.8135986328125,
      "activations/layer14_attention_weight_max": 44.30879211425781,
      "activations/layer14_attention_weight_min": -28.96697425842285,
      "activations/layer15_attention_weight_max": 40.611328125,
      "activations/layer15_attention_weight_min": -28.591394424438477,
      "activations/layer16_attention_weight_max": 40.76573944091797,
      "activations/layer16_attention_weight_min": -27.848026275634766,
      "activations/layer17_attention_weight_max": 64.83389282226562,
      "activations/layer17_attention_weight_min": -45.019073486328125,
      "activations/layer18_attention_weight_max": 53.59943389892578,
      "activations/layer18_attention_weight_min": -38.383968353271484,
      "activations/layer19_attention_weight_max": 25.427785873413086,
      "activations/layer19_attention_weight_min": -20.555212020874023,
      "activations/layer1_attention_weight_max": 18.1861515045166,
      "activations/layer1_attention_weight_min": -14.132465362548828,
      "activations/layer20_attention_weight_max": 24.97738265991211,
      "activations/layer20_attention_weight_min": -24.89419174194336,
      "activations/layer21_attention_weight_max": 41.91902542114258,
      "activations/layer21_attention_weight_min": -27.02623748779297,
      "activations/layer22_attention_weight_max": 37.40606689453125,
      "activations/layer22_attention_weight_min": -23.65030860900879,
      "activations/layer23_attention_weight_max": 35.29155349731445,
      "activations/layer23_attention_weight_min": -20.633344650268555,
      "activations/layer2_attention_weight_max": 30.708614349365234,
      "activations/layer2_attention_weight_min": -28.053733825683594,
      "activations/layer3_attention_weight_max": 84.2123031616211,
      "activations/layer3_attention_weight_min": -84.98397827148438,
      "activations/layer4_attention_weight_max": 100.48954772949219,
      "activations/layer4_attention_weight_min": -90.28424072265625,
      "activations/layer5_attention_weight_max": 99.36272430419922,
      "activations/layer5_attention_weight_min": -84.1418228149414,
      "activations/layer6_attention_weight_max": 57.11652374267578,
      "activations/layer6_attention_weight_min": -52.802371978759766,
      "activations/layer7_attention_weight_max": 73.03641510009766,
      "activations/layer7_attention_weight_min": -74.1660385131836,
      "activations/layer8_attention_weight_max": 45.459266662597656,
      "activations/layer8_attention_weight_min": -44.200538635253906,
      "activations/layer9_attention_weight_max": 50.06310272216797,
      "activations/layer9_attention_weight_min": -52.30634307861328,
      "epoch": 6.26,
      "learning_rate": 0.00011075833333333332,
      "loss": 2.826,
      "step": 107700
    },
    {
      "activations/layer0_attention_weight_max": 15.104812622070312,
      "activations/layer0_attention_weight_min": -14.711271286010742,
      "activations/layer10_attention_weight_max": 46.65962600708008,
      "activations/layer10_attention_weight_min": -43.328128814697266,
      "activations/layer11_attention_weight_max": 40.59989547729492,
      "activations/layer11_attention_weight_min": -34.13214874267578,
      "activations/layer12_attention_weight_max": 19.7747745513916,
      "activations/layer12_attention_weight_min": -27.992338180541992,
      "activations/layer13_attention_weight_max": 52.598121643066406,
      "activations/layer13_attention_weight_min": -34.98690414428711,
      "activations/layer14_attention_weight_max": 56.49858856201172,
      "activations/layer14_attention_weight_min": -30.0107364654541,
      "activations/layer15_attention_weight_max": 39.415672302246094,
      "activations/layer15_attention_weight_min": -27.573734283447266,
      "activations/layer16_attention_weight_max": 30.969066619873047,
      "activations/layer16_attention_weight_min": -26.203845977783203,
      "activations/layer17_attention_weight_max": 48.76371383666992,
      "activations/layer17_attention_weight_min": -40.378501892089844,
      "activations/layer18_attention_weight_max": 43.23836135864258,
      "activations/layer18_attention_weight_min": -35.69955825805664,
      "activations/layer19_attention_weight_max": 22.145591735839844,
      "activations/layer19_attention_weight_min": -20.46839714050293,
      "activations/layer1_attention_weight_max": 18.252525329589844,
      "activations/layer1_attention_weight_min": -16.12925910949707,
      "activations/layer20_attention_weight_max": 20.56570053100586,
      "activations/layer20_attention_weight_min": -21.285743713378906,
      "activations/layer21_attention_weight_max": 39.765892028808594,
      "activations/layer21_attention_weight_min": -23.913286209106445,
      "activations/layer22_attention_weight_max": 31.490863800048828,
      "activations/layer22_attention_weight_min": -23.814247131347656,
      "activations/layer23_attention_weight_max": 29.49067497253418,
      "activations/layer23_attention_weight_min": -19.475101470947266,
      "activations/layer2_attention_weight_max": 33.57362365722656,
      "activations/layer2_attention_weight_min": -32.217281341552734,
      "activations/layer3_attention_weight_max": 95.3783950805664,
      "activations/layer3_attention_weight_min": -97.3199234008789,
      "activations/layer4_attention_weight_max": 116.46760559082031,
      "activations/layer4_attention_weight_min": -105.78185272216797,
      "activations/layer5_attention_weight_max": 109.9202880859375,
      "activations/layer5_attention_weight_min": -97.19540405273438,
      "activations/layer6_attention_weight_max": 66.7379379272461,
      "activations/layer6_attention_weight_min": -59.12747573852539,
      "activations/layer7_attention_weight_max": 98.45512390136719,
      "activations/layer7_attention_weight_min": -77.43803405761719,
      "activations/layer8_attention_weight_max": 61.07131576538086,
      "activations/layer8_attention_weight_min": -47.43729782104492,
      "activations/layer9_attention_weight_max": 80.35580444335938,
      "activations/layer9_attention_weight_min": -61.37164306640625,
      "epoch": 6.26,
      "learning_rate": 0.00011073939393939393,
      "loss": 2.8323,
      "step": 107750
    },
    {
      "activations/layer0_attention_weight_max": 14.296041488647461,
      "activations/layer0_attention_weight_min": -14.879319190979004,
      "activations/layer10_attention_weight_max": 38.0339241027832,
      "activations/layer10_attention_weight_min": -32.858154296875,
      "activations/layer11_attention_weight_max": 30.66615104675293,
      "activations/layer11_attention_weight_min": -28.76158905029297,
      "activations/layer12_attention_weight_max": 20.797183990478516,
      "activations/layer12_attention_weight_min": -30.984474182128906,
      "activations/layer13_attention_weight_max": 61.338748931884766,
      "activations/layer13_attention_weight_min": -35.14783477783203,
      "activations/layer14_attention_weight_max": 64.20411682128906,
      "activations/layer14_attention_weight_min": -33.777801513671875,
      "activations/layer15_attention_weight_max": 47.70256805419922,
      "activations/layer15_attention_weight_min": -31.66145133972168,
      "activations/layer16_attention_weight_max": 36.58453369140625,
      "activations/layer16_attention_weight_min": -26.729379653930664,
      "activations/layer17_attention_weight_max": 61.17578125,
      "activations/layer17_attention_weight_min": -43.4600830078125,
      "activations/layer18_attention_weight_max": 52.76274871826172,
      "activations/layer18_attention_weight_min": -34.588199615478516,
      "activations/layer19_attention_weight_max": 24.291553497314453,
      "activations/layer19_attention_weight_min": -20.593692779541016,
      "activations/layer1_attention_weight_max": 18.25491714477539,
      "activations/layer1_attention_weight_min": -14.99155330657959,
      "activations/layer20_attention_weight_max": 27.021318435668945,
      "activations/layer20_attention_weight_min": -19.255815505981445,
      "activations/layer21_attention_weight_max": 58.36647415161133,
      "activations/layer21_attention_weight_min": -26.144474029541016,
      "activations/layer22_attention_weight_max": 38.42473602294922,
      "activations/layer22_attention_weight_min": -22.82756996154785,
      "activations/layer23_attention_weight_max": 32.52096176147461,
      "activations/layer23_attention_weight_min": -18.18255043029785,
      "activations/layer2_attention_weight_max": 30.25758171081543,
      "activations/layer2_attention_weight_min": -29.438899993896484,
      "activations/layer3_attention_weight_max": 77.07691192626953,
      "activations/layer3_attention_weight_min": -83.07903289794922,
      "activations/layer4_attention_weight_max": 95.74456787109375,
      "activations/layer4_attention_weight_min": -94.86544036865234,
      "activations/layer5_attention_weight_max": 93.50688171386719,
      "activations/layer5_attention_weight_min": -86.076416015625,
      "activations/layer6_attention_weight_max": 55.351627349853516,
      "activations/layer6_attention_weight_min": -50.48574447631836,
      "activations/layer7_attention_weight_max": 76.61711120605469,
      "activations/layer7_attention_weight_min": -67.10961151123047,
      "activations/layer8_attention_weight_max": 45.31529998779297,
      "activations/layer8_attention_weight_min": -42.31782913208008,
      "activations/layer9_attention_weight_max": 62.01800537109375,
      "activations/layer9_attention_weight_min": -49.24696350097656,
      "epoch": 6.26,
      "learning_rate": 0.00011072045454545452,
      "loss": 2.8449,
      "step": 107800
    },
    {
      "activations/layer0_attention_weight_max": 14.234375953674316,
      "activations/layer0_attention_weight_min": -15.49158763885498,
      "activations/layer10_attention_weight_max": 40.01499938964844,
      "activations/layer10_attention_weight_min": -34.24048614501953,
      "activations/layer11_attention_weight_max": 30.05696678161621,
      "activations/layer11_attention_weight_min": -27.174476623535156,
      "activations/layer12_attention_weight_max": 23.15441131591797,
      "activations/layer12_attention_weight_min": -28.51677894592285,
      "activations/layer13_attention_weight_max": 51.640140533447266,
      "activations/layer13_attention_weight_min": -36.487918853759766,
      "activations/layer14_attention_weight_max": 37.87275695800781,
      "activations/layer14_attention_weight_min": -30.779756546020508,
      "activations/layer15_attention_weight_max": 38.010826110839844,
      "activations/layer15_attention_weight_min": -29.997207641601562,
      "activations/layer16_attention_weight_max": 34.915470123291016,
      "activations/layer16_attention_weight_min": -25.604000091552734,
      "activations/layer17_attention_weight_max": 55.004947662353516,
      "activations/layer17_attention_weight_min": -45.555511474609375,
      "activations/layer18_attention_weight_max": 50.4488639831543,
      "activations/layer18_attention_weight_min": -35.44308090209961,
      "activations/layer19_attention_weight_max": 21.201263427734375,
      "activations/layer19_attention_weight_min": -20.889644622802734,
      "activations/layer1_attention_weight_max": 17.272493362426758,
      "activations/layer1_attention_weight_min": -14.144469261169434,
      "activations/layer20_attention_weight_max": 22.527292251586914,
      "activations/layer20_attention_weight_min": -22.375085830688477,
      "activations/layer21_attention_weight_max": 38.59827423095703,
      "activations/layer21_attention_weight_min": -25.0742244720459,
      "activations/layer22_attention_weight_max": 36.15100860595703,
      "activations/layer22_attention_weight_min": -26.07947540283203,
      "activations/layer23_attention_weight_max": 32.587581634521484,
      "activations/layer23_attention_weight_min": -18.544673919677734,
      "activations/layer2_attention_weight_max": 30.176586151123047,
      "activations/layer2_attention_weight_min": -27.498472213745117,
      "activations/layer3_attention_weight_max": 83.48006439208984,
      "activations/layer3_attention_weight_min": -83.7080078125,
      "activations/layer4_attention_weight_max": 99.35137939453125,
      "activations/layer4_attention_weight_min": -89.8748550415039,
      "activations/layer5_attention_weight_max": 94.10800170898438,
      "activations/layer5_attention_weight_min": -87.1461410522461,
      "activations/layer6_attention_weight_max": 56.938148498535156,
      "activations/layer6_attention_weight_min": -51.13404846191406,
      "activations/layer7_attention_weight_max": 71.41808319091797,
      "activations/layer7_attention_weight_min": -67.6611328125,
      "activations/layer8_attention_weight_max": 42.95564651489258,
      "activations/layer8_attention_weight_min": -40.8291130065918,
      "activations/layer9_attention_weight_max": 54.33271026611328,
      "activations/layer9_attention_weight_min": -50.569942474365234,
      "epoch": 6.27,
      "learning_rate": 0.00011070151515151514,
      "loss": 2.8424,
      "step": 107850
    },
    {
      "activations/layer0_attention_weight_max": 14.45197582244873,
      "activations/layer0_attention_weight_min": -14.66175365447998,
      "activations/layer10_attention_weight_max": 39.921852111816406,
      "activations/layer10_attention_weight_min": -39.1036376953125,
      "activations/layer11_attention_weight_max": 32.159515380859375,
      "activations/layer11_attention_weight_min": -30.773876190185547,
      "activations/layer12_attention_weight_max": 22.18033790588379,
      "activations/layer12_attention_weight_min": -25.987550735473633,
      "activations/layer13_attention_weight_max": 45.1185417175293,
      "activations/layer13_attention_weight_min": -33.77006530761719,
      "activations/layer14_attention_weight_max": 51.33674240112305,
      "activations/layer14_attention_weight_min": -33.430511474609375,
      "activations/layer15_attention_weight_max": 39.421226501464844,
      "activations/layer15_attention_weight_min": -29.363054275512695,
      "activations/layer16_attention_weight_max": 31.36069679260254,
      "activations/layer16_attention_weight_min": -29.400484085083008,
      "activations/layer17_attention_weight_max": 57.04757308959961,
      "activations/layer17_attention_weight_min": -43.41218948364258,
      "activations/layer18_attention_weight_max": 50.229888916015625,
      "activations/layer18_attention_weight_min": -37.44223403930664,
      "activations/layer19_attention_weight_max": 22.121789932250977,
      "activations/layer19_attention_weight_min": -20.417259216308594,
      "activations/layer1_attention_weight_max": 16.388797760009766,
      "activations/layer1_attention_weight_min": -17.520401000976562,
      "activations/layer20_attention_weight_max": 21.58779525756836,
      "activations/layer20_attention_weight_min": -26.3774471282959,
      "activations/layer21_attention_weight_max": 38.9541015625,
      "activations/layer21_attention_weight_min": -26.07518196105957,
      "activations/layer22_attention_weight_max": 32.544673919677734,
      "activations/layer22_attention_weight_min": -24.6855411529541,
      "activations/layer23_attention_weight_max": 31.326887130737305,
      "activations/layer23_attention_weight_min": -20.043441772460938,
      "activations/layer2_attention_weight_max": 29.996383666992188,
      "activations/layer2_attention_weight_min": -28.21945571899414,
      "activations/layer3_attention_weight_max": 80.97521209716797,
      "activations/layer3_attention_weight_min": -84.09955596923828,
      "activations/layer4_attention_weight_max": 102.55451965332031,
      "activations/layer4_attention_weight_min": -97.33856201171875,
      "activations/layer5_attention_weight_max": 97.98638916015625,
      "activations/layer5_attention_weight_min": -91.01400756835938,
      "activations/layer6_attention_weight_max": 60.747764587402344,
      "activations/layer6_attention_weight_min": -57.12255859375,
      "activations/layer7_attention_weight_max": 75.13102722167969,
      "activations/layer7_attention_weight_min": -73.73409271240234,
      "activations/layer8_attention_weight_max": 47.9767951965332,
      "activations/layer8_attention_weight_min": -46.43199157714844,
      "activations/layer9_attention_weight_max": 55.45321273803711,
      "activations/layer9_attention_weight_min": -57.00049591064453,
      "epoch": 6.27,
      "learning_rate": 0.00011068257575757575,
      "loss": 2.8371,
      "step": 107900
    },
    {
      "activations/layer0_attention_weight_max": 15.159637451171875,
      "activations/layer0_attention_weight_min": -15.14448070526123,
      "activations/layer10_attention_weight_max": 37.59750747680664,
      "activations/layer10_attention_weight_min": -36.171417236328125,
      "activations/layer11_attention_weight_max": 30.71489715576172,
      "activations/layer11_attention_weight_min": -29.2868709564209,
      "activations/layer12_attention_weight_max": 25.97636604309082,
      "activations/layer12_attention_weight_min": -26.76775360107422,
      "activations/layer13_attention_weight_max": 46.683250427246094,
      "activations/layer13_attention_weight_min": -33.80883026123047,
      "activations/layer14_attention_weight_max": 37.33308410644531,
      "activations/layer14_attention_weight_min": -30.99409294128418,
      "activations/layer15_attention_weight_max": 39.30210876464844,
      "activations/layer15_attention_weight_min": -30.502302169799805,
      "activations/layer16_attention_weight_max": 31.51262092590332,
      "activations/layer16_attention_weight_min": -27.752220153808594,
      "activations/layer17_attention_weight_max": 60.35255432128906,
      "activations/layer17_attention_weight_min": -41.884559631347656,
      "activations/layer18_attention_weight_max": 46.9323844909668,
      "activations/layer18_attention_weight_min": -34.87462615966797,
      "activations/layer19_attention_weight_max": 21.367717742919922,
      "activations/layer19_attention_weight_min": -19.527318954467773,
      "activations/layer1_attention_weight_max": 17.006248474121094,
      "activations/layer1_attention_weight_min": -14.064753532409668,
      "activations/layer20_attention_weight_max": 21.5303955078125,
      "activations/layer20_attention_weight_min": -22.308673858642578,
      "activations/layer21_attention_weight_max": 35.08216094970703,
      "activations/layer21_attention_weight_min": -24.24716567993164,
      "activations/layer22_attention_weight_max": 31.535682678222656,
      "activations/layer22_attention_weight_min": -25.296588897705078,
      "activations/layer23_attention_weight_max": 30.07912826538086,
      "activations/layer23_attention_weight_min": -19.545499801635742,
      "activations/layer2_attention_weight_max": 30.222564697265625,
      "activations/layer2_attention_weight_min": -28.87256622314453,
      "activations/layer3_attention_weight_max": 82.88003540039062,
      "activations/layer3_attention_weight_min": -81.96595001220703,
      "activations/layer4_attention_weight_max": 99.64981842041016,
      "activations/layer4_attention_weight_min": -91.94696044921875,
      "activations/layer5_attention_weight_max": 95.90536499023438,
      "activations/layer5_attention_weight_min": -86.39566040039062,
      "activations/layer6_attention_weight_max": 56.69890213012695,
      "activations/layer6_attention_weight_min": -55.034183502197266,
      "activations/layer7_attention_weight_max": 75.00595092773438,
      "activations/layer7_attention_weight_min": -74.14441680908203,
      "activations/layer8_attention_weight_max": 44.05523681640625,
      "activations/layer8_attention_weight_min": -43.49329376220703,
      "activations/layer9_attention_weight_max": 52.93557357788086,
      "activations/layer9_attention_weight_min": -51.6981201171875,
      "epoch": 6.27,
      "learning_rate": 0.00011066363636363635,
      "loss": 2.8208,
      "step": 107950
    },
    {
      "activations/layer0_attention_weight_max": 14.726523399353027,
      "activations/layer0_attention_weight_min": -15.385610580444336,
      "activations/layer10_attention_weight_max": 39.4649543762207,
      "activations/layer10_attention_weight_min": -34.14491271972656,
      "activations/layer11_attention_weight_max": 32.395606994628906,
      "activations/layer11_attention_weight_min": -28.524070739746094,
      "activations/layer12_attention_weight_max": 22.604663848876953,
      "activations/layer12_attention_weight_min": -23.23440170288086,
      "activations/layer13_attention_weight_max": 45.36067199707031,
      "activations/layer13_attention_weight_min": -36.07192611694336,
      "activations/layer14_attention_weight_max": 57.582740783691406,
      "activations/layer14_attention_weight_min": -31.975893020629883,
      "activations/layer15_attention_weight_max": 44.71704864501953,
      "activations/layer15_attention_weight_min": -30.77976417541504,
      "activations/layer16_attention_weight_max": 34.563289642333984,
      "activations/layer16_attention_weight_min": -29.607202529907227,
      "activations/layer17_attention_weight_max": 57.88368606567383,
      "activations/layer17_attention_weight_min": -39.73854064941406,
      "activations/layer18_attention_weight_max": 50.929283142089844,
      "activations/layer18_attention_weight_min": -34.02290344238281,
      "activations/layer19_attention_weight_max": 28.70816421508789,
      "activations/layer19_attention_weight_min": -18.955608367919922,
      "activations/layer1_attention_weight_max": 17.14616584777832,
      "activations/layer1_attention_weight_min": -14.693032264709473,
      "activations/layer20_attention_weight_max": 20.935977935791016,
      "activations/layer20_attention_weight_min": -20.44945526123047,
      "activations/layer21_attention_weight_max": 45.7459602355957,
      "activations/layer21_attention_weight_min": -25.456716537475586,
      "activations/layer22_attention_weight_max": 34.91175842285156,
      "activations/layer22_attention_weight_min": -22.80267906188965,
      "activations/layer23_attention_weight_max": 33.34736633300781,
      "activations/layer23_attention_weight_min": -18.26188850402832,
      "activations/layer2_attention_weight_max": 30.662322998046875,
      "activations/layer2_attention_weight_min": -29.07909393310547,
      "activations/layer3_attention_weight_max": 87.8524169921875,
      "activations/layer3_attention_weight_min": -90.86821746826172,
      "activations/layer4_attention_weight_max": 100.48440551757812,
      "activations/layer4_attention_weight_min": -99.30132293701172,
      "activations/layer5_attention_weight_max": 96.00099182128906,
      "activations/layer5_attention_weight_min": -89.70956420898438,
      "activations/layer6_attention_weight_max": 62.32352066040039,
      "activations/layer6_attention_weight_min": -56.024723052978516,
      "activations/layer7_attention_weight_max": 74.88019561767578,
      "activations/layer7_attention_weight_min": -71.74272155761719,
      "activations/layer8_attention_weight_max": 46.32839584350586,
      "activations/layer8_attention_weight_min": -44.901573181152344,
      "activations/layer9_attention_weight_max": 50.317508697509766,
      "activations/layer9_attention_weight_min": -47.584110260009766,
      "epoch": 6.28,
      "learning_rate": 0.00011064469696969696,
      "loss": 2.8157,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.5957,
      "eval_samples_per_second": 499.554,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.5957,
      "eval_openwebtext_samples_per_second": 499.554,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_wikitext_loss": 3.013671875,
      "eval_wikitext_ppl": 20.36202965350754,
      "eval_wikitext_runtime": 2.0293,
      "eval_wikitext_samples_per_second": 224.704,
      "step": 108000
    },
    {
      "epoch": 6.28,
      "eval_lambada_loss": 2.83203125,
      "eval_lambada_ppl": 16.979916273809664,
      "eval_lambada_runtime": 9.6234,
      "eval_lambada_samples_per_second": 505.956,
      "step": 108000
    },
    {
      "activations/layer0_attention_weight_max": 15.046740531921387,
      "activations/layer0_attention_weight_min": -14.94229793548584,
      "activations/layer10_attention_weight_max": 37.29412078857422,
      "activations/layer10_attention_weight_min": -34.08102035522461,
      "activations/layer11_attention_weight_max": 29.947341918945312,
      "activations/layer11_attention_weight_min": -27.673267364501953,
      "activations/layer12_attention_weight_max": 21.770736694335938,
      "activations/layer12_attention_weight_min": -27.882572174072266,
      "activations/layer13_attention_weight_max": 48.38591003417969,
      "activations/layer13_attention_weight_min": -31.67503547668457,
      "activations/layer14_attention_weight_max": 57.317710876464844,
      "activations/layer14_attention_weight_min": -32.342166900634766,
      "activations/layer15_attention_weight_max": 38.056236267089844,
      "activations/layer15_attention_weight_min": -29.456562042236328,
      "activations/layer16_attention_weight_max": 33.842857360839844,
      "activations/layer16_attention_weight_min": -26.618764877319336,
      "activations/layer17_attention_weight_max": 57.76937484741211,
      "activations/layer17_attention_weight_min": -42.30067825317383,
      "activations/layer18_attention_weight_max": 48.32426452636719,
      "activations/layer18_attention_weight_min": -34.52016067504883,
      "activations/layer19_attention_weight_max": 22.132251739501953,
      "activations/layer19_attention_weight_min": -20.5643253326416,
      "activations/layer1_attention_weight_max": 16.68295669555664,
      "activations/layer1_attention_weight_min": -15.268446922302246,
      "activations/layer20_attention_weight_max": 22.79605484008789,
      "activations/layer20_attention_weight_min": -21.141084671020508,
      "activations/layer21_attention_weight_max": 42.14780044555664,
      "activations/layer21_attention_weight_min": -25.949846267700195,
      "activations/layer22_attention_weight_max": 35.37461471557617,
      "activations/layer22_attention_weight_min": -22.638587951660156,
      "activations/layer23_attention_weight_max": 30.986801147460938,
      "activations/layer23_attention_weight_min": -20.71497344970703,
      "activations/layer2_attention_weight_max": 34.030452728271484,
      "activations/layer2_attention_weight_min": -31.34667205810547,
      "activations/layer3_attention_weight_max": 91.17759704589844,
      "activations/layer3_attention_weight_min": -93.19464111328125,
      "activations/layer4_attention_weight_max": 104.22541809082031,
      "activations/layer4_attention_weight_min": -94.11692810058594,
      "activations/layer5_attention_weight_max": 90.62364959716797,
      "activations/layer5_attention_weight_min": -85.19155883789062,
      "activations/layer6_attention_weight_max": 56.245338439941406,
      "activations/layer6_attention_weight_min": -52.32753372192383,
      "activations/layer7_attention_weight_max": 73.30915069580078,
      "activations/layer7_attention_weight_min": -70.89724731445312,
      "activations/layer8_attention_weight_max": 43.731651306152344,
      "activations/layer8_attention_weight_min": -43.63422393798828,
      "activations/layer9_attention_weight_max": 54.1539306640625,
      "activations/layer9_attention_weight_min": -52.42359161376953,
      "epoch": 6.28,
      "learning_rate": 0.00011062613636363635,
      "loss": 2.8051,
      "step": 108050
    },
    {
      "activations/layer0_attention_weight_max": 16.187124252319336,
      "activations/layer0_attention_weight_min": -15.082843780517578,
      "activations/layer10_attention_weight_max": 36.64171600341797,
      "activations/layer10_attention_weight_min": -32.9925537109375,
      "activations/layer11_attention_weight_max": 32.121131896972656,
      "activations/layer11_attention_weight_min": -27.874221801757812,
      "activations/layer12_attention_weight_max": 24.05547332763672,
      "activations/layer12_attention_weight_min": -23.047155380249023,
      "activations/layer13_attention_weight_max": 41.60609436035156,
      "activations/layer13_attention_weight_min": -33.34822082519531,
      "activations/layer14_attention_weight_max": 35.95144271850586,
      "activations/layer14_attention_weight_min": -29.835124969482422,
      "activations/layer15_attention_weight_max": 37.870574951171875,
      "activations/layer15_attention_weight_min": -28.418901443481445,
      "activations/layer16_attention_weight_max": 37.197174072265625,
      "activations/layer16_attention_weight_min": -27.59027671813965,
      "activations/layer17_attention_weight_max": 56.25552749633789,
      "activations/layer17_attention_weight_min": -40.610721588134766,
      "activations/layer18_attention_weight_max": 47.5786247253418,
      "activations/layer18_attention_weight_min": -32.54401397705078,
      "activations/layer19_attention_weight_max": 20.79142951965332,
      "activations/layer19_attention_weight_min": -18.77797508239746,
      "activations/layer1_attention_weight_max": 17.673084259033203,
      "activations/layer1_attention_weight_min": -13.956123352050781,
      "activations/layer20_attention_weight_max": 22.432016372680664,
      "activations/layer20_attention_weight_min": -18.97087860107422,
      "activations/layer21_attention_weight_max": 32.16588592529297,
      "activations/layer21_attention_weight_min": -23.311540603637695,
      "activations/layer22_attention_weight_max": 33.770320892333984,
      "activations/layer22_attention_weight_min": -23.734588623046875,
      "activations/layer23_attention_weight_max": 30.627262115478516,
      "activations/layer23_attention_weight_min": -18.238990783691406,
      "activations/layer2_attention_weight_max": 30.081180572509766,
      "activations/layer2_attention_weight_min": -28.106887817382812,
      "activations/layer3_attention_weight_max": 77.84028625488281,
      "activations/layer3_attention_weight_min": -80.7370376586914,
      "activations/layer4_attention_weight_max": 99.91817474365234,
      "activations/layer4_attention_weight_min": -92.03765106201172,
      "activations/layer5_attention_weight_max": 91.20040130615234,
      "activations/layer5_attention_weight_min": -84.12559509277344,
      "activations/layer6_attention_weight_max": 54.78617477416992,
      "activations/layer6_attention_weight_min": -51.34928512573242,
      "activations/layer7_attention_weight_max": 76.76895904541016,
      "activations/layer7_attention_weight_min": -68.42015075683594,
      "activations/layer8_attention_weight_max": 47.95256423950195,
      "activations/layer8_attention_weight_min": -39.00630569458008,
      "activations/layer9_attention_weight_max": 64.2491683959961,
      "activations/layer9_attention_weight_min": -48.95874786376953,
      "epoch": 6.28,
      "learning_rate": 0.00011060719696969696,
      "loss": 2.8102,
      "step": 108100
    },
    {
      "activations/layer0_attention_weight_max": 14.807332038879395,
      "activations/layer0_attention_weight_min": -14.882108688354492,
      "activations/layer10_attention_weight_max": 35.97486877441406,
      "activations/layer10_attention_weight_min": -33.551025390625,
      "activations/layer11_attention_weight_max": 29.034854888916016,
      "activations/layer11_attention_weight_min": -28.650634765625,
      "activations/layer12_attention_weight_max": 20.18978500366211,
      "activations/layer12_attention_weight_min": -25.689502716064453,
      "activations/layer13_attention_weight_max": 49.761470794677734,
      "activations/layer13_attention_weight_min": -33.57585906982422,
      "activations/layer14_attention_weight_max": 40.36707305908203,
      "activations/layer14_attention_weight_min": -30.756980895996094,
      "activations/layer15_attention_weight_max": 37.6203498840332,
      "activations/layer15_attention_weight_min": -29.954204559326172,
      "activations/layer16_attention_weight_max": 32.88808059692383,
      "activations/layer16_attention_weight_min": -26.641752243041992,
      "activations/layer17_attention_weight_max": 55.09890365600586,
      "activations/layer17_attention_weight_min": -39.830501556396484,
      "activations/layer18_attention_weight_max": 54.0162353515625,
      "activations/layer18_attention_weight_min": -38.69373321533203,
      "activations/layer19_attention_weight_max": 27.192991256713867,
      "activations/layer19_attention_weight_min": -21.3499755859375,
      "activations/layer1_attention_weight_max": 17.313568115234375,
      "activations/layer1_attention_weight_min": -14.446892738342285,
      "activations/layer20_attention_weight_max": 23.038301467895508,
      "activations/layer20_attention_weight_min": -22.706981658935547,
      "activations/layer21_attention_weight_max": 47.2851448059082,
      "activations/layer21_attention_weight_min": -24.372650146484375,
      "activations/layer22_attention_weight_max": 39.224853515625,
      "activations/layer22_attention_weight_min": -26.843477249145508,
      "activations/layer23_attention_weight_max": 34.866172790527344,
      "activations/layer23_attention_weight_min": -22.634288787841797,
      "activations/layer2_attention_weight_max": 30.461341857910156,
      "activations/layer2_attention_weight_min": -28.13916778564453,
      "activations/layer3_attention_weight_max": 78.75733947753906,
      "activations/layer3_attention_weight_min": -79.26873779296875,
      "activations/layer4_attention_weight_max": 94.35340118408203,
      "activations/layer4_attention_weight_min": -89.77241516113281,
      "activations/layer5_attention_weight_max": 89.64107513427734,
      "activations/layer5_attention_weight_min": -85.00406646728516,
      "activations/layer6_attention_weight_max": 55.14654541015625,
      "activations/layer6_attention_weight_min": -50.25144958496094,
      "activations/layer7_attention_weight_max": 72.3841552734375,
      "activations/layer7_attention_weight_min": -67.84310150146484,
      "activations/layer8_attention_weight_max": 41.48220443725586,
      "activations/layer8_attention_weight_min": -42.92683410644531,
      "activations/layer9_attention_weight_max": 51.691707611083984,
      "activations/layer9_attention_weight_min": -46.882633209228516,
      "epoch": 6.28,
      "learning_rate": 0.00011058825757575756,
      "loss": 2.8426,
      "step": 108150
    },
    {
      "activations/layer0_attention_weight_max": 14.840203285217285,
      "activations/layer0_attention_weight_min": -14.883031845092773,
      "activations/layer10_attention_weight_max": 35.961334228515625,
      "activations/layer10_attention_weight_min": -35.3861083984375,
      "activations/layer11_attention_weight_max": 29.4526424407959,
      "activations/layer11_attention_weight_min": -28.45174217224121,
      "activations/layer12_attention_weight_max": 22.902301788330078,
      "activations/layer12_attention_weight_min": -37.11179733276367,
      "activations/layer13_attention_weight_max": 42.26441955566406,
      "activations/layer13_attention_weight_min": -29.611087799072266,
      "activations/layer14_attention_weight_max": 36.22500991821289,
      "activations/layer14_attention_weight_min": -31.028263092041016,
      "activations/layer15_attention_weight_max": 35.25069808959961,
      "activations/layer15_attention_weight_min": -28.69619369506836,
      "activations/layer16_attention_weight_max": 33.851646423339844,
      "activations/layer16_attention_weight_min": -27.577396392822266,
      "activations/layer17_attention_weight_max": 55.37188720703125,
      "activations/layer17_attention_weight_min": -40.44954299926758,
      "activations/layer18_attention_weight_max": 44.47999954223633,
      "activations/layer18_attention_weight_min": -34.888954162597656,
      "activations/layer19_attention_weight_max": 19.979108810424805,
      "activations/layer19_attention_weight_min": -19.950666427612305,
      "activations/layer1_attention_weight_max": 17.70232582092285,
      "activations/layer1_attention_weight_min": -13.943216323852539,
      "activations/layer20_attention_weight_max": 22.596508026123047,
      "activations/layer20_attention_weight_min": -21.45375633239746,
      "activations/layer21_attention_weight_max": 38.19506072998047,
      "activations/layer21_attention_weight_min": -26.091644287109375,
      "activations/layer22_attention_weight_max": 35.55729293823242,
      "activations/layer22_attention_weight_min": -25.260345458984375,
      "activations/layer23_attention_weight_max": 31.8707332611084,
      "activations/layer23_attention_weight_min": -20.380979537963867,
      "activations/layer2_attention_weight_max": 29.773128509521484,
      "activations/layer2_attention_weight_min": -27.574445724487305,
      "activations/layer3_attention_weight_max": 77.88980102539062,
      "activations/layer3_attention_weight_min": -80.69430541992188,
      "activations/layer4_attention_weight_max": 95.2921371459961,
      "activations/layer4_attention_weight_min": -88.50438690185547,
      "activations/layer5_attention_weight_max": 90.30860900878906,
      "activations/layer5_attention_weight_min": -83.52037048339844,
      "activations/layer6_attention_weight_max": 54.1275749206543,
      "activations/layer6_attention_weight_min": -50.52763748168945,
      "activations/layer7_attention_weight_max": 76.6064224243164,
      "activations/layer7_attention_weight_min": -66.49292755126953,
      "activations/layer8_attention_weight_max": 43.7752571105957,
      "activations/layer8_attention_weight_min": -42.238853454589844,
      "activations/layer9_attention_weight_max": 59.12522506713867,
      "activations/layer9_attention_weight_min": -54.443538665771484,
      "epoch": 6.29,
      "learning_rate": 0.00011056931818181818,
      "loss": 2.8149,
      "step": 108200
    },
    {
      "activations/layer0_attention_weight_max": 15.165834426879883,
      "activations/layer0_attention_weight_min": -15.15321159362793,
      "activations/layer10_attention_weight_max": 39.22188949584961,
      "activations/layer10_attention_weight_min": -38.13945388793945,
      "activations/layer11_attention_weight_max": 30.595489501953125,
      "activations/layer11_attention_weight_min": -29.826047897338867,
      "activations/layer12_attention_weight_max": 25.11166000366211,
      "activations/layer12_attention_weight_min": -26.492429733276367,
      "activations/layer13_attention_weight_max": 46.92348861694336,
      "activations/layer13_attention_weight_min": -32.827117919921875,
      "activations/layer14_attention_weight_max": 47.590545654296875,
      "activations/layer14_attention_weight_min": -31.331727981567383,
      "activations/layer15_attention_weight_max": 37.55424880981445,
      "activations/layer15_attention_weight_min": -28.078731536865234,
      "activations/layer16_attention_weight_max": 31.178186416625977,
      "activations/layer16_attention_weight_min": -24.83930015563965,
      "activations/layer17_attention_weight_max": 52.11341857910156,
      "activations/layer17_attention_weight_min": -40.902137756347656,
      "activations/layer18_attention_weight_max": 45.819129943847656,
      "activations/layer18_attention_weight_min": -34.377342224121094,
      "activations/layer19_attention_weight_max": 21.689054489135742,
      "activations/layer19_attention_weight_min": -22.4013729095459,
      "activations/layer1_attention_weight_max": 17.993507385253906,
      "activations/layer1_attention_weight_min": -15.681035041809082,
      "activations/layer20_attention_weight_max": 20.9049072265625,
      "activations/layer20_attention_weight_min": -23.70684242248535,
      "activations/layer21_attention_weight_max": 36.87918472290039,
      "activations/layer21_attention_weight_min": -26.791837692260742,
      "activations/layer22_attention_weight_max": 34.8277702331543,
      "activations/layer22_attention_weight_min": -26.12762451171875,
      "activations/layer23_attention_weight_max": 32.610172271728516,
      "activations/layer23_attention_weight_min": -20.073549270629883,
      "activations/layer2_attention_weight_max": 33.33384323120117,
      "activations/layer2_attention_weight_min": -33.52983474731445,
      "activations/layer3_attention_weight_max": 88.00479125976562,
      "activations/layer3_attention_weight_min": -95.73332214355469,
      "activations/layer4_attention_weight_max": 106.2255630493164,
      "activations/layer4_attention_weight_min": -103.22137451171875,
      "activations/layer5_attention_weight_max": 98.22407531738281,
      "activations/layer5_attention_weight_min": -91.39800262451172,
      "activations/layer6_attention_weight_max": 58.531455993652344,
      "activations/layer6_attention_weight_min": -54.06888961791992,
      "activations/layer7_attention_weight_max": 75.53370666503906,
      "activations/layer7_attention_weight_min": -74.02249145507812,
      "activations/layer8_attention_weight_max": 46.75956726074219,
      "activations/layer8_attention_weight_min": -45.78407287597656,
      "activations/layer9_attention_weight_max": 59.36153030395508,
      "activations/layer9_attention_weight_min": -52.9672737121582,
      "epoch": 6.29,
      "learning_rate": 0.00011055037878787878,
      "loss": 2.8396,
      "step": 108250
    },
    {
      "activations/layer0_attention_weight_max": 16.40323257446289,
      "activations/layer0_attention_weight_min": -15.236088752746582,
      "activations/layer10_attention_weight_max": 38.17236328125,
      "activations/layer10_attention_weight_min": -36.47077941894531,
      "activations/layer11_attention_weight_max": 31.61383819580078,
      "activations/layer11_attention_weight_min": -35.97382354736328,
      "activations/layer12_attention_weight_max": 20.94271469116211,
      "activations/layer12_attention_weight_min": -24.153398513793945,
      "activations/layer13_attention_weight_max": 52.143070220947266,
      "activations/layer13_attention_weight_min": -35.01941680908203,
      "activations/layer14_attention_weight_max": 38.583919525146484,
      "activations/layer14_attention_weight_min": -29.24648094177246,
      "activations/layer15_attention_weight_max": 38.10720443725586,
      "activations/layer15_attention_weight_min": -29.1923828125,
      "activations/layer16_attention_weight_max": 33.403865814208984,
      "activations/layer16_attention_weight_min": -27.574445724487305,
      "activations/layer17_attention_weight_max": 57.32999801635742,
      "activations/layer17_attention_weight_min": -39.790550231933594,
      "activations/layer18_attention_weight_max": 50.62450408935547,
      "activations/layer18_attention_weight_min": -36.71175003051758,
      "activations/layer19_attention_weight_max": 28.743337631225586,
      "activations/layer19_attention_weight_min": -22.27823257446289,
      "activations/layer1_attention_weight_max": 16.608057022094727,
      "activations/layer1_attention_weight_min": -15.294206619262695,
      "activations/layer20_attention_weight_max": 24.007844924926758,
      "activations/layer20_attention_weight_min": -23.26312828063965,
      "activations/layer21_attention_weight_max": 39.456398010253906,
      "activations/layer21_attention_weight_min": -23.076648712158203,
      "activations/layer22_attention_weight_max": 36.166358947753906,
      "activations/layer22_attention_weight_min": -24.675922393798828,
      "activations/layer23_attention_weight_max": 33.460906982421875,
      "activations/layer23_attention_weight_min": -19.035940170288086,
      "activations/layer2_attention_weight_max": 30.194515228271484,
      "activations/layer2_attention_weight_min": -28.840063095092773,
      "activations/layer3_attention_weight_max": 75.99053192138672,
      "activations/layer3_attention_weight_min": -85.18727111816406,
      "activations/layer4_attention_weight_max": 96.30976104736328,
      "activations/layer4_attention_weight_min": -93.50792694091797,
      "activations/layer5_attention_weight_max": 90.18907928466797,
      "activations/layer5_attention_weight_min": -83.15533447265625,
      "activations/layer6_attention_weight_max": 57.76100540161133,
      "activations/layer6_attention_weight_min": -51.52696990966797,
      "activations/layer7_attention_weight_max": 71.6241683959961,
      "activations/layer7_attention_weight_min": -70.9577407836914,
      "activations/layer8_attention_weight_max": 43.54708480834961,
      "activations/layer8_attention_weight_min": -42.97612762451172,
      "activations/layer9_attention_weight_max": 50.50032043457031,
      "activations/layer9_attention_weight_min": -50.052730560302734,
      "epoch": 6.29,
      "learning_rate": 0.00011053143939393938,
      "loss": 2.826,
      "step": 108300
    },
    {
      "activations/layer0_attention_weight_max": 14.511992454528809,
      "activations/layer0_attention_weight_min": -15.175750732421875,
      "activations/layer10_attention_weight_max": 38.67659378051758,
      "activations/layer10_attention_weight_min": -36.10039138793945,
      "activations/layer11_attention_weight_max": 31.990951538085938,
      "activations/layer11_attention_weight_min": -29.432531356811523,
      "activations/layer12_attention_weight_max": 23.51438331604004,
      "activations/layer12_attention_weight_min": -24.79631233215332,
      "activations/layer13_attention_weight_max": 51.56663131713867,
      "activations/layer13_attention_weight_min": -38.454647064208984,
      "activations/layer14_attention_weight_max": 48.9296760559082,
      "activations/layer14_attention_weight_min": -30.373563766479492,
      "activations/layer15_attention_weight_max": 39.91923141479492,
      "activations/layer15_attention_weight_min": -30.880647659301758,
      "activations/layer16_attention_weight_max": 35.88238525390625,
      "activations/layer16_attention_weight_min": -27.422340393066406,
      "activations/layer17_attention_weight_max": 65.066650390625,
      "activations/layer17_attention_weight_min": -43.357574462890625,
      "activations/layer18_attention_weight_max": 58.29985046386719,
      "activations/layer18_attention_weight_min": -37.115997314453125,
      "activations/layer19_attention_weight_max": 28.77518081665039,
      "activations/layer19_attention_weight_min": -20.427885055541992,
      "activations/layer1_attention_weight_max": 16.411386489868164,
      "activations/layer1_attention_weight_min": -14.049295425415039,
      "activations/layer20_attention_weight_max": 27.681480407714844,
      "activations/layer20_attention_weight_min": -20.72834014892578,
      "activations/layer21_attention_weight_max": 45.554481506347656,
      "activations/layer21_attention_weight_min": -26.111175537109375,
      "activations/layer22_attention_weight_max": 38.990455627441406,
      "activations/layer22_attention_weight_min": -23.584259033203125,
      "activations/layer23_attention_weight_max": 40.3747444152832,
      "activations/layer23_attention_weight_min": -20.334320068359375,
      "activations/layer2_attention_weight_max": 32.22408676147461,
      "activations/layer2_attention_weight_min": -30.478195190429688,
      "activations/layer3_attention_weight_max": 80.08450317382812,
      "activations/layer3_attention_weight_min": -87.7585678100586,
      "activations/layer4_attention_weight_max": 98.8943099975586,
      "activations/layer4_attention_weight_min": -92.07304382324219,
      "activations/layer5_attention_weight_max": 93.01383972167969,
      "activations/layer5_attention_weight_min": -86.33538818359375,
      "activations/layer6_attention_weight_max": 57.82462692260742,
      "activations/layer6_attention_weight_min": -55.648277282714844,
      "activations/layer7_attention_weight_max": 74.43904876708984,
      "activations/layer7_attention_weight_min": -69.29743194580078,
      "activations/layer8_attention_weight_max": 46.37031936645508,
      "activations/layer8_attention_weight_min": -48.964820861816406,
      "activations/layer9_attention_weight_max": 52.82932662963867,
      "activations/layer9_attention_weight_min": -51.33097457885742,
      "epoch": 6.3,
      "learning_rate": 0.0001105125,
      "loss": 2.8293,
      "step": 108350
    },
    {
      "activations/layer0_attention_weight_max": 15.32806396484375,
      "activations/layer0_attention_weight_min": -15.076659202575684,
      "activations/layer10_attention_weight_max": 36.62458419799805,
      "activations/layer10_attention_weight_min": -34.06245803833008,
      "activations/layer11_attention_weight_max": 30.215368270874023,
      "activations/layer11_attention_weight_min": -29.67561912536621,
      "activations/layer12_attention_weight_max": 21.178104400634766,
      "activations/layer12_attention_weight_min": -24.437278747558594,
      "activations/layer13_attention_weight_max": 45.2075080871582,
      "activations/layer13_attention_weight_min": -33.847042083740234,
      "activations/layer14_attention_weight_max": 44.71292495727539,
      "activations/layer14_attention_weight_min": -31.284027099609375,
      "activations/layer15_attention_weight_max": 38.32242202758789,
      "activations/layer15_attention_weight_min": -30.357332229614258,
      "activations/layer16_attention_weight_max": 36.162845611572266,
      "activations/layer16_attention_weight_min": -26.44891929626465,
      "activations/layer17_attention_weight_max": 56.524009704589844,
      "activations/layer17_attention_weight_min": -41.115596771240234,
      "activations/layer18_attention_weight_max": 49.84410095214844,
      "activations/layer18_attention_weight_min": -35.805320739746094,
      "activations/layer19_attention_weight_max": 21.983257293701172,
      "activations/layer19_attention_weight_min": -20.902570724487305,
      "activations/layer1_attention_weight_max": 17.72499656677246,
      "activations/layer1_attention_weight_min": -13.376840591430664,
      "activations/layer20_attention_weight_max": 20.94668960571289,
      "activations/layer20_attention_weight_min": -21.682802200317383,
      "activations/layer21_attention_weight_max": 35.59131622314453,
      "activations/layer21_attention_weight_min": -24.063255310058594,
      "activations/layer22_attention_weight_max": 32.22730255126953,
      "activations/layer22_attention_weight_min": -24.09536361694336,
      "activations/layer23_attention_weight_max": 32.68833923339844,
      "activations/layer23_attention_weight_min": -20.648435592651367,
      "activations/layer2_attention_weight_max": 31.695110321044922,
      "activations/layer2_attention_weight_min": -29.233976364135742,
      "activations/layer3_attention_weight_max": 81.21006774902344,
      "activations/layer3_attention_weight_min": -87.80049133300781,
      "activations/layer4_attention_weight_max": 99.6515884399414,
      "activations/layer4_attention_weight_min": -95.28193664550781,
      "activations/layer5_attention_weight_max": 92.52984619140625,
      "activations/layer5_attention_weight_min": -86.6903305053711,
      "activations/layer6_attention_weight_max": 55.41948699951172,
      "activations/layer6_attention_weight_min": -53.697200775146484,
      "activations/layer7_attention_weight_max": 70.27301788330078,
      "activations/layer7_attention_weight_min": -70.82051086425781,
      "activations/layer8_attention_weight_max": 41.85283279418945,
      "activations/layer8_attention_weight_min": -44.077919006347656,
      "activations/layer9_attention_weight_max": 49.36792755126953,
      "activations/layer9_attention_weight_min": -50.58504867553711,
      "epoch": 6.3,
      "learning_rate": 0.00011049356060606058,
      "loss": 2.8369,
      "step": 108400
    },
    {
      "activations/layer0_attention_weight_max": 14.058241844177246,
      "activations/layer0_attention_weight_min": -15.20749568939209,
      "activations/layer10_attention_weight_max": 37.78541946411133,
      "activations/layer10_attention_weight_min": -35.11580276489258,
      "activations/layer11_attention_weight_max": 32.566551208496094,
      "activations/layer11_attention_weight_min": -28.704849243164062,
      "activations/layer12_attention_weight_max": 22.57241439819336,
      "activations/layer12_attention_weight_min": -23.785072326660156,
      "activations/layer13_attention_weight_max": 48.28220748901367,
      "activations/layer13_attention_weight_min": -35.24119186401367,
      "activations/layer14_attention_weight_max": 51.02607345581055,
      "activations/layer14_attention_weight_min": -32.68400192260742,
      "activations/layer15_attention_weight_max": 41.03835678100586,
      "activations/layer15_attention_weight_min": -30.3659610748291,
      "activations/layer16_attention_weight_max": 36.858943939208984,
      "activations/layer16_attention_weight_min": -28.107484817504883,
      "activations/layer17_attention_weight_max": 57.58939743041992,
      "activations/layer17_attention_weight_min": -48.227699279785156,
      "activations/layer18_attention_weight_max": 52.43839645385742,
      "activations/layer18_attention_weight_min": -41.55553436279297,
      "activations/layer19_attention_weight_max": 26.77577781677246,
      "activations/layer19_attention_weight_min": -21.869382858276367,
      "activations/layer1_attention_weight_max": 17.519655227661133,
      "activations/layer1_attention_weight_min": -18.07520294189453,
      "activations/layer20_attention_weight_max": 23.10204315185547,
      "activations/layer20_attention_weight_min": -21.946758270263672,
      "activations/layer21_attention_weight_max": 38.384098052978516,
      "activations/layer21_attention_weight_min": -24.203567504882812,
      "activations/layer22_attention_weight_max": 34.208274841308594,
      "activations/layer22_attention_weight_min": -25.469106674194336,
      "activations/layer23_attention_weight_max": 36.20931625366211,
      "activations/layer23_attention_weight_min": -19.740739822387695,
      "activations/layer2_attention_weight_max": 32.02742385864258,
      "activations/layer2_attention_weight_min": -29.92718505859375,
      "activations/layer3_attention_weight_max": 85.31752014160156,
      "activations/layer3_attention_weight_min": -88.3480224609375,
      "activations/layer4_attention_weight_max": 98.02729034423828,
      "activations/layer4_attention_weight_min": -95.47820281982422,
      "activations/layer5_attention_weight_max": 93.08291625976562,
      "activations/layer5_attention_weight_min": -89.12200164794922,
      "activations/layer6_attention_weight_max": 56.09199142456055,
      "activations/layer6_attention_weight_min": -57.015594482421875,
      "activations/layer7_attention_weight_max": 77.61688995361328,
      "activations/layer7_attention_weight_min": -73.60092163085938,
      "activations/layer8_attention_weight_max": 47.9592399597168,
      "activations/layer8_attention_weight_min": -43.27790832519531,
      "activations/layer9_attention_weight_max": 64.20171356201172,
      "activations/layer9_attention_weight_min": -50.5980110168457,
      "epoch": 6.3,
      "learning_rate": 0.0001104746212121212,
      "loss": 2.8196,
      "step": 108450
    },
    {
      "activations/layer0_attention_weight_max": 15.611198425292969,
      "activations/layer0_attention_weight_min": -15.272878646850586,
      "activations/layer10_attention_weight_max": 39.2327880859375,
      "activations/layer10_attention_weight_min": -36.78206253051758,
      "activations/layer11_attention_weight_max": 31.717796325683594,
      "activations/layer11_attention_weight_min": -30.644126892089844,
      "activations/layer12_attention_weight_max": 21.356666564941406,
      "activations/layer12_attention_weight_min": -23.419187545776367,
      "activations/layer13_attention_weight_max": 50.71932601928711,
      "activations/layer13_attention_weight_min": -33.23897171020508,
      "activations/layer14_attention_weight_max": 40.23821258544922,
      "activations/layer14_attention_weight_min": -30.39777374267578,
      "activations/layer15_attention_weight_max": 37.57573699951172,
      "activations/layer15_attention_weight_min": -31.24323844909668,
      "activations/layer16_attention_weight_max": 33.73017883300781,
      "activations/layer16_attention_weight_min": -27.79961585998535,
      "activations/layer17_attention_weight_max": 57.60685348510742,
      "activations/layer17_attention_weight_min": -44.46843338012695,
      "activations/layer18_attention_weight_max": 50.533103942871094,
      "activations/layer18_attention_weight_min": -37.60749435424805,
      "activations/layer19_attention_weight_max": 21.597158432006836,
      "activations/layer19_attention_weight_min": -20.80882453918457,
      "activations/layer1_attention_weight_max": 16.487834930419922,
      "activations/layer1_attention_weight_min": -14.1528902053833,
      "activations/layer20_attention_weight_max": 23.568906784057617,
      "activations/layer20_attention_weight_min": -21.449697494506836,
      "activations/layer21_attention_weight_max": 44.76506423950195,
      "activations/layer21_attention_weight_min": -27.887847900390625,
      "activations/layer22_attention_weight_max": 32.31583023071289,
      "activations/layer22_attention_weight_min": -25.931413650512695,
      "activations/layer23_attention_weight_max": 33.565223693847656,
      "activations/layer23_attention_weight_min": -22.46839714050293,
      "activations/layer2_attention_weight_max": 29.478179931640625,
      "activations/layer2_attention_weight_min": -27.268856048583984,
      "activations/layer3_attention_weight_max": 79.33892059326172,
      "activations/layer3_attention_weight_min": -80.45729064941406,
      "activations/layer4_attention_weight_max": 93.75090026855469,
      "activations/layer4_attention_weight_min": -88.76207733154297,
      "activations/layer5_attention_weight_max": 91.9395751953125,
      "activations/layer5_attention_weight_min": -86.01029968261719,
      "activations/layer6_attention_weight_max": 56.52774429321289,
      "activations/layer6_attention_weight_min": -54.06113815307617,
      "activations/layer7_attention_weight_max": 74.94113159179688,
      "activations/layer7_attention_weight_min": -70.19720458984375,
      "activations/layer8_attention_weight_max": 44.77882766723633,
      "activations/layer8_attention_weight_min": -44.2495231628418,
      "activations/layer9_attention_weight_max": 50.38452911376953,
      "activations/layer9_attention_weight_min": -47.5876579284668,
      "epoch": 6.3,
      "learning_rate": 0.00011045568181818181,
      "loss": 2.8275,
      "step": 108500
    },
    {
      "activations/layer0_attention_weight_max": 15.470295906066895,
      "activations/layer0_attention_weight_min": -15.392096519470215,
      "activations/layer10_attention_weight_max": 37.483524322509766,
      "activations/layer10_attention_weight_min": -35.68503952026367,
      "activations/layer11_attention_weight_max": 32.40324401855469,
      "activations/layer11_attention_weight_min": -31.75426483154297,
      "activations/layer12_attention_weight_max": 19.550554275512695,
      "activations/layer12_attention_weight_min": -26.916345596313477,
      "activations/layer13_attention_weight_max": 47.74867630004883,
      "activations/layer13_attention_weight_min": -36.581695556640625,
      "activations/layer14_attention_weight_max": 41.321617126464844,
      "activations/layer14_attention_weight_min": -33.65607833862305,
      "activations/layer15_attention_weight_max": 34.99297332763672,
      "activations/layer15_attention_weight_min": -28.494091033935547,
      "activations/layer16_attention_weight_max": 30.739892959594727,
      "activations/layer16_attention_weight_min": -26.053264617919922,
      "activations/layer17_attention_weight_max": 51.31201934814453,
      "activations/layer17_attention_weight_min": -39.613929748535156,
      "activations/layer18_attention_weight_max": 45.99609375,
      "activations/layer18_attention_weight_min": -32.77834701538086,
      "activations/layer19_attention_weight_max": 19.722444534301758,
      "activations/layer19_attention_weight_min": -18.062549591064453,
      "activations/layer1_attention_weight_max": 17.006755828857422,
      "activations/layer1_attention_weight_min": -15.242199897766113,
      "activations/layer20_attention_weight_max": 20.298057556152344,
      "activations/layer20_attention_weight_min": -20.346221923828125,
      "activations/layer21_attention_weight_max": 36.562355041503906,
      "activations/layer21_attention_weight_min": -23.047677993774414,
      "activations/layer22_attention_weight_max": 32.321983337402344,
      "activations/layer22_attention_weight_min": -23.720911026000977,
      "activations/layer23_attention_weight_max": 33.87117004394531,
      "activations/layer23_attention_weight_min": -21.158050537109375,
      "activations/layer2_attention_weight_max": 32.005332946777344,
      "activations/layer2_attention_weight_min": -30.95572280883789,
      "activations/layer3_attention_weight_max": 85.19742584228516,
      "activations/layer3_attention_weight_min": -88.40119934082031,
      "activations/layer4_attention_weight_max": 103.43096923828125,
      "activations/layer4_attention_weight_min": -99.0871353149414,
      "activations/layer5_attention_weight_max": 95.27526092529297,
      "activations/layer5_attention_weight_min": -88.42791748046875,
      "activations/layer6_attention_weight_max": 56.90431213378906,
      "activations/layer6_attention_weight_min": -53.07350158691406,
      "activations/layer7_attention_weight_max": 74.19078063964844,
      "activations/layer7_attention_weight_min": -74.63871002197266,
      "activations/layer8_attention_weight_max": 44.48820495605469,
      "activations/layer8_attention_weight_min": -44.454837799072266,
      "activations/layer9_attention_weight_max": 58.13765335083008,
      "activations/layer9_attention_weight_min": -52.88790512084961,
      "epoch": 6.31,
      "learning_rate": 0.0001104367424242424,
      "loss": 2.8313,
      "step": 108550
    },
    {
      "activations/layer0_attention_weight_max": 14.635015487670898,
      "activations/layer0_attention_weight_min": -15.571046829223633,
      "activations/layer10_attention_weight_max": 37.22516632080078,
      "activations/layer10_attention_weight_min": -35.40144729614258,
      "activations/layer11_attention_weight_max": 30.08913803100586,
      "activations/layer11_attention_weight_min": -28.782981872558594,
      "activations/layer12_attention_weight_max": 18.628936767578125,
      "activations/layer12_attention_weight_min": -23.26671600341797,
      "activations/layer13_attention_weight_max": 47.28291320800781,
      "activations/layer13_attention_weight_min": -36.8919677734375,
      "activations/layer14_attention_weight_max": 40.71001434326172,
      "activations/layer14_attention_weight_min": -31.06108283996582,
      "activations/layer15_attention_weight_max": 39.35983657836914,
      "activations/layer15_attention_weight_min": -28.61210823059082,
      "activations/layer16_attention_weight_max": 34.66954040527344,
      "activations/layer16_attention_weight_min": -26.233665466308594,
      "activations/layer17_attention_weight_max": 56.0599479675293,
      "activations/layer17_attention_weight_min": -39.78884506225586,
      "activations/layer18_attention_weight_max": 50.08279800415039,
      "activations/layer18_attention_weight_min": -34.10702133178711,
      "activations/layer19_attention_weight_max": 22.268606185913086,
      "activations/layer19_attention_weight_min": -18.73091697692871,
      "activations/layer1_attention_weight_max": 17.85520362854004,
      "activations/layer1_attention_weight_min": -15.51476001739502,
      "activations/layer20_attention_weight_max": 22.96245002746582,
      "activations/layer20_attention_weight_min": -21.513614654541016,
      "activations/layer21_attention_weight_max": 39.91313171386719,
      "activations/layer21_attention_weight_min": -23.100967407226562,
      "activations/layer22_attention_weight_max": 33.317466735839844,
      "activations/layer22_attention_weight_min": -24.134933471679688,
      "activations/layer23_attention_weight_max": 31.962047576904297,
      "activations/layer23_attention_weight_min": -20.330718994140625,
      "activations/layer2_attention_weight_max": 31.718496322631836,
      "activations/layer2_attention_weight_min": -27.942893981933594,
      "activations/layer3_attention_weight_max": 79.9736328125,
      "activations/layer3_attention_weight_min": -81.79192352294922,
      "activations/layer4_attention_weight_max": 98.36864471435547,
      "activations/layer4_attention_weight_min": -90.38398742675781,
      "activations/layer5_attention_weight_max": 91.31975555419922,
      "activations/layer5_attention_weight_min": -85.10485076904297,
      "activations/layer6_attention_weight_max": 56.0715446472168,
      "activations/layer6_attention_weight_min": -52.93767166137695,
      "activations/layer7_attention_weight_max": 74.17123413085938,
      "activations/layer7_attention_weight_min": -68.29118347167969,
      "activations/layer8_attention_weight_max": 45.248130798339844,
      "activations/layer8_attention_weight_min": -45.392555236816406,
      "activations/layer9_attention_weight_max": 49.10609436035156,
      "activations/layer9_attention_weight_min": -48.53107833862305,
      "epoch": 6.31,
      "learning_rate": 0.00011041780303030302,
      "loss": 2.8282,
      "step": 108600
    },
    {
      "activations/layer0_attention_weight_max": 14.740656852722168,
      "activations/layer0_attention_weight_min": -15.061768531799316,
      "activations/layer10_attention_weight_max": 40.135597229003906,
      "activations/layer10_attention_weight_min": -36.037330627441406,
      "activations/layer11_attention_weight_max": 32.349205017089844,
      "activations/layer11_attention_weight_min": -28.437854766845703,
      "activations/layer12_attention_weight_max": 23.065269470214844,
      "activations/layer12_attention_weight_min": -24.98605728149414,
      "activations/layer13_attention_weight_max": 58.42286682128906,
      "activations/layer13_attention_weight_min": -38.54774856567383,
      "activations/layer14_attention_weight_max": 44.528297424316406,
      "activations/layer14_attention_weight_min": -31.865385055541992,
      "activations/layer15_attention_weight_max": 46.44277572631836,
      "activations/layer15_attention_weight_min": -29.390241622924805,
      "activations/layer16_attention_weight_max": 34.52566909790039,
      "activations/layer16_attention_weight_min": -25.87897300720215,
      "activations/layer17_attention_weight_max": 53.10562515258789,
      "activations/layer17_attention_weight_min": -40.70772171020508,
      "activations/layer18_attention_weight_max": 50.104557037353516,
      "activations/layer18_attention_weight_min": -35.649566650390625,
      "activations/layer19_attention_weight_max": 25.169538497924805,
      "activations/layer19_attention_weight_min": -20.582706451416016,
      "activations/layer1_attention_weight_max": 17.162275314331055,
      "activations/layer1_attention_weight_min": -13.949752807617188,
      "activations/layer20_attention_weight_max": 22.25432777404785,
      "activations/layer20_attention_weight_min": -20.983196258544922,
      "activations/layer21_attention_weight_max": 39.701072692871094,
      "activations/layer21_attention_weight_min": -27.051183700561523,
      "activations/layer22_attention_weight_max": 33.50822830200195,
      "activations/layer22_attention_weight_min": -24.810470581054688,
      "activations/layer23_attention_weight_max": 32.49280548095703,
      "activations/layer23_attention_weight_min": -20.312746047973633,
      "activations/layer2_attention_weight_max": 30.273252487182617,
      "activations/layer2_attention_weight_min": -29.644119262695312,
      "activations/layer3_attention_weight_max": 83.71160125732422,
      "activations/layer3_attention_weight_min": -87.1000747680664,
      "activations/layer4_attention_weight_max": 104.56497955322266,
      "activations/layer4_attention_weight_min": -96.14797973632812,
      "activations/layer5_attention_weight_max": 97.45391845703125,
      "activations/layer5_attention_weight_min": -89.72332763671875,
      "activations/layer6_attention_weight_max": 58.1677360534668,
      "activations/layer6_attention_weight_min": -53.03323745727539,
      "activations/layer7_attention_weight_max": 75.17141723632812,
      "activations/layer7_attention_weight_min": -72.36949920654297,
      "activations/layer8_attention_weight_max": 47.62027359008789,
      "activations/layer8_attention_weight_min": -44.93503952026367,
      "activations/layer9_attention_weight_max": 54.487640380859375,
      "activations/layer9_attention_weight_min": -47.0303955078125,
      "epoch": 6.31,
      "learning_rate": 0.00011039886363636363,
      "loss": 2.8151,
      "step": 108650
    },
    {
      "activations/layer0_attention_weight_max": 16.282312393188477,
      "activations/layer0_attention_weight_min": -15.605353355407715,
      "activations/layer10_attention_weight_max": 37.9315185546875,
      "activations/layer10_attention_weight_min": -35.757415771484375,
      "activations/layer11_attention_weight_max": 30.386260986328125,
      "activations/layer11_attention_weight_min": -29.903446197509766,
      "activations/layer12_attention_weight_max": 21.61634063720703,
      "activations/layer12_attention_weight_min": -26.64051628112793,
      "activations/layer13_attention_weight_max": 45.636817932128906,
      "activations/layer13_attention_weight_min": -31.73693084716797,
      "activations/layer14_attention_weight_max": 42.83906173706055,
      "activations/layer14_attention_weight_min": -29.971969604492188,
      "activations/layer15_attention_weight_max": 38.01633834838867,
      "activations/layer15_attention_weight_min": -28.883007049560547,
      "activations/layer16_attention_weight_max": 37.41316604614258,
      "activations/layer16_attention_weight_min": -27.834775924682617,
      "activations/layer17_attention_weight_max": 61.21965408325195,
      "activations/layer17_attention_weight_min": -40.56480407714844,
      "activations/layer18_attention_weight_max": 52.91207504272461,
      "activations/layer18_attention_weight_min": -35.24974822998047,
      "activations/layer19_attention_weight_max": 28.930479049682617,
      "activations/layer19_attention_weight_min": -19.797380447387695,
      "activations/layer1_attention_weight_max": 17.256240844726562,
      "activations/layer1_attention_weight_min": -14.98962688446045,
      "activations/layer20_attention_weight_max": 24.338102340698242,
      "activations/layer20_attention_weight_min": -25.022703170776367,
      "activations/layer21_attention_weight_max": 44.023956298828125,
      "activations/layer21_attention_weight_min": -24.635343551635742,
      "activations/layer22_attention_weight_max": 38.423885345458984,
      "activations/layer22_attention_weight_min": -27.9935302734375,
      "activations/layer23_attention_weight_max": 36.28032684326172,
      "activations/layer23_attention_weight_min": -20.834047317504883,
      "activations/layer2_attention_weight_max": 30.78632926940918,
      "activations/layer2_attention_weight_min": -27.708515167236328,
      "activations/layer3_attention_weight_max": 84.84256744384766,
      "activations/layer3_attention_weight_min": -83.44856262207031,
      "activations/layer4_attention_weight_max": 102.4709243774414,
      "activations/layer4_attention_weight_min": -90.82971954345703,
      "activations/layer5_attention_weight_max": 95.5433349609375,
      "activations/layer5_attention_weight_min": -85.10533905029297,
      "activations/layer6_attention_weight_max": 56.0744743347168,
      "activations/layer6_attention_weight_min": -52.56884002685547,
      "activations/layer7_attention_weight_max": 77.25112915039062,
      "activations/layer7_attention_weight_min": -69.00113677978516,
      "activations/layer8_attention_weight_max": 44.23462677001953,
      "activations/layer8_attention_weight_min": -46.488094329833984,
      "activations/layer9_attention_weight_max": 51.60186767578125,
      "activations/layer9_attention_weight_min": -48.01215744018555,
      "epoch": 6.32,
      "learning_rate": 0.00011037992424242423,
      "loss": 2.8304,
      "step": 108700
    },
    {
      "activations/layer0_attention_weight_max": 15.575247764587402,
      "activations/layer0_attention_weight_min": -15.23698616027832,
      "activations/layer10_attention_weight_max": 39.695556640625,
      "activations/layer10_attention_weight_min": -33.69583511352539,
      "activations/layer11_attention_weight_max": 30.805706024169922,
      "activations/layer11_attention_weight_min": -27.442228317260742,
      "activations/layer12_attention_weight_max": 20.1081600189209,
      "activations/layer12_attention_weight_min": -28.9435977935791,
      "activations/layer13_attention_weight_max": 44.04119110107422,
      "activations/layer13_attention_weight_min": -31.347841262817383,
      "activations/layer14_attention_weight_max": 35.693153381347656,
      "activations/layer14_attention_weight_min": -29.791362762451172,
      "activations/layer15_attention_weight_max": 36.57102966308594,
      "activations/layer15_attention_weight_min": -28.233163833618164,
      "activations/layer16_attention_weight_max": 30.585155487060547,
      "activations/layer16_attention_weight_min": -28.663898468017578,
      "activations/layer17_attention_weight_max": 52.552005767822266,
      "activations/layer17_attention_weight_min": -42.26805114746094,
      "activations/layer18_attention_weight_max": 49.16877365112305,
      "activations/layer18_attention_weight_min": -33.65656661987305,
      "activations/layer19_attention_weight_max": 23.42535972595215,
      "activations/layer19_attention_weight_min": -18.478168487548828,
      "activations/layer1_attention_weight_max": 17.700237274169922,
      "activations/layer1_attention_weight_min": -14.644680976867676,
      "activations/layer20_attention_weight_max": 20.21099853515625,
      "activations/layer20_attention_weight_min": -21.802654266357422,
      "activations/layer21_attention_weight_max": 38.46658706665039,
      "activations/layer21_attention_weight_min": -22.5964298248291,
      "activations/layer22_attention_weight_max": 33.961090087890625,
      "activations/layer22_attention_weight_min": -22.646081924438477,
      "activations/layer23_attention_weight_max": 29.61551284790039,
      "activations/layer23_attention_weight_min": -19.846731185913086,
      "activations/layer2_attention_weight_max": 32.02553176879883,
      "activations/layer2_attention_weight_min": -33.035064697265625,
      "activations/layer3_attention_weight_max": 82.23307037353516,
      "activations/layer3_attention_weight_min": -84.21566772460938,
      "activations/layer4_attention_weight_max": 100.2354965209961,
      "activations/layer4_attention_weight_min": -92.04118347167969,
      "activations/layer5_attention_weight_max": 94.51250457763672,
      "activations/layer5_attention_weight_min": -86.05203247070312,
      "activations/layer6_attention_weight_max": 58.60334777832031,
      "activations/layer6_attention_weight_min": -54.66633605957031,
      "activations/layer7_attention_weight_max": 71.33065032958984,
      "activations/layer7_attention_weight_min": -71.13384246826172,
      "activations/layer8_attention_weight_max": 47.51779556274414,
      "activations/layer8_attention_weight_min": -47.300376892089844,
      "activations/layer9_attention_weight_max": 58.07548904418945,
      "activations/layer9_attention_weight_min": -50.613502502441406,
      "epoch": 6.32,
      "learning_rate": 0.00011036098484848484,
      "loss": 2.8297,
      "step": 108750
    },
    {
      "activations/layer0_attention_weight_max": 15.260276794433594,
      "activations/layer0_attention_weight_min": -15.55798053741455,
      "activations/layer10_attention_weight_max": 37.769439697265625,
      "activations/layer10_attention_weight_min": -35.172035217285156,
      "activations/layer11_attention_weight_max": 28.320205688476562,
      "activations/layer11_attention_weight_min": -28.205562591552734,
      "activations/layer12_attention_weight_max": 22.32009506225586,
      "activations/layer12_attention_weight_min": -25.283796310424805,
      "activations/layer13_attention_weight_max": 50.616310119628906,
      "activations/layer13_attention_weight_min": -35.82537841796875,
      "activations/layer14_attention_weight_max": 51.37203598022461,
      "activations/layer14_attention_weight_min": -30.657058715820312,
      "activations/layer15_attention_weight_max": 37.21829605102539,
      "activations/layer15_attention_weight_min": -27.229101181030273,
      "activations/layer16_attention_weight_max": 31.872758865356445,
      "activations/layer16_attention_weight_min": -25.784088134765625,
      "activations/layer17_attention_weight_max": 54.02460479736328,
      "activations/layer17_attention_weight_min": -37.46004104614258,
      "activations/layer18_attention_weight_max": 44.41531753540039,
      "activations/layer18_attention_weight_min": -34.02893829345703,
      "activations/layer19_attention_weight_max": 21.456439971923828,
      "activations/layer19_attention_weight_min": -19.706235885620117,
      "activations/layer1_attention_weight_max": 17.073335647583008,
      "activations/layer1_attention_weight_min": -13.456707000732422,
      "activations/layer20_attention_weight_max": 20.26500129699707,
      "activations/layer20_attention_weight_min": -19.93724822998047,
      "activations/layer21_attention_weight_max": 47.40317153930664,
      "activations/layer21_attention_weight_min": -25.34727668762207,
      "activations/layer22_attention_weight_max": 31.759870529174805,
      "activations/layer22_attention_weight_min": -22.52170181274414,
      "activations/layer23_attention_weight_max": 30.959365844726562,
      "activations/layer23_attention_weight_min": -21.539031982421875,
      "activations/layer2_attention_weight_max": 32.02671432495117,
      "activations/layer2_attention_weight_min": -30.636795043945312,
      "activations/layer3_attention_weight_max": 82.40612030029297,
      "activations/layer3_attention_weight_min": -86.42829132080078,
      "activations/layer4_attention_weight_max": 99.76834869384766,
      "activations/layer4_attention_weight_min": -95.99091339111328,
      "activations/layer5_attention_weight_max": 96.20098114013672,
      "activations/layer5_attention_weight_min": -89.51030731201172,
      "activations/layer6_attention_weight_max": 54.082332611083984,
      "activations/layer6_attention_weight_min": -50.59184265136719,
      "activations/layer7_attention_weight_max": 71.15434265136719,
      "activations/layer7_attention_weight_min": -69.94926452636719,
      "activations/layer8_attention_weight_max": 42.90312576293945,
      "activations/layer8_attention_weight_min": -42.61030197143555,
      "activations/layer9_attention_weight_max": 51.32495880126953,
      "activations/layer9_attention_weight_min": -47.8907585144043,
      "epoch": 6.32,
      "learning_rate": 0.00011034204545454545,
      "loss": 2.8469,
      "step": 108800
    },
    {
      "activations/layer0_attention_weight_max": 15.078145027160645,
      "activations/layer0_attention_weight_min": -15.235847473144531,
      "activations/layer10_attention_weight_max": 37.078914642333984,
      "activations/layer10_attention_weight_min": -35.13562774658203,
      "activations/layer11_attention_weight_max": 30.58986473083496,
      "activations/layer11_attention_weight_min": -30.660877227783203,
      "activations/layer12_attention_weight_max": 21.58965492248535,
      "activations/layer12_attention_weight_min": -26.72477912902832,
      "activations/layer13_attention_weight_max": 46.870384216308594,
      "activations/layer13_attention_weight_min": -34.19091796875,
      "activations/layer14_attention_weight_max": 42.12797927856445,
      "activations/layer14_attention_weight_min": -30.163162231445312,
      "activations/layer15_attention_weight_max": 39.35576629638672,
      "activations/layer15_attention_weight_min": -30.336776733398438,
      "activations/layer16_attention_weight_max": 35.89743423461914,
      "activations/layer16_attention_weight_min": -27.373807907104492,
      "activations/layer17_attention_weight_max": 56.320335388183594,
      "activations/layer17_attention_weight_min": -40.359981536865234,
      "activations/layer18_attention_weight_max": 48.872047424316406,
      "activations/layer18_attention_weight_min": -34.42998504638672,
      "activations/layer19_attention_weight_max": 25.20285415649414,
      "activations/layer19_attention_weight_min": -19.066585540771484,
      "activations/layer1_attention_weight_max": 16.74422264099121,
      "activations/layer1_attention_weight_min": -16.432836532592773,
      "activations/layer20_attention_weight_max": 23.651208877563477,
      "activations/layer20_attention_weight_min": -20.885889053344727,
      "activations/layer21_attention_weight_max": 44.01287841796875,
      "activations/layer21_attention_weight_min": -23.344053268432617,
      "activations/layer22_attention_weight_max": 37.04747772216797,
      "activations/layer22_attention_weight_min": -25.531972885131836,
      "activations/layer23_attention_weight_max": 38.00346755981445,
      "activations/layer23_attention_weight_min": -20.410484313964844,
      "activations/layer2_attention_weight_max": 30.61091423034668,
      "activations/layer2_attention_weight_min": -28.86147117614746,
      "activations/layer3_attention_weight_max": 79.84783935546875,
      "activations/layer3_attention_weight_min": -83.85120391845703,
      "activations/layer4_attention_weight_max": 98.19939422607422,
      "activations/layer4_attention_weight_min": -93.58104705810547,
      "activations/layer5_attention_weight_max": 96.70777893066406,
      "activations/layer5_attention_weight_min": -86.51614379882812,
      "activations/layer6_attention_weight_max": 57.03211975097656,
      "activations/layer6_attention_weight_min": -52.30100631713867,
      "activations/layer7_attention_weight_max": 74.38948059082031,
      "activations/layer7_attention_weight_min": -71.85324096679688,
      "activations/layer8_attention_weight_max": 43.74082946777344,
      "activations/layer8_attention_weight_min": -44.020668029785156,
      "activations/layer9_attention_weight_max": 51.634742736816406,
      "activations/layer9_attention_weight_min": -47.975807189941406,
      "epoch": 6.32,
      "learning_rate": 0.00011032310606060605,
      "loss": 2.8125,
      "step": 108850
    },
    {
      "activations/layer0_attention_weight_max": 15.449884414672852,
      "activations/layer0_attention_weight_min": -15.592617988586426,
      "activations/layer10_attention_weight_max": 39.13104248046875,
      "activations/layer10_attention_weight_min": -34.97380065917969,
      "activations/layer11_attention_weight_max": 30.346006393432617,
      "activations/layer11_attention_weight_min": -28.27407455444336,
      "activations/layer12_attention_weight_max": 19.996036529541016,
      "activations/layer12_attention_weight_min": -25.870288848876953,
      "activations/layer13_attention_weight_max": 43.7153205871582,
      "activations/layer13_attention_weight_min": -32.02573013305664,
      "activations/layer14_attention_weight_max": 37.12531280517578,
      "activations/layer14_attention_weight_min": -30.700275421142578,
      "activations/layer15_attention_weight_max": 40.89947509765625,
      "activations/layer15_attention_weight_min": -28.93674659729004,
      "activations/layer16_attention_weight_max": 31.23456573486328,
      "activations/layer16_attention_weight_min": -28.139490127563477,
      "activations/layer17_attention_weight_max": 55.7552375793457,
      "activations/layer17_attention_weight_min": -41.43254852294922,
      "activations/layer18_attention_weight_max": 45.61107635498047,
      "activations/layer18_attention_weight_min": -33.47517013549805,
      "activations/layer19_attention_weight_max": 21.39592933654785,
      "activations/layer19_attention_weight_min": -20.502670288085938,
      "activations/layer1_attention_weight_max": 16.515666961669922,
      "activations/layer1_attention_weight_min": -15.089676856994629,
      "activations/layer20_attention_weight_max": 22.48346519470215,
      "activations/layer20_attention_weight_min": -19.917879104614258,
      "activations/layer21_attention_weight_max": 41.77556228637695,
      "activations/layer21_attention_weight_min": -24.68854331970215,
      "activations/layer22_attention_weight_max": 30.19736671447754,
      "activations/layer22_attention_weight_min": -25.1716251373291,
      "activations/layer23_attention_weight_max": 31.991779327392578,
      "activations/layer23_attention_weight_min": -23.001659393310547,
      "activations/layer2_attention_weight_max": 30.167394638061523,
      "activations/layer2_attention_weight_min": -29.467016220092773,
      "activations/layer3_attention_weight_max": 83.27019500732422,
      "activations/layer3_attention_weight_min": -87.0400619506836,
      "activations/layer4_attention_weight_max": 101.69403839111328,
      "activations/layer4_attention_weight_min": -94.32488250732422,
      "activations/layer5_attention_weight_max": 95.01687622070312,
      "activations/layer5_attention_weight_min": -88.30070495605469,
      "activations/layer6_attention_weight_max": 57.22372817993164,
      "activations/layer6_attention_weight_min": -54.21022415161133,
      "activations/layer7_attention_weight_max": 72.24813842773438,
      "activations/layer7_attention_weight_min": -71.65747833251953,
      "activations/layer8_attention_weight_max": 46.459320068359375,
      "activations/layer8_attention_weight_min": -48.34464645385742,
      "activations/layer9_attention_weight_max": 53.01003646850586,
      "activations/layer9_attention_weight_min": -46.36947250366211,
      "epoch": 6.33,
      "learning_rate": 0.00011030416666666667,
      "loss": 2.8369,
      "step": 108900
    },
    {
      "activations/layer0_attention_weight_max": 14.264421463012695,
      "activations/layer0_attention_weight_min": -15.707795143127441,
      "activations/layer10_attention_weight_max": 34.44781494140625,
      "activations/layer10_attention_weight_min": -34.839378356933594,
      "activations/layer11_attention_weight_max": 27.096954345703125,
      "activations/layer11_attention_weight_min": -29.076927185058594,
      "activations/layer12_attention_weight_max": 20.329919815063477,
      "activations/layer12_attention_weight_min": -26.96528434753418,
      "activations/layer13_attention_weight_max": 44.84849166870117,
      "activations/layer13_attention_weight_min": -31.27533531188965,
      "activations/layer14_attention_weight_max": 46.15013122558594,
      "activations/layer14_attention_weight_min": -30.500802993774414,
      "activations/layer15_attention_weight_max": 37.376380920410156,
      "activations/layer15_attention_weight_min": -28.53491973876953,
      "activations/layer16_attention_weight_max": 30.830520629882812,
      "activations/layer16_attention_weight_min": -26.139421463012695,
      "activations/layer17_attention_weight_max": 51.37775802612305,
      "activations/layer17_attention_weight_min": -40.8351936340332,
      "activations/layer18_attention_weight_max": 48.18083953857422,
      "activations/layer18_attention_weight_min": -35.3752555847168,
      "activations/layer19_attention_weight_max": 23.15623664855957,
      "activations/layer19_attention_weight_min": -21.48025131225586,
      "activations/layer1_attention_weight_max": 18.07815933227539,
      "activations/layer1_attention_weight_min": -15.917495727539062,
      "activations/layer20_attention_weight_max": 23.402942657470703,
      "activations/layer20_attention_weight_min": -24.868444442749023,
      "activations/layer21_attention_weight_max": 36.707252502441406,
      "activations/layer21_attention_weight_min": -26.36410903930664,
      "activations/layer22_attention_weight_max": 29.304725646972656,
      "activations/layer22_attention_weight_min": -26.03163719177246,
      "activations/layer23_attention_weight_max": 33.00523376464844,
      "activations/layer23_attention_weight_min": -22.743019104003906,
      "activations/layer2_attention_weight_max": 31.164031982421875,
      "activations/layer2_attention_weight_min": -31.22534942626953,
      "activations/layer3_attention_weight_max": 83.4289321899414,
      "activations/layer3_attention_weight_min": -84.57469177246094,
      "activations/layer4_attention_weight_max": 97.89791107177734,
      "activations/layer4_attention_weight_min": -93.1889877319336,
      "activations/layer5_attention_weight_max": 90.51116180419922,
      "activations/layer5_attention_weight_min": -88.18658447265625,
      "activations/layer6_attention_weight_max": 54.44600296020508,
      "activations/layer6_attention_weight_min": -51.74113464355469,
      "activations/layer7_attention_weight_max": 71.4548110961914,
      "activations/layer7_attention_weight_min": -66.72937774658203,
      "activations/layer8_attention_weight_max": 41.28987121582031,
      "activations/layer8_attention_weight_min": -42.519657135009766,
      "activations/layer9_attention_weight_max": 47.189048767089844,
      "activations/layer9_attention_weight_min": -52.07357406616211,
      "epoch": 6.33,
      "learning_rate": 0.00011028522727272725,
      "loss": 2.8244,
      "step": 108950
    },
    {
      "activations/layer0_attention_weight_max": 14.623788833618164,
      "activations/layer0_attention_weight_min": -15.358744621276855,
      "activations/layer10_attention_weight_max": 38.854644775390625,
      "activations/layer10_attention_weight_min": -34.85216522216797,
      "activations/layer11_attention_weight_max": 30.84249496459961,
      "activations/layer11_attention_weight_min": -30.101776123046875,
      "activations/layer12_attention_weight_max": 22.70719337463379,
      "activations/layer12_attention_weight_min": -24.689903259277344,
      "activations/layer13_attention_weight_max": 46.18844985961914,
      "activations/layer13_attention_weight_min": -34.7929801940918,
      "activations/layer14_attention_weight_max": 43.951114654541016,
      "activations/layer14_attention_weight_min": -29.675628662109375,
      "activations/layer15_attention_weight_max": 37.32724380493164,
      "activations/layer15_attention_weight_min": -30.349973678588867,
      "activations/layer16_attention_weight_max": 33.73935317993164,
      "activations/layer16_attention_weight_min": -29.905977249145508,
      "activations/layer17_attention_weight_max": 54.57530975341797,
      "activations/layer17_attention_weight_min": -41.02607727050781,
      "activations/layer18_attention_weight_max": 45.73206329345703,
      "activations/layer18_attention_weight_min": -37.34601593017578,
      "activations/layer19_attention_weight_max": 21.222637176513672,
      "activations/layer19_attention_weight_min": -21.082292556762695,
      "activations/layer1_attention_weight_max": 16.207767486572266,
      "activations/layer1_attention_weight_min": -15.200533866882324,
      "activations/layer20_attention_weight_max": 23.657760620117188,
      "activations/layer20_attention_weight_min": -22.442434310913086,
      "activations/layer21_attention_weight_max": 39.664329528808594,
      "activations/layer21_attention_weight_min": -26.270381927490234,
      "activations/layer22_attention_weight_max": 36.86806106567383,
      "activations/layer22_attention_weight_min": -25.330469131469727,
      "activations/layer23_attention_weight_max": 32.83848571777344,
      "activations/layer23_attention_weight_min": -21.138212203979492,
      "activations/layer2_attention_weight_max": 31.724491119384766,
      "activations/layer2_attention_weight_min": -29.25067710876465,
      "activations/layer3_attention_weight_max": 83.8038558959961,
      "activations/layer3_attention_weight_min": -89.7083740234375,
      "activations/layer4_attention_weight_max": 96.8371353149414,
      "activations/layer4_attention_weight_min": -91.91958618164062,
      "activations/layer5_attention_weight_max": 91.6478500366211,
      "activations/layer5_attention_weight_min": -83.18844604492188,
      "activations/layer6_attention_weight_max": 58.51211929321289,
      "activations/layer6_attention_weight_min": -51.03539276123047,
      "activations/layer7_attention_weight_max": 70.56272888183594,
      "activations/layer7_attention_weight_min": -67.67535400390625,
      "activations/layer8_attention_weight_max": 42.835594177246094,
      "activations/layer8_attention_weight_min": -42.59284591674805,
      "activations/layer9_attention_weight_max": 50.94095993041992,
      "activations/layer9_attention_weight_min": -49.31841278076172,
      "epoch": 6.33,
      "learning_rate": 0.00011026628787878787,
      "loss": 2.827,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_loss": 2.78125,
      "eval_runtime": 8.5529,
      "eval_samples_per_second": 502.054,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_openwebtext_loss": 2.78125,
      "eval_openwebtext_ppl": 16.139182322446413,
      "eval_openwebtext_runtime": 8.5529,
      "eval_openwebtext_samples_per_second": 502.054,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_wikitext_loss": 3.017578125,
      "eval_wikitext_ppl": 20.44172438427552,
      "eval_wikitext_runtime": 2.0211,
      "eval_wikitext_samples_per_second": 225.624,
      "step": 109000
    },
    {
      "epoch": 6.33,
      "eval_lambada_loss": 2.78515625,
      "eval_lambada_ppl": 16.202349296067883,
      "eval_lambada_runtime": 9.6112,
      "eval_lambada_samples_per_second": 506.597,
      "step": 109000
    },
    {
      "activations/layer0_attention_weight_max": 14.697798728942871,
      "activations/layer0_attention_weight_min": -15.354618072509766,
      "activations/layer10_attention_weight_max": 37.40833282470703,
      "activations/layer10_attention_weight_min": -33.923553466796875,
      "activations/layer11_attention_weight_max": 31.265195846557617,
      "activations/layer11_attention_weight_min": -29.341167449951172,
      "activations/layer12_attention_weight_max": 27.335554122924805,
      "activations/layer12_attention_weight_min": -24.342161178588867,
      "activations/layer13_attention_weight_max": 40.59823226928711,
      "activations/layer13_attention_weight_min": -34.45426559448242,
      "activations/layer14_attention_weight_max": 42.360740661621094,
      "activations/layer14_attention_weight_min": -29.95632553100586,
      "activations/layer15_attention_weight_max": 35.8539924621582,
      "activations/layer15_attention_weight_min": -29.122060775756836,
      "activations/layer16_attention_weight_max": 30.193538665771484,
      "activations/layer16_attention_weight_min": -26.957109451293945,
      "activations/layer17_attention_weight_max": 53.62532043457031,
      "activations/layer17_attention_weight_min": -39.52947998046875,
      "activations/layer18_attention_weight_max": 43.82954025268555,
      "activations/layer18_attention_weight_min": -32.265708923339844,
      "activations/layer19_attention_weight_max": 22.973339080810547,
      "activations/layer19_attention_weight_min": -21.89635467529297,
      "activations/layer1_attention_weight_max": 17.240455627441406,
      "activations/layer1_attention_weight_min": -13.723739624023438,
      "activations/layer20_attention_weight_max": 18.496559143066406,
      "activations/layer20_attention_weight_min": -27.700870513916016,
      "activations/layer21_attention_weight_max": 33.96728515625,
      "activations/layer21_attention_weight_min": -23.95151710510254,
      "activations/layer22_attention_weight_max": 28.50847053527832,
      "activations/layer22_attention_weight_min": -21.530561447143555,
      "activations/layer23_attention_weight_max": 29.798871994018555,
      "activations/layer23_attention_weight_min": -18.817466735839844,
      "activations/layer2_attention_weight_max": 30.924230575561523,
      "activations/layer2_attention_weight_min": -30.584630966186523,
      "activations/layer3_attention_weight_max": 83.07605743408203,
      "activations/layer3_attention_weight_min": -89.1063232421875,
      "activations/layer4_attention_weight_max": 100.65187072753906,
      "activations/layer4_attention_weight_min": -96.44036102294922,
      "activations/layer5_attention_weight_max": 96.89595031738281,
      "activations/layer5_attention_weight_min": -90.5386962890625,
      "activations/layer6_attention_weight_max": 60.102195739746094,
      "activations/layer6_attention_weight_min": -52.93463897705078,
      "activations/layer7_attention_weight_max": 77.51972198486328,
      "activations/layer7_attention_weight_min": -69.77436828613281,
      "activations/layer8_attention_weight_max": 45.49184036254883,
      "activations/layer8_attention_weight_min": -45.56553268432617,
      "activations/layer9_attention_weight_max": 49.230648040771484,
      "activations/layer9_attention_weight_min": -48.95612716674805,
      "epoch": 6.34,
      "learning_rate": 0.00011024734848484848,
      "loss": 2.829,
      "step": 109050
    },
    {
      "activations/layer0_attention_weight_max": 15.179560661315918,
      "activations/layer0_attention_weight_min": -14.844050407409668,
      "activations/layer10_attention_weight_max": 41.071868896484375,
      "activations/layer10_attention_weight_min": -35.840362548828125,
      "activations/layer11_attention_weight_max": 34.4005126953125,
      "activations/layer11_attention_weight_min": -29.633739471435547,
      "activations/layer12_attention_weight_max": 20.023494720458984,
      "activations/layer12_attention_weight_min": -25.688430786132812,
      "activations/layer13_attention_weight_max": 49.32181167602539,
      "activations/layer13_attention_weight_min": -34.84713363647461,
      "activations/layer14_attention_weight_max": 51.755367279052734,
      "activations/layer14_attention_weight_min": -30.517183303833008,
      "activations/layer15_attention_weight_max": 39.33286666870117,
      "activations/layer15_attention_weight_min": -29.54668617248535,
      "activations/layer16_attention_weight_max": 37.728492736816406,
      "activations/layer16_attention_weight_min": -27.773218154907227,
      "activations/layer17_attention_weight_max": 57.47571563720703,
      "activations/layer17_attention_weight_min": -39.5692253112793,
      "activations/layer18_attention_weight_max": 47.57176208496094,
      "activations/layer18_attention_weight_min": -34.789512634277344,
      "activations/layer19_attention_weight_max": 25.367137908935547,
      "activations/layer19_attention_weight_min": -20.537914276123047,
      "activations/layer1_attention_weight_max": 17.234956741333008,
      "activations/layer1_attention_weight_min": -15.690262794494629,
      "activations/layer20_attention_weight_max": 24.682798385620117,
      "activations/layer20_attention_weight_min": -20.803979873657227,
      "activations/layer21_attention_weight_max": 48.595726013183594,
      "activations/layer21_attention_weight_min": -24.220815658569336,
      "activations/layer22_attention_weight_max": 33.65206527709961,
      "activations/layer22_attention_weight_min": -26.665355682373047,
      "activations/layer23_attention_weight_max": 28.37137222290039,
      "activations/layer23_attention_weight_min": -18.331924438476562,
      "activations/layer2_attention_weight_max": 34.731021881103516,
      "activations/layer2_attention_weight_min": -31.805814743041992,
      "activations/layer3_attention_weight_max": 91.71883392333984,
      "activations/layer3_attention_weight_min": -90.81428527832031,
      "activations/layer4_attention_weight_max": 110.54353332519531,
      "activations/layer4_attention_weight_min": -98.3599624633789,
      "activations/layer5_attention_weight_max": 107.17901611328125,
      "activations/layer5_attention_weight_min": -92.6282958984375,
      "activations/layer6_attention_weight_max": 60.285709381103516,
      "activations/layer6_attention_weight_min": -56.579002380371094,
      "activations/layer7_attention_weight_max": 78.43487548828125,
      "activations/layer7_attention_weight_min": -73.8755111694336,
      "activations/layer8_attention_weight_max": 52.641876220703125,
      "activations/layer8_attention_weight_min": -47.26050567626953,
      "activations/layer9_attention_weight_max": 62.1666374206543,
      "activations/layer9_attention_weight_min": -52.44548416137695,
      "epoch": 6.34,
      "learning_rate": 0.00011022840909090907,
      "loss": 2.8408,
      "step": 109100
    },
    {
      "activations/layer0_attention_weight_max": 15.51113224029541,
      "activations/layer0_attention_weight_min": -15.23995304107666,
      "activations/layer10_attention_weight_max": 38.08757400512695,
      "activations/layer10_attention_weight_min": -33.64398956298828,
      "activations/layer11_attention_weight_max": 32.120849609375,
      "activations/layer11_attention_weight_min": -28.519271850585938,
      "activations/layer12_attention_weight_max": 25.26068115234375,
      "activations/layer12_attention_weight_min": -23.163944244384766,
      "activations/layer13_attention_weight_max": 40.753990173339844,
      "activations/layer13_attention_weight_min": -34.300437927246094,
      "activations/layer14_attention_weight_max": 38.3109245300293,
      "activations/layer14_attention_weight_min": -30.022390365600586,
      "activations/layer15_attention_weight_max": 40.66669845581055,
      "activations/layer15_attention_weight_min": -31.837081909179688,
      "activations/layer16_attention_weight_max": 35.11385726928711,
      "activations/layer16_attention_weight_min": -29.120948791503906,
      "activations/layer17_attention_weight_max": 55.84584045410156,
      "activations/layer17_attention_weight_min": -42.33916473388672,
      "activations/layer18_attention_weight_max": 55.05594253540039,
      "activations/layer18_attention_weight_min": -35.87063217163086,
      "activations/layer19_attention_weight_max": 27.05002212524414,
      "activations/layer19_attention_weight_min": -19.684457778930664,
      "activations/layer1_attention_weight_max": 16.9267520904541,
      "activations/layer1_attention_weight_min": -17.934988021850586,
      "activations/layer20_attention_weight_max": 23.641643524169922,
      "activations/layer20_attention_weight_min": -24.2192440032959,
      "activations/layer21_attention_weight_max": 42.25457763671875,
      "activations/layer21_attention_weight_min": -26.659515380859375,
      "activations/layer22_attention_weight_max": 33.09935760498047,
      "activations/layer22_attention_weight_min": -25.39278793334961,
      "activations/layer23_attention_weight_max": 37.064598083496094,
      "activations/layer23_attention_weight_min": -19.7687931060791,
      "activations/layer2_attention_weight_max": 31.381263732910156,
      "activations/layer2_attention_weight_min": -29.21095085144043,
      "activations/layer3_attention_weight_max": 79.53546905517578,
      "activations/layer3_attention_weight_min": -85.84783172607422,
      "activations/layer4_attention_weight_max": 97.52632904052734,
      "activations/layer4_attention_weight_min": -91.11769104003906,
      "activations/layer5_attention_weight_max": 96.15780639648438,
      "activations/layer5_attention_weight_min": -87.81275939941406,
      "activations/layer6_attention_weight_max": 57.8271369934082,
      "activations/layer6_attention_weight_min": -53.7299690246582,
      "activations/layer7_attention_weight_max": 83.18521881103516,
      "activations/layer7_attention_weight_min": -67.78718566894531,
      "activations/layer8_attention_weight_max": 46.04023361206055,
      "activations/layer8_attention_weight_min": -42.873905181884766,
      "activations/layer9_attention_weight_max": 53.93986129760742,
      "activations/layer9_attention_weight_min": -49.73679733276367,
      "epoch": 6.34,
      "learning_rate": 0.00011020946969696969,
      "loss": 2.836,
      "step": 109150
    },
    {
      "activations/layer0_attention_weight_max": 15.041156768798828,
      "activations/layer0_attention_weight_min": -15.552288055419922,
      "activations/layer10_attention_weight_max": 37.674076080322266,
      "activations/layer10_attention_weight_min": -36.58576202392578,
      "activations/layer11_attention_weight_max": 31.453323364257812,
      "activations/layer11_attention_weight_min": -28.054235458374023,
      "activations/layer12_attention_weight_max": 21.174850463867188,
      "activations/layer12_attention_weight_min": -24.111865997314453,
      "activations/layer13_attention_weight_max": 50.95148468017578,
      "activations/layer13_attention_weight_min": -35.74460983276367,
      "activations/layer14_attention_weight_max": 46.68166732788086,
      "activations/layer14_attention_weight_min": -30.610027313232422,
      "activations/layer15_attention_weight_max": 39.29439163208008,
      "activations/layer15_attention_weight_min": -30.184310913085938,
      "activations/layer16_attention_weight_max": 34.525081634521484,
      "activations/layer16_attention_weight_min": -27.58579444885254,
      "activations/layer17_attention_weight_max": 55.65970230102539,
      "activations/layer17_attention_weight_min": -39.462158203125,
      "activations/layer18_attention_weight_max": 49.25994110107422,
      "activations/layer18_attention_weight_min": -33.66215896606445,
      "activations/layer19_attention_weight_max": 23.45490074157715,
      "activations/layer19_attention_weight_min": -19.627168655395508,
      "activations/layer1_attention_weight_max": 16.706995010375977,
      "activations/layer1_attention_weight_min": -14.875300407409668,
      "activations/layer20_attention_weight_max": 21.84129524230957,
      "activations/layer20_attention_weight_min": -20.724044799804688,
      "activations/layer21_attention_weight_max": 41.223304748535156,
      "activations/layer21_attention_weight_min": -24.653640747070312,
      "activations/layer22_attention_weight_max": 35.03622817993164,
      "activations/layer22_attention_weight_min": -23.70532989501953,
      "activations/layer23_attention_weight_max": 32.71929168701172,
      "activations/layer23_attention_weight_min": -19.742660522460938,
      "activations/layer2_attention_weight_max": 30.765151977539062,
      "activations/layer2_attention_weight_min": -28.075063705444336,
      "activations/layer3_attention_weight_max": 82.88951873779297,
      "activations/layer3_attention_weight_min": -83.27525329589844,
      "activations/layer4_attention_weight_max": 101.1631088256836,
      "activations/layer4_attention_weight_min": -93.68712615966797,
      "activations/layer5_attention_weight_max": 97.39981079101562,
      "activations/layer5_attention_weight_min": -93.6130599975586,
      "activations/layer6_attention_weight_max": 58.92034912109375,
      "activations/layer6_attention_weight_min": -57.08037185668945,
      "activations/layer7_attention_weight_max": 75.67721557617188,
      "activations/layer7_attention_weight_min": -73.45740509033203,
      "activations/layer8_attention_weight_max": 47.59260559082031,
      "activations/layer8_attention_weight_min": -45.51995086669922,
      "activations/layer9_attention_weight_max": 55.906005859375,
      "activations/layer9_attention_weight_min": -51.97038650512695,
      "epoch": 6.35,
      "learning_rate": 0.0001101905303030303,
      "loss": 2.8274,
      "step": 109200
    },
    {
      "activations/layer0_attention_weight_max": 15.160196304321289,
      "activations/layer0_attention_weight_min": -15.259188652038574,
      "activations/layer10_attention_weight_max": 39.989593505859375,
      "activations/layer10_attention_weight_min": -36.190460205078125,
      "activations/layer11_attention_weight_max": 32.980308532714844,
      "activations/layer11_attention_weight_min": -31.054698944091797,
      "activations/layer12_attention_weight_max": 19.27651023864746,
      "activations/layer12_attention_weight_min": -25.98289680480957,
      "activations/layer13_attention_weight_max": 41.62310791015625,
      "activations/layer13_attention_weight_min": -32.758731842041016,
      "activations/layer14_attention_weight_max": 36.4600944519043,
      "activations/layer14_attention_weight_min": -29.43463897705078,
      "activations/layer15_attention_weight_max": 38.73694610595703,
      "activations/layer15_attention_weight_min": -29.044218063354492,
      "activations/layer16_attention_weight_max": 34.10931396484375,
      "activations/layer16_attention_weight_min": -26.21004867553711,
      "activations/layer17_attention_weight_max": 53.90340805053711,
      "activations/layer17_attention_weight_min": -40.34351348876953,
      "activations/layer18_attention_weight_max": 47.170658111572266,
      "activations/layer18_attention_weight_min": -34.80317306518555,
      "activations/layer19_attention_weight_max": 25.558210372924805,
      "activations/layer19_attention_weight_min": -20.46272850036621,
      "activations/layer1_attention_weight_max": 17.863834381103516,
      "activations/layer1_attention_weight_min": -15.576175689697266,
      "activations/layer20_attention_weight_max": 20.592823028564453,
      "activations/layer20_attention_weight_min": -24.848594665527344,
      "activations/layer21_attention_weight_max": 36.03236770629883,
      "activations/layer21_attention_weight_min": -24.710372924804688,
      "activations/layer22_attention_weight_max": 31.016693115234375,
      "activations/layer22_attention_weight_min": -25.961912155151367,
      "activations/layer23_attention_weight_max": 30.18195343017578,
      "activations/layer23_attention_weight_min": -21.31927490234375,
      "activations/layer2_attention_weight_max": 30.318546295166016,
      "activations/layer2_attention_weight_min": -30.27973747253418,
      "activations/layer3_attention_weight_max": 80.13372802734375,
      "activations/layer3_attention_weight_min": -84.66736602783203,
      "activations/layer4_attention_weight_max": 99.25634002685547,
      "activations/layer4_attention_weight_min": -93.30701446533203,
      "activations/layer5_attention_weight_max": 93.22430419921875,
      "activations/layer5_attention_weight_min": -86.43929290771484,
      "activations/layer6_attention_weight_max": 54.537193298339844,
      "activations/layer6_attention_weight_min": -53.29694747924805,
      "activations/layer7_attention_weight_max": 71.01992797851562,
      "activations/layer7_attention_weight_min": -67.62687683105469,
      "activations/layer8_attention_weight_max": 43.09195327758789,
      "activations/layer8_attention_weight_min": -44.45778274536133,
      "activations/layer9_attention_weight_max": 51.588802337646484,
      "activations/layer9_attention_weight_min": -51.32192611694336,
      "epoch": 6.35,
      "learning_rate": 0.00011017159090909089,
      "loss": 2.8417,
      "step": 109250
    },
    {
      "activations/layer0_attention_weight_max": 14.710396766662598,
      "activations/layer0_attention_weight_min": -14.747380256652832,
      "activations/layer10_attention_weight_max": 36.70077896118164,
      "activations/layer10_attention_weight_min": -34.86830520629883,
      "activations/layer11_attention_weight_max": 27.999797821044922,
      "activations/layer11_attention_weight_min": -30.14105796813965,
      "activations/layer12_attention_weight_max": 19.89031410217285,
      "activations/layer12_attention_weight_min": -25.52825927734375,
      "activations/layer13_attention_weight_max": 43.27321243286133,
      "activations/layer13_attention_weight_min": -36.150779724121094,
      "activations/layer14_attention_weight_max": 41.59322738647461,
      "activations/layer14_attention_weight_min": -30.3632869720459,
      "activations/layer15_attention_weight_max": 35.16839599609375,
      "activations/layer15_attention_weight_min": -27.891382217407227,
      "activations/layer16_attention_weight_max": 32.971736907958984,
      "activations/layer16_attention_weight_min": -26.48399543762207,
      "activations/layer17_attention_weight_max": 54.07770919799805,
      "activations/layer17_attention_weight_min": -40.59077835083008,
      "activations/layer18_attention_weight_max": 45.34990692138672,
      "activations/layer18_attention_weight_min": -34.328590393066406,
      "activations/layer19_attention_weight_max": 22.617361068725586,
      "activations/layer19_attention_weight_min": -20.222301483154297,
      "activations/layer1_attention_weight_max": 16.545318603515625,
      "activations/layer1_attention_weight_min": -14.505631446838379,
      "activations/layer20_attention_weight_max": 22.810749053955078,
      "activations/layer20_attention_weight_min": -23.9888858795166,
      "activations/layer21_attention_weight_max": 38.70216751098633,
      "activations/layer21_attention_weight_min": -22.831954956054688,
      "activations/layer22_attention_weight_max": 31.13273811340332,
      "activations/layer22_attention_weight_min": -25.80948257446289,
      "activations/layer23_attention_weight_max": 30.933225631713867,
      "activations/layer23_attention_weight_min": -20.34209442138672,
      "activations/layer2_attention_weight_max": 28.767948150634766,
      "activations/layer2_attention_weight_min": -28.056598663330078,
      "activations/layer3_attention_weight_max": 77.54820251464844,
      "activations/layer3_attention_weight_min": -82.96829986572266,
      "activations/layer4_attention_weight_max": 91.78423309326172,
      "activations/layer4_attention_weight_min": -86.91567993164062,
      "activations/layer5_attention_weight_max": 87.00413513183594,
      "activations/layer5_attention_weight_min": -80.87751007080078,
      "activations/layer6_attention_weight_max": 52.919315338134766,
      "activations/layer6_attention_weight_min": -50.2359733581543,
      "activations/layer7_attention_weight_max": 69.96563720703125,
      "activations/layer7_attention_weight_min": -68.52948760986328,
      "activations/layer8_attention_weight_max": 39.952823638916016,
      "activations/layer8_attention_weight_min": -41.30573272705078,
      "activations/layer9_attention_weight_max": 47.4171257019043,
      "activations/layer9_attention_weight_min": -49.225364685058594,
      "epoch": 6.35,
      "learning_rate": 0.0001101526515151515,
      "loss": 2.839,
      "step": 109300
    },
    {
      "activations/layer0_attention_weight_max": 14.985060691833496,
      "activations/layer0_attention_weight_min": -14.853583335876465,
      "activations/layer10_attention_weight_max": 36.550010681152344,
      "activations/layer10_attention_weight_min": -34.13142395019531,
      "activations/layer11_attention_weight_max": 29.86288833618164,
      "activations/layer11_attention_weight_min": -28.522624969482422,
      "activations/layer12_attention_weight_max": 19.301259994506836,
      "activations/layer12_attention_weight_min": -27.97185707092285,
      "activations/layer13_attention_weight_max": 45.55928421020508,
      "activations/layer13_attention_weight_min": -36.76310348510742,
      "activations/layer14_attention_weight_max": 41.90107727050781,
      "activations/layer14_attention_weight_min": -32.59037399291992,
      "activations/layer15_attention_weight_max": 38.3972053527832,
      "activations/layer15_attention_weight_min": -30.242446899414062,
      "activations/layer16_attention_weight_max": 34.43948745727539,
      "activations/layer16_attention_weight_min": -27.003488540649414,
      "activations/layer17_attention_weight_max": 52.2426872253418,
      "activations/layer17_attention_weight_min": -42.2545051574707,
      "activations/layer18_attention_weight_max": 47.35131072998047,
      "activations/layer18_attention_weight_min": -36.35715103149414,
      "activations/layer19_attention_weight_max": 22.849618911743164,
      "activations/layer19_attention_weight_min": -21.58148765563965,
      "activations/layer1_attention_weight_max": 17.45147705078125,
      "activations/layer1_attention_weight_min": -14.37338638305664,
      "activations/layer20_attention_weight_max": 23.673986434936523,
      "activations/layer20_attention_weight_min": -22.046165466308594,
      "activations/layer21_attention_weight_max": 35.84362030029297,
      "activations/layer21_attention_weight_min": -24.069259643554688,
      "activations/layer22_attention_weight_max": 38.24118423461914,
      "activations/layer22_attention_weight_min": -23.662765502929688,
      "activations/layer23_attention_weight_max": 32.919456481933594,
      "activations/layer23_attention_weight_min": -20.089418411254883,
      "activations/layer2_attention_weight_max": 30.259017944335938,
      "activations/layer2_attention_weight_min": -27.660268783569336,
      "activations/layer3_attention_weight_max": 78.20853424072266,
      "activations/layer3_attention_weight_min": -80.5059585571289,
      "activations/layer4_attention_weight_max": 95.13253021240234,
      "activations/layer4_attention_weight_min": -87.25007629394531,
      "activations/layer5_attention_weight_max": 92.57210540771484,
      "activations/layer5_attention_weight_min": -85.54417419433594,
      "activations/layer6_attention_weight_max": 55.08659744262695,
      "activations/layer6_attention_weight_min": -55.19528579711914,
      "activations/layer7_attention_weight_max": 73.04783630371094,
      "activations/layer7_attention_weight_min": -68.71813201904297,
      "activations/layer8_attention_weight_max": 43.40183639526367,
      "activations/layer8_attention_weight_min": -43.27348709106445,
      "activations/layer9_attention_weight_max": 48.64561080932617,
      "activations/layer9_attention_weight_min": -45.88554382324219,
      "epoch": 6.35,
      "learning_rate": 0.00011013371212121212,
      "loss": 2.8378,
      "step": 109350
    },
    {
      "activations/layer0_attention_weight_max": 16.88661003112793,
      "activations/layer0_attention_weight_min": -15.321952819824219,
      "activations/layer10_attention_weight_max": 36.198158264160156,
      "activations/layer10_attention_weight_min": -36.835723876953125,
      "activations/layer11_attention_weight_max": 27.77328872680664,
      "activations/layer11_attention_weight_min": -28.651111602783203,
      "activations/layer12_attention_weight_max": 18.261903762817383,
      "activations/layer12_attention_weight_min": -25.086450576782227,
      "activations/layer13_attention_weight_max": 48.36753845214844,
      "activations/layer13_attention_weight_min": -33.77134323120117,
      "activations/layer14_attention_weight_max": 51.46388244628906,
      "activations/layer14_attention_weight_min": -31.792680740356445,
      "activations/layer15_attention_weight_max": 37.64945602416992,
      "activations/layer15_attention_weight_min": -30.438894271850586,
      "activations/layer16_attention_weight_max": 30.612829208374023,
      "activations/layer16_attention_weight_min": -26.714069366455078,
      "activations/layer17_attention_weight_max": 53.008155822753906,
      "activations/layer17_attention_weight_min": -44.14272689819336,
      "activations/layer18_attention_weight_max": 52.384559631347656,
      "activations/layer18_attention_weight_min": -35.62369918823242,
      "activations/layer19_attention_weight_max": 24.068387985229492,
      "activations/layer19_attention_weight_min": -18.905834197998047,
      "activations/layer1_attention_weight_max": 17.298742294311523,
      "activations/layer1_attention_weight_min": -14.996798515319824,
      "activations/layer20_attention_weight_max": 20.816082000732422,
      "activations/layer20_attention_weight_min": -22.658124923706055,
      "activations/layer21_attention_weight_max": 38.76588439941406,
      "activations/layer21_attention_weight_min": -24.72945213317871,
      "activations/layer22_attention_weight_max": 33.78562927246094,
      "activations/layer22_attention_weight_min": -24.43438720703125,
      "activations/layer23_attention_weight_max": 30.10686492919922,
      "activations/layer23_attention_weight_min": -17.83935546875,
      "activations/layer2_attention_weight_max": 30.85834312438965,
      "activations/layer2_attention_weight_min": -27.69228172302246,
      "activations/layer3_attention_weight_max": 83.15799713134766,
      "activations/layer3_attention_weight_min": -84.1230697631836,
      "activations/layer4_attention_weight_max": 95.5876235961914,
      "activations/layer4_attention_weight_min": -91.78719329833984,
      "activations/layer5_attention_weight_max": 90.22737121582031,
      "activations/layer5_attention_weight_min": -84.51025390625,
      "activations/layer6_attention_weight_max": 54.66391372680664,
      "activations/layer6_attention_weight_min": -53.43497848510742,
      "activations/layer7_attention_weight_max": 69.75213623046875,
      "activations/layer7_attention_weight_min": -71.60554504394531,
      "activations/layer8_attention_weight_max": 44.56425094604492,
      "activations/layer8_attention_weight_min": -48.04433059692383,
      "activations/layer9_attention_weight_max": 50.93491744995117,
      "activations/layer9_attention_weight_min": -50.12595748901367,
      "epoch": 6.36,
      "learning_rate": 0.00011011477272727271,
      "loss": 2.83,
      "step": 109400
    },
    {
      "activations/layer0_attention_weight_max": 15.354424476623535,
      "activations/layer0_attention_weight_min": -15.233460426330566,
      "activations/layer10_attention_weight_max": 40.8293342590332,
      "activations/layer10_attention_weight_min": -36.4678840637207,
      "activations/layer11_attention_weight_max": 34.77463150024414,
      "activations/layer11_attention_weight_min": -30.774032592773438,
      "activations/layer12_attention_weight_max": 18.363290786743164,
      "activations/layer12_attention_weight_min": -29.552562713623047,
      "activations/layer13_attention_weight_max": 42.37622833251953,
      "activations/layer13_attention_weight_min": -33.05433654785156,
      "activations/layer14_attention_weight_max": 35.52140808105469,
      "activations/layer14_attention_weight_min": -28.820314407348633,
      "activations/layer15_attention_weight_max": 33.617225646972656,
      "activations/layer15_attention_weight_min": -29.863910675048828,
      "activations/layer16_attention_weight_max": 30.172712326049805,
      "activations/layer16_attention_weight_min": -27.59087562561035,
      "activations/layer17_attention_weight_max": 55.645263671875,
      "activations/layer17_attention_weight_min": -40.80094909667969,
      "activations/layer18_attention_weight_max": 47.824588775634766,
      "activations/layer18_attention_weight_min": -33.505123138427734,
      "activations/layer19_attention_weight_max": 21.39497947692871,
      "activations/layer19_attention_weight_min": -19.466150283813477,
      "activations/layer1_attention_weight_max": 16.987882614135742,
      "activations/layer1_attention_weight_min": -14.20881462097168,
      "activations/layer20_attention_weight_max": 21.303619384765625,
      "activations/layer20_attention_weight_min": -19.812015533447266,
      "activations/layer21_attention_weight_max": 37.66480255126953,
      "activations/layer21_attention_weight_min": -25.981401443481445,
      "activations/layer22_attention_weight_max": 34.01433181762695,
      "activations/layer22_attention_weight_min": -25.73699951171875,
      "activations/layer23_attention_weight_max": 31.30097198486328,
      "activations/layer23_attention_weight_min": -20.68314552307129,
      "activations/layer2_attention_weight_max": 32.632835388183594,
      "activations/layer2_attention_weight_min": -30.28628921508789,
      "activations/layer3_attention_weight_max": 86.18123626708984,
      "activations/layer3_attention_weight_min": -91.20478820800781,
      "activations/layer4_attention_weight_max": 103.42679595947266,
      "activations/layer4_attention_weight_min": -98.8095703125,
      "activations/layer5_attention_weight_max": 98.52096557617188,
      "activations/layer5_attention_weight_min": -89.93093872070312,
      "activations/layer6_attention_weight_max": 56.47904968261719,
      "activations/layer6_attention_weight_min": -53.9729118347168,
      "activations/layer7_attention_weight_max": 74.13214111328125,
      "activations/layer7_attention_weight_min": -71.88764953613281,
      "activations/layer8_attention_weight_max": 46.81624984741211,
      "activations/layer8_attention_weight_min": -45.25730514526367,
      "activations/layer9_attention_weight_max": 51.14458465576172,
      "activations/layer9_attention_weight_min": -49.809661865234375,
      "epoch": 6.36,
      "learning_rate": 0.00011009583333333332,
      "loss": 2.8367,
      "step": 109450
    },
    {
      "activations/layer0_attention_weight_max": 15.770919799804688,
      "activations/layer0_attention_weight_min": -15.424800872802734,
      "activations/layer10_attention_weight_max": 37.83179473876953,
      "activations/layer10_attention_weight_min": -36.12313461303711,
      "activations/layer11_attention_weight_max": 29.823326110839844,
      "activations/layer11_attention_weight_min": -27.969635009765625,
      "activations/layer12_attention_weight_max": 18.3447208404541,
      "activations/layer12_attention_weight_min": -26.747400283813477,
      "activations/layer13_attention_weight_max": 41.94081115722656,
      "activations/layer13_attention_weight_min": -34.77743148803711,
      "activations/layer14_attention_weight_max": 36.334571838378906,
      "activations/layer14_attention_weight_min": -28.906015396118164,
      "activations/layer15_attention_weight_max": 33.19078063964844,
      "activations/layer15_attention_weight_min": -31.591459274291992,
      "activations/layer16_attention_weight_max": 28.45072364807129,
      "activations/layer16_attention_weight_min": -30.999479293823242,
      "activations/layer17_attention_weight_max": 51.25746154785156,
      "activations/layer17_attention_weight_min": -38.79750442504883,
      "activations/layer18_attention_weight_max": 44.202449798583984,
      "activations/layer18_attention_weight_min": -32.838523864746094,
      "activations/layer19_attention_weight_max": 24.0191707611084,
      "activations/layer19_attention_weight_min": -19.03478240966797,
      "activations/layer1_attention_weight_max": 16.518341064453125,
      "activations/layer1_attention_weight_min": -14.111873626708984,
      "activations/layer20_attention_weight_max": 19.09958267211914,
      "activations/layer20_attention_weight_min": -22.524614334106445,
      "activations/layer21_attention_weight_max": 35.632633209228516,
      "activations/layer21_attention_weight_min": -27.25331687927246,
      "activations/layer22_attention_weight_max": 29.945037841796875,
      "activations/layer22_attention_weight_min": -23.275869369506836,
      "activations/layer23_attention_weight_max": 30.221485137939453,
      "activations/layer23_attention_weight_min": -20.264415740966797,
      "activations/layer2_attention_weight_max": 30.079971313476562,
      "activations/layer2_attention_weight_min": -28.904630661010742,
      "activations/layer3_attention_weight_max": 81.06133270263672,
      "activations/layer3_attention_weight_min": -81.70954132080078,
      "activations/layer4_attention_weight_max": 93.2037582397461,
      "activations/layer4_attention_weight_min": -93.40451049804688,
      "activations/layer5_attention_weight_max": 88.48565673828125,
      "activations/layer5_attention_weight_min": -84.81898498535156,
      "activations/layer6_attention_weight_max": 53.2876091003418,
      "activations/layer6_attention_weight_min": -53.618412017822266,
      "activations/layer7_attention_weight_max": 72.5171890258789,
      "activations/layer7_attention_weight_min": -66.16352844238281,
      "activations/layer8_attention_weight_max": 41.911537170410156,
      "activations/layer8_attention_weight_min": -43.61369705200195,
      "activations/layer9_attention_weight_max": 52.722652435302734,
      "activations/layer9_attention_weight_min": -49.76728439331055,
      "epoch": 6.36,
      "learning_rate": 0.00011007689393939393,
      "loss": 2.8292,
      "step": 109500
    },
    {
      "activations/layer0_attention_weight_max": 16.690898895263672,
      "activations/layer0_attention_weight_min": -15.248021125793457,
      "activations/layer10_attention_weight_max": 36.3846321105957,
      "activations/layer10_attention_weight_min": -33.79091262817383,
      "activations/layer11_attention_weight_max": 27.969058990478516,
      "activations/layer11_attention_weight_min": -27.194921493530273,
      "activations/layer12_attention_weight_max": 19.609338760375977,
      "activations/layer12_attention_weight_min": -28.42207145690918,
      "activations/layer13_attention_weight_max": 41.6490478515625,
      "activations/layer13_attention_weight_min": -30.703861236572266,
      "activations/layer14_attention_weight_max": 38.949737548828125,
      "activations/layer14_attention_weight_min": -31.866968154907227,
      "activations/layer15_attention_weight_max": 36.968902587890625,
      "activations/layer15_attention_weight_min": -31.323339462280273,
      "activations/layer16_attention_weight_max": 32.988250732421875,
      "activations/layer16_attention_weight_min": -25.378814697265625,
      "activations/layer17_attention_weight_max": 55.8680419921875,
      "activations/layer17_attention_weight_min": -41.42000198364258,
      "activations/layer18_attention_weight_max": 49.4067497253418,
      "activations/layer18_attention_weight_min": -34.47894287109375,
      "activations/layer19_attention_weight_max": 22.060970306396484,
      "activations/layer19_attention_weight_min": -18.741443634033203,
      "activations/layer1_attention_weight_max": 17.139575958251953,
      "activations/layer1_attention_weight_min": -13.15133285522461,
      "activations/layer20_attention_weight_max": 18.951454162597656,
      "activations/layer20_attention_weight_min": -20.64630126953125,
      "activations/layer21_attention_weight_max": 37.0161018371582,
      "activations/layer21_attention_weight_min": -23.541812896728516,
      "activations/layer22_attention_weight_max": 31.023345947265625,
      "activations/layer22_attention_weight_min": -24.7665958404541,
      "activations/layer23_attention_weight_max": 31.62255096435547,
      "activations/layer23_attention_weight_min": -20.33939552307129,
      "activations/layer2_attention_weight_max": 29.70833969116211,
      "activations/layer2_attention_weight_min": -29.175540924072266,
      "activations/layer3_attention_weight_max": 83.73709869384766,
      "activations/layer3_attention_weight_min": -91.02581787109375,
      "activations/layer4_attention_weight_max": 99.22029876708984,
      "activations/layer4_attention_weight_min": -92.0534896850586,
      "activations/layer5_attention_weight_max": 95.72136688232422,
      "activations/layer5_attention_weight_min": -83.2159652709961,
      "activations/layer6_attention_weight_max": 57.3099479675293,
      "activations/layer6_attention_weight_min": -55.0029182434082,
      "activations/layer7_attention_weight_max": 78.01773834228516,
      "activations/layer7_attention_weight_min": -69.38542175292969,
      "activations/layer8_attention_weight_max": 44.068199157714844,
      "activations/layer8_attention_weight_min": -42.268131256103516,
      "activations/layer9_attention_weight_max": 48.9935302734375,
      "activations/layer9_attention_weight_min": -44.21114730834961,
      "epoch": 6.37,
      "learning_rate": 0.00011005833333333333,
      "loss": 2.8344,
      "step": 109550
    },
    {
      "activations/layer0_attention_weight_max": 14.635869026184082,
      "activations/layer0_attention_weight_min": -15.43105697631836,
      "activations/layer10_attention_weight_max": 42.812294006347656,
      "activations/layer10_attention_weight_min": -37.74842071533203,
      "activations/layer11_attention_weight_max": 38.48802185058594,
      "activations/layer11_attention_weight_min": -30.272781372070312,
      "activations/layer12_attention_weight_max": 22.05522346496582,
      "activations/layer12_attention_weight_min": -25.777027130126953,
      "activations/layer13_attention_weight_max": 60.68517303466797,
      "activations/layer13_attention_weight_min": -33.78337860107422,
      "activations/layer14_attention_weight_max": 51.87828826904297,
      "activations/layer14_attention_weight_min": -31.852510452270508,
      "activations/layer15_attention_weight_max": 41.789737701416016,
      "activations/layer15_attention_weight_min": -31.64981460571289,
      "activations/layer16_attention_weight_max": 38.414527893066406,
      "activations/layer16_attention_weight_min": -28.34958839416504,
      "activations/layer17_attention_weight_max": 58.43695068359375,
      "activations/layer17_attention_weight_min": -42.878135681152344,
      "activations/layer18_attention_weight_max": 52.611061096191406,
      "activations/layer18_attention_weight_min": -35.2798957824707,
      "activations/layer19_attention_weight_max": 23.43805503845215,
      "activations/layer19_attention_weight_min": -19.560382843017578,
      "activations/layer1_attention_weight_max": 17.61566734313965,
      "activations/layer1_attention_weight_min": -15.791952133178711,
      "activations/layer20_attention_weight_max": 24.54974937438965,
      "activations/layer20_attention_weight_min": -22.607070922851562,
      "activations/layer21_attention_weight_max": 44.67011260986328,
      "activations/layer21_attention_weight_min": -22.713380813598633,
      "activations/layer22_attention_weight_max": 41.295963287353516,
      "activations/layer22_attention_weight_min": -26.67816734313965,
      "activations/layer23_attention_weight_max": 35.68710708618164,
      "activations/layer23_attention_weight_min": -21.58424949645996,
      "activations/layer2_attention_weight_max": 31.087841033935547,
      "activations/layer2_attention_weight_min": -29.625545501708984,
      "activations/layer3_attention_weight_max": 84.2386245727539,
      "activations/layer3_attention_weight_min": -84.98344421386719,
      "activations/layer4_attention_weight_max": 102.9642333984375,
      "activations/layer4_attention_weight_min": -99.73159790039062,
      "activations/layer5_attention_weight_max": 96.46263885498047,
      "activations/layer5_attention_weight_min": -93.02352905273438,
      "activations/layer6_attention_weight_max": 59.81620407104492,
      "activations/layer6_attention_weight_min": -58.00773239135742,
      "activations/layer7_attention_weight_max": 84.07381439208984,
      "activations/layer7_attention_weight_min": -76.63103485107422,
      "activations/layer8_attention_weight_max": 52.19925308227539,
      "activations/layer8_attention_weight_min": -46.74736785888672,
      "activations/layer9_attention_weight_max": 72.38243103027344,
      "activations/layer9_attention_weight_min": -54.6051025390625,
      "epoch": 6.37,
      "learning_rate": 0.00011003939393939393,
      "loss": 2.8317,
      "step": 109600
    },
    {
      "activations/layer0_attention_weight_max": 14.494148254394531,
      "activations/layer0_attention_weight_min": -14.35379409790039,
      "activations/layer10_attention_weight_max": 36.45806121826172,
      "activations/layer10_attention_weight_min": -35.35676193237305,
      "activations/layer11_attention_weight_max": 29.225500106811523,
      "activations/layer11_attention_weight_min": -28.149076461791992,
      "activations/layer12_attention_weight_max": 23.74260711669922,
      "activations/layer12_attention_weight_min": -26.484342575073242,
      "activations/layer13_attention_weight_max": 47.8443717956543,
      "activations/layer13_attention_weight_min": -33.9931755065918,
      "activations/layer14_attention_weight_max": 40.78750228881836,
      "activations/layer14_attention_weight_min": -30.853193283081055,
      "activations/layer15_attention_weight_max": 38.795772552490234,
      "activations/layer15_attention_weight_min": -28.803194046020508,
      "activations/layer16_attention_weight_max": 34.02846908569336,
      "activations/layer16_attention_weight_min": -27.00465202331543,
      "activations/layer17_attention_weight_max": 54.857269287109375,
      "activations/layer17_attention_weight_min": -40.49102020263672,
      "activations/layer18_attention_weight_max": 52.04457473754883,
      "activations/layer18_attention_weight_min": -35.34408187866211,
      "activations/layer19_attention_weight_max": 22.059032440185547,
      "activations/layer19_attention_weight_min": -22.501909255981445,
      "activations/layer1_attention_weight_max": 16.750028610229492,
      "activations/layer1_attention_weight_min": -16.981842041015625,
      "activations/layer20_attention_weight_max": 20.704967498779297,
      "activations/layer20_attention_weight_min": -22.78870964050293,
      "activations/layer21_attention_weight_max": 38.7947883605957,
      "activations/layer21_attention_weight_min": -27.0667781829834,
      "activations/layer22_attention_weight_max": 33.148460388183594,
      "activations/layer22_attention_weight_min": -26.075965881347656,
      "activations/layer23_attention_weight_max": 31.46884536743164,
      "activations/layer23_attention_weight_min": -20.9726619720459,
      "activations/layer2_attention_weight_max": 31.009929656982422,
      "activations/layer2_attention_weight_min": -28.45856285095215,
      "activations/layer3_attention_weight_max": 81.18402099609375,
      "activations/layer3_attention_weight_min": -82.71512603759766,
      "activations/layer4_attention_weight_max": 96.37007141113281,
      "activations/layer4_attention_weight_min": -91.60163116455078,
      "activations/layer5_attention_weight_max": 91.72146606445312,
      "activations/layer5_attention_weight_min": -82.63203430175781,
      "activations/layer6_attention_weight_max": 60.604698181152344,
      "activations/layer6_attention_weight_min": -54.49970626831055,
      "activations/layer7_attention_weight_max": 74.35556030273438,
      "activations/layer7_attention_weight_min": -70.42364501953125,
      "activations/layer8_attention_weight_max": 43.88429641723633,
      "activations/layer8_attention_weight_min": -45.34260177612305,
      "activations/layer9_attention_weight_max": 47.24393844604492,
      "activations/layer9_attention_weight_min": -47.023651123046875,
      "epoch": 6.37,
      "learning_rate": 0.00011002045454545455,
      "loss": 2.835,
      "step": 109650
    },
    {
      "activations/layer0_attention_weight_max": 14.112991333007812,
      "activations/layer0_attention_weight_min": -14.317459106445312,
      "activations/layer10_attention_weight_max": 40.53526306152344,
      "activations/layer10_attention_weight_min": -37.905799865722656,
      "activations/layer11_attention_weight_max": 31.979108810424805,
      "activations/layer11_attention_weight_min": -29.836742401123047,
      "activations/layer12_attention_weight_max": 26.1857852935791,
      "activations/layer12_attention_weight_min": -26.599706649780273,
      "activations/layer13_attention_weight_max": 40.823265075683594,
      "activations/layer13_attention_weight_min": -27.939254760742188,
      "activations/layer14_attention_weight_max": 44.05274200439453,
      "activations/layer14_attention_weight_min": -28.787046432495117,
      "activations/layer15_attention_weight_max": 41.7320556640625,
      "activations/layer15_attention_weight_min": -29.22318458557129,
      "activations/layer16_attention_weight_max": 38.66812515258789,
      "activations/layer16_attention_weight_min": -28.19310188293457,
      "activations/layer17_attention_weight_max": 56.52117919921875,
      "activations/layer17_attention_weight_min": -40.090572357177734,
      "activations/layer18_attention_weight_max": 52.797607421875,
      "activations/layer18_attention_weight_min": -34.93144226074219,
      "activations/layer19_attention_weight_max": 27.021240234375,
      "activations/layer19_attention_weight_min": -20.620807647705078,
      "activations/layer1_attention_weight_max": 16.427278518676758,
      "activations/layer1_attention_weight_min": -16.071670532226562,
      "activations/layer20_attention_weight_max": 24.49713134765625,
      "activations/layer20_attention_weight_min": -19.508228302001953,
      "activations/layer21_attention_weight_max": 47.08938980102539,
      "activations/layer21_attention_weight_min": -29.091747283935547,
      "activations/layer22_attention_weight_max": 33.97484588623047,
      "activations/layer22_attention_weight_min": -25.48430633544922,
      "activations/layer23_attention_weight_max": 35.514678955078125,
      "activations/layer23_attention_weight_min": -21.295320510864258,
      "activations/layer2_attention_weight_max": 30.510910034179688,
      "activations/layer2_attention_weight_min": -28.82221794128418,
      "activations/layer3_attention_weight_max": 82.73230743408203,
      "activations/layer3_attention_weight_min": -85.4875259399414,
      "activations/layer4_attention_weight_max": 99.38975524902344,
      "activations/layer4_attention_weight_min": -96.11312103271484,
      "activations/layer5_attention_weight_max": 95.8061752319336,
      "activations/layer5_attention_weight_min": -87.45709228515625,
      "activations/layer6_attention_weight_max": 58.413909912109375,
      "activations/layer6_attention_weight_min": -55.32122039794922,
      "activations/layer7_attention_weight_max": 72.03663635253906,
      "activations/layer7_attention_weight_min": -70.44866943359375,
      "activations/layer8_attention_weight_max": 44.82569122314453,
      "activations/layer8_attention_weight_min": -45.764461517333984,
      "activations/layer9_attention_weight_max": 53.514625549316406,
      "activations/layer9_attention_weight_min": -52.120967864990234,
      "epoch": 6.37,
      "learning_rate": 0.00011000151515151513,
      "loss": 2.8381,
      "step": 109700
    },
    {
      "activations/layer0_attention_weight_max": 14.382938385009766,
      "activations/layer0_attention_weight_min": -14.52966594696045,
      "activations/layer10_attention_weight_max": 46.88550567626953,
      "activations/layer10_attention_weight_min": -39.58930969238281,
      "activations/layer11_attention_weight_max": 37.077178955078125,
      "activations/layer11_attention_weight_min": -32.25291442871094,
      "activations/layer12_attention_weight_max": 32.55024719238281,
      "activations/layer12_attention_weight_min": -24.785852432250977,
      "activations/layer13_attention_weight_max": 53.302120208740234,
      "activations/layer13_attention_weight_min": -31.972705841064453,
      "activations/layer14_attention_weight_max": 40.06239318847656,
      "activations/layer14_attention_weight_min": -29.494125366210938,
      "activations/layer15_attention_weight_max": 41.23929977416992,
      "activations/layer15_attention_weight_min": -30.789608001708984,
      "activations/layer16_attention_weight_max": 32.036582946777344,
      "activations/layer16_attention_weight_min": -26.59845542907715,
      "activations/layer17_attention_weight_max": 56.061561584472656,
      "activations/layer17_attention_weight_min": -40.86575698852539,
      "activations/layer18_attention_weight_max": 48.792930603027344,
      "activations/layer18_attention_weight_min": -34.87590789794922,
      "activations/layer19_attention_weight_max": 22.45097541809082,
      "activations/layer19_attention_weight_min": -20.638051986694336,
      "activations/layer1_attention_weight_max": 18.2993106842041,
      "activations/layer1_attention_weight_min": -14.358983039855957,
      "activations/layer20_attention_weight_max": 22.43747901916504,
      "activations/layer20_attention_weight_min": -27.432287216186523,
      "activations/layer21_attention_weight_max": 39.04163360595703,
      "activations/layer21_attention_weight_min": -26.7113037109375,
      "activations/layer22_attention_weight_max": 31.5612850189209,
      "activations/layer22_attention_weight_min": -24.324373245239258,
      "activations/layer23_attention_weight_max": 30.235713958740234,
      "activations/layer23_attention_weight_min": -19.861547470092773,
      "activations/layer2_attention_weight_max": 33.02278137207031,
      "activations/layer2_attention_weight_min": -30.350055694580078,
      "activations/layer3_attention_weight_max": 87.10237884521484,
      "activations/layer3_attention_weight_min": -89.17610931396484,
      "activations/layer4_attention_weight_max": 105.2746353149414,
      "activations/layer4_attention_weight_min": -97.65689849853516,
      "activations/layer5_attention_weight_max": 104.18379974365234,
      "activations/layer5_attention_weight_min": -91.65248107910156,
      "activations/layer6_attention_weight_max": 64.04683685302734,
      "activations/layer6_attention_weight_min": -57.91743087768555,
      "activations/layer7_attention_weight_max": 79.12527465820312,
      "activations/layer7_attention_weight_min": -73.75658416748047,
      "activations/layer8_attention_weight_max": 50.18351364135742,
      "activations/layer8_attention_weight_min": -50.2183837890625,
      "activations/layer9_attention_weight_max": 56.359432220458984,
      "activations/layer9_attention_weight_min": -49.78866195678711,
      "epoch": 6.38,
      "learning_rate": 0.00010998257575757575,
      "loss": 2.8049,
      "step": 109750
    },
    {
      "activations/layer0_attention_weight_max": 14.656149864196777,
      "activations/layer0_attention_weight_min": -14.614855766296387,
      "activations/layer10_attention_weight_max": 38.42789077758789,
      "activations/layer10_attention_weight_min": -36.35575485229492,
      "activations/layer11_attention_weight_max": 30.074079513549805,
      "activations/layer11_attention_weight_min": -28.0391902923584,
      "activations/layer12_attention_weight_max": 20.630517959594727,
      "activations/layer12_attention_weight_min": -23.65703010559082,
      "activations/layer13_attention_weight_max": 44.61433792114258,
      "activations/layer13_attention_weight_min": -32.782508850097656,
      "activations/layer14_attention_weight_max": 38.932071685791016,
      "activations/layer14_attention_weight_min": -30.25834846496582,
      "activations/layer15_attention_weight_max": 36.42018127441406,
      "activations/layer15_attention_weight_min": -28.765287399291992,
      "activations/layer16_attention_weight_max": 31.119625091552734,
      "activations/layer16_attention_weight_min": -26.162633895874023,
      "activations/layer17_attention_weight_max": 54.88840866088867,
      "activations/layer17_attention_weight_min": -40.88676834106445,
      "activations/layer18_attention_weight_max": 48.54975509643555,
      "activations/layer18_attention_weight_min": -35.16069793701172,
      "activations/layer19_attention_weight_max": 22.0838680267334,
      "activations/layer19_attention_weight_min": -19.42988395690918,
      "activations/layer1_attention_weight_max": 16.859485626220703,
      "activations/layer1_attention_weight_min": -16.544416427612305,
      "activations/layer20_attention_weight_max": 20.52629852294922,
      "activations/layer20_attention_weight_min": -20.630029678344727,
      "activations/layer21_attention_weight_max": 39.42561340332031,
      "activations/layer21_attention_weight_min": -24.71184730529785,
      "activations/layer22_attention_weight_max": 33.004661560058594,
      "activations/layer22_attention_weight_min": -24.665790557861328,
      "activations/layer23_attention_weight_max": 33.15083312988281,
      "activations/layer23_attention_weight_min": -22.07748794555664,
      "activations/layer2_attention_weight_max": 31.91525650024414,
      "activations/layer2_attention_weight_min": -28.749441146850586,
      "activations/layer3_attention_weight_max": 82.9620590209961,
      "activations/layer3_attention_weight_min": -87.01275634765625,
      "activations/layer4_attention_weight_max": 102.23775482177734,
      "activations/layer4_attention_weight_min": -95.52657318115234,
      "activations/layer5_attention_weight_max": 98.81401824951172,
      "activations/layer5_attention_weight_min": -89.76380920410156,
      "activations/layer6_attention_weight_max": 60.03706741333008,
      "activations/layer6_attention_weight_min": -56.54179000854492,
      "activations/layer7_attention_weight_max": 75.28607177734375,
      "activations/layer7_attention_weight_min": -72.03258514404297,
      "activations/layer8_attention_weight_max": 45.0767707824707,
      "activations/layer8_attention_weight_min": -45.58888626098633,
      "activations/layer9_attention_weight_max": 48.437095642089844,
      "activations/layer9_attention_weight_min": -47.4831657409668,
      "epoch": 6.38,
      "learning_rate": 0.00010996363636363636,
      "loss": 2.8445,
      "step": 109800
    },
    {
      "activations/layer0_attention_weight_max": 14.296274185180664,
      "activations/layer0_attention_weight_min": -14.799211502075195,
      "activations/layer10_attention_weight_max": 38.373111724853516,
      "activations/layer10_attention_weight_min": -35.527008056640625,
      "activations/layer11_attention_weight_max": 29.909360885620117,
      "activations/layer11_attention_weight_min": -28.475868225097656,
      "activations/layer12_attention_weight_max": 19.726110458374023,
      "activations/layer12_attention_weight_min": -23.83010482788086,
      "activations/layer13_attention_weight_max": 46.1968879699707,
      "activations/layer13_attention_weight_min": -34.81300735473633,
      "activations/layer14_attention_weight_max": 41.94277572631836,
      "activations/layer14_attention_weight_min": -31.164199829101562,
      "activations/layer15_attention_weight_max": 37.78190994262695,
      "activations/layer15_attention_weight_min": -30.09135627746582,
      "activations/layer16_attention_weight_max": 31.01551055908203,
      "activations/layer16_attention_weight_min": -28.745662689208984,
      "activations/layer17_attention_weight_max": 57.79804992675781,
      "activations/layer17_attention_weight_min": -43.75437545776367,
      "activations/layer18_attention_weight_max": 51.41777801513672,
      "activations/layer18_attention_weight_min": -37.51460647583008,
      "activations/layer19_attention_weight_max": 23.33303451538086,
      "activations/layer19_attention_weight_min": -21.021818161010742,
      "activations/layer1_attention_weight_max": 16.55845069885254,
      "activations/layer1_attention_weight_min": -15.155340194702148,
      "activations/layer20_attention_weight_max": 21.798025131225586,
      "activations/layer20_attention_weight_min": -22.908416748046875,
      "activations/layer21_attention_weight_max": 38.70376968383789,
      "activations/layer21_attention_weight_min": -24.51201057434082,
      "activations/layer22_attention_weight_max": 35.22867202758789,
      "activations/layer22_attention_weight_min": -27.50726890563965,
      "activations/layer23_attention_weight_max": 31.35035514831543,
      "activations/layer23_attention_weight_min": -19.362560272216797,
      "activations/layer2_attention_weight_max": 32.518455505371094,
      "activations/layer2_attention_weight_min": -29.637075424194336,
      "activations/layer3_attention_weight_max": 82.41070556640625,
      "activations/layer3_attention_weight_min": -83.012451171875,
      "activations/layer4_attention_weight_max": 98.03166961669922,
      "activations/layer4_attention_weight_min": -90.27289581298828,
      "activations/layer5_attention_weight_max": 93.04500579833984,
      "activations/layer5_attention_weight_min": -86.80590057373047,
      "activations/layer6_attention_weight_max": 58.22846221923828,
      "activations/layer6_attention_weight_min": -53.438533782958984,
      "activations/layer7_attention_weight_max": 82.25684356689453,
      "activations/layer7_attention_weight_min": -70.08070373535156,
      "activations/layer8_attention_weight_max": 47.24758529663086,
      "activations/layer8_attention_weight_min": -47.00691604614258,
      "activations/layer9_attention_weight_max": 51.211883544921875,
      "activations/layer9_attention_weight_min": -46.01900863647461,
      "epoch": 6.38,
      "learning_rate": 0.00010994469696969695,
      "loss": 2.8178,
      "step": 109850
    },
    {
      "activations/layer0_attention_weight_max": 15.926929473876953,
      "activations/layer0_attention_weight_min": -14.890144348144531,
      "activations/layer10_attention_weight_max": 36.491432189941406,
      "activations/layer10_attention_weight_min": -34.49126052856445,
      "activations/layer11_attention_weight_max": 31.182342529296875,
      "activations/layer11_attention_weight_min": -28.160518646240234,
      "activations/layer12_attention_weight_max": 20.47221565246582,
      "activations/layer12_attention_weight_min": -25.104694366455078,
      "activations/layer13_attention_weight_max": 80.97590637207031,
      "activations/layer13_attention_weight_min": -41.88128662109375,
      "activations/layer14_attention_weight_max": 76.09373474121094,
      "activations/layer14_attention_weight_min": -38.12090301513672,
      "activations/layer15_attention_weight_max": 54.38175964355469,
      "activations/layer15_attention_weight_min": -33.79523849487305,
      "activations/layer16_attention_weight_max": 40.48588562011719,
      "activations/layer16_attention_weight_min": -31.855419158935547,
      "activations/layer17_attention_weight_max": 65.57969665527344,
      "activations/layer17_attention_weight_min": -53.65082550048828,
      "activations/layer18_attention_weight_max": 62.95936965942383,
      "activations/layer18_attention_weight_min": -44.4967155456543,
      "activations/layer19_attention_weight_max": 28.922513961791992,
      "activations/layer19_attention_weight_min": -23.646785736083984,
      "activations/layer1_attention_weight_max": 17.252256393432617,
      "activations/layer1_attention_weight_min": -15.51391887664795,
      "activations/layer20_attention_weight_max": 29.72008514404297,
      "activations/layer20_attention_weight_min": -23.962116241455078,
      "activations/layer21_attention_weight_max": 56.32542037963867,
      "activations/layer21_attention_weight_min": -27.437753677368164,
      "activations/layer22_attention_weight_max": 39.202701568603516,
      "activations/layer22_attention_weight_min": -27.126367568969727,
      "activations/layer23_attention_weight_max": 37.26951599121094,
      "activations/layer23_attention_weight_min": -19.894094467163086,
      "activations/layer2_attention_weight_max": 29.63906478881836,
      "activations/layer2_attention_weight_min": -28.33466148376465,
      "activations/layer3_attention_weight_max": 77.4438705444336,
      "activations/layer3_attention_weight_min": -83.72618865966797,
      "activations/layer4_attention_weight_max": 98.1648941040039,
      "activations/layer4_attention_weight_min": -91.32990264892578,
      "activations/layer5_attention_weight_max": 98.60169982910156,
      "activations/layer5_attention_weight_min": -84.6511459350586,
      "activations/layer6_attention_weight_max": 58.299259185791016,
      "activations/layer6_attention_weight_min": -51.85414505004883,
      "activations/layer7_attention_weight_max": 82.04435729980469,
      "activations/layer7_attention_weight_min": -68.70179748535156,
      "activations/layer8_attention_weight_max": 44.495147705078125,
      "activations/layer8_attention_weight_min": -42.44380569458008,
      "activations/layer9_attention_weight_max": 53.91913604736328,
      "activations/layer9_attention_weight_min": -47.47194290161133,
      "epoch": 6.39,
      "learning_rate": 0.00010992575757575757,
      "loss": 2.8316,
      "step": 109900
    },
    {
      "activations/layer0_attention_weight_max": 15.258857727050781,
      "activations/layer0_attention_weight_min": -14.758885383605957,
      "activations/layer10_attention_weight_max": 36.18245315551758,
      "activations/layer10_attention_weight_min": -33.299652099609375,
      "activations/layer11_attention_weight_max": 31.976926803588867,
      "activations/layer11_attention_weight_min": -29.244550704956055,
      "activations/layer12_attention_weight_max": 23.043188095092773,
      "activations/layer12_attention_weight_min": -30.94746208190918,
      "activations/layer13_attention_weight_max": 43.013938903808594,
      "activations/layer13_attention_weight_min": -31.123531341552734,
      "activations/layer14_attention_weight_max": 49.492191314697266,
      "activations/layer14_attention_weight_min": -34.18281555175781,
      "activations/layer15_attention_weight_max": 38.34162902832031,
      "activations/layer15_attention_weight_min": -29.479644775390625,
      "activations/layer16_attention_weight_max": 34.77405548095703,
      "activations/layer16_attention_weight_min": -29.455230712890625,
      "activations/layer17_attention_weight_max": 55.331241607666016,
      "activations/layer17_attention_weight_min": -39.47967529296875,
      "activations/layer18_attention_weight_max": 52.4195556640625,
      "activations/layer18_attention_weight_min": -33.788169860839844,
      "activations/layer19_attention_weight_max": 21.634117126464844,
      "activations/layer19_attention_weight_min": -19.040470123291016,
      "activations/layer1_attention_weight_max": 16.613977432250977,
      "activations/layer1_attention_weight_min": -15.00504207611084,
      "activations/layer20_attention_weight_max": 23.076030731201172,
      "activations/layer20_attention_weight_min": -22.14495849609375,
      "activations/layer21_attention_weight_max": 40.246665954589844,
      "activations/layer21_attention_weight_min": -24.846155166625977,
      "activations/layer22_attention_weight_max": 36.32523727416992,
      "activations/layer22_attention_weight_min": -24.61922836303711,
      "activations/layer23_attention_weight_max": 32.24310302734375,
      "activations/layer23_attention_weight_min": -19.56612777709961,
      "activations/layer2_attention_weight_max": 31.331039428710938,
      "activations/layer2_attention_weight_min": -29.642135620117188,
      "activations/layer3_attention_weight_max": 88.36349487304688,
      "activations/layer3_attention_weight_min": -86.123046875,
      "activations/layer4_attention_weight_max": 101.946044921875,
      "activations/layer4_attention_weight_min": -96.28864288330078,
      "activations/layer5_attention_weight_max": 95.02379608154297,
      "activations/layer5_attention_weight_min": -85.11891174316406,
      "activations/layer6_attention_weight_max": 57.374176025390625,
      "activations/layer6_attention_weight_min": -53.681663513183594,
      "activations/layer7_attention_weight_max": 77.6270751953125,
      "activations/layer7_attention_weight_min": -70.31893157958984,
      "activations/layer8_attention_weight_max": 45.19985580444336,
      "activations/layer8_attention_weight_min": -44.208106994628906,
      "activations/layer9_attention_weight_max": 57.40601348876953,
      "activations/layer9_attention_weight_min": -51.47968292236328,
      "epoch": 6.39,
      "learning_rate": 0.00010990681818181818,
      "loss": 2.8379,
      "step": 109950
    },
    {
      "activations/layer0_attention_weight_max": 14.644057273864746,
      "activations/layer0_attention_weight_min": -14.86755084991455,
      "activations/layer10_attention_weight_max": 37.95036315917969,
      "activations/layer10_attention_weight_min": -34.55064010620117,
      "activations/layer11_attention_weight_max": 29.705585479736328,
      "activations/layer11_attention_weight_min": -28.776302337646484,
      "activations/layer12_attention_weight_max": 19.152193069458008,
      "activations/layer12_attention_weight_min": -22.59702491760254,
      "activations/layer13_attention_weight_max": 44.12987518310547,
      "activations/layer13_attention_weight_min": -34.63146209716797,
      "activations/layer14_attention_weight_max": 57.83492660522461,
      "activations/layer14_attention_weight_min": -37.0517463684082,
      "activations/layer15_attention_weight_max": 46.49760818481445,
      "activations/layer15_attention_weight_min": -32.651424407958984,
      "activations/layer16_attention_weight_max": 38.235015869140625,
      "activations/layer16_attention_weight_min": -27.088699340820312,
      "activations/layer17_attention_weight_max": 59.61355209350586,
      "activations/layer17_attention_weight_min": -41.7905387878418,
      "activations/layer18_attention_weight_max": 50.94894027709961,
      "activations/layer18_attention_weight_min": -35.15492248535156,
      "activations/layer19_attention_weight_max": 24.726112365722656,
      "activations/layer19_attention_weight_min": -19.732694625854492,
      "activations/layer1_attention_weight_max": 17.708053588867188,
      "activations/layer1_attention_weight_min": -15.501157760620117,
      "activations/layer20_attention_weight_max": 21.69022560119629,
      "activations/layer20_attention_weight_min": -23.586999893188477,
      "activations/layer21_attention_weight_max": 43.11325454711914,
      "activations/layer21_attention_weight_min": -24.77573585510254,
      "activations/layer22_attention_weight_max": 32.909847259521484,
      "activations/layer22_attention_weight_min": -26.876983642578125,
      "activations/layer23_attention_weight_max": 33.423519134521484,
      "activations/layer23_attention_weight_min": -22.640472412109375,
      "activations/layer2_attention_weight_max": 28.675222396850586,
      "activations/layer2_attention_weight_min": -26.876644134521484,
      "activations/layer3_attention_weight_max": 81.507568359375,
      "activations/layer3_attention_weight_min": -82.2958984375,
      "activations/layer4_attention_weight_max": 100.10236358642578,
      "activations/layer4_attention_weight_min": -93.06493377685547,
      "activations/layer5_attention_weight_max": 95.68999481201172,
      "activations/layer5_attention_weight_min": -89.30138397216797,
      "activations/layer6_attention_weight_max": 58.172813415527344,
      "activations/layer6_attention_weight_min": -54.716835021972656,
      "activations/layer7_attention_weight_max": 75.53550720214844,
      "activations/layer7_attention_weight_min": -76.46793365478516,
      "activations/layer8_attention_weight_max": 44.11707305908203,
      "activations/layer8_attention_weight_min": -44.155738830566406,
      "activations/layer9_attention_weight_max": 53.101783752441406,
      "activations/layer9_attention_weight_min": -53.4879035949707,
      "epoch": 6.39,
      "learning_rate": 0.00010988787878787877,
      "loss": 2.8312,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_loss": 2.78125,
      "eval_runtime": 8.5798,
      "eval_samples_per_second": 500.475,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_openwebtext_loss": 2.78125,
      "eval_openwebtext_ppl": 16.139182322446413,
      "eval_openwebtext_runtime": 8.5798,
      "eval_openwebtext_samples_per_second": 500.475,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_wikitext_loss": 3.009765625,
      "eval_wikitext_ppl": 20.282645623050005,
      "eval_wikitext_runtime": 2.0525,
      "eval_wikitext_samples_per_second": 222.168,
      "step": 110000
    },
    {
      "epoch": 6.39,
      "eval_lambada_loss": 2.78515625,
      "eval_lambada_ppl": 16.202349296067883,
      "eval_lambada_runtime": 9.6048,
      "eval_lambada_samples_per_second": 506.933,
      "step": 110000
    },
    {
      "activations/layer0_attention_weight_max": 14.706635475158691,
      "activations/layer0_attention_weight_min": -14.584521293640137,
      "activations/layer10_attention_weight_max": 36.828712463378906,
      "activations/layer10_attention_weight_min": -36.691497802734375,
      "activations/layer11_attention_weight_max": 29.88271141052246,
      "activations/layer11_attention_weight_min": -30.339658737182617,
      "activations/layer12_attention_weight_max": 19.61227798461914,
      "activations/layer12_attention_weight_min": -23.613317489624023,
      "activations/layer13_attention_weight_max": 40.9908561706543,
      "activations/layer13_attention_weight_min": -31.51759910583496,
      "activations/layer14_attention_weight_max": 35.87165832519531,
      "activations/layer14_attention_weight_min": -30.503721237182617,
      "activations/layer15_attention_weight_max": 36.20863723754883,
      "activations/layer15_attention_weight_min": -30.553560256958008,
      "activations/layer16_attention_weight_max": 30.953365325927734,
      "activations/layer16_attention_weight_min": -26.97481918334961,
      "activations/layer17_attention_weight_max": 52.88712692260742,
      "activations/layer17_attention_weight_min": -39.10466003417969,
      "activations/layer18_attention_weight_max": 45.74593734741211,
      "activations/layer18_attention_weight_min": -33.57613754272461,
      "activations/layer19_attention_weight_max": 19.22563362121582,
      "activations/layer19_attention_weight_min": -20.217153549194336,
      "activations/layer1_attention_weight_max": 18.070823669433594,
      "activations/layer1_attention_weight_min": -14.863917350769043,
      "activations/layer20_attention_weight_max": 22.29466438293457,
      "activations/layer20_attention_weight_min": -22.43010139465332,
      "activations/layer21_attention_weight_max": 34.52112579345703,
      "activations/layer21_attention_weight_min": -24.38371467590332,
      "activations/layer22_attention_weight_max": 33.404903411865234,
      "activations/layer22_attention_weight_min": -24.710508346557617,
      "activations/layer23_attention_weight_max": 30.783798217773438,
      "activations/layer23_attention_weight_min": -20.16349220275879,
      "activations/layer2_attention_weight_max": 30.918594360351562,
      "activations/layer2_attention_weight_min": -29.593854904174805,
      "activations/layer3_attention_weight_max": 83.41532897949219,
      "activations/layer3_attention_weight_min": -84.79666900634766,
      "activations/layer4_attention_weight_max": 101.27184295654297,
      "activations/layer4_attention_weight_min": -94.32264709472656,
      "activations/layer5_attention_weight_max": 91.89991760253906,
      "activations/layer5_attention_weight_min": -88.38356018066406,
      "activations/layer6_attention_weight_max": 56.064674377441406,
      "activations/layer6_attention_weight_min": -52.868751525878906,
      "activations/layer7_attention_weight_max": 71.21588897705078,
      "activations/layer7_attention_weight_min": -69.50038146972656,
      "activations/layer8_attention_weight_max": 43.5086784362793,
      "activations/layer8_attention_weight_min": -45.5571174621582,
      "activations/layer9_attention_weight_max": 50.3167839050293,
      "activations/layer9_attention_weight_min": -48.219173431396484,
      "epoch": 6.39,
      "learning_rate": 0.00010986893939393939,
      "loss": 2.8347,
      "step": 110050
    },
    {
      "activations/layer0_attention_weight_max": 14.716588973999023,
      "activations/layer0_attention_weight_min": -15.061558723449707,
      "activations/layer10_attention_weight_max": 35.178489685058594,
      "activations/layer10_attention_weight_min": -34.37018966674805,
      "activations/layer11_attention_weight_max": 30.21036148071289,
      "activations/layer11_attention_weight_min": -28.156978607177734,
      "activations/layer12_attention_weight_max": 22.378625869750977,
      "activations/layer12_attention_weight_min": -23.420190811157227,
      "activations/layer13_attention_weight_max": 42.24818801879883,
      "activations/layer13_attention_weight_min": -29.377578735351562,
      "activations/layer14_attention_weight_max": 52.07864761352539,
      "activations/layer14_attention_weight_min": -31.983366012573242,
      "activations/layer15_attention_weight_max": 41.58070373535156,
      "activations/layer15_attention_weight_min": -28.625516891479492,
      "activations/layer16_attention_weight_max": 31.467302322387695,
      "activations/layer16_attention_weight_min": -28.683856964111328,
      "activations/layer17_attention_weight_max": 53.39543533325195,
      "activations/layer17_attention_weight_min": -38.776283264160156,
      "activations/layer18_attention_weight_max": 49.134033203125,
      "activations/layer18_attention_weight_min": -33.450279235839844,
      "activations/layer19_attention_weight_max": 24.54143714904785,
      "activations/layer19_attention_weight_min": -19.180288314819336,
      "activations/layer1_attention_weight_max": 16.754674911499023,
      "activations/layer1_attention_weight_min": -15.682472229003906,
      "activations/layer20_attention_weight_max": 21.910783767700195,
      "activations/layer20_attention_weight_min": -22.220134735107422,
      "activations/layer21_attention_weight_max": 40.8840446472168,
      "activations/layer21_attention_weight_min": -26.201505661010742,
      "activations/layer22_attention_weight_max": 33.182106018066406,
      "activations/layer22_attention_weight_min": -24.10890769958496,
      "activations/layer23_attention_weight_max": 31.467260360717773,
      "activations/layer23_attention_weight_min": -19.932920455932617,
      "activations/layer2_attention_weight_max": 31.31664276123047,
      "activations/layer2_attention_weight_min": -29.0636043548584,
      "activations/layer3_attention_weight_max": 80.53099060058594,
      "activations/layer3_attention_weight_min": -87.36896514892578,
      "activations/layer4_attention_weight_max": 98.39498901367188,
      "activations/layer4_attention_weight_min": -93.46994018554688,
      "activations/layer5_attention_weight_max": 90.35845947265625,
      "activations/layer5_attention_weight_min": -83.77304077148438,
      "activations/layer6_attention_weight_max": 57.785465240478516,
      "activations/layer6_attention_weight_min": -51.861244201660156,
      "activations/layer7_attention_weight_max": 72.27501678466797,
      "activations/layer7_attention_weight_min": -65.95953369140625,
      "activations/layer8_attention_weight_max": 43.021820068359375,
      "activations/layer8_attention_weight_min": -42.84020233154297,
      "activations/layer9_attention_weight_max": 48.16990280151367,
      "activations/layer9_attention_weight_min": -48.51993942260742,
      "epoch": 6.4,
      "learning_rate": 0.00010984999999999999,
      "loss": 2.862,
      "step": 110100
    },
    {
      "activations/layer0_attention_weight_max": 14.8892183303833,
      "activations/layer0_attention_weight_min": -15.300046920776367,
      "activations/layer10_attention_weight_max": 38.79474639892578,
      "activations/layer10_attention_weight_min": -35.50201416015625,
      "activations/layer11_attention_weight_max": 30.56397819519043,
      "activations/layer11_attention_weight_min": -30.62228775024414,
      "activations/layer12_attention_weight_max": 22.330318450927734,
      "activations/layer12_attention_weight_min": -26.592985153198242,
      "activations/layer13_attention_weight_max": 53.72010040283203,
      "activations/layer13_attention_weight_min": -35.369728088378906,
      "activations/layer14_attention_weight_max": 53.677459716796875,
      "activations/layer14_attention_weight_min": -31.13115119934082,
      "activations/layer15_attention_weight_max": 40.02606201171875,
      "activations/layer15_attention_weight_min": -30.76796531677246,
      "activations/layer16_attention_weight_max": 39.32002639770508,
      "activations/layer16_attention_weight_min": -26.597614288330078,
      "activations/layer17_attention_weight_max": 59.78032684326172,
      "activations/layer17_attention_weight_min": -44.28050994873047,
      "activations/layer18_attention_weight_max": 47.354434967041016,
      "activations/layer18_attention_weight_min": -35.664581298828125,
      "activations/layer19_attention_weight_max": 21.42548370361328,
      "activations/layer19_attention_weight_min": -23.88982391357422,
      "activations/layer1_attention_weight_max": 17.5316104888916,
      "activations/layer1_attention_weight_min": -14.676960945129395,
      "activations/layer20_attention_weight_max": 23.689939498901367,
      "activations/layer20_attention_weight_min": -22.083463668823242,
      "activations/layer21_attention_weight_max": 42.61909103393555,
      "activations/layer21_attention_weight_min": -26.8906307220459,
      "activations/layer22_attention_weight_max": 38.68333053588867,
      "activations/layer22_attention_weight_min": -25.93195915222168,
      "activations/layer23_attention_weight_max": 37.887542724609375,
      "activations/layer23_attention_weight_min": -20.1821231842041,
      "activations/layer2_attention_weight_max": 31.044281005859375,
      "activations/layer2_attention_weight_min": -29.87123680114746,
      "activations/layer3_attention_weight_max": 87.21147155761719,
      "activations/layer3_attention_weight_min": -89.99492645263672,
      "activations/layer4_attention_weight_max": 104.38402557373047,
      "activations/layer4_attention_weight_min": -97.94264221191406,
      "activations/layer5_attention_weight_max": 99.47673034667969,
      "activations/layer5_attention_weight_min": -89.48440551757812,
      "activations/layer6_attention_weight_max": 58.93855667114258,
      "activations/layer6_attention_weight_min": -54.794010162353516,
      "activations/layer7_attention_weight_max": 75.39227294921875,
      "activations/layer7_attention_weight_min": -71.8941421508789,
      "activations/layer8_attention_weight_max": 44.74632263183594,
      "activations/layer8_attention_weight_min": -43.488853454589844,
      "activations/layer9_attention_weight_max": 49.98551559448242,
      "activations/layer9_attention_weight_min": -46.97134017944336,
      "epoch": 6.4,
      "learning_rate": 0.00010983106060606059,
      "loss": 2.8412,
      "step": 110150
    },
    {
      "activations/layer0_attention_weight_max": 14.97795581817627,
      "activations/layer0_attention_weight_min": -14.696671485900879,
      "activations/layer10_attention_weight_max": 37.93223190307617,
      "activations/layer10_attention_weight_min": -35.61867141723633,
      "activations/layer11_attention_weight_max": 31.689468383789062,
      "activations/layer11_attention_weight_min": -28.92885398864746,
      "activations/layer12_attention_weight_max": 19.18789291381836,
      "activations/layer12_attention_weight_min": -23.02976417541504,
      "activations/layer13_attention_weight_max": 42.757484436035156,
      "activations/layer13_attention_weight_min": -33.26347351074219,
      "activations/layer14_attention_weight_max": 39.261383056640625,
      "activations/layer14_attention_weight_min": -30.8548583984375,
      "activations/layer15_attention_weight_max": 36.20587158203125,
      "activations/layer15_attention_weight_min": -32.59402084350586,
      "activations/layer16_attention_weight_max": 32.37898254394531,
      "activations/layer16_attention_weight_min": -26.61775016784668,
      "activations/layer17_attention_weight_max": 55.8702278137207,
      "activations/layer17_attention_weight_min": -41.92351150512695,
      "activations/layer18_attention_weight_max": 46.615360260009766,
      "activations/layer18_attention_weight_min": -35.82579040527344,
      "activations/layer19_attention_weight_max": 26.033376693725586,
      "activations/layer19_attention_weight_min": -19.547496795654297,
      "activations/layer1_attention_weight_max": 16.632381439208984,
      "activations/layer1_attention_weight_min": -17.029932022094727,
      "activations/layer20_attention_weight_max": 24.186199188232422,
      "activations/layer20_attention_weight_min": -19.831907272338867,
      "activations/layer21_attention_weight_max": 37.42325973510742,
      "activations/layer21_attention_weight_min": -26.54595947265625,
      "activations/layer22_attention_weight_max": 34.73299026489258,
      "activations/layer22_attention_weight_min": -24.193634033203125,
      "activations/layer23_attention_weight_max": 30.84154510498047,
      "activations/layer23_attention_weight_min": -21.257204055786133,
      "activations/layer2_attention_weight_max": 30.695981979370117,
      "activations/layer2_attention_weight_min": -29.38081932067871,
      "activations/layer3_attention_weight_max": 90.1097183227539,
      "activations/layer3_attention_weight_min": -85.36119842529297,
      "activations/layer4_attention_weight_max": 99.15532684326172,
      "activations/layer4_attention_weight_min": -94.2665023803711,
      "activations/layer5_attention_weight_max": 91.92634582519531,
      "activations/layer5_attention_weight_min": -88.1317367553711,
      "activations/layer6_attention_weight_max": 56.42268371582031,
      "activations/layer6_attention_weight_min": -53.12857437133789,
      "activations/layer7_attention_weight_max": 77.06987762451172,
      "activations/layer7_attention_weight_min": -66.76271057128906,
      "activations/layer8_attention_weight_max": 43.162628173828125,
      "activations/layer8_attention_weight_min": -41.69083023071289,
      "activations/layer9_attention_weight_max": 51.596370697021484,
      "activations/layer9_attention_weight_min": -49.014156341552734,
      "epoch": 6.4,
      "learning_rate": 0.0001098121212121212,
      "loss": 2.8297,
      "step": 110200
    },
    {
      "activations/layer0_attention_weight_max": 15.70540714263916,
      "activations/layer0_attention_weight_min": -15.199052810668945,
      "activations/layer10_attention_weight_max": 39.690250396728516,
      "activations/layer10_attention_weight_min": -35.67449188232422,
      "activations/layer11_attention_weight_max": 29.66846466064453,
      "activations/layer11_attention_weight_min": -31.23105239868164,
      "activations/layer12_attention_weight_max": 20.964078903198242,
      "activations/layer12_attention_weight_min": -28.023555755615234,
      "activations/layer13_attention_weight_max": 46.02067565917969,
      "activations/layer13_attention_weight_min": -36.76430130004883,
      "activations/layer14_attention_weight_max": 46.42068099975586,
      "activations/layer14_attention_weight_min": -31.73151397705078,
      "activations/layer15_attention_weight_max": 36.68856430053711,
      "activations/layer15_attention_weight_min": -29.993967056274414,
      "activations/layer16_attention_weight_max": 33.5770149230957,
      "activations/layer16_attention_weight_min": -27.30914878845215,
      "activations/layer17_attention_weight_max": 60.77653121948242,
      "activations/layer17_attention_weight_min": -41.953189849853516,
      "activations/layer18_attention_weight_max": 49.96403503417969,
      "activations/layer18_attention_weight_min": -35.160858154296875,
      "activations/layer19_attention_weight_max": 21.01637077331543,
      "activations/layer19_attention_weight_min": -19.866004943847656,
      "activations/layer1_attention_weight_max": 17.213388442993164,
      "activations/layer1_attention_weight_min": -14.168642044067383,
      "activations/layer20_attention_weight_max": 23.61990737915039,
      "activations/layer20_attention_weight_min": -19.95670509338379,
      "activations/layer21_attention_weight_max": 37.36183547973633,
      "activations/layer21_attention_weight_min": -24.42449378967285,
      "activations/layer22_attention_weight_max": 31.572509765625,
      "activations/layer22_attention_weight_min": -25.476919174194336,
      "activations/layer23_attention_weight_max": 33.753807067871094,
      "activations/layer23_attention_weight_min": -19.374801635742188,
      "activations/layer2_attention_weight_max": 31.138090133666992,
      "activations/layer2_attention_weight_min": -28.11630630493164,
      "activations/layer3_attention_weight_max": 79.2808837890625,
      "activations/layer3_attention_weight_min": -81.22097778320312,
      "activations/layer4_attention_weight_max": 96.83885955810547,
      "activations/layer4_attention_weight_min": -92.60383605957031,
      "activations/layer5_attention_weight_max": 90.6792221069336,
      "activations/layer5_attention_weight_min": -86.94390106201172,
      "activations/layer6_attention_weight_max": 59.602272033691406,
      "activations/layer6_attention_weight_min": -54.93017578125,
      "activations/layer7_attention_weight_max": 74.94761657714844,
      "activations/layer7_attention_weight_min": -68.54100036621094,
      "activations/layer8_attention_weight_max": 46.25906753540039,
      "activations/layer8_attention_weight_min": -44.50287628173828,
      "activations/layer9_attention_weight_max": 53.0508918762207,
      "activations/layer9_attention_weight_min": -49.621437072753906,
      "epoch": 6.41,
      "learning_rate": 0.0001097931818181818,
      "loss": 2.8329,
      "step": 110250
    },
    {
      "activations/layer0_attention_weight_max": 15.271612167358398,
      "activations/layer0_attention_weight_min": -14.74666690826416,
      "activations/layer10_attention_weight_max": 36.79304885864258,
      "activations/layer10_attention_weight_min": -33.506813049316406,
      "activations/layer11_attention_weight_max": 30.273122787475586,
      "activations/layer11_attention_weight_min": -29.891056060791016,
      "activations/layer12_attention_weight_max": 18.108701705932617,
      "activations/layer12_attention_weight_min": -29.034915924072266,
      "activations/layer13_attention_weight_max": 45.79627990722656,
      "activations/layer13_attention_weight_min": -35.20831298828125,
      "activations/layer14_attention_weight_max": 58.952938079833984,
      "activations/layer14_attention_weight_min": -42.389312744140625,
      "activations/layer15_attention_weight_max": 36.92369079589844,
      "activations/layer15_attention_weight_min": -31.173721313476562,
      "activations/layer16_attention_weight_max": 30.776809692382812,
      "activations/layer16_attention_weight_min": -26.175865173339844,
      "activations/layer17_attention_weight_max": 50.513389587402344,
      "activations/layer17_attention_weight_min": -39.528465270996094,
      "activations/layer18_attention_weight_max": 41.34811019897461,
      "activations/layer18_attention_weight_min": -33.50426483154297,
      "activations/layer19_attention_weight_max": 20.290559768676758,
      "activations/layer19_attention_weight_min": -20.621610641479492,
      "activations/layer1_attention_weight_max": 16.716203689575195,
      "activations/layer1_attention_weight_min": -12.890711784362793,
      "activations/layer20_attention_weight_max": 21.258378982543945,
      "activations/layer20_attention_weight_min": -20.69595718383789,
      "activations/layer21_attention_weight_max": 37.3464241027832,
      "activations/layer21_attention_weight_min": -24.376541137695312,
      "activations/layer22_attention_weight_max": 32.6575813293457,
      "activations/layer22_attention_weight_min": -24.967655181884766,
      "activations/layer23_attention_weight_max": 30.07086181640625,
      "activations/layer23_attention_weight_min": -21.450817108154297,
      "activations/layer2_attention_weight_max": 30.401172637939453,
      "activations/layer2_attention_weight_min": -29.490432739257812,
      "activations/layer3_attention_weight_max": 83.52357482910156,
      "activations/layer3_attention_weight_min": -84.31200408935547,
      "activations/layer4_attention_weight_max": 100.58267974853516,
      "activations/layer4_attention_weight_min": -91.08976745605469,
      "activations/layer5_attention_weight_max": 91.92544555664062,
      "activations/layer5_attention_weight_min": -83.10821533203125,
      "activations/layer6_attention_weight_max": 55.80134963989258,
      "activations/layer6_attention_weight_min": -51.04659652709961,
      "activations/layer7_attention_weight_max": 69.48194885253906,
      "activations/layer7_attention_weight_min": -65.21854400634766,
      "activations/layer8_attention_weight_max": 42.25109100341797,
      "activations/layer8_attention_weight_min": -42.316993713378906,
      "activations/layer9_attention_weight_max": 47.71218490600586,
      "activations/layer9_attention_weight_min": -47.21150588989258,
      "epoch": 6.41,
      "learning_rate": 0.00010977424242424242,
      "loss": 2.8122,
      "step": 110300
    },
    {
      "activations/layer0_attention_weight_max": 15.006633758544922,
      "activations/layer0_attention_weight_min": -14.93133544921875,
      "activations/layer10_attention_weight_max": 37.03119659423828,
      "activations/layer10_attention_weight_min": -35.99699783325195,
      "activations/layer11_attention_weight_max": 28.121984481811523,
      "activations/layer11_attention_weight_min": -28.383512496948242,
      "activations/layer12_attention_weight_max": 20.17856788635254,
      "activations/layer12_attention_weight_min": -26.152196884155273,
      "activations/layer13_attention_weight_max": 49.5067024230957,
      "activations/layer13_attention_weight_min": -37.409324645996094,
      "activations/layer14_attention_weight_max": 41.65367126464844,
      "activations/layer14_attention_weight_min": -29.901966094970703,
      "activations/layer15_attention_weight_max": 41.116546630859375,
      "activations/layer15_attention_weight_min": -30.76291275024414,
      "activations/layer16_attention_weight_max": 33.53224563598633,
      "activations/layer16_attention_weight_min": -27.529125213623047,
      "activations/layer17_attention_weight_max": 57.67822265625,
      "activations/layer17_attention_weight_min": -41.61357498168945,
      "activations/layer18_attention_weight_max": 54.00487518310547,
      "activations/layer18_attention_weight_min": -35.05194854736328,
      "activations/layer19_attention_weight_max": 26.327545166015625,
      "activations/layer19_attention_weight_min": -21.55240249633789,
      "activations/layer1_attention_weight_max": 16.850191116333008,
      "activations/layer1_attention_weight_min": -14.367110252380371,
      "activations/layer20_attention_weight_max": 25.884815216064453,
      "activations/layer20_attention_weight_min": -21.34974479675293,
      "activations/layer21_attention_weight_max": 44.851776123046875,
      "activations/layer21_attention_weight_min": -24.234464645385742,
      "activations/layer22_attention_weight_max": 35.20737075805664,
      "activations/layer22_attention_weight_min": -25.38909339904785,
      "activations/layer23_attention_weight_max": 33.74313735961914,
      "activations/layer23_attention_weight_min": -18.314638137817383,
      "activations/layer2_attention_weight_max": 30.258007049560547,
      "activations/layer2_attention_weight_min": -27.342832565307617,
      "activations/layer3_attention_weight_max": 88.91473388671875,
      "activations/layer3_attention_weight_min": -92.01473999023438,
      "activations/layer4_attention_weight_max": 102.74523162841797,
      "activations/layer4_attention_weight_min": -95.47734832763672,
      "activations/layer5_attention_weight_max": 96.29450988769531,
      "activations/layer5_attention_weight_min": -87.32283020019531,
      "activations/layer6_attention_weight_max": 59.69316101074219,
      "activations/layer6_attention_weight_min": -53.29606246948242,
      "activations/layer7_attention_weight_max": 78.88680267333984,
      "activations/layer7_attention_weight_min": -68.77803802490234,
      "activations/layer8_attention_weight_max": 46.5993766784668,
      "activations/layer8_attention_weight_min": -46.9056282043457,
      "activations/layer9_attention_weight_max": 49.8687744140625,
      "activations/layer9_attention_weight_min": -48.9433708190918,
      "epoch": 6.41,
      "learning_rate": 0.00010975530303030302,
      "loss": 2.8289,
      "step": 110350
    },
    {
      "activations/layer0_attention_weight_max": 14.865448951721191,
      "activations/layer0_attention_weight_min": -15.195401191711426,
      "activations/layer10_attention_weight_max": 38.60940170288086,
      "activations/layer10_attention_weight_min": -32.42546844482422,
      "activations/layer11_attention_weight_max": 29.152599334716797,
      "activations/layer11_attention_weight_min": -27.93037223815918,
      "activations/layer12_attention_weight_max": 23.342966079711914,
      "activations/layer12_attention_weight_min": -24.43364143371582,
      "activations/layer13_attention_weight_max": 44.681148529052734,
      "activations/layer13_attention_weight_min": -31.29192543029785,
      "activations/layer14_attention_weight_max": 44.620426177978516,
      "activations/layer14_attention_weight_min": -30.88908576965332,
      "activations/layer15_attention_weight_max": 36.208885192871094,
      "activations/layer15_attention_weight_min": -29.360261917114258,
      "activations/layer16_attention_weight_max": 35.09943771362305,
      "activations/layer16_attention_weight_min": -25.46902084350586,
      "activations/layer17_attention_weight_max": 55.229637145996094,
      "activations/layer17_attention_weight_min": -40.04481887817383,
      "activations/layer18_attention_weight_max": 44.99917984008789,
      "activations/layer18_attention_weight_min": -35.251895904541016,
      "activations/layer19_attention_weight_max": 20.641403198242188,
      "activations/layer19_attention_weight_min": -20.580976486206055,
      "activations/layer1_attention_weight_max": 16.515663146972656,
      "activations/layer1_attention_weight_min": -14.782380104064941,
      "activations/layer20_attention_weight_max": 20.329748153686523,
      "activations/layer20_attention_weight_min": -20.289562225341797,
      "activations/layer21_attention_weight_max": 36.347190856933594,
      "activations/layer21_attention_weight_min": -22.67185401916504,
      "activations/layer22_attention_weight_max": 33.402740478515625,
      "activations/layer22_attention_weight_min": -25.57295799255371,
      "activations/layer23_attention_weight_max": 33.919158935546875,
      "activations/layer23_attention_weight_min": -19.500720977783203,
      "activations/layer2_attention_weight_max": 29.917034149169922,
      "activations/layer2_attention_weight_min": -27.440994262695312,
      "activations/layer3_attention_weight_max": 87.55670166015625,
      "activations/layer3_attention_weight_min": -81.13172149658203,
      "activations/layer4_attention_weight_max": 103.63335418701172,
      "activations/layer4_attention_weight_min": -90.89270782470703,
      "activations/layer5_attention_weight_max": 94.71002197265625,
      "activations/layer5_attention_weight_min": -86.43276977539062,
      "activations/layer6_attention_weight_max": 55.5129508972168,
      "activations/layer6_attention_weight_min": -50.15092468261719,
      "activations/layer7_attention_weight_max": 75.94388580322266,
      "activations/layer7_attention_weight_min": -69.79778289794922,
      "activations/layer8_attention_weight_max": 43.7993278503418,
      "activations/layer8_attention_weight_min": -40.260597229003906,
      "activations/layer9_attention_weight_max": 55.2713737487793,
      "activations/layer9_attention_weight_min": -47.679317474365234,
      "epoch": 6.41,
      "learning_rate": 0.00010973636363636362,
      "loss": 2.8328,
      "step": 110400
    },
    {
      "activations/layer0_attention_weight_max": 14.83466625213623,
      "activations/layer0_attention_weight_min": -15.409897804260254,
      "activations/layer10_attention_weight_max": 40.31060791015625,
      "activations/layer10_attention_weight_min": -34.396278381347656,
      "activations/layer11_attention_weight_max": 35.12043762207031,
      "activations/layer11_attention_weight_min": -28.9033203125,
      "activations/layer12_attention_weight_max": 20.685537338256836,
      "activations/layer12_attention_weight_min": -25.12363052368164,
      "activations/layer13_attention_weight_max": 61.7347297668457,
      "activations/layer13_attention_weight_min": -37.5077018737793,
      "activations/layer14_attention_weight_max": 59.33229446411133,
      "activations/layer14_attention_weight_min": -31.972532272338867,
      "activations/layer15_attention_weight_max": 42.3377685546875,
      "activations/layer15_attention_weight_min": -29.41184425354004,
      "activations/layer16_attention_weight_max": 37.787471771240234,
      "activations/layer16_attention_weight_min": -28.32322883605957,
      "activations/layer17_attention_weight_max": 64.31652069091797,
      "activations/layer17_attention_weight_min": -42.542972564697266,
      "activations/layer18_attention_weight_max": 55.17359161376953,
      "activations/layer18_attention_weight_min": -35.16332244873047,
      "activations/layer19_attention_weight_max": 25.37726402282715,
      "activations/layer19_attention_weight_min": -21.742794036865234,
      "activations/layer1_attention_weight_max": 18.763721466064453,
      "activations/layer1_attention_weight_min": -17.264617919921875,
      "activations/layer20_attention_weight_max": 27.833354949951172,
      "activations/layer20_attention_weight_min": -20.976457595825195,
      "activations/layer21_attention_weight_max": 56.596466064453125,
      "activations/layer21_attention_weight_min": -27.549545288085938,
      "activations/layer22_attention_weight_max": 36.266239166259766,
      "activations/layer22_attention_weight_min": -24.21849250793457,
      "activations/layer23_attention_weight_max": 38.35956573486328,
      "activations/layer23_attention_weight_min": -20.878076553344727,
      "activations/layer2_attention_weight_max": 31.267230987548828,
      "activations/layer2_attention_weight_min": -28.693805694580078,
      "activations/layer3_attention_weight_max": 86.1216812133789,
      "activations/layer3_attention_weight_min": -88.05388641357422,
      "activations/layer4_attention_weight_max": 102.7383804321289,
      "activations/layer4_attention_weight_min": -94.11580657958984,
      "activations/layer5_attention_weight_max": 96.19924926757812,
      "activations/layer5_attention_weight_min": -88.29476165771484,
      "activations/layer6_attention_weight_max": 57.31743240356445,
      "activations/layer6_attention_weight_min": -54.61821365356445,
      "activations/layer7_attention_weight_max": 78.18252563476562,
      "activations/layer7_attention_weight_min": -69.34918212890625,
      "activations/layer8_attention_weight_max": 44.820499420166016,
      "activations/layer8_attention_weight_min": -44.601097106933594,
      "activations/layer9_attention_weight_max": 53.499568939208984,
      "activations/layer9_attention_weight_min": -49.343318939208984,
      "epoch": 6.42,
      "learning_rate": 0.00010971742424242424,
      "loss": 2.8293,
      "step": 110450
    },
    {
      "activations/layer0_attention_weight_max": 14.876914978027344,
      "activations/layer0_attention_weight_min": -15.409907341003418,
      "activations/layer10_attention_weight_max": 41.40252685546875,
      "activations/layer10_attention_weight_min": -36.38460159301758,
      "activations/layer11_attention_weight_max": 32.8451042175293,
      "activations/layer11_attention_weight_min": -30.213481903076172,
      "activations/layer12_attention_weight_max": 23.686973571777344,
      "activations/layer12_attention_weight_min": -24.76007652282715,
      "activations/layer13_attention_weight_max": 67.8950424194336,
      "activations/layer13_attention_weight_min": -31.985719680786133,
      "activations/layer14_attention_weight_max": 70.97831726074219,
      "activations/layer14_attention_weight_min": -32.763389587402344,
      "activations/layer15_attention_weight_max": 54.195899963378906,
      "activations/layer15_attention_weight_min": -29.62470054626465,
      "activations/layer16_attention_weight_max": 36.351566314697266,
      "activations/layer16_attention_weight_min": -27.138805389404297,
      "activations/layer17_attention_weight_max": 57.42298126220703,
      "activations/layer17_attention_weight_min": -42.43303298950195,
      "activations/layer18_attention_weight_max": 52.59360122680664,
      "activations/layer18_attention_weight_min": -34.560970306396484,
      "activations/layer19_attention_weight_max": 22.325416564941406,
      "activations/layer19_attention_weight_min": -22.338350296020508,
      "activations/layer1_attention_weight_max": 17.233922958374023,
      "activations/layer1_attention_weight_min": -13.44941520690918,
      "activations/layer20_attention_weight_max": 23.744352340698242,
      "activations/layer20_attention_weight_min": -20.782197952270508,
      "activations/layer21_attention_weight_max": 41.03550720214844,
      "activations/layer21_attention_weight_min": -25.914060592651367,
      "activations/layer22_attention_weight_max": 30.55382537841797,
      "activations/layer22_attention_weight_min": -25.661937713623047,
      "activations/layer23_attention_weight_max": 34.177703857421875,
      "activations/layer23_attention_weight_min": -20.32932472229004,
      "activations/layer2_attention_weight_max": 32.378639221191406,
      "activations/layer2_attention_weight_min": -29.95752716064453,
      "activations/layer3_attention_weight_max": 83.1500015258789,
      "activations/layer3_attention_weight_min": -82.03541564941406,
      "activations/layer4_attention_weight_max": 100.05721282958984,
      "activations/layer4_attention_weight_min": -94.96105194091797,
      "activations/layer5_attention_weight_max": 94.4333724975586,
      "activations/layer5_attention_weight_min": -89.04698944091797,
      "activations/layer6_attention_weight_max": 59.17866134643555,
      "activations/layer6_attention_weight_min": -53.2525520324707,
      "activations/layer7_attention_weight_max": 76.51771545410156,
      "activations/layer7_attention_weight_min": -69.20877838134766,
      "activations/layer8_attention_weight_max": 48.95180892944336,
      "activations/layer8_attention_weight_min": -44.8819465637207,
      "activations/layer9_attention_weight_max": 58.05690383911133,
      "activations/layer9_attention_weight_min": -47.81897735595703,
      "epoch": 6.42,
      "learning_rate": 0.00010969848484848485,
      "loss": 2.8252,
      "step": 110500
    },
    {
      "activations/layer0_attention_weight_max": 15.36218547821045,
      "activations/layer0_attention_weight_min": -15.19663143157959,
      "activations/layer10_attention_weight_max": 37.749237060546875,
      "activations/layer10_attention_weight_min": -34.820648193359375,
      "activations/layer11_attention_weight_max": 31.547828674316406,
      "activations/layer11_attention_weight_min": -29.005645751953125,
      "activations/layer12_attention_weight_max": 21.336463928222656,
      "activations/layer12_attention_weight_min": -23.644805908203125,
      "activations/layer13_attention_weight_max": 43.99440383911133,
      "activations/layer13_attention_weight_min": -34.18375015258789,
      "activations/layer14_attention_weight_max": 37.69203186035156,
      "activations/layer14_attention_weight_min": -29.55832290649414,
      "activations/layer15_attention_weight_max": 40.13849639892578,
      "activations/layer15_attention_weight_min": -28.91598892211914,
      "activations/layer16_attention_weight_max": 36.99662780761719,
      "activations/layer16_attention_weight_min": -27.497472763061523,
      "activations/layer17_attention_weight_max": 53.646785736083984,
      "activations/layer17_attention_weight_min": -38.87068557739258,
      "activations/layer18_attention_weight_max": 47.08613204956055,
      "activations/layer18_attention_weight_min": -34.2366943359375,
      "activations/layer19_attention_weight_max": 23.960561752319336,
      "activations/layer19_attention_weight_min": -19.946693420410156,
      "activations/layer1_attention_weight_max": 17.566007614135742,
      "activations/layer1_attention_weight_min": -14.887835502624512,
      "activations/layer20_attention_weight_max": 21.446609497070312,
      "activations/layer20_attention_weight_min": -22.826780319213867,
      "activations/layer21_attention_weight_max": 39.259220123291016,
      "activations/layer21_attention_weight_min": -24.726579666137695,
      "activations/layer22_attention_weight_max": 32.987281799316406,
      "activations/layer22_attention_weight_min": -26.26866340637207,
      "activations/layer23_attention_weight_max": 29.71021842956543,
      "activations/layer23_attention_weight_min": -19.489883422851562,
      "activations/layer2_attention_weight_max": 31.216445922851562,
      "activations/layer2_attention_weight_min": -30.12430191040039,
      "activations/layer3_attention_weight_max": 85.14568328857422,
      "activations/layer3_attention_weight_min": -87.6190414428711,
      "activations/layer4_attention_weight_max": 99.42044067382812,
      "activations/layer4_attention_weight_min": -92.95166015625,
      "activations/layer5_attention_weight_max": 90.71150970458984,
      "activations/layer5_attention_weight_min": -83.9244155883789,
      "activations/layer6_attention_weight_max": 55.31221008300781,
      "activations/layer6_attention_weight_min": -53.05854797363281,
      "activations/layer7_attention_weight_max": 74.54380798339844,
      "activations/layer7_attention_weight_min": -66.5506362915039,
      "activations/layer8_attention_weight_max": 43.51169204711914,
      "activations/layer8_attention_weight_min": -41.566558837890625,
      "activations/layer9_attention_weight_max": 52.943904876708984,
      "activations/layer9_attention_weight_min": -47.04707717895508,
      "epoch": 6.42,
      "learning_rate": 0.00010967954545454544,
      "loss": 2.8322,
      "step": 110550
    },
    {
      "activations/layer0_attention_weight_max": 16.43025779724121,
      "activations/layer0_attention_weight_min": -14.755180358886719,
      "activations/layer10_attention_weight_max": 36.09206008911133,
      "activations/layer10_attention_weight_min": -32.64539337158203,
      "activations/layer11_attention_weight_max": 29.15650749206543,
      "activations/layer11_attention_weight_min": -28.620502471923828,
      "activations/layer12_attention_weight_max": 21.659976959228516,
      "activations/layer12_attention_weight_min": -26.556312561035156,
      "activations/layer13_attention_weight_max": 42.75364685058594,
      "activations/layer13_attention_weight_min": -30.42999839782715,
      "activations/layer14_attention_weight_max": 47.81795883178711,
      "activations/layer14_attention_weight_min": -31.188642501831055,
      "activations/layer15_attention_weight_max": 38.05937576293945,
      "activations/layer15_attention_weight_min": -30.554731369018555,
      "activations/layer16_attention_weight_max": 30.18695068359375,
      "activations/layer16_attention_weight_min": -26.507797241210938,
      "activations/layer17_attention_weight_max": 53.5608024597168,
      "activations/layer17_attention_weight_min": -41.892669677734375,
      "activations/layer18_attention_weight_max": 46.27062225341797,
      "activations/layer18_attention_weight_min": -35.3552360534668,
      "activations/layer19_attention_weight_max": 20.31684684753418,
      "activations/layer19_attention_weight_min": -19.690155029296875,
      "activations/layer1_attention_weight_max": 17.273052215576172,
      "activations/layer1_attention_weight_min": -15.2642822265625,
      "activations/layer20_attention_weight_max": 21.35574722290039,
      "activations/layer20_attention_weight_min": -20.687397003173828,
      "activations/layer21_attention_weight_max": 36.64887237548828,
      "activations/layer21_attention_weight_min": -25.763517379760742,
      "activations/layer22_attention_weight_max": 33.92756652832031,
      "activations/layer22_attention_weight_min": -24.660112380981445,
      "activations/layer23_attention_weight_max": 28.98744773864746,
      "activations/layer23_attention_weight_min": -19.704666137695312,
      "activations/layer2_attention_weight_max": 31.865760803222656,
      "activations/layer2_attention_weight_min": -29.794776916503906,
      "activations/layer3_attention_weight_max": 85.62954711914062,
      "activations/layer3_attention_weight_min": -93.41883850097656,
      "activations/layer4_attention_weight_max": 95.05921936035156,
      "activations/layer4_attention_weight_min": -92.94551086425781,
      "activations/layer5_attention_weight_max": 92.09844207763672,
      "activations/layer5_attention_weight_min": -83.58881378173828,
      "activations/layer6_attention_weight_max": 54.182762145996094,
      "activations/layer6_attention_weight_min": -52.50938415527344,
      "activations/layer7_attention_weight_max": 75.90641021728516,
      "activations/layer7_attention_weight_min": -67.72785949707031,
      "activations/layer8_attention_weight_max": 42.3591194152832,
      "activations/layer8_attention_weight_min": -42.11880874633789,
      "activations/layer9_attention_weight_max": 47.364437103271484,
      "activations/layer9_attention_weight_min": -46.19415283203125,
      "epoch": 6.43,
      "learning_rate": 0.00010966060606060606,
      "loss": 2.8187,
      "step": 110600
    },
    {
      "activations/layer0_attention_weight_max": 15.021484375,
      "activations/layer0_attention_weight_min": -15.046589851379395,
      "activations/layer10_attention_weight_max": 39.432273864746094,
      "activations/layer10_attention_weight_min": -34.12965774536133,
      "activations/layer11_attention_weight_max": 29.92011260986328,
      "activations/layer11_attention_weight_min": -29.226844787597656,
      "activations/layer12_attention_weight_max": 23.38523292541504,
      "activations/layer12_attention_weight_min": -24.517492294311523,
      "activations/layer13_attention_weight_max": 44.89885330200195,
      "activations/layer13_attention_weight_min": -30.193361282348633,
      "activations/layer14_attention_weight_max": 44.42057418823242,
      "activations/layer14_attention_weight_min": -30.002185821533203,
      "activations/layer15_attention_weight_max": 40.81538391113281,
      "activations/layer15_attention_weight_min": -29.82301902770996,
      "activations/layer16_attention_weight_max": 38.466495513916016,
      "activations/layer16_attention_weight_min": -26.900285720825195,
      "activations/layer17_attention_weight_max": 57.67472457885742,
      "activations/layer17_attention_weight_min": -43.249366760253906,
      "activations/layer18_attention_weight_max": 54.60569381713867,
      "activations/layer18_attention_weight_min": -36.38994598388672,
      "activations/layer19_attention_weight_max": 22.813091278076172,
      "activations/layer19_attention_weight_min": -19.875043869018555,
      "activations/layer1_attention_weight_max": 17.694887161254883,
      "activations/layer1_attention_weight_min": -13.681549072265625,
      "activations/layer20_attention_weight_max": 25.82524299621582,
      "activations/layer20_attention_weight_min": -22.377830505371094,
      "activations/layer21_attention_weight_max": 37.79762268066406,
      "activations/layer21_attention_weight_min": -25.003990173339844,
      "activations/layer22_attention_weight_max": 35.737483978271484,
      "activations/layer22_attention_weight_min": -25.91223907470703,
      "activations/layer23_attention_weight_max": 36.694725036621094,
      "activations/layer23_attention_weight_min": -20.27450180053711,
      "activations/layer2_attention_weight_max": 29.13311767578125,
      "activations/layer2_attention_weight_min": -27.39803695678711,
      "activations/layer3_attention_weight_max": 81.27384948730469,
      "activations/layer3_attention_weight_min": -81.04107666015625,
      "activations/layer4_attention_weight_max": 95.83926391601562,
      "activations/layer4_attention_weight_min": -90.03192138671875,
      "activations/layer5_attention_weight_max": 90.91253662109375,
      "activations/layer5_attention_weight_min": -82.66046142578125,
      "activations/layer6_attention_weight_max": 54.584434509277344,
      "activations/layer6_attention_weight_min": -52.551109313964844,
      "activations/layer7_attention_weight_max": 75.39373016357422,
      "activations/layer7_attention_weight_min": -69.53685760498047,
      "activations/layer8_attention_weight_max": 42.144004821777344,
      "activations/layer8_attention_weight_min": -46.21843338012695,
      "activations/layer9_attention_weight_max": 50.16218948364258,
      "activations/layer9_attention_weight_min": -50.12124252319336,
      "epoch": 6.43,
      "learning_rate": 0.00010964166666666664,
      "loss": 2.8257,
      "step": 110650
    },
    {
      "activations/layer0_attention_weight_max": 15.41821002960205,
      "activations/layer0_attention_weight_min": -15.090325355529785,
      "activations/layer10_attention_weight_max": 34.72880554199219,
      "activations/layer10_attention_weight_min": -33.10742950439453,
      "activations/layer11_attention_weight_max": 27.609649658203125,
      "activations/layer11_attention_weight_min": -28.274612426757812,
      "activations/layer12_attention_weight_max": 19.49253273010254,
      "activations/layer12_attention_weight_min": -25.324512481689453,
      "activations/layer13_attention_weight_max": 42.64325714111328,
      "activations/layer13_attention_weight_min": -32.85152816772461,
      "activations/layer14_attention_weight_max": 36.648887634277344,
      "activations/layer14_attention_weight_min": -28.4033145904541,
      "activations/layer15_attention_weight_max": 34.52275848388672,
      "activations/layer15_attention_weight_min": -28.097919464111328,
      "activations/layer16_attention_weight_max": 33.76539611816406,
      "activations/layer16_attention_weight_min": -25.07469367980957,
      "activations/layer17_attention_weight_max": 49.97621154785156,
      "activations/layer17_attention_weight_min": -39.40736770629883,
      "activations/layer18_attention_weight_max": 44.335838317871094,
      "activations/layer18_attention_weight_min": -33.41706466674805,
      "activations/layer19_attention_weight_max": 20.058130264282227,
      "activations/layer19_attention_weight_min": -19.484798431396484,
      "activations/layer1_attention_weight_max": 17.777908325195312,
      "activations/layer1_attention_weight_min": -16.20205307006836,
      "activations/layer20_attention_weight_max": 19.9412841796875,
      "activations/layer20_attention_weight_min": -21.13060760498047,
      "activations/layer21_attention_weight_max": 31.5327205657959,
      "activations/layer21_attention_weight_min": -24.385141372680664,
      "activations/layer22_attention_weight_max": 28.54851531982422,
      "activations/layer22_attention_weight_min": -25.187652587890625,
      "activations/layer23_attention_weight_max": 30.22208023071289,
      "activations/layer23_attention_weight_min": -22.921630859375,
      "activations/layer2_attention_weight_max": 30.15290069580078,
      "activations/layer2_attention_weight_min": -28.44995880126953,
      "activations/layer3_attention_weight_max": 83.5983657836914,
      "activations/layer3_attention_weight_min": -84.7691421508789,
      "activations/layer4_attention_weight_max": 94.3705062866211,
      "activations/layer4_attention_weight_min": -90.13031005859375,
      "activations/layer5_attention_weight_max": 87.90410614013672,
      "activations/layer5_attention_weight_min": -82.44437408447266,
      "activations/layer6_attention_weight_max": 54.252159118652344,
      "activations/layer6_attention_weight_min": -50.146217346191406,
      "activations/layer7_attention_weight_max": 67.49908447265625,
      "activations/layer7_attention_weight_min": -68.5447998046875,
      "activations/layer8_attention_weight_max": 40.74159240722656,
      "activations/layer8_attention_weight_min": -44.479061126708984,
      "activations/layer9_attention_weight_max": 45.49961471557617,
      "activations/layer9_attention_weight_min": -49.94050598144531,
      "epoch": 6.43,
      "learning_rate": 0.00010962272727272726,
      "loss": 2.8403,
      "step": 110700
    },
    {
      "activations/layer0_attention_weight_max": 16.67656135559082,
      "activations/layer0_attention_weight_min": -15.090030670166016,
      "activations/layer10_attention_weight_max": 37.23842239379883,
      "activations/layer10_attention_weight_min": -34.252098083496094,
      "activations/layer11_attention_weight_max": 29.873821258544922,
      "activations/layer11_attention_weight_min": -28.452655792236328,
      "activations/layer12_attention_weight_max": 28.706565856933594,
      "activations/layer12_attention_weight_min": -25.7067928314209,
      "activations/layer13_attention_weight_max": 49.42182540893555,
      "activations/layer13_attention_weight_min": -34.321170806884766,
      "activations/layer14_attention_weight_max": 42.53962326049805,
      "activations/layer14_attention_weight_min": -30.402402877807617,
      "activations/layer15_attention_weight_max": 44.29134750366211,
      "activations/layer15_attention_weight_min": -30.805376052856445,
      "activations/layer16_attention_weight_max": 31.965787887573242,
      "activations/layer16_attention_weight_min": -26.271074295043945,
      "activations/layer17_attention_weight_max": 61.96406936645508,
      "activations/layer17_attention_weight_min": -43.63262939453125,
      "activations/layer18_attention_weight_max": 50.23037338256836,
      "activations/layer18_attention_weight_min": -36.75202560424805,
      "activations/layer19_attention_weight_max": 24.792098999023438,
      "activations/layer19_attention_weight_min": -21.256853103637695,
      "activations/layer1_attention_weight_max": 17.063770294189453,
      "activations/layer1_attention_weight_min": -14.381792068481445,
      "activations/layer20_attention_weight_max": 23.27120590209961,
      "activations/layer20_attention_weight_min": -19.68966293334961,
      "activations/layer21_attention_weight_max": 40.27091979980469,
      "activations/layer21_attention_weight_min": -27.674236297607422,
      "activations/layer22_attention_weight_max": 32.84836959838867,
      "activations/layer22_attention_weight_min": -25.019229888916016,
      "activations/layer23_attention_weight_max": 30.949420928955078,
      "activations/layer23_attention_weight_min": -20.903736114501953,
      "activations/layer2_attention_weight_max": 29.725208282470703,
      "activations/layer2_attention_weight_min": -29.306867599487305,
      "activations/layer3_attention_weight_max": 82.0667495727539,
      "activations/layer3_attention_weight_min": -84.22624969482422,
      "activations/layer4_attention_weight_max": 99.70104217529297,
      "activations/layer4_attention_weight_min": -97.30995178222656,
      "activations/layer5_attention_weight_max": 94.83171844482422,
      "activations/layer5_attention_weight_min": -87.94026947021484,
      "activations/layer6_attention_weight_max": 58.163536071777344,
      "activations/layer6_attention_weight_min": -54.60968780517578,
      "activations/layer7_attention_weight_max": 79.05781555175781,
      "activations/layer7_attention_weight_min": -72.32772064208984,
      "activations/layer8_attention_weight_max": 45.7916374206543,
      "activations/layer8_attention_weight_min": -44.99224090576172,
      "activations/layer9_attention_weight_max": 51.82606887817383,
      "activations/layer9_attention_weight_min": -49.95717239379883,
      "epoch": 6.44,
      "learning_rate": 0.00010960378787878787,
      "loss": 2.8279,
      "step": 110750
    },
    {
      "activations/layer0_attention_weight_max": 15.479540824890137,
      "activations/layer0_attention_weight_min": -15.546113014221191,
      "activations/layer10_attention_weight_max": 36.389137268066406,
      "activations/layer10_attention_weight_min": -35.36907958984375,
      "activations/layer11_attention_weight_max": 30.632232666015625,
      "activations/layer11_attention_weight_min": -29.03760528564453,
      "activations/layer12_attention_weight_max": 21.645938873291016,
      "activations/layer12_attention_weight_min": -27.370271682739258,
      "activations/layer13_attention_weight_max": 42.086631774902344,
      "activations/layer13_attention_weight_min": -33.46363830566406,
      "activations/layer14_attention_weight_max": 47.391231536865234,
      "activations/layer14_attention_weight_min": -28.207523345947266,
      "activations/layer15_attention_weight_max": 39.390480041503906,
      "activations/layer15_attention_weight_min": -29.19043731689453,
      "activations/layer16_attention_weight_max": 35.46042251586914,
      "activations/layer16_attention_weight_min": -28.201406478881836,
      "activations/layer17_attention_weight_max": 55.429054260253906,
      "activations/layer17_attention_weight_min": -39.948951721191406,
      "activations/layer18_attention_weight_max": 46.29337692260742,
      "activations/layer18_attention_weight_min": -33.917354583740234,
      "activations/layer19_attention_weight_max": 20.983110427856445,
      "activations/layer19_attention_weight_min": -21.204299926757812,
      "activations/layer1_attention_weight_max": 17.25907325744629,
      "activations/layer1_attention_weight_min": -14.547744750976562,
      "activations/layer20_attention_weight_max": 21.9770565032959,
      "activations/layer20_attention_weight_min": -22.470766067504883,
      "activations/layer21_attention_weight_max": 38.63541030883789,
      "activations/layer21_attention_weight_min": -24.714237213134766,
      "activations/layer22_attention_weight_max": 32.877769470214844,
      "activations/layer22_attention_weight_min": -25.496479034423828,
      "activations/layer23_attention_weight_max": 29.577022552490234,
      "activations/layer23_attention_weight_min": -20.4666805267334,
      "activations/layer2_attention_weight_max": 29.763381958007812,
      "activations/layer2_attention_weight_min": -26.944320678710938,
      "activations/layer3_attention_weight_max": 82.07131958007812,
      "activations/layer3_attention_weight_min": -82.25371551513672,
      "activations/layer4_attention_weight_max": 95.35350799560547,
      "activations/layer4_attention_weight_min": -89.62731170654297,
      "activations/layer5_attention_weight_max": 89.12091064453125,
      "activations/layer5_attention_weight_min": -82.37750244140625,
      "activations/layer6_attention_weight_max": 54.256038665771484,
      "activations/layer6_attention_weight_min": -49.67552947998047,
      "activations/layer7_attention_weight_max": 69.87873077392578,
      "activations/layer7_attention_weight_min": -67.12088775634766,
      "activations/layer8_attention_weight_max": 44.82695770263672,
      "activations/layer8_attention_weight_min": -42.001766204833984,
      "activations/layer9_attention_weight_max": 50.112815856933594,
      "activations/layer9_attention_weight_min": -47.00507736206055,
      "epoch": 6.44,
      "learning_rate": 0.00010958484848484848,
      "loss": 2.8435,
      "step": 110800
    },
    {
      "activations/layer0_attention_weight_max": 14.665080070495605,
      "activations/layer0_attention_weight_min": -14.965383529663086,
      "activations/layer10_attention_weight_max": 36.030860900878906,
      "activations/layer10_attention_weight_min": -37.34804916381836,
      "activations/layer11_attention_weight_max": 28.864498138427734,
      "activations/layer11_attention_weight_min": -29.446399688720703,
      "activations/layer12_attention_weight_max": 21.193378448486328,
      "activations/layer12_attention_weight_min": -20.775245666503906,
      "activations/layer13_attention_weight_max": 45.41791534423828,
      "activations/layer13_attention_weight_min": -34.85860824584961,
      "activations/layer14_attention_weight_max": 43.922000885009766,
      "activations/layer14_attention_weight_min": -33.36069107055664,
      "activations/layer15_attention_weight_max": 38.14842224121094,
      "activations/layer15_attention_weight_min": -31.4644718170166,
      "activations/layer16_attention_weight_max": 34.7784538269043,
      "activations/layer16_attention_weight_min": -26.40094757080078,
      "activations/layer17_attention_weight_max": 56.387569427490234,
      "activations/layer17_attention_weight_min": -46.064178466796875,
      "activations/layer18_attention_weight_max": 51.2697639465332,
      "activations/layer18_attention_weight_min": -37.669002532958984,
      "activations/layer19_attention_weight_max": 24.159984588623047,
      "activations/layer19_attention_weight_min": -21.12958335876465,
      "activations/layer1_attention_weight_max": 17.101579666137695,
      "activations/layer1_attention_weight_min": -14.187200546264648,
      "activations/layer20_attention_weight_max": 25.55709457397461,
      "activations/layer20_attention_weight_min": -23.084184646606445,
      "activations/layer21_attention_weight_max": 39.84212112426758,
      "activations/layer21_attention_weight_min": -27.02300262451172,
      "activations/layer22_attention_weight_max": 37.949462890625,
      "activations/layer22_attention_weight_min": -26.18086814880371,
      "activations/layer23_attention_weight_max": 31.624555587768555,
      "activations/layer23_attention_weight_min": -18.668176651000977,
      "activations/layer2_attention_weight_max": 30.40389060974121,
      "activations/layer2_attention_weight_min": -29.026695251464844,
      "activations/layer3_attention_weight_max": 84.8425064086914,
      "activations/layer3_attention_weight_min": -93.08073425292969,
      "activations/layer4_attention_weight_max": 96.99922943115234,
      "activations/layer4_attention_weight_min": -96.60694885253906,
      "activations/layer5_attention_weight_max": 91.20250701904297,
      "activations/layer5_attention_weight_min": -87.15419006347656,
      "activations/layer6_attention_weight_max": 53.81895446777344,
      "activations/layer6_attention_weight_min": -54.26473617553711,
      "activations/layer7_attention_weight_max": 74.00397491455078,
      "activations/layer7_attention_weight_min": -71.4767074584961,
      "activations/layer8_attention_weight_max": 44.73832321166992,
      "activations/layer8_attention_weight_min": -43.86172103881836,
      "activations/layer9_attention_weight_max": 55.00333786010742,
      "activations/layer9_attention_weight_min": -51.488746643066406,
      "epoch": 6.44,
      "learning_rate": 0.00010956590909090908,
      "loss": 2.8324,
      "step": 110850
    },
    {
      "activations/layer0_attention_weight_max": 15.137521743774414,
      "activations/layer0_attention_weight_min": -15.634921073913574,
      "activations/layer10_attention_weight_max": 38.4880485534668,
      "activations/layer10_attention_weight_min": -34.75347900390625,
      "activations/layer11_attention_weight_max": 29.876346588134766,
      "activations/layer11_attention_weight_min": -29.31990623474121,
      "activations/layer12_attention_weight_max": 20.223844528198242,
      "activations/layer12_attention_weight_min": -26.726821899414062,
      "activations/layer13_attention_weight_max": 46.85094451904297,
      "activations/layer13_attention_weight_min": -33.111328125,
      "activations/layer14_attention_weight_max": 41.01810836791992,
      "activations/layer14_attention_weight_min": -30.443506240844727,
      "activations/layer15_attention_weight_max": 41.8659782409668,
      "activations/layer15_attention_weight_min": -29.825145721435547,
      "activations/layer16_attention_weight_max": 33.82040786743164,
      "activations/layer16_attention_weight_min": -27.930009841918945,
      "activations/layer17_attention_weight_max": 57.26430130004883,
      "activations/layer17_attention_weight_min": -42.3139533996582,
      "activations/layer18_attention_weight_max": 50.20743942260742,
      "activations/layer18_attention_weight_min": -36.308631896972656,
      "activations/layer19_attention_weight_max": 24.343372344970703,
      "activations/layer19_attention_weight_min": -21.91059684753418,
      "activations/layer1_attention_weight_max": 18.141551971435547,
      "activations/layer1_attention_weight_min": -15.060468673706055,
      "activations/layer20_attention_weight_max": 19.838333129882812,
      "activations/layer20_attention_weight_min": -20.3482723236084,
      "activations/layer21_attention_weight_max": 40.56189727783203,
      "activations/layer21_attention_weight_min": -25.155426025390625,
      "activations/layer22_attention_weight_max": 32.92417526245117,
      "activations/layer22_attention_weight_min": -23.541053771972656,
      "activations/layer23_attention_weight_max": 32.46780014038086,
      "activations/layer23_attention_weight_min": -20.723892211914062,
      "activations/layer2_attention_weight_max": 30.124427795410156,
      "activations/layer2_attention_weight_min": -28.402084350585938,
      "activations/layer3_attention_weight_max": 83.4778823852539,
      "activations/layer3_attention_weight_min": -88.04149627685547,
      "activations/layer4_attention_weight_max": 98.7918472290039,
      "activations/layer4_attention_weight_min": -89.7018814086914,
      "activations/layer5_attention_weight_max": 91.0947494506836,
      "activations/layer5_attention_weight_min": -83.16000366210938,
      "activations/layer6_attention_weight_max": 54.967185974121094,
      "activations/layer6_attention_weight_min": -51.364383697509766,
      "activations/layer7_attention_weight_max": 72.13298797607422,
      "activations/layer7_attention_weight_min": -67.64391326904297,
      "activations/layer8_attention_weight_max": 45.0264778137207,
      "activations/layer8_attention_weight_min": -43.50151443481445,
      "activations/layer9_attention_weight_max": 50.25141525268555,
      "activations/layer9_attention_weight_min": -46.391048431396484,
      "epoch": 6.44,
      "learning_rate": 0.00010954696969696969,
      "loss": 2.8276,
      "step": 110900
    },
    {
      "activations/layer0_attention_weight_max": 15.349210739135742,
      "activations/layer0_attention_weight_min": -15.120894432067871,
      "activations/layer10_attention_weight_max": 46.48858642578125,
      "activations/layer10_attention_weight_min": -42.64222717285156,
      "activations/layer11_attention_weight_max": 39.14202880859375,
      "activations/layer11_attention_weight_min": -32.29857635498047,
      "activations/layer12_attention_weight_max": 28.683088302612305,
      "activations/layer12_attention_weight_min": -25.485862731933594,
      "activations/layer13_attention_weight_max": 60.634124755859375,
      "activations/layer13_attention_weight_min": -36.937705993652344,
      "activations/layer14_attention_weight_max": 49.51055908203125,
      "activations/layer14_attention_weight_min": -31.784664154052734,
      "activations/layer15_attention_weight_max": 54.75503158569336,
      "activations/layer15_attention_weight_min": -30.149728775024414,
      "activations/layer16_attention_weight_max": 38.3768196105957,
      "activations/layer16_attention_weight_min": -31.318363189697266,
      "activations/layer17_attention_weight_max": 63.62379837036133,
      "activations/layer17_attention_weight_min": -43.36320877075195,
      "activations/layer18_attention_weight_max": 52.159507751464844,
      "activations/layer18_attention_weight_min": -35.96963119506836,
      "activations/layer19_attention_weight_max": 25.377384185791016,
      "activations/layer19_attention_weight_min": -21.397497177124023,
      "activations/layer1_attention_weight_max": 18.14105796813965,
      "activations/layer1_attention_weight_min": -14.76246452331543,
      "activations/layer20_attention_weight_max": 26.31488609313965,
      "activations/layer20_attention_weight_min": -20.25347137451172,
      "activations/layer21_attention_weight_max": 45.92213821411133,
      "activations/layer21_attention_weight_min": -25.7121524810791,
      "activations/layer22_attention_weight_max": 35.05252456665039,
      "activations/layer22_attention_weight_min": -26.13874053955078,
      "activations/layer23_attention_weight_max": 34.45365905761719,
      "activations/layer23_attention_weight_min": -22.80878257751465,
      "activations/layer2_attention_weight_max": 33.51640319824219,
      "activations/layer2_attention_weight_min": -31.24151611328125,
      "activations/layer3_attention_weight_max": 90.63127136230469,
      "activations/layer3_attention_weight_min": -90.38676452636719,
      "activations/layer4_attention_weight_max": 112.48371887207031,
      "activations/layer4_attention_weight_min": -102.93121337890625,
      "activations/layer5_attention_weight_max": 109.12260437011719,
      "activations/layer5_attention_weight_min": -97.32752990722656,
      "activations/layer6_attention_weight_max": 66.16650390625,
      "activations/layer6_attention_weight_min": -58.46015930175781,
      "activations/layer7_attention_weight_max": 83.40426635742188,
      "activations/layer7_attention_weight_min": -79.04519653320312,
      "activations/layer8_attention_weight_max": 53.953086853027344,
      "activations/layer8_attention_weight_min": -50.28105163574219,
      "activations/layer9_attention_weight_max": 60.40713119506836,
      "activations/layer9_attention_weight_min": -53.682167053222656,
      "epoch": 6.45,
      "learning_rate": 0.0001095280303030303,
      "loss": 2.8257,
      "step": 110950
    },
    {
      "activations/layer0_attention_weight_max": 16.48891830444336,
      "activations/layer0_attention_weight_min": -14.633142471313477,
      "activations/layer10_attention_weight_max": 37.929203033447266,
      "activations/layer10_attention_weight_min": -36.22111892700195,
      "activations/layer11_attention_weight_max": 29.331584930419922,
      "activations/layer11_attention_weight_min": -30.299423217773438,
      "activations/layer12_attention_weight_max": 21.478418350219727,
      "activations/layer12_attention_weight_min": -24.557247161865234,
      "activations/layer13_attention_weight_max": 48.2995491027832,
      "activations/layer13_attention_weight_min": -34.8088493347168,
      "activations/layer14_attention_weight_max": 41.60426330566406,
      "activations/layer14_attention_weight_min": -31.445592880249023,
      "activations/layer15_attention_weight_max": 38.79634475708008,
      "activations/layer15_attention_weight_min": -29.1474552154541,
      "activations/layer16_attention_weight_max": 38.027252197265625,
      "activations/layer16_attention_weight_min": -28.099533081054688,
      "activations/layer17_attention_weight_max": 54.052703857421875,
      "activations/layer17_attention_weight_min": -42.44841003417969,
      "activations/layer18_attention_weight_max": 47.17953109741211,
      "activations/layer18_attention_weight_min": -35.9152946472168,
      "activations/layer19_attention_weight_max": 22.704965591430664,
      "activations/layer19_attention_weight_min": -21.291934967041016,
      "activations/layer1_attention_weight_max": 17.789743423461914,
      "activations/layer1_attention_weight_min": -14.476105690002441,
      "activations/layer20_attention_weight_max": 21.16077995300293,
      "activations/layer20_attention_weight_min": -28.68451499938965,
      "activations/layer21_attention_weight_max": 35.50337219238281,
      "activations/layer21_attention_weight_min": -25.891693115234375,
      "activations/layer22_attention_weight_max": 34.02604293823242,
      "activations/layer22_attention_weight_min": -26.070777893066406,
      "activations/layer23_attention_weight_max": 32.52891159057617,
      "activations/layer23_attention_weight_min": -21.956632614135742,
      "activations/layer2_attention_weight_max": 30.67995834350586,
      "activations/layer2_attention_weight_min": -29.634756088256836,
      "activations/layer3_attention_weight_max": 85.23617553710938,
      "activations/layer3_attention_weight_min": -87.65508270263672,
      "activations/layer4_attention_weight_max": 99.95084381103516,
      "activations/layer4_attention_weight_min": -93.53914642333984,
      "activations/layer5_attention_weight_max": 91.037841796875,
      "activations/layer5_attention_weight_min": -84.4329833984375,
      "activations/layer6_attention_weight_max": 54.95225524902344,
      "activations/layer6_attention_weight_min": -52.78194808959961,
      "activations/layer7_attention_weight_max": 75.0985107421875,
      "activations/layer7_attention_weight_min": -73.46432495117188,
      "activations/layer8_attention_weight_max": 43.29560470581055,
      "activations/layer8_attention_weight_min": -42.718257904052734,
      "activations/layer9_attention_weight_max": 56.97930908203125,
      "activations/layer9_attention_weight_min": -49.14398193359375,
      "epoch": 6.45,
      "learning_rate": 0.00010950909090909091,
      "loss": 2.8387,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_loss": 2.78125,
      "eval_runtime": 8.5905,
      "eval_samples_per_second": 499.854,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_openwebtext_loss": 2.78125,
      "eval_openwebtext_ppl": 16.139182322446413,
      "eval_openwebtext_runtime": 8.5905,
      "eval_openwebtext_samples_per_second": 499.854,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_wikitext_loss": 3.033203125,
      "eval_wikitext_ppl": 20.763634702888442,
      "eval_wikitext_runtime": 2.0552,
      "eval_wikitext_samples_per_second": 221.881,
      "step": 111000
    },
    {
      "epoch": 6.45,
      "eval_lambada_loss": 2.708984375,
      "eval_lambada_ppl": 15.01401915561195,
      "eval_lambada_runtime": 9.6483,
      "eval_lambada_samples_per_second": 504.65,
      "step": 111000
    },
    {
      "activations/layer0_attention_weight_max": 15.137211799621582,
      "activations/layer0_attention_weight_min": -15.3172607421875,
      "activations/layer10_attention_weight_max": 38.761600494384766,
      "activations/layer10_attention_weight_min": -35.466087341308594,
      "activations/layer11_attention_weight_max": 31.638988494873047,
      "activations/layer11_attention_weight_min": -30.477460861206055,
      "activations/layer12_attention_weight_max": 26.020837783813477,
      "activations/layer12_attention_weight_min": -25.776859283447266,
      "activations/layer13_attention_weight_max": 64.65451049804688,
      "activations/layer13_attention_weight_min": -38.47662353515625,
      "activations/layer14_attention_weight_max": 43.21255874633789,
      "activations/layer14_attention_weight_min": -30.56098175048828,
      "activations/layer15_attention_weight_max": 38.16657257080078,
      "activations/layer15_attention_weight_min": -30.144704818725586,
      "activations/layer16_attention_weight_max": 33.49083709716797,
      "activations/layer16_attention_weight_min": -26.627864837646484,
      "activations/layer17_attention_weight_max": 57.30347442626953,
      "activations/layer17_attention_weight_min": -43.676788330078125,
      "activations/layer18_attention_weight_max": 47.57034683227539,
      "activations/layer18_attention_weight_min": -37.8495979309082,
      "activations/layer19_attention_weight_max": 24.61171531677246,
      "activations/layer19_attention_weight_min": -21.126392364501953,
      "activations/layer1_attention_weight_max": 17.84967041015625,
      "activations/layer1_attention_weight_min": -15.648051261901855,
      "activations/layer20_attention_weight_max": 24.592784881591797,
      "activations/layer20_attention_weight_min": -26.628684997558594,
      "activations/layer21_attention_weight_max": 44.143978118896484,
      "activations/layer21_attention_weight_min": -26.102312088012695,
      "activations/layer22_attention_weight_max": 38.916221618652344,
      "activations/layer22_attention_weight_min": -26.82188606262207,
      "activations/layer23_attention_weight_max": 35.76700973510742,
      "activations/layer23_attention_weight_min": -21.057472229003906,
      "activations/layer2_attention_weight_max": 30.98135757446289,
      "activations/layer2_attention_weight_min": -29.180397033691406,
      "activations/layer3_attention_weight_max": 86.4517593383789,
      "activations/layer3_attention_weight_min": -92.85726165771484,
      "activations/layer4_attention_weight_max": 98.69386291503906,
      "activations/layer4_attention_weight_min": -97.18147277832031,
      "activations/layer5_attention_weight_max": 93.3691635131836,
      "activations/layer5_attention_weight_min": -88.1854248046875,
      "activations/layer6_attention_weight_max": 57.78254699707031,
      "activations/layer6_attention_weight_min": -53.02593231201172,
      "activations/layer7_attention_weight_max": 78.88902282714844,
      "activations/layer7_attention_weight_min": -70.65284729003906,
      "activations/layer8_attention_weight_max": 43.5638542175293,
      "activations/layer8_attention_weight_min": -43.442718505859375,
      "activations/layer9_attention_weight_max": 58.075416564941406,
      "activations/layer9_attention_weight_min": -47.487308502197266,
      "epoch": 6.45,
      "learning_rate": 0.0001094901515151515,
      "loss": 2.8249,
      "step": 111050
    },
    {
      "activations/layer0_attention_weight_max": 15.741039276123047,
      "activations/layer0_attention_weight_min": -15.18007755279541,
      "activations/layer10_attention_weight_max": 35.926971435546875,
      "activations/layer10_attention_weight_min": -36.12949752807617,
      "activations/layer11_attention_weight_max": 29.557918548583984,
      "activations/layer11_attention_weight_min": -31.231983184814453,
      "activations/layer12_attention_weight_max": 23.385480880737305,
      "activations/layer12_attention_weight_min": -23.613601684570312,
      "activations/layer13_attention_weight_max": 52.63203811645508,
      "activations/layer13_attention_weight_min": -38.77083969116211,
      "activations/layer14_attention_weight_max": 47.82013702392578,
      "activations/layer14_attention_weight_min": -34.605831146240234,
      "activations/layer15_attention_weight_max": 39.07613754272461,
      "activations/layer15_attention_weight_min": -30.839799880981445,
      "activations/layer16_attention_weight_max": 33.20842742919922,
      "activations/layer16_attention_weight_min": -27.817598342895508,
      "activations/layer17_attention_weight_max": 59.18556213378906,
      "activations/layer17_attention_weight_min": -44.708160400390625,
      "activations/layer18_attention_weight_max": 48.5005989074707,
      "activations/layer18_attention_weight_min": -36.92271423339844,
      "activations/layer19_attention_weight_max": 22.3272762298584,
      "activations/layer19_attention_weight_min": -20.38810920715332,
      "activations/layer1_attention_weight_max": 17.865787506103516,
      "activations/layer1_attention_weight_min": -14.872261047363281,
      "activations/layer20_attention_weight_max": 21.764862060546875,
      "activations/layer20_attention_weight_min": -20.922794342041016,
      "activations/layer21_attention_weight_max": 40.79994583129883,
      "activations/layer21_attention_weight_min": -23.663497924804688,
      "activations/layer22_attention_weight_max": 33.760990142822266,
      "activations/layer22_attention_weight_min": -24.79808807373047,
      "activations/layer23_attention_weight_max": 37.46308898925781,
      "activations/layer23_attention_weight_min": -20.27817726135254,
      "activations/layer2_attention_weight_max": 30.289161682128906,
      "activations/layer2_attention_weight_min": -28.06993865966797,
      "activations/layer3_attention_weight_max": 83.83477783203125,
      "activations/layer3_attention_weight_min": -84.11654663085938,
      "activations/layer4_attention_weight_max": 100.19866180419922,
      "activations/layer4_attention_weight_min": -92.54100036621094,
      "activations/layer5_attention_weight_max": 97.3412094116211,
      "activations/layer5_attention_weight_min": -86.85406494140625,
      "activations/layer6_attention_weight_max": 59.41822814941406,
      "activations/layer6_attention_weight_min": -53.620460510253906,
      "activations/layer7_attention_weight_max": 74.78130340576172,
      "activations/layer7_attention_weight_min": -70.96546173095703,
      "activations/layer8_attention_weight_max": 44.352943420410156,
      "activations/layer8_attention_weight_min": -42.79386520385742,
      "activations/layer9_attention_weight_max": 52.6073112487793,
      "activations/layer9_attention_weight_min": -51.79913330078125,
      "epoch": 6.46,
      "learning_rate": 0.00010947121212121211,
      "loss": 2.8424,
      "step": 111100
    },
    {
      "activations/layer0_attention_weight_max": 15.653155326843262,
      "activations/layer0_attention_weight_min": -14.967061042785645,
      "activations/layer10_attention_weight_max": 39.42254638671875,
      "activations/layer10_attention_weight_min": -35.10139083862305,
      "activations/layer11_attention_weight_max": 31.82472038269043,
      "activations/layer11_attention_weight_min": -29.656417846679688,
      "activations/layer12_attention_weight_max": 21.104846954345703,
      "activations/layer12_attention_weight_min": -27.34859848022461,
      "activations/layer13_attention_weight_max": 47.77944564819336,
      "activations/layer13_attention_weight_min": -35.80796813964844,
      "activations/layer14_attention_weight_max": 36.38808059692383,
      "activations/layer14_attention_weight_min": -32.31842803955078,
      "activations/layer15_attention_weight_max": 44.22576904296875,
      "activations/layer15_attention_weight_min": -30.629432678222656,
      "activations/layer16_attention_weight_max": 32.5280647277832,
      "activations/layer16_attention_weight_min": -27.28026580810547,
      "activations/layer17_attention_weight_max": 55.86067581176758,
      "activations/layer17_attention_weight_min": -41.571346282958984,
      "activations/layer18_attention_weight_max": 46.70901107788086,
      "activations/layer18_attention_weight_min": -36.7176513671875,
      "activations/layer19_attention_weight_max": 23.48642921447754,
      "activations/layer19_attention_weight_min": -21.553762435913086,
      "activations/layer1_attention_weight_max": 17.69797134399414,
      "activations/layer1_attention_weight_min": -17.801843643188477,
      "activations/layer20_attention_weight_max": 21.756776809692383,
      "activations/layer20_attention_weight_min": -19.792064666748047,
      "activations/layer21_attention_weight_max": 37.0533561706543,
      "activations/layer21_attention_weight_min": -24.683652877807617,
      "activations/layer22_attention_weight_max": 32.43341827392578,
      "activations/layer22_attention_weight_min": -26.51289176940918,
      "activations/layer23_attention_weight_max": 31.959701538085938,
      "activations/layer23_attention_weight_min": -19.56804656982422,
      "activations/layer2_attention_weight_max": 31.283653259277344,
      "activations/layer2_attention_weight_min": -27.960071563720703,
      "activations/layer3_attention_weight_max": 84.44251251220703,
      "activations/layer3_attention_weight_min": -86.85914611816406,
      "activations/layer4_attention_weight_max": 100.60649871826172,
      "activations/layer4_attention_weight_min": -94.28170776367188,
      "activations/layer5_attention_weight_max": 91.58978271484375,
      "activations/layer5_attention_weight_min": -86.09453582763672,
      "activations/layer6_attention_weight_max": 58.849266052246094,
      "activations/layer6_attention_weight_min": -55.12519836425781,
      "activations/layer7_attention_weight_max": 73.38247680664062,
      "activations/layer7_attention_weight_min": -71.63445281982422,
      "activations/layer8_attention_weight_max": 44.67819595336914,
      "activations/layer8_attention_weight_min": -44.33015823364258,
      "activations/layer9_attention_weight_max": 51.72860336303711,
      "activations/layer9_attention_weight_min": -48.29771423339844,
      "epoch": 6.46,
      "learning_rate": 0.00010945227272727273,
      "loss": 2.8354,
      "step": 111150
    },
    {
      "activations/layer0_attention_weight_max": 14.822519302368164,
      "activations/layer0_attention_weight_min": -14.8609037399292,
      "activations/layer10_attention_weight_max": 37.66621780395508,
      "activations/layer10_attention_weight_min": -34.171024322509766,
      "activations/layer11_attention_weight_max": 29.07115936279297,
      "activations/layer11_attention_weight_min": -29.454692840576172,
      "activations/layer12_attention_weight_max": 20.350297927856445,
      "activations/layer12_attention_weight_min": -25.77764320373535,
      "activations/layer13_attention_weight_max": 51.8193244934082,
      "activations/layer13_attention_weight_min": -36.07428741455078,
      "activations/layer14_attention_weight_max": 43.30170822143555,
      "activations/layer14_attention_weight_min": -33.28974151611328,
      "activations/layer15_attention_weight_max": 37.700111389160156,
      "activations/layer15_attention_weight_min": -29.75897979736328,
      "activations/layer16_attention_weight_max": 38.80791473388672,
      "activations/layer16_attention_weight_min": -28.41391372680664,
      "activations/layer17_attention_weight_max": 56.0262565612793,
      "activations/layer17_attention_weight_min": -44.57394790649414,
      "activations/layer18_attention_weight_max": 50.31120300292969,
      "activations/layer18_attention_weight_min": -36.856910705566406,
      "activations/layer19_attention_weight_max": 21.61347770690918,
      "activations/layer19_attention_weight_min": -22.606243133544922,
      "activations/layer1_attention_weight_max": 16.78588104248047,
      "activations/layer1_attention_weight_min": -16.308923721313477,
      "activations/layer20_attention_weight_max": 21.9162540435791,
      "activations/layer20_attention_weight_min": -21.552446365356445,
      "activations/layer21_attention_weight_max": 41.2484130859375,
      "activations/layer21_attention_weight_min": -26.050254821777344,
      "activations/layer22_attention_weight_max": 34.75403594970703,
      "activations/layer22_attention_weight_min": -28.744489669799805,
      "activations/layer23_attention_weight_max": 32.38240051269531,
      "activations/layer23_attention_weight_min": -19.857421875,
      "activations/layer2_attention_weight_max": 30.703710556030273,
      "activations/layer2_attention_weight_min": -29.20532989501953,
      "activations/layer3_attention_weight_max": 84.93505096435547,
      "activations/layer3_attention_weight_min": -84.34178161621094,
      "activations/layer4_attention_weight_max": 97.40619659423828,
      "activations/layer4_attention_weight_min": -92.71836853027344,
      "activations/layer5_attention_weight_max": 90.18822479248047,
      "activations/layer5_attention_weight_min": -86.53964233398438,
      "activations/layer6_attention_weight_max": 55.321876525878906,
      "activations/layer6_attention_weight_min": -52.40546417236328,
      "activations/layer7_attention_weight_max": 74.27919006347656,
      "activations/layer7_attention_weight_min": -69.46529388427734,
      "activations/layer8_attention_weight_max": 43.16837692260742,
      "activations/layer8_attention_weight_min": -42.64842224121094,
      "activations/layer9_attention_weight_max": 47.54841613769531,
      "activations/layer9_attention_weight_min": -46.08251190185547,
      "epoch": 6.46,
      "learning_rate": 0.00010943333333333331,
      "loss": 2.8373,
      "step": 111200
    },
    {
      "activations/layer0_attention_weight_max": 15.407173156738281,
      "activations/layer0_attention_weight_min": -14.423362731933594,
      "activations/layer10_attention_weight_max": 37.330848693847656,
      "activations/layer10_attention_weight_min": -36.93726348876953,
      "activations/layer11_attention_weight_max": 29.653892517089844,
      "activations/layer11_attention_weight_min": -29.18227767944336,
      "activations/layer12_attention_weight_max": 20.466489791870117,
      "activations/layer12_attention_weight_min": -25.67791175842285,
      "activations/layer13_attention_weight_max": 45.08908462524414,
      "activations/layer13_attention_weight_min": -30.563133239746094,
      "activations/layer14_attention_weight_max": 48.54313659667969,
      "activations/layer14_attention_weight_min": -32.07088088989258,
      "activations/layer15_attention_weight_max": 44.07402038574219,
      "activations/layer15_attention_weight_min": -29.44297981262207,
      "activations/layer16_attention_weight_max": 37.534461975097656,
      "activations/layer16_attention_weight_min": -27.73491668701172,
      "activations/layer17_attention_weight_max": 58.77128219604492,
      "activations/layer17_attention_weight_min": -43.194889068603516,
      "activations/layer18_attention_weight_max": 50.59999465942383,
      "activations/layer18_attention_weight_min": -37.01423645019531,
      "activations/layer19_attention_weight_max": 27.247928619384766,
      "activations/layer19_attention_weight_min": -21.346410751342773,
      "activations/layer1_attention_weight_max": 17.743228912353516,
      "activations/layer1_attention_weight_min": -14.640918731689453,
      "activations/layer20_attention_weight_max": 21.741317749023438,
      "activations/layer20_attention_weight_min": -20.76104164123535,
      "activations/layer21_attention_weight_max": 39.19819259643555,
      "activations/layer21_attention_weight_min": -27.00260353088379,
      "activations/layer22_attention_weight_max": 39.205726623535156,
      "activations/layer22_attention_weight_min": -24.800537109375,
      "activations/layer23_attention_weight_max": 36.82981872558594,
      "activations/layer23_attention_weight_min": -19.550962448120117,
      "activations/layer2_attention_weight_max": 31.76874542236328,
      "activations/layer2_attention_weight_min": -29.174442291259766,
      "activations/layer3_attention_weight_max": 82.96138763427734,
      "activations/layer3_attention_weight_min": -83.34835815429688,
      "activations/layer4_attention_weight_max": 98.99471282958984,
      "activations/layer4_attention_weight_min": -92.79410552978516,
      "activations/layer5_attention_weight_max": 90.29247283935547,
      "activations/layer5_attention_weight_min": -84.27473449707031,
      "activations/layer6_attention_weight_max": 55.058815002441406,
      "activations/layer6_attention_weight_min": -52.35814666748047,
      "activations/layer7_attention_weight_max": 72.68231201171875,
      "activations/layer7_attention_weight_min": -67.52893829345703,
      "activations/layer8_attention_weight_max": 41.468177795410156,
      "activations/layer8_attention_weight_min": -44.39310836791992,
      "activations/layer9_attention_weight_max": 50.47967529296875,
      "activations/layer9_attention_weight_min": -47.599361419677734,
      "epoch": 6.46,
      "learning_rate": 0.00010941439393939393,
      "loss": 2.817,
      "step": 111250
    },
    {
      "activations/layer0_attention_weight_max": 15.656268119812012,
      "activations/layer0_attention_weight_min": -14.507841110229492,
      "activations/layer10_attention_weight_max": 36.85121536254883,
      "activations/layer10_attention_weight_min": -33.26453399658203,
      "activations/layer11_attention_weight_max": 28.39790153503418,
      "activations/layer11_attention_weight_min": -31.2458553314209,
      "activations/layer12_attention_weight_max": 20.270790100097656,
      "activations/layer12_attention_weight_min": -30.929855346679688,
      "activations/layer13_attention_weight_max": 39.55841827392578,
      "activations/layer13_attention_weight_min": -32.228023529052734,
      "activations/layer14_attention_weight_max": 40.663848876953125,
      "activations/layer14_attention_weight_min": -31.49419403076172,
      "activations/layer15_attention_weight_max": 34.930397033691406,
      "activations/layer15_attention_weight_min": -28.83496856689453,
      "activations/layer16_attention_weight_max": 31.079988479614258,
      "activations/layer16_attention_weight_min": -26.279930114746094,
      "activations/layer17_attention_weight_max": 56.92117691040039,
      "activations/layer17_attention_weight_min": -40.97598648071289,
      "activations/layer18_attention_weight_max": 49.721710205078125,
      "activations/layer18_attention_weight_min": -34.93189239501953,
      "activations/layer19_attention_weight_max": 23.090856552124023,
      "activations/layer19_attention_weight_min": -20.60782814025879,
      "activations/layer1_attention_weight_max": 17.076778411865234,
      "activations/layer1_attention_weight_min": -15.31332778930664,
      "activations/layer20_attention_weight_max": 22.31751251220703,
      "activations/layer20_attention_weight_min": -20.631664276123047,
      "activations/layer21_attention_weight_max": 39.253753662109375,
      "activations/layer21_attention_weight_min": -23.659597396850586,
      "activations/layer22_attention_weight_max": 36.674217224121094,
      "activations/layer22_attention_weight_min": -24.658344268798828,
      "activations/layer23_attention_weight_max": 31.49709129333496,
      "activations/layer23_attention_weight_min": -20.400453567504883,
      "activations/layer2_attention_weight_max": 31.515043258666992,
      "activations/layer2_attention_weight_min": -31.055530548095703,
      "activations/layer3_attention_weight_max": 84.17974090576172,
      "activations/layer3_attention_weight_min": -89.52267456054688,
      "activations/layer4_attention_weight_max": 100.15794372558594,
      "activations/layer4_attention_weight_min": -99.61629486083984,
      "activations/layer5_attention_weight_max": 93.82160949707031,
      "activations/layer5_attention_weight_min": -84.41094970703125,
      "activations/layer6_attention_weight_max": 57.969581604003906,
      "activations/layer6_attention_weight_min": -51.658042907714844,
      "activations/layer7_attention_weight_max": 78.68367767333984,
      "activations/layer7_attention_weight_min": -70.42320251464844,
      "activations/layer8_attention_weight_max": 43.66525650024414,
      "activations/layer8_attention_weight_min": -42.22604751586914,
      "activations/layer9_attention_weight_max": 51.879859924316406,
      "activations/layer9_attention_weight_min": -45.662296295166016,
      "epoch": 6.47,
      "learning_rate": 0.00010939583333333332,
      "loss": 2.8329,
      "step": 111300
    },
    {
      "activations/layer0_attention_weight_max": 15.211186408996582,
      "activations/layer0_attention_weight_min": -14.592618942260742,
      "activations/layer10_attention_weight_max": 38.24500274658203,
      "activations/layer10_attention_weight_min": -35.46043395996094,
      "activations/layer11_attention_weight_max": 31.903278350830078,
      "activations/layer11_attention_weight_min": -28.429595947265625,
      "activations/layer12_attention_weight_max": 20.417938232421875,
      "activations/layer12_attention_weight_min": -27.341773986816406,
      "activations/layer13_attention_weight_max": 48.14655685424805,
      "activations/layer13_attention_weight_min": -38.513389587402344,
      "activations/layer14_attention_weight_max": 49.06614303588867,
      "activations/layer14_attention_weight_min": -30.493343353271484,
      "activations/layer15_attention_weight_max": 36.28271484375,
      "activations/layer15_attention_weight_min": -30.241579055786133,
      "activations/layer16_attention_weight_max": 36.22182083129883,
      "activations/layer16_attention_weight_min": -26.811349868774414,
      "activations/layer17_attention_weight_max": 56.80472183227539,
      "activations/layer17_attention_weight_min": -41.89882278442383,
      "activations/layer18_attention_weight_max": 51.09934997558594,
      "activations/layer18_attention_weight_min": -37.38777542114258,
      "activations/layer19_attention_weight_max": 22.60456657409668,
      "activations/layer19_attention_weight_min": -20.951852798461914,
      "activations/layer1_attention_weight_max": 16.286273956298828,
      "activations/layer1_attention_weight_min": -15.91581916809082,
      "activations/layer20_attention_weight_max": 21.231552124023438,
      "activations/layer20_attention_weight_min": -21.434574127197266,
      "activations/layer21_attention_weight_max": 35.910011291503906,
      "activations/layer21_attention_weight_min": -23.982473373413086,
      "activations/layer22_attention_weight_max": 35.91364669799805,
      "activations/layer22_attention_weight_min": -25.06061363220215,
      "activations/layer23_attention_weight_max": 31.831350326538086,
      "activations/layer23_attention_weight_min": -18.685850143432617,
      "activations/layer2_attention_weight_max": 30.956838607788086,
      "activations/layer2_attention_weight_min": -27.992053985595703,
      "activations/layer3_attention_weight_max": 89.55680847167969,
      "activations/layer3_attention_weight_min": -87.30785369873047,
      "activations/layer4_attention_weight_max": 102.06868743896484,
      "activations/layer4_attention_weight_min": -95.19176483154297,
      "activations/layer5_attention_weight_max": 92.8067626953125,
      "activations/layer5_attention_weight_min": -85.37879943847656,
      "activations/layer6_attention_weight_max": 59.48087692260742,
      "activations/layer6_attention_weight_min": -52.569618225097656,
      "activations/layer7_attention_weight_max": 76.46275329589844,
      "activations/layer7_attention_weight_min": -71.70021057128906,
      "activations/layer8_attention_weight_max": 43.51221466064453,
      "activations/layer8_attention_weight_min": -44.31437683105469,
      "activations/layer9_attention_weight_max": 54.57244873046875,
      "activations/layer9_attention_weight_min": -48.90766143798828,
      "epoch": 6.47,
      "learning_rate": 0.00010937689393939394,
      "loss": 2.8329,
      "step": 111350
    },
    {
      "activations/layer0_attention_weight_max": 14.581024169921875,
      "activations/layer0_attention_weight_min": -15.042567253112793,
      "activations/layer10_attention_weight_max": 37.61717987060547,
      "activations/layer10_attention_weight_min": -37.756927490234375,
      "activations/layer11_attention_weight_max": 28.200231552124023,
      "activations/layer11_attention_weight_min": -28.011075973510742,
      "activations/layer12_attention_weight_max": 20.440185546875,
      "activations/layer12_attention_weight_min": -27.08991050720215,
      "activations/layer13_attention_weight_max": 39.57362747192383,
      "activations/layer13_attention_weight_min": -29.22003936767578,
      "activations/layer14_attention_weight_max": 41.27231979370117,
      "activations/layer14_attention_weight_min": -32.12904357910156,
      "activations/layer15_attention_weight_max": 35.679500579833984,
      "activations/layer15_attention_weight_min": -30.4129581451416,
      "activations/layer16_attention_weight_max": 31.770296096801758,
      "activations/layer16_attention_weight_min": -30.80957794189453,
      "activations/layer17_attention_weight_max": 58.6934700012207,
      "activations/layer17_attention_weight_min": -41.55251693725586,
      "activations/layer18_attention_weight_max": 52.27824020385742,
      "activations/layer18_attention_weight_min": -35.601680755615234,
      "activations/layer19_attention_weight_max": 25.890966415405273,
      "activations/layer19_attention_weight_min": -21.862140655517578,
      "activations/layer1_attention_weight_max": 16.879369735717773,
      "activations/layer1_attention_weight_min": -14.052082061767578,
      "activations/layer20_attention_weight_max": 23.62157440185547,
      "activations/layer20_attention_weight_min": -21.580827713012695,
      "activations/layer21_attention_weight_max": 45.03978729248047,
      "activations/layer21_attention_weight_min": -25.94626235961914,
      "activations/layer22_attention_weight_max": 33.81488800048828,
      "activations/layer22_attention_weight_min": -25.484455108642578,
      "activations/layer23_attention_weight_max": 31.193675994873047,
      "activations/layer23_attention_weight_min": -20.384933471679688,
      "activations/layer2_attention_weight_max": 28.942218780517578,
      "activations/layer2_attention_weight_min": -26.963802337646484,
      "activations/layer3_attention_weight_max": 79.29837036132812,
      "activations/layer3_attention_weight_min": -87.51441192626953,
      "activations/layer4_attention_weight_max": 99.16165924072266,
      "activations/layer4_attention_weight_min": -92.86663055419922,
      "activations/layer5_attention_weight_max": 92.83573913574219,
      "activations/layer5_attention_weight_min": -88.37876892089844,
      "activations/layer6_attention_weight_max": 56.3339958190918,
      "activations/layer6_attention_weight_min": -54.88966751098633,
      "activations/layer7_attention_weight_max": 70.55364990234375,
      "activations/layer7_attention_weight_min": -67.54334259033203,
      "activations/layer8_attention_weight_max": 43.02692794799805,
      "activations/layer8_attention_weight_min": -42.773048400878906,
      "activations/layer9_attention_weight_max": 48.339073181152344,
      "activations/layer9_attention_weight_min": -46.206050872802734,
      "epoch": 6.47,
      "learning_rate": 0.00010935795454545452,
      "loss": 2.8263,
      "step": 111400
    },
    {
      "activations/layer0_attention_weight_max": 14.49487018585205,
      "activations/layer0_attention_weight_min": -15.39806842803955,
      "activations/layer10_attention_weight_max": 37.905052185058594,
      "activations/layer10_attention_weight_min": -35.240962982177734,
      "activations/layer11_attention_weight_max": 31.77657699584961,
      "activations/layer11_attention_weight_min": -29.11859893798828,
      "activations/layer12_attention_weight_max": 23.531349182128906,
      "activations/layer12_attention_weight_min": -27.103918075561523,
      "activations/layer13_attention_weight_max": 53.5094108581543,
      "activations/layer13_attention_weight_min": -36.20055389404297,
      "activations/layer14_attention_weight_max": 42.98084259033203,
      "activations/layer14_attention_weight_min": -31.46223258972168,
      "activations/layer15_attention_weight_max": 40.162227630615234,
      "activations/layer15_attention_weight_min": -29.527402877807617,
      "activations/layer16_attention_weight_max": 39.923465728759766,
      "activations/layer16_attention_weight_min": -26.46620750427246,
      "activations/layer17_attention_weight_max": 56.256160736083984,
      "activations/layer17_attention_weight_min": -42.89619827270508,
      "activations/layer18_attention_weight_max": 53.19797134399414,
      "activations/layer18_attention_weight_min": -34.310630798339844,
      "activations/layer19_attention_weight_max": 23.33285903930664,
      "activations/layer19_attention_weight_min": -20.920106887817383,
      "activations/layer1_attention_weight_max": 16.972240447998047,
      "activations/layer1_attention_weight_min": -16.334035873413086,
      "activations/layer20_attention_weight_max": 25.9660701751709,
      "activations/layer20_attention_weight_min": -21.587154388427734,
      "activations/layer21_attention_weight_max": 39.93745803833008,
      "activations/layer21_attention_weight_min": -24.134756088256836,
      "activations/layer22_attention_weight_max": 36.975914001464844,
      "activations/layer22_attention_weight_min": -27.3775634765625,
      "activations/layer23_attention_weight_max": 35.877689361572266,
      "activations/layer23_attention_weight_min": -23.04736328125,
      "activations/layer2_attention_weight_max": 29.775718688964844,
      "activations/layer2_attention_weight_min": -25.060842514038086,
      "activations/layer3_attention_weight_max": 78.09529113769531,
      "activations/layer3_attention_weight_min": -77.17092895507812,
      "activations/layer4_attention_weight_max": 95.44041442871094,
      "activations/layer4_attention_weight_min": -87.29983520507812,
      "activations/layer5_attention_weight_max": 89.41988372802734,
      "activations/layer5_attention_weight_min": -84.77471923828125,
      "activations/layer6_attention_weight_max": 54.261653900146484,
      "activations/layer6_attention_weight_min": -54.43490982055664,
      "activations/layer7_attention_weight_max": 71.59928131103516,
      "activations/layer7_attention_weight_min": -70.80705261230469,
      "activations/layer8_attention_weight_max": 45.0350227355957,
      "activations/layer8_attention_weight_min": -44.31333923339844,
      "activations/layer9_attention_weight_max": 51.593807220458984,
      "activations/layer9_attention_weight_min": -51.384063720703125,
      "epoch": 6.48,
      "learning_rate": 0.00010933901515151514,
      "loss": 2.8345,
      "step": 111450
    },
    {
      "activations/layer0_attention_weight_max": 14.961968421936035,
      "activations/layer0_attention_weight_min": -14.836018562316895,
      "activations/layer10_attention_weight_max": 37.38125991821289,
      "activations/layer10_attention_weight_min": -37.41679000854492,
      "activations/layer11_attention_weight_max": 30.29910659790039,
      "activations/layer11_attention_weight_min": -31.65760040283203,
      "activations/layer12_attention_weight_max": 20.466262817382812,
      "activations/layer12_attention_weight_min": -30.321208953857422,
      "activations/layer13_attention_weight_max": 43.43916702270508,
      "activations/layer13_attention_weight_min": -35.4988899230957,
      "activations/layer14_attention_weight_max": 47.273719787597656,
      "activations/layer14_attention_weight_min": -33.67064666748047,
      "activations/layer15_attention_weight_max": 37.505043029785156,
      "activations/layer15_attention_weight_min": -30.165353775024414,
      "activations/layer16_attention_weight_max": 33.11902618408203,
      "activations/layer16_attention_weight_min": -26.917882919311523,
      "activations/layer17_attention_weight_max": 54.29039001464844,
      "activations/layer17_attention_weight_min": -41.632957458496094,
      "activations/layer18_attention_weight_max": 54.95038986206055,
      "activations/layer18_attention_weight_min": -35.6827507019043,
      "activations/layer19_attention_weight_max": 20.202144622802734,
      "activations/layer19_attention_weight_min": -20.991186141967773,
      "activations/layer1_attention_weight_max": 17.711116790771484,
      "activations/layer1_attention_weight_min": -17.364946365356445,
      "activations/layer20_attention_weight_max": 23.935514450073242,
      "activations/layer20_attention_weight_min": -22.263151168823242,
      "activations/layer21_attention_weight_max": 41.32584762573242,
      "activations/layer21_attention_weight_min": -25.137832641601562,
      "activations/layer22_attention_weight_max": 33.10890197753906,
      "activations/layer22_attention_weight_min": -25.46392250061035,
      "activations/layer23_attention_weight_max": 36.16099166870117,
      "activations/layer23_attention_weight_min": -19.869924545288086,
      "activations/layer2_attention_weight_max": 32.855445861816406,
      "activations/layer2_attention_weight_min": -29.292112350463867,
      "activations/layer3_attention_weight_max": 86.32511901855469,
      "activations/layer3_attention_weight_min": -85.80896759033203,
      "activations/layer4_attention_weight_max": 103.178466796875,
      "activations/layer4_attention_weight_min": -95.28169250488281,
      "activations/layer5_attention_weight_max": 95.00627899169922,
      "activations/layer5_attention_weight_min": -86.63035583496094,
      "activations/layer6_attention_weight_max": 60.703285217285156,
      "activations/layer6_attention_weight_min": -55.62952423095703,
      "activations/layer7_attention_weight_max": 76.98617553710938,
      "activations/layer7_attention_weight_min": -71.92707824707031,
      "activations/layer8_attention_weight_max": 43.744110107421875,
      "activations/layer8_attention_weight_min": -44.14204406738281,
      "activations/layer9_attention_weight_max": 61.837886810302734,
      "activations/layer9_attention_weight_min": -50.296119689941406,
      "epoch": 6.48,
      "learning_rate": 0.00010932007575757575,
      "loss": 2.8474,
      "step": 111500
    },
    {
      "activations/layer0_attention_weight_max": 14.73023796081543,
      "activations/layer0_attention_weight_min": -14.8691987991333,
      "activations/layer10_attention_weight_max": 41.1035041809082,
      "activations/layer10_attention_weight_min": -35.747596740722656,
      "activations/layer11_attention_weight_max": 31.26166534423828,
      "activations/layer11_attention_weight_min": -30.77968978881836,
      "activations/layer12_attention_weight_max": 27.68181610107422,
      "activations/layer12_attention_weight_min": -25.68933868408203,
      "activations/layer13_attention_weight_max": 45.64553451538086,
      "activations/layer13_attention_weight_min": -33.40248489379883,
      "activations/layer14_attention_weight_max": 35.76054382324219,
      "activations/layer14_attention_weight_min": -28.731691360473633,
      "activations/layer15_attention_weight_max": 37.515296936035156,
      "activations/layer15_attention_weight_min": -28.403514862060547,
      "activations/layer16_attention_weight_max": 33.315128326416016,
      "activations/layer16_attention_weight_min": -27.269311904907227,
      "activations/layer17_attention_weight_max": 57.16020584106445,
      "activations/layer17_attention_weight_min": -40.44788360595703,
      "activations/layer18_attention_weight_max": 49.8242073059082,
      "activations/layer18_attention_weight_min": -33.863407135009766,
      "activations/layer19_attention_weight_max": 22.395523071289062,
      "activations/layer19_attention_weight_min": -20.179485321044922,
      "activations/layer1_attention_weight_max": 16.94908332824707,
      "activations/layer1_attention_weight_min": -14.449570655822754,
      "activations/layer20_attention_weight_max": 19.772014617919922,
      "activations/layer20_attention_weight_min": -18.590049743652344,
      "activations/layer21_attention_weight_max": 37.74871826171875,
      "activations/layer21_attention_weight_min": -23.76922035217285,
      "activations/layer22_attention_weight_max": 31.86296272277832,
      "activations/layer22_attention_weight_min": -23.6346378326416,
      "activations/layer23_attention_weight_max": 31.477264404296875,
      "activations/layer23_attention_weight_min": -20.432903289794922,
      "activations/layer2_attention_weight_max": 30.88636589050293,
      "activations/layer2_attention_weight_min": -28.975156784057617,
      "activations/layer3_attention_weight_max": 84.74556732177734,
      "activations/layer3_attention_weight_min": -85.34374237060547,
      "activations/layer4_attention_weight_max": 101.60865020751953,
      "activations/layer4_attention_weight_min": -93.94559478759766,
      "activations/layer5_attention_weight_max": 93.36453247070312,
      "activations/layer5_attention_weight_min": -86.75447082519531,
      "activations/layer6_attention_weight_max": 57.648399353027344,
      "activations/layer6_attention_weight_min": -55.873260498046875,
      "activations/layer7_attention_weight_max": 75.24932861328125,
      "activations/layer7_attention_weight_min": -70.39740753173828,
      "activations/layer8_attention_weight_max": 43.75844192504883,
      "activations/layer8_attention_weight_min": -45.93708419799805,
      "activations/layer9_attention_weight_max": 48.57771682739258,
      "activations/layer9_attention_weight_min": -47.638328552246094,
      "epoch": 6.48,
      "learning_rate": 0.00010930113636363635,
      "loss": 2.8364,
      "step": 111550
    },
    {
      "activations/layer0_attention_weight_max": 15.503817558288574,
      "activations/layer0_attention_weight_min": -14.44212818145752,
      "activations/layer10_attention_weight_max": 35.49737548828125,
      "activations/layer10_attention_weight_min": -36.15184020996094,
      "activations/layer11_attention_weight_max": 29.026121139526367,
      "activations/layer11_attention_weight_min": -27.96019744873047,
      "activations/layer12_attention_weight_max": 22.303911209106445,
      "activations/layer12_attention_weight_min": -25.932788848876953,
      "activations/layer13_attention_weight_max": 41.112709045410156,
      "activations/layer13_attention_weight_min": -31.1796875,
      "activations/layer14_attention_weight_max": 38.10438537597656,
      "activations/layer14_attention_weight_min": -30.80685806274414,
      "activations/layer15_attention_weight_max": 32.690834045410156,
      "activations/layer15_attention_weight_min": -31.926969528198242,
      "activations/layer16_attention_weight_max": 31.953262329101562,
      "activations/layer16_attention_weight_min": -27.4156436920166,
      "activations/layer17_attention_weight_max": 53.80195999145508,
      "activations/layer17_attention_weight_min": -43.41255569458008,
      "activations/layer18_attention_weight_max": 44.33815383911133,
      "activations/layer18_attention_weight_min": -36.70331954956055,
      "activations/layer19_attention_weight_max": 22.315540313720703,
      "activations/layer19_attention_weight_min": -21.586591720581055,
      "activations/layer1_attention_weight_max": 16.13478660583496,
      "activations/layer1_attention_weight_min": -15.030573844909668,
      "activations/layer20_attention_weight_max": 22.215665817260742,
      "activations/layer20_attention_weight_min": -20.95063591003418,
      "activations/layer21_attention_weight_max": 40.63967514038086,
      "activations/layer21_attention_weight_min": -26.3408260345459,
      "activations/layer22_attention_weight_max": 34.88370895385742,
      "activations/layer22_attention_weight_min": -26.31599235534668,
      "activations/layer23_attention_weight_max": 33.076332092285156,
      "activations/layer23_attention_weight_min": -21.04292106628418,
      "activations/layer2_attention_weight_max": 31.031112670898438,
      "activations/layer2_attention_weight_min": -28.930402755737305,
      "activations/layer3_attention_weight_max": 84.90377807617188,
      "activations/layer3_attention_weight_min": -87.80693817138672,
      "activations/layer4_attention_weight_max": 94.99442291259766,
      "activations/layer4_attention_weight_min": -89.72200012207031,
      "activations/layer5_attention_weight_max": 88.7593994140625,
      "activations/layer5_attention_weight_min": -84.75871276855469,
      "activations/layer6_attention_weight_max": 57.891143798828125,
      "activations/layer6_attention_weight_min": -53.227928161621094,
      "activations/layer7_attention_weight_max": 73.05940246582031,
      "activations/layer7_attention_weight_min": -72.71199798583984,
      "activations/layer8_attention_weight_max": 45.27121353149414,
      "activations/layer8_attention_weight_min": -45.74603271484375,
      "activations/layer9_attention_weight_max": 49.205509185791016,
      "activations/layer9_attention_weight_min": -48.774715423583984,
      "epoch": 6.48,
      "learning_rate": 0.00010928219696969696,
      "loss": 2.8248,
      "step": 111600
    },
    {
      "activations/layer0_attention_weight_max": 15.417136192321777,
      "activations/layer0_attention_weight_min": -14.358771324157715,
      "activations/layer10_attention_weight_max": 37.771976470947266,
      "activations/layer10_attention_weight_min": -36.124027252197266,
      "activations/layer11_attention_weight_max": 28.702850341796875,
      "activations/layer11_attention_weight_min": -29.112783432006836,
      "activations/layer12_attention_weight_max": 21.533363342285156,
      "activations/layer12_attention_weight_min": -37.41542434692383,
      "activations/layer13_attention_weight_max": 53.088680267333984,
      "activations/layer13_attention_weight_min": -32.683677673339844,
      "activations/layer14_attention_weight_max": 46.535099029541016,
      "activations/layer14_attention_weight_min": -29.8433895111084,
      "activations/layer15_attention_weight_max": 34.824928283691406,
      "activations/layer15_attention_weight_min": -29.621509552001953,
      "activations/layer16_attention_weight_max": 30.786779403686523,
      "activations/layer16_attention_weight_min": -25.703466415405273,
      "activations/layer17_attention_weight_max": 54.182193756103516,
      "activations/layer17_attention_weight_min": -41.15386962890625,
      "activations/layer18_attention_weight_max": 46.81521987915039,
      "activations/layer18_attention_weight_min": -34.00575256347656,
      "activations/layer19_attention_weight_max": 22.499982833862305,
      "activations/layer19_attention_weight_min": -20.67988395690918,
      "activations/layer1_attention_weight_max": 17.635988235473633,
      "activations/layer1_attention_weight_min": -16.874422073364258,
      "activations/layer20_attention_weight_max": 21.112808227539062,
      "activations/layer20_attention_weight_min": -19.757169723510742,
      "activations/layer21_attention_weight_max": 41.9760627746582,
      "activations/layer21_attention_weight_min": -27.2030086517334,
      "activations/layer22_attention_weight_max": 32.19615936279297,
      "activations/layer22_attention_weight_min": -22.69518280029297,
      "activations/layer23_attention_weight_max": 34.50823974609375,
      "activations/layer23_attention_weight_min": -23.261716842651367,
      "activations/layer2_attention_weight_max": 28.144439697265625,
      "activations/layer2_attention_weight_min": -27.01077651977539,
      "activations/layer3_attention_weight_max": 79.1803970336914,
      "activations/layer3_attention_weight_min": -80.42057037353516,
      "activations/layer4_attention_weight_max": 97.79383087158203,
      "activations/layer4_attention_weight_min": -90.9360580444336,
      "activations/layer5_attention_weight_max": 90.63399505615234,
      "activations/layer5_attention_weight_min": -85.55297088623047,
      "activations/layer6_attention_weight_max": 56.450599670410156,
      "activations/layer6_attention_weight_min": -52.88132858276367,
      "activations/layer7_attention_weight_max": 73.32691955566406,
      "activations/layer7_attention_weight_min": -75.44207000732422,
      "activations/layer8_attention_weight_max": 43.73601150512695,
      "activations/layer8_attention_weight_min": -43.80137634277344,
      "activations/layer9_attention_weight_max": 60.053035736083984,
      "activations/layer9_attention_weight_min": -58.53750228881836,
      "epoch": 6.49,
      "learning_rate": 0.00010926325757575757,
      "loss": 2.8518,
      "step": 111650
    },
    {
      "activations/layer0_attention_weight_max": 14.250105857849121,
      "activations/layer0_attention_weight_min": -14.872300148010254,
      "activations/layer10_attention_weight_max": 38.216712951660156,
      "activations/layer10_attention_weight_min": -33.98378372192383,
      "activations/layer11_attention_weight_max": 32.45863723754883,
      "activations/layer11_attention_weight_min": -30.147626876831055,
      "activations/layer12_attention_weight_max": 19.45823860168457,
      "activations/layer12_attention_weight_min": -25.96588897705078,
      "activations/layer13_attention_weight_max": 58.18307876586914,
      "activations/layer13_attention_weight_min": -40.508583068847656,
      "activations/layer14_attention_weight_max": 43.15895462036133,
      "activations/layer14_attention_weight_min": -31.973079681396484,
      "activations/layer15_attention_weight_max": 35.57469177246094,
      "activations/layer15_attention_weight_min": -30.945446014404297,
      "activations/layer16_attention_weight_max": 36.1523323059082,
      "activations/layer16_attention_weight_min": -29.2667236328125,
      "activations/layer17_attention_weight_max": 54.36284637451172,
      "activations/layer17_attention_weight_min": -44.49405288696289,
      "activations/layer18_attention_weight_max": 48.66537857055664,
      "activations/layer18_attention_weight_min": -39.53101348876953,
      "activations/layer19_attention_weight_max": 20.206058502197266,
      "activations/layer19_attention_weight_min": -20.99422836303711,
      "activations/layer1_attention_weight_max": 16.536834716796875,
      "activations/layer1_attention_weight_min": -16.102203369140625,
      "activations/layer20_attention_weight_max": 24.635663986206055,
      "activations/layer20_attention_weight_min": -22.138057708740234,
      "activations/layer21_attention_weight_max": 41.152793884277344,
      "activations/layer21_attention_weight_min": -26.642948150634766,
      "activations/layer22_attention_weight_max": 34.503639221191406,
      "activations/layer22_attention_weight_min": -28.13132095336914,
      "activations/layer23_attention_weight_max": 33.94856262207031,
      "activations/layer23_attention_weight_min": -22.62811851501465,
      "activations/layer2_attention_weight_max": 31.08679962158203,
      "activations/layer2_attention_weight_min": -29.81338882446289,
      "activations/layer3_attention_weight_max": 84.47725677490234,
      "activations/layer3_attention_weight_min": -86.73257446289062,
      "activations/layer4_attention_weight_max": 99.89097595214844,
      "activations/layer4_attention_weight_min": -93.93698120117188,
      "activations/layer5_attention_weight_max": 92.9681396484375,
      "activations/layer5_attention_weight_min": -86.47541809082031,
      "activations/layer6_attention_weight_max": 57.12342071533203,
      "activations/layer6_attention_weight_min": -52.02199935913086,
      "activations/layer7_attention_weight_max": 74.88697052001953,
      "activations/layer7_attention_weight_min": -67.86315155029297,
      "activations/layer8_attention_weight_max": 43.00356674194336,
      "activations/layer8_attention_weight_min": -42.4302864074707,
      "activations/layer9_attention_weight_max": 48.652095794677734,
      "activations/layer9_attention_weight_min": -49.44412612915039,
      "epoch": 6.49,
      "learning_rate": 0.00010924431818181817,
      "loss": 2.8249,
      "step": 111700
    },
    {
      "activations/layer0_attention_weight_max": 14.507189750671387,
      "activations/layer0_attention_weight_min": -14.736174583435059,
      "activations/layer10_attention_weight_max": 44.52070236206055,
      "activations/layer10_attention_weight_min": -37.02028274536133,
      "activations/layer11_attention_weight_max": 34.3670654296875,
      "activations/layer11_attention_weight_min": -30.665435791015625,
      "activations/layer12_attention_weight_max": 25.695650100708008,
      "activations/layer12_attention_weight_min": -22.116849899291992,
      "activations/layer13_attention_weight_max": 78.8845443725586,
      "activations/layer13_attention_weight_min": -44.05661392211914,
      "activations/layer14_attention_weight_max": 84.34232330322266,
      "activations/layer14_attention_weight_min": -38.06993865966797,
      "activations/layer15_attention_weight_max": 57.26960754394531,
      "activations/layer15_attention_weight_min": -34.04814529418945,
      "activations/layer16_attention_weight_max": 51.607093811035156,
      "activations/layer16_attention_weight_min": -29.885709762573242,
      "activations/layer17_attention_weight_max": 78.66877746582031,
      "activations/layer17_attention_weight_min": -50.49797821044922,
      "activations/layer18_attention_weight_max": 74.68939971923828,
      "activations/layer18_attention_weight_min": -41.96833038330078,
      "activations/layer19_attention_weight_max": 30.645009994506836,
      "activations/layer19_attention_weight_min": -23.379201889038086,
      "activations/layer1_attention_weight_max": 17.148752212524414,
      "activations/layer1_attention_weight_min": -14.675621032714844,
      "activations/layer20_attention_weight_max": 39.40628433227539,
      "activations/layer20_attention_weight_min": -22.529748916625977,
      "activations/layer21_attention_weight_max": 78.0097427368164,
      "activations/layer21_attention_weight_min": -26.79729461669922,
      "activations/layer22_attention_weight_max": 46.26727294921875,
      "activations/layer22_attention_weight_min": -23.853191375732422,
      "activations/layer23_attention_weight_max": 47.350704193115234,
      "activations/layer23_attention_weight_min": -22.05201530456543,
      "activations/layer2_attention_weight_max": 31.218990325927734,
      "activations/layer2_attention_weight_min": -30.00255584716797,
      "activations/layer3_attention_weight_max": 83.23902130126953,
      "activations/layer3_attention_weight_min": -83.84241485595703,
      "activations/layer4_attention_weight_max": 105.72846984863281,
      "activations/layer4_attention_weight_min": -98.25762176513672,
      "activations/layer5_attention_weight_max": 97.39791870117188,
      "activations/layer5_attention_weight_min": -91.81671142578125,
      "activations/layer6_attention_weight_max": 60.49165344238281,
      "activations/layer6_attention_weight_min": -58.792457580566406,
      "activations/layer7_attention_weight_max": 76.92005157470703,
      "activations/layer7_attention_weight_min": -77.52317810058594,
      "activations/layer8_attention_weight_max": 49.275638580322266,
      "activations/layer8_attention_weight_min": -47.533634185791016,
      "activations/layer9_attention_weight_max": 59.03329086303711,
      "activations/layer9_attention_weight_min": -54.59456253051758,
      "epoch": 6.49,
      "learning_rate": 0.00010922537878787877,
      "loss": 2.8256,
      "step": 111750
    },
    {
      "activations/layer0_attention_weight_max": 15.493352890014648,
      "activations/layer0_attention_weight_min": -15.365365028381348,
      "activations/layer10_attention_weight_max": 40.29345703125,
      "activations/layer10_attention_weight_min": -36.582435607910156,
      "activations/layer11_attention_weight_max": 32.17433547973633,
      "activations/layer11_attention_weight_min": -31.213796615600586,
      "activations/layer12_attention_weight_max": 20.90546989440918,
      "activations/layer12_attention_weight_min": -22.865230560302734,
      "activations/layer13_attention_weight_max": 42.24897003173828,
      "activations/layer13_attention_weight_min": -35.17618179321289,
      "activations/layer14_attention_weight_max": 50.98838806152344,
      "activations/layer14_attention_weight_min": -30.586633682250977,
      "activations/layer15_attention_weight_max": 39.90397644042969,
      "activations/layer15_attention_weight_min": -30.38923454284668,
      "activations/layer16_attention_weight_max": 38.312984466552734,
      "activations/layer16_attention_weight_min": -28.580364227294922,
      "activations/layer17_attention_weight_max": 54.830265045166016,
      "activations/layer17_attention_weight_min": -41.70912551879883,
      "activations/layer18_attention_weight_max": 49.339595794677734,
      "activations/layer18_attention_weight_min": -37.05207824707031,
      "activations/layer19_attention_weight_max": 21.1882266998291,
      "activations/layer19_attention_weight_min": -21.065629959106445,
      "activations/layer1_attention_weight_max": 17.865949630737305,
      "activations/layer1_attention_weight_min": -13.8277587890625,
      "activations/layer20_attention_weight_max": 20.122112274169922,
      "activations/layer20_attention_weight_min": -22.072193145751953,
      "activations/layer21_attention_weight_max": 35.69884490966797,
      "activations/layer21_attention_weight_min": -27.288827896118164,
      "activations/layer22_attention_weight_max": 37.745262145996094,
      "activations/layer22_attention_weight_min": -26.086524963378906,
      "activations/layer23_attention_weight_max": 31.584585189819336,
      "activations/layer23_attention_weight_min": -19.37265396118164,
      "activations/layer2_attention_weight_max": 29.821165084838867,
      "activations/layer2_attention_weight_min": -28.206344604492188,
      "activations/layer3_attention_weight_max": 80.50569915771484,
      "activations/layer3_attention_weight_min": -82.02130889892578,
      "activations/layer4_attention_weight_max": 92.83549499511719,
      "activations/layer4_attention_weight_min": -89.39903259277344,
      "activations/layer5_attention_weight_max": 89.80509948730469,
      "activations/layer5_attention_weight_min": -85.84010314941406,
      "activations/layer6_attention_weight_max": 56.64392852783203,
      "activations/layer6_attention_weight_min": -51.820777893066406,
      "activations/layer7_attention_weight_max": 76.32014465332031,
      "activations/layer7_attention_weight_min": -75.67178344726562,
      "activations/layer8_attention_weight_max": 43.57872772216797,
      "activations/layer8_attention_weight_min": -44.83977127075195,
      "activations/layer9_attention_weight_max": 63.66263961791992,
      "activations/layer9_attention_weight_min": -50.55706787109375,
      "epoch": 6.5,
      "learning_rate": 0.00010920643939393938,
      "loss": 2.8174,
      "step": 111800
    },
    {
      "activations/layer0_attention_weight_max": 15.297264099121094,
      "activations/layer0_attention_weight_min": -15.364457130432129,
      "activations/layer10_attention_weight_max": 39.691062927246094,
      "activations/layer10_attention_weight_min": -35.0947380065918,
      "activations/layer11_attention_weight_max": 29.727933883666992,
      "activations/layer11_attention_weight_min": -30.069652557373047,
      "activations/layer12_attention_weight_max": 21.28424644470215,
      "activations/layer12_attention_weight_min": -25.973325729370117,
      "activations/layer13_attention_weight_max": 51.82710266113281,
      "activations/layer13_attention_weight_min": -37.86072540283203,
      "activations/layer14_attention_weight_max": 42.59590148925781,
      "activations/layer14_attention_weight_min": -31.84308433532715,
      "activations/layer15_attention_weight_max": 36.065879821777344,
      "activations/layer15_attention_weight_min": -29.761611938476562,
      "activations/layer16_attention_weight_max": 33.40963363647461,
      "activations/layer16_attention_weight_min": -26.4593505859375,
      "activations/layer17_attention_weight_max": 53.866966247558594,
      "activations/layer17_attention_weight_min": -40.33348846435547,
      "activations/layer18_attention_weight_max": 50.232093811035156,
      "activations/layer18_attention_weight_min": -35.28733825683594,
      "activations/layer19_attention_weight_max": 20.129072189331055,
      "activations/layer19_attention_weight_min": -20.854387283325195,
      "activations/layer1_attention_weight_max": 17.024911880493164,
      "activations/layer1_attention_weight_min": -13.541290283203125,
      "activations/layer20_attention_weight_max": 20.9554443359375,
      "activations/layer20_attention_weight_min": -20.43056869506836,
      "activations/layer21_attention_weight_max": 40.470863342285156,
      "activations/layer21_attention_weight_min": -25.11687469482422,
      "activations/layer22_attention_weight_max": 34.766845703125,
      "activations/layer22_attention_weight_min": -24.155658721923828,
      "activations/layer23_attention_weight_max": 29.362028121948242,
      "activations/layer23_attention_weight_min": -20.918773651123047,
      "activations/layer2_attention_weight_max": 29.74579620361328,
      "activations/layer2_attention_weight_min": -28.147397994995117,
      "activations/layer3_attention_weight_max": 80.66082000732422,
      "activations/layer3_attention_weight_min": -87.3233642578125,
      "activations/layer4_attention_weight_max": 93.98826599121094,
      "activations/layer4_attention_weight_min": -91.40069580078125,
      "activations/layer5_attention_weight_max": 90.68856811523438,
      "activations/layer5_attention_weight_min": -81.9053955078125,
      "activations/layer6_attention_weight_max": 55.152557373046875,
      "activations/layer6_attention_weight_min": -51.64537048339844,
      "activations/layer7_attention_weight_max": 75.76802825927734,
      "activations/layer7_attention_weight_min": -69.86798858642578,
      "activations/layer8_attention_weight_max": 44.05865478515625,
      "activations/layer8_attention_weight_min": -45.71689224243164,
      "activations/layer9_attention_weight_max": 51.824012756347656,
      "activations/layer9_attention_weight_min": -45.81306076049805,
      "epoch": 6.5,
      "learning_rate": 0.00010918749999999999,
      "loss": 2.8389,
      "step": 111850
    },
    {
      "activations/layer0_attention_weight_max": 15.142040252685547,
      "activations/layer0_attention_weight_min": -14.730369567871094,
      "activations/layer10_attention_weight_max": 35.85129165649414,
      "activations/layer10_attention_weight_min": -34.66898727416992,
      "activations/layer11_attention_weight_max": 29.191274642944336,
      "activations/layer11_attention_weight_min": -27.990009307861328,
      "activations/layer12_attention_weight_max": 20.009220123291016,
      "activations/layer12_attention_weight_min": -24.158058166503906,
      "activations/layer13_attention_weight_max": 39.51300811767578,
      "activations/layer13_attention_weight_min": -33.73686218261719,
      "activations/layer14_attention_weight_max": 39.26227569580078,
      "activations/layer14_attention_weight_min": -31.743574142456055,
      "activations/layer15_attention_weight_max": 38.128814697265625,
      "activations/layer15_attention_weight_min": -28.596214294433594,
      "activations/layer16_attention_weight_max": 31.677734375,
      "activations/layer16_attention_weight_min": -27.761287689208984,
      "activations/layer17_attention_weight_max": 54.7501106262207,
      "activations/layer17_attention_weight_min": -41.255210876464844,
      "activations/layer18_attention_weight_max": 46.58458709716797,
      "activations/layer18_attention_weight_min": -33.69060516357422,
      "activations/layer19_attention_weight_max": 29.147794723510742,
      "activations/layer19_attention_weight_min": -20.613264083862305,
      "activations/layer1_attention_weight_max": 15.92137622833252,
      "activations/layer1_attention_weight_min": -12.089445114135742,
      "activations/layer20_attention_weight_max": 22.99399757385254,
      "activations/layer20_attention_weight_min": -21.94302749633789,
      "activations/layer21_attention_weight_max": 35.85136032104492,
      "activations/layer21_attention_weight_min": -25.999980926513672,
      "activations/layer22_attention_weight_max": 35.54783248901367,
      "activations/layer22_attention_weight_min": -25.913015365600586,
      "activations/layer23_attention_weight_max": 29.501354217529297,
      "activations/layer23_attention_weight_min": -20.70416259765625,
      "activations/layer2_attention_weight_max": 27.12025260925293,
      "activations/layer2_attention_weight_min": -25.903745651245117,
      "activations/layer3_attention_weight_max": 74.9639663696289,
      "activations/layer3_attention_weight_min": -81.56617736816406,
      "activations/layer4_attention_weight_max": 94.61678314208984,
      "activations/layer4_attention_weight_min": -91.10265350341797,
      "activations/layer5_attention_weight_max": 91.4207992553711,
      "activations/layer5_attention_weight_min": -83.70160675048828,
      "activations/layer6_attention_weight_max": 55.46870040893555,
      "activations/layer6_attention_weight_min": -51.54821014404297,
      "activations/layer7_attention_weight_max": 75.64048767089844,
      "activations/layer7_attention_weight_min": -71.50312805175781,
      "activations/layer8_attention_weight_max": 43.00761795043945,
      "activations/layer8_attention_weight_min": -42.19312286376953,
      "activations/layer9_attention_weight_max": 48.038753509521484,
      "activations/layer9_attention_weight_min": -47.1121940612793,
      "epoch": 6.5,
      "learning_rate": 0.0001091685606060606,
      "loss": 2.8317,
      "step": 111900
    },
    {
      "activations/layer0_attention_weight_max": 14.86996841430664,
      "activations/layer0_attention_weight_min": -14.70407772064209,
      "activations/layer10_attention_weight_max": 39.899688720703125,
      "activations/layer10_attention_weight_min": -36.79668045043945,
      "activations/layer11_attention_weight_max": 32.58869552612305,
      "activations/layer11_attention_weight_min": -29.60299301147461,
      "activations/layer12_attention_weight_max": 22.69086265563965,
      "activations/layer12_attention_weight_min": -24.918460845947266,
      "activations/layer13_attention_weight_max": 51.10532760620117,
      "activations/layer13_attention_weight_min": -37.04198455810547,
      "activations/layer14_attention_weight_max": 42.99270248413086,
      "activations/layer14_attention_weight_min": -31.445158004760742,
      "activations/layer15_attention_weight_max": 41.2588005065918,
      "activations/layer15_attention_weight_min": -31.093162536621094,
      "activations/layer16_attention_weight_max": 33.87644577026367,
      "activations/layer16_attention_weight_min": -27.29950714111328,
      "activations/layer17_attention_weight_max": 58.36766815185547,
      "activations/layer17_attention_weight_min": -41.869022369384766,
      "activations/layer18_attention_weight_max": 53.2297477722168,
      "activations/layer18_attention_weight_min": -34.639739990234375,
      "activations/layer19_attention_weight_max": 24.768646240234375,
      "activations/layer19_attention_weight_min": -19.37670135498047,
      "activations/layer1_attention_weight_max": 16.598716735839844,
      "activations/layer1_attention_weight_min": -15.430985450744629,
      "activations/layer20_attention_weight_max": 24.700536727905273,
      "activations/layer20_attention_weight_min": -21.248912811279297,
      "activations/layer21_attention_weight_max": 41.68046569824219,
      "activations/layer21_attention_weight_min": -25.945724487304688,
      "activations/layer22_attention_weight_max": 34.23692321777344,
      "activations/layer22_attention_weight_min": -24.72818374633789,
      "activations/layer23_attention_weight_max": 34.554161071777344,
      "activations/layer23_attention_weight_min": -22.366317749023438,
      "activations/layer2_attention_weight_max": 31.705663681030273,
      "activations/layer2_attention_weight_min": -29.930599212646484,
      "activations/layer3_attention_weight_max": 81.09847259521484,
      "activations/layer3_attention_weight_min": -83.84746551513672,
      "activations/layer4_attention_weight_max": 103.83209228515625,
      "activations/layer4_attention_weight_min": -94.11099243164062,
      "activations/layer5_attention_weight_max": 96.96725463867188,
      "activations/layer5_attention_weight_min": -88.72319793701172,
      "activations/layer6_attention_weight_max": 59.981903076171875,
      "activations/layer6_attention_weight_min": -52.99051284790039,
      "activations/layer7_attention_weight_max": 76.45342254638672,
      "activations/layer7_attention_weight_min": -69.42695617675781,
      "activations/layer8_attention_weight_max": 46.38623046875,
      "activations/layer8_attention_weight_min": -47.05559158325195,
      "activations/layer9_attention_weight_max": 49.94512939453125,
      "activations/layer9_attention_weight_min": -51.015525817871094,
      "epoch": 6.5,
      "learning_rate": 0.0001091496212121212,
      "loss": 2.8326,
      "step": 111950
    },
    {
      "activations/layer0_attention_weight_max": 15.04183292388916,
      "activations/layer0_attention_weight_min": -14.778956413269043,
      "activations/layer10_attention_weight_max": 38.75910949707031,
      "activations/layer10_attention_weight_min": -34.59587478637695,
      "activations/layer11_attention_weight_max": 33.38459777832031,
      "activations/layer11_attention_weight_min": -28.271976470947266,
      "activations/layer12_attention_weight_max": 23.163040161132812,
      "activations/layer12_attention_weight_min": -25.28732681274414,
      "activations/layer13_attention_weight_max": 53.19645309448242,
      "activations/layer13_attention_weight_min": -34.27987289428711,
      "activations/layer14_attention_weight_max": 52.46284103393555,
      "activations/layer14_attention_weight_min": -32.46883010864258,
      "activations/layer15_attention_weight_max": 46.51545715332031,
      "activations/layer15_attention_weight_min": -30.063230514526367,
      "activations/layer16_attention_weight_max": 35.825279235839844,
      "activations/layer16_attention_weight_min": -27.389070510864258,
      "activations/layer17_attention_weight_max": 57.88184356689453,
      "activations/layer17_attention_weight_min": -43.699039459228516,
      "activations/layer18_attention_weight_max": 52.798057556152344,
      "activations/layer18_attention_weight_min": -36.91761016845703,
      "activations/layer19_attention_weight_max": 28.615453720092773,
      "activations/layer19_attention_weight_min": -21.420894622802734,
      "activations/layer1_attention_weight_max": 17.65049934387207,
      "activations/layer1_attention_weight_min": -15.960809707641602,
      "activations/layer20_attention_weight_max": 25.539100646972656,
      "activations/layer20_attention_weight_min": -20.191192626953125,
      "activations/layer21_attention_weight_max": 45.90360641479492,
      "activations/layer21_attention_weight_min": -24.626523971557617,
      "activations/layer22_attention_weight_max": 35.626731872558594,
      "activations/layer22_attention_weight_min": -23.438190460205078,
      "activations/layer23_attention_weight_max": 33.652591705322266,
      "activations/layer23_attention_weight_min": -22.03744125366211,
      "activations/layer2_attention_weight_max": 31.366178512573242,
      "activations/layer2_attention_weight_min": -28.917905807495117,
      "activations/layer3_attention_weight_max": 81.78883361816406,
      "activations/layer3_attention_weight_min": -87.34549713134766,
      "activations/layer4_attention_weight_max": 98.82547760009766,
      "activations/layer4_attention_weight_min": -95.22600555419922,
      "activations/layer5_attention_weight_max": 95.17488098144531,
      "activations/layer5_attention_weight_min": -89.8303451538086,
      "activations/layer6_attention_weight_max": 59.3984260559082,
      "activations/layer6_attention_weight_min": -57.756141662597656,
      "activations/layer7_attention_weight_max": 75.44063568115234,
      "activations/layer7_attention_weight_min": -71.51520538330078,
      "activations/layer8_attention_weight_max": 44.11819076538086,
      "activations/layer8_attention_weight_min": -45.518001556396484,
      "activations/layer9_attention_weight_max": 54.313812255859375,
      "activations/layer9_attention_weight_min": -48.56429672241211,
      "epoch": 6.51,
      "learning_rate": 0.00010913068181818181,
      "loss": 2.8277,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5499,
      "eval_samples_per_second": 502.231,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5499,
      "eval_openwebtext_samples_per_second": 502.231,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_wikitext_loss": 3.015625,
      "eval_wikitext_ppl": 20.401838105461145,
      "eval_wikitext_runtime": 2.037,
      "eval_wikitext_samples_per_second": 223.858,
      "step": 112000
    },
    {
      "epoch": 6.51,
      "eval_lambada_loss": 2.716796875,
      "eval_lambada_ppl": 15.131775567308885,
      "eval_lambada_runtime": 9.6501,
      "eval_lambada_samples_per_second": 504.554,
      "step": 112000
    },
    {
      "activations/layer0_attention_weight_max": 14.646137237548828,
      "activations/layer0_attention_weight_min": -14.835930824279785,
      "activations/layer10_attention_weight_max": 44.093257904052734,
      "activations/layer10_attention_weight_min": -39.553802490234375,
      "activations/layer11_attention_weight_max": 33.74968719482422,
      "activations/layer11_attention_weight_min": -33.79316711425781,
      "activations/layer12_attention_weight_max": 19.16109275817871,
      "activations/layer12_attention_weight_min": -29.553504943847656,
      "activations/layer13_attention_weight_max": 58.216949462890625,
      "activations/layer13_attention_weight_min": -33.41928482055664,
      "activations/layer14_attention_weight_max": 56.41759490966797,
      "activations/layer14_attention_weight_min": -34.55546188354492,
      "activations/layer15_attention_weight_max": 44.34257507324219,
      "activations/layer15_attention_weight_min": -30.745237350463867,
      "activations/layer16_attention_weight_max": 37.00586700439453,
      "activations/layer16_attention_weight_min": -27.382680892944336,
      "activations/layer17_attention_weight_max": 62.112060546875,
      "activations/layer17_attention_weight_min": -44.22323226928711,
      "activations/layer18_attention_weight_max": 55.69456100463867,
      "activations/layer18_attention_weight_min": -40.29039001464844,
      "activations/layer19_attention_weight_max": 28.41590690612793,
      "activations/layer19_attention_weight_min": -21.855609893798828,
      "activations/layer1_attention_weight_max": 17.596782684326172,
      "activations/layer1_attention_weight_min": -13.167284965515137,
      "activations/layer20_attention_weight_max": 27.382049560546875,
      "activations/layer20_attention_weight_min": -20.985637664794922,
      "activations/layer21_attention_weight_max": 54.07415771484375,
      "activations/layer21_attention_weight_min": -25.686344146728516,
      "activations/layer22_attention_weight_max": 38.76472473144531,
      "activations/layer22_attention_weight_min": -24.7886962890625,
      "activations/layer23_attention_weight_max": 36.52907180786133,
      "activations/layer23_attention_weight_min": -19.634227752685547,
      "activations/layer2_attention_weight_max": 32.123558044433594,
      "activations/layer2_attention_weight_min": -29.927371978759766,
      "activations/layer3_attention_weight_max": 85.83451843261719,
      "activations/layer3_attention_weight_min": -86.54473876953125,
      "activations/layer4_attention_weight_max": 104.86402893066406,
      "activations/layer4_attention_weight_min": -99.93596649169922,
      "activations/layer5_attention_weight_max": 97.78719329833984,
      "activations/layer5_attention_weight_min": -89.16285705566406,
      "activations/layer6_attention_weight_max": 58.00871658325195,
      "activations/layer6_attention_weight_min": -53.34595489501953,
      "activations/layer7_attention_weight_max": 77.77647399902344,
      "activations/layer7_attention_weight_min": -73.5122299194336,
      "activations/layer8_attention_weight_max": 46.86890411376953,
      "activations/layer8_attention_weight_min": -45.500762939453125,
      "activations/layer9_attention_weight_max": 61.28823471069336,
      "activations/layer9_attention_weight_min": -55.26791000366211,
      "epoch": 6.51,
      "learning_rate": 0.00010911174242424242,
      "loss": 2.8284,
      "step": 112050
    },
    {
      "activations/layer0_attention_weight_max": 15.073206901550293,
      "activations/layer0_attention_weight_min": -15.001897811889648,
      "activations/layer10_attention_weight_max": 39.7412109375,
      "activations/layer10_attention_weight_min": -36.91335678100586,
      "activations/layer11_attention_weight_max": 30.977819442749023,
      "activations/layer11_attention_weight_min": -30.624042510986328,
      "activations/layer12_attention_weight_max": 20.4710750579834,
      "activations/layer12_attention_weight_min": -30.456035614013672,
      "activations/layer13_attention_weight_max": 51.70155715942383,
      "activations/layer13_attention_weight_min": -31.028820037841797,
      "activations/layer14_attention_weight_max": 65.14543914794922,
      "activations/layer14_attention_weight_min": -33.87448501586914,
      "activations/layer15_attention_weight_max": 49.09831237792969,
      "activations/layer15_attention_weight_min": -29.838302612304688,
      "activations/layer16_attention_weight_max": 33.628299713134766,
      "activations/layer16_attention_weight_min": -26.367778778076172,
      "activations/layer17_attention_weight_max": 50.47065734863281,
      "activations/layer17_attention_weight_min": -41.25465393066406,
      "activations/layer18_attention_weight_max": 43.6948127746582,
      "activations/layer18_attention_weight_min": -35.1049919128418,
      "activations/layer19_attention_weight_max": 20.51827049255371,
      "activations/layer19_attention_weight_min": -18.90806770324707,
      "activations/layer1_attention_weight_max": 17.29128074645996,
      "activations/layer1_attention_weight_min": -14.97580623626709,
      "activations/layer20_attention_weight_max": 19.627500534057617,
      "activations/layer20_attention_weight_min": -21.17838478088379,
      "activations/layer21_attention_weight_max": 35.02486038208008,
      "activations/layer21_attention_weight_min": -25.353384017944336,
      "activations/layer22_attention_weight_max": 33.104087829589844,
      "activations/layer22_attention_weight_min": -25.22169303894043,
      "activations/layer23_attention_weight_max": 28.75228500366211,
      "activations/layer23_attention_weight_min": -20.43764877319336,
      "activations/layer2_attention_weight_max": 32.24754333496094,
      "activations/layer2_attention_weight_min": -29.32636833190918,
      "activations/layer3_attention_weight_max": 83.01921081542969,
      "activations/layer3_attention_weight_min": -87.82269287109375,
      "activations/layer4_attention_weight_max": 100.3294448852539,
      "activations/layer4_attention_weight_min": -98.15214538574219,
      "activations/layer5_attention_weight_max": 93.53981018066406,
      "activations/layer5_attention_weight_min": -86.59046936035156,
      "activations/layer6_attention_weight_max": 55.74138259887695,
      "activations/layer6_attention_weight_min": -52.12266540527344,
      "activations/layer7_attention_weight_max": 82.00109100341797,
      "activations/layer7_attention_weight_min": -70.6844253540039,
      "activations/layer8_attention_weight_max": 43.93318557739258,
      "activations/layer8_attention_weight_min": -43.54952621459961,
      "activations/layer9_attention_weight_max": 66.47447967529297,
      "activations/layer9_attention_weight_min": -54.80707550048828,
      "epoch": 6.51,
      "learning_rate": 0.00010909280303030301,
      "loss": 2.828,
      "step": 112100
    },
    {
      "activations/layer0_attention_weight_max": 16.912654876708984,
      "activations/layer0_attention_weight_min": -14.881848335266113,
      "activations/layer10_attention_weight_max": 38.810977935791016,
      "activations/layer10_attention_weight_min": -35.15340042114258,
      "activations/layer11_attention_weight_max": 29.66275405883789,
      "activations/layer11_attention_weight_min": -28.88343620300293,
      "activations/layer12_attention_weight_max": 21.020687103271484,
      "activations/layer12_attention_weight_min": -26.314895629882812,
      "activations/layer13_attention_weight_max": 45.676307678222656,
      "activations/layer13_attention_weight_min": -37.08086395263672,
      "activations/layer14_attention_weight_max": 40.28770446777344,
      "activations/layer14_attention_weight_min": -29.979961395263672,
      "activations/layer15_attention_weight_max": 39.45268249511719,
      "activations/layer15_attention_weight_min": -28.904130935668945,
      "activations/layer16_attention_weight_max": 34.314395904541016,
      "activations/layer16_attention_weight_min": -26.23858070373535,
      "activations/layer17_attention_weight_max": 56.769378662109375,
      "activations/layer17_attention_weight_min": -42.2189826965332,
      "activations/layer18_attention_weight_max": 49.450538635253906,
      "activations/layer18_attention_weight_min": -34.93478012084961,
      "activations/layer19_attention_weight_max": 22.818859100341797,
      "activations/layer19_attention_weight_min": -21.415937423706055,
      "activations/layer1_attention_weight_max": 16.9750919342041,
      "activations/layer1_attention_weight_min": -14.739992141723633,
      "activations/layer20_attention_weight_max": 24.226360321044922,
      "activations/layer20_attention_weight_min": -21.928478240966797,
      "activations/layer21_attention_weight_max": 45.57594299316406,
      "activations/layer21_attention_weight_min": -25.236967086791992,
      "activations/layer22_attention_weight_max": 35.80253219604492,
      "activations/layer22_attention_weight_min": -27.760334014892578,
      "activations/layer23_attention_weight_max": 34.590030670166016,
      "activations/layer23_attention_weight_min": -20.879920959472656,
      "activations/layer2_attention_weight_max": 29.920543670654297,
      "activations/layer2_attention_weight_min": -28.32533073425293,
      "activations/layer3_attention_weight_max": 82.14141845703125,
      "activations/layer3_attention_weight_min": -84.38554382324219,
      "activations/layer4_attention_weight_max": 98.4826889038086,
      "activations/layer4_attention_weight_min": -94.05464935302734,
      "activations/layer5_attention_weight_max": 93.56149291992188,
      "activations/layer5_attention_weight_min": -85.12687683105469,
      "activations/layer6_attention_weight_max": 55.802635192871094,
      "activations/layer6_attention_weight_min": -51.103153228759766,
      "activations/layer7_attention_weight_max": 74.5948257446289,
      "activations/layer7_attention_weight_min": -69.11038208007812,
      "activations/layer8_attention_weight_max": 46.25120162963867,
      "activations/layer8_attention_weight_min": -43.172271728515625,
      "activations/layer9_attention_weight_max": 49.66250228881836,
      "activations/layer9_attention_weight_min": -47.03056335449219,
      "epoch": 6.52,
      "learning_rate": 0.00010907386363636363,
      "loss": 2.8543,
      "step": 112150
    },
    {
      "activations/layer0_attention_weight_max": 15.078169822692871,
      "activations/layer0_attention_weight_min": -14.553482055664062,
      "activations/layer10_attention_weight_max": 40.32715606689453,
      "activations/layer10_attention_weight_min": -35.163482666015625,
      "activations/layer11_attention_weight_max": 36.26865768432617,
      "activations/layer11_attention_weight_min": -31.634868621826172,
      "activations/layer12_attention_weight_max": 23.32201385498047,
      "activations/layer12_attention_weight_min": -29.531126022338867,
      "activations/layer13_attention_weight_max": 55.353843688964844,
      "activations/layer13_attention_weight_min": -47.65425491333008,
      "activations/layer14_attention_weight_max": 44.60557174682617,
      "activations/layer14_attention_weight_min": -35.26923751831055,
      "activations/layer15_attention_weight_max": 41.19776916503906,
      "activations/layer15_attention_weight_min": -31.645235061645508,
      "activations/layer16_attention_weight_max": 38.18574142456055,
      "activations/layer16_attention_weight_min": -28.739023208618164,
      "activations/layer17_attention_weight_max": 61.22086715698242,
      "activations/layer17_attention_weight_min": -42.85227584838867,
      "activations/layer18_attention_weight_max": 52.14006805419922,
      "activations/layer18_attention_weight_min": -38.737648010253906,
      "activations/layer19_attention_weight_max": 24.534658432006836,
      "activations/layer19_attention_weight_min": -24.32937240600586,
      "activations/layer1_attention_weight_max": 17.6285400390625,
      "activations/layer1_attention_weight_min": -13.455706596374512,
      "activations/layer20_attention_weight_max": 26.534887313842773,
      "activations/layer20_attention_weight_min": -23.12342071533203,
      "activations/layer21_attention_weight_max": 50.35293960571289,
      "activations/layer21_attention_weight_min": -29.10813331604004,
      "activations/layer22_attention_weight_max": 37.32977294921875,
      "activations/layer22_attention_weight_min": -24.56235694885254,
      "activations/layer23_attention_weight_max": 37.260433197021484,
      "activations/layer23_attention_weight_min": -19.351211547851562,
      "activations/layer2_attention_weight_max": 30.987346649169922,
      "activations/layer2_attention_weight_min": -27.40387535095215,
      "activations/layer3_attention_weight_max": 83.84977722167969,
      "activations/layer3_attention_weight_min": -87.13951873779297,
      "activations/layer4_attention_weight_max": 100.4561538696289,
      "activations/layer4_attention_weight_min": -93.40248107910156,
      "activations/layer5_attention_weight_max": 96.02850341796875,
      "activations/layer5_attention_weight_min": -83.33228302001953,
      "activations/layer6_attention_weight_max": 56.92179489135742,
      "activations/layer6_attention_weight_min": -52.97310256958008,
      "activations/layer7_attention_weight_max": 74.81639099121094,
      "activations/layer7_attention_weight_min": -70.91898345947266,
      "activations/layer8_attention_weight_max": 48.1772346496582,
      "activations/layer8_attention_weight_min": -46.61903762817383,
      "activations/layer9_attention_weight_max": 53.89276885986328,
      "activations/layer9_attention_weight_min": -54.115142822265625,
      "epoch": 6.52,
      "learning_rate": 0.00010905492424242424,
      "loss": 2.8278,
      "step": 112200
    },
    {
      "activations/layer0_attention_weight_max": 14.875581741333008,
      "activations/layer0_attention_weight_min": -14.613636016845703,
      "activations/layer10_attention_weight_max": 41.97167205810547,
      "activations/layer10_attention_weight_min": -39.67061233520508,
      "activations/layer11_attention_weight_max": 36.894187927246094,
      "activations/layer11_attention_weight_min": -33.162315368652344,
      "activations/layer12_attention_weight_max": 31.93543815612793,
      "activations/layer12_attention_weight_min": -23.81853485107422,
      "activations/layer13_attention_weight_max": 50.069828033447266,
      "activations/layer13_attention_weight_min": -31.586669921875,
      "activations/layer14_attention_weight_max": 47.15339279174805,
      "activations/layer14_attention_weight_min": -29.405284881591797,
      "activations/layer15_attention_weight_max": 40.50835037231445,
      "activations/layer15_attention_weight_min": -29.786561965942383,
      "activations/layer16_attention_weight_max": 39.77425765991211,
      "activations/layer16_attention_weight_min": -28.642189025878906,
      "activations/layer17_attention_weight_max": 55.04222869873047,
      "activations/layer17_attention_weight_min": -39.57079315185547,
      "activations/layer18_attention_weight_max": 47.320159912109375,
      "activations/layer18_attention_weight_min": -34.44969940185547,
      "activations/layer19_attention_weight_max": 22.51670265197754,
      "activations/layer19_attention_weight_min": -19.37204360961914,
      "activations/layer1_attention_weight_max": 17.57135581970215,
      "activations/layer1_attention_weight_min": -15.212634086608887,
      "activations/layer20_attention_weight_max": 23.802207946777344,
      "activations/layer20_attention_weight_min": -20.525909423828125,
      "activations/layer21_attention_weight_max": 45.79066467285156,
      "activations/layer21_attention_weight_min": -26.021453857421875,
      "activations/layer22_attention_weight_max": 35.816802978515625,
      "activations/layer22_attention_weight_min": -24.84014892578125,
      "activations/layer23_attention_weight_max": 37.08269500732422,
      "activations/layer23_attention_weight_min": -20.472564697265625,
      "activations/layer2_attention_weight_max": 29.928253173828125,
      "activations/layer2_attention_weight_min": -30.92865562438965,
      "activations/layer3_attention_weight_max": 85.26829528808594,
      "activations/layer3_attention_weight_min": -89.0534896850586,
      "activations/layer4_attention_weight_max": 99.43309783935547,
      "activations/layer4_attention_weight_min": -96.26795196533203,
      "activations/layer5_attention_weight_max": 93.90618133544922,
      "activations/layer5_attention_weight_min": -87.50292205810547,
      "activations/layer6_attention_weight_max": 56.546146392822266,
      "activations/layer6_attention_weight_min": -54.12218475341797,
      "activations/layer7_attention_weight_max": 82.81546783447266,
      "activations/layer7_attention_weight_min": -75.3071060180664,
      "activations/layer8_attention_weight_max": 50.44450378417969,
      "activations/layer8_attention_weight_min": -47.31794357299805,
      "activations/layer9_attention_weight_max": 65.69344329833984,
      "activations/layer9_attention_weight_min": -53.322845458984375,
      "epoch": 6.52,
      "learning_rate": 0.00010903598484848483,
      "loss": 2.8323,
      "step": 112250
    },
    {
      "activations/layer0_attention_weight_max": 14.416253089904785,
      "activations/layer0_attention_weight_min": -14.4077730178833,
      "activations/layer10_attention_weight_max": 35.58876419067383,
      "activations/layer10_attention_weight_min": -33.458587646484375,
      "activations/layer11_attention_weight_max": 27.323745727539062,
      "activations/layer11_attention_weight_min": -27.829059600830078,
      "activations/layer12_attention_weight_max": 20.093496322631836,
      "activations/layer12_attention_weight_min": -25.155658721923828,
      "activations/layer13_attention_weight_max": 38.43468475341797,
      "activations/layer13_attention_weight_min": -33.283660888671875,
      "activations/layer14_attention_weight_max": 40.81174850463867,
      "activations/layer14_attention_weight_min": -28.379404067993164,
      "activations/layer15_attention_weight_max": 34.55781555175781,
      "activations/layer15_attention_weight_min": -28.911903381347656,
      "activations/layer16_attention_weight_max": 34.055484771728516,
      "activations/layer16_attention_weight_min": -25.019336700439453,
      "activations/layer17_attention_weight_max": 55.97154235839844,
      "activations/layer17_attention_weight_min": -41.228912353515625,
      "activations/layer18_attention_weight_max": 46.90161895751953,
      "activations/layer18_attention_weight_min": -33.57832717895508,
      "activations/layer19_attention_weight_max": 23.48931312561035,
      "activations/layer19_attention_weight_min": -21.39862060546875,
      "activations/layer1_attention_weight_max": 16.834209442138672,
      "activations/layer1_attention_weight_min": -14.827293395996094,
      "activations/layer20_attention_weight_max": 20.926698684692383,
      "activations/layer20_attention_weight_min": -22.273168563842773,
      "activations/layer21_attention_weight_max": 34.39058303833008,
      "activations/layer21_attention_weight_min": -25.984155654907227,
      "activations/layer22_attention_weight_max": 29.6180419921875,
      "activations/layer22_attention_weight_min": -27.56290626525879,
      "activations/layer23_attention_weight_max": 30.913137435913086,
      "activations/layer23_attention_weight_min": -21.600746154785156,
      "activations/layer2_attention_weight_max": 31.299222946166992,
      "activations/layer2_attention_weight_min": -28.17599868774414,
      "activations/layer3_attention_weight_max": 83.07447814941406,
      "activations/layer3_attention_weight_min": -87.80970764160156,
      "activations/layer4_attention_weight_max": 93.84803009033203,
      "activations/layer4_attention_weight_min": -93.3258285522461,
      "activations/layer5_attention_weight_max": 88.68211364746094,
      "activations/layer5_attention_weight_min": -83.63349151611328,
      "activations/layer6_attention_weight_max": 55.03824234008789,
      "activations/layer6_attention_weight_min": -51.21059799194336,
      "activations/layer7_attention_weight_max": 71.1624984741211,
      "activations/layer7_attention_weight_min": -67.85529327392578,
      "activations/layer8_attention_weight_max": 40.7305908203125,
      "activations/layer8_attention_weight_min": -40.81882095336914,
      "activations/layer9_attention_weight_max": 47.10165023803711,
      "activations/layer9_attention_weight_min": -44.24462127685547,
      "epoch": 6.53,
      "learning_rate": 0.00010901704545454545,
      "loss": 2.8118,
      "step": 112300
    },
    {
      "activations/layer0_attention_weight_max": 14.70365047454834,
      "activations/layer0_attention_weight_min": -14.320362091064453,
      "activations/layer10_attention_weight_max": 46.83417892456055,
      "activations/layer10_attention_weight_min": -44.81764602661133,
      "activations/layer11_attention_weight_max": 36.079551696777344,
      "activations/layer11_attention_weight_min": -33.49812698364258,
      "activations/layer12_attention_weight_max": 23.50741958618164,
      "activations/layer12_attention_weight_min": -26.690101623535156,
      "activations/layer13_attention_weight_max": 66.95249938964844,
      "activations/layer13_attention_weight_min": -38.423377990722656,
      "activations/layer14_attention_weight_max": 68.53256225585938,
      "activations/layer14_attention_weight_min": -38.08578872680664,
      "activations/layer15_attention_weight_max": 42.735389709472656,
      "activations/layer15_attention_weight_min": -29.071306228637695,
      "activations/layer16_attention_weight_max": 33.37863540649414,
      "activations/layer16_attention_weight_min": -28.33057975769043,
      "activations/layer17_attention_weight_max": 58.39982223510742,
      "activations/layer17_attention_weight_min": -46.759761810302734,
      "activations/layer18_attention_weight_max": 50.67796325683594,
      "activations/layer18_attention_weight_min": -40.04607391357422,
      "activations/layer19_attention_weight_max": 21.937042236328125,
      "activations/layer19_attention_weight_min": -24.408220291137695,
      "activations/layer1_attention_weight_max": 17.919763565063477,
      "activations/layer1_attention_weight_min": -16.8692569732666,
      "activations/layer20_attention_weight_max": 22.515050888061523,
      "activations/layer20_attention_weight_min": -20.221988677978516,
      "activations/layer21_attention_weight_max": 49.4874267578125,
      "activations/layer21_attention_weight_min": -27.189971923828125,
      "activations/layer22_attention_weight_max": 32.43099594116211,
      "activations/layer22_attention_weight_min": -22.80980682373047,
      "activations/layer23_attention_weight_max": 31.184621810913086,
      "activations/layer23_attention_weight_min": -20.74812889099121,
      "activations/layer2_attention_weight_max": 33.13299560546875,
      "activations/layer2_attention_weight_min": -33.09975051879883,
      "activations/layer3_attention_weight_max": 89.24752044677734,
      "activations/layer3_attention_weight_min": -90.18772888183594,
      "activations/layer4_attention_weight_max": 108.89630126953125,
      "activations/layer4_attention_weight_min": -98.8482894897461,
      "activations/layer5_attention_weight_max": 99.71148681640625,
      "activations/layer5_attention_weight_min": -93.2938461303711,
      "activations/layer6_attention_weight_max": 60.20869827270508,
      "activations/layer6_attention_weight_min": -56.95198440551758,
      "activations/layer7_attention_weight_max": 87.14828491210938,
      "activations/layer7_attention_weight_min": -84.80123138427734,
      "activations/layer8_attention_weight_max": 51.45606231689453,
      "activations/layer8_attention_weight_min": -49.93156814575195,
      "activations/layer9_attention_weight_max": 73.4940185546875,
      "activations/layer9_attention_weight_min": -57.1446647644043,
      "epoch": 6.53,
      "learning_rate": 0.00010899810606060605,
      "loss": 2.8202,
      "step": 112350
    },
    {
      "activations/layer0_attention_weight_max": 15.026741027832031,
      "activations/layer0_attention_weight_min": -14.884136199951172,
      "activations/layer10_attention_weight_max": 36.945068359375,
      "activations/layer10_attention_weight_min": -34.25887680053711,
      "activations/layer11_attention_weight_max": 30.278133392333984,
      "activations/layer11_attention_weight_min": -28.971187591552734,
      "activations/layer12_attention_weight_max": 21.55204200744629,
      "activations/layer12_attention_weight_min": -24.003747940063477,
      "activations/layer13_attention_weight_max": 44.6879997253418,
      "activations/layer13_attention_weight_min": -33.304931640625,
      "activations/layer14_attention_weight_max": 48.59501647949219,
      "activations/layer14_attention_weight_min": -29.657527923583984,
      "activations/layer15_attention_weight_max": 37.4604606628418,
      "activations/layer15_attention_weight_min": -30.661577224731445,
      "activations/layer16_attention_weight_max": 33.34294128417969,
      "activations/layer16_attention_weight_min": -26.303680419921875,
      "activations/layer17_attention_weight_max": 57.57216262817383,
      "activations/layer17_attention_weight_min": -42.21602249145508,
      "activations/layer18_attention_weight_max": 51.70369338989258,
      "activations/layer18_attention_weight_min": -36.7823600769043,
      "activations/layer19_attention_weight_max": 24.60175895690918,
      "activations/layer19_attention_weight_min": -21.164541244506836,
      "activations/layer1_attention_weight_max": 16.854764938354492,
      "activations/layer1_attention_weight_min": -14.77835464477539,
      "activations/layer20_attention_weight_max": 22.573514938354492,
      "activations/layer20_attention_weight_min": -21.819210052490234,
      "activations/layer21_attention_weight_max": 39.31332778930664,
      "activations/layer21_attention_weight_min": -30.57961654663086,
      "activations/layer22_attention_weight_max": 37.87900924682617,
      "activations/layer22_attention_weight_min": -23.9475040435791,
      "activations/layer23_attention_weight_max": 38.841552734375,
      "activations/layer23_attention_weight_min": -19.79351043701172,
      "activations/layer2_attention_weight_max": 29.956518173217773,
      "activations/layer2_attention_weight_min": -28.97127342224121,
      "activations/layer3_attention_weight_max": 81.18861389160156,
      "activations/layer3_attention_weight_min": -86.30400085449219,
      "activations/layer4_attention_weight_max": 98.01396942138672,
      "activations/layer4_attention_weight_min": -95.0971450805664,
      "activations/layer5_attention_weight_max": 90.53121185302734,
      "activations/layer5_attention_weight_min": -86.9753646850586,
      "activations/layer6_attention_weight_max": 53.04865264892578,
      "activations/layer6_attention_weight_min": -53.374080657958984,
      "activations/layer7_attention_weight_max": 72.30591583251953,
      "activations/layer7_attention_weight_min": -75.85550689697266,
      "activations/layer8_attention_weight_max": 42.54184341430664,
      "activations/layer8_attention_weight_min": -46.02825927734375,
      "activations/layer9_attention_weight_max": 49.985252380371094,
      "activations/layer9_attention_weight_min": -47.48994064331055,
      "epoch": 6.53,
      "learning_rate": 0.00010897916666666666,
      "loss": 2.8229,
      "step": 112400
    },
    {
      "activations/layer0_attention_weight_max": 14.793435096740723,
      "activations/layer0_attention_weight_min": -14.282011985778809,
      "activations/layer10_attention_weight_max": 41.54728317260742,
      "activations/layer10_attention_weight_min": -37.17244338989258,
      "activations/layer11_attention_weight_max": 31.322532653808594,
      "activations/layer11_attention_weight_min": -30.515884399414062,
      "activations/layer12_attention_weight_max": 19.48655891418457,
      "activations/layer12_attention_weight_min": -25.455785751342773,
      "activations/layer13_attention_weight_max": 41.36721420288086,
      "activations/layer13_attention_weight_min": -32.87025451660156,
      "activations/layer14_attention_weight_max": 37.04623794555664,
      "activations/layer14_attention_weight_min": -30.029205322265625,
      "activations/layer15_attention_weight_max": 34.99596405029297,
      "activations/layer15_attention_weight_min": -28.891708374023438,
      "activations/layer16_attention_weight_max": 31.181102752685547,
      "activations/layer16_attention_weight_min": -26.622238159179688,
      "activations/layer17_attention_weight_max": 51.70357894897461,
      "activations/layer17_attention_weight_min": -39.46312713623047,
      "activations/layer18_attention_weight_max": 44.86082458496094,
      "activations/layer18_attention_weight_min": -35.46052932739258,
      "activations/layer19_attention_weight_max": 21.68648338317871,
      "activations/layer19_attention_weight_min": -22.809492111206055,
      "activations/layer1_attention_weight_max": 17.225317001342773,
      "activations/layer1_attention_weight_min": -14.479923248291016,
      "activations/layer20_attention_weight_max": 24.162641525268555,
      "activations/layer20_attention_weight_min": -25.310834884643555,
      "activations/layer21_attention_weight_max": 41.5430908203125,
      "activations/layer21_attention_weight_min": -25.848251342773438,
      "activations/layer22_attention_weight_max": 31.679874420166016,
      "activations/layer22_attention_weight_min": -23.863862991333008,
      "activations/layer23_attention_weight_max": 30.002676010131836,
      "activations/layer23_attention_weight_min": -19.653430938720703,
      "activations/layer2_attention_weight_max": 31.192211151123047,
      "activations/layer2_attention_weight_min": -28.026931762695312,
      "activations/layer3_attention_weight_max": 81.22848510742188,
      "activations/layer3_attention_weight_min": -85.36113739013672,
      "activations/layer4_attention_weight_max": 100.42210388183594,
      "activations/layer4_attention_weight_min": -95.8508529663086,
      "activations/layer5_attention_weight_max": 95.56407165527344,
      "activations/layer5_attention_weight_min": -89.96470642089844,
      "activations/layer6_attention_weight_max": 59.088768005371094,
      "activations/layer6_attention_weight_min": -56.58194351196289,
      "activations/layer7_attention_weight_max": 78.70635223388672,
      "activations/layer7_attention_weight_min": -73.72402954101562,
      "activations/layer8_attention_weight_max": 44.13393020629883,
      "activations/layer8_attention_weight_min": -44.13228225708008,
      "activations/layer9_attention_weight_max": 52.75208282470703,
      "activations/layer9_attention_weight_min": -48.829593658447266,
      "epoch": 6.53,
      "learning_rate": 0.00010896022727272726,
      "loss": 2.8234,
      "step": 112450
    },
    {
      "activations/layer0_attention_weight_max": 15.251063346862793,
      "activations/layer0_attention_weight_min": -14.19269847869873,
      "activations/layer10_attention_weight_max": 37.747013092041016,
      "activations/layer10_attention_weight_min": -37.26848220825195,
      "activations/layer11_attention_weight_max": 30.073135375976562,
      "activations/layer11_attention_weight_min": -29.63057518005371,
      "activations/layer12_attention_weight_max": 20.613391876220703,
      "activations/layer12_attention_weight_min": -26.70946502685547,
      "activations/layer13_attention_weight_max": 52.107357025146484,
      "activations/layer13_attention_weight_min": -34.330772399902344,
      "activations/layer14_attention_weight_max": 41.615989685058594,
      "activations/layer14_attention_weight_min": -29.93185806274414,
      "activations/layer15_attention_weight_max": 46.25668716430664,
      "activations/layer15_attention_weight_min": -31.169965744018555,
      "activations/layer16_attention_weight_max": 34.91347885131836,
      "activations/layer16_attention_weight_min": -29.323942184448242,
      "activations/layer17_attention_weight_max": 62.23147964477539,
      "activations/layer17_attention_weight_min": -42.487510681152344,
      "activations/layer18_attention_weight_max": 56.693660736083984,
      "activations/layer18_attention_weight_min": -35.62859344482422,
      "activations/layer19_attention_weight_max": 25.256834030151367,
      "activations/layer19_attention_weight_min": -21.376447677612305,
      "activations/layer1_attention_weight_max": 17.216299057006836,
      "activations/layer1_attention_weight_min": -14.033448219299316,
      "activations/layer20_attention_weight_max": 25.749265670776367,
      "activations/layer20_attention_weight_min": -21.60610008239746,
      "activations/layer21_attention_weight_max": 42.20674514770508,
      "activations/layer21_attention_weight_min": -27.399450302124023,
      "activations/layer22_attention_weight_max": 40.643882751464844,
      "activations/layer22_attention_weight_min": -26.744308471679688,
      "activations/layer23_attention_weight_max": 34.70415115356445,
      "activations/layer23_attention_weight_min": -20.702089309692383,
      "activations/layer2_attention_weight_max": 31.206012725830078,
      "activations/layer2_attention_weight_min": -29.732383728027344,
      "activations/layer3_attention_weight_max": 82.9342041015625,
      "activations/layer3_attention_weight_min": -86.50849914550781,
      "activations/layer4_attention_weight_max": 98.23656463623047,
      "activations/layer4_attention_weight_min": -94.44001007080078,
      "activations/layer5_attention_weight_max": 96.48744201660156,
      "activations/layer5_attention_weight_min": -92.84545135498047,
      "activations/layer6_attention_weight_max": 56.52233123779297,
      "activations/layer6_attention_weight_min": -56.430511474609375,
      "activations/layer7_attention_weight_max": 75.91241455078125,
      "activations/layer7_attention_weight_min": -74.6887435913086,
      "activations/layer8_attention_weight_max": 48.20771789550781,
      "activations/layer8_attention_weight_min": -44.67118835449219,
      "activations/layer9_attention_weight_max": 51.534088134765625,
      "activations/layer9_attention_weight_min": -49.290462493896484,
      "epoch": 6.54,
      "learning_rate": 0.00010894128787878786,
      "loss": 2.8297,
      "step": 112500
    },
    {
      "activations/layer0_attention_weight_max": 15.75472354888916,
      "activations/layer0_attention_weight_min": -15.053914070129395,
      "activations/layer10_attention_weight_max": 42.10895919799805,
      "activations/layer10_attention_weight_min": -38.0670166015625,
      "activations/layer11_attention_weight_max": 35.02268981933594,
      "activations/layer11_attention_weight_min": -31.147132873535156,
      "activations/layer12_attention_weight_max": 36.012550354003906,
      "activations/layer12_attention_weight_min": -29.367597579956055,
      "activations/layer13_attention_weight_max": 58.522605895996094,
      "activations/layer13_attention_weight_min": -34.79740524291992,
      "activations/layer14_attention_weight_max": 46.484039306640625,
      "activations/layer14_attention_weight_min": -32.494842529296875,
      "activations/layer15_attention_weight_max": 47.76405715942383,
      "activations/layer15_attention_weight_min": -30.659990310668945,
      "activations/layer16_attention_weight_max": 34.470577239990234,
      "activations/layer16_attention_weight_min": -28.09422492980957,
      "activations/layer17_attention_weight_max": 68.23562622070312,
      "activations/layer17_attention_weight_min": -46.05329513549805,
      "activations/layer18_attention_weight_max": 55.49364471435547,
      "activations/layer18_attention_weight_min": -39.378108978271484,
      "activations/layer19_attention_weight_max": 23.64533042907715,
      "activations/layer19_attention_weight_min": -22.072830200195312,
      "activations/layer1_attention_weight_max": 17.18828773498535,
      "activations/layer1_attention_weight_min": -15.645355224609375,
      "activations/layer20_attention_weight_max": 26.818941116333008,
      "activations/layer20_attention_weight_min": -21.856441497802734,
      "activations/layer21_attention_weight_max": 53.438533782958984,
      "activations/layer21_attention_weight_min": -24.477312088012695,
      "activations/layer22_attention_weight_max": 36.54834747314453,
      "activations/layer22_attention_weight_min": -25.957275390625,
      "activations/layer23_attention_weight_max": 35.94036102294922,
      "activations/layer23_attention_weight_min": -21.58832359313965,
      "activations/layer2_attention_weight_max": 33.514251708984375,
      "activations/layer2_attention_weight_min": -29.850996017456055,
      "activations/layer3_attention_weight_max": 92.75495910644531,
      "activations/layer3_attention_weight_min": -88.18147277832031,
      "activations/layer4_attention_weight_max": 114.1494140625,
      "activations/layer4_attention_weight_min": -97.18571472167969,
      "activations/layer5_attention_weight_max": 103.6571044921875,
      "activations/layer5_attention_weight_min": -89.45030212402344,
      "activations/layer6_attention_weight_max": 59.95335388183594,
      "activations/layer6_attention_weight_min": -55.161293029785156,
      "activations/layer7_attention_weight_max": 77.85896301269531,
      "activations/layer7_attention_weight_min": -74.82841491699219,
      "activations/layer8_attention_weight_max": 49.109291076660156,
      "activations/layer8_attention_weight_min": -46.92457962036133,
      "activations/layer9_attention_weight_max": 53.008907318115234,
      "activations/layer9_attention_weight_min": -50.82914352416992,
      "epoch": 6.54,
      "learning_rate": 0.00010892234848484848,
      "loss": 2.8434,
      "step": 112550
    },
    {
      "activations/layer0_attention_weight_max": 13.944803237915039,
      "activations/layer0_attention_weight_min": -14.401387214660645,
      "activations/layer10_attention_weight_max": 37.59636688232422,
      "activations/layer10_attention_weight_min": -34.67868423461914,
      "activations/layer11_attention_weight_max": 30.98548698425293,
      "activations/layer11_attention_weight_min": -29.279983520507812,
      "activations/layer12_attention_weight_max": 20.289730072021484,
      "activations/layer12_attention_weight_min": -26.43644142150879,
      "activations/layer13_attention_weight_max": 39.83717727661133,
      "activations/layer13_attention_weight_min": -31.928564071655273,
      "activations/layer14_attention_weight_max": 38.26969909667969,
      "activations/layer14_attention_weight_min": -31.247591018676758,
      "activations/layer15_attention_weight_max": 33.50897979736328,
      "activations/layer15_attention_weight_min": -30.60219383239746,
      "activations/layer16_attention_weight_max": 33.56200408935547,
      "activations/layer16_attention_weight_min": -25.888837814331055,
      "activations/layer17_attention_weight_max": 53.47597122192383,
      "activations/layer17_attention_weight_min": -40.12641525268555,
      "activations/layer18_attention_weight_max": 45.34188461303711,
      "activations/layer18_attention_weight_min": -35.609840393066406,
      "activations/layer19_attention_weight_max": 21.893217086791992,
      "activations/layer19_attention_weight_min": -20.545724868774414,
      "activations/layer1_attention_weight_max": 17.258398056030273,
      "activations/layer1_attention_weight_min": -16.1929931640625,
      "activations/layer20_attention_weight_max": 19.864004135131836,
      "activations/layer20_attention_weight_min": -21.054861068725586,
      "activations/layer21_attention_weight_max": 35.53055191040039,
      "activations/layer21_attention_weight_min": -25.184329986572266,
      "activations/layer22_attention_weight_max": 31.864105224609375,
      "activations/layer22_attention_weight_min": -26.67246437072754,
      "activations/layer23_attention_weight_max": 33.326114654541016,
      "activations/layer23_attention_weight_min": -20.833919525146484,
      "activations/layer2_attention_weight_max": 30.82217788696289,
      "activations/layer2_attention_weight_min": -28.6901912689209,
      "activations/layer3_attention_weight_max": 83.85127258300781,
      "activations/layer3_attention_weight_min": -91.45401000976562,
      "activations/layer4_attention_weight_max": 98.15937805175781,
      "activations/layer4_attention_weight_min": -96.5334701538086,
      "activations/layer5_attention_weight_max": 96.30197143554688,
      "activations/layer5_attention_weight_min": -88.05845642089844,
      "activations/layer6_attention_weight_max": 56.1415901184082,
      "activations/layer6_attention_weight_min": -51.869163513183594,
      "activations/layer7_attention_weight_max": 74.06632232666016,
      "activations/layer7_attention_weight_min": -73.45730590820312,
      "activations/layer8_attention_weight_max": 46.399559020996094,
      "activations/layer8_attention_weight_min": -46.063758850097656,
      "activations/layer9_attention_weight_max": 51.99713897705078,
      "activations/layer9_attention_weight_min": -47.62026596069336,
      "epoch": 6.54,
      "learning_rate": 0.0001089034090909091,
      "loss": 2.8247,
      "step": 112600
    },
    {
      "activations/layer0_attention_weight_max": 14.149938583374023,
      "activations/layer0_attention_weight_min": -14.447823524475098,
      "activations/layer10_attention_weight_max": 37.83722686767578,
      "activations/layer10_attention_weight_min": -36.47052001953125,
      "activations/layer11_attention_weight_max": 30.845829010009766,
      "activations/layer11_attention_weight_min": -31.500669479370117,
      "activations/layer12_attention_weight_max": 24.010984420776367,
      "activations/layer12_attention_weight_min": -23.037803649902344,
      "activations/layer13_attention_weight_max": 50.810096740722656,
      "activations/layer13_attention_weight_min": -36.10719299316406,
      "activations/layer14_attention_weight_max": 41.788475036621094,
      "activations/layer14_attention_weight_min": -31.169858932495117,
      "activations/layer15_attention_weight_max": 38.25261306762695,
      "activations/layer15_attention_weight_min": -28.925430297851562,
      "activations/layer16_attention_weight_max": 34.974159240722656,
      "activations/layer16_attention_weight_min": -27.800344467163086,
      "activations/layer17_attention_weight_max": 56.26665115356445,
      "activations/layer17_attention_weight_min": -46.42344284057617,
      "activations/layer18_attention_weight_max": 50.74629211425781,
      "activations/layer18_attention_weight_min": -39.17125701904297,
      "activations/layer19_attention_weight_max": 23.692546844482422,
      "activations/layer19_attention_weight_min": -21.671234130859375,
      "activations/layer1_attention_weight_max": 17.404329299926758,
      "activations/layer1_attention_weight_min": -13.465367317199707,
      "activations/layer20_attention_weight_max": 26.655357360839844,
      "activations/layer20_attention_weight_min": -24.91399574279785,
      "activations/layer21_attention_weight_max": 42.70315170288086,
      "activations/layer21_attention_weight_min": -26.257854461669922,
      "activations/layer22_attention_weight_max": 36.71573257446289,
      "activations/layer22_attention_weight_min": -27.54136085510254,
      "activations/layer23_attention_weight_max": 34.105224609375,
      "activations/layer23_attention_weight_min": -21.49177360534668,
      "activations/layer2_attention_weight_max": 29.790584564208984,
      "activations/layer2_attention_weight_min": -29.025794982910156,
      "activations/layer3_attention_weight_max": 80.75457000732422,
      "activations/layer3_attention_weight_min": -84.25423431396484,
      "activations/layer4_attention_weight_max": 99.29759216308594,
      "activations/layer4_attention_weight_min": -94.65616607666016,
      "activations/layer5_attention_weight_max": 93.72029876708984,
      "activations/layer5_attention_weight_min": -84.4190902709961,
      "activations/layer6_attention_weight_max": 57.11628723144531,
      "activations/layer6_attention_weight_min": -56.75238037109375,
      "activations/layer7_attention_weight_max": 72.06918334960938,
      "activations/layer7_attention_weight_min": -71.7563247680664,
      "activations/layer8_attention_weight_max": 45.42629623413086,
      "activations/layer8_attention_weight_min": -46.11088562011719,
      "activations/layer9_attention_weight_max": 49.59077072143555,
      "activations/layer9_attention_weight_min": -48.79695510864258,
      "epoch": 6.55,
      "learning_rate": 0.00010888446969696968,
      "loss": 2.8239,
      "step": 112650
    },
    {
      "activations/layer0_attention_weight_max": 15.184955596923828,
      "activations/layer0_attention_weight_min": -14.68148422241211,
      "activations/layer10_attention_weight_max": 43.08125686645508,
      "activations/layer10_attention_weight_min": -40.7216682434082,
      "activations/layer11_attention_weight_max": 33.71837615966797,
      "activations/layer11_attention_weight_min": -31.642919540405273,
      "activations/layer12_attention_weight_max": 24.164894104003906,
      "activations/layer12_attention_weight_min": -27.34022331237793,
      "activations/layer13_attention_weight_max": 50.07892608642578,
      "activations/layer13_attention_weight_min": -37.69029235839844,
      "activations/layer14_attention_weight_max": 44.30952453613281,
      "activations/layer14_attention_weight_min": -33.33465576171875,
      "activations/layer15_attention_weight_max": 39.688720703125,
      "activations/layer15_attention_weight_min": -27.66420555114746,
      "activations/layer16_attention_weight_max": 31.63268280029297,
      "activations/layer16_attention_weight_min": -25.83469581604004,
      "activations/layer17_attention_weight_max": 50.670780181884766,
      "activations/layer17_attention_weight_min": -40.28467559814453,
      "activations/layer18_attention_weight_max": 43.73788833618164,
      "activations/layer18_attention_weight_min": -34.386756896972656,
      "activations/layer19_attention_weight_max": 20.29718589782715,
      "activations/layer19_attention_weight_min": -21.964513778686523,
      "activations/layer1_attention_weight_max": 18.176494598388672,
      "activations/layer1_attention_weight_min": -14.167923927307129,
      "activations/layer20_attention_weight_max": 20.105911254882812,
      "activations/layer20_attention_weight_min": -21.61534309387207,
      "activations/layer21_attention_weight_max": 35.114845275878906,
      "activations/layer21_attention_weight_min": -24.855133056640625,
      "activations/layer22_attention_weight_max": 33.02885437011719,
      "activations/layer22_attention_weight_min": -24.654552459716797,
      "activations/layer23_attention_weight_max": 32.59329605102539,
      "activations/layer23_attention_weight_min": -20.519996643066406,
      "activations/layer2_attention_weight_max": 31.23278045654297,
      "activations/layer2_attention_weight_min": -29.137550354003906,
      "activations/layer3_attention_weight_max": 85.27581787109375,
      "activations/layer3_attention_weight_min": -87.74742889404297,
      "activations/layer4_attention_weight_max": 106.89070892333984,
      "activations/layer4_attention_weight_min": -97.64617156982422,
      "activations/layer5_attention_weight_max": 96.73400115966797,
      "activations/layer5_attention_weight_min": -90.82431030273438,
      "activations/layer6_attention_weight_max": 57.837257385253906,
      "activations/layer6_attention_weight_min": -54.81928634643555,
      "activations/layer7_attention_weight_max": 77.73363494873047,
      "activations/layer7_attention_weight_min": -73.71519470214844,
      "activations/layer8_attention_weight_max": 48.286521911621094,
      "activations/layer8_attention_weight_min": -47.700286865234375,
      "activations/layer9_attention_weight_max": 59.99300003051758,
      "activations/layer9_attention_weight_min": -56.352272033691406,
      "epoch": 6.55,
      "learning_rate": 0.0001088655303030303,
      "loss": 2.8395,
      "step": 112700
    },
    {
      "activations/layer0_attention_weight_max": 14.827353477478027,
      "activations/layer0_attention_weight_min": -15.048210144042969,
      "activations/layer10_attention_weight_max": 35.42999267578125,
      "activations/layer10_attention_weight_min": -33.80385971069336,
      "activations/layer11_attention_weight_max": 32.75785827636719,
      "activations/layer11_attention_weight_min": -27.90852928161621,
      "activations/layer12_attention_weight_max": 24.859601974487305,
      "activations/layer12_attention_weight_min": -26.107511520385742,
      "activations/layer13_attention_weight_max": 48.0330924987793,
      "activations/layer13_attention_weight_min": -34.881866455078125,
      "activations/layer14_attention_weight_max": 44.89691162109375,
      "activations/layer14_attention_weight_min": -31.287397384643555,
      "activations/layer15_attention_weight_max": 34.94038391113281,
      "activations/layer15_attention_weight_min": -28.54273796081543,
      "activations/layer16_attention_weight_max": 32.29851150512695,
      "activations/layer16_attention_weight_min": -27.186595916748047,
      "activations/layer17_attention_weight_max": 54.07823181152344,
      "activations/layer17_attention_weight_min": -41.0203971862793,
      "activations/layer18_attention_weight_max": 46.08723831176758,
      "activations/layer18_attention_weight_min": -36.1817626953125,
      "activations/layer19_attention_weight_max": 20.15995216369629,
      "activations/layer19_attention_weight_min": -20.14253044128418,
      "activations/layer1_attention_weight_max": 17.08279800415039,
      "activations/layer1_attention_weight_min": -15.032941818237305,
      "activations/layer20_attention_weight_max": 20.400554656982422,
      "activations/layer20_attention_weight_min": -22.115764617919922,
      "activations/layer21_attention_weight_max": 36.50279998779297,
      "activations/layer21_attention_weight_min": -23.088464736938477,
      "activations/layer22_attention_weight_max": 31.199689865112305,
      "activations/layer22_attention_weight_min": -25.93915367126465,
      "activations/layer23_attention_weight_max": 31.59607696533203,
      "activations/layer23_attention_weight_min": -19.745704650878906,
      "activations/layer2_attention_weight_max": 31.254718780517578,
      "activations/layer2_attention_weight_min": -29.381908416748047,
      "activations/layer3_attention_weight_max": 82.4571304321289,
      "activations/layer3_attention_weight_min": -84.33261108398438,
      "activations/layer4_attention_weight_max": 99.1942138671875,
      "activations/layer4_attention_weight_min": -96.20539093017578,
      "activations/layer5_attention_weight_max": 89.08489227294922,
      "activations/layer5_attention_weight_min": -85.97528076171875,
      "activations/layer6_attention_weight_max": 54.97038650512695,
      "activations/layer6_attention_weight_min": -52.16651916503906,
      "activations/layer7_attention_weight_max": 77.0560073852539,
      "activations/layer7_attention_weight_min": -71.22782135009766,
      "activations/layer8_attention_weight_max": 43.10857009887695,
      "activations/layer8_attention_weight_min": -42.554561614990234,
      "activations/layer9_attention_weight_max": 56.450374603271484,
      "activations/layer9_attention_weight_min": -50.311458587646484,
      "epoch": 6.55,
      "learning_rate": 0.00010884659090909089,
      "loss": 2.8415,
      "step": 112750
    },
    {
      "activations/layer0_attention_weight_max": 14.704614639282227,
      "activations/layer0_attention_weight_min": -14.688166618347168,
      "activations/layer10_attention_weight_max": 42.7776985168457,
      "activations/layer10_attention_weight_min": -39.838443756103516,
      "activations/layer11_attention_weight_max": 31.70861053466797,
      "activations/layer11_attention_weight_min": -29.68532371520996,
      "activations/layer12_attention_weight_max": 21.96222496032715,
      "activations/layer12_attention_weight_min": -24.792930603027344,
      "activations/layer13_attention_weight_max": 53.188602447509766,
      "activations/layer13_attention_weight_min": -38.12089538574219,
      "activations/layer14_attention_weight_max": 55.143043518066406,
      "activations/layer14_attention_weight_min": -34.18926239013672,
      "activations/layer15_attention_weight_max": 40.46393966674805,
      "activations/layer15_attention_weight_min": -30.971712112426758,
      "activations/layer16_attention_weight_max": 39.113311767578125,
      "activations/layer16_attention_weight_min": -28.271451950073242,
      "activations/layer17_attention_weight_max": 57.73795700073242,
      "activations/layer17_attention_weight_min": -45.00642013549805,
      "activations/layer18_attention_weight_max": 48.83999252319336,
      "activations/layer18_attention_weight_min": -38.82999801635742,
      "activations/layer19_attention_weight_max": 21.272672653198242,
      "activations/layer19_attention_weight_min": -21.52272605895996,
      "activations/layer1_attention_weight_max": 16.59445571899414,
      "activations/layer1_attention_weight_min": -13.736778259277344,
      "activations/layer20_attention_weight_max": 22.415433883666992,
      "activations/layer20_attention_weight_min": -20.18335723876953,
      "activations/layer21_attention_weight_max": 44.52262496948242,
      "activations/layer21_attention_weight_min": -25.515043258666992,
      "activations/layer22_attention_weight_max": 35.25645446777344,
      "activations/layer22_attention_weight_min": -24.35746192932129,
      "activations/layer23_attention_weight_max": 34.120479583740234,
      "activations/layer23_attention_weight_min": -20.214744567871094,
      "activations/layer2_attention_weight_max": 30.19906234741211,
      "activations/layer2_attention_weight_min": -29.575725555419922,
      "activations/layer3_attention_weight_max": 85.51222229003906,
      "activations/layer3_attention_weight_min": -86.62841033935547,
      "activations/layer4_attention_weight_max": 104.89855194091797,
      "activations/layer4_attention_weight_min": -97.71074676513672,
      "activations/layer5_attention_weight_max": 92.54499816894531,
      "activations/layer5_attention_weight_min": -90.35646057128906,
      "activations/layer6_attention_weight_max": 55.16534423828125,
      "activations/layer6_attention_weight_min": -55.03068161010742,
      "activations/layer7_attention_weight_max": 79.11481475830078,
      "activations/layer7_attention_weight_min": -76.41455078125,
      "activations/layer8_attention_weight_max": 46.61362838745117,
      "activations/layer8_attention_weight_min": -48.1441535949707,
      "activations/layer9_attention_weight_max": 67.14539337158203,
      "activations/layer9_attention_weight_min": -54.17934799194336,
      "epoch": 6.55,
      "learning_rate": 0.0001088280303030303,
      "loss": 2.8348,
      "step": 112800
    },
    {
      "activations/layer0_attention_weight_max": 14.358248710632324,
      "activations/layer0_attention_weight_min": -15.009342193603516,
      "activations/layer10_attention_weight_max": 41.78911209106445,
      "activations/layer10_attention_weight_min": -35.2122917175293,
      "activations/layer11_attention_weight_max": 32.37506103515625,
      "activations/layer11_attention_weight_min": -28.55377769470215,
      "activations/layer12_attention_weight_max": 23.76042938232422,
      "activations/layer12_attention_weight_min": -24.848608016967773,
      "activations/layer13_attention_weight_max": 55.813636779785156,
      "activations/layer13_attention_weight_min": -36.548377990722656,
      "activations/layer14_attention_weight_max": 69.03722381591797,
      "activations/layer14_attention_weight_min": -38.44269561767578,
      "activations/layer15_attention_weight_max": 42.12018966674805,
      "activations/layer15_attention_weight_min": -32.06403350830078,
      "activations/layer16_attention_weight_max": 39.355499267578125,
      "activations/layer16_attention_weight_min": -27.633886337280273,
      "activations/layer17_attention_weight_max": 68.63199615478516,
      "activations/layer17_attention_weight_min": -43.51568603515625,
      "activations/layer18_attention_weight_max": 51.588863372802734,
      "activations/layer18_attention_weight_min": -39.75876998901367,
      "activations/layer19_attention_weight_max": 28.114770889282227,
      "activations/layer19_attention_weight_min": -22.153356552124023,
      "activations/layer1_attention_weight_max": 17.26607894897461,
      "activations/layer1_attention_weight_min": -17.257482528686523,
      "activations/layer20_attention_weight_max": 24.001270294189453,
      "activations/layer20_attention_weight_min": -20.434232711791992,
      "activations/layer21_attention_weight_max": 45.50492858886719,
      "activations/layer21_attention_weight_min": -25.799924850463867,
      "activations/layer22_attention_weight_max": 41.811405181884766,
      "activations/layer22_attention_weight_min": -24.89240264892578,
      "activations/layer23_attention_weight_max": 33.5103874206543,
      "activations/layer23_attention_weight_min": -20.165796279907227,
      "activations/layer2_attention_weight_max": 32.613136291503906,
      "activations/layer2_attention_weight_min": -32.795936584472656,
      "activations/layer3_attention_weight_max": 84.8896713256836,
      "activations/layer3_attention_weight_min": -88.79078674316406,
      "activations/layer4_attention_weight_max": 101.72685241699219,
      "activations/layer4_attention_weight_min": -93.1689682006836,
      "activations/layer5_attention_weight_max": 91.57102966308594,
      "activations/layer5_attention_weight_min": -84.08672332763672,
      "activations/layer6_attention_weight_max": 57.27884292602539,
      "activations/layer6_attention_weight_min": -51.64733123779297,
      "activations/layer7_attention_weight_max": 77.7806167602539,
      "activations/layer7_attention_weight_min": -69.69300079345703,
      "activations/layer8_attention_weight_max": 44.84970474243164,
      "activations/layer8_attention_weight_min": -45.574623107910156,
      "activations/layer9_attention_weight_max": 53.37593460083008,
      "activations/layer9_attention_weight_min": -48.36080551147461,
      "epoch": 6.56,
      "learning_rate": 0.00010880909090909089,
      "loss": 2.828,
      "step": 112850
    },
    {
      "activations/layer0_attention_weight_max": 14.247116088867188,
      "activations/layer0_attention_weight_min": -14.8256196975708,
      "activations/layer10_attention_weight_max": 35.697818756103516,
      "activations/layer10_attention_weight_min": -32.95914840698242,
      "activations/layer11_attention_weight_max": 29.930744171142578,
      "activations/layer11_attention_weight_min": -27.975276947021484,
      "activations/layer12_attention_weight_max": 20.23479652404785,
      "activations/layer12_attention_weight_min": -23.6724910736084,
      "activations/layer13_attention_weight_max": 40.74726104736328,
      "activations/layer13_attention_weight_min": -31.847457885742188,
      "activations/layer14_attention_weight_max": 37.656455993652344,
      "activations/layer14_attention_weight_min": -33.03349304199219,
      "activations/layer15_attention_weight_max": 38.28704833984375,
      "activations/layer15_attention_weight_min": -30.617507934570312,
      "activations/layer16_attention_weight_max": 32.57204818725586,
      "activations/layer16_attention_weight_min": -27.896577835083008,
      "activations/layer17_attention_weight_max": 55.66278076171875,
      "activations/layer17_attention_weight_min": -42.594879150390625,
      "activations/layer18_attention_weight_max": 50.14901351928711,
      "activations/layer18_attention_weight_min": -35.292564392089844,
      "activations/layer19_attention_weight_max": 25.917001724243164,
      "activations/layer19_attention_weight_min": -19.757038116455078,
      "activations/layer1_attention_weight_max": 15.612059593200684,
      "activations/layer1_attention_weight_min": -14.70211124420166,
      "activations/layer20_attention_weight_max": 21.527433395385742,
      "activations/layer20_attention_weight_min": -22.584117889404297,
      "activations/layer21_attention_weight_max": 44.510108947753906,
      "activations/layer21_attention_weight_min": -24.516429901123047,
      "activations/layer22_attention_weight_max": 33.92649841308594,
      "activations/layer22_attention_weight_min": -24.211992263793945,
      "activations/layer23_attention_weight_max": 35.178714752197266,
      "activations/layer23_attention_weight_min": -20.735559463500977,
      "activations/layer2_attention_weight_max": 30.99420738220215,
      "activations/layer2_attention_weight_min": -29.20306396484375,
      "activations/layer3_attention_weight_max": 78.85717010498047,
      "activations/layer3_attention_weight_min": -85.56697082519531,
      "activations/layer4_attention_weight_max": 93.2887191772461,
      "activations/layer4_attention_weight_min": -90.78600311279297,
      "activations/layer5_attention_weight_max": 90.400146484375,
      "activations/layer5_attention_weight_min": -89.24124145507812,
      "activations/layer6_attention_weight_max": 54.118003845214844,
      "activations/layer6_attention_weight_min": -52.83046340942383,
      "activations/layer7_attention_weight_max": 71.50315856933594,
      "activations/layer7_attention_weight_min": -71.28640747070312,
      "activations/layer8_attention_weight_max": 41.29045867919922,
      "activations/layer8_attention_weight_min": -43.94589614868164,
      "activations/layer9_attention_weight_max": 47.78853988647461,
      "activations/layer9_attention_weight_min": -46.822261810302734,
      "epoch": 6.56,
      "learning_rate": 0.0001087901515151515,
      "loss": 2.8307,
      "step": 112900
    },
    {
      "activations/layer0_attention_weight_max": 14.768192291259766,
      "activations/layer0_attention_weight_min": -14.67279052734375,
      "activations/layer10_attention_weight_max": 40.3638916015625,
      "activations/layer10_attention_weight_min": -36.136695861816406,
      "activations/layer11_attention_weight_max": 33.786529541015625,
      "activations/layer11_attention_weight_min": -30.655073165893555,
      "activations/layer12_attention_weight_max": 21.544607162475586,
      "activations/layer12_attention_weight_min": -27.00876808166504,
      "activations/layer13_attention_weight_max": 48.58185958862305,
      "activations/layer13_attention_weight_min": -39.785621643066406,
      "activations/layer14_attention_weight_max": 55.641998291015625,
      "activations/layer14_attention_weight_min": -30.96086883544922,
      "activations/layer15_attention_weight_max": 47.012351989746094,
      "activations/layer15_attention_weight_min": -29.601411819458008,
      "activations/layer16_attention_weight_max": 39.157447814941406,
      "activations/layer16_attention_weight_min": -27.249448776245117,
      "activations/layer17_attention_weight_max": 60.97446060180664,
      "activations/layer17_attention_weight_min": -39.46086883544922,
      "activations/layer18_attention_weight_max": 54.07433319091797,
      "activations/layer18_attention_weight_min": -33.091312408447266,
      "activations/layer19_attention_weight_max": 24.54375457763672,
      "activations/layer19_attention_weight_min": -21.23923110961914,
      "activations/layer1_attention_weight_max": 17.692182540893555,
      "activations/layer1_attention_weight_min": -14.487061500549316,
      "activations/layer20_attention_weight_max": 28.14706039428711,
      "activations/layer20_attention_weight_min": -22.45124053955078,
      "activations/layer21_attention_weight_max": 58.41350555419922,
      "activations/layer21_attention_weight_min": -24.20995330810547,
      "activations/layer22_attention_weight_max": 38.142578125,
      "activations/layer22_attention_weight_min": -25.515764236450195,
      "activations/layer23_attention_weight_max": 41.019187927246094,
      "activations/layer23_attention_weight_min": -22.731109619140625,
      "activations/layer2_attention_weight_max": 31.80721092224121,
      "activations/layer2_attention_weight_min": -30.967029571533203,
      "activations/layer3_attention_weight_max": 87.42546081542969,
      "activations/layer3_attention_weight_min": -90.06497955322266,
      "activations/layer4_attention_weight_max": 105.43061828613281,
      "activations/layer4_attention_weight_min": -97.68248748779297,
      "activations/layer5_attention_weight_max": 97.36970520019531,
      "activations/layer5_attention_weight_min": -87.82344055175781,
      "activations/layer6_attention_weight_max": 58.703453063964844,
      "activations/layer6_attention_weight_min": -52.85201644897461,
      "activations/layer7_attention_weight_max": 77.02267456054688,
      "activations/layer7_attention_weight_min": -71.59060668945312,
      "activations/layer8_attention_weight_max": 44.76149368286133,
      "activations/layer8_attention_weight_min": -46.88629150390625,
      "activations/layer9_attention_weight_max": 50.21247482299805,
      "activations/layer9_attention_weight_min": -53.05820846557617,
      "epoch": 6.56,
      "learning_rate": 0.00010877121212121211,
      "loss": 2.8302,
      "step": 112950
    },
    {
      "activations/layer0_attention_weight_max": 14.55655574798584,
      "activations/layer0_attention_weight_min": -14.897175788879395,
      "activations/layer10_attention_weight_max": 39.299373626708984,
      "activations/layer10_attention_weight_min": -36.16930389404297,
      "activations/layer11_attention_weight_max": 36.455352783203125,
      "activations/layer11_attention_weight_min": -31.14552879333496,
      "activations/layer12_attention_weight_max": 20.806989669799805,
      "activations/layer12_attention_weight_min": -25.293968200683594,
      "activations/layer13_attention_weight_max": 58.20245361328125,
      "activations/layer13_attention_weight_min": -39.109493255615234,
      "activations/layer14_attention_weight_max": 41.38496398925781,
      "activations/layer14_attention_weight_min": -31.183853149414062,
      "activations/layer15_attention_weight_max": 35.87358856201172,
      "activations/layer15_attention_weight_min": -30.92547035217285,
      "activations/layer16_attention_weight_max": 33.1724853515625,
      "activations/layer16_attention_weight_min": -26.034439086914062,
      "activations/layer17_attention_weight_max": 56.96573257446289,
      "activations/layer17_attention_weight_min": -41.848052978515625,
      "activations/layer18_attention_weight_max": 49.469200134277344,
      "activations/layer18_attention_weight_min": -38.0197639465332,
      "activations/layer19_attention_weight_max": 20.410314559936523,
      "activations/layer19_attention_weight_min": -19.659915924072266,
      "activations/layer1_attention_weight_max": 17.9569149017334,
      "activations/layer1_attention_weight_min": -17.093387603759766,
      "activations/layer20_attention_weight_max": 20.113021850585938,
      "activations/layer20_attention_weight_min": -21.906431198120117,
      "activations/layer21_attention_weight_max": 38.739837646484375,
      "activations/layer21_attention_weight_min": -23.431102752685547,
      "activations/layer22_attention_weight_max": 33.23094177246094,
      "activations/layer22_attention_weight_min": -25.397642135620117,
      "activations/layer23_attention_weight_max": 32.91693878173828,
      "activations/layer23_attention_weight_min": -20.417253494262695,
      "activations/layer2_attention_weight_max": 30.25666618347168,
      "activations/layer2_attention_weight_min": -29.01863670349121,
      "activations/layer3_attention_weight_max": 82.91404724121094,
      "activations/layer3_attention_weight_min": -86.4729232788086,
      "activations/layer4_attention_weight_max": 100.18496704101562,
      "activations/layer4_attention_weight_min": -97.3130111694336,
      "activations/layer5_attention_weight_max": 94.8953857421875,
      "activations/layer5_attention_weight_min": -87.67589569091797,
      "activations/layer6_attention_weight_max": 54.8699951171875,
      "activations/layer6_attention_weight_min": -52.28678894042969,
      "activations/layer7_attention_weight_max": 87.70413208007812,
      "activations/layer7_attention_weight_min": -73.76530456542969,
      "activations/layer8_attention_weight_max": 50.36709213256836,
      "activations/layer8_attention_weight_min": -45.969085693359375,
      "activations/layer9_attention_weight_max": 72.89034271240234,
      "activations/layer9_attention_weight_min": -52.870643615722656,
      "epoch": 6.57,
      "learning_rate": 0.00010875227272727271,
      "loss": 2.8413,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_loss": 2.779296875,
      "eval_runtime": 8.5953,
      "eval_samples_per_second": 499.575,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_openwebtext_loss": 2.779296875,
      "eval_openwebtext_ppl": 16.107691244988956,
      "eval_openwebtext_runtime": 8.5953,
      "eval_openwebtext_samples_per_second": 499.575,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_wikitext_loss": 3.017578125,
      "eval_wikitext_ppl": 20.44172438427552,
      "eval_wikitext_runtime": 2.0394,
      "eval_wikitext_samples_per_second": 223.599,
      "step": 113000
    },
    {
      "epoch": 6.57,
      "eval_lambada_loss": 2.736328125,
      "eval_lambada_ppl": 15.43022309819552,
      "eval_lambada_runtime": 9.661,
      "eval_lambada_samples_per_second": 503.987,
      "step": 113000
    },
    {
      "activations/layer0_attention_weight_max": 15.54328441619873,
      "activations/layer0_attention_weight_min": -14.707127571105957,
      "activations/layer10_attention_weight_max": 48.07829666137695,
      "activations/layer10_attention_weight_min": -36.493385314941406,
      "activations/layer11_attention_weight_max": 40.215049743652344,
      "activations/layer11_attention_weight_min": -30.01691436767578,
      "activations/layer12_attention_weight_max": 20.61783790588379,
      "activations/layer12_attention_weight_min": -27.258798599243164,
      "activations/layer13_attention_weight_max": 44.910743713378906,
      "activations/layer13_attention_weight_min": -32.70040512084961,
      "activations/layer14_attention_weight_max": 50.87131881713867,
      "activations/layer14_attention_weight_min": -31.71192169189453,
      "activations/layer15_attention_weight_max": 40.54680252075195,
      "activations/layer15_attention_weight_min": -29.400785446166992,
      "activations/layer16_attention_weight_max": 37.15471267700195,
      "activations/layer16_attention_weight_min": -26.33958625793457,
      "activations/layer17_attention_weight_max": 54.941593170166016,
      "activations/layer17_attention_weight_min": -41.44643020629883,
      "activations/layer18_attention_weight_max": 49.64004135131836,
      "activations/layer18_attention_weight_min": -35.32463836669922,
      "activations/layer19_attention_weight_max": 27.795791625976562,
      "activations/layer19_attention_weight_min": -22.212482452392578,
      "activations/layer1_attention_weight_max": 16.595367431640625,
      "activations/layer1_attention_weight_min": -14.941612243652344,
      "activations/layer20_attention_weight_max": 25.16161346435547,
      "activations/layer20_attention_weight_min": -21.620494842529297,
      "activations/layer21_attention_weight_max": 49.18245315551758,
      "activations/layer21_attention_weight_min": -22.91693687438965,
      "activations/layer22_attention_weight_max": 36.63301086425781,
      "activations/layer22_attention_weight_min": -26.080015182495117,
      "activations/layer23_attention_weight_max": 35.991493225097656,
      "activations/layer23_attention_weight_min": -22.423133850097656,
      "activations/layer2_attention_weight_max": 34.74443054199219,
      "activations/layer2_attention_weight_min": -30.573238372802734,
      "activations/layer3_attention_weight_max": 91.01226806640625,
      "activations/layer3_attention_weight_min": -87.2352523803711,
      "activations/layer4_attention_weight_max": 112.3331527709961,
      "activations/layer4_attention_weight_min": -102.5615234375,
      "activations/layer5_attention_weight_max": 101.53140258789062,
      "activations/layer5_attention_weight_min": -87.12925720214844,
      "activations/layer6_attention_weight_max": 59.568729400634766,
      "activations/layer6_attention_weight_min": -52.11259460449219,
      "activations/layer7_attention_weight_max": 76.75489807128906,
      "activations/layer7_attention_weight_min": -74.32791900634766,
      "activations/layer8_attention_weight_max": 48.956634521484375,
      "activations/layer8_attention_weight_min": -44.367366790771484,
      "activations/layer9_attention_weight_max": 59.25216293334961,
      "activations/layer9_attention_weight_min": -48.46449279785156,
      "epoch": 6.57,
      "learning_rate": 0.00010873333333333332,
      "loss": 2.8225,
      "step": 113050
    },
    {
      "activations/layer0_attention_weight_max": 14.34673023223877,
      "activations/layer0_attention_weight_min": -14.016629219055176,
      "activations/layer10_attention_weight_max": 40.15083694458008,
      "activations/layer10_attention_weight_min": -38.322174072265625,
      "activations/layer11_attention_weight_max": 32.004207611083984,
      "activations/layer11_attention_weight_min": -31.5311336517334,
      "activations/layer12_attention_weight_max": 20.1564998626709,
      "activations/layer12_attention_weight_min": -25.03436279296875,
      "activations/layer13_attention_weight_max": 46.11668014526367,
      "activations/layer13_attention_weight_min": -34.87580871582031,
      "activations/layer14_attention_weight_max": 47.80039596557617,
      "activations/layer14_attention_weight_min": -32.59520721435547,
      "activations/layer15_attention_weight_max": 41.231834411621094,
      "activations/layer15_attention_weight_min": -29.438657760620117,
      "activations/layer16_attention_weight_max": 36.772403717041016,
      "activations/layer16_attention_weight_min": -27.736011505126953,
      "activations/layer17_attention_weight_max": 52.7714958190918,
      "activations/layer17_attention_weight_min": -41.92327117919922,
      "activations/layer18_attention_weight_max": 46.41773986816406,
      "activations/layer18_attention_weight_min": -36.02048110961914,
      "activations/layer19_attention_weight_max": 22.471830368041992,
      "activations/layer19_attention_weight_min": -19.889591217041016,
      "activations/layer1_attention_weight_max": 16.40241241455078,
      "activations/layer1_attention_weight_min": -14.034130096435547,
      "activations/layer20_attention_weight_max": 21.068729400634766,
      "activations/layer20_attention_weight_min": -19.944047927856445,
      "activations/layer21_attention_weight_max": 43.31294631958008,
      "activations/layer21_attention_weight_min": -24.214336395263672,
      "activations/layer22_attention_weight_max": 29.98908805847168,
      "activations/layer22_attention_weight_min": -24.117408752441406,
      "activations/layer23_attention_weight_max": 34.51894760131836,
      "activations/layer23_attention_weight_min": -18.646709442138672,
      "activations/layer2_attention_weight_max": 31.694316864013672,
      "activations/layer2_attention_weight_min": -28.303462982177734,
      "activations/layer3_attention_weight_max": 74.71278381347656,
      "activations/layer3_attention_weight_min": -65.89893341064453,
      "activations/layer4_attention_weight_max": 91.77770233154297,
      "activations/layer4_attention_weight_min": -81.62300872802734,
      "activations/layer5_attention_weight_max": 89.672119140625,
      "activations/layer5_attention_weight_min": -80.82417297363281,
      "activations/layer6_attention_weight_max": 53.02540969848633,
      "activations/layer6_attention_weight_min": -49.79499435424805,
      "activations/layer7_attention_weight_max": 76.60081481933594,
      "activations/layer7_attention_weight_min": -78.63148498535156,
      "activations/layer8_attention_weight_max": 46.17311477661133,
      "activations/layer8_attention_weight_min": -45.50285339355469,
      "activations/layer9_attention_weight_max": 56.48374557495117,
      "activations/layer9_attention_weight_min": -52.62103271484375,
      "epoch": 6.57,
      "learning_rate": 0.00010871439393939393,
      "loss": 2.8344,
      "step": 113100
    },
    {
      "activations/layer0_attention_weight_max": 14.37762451171875,
      "activations/layer0_attention_weight_min": -14.34953498840332,
      "activations/layer10_attention_weight_max": 38.34099578857422,
      "activations/layer10_attention_weight_min": -33.43964385986328,
      "activations/layer11_attention_weight_max": 29.48016357421875,
      "activations/layer11_attention_weight_min": -28.10675048828125,
      "activations/layer12_attention_weight_max": 19.967920303344727,
      "activations/layer12_attention_weight_min": -25.393653869628906,
      "activations/layer13_attention_weight_max": 38.02638626098633,
      "activations/layer13_attention_weight_min": -31.062610626220703,
      "activations/layer14_attention_weight_max": 48.781185150146484,
      "activations/layer14_attention_weight_min": -33.29403305053711,
      "activations/layer15_attention_weight_max": 34.08041763305664,
      "activations/layer15_attention_weight_min": -29.87042999267578,
      "activations/layer16_attention_weight_max": 35.36053466796875,
      "activations/layer16_attention_weight_min": -27.19814682006836,
      "activations/layer17_attention_weight_max": 52.27028274536133,
      "activations/layer17_attention_weight_min": -40.02556228637695,
      "activations/layer18_attention_weight_max": 48.44990158081055,
      "activations/layer18_attention_weight_min": -35.767242431640625,
      "activations/layer19_attention_weight_max": 20.937070846557617,
      "activations/layer19_attention_weight_min": -22.13083839416504,
      "activations/layer1_attention_weight_max": 17.194316864013672,
      "activations/layer1_attention_weight_min": -16.01319694519043,
      "activations/layer20_attention_weight_max": 20.832796096801758,
      "activations/layer20_attention_weight_min": -20.922182083129883,
      "activations/layer21_attention_weight_max": 37.06013488769531,
      "activations/layer21_attention_weight_min": -25.22627830505371,
      "activations/layer22_attention_weight_max": 32.17110061645508,
      "activations/layer22_attention_weight_min": -27.402097702026367,
      "activations/layer23_attention_weight_max": 30.594938278198242,
      "activations/layer23_attention_weight_min": -21.007837295532227,
      "activations/layer2_attention_weight_max": 31.06415367126465,
      "activations/layer2_attention_weight_min": -29.96019744873047,
      "activations/layer3_attention_weight_max": 79.58848571777344,
      "activations/layer3_attention_weight_min": -78.26639556884766,
      "activations/layer4_attention_weight_max": 91.70403289794922,
      "activations/layer4_attention_weight_min": -83.70044708251953,
      "activations/layer5_attention_weight_max": 92.30809020996094,
      "activations/layer5_attention_weight_min": -79.92918395996094,
      "activations/layer6_attention_weight_max": 54.74589920043945,
      "activations/layer6_attention_weight_min": -49.16773223876953,
      "activations/layer7_attention_weight_max": 69.22844696044922,
      "activations/layer7_attention_weight_min": -65.09898376464844,
      "activations/layer8_attention_weight_max": 41.21905517578125,
      "activations/layer8_attention_weight_min": -43.35617446899414,
      "activations/layer9_attention_weight_max": 47.3678092956543,
      "activations/layer9_attention_weight_min": -47.51042175292969,
      "epoch": 6.57,
      "learning_rate": 0.00010869545454545454,
      "loss": 2.8293,
      "step": 113150
    },
    {
      "activations/layer0_attention_weight_max": 13.737466812133789,
      "activations/layer0_attention_weight_min": -14.124807357788086,
      "activations/layer10_attention_weight_max": 58.86095428466797,
      "activations/layer10_attention_weight_min": -51.858577728271484,
      "activations/layer11_attention_weight_max": 48.71746063232422,
      "activations/layer11_attention_weight_min": -41.739471435546875,
      "activations/layer12_attention_weight_max": 25.583757400512695,
      "activations/layer12_attention_weight_min": -32.549468994140625,
      "activations/layer13_attention_weight_max": 52.72536849975586,
      "activations/layer13_attention_weight_min": -36.14665985107422,
      "activations/layer14_attention_weight_max": 63.059505462646484,
      "activations/layer14_attention_weight_min": -37.60926818847656,
      "activations/layer15_attention_weight_max": 46.6160774230957,
      "activations/layer15_attention_weight_min": -29.80661392211914,
      "activations/layer16_attention_weight_max": 32.270179748535156,
      "activations/layer16_attention_weight_min": -28.21036720275879,
      "activations/layer17_attention_weight_max": 58.00187301635742,
      "activations/layer17_attention_weight_min": -44.805973052978516,
      "activations/layer18_attention_weight_max": 50.06132507324219,
      "activations/layer18_attention_weight_min": -40.05850601196289,
      "activations/layer19_attention_weight_max": 22.593626022338867,
      "activations/layer19_attention_weight_min": -21.077131271362305,
      "activations/layer1_attention_weight_max": 16.749555587768555,
      "activations/layer1_attention_weight_min": -15.300971031188965,
      "activations/layer20_attention_weight_max": 23.6009578704834,
      "activations/layer20_attention_weight_min": -21.766510009765625,
      "activations/layer21_attention_weight_max": 43.40890121459961,
      "activations/layer21_attention_weight_min": -23.60733413696289,
      "activations/layer22_attention_weight_max": 34.27485656738281,
      "activations/layer22_attention_weight_min": -24.018280029296875,
      "activations/layer23_attention_weight_max": 32.30582809448242,
      "activations/layer23_attention_weight_min": -20.13966941833496,
      "activations/layer2_attention_weight_max": 31.90821647644043,
      "activations/layer2_attention_weight_min": -32.67512512207031,
      "activations/layer3_attention_weight_max": 87.44808959960938,
      "activations/layer3_attention_weight_min": -86.1600112915039,
      "activations/layer4_attention_weight_max": 107.7829818725586,
      "activations/layer4_attention_weight_min": -98.34123992919922,
      "activations/layer5_attention_weight_max": 105.0847396850586,
      "activations/layer5_attention_weight_min": -93.10191345214844,
      "activations/layer6_attention_weight_max": 62.49363327026367,
      "activations/layer6_attention_weight_min": -53.39315414428711,
      "activations/layer7_attention_weight_max": 95.64374542236328,
      "activations/layer7_attention_weight_min": -75.55096435546875,
      "activations/layer8_attention_weight_max": 63.11604690551758,
      "activations/layer8_attention_weight_min": -52.79133987426758,
      "activations/layer9_attention_weight_max": 84.02721405029297,
      "activations/layer9_attention_weight_min": -62.6837043762207,
      "epoch": 6.58,
      "learning_rate": 0.00010867651515151514,
      "loss": 2.8253,
      "step": 113200
    },
    {
      "activations/layer0_attention_weight_max": 15.023837089538574,
      "activations/layer0_attention_weight_min": -14.910725593566895,
      "activations/layer10_attention_weight_max": 36.91228103637695,
      "activations/layer10_attention_weight_min": -35.521156311035156,
      "activations/layer11_attention_weight_max": 30.839155197143555,
      "activations/layer11_attention_weight_min": -30.517898559570312,
      "activations/layer12_attention_weight_max": 19.974994659423828,
      "activations/layer12_attention_weight_min": -23.11142349243164,
      "activations/layer13_attention_weight_max": 42.64426803588867,
      "activations/layer13_attention_weight_min": -32.928321838378906,
      "activations/layer14_attention_weight_max": 38.17451858520508,
      "activations/layer14_attention_weight_min": -30.33732032775879,
      "activations/layer15_attention_weight_max": 38.46604919433594,
      "activations/layer15_attention_weight_min": -28.32811737060547,
      "activations/layer16_attention_weight_max": 34.52180099487305,
      "activations/layer16_attention_weight_min": -28.06293296813965,
      "activations/layer17_attention_weight_max": 51.423179626464844,
      "activations/layer17_attention_weight_min": -37.59786605834961,
      "activations/layer18_attention_weight_max": 45.23839569091797,
      "activations/layer18_attention_weight_min": -35.42044448852539,
      "activations/layer19_attention_weight_max": 22.93641471862793,
      "activations/layer19_attention_weight_min": -19.684545516967773,
      "activations/layer1_attention_weight_max": 16.915283203125,
      "activations/layer1_attention_weight_min": -14.386922836303711,
      "activations/layer20_attention_weight_max": 22.04494285583496,
      "activations/layer20_attention_weight_min": -19.594440460205078,
      "activations/layer21_attention_weight_max": 35.792789459228516,
      "activations/layer21_attention_weight_min": -23.754606246948242,
      "activations/layer22_attention_weight_max": 32.72120666503906,
      "activations/layer22_attention_weight_min": -23.813880920410156,
      "activations/layer23_attention_weight_max": 30.932159423828125,
      "activations/layer23_attention_weight_min": -21.235393524169922,
      "activations/layer2_attention_weight_max": 31.781551361083984,
      "activations/layer2_attention_weight_min": -29.671802520751953,
      "activations/layer3_attention_weight_max": 84.44898223876953,
      "activations/layer3_attention_weight_min": -81.65850830078125,
      "activations/layer4_attention_weight_max": 100.20654296875,
      "activations/layer4_attention_weight_min": -91.2031021118164,
      "activations/layer5_attention_weight_max": 97.33220672607422,
      "activations/layer5_attention_weight_min": -86.8906478881836,
      "activations/layer6_attention_weight_max": 60.2333984375,
      "activations/layer6_attention_weight_min": -53.33677291870117,
      "activations/layer7_attention_weight_max": 75.93108367919922,
      "activations/layer7_attention_weight_min": -74.73121643066406,
      "activations/layer8_attention_weight_max": 46.55109786987305,
      "activations/layer8_attention_weight_min": -45.770816802978516,
      "activations/layer9_attention_weight_max": 53.79730987548828,
      "activations/layer9_attention_weight_min": -47.821224212646484,
      "epoch": 6.58,
      "learning_rate": 0.00010865757575757574,
      "loss": 2.8396,
      "step": 113250
    },
    {
      "activations/layer0_attention_weight_max": 14.54930305480957,
      "activations/layer0_attention_weight_min": -14.682990074157715,
      "activations/layer10_attention_weight_max": 42.58420181274414,
      "activations/layer10_attention_weight_min": -39.4383430480957,
      "activations/layer11_attention_weight_max": 35.008628845214844,
      "activations/layer11_attention_weight_min": -29.174482345581055,
      "activations/layer12_attention_weight_max": 19.770097732543945,
      "activations/layer12_attention_weight_min": -24.661264419555664,
      "activations/layer13_attention_weight_max": 46.547645568847656,
      "activations/layer13_attention_weight_min": -32.254005432128906,
      "activations/layer14_attention_weight_max": 51.3343620300293,
      "activations/layer14_attention_weight_min": -33.11725997924805,
      "activations/layer15_attention_weight_max": 40.67493438720703,
      "activations/layer15_attention_weight_min": -31.109806060791016,
      "activations/layer16_attention_weight_max": 33.60926055908203,
      "activations/layer16_attention_weight_min": -29.431489944458008,
      "activations/layer17_attention_weight_max": 54.16255569458008,
      "activations/layer17_attention_weight_min": -45.56502151489258,
      "activations/layer18_attention_weight_max": 45.70676803588867,
      "activations/layer18_attention_weight_min": -41.03047180175781,
      "activations/layer19_attention_weight_max": 21.359073638916016,
      "activations/layer19_attention_weight_min": -20.41878890991211,
      "activations/layer1_attention_weight_max": 17.288829803466797,
      "activations/layer1_attention_weight_min": -15.93515396118164,
      "activations/layer20_attention_weight_max": 23.296302795410156,
      "activations/layer20_attention_weight_min": -23.26209831237793,
      "activations/layer21_attention_weight_max": 36.5341911315918,
      "activations/layer21_attention_weight_min": -29.30449104309082,
      "activations/layer22_attention_weight_max": 37.30286407470703,
      "activations/layer22_attention_weight_min": -25.26983070373535,
      "activations/layer23_attention_weight_max": 32.91180419921875,
      "activations/layer23_attention_weight_min": -20.893213272094727,
      "activations/layer2_attention_weight_max": 31.63567352294922,
      "activations/layer2_attention_weight_min": -30.456363677978516,
      "activations/layer3_attention_weight_max": 82.36737823486328,
      "activations/layer3_attention_weight_min": -81.70884704589844,
      "activations/layer4_attention_weight_max": 95.91635131835938,
      "activations/layer4_attention_weight_min": -92.24514770507812,
      "activations/layer5_attention_weight_max": 93.02654266357422,
      "activations/layer5_attention_weight_min": -91.82595825195312,
      "activations/layer6_attention_weight_max": 55.026329040527344,
      "activations/layer6_attention_weight_min": -53.80532455444336,
      "activations/layer7_attention_weight_max": 79.42906188964844,
      "activations/layer7_attention_weight_min": -70.85533905029297,
      "activations/layer8_attention_weight_max": 51.84163284301758,
      "activations/layer8_attention_weight_min": -46.27186965942383,
      "activations/layer9_attention_weight_max": 69.888671875,
      "activations/layer9_attention_weight_min": -51.423187255859375,
      "epoch": 6.58,
      "learning_rate": 0.00010863863636363636,
      "loss": 2.833,
      "step": 113300
    },
    {
      "activations/layer0_attention_weight_max": 14.515491485595703,
      "activations/layer0_attention_weight_min": -14.964733123779297,
      "activations/layer10_attention_weight_max": 37.7006721496582,
      "activations/layer10_attention_weight_min": -34.25019073486328,
      "activations/layer11_attention_weight_max": 31.492507934570312,
      "activations/layer11_attention_weight_min": -28.72347068786621,
      "activations/layer12_attention_weight_max": 22.105554580688477,
      "activations/layer12_attention_weight_min": -24.693923950195312,
      "activations/layer13_attention_weight_max": 49.66535568237305,
      "activations/layer13_attention_weight_min": -35.81728744506836,
      "activations/layer14_attention_weight_max": 39.415008544921875,
      "activations/layer14_attention_weight_min": -30.99679183959961,
      "activations/layer15_attention_weight_max": 36.38105773925781,
      "activations/layer15_attention_weight_min": -28.78689193725586,
      "activations/layer16_attention_weight_max": 32.22422790527344,
      "activations/layer16_attention_weight_min": -26.194955825805664,
      "activations/layer17_attention_weight_max": 53.2761344909668,
      "activations/layer17_attention_weight_min": -40.5816764831543,
      "activations/layer18_attention_weight_max": 44.40248107910156,
      "activations/layer18_attention_weight_min": -35.822696685791016,
      "activations/layer19_attention_weight_max": 21.05644989013672,
      "activations/layer19_attention_weight_min": -21.447769165039062,
      "activations/layer1_attention_weight_max": 17.508953094482422,
      "activations/layer1_attention_weight_min": -14.893601417541504,
      "activations/layer20_attention_weight_max": 22.579875946044922,
      "activations/layer20_attention_weight_min": -27.246370315551758,
      "activations/layer21_attention_weight_max": 43.50920104980469,
      "activations/layer21_attention_weight_min": -26.809837341308594,
      "activations/layer22_attention_weight_max": 33.219417572021484,
      "activations/layer22_attention_weight_min": -25.986257553100586,
      "activations/layer23_attention_weight_max": 34.18155288696289,
      "activations/layer23_attention_weight_min": -22.141408920288086,
      "activations/layer2_attention_weight_max": 30.45937728881836,
      "activations/layer2_attention_weight_min": -28.656850814819336,
      "activations/layer3_attention_weight_max": 81.41807556152344,
      "activations/layer3_attention_weight_min": -84.45561218261719,
      "activations/layer4_attention_weight_max": 95.65132904052734,
      "activations/layer4_attention_weight_min": -87.81311798095703,
      "activations/layer5_attention_weight_max": 89.37664031982422,
      "activations/layer5_attention_weight_min": -84.0050048828125,
      "activations/layer6_attention_weight_max": 54.89593505859375,
      "activations/layer6_attention_weight_min": -52.62892532348633,
      "activations/layer7_attention_weight_max": 71.02655792236328,
      "activations/layer7_attention_weight_min": -71.3947525024414,
      "activations/layer8_attention_weight_max": 44.007389068603516,
      "activations/layer8_attention_weight_min": -45.18430709838867,
      "activations/layer9_attention_weight_max": 52.1491813659668,
      "activations/layer9_attention_weight_min": -49.45634841918945,
      "epoch": 6.59,
      "learning_rate": 0.00010861969696969697,
      "loss": 2.8228,
      "step": 113350
    },
    {
      "activations/layer0_attention_weight_max": 14.403928756713867,
      "activations/layer0_attention_weight_min": -14.590287208557129,
      "activations/layer10_attention_weight_max": 35.53476333618164,
      "activations/layer10_attention_weight_min": -34.596736907958984,
      "activations/layer11_attention_weight_max": 29.794639587402344,
      "activations/layer11_attention_weight_min": -30.194740295410156,
      "activations/layer12_attention_weight_max": 23.679920196533203,
      "activations/layer12_attention_weight_min": -26.383731842041016,
      "activations/layer13_attention_weight_max": 46.42700958251953,
      "activations/layer13_attention_weight_min": -34.777732849121094,
      "activations/layer14_attention_weight_max": 38.236202239990234,
      "activations/layer14_attention_weight_min": -30.0841121673584,
      "activations/layer15_attention_weight_max": 41.94413375854492,
      "activations/layer15_attention_weight_min": -30.337249755859375,
      "activations/layer16_attention_weight_max": 35.41910934448242,
      "activations/layer16_attention_weight_min": -26.938373565673828,
      "activations/layer17_attention_weight_max": 56.86990737915039,
      "activations/layer17_attention_weight_min": -41.54453659057617,
      "activations/layer18_attention_weight_max": 51.751346588134766,
      "activations/layer18_attention_weight_min": -38.065757751464844,
      "activations/layer19_attention_weight_max": 21.93367576599121,
      "activations/layer19_attention_weight_min": -21.83171844482422,
      "activations/layer1_attention_weight_max": 16.15402603149414,
      "activations/layer1_attention_weight_min": -13.777578353881836,
      "activations/layer20_attention_weight_max": 21.199689865112305,
      "activations/layer20_attention_weight_min": -20.540224075317383,
      "activations/layer21_attention_weight_max": 42.185176849365234,
      "activations/layer21_attention_weight_min": -25.529287338256836,
      "activations/layer22_attention_weight_max": 33.923580169677734,
      "activations/layer22_attention_weight_min": -24.848073959350586,
      "activations/layer23_attention_weight_max": 31.212797164916992,
      "activations/layer23_attention_weight_min": -19.607166290283203,
      "activations/layer2_attention_weight_max": 31.3772029876709,
      "activations/layer2_attention_weight_min": -30.36933135986328,
      "activations/layer3_attention_weight_max": 79.27234649658203,
      "activations/layer3_attention_weight_min": -84.53429412841797,
      "activations/layer4_attention_weight_max": 95.0455322265625,
      "activations/layer4_attention_weight_min": -88.07048034667969,
      "activations/layer5_attention_weight_max": 92.54508972167969,
      "activations/layer5_attention_weight_min": -87.52845764160156,
      "activations/layer6_attention_weight_max": 57.3936767578125,
      "activations/layer6_attention_weight_min": -51.007041931152344,
      "activations/layer7_attention_weight_max": 74.84967803955078,
      "activations/layer7_attention_weight_min": -65.90043640136719,
      "activations/layer8_attention_weight_max": 43.881736755371094,
      "activations/layer8_attention_weight_min": -40.219764709472656,
      "activations/layer9_attention_weight_max": 49.82175827026367,
      "activations/layer9_attention_weight_min": -45.68523025512695,
      "epoch": 6.59,
      "learning_rate": 0.00010860075757575756,
      "loss": 2.828,
      "step": 113400
    },
    {
      "activations/layer0_attention_weight_max": 14.54588508605957,
      "activations/layer0_attention_weight_min": -14.304661750793457,
      "activations/layer10_attention_weight_max": 35.86355972290039,
      "activations/layer10_attention_weight_min": -34.252220153808594,
      "activations/layer11_attention_weight_max": 30.728710174560547,
      "activations/layer11_attention_weight_min": -28.643774032592773,
      "activations/layer12_attention_weight_max": 24.814207077026367,
      "activations/layer12_attention_weight_min": -24.202327728271484,
      "activations/layer13_attention_weight_max": 52.306095123291016,
      "activations/layer13_attention_weight_min": -39.20876693725586,
      "activations/layer14_attention_weight_max": 46.43331527709961,
      "activations/layer14_attention_weight_min": -31.8308048248291,
      "activations/layer15_attention_weight_max": 39.61566925048828,
      "activations/layer15_attention_weight_min": -29.10624122619629,
      "activations/layer16_attention_weight_max": 33.37696075439453,
      "activations/layer16_attention_weight_min": -28.465482711791992,
      "activations/layer17_attention_weight_max": 55.1042594909668,
      "activations/layer17_attention_weight_min": -42.952030181884766,
      "activations/layer18_attention_weight_max": 49.990997314453125,
      "activations/layer18_attention_weight_min": -35.40899658203125,
      "activations/layer19_attention_weight_max": 26.43523597717285,
      "activations/layer19_attention_weight_min": -20.266286849975586,
      "activations/layer1_attention_weight_max": 17.60456657409668,
      "activations/layer1_attention_weight_min": -15.379743576049805,
      "activations/layer20_attention_weight_max": 25.426790237426758,
      "activations/layer20_attention_weight_min": -19.09709358215332,
      "activations/layer21_attention_weight_max": 43.32290267944336,
      "activations/layer21_attention_weight_min": -25.488637924194336,
      "activations/layer22_attention_weight_max": 35.643306732177734,
      "activations/layer22_attention_weight_min": -24.666908264160156,
      "activations/layer23_attention_weight_max": 31.976360321044922,
      "activations/layer23_attention_weight_min": -19.95777130126953,
      "activations/layer2_attention_weight_max": 31.415447235107422,
      "activations/layer2_attention_weight_min": -29.90033531188965,
      "activations/layer3_attention_weight_max": 83.95349884033203,
      "activations/layer3_attention_weight_min": -88.46888732910156,
      "activations/layer4_attention_weight_max": 100.79354858398438,
      "activations/layer4_attention_weight_min": -99.06343841552734,
      "activations/layer5_attention_weight_max": 93.20916748046875,
      "activations/layer5_attention_weight_min": -82.32835388183594,
      "activations/layer6_attention_weight_max": 58.21268844604492,
      "activations/layer6_attention_weight_min": -53.022422790527344,
      "activations/layer7_attention_weight_max": 81.10673522949219,
      "activations/layer7_attention_weight_min": -74.7911148071289,
      "activations/layer8_attention_weight_max": 44.630706787109375,
      "activations/layer8_attention_weight_min": -46.31206130981445,
      "activations/layer9_attention_weight_max": 55.98775100708008,
      "activations/layer9_attention_weight_min": -47.97793960571289,
      "epoch": 6.59,
      "learning_rate": 0.00010858181818181818,
      "loss": 2.8137,
      "step": 113450
    },
    {
      "activations/layer0_attention_weight_max": 14.212742805480957,
      "activations/layer0_attention_weight_min": -14.463153839111328,
      "activations/layer10_attention_weight_max": 40.4079475402832,
      "activations/layer10_attention_weight_min": -37.40711212158203,
      "activations/layer11_attention_weight_max": 31.224767684936523,
      "activations/layer11_attention_weight_min": -29.318191528320312,
      "activations/layer12_attention_weight_max": 26.71095085144043,
      "activations/layer12_attention_weight_min": -23.37319564819336,
      "activations/layer13_attention_weight_max": 42.552947998046875,
      "activations/layer13_attention_weight_min": -34.99088668823242,
      "activations/layer14_attention_weight_max": 36.648345947265625,
      "activations/layer14_attention_weight_min": -31.511981964111328,
      "activations/layer15_attention_weight_max": 36.96192169189453,
      "activations/layer15_attention_weight_min": -31.58400535583496,
      "activations/layer16_attention_weight_max": 33.332733154296875,
      "activations/layer16_attention_weight_min": -26.890592575073242,
      "activations/layer17_attention_weight_max": 56.88261795043945,
      "activations/layer17_attention_weight_min": -42.26657485961914,
      "activations/layer18_attention_weight_max": 45.23727798461914,
      "activations/layer18_attention_weight_min": -37.40599822998047,
      "activations/layer19_attention_weight_max": 19.530092239379883,
      "activations/layer19_attention_weight_min": -20.670114517211914,
      "activations/layer1_attention_weight_max": 16.704116821289062,
      "activations/layer1_attention_weight_min": -15.237290382385254,
      "activations/layer20_attention_weight_max": 19.99372673034668,
      "activations/layer20_attention_weight_min": -20.191051483154297,
      "activations/layer21_attention_weight_max": 42.2673225402832,
      "activations/layer21_attention_weight_min": -26.056411743164062,
      "activations/layer22_attention_weight_max": 35.05759048461914,
      "activations/layer22_attention_weight_min": -26.03932762145996,
      "activations/layer23_attention_weight_max": 32.925621032714844,
      "activations/layer23_attention_weight_min": -19.802982330322266,
      "activations/layer2_attention_weight_max": 32.28772735595703,
      "activations/layer2_attention_weight_min": -29.89700698852539,
      "activations/layer3_attention_weight_max": 84.8442611694336,
      "activations/layer3_attention_weight_min": -89.86885070800781,
      "activations/layer4_attention_weight_max": 95.75799560546875,
      "activations/layer4_attention_weight_min": -91.78865051269531,
      "activations/layer5_attention_weight_max": 95.14986419677734,
      "activations/layer5_attention_weight_min": -87.32395935058594,
      "activations/layer6_attention_weight_max": 54.42447280883789,
      "activations/layer6_attention_weight_min": -52.92278289794922,
      "activations/layer7_attention_weight_max": 68.6268081665039,
      "activations/layer7_attention_weight_min": -69.2806396484375,
      "activations/layer8_attention_weight_max": 42.398712158203125,
      "activations/layer8_attention_weight_min": -43.83543395996094,
      "activations/layer9_attention_weight_max": 52.30268478393555,
      "activations/layer9_attention_weight_min": -50.41710662841797,
      "epoch": 6.59,
      "learning_rate": 0.00010856287878787877,
      "loss": 2.8281,
      "step": 113500
    },
    {
      "activations/layer0_attention_weight_max": 14.930181503295898,
      "activations/layer0_attention_weight_min": -14.199605941772461,
      "activations/layer10_attention_weight_max": 38.20570373535156,
      "activations/layer10_attention_weight_min": -35.01401138305664,
      "activations/layer11_attention_weight_max": 30.59307098388672,
      "activations/layer11_attention_weight_min": -29.389625549316406,
      "activations/layer12_attention_weight_max": 22.00613784790039,
      "activations/layer12_attention_weight_min": -23.924636840820312,
      "activations/layer13_attention_weight_max": 44.59213638305664,
      "activations/layer13_attention_weight_min": -35.94649887084961,
      "activations/layer14_attention_weight_max": 38.50802993774414,
      "activations/layer14_attention_weight_min": -31.726110458374023,
      "activations/layer15_attention_weight_max": 35.53398895263672,
      "activations/layer15_attention_weight_min": -29.309566497802734,
      "activations/layer16_attention_weight_max": 33.88101577758789,
      "activations/layer16_attention_weight_min": -26.62251853942871,
      "activations/layer17_attention_weight_max": 57.6209716796875,
      "activations/layer17_attention_weight_min": -41.409603118896484,
      "activations/layer18_attention_weight_max": 45.96131896972656,
      "activations/layer18_attention_weight_min": -35.54116439819336,
      "activations/layer19_attention_weight_max": 20.52515983581543,
      "activations/layer19_attention_weight_min": -21.7576961517334,
      "activations/layer1_attention_weight_max": 16.079620361328125,
      "activations/layer1_attention_weight_min": -14.039141654968262,
      "activations/layer20_attention_weight_max": 22.58165168762207,
      "activations/layer20_attention_weight_min": -22.4628963470459,
      "activations/layer21_attention_weight_max": 39.015743255615234,
      "activations/layer21_attention_weight_min": -24.015487670898438,
      "activations/layer22_attention_weight_max": 30.03756332397461,
      "activations/layer22_attention_weight_min": -22.553091049194336,
      "activations/layer23_attention_weight_max": 30.104610443115234,
      "activations/layer23_attention_weight_min": -20.615886688232422,
      "activations/layer2_attention_weight_max": 30.52596664428711,
      "activations/layer2_attention_weight_min": -28.266998291015625,
      "activations/layer3_attention_weight_max": 80.99644470214844,
      "activations/layer3_attention_weight_min": -79.48998260498047,
      "activations/layer4_attention_weight_max": 94.95728302001953,
      "activations/layer4_attention_weight_min": -87.25487518310547,
      "activations/layer5_attention_weight_max": 90.34941864013672,
      "activations/layer5_attention_weight_min": -84.7967529296875,
      "activations/layer6_attention_weight_max": 54.05179214477539,
      "activations/layer6_attention_weight_min": -51.082237243652344,
      "activations/layer7_attention_weight_max": 71.13687896728516,
      "activations/layer7_attention_weight_min": -71.23986053466797,
      "activations/layer8_attention_weight_max": 42.36304473876953,
      "activations/layer8_attention_weight_min": -42.53184127807617,
      "activations/layer9_attention_weight_max": 51.78139877319336,
      "activations/layer9_attention_weight_min": -45.09357833862305,
      "epoch": 6.6,
      "learning_rate": 0.00010854393939393938,
      "loss": 2.8437,
      "step": 113550
    },
    {
      "activations/layer0_attention_weight_max": 15.421818733215332,
      "activations/layer0_attention_weight_min": -14.70414924621582,
      "activations/layer10_attention_weight_max": 43.092105865478516,
      "activations/layer10_attention_weight_min": -40.418067932128906,
      "activations/layer11_attention_weight_max": 32.57929229736328,
      "activations/layer11_attention_weight_min": -30.605220794677734,
      "activations/layer12_attention_weight_max": 31.455705642700195,
      "activations/layer12_attention_weight_min": -27.96235466003418,
      "activations/layer13_attention_weight_max": 47.9367561340332,
      "activations/layer13_attention_weight_min": -32.571964263916016,
      "activations/layer14_attention_weight_max": 43.19330596923828,
      "activations/layer14_attention_weight_min": -31.139219284057617,
      "activations/layer15_attention_weight_max": 40.316070556640625,
      "activations/layer15_attention_weight_min": -30.629972457885742,
      "activations/layer16_attention_weight_max": 33.29151153564453,
      "activations/layer16_attention_weight_min": -27.790372848510742,
      "activations/layer17_attention_weight_max": 61.75918197631836,
      "activations/layer17_attention_weight_min": -42.12663269042969,
      "activations/layer18_attention_weight_max": 45.7852897644043,
      "activations/layer18_attention_weight_min": -35.41230010986328,
      "activations/layer19_attention_weight_max": 26.034643173217773,
      "activations/layer19_attention_weight_min": -19.544736862182617,
      "activations/layer1_attention_weight_max": 17.953128814697266,
      "activations/layer1_attention_weight_min": -13.757369995117188,
      "activations/layer20_attention_weight_max": 21.02921485900879,
      "activations/layer20_attention_weight_min": -21.508153915405273,
      "activations/layer21_attention_weight_max": 36.95561599731445,
      "activations/layer21_attention_weight_min": -24.091777801513672,
      "activations/layer22_attention_weight_max": 38.02768325805664,
      "activations/layer22_attention_weight_min": -24.195878982543945,
      "activations/layer23_attention_weight_max": 32.26203155517578,
      "activations/layer23_attention_weight_min": -21.129375457763672,
      "activations/layer2_attention_weight_max": 32.12419891357422,
      "activations/layer2_attention_weight_min": -30.17225456237793,
      "activations/layer3_attention_weight_max": 82.8329086303711,
      "activations/layer3_attention_weight_min": -83.27674102783203,
      "activations/layer4_attention_weight_max": 99.67201232910156,
      "activations/layer4_attention_weight_min": -92.42046356201172,
      "activations/layer5_attention_weight_max": 94.32933807373047,
      "activations/layer5_attention_weight_min": -90.93925476074219,
      "activations/layer6_attention_weight_max": 59.42454147338867,
      "activations/layer6_attention_weight_min": -53.6904411315918,
      "activations/layer7_attention_weight_max": 76.18540954589844,
      "activations/layer7_attention_weight_min": -74.95995330810547,
      "activations/layer8_attention_weight_max": 47.44929122924805,
      "activations/layer8_attention_weight_min": -49.107791900634766,
      "activations/layer9_attention_weight_max": 55.60260009765625,
      "activations/layer9_attention_weight_min": -53.95425033569336,
      "epoch": 6.6,
      "learning_rate": 0.000108525,
      "loss": 2.8147,
      "step": 113600
    },
    {
      "activations/layer0_attention_weight_max": 14.878875732421875,
      "activations/layer0_attention_weight_min": -14.673489570617676,
      "activations/layer10_attention_weight_max": 38.40208435058594,
      "activations/layer10_attention_weight_min": -35.10867691040039,
      "activations/layer11_attention_weight_max": 30.546178817749023,
      "activations/layer11_attention_weight_min": -29.43927001953125,
      "activations/layer12_attention_weight_max": 20.183340072631836,
      "activations/layer12_attention_weight_min": -22.97844123840332,
      "activations/layer13_attention_weight_max": 43.1900520324707,
      "activations/layer13_attention_weight_min": -32.612911224365234,
      "activations/layer14_attention_weight_max": 41.897315979003906,
      "activations/layer14_attention_weight_min": -33.13465881347656,
      "activations/layer15_attention_weight_max": 39.656898498535156,
      "activations/layer15_attention_weight_min": -30.755016326904297,
      "activations/layer16_attention_weight_max": 32.60403060913086,
      "activations/layer16_attention_weight_min": -28.178356170654297,
      "activations/layer17_attention_weight_max": 58.80843734741211,
      "activations/layer17_attention_weight_min": -44.15970993041992,
      "activations/layer18_attention_weight_max": 48.253204345703125,
      "activations/layer18_attention_weight_min": -36.72220993041992,
      "activations/layer19_attention_weight_max": 23.464113235473633,
      "activations/layer19_attention_weight_min": -20.183408737182617,
      "activations/layer1_attention_weight_max": 17.054353713989258,
      "activations/layer1_attention_weight_min": -15.099117279052734,
      "activations/layer20_attention_weight_max": 22.332237243652344,
      "activations/layer20_attention_weight_min": -20.2613582611084,
      "activations/layer21_attention_weight_max": 41.35430908203125,
      "activations/layer21_attention_weight_min": -24.82477378845215,
      "activations/layer22_attention_weight_max": 33.56812286376953,
      "activations/layer22_attention_weight_min": -26.843732833862305,
      "activations/layer23_attention_weight_max": 31.637556076049805,
      "activations/layer23_attention_weight_min": -19.394001007080078,
      "activations/layer2_attention_weight_max": 30.553672790527344,
      "activations/layer2_attention_weight_min": -28.944616317749023,
      "activations/layer3_attention_weight_max": 82.73907470703125,
      "activations/layer3_attention_weight_min": -83.87811279296875,
      "activations/layer4_attention_weight_max": 95.89659118652344,
      "activations/layer4_attention_weight_min": -93.46546936035156,
      "activations/layer5_attention_weight_max": 88.4662094116211,
      "activations/layer5_attention_weight_min": -85.54817962646484,
      "activations/layer6_attention_weight_max": 53.537166595458984,
      "activations/layer6_attention_weight_min": -50.97004318237305,
      "activations/layer7_attention_weight_max": 72.58360290527344,
      "activations/layer7_attention_weight_min": -71.14632415771484,
      "activations/layer8_attention_weight_max": 43.938232421875,
      "activations/layer8_attention_weight_min": -43.29152297973633,
      "activations/layer9_attention_weight_max": 51.0818977355957,
      "activations/layer9_attention_weight_min": -45.75251770019531,
      "epoch": 6.6,
      "learning_rate": 0.0001085060606060606,
      "loss": 2.8285,
      "step": 113650
    },
    {
      "activations/layer0_attention_weight_max": 15.147379875183105,
      "activations/layer0_attention_weight_min": -14.07496166229248,
      "activations/layer10_attention_weight_max": 37.51584243774414,
      "activations/layer10_attention_weight_min": -34.87868118286133,
      "activations/layer11_attention_weight_max": 31.133712768554688,
      "activations/layer11_attention_weight_min": -31.664480209350586,
      "activations/layer12_attention_weight_max": 19.875513076782227,
      "activations/layer12_attention_weight_min": -25.496313095092773,
      "activations/layer13_attention_weight_max": 43.20634078979492,
      "activations/layer13_attention_weight_min": -32.136207580566406,
      "activations/layer14_attention_weight_max": 40.172359466552734,
      "activations/layer14_attention_weight_min": -31.897563934326172,
      "activations/layer15_attention_weight_max": 38.128150939941406,
      "activations/layer15_attention_weight_min": -30.404333114624023,
      "activations/layer16_attention_weight_max": 33.53237533569336,
      "activations/layer16_attention_weight_min": -27.280139923095703,
      "activations/layer17_attention_weight_max": 58.91355514526367,
      "activations/layer17_attention_weight_min": -41.529842376708984,
      "activations/layer18_attention_weight_max": 53.48772048950195,
      "activations/layer18_attention_weight_min": -35.21427917480469,
      "activations/layer19_attention_weight_max": 20.49115562438965,
      "activations/layer19_attention_weight_min": -20.067237854003906,
      "activations/layer1_attention_weight_max": 16.770036697387695,
      "activations/layer1_attention_weight_min": -14.723309516906738,
      "activations/layer20_attention_weight_max": 21.614606857299805,
      "activations/layer20_attention_weight_min": -21.68445587158203,
      "activations/layer21_attention_weight_max": 43.61151123046875,
      "activations/layer21_attention_weight_min": -25.76865577697754,
      "activations/layer22_attention_weight_max": 33.307579040527344,
      "activations/layer22_attention_weight_min": -25.84836196899414,
      "activations/layer23_attention_weight_max": 32.99925994873047,
      "activations/layer23_attention_weight_min": -20.22662353515625,
      "activations/layer2_attention_weight_max": 30.02448844909668,
      "activations/layer2_attention_weight_min": -28.808956146240234,
      "activations/layer3_attention_weight_max": 85.07115936279297,
      "activations/layer3_attention_weight_min": -89.26306915283203,
      "activations/layer4_attention_weight_max": 98.10755920410156,
      "activations/layer4_attention_weight_min": -90.73990631103516,
      "activations/layer5_attention_weight_max": 92.35531616210938,
      "activations/layer5_attention_weight_min": -85.44170379638672,
      "activations/layer6_attention_weight_max": 58.765220642089844,
      "activations/layer6_attention_weight_min": -54.37950134277344,
      "activations/layer7_attention_weight_max": 72.29254913330078,
      "activations/layer7_attention_weight_min": -66.73765563964844,
      "activations/layer8_attention_weight_max": 43.27036666870117,
      "activations/layer8_attention_weight_min": -42.060516357421875,
      "activations/layer9_attention_weight_max": 47.91591262817383,
      "activations/layer9_attention_weight_min": -48.82972717285156,
      "epoch": 6.61,
      "learning_rate": 0.0001084871212121212,
      "loss": 2.8386,
      "step": 113700
    },
    {
      "activations/layer0_attention_weight_max": 14.109210014343262,
      "activations/layer0_attention_weight_min": -14.436468124389648,
      "activations/layer10_attention_weight_max": 34.64407730102539,
      "activations/layer10_attention_weight_min": -35.02618408203125,
      "activations/layer11_attention_weight_max": 29.171527862548828,
      "activations/layer11_attention_weight_min": -29.58769989013672,
      "activations/layer12_attention_weight_max": 24.098575592041016,
      "activations/layer12_attention_weight_min": -25.97171401977539,
      "activations/layer13_attention_weight_max": 36.66809844970703,
      "activations/layer13_attention_weight_min": -28.909320831298828,
      "activations/layer14_attention_weight_max": 38.482357025146484,
      "activations/layer14_attention_weight_min": -30.730348587036133,
      "activations/layer15_attention_weight_max": 34.80487823486328,
      "activations/layer15_attention_weight_min": -28.44993019104004,
      "activations/layer16_attention_weight_max": 33.29410171508789,
      "activations/layer16_attention_weight_min": -27.88778305053711,
      "activations/layer17_attention_weight_max": 53.655582427978516,
      "activations/layer17_attention_weight_min": -41.2747688293457,
      "activations/layer18_attention_weight_max": 52.52423858642578,
      "activations/layer18_attention_weight_min": -33.732078552246094,
      "activations/layer19_attention_weight_max": 19.970457077026367,
      "activations/layer19_attention_weight_min": -18.27199363708496,
      "activations/layer1_attention_weight_max": 16.867082595825195,
      "activations/layer1_attention_weight_min": -17.155845642089844,
      "activations/layer20_attention_weight_max": 20.01060676574707,
      "activations/layer20_attention_weight_min": -18.414216995239258,
      "activations/layer21_attention_weight_max": 38.14908981323242,
      "activations/layer21_attention_weight_min": -23.745628356933594,
      "activations/layer22_attention_weight_max": 37.02005386352539,
      "activations/layer22_attention_weight_min": -23.37019920349121,
      "activations/layer23_attention_weight_max": 31.687925338745117,
      "activations/layer23_attention_weight_min": -20.097034454345703,
      "activations/layer2_attention_weight_max": 30.31357192993164,
      "activations/layer2_attention_weight_min": -29.096797943115234,
      "activations/layer3_attention_weight_max": 85.60352325439453,
      "activations/layer3_attention_weight_min": -82.82894897460938,
      "activations/layer4_attention_weight_max": 96.26592254638672,
      "activations/layer4_attention_weight_min": -90.51737213134766,
      "activations/layer5_attention_weight_max": 94.68568420410156,
      "activations/layer5_attention_weight_min": -83.32428741455078,
      "activations/layer6_attention_weight_max": 57.31294631958008,
      "activations/layer6_attention_weight_min": -51.34775161743164,
      "activations/layer7_attention_weight_max": 71.37184143066406,
      "activations/layer7_attention_weight_min": -73.6312026977539,
      "activations/layer8_attention_weight_max": 43.80964279174805,
      "activations/layer8_attention_weight_min": -40.345699310302734,
      "activations/layer9_attention_weight_max": 52.19985580444336,
      "activations/layer9_attention_weight_min": -49.31437683105469,
      "epoch": 6.61,
      "learning_rate": 0.00010846818181818181,
      "loss": 2.8321,
      "step": 113750
    },
    {
      "activations/layer0_attention_weight_max": 14.689908027648926,
      "activations/layer0_attention_weight_min": -14.592784881591797,
      "activations/layer10_attention_weight_max": 40.2273063659668,
      "activations/layer10_attention_weight_min": -38.06021499633789,
      "activations/layer11_attention_weight_max": 32.56324768066406,
      "activations/layer11_attention_weight_min": -31.22711181640625,
      "activations/layer12_attention_weight_max": 25.520952224731445,
      "activations/layer12_attention_weight_min": -25.166276931762695,
      "activations/layer13_attention_weight_max": 44.001346588134766,
      "activations/layer13_attention_weight_min": -32.792789459228516,
      "activations/layer14_attention_weight_max": 40.27772903442383,
      "activations/layer14_attention_weight_min": -30.492807388305664,
      "activations/layer15_attention_weight_max": 36.53532409667969,
      "activations/layer15_attention_weight_min": -29.793087005615234,
      "activations/layer16_attention_weight_max": 37.413902282714844,
      "activations/layer16_attention_weight_min": -26.020544052124023,
      "activations/layer17_attention_weight_max": 60.89275360107422,
      "activations/layer17_attention_weight_min": -42.71900939941406,
      "activations/layer18_attention_weight_max": 53.90457534790039,
      "activations/layer18_attention_weight_min": -36.67774200439453,
      "activations/layer19_attention_weight_max": 24.0322322845459,
      "activations/layer19_attention_weight_min": -21.512786865234375,
      "activations/layer1_attention_weight_max": 16.787891387939453,
      "activations/layer1_attention_weight_min": -15.232744216918945,
      "activations/layer20_attention_weight_max": 22.718822479248047,
      "activations/layer20_attention_weight_min": -25.102989196777344,
      "activations/layer21_attention_weight_max": 44.0592155456543,
      "activations/layer21_attention_weight_min": -24.813365936279297,
      "activations/layer22_attention_weight_max": 37.424190521240234,
      "activations/layer22_attention_weight_min": -25.507932662963867,
      "activations/layer23_attention_weight_max": 38.25883483886719,
      "activations/layer23_attention_weight_min": -23.285335540771484,
      "activations/layer2_attention_weight_max": 32.1064338684082,
      "activations/layer2_attention_weight_min": -29.703161239624023,
      "activations/layer3_attention_weight_max": 82.07190704345703,
      "activations/layer3_attention_weight_min": -84.40579223632812,
      "activations/layer4_attention_weight_max": 99.67204284667969,
      "activations/layer4_attention_weight_min": -93.40277099609375,
      "activations/layer5_attention_weight_max": 96.6019287109375,
      "activations/layer5_attention_weight_min": -86.23910522460938,
      "activations/layer6_attention_weight_max": 58.0226936340332,
      "activations/layer6_attention_weight_min": -55.59067153930664,
      "activations/layer7_attention_weight_max": 74.55165100097656,
      "activations/layer7_attention_weight_min": -73.34978485107422,
      "activations/layer8_attention_weight_max": 44.899173736572266,
      "activations/layer8_attention_weight_min": -44.479000091552734,
      "activations/layer9_attention_weight_max": 51.09172439575195,
      "activations/layer9_attention_weight_min": -53.123268127441406,
      "epoch": 6.61,
      "learning_rate": 0.00010844924242424241,
      "loss": 2.8303,
      "step": 113800
    },
    {
      "activations/layer0_attention_weight_max": 14.38875675201416,
      "activations/layer0_attention_weight_min": -14.634710311889648,
      "activations/layer10_attention_weight_max": 36.99496078491211,
      "activations/layer10_attention_weight_min": -33.96622085571289,
      "activations/layer11_attention_weight_max": 31.44928550720215,
      "activations/layer11_attention_weight_min": -28.15852165222168,
      "activations/layer12_attention_weight_max": 21.265918731689453,
      "activations/layer12_attention_weight_min": -25.142017364501953,
      "activations/layer13_attention_weight_max": 63.02006530761719,
      "activations/layer13_attention_weight_min": -35.12283706665039,
      "activations/layer14_attention_weight_max": 54.41579818725586,
      "activations/layer14_attention_weight_min": -31.36739730834961,
      "activations/layer15_attention_weight_max": 46.25494384765625,
      "activations/layer15_attention_weight_min": -29.843942642211914,
      "activations/layer16_attention_weight_max": 43.77507781982422,
      "activations/layer16_attention_weight_min": -26.39427375793457,
      "activations/layer17_attention_weight_max": 55.80568313598633,
      "activations/layer17_attention_weight_min": -40.69966506958008,
      "activations/layer18_attention_weight_max": 54.91752624511719,
      "activations/layer18_attention_weight_min": -35.808807373046875,
      "activations/layer19_attention_weight_max": 23.54322624206543,
      "activations/layer19_attention_weight_min": -21.26433753967285,
      "activations/layer1_attention_weight_max": 16.953052520751953,
      "activations/layer1_attention_weight_min": -14.930081367492676,
      "activations/layer20_attention_weight_max": 28.987504959106445,
      "activations/layer20_attention_weight_min": -22.08377456665039,
      "activations/layer21_attention_weight_max": 46.48041915893555,
      "activations/layer21_attention_weight_min": -22.385583877563477,
      "activations/layer22_attention_weight_max": 40.309120178222656,
      "activations/layer22_attention_weight_min": -22.861637115478516,
      "activations/layer23_attention_weight_max": 38.26395034790039,
      "activations/layer23_attention_weight_min": -23.394285202026367,
      "activations/layer2_attention_weight_max": 31.24286651611328,
      "activations/layer2_attention_weight_min": -28.73806381225586,
      "activations/layer3_attention_weight_max": 79.49886322021484,
      "activations/layer3_attention_weight_min": -82.07795715332031,
      "activations/layer4_attention_weight_max": 93.37767028808594,
      "activations/layer4_attention_weight_min": -88.9537582397461,
      "activations/layer5_attention_weight_max": 86.88762664794922,
      "activations/layer5_attention_weight_min": -82.7751235961914,
      "activations/layer6_attention_weight_max": 52.15966033935547,
      "activations/layer6_attention_weight_min": -52.190528869628906,
      "activations/layer7_attention_weight_max": 72.05538940429688,
      "activations/layer7_attention_weight_min": -73.19827270507812,
      "activations/layer8_attention_weight_max": 43.01060485839844,
      "activations/layer8_attention_weight_min": -45.92667007446289,
      "activations/layer9_attention_weight_max": 53.41676712036133,
      "activations/layer9_attention_weight_min": -49.2261848449707,
      "epoch": 6.62,
      "learning_rate": 0.00010843030303030302,
      "loss": 2.8319,
      "step": 113850
    },
    {
      "activations/layer0_attention_weight_max": 15.168868064880371,
      "activations/layer0_attention_weight_min": -14.533174514770508,
      "activations/layer10_attention_weight_max": 36.94635009765625,
      "activations/layer10_attention_weight_min": -35.26162338256836,
      "activations/layer11_attention_weight_max": 30.685184478759766,
      "activations/layer11_attention_weight_min": -28.847455978393555,
      "activations/layer12_attention_weight_max": 22.880386352539062,
      "activations/layer12_attention_weight_min": -25.749317169189453,
      "activations/layer13_attention_weight_max": 52.9109992980957,
      "activations/layer13_attention_weight_min": -36.436214447021484,
      "activations/layer14_attention_weight_max": 40.7633171081543,
      "activations/layer14_attention_weight_min": -31.743816375732422,
      "activations/layer15_attention_weight_max": 37.88500213623047,
      "activations/layer15_attention_weight_min": -28.81911849975586,
      "activations/layer16_attention_weight_max": 37.378353118896484,
      "activations/layer16_attention_weight_min": -27.7983455657959,
      "activations/layer17_attention_weight_max": 55.18346405029297,
      "activations/layer17_attention_weight_min": -41.745914459228516,
      "activations/layer18_attention_weight_max": 54.93574142456055,
      "activations/layer18_attention_weight_min": -37.14942932128906,
      "activations/layer19_attention_weight_max": 23.65237045288086,
      "activations/layer19_attention_weight_min": -22.45020866394043,
      "activations/layer1_attention_weight_max": 17.679527282714844,
      "activations/layer1_attention_weight_min": -15.585878372192383,
      "activations/layer20_attention_weight_max": 27.319185256958008,
      "activations/layer20_attention_weight_min": -22.609243392944336,
      "activations/layer21_attention_weight_max": 44.44517517089844,
      "activations/layer21_attention_weight_min": -25.257713317871094,
      "activations/layer22_attention_weight_max": 32.92964172363281,
      "activations/layer22_attention_weight_min": -24.90892219543457,
      "activations/layer23_attention_weight_max": 32.62158966064453,
      "activations/layer23_attention_weight_min": -19.798898696899414,
      "activations/layer2_attention_weight_max": 31.0185604095459,
      "activations/layer2_attention_weight_min": -30.024005889892578,
      "activations/layer3_attention_weight_max": 81.9906234741211,
      "activations/layer3_attention_weight_min": -86.4902572631836,
      "activations/layer4_attention_weight_max": 98.35645294189453,
      "activations/layer4_attention_weight_min": -90.8517837524414,
      "activations/layer5_attention_weight_max": 93.47056579589844,
      "activations/layer5_attention_weight_min": -85.35467529296875,
      "activations/layer6_attention_weight_max": 59.80500030517578,
      "activations/layer6_attention_weight_min": -51.372459411621094,
      "activations/layer7_attention_weight_max": 74.06990814208984,
      "activations/layer7_attention_weight_min": -72.10835266113281,
      "activations/layer8_attention_weight_max": 43.32425308227539,
      "activations/layer8_attention_weight_min": -42.632720947265625,
      "activations/layer9_attention_weight_max": 50.17575454711914,
      "activations/layer9_attention_weight_min": -48.02033615112305,
      "epoch": 6.62,
      "learning_rate": 0.00010841136363636363,
      "loss": 2.8338,
      "step": 113900
    },
    {
      "activations/layer0_attention_weight_max": 14.389534950256348,
      "activations/layer0_attention_weight_min": -14.581619262695312,
      "activations/layer10_attention_weight_max": 40.58734893798828,
      "activations/layer10_attention_weight_min": -37.604774475097656,
      "activations/layer11_attention_weight_max": 32.81938171386719,
      "activations/layer11_attention_weight_min": -32.59801483154297,
      "activations/layer12_attention_weight_max": 24.818557739257812,
      "activations/layer12_attention_weight_min": -25.517255783081055,
      "activations/layer13_attention_weight_max": 43.26290512084961,
      "activations/layer13_attention_weight_min": -32.1868896484375,
      "activations/layer14_attention_weight_max": 38.7795295715332,
      "activations/layer14_attention_weight_min": -32.376220703125,
      "activations/layer15_attention_weight_max": 37.243289947509766,
      "activations/layer15_attention_weight_min": -31.152313232421875,
      "activations/layer16_attention_weight_max": 32.813446044921875,
      "activations/layer16_attention_weight_min": -29.13988494873047,
      "activations/layer17_attention_weight_max": 54.06321334838867,
      "activations/layer17_attention_weight_min": -42.75686264038086,
      "activations/layer18_attention_weight_max": 46.515445709228516,
      "activations/layer18_attention_weight_min": -34.556251525878906,
      "activations/layer19_attention_weight_max": 22.559425354003906,
      "activations/layer19_attention_weight_min": -19.290678024291992,
      "activations/layer1_attention_weight_max": 17.130435943603516,
      "activations/layer1_attention_weight_min": -14.81559944152832,
      "activations/layer20_attention_weight_max": 22.512828826904297,
      "activations/layer20_attention_weight_min": -21.356035232543945,
      "activations/layer21_attention_weight_max": 38.43671417236328,
      "activations/layer21_attention_weight_min": -25.6543025970459,
      "activations/layer22_attention_weight_max": 32.81726837158203,
      "activations/layer22_attention_weight_min": -26.312349319458008,
      "activations/layer23_attention_weight_max": 31.622783660888672,
      "activations/layer23_attention_weight_min": -20.641826629638672,
      "activations/layer2_attention_weight_max": 31.447988510131836,
      "activations/layer2_attention_weight_min": -29.58403778076172,
      "activations/layer3_attention_weight_max": 86.41338348388672,
      "activations/layer3_attention_weight_min": -87.08807373046875,
      "activations/layer4_attention_weight_max": 100.24715423583984,
      "activations/layer4_attention_weight_min": -93.9192886352539,
      "activations/layer5_attention_weight_max": 93.52731323242188,
      "activations/layer5_attention_weight_min": -90.2292709350586,
      "activations/layer6_attention_weight_max": 56.97024917602539,
      "activations/layer6_attention_weight_min": -54.84341812133789,
      "activations/layer7_attention_weight_max": 73.994140625,
      "activations/layer7_attention_weight_min": -72.4507064819336,
      "activations/layer8_attention_weight_max": 48.71030807495117,
      "activations/layer8_attention_weight_min": -45.943599700927734,
      "activations/layer9_attention_weight_max": 55.52202224731445,
      "activations/layer9_attention_weight_min": -48.48204803466797,
      "epoch": 6.62,
      "learning_rate": 0.00010839242424242423,
      "loss": 2.8189,
      "step": 113950
    },
    {
      "activations/layer0_attention_weight_max": 14.527618408203125,
      "activations/layer0_attention_weight_min": -14.67170524597168,
      "activations/layer10_attention_weight_max": 41.1685676574707,
      "activations/layer10_attention_weight_min": -36.40788650512695,
      "activations/layer11_attention_weight_max": 34.1434211730957,
      "activations/layer11_attention_weight_min": -30.719730377197266,
      "activations/layer12_attention_weight_max": 22.06418228149414,
      "activations/layer12_attention_weight_min": -26.328752517700195,
      "activations/layer13_attention_weight_max": 50.205928802490234,
      "activations/layer13_attention_weight_min": -36.5688362121582,
      "activations/layer14_attention_weight_max": 55.728759765625,
      "activations/layer14_attention_weight_min": -34.3294563293457,
      "activations/layer15_attention_weight_max": 40.52322769165039,
      "activations/layer15_attention_weight_min": -29.688671112060547,
      "activations/layer16_attention_weight_max": 33.184898376464844,
      "activations/layer16_attention_weight_min": -28.247447967529297,
      "activations/layer17_attention_weight_max": 67.62601470947266,
      "activations/layer17_attention_weight_min": -42.88542938232422,
      "activations/layer18_attention_weight_max": 46.20685958862305,
      "activations/layer18_attention_weight_min": -36.52192687988281,
      "activations/layer19_attention_weight_max": 22.453325271606445,
      "activations/layer19_attention_weight_min": -19.99009895324707,
      "activations/layer1_attention_weight_max": 16.743999481201172,
      "activations/layer1_attention_weight_min": -16.629053115844727,
      "activations/layer20_attention_weight_max": 21.75615882873535,
      "activations/layer20_attention_weight_min": -23.518217086791992,
      "activations/layer21_attention_weight_max": 40.773414611816406,
      "activations/layer21_attention_weight_min": -27.78079605102539,
      "activations/layer22_attention_weight_max": 33.06216812133789,
      "activations/layer22_attention_weight_min": -23.093364715576172,
      "activations/layer23_attention_weight_max": 34.051395416259766,
      "activations/layer23_attention_weight_min": -21.942699432373047,
      "activations/layer2_attention_weight_max": 31.88542938232422,
      "activations/layer2_attention_weight_min": -30.698902130126953,
      "activations/layer3_attention_weight_max": 85.97746276855469,
      "activations/layer3_attention_weight_min": -86.59996032714844,
      "activations/layer4_attention_weight_max": 101.43810272216797,
      "activations/layer4_attention_weight_min": -96.75670623779297,
      "activations/layer5_attention_weight_max": 98.11924743652344,
      "activations/layer5_attention_weight_min": -90.28289794921875,
      "activations/layer6_attention_weight_max": 58.50164794921875,
      "activations/layer6_attention_weight_min": -53.894264221191406,
      "activations/layer7_attention_weight_max": 71.34400177001953,
      "activations/layer7_attention_weight_min": -77.15182495117188,
      "activations/layer8_attention_weight_max": 45.913455963134766,
      "activations/layer8_attention_weight_min": -44.640933990478516,
      "activations/layer9_attention_weight_max": 56.4266357421875,
      "activations/layer9_attention_weight_min": -54.1739387512207,
      "epoch": 6.62,
      "learning_rate": 0.00010837348484848485,
      "loss": 2.8236,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_loss": 2.77734375,
      "eval_runtime": 8.5371,
      "eval_samples_per_second": 502.981,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_openwebtext_loss": 2.77734375,
      "eval_openwebtext_ppl": 16.076261613516785,
      "eval_openwebtext_runtime": 8.5371,
      "eval_openwebtext_samples_per_second": 502.981,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_wikitext_loss": 3.005859375,
      "eval_wikitext_ppl": 20.203571081597193,
      "eval_wikitext_runtime": 2.0197,
      "eval_wikitext_samples_per_second": 225.773,
      "step": 114000
    },
    {
      "epoch": 6.62,
      "eval_lambada_loss": 2.73828125,
      "eval_lambada_ppl": 15.460389702669161,
      "eval_lambada_runtime": 9.6322,
      "eval_lambada_samples_per_second": 505.492,
      "step": 114000
    },
    {
      "activations/layer0_attention_weight_max": 17.339765548706055,
      "activations/layer0_attention_weight_min": -14.599020957946777,
      "activations/layer10_attention_weight_max": 39.22984313964844,
      "activations/layer10_attention_weight_min": -34.96067428588867,
      "activations/layer11_attention_weight_max": 33.62032699584961,
      "activations/layer11_attention_weight_min": -28.71149444580078,
      "activations/layer12_attention_weight_max": 21.945737838745117,
      "activations/layer12_attention_weight_min": -29.12982940673828,
      "activations/layer13_attention_weight_max": 50.92033004760742,
      "activations/layer13_attention_weight_min": -33.752685546875,
      "activations/layer14_attention_weight_max": 39.28437042236328,
      "activations/layer14_attention_weight_min": -33.749168395996094,
      "activations/layer15_attention_weight_max": 35.28279495239258,
      "activations/layer15_attention_weight_min": -31.64105987548828,
      "activations/layer16_attention_weight_max": 35.292724609375,
      "activations/layer16_attention_weight_min": -26.20317268371582,
      "activations/layer17_attention_weight_max": 52.61003494262695,
      "activations/layer17_attention_weight_min": -40.72321701049805,
      "activations/layer18_attention_weight_max": 49.18496322631836,
      "activations/layer18_attention_weight_min": -34.87039566040039,
      "activations/layer19_attention_weight_max": 23.981082916259766,
      "activations/layer19_attention_weight_min": -18.99812889099121,
      "activations/layer1_attention_weight_max": 18.373191833496094,
      "activations/layer1_attention_weight_min": -17.29709243774414,
      "activations/layer20_attention_weight_max": 20.78306007385254,
      "activations/layer20_attention_weight_min": -19.07029914855957,
      "activations/layer21_attention_weight_max": 37.86354446411133,
      "activations/layer21_attention_weight_min": -23.91820526123047,
      "activations/layer22_attention_weight_max": 32.871795654296875,
      "activations/layer22_attention_weight_min": -22.850008010864258,
      "activations/layer23_attention_weight_max": 36.04756546020508,
      "activations/layer23_attention_weight_min": -20.019935607910156,
      "activations/layer2_attention_weight_max": 31.2880859375,
      "activations/layer2_attention_weight_min": -30.05657958984375,
      "activations/layer3_attention_weight_max": 89.74332427978516,
      "activations/layer3_attention_weight_min": -90.64195251464844,
      "activations/layer4_attention_weight_max": 105.13578033447266,
      "activations/layer4_attention_weight_min": -97.53592681884766,
      "activations/layer5_attention_weight_max": 97.70977020263672,
      "activations/layer5_attention_weight_min": -88.362060546875,
      "activations/layer6_attention_weight_max": 56.54441452026367,
      "activations/layer6_attention_weight_min": -54.9775505065918,
      "activations/layer7_attention_weight_max": 74.8176498413086,
      "activations/layer7_attention_weight_min": -69.92068481445312,
      "activations/layer8_attention_weight_max": 44.55083084106445,
      "activations/layer8_attention_weight_min": -47.02299118041992,
      "activations/layer9_attention_weight_max": 52.86212921142578,
      "activations/layer9_attention_weight_min": -47.596614837646484,
      "epoch": 6.63,
      "learning_rate": 0.00010835454545454544,
      "loss": 2.8188,
      "step": 114050
    },
    {
      "activations/layer0_attention_weight_max": 15.413456916809082,
      "activations/layer0_attention_weight_min": -14.812535285949707,
      "activations/layer10_attention_weight_max": 39.468162536621094,
      "activations/layer10_attention_weight_min": -36.091007232666016,
      "activations/layer11_attention_weight_max": 31.192935943603516,
      "activations/layer11_attention_weight_min": -30.599781036376953,
      "activations/layer12_attention_weight_max": 22.64350128173828,
      "activations/layer12_attention_weight_min": -31.66384506225586,
      "activations/layer13_attention_weight_max": 45.126522064208984,
      "activations/layer13_attention_weight_min": -32.36806106567383,
      "activations/layer14_attention_weight_max": 53.941688537597656,
      "activations/layer14_attention_weight_min": -30.1423282623291,
      "activations/layer15_attention_weight_max": 45.74057388305664,
      "activations/layer15_attention_weight_min": -30.945404052734375,
      "activations/layer16_attention_weight_max": 38.42325210571289,
      "activations/layer16_attention_weight_min": -26.764820098876953,
      "activations/layer17_attention_weight_max": 59.51223373413086,
      "activations/layer17_attention_weight_min": -43.232147216796875,
      "activations/layer18_attention_weight_max": 57.0390739440918,
      "activations/layer18_attention_weight_min": -37.83341979980469,
      "activations/layer19_attention_weight_max": 22.867109298706055,
      "activations/layer19_attention_weight_min": -21.635189056396484,
      "activations/layer1_attention_weight_max": 17.201581954956055,
      "activations/layer1_attention_weight_min": -14.651339530944824,
      "activations/layer20_attention_weight_max": 23.41775131225586,
      "activations/layer20_attention_weight_min": -20.310773849487305,
      "activations/layer21_attention_weight_max": 44.81144332885742,
      "activations/layer21_attention_weight_min": -22.836820602416992,
      "activations/layer22_attention_weight_max": 34.66861343383789,
      "activations/layer22_attention_weight_min": -24.3767032623291,
      "activations/layer23_attention_weight_max": 32.57744598388672,
      "activations/layer23_attention_weight_min": -19.881200790405273,
      "activations/layer2_attention_weight_max": 31.587247848510742,
      "activations/layer2_attention_weight_min": -29.09914779663086,
      "activations/layer3_attention_weight_max": 82.71086883544922,
      "activations/layer3_attention_weight_min": -86.1318588256836,
      "activations/layer4_attention_weight_max": 100.30608367919922,
      "activations/layer4_attention_weight_min": -92.5463638305664,
      "activations/layer5_attention_weight_max": 95.17217254638672,
      "activations/layer5_attention_weight_min": -85.37755584716797,
      "activations/layer6_attention_weight_max": 57.121734619140625,
      "activations/layer6_attention_weight_min": -52.725955963134766,
      "activations/layer7_attention_weight_max": 72.41258239746094,
      "activations/layer7_attention_weight_min": -75.18500518798828,
      "activations/layer8_attention_weight_max": 44.728782653808594,
      "activations/layer8_attention_weight_min": -45.66569519042969,
      "activations/layer9_attention_weight_max": 54.538795471191406,
      "activations/layer9_attention_weight_min": -53.750587463378906,
      "epoch": 6.63,
      "learning_rate": 0.00010833560606060605,
      "loss": 2.8144,
      "step": 114100
    },
    {
      "activations/layer0_attention_weight_max": 14.286691665649414,
      "activations/layer0_attention_weight_min": -14.464899063110352,
      "activations/layer10_attention_weight_max": 48.16328048706055,
      "activations/layer10_attention_weight_min": -42.08781814575195,
      "activations/layer11_attention_weight_max": 39.596595764160156,
      "activations/layer11_attention_weight_min": -33.094520568847656,
      "activations/layer12_attention_weight_max": 22.971847534179688,
      "activations/layer12_attention_weight_min": -24.782651901245117,
      "activations/layer13_attention_weight_max": 66.3251953125,
      "activations/layer13_attention_weight_min": -38.86079025268555,
      "activations/layer14_attention_weight_max": 85.5835952758789,
      "activations/layer14_attention_weight_min": -36.109825134277344,
      "activations/layer15_attention_weight_max": 53.793827056884766,
      "activations/layer15_attention_weight_min": -29.61980438232422,
      "activations/layer16_attention_weight_max": 41.867061614990234,
      "activations/layer16_attention_weight_min": -29.031234741210938,
      "activations/layer17_attention_weight_max": 63.27265930175781,
      "activations/layer17_attention_weight_min": -47.07597351074219,
      "activations/layer18_attention_weight_max": 54.059513092041016,
      "activations/layer18_attention_weight_min": -38.998931884765625,
      "activations/layer19_attention_weight_max": 29.736616134643555,
      "activations/layer19_attention_weight_min": -23.707975387573242,
      "activations/layer1_attention_weight_max": 17.729949951171875,
      "activations/layer1_attention_weight_min": -15.841070175170898,
      "activations/layer20_attention_weight_max": 28.181371688842773,
      "activations/layer20_attention_weight_min": -20.674638748168945,
      "activations/layer21_attention_weight_max": 49.16455078125,
      "activations/layer21_attention_weight_min": -26.984111785888672,
      "activations/layer22_attention_weight_max": 33.50247573852539,
      "activations/layer22_attention_weight_min": -23.034557342529297,
      "activations/layer23_attention_weight_max": 29.578128814697266,
      "activations/layer23_attention_weight_min": -21.074325561523438,
      "activations/layer2_attention_weight_max": 33.30498123168945,
      "activations/layer2_attention_weight_min": -32.22119903564453,
      "activations/layer3_attention_weight_max": 89.29317474365234,
      "activations/layer3_attention_weight_min": -91.38339233398438,
      "activations/layer4_attention_weight_max": 108.10514068603516,
      "activations/layer4_attention_weight_min": -105.25531005859375,
      "activations/layer5_attention_weight_max": 99.58767700195312,
      "activations/layer5_attention_weight_min": -87.71074676513672,
      "activations/layer6_attention_weight_max": 59.410728454589844,
      "activations/layer6_attention_weight_min": -53.262123107910156,
      "activations/layer7_attention_weight_max": 82.53144073486328,
      "activations/layer7_attention_weight_min": -76.55144500732422,
      "activations/layer8_attention_weight_max": 53.44416809082031,
      "activations/layer8_attention_weight_min": -54.19355773925781,
      "activations/layer9_attention_weight_max": 66.36588287353516,
      "activations/layer9_attention_weight_min": -58.0308723449707,
      "epoch": 6.63,
      "learning_rate": 0.00010831666666666667,
      "loss": 2.8132,
      "step": 114150
    },
    {
      "activations/layer0_attention_weight_max": 15.646783828735352,
      "activations/layer0_attention_weight_min": -14.359187126159668,
      "activations/layer10_attention_weight_max": 35.35353088378906,
      "activations/layer10_attention_weight_min": -34.940582275390625,
      "activations/layer11_attention_weight_max": 29.19780731201172,
      "activations/layer11_attention_weight_min": -29.684799194335938,
      "activations/layer12_attention_weight_max": 19.693418502807617,
      "activations/layer12_attention_weight_min": -28.25175666809082,
      "activations/layer13_attention_weight_max": 43.34419250488281,
      "activations/layer13_attention_weight_min": -33.172752380371094,
      "activations/layer14_attention_weight_max": 35.554534912109375,
      "activations/layer14_attention_weight_min": -29.884048461914062,
      "activations/layer15_attention_weight_max": 39.19416046142578,
      "activations/layer15_attention_weight_min": -29.38176727294922,
      "activations/layer16_attention_weight_max": 32.609893798828125,
      "activations/layer16_attention_weight_min": -28.04442024230957,
      "activations/layer17_attention_weight_max": 52.089656829833984,
      "activations/layer17_attention_weight_min": -39.34950256347656,
      "activations/layer18_attention_weight_max": 45.525428771972656,
      "activations/layer18_attention_weight_min": -33.6673698425293,
      "activations/layer19_attention_weight_max": 22.836153030395508,
      "activations/layer19_attention_weight_min": -21.743240356445312,
      "activations/layer1_attention_weight_max": 17.255464553833008,
      "activations/layer1_attention_weight_min": -16.02238655090332,
      "activations/layer20_attention_weight_max": 20.79755973815918,
      "activations/layer20_attention_weight_min": -20.60784149169922,
      "activations/layer21_attention_weight_max": 37.01418685913086,
      "activations/layer21_attention_weight_min": -22.95332145690918,
      "activations/layer22_attention_weight_max": 33.01763916015625,
      "activations/layer22_attention_weight_min": -23.828495025634766,
      "activations/layer23_attention_weight_max": 29.340299606323242,
      "activations/layer23_attention_weight_min": -21.77142906188965,
      "activations/layer2_attention_weight_max": 33.21724319458008,
      "activations/layer2_attention_weight_min": -30.14626693725586,
      "activations/layer3_attention_weight_max": 85.83072662353516,
      "activations/layer3_attention_weight_min": -85.25186920166016,
      "activations/layer4_attention_weight_max": 96.783203125,
      "activations/layer4_attention_weight_min": -92.60443878173828,
      "activations/layer5_attention_weight_max": 92.44036865234375,
      "activations/layer5_attention_weight_min": -86.11526489257812,
      "activations/layer6_attention_weight_max": 53.9165153503418,
      "activations/layer6_attention_weight_min": -51.065834045410156,
      "activations/layer7_attention_weight_max": 71.34603881835938,
      "activations/layer7_attention_weight_min": -73.24266052246094,
      "activations/layer8_attention_weight_max": 41.61322784423828,
      "activations/layer8_attention_weight_min": -44.00471878051758,
      "activations/layer9_attention_weight_max": 54.88108444213867,
      "activations/layer9_attention_weight_min": -47.409427642822266,
      "epoch": 6.64,
      "learning_rate": 0.00010829772727272725,
      "loss": 2.8154,
      "step": 114200
    },
    {
      "activations/layer0_attention_weight_max": 15.912029266357422,
      "activations/layer0_attention_weight_min": -14.1237211227417,
      "activations/layer10_attention_weight_max": 39.881919860839844,
      "activations/layer10_attention_weight_min": -37.05125427246094,
      "activations/layer11_attention_weight_max": 33.08336639404297,
      "activations/layer11_attention_weight_min": -30.175796508789062,
      "activations/layer12_attention_weight_max": 22.42827033996582,
      "activations/layer12_attention_weight_min": -25.10064125061035,
      "activations/layer13_attention_weight_max": 41.415771484375,
      "activations/layer13_attention_weight_min": -34.02161407470703,
      "activations/layer14_attention_weight_max": 40.35740661621094,
      "activations/layer14_attention_weight_min": -31.426212310791016,
      "activations/layer15_attention_weight_max": 37.68680953979492,
      "activations/layer15_attention_weight_min": -30.604957580566406,
      "activations/layer16_attention_weight_max": 36.457763671875,
      "activations/layer16_attention_weight_min": -28.727005004882812,
      "activations/layer17_attention_weight_max": 57.70104217529297,
      "activations/layer17_attention_weight_min": -42.374664306640625,
      "activations/layer18_attention_weight_max": 49.065528869628906,
      "activations/layer18_attention_weight_min": -36.21082305908203,
      "activations/layer19_attention_weight_max": 21.193939208984375,
      "activations/layer19_attention_weight_min": -21.374568939208984,
      "activations/layer1_attention_weight_max": 16.693462371826172,
      "activations/layer1_attention_weight_min": -14.51231861114502,
      "activations/layer20_attention_weight_max": 24.430702209472656,
      "activations/layer20_attention_weight_min": -22.083303451538086,
      "activations/layer21_attention_weight_max": 37.2046012878418,
      "activations/layer21_attention_weight_min": -25.313627243041992,
      "activations/layer22_attention_weight_max": 33.27764129638672,
      "activations/layer22_attention_weight_min": -24.52799415588379,
      "activations/layer23_attention_weight_max": 35.65955352783203,
      "activations/layer23_attention_weight_min": -24.698932647705078,
      "activations/layer2_attention_weight_max": 31.135066986083984,
      "activations/layer2_attention_weight_min": -30.45850944519043,
      "activations/layer3_attention_weight_max": 85.35369873046875,
      "activations/layer3_attention_weight_min": -89.94013977050781,
      "activations/layer4_attention_weight_max": 100.57286834716797,
      "activations/layer4_attention_weight_min": -98.8640365600586,
      "activations/layer5_attention_weight_max": 92.54171752929688,
      "activations/layer5_attention_weight_min": -88.91668701171875,
      "activations/layer6_attention_weight_max": 56.3733024597168,
      "activations/layer6_attention_weight_min": -52.43146514892578,
      "activations/layer7_attention_weight_max": 75.55623626708984,
      "activations/layer7_attention_weight_min": -70.16084289550781,
      "activations/layer8_attention_weight_max": 45.273109436035156,
      "activations/layer8_attention_weight_min": -46.54581832885742,
      "activations/layer9_attention_weight_max": 50.9248046875,
      "activations/layer9_attention_weight_min": -51.45082473754883,
      "epoch": 6.64,
      "learning_rate": 0.00010827878787878787,
      "loss": 2.7946,
      "step": 114250
    },
    {
      "activations/layer0_attention_weight_max": 15.415705680847168,
      "activations/layer0_attention_weight_min": -14.47311782836914,
      "activations/layer10_attention_weight_max": 39.77099609375,
      "activations/layer10_attention_weight_min": -33.92158126831055,
      "activations/layer11_attention_weight_max": 30.223264694213867,
      "activations/layer11_attention_weight_min": -29.19319725036621,
      "activations/layer12_attention_weight_max": 21.74639129638672,
      "activations/layer12_attention_weight_min": -27.944076538085938,
      "activations/layer13_attention_weight_max": 44.72568893432617,
      "activations/layer13_attention_weight_min": -35.8182373046875,
      "activations/layer14_attention_weight_max": 43.15398406982422,
      "activations/layer14_attention_weight_min": -30.146833419799805,
      "activations/layer15_attention_weight_max": 36.26573181152344,
      "activations/layer15_attention_weight_min": -30.29779815673828,
      "activations/layer16_attention_weight_max": 34.344844818115234,
      "activations/layer16_attention_weight_min": -26.65926742553711,
      "activations/layer17_attention_weight_max": 53.517478942871094,
      "activations/layer17_attention_weight_min": -42.032527923583984,
      "activations/layer18_attention_weight_max": 45.00678253173828,
      "activations/layer18_attention_weight_min": -35.11119842529297,
      "activations/layer19_attention_weight_max": 20.603618621826172,
      "activations/layer19_attention_weight_min": -20.152158737182617,
      "activations/layer1_attention_weight_max": 17.805450439453125,
      "activations/layer1_attention_weight_min": -15.536749839782715,
      "activations/layer20_attention_weight_max": 20.420352935791016,
      "activations/layer20_attention_weight_min": -25.133161544799805,
      "activations/layer21_attention_weight_max": 38.29499053955078,
      "activations/layer21_attention_weight_min": -27.91509246826172,
      "activations/layer22_attention_weight_max": 31.66947364807129,
      "activations/layer22_attention_weight_min": -23.550413131713867,
      "activations/layer23_attention_weight_max": 33.62370681762695,
      "activations/layer23_attention_weight_min": -20.39794158935547,
      "activations/layer2_attention_weight_max": 32.30033493041992,
      "activations/layer2_attention_weight_min": -28.703083038330078,
      "activations/layer3_attention_weight_max": 84.0334701538086,
      "activations/layer3_attention_weight_min": -82.53536987304688,
      "activations/layer4_attention_weight_max": 100.9547348022461,
      "activations/layer4_attention_weight_min": -91.88993835449219,
      "activations/layer5_attention_weight_max": 95.60325622558594,
      "activations/layer5_attention_weight_min": -88.56716918945312,
      "activations/layer6_attention_weight_max": 59.142616271972656,
      "activations/layer6_attention_weight_min": -52.905113220214844,
      "activations/layer7_attention_weight_max": 73.09068298339844,
      "activations/layer7_attention_weight_min": -71.72101593017578,
      "activations/layer8_attention_weight_max": 45.52622985839844,
      "activations/layer8_attention_weight_min": -43.741031646728516,
      "activations/layer9_attention_weight_max": 48.7995719909668,
      "activations/layer9_attention_weight_min": -48.577762603759766,
      "epoch": 6.64,
      "learning_rate": 0.00010825984848484848,
      "loss": 2.8053,
      "step": 114300
    },
    {
      "activations/layer0_attention_weight_max": 14.952353477478027,
      "activations/layer0_attention_weight_min": -14.639347076416016,
      "activations/layer10_attention_weight_max": 38.12807846069336,
      "activations/layer10_attention_weight_min": -36.47969436645508,
      "activations/layer11_attention_weight_max": 32.535728454589844,
      "activations/layer11_attention_weight_min": -28.487991333007812,
      "activations/layer12_attention_weight_max": 22.566638946533203,
      "activations/layer12_attention_weight_min": -26.359027862548828,
      "activations/layer13_attention_weight_max": 42.03041076660156,
      "activations/layer13_attention_weight_min": -31.769081115722656,
      "activations/layer14_attention_weight_max": 39.92685317993164,
      "activations/layer14_attention_weight_min": -29.804994583129883,
      "activations/layer15_attention_weight_max": 34.52146911621094,
      "activations/layer15_attention_weight_min": -28.5826358795166,
      "activations/layer16_attention_weight_max": 32.84035110473633,
      "activations/layer16_attention_weight_min": -26.51805305480957,
      "activations/layer17_attention_weight_max": 53.717674255371094,
      "activations/layer17_attention_weight_min": -40.230987548828125,
      "activations/layer18_attention_weight_max": 45.9835205078125,
      "activations/layer18_attention_weight_min": -34.87887191772461,
      "activations/layer19_attention_weight_max": 21.660009384155273,
      "activations/layer19_attention_weight_min": -21.078184127807617,
      "activations/layer1_attention_weight_max": 16.38062286376953,
      "activations/layer1_attention_weight_min": -16.333934783935547,
      "activations/layer20_attention_weight_max": 23.5451717376709,
      "activations/layer20_attention_weight_min": -24.171388626098633,
      "activations/layer21_attention_weight_max": 35.935672760009766,
      "activations/layer21_attention_weight_min": -24.001846313476562,
      "activations/layer22_attention_weight_max": 33.50308609008789,
      "activations/layer22_attention_weight_min": -25.75398063659668,
      "activations/layer23_attention_weight_max": 30.044078826904297,
      "activations/layer23_attention_weight_min": -21.44342041015625,
      "activations/layer2_attention_weight_max": 31.92327880859375,
      "activations/layer2_attention_weight_min": -29.74260711669922,
      "activations/layer3_attention_weight_max": 84.47003173828125,
      "activations/layer3_attention_weight_min": -84.42951202392578,
      "activations/layer4_attention_weight_max": 99.5820541381836,
      "activations/layer4_attention_weight_min": -93.1135482788086,
      "activations/layer5_attention_weight_max": 98.87374877929688,
      "activations/layer5_attention_weight_min": -85.75434875488281,
      "activations/layer6_attention_weight_max": 56.97282409667969,
      "activations/layer6_attention_weight_min": -51.680240631103516,
      "activations/layer7_attention_weight_max": 74.9343032836914,
      "activations/layer7_attention_weight_min": -72.27499389648438,
      "activations/layer8_attention_weight_max": 46.50093460083008,
      "activations/layer8_attention_weight_min": -45.179447174072266,
      "activations/layer9_attention_weight_max": 59.559234619140625,
      "activations/layer9_attention_weight_min": -52.621681213378906,
      "epoch": 6.64,
      "learning_rate": 0.00010824090909090907,
      "loss": 2.8098,
      "step": 114350
    },
    {
      "activations/layer0_attention_weight_max": 17.31331443786621,
      "activations/layer0_attention_weight_min": -14.71976089477539,
      "activations/layer10_attention_weight_max": 41.24473190307617,
      "activations/layer10_attention_weight_min": -35.72764205932617,
      "activations/layer11_attention_weight_max": 32.44582748413086,
      "activations/layer11_attention_weight_min": -29.692867279052734,
      "activations/layer12_attention_weight_max": 19.659137725830078,
      "activations/layer12_attention_weight_min": -24.627769470214844,
      "activations/layer13_attention_weight_max": 48.271724700927734,
      "activations/layer13_attention_weight_min": -33.1872673034668,
      "activations/layer14_attention_weight_max": 48.167362213134766,
      "activations/layer14_attention_weight_min": -29.970918655395508,
      "activations/layer15_attention_weight_max": 35.47214126586914,
      "activations/layer15_attention_weight_min": -29.278413772583008,
      "activations/layer16_attention_weight_max": 30.920331954956055,
      "activations/layer16_attention_weight_min": -27.896129608154297,
      "activations/layer17_attention_weight_max": 54.821102142333984,
      "activations/layer17_attention_weight_min": -44.02882766723633,
      "activations/layer18_attention_weight_max": 44.205963134765625,
      "activations/layer18_attention_weight_min": -35.49080276489258,
      "activations/layer19_attention_weight_max": 20.76911163330078,
      "activations/layer19_attention_weight_min": -20.851137161254883,
      "activations/layer1_attention_weight_max": 16.60356903076172,
      "activations/layer1_attention_weight_min": -15.18315315246582,
      "activations/layer20_attention_weight_max": 21.21257781982422,
      "activations/layer20_attention_weight_min": -22.799118041992188,
      "activations/layer21_attention_weight_max": 47.89134216308594,
      "activations/layer21_attention_weight_min": -24.0317325592041,
      "activations/layer22_attention_weight_max": 29.89825439453125,
      "activations/layer22_attention_weight_min": -23.39975357055664,
      "activations/layer23_attention_weight_max": 31.942798614501953,
      "activations/layer23_attention_weight_min": -20.704998016357422,
      "activations/layer2_attention_weight_max": 33.306541442871094,
      "activations/layer2_attention_weight_min": -31.408308029174805,
      "activations/layer3_attention_weight_max": 85.02421569824219,
      "activations/layer3_attention_weight_min": -90.06141662597656,
      "activations/layer4_attention_weight_max": 100.07978057861328,
      "activations/layer4_attention_weight_min": -98.8863754272461,
      "activations/layer5_attention_weight_max": 94.27568054199219,
      "activations/layer5_attention_weight_min": -90.42681884765625,
      "activations/layer6_attention_weight_max": 56.95695877075195,
      "activations/layer6_attention_weight_min": -54.41752624511719,
      "activations/layer7_attention_weight_max": 79.01963806152344,
      "activations/layer7_attention_weight_min": -68.91696166992188,
      "activations/layer8_attention_weight_max": 44.929561614990234,
      "activations/layer8_attention_weight_min": -42.79985427856445,
      "activations/layer9_attention_weight_max": 64.41178894042969,
      "activations/layer9_attention_weight_min": -50.93988037109375,
      "epoch": 6.65,
      "learning_rate": 0.00010822196969696969,
      "loss": 2.8146,
      "step": 114400
    },
    {
      "activations/layer0_attention_weight_max": 14.262818336486816,
      "activations/layer0_attention_weight_min": -14.637592315673828,
      "activations/layer10_attention_weight_max": 42.36427307128906,
      "activations/layer10_attention_weight_min": -38.88522720336914,
      "activations/layer11_attention_weight_max": 33.17017364501953,
      "activations/layer11_attention_weight_min": -30.222919464111328,
      "activations/layer12_attention_weight_max": 24.527339935302734,
      "activations/layer12_attention_weight_min": -23.685386657714844,
      "activations/layer13_attention_weight_max": 52.461055755615234,
      "activations/layer13_attention_weight_min": -37.23440170288086,
      "activations/layer14_attention_weight_max": 49.767757415771484,
      "activations/layer14_attention_weight_min": -34.23649978637695,
      "activations/layer15_attention_weight_max": 43.31805419921875,
      "activations/layer15_attention_weight_min": -31.917667388916016,
      "activations/layer16_attention_weight_max": 37.46266174316406,
      "activations/layer16_attention_weight_min": -28.738195419311523,
      "activations/layer17_attention_weight_max": 65.98893737792969,
      "activations/layer17_attention_weight_min": -46.44816970825195,
      "activations/layer18_attention_weight_max": 59.28293991088867,
      "activations/layer18_attention_weight_min": -37.8984489440918,
      "activations/layer19_attention_weight_max": 26.89536476135254,
      "activations/layer19_attention_weight_min": -20.984922409057617,
      "activations/layer1_attention_weight_max": 16.67502784729004,
      "activations/layer1_attention_weight_min": -16.211027145385742,
      "activations/layer20_attention_weight_max": 27.660430908203125,
      "activations/layer20_attention_weight_min": -21.859891891479492,
      "activations/layer21_attention_weight_max": 49.46455001831055,
      "activations/layer21_attention_weight_min": -23.917814254760742,
      "activations/layer22_attention_weight_max": 39.33744430541992,
      "activations/layer22_attention_weight_min": -24.86229705810547,
      "activations/layer23_attention_weight_max": 37.70280075073242,
      "activations/layer23_attention_weight_min": -19.41747283935547,
      "activations/layer2_attention_weight_max": 32.12822723388672,
      "activations/layer2_attention_weight_min": -29.91579246520996,
      "activations/layer3_attention_weight_max": 83.2109375,
      "activations/layer3_attention_weight_min": -88.59313201904297,
      "activations/layer4_attention_weight_max": 98.02896881103516,
      "activations/layer4_attention_weight_min": -96.55449676513672,
      "activations/layer5_attention_weight_max": 94.82286834716797,
      "activations/layer5_attention_weight_min": -87.41131591796875,
      "activations/layer6_attention_weight_max": 58.0919189453125,
      "activations/layer6_attention_weight_min": -54.81945037841797,
      "activations/layer7_attention_weight_max": 77.72233581542969,
      "activations/layer7_attention_weight_min": -73.42342376708984,
      "activations/layer8_attention_weight_max": 46.97198486328125,
      "activations/layer8_attention_weight_min": -45.23793411254883,
      "activations/layer9_attention_weight_max": 54.2244873046875,
      "activations/layer9_attention_weight_min": -51.708492279052734,
      "epoch": 6.65,
      "learning_rate": 0.00010820303030303029,
      "loss": 2.8176,
      "step": 114450
    },
    {
      "activations/layer0_attention_weight_max": 14.829729080200195,
      "activations/layer0_attention_weight_min": -14.805606842041016,
      "activations/layer10_attention_weight_max": 38.976688385009766,
      "activations/layer10_attention_weight_min": -36.66015625,
      "activations/layer11_attention_weight_max": 30.31072235107422,
      "activations/layer11_attention_weight_min": -29.47067642211914,
      "activations/layer12_attention_weight_max": 22.0225830078125,
      "activations/layer12_attention_weight_min": -26.079456329345703,
      "activations/layer13_attention_weight_max": 48.607200622558594,
      "activations/layer13_attention_weight_min": -34.42040252685547,
      "activations/layer14_attention_weight_max": 38.46204376220703,
      "activations/layer14_attention_weight_min": -30.755643844604492,
      "activations/layer15_attention_weight_max": 34.732757568359375,
      "activations/layer15_attention_weight_min": -28.47361946105957,
      "activations/layer16_attention_weight_max": 33.345699310302734,
      "activations/layer16_attention_weight_min": -26.612350463867188,
      "activations/layer17_attention_weight_max": 57.60132598876953,
      "activations/layer17_attention_weight_min": -42.24968719482422,
      "activations/layer18_attention_weight_max": 48.28964614868164,
      "activations/layer18_attention_weight_min": -35.18278503417969,
      "activations/layer19_attention_weight_max": 23.133358001708984,
      "activations/layer19_attention_weight_min": -21.034189224243164,
      "activations/layer1_attention_weight_max": 17.70557403564453,
      "activations/layer1_attention_weight_min": -15.063431739807129,
      "activations/layer20_attention_weight_max": 21.642187118530273,
      "activations/layer20_attention_weight_min": -22.547603607177734,
      "activations/layer21_attention_weight_max": 36.62114715576172,
      "activations/layer21_attention_weight_min": -22.564271926879883,
      "activations/layer22_attention_weight_max": 35.63400650024414,
      "activations/layer22_attention_weight_min": -25.448442459106445,
      "activations/layer23_attention_weight_max": 30.747928619384766,
      "activations/layer23_attention_weight_min": -19.441675186157227,
      "activations/layer2_attention_weight_max": 32.54026794433594,
      "activations/layer2_attention_weight_min": -29.419837951660156,
      "activations/layer3_attention_weight_max": 87.22554779052734,
      "activations/layer3_attention_weight_min": -86.40357208251953,
      "activations/layer4_attention_weight_max": 100.53832244873047,
      "activations/layer4_attention_weight_min": -99.79302215576172,
      "activations/layer5_attention_weight_max": 95.66929626464844,
      "activations/layer5_attention_weight_min": -86.64999389648438,
      "activations/layer6_attention_weight_max": 56.55265426635742,
      "activations/layer6_attention_weight_min": -54.146141052246094,
      "activations/layer7_attention_weight_max": 73.42652893066406,
      "activations/layer7_attention_weight_min": -71.04405975341797,
      "activations/layer8_attention_weight_max": 44.831512451171875,
      "activations/layer8_attention_weight_min": -44.00742721557617,
      "activations/layer9_attention_weight_max": 53.04425048828125,
      "activations/layer9_attention_weight_min": -48.819149017333984,
      "epoch": 6.65,
      "learning_rate": 0.0001081840909090909,
      "loss": 2.8062,
      "step": 114500
    },
    {
      "activations/layer0_attention_weight_max": 14.246305465698242,
      "activations/layer0_attention_weight_min": -14.403426170349121,
      "activations/layer10_attention_weight_max": 39.07729721069336,
      "activations/layer10_attention_weight_min": -35.288944244384766,
      "activations/layer11_attention_weight_max": 32.071327209472656,
      "activations/layer11_attention_weight_min": -30.377315521240234,
      "activations/layer12_attention_weight_max": 24.27280616760254,
      "activations/layer12_attention_weight_min": -24.1319580078125,
      "activations/layer13_attention_weight_max": 52.68857955932617,
      "activations/layer13_attention_weight_min": -32.0567741394043,
      "activations/layer14_attention_weight_max": 42.47229766845703,
      "activations/layer14_attention_weight_min": -29.100799560546875,
      "activations/layer15_attention_weight_max": 36.43415451049805,
      "activations/layer15_attention_weight_min": -30.40517234802246,
      "activations/layer16_attention_weight_max": 36.33975601196289,
      "activations/layer16_attention_weight_min": -26.744312286376953,
      "activations/layer17_attention_weight_max": 54.26962661743164,
      "activations/layer17_attention_weight_min": -42.68589401245117,
      "activations/layer18_attention_weight_max": 43.94846725463867,
      "activations/layer18_attention_weight_min": -37.038944244384766,
      "activations/layer19_attention_weight_max": 22.65151023864746,
      "activations/layer19_attention_weight_min": -21.90106201171875,
      "activations/layer1_attention_weight_max": 17.076902389526367,
      "activations/layer1_attention_weight_min": -15.883563995361328,
      "activations/layer20_attention_weight_max": 23.52692222595215,
      "activations/layer20_attention_weight_min": -21.54003143310547,
      "activations/layer21_attention_weight_max": 37.189453125,
      "activations/layer21_attention_weight_min": -24.05863380432129,
      "activations/layer22_attention_weight_max": 29.81828498840332,
      "activations/layer22_attention_weight_min": -23.614290237426758,
      "activations/layer23_attention_weight_max": 31.80561065673828,
      "activations/layer23_attention_weight_min": -19.924816131591797,
      "activations/layer2_attention_weight_max": 35.83304977416992,
      "activations/layer2_attention_weight_min": -32.64844512939453,
      "activations/layer3_attention_weight_max": 94.412353515625,
      "activations/layer3_attention_weight_min": -96.85382843017578,
      "activations/layer4_attention_weight_max": 105.8569564819336,
      "activations/layer4_attention_weight_min": -96.88611602783203,
      "activations/layer5_attention_weight_max": 98.53591918945312,
      "activations/layer5_attention_weight_min": -88.117919921875,
      "activations/layer6_attention_weight_max": 56.704410552978516,
      "activations/layer6_attention_weight_min": -52.209632873535156,
      "activations/layer7_attention_weight_max": 72.94326782226562,
      "activations/layer7_attention_weight_min": -70.68411254882812,
      "activations/layer8_attention_weight_max": 45.587158203125,
      "activations/layer8_attention_weight_min": -49.53580856323242,
      "activations/layer9_attention_weight_max": 54.97713851928711,
      "activations/layer9_attention_weight_min": -50.16280746459961,
      "epoch": 6.66,
      "learning_rate": 0.0001081651515151515,
      "loss": 2.808,
      "step": 114550
    },
    {
      "activations/layer0_attention_weight_max": 16.052865982055664,
      "activations/layer0_attention_weight_min": -14.631300926208496,
      "activations/layer10_attention_weight_max": 39.36690139770508,
      "activations/layer10_attention_weight_min": -36.62508773803711,
      "activations/layer11_attention_weight_max": 32.04679870605469,
      "activations/layer11_attention_weight_min": -30.92233657836914,
      "activations/layer12_attention_weight_max": 21.25558853149414,
      "activations/layer12_attention_weight_min": -25.023956298828125,
      "activations/layer13_attention_weight_max": 45.04182815551758,
      "activations/layer13_attention_weight_min": -31.626150131225586,
      "activations/layer14_attention_weight_max": 38.635215759277344,
      "activations/layer14_attention_weight_min": -31.870561599731445,
      "activations/layer15_attention_weight_max": 36.606483459472656,
      "activations/layer15_attention_weight_min": -30.797569274902344,
      "activations/layer16_attention_weight_max": 30.301586151123047,
      "activations/layer16_attention_weight_min": -26.880939483642578,
      "activations/layer17_attention_weight_max": 56.33949661254883,
      "activations/layer17_attention_weight_min": -41.33642578125,
      "activations/layer18_attention_weight_max": 46.48640441894531,
      "activations/layer18_attention_weight_min": -35.53675079345703,
      "activations/layer19_attention_weight_max": 24.664003372192383,
      "activations/layer19_attention_weight_min": -20.636259078979492,
      "activations/layer1_attention_weight_max": 16.873210906982422,
      "activations/layer1_attention_weight_min": -15.467950820922852,
      "activations/layer20_attention_weight_max": 22.039180755615234,
      "activations/layer20_attention_weight_min": -19.430448532104492,
      "activations/layer21_attention_weight_max": 41.804752349853516,
      "activations/layer21_attention_weight_min": -24.349979400634766,
      "activations/layer22_attention_weight_max": 37.266380310058594,
      "activations/layer22_attention_weight_min": -23.240264892578125,
      "activations/layer23_attention_weight_max": 28.581607818603516,
      "activations/layer23_attention_weight_min": -23.601581573486328,
      "activations/layer2_attention_weight_max": 32.33985900878906,
      "activations/layer2_attention_weight_min": -29.176185607910156,
      "activations/layer3_attention_weight_max": 85.30608367919922,
      "activations/layer3_attention_weight_min": -86.26709747314453,
      "activations/layer4_attention_weight_max": 101.70853424072266,
      "activations/layer4_attention_weight_min": -95.54651641845703,
      "activations/layer5_attention_weight_max": 99.2838134765625,
      "activations/layer5_attention_weight_min": -87.12144470214844,
      "activations/layer6_attention_weight_max": 55.68430709838867,
      "activations/layer6_attention_weight_min": -53.315574645996094,
      "activations/layer7_attention_weight_max": 72.80741119384766,
      "activations/layer7_attention_weight_min": -74.714599609375,
      "activations/layer8_attention_weight_max": 45.15644836425781,
      "activations/layer8_attention_weight_min": -47.50468063354492,
      "activations/layer9_attention_weight_max": 55.84161376953125,
      "activations/layer9_attention_weight_min": -51.21997833251953,
      "epoch": 6.66,
      "learning_rate": 0.0001081462121212121,
      "loss": 2.8204,
      "step": 114600
    },
    {
      "activations/layer0_attention_weight_max": 15.190927505493164,
      "activations/layer0_attention_weight_min": -13.96795654296875,
      "activations/layer10_attention_weight_max": 40.01399612426758,
      "activations/layer10_attention_weight_min": -35.71759796142578,
      "activations/layer11_attention_weight_max": 32.36268615722656,
      "activations/layer11_attention_weight_min": -30.133052825927734,
      "activations/layer12_attention_weight_max": 20.6970272064209,
      "activations/layer12_attention_weight_min": -26.64893341064453,
      "activations/layer13_attention_weight_max": 50.90151596069336,
      "activations/layer13_attention_weight_min": -36.351112365722656,
      "activations/layer14_attention_weight_max": 41.894676208496094,
      "activations/layer14_attention_weight_min": -30.392772674560547,
      "activations/layer15_attention_weight_max": 41.38267135620117,
      "activations/layer15_attention_weight_min": -29.023923873901367,
      "activations/layer16_attention_weight_max": 33.2496223449707,
      "activations/layer16_attention_weight_min": -28.321319580078125,
      "activations/layer17_attention_weight_max": 59.36124038696289,
      "activations/layer17_attention_weight_min": -42.05152130126953,
      "activations/layer18_attention_weight_max": 48.88020324707031,
      "activations/layer18_attention_weight_min": -39.26801681518555,
      "activations/layer19_attention_weight_max": 22.014751434326172,
      "activations/layer19_attention_weight_min": -19.798269271850586,
      "activations/layer1_attention_weight_max": 17.482507705688477,
      "activations/layer1_attention_weight_min": -15.17694091796875,
      "activations/layer20_attention_weight_max": 21.292173385620117,
      "activations/layer20_attention_weight_min": -21.801908493041992,
      "activations/layer21_attention_weight_max": 36.79692077636719,
      "activations/layer21_attention_weight_min": -24.820409774780273,
      "activations/layer22_attention_weight_max": 34.090232849121094,
      "activations/layer22_attention_weight_min": -24.4822940826416,
      "activations/layer23_attention_weight_max": 32.685707092285156,
      "activations/layer23_attention_weight_min": -18.31597137451172,
      "activations/layer2_attention_weight_max": 31.151962280273438,
      "activations/layer2_attention_weight_min": -28.300979614257812,
      "activations/layer3_attention_weight_max": 83.35002899169922,
      "activations/layer3_attention_weight_min": -85.83396911621094,
      "activations/layer4_attention_weight_max": 99.43836212158203,
      "activations/layer4_attention_weight_min": -93.43858337402344,
      "activations/layer5_attention_weight_max": 95.11318969726562,
      "activations/layer5_attention_weight_min": -85.91911315917969,
      "activations/layer6_attention_weight_max": 59.53262710571289,
      "activations/layer6_attention_weight_min": -54.026344299316406,
      "activations/layer7_attention_weight_max": 77.20685577392578,
      "activations/layer7_attention_weight_min": -71.79204559326172,
      "activations/layer8_attention_weight_max": 47.17415237426758,
      "activations/layer8_attention_weight_min": -43.79119873046875,
      "activations/layer9_attention_weight_max": 50.17104721069336,
      "activations/layer9_attention_weight_min": -48.835472106933594,
      "epoch": 6.66,
      "learning_rate": 0.00010812727272727272,
      "loss": 2.8234,
      "step": 114650
    },
    {
      "activations/layer0_attention_weight_max": 14.680670738220215,
      "activations/layer0_attention_weight_min": -14.059671401977539,
      "activations/layer10_attention_weight_max": 37.34889221191406,
      "activations/layer10_attention_weight_min": -38.73743438720703,
      "activations/layer11_attention_weight_max": 28.87819480895996,
      "activations/layer11_attention_weight_min": -27.85009002685547,
      "activations/layer12_attention_weight_max": 19.996416091918945,
      "activations/layer12_attention_weight_min": -27.38035774230957,
      "activations/layer13_attention_weight_max": 48.464595794677734,
      "activations/layer13_attention_weight_min": -37.597896575927734,
      "activations/layer14_attention_weight_max": 44.43517303466797,
      "activations/layer14_attention_weight_min": -33.28608703613281,
      "activations/layer15_attention_weight_max": 38.75439453125,
      "activations/layer15_attention_weight_min": -28.971973419189453,
      "activations/layer16_attention_weight_max": 35.74570083618164,
      "activations/layer16_attention_weight_min": -27.91227912902832,
      "activations/layer17_attention_weight_max": 58.37647247314453,
      "activations/layer17_attention_weight_min": -43.09977722167969,
      "activations/layer18_attention_weight_max": 48.88119888305664,
      "activations/layer18_attention_weight_min": -38.01301193237305,
      "activations/layer19_attention_weight_max": 21.092552185058594,
      "activations/layer19_attention_weight_min": -21.98375701904297,
      "activations/layer1_attention_weight_max": 16.475086212158203,
      "activations/layer1_attention_weight_min": -15.918180465698242,
      "activations/layer20_attention_weight_max": 25.445701599121094,
      "activations/layer20_attention_weight_min": -22.28076171875,
      "activations/layer21_attention_weight_max": 47.92298126220703,
      "activations/layer21_attention_weight_min": -29.79982566833496,
      "activations/layer22_attention_weight_max": 37.77298355102539,
      "activations/layer22_attention_weight_min": -25.081398010253906,
      "activations/layer23_attention_weight_max": 41.538902282714844,
      "activations/layer23_attention_weight_min": -22.30733299255371,
      "activations/layer2_attention_weight_max": 31.20999526977539,
      "activations/layer2_attention_weight_min": -28.253522872924805,
      "activations/layer3_attention_weight_max": 86.75849914550781,
      "activations/layer3_attention_weight_min": -85.08637237548828,
      "activations/layer4_attention_weight_max": 96.10600280761719,
      "activations/layer4_attention_weight_min": -94.26107025146484,
      "activations/layer5_attention_weight_max": 91.34375,
      "activations/layer5_attention_weight_min": -82.79060363769531,
      "activations/layer6_attention_weight_max": 55.078102111816406,
      "activations/layer6_attention_weight_min": -53.24587631225586,
      "activations/layer7_attention_weight_max": 73.51947784423828,
      "activations/layer7_attention_weight_min": -74.35293579101562,
      "activations/layer8_attention_weight_max": 43.420204162597656,
      "activations/layer8_attention_weight_min": -43.134098052978516,
      "activations/layer9_attention_weight_max": 50.87632369995117,
      "activations/layer9_attention_weight_min": -48.26079559326172,
      "epoch": 6.66,
      "learning_rate": 0.00010810833333333334,
      "loss": 2.7993,
      "step": 114700
    },
    {
      "activations/layer0_attention_weight_max": 16.496158599853516,
      "activations/layer0_attention_weight_min": -14.349668502807617,
      "activations/layer10_attention_weight_max": 36.67803192138672,
      "activations/layer10_attention_weight_min": -35.33575439453125,
      "activations/layer11_attention_weight_max": 29.057249069213867,
      "activations/layer11_attention_weight_min": -30.00778579711914,
      "activations/layer12_attention_weight_max": 19.69955062866211,
      "activations/layer12_attention_weight_min": -22.01630210876465,
      "activations/layer13_attention_weight_max": 40.64888381958008,
      "activations/layer13_attention_weight_min": -30.29705238342285,
      "activations/layer14_attention_weight_max": 46.85227584838867,
      "activations/layer14_attention_weight_min": -31.456172943115234,
      "activations/layer15_attention_weight_max": 35.86561965942383,
      "activations/layer15_attention_weight_min": -30.880075454711914,
      "activations/layer16_attention_weight_max": 31.66740608215332,
      "activations/layer16_attention_weight_min": -28.93584442138672,
      "activations/layer17_attention_weight_max": 54.722747802734375,
      "activations/layer17_attention_weight_min": -46.1795539855957,
      "activations/layer18_attention_weight_max": 51.606685638427734,
      "activations/layer18_attention_weight_min": -39.75161361694336,
      "activations/layer19_attention_weight_max": 20.34706687927246,
      "activations/layer19_attention_weight_min": -23.279327392578125,
      "activations/layer1_attention_weight_max": 17.744718551635742,
      "activations/layer1_attention_weight_min": -15.202195167541504,
      "activations/layer20_attention_weight_max": 21.79448127746582,
      "activations/layer20_attention_weight_min": -23.574033737182617,
      "activations/layer21_attention_weight_max": 37.58435821533203,
      "activations/layer21_attention_weight_min": -24.34185218811035,
      "activations/layer22_attention_weight_max": 31.69074821472168,
      "activations/layer22_attention_weight_min": -28.311859130859375,
      "activations/layer23_attention_weight_max": 30.180191040039062,
      "activations/layer23_attention_weight_min": -21.044322967529297,
      "activations/layer2_attention_weight_max": 32.818519592285156,
      "activations/layer2_attention_weight_min": -30.070045471191406,
      "activations/layer3_attention_weight_max": 82.34971618652344,
      "activations/layer3_attention_weight_min": -88.00411987304688,
      "activations/layer4_attention_weight_max": 99.74625396728516,
      "activations/layer4_attention_weight_min": -94.6200180053711,
      "activations/layer5_attention_weight_max": 95.71932220458984,
      "activations/layer5_attention_weight_min": -88.37410736083984,
      "activations/layer6_attention_weight_max": 56.95939254760742,
      "activations/layer6_attention_weight_min": -53.3441047668457,
      "activations/layer7_attention_weight_max": 74.1580581665039,
      "activations/layer7_attention_weight_min": -75.59459686279297,
      "activations/layer8_attention_weight_max": 45.475921630859375,
      "activations/layer8_attention_weight_min": -44.02622604370117,
      "activations/layer9_attention_weight_max": 53.12921905517578,
      "activations/layer9_attention_weight_min": -51.9130973815918,
      "epoch": 6.67,
      "learning_rate": 0.00010808939393939392,
      "loss": 2.8222,
      "step": 114750
    },
    {
      "activations/layer0_attention_weight_max": 15.76095962524414,
      "activations/layer0_attention_weight_min": -14.25121784210205,
      "activations/layer10_attention_weight_max": 38.52907180786133,
      "activations/layer10_attention_weight_min": -37.514617919921875,
      "activations/layer11_attention_weight_max": 33.329463958740234,
      "activations/layer11_attention_weight_min": -30.851125717163086,
      "activations/layer12_attention_weight_max": 18.92631721496582,
      "activations/layer12_attention_weight_min": -25.64326286315918,
      "activations/layer13_attention_weight_max": 51.99904251098633,
      "activations/layer13_attention_weight_min": -35.381019592285156,
      "activations/layer14_attention_weight_max": 52.591583251953125,
      "activations/layer14_attention_weight_min": -31.48065757751465,
      "activations/layer15_attention_weight_max": 38.75054168701172,
      "activations/layer15_attention_weight_min": -31.03043556213379,
      "activations/layer16_attention_weight_max": 36.760562896728516,
      "activations/layer16_attention_weight_min": -26.780637741088867,
      "activations/layer17_attention_weight_max": 53.81974792480469,
      "activations/layer17_attention_weight_min": -43.28925704956055,
      "activations/layer18_attention_weight_max": 53.11228561401367,
      "activations/layer18_attention_weight_min": -35.62298583984375,
      "activations/layer19_attention_weight_max": 24.480850219726562,
      "activations/layer19_attention_weight_min": -22.125946044921875,
      "activations/layer1_attention_weight_max": 16.311540603637695,
      "activations/layer1_attention_weight_min": -14.946057319641113,
      "activations/layer20_attention_weight_max": 22.268041610717773,
      "activations/layer20_attention_weight_min": -22.9252986907959,
      "activations/layer21_attention_weight_max": 43.67355728149414,
      "activations/layer21_attention_weight_min": -23.799819946289062,
      "activations/layer22_attention_weight_max": 32.968963623046875,
      "activations/layer22_attention_weight_min": -25.547422409057617,
      "activations/layer23_attention_weight_max": 34.73626708984375,
      "activations/layer23_attention_weight_min": -21.844444274902344,
      "activations/layer2_attention_weight_max": 33.006187438964844,
      "activations/layer2_attention_weight_min": -31.559232711791992,
      "activations/layer3_attention_weight_max": 86.07771301269531,
      "activations/layer3_attention_weight_min": -91.4615478515625,
      "activations/layer4_attention_weight_max": 105.26934051513672,
      "activations/layer4_attention_weight_min": -96.0758056640625,
      "activations/layer5_attention_weight_max": 100.42105102539062,
      "activations/layer5_attention_weight_min": -84.99085998535156,
      "activations/layer6_attention_weight_max": 59.47649002075195,
      "activations/layer6_attention_weight_min": -50.93128204345703,
      "activations/layer7_attention_weight_max": 71.92957305908203,
      "activations/layer7_attention_weight_min": -71.19706726074219,
      "activations/layer8_attention_weight_max": 45.36588668823242,
      "activations/layer8_attention_weight_min": -44.34503936767578,
      "activations/layer9_attention_weight_max": 48.16655731201172,
      "activations/layer9_attention_weight_min": -46.21807098388672,
      "epoch": 6.67,
      "learning_rate": 0.00010807045454545454,
      "loss": 2.8057,
      "step": 114800
    },
    {
      "activations/layer0_attention_weight_max": 15.743752479553223,
      "activations/layer0_attention_weight_min": -14.692998886108398,
      "activations/layer10_attention_weight_max": 38.109283447265625,
      "activations/layer10_attention_weight_min": -35.78437805175781,
      "activations/layer11_attention_weight_max": 31.722217559814453,
      "activations/layer11_attention_weight_min": -32.53533172607422,
      "activations/layer12_attention_weight_max": 24.996408462524414,
      "activations/layer12_attention_weight_min": -23.220325469970703,
      "activations/layer13_attention_weight_max": 42.77632522583008,
      "activations/layer13_attention_weight_min": -34.47622299194336,
      "activations/layer14_attention_weight_max": 47.10600280761719,
      "activations/layer14_attention_weight_min": -31.404659271240234,
      "activations/layer15_attention_weight_max": 36.06085968017578,
      "activations/layer15_attention_weight_min": -31.117286682128906,
      "activations/layer16_attention_weight_max": 32.58829116821289,
      "activations/layer16_attention_weight_min": -27.640365600585938,
      "activations/layer17_attention_weight_max": 56.36377716064453,
      "activations/layer17_attention_weight_min": -42.35188674926758,
      "activations/layer18_attention_weight_max": 48.03705978393555,
      "activations/layer18_attention_weight_min": -35.59251022338867,
      "activations/layer19_attention_weight_max": 23.96535301208496,
      "activations/layer19_attention_weight_min": -21.31751823425293,
      "activations/layer1_attention_weight_max": 17.751436233520508,
      "activations/layer1_attention_weight_min": -17.575843811035156,
      "activations/layer20_attention_weight_max": 23.64979362487793,
      "activations/layer20_attention_weight_min": -22.23090171813965,
      "activations/layer21_attention_weight_max": 38.840782165527344,
      "activations/layer21_attention_weight_min": -25.804405212402344,
      "activations/layer22_attention_weight_max": 34.95920181274414,
      "activations/layer22_attention_weight_min": -24.651948928833008,
      "activations/layer23_attention_weight_max": 33.80227279663086,
      "activations/layer23_attention_weight_min": -20.814659118652344,
      "activations/layer2_attention_weight_max": 31.978137969970703,
      "activations/layer2_attention_weight_min": -30.226383209228516,
      "activations/layer3_attention_weight_max": 85.26710510253906,
      "activations/layer3_attention_weight_min": -86.96420288085938,
      "activations/layer4_attention_weight_max": 103.08575439453125,
      "activations/layer4_attention_weight_min": -96.03117370605469,
      "activations/layer5_attention_weight_max": 94.65419006347656,
      "activations/layer5_attention_weight_min": -89.5386962890625,
      "activations/layer6_attention_weight_max": 56.63327407836914,
      "activations/layer6_attention_weight_min": -54.500553131103516,
      "activations/layer7_attention_weight_max": 76.41035461425781,
      "activations/layer7_attention_weight_min": -71.57794952392578,
      "activations/layer8_attention_weight_max": 46.236473083496094,
      "activations/layer8_attention_weight_min": -44.5758171081543,
      "activations/layer9_attention_weight_max": 49.24969482421875,
      "activations/layer9_attention_weight_min": -48.829612731933594,
      "epoch": 6.67,
      "learning_rate": 0.00010805189393939393,
      "loss": 2.8071,
      "step": 114850
    },
    {
      "activations/layer0_attention_weight_max": 15.426827430725098,
      "activations/layer0_attention_weight_min": -14.870213508605957,
      "activations/layer10_attention_weight_max": 41.92832565307617,
      "activations/layer10_attention_weight_min": -36.756591796875,
      "activations/layer11_attention_weight_max": 33.297454833984375,
      "activations/layer11_attention_weight_min": -31.125234603881836,
      "activations/layer12_attention_weight_max": 24.38864517211914,
      "activations/layer12_attention_weight_min": -25.379377365112305,
      "activations/layer13_attention_weight_max": 46.45191955566406,
      "activations/layer13_attention_weight_min": -33.90774154663086,
      "activations/layer14_attention_weight_max": 43.79788589477539,
      "activations/layer14_attention_weight_min": -31.70791244506836,
      "activations/layer15_attention_weight_max": 38.018348693847656,
      "activations/layer15_attention_weight_min": -30.083362579345703,
      "activations/layer16_attention_weight_max": 35.65678024291992,
      "activations/layer16_attention_weight_min": -28.61254119873047,
      "activations/layer17_attention_weight_max": 59.51005935668945,
      "activations/layer17_attention_weight_min": -47.113121032714844,
      "activations/layer18_attention_weight_max": 56.3703498840332,
      "activations/layer18_attention_weight_min": -40.55485534667969,
      "activations/layer19_attention_weight_max": 21.828369140625,
      "activations/layer19_attention_weight_min": -22.788190841674805,
      "activations/layer1_attention_weight_max": 17.56854248046875,
      "activations/layer1_attention_weight_min": -15.969775199890137,
      "activations/layer20_attention_weight_max": 26.061912536621094,
      "activations/layer20_attention_weight_min": -21.640291213989258,
      "activations/layer21_attention_weight_max": 43.40659713745117,
      "activations/layer21_attention_weight_min": -27.028636932373047,
      "activations/layer22_attention_weight_max": 34.676658630371094,
      "activations/layer22_attention_weight_min": -25.56688117980957,
      "activations/layer23_attention_weight_max": 32.84490203857422,
      "activations/layer23_attention_weight_min": -19.731475830078125,
      "activations/layer2_attention_weight_max": 34.86554718017578,
      "activations/layer2_attention_weight_min": -32.12905502319336,
      "activations/layer3_attention_weight_max": 90.80097198486328,
      "activations/layer3_attention_weight_min": -93.9531021118164,
      "activations/layer4_attention_weight_max": 102.9327392578125,
      "activations/layer4_attention_weight_min": -97.63561248779297,
      "activations/layer5_attention_weight_max": 96.433837890625,
      "activations/layer5_attention_weight_min": -87.67122650146484,
      "activations/layer6_attention_weight_max": 60.01573181152344,
      "activations/layer6_attention_weight_min": -55.26531982421875,
      "activations/layer7_attention_weight_max": 76.2665786743164,
      "activations/layer7_attention_weight_min": -71.64427947998047,
      "activations/layer8_attention_weight_max": 46.93413543701172,
      "activations/layer8_attention_weight_min": -46.706214904785156,
      "activations/layer9_attention_weight_max": 55.332271575927734,
      "activations/layer9_attention_weight_min": -52.481929779052734,
      "epoch": 6.68,
      "learning_rate": 0.00010803295454545455,
      "loss": 2.8113,
      "step": 114900
    },
    {
      "activations/layer0_attention_weight_max": 16.00348472595215,
      "activations/layer0_attention_weight_min": -14.735442161560059,
      "activations/layer10_attention_weight_max": 36.98094940185547,
      "activations/layer10_attention_weight_min": -36.17735290527344,
      "activations/layer11_attention_weight_max": 29.85162925720215,
      "activations/layer11_attention_weight_min": -30.047740936279297,
      "activations/layer12_attention_weight_max": 23.27271842956543,
      "activations/layer12_attention_weight_min": -26.611406326293945,
      "activations/layer13_attention_weight_max": 57.76641845703125,
      "activations/layer13_attention_weight_min": -36.48419952392578,
      "activations/layer14_attention_weight_max": 58.93629455566406,
      "activations/layer14_attention_weight_min": -34.82529830932617,
      "activations/layer15_attention_weight_max": 40.98310089111328,
      "activations/layer15_attention_weight_min": -30.32991600036621,
      "activations/layer16_attention_weight_max": 39.121185302734375,
      "activations/layer16_attention_weight_min": -29.350296020507812,
      "activations/layer17_attention_weight_max": 63.85036849975586,
      "activations/layer17_attention_weight_min": -43.18738555908203,
      "activations/layer18_attention_weight_max": 57.002845764160156,
      "activations/layer18_attention_weight_min": -37.08402633666992,
      "activations/layer19_attention_weight_max": 21.857534408569336,
      "activations/layer19_attention_weight_min": -19.982786178588867,
      "activations/layer1_attention_weight_max": 15.95088005065918,
      "activations/layer1_attention_weight_min": -14.982612609863281,
      "activations/layer20_attention_weight_max": 26.919525146484375,
      "activations/layer20_attention_weight_min": -24.37018394470215,
      "activations/layer21_attention_weight_max": 42.59053039550781,
      "activations/layer21_attention_weight_min": -24.175437927246094,
      "activations/layer22_attention_weight_max": 37.30268096923828,
      "activations/layer22_attention_weight_min": -25.84306526184082,
      "activations/layer23_attention_weight_max": 35.85662841796875,
      "activations/layer23_attention_weight_min": -21.239473342895508,
      "activations/layer2_attention_weight_max": 30.994556427001953,
      "activations/layer2_attention_weight_min": -29.234472274780273,
      "activations/layer3_attention_weight_max": 87.35782623291016,
      "activations/layer3_attention_weight_min": -85.88809204101562,
      "activations/layer4_attention_weight_max": 100.4919662475586,
      "activations/layer4_attention_weight_min": -93.11043548583984,
      "activations/layer5_attention_weight_max": 95.71190643310547,
      "activations/layer5_attention_weight_min": -85.70333099365234,
      "activations/layer6_attention_weight_max": 56.83998489379883,
      "activations/layer6_attention_weight_min": -52.5926628112793,
      "activations/layer7_attention_weight_max": 72.463623046875,
      "activations/layer7_attention_weight_min": -73.62255096435547,
      "activations/layer8_attention_weight_max": 44.25691604614258,
      "activations/layer8_attention_weight_min": -44.76634216308594,
      "activations/layer9_attention_weight_max": 50.52155303955078,
      "activations/layer9_attention_weight_min": -49.702579498291016,
      "epoch": 6.68,
      "learning_rate": 0.00010801401515151513,
      "loss": 2.7961,
      "step": 114950
    },
    {
      "activations/layer0_attention_weight_max": 15.033792495727539,
      "activations/layer0_attention_weight_min": -15.007547378540039,
      "activations/layer10_attention_weight_max": 37.09625244140625,
      "activations/layer10_attention_weight_min": -34.346920013427734,
      "activations/layer11_attention_weight_max": 30.42205047607422,
      "activations/layer11_attention_weight_min": -28.606300354003906,
      "activations/layer12_attention_weight_max": 22.43553352355957,
      "activations/layer12_attention_weight_min": -27.127107620239258,
      "activations/layer13_attention_weight_max": 44.6407585144043,
      "activations/layer13_attention_weight_min": -33.48371887207031,
      "activations/layer14_attention_weight_max": 42.42325210571289,
      "activations/layer14_attention_weight_min": -31.52489471435547,
      "activations/layer15_attention_weight_max": 40.232017517089844,
      "activations/layer15_attention_weight_min": -31.36153793334961,
      "activations/layer16_attention_weight_max": 33.95683670043945,
      "activations/layer16_attention_weight_min": -26.143918991088867,
      "activations/layer17_attention_weight_max": 58.59022903442383,
      "activations/layer17_attention_weight_min": -43.01826477050781,
      "activations/layer18_attention_weight_max": 53.699951171875,
      "activations/layer18_attention_weight_min": -34.579925537109375,
      "activations/layer19_attention_weight_max": 24.38081932067871,
      "activations/layer19_attention_weight_min": -21.96320915222168,
      "activations/layer1_attention_weight_max": 17.041467666625977,
      "activations/layer1_attention_weight_min": -14.72278881072998,
      "activations/layer20_attention_weight_max": 25.48482894897461,
      "activations/layer20_attention_weight_min": -21.04777717590332,
      "activations/layer21_attention_weight_max": 41.47528076171875,
      "activations/layer21_attention_weight_min": -24.445083618164062,
      "activations/layer22_attention_weight_max": 33.8995246887207,
      "activations/layer22_attention_weight_min": -24.988889694213867,
      "activations/layer23_attention_weight_max": 33.17102813720703,
      "activations/layer23_attention_weight_min": -21.585355758666992,
      "activations/layer2_attention_weight_max": 34.45895004272461,
      "activations/layer2_attention_weight_min": -29.681020736694336,
      "activations/layer3_attention_weight_max": 84.40853118896484,
      "activations/layer3_attention_weight_min": -85.39713287353516,
      "activations/layer4_attention_weight_max": 100.02024841308594,
      "activations/layer4_attention_weight_min": -92.38005065917969,
      "activations/layer5_attention_weight_max": 95.42283630371094,
      "activations/layer5_attention_weight_min": -88.42897033691406,
      "activations/layer6_attention_weight_max": 57.70952224731445,
      "activations/layer6_attention_weight_min": -54.25368881225586,
      "activations/layer7_attention_weight_max": 75.1833724975586,
      "activations/layer7_attention_weight_min": -70.03645324707031,
      "activations/layer8_attention_weight_max": 43.72072982788086,
      "activations/layer8_attention_weight_min": -44.013423919677734,
      "activations/layer9_attention_weight_max": 48.470638275146484,
      "activations/layer9_attention_weight_min": -51.090553283691406,
      "epoch": 6.68,
      "learning_rate": 0.00010799507575757575,
      "loss": 2.8138,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_loss": 2.767578125,
      "eval_runtime": 9.0621,
      "eval_samples_per_second": 473.84,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_openwebtext_loss": 2.767578125,
      "eval_openwebtext_ppl": 15.920030957803608,
      "eval_openwebtext_runtime": 9.0621,
      "eval_openwebtext_samples_per_second": 473.84,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 2.0256,
      "eval_wikitext_samples_per_second": 225.121,
      "step": 115000
    },
    {
      "epoch": 6.68,
      "eval_lambada_loss": 2.74609375,
      "eval_lambada_ppl": 15.581647041956762,
      "eval_lambada_runtime": 9.5679,
      "eval_lambada_samples_per_second": 508.889,
      "step": 115000
    },
    {
      "activations/layer0_attention_weight_max": 14.250335693359375,
      "activations/layer0_attention_weight_min": -14.53724479675293,
      "activations/layer10_attention_weight_max": 37.31319808959961,
      "activations/layer10_attention_weight_min": -39.086734771728516,
      "activations/layer11_attention_weight_max": 28.860624313354492,
      "activations/layer11_attention_weight_min": -30.524188995361328,
      "activations/layer12_attention_weight_max": 18.88958168029785,
      "activations/layer12_attention_weight_min": -29.89261245727539,
      "activations/layer13_attention_weight_max": 57.59657287597656,
      "activations/layer13_attention_weight_min": -40.946041107177734,
      "activations/layer14_attention_weight_max": 61.43240737915039,
      "activations/layer14_attention_weight_min": -41.32542037963867,
      "activations/layer15_attention_weight_max": 42.92909240722656,
      "activations/layer15_attention_weight_min": -31.534461975097656,
      "activations/layer16_attention_weight_max": 32.37189865112305,
      "activations/layer16_attention_weight_min": -27.454748153686523,
      "activations/layer17_attention_weight_max": 55.09428024291992,
      "activations/layer17_attention_weight_min": -44.23565673828125,
      "activations/layer18_attention_weight_max": 48.53035354614258,
      "activations/layer18_attention_weight_min": -36.913787841796875,
      "activations/layer19_attention_weight_max": 22.84105682373047,
      "activations/layer19_attention_weight_min": -22.535030364990234,
      "activations/layer1_attention_weight_max": 16.659292221069336,
      "activations/layer1_attention_weight_min": -15.228869438171387,
      "activations/layer20_attention_weight_max": 25.09945297241211,
      "activations/layer20_attention_weight_min": -22.60794448852539,
      "activations/layer21_attention_weight_max": 43.27312088012695,
      "activations/layer21_attention_weight_min": -25.481542587280273,
      "activations/layer22_attention_weight_max": 30.76920509338379,
      "activations/layer22_attention_weight_min": -26.71761131286621,
      "activations/layer23_attention_weight_max": 33.132423400878906,
      "activations/layer23_attention_weight_min": -23.079626083374023,
      "activations/layer2_attention_weight_max": 31.14936065673828,
      "activations/layer2_attention_weight_min": -28.198095321655273,
      "activations/layer3_attention_weight_max": 84.32404327392578,
      "activations/layer3_attention_weight_min": -86.88304138183594,
      "activations/layer4_attention_weight_max": 101.3388671875,
      "activations/layer4_attention_weight_min": -92.78972625732422,
      "activations/layer5_attention_weight_max": 98.09072875976562,
      "activations/layer5_attention_weight_min": -84.40778350830078,
      "activations/layer6_attention_weight_max": 58.333656311035156,
      "activations/layer6_attention_weight_min": -51.53684997558594,
      "activations/layer7_attention_weight_max": 72.57270050048828,
      "activations/layer7_attention_weight_min": -69.2918930053711,
      "activations/layer8_attention_weight_max": 44.58745574951172,
      "activations/layer8_attention_weight_min": -44.83864212036133,
      "activations/layer9_attention_weight_max": 62.118648529052734,
      "activations/layer9_attention_weight_min": -48.118953704833984,
      "epoch": 6.69,
      "learning_rate": 0.00010797613636363636,
      "loss": 2.8145,
      "step": 115050
    },
    {
      "activations/layer0_attention_weight_max": 17.051742553710938,
      "activations/layer0_attention_weight_min": -14.568218231201172,
      "activations/layer10_attention_weight_max": 41.95174026489258,
      "activations/layer10_attention_weight_min": -37.363685607910156,
      "activations/layer11_attention_weight_max": 38.53404998779297,
      "activations/layer11_attention_weight_min": -29.547134399414062,
      "activations/layer12_attention_weight_max": 22.926706314086914,
      "activations/layer12_attention_weight_min": -24.301776885986328,
      "activations/layer13_attention_weight_max": 42.70328140258789,
      "activations/layer13_attention_weight_min": -29.325639724731445,
      "activations/layer14_attention_weight_max": 48.92559814453125,
      "activations/layer14_attention_weight_min": -30.420804977416992,
      "activations/layer15_attention_weight_max": 35.14270782470703,
      "activations/layer15_attention_weight_min": -29.265893936157227,
      "activations/layer16_attention_weight_max": 36.81650924682617,
      "activations/layer16_attention_weight_min": -26.44342803955078,
      "activations/layer17_attention_weight_max": 56.199588775634766,
      "activations/layer17_attention_weight_min": -38.66009521484375,
      "activations/layer18_attention_weight_max": 45.31763458251953,
      "activations/layer18_attention_weight_min": -33.98320770263672,
      "activations/layer19_attention_weight_max": 26.275053024291992,
      "activations/layer19_attention_weight_min": -19.48904037475586,
      "activations/layer1_attention_weight_max": 16.416423797607422,
      "activations/layer1_attention_weight_min": -17.028684616088867,
      "activations/layer20_attention_weight_max": 20.858488082885742,
      "activations/layer20_attention_weight_min": -19.336170196533203,
      "activations/layer21_attention_weight_max": 40.04241943359375,
      "activations/layer21_attention_weight_min": -24.056859970092773,
      "activations/layer22_attention_weight_max": 35.472007751464844,
      "activations/layer22_attention_weight_min": -22.69808006286621,
      "activations/layer23_attention_weight_max": 33.00331497192383,
      "activations/layer23_attention_weight_min": -20.883224487304688,
      "activations/layer2_attention_weight_max": 32.77701950073242,
      "activations/layer2_attention_weight_min": -30.140865325927734,
      "activations/layer3_attention_weight_max": 86.66547393798828,
      "activations/layer3_attention_weight_min": -89.6389389038086,
      "activations/layer4_attention_weight_max": 100.79277038574219,
      "activations/layer4_attention_weight_min": -97.65953063964844,
      "activations/layer5_attention_weight_max": 96.36276245117188,
      "activations/layer5_attention_weight_min": -90.41336059570312,
      "activations/layer6_attention_weight_max": 59.99973678588867,
      "activations/layer6_attention_weight_min": -53.58024978637695,
      "activations/layer7_attention_weight_max": 91.65562438964844,
      "activations/layer7_attention_weight_min": -71.2212142944336,
      "activations/layer8_attention_weight_max": 51.30968475341797,
      "activations/layer8_attention_weight_min": -46.99934768676758,
      "activations/layer9_attention_weight_max": 73.96790313720703,
      "activations/layer9_attention_weight_min": -51.99727249145508,
      "epoch": 6.69,
      "learning_rate": 0.00010795719696969695,
      "loss": 2.8261,
      "step": 115100
    },
    {
      "activations/layer0_attention_weight_max": 15.293719291687012,
      "activations/layer0_attention_weight_min": -14.333827018737793,
      "activations/layer10_attention_weight_max": 41.43779373168945,
      "activations/layer10_attention_weight_min": -36.922447204589844,
      "activations/layer11_attention_weight_max": 31.20380210876465,
      "activations/layer11_attention_weight_min": -30.420555114746094,
      "activations/layer12_attention_weight_max": 22.983577728271484,
      "activations/layer12_attention_weight_min": -24.705482482910156,
      "activations/layer13_attention_weight_max": 47.43730163574219,
      "activations/layer13_attention_weight_min": -34.635921478271484,
      "activations/layer14_attention_weight_max": 45.95244598388672,
      "activations/layer14_attention_weight_min": -32.425506591796875,
      "activations/layer15_attention_weight_max": 35.19424819946289,
      "activations/layer15_attention_weight_min": -30.731014251708984,
      "activations/layer16_attention_weight_max": 39.0799674987793,
      "activations/layer16_attention_weight_min": -27.195072174072266,
      "activations/layer17_attention_weight_max": 56.2022705078125,
      "activations/layer17_attention_weight_min": -46.69942092895508,
      "activations/layer18_attention_weight_max": 51.2702751159668,
      "activations/layer18_attention_weight_min": -38.67009735107422,
      "activations/layer19_attention_weight_max": 21.700660705566406,
      "activations/layer19_attention_weight_min": -20.370534896850586,
      "activations/layer1_attention_weight_max": 18.233732223510742,
      "activations/layer1_attention_weight_min": -15.803747177124023,
      "activations/layer20_attention_weight_max": 24.44474983215332,
      "activations/layer20_attention_weight_min": -21.66841697692871,
      "activations/layer21_attention_weight_max": 39.531005859375,
      "activations/layer21_attention_weight_min": -26.970867156982422,
      "activations/layer22_attention_weight_max": 34.23823928833008,
      "activations/layer22_attention_weight_min": -25.608829498291016,
      "activations/layer23_attention_weight_max": 33.023399353027344,
      "activations/layer23_attention_weight_min": -19.676069259643555,
      "activations/layer2_attention_weight_max": 32.474029541015625,
      "activations/layer2_attention_weight_min": -30.08242416381836,
      "activations/layer3_attention_weight_max": 83.98841094970703,
      "activations/layer3_attention_weight_min": -87.6148681640625,
      "activations/layer4_attention_weight_max": 96.65483856201172,
      "activations/layer4_attention_weight_min": -91.13997650146484,
      "activations/layer5_attention_weight_max": 91.44109344482422,
      "activations/layer5_attention_weight_min": -87.08268737792969,
      "activations/layer6_attention_weight_max": 57.701332092285156,
      "activations/layer6_attention_weight_min": -53.88554763793945,
      "activations/layer7_attention_weight_max": 74.1488265991211,
      "activations/layer7_attention_weight_min": -71.2762451171875,
      "activations/layer8_attention_weight_max": 46.233638763427734,
      "activations/layer8_attention_weight_min": -46.765472412109375,
      "activations/layer9_attention_weight_max": 52.72481918334961,
      "activations/layer9_attention_weight_min": -49.559730529785156,
      "epoch": 6.69,
      "learning_rate": 0.00010793825757575757,
      "loss": 2.8268,
      "step": 115150
    },
    {
      "activations/layer0_attention_weight_max": 16.15688133239746,
      "activations/layer0_attention_weight_min": -15.222972869873047,
      "activations/layer10_attention_weight_max": 40.78167724609375,
      "activations/layer10_attention_weight_min": -37.32622528076172,
      "activations/layer11_attention_weight_max": 30.589988708496094,
      "activations/layer11_attention_weight_min": -31.066429138183594,
      "activations/layer12_attention_weight_max": 24.68921661376953,
      "activations/layer12_attention_weight_min": -24.026569366455078,
      "activations/layer13_attention_weight_max": 42.21356964111328,
      "activations/layer13_attention_weight_min": -31.05655860900879,
      "activations/layer14_attention_weight_max": 39.88451385498047,
      "activations/layer14_attention_weight_min": -28.82578468322754,
      "activations/layer15_attention_weight_max": 33.77085494995117,
      "activations/layer15_attention_weight_min": -31.00391960144043,
      "activations/layer16_attention_weight_max": 30.751310348510742,
      "activations/layer16_attention_weight_min": -27.253252029418945,
      "activations/layer17_attention_weight_max": 54.665828704833984,
      "activations/layer17_attention_weight_min": -42.8871955871582,
      "activations/layer18_attention_weight_max": 43.70915222167969,
      "activations/layer18_attention_weight_min": -36.62553024291992,
      "activations/layer19_attention_weight_max": 22.767629623413086,
      "activations/layer19_attention_weight_min": -21.5349178314209,
      "activations/layer1_attention_weight_max": 17.835329055786133,
      "activations/layer1_attention_weight_min": -16.404882431030273,
      "activations/layer20_attention_weight_max": 22.822538375854492,
      "activations/layer20_attention_weight_min": -22.831850051879883,
      "activations/layer21_attention_weight_max": 39.29848861694336,
      "activations/layer21_attention_weight_min": -27.41800880432129,
      "activations/layer22_attention_weight_max": 30.26744270324707,
      "activations/layer22_attention_weight_min": -25.473344802856445,
      "activations/layer23_attention_weight_max": 30.514041900634766,
      "activations/layer23_attention_weight_min": -20.89271354675293,
      "activations/layer2_attention_weight_max": 34.944740295410156,
      "activations/layer2_attention_weight_min": -31.965620040893555,
      "activations/layer3_attention_weight_max": 86.92015075683594,
      "activations/layer3_attention_weight_min": -89.17626953125,
      "activations/layer4_attention_weight_max": 100.6270523071289,
      "activations/layer4_attention_weight_min": -96.61966705322266,
      "activations/layer5_attention_weight_max": 91.71508026123047,
      "activations/layer5_attention_weight_min": -87.85253143310547,
      "activations/layer6_attention_weight_max": 58.60167694091797,
      "activations/layer6_attention_weight_min": -54.94206619262695,
      "activations/layer7_attention_weight_max": 73.7806396484375,
      "activations/layer7_attention_weight_min": -70.47858428955078,
      "activations/layer8_attention_weight_max": 44.72133255004883,
      "activations/layer8_attention_weight_min": -45.27425765991211,
      "activations/layer9_attention_weight_max": 52.321834564208984,
      "activations/layer9_attention_weight_min": -50.49163055419922,
      "epoch": 6.69,
      "learning_rate": 0.00010791931818181817,
      "loss": 2.8276,
      "step": 115200
    },
    {
      "activations/layer0_attention_weight_max": 14.766522407531738,
      "activations/layer0_attention_weight_min": -14.55734920501709,
      "activations/layer10_attention_weight_max": 37.55512237548828,
      "activations/layer10_attention_weight_min": -34.853424072265625,
      "activations/layer11_attention_weight_max": 30.83747100830078,
      "activations/layer11_attention_weight_min": -29.239154815673828,
      "activations/layer12_attention_weight_max": 18.35157585144043,
      "activations/layer12_attention_weight_min": -25.03577423095703,
      "activations/layer13_attention_weight_max": 41.81696701049805,
      "activations/layer13_attention_weight_min": -33.135536193847656,
      "activations/layer14_attention_weight_max": 39.51835250854492,
      "activations/layer14_attention_weight_min": -31.947280883789062,
      "activations/layer15_attention_weight_max": 35.89570999145508,
      "activations/layer15_attention_weight_min": -32.56410598754883,
      "activations/layer16_attention_weight_max": 29.830249786376953,
      "activations/layer16_attention_weight_min": -27.531587600708008,
      "activations/layer17_attention_weight_max": 53.28857421875,
      "activations/layer17_attention_weight_min": -40.95695114135742,
      "activations/layer18_attention_weight_max": 47.743228912353516,
      "activations/layer18_attention_weight_min": -37.17601776123047,
      "activations/layer19_attention_weight_max": 22.94698715209961,
      "activations/layer19_attention_weight_min": -19.077421188354492,
      "activations/layer1_attention_weight_max": 16.75243377685547,
      "activations/layer1_attention_weight_min": -14.345125198364258,
      "activations/layer20_attention_weight_max": 22.279783248901367,
      "activations/layer20_attention_weight_min": -23.383005142211914,
      "activations/layer21_attention_weight_max": 35.581932067871094,
      "activations/layer21_attention_weight_min": -27.61751365661621,
      "activations/layer22_attention_weight_max": 33.550235748291016,
      "activations/layer22_attention_weight_min": -25.277191162109375,
      "activations/layer23_attention_weight_max": 32.3746337890625,
      "activations/layer23_attention_weight_min": -21.716854095458984,
      "activations/layer2_attention_weight_max": 31.477039337158203,
      "activations/layer2_attention_weight_min": -30.239444732666016,
      "activations/layer3_attention_weight_max": 83.94361114501953,
      "activations/layer3_attention_weight_min": -86.5389404296875,
      "activations/layer4_attention_weight_max": 102.31912231445312,
      "activations/layer4_attention_weight_min": -92.56652069091797,
      "activations/layer5_attention_weight_max": 94.10820007324219,
      "activations/layer5_attention_weight_min": -88.6316146850586,
      "activations/layer6_attention_weight_max": 59.98878860473633,
      "activations/layer6_attention_weight_min": -55.0366325378418,
      "activations/layer7_attention_weight_max": 72.0794906616211,
      "activations/layer7_attention_weight_min": -76.4604263305664,
      "activations/layer8_attention_weight_max": 43.68787384033203,
      "activations/layer8_attention_weight_min": -44.70811080932617,
      "activations/layer9_attention_weight_max": 50.09715270996094,
      "activations/layer9_attention_weight_min": -49.52435302734375,
      "epoch": 6.7,
      "learning_rate": 0.00010790037878787878,
      "loss": 2.8193,
      "step": 115250
    },
    {
      "activations/layer0_attention_weight_max": 15.354559898376465,
      "activations/layer0_attention_weight_min": -14.691141128540039,
      "activations/layer10_attention_weight_max": 35.339515686035156,
      "activations/layer10_attention_weight_min": -37.28633499145508,
      "activations/layer11_attention_weight_max": 28.325927734375,
      "activations/layer11_attention_weight_min": -32.02411651611328,
      "activations/layer12_attention_weight_max": 19.664661407470703,
      "activations/layer12_attention_weight_min": -22.48607635498047,
      "activations/layer13_attention_weight_max": 39.203792572021484,
      "activations/layer13_attention_weight_min": -32.395530700683594,
      "activations/layer14_attention_weight_max": 34.34608459472656,
      "activations/layer14_attention_weight_min": -30.16312599182129,
      "activations/layer15_attention_weight_max": 37.12440872192383,
      "activations/layer15_attention_weight_min": -31.100801467895508,
      "activations/layer16_attention_weight_max": 33.438655853271484,
      "activations/layer16_attention_weight_min": -26.529813766479492,
      "activations/layer17_attention_weight_max": 53.38652420043945,
      "activations/layer17_attention_weight_min": -43.79326629638672,
      "activations/layer18_attention_weight_max": 46.28156280517578,
      "activations/layer18_attention_weight_min": -35.088966369628906,
      "activations/layer19_attention_weight_max": 21.988698959350586,
      "activations/layer19_attention_weight_min": -19.5059757232666,
      "activations/layer1_attention_weight_max": 17.50029754638672,
      "activations/layer1_attention_weight_min": -15.287369728088379,
      "activations/layer20_attention_weight_max": 20.14311981201172,
      "activations/layer20_attention_weight_min": -21.552738189697266,
      "activations/layer21_attention_weight_max": 34.654518127441406,
      "activations/layer21_attention_weight_min": -23.40751838684082,
      "activations/layer22_attention_weight_max": 33.74838638305664,
      "activations/layer22_attention_weight_min": -25.696857452392578,
      "activations/layer23_attention_weight_max": 31.57101058959961,
      "activations/layer23_attention_weight_min": -20.4783935546875,
      "activations/layer2_attention_weight_max": 32.150108337402344,
      "activations/layer2_attention_weight_min": -29.542022705078125,
      "activations/layer3_attention_weight_max": 84.93594360351562,
      "activations/layer3_attention_weight_min": -81.84040832519531,
      "activations/layer4_attention_weight_max": 99.48503875732422,
      "activations/layer4_attention_weight_min": -91.0461654663086,
      "activations/layer5_attention_weight_max": 97.10112762451172,
      "activations/layer5_attention_weight_min": -84.7966537475586,
      "activations/layer6_attention_weight_max": 57.688804626464844,
      "activations/layer6_attention_weight_min": -53.960086822509766,
      "activations/layer7_attention_weight_max": 71.72990417480469,
      "activations/layer7_attention_weight_min": -70.64811706542969,
      "activations/layer8_attention_weight_max": 44.394744873046875,
      "activations/layer8_attention_weight_min": -49.64265823364258,
      "activations/layer9_attention_weight_max": 49.91291809082031,
      "activations/layer9_attention_weight_min": -49.490623474121094,
      "epoch": 6.7,
      "learning_rate": 0.00010788143939393938,
      "loss": 2.8258,
      "step": 115300
    },
    {
      "activations/layer0_attention_weight_max": 15.418062210083008,
      "activations/layer0_attention_weight_min": -14.662304878234863,
      "activations/layer10_attention_weight_max": 39.22723388671875,
      "activations/layer10_attention_weight_min": -36.851287841796875,
      "activations/layer11_attention_weight_max": 32.38492202758789,
      "activations/layer11_attention_weight_min": -30.643238067626953,
      "activations/layer12_attention_weight_max": 22.032047271728516,
      "activations/layer12_attention_weight_min": -26.972957611083984,
      "activations/layer13_attention_weight_max": 46.838069915771484,
      "activations/layer13_attention_weight_min": -33.76066207885742,
      "activations/layer14_attention_weight_max": 37.24314880371094,
      "activations/layer14_attention_weight_min": -29.635244369506836,
      "activations/layer15_attention_weight_max": 34.628170013427734,
      "activations/layer15_attention_weight_min": -29.06147003173828,
      "activations/layer16_attention_weight_max": 35.99113464355469,
      "activations/layer16_attention_weight_min": -25.758821487426758,
      "activations/layer17_attention_weight_max": 52.10616683959961,
      "activations/layer17_attention_weight_min": -41.465248107910156,
      "activations/layer18_attention_weight_max": 47.09560012817383,
      "activations/layer18_attention_weight_min": -36.396541595458984,
      "activations/layer19_attention_weight_max": 21.361448287963867,
      "activations/layer19_attention_weight_min": -21.44979476928711,
      "activations/layer1_attention_weight_max": 17.478330612182617,
      "activations/layer1_attention_weight_min": -18.655532836914062,
      "activations/layer20_attention_weight_max": 21.320165634155273,
      "activations/layer20_attention_weight_min": -22.837038040161133,
      "activations/layer21_attention_weight_max": 37.93781661987305,
      "activations/layer21_attention_weight_min": -25.4460391998291,
      "activations/layer22_attention_weight_max": 30.707904815673828,
      "activations/layer22_attention_weight_min": -26.252378463745117,
      "activations/layer23_attention_weight_max": 34.00571060180664,
      "activations/layer23_attention_weight_min": -20.4715518951416,
      "activations/layer2_attention_weight_max": 31.11279296875,
      "activations/layer2_attention_weight_min": -30.166393280029297,
      "activations/layer3_attention_weight_max": 85.09532928466797,
      "activations/layer3_attention_weight_min": -90.16738891601562,
      "activations/layer4_attention_weight_max": 98.30319213867188,
      "activations/layer4_attention_weight_min": -98.1126937866211,
      "activations/layer5_attention_weight_max": 91.66162109375,
      "activations/layer5_attention_weight_min": -89.53262329101562,
      "activations/layer6_attention_weight_max": 57.713104248046875,
      "activations/layer6_attention_weight_min": -55.92428970336914,
      "activations/layer7_attention_weight_max": 72.44544982910156,
      "activations/layer7_attention_weight_min": -72.66992950439453,
      "activations/layer8_attention_weight_max": 44.113643646240234,
      "activations/layer8_attention_weight_min": -44.22501754760742,
      "activations/layer9_attention_weight_max": 47.48652267456055,
      "activations/layer9_attention_weight_min": -48.146968841552734,
      "epoch": 6.7,
      "learning_rate": 0.00010786249999999999,
      "loss": 2.8144,
      "step": 115350
    },
    {
      "activations/layer0_attention_weight_max": 16.065231323242188,
      "activations/layer0_attention_weight_min": -14.240457534790039,
      "activations/layer10_attention_weight_max": 43.261634826660156,
      "activations/layer10_attention_weight_min": -35.864749908447266,
      "activations/layer11_attention_weight_max": 32.805442810058594,
      "activations/layer11_attention_weight_min": -28.619558334350586,
      "activations/layer12_attention_weight_max": 26.726993560791016,
      "activations/layer12_attention_weight_min": -25.982059478759766,
      "activations/layer13_attention_weight_max": 51.76242446899414,
      "activations/layer13_attention_weight_min": -34.73101806640625,
      "activations/layer14_attention_weight_max": 50.024932861328125,
      "activations/layer14_attention_weight_min": -30.262222290039062,
      "activations/layer15_attention_weight_max": 45.169708251953125,
      "activations/layer15_attention_weight_min": -28.879161834716797,
      "activations/layer16_attention_weight_max": 36.591468811035156,
      "activations/layer16_attention_weight_min": -27.2850284576416,
      "activations/layer17_attention_weight_max": 56.82007598876953,
      "activations/layer17_attention_weight_min": -44.649593353271484,
      "activations/layer18_attention_weight_max": 51.03996276855469,
      "activations/layer18_attention_weight_min": -37.59903335571289,
      "activations/layer19_attention_weight_max": 26.59554100036621,
      "activations/layer19_attention_weight_min": -20.238208770751953,
      "activations/layer1_attention_weight_max": 17.24370574951172,
      "activations/layer1_attention_weight_min": -15.574034690856934,
      "activations/layer20_attention_weight_max": 23.424694061279297,
      "activations/layer20_attention_weight_min": -25.20067024230957,
      "activations/layer21_attention_weight_max": 45.34525680541992,
      "activations/layer21_attention_weight_min": -22.90326499938965,
      "activations/layer22_attention_weight_max": 38.06901168823242,
      "activations/layer22_attention_weight_min": -26.212358474731445,
      "activations/layer23_attention_weight_max": 38.2674446105957,
      "activations/layer23_attention_weight_min": -21.23162841796875,
      "activations/layer2_attention_weight_max": 32.31977844238281,
      "activations/layer2_attention_weight_min": -30.765769958496094,
      "activations/layer3_attention_weight_max": 85.41258239746094,
      "activations/layer3_attention_weight_min": -86.5617904663086,
      "activations/layer4_attention_weight_max": 100.31969451904297,
      "activations/layer4_attention_weight_min": -95.9079360961914,
      "activations/layer5_attention_weight_max": 97.51689910888672,
      "activations/layer5_attention_weight_min": -90.01307678222656,
      "activations/layer6_attention_weight_max": 57.76383972167969,
      "activations/layer6_attention_weight_min": -56.032814025878906,
      "activations/layer7_attention_weight_max": 72.38951110839844,
      "activations/layer7_attention_weight_min": -71.10152435302734,
      "activations/layer8_attention_weight_max": 47.422645568847656,
      "activations/layer8_attention_weight_min": -45.596736907958984,
      "activations/layer9_attention_weight_max": 55.02703857421875,
      "activations/layer9_attention_weight_min": -50.70233154296875,
      "epoch": 6.71,
      "learning_rate": 0.0001078435606060606,
      "loss": 2.8203,
      "step": 115400
    },
    {
      "activations/layer0_attention_weight_max": 15.212559700012207,
      "activations/layer0_attention_weight_min": -14.378826141357422,
      "activations/layer10_attention_weight_max": 48.95474624633789,
      "activations/layer10_attention_weight_min": -44.54695510864258,
      "activations/layer11_attention_weight_max": 40.20087432861328,
      "activations/layer11_attention_weight_min": -33.73432922363281,
      "activations/layer12_attention_weight_max": 25.97953224182129,
      "activations/layer12_attention_weight_min": -27.186613082885742,
      "activations/layer13_attention_weight_max": 80.32083892822266,
      "activations/layer13_attention_weight_min": -37.69657516479492,
      "activations/layer14_attention_weight_max": 75.44732666015625,
      "activations/layer14_attention_weight_min": -36.64344024658203,
      "activations/layer15_attention_weight_max": 50.281192779541016,
      "activations/layer15_attention_weight_min": -29.38221549987793,
      "activations/layer16_attention_weight_max": 39.19398498535156,
      "activations/layer16_attention_weight_min": -26.790483474731445,
      "activations/layer17_attention_weight_max": 58.66696548461914,
      "activations/layer17_attention_weight_min": -41.664649963378906,
      "activations/layer18_attention_weight_max": 48.625179290771484,
      "activations/layer18_attention_weight_min": -37.387916564941406,
      "activations/layer19_attention_weight_max": 23.943052291870117,
      "activations/layer19_attention_weight_min": -20.184022903442383,
      "activations/layer1_attention_weight_max": 16.992237091064453,
      "activations/layer1_attention_weight_min": -15.410207748413086,
      "activations/layer20_attention_weight_max": 22.86625862121582,
      "activations/layer20_attention_weight_min": -22.50563621520996,
      "activations/layer21_attention_weight_max": 40.5214958190918,
      "activations/layer21_attention_weight_min": -25.135923385620117,
      "activations/layer22_attention_weight_max": 34.9612922668457,
      "activations/layer22_attention_weight_min": -26.598724365234375,
      "activations/layer23_attention_weight_max": 35.49592208862305,
      "activations/layer23_attention_weight_min": -22.143173217773438,
      "activations/layer2_attention_weight_max": 31.306259155273438,
      "activations/layer2_attention_weight_min": -29.37255859375,
      "activations/layer3_attention_weight_max": 88.39170837402344,
      "activations/layer3_attention_weight_min": -89.85322570800781,
      "activations/layer4_attention_weight_max": 103.63690185546875,
      "activations/layer4_attention_weight_min": -99.76163482666016,
      "activations/layer5_attention_weight_max": 101.16967010498047,
      "activations/layer5_attention_weight_min": -92.06922912597656,
      "activations/layer6_attention_weight_max": 62.106590270996094,
      "activations/layer6_attention_weight_min": -57.49977111816406,
      "activations/layer7_attention_weight_max": 89.91358947753906,
      "activations/layer7_attention_weight_min": -79.78903198242188,
      "activations/layer8_attention_weight_max": 59.94440460205078,
      "activations/layer8_attention_weight_min": -54.24769973754883,
      "activations/layer9_attention_weight_max": 72.72664642333984,
      "activations/layer9_attention_weight_min": -66.08997344970703,
      "epoch": 6.71,
      "learning_rate": 0.00010782462121212122,
      "loss": 2.8237,
      "step": 115450
    },
    {
      "activations/layer0_attention_weight_max": 14.573844909667969,
      "activations/layer0_attention_weight_min": -15.490616798400879,
      "activations/layer10_attention_weight_max": 36.41777038574219,
      "activations/layer10_attention_weight_min": -32.80061721801758,
      "activations/layer11_attention_weight_max": 29.372581481933594,
      "activations/layer11_attention_weight_min": -30.263580322265625,
      "activations/layer12_attention_weight_max": 20.267133712768555,
      "activations/layer12_attention_weight_min": -27.795063018798828,
      "activations/layer13_attention_weight_max": 39.64005661010742,
      "activations/layer13_attention_weight_min": -30.000137329101562,
      "activations/layer14_attention_weight_max": 39.363121032714844,
      "activations/layer14_attention_weight_min": -31.269607543945312,
      "activations/layer15_attention_weight_max": 35.28781509399414,
      "activations/layer15_attention_weight_min": -28.040803909301758,
      "activations/layer16_attention_weight_max": 34.88162612915039,
      "activations/layer16_attention_weight_min": -26.06970977783203,
      "activations/layer17_attention_weight_max": 51.450862884521484,
      "activations/layer17_attention_weight_min": -40.31770706176758,
      "activations/layer18_attention_weight_max": 43.481842041015625,
      "activations/layer18_attention_weight_min": -35.540618896484375,
      "activations/layer19_attention_weight_max": 23.0814151763916,
      "activations/layer19_attention_weight_min": -22.082948684692383,
      "activations/layer1_attention_weight_max": 16.523662567138672,
      "activations/layer1_attention_weight_min": -14.052559852600098,
      "activations/layer20_attention_weight_max": 20.83910369873047,
      "activations/layer20_attention_weight_min": -22.040590286254883,
      "activations/layer21_attention_weight_max": 35.73678207397461,
      "activations/layer21_attention_weight_min": -25.25261688232422,
      "activations/layer22_attention_weight_max": 31.883359909057617,
      "activations/layer22_attention_weight_min": -25.295320510864258,
      "activations/layer23_attention_weight_max": 32.578636169433594,
      "activations/layer23_attention_weight_min": -20.713760375976562,
      "activations/layer2_attention_weight_max": 32.15540313720703,
      "activations/layer2_attention_weight_min": -28.69873046875,
      "activations/layer3_attention_weight_max": 78.53011322021484,
      "activations/layer3_attention_weight_min": -80.76695251464844,
      "activations/layer4_attention_weight_max": 93.2340087890625,
      "activations/layer4_attention_weight_min": -89.8586654663086,
      "activations/layer5_attention_weight_max": 89.9500732421875,
      "activations/layer5_attention_weight_min": -83.1019287109375,
      "activations/layer6_attention_weight_max": 53.795326232910156,
      "activations/layer6_attention_weight_min": -50.810462951660156,
      "activations/layer7_attention_weight_max": 68.25145721435547,
      "activations/layer7_attention_weight_min": -66.59095764160156,
      "activations/layer8_attention_weight_max": 41.71236038208008,
      "activations/layer8_attention_weight_min": -43.8991584777832,
      "activations/layer9_attention_weight_max": 45.82633590698242,
      "activations/layer9_attention_weight_min": -45.8945198059082,
      "epoch": 6.71,
      "learning_rate": 0.0001078056818181818,
      "loss": 2.8275,
      "step": 115500
    },
    {
      "activations/layer0_attention_weight_max": 15.080669403076172,
      "activations/layer0_attention_weight_min": -14.344314575195312,
      "activations/layer10_attention_weight_max": 37.57737731933594,
      "activations/layer10_attention_weight_min": -35.96084213256836,
      "activations/layer11_attention_weight_max": 32.5675048828125,
      "activations/layer11_attention_weight_min": -31.651996612548828,
      "activations/layer12_attention_weight_max": 18.393930435180664,
      "activations/layer12_attention_weight_min": -27.16993522644043,
      "activations/layer13_attention_weight_max": 46.80469512939453,
      "activations/layer13_attention_weight_min": -36.711151123046875,
      "activations/layer14_attention_weight_max": 51.618717193603516,
      "activations/layer14_attention_weight_min": -37.29191207885742,
      "activations/layer15_attention_weight_max": 35.92680358886719,
      "activations/layer15_attention_weight_min": -28.4556941986084,
      "activations/layer16_attention_weight_max": 31.59006690979004,
      "activations/layer16_attention_weight_min": -25.67051124572754,
      "activations/layer17_attention_weight_max": 51.850257873535156,
      "activations/layer17_attention_weight_min": -42.47813034057617,
      "activations/layer18_attention_weight_max": 43.54318618774414,
      "activations/layer18_attention_weight_min": -36.099735260009766,
      "activations/layer19_attention_weight_max": 21.414777755737305,
      "activations/layer19_attention_weight_min": -19.915386199951172,
      "activations/layer1_attention_weight_max": 16.891695022583008,
      "activations/layer1_attention_weight_min": -14.91550350189209,
      "activations/layer20_attention_weight_max": 20.977663040161133,
      "activations/layer20_attention_weight_min": -20.88194465637207,
      "activations/layer21_attention_weight_max": 37.478363037109375,
      "activations/layer21_attention_weight_min": -25.30978775024414,
      "activations/layer22_attention_weight_max": 31.753299713134766,
      "activations/layer22_attention_weight_min": -23.95334243774414,
      "activations/layer23_attention_weight_max": 32.04094696044922,
      "activations/layer23_attention_weight_min": -20.458892822265625,
      "activations/layer2_attention_weight_max": 31.29909324645996,
      "activations/layer2_attention_weight_min": -29.17980194091797,
      "activations/layer3_attention_weight_max": 86.20992279052734,
      "activations/layer3_attention_weight_min": -83.28437042236328,
      "activations/layer4_attention_weight_max": 102.6412582397461,
      "activations/layer4_attention_weight_min": -90.54961395263672,
      "activations/layer5_attention_weight_max": 92.31462097167969,
      "activations/layer5_attention_weight_min": -83.38916778564453,
      "activations/layer6_attention_weight_max": 59.014774322509766,
      "activations/layer6_attention_weight_min": -50.955135345458984,
      "activations/layer7_attention_weight_max": 76.72135162353516,
      "activations/layer7_attention_weight_min": -67.49812316894531,
      "activations/layer8_attention_weight_max": 46.362152099609375,
      "activations/layer8_attention_weight_min": -46.14826583862305,
      "activations/layer9_attention_weight_max": 60.51028060913086,
      "activations/layer9_attention_weight_min": -48.059547424316406,
      "epoch": 6.71,
      "learning_rate": 0.00010778674242424242,
      "loss": 2.828,
      "step": 115550
    },
    {
      "activations/layer0_attention_weight_max": 15.568135261535645,
      "activations/layer0_attention_weight_min": -14.395813941955566,
      "activations/layer10_attention_weight_max": 38.70489501953125,
      "activations/layer10_attention_weight_min": -36.69677734375,
      "activations/layer11_attention_weight_max": 31.25450897216797,
      "activations/layer11_attention_weight_min": -30.78326416015625,
      "activations/layer12_attention_weight_max": 33.2575569152832,
      "activations/layer12_attention_weight_min": -26.1088924407959,
      "activations/layer13_attention_weight_max": 46.7241096496582,
      "activations/layer13_attention_weight_min": -36.27445983886719,
      "activations/layer14_attention_weight_max": 38.17009735107422,
      "activations/layer14_attention_weight_min": -31.239606857299805,
      "activations/layer15_attention_weight_max": 43.507667541503906,
      "activations/layer15_attention_weight_min": -29.293777465820312,
      "activations/layer16_attention_weight_max": 35.52271270751953,
      "activations/layer16_attention_weight_min": -26.349336624145508,
      "activations/layer17_attention_weight_max": 53.8714485168457,
      "activations/layer17_attention_weight_min": -40.37068557739258,
      "activations/layer18_attention_weight_max": 45.09250259399414,
      "activations/layer18_attention_weight_min": -34.525421142578125,
      "activations/layer19_attention_weight_max": 21.104001998901367,
      "activations/layer19_attention_weight_min": -21.016447067260742,
      "activations/layer1_attention_weight_max": 17.02909278869629,
      "activations/layer1_attention_weight_min": -15.77615737915039,
      "activations/layer20_attention_weight_max": 21.81968879699707,
      "activations/layer20_attention_weight_min": -21.144756317138672,
      "activations/layer21_attention_weight_max": 36.54768371582031,
      "activations/layer21_attention_weight_min": -23.893707275390625,
      "activations/layer22_attention_weight_max": 32.64112091064453,
      "activations/layer22_attention_weight_min": -22.588865280151367,
      "activations/layer23_attention_weight_max": 32.093997955322266,
      "activations/layer23_attention_weight_min": -19.118330001831055,
      "activations/layer2_attention_weight_max": 30.56481170654297,
      "activations/layer2_attention_weight_min": -30.453609466552734,
      "activations/layer3_attention_weight_max": 85.97067260742188,
      "activations/layer3_attention_weight_min": -92.32701873779297,
      "activations/layer4_attention_weight_max": 102.35665893554688,
      "activations/layer4_attention_weight_min": -100.90895080566406,
      "activations/layer5_attention_weight_max": 94.54127502441406,
      "activations/layer5_attention_weight_min": -88.56611633300781,
      "activations/layer6_attention_weight_max": 57.63763427734375,
      "activations/layer6_attention_weight_min": -54.08341979980469,
      "activations/layer7_attention_weight_max": 73.74607849121094,
      "activations/layer7_attention_weight_min": -71.66294860839844,
      "activations/layer8_attention_weight_max": 45.27998733520508,
      "activations/layer8_attention_weight_min": -47.783443450927734,
      "activations/layer9_attention_weight_max": 52.33515548706055,
      "activations/layer9_attention_weight_min": -50.031009674072266,
      "epoch": 6.72,
      "learning_rate": 0.00010776780303030301,
      "loss": 2.8238,
      "step": 115600
    },
    {
      "activations/layer0_attention_weight_max": 15.205890655517578,
      "activations/layer0_attention_weight_min": -15.12214469909668,
      "activations/layer10_attention_weight_max": 38.069183349609375,
      "activations/layer10_attention_weight_min": -36.188751220703125,
      "activations/layer11_attention_weight_max": 31.75251579284668,
      "activations/layer11_attention_weight_min": -30.626676559448242,
      "activations/layer12_attention_weight_max": 22.2482852935791,
      "activations/layer12_attention_weight_min": -24.304141998291016,
      "activations/layer13_attention_weight_max": 53.010093688964844,
      "activations/layer13_attention_weight_min": -36.29142761230469,
      "activations/layer14_attention_weight_max": 42.36711502075195,
      "activations/layer14_attention_weight_min": -31.5068359375,
      "activations/layer15_attention_weight_max": 37.41558074951172,
      "activations/layer15_attention_weight_min": -30.98462677001953,
      "activations/layer16_attention_weight_max": 33.521934509277344,
      "activations/layer16_attention_weight_min": -26.71310806274414,
      "activations/layer17_attention_weight_max": 55.572017669677734,
      "activations/layer17_attention_weight_min": -41.34252166748047,
      "activations/layer18_attention_weight_max": 50.25868606567383,
      "activations/layer18_attention_weight_min": -36.28336715698242,
      "activations/layer19_attention_weight_max": 22.269102096557617,
      "activations/layer19_attention_weight_min": -21.226423263549805,
      "activations/layer1_attention_weight_max": 17.540706634521484,
      "activations/layer1_attention_weight_min": -17.68916893005371,
      "activations/layer20_attention_weight_max": 22.871374130249023,
      "activations/layer20_attention_weight_min": -22.865354537963867,
      "activations/layer21_attention_weight_max": 43.31511306762695,
      "activations/layer21_attention_weight_min": -27.482112884521484,
      "activations/layer22_attention_weight_max": 37.728816986083984,
      "activations/layer22_attention_weight_min": -25.799531936645508,
      "activations/layer23_attention_weight_max": 36.32740783691406,
      "activations/layer23_attention_weight_min": -22.134855270385742,
      "activations/layer2_attention_weight_max": 31.74013900756836,
      "activations/layer2_attention_weight_min": -29.71236228942871,
      "activations/layer3_attention_weight_max": 89.52291870117188,
      "activations/layer3_attention_weight_min": -87.71401977539062,
      "activations/layer4_attention_weight_max": 101.64886474609375,
      "activations/layer4_attention_weight_min": -92.19738006591797,
      "activations/layer5_attention_weight_max": 91.27208709716797,
      "activations/layer5_attention_weight_min": -85.61500549316406,
      "activations/layer6_attention_weight_max": 55.56914520263672,
      "activations/layer6_attention_weight_min": -53.7770881652832,
      "activations/layer7_attention_weight_max": 72.43993377685547,
      "activations/layer7_attention_weight_min": -67.407958984375,
      "activations/layer8_attention_weight_max": 44.37765121459961,
      "activations/layer8_attention_weight_min": -45.012428283691406,
      "activations/layer9_attention_weight_max": 46.9522819519043,
      "activations/layer9_attention_weight_min": -47.807247161865234,
      "epoch": 6.72,
      "learning_rate": 0.00010774886363636362,
      "loss": 2.8142,
      "step": 115650
    },
    {
      "activations/layer0_attention_weight_max": 15.853632926940918,
      "activations/layer0_attention_weight_min": -14.6346435546875,
      "activations/layer10_attention_weight_max": 37.935508728027344,
      "activations/layer10_attention_weight_min": -36.230411529541016,
      "activations/layer11_attention_weight_max": 31.22391128540039,
      "activations/layer11_attention_weight_min": -31.56110954284668,
      "activations/layer12_attention_weight_max": 20.577518463134766,
      "activations/layer12_attention_weight_min": -23.19426155090332,
      "activations/layer13_attention_weight_max": 52.211814880371094,
      "activations/layer13_attention_weight_min": -37.39512252807617,
      "activations/layer14_attention_weight_max": 49.74545669555664,
      "activations/layer14_attention_weight_min": -34.97305679321289,
      "activations/layer15_attention_weight_max": 37.05501174926758,
      "activations/layer15_attention_weight_min": -30.609764099121094,
      "activations/layer16_attention_weight_max": 35.35105895996094,
      "activations/layer16_attention_weight_min": -27.588899612426758,
      "activations/layer17_attention_weight_max": 57.334228515625,
      "activations/layer17_attention_weight_min": -43.17843246459961,
      "activations/layer18_attention_weight_max": 54.18029022216797,
      "activations/layer18_attention_weight_min": -37.907867431640625,
      "activations/layer19_attention_weight_max": 24.32295799255371,
      "activations/layer19_attention_weight_min": -20.949562072753906,
      "activations/layer1_attention_weight_max": 17.19907569885254,
      "activations/layer1_attention_weight_min": -14.894962310791016,
      "activations/layer20_attention_weight_max": 23.97968864440918,
      "activations/layer20_attention_weight_min": -24.6765079498291,
      "activations/layer21_attention_weight_max": 43.15458297729492,
      "activations/layer21_attention_weight_min": -24.224624633789062,
      "activations/layer22_attention_weight_max": 36.382652282714844,
      "activations/layer22_attention_weight_min": -26.20842170715332,
      "activations/layer23_attention_weight_max": 41.07292938232422,
      "activations/layer23_attention_weight_min": -19.226858139038086,
      "activations/layer2_attention_weight_max": 32.05196762084961,
      "activations/layer2_attention_weight_min": -28.961429595947266,
      "activations/layer3_attention_weight_max": 83.46354675292969,
      "activations/layer3_attention_weight_min": -85.48562622070312,
      "activations/layer4_attention_weight_max": 95.54934692382812,
      "activations/layer4_attention_weight_min": -91.29476165771484,
      "activations/layer5_attention_weight_max": 92.01958465576172,
      "activations/layer5_attention_weight_min": -84.15428161621094,
      "activations/layer6_attention_weight_max": 57.47966384887695,
      "activations/layer6_attention_weight_min": -53.6144905090332,
      "activations/layer7_attention_weight_max": 76.45491790771484,
      "activations/layer7_attention_weight_min": -69.73783874511719,
      "activations/layer8_attention_weight_max": 43.7980842590332,
      "activations/layer8_attention_weight_min": -42.66454315185547,
      "activations/layer9_attention_weight_max": 53.01449203491211,
      "activations/layer9_attention_weight_min": -48.37565231323242,
      "epoch": 6.72,
      "learning_rate": 0.00010772992424242424,
      "loss": 2.8227,
      "step": 115700
    },
    {
      "activations/layer0_attention_weight_max": 15.575345039367676,
      "activations/layer0_attention_weight_min": -14.088058471679688,
      "activations/layer10_attention_weight_max": 41.96308898925781,
      "activations/layer10_attention_weight_min": -40.64549255371094,
      "activations/layer11_attention_weight_max": 37.674869537353516,
      "activations/layer11_attention_weight_min": -31.449771881103516,
      "activations/layer12_attention_weight_max": 19.425851821899414,
      "activations/layer12_attention_weight_min": -27.539695739746094,
      "activations/layer13_attention_weight_max": 82.23078918457031,
      "activations/layer13_attention_weight_min": -36.21488952636719,
      "activations/layer14_attention_weight_max": 84.35997009277344,
      "activations/layer14_attention_weight_min": -40.61370086669922,
      "activations/layer15_attention_weight_max": 58.62833786010742,
      "activations/layer15_attention_weight_min": -29.04572868347168,
      "activations/layer16_attention_weight_max": 34.925045013427734,
      "activations/layer16_attention_weight_min": -27.383256912231445,
      "activations/layer17_attention_weight_max": 66.07254028320312,
      "activations/layer17_attention_weight_min": -44.65851974487305,
      "activations/layer18_attention_weight_max": 55.599571228027344,
      "activations/layer18_attention_weight_min": -36.006011962890625,
      "activations/layer19_attention_weight_max": 22.438093185424805,
      "activations/layer19_attention_weight_min": -21.550071716308594,
      "activations/layer1_attention_weight_max": 16.88092041015625,
      "activations/layer1_attention_weight_min": -14.96113109588623,
      "activations/layer20_attention_weight_max": 25.2164306640625,
      "activations/layer20_attention_weight_min": -21.08218002319336,
      "activations/layer21_attention_weight_max": 53.72963333129883,
      "activations/layer21_attention_weight_min": -22.19890594482422,
      "activations/layer22_attention_weight_max": 39.11652755737305,
      "activations/layer22_attention_weight_min": -23.563690185546875,
      "activations/layer23_attention_weight_max": 35.299903869628906,
      "activations/layer23_attention_weight_min": -18.8966121673584,
      "activations/layer2_attention_weight_max": 33.177391052246094,
      "activations/layer2_attention_weight_min": -30.940135955810547,
      "activations/layer3_attention_weight_max": 93.43883514404297,
      "activations/layer3_attention_weight_min": -95.1871109008789,
      "activations/layer4_attention_weight_max": 112.94095611572266,
      "activations/layer4_attention_weight_min": -102.99195861816406,
      "activations/layer5_attention_weight_max": 101.33531951904297,
      "activations/layer5_attention_weight_min": -92.97735595703125,
      "activations/layer6_attention_weight_max": 57.57026290893555,
      "activations/layer6_attention_weight_min": -53.47771072387695,
      "activations/layer7_attention_weight_max": 81.7903823852539,
      "activations/layer7_attention_weight_min": -71.95243072509766,
      "activations/layer8_attention_weight_max": 51.150596618652344,
      "activations/layer8_attention_weight_min": -48.27916717529297,
      "activations/layer9_attention_weight_max": 63.528072357177734,
      "activations/layer9_attention_weight_min": -52.68172836303711,
      "epoch": 6.73,
      "learning_rate": 0.00010771098484848484,
      "loss": 2.8329,
      "step": 115750
    },
    {
      "activations/layer0_attention_weight_max": 15.46452808380127,
      "activations/layer0_attention_weight_min": -14.562200546264648,
      "activations/layer10_attention_weight_max": 39.9603271484375,
      "activations/layer10_attention_weight_min": -36.172515869140625,
      "activations/layer11_attention_weight_max": 32.00558853149414,
      "activations/layer11_attention_weight_min": -29.671985626220703,
      "activations/layer12_attention_weight_max": 25.1751708984375,
      "activations/layer12_attention_weight_min": -22.49942398071289,
      "activations/layer13_attention_weight_max": 46.855247497558594,
      "activations/layer13_attention_weight_min": -35.21442794799805,
      "activations/layer14_attention_weight_max": 40.81131362915039,
      "activations/layer14_attention_weight_min": -30.771316528320312,
      "activations/layer15_attention_weight_max": 39.38737106323242,
      "activations/layer15_attention_weight_min": -28.918611526489258,
      "activations/layer16_attention_weight_max": 38.47445297241211,
      "activations/layer16_attention_weight_min": -25.146331787109375,
      "activations/layer17_attention_weight_max": 57.05324172973633,
      "activations/layer17_attention_weight_min": -40.1596794128418,
      "activations/layer18_attention_weight_max": 50.0119514465332,
      "activations/layer18_attention_weight_min": -32.68830108642578,
      "activations/layer19_attention_weight_max": 22.87052345275879,
      "activations/layer19_attention_weight_min": -18.936832427978516,
      "activations/layer1_attention_weight_max": 16.867595672607422,
      "activations/layer1_attention_weight_min": -17.762447357177734,
      "activations/layer20_attention_weight_max": 21.47749900817871,
      "activations/layer20_attention_weight_min": -22.08137321472168,
      "activations/layer21_attention_weight_max": 38.8173713684082,
      "activations/layer21_attention_weight_min": -22.248838424682617,
      "activations/layer22_attention_weight_max": 32.80693054199219,
      "activations/layer22_attention_weight_min": -23.23663330078125,
      "activations/layer23_attention_weight_max": 31.81702423095703,
      "activations/layer23_attention_weight_min": -19.99881362915039,
      "activations/layer2_attention_weight_max": 32.080039978027344,
      "activations/layer2_attention_weight_min": -30.601802825927734,
      "activations/layer3_attention_weight_max": 83.64291381835938,
      "activations/layer3_attention_weight_min": -88.12655639648438,
      "activations/layer4_attention_weight_max": 101.1102523803711,
      "activations/layer4_attention_weight_min": -92.70702362060547,
      "activations/layer5_attention_weight_max": 96.8514404296875,
      "activations/layer5_attention_weight_min": -86.02394104003906,
      "activations/layer6_attention_weight_max": 60.19865798950195,
      "activations/layer6_attention_weight_min": -54.60641098022461,
      "activations/layer7_attention_weight_max": 74.46543884277344,
      "activations/layer7_attention_weight_min": -71.16458129882812,
      "activations/layer8_attention_weight_max": 51.30680465698242,
      "activations/layer8_attention_weight_min": -46.86375427246094,
      "activations/layer9_attention_weight_max": 51.02315139770508,
      "activations/layer9_attention_weight_min": -48.316463470458984,
      "epoch": 6.73,
      "learning_rate": 0.00010769204545454544,
      "loss": 2.8242,
      "step": 115800
    },
    {
      "activations/layer0_attention_weight_max": 14.97625732421875,
      "activations/layer0_attention_weight_min": -14.618919372558594,
      "activations/layer10_attention_weight_max": 37.618778228759766,
      "activations/layer10_attention_weight_min": -35.181758880615234,
      "activations/layer11_attention_weight_max": 30.076366424560547,
      "activations/layer11_attention_weight_min": -30.412885665893555,
      "activations/layer12_attention_weight_max": 19.58272361755371,
      "activations/layer12_attention_weight_min": -22.947650909423828,
      "activations/layer13_attention_weight_max": 39.110382080078125,
      "activations/layer13_attention_weight_min": -29.783239364624023,
      "activations/layer14_attention_weight_max": 35.16961669921875,
      "activations/layer14_attention_weight_min": -29.763608932495117,
      "activations/layer15_attention_weight_max": 32.88481521606445,
      "activations/layer15_attention_weight_min": -28.220626831054688,
      "activations/layer16_attention_weight_max": 29.885345458984375,
      "activations/layer16_attention_weight_min": -26.129289627075195,
      "activations/layer17_attention_weight_max": 52.75327682495117,
      "activations/layer17_attention_weight_min": -40.514713287353516,
      "activations/layer18_attention_weight_max": 45.26005554199219,
      "activations/layer18_attention_weight_min": -35.56332015991211,
      "activations/layer19_attention_weight_max": 23.94961166381836,
      "activations/layer19_attention_weight_min": -19.487895965576172,
      "activations/layer1_attention_weight_max": 16.361257553100586,
      "activations/layer1_attention_weight_min": -13.936019897460938,
      "activations/layer20_attention_weight_max": 21.096982955932617,
      "activations/layer20_attention_weight_min": -19.710121154785156,
      "activations/layer21_attention_weight_max": 34.20167922973633,
      "activations/layer21_attention_weight_min": -26.111106872558594,
      "activations/layer22_attention_weight_max": 32.05268859863281,
      "activations/layer22_attention_weight_min": -22.86897087097168,
      "activations/layer23_attention_weight_max": 29.365398406982422,
      "activations/layer23_attention_weight_min": -18.761690139770508,
      "activations/layer2_attention_weight_max": 31.97222137451172,
      "activations/layer2_attention_weight_min": -30.320220947265625,
      "activations/layer3_attention_weight_max": 83.92158508300781,
      "activations/layer3_attention_weight_min": -90.2445068359375,
      "activations/layer4_attention_weight_max": 101.94355010986328,
      "activations/layer4_attention_weight_min": -98.43846893310547,
      "activations/layer5_attention_weight_max": 94.7447509765625,
      "activations/layer5_attention_weight_min": -88.98289489746094,
      "activations/layer6_attention_weight_max": 56.098777770996094,
      "activations/layer6_attention_weight_min": -53.92481231689453,
      "activations/layer7_attention_weight_max": 76.26675415039062,
      "activations/layer7_attention_weight_min": -71.42630767822266,
      "activations/layer8_attention_weight_max": 46.60451126098633,
      "activations/layer8_attention_weight_min": -44.09360885620117,
      "activations/layer9_attention_weight_max": 51.92533493041992,
      "activations/layer9_attention_weight_min": -49.327049255371094,
      "epoch": 6.73,
      "learning_rate": 0.00010767310606060605,
      "loss": 2.8147,
      "step": 115850
    },
    {
      "activations/layer0_attention_weight_max": 15.050341606140137,
      "activations/layer0_attention_weight_min": -14.44936466217041,
      "activations/layer10_attention_weight_max": 38.49148941040039,
      "activations/layer10_attention_weight_min": -36.725521087646484,
      "activations/layer11_attention_weight_max": 31.635536193847656,
      "activations/layer11_attention_weight_min": -29.728063583374023,
      "activations/layer12_attention_weight_max": 23.542617797851562,
      "activations/layer12_attention_weight_min": -27.60268783569336,
      "activations/layer13_attention_weight_max": 45.88326644897461,
      "activations/layer13_attention_weight_min": -31.76816177368164,
      "activations/layer14_attention_weight_max": 41.21604537963867,
      "activations/layer14_attention_weight_min": -30.084644317626953,
      "activations/layer15_attention_weight_max": 41.92805480957031,
      "activations/layer15_attention_weight_min": -30.939170837402344,
      "activations/layer16_attention_weight_max": 33.48644256591797,
      "activations/layer16_attention_weight_min": -27.19058609008789,
      "activations/layer17_attention_weight_max": 60.07008361816406,
      "activations/layer17_attention_weight_min": -42.41421890258789,
      "activations/layer18_attention_weight_max": 53.29630661010742,
      "activations/layer18_attention_weight_min": -35.231605529785156,
      "activations/layer19_attention_weight_max": 29.000152587890625,
      "activations/layer19_attention_weight_min": -21.806875228881836,
      "activations/layer1_attention_weight_max": 17.53851890563965,
      "activations/layer1_attention_weight_min": -14.562082290649414,
      "activations/layer20_attention_weight_max": 26.258445739746094,
      "activations/layer20_attention_weight_min": -21.306804656982422,
      "activations/layer21_attention_weight_max": 46.596099853515625,
      "activations/layer21_attention_weight_min": -24.43191909790039,
      "activations/layer22_attention_weight_max": 37.942054748535156,
      "activations/layer22_attention_weight_min": -24.090166091918945,
      "activations/layer23_attention_weight_max": 35.77610778808594,
      "activations/layer23_attention_weight_min": -22.134807586669922,
      "activations/layer2_attention_weight_max": 30.571651458740234,
      "activations/layer2_attention_weight_min": -29.154815673828125,
      "activations/layer3_attention_weight_max": 83.90555572509766,
      "activations/layer3_attention_weight_min": -85.87032318115234,
      "activations/layer4_attention_weight_max": 100.7589111328125,
      "activations/layer4_attention_weight_min": -93.16459655761719,
      "activations/layer5_attention_weight_max": 100.4052963256836,
      "activations/layer5_attention_weight_min": -85.29717254638672,
      "activations/layer6_attention_weight_max": 60.455810546875,
      "activations/layer6_attention_weight_min": -51.89292526245117,
      "activations/layer7_attention_weight_max": 70.55941009521484,
      "activations/layer7_attention_weight_min": -67.75154113769531,
      "activations/layer8_attention_weight_max": 44.63248062133789,
      "activations/layer8_attention_weight_min": -44.38790512084961,
      "activations/layer9_attention_weight_max": 49.53643035888672,
      "activations/layer9_attention_weight_min": -47.58120346069336,
      "epoch": 6.73,
      "learning_rate": 0.00010765416666666666,
      "loss": 2.8101,
      "step": 115900
    },
    {
      "activations/layer0_attention_weight_max": 15.587981224060059,
      "activations/layer0_attention_weight_min": -14.68897819519043,
      "activations/layer10_attention_weight_max": 37.14472198486328,
      "activations/layer10_attention_weight_min": -36.81400680541992,
      "activations/layer11_attention_weight_max": 30.663129806518555,
      "activations/layer11_attention_weight_min": -31.77949333190918,
      "activations/layer12_attention_weight_max": 27.443307876586914,
      "activations/layer12_attention_weight_min": -26.912696838378906,
      "activations/layer13_attention_weight_max": 45.910377502441406,
      "activations/layer13_attention_weight_min": -32.984710693359375,
      "activations/layer14_attention_weight_max": 44.155250549316406,
      "activations/layer14_attention_weight_min": -30.716096878051758,
      "activations/layer15_attention_weight_max": 36.90091323852539,
      "activations/layer15_attention_weight_min": -29.068273544311523,
      "activations/layer16_attention_weight_max": 34.206417083740234,
      "activations/layer16_attention_weight_min": -25.914505004882812,
      "activations/layer17_attention_weight_max": 55.34246063232422,
      "activations/layer17_attention_weight_min": -41.54531478881836,
      "activations/layer18_attention_weight_max": 48.10016632080078,
      "activations/layer18_attention_weight_min": -35.030242919921875,
      "activations/layer19_attention_weight_max": 21.969757080078125,
      "activations/layer19_attention_weight_min": -20.78022575378418,
      "activations/layer1_attention_weight_max": 17.080957412719727,
      "activations/layer1_attention_weight_min": -15.237698554992676,
      "activations/layer20_attention_weight_max": 24.065813064575195,
      "activations/layer20_attention_weight_min": -19.480392456054688,
      "activations/layer21_attention_weight_max": 35.41809844970703,
      "activations/layer21_attention_weight_min": -24.368669509887695,
      "activations/layer22_attention_weight_max": 32.72635269165039,
      "activations/layer22_attention_weight_min": -23.30758285522461,
      "activations/layer23_attention_weight_max": 32.335166931152344,
      "activations/layer23_attention_weight_min": -19.274442672729492,
      "activations/layer2_attention_weight_max": 30.30482292175293,
      "activations/layer2_attention_weight_min": -29.75063705444336,
      "activations/layer3_attention_weight_max": 82.2852554321289,
      "activations/layer3_attention_weight_min": -83.10212707519531,
      "activations/layer4_attention_weight_max": 98.55119323730469,
      "activations/layer4_attention_weight_min": -94.40110778808594,
      "activations/layer5_attention_weight_max": 90.90087127685547,
      "activations/layer5_attention_weight_min": -86.63691711425781,
      "activations/layer6_attention_weight_max": 56.73436737060547,
      "activations/layer6_attention_weight_min": -54.61491394042969,
      "activations/layer7_attention_weight_max": 80.90290832519531,
      "activations/layer7_attention_weight_min": -70.44935607910156,
      "activations/layer8_attention_weight_max": 45.08445358276367,
      "activations/layer8_attention_weight_min": -44.198543548583984,
      "activations/layer9_attention_weight_max": 55.137813568115234,
      "activations/layer9_attention_weight_min": -54.0350227355957,
      "epoch": 6.74,
      "learning_rate": 0.00010763560606060605,
      "loss": 2.8198,
      "step": 115950
    },
    {
      "activations/layer0_attention_weight_max": 14.979083061218262,
      "activations/layer0_attention_weight_min": -14.991423606872559,
      "activations/layer10_attention_weight_max": 37.08021926879883,
      "activations/layer10_attention_weight_min": -34.796722412109375,
      "activations/layer11_attention_weight_max": 30.202503204345703,
      "activations/layer11_attention_weight_min": -28.670995712280273,
      "activations/layer12_attention_weight_max": 19.433460235595703,
      "activations/layer12_attention_weight_min": -26.764265060424805,
      "activations/layer13_attention_weight_max": 43.179866790771484,
      "activations/layer13_attention_weight_min": -33.426658630371094,
      "activations/layer14_attention_weight_max": 46.4486198425293,
      "activations/layer14_attention_weight_min": -30.444936752319336,
      "activations/layer15_attention_weight_max": 39.4334831237793,
      "activations/layer15_attention_weight_min": -28.266620635986328,
      "activations/layer16_attention_weight_max": 30.261341094970703,
      "activations/layer16_attention_weight_min": -26.967336654663086,
      "activations/layer17_attention_weight_max": 54.53340530395508,
      "activations/layer17_attention_weight_min": -39.2871208190918,
      "activations/layer18_attention_weight_max": 47.75957489013672,
      "activations/layer18_attention_weight_min": -35.488460540771484,
      "activations/layer19_attention_weight_max": 24.854578018188477,
      "activations/layer19_attention_weight_min": -21.315441131591797,
      "activations/layer1_attention_weight_max": 16.96396255493164,
      "activations/layer1_attention_weight_min": -16.22254753112793,
      "activations/layer20_attention_weight_max": 21.194276809692383,
      "activations/layer20_attention_weight_min": -19.52196502685547,
      "activations/layer21_attention_weight_max": 44.254634857177734,
      "activations/layer21_attention_weight_min": -23.429773330688477,
      "activations/layer22_attention_weight_max": 31.870906829833984,
      "activations/layer22_attention_weight_min": -25.208690643310547,
      "activations/layer23_attention_weight_max": 32.97203826904297,
      "activations/layer23_attention_weight_min": -19.548870086669922,
      "activations/layer2_attention_weight_max": 31.69472312927246,
      "activations/layer2_attention_weight_min": -29.649267196655273,
      "activations/layer3_attention_weight_max": 90.21087646484375,
      "activations/layer3_attention_weight_min": -89.48397064208984,
      "activations/layer4_attention_weight_max": 96.67635345458984,
      "activations/layer4_attention_weight_min": -90.5261459350586,
      "activations/layer5_attention_weight_max": 91.50480651855469,
      "activations/layer5_attention_weight_min": -82.687255859375,
      "activations/layer6_attention_weight_max": 55.396446228027344,
      "activations/layer6_attention_weight_min": -52.44302749633789,
      "activations/layer7_attention_weight_max": 70.53683471679688,
      "activations/layer7_attention_weight_min": -68.87832641601562,
      "activations/layer8_attention_weight_max": 42.596988677978516,
      "activations/layer8_attention_weight_min": -45.1431770324707,
      "activations/layer9_attention_weight_max": 47.88373947143555,
      "activations/layer9_attention_weight_min": -46.725399017333984,
      "epoch": 6.74,
      "learning_rate": 0.00010761704545454545,
      "loss": 2.8492,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.518,
      "eval_samples_per_second": 504.109,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.518,
      "eval_openwebtext_samples_per_second": 504.109,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 2.011,
      "eval_wikitext_samples_per_second": 226.752,
      "step": 116000
    },
    {
      "epoch": 6.74,
      "eval_lambada_loss": 2.806640625,
      "eval_lambada_ppl": 16.554212894695087,
      "eval_lambada_runtime": 9.6624,
      "eval_lambada_samples_per_second": 503.91,
      "step": 116000
    },
    {
      "activations/layer0_attention_weight_max": 14.374351501464844,
      "activations/layer0_attention_weight_min": -14.829833984375,
      "activations/layer10_attention_weight_max": 38.52055740356445,
      "activations/layer10_attention_weight_min": -34.91146469116211,
      "activations/layer11_attention_weight_max": 30.73756980895996,
      "activations/layer11_attention_weight_min": -30.276737213134766,
      "activations/layer12_attention_weight_max": 21.53853988647461,
      "activations/layer12_attention_weight_min": -22.219511032104492,
      "activations/layer13_attention_weight_max": 40.25266647338867,
      "activations/layer13_attention_weight_min": -32.40871047973633,
      "activations/layer14_attention_weight_max": 41.234493255615234,
      "activations/layer14_attention_weight_min": -30.995304107666016,
      "activations/layer15_attention_weight_max": 37.301544189453125,
      "activations/layer15_attention_weight_min": -29.576108932495117,
      "activations/layer16_attention_weight_max": 31.37206268310547,
      "activations/layer16_attention_weight_min": -26.704191207885742,
      "activations/layer17_attention_weight_max": 58.01389694213867,
      "activations/layer17_attention_weight_min": -41.37198257446289,
      "activations/layer18_attention_weight_max": 46.91150665283203,
      "activations/layer18_attention_weight_min": -36.752479553222656,
      "activations/layer19_attention_weight_max": 24.70380973815918,
      "activations/layer19_attention_weight_min": -21.760711669921875,
      "activations/layer1_attention_weight_max": 17.146026611328125,
      "activations/layer1_attention_weight_min": -13.97368049621582,
      "activations/layer20_attention_weight_max": 25.74010467529297,
      "activations/layer20_attention_weight_min": -21.02799415588379,
      "activations/layer21_attention_weight_max": 44.59012222290039,
      "activations/layer21_attention_weight_min": -23.026708602905273,
      "activations/layer22_attention_weight_max": 33.54219055175781,
      "activations/layer22_attention_weight_min": -24.442434310913086,
      "activations/layer23_attention_weight_max": 39.435821533203125,
      "activations/layer23_attention_weight_min": -20.752031326293945,
      "activations/layer2_attention_weight_max": 31.573989868164062,
      "activations/layer2_attention_weight_min": -29.542327880859375,
      "activations/layer3_attention_weight_max": 84.5375747680664,
      "activations/layer3_attention_weight_min": -86.47549438476562,
      "activations/layer4_attention_weight_max": 99.07989501953125,
      "activations/layer4_attention_weight_min": -94.12909698486328,
      "activations/layer5_attention_weight_max": 90.38998413085938,
      "activations/layer5_attention_weight_min": -82.71647644042969,
      "activations/layer6_attention_weight_max": 54.035972595214844,
      "activations/layer6_attention_weight_min": -50.55109405517578,
      "activations/layer7_attention_weight_max": 76.31452178955078,
      "activations/layer7_attention_weight_min": -69.33319091796875,
      "activations/layer8_attention_weight_max": 42.31458282470703,
      "activations/layer8_attention_weight_min": -44.132301330566406,
      "activations/layer9_attention_weight_max": 48.10597229003906,
      "activations/layer9_attention_weight_min": -48.35174560546875,
      "epoch": 6.74,
      "learning_rate": 0.00010759810606060605,
      "loss": 2.8341,
      "step": 116050
    },
    {
      "activations/layer0_attention_weight_max": 15.091727256774902,
      "activations/layer0_attention_weight_min": -14.708191871643066,
      "activations/layer10_attention_weight_max": 36.63840103149414,
      "activations/layer10_attention_weight_min": -33.831966400146484,
      "activations/layer11_attention_weight_max": 29.790603637695312,
      "activations/layer11_attention_weight_min": -28.494487762451172,
      "activations/layer12_attention_weight_max": 19.46688461303711,
      "activations/layer12_attention_weight_min": -25.340225219726562,
      "activations/layer13_attention_weight_max": 44.259132385253906,
      "activations/layer13_attention_weight_min": -32.062347412109375,
      "activations/layer14_attention_weight_max": 38.81351852416992,
      "activations/layer14_attention_weight_min": -29.41942024230957,
      "activations/layer15_attention_weight_max": 36.61530303955078,
      "activations/layer15_attention_weight_min": -28.48450469970703,
      "activations/layer16_attention_weight_max": 31.769287109375,
      "activations/layer16_attention_weight_min": -26.24522590637207,
      "activations/layer17_attention_weight_max": 56.71733474731445,
      "activations/layer17_attention_weight_min": -41.586997985839844,
      "activations/layer18_attention_weight_max": 50.42402267456055,
      "activations/layer18_attention_weight_min": -36.510772705078125,
      "activations/layer19_attention_weight_max": 21.1063175201416,
      "activations/layer19_attention_weight_min": -21.807004928588867,
      "activations/layer1_attention_weight_max": 16.649511337280273,
      "activations/layer1_attention_weight_min": -14.704687118530273,
      "activations/layer20_attention_weight_max": 23.025453567504883,
      "activations/layer20_attention_weight_min": -22.524751663208008,
      "activations/layer21_attention_weight_max": 46.17522048950195,
      "activations/layer21_attention_weight_min": -26.776611328125,
      "activations/layer22_attention_weight_max": 35.841156005859375,
      "activations/layer22_attention_weight_min": -26.613956451416016,
      "activations/layer23_attention_weight_max": 37.34276580810547,
      "activations/layer23_attention_weight_min": -21.541244506835938,
      "activations/layer2_attention_weight_max": 32.270389556884766,
      "activations/layer2_attention_weight_min": -31.001407623291016,
      "activations/layer3_attention_weight_max": 86.04446411132812,
      "activations/layer3_attention_weight_min": -86.37592315673828,
      "activations/layer4_attention_weight_max": 103.9246597290039,
      "activations/layer4_attention_weight_min": -97.42793273925781,
      "activations/layer5_attention_weight_max": 96.46286010742188,
      "activations/layer5_attention_weight_min": -91.48522186279297,
      "activations/layer6_attention_weight_max": 59.95570373535156,
      "activations/layer6_attention_weight_min": -53.37178039550781,
      "activations/layer7_attention_weight_max": 76.65107727050781,
      "activations/layer7_attention_weight_min": -71.23656463623047,
      "activations/layer8_attention_weight_max": 47.318824768066406,
      "activations/layer8_attention_weight_min": -44.49391555786133,
      "activations/layer9_attention_weight_max": 49.178924560546875,
      "activations/layer9_attention_weight_min": -44.27131271362305,
      "epoch": 6.75,
      "learning_rate": 0.00010757954545454544,
      "loss": 2.8381,
      "step": 116100
    },
    {
      "activations/layer0_attention_weight_max": 14.979820251464844,
      "activations/layer0_attention_weight_min": -15.381678581237793,
      "activations/layer10_attention_weight_max": 35.99407958984375,
      "activations/layer10_attention_weight_min": -35.538021087646484,
      "activations/layer11_attention_weight_max": 29.497404098510742,
      "activations/layer11_attention_weight_min": -29.67477798461914,
      "activations/layer12_attention_weight_max": 17.593233108520508,
      "activations/layer12_attention_weight_min": -34.17823791503906,
      "activations/layer13_attention_weight_max": 46.72480392456055,
      "activations/layer13_attention_weight_min": -35.80900955200195,
      "activations/layer14_attention_weight_max": 47.45896530151367,
      "activations/layer14_attention_weight_min": -33.01731491088867,
      "activations/layer15_attention_weight_max": 34.557403564453125,
      "activations/layer15_attention_weight_min": -29.022037506103516,
      "activations/layer16_attention_weight_max": 30.32151222229004,
      "activations/layer16_attention_weight_min": -26.15846824645996,
      "activations/layer17_attention_weight_max": 54.599998474121094,
      "activations/layer17_attention_weight_min": -42.14178466796875,
      "activations/layer18_attention_weight_max": 48.53605270385742,
      "activations/layer18_attention_weight_min": -37.238616943359375,
      "activations/layer19_attention_weight_max": 22.97637939453125,
      "activations/layer19_attention_weight_min": -21.52821159362793,
      "activations/layer1_attention_weight_max": 16.907543182373047,
      "activations/layer1_attention_weight_min": -16.293338775634766,
      "activations/layer20_attention_weight_max": 23.31139373779297,
      "activations/layer20_attention_weight_min": -20.54128646850586,
      "activations/layer21_attention_weight_max": 42.11349868774414,
      "activations/layer21_attention_weight_min": -24.439037322998047,
      "activations/layer22_attention_weight_max": 32.30854797363281,
      "activations/layer22_attention_weight_min": -26.54407501220703,
      "activations/layer23_attention_weight_max": 29.803730010986328,
      "activations/layer23_attention_weight_min": -22.539165496826172,
      "activations/layer2_attention_weight_max": 32.39983367919922,
      "activations/layer2_attention_weight_min": -31.706510543823242,
      "activations/layer3_attention_weight_max": 89.78862762451172,
      "activations/layer3_attention_weight_min": -88.82315826416016,
      "activations/layer4_attention_weight_max": 103.18033599853516,
      "activations/layer4_attention_weight_min": -94.13570404052734,
      "activations/layer5_attention_weight_max": 97.1104965209961,
      "activations/layer5_attention_weight_min": -84.98222351074219,
      "activations/layer6_attention_weight_max": 58.67624282836914,
      "activations/layer6_attention_weight_min": -53.08803176879883,
      "activations/layer7_attention_weight_max": 76.05077362060547,
      "activations/layer7_attention_weight_min": -70.45040130615234,
      "activations/layer8_attention_weight_max": 44.4755973815918,
      "activations/layer8_attention_weight_min": -43.61488723754883,
      "activations/layer9_attention_weight_max": 61.06605911254883,
      "activations/layer9_attention_weight_min": -52.48164749145508,
      "epoch": 6.75,
      "learning_rate": 0.00010756060606060606,
      "loss": 2.8164,
      "step": 116150
    },
    {
      "activations/layer0_attention_weight_max": 15.228414535522461,
      "activations/layer0_attention_weight_min": -14.639599800109863,
      "activations/layer10_attention_weight_max": 44.01034164428711,
      "activations/layer10_attention_weight_min": -34.842918395996094,
      "activations/layer11_attention_weight_max": 34.9365119934082,
      "activations/layer11_attention_weight_min": -29.941762924194336,
      "activations/layer12_attention_weight_max": 20.2755069732666,
      "activations/layer12_attention_weight_min": -26.486360549926758,
      "activations/layer13_attention_weight_max": 47.72311019897461,
      "activations/layer13_attention_weight_min": -31.312196731567383,
      "activations/layer14_attention_weight_max": 43.2513313293457,
      "activations/layer14_attention_weight_min": -29.187007904052734,
      "activations/layer15_attention_weight_max": 38.894771575927734,
      "activations/layer15_attention_weight_min": -29.31541633605957,
      "activations/layer16_attention_weight_max": 33.64080810546875,
      "activations/layer16_attention_weight_min": -25.872989654541016,
      "activations/layer17_attention_weight_max": 63.25571060180664,
      "activations/layer17_attention_weight_min": -39.4986457824707,
      "activations/layer18_attention_weight_max": 51.3950309753418,
      "activations/layer18_attention_weight_min": -33.5435905456543,
      "activations/layer19_attention_weight_max": 22.74188804626465,
      "activations/layer19_attention_weight_min": -20.20481300354004,
      "activations/layer1_attention_weight_max": 17.52848243713379,
      "activations/layer1_attention_weight_min": -14.825562477111816,
      "activations/layer20_attention_weight_max": 22.852642059326172,
      "activations/layer20_attention_weight_min": -25.137022018432617,
      "activations/layer21_attention_weight_max": 44.597999572753906,
      "activations/layer21_attention_weight_min": -22.052087783813477,
      "activations/layer22_attention_weight_max": 32.80316925048828,
      "activations/layer22_attention_weight_min": -23.613189697265625,
      "activations/layer23_attention_weight_max": 32.52647399902344,
      "activations/layer23_attention_weight_min": -19.632957458496094,
      "activations/layer2_attention_weight_max": 31.166709899902344,
      "activations/layer2_attention_weight_min": -29.62726402282715,
      "activations/layer3_attention_weight_max": 84.14081573486328,
      "activations/layer3_attention_weight_min": -85.5042953491211,
      "activations/layer4_attention_weight_max": 99.75481414794922,
      "activations/layer4_attention_weight_min": -92.83959197998047,
      "activations/layer5_attention_weight_max": 94.40431213378906,
      "activations/layer5_attention_weight_min": -86.79782104492188,
      "activations/layer6_attention_weight_max": 56.494407653808594,
      "activations/layer6_attention_weight_min": -55.5609016418457,
      "activations/layer7_attention_weight_max": 75.1111068725586,
      "activations/layer7_attention_weight_min": -66.6269302368164,
      "activations/layer8_attention_weight_max": 46.14447021484375,
      "activations/layer8_attention_weight_min": -45.133094787597656,
      "activations/layer9_attention_weight_max": 58.0809326171875,
      "activations/layer9_attention_weight_min": -45.87761688232422,
      "epoch": 6.75,
      "learning_rate": 0.00010754166666666666,
      "loss": 2.8198,
      "step": 116200
    },
    {
      "activations/layer0_attention_weight_max": 14.908252716064453,
      "activations/layer0_attention_weight_min": -14.636862754821777,
      "activations/layer10_attention_weight_max": 39.569156646728516,
      "activations/layer10_attention_weight_min": -34.865806579589844,
      "activations/layer11_attention_weight_max": 31.437612533569336,
      "activations/layer11_attention_weight_min": -29.534122467041016,
      "activations/layer12_attention_weight_max": 21.319576263427734,
      "activations/layer12_attention_weight_min": -23.051603317260742,
      "activations/layer13_attention_weight_max": 42.36834716796875,
      "activations/layer13_attention_weight_min": -31.482921600341797,
      "activations/layer14_attention_weight_max": 37.726009368896484,
      "activations/layer14_attention_weight_min": -29.484588623046875,
      "activations/layer15_attention_weight_max": 34.21363830566406,
      "activations/layer15_attention_weight_min": -29.064531326293945,
      "activations/layer16_attention_weight_max": 30.63445281982422,
      "activations/layer16_attention_weight_min": -26.730669021606445,
      "activations/layer17_attention_weight_max": 49.31120681762695,
      "activations/layer17_attention_weight_min": -40.959293365478516,
      "activations/layer18_attention_weight_max": 43.47052001953125,
      "activations/layer18_attention_weight_min": -34.68640899658203,
      "activations/layer19_attention_weight_max": 21.736286163330078,
      "activations/layer19_attention_weight_min": -22.225440979003906,
      "activations/layer1_attention_weight_max": 17.71338653564453,
      "activations/layer1_attention_weight_min": -14.36705493927002,
      "activations/layer20_attention_weight_max": 19.888992309570312,
      "activations/layer20_attention_weight_min": -23.676820755004883,
      "activations/layer21_attention_weight_max": 33.20439147949219,
      "activations/layer21_attention_weight_min": -27.64708137512207,
      "activations/layer22_attention_weight_max": 31.98348045349121,
      "activations/layer22_attention_weight_min": -26.653650283813477,
      "activations/layer23_attention_weight_max": 27.735523223876953,
      "activations/layer23_attention_weight_min": -20.74144172668457,
      "activations/layer2_attention_weight_max": 31.142202377319336,
      "activations/layer2_attention_weight_min": -30.99468231201172,
      "activations/layer3_attention_weight_max": 83.60523223876953,
      "activations/layer3_attention_weight_min": -85.03025817871094,
      "activations/layer4_attention_weight_max": 100.01854705810547,
      "activations/layer4_attention_weight_min": -88.1633529663086,
      "activations/layer5_attention_weight_max": 91.04888153076172,
      "activations/layer5_attention_weight_min": -81.10794830322266,
      "activations/layer6_attention_weight_max": 55.52922058105469,
      "activations/layer6_attention_weight_min": -50.208885192871094,
      "activations/layer7_attention_weight_max": 71.58992767333984,
      "activations/layer7_attention_weight_min": -66.72943115234375,
      "activations/layer8_attention_weight_max": 45.559043884277344,
      "activations/layer8_attention_weight_min": -44.45088195800781,
      "activations/layer9_attention_weight_max": 49.49152755737305,
      "activations/layer9_attention_weight_min": -47.02988815307617,
      "epoch": 6.75,
      "learning_rate": 0.00010752310606060605,
      "loss": 2.8207,
      "step": 116250
    },
    {
      "activations/layer0_attention_weight_max": 14.96445369720459,
      "activations/layer0_attention_weight_min": -15.390203475952148,
      "activations/layer10_attention_weight_max": 35.20650100708008,
      "activations/layer10_attention_weight_min": -32.863311767578125,
      "activations/layer11_attention_weight_max": 28.392024993896484,
      "activations/layer11_attention_weight_min": -29.698741912841797,
      "activations/layer12_attention_weight_max": 22.197275161743164,
      "activations/layer12_attention_weight_min": -26.07324981689453,
      "activations/layer13_attention_weight_max": 44.30052185058594,
      "activations/layer13_attention_weight_min": -34.529964447021484,
      "activations/layer14_attention_weight_max": 39.77980041503906,
      "activations/layer14_attention_weight_min": -30.77530288696289,
      "activations/layer15_attention_weight_max": 38.3768424987793,
      "activations/layer15_attention_weight_min": -30.002042770385742,
      "activations/layer16_attention_weight_max": 31.005212783813477,
      "activations/layer16_attention_weight_min": -26.424789428710938,
      "activations/layer17_attention_weight_max": 49.36494064331055,
      "activations/layer17_attention_weight_min": -42.64638900756836,
      "activations/layer18_attention_weight_max": 42.436927795410156,
      "activations/layer18_attention_weight_min": -34.81622314453125,
      "activations/layer19_attention_weight_max": 19.801198959350586,
      "activations/layer19_attention_weight_min": -21.413021087646484,
      "activations/layer1_attention_weight_max": 18.011064529418945,
      "activations/layer1_attention_weight_min": -14.470016479492188,
      "activations/layer20_attention_weight_max": 21.37764549255371,
      "activations/layer20_attention_weight_min": -21.78105354309082,
      "activations/layer21_attention_weight_max": 35.23772430419922,
      "activations/layer21_attention_weight_min": -24.174694061279297,
      "activations/layer22_attention_weight_max": 30.484590530395508,
      "activations/layer22_attention_weight_min": -23.806957244873047,
      "activations/layer23_attention_weight_max": 32.70624542236328,
      "activations/layer23_attention_weight_min": -19.79731559753418,
      "activations/layer2_attention_weight_max": 31.992361068725586,
      "activations/layer2_attention_weight_min": -30.940916061401367,
      "activations/layer3_attention_weight_max": 86.86666870117188,
      "activations/layer3_attention_weight_min": -88.52198028564453,
      "activations/layer4_attention_weight_max": 95.67929077148438,
      "activations/layer4_attention_weight_min": -92.3338623046875,
      "activations/layer5_attention_weight_max": 87.6273422241211,
      "activations/layer5_attention_weight_min": -84.7618637084961,
      "activations/layer6_attention_weight_max": 53.3105583190918,
      "activations/layer6_attention_weight_min": -50.68830490112305,
      "activations/layer7_attention_weight_max": 70.49483489990234,
      "activations/layer7_attention_weight_min": -71.51029205322266,
      "activations/layer8_attention_weight_max": 39.21512985229492,
      "activations/layer8_attention_weight_min": -41.22496795654297,
      "activations/layer9_attention_weight_max": 47.93338394165039,
      "activations/layer9_attention_weight_min": -48.15652084350586,
      "epoch": 6.76,
      "learning_rate": 0.00010750416666666665,
      "loss": 2.8312,
      "step": 116300
    },
    {
      "activations/layer0_attention_weight_max": 15.30247688293457,
      "activations/layer0_attention_weight_min": -14.63171672821045,
      "activations/layer10_attention_weight_max": 37.56834030151367,
      "activations/layer10_attention_weight_min": -35.720489501953125,
      "activations/layer11_attention_weight_max": 32.70774841308594,
      "activations/layer11_attention_weight_min": -30.002241134643555,
      "activations/layer12_attention_weight_max": 20.801315307617188,
      "activations/layer12_attention_weight_min": -22.373458862304688,
      "activations/layer13_attention_weight_max": 42.362632751464844,
      "activations/layer13_attention_weight_min": -31.431753158569336,
      "activations/layer14_attention_weight_max": 42.800987243652344,
      "activations/layer14_attention_weight_min": -30.586692810058594,
      "activations/layer15_attention_weight_max": 37.3664665222168,
      "activations/layer15_attention_weight_min": -29.36553955078125,
      "activations/layer16_attention_weight_max": 34.75922775268555,
      "activations/layer16_attention_weight_min": -27.290922164916992,
      "activations/layer17_attention_weight_max": 58.88229751586914,
      "activations/layer17_attention_weight_min": -42.14273452758789,
      "activations/layer18_attention_weight_max": 55.80844497680664,
      "activations/layer18_attention_weight_min": -35.89732360839844,
      "activations/layer19_attention_weight_max": 25.860931396484375,
      "activations/layer19_attention_weight_min": -22.88716697692871,
      "activations/layer1_attention_weight_max": 16.270401000976562,
      "activations/layer1_attention_weight_min": -14.573404312133789,
      "activations/layer20_attention_weight_max": 25.26751136779785,
      "activations/layer20_attention_weight_min": -22.09894561767578,
      "activations/layer21_attention_weight_max": 36.122928619384766,
      "activations/layer21_attention_weight_min": -24.75157928466797,
      "activations/layer22_attention_weight_max": 38.34870147705078,
      "activations/layer22_attention_weight_min": -24.807344436645508,
      "activations/layer23_attention_weight_max": 35.49314880371094,
      "activations/layer23_attention_weight_min": -20.95579719543457,
      "activations/layer2_attention_weight_max": 33.6007080078125,
      "activations/layer2_attention_weight_min": -28.796794891357422,
      "activations/layer3_attention_weight_max": 85.31733703613281,
      "activations/layer3_attention_weight_min": -88.2528305053711,
      "activations/layer4_attention_weight_max": 100.50293731689453,
      "activations/layer4_attention_weight_min": -95.57270812988281,
      "activations/layer5_attention_weight_max": 93.07344055175781,
      "activations/layer5_attention_weight_min": -86.78701782226562,
      "activations/layer6_attention_weight_max": 55.25404357910156,
      "activations/layer6_attention_weight_min": -52.695186614990234,
      "activations/layer7_attention_weight_max": 76.39602661132812,
      "activations/layer7_attention_weight_min": -68.18550109863281,
      "activations/layer8_attention_weight_max": 43.843448638916016,
      "activations/layer8_attention_weight_min": -43.1649055480957,
      "activations/layer9_attention_weight_max": 49.46946334838867,
      "activations/layer9_attention_weight_min": -49.7050895690918,
      "epoch": 6.76,
      "learning_rate": 0.00010748522727272727,
      "loss": 2.8352,
      "step": 116350
    },
    {
      "activations/layer0_attention_weight_max": 14.668816566467285,
      "activations/layer0_attention_weight_min": -14.620061874389648,
      "activations/layer10_attention_weight_max": 39.810176849365234,
      "activations/layer10_attention_weight_min": -35.826358795166016,
      "activations/layer11_attention_weight_max": 34.206050872802734,
      "activations/layer11_attention_weight_min": -30.259204864501953,
      "activations/layer12_attention_weight_max": 22.411840438842773,
      "activations/layer12_attention_weight_min": -26.034622192382812,
      "activations/layer13_attention_weight_max": 56.40751266479492,
      "activations/layer13_attention_weight_min": -34.98899459838867,
      "activations/layer14_attention_weight_max": 61.55995559692383,
      "activations/layer14_attention_weight_min": -35.437557220458984,
      "activations/layer15_attention_weight_max": 45.43217086791992,
      "activations/layer15_attention_weight_min": -29.425321578979492,
      "activations/layer16_attention_weight_max": 37.98324966430664,
      "activations/layer16_attention_weight_min": -27.117263793945312,
      "activations/layer17_attention_weight_max": 55.55169677734375,
      "activations/layer17_attention_weight_min": -42.1061897277832,
      "activations/layer18_attention_weight_max": 52.90885925292969,
      "activations/layer18_attention_weight_min": -35.67720031738281,
      "activations/layer19_attention_weight_max": 25.15985679626465,
      "activations/layer19_attention_weight_min": -21.0489444732666,
      "activations/layer1_attention_weight_max": 17.309978485107422,
      "activations/layer1_attention_weight_min": -16.356781005859375,
      "activations/layer20_attention_weight_max": 22.60814094543457,
      "activations/layer20_attention_weight_min": -21.9466609954834,
      "activations/layer21_attention_weight_max": 36.78380584716797,
      "activations/layer21_attention_weight_min": -26.484813690185547,
      "activations/layer22_attention_weight_max": 33.39759063720703,
      "activations/layer22_attention_weight_min": -24.77349090576172,
      "activations/layer23_attention_weight_max": 36.13072204589844,
      "activations/layer23_attention_weight_min": -22.939382553100586,
      "activations/layer2_attention_weight_max": 31.11852264404297,
      "activations/layer2_attention_weight_min": -28.737178802490234,
      "activations/layer3_attention_weight_max": 82.48210906982422,
      "activations/layer3_attention_weight_min": -81.8817367553711,
      "activations/layer4_attention_weight_max": 100.80601501464844,
      "activations/layer4_attention_weight_min": -87.605224609375,
      "activations/layer5_attention_weight_max": 97.19200134277344,
      "activations/layer5_attention_weight_min": -82.39132690429688,
      "activations/layer6_attention_weight_max": 62.785743713378906,
      "activations/layer6_attention_weight_min": -52.74649429321289,
      "activations/layer7_attention_weight_max": 81.70772552490234,
      "activations/layer7_attention_weight_min": -70.27375793457031,
      "activations/layer8_attention_weight_max": 49.850643157958984,
      "activations/layer8_attention_weight_min": -46.0826301574707,
      "activations/layer9_attention_weight_max": 60.27993392944336,
      "activations/layer9_attention_weight_min": -48.613948822021484,
      "epoch": 6.76,
      "learning_rate": 0.00010746628787878788,
      "loss": 2.8247,
      "step": 116400
    },
    {
      "activations/layer0_attention_weight_max": 15.240861892700195,
      "activations/layer0_attention_weight_min": -14.650287628173828,
      "activations/layer10_attention_weight_max": 38.186893463134766,
      "activations/layer10_attention_weight_min": -33.1825065612793,
      "activations/layer11_attention_weight_max": 30.932418823242188,
      "activations/layer11_attention_weight_min": -29.33502197265625,
      "activations/layer12_attention_weight_max": 18.665380477905273,
      "activations/layer12_attention_weight_min": -25.686769485473633,
      "activations/layer13_attention_weight_max": 41.43354415893555,
      "activations/layer13_attention_weight_min": -29.522464752197266,
      "activations/layer14_attention_weight_max": 46.25595474243164,
      "activations/layer14_attention_weight_min": -29.166484832763672,
      "activations/layer15_attention_weight_max": 41.10454559326172,
      "activations/layer15_attention_weight_min": -29.471038818359375,
      "activations/layer16_attention_weight_max": 33.44190216064453,
      "activations/layer16_attention_weight_min": -25.881444931030273,
      "activations/layer17_attention_weight_max": 56.40212631225586,
      "activations/layer17_attention_weight_min": -41.10778045654297,
      "activations/layer18_attention_weight_max": 49.5964469909668,
      "activations/layer18_attention_weight_min": -34.45835494995117,
      "activations/layer19_attention_weight_max": 20.600486755371094,
      "activations/layer19_attention_weight_min": -20.433334350585938,
      "activations/layer1_attention_weight_max": 17.122936248779297,
      "activations/layer1_attention_weight_min": -14.0000638961792,
      "activations/layer20_attention_weight_max": 21.85901641845703,
      "activations/layer20_attention_weight_min": -20.08041763305664,
      "activations/layer21_attention_weight_max": 43.952545166015625,
      "activations/layer21_attention_weight_min": -23.995254516601562,
      "activations/layer22_attention_weight_max": 36.01956558227539,
      "activations/layer22_attention_weight_min": -26.0610294342041,
      "activations/layer23_attention_weight_max": 30.690113067626953,
      "activations/layer23_attention_weight_min": -20.67355728149414,
      "activations/layer2_attention_weight_max": 30.767398834228516,
      "activations/layer2_attention_weight_min": -29.08820343017578,
      "activations/layer3_attention_weight_max": 81.62590789794922,
      "activations/layer3_attention_weight_min": -83.9462890625,
      "activations/layer4_attention_weight_max": 96.00963592529297,
      "activations/layer4_attention_weight_min": -87.11812591552734,
      "activations/layer5_attention_weight_max": 84.52681732177734,
      "activations/layer5_attention_weight_min": -80.47857666015625,
      "activations/layer6_attention_weight_max": 54.197776794433594,
      "activations/layer6_attention_weight_min": -50.126224517822266,
      "activations/layer7_attention_weight_max": 70.86304473876953,
      "activations/layer7_attention_weight_min": -67.84771728515625,
      "activations/layer8_attention_weight_max": 41.6837272644043,
      "activations/layer8_attention_weight_min": -42.90541076660156,
      "activations/layer9_attention_weight_max": 47.6329231262207,
      "activations/layer9_attention_weight_min": -47.97774124145508,
      "epoch": 6.77,
      "learning_rate": 0.00010744734848484847,
      "loss": 2.8233,
      "step": 116450
    },
    {
      "activations/layer0_attention_weight_max": 15.638932228088379,
      "activations/layer0_attention_weight_min": -14.492619514465332,
      "activations/layer10_attention_weight_max": 45.92218780517578,
      "activations/layer10_attention_weight_min": -40.51361083984375,
      "activations/layer11_attention_weight_max": 35.05835723876953,
      "activations/layer11_attention_weight_min": -31.75472640991211,
      "activations/layer12_attention_weight_max": 20.91109848022461,
      "activations/layer12_attention_weight_min": -25.43275260925293,
      "activations/layer13_attention_weight_max": 48.534156799316406,
      "activations/layer13_attention_weight_min": -38.37947082519531,
      "activations/layer14_attention_weight_max": 54.916385650634766,
      "activations/layer14_attention_weight_min": -33.29114532470703,
      "activations/layer15_attention_weight_max": 41.944461822509766,
      "activations/layer15_attention_weight_min": -28.999740600585938,
      "activations/layer16_attention_weight_max": 34.187034606933594,
      "activations/layer16_attention_weight_min": -27.368534088134766,
      "activations/layer17_attention_weight_max": 59.6486701965332,
      "activations/layer17_attention_weight_min": -41.688880920410156,
      "activations/layer18_attention_weight_max": 51.9519157409668,
      "activations/layer18_attention_weight_min": -36.2495002746582,
      "activations/layer19_attention_weight_max": 24.4728946685791,
      "activations/layer19_attention_weight_min": -21.058927536010742,
      "activations/layer1_attention_weight_max": 17.66824722290039,
      "activations/layer1_attention_weight_min": -15.466405868530273,
      "activations/layer20_attention_weight_max": 22.18266487121582,
      "activations/layer20_attention_weight_min": -23.95342445373535,
      "activations/layer21_attention_weight_max": 40.181060791015625,
      "activations/layer21_attention_weight_min": -23.07563591003418,
      "activations/layer22_attention_weight_max": 35.2723274230957,
      "activations/layer22_attention_weight_min": -23.76021957397461,
      "activations/layer23_attention_weight_max": 31.58723258972168,
      "activations/layer23_attention_weight_min": -21.181583404541016,
      "activations/layer2_attention_weight_max": 33.277320861816406,
      "activations/layer2_attention_weight_min": -30.936527252197266,
      "activations/layer3_attention_weight_max": 95.19735717773438,
      "activations/layer3_attention_weight_min": -92.79324340820312,
      "activations/layer4_attention_weight_max": 105.79918670654297,
      "activations/layer4_attention_weight_min": -102.80763244628906,
      "activations/layer5_attention_weight_max": 94.22175598144531,
      "activations/layer5_attention_weight_min": -91.48332977294922,
      "activations/layer6_attention_weight_max": 63.47797393798828,
      "activations/layer6_attention_weight_min": -58.213382720947266,
      "activations/layer7_attention_weight_max": 88.7226791381836,
      "activations/layer7_attention_weight_min": -75.07547760009766,
      "activations/layer8_attention_weight_max": 56.86252212524414,
      "activations/layer8_attention_weight_min": -53.30018997192383,
      "activations/layer9_attention_weight_max": 70.05699920654297,
      "activations/layer9_attention_weight_min": -56.3917350769043,
      "epoch": 6.77,
      "learning_rate": 0.00010742840909090909,
      "loss": 2.8448,
      "step": 116500
    },
    {
      "activations/layer0_attention_weight_max": 16.201011657714844,
      "activations/layer0_attention_weight_min": -14.5363130569458,
      "activations/layer10_attention_weight_max": 39.306461334228516,
      "activations/layer10_attention_weight_min": -35.26057815551758,
      "activations/layer11_attention_weight_max": 31.301490783691406,
      "activations/layer11_attention_weight_min": -29.13014030456543,
      "activations/layer12_attention_weight_max": 19.03015899658203,
      "activations/layer12_attention_weight_min": -27.567073822021484,
      "activations/layer13_attention_weight_max": 43.90355682373047,
      "activations/layer13_attention_weight_min": -28.787378311157227,
      "activations/layer14_attention_weight_max": 51.94770050048828,
      "activations/layer14_attention_weight_min": -32.671077728271484,
      "activations/layer15_attention_weight_max": 47.91781997680664,
      "activations/layer15_attention_weight_min": -30.38372039794922,
      "activations/layer16_attention_weight_max": 32.64350891113281,
      "activations/layer16_attention_weight_min": -26.82364273071289,
      "activations/layer17_attention_weight_max": 52.51190948486328,
      "activations/layer17_attention_weight_min": -42.27878952026367,
      "activations/layer18_attention_weight_max": 44.491947174072266,
      "activations/layer18_attention_weight_min": -34.47026824951172,
      "activations/layer19_attention_weight_max": 19.331632614135742,
      "activations/layer19_attention_weight_min": -19.390668869018555,
      "activations/layer1_attention_weight_max": 17.727388381958008,
      "activations/layer1_attention_weight_min": -14.554123878479004,
      "activations/layer20_attention_weight_max": 21.640125274658203,
      "activations/layer20_attention_weight_min": -21.943662643432617,
      "activations/layer21_attention_weight_max": 31.072813034057617,
      "activations/layer21_attention_weight_min": -24.63463592529297,
      "activations/layer22_attention_weight_max": 29.03605842590332,
      "activations/layer22_attention_weight_min": -24.6982479095459,
      "activations/layer23_attention_weight_max": 29.85328483581543,
      "activations/layer23_attention_weight_min": -21.066001892089844,
      "activations/layer2_attention_weight_max": 31.800865173339844,
      "activations/layer2_attention_weight_min": -30.106853485107422,
      "activations/layer3_attention_weight_max": 83.17794036865234,
      "activations/layer3_attention_weight_min": -89.4952392578125,
      "activations/layer4_attention_weight_max": 98.66902923583984,
      "activations/layer4_attention_weight_min": -97.39427185058594,
      "activations/layer5_attention_weight_max": 90.81874084472656,
      "activations/layer5_attention_weight_min": -86.1212387084961,
      "activations/layer6_attention_weight_max": 53.682132720947266,
      "activations/layer6_attention_weight_min": -52.066429138183594,
      "activations/layer7_attention_weight_max": 69.47985076904297,
      "activations/layer7_attention_weight_min": -70.02415466308594,
      "activations/layer8_attention_weight_max": 43.849605560302734,
      "activations/layer8_attention_weight_min": -44.08403015136719,
      "activations/layer9_attention_weight_max": 50.84760665893555,
      "activations/layer9_attention_weight_min": -48.01609802246094,
      "epoch": 6.77,
      "learning_rate": 0.00010740946969696967,
      "loss": 2.8118,
      "step": 116550
    },
    {
      "activations/layer0_attention_weight_max": 14.818182945251465,
      "activations/layer0_attention_weight_min": -14.985917091369629,
      "activations/layer10_attention_weight_max": 36.761226654052734,
      "activations/layer10_attention_weight_min": -35.151607513427734,
      "activations/layer11_attention_weight_max": 28.073894500732422,
      "activations/layer11_attention_weight_min": -29.80970573425293,
      "activations/layer12_attention_weight_max": 18.914955139160156,
      "activations/layer12_attention_weight_min": -26.781818389892578,
      "activations/layer13_attention_weight_max": 45.732276916503906,
      "activations/layer13_attention_weight_min": -31.234766006469727,
      "activations/layer14_attention_weight_max": 37.14472579956055,
      "activations/layer14_attention_weight_min": -29.098064422607422,
      "activations/layer15_attention_weight_max": 36.46151351928711,
      "activations/layer15_attention_weight_min": -28.801830291748047,
      "activations/layer16_attention_weight_max": 31.391357421875,
      "activations/layer16_attention_weight_min": -27.60885238647461,
      "activations/layer17_attention_weight_max": 54.04413604736328,
      "activations/layer17_attention_weight_min": -40.9015998840332,
      "activations/layer18_attention_weight_max": 51.98131561279297,
      "activations/layer18_attention_weight_min": -32.45040512084961,
      "activations/layer19_attention_weight_max": 21.602859497070312,
      "activations/layer19_attention_weight_min": -21.082496643066406,
      "activations/layer1_attention_weight_max": 16.54945182800293,
      "activations/layer1_attention_weight_min": -17.11576271057129,
      "activations/layer20_attention_weight_max": 23.17003631591797,
      "activations/layer20_attention_weight_min": -23.96002769470215,
      "activations/layer21_attention_weight_max": 42.75983810424805,
      "activations/layer21_attention_weight_min": -24.36363983154297,
      "activations/layer22_attention_weight_max": 31.34477996826172,
      "activations/layer22_attention_weight_min": -25.43185806274414,
      "activations/layer23_attention_weight_max": 34.98430633544922,
      "activations/layer23_attention_weight_min": -20.543169021606445,
      "activations/layer2_attention_weight_max": 30.146575927734375,
      "activations/layer2_attention_weight_min": -27.68610191345215,
      "activations/layer3_attention_weight_max": 78.06317901611328,
      "activations/layer3_attention_weight_min": -81.30551147460938,
      "activations/layer4_attention_weight_max": 94.9288558959961,
      "activations/layer4_attention_weight_min": -90.86943817138672,
      "activations/layer5_attention_weight_max": 85.35232543945312,
      "activations/layer5_attention_weight_min": -83.73307037353516,
      "activations/layer6_attention_weight_max": 53.0309944152832,
      "activations/layer6_attention_weight_min": -51.21473693847656,
      "activations/layer7_attention_weight_max": 69.99681091308594,
      "activations/layer7_attention_weight_min": -69.51106262207031,
      "activations/layer8_attention_weight_max": 42.34278869628906,
      "activations/layer8_attention_weight_min": -41.31425857543945,
      "activations/layer9_attention_weight_max": 47.68291473388672,
      "activations/layer9_attention_weight_min": -46.29665756225586,
      "epoch": 6.78,
      "learning_rate": 0.00010739053030303029,
      "loss": 2.8181,
      "step": 116600
    },
    {
      "activations/layer0_attention_weight_max": 15.296351432800293,
      "activations/layer0_attention_weight_min": -15.03638744354248,
      "activations/layer10_attention_weight_max": 37.297096252441406,
      "activations/layer10_attention_weight_min": -33.905982971191406,
      "activations/layer11_attention_weight_max": 30.149290084838867,
      "activations/layer11_attention_weight_min": -29.070905685424805,
      "activations/layer12_attention_weight_max": 27.652362823486328,
      "activations/layer12_attention_weight_min": -24.675724029541016,
      "activations/layer13_attention_weight_max": 46.15909957885742,
      "activations/layer13_attention_weight_min": -36.38935089111328,
      "activations/layer14_attention_weight_max": 45.26795959472656,
      "activations/layer14_attention_weight_min": -32.07842254638672,
      "activations/layer15_attention_weight_max": 36.16765594482422,
      "activations/layer15_attention_weight_min": -34.823368072509766,
      "activations/layer16_attention_weight_max": 33.43077850341797,
      "activations/layer16_attention_weight_min": -29.430408477783203,
      "activations/layer17_attention_weight_max": 53.1407585144043,
      "activations/layer17_attention_weight_min": -42.28778839111328,
      "activations/layer18_attention_weight_max": 47.392024993896484,
      "activations/layer18_attention_weight_min": -35.945045471191406,
      "activations/layer19_attention_weight_max": 23.988630294799805,
      "activations/layer19_attention_weight_min": -21.067468643188477,
      "activations/layer1_attention_weight_max": 18.153003692626953,
      "activations/layer1_attention_weight_min": -17.43181800842285,
      "activations/layer20_attention_weight_max": 22.246845245361328,
      "activations/layer20_attention_weight_min": -22.406164169311523,
      "activations/layer21_attention_weight_max": 43.86775588989258,
      "activations/layer21_attention_weight_min": -26.75111961364746,
      "activations/layer22_attention_weight_max": 33.39032745361328,
      "activations/layer22_attention_weight_min": -26.30779457092285,
      "activations/layer23_attention_weight_max": 33.30939483642578,
      "activations/layer23_attention_weight_min": -20.255760192871094,
      "activations/layer2_attention_weight_max": 31.178165435791016,
      "activations/layer2_attention_weight_min": -30.141822814941406,
      "activations/layer3_attention_weight_max": 85.61747741699219,
      "activations/layer3_attention_weight_min": -88.29930877685547,
      "activations/layer4_attention_weight_max": 99.4903793334961,
      "activations/layer4_attention_weight_min": -93.60743713378906,
      "activations/layer5_attention_weight_max": 93.8775634765625,
      "activations/layer5_attention_weight_min": -85.20260620117188,
      "activations/layer6_attention_weight_max": 59.18495559692383,
      "activations/layer6_attention_weight_min": -51.99272918701172,
      "activations/layer7_attention_weight_max": 74.19569396972656,
      "activations/layer7_attention_weight_min": -71.83869934082031,
      "activations/layer8_attention_weight_max": 42.134395599365234,
      "activations/layer8_attention_weight_min": -42.2905158996582,
      "activations/layer9_attention_weight_max": 46.00778579711914,
      "activations/layer9_attention_weight_min": -47.13275909423828,
      "epoch": 6.78,
      "learning_rate": 0.0001073715909090909,
      "loss": 2.8165,
      "step": 116650
    },
    {
      "activations/layer0_attention_weight_max": 15.50414752960205,
      "activations/layer0_attention_weight_min": -14.89995002746582,
      "activations/layer10_attention_weight_max": 36.56454849243164,
      "activations/layer10_attention_weight_min": -36.238807678222656,
      "activations/layer11_attention_weight_max": 30.897993087768555,
      "activations/layer11_attention_weight_min": -29.037296295166016,
      "activations/layer12_attention_weight_max": 19.65850830078125,
      "activations/layer12_attention_weight_min": -25.505216598510742,
      "activations/layer13_attention_weight_max": 41.3151969909668,
      "activations/layer13_attention_weight_min": -34.271793365478516,
      "activations/layer14_attention_weight_max": 40.21449279785156,
      "activations/layer14_attention_weight_min": -29.966344833374023,
      "activations/layer15_attention_weight_max": 36.403751373291016,
      "activations/layer15_attention_weight_min": -28.999582290649414,
      "activations/layer16_attention_weight_max": 34.609954833984375,
      "activations/layer16_attention_weight_min": -27.149187088012695,
      "activations/layer17_attention_weight_max": 51.809547424316406,
      "activations/layer17_attention_weight_min": -40.92616271972656,
      "activations/layer18_attention_weight_max": 45.87943649291992,
      "activations/layer18_attention_weight_min": -33.586814880371094,
      "activations/layer19_attention_weight_max": 22.747465133666992,
      "activations/layer19_attention_weight_min": -19.137971878051758,
      "activations/layer1_attention_weight_max": 16.692325592041016,
      "activations/layer1_attention_weight_min": -15.22524642944336,
      "activations/layer20_attention_weight_max": 22.14966583251953,
      "activations/layer20_attention_weight_min": -23.25811004638672,
      "activations/layer21_attention_weight_max": 38.72953796386719,
      "activations/layer21_attention_weight_min": -24.910484313964844,
      "activations/layer22_attention_weight_max": 33.19435501098633,
      "activations/layer22_attention_weight_min": -25.50583267211914,
      "activations/layer23_attention_weight_max": 36.25095748901367,
      "activations/layer23_attention_weight_min": -20.52477264404297,
      "activations/layer2_attention_weight_max": 31.392772674560547,
      "activations/layer2_attention_weight_min": -30.12557029724121,
      "activations/layer3_attention_weight_max": 87.0836410522461,
      "activations/layer3_attention_weight_min": -90.86246490478516,
      "activations/layer4_attention_weight_max": 99.83586883544922,
      "activations/layer4_attention_weight_min": -95.03788757324219,
      "activations/layer5_attention_weight_max": 93.92633056640625,
      "activations/layer5_attention_weight_min": -86.74919128417969,
      "activations/layer6_attention_weight_max": 56.26637649536133,
      "activations/layer6_attention_weight_min": -52.97639846801758,
      "activations/layer7_attention_weight_max": 72.46331787109375,
      "activations/layer7_attention_weight_min": -71.15338897705078,
      "activations/layer8_attention_weight_max": 43.534603118896484,
      "activations/layer8_attention_weight_min": -46.89624786376953,
      "activations/layer9_attention_weight_max": 49.169612884521484,
      "activations/layer9_attention_weight_min": -50.803401947021484,
      "epoch": 6.78,
      "learning_rate": 0.0001073526515151515,
      "loss": 2.8261,
      "step": 116700
    },
    {
      "activations/layer0_attention_weight_max": 15.47996997833252,
      "activations/layer0_attention_weight_min": -14.737548828125,
      "activations/layer10_attention_weight_max": 38.09104537963867,
      "activations/layer10_attention_weight_min": -35.58911895751953,
      "activations/layer11_attention_weight_max": 31.67224884033203,
      "activations/layer11_attention_weight_min": -31.5489501953125,
      "activations/layer12_attention_weight_max": 20.530258178710938,
      "activations/layer12_attention_weight_min": -24.210203170776367,
      "activations/layer13_attention_weight_max": 44.50190734863281,
      "activations/layer13_attention_weight_min": -29.255207061767578,
      "activations/layer14_attention_weight_max": 38.91563034057617,
      "activations/layer14_attention_weight_min": -29.373699188232422,
      "activations/layer15_attention_weight_max": 34.95206832885742,
      "activations/layer15_attention_weight_min": -29.445589065551758,
      "activations/layer16_attention_weight_max": 33.92317581176758,
      "activations/layer16_attention_weight_min": -27.107547760009766,
      "activations/layer17_attention_weight_max": 56.57777786254883,
      "activations/layer17_attention_weight_min": -41.41316223144531,
      "activations/layer18_attention_weight_max": 49.28787612915039,
      "activations/layer18_attention_weight_min": -35.36200714111328,
      "activations/layer19_attention_weight_max": 20.276752471923828,
      "activations/layer19_attention_weight_min": -21.152936935424805,
      "activations/layer1_attention_weight_max": 16.31492805480957,
      "activations/layer1_attention_weight_min": -14.668122291564941,
      "activations/layer20_attention_weight_max": 20.970779418945312,
      "activations/layer20_attention_weight_min": -22.044294357299805,
      "activations/layer21_attention_weight_max": 39.26538848876953,
      "activations/layer21_attention_weight_min": -24.943496704101562,
      "activations/layer22_attention_weight_max": 35.62683868408203,
      "activations/layer22_attention_weight_min": -25.83171844482422,
      "activations/layer23_attention_weight_max": 32.85931396484375,
      "activations/layer23_attention_weight_min": -19.80827522277832,
      "activations/layer2_attention_weight_max": 31.15688705444336,
      "activations/layer2_attention_weight_min": -30.042539596557617,
      "activations/layer3_attention_weight_max": 85.58576202392578,
      "activations/layer3_attention_weight_min": -85.88243103027344,
      "activations/layer4_attention_weight_max": 101.47606658935547,
      "activations/layer4_attention_weight_min": -92.48860931396484,
      "activations/layer5_attention_weight_max": 96.82945251464844,
      "activations/layer5_attention_weight_min": -88.79264831542969,
      "activations/layer6_attention_weight_max": 59.462310791015625,
      "activations/layer6_attention_weight_min": -55.268577575683594,
      "activations/layer7_attention_weight_max": 77.76113891601562,
      "activations/layer7_attention_weight_min": -74.56352996826172,
      "activations/layer8_attention_weight_max": 46.0628547668457,
      "activations/layer8_attention_weight_min": -45.48537063598633,
      "activations/layer9_attention_weight_max": 54.41841506958008,
      "activations/layer9_attention_weight_min": -50.56428527832031,
      "epoch": 6.78,
      "learning_rate": 0.00010733371212121211,
      "loss": 2.8107,
      "step": 116750
    },
    {
      "activations/layer0_attention_weight_max": 14.499784469604492,
      "activations/layer0_attention_weight_min": -14.729022026062012,
      "activations/layer10_attention_weight_max": 34.89134216308594,
      "activations/layer10_attention_weight_min": -32.724220275878906,
      "activations/layer11_attention_weight_max": 28.478612899780273,
      "activations/layer11_attention_weight_min": -29.364242553710938,
      "activations/layer12_attention_weight_max": 21.08759307861328,
      "activations/layer12_attention_weight_min": -24.47210121154785,
      "activations/layer13_attention_weight_max": 46.33540344238281,
      "activations/layer13_attention_weight_min": -32.84963607788086,
      "activations/layer14_attention_weight_max": 38.529510498046875,
      "activations/layer14_attention_weight_min": -28.744516372680664,
      "activations/layer15_attention_weight_max": 36.095916748046875,
      "activations/layer15_attention_weight_min": -28.883943557739258,
      "activations/layer16_attention_weight_max": 32.157344818115234,
      "activations/layer16_attention_weight_min": -25.658605575561523,
      "activations/layer17_attention_weight_max": 56.250404357910156,
      "activations/layer17_attention_weight_min": -40.67343521118164,
      "activations/layer18_attention_weight_max": 47.03010177612305,
      "activations/layer18_attention_weight_min": -31.868961334228516,
      "activations/layer19_attention_weight_max": 25.094013214111328,
      "activations/layer19_attention_weight_min": -20.359638214111328,
      "activations/layer1_attention_weight_max": 16.665433883666992,
      "activations/layer1_attention_weight_min": -15.463679313659668,
      "activations/layer20_attention_weight_max": 23.863441467285156,
      "activations/layer20_attention_weight_min": -21.666385650634766,
      "activations/layer21_attention_weight_max": 40.96567916870117,
      "activations/layer21_attention_weight_min": -24.77511978149414,
      "activations/layer22_attention_weight_max": 34.80693435668945,
      "activations/layer22_attention_weight_min": -23.844987869262695,
      "activations/layer23_attention_weight_max": 35.898983001708984,
      "activations/layer23_attention_weight_min": -21.634122848510742,
      "activations/layer2_attention_weight_max": 31.621183395385742,
      "activations/layer2_attention_weight_min": -30.00339126586914,
      "activations/layer3_attention_weight_max": 83.50174713134766,
      "activations/layer3_attention_weight_min": -85.63124084472656,
      "activations/layer4_attention_weight_max": 95.45995330810547,
      "activations/layer4_attention_weight_min": -90.66854095458984,
      "activations/layer5_attention_weight_max": 90.89009857177734,
      "activations/layer5_attention_weight_min": -85.42909240722656,
      "activations/layer6_attention_weight_max": 54.95660400390625,
      "activations/layer6_attention_weight_min": -51.81993865966797,
      "activations/layer7_attention_weight_max": 72.69745635986328,
      "activations/layer7_attention_weight_min": -69.65739440917969,
      "activations/layer8_attention_weight_max": 43.549781799316406,
      "activations/layer8_attention_weight_min": -45.17482376098633,
      "activations/layer9_attention_weight_max": 49.04025650024414,
      "activations/layer9_attention_weight_min": -45.55548858642578,
      "epoch": 6.79,
      "learning_rate": 0.00010731477272727272,
      "loss": 2.8267,
      "step": 116800
    },
    {
      "activations/layer0_attention_weight_max": 15.056107521057129,
      "activations/layer0_attention_weight_min": -14.65869426727295,
      "activations/layer10_attention_weight_max": 40.20888137817383,
      "activations/layer10_attention_weight_min": -36.53461837768555,
      "activations/layer11_attention_weight_max": 33.27391052246094,
      "activations/layer11_attention_weight_min": -30.008800506591797,
      "activations/layer12_attention_weight_max": 27.042667388916016,
      "activations/layer12_attention_weight_min": -29.299863815307617,
      "activations/layer13_attention_weight_max": 41.67192459106445,
      "activations/layer13_attention_weight_min": -33.440670013427734,
      "activations/layer14_attention_weight_max": 37.62330627441406,
      "activations/layer14_attention_weight_min": -31.19062042236328,
      "activations/layer15_attention_weight_max": 37.03451919555664,
      "activations/layer15_attention_weight_min": -30.923948287963867,
      "activations/layer16_attention_weight_max": 31.41124153137207,
      "activations/layer16_attention_weight_min": -25.44407081604004,
      "activations/layer17_attention_weight_max": 56.299964904785156,
      "activations/layer17_attention_weight_min": -40.56455612182617,
      "activations/layer18_attention_weight_max": 44.93429946899414,
      "activations/layer18_attention_weight_min": -34.52397155761719,
      "activations/layer19_attention_weight_max": 19.384855270385742,
      "activations/layer19_attention_weight_min": -19.555253982543945,
      "activations/layer1_attention_weight_max": 17.213851928710938,
      "activations/layer1_attention_weight_min": -14.404729843139648,
      "activations/layer20_attention_weight_max": 20.356040954589844,
      "activations/layer20_attention_weight_min": -23.706132888793945,
      "activations/layer21_attention_weight_max": 36.78254318237305,
      "activations/layer21_attention_weight_min": -23.72355842590332,
      "activations/layer22_attention_weight_max": 34.54312515258789,
      "activations/layer22_attention_weight_min": -25.54863166809082,
      "activations/layer23_attention_weight_max": 32.18494415283203,
      "activations/layer23_attention_weight_min": -21.01825714111328,
      "activations/layer2_attention_weight_max": 32.79830551147461,
      "activations/layer2_attention_weight_min": -30.48242950439453,
      "activations/layer3_attention_weight_max": 89.30377197265625,
      "activations/layer3_attention_weight_min": -87.55803680419922,
      "activations/layer4_attention_weight_max": 105.31353759765625,
      "activations/layer4_attention_weight_min": -96.74873352050781,
      "activations/layer5_attention_weight_max": 100.14083099365234,
      "activations/layer5_attention_weight_min": -86.72747039794922,
      "activations/layer6_attention_weight_max": 59.50345230102539,
      "activations/layer6_attention_weight_min": -54.46523666381836,
      "activations/layer7_attention_weight_max": 80.89313507080078,
      "activations/layer7_attention_weight_min": -72.49193572998047,
      "activations/layer8_attention_weight_max": 47.34871292114258,
      "activations/layer8_attention_weight_min": -44.956817626953125,
      "activations/layer9_attention_weight_max": 56.15102005004883,
      "activations/layer9_attention_weight_min": -53.40321731567383,
      "epoch": 6.79,
      "learning_rate": 0.00010729583333333332,
      "loss": 2.8272,
      "step": 116850
    },
    {
      "activations/layer0_attention_weight_max": 15.097258567810059,
      "activations/layer0_attention_weight_min": -14.643547058105469,
      "activations/layer10_attention_weight_max": 37.40190887451172,
      "activations/layer10_attention_weight_min": -35.709144592285156,
      "activations/layer11_attention_weight_max": 30.932945251464844,
      "activations/layer11_attention_weight_min": -28.507709503173828,
      "activations/layer12_attention_weight_max": 23.093503952026367,
      "activations/layer12_attention_weight_min": -31.14533042907715,
      "activations/layer13_attention_weight_max": 44.68601608276367,
      "activations/layer13_attention_weight_min": -35.21038818359375,
      "activations/layer14_attention_weight_max": 37.63383483886719,
      "activations/layer14_attention_weight_min": -31.1826229095459,
      "activations/layer15_attention_weight_max": 36.009891510009766,
      "activations/layer15_attention_weight_min": -28.82520294189453,
      "activations/layer16_attention_weight_max": 31.54227638244629,
      "activations/layer16_attention_weight_min": -28.866100311279297,
      "activations/layer17_attention_weight_max": 55.50056457519531,
      "activations/layer17_attention_weight_min": -46.51509094238281,
      "activations/layer18_attention_weight_max": 48.95454788208008,
      "activations/layer18_attention_weight_min": -35.944854736328125,
      "activations/layer19_attention_weight_max": 23.573684692382812,
      "activations/layer19_attention_weight_min": -22.15297508239746,
      "activations/layer1_attention_weight_max": 18.3106689453125,
      "activations/layer1_attention_weight_min": -14.605291366577148,
      "activations/layer20_attention_weight_max": 22.24286460876465,
      "activations/layer20_attention_weight_min": -23.21975326538086,
      "activations/layer21_attention_weight_max": 39.894073486328125,
      "activations/layer21_attention_weight_min": -25.66155242919922,
      "activations/layer22_attention_weight_max": 31.48603057861328,
      "activations/layer22_attention_weight_min": -25.645477294921875,
      "activations/layer23_attention_weight_max": 34.7108154296875,
      "activations/layer23_attention_weight_min": -19.95314598083496,
      "activations/layer2_attention_weight_max": 32.42525863647461,
      "activations/layer2_attention_weight_min": -30.61408042907715,
      "activations/layer3_attention_weight_max": 87.3841323852539,
      "activations/layer3_attention_weight_min": -86.9683837890625,
      "activations/layer4_attention_weight_max": 105.3304214477539,
      "activations/layer4_attention_weight_min": -97.57657623291016,
      "activations/layer5_attention_weight_max": 96.85818481445312,
      "activations/layer5_attention_weight_min": -85.22078704833984,
      "activations/layer6_attention_weight_max": 57.781776428222656,
      "activations/layer6_attention_weight_min": -52.729393005371094,
      "activations/layer7_attention_weight_max": 74.0468521118164,
      "activations/layer7_attention_weight_min": -71.81864166259766,
      "activations/layer8_attention_weight_max": 45.889896392822266,
      "activations/layer8_attention_weight_min": -49.69805145263672,
      "activations/layer9_attention_weight_max": 54.94777297973633,
      "activations/layer9_attention_weight_min": -49.924285888671875,
      "epoch": 6.79,
      "learning_rate": 0.00010727689393939394,
      "loss": 2.8431,
      "step": 116900
    },
    {
      "activations/layer0_attention_weight_max": 16.200563430786133,
      "activations/layer0_attention_weight_min": -15.045090675354004,
      "activations/layer10_attention_weight_max": 42.12523651123047,
      "activations/layer10_attention_weight_min": -36.87739944458008,
      "activations/layer11_attention_weight_max": 32.53263854980469,
      "activations/layer11_attention_weight_min": -31.15644645690918,
      "activations/layer12_attention_weight_max": 21.234548568725586,
      "activations/layer12_attention_weight_min": -30.701461791992188,
      "activations/layer13_attention_weight_max": 56.957672119140625,
      "activations/layer13_attention_weight_min": -37.876930236816406,
      "activations/layer14_attention_weight_max": 48.27630615234375,
      "activations/layer14_attention_weight_min": -30.80133819580078,
      "activations/layer15_attention_weight_max": 45.45479965209961,
      "activations/layer15_attention_weight_min": -30.613643646240234,
      "activations/layer16_attention_weight_max": 36.81886291503906,
      "activations/layer16_attention_weight_min": -29.343950271606445,
      "activations/layer17_attention_weight_max": 59.22987365722656,
      "activations/layer17_attention_weight_min": -40.45961380004883,
      "activations/layer18_attention_weight_max": 50.31178665161133,
      "activations/layer18_attention_weight_min": -35.01902770996094,
      "activations/layer19_attention_weight_max": 25.72356605529785,
      "activations/layer19_attention_weight_min": -19.889549255371094,
      "activations/layer1_attention_weight_max": 16.37779426574707,
      "activations/layer1_attention_weight_min": -17.749755859375,
      "activations/layer20_attention_weight_max": 24.765745162963867,
      "activations/layer20_attention_weight_min": -20.325342178344727,
      "activations/layer21_attention_weight_max": 51.589500427246094,
      "activations/layer21_attention_weight_min": -23.59234046936035,
      "activations/layer22_attention_weight_max": 37.233421325683594,
      "activations/layer22_attention_weight_min": -23.2886962890625,
      "activations/layer23_attention_weight_max": 35.80965042114258,
      "activations/layer23_attention_weight_min": -18.698352813720703,
      "activations/layer2_attention_weight_max": 35.61241912841797,
      "activations/layer2_attention_weight_min": -34.064701080322266,
      "activations/layer3_attention_weight_max": 98.08413696289062,
      "activations/layer3_attention_weight_min": -98.4112548828125,
      "activations/layer4_attention_weight_max": 105.33100891113281,
      "activations/layer4_attention_weight_min": -102.89556884765625,
      "activations/layer5_attention_weight_max": 99.53630828857422,
      "activations/layer5_attention_weight_min": -92.97959899902344,
      "activations/layer6_attention_weight_max": 60.14335250854492,
      "activations/layer6_attention_weight_min": -57.137840270996094,
      "activations/layer7_attention_weight_max": 82.70384216308594,
      "activations/layer7_attention_weight_min": -73.14682006835938,
      "activations/layer8_attention_weight_max": 53.12450408935547,
      "activations/layer8_attention_weight_min": -46.13874053955078,
      "activations/layer9_attention_weight_max": 64.45223236083984,
      "activations/layer9_attention_weight_min": -52.805843353271484,
      "epoch": 6.8,
      "learning_rate": 0.00010725795454545454,
      "loss": 2.8282,
      "step": 116950
    },
    {
      "activations/layer0_attention_weight_max": 15.331427574157715,
      "activations/layer0_attention_weight_min": -15.254989624023438,
      "activations/layer10_attention_weight_max": 44.98067855834961,
      "activations/layer10_attention_weight_min": -37.94190979003906,
      "activations/layer11_attention_weight_max": 41.8897705078125,
      "activations/layer11_attention_weight_min": -31.819957733154297,
      "activations/layer12_attention_weight_max": 31.061235427856445,
      "activations/layer12_attention_weight_min": -26.871177673339844,
      "activations/layer13_attention_weight_max": 46.72501754760742,
      "activations/layer13_attention_weight_min": -37.18679428100586,
      "activations/layer14_attention_weight_max": 42.49347686767578,
      "activations/layer14_attention_weight_min": -30.79424476623535,
      "activations/layer15_attention_weight_max": 36.5133056640625,
      "activations/layer15_attention_weight_min": -29.164628982543945,
      "activations/layer16_attention_weight_max": 31.976627349853516,
      "activations/layer16_attention_weight_min": -26.651418685913086,
      "activations/layer17_attention_weight_max": 59.02888870239258,
      "activations/layer17_attention_weight_min": -44.00862121582031,
      "activations/layer18_attention_weight_max": 46.47486877441406,
      "activations/layer18_attention_weight_min": -34.876129150390625,
      "activations/layer19_attention_weight_max": 24.173093795776367,
      "activations/layer19_attention_weight_min": -18.424856185913086,
      "activations/layer1_attention_weight_max": 17.947586059570312,
      "activations/layer1_attention_weight_min": -16.75019645690918,
      "activations/layer20_attention_weight_max": 22.746068954467773,
      "activations/layer20_attention_weight_min": -19.226652145385742,
      "activations/layer21_attention_weight_max": 38.8350830078125,
      "activations/layer21_attention_weight_min": -23.732877731323242,
      "activations/layer22_attention_weight_max": 36.10271072387695,
      "activations/layer22_attention_weight_min": -25.9100341796875,
      "activations/layer23_attention_weight_max": 32.95991897583008,
      "activations/layer23_attention_weight_min": -20.3519229888916,
      "activations/layer2_attention_weight_max": 34.52610397338867,
      "activations/layer2_attention_weight_min": -30.857704162597656,
      "activations/layer3_attention_weight_max": 84.76515197753906,
      "activations/layer3_attention_weight_min": -85.35200500488281,
      "activations/layer4_attention_weight_max": 99.42604064941406,
      "activations/layer4_attention_weight_min": -90.90321350097656,
      "activations/layer5_attention_weight_max": 98.88121795654297,
      "activations/layer5_attention_weight_min": -84.04378509521484,
      "activations/layer6_attention_weight_max": 60.64229202270508,
      "activations/layer6_attention_weight_min": -55.5224494934082,
      "activations/layer7_attention_weight_max": 92.00370025634766,
      "activations/layer7_attention_weight_min": -75.23079681396484,
      "activations/layer8_attention_weight_max": 57.65240478515625,
      "activations/layer8_attention_weight_min": -43.26311111450195,
      "activations/layer9_attention_weight_max": 82.74186706542969,
      "activations/layer9_attention_weight_min": -55.026851654052734,
      "epoch": 6.8,
      "learning_rate": 0.00010723901515151514,
      "loss": 2.8323,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.5164,
      "eval_samples_per_second": 504.206,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.5164,
      "eval_openwebtext_samples_per_second": 504.206,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 2.0126,
      "eval_wikitext_samples_per_second": 226.571,
      "step": 117000
    },
    {
      "epoch": 6.8,
      "eval_lambada_loss": 2.759765625,
      "eval_lambada_ppl": 15.796140293991145,
      "eval_lambada_runtime": 9.5944,
      "eval_lambada_samples_per_second": 507.484,
      "step": 117000
    },
    {
      "activations/layer0_attention_weight_max": 14.868195533752441,
      "activations/layer0_attention_weight_min": -15.089506149291992,
      "activations/layer10_attention_weight_max": 43.41025161743164,
      "activations/layer10_attention_weight_min": -41.65595626831055,
      "activations/layer11_attention_weight_max": 39.255619049072266,
      "activations/layer11_attention_weight_min": -33.474910736083984,
      "activations/layer12_attention_weight_max": 22.95238494873047,
      "activations/layer12_attention_weight_min": -24.375507354736328,
      "activations/layer13_attention_weight_max": 54.335350036621094,
      "activations/layer13_attention_weight_min": -37.72911834716797,
      "activations/layer14_attention_weight_max": 64.31333923339844,
      "activations/layer14_attention_weight_min": -31.344877243041992,
      "activations/layer15_attention_weight_max": 53.29979705810547,
      "activations/layer15_attention_weight_min": -27.515478134155273,
      "activations/layer16_attention_weight_max": 34.94326400756836,
      "activations/layer16_attention_weight_min": -25.333803176879883,
      "activations/layer17_attention_weight_max": 56.02431106567383,
      "activations/layer17_attention_weight_min": -40.491050720214844,
      "activations/layer18_attention_weight_max": 49.646263122558594,
      "activations/layer18_attention_weight_min": -34.35491180419922,
      "activations/layer19_attention_weight_max": 22.214208602905273,
      "activations/layer19_attention_weight_min": -18.60335350036621,
      "activations/layer1_attention_weight_max": 17.663307189941406,
      "activations/layer1_attention_weight_min": -16.221630096435547,
      "activations/layer20_attention_weight_max": 22.2808895111084,
      "activations/layer20_attention_weight_min": -20.596885681152344,
      "activations/layer21_attention_weight_max": 39.25489807128906,
      "activations/layer21_attention_weight_min": -22.9165096282959,
      "activations/layer22_attention_weight_max": 32.97981643676758,
      "activations/layer22_attention_weight_min": -22.81800079345703,
      "activations/layer23_attention_weight_max": 35.2218017578125,
      "activations/layer23_attention_weight_min": -18.01302146911621,
      "activations/layer2_attention_weight_max": 31.072341918945312,
      "activations/layer2_attention_weight_min": -29.011343002319336,
      "activations/layer3_attention_weight_max": 83.47161102294922,
      "activations/layer3_attention_weight_min": -87.70637512207031,
      "activations/layer4_attention_weight_max": 103.20494842529297,
      "activations/layer4_attention_weight_min": -94.7088851928711,
      "activations/layer5_attention_weight_max": 96.18165588378906,
      "activations/layer5_attention_weight_min": -86.8106918334961,
      "activations/layer6_attention_weight_max": 58.846317291259766,
      "activations/layer6_attention_weight_min": -56.09191131591797,
      "activations/layer7_attention_weight_max": 93.48832702636719,
      "activations/layer7_attention_weight_min": -71.41190338134766,
      "activations/layer8_attention_weight_max": 57.69593811035156,
      "activations/layer8_attention_weight_min": -45.72067642211914,
      "activations/layer9_attention_weight_max": 72.49002075195312,
      "activations/layer9_attention_weight_min": -54.80402755737305,
      "epoch": 6.8,
      "learning_rate": 0.00010722007575757576,
      "loss": 2.8304,
      "step": 117050
    },
    {
      "activations/layer0_attention_weight_max": 15.765462875366211,
      "activations/layer0_attention_weight_min": -14.989177703857422,
      "activations/layer10_attention_weight_max": 37.052032470703125,
      "activations/layer10_attention_weight_min": -33.64040756225586,
      "activations/layer11_attention_weight_max": 29.718994140625,
      "activations/layer11_attention_weight_min": -29.120529174804688,
      "activations/layer12_attention_weight_max": 23.69601821899414,
      "activations/layer12_attention_weight_min": -25.02336883544922,
      "activations/layer13_attention_weight_max": 49.055484771728516,
      "activations/layer13_attention_weight_min": -37.47557067871094,
      "activations/layer14_attention_weight_max": 50.81332015991211,
      "activations/layer14_attention_weight_min": -33.55063247680664,
      "activations/layer15_attention_weight_max": 37.782711029052734,
      "activations/layer15_attention_weight_min": -29.688030242919922,
      "activations/layer16_attention_weight_max": 34.21817398071289,
      "activations/layer16_attention_weight_min": -25.98029136657715,
      "activations/layer17_attention_weight_max": 51.35184097290039,
      "activations/layer17_attention_weight_min": -39.602779388427734,
      "activations/layer18_attention_weight_max": 44.000362396240234,
      "activations/layer18_attention_weight_min": -34.864898681640625,
      "activations/layer19_attention_weight_max": 22.995559692382812,
      "activations/layer19_attention_weight_min": -20.056381225585938,
      "activations/layer1_attention_weight_max": 16.898849487304688,
      "activations/layer1_attention_weight_min": -15.658717155456543,
      "activations/layer20_attention_weight_max": 24.872657775878906,
      "activations/layer20_attention_weight_min": -21.076560974121094,
      "activations/layer21_attention_weight_max": 44.663658142089844,
      "activations/layer21_attention_weight_min": -24.483909606933594,
      "activations/layer22_attention_weight_max": 30.449092864990234,
      "activations/layer22_attention_weight_min": -24.263713836669922,
      "activations/layer23_attention_weight_max": 30.642414093017578,
      "activations/layer23_attention_weight_min": -21.12786865234375,
      "activations/layer2_attention_weight_max": 32.206809997558594,
      "activations/layer2_attention_weight_min": -30.43380355834961,
      "activations/layer3_attention_weight_max": 85.79793548583984,
      "activations/layer3_attention_weight_min": -91.21403503417969,
      "activations/layer4_attention_weight_max": 97.6772232055664,
      "activations/layer4_attention_weight_min": -96.32069396972656,
      "activations/layer5_attention_weight_max": 90.13458251953125,
      "activations/layer5_attention_weight_min": -83.85017395019531,
      "activations/layer6_attention_weight_max": 57.31829071044922,
      "activations/layer6_attention_weight_min": -51.46681213378906,
      "activations/layer7_attention_weight_max": 72.62350463867188,
      "activations/layer7_attention_weight_min": -68.62367248535156,
      "activations/layer8_attention_weight_max": 43.86159896850586,
      "activations/layer8_attention_weight_min": -41.78303527832031,
      "activations/layer9_attention_weight_max": 51.02528762817383,
      "activations/layer9_attention_weight_min": -48.82973861694336,
      "epoch": 6.8,
      "learning_rate": 0.00010720113636363634,
      "loss": 2.84,
      "step": 117100
    },
    {
      "activations/layer0_attention_weight_max": 14.235191345214844,
      "activations/layer0_attention_weight_min": -15.45898723602295,
      "activations/layer10_attention_weight_max": 35.075286865234375,
      "activations/layer10_attention_weight_min": -33.93771743774414,
      "activations/layer11_attention_weight_max": 29.496984481811523,
      "activations/layer11_attention_weight_min": -28.418689727783203,
      "activations/layer12_attention_weight_max": 22.32695198059082,
      "activations/layer12_attention_weight_min": -27.149091720581055,
      "activations/layer13_attention_weight_max": 39.20783233642578,
      "activations/layer13_attention_weight_min": -30.787939071655273,
      "activations/layer14_attention_weight_max": 37.31901550292969,
      "activations/layer14_attention_weight_min": -30.062721252441406,
      "activations/layer15_attention_weight_max": 35.29692459106445,
      "activations/layer15_attention_weight_min": -30.23495101928711,
      "activations/layer16_attention_weight_max": 32.41062545776367,
      "activations/layer16_attention_weight_min": -26.41499137878418,
      "activations/layer17_attention_weight_max": 55.604270935058594,
      "activations/layer17_attention_weight_min": -42.048248291015625,
      "activations/layer18_attention_weight_max": 48.478721618652344,
      "activations/layer18_attention_weight_min": -35.75026321411133,
      "activations/layer19_attention_weight_max": 21.234891891479492,
      "activations/layer19_attention_weight_min": -21.210485458374023,
      "activations/layer1_attention_weight_max": 16.324424743652344,
      "activations/layer1_attention_weight_min": -14.501058578491211,
      "activations/layer20_attention_weight_max": 21.388093948364258,
      "activations/layer20_attention_weight_min": -26.683204650878906,
      "activations/layer21_attention_weight_max": 39.955482482910156,
      "activations/layer21_attention_weight_min": -25.65117645263672,
      "activations/layer22_attention_weight_max": 32.00011444091797,
      "activations/layer22_attention_weight_min": -26.00420379638672,
      "activations/layer23_attention_weight_max": 33.408721923828125,
      "activations/layer23_attention_weight_min": -20.528274536132812,
      "activations/layer2_attention_weight_max": 32.69303512573242,
      "activations/layer2_attention_weight_min": -29.38947296142578,
      "activations/layer3_attention_weight_max": 85.24801635742188,
      "activations/layer3_attention_weight_min": -87.45601654052734,
      "activations/layer4_attention_weight_max": 100.5345458984375,
      "activations/layer4_attention_weight_min": -99.34004974365234,
      "activations/layer5_attention_weight_max": 96.63212585449219,
      "activations/layer5_attention_weight_min": -88.48124694824219,
      "activations/layer6_attention_weight_max": 59.50959014892578,
      "activations/layer6_attention_weight_min": -55.806610107421875,
      "activations/layer7_attention_weight_max": 76.92340087890625,
      "activations/layer7_attention_weight_min": -70.87606048583984,
      "activations/layer8_attention_weight_max": 43.58027267456055,
      "activations/layer8_attention_weight_min": -47.816673278808594,
      "activations/layer9_attention_weight_max": 48.66787338256836,
      "activations/layer9_attention_weight_min": -50.80007553100586,
      "epoch": 6.81,
      "learning_rate": 0.00010718219696969696,
      "loss": 2.8215,
      "step": 117150
    },
    {
      "activations/layer0_attention_weight_max": 15.529305458068848,
      "activations/layer0_attention_weight_min": -15.234002113342285,
      "activations/layer10_attention_weight_max": 35.109657287597656,
      "activations/layer10_attention_weight_min": -35.86111068725586,
      "activations/layer11_attention_weight_max": 28.07742691040039,
      "activations/layer11_attention_weight_min": -30.050628662109375,
      "activations/layer12_attention_weight_max": 19.973875045776367,
      "activations/layer12_attention_weight_min": -23.614540100097656,
      "activations/layer13_attention_weight_max": 45.50454330444336,
      "activations/layer13_attention_weight_min": -34.163909912109375,
      "activations/layer14_attention_weight_max": 38.63886642456055,
      "activations/layer14_attention_weight_min": -27.90043067932129,
      "activations/layer15_attention_weight_max": 33.75298309326172,
      "activations/layer15_attention_weight_min": -29.301273345947266,
      "activations/layer16_attention_weight_max": 31.092012405395508,
      "activations/layer16_attention_weight_min": -26.857349395751953,
      "activations/layer17_attention_weight_max": 49.799896240234375,
      "activations/layer17_attention_weight_min": -42.99879837036133,
      "activations/layer18_attention_weight_max": 45.72016143798828,
      "activations/layer18_attention_weight_min": -34.297271728515625,
      "activations/layer19_attention_weight_max": 23.014432907104492,
      "activations/layer19_attention_weight_min": -18.328964233398438,
      "activations/layer1_attention_weight_max": 16.874956130981445,
      "activations/layer1_attention_weight_min": -17.86363410949707,
      "activations/layer20_attention_weight_max": 24.079479217529297,
      "activations/layer20_attention_weight_min": -20.182933807373047,
      "activations/layer21_attention_weight_max": 36.11934280395508,
      "activations/layer21_attention_weight_min": -23.947952270507812,
      "activations/layer22_attention_weight_max": 32.29191970825195,
      "activations/layer22_attention_weight_min": -24.96026611328125,
      "activations/layer23_attention_weight_max": 30.285293579101562,
      "activations/layer23_attention_weight_min": -21.86276626586914,
      "activations/layer2_attention_weight_max": 32.70732498168945,
      "activations/layer2_attention_weight_min": -29.69539451599121,
      "activations/layer3_attention_weight_max": 81.95576477050781,
      "activations/layer3_attention_weight_min": -83.45308685302734,
      "activations/layer4_attention_weight_max": 99.29637908935547,
      "activations/layer4_attention_weight_min": -93.42964935302734,
      "activations/layer5_attention_weight_max": 92.69469451904297,
      "activations/layer5_attention_weight_min": -87.20588684082031,
      "activations/layer6_attention_weight_max": 55.46057891845703,
      "activations/layer6_attention_weight_min": -53.140724182128906,
      "activations/layer7_attention_weight_max": 86.0577392578125,
      "activations/layer7_attention_weight_min": -74.86212921142578,
      "activations/layer8_attention_weight_max": 46.18715286254883,
      "activations/layer8_attention_weight_min": -46.069576263427734,
      "activations/layer9_attention_weight_max": 56.1893196105957,
      "activations/layer9_attention_weight_min": -48.744388580322266,
      "epoch": 6.81,
      "learning_rate": 0.00010716325757575757,
      "loss": 2.826,
      "step": 117200
    },
    {
      "activations/layer0_attention_weight_max": 14.870879173278809,
      "activations/layer0_attention_weight_min": -15.74935531616211,
      "activations/layer10_attention_weight_max": 40.48048782348633,
      "activations/layer10_attention_weight_min": -34.3829231262207,
      "activations/layer11_attention_weight_max": 31.96706199645996,
      "activations/layer11_attention_weight_min": -29.8154296875,
      "activations/layer12_attention_weight_max": 31.569129943847656,
      "activations/layer12_attention_weight_min": -25.570775985717773,
      "activations/layer13_attention_weight_max": 40.3305778503418,
      "activations/layer13_attention_weight_min": -28.026988983154297,
      "activations/layer14_attention_weight_max": 37.298667907714844,
      "activations/layer14_attention_weight_min": -30.08775520324707,
      "activations/layer15_attention_weight_max": 36.20738220214844,
      "activations/layer15_attention_weight_min": -28.876235961914062,
      "activations/layer16_attention_weight_max": 35.441802978515625,
      "activations/layer16_attention_weight_min": -26.759145736694336,
      "activations/layer17_attention_weight_max": 52.48795700073242,
      "activations/layer17_attention_weight_min": -41.969879150390625,
      "activations/layer18_attention_weight_max": 46.68522644042969,
      "activations/layer18_attention_weight_min": -39.041709899902344,
      "activations/layer19_attention_weight_max": 23.01909828186035,
      "activations/layer19_attention_weight_min": -20.697906494140625,
      "activations/layer1_attention_weight_max": 17.79969596862793,
      "activations/layer1_attention_weight_min": -13.78262996673584,
      "activations/layer20_attention_weight_max": 20.42850685119629,
      "activations/layer20_attention_weight_min": -22.62331771850586,
      "activations/layer21_attention_weight_max": 33.46037292480469,
      "activations/layer21_attention_weight_min": -24.66208267211914,
      "activations/layer22_attention_weight_max": 32.12137222290039,
      "activations/layer22_attention_weight_min": -25.139198303222656,
      "activations/layer23_attention_weight_max": 30.123706817626953,
      "activations/layer23_attention_weight_min": -21.145885467529297,
      "activations/layer2_attention_weight_max": 30.82630729675293,
      "activations/layer2_attention_weight_min": -29.9443416595459,
      "activations/layer3_attention_weight_max": 87.1280746459961,
      "activations/layer3_attention_weight_min": -93.54792022705078,
      "activations/layer4_attention_weight_max": 98.0650405883789,
      "activations/layer4_attention_weight_min": -95.42438507080078,
      "activations/layer5_attention_weight_max": 93.54371643066406,
      "activations/layer5_attention_weight_min": -82.01616668701172,
      "activations/layer6_attention_weight_max": 59.02687454223633,
      "activations/layer6_attention_weight_min": -55.70348358154297,
      "activations/layer7_attention_weight_max": 74.42774963378906,
      "activations/layer7_attention_weight_min": -67.69871520996094,
      "activations/layer8_attention_weight_max": 45.69167709350586,
      "activations/layer8_attention_weight_min": -44.46043395996094,
      "activations/layer9_attention_weight_max": 51.33009719848633,
      "activations/layer9_attention_weight_min": -44.9616584777832,
      "epoch": 6.81,
      "learning_rate": 0.00010714431818181816,
      "loss": 2.8319,
      "step": 117250
    },
    {
      "activations/layer0_attention_weight_max": 14.901941299438477,
      "activations/layer0_attention_weight_min": -14.98534107208252,
      "activations/layer10_attention_weight_max": 35.88957595825195,
      "activations/layer10_attention_weight_min": -33.2519645690918,
      "activations/layer11_attention_weight_max": 29.89236831665039,
      "activations/layer11_attention_weight_min": -28.553855895996094,
      "activations/layer12_attention_weight_max": 18.292278289794922,
      "activations/layer12_attention_weight_min": -26.874025344848633,
      "activations/layer13_attention_weight_max": 44.64265823364258,
      "activations/layer13_attention_weight_min": -33.594173431396484,
      "activations/layer14_attention_weight_max": 56.2159309387207,
      "activations/layer14_attention_weight_min": -37.743778228759766,
      "activations/layer15_attention_weight_max": 39.56631851196289,
      "activations/layer15_attention_weight_min": -31.95516586303711,
      "activations/layer16_attention_weight_max": 32.2113151550293,
      "activations/layer16_attention_weight_min": -28.30129051208496,
      "activations/layer17_attention_weight_max": 53.85584259033203,
      "activations/layer17_attention_weight_min": -39.71300506591797,
      "activations/layer18_attention_weight_max": 49.38957595825195,
      "activations/layer18_attention_weight_min": -34.847286224365234,
      "activations/layer19_attention_weight_max": 21.415788650512695,
      "activations/layer19_attention_weight_min": -20.44916534423828,
      "activations/layer1_attention_weight_max": 16.910640716552734,
      "activations/layer1_attention_weight_min": -16.5604248046875,
      "activations/layer20_attention_weight_max": 22.75560760498047,
      "activations/layer20_attention_weight_min": -21.8564510345459,
      "activations/layer21_attention_weight_max": 39.571353912353516,
      "activations/layer21_attention_weight_min": -25.34354591369629,
      "activations/layer22_attention_weight_max": 35.86370849609375,
      "activations/layer22_attention_weight_min": -25.265968322753906,
      "activations/layer23_attention_weight_max": 35.53801727294922,
      "activations/layer23_attention_weight_min": -22.04817008972168,
      "activations/layer2_attention_weight_max": 31.520170211791992,
      "activations/layer2_attention_weight_min": -28.962661743164062,
      "activations/layer3_attention_weight_max": 85.07672119140625,
      "activations/layer3_attention_weight_min": -88.10345458984375,
      "activations/layer4_attention_weight_max": 97.0454330444336,
      "activations/layer4_attention_weight_min": -92.29701232910156,
      "activations/layer5_attention_weight_max": 88.5091552734375,
      "activations/layer5_attention_weight_min": -80.36002349853516,
      "activations/layer6_attention_weight_max": 54.34153747558594,
      "activations/layer6_attention_weight_min": -51.285648345947266,
      "activations/layer7_attention_weight_max": 71.12592315673828,
      "activations/layer7_attention_weight_min": -68.7616195678711,
      "activations/layer8_attention_weight_max": 43.33205032348633,
      "activations/layer8_attention_weight_min": -40.81875228881836,
      "activations/layer9_attention_weight_max": 51.0639762878418,
      "activations/layer9_attention_weight_min": -47.92254638671875,
      "epoch": 6.82,
      "learning_rate": 0.00010712537878787878,
      "loss": 2.8305,
      "step": 117300
    },
    {
      "activations/layer0_attention_weight_max": 15.331022262573242,
      "activations/layer0_attention_weight_min": -15.674049377441406,
      "activations/layer10_attention_weight_max": 38.21247863769531,
      "activations/layer10_attention_weight_min": -35.98858642578125,
      "activations/layer11_attention_weight_max": 30.522689819335938,
      "activations/layer11_attention_weight_min": -30.204898834228516,
      "activations/layer12_attention_weight_max": 17.75974464416504,
      "activations/layer12_attention_weight_min": -25.007200241088867,
      "activations/layer13_attention_weight_max": 50.1663932800293,
      "activations/layer13_attention_weight_min": -33.134033203125,
      "activations/layer14_attention_weight_max": 38.435585021972656,
      "activations/layer14_attention_weight_min": -30.24135398864746,
      "activations/layer15_attention_weight_max": 37.88504409790039,
      "activations/layer15_attention_weight_min": -29.126127243041992,
      "activations/layer16_attention_weight_max": 30.632585525512695,
      "activations/layer16_attention_weight_min": -28.73287582397461,
      "activations/layer17_attention_weight_max": 52.63982009887695,
      "activations/layer17_attention_weight_min": -42.388641357421875,
      "activations/layer18_attention_weight_max": 44.158233642578125,
      "activations/layer18_attention_weight_min": -33.655296325683594,
      "activations/layer19_attention_weight_max": 20.6544189453125,
      "activations/layer19_attention_weight_min": -20.589807510375977,
      "activations/layer1_attention_weight_max": 17.24745750427246,
      "activations/layer1_attention_weight_min": -16.056522369384766,
      "activations/layer20_attention_weight_max": 19.727462768554688,
      "activations/layer20_attention_weight_min": -20.70294189453125,
      "activations/layer21_attention_weight_max": 37.71651077270508,
      "activations/layer21_attention_weight_min": -25.060314178466797,
      "activations/layer22_attention_weight_max": 29.16461944580078,
      "activations/layer22_attention_weight_min": -22.96976661682129,
      "activations/layer23_attention_weight_max": 31.566009521484375,
      "activations/layer23_attention_weight_min": -20.09127426147461,
      "activations/layer2_attention_weight_max": 30.57329559326172,
      "activations/layer2_attention_weight_min": -30.69615936279297,
      "activations/layer3_attention_weight_max": 85.67741394042969,
      "activations/layer3_attention_weight_min": -90.76685333251953,
      "activations/layer4_attention_weight_max": 101.46468353271484,
      "activations/layer4_attention_weight_min": -96.67473602294922,
      "activations/layer5_attention_weight_max": 92.67707824707031,
      "activations/layer5_attention_weight_min": -88.56710052490234,
      "activations/layer6_attention_weight_max": 55.08957290649414,
      "activations/layer6_attention_weight_min": -54.8592643737793,
      "activations/layer7_attention_weight_max": 71.43949890136719,
      "activations/layer7_attention_weight_min": -70.41658020019531,
      "activations/layer8_attention_weight_max": 43.870731353759766,
      "activations/layer8_attention_weight_min": -46.56843948364258,
      "activations/layer9_attention_weight_max": 50.75133514404297,
      "activations/layer9_attention_weight_min": -47.96205139160156,
      "epoch": 6.82,
      "learning_rate": 0.00010710643939393939,
      "loss": 2.8062,
      "step": 117350
    },
    {
      "activations/layer0_attention_weight_max": 15.19324779510498,
      "activations/layer0_attention_weight_min": -15.519485473632812,
      "activations/layer10_attention_weight_max": 39.35634231567383,
      "activations/layer10_attention_weight_min": -35.89844512939453,
      "activations/layer11_attention_weight_max": 33.00288772583008,
      "activations/layer11_attention_weight_min": -31.027999877929688,
      "activations/layer12_attention_weight_max": 21.942012786865234,
      "activations/layer12_attention_weight_min": -24.49579429626465,
      "activations/layer13_attention_weight_max": 51.453861236572266,
      "activations/layer13_attention_weight_min": -35.69844055175781,
      "activations/layer14_attention_weight_max": 44.37651824951172,
      "activations/layer14_attention_weight_min": -31.446298599243164,
      "activations/layer15_attention_weight_max": 36.063785552978516,
      "activations/layer15_attention_weight_min": -30.58635902404785,
      "activations/layer16_attention_weight_max": 32.48979187011719,
      "activations/layer16_attention_weight_min": -26.349491119384766,
      "activations/layer17_attention_weight_max": 55.350181579589844,
      "activations/layer17_attention_weight_min": -43.476722717285156,
      "activations/layer18_attention_weight_max": 48.3402099609375,
      "activations/layer18_attention_weight_min": -36.447120666503906,
      "activations/layer19_attention_weight_max": 22.753122329711914,
      "activations/layer19_attention_weight_min": -22.05419921875,
      "activations/layer1_attention_weight_max": 17.268400192260742,
      "activations/layer1_attention_weight_min": -15.825139045715332,
      "activations/layer20_attention_weight_max": 22.674480438232422,
      "activations/layer20_attention_weight_min": -20.3547306060791,
      "activations/layer21_attention_weight_max": 43.429718017578125,
      "activations/layer21_attention_weight_min": -26.076583862304688,
      "activations/layer22_attention_weight_max": 33.33439254760742,
      "activations/layer22_attention_weight_min": -25.17318344116211,
      "activations/layer23_attention_weight_max": 36.515689849853516,
      "activations/layer23_attention_weight_min": -21.00634765625,
      "activations/layer2_attention_weight_max": 32.888824462890625,
      "activations/layer2_attention_weight_min": -29.935741424560547,
      "activations/layer3_attention_weight_max": 82.85762786865234,
      "activations/layer3_attention_weight_min": -87.2904281616211,
      "activations/layer4_attention_weight_max": 101.4359359741211,
      "activations/layer4_attention_weight_min": -96.65856170654297,
      "activations/layer5_attention_weight_max": 97.14742279052734,
      "activations/layer5_attention_weight_min": -87.72611999511719,
      "activations/layer6_attention_weight_max": 56.2952995300293,
      "activations/layer6_attention_weight_min": -53.25359344482422,
      "activations/layer7_attention_weight_max": 82.18556213378906,
      "activations/layer7_attention_weight_min": -70.11717987060547,
      "activations/layer8_attention_weight_max": 51.21516036987305,
      "activations/layer8_attention_weight_min": -47.4990119934082,
      "activations/layer9_attention_weight_max": 63.62897491455078,
      "activations/layer9_attention_weight_min": -53.909889221191406,
      "epoch": 6.82,
      "learning_rate": 0.00010708749999999998,
      "loss": 2.8277,
      "step": 117400
    },
    {
      "activations/layer0_attention_weight_max": 15.406737327575684,
      "activations/layer0_attention_weight_min": -15.360024452209473,
      "activations/layer10_attention_weight_max": 39.42744445800781,
      "activations/layer10_attention_weight_min": -36.83732223510742,
      "activations/layer11_attention_weight_max": 32.50160598754883,
      "activations/layer11_attention_weight_min": -31.126989364624023,
      "activations/layer12_attention_weight_max": 19.01812171936035,
      "activations/layer12_attention_weight_min": -24.629655838012695,
      "activations/layer13_attention_weight_max": 48.829200744628906,
      "activations/layer13_attention_weight_min": -36.51155090332031,
      "activations/layer14_attention_weight_max": 37.89713668823242,
      "activations/layer14_attention_weight_min": -29.95947265625,
      "activations/layer15_attention_weight_max": 37.751014709472656,
      "activations/layer15_attention_weight_min": -29.801837921142578,
      "activations/layer16_attention_weight_max": 35.896968841552734,
      "activations/layer16_attention_weight_min": -26.671781539916992,
      "activations/layer17_attention_weight_max": 55.200843811035156,
      "activations/layer17_attention_weight_min": -40.92201232910156,
      "activations/layer18_attention_weight_max": 47.620296478271484,
      "activations/layer18_attention_weight_min": -34.20652389526367,
      "activations/layer19_attention_weight_max": 23.959217071533203,
      "activations/layer19_attention_weight_min": -19.967147827148438,
      "activations/layer1_attention_weight_max": 17.45431900024414,
      "activations/layer1_attention_weight_min": -15.70827579498291,
      "activations/layer20_attention_weight_max": 21.721872329711914,
      "activations/layer20_attention_weight_min": -19.370250701904297,
      "activations/layer21_attention_weight_max": 40.654911041259766,
      "activations/layer21_attention_weight_min": -24.4044189453125,
      "activations/layer22_attention_weight_max": 31.524595260620117,
      "activations/layer22_attention_weight_min": -23.914751052856445,
      "activations/layer23_attention_weight_max": 33.48439025878906,
      "activations/layer23_attention_weight_min": -19.237865447998047,
      "activations/layer2_attention_weight_max": 32.92884063720703,
      "activations/layer2_attention_weight_min": -30.702688217163086,
      "activations/layer3_attention_weight_max": 85.84657287597656,
      "activations/layer3_attention_weight_min": -91.98143768310547,
      "activations/layer4_attention_weight_max": 97.81380462646484,
      "activations/layer4_attention_weight_min": -91.72008514404297,
      "activations/layer5_attention_weight_max": 95.08195495605469,
      "activations/layer5_attention_weight_min": -86.5172348022461,
      "activations/layer6_attention_weight_max": 58.30379104614258,
      "activations/layer6_attention_weight_min": -55.49609375,
      "activations/layer7_attention_weight_max": 74.70700073242188,
      "activations/layer7_attention_weight_min": -72.18280792236328,
      "activations/layer8_attention_weight_max": 45.236602783203125,
      "activations/layer8_attention_weight_min": -46.355892181396484,
      "activations/layer9_attention_weight_max": 50.821651458740234,
      "activations/layer9_attention_weight_min": -50.70446014404297,
      "epoch": 6.82,
      "learning_rate": 0.0001070685606060606,
      "loss": 2.8315,
      "step": 117450
    },
    {
      "activations/layer0_attention_weight_max": 15.764955520629883,
      "activations/layer0_attention_weight_min": -15.017088890075684,
      "activations/layer10_attention_weight_max": 35.10163116455078,
      "activations/layer10_attention_weight_min": -34.24338912963867,
      "activations/layer11_attention_weight_max": 30.882062911987305,
      "activations/layer11_attention_weight_min": -28.423336029052734,
      "activations/layer12_attention_weight_max": 23.972742080688477,
      "activations/layer12_attention_weight_min": -27.975940704345703,
      "activations/layer13_attention_weight_max": 71.39274597167969,
      "activations/layer13_attention_weight_min": -38.34999465942383,
      "activations/layer14_attention_weight_max": 95.11307525634766,
      "activations/layer14_attention_weight_min": -45.24232864379883,
      "activations/layer15_attention_weight_max": 72.26860046386719,
      "activations/layer15_attention_weight_min": -37.13383865356445,
      "activations/layer16_attention_weight_max": 38.6728515625,
      "activations/layer16_attention_weight_min": -28.700183868408203,
      "activations/layer17_attention_weight_max": 64.09620666503906,
      "activations/layer17_attention_weight_min": -45.45197677612305,
      "activations/layer18_attention_weight_max": 51.6605110168457,
      "activations/layer18_attention_weight_min": -40.0081672668457,
      "activations/layer19_attention_weight_max": 32.22332000732422,
      "activations/layer19_attention_weight_min": -23.118860244750977,
      "activations/layer1_attention_weight_max": 16.816848754882812,
      "activations/layer1_attention_weight_min": -15.518461227416992,
      "activations/layer20_attention_weight_max": 29.99428939819336,
      "activations/layer20_attention_weight_min": -26.506492614746094,
      "activations/layer21_attention_weight_max": 57.08906173706055,
      "activations/layer21_attention_weight_min": -27.61266326904297,
      "activations/layer22_attention_weight_max": 35.45132827758789,
      "activations/layer22_attention_weight_min": -27.11849021911621,
      "activations/layer23_attention_weight_max": 32.980804443359375,
      "activations/layer23_attention_weight_min": -21.000593185424805,
      "activations/layer2_attention_weight_max": 33.882667541503906,
      "activations/layer2_attention_weight_min": -29.942289352416992,
      "activations/layer3_attention_weight_max": 86.42138671875,
      "activations/layer3_attention_weight_min": -86.8219985961914,
      "activations/layer4_attention_weight_max": 98.28734588623047,
      "activations/layer4_attention_weight_min": -94.80675506591797,
      "activations/layer5_attention_weight_max": 88.30809783935547,
      "activations/layer5_attention_weight_min": -80.6092529296875,
      "activations/layer6_attention_weight_max": 53.171592712402344,
      "activations/layer6_attention_weight_min": -48.16195297241211,
      "activations/layer7_attention_weight_max": 70.0438003540039,
      "activations/layer7_attention_weight_min": -63.204978942871094,
      "activations/layer8_attention_weight_max": 40.0140495300293,
      "activations/layer8_attention_weight_min": -42.67987823486328,
      "activations/layer9_attention_weight_max": 56.11738204956055,
      "activations/layer9_attention_weight_min": -52.620819091796875,
      "epoch": 6.83,
      "learning_rate": 0.0001070496212121212,
      "loss": 2.8219,
      "step": 117500
    },
    {
      "activations/layer0_attention_weight_max": 14.732178688049316,
      "activations/layer0_attention_weight_min": -15.497998237609863,
      "activations/layer10_attention_weight_max": 37.195526123046875,
      "activations/layer10_attention_weight_min": -37.14789962768555,
      "activations/layer11_attention_weight_max": 30.17871856689453,
      "activations/layer11_attention_weight_min": -30.725393295288086,
      "activations/layer12_attention_weight_max": 25.086944580078125,
      "activations/layer12_attention_weight_min": -24.742834091186523,
      "activations/layer13_attention_weight_max": 41.4893684387207,
      "activations/layer13_attention_weight_min": -32.175636291503906,
      "activations/layer14_attention_weight_max": 38.183563232421875,
      "activations/layer14_attention_weight_min": -30.518674850463867,
      "activations/layer15_attention_weight_max": 35.123199462890625,
      "activations/layer15_attention_weight_min": -29.56864356994629,
      "activations/layer16_attention_weight_max": 32.98983383178711,
      "activations/layer16_attention_weight_min": -28.587770462036133,
      "activations/layer17_attention_weight_max": 59.59952163696289,
      "activations/layer17_attention_weight_min": -42.06318664550781,
      "activations/layer18_attention_weight_max": 50.1220588684082,
      "activations/layer18_attention_weight_min": -35.65774154663086,
      "activations/layer19_attention_weight_max": 24.706104278564453,
      "activations/layer19_attention_weight_min": -19.801307678222656,
      "activations/layer1_attention_weight_max": 17.15485954284668,
      "activations/layer1_attention_weight_min": -14.295831680297852,
      "activations/layer20_attention_weight_max": 23.670991897583008,
      "activations/layer20_attention_weight_min": -20.872760772705078,
      "activations/layer21_attention_weight_max": 39.051509857177734,
      "activations/layer21_attention_weight_min": -25.680828094482422,
      "activations/layer22_attention_weight_max": 37.63710021972656,
      "activations/layer22_attention_weight_min": -24.0661563873291,
      "activations/layer23_attention_weight_max": 33.8040657043457,
      "activations/layer23_attention_weight_min": -23.402875900268555,
      "activations/layer2_attention_weight_max": 31.15126609802246,
      "activations/layer2_attention_weight_min": -29.552085876464844,
      "activations/layer3_attention_weight_max": 87.25874328613281,
      "activations/layer3_attention_weight_min": -88.62216186523438,
      "activations/layer4_attention_weight_max": 102.23104858398438,
      "activations/layer4_attention_weight_min": -93.41638946533203,
      "activations/layer5_attention_weight_max": 94.39299011230469,
      "activations/layer5_attention_weight_min": -87.71139526367188,
      "activations/layer6_attention_weight_max": 57.24192810058594,
      "activations/layer6_attention_weight_min": -56.65744400024414,
      "activations/layer7_attention_weight_max": 78.4677505493164,
      "activations/layer7_attention_weight_min": -77.9645767211914,
      "activations/layer8_attention_weight_max": 45.124671936035156,
      "activations/layer8_attention_weight_min": -46.824798583984375,
      "activations/layer9_attention_weight_max": 50.58537292480469,
      "activations/layer9_attention_weight_min": -51.10772705078125,
      "epoch": 6.83,
      "learning_rate": 0.00010703068181818181,
      "loss": 2.8328,
      "step": 117550
    },
    {
      "activations/layer0_attention_weight_max": 14.85932731628418,
      "activations/layer0_attention_weight_min": -15.550604820251465,
      "activations/layer10_attention_weight_max": 39.628807067871094,
      "activations/layer10_attention_weight_min": -35.822879791259766,
      "activations/layer11_attention_weight_max": 32.74409484863281,
      "activations/layer11_attention_weight_min": -31.552831649780273,
      "activations/layer12_attention_weight_max": 20.58547019958496,
      "activations/layer12_attention_weight_min": -24.83018684387207,
      "activations/layer13_attention_weight_max": 44.743736267089844,
      "activations/layer13_attention_weight_min": -36.213340759277344,
      "activations/layer14_attention_weight_max": 36.837581634521484,
      "activations/layer14_attention_weight_min": -30.059452056884766,
      "activations/layer15_attention_weight_max": 36.97492218017578,
      "activations/layer15_attention_weight_min": -29.960039138793945,
      "activations/layer16_attention_weight_max": 30.93360137939453,
      "activations/layer16_attention_weight_min": -26.90123176574707,
      "activations/layer17_attention_weight_max": 56.25055694580078,
      "activations/layer17_attention_weight_min": -44.3620491027832,
      "activations/layer18_attention_weight_max": 51.04990005493164,
      "activations/layer18_attention_weight_min": -35.580833435058594,
      "activations/layer19_attention_weight_max": 21.432344436645508,
      "activations/layer19_attention_weight_min": -21.795108795166016,
      "activations/layer1_attention_weight_max": 16.73388671875,
      "activations/layer1_attention_weight_min": -15.099808692932129,
      "activations/layer20_attention_weight_max": 21.833372116088867,
      "activations/layer20_attention_weight_min": -20.11222267150879,
      "activations/layer21_attention_weight_max": 36.280235290527344,
      "activations/layer21_attention_weight_min": -23.292495727539062,
      "activations/layer22_attention_weight_max": 29.547082901000977,
      "activations/layer22_attention_weight_min": -26.15939712524414,
      "activations/layer23_attention_weight_max": 30.533618927001953,
      "activations/layer23_attention_weight_min": -18.818471908569336,
      "activations/layer2_attention_weight_max": 32.149391174316406,
      "activations/layer2_attention_weight_min": -30.447750091552734,
      "activations/layer3_attention_weight_max": 86.92532348632812,
      "activations/layer3_attention_weight_min": -96.17098999023438,
      "activations/layer4_attention_weight_max": 103.14497375488281,
      "activations/layer4_attention_weight_min": -99.82733154296875,
      "activations/layer5_attention_weight_max": 92.32598114013672,
      "activations/layer5_attention_weight_min": -89.74837493896484,
      "activations/layer6_attention_weight_max": 57.63554382324219,
      "activations/layer6_attention_weight_min": -54.42385482788086,
      "activations/layer7_attention_weight_max": 73.38140869140625,
      "activations/layer7_attention_weight_min": -73.96502685546875,
      "activations/layer8_attention_weight_max": 45.033653259277344,
      "activations/layer8_attention_weight_min": -49.2156867980957,
      "activations/layer9_attention_weight_max": 52.35573196411133,
      "activations/layer9_attention_weight_min": -49.817535400390625,
      "epoch": 6.83,
      "learning_rate": 0.00010701174242424241,
      "loss": 2.8229,
      "step": 117600
    },
    {
      "activations/layer0_attention_weight_max": 15.502764701843262,
      "activations/layer0_attention_weight_min": -15.225362777709961,
      "activations/layer10_attention_weight_max": 37.509376525878906,
      "activations/layer10_attention_weight_min": -34.7103157043457,
      "activations/layer11_attention_weight_max": 31.796613693237305,
      "activations/layer11_attention_weight_min": -29.076139450073242,
      "activations/layer12_attention_weight_max": 22.231529235839844,
      "activations/layer12_attention_weight_min": -29.04178810119629,
      "activations/layer13_attention_weight_max": 39.35481262207031,
      "activations/layer13_attention_weight_min": -33.39439010620117,
      "activations/layer14_attention_weight_max": 38.83094024658203,
      "activations/layer14_attention_weight_min": -30.45730209350586,
      "activations/layer15_attention_weight_max": 39.4622917175293,
      "activations/layer15_attention_weight_min": -29.73964500427246,
      "activations/layer16_attention_weight_max": 32.72193908691406,
      "activations/layer16_attention_weight_min": -28.948331832885742,
      "activations/layer17_attention_weight_max": 53.22633361816406,
      "activations/layer17_attention_weight_min": -43.677940368652344,
      "activations/layer18_attention_weight_max": 46.15453338623047,
      "activations/layer18_attention_weight_min": -37.68151092529297,
      "activations/layer19_attention_weight_max": 20.521129608154297,
      "activations/layer19_attention_weight_min": -21.761289596557617,
      "activations/layer1_attention_weight_max": 16.598434448242188,
      "activations/layer1_attention_weight_min": -17.690080642700195,
      "activations/layer20_attention_weight_max": 20.57243537902832,
      "activations/layer20_attention_weight_min": -20.510467529296875,
      "activations/layer21_attention_weight_max": 35.87772750854492,
      "activations/layer21_attention_weight_min": -23.242916107177734,
      "activations/layer22_attention_weight_max": 34.93431854248047,
      "activations/layer22_attention_weight_min": -26.898284912109375,
      "activations/layer23_attention_weight_max": 29.16681480407715,
      "activations/layer23_attention_weight_min": -20.685041427612305,
      "activations/layer2_attention_weight_max": 36.41759490966797,
      "activations/layer2_attention_weight_min": -32.795021057128906,
      "activations/layer3_attention_weight_max": 94.93812561035156,
      "activations/layer3_attention_weight_min": -94.77128601074219,
      "activations/layer4_attention_weight_max": 108.3411636352539,
      "activations/layer4_attention_weight_min": -103.3033447265625,
      "activations/layer5_attention_weight_max": 99.20401000976562,
      "activations/layer5_attention_weight_min": -84.42655944824219,
      "activations/layer6_attention_weight_max": 55.99650192260742,
      "activations/layer6_attention_weight_min": -52.229164123535156,
      "activations/layer7_attention_weight_max": 80.01398468017578,
      "activations/layer7_attention_weight_min": -71.31802368164062,
      "activations/layer8_attention_weight_max": 46.71826934814453,
      "activations/layer8_attention_weight_min": -44.081233978271484,
      "activations/layer9_attention_weight_max": 58.517330169677734,
      "activations/layer9_attention_weight_min": -50.50276184082031,
      "epoch": 6.84,
      "learning_rate": 0.00010699280303030302,
      "loss": 2.8357,
      "step": 117650
    },
    {
      "activations/layer0_attention_weight_max": 14.758308410644531,
      "activations/layer0_attention_weight_min": -14.875938415527344,
      "activations/layer10_attention_weight_max": 39.82223892211914,
      "activations/layer10_attention_weight_min": -35.642333984375,
      "activations/layer11_attention_weight_max": 31.82002067565918,
      "activations/layer11_attention_weight_min": -32.62061309814453,
      "activations/layer12_attention_weight_max": 22.81962013244629,
      "activations/layer12_attention_weight_min": -24.198184967041016,
      "activations/layer13_attention_weight_max": 48.228816986083984,
      "activations/layer13_attention_weight_min": -36.35642623901367,
      "activations/layer14_attention_weight_max": 44.35469055175781,
      "activations/layer14_attention_weight_min": -30.776376724243164,
      "activations/layer15_attention_weight_max": 41.53963088989258,
      "activations/layer15_attention_weight_min": -30.62518882751465,
      "activations/layer16_attention_weight_max": 36.29151153564453,
      "activations/layer16_attention_weight_min": -25.75934600830078,
      "activations/layer17_attention_weight_max": 56.95948791503906,
      "activations/layer17_attention_weight_min": -43.05852127075195,
      "activations/layer18_attention_weight_max": 52.931427001953125,
      "activations/layer18_attention_weight_min": -35.24913787841797,
      "activations/layer19_attention_weight_max": 23.858243942260742,
      "activations/layer19_attention_weight_min": -20.482433319091797,
      "activations/layer1_attention_weight_max": 17.652982711791992,
      "activations/layer1_attention_weight_min": -14.919188499450684,
      "activations/layer20_attention_weight_max": 21.87902069091797,
      "activations/layer20_attention_weight_min": -19.34115219116211,
      "activations/layer21_attention_weight_max": 38.56856918334961,
      "activations/layer21_attention_weight_min": -24.1077823638916,
      "activations/layer22_attention_weight_max": 36.70450973510742,
      "activations/layer22_attention_weight_min": -23.06313133239746,
      "activations/layer23_attention_weight_max": 37.98710250854492,
      "activations/layer23_attention_weight_min": -19.171785354614258,
      "activations/layer2_attention_weight_max": 31.08990478515625,
      "activations/layer2_attention_weight_min": -30.41457748413086,
      "activations/layer3_attention_weight_max": 85.7901382446289,
      "activations/layer3_attention_weight_min": -86.05536651611328,
      "activations/layer4_attention_weight_max": 104.78226470947266,
      "activations/layer4_attention_weight_min": -100.58421325683594,
      "activations/layer5_attention_weight_max": 96.1887435913086,
      "activations/layer5_attention_weight_min": -93.49517822265625,
      "activations/layer6_attention_weight_max": 59.5640869140625,
      "activations/layer6_attention_weight_min": -55.271636962890625,
      "activations/layer7_attention_weight_max": 75.41099548339844,
      "activations/layer7_attention_weight_min": -78.29569244384766,
      "activations/layer8_attention_weight_max": 46.76576614379883,
      "activations/layer8_attention_weight_min": -46.12450408935547,
      "activations/layer9_attention_weight_max": 50.95621871948242,
      "activations/layer9_attention_weight_min": -48.78873062133789,
      "epoch": 6.84,
      "learning_rate": 0.00010697386363636363,
      "loss": 2.826,
      "step": 117700
    },
    {
      "activations/layer0_attention_weight_max": 15.566666603088379,
      "activations/layer0_attention_weight_min": -15.320652961730957,
      "activations/layer10_attention_weight_max": 40.09215545654297,
      "activations/layer10_attention_weight_min": -36.030704498291016,
      "activations/layer11_attention_weight_max": 33.98393630981445,
      "activations/layer11_attention_weight_min": -32.14671325683594,
      "activations/layer12_attention_weight_max": 24.493127822875977,
      "activations/layer12_attention_weight_min": -24.63913917541504,
      "activations/layer13_attention_weight_max": 50.6437873840332,
      "activations/layer13_attention_weight_min": -35.38538360595703,
      "activations/layer14_attention_weight_max": 48.288230895996094,
      "activations/layer14_attention_weight_min": -30.211288452148438,
      "activations/layer15_attention_weight_max": 41.480628967285156,
      "activations/layer15_attention_weight_min": -30.104379653930664,
      "activations/layer16_attention_weight_max": 35.397884368896484,
      "activations/layer16_attention_weight_min": -25.71343421936035,
      "activations/layer17_attention_weight_max": 57.8671875,
      "activations/layer17_attention_weight_min": -43.667076110839844,
      "activations/layer18_attention_weight_max": 49.89251708984375,
      "activations/layer18_attention_weight_min": -36.3722038269043,
      "activations/layer19_attention_weight_max": 23.535429000854492,
      "activations/layer19_attention_weight_min": -22.581274032592773,
      "activations/layer1_attention_weight_max": 16.373382568359375,
      "activations/layer1_attention_weight_min": -15.592206001281738,
      "activations/layer20_attention_weight_max": 25.915599822998047,
      "activations/layer20_attention_weight_min": -19.743595123291016,
      "activations/layer21_attention_weight_max": 41.199745178222656,
      "activations/layer21_attention_weight_min": -23.4047908782959,
      "activations/layer22_attention_weight_max": 33.598533630371094,
      "activations/layer22_attention_weight_min": -23.806224822998047,
      "activations/layer23_attention_weight_max": 35.09082794189453,
      "activations/layer23_attention_weight_min": -20.889249801635742,
      "activations/layer2_attention_weight_max": 32.943302154541016,
      "activations/layer2_attention_weight_min": -30.37600326538086,
      "activations/layer3_attention_weight_max": 86.68189239501953,
      "activations/layer3_attention_weight_min": -90.92481231689453,
      "activations/layer4_attention_weight_max": 102.9073486328125,
      "activations/layer4_attention_weight_min": -97.00948333740234,
      "activations/layer5_attention_weight_max": 94.02001190185547,
      "activations/layer5_attention_weight_min": -83.86875915527344,
      "activations/layer6_attention_weight_max": 57.515933990478516,
      "activations/layer6_attention_weight_min": -52.823402404785156,
      "activations/layer7_attention_weight_max": 76.11670684814453,
      "activations/layer7_attention_weight_min": -71.74201965332031,
      "activations/layer8_attention_weight_max": 51.42631912231445,
      "activations/layer8_attention_weight_min": -46.95432662963867,
      "activations/layer9_attention_weight_max": 61.49915313720703,
      "activations/layer9_attention_weight_min": -53.898193359375,
      "epoch": 6.84,
      "learning_rate": 0.00010695492424242425,
      "loss": 2.8248,
      "step": 117750
    },
    {
      "activations/layer0_attention_weight_max": 15.20647144317627,
      "activations/layer0_attention_weight_min": -14.913595199584961,
      "activations/layer10_attention_weight_max": 43.602134704589844,
      "activations/layer10_attention_weight_min": -37.796295166015625,
      "activations/layer11_attention_weight_max": 33.40269470214844,
      "activations/layer11_attention_weight_min": -31.613815307617188,
      "activations/layer12_attention_weight_max": 23.249351501464844,
      "activations/layer12_attention_weight_min": -25.973758697509766,
      "activations/layer13_attention_weight_max": 50.22035217285156,
      "activations/layer13_attention_weight_min": -36.415645599365234,
      "activations/layer14_attention_weight_max": 53.87403869628906,
      "activations/layer14_attention_weight_min": -33.373138427734375,
      "activations/layer15_attention_weight_max": 38.51206970214844,
      "activations/layer15_attention_weight_min": -30.206193923950195,
      "activations/layer16_attention_weight_max": 35.80173110961914,
      "activations/layer16_attention_weight_min": -27.801773071289062,
      "activations/layer17_attention_weight_max": 59.95753479003906,
      "activations/layer17_attention_weight_min": -44.31264877319336,
      "activations/layer18_attention_weight_max": 48.388397216796875,
      "activations/layer18_attention_weight_min": -37.44468307495117,
      "activations/layer19_attention_weight_max": 24.903390884399414,
      "activations/layer19_attention_weight_min": -20.628742218017578,
      "activations/layer1_attention_weight_max": 17.100067138671875,
      "activations/layer1_attention_weight_min": -13.30887508392334,
      "activations/layer20_attention_weight_max": 25.014747619628906,
      "activations/layer20_attention_weight_min": -23.228050231933594,
      "activations/layer21_attention_weight_max": 39.47834396362305,
      "activations/layer21_attention_weight_min": -25.141311645507812,
      "activations/layer22_attention_weight_max": 33.90321350097656,
      "activations/layer22_attention_weight_min": -27.10070037841797,
      "activations/layer23_attention_weight_max": 39.07186508178711,
      "activations/layer23_attention_weight_min": -20.634868621826172,
      "activations/layer2_attention_weight_max": 32.03062438964844,
      "activations/layer2_attention_weight_min": -32.437164306640625,
      "activations/layer3_attention_weight_max": 83.95063018798828,
      "activations/layer3_attention_weight_min": -89.75321197509766,
      "activations/layer4_attention_weight_max": 97.06156158447266,
      "activations/layer4_attention_weight_min": -93.92091369628906,
      "activations/layer5_attention_weight_max": 92.17904663085938,
      "activations/layer5_attention_weight_min": -88.08238220214844,
      "activations/layer6_attention_weight_max": 58.71910095214844,
      "activations/layer6_attention_weight_min": -55.8541259765625,
      "activations/layer7_attention_weight_max": 86.11551666259766,
      "activations/layer7_attention_weight_min": -80.27698516845703,
      "activations/layer8_attention_weight_max": 50.019874572753906,
      "activations/layer8_attention_weight_min": -46.75375747680664,
      "activations/layer9_attention_weight_max": 58.82801055908203,
      "activations/layer9_attention_weight_min": -49.50056457519531,
      "epoch": 6.84,
      "learning_rate": 0.00010693598484848483,
      "loss": 2.8104,
      "step": 117800
    },
    {
      "activations/layer0_attention_weight_max": 15.214031219482422,
      "activations/layer0_attention_weight_min": -15.780536651611328,
      "activations/layer10_attention_weight_max": 39.66093063354492,
      "activations/layer10_attention_weight_min": -37.5378532409668,
      "activations/layer11_attention_weight_max": 32.58220672607422,
      "activations/layer11_attention_weight_min": -29.660789489746094,
      "activations/layer12_attention_weight_max": 26.69688606262207,
      "activations/layer12_attention_weight_min": -28.47622299194336,
      "activations/layer13_attention_weight_max": 49.78188705444336,
      "activations/layer13_attention_weight_min": -33.88543701171875,
      "activations/layer14_attention_weight_max": 50.93546676635742,
      "activations/layer14_attention_weight_min": -31.857887268066406,
      "activations/layer15_attention_weight_max": 37.7530517578125,
      "activations/layer15_attention_weight_min": -30.9964656829834,
      "activations/layer16_attention_weight_max": 32.77364730834961,
      "activations/layer16_attention_weight_min": -28.66330909729004,
      "activations/layer17_attention_weight_max": 60.22345733642578,
      "activations/layer17_attention_weight_min": -46.44218826293945,
      "activations/layer18_attention_weight_max": 51.078956604003906,
      "activations/layer18_attention_weight_min": -36.70632553100586,
      "activations/layer19_attention_weight_max": 23.11710548400879,
      "activations/layer19_attention_weight_min": -19.09966468811035,
      "activations/layer1_attention_weight_max": 16.770553588867188,
      "activations/layer1_attention_weight_min": -18.219209671020508,
      "activations/layer20_attention_weight_max": 24.272838592529297,
      "activations/layer20_attention_weight_min": -20.87346839904785,
      "activations/layer21_attention_weight_max": 40.699832916259766,
      "activations/layer21_attention_weight_min": -25.548303604125977,
      "activations/layer22_attention_weight_max": 32.67940139770508,
      "activations/layer22_attention_weight_min": -25.10247802734375,
      "activations/layer23_attention_weight_max": 35.18191146850586,
      "activations/layer23_attention_weight_min": -21.023822784423828,
      "activations/layer2_attention_weight_max": 33.486175537109375,
      "activations/layer2_attention_weight_min": -30.06960678100586,
      "activations/layer3_attention_weight_max": 85.50159454345703,
      "activations/layer3_attention_weight_min": -90.56598663330078,
      "activations/layer4_attention_weight_max": 100.99807739257812,
      "activations/layer4_attention_weight_min": -94.76220703125,
      "activations/layer5_attention_weight_max": 92.03517150878906,
      "activations/layer5_attention_weight_min": -84.89855194091797,
      "activations/layer6_attention_weight_max": 60.12775421142578,
      "activations/layer6_attention_weight_min": -56.143310546875,
      "activations/layer7_attention_weight_max": 78.11404418945312,
      "activations/layer7_attention_weight_min": -69.44188690185547,
      "activations/layer8_attention_weight_max": 49.52965545654297,
      "activations/layer8_attention_weight_min": -50.718048095703125,
      "activations/layer9_attention_weight_max": 60.618629455566406,
      "activations/layer9_attention_weight_min": -49.11925506591797,
      "epoch": 6.85,
      "learning_rate": 0.00010691704545454545,
      "loss": 2.817,
      "step": 117850
    },
    {
      "activations/layer0_attention_weight_max": 15.217150688171387,
      "activations/layer0_attention_weight_min": -15.047993659973145,
      "activations/layer10_attention_weight_max": 35.363128662109375,
      "activations/layer10_attention_weight_min": -35.869876861572266,
      "activations/layer11_attention_weight_max": 28.638565063476562,
      "activations/layer11_attention_weight_min": -29.504108428955078,
      "activations/layer12_attention_weight_max": 21.659624099731445,
      "activations/layer12_attention_weight_min": -27.110193252563477,
      "activations/layer13_attention_weight_max": 38.95372009277344,
      "activations/layer13_attention_weight_min": -31.36252784729004,
      "activations/layer14_attention_weight_max": 38.294620513916016,
      "activations/layer14_attention_weight_min": -30.977548599243164,
      "activations/layer15_attention_weight_max": 35.090003967285156,
      "activations/layer15_attention_weight_min": -29.535236358642578,
      "activations/layer16_attention_weight_max": 34.52898406982422,
      "activations/layer16_attention_weight_min": -27.661962509155273,
      "activations/layer17_attention_weight_max": 52.18986511230469,
      "activations/layer17_attention_weight_min": -43.521934509277344,
      "activations/layer18_attention_weight_max": 48.51360321044922,
      "activations/layer18_attention_weight_min": -36.34244918823242,
      "activations/layer19_attention_weight_max": 20.927051544189453,
      "activations/layer19_attention_weight_min": -21.59364891052246,
      "activations/layer1_attention_weight_max": 17.300334930419922,
      "activations/layer1_attention_weight_min": -15.5077543258667,
      "activations/layer20_attention_weight_max": 21.925567626953125,
      "activations/layer20_attention_weight_min": -20.990501403808594,
      "activations/layer21_attention_weight_max": 35.74262619018555,
      "activations/layer21_attention_weight_min": -25.98958396911621,
      "activations/layer22_attention_weight_max": 35.02080154418945,
      "activations/layer22_attention_weight_min": -25.23569107055664,
      "activations/layer23_attention_weight_max": 32.032958984375,
      "activations/layer23_attention_weight_min": -22.032155990600586,
      "activations/layer2_attention_weight_max": 32.7408447265625,
      "activations/layer2_attention_weight_min": -29.481216430664062,
      "activations/layer3_attention_weight_max": 85.06059265136719,
      "activations/layer3_attention_weight_min": -86.26050567626953,
      "activations/layer4_attention_weight_max": 101.8731689453125,
      "activations/layer4_attention_weight_min": -92.23982238769531,
      "activations/layer5_attention_weight_max": 88.76315307617188,
      "activations/layer5_attention_weight_min": -81.52434539794922,
      "activations/layer6_attention_weight_max": 55.24812316894531,
      "activations/layer6_attention_weight_min": -50.289119720458984,
      "activations/layer7_attention_weight_max": 72.92327117919922,
      "activations/layer7_attention_weight_min": -68.28153228759766,
      "activations/layer8_attention_weight_max": 40.87484359741211,
      "activations/layer8_attention_weight_min": -41.9135627746582,
      "activations/layer9_attention_weight_max": 48.728397369384766,
      "activations/layer9_attention_weight_min": -50.86248779296875,
      "epoch": 6.85,
      "learning_rate": 0.00010689810606060606,
      "loss": 2.8219,
      "step": 117900
    },
    {
      "activations/layer0_attention_weight_max": 15.124835014343262,
      "activations/layer0_attention_weight_min": -15.232934951782227,
      "activations/layer10_attention_weight_max": 38.39262771606445,
      "activations/layer10_attention_weight_min": -36.35146713256836,
      "activations/layer11_attention_weight_max": 30.31917953491211,
      "activations/layer11_attention_weight_min": -32.71326446533203,
      "activations/layer12_attention_weight_max": 20.397403717041016,
      "activations/layer12_attention_weight_min": -30.871822357177734,
      "activations/layer13_attention_weight_max": 40.48089599609375,
      "activations/layer13_attention_weight_min": -32.24088668823242,
      "activations/layer14_attention_weight_max": 35.811553955078125,
      "activations/layer14_attention_weight_min": -30.71976661682129,
      "activations/layer15_attention_weight_max": 35.26627731323242,
      "activations/layer15_attention_weight_min": -29.096067428588867,
      "activations/layer16_attention_weight_max": 28.972414016723633,
      "activations/layer16_attention_weight_min": -25.875768661499023,
      "activations/layer17_attention_weight_max": 55.546905517578125,
      "activations/layer17_attention_weight_min": -38.998844146728516,
      "activations/layer18_attention_weight_max": 46.276222229003906,
      "activations/layer18_attention_weight_min": -32.3133544921875,
      "activations/layer19_attention_weight_max": 23.32636070251465,
      "activations/layer19_attention_weight_min": -20.40411949157715,
      "activations/layer1_attention_weight_max": 16.52922821044922,
      "activations/layer1_attention_weight_min": -18.657812118530273,
      "activations/layer20_attention_weight_max": 21.929466247558594,
      "activations/layer20_attention_weight_min": -19.95607566833496,
      "activations/layer21_attention_weight_max": 36.42149353027344,
      "activations/layer21_attention_weight_min": -24.191165924072266,
      "activations/layer22_attention_weight_max": 31.379236221313477,
      "activations/layer22_attention_weight_min": -22.88321304321289,
      "activations/layer23_attention_weight_max": 30.231647491455078,
      "activations/layer23_attention_weight_min": -20.830768585205078,
      "activations/layer2_attention_weight_max": 33.466835021972656,
      "activations/layer2_attention_weight_min": -30.119449615478516,
      "activations/layer3_attention_weight_max": 93.93116760253906,
      "activations/layer3_attention_weight_min": -91.10273742675781,
      "activations/layer4_attention_weight_max": 107.09069061279297,
      "activations/layer4_attention_weight_min": -95.35142517089844,
      "activations/layer5_attention_weight_max": 94.7887191772461,
      "activations/layer5_attention_weight_min": -86.34046936035156,
      "activations/layer6_attention_weight_max": 58.023345947265625,
      "activations/layer6_attention_weight_min": -53.539859771728516,
      "activations/layer7_attention_weight_max": 77.49327087402344,
      "activations/layer7_attention_weight_min": -73.91191864013672,
      "activations/layer8_attention_weight_max": 48.732295989990234,
      "activations/layer8_attention_weight_min": -44.46306228637695,
      "activations/layer9_attention_weight_max": 57.993709564208984,
      "activations/layer9_attention_weight_min": -54.556941986083984,
      "epoch": 6.85,
      "learning_rate": 0.00010687916666666665,
      "loss": 2.8357,
      "step": 117950
    },
    {
      "activations/layer0_attention_weight_max": 15.413658142089844,
      "activations/layer0_attention_weight_min": -15.507259368896484,
      "activations/layer10_attention_weight_max": 37.45469284057617,
      "activations/layer10_attention_weight_min": -34.80748748779297,
      "activations/layer11_attention_weight_max": 29.72210693359375,
      "activations/layer11_attention_weight_min": -29.90985870361328,
      "activations/layer12_attention_weight_max": 25.962858200073242,
      "activations/layer12_attention_weight_min": -24.443506240844727,
      "activations/layer13_attention_weight_max": 50.35074234008789,
      "activations/layer13_attention_weight_min": -32.73602294921875,
      "activations/layer14_attention_weight_max": 42.126441955566406,
      "activations/layer14_attention_weight_min": -30.311742782592773,
      "activations/layer15_attention_weight_max": 43.6541862487793,
      "activations/layer15_attention_weight_min": -29.080366134643555,
      "activations/layer16_attention_weight_max": 37.15199661254883,
      "activations/layer16_attention_weight_min": -27.606258392333984,
      "activations/layer17_attention_weight_max": 60.08910369873047,
      "activations/layer17_attention_weight_min": -43.39714813232422,
      "activations/layer18_attention_weight_max": 49.89336395263672,
      "activations/layer18_attention_weight_min": -38.57830047607422,
      "activations/layer19_attention_weight_max": 26.36737060546875,
      "activations/layer19_attention_weight_min": -23.581605911254883,
      "activations/layer1_attention_weight_max": 16.791915893554688,
      "activations/layer1_attention_weight_min": -14.780461311340332,
      "activations/layer20_attention_weight_max": 27.029497146606445,
      "activations/layer20_attention_weight_min": -21.76302719116211,
      "activations/layer21_attention_weight_max": 39.768253326416016,
      "activations/layer21_attention_weight_min": -24.515295028686523,
      "activations/layer22_attention_weight_max": 32.61082077026367,
      "activations/layer22_attention_weight_min": -23.955524444580078,
      "activations/layer23_attention_weight_max": 34.74391555786133,
      "activations/layer23_attention_weight_min": -22.740863800048828,
      "activations/layer2_attention_weight_max": 31.921894073486328,
      "activations/layer2_attention_weight_min": -29.51409149169922,
      "activations/layer3_attention_weight_max": 85.78547668457031,
      "activations/layer3_attention_weight_min": -87.8744125366211,
      "activations/layer4_attention_weight_max": 99.6676025390625,
      "activations/layer4_attention_weight_min": -91.22150421142578,
      "activations/layer5_attention_weight_max": 93.65478515625,
      "activations/layer5_attention_weight_min": -93.06526947021484,
      "activations/layer6_attention_weight_max": 56.81312942504883,
      "activations/layer6_attention_weight_min": -52.295223236083984,
      "activations/layer7_attention_weight_max": 74.29460144042969,
      "activations/layer7_attention_weight_min": -68.09365844726562,
      "activations/layer8_attention_weight_max": 46.63896560668945,
      "activations/layer8_attention_weight_min": -47.23958969116211,
      "activations/layer9_attention_weight_max": 54.11361312866211,
      "activations/layer9_attention_weight_min": -46.53466033935547,
      "epoch": 6.86,
      "learning_rate": 0.00010686022727272727,
      "loss": 2.8383,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.5334,
      "eval_samples_per_second": 503.2,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.5334,
      "eval_openwebtext_samples_per_second": 503.2,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_wikitext_loss": 3.01953125,
      "eval_wikitext_ppl": 20.4816886421048,
      "eval_wikitext_runtime": 2.027,
      "eval_wikitext_samples_per_second": 224.965,
      "step": 118000
    },
    {
      "epoch": 6.86,
      "eval_lambada_loss": 2.810546875,
      "eval_lambada_ppl": 16.61900425204837,
      "eval_lambada_runtime": 9.6231,
      "eval_lambada_samples_per_second": 505.972,
      "step": 118000
    },
    {
      "activations/layer0_attention_weight_max": 15.422438621520996,
      "activations/layer0_attention_weight_min": -15.26569938659668,
      "activations/layer10_attention_weight_max": 38.61924743652344,
      "activations/layer10_attention_weight_min": -34.843624114990234,
      "activations/layer11_attention_weight_max": 30.332977294921875,
      "activations/layer11_attention_weight_min": -30.34002685546875,
      "activations/layer12_attention_weight_max": 21.194046020507812,
      "activations/layer12_attention_weight_min": -24.936552047729492,
      "activations/layer13_attention_weight_max": 59.63458251953125,
      "activations/layer13_attention_weight_min": -35.88795852661133,
      "activations/layer14_attention_weight_max": 45.876747131347656,
      "activations/layer14_attention_weight_min": -30.218347549438477,
      "activations/layer15_attention_weight_max": 43.36250686645508,
      "activations/layer15_attention_weight_min": -30.304248809814453,
      "activations/layer16_attention_weight_max": 37.895931243896484,
      "activations/layer16_attention_weight_min": -27.465286254882812,
      "activations/layer17_attention_weight_max": 59.9515266418457,
      "activations/layer17_attention_weight_min": -42.76078796386719,
      "activations/layer18_attention_weight_max": 52.23260498046875,
      "activations/layer18_attention_weight_min": -36.052650451660156,
      "activations/layer19_attention_weight_max": 27.637664794921875,
      "activations/layer19_attention_weight_min": -20.32917594909668,
      "activations/layer1_attention_weight_max": 18.484210968017578,
      "activations/layer1_attention_weight_min": -17.67912483215332,
      "activations/layer20_attention_weight_max": 26.833267211914062,
      "activations/layer20_attention_weight_min": -23.537267684936523,
      "activations/layer21_attention_weight_max": 47.47598648071289,
      "activations/layer21_attention_weight_min": -27.09072494506836,
      "activations/layer22_attention_weight_max": 39.34364700317383,
      "activations/layer22_attention_weight_min": -26.44533920288086,
      "activations/layer23_attention_weight_max": 43.119537353515625,
      "activations/layer23_attention_weight_min": -22.849376678466797,
      "activations/layer2_attention_weight_max": 33.52323532104492,
      "activations/layer2_attention_weight_min": -31.265724182128906,
      "activations/layer3_attention_weight_max": 86.08696746826172,
      "activations/layer3_attention_weight_min": -90.26117706298828,
      "activations/layer4_attention_weight_max": 107.05360412597656,
      "activations/layer4_attention_weight_min": -96.56258392333984,
      "activations/layer5_attention_weight_max": 95.65580749511719,
      "activations/layer5_attention_weight_min": -87.21968841552734,
      "activations/layer6_attention_weight_max": 59.631629943847656,
      "activations/layer6_attention_weight_min": -54.64759826660156,
      "activations/layer7_attention_weight_max": 75.1888427734375,
      "activations/layer7_attention_weight_min": -69.54778289794922,
      "activations/layer8_attention_weight_max": 45.16035842895508,
      "activations/layer8_attention_weight_min": -44.17793655395508,
      "activations/layer9_attention_weight_max": 51.48641586303711,
      "activations/layer9_attention_weight_min": -49.65325164794922,
      "epoch": 6.86,
      "learning_rate": 0.00010684128787878787,
      "loss": 2.8243,
      "step": 118050
    },
    {
      "activations/layer0_attention_weight_max": 15.273852348327637,
      "activations/layer0_attention_weight_min": -14.902813911437988,
      "activations/layer10_attention_weight_max": 39.664424896240234,
      "activations/layer10_attention_weight_min": -36.71266174316406,
      "activations/layer11_attention_weight_max": 31.933563232421875,
      "activations/layer11_attention_weight_min": -28.562992095947266,
      "activations/layer12_attention_weight_max": 24.928964614868164,
      "activations/layer12_attention_weight_min": -22.070316314697266,
      "activations/layer13_attention_weight_max": 44.56250762939453,
      "activations/layer13_attention_weight_min": -33.11094284057617,
      "activations/layer14_attention_weight_max": 39.57120895385742,
      "activations/layer14_attention_weight_min": -30.1772403717041,
      "activations/layer15_attention_weight_max": 38.76116943359375,
      "activations/layer15_attention_weight_min": -31.545089721679688,
      "activations/layer16_attention_weight_max": 33.04739761352539,
      "activations/layer16_attention_weight_min": -27.75190544128418,
      "activations/layer17_attention_weight_max": 56.493831634521484,
      "activations/layer17_attention_weight_min": -42.276554107666016,
      "activations/layer18_attention_weight_max": 47.1431999206543,
      "activations/layer18_attention_weight_min": -36.029747009277344,
      "activations/layer19_attention_weight_max": 24.33254623413086,
      "activations/layer19_attention_weight_min": -22.217830657958984,
      "activations/layer1_attention_weight_max": 16.992238998413086,
      "activations/layer1_attention_weight_min": -18.119552612304688,
      "activations/layer20_attention_weight_max": 22.636821746826172,
      "activations/layer20_attention_weight_min": -21.273202896118164,
      "activations/layer21_attention_weight_max": 42.609771728515625,
      "activations/layer21_attention_weight_min": -28.935239791870117,
      "activations/layer22_attention_weight_max": 33.5082893371582,
      "activations/layer22_attention_weight_min": -25.512104034423828,
      "activations/layer23_attention_weight_max": 32.77717971801758,
      "activations/layer23_attention_weight_min": -20.333005905151367,
      "activations/layer2_attention_weight_max": 31.978370666503906,
      "activations/layer2_attention_weight_min": -29.714710235595703,
      "activations/layer3_attention_weight_max": 90.6006851196289,
      "activations/layer3_attention_weight_min": -92.80859375,
      "activations/layer4_attention_weight_max": 100.53734588623047,
      "activations/layer4_attention_weight_min": -98.4018783569336,
      "activations/layer5_attention_weight_max": 93.27694702148438,
      "activations/layer5_attention_weight_min": -88.30244445800781,
      "activations/layer6_attention_weight_max": 59.17499923706055,
      "activations/layer6_attention_weight_min": -55.87336730957031,
      "activations/layer7_attention_weight_max": 74.55545806884766,
      "activations/layer7_attention_weight_min": -72.57353210449219,
      "activations/layer8_attention_weight_max": 46.462120056152344,
      "activations/layer8_attention_weight_min": -47.676612854003906,
      "activations/layer9_attention_weight_max": 49.10647964477539,
      "activations/layer9_attention_weight_min": -48.27333068847656,
      "epoch": 6.86,
      "learning_rate": 0.00010682234848484847,
      "loss": 2.8271,
      "step": 118100
    },
    {
      "activations/layer0_attention_weight_max": 15.395709037780762,
      "activations/layer0_attention_weight_min": -14.808638572692871,
      "activations/layer10_attention_weight_max": 44.34555435180664,
      "activations/layer10_attention_weight_min": -39.89752197265625,
      "activations/layer11_attention_weight_max": 40.46868896484375,
      "activations/layer11_attention_weight_min": -30.95947265625,
      "activations/layer12_attention_weight_max": 31.20409393310547,
      "activations/layer12_attention_weight_min": -23.280473709106445,
      "activations/layer13_attention_weight_max": 47.95383071899414,
      "activations/layer13_attention_weight_min": -37.6146354675293,
      "activations/layer14_attention_weight_max": 64.0617904663086,
      "activations/layer14_attention_weight_min": -34.603797912597656,
      "activations/layer15_attention_weight_max": 52.15122604370117,
      "activations/layer15_attention_weight_min": -29.712305068969727,
      "activations/layer16_attention_weight_max": 39.63935470581055,
      "activations/layer16_attention_weight_min": -27.26076889038086,
      "activations/layer17_attention_weight_max": 66.74222564697266,
      "activations/layer17_attention_weight_min": -48.1347541809082,
      "activations/layer18_attention_weight_max": 52.412288665771484,
      "activations/layer18_attention_weight_min": -41.14520263671875,
      "activations/layer19_attention_weight_max": 24.10262107849121,
      "activations/layer19_attention_weight_min": -23.252201080322266,
      "activations/layer1_attention_weight_max": 17.46848487854004,
      "activations/layer1_attention_weight_min": -14.601034164428711,
      "activations/layer20_attention_weight_max": 26.521202087402344,
      "activations/layer20_attention_weight_min": -22.26616668701172,
      "activations/layer21_attention_weight_max": 53.91849899291992,
      "activations/layer21_attention_weight_min": -27.403472900390625,
      "activations/layer22_attention_weight_max": 37.497013092041016,
      "activations/layer22_attention_weight_min": -25.261051177978516,
      "activations/layer23_attention_weight_max": 38.711753845214844,
      "activations/layer23_attention_weight_min": -21.413928985595703,
      "activations/layer2_attention_weight_max": 33.24359893798828,
      "activations/layer2_attention_weight_min": -31.697660446166992,
      "activations/layer3_attention_weight_max": 91.51927185058594,
      "activations/layer3_attention_weight_min": -94.72346496582031,
      "activations/layer4_attention_weight_max": 108.27520751953125,
      "activations/layer4_attention_weight_min": -105.31685638427734,
      "activations/layer5_attention_weight_max": 98.96292114257812,
      "activations/layer5_attention_weight_min": -91.52766418457031,
      "activations/layer6_attention_weight_max": 60.918365478515625,
      "activations/layer6_attention_weight_min": -56.6977424621582,
      "activations/layer7_attention_weight_max": 84.2568588256836,
      "activations/layer7_attention_weight_min": -77.92543029785156,
      "activations/layer8_attention_weight_max": 51.919769287109375,
      "activations/layer8_attention_weight_min": -47.4996223449707,
      "activations/layer9_attention_weight_max": 66.63961029052734,
      "activations/layer9_attention_weight_min": -52.199310302734375,
      "epoch": 6.87,
      "learning_rate": 0.00010680340909090908,
      "loss": 2.8301,
      "step": 118150
    },
    {
      "activations/layer0_attention_weight_max": 14.058769226074219,
      "activations/layer0_attention_weight_min": -14.975471496582031,
      "activations/layer10_attention_weight_max": 41.999332427978516,
      "activations/layer10_attention_weight_min": -38.42565155029297,
      "activations/layer11_attention_weight_max": 32.41245651245117,
      "activations/layer11_attention_weight_min": -31.923992156982422,
      "activations/layer12_attention_weight_max": 20.477039337158203,
      "activations/layer12_attention_weight_min": -26.81833839416504,
      "activations/layer13_attention_weight_max": 58.67439651489258,
      "activations/layer13_attention_weight_min": -36.23396301269531,
      "activations/layer14_attention_weight_max": 52.960365295410156,
      "activations/layer14_attention_weight_min": -38.55917739868164,
      "activations/layer15_attention_weight_max": 45.919647216796875,
      "activations/layer15_attention_weight_min": -28.361858367919922,
      "activations/layer16_attention_weight_max": 31.87591552734375,
      "activations/layer16_attention_weight_min": -25.798189163208008,
      "activations/layer17_attention_weight_max": 55.254432678222656,
      "activations/layer17_attention_weight_min": -41.60860061645508,
      "activations/layer18_attention_weight_max": 46.7979736328125,
      "activations/layer18_attention_weight_min": -35.867427825927734,
      "activations/layer19_attention_weight_max": 23.044891357421875,
      "activations/layer19_attention_weight_min": -21.9506778717041,
      "activations/layer1_attention_weight_max": 17.421091079711914,
      "activations/layer1_attention_weight_min": -17.674577713012695,
      "activations/layer20_attention_weight_max": 21.540264129638672,
      "activations/layer20_attention_weight_min": -21.05476188659668,
      "activations/layer21_attention_weight_max": 40.89183807373047,
      "activations/layer21_attention_weight_min": -23.740325927734375,
      "activations/layer22_attention_weight_max": 32.1541748046875,
      "activations/layer22_attention_weight_min": -24.547393798828125,
      "activations/layer23_attention_weight_max": 36.55693435668945,
      "activations/layer23_attention_weight_min": -18.67969512939453,
      "activations/layer2_attention_weight_max": 32.35591125488281,
      "activations/layer2_attention_weight_min": -30.396564483642578,
      "activations/layer3_attention_weight_max": 82.28941345214844,
      "activations/layer3_attention_weight_min": -85.75656127929688,
      "activations/layer4_attention_weight_max": 102.91639709472656,
      "activations/layer4_attention_weight_min": -93.9056396484375,
      "activations/layer5_attention_weight_max": 95.30184173583984,
      "activations/layer5_attention_weight_min": -84.6294937133789,
      "activations/layer6_attention_weight_max": 58.88197326660156,
      "activations/layer6_attention_weight_min": -53.29910659790039,
      "activations/layer7_attention_weight_max": 78.92595672607422,
      "activations/layer7_attention_weight_min": -73.5025634765625,
      "activations/layer8_attention_weight_max": 48.16497039794922,
      "activations/layer8_attention_weight_min": -48.59358596801758,
      "activations/layer9_attention_weight_max": 55.013526916503906,
      "activations/layer9_attention_weight_min": -49.87958526611328,
      "epoch": 6.87,
      "learning_rate": 0.00010678446969696969,
      "loss": 2.8261,
      "step": 118200
    },
    {
      "activations/layer0_attention_weight_max": 16.123620986938477,
      "activations/layer0_attention_weight_min": -15.0160551071167,
      "activations/layer10_attention_weight_max": 37.48569869995117,
      "activations/layer10_attention_weight_min": -34.074642181396484,
      "activations/layer11_attention_weight_max": 30.323949813842773,
      "activations/layer11_attention_weight_min": -29.469196319580078,
      "activations/layer12_attention_weight_max": 20.15172004699707,
      "activations/layer12_attention_weight_min": -25.413360595703125,
      "activations/layer13_attention_weight_max": 40.10659408569336,
      "activations/layer13_attention_weight_min": -31.62006378173828,
      "activations/layer14_attention_weight_max": 38.041229248046875,
      "activations/layer14_attention_weight_min": -29.572635650634766,
      "activations/layer15_attention_weight_max": 37.7384147644043,
      "activations/layer15_attention_weight_min": -30.105735778808594,
      "activations/layer16_attention_weight_max": 33.60499954223633,
      "activations/layer16_attention_weight_min": -26.74424934387207,
      "activations/layer17_attention_weight_max": 56.39273452758789,
      "activations/layer17_attention_weight_min": -41.45277404785156,
      "activations/layer18_attention_weight_max": 47.617271423339844,
      "activations/layer18_attention_weight_min": -36.15520477294922,
      "activations/layer19_attention_weight_max": 21.022268295288086,
      "activations/layer19_attention_weight_min": -19.96900749206543,
      "activations/layer1_attention_weight_max": 16.988019943237305,
      "activations/layer1_attention_weight_min": -15.88880443572998,
      "activations/layer20_attention_weight_max": 23.37682342529297,
      "activations/layer20_attention_weight_min": -23.089519500732422,
      "activations/layer21_attention_weight_max": 40.29724884033203,
      "activations/layer21_attention_weight_min": -25.08109474182129,
      "activations/layer22_attention_weight_max": 30.534332275390625,
      "activations/layer22_attention_weight_min": -24.092275619506836,
      "activations/layer23_attention_weight_max": 33.29945373535156,
      "activations/layer23_attention_weight_min": -21.065364837646484,
      "activations/layer2_attention_weight_max": 29.97535514831543,
      "activations/layer2_attention_weight_min": -28.513320922851562,
      "activations/layer3_attention_weight_max": 79.47530364990234,
      "activations/layer3_attention_weight_min": -83.98573303222656,
      "activations/layer4_attention_weight_max": 97.65873718261719,
      "activations/layer4_attention_weight_min": -95.86727905273438,
      "activations/layer5_attention_weight_max": 90.13114929199219,
      "activations/layer5_attention_weight_min": -87.5784912109375,
      "activations/layer6_attention_weight_max": 57.2417106628418,
      "activations/layer6_attention_weight_min": -52.957542419433594,
      "activations/layer7_attention_weight_max": 73.87512969970703,
      "activations/layer7_attention_weight_min": -74.06942749023438,
      "activations/layer8_attention_weight_max": 46.90605926513672,
      "activations/layer8_attention_weight_min": -44.263877868652344,
      "activations/layer9_attention_weight_max": 50.5838508605957,
      "activations/layer9_attention_weight_min": -50.5500373840332,
      "epoch": 6.87,
      "learning_rate": 0.0001067655303030303,
      "loss": 2.8411,
      "step": 118250
    },
    {
      "activations/layer0_attention_weight_max": 14.902825355529785,
      "activations/layer0_attention_weight_min": -15.074799537658691,
      "activations/layer10_attention_weight_max": 38.502593994140625,
      "activations/layer10_attention_weight_min": -36.778778076171875,
      "activations/layer11_attention_weight_max": 33.550384521484375,
      "activations/layer11_attention_weight_min": -30.681119918823242,
      "activations/layer12_attention_weight_max": 20.68026351928711,
      "activations/layer12_attention_weight_min": -25.5279598236084,
      "activations/layer13_attention_weight_max": 39.93299102783203,
      "activations/layer13_attention_weight_min": -31.450464248657227,
      "activations/layer14_attention_weight_max": 45.981483459472656,
      "activations/layer14_attention_weight_min": -30.30365562438965,
      "activations/layer15_attention_weight_max": 39.580039978027344,
      "activations/layer15_attention_weight_min": -29.420289993286133,
      "activations/layer16_attention_weight_max": 32.092369079589844,
      "activations/layer16_attention_weight_min": -25.81087875366211,
      "activations/layer17_attention_weight_max": 56.815582275390625,
      "activations/layer17_attention_weight_min": -43.42692947387695,
      "activations/layer18_attention_weight_max": 51.90989685058594,
      "activations/layer18_attention_weight_min": -37.661128997802734,
      "activations/layer19_attention_weight_max": 21.49178123474121,
      "activations/layer19_attention_weight_min": -20.1566162109375,
      "activations/layer1_attention_weight_max": 16.604143142700195,
      "activations/layer1_attention_weight_min": -16.426918029785156,
      "activations/layer20_attention_weight_max": 23.243301391601562,
      "activations/layer20_attention_weight_min": -21.761741638183594,
      "activations/layer21_attention_weight_max": 36.67791748046875,
      "activations/layer21_attention_weight_min": -24.168853759765625,
      "activations/layer22_attention_weight_max": 31.956119537353516,
      "activations/layer22_attention_weight_min": -23.87421417236328,
      "activations/layer23_attention_weight_max": 30.500295639038086,
      "activations/layer23_attention_weight_min": -19.013628005981445,
      "activations/layer2_attention_weight_max": 32.82440948486328,
      "activations/layer2_attention_weight_min": -29.917102813720703,
      "activations/layer3_attention_weight_max": 86.0566635131836,
      "activations/layer3_attention_weight_min": -91.76853942871094,
      "activations/layer4_attention_weight_max": 99.1019058227539,
      "activations/layer4_attention_weight_min": -96.32128143310547,
      "activations/layer5_attention_weight_max": 89.92498016357422,
      "activations/layer5_attention_weight_min": -85.33299255371094,
      "activations/layer6_attention_weight_max": 58.21220016479492,
      "activations/layer6_attention_weight_min": -57.96775817871094,
      "activations/layer7_attention_weight_max": 73.52790069580078,
      "activations/layer7_attention_weight_min": -73.74496459960938,
      "activations/layer8_attention_weight_max": 47.44149398803711,
      "activations/layer8_attention_weight_min": -45.49446487426758,
      "activations/layer9_attention_weight_max": 51.28145217895508,
      "activations/layer9_attention_weight_min": -54.773353576660156,
      "epoch": 6.87,
      "learning_rate": 0.0001067465909090909,
      "loss": 2.8211,
      "step": 118300
    },
    {
      "activations/layer0_attention_weight_max": 15.139444351196289,
      "activations/layer0_attention_weight_min": -14.644231796264648,
      "activations/layer10_attention_weight_max": 40.163612365722656,
      "activations/layer10_attention_weight_min": -39.429237365722656,
      "activations/layer11_attention_weight_max": 30.200756072998047,
      "activations/layer11_attention_weight_min": -30.31884765625,
      "activations/layer12_attention_weight_max": 21.84815788269043,
      "activations/layer12_attention_weight_min": -26.77728843688965,
      "activations/layer13_attention_weight_max": 44.644168853759766,
      "activations/layer13_attention_weight_min": -35.18158721923828,
      "activations/layer14_attention_weight_max": 41.410255432128906,
      "activations/layer14_attention_weight_min": -30.627220153808594,
      "activations/layer15_attention_weight_max": 36.410125732421875,
      "activations/layer15_attention_weight_min": -28.888351440429688,
      "activations/layer16_attention_weight_max": 36.398902893066406,
      "activations/layer16_attention_weight_min": -27.02344512939453,
      "activations/layer17_attention_weight_max": 53.296234130859375,
      "activations/layer17_attention_weight_min": -41.64668655395508,
      "activations/layer18_attention_weight_max": 47.76361846923828,
      "activations/layer18_attention_weight_min": -35.1552848815918,
      "activations/layer19_attention_weight_max": 21.220701217651367,
      "activations/layer19_attention_weight_min": -20.747318267822266,
      "activations/layer1_attention_weight_max": 16.792434692382812,
      "activations/layer1_attention_weight_min": -14.743722915649414,
      "activations/layer20_attention_weight_max": 21.838390350341797,
      "activations/layer20_attention_weight_min": -20.378633499145508,
      "activations/layer21_attention_weight_max": 35.10493469238281,
      "activations/layer21_attention_weight_min": -23.607988357543945,
      "activations/layer22_attention_weight_max": 34.28092575073242,
      "activations/layer22_attention_weight_min": -25.737459182739258,
      "activations/layer23_attention_weight_max": 35.3624267578125,
      "activations/layer23_attention_weight_min": -19.96487808227539,
      "activations/layer2_attention_weight_max": 32.02838134765625,
      "activations/layer2_attention_weight_min": -29.07583999633789,
      "activations/layer3_attention_weight_max": 85.08413696289062,
      "activations/layer3_attention_weight_min": -89.72686767578125,
      "activations/layer4_attention_weight_max": 102.13542938232422,
      "activations/layer4_attention_weight_min": -93.95662689208984,
      "activations/layer5_attention_weight_max": 87.23445892333984,
      "activations/layer5_attention_weight_min": -83.14037322998047,
      "activations/layer6_attention_weight_max": 53.02803421020508,
      "activations/layer6_attention_weight_min": -53.898597717285156,
      "activations/layer7_attention_weight_max": 72.57569885253906,
      "activations/layer7_attention_weight_min": -75.3955307006836,
      "activations/layer8_attention_weight_max": 48.25101852416992,
      "activations/layer8_attention_weight_min": -52.06773376464844,
      "activations/layer9_attention_weight_max": 51.731441497802734,
      "activations/layer9_attention_weight_min": -49.66267013549805,
      "epoch": 6.88,
      "learning_rate": 0.0001067276515151515,
      "loss": 2.8324,
      "step": 118350
    },
    {
      "activations/layer0_attention_weight_max": 14.956080436706543,
      "activations/layer0_attention_weight_min": -14.732771873474121,
      "activations/layer10_attention_weight_max": 39.817691802978516,
      "activations/layer10_attention_weight_min": -33.47612380981445,
      "activations/layer11_attention_weight_max": 31.73332977294922,
      "activations/layer11_attention_weight_min": -29.528026580810547,
      "activations/layer12_attention_weight_max": 20.17039680480957,
      "activations/layer12_attention_weight_min": -26.418363571166992,
      "activations/layer13_attention_weight_max": 42.260597229003906,
      "activations/layer13_attention_weight_min": -31.504776000976562,
      "activations/layer14_attention_weight_max": 36.0755500793457,
      "activations/layer14_attention_weight_min": -30.506885528564453,
      "activations/layer15_attention_weight_max": 35.63331604003906,
      "activations/layer15_attention_weight_min": -30.780887603759766,
      "activations/layer16_attention_weight_max": 29.99369239807129,
      "activations/layer16_attention_weight_min": -26.085628509521484,
      "activations/layer17_attention_weight_max": 55.883888244628906,
      "activations/layer17_attention_weight_min": -39.81675720214844,
      "activations/layer18_attention_weight_max": 52.06632614135742,
      "activations/layer18_attention_weight_min": -35.09043884277344,
      "activations/layer19_attention_weight_max": 20.18433952331543,
      "activations/layer19_attention_weight_min": -20.73570442199707,
      "activations/layer1_attention_weight_max": 18.029951095581055,
      "activations/layer1_attention_weight_min": -15.677308082580566,
      "activations/layer20_attention_weight_max": 22.700435638427734,
      "activations/layer20_attention_weight_min": -21.173093795776367,
      "activations/layer21_attention_weight_max": 36.052825927734375,
      "activations/layer21_attention_weight_min": -24.862957000732422,
      "activations/layer22_attention_weight_max": 36.01152420043945,
      "activations/layer22_attention_weight_min": -25.01743507385254,
      "activations/layer23_attention_weight_max": 34.50234603881836,
      "activations/layer23_attention_weight_min": -21.358108520507812,
      "activations/layer2_attention_weight_max": 33.027061462402344,
      "activations/layer2_attention_weight_min": -30.28607940673828,
      "activations/layer3_attention_weight_max": 86.69091033935547,
      "activations/layer3_attention_weight_min": -86.595947265625,
      "activations/layer4_attention_weight_max": 100.1410903930664,
      "activations/layer4_attention_weight_min": -97.79739379882812,
      "activations/layer5_attention_weight_max": 95.73036193847656,
      "activations/layer5_attention_weight_min": -88.58572387695312,
      "activations/layer6_attention_weight_max": 57.24415588378906,
      "activations/layer6_attention_weight_min": -53.02043914794922,
      "activations/layer7_attention_weight_max": 76.03882598876953,
      "activations/layer7_attention_weight_min": -72.07804107666016,
      "activations/layer8_attention_weight_max": 47.29663848876953,
      "activations/layer8_attention_weight_min": -44.76630401611328,
      "activations/layer9_attention_weight_max": 52.18994140625,
      "activations/layer9_attention_weight_min": -50.60846710205078,
      "epoch": 6.88,
      "learning_rate": 0.00010670871212121212,
      "loss": 2.8483,
      "step": 118400
    },
    {
      "activations/layer0_attention_weight_max": 15.975235939025879,
      "activations/layer0_attention_weight_min": -15.011661529541016,
      "activations/layer10_attention_weight_max": 40.30237579345703,
      "activations/layer10_attention_weight_min": -34.34109878540039,
      "activations/layer11_attention_weight_max": 34.47772216796875,
      "activations/layer11_attention_weight_min": -31.567455291748047,
      "activations/layer12_attention_weight_max": 25.874832153320312,
      "activations/layer12_attention_weight_min": -26.672643661499023,
      "activations/layer13_attention_weight_max": 63.19260787963867,
      "activations/layer13_attention_weight_min": -39.946250915527344,
      "activations/layer14_attention_weight_max": 96.18421936035156,
      "activations/layer14_attention_weight_min": -48.23601150512695,
      "activations/layer15_attention_weight_max": 72.06523895263672,
      "activations/layer15_attention_weight_min": -40.159446716308594,
      "activations/layer16_attention_weight_max": 38.82633972167969,
      "activations/layer16_attention_weight_min": -27.080026626586914,
      "activations/layer17_attention_weight_max": 58.62895584106445,
      "activations/layer17_attention_weight_min": -44.54019546508789,
      "activations/layer18_attention_weight_max": 49.44245910644531,
      "activations/layer18_attention_weight_min": -36.77107238769531,
      "activations/layer19_attention_weight_max": 33.774784088134766,
      "activations/layer19_attention_weight_min": -21.166250228881836,
      "activations/layer1_attention_weight_max": 16.51329803466797,
      "activations/layer1_attention_weight_min": -16.213035583496094,
      "activations/layer20_attention_weight_max": 34.79581069946289,
      "activations/layer20_attention_weight_min": -28.978715896606445,
      "activations/layer21_attention_weight_max": 65.35126495361328,
      "activations/layer21_attention_weight_min": -27.452098846435547,
      "activations/layer22_attention_weight_max": 37.579566955566406,
      "activations/layer22_attention_weight_min": -25.051939010620117,
      "activations/layer23_attention_weight_max": 34.1832275390625,
      "activations/layer23_attention_weight_min": -24.33580780029297,
      "activations/layer2_attention_weight_max": 32.07280349731445,
      "activations/layer2_attention_weight_min": -29.370304107666016,
      "activations/layer3_attention_weight_max": 87.51383972167969,
      "activations/layer3_attention_weight_min": -85.90554809570312,
      "activations/layer4_attention_weight_max": 99.8104476928711,
      "activations/layer4_attention_weight_min": -91.43865203857422,
      "activations/layer5_attention_weight_max": 91.24273681640625,
      "activations/layer5_attention_weight_min": -84.27263641357422,
      "activations/layer6_attention_weight_max": 53.48194885253906,
      "activations/layer6_attention_weight_min": -52.613826751708984,
      "activations/layer7_attention_weight_max": 75.04048156738281,
      "activations/layer7_attention_weight_min": -72.39511108398438,
      "activations/layer8_attention_weight_max": 44.577003479003906,
      "activations/layer8_attention_weight_min": -42.79037094116211,
      "activations/layer9_attention_weight_max": 64.74251556396484,
      "activations/layer9_attention_weight_min": -52.949275970458984,
      "epoch": 6.88,
      "learning_rate": 0.00010668977272727271,
      "loss": 2.8378,
      "step": 118450
    },
    {
      "activations/layer0_attention_weight_max": 14.51435375213623,
      "activations/layer0_attention_weight_min": -14.791476249694824,
      "activations/layer10_attention_weight_max": 38.806095123291016,
      "activations/layer10_attention_weight_min": -35.17954635620117,
      "activations/layer11_attention_weight_max": 31.268489837646484,
      "activations/layer11_attention_weight_min": -30.152633666992188,
      "activations/layer12_attention_weight_max": 29.79365348815918,
      "activations/layer12_attention_weight_min": -22.474918365478516,
      "activations/layer13_attention_weight_max": 41.722076416015625,
      "activations/layer13_attention_weight_min": -33.589561462402344,
      "activations/layer14_attention_weight_max": 39.40239715576172,
      "activations/layer14_attention_weight_min": -32.932037353515625,
      "activations/layer15_attention_weight_max": 39.27372360229492,
      "activations/layer15_attention_weight_min": -28.756465911865234,
      "activations/layer16_attention_weight_max": 31.796171188354492,
      "activations/layer16_attention_weight_min": -26.637557983398438,
      "activations/layer17_attention_weight_max": 56.582210540771484,
      "activations/layer17_attention_weight_min": -40.0400390625,
      "activations/layer18_attention_weight_max": 50.64410400390625,
      "activations/layer18_attention_weight_min": -33.282257080078125,
      "activations/layer19_attention_weight_max": 21.59767723083496,
      "activations/layer19_attention_weight_min": -19.862342834472656,
      "activations/layer1_attention_weight_max": 16.639957427978516,
      "activations/layer1_attention_weight_min": -15.339096069335938,
      "activations/layer20_attention_weight_max": 24.657567977905273,
      "activations/layer20_attention_weight_min": -21.86622428894043,
      "activations/layer21_attention_weight_max": 43.10695266723633,
      "activations/layer21_attention_weight_min": -24.080373764038086,
      "activations/layer22_attention_weight_max": 33.16347885131836,
      "activations/layer22_attention_weight_min": -24.82939338684082,
      "activations/layer23_attention_weight_max": 33.772029876708984,
      "activations/layer23_attention_weight_min": -20.10702896118164,
      "activations/layer2_attention_weight_max": 29.459548950195312,
      "activations/layer2_attention_weight_min": -29.451229095458984,
      "activations/layer3_attention_weight_max": 83.77362823486328,
      "activations/layer3_attention_weight_min": -80.21910095214844,
      "activations/layer4_attention_weight_max": 97.50267791748047,
      "activations/layer4_attention_weight_min": -90.13640594482422,
      "activations/layer5_attention_weight_max": 90.48180389404297,
      "activations/layer5_attention_weight_min": -83.40655517578125,
      "activations/layer6_attention_weight_max": 53.061058044433594,
      "activations/layer6_attention_weight_min": -52.129215240478516,
      "activations/layer7_attention_weight_max": 71.07987976074219,
      "activations/layer7_attention_weight_min": -72.69694519042969,
      "activations/layer8_attention_weight_max": 44.50676727294922,
      "activations/layer8_attention_weight_min": -46.510372161865234,
      "activations/layer9_attention_weight_max": 48.706180572509766,
      "activations/layer9_attention_weight_min": -51.52667999267578,
      "epoch": 6.89,
      "learning_rate": 0.00010667083333333332,
      "loss": 2.8276,
      "step": 118500
    },
    {
      "activations/layer0_attention_weight_max": 14.545605659484863,
      "activations/layer0_attention_weight_min": -14.9757719039917,
      "activations/layer10_attention_weight_max": 36.77535629272461,
      "activations/layer10_attention_weight_min": -38.65044021606445,
      "activations/layer11_attention_weight_max": 30.191604614257812,
      "activations/layer11_attention_weight_min": -29.257888793945312,
      "activations/layer12_attention_weight_max": 18.730249404907227,
      "activations/layer12_attention_weight_min": -27.54416275024414,
      "activations/layer13_attention_weight_max": 48.938636779785156,
      "activations/layer13_attention_weight_min": -36.2352409362793,
      "activations/layer14_attention_weight_max": 50.78811264038086,
      "activations/layer14_attention_weight_min": -32.10175323486328,
      "activations/layer15_attention_weight_max": 36.47350311279297,
      "activations/layer15_attention_weight_min": -28.56338119506836,
      "activations/layer16_attention_weight_max": 37.00239181518555,
      "activations/layer16_attention_weight_min": -26.01532554626465,
      "activations/layer17_attention_weight_max": 52.66202163696289,
      "activations/layer17_attention_weight_min": -40.14907455444336,
      "activations/layer18_attention_weight_max": 46.70889663696289,
      "activations/layer18_attention_weight_min": -34.99130630493164,
      "activations/layer19_attention_weight_max": 24.111804962158203,
      "activations/layer19_attention_weight_min": -19.965662002563477,
      "activations/layer1_attention_weight_max": 17.79642677307129,
      "activations/layer1_attention_weight_min": -19.45318031311035,
      "activations/layer20_attention_weight_max": 20.72730827331543,
      "activations/layer20_attention_weight_min": -20.40022087097168,
      "activations/layer21_attention_weight_max": 43.20769119262695,
      "activations/layer21_attention_weight_min": -25.45433235168457,
      "activations/layer22_attention_weight_max": 35.89773941040039,
      "activations/layer22_attention_weight_min": -25.12347984313965,
      "activations/layer23_attention_weight_max": 35.70485305786133,
      "activations/layer23_attention_weight_min": -21.55014419555664,
      "activations/layer2_attention_weight_max": 34.73621368408203,
      "activations/layer2_attention_weight_min": -31.051197052001953,
      "activations/layer3_attention_weight_max": 83.20072937011719,
      "activations/layer3_attention_weight_min": -83.60230255126953,
      "activations/layer4_attention_weight_max": 99.93739318847656,
      "activations/layer4_attention_weight_min": -89.3392562866211,
      "activations/layer5_attention_weight_max": 90.82662963867188,
      "activations/layer5_attention_weight_min": -85.08049774169922,
      "activations/layer6_attention_weight_max": 57.564109802246094,
      "activations/layer6_attention_weight_min": -52.25438690185547,
      "activations/layer7_attention_weight_max": 71.75264739990234,
      "activations/layer7_attention_weight_min": -68.67366027832031,
      "activations/layer8_attention_weight_max": 44.024017333984375,
      "activations/layer8_attention_weight_min": -45.77397918701172,
      "activations/layer9_attention_weight_max": 61.354854583740234,
      "activations/layer9_attention_weight_min": -50.30363082885742,
      "epoch": 6.89,
      "learning_rate": 0.00010665189393939394,
      "loss": 2.8194,
      "step": 118550
    },
    {
      "activations/layer0_attention_weight_max": 13.899770736694336,
      "activations/layer0_attention_weight_min": -15.490304946899414,
      "activations/layer10_attention_weight_max": 37.6264762878418,
      "activations/layer10_attention_weight_min": -35.812095642089844,
      "activations/layer11_attention_weight_max": 29.920194625854492,
      "activations/layer11_attention_weight_min": -29.605506896972656,
      "activations/layer12_attention_weight_max": 21.019399642944336,
      "activations/layer12_attention_weight_min": -29.10886573791504,
      "activations/layer13_attention_weight_max": 52.23744583129883,
      "activations/layer13_attention_weight_min": -31.08762550354004,
      "activations/layer14_attention_weight_max": 67.93740844726562,
      "activations/layer14_attention_weight_min": -35.56938171386719,
      "activations/layer15_attention_weight_max": 45.27497482299805,
      "activations/layer15_attention_weight_min": -28.201181411743164,
      "activations/layer16_attention_weight_max": 32.02592468261719,
      "activations/layer16_attention_weight_min": -26.891490936279297,
      "activations/layer17_attention_weight_max": 55.32167434692383,
      "activations/layer17_attention_weight_min": -38.802154541015625,
      "activations/layer18_attention_weight_max": 46.844886779785156,
      "activations/layer18_attention_weight_min": -32.773468017578125,
      "activations/layer19_attention_weight_max": 22.740543365478516,
      "activations/layer19_attention_weight_min": -18.43262481689453,
      "activations/layer1_attention_weight_max": 16.94057846069336,
      "activations/layer1_attention_weight_min": -15.908875465393066,
      "activations/layer20_attention_weight_max": 19.78543472290039,
      "activations/layer20_attention_weight_min": -23.083415985107422,
      "activations/layer21_attention_weight_max": 34.7254753112793,
      "activations/layer21_attention_weight_min": -23.831558227539062,
      "activations/layer22_attention_weight_max": 32.97415542602539,
      "activations/layer22_attention_weight_min": -26.629560470581055,
      "activations/layer23_attention_weight_max": 34.11529541015625,
      "activations/layer23_attention_weight_min": -18.355789184570312,
      "activations/layer2_attention_weight_max": 29.875102996826172,
      "activations/layer2_attention_weight_min": -30.194974899291992,
      "activations/layer3_attention_weight_max": 82.50407409667969,
      "activations/layer3_attention_weight_min": -89.01512908935547,
      "activations/layer4_attention_weight_max": 97.12398529052734,
      "activations/layer4_attention_weight_min": -89.3706283569336,
      "activations/layer5_attention_weight_max": 86.31829833984375,
      "activations/layer5_attention_weight_min": -82.4159927368164,
      "activations/layer6_attention_weight_max": 54.20936584472656,
      "activations/layer6_attention_weight_min": -51.436134338378906,
      "activations/layer7_attention_weight_max": 73.31254577636719,
      "activations/layer7_attention_weight_min": -69.00685119628906,
      "activations/layer8_attention_weight_max": 42.298927307128906,
      "activations/layer8_attention_weight_min": -44.00978088378906,
      "activations/layer9_attention_weight_max": 57.7626953125,
      "activations/layer9_attention_weight_min": -50.42648696899414,
      "epoch": 6.89,
      "learning_rate": 0.00010663295454545452,
      "loss": 2.8303,
      "step": 118600
    },
    {
      "activations/layer0_attention_weight_max": 15.204063415527344,
      "activations/layer0_attention_weight_min": -14.89274787902832,
      "activations/layer10_attention_weight_max": 35.98993682861328,
      "activations/layer10_attention_weight_min": -34.105350494384766,
      "activations/layer11_attention_weight_max": 29.435575485229492,
      "activations/layer11_attention_weight_min": -30.26619529724121,
      "activations/layer12_attention_weight_max": 20.304489135742188,
      "activations/layer12_attention_weight_min": -30.37481117248535,
      "activations/layer13_attention_weight_max": 46.583492279052734,
      "activations/layer13_attention_weight_min": -29.75225067138672,
      "activations/layer14_attention_weight_max": 39.345420837402344,
      "activations/layer14_attention_weight_min": -29.49564552307129,
      "activations/layer15_attention_weight_max": 37.70621109008789,
      "activations/layer15_attention_weight_min": -27.237186431884766,
      "activations/layer16_attention_weight_max": 37.007965087890625,
      "activations/layer16_attention_weight_min": -26.07335090637207,
      "activations/layer17_attention_weight_max": 60.26063537597656,
      "activations/layer17_attention_weight_min": -42.21836471557617,
      "activations/layer18_attention_weight_max": 51.00014877319336,
      "activations/layer18_attention_weight_min": -37.6287956237793,
      "activations/layer19_attention_weight_max": 22.86980628967285,
      "activations/layer19_attention_weight_min": -20.398801803588867,
      "activations/layer1_attention_weight_max": 17.368404388427734,
      "activations/layer1_attention_weight_min": -16.753700256347656,
      "activations/layer20_attention_weight_max": 22.550182342529297,
      "activations/layer20_attention_weight_min": -24.093265533447266,
      "activations/layer21_attention_weight_max": 44.24275588989258,
      "activations/layer21_attention_weight_min": -24.51445198059082,
      "activations/layer22_attention_weight_max": 32.709354400634766,
      "activations/layer22_attention_weight_min": -25.226701736450195,
      "activations/layer23_attention_weight_max": 32.835716247558594,
      "activations/layer23_attention_weight_min": -23.140216827392578,
      "activations/layer2_attention_weight_max": 33.124656677246094,
      "activations/layer2_attention_weight_min": -30.855045318603516,
      "activations/layer3_attention_weight_max": 88.48330688476562,
      "activations/layer3_attention_weight_min": -94.65658569335938,
      "activations/layer4_attention_weight_max": 102.47216033935547,
      "activations/layer4_attention_weight_min": -100.69731903076172,
      "activations/layer5_attention_weight_max": 93.18989562988281,
      "activations/layer5_attention_weight_min": -84.64824676513672,
      "activations/layer6_attention_weight_max": 52.72615051269531,
      "activations/layer6_attention_weight_min": -50.86650848388672,
      "activations/layer7_attention_weight_max": 70.64823913574219,
      "activations/layer7_attention_weight_min": -68.18995666503906,
      "activations/layer8_attention_weight_max": 44.2762336730957,
      "activations/layer8_attention_weight_min": -44.652381896972656,
      "activations/layer9_attention_weight_max": 49.37870788574219,
      "activations/layer9_attention_weight_min": -48.73499298095703,
      "epoch": 6.89,
      "learning_rate": 0.00010661401515151514,
      "loss": 2.8122,
      "step": 118650
    },
    {
      "activations/layer0_attention_weight_max": 15.228278160095215,
      "activations/layer0_attention_weight_min": -15.057491302490234,
      "activations/layer10_attention_weight_max": 41.59038162231445,
      "activations/layer10_attention_weight_min": -37.771759033203125,
      "activations/layer11_attention_weight_max": 32.80914306640625,
      "activations/layer11_attention_weight_min": -30.986303329467773,
      "activations/layer12_attention_weight_max": 20.119810104370117,
      "activations/layer12_attention_weight_min": -25.26202964782715,
      "activations/layer13_attention_weight_max": 40.871150970458984,
      "activations/layer13_attention_weight_min": -29.097492218017578,
      "activations/layer14_attention_weight_max": 38.33436965942383,
      "activations/layer14_attention_weight_min": -32.530460357666016,
      "activations/layer15_attention_weight_max": 37.87654495239258,
      "activations/layer15_attention_weight_min": -31.7984676361084,
      "activations/layer16_attention_weight_max": 32.39626693725586,
      "activations/layer16_attention_weight_min": -28.19769287109375,
      "activations/layer17_attention_weight_max": 56.47812271118164,
      "activations/layer17_attention_weight_min": -41.835777282714844,
      "activations/layer18_attention_weight_max": 48.24439239501953,
      "activations/layer18_attention_weight_min": -36.0692253112793,
      "activations/layer19_attention_weight_max": 24.604528427124023,
      "activations/layer19_attention_weight_min": -20.22626495361328,
      "activations/layer1_attention_weight_max": 17.543766021728516,
      "activations/layer1_attention_weight_min": -18.30300521850586,
      "activations/layer20_attention_weight_max": 22.509132385253906,
      "activations/layer20_attention_weight_min": -22.26900291442871,
      "activations/layer21_attention_weight_max": 40.905906677246094,
      "activations/layer21_attention_weight_min": -23.261581420898438,
      "activations/layer22_attention_weight_max": 33.510597229003906,
      "activations/layer22_attention_weight_min": -27.22681999206543,
      "activations/layer23_attention_weight_max": 37.54209899902344,
      "activations/layer23_attention_weight_min": -20.812053680419922,
      "activations/layer2_attention_weight_max": 30.868656158447266,
      "activations/layer2_attention_weight_min": -29.337491989135742,
      "activations/layer3_attention_weight_max": 87.3024673461914,
      "activations/layer3_attention_weight_min": -88.50118255615234,
      "activations/layer4_attention_weight_max": 102.0587158203125,
      "activations/layer4_attention_weight_min": -95.8839340209961,
      "activations/layer5_attention_weight_max": 92.35594177246094,
      "activations/layer5_attention_weight_min": -85.90721130371094,
      "activations/layer6_attention_weight_max": 59.673057556152344,
      "activations/layer6_attention_weight_min": -54.04130935668945,
      "activations/layer7_attention_weight_max": 78.15736389160156,
      "activations/layer7_attention_weight_min": -73.4395980834961,
      "activations/layer8_attention_weight_max": 46.30277633666992,
      "activations/layer8_attention_weight_min": -46.13660430908203,
      "activations/layer9_attention_weight_max": 53.98347854614258,
      "activations/layer9_attention_weight_min": -48.709041595458984,
      "epoch": 6.9,
      "learning_rate": 0.00010659507575757575,
      "loss": 2.8376,
      "step": 118700
    },
    {
      "activations/layer0_attention_weight_max": 15.047788619995117,
      "activations/layer0_attention_weight_min": -14.999579429626465,
      "activations/layer10_attention_weight_max": 43.64611053466797,
      "activations/layer10_attention_weight_min": -36.940155029296875,
      "activations/layer11_attention_weight_max": 37.482765197753906,
      "activations/layer11_attention_weight_min": -30.290769577026367,
      "activations/layer12_attention_weight_max": 31.862571716308594,
      "activations/layer12_attention_weight_min": -24.12326431274414,
      "activations/layer13_attention_weight_max": 51.36421585083008,
      "activations/layer13_attention_weight_min": -30.946802139282227,
      "activations/layer14_attention_weight_max": 43.682029724121094,
      "activations/layer14_attention_weight_min": -30.93065071105957,
      "activations/layer15_attention_weight_max": 40.17377471923828,
      "activations/layer15_attention_weight_min": -30.15144920349121,
      "activations/layer16_attention_weight_max": 38.254730224609375,
      "activations/layer16_attention_weight_min": -26.592426300048828,
      "activations/layer17_attention_weight_max": 58.5067138671875,
      "activations/layer17_attention_weight_min": -44.95804977416992,
      "activations/layer18_attention_weight_max": 53.0048828125,
      "activations/layer18_attention_weight_min": -35.529510498046875,
      "activations/layer19_attention_weight_max": 28.97539710998535,
      "activations/layer19_attention_weight_min": -20.152971267700195,
      "activations/layer1_attention_weight_max": 18.271663665771484,
      "activations/layer1_attention_weight_min": -15.338848114013672,
      "activations/layer20_attention_weight_max": 25.808921813964844,
      "activations/layer20_attention_weight_min": -21.16001319885254,
      "activations/layer21_attention_weight_max": 48.282798767089844,
      "activations/layer21_attention_weight_min": -24.239660263061523,
      "activations/layer22_attention_weight_max": 39.99964904785156,
      "activations/layer22_attention_weight_min": -22.921037673950195,
      "activations/layer23_attention_weight_max": 37.3104248046875,
      "activations/layer23_attention_weight_min": -16.27910804748535,
      "activations/layer2_attention_weight_max": 31.787044525146484,
      "activations/layer2_attention_weight_min": -30.35847282409668,
      "activations/layer3_attention_weight_max": 87.02986145019531,
      "activations/layer3_attention_weight_min": -92.80548095703125,
      "activations/layer4_attention_weight_max": 98.5885238647461,
      "activations/layer4_attention_weight_min": -94.83082580566406,
      "activations/layer5_attention_weight_max": 94.43901824951172,
      "activations/layer5_attention_weight_min": -83.95172119140625,
      "activations/layer6_attention_weight_max": 58.448036193847656,
      "activations/layer6_attention_weight_min": -53.03895568847656,
      "activations/layer7_attention_weight_max": 74.58457946777344,
      "activations/layer7_attention_weight_min": -75.49961853027344,
      "activations/layer8_attention_weight_max": 50.81331253051758,
      "activations/layer8_attention_weight_min": -46.764488220214844,
      "activations/layer9_attention_weight_max": 52.663299560546875,
      "activations/layer9_attention_weight_min": -51.4688720703125,
      "epoch": 6.9,
      "learning_rate": 0.00010657613636363636,
      "loss": 2.8077,
      "step": 118750
    },
    {
      "activations/layer0_attention_weight_max": 14.654045104980469,
      "activations/layer0_attention_weight_min": -15.376862525939941,
      "activations/layer10_attention_weight_max": 37.21464157104492,
      "activations/layer10_attention_weight_min": -33.750797271728516,
      "activations/layer11_attention_weight_max": 29.709211349487305,
      "activations/layer11_attention_weight_min": -29.990753173828125,
      "activations/layer12_attention_weight_max": 21.702346801757812,
      "activations/layer12_attention_weight_min": -26.29214859008789,
      "activations/layer13_attention_weight_max": 52.03772735595703,
      "activations/layer13_attention_weight_min": -34.645267486572266,
      "activations/layer14_attention_weight_max": 48.55813980102539,
      "activations/layer14_attention_weight_min": -30.24970817565918,
      "activations/layer15_attention_weight_max": 39.29924774169922,
      "activations/layer15_attention_weight_min": -29.435218811035156,
      "activations/layer16_attention_weight_max": 33.86832046508789,
      "activations/layer16_attention_weight_min": -27.254138946533203,
      "activations/layer17_attention_weight_max": 61.12424087524414,
      "activations/layer17_attention_weight_min": -42.77785110473633,
      "activations/layer18_attention_weight_max": 55.60725021362305,
      "activations/layer18_attention_weight_min": -36.391414642333984,
      "activations/layer19_attention_weight_max": 23.701265335083008,
      "activations/layer19_attention_weight_min": -23.36666488647461,
      "activations/layer1_attention_weight_max": 18.256826400756836,
      "activations/layer1_attention_weight_min": -18.161224365234375,
      "activations/layer20_attention_weight_max": 26.515207290649414,
      "activations/layer20_attention_weight_min": -26.80324935913086,
      "activations/layer21_attention_weight_max": 49.431884765625,
      "activations/layer21_attention_weight_min": -24.711498260498047,
      "activations/layer22_attention_weight_max": 39.875389099121094,
      "activations/layer22_attention_weight_min": -26.565156936645508,
      "activations/layer23_attention_weight_max": 39.01893997192383,
      "activations/layer23_attention_weight_min": -21.554996490478516,
      "activations/layer2_attention_weight_max": 31.7504940032959,
      "activations/layer2_attention_weight_min": -30.09925079345703,
      "activations/layer3_attention_weight_max": 85.50196075439453,
      "activations/layer3_attention_weight_min": -86.4410629272461,
      "activations/layer4_attention_weight_max": 97.30732727050781,
      "activations/layer4_attention_weight_min": -90.6982192993164,
      "activations/layer5_attention_weight_max": 85.44842529296875,
      "activations/layer5_attention_weight_min": -76.73263549804688,
      "activations/layer6_attention_weight_max": 53.53141784667969,
      "activations/layer6_attention_weight_min": -49.758113861083984,
      "activations/layer7_attention_weight_max": 68.68455505371094,
      "activations/layer7_attention_weight_min": -65.9772720336914,
      "activations/layer8_attention_weight_max": 44.2466926574707,
      "activations/layer8_attention_weight_min": -42.878448486328125,
      "activations/layer9_attention_weight_max": 51.59758377075195,
      "activations/layer9_attention_weight_min": -47.18546676635742,
      "epoch": 6.9,
      "learning_rate": 0.00010655719696969696,
      "loss": 2.8284,
      "step": 118800
    },
    {
      "activations/layer0_attention_weight_max": 15.29213809967041,
      "activations/layer0_attention_weight_min": -14.711302757263184,
      "activations/layer10_attention_weight_max": 38.628936767578125,
      "activations/layer10_attention_weight_min": -34.55512237548828,
      "activations/layer11_attention_weight_max": 29.832855224609375,
      "activations/layer11_attention_weight_min": -30.555194854736328,
      "activations/layer12_attention_weight_max": 34.682861328125,
      "activations/layer12_attention_weight_min": -25.03995132446289,
      "activations/layer13_attention_weight_max": 46.31632995605469,
      "activations/layer13_attention_weight_min": -34.45069885253906,
      "activations/layer14_attention_weight_max": 43.35780334472656,
      "activations/layer14_attention_weight_min": -32.23478317260742,
      "activations/layer15_attention_weight_max": 42.850440979003906,
      "activations/layer15_attention_weight_min": -28.841066360473633,
      "activations/layer16_attention_weight_max": 32.51340103149414,
      "activations/layer16_attention_weight_min": -26.550188064575195,
      "activations/layer17_attention_weight_max": 53.886390686035156,
      "activations/layer17_attention_weight_min": -42.274845123291016,
      "activations/layer18_attention_weight_max": 49.83205795288086,
      "activations/layer18_attention_weight_min": -34.947975158691406,
      "activations/layer19_attention_weight_max": 22.421079635620117,
      "activations/layer19_attention_weight_min": -21.020357131958008,
      "activations/layer1_attention_weight_max": 16.778966903686523,
      "activations/layer1_attention_weight_min": -16.111499786376953,
      "activations/layer20_attention_weight_max": 22.753503799438477,
      "activations/layer20_attention_weight_min": -22.064416885375977,
      "activations/layer21_attention_weight_max": 41.999542236328125,
      "activations/layer21_attention_weight_min": -26.71877670288086,
      "activations/layer22_attention_weight_max": 32.44687271118164,
      "activations/layer22_attention_weight_min": -25.71272087097168,
      "activations/layer23_attention_weight_max": 33.33403015136719,
      "activations/layer23_attention_weight_min": -20.352909088134766,
      "activations/layer2_attention_weight_max": 33.280906677246094,
      "activations/layer2_attention_weight_min": -31.202573776245117,
      "activations/layer3_attention_weight_max": 83.10536193847656,
      "activations/layer3_attention_weight_min": -87.17752075195312,
      "activations/layer4_attention_weight_max": 95.64049530029297,
      "activations/layer4_attention_weight_min": -94.06163024902344,
      "activations/layer5_attention_weight_max": 86.37822723388672,
      "activations/layer5_attention_weight_min": -79.79488372802734,
      "activations/layer6_attention_weight_max": 53.89482879638672,
      "activations/layer6_attention_weight_min": -52.08405303955078,
      "activations/layer7_attention_weight_max": 69.83612060546875,
      "activations/layer7_attention_weight_min": -66.24917602539062,
      "activations/layer8_attention_weight_max": 44.72561264038086,
      "activations/layer8_attention_weight_min": -44.84343338012695,
      "activations/layer9_attention_weight_max": 48.673439025878906,
      "activations/layer9_attention_weight_min": -46.9495964050293,
      "epoch": 6.91,
      "learning_rate": 0.00010653825757575757,
      "loss": 2.8314,
      "step": 118850
    },
    {
      "activations/layer0_attention_weight_max": 14.424871444702148,
      "activations/layer0_attention_weight_min": -14.955074310302734,
      "activations/layer10_attention_weight_max": 36.86191940307617,
      "activations/layer10_attention_weight_min": -32.71687316894531,
      "activations/layer11_attention_weight_max": 30.172767639160156,
      "activations/layer11_attention_weight_min": -28.916906356811523,
      "activations/layer12_attention_weight_max": 18.61280059814453,
      "activations/layer12_attention_weight_min": -23.66584587097168,
      "activations/layer13_attention_weight_max": 42.12495422363281,
      "activations/layer13_attention_weight_min": -34.471275329589844,
      "activations/layer14_attention_weight_max": 42.57697296142578,
      "activations/layer14_attention_weight_min": -30.08405113220215,
      "activations/layer15_attention_weight_max": 32.28969192504883,
      "activations/layer15_attention_weight_min": -28.884347915649414,
      "activations/layer16_attention_weight_max": 29.230911254882812,
      "activations/layer16_attention_weight_min": -27.355119705200195,
      "activations/layer17_attention_weight_max": 48.33757400512695,
      "activations/layer17_attention_weight_min": -40.73451614379883,
      "activations/layer18_attention_weight_max": 41.89218521118164,
      "activations/layer18_attention_weight_min": -35.32778549194336,
      "activations/layer19_attention_weight_max": 19.378442764282227,
      "activations/layer19_attention_weight_min": -19.11897087097168,
      "activations/layer1_attention_weight_max": 17.30746841430664,
      "activations/layer1_attention_weight_min": -16.62384796142578,
      "activations/layer20_attention_weight_max": 21.417564392089844,
      "activations/layer20_attention_weight_min": -21.077871322631836,
      "activations/layer21_attention_weight_max": 37.14853286743164,
      "activations/layer21_attention_weight_min": -24.7669734954834,
      "activations/layer22_attention_weight_max": 29.40347671508789,
      "activations/layer22_attention_weight_min": -24.9219913482666,
      "activations/layer23_attention_weight_max": 30.070838928222656,
      "activations/layer23_attention_weight_min": -20.782655715942383,
      "activations/layer2_attention_weight_max": 31.15852928161621,
      "activations/layer2_attention_weight_min": -30.00502586364746,
      "activations/layer3_attention_weight_max": 83.78007507324219,
      "activations/layer3_attention_weight_min": -86.43196105957031,
      "activations/layer4_attention_weight_max": 95.5203857421875,
      "activations/layer4_attention_weight_min": -91.95189666748047,
      "activations/layer5_attention_weight_max": 86.28091430664062,
      "activations/layer5_attention_weight_min": -79.69384765625,
      "activations/layer6_attention_weight_max": 53.92012405395508,
      "activations/layer6_attention_weight_min": -50.98265838623047,
      "activations/layer7_attention_weight_max": 75.12339782714844,
      "activations/layer7_attention_weight_min": -65.4308090209961,
      "activations/layer8_attention_weight_max": 40.97275161743164,
      "activations/layer8_attention_weight_min": -43.16061782836914,
      "activations/layer9_attention_weight_max": 56.22147750854492,
      "activations/layer9_attention_weight_min": -45.10081100463867,
      "epoch": 6.91,
      "learning_rate": 0.00010651931818181817,
      "loss": 2.8452,
      "step": 118900
    },
    {
      "activations/layer0_attention_weight_max": 14.243425369262695,
      "activations/layer0_attention_weight_min": -14.88893985748291,
      "activations/layer10_attention_weight_max": 40.84690475463867,
      "activations/layer10_attention_weight_min": -38.49702835083008,
      "activations/layer11_attention_weight_max": 31.63868522644043,
      "activations/layer11_attention_weight_min": -31.027158737182617,
      "activations/layer12_attention_weight_max": 19.65875816345215,
      "activations/layer12_attention_weight_min": -24.583627700805664,
      "activations/layer13_attention_weight_max": 39.547218322753906,
      "activations/layer13_attention_weight_min": -29.412574768066406,
      "activations/layer14_attention_weight_max": 37.69351577758789,
      "activations/layer14_attention_weight_min": -31.05454444885254,
      "activations/layer15_attention_weight_max": 36.221866607666016,
      "activations/layer15_attention_weight_min": -30.146568298339844,
      "activations/layer16_attention_weight_max": 33.69865798950195,
      "activations/layer16_attention_weight_min": -28.73047637939453,
      "activations/layer17_attention_weight_max": 54.017181396484375,
      "activations/layer17_attention_weight_min": -42.158302307128906,
      "activations/layer18_attention_weight_max": 47.60383224487305,
      "activations/layer18_attention_weight_min": -37.14554214477539,
      "activations/layer19_attention_weight_max": 22.886089324951172,
      "activations/layer19_attention_weight_min": -20.082765579223633,
      "activations/layer1_attention_weight_max": 16.66692543029785,
      "activations/layer1_attention_weight_min": -14.634885787963867,
      "activations/layer20_attention_weight_max": 22.514060974121094,
      "activations/layer20_attention_weight_min": -20.16488265991211,
      "activations/layer21_attention_weight_max": 38.160430908203125,
      "activations/layer21_attention_weight_min": -26.46111297607422,
      "activations/layer22_attention_weight_max": 32.71240997314453,
      "activations/layer22_attention_weight_min": -26.025415420532227,
      "activations/layer23_attention_weight_max": 34.72815704345703,
      "activations/layer23_attention_weight_min": -23.872434616088867,
      "activations/layer2_attention_weight_max": 31.936138153076172,
      "activations/layer2_attention_weight_min": -30.257400512695312,
      "activations/layer3_attention_weight_max": 85.80261993408203,
      "activations/layer3_attention_weight_min": -93.0224838256836,
      "activations/layer4_attention_weight_max": 102.5299301147461,
      "activations/layer4_attention_weight_min": -98.71112060546875,
      "activations/layer5_attention_weight_max": 91.26632690429688,
      "activations/layer5_attention_weight_min": -89.68429565429688,
      "activations/layer6_attention_weight_max": 57.535858154296875,
      "activations/layer6_attention_weight_min": -54.86653518676758,
      "activations/layer7_attention_weight_max": 72.5352554321289,
      "activations/layer7_attention_weight_min": -68.72634887695312,
      "activations/layer8_attention_weight_max": 45.220523834228516,
      "activations/layer8_attention_weight_min": -44.77831268310547,
      "activations/layer9_attention_weight_max": 52.27956771850586,
      "activations/layer9_attention_weight_min": -48.50385665893555,
      "epoch": 6.91,
      "learning_rate": 0.00010650037878787878,
      "loss": 2.8177,
      "step": 118950
    },
    {
      "activations/layer0_attention_weight_max": 15.302851676940918,
      "activations/layer0_attention_weight_min": -14.594059944152832,
      "activations/layer10_attention_weight_max": 41.21969223022461,
      "activations/layer10_attention_weight_min": -37.56050109863281,
      "activations/layer11_attention_weight_max": 32.826133728027344,
      "activations/layer11_attention_weight_min": -32.169532775878906,
      "activations/layer12_attention_weight_max": 21.88031768798828,
      "activations/layer12_attention_weight_min": -25.505847930908203,
      "activations/layer13_attention_weight_max": 54.446311950683594,
      "activations/layer13_attention_weight_min": -37.52198028564453,
      "activations/layer14_attention_weight_max": 52.854366302490234,
      "activations/layer14_attention_weight_min": -32.866065979003906,
      "activations/layer15_attention_weight_max": 39.475341796875,
      "activations/layer15_attention_weight_min": -30.185152053833008,
      "activations/layer16_attention_weight_max": 35.42975997924805,
      "activations/layer16_attention_weight_min": -27.414016723632812,
      "activations/layer17_attention_weight_max": 60.87813186645508,
      "activations/layer17_attention_weight_min": -44.6259651184082,
      "activations/layer18_attention_weight_max": 52.0114860534668,
      "activations/layer18_attention_weight_min": -37.55891036987305,
      "activations/layer19_attention_weight_max": 22.533924102783203,
      "activations/layer19_attention_weight_min": -21.32074546813965,
      "activations/layer1_attention_weight_max": 17.188770294189453,
      "activations/layer1_attention_weight_min": -17.507156372070312,
      "activations/layer20_attention_weight_max": 23.161794662475586,
      "activations/layer20_attention_weight_min": -21.061084747314453,
      "activations/layer21_attention_weight_max": 48.49957275390625,
      "activations/layer21_attention_weight_min": -26.35373306274414,
      "activations/layer22_attention_weight_max": 34.70851135253906,
      "activations/layer22_attention_weight_min": -24.27573585510254,
      "activations/layer23_attention_weight_max": 37.640541076660156,
      "activations/layer23_attention_weight_min": -22.400455474853516,
      "activations/layer2_attention_weight_max": 31.720760345458984,
      "activations/layer2_attention_weight_min": -30.06904411315918,
      "activations/layer3_attention_weight_max": 83.36224365234375,
      "activations/layer3_attention_weight_min": -89.00171661376953,
      "activations/layer4_attention_weight_max": 98.9183578491211,
      "activations/layer4_attention_weight_min": -94.68328857421875,
      "activations/layer5_attention_weight_max": 93.0555648803711,
      "activations/layer5_attention_weight_min": -86.1986312866211,
      "activations/layer6_attention_weight_max": 59.2548713684082,
      "activations/layer6_attention_weight_min": -53.89087677001953,
      "activations/layer7_attention_weight_max": 78.54014587402344,
      "activations/layer7_attention_weight_min": -75.81367492675781,
      "activations/layer8_attention_weight_max": 48.2428092956543,
      "activations/layer8_attention_weight_min": -50.74351119995117,
      "activations/layer9_attention_weight_max": 51.131858825683594,
      "activations/layer9_attention_weight_min": -51.290069580078125,
      "epoch": 6.91,
      "learning_rate": 0.00010648143939393938,
      "loss": 2.8406,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_loss": 2.7734375,
      "eval_runtime": 8.5361,
      "eval_samples_per_second": 503.039,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_openwebtext_loss": 2.7734375,
      "eval_openwebtext_ppl": 16.01358620918395,
      "eval_openwebtext_runtime": 8.5361,
      "eval_openwebtext_samples_per_second": 503.039,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_wikitext_loss": 3.00390625,
      "eval_wikitext_ppl": 20.16414949200632,
      "eval_wikitext_runtime": 1.9868,
      "eval_wikitext_samples_per_second": 229.519,
      "step": 119000
    },
    {
      "epoch": 6.91,
      "eval_lambada_loss": 2.666015625,
      "eval_lambada_ppl": 14.382549407491597,
      "eval_lambada_runtime": 9.5806,
      "eval_lambada_samples_per_second": 508.214,
      "step": 119000
    },
    {
      "activations/layer0_attention_weight_max": 15.477699279785156,
      "activations/layer0_attention_weight_min": -14.954901695251465,
      "activations/layer10_attention_weight_max": 44.38005447387695,
      "activations/layer10_attention_weight_min": -39.79884719848633,
      "activations/layer11_attention_weight_max": 33.982181549072266,
      "activations/layer11_attention_weight_min": -32.348690032958984,
      "activations/layer12_attention_weight_max": 21.67953872680664,
      "activations/layer12_attention_weight_min": -24.967960357666016,
      "activations/layer13_attention_weight_max": 58.338077545166016,
      "activations/layer13_attention_weight_min": -40.975830078125,
      "activations/layer14_attention_weight_max": 50.492652893066406,
      "activations/layer14_attention_weight_min": -32.70808792114258,
      "activations/layer15_attention_weight_max": 42.35171890258789,
      "activations/layer15_attention_weight_min": -31.567598342895508,
      "activations/layer16_attention_weight_max": 34.5938720703125,
      "activations/layer16_attention_weight_min": -27.081159591674805,
      "activations/layer17_attention_weight_max": 54.35990524291992,
      "activations/layer17_attention_weight_min": -40.998836517333984,
      "activations/layer18_attention_weight_max": 52.32099533081055,
      "activations/layer18_attention_weight_min": -35.40824890136719,
      "activations/layer19_attention_weight_max": 30.379684448242188,
      "activations/layer19_attention_weight_min": -20.400388717651367,
      "activations/layer1_attention_weight_max": 17.34379768371582,
      "activations/layer1_attention_weight_min": -17.538787841796875,
      "activations/layer20_attention_weight_max": 26.331480026245117,
      "activations/layer20_attention_weight_min": -21.068836212158203,
      "activations/layer21_attention_weight_max": 45.3434944152832,
      "activations/layer21_attention_weight_min": -24.475431442260742,
      "activations/layer22_attention_weight_max": 35.28239440917969,
      "activations/layer22_attention_weight_min": -26.731924057006836,
      "activations/layer23_attention_weight_max": 36.41207504272461,
      "activations/layer23_attention_weight_min": -19.38046646118164,
      "activations/layer2_attention_weight_max": 33.56718444824219,
      "activations/layer2_attention_weight_min": -28.790237426757812,
      "activations/layer3_attention_weight_max": 85.29112243652344,
      "activations/layer3_attention_weight_min": -86.8275146484375,
      "activations/layer4_attention_weight_max": 100.91244506835938,
      "activations/layer4_attention_weight_min": -94.94729614257812,
      "activations/layer5_attention_weight_max": 93.56160736083984,
      "activations/layer5_attention_weight_min": -88.88631439208984,
      "activations/layer6_attention_weight_max": 57.28514862060547,
      "activations/layer6_attention_weight_min": -53.018123626708984,
      "activations/layer7_attention_weight_max": 77.89859008789062,
      "activations/layer7_attention_weight_min": -76.7861328125,
      "activations/layer8_attention_weight_max": 50.430519104003906,
      "activations/layer8_attention_weight_min": -48.24319076538086,
      "activations/layer9_attention_weight_max": 62.31492233276367,
      "activations/layer9_attention_weight_min": -50.329586029052734,
      "epoch": 6.92,
      "learning_rate": 0.00010646249999999999,
      "loss": 2.8135,
      "step": 119050
    },
    {
      "activations/layer0_attention_weight_max": 14.586870193481445,
      "activations/layer0_attention_weight_min": -14.59485912322998,
      "activations/layer10_attention_weight_max": 40.73286437988281,
      "activations/layer10_attention_weight_min": -38.097171783447266,
      "activations/layer11_attention_weight_max": 32.996341705322266,
      "activations/layer11_attention_weight_min": -29.043638229370117,
      "activations/layer12_attention_weight_max": 19.885868072509766,
      "activations/layer12_attention_weight_min": -24.740360260009766,
      "activations/layer13_attention_weight_max": 43.002464294433594,
      "activations/layer13_attention_weight_min": -31.722633361816406,
      "activations/layer14_attention_weight_max": 53.51216506958008,
      "activations/layer14_attention_weight_min": -35.1356201171875,
      "activations/layer15_attention_weight_max": 38.8547248840332,
      "activations/layer15_attention_weight_min": -27.491634368896484,
      "activations/layer16_attention_weight_max": 31.901166915893555,
      "activations/layer16_attention_weight_min": -25.005615234375,
      "activations/layer17_attention_weight_max": 51.068756103515625,
      "activations/layer17_attention_weight_min": -40.35572052001953,
      "activations/layer18_attention_weight_max": 47.47981262207031,
      "activations/layer18_attention_weight_min": -34.95684814453125,
      "activations/layer19_attention_weight_max": 22.215065002441406,
      "activations/layer19_attention_weight_min": -21.02067756652832,
      "activations/layer1_attention_weight_max": 17.948394775390625,
      "activations/layer1_attention_weight_min": -17.882619857788086,
      "activations/layer20_attention_weight_max": 21.835796356201172,
      "activations/layer20_attention_weight_min": -22.3558292388916,
      "activations/layer21_attention_weight_max": 35.729827880859375,
      "activations/layer21_attention_weight_min": -24.4494686126709,
      "activations/layer22_attention_weight_max": 33.81437683105469,
      "activations/layer22_attention_weight_min": -25.316009521484375,
      "activations/layer23_attention_weight_max": 31.484691619873047,
      "activations/layer23_attention_weight_min": -18.973388671875,
      "activations/layer2_attention_weight_max": 32.37309265136719,
      "activations/layer2_attention_weight_min": -30.832279205322266,
      "activations/layer3_attention_weight_max": 89.81279754638672,
      "activations/layer3_attention_weight_min": -90.04691314697266,
      "activations/layer4_attention_weight_max": 101.4375991821289,
      "activations/layer4_attention_weight_min": -99.61128997802734,
      "activations/layer5_attention_weight_max": 92.69414520263672,
      "activations/layer5_attention_weight_min": -87.05236053466797,
      "activations/layer6_attention_weight_max": 54.54032516479492,
      "activations/layer6_attention_weight_min": -52.038780212402344,
      "activations/layer7_attention_weight_max": 77.83522033691406,
      "activations/layer7_attention_weight_min": -68.57249450683594,
      "activations/layer8_attention_weight_max": 49.39253234863281,
      "activations/layer8_attention_weight_min": -42.90522766113281,
      "activations/layer9_attention_weight_max": 59.51947021484375,
      "activations/layer9_attention_weight_min": -47.71525955200195,
      "epoch": 6.92,
      "learning_rate": 0.00010644356060606061,
      "loss": 2.8257,
      "step": 119100
    },
    {
      "activations/layer0_attention_weight_max": 15.188121795654297,
      "activations/layer0_attention_weight_min": -14.877720832824707,
      "activations/layer10_attention_weight_max": 41.26194763183594,
      "activations/layer10_attention_weight_min": -39.11206817626953,
      "activations/layer11_attention_weight_max": 35.509029388427734,
      "activations/layer11_attention_weight_min": -31.55937957763672,
      "activations/layer12_attention_weight_max": 21.16132164001465,
      "activations/layer12_attention_weight_min": -24.044265747070312,
      "activations/layer13_attention_weight_max": 44.0652961730957,
      "activations/layer13_attention_weight_min": -33.09130859375,
      "activations/layer14_attention_weight_max": 43.088233947753906,
      "activations/layer14_attention_weight_min": -32.89107131958008,
      "activations/layer15_attention_weight_max": 36.83051300048828,
      "activations/layer15_attention_weight_min": -28.234289169311523,
      "activations/layer16_attention_weight_max": 31.79126739501953,
      "activations/layer16_attention_weight_min": -28.738067626953125,
      "activations/layer17_attention_weight_max": 55.56510925292969,
      "activations/layer17_attention_weight_min": -43.33523941040039,
      "activations/layer18_attention_weight_max": 46.08320236206055,
      "activations/layer18_attention_weight_min": -35.7753791809082,
      "activations/layer19_attention_weight_max": 24.058704376220703,
      "activations/layer19_attention_weight_min": -19.801162719726562,
      "activations/layer1_attention_weight_max": 16.674205780029297,
      "activations/layer1_attention_weight_min": -17.00602149963379,
      "activations/layer20_attention_weight_max": 20.224502563476562,
      "activations/layer20_attention_weight_min": -22.215974807739258,
      "activations/layer21_attention_weight_max": 37.852630615234375,
      "activations/layer21_attention_weight_min": -24.27939796447754,
      "activations/layer22_attention_weight_max": 29.7515926361084,
      "activations/layer22_attention_weight_min": -26.500715255737305,
      "activations/layer23_attention_weight_max": 31.280128479003906,
      "activations/layer23_attention_weight_min": -22.938396453857422,
      "activations/layer2_attention_weight_max": 33.54973602294922,
      "activations/layer2_attention_weight_min": -31.14859390258789,
      "activations/layer3_attention_weight_max": 88.89117431640625,
      "activations/layer3_attention_weight_min": -90.93565368652344,
      "activations/layer4_attention_weight_max": 107.17960357666016,
      "activations/layer4_attention_weight_min": -98.81566619873047,
      "activations/layer5_attention_weight_max": 96.91488647460938,
      "activations/layer5_attention_weight_min": -90.73072814941406,
      "activations/layer6_attention_weight_max": 60.944358825683594,
      "activations/layer6_attention_weight_min": -55.88307189941406,
      "activations/layer7_attention_weight_max": 78.98108673095703,
      "activations/layer7_attention_weight_min": -73.95675659179688,
      "activations/layer8_attention_weight_max": 48.1895866394043,
      "activations/layer8_attention_weight_min": -52.40538787841797,
      "activations/layer9_attention_weight_max": 54.40877914428711,
      "activations/layer9_attention_weight_min": -52.76837158203125,
      "epoch": 6.92,
      "learning_rate": 0.0001064246212121212,
      "loss": 2.828,
      "step": 119150
    },
    {
      "activations/layer0_attention_weight_max": 14.73166275024414,
      "activations/layer0_attention_weight_min": -15.407411575317383,
      "activations/layer10_attention_weight_max": 39.98743438720703,
      "activations/layer10_attention_weight_min": -37.177589416503906,
      "activations/layer11_attention_weight_max": 32.87061309814453,
      "activations/layer11_attention_weight_min": -31.53949737548828,
      "activations/layer12_attention_weight_max": 23.612075805664062,
      "activations/layer12_attention_weight_min": -24.31841278076172,
      "activations/layer13_attention_weight_max": 70.83258819580078,
      "activations/layer13_attention_weight_min": -46.90046691894531,
      "activations/layer14_attention_weight_max": 50.20363998413086,
      "activations/layer14_attention_weight_min": -29.94289779663086,
      "activations/layer15_attention_weight_max": 38.85810852050781,
      "activations/layer15_attention_weight_min": -30.71246337890625,
      "activations/layer16_attention_weight_max": 35.988136291503906,
      "activations/layer16_attention_weight_min": -26.261009216308594,
      "activations/layer17_attention_weight_max": 56.89823532104492,
      "activations/layer17_attention_weight_min": -40.531227111816406,
      "activations/layer18_attention_weight_max": 47.60857391357422,
      "activations/layer18_attention_weight_min": -34.792789459228516,
      "activations/layer19_attention_weight_max": 21.342023849487305,
      "activations/layer19_attention_weight_min": -21.7784481048584,
      "activations/layer1_attention_weight_max": 16.93333625793457,
      "activations/layer1_attention_weight_min": -16.874011993408203,
      "activations/layer20_attention_weight_max": 22.84210777282715,
      "activations/layer20_attention_weight_min": -20.448625564575195,
      "activations/layer21_attention_weight_max": 43.24489212036133,
      "activations/layer21_attention_weight_min": -22.876501083374023,
      "activations/layer22_attention_weight_max": 35.51905822753906,
      "activations/layer22_attention_weight_min": -24.293046951293945,
      "activations/layer23_attention_weight_max": 37.6193733215332,
      "activations/layer23_attention_weight_min": -19.956520080566406,
      "activations/layer2_attention_weight_max": 31.51064682006836,
      "activations/layer2_attention_weight_min": -29.901243209838867,
      "activations/layer3_attention_weight_max": 84.00776672363281,
      "activations/layer3_attention_weight_min": -88.33960723876953,
      "activations/layer4_attention_weight_max": 102.2707748413086,
      "activations/layer4_attention_weight_min": -96.95647430419922,
      "activations/layer5_attention_weight_max": 94.9040756225586,
      "activations/layer5_attention_weight_min": -85.1316909790039,
      "activations/layer6_attention_weight_max": 56.61368942260742,
      "activations/layer6_attention_weight_min": -54.156673431396484,
      "activations/layer7_attention_weight_max": 73.19867706298828,
      "activations/layer7_attention_weight_min": -73.1335220336914,
      "activations/layer8_attention_weight_max": 45.536354064941406,
      "activations/layer8_attention_weight_min": -46.760345458984375,
      "activations/layer9_attention_weight_max": 52.43315505981445,
      "activations/layer9_attention_weight_min": -52.421119689941406,
      "epoch": 6.93,
      "learning_rate": 0.00010640568181818181,
      "loss": 2.8445,
      "step": 119200
    },
    {
      "activations/layer0_attention_weight_max": 15.56653118133545,
      "activations/layer0_attention_weight_min": -15.105719566345215,
      "activations/layer10_attention_weight_max": 34.38972473144531,
      "activations/layer10_attention_weight_min": -32.698448181152344,
      "activations/layer11_attention_weight_max": 30.579076766967773,
      "activations/layer11_attention_weight_min": -28.238431930541992,
      "activations/layer12_attention_weight_max": 20.284366607666016,
      "activations/layer12_attention_weight_min": -24.28561019897461,
      "activations/layer13_attention_weight_max": 36.99711990356445,
      "activations/layer13_attention_weight_min": -30.82672119140625,
      "activations/layer14_attention_weight_max": 35.30520248413086,
      "activations/layer14_attention_weight_min": -29.327238082885742,
      "activations/layer15_attention_weight_max": 33.45587921142578,
      "activations/layer15_attention_weight_min": -29.416078567504883,
      "activations/layer16_attention_weight_max": 28.919769287109375,
      "activations/layer16_attention_weight_min": -27.19719123840332,
      "activations/layer17_attention_weight_max": 49.08595657348633,
      "activations/layer17_attention_weight_min": -42.80000686645508,
      "activations/layer18_attention_weight_max": 43.10243606567383,
      "activations/layer18_attention_weight_min": -35.75284194946289,
      "activations/layer19_attention_weight_max": 19.12477684020996,
      "activations/layer19_attention_weight_min": -18.92058753967285,
      "activations/layer1_attention_weight_max": 16.876951217651367,
      "activations/layer1_attention_weight_min": -14.821732521057129,
      "activations/layer20_attention_weight_max": 18.18151092529297,
      "activations/layer20_attention_weight_min": -21.638639450073242,
      "activations/layer21_attention_weight_max": 34.619747161865234,
      "activations/layer21_attention_weight_min": -25.850658416748047,
      "activations/layer22_attention_weight_max": 29.62862205505371,
      "activations/layer22_attention_weight_min": -26.974159240722656,
      "activations/layer23_attention_weight_max": 29.185527801513672,
      "activations/layer23_attention_weight_min": -19.967370986938477,
      "activations/layer2_attention_weight_max": 31.02853775024414,
      "activations/layer2_attention_weight_min": -30.559486389160156,
      "activations/layer3_attention_weight_max": 80.29742431640625,
      "activations/layer3_attention_weight_min": -82.5260009765625,
      "activations/layer4_attention_weight_max": 96.04558563232422,
      "activations/layer4_attention_weight_min": -87.45500946044922,
      "activations/layer5_attention_weight_max": 84.09642791748047,
      "activations/layer5_attention_weight_min": -78.38778686523438,
      "activations/layer6_attention_weight_max": 53.97239303588867,
      "activations/layer6_attention_weight_min": -50.741966247558594,
      "activations/layer7_attention_weight_max": 71.83546447753906,
      "activations/layer7_attention_weight_min": -66.3866195678711,
      "activations/layer8_attention_weight_max": 40.94136428833008,
      "activations/layer8_attention_weight_min": -41.39706039428711,
      "activations/layer9_attention_weight_max": 46.585575103759766,
      "activations/layer9_attention_weight_min": -47.904109954833984,
      "epoch": 6.93,
      "learning_rate": 0.00010638674242424243,
      "loss": 2.8135,
      "step": 119250
    },
    {
      "activations/layer0_attention_weight_max": 15.422521591186523,
      "activations/layer0_attention_weight_min": -15.091928482055664,
      "activations/layer10_attention_weight_max": 37.988426208496094,
      "activations/layer10_attention_weight_min": -35.155094146728516,
      "activations/layer11_attention_weight_max": 30.853628158569336,
      "activations/layer11_attention_weight_min": -29.897884368896484,
      "activations/layer12_attention_weight_max": 22.893253326416016,
      "activations/layer12_attention_weight_min": -25.448322296142578,
      "activations/layer13_attention_weight_max": 52.63717269897461,
      "activations/layer13_attention_weight_min": -34.22207260131836,
      "activations/layer14_attention_weight_max": 59.120853424072266,
      "activations/layer14_attention_weight_min": -32.96587371826172,
      "activations/layer15_attention_weight_max": 45.341087341308594,
      "activations/layer15_attention_weight_min": -30.02846336364746,
      "activations/layer16_attention_weight_max": 31.811840057373047,
      "activations/layer16_attention_weight_min": -26.78145980834961,
      "activations/layer17_attention_weight_max": 56.57045364379883,
      "activations/layer17_attention_weight_min": -40.328704833984375,
      "activations/layer18_attention_weight_max": 46.151248931884766,
      "activations/layer18_attention_weight_min": -35.64563751220703,
      "activations/layer19_attention_weight_max": 27.1698055267334,
      "activations/layer19_attention_weight_min": -19.316261291503906,
      "activations/layer1_attention_weight_max": 18.69182777404785,
      "activations/layer1_attention_weight_min": -18.48287010192871,
      "activations/layer20_attention_weight_max": 24.617143630981445,
      "activations/layer20_attention_weight_min": -24.5264835357666,
      "activations/layer21_attention_weight_max": 41.914520263671875,
      "activations/layer21_attention_weight_min": -22.13612937927246,
      "activations/layer22_attention_weight_max": 33.50616455078125,
      "activations/layer22_attention_weight_min": -23.840251922607422,
      "activations/layer23_attention_weight_max": 32.96595764160156,
      "activations/layer23_attention_weight_min": -21.056150436401367,
      "activations/layer2_attention_weight_max": 31.934471130371094,
      "activations/layer2_attention_weight_min": -31.93989372253418,
      "activations/layer3_attention_weight_max": 89.86001586914062,
      "activations/layer3_attention_weight_min": -98.62400817871094,
      "activations/layer4_attention_weight_max": 97.2137451171875,
      "activations/layer4_attention_weight_min": -95.42355346679688,
      "activations/layer5_attention_weight_max": 89.15391540527344,
      "activations/layer5_attention_weight_min": -85.17689514160156,
      "activations/layer6_attention_weight_max": 59.13212966918945,
      "activations/layer6_attention_weight_min": -57.46477127075195,
      "activations/layer7_attention_weight_max": 72.10723876953125,
      "activations/layer7_attention_weight_min": -77.34387969970703,
      "activations/layer8_attention_weight_max": 42.90461349487305,
      "activations/layer8_attention_weight_min": -43.7421760559082,
      "activations/layer9_attention_weight_max": 50.200801849365234,
      "activations/layer9_attention_weight_min": -48.78598403930664,
      "epoch": 6.93,
      "learning_rate": 0.00010636780303030301,
      "loss": 2.8347,
      "step": 119300
    },
    {
      "activations/layer0_attention_weight_max": 15.02265739440918,
      "activations/layer0_attention_weight_min": -14.932348251342773,
      "activations/layer10_attention_weight_max": 41.7337646484375,
      "activations/layer10_attention_weight_min": -37.7151985168457,
      "activations/layer11_attention_weight_max": 32.11835479736328,
      "activations/layer11_attention_weight_min": -30.850460052490234,
      "activations/layer12_attention_weight_max": 23.26922035217285,
      "activations/layer12_attention_weight_min": -35.25979995727539,
      "activations/layer13_attention_weight_max": 44.13905715942383,
      "activations/layer13_attention_weight_min": -33.182804107666016,
      "activations/layer14_attention_weight_max": 36.53825759887695,
      "activations/layer14_attention_weight_min": -28.990270614624023,
      "activations/layer15_attention_weight_max": 36.4799919128418,
      "activations/layer15_attention_weight_min": -29.664384841918945,
      "activations/layer16_attention_weight_max": 30.914155960083008,
      "activations/layer16_attention_weight_min": -25.221620559692383,
      "activations/layer17_attention_weight_max": 49.819644927978516,
      "activations/layer17_attention_weight_min": -39.8688850402832,
      "activations/layer18_attention_weight_max": 43.25418472290039,
      "activations/layer18_attention_weight_min": -31.730581283569336,
      "activations/layer19_attention_weight_max": 20.85110092163086,
      "activations/layer19_attention_weight_min": -19.09334373474121,
      "activations/layer1_attention_weight_max": 17.45209503173828,
      "activations/layer1_attention_weight_min": -16.061931610107422,
      "activations/layer20_attention_weight_max": 21.018041610717773,
      "activations/layer20_attention_weight_min": -21.79001235961914,
      "activations/layer21_attention_weight_max": 39.2339973449707,
      "activations/layer21_attention_weight_min": -24.827835083007812,
      "activations/layer22_attention_weight_max": 33.91974639892578,
      "activations/layer22_attention_weight_min": -22.787811279296875,
      "activations/layer23_attention_weight_max": 31.410518646240234,
      "activations/layer23_attention_weight_min": -19.290573120117188,
      "activations/layer2_attention_weight_max": 34.336368560791016,
      "activations/layer2_attention_weight_min": -30.37238121032715,
      "activations/layer3_attention_weight_max": 87.41503143310547,
      "activations/layer3_attention_weight_min": -89.12722778320312,
      "activations/layer4_attention_weight_max": 105.97846984863281,
      "activations/layer4_attention_weight_min": -98.67112731933594,
      "activations/layer5_attention_weight_max": 95.02003479003906,
      "activations/layer5_attention_weight_min": -85.58322143554688,
      "activations/layer6_attention_weight_max": 55.274009704589844,
      "activations/layer6_attention_weight_min": -52.75295639038086,
      "activations/layer7_attention_weight_max": 80.98131561279297,
      "activations/layer7_attention_weight_min": -73.7515869140625,
      "activations/layer8_attention_weight_max": 47.571144104003906,
      "activations/layer8_attention_weight_min": -46.93369674682617,
      "activations/layer9_attention_weight_max": 65.12406158447266,
      "activations/layer9_attention_weight_min": -53.96723556518555,
      "epoch": 6.93,
      "learning_rate": 0.00010634886363636363,
      "loss": 2.8234,
      "step": 119350
    },
    {
      "activations/layer0_attention_weight_max": 15.571593284606934,
      "activations/layer0_attention_weight_min": -15.040562629699707,
      "activations/layer10_attention_weight_max": 37.802772521972656,
      "activations/layer10_attention_weight_min": -35.31977081298828,
      "activations/layer11_attention_weight_max": 29.755857467651367,
      "activations/layer11_attention_weight_min": -29.277873992919922,
      "activations/layer12_attention_weight_max": 21.8536319732666,
      "activations/layer12_attention_weight_min": -25.205488204956055,
      "activations/layer13_attention_weight_max": 47.40938186645508,
      "activations/layer13_attention_weight_min": -34.4123420715332,
      "activations/layer14_attention_weight_max": 44.18623733520508,
      "activations/layer14_attention_weight_min": -31.42865753173828,
      "activations/layer15_attention_weight_max": 34.35132598876953,
      "activations/layer15_attention_weight_min": -29.458742141723633,
      "activations/layer16_attention_weight_max": 28.559253692626953,
      "activations/layer16_attention_weight_min": -26.564058303833008,
      "activations/layer17_attention_weight_max": 55.56193923950195,
      "activations/layer17_attention_weight_min": -41.88359069824219,
      "activations/layer18_attention_weight_max": 43.30386734008789,
      "activations/layer18_attention_weight_min": -33.23412322998047,
      "activations/layer19_attention_weight_max": 19.70525550842285,
      "activations/layer19_attention_weight_min": -18.41089630126953,
      "activations/layer1_attention_weight_max": 17.734304428100586,
      "activations/layer1_attention_weight_min": -16.089826583862305,
      "activations/layer20_attention_weight_max": 21.01134490966797,
      "activations/layer20_attention_weight_min": -22.29607391357422,
      "activations/layer21_attention_weight_max": 34.658180236816406,
      "activations/layer21_attention_weight_min": -24.98879051208496,
      "activations/layer22_attention_weight_max": 29.35445213317871,
      "activations/layer22_attention_weight_min": -25.291555404663086,
      "activations/layer23_attention_weight_max": 29.285879135131836,
      "activations/layer23_attention_weight_min": -21.341842651367188,
      "activations/layer2_attention_weight_max": 32.95539093017578,
      "activations/layer2_attention_weight_min": -30.984790802001953,
      "activations/layer3_attention_weight_max": 86.44762420654297,
      "activations/layer3_attention_weight_min": -86.47013092041016,
      "activations/layer4_attention_weight_max": 98.36638641357422,
      "activations/layer4_attention_weight_min": -93.3238296508789,
      "activations/layer5_attention_weight_max": 91.18010711669922,
      "activations/layer5_attention_weight_min": -80.30917358398438,
      "activations/layer6_attention_weight_max": 59.110328674316406,
      "activations/layer6_attention_weight_min": -51.58059310913086,
      "activations/layer7_attention_weight_max": 75.78874969482422,
      "activations/layer7_attention_weight_min": -67.95663452148438,
      "activations/layer8_attention_weight_max": 47.01137161254883,
      "activations/layer8_attention_weight_min": -40.8067626953125,
      "activations/layer9_attention_weight_max": 52.197792053222656,
      "activations/layer9_attention_weight_min": -48.29197311401367,
      "epoch": 6.94,
      "learning_rate": 0.00010632992424242423,
      "loss": 2.8082,
      "step": 119400
    },
    {
      "activations/layer0_attention_weight_max": 14.456950187683105,
      "activations/layer0_attention_weight_min": -15.369041442871094,
      "activations/layer10_attention_weight_max": 39.57073974609375,
      "activations/layer10_attention_weight_min": -37.57857894897461,
      "activations/layer11_attention_weight_max": 31.522300720214844,
      "activations/layer11_attention_weight_min": -29.952842712402344,
      "activations/layer12_attention_weight_max": 19.59755516052246,
      "activations/layer12_attention_weight_min": -29.904312133789062,
      "activations/layer13_attention_weight_max": 49.83869552612305,
      "activations/layer13_attention_weight_min": -34.480499267578125,
      "activations/layer14_attention_weight_max": 42.079410552978516,
      "activations/layer14_attention_weight_min": -29.80677032470703,
      "activations/layer15_attention_weight_max": 35.83908462524414,
      "activations/layer15_attention_weight_min": -29.219940185546875,
      "activations/layer16_attention_weight_max": 30.42029571533203,
      "activations/layer16_attention_weight_min": -26.27901268005371,
      "activations/layer17_attention_weight_max": 50.60400390625,
      "activations/layer17_attention_weight_min": -40.14082717895508,
      "activations/layer18_attention_weight_max": 43.50153732299805,
      "activations/layer18_attention_weight_min": -34.285614013671875,
      "activations/layer19_attention_weight_max": 22.54668617248535,
      "activations/layer19_attention_weight_min": -19.677452087402344,
      "activations/layer1_attention_weight_max": 17.34248924255371,
      "activations/layer1_attention_weight_min": -18.64394187927246,
      "activations/layer20_attention_weight_max": 20.792003631591797,
      "activations/layer20_attention_weight_min": -19.33531951904297,
      "activations/layer21_attention_weight_max": 35.90211868286133,
      "activations/layer21_attention_weight_min": -23.60484504699707,
      "activations/layer22_attention_weight_max": 31.43268585205078,
      "activations/layer22_attention_weight_min": -25.27028465270996,
      "activations/layer23_attention_weight_max": 29.45755386352539,
      "activations/layer23_attention_weight_min": -19.924114227294922,
      "activations/layer2_attention_weight_max": 32.2261962890625,
      "activations/layer2_attention_weight_min": -28.253372192382812,
      "activations/layer3_attention_weight_max": 83.82420349121094,
      "activations/layer3_attention_weight_min": -88.78482055664062,
      "activations/layer4_attention_weight_max": 95.30269622802734,
      "activations/layer4_attention_weight_min": -94.11151885986328,
      "activations/layer5_attention_weight_max": 88.50737762451172,
      "activations/layer5_attention_weight_min": -80.45655822753906,
      "activations/layer6_attention_weight_max": 55.81154251098633,
      "activations/layer6_attention_weight_min": -52.16202163696289,
      "activations/layer7_attention_weight_max": 75.11105346679688,
      "activations/layer7_attention_weight_min": -70.68861389160156,
      "activations/layer8_attention_weight_max": 46.66651916503906,
      "activations/layer8_attention_weight_min": -45.45499038696289,
      "activations/layer9_attention_weight_max": 58.06498336791992,
      "activations/layer9_attention_weight_min": -49.364830017089844,
      "epoch": 6.94,
      "learning_rate": 0.00010631098484848483,
      "loss": 2.8353,
      "step": 119450
    },
    {
      "activations/layer0_attention_weight_max": 14.901299476623535,
      "activations/layer0_attention_weight_min": -14.732063293457031,
      "activations/layer10_attention_weight_max": 35.76874542236328,
      "activations/layer10_attention_weight_min": -33.428192138671875,
      "activations/layer11_attention_weight_max": 30.07030487060547,
      "activations/layer11_attention_weight_min": -28.326948165893555,
      "activations/layer12_attention_weight_max": 18.321508407592773,
      "activations/layer12_attention_weight_min": -26.03651237487793,
      "activations/layer13_attention_weight_max": 36.52216339111328,
      "activations/layer13_attention_weight_min": -31.439897537231445,
      "activations/layer14_attention_weight_max": 34.387351989746094,
      "activations/layer14_attention_weight_min": -30.54784393310547,
      "activations/layer15_attention_weight_max": 37.52421188354492,
      "activations/layer15_attention_weight_min": -31.39260482788086,
      "activations/layer16_attention_weight_max": 31.949426651000977,
      "activations/layer16_attention_weight_min": -26.889253616333008,
      "activations/layer17_attention_weight_max": 52.71422576904297,
      "activations/layer17_attention_weight_min": -40.48200225830078,
      "activations/layer18_attention_weight_max": 45.85921859741211,
      "activations/layer18_attention_weight_min": -34.85359191894531,
      "activations/layer19_attention_weight_max": 20.56817054748535,
      "activations/layer19_attention_weight_min": -22.22225570678711,
      "activations/layer1_attention_weight_max": 15.676090240478516,
      "activations/layer1_attention_weight_min": -15.164360046386719,
      "activations/layer20_attention_weight_max": 21.8620662689209,
      "activations/layer20_attention_weight_min": -22.096973419189453,
      "activations/layer21_attention_weight_max": 34.557708740234375,
      "activations/layer21_attention_weight_min": -22.61948013305664,
      "activations/layer22_attention_weight_max": 33.537994384765625,
      "activations/layer22_attention_weight_min": -24.747060775756836,
      "activations/layer23_attention_weight_max": 31.84285545349121,
      "activations/layer23_attention_weight_min": -19.661157608032227,
      "activations/layer2_attention_weight_max": 30.054370880126953,
      "activations/layer2_attention_weight_min": -27.802230834960938,
      "activations/layer3_attention_weight_max": 80.56587219238281,
      "activations/layer3_attention_weight_min": -83.10128784179688,
      "activations/layer4_attention_weight_max": 97.4118423461914,
      "activations/layer4_attention_weight_min": -93.68245697021484,
      "activations/layer5_attention_weight_max": 90.38690185546875,
      "activations/layer5_attention_weight_min": -84.09648132324219,
      "activations/layer6_attention_weight_max": 56.89607238769531,
      "activations/layer6_attention_weight_min": -52.158119201660156,
      "activations/layer7_attention_weight_max": 72.86944580078125,
      "activations/layer7_attention_weight_min": -70.07542419433594,
      "activations/layer8_attention_weight_max": 43.23064041137695,
      "activations/layer8_attention_weight_min": -43.708778381347656,
      "activations/layer9_attention_weight_max": 49.488922119140625,
      "activations/layer9_attention_weight_min": -45.397850036621094,
      "epoch": 6.94,
      "learning_rate": 0.00010629204545454545,
      "loss": 2.8185,
      "step": 119500
    },
    {
      "activations/layer0_attention_weight_max": 16.322542190551758,
      "activations/layer0_attention_weight_min": -15.049857139587402,
      "activations/layer10_attention_weight_max": 37.75259780883789,
      "activations/layer10_attention_weight_min": -34.888343811035156,
      "activations/layer11_attention_weight_max": 32.37849426269531,
      "activations/layer11_attention_weight_min": -29.669273376464844,
      "activations/layer12_attention_weight_max": 24.585758209228516,
      "activations/layer12_attention_weight_min": -24.17689323425293,
      "activations/layer13_attention_weight_max": 49.17292785644531,
      "activations/layer13_attention_weight_min": -36.07270431518555,
      "activations/layer14_attention_weight_max": 39.44374465942383,
      "activations/layer14_attention_weight_min": -34.03089904785156,
      "activations/layer15_attention_weight_max": 37.65645217895508,
      "activations/layer15_attention_weight_min": -31.519994735717773,
      "activations/layer16_attention_weight_max": 34.65861892700195,
      "activations/layer16_attention_weight_min": -28.838607788085938,
      "activations/layer17_attention_weight_max": 55.106956481933594,
      "activations/layer17_attention_weight_min": -41.66242980957031,
      "activations/layer18_attention_weight_max": 48.47500991821289,
      "activations/layer18_attention_weight_min": -35.89678192138672,
      "activations/layer19_attention_weight_max": 23.37029266357422,
      "activations/layer19_attention_weight_min": -18.690229415893555,
      "activations/layer1_attention_weight_max": 16.901226043701172,
      "activations/layer1_attention_weight_min": -15.37932014465332,
      "activations/layer20_attention_weight_max": 21.90312957763672,
      "activations/layer20_attention_weight_min": -20.984424591064453,
      "activations/layer21_attention_weight_max": 41.44345474243164,
      "activations/layer21_attention_weight_min": -26.262264251708984,
      "activations/layer22_attention_weight_max": 32.39767074584961,
      "activations/layer22_attention_weight_min": -23.38922882080078,
      "activations/layer23_attention_weight_max": 35.88559341430664,
      "activations/layer23_attention_weight_min": -23.03097915649414,
      "activations/layer2_attention_weight_max": 30.961429595947266,
      "activations/layer2_attention_weight_min": -28.324337005615234,
      "activations/layer3_attention_weight_max": 80.58126831054688,
      "activations/layer3_attention_weight_min": -82.67475128173828,
      "activations/layer4_attention_weight_max": 100.31038665771484,
      "activations/layer4_attention_weight_min": -90.4413833618164,
      "activations/layer5_attention_weight_max": 87.33515930175781,
      "activations/layer5_attention_weight_min": -81.38990020751953,
      "activations/layer6_attention_weight_max": 54.85317611694336,
      "activations/layer6_attention_weight_min": -51.73379135131836,
      "activations/layer7_attention_weight_max": 75.43705749511719,
      "activations/layer7_attention_weight_min": -69.17587280273438,
      "activations/layer8_attention_weight_max": 43.11901092529297,
      "activations/layer8_attention_weight_min": -43.55160140991211,
      "activations/layer9_attention_weight_max": 54.840431213378906,
      "activations/layer9_attention_weight_min": -47.79783248901367,
      "epoch": 6.95,
      "learning_rate": 0.00010627310606060605,
      "loss": 2.8187,
      "step": 119550
    },
    {
      "activations/layer0_attention_weight_max": 15.352167129516602,
      "activations/layer0_attention_weight_min": -15.13209056854248,
      "activations/layer10_attention_weight_max": 39.124019622802734,
      "activations/layer10_attention_weight_min": -36.1282958984375,
      "activations/layer11_attention_weight_max": 33.637855529785156,
      "activations/layer11_attention_weight_min": -32.3489990234375,
      "activations/layer12_attention_weight_max": 28.11551856994629,
      "activations/layer12_attention_weight_min": -25.39739990234375,
      "activations/layer13_attention_weight_max": 46.2647819519043,
      "activations/layer13_attention_weight_min": -30.935914993286133,
      "activations/layer14_attention_weight_max": 50.79842758178711,
      "activations/layer14_attention_weight_min": -33.75323486328125,
      "activations/layer15_attention_weight_max": 39.458946228027344,
      "activations/layer15_attention_weight_min": -32.62663650512695,
      "activations/layer16_attention_weight_max": 44.42522430419922,
      "activations/layer16_attention_weight_min": -27.6330623626709,
      "activations/layer17_attention_weight_max": 61.12643051147461,
      "activations/layer17_attention_weight_min": -42.98103713989258,
      "activations/layer18_attention_weight_max": 55.13210678100586,
      "activations/layer18_attention_weight_min": -34.78577423095703,
      "activations/layer19_attention_weight_max": 20.114192962646484,
      "activations/layer19_attention_weight_min": -20.72726821899414,
      "activations/layer1_attention_weight_max": 17.119112014770508,
      "activations/layer1_attention_weight_min": -15.914727210998535,
      "activations/layer20_attention_weight_max": 24.08066749572754,
      "activations/layer20_attention_weight_min": -21.746654510498047,
      "activations/layer21_attention_weight_max": 38.12997055053711,
      "activations/layer21_attention_weight_min": -24.10093879699707,
      "activations/layer22_attention_weight_max": 38.765281677246094,
      "activations/layer22_attention_weight_min": -25.487060546875,
      "activations/layer23_attention_weight_max": 36.49961853027344,
      "activations/layer23_attention_weight_min": -20.42359733581543,
      "activations/layer2_attention_weight_max": 31.448225021362305,
      "activations/layer2_attention_weight_min": -28.70623779296875,
      "activations/layer3_attention_weight_max": 79.1229019165039,
      "activations/layer3_attention_weight_min": -82.51144409179688,
      "activations/layer4_attention_weight_max": 96.59838104248047,
      "activations/layer4_attention_weight_min": -90.95806121826172,
      "activations/layer5_attention_weight_max": 90.22300720214844,
      "activations/layer5_attention_weight_min": -83.63167572021484,
      "activations/layer6_attention_weight_max": 56.26033020019531,
      "activations/layer6_attention_weight_min": -53.80732727050781,
      "activations/layer7_attention_weight_max": 80.11431884765625,
      "activations/layer7_attention_weight_min": -71.86212921142578,
      "activations/layer8_attention_weight_max": 45.16020584106445,
      "activations/layer8_attention_weight_min": -45.0771369934082,
      "activations/layer9_attention_weight_max": 53.837154388427734,
      "activations/layer9_attention_weight_min": -51.85612106323242,
      "epoch": 6.95,
      "learning_rate": 0.00010625454545454545,
      "loss": 2.8446,
      "step": 119600
    },
    {
      "activations/layer0_attention_weight_max": 15.68387222290039,
      "activations/layer0_attention_weight_min": -15.008633613586426,
      "activations/layer10_attention_weight_max": 39.09996795654297,
      "activations/layer10_attention_weight_min": -36.75898361206055,
      "activations/layer11_attention_weight_max": 37.11041259765625,
      "activations/layer11_attention_weight_min": -30.050643920898438,
      "activations/layer12_attention_weight_max": 19.80764389038086,
      "activations/layer12_attention_weight_min": -25.0825138092041,
      "activations/layer13_attention_weight_max": 43.485328674316406,
      "activations/layer13_attention_weight_min": -33.16981506347656,
      "activations/layer14_attention_weight_max": 39.02687454223633,
      "activations/layer14_attention_weight_min": -30.666160583496094,
      "activations/layer15_attention_weight_max": 39.6024055480957,
      "activations/layer15_attention_weight_min": -29.998058319091797,
      "activations/layer16_attention_weight_max": 38.832763671875,
      "activations/layer16_attention_weight_min": -27.02525520324707,
      "activations/layer17_attention_weight_max": 61.32503128051758,
      "activations/layer17_attention_weight_min": -43.225914001464844,
      "activations/layer18_attention_weight_max": 48.455562591552734,
      "activations/layer18_attention_weight_min": -36.51055145263672,
      "activations/layer19_attention_weight_max": 27.808156967163086,
      "activations/layer19_attention_weight_min": -21.107664108276367,
      "activations/layer1_attention_weight_max": 16.9964656829834,
      "activations/layer1_attention_weight_min": -15.210345268249512,
      "activations/layer20_attention_weight_max": 24.725200653076172,
      "activations/layer20_attention_weight_min": -20.373258590698242,
      "activations/layer21_attention_weight_max": 42.657554626464844,
      "activations/layer21_attention_weight_min": -23.857391357421875,
      "activations/layer22_attention_weight_max": 41.68626022338867,
      "activations/layer22_attention_weight_min": -25.020296096801758,
      "activations/layer23_attention_weight_max": 35.99856948852539,
      "activations/layer23_attention_weight_min": -20.19527816772461,
      "activations/layer2_attention_weight_max": 33.35389709472656,
      "activations/layer2_attention_weight_min": -31.244598388671875,
      "activations/layer3_attention_weight_max": 87.80353546142578,
      "activations/layer3_attention_weight_min": -95.36030578613281,
      "activations/layer4_attention_weight_max": 101.20901489257812,
      "activations/layer4_attention_weight_min": -100.10345458984375,
      "activations/layer5_attention_weight_max": 95.27842712402344,
      "activations/layer5_attention_weight_min": -93.01530456542969,
      "activations/layer6_attention_weight_max": 56.48042297363281,
      "activations/layer6_attention_weight_min": -57.24138641357422,
      "activations/layer7_attention_weight_max": 78.32514190673828,
      "activations/layer7_attention_weight_min": -74.19722747802734,
      "activations/layer8_attention_weight_max": 47.72859573364258,
      "activations/layer8_attention_weight_min": -46.36579895019531,
      "activations/layer9_attention_weight_max": 70.30641174316406,
      "activations/layer9_attention_weight_min": -50.809043884277344,
      "epoch": 6.95,
      "learning_rate": 0.00010623560606060605,
      "loss": 2.829,
      "step": 119650
    },
    {
      "activations/layer0_attention_weight_max": 14.423968315124512,
      "activations/layer0_attention_weight_min": -14.996100425720215,
      "activations/layer10_attention_weight_max": 33.439430236816406,
      "activations/layer10_attention_weight_min": -32.45867919921875,
      "activations/layer11_attention_weight_max": 28.363868713378906,
      "activations/layer11_attention_weight_min": -28.520261764526367,
      "activations/layer12_attention_weight_max": 20.730615615844727,
      "activations/layer12_attention_weight_min": -24.54654884338379,
      "activations/layer13_attention_weight_max": 44.34253692626953,
      "activations/layer13_attention_weight_min": -34.576622009277344,
      "activations/layer14_attention_weight_max": 38.20195388793945,
      "activations/layer14_attention_weight_min": -29.036977767944336,
      "activations/layer15_attention_weight_max": 39.34388732910156,
      "activations/layer15_attention_weight_min": -29.462778091430664,
      "activations/layer16_attention_weight_max": 30.862682342529297,
      "activations/layer16_attention_weight_min": -27.33218765258789,
      "activations/layer17_attention_weight_max": 56.6288948059082,
      "activations/layer17_attention_weight_min": -39.879337310791016,
      "activations/layer18_attention_weight_max": 54.00117492675781,
      "activations/layer18_attention_weight_min": -33.60274887084961,
      "activations/layer19_attention_weight_max": 24.670635223388672,
      "activations/layer19_attention_weight_min": -20.335304260253906,
      "activations/layer1_attention_weight_max": 16.61677360534668,
      "activations/layer1_attention_weight_min": -16.53186798095703,
      "activations/layer20_attention_weight_max": 22.330223083496094,
      "activations/layer20_attention_weight_min": -22.05554962158203,
      "activations/layer21_attention_weight_max": 40.843963623046875,
      "activations/layer21_attention_weight_min": -23.15635871887207,
      "activations/layer22_attention_weight_max": 29.751327514648438,
      "activations/layer22_attention_weight_min": -26.220186233520508,
      "activations/layer23_attention_weight_max": 36.55569076538086,
      "activations/layer23_attention_weight_min": -20.10645294189453,
      "activations/layer2_attention_weight_max": 29.786733627319336,
      "activations/layer2_attention_weight_min": -30.060483932495117,
      "activations/layer3_attention_weight_max": 83.75452423095703,
      "activations/layer3_attention_weight_min": -85.1009292602539,
      "activations/layer4_attention_weight_max": 96.7967300415039,
      "activations/layer4_attention_weight_min": -91.8857421875,
      "activations/layer5_attention_weight_max": 85.5965576171875,
      "activations/layer5_attention_weight_min": -80.36264038085938,
      "activations/layer6_attention_weight_max": 53.08809280395508,
      "activations/layer6_attention_weight_min": -52.26655197143555,
      "activations/layer7_attention_weight_max": 69.75491333007812,
      "activations/layer7_attention_weight_min": -67.07102966308594,
      "activations/layer8_attention_weight_max": 40.65041732788086,
      "activations/layer8_attention_weight_min": -42.92265701293945,
      "activations/layer9_attention_weight_max": 44.54025650024414,
      "activations/layer9_attention_weight_min": -44.63706970214844,
      "epoch": 6.96,
      "learning_rate": 0.00010621666666666666,
      "loss": 2.8178,
      "step": 119700
    },
    {
      "activations/layer0_attention_weight_max": 15.184813499450684,
      "activations/layer0_attention_weight_min": -15.434910774230957,
      "activations/layer10_attention_weight_max": 38.33734130859375,
      "activations/layer10_attention_weight_min": -36.76496505737305,
      "activations/layer11_attention_weight_max": 31.209104537963867,
      "activations/layer11_attention_weight_min": -32.179359436035156,
      "activations/layer12_attention_weight_max": 21.667638778686523,
      "activations/layer12_attention_weight_min": -25.646638870239258,
      "activations/layer13_attention_weight_max": 49.57444763183594,
      "activations/layer13_attention_weight_min": -31.883684158325195,
      "activations/layer14_attention_weight_max": 40.94145965576172,
      "activations/layer14_attention_weight_min": -31.64600372314453,
      "activations/layer15_attention_weight_max": 40.269866943359375,
      "activations/layer15_attention_weight_min": -29.242536544799805,
      "activations/layer16_attention_weight_max": 38.62394332885742,
      "activations/layer16_attention_weight_min": -27.56410789489746,
      "activations/layer17_attention_weight_max": 54.915523529052734,
      "activations/layer17_attention_weight_min": -43.65690231323242,
      "activations/layer18_attention_weight_max": 48.5896110534668,
      "activations/layer18_attention_weight_min": -36.142112731933594,
      "activations/layer19_attention_weight_max": 29.19742774963379,
      "activations/layer19_attention_weight_min": -19.782672882080078,
      "activations/layer1_attention_weight_max": 17.037715911865234,
      "activations/layer1_attention_weight_min": -17.248687744140625,
      "activations/layer20_attention_weight_max": 23.886533737182617,
      "activations/layer20_attention_weight_min": -20.858333587646484,
      "activations/layer21_attention_weight_max": 39.34437942504883,
      "activations/layer21_attention_weight_min": -24.57684326171875,
      "activations/layer22_attention_weight_max": 32.5569953918457,
      "activations/layer22_attention_weight_min": -25.20587921142578,
      "activations/layer23_attention_weight_max": 34.61839294433594,
      "activations/layer23_attention_weight_min": -21.078094482421875,
      "activations/layer2_attention_weight_max": 31.612735748291016,
      "activations/layer2_attention_weight_min": -29.184804916381836,
      "activations/layer3_attention_weight_max": 87.0475845336914,
      "activations/layer3_attention_weight_min": -84.8166275024414,
      "activations/layer4_attention_weight_max": 99.75704956054688,
      "activations/layer4_attention_weight_min": -93.42607116699219,
      "activations/layer5_attention_weight_max": 92.19587707519531,
      "activations/layer5_attention_weight_min": -88.94271087646484,
      "activations/layer6_attention_weight_max": 55.868656158447266,
      "activations/layer6_attention_weight_min": -53.40827941894531,
      "activations/layer7_attention_weight_max": 74.3617935180664,
      "activations/layer7_attention_weight_min": -68.75149536132812,
      "activations/layer8_attention_weight_max": 43.28185272216797,
      "activations/layer8_attention_weight_min": -43.16231155395508,
      "activations/layer9_attention_weight_max": 49.17567825317383,
      "activations/layer9_attention_weight_min": -47.75431823730469,
      "epoch": 6.96,
      "learning_rate": 0.00010619772727272726,
      "loss": 2.8316,
      "step": 119750
    },
    {
      "activations/layer0_attention_weight_max": 15.157482147216797,
      "activations/layer0_attention_weight_min": -14.825776100158691,
      "activations/layer10_attention_weight_max": 36.033660888671875,
      "activations/layer10_attention_weight_min": -36.45140075683594,
      "activations/layer11_attention_weight_max": 30.733196258544922,
      "activations/layer11_attention_weight_min": -32.01765823364258,
      "activations/layer12_attention_weight_max": 24.127607345581055,
      "activations/layer12_attention_weight_min": -23.77309799194336,
      "activations/layer13_attention_weight_max": 38.38710021972656,
      "activations/layer13_attention_weight_min": -31.339115142822266,
      "activations/layer14_attention_weight_max": 35.519561767578125,
      "activations/layer14_attention_weight_min": -31.66332244873047,
      "activations/layer15_attention_weight_max": 34.14826965332031,
      "activations/layer15_attention_weight_min": -30.13751792907715,
      "activations/layer16_attention_weight_max": 33.46829605102539,
      "activations/layer16_attention_weight_min": -29.256685256958008,
      "activations/layer17_attention_weight_max": 50.48039627075195,
      "activations/layer17_attention_weight_min": -41.041019439697266,
      "activations/layer18_attention_weight_max": 46.55281448364258,
      "activations/layer18_attention_weight_min": -34.064903259277344,
      "activations/layer19_attention_weight_max": 18.765623092651367,
      "activations/layer19_attention_weight_min": -18.607728958129883,
      "activations/layer1_attention_weight_max": 17.301767349243164,
      "activations/layer1_attention_weight_min": -18.739524841308594,
      "activations/layer20_attention_weight_max": 22.024444580078125,
      "activations/layer20_attention_weight_min": -18.88697624206543,
      "activations/layer21_attention_weight_max": 37.289573669433594,
      "activations/layer21_attention_weight_min": -22.196069717407227,
      "activations/layer22_attention_weight_max": 32.626956939697266,
      "activations/layer22_attention_weight_min": -28.008604049682617,
      "activations/layer23_attention_weight_max": 33.338951110839844,
      "activations/layer23_attention_weight_min": -20.736492156982422,
      "activations/layer2_attention_weight_max": 32.937355041503906,
      "activations/layer2_attention_weight_min": -29.51814079284668,
      "activations/layer3_attention_weight_max": 80.09646606445312,
      "activations/layer3_attention_weight_min": -86.47318267822266,
      "activations/layer4_attention_weight_max": 98.8765640258789,
      "activations/layer4_attention_weight_min": -93.619140625,
      "activations/layer5_attention_weight_max": 89.30855560302734,
      "activations/layer5_attention_weight_min": -81.46353149414062,
      "activations/layer6_attention_weight_max": 55.88797378540039,
      "activations/layer6_attention_weight_min": -52.752750396728516,
      "activations/layer7_attention_weight_max": 76.73306274414062,
      "activations/layer7_attention_weight_min": -71.40750122070312,
      "activations/layer8_attention_weight_max": 45.927589416503906,
      "activations/layer8_attention_weight_min": -45.10826110839844,
      "activations/layer9_attention_weight_max": 52.686946868896484,
      "activations/layer9_attention_weight_min": -51.7270622253418,
      "epoch": 6.96,
      "learning_rate": 0.00010617878787878787,
      "loss": 2.8379,
      "step": 119800
    },
    {
      "activations/layer0_attention_weight_max": 15.647279739379883,
      "activations/layer0_attention_weight_min": -15.000981330871582,
      "activations/layer10_attention_weight_max": 44.54243850708008,
      "activations/layer10_attention_weight_min": -40.88267517089844,
      "activations/layer11_attention_weight_max": 38.85801696777344,
      "activations/layer11_attention_weight_min": -32.353240966796875,
      "activations/layer12_attention_weight_max": 26.334522247314453,
      "activations/layer12_attention_weight_min": -25.85221290588379,
      "activations/layer13_attention_weight_max": 44.78727340698242,
      "activations/layer13_attention_weight_min": -35.89009094238281,
      "activations/layer14_attention_weight_max": 46.604576110839844,
      "activations/layer14_attention_weight_min": -32.30402755737305,
      "activations/layer15_attention_weight_max": 36.30389404296875,
      "activations/layer15_attention_weight_min": -30.835988998413086,
      "activations/layer16_attention_weight_max": 33.9183235168457,
      "activations/layer16_attention_weight_min": -27.42137336730957,
      "activations/layer17_attention_weight_max": 57.07486343383789,
      "activations/layer17_attention_weight_min": -43.55488967895508,
      "activations/layer18_attention_weight_max": 43.524295806884766,
      "activations/layer18_attention_weight_min": -35.44746398925781,
      "activations/layer19_attention_weight_max": 21.216976165771484,
      "activations/layer19_attention_weight_min": -20.54901695251465,
      "activations/layer1_attention_weight_max": 16.43157386779785,
      "activations/layer1_attention_weight_min": -16.03664207458496,
      "activations/layer20_attention_weight_max": 21.79387664794922,
      "activations/layer20_attention_weight_min": -25.89252281188965,
      "activations/layer21_attention_weight_max": 39.06927490234375,
      "activations/layer21_attention_weight_min": -24.468399047851562,
      "activations/layer22_attention_weight_max": 31.765764236450195,
      "activations/layer22_attention_weight_min": -23.075620651245117,
      "activations/layer23_attention_weight_max": 31.402454376220703,
      "activations/layer23_attention_weight_min": -21.7772274017334,
      "activations/layer2_attention_weight_max": 31.986371994018555,
      "activations/layer2_attention_weight_min": -28.641071319580078,
      "activations/layer3_attention_weight_max": 84.87933349609375,
      "activations/layer3_attention_weight_min": -86.08684539794922,
      "activations/layer4_attention_weight_max": 98.10625457763672,
      "activations/layer4_attention_weight_min": -91.25019836425781,
      "activations/layer5_attention_weight_max": 91.49523162841797,
      "activations/layer5_attention_weight_min": -81.03662109375,
      "activations/layer6_attention_weight_max": 55.59279251098633,
      "activations/layer6_attention_weight_min": -51.91923522949219,
      "activations/layer7_attention_weight_max": 72.8036117553711,
      "activations/layer7_attention_weight_min": -69.6031494140625,
      "activations/layer8_attention_weight_max": 46.38092041015625,
      "activations/layer8_attention_weight_min": -48.03175354003906,
      "activations/layer9_attention_weight_max": 58.65838623046875,
      "activations/layer9_attention_weight_min": -49.60430145263672,
      "epoch": 6.96,
      "learning_rate": 0.00010615984848484849,
      "loss": 2.832,
      "step": 119850
    },
    {
      "activations/layer0_attention_weight_max": 15.395054817199707,
      "activations/layer0_attention_weight_min": -14.967976570129395,
      "activations/layer10_attention_weight_max": 38.41549301147461,
      "activations/layer10_attention_weight_min": -35.18815994262695,
      "activations/layer11_attention_weight_max": 33.52891159057617,
      "activations/layer11_attention_weight_min": -29.59677505493164,
      "activations/layer12_attention_weight_max": 23.882726669311523,
      "activations/layer12_attention_weight_min": -28.088504791259766,
      "activations/layer13_attention_weight_max": 45.63932418823242,
      "activations/layer13_attention_weight_min": -29.112390518188477,
      "activations/layer14_attention_weight_max": 50.622337341308594,
      "activations/layer14_attention_weight_min": -30.014265060424805,
      "activations/layer15_attention_weight_max": 38.32273483276367,
      "activations/layer15_attention_weight_min": -29.208261489868164,
      "activations/layer16_attention_weight_max": 36.04579162597656,
      "activations/layer16_attention_weight_min": -26.841999053955078,
      "activations/layer17_attention_weight_max": 60.97220993041992,
      "activations/layer17_attention_weight_min": -47.27134704589844,
      "activations/layer18_attention_weight_max": 53.89244842529297,
      "activations/layer18_attention_weight_min": -38.458553314208984,
      "activations/layer19_attention_weight_max": 21.78137969970703,
      "activations/layer19_attention_weight_min": -22.318500518798828,
      "activations/layer1_attention_weight_max": 16.55840492248535,
      "activations/layer1_attention_weight_min": -16.67124366760254,
      "activations/layer20_attention_weight_max": 22.98343276977539,
      "activations/layer20_attention_weight_min": -22.600120544433594,
      "activations/layer21_attention_weight_max": 42.71119689941406,
      "activations/layer21_attention_weight_min": -23.12276268005371,
      "activations/layer22_attention_weight_max": 37.55996322631836,
      "activations/layer22_attention_weight_min": -24.73421859741211,
      "activations/layer23_attention_weight_max": 33.13797378540039,
      "activations/layer23_attention_weight_min": -20.937423706054688,
      "activations/layer2_attention_weight_max": 33.45679473876953,
      "activations/layer2_attention_weight_min": -29.633230209350586,
      "activations/layer3_attention_weight_max": 87.83271026611328,
      "activations/layer3_attention_weight_min": -84.73953247070312,
      "activations/layer4_attention_weight_max": 101.86045837402344,
      "activations/layer4_attention_weight_min": -94.01869201660156,
      "activations/layer5_attention_weight_max": 94.42003631591797,
      "activations/layer5_attention_weight_min": -84.79515075683594,
      "activations/layer6_attention_weight_max": 56.274444580078125,
      "activations/layer6_attention_weight_min": -51.95650100708008,
      "activations/layer7_attention_weight_max": 77.72513580322266,
      "activations/layer7_attention_weight_min": -68.72748565673828,
      "activations/layer8_attention_weight_max": 47.586910247802734,
      "activations/layer8_attention_weight_min": -43.252960205078125,
      "activations/layer9_attention_weight_max": 50.770729064941406,
      "activations/layer9_attention_weight_min": -47.95094299316406,
      "epoch": 6.97,
      "learning_rate": 0.00010614090909090907,
      "loss": 2.8225,
      "step": 119900
    },
    {
      "activations/layer0_attention_weight_max": 14.138265609741211,
      "activations/layer0_attention_weight_min": -15.043506622314453,
      "activations/layer10_attention_weight_max": 41.889339447021484,
      "activations/layer10_attention_weight_min": -39.8077278137207,
      "activations/layer11_attention_weight_max": 33.4352912902832,
      "activations/layer11_attention_weight_min": -31.101390838623047,
      "activations/layer12_attention_weight_max": 20.05304718017578,
      "activations/layer12_attention_weight_min": -28.266191482543945,
      "activations/layer13_attention_weight_max": 43.29172134399414,
      "activations/layer13_attention_weight_min": -33.79616928100586,
      "activations/layer14_attention_weight_max": 37.133811950683594,
      "activations/layer14_attention_weight_min": -31.573505401611328,
      "activations/layer15_attention_weight_max": 40.087860107421875,
      "activations/layer15_attention_weight_min": -31.305173873901367,
      "activations/layer16_attention_weight_max": 32.40083312988281,
      "activations/layer16_attention_weight_min": -26.8249568939209,
      "activations/layer17_attention_weight_max": 55.32730484008789,
      "activations/layer17_attention_weight_min": -42.74755859375,
      "activations/layer18_attention_weight_max": 44.93856430053711,
      "activations/layer18_attention_weight_min": -35.400474548339844,
      "activations/layer19_attention_weight_max": 21.153823852539062,
      "activations/layer19_attention_weight_min": -20.53037452697754,
      "activations/layer1_attention_weight_max": 18.57978630065918,
      "activations/layer1_attention_weight_min": -13.668244361877441,
      "activations/layer20_attention_weight_max": 20.633573532104492,
      "activations/layer20_attention_weight_min": -20.721891403198242,
      "activations/layer21_attention_weight_max": 37.77189254760742,
      "activations/layer21_attention_weight_min": -23.93837547302246,
      "activations/layer22_attention_weight_max": 33.588401794433594,
      "activations/layer22_attention_weight_min": -25.9226131439209,
      "activations/layer23_attention_weight_max": 35.46765899658203,
      "activations/layer23_attention_weight_min": -20.14753532409668,
      "activations/layer2_attention_weight_max": 31.32405662536621,
      "activations/layer2_attention_weight_min": -29.35902214050293,
      "activations/layer3_attention_weight_max": 79.89797973632812,
      "activations/layer3_attention_weight_min": -83.93722534179688,
      "activations/layer4_attention_weight_max": 97.23358154296875,
      "activations/layer4_attention_weight_min": -98.44957733154297,
      "activations/layer5_attention_weight_max": 88.27973937988281,
      "activations/layer5_attention_weight_min": -85.74595642089844,
      "activations/layer6_attention_weight_max": 56.328773498535156,
      "activations/layer6_attention_weight_min": -56.70159149169922,
      "activations/layer7_attention_weight_max": 72.81389617919922,
      "activations/layer7_attention_weight_min": -79.19185638427734,
      "activations/layer8_attention_weight_max": 48.01742172241211,
      "activations/layer8_attention_weight_min": -46.82497024536133,
      "activations/layer9_attention_weight_max": 55.7459716796875,
      "activations/layer9_attention_weight_min": -52.325069427490234,
      "epoch": 6.97,
      "learning_rate": 0.00010612196969696969,
      "loss": 2.8123,
      "step": 119950
    },
    {
      "activations/layer0_attention_weight_max": 15.137930870056152,
      "activations/layer0_attention_weight_min": -15.388379096984863,
      "activations/layer10_attention_weight_max": 42.052188873291016,
      "activations/layer10_attention_weight_min": -36.730587005615234,
      "activations/layer11_attention_weight_max": 34.34113311767578,
      "activations/layer11_attention_weight_min": -31.174365997314453,
      "activations/layer12_attention_weight_max": 30.521390914916992,
      "activations/layer12_attention_weight_min": -23.34592056274414,
      "activations/layer13_attention_weight_max": 47.22134780883789,
      "activations/layer13_attention_weight_min": -34.97982406616211,
      "activations/layer14_attention_weight_max": 40.52547836303711,
      "activations/layer14_attention_weight_min": -30.40290069580078,
      "activations/layer15_attention_weight_max": 37.86564636230469,
      "activations/layer15_attention_weight_min": -30.189987182617188,
      "activations/layer16_attention_weight_max": 38.78893280029297,
      "activations/layer16_attention_weight_min": -27.20770263671875,
      "activations/layer17_attention_weight_max": 56.666358947753906,
      "activations/layer17_attention_weight_min": -41.11726379394531,
      "activations/layer18_attention_weight_max": 46.81705093383789,
      "activations/layer18_attention_weight_min": -36.67382049560547,
      "activations/layer19_attention_weight_max": 23.416427612304688,
      "activations/layer19_attention_weight_min": -20.11629867553711,
      "activations/layer1_attention_weight_max": 17.17426872253418,
      "activations/layer1_attention_weight_min": -15.455286026000977,
      "activations/layer20_attention_weight_max": 21.52350425720215,
      "activations/layer20_attention_weight_min": -29.5612850189209,
      "activations/layer21_attention_weight_max": 40.6456413269043,
      "activations/layer21_attention_weight_min": -24.305089950561523,
      "activations/layer22_attention_weight_max": 35.423095703125,
      "activations/layer22_attention_weight_min": -24.687084197998047,
      "activations/layer23_attention_weight_max": 34.63461685180664,
      "activations/layer23_attention_weight_min": -19.478178024291992,
      "activations/layer2_attention_weight_max": 31.697772979736328,
      "activations/layer2_attention_weight_min": -29.92426300048828,
      "activations/layer3_attention_weight_max": 90.07239532470703,
      "activations/layer3_attention_weight_min": -88.98655700683594,
      "activations/layer4_attention_weight_max": 104.9658432006836,
      "activations/layer4_attention_weight_min": -98.65686798095703,
      "activations/layer5_attention_weight_max": 96.18431091308594,
      "activations/layer5_attention_weight_min": -87.43266296386719,
      "activations/layer6_attention_weight_max": 57.66753005981445,
      "activations/layer6_attention_weight_min": -55.21757125854492,
      "activations/layer7_attention_weight_max": 78.34317016601562,
      "activations/layer7_attention_weight_min": -73.39614868164062,
      "activations/layer8_attention_weight_max": 45.44047546386719,
      "activations/layer8_attention_weight_min": -45.76689147949219,
      "activations/layer9_attention_weight_max": 56.41569900512695,
      "activations/layer9_attention_weight_min": -50.82125473022461,
      "epoch": 6.97,
      "learning_rate": 0.0001061030303030303,
      "loss": 2.8059,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5484,
      "eval_samples_per_second": 502.313,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5484,
      "eval_openwebtext_samples_per_second": 502.313,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_wikitext_loss": 3.009765625,
      "eval_wikitext_ppl": 20.282645623050005,
      "eval_wikitext_runtime": 2.0423,
      "eval_wikitext_samples_per_second": 223.283,
      "step": 120000
    },
    {
      "epoch": 6.97,
      "eval_lambada_loss": 2.748046875,
      "eval_lambada_ppl": 15.612109685326937,
      "eval_lambada_runtime": 9.6014,
      "eval_lambada_samples_per_second": 507.116,
      "step": 120000
    },
    {
      "activations/layer0_attention_weight_max": 14.613058090209961,
      "activations/layer0_attention_weight_min": -15.472200393676758,
      "activations/layer10_attention_weight_max": 39.15303421020508,
      "activations/layer10_attention_weight_min": -34.86148452758789,
      "activations/layer11_attention_weight_max": 29.642126083374023,
      "activations/layer11_attention_weight_min": -28.925457000732422,
      "activations/layer12_attention_weight_max": 28.494098663330078,
      "activations/layer12_attention_weight_min": -24.347640991210938,
      "activations/layer13_attention_weight_max": 40.828521728515625,
      "activations/layer13_attention_weight_min": -36.02143478393555,
      "activations/layer14_attention_weight_max": 39.78129577636719,
      "activations/layer14_attention_weight_min": -32.379180908203125,
      "activations/layer15_attention_weight_max": 37.68585968017578,
      "activations/layer15_attention_weight_min": -31.412269592285156,
      "activations/layer16_attention_weight_max": 36.95313262939453,
      "activations/layer16_attention_weight_min": -28.126386642456055,
      "activations/layer17_attention_weight_max": 59.00432586669922,
      "activations/layer17_attention_weight_min": -45.10620880126953,
      "activations/layer18_attention_weight_max": 51.60409164428711,
      "activations/layer18_attention_weight_min": -38.2716064453125,
      "activations/layer19_attention_weight_max": 23.31709098815918,
      "activations/layer19_attention_weight_min": -20.448225021362305,
      "activations/layer1_attention_weight_max": 17.04168128967285,
      "activations/layer1_attention_weight_min": -14.966628074645996,
      "activations/layer20_attention_weight_max": 25.91561508178711,
      "activations/layer20_attention_weight_min": -25.648418426513672,
      "activations/layer21_attention_weight_max": 42.43599319458008,
      "activations/layer21_attention_weight_min": -23.627120971679688,
      "activations/layer22_attention_weight_max": 35.820194244384766,
      "activations/layer22_attention_weight_min": -24.89641571044922,
      "activations/layer23_attention_weight_max": 35.87108612060547,
      "activations/layer23_attention_weight_min": -19.824106216430664,
      "activations/layer2_attention_weight_max": 31.674522399902344,
      "activations/layer2_attention_weight_min": -28.577171325683594,
      "activations/layer3_attention_weight_max": 83.51344299316406,
      "activations/layer3_attention_weight_min": -85.82814025878906,
      "activations/layer4_attention_weight_max": 99.56929779052734,
      "activations/layer4_attention_weight_min": -99.55120849609375,
      "activations/layer5_attention_weight_max": 96.23332214355469,
      "activations/layer5_attention_weight_min": -86.60140991210938,
      "activations/layer6_attention_weight_max": 56.75139236450195,
      "activations/layer6_attention_weight_min": -52.516517639160156,
      "activations/layer7_attention_weight_max": 74.90023040771484,
      "activations/layer7_attention_weight_min": -69.83058166503906,
      "activations/layer8_attention_weight_max": 44.43531036376953,
      "activations/layer8_attention_weight_min": -46.33687210083008,
      "activations/layer9_attention_weight_max": 52.01894760131836,
      "activations/layer9_attention_weight_min": -47.79104995727539,
      "epoch": 6.98,
      "learning_rate": 0.00010608409090909089,
      "loss": 2.83,
      "step": 120050
    },
    {
      "activations/layer0_attention_weight_max": 15.626237869262695,
      "activations/layer0_attention_weight_min": -15.240216255187988,
      "activations/layer10_attention_weight_max": 39.864418029785156,
      "activations/layer10_attention_weight_min": -35.88910675048828,
      "activations/layer11_attention_weight_max": 30.7298583984375,
      "activations/layer11_attention_weight_min": -29.796356201171875,
      "activations/layer12_attention_weight_max": 20.060476303100586,
      "activations/layer12_attention_weight_min": -23.35393524169922,
      "activations/layer13_attention_weight_max": 45.55304718017578,
      "activations/layer13_attention_weight_min": -30.367136001586914,
      "activations/layer14_attention_weight_max": 38.59791564941406,
      "activations/layer14_attention_weight_min": -30.119855880737305,
      "activations/layer15_attention_weight_max": 37.603431701660156,
      "activations/layer15_attention_weight_min": -30.6508731842041,
      "activations/layer16_attention_weight_max": 32.32994079589844,
      "activations/layer16_attention_weight_min": -26.04633140563965,
      "activations/layer17_attention_weight_max": 55.115203857421875,
      "activations/layer17_attention_weight_min": -40.27462387084961,
      "activations/layer18_attention_weight_max": 50.155052185058594,
      "activations/layer18_attention_weight_min": -35.467620849609375,
      "activations/layer19_attention_weight_max": 23.185565948486328,
      "activations/layer19_attention_weight_min": -21.81633186340332,
      "activations/layer1_attention_weight_max": 16.938720703125,
      "activations/layer1_attention_weight_min": -16.6297607421875,
      "activations/layer20_attention_weight_max": 22.22931480407715,
      "activations/layer20_attention_weight_min": -20.793773651123047,
      "activations/layer21_attention_weight_max": 40.82135009765625,
      "activations/layer21_attention_weight_min": -25.21077537536621,
      "activations/layer22_attention_weight_max": 35.21195602416992,
      "activations/layer22_attention_weight_min": -26.95606231689453,
      "activations/layer23_attention_weight_max": 33.721954345703125,
      "activations/layer23_attention_weight_min": -22.02206039428711,
      "activations/layer2_attention_weight_max": 31.354045867919922,
      "activations/layer2_attention_weight_min": -28.778366088867188,
      "activations/layer3_attention_weight_max": 85.42446899414062,
      "activations/layer3_attention_weight_min": -88.186279296875,
      "activations/layer4_attention_weight_max": 100.33818054199219,
      "activations/layer4_attention_weight_min": -94.24336242675781,
      "activations/layer5_attention_weight_max": 90.63528442382812,
      "activations/layer5_attention_weight_min": -86.04451751708984,
      "activations/layer6_attention_weight_max": 56.72315979003906,
      "activations/layer6_attention_weight_min": -53.0241813659668,
      "activations/layer7_attention_weight_max": 76.0836410522461,
      "activations/layer7_attention_weight_min": -71.37371063232422,
      "activations/layer8_attention_weight_max": 44.905975341796875,
      "activations/layer8_attention_weight_min": -42.675804138183594,
      "activations/layer9_attention_weight_max": 50.077842712402344,
      "activations/layer9_attention_weight_min": -48.13517761230469,
      "epoch": 6.98,
      "learning_rate": 0.00010606515151515151,
      "loss": 2.821,
      "step": 120100
    },
    {
      "activations/layer0_attention_weight_max": 15.887332916259766,
      "activations/layer0_attention_weight_min": -14.926180839538574,
      "activations/layer10_attention_weight_max": 36.861244201660156,
      "activations/layer10_attention_weight_min": -34.40367126464844,
      "activations/layer11_attention_weight_max": 30.808704376220703,
      "activations/layer11_attention_weight_min": -29.55007553100586,
      "activations/layer12_attention_weight_max": 17.87201499938965,
      "activations/layer12_attention_weight_min": -25.40650749206543,
      "activations/layer13_attention_weight_max": 47.495487213134766,
      "activations/layer13_attention_weight_min": -35.26945114135742,
      "activations/layer14_attention_weight_max": 42.03363800048828,
      "activations/layer14_attention_weight_min": -30.33024787902832,
      "activations/layer15_attention_weight_max": 35.22514343261719,
      "activations/layer15_attention_weight_min": -30.449338912963867,
      "activations/layer16_attention_weight_max": 33.86027145385742,
      "activations/layer16_attention_weight_min": -28.925941467285156,
      "activations/layer17_attention_weight_max": 52.45391845703125,
      "activations/layer17_attention_weight_min": -43.65694808959961,
      "activations/layer18_attention_weight_max": 47.29849624633789,
      "activations/layer18_attention_weight_min": -35.64584732055664,
      "activations/layer19_attention_weight_max": 22.62022590637207,
      "activations/layer19_attention_weight_min": -21.021425247192383,
      "activations/layer1_attention_weight_max": 17.859086990356445,
      "activations/layer1_attention_weight_min": -15.959054946899414,
      "activations/layer20_attention_weight_max": 22.712928771972656,
      "activations/layer20_attention_weight_min": -21.129186630249023,
      "activations/layer21_attention_weight_max": 40.757415771484375,
      "activations/layer21_attention_weight_min": -24.982097625732422,
      "activations/layer22_attention_weight_max": 35.314205169677734,
      "activations/layer22_attention_weight_min": -26.42507553100586,
      "activations/layer23_attention_weight_max": 33.137786865234375,
      "activations/layer23_attention_weight_min": -21.608243942260742,
      "activations/layer2_attention_weight_max": 32.43946838378906,
      "activations/layer2_attention_weight_min": -29.9700984954834,
      "activations/layer3_attention_weight_max": 84.03235626220703,
      "activations/layer3_attention_weight_min": -87.7265625,
      "activations/layer4_attention_weight_max": 99.35385131835938,
      "activations/layer4_attention_weight_min": -98.02457427978516,
      "activations/layer5_attention_weight_max": 88.01359558105469,
      "activations/layer5_attention_weight_min": -85.05440521240234,
      "activations/layer6_attention_weight_max": 55.09547424316406,
      "activations/layer6_attention_weight_min": -54.63465118408203,
      "activations/layer7_attention_weight_max": 77.34436798095703,
      "activations/layer7_attention_weight_min": -74.85487365722656,
      "activations/layer8_attention_weight_max": 44.15757369995117,
      "activations/layer8_attention_weight_min": -43.741920471191406,
      "activations/layer9_attention_weight_max": 51.02674865722656,
      "activations/layer9_attention_weight_min": -48.16864013671875,
      "epoch": 6.98,
      "learning_rate": 0.00010604621212121211,
      "loss": 2.8257,
      "step": 120150
    },
    {
      "activations/layer0_attention_weight_max": 13.960762977600098,
      "activations/layer0_attention_weight_min": -15.500508308410645,
      "activations/layer10_attention_weight_max": 38.6346435546875,
      "activations/layer10_attention_weight_min": -35.9259033203125,
      "activations/layer11_attention_weight_max": 30.222339630126953,
      "activations/layer11_attention_weight_min": -31.323829650878906,
      "activations/layer12_attention_weight_max": 29.741579055786133,
      "activations/layer12_attention_weight_min": -25.245004653930664,
      "activations/layer13_attention_weight_max": 39.984596252441406,
      "activations/layer13_attention_weight_min": -31.03707504272461,
      "activations/layer14_attention_weight_max": 36.96962356567383,
      "activations/layer14_attention_weight_min": -30.124895095825195,
      "activations/layer15_attention_weight_max": 36.55342483520508,
      "activations/layer15_attention_weight_min": -29.87303352355957,
      "activations/layer16_attention_weight_max": 32.198184967041016,
      "activations/layer16_attention_weight_min": -26.57529640197754,
      "activations/layer17_attention_weight_max": 55.930320739746094,
      "activations/layer17_attention_weight_min": -42.6434440612793,
      "activations/layer18_attention_weight_max": 47.5673942565918,
      "activations/layer18_attention_weight_min": -34.72148895263672,
      "activations/layer19_attention_weight_max": 22.614625930786133,
      "activations/layer19_attention_weight_min": -19.67404556274414,
      "activations/layer1_attention_weight_max": 17.169780731201172,
      "activations/layer1_attention_weight_min": -14.857057571411133,
      "activations/layer20_attention_weight_max": 20.47724723815918,
      "activations/layer20_attention_weight_min": -21.194406509399414,
      "activations/layer21_attention_weight_max": 36.431678771972656,
      "activations/layer21_attention_weight_min": -25.198196411132812,
      "activations/layer22_attention_weight_max": 34.45633316040039,
      "activations/layer22_attention_weight_min": -25.775821685791016,
      "activations/layer23_attention_weight_max": 35.28070831298828,
      "activations/layer23_attention_weight_min": -24.4818172454834,
      "activations/layer2_attention_weight_max": 32.19357681274414,
      "activations/layer2_attention_weight_min": -30.414012908935547,
      "activations/layer3_attention_weight_max": 87.41727447509766,
      "activations/layer3_attention_weight_min": -90.43133544921875,
      "activations/layer4_attention_weight_max": 101.45773315429688,
      "activations/layer4_attention_weight_min": -95.5837631225586,
      "activations/layer5_attention_weight_max": 91.05903625488281,
      "activations/layer5_attention_weight_min": -88.69524383544922,
      "activations/layer6_attention_weight_max": 57.52641296386719,
      "activations/layer6_attention_weight_min": -56.76410675048828,
      "activations/layer7_attention_weight_max": 73.04661560058594,
      "activations/layer7_attention_weight_min": -76.54705810546875,
      "activations/layer8_attention_weight_max": 49.929744720458984,
      "activations/layer8_attention_weight_min": -49.2577018737793,
      "activations/layer9_attention_weight_max": 51.90436935424805,
      "activations/layer9_attention_weight_min": -47.31059265136719,
      "epoch": 6.98,
      "learning_rate": 0.00010602727272727271,
      "loss": 2.8282,
      "step": 120200
    },
    {
      "activations/layer0_attention_weight_max": 15.573963165283203,
      "activations/layer0_attention_weight_min": -14.984025955200195,
      "activations/layer10_attention_weight_max": 39.50749206542969,
      "activations/layer10_attention_weight_min": -36.56769943237305,
      "activations/layer11_attention_weight_max": 32.256263732910156,
      "activations/layer11_attention_weight_min": -29.648303985595703,
      "activations/layer12_attention_weight_max": 30.241666793823242,
      "activations/layer12_attention_weight_min": -24.364763259887695,
      "activations/layer13_attention_weight_max": 46.982948303222656,
      "activations/layer13_attention_weight_min": -34.91053771972656,
      "activations/layer14_attention_weight_max": 45.82362747192383,
      "activations/layer14_attention_weight_min": -33.41477584838867,
      "activations/layer15_attention_weight_max": 39.681827545166016,
      "activations/layer15_attention_weight_min": -30.637035369873047,
      "activations/layer16_attention_weight_max": 36.731536865234375,
      "activations/layer16_attention_weight_min": -28.580244064331055,
      "activations/layer17_attention_weight_max": 55.775630950927734,
      "activations/layer17_attention_weight_min": -46.04399108886719,
      "activations/layer18_attention_weight_max": 49.349639892578125,
      "activations/layer18_attention_weight_min": -36.82333755493164,
      "activations/layer19_attention_weight_max": 24.38251304626465,
      "activations/layer19_attention_weight_min": -21.181049346923828,
      "activations/layer1_attention_weight_max": 17.62020492553711,
      "activations/layer1_attention_weight_min": -13.946770668029785,
      "activations/layer20_attention_weight_max": 22.792463302612305,
      "activations/layer20_attention_weight_min": -21.46779441833496,
      "activations/layer21_attention_weight_max": 52.93288040161133,
      "activations/layer21_attention_weight_min": -28.602657318115234,
      "activations/layer22_attention_weight_max": 36.50484085083008,
      "activations/layer22_attention_weight_min": -27.74822235107422,
      "activations/layer23_attention_weight_max": 39.77625274658203,
      "activations/layer23_attention_weight_min": -20.1937198638916,
      "activations/layer2_attention_weight_max": 31.99933433532715,
      "activations/layer2_attention_weight_min": -32.315940856933594,
      "activations/layer3_attention_weight_max": 90.78123474121094,
      "activations/layer3_attention_weight_min": -90.0631332397461,
      "activations/layer4_attention_weight_max": 104.94756317138672,
      "activations/layer4_attention_weight_min": -98.43997955322266,
      "activations/layer5_attention_weight_max": 96.0328598022461,
      "activations/layer5_attention_weight_min": -91.26439666748047,
      "activations/layer6_attention_weight_max": 58.82830047607422,
      "activations/layer6_attention_weight_min": -55.97645950317383,
      "activations/layer7_attention_weight_max": 78.59412384033203,
      "activations/layer7_attention_weight_min": -72.58154296875,
      "activations/layer8_attention_weight_max": 50.06818389892578,
      "activations/layer8_attention_weight_min": -49.34544372558594,
      "activations/layer9_attention_weight_max": 52.82563018798828,
      "activations/layer9_attention_weight_min": -48.580196380615234,
      "epoch": 6.99,
      "learning_rate": 0.00010600833333333333,
      "loss": 2.85,
      "step": 120250
    },
    {
      "activations/layer0_attention_weight_max": 15.926125526428223,
      "activations/layer0_attention_weight_min": -15.391554832458496,
      "activations/layer10_attention_weight_max": 38.9590950012207,
      "activations/layer10_attention_weight_min": -35.397979736328125,
      "activations/layer11_attention_weight_max": 33.607696533203125,
      "activations/layer11_attention_weight_min": -31.527568817138672,
      "activations/layer12_attention_weight_max": 25.02524185180664,
      "activations/layer12_attention_weight_min": -37.10515213012695,
      "activations/layer13_attention_weight_max": 51.181217193603516,
      "activations/layer13_attention_weight_min": -38.69423294067383,
      "activations/layer14_attention_weight_max": 39.224422454833984,
      "activations/layer14_attention_weight_min": -29.8665828704834,
      "activations/layer15_attention_weight_max": 33.94987106323242,
      "activations/layer15_attention_weight_min": -31.159442901611328,
      "activations/layer16_attention_weight_max": 28.34343719482422,
      "activations/layer16_attention_weight_min": -27.933578491210938,
      "activations/layer17_attention_weight_max": 53.43165588378906,
      "activations/layer17_attention_weight_min": -41.293235778808594,
      "activations/layer18_attention_weight_max": 46.869773864746094,
      "activations/layer18_attention_weight_min": -34.69987106323242,
      "activations/layer19_attention_weight_max": 19.957889556884766,
      "activations/layer19_attention_weight_min": -19.255659103393555,
      "activations/layer1_attention_weight_max": 17.07207679748535,
      "activations/layer1_attention_weight_min": -14.771723747253418,
      "activations/layer20_attention_weight_max": 19.9049129486084,
      "activations/layer20_attention_weight_min": -20.60922622680664,
      "activations/layer21_attention_weight_max": 32.86699676513672,
      "activations/layer21_attention_weight_min": -28.30133056640625,
      "activations/layer22_attention_weight_max": 31.70134162902832,
      "activations/layer22_attention_weight_min": -26.168245315551758,
      "activations/layer23_attention_weight_max": 29.72193145751953,
      "activations/layer23_attention_weight_min": -19.291275024414062,
      "activations/layer2_attention_weight_max": 29.89769744873047,
      "activations/layer2_attention_weight_min": -27.59016227722168,
      "activations/layer3_attention_weight_max": 89.49592590332031,
      "activations/layer3_attention_weight_min": -84.85966491699219,
      "activations/layer4_attention_weight_max": 99.41143798828125,
      "activations/layer4_attention_weight_min": -94.37104797363281,
      "activations/layer5_attention_weight_max": 89.86418914794922,
      "activations/layer5_attention_weight_min": -85.15359497070312,
      "activations/layer6_attention_weight_max": 54.731807708740234,
      "activations/layer6_attention_weight_min": -53.89589309692383,
      "activations/layer7_attention_weight_max": 80.0338134765625,
      "activations/layer7_attention_weight_min": -72.31065368652344,
      "activations/layer8_attention_weight_max": 46.582462310791016,
      "activations/layer8_attention_weight_min": -45.34269714355469,
      "activations/layer9_attention_weight_max": 61.132415771484375,
      "activations/layer9_attention_weight_min": -53.63508987426758,
      "epoch": 6.99,
      "learning_rate": 0.00010598939393939393,
      "loss": 2.8113,
      "step": 120300
    },
    {
      "activations/layer0_attention_weight_max": 14.901386260986328,
      "activations/layer0_attention_weight_min": -15.212503433227539,
      "activations/layer10_attention_weight_max": 40.41350173950195,
      "activations/layer10_attention_weight_min": -35.98695755004883,
      "activations/layer11_attention_weight_max": 36.623844146728516,
      "activations/layer11_attention_weight_min": -31.25270652770996,
      "activations/layer12_attention_weight_max": 22.92405891418457,
      "activations/layer12_attention_weight_min": -24.84491729736328,
      "activations/layer13_attention_weight_max": 61.36833953857422,
      "activations/layer13_attention_weight_min": -36.972686767578125,
      "activations/layer14_attention_weight_max": 86.33615112304688,
      "activations/layer14_attention_weight_min": -39.3027458190918,
      "activations/layer15_attention_weight_max": 63.272300720214844,
      "activations/layer15_attention_weight_min": -32.24767303466797,
      "activations/layer16_attention_weight_max": 36.29074478149414,
      "activations/layer16_attention_weight_min": -28.336076736450195,
      "activations/layer17_attention_weight_max": 61.48173904418945,
      "activations/layer17_attention_weight_min": -45.112998962402344,
      "activations/layer18_attention_weight_max": 53.06968307495117,
      "activations/layer18_attention_weight_min": -38.47379684448242,
      "activations/layer19_attention_weight_max": 27.898290634155273,
      "activations/layer19_attention_weight_min": -22.28092384338379,
      "activations/layer1_attention_weight_max": 18.614925384521484,
      "activations/layer1_attention_weight_min": -17.91036033630371,
      "activations/layer20_attention_weight_max": 28.36669158935547,
      "activations/layer20_attention_weight_min": -22.277694702148438,
      "activations/layer21_attention_weight_max": 52.558197021484375,
      "activations/layer21_attention_weight_min": -24.845378875732422,
      "activations/layer22_attention_weight_max": 37.08954620361328,
      "activations/layer22_attention_weight_min": -26.4958438873291,
      "activations/layer23_attention_weight_max": 42.05949783325195,
      "activations/layer23_attention_weight_min": -18.770801544189453,
      "activations/layer2_attention_weight_max": 33.31243133544922,
      "activations/layer2_attention_weight_min": -31.06439208984375,
      "activations/layer3_attention_weight_max": 90.54899597167969,
      "activations/layer3_attention_weight_min": -94.50242614746094,
      "activations/layer4_attention_weight_max": 103.72945404052734,
      "activations/layer4_attention_weight_min": -97.70671081542969,
      "activations/layer5_attention_weight_max": 91.35600280761719,
      "activations/layer5_attention_weight_min": -83.99752044677734,
      "activations/layer6_attention_weight_max": 56.60371780395508,
      "activations/layer6_attention_weight_min": -52.61227798461914,
      "activations/layer7_attention_weight_max": 82.68915557861328,
      "activations/layer7_attention_weight_min": -74.65968322753906,
      "activations/layer8_attention_weight_max": 50.632572174072266,
      "activations/layer8_attention_weight_min": -47.78947830200195,
      "activations/layer9_attention_weight_max": 65.72982025146484,
      "activations/layer9_attention_weight_min": -57.48401641845703,
      "epoch": 6.99,
      "learning_rate": 0.00010597045454545454,
      "loss": 2.8277,
      "step": 120350
    },
    {
      "activations/layer0_attention_weight_max": 15.73917293548584,
      "activations/layer0_attention_weight_min": -15.434304237365723,
      "activations/layer10_attention_weight_max": 42.3137321472168,
      "activations/layer10_attention_weight_min": -37.14369583129883,
      "activations/layer11_attention_weight_max": 35.801780700683594,
      "activations/layer11_attention_weight_min": -30.20196533203125,
      "activations/layer12_attention_weight_max": 38.33899688720703,
      "activations/layer12_attention_weight_min": -29.332027435302734,
      "activations/layer13_attention_weight_max": 55.03403854370117,
      "activations/layer13_attention_weight_min": -35.10258865356445,
      "activations/layer14_attention_weight_max": 55.22751235961914,
      "activations/layer14_attention_weight_min": -35.04785919189453,
      "activations/layer15_attention_weight_max": 47.77470016479492,
      "activations/layer15_attention_weight_min": -28.700313568115234,
      "activations/layer16_attention_weight_max": 34.5461540222168,
      "activations/layer16_attention_weight_min": -26.26321792602539,
      "activations/layer17_attention_weight_max": 54.34209442138672,
      "activations/layer17_attention_weight_min": -41.7956428527832,
      "activations/layer18_attention_weight_max": 49.857906341552734,
      "activations/layer18_attention_weight_min": -35.74813461303711,
      "activations/layer19_attention_weight_max": 23.209325790405273,
      "activations/layer19_attention_weight_min": -19.475812911987305,
      "activations/layer1_attention_weight_max": 16.744441986083984,
      "activations/layer1_attention_weight_min": -13.894847869873047,
      "activations/layer20_attention_weight_max": 22.340818405151367,
      "activations/layer20_attention_weight_min": -20.723237991333008,
      "activations/layer21_attention_weight_max": 37.835853576660156,
      "activations/layer21_attention_weight_min": -23.754905700683594,
      "activations/layer22_attention_weight_max": 33.670841217041016,
      "activations/layer22_attention_weight_min": -24.454811096191406,
      "activations/layer23_attention_weight_max": 34.114402770996094,
      "activations/layer23_attention_weight_min": -19.298620223999023,
      "activations/layer2_attention_weight_max": 31.50716781616211,
      "activations/layer2_attention_weight_min": -29.33136558532715,
      "activations/layer3_attention_weight_max": 90.04570007324219,
      "activations/layer3_attention_weight_min": -88.74883270263672,
      "activations/layer4_attention_weight_max": 101.46027374267578,
      "activations/layer4_attention_weight_min": -99.47068786621094,
      "activations/layer5_attention_weight_max": 93.06478881835938,
      "activations/layer5_attention_weight_min": -87.51742553710938,
      "activations/layer6_attention_weight_max": 59.671531677246094,
      "activations/layer6_attention_weight_min": -55.954856872558594,
      "activations/layer7_attention_weight_max": 85.14791870117188,
      "activations/layer7_attention_weight_min": -78.19721984863281,
      "activations/layer8_attention_weight_max": 53.89178466796875,
      "activations/layer8_attention_weight_min": -47.11948013305664,
      "activations/layer9_attention_weight_max": 68.18307495117188,
      "activations/layer9_attention_weight_min": -55.63972854614258,
      "epoch": 7.0,
      "learning_rate": 0.00010595151515151514,
      "loss": 2.8297,
      "step": 120400
    },
    {
      "activations/layer0_attention_weight_max": 15.646245956420898,
      "activations/layer0_attention_weight_min": -14.917984008789062,
      "activations/layer10_attention_weight_max": 35.48286819458008,
      "activations/layer10_attention_weight_min": -35.0030517578125,
      "activations/layer11_attention_weight_max": 29.000743865966797,
      "activations/layer11_attention_weight_min": -29.12389373779297,
      "activations/layer12_attention_weight_max": 19.87908363342285,
      "activations/layer12_attention_weight_min": -27.872377395629883,
      "activations/layer13_attention_weight_max": 44.626712799072266,
      "activations/layer13_attention_weight_min": -32.379756927490234,
      "activations/layer14_attention_weight_max": 36.84707260131836,
      "activations/layer14_attention_weight_min": -28.858644485473633,
      "activations/layer15_attention_weight_max": 34.52979278564453,
      "activations/layer15_attention_weight_min": -29.108613967895508,
      "activations/layer16_attention_weight_max": 29.96544075012207,
      "activations/layer16_attention_weight_min": -26.19221305847168,
      "activations/layer17_attention_weight_max": 54.86454772949219,
      "activations/layer17_attention_weight_min": -40.92650604248047,
      "activations/layer18_attention_weight_max": 48.46055603027344,
      "activations/layer18_attention_weight_min": -34.2921257019043,
      "activations/layer19_attention_weight_max": 19.756521224975586,
      "activations/layer19_attention_weight_min": -18.549522399902344,
      "activations/layer1_attention_weight_max": 17.30588722229004,
      "activations/layer1_attention_weight_min": -16.162616729736328,
      "activations/layer20_attention_weight_max": 21.943147659301758,
      "activations/layer20_attention_weight_min": -21.304996490478516,
      "activations/layer21_attention_weight_max": 39.84511184692383,
      "activations/layer21_attention_weight_min": -23.06983184814453,
      "activations/layer22_attention_weight_max": 31.497407913208008,
      "activations/layer22_attention_weight_min": -25.084978103637695,
      "activations/layer23_attention_weight_max": 31.55359649658203,
      "activations/layer23_attention_weight_min": -20.346681594848633,
      "activations/layer2_attention_weight_max": 31.885284423828125,
      "activations/layer2_attention_weight_min": -29.478483200073242,
      "activations/layer3_attention_weight_max": 83.10369873046875,
      "activations/layer3_attention_weight_min": -83.59764099121094,
      "activations/layer4_attention_weight_max": 94.93621063232422,
      "activations/layer4_attention_weight_min": -88.16121673583984,
      "activations/layer5_attention_weight_max": 87.69223022460938,
      "activations/layer5_attention_weight_min": -83.68333435058594,
      "activations/layer6_attention_weight_max": 54.34954071044922,
      "activations/layer6_attention_weight_min": -50.019020080566406,
      "activations/layer7_attention_weight_max": 75.05731201171875,
      "activations/layer7_attention_weight_min": -69.0060043334961,
      "activations/layer8_attention_weight_max": 43.65633010864258,
      "activations/layer8_attention_weight_min": -42.851497650146484,
      "activations/layer9_attention_weight_max": 48.167091369628906,
      "activations/layer9_attention_weight_min": -47.641761779785156,
      "epoch": 7.0,
      "learning_rate": 0.00010593257575757575,
      "loss": 2.8212,
      "step": 120450
    },
    {
      "activations/layer0_attention_weight_max": 15.164519309997559,
      "activations/layer0_attention_weight_min": -15.095009803771973,
      "activations/layer10_attention_weight_max": 39.20775604248047,
      "activations/layer10_attention_weight_min": -35.47636795043945,
      "activations/layer11_attention_weight_max": 30.394384384155273,
      "activations/layer11_attention_weight_min": -31.718997955322266,
      "activations/layer12_attention_weight_max": 22.45124053955078,
      "activations/layer12_attention_weight_min": -26.812868118286133,
      "activations/layer13_attention_weight_max": 47.375892639160156,
      "activations/layer13_attention_weight_min": -35.82701110839844,
      "activations/layer14_attention_weight_max": 39.178157806396484,
      "activations/layer14_attention_weight_min": -31.417320251464844,
      "activations/layer15_attention_weight_max": 36.39195251464844,
      "activations/layer15_attention_weight_min": -29.63124656677246,
      "activations/layer16_attention_weight_max": 34.02793884277344,
      "activations/layer16_attention_weight_min": -26.927587509155273,
      "activations/layer17_attention_weight_max": 54.494407653808594,
      "activations/layer17_attention_weight_min": -42.67244338989258,
      "activations/layer18_attention_weight_max": 46.713356018066406,
      "activations/layer18_attention_weight_min": -37.395233154296875,
      "activations/layer19_attention_weight_max": 21.513036727905273,
      "activations/layer19_attention_weight_min": -21.284391403198242,
      "activations/layer1_attention_weight_max": 17.417236328125,
      "activations/layer1_attention_weight_min": -15.49023723602295,
      "activations/layer20_attention_weight_max": 22.189128875732422,
      "activations/layer20_attention_weight_min": -22.98173713684082,
      "activations/layer21_attention_weight_max": 42.2598876953125,
      "activations/layer21_attention_weight_min": -23.86956787109375,
      "activations/layer22_attention_weight_max": 31.83344841003418,
      "activations/layer22_attention_weight_min": -27.13783073425293,
      "activations/layer23_attention_weight_max": 32.48384094238281,
      "activations/layer23_attention_weight_min": -20.41347885131836,
      "activations/layer2_attention_weight_max": 32.91416931152344,
      "activations/layer2_attention_weight_min": -29.336212158203125,
      "activations/layer3_attention_weight_max": 84.66542053222656,
      "activations/layer3_attention_weight_min": -86.53023529052734,
      "activations/layer4_attention_weight_max": 98.58812713623047,
      "activations/layer4_attention_weight_min": -94.2979736328125,
      "activations/layer5_attention_weight_max": 95.29005432128906,
      "activations/layer5_attention_weight_min": -87.21617889404297,
      "activations/layer6_attention_weight_max": 57.56460952758789,
      "activations/layer6_attention_weight_min": -53.72969055175781,
      "activations/layer7_attention_weight_max": 78.47972869873047,
      "activations/layer7_attention_weight_min": -66.66062927246094,
      "activations/layer8_attention_weight_max": 44.05743408203125,
      "activations/layer8_attention_weight_min": -42.52362823486328,
      "activations/layer9_attention_weight_max": 47.53511428833008,
      "activations/layer9_attention_weight_min": -47.582801818847656,
      "epoch": 7.0,
      "learning_rate": 0.00010591363636363636,
      "loss": 2.8746,
      "step": 120500
    },
    {
      "activations/layer0_attention_weight_max": 14.92524242401123,
      "activations/layer0_attention_weight_min": -15.140325546264648,
      "activations/layer10_attention_weight_max": 37.191749572753906,
      "activations/layer10_attention_weight_min": -34.335914611816406,
      "activations/layer11_attention_weight_max": 29.22821617126465,
      "activations/layer11_attention_weight_min": -32.09126663208008,
      "activations/layer12_attention_weight_max": 22.563201904296875,
      "activations/layer12_attention_weight_min": -24.066715240478516,
      "activations/layer13_attention_weight_max": 43.5733528137207,
      "activations/layer13_attention_weight_min": -34.11562728881836,
      "activations/layer14_attention_weight_max": 42.21074295043945,
      "activations/layer14_attention_weight_min": -32.012054443359375,
      "activations/layer15_attention_weight_max": 36.634742736816406,
      "activations/layer15_attention_weight_min": -29.610206604003906,
      "activations/layer16_attention_weight_max": 31.46730613708496,
      "activations/layer16_attention_weight_min": -27.848703384399414,
      "activations/layer17_attention_weight_max": 54.10124588012695,
      "activations/layer17_attention_weight_min": -43.455528259277344,
      "activations/layer18_attention_weight_max": 48.65729904174805,
      "activations/layer18_attention_weight_min": -36.78828430175781,
      "activations/layer19_attention_weight_max": 23.782344818115234,
      "activations/layer19_attention_weight_min": -25.21687126159668,
      "activations/layer1_attention_weight_max": 16.70864486694336,
      "activations/layer1_attention_weight_min": -17.2902889251709,
      "activations/layer20_attention_weight_max": 21.316659927368164,
      "activations/layer20_attention_weight_min": -24.096345901489258,
      "activations/layer21_attention_weight_max": 37.6441535949707,
      "activations/layer21_attention_weight_min": -24.84453582763672,
      "activations/layer22_attention_weight_max": 30.782821655273438,
      "activations/layer22_attention_weight_min": -25.47029685974121,
      "activations/layer23_attention_weight_max": 32.888572692871094,
      "activations/layer23_attention_weight_min": -21.154645919799805,
      "activations/layer2_attention_weight_max": 32.99287033081055,
      "activations/layer2_attention_weight_min": -30.465665817260742,
      "activations/layer3_attention_weight_max": 89.84171295166016,
      "activations/layer3_attention_weight_min": -86.74906158447266,
      "activations/layer4_attention_weight_max": 101.17327117919922,
      "activations/layer4_attention_weight_min": -93.69512939453125,
      "activations/layer5_attention_weight_max": 90.48320007324219,
      "activations/layer5_attention_weight_min": -88.1505355834961,
      "activations/layer6_attention_weight_max": 60.86015319824219,
      "activations/layer6_attention_weight_min": -58.58238983154297,
      "activations/layer7_attention_weight_max": 79.23615264892578,
      "activations/layer7_attention_weight_min": -74.97232055664062,
      "activations/layer8_attention_weight_max": 45.123939514160156,
      "activations/layer8_attention_weight_min": -45.15467834472656,
      "activations/layer9_attention_weight_max": 50.08489227294922,
      "activations/layer9_attention_weight_min": -48.32441329956055,
      "epoch": 7.0,
      "learning_rate": 0.00010589469696969696,
      "loss": 2.8078,
      "step": 120550
    },
    {
      "activations/layer0_attention_weight_max": 14.405620574951172,
      "activations/layer0_attention_weight_min": -14.809850692749023,
      "activations/layer10_attention_weight_max": 35.606842041015625,
      "activations/layer10_attention_weight_min": -34.86199951171875,
      "activations/layer11_attention_weight_max": 30.920574188232422,
      "activations/layer11_attention_weight_min": -30.241575241088867,
      "activations/layer12_attention_weight_max": 23.451644897460938,
      "activations/layer12_attention_weight_min": -23.6448917388916,
      "activations/layer13_attention_weight_max": 42.156307220458984,
      "activations/layer13_attention_weight_min": -30.99301528930664,
      "activations/layer14_attention_weight_max": 41.02382278442383,
      "activations/layer14_attention_weight_min": -30.421573638916016,
      "activations/layer15_attention_weight_max": 35.56816482543945,
      "activations/layer15_attention_weight_min": -29.245098114013672,
      "activations/layer16_attention_weight_max": 32.627899169921875,
      "activations/layer16_attention_weight_min": -30.601106643676758,
      "activations/layer17_attention_weight_max": 53.06807327270508,
      "activations/layer17_attention_weight_min": -44.02381896972656,
      "activations/layer18_attention_weight_max": 46.87609100341797,
      "activations/layer18_attention_weight_min": -35.019412994384766,
      "activations/layer19_attention_weight_max": 20.436447143554688,
      "activations/layer19_attention_weight_min": -18.25319480895996,
      "activations/layer1_attention_weight_max": 17.92510223388672,
      "activations/layer1_attention_weight_min": -15.893926620483398,
      "activations/layer20_attention_weight_max": 20.475746154785156,
      "activations/layer20_attention_weight_min": -22.560026168823242,
      "activations/layer21_attention_weight_max": 37.63321304321289,
      "activations/layer21_attention_weight_min": -25.59661102294922,
      "activations/layer22_attention_weight_max": 31.072771072387695,
      "activations/layer22_attention_weight_min": -24.44003677368164,
      "activations/layer23_attention_weight_max": 31.294857025146484,
      "activations/layer23_attention_weight_min": -22.817787170410156,
      "activations/layer2_attention_weight_max": 29.303829193115234,
      "activations/layer2_attention_weight_min": -27.80181121826172,
      "activations/layer3_attention_weight_max": 81.37019348144531,
      "activations/layer3_attention_weight_min": -79.84791564941406,
      "activations/layer4_attention_weight_max": 94.23017883300781,
      "activations/layer4_attention_weight_min": -88.68830871582031,
      "activations/layer5_attention_weight_max": 89.23030090332031,
      "activations/layer5_attention_weight_min": -85.26228332519531,
      "activations/layer6_attention_weight_max": 55.48046875,
      "activations/layer6_attention_weight_min": -52.48250961303711,
      "activations/layer7_attention_weight_max": 74.82379913330078,
      "activations/layer7_attention_weight_min": -72.14556121826172,
      "activations/layer8_attention_weight_max": 44.89106750488281,
      "activations/layer8_attention_weight_min": -44.39173889160156,
      "activations/layer9_attention_weight_max": 52.88240432739258,
      "activations/layer9_attention_weight_min": -48.8319206237793,
      "epoch": 7.01,
      "learning_rate": 0.00010587613636363637,
      "loss": 2.8315,
      "step": 120600
    },
    {
      "activations/layer0_attention_weight_max": 14.932025909423828,
      "activations/layer0_attention_weight_min": -14.920652389526367,
      "activations/layer10_attention_weight_max": 38.742462158203125,
      "activations/layer10_attention_weight_min": -35.447547912597656,
      "activations/layer11_attention_weight_max": 30.518146514892578,
      "activations/layer11_attention_weight_min": -30.968442916870117,
      "activations/layer12_attention_weight_max": 21.24921989440918,
      "activations/layer12_attention_weight_min": -25.20684051513672,
      "activations/layer13_attention_weight_max": 46.13082504272461,
      "activations/layer13_attention_weight_min": -33.428443908691406,
      "activations/layer14_attention_weight_max": 48.36946487426758,
      "activations/layer14_attention_weight_min": -35.09892272949219,
      "activations/layer15_attention_weight_max": 35.51811218261719,
      "activations/layer15_attention_weight_min": -28.608854293823242,
      "activations/layer16_attention_weight_max": 32.02116775512695,
      "activations/layer16_attention_weight_min": -26.72382354736328,
      "activations/layer17_attention_weight_max": 52.18243408203125,
      "activations/layer17_attention_weight_min": -40.67987060546875,
      "activations/layer18_attention_weight_max": 50.92388153076172,
      "activations/layer18_attention_weight_min": -35.63947677612305,
      "activations/layer19_attention_weight_max": 19.588850021362305,
      "activations/layer19_attention_weight_min": -20.63555335998535,
      "activations/layer1_attention_weight_max": 17.656457901000977,
      "activations/layer1_attention_weight_min": -15.285021781921387,
      "activations/layer20_attention_weight_max": 21.312522888183594,
      "activations/layer20_attention_weight_min": -25.713760375976562,
      "activations/layer21_attention_weight_max": 37.54793930053711,
      "activations/layer21_attention_weight_min": -28.461544036865234,
      "activations/layer22_attention_weight_max": 31.13517189025879,
      "activations/layer22_attention_weight_min": -25.80995750427246,
      "activations/layer23_attention_weight_max": 33.468326568603516,
      "activations/layer23_attention_weight_min": -25.683490753173828,
      "activations/layer2_attention_weight_max": 29.892719268798828,
      "activations/layer2_attention_weight_min": -28.41139030456543,
      "activations/layer3_attention_weight_max": 84.50743865966797,
      "activations/layer3_attention_weight_min": -81.72747039794922,
      "activations/layer4_attention_weight_max": 96.83580017089844,
      "activations/layer4_attention_weight_min": -91.9133529663086,
      "activations/layer5_attention_weight_max": 91.00115966796875,
      "activations/layer5_attention_weight_min": -85.21443939208984,
      "activations/layer6_attention_weight_max": 55.83745193481445,
      "activations/layer6_attention_weight_min": -55.47228240966797,
      "activations/layer7_attention_weight_max": 75.17607879638672,
      "activations/layer7_attention_weight_min": -70.04270935058594,
      "activations/layer8_attention_weight_max": 48.02241897583008,
      "activations/layer8_attention_weight_min": -44.132259368896484,
      "activations/layer9_attention_weight_max": 54.06391143798828,
      "activations/layer9_attention_weight_min": -53.07046127319336,
      "epoch": 7.01,
      "learning_rate": 0.00010585719696969695,
      "loss": 2.8153,
      "step": 120650
    },
    {
      "activations/layer0_attention_weight_max": 15.892376899719238,
      "activations/layer0_attention_weight_min": -14.929997444152832,
      "activations/layer10_attention_weight_max": 40.83405303955078,
      "activations/layer10_attention_weight_min": -36.538490295410156,
      "activations/layer11_attention_weight_max": 33.20674133300781,
      "activations/layer11_attention_weight_min": -30.942899703979492,
      "activations/layer12_attention_weight_max": 32.14257049560547,
      "activations/layer12_attention_weight_min": -24.044347763061523,
      "activations/layer13_attention_weight_max": 47.07939147949219,
      "activations/layer13_attention_weight_min": -32.6913948059082,
      "activations/layer14_attention_weight_max": 48.95241928100586,
      "activations/layer14_attention_weight_min": -30.53857421875,
      "activations/layer15_attention_weight_max": 43.34638214111328,
      "activations/layer15_attention_weight_min": -31.586502075195312,
      "activations/layer16_attention_weight_max": 34.04265594482422,
      "activations/layer16_attention_weight_min": -26.519081115722656,
      "activations/layer17_attention_weight_max": 56.68178939819336,
      "activations/layer17_attention_weight_min": -41.25438690185547,
      "activations/layer18_attention_weight_max": 48.29973602294922,
      "activations/layer18_attention_weight_min": -36.297786712646484,
      "activations/layer19_attention_weight_max": 24.950246810913086,
      "activations/layer19_attention_weight_min": -21.383766174316406,
      "activations/layer1_attention_weight_max": 18.587202072143555,
      "activations/layer1_attention_weight_min": -16.80868148803711,
      "activations/layer20_attention_weight_max": 26.04222297668457,
      "activations/layer20_attention_weight_min": -22.86830711364746,
      "activations/layer21_attention_weight_max": 55.96861267089844,
      "activations/layer21_attention_weight_min": -25.445219039916992,
      "activations/layer22_attention_weight_max": 34.22427749633789,
      "activations/layer22_attention_weight_min": -25.959260940551758,
      "activations/layer23_attention_weight_max": 37.84780502319336,
      "activations/layer23_attention_weight_min": -19.204742431640625,
      "activations/layer2_attention_weight_max": 33.33018112182617,
      "activations/layer2_attention_weight_min": -30.222789764404297,
      "activations/layer3_attention_weight_max": 84.20437622070312,
      "activations/layer3_attention_weight_min": -86.46746063232422,
      "activations/layer4_attention_weight_max": 102.26399993896484,
      "activations/layer4_attention_weight_min": -95.1437759399414,
      "activations/layer5_attention_weight_max": 92.13665771484375,
      "activations/layer5_attention_weight_min": -84.74200439453125,
      "activations/layer6_attention_weight_max": 57.79854202270508,
      "activations/layer6_attention_weight_min": -52.718265533447266,
      "activations/layer7_attention_weight_max": 74.84036254882812,
      "activations/layer7_attention_weight_min": -74.75772094726562,
      "activations/layer8_attention_weight_max": 46.323280334472656,
      "activations/layer8_attention_weight_min": -44.049373626708984,
      "activations/layer9_attention_weight_max": 52.551971435546875,
      "activations/layer9_attention_weight_min": -48.98170471191406,
      "epoch": 7.01,
      "learning_rate": 0.00010583825757575757,
      "loss": 2.8109,
      "step": 120700
    },
    {
      "activations/layer0_attention_weight_max": 15.539027214050293,
      "activations/layer0_attention_weight_min": -15.330220222473145,
      "activations/layer10_attention_weight_max": 41.141754150390625,
      "activations/layer10_attention_weight_min": -36.776268005371094,
      "activations/layer11_attention_weight_max": 37.60712432861328,
      "activations/layer11_attention_weight_min": -30.127849578857422,
      "activations/layer12_attention_weight_max": 20.631277084350586,
      "activations/layer12_attention_weight_min": -27.53632354736328,
      "activations/layer13_attention_weight_max": 52.639583587646484,
      "activations/layer13_attention_weight_min": -35.42159652709961,
      "activations/layer14_attention_weight_max": 66.25520324707031,
      "activations/layer14_attention_weight_min": -35.01298141479492,
      "activations/layer15_attention_weight_max": 52.48210906982422,
      "activations/layer15_attention_weight_min": -30.005901336669922,
      "activations/layer16_attention_weight_max": 39.53346633911133,
      "activations/layer16_attention_weight_min": -26.384675979614258,
      "activations/layer17_attention_weight_max": 59.43407440185547,
      "activations/layer17_attention_weight_min": -41.88118362426758,
      "activations/layer18_attention_weight_max": 50.333221435546875,
      "activations/layer18_attention_weight_min": -38.77045822143555,
      "activations/layer19_attention_weight_max": 22.480897903442383,
      "activations/layer19_attention_weight_min": -19.87860679626465,
      "activations/layer1_attention_weight_max": 16.261131286621094,
      "activations/layer1_attention_weight_min": -15.784878730773926,
      "activations/layer20_attention_weight_max": 26.493900299072266,
      "activations/layer20_attention_weight_min": -21.25391960144043,
      "activations/layer21_attention_weight_max": 46.94602966308594,
      "activations/layer21_attention_weight_min": -23.709524154663086,
      "activations/layer22_attention_weight_max": 30.979755401611328,
      "activations/layer22_attention_weight_min": -24.390689849853516,
      "activations/layer23_attention_weight_max": 30.759174346923828,
      "activations/layer23_attention_weight_min": -22.392841339111328,
      "activations/layer2_attention_weight_max": 32.42161560058594,
      "activations/layer2_attention_weight_min": -28.718429565429688,
      "activations/layer3_attention_weight_max": 87.92948913574219,
      "activations/layer3_attention_weight_min": -82.71297454833984,
      "activations/layer4_attention_weight_max": 104.1453628540039,
      "activations/layer4_attention_weight_min": -95.99969482421875,
      "activations/layer5_attention_weight_max": 92.27306365966797,
      "activations/layer5_attention_weight_min": -85.51580047607422,
      "activations/layer6_attention_weight_max": 54.33541488647461,
      "activations/layer6_attention_weight_min": -52.454017639160156,
      "activations/layer7_attention_weight_max": 78.6659164428711,
      "activations/layer7_attention_weight_min": -74.28207397460938,
      "activations/layer8_attention_weight_max": 49.46593475341797,
      "activations/layer8_attention_weight_min": -48.705963134765625,
      "activations/layer9_attention_weight_max": 62.541481018066406,
      "activations/layer9_attention_weight_min": -60.252986907958984,
      "epoch": 7.02,
      "learning_rate": 0.00010581931818181818,
      "loss": 2.8027,
      "step": 120750
    },
    {
      "activations/layer0_attention_weight_max": 15.274957656860352,
      "activations/layer0_attention_weight_min": -15.06987476348877,
      "activations/layer10_attention_weight_max": 37.37561798095703,
      "activations/layer10_attention_weight_min": -34.55939865112305,
      "activations/layer11_attention_weight_max": 29.659069061279297,
      "activations/layer11_attention_weight_min": -30.146034240722656,
      "activations/layer12_attention_weight_max": 21.654462814331055,
      "activations/layer12_attention_weight_min": -26.41937828063965,
      "activations/layer13_attention_weight_max": 49.73971939086914,
      "activations/layer13_attention_weight_min": -33.26105499267578,
      "activations/layer14_attention_weight_max": 47.32778549194336,
      "activations/layer14_attention_weight_min": -31.353561401367188,
      "activations/layer15_attention_weight_max": 41.10000991821289,
      "activations/layer15_attention_weight_min": -31.719486236572266,
      "activations/layer16_attention_weight_max": 34.19560241699219,
      "activations/layer16_attention_weight_min": -28.803817749023438,
      "activations/layer17_attention_weight_max": 60.31410598754883,
      "activations/layer17_attention_weight_min": -42.27979278564453,
      "activations/layer18_attention_weight_max": 50.95126724243164,
      "activations/layer18_attention_weight_min": -36.26081466674805,
      "activations/layer19_attention_weight_max": 25.513059616088867,
      "activations/layer19_attention_weight_min": -21.716167449951172,
      "activations/layer1_attention_weight_max": 16.980575561523438,
      "activations/layer1_attention_weight_min": -18.911699295043945,
      "activations/layer20_attention_weight_max": 24.657020568847656,
      "activations/layer20_attention_weight_min": -22.439109802246094,
      "activations/layer21_attention_weight_max": 45.35535430908203,
      "activations/layer21_attention_weight_min": -25.227127075195312,
      "activations/layer22_attention_weight_max": 35.8502082824707,
      "activations/layer22_attention_weight_min": -25.67748260498047,
      "activations/layer23_attention_weight_max": 39.755699157714844,
      "activations/layer23_attention_weight_min": -21.063648223876953,
      "activations/layer2_attention_weight_max": 33.40153884887695,
      "activations/layer2_attention_weight_min": -29.058578491210938,
      "activations/layer3_attention_weight_max": 85.54890441894531,
      "activations/layer3_attention_weight_min": -91.28445434570312,
      "activations/layer4_attention_weight_max": 96.22866821289062,
      "activations/layer4_attention_weight_min": -88.98866271972656,
      "activations/layer5_attention_weight_max": 89.37582397460938,
      "activations/layer5_attention_weight_min": -80.48673248291016,
      "activations/layer6_attention_weight_max": 55.1587028503418,
      "activations/layer6_attention_weight_min": -51.79142379760742,
      "activations/layer7_attention_weight_max": 73.6205825805664,
      "activations/layer7_attention_weight_min": -67.9227294921875,
      "activations/layer8_attention_weight_max": 40.4793701171875,
      "activations/layer8_attention_weight_min": -42.23123550415039,
      "activations/layer9_attention_weight_max": 46.99568176269531,
      "activations/layer9_attention_weight_min": -46.1851921081543,
      "epoch": 7.02,
      "learning_rate": 0.00010580037878787877,
      "loss": 2.8125,
      "step": 120800
    },
    {
      "activations/layer0_attention_weight_max": 14.869953155517578,
      "activations/layer0_attention_weight_min": -14.730790138244629,
      "activations/layer10_attention_weight_max": 38.85650634765625,
      "activations/layer10_attention_weight_min": -35.9104118347168,
      "activations/layer11_attention_weight_max": 33.17210388183594,
      "activations/layer11_attention_weight_min": -32.14692306518555,
      "activations/layer12_attention_weight_max": 25.294431686401367,
      "activations/layer12_attention_weight_min": -24.225303649902344,
      "activations/layer13_attention_weight_max": 47.742530822753906,
      "activations/layer13_attention_weight_min": -34.40196228027344,
      "activations/layer14_attention_weight_max": 46.14649963378906,
      "activations/layer14_attention_weight_min": -30.61962890625,
      "activations/layer15_attention_weight_max": 38.661251068115234,
      "activations/layer15_attention_weight_min": -29.23030662536621,
      "activations/layer16_attention_weight_max": 33.49784469604492,
      "activations/layer16_attention_weight_min": -27.094844818115234,
      "activations/layer17_attention_weight_max": 53.534027099609375,
      "activations/layer17_attention_weight_min": -41.1736946105957,
      "activations/layer18_attention_weight_max": 48.17316436767578,
      "activations/layer18_attention_weight_min": -36.49760818481445,
      "activations/layer19_attention_weight_max": 20.57070541381836,
      "activations/layer19_attention_weight_min": -21.361679077148438,
      "activations/layer1_attention_weight_max": 16.696950912475586,
      "activations/layer1_attention_weight_min": -16.508846282958984,
      "activations/layer20_attention_weight_max": 21.776268005371094,
      "activations/layer20_attention_weight_min": -21.570098876953125,
      "activations/layer21_attention_weight_max": 42.17953109741211,
      "activations/layer21_attention_weight_min": -26.186534881591797,
      "activations/layer22_attention_weight_max": 34.81218338012695,
      "activations/layer22_attention_weight_min": -26.18709945678711,
      "activations/layer23_attention_weight_max": 37.350250244140625,
      "activations/layer23_attention_weight_min": -19.59234046936035,
      "activations/layer2_attention_weight_max": 34.71026611328125,
      "activations/layer2_attention_weight_min": -31.327091217041016,
      "activations/layer3_attention_weight_max": 94.2773666381836,
      "activations/layer3_attention_weight_min": -90.96159362792969,
      "activations/layer4_attention_weight_max": 103.52921295166016,
      "activations/layer4_attention_weight_min": -96.3885726928711,
      "activations/layer5_attention_weight_max": 100.03923797607422,
      "activations/layer5_attention_weight_min": -85.10196685791016,
      "activations/layer6_attention_weight_max": 65.35963439941406,
      "activations/layer6_attention_weight_min": -54.38672637939453,
      "activations/layer7_attention_weight_max": 76.15044403076172,
      "activations/layer7_attention_weight_min": -69.79573822021484,
      "activations/layer8_attention_weight_max": 49.6436767578125,
      "activations/layer8_attention_weight_min": -45.548213958740234,
      "activations/layer9_attention_weight_max": 61.14223098754883,
      "activations/layer9_attention_weight_min": -47.69135665893555,
      "epoch": 7.02,
      "learning_rate": 0.00010578143939393939,
      "loss": 2.8226,
      "step": 120850
    },
    {
      "activations/layer0_attention_weight_max": 14.809226036071777,
      "activations/layer0_attention_weight_min": -15.325804710388184,
      "activations/layer10_attention_weight_max": 41.98158264160156,
      "activations/layer10_attention_weight_min": -37.070518493652344,
      "activations/layer11_attention_weight_max": 34.119598388671875,
      "activations/layer11_attention_weight_min": -32.897666931152344,
      "activations/layer12_attention_weight_max": 23.8593807220459,
      "activations/layer12_attention_weight_min": -27.663663864135742,
      "activations/layer13_attention_weight_max": 45.01346206665039,
      "activations/layer13_attention_weight_min": -37.564170837402344,
      "activations/layer14_attention_weight_max": 41.22732925415039,
      "activations/layer14_attention_weight_min": -30.62317657470703,
      "activations/layer15_attention_weight_max": 36.641204833984375,
      "activations/layer15_attention_weight_min": -28.932655334472656,
      "activations/layer16_attention_weight_max": 32.0797233581543,
      "activations/layer16_attention_weight_min": -25.161399841308594,
      "activations/layer17_attention_weight_max": 54.52918243408203,
      "activations/layer17_attention_weight_min": -41.26078796386719,
      "activations/layer18_attention_weight_max": 46.6359748840332,
      "activations/layer18_attention_weight_min": -33.788856506347656,
      "activations/layer19_attention_weight_max": 22.7089786529541,
      "activations/layer19_attention_weight_min": -19.5471134185791,
      "activations/layer1_attention_weight_max": 16.4450740814209,
      "activations/layer1_attention_weight_min": -14.463369369506836,
      "activations/layer20_attention_weight_max": 21.989702224731445,
      "activations/layer20_attention_weight_min": -18.746139526367188,
      "activations/layer21_attention_weight_max": 38.79678726196289,
      "activations/layer21_attention_weight_min": -23.366012573242188,
      "activations/layer22_attention_weight_max": 33.74162292480469,
      "activations/layer22_attention_weight_min": -24.1109619140625,
      "activations/layer23_attention_weight_max": 30.49779510498047,
      "activations/layer23_attention_weight_min": -19.638957977294922,
      "activations/layer2_attention_weight_max": 33.12595748901367,
      "activations/layer2_attention_weight_min": -30.291547775268555,
      "activations/layer3_attention_weight_max": 85.71392822265625,
      "activations/layer3_attention_weight_min": -89.3875961303711,
      "activations/layer4_attention_weight_max": 100.4992904663086,
      "activations/layer4_attention_weight_min": -93.56069946289062,
      "activations/layer5_attention_weight_max": 93.60365295410156,
      "activations/layer5_attention_weight_min": -89.02140808105469,
      "activations/layer6_attention_weight_max": 57.63861083984375,
      "activations/layer6_attention_weight_min": -54.73140335083008,
      "activations/layer7_attention_weight_max": 78.20266723632812,
      "activations/layer7_attention_weight_min": -73.59876251220703,
      "activations/layer8_attention_weight_max": 47.53978729248047,
      "activations/layer8_attention_weight_min": -46.00385284423828,
      "activations/layer9_attention_weight_max": 57.735023498535156,
      "activations/layer9_attention_weight_min": -52.8245964050293,
      "epoch": 7.02,
      "learning_rate": 0.00010576249999999999,
      "loss": 2.8041,
      "step": 120900
    },
    {
      "activations/layer0_attention_weight_max": 15.279351234436035,
      "activations/layer0_attention_weight_min": -15.283302307128906,
      "activations/layer10_attention_weight_max": 38.6798095703125,
      "activations/layer10_attention_weight_min": -36.49095916748047,
      "activations/layer11_attention_weight_max": 30.815975189208984,
      "activations/layer11_attention_weight_min": -30.395633697509766,
      "activations/layer12_attention_weight_max": 21.666425704956055,
      "activations/layer12_attention_weight_min": -25.215288162231445,
      "activations/layer13_attention_weight_max": 40.36941146850586,
      "activations/layer13_attention_weight_min": -31.53959083557129,
      "activations/layer14_attention_weight_max": 39.805233001708984,
      "activations/layer14_attention_weight_min": -29.324586868286133,
      "activations/layer15_attention_weight_max": 36.14002227783203,
      "activations/layer15_attention_weight_min": -29.88588523864746,
      "activations/layer16_attention_weight_max": 34.834632873535156,
      "activations/layer16_attention_weight_min": -27.322235107421875,
      "activations/layer17_attention_weight_max": 54.814239501953125,
      "activations/layer17_attention_weight_min": -42.65370559692383,
      "activations/layer18_attention_weight_max": 50.99930191040039,
      "activations/layer18_attention_weight_min": -36.132877349853516,
      "activations/layer19_attention_weight_max": 23.013517379760742,
      "activations/layer19_attention_weight_min": -19.574264526367188,
      "activations/layer1_attention_weight_max": 16.088184356689453,
      "activations/layer1_attention_weight_min": -15.147232055664062,
      "activations/layer20_attention_weight_max": 21.987350463867188,
      "activations/layer20_attention_weight_min": -21.335222244262695,
      "activations/layer21_attention_weight_max": 36.48537063598633,
      "activations/layer21_attention_weight_min": -22.785446166992188,
      "activations/layer22_attention_weight_max": 34.18646240234375,
      "activations/layer22_attention_weight_min": -24.572996139526367,
      "activations/layer23_attention_weight_max": 33.22526550292969,
      "activations/layer23_attention_weight_min": -18.75019645690918,
      "activations/layer2_attention_weight_max": 29.766021728515625,
      "activations/layer2_attention_weight_min": -28.720067977905273,
      "activations/layer3_attention_weight_max": 83.38320922851562,
      "activations/layer3_attention_weight_min": -84.3460464477539,
      "activations/layer4_attention_weight_max": 96.20328521728516,
      "activations/layer4_attention_weight_min": -89.71664428710938,
      "activations/layer5_attention_weight_max": 87.20863342285156,
      "activations/layer5_attention_weight_min": -79.7362060546875,
      "activations/layer6_attention_weight_max": 51.951560974121094,
      "activations/layer6_attention_weight_min": -51.89811325073242,
      "activations/layer7_attention_weight_max": 72.72638702392578,
      "activations/layer7_attention_weight_min": -67.94698333740234,
      "activations/layer8_attention_weight_max": 43.335601806640625,
      "activations/layer8_attention_weight_min": -43.444271087646484,
      "activations/layer9_attention_weight_max": 46.62208938598633,
      "activations/layer9_attention_weight_min": -46.065670013427734,
      "epoch": 7.03,
      "learning_rate": 0.00010574356060606059,
      "loss": 2.7975,
      "step": 120950
    },
    {
      "activations/layer0_attention_weight_max": 15.705588340759277,
      "activations/layer0_attention_weight_min": -14.901782035827637,
      "activations/layer10_attention_weight_max": 38.84122848510742,
      "activations/layer10_attention_weight_min": -36.78802490234375,
      "activations/layer11_attention_weight_max": 32.190555572509766,
      "activations/layer11_attention_weight_min": -30.606388092041016,
      "activations/layer12_attention_weight_max": 22.491641998291016,
      "activations/layer12_attention_weight_min": -25.548858642578125,
      "activations/layer13_attention_weight_max": 43.29429244995117,
      "activations/layer13_attention_weight_min": -31.876163482666016,
      "activations/layer14_attention_weight_max": 38.355812072753906,
      "activations/layer14_attention_weight_min": -30.717559814453125,
      "activations/layer15_attention_weight_max": 36.20040512084961,
      "activations/layer15_attention_weight_min": -28.49857521057129,
      "activations/layer16_attention_weight_max": 32.57504653930664,
      "activations/layer16_attention_weight_min": -27.736982345581055,
      "activations/layer17_attention_weight_max": 57.05310821533203,
      "activations/layer17_attention_weight_min": -42.368770599365234,
      "activations/layer18_attention_weight_max": 47.319461822509766,
      "activations/layer18_attention_weight_min": -34.601497650146484,
      "activations/layer19_attention_weight_max": 21.154037475585938,
      "activations/layer19_attention_weight_min": -20.567461013793945,
      "activations/layer1_attention_weight_max": 16.90143394470215,
      "activations/layer1_attention_weight_min": -14.600578308105469,
      "activations/layer20_attention_weight_max": 22.057600021362305,
      "activations/layer20_attention_weight_min": -22.449785232543945,
      "activations/layer21_attention_weight_max": 41.136924743652344,
      "activations/layer21_attention_weight_min": -24.103313446044922,
      "activations/layer22_attention_weight_max": 33.5054817199707,
      "activations/layer22_attention_weight_min": -27.311134338378906,
      "activations/layer23_attention_weight_max": 34.88862609863281,
      "activations/layer23_attention_weight_min": -21.837665557861328,
      "activations/layer2_attention_weight_max": 32.35092544555664,
      "activations/layer2_attention_weight_min": -30.147144317626953,
      "activations/layer3_attention_weight_max": 91.1236343383789,
      "activations/layer3_attention_weight_min": -93.4784164428711,
      "activations/layer4_attention_weight_max": 103.62660217285156,
      "activations/layer4_attention_weight_min": -97.96672058105469,
      "activations/layer5_attention_weight_max": 95.32046508789062,
      "activations/layer5_attention_weight_min": -88.73204803466797,
      "activations/layer6_attention_weight_max": 62.18427276611328,
      "activations/layer6_attention_weight_min": -56.11173629760742,
      "activations/layer7_attention_weight_max": 75.88347625732422,
      "activations/layer7_attention_weight_min": -73.3816909790039,
      "activations/layer8_attention_weight_max": 45.810115814208984,
      "activations/layer8_attention_weight_min": -45.573944091796875,
      "activations/layer9_attention_weight_max": 55.84876251220703,
      "activations/layer9_attention_weight_min": -51.01226043701172,
      "epoch": 7.03,
      "learning_rate": 0.0001057246212121212,
      "loss": 2.809,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5737,
      "eval_samples_per_second": 500.831,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5737,
      "eval_openwebtext_samples_per_second": 500.831,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_wikitext_loss": 3.005859375,
      "eval_wikitext_ppl": 20.203571081597193,
      "eval_wikitext_runtime": 2.0174,
      "eval_wikitext_samples_per_second": 226.036,
      "step": 121000
    },
    {
      "epoch": 7.03,
      "eval_lambada_loss": 2.767578125,
      "eval_lambada_ppl": 15.920030957803608,
      "eval_lambada_runtime": 9.6206,
      "eval_lambada_samples_per_second": 506.101,
      "step": 121000
    },
    {
      "activations/layer0_attention_weight_max": 15.31359577178955,
      "activations/layer0_attention_weight_min": -14.827256202697754,
      "activations/layer10_attention_weight_max": 40.02018356323242,
      "activations/layer10_attention_weight_min": -37.75828552246094,
      "activations/layer11_attention_weight_max": 31.451576232910156,
      "activations/layer11_attention_weight_min": -31.64220428466797,
      "activations/layer12_attention_weight_max": 21.064085006713867,
      "activations/layer12_attention_weight_min": -24.9357967376709,
      "activations/layer13_attention_weight_max": 57.65156173706055,
      "activations/layer13_attention_weight_min": -34.716285705566406,
      "activations/layer14_attention_weight_max": 39.3597297668457,
      "activations/layer14_attention_weight_min": -32.253726959228516,
      "activations/layer15_attention_weight_max": 38.89635467529297,
      "activations/layer15_attention_weight_min": -29.262514114379883,
      "activations/layer16_attention_weight_max": 33.25448989868164,
      "activations/layer16_attention_weight_min": -27.30010414123535,
      "activations/layer17_attention_weight_max": 57.27167510986328,
      "activations/layer17_attention_weight_min": -42.56884002685547,
      "activations/layer18_attention_weight_max": 52.17605972290039,
      "activations/layer18_attention_weight_min": -34.90363693237305,
      "activations/layer19_attention_weight_max": 20.98627281188965,
      "activations/layer19_attention_weight_min": -19.9472713470459,
      "activations/layer1_attention_weight_max": 16.66654396057129,
      "activations/layer1_attention_weight_min": -15.386125564575195,
      "activations/layer20_attention_weight_max": 22.6832218170166,
      "activations/layer20_attention_weight_min": -23.68036460876465,
      "activations/layer21_attention_weight_max": 38.83101272583008,
      "activations/layer21_attention_weight_min": -23.069091796875,
      "activations/layer22_attention_weight_max": 32.550636291503906,
      "activations/layer22_attention_weight_min": -24.518972396850586,
      "activations/layer23_attention_weight_max": 34.17955780029297,
      "activations/layer23_attention_weight_min": -20.169212341308594,
      "activations/layer2_attention_weight_max": 30.334156036376953,
      "activations/layer2_attention_weight_min": -29.435775756835938,
      "activations/layer3_attention_weight_max": 83.43428039550781,
      "activations/layer3_attention_weight_min": -83.93558502197266,
      "activations/layer4_attention_weight_max": 97.61279296875,
      "activations/layer4_attention_weight_min": -91.60652923583984,
      "activations/layer5_attention_weight_max": 89.48548889160156,
      "activations/layer5_attention_weight_min": -81.9507827758789,
      "activations/layer6_attention_weight_max": 54.99335479736328,
      "activations/layer6_attention_weight_min": -51.13396072387695,
      "activations/layer7_attention_weight_max": 78.0972671508789,
      "activations/layer7_attention_weight_min": -74.70480346679688,
      "activations/layer8_attention_weight_max": 47.32124328613281,
      "activations/layer8_attention_weight_min": -44.42274856567383,
      "activations/layer9_attention_weight_max": 51.31843185424805,
      "activations/layer9_attention_weight_min": -47.74766540527344,
      "epoch": 7.03,
      "learning_rate": 0.00010570568181818181,
      "loss": 2.8016,
      "step": 121050
    },
    {
      "activations/layer0_attention_weight_max": 15.3262357711792,
      "activations/layer0_attention_weight_min": -14.870989799499512,
      "activations/layer10_attention_weight_max": 36.47564697265625,
      "activations/layer10_attention_weight_min": -37.64897537231445,
      "activations/layer11_attention_weight_max": 32.0673942565918,
      "activations/layer11_attention_weight_min": -30.003597259521484,
      "activations/layer12_attention_weight_max": 30.807401657104492,
      "activations/layer12_attention_weight_min": -24.772445678710938,
      "activations/layer13_attention_weight_max": 46.53505325317383,
      "activations/layer13_attention_weight_min": -30.99584197998047,
      "activations/layer14_attention_weight_max": 37.42837142944336,
      "activations/layer14_attention_weight_min": -30.81953239440918,
      "activations/layer15_attention_weight_max": 39.87548828125,
      "activations/layer15_attention_weight_min": -29.63238525390625,
      "activations/layer16_attention_weight_max": 32.98973083496094,
      "activations/layer16_attention_weight_min": -27.87896156311035,
      "activations/layer17_attention_weight_max": 54.898353576660156,
      "activations/layer17_attention_weight_min": -42.975494384765625,
      "activations/layer18_attention_weight_max": 47.88646697998047,
      "activations/layer18_attention_weight_min": -35.27499008178711,
      "activations/layer19_attention_weight_max": 21.964059829711914,
      "activations/layer19_attention_weight_min": -20.48038673400879,
      "activations/layer1_attention_weight_max": 16.647157669067383,
      "activations/layer1_attention_weight_min": -16.002243041992188,
      "activations/layer20_attention_weight_max": 24.61863899230957,
      "activations/layer20_attention_weight_min": -20.881059646606445,
      "activations/layer21_attention_weight_max": 40.74372482299805,
      "activations/layer21_attention_weight_min": -24.182205200195312,
      "activations/layer22_attention_weight_max": 36.49224853515625,
      "activations/layer22_attention_weight_min": -25.27248191833496,
      "activations/layer23_attention_weight_max": 34.05622100830078,
      "activations/layer23_attention_weight_min": -22.911865234375,
      "activations/layer2_attention_weight_max": 28.55455780029297,
      "activations/layer2_attention_weight_min": -28.716381072998047,
      "activations/layer3_attention_weight_max": 75.37071990966797,
      "activations/layer3_attention_weight_min": -81.13015747070312,
      "activations/layer4_attention_weight_max": 92.46379089355469,
      "activations/layer4_attention_weight_min": -88.36331939697266,
      "activations/layer5_attention_weight_max": 84.54794311523438,
      "activations/layer5_attention_weight_min": -82.20670318603516,
      "activations/layer6_attention_weight_max": 52.798377990722656,
      "activations/layer6_attention_weight_min": -50.35884094238281,
      "activations/layer7_attention_weight_max": 73.27730560302734,
      "activations/layer7_attention_weight_min": -70.6908950805664,
      "activations/layer8_attention_weight_max": 41.292945861816406,
      "activations/layer8_attention_weight_min": -43.959415435791016,
      "activations/layer9_attention_weight_max": 53.9771842956543,
      "activations/layer9_attention_weight_min": -47.688053131103516,
      "epoch": 7.04,
      "learning_rate": 0.00010568674242424242,
      "loss": 2.8237,
      "step": 121100
    },
    {
      "activations/layer0_attention_weight_max": 15.52401351928711,
      "activations/layer0_attention_weight_min": -15.262781143188477,
      "activations/layer10_attention_weight_max": 37.443519592285156,
      "activations/layer10_attention_weight_min": -35.285186767578125,
      "activations/layer11_attention_weight_max": 30.545108795166016,
      "activations/layer11_attention_weight_min": -31.950353622436523,
      "activations/layer12_attention_weight_max": 24.9517879486084,
      "activations/layer12_attention_weight_min": -31.077526092529297,
      "activations/layer13_attention_weight_max": 46.575565338134766,
      "activations/layer13_attention_weight_min": -30.446428298950195,
      "activations/layer14_attention_weight_max": 52.03630065917969,
      "activations/layer14_attention_weight_min": -30.543468475341797,
      "activations/layer15_attention_weight_max": 40.89594650268555,
      "activations/layer15_attention_weight_min": -30.141569137573242,
      "activations/layer16_attention_weight_max": 38.345863342285156,
      "activations/layer16_attention_weight_min": -26.17717742919922,
      "activations/layer17_attention_weight_max": 62.23702621459961,
      "activations/layer17_attention_weight_min": -42.97542190551758,
      "activations/layer18_attention_weight_max": 54.002769470214844,
      "activations/layer18_attention_weight_min": -36.57386016845703,
      "activations/layer19_attention_weight_max": 25.150672912597656,
      "activations/layer19_attention_weight_min": -21.130170822143555,
      "activations/layer1_attention_weight_max": 16.6925106048584,
      "activations/layer1_attention_weight_min": -15.717989921569824,
      "activations/layer20_attention_weight_max": 29.031675338745117,
      "activations/layer20_attention_weight_min": -24.209287643432617,
      "activations/layer21_attention_weight_max": 51.714080810546875,
      "activations/layer21_attention_weight_min": -27.9003963470459,
      "activations/layer22_attention_weight_max": 41.802764892578125,
      "activations/layer22_attention_weight_min": -25.97534942626953,
      "activations/layer23_attention_weight_max": 38.32051086425781,
      "activations/layer23_attention_weight_min": -24.396047592163086,
      "activations/layer2_attention_weight_max": 30.83684539794922,
      "activations/layer2_attention_weight_min": -27.943592071533203,
      "activations/layer3_attention_weight_max": 81.20633697509766,
      "activations/layer3_attention_weight_min": -84.19140625,
      "activations/layer4_attention_weight_max": 95.32706451416016,
      "activations/layer4_attention_weight_min": -93.56365203857422,
      "activations/layer5_attention_weight_max": 86.89458465576172,
      "activations/layer5_attention_weight_min": -82.64722442626953,
      "activations/layer6_attention_weight_max": 53.421897888183594,
      "activations/layer6_attention_weight_min": -51.11500930786133,
      "activations/layer7_attention_weight_max": 72.08035278320312,
      "activations/layer7_attention_weight_min": -67.17072296142578,
      "activations/layer8_attention_weight_max": 42.054931640625,
      "activations/layer8_attention_weight_min": -42.34898376464844,
      "activations/layer9_attention_weight_max": 47.201629638671875,
      "activations/layer9_attention_weight_min": -47.275997161865234,
      "epoch": 7.04,
      "learning_rate": 0.00010566780303030302,
      "loss": 2.8102,
      "step": 121150
    },
    {
      "activations/layer0_attention_weight_max": 14.874908447265625,
      "activations/layer0_attention_weight_min": -15.1217041015625,
      "activations/layer10_attention_weight_max": 37.12616729736328,
      "activations/layer10_attention_weight_min": -36.067989349365234,
      "activations/layer11_attention_weight_max": 31.353683471679688,
      "activations/layer11_attention_weight_min": -29.86615753173828,
      "activations/layer12_attention_weight_max": 17.982837677001953,
      "activations/layer12_attention_weight_min": -26.556842803955078,
      "activations/layer13_attention_weight_max": 45.62314987182617,
      "activations/layer13_attention_weight_min": -35.08051300048828,
      "activations/layer14_attention_weight_max": 46.048439025878906,
      "activations/layer14_attention_weight_min": -30.042543411254883,
      "activations/layer15_attention_weight_max": 35.866641998291016,
      "activations/layer15_attention_weight_min": -28.187307357788086,
      "activations/layer16_attention_weight_max": 35.21537780761719,
      "activations/layer16_attention_weight_min": -28.827062606811523,
      "activations/layer17_attention_weight_max": 52.29052734375,
      "activations/layer17_attention_weight_min": -40.60390853881836,
      "activations/layer18_attention_weight_max": 43.27808380126953,
      "activations/layer18_attention_weight_min": -34.84974670410156,
      "activations/layer19_attention_weight_max": 21.660480499267578,
      "activations/layer19_attention_weight_min": -20.501371383666992,
      "activations/layer1_attention_weight_max": 16.88503074645996,
      "activations/layer1_attention_weight_min": -14.82181453704834,
      "activations/layer20_attention_weight_max": 21.66977882385254,
      "activations/layer20_attention_weight_min": -22.592815399169922,
      "activations/layer21_attention_weight_max": 38.219913482666016,
      "activations/layer21_attention_weight_min": -26.93793296813965,
      "activations/layer22_attention_weight_max": 32.828147888183594,
      "activations/layer22_attention_weight_min": -27.843292236328125,
      "activations/layer23_attention_weight_max": 32.7332649230957,
      "activations/layer23_attention_weight_min": -19.350494384765625,
      "activations/layer2_attention_weight_max": 30.082563400268555,
      "activations/layer2_attention_weight_min": -28.227645874023438,
      "activations/layer3_attention_weight_max": 82.12104034423828,
      "activations/layer3_attention_weight_min": -85.88912200927734,
      "activations/layer4_attention_weight_max": 95.72030639648438,
      "activations/layer4_attention_weight_min": -92.88440704345703,
      "activations/layer5_attention_weight_max": 88.72783660888672,
      "activations/layer5_attention_weight_min": -86.6904296875,
      "activations/layer6_attention_weight_max": 54.53779983520508,
      "activations/layer6_attention_weight_min": -53.379234313964844,
      "activations/layer7_attention_weight_max": 74.82830810546875,
      "activations/layer7_attention_weight_min": -72.3262939453125,
      "activations/layer8_attention_weight_max": 45.03308868408203,
      "activations/layer8_attention_weight_min": -45.38410949707031,
      "activations/layer9_attention_weight_max": 54.6298828125,
      "activations/layer9_attention_weight_min": -49.40920639038086,
      "epoch": 7.04,
      "learning_rate": 0.00010564886363636363,
      "loss": 2.8212,
      "step": 121200
    },
    {
      "activations/layer0_attention_weight_max": 16.092910766601562,
      "activations/layer0_attention_weight_min": -15.206538200378418,
      "activations/layer10_attention_weight_max": 36.80887985229492,
      "activations/layer10_attention_weight_min": -33.003326416015625,
      "activations/layer11_attention_weight_max": 29.214435577392578,
      "activations/layer11_attention_weight_min": -29.240787506103516,
      "activations/layer12_attention_weight_max": 16.830923080444336,
      "activations/layer12_attention_weight_min": -28.514352798461914,
      "activations/layer13_attention_weight_max": 42.20168685913086,
      "activations/layer13_attention_weight_min": -32.065277099609375,
      "activations/layer14_attention_weight_max": 52.70120620727539,
      "activations/layer14_attention_weight_min": -34.460365295410156,
      "activations/layer15_attention_weight_max": 42.072288513183594,
      "activations/layer15_attention_weight_min": -30.280702590942383,
      "activations/layer16_attention_weight_max": 28.485944747924805,
      "activations/layer16_attention_weight_min": -27.703088760375977,
      "activations/layer17_attention_weight_max": 50.909088134765625,
      "activations/layer17_attention_weight_min": -43.71961212158203,
      "activations/layer18_attention_weight_max": 42.32570266723633,
      "activations/layer18_attention_weight_min": -34.0941276550293,
      "activations/layer19_attention_weight_max": 20.817434310913086,
      "activations/layer19_attention_weight_min": -21.163572311401367,
      "activations/layer1_attention_weight_max": 16.837726593017578,
      "activations/layer1_attention_weight_min": -14.067011833190918,
      "activations/layer20_attention_weight_max": 22.821640014648438,
      "activations/layer20_attention_weight_min": -21.790781021118164,
      "activations/layer21_attention_weight_max": 40.363929748535156,
      "activations/layer21_attention_weight_min": -23.717302322387695,
      "activations/layer22_attention_weight_max": 32.63434600830078,
      "activations/layer22_attention_weight_min": -24.88129234313965,
      "activations/layer23_attention_weight_max": 32.04267501831055,
      "activations/layer23_attention_weight_min": -21.00436782836914,
      "activations/layer2_attention_weight_max": 32.32710266113281,
      "activations/layer2_attention_weight_min": -29.01728057861328,
      "activations/layer3_attention_weight_max": 80.95660400390625,
      "activations/layer3_attention_weight_min": -80.85362243652344,
      "activations/layer4_attention_weight_max": 93.14239501953125,
      "activations/layer4_attention_weight_min": -86.3843994140625,
      "activations/layer5_attention_weight_max": 87.49047088623047,
      "activations/layer5_attention_weight_min": -78.31409454345703,
      "activations/layer6_attention_weight_max": 56.02020263671875,
      "activations/layer6_attention_weight_min": -48.520469665527344,
      "activations/layer7_attention_weight_max": 74.55560302734375,
      "activations/layer7_attention_weight_min": -68.0224380493164,
      "activations/layer8_attention_weight_max": 43.722904205322266,
      "activations/layer8_attention_weight_min": -42.45700454711914,
      "activations/layer9_attention_weight_max": 48.59458541870117,
      "activations/layer9_attention_weight_min": -46.00613784790039,
      "epoch": 7.05,
      "learning_rate": 0.00010562992424242424,
      "loss": 2.8282,
      "step": 121250
    },
    {
      "activations/layer0_attention_weight_max": 14.650347709655762,
      "activations/layer0_attention_weight_min": -15.518706321716309,
      "activations/layer10_attention_weight_max": 37.93260955810547,
      "activations/layer10_attention_weight_min": -33.48405456542969,
      "activations/layer11_attention_weight_max": 29.95250701904297,
      "activations/layer11_attention_weight_min": -29.215877532958984,
      "activations/layer12_attention_weight_max": 21.98422622680664,
      "activations/layer12_attention_weight_min": -23.472375869750977,
      "activations/layer13_attention_weight_max": 51.480308532714844,
      "activations/layer13_attention_weight_min": -38.04763412475586,
      "activations/layer14_attention_weight_max": 36.87868881225586,
      "activations/layer14_attention_weight_min": -27.91644859313965,
      "activations/layer15_attention_weight_max": 39.238590240478516,
      "activations/layer15_attention_weight_min": -27.862442016601562,
      "activations/layer16_attention_weight_max": 33.00772476196289,
      "activations/layer16_attention_weight_min": -25.566320419311523,
      "activations/layer17_attention_weight_max": 57.15312957763672,
      "activations/layer17_attention_weight_min": -40.343143463134766,
      "activations/layer18_attention_weight_max": 50.50338363647461,
      "activations/layer18_attention_weight_min": -34.99291229248047,
      "activations/layer19_attention_weight_max": 24.77830696105957,
      "activations/layer19_attention_weight_min": -20.593284606933594,
      "activations/layer1_attention_weight_max": 16.710851669311523,
      "activations/layer1_attention_weight_min": -17.483768463134766,
      "activations/layer20_attention_weight_max": 22.547780990600586,
      "activations/layer20_attention_weight_min": -22.95478057861328,
      "activations/layer21_attention_weight_max": 36.7898063659668,
      "activations/layer21_attention_weight_min": -23.68035316467285,
      "activations/layer22_attention_weight_max": 31.390071868896484,
      "activations/layer22_attention_weight_min": -23.1044979095459,
      "activations/layer23_attention_weight_max": 32.55534362792969,
      "activations/layer23_attention_weight_min": -19.435319900512695,
      "activations/layer2_attention_weight_max": 32.44073486328125,
      "activations/layer2_attention_weight_min": -30.598899841308594,
      "activations/layer3_attention_weight_max": 85.59689331054688,
      "activations/layer3_attention_weight_min": -90.36995697021484,
      "activations/layer4_attention_weight_max": 97.9518814086914,
      "activations/layer4_attention_weight_min": -93.56755828857422,
      "activations/layer5_attention_weight_max": 89.49996185302734,
      "activations/layer5_attention_weight_min": -83.93804931640625,
      "activations/layer6_attention_weight_max": 56.304481506347656,
      "activations/layer6_attention_weight_min": -52.97352600097656,
      "activations/layer7_attention_weight_max": 76.30949401855469,
      "activations/layer7_attention_weight_min": -70.26790618896484,
      "activations/layer8_attention_weight_max": 44.55984115600586,
      "activations/layer8_attention_weight_min": -46.40248107910156,
      "activations/layer9_attention_weight_max": 49.842098236083984,
      "activations/layer9_attention_weight_min": -45.70658874511719,
      "epoch": 7.05,
      "learning_rate": 0.00010561136363636363,
      "loss": 2.8207,
      "step": 121300
    },
    {
      "activations/layer0_attention_weight_max": 14.38223648071289,
      "activations/layer0_attention_weight_min": -15.122502326965332,
      "activations/layer10_attention_weight_max": 40.2894401550293,
      "activations/layer10_attention_weight_min": -38.81766891479492,
      "activations/layer11_attention_weight_max": 30.515243530273438,
      "activations/layer11_attention_weight_min": -30.87984275817871,
      "activations/layer12_attention_weight_max": 21.028827667236328,
      "activations/layer12_attention_weight_min": -26.672603607177734,
      "activations/layer13_attention_weight_max": 46.0778694152832,
      "activations/layer13_attention_weight_min": -31.93553352355957,
      "activations/layer14_attention_weight_max": 37.375247955322266,
      "activations/layer14_attention_weight_min": -30.92029571533203,
      "activations/layer15_attention_weight_max": 35.66863250732422,
      "activations/layer15_attention_weight_min": -31.39118003845215,
      "activations/layer16_attention_weight_max": 36.24656295776367,
      "activations/layer16_attention_weight_min": -27.380558013916016,
      "activations/layer17_attention_weight_max": 57.09952926635742,
      "activations/layer17_attention_weight_min": -40.310081481933594,
      "activations/layer18_attention_weight_max": 47.57697677612305,
      "activations/layer18_attention_weight_min": -35.16804122924805,
      "activations/layer19_attention_weight_max": 20.39986228942871,
      "activations/layer19_attention_weight_min": -18.8765869140625,
      "activations/layer1_attention_weight_max": 17.17643165588379,
      "activations/layer1_attention_weight_min": -14.921250343322754,
      "activations/layer20_attention_weight_max": 21.707456588745117,
      "activations/layer20_attention_weight_min": -21.871843338012695,
      "activations/layer21_attention_weight_max": 46.01019287109375,
      "activations/layer21_attention_weight_min": -24.501079559326172,
      "activations/layer22_attention_weight_max": 35.23195266723633,
      "activations/layer22_attention_weight_min": -25.94681739807129,
      "activations/layer23_attention_weight_max": 34.82286071777344,
      "activations/layer23_attention_weight_min": -22.013111114501953,
      "activations/layer2_attention_weight_max": 28.934368133544922,
      "activations/layer2_attention_weight_min": -28.575763702392578,
      "activations/layer3_attention_weight_max": 83.66985321044922,
      "activations/layer3_attention_weight_min": -82.60780334472656,
      "activations/layer4_attention_weight_max": 97.74007415771484,
      "activations/layer4_attention_weight_min": -96.1300277709961,
      "activations/layer5_attention_weight_max": 88.7622299194336,
      "activations/layer5_attention_weight_min": -82.47762298583984,
      "activations/layer6_attention_weight_max": 55.901519775390625,
      "activations/layer6_attention_weight_min": -52.7946662902832,
      "activations/layer7_attention_weight_max": 76.51685333251953,
      "activations/layer7_attention_weight_min": -70.50624084472656,
      "activations/layer8_attention_weight_max": 43.0767707824707,
      "activations/layer8_attention_weight_min": -44.586212158203125,
      "activations/layer9_attention_weight_max": 51.51667404174805,
      "activations/layer9_attention_weight_min": -53.838356018066406,
      "epoch": 7.05,
      "learning_rate": 0.00010559280303030302,
      "loss": 2.8254,
      "step": 121350
    },
    {
      "activations/layer0_attention_weight_max": 15.11495590209961,
      "activations/layer0_attention_weight_min": -15.133163452148438,
      "activations/layer10_attention_weight_max": 36.36280059814453,
      "activations/layer10_attention_weight_min": -32.89508819580078,
      "activations/layer11_attention_weight_max": 30.104839324951172,
      "activations/layer11_attention_weight_min": -27.658313751220703,
      "activations/layer12_attention_weight_max": 21.346961975097656,
      "activations/layer12_attention_weight_min": -24.383346557617188,
      "activations/layer13_attention_weight_max": 40.552059173583984,
      "activations/layer13_attention_weight_min": -32.715518951416016,
      "activations/layer14_attention_weight_max": 44.28536605834961,
      "activations/layer14_attention_weight_min": -31.045631408691406,
      "activations/layer15_attention_weight_max": 36.67853546142578,
      "activations/layer15_attention_weight_min": -28.328699111938477,
      "activations/layer16_attention_weight_max": 33.39471435546875,
      "activations/layer16_attention_weight_min": -25.845800399780273,
      "activations/layer17_attention_weight_max": 52.39400863647461,
      "activations/layer17_attention_weight_min": -39.863136291503906,
      "activations/layer18_attention_weight_max": 49.06755065917969,
      "activations/layer18_attention_weight_min": -34.705291748046875,
      "activations/layer19_attention_weight_max": 24.32736587524414,
      "activations/layer19_attention_weight_min": -19.64644432067871,
      "activations/layer1_attention_weight_max": 17.4461612701416,
      "activations/layer1_attention_weight_min": -15.36983585357666,
      "activations/layer20_attention_weight_max": 23.16623878479004,
      "activations/layer20_attention_weight_min": -22.5236873626709,
      "activations/layer21_attention_weight_max": 37.94462585449219,
      "activations/layer21_attention_weight_min": -23.926042556762695,
      "activations/layer22_attention_weight_max": 31.94740104675293,
      "activations/layer22_attention_weight_min": -27.010116577148438,
      "activations/layer23_attention_weight_max": 33.344078063964844,
      "activations/layer23_attention_weight_min": -22.003192901611328,
      "activations/layer2_attention_weight_max": 28.990989685058594,
      "activations/layer2_attention_weight_min": -27.933425903320312,
      "activations/layer3_attention_weight_max": 78.94923400878906,
      "activations/layer3_attention_weight_min": -81.5594482421875,
      "activations/layer4_attention_weight_max": 99.8652572631836,
      "activations/layer4_attention_weight_min": -93.06807708740234,
      "activations/layer5_attention_weight_max": 91.08492279052734,
      "activations/layer5_attention_weight_min": -81.16899108886719,
      "activations/layer6_attention_weight_max": 55.29340744018555,
      "activations/layer6_attention_weight_min": -51.81178665161133,
      "activations/layer7_attention_weight_max": 75.2607421875,
      "activations/layer7_attention_weight_min": -70.44789123535156,
      "activations/layer8_attention_weight_max": 41.41121292114258,
      "activations/layer8_attention_weight_min": -41.43000411987305,
      "activations/layer9_attention_weight_max": 47.1836051940918,
      "activations/layer9_attention_weight_min": -46.30194091796875,
      "epoch": 7.05,
      "learning_rate": 0.00010557386363636364,
      "loss": 2.8216,
      "step": 121400
    },
    {
      "activations/layer0_attention_weight_max": 16.07452964782715,
      "activations/layer0_attention_weight_min": -14.822976112365723,
      "activations/layer10_attention_weight_max": 37.84807586669922,
      "activations/layer10_attention_weight_min": -34.7436408996582,
      "activations/layer11_attention_weight_max": 30.719982147216797,
      "activations/layer11_attention_weight_min": -29.769691467285156,
      "activations/layer12_attention_weight_max": 21.30557632446289,
      "activations/layer12_attention_weight_min": -23.64785385131836,
      "activations/layer13_attention_weight_max": 53.74590301513672,
      "activations/layer13_attention_weight_min": -33.513648986816406,
      "activations/layer14_attention_weight_max": 45.253082275390625,
      "activations/layer14_attention_weight_min": -30.941875457763672,
      "activations/layer15_attention_weight_max": 38.82920837402344,
      "activations/layer15_attention_weight_min": -30.1197566986084,
      "activations/layer16_attention_weight_max": 35.129451751708984,
      "activations/layer16_attention_weight_min": -26.17087745666504,
      "activations/layer17_attention_weight_max": 59.76417922973633,
      "activations/layer17_attention_weight_min": -41.90341567993164,
      "activations/layer18_attention_weight_max": 55.6002197265625,
      "activations/layer18_attention_weight_min": -34.86934280395508,
      "activations/layer19_attention_weight_max": 20.942909240722656,
      "activations/layer19_attention_weight_min": -21.06772804260254,
      "activations/layer1_attention_weight_max": 16.271575927734375,
      "activations/layer1_attention_weight_min": -14.846224784851074,
      "activations/layer20_attention_weight_max": 27.04869842529297,
      "activations/layer20_attention_weight_min": -23.69808006286621,
      "activations/layer21_attention_weight_max": 43.328369140625,
      "activations/layer21_attention_weight_min": -25.792259216308594,
      "activations/layer22_attention_weight_max": 34.84831619262695,
      "activations/layer22_attention_weight_min": -23.353601455688477,
      "activations/layer23_attention_weight_max": 37.602378845214844,
      "activations/layer23_attention_weight_min": -21.900390625,
      "activations/layer2_attention_weight_max": 30.118764877319336,
      "activations/layer2_attention_weight_min": -27.721830368041992,
      "activations/layer3_attention_weight_max": 83.82896423339844,
      "activations/layer3_attention_weight_min": -83.96412658691406,
      "activations/layer4_attention_weight_max": 97.07958221435547,
      "activations/layer4_attention_weight_min": -90.46810150146484,
      "activations/layer5_attention_weight_max": 89.79720306396484,
      "activations/layer5_attention_weight_min": -83.8438949584961,
      "activations/layer6_attention_weight_max": 54.89330291748047,
      "activations/layer6_attention_weight_min": -53.725711822509766,
      "activations/layer7_attention_weight_max": 75.60585021972656,
      "activations/layer7_attention_weight_min": -69.9248046875,
      "activations/layer8_attention_weight_max": 44.9293212890625,
      "activations/layer8_attention_weight_min": -46.456153869628906,
      "activations/layer9_attention_weight_max": 51.89044952392578,
      "activations/layer9_attention_weight_min": -48.63943862915039,
      "epoch": 7.06,
      "learning_rate": 0.00010555492424242422,
      "loss": 2.8095,
      "step": 121450
    },
    {
      "activations/layer0_attention_weight_max": 15.697200775146484,
      "activations/layer0_attention_weight_min": -14.783796310424805,
      "activations/layer10_attention_weight_max": 38.90637969970703,
      "activations/layer10_attention_weight_min": -35.70401382446289,
      "activations/layer11_attention_weight_max": 31.1471004486084,
      "activations/layer11_attention_weight_min": -30.258512496948242,
      "activations/layer12_attention_weight_max": 20.251564025878906,
      "activations/layer12_attention_weight_min": -30.40475845336914,
      "activations/layer13_attention_weight_max": 59.669071197509766,
      "activations/layer13_attention_weight_min": -35.36498260498047,
      "activations/layer14_attention_weight_max": 47.16312789916992,
      "activations/layer14_attention_weight_min": -30.70815658569336,
      "activations/layer15_attention_weight_max": 41.80411911010742,
      "activations/layer15_attention_weight_min": -30.041465759277344,
      "activations/layer16_attention_weight_max": 31.78455352783203,
      "activations/layer16_attention_weight_min": -26.15031623840332,
      "activations/layer17_attention_weight_max": 56.50221252441406,
      "activations/layer17_attention_weight_min": -43.98848342895508,
      "activations/layer18_attention_weight_max": 48.03534698486328,
      "activations/layer18_attention_weight_min": -37.805885314941406,
      "activations/layer19_attention_weight_max": 21.34332847595215,
      "activations/layer19_attention_weight_min": -21.384546279907227,
      "activations/layer1_attention_weight_max": 17.237598419189453,
      "activations/layer1_attention_weight_min": -14.85719108581543,
      "activations/layer20_attention_weight_max": 21.67494010925293,
      "activations/layer20_attention_weight_min": -23.599855422973633,
      "activations/layer21_attention_weight_max": 37.40834045410156,
      "activations/layer21_attention_weight_min": -23.045766830444336,
      "activations/layer22_attention_weight_max": 32.41801071166992,
      "activations/layer22_attention_weight_min": -25.4233341217041,
      "activations/layer23_attention_weight_max": 39.59248733520508,
      "activations/layer23_attention_weight_min": -21.49793243408203,
      "activations/layer2_attention_weight_max": 32.1253776550293,
      "activations/layer2_attention_weight_min": -28.725584030151367,
      "activations/layer3_attention_weight_max": 80.86409759521484,
      "activations/layer3_attention_weight_min": -88.79249572753906,
      "activations/layer4_attention_weight_max": 93.18644714355469,
      "activations/layer4_attention_weight_min": -90.66390228271484,
      "activations/layer5_attention_weight_max": 87.5002670288086,
      "activations/layer5_attention_weight_min": -82.04766845703125,
      "activations/layer6_attention_weight_max": 53.785221099853516,
      "activations/layer6_attention_weight_min": -53.025306701660156,
      "activations/layer7_attention_weight_max": 72.7285385131836,
      "activations/layer7_attention_weight_min": -74.83133697509766,
      "activations/layer8_attention_weight_max": 43.743656158447266,
      "activations/layer8_attention_weight_min": -45.953426361083984,
      "activations/layer9_attention_weight_max": 50.48361587524414,
      "activations/layer9_attention_weight_min": -48.455204010009766,
      "epoch": 7.06,
      "learning_rate": 0.00010553598484848484,
      "loss": 2.8099,
      "step": 121500
    },
    {
      "activations/layer0_attention_weight_max": 14.769732475280762,
      "activations/layer0_attention_weight_min": -15.100747108459473,
      "activations/layer10_attention_weight_max": 39.20535659790039,
      "activations/layer10_attention_weight_min": -34.66475296020508,
      "activations/layer11_attention_weight_max": 30.742734909057617,
      "activations/layer11_attention_weight_min": -28.837387084960938,
      "activations/layer12_attention_weight_max": 25.125648498535156,
      "activations/layer12_attention_weight_min": -25.879472732543945,
      "activations/layer13_attention_weight_max": 44.30426788330078,
      "activations/layer13_attention_weight_min": -34.78824996948242,
      "activations/layer14_attention_weight_max": 41.6806640625,
      "activations/layer14_attention_weight_min": -30.282651901245117,
      "activations/layer15_attention_weight_max": 38.98037338256836,
      "activations/layer15_attention_weight_min": -28.99721336364746,
      "activations/layer16_attention_weight_max": 32.97561264038086,
      "activations/layer16_attention_weight_min": -29.24001693725586,
      "activations/layer17_attention_weight_max": 54.92633056640625,
      "activations/layer17_attention_weight_min": -39.24641418457031,
      "activations/layer18_attention_weight_max": 48.03803253173828,
      "activations/layer18_attention_weight_min": -35.459842681884766,
      "activations/layer19_attention_weight_max": 23.426876068115234,
      "activations/layer19_attention_weight_min": -22.072690963745117,
      "activations/layer1_attention_weight_max": 16.64222526550293,
      "activations/layer1_attention_weight_min": -16.389801025390625,
      "activations/layer20_attention_weight_max": 21.429630279541016,
      "activations/layer20_attention_weight_min": -26.862701416015625,
      "activations/layer21_attention_weight_max": 37.653297424316406,
      "activations/layer21_attention_weight_min": -23.966262817382812,
      "activations/layer22_attention_weight_max": 33.61956024169922,
      "activations/layer22_attention_weight_min": -25.016443252563477,
      "activations/layer23_attention_weight_max": 37.10149383544922,
      "activations/layer23_attention_weight_min": -21.026771545410156,
      "activations/layer2_attention_weight_max": 32.14732360839844,
      "activations/layer2_attention_weight_min": -28.621898651123047,
      "activations/layer3_attention_weight_max": 81.91719818115234,
      "activations/layer3_attention_weight_min": -85.35218811035156,
      "activations/layer4_attention_weight_max": 95.6900634765625,
      "activations/layer4_attention_weight_min": -89.36878967285156,
      "activations/layer5_attention_weight_max": 87.6881332397461,
      "activations/layer5_attention_weight_min": -77.83137512207031,
      "activations/layer6_attention_weight_max": 53.293880462646484,
      "activations/layer6_attention_weight_min": -52.80470275878906,
      "activations/layer7_attention_weight_max": 74.69100189208984,
      "activations/layer7_attention_weight_min": -68.53350067138672,
      "activations/layer8_attention_weight_max": 44.662784576416016,
      "activations/layer8_attention_weight_min": -47.002933502197266,
      "activations/layer9_attention_weight_max": 51.91719436645508,
      "activations/layer9_attention_weight_min": -45.55613327026367,
      "epoch": 7.06,
      "learning_rate": 0.00010551704545454545,
      "loss": 2.8237,
      "step": 121550
    },
    {
      "activations/layer0_attention_weight_max": 14.331149101257324,
      "activations/layer0_attention_weight_min": -14.71835994720459,
      "activations/layer10_attention_weight_max": 36.196929931640625,
      "activations/layer10_attention_weight_min": -33.693153381347656,
      "activations/layer11_attention_weight_max": 29.92586898803711,
      "activations/layer11_attention_weight_min": -28.824787139892578,
      "activations/layer12_attention_weight_max": 21.63593292236328,
      "activations/layer12_attention_weight_min": -25.320348739624023,
      "activations/layer13_attention_weight_max": 55.93433380126953,
      "activations/layer13_attention_weight_min": -41.3853645324707,
      "activations/layer14_attention_weight_max": 43.430118560791016,
      "activations/layer14_attention_weight_min": -31.84970474243164,
      "activations/layer15_attention_weight_max": 39.361427307128906,
      "activations/layer15_attention_weight_min": -30.54686164855957,
      "activations/layer16_attention_weight_max": 33.82667541503906,
      "activations/layer16_attention_weight_min": -28.22391128540039,
      "activations/layer17_attention_weight_max": 56.12327194213867,
      "activations/layer17_attention_weight_min": -43.57148742675781,
      "activations/layer18_attention_weight_max": 55.18422317504883,
      "activations/layer18_attention_weight_min": -37.305023193359375,
      "activations/layer19_attention_weight_max": 23.485977172851562,
      "activations/layer19_attention_weight_min": -21.356325149536133,
      "activations/layer1_attention_weight_max": 16.39058494567871,
      "activations/layer1_attention_weight_min": -14.323517799377441,
      "activations/layer20_attention_weight_max": 25.244056701660156,
      "activations/layer20_attention_weight_min": -23.64031410217285,
      "activations/layer21_attention_weight_max": 42.10785675048828,
      "activations/layer21_attention_weight_min": -25.980880737304688,
      "activations/layer22_attention_weight_max": 36.336021423339844,
      "activations/layer22_attention_weight_min": -26.83284568786621,
      "activations/layer23_attention_weight_max": 39.30073165893555,
      "activations/layer23_attention_weight_min": -22.222036361694336,
      "activations/layer2_attention_weight_max": 28.840551376342773,
      "activations/layer2_attention_weight_min": -28.22688865661621,
      "activations/layer3_attention_weight_max": 82.69847869873047,
      "activations/layer3_attention_weight_min": -87.42750549316406,
      "activations/layer4_attention_weight_max": 101.4259262084961,
      "activations/layer4_attention_weight_min": -93.15471649169922,
      "activations/layer5_attention_weight_max": 91.11163330078125,
      "activations/layer5_attention_weight_min": -83.34474182128906,
      "activations/layer6_attention_weight_max": 55.67869186401367,
      "activations/layer6_attention_weight_min": -51.10564041137695,
      "activations/layer7_attention_weight_max": 73.35459899902344,
      "activations/layer7_attention_weight_min": -70.20265197753906,
      "activations/layer8_attention_weight_max": 43.84934616088867,
      "activations/layer8_attention_weight_min": -42.78010177612305,
      "activations/layer9_attention_weight_max": 47.68503952026367,
      "activations/layer9_attention_weight_min": -44.48871994018555,
      "epoch": 7.07,
      "learning_rate": 0.00010549810606060604,
      "loss": 2.8557,
      "step": 121600
    },
    {
      "activations/layer0_attention_weight_max": 14.92015266418457,
      "activations/layer0_attention_weight_min": -14.769970893859863,
      "activations/layer10_attention_weight_max": 37.75896453857422,
      "activations/layer10_attention_weight_min": -37.411590576171875,
      "activations/layer11_attention_weight_max": 30.063932418823242,
      "activations/layer11_attention_weight_min": -30.54123306274414,
      "activations/layer12_attention_weight_max": 21.369144439697266,
      "activations/layer12_attention_weight_min": -24.61080551147461,
      "activations/layer13_attention_weight_max": 43.421485900878906,
      "activations/layer13_attention_weight_min": -32.315093994140625,
      "activations/layer14_attention_weight_max": 39.02659606933594,
      "activations/layer14_attention_weight_min": -31.570907592773438,
      "activations/layer15_attention_weight_max": 36.23280715942383,
      "activations/layer15_attention_weight_min": -33.02178192138672,
      "activations/layer16_attention_weight_max": 32.31058120727539,
      "activations/layer16_attention_weight_min": -28.777862548828125,
      "activations/layer17_attention_weight_max": 56.92598342895508,
      "activations/layer17_attention_weight_min": -45.25849533081055,
      "activations/layer18_attention_weight_max": 46.73241424560547,
      "activations/layer18_attention_weight_min": -37.38987731933594,
      "activations/layer19_attention_weight_max": 22.686248779296875,
      "activations/layer19_attention_weight_min": -22.27213478088379,
      "activations/layer1_attention_weight_max": 16.958101272583008,
      "activations/layer1_attention_weight_min": -16.623991012573242,
      "activations/layer20_attention_weight_max": 24.185428619384766,
      "activations/layer20_attention_weight_min": -23.98822593688965,
      "activations/layer21_attention_weight_max": 40.662940979003906,
      "activations/layer21_attention_weight_min": -24.939537048339844,
      "activations/layer22_attention_weight_max": 30.20515251159668,
      "activations/layer22_attention_weight_min": -25.574514389038086,
      "activations/layer23_attention_weight_max": 36.19021224975586,
      "activations/layer23_attention_weight_min": -21.019250869750977,
      "activations/layer2_attention_weight_max": 32.52424621582031,
      "activations/layer2_attention_weight_min": -30.97869110107422,
      "activations/layer3_attention_weight_max": 83.4670639038086,
      "activations/layer3_attention_weight_min": -86.90787506103516,
      "activations/layer4_attention_weight_max": 98.80310821533203,
      "activations/layer4_attention_weight_min": -93.8432846069336,
      "activations/layer5_attention_weight_max": 93.40794372558594,
      "activations/layer5_attention_weight_min": -86.16021728515625,
      "activations/layer6_attention_weight_max": 58.10690689086914,
      "activations/layer6_attention_weight_min": -53.79755783081055,
      "activations/layer7_attention_weight_max": 77.96085357666016,
      "activations/layer7_attention_weight_min": -76.30201721191406,
      "activations/layer8_attention_weight_max": 47.70513916015625,
      "activations/layer8_attention_weight_min": -51.31986618041992,
      "activations/layer9_attention_weight_max": 55.37908935546875,
      "activations/layer9_attention_weight_min": -50.38701248168945,
      "epoch": 7.07,
      "learning_rate": 0.00010547916666666666,
      "loss": 2.8084,
      "step": 121650
    },
    {
      "activations/layer0_attention_weight_max": 15.50965690612793,
      "activations/layer0_attention_weight_min": -14.531322479248047,
      "activations/layer10_attention_weight_max": 35.05242919921875,
      "activations/layer10_attention_weight_min": -35.204261779785156,
      "activations/layer11_attention_weight_max": 30.07417869567871,
      "activations/layer11_attention_weight_min": -28.98446273803711,
      "activations/layer12_attention_weight_max": 17.851577758789062,
      "activations/layer12_attention_weight_min": -30.13551139831543,
      "activations/layer13_attention_weight_max": 39.21171569824219,
      "activations/layer13_attention_weight_min": -32.30937576293945,
      "activations/layer14_attention_weight_max": 36.35237121582031,
      "activations/layer14_attention_weight_min": -31.325790405273438,
      "activations/layer15_attention_weight_max": 33.237335205078125,
      "activations/layer15_attention_weight_min": -29.89535903930664,
      "activations/layer16_attention_weight_max": 30.93162727355957,
      "activations/layer16_attention_weight_min": -26.049345016479492,
      "activations/layer17_attention_weight_max": 51.731021881103516,
      "activations/layer17_attention_weight_min": -42.22378158569336,
      "activations/layer18_attention_weight_max": 43.94935989379883,
      "activations/layer18_attention_weight_min": -34.99584197998047,
      "activations/layer19_attention_weight_max": 23.7170467376709,
      "activations/layer19_attention_weight_min": -19.584260940551758,
      "activations/layer1_attention_weight_max": 16.764768600463867,
      "activations/layer1_attention_weight_min": -13.736746788024902,
      "activations/layer20_attention_weight_max": 23.92824935913086,
      "activations/layer20_attention_weight_min": -21.120800018310547,
      "activations/layer21_attention_weight_max": 32.60805130004883,
      "activations/layer21_attention_weight_min": -23.200632095336914,
      "activations/layer22_attention_weight_max": 33.06190490722656,
      "activations/layer22_attention_weight_min": -27.58417320251465,
      "activations/layer23_attention_weight_max": 33.613826751708984,
      "activations/layer23_attention_weight_min": -19.290618896484375,
      "activations/layer2_attention_weight_max": 32.27851867675781,
      "activations/layer2_attention_weight_min": -30.263259887695312,
      "activations/layer3_attention_weight_max": 88.8753433227539,
      "activations/layer3_attention_weight_min": -88.8634262084961,
      "activations/layer4_attention_weight_max": 96.46029663085938,
      "activations/layer4_attention_weight_min": -91.34008026123047,
      "activations/layer5_attention_weight_max": 84.36241149902344,
      "activations/layer5_attention_weight_min": -81.65716552734375,
      "activations/layer6_attention_weight_max": 53.10850143432617,
      "activations/layer6_attention_weight_min": -50.91676712036133,
      "activations/layer7_attention_weight_max": 70.88414001464844,
      "activations/layer7_attention_weight_min": -72.6685562133789,
      "activations/layer8_attention_weight_max": 41.479331970214844,
      "activations/layer8_attention_weight_min": -42.69171905517578,
      "activations/layer9_attention_weight_max": 51.57770919799805,
      "activations/layer9_attention_weight_min": -48.345394134521484,
      "epoch": 7.07,
      "learning_rate": 0.00010546060606060605,
      "loss": 2.8143,
      "step": 121700
    },
    {
      "activations/layer0_attention_weight_max": 15.013908386230469,
      "activations/layer0_attention_weight_min": -14.025632858276367,
      "activations/layer10_attention_weight_max": 37.31264114379883,
      "activations/layer10_attention_weight_min": -35.537025451660156,
      "activations/layer11_attention_weight_max": 33.35899353027344,
      "activations/layer11_attention_weight_min": -30.363101959228516,
      "activations/layer12_attention_weight_max": 18.260026931762695,
      "activations/layer12_attention_weight_min": -25.510950088500977,
      "activations/layer13_attention_weight_max": 40.658119201660156,
      "activations/layer13_attention_weight_min": -30.78231430053711,
      "activations/layer14_attention_weight_max": 37.59611511230469,
      "activations/layer14_attention_weight_min": -28.74509620666504,
      "activations/layer15_attention_weight_max": 34.01713943481445,
      "activations/layer15_attention_weight_min": -28.948589324951172,
      "activations/layer16_attention_weight_max": 30.02428436279297,
      "activations/layer16_attention_weight_min": -25.497655868530273,
      "activations/layer17_attention_weight_max": 53.08000183105469,
      "activations/layer17_attention_weight_min": -40.133113861083984,
      "activations/layer18_attention_weight_max": 48.11263656616211,
      "activations/layer18_attention_weight_min": -35.65390396118164,
      "activations/layer19_attention_weight_max": 20.349905014038086,
      "activations/layer19_attention_weight_min": -21.02216148376465,
      "activations/layer1_attention_weight_max": 17.792526245117188,
      "activations/layer1_attention_weight_min": -13.408492088317871,
      "activations/layer20_attention_weight_max": 20.06468391418457,
      "activations/layer20_attention_weight_min": -25.0134334564209,
      "activations/layer21_attention_weight_max": 36.66053771972656,
      "activations/layer21_attention_weight_min": -23.307790756225586,
      "activations/layer22_attention_weight_max": 35.62949752807617,
      "activations/layer22_attention_weight_min": -26.08152961730957,
      "activations/layer23_attention_weight_max": 34.40595626831055,
      "activations/layer23_attention_weight_min": -21.84389877319336,
      "activations/layer2_attention_weight_max": 31.745403289794922,
      "activations/layer2_attention_weight_min": -29.102764129638672,
      "activations/layer3_attention_weight_max": 82.10395812988281,
      "activations/layer3_attention_weight_min": -81.52140045166016,
      "activations/layer4_attention_weight_max": 97.9875717163086,
      "activations/layer4_attention_weight_min": -90.07050323486328,
      "activations/layer5_attention_weight_max": 90.08680725097656,
      "activations/layer5_attention_weight_min": -79.69624328613281,
      "activations/layer6_attention_weight_max": 54.56779479980469,
      "activations/layer6_attention_weight_min": -52.51523208618164,
      "activations/layer7_attention_weight_max": 69.86254119873047,
      "activations/layer7_attention_weight_min": -71.61573791503906,
      "activations/layer8_attention_weight_max": 42.80379104614258,
      "activations/layer8_attention_weight_min": -42.526554107666016,
      "activations/layer9_attention_weight_max": 52.4182014465332,
      "activations/layer9_attention_weight_min": -46.642635345458984,
      "epoch": 7.07,
      "learning_rate": 0.00010544166666666666,
      "loss": 2.8168,
      "step": 121750
    },
    {
      "activations/layer0_attention_weight_max": 14.146659851074219,
      "activations/layer0_attention_weight_min": -14.667244911193848,
      "activations/layer10_attention_weight_max": 37.46364974975586,
      "activations/layer10_attention_weight_min": -35.70248794555664,
      "activations/layer11_attention_weight_max": 32.0168342590332,
      "activations/layer11_attention_weight_min": -31.194446563720703,
      "activations/layer12_attention_weight_max": 20.492347717285156,
      "activations/layer12_attention_weight_min": -30.062026977539062,
      "activations/layer13_attention_weight_max": 43.69816589355469,
      "activations/layer13_attention_weight_min": -35.88264846801758,
      "activations/layer14_attention_weight_max": 64.15694427490234,
      "activations/layer14_attention_weight_min": -39.018070220947266,
      "activations/layer15_attention_weight_max": 41.65817642211914,
      "activations/layer15_attention_weight_min": -29.608051300048828,
      "activations/layer16_attention_weight_max": 29.74567985534668,
      "activations/layer16_attention_weight_min": -26.390911102294922,
      "activations/layer17_attention_weight_max": 53.47921371459961,
      "activations/layer17_attention_weight_min": -43.00383758544922,
      "activations/layer18_attention_weight_max": 44.01441192626953,
      "activations/layer18_attention_weight_min": -36.80925750732422,
      "activations/layer19_attention_weight_max": 22.099267959594727,
      "activations/layer19_attention_weight_min": -20.339065551757812,
      "activations/layer1_attention_weight_max": 17.490148544311523,
      "activations/layer1_attention_weight_min": -14.314979553222656,
      "activations/layer20_attention_weight_max": 25.841520309448242,
      "activations/layer20_attention_weight_min": -23.796064376831055,
      "activations/layer21_attention_weight_max": 41.07719802856445,
      "activations/layer21_attention_weight_min": -24.425888061523438,
      "activations/layer22_attention_weight_max": 32.41773986816406,
      "activations/layer22_attention_weight_min": -25.97446632385254,
      "activations/layer23_attention_weight_max": 33.771141052246094,
      "activations/layer23_attention_weight_min": -21.2955379486084,
      "activations/layer2_attention_weight_max": 33.056488037109375,
      "activations/layer2_attention_weight_min": -29.318153381347656,
      "activations/layer3_attention_weight_max": 89.38375854492188,
      "activations/layer3_attention_weight_min": -87.12295532226562,
      "activations/layer4_attention_weight_max": 106.6728515625,
      "activations/layer4_attention_weight_min": -103.72025299072266,
      "activations/layer5_attention_weight_max": 90.06466674804688,
      "activations/layer5_attention_weight_min": -89.53224182128906,
      "activations/layer6_attention_weight_max": 56.6801872253418,
      "activations/layer6_attention_weight_min": -55.7705192565918,
      "activations/layer7_attention_weight_max": 79.17076110839844,
      "activations/layer7_attention_weight_min": -75.2685775756836,
      "activations/layer8_attention_weight_max": 46.78986740112305,
      "activations/layer8_attention_weight_min": -48.1063117980957,
      "activations/layer9_attention_weight_max": 59.67670822143555,
      "activations/layer9_attention_weight_min": -54.61533737182617,
      "epoch": 7.08,
      "learning_rate": 0.00010542272727272727,
      "loss": 2.8183,
      "step": 121800
    },
    {
      "activations/layer0_attention_weight_max": 14.205251693725586,
      "activations/layer0_attention_weight_min": -14.881488800048828,
      "activations/layer10_attention_weight_max": 37.31327819824219,
      "activations/layer10_attention_weight_min": -35.08913040161133,
      "activations/layer11_attention_weight_max": 28.238967895507812,
      "activations/layer11_attention_weight_min": -31.049001693725586,
      "activations/layer12_attention_weight_max": 22.42291259765625,
      "activations/layer12_attention_weight_min": -23.16239356994629,
      "activations/layer13_attention_weight_max": 45.58335494995117,
      "activations/layer13_attention_weight_min": -32.28776168823242,
      "activations/layer14_attention_weight_max": 39.92792892456055,
      "activations/layer14_attention_weight_min": -30.54856300354004,
      "activations/layer15_attention_weight_max": 41.56449508666992,
      "activations/layer15_attention_weight_min": -29.0162353515625,
      "activations/layer16_attention_weight_max": 35.48417663574219,
      "activations/layer16_attention_weight_min": -29.00493049621582,
      "activations/layer17_attention_weight_max": 60.66889190673828,
      "activations/layer17_attention_weight_min": -43.48195266723633,
      "activations/layer18_attention_weight_max": 54.769615173339844,
      "activations/layer18_attention_weight_min": -35.976497650146484,
      "activations/layer19_attention_weight_max": 26.768415451049805,
      "activations/layer19_attention_weight_min": -20.564266204833984,
      "activations/layer1_attention_weight_max": 16.813570022583008,
      "activations/layer1_attention_weight_min": -14.782187461853027,
      "activations/layer20_attention_weight_max": 27.626380920410156,
      "activations/layer20_attention_weight_min": -22.61857795715332,
      "activations/layer21_attention_weight_max": 45.05744934082031,
      "activations/layer21_attention_weight_min": -24.469778060913086,
      "activations/layer22_attention_weight_max": 39.64424133300781,
      "activations/layer22_attention_weight_min": -25.07847785949707,
      "activations/layer23_attention_weight_max": 39.31793212890625,
      "activations/layer23_attention_weight_min": -21.250125885009766,
      "activations/layer2_attention_weight_max": 29.969030380249023,
      "activations/layer2_attention_weight_min": -27.866180419921875,
      "activations/layer3_attention_weight_max": 84.38728332519531,
      "activations/layer3_attention_weight_min": -83.29013061523438,
      "activations/layer4_attention_weight_max": 95.6017837524414,
      "activations/layer4_attention_weight_min": -91.59852600097656,
      "activations/layer5_attention_weight_max": 89.67042541503906,
      "activations/layer5_attention_weight_min": -87.62234497070312,
      "activations/layer6_attention_weight_max": 55.08348083496094,
      "activations/layer6_attention_weight_min": -58.06395721435547,
      "activations/layer7_attention_weight_max": 74.50546264648438,
      "activations/layer7_attention_weight_min": -76.70426177978516,
      "activations/layer8_attention_weight_max": 46.371910095214844,
      "activations/layer8_attention_weight_min": -47.70626449584961,
      "activations/layer9_attention_weight_max": 56.40876007080078,
      "activations/layer9_attention_weight_min": -49.12588119506836,
      "epoch": 7.08,
      "learning_rate": 0.00010540378787878787,
      "loss": 2.8177,
      "step": 121850
    },
    {
      "activations/layer0_attention_weight_max": 14.310673713684082,
      "activations/layer0_attention_weight_min": -15.387742042541504,
      "activations/layer10_attention_weight_max": 35.18083953857422,
      "activations/layer10_attention_weight_min": -35.07514953613281,
      "activations/layer11_attention_weight_max": 29.743900299072266,
      "activations/layer11_attention_weight_min": -30.340307235717773,
      "activations/layer12_attention_weight_max": 20.414779663085938,
      "activations/layer12_attention_weight_min": -26.655576705932617,
      "activations/layer13_attention_weight_max": 44.34526824951172,
      "activations/layer13_attention_weight_min": -32.51708984375,
      "activations/layer14_attention_weight_max": 36.24603271484375,
      "activations/layer14_attention_weight_min": -30.842344284057617,
      "activations/layer15_attention_weight_max": 35.293617248535156,
      "activations/layer15_attention_weight_min": -30.363727569580078,
      "activations/layer16_attention_weight_max": 30.86795997619629,
      "activations/layer16_attention_weight_min": -27.764873504638672,
      "activations/layer17_attention_weight_max": 52.171878814697266,
      "activations/layer17_attention_weight_min": -41.78265380859375,
      "activations/layer18_attention_weight_max": 41.87154006958008,
      "activations/layer18_attention_weight_min": -34.673439025878906,
      "activations/layer19_attention_weight_max": 22.43315315246582,
      "activations/layer19_attention_weight_min": -20.534194946289062,
      "activations/layer1_attention_weight_max": 16.641639709472656,
      "activations/layer1_attention_weight_min": -14.463205337524414,
      "activations/layer20_attention_weight_max": 21.260868072509766,
      "activations/layer20_attention_weight_min": -22.27231216430664,
      "activations/layer21_attention_weight_max": 38.312625885009766,
      "activations/layer21_attention_weight_min": -25.070547103881836,
      "activations/layer22_attention_weight_max": 30.882051467895508,
      "activations/layer22_attention_weight_min": -24.56226348876953,
      "activations/layer23_attention_weight_max": 31.141958236694336,
      "activations/layer23_attention_weight_min": -22.451335906982422,
      "activations/layer2_attention_weight_max": 29.848224639892578,
      "activations/layer2_attention_weight_min": -29.39220428466797,
      "activations/layer3_attention_weight_max": 83.31185913085938,
      "activations/layer3_attention_weight_min": -84.76983642578125,
      "activations/layer4_attention_weight_max": 97.73799896240234,
      "activations/layer4_attention_weight_min": -90.3882827758789,
      "activations/layer5_attention_weight_max": 89.1643295288086,
      "activations/layer5_attention_weight_min": -83.52423858642578,
      "activations/layer6_attention_weight_max": 56.30830001831055,
      "activations/layer6_attention_weight_min": -55.448448181152344,
      "activations/layer7_attention_weight_max": 75.3946304321289,
      "activations/layer7_attention_weight_min": -69.09917449951172,
      "activations/layer8_attention_weight_max": 44.240482330322266,
      "activations/layer8_attention_weight_min": -45.6363525390625,
      "activations/layer9_attention_weight_max": 52.32693099975586,
      "activations/layer9_attention_weight_min": -48.905582427978516,
      "epoch": 7.08,
      "learning_rate": 0.00010538484848484848,
      "loss": 2.8089,
      "step": 121900
    },
    {
      "activations/layer0_attention_weight_max": 14.7494478225708,
      "activations/layer0_attention_weight_min": -15.309866905212402,
      "activations/layer10_attention_weight_max": 36.69154739379883,
      "activations/layer10_attention_weight_min": -34.033809661865234,
      "activations/layer11_attention_weight_max": 29.826950073242188,
      "activations/layer11_attention_weight_min": -29.887557983398438,
      "activations/layer12_attention_weight_max": 21.20025634765625,
      "activations/layer12_attention_weight_min": -25.574966430664062,
      "activations/layer13_attention_weight_max": 41.83448791503906,
      "activations/layer13_attention_weight_min": -33.12393569946289,
      "activations/layer14_attention_weight_max": 37.95820617675781,
      "activations/layer14_attention_weight_min": -30.61359214782715,
      "activations/layer15_attention_weight_max": 35.561710357666016,
      "activations/layer15_attention_weight_min": -31.924795150756836,
      "activations/layer16_attention_weight_max": 31.625577926635742,
      "activations/layer16_attention_weight_min": -27.597766876220703,
      "activations/layer17_attention_weight_max": 52.23981857299805,
      "activations/layer17_attention_weight_min": -40.964805603027344,
      "activations/layer18_attention_weight_max": 48.623538970947266,
      "activations/layer18_attention_weight_min": -34.73086929321289,
      "activations/layer19_attention_weight_max": 21.174528121948242,
      "activations/layer19_attention_weight_min": -21.462589263916016,
      "activations/layer1_attention_weight_max": 16.74311065673828,
      "activations/layer1_attention_weight_min": -14.842681884765625,
      "activations/layer20_attention_weight_max": 22.489036560058594,
      "activations/layer20_attention_weight_min": -23.598859786987305,
      "activations/layer21_attention_weight_max": 43.93900680541992,
      "activations/layer21_attention_weight_min": -29.1602783203125,
      "activations/layer22_attention_weight_max": 31.21836280822754,
      "activations/layer22_attention_weight_min": -22.79737091064453,
      "activations/layer23_attention_weight_max": 32.82833480834961,
      "activations/layer23_attention_weight_min": -24.218711853027344,
      "activations/layer2_attention_weight_max": 30.698406219482422,
      "activations/layer2_attention_weight_min": -27.899160385131836,
      "activations/layer3_attention_weight_max": 78.58331298828125,
      "activations/layer3_attention_weight_min": -86.9004135131836,
      "activations/layer4_attention_weight_max": 92.71900939941406,
      "activations/layer4_attention_weight_min": -89.88607025146484,
      "activations/layer5_attention_weight_max": 87.78878021240234,
      "activations/layer5_attention_weight_min": -80.75331115722656,
      "activations/layer6_attention_weight_max": 55.3618049621582,
      "activations/layer6_attention_weight_min": -49.8799934387207,
      "activations/layer7_attention_weight_max": 84.52947998046875,
      "activations/layer7_attention_weight_min": -66.64897918701172,
      "activations/layer8_attention_weight_max": 43.89397430419922,
      "activations/layer8_attention_weight_min": -45.299232482910156,
      "activations/layer9_attention_weight_max": 48.35419464111328,
      "activations/layer9_attention_weight_min": -47.28158187866211,
      "epoch": 7.09,
      "learning_rate": 0.00010536590909090908,
      "loss": 2.8205,
      "step": 121950
    },
    {
      "activations/layer0_attention_weight_max": 15.095086097717285,
      "activations/layer0_attention_weight_min": -14.83113956451416,
      "activations/layer10_attention_weight_max": 44.56039047241211,
      "activations/layer10_attention_weight_min": -36.72303771972656,
      "activations/layer11_attention_weight_max": 36.42631530761719,
      "activations/layer11_attention_weight_min": -31.459808349609375,
      "activations/layer12_attention_weight_max": 38.87593078613281,
      "activations/layer12_attention_weight_min": -24.314777374267578,
      "activations/layer13_attention_weight_max": 47.236412048339844,
      "activations/layer13_attention_weight_min": -35.7301139831543,
      "activations/layer14_attention_weight_max": 40.26915740966797,
      "activations/layer14_attention_weight_min": -30.26783561706543,
      "activations/layer15_attention_weight_max": 42.32505416870117,
      "activations/layer15_attention_weight_min": -28.078737258911133,
      "activations/layer16_attention_weight_max": 36.28373336791992,
      "activations/layer16_attention_weight_min": -26.144855499267578,
      "activations/layer17_attention_weight_max": 59.87164306640625,
      "activations/layer17_attention_weight_min": -41.56611633300781,
      "activations/layer18_attention_weight_max": 52.58635330200195,
      "activations/layer18_attention_weight_min": -36.40962600708008,
      "activations/layer19_attention_weight_max": 23.984895706176758,
      "activations/layer19_attention_weight_min": -20.817245483398438,
      "activations/layer1_attention_weight_max": 17.184728622436523,
      "activations/layer1_attention_weight_min": -13.964734077453613,
      "activations/layer20_attention_weight_max": 23.25479507446289,
      "activations/layer20_attention_weight_min": -20.065000534057617,
      "activations/layer21_attention_weight_max": 41.75120544433594,
      "activations/layer21_attention_weight_min": -23.404150009155273,
      "activations/layer22_attention_weight_max": 34.78624725341797,
      "activations/layer22_attention_weight_min": -25.174348831176758,
      "activations/layer23_attention_weight_max": 32.81190490722656,
      "activations/layer23_attention_weight_min": -22.721960067749023,
      "activations/layer2_attention_weight_max": 32.52922058105469,
      "activations/layer2_attention_weight_min": -29.475271224975586,
      "activations/layer3_attention_weight_max": 82.5147476196289,
      "activations/layer3_attention_weight_min": -87.89155578613281,
      "activations/layer4_attention_weight_max": 101.52269744873047,
      "activations/layer4_attention_weight_min": -91.5785140991211,
      "activations/layer5_attention_weight_max": 92.69991302490234,
      "activations/layer5_attention_weight_min": -83.6666488647461,
      "activations/layer6_attention_weight_max": 56.82598876953125,
      "activations/layer6_attention_weight_min": -54.111637115478516,
      "activations/layer7_attention_weight_max": 76.28141784667969,
      "activations/layer7_attention_weight_min": -71.78596496582031,
      "activations/layer8_attention_weight_max": 49.98467254638672,
      "activations/layer8_attention_weight_min": -48.06059646606445,
      "activations/layer9_attention_weight_max": 54.816036224365234,
      "activations/layer9_attention_weight_min": -51.7396354675293,
      "epoch": 7.09,
      "learning_rate": 0.00010534696969696968,
      "loss": 2.8202,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5323,
      "eval_samples_per_second": 503.266,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5323,
      "eval_openwebtext_samples_per_second": 503.266,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_wikitext_loss": 3.009765625,
      "eval_wikitext_ppl": 20.282645623050005,
      "eval_wikitext_runtime": 2.0275,
      "eval_wikitext_samples_per_second": 224.903,
      "step": 122000
    },
    {
      "epoch": 7.09,
      "eval_lambada_loss": 2.751953125,
      "eval_lambada_ppl": 15.673213754973451,
      "eval_lambada_runtime": 9.6648,
      "eval_lambada_samples_per_second": 503.789,
      "step": 122000
    },
    {
      "activations/layer0_attention_weight_max": 13.813150405883789,
      "activations/layer0_attention_weight_min": -14.518444061279297,
      "activations/layer10_attention_weight_max": 42.22559356689453,
      "activations/layer10_attention_weight_min": -36.24254608154297,
      "activations/layer11_attention_weight_max": 34.12831115722656,
      "activations/layer11_attention_weight_min": -31.912656784057617,
      "activations/layer12_attention_weight_max": 20.287057876586914,
      "activations/layer12_attention_weight_min": -23.243896484375,
      "activations/layer13_attention_weight_max": 52.018428802490234,
      "activations/layer13_attention_weight_min": -33.29513168334961,
      "activations/layer14_attention_weight_max": 39.7593879699707,
      "activations/layer14_attention_weight_min": -34.90110778808594,
      "activations/layer15_attention_weight_max": 38.087196350097656,
      "activations/layer15_attention_weight_min": -31.660795211791992,
      "activations/layer16_attention_weight_max": 33.58296203613281,
      "activations/layer16_attention_weight_min": -27.003297805786133,
      "activations/layer17_attention_weight_max": 59.99711227416992,
      "activations/layer17_attention_weight_min": -43.39139175415039,
      "activations/layer18_attention_weight_max": 48.90607833862305,
      "activations/layer18_attention_weight_min": -34.7136344909668,
      "activations/layer19_attention_weight_max": 25.21665382385254,
      "activations/layer19_attention_weight_min": -17.781972885131836,
      "activations/layer1_attention_weight_max": 16.728626251220703,
      "activations/layer1_attention_weight_min": -16.53374671936035,
      "activations/layer20_attention_weight_max": 21.94794464111328,
      "activations/layer20_attention_weight_min": -22.750633239746094,
      "activations/layer21_attention_weight_max": 39.6788215637207,
      "activations/layer21_attention_weight_min": -24.034103393554688,
      "activations/layer22_attention_weight_max": 32.94816207885742,
      "activations/layer22_attention_weight_min": -26.0902099609375,
      "activations/layer23_attention_weight_max": 33.334293365478516,
      "activations/layer23_attention_weight_min": -20.218900680541992,
      "activations/layer2_attention_weight_max": 30.5867919921875,
      "activations/layer2_attention_weight_min": -29.507612228393555,
      "activations/layer3_attention_weight_max": 85.39075469970703,
      "activations/layer3_attention_weight_min": -91.19310760498047,
      "activations/layer4_attention_weight_max": 105.03279113769531,
      "activations/layer4_attention_weight_min": -96.57852172851562,
      "activations/layer5_attention_weight_max": 95.9859619140625,
      "activations/layer5_attention_weight_min": -84.65216827392578,
      "activations/layer6_attention_weight_max": 57.81648254394531,
      "activations/layer6_attention_weight_min": -53.894691467285156,
      "activations/layer7_attention_weight_max": 74.43576049804688,
      "activations/layer7_attention_weight_min": -72.07054138183594,
      "activations/layer8_attention_weight_max": 43.546661376953125,
      "activations/layer8_attention_weight_min": -47.61785888671875,
      "activations/layer9_attention_weight_max": 51.3558349609375,
      "activations/layer9_attention_weight_min": -49.272823333740234,
      "epoch": 7.09,
      "learning_rate": 0.00010532803030303029,
      "loss": 2.8242,
      "step": 122050
    },
    {
      "activations/layer0_attention_weight_max": 14.045696258544922,
      "activations/layer0_attention_weight_min": -14.497321128845215,
      "activations/layer10_attention_weight_max": 38.7244873046875,
      "activations/layer10_attention_weight_min": -36.81391906738281,
      "activations/layer11_attention_weight_max": 33.839561462402344,
      "activations/layer11_attention_weight_min": -31.978416442871094,
      "activations/layer12_attention_weight_max": 27.364351272583008,
      "activations/layer12_attention_weight_min": -28.61216926574707,
      "activations/layer13_attention_weight_max": 53.43423080444336,
      "activations/layer13_attention_weight_min": -40.277923583984375,
      "activations/layer14_attention_weight_max": 72.594970703125,
      "activations/layer14_attention_weight_min": -45.25931930541992,
      "activations/layer15_attention_weight_max": 54.52411651611328,
      "activations/layer15_attention_weight_min": -38.85594177246094,
      "activations/layer16_attention_weight_max": 34.760032653808594,
      "activations/layer16_attention_weight_min": -27.947715759277344,
      "activations/layer17_attention_weight_max": 59.97591018676758,
      "activations/layer17_attention_weight_min": -43.61582946777344,
      "activations/layer18_attention_weight_max": 54.49022674560547,
      "activations/layer18_attention_weight_min": -36.296783447265625,
      "activations/layer19_attention_weight_max": 24.638397216796875,
      "activations/layer19_attention_weight_min": -21.081350326538086,
      "activations/layer1_attention_weight_max": 16.855587005615234,
      "activations/layer1_attention_weight_min": -14.447497367858887,
      "activations/layer20_attention_weight_max": 23.997488021850586,
      "activations/layer20_attention_weight_min": -26.724613189697266,
      "activations/layer21_attention_weight_max": 38.81721878051758,
      "activations/layer21_attention_weight_min": -24.704803466796875,
      "activations/layer22_attention_weight_max": 41.6347770690918,
      "activations/layer22_attention_weight_min": -28.29671287536621,
      "activations/layer23_attention_weight_max": 34.20338821411133,
      "activations/layer23_attention_weight_min": -20.87617301940918,
      "activations/layer2_attention_weight_max": 31.51702880859375,
      "activations/layer2_attention_weight_min": -30.403274536132812,
      "activations/layer3_attention_weight_max": 85.49909210205078,
      "activations/layer3_attention_weight_min": -87.16166687011719,
      "activations/layer4_attention_weight_max": 101.23384857177734,
      "activations/layer4_attention_weight_min": -92.23824310302734,
      "activations/layer5_attention_weight_max": 88.6453857421875,
      "activations/layer5_attention_weight_min": -78.47532653808594,
      "activations/layer6_attention_weight_max": 57.29442596435547,
      "activations/layer6_attention_weight_min": -52.465965270996094,
      "activations/layer7_attention_weight_max": 80.78968811035156,
      "activations/layer7_attention_weight_min": -71.02500915527344,
      "activations/layer8_attention_weight_max": 45.532989501953125,
      "activations/layer8_attention_weight_min": -43.43172073364258,
      "activations/layer9_attention_weight_max": 66.5277328491211,
      "activations/layer9_attention_weight_min": -56.742950439453125,
      "epoch": 7.09,
      "learning_rate": 0.0001053090909090909,
      "loss": 2.8131,
      "step": 122100
    },
    {
      "activations/layer0_attention_weight_max": 16.71401023864746,
      "activations/layer0_attention_weight_min": -14.738309860229492,
      "activations/layer10_attention_weight_max": 36.7260856628418,
      "activations/layer10_attention_weight_min": -35.99839782714844,
      "activations/layer11_attention_weight_max": 31.29327964782715,
      "activations/layer11_attention_weight_min": -30.3842830657959,
      "activations/layer12_attention_weight_max": 22.855253219604492,
      "activations/layer12_attention_weight_min": -24.103626251220703,
      "activations/layer13_attention_weight_max": 41.087242126464844,
      "activations/layer13_attention_weight_min": -30.26784324645996,
      "activations/layer14_attention_weight_max": 52.55187225341797,
      "activations/layer14_attention_weight_min": -32.46046829223633,
      "activations/layer15_attention_weight_max": 38.14663314819336,
      "activations/layer15_attention_weight_min": -29.935894012451172,
      "activations/layer16_attention_weight_max": 32.84414291381836,
      "activations/layer16_attention_weight_min": -27.94318962097168,
      "activations/layer17_attention_weight_max": 53.991966247558594,
      "activations/layer17_attention_weight_min": -43.88840866088867,
      "activations/layer18_attention_weight_max": 50.18828582763672,
      "activations/layer18_attention_weight_min": -38.22154998779297,
      "activations/layer19_attention_weight_max": 20.14082145690918,
      "activations/layer19_attention_weight_min": -24.110830307006836,
      "activations/layer1_attention_weight_max": 16.122034072875977,
      "activations/layer1_attention_weight_min": -15.721385955810547,
      "activations/layer20_attention_weight_max": 22.5579891204834,
      "activations/layer20_attention_weight_min": -21.004375457763672,
      "activations/layer21_attention_weight_max": 35.16370391845703,
      "activations/layer21_attention_weight_min": -24.844491958618164,
      "activations/layer22_attention_weight_max": 32.60602951049805,
      "activations/layer22_attention_weight_min": -27.438154220581055,
      "activations/layer23_attention_weight_max": 33.97031784057617,
      "activations/layer23_attention_weight_min": -21.34225082397461,
      "activations/layer2_attention_weight_max": 30.724002838134766,
      "activations/layer2_attention_weight_min": -30.37409019470215,
      "activations/layer3_attention_weight_max": 85.4123764038086,
      "activations/layer3_attention_weight_min": -89.78410339355469,
      "activations/layer4_attention_weight_max": 102.458251953125,
      "activations/layer4_attention_weight_min": -98.56324768066406,
      "activations/layer5_attention_weight_max": 95.17857360839844,
      "activations/layer5_attention_weight_min": -87.01963806152344,
      "activations/layer6_attention_weight_max": 57.007171630859375,
      "activations/layer6_attention_weight_min": -53.94772720336914,
      "activations/layer7_attention_weight_max": 78.39512634277344,
      "activations/layer7_attention_weight_min": -71.32080841064453,
      "activations/layer8_attention_weight_max": 47.63911819458008,
      "activations/layer8_attention_weight_min": -45.7270393371582,
      "activations/layer9_attention_weight_max": 48.289608001708984,
      "activations/layer9_attention_weight_min": -48.8430061340332,
      "epoch": 7.1,
      "learning_rate": 0.00010529015151515152,
      "loss": 2.8047,
      "step": 122150
    },
    {
      "activations/layer0_attention_weight_max": 15.134604454040527,
      "activations/layer0_attention_weight_min": -14.992325782775879,
      "activations/layer10_attention_weight_max": 35.50246047973633,
      "activations/layer10_attention_weight_min": -35.49095916748047,
      "activations/layer11_attention_weight_max": 28.699790954589844,
      "activations/layer11_attention_weight_min": -29.39539909362793,
      "activations/layer12_attention_weight_max": 23.29539680480957,
      "activations/layer12_attention_weight_min": -24.736982345581055,
      "activations/layer13_attention_weight_max": 39.06671142578125,
      "activations/layer13_attention_weight_min": -32.091922760009766,
      "activations/layer14_attention_weight_max": 38.577415466308594,
      "activations/layer14_attention_weight_min": -30.94216537475586,
      "activations/layer15_attention_weight_max": 36.33443069458008,
      "activations/layer15_attention_weight_min": -31.22025489807129,
      "activations/layer16_attention_weight_max": 36.24668502807617,
      "activations/layer16_attention_weight_min": -27.01140594482422,
      "activations/layer17_attention_weight_max": 52.968021392822266,
      "activations/layer17_attention_weight_min": -42.52909469604492,
      "activations/layer18_attention_weight_max": 52.32221603393555,
      "activations/layer18_attention_weight_min": -34.158538818359375,
      "activations/layer19_attention_weight_max": 23.320293426513672,
      "activations/layer19_attention_weight_min": -19.26325225830078,
      "activations/layer1_attention_weight_max": 16.485158920288086,
      "activations/layer1_attention_weight_min": -13.939644813537598,
      "activations/layer20_attention_weight_max": 22.75966453552246,
      "activations/layer20_attention_weight_min": -23.09664535522461,
      "activations/layer21_attention_weight_max": 39.16646957397461,
      "activations/layer21_attention_weight_min": -23.201568603515625,
      "activations/layer22_attention_weight_max": 35.072391510009766,
      "activations/layer22_attention_weight_min": -24.68864631652832,
      "activations/layer23_attention_weight_max": 35.505611419677734,
      "activations/layer23_attention_weight_min": -19.097440719604492,
      "activations/layer2_attention_weight_max": 30.21392822265625,
      "activations/layer2_attention_weight_min": -29.30148696899414,
      "activations/layer3_attention_weight_max": 82.1866455078125,
      "activations/layer3_attention_weight_min": -84.01042175292969,
      "activations/layer4_attention_weight_max": 92.20574188232422,
      "activations/layer4_attention_weight_min": -93.50920867919922,
      "activations/layer5_attention_weight_max": 84.0271224975586,
      "activations/layer5_attention_weight_min": -82.07894897460938,
      "activations/layer6_attention_weight_max": 52.429935455322266,
      "activations/layer6_attention_weight_min": -53.194698333740234,
      "activations/layer7_attention_weight_max": 73.58906555175781,
      "activations/layer7_attention_weight_min": -71.86772155761719,
      "activations/layer8_attention_weight_max": 43.731143951416016,
      "activations/layer8_attention_weight_min": -42.928104400634766,
      "activations/layer9_attention_weight_max": 49.46027755737305,
      "activations/layer9_attention_weight_min": -44.94451141357422,
      "epoch": 7.1,
      "learning_rate": 0.0001052712121212121,
      "loss": 2.8183,
      "step": 122200
    },
    {
      "activations/layer0_attention_weight_max": 15.48859691619873,
      "activations/layer0_attention_weight_min": -14.977060317993164,
      "activations/layer10_attention_weight_max": 43.933685302734375,
      "activations/layer10_attention_weight_min": -38.02509307861328,
      "activations/layer11_attention_weight_max": 36.24103927612305,
      "activations/layer11_attention_weight_min": -33.214698791503906,
      "activations/layer12_attention_weight_max": 21.408260345458984,
      "activations/layer12_attention_weight_min": -25.662294387817383,
      "activations/layer13_attention_weight_max": 50.02608108520508,
      "activations/layer13_attention_weight_min": -33.75230407714844,
      "activations/layer14_attention_weight_max": 53.35853958129883,
      "activations/layer14_attention_weight_min": -34.06753158569336,
      "activations/layer15_attention_weight_max": 42.75844955444336,
      "activations/layer15_attention_weight_min": -32.83905792236328,
      "activations/layer16_attention_weight_max": 37.77226638793945,
      "activations/layer16_attention_weight_min": -31.91790771484375,
      "activations/layer17_attention_weight_max": 63.46433639526367,
      "activations/layer17_attention_weight_min": -49.03661346435547,
      "activations/layer18_attention_weight_max": 55.8742561340332,
      "activations/layer18_attention_weight_min": -41.836517333984375,
      "activations/layer19_attention_weight_max": 22.237363815307617,
      "activations/layer19_attention_weight_min": -23.752647399902344,
      "activations/layer1_attention_weight_max": 16.925012588500977,
      "activations/layer1_attention_weight_min": -15.827249526977539,
      "activations/layer20_attention_weight_max": 26.07002830505371,
      "activations/layer20_attention_weight_min": -25.690967559814453,
      "activations/layer21_attention_weight_max": 45.951026916503906,
      "activations/layer21_attention_weight_min": -34.57082748413086,
      "activations/layer22_attention_weight_max": 36.63383483886719,
      "activations/layer22_attention_weight_min": -30.955841064453125,
      "activations/layer23_attention_weight_max": 36.432403564453125,
      "activations/layer23_attention_weight_min": -23.61603546142578,
      "activations/layer2_attention_weight_max": 30.565799713134766,
      "activations/layer2_attention_weight_min": -28.11476707458496,
      "activations/layer3_attention_weight_max": 87.61373138427734,
      "activations/layer3_attention_weight_min": -86.27556610107422,
      "activations/layer4_attention_weight_max": 99.17190551757812,
      "activations/layer4_attention_weight_min": -95.57463836669922,
      "activations/layer5_attention_weight_max": 88.63093566894531,
      "activations/layer5_attention_weight_min": -85.10186767578125,
      "activations/layer6_attention_weight_max": 55.256473541259766,
      "activations/layer6_attention_weight_min": -55.36670684814453,
      "activations/layer7_attention_weight_max": 78.01487731933594,
      "activations/layer7_attention_weight_min": -72.74494934082031,
      "activations/layer8_attention_weight_max": 45.98217010498047,
      "activations/layer8_attention_weight_min": -46.97710037231445,
      "activations/layer9_attention_weight_max": 57.49624252319336,
      "activations/layer9_attention_weight_min": -50.36320495605469,
      "epoch": 7.1,
      "learning_rate": 0.00010525227272727272,
      "loss": 2.8136,
      "step": 122250
    },
    {
      "activations/layer0_attention_weight_max": 14.693307876586914,
      "activations/layer0_attention_weight_min": -15.037261009216309,
      "activations/layer10_attention_weight_max": 35.855987548828125,
      "activations/layer10_attention_weight_min": -34.369842529296875,
      "activations/layer11_attention_weight_max": 30.43103790283203,
      "activations/layer11_attention_weight_min": -29.92878532409668,
      "activations/layer12_attention_weight_max": 20.480077743530273,
      "activations/layer12_attention_weight_min": -27.232999801635742,
      "activations/layer13_attention_weight_max": 49.08384323120117,
      "activations/layer13_attention_weight_min": -39.69206237792969,
      "activations/layer14_attention_weight_max": 45.57986068725586,
      "activations/layer14_attention_weight_min": -34.41831588745117,
      "activations/layer15_attention_weight_max": 33.73106002807617,
      "activations/layer15_attention_weight_min": -30.9997615814209,
      "activations/layer16_attention_weight_max": 34.203060150146484,
      "activations/layer16_attention_weight_min": -25.624347686767578,
      "activations/layer17_attention_weight_max": 53.059417724609375,
      "activations/layer17_attention_weight_min": -42.18698501586914,
      "activations/layer18_attention_weight_max": 44.66685485839844,
      "activations/layer18_attention_weight_min": -34.04289627075195,
      "activations/layer19_attention_weight_max": 23.937686920166016,
      "activations/layer19_attention_weight_min": -20.79990577697754,
      "activations/layer1_attention_weight_max": 17.690399169921875,
      "activations/layer1_attention_weight_min": -14.040519714355469,
      "activations/layer20_attention_weight_max": 20.75563621520996,
      "activations/layer20_attention_weight_min": -23.004636764526367,
      "activations/layer21_attention_weight_max": 36.68596267700195,
      "activations/layer21_attention_weight_min": -26.9019832611084,
      "activations/layer22_attention_weight_max": 33.14459228515625,
      "activations/layer22_attention_weight_min": -26.14164924621582,
      "activations/layer23_attention_weight_max": 33.803104400634766,
      "activations/layer23_attention_weight_min": -19.503890991210938,
      "activations/layer2_attention_weight_max": 29.981578826904297,
      "activations/layer2_attention_weight_min": -28.215518951416016,
      "activations/layer3_attention_weight_max": 82.22207641601562,
      "activations/layer3_attention_weight_min": -87.47085571289062,
      "activations/layer4_attention_weight_max": 95.30826568603516,
      "activations/layer4_attention_weight_min": -97.13526916503906,
      "activations/layer5_attention_weight_max": 90.73809051513672,
      "activations/layer5_attention_weight_min": -82.90910339355469,
      "activations/layer6_attention_weight_max": 57.35580825805664,
      "activations/layer6_attention_weight_min": -54.50984191894531,
      "activations/layer7_attention_weight_max": 77.54542541503906,
      "activations/layer7_attention_weight_min": -69.25442504882812,
      "activations/layer8_attention_weight_max": 42.81814956665039,
      "activations/layer8_attention_weight_min": -43.55628204345703,
      "activations/layer9_attention_weight_max": 49.73988723754883,
      "activations/layer9_attention_weight_min": -47.263675689697266,
      "epoch": 7.11,
      "learning_rate": 0.00010523333333333333,
      "loss": 2.8114,
      "step": 122300
    },
    {
      "activations/layer0_attention_weight_max": 14.272216796875,
      "activations/layer0_attention_weight_min": -15.108449935913086,
      "activations/layer10_attention_weight_max": 42.75254821777344,
      "activations/layer10_attention_weight_min": -36.114784240722656,
      "activations/layer11_attention_weight_max": 33.57310485839844,
      "activations/layer11_attention_weight_min": -30.812808990478516,
      "activations/layer12_attention_weight_max": 20.65400505065918,
      "activations/layer12_attention_weight_min": -23.59324836730957,
      "activations/layer13_attention_weight_max": 43.77608108520508,
      "activations/layer13_attention_weight_min": -31.705713272094727,
      "activations/layer14_attention_weight_max": 47.53448486328125,
      "activations/layer14_attention_weight_min": -31.21636199951172,
      "activations/layer15_attention_weight_max": 39.43937683105469,
      "activations/layer15_attention_weight_min": -29.64472198486328,
      "activations/layer16_attention_weight_max": 32.492496490478516,
      "activations/layer16_attention_weight_min": -28.522377014160156,
      "activations/layer17_attention_weight_max": 53.90153884887695,
      "activations/layer17_attention_weight_min": -44.92336654663086,
      "activations/layer18_attention_weight_max": 47.7756233215332,
      "activations/layer18_attention_weight_min": -36.96701431274414,
      "activations/layer19_attention_weight_max": 23.250164031982422,
      "activations/layer19_attention_weight_min": -20.801572799682617,
      "activations/layer1_attention_weight_max": 16.40121078491211,
      "activations/layer1_attention_weight_min": -14.22218132019043,
      "activations/layer20_attention_weight_max": 23.249055862426758,
      "activations/layer20_attention_weight_min": -21.91897201538086,
      "activations/layer21_attention_weight_max": 39.126094818115234,
      "activations/layer21_attention_weight_min": -26.102642059326172,
      "activations/layer22_attention_weight_max": 31.700708389282227,
      "activations/layer22_attention_weight_min": -25.260114669799805,
      "activations/layer23_attention_weight_max": 35.564876556396484,
      "activations/layer23_attention_weight_min": -20.380172729492188,
      "activations/layer2_attention_weight_max": 31.666645050048828,
      "activations/layer2_attention_weight_min": -29.981435775756836,
      "activations/layer3_attention_weight_max": 86.24911499023438,
      "activations/layer3_attention_weight_min": -89.62067413330078,
      "activations/layer4_attention_weight_max": 98.3807601928711,
      "activations/layer4_attention_weight_min": -91.54879760742188,
      "activations/layer5_attention_weight_max": 93.3967514038086,
      "activations/layer5_attention_weight_min": -84.2861328125,
      "activations/layer6_attention_weight_max": 57.99594497680664,
      "activations/layer6_attention_weight_min": -55.56827926635742,
      "activations/layer7_attention_weight_max": 83.65277862548828,
      "activations/layer7_attention_weight_min": -74.79315185546875,
      "activations/layer8_attention_weight_max": 48.05274963378906,
      "activations/layer8_attention_weight_min": -45.79929733276367,
      "activations/layer9_attention_weight_max": 57.953426361083984,
      "activations/layer9_attention_weight_min": -49.99332809448242,
      "epoch": 7.11,
      "learning_rate": 0.00010521439393939392,
      "loss": 2.8197,
      "step": 122350
    },
    {
      "activations/layer0_attention_weight_max": 15.468127250671387,
      "activations/layer0_attention_weight_min": -14.91594123840332,
      "activations/layer10_attention_weight_max": 37.99127960205078,
      "activations/layer10_attention_weight_min": -36.39381790161133,
      "activations/layer11_attention_weight_max": 30.605031967163086,
      "activations/layer11_attention_weight_min": -30.590246200561523,
      "activations/layer12_attention_weight_max": 21.346622467041016,
      "activations/layer12_attention_weight_min": -27.36794662475586,
      "activations/layer13_attention_weight_max": 44.737220764160156,
      "activations/layer13_attention_weight_min": -34.4206657409668,
      "activations/layer14_attention_weight_max": 43.9329833984375,
      "activations/layer14_attention_weight_min": -30.710002899169922,
      "activations/layer15_attention_weight_max": 38.25960922241211,
      "activations/layer15_attention_weight_min": -29.272111892700195,
      "activations/layer16_attention_weight_max": 38.350067138671875,
      "activations/layer16_attention_weight_min": -27.538379669189453,
      "activations/layer17_attention_weight_max": 56.43290710449219,
      "activations/layer17_attention_weight_min": -44.60356903076172,
      "activations/layer18_attention_weight_max": 46.50375747680664,
      "activations/layer18_attention_weight_min": -36.35690689086914,
      "activations/layer19_attention_weight_max": 25.63875389099121,
      "activations/layer19_attention_weight_min": -20.484838485717773,
      "activations/layer1_attention_weight_max": 17.424591064453125,
      "activations/layer1_attention_weight_min": -15.655669212341309,
      "activations/layer20_attention_weight_max": 22.79403305053711,
      "activations/layer20_attention_weight_min": -21.3150691986084,
      "activations/layer21_attention_weight_max": 44.44131851196289,
      "activations/layer21_attention_weight_min": -29.952939987182617,
      "activations/layer22_attention_weight_max": 35.104732513427734,
      "activations/layer22_attention_weight_min": -26.799612045288086,
      "activations/layer23_attention_weight_max": 35.88706970214844,
      "activations/layer23_attention_weight_min": -22.0036678314209,
      "activations/layer2_attention_weight_max": 30.249929428100586,
      "activations/layer2_attention_weight_min": -28.62903594970703,
      "activations/layer3_attention_weight_max": 83.94176483154297,
      "activations/layer3_attention_weight_min": -91.89884948730469,
      "activations/layer4_attention_weight_max": 96.30268096923828,
      "activations/layer4_attention_weight_min": -93.0072021484375,
      "activations/layer5_attention_weight_max": 90.28547668457031,
      "activations/layer5_attention_weight_min": -80.82829284667969,
      "activations/layer6_attention_weight_max": 58.73671340942383,
      "activations/layer6_attention_weight_min": -53.07563400268555,
      "activations/layer7_attention_weight_max": 79.10540008544922,
      "activations/layer7_attention_weight_min": -74.03633880615234,
      "activations/layer8_attention_weight_max": 44.48617172241211,
      "activations/layer8_attention_weight_min": -46.35211181640625,
      "activations/layer9_attention_weight_max": 51.5775260925293,
      "activations/layer9_attention_weight_min": -49.06556701660156,
      "epoch": 7.11,
      "learning_rate": 0.00010519545454545454,
      "loss": 2.8148,
      "step": 122400
    },
    {
      "activations/layer0_attention_weight_max": 15.60653305053711,
      "activations/layer0_attention_weight_min": -14.975096702575684,
      "activations/layer10_attention_weight_max": 42.34095764160156,
      "activations/layer10_attention_weight_min": -38.493385314941406,
      "activations/layer11_attention_weight_max": 32.79962921142578,
      "activations/layer11_attention_weight_min": -30.839582443237305,
      "activations/layer12_attention_weight_max": 21.74164390563965,
      "activations/layer12_attention_weight_min": -26.32897186279297,
      "activations/layer13_attention_weight_max": 70.10203552246094,
      "activations/layer13_attention_weight_min": -40.85050964355469,
      "activations/layer14_attention_weight_max": 67.63922119140625,
      "activations/layer14_attention_weight_min": -40.40859603881836,
      "activations/layer15_attention_weight_max": 49.34866714477539,
      "activations/layer15_attention_weight_min": -31.728431701660156,
      "activations/layer16_attention_weight_max": 38.059024810791016,
      "activations/layer16_attention_weight_min": -28.65948486328125,
      "activations/layer17_attention_weight_max": 60.55470275878906,
      "activations/layer17_attention_weight_min": -43.23539733886719,
      "activations/layer18_attention_weight_max": 53.9302978515625,
      "activations/layer18_attention_weight_min": -37.47659683227539,
      "activations/layer19_attention_weight_max": 23.68010139465332,
      "activations/layer19_attention_weight_min": -19.25306510925293,
      "activations/layer1_attention_weight_max": 17.73245620727539,
      "activations/layer1_attention_weight_min": -13.554974555969238,
      "activations/layer20_attention_weight_max": 23.256254196166992,
      "activations/layer20_attention_weight_min": -20.026521682739258,
      "activations/layer21_attention_weight_max": 41.75112533569336,
      "activations/layer21_attention_weight_min": -25.897289276123047,
      "activations/layer22_attention_weight_max": 34.52943801879883,
      "activations/layer22_attention_weight_min": -26.089876174926758,
      "activations/layer23_attention_weight_max": 35.19173812866211,
      "activations/layer23_attention_weight_min": -19.103736877441406,
      "activations/layer2_attention_weight_max": 31.049213409423828,
      "activations/layer2_attention_weight_min": -29.662368774414062,
      "activations/layer3_attention_weight_max": 86.31320190429688,
      "activations/layer3_attention_weight_min": -87.39601135253906,
      "activations/layer4_attention_weight_max": 100.77694702148438,
      "activations/layer4_attention_weight_min": -92.7048110961914,
      "activations/layer5_attention_weight_max": 95.6391830444336,
      "activations/layer5_attention_weight_min": -85.60639953613281,
      "activations/layer6_attention_weight_max": 56.822757720947266,
      "activations/layer6_attention_weight_min": -54.085182189941406,
      "activations/layer7_attention_weight_max": 82.3475570678711,
      "activations/layer7_attention_weight_min": -73.68832397460938,
      "activations/layer8_attention_weight_max": 48.528892517089844,
      "activations/layer8_attention_weight_min": -47.89199447631836,
      "activations/layer9_attention_weight_max": 59.87813186645508,
      "activations/layer9_attention_weight_min": -50.6746940612793,
      "epoch": 7.12,
      "learning_rate": 0.00010517651515151515,
      "loss": 2.815,
      "step": 122450
    },
    {
      "activations/layer0_attention_weight_max": 14.639485359191895,
      "activations/layer0_attention_weight_min": -15.297914505004883,
      "activations/layer10_attention_weight_max": 36.68861770629883,
      "activations/layer10_attention_weight_min": -36.596595764160156,
      "activations/layer11_attention_weight_max": 31.168224334716797,
      "activations/layer11_attention_weight_min": -30.66576385498047,
      "activations/layer12_attention_weight_max": 21.011091232299805,
      "activations/layer12_attention_weight_min": -27.367876052856445,
      "activations/layer13_attention_weight_max": 45.506324768066406,
      "activations/layer13_attention_weight_min": -35.88686752319336,
      "activations/layer14_attention_weight_max": 37.70542907714844,
      "activations/layer14_attention_weight_min": -32.023406982421875,
      "activations/layer15_attention_weight_max": 34.61052322387695,
      "activations/layer15_attention_weight_min": -29.417543411254883,
      "activations/layer16_attention_weight_max": 34.365196228027344,
      "activations/layer16_attention_weight_min": -27.772212982177734,
      "activations/layer17_attention_weight_max": 52.73475646972656,
      "activations/layer17_attention_weight_min": -44.663856506347656,
      "activations/layer18_attention_weight_max": 48.33621597290039,
      "activations/layer18_attention_weight_min": -35.68021011352539,
      "activations/layer19_attention_weight_max": 29.911630630493164,
      "activations/layer19_attention_weight_min": -21.332839965820312,
      "activations/layer1_attention_weight_max": 17.37839126586914,
      "activations/layer1_attention_weight_min": -14.393867492675781,
      "activations/layer20_attention_weight_max": 22.92513084411621,
      "activations/layer20_attention_weight_min": -21.126567840576172,
      "activations/layer21_attention_weight_max": 38.70341873168945,
      "activations/layer21_attention_weight_min": -25.195476531982422,
      "activations/layer22_attention_weight_max": 38.90290069580078,
      "activations/layer22_attention_weight_min": -27.460054397583008,
      "activations/layer23_attention_weight_max": 33.02460479736328,
      "activations/layer23_attention_weight_min": -24.698760986328125,
      "activations/layer2_attention_weight_max": 31.324626922607422,
      "activations/layer2_attention_weight_min": -30.57430648803711,
      "activations/layer3_attention_weight_max": 84.49577331542969,
      "activations/layer3_attention_weight_min": -88.41919708251953,
      "activations/layer4_attention_weight_max": 97.70465087890625,
      "activations/layer4_attention_weight_min": -92.2890853881836,
      "activations/layer5_attention_weight_max": 90.68241882324219,
      "activations/layer5_attention_weight_min": -84.130615234375,
      "activations/layer6_attention_weight_max": 56.94115447998047,
      "activations/layer6_attention_weight_min": -56.217926025390625,
      "activations/layer7_attention_weight_max": 82.16272735595703,
      "activations/layer7_attention_weight_min": -71.50963592529297,
      "activations/layer8_attention_weight_max": 46.07337951660156,
      "activations/layer8_attention_weight_min": -45.151363372802734,
      "activations/layer9_attention_weight_max": 49.309688568115234,
      "activations/layer9_attention_weight_min": -48.91865158081055,
      "epoch": 7.12,
      "learning_rate": 0.00010515757575757574,
      "loss": 2.8164,
      "step": 122500
    },
    {
      "activations/layer0_attention_weight_max": 13.741206169128418,
      "activations/layer0_attention_weight_min": -15.227158546447754,
      "activations/layer10_attention_weight_max": 40.421932220458984,
      "activations/layer10_attention_weight_min": -36.199928283691406,
      "activations/layer11_attention_weight_max": 34.66205596923828,
      "activations/layer11_attention_weight_min": -30.393314361572266,
      "activations/layer12_attention_weight_max": 26.254047393798828,
      "activations/layer12_attention_weight_min": -24.46155548095703,
      "activations/layer13_attention_weight_max": 55.068023681640625,
      "activations/layer13_attention_weight_min": -33.25848388671875,
      "activations/layer14_attention_weight_max": 48.88198471069336,
      "activations/layer14_attention_weight_min": -31.061975479125977,
      "activations/layer15_attention_weight_max": 47.457157135009766,
      "activations/layer15_attention_weight_min": -30.030654907226562,
      "activations/layer16_attention_weight_max": 38.03148651123047,
      "activations/layer16_attention_weight_min": -27.7265567779541,
      "activations/layer17_attention_weight_max": 63.461669921875,
      "activations/layer17_attention_weight_min": -44.25918960571289,
      "activations/layer18_attention_weight_max": 57.936702728271484,
      "activations/layer18_attention_weight_min": -36.80337142944336,
      "activations/layer19_attention_weight_max": 24.15685272216797,
      "activations/layer19_attention_weight_min": -22.161514282226562,
      "activations/layer1_attention_weight_max": 17.00331687927246,
      "activations/layer1_attention_weight_min": -17.107301712036133,
      "activations/layer20_attention_weight_max": 26.28231430053711,
      "activations/layer20_attention_weight_min": -22.103618621826172,
      "activations/layer21_attention_weight_max": 46.93880081176758,
      "activations/layer21_attention_weight_min": -26.35197639465332,
      "activations/layer22_attention_weight_max": 42.7018928527832,
      "activations/layer22_attention_weight_min": -25.005233764648438,
      "activations/layer23_attention_weight_max": 42.62784194946289,
      "activations/layer23_attention_weight_min": -20.469280242919922,
      "activations/layer2_attention_weight_max": 31.263547897338867,
      "activations/layer2_attention_weight_min": -30.3099422454834,
      "activations/layer3_attention_weight_max": 84.6189193725586,
      "activations/layer3_attention_weight_min": -89.68528747558594,
      "activations/layer4_attention_weight_max": 99.8510971069336,
      "activations/layer4_attention_weight_min": -93.10015106201172,
      "activations/layer5_attention_weight_max": 88.33275604248047,
      "activations/layer5_attention_weight_min": -85.03965759277344,
      "activations/layer6_attention_weight_max": 56.81977462768555,
      "activations/layer6_attention_weight_min": -52.586158752441406,
      "activations/layer7_attention_weight_max": 78.14933013916016,
      "activations/layer7_attention_weight_min": -72.19471740722656,
      "activations/layer8_attention_weight_max": 44.71297073364258,
      "activations/layer8_attention_weight_min": -46.872283935546875,
      "activations/layer9_attention_weight_max": 53.203487396240234,
      "activations/layer9_attention_weight_min": -52.881019592285156,
      "epoch": 7.12,
      "learning_rate": 0.00010513863636363636,
      "loss": 2.8126,
      "step": 122550
    },
    {
      "activations/layer0_attention_weight_max": 14.739134788513184,
      "activations/layer0_attention_weight_min": -15.040495872497559,
      "activations/layer10_attention_weight_max": 39.78316879272461,
      "activations/layer10_attention_weight_min": -38.88685607910156,
      "activations/layer11_attention_weight_max": 32.86741638183594,
      "activations/layer11_attention_weight_min": -32.1475944519043,
      "activations/layer12_attention_weight_max": 22.261573791503906,
      "activations/layer12_attention_weight_min": -30.497512817382812,
      "activations/layer13_attention_weight_max": 45.128639221191406,
      "activations/layer13_attention_weight_min": -35.50886154174805,
      "activations/layer14_attention_weight_max": 41.900718688964844,
      "activations/layer14_attention_weight_min": -30.206661224365234,
      "activations/layer15_attention_weight_max": 35.164920806884766,
      "activations/layer15_attention_weight_min": -29.036590576171875,
      "activations/layer16_attention_weight_max": 31.16343116760254,
      "activations/layer16_attention_weight_min": -26.199138641357422,
      "activations/layer17_attention_weight_max": 54.03480911254883,
      "activations/layer17_attention_weight_min": -42.643009185791016,
      "activations/layer18_attention_weight_max": 47.17626953125,
      "activations/layer18_attention_weight_min": -38.49674987792969,
      "activations/layer19_attention_weight_max": 27.166290283203125,
      "activations/layer19_attention_weight_min": -21.08662986755371,
      "activations/layer1_attention_weight_max": 17.087078094482422,
      "activations/layer1_attention_weight_min": -15.375147819519043,
      "activations/layer20_attention_weight_max": 23.486713409423828,
      "activations/layer20_attention_weight_min": -21.17284393310547,
      "activations/layer21_attention_weight_max": 42.221317291259766,
      "activations/layer21_attention_weight_min": -24.2120418548584,
      "activations/layer22_attention_weight_max": 35.28333282470703,
      "activations/layer22_attention_weight_min": -25.080726623535156,
      "activations/layer23_attention_weight_max": 36.56803512573242,
      "activations/layer23_attention_weight_min": -20.49817657470703,
      "activations/layer2_attention_weight_max": 29.924589157104492,
      "activations/layer2_attention_weight_min": -28.31219482421875,
      "activations/layer3_attention_weight_max": 84.72723388671875,
      "activations/layer3_attention_weight_min": -87.41535186767578,
      "activations/layer4_attention_weight_max": 97.38994598388672,
      "activations/layer4_attention_weight_min": -92.9098129272461,
      "activations/layer5_attention_weight_max": 92.69554138183594,
      "activations/layer5_attention_weight_min": -84.23106384277344,
      "activations/layer6_attention_weight_max": 57.09123229980469,
      "activations/layer6_attention_weight_min": -50.55852508544922,
      "activations/layer7_attention_weight_max": 75.3258056640625,
      "activations/layer7_attention_weight_min": -71.29154968261719,
      "activations/layer8_attention_weight_max": 46.919715881347656,
      "activations/layer8_attention_weight_min": -46.52362823486328,
      "activations/layer9_attention_weight_max": 51.453060150146484,
      "activations/layer9_attention_weight_min": -52.157623291015625,
      "epoch": 7.12,
      "learning_rate": 0.00010511969696969696,
      "loss": 2.8215,
      "step": 122600
    },
    {
      "activations/layer0_attention_weight_max": 14.632646560668945,
      "activations/layer0_attention_weight_min": -15.148036003112793,
      "activations/layer10_attention_weight_max": 40.24066925048828,
      "activations/layer10_attention_weight_min": -35.64691925048828,
      "activations/layer11_attention_weight_max": 31.550182342529297,
      "activations/layer11_attention_weight_min": -30.272756576538086,
      "activations/layer12_attention_weight_max": 20.05938148498535,
      "activations/layer12_attention_weight_min": -25.2930908203125,
      "activations/layer13_attention_weight_max": 43.34076690673828,
      "activations/layer13_attention_weight_min": -28.65350341796875,
      "activations/layer14_attention_weight_max": 38.252559661865234,
      "activations/layer14_attention_weight_min": -29.6580810546875,
      "activations/layer15_attention_weight_max": 39.912803649902344,
      "activations/layer15_attention_weight_min": -28.789073944091797,
      "activations/layer16_attention_weight_max": 30.77567481994629,
      "activations/layer16_attention_weight_min": -26.293031692504883,
      "activations/layer17_attention_weight_max": 58.28676223754883,
      "activations/layer17_attention_weight_min": -39.6805419921875,
      "activations/layer18_attention_weight_max": 50.59909439086914,
      "activations/layer18_attention_weight_min": -33.792232513427734,
      "activations/layer19_attention_weight_max": 21.185680389404297,
      "activations/layer19_attention_weight_min": -19.20234489440918,
      "activations/layer1_attention_weight_max": 17.562307357788086,
      "activations/layer1_attention_weight_min": -14.264341354370117,
      "activations/layer20_attention_weight_max": 21.925142288208008,
      "activations/layer20_attention_weight_min": -20.318510055541992,
      "activations/layer21_attention_weight_max": 37.1562385559082,
      "activations/layer21_attention_weight_min": -22.2905216217041,
      "activations/layer22_attention_weight_max": 30.76012420654297,
      "activations/layer22_attention_weight_min": -25.216754913330078,
      "activations/layer23_attention_weight_max": 31.47869110107422,
      "activations/layer23_attention_weight_min": -20.406265258789062,
      "activations/layer2_attention_weight_max": 32.89543914794922,
      "activations/layer2_attention_weight_min": -29.493362426757812,
      "activations/layer3_attention_weight_max": 84.7157974243164,
      "activations/layer3_attention_weight_min": -84.97514343261719,
      "activations/layer4_attention_weight_max": 99.99173736572266,
      "activations/layer4_attention_weight_min": -89.89802551269531,
      "activations/layer5_attention_weight_max": 89.84966278076172,
      "activations/layer5_attention_weight_min": -80.50920867919922,
      "activations/layer6_attention_weight_max": 53.70084762573242,
      "activations/layer6_attention_weight_min": -50.45536804199219,
      "activations/layer7_attention_weight_max": 71.13299560546875,
      "activations/layer7_attention_weight_min": -70.33372497558594,
      "activations/layer8_attention_weight_max": 42.70854568481445,
      "activations/layer8_attention_weight_min": -42.80158615112305,
      "activations/layer9_attention_weight_max": 49.404788970947266,
      "activations/layer9_attention_weight_min": -49.460201263427734,
      "epoch": 7.13,
      "learning_rate": 0.00010510075757575757,
      "loss": 2.8118,
      "step": 122650
    },
    {
      "activations/layer0_attention_weight_max": 16.086626052856445,
      "activations/layer0_attention_weight_min": -14.334030151367188,
      "activations/layer10_attention_weight_max": 44.294002532958984,
      "activations/layer10_attention_weight_min": -37.38831329345703,
      "activations/layer11_attention_weight_max": 34.250064849853516,
      "activations/layer11_attention_weight_min": -33.43733596801758,
      "activations/layer12_attention_weight_max": 21.211929321289062,
      "activations/layer12_attention_weight_min": -24.378585815429688,
      "activations/layer13_attention_weight_max": 46.39543151855469,
      "activations/layer13_attention_weight_min": -33.28668975830078,
      "activations/layer14_attention_weight_max": 42.40095901489258,
      "activations/layer14_attention_weight_min": -31.808696746826172,
      "activations/layer15_attention_weight_max": 41.12751388549805,
      "activations/layer15_attention_weight_min": -31.840227127075195,
      "activations/layer16_attention_weight_max": 39.30347442626953,
      "activations/layer16_attention_weight_min": -28.446441650390625,
      "activations/layer17_attention_weight_max": 64.39314270019531,
      "activations/layer17_attention_weight_min": -45.489383697509766,
      "activations/layer18_attention_weight_max": 57.12841033935547,
      "activations/layer18_attention_weight_min": -38.170509338378906,
      "activations/layer19_attention_weight_max": 23.744314193725586,
      "activations/layer19_attention_weight_min": -20.45503044128418,
      "activations/layer1_attention_weight_max": 17.341981887817383,
      "activations/layer1_attention_weight_min": -14.164648056030273,
      "activations/layer20_attention_weight_max": 24.876232147216797,
      "activations/layer20_attention_weight_min": -22.49186134338379,
      "activations/layer21_attention_weight_max": 41.42340087890625,
      "activations/layer21_attention_weight_min": -24.240734100341797,
      "activations/layer22_attention_weight_max": 39.151390075683594,
      "activations/layer22_attention_weight_min": -26.812633514404297,
      "activations/layer23_attention_weight_max": 40.83954620361328,
      "activations/layer23_attention_weight_min": -19.524940490722656,
      "activations/layer2_attention_weight_max": 31.659276962280273,
      "activations/layer2_attention_weight_min": -31.255918502807617,
      "activations/layer3_attention_weight_max": 83.38369750976562,
      "activations/layer3_attention_weight_min": -89.73330688476562,
      "activations/layer4_attention_weight_max": 100.94218444824219,
      "activations/layer4_attention_weight_min": -98.83576965332031,
      "activations/layer5_attention_weight_max": 95.93856048583984,
      "activations/layer5_attention_weight_min": -87.24325561523438,
      "activations/layer6_attention_weight_max": 60.58151626586914,
      "activations/layer6_attention_weight_min": -55.82978057861328,
      "activations/layer7_attention_weight_max": 82.67500305175781,
      "activations/layer7_attention_weight_min": -78.37430572509766,
      "activations/layer8_attention_weight_max": 50.773582458496094,
      "activations/layer8_attention_weight_min": -47.37297439575195,
      "activations/layer9_attention_weight_max": 55.94073486328125,
      "activations/layer9_attention_weight_min": -49.87517547607422,
      "epoch": 7.13,
      "learning_rate": 0.00010508181818181817,
      "loss": 2.8014,
      "step": 122700
    },
    {
      "activations/layer0_attention_weight_max": 15.424124717712402,
      "activations/layer0_attention_weight_min": -15.284093856811523,
      "activations/layer10_attention_weight_max": 38.53874969482422,
      "activations/layer10_attention_weight_min": -36.54866409301758,
      "activations/layer11_attention_weight_max": 31.884197235107422,
      "activations/layer11_attention_weight_min": -31.34466552734375,
      "activations/layer12_attention_weight_max": 24.775346755981445,
      "activations/layer12_attention_weight_min": -25.09388542175293,
      "activations/layer13_attention_weight_max": 48.48067855834961,
      "activations/layer13_attention_weight_min": -33.662105560302734,
      "activations/layer14_attention_weight_max": 44.26984786987305,
      "activations/layer14_attention_weight_min": -31.123624801635742,
      "activations/layer15_attention_weight_max": 37.304012298583984,
      "activations/layer15_attention_weight_min": -30.131399154663086,
      "activations/layer16_attention_weight_max": 33.31491470336914,
      "activations/layer16_attention_weight_min": -27.53448486328125,
      "activations/layer17_attention_weight_max": 62.16407012939453,
      "activations/layer17_attention_weight_min": -43.37188720703125,
      "activations/layer18_attention_weight_max": 54.677791595458984,
      "activations/layer18_attention_weight_min": -37.1581916809082,
      "activations/layer19_attention_weight_max": 22.44432830810547,
      "activations/layer19_attention_weight_min": -22.029457092285156,
      "activations/layer1_attention_weight_max": 17.882564544677734,
      "activations/layer1_attention_weight_min": -14.287345886230469,
      "activations/layer20_attention_weight_max": 24.741361618041992,
      "activations/layer20_attention_weight_min": -19.923032760620117,
      "activations/layer21_attention_weight_max": 39.885616302490234,
      "activations/layer21_attention_weight_min": -23.651226043701172,
      "activations/layer22_attention_weight_max": 35.73480987548828,
      "activations/layer22_attention_weight_min": -26.27242088317871,
      "activations/layer23_attention_weight_max": 34.21546173095703,
      "activations/layer23_attention_weight_min": -21.375465393066406,
      "activations/layer2_attention_weight_max": 30.69808006286621,
      "activations/layer2_attention_weight_min": -30.056476593017578,
      "activations/layer3_attention_weight_max": 82.7821273803711,
      "activations/layer3_attention_weight_min": -84.16757202148438,
      "activations/layer4_attention_weight_max": 99.72787475585938,
      "activations/layer4_attention_weight_min": -95.05987548828125,
      "activations/layer5_attention_weight_max": 91.15748596191406,
      "activations/layer5_attention_weight_min": -83.13397216796875,
      "activations/layer6_attention_weight_max": 57.280052185058594,
      "activations/layer6_attention_weight_min": -54.273582458496094,
      "activations/layer7_attention_weight_max": 82.81893920898438,
      "activations/layer7_attention_weight_min": -80.06391906738281,
      "activations/layer8_attention_weight_max": 45.93557357788086,
      "activations/layer8_attention_weight_min": -45.71931076049805,
      "activations/layer9_attention_weight_max": 50.965476989746094,
      "activations/layer9_attention_weight_min": -51.16992950439453,
      "epoch": 7.13,
      "learning_rate": 0.00010506287878787877,
      "loss": 2.8032,
      "step": 122750
    },
    {
      "activations/layer0_attention_weight_max": 14.077278137207031,
      "activations/layer0_attention_weight_min": -14.910857200622559,
      "activations/layer10_attention_weight_max": 39.630489349365234,
      "activations/layer10_attention_weight_min": -33.517250061035156,
      "activations/layer11_attention_weight_max": 29.97054672241211,
      "activations/layer11_attention_weight_min": -30.923389434814453,
      "activations/layer12_attention_weight_max": 20.147153854370117,
      "activations/layer12_attention_weight_min": -24.035417556762695,
      "activations/layer13_attention_weight_max": 61.598243713378906,
      "activations/layer13_attention_weight_min": -38.3264045715332,
      "activations/layer14_attention_weight_max": 61.55290985107422,
      "activations/layer14_attention_weight_min": -34.416473388671875,
      "activations/layer15_attention_weight_max": 48.33216857910156,
      "activations/layer15_attention_weight_min": -32.66579818725586,
      "activations/layer16_attention_weight_max": 40.61447525024414,
      "activations/layer16_attention_weight_min": -27.458309173583984,
      "activations/layer17_attention_weight_max": 63.94065475463867,
      "activations/layer17_attention_weight_min": -49.116493225097656,
      "activations/layer18_attention_weight_max": 57.539634704589844,
      "activations/layer18_attention_weight_min": -42.79556655883789,
      "activations/layer19_attention_weight_max": 23.821800231933594,
      "activations/layer19_attention_weight_min": -22.813945770263672,
      "activations/layer1_attention_weight_max": 16.68548011779785,
      "activations/layer1_attention_weight_min": -15.654191970825195,
      "activations/layer20_attention_weight_max": 27.247833251953125,
      "activations/layer20_attention_weight_min": -24.940959930419922,
      "activations/layer21_attention_weight_max": 57.219207763671875,
      "activations/layer21_attention_weight_min": -31.652618408203125,
      "activations/layer22_attention_weight_max": 36.9528694152832,
      "activations/layer22_attention_weight_min": -27.095924377441406,
      "activations/layer23_attention_weight_max": 38.266822814941406,
      "activations/layer23_attention_weight_min": -21.127033233642578,
      "activations/layer2_attention_weight_max": 30.9725399017334,
      "activations/layer2_attention_weight_min": -30.099365234375,
      "activations/layer3_attention_weight_max": 82.3229751586914,
      "activations/layer3_attention_weight_min": -87.2544174194336,
      "activations/layer4_attention_weight_max": 94.15936279296875,
      "activations/layer4_attention_weight_min": -91.15951538085938,
      "activations/layer5_attention_weight_max": 86.30987548828125,
      "activations/layer5_attention_weight_min": -79.50139617919922,
      "activations/layer6_attention_weight_max": 56.143924713134766,
      "activations/layer6_attention_weight_min": -51.96862030029297,
      "activations/layer7_attention_weight_max": 71.27886962890625,
      "activations/layer7_attention_weight_min": -70.85839080810547,
      "activations/layer8_attention_weight_max": 45.037288665771484,
      "activations/layer8_attention_weight_min": -43.37083435058594,
      "activations/layer9_attention_weight_max": 49.19090270996094,
      "activations/layer9_attention_weight_min": -46.6224479675293,
      "epoch": 7.14,
      "learning_rate": 0.00010504393939393939,
      "loss": 2.8173,
      "step": 122800
    },
    {
      "activations/layer0_attention_weight_max": 17.866432189941406,
      "activations/layer0_attention_weight_min": -15.026323318481445,
      "activations/layer10_attention_weight_max": 46.358642578125,
      "activations/layer10_attention_weight_min": -37.9669189453125,
      "activations/layer11_attention_weight_max": 38.159828186035156,
      "activations/layer11_attention_weight_min": -34.473548889160156,
      "activations/layer12_attention_weight_max": 23.828292846679688,
      "activations/layer12_attention_weight_min": -23.654468536376953,
      "activations/layer13_attention_weight_max": 61.03555679321289,
      "activations/layer13_attention_weight_min": -37.7746696472168,
      "activations/layer14_attention_weight_max": 65.31175994873047,
      "activations/layer14_attention_weight_min": -37.25544738769531,
      "activations/layer15_attention_weight_max": 48.47079086303711,
      "activations/layer15_attention_weight_min": -30.735204696655273,
      "activations/layer16_attention_weight_max": 38.48598098754883,
      "activations/layer16_attention_weight_min": -28.106157302856445,
      "activations/layer17_attention_weight_max": 58.16798400878906,
      "activations/layer17_attention_weight_min": -42.82115173339844,
      "activations/layer18_attention_weight_max": 50.92837905883789,
      "activations/layer18_attention_weight_min": -35.69429016113281,
      "activations/layer19_attention_weight_max": 26.18245506286621,
      "activations/layer19_attention_weight_min": -21.966354370117188,
      "activations/layer1_attention_weight_max": 17.561059951782227,
      "activations/layer1_attention_weight_min": -14.975030899047852,
      "activations/layer20_attention_weight_max": 25.24502182006836,
      "activations/layer20_attention_weight_min": -21.34337043762207,
      "activations/layer21_attention_weight_max": 48.80281448364258,
      "activations/layer21_attention_weight_min": -27.355712890625,
      "activations/layer22_attention_weight_max": 35.34954833984375,
      "activations/layer22_attention_weight_min": -27.910512924194336,
      "activations/layer23_attention_weight_max": 34.058937072753906,
      "activations/layer23_attention_weight_min": -22.051908493041992,
      "activations/layer2_attention_weight_max": 33.993526458740234,
      "activations/layer2_attention_weight_min": -29.618305206298828,
      "activations/layer3_attention_weight_max": 89.45814514160156,
      "activations/layer3_attention_weight_min": -91.99687194824219,
      "activations/layer4_attention_weight_max": 97.95690155029297,
      "activations/layer4_attention_weight_min": -97.45526123046875,
      "activations/layer5_attention_weight_max": 90.99836730957031,
      "activations/layer5_attention_weight_min": -85.44139862060547,
      "activations/layer6_attention_weight_max": 60.177528381347656,
      "activations/layer6_attention_weight_min": -52.79377746582031,
      "activations/layer7_attention_weight_max": 84.14016723632812,
      "activations/layer7_attention_weight_min": -78.43315887451172,
      "activations/layer8_attention_weight_max": 57.59230041503906,
      "activations/layer8_attention_weight_min": -47.917606353759766,
      "activations/layer9_attention_weight_max": 65.24562072753906,
      "activations/layer9_attention_weight_min": -50.94094467163086,
      "epoch": 7.14,
      "learning_rate": 0.00010502499999999999,
      "loss": 2.809,
      "step": 122850
    },
    {
      "activations/layer0_attention_weight_max": 15.082942008972168,
      "activations/layer0_attention_weight_min": -14.942366600036621,
      "activations/layer10_attention_weight_max": 36.40419387817383,
      "activations/layer10_attention_weight_min": -33.61655807495117,
      "activations/layer11_attention_weight_max": 30.115005493164062,
      "activations/layer11_attention_weight_min": -29.29167938232422,
      "activations/layer12_attention_weight_max": 22.012001037597656,
      "activations/layer12_attention_weight_min": -24.547197341918945,
      "activations/layer13_attention_weight_max": 45.543006896972656,
      "activations/layer13_attention_weight_min": -32.89826202392578,
      "activations/layer14_attention_weight_max": 40.995914459228516,
      "activations/layer14_attention_weight_min": -29.424692153930664,
      "activations/layer15_attention_weight_max": 42.6188850402832,
      "activations/layer15_attention_weight_min": -30.819320678710938,
      "activations/layer16_attention_weight_max": 33.76377868652344,
      "activations/layer16_attention_weight_min": -28.63159942626953,
      "activations/layer17_attention_weight_max": 58.16130828857422,
      "activations/layer17_attention_weight_min": -41.534027099609375,
      "activations/layer18_attention_weight_max": 47.50511932373047,
      "activations/layer18_attention_weight_min": -36.98081970214844,
      "activations/layer19_attention_weight_max": 21.342058181762695,
      "activations/layer19_attention_weight_min": -19.839378356933594,
      "activations/layer1_attention_weight_max": 18.017948150634766,
      "activations/layer1_attention_weight_min": -16.97857093811035,
      "activations/layer20_attention_weight_max": 21.897483825683594,
      "activations/layer20_attention_weight_min": -24.149507522583008,
      "activations/layer21_attention_weight_max": 37.306312561035156,
      "activations/layer21_attention_weight_min": -22.714689254760742,
      "activations/layer22_attention_weight_max": 34.3873291015625,
      "activations/layer22_attention_weight_min": -26.387725830078125,
      "activations/layer23_attention_weight_max": 34.44216537475586,
      "activations/layer23_attention_weight_min": -20.617963790893555,
      "activations/layer2_attention_weight_max": 30.4610652923584,
      "activations/layer2_attention_weight_min": -29.043590545654297,
      "activations/layer3_attention_weight_max": 85.10482788085938,
      "activations/layer3_attention_weight_min": -87.56047821044922,
      "activations/layer4_attention_weight_max": 95.14924621582031,
      "activations/layer4_attention_weight_min": -91.39622497558594,
      "activations/layer5_attention_weight_max": 88.57511901855469,
      "activations/layer5_attention_weight_min": -80.44530487060547,
      "activations/layer6_attention_weight_max": 53.62936019897461,
      "activations/layer6_attention_weight_min": -51.3670768737793,
      "activations/layer7_attention_weight_max": 76.01580810546875,
      "activations/layer7_attention_weight_min": -68.07349395751953,
      "activations/layer8_attention_weight_max": 41.32822036743164,
      "activations/layer8_attention_weight_min": -44.582435607910156,
      "activations/layer9_attention_weight_max": 46.58022689819336,
      "activations/layer9_attention_weight_min": -50.05636978149414,
      "epoch": 7.14,
      "learning_rate": 0.00010500606060606059,
      "loss": 2.8012,
      "step": 122900
    },
    {
      "activations/layer0_attention_weight_max": 15.203118324279785,
      "activations/layer0_attention_weight_min": -14.58328914642334,
      "activations/layer10_attention_weight_max": 39.2390022277832,
      "activations/layer10_attention_weight_min": -36.21299362182617,
      "activations/layer11_attention_weight_max": 30.796417236328125,
      "activations/layer11_attention_weight_min": -31.538108825683594,
      "activations/layer12_attention_weight_max": 19.848161697387695,
      "activations/layer12_attention_weight_min": -23.60150146484375,
      "activations/layer13_attention_weight_max": 50.91279983520508,
      "activations/layer13_attention_weight_min": -32.32544708251953,
      "activations/layer14_attention_weight_max": 56.129295349121094,
      "activations/layer14_attention_weight_min": -32.00922775268555,
      "activations/layer15_attention_weight_max": 49.04032516479492,
      "activations/layer15_attention_weight_min": -29.995742797851562,
      "activations/layer16_attention_weight_max": 32.429222106933594,
      "activations/layer16_attention_weight_min": -26.851696014404297,
      "activations/layer17_attention_weight_max": 60.29933166503906,
      "activations/layer17_attention_weight_min": -42.91816329956055,
      "activations/layer18_attention_weight_max": 54.269866943359375,
      "activations/layer18_attention_weight_min": -36.573089599609375,
      "activations/layer19_attention_weight_max": 24.043676376342773,
      "activations/layer19_attention_weight_min": -20.4688720703125,
      "activations/layer1_attention_weight_max": 16.68901824951172,
      "activations/layer1_attention_weight_min": -16.518043518066406,
      "activations/layer20_attention_weight_max": 24.832256317138672,
      "activations/layer20_attention_weight_min": -21.42217254638672,
      "activations/layer21_attention_weight_max": 39.798824310302734,
      "activations/layer21_attention_weight_min": -23.87718963623047,
      "activations/layer22_attention_weight_max": 35.68546676635742,
      "activations/layer22_attention_weight_min": -24.28546142578125,
      "activations/layer23_attention_weight_max": 33.7515983581543,
      "activations/layer23_attention_weight_min": -21.299697875976562,
      "activations/layer2_attention_weight_max": 32.52734375,
      "activations/layer2_attention_weight_min": -31.082395553588867,
      "activations/layer3_attention_weight_max": 85.14325714111328,
      "activations/layer3_attention_weight_min": -92.96867370605469,
      "activations/layer4_attention_weight_max": 97.7917251586914,
      "activations/layer4_attention_weight_min": -93.32211303710938,
      "activations/layer5_attention_weight_max": 92.29885864257812,
      "activations/layer5_attention_weight_min": -81.20984649658203,
      "activations/layer6_attention_weight_max": 56.910865783691406,
      "activations/layer6_attention_weight_min": -56.01537322998047,
      "activations/layer7_attention_weight_max": 75.0152587890625,
      "activations/layer7_attention_weight_min": -68.07062530517578,
      "activations/layer8_attention_weight_max": 47.02737045288086,
      "activations/layer8_attention_weight_min": -46.66769027709961,
      "activations/layer9_attention_weight_max": 56.81195831298828,
      "activations/layer9_attention_weight_min": -48.231651306152344,
      "epoch": 7.14,
      "learning_rate": 0.00010498712121212121,
      "loss": 2.8242,
      "step": 122950
    },
    {
      "activations/layer0_attention_weight_max": 15.032829284667969,
      "activations/layer0_attention_weight_min": -14.79397964477539,
      "activations/layer10_attention_weight_max": 38.15381622314453,
      "activations/layer10_attention_weight_min": -35.38210678100586,
      "activations/layer11_attention_weight_max": 31.295970916748047,
      "activations/layer11_attention_weight_min": -29.94831657409668,
      "activations/layer12_attention_weight_max": 22.786420822143555,
      "activations/layer12_attention_weight_min": -24.277835845947266,
      "activations/layer13_attention_weight_max": 47.3554573059082,
      "activations/layer13_attention_weight_min": -35.08362579345703,
      "activations/layer14_attention_weight_max": 50.62394714355469,
      "activations/layer14_attention_weight_min": -32.32829284667969,
      "activations/layer15_attention_weight_max": 38.1306266784668,
      "activations/layer15_attention_weight_min": -30.51203155517578,
      "activations/layer16_attention_weight_max": 38.972930908203125,
      "activations/layer16_attention_weight_min": -27.841989517211914,
      "activations/layer17_attention_weight_max": 57.4501953125,
      "activations/layer17_attention_weight_min": -44.66239547729492,
      "activations/layer18_attention_weight_max": 52.84455871582031,
      "activations/layer18_attention_weight_min": -36.7903938293457,
      "activations/layer19_attention_weight_max": 25.522624969482422,
      "activations/layer19_attention_weight_min": -20.56694793701172,
      "activations/layer1_attention_weight_max": 17.783443450927734,
      "activations/layer1_attention_weight_min": -17.524003982543945,
      "activations/layer20_attention_weight_max": 23.61309051513672,
      "activations/layer20_attention_weight_min": -19.73082160949707,
      "activations/layer21_attention_weight_max": 46.526973724365234,
      "activations/layer21_attention_weight_min": -24.440771102905273,
      "activations/layer22_attention_weight_max": 32.52841567993164,
      "activations/layer22_attention_weight_min": -26.130693435668945,
      "activations/layer23_attention_weight_max": 31.46399688720703,
      "activations/layer23_attention_weight_min": -19.379497528076172,
      "activations/layer2_attention_weight_max": 33.40581512451172,
      "activations/layer2_attention_weight_min": -32.324073791503906,
      "activations/layer3_attention_weight_max": 81.99713134765625,
      "activations/layer3_attention_weight_min": -83.69467163085938,
      "activations/layer4_attention_weight_max": 93.00031280517578,
      "activations/layer4_attention_weight_min": -87.38860321044922,
      "activations/layer5_attention_weight_max": 86.93965911865234,
      "activations/layer5_attention_weight_min": -78.1158447265625,
      "activations/layer6_attention_weight_max": 55.165321350097656,
      "activations/layer6_attention_weight_min": -51.31218338012695,
      "activations/layer7_attention_weight_max": 76.50988006591797,
      "activations/layer7_attention_weight_min": -73.41645812988281,
      "activations/layer8_attention_weight_max": 46.515777587890625,
      "activations/layer8_attention_weight_min": -43.64792251586914,
      "activations/layer9_attention_weight_max": 58.0316047668457,
      "activations/layer9_attention_weight_min": -47.90935134887695,
      "epoch": 7.15,
      "learning_rate": 0.0001049681818181818,
      "loss": 2.838,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_loss": 2.771484375,
      "eval_runtime": 8.5462,
      "eval_samples_per_second": 502.447,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_openwebtext_loss": 2.771484375,
      "eval_openwebtext_ppl": 15.98234019723552,
      "eval_openwebtext_runtime": 8.5462,
      "eval_openwebtext_samples_per_second": 502.447,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 2.0054,
      "eval_wikitext_samples_per_second": 227.389,
      "step": 123000
    },
    {
      "epoch": 7.15,
      "eval_lambada_loss": 2.6953125,
      "eval_lambada_ppl": 14.81014620089916,
      "eval_lambada_runtime": 9.574,
      "eval_lambada_samples_per_second": 508.563,
      "step": 123000
    },
    {
      "activations/layer0_attention_weight_max": 15.079237937927246,
      "activations/layer0_attention_weight_min": -15.081277847290039,
      "activations/layer10_attention_weight_max": 43.035179138183594,
      "activations/layer10_attention_weight_min": -39.06951904296875,
      "activations/layer11_attention_weight_max": 34.64521789550781,
      "activations/layer11_attention_weight_min": -31.26711082458496,
      "activations/layer12_attention_weight_max": 21.789588928222656,
      "activations/layer12_attention_weight_min": -25.170881271362305,
      "activations/layer13_attention_weight_max": 41.82511901855469,
      "activations/layer13_attention_weight_min": -30.080984115600586,
      "activations/layer14_attention_weight_max": 38.01968002319336,
      "activations/layer14_attention_weight_min": -29.69972038269043,
      "activations/layer15_attention_weight_max": 34.83263397216797,
      "activations/layer15_attention_weight_min": -29.75884246826172,
      "activations/layer16_attention_weight_max": 34.472572326660156,
      "activations/layer16_attention_weight_min": -26.137794494628906,
      "activations/layer17_attention_weight_max": 53.170562744140625,
      "activations/layer17_attention_weight_min": -42.64809799194336,
      "activations/layer18_attention_weight_max": 47.019798278808594,
      "activations/layer18_attention_weight_min": -35.5678825378418,
      "activations/layer19_attention_weight_max": 19.62428092956543,
      "activations/layer19_attention_weight_min": -20.12909507751465,
      "activations/layer1_attention_weight_max": 16.74256134033203,
      "activations/layer1_attention_weight_min": -15.240232467651367,
      "activations/layer20_attention_weight_max": 22.332277297973633,
      "activations/layer20_attention_weight_min": -21.945920944213867,
      "activations/layer21_attention_weight_max": 42.45009231567383,
      "activations/layer21_attention_weight_min": -24.150793075561523,
      "activations/layer22_attention_weight_max": 32.512332916259766,
      "activations/layer22_attention_weight_min": -22.59674835205078,
      "activations/layer23_attention_weight_max": 32.06788635253906,
      "activations/layer23_attention_weight_min": -21.977432250976562,
      "activations/layer2_attention_weight_max": 31.69070053100586,
      "activations/layer2_attention_weight_min": -29.791210174560547,
      "activations/layer3_attention_weight_max": 81.86798858642578,
      "activations/layer3_attention_weight_min": -84.94566345214844,
      "activations/layer4_attention_weight_max": 97.59979248046875,
      "activations/layer4_attention_weight_min": -93.13410186767578,
      "activations/layer5_attention_weight_max": 89.11177825927734,
      "activations/layer5_attention_weight_min": -80.91804504394531,
      "activations/layer6_attention_weight_max": 55.27972412109375,
      "activations/layer6_attention_weight_min": -53.038631439208984,
      "activations/layer7_attention_weight_max": 78.21430969238281,
      "activations/layer7_attention_weight_min": -76.23848724365234,
      "activations/layer8_attention_weight_max": 46.844451904296875,
      "activations/layer8_attention_weight_min": -46.419342041015625,
      "activations/layer9_attention_weight_max": 63.00545120239258,
      "activations/layer9_attention_weight_min": -51.165679931640625,
      "epoch": 7.15,
      "learning_rate": 0.00010494924242424241,
      "loss": 2.8182,
      "step": 123050
    },
    {
      "activations/layer0_attention_weight_max": 15.790464401245117,
      "activations/layer0_attention_weight_min": -14.572427749633789,
      "activations/layer10_attention_weight_max": 37.7379150390625,
      "activations/layer10_attention_weight_min": -35.97186279296875,
      "activations/layer11_attention_weight_max": 31.04977798461914,
      "activations/layer11_attention_weight_min": -30.58331298828125,
      "activations/layer12_attention_weight_max": 16.526437759399414,
      "activations/layer12_attention_weight_min": -23.372093200683594,
      "activations/layer13_attention_weight_max": 40.3999137878418,
      "activations/layer13_attention_weight_min": -31.15474510192871,
      "activations/layer14_attention_weight_max": 42.79758071899414,
      "activations/layer14_attention_weight_min": -28.365190505981445,
      "activations/layer15_attention_weight_max": 34.35275650024414,
      "activations/layer15_attention_weight_min": -28.467578887939453,
      "activations/layer16_attention_weight_max": 28.115215301513672,
      "activations/layer16_attention_weight_min": -26.36334991455078,
      "activations/layer17_attention_weight_max": 50.19456100463867,
      "activations/layer17_attention_weight_min": -39.32624435424805,
      "activations/layer18_attention_weight_max": 39.546669006347656,
      "activations/layer18_attention_weight_min": -35.57316207885742,
      "activations/layer19_attention_weight_max": 17.535846710205078,
      "activations/layer19_attention_weight_min": -22.3154354095459,
      "activations/layer1_attention_weight_max": 16.102840423583984,
      "activations/layer1_attention_weight_min": -15.717262268066406,
      "activations/layer20_attention_weight_max": 18.590185165405273,
      "activations/layer20_attention_weight_min": -21.594785690307617,
      "activations/layer21_attention_weight_max": 31.54221534729004,
      "activations/layer21_attention_weight_min": -24.48740577697754,
      "activations/layer22_attention_weight_max": 29.57552146911621,
      "activations/layer22_attention_weight_min": -23.5168399810791,
      "activations/layer23_attention_weight_max": 27.145803451538086,
      "activations/layer23_attention_weight_min": -22.230979919433594,
      "activations/layer2_attention_weight_max": 32.278717041015625,
      "activations/layer2_attention_weight_min": -29.859012603759766,
      "activations/layer3_attention_weight_max": 86.35054016113281,
      "activations/layer3_attention_weight_min": -86.62410736083984,
      "activations/layer4_attention_weight_max": 103.19744873046875,
      "activations/layer4_attention_weight_min": -93.35999298095703,
      "activations/layer5_attention_weight_max": 92.45967102050781,
      "activations/layer5_attention_weight_min": -84.80178833007812,
      "activations/layer6_attention_weight_max": 60.51911163330078,
      "activations/layer6_attention_weight_min": -57.08584976196289,
      "activations/layer7_attention_weight_max": 74.36686706542969,
      "activations/layer7_attention_weight_min": -75.86823272705078,
      "activations/layer8_attention_weight_max": 44.82788848876953,
      "activations/layer8_attention_weight_min": -43.97380828857422,
      "activations/layer9_attention_weight_max": 52.56303024291992,
      "activations/layer9_attention_weight_min": -52.12207794189453,
      "epoch": 7.15,
      "learning_rate": 0.00010493030303030303,
      "loss": 2.8152,
      "step": 123100
    },
    {
      "activations/layer0_attention_weight_max": 15.441786766052246,
      "activations/layer0_attention_weight_min": -14.536014556884766,
      "activations/layer10_attention_weight_max": 39.29022979736328,
      "activations/layer10_attention_weight_min": -36.71942138671875,
      "activations/layer11_attention_weight_max": 34.905433654785156,
      "activations/layer11_attention_weight_min": -30.176639556884766,
      "activations/layer12_attention_weight_max": 25.743019104003906,
      "activations/layer12_attention_weight_min": -24.2622127532959,
      "activations/layer13_attention_weight_max": 55.33732986450195,
      "activations/layer13_attention_weight_min": -36.57137680053711,
      "activations/layer14_attention_weight_max": 51.99796676635742,
      "activations/layer14_attention_weight_min": -32.83585739135742,
      "activations/layer15_attention_weight_max": 41.05203628540039,
      "activations/layer15_attention_weight_min": -29.149917602539062,
      "activations/layer16_attention_weight_max": 34.48626708984375,
      "activations/layer16_attention_weight_min": -27.111356735229492,
      "activations/layer17_attention_weight_max": 56.7172737121582,
      "activations/layer17_attention_weight_min": -41.9879264831543,
      "activations/layer18_attention_weight_max": 55.13465118408203,
      "activations/layer18_attention_weight_min": -35.945438385009766,
      "activations/layer19_attention_weight_max": 23.40108299255371,
      "activations/layer19_attention_weight_min": -20.34611701965332,
      "activations/layer1_attention_weight_max": 16.603628158569336,
      "activations/layer1_attention_weight_min": -14.722000122070312,
      "activations/layer20_attention_weight_max": 26.441143035888672,
      "activations/layer20_attention_weight_min": -25.757957458496094,
      "activations/layer21_attention_weight_max": 45.45309066772461,
      "activations/layer21_attention_weight_min": -26.514047622680664,
      "activations/layer22_attention_weight_max": 36.59656524658203,
      "activations/layer22_attention_weight_min": -24.855806350708008,
      "activations/layer23_attention_weight_max": 41.22544860839844,
      "activations/layer23_attention_weight_min": -20.06684112548828,
      "activations/layer2_attention_weight_max": 30.39417266845703,
      "activations/layer2_attention_weight_min": -28.71387481689453,
      "activations/layer3_attention_weight_max": 86.89132690429688,
      "activations/layer3_attention_weight_min": -89.19393157958984,
      "activations/layer4_attention_weight_max": 98.46172332763672,
      "activations/layer4_attention_weight_min": -94.68062591552734,
      "activations/layer5_attention_weight_max": 91.47749328613281,
      "activations/layer5_attention_weight_min": -81.98282623291016,
      "activations/layer6_attention_weight_max": 55.93503189086914,
      "activations/layer6_attention_weight_min": -52.297672271728516,
      "activations/layer7_attention_weight_max": 76.58967590332031,
      "activations/layer7_attention_weight_min": -73.74293518066406,
      "activations/layer8_attention_weight_max": 46.13600540161133,
      "activations/layer8_attention_weight_min": -45.95568084716797,
      "activations/layer9_attention_weight_max": 49.56093215942383,
      "activations/layer9_attention_weight_min": -46.98208236694336,
      "epoch": 7.16,
      "learning_rate": 0.00010491136363636363,
      "loss": 2.8135,
      "step": 123150
    },
    {
      "activations/layer0_attention_weight_max": 13.969396591186523,
      "activations/layer0_attention_weight_min": -14.694543838500977,
      "activations/layer10_attention_weight_max": 40.78385543823242,
      "activations/layer10_attention_weight_min": -37.861175537109375,
      "activations/layer11_attention_weight_max": 32.20071792602539,
      "activations/layer11_attention_weight_min": -28.65723991394043,
      "activations/layer12_attention_weight_max": 24.988107681274414,
      "activations/layer12_attention_weight_min": -21.03798484802246,
      "activations/layer13_attention_weight_max": 50.09322738647461,
      "activations/layer13_attention_weight_min": -31.833892822265625,
      "activations/layer14_attention_weight_max": 40.38141632080078,
      "activations/layer14_attention_weight_min": -30.144304275512695,
      "activations/layer15_attention_weight_max": 42.3069953918457,
      "activations/layer15_attention_weight_min": -28.97760009765625,
      "activations/layer16_attention_weight_max": 33.740970611572266,
      "activations/layer16_attention_weight_min": -25.388856887817383,
      "activations/layer17_attention_weight_max": 52.34617233276367,
      "activations/layer17_attention_weight_min": -40.85087585449219,
      "activations/layer18_attention_weight_max": 48.73844528198242,
      "activations/layer18_attention_weight_min": -34.56553649902344,
      "activations/layer19_attention_weight_max": 23.23959732055664,
      "activations/layer19_attention_weight_min": -19.472158432006836,
      "activations/layer1_attention_weight_max": 16.223588943481445,
      "activations/layer1_attention_weight_min": -13.967179298400879,
      "activations/layer20_attention_weight_max": 21.255905151367188,
      "activations/layer20_attention_weight_min": -25.110595703125,
      "activations/layer21_attention_weight_max": 33.53880310058594,
      "activations/layer21_attention_weight_min": -23.710796356201172,
      "activations/layer22_attention_weight_max": 35.8222541809082,
      "activations/layer22_attention_weight_min": -23.639083862304688,
      "activations/layer23_attention_weight_max": 35.7227783203125,
      "activations/layer23_attention_weight_min": -20.319856643676758,
      "activations/layer2_attention_weight_max": 31.184165954589844,
      "activations/layer2_attention_weight_min": -29.653974533081055,
      "activations/layer3_attention_weight_max": 84.37847900390625,
      "activations/layer3_attention_weight_min": -81.57005310058594,
      "activations/layer4_attention_weight_max": 92.80562591552734,
      "activations/layer4_attention_weight_min": -89.64502716064453,
      "activations/layer5_attention_weight_max": 86.83769226074219,
      "activations/layer5_attention_weight_min": -82.80133819580078,
      "activations/layer6_attention_weight_max": 52.924007415771484,
      "activations/layer6_attention_weight_min": -53.458839416503906,
      "activations/layer7_attention_weight_max": 72.56830596923828,
      "activations/layer7_attention_weight_min": -72.0591049194336,
      "activations/layer8_attention_weight_max": 43.72050476074219,
      "activations/layer8_attention_weight_min": -45.7325553894043,
      "activations/layer9_attention_weight_max": 53.4993896484375,
      "activations/layer9_attention_weight_min": -49.80712127685547,
      "epoch": 7.16,
      "learning_rate": 0.00010489242424242423,
      "loss": 2.8077,
      "step": 123200
    },
    {
      "activations/layer0_attention_weight_max": 17.589248657226562,
      "activations/layer0_attention_weight_min": -14.563167572021484,
      "activations/layer10_attention_weight_max": 38.66136932373047,
      "activations/layer10_attention_weight_min": -36.634273529052734,
      "activations/layer11_attention_weight_max": 31.212553024291992,
      "activations/layer11_attention_weight_min": -33.6074104309082,
      "activations/layer12_attention_weight_max": 19.86979103088379,
      "activations/layer12_attention_weight_min": -28.428733825683594,
      "activations/layer13_attention_weight_max": 49.123680114746094,
      "activations/layer13_attention_weight_min": -33.61481857299805,
      "activations/layer14_attention_weight_max": 36.99965286254883,
      "activations/layer14_attention_weight_min": -30.775550842285156,
      "activations/layer15_attention_weight_max": 36.571407318115234,
      "activations/layer15_attention_weight_min": -30.447872161865234,
      "activations/layer16_attention_weight_max": 31.803585052490234,
      "activations/layer16_attention_weight_min": -30.08087730407715,
      "activations/layer17_attention_weight_max": 54.77680969238281,
      "activations/layer17_attention_weight_min": -43.368709564208984,
      "activations/layer18_attention_weight_max": 48.78491973876953,
      "activations/layer18_attention_weight_min": -36.90562057495117,
      "activations/layer19_attention_weight_max": 25.92519187927246,
      "activations/layer19_attention_weight_min": -21.952966690063477,
      "activations/layer1_attention_weight_max": 15.742152214050293,
      "activations/layer1_attention_weight_min": -13.765172958374023,
      "activations/layer20_attention_weight_max": 24.32689666748047,
      "activations/layer20_attention_weight_min": -21.764902114868164,
      "activations/layer21_attention_weight_max": 38.40385818481445,
      "activations/layer21_attention_weight_min": -24.77006721496582,
      "activations/layer22_attention_weight_max": 34.90808868408203,
      "activations/layer22_attention_weight_min": -23.775665283203125,
      "activations/layer23_attention_weight_max": 34.55376052856445,
      "activations/layer23_attention_weight_min": -22.114553451538086,
      "activations/layer2_attention_weight_max": 30.895421981811523,
      "activations/layer2_attention_weight_min": -31.119400024414062,
      "activations/layer3_attention_weight_max": 86.04402160644531,
      "activations/layer3_attention_weight_min": -90.95811462402344,
      "activations/layer4_attention_weight_max": 100.7254867553711,
      "activations/layer4_attention_weight_min": -96.57788848876953,
      "activations/layer5_attention_weight_max": 91.00218963623047,
      "activations/layer5_attention_weight_min": -84.47821044921875,
      "activations/layer6_attention_weight_max": 55.63727951049805,
      "activations/layer6_attention_weight_min": -52.193580627441406,
      "activations/layer7_attention_weight_max": 74.72175598144531,
      "activations/layer7_attention_weight_min": -77.55583190917969,
      "activations/layer8_attention_weight_max": 46.59724044799805,
      "activations/layer8_attention_weight_min": -45.83019256591797,
      "activations/layer9_attention_weight_max": 51.37369918823242,
      "activations/layer9_attention_weight_min": -51.13692092895508,
      "epoch": 7.16,
      "learning_rate": 0.00010487348484848484,
      "loss": 2.8277,
      "step": 123250
    },
    {
      "activations/layer0_attention_weight_max": 15.088236808776855,
      "activations/layer0_attention_weight_min": -14.935766220092773,
      "activations/layer10_attention_weight_max": 36.86213684082031,
      "activations/layer10_attention_weight_min": -36.234222412109375,
      "activations/layer11_attention_weight_max": 32.11426544189453,
      "activations/layer11_attention_weight_min": -31.198474884033203,
      "activations/layer12_attention_weight_max": 17.45750617980957,
      "activations/layer12_attention_weight_min": -25.764192581176758,
      "activations/layer13_attention_weight_max": 44.750545501708984,
      "activations/layer13_attention_weight_min": -34.09120559692383,
      "activations/layer14_attention_weight_max": 45.06794357299805,
      "activations/layer14_attention_weight_min": -31.576112747192383,
      "activations/layer15_attention_weight_max": 34.74904251098633,
      "activations/layer15_attention_weight_min": -30.49005889892578,
      "activations/layer16_attention_weight_max": 38.895965576171875,
      "activations/layer16_attention_weight_min": -26.205472946166992,
      "activations/layer17_attention_weight_max": 53.22500228881836,
      "activations/layer17_attention_weight_min": -40.58988571166992,
      "activations/layer18_attention_weight_max": 45.228538513183594,
      "activations/layer18_attention_weight_min": -34.805908203125,
      "activations/layer19_attention_weight_max": 19.679271697998047,
      "activations/layer19_attention_weight_min": -21.670949935913086,
      "activations/layer1_attention_weight_max": 17.938156127929688,
      "activations/layer1_attention_weight_min": -15.304403305053711,
      "activations/layer20_attention_weight_max": 19.57310676574707,
      "activations/layer20_attention_weight_min": -23.98593521118164,
      "activations/layer21_attention_weight_max": 37.400115966796875,
      "activations/layer21_attention_weight_min": -24.99457359313965,
      "activations/layer22_attention_weight_max": 31.370948791503906,
      "activations/layer22_attention_weight_min": -25.943193435668945,
      "activations/layer23_attention_weight_max": 31.743122100830078,
      "activations/layer23_attention_weight_min": -21.018918991088867,
      "activations/layer2_attention_weight_max": 33.21083068847656,
      "activations/layer2_attention_weight_min": -30.999237060546875,
      "activations/layer3_attention_weight_max": 90.69185638427734,
      "activations/layer3_attention_weight_min": -87.90389251708984,
      "activations/layer4_attention_weight_max": 106.85283660888672,
      "activations/layer4_attention_weight_min": -96.03244018554688,
      "activations/layer5_attention_weight_max": 97.34893035888672,
      "activations/layer5_attention_weight_min": -85.01557159423828,
      "activations/layer6_attention_weight_max": 61.92365646362305,
      "activations/layer6_attention_weight_min": -54.20561981201172,
      "activations/layer7_attention_weight_max": 73.14364624023438,
      "activations/layer7_attention_weight_min": -69.646240234375,
      "activations/layer8_attention_weight_max": 44.90042495727539,
      "activations/layer8_attention_weight_min": -44.12604522705078,
      "activations/layer9_attention_weight_max": 53.38594436645508,
      "activations/layer9_attention_weight_min": -51.31394577026367,
      "epoch": 7.16,
      "learning_rate": 0.00010485454545454545,
      "loss": 2.8135,
      "step": 123300
    },
    {
      "activations/layer0_attention_weight_max": 15.525052070617676,
      "activations/layer0_attention_weight_min": -14.849995613098145,
      "activations/layer10_attention_weight_max": 38.62617492675781,
      "activations/layer10_attention_weight_min": -35.522220611572266,
      "activations/layer11_attention_weight_max": 31.637487411499023,
      "activations/layer11_attention_weight_min": -29.396282196044922,
      "activations/layer12_attention_weight_max": 32.16294860839844,
      "activations/layer12_attention_weight_min": -25.83180046081543,
      "activations/layer13_attention_weight_max": 45.51081466674805,
      "activations/layer13_attention_weight_min": -32.88153839111328,
      "activations/layer14_attention_weight_max": 43.7403678894043,
      "activations/layer14_attention_weight_min": -29.620351791381836,
      "activations/layer15_attention_weight_max": 37.208152770996094,
      "activations/layer15_attention_weight_min": -27.54005241394043,
      "activations/layer16_attention_weight_max": 38.03230667114258,
      "activations/layer16_attention_weight_min": -26.688880920410156,
      "activations/layer17_attention_weight_max": 58.97319793701172,
      "activations/layer17_attention_weight_min": -45.48689270019531,
      "activations/layer18_attention_weight_max": 46.91579818725586,
      "activations/layer18_attention_weight_min": -38.0665168762207,
      "activations/layer19_attention_weight_max": 24.09328842163086,
      "activations/layer19_attention_weight_min": -21.383947372436523,
      "activations/layer1_attention_weight_max": 16.817424774169922,
      "activations/layer1_attention_weight_min": -16.231510162353516,
      "activations/layer20_attention_weight_max": 22.374771118164062,
      "activations/layer20_attention_weight_min": -23.47543716430664,
      "activations/layer21_attention_weight_max": 42.4346809387207,
      "activations/layer21_attention_weight_min": -23.666162490844727,
      "activations/layer22_attention_weight_max": 30.287080764770508,
      "activations/layer22_attention_weight_min": -29.502458572387695,
      "activations/layer23_attention_weight_max": 31.059768676757812,
      "activations/layer23_attention_weight_min": -20.921510696411133,
      "activations/layer2_attention_weight_max": 31.485868453979492,
      "activations/layer2_attention_weight_min": -29.608882904052734,
      "activations/layer3_attention_weight_max": 86.87973022460938,
      "activations/layer3_attention_weight_min": -86.75847625732422,
      "activations/layer4_attention_weight_max": 103.8080825805664,
      "activations/layer4_attention_weight_min": -92.9867172241211,
      "activations/layer5_attention_weight_max": 93.07415008544922,
      "activations/layer5_attention_weight_min": -82.78816223144531,
      "activations/layer6_attention_weight_max": 57.89653015136719,
      "activations/layer6_attention_weight_min": -51.74227523803711,
      "activations/layer7_attention_weight_max": 76.21704864501953,
      "activations/layer7_attention_weight_min": -68.9256591796875,
      "activations/layer8_attention_weight_max": 46.2870979309082,
      "activations/layer8_attention_weight_min": -45.77214431762695,
      "activations/layer9_attention_weight_max": 51.77020263671875,
      "activations/layer9_attention_weight_min": -47.63125991821289,
      "epoch": 7.17,
      "learning_rate": 0.00010483560606060605,
      "loss": 2.8079,
      "step": 123350
    },
    {
      "activations/layer0_attention_weight_max": 14.710867881774902,
      "activations/layer0_attention_weight_min": -14.945897102355957,
      "activations/layer10_attention_weight_max": 38.210060119628906,
      "activations/layer10_attention_weight_min": -33.933631896972656,
      "activations/layer11_attention_weight_max": 33.04875183105469,
      "activations/layer11_attention_weight_min": -29.70319938659668,
      "activations/layer12_attention_weight_max": 20.574411392211914,
      "activations/layer12_attention_weight_min": -24.054298400878906,
      "activations/layer13_attention_weight_max": 48.552974700927734,
      "activations/layer13_attention_weight_min": -37.172882080078125,
      "activations/layer14_attention_weight_max": 40.02138900756836,
      "activations/layer14_attention_weight_min": -31.88727378845215,
      "activations/layer15_attention_weight_max": 35.945594787597656,
      "activations/layer15_attention_weight_min": -29.866241455078125,
      "activations/layer16_attention_weight_max": 35.8712158203125,
      "activations/layer16_attention_weight_min": -28.035036087036133,
      "activations/layer17_attention_weight_max": 56.238807678222656,
      "activations/layer17_attention_weight_min": -39.59627914428711,
      "activations/layer18_attention_weight_max": 50.6947021484375,
      "activations/layer18_attention_weight_min": -36.12461471557617,
      "activations/layer19_attention_weight_max": 20.605073928833008,
      "activations/layer19_attention_weight_min": -21.457738876342773,
      "activations/layer1_attention_weight_max": 17.24297523498535,
      "activations/layer1_attention_weight_min": -15.260783195495605,
      "activations/layer20_attention_weight_max": 22.869853973388672,
      "activations/layer20_attention_weight_min": -20.162761688232422,
      "activations/layer21_attention_weight_max": 44.29903030395508,
      "activations/layer21_attention_weight_min": -21.89383888244629,
      "activations/layer22_attention_weight_max": 32.95802307128906,
      "activations/layer22_attention_weight_min": -25.15416717529297,
      "activations/layer23_attention_weight_max": 38.526222229003906,
      "activations/layer23_attention_weight_min": -19.644916534423828,
      "activations/layer2_attention_weight_max": 31.79901123046875,
      "activations/layer2_attention_weight_min": -30.245182037353516,
      "activations/layer3_attention_weight_max": 84.91818237304688,
      "activations/layer3_attention_weight_min": -90.86373901367188,
      "activations/layer4_attention_weight_max": 96.29007720947266,
      "activations/layer4_attention_weight_min": -88.4002914428711,
      "activations/layer5_attention_weight_max": 88.75689697265625,
      "activations/layer5_attention_weight_min": -82.65096282958984,
      "activations/layer6_attention_weight_max": 54.33942413330078,
      "activations/layer6_attention_weight_min": -51.858638763427734,
      "activations/layer7_attention_weight_max": 73.48450469970703,
      "activations/layer7_attention_weight_min": -71.06578826904297,
      "activations/layer8_attention_weight_max": 43.87959671020508,
      "activations/layer8_attention_weight_min": -47.43172836303711,
      "activations/layer9_attention_weight_max": 50.41329574584961,
      "activations/layer9_attention_weight_min": -48.92243194580078,
      "epoch": 7.17,
      "learning_rate": 0.00010481666666666666,
      "loss": 2.7936,
      "step": 123400
    },
    {
      "activations/layer0_attention_weight_max": 14.983415603637695,
      "activations/layer0_attention_weight_min": -14.931559562683105,
      "activations/layer10_attention_weight_max": 38.906890869140625,
      "activations/layer10_attention_weight_min": -37.061073303222656,
      "activations/layer11_attention_weight_max": 30.497352600097656,
      "activations/layer11_attention_weight_min": -29.473203659057617,
      "activations/layer12_attention_weight_max": 18.85129165649414,
      "activations/layer12_attention_weight_min": -24.972129821777344,
      "activations/layer13_attention_weight_max": 47.443660736083984,
      "activations/layer13_attention_weight_min": -34.96495056152344,
      "activations/layer14_attention_weight_max": 35.51172637939453,
      "activations/layer14_attention_weight_min": -28.935625076293945,
      "activations/layer15_attention_weight_max": 31.88965606689453,
      "activations/layer15_attention_weight_min": -29.300642013549805,
      "activations/layer16_attention_weight_max": 31.15403938293457,
      "activations/layer16_attention_weight_min": -27.330495834350586,
      "activations/layer17_attention_weight_max": 51.648868560791016,
      "activations/layer17_attention_weight_min": -39.597042083740234,
      "activations/layer18_attention_weight_max": 45.53784942626953,
      "activations/layer18_attention_weight_min": -33.26570510864258,
      "activations/layer19_attention_weight_max": 19.156370162963867,
      "activations/layer19_attention_weight_min": -22.892704010009766,
      "activations/layer1_attention_weight_max": 17.280668258666992,
      "activations/layer1_attention_weight_min": -15.525808334350586,
      "activations/layer20_attention_weight_max": 19.80369758605957,
      "activations/layer20_attention_weight_min": -22.659873962402344,
      "activations/layer21_attention_weight_max": 33.791893005371094,
      "activations/layer21_attention_weight_min": -25.292057037353516,
      "activations/layer22_attention_weight_max": 28.19516372680664,
      "activations/layer22_attention_weight_min": -23.249048233032227,
      "activations/layer23_attention_weight_max": 29.280712127685547,
      "activations/layer23_attention_weight_min": -22.560897827148438,
      "activations/layer2_attention_weight_max": 32.93157958984375,
      "activations/layer2_attention_weight_min": -30.341915130615234,
      "activations/layer3_attention_weight_max": 87.31476593017578,
      "activations/layer3_attention_weight_min": -95.9982681274414,
      "activations/layer4_attention_weight_max": 100.22439575195312,
      "activations/layer4_attention_weight_min": -100.88070678710938,
      "activations/layer5_attention_weight_max": 91.15937805175781,
      "activations/layer5_attention_weight_min": -83.88001251220703,
      "activations/layer6_attention_weight_max": 57.0205192565918,
      "activations/layer6_attention_weight_min": -53.104896545410156,
      "activations/layer7_attention_weight_max": 76.99300384521484,
      "activations/layer7_attention_weight_min": -71.19902038574219,
      "activations/layer8_attention_weight_max": 45.05842590332031,
      "activations/layer8_attention_weight_min": -46.48649597167969,
      "activations/layer9_attention_weight_max": 51.7509880065918,
      "activations/layer9_attention_weight_min": -50.48006057739258,
      "epoch": 7.17,
      "learning_rate": 0.00010479772727272726,
      "loss": 2.8169,
      "step": 123450
    },
    {
      "activations/layer0_attention_weight_max": 14.911726951599121,
      "activations/layer0_attention_weight_min": -14.449353218078613,
      "activations/layer10_attention_weight_max": 39.84526443481445,
      "activations/layer10_attention_weight_min": -34.06720733642578,
      "activations/layer11_attention_weight_max": 38.667606353759766,
      "activations/layer11_attention_weight_min": -29.37322998046875,
      "activations/layer12_attention_weight_max": 18.88640785217285,
      "activations/layer12_attention_weight_min": -27.509912490844727,
      "activations/layer13_attention_weight_max": 38.504913330078125,
      "activations/layer13_attention_weight_min": -27.75216293334961,
      "activations/layer14_attention_weight_max": 37.57752990722656,
      "activations/layer14_attention_weight_min": -30.85321617126465,
      "activations/layer15_attention_weight_max": 34.95161056518555,
      "activations/layer15_attention_weight_min": -28.787376403808594,
      "activations/layer16_attention_weight_max": 32.07368087768555,
      "activations/layer16_attention_weight_min": -27.804685592651367,
      "activations/layer17_attention_weight_max": 50.01297378540039,
      "activations/layer17_attention_weight_min": -40.852909088134766,
      "activations/layer18_attention_weight_max": 47.47315216064453,
      "activations/layer18_attention_weight_min": -34.96675491333008,
      "activations/layer19_attention_weight_max": 18.673065185546875,
      "activations/layer19_attention_weight_min": -20.456558227539062,
      "activations/layer1_attention_weight_max": 17.7078857421875,
      "activations/layer1_attention_weight_min": -16.850440979003906,
      "activations/layer20_attention_weight_max": 24.280651092529297,
      "activations/layer20_attention_weight_min": -23.604534149169922,
      "activations/layer21_attention_weight_max": 35.86116027832031,
      "activations/layer21_attention_weight_min": -23.372026443481445,
      "activations/layer22_attention_weight_max": 29.95688247680664,
      "activations/layer22_attention_weight_min": -26.654449462890625,
      "activations/layer23_attention_weight_max": 30.98736572265625,
      "activations/layer23_attention_weight_min": -20.422344207763672,
      "activations/layer2_attention_weight_max": 32.576744079589844,
      "activations/layer2_attention_weight_min": -31.750083923339844,
      "activations/layer3_attention_weight_max": 86.2496337890625,
      "activations/layer3_attention_weight_min": -89.99980926513672,
      "activations/layer4_attention_weight_max": 98.82005310058594,
      "activations/layer4_attention_weight_min": -94.24993133544922,
      "activations/layer5_attention_weight_max": 92.86956787109375,
      "activations/layer5_attention_weight_min": -83.0545654296875,
      "activations/layer6_attention_weight_max": 57.71734619140625,
      "activations/layer6_attention_weight_min": -53.68907165527344,
      "activations/layer7_attention_weight_max": 78.31653594970703,
      "activations/layer7_attention_weight_min": -74.54605102539062,
      "activations/layer8_attention_weight_max": 44.898460388183594,
      "activations/layer8_attention_weight_min": -45.388423919677734,
      "activations/layer9_attention_weight_max": 67.45404052734375,
      "activations/layer9_attention_weight_min": -51.08052444458008,
      "epoch": 7.18,
      "learning_rate": 0.00010477878787878788,
      "loss": 2.8122,
      "step": 123500
    },
    {
      "activations/layer0_attention_weight_max": 15.018538475036621,
      "activations/layer0_attention_weight_min": -15.09945011138916,
      "activations/layer10_attention_weight_max": 37.584197998046875,
      "activations/layer10_attention_weight_min": -35.30389404296875,
      "activations/layer11_attention_weight_max": 29.750486373901367,
      "activations/layer11_attention_weight_min": -30.50757598876953,
      "activations/layer12_attention_weight_max": 20.357112884521484,
      "activations/layer12_attention_weight_min": -27.385433197021484,
      "activations/layer13_attention_weight_max": 57.1174430847168,
      "activations/layer13_attention_weight_min": -37.72341537475586,
      "activations/layer14_attention_weight_max": 61.24862289428711,
      "activations/layer14_attention_weight_min": -34.210365295410156,
      "activations/layer15_attention_weight_max": 46.58985900878906,
      "activations/layer15_attention_weight_min": -29.46019172668457,
      "activations/layer16_attention_weight_max": 35.37517166137695,
      "activations/layer16_attention_weight_min": -27.036928176879883,
      "activations/layer17_attention_weight_max": 58.74072265625,
      "activations/layer17_attention_weight_min": -41.208370208740234,
      "activations/layer18_attention_weight_max": 44.955326080322266,
      "activations/layer18_attention_weight_min": -35.44171142578125,
      "activations/layer19_attention_weight_max": 21.22560691833496,
      "activations/layer19_attention_weight_min": -20.369476318359375,
      "activations/layer1_attention_weight_max": 16.91028594970703,
      "activations/layer1_attention_weight_min": -16.054195404052734,
      "activations/layer20_attention_weight_max": 22.481292724609375,
      "activations/layer20_attention_weight_min": -19.290346145629883,
      "activations/layer21_attention_weight_max": 33.83654022216797,
      "activations/layer21_attention_weight_min": -24.208024978637695,
      "activations/layer22_attention_weight_max": 33.90386199951172,
      "activations/layer22_attention_weight_min": -23.43006134033203,
      "activations/layer23_attention_weight_max": 34.774715423583984,
      "activations/layer23_attention_weight_min": -19.62613296508789,
      "activations/layer2_attention_weight_max": 33.97008514404297,
      "activations/layer2_attention_weight_min": -30.288389205932617,
      "activations/layer3_attention_weight_max": 88.12590026855469,
      "activations/layer3_attention_weight_min": -88.00414276123047,
      "activations/layer4_attention_weight_max": 101.52667236328125,
      "activations/layer4_attention_weight_min": -98.24301147460938,
      "activations/layer5_attention_weight_max": 94.17430114746094,
      "activations/layer5_attention_weight_min": -90.8712158203125,
      "activations/layer6_attention_weight_max": 57.255462646484375,
      "activations/layer6_attention_weight_min": -56.3608512878418,
      "activations/layer7_attention_weight_max": 76.90017700195312,
      "activations/layer7_attention_weight_min": -72.34284973144531,
      "activations/layer8_attention_weight_max": 45.91438674926758,
      "activations/layer8_attention_weight_min": -45.63218688964844,
      "activations/layer9_attention_weight_max": 54.951202392578125,
      "activations/layer9_attention_weight_min": -48.968441009521484,
      "epoch": 7.18,
      "learning_rate": 0.00010475984848484847,
      "loss": 2.8107,
      "step": 123550
    },
    {
      "activations/layer0_attention_weight_max": 15.503159523010254,
      "activations/layer0_attention_weight_min": -14.449859619140625,
      "activations/layer10_attention_weight_max": 44.94691848754883,
      "activations/layer10_attention_weight_min": -39.704322814941406,
      "activations/layer11_attention_weight_max": 36.175750732421875,
      "activations/layer11_attention_weight_min": -31.9086971282959,
      "activations/layer12_attention_weight_max": 19.74872398376465,
      "activations/layer12_attention_weight_min": -26.734086990356445,
      "activations/layer13_attention_weight_max": 51.611228942871094,
      "activations/layer13_attention_weight_min": -35.497955322265625,
      "activations/layer14_attention_weight_max": 42.237274169921875,
      "activations/layer14_attention_weight_min": -30.96463394165039,
      "activations/layer15_attention_weight_max": 35.63808822631836,
      "activations/layer15_attention_weight_min": -29.85276222229004,
      "activations/layer16_attention_weight_max": 37.41643524169922,
      "activations/layer16_attention_weight_min": -26.659713745117188,
      "activations/layer17_attention_weight_max": 53.65517807006836,
      "activations/layer17_attention_weight_min": -41.53642654418945,
      "activations/layer18_attention_weight_max": 49.928321838378906,
      "activations/layer18_attention_weight_min": -35.2324104309082,
      "activations/layer19_attention_weight_max": 21.594560623168945,
      "activations/layer19_attention_weight_min": -20.86524200439453,
      "activations/layer1_attention_weight_max": 16.526966094970703,
      "activations/layer1_attention_weight_min": -17.497453689575195,
      "activations/layer20_attention_weight_max": 21.44545555114746,
      "activations/layer20_attention_weight_min": -22.637041091918945,
      "activations/layer21_attention_weight_max": 34.87567138671875,
      "activations/layer21_attention_weight_min": -24.37337303161621,
      "activations/layer22_attention_weight_max": 32.272125244140625,
      "activations/layer22_attention_weight_min": -23.677642822265625,
      "activations/layer23_attention_weight_max": 32.5091552734375,
      "activations/layer23_attention_weight_min": -18.673784255981445,
      "activations/layer2_attention_weight_max": 32.76612854003906,
      "activations/layer2_attention_weight_min": -29.440488815307617,
      "activations/layer3_attention_weight_max": 84.96891784667969,
      "activations/layer3_attention_weight_min": -85.79503631591797,
      "activations/layer4_attention_weight_max": 100.63843536376953,
      "activations/layer4_attention_weight_min": -93.07743835449219,
      "activations/layer5_attention_weight_max": 88.85153198242188,
      "activations/layer5_attention_weight_min": -85.27666473388672,
      "activations/layer6_attention_weight_max": 55.46968460083008,
      "activations/layer6_attention_weight_min": -54.790672302246094,
      "activations/layer7_attention_weight_max": 79.18499755859375,
      "activations/layer7_attention_weight_min": -73.271728515625,
      "activations/layer8_attention_weight_max": 50.466552734375,
      "activations/layer8_attention_weight_min": -46.919246673583984,
      "activations/layer9_attention_weight_max": 60.81732940673828,
      "activations/layer9_attention_weight_min": -50.46931076049805,
      "epoch": 7.18,
      "learning_rate": 0.00010474090909090908,
      "loss": 2.8383,
      "step": 123600
    },
    {
      "activations/layer0_attention_weight_max": 15.059591293334961,
      "activations/layer0_attention_weight_min": -14.780792236328125,
      "activations/layer10_attention_weight_max": 44.81744384765625,
      "activations/layer10_attention_weight_min": -37.81462860107422,
      "activations/layer11_attention_weight_max": 40.38269805908203,
      "activations/layer11_attention_weight_min": -31.41727066040039,
      "activations/layer12_attention_weight_max": 19.630643844604492,
      "activations/layer12_attention_weight_min": -28.87966537475586,
      "activations/layer13_attention_weight_max": 63.57756042480469,
      "activations/layer13_attention_weight_min": -32.969451904296875,
      "activations/layer14_attention_weight_max": 62.120723724365234,
      "activations/layer14_attention_weight_min": -30.871553421020508,
      "activations/layer15_attention_weight_max": 46.89136505126953,
      "activations/layer15_attention_weight_min": -30.117830276489258,
      "activations/layer16_attention_weight_max": 37.54666519165039,
      "activations/layer16_attention_weight_min": -26.16240119934082,
      "activations/layer17_attention_weight_max": 63.61130142211914,
      "activations/layer17_attention_weight_min": -46.90166473388672,
      "activations/layer18_attention_weight_max": 55.42121887207031,
      "activations/layer18_attention_weight_min": -36.95632553100586,
      "activations/layer19_attention_weight_max": 23.953201293945312,
      "activations/layer19_attention_weight_min": -20.59661102294922,
      "activations/layer1_attention_weight_max": 18.044286727905273,
      "activations/layer1_attention_weight_min": -16.931909561157227,
      "activations/layer20_attention_weight_max": 25.326976776123047,
      "activations/layer20_attention_weight_min": -22.171735763549805,
      "activations/layer21_attention_weight_max": 53.95953369140625,
      "activations/layer21_attention_weight_min": -23.50430679321289,
      "activations/layer22_attention_weight_max": 34.02763748168945,
      "activations/layer22_attention_weight_min": -23.78225326538086,
      "activations/layer23_attention_weight_max": 33.75465393066406,
      "activations/layer23_attention_weight_min": -19.519969940185547,
      "activations/layer2_attention_weight_max": 34.494300842285156,
      "activations/layer2_attention_weight_min": -32.12476348876953,
      "activations/layer3_attention_weight_max": 95.05510711669922,
      "activations/layer3_attention_weight_min": -94.75447082519531,
      "activations/layer4_attention_weight_max": 109.15545654296875,
      "activations/layer4_attention_weight_min": -97.9188232421875,
      "activations/layer5_attention_weight_max": 93.84898376464844,
      "activations/layer5_attention_weight_min": -86.64933776855469,
      "activations/layer6_attention_weight_max": 55.5887565612793,
      "activations/layer6_attention_weight_min": -52.528629302978516,
      "activations/layer7_attention_weight_max": 80.82916259765625,
      "activations/layer7_attention_weight_min": -77.40960693359375,
      "activations/layer8_attention_weight_max": 47.84383010864258,
      "activations/layer8_attention_weight_min": -44.91838073730469,
      "activations/layer9_attention_weight_max": 63.56201934814453,
      "activations/layer9_attention_weight_min": -53.99652099609375,
      "epoch": 7.18,
      "learning_rate": 0.0001047219696969697,
      "loss": 2.8128,
      "step": 123650
    },
    {
      "activations/layer0_attention_weight_max": 14.754151344299316,
      "activations/layer0_attention_weight_min": -14.941903114318848,
      "activations/layer10_attention_weight_max": 35.659759521484375,
      "activations/layer10_attention_weight_min": -34.16376876831055,
      "activations/layer11_attention_weight_max": 29.69760513305664,
      "activations/layer11_attention_weight_min": -30.46951675415039,
      "activations/layer12_attention_weight_max": 36.181400299072266,
      "activations/layer12_attention_weight_min": -24.4368953704834,
      "activations/layer13_attention_weight_max": 48.71550369262695,
      "activations/layer13_attention_weight_min": -32.81449508666992,
      "activations/layer14_attention_weight_max": 37.67127227783203,
      "activations/layer14_attention_weight_min": -30.61260414123535,
      "activations/layer15_attention_weight_max": 37.904754638671875,
      "activations/layer15_attention_weight_min": -29.949586868286133,
      "activations/layer16_attention_weight_max": 31.859338760375977,
      "activations/layer16_attention_weight_min": -28.20193862915039,
      "activations/layer17_attention_weight_max": 53.805057525634766,
      "activations/layer17_attention_weight_min": -40.29701614379883,
      "activations/layer18_attention_weight_max": 47.37176513671875,
      "activations/layer18_attention_weight_min": -36.165748596191406,
      "activations/layer19_attention_weight_max": 22.279111862182617,
      "activations/layer19_attention_weight_min": -20.852384567260742,
      "activations/layer1_attention_weight_max": 17.089508056640625,
      "activations/layer1_attention_weight_min": -15.915679931640625,
      "activations/layer20_attention_weight_max": 22.246585845947266,
      "activations/layer20_attention_weight_min": -23.739498138427734,
      "activations/layer21_attention_weight_max": 45.95140075683594,
      "activations/layer21_attention_weight_min": -24.78118896484375,
      "activations/layer22_attention_weight_max": 30.837373733520508,
      "activations/layer22_attention_weight_min": -24.63360595703125,
      "activations/layer23_attention_weight_max": 33.473106384277344,
      "activations/layer23_attention_weight_min": -21.502470016479492,
      "activations/layer2_attention_weight_max": 32.88521194458008,
      "activations/layer2_attention_weight_min": -31.59087371826172,
      "activations/layer3_attention_weight_max": 84.6319580078125,
      "activations/layer3_attention_weight_min": -87.89210510253906,
      "activations/layer4_attention_weight_max": 98.51473236083984,
      "activations/layer4_attention_weight_min": -95.3406753540039,
      "activations/layer5_attention_weight_max": 91.19477844238281,
      "activations/layer5_attention_weight_min": -85.11114501953125,
      "activations/layer6_attention_weight_max": 55.29889678955078,
      "activations/layer6_attention_weight_min": -54.12540054321289,
      "activations/layer7_attention_weight_max": 72.60142517089844,
      "activations/layer7_attention_weight_min": -72.63034057617188,
      "activations/layer8_attention_weight_max": 44.58892059326172,
      "activations/layer8_attention_weight_min": -42.8388671875,
      "activations/layer9_attention_weight_max": 47.71928405761719,
      "activations/layer9_attention_weight_min": -46.15267562866211,
      "epoch": 7.19,
      "learning_rate": 0.00010470303030303028,
      "loss": 2.814,
      "step": 123700
    },
    {
      "activations/layer0_attention_weight_max": 14.61770248413086,
      "activations/layer0_attention_weight_min": -14.614835739135742,
      "activations/layer10_attention_weight_max": 37.37481689453125,
      "activations/layer10_attention_weight_min": -36.29415512084961,
      "activations/layer11_attention_weight_max": 29.801063537597656,
      "activations/layer11_attention_weight_min": -30.286611557006836,
      "activations/layer12_attention_weight_max": 19.26042938232422,
      "activations/layer12_attention_weight_min": -24.283935546875,
      "activations/layer13_attention_weight_max": 41.42906188964844,
      "activations/layer13_attention_weight_min": -30.60983657836914,
      "activations/layer14_attention_weight_max": 39.544464111328125,
      "activations/layer14_attention_weight_min": -30.57933235168457,
      "activations/layer15_attention_weight_max": 36.44007110595703,
      "activations/layer15_attention_weight_min": -28.80946922302246,
      "activations/layer16_attention_weight_max": 36.010501861572266,
      "activations/layer16_attention_weight_min": -28.19562530517578,
      "activations/layer17_attention_weight_max": 55.5324592590332,
      "activations/layer17_attention_weight_min": -41.34212112426758,
      "activations/layer18_attention_weight_max": 49.85458755493164,
      "activations/layer18_attention_weight_min": -33.961971282958984,
      "activations/layer19_attention_weight_max": 23.988893508911133,
      "activations/layer19_attention_weight_min": -20.14980125427246,
      "activations/layer1_attention_weight_max": 16.988954544067383,
      "activations/layer1_attention_weight_min": -15.252530097961426,
      "activations/layer20_attention_weight_max": 21.616268157958984,
      "activations/layer20_attention_weight_min": -20.791288375854492,
      "activations/layer21_attention_weight_max": 35.86486053466797,
      "activations/layer21_attention_weight_min": -23.3320255279541,
      "activations/layer22_attention_weight_max": 31.73349380493164,
      "activations/layer22_attention_weight_min": -25.85890769958496,
      "activations/layer23_attention_weight_max": 32.04543685913086,
      "activations/layer23_attention_weight_min": -21.59423065185547,
      "activations/layer2_attention_weight_max": 30.966421127319336,
      "activations/layer2_attention_weight_min": -30.646793365478516,
      "activations/layer3_attention_weight_max": 87.20850372314453,
      "activations/layer3_attention_weight_min": -87.45635223388672,
      "activations/layer4_attention_weight_max": 101.28895568847656,
      "activations/layer4_attention_weight_min": -96.46723175048828,
      "activations/layer5_attention_weight_max": 91.055908203125,
      "activations/layer5_attention_weight_min": -87.00484466552734,
      "activations/layer6_attention_weight_max": 57.80320739746094,
      "activations/layer6_attention_weight_min": -53.295345306396484,
      "activations/layer7_attention_weight_max": 72.52120971679688,
      "activations/layer7_attention_weight_min": -69.38111114501953,
      "activations/layer8_attention_weight_max": 43.22142028808594,
      "activations/layer8_attention_weight_min": -45.93844985961914,
      "activations/layer9_attention_weight_max": 54.67489242553711,
      "activations/layer9_attention_weight_min": -47.934715270996094,
      "epoch": 7.19,
      "learning_rate": 0.0001046840909090909,
      "loss": 2.8157,
      "step": 123750
    },
    {
      "activations/layer0_attention_weight_max": 15.206650733947754,
      "activations/layer0_attention_weight_min": -14.935912132263184,
      "activations/layer10_attention_weight_max": 42.61854934692383,
      "activations/layer10_attention_weight_min": -41.948944091796875,
      "activations/layer11_attention_weight_max": 35.412532806396484,
      "activations/layer11_attention_weight_min": -31.543601989746094,
      "activations/layer12_attention_weight_max": 20.424352645874023,
      "activations/layer12_attention_weight_min": -25.40394401550293,
      "activations/layer13_attention_weight_max": 92.28572082519531,
      "activations/layer13_attention_weight_min": -50.57424545288086,
      "activations/layer14_attention_weight_max": 87.42008209228516,
      "activations/layer14_attention_weight_min": -41.89036560058594,
      "activations/layer15_attention_weight_max": 61.79387664794922,
      "activations/layer15_attention_weight_min": -33.1989631652832,
      "activations/layer16_attention_weight_max": 42.589378356933594,
      "activations/layer16_attention_weight_min": -30.69169044494629,
      "activations/layer17_attention_weight_max": 68.36695098876953,
      "activations/layer17_attention_weight_min": -47.189659118652344,
      "activations/layer18_attention_weight_max": 62.67304992675781,
      "activations/layer18_attention_weight_min": -42.058162689208984,
      "activations/layer19_attention_weight_max": 30.51766014099121,
      "activations/layer19_attention_weight_min": -23.66208839416504,
      "activations/layer1_attention_weight_max": 18.094770431518555,
      "activations/layer1_attention_weight_min": -15.552545547485352,
      "activations/layer20_attention_weight_max": 33.68413543701172,
      "activations/layer20_attention_weight_min": -22.105791091918945,
      "activations/layer21_attention_weight_max": 68.26522064208984,
      "activations/layer21_attention_weight_min": -27.561004638671875,
      "activations/layer22_attention_weight_max": 40.19966506958008,
      "activations/layer22_attention_weight_min": -26.418956756591797,
      "activations/layer23_attention_weight_max": 42.8662109375,
      "activations/layer23_attention_weight_min": -21.022762298583984,
      "activations/layer2_attention_weight_max": 33.68446731567383,
      "activations/layer2_attention_weight_min": -32.98691177368164,
      "activations/layer3_attention_weight_max": 94.12055969238281,
      "activations/layer3_attention_weight_min": -98.15756225585938,
      "activations/layer4_attention_weight_max": 103.9006576538086,
      "activations/layer4_attention_weight_min": -102.4897232055664,
      "activations/layer5_attention_weight_max": 91.92403411865234,
      "activations/layer5_attention_weight_min": -89.12101745605469,
      "activations/layer6_attention_weight_max": 57.59700012207031,
      "activations/layer6_attention_weight_min": -56.43650817871094,
      "activations/layer7_attention_weight_max": 81.29759216308594,
      "activations/layer7_attention_weight_min": -76.83486938476562,
      "activations/layer8_attention_weight_max": 48.06545639038086,
      "activations/layer8_attention_weight_min": -48.01620101928711,
      "activations/layer9_attention_weight_max": 61.90851593017578,
      "activations/layer9_attention_weight_min": -53.154510498046875,
      "epoch": 7.19,
      "learning_rate": 0.00010466515151515151,
      "loss": 2.8149,
      "step": 123800
    },
    {
      "activations/layer0_attention_weight_max": 15.34766960144043,
      "activations/layer0_attention_weight_min": -15.227726936340332,
      "activations/layer10_attention_weight_max": 43.802852630615234,
      "activations/layer10_attention_weight_min": -40.989105224609375,
      "activations/layer11_attention_weight_max": 36.281986236572266,
      "activations/layer11_attention_weight_min": -33.65717697143555,
      "activations/layer12_attention_weight_max": 33.898040771484375,
      "activations/layer12_attention_weight_min": -24.86966323852539,
      "activations/layer13_attention_weight_max": 61.732269287109375,
      "activations/layer13_attention_weight_min": -36.86161422729492,
      "activations/layer14_attention_weight_max": 64.9879379272461,
      "activations/layer14_attention_weight_min": -35.2235221862793,
      "activations/layer15_attention_weight_max": 47.85725021362305,
      "activations/layer15_attention_weight_min": -28.657896041870117,
      "activations/layer16_attention_weight_max": 32.95536804199219,
      "activations/layer16_attention_weight_min": -24.486717224121094,
      "activations/layer17_attention_weight_max": 56.88807678222656,
      "activations/layer17_attention_weight_min": -40.192256927490234,
      "activations/layer18_attention_weight_max": 47.617183685302734,
      "activations/layer18_attention_weight_min": -34.714073181152344,
      "activations/layer19_attention_weight_max": 21.149898529052734,
      "activations/layer19_attention_weight_min": -20.64256477355957,
      "activations/layer1_attention_weight_max": 17.03827667236328,
      "activations/layer1_attention_weight_min": -14.191762924194336,
      "activations/layer20_attention_weight_max": 23.889047622680664,
      "activations/layer20_attention_weight_min": -25.006628036499023,
      "activations/layer21_attention_weight_max": 38.081512451171875,
      "activations/layer21_attention_weight_min": -25.324296951293945,
      "activations/layer22_attention_weight_max": 32.98341369628906,
      "activations/layer22_attention_weight_min": -23.228853225708008,
      "activations/layer23_attention_weight_max": 32.1749267578125,
      "activations/layer23_attention_weight_min": -20.056570053100586,
      "activations/layer2_attention_weight_max": 31.97281265258789,
      "activations/layer2_attention_weight_min": -29.538616180419922,
      "activations/layer3_attention_weight_max": 90.02289581298828,
      "activations/layer3_attention_weight_min": -87.9022445678711,
      "activations/layer4_attention_weight_max": 103.12467193603516,
      "activations/layer4_attention_weight_min": -98.8642349243164,
      "activations/layer5_attention_weight_max": 93.38710021972656,
      "activations/layer5_attention_weight_min": -86.4256591796875,
      "activations/layer6_attention_weight_max": 57.16080856323242,
      "activations/layer6_attention_weight_min": -57.34306716918945,
      "activations/layer7_attention_weight_max": 79.7548828125,
      "activations/layer7_attention_weight_min": -76.33236694335938,
      "activations/layer8_attention_weight_max": 48.74796676635742,
      "activations/layer8_attention_weight_min": -45.918365478515625,
      "activations/layer9_attention_weight_max": 63.04267501831055,
      "activations/layer9_attention_weight_min": -57.025875091552734,
      "epoch": 7.2,
      "learning_rate": 0.0001046462121212121,
      "loss": 2.809,
      "step": 123850
    },
    {
      "activations/layer0_attention_weight_max": 16.202882766723633,
      "activations/layer0_attention_weight_min": -14.603151321411133,
      "activations/layer10_attention_weight_max": 36.943912506103516,
      "activations/layer10_attention_weight_min": -34.00276565551758,
      "activations/layer11_attention_weight_max": 29.516647338867188,
      "activations/layer11_attention_weight_min": -28.92896270751953,
      "activations/layer12_attention_weight_max": 22.7897891998291,
      "activations/layer12_attention_weight_min": -28.29505729675293,
      "activations/layer13_attention_weight_max": 40.901084899902344,
      "activations/layer13_attention_weight_min": -31.43880844116211,
      "activations/layer14_attention_weight_max": 37.3967399597168,
      "activations/layer14_attention_weight_min": -30.369298934936523,
      "activations/layer15_attention_weight_max": 34.94068145751953,
      "activations/layer15_attention_weight_min": -28.965993881225586,
      "activations/layer16_attention_weight_max": 31.02184295654297,
      "activations/layer16_attention_weight_min": -26.22397804260254,
      "activations/layer17_attention_weight_max": 55.564544677734375,
      "activations/layer17_attention_weight_min": -44.01226806640625,
      "activations/layer18_attention_weight_max": 48.63264083862305,
      "activations/layer18_attention_weight_min": -35.254905700683594,
      "activations/layer19_attention_weight_max": 24.99921226501465,
      "activations/layer19_attention_weight_min": -21.207157135009766,
      "activations/layer1_attention_weight_max": 17.284746170043945,
      "activations/layer1_attention_weight_min": -14.987669944763184,
      "activations/layer20_attention_weight_max": 21.789676666259766,
      "activations/layer20_attention_weight_min": -21.681589126586914,
      "activations/layer21_attention_weight_max": 35.573978424072266,
      "activations/layer21_attention_weight_min": -23.28788185119629,
      "activations/layer22_attention_weight_max": 30.596881866455078,
      "activations/layer22_attention_weight_min": -23.710039138793945,
      "activations/layer23_attention_weight_max": 35.37274169921875,
      "activations/layer23_attention_weight_min": -21.795917510986328,
      "activations/layer2_attention_weight_max": 31.105815887451172,
      "activations/layer2_attention_weight_min": -29.453582763671875,
      "activations/layer3_attention_weight_max": 87.66585540771484,
      "activations/layer3_attention_weight_min": -88.14362335205078,
      "activations/layer4_attention_weight_max": 99.3247299194336,
      "activations/layer4_attention_weight_min": -93.4321517944336,
      "activations/layer5_attention_weight_max": 84.75977325439453,
      "activations/layer5_attention_weight_min": -82.22862243652344,
      "activations/layer6_attention_weight_max": 55.67372131347656,
      "activations/layer6_attention_weight_min": -52.20885467529297,
      "activations/layer7_attention_weight_max": 72.94827270507812,
      "activations/layer7_attention_weight_min": -67.25200653076172,
      "activations/layer8_attention_weight_max": 42.16939163208008,
      "activations/layer8_attention_weight_min": -42.51151657104492,
      "activations/layer9_attention_weight_max": 49.942012786865234,
      "activations/layer9_attention_weight_min": -50.58706283569336,
      "epoch": 7.2,
      "learning_rate": 0.00010462727272727272,
      "loss": 2.8056,
      "step": 123900
    },
    {
      "activations/layer0_attention_weight_max": 15.014322280883789,
      "activations/layer0_attention_weight_min": -14.781100273132324,
      "activations/layer10_attention_weight_max": 38.5030517578125,
      "activations/layer10_attention_weight_min": -38.9747314453125,
      "activations/layer11_attention_weight_max": 30.45207977294922,
      "activations/layer11_attention_weight_min": -30.811628341674805,
      "activations/layer12_attention_weight_max": 18.308412551879883,
      "activations/layer12_attention_weight_min": -27.910140991210938,
      "activations/layer13_attention_weight_max": 38.92125701904297,
      "activations/layer13_attention_weight_min": -32.18754577636719,
      "activations/layer14_attention_weight_max": 36.41928482055664,
      "activations/layer14_attention_weight_min": -30.172893524169922,
      "activations/layer15_attention_weight_max": 35.00728988647461,
      "activations/layer15_attention_weight_min": -29.451175689697266,
      "activations/layer16_attention_weight_max": 33.11933135986328,
      "activations/layer16_attention_weight_min": -27.73866844177246,
      "activations/layer17_attention_weight_max": 55.44691848754883,
      "activations/layer17_attention_weight_min": -45.990055084228516,
      "activations/layer18_attention_weight_max": 47.623741149902344,
      "activations/layer18_attention_weight_min": -39.1363410949707,
      "activations/layer19_attention_weight_max": 21.657350540161133,
      "activations/layer19_attention_weight_min": -22.635221481323242,
      "activations/layer1_attention_weight_max": 17.18928337097168,
      "activations/layer1_attention_weight_min": -14.675692558288574,
      "activations/layer20_attention_weight_max": 22.319211959838867,
      "activations/layer20_attention_weight_min": -21.8967227935791,
      "activations/layer21_attention_weight_max": 43.96992111206055,
      "activations/layer21_attention_weight_min": -24.91594123840332,
      "activations/layer22_attention_weight_max": 33.91120910644531,
      "activations/layer22_attention_weight_min": -26.387496948242188,
      "activations/layer23_attention_weight_max": 37.97502899169922,
      "activations/layer23_attention_weight_min": -25.327741622924805,
      "activations/layer2_attention_weight_max": 30.75054931640625,
      "activations/layer2_attention_weight_min": -30.83541488647461,
      "activations/layer3_attention_weight_max": 82.27052307128906,
      "activations/layer3_attention_weight_min": -87.83584594726562,
      "activations/layer4_attention_weight_max": 97.32390594482422,
      "activations/layer4_attention_weight_min": -89.11427307128906,
      "activations/layer5_attention_weight_max": 89.77177429199219,
      "activations/layer5_attention_weight_min": -83.88362884521484,
      "activations/layer6_attention_weight_max": 56.47869873046875,
      "activations/layer6_attention_weight_min": -54.10435485839844,
      "activations/layer7_attention_weight_max": 81.04022216796875,
      "activations/layer7_attention_weight_min": -79.99987030029297,
      "activations/layer8_attention_weight_max": 44.15375518798828,
      "activations/layer8_attention_weight_min": -49.38331985473633,
      "activations/layer9_attention_weight_max": 51.442378997802734,
      "activations/layer9_attention_weight_min": -55.00098419189453,
      "epoch": 7.2,
      "learning_rate": 0.00010460833333333332,
      "loss": 2.8135,
      "step": 123950
    },
    {
      "activations/layer0_attention_weight_max": 15.002287864685059,
      "activations/layer0_attention_weight_min": -14.492311477661133,
      "activations/layer10_attention_weight_max": 40.91507339477539,
      "activations/layer10_attention_weight_min": -36.6012077331543,
      "activations/layer11_attention_weight_max": 31.0433349609375,
      "activations/layer11_attention_weight_min": -30.647855758666992,
      "activations/layer12_attention_weight_max": 25.820390701293945,
      "activations/layer12_attention_weight_min": -26.98916244506836,
      "activations/layer13_attention_weight_max": 38.968971252441406,
      "activations/layer13_attention_weight_min": -28.59462547302246,
      "activations/layer14_attention_weight_max": 37.37527847290039,
      "activations/layer14_attention_weight_min": -28.39752769470215,
      "activations/layer15_attention_weight_max": 31.999000549316406,
      "activations/layer15_attention_weight_min": -27.858497619628906,
      "activations/layer16_attention_weight_max": 30.09592056274414,
      "activations/layer16_attention_weight_min": -27.00496482849121,
      "activations/layer17_attention_weight_max": 50.1734619140625,
      "activations/layer17_attention_weight_min": -40.4273567199707,
      "activations/layer18_attention_weight_max": 42.874908447265625,
      "activations/layer18_attention_weight_min": -34.528472900390625,
      "activations/layer19_attention_weight_max": 18.520566940307617,
      "activations/layer19_attention_weight_min": -20.69254493713379,
      "activations/layer1_attention_weight_max": 16.629573822021484,
      "activations/layer1_attention_weight_min": -15.98379135131836,
      "activations/layer20_attention_weight_max": 19.387601852416992,
      "activations/layer20_attention_weight_min": -20.154508590698242,
      "activations/layer21_attention_weight_max": 33.30498123168945,
      "activations/layer21_attention_weight_min": -26.278703689575195,
      "activations/layer22_attention_weight_max": 29.583940505981445,
      "activations/layer22_attention_weight_min": -25.91160011291504,
      "activations/layer23_attention_weight_max": 31.087987899780273,
      "activations/layer23_attention_weight_min": -20.227895736694336,
      "activations/layer2_attention_weight_max": 32.0635871887207,
      "activations/layer2_attention_weight_min": -30.549678802490234,
      "activations/layer3_attention_weight_max": 88.0902328491211,
      "activations/layer3_attention_weight_min": -86.96314239501953,
      "activations/layer4_attention_weight_max": 102.3830337524414,
      "activations/layer4_attention_weight_min": -95.07424926757812,
      "activations/layer5_attention_weight_max": 90.03539276123047,
      "activations/layer5_attention_weight_min": -86.38053894042969,
      "activations/layer6_attention_weight_max": 57.677677154541016,
      "activations/layer6_attention_weight_min": -55.580299377441406,
      "activations/layer7_attention_weight_max": 74.81980895996094,
      "activations/layer7_attention_weight_min": -71.40205383300781,
      "activations/layer8_attention_weight_max": 48.51674270629883,
      "activations/layer8_attention_weight_min": -47.830142974853516,
      "activations/layer9_attention_weight_max": 52.6270751953125,
      "activations/layer9_attention_weight_min": -48.03925323486328,
      "epoch": 7.21,
      "learning_rate": 0.00010458939393939393,
      "loss": 2.8107,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5375,
      "eval_samples_per_second": 502.957,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5375,
      "eval_openwebtext_samples_per_second": 502.957,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_wikitext_loss": 3.00390625,
      "eval_wikitext_ppl": 20.16414949200632,
      "eval_wikitext_runtime": 2.0226,
      "eval_wikitext_samples_per_second": 225.449,
      "step": 124000
    },
    {
      "epoch": 7.21,
      "eval_lambada_loss": 2.7109375,
      "eval_lambada_ppl": 15.04337206739713,
      "eval_lambada_runtime": 9.6396,
      "eval_lambada_samples_per_second": 505.106,
      "step": 124000
    },
    {
      "activations/layer0_attention_weight_max": 14.281102180480957,
      "activations/layer0_attention_weight_min": -14.4963960647583,
      "activations/layer10_attention_weight_max": 40.970977783203125,
      "activations/layer10_attention_weight_min": -38.22795867919922,
      "activations/layer11_attention_weight_max": 33.17457580566406,
      "activations/layer11_attention_weight_min": -32.22852325439453,
      "activations/layer12_attention_weight_max": 20.897010803222656,
      "activations/layer12_attention_weight_min": -28.682035446166992,
      "activations/layer13_attention_weight_max": 49.33481979370117,
      "activations/layer13_attention_weight_min": -35.04512405395508,
      "activations/layer14_attention_weight_max": 44.7056999206543,
      "activations/layer14_attention_weight_min": -29.784500122070312,
      "activations/layer15_attention_weight_max": 37.51871109008789,
      "activations/layer15_attention_weight_min": -28.12002182006836,
      "activations/layer16_attention_weight_max": 34.39615249633789,
      "activations/layer16_attention_weight_min": -27.82267951965332,
      "activations/layer17_attention_weight_max": 57.90904998779297,
      "activations/layer17_attention_weight_min": -41.76417922973633,
      "activations/layer18_attention_weight_max": 51.22811508178711,
      "activations/layer18_attention_weight_min": -34.655757904052734,
      "activations/layer19_attention_weight_max": 22.218584060668945,
      "activations/layer19_attention_weight_min": -20.33462905883789,
      "activations/layer1_attention_weight_max": 16.18314552307129,
      "activations/layer1_attention_weight_min": -15.58696174621582,
      "activations/layer20_attention_weight_max": 24.43315887451172,
      "activations/layer20_attention_weight_min": -20.22214126586914,
      "activations/layer21_attention_weight_max": 41.19681167602539,
      "activations/layer21_attention_weight_min": -26.360881805419922,
      "activations/layer22_attention_weight_max": 32.582942962646484,
      "activations/layer22_attention_weight_min": -24.534461975097656,
      "activations/layer23_attention_weight_max": 34.041534423828125,
      "activations/layer23_attention_weight_min": -19.657936096191406,
      "activations/layer2_attention_weight_max": 32.153907775878906,
      "activations/layer2_attention_weight_min": -29.822917938232422,
      "activations/layer3_attention_weight_max": 87.55477905273438,
      "activations/layer3_attention_weight_min": -90.41058349609375,
      "activations/layer4_attention_weight_max": 101.31684112548828,
      "activations/layer4_attention_weight_min": -94.18025207519531,
      "activations/layer5_attention_weight_max": 93.20976257324219,
      "activations/layer5_attention_weight_min": -86.94099426269531,
      "activations/layer6_attention_weight_max": 58.58092498779297,
      "activations/layer6_attention_weight_min": -55.70817565917969,
      "activations/layer7_attention_weight_max": 83.16557312011719,
      "activations/layer7_attention_weight_min": -75.95975494384766,
      "activations/layer8_attention_weight_max": 47.24708557128906,
      "activations/layer8_attention_weight_min": -48.73213577270508,
      "activations/layer9_attention_weight_max": 57.140113830566406,
      "activations/layer9_attention_weight_min": -54.233184814453125,
      "epoch": 7.21,
      "learning_rate": 0.00010457045454545454,
      "loss": 2.8208,
      "step": 124050
    },
    {
      "activations/layer0_attention_weight_max": 14.188986778259277,
      "activations/layer0_attention_weight_min": -14.856266021728516,
      "activations/layer10_attention_weight_max": 40.492557525634766,
      "activations/layer10_attention_weight_min": -36.5312385559082,
      "activations/layer11_attention_weight_max": 32.45094299316406,
      "activations/layer11_attention_weight_min": -30.495702743530273,
      "activations/layer12_attention_weight_max": 21.679845809936523,
      "activations/layer12_attention_weight_min": -26.822080612182617,
      "activations/layer13_attention_weight_max": 46.62562942504883,
      "activations/layer13_attention_weight_min": -35.703250885009766,
      "activations/layer14_attention_weight_max": 37.43203353881836,
      "activations/layer14_attention_weight_min": -31.61865997314453,
      "activations/layer15_attention_weight_max": 34.54036331176758,
      "activations/layer15_attention_weight_min": -27.698856353759766,
      "activations/layer16_attention_weight_max": 32.37028121948242,
      "activations/layer16_attention_weight_min": -27.384035110473633,
      "activations/layer17_attention_weight_max": 55.85902786254883,
      "activations/layer17_attention_weight_min": -42.17390823364258,
      "activations/layer18_attention_weight_max": 48.23481369018555,
      "activations/layer18_attention_weight_min": -34.840335845947266,
      "activations/layer19_attention_weight_max": 23.268932342529297,
      "activations/layer19_attention_weight_min": -19.954092025756836,
      "activations/layer1_attention_weight_max": 16.431631088256836,
      "activations/layer1_attention_weight_min": -15.218579292297363,
      "activations/layer20_attention_weight_max": 22.60027313232422,
      "activations/layer20_attention_weight_min": -24.255535125732422,
      "activations/layer21_attention_weight_max": 40.555419921875,
      "activations/layer21_attention_weight_min": -22.694244384765625,
      "activations/layer22_attention_weight_max": 35.19110870361328,
      "activations/layer22_attention_weight_min": -24.322845458984375,
      "activations/layer23_attention_weight_max": 34.83221435546875,
      "activations/layer23_attention_weight_min": -19.747982025146484,
      "activations/layer2_attention_weight_max": 31.55792236328125,
      "activations/layer2_attention_weight_min": -29.86357879638672,
      "activations/layer3_attention_weight_max": 84.21398162841797,
      "activations/layer3_attention_weight_min": -87.0305404663086,
      "activations/layer4_attention_weight_max": 102.03055572509766,
      "activations/layer4_attention_weight_min": -95.24064636230469,
      "activations/layer5_attention_weight_max": 93.11602020263672,
      "activations/layer5_attention_weight_min": -86.30459594726562,
      "activations/layer6_attention_weight_max": 57.58067321777344,
      "activations/layer6_attention_weight_min": -50.95920181274414,
      "activations/layer7_attention_weight_max": 78.10803985595703,
      "activations/layer7_attention_weight_min": -70.73480987548828,
      "activations/layer8_attention_weight_max": 45.83198928833008,
      "activations/layer8_attention_weight_min": -45.217708587646484,
      "activations/layer9_attention_weight_max": 51.8537712097168,
      "activations/layer9_attention_weight_min": -48.18833541870117,
      "epoch": 7.21,
      "learning_rate": 0.00010455151515151514,
      "loss": 2.8275,
      "step": 124100
    },
    {
      "activations/layer0_attention_weight_max": 14.588692665100098,
      "activations/layer0_attention_weight_min": -14.583202362060547,
      "activations/layer10_attention_weight_max": 39.70451354980469,
      "activations/layer10_attention_weight_min": -38.067447662353516,
      "activations/layer11_attention_weight_max": 31.492971420288086,
      "activations/layer11_attention_weight_min": -32.29695129394531,
      "activations/layer12_attention_weight_max": 20.5635986328125,
      "activations/layer12_attention_weight_min": -33.68060302734375,
      "activations/layer13_attention_weight_max": 40.247772216796875,
      "activations/layer13_attention_weight_min": -31.513080596923828,
      "activations/layer14_attention_weight_max": 40.6839714050293,
      "activations/layer14_attention_weight_min": -31.956912994384766,
      "activations/layer15_attention_weight_max": 35.445220947265625,
      "activations/layer15_attention_weight_min": -31.01780891418457,
      "activations/layer16_attention_weight_max": 31.948535919189453,
      "activations/layer16_attention_weight_min": -26.53976821899414,
      "activations/layer17_attention_weight_max": 54.07669448852539,
      "activations/layer17_attention_weight_min": -41.99983215332031,
      "activations/layer18_attention_weight_max": 48.19255828857422,
      "activations/layer18_attention_weight_min": -36.61860656738281,
      "activations/layer19_attention_weight_max": 21.625839233398438,
      "activations/layer19_attention_weight_min": -20.37811851501465,
      "activations/layer1_attention_weight_max": 15.87010383605957,
      "activations/layer1_attention_weight_min": -12.946175575256348,
      "activations/layer20_attention_weight_max": 20.42917251586914,
      "activations/layer20_attention_weight_min": -20.395776748657227,
      "activations/layer21_attention_weight_max": 40.03119659423828,
      "activations/layer21_attention_weight_min": -26.07638931274414,
      "activations/layer22_attention_weight_max": 36.270111083984375,
      "activations/layer22_attention_weight_min": -26.367952346801758,
      "activations/layer23_attention_weight_max": 34.31983947753906,
      "activations/layer23_attention_weight_min": -21.658437728881836,
      "activations/layer2_attention_weight_max": 30.695941925048828,
      "activations/layer2_attention_weight_min": -27.7891845703125,
      "activations/layer3_attention_weight_max": 84.95484161376953,
      "activations/layer3_attention_weight_min": -82.23975372314453,
      "activations/layer4_attention_weight_max": 101.72631072998047,
      "activations/layer4_attention_weight_min": -91.48558807373047,
      "activations/layer5_attention_weight_max": 86.03472137451172,
      "activations/layer5_attention_weight_min": -78.9234390258789,
      "activations/layer6_attention_weight_max": 55.57914733886719,
      "activations/layer6_attention_weight_min": -48.8595085144043,
      "activations/layer7_attention_weight_max": 75.53096008300781,
      "activations/layer7_attention_weight_min": -70.0127944946289,
      "activations/layer8_attention_weight_max": 44.71753692626953,
      "activations/layer8_attention_weight_min": -43.7398681640625,
      "activations/layer9_attention_weight_max": 56.138832092285156,
      "activations/layer9_attention_weight_min": -50.42814254760742,
      "epoch": 7.21,
      "learning_rate": 0.00010453295454545453,
      "loss": 2.8127,
      "step": 124150
    },
    {
      "activations/layer0_attention_weight_max": 15.01643180847168,
      "activations/layer0_attention_weight_min": -14.423678398132324,
      "activations/layer10_attention_weight_max": 37.22222900390625,
      "activations/layer10_attention_weight_min": -36.9576301574707,
      "activations/layer11_attention_weight_max": 30.71695327758789,
      "activations/layer11_attention_weight_min": -31.41059684753418,
      "activations/layer12_attention_weight_max": 18.373193740844727,
      "activations/layer12_attention_weight_min": -23.873804092407227,
      "activations/layer13_attention_weight_max": 38.247413635253906,
      "activations/layer13_attention_weight_min": -30.498729705810547,
      "activations/layer14_attention_weight_max": 34.371307373046875,
      "activations/layer14_attention_weight_min": -28.89456558227539,
      "activations/layer15_attention_weight_max": 33.14803695678711,
      "activations/layer15_attention_weight_min": -28.66424560546875,
      "activations/layer16_attention_weight_max": 29.199365615844727,
      "activations/layer16_attention_weight_min": -28.012563705444336,
      "activations/layer17_attention_weight_max": 52.20867156982422,
      "activations/layer17_attention_weight_min": -38.57788848876953,
      "activations/layer18_attention_weight_max": 44.51801300048828,
      "activations/layer18_attention_weight_min": -33.24639129638672,
      "activations/layer19_attention_weight_max": 19.444442749023438,
      "activations/layer19_attention_weight_min": -20.481321334838867,
      "activations/layer1_attention_weight_max": 16.68868637084961,
      "activations/layer1_attention_weight_min": -13.173962593078613,
      "activations/layer20_attention_weight_max": 22.180850982666016,
      "activations/layer20_attention_weight_min": -27.008310317993164,
      "activations/layer21_attention_weight_max": 30.773113250732422,
      "activations/layer21_attention_weight_min": -22.201562881469727,
      "activations/layer22_attention_weight_max": 31.4846248626709,
      "activations/layer22_attention_weight_min": -29.0319881439209,
      "activations/layer23_attention_weight_max": 33.28593444824219,
      "activations/layer23_attention_weight_min": -23.425901412963867,
      "activations/layer2_attention_weight_max": 32.011322021484375,
      "activations/layer2_attention_weight_min": -29.257061004638672,
      "activations/layer3_attention_weight_max": 88.49974822998047,
      "activations/layer3_attention_weight_min": -90.4803237915039,
      "activations/layer4_attention_weight_max": 109.52555084228516,
      "activations/layer4_attention_weight_min": -98.08547973632812,
      "activations/layer5_attention_weight_max": 94.47874450683594,
      "activations/layer5_attention_weight_min": -85.40504455566406,
      "activations/layer6_attention_weight_max": 55.66387939453125,
      "activations/layer6_attention_weight_min": -53.56901550292969,
      "activations/layer7_attention_weight_max": 79.62734985351562,
      "activations/layer7_attention_weight_min": -74.17797088623047,
      "activations/layer8_attention_weight_max": 44.632415771484375,
      "activations/layer8_attention_weight_min": -46.67238998413086,
      "activations/layer9_attention_weight_max": 50.24635314941406,
      "activations/layer9_attention_weight_min": -50.28974533081055,
      "epoch": 7.22,
      "learning_rate": 0.00010451401515151514,
      "loss": 2.8208,
      "step": 124200
    },
    {
      "activations/layer0_attention_weight_max": 14.928631782531738,
      "activations/layer0_attention_weight_min": -14.269477844238281,
      "activations/layer10_attention_weight_max": 38.03230285644531,
      "activations/layer10_attention_weight_min": -34.44004821777344,
      "activations/layer11_attention_weight_max": 31.806516647338867,
      "activations/layer11_attention_weight_min": -29.69692039489746,
      "activations/layer12_attention_weight_max": 19.72426986694336,
      "activations/layer12_attention_weight_min": -23.14915657043457,
      "activations/layer13_attention_weight_max": 43.3917121887207,
      "activations/layer13_attention_weight_min": -32.20361328125,
      "activations/layer14_attention_weight_max": 41.023231506347656,
      "activations/layer14_attention_weight_min": -31.55534553527832,
      "activations/layer15_attention_weight_max": 36.17276382446289,
      "activations/layer15_attention_weight_min": -29.37550163269043,
      "activations/layer16_attention_weight_max": 32.23735809326172,
      "activations/layer16_attention_weight_min": -27.0668888092041,
      "activations/layer17_attention_weight_max": 55.70122528076172,
      "activations/layer17_attention_weight_min": -44.869529724121094,
      "activations/layer18_attention_weight_max": 49.45933532714844,
      "activations/layer18_attention_weight_min": -35.220577239990234,
      "activations/layer19_attention_weight_max": 21.92409324645996,
      "activations/layer19_attention_weight_min": -22.6845645904541,
      "activations/layer1_attention_weight_max": 18.045866012573242,
      "activations/layer1_attention_weight_min": -13.753040313720703,
      "activations/layer20_attention_weight_max": 22.573772430419922,
      "activations/layer20_attention_weight_min": -22.18903923034668,
      "activations/layer21_attention_weight_max": 44.1319580078125,
      "activations/layer21_attention_weight_min": -24.240446090698242,
      "activations/layer22_attention_weight_max": 31.362567901611328,
      "activations/layer22_attention_weight_min": -24.326066970825195,
      "activations/layer23_attention_weight_max": 34.66919708251953,
      "activations/layer23_attention_weight_min": -21.830137252807617,
      "activations/layer2_attention_weight_max": 29.60222625732422,
      "activations/layer2_attention_weight_min": -29.079410552978516,
      "activations/layer3_attention_weight_max": 78.7599868774414,
      "activations/layer3_attention_weight_min": -83.42291259765625,
      "activations/layer4_attention_weight_max": 95.75330352783203,
      "activations/layer4_attention_weight_min": -89.28401184082031,
      "activations/layer5_attention_weight_max": 90.19699096679688,
      "activations/layer5_attention_weight_min": -79.14168548583984,
      "activations/layer6_attention_weight_max": 53.63667678833008,
      "activations/layer6_attention_weight_min": -49.20559310913086,
      "activations/layer7_attention_weight_max": 70.06836700439453,
      "activations/layer7_attention_weight_min": -71.19062805175781,
      "activations/layer8_attention_weight_max": 45.68918228149414,
      "activations/layer8_attention_weight_min": -44.7713508605957,
      "activations/layer9_attention_weight_max": 53.4959831237793,
      "activations/layer9_attention_weight_min": -47.9625129699707,
      "epoch": 7.22,
      "learning_rate": 0.00010449507575757576,
      "loss": 2.8299,
      "step": 124250
    },
    {
      "activations/layer0_attention_weight_max": 14.37922191619873,
      "activations/layer0_attention_weight_min": -14.107213973999023,
      "activations/layer10_attention_weight_max": 40.52899932861328,
      "activations/layer10_attention_weight_min": -37.62052917480469,
      "activations/layer11_attention_weight_max": 31.793983459472656,
      "activations/layer11_attention_weight_min": -32.62834167480469,
      "activations/layer12_attention_weight_max": 24.770462036132812,
      "activations/layer12_attention_weight_min": -23.918231964111328,
      "activations/layer13_attention_weight_max": 55.02916717529297,
      "activations/layer13_attention_weight_min": -34.56586837768555,
      "activations/layer14_attention_weight_max": 59.169376373291016,
      "activations/layer14_attention_weight_min": -34.65135955810547,
      "activations/layer15_attention_weight_max": 44.69207763671875,
      "activations/layer15_attention_weight_min": -29.611814498901367,
      "activations/layer16_attention_weight_max": 31.957548141479492,
      "activations/layer16_attention_weight_min": -29.552087783813477,
      "activations/layer17_attention_weight_max": 55.37395095825195,
      "activations/layer17_attention_weight_min": -47.53293228149414,
      "activations/layer18_attention_weight_max": 46.69495391845703,
      "activations/layer18_attention_weight_min": -37.563167572021484,
      "activations/layer19_attention_weight_max": 24.623802185058594,
      "activations/layer19_attention_weight_min": -21.13998794555664,
      "activations/layer1_attention_weight_max": 18.44957160949707,
      "activations/layer1_attention_weight_min": -13.359292030334473,
      "activations/layer20_attention_weight_max": 21.496952056884766,
      "activations/layer20_attention_weight_min": -22.06916046142578,
      "activations/layer21_attention_weight_max": 46.609100341796875,
      "activations/layer21_attention_weight_min": -28.30816650390625,
      "activations/layer22_attention_weight_max": 32.15927505493164,
      "activations/layer22_attention_weight_min": -26.5245304107666,
      "activations/layer23_attention_weight_max": 31.955669403076172,
      "activations/layer23_attention_weight_min": -21.57494354248047,
      "activations/layer2_attention_weight_max": 30.504722595214844,
      "activations/layer2_attention_weight_min": -26.95409393310547,
      "activations/layer3_attention_weight_max": 84.90623474121094,
      "activations/layer3_attention_weight_min": -87.0431900024414,
      "activations/layer4_attention_weight_max": 98.25366973876953,
      "activations/layer4_attention_weight_min": -93.41393280029297,
      "activations/layer5_attention_weight_max": 89.27630615234375,
      "activations/layer5_attention_weight_min": -84.26136779785156,
      "activations/layer6_attention_weight_max": 56.788360595703125,
      "activations/layer6_attention_weight_min": -51.974586486816406,
      "activations/layer7_attention_weight_max": 74.5735092163086,
      "activations/layer7_attention_weight_min": -74.03947448730469,
      "activations/layer8_attention_weight_max": 45.770729064941406,
      "activations/layer8_attention_weight_min": -48.39430618286133,
      "activations/layer9_attention_weight_max": 60.545745849609375,
      "activations/layer9_attention_weight_min": -51.968238830566406,
      "epoch": 7.22,
      "learning_rate": 0.00010447613636363635,
      "loss": 2.8234,
      "step": 124300
    },
    {
      "activations/layer0_attention_weight_max": 14.759317398071289,
      "activations/layer0_attention_weight_min": -14.840974807739258,
      "activations/layer10_attention_weight_max": 36.177757263183594,
      "activations/layer10_attention_weight_min": -34.30681228637695,
      "activations/layer11_attention_weight_max": 29.416982650756836,
      "activations/layer11_attention_weight_min": -29.55103302001953,
      "activations/layer12_attention_weight_max": 20.08658218383789,
      "activations/layer12_attention_weight_min": -23.65188217163086,
      "activations/layer13_attention_weight_max": 38.153419494628906,
      "activations/layer13_attention_weight_min": -32.65915298461914,
      "activations/layer14_attention_weight_max": 37.37614822387695,
      "activations/layer14_attention_weight_min": -31.00399398803711,
      "activations/layer15_attention_weight_max": 33.8594970703125,
      "activations/layer15_attention_weight_min": -30.068870544433594,
      "activations/layer16_attention_weight_max": 30.87607765197754,
      "activations/layer16_attention_weight_min": -26.981477737426758,
      "activations/layer17_attention_weight_max": 56.10417938232422,
      "activations/layer17_attention_weight_min": -45.93719482421875,
      "activations/layer18_attention_weight_max": 49.72380065917969,
      "activations/layer18_attention_weight_min": -38.02436447143555,
      "activations/layer19_attention_weight_max": 23.963260650634766,
      "activations/layer19_attention_weight_min": -21.08283805847168,
      "activations/layer1_attention_weight_max": 16.813434600830078,
      "activations/layer1_attention_weight_min": -14.718082427978516,
      "activations/layer20_attention_weight_max": 20.541276931762695,
      "activations/layer20_attention_weight_min": -22.114913940429688,
      "activations/layer21_attention_weight_max": 42.74233627319336,
      "activations/layer21_attention_weight_min": -25.89318084716797,
      "activations/layer22_attention_weight_max": 32.13084411621094,
      "activations/layer22_attention_weight_min": -25.452226638793945,
      "activations/layer23_attention_weight_max": 34.78669738769531,
      "activations/layer23_attention_weight_min": -21.51025390625,
      "activations/layer2_attention_weight_max": 29.875450134277344,
      "activations/layer2_attention_weight_min": -27.2935848236084,
      "activations/layer3_attention_weight_max": 81.62989044189453,
      "activations/layer3_attention_weight_min": -86.11627960205078,
      "activations/layer4_attention_weight_max": 93.08035278320312,
      "activations/layer4_attention_weight_min": -95.1201400756836,
      "activations/layer5_attention_weight_max": 87.40550231933594,
      "activations/layer5_attention_weight_min": -81.81788635253906,
      "activations/layer6_attention_weight_max": 57.83113098144531,
      "activations/layer6_attention_weight_min": -52.87149429321289,
      "activations/layer7_attention_weight_max": 75.20574951171875,
      "activations/layer7_attention_weight_min": -72.02950286865234,
      "activations/layer8_attention_weight_max": 44.076072692871094,
      "activations/layer8_attention_weight_min": -44.4686164855957,
      "activations/layer9_attention_weight_max": 46.90371322631836,
      "activations/layer9_attention_weight_min": -49.64823532104492,
      "epoch": 7.23,
      "learning_rate": 0.00010445719696969696,
      "loss": 2.8193,
      "step": 124350
    },
    {
      "activations/layer0_attention_weight_max": 14.491392135620117,
      "activations/layer0_attention_weight_min": -14.889983177185059,
      "activations/layer10_attention_weight_max": 40.301544189453125,
      "activations/layer10_attention_weight_min": -37.73760986328125,
      "activations/layer11_attention_weight_max": 32.35894012451172,
      "activations/layer11_attention_weight_min": -30.743213653564453,
      "activations/layer12_attention_weight_max": 21.480497360229492,
      "activations/layer12_attention_weight_min": -28.51264762878418,
      "activations/layer13_attention_weight_max": 44.20918273925781,
      "activations/layer13_attention_weight_min": -30.603120803833008,
      "activations/layer14_attention_weight_max": 39.14278793334961,
      "activations/layer14_attention_weight_min": -33.222412109375,
      "activations/layer15_attention_weight_max": 38.59809112548828,
      "activations/layer15_attention_weight_min": -29.756053924560547,
      "activations/layer16_attention_weight_max": 32.56001663208008,
      "activations/layer16_attention_weight_min": -27.209341049194336,
      "activations/layer17_attention_weight_max": 54.766937255859375,
      "activations/layer17_attention_weight_min": -41.55754852294922,
      "activations/layer18_attention_weight_max": 49.07101058959961,
      "activations/layer18_attention_weight_min": -34.760284423828125,
      "activations/layer19_attention_weight_max": 22.944570541381836,
      "activations/layer19_attention_weight_min": -19.611478805541992,
      "activations/layer1_attention_weight_max": 16.38678741455078,
      "activations/layer1_attention_weight_min": -15.16421890258789,
      "activations/layer20_attention_weight_max": 20.6995849609375,
      "activations/layer20_attention_weight_min": -24.48638916015625,
      "activations/layer21_attention_weight_max": 37.37714385986328,
      "activations/layer21_attention_weight_min": -23.640289306640625,
      "activations/layer22_attention_weight_max": 34.790740966796875,
      "activations/layer22_attention_weight_min": -25.578250885009766,
      "activations/layer23_attention_weight_max": 38.63795471191406,
      "activations/layer23_attention_weight_min": -23.162452697753906,
      "activations/layer2_attention_weight_max": 31.914058685302734,
      "activations/layer2_attention_weight_min": -28.6800594329834,
      "activations/layer3_attention_weight_max": 80.84636688232422,
      "activations/layer3_attention_weight_min": -85.7416000366211,
      "activations/layer4_attention_weight_max": 97.15550994873047,
      "activations/layer4_attention_weight_min": -94.20098114013672,
      "activations/layer5_attention_weight_max": 89.75502014160156,
      "activations/layer5_attention_weight_min": -79.35438537597656,
      "activations/layer6_attention_weight_max": 55.04767990112305,
      "activations/layer6_attention_weight_min": -49.82018280029297,
      "activations/layer7_attention_weight_max": 72.19229888916016,
      "activations/layer7_attention_weight_min": -69.32540130615234,
      "activations/layer8_attention_weight_max": 44.00493240356445,
      "activations/layer8_attention_weight_min": -43.07910919189453,
      "activations/layer9_attention_weight_max": 47.0145263671875,
      "activations/layer9_attention_weight_min": -46.2616081237793,
      "epoch": 7.23,
      "learning_rate": 0.00010443825757575758,
      "loss": 2.8352,
      "step": 124400
    },
    {
      "activations/layer0_attention_weight_max": 15.411182403564453,
      "activations/layer0_attention_weight_min": -14.884876251220703,
      "activations/layer10_attention_weight_max": 41.1189079284668,
      "activations/layer10_attention_weight_min": -36.93007278442383,
      "activations/layer11_attention_weight_max": 34.76757049560547,
      "activations/layer11_attention_weight_min": -29.39228630065918,
      "activations/layer12_attention_weight_max": 19.854894638061523,
      "activations/layer12_attention_weight_min": -28.92015266418457,
      "activations/layer13_attention_weight_max": 46.94447708129883,
      "activations/layer13_attention_weight_min": -34.405914306640625,
      "activations/layer14_attention_weight_max": 47.73926544189453,
      "activations/layer14_attention_weight_min": -32.57184600830078,
      "activations/layer15_attention_weight_max": 41.46134567260742,
      "activations/layer15_attention_weight_min": -29.22587776184082,
      "activations/layer16_attention_weight_max": 39.43278884887695,
      "activations/layer16_attention_weight_min": -27.25522232055664,
      "activations/layer17_attention_weight_max": 58.99650955200195,
      "activations/layer17_attention_weight_min": -41.85502624511719,
      "activations/layer18_attention_weight_max": 48.946475982666016,
      "activations/layer18_attention_weight_min": -36.314910888671875,
      "activations/layer19_attention_weight_max": 21.927885055541992,
      "activations/layer19_attention_weight_min": -21.449060440063477,
      "activations/layer1_attention_weight_max": 17.67784309387207,
      "activations/layer1_attention_weight_min": -17.015968322753906,
      "activations/layer20_attention_weight_max": 25.663658142089844,
      "activations/layer20_attention_weight_min": -21.714689254760742,
      "activations/layer21_attention_weight_max": 40.14656448364258,
      "activations/layer21_attention_weight_min": -25.993576049804688,
      "activations/layer22_attention_weight_max": 33.94676208496094,
      "activations/layer22_attention_weight_min": -25.36918830871582,
      "activations/layer23_attention_weight_max": 35.79984664916992,
      "activations/layer23_attention_weight_min": -22.050132751464844,
      "activations/layer2_attention_weight_max": 33.70637512207031,
      "activations/layer2_attention_weight_min": -31.215648651123047,
      "activations/layer3_attention_weight_max": 85.45346069335938,
      "activations/layer3_attention_weight_min": -86.47417449951172,
      "activations/layer4_attention_weight_max": 100.11415100097656,
      "activations/layer4_attention_weight_min": -92.92037200927734,
      "activations/layer5_attention_weight_max": 90.71467590332031,
      "activations/layer5_attention_weight_min": -81.66387939453125,
      "activations/layer6_attention_weight_max": 54.84373092651367,
      "activations/layer6_attention_weight_min": -52.039466857910156,
      "activations/layer7_attention_weight_max": 80.55286407470703,
      "activations/layer7_attention_weight_min": -72.77088928222656,
      "activations/layer8_attention_weight_max": 47.72181701660156,
      "activations/layer8_attention_weight_min": -44.35004425048828,
      "activations/layer9_attention_weight_max": 58.43775177001953,
      "activations/layer9_attention_weight_min": -51.2713623046875,
      "epoch": 7.23,
      "learning_rate": 0.00010441931818181816,
      "loss": 2.8091,
      "step": 124450
    },
    {
      "activations/layer0_attention_weight_max": 15.692632675170898,
      "activations/layer0_attention_weight_min": -14.535440444946289,
      "activations/layer10_attention_weight_max": 41.934967041015625,
      "activations/layer10_attention_weight_min": -36.741485595703125,
      "activations/layer11_attention_weight_max": 34.71533966064453,
      "activations/layer11_attention_weight_min": -30.209163665771484,
      "activations/layer12_attention_weight_max": 25.862688064575195,
      "activations/layer12_attention_weight_min": -24.00766944885254,
      "activations/layer13_attention_weight_max": 46.09022521972656,
      "activations/layer13_attention_weight_min": -30.861263275146484,
      "activations/layer14_attention_weight_max": 49.0921630859375,
      "activations/layer14_attention_weight_min": -30.866756439208984,
      "activations/layer15_attention_weight_max": 41.379852294921875,
      "activations/layer15_attention_weight_min": -29.912933349609375,
      "activations/layer16_attention_weight_max": 35.10660171508789,
      "activations/layer16_attention_weight_min": -26.239042282104492,
      "activations/layer17_attention_weight_max": 56.64651870727539,
      "activations/layer17_attention_weight_min": -41.646636962890625,
      "activations/layer18_attention_weight_max": 49.76118850708008,
      "activations/layer18_attention_weight_min": -35.448089599609375,
      "activations/layer19_attention_weight_max": 23.22549057006836,
      "activations/layer19_attention_weight_min": -20.997699737548828,
      "activations/layer1_attention_weight_max": 18.498310089111328,
      "activations/layer1_attention_weight_min": -14.357266426086426,
      "activations/layer20_attention_weight_max": 22.249853134155273,
      "activations/layer20_attention_weight_min": -20.114837646484375,
      "activations/layer21_attention_weight_max": 41.360504150390625,
      "activations/layer21_attention_weight_min": -24.16940689086914,
      "activations/layer22_attention_weight_max": 29.469112396240234,
      "activations/layer22_attention_weight_min": -26.992172241210938,
      "activations/layer23_attention_weight_max": 33.36070251464844,
      "activations/layer23_attention_weight_min": -22.92660140991211,
      "activations/layer2_attention_weight_max": 31.89594078063965,
      "activations/layer2_attention_weight_min": -29.21700668334961,
      "activations/layer3_attention_weight_max": 85.31653594970703,
      "activations/layer3_attention_weight_min": -86.12706756591797,
      "activations/layer4_attention_weight_max": 97.6429443359375,
      "activations/layer4_attention_weight_min": -89.08055114746094,
      "activations/layer5_attention_weight_max": 86.58419036865234,
      "activations/layer5_attention_weight_min": -78.01438903808594,
      "activations/layer6_attention_weight_max": 57.61765670776367,
      "activations/layer6_attention_weight_min": -52.92073440551758,
      "activations/layer7_attention_weight_max": 79.67623138427734,
      "activations/layer7_attention_weight_min": -72.04478454589844,
      "activations/layer8_attention_weight_max": 48.276893615722656,
      "activations/layer8_attention_weight_min": -48.63404083251953,
      "activations/layer9_attention_weight_max": 54.446685791015625,
      "activations/layer9_attention_weight_min": -48.18231964111328,
      "epoch": 7.23,
      "learning_rate": 0.00010440037878787878,
      "loss": 2.8097,
      "step": 124500
    },
    {
      "activations/layer0_attention_weight_max": 14.797295570373535,
      "activations/layer0_attention_weight_min": -15.201974868774414,
      "activations/layer10_attention_weight_max": 35.306968688964844,
      "activations/layer10_attention_weight_min": -34.470848083496094,
      "activations/layer11_attention_weight_max": 30.956764221191406,
      "activations/layer11_attention_weight_min": -30.897977828979492,
      "activations/layer12_attention_weight_max": 20.400602340698242,
      "activations/layer12_attention_weight_min": -25.765745162963867,
      "activations/layer13_attention_weight_max": 37.253623962402344,
      "activations/layer13_attention_weight_min": -28.79354476928711,
      "activations/layer14_attention_weight_max": 36.68803787231445,
      "activations/layer14_attention_weight_min": -28.192001342773438,
      "activations/layer15_attention_weight_max": 32.380279541015625,
      "activations/layer15_attention_weight_min": -28.921728134155273,
      "activations/layer16_attention_weight_max": 31.051851272583008,
      "activations/layer16_attention_weight_min": -26.385684967041016,
      "activations/layer17_attention_weight_max": 51.05440139770508,
      "activations/layer17_attention_weight_min": -39.934165954589844,
      "activations/layer18_attention_weight_max": 47.337100982666016,
      "activations/layer18_attention_weight_min": -33.22431945800781,
      "activations/layer19_attention_weight_max": 22.042810440063477,
      "activations/layer19_attention_weight_min": -20.1407527923584,
      "activations/layer1_attention_weight_max": 17.58820343017578,
      "activations/layer1_attention_weight_min": -14.113055229187012,
      "activations/layer20_attention_weight_max": 21.595312118530273,
      "activations/layer20_attention_weight_min": -21.559452056884766,
      "activations/layer21_attention_weight_max": 42.75007247924805,
      "activations/layer21_attention_weight_min": -24.816593170166016,
      "activations/layer22_attention_weight_max": 31.0658016204834,
      "activations/layer22_attention_weight_min": -24.43999481201172,
      "activations/layer23_attention_weight_max": 31.657371520996094,
      "activations/layer23_attention_weight_min": -23.60568618774414,
      "activations/layer2_attention_weight_max": 31.327125549316406,
      "activations/layer2_attention_weight_min": -28.429218292236328,
      "activations/layer3_attention_weight_max": 84.17249298095703,
      "activations/layer3_attention_weight_min": -86.01639556884766,
      "activations/layer4_attention_weight_max": 96.43868255615234,
      "activations/layer4_attention_weight_min": -89.10421752929688,
      "activations/layer5_attention_weight_max": 87.57229614257812,
      "activations/layer5_attention_weight_min": -80.4975814819336,
      "activations/layer6_attention_weight_max": 53.576637268066406,
      "activations/layer6_attention_weight_min": -52.28008270263672,
      "activations/layer7_attention_weight_max": 75.16329193115234,
      "activations/layer7_attention_weight_min": -71.87866973876953,
      "activations/layer8_attention_weight_max": 43.855865478515625,
      "activations/layer8_attention_weight_min": -42.34025192260742,
      "activations/layer9_attention_weight_max": 56.96149444580078,
      "activations/layer9_attention_weight_min": -47.14518737792969,
      "epoch": 7.24,
      "learning_rate": 0.0001043814393939394,
      "loss": 2.8148,
      "step": 124550
    },
    {
      "activations/layer0_attention_weight_max": 14.426751136779785,
      "activations/layer0_attention_weight_min": -14.896056175231934,
      "activations/layer10_attention_weight_max": 42.89369583129883,
      "activations/layer10_attention_weight_min": -39.0371208190918,
      "activations/layer11_attention_weight_max": 34.59048843383789,
      "activations/layer11_attention_weight_min": -33.50991439819336,
      "activations/layer12_attention_weight_max": 24.163516998291016,
      "activations/layer12_attention_weight_min": -24.72442626953125,
      "activations/layer13_attention_weight_max": 47.04948043823242,
      "activations/layer13_attention_weight_min": -38.240962982177734,
      "activations/layer14_attention_weight_max": 47.071285247802734,
      "activations/layer14_attention_weight_min": -30.74479103088379,
      "activations/layer15_attention_weight_max": 37.87635803222656,
      "activations/layer15_attention_weight_min": -29.990802764892578,
      "activations/layer16_attention_weight_max": 34.75242233276367,
      "activations/layer16_attention_weight_min": -27.165239334106445,
      "activations/layer17_attention_weight_max": 58.24889373779297,
      "activations/layer17_attention_weight_min": -43.73448944091797,
      "activations/layer18_attention_weight_max": 54.51748275756836,
      "activations/layer18_attention_weight_min": -37.28291702270508,
      "activations/layer19_attention_weight_max": 22.343263626098633,
      "activations/layer19_attention_weight_min": -20.533044815063477,
      "activations/layer1_attention_weight_max": 15.425939559936523,
      "activations/layer1_attention_weight_min": -12.820630073547363,
      "activations/layer20_attention_weight_max": 27.858436584472656,
      "activations/layer20_attention_weight_min": -20.581417083740234,
      "activations/layer21_attention_weight_max": 45.01156997680664,
      "activations/layer21_attention_weight_min": -25.36679458618164,
      "activations/layer22_attention_weight_max": 33.82776641845703,
      "activations/layer22_attention_weight_min": -24.22463607788086,
      "activations/layer23_attention_weight_max": 35.75275421142578,
      "activations/layer23_attention_weight_min": -20.159034729003906,
      "activations/layer2_attention_weight_max": 30.784664154052734,
      "activations/layer2_attention_weight_min": -27.425594329833984,
      "activations/layer3_attention_weight_max": 88.82035827636719,
      "activations/layer3_attention_weight_min": -84.06476593017578,
      "activations/layer4_attention_weight_max": 106.94266510009766,
      "activations/layer4_attention_weight_min": -98.69310760498047,
      "activations/layer5_attention_weight_max": 98.45198059082031,
      "activations/layer5_attention_weight_min": -90.48638153076172,
      "activations/layer6_attention_weight_max": 62.34619903564453,
      "activations/layer6_attention_weight_min": -57.244300842285156,
      "activations/layer7_attention_weight_max": 81.60820007324219,
      "activations/layer7_attention_weight_min": -79.1222915649414,
      "activations/layer8_attention_weight_max": 49.99709701538086,
      "activations/layer8_attention_weight_min": -50.037452697753906,
      "activations/layer9_attention_weight_max": 55.18905258178711,
      "activations/layer9_attention_weight_min": -54.55705642700195,
      "epoch": 7.24,
      "learning_rate": 0.00010436249999999998,
      "loss": 2.8231,
      "step": 124600
    },
    {
      "activations/layer0_attention_weight_max": 15.152637481689453,
      "activations/layer0_attention_weight_min": -14.935446739196777,
      "activations/layer10_attention_weight_max": 41.069793701171875,
      "activations/layer10_attention_weight_min": -37.220375061035156,
      "activations/layer11_attention_weight_max": 33.90271759033203,
      "activations/layer11_attention_weight_min": -30.725996017456055,
      "activations/layer12_attention_weight_max": 22.08666229248047,
      "activations/layer12_attention_weight_min": -23.518341064453125,
      "activations/layer13_attention_weight_max": 41.6075325012207,
      "activations/layer13_attention_weight_min": -32.97042465209961,
      "activations/layer14_attention_weight_max": 39.66463088989258,
      "activations/layer14_attention_weight_min": -30.93198585510254,
      "activations/layer15_attention_weight_max": 36.85744857788086,
      "activations/layer15_attention_weight_min": -29.533937454223633,
      "activations/layer16_attention_weight_max": 36.16347122192383,
      "activations/layer16_attention_weight_min": -26.34931182861328,
      "activations/layer17_attention_weight_max": 55.51740646362305,
      "activations/layer17_attention_weight_min": -40.39271545410156,
      "activations/layer18_attention_weight_max": 47.739322662353516,
      "activations/layer18_attention_weight_min": -34.56596374511719,
      "activations/layer19_attention_weight_max": 20.017820358276367,
      "activations/layer19_attention_weight_min": -18.97490692138672,
      "activations/layer1_attention_weight_max": 17.796316146850586,
      "activations/layer1_attention_weight_min": -14.666589736938477,
      "activations/layer20_attention_weight_max": 22.773523330688477,
      "activations/layer20_attention_weight_min": -18.959951400756836,
      "activations/layer21_attention_weight_max": 36.045082092285156,
      "activations/layer21_attention_weight_min": -23.16036033630371,
      "activations/layer22_attention_weight_max": 33.95130920410156,
      "activations/layer22_attention_weight_min": -23.40693473815918,
      "activations/layer23_attention_weight_max": 37.77833557128906,
      "activations/layer23_attention_weight_min": -23.466405868530273,
      "activations/layer2_attention_weight_max": 32.15214157104492,
      "activations/layer2_attention_weight_min": -29.279569625854492,
      "activations/layer3_attention_weight_max": 86.87586212158203,
      "activations/layer3_attention_weight_min": -86.9117431640625,
      "activations/layer4_attention_weight_max": 106.4087142944336,
      "activations/layer4_attention_weight_min": -95.65855407714844,
      "activations/layer5_attention_weight_max": 90.3281478881836,
      "activations/layer5_attention_weight_min": -83.52972412109375,
      "activations/layer6_attention_weight_max": 56.774810791015625,
      "activations/layer6_attention_weight_min": -52.53158950805664,
      "activations/layer7_attention_weight_max": 76.16085052490234,
      "activations/layer7_attention_weight_min": -71.32237243652344,
      "activations/layer8_attention_weight_max": 44.40625762939453,
      "activations/layer8_attention_weight_min": -47.05326461791992,
      "activations/layer9_attention_weight_max": 54.3513069152832,
      "activations/layer9_attention_weight_min": -50.4386100769043,
      "epoch": 7.24,
      "learning_rate": 0.0001043435606060606,
      "loss": 2.8169,
      "step": 124650
    },
    {
      "activations/layer0_attention_weight_max": 15.539582252502441,
      "activations/layer0_attention_weight_min": -15.09390926361084,
      "activations/layer10_attention_weight_max": 38.070594787597656,
      "activations/layer10_attention_weight_min": -34.75690841674805,
      "activations/layer11_attention_weight_max": 31.32114028930664,
      "activations/layer11_attention_weight_min": -30.034088134765625,
      "activations/layer12_attention_weight_max": 21.162290573120117,
      "activations/layer12_attention_weight_min": -23.51139259338379,
      "activations/layer13_attention_weight_max": 43.89357376098633,
      "activations/layer13_attention_weight_min": -31.036235809326172,
      "activations/layer14_attention_weight_max": 39.582969665527344,
      "activations/layer14_attention_weight_min": -28.96286964416504,
      "activations/layer15_attention_weight_max": 34.808902740478516,
      "activations/layer15_attention_weight_min": -29.779552459716797,
      "activations/layer16_attention_weight_max": 32.103057861328125,
      "activations/layer16_attention_weight_min": -29.0578670501709,
      "activations/layer17_attention_weight_max": 52.9953498840332,
      "activations/layer17_attention_weight_min": -40.42323303222656,
      "activations/layer18_attention_weight_max": 47.81007766723633,
      "activations/layer18_attention_weight_min": -35.34568405151367,
      "activations/layer19_attention_weight_max": 22.499258041381836,
      "activations/layer19_attention_weight_min": -19.30890464782715,
      "activations/layer1_attention_weight_max": 17.1881046295166,
      "activations/layer1_attention_weight_min": -15.735321998596191,
      "activations/layer20_attention_weight_max": 22.849775314331055,
      "activations/layer20_attention_weight_min": -24.157560348510742,
      "activations/layer21_attention_weight_max": 40.374542236328125,
      "activations/layer21_attention_weight_min": -25.520431518554688,
      "activations/layer22_attention_weight_max": 34.957054138183594,
      "activations/layer22_attention_weight_min": -27.80841064453125,
      "activations/layer23_attention_weight_max": 35.261146545410156,
      "activations/layer23_attention_weight_min": -22.02214813232422,
      "activations/layer2_attention_weight_max": 30.751819610595703,
      "activations/layer2_attention_weight_min": -28.353506088256836,
      "activations/layer3_attention_weight_max": 81.40333557128906,
      "activations/layer3_attention_weight_min": -87.39413452148438,
      "activations/layer4_attention_weight_max": 94.45845794677734,
      "activations/layer4_attention_weight_min": -92.8892593383789,
      "activations/layer5_attention_weight_max": 88.00618743896484,
      "activations/layer5_attention_weight_min": -80.28584289550781,
      "activations/layer6_attention_weight_max": 57.25278854370117,
      "activations/layer6_attention_weight_min": -54.38755798339844,
      "activations/layer7_attention_weight_max": 73.1956787109375,
      "activations/layer7_attention_weight_min": -78.16036987304688,
      "activations/layer8_attention_weight_max": 45.09009552001953,
      "activations/layer8_attention_weight_min": -43.283485412597656,
      "activations/layer9_attention_weight_max": 51.50347900390625,
      "activations/layer9_attention_weight_min": -50.91032791137695,
      "epoch": 7.25,
      "learning_rate": 0.0001043246212121212,
      "loss": 2.8057,
      "step": 124700
    },
    {
      "activations/layer0_attention_weight_max": 14.28463363647461,
      "activations/layer0_attention_weight_min": -14.980056762695312,
      "activations/layer10_attention_weight_max": 38.157833099365234,
      "activations/layer10_attention_weight_min": -35.36820983886719,
      "activations/layer11_attention_weight_max": 33.036048889160156,
      "activations/layer11_attention_weight_min": -32.13587951660156,
      "activations/layer12_attention_weight_max": 23.82387924194336,
      "activations/layer12_attention_weight_min": -25.244836807250977,
      "activations/layer13_attention_weight_max": 38.680110931396484,
      "activations/layer13_attention_weight_min": -29.67999839782715,
      "activations/layer14_attention_weight_max": 38.30692672729492,
      "activations/layer14_attention_weight_min": -28.652923583984375,
      "activations/layer15_attention_weight_max": 32.33198165893555,
      "activations/layer15_attention_weight_min": -27.57756996154785,
      "activations/layer16_attention_weight_max": 31.733531951904297,
      "activations/layer16_attention_weight_min": -31.084619522094727,
      "activations/layer17_attention_weight_max": 47.41286087036133,
      "activations/layer17_attention_weight_min": -40.563079833984375,
      "activations/layer18_attention_weight_max": 44.55411911010742,
      "activations/layer18_attention_weight_min": -33.946693420410156,
      "activations/layer19_attention_weight_max": 20.630395889282227,
      "activations/layer19_attention_weight_min": -20.059589385986328,
      "activations/layer1_attention_weight_max": 17.562498092651367,
      "activations/layer1_attention_weight_min": -14.481918334960938,
      "activations/layer20_attention_weight_max": 21.129192352294922,
      "activations/layer20_attention_weight_min": -21.40151596069336,
      "activations/layer21_attention_weight_max": 40.113792419433594,
      "activations/layer21_attention_weight_min": -28.951034545898438,
      "activations/layer22_attention_weight_max": 31.29982566833496,
      "activations/layer22_attention_weight_min": -24.36359214782715,
      "activations/layer23_attention_weight_max": 30.061315536499023,
      "activations/layer23_attention_weight_min": -22.09792137145996,
      "activations/layer2_attention_weight_max": 31.60002899169922,
      "activations/layer2_attention_weight_min": -28.644607543945312,
      "activations/layer3_attention_weight_max": 84.65966033935547,
      "activations/layer3_attention_weight_min": -84.95479583740234,
      "activations/layer4_attention_weight_max": 103.84090423583984,
      "activations/layer4_attention_weight_min": -94.32250213623047,
      "activations/layer5_attention_weight_max": 95.63655853271484,
      "activations/layer5_attention_weight_min": -85.2589111328125,
      "activations/layer6_attention_weight_max": 56.654502868652344,
      "activations/layer6_attention_weight_min": -56.05687713623047,
      "activations/layer7_attention_weight_max": 77.02445983886719,
      "activations/layer7_attention_weight_min": -75.10765075683594,
      "activations/layer8_attention_weight_max": 45.597747802734375,
      "activations/layer8_attention_weight_min": -45.03916549682617,
      "activations/layer9_attention_weight_max": 53.39492416381836,
      "activations/layer9_attention_weight_min": -49.05557632446289,
      "epoch": 7.25,
      "learning_rate": 0.00010430568181818181,
      "loss": 2.8348,
      "step": 124750
    },
    {
      "activations/layer0_attention_weight_max": 14.938755989074707,
      "activations/layer0_attention_weight_min": -15.020625114440918,
      "activations/layer10_attention_weight_max": 41.880516052246094,
      "activations/layer10_attention_weight_min": -36.86669921875,
      "activations/layer11_attention_weight_max": 33.779319763183594,
      "activations/layer11_attention_weight_min": -31.846446990966797,
      "activations/layer12_attention_weight_max": 35.84705352783203,
      "activations/layer12_attention_weight_min": -25.39897346496582,
      "activations/layer13_attention_weight_max": 48.37883758544922,
      "activations/layer13_attention_weight_min": -34.41419982910156,
      "activations/layer14_attention_weight_max": 39.5991325378418,
      "activations/layer14_attention_weight_min": -31.51220703125,
      "activations/layer15_attention_weight_max": 46.29176330566406,
      "activations/layer15_attention_weight_min": -31.270151138305664,
      "activations/layer16_attention_weight_max": 30.753490447998047,
      "activations/layer16_attention_weight_min": -28.692461013793945,
      "activations/layer17_attention_weight_max": 60.9216194152832,
      "activations/layer17_attention_weight_min": -45.79866409301758,
      "activations/layer18_attention_weight_max": 50.458744049072266,
      "activations/layer18_attention_weight_min": -34.39024353027344,
      "activations/layer19_attention_weight_max": 25.192256927490234,
      "activations/layer19_attention_weight_min": -19.963964462280273,
      "activations/layer1_attention_weight_max": 16.301395416259766,
      "activations/layer1_attention_weight_min": -15.100776672363281,
      "activations/layer20_attention_weight_max": 26.266033172607422,
      "activations/layer20_attention_weight_min": -20.766218185424805,
      "activations/layer21_attention_weight_max": 45.754669189453125,
      "activations/layer21_attention_weight_min": -24.897464752197266,
      "activations/layer22_attention_weight_max": 31.501550674438477,
      "activations/layer22_attention_weight_min": -25.430011749267578,
      "activations/layer23_attention_weight_max": 31.115079879760742,
      "activations/layer23_attention_weight_min": -19.191072463989258,
      "activations/layer2_attention_weight_max": 33.523109436035156,
      "activations/layer2_attention_weight_min": -32.164520263671875,
      "activations/layer3_attention_weight_max": 88.47257232666016,
      "activations/layer3_attention_weight_min": -88.56449127197266,
      "activations/layer4_attention_weight_max": 104.4029769897461,
      "activations/layer4_attention_weight_min": -96.74124145507812,
      "activations/layer5_attention_weight_max": 96.28783416748047,
      "activations/layer5_attention_weight_min": -89.93971252441406,
      "activations/layer6_attention_weight_max": 58.90293884277344,
      "activations/layer6_attention_weight_min": -56.78205871582031,
      "activations/layer7_attention_weight_max": 81.01371765136719,
      "activations/layer7_attention_weight_min": -76.82230377197266,
      "activations/layer8_attention_weight_max": 49.672828674316406,
      "activations/layer8_attention_weight_min": -49.154212951660156,
      "activations/layer9_attention_weight_max": 56.51558303833008,
      "activations/layer9_attention_weight_min": -51.55970001220703,
      "epoch": 7.25,
      "learning_rate": 0.00010428674242424242,
      "loss": 2.8134,
      "step": 124800
    },
    {
      "activations/layer0_attention_weight_max": 15.139769554138184,
      "activations/layer0_attention_weight_min": -14.840313911437988,
      "activations/layer10_attention_weight_max": 41.02893829345703,
      "activations/layer10_attention_weight_min": -37.21269226074219,
      "activations/layer11_attention_weight_max": 29.911291122436523,
      "activations/layer11_attention_weight_min": -30.655746459960938,
      "activations/layer12_attention_weight_max": 18.364727020263672,
      "activations/layer12_attention_weight_min": -24.624664306640625,
      "activations/layer13_attention_weight_max": 44.007774353027344,
      "activations/layer13_attention_weight_min": -34.98179626464844,
      "activations/layer14_attention_weight_max": 40.136199951171875,
      "activations/layer14_attention_weight_min": -32.22503662109375,
      "activations/layer15_attention_weight_max": 37.82916259765625,
      "activations/layer15_attention_weight_min": -30.21540641784668,
      "activations/layer16_attention_weight_max": 31.195520401000977,
      "activations/layer16_attention_weight_min": -27.477630615234375,
      "activations/layer17_attention_weight_max": 58.82662582397461,
      "activations/layer17_attention_weight_min": -44.65054702758789,
      "activations/layer18_attention_weight_max": 46.60541915893555,
      "activations/layer18_attention_weight_min": -37.0501594543457,
      "activations/layer19_attention_weight_max": 24.044546127319336,
      "activations/layer19_attention_weight_min": -19.504758834838867,
      "activations/layer1_attention_weight_max": 17.010744094848633,
      "activations/layer1_attention_weight_min": -14.644336700439453,
      "activations/layer20_attention_weight_max": 21.500267028808594,
      "activations/layer20_attention_weight_min": -24.043729782104492,
      "activations/layer21_attention_weight_max": 35.44716262817383,
      "activations/layer21_attention_weight_min": -22.521650314331055,
      "activations/layer22_attention_weight_max": 33.0373420715332,
      "activations/layer22_attention_weight_min": -24.822418212890625,
      "activations/layer23_attention_weight_max": 30.77149200439453,
      "activations/layer23_attention_weight_min": -22.406780242919922,
      "activations/layer2_attention_weight_max": 30.17599105834961,
      "activations/layer2_attention_weight_min": -28.468013763427734,
      "activations/layer3_attention_weight_max": 83.24247741699219,
      "activations/layer3_attention_weight_min": -82.85449981689453,
      "activations/layer4_attention_weight_max": 102.24005126953125,
      "activations/layer4_attention_weight_min": -93.4756088256836,
      "activations/layer5_attention_weight_max": 96.55422973632812,
      "activations/layer5_attention_weight_min": -84.99551391601562,
      "activations/layer6_attention_weight_max": 58.062721252441406,
      "activations/layer6_attention_weight_min": -54.06550979614258,
      "activations/layer7_attention_weight_max": 76.28257751464844,
      "activations/layer7_attention_weight_min": -74.50712585449219,
      "activations/layer8_attention_weight_max": 44.313140869140625,
      "activations/layer8_attention_weight_min": -45.034793853759766,
      "activations/layer9_attention_weight_max": 53.98853302001953,
      "activations/layer9_attention_weight_min": -48.16105270385742,
      "epoch": 7.25,
      "learning_rate": 0.00010426780303030302,
      "loss": 2.8283,
      "step": 124850
    },
    {
      "activations/layer0_attention_weight_max": 15.516073226928711,
      "activations/layer0_attention_weight_min": -14.82764720916748,
      "activations/layer10_attention_weight_max": 37.683902740478516,
      "activations/layer10_attention_weight_min": -34.34962463378906,
      "activations/layer11_attention_weight_max": 29.571197509765625,
      "activations/layer11_attention_weight_min": -29.766395568847656,
      "activations/layer12_attention_weight_max": 26.749494552612305,
      "activations/layer12_attention_weight_min": -24.975147247314453,
      "activations/layer13_attention_weight_max": 48.43092346191406,
      "activations/layer13_attention_weight_min": -31.754125595092773,
      "activations/layer14_attention_weight_max": 46.22312545776367,
      "activations/layer14_attention_weight_min": -30.203519821166992,
      "activations/layer15_attention_weight_max": 36.80564498901367,
      "activations/layer15_attention_weight_min": -28.963674545288086,
      "activations/layer16_attention_weight_max": 32.41439437866211,
      "activations/layer16_attention_weight_min": -28.208744049072266,
      "activations/layer17_attention_weight_max": 55.436859130859375,
      "activations/layer17_attention_weight_min": -46.368839263916016,
      "activations/layer18_attention_weight_max": 47.9305419921875,
      "activations/layer18_attention_weight_min": -35.50767135620117,
      "activations/layer19_attention_weight_max": 22.830713272094727,
      "activations/layer19_attention_weight_min": -22.762672424316406,
      "activations/layer1_attention_weight_max": 17.448711395263672,
      "activations/layer1_attention_weight_min": -14.6283540725708,
      "activations/layer20_attention_weight_max": 23.27464485168457,
      "activations/layer20_attention_weight_min": -21.325891494750977,
      "activations/layer21_attention_weight_max": 39.654293060302734,
      "activations/layer21_attention_weight_min": -24.396169662475586,
      "activations/layer22_attention_weight_max": 32.65225601196289,
      "activations/layer22_attention_weight_min": -25.019975662231445,
      "activations/layer23_attention_weight_max": 33.40636444091797,
      "activations/layer23_attention_weight_min": -21.632144927978516,
      "activations/layer2_attention_weight_max": 31.704978942871094,
      "activations/layer2_attention_weight_min": -28.76702117919922,
      "activations/layer3_attention_weight_max": 88.76264190673828,
      "activations/layer3_attention_weight_min": -84.98530578613281,
      "activations/layer4_attention_weight_max": 101.54949188232422,
      "activations/layer4_attention_weight_min": -89.89962005615234,
      "activations/layer5_attention_weight_max": 90.01699829101562,
      "activations/layer5_attention_weight_min": -81.34821319580078,
      "activations/layer6_attention_weight_max": 53.94620132446289,
      "activations/layer6_attention_weight_min": -50.3886604309082,
      "activations/layer7_attention_weight_max": 77.146728515625,
      "activations/layer7_attention_weight_min": -72.222412109375,
      "activations/layer8_attention_weight_max": 43.494873046875,
      "activations/layer8_attention_weight_min": -43.38170623779297,
      "activations/layer9_attention_weight_max": 51.958187103271484,
      "activations/layer9_attention_weight_min": -49.71610641479492,
      "epoch": 7.26,
      "learning_rate": 0.00010424886363636363,
      "loss": 2.8232,
      "step": 124900
    },
    {
      "activations/layer0_attention_weight_max": 15.232583999633789,
      "activations/layer0_attention_weight_min": -14.439318656921387,
      "activations/layer10_attention_weight_max": 39.06822967529297,
      "activations/layer10_attention_weight_min": -35.3779411315918,
      "activations/layer11_attention_weight_max": 32.761356353759766,
      "activations/layer11_attention_weight_min": -31.3139705657959,
      "activations/layer12_attention_weight_max": 22.67871856689453,
      "activations/layer12_attention_weight_min": -25.959718704223633,
      "activations/layer13_attention_weight_max": 53.35531234741211,
      "activations/layer13_attention_weight_min": -38.0803108215332,
      "activations/layer14_attention_weight_max": 45.880035400390625,
      "activations/layer14_attention_weight_min": -31.978837966918945,
      "activations/layer15_attention_weight_max": 40.85519790649414,
      "activations/layer15_attention_weight_min": -31.428918838500977,
      "activations/layer16_attention_weight_max": 35.56167984008789,
      "activations/layer16_attention_weight_min": -28.30605697631836,
      "activations/layer17_attention_weight_max": 62.92426300048828,
      "activations/layer17_attention_weight_min": -46.84822463989258,
      "activations/layer18_attention_weight_max": 56.02257537841797,
      "activations/layer18_attention_weight_min": -39.254642486572266,
      "activations/layer19_attention_weight_max": 22.54987144470215,
      "activations/layer19_attention_weight_min": -22.412675857543945,
      "activations/layer1_attention_weight_max": 18.37772560119629,
      "activations/layer1_attention_weight_min": -15.375279426574707,
      "activations/layer20_attention_weight_max": 22.918628692626953,
      "activations/layer20_attention_weight_min": -23.41478729248047,
      "activations/layer21_attention_weight_max": 45.30290603637695,
      "activations/layer21_attention_weight_min": -23.316781997680664,
      "activations/layer22_attention_weight_max": 33.9225959777832,
      "activations/layer22_attention_weight_min": -25.366323471069336,
      "activations/layer23_attention_weight_max": 32.73583221435547,
      "activations/layer23_attention_weight_min": -24.076904296875,
      "activations/layer2_attention_weight_max": 30.12869644165039,
      "activations/layer2_attention_weight_min": -29.032249450683594,
      "activations/layer3_attention_weight_max": 83.02050018310547,
      "activations/layer3_attention_weight_min": -87.30403900146484,
      "activations/layer4_attention_weight_max": 97.23490905761719,
      "activations/layer4_attention_weight_min": -89.06185150146484,
      "activations/layer5_attention_weight_max": 88.11851501464844,
      "activations/layer5_attention_weight_min": -85.5595932006836,
      "activations/layer6_attention_weight_max": 55.34546661376953,
      "activations/layer6_attention_weight_min": -53.37006378173828,
      "activations/layer7_attention_weight_max": 73.43120574951172,
      "activations/layer7_attention_weight_min": -69.54718780517578,
      "activations/layer8_attention_weight_max": 44.52747344970703,
      "activations/layer8_attention_weight_min": -45.02791213989258,
      "activations/layer9_attention_weight_max": 52.08225631713867,
      "activations/layer9_attention_weight_min": -49.730403900146484,
      "epoch": 7.26,
      "learning_rate": 0.00010422992424242423,
      "loss": 2.805,
      "step": 124950
    },
    {
      "activations/layer0_attention_weight_max": 15.46579647064209,
      "activations/layer0_attention_weight_min": -15.054244995117188,
      "activations/layer10_attention_weight_max": 36.051727294921875,
      "activations/layer10_attention_weight_min": -34.11912155151367,
      "activations/layer11_attention_weight_max": 30.943561553955078,
      "activations/layer11_attention_weight_min": -29.97003173828125,
      "activations/layer12_attention_weight_max": 17.84551429748535,
      "activations/layer12_attention_weight_min": -31.2281551361084,
      "activations/layer13_attention_weight_max": 36.69153594970703,
      "activations/layer13_attention_weight_min": -30.00936508178711,
      "activations/layer14_attention_weight_max": 35.110565185546875,
      "activations/layer14_attention_weight_min": -29.707256317138672,
      "activations/layer15_attention_weight_max": 34.134071350097656,
      "activations/layer15_attention_weight_min": -29.418210983276367,
      "activations/layer16_attention_weight_max": 29.183528900146484,
      "activations/layer16_attention_weight_min": -27.167430877685547,
      "activations/layer17_attention_weight_max": 52.941978454589844,
      "activations/layer17_attention_weight_min": -39.874671936035156,
      "activations/layer18_attention_weight_max": 42.7386360168457,
      "activations/layer18_attention_weight_min": -32.525390625,
      "activations/layer19_attention_weight_max": 21.47220230102539,
      "activations/layer19_attention_weight_min": -19.37691307067871,
      "activations/layer1_attention_weight_max": 17.27782440185547,
      "activations/layer1_attention_weight_min": -15.11296272277832,
      "activations/layer20_attention_weight_max": 20.055606842041016,
      "activations/layer20_attention_weight_min": -23.620819091796875,
      "activations/layer21_attention_weight_max": 33.08843994140625,
      "activations/layer21_attention_weight_min": -21.067440032958984,
      "activations/layer22_attention_weight_max": 29.43397331237793,
      "activations/layer22_attention_weight_min": -23.30152702331543,
      "activations/layer23_attention_weight_max": 29.226903915405273,
      "activations/layer23_attention_weight_min": -23.28814697265625,
      "activations/layer2_attention_weight_max": 29.7613582611084,
      "activations/layer2_attention_weight_min": -27.97863006591797,
      "activations/layer3_attention_weight_max": 82.59461975097656,
      "activations/layer3_attention_weight_min": -85.04817199707031,
      "activations/layer4_attention_weight_max": 95.95039367675781,
      "activations/layer4_attention_weight_min": -92.52034759521484,
      "activations/layer5_attention_weight_max": 88.61929321289062,
      "activations/layer5_attention_weight_min": -81.8914566040039,
      "activations/layer6_attention_weight_max": 53.458072662353516,
      "activations/layer6_attention_weight_min": -52.802268981933594,
      "activations/layer7_attention_weight_max": 74.17072296142578,
      "activations/layer7_attention_weight_min": -70.8724365234375,
      "activations/layer8_attention_weight_max": 42.02974319458008,
      "activations/layer8_attention_weight_min": -42.42049026489258,
      "activations/layer9_attention_weight_max": 47.079532623291016,
      "activations/layer9_attention_weight_min": -49.467281341552734,
      "epoch": 7.26,
      "learning_rate": 0.00010421098484848483,
      "loss": 2.828,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.536,
      "eval_samples_per_second": 503.046,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.536,
      "eval_openwebtext_samples_per_second": 503.046,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_wikitext_loss": 3.00390625,
      "eval_wikitext_ppl": 20.16414949200632,
      "eval_wikitext_runtime": 2.0153,
      "eval_wikitext_samples_per_second": 226.272,
      "step": 125000
    },
    {
      "epoch": 7.26,
      "eval_lambada_loss": 2.71484375,
      "eval_lambada_ppl": 15.10225016094451,
      "eval_lambada_runtime": 9.5899,
      "eval_lambada_samples_per_second": 507.723,
      "step": 125000
    },
    {
      "activations/layer0_attention_weight_max": 15.113007545471191,
      "activations/layer0_attention_weight_min": -14.716175079345703,
      "activations/layer10_attention_weight_max": 38.43388366699219,
      "activations/layer10_attention_weight_min": -35.88114929199219,
      "activations/layer11_attention_weight_max": 33.04194641113281,
      "activations/layer11_attention_weight_min": -30.465717315673828,
      "activations/layer12_attention_weight_max": 29.340213775634766,
      "activations/layer12_attention_weight_min": -27.131261825561523,
      "activations/layer13_attention_weight_max": 52.54310607910156,
      "activations/layer13_attention_weight_min": -35.68854522705078,
      "activations/layer14_attention_weight_max": 43.71460723876953,
      "activations/layer14_attention_weight_min": -31.2856388092041,
      "activations/layer15_attention_weight_max": 42.543514251708984,
      "activations/layer15_attention_weight_min": -28.76788330078125,
      "activations/layer16_attention_weight_max": 32.81763458251953,
      "activations/layer16_attention_weight_min": -26.688127517700195,
      "activations/layer17_attention_weight_max": 53.84437561035156,
      "activations/layer17_attention_weight_min": -41.382747650146484,
      "activations/layer18_attention_weight_max": 45.82484817504883,
      "activations/layer18_attention_weight_min": -34.76152038574219,
      "activations/layer19_attention_weight_max": 23.344484329223633,
      "activations/layer19_attention_weight_min": -19.706804275512695,
      "activations/layer1_attention_weight_max": 17.131738662719727,
      "activations/layer1_attention_weight_min": -15.37097454071045,
      "activations/layer20_attention_weight_max": 22.880163192749023,
      "activations/layer20_attention_weight_min": -20.69243621826172,
      "activations/layer21_attention_weight_max": 34.046451568603516,
      "activations/layer21_attention_weight_min": -26.144256591796875,
      "activations/layer22_attention_weight_max": 32.24803924560547,
      "activations/layer22_attention_weight_min": -26.285846710205078,
      "activations/layer23_attention_weight_max": 31.706592559814453,
      "activations/layer23_attention_weight_min": -22.145132064819336,
      "activations/layer2_attention_weight_max": 30.733768463134766,
      "activations/layer2_attention_weight_min": -28.32879638671875,
      "activations/layer3_attention_weight_max": 81.69595336914062,
      "activations/layer3_attention_weight_min": -83.9951400756836,
      "activations/layer4_attention_weight_max": 96.4860610961914,
      "activations/layer4_attention_weight_min": -92.58101654052734,
      "activations/layer5_attention_weight_max": 88.62068176269531,
      "activations/layer5_attention_weight_min": -86.68197631835938,
      "activations/layer6_attention_weight_max": 54.622108459472656,
      "activations/layer6_attention_weight_min": -55.19806671142578,
      "activations/layer7_attention_weight_max": 77.02747344970703,
      "activations/layer7_attention_weight_min": -74.0733871459961,
      "activations/layer8_attention_weight_max": 49.76922607421875,
      "activations/layer8_attention_weight_min": -51.2927131652832,
      "activations/layer9_attention_weight_max": 50.78293991088867,
      "activations/layer9_attention_weight_min": -48.384864807128906,
      "epoch": 7.27,
      "learning_rate": 0.00010419204545454545,
      "loss": 2.8095,
      "step": 125050
    },
    {
      "activations/layer0_attention_weight_max": 14.350049018859863,
      "activations/layer0_attention_weight_min": -15.276457786560059,
      "activations/layer10_attention_weight_max": 38.16325759887695,
      "activations/layer10_attention_weight_min": -37.52557373046875,
      "activations/layer11_attention_weight_max": 32.860538482666016,
      "activations/layer11_attention_weight_min": -30.64745330810547,
      "activations/layer12_attention_weight_max": 19.502323150634766,
      "activations/layer12_attention_weight_min": -25.32689666748047,
      "activations/layer13_attention_weight_max": 44.91867446899414,
      "activations/layer13_attention_weight_min": -32.766761779785156,
      "activations/layer14_attention_weight_max": 37.65375900268555,
      "activations/layer14_attention_weight_min": -32.145076751708984,
      "activations/layer15_attention_weight_max": 39.2315788269043,
      "activations/layer15_attention_weight_min": -30.761816024780273,
      "activations/layer16_attention_weight_max": 34.237876892089844,
      "activations/layer16_attention_weight_min": -28.04561424255371,
      "activations/layer17_attention_weight_max": 57.45062255859375,
      "activations/layer17_attention_weight_min": -42.6361083984375,
      "activations/layer18_attention_weight_max": 49.99118423461914,
      "activations/layer18_attention_weight_min": -36.64986038208008,
      "activations/layer19_attention_weight_max": 26.912418365478516,
      "activations/layer19_attention_weight_min": -20.995450973510742,
      "activations/layer1_attention_weight_max": 17.26362419128418,
      "activations/layer1_attention_weight_min": -16.13652801513672,
      "activations/layer20_attention_weight_max": 23.53689193725586,
      "activations/layer20_attention_weight_min": -21.948577880859375,
      "activations/layer21_attention_weight_max": 38.06991958618164,
      "activations/layer21_attention_weight_min": -24.39657974243164,
      "activations/layer22_attention_weight_max": 33.235939025878906,
      "activations/layer22_attention_weight_min": -24.289291381835938,
      "activations/layer23_attention_weight_max": 32.8828125,
      "activations/layer23_attention_weight_min": -21.308837890625,
      "activations/layer2_attention_weight_max": 30.5004940032959,
      "activations/layer2_attention_weight_min": -29.246992111206055,
      "activations/layer3_attention_weight_max": 87.4698486328125,
      "activations/layer3_attention_weight_min": -93.41999053955078,
      "activations/layer4_attention_weight_max": 100.8696060180664,
      "activations/layer4_attention_weight_min": -93.91184997558594,
      "activations/layer5_attention_weight_max": 90.59797668457031,
      "activations/layer5_attention_weight_min": -83.48580932617188,
      "activations/layer6_attention_weight_max": 57.08385467529297,
      "activations/layer6_attention_weight_min": -56.0601921081543,
      "activations/layer7_attention_weight_max": 74.83541107177734,
      "activations/layer7_attention_weight_min": -75.3212890625,
      "activations/layer8_attention_weight_max": 48.22662353515625,
      "activations/layer8_attention_weight_min": -50.665836334228516,
      "activations/layer9_attention_weight_max": 59.56005859375,
      "activations/layer9_attention_weight_min": -49.8060417175293,
      "epoch": 7.27,
      "learning_rate": 0.00010417310606060606,
      "loss": 2.8234,
      "step": 125100
    },
    {
      "activations/layer0_attention_weight_max": 14.812591552734375,
      "activations/layer0_attention_weight_min": -14.595519065856934,
      "activations/layer10_attention_weight_max": 39.69168472290039,
      "activations/layer10_attention_weight_min": -36.37075424194336,
      "activations/layer11_attention_weight_max": 33.0272102355957,
      "activations/layer11_attention_weight_min": -31.810583114624023,
      "activations/layer12_attention_weight_max": 24.152433395385742,
      "activations/layer12_attention_weight_min": -23.65647315979004,
      "activations/layer13_attention_weight_max": 58.79713439941406,
      "activations/layer13_attention_weight_min": -33.18075180053711,
      "activations/layer14_attention_weight_max": 50.565834045410156,
      "activations/layer14_attention_weight_min": -31.985925674438477,
      "activations/layer15_attention_weight_max": 43.99480056762695,
      "activations/layer15_attention_weight_min": -30.880613327026367,
      "activations/layer16_attention_weight_max": 36.94333267211914,
      "activations/layer16_attention_weight_min": -27.747543334960938,
      "activations/layer17_attention_weight_max": 67.49981689453125,
      "activations/layer17_attention_weight_min": -46.367000579833984,
      "activations/layer18_attention_weight_max": 59.96101379394531,
      "activations/layer18_attention_weight_min": -40.88371276855469,
      "activations/layer19_attention_weight_max": 27.572315216064453,
      "activations/layer19_attention_weight_min": -22.717500686645508,
      "activations/layer1_attention_weight_max": 16.63054847717285,
      "activations/layer1_attention_weight_min": -16.388010025024414,
      "activations/layer20_attention_weight_max": 25.162242889404297,
      "activations/layer20_attention_weight_min": -22.99358558654785,
      "activations/layer21_attention_weight_max": 54.054046630859375,
      "activations/layer21_attention_weight_min": -26.356353759765625,
      "activations/layer22_attention_weight_max": 41.3240966796875,
      "activations/layer22_attention_weight_min": -27.29486083984375,
      "activations/layer23_attention_weight_max": 39.39850616455078,
      "activations/layer23_attention_weight_min": -21.81264305114746,
      "activations/layer2_attention_weight_max": 30.7135066986084,
      "activations/layer2_attention_weight_min": -28.152585983276367,
      "activations/layer3_attention_weight_max": 81.33256530761719,
      "activations/layer3_attention_weight_min": -81.9703140258789,
      "activations/layer4_attention_weight_max": 95.97037506103516,
      "activations/layer4_attention_weight_min": -94.15577697753906,
      "activations/layer5_attention_weight_max": 87.94905090332031,
      "activations/layer5_attention_weight_min": -80.37464904785156,
      "activations/layer6_attention_weight_max": 53.43333053588867,
      "activations/layer6_attention_weight_min": -49.780277252197266,
      "activations/layer7_attention_weight_max": 72.99999237060547,
      "activations/layer7_attention_weight_min": -76.17607879638672,
      "activations/layer8_attention_weight_max": 48.429466247558594,
      "activations/layer8_attention_weight_min": -43.924774169921875,
      "activations/layer9_attention_weight_max": 51.86491012573242,
      "activations/layer9_attention_weight_min": -49.68935012817383,
      "epoch": 7.27,
      "learning_rate": 0.00010415416666666665,
      "loss": 2.8187,
      "step": 125150
    },
    {
      "activations/layer0_attention_weight_max": 15.881379127502441,
      "activations/layer0_attention_weight_min": -14.574545860290527,
      "activations/layer10_attention_weight_max": 34.62162780761719,
      "activations/layer10_attention_weight_min": -33.20033645629883,
      "activations/layer11_attention_weight_max": 29.42422103881836,
      "activations/layer11_attention_weight_min": -30.969669342041016,
      "activations/layer12_attention_weight_max": 21.848812103271484,
      "activations/layer12_attention_weight_min": -27.99361801147461,
      "activations/layer13_attention_weight_max": 50.41152572631836,
      "activations/layer13_attention_weight_min": -33.28541564941406,
      "activations/layer14_attention_weight_max": 37.630767822265625,
      "activations/layer14_attention_weight_min": -30.63865852355957,
      "activations/layer15_attention_weight_max": 35.74703598022461,
      "activations/layer15_attention_weight_min": -30.134632110595703,
      "activations/layer16_attention_weight_max": 33.78452682495117,
      "activations/layer16_attention_weight_min": -27.264522552490234,
      "activations/layer17_attention_weight_max": 55.015316009521484,
      "activations/layer17_attention_weight_min": -41.22753143310547,
      "activations/layer18_attention_weight_max": 47.11491775512695,
      "activations/layer18_attention_weight_min": -35.541656494140625,
      "activations/layer19_attention_weight_max": 21.514766693115234,
      "activations/layer19_attention_weight_min": -21.441614151000977,
      "activations/layer1_attention_weight_max": 17.893404006958008,
      "activations/layer1_attention_weight_min": -17.121431350708008,
      "activations/layer20_attention_weight_max": 23.25893783569336,
      "activations/layer20_attention_weight_min": -20.9725284576416,
      "activations/layer21_attention_weight_max": 38.79332733154297,
      "activations/layer21_attention_weight_min": -23.728546142578125,
      "activations/layer22_attention_weight_max": 30.63971519470215,
      "activations/layer22_attention_weight_min": -26.35985565185547,
      "activations/layer23_attention_weight_max": 31.902488708496094,
      "activations/layer23_attention_weight_min": -22.68158721923828,
      "activations/layer2_attention_weight_max": 31.71937370300293,
      "activations/layer2_attention_weight_min": -30.654844284057617,
      "activations/layer3_attention_weight_max": 85.00487518310547,
      "activations/layer3_attention_weight_min": -87.03682708740234,
      "activations/layer4_attention_weight_max": 99.64891815185547,
      "activations/layer4_attention_weight_min": -91.83830261230469,
      "activations/layer5_attention_weight_max": 88.67364501953125,
      "activations/layer5_attention_weight_min": -81.76648712158203,
      "activations/layer6_attention_weight_max": 56.05548858642578,
      "activations/layer6_attention_weight_min": -53.47501754760742,
      "activations/layer7_attention_weight_max": 73.61266326904297,
      "activations/layer7_attention_weight_min": -72.4324722290039,
      "activations/layer8_attention_weight_max": 42.62206268310547,
      "activations/layer8_attention_weight_min": -43.119747161865234,
      "activations/layer9_attention_weight_max": 50.144229888916016,
      "activations/layer9_attention_weight_min": -50.420772552490234,
      "epoch": 7.27,
      "learning_rate": 0.00010413522727272727,
      "loss": 2.8094,
      "step": 125200
    },
    {
      "activations/layer0_attention_weight_max": 14.617165565490723,
      "activations/layer0_attention_weight_min": -14.62330150604248,
      "activations/layer10_attention_weight_max": 40.558021545410156,
      "activations/layer10_attention_weight_min": -37.05254364013672,
      "activations/layer11_attention_weight_max": 33.027137756347656,
      "activations/layer11_attention_weight_min": -31.706401824951172,
      "activations/layer12_attention_weight_max": 17.907995223999023,
      "activations/layer12_attention_weight_min": -25.39625358581543,
      "activations/layer13_attention_weight_max": 49.81259536743164,
      "activations/layer13_attention_weight_min": -39.26317596435547,
      "activations/layer14_attention_weight_max": 43.70956039428711,
      "activations/layer14_attention_weight_min": -31.10476303100586,
      "activations/layer15_attention_weight_max": 35.29343795776367,
      "activations/layer15_attention_weight_min": -29.90034294128418,
      "activations/layer16_attention_weight_max": 32.22786331176758,
      "activations/layer16_attention_weight_min": -27.0611629486084,
      "activations/layer17_attention_weight_max": 54.20543670654297,
      "activations/layer17_attention_weight_min": -40.53772735595703,
      "activations/layer18_attention_weight_max": 46.147987365722656,
      "activations/layer18_attention_weight_min": -35.38679122924805,
      "activations/layer19_attention_weight_max": 22.381690979003906,
      "activations/layer19_attention_weight_min": -20.515905380249023,
      "activations/layer1_attention_weight_max": 16.792062759399414,
      "activations/layer1_attention_weight_min": -14.988253593444824,
      "activations/layer20_attention_weight_max": 22.274694442749023,
      "activations/layer20_attention_weight_min": -23.0212459564209,
      "activations/layer21_attention_weight_max": 39.530006408691406,
      "activations/layer21_attention_weight_min": -23.151107788085938,
      "activations/layer22_attention_weight_max": 31.27863121032715,
      "activations/layer22_attention_weight_min": -25.94375228881836,
      "activations/layer23_attention_weight_max": 34.318424224853516,
      "activations/layer23_attention_weight_min": -24.224262237548828,
      "activations/layer2_attention_weight_max": 31.026878356933594,
      "activations/layer2_attention_weight_min": -29.186002731323242,
      "activations/layer3_attention_weight_max": 85.27928924560547,
      "activations/layer3_attention_weight_min": -85.87115478515625,
      "activations/layer4_attention_weight_max": 95.68084716796875,
      "activations/layer4_attention_weight_min": -90.99848937988281,
      "activations/layer5_attention_weight_max": 88.81492614746094,
      "activations/layer5_attention_weight_min": -84.76754760742188,
      "activations/layer6_attention_weight_max": 57.78133010864258,
      "activations/layer6_attention_weight_min": -58.26359558105469,
      "activations/layer7_attention_weight_max": 80.43162536621094,
      "activations/layer7_attention_weight_min": -76.90299224853516,
      "activations/layer8_attention_weight_max": 47.83561706542969,
      "activations/layer8_attention_weight_min": -48.21162414550781,
      "activations/layer9_attention_weight_max": 53.121395111083984,
      "activations/layer9_attention_weight_min": -50.528419494628906,
      "epoch": 7.28,
      "learning_rate": 0.00010411628787878787,
      "loss": 2.8149,
      "step": 125250
    },
    {
      "activations/layer0_attention_weight_max": 15.290775299072266,
      "activations/layer0_attention_weight_min": -14.910608291625977,
      "activations/layer10_attention_weight_max": 37.36960220336914,
      "activations/layer10_attention_weight_min": -34.694801330566406,
      "activations/layer11_attention_weight_max": 29.567401885986328,
      "activations/layer11_attention_weight_min": -29.446027755737305,
      "activations/layer12_attention_weight_max": 21.124408721923828,
      "activations/layer12_attention_weight_min": -28.200170516967773,
      "activations/layer13_attention_weight_max": 44.055091857910156,
      "activations/layer13_attention_weight_min": -34.60472869873047,
      "activations/layer14_attention_weight_max": 45.99341583251953,
      "activations/layer14_attention_weight_min": -38.237545013427734,
      "activations/layer15_attention_weight_max": 34.79420471191406,
      "activations/layer15_attention_weight_min": -29.742502212524414,
      "activations/layer16_attention_weight_max": 34.93102264404297,
      "activations/layer16_attention_weight_min": -27.160377502441406,
      "activations/layer17_attention_weight_max": 52.66011428833008,
      "activations/layer17_attention_weight_min": -43.84343338012695,
      "activations/layer18_attention_weight_max": 45.60116958618164,
      "activations/layer18_attention_weight_min": -36.16985321044922,
      "activations/layer19_attention_weight_max": 22.417524337768555,
      "activations/layer19_attention_weight_min": -21.571794509887695,
      "activations/layer1_attention_weight_max": 17.13189125061035,
      "activations/layer1_attention_weight_min": -16.113208770751953,
      "activations/layer20_attention_weight_max": 23.543102264404297,
      "activations/layer20_attention_weight_min": -23.710063934326172,
      "activations/layer21_attention_weight_max": 40.83904266357422,
      "activations/layer21_attention_weight_min": -25.077672958374023,
      "activations/layer22_attention_weight_max": 36.41612243652344,
      "activations/layer22_attention_weight_min": -25.949615478515625,
      "activations/layer23_attention_weight_max": 39.65229034423828,
      "activations/layer23_attention_weight_min": -26.10743522644043,
      "activations/layer2_attention_weight_max": 30.61293601989746,
      "activations/layer2_attention_weight_min": -28.37261199951172,
      "activations/layer3_attention_weight_max": 85.5723876953125,
      "activations/layer3_attention_weight_min": -89.40961456298828,
      "activations/layer4_attention_weight_max": 95.95500183105469,
      "activations/layer4_attention_weight_min": -93.04381561279297,
      "activations/layer5_attention_weight_max": 86.57510375976562,
      "activations/layer5_attention_weight_min": -85.33688354492188,
      "activations/layer6_attention_weight_max": 56.748600006103516,
      "activations/layer6_attention_weight_min": -52.056941986083984,
      "activations/layer7_attention_weight_max": 73.27262115478516,
      "activations/layer7_attention_weight_min": -73.23300170898438,
      "activations/layer8_attention_weight_max": 45.03028106689453,
      "activations/layer8_attention_weight_min": -46.69267654418945,
      "activations/layer9_attention_weight_max": 50.87552261352539,
      "activations/layer9_attention_weight_min": -50.00741958618164,
      "epoch": 7.28,
      "learning_rate": 0.00010409734848484847,
      "loss": 2.8244,
      "step": 125300
    },
    {
      "activations/layer0_attention_weight_max": 15.302837371826172,
      "activations/layer0_attention_weight_min": -15.421540260314941,
      "activations/layer10_attention_weight_max": 41.3928108215332,
      "activations/layer10_attention_weight_min": -34.95442581176758,
      "activations/layer11_attention_weight_max": 33.744110107421875,
      "activations/layer11_attention_weight_min": -31.153318405151367,
      "activations/layer12_attention_weight_max": 19.80647087097168,
      "activations/layer12_attention_weight_min": -26.009681701660156,
      "activations/layer13_attention_weight_max": 40.698055267333984,
      "activations/layer13_attention_weight_min": -31.420066833496094,
      "activations/layer14_attention_weight_max": 39.2994270324707,
      "activations/layer14_attention_weight_min": -30.393531799316406,
      "activations/layer15_attention_weight_max": 38.02964782714844,
      "activations/layer15_attention_weight_min": -28.820207595825195,
      "activations/layer16_attention_weight_max": 34.30356216430664,
      "activations/layer16_attention_weight_min": -26.817920684814453,
      "activations/layer17_attention_weight_max": 56.79790496826172,
      "activations/layer17_attention_weight_min": -42.853878021240234,
      "activations/layer18_attention_weight_max": 48.80641174316406,
      "activations/layer18_attention_weight_min": -35.66128158569336,
      "activations/layer19_attention_weight_max": 20.400686264038086,
      "activations/layer19_attention_weight_min": -21.425268173217773,
      "activations/layer1_attention_weight_max": 17.499032974243164,
      "activations/layer1_attention_weight_min": -14.64273452758789,
      "activations/layer20_attention_weight_max": 22.426633834838867,
      "activations/layer20_attention_weight_min": -22.381019592285156,
      "activations/layer21_attention_weight_max": 44.25189971923828,
      "activations/layer21_attention_weight_min": -25.479843139648438,
      "activations/layer22_attention_weight_max": 33.87173080444336,
      "activations/layer22_attention_weight_min": -25.495250701904297,
      "activations/layer23_attention_weight_max": 32.90428161621094,
      "activations/layer23_attention_weight_min": -19.91752052307129,
      "activations/layer2_attention_weight_max": 32.247703552246094,
      "activations/layer2_attention_weight_min": -29.93842315673828,
      "activations/layer3_attention_weight_max": 87.3580551147461,
      "activations/layer3_attention_weight_min": -88.1329345703125,
      "activations/layer4_attention_weight_max": 101.13530731201172,
      "activations/layer4_attention_weight_min": -94.4321517944336,
      "activations/layer5_attention_weight_max": 93.47138977050781,
      "activations/layer5_attention_weight_min": -87.10002899169922,
      "activations/layer6_attention_weight_max": 56.503990173339844,
      "activations/layer6_attention_weight_min": -53.13233184814453,
      "activations/layer7_attention_weight_max": 75.50543212890625,
      "activations/layer7_attention_weight_min": -74.46147155761719,
      "activations/layer8_attention_weight_max": 46.79192352294922,
      "activations/layer8_attention_weight_min": -44.75930404663086,
      "activations/layer9_attention_weight_max": 51.19900131225586,
      "activations/layer9_attention_weight_min": -48.34813690185547,
      "epoch": 7.28,
      "learning_rate": 0.00010407840909090909,
      "loss": 2.8234,
      "step": 125350
    },
    {
      "activations/layer0_attention_weight_max": 14.869476318359375,
      "activations/layer0_attention_weight_min": -14.730401039123535,
      "activations/layer10_attention_weight_max": 41.03379440307617,
      "activations/layer10_attention_weight_min": -38.512611389160156,
      "activations/layer11_attention_weight_max": 31.98780059814453,
      "activations/layer11_attention_weight_min": -30.91779136657715,
      "activations/layer12_attention_weight_max": 20.862186431884766,
      "activations/layer12_attention_weight_min": -25.22878646850586,
      "activations/layer13_attention_weight_max": 46.07851028442383,
      "activations/layer13_attention_weight_min": -35.444679260253906,
      "activations/layer14_attention_weight_max": 60.40651321411133,
      "activations/layer14_attention_weight_min": -40.980018615722656,
      "activations/layer15_attention_weight_max": 45.719032287597656,
      "activations/layer15_attention_weight_min": -30.51126480102539,
      "activations/layer16_attention_weight_max": 36.19103240966797,
      "activations/layer16_attention_weight_min": -28.458457946777344,
      "activations/layer17_attention_weight_max": 53.9221076965332,
      "activations/layer17_attention_weight_min": -42.39923858642578,
      "activations/layer18_attention_weight_max": 49.24065399169922,
      "activations/layer18_attention_weight_min": -37.03903579711914,
      "activations/layer19_attention_weight_max": 24.31182098388672,
      "activations/layer19_attention_weight_min": -23.19104766845703,
      "activations/layer1_attention_weight_max": 16.653759002685547,
      "activations/layer1_attention_weight_min": -15.283671379089355,
      "activations/layer20_attention_weight_max": 23.215177536010742,
      "activations/layer20_attention_weight_min": -22.537113189697266,
      "activations/layer21_attention_weight_max": 40.83283615112305,
      "activations/layer21_attention_weight_min": -25.339136123657227,
      "activations/layer22_attention_weight_max": 35.91963195800781,
      "activations/layer22_attention_weight_min": -27.74442481994629,
      "activations/layer23_attention_weight_max": 36.11295700073242,
      "activations/layer23_attention_weight_min": -24.696632385253906,
      "activations/layer2_attention_weight_max": 32.82302474975586,
      "activations/layer2_attention_weight_min": -29.814373016357422,
      "activations/layer3_attention_weight_max": 84.92085266113281,
      "activations/layer3_attention_weight_min": -85.05046081542969,
      "activations/layer4_attention_weight_max": 97.11843872070312,
      "activations/layer4_attention_weight_min": -97.14610290527344,
      "activations/layer5_attention_weight_max": 91.7705307006836,
      "activations/layer5_attention_weight_min": -84.7691650390625,
      "activations/layer6_attention_weight_max": 58.639381408691406,
      "activations/layer6_attention_weight_min": -54.54114532470703,
      "activations/layer7_attention_weight_max": 81.93524169921875,
      "activations/layer7_attention_weight_min": -74.6535873413086,
      "activations/layer8_attention_weight_max": 44.56126403808594,
      "activations/layer8_attention_weight_min": -48.30876541137695,
      "activations/layer9_attention_weight_max": 59.86664962768555,
      "activations/layer9_attention_weight_min": -54.62122344970703,
      "epoch": 7.29,
      "learning_rate": 0.00010405946969696969,
      "loss": 2.8179,
      "step": 125400
    },
    {
      "activations/layer0_attention_weight_max": 14.948524475097656,
      "activations/layer0_attention_weight_min": -14.984841346740723,
      "activations/layer10_attention_weight_max": 42.85820388793945,
      "activations/layer10_attention_weight_min": -41.329193115234375,
      "activations/layer11_attention_weight_max": 34.609798431396484,
      "activations/layer11_attention_weight_min": -31.931114196777344,
      "activations/layer12_attention_weight_max": 18.77184295654297,
      "activations/layer12_attention_weight_min": -29.20766830444336,
      "activations/layer13_attention_weight_max": 47.964412689208984,
      "activations/layer13_attention_weight_min": -36.43252182006836,
      "activations/layer14_attention_weight_max": 48.273799896240234,
      "activations/layer14_attention_weight_min": -33.50075149536133,
      "activations/layer15_attention_weight_max": 39.2703971862793,
      "activations/layer15_attention_weight_min": -29.67621612548828,
      "activations/layer16_attention_weight_max": 32.511878967285156,
      "activations/layer16_attention_weight_min": -27.124025344848633,
      "activations/layer17_attention_weight_max": 55.386837005615234,
      "activations/layer17_attention_weight_min": -43.874149322509766,
      "activations/layer18_attention_weight_max": 47.77145767211914,
      "activations/layer18_attention_weight_min": -39.66863250732422,
      "activations/layer19_attention_weight_max": 23.828399658203125,
      "activations/layer19_attention_weight_min": -23.01002311706543,
      "activations/layer1_attention_weight_max": 17.054533004760742,
      "activations/layer1_attention_weight_min": -15.783077239990234,
      "activations/layer20_attention_weight_max": 22.964479446411133,
      "activations/layer20_attention_weight_min": -22.313570022583008,
      "activations/layer21_attention_weight_max": 37.81913757324219,
      "activations/layer21_attention_weight_min": -24.985620498657227,
      "activations/layer22_attention_weight_max": 33.90980529785156,
      "activations/layer22_attention_weight_min": -26.21713638305664,
      "activations/layer23_attention_weight_max": 31.80710220336914,
      "activations/layer23_attention_weight_min": -24.517202377319336,
      "activations/layer2_attention_weight_max": 31.977195739746094,
      "activations/layer2_attention_weight_min": -30.158374786376953,
      "activations/layer3_attention_weight_max": 94.87445068359375,
      "activations/layer3_attention_weight_min": -94.78656005859375,
      "activations/layer4_attention_weight_max": 113.19129943847656,
      "activations/layer4_attention_weight_min": -103.74005126953125,
      "activations/layer5_attention_weight_max": 101.37669372558594,
      "activations/layer5_attention_weight_min": -89.468017578125,
      "activations/layer6_attention_weight_max": 59.6456298828125,
      "activations/layer6_attention_weight_min": -57.84824752807617,
      "activations/layer7_attention_weight_max": 78.79791259765625,
      "activations/layer7_attention_weight_min": -74.30851745605469,
      "activations/layer8_attention_weight_max": 50.33089065551758,
      "activations/layer8_attention_weight_min": -47.75404357910156,
      "activations/layer9_attention_weight_max": 58.17689895629883,
      "activations/layer9_attention_weight_min": -55.811309814453125,
      "epoch": 7.29,
      "learning_rate": 0.00010404053030303029,
      "loss": 2.8103,
      "step": 125450
    },
    {
      "activations/layer0_attention_weight_max": 15.334067344665527,
      "activations/layer0_attention_weight_min": -14.988676071166992,
      "activations/layer10_attention_weight_max": 40.455078125,
      "activations/layer10_attention_weight_min": -35.430885314941406,
      "activations/layer11_attention_weight_max": 34.05885314941406,
      "activations/layer11_attention_weight_min": -30.51027488708496,
      "activations/layer12_attention_weight_max": 23.956226348876953,
      "activations/layer12_attention_weight_min": -25.411542892456055,
      "activations/layer13_attention_weight_max": 46.75940704345703,
      "activations/layer13_attention_weight_min": -32.99131774902344,
      "activations/layer14_attention_weight_max": 39.90945053100586,
      "activations/layer14_attention_weight_min": -30.69450569152832,
      "activations/layer15_attention_weight_max": 37.398136138916016,
      "activations/layer15_attention_weight_min": -30.833688735961914,
      "activations/layer16_attention_weight_max": 34.152198791503906,
      "activations/layer16_attention_weight_min": -27.7778263092041,
      "activations/layer17_attention_weight_max": 52.35658645629883,
      "activations/layer17_attention_weight_min": -40.830570220947266,
      "activations/layer18_attention_weight_max": 49.36827850341797,
      "activations/layer18_attention_weight_min": -34.5666618347168,
      "activations/layer19_attention_weight_max": 21.574716567993164,
      "activations/layer19_attention_weight_min": -21.947690963745117,
      "activations/layer1_attention_weight_max": 17.42230796813965,
      "activations/layer1_attention_weight_min": -15.297354698181152,
      "activations/layer20_attention_weight_max": 23.949352264404297,
      "activations/layer20_attention_weight_min": -24.926977157592773,
      "activations/layer21_attention_weight_max": 37.3323974609375,
      "activations/layer21_attention_weight_min": -24.05325698852539,
      "activations/layer22_attention_weight_max": 33.68976593017578,
      "activations/layer22_attention_weight_min": -26.26026725769043,
      "activations/layer23_attention_weight_max": 31.177345275878906,
      "activations/layer23_attention_weight_min": -21.960662841796875,
      "activations/layer2_attention_weight_max": 30.987699508666992,
      "activations/layer2_attention_weight_min": -29.372787475585938,
      "activations/layer3_attention_weight_max": 82.13220977783203,
      "activations/layer3_attention_weight_min": -86.41238403320312,
      "activations/layer4_attention_weight_max": 94.35948944091797,
      "activations/layer4_attention_weight_min": -92.68560028076172,
      "activations/layer5_attention_weight_max": 85.56780242919922,
      "activations/layer5_attention_weight_min": -83.64324951171875,
      "activations/layer6_attention_weight_max": 53.03705596923828,
      "activations/layer6_attention_weight_min": -53.18943786621094,
      "activations/layer7_attention_weight_max": 74.16181945800781,
      "activations/layer7_attention_weight_min": -73.85800170898438,
      "activations/layer8_attention_weight_max": 47.15424728393555,
      "activations/layer8_attention_weight_min": -46.401432037353516,
      "activations/layer9_attention_weight_max": 56.055355072021484,
      "activations/layer9_attention_weight_min": -50.68703842163086,
      "epoch": 7.29,
      "learning_rate": 0.0001040215909090909,
      "loss": 2.8285,
      "step": 125500
    },
    {
      "activations/layer0_attention_weight_max": 14.394124031066895,
      "activations/layer0_attention_weight_min": -14.591479301452637,
      "activations/layer10_attention_weight_max": 40.6328125,
      "activations/layer10_attention_weight_min": -36.70574951171875,
      "activations/layer11_attention_weight_max": 30.638059616088867,
      "activations/layer11_attention_weight_min": -31.388931274414062,
      "activations/layer12_attention_weight_max": 20.210371017456055,
      "activations/layer12_attention_weight_min": -25.194934844970703,
      "activations/layer13_attention_weight_max": 42.26655960083008,
      "activations/layer13_attention_weight_min": -31.139307022094727,
      "activations/layer14_attention_weight_max": 40.814334869384766,
      "activations/layer14_attention_weight_min": -30.76587677001953,
      "activations/layer15_attention_weight_max": 34.79745101928711,
      "activations/layer15_attention_weight_min": -29.672834396362305,
      "activations/layer16_attention_weight_max": 35.30698013305664,
      "activations/layer16_attention_weight_min": -27.51544761657715,
      "activations/layer17_attention_weight_max": 52.920684814453125,
      "activations/layer17_attention_weight_min": -41.8477783203125,
      "activations/layer18_attention_weight_max": 51.08509063720703,
      "activations/layer18_attention_weight_min": -36.8864860534668,
      "activations/layer19_attention_weight_max": 20.73287010192871,
      "activations/layer19_attention_weight_min": -21.699865341186523,
      "activations/layer1_attention_weight_max": 17.796052932739258,
      "activations/layer1_attention_weight_min": -15.165152549743652,
      "activations/layer20_attention_weight_max": 21.47885513305664,
      "activations/layer20_attention_weight_min": -23.772098541259766,
      "activations/layer21_attention_weight_max": 39.12809753417969,
      "activations/layer21_attention_weight_min": -24.54112434387207,
      "activations/layer22_attention_weight_max": 33.442596435546875,
      "activations/layer22_attention_weight_min": -26.25058937072754,
      "activations/layer23_attention_weight_max": 35.875885009765625,
      "activations/layer23_attention_weight_min": -21.088998794555664,
      "activations/layer2_attention_weight_max": 30.845335006713867,
      "activations/layer2_attention_weight_min": -28.095273971557617,
      "activations/layer3_attention_weight_max": 83.17927551269531,
      "activations/layer3_attention_weight_min": -83.23890686035156,
      "activations/layer4_attention_weight_max": 101.55918884277344,
      "activations/layer4_attention_weight_min": -93.8929443359375,
      "activations/layer5_attention_weight_max": 92.39134216308594,
      "activations/layer5_attention_weight_min": -86.58241271972656,
      "activations/layer6_attention_weight_max": 58.94459915161133,
      "activations/layer6_attention_weight_min": -53.61107635498047,
      "activations/layer7_attention_weight_max": 75.16252136230469,
      "activations/layer7_attention_weight_min": -73.13670349121094,
      "activations/layer8_attention_weight_max": 44.762977600097656,
      "activations/layer8_attention_weight_min": -45.228126525878906,
      "activations/layer9_attention_weight_max": 48.70310592651367,
      "activations/layer9_attention_weight_min": -51.049312591552734,
      "epoch": 7.3,
      "learning_rate": 0.0001040026515151515,
      "loss": 2.82,
      "step": 125550
    },
    {
      "activations/layer0_attention_weight_max": 15.777585983276367,
      "activations/layer0_attention_weight_min": -15.029186248779297,
      "activations/layer10_attention_weight_max": 37.78910446166992,
      "activations/layer10_attention_weight_min": -34.844932556152344,
      "activations/layer11_attention_weight_max": 32.50941467285156,
      "activations/layer11_attention_weight_min": -31.14995574951172,
      "activations/layer12_attention_weight_max": 21.35926628112793,
      "activations/layer12_attention_weight_min": -28.68790054321289,
      "activations/layer13_attention_weight_max": 54.8679313659668,
      "activations/layer13_attention_weight_min": -32.907718658447266,
      "activations/layer14_attention_weight_max": 47.596282958984375,
      "activations/layer14_attention_weight_min": -31.328414916992188,
      "activations/layer15_attention_weight_max": 39.131107330322266,
      "activations/layer15_attention_weight_min": -27.866361618041992,
      "activations/layer16_attention_weight_max": 39.10782241821289,
      "activations/layer16_attention_weight_min": -27.46487045288086,
      "activations/layer17_attention_weight_max": 55.155372619628906,
      "activations/layer17_attention_weight_min": -42.025978088378906,
      "activations/layer18_attention_weight_max": 51.90557861328125,
      "activations/layer18_attention_weight_min": -36.784767150878906,
      "activations/layer19_attention_weight_max": 21.637514114379883,
      "activations/layer19_attention_weight_min": -22.586627960205078,
      "activations/layer1_attention_weight_max": 17.130355834960938,
      "activations/layer1_attention_weight_min": -16.405467987060547,
      "activations/layer20_attention_weight_max": 25.174577713012695,
      "activations/layer20_attention_weight_min": -23.594745635986328,
      "activations/layer21_attention_weight_max": 45.8399543762207,
      "activations/layer21_attention_weight_min": -23.117494583129883,
      "activations/layer22_attention_weight_max": 33.152976989746094,
      "activations/layer22_attention_weight_min": -28.853500366210938,
      "activations/layer23_attention_weight_max": 40.504940032958984,
      "activations/layer23_attention_weight_min": -21.376911163330078,
      "activations/layer2_attention_weight_max": 33.47407913208008,
      "activations/layer2_attention_weight_min": -32.81406784057617,
      "activations/layer3_attention_weight_max": 91.04222106933594,
      "activations/layer3_attention_weight_min": -88.42354583740234,
      "activations/layer4_attention_weight_max": 107.38329315185547,
      "activations/layer4_attention_weight_min": -98.95929718017578,
      "activations/layer5_attention_weight_max": 92.12459564208984,
      "activations/layer5_attention_weight_min": -86.2981185913086,
      "activations/layer6_attention_weight_max": 57.16874694824219,
      "activations/layer6_attention_weight_min": -51.73700714111328,
      "activations/layer7_attention_weight_max": 80.8425064086914,
      "activations/layer7_attention_weight_min": -75.02186584472656,
      "activations/layer8_attention_weight_max": 45.80897521972656,
      "activations/layer8_attention_weight_min": -44.14881896972656,
      "activations/layer9_attention_weight_max": 60.506690979003906,
      "activations/layer9_attention_weight_min": -58.052978515625,
      "epoch": 7.3,
      "learning_rate": 0.00010398371212121212,
      "loss": 2.8248,
      "step": 125600
    },
    {
      "activations/layer0_attention_weight_max": 15.548821449279785,
      "activations/layer0_attention_weight_min": -14.629541397094727,
      "activations/layer10_attention_weight_max": 37.778404235839844,
      "activations/layer10_attention_weight_min": -36.07813262939453,
      "activations/layer11_attention_weight_max": 32.54920196533203,
      "activations/layer11_attention_weight_min": -29.347715377807617,
      "activations/layer12_attention_weight_max": 29.76229476928711,
      "activations/layer12_attention_weight_min": -25.20814323425293,
      "activations/layer13_attention_weight_max": 48.4967155456543,
      "activations/layer13_attention_weight_min": -33.904296875,
      "activations/layer14_attention_weight_max": 49.02247619628906,
      "activations/layer14_attention_weight_min": -29.337753295898438,
      "activations/layer15_attention_weight_max": 37.7593994140625,
      "activations/layer15_attention_weight_min": -29.58754539489746,
      "activations/layer16_attention_weight_max": 33.03532028198242,
      "activations/layer16_attention_weight_min": -27.118701934814453,
      "activations/layer17_attention_weight_max": 55.33546829223633,
      "activations/layer17_attention_weight_min": -42.07861328125,
      "activations/layer18_attention_weight_max": 48.61614990234375,
      "activations/layer18_attention_weight_min": -36.20366668701172,
      "activations/layer19_attention_weight_max": 21.806257247924805,
      "activations/layer19_attention_weight_min": -20.20294761657715,
      "activations/layer1_attention_weight_max": 16.702327728271484,
      "activations/layer1_attention_weight_min": -15.112975120544434,
      "activations/layer20_attention_weight_max": 23.084692001342773,
      "activations/layer20_attention_weight_min": -20.34657859802246,
      "activations/layer21_attention_weight_max": 36.684913635253906,
      "activations/layer21_attention_weight_min": -23.80655860900879,
      "activations/layer22_attention_weight_max": 30.64027214050293,
      "activations/layer22_attention_weight_min": -24.207605361938477,
      "activations/layer23_attention_weight_max": 34.50143814086914,
      "activations/layer23_attention_weight_min": -17.828075408935547,
      "activations/layer2_attention_weight_max": 31.287883758544922,
      "activations/layer2_attention_weight_min": -28.877288818359375,
      "activations/layer3_attention_weight_max": 83.25335693359375,
      "activations/layer3_attention_weight_min": -83.5444564819336,
      "activations/layer4_attention_weight_max": 98.7667465209961,
      "activations/layer4_attention_weight_min": -94.05174255371094,
      "activations/layer5_attention_weight_max": 89.40313720703125,
      "activations/layer5_attention_weight_min": -82.98974609375,
      "activations/layer6_attention_weight_max": 59.1226692199707,
      "activations/layer6_attention_weight_min": -51.42496871948242,
      "activations/layer7_attention_weight_max": 75.37568664550781,
      "activations/layer7_attention_weight_min": -69.46460723876953,
      "activations/layer8_attention_weight_max": 45.98429870605469,
      "activations/layer8_attention_weight_min": -44.125240325927734,
      "activations/layer9_attention_weight_max": 51.50604248046875,
      "activations/layer9_attention_weight_min": -47.908355712890625,
      "epoch": 7.3,
      "learning_rate": 0.00010396477272727271,
      "loss": 2.8138,
      "step": 125650
    },
    {
      "activations/layer0_attention_weight_max": 15.48073673248291,
      "activations/layer0_attention_weight_min": -14.862457275390625,
      "activations/layer10_attention_weight_max": 37.83135223388672,
      "activations/layer10_attention_weight_min": -34.34224319458008,
      "activations/layer11_attention_weight_max": 31.16851806640625,
      "activations/layer11_attention_weight_min": -30.1960506439209,
      "activations/layer12_attention_weight_max": 21.850448608398438,
      "activations/layer12_attention_weight_min": -25.31510353088379,
      "activations/layer13_attention_weight_max": 53.25410842895508,
      "activations/layer13_attention_weight_min": -35.85261917114258,
      "activations/layer14_attention_weight_max": 43.9641227722168,
      "activations/layer14_attention_weight_min": -29.672693252563477,
      "activations/layer15_attention_weight_max": 40.16965103149414,
      "activations/layer15_attention_weight_min": -29.548004150390625,
      "activations/layer16_attention_weight_max": 33.295902252197266,
      "activations/layer16_attention_weight_min": -27.243093490600586,
      "activations/layer17_attention_weight_max": 58.08859634399414,
      "activations/layer17_attention_weight_min": -43.23161697387695,
      "activations/layer18_attention_weight_max": 48.93411636352539,
      "activations/layer18_attention_weight_min": -37.70062255859375,
      "activations/layer19_attention_weight_max": 25.0175724029541,
      "activations/layer19_attention_weight_min": -21.586700439453125,
      "activations/layer1_attention_weight_max": 17.171846389770508,
      "activations/layer1_attention_weight_min": -13.328378677368164,
      "activations/layer20_attention_weight_max": 23.427366256713867,
      "activations/layer20_attention_weight_min": -20.519376754760742,
      "activations/layer21_attention_weight_max": 38.9765625,
      "activations/layer21_attention_weight_min": -26.079030990600586,
      "activations/layer22_attention_weight_max": 38.307273864746094,
      "activations/layer22_attention_weight_min": -26.32817840576172,
      "activations/layer23_attention_weight_max": 37.022193908691406,
      "activations/layer23_attention_weight_min": -21.614133834838867,
      "activations/layer2_attention_weight_max": 30.56641960144043,
      "activations/layer2_attention_weight_min": -29.38825225830078,
      "activations/layer3_attention_weight_max": 85.3759994506836,
      "activations/layer3_attention_weight_min": -87.60257720947266,
      "activations/layer4_attention_weight_max": 102.83863830566406,
      "activations/layer4_attention_weight_min": -100.0108871459961,
      "activations/layer5_attention_weight_max": 93.77151489257812,
      "activations/layer5_attention_weight_min": -89.95845794677734,
      "activations/layer6_attention_weight_max": 60.65162658691406,
      "activations/layer6_attention_weight_min": -55.42887496948242,
      "activations/layer7_attention_weight_max": 79.09760284423828,
      "activations/layer7_attention_weight_min": -76.54454803466797,
      "activations/layer8_attention_weight_max": 47.644649505615234,
      "activations/layer8_attention_weight_min": -50.46249008178711,
      "activations/layer9_attention_weight_max": 50.972816467285156,
      "activations/layer9_attention_weight_min": -49.94902420043945,
      "epoch": 7.3,
      "learning_rate": 0.00010394583333333332,
      "loss": 2.8222,
      "step": 125700
    },
    {
      "activations/layer0_attention_weight_max": 15.382061958312988,
      "activations/layer0_attention_weight_min": -14.901358604431152,
      "activations/layer10_attention_weight_max": 37.71235275268555,
      "activations/layer10_attention_weight_min": -37.16537857055664,
      "activations/layer11_attention_weight_max": 31.385143280029297,
      "activations/layer11_attention_weight_min": -30.22024917602539,
      "activations/layer12_attention_weight_max": 20.240324020385742,
      "activations/layer12_attention_weight_min": -23.47452735900879,
      "activations/layer13_attention_weight_max": 46.633052825927734,
      "activations/layer13_attention_weight_min": -34.27981948852539,
      "activations/layer14_attention_weight_max": 38.564796447753906,
      "activations/layer14_attention_weight_min": -29.71755599975586,
      "activations/layer15_attention_weight_max": 37.08338928222656,
      "activations/layer15_attention_weight_min": -29.34571075439453,
      "activations/layer16_attention_weight_max": 31.103759765625,
      "activations/layer16_attention_weight_min": -28.261627197265625,
      "activations/layer17_attention_weight_max": 54.84254455566406,
      "activations/layer17_attention_weight_min": -43.277530670166016,
      "activations/layer18_attention_weight_max": 47.469970703125,
      "activations/layer18_attention_weight_min": -36.747745513916016,
      "activations/layer19_attention_weight_max": 20.99654197692871,
      "activations/layer19_attention_weight_min": -21.94035530090332,
      "activations/layer1_attention_weight_max": 16.863027572631836,
      "activations/layer1_attention_weight_min": -15.2613525390625,
      "activations/layer20_attention_weight_max": 25.925817489624023,
      "activations/layer20_attention_weight_min": -22.819568634033203,
      "activations/layer21_attention_weight_max": 41.669273376464844,
      "activations/layer21_attention_weight_min": -23.790630340576172,
      "activations/layer22_attention_weight_max": 35.27277374267578,
      "activations/layer22_attention_weight_min": -25.18548011779785,
      "activations/layer23_attention_weight_max": 36.57305908203125,
      "activations/layer23_attention_weight_min": -22.193395614624023,
      "activations/layer2_attention_weight_max": 30.79167938232422,
      "activations/layer2_attention_weight_min": -29.35712432861328,
      "activations/layer3_attention_weight_max": 90.5302963256836,
      "activations/layer3_attention_weight_min": -92.28375244140625,
      "activations/layer4_attention_weight_max": 103.49417877197266,
      "activations/layer4_attention_weight_min": -96.15901184082031,
      "activations/layer5_attention_weight_max": 94.70716857910156,
      "activations/layer5_attention_weight_min": -87.46231079101562,
      "activations/layer6_attention_weight_max": 57.63486099243164,
      "activations/layer6_attention_weight_min": -54.23764419555664,
      "activations/layer7_attention_weight_max": 77.64244842529297,
      "activations/layer7_attention_weight_min": -72.67630004882812,
      "activations/layer8_attention_weight_max": 45.1918830871582,
      "activations/layer8_attention_weight_min": -45.421241760253906,
      "activations/layer9_attention_weight_max": 48.873939514160156,
      "activations/layer9_attention_weight_min": -50.78860092163086,
      "epoch": 7.31,
      "learning_rate": 0.00010392689393939394,
      "loss": 2.81,
      "step": 125750
    },
    {
      "activations/layer0_attention_weight_max": 15.609674453735352,
      "activations/layer0_attention_weight_min": -14.381266593933105,
      "activations/layer10_attention_weight_max": 38.83176040649414,
      "activations/layer10_attention_weight_min": -35.795040130615234,
      "activations/layer11_attention_weight_max": 32.00288772583008,
      "activations/layer11_attention_weight_min": -30.41309928894043,
      "activations/layer12_attention_weight_max": 22.37391471862793,
      "activations/layer12_attention_weight_min": -25.32500648498535,
      "activations/layer13_attention_weight_max": 40.576881408691406,
      "activations/layer13_attention_weight_min": -34.40305709838867,
      "activations/layer14_attention_weight_max": 36.62547302246094,
      "activations/layer14_attention_weight_min": -31.782337188720703,
      "activations/layer15_attention_weight_max": 35.62946701049805,
      "activations/layer15_attention_weight_min": -31.146244049072266,
      "activations/layer16_attention_weight_max": 33.62208938598633,
      "activations/layer16_attention_weight_min": -27.85072898864746,
      "activations/layer17_attention_weight_max": 53.01247024536133,
      "activations/layer17_attention_weight_min": -41.96510314941406,
      "activations/layer18_attention_weight_max": 45.30187225341797,
      "activations/layer18_attention_weight_min": -34.14075469970703,
      "activations/layer19_attention_weight_max": 20.845664978027344,
      "activations/layer19_attention_weight_min": -20.15291404724121,
      "activations/layer1_attention_weight_max": 16.567787170410156,
      "activations/layer1_attention_weight_min": -14.34325885772705,
      "activations/layer20_attention_weight_max": 21.231191635131836,
      "activations/layer20_attention_weight_min": -24.25694465637207,
      "activations/layer21_attention_weight_max": 38.81489181518555,
      "activations/layer21_attention_weight_min": -24.795822143554688,
      "activations/layer22_attention_weight_max": 30.826740264892578,
      "activations/layer22_attention_weight_min": -24.64564323425293,
      "activations/layer23_attention_weight_max": 32.531558990478516,
      "activations/layer23_attention_weight_min": -22.228900909423828,
      "activations/layer2_attention_weight_max": 32.38505554199219,
      "activations/layer2_attention_weight_min": -28.827041625976562,
      "activations/layer3_attention_weight_max": 87.5673828125,
      "activations/layer3_attention_weight_min": -88.98494720458984,
      "activations/layer4_attention_weight_max": 104.43177795410156,
      "activations/layer4_attention_weight_min": -94.43888092041016,
      "activations/layer5_attention_weight_max": 90.78813934326172,
      "activations/layer5_attention_weight_min": -81.52445983886719,
      "activations/layer6_attention_weight_max": 57.176212310791016,
      "activations/layer6_attention_weight_min": -54.18089294433594,
      "activations/layer7_attention_weight_max": 74.4677734375,
      "activations/layer7_attention_weight_min": -69.95774841308594,
      "activations/layer8_attention_weight_max": 45.58224105834961,
      "activations/layer8_attention_weight_min": -44.31488800048828,
      "activations/layer9_attention_weight_max": 50.169795989990234,
      "activations/layer9_attention_weight_min": -46.99265670776367,
      "epoch": 7.31,
      "learning_rate": 0.00010390795454545453,
      "loss": 2.8137,
      "step": 125800
    },
    {
      "activations/layer0_attention_weight_max": 16.388843536376953,
      "activations/layer0_attention_weight_min": -14.500092506408691,
      "activations/layer10_attention_weight_max": 37.628211975097656,
      "activations/layer10_attention_weight_min": -35.36665725708008,
      "activations/layer11_attention_weight_max": 31.624282836914062,
      "activations/layer11_attention_weight_min": -30.224388122558594,
      "activations/layer12_attention_weight_max": 22.31195068359375,
      "activations/layer12_attention_weight_min": -23.835678100585938,
      "activations/layer13_attention_weight_max": 45.1315803527832,
      "activations/layer13_attention_weight_min": -31.860061645507812,
      "activations/layer14_attention_weight_max": 39.12443161010742,
      "activations/layer14_attention_weight_min": -29.189861297607422,
      "activations/layer15_attention_weight_max": 37.6741828918457,
      "activations/layer15_attention_weight_min": -28.81356430053711,
      "activations/layer16_attention_weight_max": 30.497652053833008,
      "activations/layer16_attention_weight_min": -26.843095779418945,
      "activations/layer17_attention_weight_max": 56.501583099365234,
      "activations/layer17_attention_weight_min": -41.57562255859375,
      "activations/layer18_attention_weight_max": 48.36143493652344,
      "activations/layer18_attention_weight_min": -35.91023635864258,
      "activations/layer19_attention_weight_max": 22.24729347229004,
      "activations/layer19_attention_weight_min": -20.653884887695312,
      "activations/layer1_attention_weight_max": 16.953794479370117,
      "activations/layer1_attention_weight_min": -13.728154182434082,
      "activations/layer20_attention_weight_max": 22.84589195251465,
      "activations/layer20_attention_weight_min": -22.627788543701172,
      "activations/layer21_attention_weight_max": 37.568084716796875,
      "activations/layer21_attention_weight_min": -22.1156063079834,
      "activations/layer22_attention_weight_max": 30.47066879272461,
      "activations/layer22_attention_weight_min": -24.633710861206055,
      "activations/layer23_attention_weight_max": 33.01581573486328,
      "activations/layer23_attention_weight_min": -19.84407615661621,
      "activations/layer2_attention_weight_max": 30.699329376220703,
      "activations/layer2_attention_weight_min": -28.57160186767578,
      "activations/layer3_attention_weight_max": 82.5312728881836,
      "activations/layer3_attention_weight_min": -83.73480224609375,
      "activations/layer4_attention_weight_max": 97.1450424194336,
      "activations/layer4_attention_weight_min": -92.31896209716797,
      "activations/layer5_attention_weight_max": 88.70549774169922,
      "activations/layer5_attention_weight_min": -85.3603515625,
      "activations/layer6_attention_weight_max": 54.32518005371094,
      "activations/layer6_attention_weight_min": -52.81532287597656,
      "activations/layer7_attention_weight_max": 71.77497100830078,
      "activations/layer7_attention_weight_min": -72.46401977539062,
      "activations/layer8_attention_weight_max": 44.84819793701172,
      "activations/layer8_attention_weight_min": -46.657936096191406,
      "activations/layer9_attention_weight_max": 50.599639892578125,
      "activations/layer9_attention_weight_min": -46.91801071166992,
      "epoch": 7.31,
      "learning_rate": 0.00010388901515151514,
      "loss": 2.8068,
      "step": 125850
    },
    {
      "activations/layer0_attention_weight_max": 13.80127239227295,
      "activations/layer0_attention_weight_min": -14.4654541015625,
      "activations/layer10_attention_weight_max": 41.26499557495117,
      "activations/layer10_attention_weight_min": -37.35417938232422,
      "activations/layer11_attention_weight_max": 32.75644302368164,
      "activations/layer11_attention_weight_min": -31.890804290771484,
      "activations/layer12_attention_weight_max": 19.773855209350586,
      "activations/layer12_attention_weight_min": -24.790115356445312,
      "activations/layer13_attention_weight_max": 59.87425994873047,
      "activations/layer13_attention_weight_min": -35.789825439453125,
      "activations/layer14_attention_weight_max": 57.512107849121094,
      "activations/layer14_attention_weight_min": -30.441801071166992,
      "activations/layer15_attention_weight_max": 45.91780471801758,
      "activations/layer15_attention_weight_min": -29.350011825561523,
      "activations/layer16_attention_weight_max": 39.87107849121094,
      "activations/layer16_attention_weight_min": -28.982454299926758,
      "activations/layer17_attention_weight_max": 64.97432708740234,
      "activations/layer17_attention_weight_min": -46.20501708984375,
      "activations/layer18_attention_weight_max": 56.53339767456055,
      "activations/layer18_attention_weight_min": -40.13242721557617,
      "activations/layer19_attention_weight_max": 23.2471981048584,
      "activations/layer19_attention_weight_min": -23.426414489746094,
      "activations/layer1_attention_weight_max": 17.700767517089844,
      "activations/layer1_attention_weight_min": -16.7017822265625,
      "activations/layer20_attention_weight_max": 27.51287078857422,
      "activations/layer20_attention_weight_min": -22.868627548217773,
      "activations/layer21_attention_weight_max": 47.698699951171875,
      "activations/layer21_attention_weight_min": -24.33669090270996,
      "activations/layer22_attention_weight_max": 39.513423919677734,
      "activations/layer22_attention_weight_min": -26.86180877685547,
      "activations/layer23_attention_weight_max": 41.029052734375,
      "activations/layer23_attention_weight_min": -20.3136043548584,
      "activations/layer2_attention_weight_max": 32.395286560058594,
      "activations/layer2_attention_weight_min": -29.877403259277344,
      "activations/layer3_attention_weight_max": 83.79834747314453,
      "activations/layer3_attention_weight_min": -85.86222076416016,
      "activations/layer4_attention_weight_max": 99.79096221923828,
      "activations/layer4_attention_weight_min": -94.83146667480469,
      "activations/layer5_attention_weight_max": 90.61921691894531,
      "activations/layer5_attention_weight_min": -86.1412582397461,
      "activations/layer6_attention_weight_max": 56.661895751953125,
      "activations/layer6_attention_weight_min": -54.176944732666016,
      "activations/layer7_attention_weight_max": 78.62684631347656,
      "activations/layer7_attention_weight_min": -74.96807861328125,
      "activations/layer8_attention_weight_max": 48.00794982910156,
      "activations/layer8_attention_weight_min": -49.9432487487793,
      "activations/layer9_attention_weight_max": 53.39439010620117,
      "activations/layer9_attention_weight_min": -52.692909240722656,
      "epoch": 7.32,
      "learning_rate": 0.00010387007575757576,
      "loss": 2.8183,
      "step": 125900
    },
    {
      "activations/layer0_attention_weight_max": 16.925596237182617,
      "activations/layer0_attention_weight_min": -14.95623779296875,
      "activations/layer10_attention_weight_max": 42.14378356933594,
      "activations/layer10_attention_weight_min": -41.798248291015625,
      "activations/layer11_attention_weight_max": 34.01987075805664,
      "activations/layer11_attention_weight_min": -34.64031982421875,
      "activations/layer12_attention_weight_max": 21.718223571777344,
      "activations/layer12_attention_weight_min": -25.202449798583984,
      "activations/layer13_attention_weight_max": 47.38650131225586,
      "activations/layer13_attention_weight_min": -33.08754348754883,
      "activations/layer14_attention_weight_max": 40.3608512878418,
      "activations/layer14_attention_weight_min": -30.392793655395508,
      "activations/layer15_attention_weight_max": 38.75163650512695,
      "activations/layer15_attention_weight_min": -29.481765747070312,
      "activations/layer16_attention_weight_max": 34.52347946166992,
      "activations/layer16_attention_weight_min": -28.199234008789062,
      "activations/layer17_attention_weight_max": 59.98670959472656,
      "activations/layer17_attention_weight_min": -42.70243835449219,
      "activations/layer18_attention_weight_max": 50.62735366821289,
      "activations/layer18_attention_weight_min": -36.24424362182617,
      "activations/layer19_attention_weight_max": 23.65335464477539,
      "activations/layer19_attention_weight_min": -21.69306755065918,
      "activations/layer1_attention_weight_max": 16.394287109375,
      "activations/layer1_attention_weight_min": -14.565013885498047,
      "activations/layer20_attention_weight_max": 22.219139099121094,
      "activations/layer20_attention_weight_min": -20.188676834106445,
      "activations/layer21_attention_weight_max": 39.67259216308594,
      "activations/layer21_attention_weight_min": -26.547454833984375,
      "activations/layer22_attention_weight_max": 37.2142219543457,
      "activations/layer22_attention_weight_min": -25.623212814331055,
      "activations/layer23_attention_weight_max": 35.24370574951172,
      "activations/layer23_attention_weight_min": -19.50783920288086,
      "activations/layer2_attention_weight_max": 31.840763092041016,
      "activations/layer2_attention_weight_min": -29.258146286010742,
      "activations/layer3_attention_weight_max": 85.8853988647461,
      "activations/layer3_attention_weight_min": -89.47872161865234,
      "activations/layer4_attention_weight_max": 104.6241226196289,
      "activations/layer4_attention_weight_min": -100.32876586914062,
      "activations/layer5_attention_weight_max": 94.03358459472656,
      "activations/layer5_attention_weight_min": -92.08677673339844,
      "activations/layer6_attention_weight_max": 55.0240478515625,
      "activations/layer6_attention_weight_min": -55.95294952392578,
      "activations/layer7_attention_weight_max": 78.16390991210938,
      "activations/layer7_attention_weight_min": -76.1165771484375,
      "activations/layer8_attention_weight_max": 47.09759521484375,
      "activations/layer8_attention_weight_min": -49.3763313293457,
      "activations/layer9_attention_weight_max": 54.576576232910156,
      "activations/layer9_attention_weight_min": -55.14794921875,
      "epoch": 7.32,
      "learning_rate": 0.00010385113636363634,
      "loss": 2.8145,
      "step": 125950
    },
    {
      "activations/layer0_attention_weight_max": 16.2772216796875,
      "activations/layer0_attention_weight_min": -14.568653106689453,
      "activations/layer10_attention_weight_max": 36.99452590942383,
      "activations/layer10_attention_weight_min": -34.23252487182617,
      "activations/layer11_attention_weight_max": 30.117992401123047,
      "activations/layer11_attention_weight_min": -28.695898056030273,
      "activations/layer12_attention_weight_max": 17.556785583496094,
      "activations/layer12_attention_weight_min": -26.380285263061523,
      "activations/layer13_attention_weight_max": 46.11347961425781,
      "activations/layer13_attention_weight_min": -33.60285949707031,
      "activations/layer14_attention_weight_max": 47.687442779541016,
      "activations/layer14_attention_weight_min": -30.6116886138916,
      "activations/layer15_attention_weight_max": 36.411678314208984,
      "activations/layer15_attention_weight_min": -29.921003341674805,
      "activations/layer16_attention_weight_max": 33.51100158691406,
      "activations/layer16_attention_weight_min": -27.009979248046875,
      "activations/layer17_attention_weight_max": 53.52858352661133,
      "activations/layer17_attention_weight_min": -45.80874252319336,
      "activations/layer18_attention_weight_max": 46.416893005371094,
      "activations/layer18_attention_weight_min": -37.63179016113281,
      "activations/layer19_attention_weight_max": 22.123672485351562,
      "activations/layer19_attention_weight_min": -22.305192947387695,
      "activations/layer1_attention_weight_max": 16.95722770690918,
      "activations/layer1_attention_weight_min": -15.433235168457031,
      "activations/layer20_attention_weight_max": 25.49431037902832,
      "activations/layer20_attention_weight_min": -20.744993209838867,
      "activations/layer21_attention_weight_max": 44.50819778442383,
      "activations/layer21_attention_weight_min": -23.07024574279785,
      "activations/layer22_attention_weight_max": 35.87939453125,
      "activations/layer22_attention_weight_min": -25.32557487487793,
      "activations/layer23_attention_weight_max": 32.033103942871094,
      "activations/layer23_attention_weight_min": -19.72635269165039,
      "activations/layer2_attention_weight_max": 31.369670867919922,
      "activations/layer2_attention_weight_min": -29.234968185424805,
      "activations/layer3_attention_weight_max": 86.13837432861328,
      "activations/layer3_attention_weight_min": -82.38946533203125,
      "activations/layer4_attention_weight_max": 96.37560272216797,
      "activations/layer4_attention_weight_min": -94.03600311279297,
      "activations/layer5_attention_weight_max": 86.46029663085938,
      "activations/layer5_attention_weight_min": -80.80049133300781,
      "activations/layer6_attention_weight_max": 53.69401168823242,
      "activations/layer6_attention_weight_min": -51.20286560058594,
      "activations/layer7_attention_weight_max": 73.81962585449219,
      "activations/layer7_attention_weight_min": -70.28169250488281,
      "activations/layer8_attention_weight_max": 43.177738189697266,
      "activations/layer8_attention_weight_min": -44.40816116333008,
      "activations/layer9_attention_weight_max": 52.96694564819336,
      "activations/layer9_attention_weight_min": -48.674720764160156,
      "epoch": 7.32,
      "learning_rate": 0.00010383219696969696,
      "loss": 2.8273,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5699,
      "eval_samples_per_second": 501.059,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5699,
      "eval_openwebtext_samples_per_second": 501.059,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 1.9753,
      "eval_wikitext_samples_per_second": 230.853,
      "step": 126000
    },
    {
      "epoch": 7.32,
      "eval_lambada_loss": 2.734375,
      "eval_lambada_ppl": 15.400115355370453,
      "eval_lambada_runtime": 9.6346,
      "eval_lambada_samples_per_second": 505.368,
      "step": 126000
    },
    {
      "activations/layer0_attention_weight_max": 14.40738296508789,
      "activations/layer0_attention_weight_min": -14.892167091369629,
      "activations/layer10_attention_weight_max": 43.984100341796875,
      "activations/layer10_attention_weight_min": -39.8527717590332,
      "activations/layer11_attention_weight_max": 36.05872344970703,
      "activations/layer11_attention_weight_min": -32.29278564453125,
      "activations/layer12_attention_weight_max": 19.38205909729004,
      "activations/layer12_attention_weight_min": -24.694984436035156,
      "activations/layer13_attention_weight_max": 49.6482048034668,
      "activations/layer13_attention_weight_min": -32.377410888671875,
      "activations/layer14_attention_weight_max": 50.22611999511719,
      "activations/layer14_attention_weight_min": -30.061010360717773,
      "activations/layer15_attention_weight_max": 49.54250717163086,
      "activations/layer15_attention_weight_min": -28.96988296508789,
      "activations/layer16_attention_weight_max": 33.95061492919922,
      "activations/layer16_attention_weight_min": -26.346948623657227,
      "activations/layer17_attention_weight_max": 60.17024612426758,
      "activations/layer17_attention_weight_min": -44.80638122558594,
      "activations/layer18_attention_weight_max": 52.28057861328125,
      "activations/layer18_attention_weight_min": -38.646724700927734,
      "activations/layer19_attention_weight_max": 23.8825626373291,
      "activations/layer19_attention_weight_min": -23.511140823364258,
      "activations/layer1_attention_weight_max": 16.939821243286133,
      "activations/layer1_attention_weight_min": -14.367990493774414,
      "activations/layer20_attention_weight_max": 24.055253982543945,
      "activations/layer20_attention_weight_min": -22.52182388305664,
      "activations/layer21_attention_weight_max": 53.16746520996094,
      "activations/layer21_attention_weight_min": -23.999834060668945,
      "activations/layer22_attention_weight_max": 35.42347717285156,
      "activations/layer22_attention_weight_min": -25.04885482788086,
      "activations/layer23_attention_weight_max": 39.79554748535156,
      "activations/layer23_attention_weight_min": -19.855741500854492,
      "activations/layer2_attention_weight_max": 32.16996765136719,
      "activations/layer2_attention_weight_min": -29.481407165527344,
      "activations/layer3_attention_weight_max": 91.20417785644531,
      "activations/layer3_attention_weight_min": -91.15950775146484,
      "activations/layer4_attention_weight_max": 104.11760711669922,
      "activations/layer4_attention_weight_min": -98.04656219482422,
      "activations/layer5_attention_weight_max": 89.56388854980469,
      "activations/layer5_attention_weight_min": -87.498779296875,
      "activations/layer6_attention_weight_max": 54.89790725708008,
      "activations/layer6_attention_weight_min": -53.69466781616211,
      "activations/layer7_attention_weight_max": 78.81776428222656,
      "activations/layer7_attention_weight_min": -70.79352569580078,
      "activations/layer8_attention_weight_max": 48.16532516479492,
      "activations/layer8_attention_weight_min": -47.68146514892578,
      "activations/layer9_attention_weight_max": 60.81122589111328,
      "activations/layer9_attention_weight_min": -49.84798812866211,
      "epoch": 7.32,
      "learning_rate": 0.00010381325757575757,
      "loss": 2.8149,
      "step": 126050
    },
    {
      "activations/layer0_attention_weight_max": 14.529928207397461,
      "activations/layer0_attention_weight_min": -14.737687110900879,
      "activations/layer10_attention_weight_max": 37.89750289916992,
      "activations/layer10_attention_weight_min": -33.107444763183594,
      "activations/layer11_attention_weight_max": 30.234342575073242,
      "activations/layer11_attention_weight_min": -29.910659790039062,
      "activations/layer12_attention_weight_max": 24.176822662353516,
      "activations/layer12_attention_weight_min": -27.469987869262695,
      "activations/layer13_attention_weight_max": 41.27545166015625,
      "activations/layer13_attention_weight_min": -32.422733306884766,
      "activations/layer14_attention_weight_max": 40.94571304321289,
      "activations/layer14_attention_weight_min": -29.388904571533203,
      "activations/layer15_attention_weight_max": 37.55022430419922,
      "activations/layer15_attention_weight_min": -29.37919044494629,
      "activations/layer16_attention_weight_max": 33.451629638671875,
      "activations/layer16_attention_weight_min": -26.507415771484375,
      "activations/layer17_attention_weight_max": 54.87297821044922,
      "activations/layer17_attention_weight_min": -40.63718032836914,
      "activations/layer18_attention_weight_max": 51.91065979003906,
      "activations/layer18_attention_weight_min": -32.86009979248047,
      "activations/layer19_attention_weight_max": 21.40400505065918,
      "activations/layer19_attention_weight_min": -20.114974975585938,
      "activations/layer1_attention_weight_max": 16.75540542602539,
      "activations/layer1_attention_weight_min": -15.011750221252441,
      "activations/layer20_attention_weight_max": 23.394105911254883,
      "activations/layer20_attention_weight_min": -22.609752655029297,
      "activations/layer21_attention_weight_max": 44.49795150756836,
      "activations/layer21_attention_weight_min": -23.6335391998291,
      "activations/layer22_attention_weight_max": 33.20532989501953,
      "activations/layer22_attention_weight_min": -23.830604553222656,
      "activations/layer23_attention_weight_max": 32.25138854980469,
      "activations/layer23_attention_weight_min": -21.10948944091797,
      "activations/layer2_attention_weight_max": 33.98753356933594,
      "activations/layer2_attention_weight_min": -29.547672271728516,
      "activations/layer3_attention_weight_max": 83.48341369628906,
      "activations/layer3_attention_weight_min": -84.6509780883789,
      "activations/layer4_attention_weight_max": 95.55918884277344,
      "activations/layer4_attention_weight_min": -91.89334869384766,
      "activations/layer5_attention_weight_max": 86.89611053466797,
      "activations/layer5_attention_weight_min": -81.42609405517578,
      "activations/layer6_attention_weight_max": 54.561744689941406,
      "activations/layer6_attention_weight_min": -53.12215042114258,
      "activations/layer7_attention_weight_max": 75.90613555908203,
      "activations/layer7_attention_weight_min": -68.25492095947266,
      "activations/layer8_attention_weight_max": 43.38724899291992,
      "activations/layer8_attention_weight_min": -42.860355377197266,
      "activations/layer9_attention_weight_max": 56.5927734375,
      "activations/layer9_attention_weight_min": -49.408443450927734,
      "epoch": 7.33,
      "learning_rate": 0.00010379431818181818,
      "loss": 2.8053,
      "step": 126100
    },
    {
      "activations/layer0_attention_weight_max": 14.264168739318848,
      "activations/layer0_attention_weight_min": -14.69811725616455,
      "activations/layer10_attention_weight_max": 36.6532096862793,
      "activations/layer10_attention_weight_min": -34.29061508178711,
      "activations/layer11_attention_weight_max": 31.590795516967773,
      "activations/layer11_attention_weight_min": -30.27326202392578,
      "activations/layer12_attention_weight_max": 18.86281394958496,
      "activations/layer12_attention_weight_min": -30.82233238220215,
      "activations/layer13_attention_weight_max": 49.82400894165039,
      "activations/layer13_attention_weight_min": -37.13374328613281,
      "activations/layer14_attention_weight_max": 41.35102462768555,
      "activations/layer14_attention_weight_min": -31.282054901123047,
      "activations/layer15_attention_weight_max": 38.139892578125,
      "activations/layer15_attention_weight_min": -28.982351303100586,
      "activations/layer16_attention_weight_max": 35.1863899230957,
      "activations/layer16_attention_weight_min": -27.84415054321289,
      "activations/layer17_attention_weight_max": 56.9582405090332,
      "activations/layer17_attention_weight_min": -47.828102111816406,
      "activations/layer18_attention_weight_max": 44.9928092956543,
      "activations/layer18_attention_weight_min": -38.43731689453125,
      "activations/layer19_attention_weight_max": 23.9090576171875,
      "activations/layer19_attention_weight_min": -21.637985229492188,
      "activations/layer1_attention_weight_max": 17.01813507080078,
      "activations/layer1_attention_weight_min": -12.982870101928711,
      "activations/layer20_attention_weight_max": 22.963346481323242,
      "activations/layer20_attention_weight_min": -25.57000160217285,
      "activations/layer21_attention_weight_max": 44.27400207519531,
      "activations/layer21_attention_weight_min": -25.310283660888672,
      "activations/layer22_attention_weight_max": 31.713850021362305,
      "activations/layer22_attention_weight_min": -24.28238868713379,
      "activations/layer23_attention_weight_max": 33.93985366821289,
      "activations/layer23_attention_weight_min": -20.166587829589844,
      "activations/layer2_attention_weight_max": 32.08393096923828,
      "activations/layer2_attention_weight_min": -29.50322723388672,
      "activations/layer3_attention_weight_max": 91.53543853759766,
      "activations/layer3_attention_weight_min": -89.24397277832031,
      "activations/layer4_attention_weight_max": 102.67662811279297,
      "activations/layer4_attention_weight_min": -96.04401397705078,
      "activations/layer5_attention_weight_max": 89.89351654052734,
      "activations/layer5_attention_weight_min": -80.77513122558594,
      "activations/layer6_attention_weight_max": 52.70799255371094,
      "activations/layer6_attention_weight_min": -52.13481521606445,
      "activations/layer7_attention_weight_max": 76.10130310058594,
      "activations/layer7_attention_weight_min": -71.5018310546875,
      "activations/layer8_attention_weight_max": 42.94923400878906,
      "activations/layer8_attention_weight_min": -44.62342834472656,
      "activations/layer9_attention_weight_max": 58.417701721191406,
      "activations/layer9_attention_weight_min": -50.563716888427734,
      "epoch": 7.33,
      "learning_rate": 0.00010377537878787878,
      "loss": 2.8205,
      "step": 126150
    },
    {
      "activations/layer0_attention_weight_max": 14.649083137512207,
      "activations/layer0_attention_weight_min": -15.035511016845703,
      "activations/layer10_attention_weight_max": 39.341739654541016,
      "activations/layer10_attention_weight_min": -33.865150451660156,
      "activations/layer11_attention_weight_max": 31.605567932128906,
      "activations/layer11_attention_weight_min": -28.64606475830078,
      "activations/layer12_attention_weight_max": 19.101482391357422,
      "activations/layer12_attention_weight_min": -25.274070739746094,
      "activations/layer13_attention_weight_max": 43.11854553222656,
      "activations/layer13_attention_weight_min": -33.48690414428711,
      "activations/layer14_attention_weight_max": 40.46052551269531,
      "activations/layer14_attention_weight_min": -32.4615478515625,
      "activations/layer15_attention_weight_max": 39.302268981933594,
      "activations/layer15_attention_weight_min": -29.57369613647461,
      "activations/layer16_attention_weight_max": 36.53714370727539,
      "activations/layer16_attention_weight_min": -28.921422958374023,
      "activations/layer17_attention_weight_max": 61.8054084777832,
      "activations/layer17_attention_weight_min": -46.385250091552734,
      "activations/layer18_attention_weight_max": 57.54287338256836,
      "activations/layer18_attention_weight_min": -40.193809509277344,
      "activations/layer19_attention_weight_max": 22.58568572998047,
      "activations/layer19_attention_weight_min": -21.40583038330078,
      "activations/layer1_attention_weight_max": 18.017335891723633,
      "activations/layer1_attention_weight_min": -14.289047241210938,
      "activations/layer20_attention_weight_max": 26.780651092529297,
      "activations/layer20_attention_weight_min": -22.78730010986328,
      "activations/layer21_attention_weight_max": 52.026058197021484,
      "activations/layer21_attention_weight_min": -23.499637603759766,
      "activations/layer22_attention_weight_max": 39.09352493286133,
      "activations/layer22_attention_weight_min": -25.885509490966797,
      "activations/layer23_attention_weight_max": 43.05245590209961,
      "activations/layer23_attention_weight_min": -22.571971893310547,
      "activations/layer2_attention_weight_max": 31.597797393798828,
      "activations/layer2_attention_weight_min": -29.77707290649414,
      "activations/layer3_attention_weight_max": 83.5823745727539,
      "activations/layer3_attention_weight_min": -87.62639617919922,
      "activations/layer4_attention_weight_max": 99.35835266113281,
      "activations/layer4_attention_weight_min": -89.89155578613281,
      "activations/layer5_attention_weight_max": 89.74546813964844,
      "activations/layer5_attention_weight_min": -82.23046875,
      "activations/layer6_attention_weight_max": 57.607704162597656,
      "activations/layer6_attention_weight_min": -50.05735397338867,
      "activations/layer7_attention_weight_max": 73.54008483886719,
      "activations/layer7_attention_weight_min": -71.01969909667969,
      "activations/layer8_attention_weight_max": 45.015018463134766,
      "activations/layer8_attention_weight_min": -44.16561508178711,
      "activations/layer9_attention_weight_max": 50.021484375,
      "activations/layer9_attention_weight_min": -46.603294372558594,
      "epoch": 7.33,
      "learning_rate": 0.00010375643939393938,
      "loss": 2.8262,
      "step": 126200
    },
    {
      "activations/layer0_attention_weight_max": 15.197134971618652,
      "activations/layer0_attention_weight_min": -14.782812118530273,
      "activations/layer10_attention_weight_max": 41.32906723022461,
      "activations/layer10_attention_weight_min": -37.51189041137695,
      "activations/layer11_attention_weight_max": 31.98414421081543,
      "activations/layer11_attention_weight_min": -32.23820114135742,
      "activations/layer12_attention_weight_max": 23.32102394104004,
      "activations/layer12_attention_weight_min": -24.11150550842285,
      "activations/layer13_attention_weight_max": 46.55175018310547,
      "activations/layer13_attention_weight_min": -34.909297943115234,
      "activations/layer14_attention_weight_max": 45.32377624511719,
      "activations/layer14_attention_weight_min": -33.055145263671875,
      "activations/layer15_attention_weight_max": 35.736629486083984,
      "activations/layer15_attention_weight_min": -31.597322463989258,
      "activations/layer16_attention_weight_max": 34.46477508544922,
      "activations/layer16_attention_weight_min": -27.79950523376465,
      "activations/layer17_attention_weight_max": 59.539794921875,
      "activations/layer17_attention_weight_min": -46.19927978515625,
      "activations/layer18_attention_weight_max": 51.609222412109375,
      "activations/layer18_attention_weight_min": -37.69774627685547,
      "activations/layer19_attention_weight_max": 23.632722854614258,
      "activations/layer19_attention_weight_min": -20.21433448791504,
      "activations/layer1_attention_weight_max": 17.144559860229492,
      "activations/layer1_attention_weight_min": -16.864789962768555,
      "activations/layer20_attention_weight_max": 24.311220169067383,
      "activations/layer20_attention_weight_min": -23.530502319335938,
      "activations/layer21_attention_weight_max": 43.05085372924805,
      "activations/layer21_attention_weight_min": -26.10744285583496,
      "activations/layer22_attention_weight_max": 39.41067123413086,
      "activations/layer22_attention_weight_min": -26.26679801940918,
      "activations/layer23_attention_weight_max": 42.82587432861328,
      "activations/layer23_attention_weight_min": -20.643436431884766,
      "activations/layer2_attention_weight_max": 31.606870651245117,
      "activations/layer2_attention_weight_min": -29.869428634643555,
      "activations/layer3_attention_weight_max": 84.72199249267578,
      "activations/layer3_attention_weight_min": -91.0878677368164,
      "activations/layer4_attention_weight_max": 99.41952514648438,
      "activations/layer4_attention_weight_min": -94.52107238769531,
      "activations/layer5_attention_weight_max": 92.98567962646484,
      "activations/layer5_attention_weight_min": -83.92578887939453,
      "activations/layer6_attention_weight_max": 57.674644470214844,
      "activations/layer6_attention_weight_min": -52.37812423706055,
      "activations/layer7_attention_weight_max": 79.8718490600586,
      "activations/layer7_attention_weight_min": -72.66959381103516,
      "activations/layer8_attention_weight_max": 47.829410552978516,
      "activations/layer8_attention_weight_min": -45.21572494506836,
      "activations/layer9_attention_weight_max": 54.47388458251953,
      "activations/layer9_attention_weight_min": -51.22304916381836,
      "epoch": 7.34,
      "learning_rate": 0.00010373787878787878,
      "loss": 2.819,
      "step": 126250
    },
    {
      "activations/layer0_attention_weight_max": 16.061885833740234,
      "activations/layer0_attention_weight_min": -15.018658638000488,
      "activations/layer10_attention_weight_max": 33.71512222290039,
      "activations/layer10_attention_weight_min": -34.252166748046875,
      "activations/layer11_attention_weight_max": 29.574033737182617,
      "activations/layer11_attention_weight_min": -28.345653533935547,
      "activations/layer12_attention_weight_max": 21.161991119384766,
      "activations/layer12_attention_weight_min": -23.20445442199707,
      "activations/layer13_attention_weight_max": 40.03744125366211,
      "activations/layer13_attention_weight_min": -31.648889541625977,
      "activations/layer14_attention_weight_max": 35.543365478515625,
      "activations/layer14_attention_weight_min": -31.909011840820312,
      "activations/layer15_attention_weight_max": 31.410917282104492,
      "activations/layer15_attention_weight_min": -29.81973648071289,
      "activations/layer16_attention_weight_max": 32.72905731201172,
      "activations/layer16_attention_weight_min": -26.288103103637695,
      "activations/layer17_attention_weight_max": 52.6416015625,
      "activations/layer17_attention_weight_min": -40.260581970214844,
      "activations/layer18_attention_weight_max": 46.60646438598633,
      "activations/layer18_attention_weight_min": -34.557491302490234,
      "activations/layer19_attention_weight_max": 20.289947509765625,
      "activations/layer19_attention_weight_min": -19.866605758666992,
      "activations/layer1_attention_weight_max": 17.54844093322754,
      "activations/layer1_attention_weight_min": -16.386844635009766,
      "activations/layer20_attention_weight_max": 19.795198440551758,
      "activations/layer20_attention_weight_min": -23.310625076293945,
      "activations/layer21_attention_weight_max": 38.68043899536133,
      "activations/layer21_attention_weight_min": -24.76416015625,
      "activations/layer22_attention_weight_max": 31.84522247314453,
      "activations/layer22_attention_weight_min": -25.17876434326172,
      "activations/layer23_attention_weight_max": 31.281421661376953,
      "activations/layer23_attention_weight_min": -20.5430965423584,
      "activations/layer2_attention_weight_max": 32.572654724121094,
      "activations/layer2_attention_weight_min": -30.83951187133789,
      "activations/layer3_attention_weight_max": 86.25859832763672,
      "activations/layer3_attention_weight_min": -87.49188995361328,
      "activations/layer4_attention_weight_max": 94.0530014038086,
      "activations/layer4_attention_weight_min": -89.58682250976562,
      "activations/layer5_attention_weight_max": 85.47491455078125,
      "activations/layer5_attention_weight_min": -76.74962615966797,
      "activations/layer6_attention_weight_max": 55.19703674316406,
      "activations/layer6_attention_weight_min": -49.71359634399414,
      "activations/layer7_attention_weight_max": 73.75202941894531,
      "activations/layer7_attention_weight_min": -66.18655395507812,
      "activations/layer8_attention_weight_max": 43.913082122802734,
      "activations/layer8_attention_weight_min": -43.45430374145508,
      "activations/layer9_attention_weight_max": 49.35316848754883,
      "activations/layer9_attention_weight_min": -45.59469223022461,
      "epoch": 7.34,
      "learning_rate": 0.00010371893939393938,
      "loss": 2.8299,
      "step": 126300
    },
    {
      "activations/layer0_attention_weight_max": 14.757247924804688,
      "activations/layer0_attention_weight_min": -14.786919593811035,
      "activations/layer10_attention_weight_max": 39.77085876464844,
      "activations/layer10_attention_weight_min": -37.13437271118164,
      "activations/layer11_attention_weight_max": 33.50981140136719,
      "activations/layer11_attention_weight_min": -32.24116897583008,
      "activations/layer12_attention_weight_max": 21.043325424194336,
      "activations/layer12_attention_weight_min": -26.031612396240234,
      "activations/layer13_attention_weight_max": 62.11128616333008,
      "activations/layer13_attention_weight_min": -34.27448654174805,
      "activations/layer14_attention_weight_max": 76.69452667236328,
      "activations/layer14_attention_weight_min": -38.042388916015625,
      "activations/layer15_attention_weight_max": 61.52016830444336,
      "activations/layer15_attention_weight_min": -31.70982551574707,
      "activations/layer16_attention_weight_max": 39.009796142578125,
      "activations/layer16_attention_weight_min": -27.350723266601562,
      "activations/layer17_attention_weight_max": 60.44490432739258,
      "activations/layer17_attention_weight_min": -44.730072021484375,
      "activations/layer18_attention_weight_max": 51.83827590942383,
      "activations/layer18_attention_weight_min": -40.30341339111328,
      "activations/layer19_attention_weight_max": 25.58949851989746,
      "activations/layer19_attention_weight_min": -22.266605377197266,
      "activations/layer1_attention_weight_max": 17.0228214263916,
      "activations/layer1_attention_weight_min": -15.659441947937012,
      "activations/layer20_attention_weight_max": 23.321916580200195,
      "activations/layer20_attention_weight_min": -22.02042579650879,
      "activations/layer21_attention_weight_max": 50.138545989990234,
      "activations/layer21_attention_weight_min": -25.241453170776367,
      "activations/layer22_attention_weight_max": 37.40801239013672,
      "activations/layer22_attention_weight_min": -28.29484748840332,
      "activations/layer23_attention_weight_max": 37.27122497558594,
      "activations/layer23_attention_weight_min": -20.107810974121094,
      "activations/layer2_attention_weight_max": 34.075477600097656,
      "activations/layer2_attention_weight_min": -30.665172576904297,
      "activations/layer3_attention_weight_max": 90.970703125,
      "activations/layer3_attention_weight_min": -91.87126159667969,
      "activations/layer4_attention_weight_max": 106.78531646728516,
      "activations/layer4_attention_weight_min": -105.24605560302734,
      "activations/layer5_attention_weight_max": 97.64366149902344,
      "activations/layer5_attention_weight_min": -98.27272033691406,
      "activations/layer6_attention_weight_max": 60.461490631103516,
      "activations/layer6_attention_weight_min": -56.071434020996094,
      "activations/layer7_attention_weight_max": 83.59827423095703,
      "activations/layer7_attention_weight_min": -73.89250183105469,
      "activations/layer8_attention_weight_max": 47.72999572753906,
      "activations/layer8_attention_weight_min": -48.81037139892578,
      "activations/layer9_attention_weight_max": 61.32936477661133,
      "activations/layer9_attention_weight_min": -52.739437103271484,
      "epoch": 7.34,
      "learning_rate": 0.0001037,
      "loss": 2.8284,
      "step": 126350
    },
    {
      "activations/layer0_attention_weight_max": 13.863378524780273,
      "activations/layer0_attention_weight_min": -14.683399200439453,
      "activations/layer10_attention_weight_max": 43.07125473022461,
      "activations/layer10_attention_weight_min": -39.354881286621094,
      "activations/layer11_attention_weight_max": 33.742610931396484,
      "activations/layer11_attention_weight_min": -33.67917251586914,
      "activations/layer12_attention_weight_max": 22.40721321105957,
      "activations/layer12_attention_weight_min": -22.9438533782959,
      "activations/layer13_attention_weight_max": 44.4754753112793,
      "activations/layer13_attention_weight_min": -35.483642578125,
      "activations/layer14_attention_weight_max": 39.4715461730957,
      "activations/layer14_attention_weight_min": -30.895980834960938,
      "activations/layer15_attention_weight_max": 36.48674392700195,
      "activations/layer15_attention_weight_min": -29.603092193603516,
      "activations/layer16_attention_weight_max": 31.76792335510254,
      "activations/layer16_attention_weight_min": -26.939050674438477,
      "activations/layer17_attention_weight_max": 52.20050811767578,
      "activations/layer17_attention_weight_min": -42.27452850341797,
      "activations/layer18_attention_weight_max": 49.561981201171875,
      "activations/layer18_attention_weight_min": -36.0189323425293,
      "activations/layer19_attention_weight_max": 26.076217651367188,
      "activations/layer19_attention_weight_min": -20.834903717041016,
      "activations/layer1_attention_weight_max": 18.268415451049805,
      "activations/layer1_attention_weight_min": -15.241789817810059,
      "activations/layer20_attention_weight_max": 23.4552059173584,
      "activations/layer20_attention_weight_min": -23.098033905029297,
      "activations/layer21_attention_weight_max": 43.24567794799805,
      "activations/layer21_attention_weight_min": -24.370351791381836,
      "activations/layer22_attention_weight_max": 35.4713020324707,
      "activations/layer22_attention_weight_min": -23.888778686523438,
      "activations/layer23_attention_weight_max": 36.78472137451172,
      "activations/layer23_attention_weight_min": -23.117294311523438,
      "activations/layer2_attention_weight_max": 32.515098571777344,
      "activations/layer2_attention_weight_min": -29.240575790405273,
      "activations/layer3_attention_weight_max": 89.8288803100586,
      "activations/layer3_attention_weight_min": -92.65042114257812,
      "activations/layer4_attention_weight_max": 110.42304992675781,
      "activations/layer4_attention_weight_min": -101.7105941772461,
      "activations/layer5_attention_weight_max": 98.28962707519531,
      "activations/layer5_attention_weight_min": -88.21260070800781,
      "activations/layer6_attention_weight_max": 59.74885940551758,
      "activations/layer6_attention_weight_min": -57.834571838378906,
      "activations/layer7_attention_weight_max": 81.92552185058594,
      "activations/layer7_attention_weight_min": -76.37238311767578,
      "activations/layer8_attention_weight_max": 49.4372673034668,
      "activations/layer8_attention_weight_min": -50.18644332885742,
      "activations/layer9_attention_weight_max": 57.103485107421875,
      "activations/layer9_attention_weight_min": -58.0582389831543,
      "epoch": 7.34,
      "learning_rate": 0.00010368106060606059,
      "loss": 2.8286,
      "step": 126400
    },
    {
      "activations/layer0_attention_weight_max": 15.332184791564941,
      "activations/layer0_attention_weight_min": -14.96279525756836,
      "activations/layer10_attention_weight_max": 39.547481536865234,
      "activations/layer10_attention_weight_min": -35.82075119018555,
      "activations/layer11_attention_weight_max": 33.16484832763672,
      "activations/layer11_attention_weight_min": -32.261348724365234,
      "activations/layer12_attention_weight_max": 17.586856842041016,
      "activations/layer12_attention_weight_min": -27.82444953918457,
      "activations/layer13_attention_weight_max": 37.963645935058594,
      "activations/layer13_attention_weight_min": -30.78139877319336,
      "activations/layer14_attention_weight_max": 35.70359802246094,
      "activations/layer14_attention_weight_min": -30.03803062438965,
      "activations/layer15_attention_weight_max": 35.5081787109375,
      "activations/layer15_attention_weight_min": -30.186054229736328,
      "activations/layer16_attention_weight_max": 33.202884674072266,
      "activations/layer16_attention_weight_min": -27.488624572753906,
      "activations/layer17_attention_weight_max": 55.41917037963867,
      "activations/layer17_attention_weight_min": -43.380741119384766,
      "activations/layer18_attention_weight_max": 48.182064056396484,
      "activations/layer18_attention_weight_min": -37.62898635864258,
      "activations/layer19_attention_weight_max": 22.317216873168945,
      "activations/layer19_attention_weight_min": -21.890962600708008,
      "activations/layer1_attention_weight_max": 16.473064422607422,
      "activations/layer1_attention_weight_min": -14.910186767578125,
      "activations/layer20_attention_weight_max": 23.312673568725586,
      "activations/layer20_attention_weight_min": -22.468978881835938,
      "activations/layer21_attention_weight_max": 32.22644805908203,
      "activations/layer21_attention_weight_min": -26.272014617919922,
      "activations/layer22_attention_weight_max": 32.90629577636719,
      "activations/layer22_attention_weight_min": -25.83184051513672,
      "activations/layer23_attention_weight_max": 32.87115478515625,
      "activations/layer23_attention_weight_min": -20.76453399658203,
      "activations/layer2_attention_weight_max": 32.366119384765625,
      "activations/layer2_attention_weight_min": -30.319854736328125,
      "activations/layer3_attention_weight_max": 89.2988510131836,
      "activations/layer3_attention_weight_min": -93.2830581665039,
      "activations/layer4_attention_weight_max": 98.66168212890625,
      "activations/layer4_attention_weight_min": -95.726318359375,
      "activations/layer5_attention_weight_max": 90.33403778076172,
      "activations/layer5_attention_weight_min": -83.72911834716797,
      "activations/layer6_attention_weight_max": 58.13978576660156,
      "activations/layer6_attention_weight_min": -53.31968688964844,
      "activations/layer7_attention_weight_max": 76.10737609863281,
      "activations/layer7_attention_weight_min": -72.49662017822266,
      "activations/layer8_attention_weight_max": 46.85792922973633,
      "activations/layer8_attention_weight_min": -45.92045593261719,
      "activations/layer9_attention_weight_max": 60.39486312866211,
      "activations/layer9_attention_weight_min": -50.27579116821289,
      "epoch": 7.35,
      "learning_rate": 0.0001036621212121212,
      "loss": 2.8074,
      "step": 126450
    },
    {
      "activations/layer0_attention_weight_max": 14.780965805053711,
      "activations/layer0_attention_weight_min": -14.877989768981934,
      "activations/layer10_attention_weight_max": 37.32037353515625,
      "activations/layer10_attention_weight_min": -35.26026153564453,
      "activations/layer11_attention_weight_max": 29.77292251586914,
      "activations/layer11_attention_weight_min": -30.229576110839844,
      "activations/layer12_attention_weight_max": 19.929656982421875,
      "activations/layer12_attention_weight_min": -25.692121505737305,
      "activations/layer13_attention_weight_max": 40.102638244628906,
      "activations/layer13_attention_weight_min": -29.104534149169922,
      "activations/layer14_attention_weight_max": 40.754722595214844,
      "activations/layer14_attention_weight_min": -30.071975708007812,
      "activations/layer15_attention_weight_max": 36.47948455810547,
      "activations/layer15_attention_weight_min": -29.8317813873291,
      "activations/layer16_attention_weight_max": 33.5826416015625,
      "activations/layer16_attention_weight_min": -27.252424240112305,
      "activations/layer17_attention_weight_max": 53.02042770385742,
      "activations/layer17_attention_weight_min": -41.42933654785156,
      "activations/layer18_attention_weight_max": 46.98225021362305,
      "activations/layer18_attention_weight_min": -35.17417526245117,
      "activations/layer19_attention_weight_max": 22.631872177124023,
      "activations/layer19_attention_weight_min": -20.752477645874023,
      "activations/layer1_attention_weight_max": 17.30606460571289,
      "activations/layer1_attention_weight_min": -17.47789764404297,
      "activations/layer20_attention_weight_max": 22.23711585998535,
      "activations/layer20_attention_weight_min": -22.031028747558594,
      "activations/layer21_attention_weight_max": 40.69489288330078,
      "activations/layer21_attention_weight_min": -24.093042373657227,
      "activations/layer22_attention_weight_max": 31.92290496826172,
      "activations/layer22_attention_weight_min": -24.22362518310547,
      "activations/layer23_attention_weight_max": 32.339351654052734,
      "activations/layer23_attention_weight_min": -21.84718894958496,
      "activations/layer2_attention_weight_max": 31.37153434753418,
      "activations/layer2_attention_weight_min": -29.872251510620117,
      "activations/layer3_attention_weight_max": 86.74627685546875,
      "activations/layer3_attention_weight_min": -89.8127212524414,
      "activations/layer4_attention_weight_max": 104.1330795288086,
      "activations/layer4_attention_weight_min": -94.8218002319336,
      "activations/layer5_attention_weight_max": 92.7672119140625,
      "activations/layer5_attention_weight_min": -87.67280578613281,
      "activations/layer6_attention_weight_max": 58.95334243774414,
      "activations/layer6_attention_weight_min": -54.245384216308594,
      "activations/layer7_attention_weight_max": 77.51126861572266,
      "activations/layer7_attention_weight_min": -73.02113342285156,
      "activations/layer8_attention_weight_max": 45.10637283325195,
      "activations/layer8_attention_weight_min": -45.763336181640625,
      "activations/layer9_attention_weight_max": 52.036773681640625,
      "activations/layer9_attention_weight_min": -47.23448181152344,
      "epoch": 7.35,
      "learning_rate": 0.00010364318181818182,
      "loss": 2.8307,
      "step": 126500
    },
    {
      "activations/layer0_attention_weight_max": 14.761055946350098,
      "activations/layer0_attention_weight_min": -14.832514762878418,
      "activations/layer10_attention_weight_max": 36.13374710083008,
      "activations/layer10_attention_weight_min": -34.1763916015625,
      "activations/layer11_attention_weight_max": 32.179222106933594,
      "activations/layer11_attention_weight_min": -30.998031616210938,
      "activations/layer12_attention_weight_max": 17.833051681518555,
      "activations/layer12_attention_weight_min": -27.519392013549805,
      "activations/layer13_attention_weight_max": 38.367820739746094,
      "activations/layer13_attention_weight_min": -28.145334243774414,
      "activations/layer14_attention_weight_max": 41.768741607666016,
      "activations/layer14_attention_weight_min": -27.83820915222168,
      "activations/layer15_attention_weight_max": 34.53203582763672,
      "activations/layer15_attention_weight_min": -28.74390983581543,
      "activations/layer16_attention_weight_max": 31.428922653198242,
      "activations/layer16_attention_weight_min": -27.47552490234375,
      "activations/layer17_attention_weight_max": 55.334922790527344,
      "activations/layer17_attention_weight_min": -40.87702941894531,
      "activations/layer18_attention_weight_max": 45.22797775268555,
      "activations/layer18_attention_weight_min": -34.04695129394531,
      "activations/layer19_attention_weight_max": 21.70768165588379,
      "activations/layer19_attention_weight_min": -22.140487670898438,
      "activations/layer1_attention_weight_max": 17.548891067504883,
      "activations/layer1_attention_weight_min": -14.787599563598633,
      "activations/layer20_attention_weight_max": 23.226919174194336,
      "activations/layer20_attention_weight_min": -21.157155990600586,
      "activations/layer21_attention_weight_max": 47.87101745605469,
      "activations/layer21_attention_weight_min": -27.25810432434082,
      "activations/layer22_attention_weight_max": 31.73478126525879,
      "activations/layer22_attention_weight_min": -24.179035186767578,
      "activations/layer23_attention_weight_max": 32.35346603393555,
      "activations/layer23_attention_weight_min": -20.442550659179688,
      "activations/layer2_attention_weight_max": 29.697385787963867,
      "activations/layer2_attention_weight_min": -28.79651641845703,
      "activations/layer3_attention_weight_max": 84.27820587158203,
      "activations/layer3_attention_weight_min": -84.18936920166016,
      "activations/layer4_attention_weight_max": 97.1081771850586,
      "activations/layer4_attention_weight_min": -87.85930633544922,
      "activations/layer5_attention_weight_max": 87.79354095458984,
      "activations/layer5_attention_weight_min": -76.59298706054688,
      "activations/layer6_attention_weight_max": 52.81269073486328,
      "activations/layer6_attention_weight_min": -49.16434097290039,
      "activations/layer7_attention_weight_max": 74.30083465576172,
      "activations/layer7_attention_weight_min": -65.33649444580078,
      "activations/layer8_attention_weight_max": 41.62651824951172,
      "activations/layer8_attention_weight_min": -42.64882278442383,
      "activations/layer9_attention_weight_max": 57.242733001708984,
      "activations/layer9_attention_weight_min": -53.550148010253906,
      "epoch": 7.35,
      "learning_rate": 0.0001036242424242424,
      "loss": 2.8358,
      "step": 126550
    },
    {
      "activations/layer0_attention_weight_max": 14.629769325256348,
      "activations/layer0_attention_weight_min": -14.650104522705078,
      "activations/layer10_attention_weight_max": 36.976314544677734,
      "activations/layer10_attention_weight_min": -34.74888610839844,
      "activations/layer11_attention_weight_max": 29.661109924316406,
      "activations/layer11_attention_weight_min": -30.188873291015625,
      "activations/layer12_attention_weight_max": 20.404932022094727,
      "activations/layer12_attention_weight_min": -21.847848892211914,
      "activations/layer13_attention_weight_max": 47.80351638793945,
      "activations/layer13_attention_weight_min": -35.644649505615234,
      "activations/layer14_attention_weight_max": 54.79551315307617,
      "activations/layer14_attention_weight_min": -34.56963348388672,
      "activations/layer15_attention_weight_max": 43.16075897216797,
      "activations/layer15_attention_weight_min": -28.729793548583984,
      "activations/layer16_attention_weight_max": 31.286617279052734,
      "activations/layer16_attention_weight_min": -26.16801643371582,
      "activations/layer17_attention_weight_max": 52.29726791381836,
      "activations/layer17_attention_weight_min": -40.41615676879883,
      "activations/layer18_attention_weight_max": 47.16963195800781,
      "activations/layer18_attention_weight_min": -35.597015380859375,
      "activations/layer19_attention_weight_max": 21.669958114624023,
      "activations/layer19_attention_weight_min": -19.436845779418945,
      "activations/layer1_attention_weight_max": 17.07818603515625,
      "activations/layer1_attention_weight_min": -15.052140235900879,
      "activations/layer20_attention_weight_max": 20.763944625854492,
      "activations/layer20_attention_weight_min": -21.148408889770508,
      "activations/layer21_attention_weight_max": 33.80198669433594,
      "activations/layer21_attention_weight_min": -22.956161499023438,
      "activations/layer22_attention_weight_max": 33.42766189575195,
      "activations/layer22_attention_weight_min": -27.671724319458008,
      "activations/layer23_attention_weight_max": 36.71040344238281,
      "activations/layer23_attention_weight_min": -22.99310302734375,
      "activations/layer2_attention_weight_max": 32.294677734375,
      "activations/layer2_attention_weight_min": -29.002981185913086,
      "activations/layer3_attention_weight_max": 82.73370361328125,
      "activations/layer3_attention_weight_min": -82.00605773925781,
      "activations/layer4_attention_weight_max": 98.24346160888672,
      "activations/layer4_attention_weight_min": -92.57075500488281,
      "activations/layer5_attention_weight_max": 86.69657897949219,
      "activations/layer5_attention_weight_min": -83.29020690917969,
      "activations/layer6_attention_weight_max": 54.495243072509766,
      "activations/layer6_attention_weight_min": -51.57277297973633,
      "activations/layer7_attention_weight_max": 75.4822769165039,
      "activations/layer7_attention_weight_min": -71.19107818603516,
      "activations/layer8_attention_weight_max": 44.88253402709961,
      "activations/layer8_attention_weight_min": -44.42479705810547,
      "activations/layer9_attention_weight_max": 51.157508850097656,
      "activations/layer9_attention_weight_min": -49.30491256713867,
      "epoch": 7.36,
      "learning_rate": 0.0001036056818181818,
      "loss": 2.7893,
      "step": 126600
    },
    {
      "activations/layer0_attention_weight_max": 16.197208404541016,
      "activations/layer0_attention_weight_min": -14.957296371459961,
      "activations/layer10_attention_weight_max": 43.3517951965332,
      "activations/layer10_attention_weight_min": -36.258235931396484,
      "activations/layer11_attention_weight_max": 33.018310546875,
      "activations/layer11_attention_weight_min": -31.613941192626953,
      "activations/layer12_attention_weight_max": 20.76594352722168,
      "activations/layer12_attention_weight_min": -22.29990577697754,
      "activations/layer13_attention_weight_max": 61.65823745727539,
      "activations/layer13_attention_weight_min": -33.39015197753906,
      "activations/layer14_attention_weight_max": 70.06119537353516,
      "activations/layer14_attention_weight_min": -37.14106750488281,
      "activations/layer15_attention_weight_max": 53.28115463256836,
      "activations/layer15_attention_weight_min": -29.196027755737305,
      "activations/layer16_attention_weight_max": 37.770565032958984,
      "activations/layer16_attention_weight_min": -28.372568130493164,
      "activations/layer17_attention_weight_max": 62.00760269165039,
      "activations/layer17_attention_weight_min": -43.66563415527344,
      "activations/layer18_attention_weight_max": 56.05590057373047,
      "activations/layer18_attention_weight_min": -39.86629104614258,
      "activations/layer19_attention_weight_max": 23.68935203552246,
      "activations/layer19_attention_weight_min": -21.349348068237305,
      "activations/layer1_attention_weight_max": 16.243751525878906,
      "activations/layer1_attention_weight_min": -14.212442398071289,
      "activations/layer20_attention_weight_max": 24.307836532592773,
      "activations/layer20_attention_weight_min": -21.272626876831055,
      "activations/layer21_attention_weight_max": 50.32376480102539,
      "activations/layer21_attention_weight_min": -27.773502349853516,
      "activations/layer22_attention_weight_max": 41.98145294189453,
      "activations/layer22_attention_weight_min": -29.60866355895996,
      "activations/layer23_attention_weight_max": 38.56117248535156,
      "activations/layer23_attention_weight_min": -20.265907287597656,
      "activations/layer2_attention_weight_max": 30.26533317565918,
      "activations/layer2_attention_weight_min": -27.827558517456055,
      "activations/layer3_attention_weight_max": 83.16873168945312,
      "activations/layer3_attention_weight_min": -85.91310119628906,
      "activations/layer4_attention_weight_max": 95.50920867919922,
      "activations/layer4_attention_weight_min": -91.3200912475586,
      "activations/layer5_attention_weight_max": 85.68893432617188,
      "activations/layer5_attention_weight_min": -79.42491149902344,
      "activations/layer6_attention_weight_max": 54.02657699584961,
      "activations/layer6_attention_weight_min": -52.91560363769531,
      "activations/layer7_attention_weight_max": 76.44571685791016,
      "activations/layer7_attention_weight_min": -69.12432861328125,
      "activations/layer8_attention_weight_max": 44.16699981689453,
      "activations/layer8_attention_weight_min": -46.95793914794922,
      "activations/layer9_attention_weight_max": 58.08102035522461,
      "activations/layer9_attention_weight_min": -50.63965606689453,
      "epoch": 7.36,
      "learning_rate": 0.00010358674242424241,
      "loss": 2.8188,
      "step": 126650
    },
    {
      "activations/layer0_attention_weight_max": 14.686738967895508,
      "activations/layer0_attention_weight_min": -15.033394813537598,
      "activations/layer10_attention_weight_max": 48.3614616394043,
      "activations/layer10_attention_weight_min": -43.49066162109375,
      "activations/layer11_attention_weight_max": 39.7937126159668,
      "activations/layer11_attention_weight_min": -34.506561279296875,
      "activations/layer12_attention_weight_max": 28.227500915527344,
      "activations/layer12_attention_weight_min": -22.866289138793945,
      "activations/layer13_attention_weight_max": 49.23250198364258,
      "activations/layer13_attention_weight_min": -33.231502532958984,
      "activations/layer14_attention_weight_max": 51.3354606628418,
      "activations/layer14_attention_weight_min": -33.044715881347656,
      "activations/layer15_attention_weight_max": 40.80006408691406,
      "activations/layer15_attention_weight_min": -31.596332550048828,
      "activations/layer16_attention_weight_max": 36.3234977722168,
      "activations/layer16_attention_weight_min": -29.593374252319336,
      "activations/layer17_attention_weight_max": 60.74916458129883,
      "activations/layer17_attention_weight_min": -45.711212158203125,
      "activations/layer18_attention_weight_max": 56.23221206665039,
      "activations/layer18_attention_weight_min": -38.722320556640625,
      "activations/layer19_attention_weight_max": 23.29745864868164,
      "activations/layer19_attention_weight_min": -22.486827850341797,
      "activations/layer1_attention_weight_max": 17.725914001464844,
      "activations/layer1_attention_weight_min": -15.845267295837402,
      "activations/layer20_attention_weight_max": 26.271581649780273,
      "activations/layer20_attention_weight_min": -23.453643798828125,
      "activations/layer21_attention_weight_max": 45.539588928222656,
      "activations/layer21_attention_weight_min": -22.711524963378906,
      "activations/layer22_attention_weight_max": 34.90144729614258,
      "activations/layer22_attention_weight_min": -24.860416412353516,
      "activations/layer23_attention_weight_max": 38.6876335144043,
      "activations/layer23_attention_weight_min": -22.981098175048828,
      "activations/layer2_attention_weight_max": 31.63324546813965,
      "activations/layer2_attention_weight_min": -29.684425354003906,
      "activations/layer3_attention_weight_max": 81.75037384033203,
      "activations/layer3_attention_weight_min": -86.0465087890625,
      "activations/layer4_attention_weight_max": 100.93294525146484,
      "activations/layer4_attention_weight_min": -91.27082061767578,
      "activations/layer5_attention_weight_max": 92.57008361816406,
      "activations/layer5_attention_weight_min": -81.47057342529297,
      "activations/layer6_attention_weight_max": 58.95073318481445,
      "activations/layer6_attention_weight_min": -52.35170364379883,
      "activations/layer7_attention_weight_max": 85.3987045288086,
      "activations/layer7_attention_weight_min": -74.92121887207031,
      "activations/layer8_attention_weight_max": 54.8447151184082,
      "activations/layer8_attention_weight_min": -48.244178771972656,
      "activations/layer9_attention_weight_max": 69.59820556640625,
      "activations/layer9_attention_weight_min": -56.93459701538086,
      "epoch": 7.36,
      "learning_rate": 0.00010356780303030303,
      "loss": 2.8157,
      "step": 126700
    },
    {
      "activations/layer0_attention_weight_max": 15.374886512756348,
      "activations/layer0_attention_weight_min": -14.901095390319824,
      "activations/layer10_attention_weight_max": 39.579322814941406,
      "activations/layer10_attention_weight_min": -33.83758544921875,
      "activations/layer11_attention_weight_max": 30.959651947021484,
      "activations/layer11_attention_weight_min": -31.695859909057617,
      "activations/layer12_attention_weight_max": 20.675373077392578,
      "activations/layer12_attention_weight_min": -22.917274475097656,
      "activations/layer13_attention_weight_max": 43.91014862060547,
      "activations/layer13_attention_weight_min": -31.199851989746094,
      "activations/layer14_attention_weight_max": 38.36321258544922,
      "activations/layer14_attention_weight_min": -32.54460906982422,
      "activations/layer15_attention_weight_max": 35.39287567138672,
      "activations/layer15_attention_weight_min": -29.811473846435547,
      "activations/layer16_attention_weight_max": 31.602123260498047,
      "activations/layer16_attention_weight_min": -28.19633674621582,
      "activations/layer17_attention_weight_max": 56.60539627075195,
      "activations/layer17_attention_weight_min": -42.1768913269043,
      "activations/layer18_attention_weight_max": 50.51625061035156,
      "activations/layer18_attention_weight_min": -39.37126922607422,
      "activations/layer19_attention_weight_max": 22.37647819519043,
      "activations/layer19_attention_weight_min": -21.97136688232422,
      "activations/layer1_attention_weight_max": 16.845014572143555,
      "activations/layer1_attention_weight_min": -15.921080589294434,
      "activations/layer20_attention_weight_max": 21.955766677856445,
      "activations/layer20_attention_weight_min": -21.06708526611328,
      "activations/layer21_attention_weight_max": 34.167686462402344,
      "activations/layer21_attention_weight_min": -26.479204177856445,
      "activations/layer22_attention_weight_max": 31.996763229370117,
      "activations/layer22_attention_weight_min": -25.717567443847656,
      "activations/layer23_attention_weight_max": 34.00578689575195,
      "activations/layer23_attention_weight_min": -25.384891510009766,
      "activations/layer2_attention_weight_max": 31.788265228271484,
      "activations/layer2_attention_weight_min": -31.124645233154297,
      "activations/layer3_attention_weight_max": 83.7205810546875,
      "activations/layer3_attention_weight_min": -88.50456237792969,
      "activations/layer4_attention_weight_max": 100.55609130859375,
      "activations/layer4_attention_weight_min": -94.52230072021484,
      "activations/layer5_attention_weight_max": 89.16537475585938,
      "activations/layer5_attention_weight_min": -86.3086166381836,
      "activations/layer6_attention_weight_max": 56.43259811401367,
      "activations/layer6_attention_weight_min": -54.26829528808594,
      "activations/layer7_attention_weight_max": 78.95430755615234,
      "activations/layer7_attention_weight_min": -74.08780670166016,
      "activations/layer8_attention_weight_max": 47.4882698059082,
      "activations/layer8_attention_weight_min": -47.815189361572266,
      "activations/layer9_attention_weight_max": 50.203243255615234,
      "activations/layer9_attention_weight_min": -48.15518569946289,
      "epoch": 7.36,
      "learning_rate": 0.00010354886363636363,
      "loss": 2.8159,
      "step": 126750
    },
    {
      "activations/layer0_attention_weight_max": 14.945563316345215,
      "activations/layer0_attention_weight_min": -14.339813232421875,
      "activations/layer10_attention_weight_max": 36.34749221801758,
      "activations/layer10_attention_weight_min": -36.37082290649414,
      "activations/layer11_attention_weight_max": 30.466598510742188,
      "activations/layer11_attention_weight_min": -30.93978500366211,
      "activations/layer12_attention_weight_max": 25.64183807373047,
      "activations/layer12_attention_weight_min": -24.565610885620117,
      "activations/layer13_attention_weight_max": 43.46781921386719,
      "activations/layer13_attention_weight_min": -31.312923431396484,
      "activations/layer14_attention_weight_max": 54.17177200317383,
      "activations/layer14_attention_weight_min": -34.47014236450195,
      "activations/layer15_attention_weight_max": 39.931007385253906,
      "activations/layer15_attention_weight_min": -31.605026245117188,
      "activations/layer16_attention_weight_max": 32.055484771728516,
      "activations/layer16_attention_weight_min": -29.309476852416992,
      "activations/layer17_attention_weight_max": 58.45558547973633,
      "activations/layer17_attention_weight_min": -44.538734436035156,
      "activations/layer18_attention_weight_max": 50.13400650024414,
      "activations/layer18_attention_weight_min": -37.61161804199219,
      "activations/layer19_attention_weight_max": 20.339984893798828,
      "activations/layer19_attention_weight_min": -19.9920711517334,
      "activations/layer1_attention_weight_max": 19.288755416870117,
      "activations/layer1_attention_weight_min": -15.181812286376953,
      "activations/layer20_attention_weight_max": 22.04912567138672,
      "activations/layer20_attention_weight_min": -21.408231735229492,
      "activations/layer21_attention_weight_max": 37.6229362487793,
      "activations/layer21_attention_weight_min": -24.573881149291992,
      "activations/layer22_attention_weight_max": 31.92774200439453,
      "activations/layer22_attention_weight_min": -23.820171356201172,
      "activations/layer23_attention_weight_max": 34.72395324707031,
      "activations/layer23_attention_weight_min": -24.31308364868164,
      "activations/layer2_attention_weight_max": 31.921829223632812,
      "activations/layer2_attention_weight_min": -30.495925903320312,
      "activations/layer3_attention_weight_max": 84.07504272460938,
      "activations/layer3_attention_weight_min": -89.30953216552734,
      "activations/layer4_attention_weight_max": 100.71214294433594,
      "activations/layer4_attention_weight_min": -96.17993927001953,
      "activations/layer5_attention_weight_max": 89.88005065917969,
      "activations/layer5_attention_weight_min": -85.79109191894531,
      "activations/layer6_attention_weight_max": 56.255943298339844,
      "activations/layer6_attention_weight_min": -53.57734298706055,
      "activations/layer7_attention_weight_max": 79.54450225830078,
      "activations/layer7_attention_weight_min": -72.24967193603516,
      "activations/layer8_attention_weight_max": 43.899192810058594,
      "activations/layer8_attention_weight_min": -46.50746536254883,
      "activations/layer9_attention_weight_max": 52.10023880004883,
      "activations/layer9_attention_weight_min": -47.798763275146484,
      "epoch": 7.37,
      "learning_rate": 0.00010352992424242423,
      "loss": 2.8035,
      "step": 126800
    },
    {
      "activations/layer0_attention_weight_max": 15.654061317443848,
      "activations/layer0_attention_weight_min": -15.000368118286133,
      "activations/layer10_attention_weight_max": 42.371070861816406,
      "activations/layer10_attention_weight_min": -38.6529426574707,
      "activations/layer11_attention_weight_max": 35.41565704345703,
      "activations/layer11_attention_weight_min": -33.212074279785156,
      "activations/layer12_attention_weight_max": 34.55150604248047,
      "activations/layer12_attention_weight_min": -25.5670166015625,
      "activations/layer13_attention_weight_max": 63.45297622680664,
      "activations/layer13_attention_weight_min": -41.76070785522461,
      "activations/layer14_attention_weight_max": 49.63310623168945,
      "activations/layer14_attention_weight_min": -32.07261657714844,
      "activations/layer15_attention_weight_max": 45.97837829589844,
      "activations/layer15_attention_weight_min": -30.26317596435547,
      "activations/layer16_attention_weight_max": 34.347145080566406,
      "activations/layer16_attention_weight_min": -28.330541610717773,
      "activations/layer17_attention_weight_max": 60.07808303833008,
      "activations/layer17_attention_weight_min": -47.326507568359375,
      "activations/layer18_attention_weight_max": 53.71840286254883,
      "activations/layer18_attention_weight_min": -38.864559173583984,
      "activations/layer19_attention_weight_max": 20.49251365661621,
      "activations/layer19_attention_weight_min": -20.980905532836914,
      "activations/layer1_attention_weight_max": 16.97169303894043,
      "activations/layer1_attention_weight_min": -15.378989219665527,
      "activations/layer20_attention_weight_max": 23.815523147583008,
      "activations/layer20_attention_weight_min": -24.341583251953125,
      "activations/layer21_attention_weight_max": 37.46845245361328,
      "activations/layer21_attention_weight_min": -22.729291915893555,
      "activations/layer22_attention_weight_max": 33.104190826416016,
      "activations/layer22_attention_weight_min": -25.38748550415039,
      "activations/layer23_attention_weight_max": 38.594398498535156,
      "activations/layer23_attention_weight_min": -22.708698272705078,
      "activations/layer2_attention_weight_max": 30.387449264526367,
      "activations/layer2_attention_weight_min": -28.7249813079834,
      "activations/layer3_attention_weight_max": 88.31856536865234,
      "activations/layer3_attention_weight_min": -85.49435424804688,
      "activations/layer4_attention_weight_max": 103.75139617919922,
      "activations/layer4_attention_weight_min": -95.02233123779297,
      "activations/layer5_attention_weight_max": 91.06809997558594,
      "activations/layer5_attention_weight_min": -86.13179016113281,
      "activations/layer6_attention_weight_max": 57.5389289855957,
      "activations/layer6_attention_weight_min": -56.626434326171875,
      "activations/layer7_attention_weight_max": 77.31568145751953,
      "activations/layer7_attention_weight_min": -74.63258361816406,
      "activations/layer8_attention_weight_max": 49.10551834106445,
      "activations/layer8_attention_weight_min": -48.8475227355957,
      "activations/layer9_attention_weight_max": 62.25238800048828,
      "activations/layer9_attention_weight_min": -51.292842864990234,
      "epoch": 7.37,
      "learning_rate": 0.00010351098484848484,
      "loss": 2.8091,
      "step": 126850
    },
    {
      "activations/layer0_attention_weight_max": 14.273797035217285,
      "activations/layer0_attention_weight_min": -14.960622787475586,
      "activations/layer10_attention_weight_max": 44.54751205444336,
      "activations/layer10_attention_weight_min": -41.073974609375,
      "activations/layer11_attention_weight_max": 38.64032745361328,
      "activations/layer11_attention_weight_min": -31.476119995117188,
      "activations/layer12_attention_weight_max": 23.3530216217041,
      "activations/layer12_attention_weight_min": -28.971738815307617,
      "activations/layer13_attention_weight_max": 42.68098449707031,
      "activations/layer13_attention_weight_min": -33.26996612548828,
      "activations/layer14_attention_weight_max": 46.837615966796875,
      "activations/layer14_attention_weight_min": -30.29494285583496,
      "activations/layer15_attention_weight_max": 43.88408660888672,
      "activations/layer15_attention_weight_min": -28.629085540771484,
      "activations/layer16_attention_weight_max": 32.83832550048828,
      "activations/layer16_attention_weight_min": -27.950176239013672,
      "activations/layer17_attention_weight_max": 54.96096420288086,
      "activations/layer17_attention_weight_min": -44.38906478881836,
      "activations/layer18_attention_weight_max": 48.68891906738281,
      "activations/layer18_attention_weight_min": -36.89268493652344,
      "activations/layer19_attention_weight_max": 22.45531463623047,
      "activations/layer19_attention_weight_min": -21.33356285095215,
      "activations/layer1_attention_weight_max": 18.83672523498535,
      "activations/layer1_attention_weight_min": -15.691910743713379,
      "activations/layer20_attention_weight_max": 23.4122314453125,
      "activations/layer20_attention_weight_min": -23.516658782958984,
      "activations/layer21_attention_weight_max": 35.87435531616211,
      "activations/layer21_attention_weight_min": -25.83555793762207,
      "activations/layer22_attention_weight_max": 33.20048904418945,
      "activations/layer22_attention_weight_min": -25.314247131347656,
      "activations/layer23_attention_weight_max": 34.54027557373047,
      "activations/layer23_attention_weight_min": -20.8572940826416,
      "activations/layer2_attention_weight_max": 34.55419921875,
      "activations/layer2_attention_weight_min": -32.020263671875,
      "activations/layer3_attention_weight_max": 91.9084243774414,
      "activations/layer3_attention_weight_min": -94.47406768798828,
      "activations/layer4_attention_weight_max": 112.41643524169922,
      "activations/layer4_attention_weight_min": -106.93746185302734,
      "activations/layer5_attention_weight_max": 98.42509460449219,
      "activations/layer5_attention_weight_min": -86.36305236816406,
      "activations/layer6_attention_weight_max": 61.680564880371094,
      "activations/layer6_attention_weight_min": -55.86613845825195,
      "activations/layer7_attention_weight_max": 94.89575958251953,
      "activations/layer7_attention_weight_min": -76.99620819091797,
      "activations/layer8_attention_weight_max": 55.98301315307617,
      "activations/layer8_attention_weight_min": -47.90974426269531,
      "activations/layer9_attention_weight_max": 72.61856079101562,
      "activations/layer9_attention_weight_min": -55.41497802734375,
      "epoch": 7.37,
      "learning_rate": 0.00010349204545454545,
      "loss": 2.8161,
      "step": 126900
    },
    {
      "activations/layer0_attention_weight_max": 15.919736862182617,
      "activations/layer0_attention_weight_min": -14.390193939208984,
      "activations/layer10_attention_weight_max": 39.16291809082031,
      "activations/layer10_attention_weight_min": -35.823974609375,
      "activations/layer11_attention_weight_max": 31.658668518066406,
      "activations/layer11_attention_weight_min": -29.864139556884766,
      "activations/layer12_attention_weight_max": 19.637142181396484,
      "activations/layer12_attention_weight_min": -25.598281860351562,
      "activations/layer13_attention_weight_max": 46.58070755004883,
      "activations/layer13_attention_weight_min": -29.341136932373047,
      "activations/layer14_attention_weight_max": 40.65656661987305,
      "activations/layer14_attention_weight_min": -30.396169662475586,
      "activations/layer15_attention_weight_max": 40.358123779296875,
      "activations/layer15_attention_weight_min": -29.883502960205078,
      "activations/layer16_attention_weight_max": 31.00714874267578,
      "activations/layer16_attention_weight_min": -26.360637664794922,
      "activations/layer17_attention_weight_max": 55.24519729614258,
      "activations/layer17_attention_weight_min": -43.69023895263672,
      "activations/layer18_attention_weight_max": 49.37796401977539,
      "activations/layer18_attention_weight_min": -37.46074676513672,
      "activations/layer19_attention_weight_max": 23.756933212280273,
      "activations/layer19_attention_weight_min": -20.183837890625,
      "activations/layer1_attention_weight_max": 17.102489471435547,
      "activations/layer1_attention_weight_min": -15.194933891296387,
      "activations/layer20_attention_weight_max": 24.589139938354492,
      "activations/layer20_attention_weight_min": -20.41020965576172,
      "activations/layer21_attention_weight_max": 42.346588134765625,
      "activations/layer21_attention_weight_min": -22.762704849243164,
      "activations/layer22_attention_weight_max": 35.730525970458984,
      "activations/layer22_attention_weight_min": -25.552656173706055,
      "activations/layer23_attention_weight_max": 34.25115966796875,
      "activations/layer23_attention_weight_min": -21.775938034057617,
      "activations/layer2_attention_weight_max": 34.20794677734375,
      "activations/layer2_attention_weight_min": -31.524747848510742,
      "activations/layer3_attention_weight_max": 89.8578872680664,
      "activations/layer3_attention_weight_min": -87.84770965576172,
      "activations/layer4_attention_weight_max": 104.14163970947266,
      "activations/layer4_attention_weight_min": -93.77788543701172,
      "activations/layer5_attention_weight_max": 92.0439682006836,
      "activations/layer5_attention_weight_min": -83.9251480102539,
      "activations/layer6_attention_weight_max": 56.87432098388672,
      "activations/layer6_attention_weight_min": -53.21128845214844,
      "activations/layer7_attention_weight_max": 75.15596008300781,
      "activations/layer7_attention_weight_min": -72.56486511230469,
      "activations/layer8_attention_weight_max": 46.52961730957031,
      "activations/layer8_attention_weight_min": -42.45866775512695,
      "activations/layer9_attention_weight_max": 48.90049743652344,
      "activations/layer9_attention_weight_min": -47.31985092163086,
      "epoch": 7.38,
      "learning_rate": 0.00010347310606060605,
      "loss": 2.8088,
      "step": 126950
    },
    {
      "activations/layer0_attention_weight_max": 15.479488372802734,
      "activations/layer0_attention_weight_min": -14.287531852722168,
      "activations/layer10_attention_weight_max": 37.28900146484375,
      "activations/layer10_attention_weight_min": -33.89623260498047,
      "activations/layer11_attention_weight_max": 30.233776092529297,
      "activations/layer11_attention_weight_min": -29.09341049194336,
      "activations/layer12_attention_weight_max": 19.985450744628906,
      "activations/layer12_attention_weight_min": -23.171911239624023,
      "activations/layer13_attention_weight_max": 38.73183822631836,
      "activations/layer13_attention_weight_min": -29.667240142822266,
      "activations/layer14_attention_weight_max": 38.19236755371094,
      "activations/layer14_attention_weight_min": -31.09975814819336,
      "activations/layer15_attention_weight_max": 34.2220344543457,
      "activations/layer15_attention_weight_min": -29.94497299194336,
      "activations/layer16_attention_weight_max": 29.705066680908203,
      "activations/layer16_attention_weight_min": -27.510297775268555,
      "activations/layer17_attention_weight_max": 52.501808166503906,
      "activations/layer17_attention_weight_min": -40.50886917114258,
      "activations/layer18_attention_weight_max": 47.72611999511719,
      "activations/layer18_attention_weight_min": -33.856781005859375,
      "activations/layer19_attention_weight_max": 21.212785720825195,
      "activations/layer19_attention_weight_min": -19.886489868164062,
      "activations/layer1_attention_weight_max": 17.28604507446289,
      "activations/layer1_attention_weight_min": -16.534452438354492,
      "activations/layer20_attention_weight_max": 20.727006912231445,
      "activations/layer20_attention_weight_min": -22.309337615966797,
      "activations/layer21_attention_weight_max": 36.05162811279297,
      "activations/layer21_attention_weight_min": -26.734024047851562,
      "activations/layer22_attention_weight_max": 30.800031661987305,
      "activations/layer22_attention_weight_min": -26.82475471496582,
      "activations/layer23_attention_weight_max": 34.467655181884766,
      "activations/layer23_attention_weight_min": -22.33055877685547,
      "activations/layer2_attention_weight_max": 30.741186141967773,
      "activations/layer2_attention_weight_min": -29.098318099975586,
      "activations/layer3_attention_weight_max": 81.98352813720703,
      "activations/layer3_attention_weight_min": -81.7038345336914,
      "activations/layer4_attention_weight_max": 96.6496810913086,
      "activations/layer4_attention_weight_min": -89.65288543701172,
      "activations/layer5_attention_weight_max": 85.25152587890625,
      "activations/layer5_attention_weight_min": -80.01351165771484,
      "activations/layer6_attention_weight_max": 55.71943664550781,
      "activations/layer6_attention_weight_min": -50.18085479736328,
      "activations/layer7_attention_weight_max": 75.07032775878906,
      "activations/layer7_attention_weight_min": -69.77207946777344,
      "activations/layer8_attention_weight_max": 43.425289154052734,
      "activations/layer8_attention_weight_min": -42.35865020751953,
      "activations/layer9_attention_weight_max": 48.55068588256836,
      "activations/layer9_attention_weight_min": -47.037757873535156,
      "epoch": 7.38,
      "learning_rate": 0.00010345416666666666,
      "loss": 2.81,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5457,
      "eval_samples_per_second": 502.477,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5457,
      "eval_openwebtext_samples_per_second": 502.477,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_wikitext_loss": 3.005859375,
      "eval_wikitext_ppl": 20.203571081597193,
      "eval_wikitext_runtime": 2.0088,
      "eval_wikitext_samples_per_second": 227.002,
      "step": 127000
    },
    {
      "epoch": 7.38,
      "eval_lambada_loss": 2.732421875,
      "eval_lambada_ppl": 15.370066359341997,
      "eval_lambada_runtime": 9.6023,
      "eval_lambada_samples_per_second": 507.065,
      "step": 127000
    },
    {
      "activations/layer0_attention_weight_max": 14.920516014099121,
      "activations/layer0_attention_weight_min": -14.771195411682129,
      "activations/layer10_attention_weight_max": 37.827423095703125,
      "activations/layer10_attention_weight_min": -36.692665100097656,
      "activations/layer11_attention_weight_max": 31.54352569580078,
      "activations/layer11_attention_weight_min": -30.924772262573242,
      "activations/layer12_attention_weight_max": 20.618345260620117,
      "activations/layer12_attention_weight_min": -23.854347229003906,
      "activations/layer13_attention_weight_max": 54.697269439697266,
      "activations/layer13_attention_weight_min": -37.64361572265625,
      "activations/layer14_attention_weight_max": 62.64363479614258,
      "activations/layer14_attention_weight_min": -34.70844268798828,
      "activations/layer15_attention_weight_max": 44.754913330078125,
      "activations/layer15_attention_weight_min": -29.376800537109375,
      "activations/layer16_attention_weight_max": 34.69015121459961,
      "activations/layer16_attention_weight_min": -28.146102905273438,
      "activations/layer17_attention_weight_max": 58.99153518676758,
      "activations/layer17_attention_weight_min": -43.174827575683594,
      "activations/layer18_attention_weight_max": 47.753929138183594,
      "activations/layer18_attention_weight_min": -37.70037078857422,
      "activations/layer19_attention_weight_max": 24.84688377380371,
      "activations/layer19_attention_weight_min": -21.87660789489746,
      "activations/layer1_attention_weight_max": 16.86859703063965,
      "activations/layer1_attention_weight_min": -14.353093147277832,
      "activations/layer20_attention_weight_max": 22.321491241455078,
      "activations/layer20_attention_weight_min": -22.29768943786621,
      "activations/layer21_attention_weight_max": 46.39509582519531,
      "activations/layer21_attention_weight_min": -24.130205154418945,
      "activations/layer22_attention_weight_max": 34.70648956298828,
      "activations/layer22_attention_weight_min": -25.98140525817871,
      "activations/layer23_attention_weight_max": 37.50994873046875,
      "activations/layer23_attention_weight_min": -21.882644653320312,
      "activations/layer2_attention_weight_max": 30.073200225830078,
      "activations/layer2_attention_weight_min": -29.480934143066406,
      "activations/layer3_attention_weight_max": 87.21810150146484,
      "activations/layer3_attention_weight_min": -83.94200897216797,
      "activations/layer4_attention_weight_max": 98.8730697631836,
      "activations/layer4_attention_weight_min": -92.84728240966797,
      "activations/layer5_attention_weight_max": 88.55876922607422,
      "activations/layer5_attention_weight_min": -82.11773681640625,
      "activations/layer6_attention_weight_max": 55.61268997192383,
      "activations/layer6_attention_weight_min": -53.005123138427734,
      "activations/layer7_attention_weight_max": 74.71654510498047,
      "activations/layer7_attention_weight_min": -69.85579681396484,
      "activations/layer8_attention_weight_max": 46.62779998779297,
      "activations/layer8_attention_weight_min": -45.636070251464844,
      "activations/layer9_attention_weight_max": 56.948280334472656,
      "activations/layer9_attention_weight_min": -48.66532897949219,
      "epoch": 7.38,
      "learning_rate": 0.00010343522727272726,
      "loss": 2.815,
      "step": 127050
    },
    {
      "activations/layer0_attention_weight_max": 14.261434555053711,
      "activations/layer0_attention_weight_min": -14.841577529907227,
      "activations/layer10_attention_weight_max": 41.33073806762695,
      "activations/layer10_attention_weight_min": -36.829933166503906,
      "activations/layer11_attention_weight_max": 33.88363265991211,
      "activations/layer11_attention_weight_min": -30.249412536621094,
      "activations/layer12_attention_weight_max": 28.054906845092773,
      "activations/layer12_attention_weight_min": -27.009254455566406,
      "activations/layer13_attention_weight_max": 45.27767562866211,
      "activations/layer13_attention_weight_min": -32.712425231933594,
      "activations/layer14_attention_weight_max": 37.747406005859375,
      "activations/layer14_attention_weight_min": -30.674041748046875,
      "activations/layer15_attention_weight_max": 34.86354446411133,
      "activations/layer15_attention_weight_min": -27.845855712890625,
      "activations/layer16_attention_weight_max": 33.77415084838867,
      "activations/layer16_attention_weight_min": -25.395183563232422,
      "activations/layer17_attention_weight_max": 52.69212341308594,
      "activations/layer17_attention_weight_min": -39.710716247558594,
      "activations/layer18_attention_weight_max": 48.87837219238281,
      "activations/layer18_attention_weight_min": -35.057281494140625,
      "activations/layer19_attention_weight_max": 20.846763610839844,
      "activations/layer19_attention_weight_min": -18.016441345214844,
      "activations/layer1_attention_weight_max": 16.921491622924805,
      "activations/layer1_attention_weight_min": -13.749691009521484,
      "activations/layer20_attention_weight_max": 20.74734878540039,
      "activations/layer20_attention_weight_min": -19.14354705810547,
      "activations/layer21_attention_weight_max": 32.986419677734375,
      "activations/layer21_attention_weight_min": -23.8905086517334,
      "activations/layer22_attention_weight_max": 31.726303100585938,
      "activations/layer22_attention_weight_min": -23.776466369628906,
      "activations/layer23_attention_weight_max": 33.776519775390625,
      "activations/layer23_attention_weight_min": -18.9888973236084,
      "activations/layer2_attention_weight_max": 30.962018966674805,
      "activations/layer2_attention_weight_min": -29.72541046142578,
      "activations/layer3_attention_weight_max": 87.65631866455078,
      "activations/layer3_attention_weight_min": -85.70709228515625,
      "activations/layer4_attention_weight_max": 101.25054931640625,
      "activations/layer4_attention_weight_min": -93.52439880371094,
      "activations/layer5_attention_weight_max": 92.898193359375,
      "activations/layer5_attention_weight_min": -80.86016082763672,
      "activations/layer6_attention_weight_max": 58.02403259277344,
      "activations/layer6_attention_weight_min": -53.77861785888672,
      "activations/layer7_attention_weight_max": 78.75728607177734,
      "activations/layer7_attention_weight_min": -74.23883056640625,
      "activations/layer8_attention_weight_max": 49.685028076171875,
      "activations/layer8_attention_weight_min": -47.209407806396484,
      "activations/layer9_attention_weight_max": 52.07917022705078,
      "activations/layer9_attention_weight_min": -50.314701080322266,
      "epoch": 7.39,
      "learning_rate": 0.00010341628787878788,
      "loss": 2.8148,
      "step": 127100
    },
    {
      "activations/layer0_attention_weight_max": 14.845008850097656,
      "activations/layer0_attention_weight_min": -14.755511283874512,
      "activations/layer10_attention_weight_max": 40.53301239013672,
      "activations/layer10_attention_weight_min": -36.161495208740234,
      "activations/layer11_attention_weight_max": 36.33190155029297,
      "activations/layer11_attention_weight_min": -32.971031188964844,
      "activations/layer12_attention_weight_max": 18.82141876220703,
      "activations/layer12_attention_weight_min": -27.862720489501953,
      "activations/layer13_attention_weight_max": 36.9467887878418,
      "activations/layer13_attention_weight_min": -31.274831771850586,
      "activations/layer14_attention_weight_max": 36.356475830078125,
      "activations/layer14_attention_weight_min": -31.043235778808594,
      "activations/layer15_attention_weight_max": 34.8604736328125,
      "activations/layer15_attention_weight_min": -28.631135940551758,
      "activations/layer16_attention_weight_max": 33.16510009765625,
      "activations/layer16_attention_weight_min": -27.485612869262695,
      "activations/layer17_attention_weight_max": 57.48648452758789,
      "activations/layer17_attention_weight_min": -42.01660919189453,
      "activations/layer18_attention_weight_max": 49.09485626220703,
      "activations/layer18_attention_weight_min": -35.69997787475586,
      "activations/layer19_attention_weight_max": 22.150686264038086,
      "activations/layer19_attention_weight_min": -21.63321304321289,
      "activations/layer1_attention_weight_max": 17.117361068725586,
      "activations/layer1_attention_weight_min": -14.864583969116211,
      "activations/layer20_attention_weight_max": 20.6960391998291,
      "activations/layer20_attention_weight_min": -20.416641235351562,
      "activations/layer21_attention_weight_max": 37.71770095825195,
      "activations/layer21_attention_weight_min": -25.645055770874023,
      "activations/layer22_attention_weight_max": 31.204038619995117,
      "activations/layer22_attention_weight_min": -25.48613166809082,
      "activations/layer23_attention_weight_max": 34.14255142211914,
      "activations/layer23_attention_weight_min": -20.580272674560547,
      "activations/layer2_attention_weight_max": 31.11762046813965,
      "activations/layer2_attention_weight_min": -30.3648681640625,
      "activations/layer3_attention_weight_max": 87.60826873779297,
      "activations/layer3_attention_weight_min": -90.27226257324219,
      "activations/layer4_attention_weight_max": 104.13823699951172,
      "activations/layer4_attention_weight_min": -98.46937561035156,
      "activations/layer5_attention_weight_max": 94.18380737304688,
      "activations/layer5_attention_weight_min": -86.27672576904297,
      "activations/layer6_attention_weight_max": 60.94546890258789,
      "activations/layer6_attention_weight_min": -57.34116744995117,
      "activations/layer7_attention_weight_max": 77.92886352539062,
      "activations/layer7_attention_weight_min": -76.69837951660156,
      "activations/layer8_attention_weight_max": 49.72236633300781,
      "activations/layer8_attention_weight_min": -48.5224494934082,
      "activations/layer9_attention_weight_max": 58.12550735473633,
      "activations/layer9_attention_weight_min": -50.922786712646484,
      "epoch": 7.39,
      "learning_rate": 0.00010339734848484847,
      "loss": 2.8108,
      "step": 127150
    },
    {
      "activations/layer0_attention_weight_max": 14.151412963867188,
      "activations/layer0_attention_weight_min": -14.327903747558594,
      "activations/layer10_attention_weight_max": 38.25639724731445,
      "activations/layer10_attention_weight_min": -37.21702194213867,
      "activations/layer11_attention_weight_max": 29.490652084350586,
      "activations/layer11_attention_weight_min": -30.224742889404297,
      "activations/layer12_attention_weight_max": 19.30813217163086,
      "activations/layer12_attention_weight_min": -26.865238189697266,
      "activations/layer13_attention_weight_max": 47.8619270324707,
      "activations/layer13_attention_weight_min": -31.571630477905273,
      "activations/layer14_attention_weight_max": 38.13438415527344,
      "activations/layer14_attention_weight_min": -32.46295928955078,
      "activations/layer15_attention_weight_max": 36.46489715576172,
      "activations/layer15_attention_weight_min": -30.585369110107422,
      "activations/layer16_attention_weight_max": 31.18137550354004,
      "activations/layer16_attention_weight_min": -27.747617721557617,
      "activations/layer17_attention_weight_max": 57.892822265625,
      "activations/layer17_attention_weight_min": -44.143524169921875,
      "activations/layer18_attention_weight_max": 46.17420196533203,
      "activations/layer18_attention_weight_min": -36.994895935058594,
      "activations/layer19_attention_weight_max": 22.015710830688477,
      "activations/layer19_attention_weight_min": -21.377172470092773,
      "activations/layer1_attention_weight_max": 17.92531394958496,
      "activations/layer1_attention_weight_min": -15.367953300476074,
      "activations/layer20_attention_weight_max": 22.829883575439453,
      "activations/layer20_attention_weight_min": -21.595012664794922,
      "activations/layer21_attention_weight_max": 39.80170440673828,
      "activations/layer21_attention_weight_min": -24.94738006591797,
      "activations/layer22_attention_weight_max": 30.32113265991211,
      "activations/layer22_attention_weight_min": -24.465906143188477,
      "activations/layer23_attention_weight_max": 36.760986328125,
      "activations/layer23_attention_weight_min": -25.280963897705078,
      "activations/layer2_attention_weight_max": 32.44630432128906,
      "activations/layer2_attention_weight_min": -29.97090721130371,
      "activations/layer3_attention_weight_max": 90.39350891113281,
      "activations/layer3_attention_weight_min": -92.15684509277344,
      "activations/layer4_attention_weight_max": 107.2912826538086,
      "activations/layer4_attention_weight_min": -98.97820281982422,
      "activations/layer5_attention_weight_max": 90.22663879394531,
      "activations/layer5_attention_weight_min": -83.62124633789062,
      "activations/layer6_attention_weight_max": 55.01924514770508,
      "activations/layer6_attention_weight_min": -52.74616622924805,
      "activations/layer7_attention_weight_max": 79.05326843261719,
      "activations/layer7_attention_weight_min": -70.91177368164062,
      "activations/layer8_attention_weight_max": 47.01734924316406,
      "activations/layer8_attention_weight_min": -45.18837356567383,
      "activations/layer9_attention_weight_max": 48.768802642822266,
      "activations/layer9_attention_weight_min": -49.0949821472168,
      "epoch": 7.39,
      "learning_rate": 0.00010337840909090908,
      "loss": 2.815,
      "step": 127200
    },
    {
      "activations/layer0_attention_weight_max": 14.183733940124512,
      "activations/layer0_attention_weight_min": -14.48957347869873,
      "activations/layer10_attention_weight_max": 40.36830139160156,
      "activations/layer10_attention_weight_min": -38.4067268371582,
      "activations/layer11_attention_weight_max": 36.350345611572266,
      "activations/layer11_attention_weight_min": -31.9100341796875,
      "activations/layer12_attention_weight_max": 21.27503204345703,
      "activations/layer12_attention_weight_min": -27.777204513549805,
      "activations/layer13_attention_weight_max": 41.30354309082031,
      "activations/layer13_attention_weight_min": -35.51020812988281,
      "activations/layer14_attention_weight_max": 43.63184356689453,
      "activations/layer14_attention_weight_min": -30.439178466796875,
      "activations/layer15_attention_weight_max": 37.47828674316406,
      "activations/layer15_attention_weight_min": -28.42467498779297,
      "activations/layer16_attention_weight_max": 30.942174911499023,
      "activations/layer16_attention_weight_min": -26.66588020324707,
      "activations/layer17_attention_weight_max": 52.92830276489258,
      "activations/layer17_attention_weight_min": -39.3653678894043,
      "activations/layer18_attention_weight_max": 47.725643157958984,
      "activations/layer18_attention_weight_min": -33.73649215698242,
      "activations/layer19_attention_weight_max": 19.48243522644043,
      "activations/layer19_attention_weight_min": -20.39812469482422,
      "activations/layer1_attention_weight_max": 16.93157386779785,
      "activations/layer1_attention_weight_min": -15.478710174560547,
      "activations/layer20_attention_weight_max": 21.038881301879883,
      "activations/layer20_attention_weight_min": -22.56644058227539,
      "activations/layer21_attention_weight_max": 36.47513198852539,
      "activations/layer21_attention_weight_min": -23.002870559692383,
      "activations/layer22_attention_weight_max": 29.995168685913086,
      "activations/layer22_attention_weight_min": -25.665233612060547,
      "activations/layer23_attention_weight_max": 35.8269157409668,
      "activations/layer23_attention_weight_min": -19.64727783203125,
      "activations/layer2_attention_weight_max": 31.350387573242188,
      "activations/layer2_attention_weight_min": -29.509000778198242,
      "activations/layer3_attention_weight_max": 83.05575561523438,
      "activations/layer3_attention_weight_min": -85.02255249023438,
      "activations/layer4_attention_weight_max": 96.55477905273438,
      "activations/layer4_attention_weight_min": -93.61224365234375,
      "activations/layer5_attention_weight_max": 86.64115905761719,
      "activations/layer5_attention_weight_min": -83.10301208496094,
      "activations/layer6_attention_weight_max": 54.5362434387207,
      "activations/layer6_attention_weight_min": -53.61870193481445,
      "activations/layer7_attention_weight_max": 81.24744415283203,
      "activations/layer7_attention_weight_min": -75.97801971435547,
      "activations/layer8_attention_weight_max": 44.81630325317383,
      "activations/layer8_attention_weight_min": -46.253028869628906,
      "activations/layer9_attention_weight_max": 62.008541107177734,
      "activations/layer9_attention_weight_min": -55.06764602661133,
      "epoch": 7.39,
      "learning_rate": 0.0001033594696969697,
      "loss": 2.8128,
      "step": 127250
    },
    {
      "activations/layer0_attention_weight_max": 14.912737846374512,
      "activations/layer0_attention_weight_min": -14.479557037353516,
      "activations/layer10_attention_weight_max": 38.999671936035156,
      "activations/layer10_attention_weight_min": -38.36695861816406,
      "activations/layer11_attention_weight_max": 32.71363830566406,
      "activations/layer11_attention_weight_min": -31.389026641845703,
      "activations/layer12_attention_weight_max": 21.511930465698242,
      "activations/layer12_attention_weight_min": -24.994970321655273,
      "activations/layer13_attention_weight_max": 48.484867095947266,
      "activations/layer13_attention_weight_min": -36.64015197753906,
      "activations/layer14_attention_weight_max": 56.02033615112305,
      "activations/layer14_attention_weight_min": -35.130577087402344,
      "activations/layer15_attention_weight_max": 51.399723052978516,
      "activations/layer15_attention_weight_min": -30.41315460205078,
      "activations/layer16_attention_weight_max": 35.06204605102539,
      "activations/layer16_attention_weight_min": -30.25200080871582,
      "activations/layer17_attention_weight_max": 59.25996780395508,
      "activations/layer17_attention_weight_min": -45.06489562988281,
      "activations/layer18_attention_weight_max": 50.4248161315918,
      "activations/layer18_attention_weight_min": -39.184165954589844,
      "activations/layer19_attention_weight_max": 24.863916397094727,
      "activations/layer19_attention_weight_min": -21.1544189453125,
      "activations/layer1_attention_weight_max": 16.998668670654297,
      "activations/layer1_attention_weight_min": -14.512938499450684,
      "activations/layer20_attention_weight_max": 21.72293472290039,
      "activations/layer20_attention_weight_min": -19.639768600463867,
      "activations/layer21_attention_weight_max": 45.47731018066406,
      "activations/layer21_attention_weight_min": -23.009424209594727,
      "activations/layer22_attention_weight_max": 36.87266159057617,
      "activations/layer22_attention_weight_min": -25.728513717651367,
      "activations/layer23_attention_weight_max": 40.031837463378906,
      "activations/layer23_attention_weight_min": -22.497343063354492,
      "activations/layer2_attention_weight_max": 31.55901336669922,
      "activations/layer2_attention_weight_min": -29.343387603759766,
      "activations/layer3_attention_weight_max": 90.78308868408203,
      "activations/layer3_attention_weight_min": -92.27271270751953,
      "activations/layer4_attention_weight_max": 106.70348358154297,
      "activations/layer4_attention_weight_min": -96.43033599853516,
      "activations/layer5_attention_weight_max": 93.62953186035156,
      "activations/layer5_attention_weight_min": -87.52700805664062,
      "activations/layer6_attention_weight_max": 57.47154235839844,
      "activations/layer6_attention_weight_min": -53.5383415222168,
      "activations/layer7_attention_weight_max": 80.34091186523438,
      "activations/layer7_attention_weight_min": -74.0018539428711,
      "activations/layer8_attention_weight_max": 47.817474365234375,
      "activations/layer8_attention_weight_min": -46.5156135559082,
      "activations/layer9_attention_weight_max": 56.39863967895508,
      "activations/layer9_attention_weight_min": -54.710609436035156,
      "epoch": 7.4,
      "learning_rate": 0.00010334053030303029,
      "loss": 2.8257,
      "step": 127300
    },
    {
      "activations/layer0_attention_weight_max": 14.799753189086914,
      "activations/layer0_attention_weight_min": -14.655780792236328,
      "activations/layer10_attention_weight_max": 36.96620559692383,
      "activations/layer10_attention_weight_min": -36.39337158203125,
      "activations/layer11_attention_weight_max": 30.4881591796875,
      "activations/layer11_attention_weight_min": -29.606307983398438,
      "activations/layer12_attention_weight_max": 20.99593734741211,
      "activations/layer12_attention_weight_min": -24.1154842376709,
      "activations/layer13_attention_weight_max": 43.54570388793945,
      "activations/layer13_attention_weight_min": -29.83475685119629,
      "activations/layer14_attention_weight_max": 38.65381622314453,
      "activations/layer14_attention_weight_min": -31.929302215576172,
      "activations/layer15_attention_weight_max": 37.97649002075195,
      "activations/layer15_attention_weight_min": -30.140583038330078,
      "activations/layer16_attention_weight_max": 31.779895782470703,
      "activations/layer16_attention_weight_min": -26.410390853881836,
      "activations/layer17_attention_weight_max": 52.464107513427734,
      "activations/layer17_attention_weight_min": -40.839969635009766,
      "activations/layer18_attention_weight_max": 47.20221710205078,
      "activations/layer18_attention_weight_min": -33.7816276550293,
      "activations/layer19_attention_weight_max": 22.39937400817871,
      "activations/layer19_attention_weight_min": -21.66922378540039,
      "activations/layer1_attention_weight_max": 17.206388473510742,
      "activations/layer1_attention_weight_min": -14.96520709991455,
      "activations/layer20_attention_weight_max": 23.60272789001465,
      "activations/layer20_attention_weight_min": -20.993976593017578,
      "activations/layer21_attention_weight_max": 35.07625961303711,
      "activations/layer21_attention_weight_min": -23.233022689819336,
      "activations/layer22_attention_weight_max": 33.66932678222656,
      "activations/layer22_attention_weight_min": -24.267215728759766,
      "activations/layer23_attention_weight_max": 33.0255241394043,
      "activations/layer23_attention_weight_min": -19.893999099731445,
      "activations/layer2_attention_weight_max": 31.58016014099121,
      "activations/layer2_attention_weight_min": -30.30963897705078,
      "activations/layer3_attention_weight_max": 83.6550064086914,
      "activations/layer3_attention_weight_min": -85.52947998046875,
      "activations/layer4_attention_weight_max": 97.65117645263672,
      "activations/layer4_attention_weight_min": -96.71459197998047,
      "activations/layer5_attention_weight_max": 90.97786712646484,
      "activations/layer5_attention_weight_min": -83.19925689697266,
      "activations/layer6_attention_weight_max": 55.45536804199219,
      "activations/layer6_attention_weight_min": -51.41832733154297,
      "activations/layer7_attention_weight_max": 72.23421478271484,
      "activations/layer7_attention_weight_min": -67.88298034667969,
      "activations/layer8_attention_weight_max": 43.43812942504883,
      "activations/layer8_attention_weight_min": -45.44047927856445,
      "activations/layer9_attention_weight_max": 50.971092224121094,
      "activations/layer9_attention_weight_min": -48.85615921020508,
      "epoch": 7.4,
      "learning_rate": 0.0001033215909090909,
      "loss": 2.8069,
      "step": 127350
    },
    {
      "activations/layer0_attention_weight_max": 15.579375267028809,
      "activations/layer0_attention_weight_min": -14.533573150634766,
      "activations/layer10_attention_weight_max": 41.593994140625,
      "activations/layer10_attention_weight_min": -37.43610382080078,
      "activations/layer11_attention_weight_max": 32.51603698730469,
      "activations/layer11_attention_weight_min": -32.770816802978516,
      "activations/layer12_attention_weight_max": 26.311750411987305,
      "activations/layer12_attention_weight_min": -24.400100708007812,
      "activations/layer13_attention_weight_max": 52.93484878540039,
      "activations/layer13_attention_weight_min": -32.3249626159668,
      "activations/layer14_attention_weight_max": 46.96235656738281,
      "activations/layer14_attention_weight_min": -33.144466400146484,
      "activations/layer15_attention_weight_max": 43.527767181396484,
      "activations/layer15_attention_weight_min": -28.992870330810547,
      "activations/layer16_attention_weight_max": 33.333736419677734,
      "activations/layer16_attention_weight_min": -27.926015853881836,
      "activations/layer17_attention_weight_max": 60.27637481689453,
      "activations/layer17_attention_weight_min": -44.443119049072266,
      "activations/layer18_attention_weight_max": 50.879852294921875,
      "activations/layer18_attention_weight_min": -35.41169738769531,
      "activations/layer19_attention_weight_max": 25.10590934753418,
      "activations/layer19_attention_weight_min": -20.429243087768555,
      "activations/layer1_attention_weight_max": 19.269628524780273,
      "activations/layer1_attention_weight_min": -16.461633682250977,
      "activations/layer20_attention_weight_max": 25.3284854888916,
      "activations/layer20_attention_weight_min": -22.938316345214844,
      "activations/layer21_attention_weight_max": 45.17390823364258,
      "activations/layer21_attention_weight_min": -23.134035110473633,
      "activations/layer22_attention_weight_max": 37.133731842041016,
      "activations/layer22_attention_weight_min": -23.61332893371582,
      "activations/layer23_attention_weight_max": 40.35198974609375,
      "activations/layer23_attention_weight_min": -20.925273895263672,
      "activations/layer2_attention_weight_max": 30.572843551635742,
      "activations/layer2_attention_weight_min": -29.2618465423584,
      "activations/layer3_attention_weight_max": 86.56169128417969,
      "activations/layer3_attention_weight_min": -87.35294342041016,
      "activations/layer4_attention_weight_max": 100.93997192382812,
      "activations/layer4_attention_weight_min": -95.41309356689453,
      "activations/layer5_attention_weight_max": 89.7405014038086,
      "activations/layer5_attention_weight_min": -87.69671630859375,
      "activations/layer6_attention_weight_max": 57.04446792602539,
      "activations/layer6_attention_weight_min": -57.06721115112305,
      "activations/layer7_attention_weight_max": 78.26576232910156,
      "activations/layer7_attention_weight_min": -76.44347381591797,
      "activations/layer8_attention_weight_max": 46.60465621948242,
      "activations/layer8_attention_weight_min": -47.77390670776367,
      "activations/layer9_attention_weight_max": 51.18680953979492,
      "activations/layer9_attention_weight_min": -49.71049118041992,
      "epoch": 7.4,
      "learning_rate": 0.00010330265151515152,
      "loss": 2.8098,
      "step": 127400
    },
    {
      "activations/layer0_attention_weight_max": 17.454994201660156,
      "activations/layer0_attention_weight_min": -14.89248275756836,
      "activations/layer10_attention_weight_max": 36.23136520385742,
      "activations/layer10_attention_weight_min": -34.81589126586914,
      "activations/layer11_attention_weight_max": 29.96611785888672,
      "activations/layer11_attention_weight_min": -30.37958335876465,
      "activations/layer12_attention_weight_max": 24.77604866027832,
      "activations/layer12_attention_weight_min": -22.913414001464844,
      "activations/layer13_attention_weight_max": 60.290313720703125,
      "activations/layer13_attention_weight_min": -35.43500900268555,
      "activations/layer14_attention_weight_max": 40.02273941040039,
      "activations/layer14_attention_weight_min": -29.392227172851562,
      "activations/layer15_attention_weight_max": 40.20817565917969,
      "activations/layer15_attention_weight_min": -28.638185501098633,
      "activations/layer16_attention_weight_max": 35.08361053466797,
      "activations/layer16_attention_weight_min": -28.601778030395508,
      "activations/layer17_attention_weight_max": 58.55274963378906,
      "activations/layer17_attention_weight_min": -43.058509826660156,
      "activations/layer18_attention_weight_max": 56.729549407958984,
      "activations/layer18_attention_weight_min": -36.175811767578125,
      "activations/layer19_attention_weight_max": 22.229576110839844,
      "activations/layer19_attention_weight_min": -21.26218032836914,
      "activations/layer1_attention_weight_max": 17.756248474121094,
      "activations/layer1_attention_weight_min": -16.383249282836914,
      "activations/layer20_attention_weight_max": 24.58522605895996,
      "activations/layer20_attention_weight_min": -20.849571228027344,
      "activations/layer21_attention_weight_max": 45.30112838745117,
      "activations/layer21_attention_weight_min": -23.237003326416016,
      "activations/layer22_attention_weight_max": 37.2370491027832,
      "activations/layer22_attention_weight_min": -25.43010902404785,
      "activations/layer23_attention_weight_max": 39.104862213134766,
      "activations/layer23_attention_weight_min": -20.85256576538086,
      "activations/layer2_attention_weight_max": 31.816730499267578,
      "activations/layer2_attention_weight_min": -29.470252990722656,
      "activations/layer3_attention_weight_max": 91.2391357421875,
      "activations/layer3_attention_weight_min": -89.4913101196289,
      "activations/layer4_attention_weight_max": 104.92815399169922,
      "activations/layer4_attention_weight_min": -95.96392059326172,
      "activations/layer5_attention_weight_max": 93.67201232910156,
      "activations/layer5_attention_weight_min": -83.45774841308594,
      "activations/layer6_attention_weight_max": 57.07793045043945,
      "activations/layer6_attention_weight_min": -52.22633361816406,
      "activations/layer7_attention_weight_max": 78.56476593017578,
      "activations/layer7_attention_weight_min": -70.16178131103516,
      "activations/layer8_attention_weight_max": 46.31278610229492,
      "activations/layer8_attention_weight_min": -42.6296501159668,
      "activations/layer9_attention_weight_max": 52.12982177734375,
      "activations/layer9_attention_weight_min": -46.856285095214844,
      "epoch": 7.41,
      "learning_rate": 0.0001032837121212121,
      "loss": 2.8159,
      "step": 127450
    },
    {
      "activations/layer0_attention_weight_max": 14.63905143737793,
      "activations/layer0_attention_weight_min": -14.700030326843262,
      "activations/layer10_attention_weight_max": 41.578582763671875,
      "activations/layer10_attention_weight_min": -37.19963073730469,
      "activations/layer11_attention_weight_max": 34.41619873046875,
      "activations/layer11_attention_weight_min": -31.8266658782959,
      "activations/layer12_attention_weight_max": 20.953683853149414,
      "activations/layer12_attention_weight_min": -25.407745361328125,
      "activations/layer13_attention_weight_max": 58.47294616699219,
      "activations/layer13_attention_weight_min": -38.31595993041992,
      "activations/layer14_attention_weight_max": 80.43040466308594,
      "activations/layer14_attention_weight_min": -41.68180465698242,
      "activations/layer15_attention_weight_max": 58.12939453125,
      "activations/layer15_attention_weight_min": -31.10556411743164,
      "activations/layer16_attention_weight_max": 38.79743957519531,
      "activations/layer16_attention_weight_min": -29.21879005432129,
      "activations/layer17_attention_weight_max": 60.54692459106445,
      "activations/layer17_attention_weight_min": -43.140201568603516,
      "activations/layer18_attention_weight_max": 55.79408645629883,
      "activations/layer18_attention_weight_min": -38.938575744628906,
      "activations/layer19_attention_weight_max": 30.504430770874023,
      "activations/layer19_attention_weight_min": -23.333730697631836,
      "activations/layer1_attention_weight_max": 17.6676025390625,
      "activations/layer1_attention_weight_min": -14.552056312561035,
      "activations/layer20_attention_weight_max": 31.414138793945312,
      "activations/layer20_attention_weight_min": -23.54778289794922,
      "activations/layer21_attention_weight_max": 48.79130935668945,
      "activations/layer21_attention_weight_min": -26.14903450012207,
      "activations/layer22_attention_weight_max": 36.979347229003906,
      "activations/layer22_attention_weight_min": -25.60956573486328,
      "activations/layer23_attention_weight_max": 40.502044677734375,
      "activations/layer23_attention_weight_min": -18.79407501220703,
      "activations/layer2_attention_weight_max": 32.2231330871582,
      "activations/layer2_attention_weight_min": -28.865095138549805,
      "activations/layer3_attention_weight_max": 84.95452880859375,
      "activations/layer3_attention_weight_min": -85.13774108886719,
      "activations/layer4_attention_weight_max": 104.31312561035156,
      "activations/layer4_attention_weight_min": -92.84648895263672,
      "activations/layer5_attention_weight_max": 92.37948608398438,
      "activations/layer5_attention_weight_min": -81.89485168457031,
      "activations/layer6_attention_weight_max": 59.19110870361328,
      "activations/layer6_attention_weight_min": -52.230628967285156,
      "activations/layer7_attention_weight_max": 77.26641082763672,
      "activations/layer7_attention_weight_min": -76.2027816772461,
      "activations/layer8_attention_weight_max": 46.27113342285156,
      "activations/layer8_attention_weight_min": -46.467620849609375,
      "activations/layer9_attention_weight_max": 55.74143600463867,
      "activations/layer9_attention_weight_min": -51.21394729614258,
      "epoch": 7.41,
      "learning_rate": 0.00010326477272727272,
      "loss": 2.8257,
      "step": 127500
    },
    {
      "activations/layer0_attention_weight_max": 14.489776611328125,
      "activations/layer0_attention_weight_min": -14.417612075805664,
      "activations/layer10_attention_weight_max": 38.28253173828125,
      "activations/layer10_attention_weight_min": -36.145851135253906,
      "activations/layer11_attention_weight_max": 32.474769592285156,
      "activations/layer11_attention_weight_min": -29.77875328063965,
      "activations/layer12_attention_weight_max": 18.406206130981445,
      "activations/layer12_attention_weight_min": -24.176279067993164,
      "activations/layer13_attention_weight_max": 38.91912841796875,
      "activations/layer13_attention_weight_min": -33.58765411376953,
      "activations/layer14_attention_weight_max": 37.38351821899414,
      "activations/layer14_attention_weight_min": -29.606372833251953,
      "activations/layer15_attention_weight_max": 34.95747756958008,
      "activations/layer15_attention_weight_min": -30.152515411376953,
      "activations/layer16_attention_weight_max": 32.4036979675293,
      "activations/layer16_attention_weight_min": -26.838796615600586,
      "activations/layer17_attention_weight_max": 54.888668060302734,
      "activations/layer17_attention_weight_min": -40.384395599365234,
      "activations/layer18_attention_weight_max": 46.02254867553711,
      "activations/layer18_attention_weight_min": -35.361202239990234,
      "activations/layer19_attention_weight_max": 21.0935001373291,
      "activations/layer19_attention_weight_min": -20.987302780151367,
      "activations/layer1_attention_weight_max": 16.906286239624023,
      "activations/layer1_attention_weight_min": -14.75110912322998,
      "activations/layer20_attention_weight_max": 20.75615119934082,
      "activations/layer20_attention_weight_min": -23.56976890563965,
      "activations/layer21_attention_weight_max": 37.70369338989258,
      "activations/layer21_attention_weight_min": -23.81082534790039,
      "activations/layer22_attention_weight_max": 31.13951301574707,
      "activations/layer22_attention_weight_min": -25.773849487304688,
      "activations/layer23_attention_weight_max": 32.72199249267578,
      "activations/layer23_attention_weight_min": -23.79525375366211,
      "activations/layer2_attention_weight_max": 31.381793975830078,
      "activations/layer2_attention_weight_min": -28.607913970947266,
      "activations/layer3_attention_weight_max": 89.9153823852539,
      "activations/layer3_attention_weight_min": -91.1072006225586,
      "activations/layer4_attention_weight_max": 106.7783432006836,
      "activations/layer4_attention_weight_min": -101.51026916503906,
      "activations/layer5_attention_weight_max": 90.4840087890625,
      "activations/layer5_attention_weight_min": -85.22486877441406,
      "activations/layer6_attention_weight_max": 55.43909454345703,
      "activations/layer6_attention_weight_min": -54.12028884887695,
      "activations/layer7_attention_weight_max": 75.99889373779297,
      "activations/layer7_attention_weight_min": -70.97400665283203,
      "activations/layer8_attention_weight_max": 44.58029556274414,
      "activations/layer8_attention_weight_min": -45.35408401489258,
      "activations/layer9_attention_weight_max": 51.58765411376953,
      "activations/layer9_attention_weight_min": -50.907691955566406,
      "epoch": 7.41,
      "learning_rate": 0.00010324583333333332,
      "loss": 2.8144,
      "step": 127550
    },
    {
      "activations/layer0_attention_weight_max": 15.590907096862793,
      "activations/layer0_attention_weight_min": -14.649723052978516,
      "activations/layer10_attention_weight_max": 38.98938751220703,
      "activations/layer10_attention_weight_min": -37.76569366455078,
      "activations/layer11_attention_weight_max": 32.70276641845703,
      "activations/layer11_attention_weight_min": -32.126426696777344,
      "activations/layer12_attention_weight_max": 20.278705596923828,
      "activations/layer12_attention_weight_min": -26.320600509643555,
      "activations/layer13_attention_weight_max": 46.964439392089844,
      "activations/layer13_attention_weight_min": -30.20417022705078,
      "activations/layer14_attention_weight_max": 38.54770278930664,
      "activations/layer14_attention_weight_min": -29.603668212890625,
      "activations/layer15_attention_weight_max": 32.98061752319336,
      "activations/layer15_attention_weight_min": -27.958364486694336,
      "activations/layer16_attention_weight_max": 32.48168182373047,
      "activations/layer16_attention_weight_min": -25.657243728637695,
      "activations/layer17_attention_weight_max": 54.37152099609375,
      "activations/layer17_attention_weight_min": -41.2935676574707,
      "activations/layer18_attention_weight_max": 42.75301742553711,
      "activations/layer18_attention_weight_min": -33.52811050415039,
      "activations/layer19_attention_weight_max": 20.455888748168945,
      "activations/layer19_attention_weight_min": -22.232648849487305,
      "activations/layer1_attention_weight_max": 16.72745704650879,
      "activations/layer1_attention_weight_min": -14.587604522705078,
      "activations/layer20_attention_weight_max": 20.100677490234375,
      "activations/layer20_attention_weight_min": -22.154441833496094,
      "activations/layer21_attention_weight_max": 31.734477996826172,
      "activations/layer21_attention_weight_min": -23.19045066833496,
      "activations/layer22_attention_weight_max": 32.47407531738281,
      "activations/layer22_attention_weight_min": -23.902376174926758,
      "activations/layer23_attention_weight_max": 32.453407287597656,
      "activations/layer23_attention_weight_min": -19.56694221496582,
      "activations/layer2_attention_weight_max": 31.494705200195312,
      "activations/layer2_attention_weight_min": -29.570043563842773,
      "activations/layer3_attention_weight_max": 86.45816040039062,
      "activations/layer3_attention_weight_min": -89.55644226074219,
      "activations/layer4_attention_weight_max": 108.0230484008789,
      "activations/layer4_attention_weight_min": -99.05170440673828,
      "activations/layer5_attention_weight_max": 92.20189666748047,
      "activations/layer5_attention_weight_min": -85.4140853881836,
      "activations/layer6_attention_weight_max": 57.50239181518555,
      "activations/layer6_attention_weight_min": -54.33613586425781,
      "activations/layer7_attention_weight_max": 78.52429962158203,
      "activations/layer7_attention_weight_min": -74.79602813720703,
      "activations/layer8_attention_weight_max": 45.64859390258789,
      "activations/layer8_attention_weight_min": -47.660484313964844,
      "activations/layer9_attention_weight_max": 53.0573844909668,
      "activations/layer9_attention_weight_min": -51.580379486083984,
      "epoch": 7.41,
      "learning_rate": 0.00010322689393939393,
      "loss": 2.7972,
      "step": 127600
    },
    {
      "activations/layer0_attention_weight_max": 15.418739318847656,
      "activations/layer0_attention_weight_min": -14.768187522888184,
      "activations/layer10_attention_weight_max": 37.96472930908203,
      "activations/layer10_attention_weight_min": -35.5743408203125,
      "activations/layer11_attention_weight_max": 29.859764099121094,
      "activations/layer11_attention_weight_min": -30.12348747253418,
      "activations/layer12_attention_weight_max": 19.763696670532227,
      "activations/layer12_attention_weight_min": -28.143346786499023,
      "activations/layer13_attention_weight_max": 52.05139923095703,
      "activations/layer13_attention_weight_min": -32.710121154785156,
      "activations/layer14_attention_weight_max": 37.03865051269531,
      "activations/layer14_attention_weight_min": -33.0063591003418,
      "activations/layer15_attention_weight_max": 35.96974563598633,
      "activations/layer15_attention_weight_min": -30.434200286865234,
      "activations/layer16_attention_weight_max": 31.698637008666992,
      "activations/layer16_attention_weight_min": -28.283279418945312,
      "activations/layer17_attention_weight_max": 54.875335693359375,
      "activations/layer17_attention_weight_min": -43.99625778198242,
      "activations/layer18_attention_weight_max": 43.63451385498047,
      "activations/layer18_attention_weight_min": -36.9111442565918,
      "activations/layer19_attention_weight_max": 24.67975425720215,
      "activations/layer19_attention_weight_min": -21.58354949951172,
      "activations/layer1_attention_weight_max": 16.51969337463379,
      "activations/layer1_attention_weight_min": -15.641550064086914,
      "activations/layer20_attention_weight_max": 22.132930755615234,
      "activations/layer20_attention_weight_min": -21.110637664794922,
      "activations/layer21_attention_weight_max": 43.65220642089844,
      "activations/layer21_attention_weight_min": -25.514358520507812,
      "activations/layer22_attention_weight_max": 33.02029800415039,
      "activations/layer22_attention_weight_min": -25.160568237304688,
      "activations/layer23_attention_weight_max": 34.2906608581543,
      "activations/layer23_attention_weight_min": -21.47869873046875,
      "activations/layer2_attention_weight_max": 31.087890625,
      "activations/layer2_attention_weight_min": -29.474477767944336,
      "activations/layer3_attention_weight_max": 86.21342468261719,
      "activations/layer3_attention_weight_min": -83.85102081298828,
      "activations/layer4_attention_weight_max": 103.26055908203125,
      "activations/layer4_attention_weight_min": -96.07537841796875,
      "activations/layer5_attention_weight_max": 89.69023132324219,
      "activations/layer5_attention_weight_min": -82.87832641601562,
      "activations/layer6_attention_weight_max": 55.33183288574219,
      "activations/layer6_attention_weight_min": -52.4725227355957,
      "activations/layer7_attention_weight_max": 73.30635833740234,
      "activations/layer7_attention_weight_min": -72.3329086303711,
      "activations/layer8_attention_weight_max": 43.666500091552734,
      "activations/layer8_attention_weight_min": -45.2807731628418,
      "activations/layer9_attention_weight_max": 48.02565002441406,
      "activations/layer9_attention_weight_min": -48.626441955566406,
      "epoch": 7.42,
      "learning_rate": 0.00010320795454545454,
      "loss": 2.8225,
      "step": 127650
    },
    {
      "activations/layer0_attention_weight_max": 15.148137092590332,
      "activations/layer0_attention_weight_min": -14.85110092163086,
      "activations/layer10_attention_weight_max": 35.917724609375,
      "activations/layer10_attention_weight_min": -35.3770637512207,
      "activations/layer11_attention_weight_max": 30.23121452331543,
      "activations/layer11_attention_weight_min": -29.998037338256836,
      "activations/layer12_attention_weight_max": 19.924312591552734,
      "activations/layer12_attention_weight_min": -25.848600387573242,
      "activations/layer13_attention_weight_max": 42.573158264160156,
      "activations/layer13_attention_weight_min": -31.340442657470703,
      "activations/layer14_attention_weight_max": 39.818504333496094,
      "activations/layer14_attention_weight_min": -32.18346405029297,
      "activations/layer15_attention_weight_max": 37.0101432800293,
      "activations/layer15_attention_weight_min": -29.57200813293457,
      "activations/layer16_attention_weight_max": 33.7393913269043,
      "activations/layer16_attention_weight_min": -27.727296829223633,
      "activations/layer17_attention_weight_max": 53.708656311035156,
      "activations/layer17_attention_weight_min": -43.244022369384766,
      "activations/layer18_attention_weight_max": 46.14065170288086,
      "activations/layer18_attention_weight_min": -35.87445831298828,
      "activations/layer19_attention_weight_max": 20.851316452026367,
      "activations/layer19_attention_weight_min": -20.03131103515625,
      "activations/layer1_attention_weight_max": 16.620574951171875,
      "activations/layer1_attention_weight_min": -15.067469596862793,
      "activations/layer20_attention_weight_max": 23.748441696166992,
      "activations/layer20_attention_weight_min": -21.461381912231445,
      "activations/layer21_attention_weight_max": 44.712013244628906,
      "activations/layer21_attention_weight_min": -23.91241455078125,
      "activations/layer22_attention_weight_max": 34.48149490356445,
      "activations/layer22_attention_weight_min": -23.37609100341797,
      "activations/layer23_attention_weight_max": 34.45704650878906,
      "activations/layer23_attention_weight_min": -19.028947830200195,
      "activations/layer2_attention_weight_max": 31.069259643554688,
      "activations/layer2_attention_weight_min": -29.480777740478516,
      "activations/layer3_attention_weight_max": 89.21869659423828,
      "activations/layer3_attention_weight_min": -89.51631164550781,
      "activations/layer4_attention_weight_max": 102.74259185791016,
      "activations/layer4_attention_weight_min": -96.29385375976562,
      "activations/layer5_attention_weight_max": 87.98776245117188,
      "activations/layer5_attention_weight_min": -80.80644226074219,
      "activations/layer6_attention_weight_max": 54.0890007019043,
      "activations/layer6_attention_weight_min": -50.88847732543945,
      "activations/layer7_attention_weight_max": 75.21762084960938,
      "activations/layer7_attention_weight_min": -69.97075653076172,
      "activations/layer8_attention_weight_max": 44.154720306396484,
      "activations/layer8_attention_weight_min": -42.902286529541016,
      "activations/layer9_attention_weight_max": 50.43206787109375,
      "activations/layer9_attention_weight_min": -47.5571174621582,
      "epoch": 7.42,
      "learning_rate": 0.00010318901515151514,
      "loss": 2.8014,
      "step": 127700
    },
    {
      "activations/layer0_attention_weight_max": 14.399515151977539,
      "activations/layer0_attention_weight_min": -15.296052932739258,
      "activations/layer10_attention_weight_max": 38.643638610839844,
      "activations/layer10_attention_weight_min": -36.3218994140625,
      "activations/layer11_attention_weight_max": 30.86446762084961,
      "activations/layer11_attention_weight_min": -30.453102111816406,
      "activations/layer12_attention_weight_max": 18.494232177734375,
      "activations/layer12_attention_weight_min": -27.67612075805664,
      "activations/layer13_attention_weight_max": 48.44758605957031,
      "activations/layer13_attention_weight_min": -34.94145202636719,
      "activations/layer14_attention_weight_max": 36.30011749267578,
      "activations/layer14_attention_weight_min": -30.085908889770508,
      "activations/layer15_attention_weight_max": 37.51728439331055,
      "activations/layer15_attention_weight_min": -29.308420181274414,
      "activations/layer16_attention_weight_max": 35.3265266418457,
      "activations/layer16_attention_weight_min": -26.7155818939209,
      "activations/layer17_attention_weight_max": 56.13160705566406,
      "activations/layer17_attention_weight_min": -42.53360366821289,
      "activations/layer18_attention_weight_max": 47.34673309326172,
      "activations/layer18_attention_weight_min": -37.622798919677734,
      "activations/layer19_attention_weight_max": 28.613306045532227,
      "activations/layer19_attention_weight_min": -23.630102157592773,
      "activations/layer1_attention_weight_max": 16.969112396240234,
      "activations/layer1_attention_weight_min": -17.49920654296875,
      "activations/layer20_attention_weight_max": 20.518163681030273,
      "activations/layer20_attention_weight_min": -23.030319213867188,
      "activations/layer21_attention_weight_max": 36.16582107543945,
      "activations/layer21_attention_weight_min": -25.39558982849121,
      "activations/layer22_attention_weight_max": 28.34491539001465,
      "activations/layer22_attention_weight_min": -25.580665588378906,
      "activations/layer23_attention_weight_max": 34.36593246459961,
      "activations/layer23_attention_weight_min": -25.180997848510742,
      "activations/layer2_attention_weight_max": 31.09463119506836,
      "activations/layer2_attention_weight_min": -28.31684112548828,
      "activations/layer3_attention_weight_max": 90.6565170288086,
      "activations/layer3_attention_weight_min": -86.49594116210938,
      "activations/layer4_attention_weight_max": 105.36468505859375,
      "activations/layer4_attention_weight_min": -93.1231460571289,
      "activations/layer5_attention_weight_max": 93.85028076171875,
      "activations/layer5_attention_weight_min": -83.75797271728516,
      "activations/layer6_attention_weight_max": 57.18708038330078,
      "activations/layer6_attention_weight_min": -53.50515365600586,
      "activations/layer7_attention_weight_max": 76.69960021972656,
      "activations/layer7_attention_weight_min": -69.81722259521484,
      "activations/layer8_attention_weight_max": 44.099308013916016,
      "activations/layer8_attention_weight_min": -44.26312255859375,
      "activations/layer9_attention_weight_max": 51.71352005004883,
      "activations/layer9_attention_weight_min": -52.36894607543945,
      "epoch": 7.42,
      "learning_rate": 0.00010317007575757575,
      "loss": 2.8174,
      "step": 127750
    },
    {
      "activations/layer0_attention_weight_max": 14.407387733459473,
      "activations/layer0_attention_weight_min": -14.931207656860352,
      "activations/layer10_attention_weight_max": 39.97089385986328,
      "activations/layer10_attention_weight_min": -37.02068328857422,
      "activations/layer11_attention_weight_max": 33.47279357910156,
      "activations/layer11_attention_weight_min": -32.58995819091797,
      "activations/layer12_attention_weight_max": 32.23857498168945,
      "activations/layer12_attention_weight_min": -21.47115135192871,
      "activations/layer13_attention_weight_max": 45.12778091430664,
      "activations/layer13_attention_weight_min": -32.29380798339844,
      "activations/layer14_attention_weight_max": 37.49873352050781,
      "activations/layer14_attention_weight_min": -30.406282424926758,
      "activations/layer15_attention_weight_max": 37.95777893066406,
      "activations/layer15_attention_weight_min": -29.61665153503418,
      "activations/layer16_attention_weight_max": 32.41379165649414,
      "activations/layer16_attention_weight_min": -27.089385986328125,
      "activations/layer17_attention_weight_max": 55.96084976196289,
      "activations/layer17_attention_weight_min": -41.3572998046875,
      "activations/layer18_attention_weight_max": 43.66078567504883,
      "activations/layer18_attention_weight_min": -36.063453674316406,
      "activations/layer19_attention_weight_max": 23.688068389892578,
      "activations/layer19_attention_weight_min": -20.806413650512695,
      "activations/layer1_attention_weight_max": 17.121883392333984,
      "activations/layer1_attention_weight_min": -15.561041831970215,
      "activations/layer20_attention_weight_max": 23.62825584411621,
      "activations/layer20_attention_weight_min": -23.083345413208008,
      "activations/layer21_attention_weight_max": 39.00368881225586,
      "activations/layer21_attention_weight_min": -23.803306579589844,
      "activations/layer22_attention_weight_max": 32.1854133605957,
      "activations/layer22_attention_weight_min": -26.9049015045166,
      "activations/layer23_attention_weight_max": 33.088279724121094,
      "activations/layer23_attention_weight_min": -21.294782638549805,
      "activations/layer2_attention_weight_max": 34.54059600830078,
      "activations/layer2_attention_weight_min": -29.40277862548828,
      "activations/layer3_attention_weight_max": 90.3558120727539,
      "activations/layer3_attention_weight_min": -89.02622985839844,
      "activations/layer4_attention_weight_max": 104.54351043701172,
      "activations/layer4_attention_weight_min": -100.5143814086914,
      "activations/layer5_attention_weight_max": 92.89088439941406,
      "activations/layer5_attention_weight_min": -88.67915344238281,
      "activations/layer6_attention_weight_max": 59.121856689453125,
      "activations/layer6_attention_weight_min": -54.02222442626953,
      "activations/layer7_attention_weight_max": 82.09011840820312,
      "activations/layer7_attention_weight_min": -77.45259857177734,
      "activations/layer8_attention_weight_max": 49.865604400634766,
      "activations/layer8_attention_weight_min": -47.4062614440918,
      "activations/layer9_attention_weight_max": 60.83956527709961,
      "activations/layer9_attention_weight_min": -54.40949249267578,
      "epoch": 7.43,
      "learning_rate": 0.00010315113636363635,
      "loss": 2.8153,
      "step": 127800
    },
    {
      "activations/layer0_attention_weight_max": 15.180766105651855,
      "activations/layer0_attention_weight_min": -15.047879219055176,
      "activations/layer10_attention_weight_max": 37.315277099609375,
      "activations/layer10_attention_weight_min": -34.25408935546875,
      "activations/layer11_attention_weight_max": 29.66185760498047,
      "activations/layer11_attention_weight_min": -31.69501304626465,
      "activations/layer12_attention_weight_max": 21.974348068237305,
      "activations/layer12_attention_weight_min": -25.450048446655273,
      "activations/layer13_attention_weight_max": 42.13190841674805,
      "activations/layer13_attention_weight_min": -29.89586067199707,
      "activations/layer14_attention_weight_max": 40.33063507080078,
      "activations/layer14_attention_weight_min": -31.75421714782715,
      "activations/layer15_attention_weight_max": 34.71674728393555,
      "activations/layer15_attention_weight_min": -30.08938980102539,
      "activations/layer16_attention_weight_max": 32.118228912353516,
      "activations/layer16_attention_weight_min": -28.117856979370117,
      "activations/layer17_attention_weight_max": 53.279537200927734,
      "activations/layer17_attention_weight_min": -44.264915466308594,
      "activations/layer18_attention_weight_max": 48.457576751708984,
      "activations/layer18_attention_weight_min": -34.4194450378418,
      "activations/layer19_attention_weight_max": 28.925992965698242,
      "activations/layer19_attention_weight_min": -20.75868034362793,
      "activations/layer1_attention_weight_max": 17.501840591430664,
      "activations/layer1_attention_weight_min": -18.0583553314209,
      "activations/layer20_attention_weight_max": 22.745738983154297,
      "activations/layer20_attention_weight_min": -22.558061599731445,
      "activations/layer21_attention_weight_max": 41.02854537963867,
      "activations/layer21_attention_weight_min": -23.627614974975586,
      "activations/layer22_attention_weight_max": 38.14598083496094,
      "activations/layer22_attention_weight_min": -24.854856491088867,
      "activations/layer23_attention_weight_max": 33.441551208496094,
      "activations/layer23_attention_weight_min": -21.51582145690918,
      "activations/layer2_attention_weight_max": 30.695903778076172,
      "activations/layer2_attention_weight_min": -29.482215881347656,
      "activations/layer3_attention_weight_max": 82.79901123046875,
      "activations/layer3_attention_weight_min": -85.0173568725586,
      "activations/layer4_attention_weight_max": 97.74691009521484,
      "activations/layer4_attention_weight_min": -91.6463623046875,
      "activations/layer5_attention_weight_max": 85.33552551269531,
      "activations/layer5_attention_weight_min": -82.52246856689453,
      "activations/layer6_attention_weight_max": 54.63649368286133,
      "activations/layer6_attention_weight_min": -51.18019104003906,
      "activations/layer7_attention_weight_max": 74.9321517944336,
      "activations/layer7_attention_weight_min": -71.76693725585938,
      "activations/layer8_attention_weight_max": 43.91880416870117,
      "activations/layer8_attention_weight_min": -43.40485763549805,
      "activations/layer9_attention_weight_max": 49.408226013183594,
      "activations/layer9_attention_weight_min": -46.83512496948242,
      "epoch": 7.43,
      "learning_rate": 0.00010313219696969696,
      "loss": 2.8062,
      "step": 127850
    },
    {
      "activations/layer0_attention_weight_max": 15.50505256652832,
      "activations/layer0_attention_weight_min": -14.673952102661133,
      "activations/layer10_attention_weight_max": 44.13068389892578,
      "activations/layer10_attention_weight_min": -40.582313537597656,
      "activations/layer11_attention_weight_max": 35.18816375732422,
      "activations/layer11_attention_weight_min": -34.11558532714844,
      "activations/layer12_attention_weight_max": 18.859312057495117,
      "activations/layer12_attention_weight_min": -27.954036712646484,
      "activations/layer13_attention_weight_max": 55.9860954284668,
      "activations/layer13_attention_weight_min": -36.94264602661133,
      "activations/layer14_attention_weight_max": 61.18918228149414,
      "activations/layer14_attention_weight_min": -34.72357940673828,
      "activations/layer15_attention_weight_max": 47.544551849365234,
      "activations/layer15_attention_weight_min": -29.690046310424805,
      "activations/layer16_attention_weight_max": 41.201255798339844,
      "activations/layer16_attention_weight_min": -28.260536193847656,
      "activations/layer17_attention_weight_max": 57.87056350708008,
      "activations/layer17_attention_weight_min": -48.968772888183594,
      "activations/layer18_attention_weight_max": 49.0998420715332,
      "activations/layer18_attention_weight_min": -39.06224060058594,
      "activations/layer19_attention_weight_max": 22.113086700439453,
      "activations/layer19_attention_weight_min": -21.029876708984375,
      "activations/layer1_attention_weight_max": 16.5275821685791,
      "activations/layer1_attention_weight_min": -15.283228874206543,
      "activations/layer20_attention_weight_max": 27.12394905090332,
      "activations/layer20_attention_weight_min": -20.13298797607422,
      "activations/layer21_attention_weight_max": 58.97150421142578,
      "activations/layer21_attention_weight_min": -26.113096237182617,
      "activations/layer22_attention_weight_max": 36.926151275634766,
      "activations/layer22_attention_weight_min": -26.651582717895508,
      "activations/layer23_attention_weight_max": 42.47297668457031,
      "activations/layer23_attention_weight_min": -24.16834831237793,
      "activations/layer2_attention_weight_max": 32.75999069213867,
      "activations/layer2_attention_weight_min": -30.499664306640625,
      "activations/layer3_attention_weight_max": 90.99945831298828,
      "activations/layer3_attention_weight_min": -90.2207260131836,
      "activations/layer4_attention_weight_max": 104.90892791748047,
      "activations/layer4_attention_weight_min": -99.91333770751953,
      "activations/layer5_attention_weight_max": 94.03978729248047,
      "activations/layer5_attention_weight_min": -86.98030853271484,
      "activations/layer6_attention_weight_max": 58.71952438354492,
      "activations/layer6_attention_weight_min": -55.73091125488281,
      "activations/layer7_attention_weight_max": 78.3740234375,
      "activations/layer7_attention_weight_min": -74.85153198242188,
      "activations/layer8_attention_weight_max": 48.59578323364258,
      "activations/layer8_attention_weight_min": -48.24191665649414,
      "activations/layer9_attention_weight_max": 54.04134750366211,
      "activations/layer9_attention_weight_min": -52.79478073120117,
      "epoch": 7.43,
      "learning_rate": 0.00010311325757575757,
      "loss": 2.8012,
      "step": 127900
    },
    {
      "activations/layer0_attention_weight_max": 15.085248947143555,
      "activations/layer0_attention_weight_min": -14.967418670654297,
      "activations/layer10_attention_weight_max": 38.44008255004883,
      "activations/layer10_attention_weight_min": -36.48468017578125,
      "activations/layer11_attention_weight_max": 31.459335327148438,
      "activations/layer11_attention_weight_min": -32.07176208496094,
      "activations/layer12_attention_weight_max": 21.19002342224121,
      "activations/layer12_attention_weight_min": -26.877193450927734,
      "activations/layer13_attention_weight_max": 48.36274337768555,
      "activations/layer13_attention_weight_min": -34.75596618652344,
      "activations/layer14_attention_weight_max": 42.63930892944336,
      "activations/layer14_attention_weight_min": -31.602317810058594,
      "activations/layer15_attention_weight_max": 37.92890167236328,
      "activations/layer15_attention_weight_min": -30.68807601928711,
      "activations/layer16_attention_weight_max": 33.237709045410156,
      "activations/layer16_attention_weight_min": -27.512115478515625,
      "activations/layer17_attention_weight_max": 55.399837493896484,
      "activations/layer17_attention_weight_min": -41.0179557800293,
      "activations/layer18_attention_weight_max": 47.48139572143555,
      "activations/layer18_attention_weight_min": -37.33230972290039,
      "activations/layer19_attention_weight_max": 26.66128921508789,
      "activations/layer19_attention_weight_min": -24.18008804321289,
      "activations/layer1_attention_weight_max": 17.94752311706543,
      "activations/layer1_attention_weight_min": -15.17593765258789,
      "activations/layer20_attention_weight_max": 23.112693786621094,
      "activations/layer20_attention_weight_min": -24.423145294189453,
      "activations/layer21_attention_weight_max": 44.11628341674805,
      "activations/layer21_attention_weight_min": -25.151086807250977,
      "activations/layer22_attention_weight_max": 31.752113342285156,
      "activations/layer22_attention_weight_min": -26.19917869567871,
      "activations/layer23_attention_weight_max": 37.194862365722656,
      "activations/layer23_attention_weight_min": -21.71854019165039,
      "activations/layer2_attention_weight_max": 31.27414321899414,
      "activations/layer2_attention_weight_min": -29.857452392578125,
      "activations/layer3_attention_weight_max": 87.27947235107422,
      "activations/layer3_attention_weight_min": -86.02018737792969,
      "activations/layer4_attention_weight_max": 100.92371368408203,
      "activations/layer4_attention_weight_min": -93.02344512939453,
      "activations/layer5_attention_weight_max": 85.30899047851562,
      "activations/layer5_attention_weight_min": -82.44078826904297,
      "activations/layer6_attention_weight_max": 56.57765579223633,
      "activations/layer6_attention_weight_min": -55.06401062011719,
      "activations/layer7_attention_weight_max": 76.95574188232422,
      "activations/layer7_attention_weight_min": -74.3508071899414,
      "activations/layer8_attention_weight_max": 46.588226318359375,
      "activations/layer8_attention_weight_min": -49.466453552246094,
      "activations/layer9_attention_weight_max": 51.25696563720703,
      "activations/layer9_attention_weight_min": -51.72085189819336,
      "epoch": 7.43,
      "learning_rate": 0.00010309431818181819,
      "loss": 2.8339,
      "step": 127950
    },
    {
      "activations/layer0_attention_weight_max": 14.741786003112793,
      "activations/layer0_attention_weight_min": -14.670563697814941,
      "activations/layer10_attention_weight_max": 40.63214111328125,
      "activations/layer10_attention_weight_min": -36.23684310913086,
      "activations/layer11_attention_weight_max": 34.09324645996094,
      "activations/layer11_attention_weight_min": -30.702533721923828,
      "activations/layer12_attention_weight_max": 20.102985382080078,
      "activations/layer12_attention_weight_min": -24.38761329650879,
      "activations/layer13_attention_weight_max": 53.464656829833984,
      "activations/layer13_attention_weight_min": -32.60258483886719,
      "activations/layer14_attention_weight_max": 47.13981628417969,
      "activations/layer14_attention_weight_min": -29.294065475463867,
      "activations/layer15_attention_weight_max": 37.91588592529297,
      "activations/layer15_attention_weight_min": -29.16460418701172,
      "activations/layer16_attention_weight_max": 36.681583404541016,
      "activations/layer16_attention_weight_min": -28.684701919555664,
      "activations/layer17_attention_weight_max": 58.30799102783203,
      "activations/layer17_attention_weight_min": -46.0269660949707,
      "activations/layer18_attention_weight_max": 50.77516174316406,
      "activations/layer18_attention_weight_min": -36.54954528808594,
      "activations/layer19_attention_weight_max": 24.63079833984375,
      "activations/layer19_attention_weight_min": -21.97490692138672,
      "activations/layer1_attention_weight_max": 16.643829345703125,
      "activations/layer1_attention_weight_min": -16.39755630493164,
      "activations/layer20_attention_weight_max": 25.526216506958008,
      "activations/layer20_attention_weight_min": -23.763153076171875,
      "activations/layer21_attention_weight_max": 39.52937698364258,
      "activations/layer21_attention_weight_min": -23.224109649658203,
      "activations/layer22_attention_weight_max": 33.89955520629883,
      "activations/layer22_attention_weight_min": -27.003864288330078,
      "activations/layer23_attention_weight_max": 40.15203857421875,
      "activations/layer23_attention_weight_min": -21.80251693725586,
      "activations/layer2_attention_weight_max": 32.149837493896484,
      "activations/layer2_attention_weight_min": -29.747846603393555,
      "activations/layer3_attention_weight_max": 91.67808532714844,
      "activations/layer3_attention_weight_min": -91.8109359741211,
      "activations/layer4_attention_weight_max": 104.45800018310547,
      "activations/layer4_attention_weight_min": -102.46646881103516,
      "activations/layer5_attention_weight_max": 91.86553955078125,
      "activations/layer5_attention_weight_min": -85.64492797851562,
      "activations/layer6_attention_weight_max": 60.930381774902344,
      "activations/layer6_attention_weight_min": -56.67121505737305,
      "activations/layer7_attention_weight_max": 82.15326690673828,
      "activations/layer7_attention_weight_min": -71.26515197753906,
      "activations/layer8_attention_weight_max": 47.082008361816406,
      "activations/layer8_attention_weight_min": -46.77644729614258,
      "activations/layer9_attention_weight_max": 64.71851348876953,
      "activations/layer9_attention_weight_min": -53.2576789855957,
      "epoch": 7.44,
      "learning_rate": 0.00010307537878787877,
      "loss": 2.8216,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_loss": 2.767578125,
      "eval_runtime": 8.5551,
      "eval_samples_per_second": 501.922,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_openwebtext_loss": 2.767578125,
      "eval_openwebtext_ppl": 15.920030957803608,
      "eval_openwebtext_runtime": 8.5551,
      "eval_openwebtext_samples_per_second": 501.922,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 2.0056,
      "eval_wikitext_samples_per_second": 227.367,
      "step": 128000
    },
    {
      "epoch": 7.44,
      "eval_lambada_loss": 2.802734375,
      "eval_lambada_ppl": 16.489674134905655,
      "eval_lambada_runtime": 9.6458,
      "eval_lambada_samples_per_second": 504.779,
      "step": 128000
    },
    {
      "activations/layer0_attention_weight_max": 14.991262435913086,
      "activations/layer0_attention_weight_min": -14.603874206542969,
      "activations/layer10_attention_weight_max": 36.9638671875,
      "activations/layer10_attention_weight_min": -35.92094802856445,
      "activations/layer11_attention_weight_max": 31.434354782104492,
      "activations/layer11_attention_weight_min": -30.509937286376953,
      "activations/layer12_attention_weight_max": 18.829824447631836,
      "activations/layer12_attention_weight_min": -27.944982528686523,
      "activations/layer13_attention_weight_max": 48.93742752075195,
      "activations/layer13_attention_weight_min": -34.75313949584961,
      "activations/layer14_attention_weight_max": 51.1041145324707,
      "activations/layer14_attention_weight_min": -35.0379753112793,
      "activations/layer15_attention_weight_max": 38.559513092041016,
      "activations/layer15_attention_weight_min": -29.836936950683594,
      "activations/layer16_attention_weight_max": 32.08121109008789,
      "activations/layer16_attention_weight_min": -27.697763442993164,
      "activations/layer17_attention_weight_max": 56.731414794921875,
      "activations/layer17_attention_weight_min": -43.0428581237793,
      "activations/layer18_attention_weight_max": 46.8500862121582,
      "activations/layer18_attention_weight_min": -35.068790435791016,
      "activations/layer19_attention_weight_max": 22.901166915893555,
      "activations/layer19_attention_weight_min": -19.482664108276367,
      "activations/layer1_attention_weight_max": 17.5762939453125,
      "activations/layer1_attention_weight_min": -14.98438835144043,
      "activations/layer20_attention_weight_max": 22.004119873046875,
      "activations/layer20_attention_weight_min": -20.02512550354004,
      "activations/layer21_attention_weight_max": 37.046119689941406,
      "activations/layer21_attention_weight_min": -24.470340728759766,
      "activations/layer22_attention_weight_max": 31.65589714050293,
      "activations/layer22_attention_weight_min": -25.195905685424805,
      "activations/layer23_attention_weight_max": 34.3463249206543,
      "activations/layer23_attention_weight_min": -20.702707290649414,
      "activations/layer2_attention_weight_max": 31.041946411132812,
      "activations/layer2_attention_weight_min": -29.459449768066406,
      "activations/layer3_attention_weight_max": 85.65454864501953,
      "activations/layer3_attention_weight_min": -87.87779998779297,
      "activations/layer4_attention_weight_max": 101.20008850097656,
      "activations/layer4_attention_weight_min": -97.51835632324219,
      "activations/layer5_attention_weight_max": 90.99568176269531,
      "activations/layer5_attention_weight_min": -82.0711669921875,
      "activations/layer6_attention_weight_max": 56.95823287963867,
      "activations/layer6_attention_weight_min": -52.68014144897461,
      "activations/layer7_attention_weight_max": 79.3770980834961,
      "activations/layer7_attention_weight_min": -68.73977661132812,
      "activations/layer8_attention_weight_max": 44.4647102355957,
      "activations/layer8_attention_weight_min": -44.0722541809082,
      "activations/layer9_attention_weight_max": 50.3854866027832,
      "activations/layer9_attention_weight_min": -47.47249984741211,
      "epoch": 7.44,
      "learning_rate": 0.00010305643939393939,
      "loss": 2.8144,
      "step": 128050
    },
    {
      "activations/layer0_attention_weight_max": 15.06785774230957,
      "activations/layer0_attention_weight_min": -14.402274131774902,
      "activations/layer10_attention_weight_max": 41.96542739868164,
      "activations/layer10_attention_weight_min": -39.56039810180664,
      "activations/layer11_attention_weight_max": 34.830360412597656,
      "activations/layer11_attention_weight_min": -30.861103057861328,
      "activations/layer12_attention_weight_max": 20.89026641845703,
      "activations/layer12_attention_weight_min": -25.20460319519043,
      "activations/layer13_attention_weight_max": 60.783912658691406,
      "activations/layer13_attention_weight_min": -37.2473030090332,
      "activations/layer14_attention_weight_max": 61.32313537597656,
      "activations/layer14_attention_weight_min": -31.129541397094727,
      "activations/layer15_attention_weight_max": 54.24831008911133,
      "activations/layer15_attention_weight_min": -30.315078735351562,
      "activations/layer16_attention_weight_max": 34.0509033203125,
      "activations/layer16_attention_weight_min": -28.48692512512207,
      "activations/layer17_attention_weight_max": 56.99456024169922,
      "activations/layer17_attention_weight_min": -43.6945686340332,
      "activations/layer18_attention_weight_max": 47.53837585449219,
      "activations/layer18_attention_weight_min": -37.51509094238281,
      "activations/layer19_attention_weight_max": 21.16399574279785,
      "activations/layer19_attention_weight_min": -20.714269638061523,
      "activations/layer1_attention_weight_max": 16.092464447021484,
      "activations/layer1_attention_weight_min": -14.822038650512695,
      "activations/layer20_attention_weight_max": 24.269323348999023,
      "activations/layer20_attention_weight_min": -21.536846160888672,
      "activations/layer21_attention_weight_max": 44.292999267578125,
      "activations/layer21_attention_weight_min": -25.438974380493164,
      "activations/layer22_attention_weight_max": 34.50870132446289,
      "activations/layer22_attention_weight_min": -26.52219009399414,
      "activations/layer23_attention_weight_max": 33.98186492919922,
      "activations/layer23_attention_weight_min": -21.556509017944336,
      "activations/layer2_attention_weight_max": 31.918132781982422,
      "activations/layer2_attention_weight_min": -29.673660278320312,
      "activations/layer3_attention_weight_max": 85.88343048095703,
      "activations/layer3_attention_weight_min": -87.43040466308594,
      "activations/layer4_attention_weight_max": 102.60282135009766,
      "activations/layer4_attention_weight_min": -97.88231658935547,
      "activations/layer5_attention_weight_max": 93.1561508178711,
      "activations/layer5_attention_weight_min": -80.582763671875,
      "activations/layer6_attention_weight_max": 55.70155715942383,
      "activations/layer6_attention_weight_min": -49.87129211425781,
      "activations/layer7_attention_weight_max": 77.35167694091797,
      "activations/layer7_attention_weight_min": -69.83540344238281,
      "activations/layer8_attention_weight_max": 47.559043884277344,
      "activations/layer8_attention_weight_min": -45.822940826416016,
      "activations/layer9_attention_weight_max": 56.927223205566406,
      "activations/layer9_attention_weight_min": -50.001861572265625,
      "epoch": 7.44,
      "learning_rate": 0.00010303749999999999,
      "loss": 2.806,
      "step": 128100
    },
    {
      "activations/layer0_attention_weight_max": 15.501151084899902,
      "activations/layer0_attention_weight_min": -14.851634979248047,
      "activations/layer10_attention_weight_max": 34.261417388916016,
      "activations/layer10_attention_weight_min": -33.098388671875,
      "activations/layer11_attention_weight_max": 29.231525421142578,
      "activations/layer11_attention_weight_min": -30.790189743041992,
      "activations/layer12_attention_weight_max": 23.2784481048584,
      "activations/layer12_attention_weight_min": -26.86786651611328,
      "activations/layer13_attention_weight_max": 39.075042724609375,
      "activations/layer13_attention_weight_min": -29.151643753051758,
      "activations/layer14_attention_weight_max": 33.680633544921875,
      "activations/layer14_attention_weight_min": -28.37384796142578,
      "activations/layer15_attention_weight_max": 32.39485549926758,
      "activations/layer15_attention_weight_min": -27.94384002685547,
      "activations/layer16_attention_weight_max": 29.525575637817383,
      "activations/layer16_attention_weight_min": -26.225635528564453,
      "activations/layer17_attention_weight_max": 49.26920700073242,
      "activations/layer17_attention_weight_min": -39.623313903808594,
      "activations/layer18_attention_weight_max": 43.15779113769531,
      "activations/layer18_attention_weight_min": -33.833621978759766,
      "activations/layer19_attention_weight_max": 27.452463150024414,
      "activations/layer19_attention_weight_min": -19.6202392578125,
      "activations/layer1_attention_weight_max": 17.639789581298828,
      "activations/layer1_attention_weight_min": -18.31049346923828,
      "activations/layer20_attention_weight_max": 22.415864944458008,
      "activations/layer20_attention_weight_min": -20.260684967041016,
      "activations/layer21_attention_weight_max": 45.454559326171875,
      "activations/layer21_attention_weight_min": -27.330495834350586,
      "activations/layer22_attention_weight_max": 29.83445167541504,
      "activations/layer22_attention_weight_min": -24.196063995361328,
      "activations/layer23_attention_weight_max": 32.3706169128418,
      "activations/layer23_attention_weight_min": -21.624998092651367,
      "activations/layer2_attention_weight_max": 33.60191345214844,
      "activations/layer2_attention_weight_min": -30.52948760986328,
      "activations/layer3_attention_weight_max": 87.55843353271484,
      "activations/layer3_attention_weight_min": -91.3077392578125,
      "activations/layer4_attention_weight_max": 101.40188598632812,
      "activations/layer4_attention_weight_min": -98.62421417236328,
      "activations/layer5_attention_weight_max": 88.75189208984375,
      "activations/layer5_attention_weight_min": -88.3434066772461,
      "activations/layer6_attention_weight_max": 56.5505256652832,
      "activations/layer6_attention_weight_min": -52.089698791503906,
      "activations/layer7_attention_weight_max": 79.70829010009766,
      "activations/layer7_attention_weight_min": -71.05056762695312,
      "activations/layer8_attention_weight_max": 41.6734619140625,
      "activations/layer8_attention_weight_min": -43.58043670654297,
      "activations/layer9_attention_weight_max": 55.2463493347168,
      "activations/layer9_attention_weight_min": -46.860774993896484,
      "epoch": 7.45,
      "learning_rate": 0.00010301856060606059,
      "loss": 2.8112,
      "step": 128150
    },
    {
      "activations/layer0_attention_weight_max": 14.788331985473633,
      "activations/layer0_attention_weight_min": -14.444536209106445,
      "activations/layer10_attention_weight_max": 36.423255920410156,
      "activations/layer10_attention_weight_min": -33.648902893066406,
      "activations/layer11_attention_weight_max": 30.884857177734375,
      "activations/layer11_attention_weight_min": -29.072067260742188,
      "activations/layer12_attention_weight_max": 17.617509841918945,
      "activations/layer12_attention_weight_min": -28.739484786987305,
      "activations/layer13_attention_weight_max": 42.01420593261719,
      "activations/layer13_attention_weight_min": -34.216556549072266,
      "activations/layer14_attention_weight_max": 39.6349983215332,
      "activations/layer14_attention_weight_min": -32.360897064208984,
      "activations/layer15_attention_weight_max": 39.36344528198242,
      "activations/layer15_attention_weight_min": -30.274240493774414,
      "activations/layer16_attention_weight_max": 32.03590393066406,
      "activations/layer16_attention_weight_min": -27.105670928955078,
      "activations/layer17_attention_weight_max": 50.589210510253906,
      "activations/layer17_attention_weight_min": -43.446327209472656,
      "activations/layer18_attention_weight_max": 44.657554626464844,
      "activations/layer18_attention_weight_min": -34.97276306152344,
      "activations/layer19_attention_weight_max": 20.67057228088379,
      "activations/layer19_attention_weight_min": -21.276824951171875,
      "activations/layer1_attention_weight_max": 17.141815185546875,
      "activations/layer1_attention_weight_min": -14.169013023376465,
      "activations/layer20_attention_weight_max": 21.827510833740234,
      "activations/layer20_attention_weight_min": -21.04142189025879,
      "activations/layer21_attention_weight_max": 34.50835037231445,
      "activations/layer21_attention_weight_min": -24.069042205810547,
      "activations/layer22_attention_weight_max": 33.315303802490234,
      "activations/layer22_attention_weight_min": -23.64022445678711,
      "activations/layer23_attention_weight_max": 32.38691329956055,
      "activations/layer23_attention_weight_min": -23.28130340576172,
      "activations/layer2_attention_weight_max": 34.15483093261719,
      "activations/layer2_attention_weight_min": -32.37776184082031,
      "activations/layer3_attention_weight_max": 92.46717071533203,
      "activations/layer3_attention_weight_min": -95.75797271728516,
      "activations/layer4_attention_weight_max": 105.18090057373047,
      "activations/layer4_attention_weight_min": -97.04158782958984,
      "activations/layer5_attention_weight_max": 90.27191162109375,
      "activations/layer5_attention_weight_min": -83.27081298828125,
      "activations/layer6_attention_weight_max": 57.85667419433594,
      "activations/layer6_attention_weight_min": -51.08241271972656,
      "activations/layer7_attention_weight_max": 72.366455078125,
      "activations/layer7_attention_weight_min": -78.36588287353516,
      "activations/layer8_attention_weight_max": 44.70772171020508,
      "activations/layer8_attention_weight_min": -42.63754653930664,
      "activations/layer9_attention_weight_max": 53.28207015991211,
      "activations/layer9_attention_weight_min": -50.70634078979492,
      "epoch": 7.45,
      "learning_rate": 0.00010299962121212121,
      "loss": 2.8131,
      "step": 128200
    },
    {
      "activations/layer0_attention_weight_max": 15.66584587097168,
      "activations/layer0_attention_weight_min": -14.975550651550293,
      "activations/layer10_attention_weight_max": 38.773468017578125,
      "activations/layer10_attention_weight_min": -36.16680145263672,
      "activations/layer11_attention_weight_max": 31.07329559326172,
      "activations/layer11_attention_weight_min": -30.467063903808594,
      "activations/layer12_attention_weight_max": 23.006881713867188,
      "activations/layer12_attention_weight_min": -22.97052001953125,
      "activations/layer13_attention_weight_max": 44.8159294128418,
      "activations/layer13_attention_weight_min": -31.119876861572266,
      "activations/layer14_attention_weight_max": 36.5012321472168,
      "activations/layer14_attention_weight_min": -30.592994689941406,
      "activations/layer15_attention_weight_max": 33.80984878540039,
      "activations/layer15_attention_weight_min": -30.26011848449707,
      "activations/layer16_attention_weight_max": 31.873523712158203,
      "activations/layer16_attention_weight_min": -28.847620010375977,
      "activations/layer17_attention_weight_max": 53.21173858642578,
      "activations/layer17_attention_weight_min": -43.59718322753906,
      "activations/layer18_attention_weight_max": 44.75772476196289,
      "activations/layer18_attention_weight_min": -36.891658782958984,
      "activations/layer19_attention_weight_max": 22.888721466064453,
      "activations/layer19_attention_weight_min": -21.6614990234375,
      "activations/layer1_attention_weight_max": 17.479774475097656,
      "activations/layer1_attention_weight_min": -14.731192588806152,
      "activations/layer20_attention_weight_max": 20.57815170288086,
      "activations/layer20_attention_weight_min": -22.7329044342041,
      "activations/layer21_attention_weight_max": 39.349300384521484,
      "activations/layer21_attention_weight_min": -25.253612518310547,
      "activations/layer22_attention_weight_max": 33.22032165527344,
      "activations/layer22_attention_weight_min": -26.950790405273438,
      "activations/layer23_attention_weight_max": 34.05436706542969,
      "activations/layer23_attention_weight_min": -22.194965362548828,
      "activations/layer2_attention_weight_max": 31.784744262695312,
      "activations/layer2_attention_weight_min": -30.730934143066406,
      "activations/layer3_attention_weight_max": 93.2717514038086,
      "activations/layer3_attention_weight_min": -93.60023498535156,
      "activations/layer4_attention_weight_max": 104.70121002197266,
      "activations/layer4_attention_weight_min": -97.97049713134766,
      "activations/layer5_attention_weight_max": 90.04611206054688,
      "activations/layer5_attention_weight_min": -85.26331329345703,
      "activations/layer6_attention_weight_max": 57.85673522949219,
      "activations/layer6_attention_weight_min": -53.3162956237793,
      "activations/layer7_attention_weight_max": 79.92082214355469,
      "activations/layer7_attention_weight_min": -74.96442413330078,
      "activations/layer8_attention_weight_max": 49.16201400756836,
      "activations/layer8_attention_weight_min": -47.016014099121094,
      "activations/layer9_attention_weight_max": 52.39724349975586,
      "activations/layer9_attention_weight_min": -47.35627365112305,
      "epoch": 7.45,
      "learning_rate": 0.00010298068181818181,
      "loss": 2.7967,
      "step": 128250
    },
    {
      "activations/layer0_attention_weight_max": 15.06865119934082,
      "activations/layer0_attention_weight_min": -14.472244262695312,
      "activations/layer10_attention_weight_max": 37.537452697753906,
      "activations/layer10_attention_weight_min": -36.42272186279297,
      "activations/layer11_attention_weight_max": 29.91554832458496,
      "activations/layer11_attention_weight_min": -29.99872398376465,
      "activations/layer12_attention_weight_max": 20.655553817749023,
      "activations/layer12_attention_weight_min": -26.11272621154785,
      "activations/layer13_attention_weight_max": 43.28706741333008,
      "activations/layer13_attention_weight_min": -35.88451385498047,
      "activations/layer14_attention_weight_max": 40.842369079589844,
      "activations/layer14_attention_weight_min": -30.337448120117188,
      "activations/layer15_attention_weight_max": 38.54072952270508,
      "activations/layer15_attention_weight_min": -29.214950561523438,
      "activations/layer16_attention_weight_max": 31.750532150268555,
      "activations/layer16_attention_weight_min": -26.827909469604492,
      "activations/layer17_attention_weight_max": 55.5795783996582,
      "activations/layer17_attention_weight_min": -41.50200271606445,
      "activations/layer18_attention_weight_max": 50.56517028808594,
      "activations/layer18_attention_weight_min": -35.67668914794922,
      "activations/layer19_attention_weight_max": 20.963045120239258,
      "activations/layer19_attention_weight_min": -21.32358169555664,
      "activations/layer1_attention_weight_max": 17.9216251373291,
      "activations/layer1_attention_weight_min": -13.99498176574707,
      "activations/layer20_attention_weight_max": 24.76363754272461,
      "activations/layer20_attention_weight_min": -21.640230178833008,
      "activations/layer21_attention_weight_max": 41.70054244995117,
      "activations/layer21_attention_weight_min": -23.430858612060547,
      "activations/layer22_attention_weight_max": 33.25472640991211,
      "activations/layer22_attention_weight_min": -25.738922119140625,
      "activations/layer23_attention_weight_max": 36.107635498046875,
      "activations/layer23_attention_weight_min": -20.4089298248291,
      "activations/layer2_attention_weight_max": 31.663955688476562,
      "activations/layer2_attention_weight_min": -30.603750228881836,
      "activations/layer3_attention_weight_max": 88.59427642822266,
      "activations/layer3_attention_weight_min": -89.88906860351562,
      "activations/layer4_attention_weight_max": 106.65865325927734,
      "activations/layer4_attention_weight_min": -100.14993286132812,
      "activations/layer5_attention_weight_max": 90.98604583740234,
      "activations/layer5_attention_weight_min": -85.919677734375,
      "activations/layer6_attention_weight_max": 54.92988204956055,
      "activations/layer6_attention_weight_min": -54.594112396240234,
      "activations/layer7_attention_weight_max": 76.66695404052734,
      "activations/layer7_attention_weight_min": -73.4063720703125,
      "activations/layer8_attention_weight_max": 44.58452606201172,
      "activations/layer8_attention_weight_min": -46.18047332763672,
      "activations/layer9_attention_weight_max": 48.5543098449707,
      "activations/layer9_attention_weight_min": -47.60247039794922,
      "epoch": 7.45,
      "learning_rate": 0.00010296174242424241,
      "loss": 2.8411,
      "step": 128300
    },
    {
      "activations/layer0_attention_weight_max": 14.701236724853516,
      "activations/layer0_attention_weight_min": -14.363465309143066,
      "activations/layer10_attention_weight_max": 37.17766189575195,
      "activations/layer10_attention_weight_min": -33.21819305419922,
      "activations/layer11_attention_weight_max": 30.553340911865234,
      "activations/layer11_attention_weight_min": -30.55850601196289,
      "activations/layer12_attention_weight_max": 19.22281837463379,
      "activations/layer12_attention_weight_min": -27.254493713378906,
      "activations/layer13_attention_weight_max": 48.60944747924805,
      "activations/layer13_attention_weight_min": -35.21091842651367,
      "activations/layer14_attention_weight_max": 52.34986114501953,
      "activations/layer14_attention_weight_min": -35.89458084106445,
      "activations/layer15_attention_weight_max": 43.420753479003906,
      "activations/layer15_attention_weight_min": -29.503768920898438,
      "activations/layer16_attention_weight_max": 33.37663269042969,
      "activations/layer16_attention_weight_min": -26.379858016967773,
      "activations/layer17_attention_weight_max": 52.22831344604492,
      "activations/layer17_attention_weight_min": -42.15988540649414,
      "activations/layer18_attention_weight_max": 45.26073455810547,
      "activations/layer18_attention_weight_min": -37.23430633544922,
      "activations/layer19_attention_weight_max": 22.284931182861328,
      "activations/layer19_attention_weight_min": -21.287593841552734,
      "activations/layer1_attention_weight_max": 17.46872329711914,
      "activations/layer1_attention_weight_min": -15.368365287780762,
      "activations/layer20_attention_weight_max": 21.802978515625,
      "activations/layer20_attention_weight_min": -23.52940559387207,
      "activations/layer21_attention_weight_max": 38.78636932373047,
      "activations/layer21_attention_weight_min": -25.541515350341797,
      "activations/layer22_attention_weight_max": 33.4798469543457,
      "activations/layer22_attention_weight_min": -28.344221115112305,
      "activations/layer23_attention_weight_max": 32.473812103271484,
      "activations/layer23_attention_weight_min": -25.537099838256836,
      "activations/layer2_attention_weight_max": 32.571495056152344,
      "activations/layer2_attention_weight_min": -29.435514450073242,
      "activations/layer3_attention_weight_max": 86.31159973144531,
      "activations/layer3_attention_weight_min": -92.63577270507812,
      "activations/layer4_attention_weight_max": 98.6169204711914,
      "activations/layer4_attention_weight_min": -98.21858978271484,
      "activations/layer5_attention_weight_max": 90.77608489990234,
      "activations/layer5_attention_weight_min": -82.615478515625,
      "activations/layer6_attention_weight_max": 58.29994201660156,
      "activations/layer6_attention_weight_min": -54.12674331665039,
      "activations/layer7_attention_weight_max": 77.60520935058594,
      "activations/layer7_attention_weight_min": -78.56657409667969,
      "activations/layer8_attention_weight_max": 46.17500686645508,
      "activations/layer8_attention_weight_min": -44.299644470214844,
      "activations/layer9_attention_weight_max": 52.173763275146484,
      "activations/layer9_attention_weight_min": -49.58671188354492,
      "epoch": 7.46,
      "learning_rate": 0.00010294280303030303,
      "loss": 2.8185,
      "step": 128350
    },
    {
      "activations/layer0_attention_weight_max": 14.969583511352539,
      "activations/layer0_attention_weight_min": -14.549036026000977,
      "activations/layer10_attention_weight_max": 36.60113525390625,
      "activations/layer10_attention_weight_min": -36.995941162109375,
      "activations/layer11_attention_weight_max": 31.288883209228516,
      "activations/layer11_attention_weight_min": -30.366086959838867,
      "activations/layer12_attention_weight_max": 20.52732276916504,
      "activations/layer12_attention_weight_min": -29.44355010986328,
      "activations/layer13_attention_weight_max": 39.706268310546875,
      "activations/layer13_attention_weight_min": -30.271080017089844,
      "activations/layer14_attention_weight_max": 33.901248931884766,
      "activations/layer14_attention_weight_min": -31.210254669189453,
      "activations/layer15_attention_weight_max": 33.98833084106445,
      "activations/layer15_attention_weight_min": -29.569822311401367,
      "activations/layer16_attention_weight_max": 35.27998733520508,
      "activations/layer16_attention_weight_min": -27.18687629699707,
      "activations/layer17_attention_weight_max": 49.21429443359375,
      "activations/layer17_attention_weight_min": -39.988792419433594,
      "activations/layer18_attention_weight_max": 49.25545883178711,
      "activations/layer18_attention_weight_min": -38.2083625793457,
      "activations/layer19_attention_weight_max": 22.856035232543945,
      "activations/layer19_attention_weight_min": -21.96332359313965,
      "activations/layer1_attention_weight_max": 18.211299896240234,
      "activations/layer1_attention_weight_min": -17.348873138427734,
      "activations/layer20_attention_weight_max": 22.145383834838867,
      "activations/layer20_attention_weight_min": -23.235443115234375,
      "activations/layer21_attention_weight_max": 38.031497955322266,
      "activations/layer21_attention_weight_min": -25.10626220703125,
      "activations/layer22_attention_weight_max": 32.54527282714844,
      "activations/layer22_attention_weight_min": -28.583467483520508,
      "activations/layer23_attention_weight_max": 34.53889465332031,
      "activations/layer23_attention_weight_min": -22.94417381286621,
      "activations/layer2_attention_weight_max": 33.32538604736328,
      "activations/layer2_attention_weight_min": -33.75352096557617,
      "activations/layer3_attention_weight_max": 85.44819641113281,
      "activations/layer3_attention_weight_min": -91.57698059082031,
      "activations/layer4_attention_weight_max": 100.0659408569336,
      "activations/layer4_attention_weight_min": -90.9598617553711,
      "activations/layer5_attention_weight_max": 89.37519836425781,
      "activations/layer5_attention_weight_min": -80.89077758789062,
      "activations/layer6_attention_weight_max": 55.426780700683594,
      "activations/layer6_attention_weight_min": -52.69048309326172,
      "activations/layer7_attention_weight_max": 77.93159484863281,
      "activations/layer7_attention_weight_min": -73.367431640625,
      "activations/layer8_attention_weight_max": 44.665313720703125,
      "activations/layer8_attention_weight_min": -44.433563232421875,
      "activations/layer9_attention_weight_max": 59.9028205871582,
      "activations/layer9_attention_weight_min": -49.093502044677734,
      "epoch": 7.46,
      "learning_rate": 0.00010292386363636363,
      "loss": 2.8069,
      "step": 128400
    },
    {
      "activations/layer0_attention_weight_max": 14.683796882629395,
      "activations/layer0_attention_weight_min": -15.207602500915527,
      "activations/layer10_attention_weight_max": 38.80241775512695,
      "activations/layer10_attention_weight_min": -37.54633712768555,
      "activations/layer11_attention_weight_max": 31.64411163330078,
      "activations/layer11_attention_weight_min": -31.368505477905273,
      "activations/layer12_attention_weight_max": 29.471330642700195,
      "activations/layer12_attention_weight_min": -23.926498413085938,
      "activations/layer13_attention_weight_max": 43.36223220825195,
      "activations/layer13_attention_weight_min": -34.446205139160156,
      "activations/layer14_attention_weight_max": 37.20207214355469,
      "activations/layer14_attention_weight_min": -32.837249755859375,
      "activations/layer15_attention_weight_max": 36.326080322265625,
      "activations/layer15_attention_weight_min": -33.577178955078125,
      "activations/layer16_attention_weight_max": 32.37276077270508,
      "activations/layer16_attention_weight_min": -29.854276657104492,
      "activations/layer17_attention_weight_max": 57.46084976196289,
      "activations/layer17_attention_weight_min": -41.17158889770508,
      "activations/layer18_attention_weight_max": 48.57146453857422,
      "activations/layer18_attention_weight_min": -36.23780059814453,
      "activations/layer19_attention_weight_max": 25.367773056030273,
      "activations/layer19_attention_weight_min": -21.116249084472656,
      "activations/layer1_attention_weight_max": 16.7385196685791,
      "activations/layer1_attention_weight_min": -14.747496604919434,
      "activations/layer20_attention_weight_max": 22.46044158935547,
      "activations/layer20_attention_weight_min": -21.910661697387695,
      "activations/layer21_attention_weight_max": 40.85858154296875,
      "activations/layer21_attention_weight_min": -25.16253089904785,
      "activations/layer22_attention_weight_max": 32.30875015258789,
      "activations/layer22_attention_weight_min": -25.17618179321289,
      "activations/layer23_attention_weight_max": 34.48810577392578,
      "activations/layer23_attention_weight_min": -20.056087493896484,
      "activations/layer2_attention_weight_max": 29.667325973510742,
      "activations/layer2_attention_weight_min": -28.121854782104492,
      "activations/layer3_attention_weight_max": 83.31292724609375,
      "activations/layer3_attention_weight_min": -84.6065673828125,
      "activations/layer4_attention_weight_max": 99.19342803955078,
      "activations/layer4_attention_weight_min": -95.1855697631836,
      "activations/layer5_attention_weight_max": 87.9327621459961,
      "activations/layer5_attention_weight_min": -85.69417572021484,
      "activations/layer6_attention_weight_max": 52.98775863647461,
      "activations/layer6_attention_weight_min": -52.26323318481445,
      "activations/layer7_attention_weight_max": 77.87671661376953,
      "activations/layer7_attention_weight_min": -72.46234893798828,
      "activations/layer8_attention_weight_max": 46.94835662841797,
      "activations/layer8_attention_weight_min": -45.67805480957031,
      "activations/layer9_attention_weight_max": 53.8754768371582,
      "activations/layer9_attention_weight_min": -48.712947845458984,
      "epoch": 7.46,
      "learning_rate": 0.00010290492424242424,
      "loss": 2.8148,
      "step": 128450
    },
    {
      "activations/layer0_attention_weight_max": 14.462325096130371,
      "activations/layer0_attention_weight_min": -14.719355583190918,
      "activations/layer10_attention_weight_max": 37.89527130126953,
      "activations/layer10_attention_weight_min": -34.99113082885742,
      "activations/layer11_attention_weight_max": 29.753742218017578,
      "activations/layer11_attention_weight_min": -31.754819869995117,
      "activations/layer12_attention_weight_max": 22.096147537231445,
      "activations/layer12_attention_weight_min": -27.597129821777344,
      "activations/layer13_attention_weight_max": 38.359336853027344,
      "activations/layer13_attention_weight_min": -29.029150009155273,
      "activations/layer14_attention_weight_max": 36.80750274658203,
      "activations/layer14_attention_weight_min": -30.622621536254883,
      "activations/layer15_attention_weight_max": 35.47993087768555,
      "activations/layer15_attention_weight_min": -29.285598754882812,
      "activations/layer16_attention_weight_max": 28.963542938232422,
      "activations/layer16_attention_weight_min": -24.71826171875,
      "activations/layer17_attention_weight_max": 54.85013961791992,
      "activations/layer17_attention_weight_min": -42.837310791015625,
      "activations/layer18_attention_weight_max": 43.4732666015625,
      "activations/layer18_attention_weight_min": -35.243988037109375,
      "activations/layer19_attention_weight_max": 21.990018844604492,
      "activations/layer19_attention_weight_min": -21.949695587158203,
      "activations/layer1_attention_weight_max": 18.372997283935547,
      "activations/layer1_attention_weight_min": -15.017548561096191,
      "activations/layer20_attention_weight_max": 21.266624450683594,
      "activations/layer20_attention_weight_min": -21.60188865661621,
      "activations/layer21_attention_weight_max": 32.18887710571289,
      "activations/layer21_attention_weight_min": -21.9084529876709,
      "activations/layer22_attention_weight_max": 30.662900924682617,
      "activations/layer22_attention_weight_min": -23.293163299560547,
      "activations/layer23_attention_weight_max": 31.996734619140625,
      "activations/layer23_attention_weight_min": -21.520374298095703,
      "activations/layer2_attention_weight_max": 31.355573654174805,
      "activations/layer2_attention_weight_min": -31.542186737060547,
      "activations/layer3_attention_weight_max": 82.51464080810547,
      "activations/layer3_attention_weight_min": -89.56761169433594,
      "activations/layer4_attention_weight_max": 98.28599548339844,
      "activations/layer4_attention_weight_min": -93.07901763916016,
      "activations/layer5_attention_weight_max": 90.00697326660156,
      "activations/layer5_attention_weight_min": -77.68295288085938,
      "activations/layer6_attention_weight_max": 52.92362594604492,
      "activations/layer6_attention_weight_min": -49.918052673339844,
      "activations/layer7_attention_weight_max": 75.88340759277344,
      "activations/layer7_attention_weight_min": -70.270263671875,
      "activations/layer8_attention_weight_max": 43.7253303527832,
      "activations/layer8_attention_weight_min": -43.195350646972656,
      "activations/layer9_attention_weight_max": 47.11749267578125,
      "activations/layer9_attention_weight_min": -46.654964447021484,
      "epoch": 7.47,
      "learning_rate": 0.00010288598484848483,
      "loss": 2.8277,
      "step": 128500
    },
    {
      "activations/layer0_attention_weight_max": 14.820755958557129,
      "activations/layer0_attention_weight_min": -14.866663932800293,
      "activations/layer10_attention_weight_max": 35.79531478881836,
      "activations/layer10_attention_weight_min": -33.34532928466797,
      "activations/layer11_attention_weight_max": 28.532066345214844,
      "activations/layer11_attention_weight_min": -29.901687622070312,
      "activations/layer12_attention_weight_max": 23.169137954711914,
      "activations/layer12_attention_weight_min": -24.244007110595703,
      "activations/layer13_attention_weight_max": 42.350341796875,
      "activations/layer13_attention_weight_min": -35.9083366394043,
      "activations/layer14_attention_weight_max": 43.06691360473633,
      "activations/layer14_attention_weight_min": -35.02278518676758,
      "activations/layer15_attention_weight_max": 34.6217155456543,
      "activations/layer15_attention_weight_min": -28.642698287963867,
      "activations/layer16_attention_weight_max": 34.8576774597168,
      "activations/layer16_attention_weight_min": -27.101741790771484,
      "activations/layer17_attention_weight_max": 55.46119689941406,
      "activations/layer17_attention_weight_min": -41.18622970581055,
      "activations/layer18_attention_weight_max": 49.407501220703125,
      "activations/layer18_attention_weight_min": -34.3369255065918,
      "activations/layer19_attention_weight_max": 23.546524047851562,
      "activations/layer19_attention_weight_min": -20.56481170654297,
      "activations/layer1_attention_weight_max": 16.82350730895996,
      "activations/layer1_attention_weight_min": -15.61712646484375,
      "activations/layer20_attention_weight_max": 23.397409439086914,
      "activations/layer20_attention_weight_min": -25.088306427001953,
      "activations/layer21_attention_weight_max": 38.53759765625,
      "activations/layer21_attention_weight_min": -22.83671760559082,
      "activations/layer22_attention_weight_max": 34.833885192871094,
      "activations/layer22_attention_weight_min": -24.638872146606445,
      "activations/layer23_attention_weight_max": 42.314056396484375,
      "activations/layer23_attention_weight_min": -21.751365661621094,
      "activations/layer2_attention_weight_max": 30.450300216674805,
      "activations/layer2_attention_weight_min": -30.252113342285156,
      "activations/layer3_attention_weight_max": 84.89849853515625,
      "activations/layer3_attention_weight_min": -88.13024139404297,
      "activations/layer4_attention_weight_max": 97.29399871826172,
      "activations/layer4_attention_weight_min": -95.0028305053711,
      "activations/layer5_attention_weight_max": 86.51446533203125,
      "activations/layer5_attention_weight_min": -81.07421875,
      "activations/layer6_attention_weight_max": 53.787376403808594,
      "activations/layer6_attention_weight_min": -49.29069519042969,
      "activations/layer7_attention_weight_max": 71.46173858642578,
      "activations/layer7_attention_weight_min": -71.82235717773438,
      "activations/layer8_attention_weight_max": 45.295475006103516,
      "activations/layer8_attention_weight_min": -44.47172927856445,
      "activations/layer9_attention_weight_max": 50.372840881347656,
      "activations/layer9_attention_weight_min": -48.79375457763672,
      "epoch": 7.47,
      "learning_rate": 0.00010286704545454544,
      "loss": 2.8053,
      "step": 128550
    },
    {
      "activations/layer0_attention_weight_max": 15.03216552734375,
      "activations/layer0_attention_weight_min": -14.628507614135742,
      "activations/layer10_attention_weight_max": 34.61977005004883,
      "activations/layer10_attention_weight_min": -34.337196350097656,
      "activations/layer11_attention_weight_max": 28.55795669555664,
      "activations/layer11_attention_weight_min": -30.96670150756836,
      "activations/layer12_attention_weight_max": 19.59025001525879,
      "activations/layer12_attention_weight_min": -24.317665100097656,
      "activations/layer13_attention_weight_max": 42.67593765258789,
      "activations/layer13_attention_weight_min": -30.92094612121582,
      "activations/layer14_attention_weight_max": 39.49174880981445,
      "activations/layer14_attention_weight_min": -30.26258659362793,
      "activations/layer15_attention_weight_max": 34.13453674316406,
      "activations/layer15_attention_weight_min": -32.992671966552734,
      "activations/layer16_attention_weight_max": 30.159399032592773,
      "activations/layer16_attention_weight_min": -28.434539794921875,
      "activations/layer17_attention_weight_max": 55.83015060424805,
      "activations/layer17_attention_weight_min": -42.023162841796875,
      "activations/layer18_attention_weight_max": 48.28252410888672,
      "activations/layer18_attention_weight_min": -37.931941986083984,
      "activations/layer19_attention_weight_max": 23.834199905395508,
      "activations/layer19_attention_weight_min": -22.93677520751953,
      "activations/layer1_attention_weight_max": 16.47443962097168,
      "activations/layer1_attention_weight_min": -15.719842910766602,
      "activations/layer20_attention_weight_max": 22.242206573486328,
      "activations/layer20_attention_weight_min": -21.787044525146484,
      "activations/layer21_attention_weight_max": 39.54448699951172,
      "activations/layer21_attention_weight_min": -24.970529556274414,
      "activations/layer22_attention_weight_max": 32.75874328613281,
      "activations/layer22_attention_weight_min": -26.381122589111328,
      "activations/layer23_attention_weight_max": 34.269500732421875,
      "activations/layer23_attention_weight_min": -19.58169174194336,
      "activations/layer2_attention_weight_max": 31.263410568237305,
      "activations/layer2_attention_weight_min": -29.07314682006836,
      "activations/layer3_attention_weight_max": 89.7843017578125,
      "activations/layer3_attention_weight_min": -94.74577331542969,
      "activations/layer4_attention_weight_max": 97.92764282226562,
      "activations/layer4_attention_weight_min": -106.31963348388672,
      "activations/layer5_attention_weight_max": 81.93663024902344,
      "activations/layer5_attention_weight_min": -84.83695220947266,
      "activations/layer6_attention_weight_max": 53.256752014160156,
      "activations/layer6_attention_weight_min": -52.7391471862793,
      "activations/layer7_attention_weight_max": 73.69634246826172,
      "activations/layer7_attention_weight_min": -73.11760711669922,
      "activations/layer8_attention_weight_max": 42.19862365722656,
      "activations/layer8_attention_weight_min": -42.61380386352539,
      "activations/layer9_attention_weight_max": 49.01584243774414,
      "activations/layer9_attention_weight_min": -49.829505920410156,
      "epoch": 7.47,
      "learning_rate": 0.00010284810606060606,
      "loss": 2.8067,
      "step": 128600
    },
    {
      "activations/layer0_attention_weight_max": 13.847370147705078,
      "activations/layer0_attention_weight_min": -15.244597434997559,
      "activations/layer10_attention_weight_max": 38.38053512573242,
      "activations/layer10_attention_weight_min": -37.159542083740234,
      "activations/layer11_attention_weight_max": 31.548839569091797,
      "activations/layer11_attention_weight_min": -32.60928726196289,
      "activations/layer12_attention_weight_max": 24.36620330810547,
      "activations/layer12_attention_weight_min": -24.204116821289062,
      "activations/layer13_attention_weight_max": 47.93046951293945,
      "activations/layer13_attention_weight_min": -33.46135711669922,
      "activations/layer14_attention_weight_max": 43.260379791259766,
      "activations/layer14_attention_weight_min": -32.78751754760742,
      "activations/layer15_attention_weight_max": 35.081199645996094,
      "activations/layer15_attention_weight_min": -32.33734893798828,
      "activations/layer16_attention_weight_max": 34.57933807373047,
      "activations/layer16_attention_weight_min": -27.79136848449707,
      "activations/layer17_attention_weight_max": 58.0703125,
      "activations/layer17_attention_weight_min": -45.837547302246094,
      "activations/layer18_attention_weight_max": 46.335025787353516,
      "activations/layer18_attention_weight_min": -36.35926818847656,
      "activations/layer19_attention_weight_max": 21.313980102539062,
      "activations/layer19_attention_weight_min": -19.482975006103516,
      "activations/layer1_attention_weight_max": 16.395925521850586,
      "activations/layer1_attention_weight_min": -15.661314010620117,
      "activations/layer20_attention_weight_max": 19.043468475341797,
      "activations/layer20_attention_weight_min": -19.826406478881836,
      "activations/layer21_attention_weight_max": 40.69925308227539,
      "activations/layer21_attention_weight_min": -25.69873046875,
      "activations/layer22_attention_weight_max": 31.738977432250977,
      "activations/layer22_attention_weight_min": -26.896718978881836,
      "activations/layer23_attention_weight_max": 30.109119415283203,
      "activations/layer23_attention_weight_min": -21.533103942871094,
      "activations/layer2_attention_weight_max": 33.17889404296875,
      "activations/layer2_attention_weight_min": -31.685871124267578,
      "activations/layer3_attention_weight_max": 94.77051544189453,
      "activations/layer3_attention_weight_min": -98.66789245605469,
      "activations/layer4_attention_weight_max": 103.60453033447266,
      "activations/layer4_attention_weight_min": -106.41446685791016,
      "activations/layer5_attention_weight_max": 96.93516540527344,
      "activations/layer5_attention_weight_min": -85.80027770996094,
      "activations/layer6_attention_weight_max": 55.60436248779297,
      "activations/layer6_attention_weight_min": -53.05473709106445,
      "activations/layer7_attention_weight_max": 77.83158111572266,
      "activations/layer7_attention_weight_min": -73.19568634033203,
      "activations/layer8_attention_weight_max": 46.41450500488281,
      "activations/layer8_attention_weight_min": -46.52572250366211,
      "activations/layer9_attention_weight_max": 52.4621467590332,
      "activations/layer9_attention_weight_min": -51.13309860229492,
      "epoch": 7.48,
      "learning_rate": 0.00010282916666666665,
      "loss": 2.8066,
      "step": 128650
    },
    {
      "activations/layer0_attention_weight_max": 14.418850898742676,
      "activations/layer0_attention_weight_min": -14.788736343383789,
      "activations/layer10_attention_weight_max": 36.406349182128906,
      "activations/layer10_attention_weight_min": -35.8988151550293,
      "activations/layer11_attention_weight_max": 29.860593795776367,
      "activations/layer11_attention_weight_min": -29.536956787109375,
      "activations/layer12_attention_weight_max": 22.056222915649414,
      "activations/layer12_attention_weight_min": -29.48019790649414,
      "activations/layer13_attention_weight_max": 42.976078033447266,
      "activations/layer13_attention_weight_min": -32.42061233520508,
      "activations/layer14_attention_weight_max": 47.31632614135742,
      "activations/layer14_attention_weight_min": -32.340614318847656,
      "activations/layer15_attention_weight_max": 40.88055419921875,
      "activations/layer15_attention_weight_min": -30.187088012695312,
      "activations/layer16_attention_weight_max": 31.436967849731445,
      "activations/layer16_attention_weight_min": -29.42108726501465,
      "activations/layer17_attention_weight_max": 60.96803283691406,
      "activations/layer17_attention_weight_min": -42.81264877319336,
      "activations/layer18_attention_weight_max": 47.08183670043945,
      "activations/layer18_attention_weight_min": -37.531131744384766,
      "activations/layer19_attention_weight_max": 23.518770217895508,
      "activations/layer19_attention_weight_min": -20.98311424255371,
      "activations/layer1_attention_weight_max": 16.37693214416504,
      "activations/layer1_attention_weight_min": -14.355999946594238,
      "activations/layer20_attention_weight_max": 21.386234283447266,
      "activations/layer20_attention_weight_min": -21.39751434326172,
      "activations/layer21_attention_weight_max": 44.31991958618164,
      "activations/layer21_attention_weight_min": -24.574052810668945,
      "activations/layer22_attention_weight_max": 31.50507926940918,
      "activations/layer22_attention_weight_min": -26.826129913330078,
      "activations/layer23_attention_weight_max": 35.6986083984375,
      "activations/layer23_attention_weight_min": -20.628482818603516,
      "activations/layer2_attention_weight_max": 30.884727478027344,
      "activations/layer2_attention_weight_min": -29.50428009033203,
      "activations/layer3_attention_weight_max": 85.97125244140625,
      "activations/layer3_attention_weight_min": -84.96062469482422,
      "activations/layer4_attention_weight_max": 101.53079223632812,
      "activations/layer4_attention_weight_min": -92.20549774169922,
      "activations/layer5_attention_weight_max": 86.6530990600586,
      "activations/layer5_attention_weight_min": -82.39512634277344,
      "activations/layer6_attention_weight_max": 53.02942657470703,
      "activations/layer6_attention_weight_min": -52.39042282104492,
      "activations/layer7_attention_weight_max": 75.43048095703125,
      "activations/layer7_attention_weight_min": -68.4805908203125,
      "activations/layer8_attention_weight_max": 42.59972381591797,
      "activations/layer8_attention_weight_min": -44.61637878417969,
      "activations/layer9_attention_weight_max": 49.39106369018555,
      "activations/layer9_attention_weight_min": -47.16556930541992,
      "epoch": 7.48,
      "learning_rate": 0.00010281022727272726,
      "loss": 2.8192,
      "step": 128700
    },
    {
      "activations/layer0_attention_weight_max": 14.64539623260498,
      "activations/layer0_attention_weight_min": -14.429030418395996,
      "activations/layer10_attention_weight_max": 37.63605499267578,
      "activations/layer10_attention_weight_min": -36.65536117553711,
      "activations/layer11_attention_weight_max": 30.40129852294922,
      "activations/layer11_attention_weight_min": -34.53797149658203,
      "activations/layer12_attention_weight_max": 19.96871566772461,
      "activations/layer12_attention_weight_min": -25.482770919799805,
      "activations/layer13_attention_weight_max": 52.44853591918945,
      "activations/layer13_attention_weight_min": -43.20112991333008,
      "activations/layer14_attention_weight_max": 43.48400115966797,
      "activations/layer14_attention_weight_min": -32.509281158447266,
      "activations/layer15_attention_weight_max": 36.07649230957031,
      "activations/layer15_attention_weight_min": -29.079076766967773,
      "activations/layer16_attention_weight_max": 34.3678092956543,
      "activations/layer16_attention_weight_min": -27.881973266601562,
      "activations/layer17_attention_weight_max": 56.03911209106445,
      "activations/layer17_attention_weight_min": -43.41324996948242,
      "activations/layer18_attention_weight_max": 51.7835693359375,
      "activations/layer18_attention_weight_min": -36.00775146484375,
      "activations/layer19_attention_weight_max": 21.371810913085938,
      "activations/layer19_attention_weight_min": -21.01993179321289,
      "activations/layer1_attention_weight_max": 17.55603790283203,
      "activations/layer1_attention_weight_min": -15.225893020629883,
      "activations/layer20_attention_weight_max": 24.291772842407227,
      "activations/layer20_attention_weight_min": -23.785594940185547,
      "activations/layer21_attention_weight_max": 44.04365921020508,
      "activations/layer21_attention_weight_min": -24.243576049804688,
      "activations/layer22_attention_weight_max": 38.129093170166016,
      "activations/layer22_attention_weight_min": -26.108131408691406,
      "activations/layer23_attention_weight_max": 39.21753692626953,
      "activations/layer23_attention_weight_min": -20.294429779052734,
      "activations/layer2_attention_weight_max": 31.97136688232422,
      "activations/layer2_attention_weight_min": -30.05914878845215,
      "activations/layer3_attention_weight_max": 87.93439483642578,
      "activations/layer3_attention_weight_min": -87.92255401611328,
      "activations/layer4_attention_weight_max": 101.3261947631836,
      "activations/layer4_attention_weight_min": -96.64158630371094,
      "activations/layer5_attention_weight_max": 88.79183197021484,
      "activations/layer5_attention_weight_min": -85.02586364746094,
      "activations/layer6_attention_weight_max": 57.860984802246094,
      "activations/layer6_attention_weight_min": -54.046974182128906,
      "activations/layer7_attention_weight_max": 74.11847686767578,
      "activations/layer7_attention_weight_min": -70.25770568847656,
      "activations/layer8_attention_weight_max": 43.361228942871094,
      "activations/layer8_attention_weight_min": -46.32436752319336,
      "activations/layer9_attention_weight_max": 50.46479415893555,
      "activations/layer9_attention_weight_min": -47.469112396240234,
      "epoch": 7.48,
      "learning_rate": 0.00010279128787878788,
      "loss": 2.8248,
      "step": 128750
    },
    {
      "activations/layer0_attention_weight_max": 14.699334144592285,
      "activations/layer0_attention_weight_min": -14.762547492980957,
      "activations/layer10_attention_weight_max": 36.72162628173828,
      "activations/layer10_attention_weight_min": -35.49650955200195,
      "activations/layer11_attention_weight_max": 30.267135620117188,
      "activations/layer11_attention_weight_min": -33.48734664916992,
      "activations/layer12_attention_weight_max": 22.111557006835938,
      "activations/layer12_attention_weight_min": -26.467744827270508,
      "activations/layer13_attention_weight_max": 41.014244079589844,
      "activations/layer13_attention_weight_min": -34.205780029296875,
      "activations/layer14_attention_weight_max": 41.2703742980957,
      "activations/layer14_attention_weight_min": -30.753694534301758,
      "activations/layer15_attention_weight_max": 34.82420349121094,
      "activations/layer15_attention_weight_min": -28.753162384033203,
      "activations/layer16_attention_weight_max": 33.354530334472656,
      "activations/layer16_attention_weight_min": -28.160715103149414,
      "activations/layer17_attention_weight_max": 49.53548049926758,
      "activations/layer17_attention_weight_min": -42.60626220703125,
      "activations/layer18_attention_weight_max": 45.54348373413086,
      "activations/layer18_attention_weight_min": -33.802085876464844,
      "activations/layer19_attention_weight_max": 25.067941665649414,
      "activations/layer19_attention_weight_min": -21.713565826416016,
      "activations/layer1_attention_weight_max": 16.825864791870117,
      "activations/layer1_attention_weight_min": -16.39432716369629,
      "activations/layer20_attention_weight_max": 23.361888885498047,
      "activations/layer20_attention_weight_min": -23.05926513671875,
      "activations/layer21_attention_weight_max": 40.691986083984375,
      "activations/layer21_attention_weight_min": -25.129289627075195,
      "activations/layer22_attention_weight_max": 36.11471176147461,
      "activations/layer22_attention_weight_min": -29.67034149169922,
      "activations/layer23_attention_weight_max": 31.56600570678711,
      "activations/layer23_attention_weight_min": -22.47763442993164,
      "activations/layer2_attention_weight_max": 31.380390167236328,
      "activations/layer2_attention_weight_min": -30.661205291748047,
      "activations/layer3_attention_weight_max": 83.56211853027344,
      "activations/layer3_attention_weight_min": -87.47608184814453,
      "activations/layer4_attention_weight_max": 97.85675811767578,
      "activations/layer4_attention_weight_min": -97.28401184082031,
      "activations/layer5_attention_weight_max": 86.61660766601562,
      "activations/layer5_attention_weight_min": -81.90262603759766,
      "activations/layer6_attention_weight_max": 56.44361877441406,
      "activations/layer6_attention_weight_min": -53.91950225830078,
      "activations/layer7_attention_weight_max": 80.8062744140625,
      "activations/layer7_attention_weight_min": -72.24314880371094,
      "activations/layer8_attention_weight_max": 46.03691482543945,
      "activations/layer8_attention_weight_min": -46.5021858215332,
      "activations/layer9_attention_weight_max": 54.61606979370117,
      "activations/layer9_attention_weight_min": -50.35763168334961,
      "epoch": 7.48,
      "learning_rate": 0.00010277234848484847,
      "loss": 2.8192,
      "step": 128800
    },
    {
      "activations/layer0_attention_weight_max": 16.729686737060547,
      "activations/layer0_attention_weight_min": -15.14134407043457,
      "activations/layer10_attention_weight_max": 39.36908721923828,
      "activations/layer10_attention_weight_min": -37.426090240478516,
      "activations/layer11_attention_weight_max": 33.4885368347168,
      "activations/layer11_attention_weight_min": -30.865711212158203,
      "activations/layer12_attention_weight_max": 23.663633346557617,
      "activations/layer12_attention_weight_min": -28.432836532592773,
      "activations/layer13_attention_weight_max": 45.28525161743164,
      "activations/layer13_attention_weight_min": -35.703651428222656,
      "activations/layer14_attention_weight_max": 39.97567367553711,
      "activations/layer14_attention_weight_min": -32.30393600463867,
      "activations/layer15_attention_weight_max": 37.978599548339844,
      "activations/layer15_attention_weight_min": -30.73358726501465,
      "activations/layer16_attention_weight_max": 34.46642303466797,
      "activations/layer16_attention_weight_min": -28.006929397583008,
      "activations/layer17_attention_weight_max": 56.28205490112305,
      "activations/layer17_attention_weight_min": -48.83766555786133,
      "activations/layer18_attention_weight_max": 50.93825149536133,
      "activations/layer18_attention_weight_min": -37.60069274902344,
      "activations/layer19_attention_weight_max": 27.16596794128418,
      "activations/layer19_attention_weight_min": -20.705324172973633,
      "activations/layer1_attention_weight_max": 16.980783462524414,
      "activations/layer1_attention_weight_min": -14.33314037322998,
      "activations/layer20_attention_weight_max": 22.768016815185547,
      "activations/layer20_attention_weight_min": -23.551122665405273,
      "activations/layer21_attention_weight_max": 39.526885986328125,
      "activations/layer21_attention_weight_min": -23.232704162597656,
      "activations/layer22_attention_weight_max": 33.105003356933594,
      "activations/layer22_attention_weight_min": -24.110107421875,
      "activations/layer23_attention_weight_max": 35.94917678833008,
      "activations/layer23_attention_weight_min": -20.317716598510742,
      "activations/layer2_attention_weight_max": 32.61882400512695,
      "activations/layer2_attention_weight_min": -29.846942901611328,
      "activations/layer3_attention_weight_max": 84.87307739257812,
      "activations/layer3_attention_weight_min": -86.3296890258789,
      "activations/layer4_attention_weight_max": 101.75208282470703,
      "activations/layer4_attention_weight_min": -95.74310302734375,
      "activations/layer5_attention_weight_max": 92.00203704833984,
      "activations/layer5_attention_weight_min": -85.19811248779297,
      "activations/layer6_attention_weight_max": 56.17716598510742,
      "activations/layer6_attention_weight_min": -52.871585845947266,
      "activations/layer7_attention_weight_max": 77.96833801269531,
      "activations/layer7_attention_weight_min": -74.8045883178711,
      "activations/layer8_attention_weight_max": 45.88643264770508,
      "activations/layer8_attention_weight_min": -49.22117614746094,
      "activations/layer9_attention_weight_max": 53.47510528564453,
      "activations/layer9_attention_weight_min": -48.76010513305664,
      "epoch": 7.49,
      "learning_rate": 0.00010275340909090908,
      "loss": 2.8211,
      "step": 128850
    },
    {
      "activations/layer0_attention_weight_max": 16.882001876831055,
      "activations/layer0_attention_weight_min": -15.142416954040527,
      "activations/layer10_attention_weight_max": 37.9821891784668,
      "activations/layer10_attention_weight_min": -34.95880126953125,
      "activations/layer11_attention_weight_max": 30.439125061035156,
      "activations/layer11_attention_weight_min": -31.74596405029297,
      "activations/layer12_attention_weight_max": 19.222929000854492,
      "activations/layer12_attention_weight_min": -25.238666534423828,
      "activations/layer13_attention_weight_max": 44.30355453491211,
      "activations/layer13_attention_weight_min": -34.41541290283203,
      "activations/layer14_attention_weight_max": 47.76335525512695,
      "activations/layer14_attention_weight_min": -30.768280029296875,
      "activations/layer15_attention_weight_max": 38.118614196777344,
      "activations/layer15_attention_weight_min": -29.370586395263672,
      "activations/layer16_attention_weight_max": 31.705793380737305,
      "activations/layer16_attention_weight_min": -28.881175994873047,
      "activations/layer17_attention_weight_max": 56.32671356201172,
      "activations/layer17_attention_weight_min": -42.03044128417969,
      "activations/layer18_attention_weight_max": 48.134822845458984,
      "activations/layer18_attention_weight_min": -36.36490249633789,
      "activations/layer19_attention_weight_max": 25.451038360595703,
      "activations/layer19_attention_weight_min": -20.604795455932617,
      "activations/layer1_attention_weight_max": 18.547550201416016,
      "activations/layer1_attention_weight_min": -17.00958251953125,
      "activations/layer20_attention_weight_max": 22.861656188964844,
      "activations/layer20_attention_weight_min": -24.881452560424805,
      "activations/layer21_attention_weight_max": 38.78194808959961,
      "activations/layer21_attention_weight_min": -24.783824920654297,
      "activations/layer22_attention_weight_max": 33.982078552246094,
      "activations/layer22_attention_weight_min": -26.149547576904297,
      "activations/layer23_attention_weight_max": 35.08967590332031,
      "activations/layer23_attention_weight_min": -22.435211181640625,
      "activations/layer2_attention_weight_max": 30.361913681030273,
      "activations/layer2_attention_weight_min": -27.682851791381836,
      "activations/layer3_attention_weight_max": 87.5000991821289,
      "activations/layer3_attention_weight_min": -86.45941925048828,
      "activations/layer4_attention_weight_max": 99.77536010742188,
      "activations/layer4_attention_weight_min": -95.09899139404297,
      "activations/layer5_attention_weight_max": 90.45951843261719,
      "activations/layer5_attention_weight_min": -83.56434631347656,
      "activations/layer6_attention_weight_max": 54.95487976074219,
      "activations/layer6_attention_weight_min": -56.3768196105957,
      "activations/layer7_attention_weight_max": 75.3514175415039,
      "activations/layer7_attention_weight_min": -70.55574798583984,
      "activations/layer8_attention_weight_max": 43.410362243652344,
      "activations/layer8_attention_weight_min": -44.091800689697266,
      "activations/layer9_attention_weight_max": 49.79056930541992,
      "activations/layer9_attention_weight_min": -49.66775131225586,
      "epoch": 7.49,
      "learning_rate": 0.0001027344696969697,
      "loss": 2.814,
      "step": 128900
    },
    {
      "activations/layer0_attention_weight_max": 14.73603343963623,
      "activations/layer0_attention_weight_min": -14.863813400268555,
      "activations/layer10_attention_weight_max": 40.192203521728516,
      "activations/layer10_attention_weight_min": -37.38176345825195,
      "activations/layer11_attention_weight_max": 30.983631134033203,
      "activations/layer11_attention_weight_min": -31.367918014526367,
      "activations/layer12_attention_weight_max": 21.611507415771484,
      "activations/layer12_attention_weight_min": -26.4084415435791,
      "activations/layer13_attention_weight_max": 49.12673568725586,
      "activations/layer13_attention_weight_min": -36.8603630065918,
      "activations/layer14_attention_weight_max": 40.303565979003906,
      "activations/layer14_attention_weight_min": -30.50465202331543,
      "activations/layer15_attention_weight_max": 36.94121551513672,
      "activations/layer15_attention_weight_min": -29.44657325744629,
      "activations/layer16_attention_weight_max": 33.28238296508789,
      "activations/layer16_attention_weight_min": -25.91169548034668,
      "activations/layer17_attention_weight_max": 57.43820571899414,
      "activations/layer17_attention_weight_min": -44.80412673950195,
      "activations/layer18_attention_weight_max": 52.0251579284668,
      "activations/layer18_attention_weight_min": -38.196868896484375,
      "activations/layer19_attention_weight_max": 24.346111297607422,
      "activations/layer19_attention_weight_min": -22.6512508392334,
      "activations/layer1_attention_weight_max": 17.48920440673828,
      "activations/layer1_attention_weight_min": -14.32448959350586,
      "activations/layer20_attention_weight_max": 24.489089965820312,
      "activations/layer20_attention_weight_min": -22.00222396850586,
      "activations/layer21_attention_weight_max": 39.568817138671875,
      "activations/layer21_attention_weight_min": -27.189786911010742,
      "activations/layer22_attention_weight_max": 33.71168518066406,
      "activations/layer22_attention_weight_min": -25.70384407043457,
      "activations/layer23_attention_weight_max": 37.357749938964844,
      "activations/layer23_attention_weight_min": -23.47018051147461,
      "activations/layer2_attention_weight_max": 30.749418258666992,
      "activations/layer2_attention_weight_min": -27.89289093017578,
      "activations/layer3_attention_weight_max": 85.71833038330078,
      "activations/layer3_attention_weight_min": -89.29269409179688,
      "activations/layer4_attention_weight_max": 100.96797943115234,
      "activations/layer4_attention_weight_min": -98.35335540771484,
      "activations/layer5_attention_weight_max": 87.41752624511719,
      "activations/layer5_attention_weight_min": -84.95478820800781,
      "activations/layer6_attention_weight_max": 57.062503814697266,
      "activations/layer6_attention_weight_min": -55.09650421142578,
      "activations/layer7_attention_weight_max": 81.12496185302734,
      "activations/layer7_attention_weight_min": -76.7017822265625,
      "activations/layer8_attention_weight_max": 47.76195526123047,
      "activations/layer8_attention_weight_min": -51.0131721496582,
      "activations/layer9_attention_weight_max": 54.5294075012207,
      "activations/layer9_attention_weight_min": -50.69607162475586,
      "epoch": 7.49,
      "learning_rate": 0.0001027155303030303,
      "loss": 2.8096,
      "step": 128950
    },
    {
      "activations/layer0_attention_weight_max": 14.34924030303955,
      "activations/layer0_attention_weight_min": -14.67187213897705,
      "activations/layer10_attention_weight_max": 38.985897064208984,
      "activations/layer10_attention_weight_min": -36.46290969848633,
      "activations/layer11_attention_weight_max": 30.72077751159668,
      "activations/layer11_attention_weight_min": -31.520763397216797,
      "activations/layer12_attention_weight_max": 23.338130950927734,
      "activations/layer12_attention_weight_min": -26.529438018798828,
      "activations/layer13_attention_weight_max": 42.861454010009766,
      "activations/layer13_attention_weight_min": -33.445030212402344,
      "activations/layer14_attention_weight_max": 36.61973190307617,
      "activations/layer14_attention_weight_min": -31.26654052734375,
      "activations/layer15_attention_weight_max": 32.66938018798828,
      "activations/layer15_attention_weight_min": -30.502635955810547,
      "activations/layer16_attention_weight_max": 32.55142593383789,
      "activations/layer16_attention_weight_min": -29.341644287109375,
      "activations/layer17_attention_weight_max": 50.48649978637695,
      "activations/layer17_attention_weight_min": -42.20121765136719,
      "activations/layer18_attention_weight_max": 45.196937561035156,
      "activations/layer18_attention_weight_min": -34.256004333496094,
      "activations/layer19_attention_weight_max": 17.90338897705078,
      "activations/layer19_attention_weight_min": -18.72987174987793,
      "activations/layer1_attention_weight_max": 17.29287338256836,
      "activations/layer1_attention_weight_min": -14.68392276763916,
      "activations/layer20_attention_weight_max": 20.08073616027832,
      "activations/layer20_attention_weight_min": -20.613601684570312,
      "activations/layer21_attention_weight_max": 38.467262268066406,
      "activations/layer21_attention_weight_min": -25.068212509155273,
      "activations/layer22_attention_weight_max": 29.9561824798584,
      "activations/layer22_attention_weight_min": -27.37002944946289,
      "activations/layer23_attention_weight_max": 31.578514099121094,
      "activations/layer23_attention_weight_min": -20.807374954223633,
      "activations/layer2_attention_weight_max": 34.41883850097656,
      "activations/layer2_attention_weight_min": -30.339935302734375,
      "activations/layer3_attention_weight_max": 91.07208251953125,
      "activations/layer3_attention_weight_min": -86.66465759277344,
      "activations/layer4_attention_weight_max": 104.15289306640625,
      "activations/layer4_attention_weight_min": -94.53194427490234,
      "activations/layer5_attention_weight_max": 90.03927612304688,
      "activations/layer5_attention_weight_min": -83.24447631835938,
      "activations/layer6_attention_weight_max": 56.6142463684082,
      "activations/layer6_attention_weight_min": -54.75376892089844,
      "activations/layer7_attention_weight_max": 75.2890625,
      "activations/layer7_attention_weight_min": -77.00743103027344,
      "activations/layer8_attention_weight_max": 43.70421600341797,
      "activations/layer8_attention_weight_min": -45.962120056152344,
      "activations/layer9_attention_weight_max": 50.81061935424805,
      "activations/layer9_attention_weight_min": -50.39156723022461,
      "epoch": 7.5,
      "learning_rate": 0.0001026965909090909,
      "loss": 2.8062,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_loss": 2.76953125,
      "eval_runtime": 8.5597,
      "eval_samples_per_second": 501.656,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_openwebtext_loss": 2.76953125,
      "eval_openwebtext_ppl": 15.951155153095918,
      "eval_openwebtext_runtime": 8.5597,
      "eval_openwebtext_samples_per_second": 501.656,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 1.9985,
      "eval_wikitext_samples_per_second": 228.176,
      "step": 129000
    },
    {
      "epoch": 7.5,
      "eval_lambada_loss": 2.689453125,
      "eval_lambada_ppl": 14.723621737689907,
      "eval_lambada_runtime": 9.6189,
      "eval_lambada_samples_per_second": 506.189,
      "step": 129000
    },
    {
      "activations/layer0_attention_weight_max": 15.35610580444336,
      "activations/layer0_attention_weight_min": -14.669989585876465,
      "activations/layer10_attention_weight_max": 39.39300537109375,
      "activations/layer10_attention_weight_min": -35.247493743896484,
      "activations/layer11_attention_weight_max": 30.759952545166016,
      "activations/layer11_attention_weight_min": -30.618629455566406,
      "activations/layer12_attention_weight_max": 21.056324005126953,
      "activations/layer12_attention_weight_min": -24.793790817260742,
      "activations/layer13_attention_weight_max": 43.7381477355957,
      "activations/layer13_attention_weight_min": -30.517623901367188,
      "activations/layer14_attention_weight_max": 40.47809982299805,
      "activations/layer14_attention_weight_min": -32.15549087524414,
      "activations/layer15_attention_weight_max": 41.18541717529297,
      "activations/layer15_attention_weight_min": -28.55110740661621,
      "activations/layer16_attention_weight_max": 36.75215530395508,
      "activations/layer16_attention_weight_min": -27.896461486816406,
      "activations/layer17_attention_weight_max": 62.55330276489258,
      "activations/layer17_attention_weight_min": -42.192771911621094,
      "activations/layer18_attention_weight_max": 54.4217529296875,
      "activations/layer18_attention_weight_min": -36.534122467041016,
      "activations/layer19_attention_weight_max": 23.522043228149414,
      "activations/layer19_attention_weight_min": -19.814044952392578,
      "activations/layer1_attention_weight_max": 16.568490982055664,
      "activations/layer1_attention_weight_min": -13.699501037597656,
      "activations/layer20_attention_weight_max": 22.234086990356445,
      "activations/layer20_attention_weight_min": -20.408823013305664,
      "activations/layer21_attention_weight_max": 47.76361846923828,
      "activations/layer21_attention_weight_min": -25.761564254760742,
      "activations/layer22_attention_weight_max": 34.06764221191406,
      "activations/layer22_attention_weight_min": -23.513038635253906,
      "activations/layer23_attention_weight_max": 34.929683685302734,
      "activations/layer23_attention_weight_min": -20.088760375976562,
      "activations/layer2_attention_weight_max": 32.08818054199219,
      "activations/layer2_attention_weight_min": -28.506298065185547,
      "activations/layer3_attention_weight_max": 85.69612121582031,
      "activations/layer3_attention_weight_min": -85.780029296875,
      "activations/layer4_attention_weight_max": 103.41080474853516,
      "activations/layer4_attention_weight_min": -95.97664642333984,
      "activations/layer5_attention_weight_max": 88.35368347167969,
      "activations/layer5_attention_weight_min": -82.2832260131836,
      "activations/layer6_attention_weight_max": 57.44544982910156,
      "activations/layer6_attention_weight_min": -55.6247673034668,
      "activations/layer7_attention_weight_max": 80.54141235351562,
      "activations/layer7_attention_weight_min": -75.92913818359375,
      "activations/layer8_attention_weight_max": 46.59253692626953,
      "activations/layer8_attention_weight_min": -49.087974548339844,
      "activations/layer9_attention_weight_max": 53.87809371948242,
      "activations/layer9_attention_weight_min": -51.30820846557617,
      "epoch": 7.5,
      "learning_rate": 0.0001026776515151515,
      "loss": 2.8374,
      "step": 129050
    },
    {
      "activations/layer0_attention_weight_max": 15.432425498962402,
      "activations/layer0_attention_weight_min": -14.248409271240234,
      "activations/layer10_attention_weight_max": 37.91590118408203,
      "activations/layer10_attention_weight_min": -35.27153396606445,
      "activations/layer11_attention_weight_max": 32.82357406616211,
      "activations/layer11_attention_weight_min": -31.18126678466797,
      "activations/layer12_attention_weight_max": 21.608802795410156,
      "activations/layer12_attention_weight_min": -23.036462783813477,
      "activations/layer13_attention_weight_max": 54.10746383666992,
      "activations/layer13_attention_weight_min": -36.18263244628906,
      "activations/layer14_attention_weight_max": 51.7563591003418,
      "activations/layer14_attention_weight_min": -31.903499603271484,
      "activations/layer15_attention_weight_max": 39.81230545043945,
      "activations/layer15_attention_weight_min": -29.215417861938477,
      "activations/layer16_attention_weight_max": 33.51234817504883,
      "activations/layer16_attention_weight_min": -27.295917510986328,
      "activations/layer17_attention_weight_max": 55.935054779052734,
      "activations/layer17_attention_weight_min": -42.28699493408203,
      "activations/layer18_attention_weight_max": 50.046913146972656,
      "activations/layer18_attention_weight_min": -36.18706512451172,
      "activations/layer19_attention_weight_max": 22.997501373291016,
      "activations/layer19_attention_weight_min": -21.149145126342773,
      "activations/layer1_attention_weight_max": 16.64655303955078,
      "activations/layer1_attention_weight_min": -17.940757751464844,
      "activations/layer20_attention_weight_max": 24.62273406982422,
      "activations/layer20_attention_weight_min": -23.957983016967773,
      "activations/layer21_attention_weight_max": 41.34523391723633,
      "activations/layer21_attention_weight_min": -23.700613021850586,
      "activations/layer22_attention_weight_max": 35.21918869018555,
      "activations/layer22_attention_weight_min": -27.886947631835938,
      "activations/layer23_attention_weight_max": 40.01365661621094,
      "activations/layer23_attention_weight_min": -21.12700080871582,
      "activations/layer2_attention_weight_max": 32.39583206176758,
      "activations/layer2_attention_weight_min": -30.29840850830078,
      "activations/layer3_attention_weight_max": 89.90669250488281,
      "activations/layer3_attention_weight_min": -90.0453872680664,
      "activations/layer4_attention_weight_max": 109.7400894165039,
      "activations/layer4_attention_weight_min": -102.9072265625,
      "activations/layer5_attention_weight_max": 93.91014099121094,
      "activations/layer5_attention_weight_min": -87.09750366210938,
      "activations/layer6_attention_weight_max": 59.3980827331543,
      "activations/layer6_attention_weight_min": -55.816505432128906,
      "activations/layer7_attention_weight_max": 80.55502319335938,
      "activations/layer7_attention_weight_min": -72.42583465576172,
      "activations/layer8_attention_weight_max": 46.22816467285156,
      "activations/layer8_attention_weight_min": -46.10539245605469,
      "activations/layer9_attention_weight_max": 52.44272994995117,
      "activations/layer9_attention_weight_min": -49.55620574951172,
      "epoch": 7.5,
      "learning_rate": 0.00010265871212121212,
      "loss": 2.8247,
      "step": 129100
    },
    {
      "activations/layer0_attention_weight_max": 15.21371841430664,
      "activations/layer0_attention_weight_min": -14.369572639465332,
      "activations/layer10_attention_weight_max": 39.821044921875,
      "activations/layer10_attention_weight_min": -37.42476272583008,
      "activations/layer11_attention_weight_max": 34.80784606933594,
      "activations/layer11_attention_weight_min": -31.30797576904297,
      "activations/layer12_attention_weight_max": 18.24715232849121,
      "activations/layer12_attention_weight_min": -29.128189086914062,
      "activations/layer13_attention_weight_max": 47.24745178222656,
      "activations/layer13_attention_weight_min": -37.72080612182617,
      "activations/layer14_attention_weight_max": 42.1210823059082,
      "activations/layer14_attention_weight_min": -32.816253662109375,
      "activations/layer15_attention_weight_max": 34.55672836303711,
      "activations/layer15_attention_weight_min": -29.05287742614746,
      "activations/layer16_attention_weight_max": 31.36890411376953,
      "activations/layer16_attention_weight_min": -27.232757568359375,
      "activations/layer17_attention_weight_max": 50.6933708190918,
      "activations/layer17_attention_weight_min": -45.81610107421875,
      "activations/layer18_attention_weight_max": 44.3344841003418,
      "activations/layer18_attention_weight_min": -39.30311584472656,
      "activations/layer19_attention_weight_max": 21.72701644897461,
      "activations/layer19_attention_weight_min": -23.519895553588867,
      "activations/layer1_attention_weight_max": 18.287919998168945,
      "activations/layer1_attention_weight_min": -13.113158226013184,
      "activations/layer20_attention_weight_max": 22.261449813842773,
      "activations/layer20_attention_weight_min": -23.030807495117188,
      "activations/layer21_attention_weight_max": 39.43094253540039,
      "activations/layer21_attention_weight_min": -23.966894149780273,
      "activations/layer22_attention_weight_max": 31.94698143005371,
      "activations/layer22_attention_weight_min": -24.6451358795166,
      "activations/layer23_attention_weight_max": 36.421287536621094,
      "activations/layer23_attention_weight_min": -20.909034729003906,
      "activations/layer2_attention_weight_max": 32.59191131591797,
      "activations/layer2_attention_weight_min": -28.928592681884766,
      "activations/layer3_attention_weight_max": 86.29957580566406,
      "activations/layer3_attention_weight_min": -88.6993408203125,
      "activations/layer4_attention_weight_max": 100.77210235595703,
      "activations/layer4_attention_weight_min": -98.69623565673828,
      "activations/layer5_attention_weight_max": 86.78839111328125,
      "activations/layer5_attention_weight_min": -86.12333679199219,
      "activations/layer6_attention_weight_max": 56.03952407836914,
      "activations/layer6_attention_weight_min": -53.474037170410156,
      "activations/layer7_attention_weight_max": 82.18814849853516,
      "activations/layer7_attention_weight_min": -76.13578796386719,
      "activations/layer8_attention_weight_max": 48.59606170654297,
      "activations/layer8_attention_weight_min": -44.915042877197266,
      "activations/layer9_attention_weight_max": 62.03154373168945,
      "activations/layer9_attention_weight_min": -54.32540512084961,
      "epoch": 7.5,
      "learning_rate": 0.00010263977272727272,
      "loss": 2.8099,
      "step": 129150
    },
    {
      "activations/layer0_attention_weight_max": 15.277215957641602,
      "activations/layer0_attention_weight_min": -14.756804466247559,
      "activations/layer10_attention_weight_max": 38.474891662597656,
      "activations/layer10_attention_weight_min": -35.86577606201172,
      "activations/layer11_attention_weight_max": 31.937850952148438,
      "activations/layer11_attention_weight_min": -29.946895599365234,
      "activations/layer12_attention_weight_max": 22.762527465820312,
      "activations/layer12_attention_weight_min": -27.81951332092285,
      "activations/layer13_attention_weight_max": 37.520286560058594,
      "activations/layer13_attention_weight_min": -29.517547607421875,
      "activations/layer14_attention_weight_max": 34.07231521606445,
      "activations/layer14_attention_weight_min": -30.3175048828125,
      "activations/layer15_attention_weight_max": 35.54588317871094,
      "activations/layer15_attention_weight_min": -30.478445053100586,
      "activations/layer16_attention_weight_max": 31.716812133789062,
      "activations/layer16_attention_weight_min": -28.052751541137695,
      "activations/layer17_attention_weight_max": 57.83319091796875,
      "activations/layer17_attention_weight_min": -43.98427200317383,
      "activations/layer18_attention_weight_max": 52.57889175415039,
      "activations/layer18_attention_weight_min": -38.746482849121094,
      "activations/layer19_attention_weight_max": 22.455320358276367,
      "activations/layer19_attention_weight_min": -22.149967193603516,
      "activations/layer1_attention_weight_max": 18.08544158935547,
      "activations/layer1_attention_weight_min": -15.770787239074707,
      "activations/layer20_attention_weight_max": 23.938701629638672,
      "activations/layer20_attention_weight_min": -21.83761978149414,
      "activations/layer21_attention_weight_max": 40.64842224121094,
      "activations/layer21_attention_weight_min": -25.2314395904541,
      "activations/layer22_attention_weight_max": 34.390419006347656,
      "activations/layer22_attention_weight_min": -28.1522159576416,
      "activations/layer23_attention_weight_max": 33.32032775878906,
      "activations/layer23_attention_weight_min": -23.81229019165039,
      "activations/layer2_attention_weight_max": 31.7764892578125,
      "activations/layer2_attention_weight_min": -29.250774383544922,
      "activations/layer3_attention_weight_max": 88.35663604736328,
      "activations/layer3_attention_weight_min": -89.46111297607422,
      "activations/layer4_attention_weight_max": 110.14350891113281,
      "activations/layer4_attention_weight_min": -100.99431610107422,
      "activations/layer5_attention_weight_max": 91.95408630371094,
      "activations/layer5_attention_weight_min": -86.13958740234375,
      "activations/layer6_attention_weight_max": 55.59592819213867,
      "activations/layer6_attention_weight_min": -53.15559768676758,
      "activations/layer7_attention_weight_max": 77.20674133300781,
      "activations/layer7_attention_weight_min": -70.17501068115234,
      "activations/layer8_attention_weight_max": 47.11687469482422,
      "activations/layer8_attention_weight_min": -43.843162536621094,
      "activations/layer9_attention_weight_max": 60.8546257019043,
      "activations/layer9_attention_weight_min": -49.88352584838867,
      "epoch": 7.51,
      "learning_rate": 0.00010262083333333332,
      "loss": 2.8097,
      "step": 129200
    },
    {
      "activations/layer0_attention_weight_max": 14.834088325500488,
      "activations/layer0_attention_weight_min": -14.80064582824707,
      "activations/layer10_attention_weight_max": 35.5187873840332,
      "activations/layer10_attention_weight_min": -34.542083740234375,
      "activations/layer11_attention_weight_max": 29.956466674804688,
      "activations/layer11_attention_weight_min": -30.226171493530273,
      "activations/layer12_attention_weight_max": 18.081344604492188,
      "activations/layer12_attention_weight_min": -26.57908821105957,
      "activations/layer13_attention_weight_max": 48.08962631225586,
      "activations/layer13_attention_weight_min": -30.6425838470459,
      "activations/layer14_attention_weight_max": 39.72574996948242,
      "activations/layer14_attention_weight_min": -30.347087860107422,
      "activations/layer15_attention_weight_max": 36.56966781616211,
      "activations/layer15_attention_weight_min": -28.243139266967773,
      "activations/layer16_attention_weight_max": 32.27033233642578,
      "activations/layer16_attention_weight_min": -29.714603424072266,
      "activations/layer17_attention_weight_max": 53.794464111328125,
      "activations/layer17_attention_weight_min": -41.35530471801758,
      "activations/layer18_attention_weight_max": 49.4581413269043,
      "activations/layer18_attention_weight_min": -34.28315734863281,
      "activations/layer19_attention_weight_max": 21.494077682495117,
      "activations/layer19_attention_weight_min": -21.81585121154785,
      "activations/layer1_attention_weight_max": 16.939453125,
      "activations/layer1_attention_weight_min": -14.020391464233398,
      "activations/layer20_attention_weight_max": 20.76107406616211,
      "activations/layer20_attention_weight_min": -21.426029205322266,
      "activations/layer21_attention_weight_max": 42.7657356262207,
      "activations/layer21_attention_weight_min": -24.736631393432617,
      "activations/layer22_attention_weight_max": 34.80320358276367,
      "activations/layer22_attention_weight_min": -25.729944229125977,
      "activations/layer23_attention_weight_max": 40.20112991333008,
      "activations/layer23_attention_weight_min": -23.893714904785156,
      "activations/layer2_attention_weight_max": 31.97638511657715,
      "activations/layer2_attention_weight_min": -31.161230087280273,
      "activations/layer3_attention_weight_max": 88.18067169189453,
      "activations/layer3_attention_weight_min": -94.63929748535156,
      "activations/layer4_attention_weight_max": 100.459228515625,
      "activations/layer4_attention_weight_min": -92.67434692382812,
      "activations/layer5_attention_weight_max": 85.72747039794922,
      "activations/layer5_attention_weight_min": -79.91707611083984,
      "activations/layer6_attention_weight_max": 54.595218658447266,
      "activations/layer6_attention_weight_min": -51.91122817993164,
      "activations/layer7_attention_weight_max": 75.87138366699219,
      "activations/layer7_attention_weight_min": -71.82821655273438,
      "activations/layer8_attention_weight_max": 46.11576843261719,
      "activations/layer8_attention_weight_min": -47.9582405090332,
      "activations/layer9_attention_weight_max": 47.39501953125,
      "activations/layer9_attention_weight_min": -48.34161376953125,
      "epoch": 7.51,
      "learning_rate": 0.00010260189393939393,
      "loss": 2.8275,
      "step": 129250
    },
    {
      "activations/layer0_attention_weight_max": 14.92812728881836,
      "activations/layer0_attention_weight_min": -15.079509735107422,
      "activations/layer10_attention_weight_max": 36.7443962097168,
      "activations/layer10_attention_weight_min": -35.39313888549805,
      "activations/layer11_attention_weight_max": 33.92054748535156,
      "activations/layer11_attention_weight_min": -30.672466278076172,
      "activations/layer12_attention_weight_max": 18.833498001098633,
      "activations/layer12_attention_weight_min": -28.21609115600586,
      "activations/layer13_attention_weight_max": 44.78871154785156,
      "activations/layer13_attention_weight_min": -30.397891998291016,
      "activations/layer14_attention_weight_max": 40.205665588378906,
      "activations/layer14_attention_weight_min": -31.56706428527832,
      "activations/layer15_attention_weight_max": 35.65896987915039,
      "activations/layer15_attention_weight_min": -30.374061584472656,
      "activations/layer16_attention_weight_max": 34.07234191894531,
      "activations/layer16_attention_weight_min": -27.483240127563477,
      "activations/layer17_attention_weight_max": 55.54690933227539,
      "activations/layer17_attention_weight_min": -45.14726638793945,
      "activations/layer18_attention_weight_max": 52.25221633911133,
      "activations/layer18_attention_weight_min": -36.1890869140625,
      "activations/layer19_attention_weight_max": 22.257108688354492,
      "activations/layer19_attention_weight_min": -21.935142517089844,
      "activations/layer1_attention_weight_max": 17.449981689453125,
      "activations/layer1_attention_weight_min": -13.278539657592773,
      "activations/layer20_attention_weight_max": 21.2504825592041,
      "activations/layer20_attention_weight_min": -21.789817810058594,
      "activations/layer21_attention_weight_max": 37.42241668701172,
      "activations/layer21_attention_weight_min": -24.329851150512695,
      "activations/layer22_attention_weight_max": 36.63313674926758,
      "activations/layer22_attention_weight_min": -24.936235427856445,
      "activations/layer23_attention_weight_max": 35.167640686035156,
      "activations/layer23_attention_weight_min": -23.930084228515625,
      "activations/layer2_attention_weight_max": 31.196096420288086,
      "activations/layer2_attention_weight_min": -30.60462760925293,
      "activations/layer3_attention_weight_max": 84.97359466552734,
      "activations/layer3_attention_weight_min": -94.46690368652344,
      "activations/layer4_attention_weight_max": 98.97941589355469,
      "activations/layer4_attention_weight_min": -98.8017578125,
      "activations/layer5_attention_weight_max": 87.52230834960938,
      "activations/layer5_attention_weight_min": -86.42192840576172,
      "activations/layer6_attention_weight_max": 55.36616516113281,
      "activations/layer6_attention_weight_min": -55.104156494140625,
      "activations/layer7_attention_weight_max": 74.12580108642578,
      "activations/layer7_attention_weight_min": -79.72539520263672,
      "activations/layer8_attention_weight_max": 44.466041564941406,
      "activations/layer8_attention_weight_min": -44.565853118896484,
      "activations/layer9_attention_weight_max": 50.077293395996094,
      "activations/layer9_attention_weight_min": -47.15028762817383,
      "epoch": 7.51,
      "learning_rate": 0.00010258295454545455,
      "loss": 2.8215,
      "step": 129300
    },
    {
      "activations/layer0_attention_weight_max": 14.787968635559082,
      "activations/layer0_attention_weight_min": -14.823559761047363,
      "activations/layer10_attention_weight_max": 40.408424377441406,
      "activations/layer10_attention_weight_min": -38.32954788208008,
      "activations/layer11_attention_weight_max": 33.86992645263672,
      "activations/layer11_attention_weight_min": -34.13926696777344,
      "activations/layer12_attention_weight_max": 27.51056671142578,
      "activations/layer12_attention_weight_min": -26.132057189941406,
      "activations/layer13_attention_weight_max": 41.16261672973633,
      "activations/layer13_attention_weight_min": -31.955965042114258,
      "activations/layer14_attention_weight_max": 37.31931686401367,
      "activations/layer14_attention_weight_min": -30.038301467895508,
      "activations/layer15_attention_weight_max": 35.8598518371582,
      "activations/layer15_attention_weight_min": -28.842222213745117,
      "activations/layer16_attention_weight_max": 31.50763702392578,
      "activations/layer16_attention_weight_min": -28.546064376831055,
      "activations/layer17_attention_weight_max": 58.55879211425781,
      "activations/layer17_attention_weight_min": -45.407772064208984,
      "activations/layer18_attention_weight_max": 47.592830657958984,
      "activations/layer18_attention_weight_min": -37.669559478759766,
      "activations/layer19_attention_weight_max": 20.29213523864746,
      "activations/layer19_attention_weight_min": -20.334415435791016,
      "activations/layer1_attention_weight_max": 16.405071258544922,
      "activations/layer1_attention_weight_min": -15.367412567138672,
      "activations/layer20_attention_weight_max": 20.921903610229492,
      "activations/layer20_attention_weight_min": -25.75322914123535,
      "activations/layer21_attention_weight_max": 32.032073974609375,
      "activations/layer21_attention_weight_min": -23.982242584228516,
      "activations/layer22_attention_weight_max": 31.92800521850586,
      "activations/layer22_attention_weight_min": -25.79017448425293,
      "activations/layer23_attention_weight_max": 30.544986724853516,
      "activations/layer23_attention_weight_min": -21.03568458557129,
      "activations/layer2_attention_weight_max": 37.191184997558594,
      "activations/layer2_attention_weight_min": -36.628082275390625,
      "activations/layer3_attention_weight_max": 88.30513763427734,
      "activations/layer3_attention_weight_min": -90.70796966552734,
      "activations/layer4_attention_weight_max": 97.13826751708984,
      "activations/layer4_attention_weight_min": -94.54315948486328,
      "activations/layer5_attention_weight_max": 86.98857879638672,
      "activations/layer5_attention_weight_min": -81.57755279541016,
      "activations/layer6_attention_weight_max": 55.509193420410156,
      "activations/layer6_attention_weight_min": -54.15849304199219,
      "activations/layer7_attention_weight_max": 86.57501983642578,
      "activations/layer7_attention_weight_min": -83.39896392822266,
      "activations/layer8_attention_weight_max": 50.72954177856445,
      "activations/layer8_attention_weight_min": -52.08244705200195,
      "activations/layer9_attention_weight_max": 53.969947814941406,
      "activations/layer9_attention_weight_min": -52.44974899291992,
      "epoch": 7.52,
      "learning_rate": 0.00010256401515151514,
      "loss": 2.7944,
      "step": 129350
    },
    {
      "activations/layer0_attention_weight_max": 14.400928497314453,
      "activations/layer0_attention_weight_min": -14.56272029876709,
      "activations/layer10_attention_weight_max": 36.83506393432617,
      "activations/layer10_attention_weight_min": -33.1539306640625,
      "activations/layer11_attention_weight_max": 30.133800506591797,
      "activations/layer11_attention_weight_min": -29.879011154174805,
      "activations/layer12_attention_weight_max": 24.27926254272461,
      "activations/layer12_attention_weight_min": -23.8221378326416,
      "activations/layer13_attention_weight_max": 39.57733917236328,
      "activations/layer13_attention_weight_min": -32.488563537597656,
      "activations/layer14_attention_weight_max": 35.89870071411133,
      "activations/layer14_attention_weight_min": -30.467140197753906,
      "activations/layer15_attention_weight_max": 36.41434860229492,
      "activations/layer15_attention_weight_min": -29.01373291015625,
      "activations/layer16_attention_weight_max": 33.77210998535156,
      "activations/layer16_attention_weight_min": -27.16777229309082,
      "activations/layer17_attention_weight_max": 56.41642379760742,
      "activations/layer17_attention_weight_min": -40.561195373535156,
      "activations/layer18_attention_weight_max": 48.343345642089844,
      "activations/layer18_attention_weight_min": -35.08906173706055,
      "activations/layer19_attention_weight_max": 24.98505973815918,
      "activations/layer19_attention_weight_min": -19.78135108947754,
      "activations/layer1_attention_weight_max": 16.093914031982422,
      "activations/layer1_attention_weight_min": -14.738321304321289,
      "activations/layer20_attention_weight_max": 22.097257614135742,
      "activations/layer20_attention_weight_min": -20.614763259887695,
      "activations/layer21_attention_weight_max": 41.63187789916992,
      "activations/layer21_attention_weight_min": -26.823850631713867,
      "activations/layer22_attention_weight_max": 33.32528305053711,
      "activations/layer22_attention_weight_min": -25.10068130493164,
      "activations/layer23_attention_weight_max": 35.839630126953125,
      "activations/layer23_attention_weight_min": -23.11520767211914,
      "activations/layer2_attention_weight_max": 35.21369934082031,
      "activations/layer2_attention_weight_min": -36.114280700683594,
      "activations/layer3_attention_weight_max": 85.52143859863281,
      "activations/layer3_attention_weight_min": -87.35784149169922,
      "activations/layer4_attention_weight_max": 97.36368560791016,
      "activations/layer4_attention_weight_min": -89.25662231445312,
      "activations/layer5_attention_weight_max": 82.67720031738281,
      "activations/layer5_attention_weight_min": -80.49164581298828,
      "activations/layer6_attention_weight_max": 53.33190155029297,
      "activations/layer6_attention_weight_min": -52.31340026855469,
      "activations/layer7_attention_weight_max": 72.88086700439453,
      "activations/layer7_attention_weight_min": -69.49642181396484,
      "activations/layer8_attention_weight_max": 41.74379348754883,
      "activations/layer8_attention_weight_min": -44.562923431396484,
      "activations/layer9_attention_weight_max": 49.344749450683594,
      "activations/layer9_attention_weight_min": -47.28115463256836,
      "epoch": 7.52,
      "learning_rate": 0.00010254507575757575,
      "loss": 2.8201,
      "step": 129400
    },
    {
      "activations/layer0_attention_weight_max": 15.234099388122559,
      "activations/layer0_attention_weight_min": -14.78419017791748,
      "activations/layer10_attention_weight_max": 35.347496032714844,
      "activations/layer10_attention_weight_min": -35.534278869628906,
      "activations/layer11_attention_weight_max": 30.704978942871094,
      "activations/layer11_attention_weight_min": -32.09137725830078,
      "activations/layer12_attention_weight_max": 17.12152862548828,
      "activations/layer12_attention_weight_min": -25.60775375366211,
      "activations/layer13_attention_weight_max": 36.159088134765625,
      "activations/layer13_attention_weight_min": -31.226055145263672,
      "activations/layer14_attention_weight_max": 39.90108108520508,
      "activations/layer14_attention_weight_min": -30.275882720947266,
      "activations/layer15_attention_weight_max": 34.944664001464844,
      "activations/layer15_attention_weight_min": -29.34772300720215,
      "activations/layer16_attention_weight_max": 32.129703521728516,
      "activations/layer16_attention_weight_min": -27.090648651123047,
      "activations/layer17_attention_weight_max": 57.74696731567383,
      "activations/layer17_attention_weight_min": -42.217750549316406,
      "activations/layer18_attention_weight_max": 51.639930725097656,
      "activations/layer18_attention_weight_min": -36.10631561279297,
      "activations/layer19_attention_weight_max": 22.145231246948242,
      "activations/layer19_attention_weight_min": -21.841028213500977,
      "activations/layer1_attention_weight_max": 17.557058334350586,
      "activations/layer1_attention_weight_min": -13.884195327758789,
      "activations/layer20_attention_weight_max": 21.714704513549805,
      "activations/layer20_attention_weight_min": -23.893205642700195,
      "activations/layer21_attention_weight_max": 41.641883850097656,
      "activations/layer21_attention_weight_min": -23.48409080505371,
      "activations/layer22_attention_weight_max": 34.49126052856445,
      "activations/layer22_attention_weight_min": -24.432804107666016,
      "activations/layer23_attention_weight_max": 37.707855224609375,
      "activations/layer23_attention_weight_min": -21.999744415283203,
      "activations/layer2_attention_weight_max": 30.164310455322266,
      "activations/layer2_attention_weight_min": -29.549461364746094,
      "activations/layer3_attention_weight_max": 79.0314712524414,
      "activations/layer3_attention_weight_min": -82.69380187988281,
      "activations/layer4_attention_weight_max": 94.53286743164062,
      "activations/layer4_attention_weight_min": -89.14022064208984,
      "activations/layer5_attention_weight_max": 81.24290466308594,
      "activations/layer5_attention_weight_min": -80.68126678466797,
      "activations/layer6_attention_weight_max": 51.99711608886719,
      "activations/layer6_attention_weight_min": -53.632484436035156,
      "activations/layer7_attention_weight_max": 74.27643585205078,
      "activations/layer7_attention_weight_min": -74.88148498535156,
      "activations/layer8_attention_weight_max": 41.88551712036133,
      "activations/layer8_attention_weight_min": -42.65458297729492,
      "activations/layer9_attention_weight_max": 47.84770584106445,
      "activations/layer9_attention_weight_min": -48.80646514892578,
      "epoch": 7.52,
      "learning_rate": 0.00010252613636363635,
      "loss": 2.8101,
      "step": 129450
    },
    {
      "activations/layer0_attention_weight_max": 14.670660018920898,
      "activations/layer0_attention_weight_min": -15.096155166625977,
      "activations/layer10_attention_weight_max": 39.16019058227539,
      "activations/layer10_attention_weight_min": -35.959320068359375,
      "activations/layer11_attention_weight_max": 31.145389556884766,
      "activations/layer11_attention_weight_min": -31.253555297851562,
      "activations/layer12_attention_weight_max": 32.94321060180664,
      "activations/layer12_attention_weight_min": -23.778076171875,
      "activations/layer13_attention_weight_max": 44.950321197509766,
      "activations/layer13_attention_weight_min": -32.10976791381836,
      "activations/layer14_attention_weight_max": 47.984703063964844,
      "activations/layer14_attention_weight_min": -31.04863166809082,
      "activations/layer15_attention_weight_max": 40.52811813354492,
      "activations/layer15_attention_weight_min": -29.73944091796875,
      "activations/layer16_attention_weight_max": 31.006071090698242,
      "activations/layer16_attention_weight_min": -28.31534194946289,
      "activations/layer17_attention_weight_max": 54.204917907714844,
      "activations/layer17_attention_weight_min": -44.097145080566406,
      "activations/layer18_attention_weight_max": 47.71245574951172,
      "activations/layer18_attention_weight_min": -35.205596923828125,
      "activations/layer19_attention_weight_max": 21.22281265258789,
      "activations/layer19_attention_weight_min": -21.7763729095459,
      "activations/layer1_attention_weight_max": 17.087905883789062,
      "activations/layer1_attention_weight_min": -19.88265609741211,
      "activations/layer20_attention_weight_max": 22.080795288085938,
      "activations/layer20_attention_weight_min": -19.683652877807617,
      "activations/layer21_attention_weight_max": 38.379634857177734,
      "activations/layer21_attention_weight_min": -26.681264877319336,
      "activations/layer22_attention_weight_max": 32.76055908203125,
      "activations/layer22_attention_weight_min": -25.89220428466797,
      "activations/layer23_attention_weight_max": 37.60697937011719,
      "activations/layer23_attention_weight_min": -22.83975601196289,
      "activations/layer2_attention_weight_max": 34.32526397705078,
      "activations/layer2_attention_weight_min": -31.63551139831543,
      "activations/layer3_attention_weight_max": 96.44744110107422,
      "activations/layer3_attention_weight_min": -91.11710357666016,
      "activations/layer4_attention_weight_max": 100.75863647460938,
      "activations/layer4_attention_weight_min": -91.99104309082031,
      "activations/layer5_attention_weight_max": 86.73876190185547,
      "activations/layer5_attention_weight_min": -82.06217193603516,
      "activations/layer6_attention_weight_max": 57.79853820800781,
      "activations/layer6_attention_weight_min": -54.13713455200195,
      "activations/layer7_attention_weight_max": 74.37815856933594,
      "activations/layer7_attention_weight_min": -75.22516632080078,
      "activations/layer8_attention_weight_max": 45.99141311645508,
      "activations/layer8_attention_weight_min": -46.34213638305664,
      "activations/layer9_attention_weight_max": 48.93790817260742,
      "activations/layer9_attention_weight_min": -49.171119689941406,
      "epoch": 7.52,
      "learning_rate": 0.00010250719696969695,
      "loss": 2.8182,
      "step": 129500
    },
    {
      "activations/layer0_attention_weight_max": 14.620931625366211,
      "activations/layer0_attention_weight_min": -15.082515716552734,
      "activations/layer10_attention_weight_max": 35.974246978759766,
      "activations/layer10_attention_weight_min": -35.504127502441406,
      "activations/layer11_attention_weight_max": 30.421489715576172,
      "activations/layer11_attention_weight_min": -32.50513458251953,
      "activations/layer12_attention_weight_max": 16.87908935546875,
      "activations/layer12_attention_weight_min": -24.212177276611328,
      "activations/layer13_attention_weight_max": 48.55891418457031,
      "activations/layer13_attention_weight_min": -31.243526458740234,
      "activations/layer14_attention_weight_max": 35.947349548339844,
      "activations/layer14_attention_weight_min": -30.85016632080078,
      "activations/layer15_attention_weight_max": 36.29339599609375,
      "activations/layer15_attention_weight_min": -29.522546768188477,
      "activations/layer16_attention_weight_max": 32.164794921875,
      "activations/layer16_attention_weight_min": -27.237262725830078,
      "activations/layer17_attention_weight_max": 53.3976936340332,
      "activations/layer17_attention_weight_min": -44.92830276489258,
      "activations/layer18_attention_weight_max": 48.48176956176758,
      "activations/layer18_attention_weight_min": -37.54285430908203,
      "activations/layer19_attention_weight_max": 20.73600959777832,
      "activations/layer19_attention_weight_min": -22.730836868286133,
      "activations/layer1_attention_weight_max": 19.277408599853516,
      "activations/layer1_attention_weight_min": -13.174664497375488,
      "activations/layer20_attention_weight_max": 22.87204360961914,
      "activations/layer20_attention_weight_min": -22.444730758666992,
      "activations/layer21_attention_weight_max": 33.63759231567383,
      "activations/layer21_attention_weight_min": -24.545482635498047,
      "activations/layer22_attention_weight_max": 36.981204986572266,
      "activations/layer22_attention_weight_min": -28.99881935119629,
      "activations/layer23_attention_weight_max": 34.63056182861328,
      "activations/layer23_attention_weight_min": -20.134580612182617,
      "activations/layer2_attention_weight_max": 31.431907653808594,
      "activations/layer2_attention_weight_min": -30.011348724365234,
      "activations/layer3_attention_weight_max": 86.97782135009766,
      "activations/layer3_attention_weight_min": -91.8387451171875,
      "activations/layer4_attention_weight_max": 100.79727935791016,
      "activations/layer4_attention_weight_min": -98.25726318359375,
      "activations/layer5_attention_weight_max": 86.24162292480469,
      "activations/layer5_attention_weight_min": -82.36393737792969,
      "activations/layer6_attention_weight_max": 55.42637634277344,
      "activations/layer6_attention_weight_min": -53.58440017700195,
      "activations/layer7_attention_weight_max": 84.63858795166016,
      "activations/layer7_attention_weight_min": -73.17463684082031,
      "activations/layer8_attention_weight_max": 45.90133285522461,
      "activations/layer8_attention_weight_min": -46.367977142333984,
      "activations/layer9_attention_weight_max": 52.2437744140625,
      "activations/layer9_attention_weight_min": -48.1979866027832,
      "epoch": 7.53,
      "learning_rate": 0.00010248825757575757,
      "loss": 2.8241,
      "step": 129550
    },
    {
      "activations/layer0_attention_weight_max": 14.473374366760254,
      "activations/layer0_attention_weight_min": -15.246894836425781,
      "activations/layer10_attention_weight_max": 37.1569709777832,
      "activations/layer10_attention_weight_min": -36.46735763549805,
      "activations/layer11_attention_weight_max": 31.269506454467773,
      "activations/layer11_attention_weight_min": -30.123218536376953,
      "activations/layer12_attention_weight_max": 21.483070373535156,
      "activations/layer12_attention_weight_min": -23.721769332885742,
      "activations/layer13_attention_weight_max": 39.466796875,
      "activations/layer13_attention_weight_min": -30.499481201171875,
      "activations/layer14_attention_weight_max": 36.2152099609375,
      "activations/layer14_attention_weight_min": -31.117956161499023,
      "activations/layer15_attention_weight_max": 35.01076889038086,
      "activations/layer15_attention_weight_min": -29.360681533813477,
      "activations/layer16_attention_weight_max": 29.28761863708496,
      "activations/layer16_attention_weight_min": -29.4785099029541,
      "activations/layer17_attention_weight_max": 56.285343170166016,
      "activations/layer17_attention_weight_min": -43.09249496459961,
      "activations/layer18_attention_weight_max": 51.47534942626953,
      "activations/layer18_attention_weight_min": -36.709285736083984,
      "activations/layer19_attention_weight_max": 21.27629852294922,
      "activations/layer19_attention_weight_min": -24.433656692504883,
      "activations/layer1_attention_weight_max": 17.139341354370117,
      "activations/layer1_attention_weight_min": -14.801657676696777,
      "activations/layer20_attention_weight_max": 21.811491012573242,
      "activations/layer20_attention_weight_min": -21.73564910888672,
      "activations/layer21_attention_weight_max": 38.33369445800781,
      "activations/layer21_attention_weight_min": -23.030954360961914,
      "activations/layer22_attention_weight_max": 31.724132537841797,
      "activations/layer22_attention_weight_min": -26.53343391418457,
      "activations/layer23_attention_weight_max": 34.01468276977539,
      "activations/layer23_attention_weight_min": -22.48554039001465,
      "activations/layer2_attention_weight_max": 32.796871185302734,
      "activations/layer2_attention_weight_min": -29.375389099121094,
      "activations/layer3_attention_weight_max": 88.41464233398438,
      "activations/layer3_attention_weight_min": -89.35225677490234,
      "activations/layer4_attention_weight_max": 105.1845932006836,
      "activations/layer4_attention_weight_min": -99.54135131835938,
      "activations/layer5_attention_weight_max": 91.64361572265625,
      "activations/layer5_attention_weight_min": -85.18943786621094,
      "activations/layer6_attention_weight_max": 56.149497985839844,
      "activations/layer6_attention_weight_min": -53.64658737182617,
      "activations/layer7_attention_weight_max": 75.09376525878906,
      "activations/layer7_attention_weight_min": -73.86907958984375,
      "activations/layer8_attention_weight_max": 46.380409240722656,
      "activations/layer8_attention_weight_min": -46.6210823059082,
      "activations/layer9_attention_weight_max": 49.06586837768555,
      "activations/layer9_attention_weight_min": -51.36747360229492,
      "epoch": 7.53,
      "learning_rate": 0.00010246969696969696,
      "loss": 2.8272,
      "step": 129600
    },
    {
      "activations/layer0_attention_weight_max": 15.066938400268555,
      "activations/layer0_attention_weight_min": -14.992769241333008,
      "activations/layer10_attention_weight_max": 39.937477111816406,
      "activations/layer10_attention_weight_min": -37.62944793701172,
      "activations/layer11_attention_weight_max": 30.19375228881836,
      "activations/layer11_attention_weight_min": -31.207698822021484,
      "activations/layer12_attention_weight_max": 23.46488380432129,
      "activations/layer12_attention_weight_min": -26.58795928955078,
      "activations/layer13_attention_weight_max": 45.84563064575195,
      "activations/layer13_attention_weight_min": -31.650793075561523,
      "activations/layer14_attention_weight_max": 42.06601333618164,
      "activations/layer14_attention_weight_min": -30.094541549682617,
      "activations/layer15_attention_weight_max": 37.26049041748047,
      "activations/layer15_attention_weight_min": -29.805686950683594,
      "activations/layer16_attention_weight_max": 37.12908935546875,
      "activations/layer16_attention_weight_min": -27.208335876464844,
      "activations/layer17_attention_weight_max": 60.78091812133789,
      "activations/layer17_attention_weight_min": -43.9851188659668,
      "activations/layer18_attention_weight_max": 48.893409729003906,
      "activations/layer18_attention_weight_min": -37.500553131103516,
      "activations/layer19_attention_weight_max": 25.543622970581055,
      "activations/layer19_attention_weight_min": -22.48599624633789,
      "activations/layer1_attention_weight_max": 17.779979705810547,
      "activations/layer1_attention_weight_min": -14.252972602844238,
      "activations/layer20_attention_weight_max": 23.615123748779297,
      "activations/layer20_attention_weight_min": -21.59395980834961,
      "activations/layer21_attention_weight_max": 45.44615936279297,
      "activations/layer21_attention_weight_min": -26.651023864746094,
      "activations/layer22_attention_weight_max": 38.517269134521484,
      "activations/layer22_attention_weight_min": -25.39321517944336,
      "activations/layer23_attention_weight_max": 39.71858596801758,
      "activations/layer23_attention_weight_min": -20.765052795410156,
      "activations/layer2_attention_weight_max": 31.343551635742188,
      "activations/layer2_attention_weight_min": -29.443769454956055,
      "activations/layer3_attention_weight_max": 83.55042266845703,
      "activations/layer3_attention_weight_min": -83.18051147460938,
      "activations/layer4_attention_weight_max": 99.41590881347656,
      "activations/layer4_attention_weight_min": -94.35258483886719,
      "activations/layer5_attention_weight_max": 86.10113525390625,
      "activations/layer5_attention_weight_min": -84.64214324951172,
      "activations/layer6_attention_weight_max": 56.948486328125,
      "activations/layer6_attention_weight_min": -54.72722625732422,
      "activations/layer7_attention_weight_max": 79.7370834350586,
      "activations/layer7_attention_weight_min": -76.21609497070312,
      "activations/layer8_attention_weight_max": 46.708282470703125,
      "activations/layer8_attention_weight_min": -48.85346221923828,
      "activations/layer9_attention_weight_max": 51.36341094970703,
      "activations/layer9_attention_weight_min": -50.42027282714844,
      "epoch": 7.53,
      "learning_rate": 0.00010245075757575758,
      "loss": 2.8299,
      "step": 129650
    },
    {
      "activations/layer0_attention_weight_max": 15.744674682617188,
      "activations/layer0_attention_weight_min": -15.167619705200195,
      "activations/layer10_attention_weight_max": 37.11210250854492,
      "activations/layer10_attention_weight_min": -36.83009719848633,
      "activations/layer11_attention_weight_max": 33.32798767089844,
      "activations/layer11_attention_weight_min": -31.977758407592773,
      "activations/layer12_attention_weight_max": 21.49366569519043,
      "activations/layer12_attention_weight_min": -32.41994094848633,
      "activations/layer13_attention_weight_max": 39.9763069152832,
      "activations/layer13_attention_weight_min": -30.420333862304688,
      "activations/layer14_attention_weight_max": 36.91742706298828,
      "activations/layer14_attention_weight_min": -29.866086959838867,
      "activations/layer15_attention_weight_max": 35.781005859375,
      "activations/layer15_attention_weight_min": -28.33919334411621,
      "activations/layer16_attention_weight_max": 32.586177825927734,
      "activations/layer16_attention_weight_min": -27.84105682373047,
      "activations/layer17_attention_weight_max": 58.38839340209961,
      "activations/layer17_attention_weight_min": -45.061553955078125,
      "activations/layer18_attention_weight_max": 48.10551834106445,
      "activations/layer18_attention_weight_min": -35.41575622558594,
      "activations/layer19_attention_weight_max": 19.56389617919922,
      "activations/layer19_attention_weight_min": -20.691986083984375,
      "activations/layer1_attention_weight_max": 16.473623275756836,
      "activations/layer1_attention_weight_min": -13.68218994140625,
      "activations/layer20_attention_weight_max": 23.992042541503906,
      "activations/layer20_attention_weight_min": -20.712892532348633,
      "activations/layer21_attention_weight_max": 36.378170013427734,
      "activations/layer21_attention_weight_min": -25.72966194152832,
      "activations/layer22_attention_weight_max": 29.22156524658203,
      "activations/layer22_attention_weight_min": -24.070634841918945,
      "activations/layer23_attention_weight_max": 32.19676208496094,
      "activations/layer23_attention_weight_min": -21.950803756713867,
      "activations/layer2_attention_weight_max": 31.344749450683594,
      "activations/layer2_attention_weight_min": -30.13167953491211,
      "activations/layer3_attention_weight_max": 85.85921478271484,
      "activations/layer3_attention_weight_min": -85.83512115478516,
      "activations/layer4_attention_weight_max": 98.92626190185547,
      "activations/layer4_attention_weight_min": -93.620849609375,
      "activations/layer5_attention_weight_max": 85.45711517333984,
      "activations/layer5_attention_weight_min": -82.8407211303711,
      "activations/layer6_attention_weight_max": 53.89693069458008,
      "activations/layer6_attention_weight_min": -51.99831771850586,
      "activations/layer7_attention_weight_max": 81.6017074584961,
      "activations/layer7_attention_weight_min": -71.93637084960938,
      "activations/layer8_attention_weight_max": 46.281307220458984,
      "activations/layer8_attention_weight_min": -45.110191345214844,
      "activations/layer9_attention_weight_max": 64.97051239013672,
      "activations/layer9_attention_weight_min": -50.610435485839844,
      "epoch": 7.54,
      "learning_rate": 0.00010243181818181818,
      "loss": 2.816,
      "step": 129700
    },
    {
      "activations/layer0_attention_weight_max": 14.675708770751953,
      "activations/layer0_attention_weight_min": -14.870190620422363,
      "activations/layer10_attention_weight_max": 36.870018005371094,
      "activations/layer10_attention_weight_min": -36.088748931884766,
      "activations/layer11_attention_weight_max": 28.359519958496094,
      "activations/layer11_attention_weight_min": -29.280805587768555,
      "activations/layer12_attention_weight_max": 19.37439727783203,
      "activations/layer12_attention_weight_min": -24.379972457885742,
      "activations/layer13_attention_weight_max": 44.49437713623047,
      "activations/layer13_attention_weight_min": -34.873931884765625,
      "activations/layer14_attention_weight_max": 39.198280334472656,
      "activations/layer14_attention_weight_min": -32.56544494628906,
      "activations/layer15_attention_weight_max": 36.67807388305664,
      "activations/layer15_attention_weight_min": -28.29721450805664,
      "activations/layer16_attention_weight_max": 31.71630096435547,
      "activations/layer16_attention_weight_min": -27.335988998413086,
      "activations/layer17_attention_weight_max": 55.074256896972656,
      "activations/layer17_attention_weight_min": -43.88343811035156,
      "activations/layer18_attention_weight_max": 48.63418960571289,
      "activations/layer18_attention_weight_min": -35.658145904541016,
      "activations/layer19_attention_weight_max": 22.447509765625,
      "activations/layer19_attention_weight_min": -19.92761993408203,
      "activations/layer1_attention_weight_max": 16.734020233154297,
      "activations/layer1_attention_weight_min": -17.1943359375,
      "activations/layer20_attention_weight_max": 21.25510025024414,
      "activations/layer20_attention_weight_min": -18.96263313293457,
      "activations/layer21_attention_weight_max": 40.65629577636719,
      "activations/layer21_attention_weight_min": -24.21782875061035,
      "activations/layer22_attention_weight_max": 33.78730392456055,
      "activations/layer22_attention_weight_min": -24.09109115600586,
      "activations/layer23_attention_weight_max": 33.452693939208984,
      "activations/layer23_attention_weight_min": -20.845687866210938,
      "activations/layer2_attention_weight_max": 30.77401351928711,
      "activations/layer2_attention_weight_min": -28.409420013427734,
      "activations/layer3_attention_weight_max": 81.44796752929688,
      "activations/layer3_attention_weight_min": -85.83574676513672,
      "activations/layer4_attention_weight_max": 97.4005126953125,
      "activations/layer4_attention_weight_min": -91.63627624511719,
      "activations/layer5_attention_weight_max": 84.14286804199219,
      "activations/layer5_attention_weight_min": -79.17782592773438,
      "activations/layer6_attention_weight_max": 55.44504928588867,
      "activations/layer6_attention_weight_min": -50.94796371459961,
      "activations/layer7_attention_weight_max": 75.13046264648438,
      "activations/layer7_attention_weight_min": -74.64447784423828,
      "activations/layer8_attention_weight_max": 44.44797897338867,
      "activations/layer8_attention_weight_min": -45.44594955444336,
      "activations/layer9_attention_weight_max": 51.92445755004883,
      "activations/layer9_attention_weight_min": -47.934898376464844,
      "epoch": 7.54,
      "learning_rate": 0.00010241325757575757,
      "loss": 2.8195,
      "step": 129750
    },
    {
      "activations/layer0_attention_weight_max": 14.903959274291992,
      "activations/layer0_attention_weight_min": -15.024282455444336,
      "activations/layer10_attention_weight_max": 37.72050857543945,
      "activations/layer10_attention_weight_min": -35.314598083496094,
      "activations/layer11_attention_weight_max": 31.48387908935547,
      "activations/layer11_attention_weight_min": -32.43064498901367,
      "activations/layer12_attention_weight_max": 21.542936325073242,
      "activations/layer12_attention_weight_min": -25.931978225708008,
      "activations/layer13_attention_weight_max": 46.967262268066406,
      "activations/layer13_attention_weight_min": -35.547367095947266,
      "activations/layer14_attention_weight_max": 54.146942138671875,
      "activations/layer14_attention_weight_min": -34.30903244018555,
      "activations/layer15_attention_weight_max": 41.58317565917969,
      "activations/layer15_attention_weight_min": -29.687498092651367,
      "activations/layer16_attention_weight_max": 34.49606704711914,
      "activations/layer16_attention_weight_min": -26.970462799072266,
      "activations/layer17_attention_weight_max": 54.00654983520508,
      "activations/layer17_attention_weight_min": -43.5829963684082,
      "activations/layer18_attention_weight_max": 47.79344940185547,
      "activations/layer18_attention_weight_min": -36.35753631591797,
      "activations/layer19_attention_weight_max": 24.520723342895508,
      "activations/layer19_attention_weight_min": -21.3182373046875,
      "activations/layer1_attention_weight_max": 16.76085090637207,
      "activations/layer1_attention_weight_min": -14.615289688110352,
      "activations/layer20_attention_weight_max": 23.467533111572266,
      "activations/layer20_attention_weight_min": -21.13617515563965,
      "activations/layer21_attention_weight_max": 45.220401763916016,
      "activations/layer21_attention_weight_min": -22.402612686157227,
      "activations/layer22_attention_weight_max": 33.2009162902832,
      "activations/layer22_attention_weight_min": -26.753599166870117,
      "activations/layer23_attention_weight_max": 40.56101989746094,
      "activations/layer23_attention_weight_min": -22.0299129486084,
      "activations/layer2_attention_weight_max": 31.86606216430664,
      "activations/layer2_attention_weight_min": -29.32677459716797,
      "activations/layer3_attention_weight_max": 85.15727233886719,
      "activations/layer3_attention_weight_min": -89.4166488647461,
      "activations/layer4_attention_weight_max": 101.23377990722656,
      "activations/layer4_attention_weight_min": -95.34004211425781,
      "activations/layer5_attention_weight_max": 89.96157836914062,
      "activations/layer5_attention_weight_min": -85.13092803955078,
      "activations/layer6_attention_weight_max": 55.043670654296875,
      "activations/layer6_attention_weight_min": -52.61839294433594,
      "activations/layer7_attention_weight_max": 77.34748077392578,
      "activations/layer7_attention_weight_min": -73.75000762939453,
      "activations/layer8_attention_weight_max": 45.84251403808594,
      "activations/layer8_attention_weight_min": -45.50690460205078,
      "activations/layer9_attention_weight_max": 51.03990173339844,
      "activations/layer9_attention_weight_min": -46.3145637512207,
      "epoch": 7.54,
      "learning_rate": 0.00010239431818181817,
      "loss": 2.8166,
      "step": 129800
    },
    {
      "activations/layer0_attention_weight_max": 14.617191314697266,
      "activations/layer0_attention_weight_min": -15.181845664978027,
      "activations/layer10_attention_weight_max": 37.007747650146484,
      "activations/layer10_attention_weight_min": -35.2362060546875,
      "activations/layer11_attention_weight_max": 29.984943389892578,
      "activations/layer11_attention_weight_min": -28.45665740966797,
      "activations/layer12_attention_weight_max": 19.1031494140625,
      "activations/layer12_attention_weight_min": -27.20773696899414,
      "activations/layer13_attention_weight_max": 39.97788619995117,
      "activations/layer13_attention_weight_min": -28.30902099609375,
      "activations/layer14_attention_weight_max": 37.18772506713867,
      "activations/layer14_attention_weight_min": -30.18294906616211,
      "activations/layer15_attention_weight_max": 36.749732971191406,
      "activations/layer15_attention_weight_min": -28.21323585510254,
      "activations/layer16_attention_weight_max": 32.52214813232422,
      "activations/layer16_attention_weight_min": -27.13755989074707,
      "activations/layer17_attention_weight_max": 53.20581817626953,
      "activations/layer17_attention_weight_min": -43.03182601928711,
      "activations/layer18_attention_weight_max": 50.07366180419922,
      "activations/layer18_attention_weight_min": -34.2849235534668,
      "activations/layer19_attention_weight_max": 22.56527328491211,
      "activations/layer19_attention_weight_min": -20.180295944213867,
      "activations/layer1_attention_weight_max": 17.221227645874023,
      "activations/layer1_attention_weight_min": -13.908000946044922,
      "activations/layer20_attention_weight_max": 22.828336715698242,
      "activations/layer20_attention_weight_min": -19.486658096313477,
      "activations/layer21_attention_weight_max": 35.003211975097656,
      "activations/layer21_attention_weight_min": -21.249216079711914,
      "activations/layer22_attention_weight_max": 32.901084899902344,
      "activations/layer22_attention_weight_min": -23.59695053100586,
      "activations/layer23_attention_weight_max": 34.292579650878906,
      "activations/layer23_attention_weight_min": -23.596609115600586,
      "activations/layer2_attention_weight_max": 30.73475456237793,
      "activations/layer2_attention_weight_min": -28.47745132446289,
      "activations/layer3_attention_weight_max": 89.1967544555664,
      "activations/layer3_attention_weight_min": -87.77306365966797,
      "activations/layer4_attention_weight_max": 103.12285614013672,
      "activations/layer4_attention_weight_min": -95.9018783569336,
      "activations/layer5_attention_weight_max": 89.9349365234375,
      "activations/layer5_attention_weight_min": -81.35767364501953,
      "activations/layer6_attention_weight_max": 55.13978958129883,
      "activations/layer6_attention_weight_min": -52.15162658691406,
      "activations/layer7_attention_weight_max": 75.63621520996094,
      "activations/layer7_attention_weight_min": -70.19386291503906,
      "activations/layer8_attention_weight_max": 45.94514846801758,
      "activations/layer8_attention_weight_min": -46.734737396240234,
      "activations/layer9_attention_weight_max": 47.09300994873047,
      "activations/layer9_attention_weight_min": -47.52029800415039,
      "epoch": 7.55,
      "learning_rate": 0.00010237537878787878,
      "loss": 2.816,
      "step": 129850
    },
    {
      "activations/layer0_attention_weight_max": 14.274642944335938,
      "activations/layer0_attention_weight_min": -14.796192169189453,
      "activations/layer10_attention_weight_max": 39.61029815673828,
      "activations/layer10_attention_weight_min": -36.58477783203125,
      "activations/layer11_attention_weight_max": 33.562744140625,
      "activations/layer11_attention_weight_min": -32.225032806396484,
      "activations/layer12_attention_weight_max": 22.518543243408203,
      "activations/layer12_attention_weight_min": -24.38323974609375,
      "activations/layer13_attention_weight_max": 43.79197692871094,
      "activations/layer13_attention_weight_min": -31.94977378845215,
      "activations/layer14_attention_weight_max": 41.82059860229492,
      "activations/layer14_attention_weight_min": -29.014698028564453,
      "activations/layer15_attention_weight_max": 33.038150787353516,
      "activations/layer15_attention_weight_min": -28.199905395507812,
      "activations/layer16_attention_weight_max": 31.519237518310547,
      "activations/layer16_attention_weight_min": -25.89787483215332,
      "activations/layer17_attention_weight_max": 51.04734420776367,
      "activations/layer17_attention_weight_min": -39.78646469116211,
      "activations/layer18_attention_weight_max": 45.35209655761719,
      "activations/layer18_attention_weight_min": -32.82564163208008,
      "activations/layer19_attention_weight_max": 20.730253219604492,
      "activations/layer19_attention_weight_min": -22.621662139892578,
      "activations/layer1_attention_weight_max": 16.774629592895508,
      "activations/layer1_attention_weight_min": -14.248103141784668,
      "activations/layer20_attention_weight_max": 20.027894973754883,
      "activations/layer20_attention_weight_min": -23.128562927246094,
      "activations/layer21_attention_weight_max": 36.88931655883789,
      "activations/layer21_attention_weight_min": -22.599163055419922,
      "activations/layer22_attention_weight_max": 31.96675682067871,
      "activations/layer22_attention_weight_min": -24.823566436767578,
      "activations/layer23_attention_weight_max": 32.53578567504883,
      "activations/layer23_attention_weight_min": -21.772214889526367,
      "activations/layer2_attention_weight_max": 31.27385711669922,
      "activations/layer2_attention_weight_min": -29.552593231201172,
      "activations/layer3_attention_weight_max": 85.98558044433594,
      "activations/layer3_attention_weight_min": -88.18920135498047,
      "activations/layer4_attention_weight_max": 106.44083404541016,
      "activations/layer4_attention_weight_min": -102.6240463256836,
      "activations/layer5_attention_weight_max": 91.0991439819336,
      "activations/layer5_attention_weight_min": -85.77560424804688,
      "activations/layer6_attention_weight_max": 57.053199768066406,
      "activations/layer6_attention_weight_min": -54.593223571777344,
      "activations/layer7_attention_weight_max": 76.31806945800781,
      "activations/layer7_attention_weight_min": -74.78146362304688,
      "activations/layer8_attention_weight_max": 46.534019470214844,
      "activations/layer8_attention_weight_min": -45.63334274291992,
      "activations/layer9_attention_weight_max": 53.38480758666992,
      "activations/layer9_attention_weight_min": -51.064144134521484,
      "epoch": 7.55,
      "learning_rate": 0.00010235643939393939,
      "loss": 2.826,
      "step": 129900
    },
    {
      "activations/layer0_attention_weight_max": 15.041351318359375,
      "activations/layer0_attention_weight_min": -15.052685737609863,
      "activations/layer10_attention_weight_max": 36.62154006958008,
      "activations/layer10_attention_weight_min": -35.03055191040039,
      "activations/layer11_attention_weight_max": 30.153928756713867,
      "activations/layer11_attention_weight_min": -30.323028564453125,
      "activations/layer12_attention_weight_max": 20.228424072265625,
      "activations/layer12_attention_weight_min": -23.22329330444336,
      "activations/layer13_attention_weight_max": 40.30650329589844,
      "activations/layer13_attention_weight_min": -29.517330169677734,
      "activations/layer14_attention_weight_max": 36.06217575073242,
      "activations/layer14_attention_weight_min": -30.09857177734375,
      "activations/layer15_attention_weight_max": 34.76331329345703,
      "activations/layer15_attention_weight_min": -30.742328643798828,
      "activations/layer16_attention_weight_max": 31.733495712280273,
      "activations/layer16_attention_weight_min": -28.710371017456055,
      "activations/layer17_attention_weight_max": 56.99269104003906,
      "activations/layer17_attention_weight_min": -43.612548828125,
      "activations/layer18_attention_weight_max": 49.02404022216797,
      "activations/layer18_attention_weight_min": -38.10817337036133,
      "activations/layer19_attention_weight_max": 20.1669979095459,
      "activations/layer19_attention_weight_min": -22.166593551635742,
      "activations/layer1_attention_weight_max": 17.247844696044922,
      "activations/layer1_attention_weight_min": -15.338248252868652,
      "activations/layer20_attention_weight_max": 20.992305755615234,
      "activations/layer20_attention_weight_min": -23.74094581604004,
      "activations/layer21_attention_weight_max": 36.35353469848633,
      "activations/layer21_attention_weight_min": -24.79276466369629,
      "activations/layer22_attention_weight_max": 33.89154815673828,
      "activations/layer22_attention_weight_min": -25.5430965423584,
      "activations/layer23_attention_weight_max": 34.243309020996094,
      "activations/layer23_attention_weight_min": -21.37678337097168,
      "activations/layer2_attention_weight_max": 32.746063232421875,
      "activations/layer2_attention_weight_min": -30.010116577148438,
      "activations/layer3_attention_weight_max": 88.26158905029297,
      "activations/layer3_attention_weight_min": -87.81790161132812,
      "activations/layer4_attention_weight_max": 100.0886459350586,
      "activations/layer4_attention_weight_min": -94.0265121459961,
      "activations/layer5_attention_weight_max": 88.07186889648438,
      "activations/layer5_attention_weight_min": -81.6768569946289,
      "activations/layer6_attention_weight_max": 53.74391555786133,
      "activations/layer6_attention_weight_min": -54.655906677246094,
      "activations/layer7_attention_weight_max": 76.38618469238281,
      "activations/layer7_attention_weight_min": -72.5584487915039,
      "activations/layer8_attention_weight_max": 46.01218032836914,
      "activations/layer8_attention_weight_min": -45.2490234375,
      "activations/layer9_attention_weight_max": 47.722625732421875,
      "activations/layer9_attention_weight_min": -48.80807113647461,
      "epoch": 7.55,
      "learning_rate": 0.0001023375,
      "loss": 2.8259,
      "step": 129950
    },
    {
      "activations/layer0_attention_weight_max": 15.05258846282959,
      "activations/layer0_attention_weight_min": -14.115251541137695,
      "activations/layer10_attention_weight_max": 36.97039031982422,
      "activations/layer10_attention_weight_min": -32.76766586303711,
      "activations/layer11_attention_weight_max": 32.738014221191406,
      "activations/layer11_attention_weight_min": -28.319793701171875,
      "activations/layer12_attention_weight_max": 19.08963966369629,
      "activations/layer12_attention_weight_min": -25.65676498413086,
      "activations/layer13_attention_weight_max": 43.55268478393555,
      "activations/layer13_attention_weight_min": -31.472789764404297,
      "activations/layer14_attention_weight_max": 38.060848236083984,
      "activations/layer14_attention_weight_min": -29.459243774414062,
      "activations/layer15_attention_weight_max": 36.10470199584961,
      "activations/layer15_attention_weight_min": -32.550662994384766,
      "activations/layer16_attention_weight_max": 33.369590759277344,
      "activations/layer16_attention_weight_min": -27.121389389038086,
      "activations/layer17_attention_weight_max": 54.375511169433594,
      "activations/layer17_attention_weight_min": -44.018104553222656,
      "activations/layer18_attention_weight_max": 46.84239959716797,
      "activations/layer18_attention_weight_min": -35.17585372924805,
      "activations/layer19_attention_weight_max": 23.44264030456543,
      "activations/layer19_attention_weight_min": -19.12914276123047,
      "activations/layer1_attention_weight_max": 17.368223190307617,
      "activations/layer1_attention_weight_min": -11.422952651977539,
      "activations/layer20_attention_weight_max": 22.590179443359375,
      "activations/layer20_attention_weight_min": -20.643526077270508,
      "activations/layer21_attention_weight_max": 41.611328125,
      "activations/layer21_attention_weight_min": -23.160165786743164,
      "activations/layer22_attention_weight_max": 35.52205276489258,
      "activations/layer22_attention_weight_min": -27.32004737854004,
      "activations/layer23_attention_weight_max": 38.06963348388672,
      "activations/layer23_attention_weight_min": -22.371761322021484,
      "activations/layer2_attention_weight_max": 25.84115219116211,
      "activations/layer2_attention_weight_min": -23.032865524291992,
      "activations/layer3_attention_weight_max": 63.168975830078125,
      "activations/layer3_attention_weight_min": -64.2774658203125,
      "activations/layer4_attention_weight_max": 85.72622680664062,
      "activations/layer4_attention_weight_min": -78.3382568359375,
      "activations/layer5_attention_weight_max": 78.92842102050781,
      "activations/layer5_attention_weight_min": -68.47050476074219,
      "activations/layer6_attention_weight_max": 49.28915023803711,
      "activations/layer6_attention_weight_min": -45.90628433227539,
      "activations/layer7_attention_weight_max": 75.76156616210938,
      "activations/layer7_attention_weight_min": -72.08848571777344,
      "activations/layer8_attention_weight_max": 43.27189254760742,
      "activations/layer8_attention_weight_min": -39.03855514526367,
      "activations/layer9_attention_weight_max": 56.94093704223633,
      "activations/layer9_attention_weight_min": -50.515811920166016,
      "epoch": 7.55,
      "learning_rate": 0.00010231856060606059,
      "loss": 2.8283,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_loss": 2.783203125,
      "eval_runtime": 8.5532,
      "eval_samples_per_second": 502.032,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_openwebtext_loss": 2.783203125,
      "eval_openwebtext_ppl": 16.170734966018113,
      "eval_openwebtext_runtime": 8.5532,
      "eval_openwebtext_samples_per_second": 502.032,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 2.041,
      "eval_wikitext_samples_per_second": 223.422,
      "step": 130000
    },
    {
      "epoch": 7.55,
      "eval_lambada_loss": 2.728515625,
      "eval_lambada_ppl": 15.310144149387147,
      "eval_lambada_runtime": 9.617,
      "eval_lambada_samples_per_second": 506.293,
      "step": 130000
    },
    {
      "activations/layer0_attention_weight_max": 15.827919006347656,
      "activations/layer0_attention_weight_min": -12.876913070678711,
      "activations/layer10_attention_weight_max": 34.88835144042969,
      "activations/layer10_attention_weight_min": -34.96142578125,
      "activations/layer11_attention_weight_max": 29.803213119506836,
      "activations/layer11_attention_weight_min": -30.88728904724121,
      "activations/layer12_attention_weight_max": 18.275693893432617,
      "activations/layer12_attention_weight_min": -28.548721313476562,
      "activations/layer13_attention_weight_max": 47.983619689941406,
      "activations/layer13_attention_weight_min": -33.349998474121094,
      "activations/layer14_attention_weight_max": 45.42275619506836,
      "activations/layer14_attention_weight_min": -30.590787887573242,
      "activations/layer15_attention_weight_max": 36.03787612915039,
      "activations/layer15_attention_weight_min": -30.039222717285156,
      "activations/layer16_attention_weight_max": 32.664451599121094,
      "activations/layer16_attention_weight_min": -30.45021629333496,
      "activations/layer17_attention_weight_max": 53.341278076171875,
      "activations/layer17_attention_weight_min": -45.68318557739258,
      "activations/layer18_attention_weight_max": 45.7301025390625,
      "activations/layer18_attention_weight_min": -36.6160774230957,
      "activations/layer19_attention_weight_max": 22.397130966186523,
      "activations/layer19_attention_weight_min": -20.160329818725586,
      "activations/layer1_attention_weight_max": 17.499290466308594,
      "activations/layer1_attention_weight_min": -16.238346099853516,
      "activations/layer20_attention_weight_max": 23.633764266967773,
      "activations/layer20_attention_weight_min": -22.229719161987305,
      "activations/layer21_attention_weight_max": 43.7586669921875,
      "activations/layer21_attention_weight_min": -26.158580780029297,
      "activations/layer22_attention_weight_max": 33.5495491027832,
      "activations/layer22_attention_weight_min": -25.163042068481445,
      "activations/layer23_attention_weight_max": 41.33919906616211,
      "activations/layer23_attention_weight_min": -20.799968719482422,
      "activations/layer2_attention_weight_max": 30.362651824951172,
      "activations/layer2_attention_weight_min": -29.220661163330078,
      "activations/layer3_attention_weight_max": 88.11772918701172,
      "activations/layer3_attention_weight_min": -84.3165283203125,
      "activations/layer4_attention_weight_max": 98.54517364501953,
      "activations/layer4_attention_weight_min": -93.63290405273438,
      "activations/layer5_attention_weight_max": 86.83869934082031,
      "activations/layer5_attention_weight_min": -80.32816314697266,
      "activations/layer6_attention_weight_max": 52.43956756591797,
      "activations/layer6_attention_weight_min": -52.324771881103516,
      "activations/layer7_attention_weight_max": 79.0089340209961,
      "activations/layer7_attention_weight_min": -73.67961120605469,
      "activations/layer8_attention_weight_max": 43.526790618896484,
      "activations/layer8_attention_weight_min": -42.64249801635742,
      "activations/layer9_attention_weight_max": 55.99720001220703,
      "activations/layer9_attention_weight_min": -54.25327682495117,
      "epoch": 7.56,
      "learning_rate": 0.0001022996212121212,
      "loss": 2.8269,
      "step": 130050
    },
    {
      "activations/layer0_attention_weight_max": 14.940896034240723,
      "activations/layer0_attention_weight_min": -13.256429672241211,
      "activations/layer10_attention_weight_max": 37.472373962402344,
      "activations/layer10_attention_weight_min": -35.0834846496582,
      "activations/layer11_attention_weight_max": 29.754959106445312,
      "activations/layer11_attention_weight_min": -30.473434448242188,
      "activations/layer12_attention_weight_max": 21.83792495727539,
      "activations/layer12_attention_weight_min": -26.13941764831543,
      "activations/layer13_attention_weight_max": 43.67988586425781,
      "activations/layer13_attention_weight_min": -30.764333724975586,
      "activations/layer14_attention_weight_max": 38.54142379760742,
      "activations/layer14_attention_weight_min": -30.40557861328125,
      "activations/layer15_attention_weight_max": 36.02544403076172,
      "activations/layer15_attention_weight_min": -30.151254653930664,
      "activations/layer16_attention_weight_max": 32.676326751708984,
      "activations/layer16_attention_weight_min": -28.340105056762695,
      "activations/layer17_attention_weight_max": 51.97927474975586,
      "activations/layer17_attention_weight_min": -43.21371841430664,
      "activations/layer18_attention_weight_max": 44.63386535644531,
      "activations/layer18_attention_weight_min": -35.32442092895508,
      "activations/layer19_attention_weight_max": 21.85637664794922,
      "activations/layer19_attention_weight_min": -20.00336265563965,
      "activations/layer1_attention_weight_max": 16.46110725402832,
      "activations/layer1_attention_weight_min": -13.921342849731445,
      "activations/layer20_attention_weight_max": 22.874589920043945,
      "activations/layer20_attention_weight_min": -21.163166046142578,
      "activations/layer21_attention_weight_max": 38.982112884521484,
      "activations/layer21_attention_weight_min": -24.36758804321289,
      "activations/layer22_attention_weight_max": 33.72809600830078,
      "activations/layer22_attention_weight_min": -26.752229690551758,
      "activations/layer23_attention_weight_max": 37.06480026245117,
      "activations/layer23_attention_weight_min": -20.782548904418945,
      "activations/layer2_attention_weight_max": 30.727519989013672,
      "activations/layer2_attention_weight_min": -29.666728973388672,
      "activations/layer3_attention_weight_max": 79.41655731201172,
      "activations/layer3_attention_weight_min": -81.74320220947266,
      "activations/layer4_attention_weight_max": 93.62444305419922,
      "activations/layer4_attention_weight_min": -89.46770477294922,
      "activations/layer5_attention_weight_max": 83.07577514648438,
      "activations/layer5_attention_weight_min": -76.89543151855469,
      "activations/layer6_attention_weight_max": 54.11333084106445,
      "activations/layer6_attention_weight_min": -50.460697174072266,
      "activations/layer7_attention_weight_max": 70.54293060302734,
      "activations/layer7_attention_weight_min": -71.7951889038086,
      "activations/layer8_attention_weight_max": 43.65126419067383,
      "activations/layer8_attention_weight_min": -44.483245849609375,
      "activations/layer9_attention_weight_max": 51.41705322265625,
      "activations/layer9_attention_weight_min": -50.224639892578125,
      "epoch": 7.56,
      "learning_rate": 0.00010228068181818182,
      "loss": 2.816,
      "step": 130100
    },
    {
      "activations/layer0_attention_weight_max": 15.189011573791504,
      "activations/layer0_attention_weight_min": -13.505590438842773,
      "activations/layer10_attention_weight_max": 37.58879470825195,
      "activations/layer10_attention_weight_min": -36.417572021484375,
      "activations/layer11_attention_weight_max": 37.76215362548828,
      "activations/layer11_attention_weight_min": -30.028135299682617,
      "activations/layer12_attention_weight_max": 21.054651260375977,
      "activations/layer12_attention_weight_min": -24.694595336914062,
      "activations/layer13_attention_weight_max": 40.270973205566406,
      "activations/layer13_attention_weight_min": -30.552629470825195,
      "activations/layer14_attention_weight_max": 35.67148208618164,
      "activations/layer14_attention_weight_min": -29.027408599853516,
      "activations/layer15_attention_weight_max": 33.030174255371094,
      "activations/layer15_attention_weight_min": -27.729101181030273,
      "activations/layer16_attention_weight_max": 31.471996307373047,
      "activations/layer16_attention_weight_min": -28.5167236328125,
      "activations/layer17_attention_weight_max": 53.6141242980957,
      "activations/layer17_attention_weight_min": -42.22652816772461,
      "activations/layer18_attention_weight_max": 43.59185791015625,
      "activations/layer18_attention_weight_min": -35.125301361083984,
      "activations/layer19_attention_weight_max": 24.062110900878906,
      "activations/layer19_attention_weight_min": -20.202362060546875,
      "activations/layer1_attention_weight_max": 17.367202758789062,
      "activations/layer1_attention_weight_min": -14.237881660461426,
      "activations/layer20_attention_weight_max": 21.019548416137695,
      "activations/layer20_attention_weight_min": -23.181140899658203,
      "activations/layer21_attention_weight_max": 34.36325454711914,
      "activations/layer21_attention_weight_min": -24.79656982421875,
      "activations/layer22_attention_weight_max": 33.01652908325195,
      "activations/layer22_attention_weight_min": -25.598529815673828,
      "activations/layer23_attention_weight_max": 32.570762634277344,
      "activations/layer23_attention_weight_min": -22.280963897705078,
      "activations/layer2_attention_weight_max": 31.14374542236328,
      "activations/layer2_attention_weight_min": -28.31771469116211,
      "activations/layer3_attention_weight_max": 86.08845520019531,
      "activations/layer3_attention_weight_min": -83.72952270507812,
      "activations/layer4_attention_weight_max": 97.20147705078125,
      "activations/layer4_attention_weight_min": -90.86795806884766,
      "activations/layer5_attention_weight_max": 86.44584655761719,
      "activations/layer5_attention_weight_min": -81.23841857910156,
      "activations/layer6_attention_weight_max": 53.14781188964844,
      "activations/layer6_attention_weight_min": -50.24598693847656,
      "activations/layer7_attention_weight_max": 82.23776245117188,
      "activations/layer7_attention_weight_min": -70.48280334472656,
      "activations/layer8_attention_weight_max": 52.73304748535156,
      "activations/layer8_attention_weight_min": -46.353694915771484,
      "activations/layer9_attention_weight_max": 64.81904602050781,
      "activations/layer9_attention_weight_min": -49.19697952270508,
      "epoch": 7.56,
      "learning_rate": 0.0001022617424242424,
      "loss": 2.8135,
      "step": 130150
    },
    {
      "activations/layer0_attention_weight_max": 14.498908996582031,
      "activations/layer0_attention_weight_min": -14.022768020629883,
      "activations/layer10_attention_weight_max": 39.184906005859375,
      "activations/layer10_attention_weight_min": -35.85124588012695,
      "activations/layer11_attention_weight_max": 30.39175033569336,
      "activations/layer11_attention_weight_min": -30.957351684570312,
      "activations/layer12_attention_weight_max": 20.14864158630371,
      "activations/layer12_attention_weight_min": -24.13408660888672,
      "activations/layer13_attention_weight_max": 53.29619216918945,
      "activations/layer13_attention_weight_min": -35.16537094116211,
      "activations/layer14_attention_weight_max": 50.292633056640625,
      "activations/layer14_attention_weight_min": -31.764427185058594,
      "activations/layer15_attention_weight_max": 39.019309997558594,
      "activations/layer15_attention_weight_min": -28.975393295288086,
      "activations/layer16_attention_weight_max": 39.959632873535156,
      "activations/layer16_attention_weight_min": -29.927560806274414,
      "activations/layer17_attention_weight_max": 59.48965835571289,
      "activations/layer17_attention_weight_min": -43.44562911987305,
      "activations/layer18_attention_weight_max": 49.4141960144043,
      "activations/layer18_attention_weight_min": -35.07880783081055,
      "activations/layer19_attention_weight_max": 22.426488876342773,
      "activations/layer19_attention_weight_min": -21.94562339782715,
      "activations/layer1_attention_weight_max": 16.31357765197754,
      "activations/layer1_attention_weight_min": -14.246865272521973,
      "activations/layer20_attention_weight_max": 25.294780731201172,
      "activations/layer20_attention_weight_min": -22.67559242248535,
      "activations/layer21_attention_weight_max": 45.45661926269531,
      "activations/layer21_attention_weight_min": -25.125131607055664,
      "activations/layer22_attention_weight_max": 33.127830505371094,
      "activations/layer22_attention_weight_min": -26.438739776611328,
      "activations/layer23_attention_weight_max": 37.843963623046875,
      "activations/layer23_attention_weight_min": -20.55902862548828,
      "activations/layer2_attention_weight_max": 30.554349899291992,
      "activations/layer2_attention_weight_min": -28.87511444091797,
      "activations/layer3_attention_weight_max": 81.40179443359375,
      "activations/layer3_attention_weight_min": -89.50308990478516,
      "activations/layer4_attention_weight_max": 99.90414428710938,
      "activations/layer4_attention_weight_min": -96.47251892089844,
      "activations/layer5_attention_weight_max": 88.3448486328125,
      "activations/layer5_attention_weight_min": -83.50955963134766,
      "activations/layer6_attention_weight_max": 57.1999626159668,
      "activations/layer6_attention_weight_min": -57.95039749145508,
      "activations/layer7_attention_weight_max": 74.8000717163086,
      "activations/layer7_attention_weight_min": -73.33049011230469,
      "activations/layer8_attention_weight_max": 44.623008728027344,
      "activations/layer8_attention_weight_min": -46.298282623291016,
      "activations/layer9_attention_weight_max": 50.28834533691406,
      "activations/layer9_attention_weight_min": -48.64297103881836,
      "epoch": 7.57,
      "learning_rate": 0.00010224280303030302,
      "loss": 2.8253,
      "step": 130200
    },
    {
      "activations/layer0_attention_weight_max": 14.04308795928955,
      "activations/layer0_attention_weight_min": -14.259684562683105,
      "activations/layer10_attention_weight_max": 36.43006134033203,
      "activations/layer10_attention_weight_min": -35.29030227661133,
      "activations/layer11_attention_weight_max": 33.00624084472656,
      "activations/layer11_attention_weight_min": -29.68390655517578,
      "activations/layer12_attention_weight_max": 19.768531799316406,
      "activations/layer12_attention_weight_min": -25.04498863220215,
      "activations/layer13_attention_weight_max": 51.76408386230469,
      "activations/layer13_attention_weight_min": -37.02074432373047,
      "activations/layer14_attention_weight_max": 53.090476989746094,
      "activations/layer14_attention_weight_min": -39.00632095336914,
      "activations/layer15_attention_weight_max": 37.89826965332031,
      "activations/layer15_attention_weight_min": -29.519128799438477,
      "activations/layer16_attention_weight_max": 31.77124786376953,
      "activations/layer16_attention_weight_min": -26.695518493652344,
      "activations/layer17_attention_weight_max": 53.21315383911133,
      "activations/layer17_attention_weight_min": -45.419742584228516,
      "activations/layer18_attention_weight_max": 48.19990158081055,
      "activations/layer18_attention_weight_min": -38.51060485839844,
      "activations/layer19_attention_weight_max": 24.66803550720215,
      "activations/layer19_attention_weight_min": -21.048139572143555,
      "activations/layer1_attention_weight_max": 18.2618408203125,
      "activations/layer1_attention_weight_min": -14.106243133544922,
      "activations/layer20_attention_weight_max": 21.80073356628418,
      "activations/layer20_attention_weight_min": -22.521629333496094,
      "activations/layer21_attention_weight_max": 42.143829345703125,
      "activations/layer21_attention_weight_min": -26.57029914855957,
      "activations/layer22_attention_weight_max": 34.451961517333984,
      "activations/layer22_attention_weight_min": -28.588533401489258,
      "activations/layer23_attention_weight_max": 37.290931701660156,
      "activations/layer23_attention_weight_min": -24.180870056152344,
      "activations/layer2_attention_weight_max": 29.542156219482422,
      "activations/layer2_attention_weight_min": -28.387413024902344,
      "activations/layer3_attention_weight_max": 87.53743743896484,
      "activations/layer3_attention_weight_min": -85.78099060058594,
      "activations/layer4_attention_weight_max": 104.0014877319336,
      "activations/layer4_attention_weight_min": -97.00813293457031,
      "activations/layer5_attention_weight_max": 90.98876953125,
      "activations/layer5_attention_weight_min": -85.1221694946289,
      "activations/layer6_attention_weight_max": 54.81895446777344,
      "activations/layer6_attention_weight_min": -53.38331604003906,
      "activations/layer7_attention_weight_max": 78.01472473144531,
      "activations/layer7_attention_weight_min": -75.91090393066406,
      "activations/layer8_attention_weight_max": 47.2415657043457,
      "activations/layer8_attention_weight_min": -47.49003601074219,
      "activations/layer9_attention_weight_max": 60.029884338378906,
      "activations/layer9_attention_weight_min": -49.84581756591797,
      "epoch": 7.57,
      "learning_rate": 0.00010222386363636364,
      "loss": 2.8047,
      "step": 130250
    },
    {
      "activations/layer0_attention_weight_max": 14.187094688415527,
      "activations/layer0_attention_weight_min": -13.976240158081055,
      "activations/layer10_attention_weight_max": 38.443729400634766,
      "activations/layer10_attention_weight_min": -36.191532135009766,
      "activations/layer11_attention_weight_max": 31.45745849609375,
      "activations/layer11_attention_weight_min": -30.211593627929688,
      "activations/layer12_attention_weight_max": 19.767765045166016,
      "activations/layer12_attention_weight_min": -24.543970108032227,
      "activations/layer13_attention_weight_max": 42.20402908325195,
      "activations/layer13_attention_weight_min": -33.63315200805664,
      "activations/layer14_attention_weight_max": 36.619972229003906,
      "activations/layer14_attention_weight_min": -29.61263084411621,
      "activations/layer15_attention_weight_max": 36.67006301879883,
      "activations/layer15_attention_weight_min": -29.080286026000977,
      "activations/layer16_attention_weight_max": 31.34467315673828,
      "activations/layer16_attention_weight_min": -26.280555725097656,
      "activations/layer17_attention_weight_max": 56.95729446411133,
      "activations/layer17_attention_weight_min": -41.31366729736328,
      "activations/layer18_attention_weight_max": 50.96079635620117,
      "activations/layer18_attention_weight_min": -34.56166458129883,
      "activations/layer19_attention_weight_max": 24.94522476196289,
      "activations/layer19_attention_weight_min": -20.868234634399414,
      "activations/layer1_attention_weight_max": 15.65844440460205,
      "activations/layer1_attention_weight_min": -14.136175155639648,
      "activations/layer20_attention_weight_max": 22.730224609375,
      "activations/layer20_attention_weight_min": -19.766948699951172,
      "activations/layer21_attention_weight_max": 37.75141525268555,
      "activations/layer21_attention_weight_min": -25.1134033203125,
      "activations/layer22_attention_weight_max": 33.2095947265625,
      "activations/layer22_attention_weight_min": -26.79974937438965,
      "activations/layer23_attention_weight_max": 35.578277587890625,
      "activations/layer23_attention_weight_min": -20.72883415222168,
      "activations/layer2_attention_weight_max": 30.327604293823242,
      "activations/layer2_attention_weight_min": -28.215213775634766,
      "activations/layer3_attention_weight_max": 85.65674591064453,
      "activations/layer3_attention_weight_min": -87.07198333740234,
      "activations/layer4_attention_weight_max": 97.47547912597656,
      "activations/layer4_attention_weight_min": -95.82688903808594,
      "activations/layer5_attention_weight_max": 87.79888916015625,
      "activations/layer5_attention_weight_min": -84.608154296875,
      "activations/layer6_attention_weight_max": 55.82080841064453,
      "activations/layer6_attention_weight_min": -52.86001205444336,
      "activations/layer7_attention_weight_max": 74.9491195678711,
      "activations/layer7_attention_weight_min": -77.35396575927734,
      "activations/layer8_attention_weight_max": 44.15158462524414,
      "activations/layer8_attention_weight_min": -44.08085250854492,
      "activations/layer9_attention_weight_max": 50.235164642333984,
      "activations/layer9_attention_weight_min": -49.43197250366211,
      "epoch": 7.57,
      "learning_rate": 0.00010220492424242422,
      "loss": 2.8162,
      "step": 130300
    },
    {
      "activations/layer0_attention_weight_max": 14.065475463867188,
      "activations/layer0_attention_weight_min": -14.587691307067871,
      "activations/layer10_attention_weight_max": 40.19364547729492,
      "activations/layer10_attention_weight_min": -37.713844299316406,
      "activations/layer11_attention_weight_max": 34.31963348388672,
      "activations/layer11_attention_weight_min": -31.084508895874023,
      "activations/layer12_attention_weight_max": 19.103899002075195,
      "activations/layer12_attention_weight_min": -28.16751480102539,
      "activations/layer13_attention_weight_max": 49.55937576293945,
      "activations/layer13_attention_weight_min": -38.66144561767578,
      "activations/layer14_attention_weight_max": 51.68526840209961,
      "activations/layer14_attention_weight_min": -33.10165023803711,
      "activations/layer15_attention_weight_max": 40.34786605834961,
      "activations/layer15_attention_weight_min": -27.965211868286133,
      "activations/layer16_attention_weight_max": 34.12321472167969,
      "activations/layer16_attention_weight_min": -27.08458137512207,
      "activations/layer17_attention_weight_max": 52.5889892578125,
      "activations/layer17_attention_weight_min": -42.03311538696289,
      "activations/layer18_attention_weight_max": 47.56163787841797,
      "activations/layer18_attention_weight_min": -34.59080505371094,
      "activations/layer19_attention_weight_max": 21.8406925201416,
      "activations/layer19_attention_weight_min": -19.858009338378906,
      "activations/layer1_attention_weight_max": 20.05849266052246,
      "activations/layer1_attention_weight_min": -15.488204956054688,
      "activations/layer20_attention_weight_max": 22.78827476501465,
      "activations/layer20_attention_weight_min": -21.7598819732666,
      "activations/layer21_attention_weight_max": 39.68109893798828,
      "activations/layer21_attention_weight_min": -24.17685317993164,
      "activations/layer22_attention_weight_max": 31.957950592041016,
      "activations/layer22_attention_weight_min": -25.960676193237305,
      "activations/layer23_attention_weight_max": 34.416290283203125,
      "activations/layer23_attention_weight_min": -21.953832626342773,
      "activations/layer2_attention_weight_max": 31.081998825073242,
      "activations/layer2_attention_weight_min": -28.52062225341797,
      "activations/layer3_attention_weight_max": 88.00467681884766,
      "activations/layer3_attention_weight_min": -89.40104675292969,
      "activations/layer4_attention_weight_max": 107.33662414550781,
      "activations/layer4_attention_weight_min": -97.81826782226562,
      "activations/layer5_attention_weight_max": 93.0755844116211,
      "activations/layer5_attention_weight_min": -82.89368438720703,
      "activations/layer6_attention_weight_max": 59.490360260009766,
      "activations/layer6_attention_weight_min": -55.077392578125,
      "activations/layer7_attention_weight_max": 78.41719818115234,
      "activations/layer7_attention_weight_min": -75.44558715820312,
      "activations/layer8_attention_weight_max": 49.19852066040039,
      "activations/layer8_attention_weight_min": -48.70869445800781,
      "activations/layer9_attention_weight_max": 52.27128219604492,
      "activations/layer9_attention_weight_min": -51.00802230834961,
      "epoch": 7.57,
      "learning_rate": 0.00010218598484848484,
      "loss": 2.8018,
      "step": 130350
    },
    {
      "activations/layer0_attention_weight_max": 14.662149429321289,
      "activations/layer0_attention_weight_min": -14.206884384155273,
      "activations/layer10_attention_weight_max": 36.2600212097168,
      "activations/layer10_attention_weight_min": -34.64248275756836,
      "activations/layer11_attention_weight_max": 30.428007125854492,
      "activations/layer11_attention_weight_min": -30.870941162109375,
      "activations/layer12_attention_weight_max": 20.00191307067871,
      "activations/layer12_attention_weight_min": -27.62026596069336,
      "activations/layer13_attention_weight_max": 42.24327087402344,
      "activations/layer13_attention_weight_min": -32.6844367980957,
      "activations/layer14_attention_weight_max": 36.70146560668945,
      "activations/layer14_attention_weight_min": -31.177705764770508,
      "activations/layer15_attention_weight_max": 34.91435623168945,
      "activations/layer15_attention_weight_min": -29.55160140991211,
      "activations/layer16_attention_weight_max": 32.1765251159668,
      "activations/layer16_attention_weight_min": -27.72007942199707,
      "activations/layer17_attention_weight_max": 54.67156982421875,
      "activations/layer17_attention_weight_min": -43.3780517578125,
      "activations/layer18_attention_weight_max": 47.5634765625,
      "activations/layer18_attention_weight_min": -35.5781135559082,
      "activations/layer19_attention_weight_max": 19.582456588745117,
      "activations/layer19_attention_weight_min": -21.09695053100586,
      "activations/layer1_attention_weight_max": 17.519622802734375,
      "activations/layer1_attention_weight_min": -16.12531280517578,
      "activations/layer20_attention_weight_max": 22.129009246826172,
      "activations/layer20_attention_weight_min": -22.031597137451172,
      "activations/layer21_attention_weight_max": 41.178436279296875,
      "activations/layer21_attention_weight_min": -25.76291275024414,
      "activations/layer22_attention_weight_max": 31.821748733520508,
      "activations/layer22_attention_weight_min": -23.90626335144043,
      "activations/layer23_attention_weight_max": 35.36037063598633,
      "activations/layer23_attention_weight_min": -19.78732681274414,
      "activations/layer2_attention_weight_max": 30.666202545166016,
      "activations/layer2_attention_weight_min": -28.115842819213867,
      "activations/layer3_attention_weight_max": 87.18646240234375,
      "activations/layer3_attention_weight_min": -89.47700500488281,
      "activations/layer4_attention_weight_max": 100.40792846679688,
      "activations/layer4_attention_weight_min": -94.41605377197266,
      "activations/layer5_attention_weight_max": 85.61568450927734,
      "activations/layer5_attention_weight_min": -81.62769317626953,
      "activations/layer6_attention_weight_max": 55.626312255859375,
      "activations/layer6_attention_weight_min": -52.41427230834961,
      "activations/layer7_attention_weight_max": 77.20670318603516,
      "activations/layer7_attention_weight_min": -73.95647430419922,
      "activations/layer8_attention_weight_max": 42.954463958740234,
      "activations/layer8_attention_weight_min": -44.29750061035156,
      "activations/layer9_attention_weight_max": 49.2426872253418,
      "activations/layer9_attention_weight_min": -50.1200065612793,
      "epoch": 7.58,
      "learning_rate": 0.00010216704545454544,
      "loss": 2.8141,
      "step": 130400
    },
    {
      "activations/layer0_attention_weight_max": 14.828852653503418,
      "activations/layer0_attention_weight_min": -13.873007774353027,
      "activations/layer10_attention_weight_max": 38.29945755004883,
      "activations/layer10_attention_weight_min": -34.734012603759766,
      "activations/layer11_attention_weight_max": 30.74873161315918,
      "activations/layer11_attention_weight_min": -30.97230339050293,
      "activations/layer12_attention_weight_max": 17.83998680114746,
      "activations/layer12_attention_weight_min": -24.708927154541016,
      "activations/layer13_attention_weight_max": 43.2915153503418,
      "activations/layer13_attention_weight_min": -32.87325668334961,
      "activations/layer14_attention_weight_max": 32.924644470214844,
      "activations/layer14_attention_weight_min": -31.001598358154297,
      "activations/layer15_attention_weight_max": 32.7086296081543,
      "activations/layer15_attention_weight_min": -30.29639434814453,
      "activations/layer16_attention_weight_max": 29.90216636657715,
      "activations/layer16_attention_weight_min": -25.48668098449707,
      "activations/layer17_attention_weight_max": 50.30373001098633,
      "activations/layer17_attention_weight_min": -41.61913299560547,
      "activations/layer18_attention_weight_max": 41.72673797607422,
      "activations/layer18_attention_weight_min": -34.94778823852539,
      "activations/layer19_attention_weight_max": 19.626943588256836,
      "activations/layer19_attention_weight_min": -22.030366897583008,
      "activations/layer1_attention_weight_max": 17.28361701965332,
      "activations/layer1_attention_weight_min": -14.229445457458496,
      "activations/layer20_attention_weight_max": 20.869857788085938,
      "activations/layer20_attention_weight_min": -21.366687774658203,
      "activations/layer21_attention_weight_max": 40.93507766723633,
      "activations/layer21_attention_weight_min": -25.652751922607422,
      "activations/layer22_attention_weight_max": 31.297977447509766,
      "activations/layer22_attention_weight_min": -24.906335830688477,
      "activations/layer23_attention_weight_max": 35.40786361694336,
      "activations/layer23_attention_weight_min": -22.27707290649414,
      "activations/layer2_attention_weight_max": 30.756546020507812,
      "activations/layer2_attention_weight_min": -27.875967025756836,
      "activations/layer3_attention_weight_max": 85.14483642578125,
      "activations/layer3_attention_weight_min": -90.1315689086914,
      "activations/layer4_attention_weight_max": 97.2845687866211,
      "activations/layer4_attention_weight_min": -95.5332260131836,
      "activations/layer5_attention_weight_max": 86.09164428710938,
      "activations/layer5_attention_weight_min": -80.85444641113281,
      "activations/layer6_attention_weight_max": 54.75646209716797,
      "activations/layer6_attention_weight_min": -50.61949920654297,
      "activations/layer7_attention_weight_max": 75.62657928466797,
      "activations/layer7_attention_weight_min": -76.38824462890625,
      "activations/layer8_attention_weight_max": 44.17085266113281,
      "activations/layer8_attention_weight_min": -43.62918472290039,
      "activations/layer9_attention_weight_max": 48.25644302368164,
      "activations/layer9_attention_weight_min": -48.371089935302734,
      "epoch": 7.58,
      "learning_rate": 0.00010214810606060606,
      "loss": 2.8172,
      "step": 130450
    },
    {
      "activations/layer0_attention_weight_max": 15.059139251708984,
      "activations/layer0_attention_weight_min": -14.269408226013184,
      "activations/layer10_attention_weight_max": 38.02825927734375,
      "activations/layer10_attention_weight_min": -36.02360153198242,
      "activations/layer11_attention_weight_max": 30.976919174194336,
      "activations/layer11_attention_weight_min": -33.04335021972656,
      "activations/layer12_attention_weight_max": 18.51116371154785,
      "activations/layer12_attention_weight_min": -25.133472442626953,
      "activations/layer13_attention_weight_max": 38.601539611816406,
      "activations/layer13_attention_weight_min": -31.227611541748047,
      "activations/layer14_attention_weight_max": 42.14659881591797,
      "activations/layer14_attention_weight_min": -30.627702713012695,
      "activations/layer15_attention_weight_max": 32.01819610595703,
      "activations/layer15_attention_weight_min": -29.67998695373535,
      "activations/layer16_attention_weight_max": 30.145069122314453,
      "activations/layer16_attention_weight_min": -26.765470504760742,
      "activations/layer17_attention_weight_max": 52.55743408203125,
      "activations/layer17_attention_weight_min": -41.328285217285156,
      "activations/layer18_attention_weight_max": 44.77782440185547,
      "activations/layer18_attention_weight_min": -34.113983154296875,
      "activations/layer19_attention_weight_max": 18.215036392211914,
      "activations/layer19_attention_weight_min": -18.831398010253906,
      "activations/layer1_attention_weight_max": 18.25322151184082,
      "activations/layer1_attention_weight_min": -17.55814552307129,
      "activations/layer20_attention_weight_max": 21.623376846313477,
      "activations/layer20_attention_weight_min": -22.58263397216797,
      "activations/layer21_attention_weight_max": 36.68470001220703,
      "activations/layer21_attention_weight_min": -24.647802352905273,
      "activations/layer22_attention_weight_max": 29.799556732177734,
      "activations/layer22_attention_weight_min": -26.31834602355957,
      "activations/layer23_attention_weight_max": 31.72858238220215,
      "activations/layer23_attention_weight_min": -20.205123901367188,
      "activations/layer2_attention_weight_max": 31.84956932067871,
      "activations/layer2_attention_weight_min": -29.87434196472168,
      "activations/layer3_attention_weight_max": 85.75839233398438,
      "activations/layer3_attention_weight_min": -85.43268585205078,
      "activations/layer4_attention_weight_max": 102.26013946533203,
      "activations/layer4_attention_weight_min": -92.9975814819336,
      "activations/layer5_attention_weight_max": 89.11042785644531,
      "activations/layer5_attention_weight_min": -81.21453857421875,
      "activations/layer6_attention_weight_max": 58.32037353515625,
      "activations/layer6_attention_weight_min": -50.816505432128906,
      "activations/layer7_attention_weight_max": 71.95354461669922,
      "activations/layer7_attention_weight_min": -72.27171325683594,
      "activations/layer8_attention_weight_max": 42.726383209228516,
      "activations/layer8_attention_weight_min": -43.87736892700195,
      "activations/layer9_attention_weight_max": 48.51876449584961,
      "activations/layer9_attention_weight_min": -50.83749008178711,
      "epoch": 7.58,
      "learning_rate": 0.00010212916666666666,
      "loss": 2.8125,
      "step": 130500
    },
    {
      "activations/layer0_attention_weight_max": 14.579421997070312,
      "activations/layer0_attention_weight_min": -13.620792388916016,
      "activations/layer10_attention_weight_max": 37.83528137207031,
      "activations/layer10_attention_weight_min": -35.66536331176758,
      "activations/layer11_attention_weight_max": 31.131336212158203,
      "activations/layer11_attention_weight_min": -31.45925521850586,
      "activations/layer12_attention_weight_max": 25.00101089477539,
      "activations/layer12_attention_weight_min": -24.30333137512207,
      "activations/layer13_attention_weight_max": 45.52848815917969,
      "activations/layer13_attention_weight_min": -31.119853973388672,
      "activations/layer14_attention_weight_max": 38.125450134277344,
      "activations/layer14_attention_weight_min": -31.259925842285156,
      "activations/layer15_attention_weight_max": 35.94216537475586,
      "activations/layer15_attention_weight_min": -29.41276741027832,
      "activations/layer16_attention_weight_max": 34.54354476928711,
      "activations/layer16_attention_weight_min": -26.75518798828125,
      "activations/layer17_attention_weight_max": 55.737213134765625,
      "activations/layer17_attention_weight_min": -46.97089385986328,
      "activations/layer18_attention_weight_max": 48.06071090698242,
      "activations/layer18_attention_weight_min": -36.74143600463867,
      "activations/layer19_attention_weight_max": 24.314786911010742,
      "activations/layer19_attention_weight_min": -21.399864196777344,
      "activations/layer1_attention_weight_max": 17.479122161865234,
      "activations/layer1_attention_weight_min": -15.617410659790039,
      "activations/layer20_attention_weight_max": 21.23539924621582,
      "activations/layer20_attention_weight_min": -20.149442672729492,
      "activations/layer21_attention_weight_max": 39.16302490234375,
      "activations/layer21_attention_weight_min": -25.24767303466797,
      "activations/layer22_attention_weight_max": 32.57312774658203,
      "activations/layer22_attention_weight_min": -26.026397705078125,
      "activations/layer23_attention_weight_max": 35.651493072509766,
      "activations/layer23_attention_weight_min": -23.783475875854492,
      "activations/layer2_attention_weight_max": 30.64935874938965,
      "activations/layer2_attention_weight_min": -26.896453857421875,
      "activations/layer3_attention_weight_max": 80.66710662841797,
      "activations/layer3_attention_weight_min": -80.54183959960938,
      "activations/layer4_attention_weight_max": 101.33877563476562,
      "activations/layer4_attention_weight_min": -92.6698226928711,
      "activations/layer5_attention_weight_max": 85.12808227539062,
      "activations/layer5_attention_weight_min": -79.84835815429688,
      "activations/layer6_attention_weight_max": 52.74515914916992,
      "activations/layer6_attention_weight_min": -53.032264709472656,
      "activations/layer7_attention_weight_max": 76.8556137084961,
      "activations/layer7_attention_weight_min": -72.30934143066406,
      "activations/layer8_attention_weight_max": 45.122520446777344,
      "activations/layer8_attention_weight_min": -45.225589752197266,
      "activations/layer9_attention_weight_max": 50.95896530151367,
      "activations/layer9_attention_weight_min": -47.2827262878418,
      "epoch": 7.59,
      "learning_rate": 0.00010211022727272726,
      "loss": 2.8078,
      "step": 130550
    },
    {
      "activations/layer0_attention_weight_max": 15.320686340332031,
      "activations/layer0_attention_weight_min": -13.623254776000977,
      "activations/layer10_attention_weight_max": 39.3365364074707,
      "activations/layer10_attention_weight_min": -34.689598083496094,
      "activations/layer11_attention_weight_max": 30.622467041015625,
      "activations/layer11_attention_weight_min": -31.039724349975586,
      "activations/layer12_attention_weight_max": 18.61363410949707,
      "activations/layer12_attention_weight_min": -25.149633407592773,
      "activations/layer13_attention_weight_max": 40.79802703857422,
      "activations/layer13_attention_weight_min": -33.160430908203125,
      "activations/layer14_attention_weight_max": 36.39798355102539,
      "activations/layer14_attention_weight_min": -30.465694427490234,
      "activations/layer15_attention_weight_max": 35.21853256225586,
      "activations/layer15_attention_weight_min": -29.767114639282227,
      "activations/layer16_attention_weight_max": 33.30605697631836,
      "activations/layer16_attention_weight_min": -29.43829917907715,
      "activations/layer17_attention_weight_max": 55.19651412963867,
      "activations/layer17_attention_weight_min": -44.406105041503906,
      "activations/layer18_attention_weight_max": 49.619178771972656,
      "activations/layer18_attention_weight_min": -36.5225830078125,
      "activations/layer19_attention_weight_max": 24.842077255249023,
      "activations/layer19_attention_weight_min": -22.577144622802734,
      "activations/layer1_attention_weight_max": 16.964092254638672,
      "activations/layer1_attention_weight_min": -14.734539985656738,
      "activations/layer20_attention_weight_max": 25.05088996887207,
      "activations/layer20_attention_weight_min": -21.83951187133789,
      "activations/layer21_attention_weight_max": 42.620094299316406,
      "activations/layer21_attention_weight_min": -26.194570541381836,
      "activations/layer22_attention_weight_max": 33.67699432373047,
      "activations/layer22_attention_weight_min": -24.2696590423584,
      "activations/layer23_attention_weight_max": 34.59345245361328,
      "activations/layer23_attention_weight_min": -23.975547790527344,
      "activations/layer2_attention_weight_max": 30.045413970947266,
      "activations/layer2_attention_weight_min": -27.741676330566406,
      "activations/layer3_attention_weight_max": 80.71431732177734,
      "activations/layer3_attention_weight_min": -82.24665069580078,
      "activations/layer4_attention_weight_max": 99.84622955322266,
      "activations/layer4_attention_weight_min": -91.91480255126953,
      "activations/layer5_attention_weight_max": 89.90715026855469,
      "activations/layer5_attention_weight_min": -80.70649719238281,
      "activations/layer6_attention_weight_max": 56.04037857055664,
      "activations/layer6_attention_weight_min": -51.72151184082031,
      "activations/layer7_attention_weight_max": 80.91138458251953,
      "activations/layer7_attention_weight_min": -73.74121856689453,
      "activations/layer8_attention_weight_max": 51.05678176879883,
      "activations/layer8_attention_weight_min": -46.55050277709961,
      "activations/layer9_attention_weight_max": 50.94321060180664,
      "activations/layer9_attention_weight_min": -49.85017013549805,
      "epoch": 7.59,
      "learning_rate": 0.00010209128787878787,
      "loss": 2.8211,
      "step": 130600
    },
    {
      "activations/layer0_attention_weight_max": 14.454296112060547,
      "activations/layer0_attention_weight_min": -14.064713478088379,
      "activations/layer10_attention_weight_max": 39.41985321044922,
      "activations/layer10_attention_weight_min": -35.53502655029297,
      "activations/layer11_attention_weight_max": 31.96111297607422,
      "activations/layer11_attention_weight_min": -31.26576805114746,
      "activations/layer12_attention_weight_max": 38.55951690673828,
      "activations/layer12_attention_weight_min": -24.401105880737305,
      "activations/layer13_attention_weight_max": 46.6221809387207,
      "activations/layer13_attention_weight_min": -34.30768966674805,
      "activations/layer14_attention_weight_max": 38.62894821166992,
      "activations/layer14_attention_weight_min": -31.69172477722168,
      "activations/layer15_attention_weight_max": 36.77741622924805,
      "activations/layer15_attention_weight_min": -30.159835815429688,
      "activations/layer16_attention_weight_max": 28.645557403564453,
      "activations/layer16_attention_weight_min": -27.190217971801758,
      "activations/layer17_attention_weight_max": 53.00870132446289,
      "activations/layer17_attention_weight_min": -43.39878463745117,
      "activations/layer18_attention_weight_max": 43.77096939086914,
      "activations/layer18_attention_weight_min": -37.11655044555664,
      "activations/layer19_attention_weight_max": 21.715085983276367,
      "activations/layer19_attention_weight_min": -20.3466854095459,
      "activations/layer1_attention_weight_max": 16.113018035888672,
      "activations/layer1_attention_weight_min": -15.202855110168457,
      "activations/layer20_attention_weight_max": 19.95241355895996,
      "activations/layer20_attention_weight_min": -24.605546951293945,
      "activations/layer21_attention_weight_max": 34.29601287841797,
      "activations/layer21_attention_weight_min": -24.332841873168945,
      "activations/layer22_attention_weight_max": 30.75715446472168,
      "activations/layer22_attention_weight_min": -25.36726188659668,
      "activations/layer23_attention_weight_max": 31.073179244995117,
      "activations/layer23_attention_weight_min": -21.917083740234375,
      "activations/layer2_attention_weight_max": 30.683509826660156,
      "activations/layer2_attention_weight_min": -28.383991241455078,
      "activations/layer3_attention_weight_max": 83.7245101928711,
      "activations/layer3_attention_weight_min": -87.91381072998047,
      "activations/layer4_attention_weight_max": 100.57530212402344,
      "activations/layer4_attention_weight_min": -93.9664077758789,
      "activations/layer5_attention_weight_max": 86.64944458007812,
      "activations/layer5_attention_weight_min": -80.38080596923828,
      "activations/layer6_attention_weight_max": 53.91732406616211,
      "activations/layer6_attention_weight_min": -52.345802307128906,
      "activations/layer7_attention_weight_max": 76.16412353515625,
      "activations/layer7_attention_weight_min": -69.79161834716797,
      "activations/layer8_attention_weight_max": 45.12881088256836,
      "activations/layer8_attention_weight_min": -47.78455352783203,
      "activations/layer9_attention_weight_max": 47.65650939941406,
      "activations/layer9_attention_weight_min": -48.09869384765625,
      "epoch": 7.59,
      "learning_rate": 0.00010207234848484848,
      "loss": 2.8177,
      "step": 130650
    },
    {
      "activations/layer0_attention_weight_max": 14.337240219116211,
      "activations/layer0_attention_weight_min": -14.255179405212402,
      "activations/layer10_attention_weight_max": 38.49538803100586,
      "activations/layer10_attention_weight_min": -36.53059768676758,
      "activations/layer11_attention_weight_max": 31.263553619384766,
      "activations/layer11_attention_weight_min": -33.107887268066406,
      "activations/layer12_attention_weight_max": 22.602750778198242,
      "activations/layer12_attention_weight_min": -27.975900650024414,
      "activations/layer13_attention_weight_max": 53.262203216552734,
      "activations/layer13_attention_weight_min": -37.00086212158203,
      "activations/layer14_attention_weight_max": 41.64971923828125,
      "activations/layer14_attention_weight_min": -30.242137908935547,
      "activations/layer15_attention_weight_max": 36.459129333496094,
      "activations/layer15_attention_weight_min": -30.098705291748047,
      "activations/layer16_attention_weight_max": 32.14057922363281,
      "activations/layer16_attention_weight_min": -27.207679748535156,
      "activations/layer17_attention_weight_max": 52.265716552734375,
      "activations/layer17_attention_weight_min": -43.85165023803711,
      "activations/layer18_attention_weight_max": 47.904273986816406,
      "activations/layer18_attention_weight_min": -38.229217529296875,
      "activations/layer19_attention_weight_max": 21.261768341064453,
      "activations/layer19_attention_weight_min": -22.148962020874023,
      "activations/layer1_attention_weight_max": 18.058263778686523,
      "activations/layer1_attention_weight_min": -16.922502517700195,
      "activations/layer20_attention_weight_max": 21.102882385253906,
      "activations/layer20_attention_weight_min": -21.682809829711914,
      "activations/layer21_attention_weight_max": 43.217071533203125,
      "activations/layer21_attention_weight_min": -24.766876220703125,
      "activations/layer22_attention_weight_max": 33.67295837402344,
      "activations/layer22_attention_weight_min": -26.237899780273438,
      "activations/layer23_attention_weight_max": 32.40204620361328,
      "activations/layer23_attention_weight_min": -21.154560089111328,
      "activations/layer2_attention_weight_max": 30.311420440673828,
      "activations/layer2_attention_weight_min": -28.474853515625,
      "activations/layer3_attention_weight_max": 84.38630676269531,
      "activations/layer3_attention_weight_min": -88.99606323242188,
      "activations/layer4_attention_weight_max": 98.71147918701172,
      "activations/layer4_attention_weight_min": -93.47067260742188,
      "activations/layer5_attention_weight_max": 88.55294799804688,
      "activations/layer5_attention_weight_min": -81.95387268066406,
      "activations/layer6_attention_weight_max": 55.15109634399414,
      "activations/layer6_attention_weight_min": -52.478309631347656,
      "activations/layer7_attention_weight_max": 77.8250732421875,
      "activations/layer7_attention_weight_min": -74.62291717529297,
      "activations/layer8_attention_weight_max": 44.98478317260742,
      "activations/layer8_attention_weight_min": -44.663700103759766,
      "activations/layer9_attention_weight_max": 54.903900146484375,
      "activations/layer9_attention_weight_min": -47.651615142822266,
      "epoch": 7.59,
      "learning_rate": 0.00010205340909090908,
      "loss": 2.8192,
      "step": 130700
    },
    {
      "activations/layer0_attention_weight_max": 15.327489852905273,
      "activations/layer0_attention_weight_min": -14.258893966674805,
      "activations/layer10_attention_weight_max": 39.90520095825195,
      "activations/layer10_attention_weight_min": -38.06128692626953,
      "activations/layer11_attention_weight_max": 31.56749725341797,
      "activations/layer11_attention_weight_min": -31.000137329101562,
      "activations/layer12_attention_weight_max": 18.50570297241211,
      "activations/layer12_attention_weight_min": -25.295358657836914,
      "activations/layer13_attention_weight_max": 46.23839569091797,
      "activations/layer13_attention_weight_min": -29.783287048339844,
      "activations/layer14_attention_weight_max": 44.407840728759766,
      "activations/layer14_attention_weight_min": -29.379728317260742,
      "activations/layer15_attention_weight_max": 34.14022445678711,
      "activations/layer15_attention_weight_min": -28.541624069213867,
      "activations/layer16_attention_weight_max": 30.179821014404297,
      "activations/layer16_attention_weight_min": -27.046031951904297,
      "activations/layer17_attention_weight_max": 52.25690460205078,
      "activations/layer17_attention_weight_min": -41.60287094116211,
      "activations/layer18_attention_weight_max": 45.34954071044922,
      "activations/layer18_attention_weight_min": -35.70154571533203,
      "activations/layer19_attention_weight_max": 21.566143035888672,
      "activations/layer19_attention_weight_min": -20.222036361694336,
      "activations/layer1_attention_weight_max": 17.32733154296875,
      "activations/layer1_attention_weight_min": -15.022660255432129,
      "activations/layer20_attention_weight_max": 21.771240234375,
      "activations/layer20_attention_weight_min": -25.449710845947266,
      "activations/layer21_attention_weight_max": 42.288414001464844,
      "activations/layer21_attention_weight_min": -25.869043350219727,
      "activations/layer22_attention_weight_max": 30.072988510131836,
      "activations/layer22_attention_weight_min": -26.70865821838379,
      "activations/layer23_attention_weight_max": 30.56798553466797,
      "activations/layer23_attention_weight_min": -20.85144805908203,
      "activations/layer2_attention_weight_max": 30.428936004638672,
      "activations/layer2_attention_weight_min": -28.669538497924805,
      "activations/layer3_attention_weight_max": 79.75741577148438,
      "activations/layer3_attention_weight_min": -85.3946762084961,
      "activations/layer4_attention_weight_max": 100.64215087890625,
      "activations/layer4_attention_weight_min": -98.46102905273438,
      "activations/layer5_attention_weight_max": 88.03793334960938,
      "activations/layer5_attention_weight_min": -81.27809143066406,
      "activations/layer6_attention_weight_max": 55.76274871826172,
      "activations/layer6_attention_weight_min": -53.4954948425293,
      "activations/layer7_attention_weight_max": 78.51041412353516,
      "activations/layer7_attention_weight_min": -74.867431640625,
      "activations/layer8_attention_weight_max": 45.38350296020508,
      "activations/layer8_attention_weight_min": -44.811946868896484,
      "activations/layer9_attention_weight_max": 56.240020751953125,
      "activations/layer9_attention_weight_min": -49.65378189086914,
      "epoch": 7.6,
      "learning_rate": 0.00010203446969696969,
      "loss": 2.8317,
      "step": 130750
    },
    {
      "activations/layer0_attention_weight_max": 14.884336471557617,
      "activations/layer0_attention_weight_min": -14.308506965637207,
      "activations/layer10_attention_weight_max": 41.02810287475586,
      "activations/layer10_attention_weight_min": -39.24604034423828,
      "activations/layer11_attention_weight_max": 32.66587829589844,
      "activations/layer11_attention_weight_min": -34.344364166259766,
      "activations/layer12_attention_weight_max": 18.65924072265625,
      "activations/layer12_attention_weight_min": -26.50429916381836,
      "activations/layer13_attention_weight_max": 38.28022766113281,
      "activations/layer13_attention_weight_min": -33.26691436767578,
      "activations/layer14_attention_weight_max": 34.974361419677734,
      "activations/layer14_attention_weight_min": -30.570575714111328,
      "activations/layer15_attention_weight_max": 33.049598693847656,
      "activations/layer15_attention_weight_min": -30.00861930847168,
      "activations/layer16_attention_weight_max": 32.93599319458008,
      "activations/layer16_attention_weight_min": -26.82495880126953,
      "activations/layer17_attention_weight_max": 53.12705612182617,
      "activations/layer17_attention_weight_min": -45.91023635864258,
      "activations/layer18_attention_weight_max": 47.56053924560547,
      "activations/layer18_attention_weight_min": -37.838871002197266,
      "activations/layer19_attention_weight_max": 23.219463348388672,
      "activations/layer19_attention_weight_min": -21.944028854370117,
      "activations/layer1_attention_weight_max": 17.29898452758789,
      "activations/layer1_attention_weight_min": -15.298542022705078,
      "activations/layer20_attention_weight_max": 21.259347915649414,
      "activations/layer20_attention_weight_min": -23.931747436523438,
      "activations/layer21_attention_weight_max": 37.813446044921875,
      "activations/layer21_attention_weight_min": -23.530691146850586,
      "activations/layer22_attention_weight_max": 30.220226287841797,
      "activations/layer22_attention_weight_min": -24.893035888671875,
      "activations/layer23_attention_weight_max": 31.93972396850586,
      "activations/layer23_attention_weight_min": -20.929676055908203,
      "activations/layer2_attention_weight_max": 31.540081024169922,
      "activations/layer2_attention_weight_min": -28.08099365234375,
      "activations/layer3_attention_weight_max": 82.38261413574219,
      "activations/layer3_attention_weight_min": -88.78312683105469,
      "activations/layer4_attention_weight_max": 99.15345764160156,
      "activations/layer4_attention_weight_min": -100.78914642333984,
      "activations/layer5_attention_weight_max": 88.73283386230469,
      "activations/layer5_attention_weight_min": -82.0482177734375,
      "activations/layer6_attention_weight_max": 55.99129104614258,
      "activations/layer6_attention_weight_min": -54.62605285644531,
      "activations/layer7_attention_weight_max": 77.95155334472656,
      "activations/layer7_attention_weight_min": -72.94769287109375,
      "activations/layer8_attention_weight_max": 46.887569427490234,
      "activations/layer8_attention_weight_min": -47.97814178466797,
      "activations/layer9_attention_weight_max": 52.03544235229492,
      "activations/layer9_attention_weight_min": -48.911861419677734,
      "epoch": 7.6,
      "learning_rate": 0.00010201553030303031,
      "loss": 2.8151,
      "step": 130800
    },
    {
      "activations/layer0_attention_weight_max": 14.657804489135742,
      "activations/layer0_attention_weight_min": -14.331143379211426,
      "activations/layer10_attention_weight_max": 48.243804931640625,
      "activations/layer10_attention_weight_min": -45.1253776550293,
      "activations/layer11_attention_weight_max": 43.64696502685547,
      "activations/layer11_attention_weight_min": -36.34422302246094,
      "activations/layer12_attention_weight_max": 19.68195915222168,
      "activations/layer12_attention_weight_min": -27.175195693969727,
      "activations/layer13_attention_weight_max": 58.19523620605469,
      "activations/layer13_attention_weight_min": -34.65361022949219,
      "activations/layer14_attention_weight_max": 72.62277221679688,
      "activations/layer14_attention_weight_min": -37.38175964355469,
      "activations/layer15_attention_weight_max": 55.659603118896484,
      "activations/layer15_attention_weight_min": -32.11431884765625,
      "activations/layer16_attention_weight_max": 36.068084716796875,
      "activations/layer16_attention_weight_min": -28.674440383911133,
      "activations/layer17_attention_weight_max": 61.60205078125,
      "activations/layer17_attention_weight_min": -51.043148040771484,
      "activations/layer18_attention_weight_max": 54.24675750732422,
      "activations/layer18_attention_weight_min": -46.35944747924805,
      "activations/layer19_attention_weight_max": 26.505300521850586,
      "activations/layer19_attention_weight_min": -22.987165451049805,
      "activations/layer1_attention_weight_max": 16.82461166381836,
      "activations/layer1_attention_weight_min": -13.26507568359375,
      "activations/layer20_attention_weight_max": 28.98101043701172,
      "activations/layer20_attention_weight_min": -21.050518035888672,
      "activations/layer21_attention_weight_max": 51.14952087402344,
      "activations/layer21_attention_weight_min": -24.625865936279297,
      "activations/layer22_attention_weight_max": 38.21348190307617,
      "activations/layer22_attention_weight_min": -27.73874855041504,
      "activations/layer23_attention_weight_max": 39.8117790222168,
      "activations/layer23_attention_weight_min": -21.827260971069336,
      "activations/layer2_attention_weight_max": 31.88119888305664,
      "activations/layer2_attention_weight_min": -28.205726623535156,
      "activations/layer3_attention_weight_max": 96.79261016845703,
      "activations/layer3_attention_weight_min": -96.37798309326172,
      "activations/layer4_attention_weight_max": 112.156982421875,
      "activations/layer4_attention_weight_min": -101.46465301513672,
      "activations/layer5_attention_weight_max": 91.37528991699219,
      "activations/layer5_attention_weight_min": -85.26592254638672,
      "activations/layer6_attention_weight_max": 56.21464920043945,
      "activations/layer6_attention_weight_min": -56.04042434692383,
      "activations/layer7_attention_weight_max": 91.07090759277344,
      "activations/layer7_attention_weight_min": -78.72249603271484,
      "activations/layer8_attention_weight_max": 54.243682861328125,
      "activations/layer8_attention_weight_min": -52.153221130371094,
      "activations/layer9_attention_weight_max": 72.67215728759766,
      "activations/layer9_attention_weight_min": -58.05506134033203,
      "epoch": 7.6,
      "learning_rate": 0.0001019965909090909,
      "loss": 2.8117,
      "step": 130850
    },
    {
      "activations/layer0_attention_weight_max": 15.03821849822998,
      "activations/layer0_attention_weight_min": -14.424962997436523,
      "activations/layer10_attention_weight_max": 40.77455520629883,
      "activations/layer10_attention_weight_min": -36.44764709472656,
      "activations/layer11_attention_weight_max": 35.315608978271484,
      "activations/layer11_attention_weight_min": -30.464046478271484,
      "activations/layer12_attention_weight_max": 21.96880340576172,
      "activations/layer12_attention_weight_min": -24.806615829467773,
      "activations/layer13_attention_weight_max": 38.780189514160156,
      "activations/layer13_attention_weight_min": -32.12235641479492,
      "activations/layer14_attention_weight_max": 41.234046936035156,
      "activations/layer14_attention_weight_min": -29.42702865600586,
      "activations/layer15_attention_weight_max": 31.586790084838867,
      "activations/layer15_attention_weight_min": -29.203657150268555,
      "activations/layer16_attention_weight_max": 29.52738380432129,
      "activations/layer16_attention_weight_min": -26.703012466430664,
      "activations/layer17_attention_weight_max": 51.342987060546875,
      "activations/layer17_attention_weight_min": -44.1643180847168,
      "activations/layer18_attention_weight_max": 42.59499740600586,
      "activations/layer18_attention_weight_min": -39.05687713623047,
      "activations/layer19_attention_weight_max": 20.819669723510742,
      "activations/layer19_attention_weight_min": -21.61902618408203,
      "activations/layer1_attention_weight_max": 17.37407875061035,
      "activations/layer1_attention_weight_min": -15.45203971862793,
      "activations/layer20_attention_weight_max": 18.971988677978516,
      "activations/layer20_attention_weight_min": -19.792827606201172,
      "activations/layer21_attention_weight_max": 31.680782318115234,
      "activations/layer21_attention_weight_min": -22.79681396484375,
      "activations/layer22_attention_weight_max": 31.991230010986328,
      "activations/layer22_attention_weight_min": -27.886547088623047,
      "activations/layer23_attention_weight_max": 33.00141906738281,
      "activations/layer23_attention_weight_min": -21.280689239501953,
      "activations/layer2_attention_weight_max": 33.602420806884766,
      "activations/layer2_attention_weight_min": -30.65199089050293,
      "activations/layer3_attention_weight_max": 83.19902038574219,
      "activations/layer3_attention_weight_min": -85.90599060058594,
      "activations/layer4_attention_weight_max": 98.3432388305664,
      "activations/layer4_attention_weight_min": -92.9502182006836,
      "activations/layer5_attention_weight_max": 84.62109375,
      "activations/layer5_attention_weight_min": -79.28044128417969,
      "activations/layer6_attention_weight_max": 54.04863357543945,
      "activations/layer6_attention_weight_min": -52.27886199951172,
      "activations/layer7_attention_weight_max": 81.9021224975586,
      "activations/layer7_attention_weight_min": -76.86017608642578,
      "activations/layer8_attention_weight_max": 48.283111572265625,
      "activations/layer8_attention_weight_min": -42.49363708496094,
      "activations/layer9_attention_weight_max": 62.57686996459961,
      "activations/layer9_attention_weight_min": -52.2636833190918,
      "epoch": 7.61,
      "learning_rate": 0.00010197765151515151,
      "loss": 2.8151,
      "step": 130900
    },
    {
      "activations/layer0_attention_weight_max": 15.317902565002441,
      "activations/layer0_attention_weight_min": -14.463732719421387,
      "activations/layer10_attention_weight_max": 39.99549102783203,
      "activations/layer10_attention_weight_min": -36.20927429199219,
      "activations/layer11_attention_weight_max": 31.020751953125,
      "activations/layer11_attention_weight_min": -32.38402557373047,
      "activations/layer12_attention_weight_max": 28.585189819335938,
      "activations/layer12_attention_weight_min": -24.264488220214844,
      "activations/layer13_attention_weight_max": 41.66393280029297,
      "activations/layer13_attention_weight_min": -31.86386489868164,
      "activations/layer14_attention_weight_max": 40.46741485595703,
      "activations/layer14_attention_weight_min": -30.307348251342773,
      "activations/layer15_attention_weight_max": 38.472286224365234,
      "activations/layer15_attention_weight_min": -29.238689422607422,
      "activations/layer16_attention_weight_max": 32.57789993286133,
      "activations/layer16_attention_weight_min": -28.195161819458008,
      "activations/layer17_attention_weight_max": 60.60007858276367,
      "activations/layer17_attention_weight_min": -44.71554183959961,
      "activations/layer18_attention_weight_max": 57.6019287109375,
      "activations/layer18_attention_weight_min": -37.83282470703125,
      "activations/layer19_attention_weight_max": 22.21891212463379,
      "activations/layer19_attention_weight_min": -20.491369247436523,
      "activations/layer1_attention_weight_max": 17.221853256225586,
      "activations/layer1_attention_weight_min": -14.174750328063965,
      "activations/layer20_attention_weight_max": 22.382829666137695,
      "activations/layer20_attention_weight_min": -20.312429428100586,
      "activations/layer21_attention_weight_max": 38.1729850769043,
      "activations/layer21_attention_weight_min": -23.09140968322754,
      "activations/layer22_attention_weight_max": 31.403533935546875,
      "activations/layer22_attention_weight_min": -27.160993576049805,
      "activations/layer23_attention_weight_max": 37.3375244140625,
      "activations/layer23_attention_weight_min": -22.4276123046875,
      "activations/layer2_attention_weight_max": 31.984722137451172,
      "activations/layer2_attention_weight_min": -31.776077270507812,
      "activations/layer3_attention_weight_max": 88.99995422363281,
      "activations/layer3_attention_weight_min": -92.03521728515625,
      "activations/layer4_attention_weight_max": 103.17860412597656,
      "activations/layer4_attention_weight_min": -97.89749145507812,
      "activations/layer5_attention_weight_max": 88.93069458007812,
      "activations/layer5_attention_weight_min": -84.73248291015625,
      "activations/layer6_attention_weight_max": 55.56354522705078,
      "activations/layer6_attention_weight_min": -54.120784759521484,
      "activations/layer7_attention_weight_max": 74.20784759521484,
      "activations/layer7_attention_weight_min": -76.31659698486328,
      "activations/layer8_attention_weight_max": 48.62095260620117,
      "activations/layer8_attention_weight_min": -50.26776885986328,
      "activations/layer9_attention_weight_max": 51.805660247802734,
      "activations/layer9_attention_weight_min": -49.029014587402344,
      "epoch": 7.61,
      "learning_rate": 0.0001019587121212121,
      "loss": 2.8249,
      "step": 130950
    },
    {
      "activations/layer0_attention_weight_max": 14.783919334411621,
      "activations/layer0_attention_weight_min": -14.648425102233887,
      "activations/layer10_attention_weight_max": 37.323997497558594,
      "activations/layer10_attention_weight_min": -36.15443420410156,
      "activations/layer11_attention_weight_max": 32.45429992675781,
      "activations/layer11_attention_weight_min": -30.699947357177734,
      "activations/layer12_attention_weight_max": 21.3797607421875,
      "activations/layer12_attention_weight_min": -24.58226776123047,
      "activations/layer13_attention_weight_max": 46.19705581665039,
      "activations/layer13_attention_weight_min": -32.42103576660156,
      "activations/layer14_attention_weight_max": 42.91464614868164,
      "activations/layer14_attention_weight_min": -30.995271682739258,
      "activations/layer15_attention_weight_max": 34.89817810058594,
      "activations/layer15_attention_weight_min": -29.878223419189453,
      "activations/layer16_attention_weight_max": 37.65842056274414,
      "activations/layer16_attention_weight_min": -28.276756286621094,
      "activations/layer17_attention_weight_max": 60.51551818847656,
      "activations/layer17_attention_weight_min": -45.78797912597656,
      "activations/layer18_attention_weight_max": 51.32864761352539,
      "activations/layer18_attention_weight_min": -37.09438705444336,
      "activations/layer19_attention_weight_max": 22.50823211669922,
      "activations/layer19_attention_weight_min": -21.286344528198242,
      "activations/layer1_attention_weight_max": 17.33382797241211,
      "activations/layer1_attention_weight_min": -14.497602462768555,
      "activations/layer20_attention_weight_max": 25.924291610717773,
      "activations/layer20_attention_weight_min": -20.308595657348633,
      "activations/layer21_attention_weight_max": 42.680267333984375,
      "activations/layer21_attention_weight_min": -24.862247467041016,
      "activations/layer22_attention_weight_max": 33.752220153808594,
      "activations/layer22_attention_weight_min": -27.150007247924805,
      "activations/layer23_attention_weight_max": 39.77251052856445,
      "activations/layer23_attention_weight_min": -25.94332504272461,
      "activations/layer2_attention_weight_max": 31.164518356323242,
      "activations/layer2_attention_weight_min": -28.41206932067871,
      "activations/layer3_attention_weight_max": 86.78316497802734,
      "activations/layer3_attention_weight_min": -92.20020294189453,
      "activations/layer4_attention_weight_max": 100.84514617919922,
      "activations/layer4_attention_weight_min": -93.22135162353516,
      "activations/layer5_attention_weight_max": 85.2841796875,
      "activations/layer5_attention_weight_min": -76.93346405029297,
      "activations/layer6_attention_weight_max": 56.85606384277344,
      "activations/layer6_attention_weight_min": -52.43910217285156,
      "activations/layer7_attention_weight_max": 76.0672836303711,
      "activations/layer7_attention_weight_min": -72.28150177001953,
      "activations/layer8_attention_weight_max": 42.6688117980957,
      "activations/layer8_attention_weight_min": -44.11734390258789,
      "activations/layer9_attention_weight_max": 50.51049041748047,
      "activations/layer9_attention_weight_min": -48.962650299072266,
      "epoch": 7.61,
      "learning_rate": 0.00010193977272727271,
      "loss": 2.8211,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5593,
      "eval_samples_per_second": 501.674,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5593,
      "eval_openwebtext_samples_per_second": 501.674,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 2.0252,
      "eval_wikitext_samples_per_second": 225.163,
      "step": 131000
    },
    {
      "epoch": 7.61,
      "eval_lambada_loss": 2.71875,
      "eval_lambada_ppl": 15.16135869683449,
      "eval_lambada_runtime": 9.618,
      "eval_lambada_samples_per_second": 506.239,
      "step": 131000
    },
    {
      "activations/layer0_attention_weight_max": 15.126457214355469,
      "activations/layer0_attention_weight_min": -14.646333694458008,
      "activations/layer10_attention_weight_max": 38.1695556640625,
      "activations/layer10_attention_weight_min": -35.38951873779297,
      "activations/layer11_attention_weight_max": 31.54812240600586,
      "activations/layer11_attention_weight_min": -30.101978302001953,
      "activations/layer12_attention_weight_max": 21.0161190032959,
      "activations/layer12_attention_weight_min": -26.039106369018555,
      "activations/layer13_attention_weight_max": 43.380855560302734,
      "activations/layer13_attention_weight_min": -29.87937355041504,
      "activations/layer14_attention_weight_max": 37.97233581542969,
      "activations/layer14_attention_weight_min": -29.639192581176758,
      "activations/layer15_attention_weight_max": 36.04066467285156,
      "activations/layer15_attention_weight_min": -27.68170928955078,
      "activations/layer16_attention_weight_max": 32.23680114746094,
      "activations/layer16_attention_weight_min": -26.430538177490234,
      "activations/layer17_attention_weight_max": 53.672645568847656,
      "activations/layer17_attention_weight_min": -41.66358947753906,
      "activations/layer18_attention_weight_max": 46.17831802368164,
      "activations/layer18_attention_weight_min": -34.260284423828125,
      "activations/layer19_attention_weight_max": 23.62614631652832,
      "activations/layer19_attention_weight_min": -20.535993576049805,
      "activations/layer1_attention_weight_max": 17.362035751342773,
      "activations/layer1_attention_weight_min": -13.672125816345215,
      "activations/layer20_attention_weight_max": 23.837242126464844,
      "activations/layer20_attention_weight_min": -21.725440979003906,
      "activations/layer21_attention_weight_max": 33.74420166015625,
      "activations/layer21_attention_weight_min": -22.073875427246094,
      "activations/layer22_attention_weight_max": 31.84751319885254,
      "activations/layer22_attention_weight_min": -26.035472869873047,
      "activations/layer23_attention_weight_max": 34.6888427734375,
      "activations/layer23_attention_weight_min": -23.37665557861328,
      "activations/layer2_attention_weight_max": 30.688627243041992,
      "activations/layer2_attention_weight_min": -27.719764709472656,
      "activations/layer3_attention_weight_max": 83.44058990478516,
      "activations/layer3_attention_weight_min": -90.19732666015625,
      "activations/layer4_attention_weight_max": 101.41133880615234,
      "activations/layer4_attention_weight_min": -99.53255462646484,
      "activations/layer5_attention_weight_max": 88.54716491699219,
      "activations/layer5_attention_weight_min": -81.71113586425781,
      "activations/layer6_attention_weight_max": 54.02913284301758,
      "activations/layer6_attention_weight_min": -53.42741775512695,
      "activations/layer7_attention_weight_max": 75.8841323852539,
      "activations/layer7_attention_weight_min": -69.62757110595703,
      "activations/layer8_attention_weight_max": 46.442588806152344,
      "activations/layer8_attention_weight_min": -45.24939727783203,
      "activations/layer9_attention_weight_max": 49.43347930908203,
      "activations/layer9_attention_weight_min": -49.4233512878418,
      "epoch": 7.61,
      "learning_rate": 0.00010192083333333333,
      "loss": 2.8105,
      "step": 131050
    },
    {
      "activations/layer0_attention_weight_max": 14.19638442993164,
      "activations/layer0_attention_weight_min": -14.363991737365723,
      "activations/layer10_attention_weight_max": 41.3974609375,
      "activations/layer10_attention_weight_min": -37.199806213378906,
      "activations/layer11_attention_weight_max": 34.07851791381836,
      "activations/layer11_attention_weight_min": -33.586158752441406,
      "activations/layer12_attention_weight_max": 20.519384384155273,
      "activations/layer12_attention_weight_min": -26.548141479492188,
      "activations/layer13_attention_weight_max": 39.19083786010742,
      "activations/layer13_attention_weight_min": -32.63857650756836,
      "activations/layer14_attention_weight_max": 37.968910217285156,
      "activations/layer14_attention_weight_min": -30.06928825378418,
      "activations/layer15_attention_weight_max": 33.757843017578125,
      "activations/layer15_attention_weight_min": -29.69791030883789,
      "activations/layer16_attention_weight_max": 34.32876205444336,
      "activations/layer16_attention_weight_min": -28.751649856567383,
      "activations/layer17_attention_weight_max": 52.686283111572266,
      "activations/layer17_attention_weight_min": -41.45703887939453,
      "activations/layer18_attention_weight_max": 45.25700378417969,
      "activations/layer18_attention_weight_min": -34.203590393066406,
      "activations/layer19_attention_weight_max": 20.932222366333008,
      "activations/layer19_attention_weight_min": -23.199459075927734,
      "activations/layer1_attention_weight_max": 18.19191551208496,
      "activations/layer1_attention_weight_min": -15.087990760803223,
      "activations/layer20_attention_weight_max": 19.783056259155273,
      "activations/layer20_attention_weight_min": -20.807146072387695,
      "activations/layer21_attention_weight_max": 39.393829345703125,
      "activations/layer21_attention_weight_min": -25.068143844604492,
      "activations/layer22_attention_weight_max": 29.835493087768555,
      "activations/layer22_attention_weight_min": -28.008338928222656,
      "activations/layer23_attention_weight_max": 36.470314025878906,
      "activations/layer23_attention_weight_min": -23.97244644165039,
      "activations/layer2_attention_weight_max": 31.827980041503906,
      "activations/layer2_attention_weight_min": -29.937496185302734,
      "activations/layer3_attention_weight_max": 88.0593490600586,
      "activations/layer3_attention_weight_min": -92.7569808959961,
      "activations/layer4_attention_weight_max": 108.01068115234375,
      "activations/layer4_attention_weight_min": -102.02669525146484,
      "activations/layer5_attention_weight_max": 93.0941162109375,
      "activations/layer5_attention_weight_min": -87.24803924560547,
      "activations/layer6_attention_weight_max": 57.39811325073242,
      "activations/layer6_attention_weight_min": -54.16950988769531,
      "activations/layer7_attention_weight_max": 76.58209991455078,
      "activations/layer7_attention_weight_min": -73.94221496582031,
      "activations/layer8_attention_weight_max": 48.846744537353516,
      "activations/layer8_attention_weight_min": -49.45649337768555,
      "activations/layer9_attention_weight_max": 52.10062789916992,
      "activations/layer9_attention_weight_min": -50.611534118652344,
      "epoch": 7.62,
      "learning_rate": 0.00010190189393939393,
      "loss": 2.8209,
      "step": 131100
    },
    {
      "activations/layer0_attention_weight_max": 14.757458686828613,
      "activations/layer0_attention_weight_min": -14.495482444763184,
      "activations/layer10_attention_weight_max": 37.43034362792969,
      "activations/layer10_attention_weight_min": -37.182594299316406,
      "activations/layer11_attention_weight_max": 31.92058563232422,
      "activations/layer11_attention_weight_min": -30.38667106628418,
      "activations/layer12_attention_weight_max": 19.236459732055664,
      "activations/layer12_attention_weight_min": -26.137231826782227,
      "activations/layer13_attention_weight_max": 44.27471160888672,
      "activations/layer13_attention_weight_min": -32.35836410522461,
      "activations/layer14_attention_weight_max": 43.83993148803711,
      "activations/layer14_attention_weight_min": -29.71680450439453,
      "activations/layer15_attention_weight_max": 35.964500427246094,
      "activations/layer15_attention_weight_min": -29.537790298461914,
      "activations/layer16_attention_weight_max": 34.04107666015625,
      "activations/layer16_attention_weight_min": -30.82346534729004,
      "activations/layer17_attention_weight_max": 56.86753845214844,
      "activations/layer17_attention_weight_min": -42.771324157714844,
      "activations/layer18_attention_weight_max": 48.84446716308594,
      "activations/layer18_attention_weight_min": -37.73016357421875,
      "activations/layer19_attention_weight_max": 23.478742599487305,
      "activations/layer19_attention_weight_min": -20.91888999938965,
      "activations/layer1_attention_weight_max": 17.233673095703125,
      "activations/layer1_attention_weight_min": -16.787199020385742,
      "activations/layer20_attention_weight_max": 23.469013214111328,
      "activations/layer20_attention_weight_min": -19.980440139770508,
      "activations/layer21_attention_weight_max": 44.84041213989258,
      "activations/layer21_attention_weight_min": -25.46269989013672,
      "activations/layer22_attention_weight_max": 31.510175704956055,
      "activations/layer22_attention_weight_min": -22.893043518066406,
      "activations/layer23_attention_weight_max": 36.90167999267578,
      "activations/layer23_attention_weight_min": -22.24080467224121,
      "activations/layer2_attention_weight_max": 31.373044967651367,
      "activations/layer2_attention_weight_min": -29.21564292907715,
      "activations/layer3_attention_weight_max": 85.4493637084961,
      "activations/layer3_attention_weight_min": -87.74205780029297,
      "activations/layer4_attention_weight_max": 103.15914154052734,
      "activations/layer4_attention_weight_min": -96.74617004394531,
      "activations/layer5_attention_weight_max": 89.59304809570312,
      "activations/layer5_attention_weight_min": -79.67633056640625,
      "activations/layer6_attention_weight_max": 53.847835540771484,
      "activations/layer6_attention_weight_min": -50.32589340209961,
      "activations/layer7_attention_weight_max": 71.78959655761719,
      "activations/layer7_attention_weight_min": -72.0174331665039,
      "activations/layer8_attention_weight_max": 47.278465270996094,
      "activations/layer8_attention_weight_min": -46.453346252441406,
      "activations/layer9_attention_weight_max": 52.36652755737305,
      "activations/layer9_attention_weight_min": -49.7780876159668,
      "epoch": 7.62,
      "learning_rate": 0.00010188295454545453,
      "loss": 2.8106,
      "step": 131150
    },
    {
      "activations/layer0_attention_weight_max": 14.99864673614502,
      "activations/layer0_attention_weight_min": -14.67432689666748,
      "activations/layer10_attention_weight_max": 35.22626495361328,
      "activations/layer10_attention_weight_min": -34.5303840637207,
      "activations/layer11_attention_weight_max": 31.085430145263672,
      "activations/layer11_attention_weight_min": -30.781391143798828,
      "activations/layer12_attention_weight_max": 21.32906150817871,
      "activations/layer12_attention_weight_min": -25.667631149291992,
      "activations/layer13_attention_weight_max": 36.960693359375,
      "activations/layer13_attention_weight_min": -30.117895126342773,
      "activations/layer14_attention_weight_max": 40.34005355834961,
      "activations/layer14_attention_weight_min": -31.171104431152344,
      "activations/layer15_attention_weight_max": 34.96359634399414,
      "activations/layer15_attention_weight_min": -29.85657501220703,
      "activations/layer16_attention_weight_max": 33.76203536987305,
      "activations/layer16_attention_weight_min": -27.372116088867188,
      "activations/layer17_attention_weight_max": 55.150901794433594,
      "activations/layer17_attention_weight_min": -45.58510208129883,
      "activations/layer18_attention_weight_max": 48.272796630859375,
      "activations/layer18_attention_weight_min": -40.26500701904297,
      "activations/layer19_attention_weight_max": 22.853458404541016,
      "activations/layer19_attention_weight_min": -21.359195709228516,
      "activations/layer1_attention_weight_max": 16.2913761138916,
      "activations/layer1_attention_weight_min": -13.861851692199707,
      "activations/layer20_attention_weight_max": 23.614036560058594,
      "activations/layer20_attention_weight_min": -24.662996292114258,
      "activations/layer21_attention_weight_max": 37.79018020629883,
      "activations/layer21_attention_weight_min": -24.67214012145996,
      "activations/layer22_attention_weight_max": 42.15430450439453,
      "activations/layer22_attention_weight_min": -28.240571975708008,
      "activations/layer23_attention_weight_max": 38.07066345214844,
      "activations/layer23_attention_weight_min": -23.24782371520996,
      "activations/layer2_attention_weight_max": 31.418981552124023,
      "activations/layer2_attention_weight_min": -27.65204429626465,
      "activations/layer3_attention_weight_max": 90.66960144042969,
      "activations/layer3_attention_weight_min": -87.39137268066406,
      "activations/layer4_attention_weight_max": 101.76900482177734,
      "activations/layer4_attention_weight_min": -95.67659759521484,
      "activations/layer5_attention_weight_max": 87.55760192871094,
      "activations/layer5_attention_weight_min": -81.81616973876953,
      "activations/layer6_attention_weight_max": 55.61730194091797,
      "activations/layer6_attention_weight_min": -53.545860290527344,
      "activations/layer7_attention_weight_max": 81.7171859741211,
      "activations/layer7_attention_weight_min": -71.49073028564453,
      "activations/layer8_attention_weight_max": 43.48683547973633,
      "activations/layer8_attention_weight_min": -45.72771072387695,
      "activations/layer9_attention_weight_max": 58.53794479370117,
      "activations/layer9_attention_weight_min": -48.10164260864258,
      "epoch": 7.62,
      "learning_rate": 0.00010186401515151515,
      "loss": 2.81,
      "step": 131200
    },
    {
      "activations/layer0_attention_weight_max": 14.127254486083984,
      "activations/layer0_attention_weight_min": -13.783924102783203,
      "activations/layer10_attention_weight_max": 38.167808532714844,
      "activations/layer10_attention_weight_min": -37.09246826171875,
      "activations/layer11_attention_weight_max": 33.03199768066406,
      "activations/layer11_attention_weight_min": -30.345182418823242,
      "activations/layer12_attention_weight_max": 21.386838912963867,
      "activations/layer12_attention_weight_min": -23.150287628173828,
      "activations/layer13_attention_weight_max": 45.0673942565918,
      "activations/layer13_attention_weight_min": -33.76264572143555,
      "activations/layer14_attention_weight_max": 41.58964920043945,
      "activations/layer14_attention_weight_min": -29.583599090576172,
      "activations/layer15_attention_weight_max": 37.02820587158203,
      "activations/layer15_attention_weight_min": -28.4224796295166,
      "activations/layer16_attention_weight_max": 32.85702896118164,
      "activations/layer16_attention_weight_min": -26.61186408996582,
      "activations/layer17_attention_weight_max": 57.93765640258789,
      "activations/layer17_attention_weight_min": -39.9858283996582,
      "activations/layer18_attention_weight_max": 53.38656997680664,
      "activations/layer18_attention_weight_min": -33.587974548339844,
      "activations/layer19_attention_weight_max": 22.260112762451172,
      "activations/layer19_attention_weight_min": -19.677175521850586,
      "activations/layer1_attention_weight_max": 16.82083511352539,
      "activations/layer1_attention_weight_min": -13.995485305786133,
      "activations/layer20_attention_weight_max": 23.892118453979492,
      "activations/layer20_attention_weight_min": -20.184682846069336,
      "activations/layer21_attention_weight_max": 38.77070236206055,
      "activations/layer21_attention_weight_min": -22.58307456970215,
      "activations/layer22_attention_weight_max": 33.479637145996094,
      "activations/layer22_attention_weight_min": -26.036376953125,
      "activations/layer23_attention_weight_max": 36.57891082763672,
      "activations/layer23_attention_weight_min": -23.270544052124023,
      "activations/layer2_attention_weight_max": 29.25824546813965,
      "activations/layer2_attention_weight_min": -28.874427795410156,
      "activations/layer3_attention_weight_max": 81.7900161743164,
      "activations/layer3_attention_weight_min": -82.64706420898438,
      "activations/layer4_attention_weight_max": 99.3410415649414,
      "activations/layer4_attention_weight_min": -94.42572021484375,
      "activations/layer5_attention_weight_max": 86.76239013671875,
      "activations/layer5_attention_weight_min": -83.5462646484375,
      "activations/layer6_attention_weight_max": 55.372283935546875,
      "activations/layer6_attention_weight_min": -52.80472183227539,
      "activations/layer7_attention_weight_max": 74.35198211669922,
      "activations/layer7_attention_weight_min": -73.49955749511719,
      "activations/layer8_attention_weight_max": 45.685848236083984,
      "activations/layer8_attention_weight_min": -47.000511169433594,
      "activations/layer9_attention_weight_max": 51.08462142944336,
      "activations/layer9_attention_weight_min": -50.482696533203125,
      "epoch": 7.63,
      "learning_rate": 0.00010184507575757575,
      "loss": 2.8075,
      "step": 131250
    },
    {
      "activations/layer0_attention_weight_max": 15.471653938293457,
      "activations/layer0_attention_weight_min": -14.40471076965332,
      "activations/layer10_attention_weight_max": 36.3381233215332,
      "activations/layer10_attention_weight_min": -34.73919677734375,
      "activations/layer11_attention_weight_max": 30.180862426757812,
      "activations/layer11_attention_weight_min": -30.743824005126953,
      "activations/layer12_attention_weight_max": 17.683422088623047,
      "activations/layer12_attention_weight_min": -23.779367446899414,
      "activations/layer13_attention_weight_max": 40.720855712890625,
      "activations/layer13_attention_weight_min": -29.729480743408203,
      "activations/layer14_attention_weight_max": 38.68010711669922,
      "activations/layer14_attention_weight_min": -29.808712005615234,
      "activations/layer15_attention_weight_max": 35.36957931518555,
      "activations/layer15_attention_weight_min": -27.9996337890625,
      "activations/layer16_attention_weight_max": 30.378141403198242,
      "activations/layer16_attention_weight_min": -28.146831512451172,
      "activations/layer17_attention_weight_max": 56.110023498535156,
      "activations/layer17_attention_weight_min": -41.84641647338867,
      "activations/layer18_attention_weight_max": 47.92148208618164,
      "activations/layer18_attention_weight_min": -37.109619140625,
      "activations/layer19_attention_weight_max": 22.3140926361084,
      "activations/layer19_attention_weight_min": -20.893457412719727,
      "activations/layer1_attention_weight_max": 16.934097290039062,
      "activations/layer1_attention_weight_min": -14.793471336364746,
      "activations/layer20_attention_weight_max": 21.522539138793945,
      "activations/layer20_attention_weight_min": -21.166221618652344,
      "activations/layer21_attention_weight_max": 37.521339416503906,
      "activations/layer21_attention_weight_min": -24.31004524230957,
      "activations/layer22_attention_weight_max": 34.30812072753906,
      "activations/layer22_attention_weight_min": -24.849628448486328,
      "activations/layer23_attention_weight_max": 35.88078308105469,
      "activations/layer23_attention_weight_min": -24.117305755615234,
      "activations/layer2_attention_weight_max": 31.500560760498047,
      "activations/layer2_attention_weight_min": -27.847850799560547,
      "activations/layer3_attention_weight_max": 80.5807113647461,
      "activations/layer3_attention_weight_min": -81.8078384399414,
      "activations/layer4_attention_weight_max": 95.28144836425781,
      "activations/layer4_attention_weight_min": -94.17302703857422,
      "activations/layer5_attention_weight_max": 84.61421966552734,
      "activations/layer5_attention_weight_min": -79.34158325195312,
      "activations/layer6_attention_weight_max": 53.91317367553711,
      "activations/layer6_attention_weight_min": -50.923213958740234,
      "activations/layer7_attention_weight_max": 71.10914611816406,
      "activations/layer7_attention_weight_min": -70.17117309570312,
      "activations/layer8_attention_weight_max": 42.40276336669922,
      "activations/layer8_attention_weight_min": -44.61254119873047,
      "activations/layer9_attention_weight_max": 49.14203643798828,
      "activations/layer9_attention_weight_min": -49.50698471069336,
      "epoch": 7.63,
      "learning_rate": 0.00010182613636363636,
      "loss": 2.825,
      "step": 131300
    },
    {
      "activations/layer0_attention_weight_max": 16.33195686340332,
      "activations/layer0_attention_weight_min": -14.209996223449707,
      "activations/layer10_attention_weight_max": 37.74322509765625,
      "activations/layer10_attention_weight_min": -36.46108627319336,
      "activations/layer11_attention_weight_max": 30.228042602539062,
      "activations/layer11_attention_weight_min": -30.002105712890625,
      "activations/layer12_attention_weight_max": 20.31533432006836,
      "activations/layer12_attention_weight_min": -24.07738494873047,
      "activations/layer13_attention_weight_max": 45.87342071533203,
      "activations/layer13_attention_weight_min": -34.47256088256836,
      "activations/layer14_attention_weight_max": 41.70466613769531,
      "activations/layer14_attention_weight_min": -30.260711669921875,
      "activations/layer15_attention_weight_max": 37.394134521484375,
      "activations/layer15_attention_weight_min": -29.54653549194336,
      "activations/layer16_attention_weight_max": 31.89347267150879,
      "activations/layer16_attention_weight_min": -26.60992431640625,
      "activations/layer17_attention_weight_max": 54.17267608642578,
      "activations/layer17_attention_weight_min": -42.6362419128418,
      "activations/layer18_attention_weight_max": 47.76201629638672,
      "activations/layer18_attention_weight_min": -35.51079559326172,
      "activations/layer19_attention_weight_max": 21.789016723632812,
      "activations/layer19_attention_weight_min": -19.707509994506836,
      "activations/layer1_attention_weight_max": 17.375778198242188,
      "activations/layer1_attention_weight_min": -15.058771133422852,
      "activations/layer20_attention_weight_max": 21.832483291625977,
      "activations/layer20_attention_weight_min": -20.456499099731445,
      "activations/layer21_attention_weight_max": 41.1325569152832,
      "activations/layer21_attention_weight_min": -24.346153259277344,
      "activations/layer22_attention_weight_max": 32.70682144165039,
      "activations/layer22_attention_weight_min": -25.052289962768555,
      "activations/layer23_attention_weight_max": 33.31562423706055,
      "activations/layer23_attention_weight_min": -22.207107543945312,
      "activations/layer2_attention_weight_max": 32.027610778808594,
      "activations/layer2_attention_weight_min": -31.680936813354492,
      "activations/layer3_attention_weight_max": 85.96830749511719,
      "activations/layer3_attention_weight_min": -88.44231414794922,
      "activations/layer4_attention_weight_max": 98.5385971069336,
      "activations/layer4_attention_weight_min": -96.60539245605469,
      "activations/layer5_attention_weight_max": 84.56298065185547,
      "activations/layer5_attention_weight_min": -80.8983154296875,
      "activations/layer6_attention_weight_max": 52.60383224487305,
      "activations/layer6_attention_weight_min": -52.08118438720703,
      "activations/layer7_attention_weight_max": 75.95142364501953,
      "activations/layer7_attention_weight_min": -74.99877166748047,
      "activations/layer8_attention_weight_max": 42.31906509399414,
      "activations/layer8_attention_weight_min": -46.561927795410156,
      "activations/layer9_attention_weight_max": 52.36554718017578,
      "activations/layer9_attention_weight_min": -49.50680923461914,
      "epoch": 7.63,
      "learning_rate": 0.00010180719696969695,
      "loss": 2.8217,
      "step": 131350
    },
    {
      "activations/layer0_attention_weight_max": 15.000802040100098,
      "activations/layer0_attention_weight_min": -14.41766357421875,
      "activations/layer10_attention_weight_max": 37.79073715209961,
      "activations/layer10_attention_weight_min": -35.4185791015625,
      "activations/layer11_attention_weight_max": 31.619535446166992,
      "activations/layer11_attention_weight_min": -32.648136138916016,
      "activations/layer12_attention_weight_max": 19.394399642944336,
      "activations/layer12_attention_weight_min": -26.63485336303711,
      "activations/layer13_attention_weight_max": 40.33879470825195,
      "activations/layer13_attention_weight_min": -33.82093048095703,
      "activations/layer14_attention_weight_max": 38.60359191894531,
      "activations/layer14_attention_weight_min": -31.97971534729004,
      "activations/layer15_attention_weight_max": 33.84433364868164,
      "activations/layer15_attention_weight_min": -30.915651321411133,
      "activations/layer16_attention_weight_max": 33.564430236816406,
      "activations/layer16_attention_weight_min": -27.77068519592285,
      "activations/layer17_attention_weight_max": 52.8342170715332,
      "activations/layer17_attention_weight_min": -41.5417366027832,
      "activations/layer18_attention_weight_max": 46.3701286315918,
      "activations/layer18_attention_weight_min": -35.401268005371094,
      "activations/layer19_attention_weight_max": 22.168357849121094,
      "activations/layer19_attention_weight_min": -22.155515670776367,
      "activations/layer1_attention_weight_max": 17.554033279418945,
      "activations/layer1_attention_weight_min": -15.932239532470703,
      "activations/layer20_attention_weight_max": 22.16870880126953,
      "activations/layer20_attention_weight_min": -23.167783737182617,
      "activations/layer21_attention_weight_max": 37.87334060668945,
      "activations/layer21_attention_weight_min": -24.299570083618164,
      "activations/layer22_attention_weight_max": 31.002532958984375,
      "activations/layer22_attention_weight_min": -25.51993751525879,
      "activations/layer23_attention_weight_max": 35.80424880981445,
      "activations/layer23_attention_weight_min": -22.948158264160156,
      "activations/layer2_attention_weight_max": 30.55872917175293,
      "activations/layer2_attention_weight_min": -28.142234802246094,
      "activations/layer3_attention_weight_max": 86.39015197753906,
      "activations/layer3_attention_weight_min": -89.09197998046875,
      "activations/layer4_attention_weight_max": 101.23677062988281,
      "activations/layer4_attention_weight_min": -98.72171020507812,
      "activations/layer5_attention_weight_max": 88.76370239257812,
      "activations/layer5_attention_weight_min": -81.64430236816406,
      "activations/layer6_attention_weight_max": 56.00455093383789,
      "activations/layer6_attention_weight_min": -52.1241569519043,
      "activations/layer7_attention_weight_max": 76.34505462646484,
      "activations/layer7_attention_weight_min": -71.3691635131836,
      "activations/layer8_attention_weight_max": 44.857215881347656,
      "activations/layer8_attention_weight_min": -45.20830154418945,
      "activations/layer9_attention_weight_max": 55.130977630615234,
      "activations/layer9_attention_weight_min": -47.4390869140625,
      "epoch": 7.64,
      "learning_rate": 0.00010178825757575757,
      "loss": 2.7954,
      "step": 131400
    },
    {
      "activations/layer0_attention_weight_max": 16.32613754272461,
      "activations/layer0_attention_weight_min": -14.03659725189209,
      "activations/layer10_attention_weight_max": 35.91754150390625,
      "activations/layer10_attention_weight_min": -35.45365905761719,
      "activations/layer11_attention_weight_max": 30.81570053100586,
      "activations/layer11_attention_weight_min": -31.565330505371094,
      "activations/layer12_attention_weight_max": 21.26264762878418,
      "activations/layer12_attention_weight_min": -26.94268035888672,
      "activations/layer13_attention_weight_max": 53.6771125793457,
      "activations/layer13_attention_weight_min": -30.953508377075195,
      "activations/layer14_attention_weight_max": 61.738624572753906,
      "activations/layer14_attention_weight_min": -36.95363998413086,
      "activations/layer15_attention_weight_max": 41.575721740722656,
      "activations/layer15_attention_weight_min": -31.91819190979004,
      "activations/layer16_attention_weight_max": 32.59020233154297,
      "activations/layer16_attention_weight_min": -26.851543426513672,
      "activations/layer17_attention_weight_max": 53.801387786865234,
      "activations/layer17_attention_weight_min": -43.67264938354492,
      "activations/layer18_attention_weight_max": 47.10450744628906,
      "activations/layer18_attention_weight_min": -34.3352165222168,
      "activations/layer19_attention_weight_max": 19.419445037841797,
      "activations/layer19_attention_weight_min": -20.97389030456543,
      "activations/layer1_attention_weight_max": 17.71468734741211,
      "activations/layer1_attention_weight_min": -14.729593276977539,
      "activations/layer20_attention_weight_max": 20.167448043823242,
      "activations/layer20_attention_weight_min": -24.220720291137695,
      "activations/layer21_attention_weight_max": 35.713008880615234,
      "activations/layer21_attention_weight_min": -24.5443058013916,
      "activations/layer22_attention_weight_max": 31.044986724853516,
      "activations/layer22_attention_weight_min": -25.34693145751953,
      "activations/layer23_attention_weight_max": 30.94928550720215,
      "activations/layer23_attention_weight_min": -21.638214111328125,
      "activations/layer2_attention_weight_max": 31.941818237304688,
      "activations/layer2_attention_weight_min": -29.87221336364746,
      "activations/layer3_attention_weight_max": 84.97538757324219,
      "activations/layer3_attention_weight_min": -86.8305435180664,
      "activations/layer4_attention_weight_max": 98.59029388427734,
      "activations/layer4_attention_weight_min": -94.09942626953125,
      "activations/layer5_attention_weight_max": 82.3540267944336,
      "activations/layer5_attention_weight_min": -79.70201873779297,
      "activations/layer6_attention_weight_max": 53.810020446777344,
      "activations/layer6_attention_weight_min": -50.34373092651367,
      "activations/layer7_attention_weight_max": 75.70655059814453,
      "activations/layer7_attention_weight_min": -71.25730895996094,
      "activations/layer8_attention_weight_max": 42.58287048339844,
      "activations/layer8_attention_weight_min": -43.9806022644043,
      "activations/layer9_attention_weight_max": 50.98063659667969,
      "activations/layer9_attention_weight_min": -52.094146728515625,
      "epoch": 7.64,
      "learning_rate": 0.00010176931818181818,
      "loss": 2.807,
      "step": 131450
    },
    {
      "activations/layer0_attention_weight_max": 16.031700134277344,
      "activations/layer0_attention_weight_min": -14.418380737304688,
      "activations/layer10_attention_weight_max": 41.55633544921875,
      "activations/layer10_attention_weight_min": -41.38501739501953,
      "activations/layer11_attention_weight_max": 35.369544982910156,
      "activations/layer11_attention_weight_min": -35.96928405761719,
      "activations/layer12_attention_weight_max": 22.43443489074707,
      "activations/layer12_attention_weight_min": -33.72937774658203,
      "activations/layer13_attention_weight_max": 40.74794387817383,
      "activations/layer13_attention_weight_min": -30.485671997070312,
      "activations/layer14_attention_weight_max": 39.76937484741211,
      "activations/layer14_attention_weight_min": -29.732290267944336,
      "activations/layer15_attention_weight_max": 37.92747116088867,
      "activations/layer15_attention_weight_min": -28.307558059692383,
      "activations/layer16_attention_weight_max": 33.99224853515625,
      "activations/layer16_attention_weight_min": -27.423887252807617,
      "activations/layer17_attention_weight_max": 57.34833526611328,
      "activations/layer17_attention_weight_min": -41.18095779418945,
      "activations/layer18_attention_weight_max": 48.69353485107422,
      "activations/layer18_attention_weight_min": -36.41144943237305,
      "activations/layer19_attention_weight_max": 24.42845916748047,
      "activations/layer19_attention_weight_min": -20.15821647644043,
      "activations/layer1_attention_weight_max": 16.63810920715332,
      "activations/layer1_attention_weight_min": -14.283014297485352,
      "activations/layer20_attention_weight_max": 23.932111740112305,
      "activations/layer20_attention_weight_min": -23.32831382751465,
      "activations/layer21_attention_weight_max": 42.842796325683594,
      "activations/layer21_attention_weight_min": -23.617021560668945,
      "activations/layer22_attention_weight_max": 33.21847915649414,
      "activations/layer22_attention_weight_min": -27.845273971557617,
      "activations/layer23_attention_weight_max": 36.599609375,
      "activations/layer23_attention_weight_min": -24.04371452331543,
      "activations/layer2_attention_weight_max": 34.56005096435547,
      "activations/layer2_attention_weight_min": -31.93691062927246,
      "activations/layer3_attention_weight_max": 102.4866714477539,
      "activations/layer3_attention_weight_min": -102.21072387695312,
      "activations/layer4_attention_weight_max": 114.35929107666016,
      "activations/layer4_attention_weight_min": -116.95765686035156,
      "activations/layer5_attention_weight_max": 95.08804321289062,
      "activations/layer5_attention_weight_min": -94.10321044921875,
      "activations/layer6_attention_weight_max": 56.528526306152344,
      "activations/layer6_attention_weight_min": -57.2634162902832,
      "activations/layer7_attention_weight_max": 79.81564331054688,
      "activations/layer7_attention_weight_min": -79.60627746582031,
      "activations/layer8_attention_weight_max": 47.777503967285156,
      "activations/layer8_attention_weight_min": -49.700279235839844,
      "activations/layer9_attention_weight_max": 55.707496643066406,
      "activations/layer9_attention_weight_min": -55.40131378173828,
      "epoch": 7.64,
      "learning_rate": 0.00010175037878787877,
      "loss": 2.8331,
      "step": 131500
    },
    {
      "activations/layer0_attention_weight_max": 14.856112480163574,
      "activations/layer0_attention_weight_min": -14.945436477661133,
      "activations/layer10_attention_weight_max": 34.53145217895508,
      "activations/layer10_attention_weight_min": -33.23084259033203,
      "activations/layer11_attention_weight_max": 28.596891403198242,
      "activations/layer11_attention_weight_min": -28.505712509155273,
      "activations/layer12_attention_weight_max": 18.817689895629883,
      "activations/layer12_attention_weight_min": -23.693649291992188,
      "activations/layer13_attention_weight_max": 43.61931228637695,
      "activations/layer13_attention_weight_min": -31.73788833618164,
      "activations/layer14_attention_weight_max": 36.03816604614258,
      "activations/layer14_attention_weight_min": -30.389673233032227,
      "activations/layer15_attention_weight_max": 34.39094161987305,
      "activations/layer15_attention_weight_min": -29.320186614990234,
      "activations/layer16_attention_weight_max": 30.871301651000977,
      "activations/layer16_attention_weight_min": -29.189924240112305,
      "activations/layer17_attention_weight_max": 54.416114807128906,
      "activations/layer17_attention_weight_min": -46.296913146972656,
      "activations/layer18_attention_weight_max": 48.86724853515625,
      "activations/layer18_attention_weight_min": -35.88627243041992,
      "activations/layer19_attention_weight_max": 24.837562561035156,
      "activations/layer19_attention_weight_min": -22.339988708496094,
      "activations/layer1_attention_weight_max": 16.81327247619629,
      "activations/layer1_attention_weight_min": -13.76594352722168,
      "activations/layer20_attention_weight_max": 22.381027221679688,
      "activations/layer20_attention_weight_min": -25.219152450561523,
      "activations/layer21_attention_weight_max": 41.423858642578125,
      "activations/layer21_attention_weight_min": -26.201400756835938,
      "activations/layer22_attention_weight_max": 31.94434928894043,
      "activations/layer22_attention_weight_min": -25.680274963378906,
      "activations/layer23_attention_weight_max": 35.86692810058594,
      "activations/layer23_attention_weight_min": -22.189300537109375,
      "activations/layer2_attention_weight_max": 31.336774826049805,
      "activations/layer2_attention_weight_min": -29.358844757080078,
      "activations/layer3_attention_weight_max": 84.95906066894531,
      "activations/layer3_attention_weight_min": -87.88224792480469,
      "activations/layer4_attention_weight_max": 99.08321380615234,
      "activations/layer4_attention_weight_min": -90.82170867919922,
      "activations/layer5_attention_weight_max": 85.9140853881836,
      "activations/layer5_attention_weight_min": -81.05860900878906,
      "activations/layer6_attention_weight_max": 52.31220245361328,
      "activations/layer6_attention_weight_min": -51.00156784057617,
      "activations/layer7_attention_weight_max": 68.71317291259766,
      "activations/layer7_attention_weight_min": -68.9332275390625,
      "activations/layer8_attention_weight_max": 42.204288482666016,
      "activations/layer8_attention_weight_min": -42.85450744628906,
      "activations/layer9_attention_weight_max": 47.7941780090332,
      "activations/layer9_attention_weight_min": -44.8812370300293,
      "epoch": 7.64,
      "learning_rate": 0.00010173143939393938,
      "loss": 2.8164,
      "step": 131550
    },
    {
      "activations/layer0_attention_weight_max": 14.915265083312988,
      "activations/layer0_attention_weight_min": -14.128092765808105,
      "activations/layer10_attention_weight_max": 36.91059494018555,
      "activations/layer10_attention_weight_min": -35.483009338378906,
      "activations/layer11_attention_weight_max": 30.65785789489746,
      "activations/layer11_attention_weight_min": -31.49622344970703,
      "activations/layer12_attention_weight_max": 19.282955169677734,
      "activations/layer12_attention_weight_min": -23.121007919311523,
      "activations/layer13_attention_weight_max": 45.08546447753906,
      "activations/layer13_attention_weight_min": -28.924673080444336,
      "activations/layer14_attention_weight_max": 39.63045883178711,
      "activations/layer14_attention_weight_min": -29.513593673706055,
      "activations/layer15_attention_weight_max": 38.53608703613281,
      "activations/layer15_attention_weight_min": -28.540000915527344,
      "activations/layer16_attention_weight_max": 33.23648452758789,
      "activations/layer16_attention_weight_min": -28.567163467407227,
      "activations/layer17_attention_weight_max": 55.64067459106445,
      "activations/layer17_attention_weight_min": -43.66708755493164,
      "activations/layer18_attention_weight_max": 48.02522659301758,
      "activations/layer18_attention_weight_min": -37.17594528198242,
      "activations/layer19_attention_weight_max": 23.632144927978516,
      "activations/layer19_attention_weight_min": -23.088661193847656,
      "activations/layer1_attention_weight_max": 17.614151000976562,
      "activations/layer1_attention_weight_min": -17.24068260192871,
      "activations/layer20_attention_weight_max": 25.0211124420166,
      "activations/layer20_attention_weight_min": -24.783613204956055,
      "activations/layer21_attention_weight_max": 42.11126708984375,
      "activations/layer21_attention_weight_min": -24.2226619720459,
      "activations/layer22_attention_weight_max": 33.37550354003906,
      "activations/layer22_attention_weight_min": -24.39518928527832,
      "activations/layer23_attention_weight_max": 36.734405517578125,
      "activations/layer23_attention_weight_min": -21.289405822753906,
      "activations/layer2_attention_weight_max": 33.22016143798828,
      "activations/layer2_attention_weight_min": -30.224180221557617,
      "activations/layer3_attention_weight_max": 86.4380874633789,
      "activations/layer3_attention_weight_min": -89.55642700195312,
      "activations/layer4_attention_weight_max": 103.12159729003906,
      "activations/layer4_attention_weight_min": -96.7300033569336,
      "activations/layer5_attention_weight_max": 89.64714050292969,
      "activations/layer5_attention_weight_min": -85.38531494140625,
      "activations/layer6_attention_weight_max": 57.38798904418945,
      "activations/layer6_attention_weight_min": -54.307960510253906,
      "activations/layer7_attention_weight_max": 77.92851257324219,
      "activations/layer7_attention_weight_min": -73.84217071533203,
      "activations/layer8_attention_weight_max": 43.18202590942383,
      "activations/layer8_attention_weight_min": -44.58544158935547,
      "activations/layer9_attention_weight_max": 47.82490158081055,
      "activations/layer9_attention_weight_min": -47.224327087402344,
      "epoch": 7.65,
      "learning_rate": 0.0001017125,
      "loss": 2.8007,
      "step": 131600
    },
    {
      "activations/layer0_attention_weight_max": 14.773435592651367,
      "activations/layer0_attention_weight_min": -14.237324714660645,
      "activations/layer10_attention_weight_max": 42.689857482910156,
      "activations/layer10_attention_weight_min": -38.664005279541016,
      "activations/layer11_attention_weight_max": 36.21021270751953,
      "activations/layer11_attention_weight_min": -34.33033752441406,
      "activations/layer12_attention_weight_max": 19.772573471069336,
      "activations/layer12_attention_weight_min": -24.79184341430664,
      "activations/layer13_attention_weight_max": 52.183895111083984,
      "activations/layer13_attention_weight_min": -36.995689392089844,
      "activations/layer14_attention_weight_max": 45.96908950805664,
      "activations/layer14_attention_weight_min": -32.809871673583984,
      "activations/layer15_attention_weight_max": 36.652366638183594,
      "activations/layer15_attention_weight_min": -30.303030014038086,
      "activations/layer16_attention_weight_max": 34.13274383544922,
      "activations/layer16_attention_weight_min": -27.42777442932129,
      "activations/layer17_attention_weight_max": 57.34058380126953,
      "activations/layer17_attention_weight_min": -43.31489944458008,
      "activations/layer18_attention_weight_max": 47.7105827331543,
      "activations/layer18_attention_weight_min": -36.947784423828125,
      "activations/layer19_attention_weight_max": 22.39710807800293,
      "activations/layer19_attention_weight_min": -20.608327865600586,
      "activations/layer1_attention_weight_max": 18.33135414123535,
      "activations/layer1_attention_weight_min": -16.97555923461914,
      "activations/layer20_attention_weight_max": 22.642507553100586,
      "activations/layer20_attention_weight_min": -22.126667022705078,
      "activations/layer21_attention_weight_max": 43.20399856567383,
      "activations/layer21_attention_weight_min": -25.877025604248047,
      "activations/layer22_attention_weight_max": 34.85009765625,
      "activations/layer22_attention_weight_min": -26.234243392944336,
      "activations/layer23_attention_weight_max": 40.3480224609375,
      "activations/layer23_attention_weight_min": -22.026790618896484,
      "activations/layer2_attention_weight_max": 31.2601375579834,
      "activations/layer2_attention_weight_min": -29.47589111328125,
      "activations/layer3_attention_weight_max": 88.87068176269531,
      "activations/layer3_attention_weight_min": -89.35907745361328,
      "activations/layer4_attention_weight_max": 108.77591705322266,
      "activations/layer4_attention_weight_min": -96.87841796875,
      "activations/layer5_attention_weight_max": 89.5628662109375,
      "activations/layer5_attention_weight_min": -80.46022033691406,
      "activations/layer6_attention_weight_max": 55.94904708862305,
      "activations/layer6_attention_weight_min": -54.168968200683594,
      "activations/layer7_attention_weight_max": 80.39085388183594,
      "activations/layer7_attention_weight_min": -72.90853118896484,
      "activations/layer8_attention_weight_max": 47.95982360839844,
      "activations/layer8_attention_weight_min": -46.3017578125,
      "activations/layer9_attention_weight_max": 60.93085861206055,
      "activations/layer9_attention_weight_min": -52.5198860168457,
      "epoch": 7.65,
      "learning_rate": 0.00010169356060606059,
      "loss": 2.8211,
      "step": 131650
    },
    {
      "activations/layer0_attention_weight_max": 15.499754905700684,
      "activations/layer0_attention_weight_min": -14.18634033203125,
      "activations/layer10_attention_weight_max": 37.79106521606445,
      "activations/layer10_attention_weight_min": -35.203399658203125,
      "activations/layer11_attention_weight_max": 31.298873901367188,
      "activations/layer11_attention_weight_min": -30.794841766357422,
      "activations/layer12_attention_weight_max": 18.75763511657715,
      "activations/layer12_attention_weight_min": -28.754854202270508,
      "activations/layer13_attention_weight_max": 48.65635299682617,
      "activations/layer13_attention_weight_min": -35.34782409667969,
      "activations/layer14_attention_weight_max": 45.50899124145508,
      "activations/layer14_attention_weight_min": -31.090015411376953,
      "activations/layer15_attention_weight_max": 38.44871520996094,
      "activations/layer15_attention_weight_min": -30.731569290161133,
      "activations/layer16_attention_weight_max": 40.61602020263672,
      "activations/layer16_attention_weight_min": -32.64625549316406,
      "activations/layer17_attention_weight_max": 55.92879867553711,
      "activations/layer17_attention_weight_min": -43.920101165771484,
      "activations/layer18_attention_weight_max": 51.14738082885742,
      "activations/layer18_attention_weight_min": -38.495731353759766,
      "activations/layer19_attention_weight_max": 23.282684326171875,
      "activations/layer19_attention_weight_min": -22.636762619018555,
      "activations/layer1_attention_weight_max": 17.0882511138916,
      "activations/layer1_attention_weight_min": -18.185598373413086,
      "activations/layer20_attention_weight_max": 25.737688064575195,
      "activations/layer20_attention_weight_min": -21.20055389404297,
      "activations/layer21_attention_weight_max": 39.33348846435547,
      "activations/layer21_attention_weight_min": -25.496837615966797,
      "activations/layer22_attention_weight_max": 37.07005310058594,
      "activations/layer22_attention_weight_min": -27.64668846130371,
      "activations/layer23_attention_weight_max": 38.87128829956055,
      "activations/layer23_attention_weight_min": -24.594778060913086,
      "activations/layer2_attention_weight_max": 31.14028549194336,
      "activations/layer2_attention_weight_min": -29.69185447692871,
      "activations/layer3_attention_weight_max": 86.30956268310547,
      "activations/layer3_attention_weight_min": -88.73258209228516,
      "activations/layer4_attention_weight_max": 98.6726303100586,
      "activations/layer4_attention_weight_min": -94.03792572021484,
      "activations/layer5_attention_weight_max": 85.3272705078125,
      "activations/layer5_attention_weight_min": -77.04106140136719,
      "activations/layer6_attention_weight_max": 54.52832794189453,
      "activations/layer6_attention_weight_min": -51.50190353393555,
      "activations/layer7_attention_weight_max": 79.94127655029297,
      "activations/layer7_attention_weight_min": -70.74000549316406,
      "activations/layer8_attention_weight_max": 42.96396255493164,
      "activations/layer8_attention_weight_min": -43.599544525146484,
      "activations/layer9_attention_weight_max": 52.1336555480957,
      "activations/layer9_attention_weight_min": -46.26187515258789,
      "epoch": 7.65,
      "learning_rate": 0.0001016746212121212,
      "loss": 2.8071,
      "step": 131700
    },
    {
      "activations/layer0_attention_weight_max": 16.499664306640625,
      "activations/layer0_attention_weight_min": -14.585816383361816,
      "activations/layer10_attention_weight_max": 37.33741760253906,
      "activations/layer10_attention_weight_min": -36.9814338684082,
      "activations/layer11_attention_weight_max": 30.162992477416992,
      "activations/layer11_attention_weight_min": -30.060916900634766,
      "activations/layer12_attention_weight_max": 17.942352294921875,
      "activations/layer12_attention_weight_min": -24.183263778686523,
      "activations/layer13_attention_weight_max": 45.66613006591797,
      "activations/layer13_attention_weight_min": -33.707828521728516,
      "activations/layer14_attention_weight_max": 40.106266021728516,
      "activations/layer14_attention_weight_min": -30.51933479309082,
      "activations/layer15_attention_weight_max": 35.000423431396484,
      "activations/layer15_attention_weight_min": -29.112789154052734,
      "activations/layer16_attention_weight_max": 31.533645629882812,
      "activations/layer16_attention_weight_min": -29.179744720458984,
      "activations/layer17_attention_weight_max": 55.927669525146484,
      "activations/layer17_attention_weight_min": -43.81198501586914,
      "activations/layer18_attention_weight_max": 48.67380905151367,
      "activations/layer18_attention_weight_min": -37.24602508544922,
      "activations/layer19_attention_weight_max": 23.579975128173828,
      "activations/layer19_attention_weight_min": -19.971311569213867,
      "activations/layer1_attention_weight_max": 16.418569564819336,
      "activations/layer1_attention_weight_min": -15.29351806640625,
      "activations/layer20_attention_weight_max": 22.68384552001953,
      "activations/layer20_attention_weight_min": -19.893312454223633,
      "activations/layer21_attention_weight_max": 36.099544525146484,
      "activations/layer21_attention_weight_min": -24.26593017578125,
      "activations/layer22_attention_weight_max": 31.337182998657227,
      "activations/layer22_attention_weight_min": -24.163700103759766,
      "activations/layer23_attention_weight_max": 34.22283935546875,
      "activations/layer23_attention_weight_min": -21.211299896240234,
      "activations/layer2_attention_weight_max": 33.379180908203125,
      "activations/layer2_attention_weight_min": -30.160385131835938,
      "activations/layer3_attention_weight_max": 87.99226379394531,
      "activations/layer3_attention_weight_min": -87.04967498779297,
      "activations/layer4_attention_weight_max": 103.93714904785156,
      "activations/layer4_attention_weight_min": -98.41533660888672,
      "activations/layer5_attention_weight_max": 91.50243377685547,
      "activations/layer5_attention_weight_min": -82.33981323242188,
      "activations/layer6_attention_weight_max": 57.84260940551758,
      "activations/layer6_attention_weight_min": -56.360267639160156,
      "activations/layer7_attention_weight_max": 79.3936996459961,
      "activations/layer7_attention_weight_min": -73.63450622558594,
      "activations/layer8_attention_weight_max": 45.7265510559082,
      "activations/layer8_attention_weight_min": -45.92686462402344,
      "activations/layer9_attention_weight_max": 53.074501037597656,
      "activations/layer9_attention_weight_min": -47.94029998779297,
      "epoch": 7.66,
      "learning_rate": 0.00010165568181818182,
      "loss": 2.8234,
      "step": 131750
    },
    {
      "activations/layer0_attention_weight_max": 14.423340797424316,
      "activations/layer0_attention_weight_min": -14.568299293518066,
      "activations/layer10_attention_weight_max": 41.06040954589844,
      "activations/layer10_attention_weight_min": -38.9713134765625,
      "activations/layer11_attention_weight_max": 32.932491302490234,
      "activations/layer11_attention_weight_min": -31.781272888183594,
      "activations/layer12_attention_weight_max": 20.776281356811523,
      "activations/layer12_attention_weight_min": -27.4389705657959,
      "activations/layer13_attention_weight_max": 46.35887145996094,
      "activations/layer13_attention_weight_min": -32.74606704711914,
      "activations/layer14_attention_weight_max": 35.95178985595703,
      "activations/layer14_attention_weight_min": -30.33071517944336,
      "activations/layer15_attention_weight_max": 34.281715393066406,
      "activations/layer15_attention_weight_min": -30.654922485351562,
      "activations/layer16_attention_weight_max": 31.900114059448242,
      "activations/layer16_attention_weight_min": -27.886507034301758,
      "activations/layer17_attention_weight_max": 55.504676818847656,
      "activations/layer17_attention_weight_min": -44.88014221191406,
      "activations/layer18_attention_weight_max": 44.46123504638672,
      "activations/layer18_attention_weight_min": -39.04067611694336,
      "activations/layer19_attention_weight_max": 21.4589786529541,
      "activations/layer19_attention_weight_min": -21.456838607788086,
      "activations/layer1_attention_weight_max": 17.38056182861328,
      "activations/layer1_attention_weight_min": -14.33034896850586,
      "activations/layer20_attention_weight_max": 19.778099060058594,
      "activations/layer20_attention_weight_min": -21.802457809448242,
      "activations/layer21_attention_weight_max": 37.70064926147461,
      "activations/layer21_attention_weight_min": -23.005638122558594,
      "activations/layer22_attention_weight_max": 29.419878005981445,
      "activations/layer22_attention_weight_min": -27.320663452148438,
      "activations/layer23_attention_weight_max": 33.35310745239258,
      "activations/layer23_attention_weight_min": -22.963016510009766,
      "activations/layer2_attention_weight_max": 32.68601989746094,
      "activations/layer2_attention_weight_min": -31.129798889160156,
      "activations/layer3_attention_weight_max": 91.07623291015625,
      "activations/layer3_attention_weight_min": -91.35256958007812,
      "activations/layer4_attention_weight_max": 105.78663635253906,
      "activations/layer4_attention_weight_min": -102.03986358642578,
      "activations/layer5_attention_weight_max": 88.4275131225586,
      "activations/layer5_attention_weight_min": -85.45406341552734,
      "activations/layer6_attention_weight_max": 55.59656524658203,
      "activations/layer6_attention_weight_min": -54.61777114868164,
      "activations/layer7_attention_weight_max": 81.29463958740234,
      "activations/layer7_attention_weight_min": -72.73634338378906,
      "activations/layer8_attention_weight_max": 46.279781341552734,
      "activations/layer8_attention_weight_min": -48.631439208984375,
      "activations/layer9_attention_weight_max": 58.744720458984375,
      "activations/layer9_attention_weight_min": -50.651737213134766,
      "epoch": 7.66,
      "learning_rate": 0.00010163712121212121,
      "loss": 2.8193,
      "step": 131800
    },
    {
      "activations/layer0_attention_weight_max": 14.572236061096191,
      "activations/layer0_attention_weight_min": -14.484380722045898,
      "activations/layer10_attention_weight_max": 37.78340148925781,
      "activations/layer10_attention_weight_min": -34.63251495361328,
      "activations/layer11_attention_weight_max": 30.558887481689453,
      "activations/layer11_attention_weight_min": -30.513629913330078,
      "activations/layer12_attention_weight_max": 20.183399200439453,
      "activations/layer12_attention_weight_min": -27.776079177856445,
      "activations/layer13_attention_weight_max": 40.234432220458984,
      "activations/layer13_attention_weight_min": -32.307918548583984,
      "activations/layer14_attention_weight_max": 41.26157760620117,
      "activations/layer14_attention_weight_min": -30.266902923583984,
      "activations/layer15_attention_weight_max": 36.00448226928711,
      "activations/layer15_attention_weight_min": -28.26146125793457,
      "activations/layer16_attention_weight_max": 33.836429595947266,
      "activations/layer16_attention_weight_min": -27.11028480529785,
      "activations/layer17_attention_weight_max": 53.1026496887207,
      "activations/layer17_attention_weight_min": -39.29503631591797,
      "activations/layer18_attention_weight_max": 47.90415573120117,
      "activations/layer18_attention_weight_min": -33.549591064453125,
      "activations/layer19_attention_weight_max": 19.882123947143555,
      "activations/layer19_attention_weight_min": -20.264009475708008,
      "activations/layer1_attention_weight_max": 16.626354217529297,
      "activations/layer1_attention_weight_min": -15.497934341430664,
      "activations/layer20_attention_weight_max": 22.02048683166504,
      "activations/layer20_attention_weight_min": -24.35372543334961,
      "activations/layer21_attention_weight_max": 35.51416778564453,
      "activations/layer21_attention_weight_min": -22.329702377319336,
      "activations/layer22_attention_weight_max": 29.770938873291016,
      "activations/layer22_attention_weight_min": -24.05845069885254,
      "activations/layer23_attention_weight_max": 35.920108795166016,
      "activations/layer23_attention_weight_min": -21.856306076049805,
      "activations/layer2_attention_weight_max": 30.841772079467773,
      "activations/layer2_attention_weight_min": -31.33759880065918,
      "activations/layer3_attention_weight_max": 86.89971923828125,
      "activations/layer3_attention_weight_min": -92.01798248291016,
      "activations/layer4_attention_weight_max": 99.15437316894531,
      "activations/layer4_attention_weight_min": -95.08589935302734,
      "activations/layer5_attention_weight_max": 84.72941589355469,
      "activations/layer5_attention_weight_min": -80.92776489257812,
      "activations/layer6_attention_weight_max": 53.87282180786133,
      "activations/layer6_attention_weight_min": -51.34882736206055,
      "activations/layer7_attention_weight_max": 74.15390014648438,
      "activations/layer7_attention_weight_min": -74.09517669677734,
      "activations/layer8_attention_weight_max": 43.77394104003906,
      "activations/layer8_attention_weight_min": -45.98948669433594,
      "activations/layer9_attention_weight_max": 51.02228927612305,
      "activations/layer9_attention_weight_min": -52.5278205871582,
      "epoch": 7.66,
      "learning_rate": 0.00010161818181818181,
      "loss": 2.8235,
      "step": 131850
    },
    {
      "activations/layer0_attention_weight_max": 15.544488906860352,
      "activations/layer0_attention_weight_min": -14.41427993774414,
      "activations/layer10_attention_weight_max": 37.37641143798828,
      "activations/layer10_attention_weight_min": -36.603858947753906,
      "activations/layer11_attention_weight_max": 32.9161376953125,
      "activations/layer11_attention_weight_min": -30.500478744506836,
      "activations/layer12_attention_weight_max": 27.740554809570312,
      "activations/layer12_attention_weight_min": -32.27082443237305,
      "activations/layer13_attention_weight_max": 39.29512023925781,
      "activations/layer13_attention_weight_min": -29.608484268188477,
      "activations/layer14_attention_weight_max": 37.871864318847656,
      "activations/layer14_attention_weight_min": -29.711151123046875,
      "activations/layer15_attention_weight_max": 35.00778579711914,
      "activations/layer15_attention_weight_min": -28.85300636291504,
      "activations/layer16_attention_weight_max": 30.43467903137207,
      "activations/layer16_attention_weight_min": -26.948692321777344,
      "activations/layer17_attention_weight_max": 50.539405822753906,
      "activations/layer17_attention_weight_min": -41.7028923034668,
      "activations/layer18_attention_weight_max": 45.52403259277344,
      "activations/layer18_attention_weight_min": -34.097084045410156,
      "activations/layer19_attention_weight_max": 22.877338409423828,
      "activations/layer19_attention_weight_min": -19.45840835571289,
      "activations/layer1_attention_weight_max": 17.689502716064453,
      "activations/layer1_attention_weight_min": -14.217805862426758,
      "activations/layer20_attention_weight_max": 21.36465835571289,
      "activations/layer20_attention_weight_min": -18.476604461669922,
      "activations/layer21_attention_weight_max": 33.764469146728516,
      "activations/layer21_attention_weight_min": -22.058698654174805,
      "activations/layer22_attention_weight_max": 28.089462280273438,
      "activations/layer22_attention_weight_min": -24.974103927612305,
      "activations/layer23_attention_weight_max": 30.252483367919922,
      "activations/layer23_attention_weight_min": -20.56464958190918,
      "activations/layer2_attention_weight_max": 33.023651123046875,
      "activations/layer2_attention_weight_min": -29.722349166870117,
      "activations/layer3_attention_weight_max": 101.19927215576172,
      "activations/layer3_attention_weight_min": -97.039794921875,
      "activations/layer4_attention_weight_max": 117.1709213256836,
      "activations/layer4_attention_weight_min": -105.61429595947266,
      "activations/layer5_attention_weight_max": 92.22610473632812,
      "activations/layer5_attention_weight_min": -79.27458953857422,
      "activations/layer6_attention_weight_max": 58.0214958190918,
      "activations/layer6_attention_weight_min": -51.74681854248047,
      "activations/layer7_attention_weight_max": 80.09849548339844,
      "activations/layer7_attention_weight_min": -74.4896240234375,
      "activations/layer8_attention_weight_max": 47.18503952026367,
      "activations/layer8_attention_weight_min": -48.80467224121094,
      "activations/layer9_attention_weight_max": 63.96491622924805,
      "activations/layer9_attention_weight_min": -54.028228759765625,
      "epoch": 7.66,
      "learning_rate": 0.00010159924242424241,
      "loss": 2.8117,
      "step": 131900
    },
    {
      "activations/layer0_attention_weight_max": 15.720778465270996,
      "activations/layer0_attention_weight_min": -14.118998527526855,
      "activations/layer10_attention_weight_max": 37.18060302734375,
      "activations/layer10_attention_weight_min": -38.07278060913086,
      "activations/layer11_attention_weight_max": 29.92499351501465,
      "activations/layer11_attention_weight_min": -29.8710994720459,
      "activations/layer12_attention_weight_max": 22.378381729125977,
      "activations/layer12_attention_weight_min": -25.809711456298828,
      "activations/layer13_attention_weight_max": 52.08046340942383,
      "activations/layer13_attention_weight_min": -34.74185562133789,
      "activations/layer14_attention_weight_max": 60.699188232421875,
      "activations/layer14_attention_weight_min": -41.09025192260742,
      "activations/layer15_attention_weight_max": 49.17877960205078,
      "activations/layer15_attention_weight_min": -35.33503341674805,
      "activations/layer16_attention_weight_max": 32.03605270385742,
      "activations/layer16_attention_weight_min": -25.985769271850586,
      "activations/layer17_attention_weight_max": 49.39040756225586,
      "activations/layer17_attention_weight_min": -42.439674377441406,
      "activations/layer18_attention_weight_max": 45.49576187133789,
      "activations/layer18_attention_weight_min": -35.3621940612793,
      "activations/layer19_attention_weight_max": 21.18016815185547,
      "activations/layer19_attention_weight_min": -21.10848045349121,
      "activations/layer1_attention_weight_max": 17.084421157836914,
      "activations/layer1_attention_weight_min": -16.98093605041504,
      "activations/layer20_attention_weight_max": 23.369815826416016,
      "activations/layer20_attention_weight_min": -25.036649703979492,
      "activations/layer21_attention_weight_max": 43.78825759887695,
      "activations/layer21_attention_weight_min": -23.302488327026367,
      "activations/layer22_attention_weight_max": 31.063232421875,
      "activations/layer22_attention_weight_min": -24.056821823120117,
      "activations/layer23_attention_weight_max": 33.138092041015625,
      "activations/layer23_attention_weight_min": -22.379674911499023,
      "activations/layer2_attention_weight_max": 31.22134780883789,
      "activations/layer2_attention_weight_min": -30.34512710571289,
      "activations/layer3_attention_weight_max": 87.13764190673828,
      "activations/layer3_attention_weight_min": -86.85987091064453,
      "activations/layer4_attention_weight_max": 102.93560028076172,
      "activations/layer4_attention_weight_min": -94.14974212646484,
      "activations/layer5_attention_weight_max": 85.17474365234375,
      "activations/layer5_attention_weight_min": -79.56482696533203,
      "activations/layer6_attention_weight_max": 55.267539978027344,
      "activations/layer6_attention_weight_min": -52.63937759399414,
      "activations/layer7_attention_weight_max": 76.74401092529297,
      "activations/layer7_attention_weight_min": -71.5653305053711,
      "activations/layer8_attention_weight_max": 43.92109680175781,
      "activations/layer8_attention_weight_min": -45.09881591796875,
      "activations/layer9_attention_weight_max": 49.58134078979492,
      "activations/layer9_attention_weight_min": -50.81956100463867,
      "epoch": 7.67,
      "learning_rate": 0.00010158030303030303,
      "loss": 2.819,
      "step": 131950
    },
    {
      "activations/layer0_attention_weight_max": 14.704378128051758,
      "activations/layer0_attention_weight_min": -14.01954460144043,
      "activations/layer10_attention_weight_max": 39.24480438232422,
      "activations/layer10_attention_weight_min": -35.3587760925293,
      "activations/layer11_attention_weight_max": 33.09002685546875,
      "activations/layer11_attention_weight_min": -29.856914520263672,
      "activations/layer12_attention_weight_max": 25.180171966552734,
      "activations/layer12_attention_weight_min": -25.743675231933594,
      "activations/layer13_attention_weight_max": 51.1248664855957,
      "activations/layer13_attention_weight_min": -35.324920654296875,
      "activations/layer14_attention_weight_max": 39.982765197753906,
      "activations/layer14_attention_weight_min": -31.95105743408203,
      "activations/layer15_attention_weight_max": 36.61402893066406,
      "activations/layer15_attention_weight_min": -30.479284286499023,
      "activations/layer16_attention_weight_max": 34.25827407836914,
      "activations/layer16_attention_weight_min": -27.901325225830078,
      "activations/layer17_attention_weight_max": 55.42776870727539,
      "activations/layer17_attention_weight_min": -45.08618927001953,
      "activations/layer18_attention_weight_max": 50.62646484375,
      "activations/layer18_attention_weight_min": -39.101768493652344,
      "activations/layer19_attention_weight_max": 24.342926025390625,
      "activations/layer19_attention_weight_min": -21.08576011657715,
      "activations/layer1_attention_weight_max": 16.08480453491211,
      "activations/layer1_attention_weight_min": -13.904534339904785,
      "activations/layer20_attention_weight_max": 24.745182037353516,
      "activations/layer20_attention_weight_min": -23.123807907104492,
      "activations/layer21_attention_weight_max": 44.02741622924805,
      "activations/layer21_attention_weight_min": -24.734689712524414,
      "activations/layer22_attention_weight_max": 34.22780990600586,
      "activations/layer22_attention_weight_min": -26.60702133178711,
      "activations/layer23_attention_weight_max": 35.78291320800781,
      "activations/layer23_attention_weight_min": -23.392553329467773,
      "activations/layer2_attention_weight_max": 33.30376052856445,
      "activations/layer2_attention_weight_min": -30.028236389160156,
      "activations/layer3_attention_weight_max": 89.67218017578125,
      "activations/layer3_attention_weight_min": -95.3015365600586,
      "activations/layer4_attention_weight_max": 101.09796142578125,
      "activations/layer4_attention_weight_min": -100.41106414794922,
      "activations/layer5_attention_weight_max": 90.1986083984375,
      "activations/layer5_attention_weight_min": -81.17143249511719,
      "activations/layer6_attention_weight_max": 57.95499038696289,
      "activations/layer6_attention_weight_min": -52.65922546386719,
      "activations/layer7_attention_weight_max": 75.9796371459961,
      "activations/layer7_attention_weight_min": -71.91340637207031,
      "activations/layer8_attention_weight_max": 46.64485549926758,
      "activations/layer8_attention_weight_min": -47.790103912353516,
      "activations/layer9_attention_weight_max": 51.769344329833984,
      "activations/layer9_attention_weight_min": -47.70928955078125,
      "epoch": 7.67,
      "learning_rate": 0.00010156136363636363,
      "loss": 2.8055,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_loss": 2.765625,
      "eval_runtime": 8.4984,
      "eval_samples_per_second": 505.273,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.4984,
      "eval_openwebtext_samples_per_second": 505.273,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_wikitext_loss": 3.0078125,
      "eval_wikitext_ppl": 20.24306974171993,
      "eval_wikitext_runtime": 2.0204,
      "eval_wikitext_samples_per_second": 225.701,
      "step": 132000
    },
    {
      "epoch": 7.67,
      "eval_lambada_loss": 2.76953125,
      "eval_lambada_ppl": 15.951155153095918,
      "eval_lambada_runtime": 9.5857,
      "eval_lambada_samples_per_second": 507.945,
      "step": 132000
    },
    {
      "activations/layer0_attention_weight_max": 14.903518676757812,
      "activations/layer0_attention_weight_min": -14.319818496704102,
      "activations/layer10_attention_weight_max": 36.425724029541016,
      "activations/layer10_attention_weight_min": -34.435791015625,
      "activations/layer11_attention_weight_max": 31.110980987548828,
      "activations/layer11_attention_weight_min": -29.618154525756836,
      "activations/layer12_attention_weight_max": 18.273033142089844,
      "activations/layer12_attention_weight_min": -25.401681900024414,
      "activations/layer13_attention_weight_max": 44.495967864990234,
      "activations/layer13_attention_weight_min": -30.88182258605957,
      "activations/layer14_attention_weight_max": 37.377662658691406,
      "activations/layer14_attention_weight_min": -31.58672523498535,
      "activations/layer15_attention_weight_max": 34.82645797729492,
      "activations/layer15_attention_weight_min": -30.41744041442871,
      "activations/layer16_attention_weight_max": 30.064491271972656,
      "activations/layer16_attention_weight_min": -28.699047088623047,
      "activations/layer17_attention_weight_max": 52.57307815551758,
      "activations/layer17_attention_weight_min": -44.188819885253906,
      "activations/layer18_attention_weight_max": 47.45767593383789,
      "activations/layer18_attention_weight_min": -38.007389068603516,
      "activations/layer19_attention_weight_max": 21.55901336669922,
      "activations/layer19_attention_weight_min": -21.0357723236084,
      "activations/layer1_attention_weight_max": 16.299625396728516,
      "activations/layer1_attention_weight_min": -15.242136001586914,
      "activations/layer20_attention_weight_max": 21.014036178588867,
      "activations/layer20_attention_weight_min": -21.405912399291992,
      "activations/layer21_attention_weight_max": 39.3057746887207,
      "activations/layer21_attention_weight_min": -24.52939224243164,
      "activations/layer22_attention_weight_max": 31.624414443969727,
      "activations/layer22_attention_weight_min": -25.58306884765625,
      "activations/layer23_attention_weight_max": 35.23383331298828,
      "activations/layer23_attention_weight_min": -21.980052947998047,
      "activations/layer2_attention_weight_max": 32.25335693359375,
      "activations/layer2_attention_weight_min": -29.448986053466797,
      "activations/layer3_attention_weight_max": 89.21856689453125,
      "activations/layer3_attention_weight_min": -89.48392486572266,
      "activations/layer4_attention_weight_max": 98.88094329833984,
      "activations/layer4_attention_weight_min": -97.83364868164062,
      "activations/layer5_attention_weight_max": 86.34584045410156,
      "activations/layer5_attention_weight_min": -82.92854309082031,
      "activations/layer6_attention_weight_max": 55.70990753173828,
      "activations/layer6_attention_weight_min": -53.060874938964844,
      "activations/layer7_attention_weight_max": 82.84566497802734,
      "activations/layer7_attention_weight_min": -75.05497741699219,
      "activations/layer8_attention_weight_max": 45.30156326293945,
      "activations/layer8_attention_weight_min": -45.861392974853516,
      "activations/layer9_attention_weight_max": 55.59109115600586,
      "activations/layer9_attention_weight_min": -48.07500457763672,
      "epoch": 7.67,
      "learning_rate": 0.00010154242424242424,
      "loss": 2.8216,
      "step": 132050
    },
    {
      "activations/layer0_attention_weight_max": 15.303240776062012,
      "activations/layer0_attention_weight_min": -13.990535736083984,
      "activations/layer10_attention_weight_max": 36.935970306396484,
      "activations/layer10_attention_weight_min": -34.33381652832031,
      "activations/layer11_attention_weight_max": 30.36009979248047,
      "activations/layer11_attention_weight_min": -32.804962158203125,
      "activations/layer12_attention_weight_max": 24.28009796142578,
      "activations/layer12_attention_weight_min": -23.92028045654297,
      "activations/layer13_attention_weight_max": 45.92479705810547,
      "activations/layer13_attention_weight_min": -30.922836303710938,
      "activations/layer14_attention_weight_max": 37.99314498901367,
      "activations/layer14_attention_weight_min": -29.642627716064453,
      "activations/layer15_attention_weight_max": 36.695411682128906,
      "activations/layer15_attention_weight_min": -29.945533752441406,
      "activations/layer16_attention_weight_max": 30.110918045043945,
      "activations/layer16_attention_weight_min": -27.109737396240234,
      "activations/layer17_attention_weight_max": 52.319000244140625,
      "activations/layer17_attention_weight_min": -41.42355728149414,
      "activations/layer18_attention_weight_max": 46.23323440551758,
      "activations/layer18_attention_weight_min": -35.26757049560547,
      "activations/layer19_attention_weight_max": 21.793609619140625,
      "activations/layer19_attention_weight_min": -20.545114517211914,
      "activations/layer1_attention_weight_max": 16.131755828857422,
      "activations/layer1_attention_weight_min": -14.850702285766602,
      "activations/layer20_attention_weight_max": 22.769107818603516,
      "activations/layer20_attention_weight_min": -21.613895416259766,
      "activations/layer21_attention_weight_max": 38.66096496582031,
      "activations/layer21_attention_weight_min": -24.309707641601562,
      "activations/layer22_attention_weight_max": 33.604923248291016,
      "activations/layer22_attention_weight_min": -26.222396850585938,
      "activations/layer23_attention_weight_max": 33.19879913330078,
      "activations/layer23_attention_weight_min": -20.862701416015625,
      "activations/layer2_attention_weight_max": 33.74445724487305,
      "activations/layer2_attention_weight_min": -32.742774963378906,
      "activations/layer3_attention_weight_max": 87.87345123291016,
      "activations/layer3_attention_weight_min": -92.65043640136719,
      "activations/layer4_attention_weight_max": 98.68512725830078,
      "activations/layer4_attention_weight_min": -91.88302612304688,
      "activations/layer5_attention_weight_max": 88.1570053100586,
      "activations/layer5_attention_weight_min": -79.68364715576172,
      "activations/layer6_attention_weight_max": 56.554649353027344,
      "activations/layer6_attention_weight_min": -54.56033706665039,
      "activations/layer7_attention_weight_max": 73.86054992675781,
      "activations/layer7_attention_weight_min": -71.10970306396484,
      "activations/layer8_attention_weight_max": 44.924346923828125,
      "activations/layer8_attention_weight_min": -46.28375244140625,
      "activations/layer9_attention_weight_max": 51.22188186645508,
      "activations/layer9_attention_weight_min": -46.30132293701172,
      "epoch": 7.68,
      "learning_rate": 0.00010152348484848483,
      "loss": 2.8147,
      "step": 132100
    },
    {
      "activations/layer0_attention_weight_max": 15.585240364074707,
      "activations/layer0_attention_weight_min": -14.775238037109375,
      "activations/layer10_attention_weight_max": 41.06304168701172,
      "activations/layer10_attention_weight_min": -36.73202896118164,
      "activations/layer11_attention_weight_max": 32.299705505371094,
      "activations/layer11_attention_weight_min": -32.28502655029297,
      "activations/layer12_attention_weight_max": 19.286144256591797,
      "activations/layer12_attention_weight_min": -25.25727081298828,
      "activations/layer13_attention_weight_max": 46.02873611450195,
      "activations/layer13_attention_weight_min": -34.75819778442383,
      "activations/layer14_attention_weight_max": 38.099552154541016,
      "activations/layer14_attention_weight_min": -31.36774253845215,
      "activations/layer15_attention_weight_max": 37.55739212036133,
      "activations/layer15_attention_weight_min": -30.589445114135742,
      "activations/layer16_attention_weight_max": 34.06619644165039,
      "activations/layer16_attention_weight_min": -28.245479583740234,
      "activations/layer17_attention_weight_max": 52.51439666748047,
      "activations/layer17_attention_weight_min": -43.069190979003906,
      "activations/layer18_attention_weight_max": 47.442726135253906,
      "activations/layer18_attention_weight_min": -37.525672912597656,
      "activations/layer19_attention_weight_max": 24.730052947998047,
      "activations/layer19_attention_weight_min": -20.93783950805664,
      "activations/layer1_attention_weight_max": 16.683486938476562,
      "activations/layer1_attention_weight_min": -14.760077476501465,
      "activations/layer20_attention_weight_max": 22.277204513549805,
      "activations/layer20_attention_weight_min": -20.08850860595703,
      "activations/layer21_attention_weight_max": 31.88100242614746,
      "activations/layer21_attention_weight_min": -25.29733657836914,
      "activations/layer22_attention_weight_max": 34.480552673339844,
      "activations/layer22_attention_weight_min": -23.90021324157715,
      "activations/layer23_attention_weight_max": 38.1252555847168,
      "activations/layer23_attention_weight_min": -20.19937515258789,
      "activations/layer2_attention_weight_max": 30.491455078125,
      "activations/layer2_attention_weight_min": -28.742210388183594,
      "activations/layer3_attention_weight_max": 85.05425262451172,
      "activations/layer3_attention_weight_min": -85.34175872802734,
      "activations/layer4_attention_weight_max": 99.5579833984375,
      "activations/layer4_attention_weight_min": -95.22673797607422,
      "activations/layer5_attention_weight_max": 86.99386596679688,
      "activations/layer5_attention_weight_min": -81.74127960205078,
      "activations/layer6_attention_weight_max": 54.68042755126953,
      "activations/layer6_attention_weight_min": -53.437808990478516,
      "activations/layer7_attention_weight_max": 77.2442626953125,
      "activations/layer7_attention_weight_min": -73.12315368652344,
      "activations/layer8_attention_weight_max": 45.08120346069336,
      "activations/layer8_attention_weight_min": -45.75660705566406,
      "activations/layer9_attention_weight_max": 53.478729248046875,
      "activations/layer9_attention_weight_min": -48.3625602722168,
      "epoch": 7.68,
      "learning_rate": 0.00010150454545454545,
      "loss": 2.8146,
      "step": 132150
    },
    {
      "activations/layer0_attention_weight_max": 14.997268676757812,
      "activations/layer0_attention_weight_min": -14.534574508666992,
      "activations/layer10_attention_weight_max": 36.5901985168457,
      "activations/layer10_attention_weight_min": -34.95288848876953,
      "activations/layer11_attention_weight_max": 30.471179962158203,
      "activations/layer11_attention_weight_min": -31.131771087646484,
      "activations/layer12_attention_weight_max": 23.099166870117188,
      "activations/layer12_attention_weight_min": -23.97466278076172,
      "activations/layer13_attention_weight_max": 57.88323211669922,
      "activations/layer13_attention_weight_min": -39.79353713989258,
      "activations/layer14_attention_weight_max": 39.422637939453125,
      "activations/layer14_attention_weight_min": -31.158098220825195,
      "activations/layer15_attention_weight_max": 36.1386833190918,
      "activations/layer15_attention_weight_min": -29.4678955078125,
      "activations/layer16_attention_weight_max": 28.860506057739258,
      "activations/layer16_attention_weight_min": -27.075185775756836,
      "activations/layer17_attention_weight_max": 52.0521125793457,
      "activations/layer17_attention_weight_min": -44.388816833496094,
      "activations/layer18_attention_weight_max": 47.08950424194336,
      "activations/layer18_attention_weight_min": -35.59156036376953,
      "activations/layer19_attention_weight_max": 19.349098205566406,
      "activations/layer19_attention_weight_min": -19.76655387878418,
      "activations/layer1_attention_weight_max": 16.36239242553711,
      "activations/layer1_attention_weight_min": -14.02213191986084,
      "activations/layer20_attention_weight_max": 21.90756607055664,
      "activations/layer20_attention_weight_min": -21.57465171813965,
      "activations/layer21_attention_weight_max": 37.7094841003418,
      "activations/layer21_attention_weight_min": -25.16975975036621,
      "activations/layer22_attention_weight_max": 32.56645584106445,
      "activations/layer22_attention_weight_min": -24.307113647460938,
      "activations/layer23_attention_weight_max": 32.13166809082031,
      "activations/layer23_attention_weight_min": -23.943517684936523,
      "activations/layer2_attention_weight_max": 31.9491024017334,
      "activations/layer2_attention_weight_min": -29.739776611328125,
      "activations/layer3_attention_weight_max": 92.45381927490234,
      "activations/layer3_attention_weight_min": -88.76901245117188,
      "activations/layer4_attention_weight_max": 103.3185806274414,
      "activations/layer4_attention_weight_min": -96.44339752197266,
      "activations/layer5_attention_weight_max": 91.32037353515625,
      "activations/layer5_attention_weight_min": -81.20623779296875,
      "activations/layer6_attention_weight_max": 56.45499801635742,
      "activations/layer6_attention_weight_min": -55.8718376159668,
      "activations/layer7_attention_weight_max": 78.20265197753906,
      "activations/layer7_attention_weight_min": -75.65904235839844,
      "activations/layer8_attention_weight_max": 46.00971984863281,
      "activations/layer8_attention_weight_min": -45.09800338745117,
      "activations/layer9_attention_weight_max": 49.1146354675293,
      "activations/layer9_attention_weight_min": -48.62055587768555,
      "epoch": 7.68,
      "learning_rate": 0.00010148560606060606,
      "loss": 2.801,
      "step": 132200
    },
    {
      "activations/layer0_attention_weight_max": 15.803128242492676,
      "activations/layer0_attention_weight_min": -14.528851509094238,
      "activations/layer10_attention_weight_max": 37.10767364501953,
      "activations/layer10_attention_weight_min": -34.67497253417969,
      "activations/layer11_attention_weight_max": 29.39368438720703,
      "activations/layer11_attention_weight_min": -30.51318359375,
      "activations/layer12_attention_weight_max": 21.99729347229004,
      "activations/layer12_attention_weight_min": -25.80030632019043,
      "activations/layer13_attention_weight_max": 43.49201202392578,
      "activations/layer13_attention_weight_min": -32.323883056640625,
      "activations/layer14_attention_weight_max": 41.03870391845703,
      "activations/layer14_attention_weight_min": -31.081989288330078,
      "activations/layer15_attention_weight_max": 35.121917724609375,
      "activations/layer15_attention_weight_min": -30.74244499206543,
      "activations/layer16_attention_weight_max": 32.311866760253906,
      "activations/layer16_attention_weight_min": -26.7479190826416,
      "activations/layer17_attention_weight_max": 53.70072555541992,
      "activations/layer17_attention_weight_min": -40.85664367675781,
      "activations/layer18_attention_weight_max": 46.6768913269043,
      "activations/layer18_attention_weight_min": -34.952880859375,
      "activations/layer19_attention_weight_max": 20.782556533813477,
      "activations/layer19_attention_weight_min": -20.2333984375,
      "activations/layer1_attention_weight_max": 17.262907028198242,
      "activations/layer1_attention_weight_min": -16.544677734375,
      "activations/layer20_attention_weight_max": 22.206727981567383,
      "activations/layer20_attention_weight_min": -21.18634033203125,
      "activations/layer21_attention_weight_max": 34.75306701660156,
      "activations/layer21_attention_weight_min": -22.732099533081055,
      "activations/layer22_attention_weight_max": 34.23493576049805,
      "activations/layer22_attention_weight_min": -24.59322166442871,
      "activations/layer23_attention_weight_max": 37.001888275146484,
      "activations/layer23_attention_weight_min": -22.90047836303711,
      "activations/layer2_attention_weight_max": 32.041748046875,
      "activations/layer2_attention_weight_min": -29.1903133392334,
      "activations/layer3_attention_weight_max": 82.48250579833984,
      "activations/layer3_attention_weight_min": -84.05128479003906,
      "activations/layer4_attention_weight_max": 98.67229461669922,
      "activations/layer4_attention_weight_min": -90.54150390625,
      "activations/layer5_attention_weight_max": 85.9742431640625,
      "activations/layer5_attention_weight_min": -77.84141540527344,
      "activations/layer6_attention_weight_max": 52.44763946533203,
      "activations/layer6_attention_weight_min": -51.207374572753906,
      "activations/layer7_attention_weight_max": 73.64144134521484,
      "activations/layer7_attention_weight_min": -72.84966278076172,
      "activations/layer8_attention_weight_max": 43.49384307861328,
      "activations/layer8_attention_weight_min": -44.3295783996582,
      "activations/layer9_attention_weight_max": 48.58547592163086,
      "activations/layer9_attention_weight_min": -46.473785400390625,
      "epoch": 7.68,
      "learning_rate": 0.00010146666666666665,
      "loss": 2.8165,
      "step": 132250
    },
    {
      "activations/layer0_attention_weight_max": 15.368961334228516,
      "activations/layer0_attention_weight_min": -14.739578247070312,
      "activations/layer10_attention_weight_max": 36.43925857543945,
      "activations/layer10_attention_weight_min": -32.052032470703125,
      "activations/layer11_attention_weight_max": 30.05889129638672,
      "activations/layer11_attention_weight_min": -29.637189865112305,
      "activations/layer12_attention_weight_max": 18.063777923583984,
      "activations/layer12_attention_weight_min": -23.725000381469727,
      "activations/layer13_attention_weight_max": 35.92134094238281,
      "activations/layer13_attention_weight_min": -31.67020606994629,
      "activations/layer14_attention_weight_max": 35.941375732421875,
      "activations/layer14_attention_weight_min": -30.810800552368164,
      "activations/layer15_attention_weight_max": 32.19488525390625,
      "activations/layer15_attention_weight_min": -29.35841178894043,
      "activations/layer16_attention_weight_max": 27.876873016357422,
      "activations/layer16_attention_weight_min": -25.98951530456543,
      "activations/layer17_attention_weight_max": 50.73393249511719,
      "activations/layer17_attention_weight_min": -43.627220153808594,
      "activations/layer18_attention_weight_max": 44.78852462768555,
      "activations/layer18_attention_weight_min": -35.97446823120117,
      "activations/layer19_attention_weight_max": 19.197811126708984,
      "activations/layer19_attention_weight_min": -19.27924156188965,
      "activations/layer1_attention_weight_max": 16.27157974243164,
      "activations/layer1_attention_weight_min": -13.662802696228027,
      "activations/layer20_attention_weight_max": 21.106700897216797,
      "activations/layer20_attention_weight_min": -22.93061065673828,
      "activations/layer21_attention_weight_max": 34.326908111572266,
      "activations/layer21_attention_weight_min": -22.51776695251465,
      "activations/layer22_attention_weight_max": 29.59615135192871,
      "activations/layer22_attention_weight_min": -23.557695388793945,
      "activations/layer23_attention_weight_max": 32.458927154541016,
      "activations/layer23_attention_weight_min": -21.775184631347656,
      "activations/layer2_attention_weight_max": 29.47500991821289,
      "activations/layer2_attention_weight_min": -27.688039779663086,
      "activations/layer3_attention_weight_max": 81.57209014892578,
      "activations/layer3_attention_weight_min": -88.00022888183594,
      "activations/layer4_attention_weight_max": 95.86089324951172,
      "activations/layer4_attention_weight_min": -91.74750518798828,
      "activations/layer5_attention_weight_max": 87.20234680175781,
      "activations/layer5_attention_weight_min": -79.98997497558594,
      "activations/layer6_attention_weight_max": 53.40553283691406,
      "activations/layer6_attention_weight_min": -50.132320404052734,
      "activations/layer7_attention_weight_max": 72.2827377319336,
      "activations/layer7_attention_weight_min": -73.08804321289062,
      "activations/layer8_attention_weight_max": 42.8476448059082,
      "activations/layer8_attention_weight_min": -45.4683837890625,
      "activations/layer9_attention_weight_max": 49.39738845825195,
      "activations/layer9_attention_weight_min": -46.32236862182617,
      "epoch": 7.69,
      "learning_rate": 0.00010144772727272726,
      "loss": 2.8196,
      "step": 132300
    },
    {
      "activations/layer0_attention_weight_max": 14.839032173156738,
      "activations/layer0_attention_weight_min": -14.18338394165039,
      "activations/layer10_attention_weight_max": 38.33045196533203,
      "activations/layer10_attention_weight_min": -36.48073196411133,
      "activations/layer11_attention_weight_max": 32.45731735229492,
      "activations/layer11_attention_weight_min": -31.71953582763672,
      "activations/layer12_attention_weight_max": 18.875268936157227,
      "activations/layer12_attention_weight_min": -25.39277458190918,
      "activations/layer13_attention_weight_max": 47.38679122924805,
      "activations/layer13_attention_weight_min": -35.83131408691406,
      "activations/layer14_attention_weight_max": 53.315589904785156,
      "activations/layer14_attention_weight_min": -44.090415954589844,
      "activations/layer15_attention_weight_max": 42.71337127685547,
      "activations/layer15_attention_weight_min": -35.87788391113281,
      "activations/layer16_attention_weight_max": 32.326778411865234,
      "activations/layer16_attention_weight_min": -27.213977813720703,
      "activations/layer17_attention_weight_max": 56.738685607910156,
      "activations/layer17_attention_weight_min": -42.70228958129883,
      "activations/layer18_attention_weight_max": 48.315284729003906,
      "activations/layer18_attention_weight_min": -36.17402267456055,
      "activations/layer19_attention_weight_max": 22.048799514770508,
      "activations/layer19_attention_weight_min": -20.73894691467285,
      "activations/layer1_attention_weight_max": 16.98349380493164,
      "activations/layer1_attention_weight_min": -15.0821533203125,
      "activations/layer20_attention_weight_max": 24.79460906982422,
      "activations/layer20_attention_weight_min": -20.539043426513672,
      "activations/layer21_attention_weight_max": 38.5467529296875,
      "activations/layer21_attention_weight_min": -25.280315399169922,
      "activations/layer22_attention_weight_max": 30.821401596069336,
      "activations/layer22_attention_weight_min": -25.523408889770508,
      "activations/layer23_attention_weight_max": 34.907562255859375,
      "activations/layer23_attention_weight_min": -22.596675872802734,
      "activations/layer2_attention_weight_max": 30.499935150146484,
      "activations/layer2_attention_weight_min": -30.20165252685547,
      "activations/layer3_attention_weight_max": 87.97265625,
      "activations/layer3_attention_weight_min": -90.16194915771484,
      "activations/layer4_attention_weight_max": 102.23148345947266,
      "activations/layer4_attention_weight_min": -98.6858139038086,
      "activations/layer5_attention_weight_max": 88.85332489013672,
      "activations/layer5_attention_weight_min": -81.12443542480469,
      "activations/layer6_attention_weight_max": 55.876495361328125,
      "activations/layer6_attention_weight_min": -54.920684814453125,
      "activations/layer7_attention_weight_max": 76.12940216064453,
      "activations/layer7_attention_weight_min": -73.46912384033203,
      "activations/layer8_attention_weight_max": 50.06819534301758,
      "activations/layer8_attention_weight_min": -45.81547927856445,
      "activations/layer9_attention_weight_max": 52.85334396362305,
      "activations/layer9_attention_weight_min": -49.99997329711914,
      "epoch": 7.69,
      "learning_rate": 0.00010142878787878788,
      "loss": 2.8326,
      "step": 132350
    },
    {
      "activations/layer0_attention_weight_max": 15.24423885345459,
      "activations/layer0_attention_weight_min": -14.559629440307617,
      "activations/layer10_attention_weight_max": 35.568992614746094,
      "activations/layer10_attention_weight_min": -34.31584930419922,
      "activations/layer11_attention_weight_max": 29.34510040283203,
      "activations/layer11_attention_weight_min": -30.83617401123047,
      "activations/layer12_attention_weight_max": 19.93556785583496,
      "activations/layer12_attention_weight_min": -27.151426315307617,
      "activations/layer13_attention_weight_max": 42.51328659057617,
      "activations/layer13_attention_weight_min": -31.21493911743164,
      "activations/layer14_attention_weight_max": 40.558677673339844,
      "activations/layer14_attention_weight_min": -31.89599609375,
      "activations/layer15_attention_weight_max": 38.0150146484375,
      "activations/layer15_attention_weight_min": -29.811281204223633,
      "activations/layer16_attention_weight_max": 34.06464385986328,
      "activations/layer16_attention_weight_min": -30.551958084106445,
      "activations/layer17_attention_weight_max": 55.179107666015625,
      "activations/layer17_attention_weight_min": -44.83879470825195,
      "activations/layer18_attention_weight_max": 54.568817138671875,
      "activations/layer18_attention_weight_min": -36.930023193359375,
      "activations/layer19_attention_weight_max": 24.7492733001709,
      "activations/layer19_attention_weight_min": -19.883865356445312,
      "activations/layer1_attention_weight_max": 17.856060028076172,
      "activations/layer1_attention_weight_min": -17.517309188842773,
      "activations/layer20_attention_weight_max": 25.176528930664062,
      "activations/layer20_attention_weight_min": -22.924278259277344,
      "activations/layer21_attention_weight_max": 48.47196578979492,
      "activations/layer21_attention_weight_min": -22.982807159423828,
      "activations/layer22_attention_weight_max": 34.161617279052734,
      "activations/layer22_attention_weight_min": -25.851642608642578,
      "activations/layer23_attention_weight_max": 37.251930236816406,
      "activations/layer23_attention_weight_min": -22.226964950561523,
      "activations/layer2_attention_weight_max": 29.389545440673828,
      "activations/layer2_attention_weight_min": -28.146198272705078,
      "activations/layer3_attention_weight_max": 85.64543151855469,
      "activations/layer3_attention_weight_min": -86.1799087524414,
      "activations/layer4_attention_weight_max": 100.62461853027344,
      "activations/layer4_attention_weight_min": -93.0406265258789,
      "activations/layer5_attention_weight_max": 86.62374114990234,
      "activations/layer5_attention_weight_min": -82.692626953125,
      "activations/layer6_attention_weight_max": 54.90815734863281,
      "activations/layer6_attention_weight_min": -53.15897750854492,
      "activations/layer7_attention_weight_max": 79.29081726074219,
      "activations/layer7_attention_weight_min": -81.37491607666016,
      "activations/layer8_attention_weight_max": 43.12527084350586,
      "activations/layer8_attention_weight_min": -45.66643524169922,
      "activations/layer9_attention_weight_max": 48.668128967285156,
      "activations/layer9_attention_weight_min": -48.75908660888672,
      "epoch": 7.69,
      "learning_rate": 0.00010140984848484847,
      "loss": 2.8059,
      "step": 132400
    },
    {
      "activations/layer0_attention_weight_max": 14.261214256286621,
      "activations/layer0_attention_weight_min": -14.5852689743042,
      "activations/layer10_attention_weight_max": 38.57421875,
      "activations/layer10_attention_weight_min": -33.99496078491211,
      "activations/layer11_attention_weight_max": 31.471847534179688,
      "activations/layer11_attention_weight_min": -31.187801361083984,
      "activations/layer12_attention_weight_max": 21.44464111328125,
      "activations/layer12_attention_weight_min": -24.480871200561523,
      "activations/layer13_attention_weight_max": 40.63485336303711,
      "activations/layer13_attention_weight_min": -30.055946350097656,
      "activations/layer14_attention_weight_max": 37.12411117553711,
      "activations/layer14_attention_weight_min": -31.93264389038086,
      "activations/layer15_attention_weight_max": 33.29393768310547,
      "activations/layer15_attention_weight_min": -30.115041732788086,
      "activations/layer16_attention_weight_max": 31.367015838623047,
      "activations/layer16_attention_weight_min": -27.864683151245117,
      "activations/layer17_attention_weight_max": 55.355403900146484,
      "activations/layer17_attention_weight_min": -42.84308624267578,
      "activations/layer18_attention_weight_max": 46.915348052978516,
      "activations/layer18_attention_weight_min": -36.59713363647461,
      "activations/layer19_attention_weight_max": 20.682846069335938,
      "activations/layer19_attention_weight_min": -20.511884689331055,
      "activations/layer1_attention_weight_max": 17.3026180267334,
      "activations/layer1_attention_weight_min": -16.03282928466797,
      "activations/layer20_attention_weight_max": 21.838016510009766,
      "activations/layer20_attention_weight_min": -20.759977340698242,
      "activations/layer21_attention_weight_max": 34.13995361328125,
      "activations/layer21_attention_weight_min": -23.75011444091797,
      "activations/layer22_attention_weight_max": 32.688655853271484,
      "activations/layer22_attention_weight_min": -25.37587547302246,
      "activations/layer23_attention_weight_max": 33.64565658569336,
      "activations/layer23_attention_weight_min": -20.5888671875,
      "activations/layer2_attention_weight_max": 32.071563720703125,
      "activations/layer2_attention_weight_min": -28.441654205322266,
      "activations/layer3_attention_weight_max": 85.735107421875,
      "activations/layer3_attention_weight_min": -87.51496887207031,
      "activations/layer4_attention_weight_max": 99.72704315185547,
      "activations/layer4_attention_weight_min": -94.44900512695312,
      "activations/layer5_attention_weight_max": 87.4898681640625,
      "activations/layer5_attention_weight_min": -82.42733764648438,
      "activations/layer6_attention_weight_max": 54.94318771362305,
      "activations/layer6_attention_weight_min": -52.80327606201172,
      "activations/layer7_attention_weight_max": 77.08270263671875,
      "activations/layer7_attention_weight_min": -69.53726959228516,
      "activations/layer8_attention_weight_max": 42.7604866027832,
      "activations/layer8_attention_weight_min": -43.18925094604492,
      "activations/layer9_attention_weight_max": 47.48651885986328,
      "activations/layer9_attention_weight_min": -46.181941986083984,
      "epoch": 7.7,
      "learning_rate": 0.00010139090909090908,
      "loss": 2.8175,
      "step": 132450
    },
    {
      "activations/layer0_attention_weight_max": 14.598710060119629,
      "activations/layer0_attention_weight_min": -14.529370307922363,
      "activations/layer10_attention_weight_max": 38.097267150878906,
      "activations/layer10_attention_weight_min": -37.31953811645508,
      "activations/layer11_attention_weight_max": 34.718849182128906,
      "activations/layer11_attention_weight_min": -31.548742294311523,
      "activations/layer12_attention_weight_max": 21.459091186523438,
      "activations/layer12_attention_weight_min": -33.272056579589844,
      "activations/layer13_attention_weight_max": 34.92621994018555,
      "activations/layer13_attention_weight_min": -28.692689895629883,
      "activations/layer14_attention_weight_max": 36.42974090576172,
      "activations/layer14_attention_weight_min": -29.487558364868164,
      "activations/layer15_attention_weight_max": 31.638818740844727,
      "activations/layer15_attention_weight_min": -28.487043380737305,
      "activations/layer16_attention_weight_max": 30.492671966552734,
      "activations/layer16_attention_weight_min": -26.081266403198242,
      "activations/layer17_attention_weight_max": 51.05986022949219,
      "activations/layer17_attention_weight_min": -42.36685562133789,
      "activations/layer18_attention_weight_max": 43.36205291748047,
      "activations/layer18_attention_weight_min": -36.42942810058594,
      "activations/layer19_attention_weight_max": 21.423391342163086,
      "activations/layer19_attention_weight_min": -21.311880111694336,
      "activations/layer1_attention_weight_max": 17.873106002807617,
      "activations/layer1_attention_weight_min": -15.167821884155273,
      "activations/layer20_attention_weight_max": 21.40500259399414,
      "activations/layer20_attention_weight_min": -21.267820358276367,
      "activations/layer21_attention_weight_max": 39.67285919189453,
      "activations/layer21_attention_weight_min": -27.325942993164062,
      "activations/layer22_attention_weight_max": 30.86160659790039,
      "activations/layer22_attention_weight_min": -27.24599266052246,
      "activations/layer23_attention_weight_max": 30.099937438964844,
      "activations/layer23_attention_weight_min": -19.00766372680664,
      "activations/layer2_attention_weight_max": 33.0871696472168,
      "activations/layer2_attention_weight_min": -31.532608032226562,
      "activations/layer3_attention_weight_max": 91.69444274902344,
      "activations/layer3_attention_weight_min": -91.46273803710938,
      "activations/layer4_attention_weight_max": 101.04646301269531,
      "activations/layer4_attention_weight_min": -97.28919982910156,
      "activations/layer5_attention_weight_max": 86.80946350097656,
      "activations/layer5_attention_weight_min": -81.54698181152344,
      "activations/layer6_attention_weight_max": 53.99463653564453,
      "activations/layer6_attention_weight_min": -52.3546028137207,
      "activations/layer7_attention_weight_max": 85.57806396484375,
      "activations/layer7_attention_weight_min": -75.80712127685547,
      "activations/layer8_attention_weight_max": 47.99462890625,
      "activations/layer8_attention_weight_min": -49.26741027832031,
      "activations/layer9_attention_weight_max": 67.44732666015625,
      "activations/layer9_attention_weight_min": -54.062225341796875,
      "epoch": 7.7,
      "learning_rate": 0.0001013719696969697,
      "loss": 2.8098,
      "step": 132500
    },
    {
      "activations/layer0_attention_weight_max": 14.75019359588623,
      "activations/layer0_attention_weight_min": -14.525165557861328,
      "activations/layer10_attention_weight_max": 36.59703063964844,
      "activations/layer10_attention_weight_min": -36.272804260253906,
      "activations/layer11_attention_weight_max": 31.68158531188965,
      "activations/layer11_attention_weight_min": -31.49744987487793,
      "activations/layer12_attention_weight_max": 20.701955795288086,
      "activations/layer12_attention_weight_min": -29.566890716552734,
      "activations/layer13_attention_weight_max": 48.629940032958984,
      "activations/layer13_attention_weight_min": -35.591331481933594,
      "activations/layer14_attention_weight_max": 44.58169174194336,
      "activations/layer14_attention_weight_min": -33.8314208984375,
      "activations/layer15_attention_weight_max": 37.12725067138672,
      "activations/layer15_attention_weight_min": -31.28178596496582,
      "activations/layer16_attention_weight_max": 36.363792419433594,
      "activations/layer16_attention_weight_min": -29.01079750061035,
      "activations/layer17_attention_weight_max": 54.647220611572266,
      "activations/layer17_attention_weight_min": -44.46488571166992,
      "activations/layer18_attention_weight_max": 51.65287780761719,
      "activations/layer18_attention_weight_min": -37.3415641784668,
      "activations/layer19_attention_weight_max": 21.702468872070312,
      "activations/layer19_attention_weight_min": -22.244718551635742,
      "activations/layer1_attention_weight_max": 16.37428092956543,
      "activations/layer1_attention_weight_min": -14.434075355529785,
      "activations/layer20_attention_weight_max": 24.48505210876465,
      "activations/layer20_attention_weight_min": -22.921049118041992,
      "activations/layer21_attention_weight_max": 40.51588821411133,
      "activations/layer21_attention_weight_min": -24.467477798461914,
      "activations/layer22_attention_weight_max": 36.99510192871094,
      "activations/layer22_attention_weight_min": -27.66913604736328,
      "activations/layer23_attention_weight_max": 39.260658264160156,
      "activations/layer23_attention_weight_min": -21.659191131591797,
      "activations/layer2_attention_weight_max": 32.04754638671875,
      "activations/layer2_attention_weight_min": -31.034324645996094,
      "activations/layer3_attention_weight_max": 85.33885955810547,
      "activations/layer3_attention_weight_min": -86.34337615966797,
      "activations/layer4_attention_weight_max": 101.54883575439453,
      "activations/layer4_attention_weight_min": -92.01314544677734,
      "activations/layer5_attention_weight_max": 87.50725555419922,
      "activations/layer5_attention_weight_min": -78.03492736816406,
      "activations/layer6_attention_weight_max": 55.163818359375,
      "activations/layer6_attention_weight_min": -52.78816223144531,
      "activations/layer7_attention_weight_max": 83.73651885986328,
      "activations/layer7_attention_weight_min": -73.11087799072266,
      "activations/layer8_attention_weight_max": 45.46785354614258,
      "activations/layer8_attention_weight_min": -44.620113372802734,
      "activations/layer9_attention_weight_max": 50.84452438354492,
      "activations/layer9_attention_weight_min": -48.03599166870117,
      "epoch": 7.7,
      "learning_rate": 0.0001013530303030303,
      "loss": 2.8188,
      "step": 132550
    },
    {
      "activations/layer0_attention_weight_max": 15.076494216918945,
      "activations/layer0_attention_weight_min": -14.79666519165039,
      "activations/layer10_attention_weight_max": 35.26643371582031,
      "activations/layer10_attention_weight_min": -34.66523361206055,
      "activations/layer11_attention_weight_max": 30.30874252319336,
      "activations/layer11_attention_weight_min": -30.097900390625,
      "activations/layer12_attention_weight_max": 19.95561408996582,
      "activations/layer12_attention_weight_min": -25.239337921142578,
      "activations/layer13_attention_weight_max": 46.19515609741211,
      "activations/layer13_attention_weight_min": -32.26422882080078,
      "activations/layer14_attention_weight_max": 41.17702102661133,
      "activations/layer14_attention_weight_min": -30.963529586791992,
      "activations/layer15_attention_weight_max": 34.59388732910156,
      "activations/layer15_attention_weight_min": -29.15421485900879,
      "activations/layer16_attention_weight_max": 36.120304107666016,
      "activations/layer16_attention_weight_min": -27.95060920715332,
      "activations/layer17_attention_weight_max": 53.712974548339844,
      "activations/layer17_attention_weight_min": -42.55675506591797,
      "activations/layer18_attention_weight_max": 46.03050994873047,
      "activations/layer18_attention_weight_min": -35.73346710205078,
      "activations/layer19_attention_weight_max": 21.939353942871094,
      "activations/layer19_attention_weight_min": -21.431488037109375,
      "activations/layer1_attention_weight_max": 17.498044967651367,
      "activations/layer1_attention_weight_min": -13.646148681640625,
      "activations/layer20_attention_weight_max": 22.24723243713379,
      "activations/layer20_attention_weight_min": -22.825021743774414,
      "activations/layer21_attention_weight_max": 42.605045318603516,
      "activations/layer21_attention_weight_min": -24.105199813842773,
      "activations/layer22_attention_weight_max": 34.75555419921875,
      "activations/layer22_attention_weight_min": -24.871294021606445,
      "activations/layer23_attention_weight_max": 38.696353912353516,
      "activations/layer23_attention_weight_min": -22.557546615600586,
      "activations/layer2_attention_weight_max": 31.243629455566406,
      "activations/layer2_attention_weight_min": -29.615718841552734,
      "activations/layer3_attention_weight_max": 87.6329574584961,
      "activations/layer3_attention_weight_min": -90.76052856445312,
      "activations/layer4_attention_weight_max": 99.87561798095703,
      "activations/layer4_attention_weight_min": -92.22472381591797,
      "activations/layer5_attention_weight_max": 84.53897857666016,
      "activations/layer5_attention_weight_min": -78.786865234375,
      "activations/layer6_attention_weight_max": 52.81766128540039,
      "activations/layer6_attention_weight_min": -53.30349349975586,
      "activations/layer7_attention_weight_max": 74.33619689941406,
      "activations/layer7_attention_weight_min": -71.6952133178711,
      "activations/layer8_attention_weight_max": 42.97709655761719,
      "activations/layer8_attention_weight_min": -42.96318054199219,
      "activations/layer9_attention_weight_max": 49.6804084777832,
      "activations/layer9_attention_weight_min": -48.02858352661133,
      "epoch": 7.7,
      "learning_rate": 0.0001013340909090909,
      "loss": 2.8133,
      "step": 132600
    },
    {
      "activations/layer0_attention_weight_max": 14.355185508728027,
      "activations/layer0_attention_weight_min": -14.370885848999023,
      "activations/layer10_attention_weight_max": 36.791290283203125,
      "activations/layer10_attention_weight_min": -34.15134811401367,
      "activations/layer11_attention_weight_max": 31.320877075195312,
      "activations/layer11_attention_weight_min": -29.824676513671875,
      "activations/layer12_attention_weight_max": 21.038421630859375,
      "activations/layer12_attention_weight_min": -26.68762969970703,
      "activations/layer13_attention_weight_max": 39.969234466552734,
      "activations/layer13_attention_weight_min": -31.89996337890625,
      "activations/layer14_attention_weight_max": 34.166133880615234,
      "activations/layer14_attention_weight_min": -29.187456130981445,
      "activations/layer15_attention_weight_max": 36.6837043762207,
      "activations/layer15_attention_weight_min": -28.23089027404785,
      "activations/layer16_attention_weight_max": 30.341840744018555,
      "activations/layer16_attention_weight_min": -25.976848602294922,
      "activations/layer17_attention_weight_max": 52.5129280090332,
      "activations/layer17_attention_weight_min": -39.125972747802734,
      "activations/layer18_attention_weight_max": 43.26138687133789,
      "activations/layer18_attention_weight_min": -31.958477020263672,
      "activations/layer19_attention_weight_max": 21.32071876525879,
      "activations/layer19_attention_weight_min": -18.59330177307129,
      "activations/layer1_attention_weight_max": 17.517690658569336,
      "activations/layer1_attention_weight_min": -16.861038208007812,
      "activations/layer20_attention_weight_max": 21.472352981567383,
      "activations/layer20_attention_weight_min": -20.024152755737305,
      "activations/layer21_attention_weight_max": 32.174110412597656,
      "activations/layer21_attention_weight_min": -21.09078598022461,
      "activations/layer22_attention_weight_max": 30.781295776367188,
      "activations/layer22_attention_weight_min": -22.878887176513672,
      "activations/layer23_attention_weight_max": 31.82369613647461,
      "activations/layer23_attention_weight_min": -21.750473022460938,
      "activations/layer2_attention_weight_max": 32.048126220703125,
      "activations/layer2_attention_weight_min": -31.692310333251953,
      "activations/layer3_attention_weight_max": 84.7626953125,
      "activations/layer3_attention_weight_min": -90.42626953125,
      "activations/layer4_attention_weight_max": 99.6270523071289,
      "activations/layer4_attention_weight_min": -98.10154724121094,
      "activations/layer5_attention_weight_max": 88.84963989257812,
      "activations/layer5_attention_weight_min": -84.3720474243164,
      "activations/layer6_attention_weight_max": 56.01081848144531,
      "activations/layer6_attention_weight_min": -53.58480453491211,
      "activations/layer7_attention_weight_max": 72.73159790039062,
      "activations/layer7_attention_weight_min": -72.8907699584961,
      "activations/layer8_attention_weight_max": 46.044090270996094,
      "activations/layer8_attention_weight_min": -47.53926467895508,
      "activations/layer9_attention_weight_max": 47.68170928955078,
      "activations/layer9_attention_weight_min": -47.22626876831055,
      "epoch": 7.71,
      "learning_rate": 0.0001013151515151515,
      "loss": 2.8111,
      "step": 132650
    },
    {
      "activations/layer0_attention_weight_max": 15.279287338256836,
      "activations/layer0_attention_weight_min": -14.653200149536133,
      "activations/layer10_attention_weight_max": 39.78126907348633,
      "activations/layer10_attention_weight_min": -39.80888748168945,
      "activations/layer11_attention_weight_max": 34.423091888427734,
      "activations/layer11_attention_weight_min": -30.908071517944336,
      "activations/layer12_attention_weight_max": 20.998647689819336,
      "activations/layer12_attention_weight_min": -26.375999450683594,
      "activations/layer13_attention_weight_max": 50.53449630737305,
      "activations/layer13_attention_weight_min": -39.719966888427734,
      "activations/layer14_attention_weight_max": 54.44648742675781,
      "activations/layer14_attention_weight_min": -41.68453598022461,
      "activations/layer15_attention_weight_max": 41.65260696411133,
      "activations/layer15_attention_weight_min": -31.938865661621094,
      "activations/layer16_attention_weight_max": 34.59989547729492,
      "activations/layer16_attention_weight_min": -28.23602294921875,
      "activations/layer17_attention_weight_max": 54.133399963378906,
      "activations/layer17_attention_weight_min": -44.255165100097656,
      "activations/layer18_attention_weight_max": 47.0337028503418,
      "activations/layer18_attention_weight_min": -39.582122802734375,
      "activations/layer19_attention_weight_max": 21.436302185058594,
      "activations/layer19_attention_weight_min": -23.506000518798828,
      "activations/layer1_attention_weight_max": 17.77562713623047,
      "activations/layer1_attention_weight_min": -15.817700386047363,
      "activations/layer20_attention_weight_max": 21.582313537597656,
      "activations/layer20_attention_weight_min": -22.232152938842773,
      "activations/layer21_attention_weight_max": 39.44419860839844,
      "activations/layer21_attention_weight_min": -25.85787010192871,
      "activations/layer22_attention_weight_max": 34.26551055908203,
      "activations/layer22_attention_weight_min": -27.130109786987305,
      "activations/layer23_attention_weight_max": 37.439720153808594,
      "activations/layer23_attention_weight_min": -23.516468048095703,
      "activations/layer2_attention_weight_max": 32.033905029296875,
      "activations/layer2_attention_weight_min": -31.31804656982422,
      "activations/layer3_attention_weight_max": 91.59008026123047,
      "activations/layer3_attention_weight_min": -94.0146484375,
      "activations/layer4_attention_weight_max": 103.0427017211914,
      "activations/layer4_attention_weight_min": -97.89160919189453,
      "activations/layer5_attention_weight_max": 86.40531921386719,
      "activations/layer5_attention_weight_min": -82.37606811523438,
      "activations/layer6_attention_weight_max": 54.41618347167969,
      "activations/layer6_attention_weight_min": -51.77613067626953,
      "activations/layer7_attention_weight_max": 75.86076354980469,
      "activations/layer7_attention_weight_min": -75.10811614990234,
      "activations/layer8_attention_weight_max": 48.30330276489258,
      "activations/layer8_attention_weight_min": -48.943199157714844,
      "activations/layer9_attention_weight_max": 65.26072692871094,
      "activations/layer9_attention_weight_min": -50.83222579956055,
      "epoch": 7.71,
      "learning_rate": 0.00010129621212121212,
      "loss": 2.8146,
      "step": 132700
    },
    {
      "activations/layer0_attention_weight_max": 15.340989112854004,
      "activations/layer0_attention_weight_min": -14.4791259765625,
      "activations/layer10_attention_weight_max": 37.44221878051758,
      "activations/layer10_attention_weight_min": -33.62527084350586,
      "activations/layer11_attention_weight_max": 29.854385375976562,
      "activations/layer11_attention_weight_min": -30.943937301635742,
      "activations/layer12_attention_weight_max": 19.452484130859375,
      "activations/layer12_attention_weight_min": -23.46894645690918,
      "activations/layer13_attention_weight_max": 37.4392204284668,
      "activations/layer13_attention_weight_min": -29.450841903686523,
      "activations/layer14_attention_weight_max": 37.94513702392578,
      "activations/layer14_attention_weight_min": -30.858661651611328,
      "activations/layer15_attention_weight_max": 32.37205505371094,
      "activations/layer15_attention_weight_min": -29.801481246948242,
      "activations/layer16_attention_weight_max": 30.872270584106445,
      "activations/layer16_attention_weight_min": -27.65867805480957,
      "activations/layer17_attention_weight_max": 55.08818054199219,
      "activations/layer17_attention_weight_min": -45.52866744995117,
      "activations/layer18_attention_weight_max": 54.222225189208984,
      "activations/layer18_attention_weight_min": -41.174652099609375,
      "activations/layer19_attention_weight_max": 22.68773078918457,
      "activations/layer19_attention_weight_min": -21.483295440673828,
      "activations/layer1_attention_weight_max": 17.185258865356445,
      "activations/layer1_attention_weight_min": -14.274442672729492,
      "activations/layer20_attention_weight_max": 22.55818748474121,
      "activations/layer20_attention_weight_min": -21.734758377075195,
      "activations/layer21_attention_weight_max": 40.625587463378906,
      "activations/layer21_attention_weight_min": -23.613367080688477,
      "activations/layer22_attention_weight_max": 32.44035720825195,
      "activations/layer22_attention_weight_min": -27.373533248901367,
      "activations/layer23_attention_weight_max": 35.24581527709961,
      "activations/layer23_attention_weight_min": -23.65749740600586,
      "activations/layer2_attention_weight_max": 31.373167037963867,
      "activations/layer2_attention_weight_min": -29.33292007446289,
      "activations/layer3_attention_weight_max": 86.79571533203125,
      "activations/layer3_attention_weight_min": -90.31340789794922,
      "activations/layer4_attention_weight_max": 101.09999084472656,
      "activations/layer4_attention_weight_min": -92.53279876708984,
      "activations/layer5_attention_weight_max": 87.01898193359375,
      "activations/layer5_attention_weight_min": -80.72872161865234,
      "activations/layer6_attention_weight_max": 55.544334411621094,
      "activations/layer6_attention_weight_min": -52.403465270996094,
      "activations/layer7_attention_weight_max": 74.9723892211914,
      "activations/layer7_attention_weight_min": -71.17729949951172,
      "activations/layer8_attention_weight_max": 43.83639907836914,
      "activations/layer8_attention_weight_min": -44.603668212890625,
      "activations/layer9_attention_weight_max": 47.31525421142578,
      "activations/layer9_attention_weight_min": -49.36082458496094,
      "epoch": 7.71,
      "learning_rate": 0.00010127727272727272,
      "loss": 2.8196,
      "step": 132750
    },
    {
      "activations/layer0_attention_weight_max": 14.510661125183105,
      "activations/layer0_attention_weight_min": -14.96270751953125,
      "activations/layer10_attention_weight_max": 38.8410530090332,
      "activations/layer10_attention_weight_min": -36.15662384033203,
      "activations/layer11_attention_weight_max": 30.377790451049805,
      "activations/layer11_attention_weight_min": -29.533512115478516,
      "activations/layer12_attention_weight_max": 19.430286407470703,
      "activations/layer12_attention_weight_min": -29.052268981933594,
      "activations/layer13_attention_weight_max": 40.48400115966797,
      "activations/layer13_attention_weight_min": -31.210983276367188,
      "activations/layer14_attention_weight_max": 38.29496383666992,
      "activations/layer14_attention_weight_min": -32.248409271240234,
      "activations/layer15_attention_weight_max": 36.3253288269043,
      "activations/layer15_attention_weight_min": -28.78348159790039,
      "activations/layer16_attention_weight_max": 34.18396759033203,
      "activations/layer16_attention_weight_min": -26.749370574951172,
      "activations/layer17_attention_weight_max": 54.39936065673828,
      "activations/layer17_attention_weight_min": -42.15516662597656,
      "activations/layer18_attention_weight_max": 48.389068603515625,
      "activations/layer18_attention_weight_min": -37.164031982421875,
      "activations/layer19_attention_weight_max": 21.259092330932617,
      "activations/layer19_attention_weight_min": -21.78561019897461,
      "activations/layer1_attention_weight_max": 16.411502838134766,
      "activations/layer1_attention_weight_min": -15.160573959350586,
      "activations/layer20_attention_weight_max": 22.490175247192383,
      "activations/layer20_attention_weight_min": -21.025493621826172,
      "activations/layer21_attention_weight_max": 38.831546783447266,
      "activations/layer21_attention_weight_min": -23.33390235900879,
      "activations/layer22_attention_weight_max": 30.40379524230957,
      "activations/layer22_attention_weight_min": -24.208059310913086,
      "activations/layer23_attention_weight_max": 35.532474517822266,
      "activations/layer23_attention_weight_min": -21.09512710571289,
      "activations/layer2_attention_weight_max": 32.263771057128906,
      "activations/layer2_attention_weight_min": -29.0980281829834,
      "activations/layer3_attention_weight_max": 83.43986511230469,
      "activations/layer3_attention_weight_min": -88.28343963623047,
      "activations/layer4_attention_weight_max": 98.7618408203125,
      "activations/layer4_attention_weight_min": -101.15540313720703,
      "activations/layer5_attention_weight_max": 84.08045959472656,
      "activations/layer5_attention_weight_min": -80.90768432617188,
      "activations/layer6_attention_weight_max": 53.31757354736328,
      "activations/layer6_attention_weight_min": -54.49822235107422,
      "activations/layer7_attention_weight_max": 76.53197479248047,
      "activations/layer7_attention_weight_min": -71.4166259765625,
      "activations/layer8_attention_weight_max": 45.03657913208008,
      "activations/layer8_attention_weight_min": -49.66047668457031,
      "activations/layer9_attention_weight_max": 48.15869140625,
      "activations/layer9_attention_weight_min": -47.68892288208008,
      "epoch": 7.72,
      "learning_rate": 0.00010125833333333332,
      "loss": 2.803,
      "step": 132800
    },
    {
      "activations/layer0_attention_weight_max": 15.64553165435791,
      "activations/layer0_attention_weight_min": -14.194757461547852,
      "activations/layer10_attention_weight_max": 41.00984573364258,
      "activations/layer10_attention_weight_min": -40.2929573059082,
      "activations/layer11_attention_weight_max": 32.967220306396484,
      "activations/layer11_attention_weight_min": -33.884490966796875,
      "activations/layer12_attention_weight_max": 20.5653133392334,
      "activations/layer12_attention_weight_min": -26.114891052246094,
      "activations/layer13_attention_weight_max": 55.491058349609375,
      "activations/layer13_attention_weight_min": -32.68326950073242,
      "activations/layer14_attention_weight_max": 42.34253692626953,
      "activations/layer14_attention_weight_min": -32.00674819946289,
      "activations/layer15_attention_weight_max": 37.66341781616211,
      "activations/layer15_attention_weight_min": -30.307266235351562,
      "activations/layer16_attention_weight_max": 28.947696685791016,
      "activations/layer16_attention_weight_min": -27.961523056030273,
      "activations/layer17_attention_weight_max": 54.092750549316406,
      "activations/layer17_attention_weight_min": -44.047950744628906,
      "activations/layer18_attention_weight_max": 45.72738265991211,
      "activations/layer18_attention_weight_min": -36.849830627441406,
      "activations/layer19_attention_weight_max": 22.64392852783203,
      "activations/layer19_attention_weight_min": -20.092649459838867,
      "activations/layer1_attention_weight_max": 16.82610321044922,
      "activations/layer1_attention_weight_min": -16.92190933227539,
      "activations/layer20_attention_weight_max": 22.483720779418945,
      "activations/layer20_attention_weight_min": -20.666736602783203,
      "activations/layer21_attention_weight_max": 41.55609893798828,
      "activations/layer21_attention_weight_min": -25.545028686523438,
      "activations/layer22_attention_weight_max": 31.596120834350586,
      "activations/layer22_attention_weight_min": -27.251550674438477,
      "activations/layer23_attention_weight_max": 35.062129974365234,
      "activations/layer23_attention_weight_min": -20.799739837646484,
      "activations/layer2_attention_weight_max": 31.400243759155273,
      "activations/layer2_attention_weight_min": -28.365087509155273,
      "activations/layer3_attention_weight_max": 85.85559844970703,
      "activations/layer3_attention_weight_min": -89.4090347290039,
      "activations/layer4_attention_weight_max": 101.03624725341797,
      "activations/layer4_attention_weight_min": -99.42670440673828,
      "activations/layer5_attention_weight_max": 88.09587860107422,
      "activations/layer5_attention_weight_min": -82.4825668334961,
      "activations/layer6_attention_weight_max": 56.52885055541992,
      "activations/layer6_attention_weight_min": -54.88898849487305,
      "activations/layer7_attention_weight_max": 80.80486297607422,
      "activations/layer7_attention_weight_min": -78.6387939453125,
      "activations/layer8_attention_weight_max": 50.13221740722656,
      "activations/layer8_attention_weight_min": -46.386714935302734,
      "activations/layer9_attention_weight_max": 59.47821044921875,
      "activations/layer9_attention_weight_min": -51.97480392456055,
      "epoch": 7.72,
      "learning_rate": 0.00010123939393939393,
      "loss": 2.8275,
      "step": 132850
    },
    {
      "activations/layer0_attention_weight_max": 14.310338973999023,
      "activations/layer0_attention_weight_min": -14.634167671203613,
      "activations/layer10_attention_weight_max": 37.67810821533203,
      "activations/layer10_attention_weight_min": -36.782527923583984,
      "activations/layer11_attention_weight_max": 31.325101852416992,
      "activations/layer11_attention_weight_min": -33.115966796875,
      "activations/layer12_attention_weight_max": 20.39227867126465,
      "activations/layer12_attention_weight_min": -26.502511978149414,
      "activations/layer13_attention_weight_max": 43.99461364746094,
      "activations/layer13_attention_weight_min": -33.12069320678711,
      "activations/layer14_attention_weight_max": 44.65360641479492,
      "activations/layer14_attention_weight_min": -31.444316864013672,
      "activations/layer15_attention_weight_max": 35.155967712402344,
      "activations/layer15_attention_weight_min": -30.18231773376465,
      "activations/layer16_attention_weight_max": 32.443878173828125,
      "activations/layer16_attention_weight_min": -29.09258270263672,
      "activations/layer17_attention_weight_max": 53.03095626831055,
      "activations/layer17_attention_weight_min": -44.14437484741211,
      "activations/layer18_attention_weight_max": 49.46310043334961,
      "activations/layer18_attention_weight_min": -39.65162658691406,
      "activations/layer19_attention_weight_max": 22.714262008666992,
      "activations/layer19_attention_weight_min": -22.38240623474121,
      "activations/layer1_attention_weight_max": 17.57459259033203,
      "activations/layer1_attention_weight_min": -14.850421905517578,
      "activations/layer20_attention_weight_max": 23.55551528930664,
      "activations/layer20_attention_weight_min": -22.054441452026367,
      "activations/layer21_attention_weight_max": 46.60557556152344,
      "activations/layer21_attention_weight_min": -25.186996459960938,
      "activations/layer22_attention_weight_max": 37.59302520751953,
      "activations/layer22_attention_weight_min": -27.11131477355957,
      "activations/layer23_attention_weight_max": 37.261199951171875,
      "activations/layer23_attention_weight_min": -24.730777740478516,
      "activations/layer2_attention_weight_max": 32.186851501464844,
      "activations/layer2_attention_weight_min": -30.11385154724121,
      "activations/layer3_attention_weight_max": 91.25138854980469,
      "activations/layer3_attention_weight_min": -90.21994018554688,
      "activations/layer4_attention_weight_max": 104.30670166015625,
      "activations/layer4_attention_weight_min": -100.18974304199219,
      "activations/layer5_attention_weight_max": 90.64950561523438,
      "activations/layer5_attention_weight_min": -86.65531921386719,
      "activations/layer6_attention_weight_max": 59.23755645751953,
      "activations/layer6_attention_weight_min": -58.19717788696289,
      "activations/layer7_attention_weight_max": 78.43338775634766,
      "activations/layer7_attention_weight_min": -77.65972137451172,
      "activations/layer8_attention_weight_max": 47.552371978759766,
      "activations/layer8_attention_weight_min": -47.479820251464844,
      "activations/layer9_attention_weight_max": 55.977455139160156,
      "activations/layer9_attention_weight_min": -52.111202239990234,
      "epoch": 7.72,
      "learning_rate": 0.00010122045454545455,
      "loss": 2.8257,
      "step": 132900
    },
    {
      "activations/layer0_attention_weight_max": 15.410046577453613,
      "activations/layer0_attention_weight_min": -15.011824607849121,
      "activations/layer10_attention_weight_max": 35.97718811035156,
      "activations/layer10_attention_weight_min": -33.06389617919922,
      "activations/layer11_attention_weight_max": 30.94768524169922,
      "activations/layer11_attention_weight_min": -29.689613342285156,
      "activations/layer12_attention_weight_max": 19.775590896606445,
      "activations/layer12_attention_weight_min": -23.874576568603516,
      "activations/layer13_attention_weight_max": 41.099586486816406,
      "activations/layer13_attention_weight_min": -32.26747131347656,
      "activations/layer14_attention_weight_max": 35.05030822753906,
      "activations/layer14_attention_weight_min": -35.26173400878906,
      "activations/layer15_attention_weight_max": 33.161163330078125,
      "activations/layer15_attention_weight_min": -32.183895111083984,
      "activations/layer16_attention_weight_max": 30.958646774291992,
      "activations/layer16_attention_weight_min": -27.634435653686523,
      "activations/layer17_attention_weight_max": 51.40854263305664,
      "activations/layer17_attention_weight_min": -42.51332473754883,
      "activations/layer18_attention_weight_max": 46.558998107910156,
      "activations/layer18_attention_weight_min": -35.632022857666016,
      "activations/layer19_attention_weight_max": 20.95386505126953,
      "activations/layer19_attention_weight_min": -19.479103088378906,
      "activations/layer1_attention_weight_max": 17.77069664001465,
      "activations/layer1_attention_weight_min": -15.72823715209961,
      "activations/layer20_attention_weight_max": 21.342025756835938,
      "activations/layer20_attention_weight_min": -22.322185516357422,
      "activations/layer21_attention_weight_max": 36.32500457763672,
      "activations/layer21_attention_weight_min": -22.911489486694336,
      "activations/layer22_attention_weight_max": 31.393844604492188,
      "activations/layer22_attention_weight_min": -24.58391571044922,
      "activations/layer23_attention_weight_max": 35.342613220214844,
      "activations/layer23_attention_weight_min": -20.432241439819336,
      "activations/layer2_attention_weight_max": 30.521377563476562,
      "activations/layer2_attention_weight_min": -28.960948944091797,
      "activations/layer3_attention_weight_max": 87.5052719116211,
      "activations/layer3_attention_weight_min": -88.55564880371094,
      "activations/layer4_attention_weight_max": 101.04216766357422,
      "activations/layer4_attention_weight_min": -90.7657699584961,
      "activations/layer5_attention_weight_max": 86.43047332763672,
      "activations/layer5_attention_weight_min": -77.42152404785156,
      "activations/layer6_attention_weight_max": 56.7938117980957,
      "activations/layer6_attention_weight_min": -50.71613311767578,
      "activations/layer7_attention_weight_max": 74.87779998779297,
      "activations/layer7_attention_weight_min": -71.2945785522461,
      "activations/layer8_attention_weight_max": 44.794395446777344,
      "activations/layer8_attention_weight_min": -44.91254425048828,
      "activations/layer9_attention_weight_max": 48.84306335449219,
      "activations/layer9_attention_weight_min": -48.27346420288086,
      "epoch": 7.73,
      "learning_rate": 0.00010120151515151514,
      "loss": 2.8236,
      "step": 132950
    },
    {
      "activations/layer0_attention_weight_max": 15.041889190673828,
      "activations/layer0_attention_weight_min": -14.751001358032227,
      "activations/layer10_attention_weight_max": 35.637474060058594,
      "activations/layer10_attention_weight_min": -33.97198486328125,
      "activations/layer11_attention_weight_max": 29.345149993896484,
      "activations/layer11_attention_weight_min": -30.147994995117188,
      "activations/layer12_attention_weight_max": 20.41738510131836,
      "activations/layer12_attention_weight_min": -26.9368953704834,
      "activations/layer13_attention_weight_max": 43.51698303222656,
      "activations/layer13_attention_weight_min": -34.41035461425781,
      "activations/layer14_attention_weight_max": 40.52567672729492,
      "activations/layer14_attention_weight_min": -30.77779769897461,
      "activations/layer15_attention_weight_max": 36.69572448730469,
      "activations/layer15_attention_weight_min": -29.958831787109375,
      "activations/layer16_attention_weight_max": 34.61153030395508,
      "activations/layer16_attention_weight_min": -26.414688110351562,
      "activations/layer17_attention_weight_max": 56.679405212402344,
      "activations/layer17_attention_weight_min": -42.81840896606445,
      "activations/layer18_attention_weight_max": 48.48701477050781,
      "activations/layer18_attention_weight_min": -37.901649475097656,
      "activations/layer19_attention_weight_max": 24.082345962524414,
      "activations/layer19_attention_weight_min": -21.65935707092285,
      "activations/layer1_attention_weight_max": 16.887311935424805,
      "activations/layer1_attention_weight_min": -16.897783279418945,
      "activations/layer20_attention_weight_max": 23.808971405029297,
      "activations/layer20_attention_weight_min": -21.002397537231445,
      "activations/layer21_attention_weight_max": 43.0311279296875,
      "activations/layer21_attention_weight_min": -24.319780349731445,
      "activations/layer22_attention_weight_max": 32.39377212524414,
      "activations/layer22_attention_weight_min": -26.611352920532227,
      "activations/layer23_attention_weight_max": 38.49123001098633,
      "activations/layer23_attention_weight_min": -21.77832794189453,
      "activations/layer2_attention_weight_max": 31.883316040039062,
      "activations/layer2_attention_weight_min": -30.034099578857422,
      "activations/layer3_attention_weight_max": 87.25505065917969,
      "activations/layer3_attention_weight_min": -87.72404479980469,
      "activations/layer4_attention_weight_max": 100.83598327636719,
      "activations/layer4_attention_weight_min": -92.2044906616211,
      "activations/layer5_attention_weight_max": 85.08073425292969,
      "activations/layer5_attention_weight_min": -77.10615539550781,
      "activations/layer6_attention_weight_max": 54.01150131225586,
      "activations/layer6_attention_weight_min": -53.369712829589844,
      "activations/layer7_attention_weight_max": 73.39734649658203,
      "activations/layer7_attention_weight_min": -69.18605041503906,
      "activations/layer8_attention_weight_max": 44.9703369140625,
      "activations/layer8_attention_weight_min": -44.71417236328125,
      "activations/layer9_attention_weight_max": 50.02349090576172,
      "activations/layer9_attention_weight_min": -49.906490325927734,
      "epoch": 7.73,
      "learning_rate": 0.00010118257575757575,
      "loss": 2.8059,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5964,
      "eval_samples_per_second": 499.512,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5964,
      "eval_openwebtext_samples_per_second": 499.512,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_wikitext_loss": 3.0,
      "eval_wikitext_ppl": 20.085536923187668,
      "eval_wikitext_runtime": 2.0369,
      "eval_wikitext_samples_per_second": 223.867,
      "step": 133000
    },
    {
      "epoch": 7.73,
      "eval_lambada_loss": 2.7265625,
      "eval_lambada_ppl": 15.280270706875589,
      "eval_lambada_runtime": 9.6159,
      "eval_lambada_samples_per_second": 506.346,
      "step": 133000
    },
    {
      "activations/layer0_attention_weight_max": 14.999435424804688,
      "activations/layer0_attention_weight_min": -15.172760963439941,
      "activations/layer10_attention_weight_max": 38.607696533203125,
      "activations/layer10_attention_weight_min": -36.9158821105957,
      "activations/layer11_attention_weight_max": 33.56407928466797,
      "activations/layer11_attention_weight_min": -31.967147827148438,
      "activations/layer12_attention_weight_max": 20.4647274017334,
      "activations/layer12_attention_weight_min": -27.209278106689453,
      "activations/layer13_attention_weight_max": 47.15958786010742,
      "activations/layer13_attention_weight_min": -34.6541748046875,
      "activations/layer14_attention_weight_max": 43.92257308959961,
      "activations/layer14_attention_weight_min": -32.77104949951172,
      "activations/layer15_attention_weight_max": 39.74248123168945,
      "activations/layer15_attention_weight_min": -31.211820602416992,
      "activations/layer16_attention_weight_max": 37.484222412109375,
      "activations/layer16_attention_weight_min": -29.19056510925293,
      "activations/layer17_attention_weight_max": 61.10961151123047,
      "activations/layer17_attention_weight_min": -45.546470642089844,
      "activations/layer18_attention_weight_max": 57.08750534057617,
      "activations/layer18_attention_weight_min": -37.87157440185547,
      "activations/layer19_attention_weight_max": 22.598896026611328,
      "activations/layer19_attention_weight_min": -22.887479782104492,
      "activations/layer1_attention_weight_max": 18.073514938354492,
      "activations/layer1_attention_weight_min": -17.160245895385742,
      "activations/layer20_attention_weight_max": 26.307470321655273,
      "activations/layer20_attention_weight_min": -19.865589141845703,
      "activations/layer21_attention_weight_max": 42.38160705566406,
      "activations/layer21_attention_weight_min": -25.287487030029297,
      "activations/layer22_attention_weight_max": 33.20777893066406,
      "activations/layer22_attention_weight_min": -26.225360870361328,
      "activations/layer23_attention_weight_max": 30.649002075195312,
      "activations/layer23_attention_weight_min": -22.99294662475586,
      "activations/layer2_attention_weight_max": 31.665504455566406,
      "activations/layer2_attention_weight_min": -30.028213500976562,
      "activations/layer3_attention_weight_max": 86.59589385986328,
      "activations/layer3_attention_weight_min": -86.21484375,
      "activations/layer4_attention_weight_max": 98.83317565917969,
      "activations/layer4_attention_weight_min": -97.04161071777344,
      "activations/layer5_attention_weight_max": 91.01412200927734,
      "activations/layer5_attention_weight_min": -83.82735443115234,
      "activations/layer6_attention_weight_max": 55.46863555908203,
      "activations/layer6_attention_weight_min": -54.42559051513672,
      "activations/layer7_attention_weight_max": 78.8096694946289,
      "activations/layer7_attention_weight_min": -74.08048248291016,
      "activations/layer8_attention_weight_max": 45.9410285949707,
      "activations/layer8_attention_weight_min": -46.51101303100586,
      "activations/layer9_attention_weight_max": 56.563720703125,
      "activations/layer9_attention_weight_min": -52.28352737426758,
      "epoch": 7.73,
      "learning_rate": 0.00010116363636363634,
      "loss": 2.8155,
      "step": 133050
    },
    {
      "activations/layer0_attention_weight_max": 15.239249229431152,
      "activations/layer0_attention_weight_min": -15.075045585632324,
      "activations/layer10_attention_weight_max": 39.8094482421875,
      "activations/layer10_attention_weight_min": -35.503387451171875,
      "activations/layer11_attention_weight_max": 31.958240509033203,
      "activations/layer11_attention_weight_min": -31.92487335205078,
      "activations/layer12_attention_weight_max": 20.559207916259766,
      "activations/layer12_attention_weight_min": -24.42291259765625,
      "activations/layer13_attention_weight_max": 38.038330078125,
      "activations/layer13_attention_weight_min": -31.073144912719727,
      "activations/layer14_attention_weight_max": 37.192386627197266,
      "activations/layer14_attention_weight_min": -29.882848739624023,
      "activations/layer15_attention_weight_max": 34.531375885009766,
      "activations/layer15_attention_weight_min": -28.460601806640625,
      "activations/layer16_attention_weight_max": 32.02054214477539,
      "activations/layer16_attention_weight_min": -27.830909729003906,
      "activations/layer17_attention_weight_max": 54.9079475402832,
      "activations/layer17_attention_weight_min": -41.91524124145508,
      "activations/layer18_attention_weight_max": 45.40843963623047,
      "activations/layer18_attention_weight_min": -35.63912582397461,
      "activations/layer19_attention_weight_max": 22.175731658935547,
      "activations/layer19_attention_weight_min": -22.8588924407959,
      "activations/layer1_attention_weight_max": 17.872589111328125,
      "activations/layer1_attention_weight_min": -14.64444637298584,
      "activations/layer20_attention_weight_max": 23.434391021728516,
      "activations/layer20_attention_weight_min": -24.041501998901367,
      "activations/layer21_attention_weight_max": 40.37581253051758,
      "activations/layer21_attention_weight_min": -24.02707862854004,
      "activations/layer22_attention_weight_max": 31.41530990600586,
      "activations/layer22_attention_weight_min": -24.560972213745117,
      "activations/layer23_attention_weight_max": 34.6939697265625,
      "activations/layer23_attention_weight_min": -23.298397064208984,
      "activations/layer2_attention_weight_max": 31.333799362182617,
      "activations/layer2_attention_weight_min": -29.405729293823242,
      "activations/layer3_attention_weight_max": 86.72295379638672,
      "activations/layer3_attention_weight_min": -90.33553314208984,
      "activations/layer4_attention_weight_max": 101.60869598388672,
      "activations/layer4_attention_weight_min": -97.2134780883789,
      "activations/layer5_attention_weight_max": 88.78146362304688,
      "activations/layer5_attention_weight_min": -85.14116668701172,
      "activations/layer6_attention_weight_max": 54.61711502075195,
      "activations/layer6_attention_weight_min": -55.70951461791992,
      "activations/layer7_attention_weight_max": 74.83638000488281,
      "activations/layer7_attention_weight_min": -73.20156860351562,
      "activations/layer8_attention_weight_max": 45.00709915161133,
      "activations/layer8_attention_weight_min": -45.25126266479492,
      "activations/layer9_attention_weight_max": 50.9143180847168,
      "activations/layer9_attention_weight_min": -45.72952651977539,
      "epoch": 7.73,
      "learning_rate": 0.00010114469696969696,
      "loss": 2.8161,
      "step": 133100
    },
    {
      "activations/layer0_attention_weight_max": 15.64384937286377,
      "activations/layer0_attention_weight_min": -15.077900886535645,
      "activations/layer10_attention_weight_max": 40.034889221191406,
      "activations/layer10_attention_weight_min": -36.59187698364258,
      "activations/layer11_attention_weight_max": 34.56199645996094,
      "activations/layer11_attention_weight_min": -32.966712951660156,
      "activations/layer12_attention_weight_max": 20.234928131103516,
      "activations/layer12_attention_weight_min": -21.620786666870117,
      "activations/layer13_attention_weight_max": 45.80553436279297,
      "activations/layer13_attention_weight_min": -31.10076904296875,
      "activations/layer14_attention_weight_max": 45.78691482543945,
      "activations/layer14_attention_weight_min": -29.961435317993164,
      "activations/layer15_attention_weight_max": 35.30180740356445,
      "activations/layer15_attention_weight_min": -28.97454071044922,
      "activations/layer16_attention_weight_max": 30.703580856323242,
      "activations/layer16_attention_weight_min": -26.024478912353516,
      "activations/layer17_attention_weight_max": 53.140560150146484,
      "activations/layer17_attention_weight_min": -39.3162841796875,
      "activations/layer18_attention_weight_max": 47.24896240234375,
      "activations/layer18_attention_weight_min": -36.12308120727539,
      "activations/layer19_attention_weight_max": 23.142452239990234,
      "activations/layer19_attention_weight_min": -22.619382858276367,
      "activations/layer1_attention_weight_max": 16.527801513671875,
      "activations/layer1_attention_weight_min": -13.773852348327637,
      "activations/layer20_attention_weight_max": 21.974449157714844,
      "activations/layer20_attention_weight_min": -21.180919647216797,
      "activations/layer21_attention_weight_max": 36.913875579833984,
      "activations/layer21_attention_weight_min": -22.768564224243164,
      "activations/layer22_attention_weight_max": 33.347877502441406,
      "activations/layer22_attention_weight_min": -24.008821487426758,
      "activations/layer23_attention_weight_max": 36.6790771484375,
      "activations/layer23_attention_weight_min": -24.181591033935547,
      "activations/layer2_attention_weight_max": 30.544395446777344,
      "activations/layer2_attention_weight_min": -29.525859832763672,
      "activations/layer3_attention_weight_max": 85.66900634765625,
      "activations/layer3_attention_weight_min": -88.84105682373047,
      "activations/layer4_attention_weight_max": 100.83087921142578,
      "activations/layer4_attention_weight_min": -98.52383422851562,
      "activations/layer5_attention_weight_max": 90.16130828857422,
      "activations/layer5_attention_weight_min": -82.64517211914062,
      "activations/layer6_attention_weight_max": 60.16960906982422,
      "activations/layer6_attention_weight_min": -54.82162094116211,
      "activations/layer7_attention_weight_max": 86.02546691894531,
      "activations/layer7_attention_weight_min": -83.8224105834961,
      "activations/layer8_attention_weight_max": 52.4755744934082,
      "activations/layer8_attention_weight_min": -48.10182571411133,
      "activations/layer9_attention_weight_max": 62.50154495239258,
      "activations/layer9_attention_weight_min": -53.36613845825195,
      "epoch": 7.74,
      "learning_rate": 0.00010112575757575757,
      "loss": 2.7932,
      "step": 133150
    },
    {
      "activations/layer0_attention_weight_max": 14.944841384887695,
      "activations/layer0_attention_weight_min": -14.609431266784668,
      "activations/layer10_attention_weight_max": 40.251644134521484,
      "activations/layer10_attention_weight_min": -37.1562385559082,
      "activations/layer11_attention_weight_max": 36.123077392578125,
      "activations/layer11_attention_weight_min": -31.688575744628906,
      "activations/layer12_attention_weight_max": 19.93011474609375,
      "activations/layer12_attention_weight_min": -28.94251823425293,
      "activations/layer13_attention_weight_max": 43.18576431274414,
      "activations/layer13_attention_weight_min": -32.65357971191406,
      "activations/layer14_attention_weight_max": 37.564517974853516,
      "activations/layer14_attention_weight_min": -29.933170318603516,
      "activations/layer15_attention_weight_max": 34.471981048583984,
      "activations/layer15_attention_weight_min": -28.332069396972656,
      "activations/layer16_attention_weight_max": 34.21535110473633,
      "activations/layer16_attention_weight_min": -27.46605682373047,
      "activations/layer17_attention_weight_max": 54.96363067626953,
      "activations/layer17_attention_weight_min": -40.88547134399414,
      "activations/layer18_attention_weight_max": 45.70085906982422,
      "activations/layer18_attention_weight_min": -37.04407501220703,
      "activations/layer19_attention_weight_max": 19.334041595458984,
      "activations/layer19_attention_weight_min": -20.271167755126953,
      "activations/layer1_attention_weight_max": 17.796897888183594,
      "activations/layer1_attention_weight_min": -14.216667175292969,
      "activations/layer20_attention_weight_max": 21.508718490600586,
      "activations/layer20_attention_weight_min": -20.987829208374023,
      "activations/layer21_attention_weight_max": 36.29545593261719,
      "activations/layer21_attention_weight_min": -22.87395668029785,
      "activations/layer22_attention_weight_max": 28.188785552978516,
      "activations/layer22_attention_weight_min": -25.09391212463379,
      "activations/layer23_attention_weight_max": 31.53814697265625,
      "activations/layer23_attention_weight_min": -22.735187530517578,
      "activations/layer2_attention_weight_max": 33.163352966308594,
      "activations/layer2_attention_weight_min": -33.15284729003906,
      "activations/layer3_attention_weight_max": 91.27798461914062,
      "activations/layer3_attention_weight_min": -91.31074523925781,
      "activations/layer4_attention_weight_max": 105.6814193725586,
      "activations/layer4_attention_weight_min": -100.4368667602539,
      "activations/layer5_attention_weight_max": 94.70587158203125,
      "activations/layer5_attention_weight_min": -83.93025207519531,
      "activations/layer6_attention_weight_max": 59.1467399597168,
      "activations/layer6_attention_weight_min": -53.76914596557617,
      "activations/layer7_attention_weight_max": 83.09379577636719,
      "activations/layer7_attention_weight_min": -74.65337371826172,
      "activations/layer8_attention_weight_max": 48.96319580078125,
      "activations/layer8_attention_weight_min": -46.51654052734375,
      "activations/layer9_attention_weight_max": 59.01980209350586,
      "activations/layer9_attention_weight_min": -53.75347900390625,
      "epoch": 7.74,
      "learning_rate": 0.00010110681818181817,
      "loss": 2.8154,
      "step": 133200
    },
    {
      "activations/layer0_attention_weight_max": 15.247730255126953,
      "activations/layer0_attention_weight_min": -14.64529800415039,
      "activations/layer10_attention_weight_max": 41.76433181762695,
      "activations/layer10_attention_weight_min": -39.716556549072266,
      "activations/layer11_attention_weight_max": 31.236989974975586,
      "activations/layer11_attention_weight_min": -31.18069839477539,
      "activations/layer12_attention_weight_max": 21.08405303955078,
      "activations/layer12_attention_weight_min": -30.090421676635742,
      "activations/layer13_attention_weight_max": 46.66897964477539,
      "activations/layer13_attention_weight_min": -36.46763610839844,
      "activations/layer14_attention_weight_max": 37.845237731933594,
      "activations/layer14_attention_weight_min": -29.022687911987305,
      "activations/layer15_attention_weight_max": 38.58137130737305,
      "activations/layer15_attention_weight_min": -27.590173721313477,
      "activations/layer16_attention_weight_max": 33.10666275024414,
      "activations/layer16_attention_weight_min": -27.7786922454834,
      "activations/layer17_attention_weight_max": 60.85852813720703,
      "activations/layer17_attention_weight_min": -43.79851531982422,
      "activations/layer18_attention_weight_max": 48.82803726196289,
      "activations/layer18_attention_weight_min": -39.25774002075195,
      "activations/layer19_attention_weight_max": 22.916135787963867,
      "activations/layer19_attention_weight_min": -21.596187591552734,
      "activations/layer1_attention_weight_max": 17.436767578125,
      "activations/layer1_attention_weight_min": -13.855608940124512,
      "activations/layer20_attention_weight_max": 21.986225128173828,
      "activations/layer20_attention_weight_min": -20.99881935119629,
      "activations/layer21_attention_weight_max": 37.88470458984375,
      "activations/layer21_attention_weight_min": -22.615880966186523,
      "activations/layer22_attention_weight_max": 31.63764190673828,
      "activations/layer22_attention_weight_min": -24.2327938079834,
      "activations/layer23_attention_weight_max": 38.154205322265625,
      "activations/layer23_attention_weight_min": -21.069246292114258,
      "activations/layer2_attention_weight_max": 31.79327392578125,
      "activations/layer2_attention_weight_min": -30.379074096679688,
      "activations/layer3_attention_weight_max": 92.96177673339844,
      "activations/layer3_attention_weight_min": -98.19245147705078,
      "activations/layer4_attention_weight_max": 103.01496887207031,
      "activations/layer4_attention_weight_min": -105.35919189453125,
      "activations/layer5_attention_weight_max": 93.28390502929688,
      "activations/layer5_attention_weight_min": -88.28941345214844,
      "activations/layer6_attention_weight_max": 57.33461380004883,
      "activations/layer6_attention_weight_min": -56.33378219604492,
      "activations/layer7_attention_weight_max": 77.80058288574219,
      "activations/layer7_attention_weight_min": -75.76533508300781,
      "activations/layer8_attention_weight_max": 46.481056213378906,
      "activations/layer8_attention_weight_min": -47.553611755371094,
      "activations/layer9_attention_weight_max": 52.802913665771484,
      "activations/layer9_attention_weight_min": -51.16374969482422,
      "epoch": 7.74,
      "learning_rate": 0.00010108787878787877,
      "loss": 2.8267,
      "step": 133250
    },
    {
      "activations/layer0_attention_weight_max": 14.80009651184082,
      "activations/layer0_attention_weight_min": -14.671684265136719,
      "activations/layer10_attention_weight_max": 35.42832565307617,
      "activations/layer10_attention_weight_min": -36.50767135620117,
      "activations/layer11_attention_weight_max": 31.050575256347656,
      "activations/layer11_attention_weight_min": -29.708538055419922,
      "activations/layer12_attention_weight_max": 19.085046768188477,
      "activations/layer12_attention_weight_min": -34.40583038330078,
      "activations/layer13_attention_weight_max": 40.49540710449219,
      "activations/layer13_attention_weight_min": -35.8444938659668,
      "activations/layer14_attention_weight_max": 43.93928527832031,
      "activations/layer14_attention_weight_min": -30.80514907836914,
      "activations/layer15_attention_weight_max": 32.43949508666992,
      "activations/layer15_attention_weight_min": -30.756807327270508,
      "activations/layer16_attention_weight_max": 31.504961013793945,
      "activations/layer16_attention_weight_min": -29.04892921447754,
      "activations/layer17_attention_weight_max": 52.55018615722656,
      "activations/layer17_attention_weight_min": -42.477176666259766,
      "activations/layer18_attention_weight_max": 44.78439712524414,
      "activations/layer18_attention_weight_min": -35.170677185058594,
      "activations/layer19_attention_weight_max": 21.3569393157959,
      "activations/layer19_attention_weight_min": -19.98445701599121,
      "activations/layer1_attention_weight_max": 16.22895050048828,
      "activations/layer1_attention_weight_min": -12.69274616241455,
      "activations/layer20_attention_weight_max": 21.54410171508789,
      "activations/layer20_attention_weight_min": -20.31370735168457,
      "activations/layer21_attention_weight_max": 40.978389739990234,
      "activations/layer21_attention_weight_min": -26.31172752380371,
      "activations/layer22_attention_weight_max": 36.68205261230469,
      "activations/layer22_attention_weight_min": -23.810993194580078,
      "activations/layer23_attention_weight_max": 34.59953308105469,
      "activations/layer23_attention_weight_min": -21.80373191833496,
      "activations/layer2_attention_weight_max": 32.8431396484375,
      "activations/layer2_attention_weight_min": -29.268081665039062,
      "activations/layer3_attention_weight_max": 92.55370330810547,
      "activations/layer3_attention_weight_min": -89.18511962890625,
      "activations/layer4_attention_weight_max": 103.4525375366211,
      "activations/layer4_attention_weight_min": -92.69124603271484,
      "activations/layer5_attention_weight_max": 85.06724548339844,
      "activations/layer5_attention_weight_min": -83.17848205566406,
      "activations/layer6_attention_weight_max": 55.12940216064453,
      "activations/layer6_attention_weight_min": -53.76227951049805,
      "activations/layer7_attention_weight_max": 77.89302062988281,
      "activations/layer7_attention_weight_min": -73.14167022705078,
      "activations/layer8_attention_weight_max": 45.61431121826172,
      "activations/layer8_attention_weight_min": -44.40321731567383,
      "activations/layer9_attention_weight_max": 63.9423942565918,
      "activations/layer9_attention_weight_min": -52.46148681640625,
      "epoch": 7.75,
      "learning_rate": 0.00010106893939393939,
      "loss": 2.7928,
      "step": 133300
    },
    {
      "activations/layer0_attention_weight_max": 14.960312843322754,
      "activations/layer0_attention_weight_min": -14.308262825012207,
      "activations/layer10_attention_weight_max": 38.98797607421875,
      "activations/layer10_attention_weight_min": -36.609527587890625,
      "activations/layer11_attention_weight_max": 31.19422149658203,
      "activations/layer11_attention_weight_min": -30.794750213623047,
      "activations/layer12_attention_weight_max": 18.29842185974121,
      "activations/layer12_attention_weight_min": -26.516075134277344,
      "activations/layer13_attention_weight_max": 43.408573150634766,
      "activations/layer13_attention_weight_min": -35.08892059326172,
      "activations/layer14_attention_weight_max": 43.78398895263672,
      "activations/layer14_attention_weight_min": -32.409358978271484,
      "activations/layer15_attention_weight_max": 33.4153938293457,
      "activations/layer15_attention_weight_min": -31.619598388671875,
      "activations/layer16_attention_weight_max": 29.98660659790039,
      "activations/layer16_attention_weight_min": -27.993284225463867,
      "activations/layer17_attention_weight_max": 51.36376953125,
      "activations/layer17_attention_weight_min": -43.19927215576172,
      "activations/layer18_attention_weight_max": 43.05850601196289,
      "activations/layer18_attention_weight_min": -37.29621505737305,
      "activations/layer19_attention_weight_max": 21.669469833374023,
      "activations/layer19_attention_weight_min": -25.969221115112305,
      "activations/layer1_attention_weight_max": 16.737070083618164,
      "activations/layer1_attention_weight_min": -14.995171546936035,
      "activations/layer20_attention_weight_max": 23.201358795166016,
      "activations/layer20_attention_weight_min": -22.03053092956543,
      "activations/layer21_attention_weight_max": 34.86043167114258,
      "activations/layer21_attention_weight_min": -25.72837257385254,
      "activations/layer22_attention_weight_max": 28.0618896484375,
      "activations/layer22_attention_weight_min": -26.8217830657959,
      "activations/layer23_attention_weight_max": 33.40190887451172,
      "activations/layer23_attention_weight_min": -26.262563705444336,
      "activations/layer2_attention_weight_max": 31.57887077331543,
      "activations/layer2_attention_weight_min": -29.153121948242188,
      "activations/layer3_attention_weight_max": 84.34138488769531,
      "activations/layer3_attention_weight_min": -87.18769073486328,
      "activations/layer4_attention_weight_max": 99.5870590209961,
      "activations/layer4_attention_weight_min": -93.99723815917969,
      "activations/layer5_attention_weight_max": 87.829345703125,
      "activations/layer5_attention_weight_min": -81.13838958740234,
      "activations/layer6_attention_weight_max": 55.61456298828125,
      "activations/layer6_attention_weight_min": -52.563209533691406,
      "activations/layer7_attention_weight_max": 80.11064147949219,
      "activations/layer7_attention_weight_min": -73.67726135253906,
      "activations/layer8_attention_weight_max": 46.70394515991211,
      "activations/layer8_attention_weight_min": -47.07526779174805,
      "activations/layer9_attention_weight_max": 49.99613571166992,
      "activations/layer9_attention_weight_min": -48.68587875366211,
      "epoch": 7.75,
      "learning_rate": 0.00010104999999999999,
      "loss": 2.8025,
      "step": 133350
    },
    {
      "activations/layer0_attention_weight_max": 15.603561401367188,
      "activations/layer0_attention_weight_min": -14.602630615234375,
      "activations/layer10_attention_weight_max": 42.14363479614258,
      "activations/layer10_attention_weight_min": -36.93442153930664,
      "activations/layer11_attention_weight_max": 34.311927795410156,
      "activations/layer11_attention_weight_min": -31.865591049194336,
      "activations/layer12_attention_weight_max": 21.61285972595215,
      "activations/layer12_attention_weight_min": -25.163108825683594,
      "activations/layer13_attention_weight_max": 46.51945495605469,
      "activations/layer13_attention_weight_min": -31.659414291381836,
      "activations/layer14_attention_weight_max": 46.09785461425781,
      "activations/layer14_attention_weight_min": -32.35340881347656,
      "activations/layer15_attention_weight_max": 42.96731185913086,
      "activations/layer15_attention_weight_min": -30.43813705444336,
      "activations/layer16_attention_weight_max": 36.027469635009766,
      "activations/layer16_attention_weight_min": -29.798765182495117,
      "activations/layer17_attention_weight_max": 65.1843032836914,
      "activations/layer17_attention_weight_min": -45.662750244140625,
      "activations/layer18_attention_weight_max": 60.17256164550781,
      "activations/layer18_attention_weight_min": -38.70662307739258,
      "activations/layer19_attention_weight_max": 28.126632690429688,
      "activations/layer19_attention_weight_min": -23.603620529174805,
      "activations/layer1_attention_weight_max": 17.543254852294922,
      "activations/layer1_attention_weight_min": -14.617793083190918,
      "activations/layer20_attention_weight_max": 31.39189910888672,
      "activations/layer20_attention_weight_min": -21.873577117919922,
      "activations/layer21_attention_weight_max": 50.0581169128418,
      "activations/layer21_attention_weight_min": -23.564321517944336,
      "activations/layer22_attention_weight_max": 39.13886260986328,
      "activations/layer22_attention_weight_min": -25.662019729614258,
      "activations/layer23_attention_weight_max": 45.855628967285156,
      "activations/layer23_attention_weight_min": -23.715579986572266,
      "activations/layer2_attention_weight_max": 31.930068969726562,
      "activations/layer2_attention_weight_min": -29.834760665893555,
      "activations/layer3_attention_weight_max": 91.25003814697266,
      "activations/layer3_attention_weight_min": -85.91472625732422,
      "activations/layer4_attention_weight_max": 102.89891815185547,
      "activations/layer4_attention_weight_min": -98.77193450927734,
      "activations/layer5_attention_weight_max": 87.60979461669922,
      "activations/layer5_attention_weight_min": -82.79624938964844,
      "activations/layer6_attention_weight_max": 61.568206787109375,
      "activations/layer6_attention_weight_min": -57.08323669433594,
      "activations/layer7_attention_weight_max": 84.28764343261719,
      "activations/layer7_attention_weight_min": -75.24964141845703,
      "activations/layer8_attention_weight_max": 47.35131072998047,
      "activations/layer8_attention_weight_min": -51.85637283325195,
      "activations/layer9_attention_weight_max": 52.572052001953125,
      "activations/layer9_attention_weight_min": -48.89423751831055,
      "epoch": 7.75,
      "learning_rate": 0.0001010310606060606,
      "loss": 2.7988,
      "step": 133400
    },
    {
      "activations/layer0_attention_weight_max": 15.303753852844238,
      "activations/layer0_attention_weight_min": -14.619993209838867,
      "activations/layer10_attention_weight_max": 37.89863586425781,
      "activations/layer10_attention_weight_min": -36.91109085083008,
      "activations/layer11_attention_weight_max": 30.569232940673828,
      "activations/layer11_attention_weight_min": -30.645221710205078,
      "activations/layer12_attention_weight_max": 19.706499099731445,
      "activations/layer12_attention_weight_min": -23.2618465423584,
      "activations/layer13_attention_weight_max": 47.90736389160156,
      "activations/layer13_attention_weight_min": -35.14347457885742,
      "activations/layer14_attention_weight_max": 37.40093231201172,
      "activations/layer14_attention_weight_min": -29.081226348876953,
      "activations/layer15_attention_weight_max": 33.7555046081543,
      "activations/layer15_attention_weight_min": -30.39467430114746,
      "activations/layer16_attention_weight_max": 30.437360763549805,
      "activations/layer16_attention_weight_min": -26.98641586303711,
      "activations/layer17_attention_weight_max": 50.29085159301758,
      "activations/layer17_attention_weight_min": -41.91890335083008,
      "activations/layer18_attention_weight_max": 44.55659484863281,
      "activations/layer18_attention_weight_min": -35.61518478393555,
      "activations/layer19_attention_weight_max": 20.363012313842773,
      "activations/layer19_attention_weight_min": -20.462553024291992,
      "activations/layer1_attention_weight_max": 16.23206901550293,
      "activations/layer1_attention_weight_min": -15.939924240112305,
      "activations/layer20_attention_weight_max": 21.72316551208496,
      "activations/layer20_attention_weight_min": -20.78341293334961,
      "activations/layer21_attention_weight_max": 36.501922607421875,
      "activations/layer21_attention_weight_min": -25.720443725585938,
      "activations/layer22_attention_weight_max": 29.186994552612305,
      "activations/layer22_attention_weight_min": -23.91048240661621,
      "activations/layer23_attention_weight_max": 33.51950454711914,
      "activations/layer23_attention_weight_min": -23.136627197265625,
      "activations/layer2_attention_weight_max": 31.884143829345703,
      "activations/layer2_attention_weight_min": -29.218746185302734,
      "activations/layer3_attention_weight_max": 89.57703399658203,
      "activations/layer3_attention_weight_min": -93.75028228759766,
      "activations/layer4_attention_weight_max": 107.69148254394531,
      "activations/layer4_attention_weight_min": -96.82364654541016,
      "activations/layer5_attention_weight_max": 92.42500305175781,
      "activations/layer5_attention_weight_min": -81.72486877441406,
      "activations/layer6_attention_weight_max": 58.3872184753418,
      "activations/layer6_attention_weight_min": -54.95393371582031,
      "activations/layer7_attention_weight_max": 76.90394592285156,
      "activations/layer7_attention_weight_min": -73.2985610961914,
      "activations/layer8_attention_weight_max": 46.5569953918457,
      "activations/layer8_attention_weight_min": -49.4788932800293,
      "activations/layer9_attention_weight_max": 52.6812629699707,
      "activations/layer9_attention_weight_min": -48.381656646728516,
      "epoch": 7.75,
      "learning_rate": 0.0001010121212121212,
      "loss": 2.8308,
      "step": 133450
    },
    {
      "activations/layer0_attention_weight_max": 14.744914054870605,
      "activations/layer0_attention_weight_min": -14.525304794311523,
      "activations/layer10_attention_weight_max": 37.80004119873047,
      "activations/layer10_attention_weight_min": -36.960025787353516,
      "activations/layer11_attention_weight_max": 31.241376876831055,
      "activations/layer11_attention_weight_min": -30.46765899658203,
      "activations/layer12_attention_weight_max": 18.590770721435547,
      "activations/layer12_attention_weight_min": -24.897241592407227,
      "activations/layer13_attention_weight_max": 41.92615509033203,
      "activations/layer13_attention_weight_min": -33.32339096069336,
      "activations/layer14_attention_weight_max": 35.82087707519531,
      "activations/layer14_attention_weight_min": -29.550052642822266,
      "activations/layer15_attention_weight_max": 37.24622344970703,
      "activations/layer15_attention_weight_min": -30.335758209228516,
      "activations/layer16_attention_weight_max": 31.583528518676758,
      "activations/layer16_attention_weight_min": -26.798322677612305,
      "activations/layer17_attention_weight_max": 52.394378662109375,
      "activations/layer17_attention_weight_min": -40.666526794433594,
      "activations/layer18_attention_weight_max": 46.315799713134766,
      "activations/layer18_attention_weight_min": -34.42833709716797,
      "activations/layer19_attention_weight_max": 23.82571029663086,
      "activations/layer19_attention_weight_min": -21.652082443237305,
      "activations/layer1_attention_weight_max": 18.041555404663086,
      "activations/layer1_attention_weight_min": -14.766039848327637,
      "activations/layer20_attention_weight_max": 23.271394729614258,
      "activations/layer20_attention_weight_min": -22.468870162963867,
      "activations/layer21_attention_weight_max": 38.23109817504883,
      "activations/layer21_attention_weight_min": -23.461645126342773,
      "activations/layer22_attention_weight_max": 38.345237731933594,
      "activations/layer22_attention_weight_min": -26.91993522644043,
      "activations/layer23_attention_weight_max": 34.75670623779297,
      "activations/layer23_attention_weight_min": -23.67360496520996,
      "activations/layer2_attention_weight_max": 33.25803756713867,
      "activations/layer2_attention_weight_min": -29.646909713745117,
      "activations/layer3_attention_weight_max": 86.060546875,
      "activations/layer3_attention_weight_min": -88.17079162597656,
      "activations/layer4_attention_weight_max": 97.9241714477539,
      "activations/layer4_attention_weight_min": -95.95782470703125,
      "activations/layer5_attention_weight_max": 85.1487045288086,
      "activations/layer5_attention_weight_min": -80.6871109008789,
      "activations/layer6_attention_weight_max": 52.212074279785156,
      "activations/layer6_attention_weight_min": -53.135135650634766,
      "activations/layer7_attention_weight_max": 73.57377624511719,
      "activations/layer7_attention_weight_min": -74.193359375,
      "activations/layer8_attention_weight_max": 44.81816482543945,
      "activations/layer8_attention_weight_min": -49.461769104003906,
      "activations/layer9_attention_weight_max": 48.274658203125,
      "activations/layer9_attention_weight_min": -48.02861022949219,
      "epoch": 7.76,
      "learning_rate": 0.00010099318181818181,
      "loss": 2.8139,
      "step": 133500
    },
    {
      "activations/layer0_attention_weight_max": 15.72021198272705,
      "activations/layer0_attention_weight_min": -14.813812255859375,
      "activations/layer10_attention_weight_max": 33.2908935546875,
      "activations/layer10_attention_weight_min": -33.4857063293457,
      "activations/layer11_attention_weight_max": 28.542943954467773,
      "activations/layer11_attention_weight_min": -29.045291900634766,
      "activations/layer12_attention_weight_max": 19.024803161621094,
      "activations/layer12_attention_weight_min": -26.721349716186523,
      "activations/layer13_attention_weight_max": 41.232261657714844,
      "activations/layer13_attention_weight_min": -31.021385192871094,
      "activations/layer14_attention_weight_max": 36.09815979003906,
      "activations/layer14_attention_weight_min": -30.834714889526367,
      "activations/layer15_attention_weight_max": 33.47947692871094,
      "activations/layer15_attention_weight_min": -28.638877868652344,
      "activations/layer16_attention_weight_max": 31.26285743713379,
      "activations/layer16_attention_weight_min": -28.292465209960938,
      "activations/layer17_attention_weight_max": 55.2763786315918,
      "activations/layer17_attention_weight_min": -42.41322326660156,
      "activations/layer18_attention_weight_max": 45.13732147216797,
      "activations/layer18_attention_weight_min": -36.18098068237305,
      "activations/layer19_attention_weight_max": 20.452566146850586,
      "activations/layer19_attention_weight_min": -20.29344367980957,
      "activations/layer1_attention_weight_max": 17.017478942871094,
      "activations/layer1_attention_weight_min": -14.417987823486328,
      "activations/layer20_attention_weight_max": 19.373075485229492,
      "activations/layer20_attention_weight_min": -22.244897842407227,
      "activations/layer21_attention_weight_max": 39.167720794677734,
      "activations/layer21_attention_weight_min": -22.954214096069336,
      "activations/layer22_attention_weight_max": 32.29594802856445,
      "activations/layer22_attention_weight_min": -26.44860076904297,
      "activations/layer23_attention_weight_max": 35.385318756103516,
      "activations/layer23_attention_weight_min": -23.63239097595215,
      "activations/layer2_attention_weight_max": 33.70283508300781,
      "activations/layer2_attention_weight_min": -31.46579360961914,
      "activations/layer3_attention_weight_max": 88.20638275146484,
      "activations/layer3_attention_weight_min": -86.27813720703125,
      "activations/layer4_attention_weight_max": 100.7152328491211,
      "activations/layer4_attention_weight_min": -93.32772064208984,
      "activations/layer5_attention_weight_max": 83.13327026367188,
      "activations/layer5_attention_weight_min": -78.53953552246094,
      "activations/layer6_attention_weight_max": 52.70845031738281,
      "activations/layer6_attention_weight_min": -51.76055145263672,
      "activations/layer7_attention_weight_max": 73.59912109375,
      "activations/layer7_attention_weight_min": -69.74771881103516,
      "activations/layer8_attention_weight_max": 41.58797073364258,
      "activations/layer8_attention_weight_min": -42.356258392333984,
      "activations/layer9_attention_weight_max": 47.97858428955078,
      "activations/layer9_attention_weight_min": -46.37726974487305,
      "epoch": 7.76,
      "learning_rate": 0.00010097424242424242,
      "loss": 2.8076,
      "step": 133550
    },
    {
      "activations/layer0_attention_weight_max": 15.02959156036377,
      "activations/layer0_attention_weight_min": -14.620644569396973,
      "activations/layer10_attention_weight_max": 39.6719856262207,
      "activations/layer10_attention_weight_min": -38.70085144042969,
      "activations/layer11_attention_weight_max": 35.15705108642578,
      "activations/layer11_attention_weight_min": -32.301239013671875,
      "activations/layer12_attention_weight_max": 19.889123916625977,
      "activations/layer12_attention_weight_min": -23.615625381469727,
      "activations/layer13_attention_weight_max": 45.147377014160156,
      "activations/layer13_attention_weight_min": -32.023681640625,
      "activations/layer14_attention_weight_max": 38.36327362060547,
      "activations/layer14_attention_weight_min": -33.522613525390625,
      "activations/layer15_attention_weight_max": 34.0431022644043,
      "activations/layer15_attention_weight_min": -28.383827209472656,
      "activations/layer16_attention_weight_max": 32.10736846923828,
      "activations/layer16_attention_weight_min": -26.253068923950195,
      "activations/layer17_attention_weight_max": 50.0889892578125,
      "activations/layer17_attention_weight_min": -39.40622329711914,
      "activations/layer18_attention_weight_max": 45.381317138671875,
      "activations/layer18_attention_weight_min": -34.666290283203125,
      "activations/layer19_attention_weight_max": 24.20534896850586,
      "activations/layer19_attention_weight_min": -19.882131576538086,
      "activations/layer1_attention_weight_max": 16.906869888305664,
      "activations/layer1_attention_weight_min": -13.729676246643066,
      "activations/layer20_attention_weight_max": 21.747180938720703,
      "activations/layer20_attention_weight_min": -20.72349739074707,
      "activations/layer21_attention_weight_max": 41.3540153503418,
      "activations/layer21_attention_weight_min": -25.688980102539062,
      "activations/layer22_attention_weight_max": 35.93394470214844,
      "activations/layer22_attention_weight_min": -25.79277229309082,
      "activations/layer23_attention_weight_max": 33.74647521972656,
      "activations/layer23_attention_weight_min": -21.518207550048828,
      "activations/layer2_attention_weight_max": 33.848731994628906,
      "activations/layer2_attention_weight_min": -29.64324951171875,
      "activations/layer3_attention_weight_max": 88.72254180908203,
      "activations/layer3_attention_weight_min": -89.61172485351562,
      "activations/layer4_attention_weight_max": 106.6283950805664,
      "activations/layer4_attention_weight_min": -101.48888397216797,
      "activations/layer5_attention_weight_max": 91.95016479492188,
      "activations/layer5_attention_weight_min": -83.08665466308594,
      "activations/layer6_attention_weight_max": 56.64225769042969,
      "activations/layer6_attention_weight_min": -55.377052307128906,
      "activations/layer7_attention_weight_max": 80.58419036865234,
      "activations/layer7_attention_weight_min": -73.34201049804688,
      "activations/layer8_attention_weight_max": 45.35849380493164,
      "activations/layer8_attention_weight_min": -47.433990478515625,
      "activations/layer9_attention_weight_max": 56.69388961791992,
      "activations/layer9_attention_weight_min": -49.49699020385742,
      "epoch": 7.76,
      "learning_rate": 0.00010095530303030301,
      "loss": 2.814,
      "step": 133600
    },
    {
      "activations/layer0_attention_weight_max": 14.745623588562012,
      "activations/layer0_attention_weight_min": -14.889387130737305,
      "activations/layer10_attention_weight_max": 38.18772888183594,
      "activations/layer10_attention_weight_min": -35.1586799621582,
      "activations/layer11_attention_weight_max": 31.797271728515625,
      "activations/layer11_attention_weight_min": -31.226970672607422,
      "activations/layer12_attention_weight_max": 24.48513412475586,
      "activations/layer12_attention_weight_min": -24.2828426361084,
      "activations/layer13_attention_weight_max": 46.63496398925781,
      "activations/layer13_attention_weight_min": -33.149776458740234,
      "activations/layer14_attention_weight_max": 43.921546936035156,
      "activations/layer14_attention_weight_min": -30.52959632873535,
      "activations/layer15_attention_weight_max": 38.271942138671875,
      "activations/layer15_attention_weight_min": -30.39336395263672,
      "activations/layer16_attention_weight_max": 38.99341583251953,
      "activations/layer16_attention_weight_min": -28.664134979248047,
      "activations/layer17_attention_weight_max": 57.87343215942383,
      "activations/layer17_attention_weight_min": -44.62131118774414,
      "activations/layer18_attention_weight_max": 50.23137664794922,
      "activations/layer18_attention_weight_min": -36.97636413574219,
      "activations/layer19_attention_weight_max": 24.983545303344727,
      "activations/layer19_attention_weight_min": -22.668968200683594,
      "activations/layer1_attention_weight_max": 18.170623779296875,
      "activations/layer1_attention_weight_min": -13.83899974822998,
      "activations/layer20_attention_weight_max": 22.71453285217285,
      "activations/layer20_attention_weight_min": -23.320425033569336,
      "activations/layer21_attention_weight_max": 39.173702239990234,
      "activations/layer21_attention_weight_min": -23.692241668701172,
      "activations/layer22_attention_weight_max": 33.30482864379883,
      "activations/layer22_attention_weight_min": -25.36734962463379,
      "activations/layer23_attention_weight_max": 34.7325439453125,
      "activations/layer23_attention_weight_min": -21.332252502441406,
      "activations/layer2_attention_weight_max": 31.240135192871094,
      "activations/layer2_attention_weight_min": -29.613492965698242,
      "activations/layer3_attention_weight_max": 85.94123840332031,
      "activations/layer3_attention_weight_min": -85.86162567138672,
      "activations/layer4_attention_weight_max": 97.65886688232422,
      "activations/layer4_attention_weight_min": -95.76480865478516,
      "activations/layer5_attention_weight_max": 85.34895324707031,
      "activations/layer5_attention_weight_min": -80.0644302368164,
      "activations/layer6_attention_weight_max": 53.41378402709961,
      "activations/layer6_attention_weight_min": -54.78477096557617,
      "activations/layer7_attention_weight_max": 76.2094955444336,
      "activations/layer7_attention_weight_min": -77.20367431640625,
      "activations/layer8_attention_weight_max": 42.81150817871094,
      "activations/layer8_attention_weight_min": -44.500553131103516,
      "activations/layer9_attention_weight_max": 52.403968811035156,
      "activations/layer9_attention_weight_min": -48.924198150634766,
      "epoch": 7.77,
      "learning_rate": 0.00010093636363636363,
      "loss": 2.8185,
      "step": 133650
    },
    {
      "activations/layer0_attention_weight_max": 15.399032592773438,
      "activations/layer0_attention_weight_min": -14.47094440460205,
      "activations/layer10_attention_weight_max": 39.50122833251953,
      "activations/layer10_attention_weight_min": -38.2672004699707,
      "activations/layer11_attention_weight_max": 30.516948699951172,
      "activations/layer11_attention_weight_min": -32.332969665527344,
      "activations/layer12_attention_weight_max": 21.787662506103516,
      "activations/layer12_attention_weight_min": -23.116479873657227,
      "activations/layer13_attention_weight_max": 44.34654998779297,
      "activations/layer13_attention_weight_min": -30.459774017333984,
      "activations/layer14_attention_weight_max": 53.323760986328125,
      "activations/layer14_attention_weight_min": -30.42584228515625,
      "activations/layer15_attention_weight_max": 39.242366790771484,
      "activations/layer15_attention_weight_min": -28.634143829345703,
      "activations/layer16_attention_weight_max": 32.949951171875,
      "activations/layer16_attention_weight_min": -27.81622314453125,
      "activations/layer17_attention_weight_max": 56.084373474121094,
      "activations/layer17_attention_weight_min": -41.66462326049805,
      "activations/layer18_attention_weight_max": 52.270751953125,
      "activations/layer18_attention_weight_min": -34.17043685913086,
      "activations/layer19_attention_weight_max": 25.065431594848633,
      "activations/layer19_attention_weight_min": -20.09586524963379,
      "activations/layer1_attention_weight_max": 16.741361618041992,
      "activations/layer1_attention_weight_min": -14.022242546081543,
      "activations/layer20_attention_weight_max": 23.531965255737305,
      "activations/layer20_attention_weight_min": -20.194700241088867,
      "activations/layer21_attention_weight_max": 41.7912712097168,
      "activations/layer21_attention_weight_min": -25.175662994384766,
      "activations/layer22_attention_weight_max": 36.30628967285156,
      "activations/layer22_attention_weight_min": -23.47909164428711,
      "activations/layer23_attention_weight_max": 38.749794006347656,
      "activations/layer23_attention_weight_min": -21.358036041259766,
      "activations/layer2_attention_weight_max": 32.167694091796875,
      "activations/layer2_attention_weight_min": -31.533870697021484,
      "activations/layer3_attention_weight_max": 86.2185287475586,
      "activations/layer3_attention_weight_min": -88.72466278076172,
      "activations/layer4_attention_weight_max": 103.67142486572266,
      "activations/layer4_attention_weight_min": -96.0814208984375,
      "activations/layer5_attention_weight_max": 90.30369567871094,
      "activations/layer5_attention_weight_min": -81.18247985839844,
      "activations/layer6_attention_weight_max": 55.274742126464844,
      "activations/layer6_attention_weight_min": -55.39814758300781,
      "activations/layer7_attention_weight_max": 77.45207214355469,
      "activations/layer7_attention_weight_min": -76.45916748046875,
      "activations/layer8_attention_weight_max": 48.0433235168457,
      "activations/layer8_attention_weight_min": -50.7444953918457,
      "activations/layer9_attention_weight_max": 53.82954025268555,
      "activations/layer9_attention_weight_min": -49.44801712036133,
      "epoch": 7.77,
      "learning_rate": 0.00010091742424242424,
      "loss": 2.804,
      "step": 133700
    },
    {
      "activations/layer0_attention_weight_max": 14.822083473205566,
      "activations/layer0_attention_weight_min": -14.106661796569824,
      "activations/layer10_attention_weight_max": 37.02353286743164,
      "activations/layer10_attention_weight_min": -36.8277587890625,
      "activations/layer11_attention_weight_max": 31.725139617919922,
      "activations/layer11_attention_weight_min": -31.459728240966797,
      "activations/layer12_attention_weight_max": 23.37464714050293,
      "activations/layer12_attention_weight_min": -26.062824249267578,
      "activations/layer13_attention_weight_max": 53.52547073364258,
      "activations/layer13_attention_weight_min": -34.876625061035156,
      "activations/layer14_attention_weight_max": 55.003761291503906,
      "activations/layer14_attention_weight_min": -30.160884857177734,
      "activations/layer15_attention_weight_max": 37.35047912597656,
      "activations/layer15_attention_weight_min": -28.94222640991211,
      "activations/layer16_attention_weight_max": 40.02216720581055,
      "activations/layer16_attention_weight_min": -27.129440307617188,
      "activations/layer17_attention_weight_max": 57.93096160888672,
      "activations/layer17_attention_weight_min": -40.701412200927734,
      "activations/layer18_attention_weight_max": 48.02363967895508,
      "activations/layer18_attention_weight_min": -35.48426818847656,
      "activations/layer19_attention_weight_max": 21.696765899658203,
      "activations/layer19_attention_weight_min": -23.38199234008789,
      "activations/layer1_attention_weight_max": 17.46030044555664,
      "activations/layer1_attention_weight_min": -14.867741584777832,
      "activations/layer20_attention_weight_max": 23.82794952392578,
      "activations/layer20_attention_weight_min": -22.714262008666992,
      "activations/layer21_attention_weight_max": 43.6004638671875,
      "activations/layer21_attention_weight_min": -24.569278717041016,
      "activations/layer22_attention_weight_max": 31.91114044189453,
      "activations/layer22_attention_weight_min": -26.124755859375,
      "activations/layer23_attention_weight_max": 36.82545471191406,
      "activations/layer23_attention_weight_min": -21.249570846557617,
      "activations/layer2_attention_weight_max": 31.19814682006836,
      "activations/layer2_attention_weight_min": -30.235397338867188,
      "activations/layer3_attention_weight_max": 88.75196838378906,
      "activations/layer3_attention_weight_min": -91.26995849609375,
      "activations/layer4_attention_weight_max": 103.5338134765625,
      "activations/layer4_attention_weight_min": -94.43183135986328,
      "activations/layer5_attention_weight_max": 85.26935577392578,
      "activations/layer5_attention_weight_min": -78.03052520751953,
      "activations/layer6_attention_weight_max": 55.2344856262207,
      "activations/layer6_attention_weight_min": -52.697914123535156,
      "activations/layer7_attention_weight_max": 77.96187591552734,
      "activations/layer7_attention_weight_min": -74.27408599853516,
      "activations/layer8_attention_weight_max": 44.14702224731445,
      "activations/layer8_attention_weight_min": -46.02824783325195,
      "activations/layer9_attention_weight_max": 53.80082321166992,
      "activations/layer9_attention_weight_min": -50.43170928955078,
      "epoch": 7.77,
      "learning_rate": 0.00010089848484848483,
      "loss": 2.8143,
      "step": 133750
    },
    {
      "activations/layer0_attention_weight_max": 14.56005859375,
      "activations/layer0_attention_weight_min": -14.975972175598145,
      "activations/layer10_attention_weight_max": 39.71831512451172,
      "activations/layer10_attention_weight_min": -35.52191925048828,
      "activations/layer11_attention_weight_max": 33.6062126159668,
      "activations/layer11_attention_weight_min": -30.254703521728516,
      "activations/layer12_attention_weight_max": 28.854633331298828,
      "activations/layer12_attention_weight_min": -25.452213287353516,
      "activations/layer13_attention_weight_max": 51.44139099121094,
      "activations/layer13_attention_weight_min": -35.072608947753906,
      "activations/layer14_attention_weight_max": 43.86216735839844,
      "activations/layer14_attention_weight_min": -31.655376434326172,
      "activations/layer15_attention_weight_max": 36.294334411621094,
      "activations/layer15_attention_weight_min": -30.377479553222656,
      "activations/layer16_attention_weight_max": 32.1765251159668,
      "activations/layer16_attention_weight_min": -30.75518035888672,
      "activations/layer17_attention_weight_max": 53.66088104248047,
      "activations/layer17_attention_weight_min": -41.9671745300293,
      "activations/layer18_attention_weight_max": 45.939453125,
      "activations/layer18_attention_weight_min": -35.49455642700195,
      "activations/layer19_attention_weight_max": 23.83513069152832,
      "activations/layer19_attention_weight_min": -20.7762393951416,
      "activations/layer1_attention_weight_max": 17.268266677856445,
      "activations/layer1_attention_weight_min": -16.864839553833008,
      "activations/layer20_attention_weight_max": 21.461244583129883,
      "activations/layer20_attention_weight_min": -22.703670501708984,
      "activations/layer21_attention_weight_max": 41.06794357299805,
      "activations/layer21_attention_weight_min": -23.967702865600586,
      "activations/layer22_attention_weight_max": 34.046382904052734,
      "activations/layer22_attention_weight_min": -23.61298942565918,
      "activations/layer23_attention_weight_max": 35.655067443847656,
      "activations/layer23_attention_weight_min": -22.088659286499023,
      "activations/layer2_attention_weight_max": 33.40870666503906,
      "activations/layer2_attention_weight_min": -31.39181137084961,
      "activations/layer3_attention_weight_max": 89.8625259399414,
      "activations/layer3_attention_weight_min": -89.80569458007812,
      "activations/layer4_attention_weight_max": 100.9866943359375,
      "activations/layer4_attention_weight_min": -94.34468841552734,
      "activations/layer5_attention_weight_max": 89.257568359375,
      "activations/layer5_attention_weight_min": -80.77558135986328,
      "activations/layer6_attention_weight_max": 57.05743408203125,
      "activations/layer6_attention_weight_min": -51.930904388427734,
      "activations/layer7_attention_weight_max": 76.1690444946289,
      "activations/layer7_attention_weight_min": -70.92670440673828,
      "activations/layer8_attention_weight_max": 48.37991714477539,
      "activations/layer8_attention_weight_min": -47.0670051574707,
      "activations/layer9_attention_weight_max": 53.1395263671875,
      "activations/layer9_attention_weight_min": -47.66410446166992,
      "epoch": 7.77,
      "learning_rate": 0.00010087954545454544,
      "loss": 2.8181,
      "step": 133800
    },
    {
      "activations/layer0_attention_weight_max": 14.244376182556152,
      "activations/layer0_attention_weight_min": -14.321946144104004,
      "activations/layer10_attention_weight_max": 45.718406677246094,
      "activations/layer10_attention_weight_min": -41.44549560546875,
      "activations/layer11_attention_weight_max": 38.43147277832031,
      "activations/layer11_attention_weight_min": -35.72042465209961,
      "activations/layer12_attention_weight_max": 21.596418380737305,
      "activations/layer12_attention_weight_min": -29.44397735595703,
      "activations/layer13_attention_weight_max": 40.90081787109375,
      "activations/layer13_attention_weight_min": -31.38068389892578,
      "activations/layer14_attention_weight_max": 41.575775146484375,
      "activations/layer14_attention_weight_min": -35.5379753112793,
      "activations/layer15_attention_weight_max": 37.382362365722656,
      "activations/layer15_attention_weight_min": -30.382686614990234,
      "activations/layer16_attention_weight_max": 30.516357421875,
      "activations/layer16_attention_weight_min": -28.46507453918457,
      "activations/layer17_attention_weight_max": 54.7773323059082,
      "activations/layer17_attention_weight_min": -46.06538772583008,
      "activations/layer18_attention_weight_max": 47.91091537475586,
      "activations/layer18_attention_weight_min": -41.37345504760742,
      "activations/layer19_attention_weight_max": 25.543882369995117,
      "activations/layer19_attention_weight_min": -22.631450653076172,
      "activations/layer1_attention_weight_max": 17.570398330688477,
      "activations/layer1_attention_weight_min": -16.75800323486328,
      "activations/layer20_attention_weight_max": 21.538454055786133,
      "activations/layer20_attention_weight_min": -22.398948669433594,
      "activations/layer21_attention_weight_max": 41.25799560546875,
      "activations/layer21_attention_weight_min": -26.53965950012207,
      "activations/layer22_attention_weight_max": 35.1331787109375,
      "activations/layer22_attention_weight_min": -26.910181045532227,
      "activations/layer23_attention_weight_max": 36.75456619262695,
      "activations/layer23_attention_weight_min": -25.93446922302246,
      "activations/layer2_attention_weight_max": 31.05712127685547,
      "activations/layer2_attention_weight_min": -30.03567886352539,
      "activations/layer3_attention_weight_max": 88.9420166015625,
      "activations/layer3_attention_weight_min": -89.09866333007812,
      "activations/layer4_attention_weight_max": 103.9639663696289,
      "activations/layer4_attention_weight_min": -97.32296752929688,
      "activations/layer5_attention_weight_max": 91.18580627441406,
      "activations/layer5_attention_weight_min": -81.71406555175781,
      "activations/layer6_attention_weight_max": 59.93245315551758,
      "activations/layer6_attention_weight_min": -54.78617858886719,
      "activations/layer7_attention_weight_max": 81.02621459960938,
      "activations/layer7_attention_weight_min": -79.3799819946289,
      "activations/layer8_attention_weight_max": 53.592777252197266,
      "activations/layer8_attention_weight_min": -52.96852493286133,
      "activations/layer9_attention_weight_max": 58.401241302490234,
      "activations/layer9_attention_weight_min": -54.72900390625,
      "epoch": 7.78,
      "learning_rate": 0.00010086098484848483,
      "loss": 2.8156,
      "step": 133850
    },
    {
      "activations/layer0_attention_weight_max": 16.871089935302734,
      "activations/layer0_attention_weight_min": -14.25300121307373,
      "activations/layer10_attention_weight_max": 37.69525146484375,
      "activations/layer10_attention_weight_min": -36.53611755371094,
      "activations/layer11_attention_weight_max": 31.026662826538086,
      "activations/layer11_attention_weight_min": -31.019119262695312,
      "activations/layer12_attention_weight_max": 19.004215240478516,
      "activations/layer12_attention_weight_min": -27.156375885009766,
      "activations/layer13_attention_weight_max": 42.19206237792969,
      "activations/layer13_attention_weight_min": -35.25218200683594,
      "activations/layer14_attention_weight_max": 38.981040954589844,
      "activations/layer14_attention_weight_min": -31.77250099182129,
      "activations/layer15_attention_weight_max": 34.52473449707031,
      "activations/layer15_attention_weight_min": -29.471601486206055,
      "activations/layer16_attention_weight_max": 31.55058479309082,
      "activations/layer16_attention_weight_min": -28.5156192779541,
      "activations/layer17_attention_weight_max": 57.97802734375,
      "activations/layer17_attention_weight_min": -44.41845703125,
      "activations/layer18_attention_weight_max": 52.106204986572266,
      "activations/layer18_attention_weight_min": -38.37187957763672,
      "activations/layer19_attention_weight_max": 22.91595458984375,
      "activations/layer19_attention_weight_min": -21.231487274169922,
      "activations/layer1_attention_weight_max": 16.251724243164062,
      "activations/layer1_attention_weight_min": -15.619154930114746,
      "activations/layer20_attention_weight_max": 22.424345016479492,
      "activations/layer20_attention_weight_min": -23.662490844726562,
      "activations/layer21_attention_weight_max": 45.40237045288086,
      "activations/layer21_attention_weight_min": -25.889774322509766,
      "activations/layer22_attention_weight_max": 32.5498161315918,
      "activations/layer22_attention_weight_min": -27.929035186767578,
      "activations/layer23_attention_weight_max": 34.11213684082031,
      "activations/layer23_attention_weight_min": -19.601951599121094,
      "activations/layer2_attention_weight_max": 32.664947509765625,
      "activations/layer2_attention_weight_min": -29.862333297729492,
      "activations/layer3_attention_weight_max": 89.92572784423828,
      "activations/layer3_attention_weight_min": -92.48805236816406,
      "activations/layer4_attention_weight_max": 103.61454010009766,
      "activations/layer4_attention_weight_min": -95.67326354980469,
      "activations/layer5_attention_weight_max": 88.1380844116211,
      "activations/layer5_attention_weight_min": -81.04495239257812,
      "activations/layer6_attention_weight_max": 55.88127136230469,
      "activations/layer6_attention_weight_min": -54.1628303527832,
      "activations/layer7_attention_weight_max": 73.41854095458984,
      "activations/layer7_attention_weight_min": -79.32160186767578,
      "activations/layer8_attention_weight_max": 45.645652770996094,
      "activations/layer8_attention_weight_min": -46.167823791503906,
      "activations/layer9_attention_weight_max": 53.8064079284668,
      "activations/layer9_attention_weight_min": -47.81882858276367,
      "epoch": 7.78,
      "learning_rate": 0.00010084204545454545,
      "loss": 2.8216,
      "step": 133900
    },
    {
      "activations/layer0_attention_weight_max": 15.286934852600098,
      "activations/layer0_attention_weight_min": -14.598573684692383,
      "activations/layer10_attention_weight_max": 37.800350189208984,
      "activations/layer10_attention_weight_min": -36.85652542114258,
      "activations/layer11_attention_weight_max": 32.19453430175781,
      "activations/layer11_attention_weight_min": -31.76757049560547,
      "activations/layer12_attention_weight_max": 20.4140567779541,
      "activations/layer12_attention_weight_min": -24.713829040527344,
      "activations/layer13_attention_weight_max": 41.62961959838867,
      "activations/layer13_attention_weight_min": -28.88800048828125,
      "activations/layer14_attention_weight_max": 38.480438232421875,
      "activations/layer14_attention_weight_min": -30.475252151489258,
      "activations/layer15_attention_weight_max": 33.157997131347656,
      "activations/layer15_attention_weight_min": -28.039915084838867,
      "activations/layer16_attention_weight_max": 33.02379608154297,
      "activations/layer16_attention_weight_min": -28.757484436035156,
      "activations/layer17_attention_weight_max": 51.32570266723633,
      "activations/layer17_attention_weight_min": -39.16070556640625,
      "activations/layer18_attention_weight_max": 43.764259338378906,
      "activations/layer18_attention_weight_min": -33.61811828613281,
      "activations/layer19_attention_weight_max": 19.91468048095703,
      "activations/layer19_attention_weight_min": -21.056241989135742,
      "activations/layer1_attention_weight_max": 17.5433292388916,
      "activations/layer1_attention_weight_min": -15.920947074890137,
      "activations/layer20_attention_weight_max": 19.807510375976562,
      "activations/layer20_attention_weight_min": -23.62420654296875,
      "activations/layer21_attention_weight_max": 33.564666748046875,
      "activations/layer21_attention_weight_min": -21.478713989257812,
      "activations/layer22_attention_weight_max": 30.55853843688965,
      "activations/layer22_attention_weight_min": -27.0664119720459,
      "activations/layer23_attention_weight_max": 31.2055721282959,
      "activations/layer23_attention_weight_min": -19.514928817749023,
      "activations/layer2_attention_weight_max": 31.388463973999023,
      "activations/layer2_attention_weight_min": -29.644512176513672,
      "activations/layer3_attention_weight_max": 83.60272979736328,
      "activations/layer3_attention_weight_min": -87.59650421142578,
      "activations/layer4_attention_weight_max": 99.99193572998047,
      "activations/layer4_attention_weight_min": -93.57927703857422,
      "activations/layer5_attention_weight_max": 87.98231506347656,
      "activations/layer5_attention_weight_min": -81.25035095214844,
      "activations/layer6_attention_weight_max": 57.3292350769043,
      "activations/layer6_attention_weight_min": -52.94192886352539,
      "activations/layer7_attention_weight_max": 75.0612564086914,
      "activations/layer7_attention_weight_min": -70.7202377319336,
      "activations/layer8_attention_weight_max": 44.48948669433594,
      "activations/layer8_attention_weight_min": -45.597469329833984,
      "activations/layer9_attention_weight_max": 48.928524017333984,
      "activations/layer9_attention_weight_min": -46.0051383972168,
      "epoch": 7.78,
      "learning_rate": 0.00010082310606060605,
      "loss": 2.8018,
      "step": 133950
    },
    {
      "activations/layer0_attention_weight_max": 15.370811462402344,
      "activations/layer0_attention_weight_min": -14.257949829101562,
      "activations/layer10_attention_weight_max": 34.90951156616211,
      "activations/layer10_attention_weight_min": -34.028194427490234,
      "activations/layer11_attention_weight_max": 30.753528594970703,
      "activations/layer11_attention_weight_min": -28.999637603759766,
      "activations/layer12_attention_weight_max": 21.293474197387695,
      "activations/layer12_attention_weight_min": -25.79730796813965,
      "activations/layer13_attention_weight_max": 48.416221618652344,
      "activations/layer13_attention_weight_min": -30.814868927001953,
      "activations/layer14_attention_weight_max": 40.99227523803711,
      "activations/layer14_attention_weight_min": -29.655290603637695,
      "activations/layer15_attention_weight_max": 35.95643997192383,
      "activations/layer15_attention_weight_min": -29.595191955566406,
      "activations/layer16_attention_weight_max": 30.98197364807129,
      "activations/layer16_attention_weight_min": -27.772560119628906,
      "activations/layer17_attention_weight_max": 55.06110382080078,
      "activations/layer17_attention_weight_min": -43.93069839477539,
      "activations/layer18_attention_weight_max": 46.65119552612305,
      "activations/layer18_attention_weight_min": -34.82331466674805,
      "activations/layer19_attention_weight_max": 20.227893829345703,
      "activations/layer19_attention_weight_min": -22.194360733032227,
      "activations/layer1_attention_weight_max": 18.51600456237793,
      "activations/layer1_attention_weight_min": -18.01974868774414,
      "activations/layer20_attention_weight_max": 22.535680770874023,
      "activations/layer20_attention_weight_min": -22.51227378845215,
      "activations/layer21_attention_weight_max": 35.32390213012695,
      "activations/layer21_attention_weight_min": -24.268083572387695,
      "activations/layer22_attention_weight_max": 31.66606330871582,
      "activations/layer22_attention_weight_min": -25.23528480529785,
      "activations/layer23_attention_weight_max": 34.413360595703125,
      "activations/layer23_attention_weight_min": -20.79705810546875,
      "activations/layer2_attention_weight_max": 30.8403377532959,
      "activations/layer2_attention_weight_min": -28.638751983642578,
      "activations/layer3_attention_weight_max": 86.32002258300781,
      "activations/layer3_attention_weight_min": -87.2816162109375,
      "activations/layer4_attention_weight_max": 99.64946746826172,
      "activations/layer4_attention_weight_min": -98.02764129638672,
      "activations/layer5_attention_weight_max": 82.98753356933594,
      "activations/layer5_attention_weight_min": -81.68063354492188,
      "activations/layer6_attention_weight_max": 55.11170196533203,
      "activations/layer6_attention_weight_min": -52.79778289794922,
      "activations/layer7_attention_weight_max": 72.82426452636719,
      "activations/layer7_attention_weight_min": -72.77767181396484,
      "activations/layer8_attention_weight_max": 42.15058517456055,
      "activations/layer8_attention_weight_min": -44.97248458862305,
      "activations/layer9_attention_weight_max": 54.52863311767578,
      "activations/layer9_attention_weight_min": -45.67741394042969,
      "epoch": 7.79,
      "learning_rate": 0.00010080416666666665,
      "loss": 2.819,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5367,
      "eval_samples_per_second": 503.006,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5367,
      "eval_openwebtext_samples_per_second": 503.006,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_wikitext_loss": 2.994140625,
      "eval_wikitext_ppl": 19.96819234893607,
      "eval_wikitext_runtime": 2.0625,
      "eval_wikitext_samples_per_second": 221.094,
      "step": 134000
    },
    {
      "epoch": 7.79,
      "eval_lambada_loss": 2.732421875,
      "eval_lambada_ppl": 15.370066359341997,
      "eval_lambada_runtime": 9.7184,
      "eval_lambada_samples_per_second": 501.006,
      "step": 134000
    },
    {
      "activations/layer0_attention_weight_max": 14.513733863830566,
      "activations/layer0_attention_weight_min": -14.864580154418945,
      "activations/layer10_attention_weight_max": 39.722537994384766,
      "activations/layer10_attention_weight_min": -38.82685470581055,
      "activations/layer11_attention_weight_max": 31.997398376464844,
      "activations/layer11_attention_weight_min": -31.28972625732422,
      "activations/layer12_attention_weight_max": 20.026111602783203,
      "activations/layer12_attention_weight_min": -26.234437942504883,
      "activations/layer13_attention_weight_max": 49.09070587158203,
      "activations/layer13_attention_weight_min": -35.94166946411133,
      "activations/layer14_attention_weight_max": 42.187103271484375,
      "activations/layer14_attention_weight_min": -33.55378341674805,
      "activations/layer15_attention_weight_max": 38.07281494140625,
      "activations/layer15_attention_weight_min": -30.09008026123047,
      "activations/layer16_attention_weight_max": 35.51673889160156,
      "activations/layer16_attention_weight_min": -28.765151977539062,
      "activations/layer17_attention_weight_max": 56.02174758911133,
      "activations/layer17_attention_weight_min": -45.28410339355469,
      "activations/layer18_attention_weight_max": 48.068511962890625,
      "activations/layer18_attention_weight_min": -39.90985107421875,
      "activations/layer19_attention_weight_max": 23.17294692993164,
      "activations/layer19_attention_weight_min": -23.10340690612793,
      "activations/layer1_attention_weight_max": 16.58230209350586,
      "activations/layer1_attention_weight_min": -15.188329696655273,
      "activations/layer20_attention_weight_max": 25.85769271850586,
      "activations/layer20_attention_weight_min": -22.724992752075195,
      "activations/layer21_attention_weight_max": 41.33102798461914,
      "activations/layer21_attention_weight_min": -23.27293586730957,
      "activations/layer22_attention_weight_max": 34.00246047973633,
      "activations/layer22_attention_weight_min": -27.001014709472656,
      "activations/layer23_attention_weight_max": 35.8472785949707,
      "activations/layer23_attention_weight_min": -21.97939109802246,
      "activations/layer2_attention_weight_max": 33.083106994628906,
      "activations/layer2_attention_weight_min": -29.968494415283203,
      "activations/layer3_attention_weight_max": 90.95877075195312,
      "activations/layer3_attention_weight_min": -89.55642700195312,
      "activations/layer4_attention_weight_max": 108.2717056274414,
      "activations/layer4_attention_weight_min": -104.6157455444336,
      "activations/layer5_attention_weight_max": 94.67897033691406,
      "activations/layer5_attention_weight_min": -88.34810638427734,
      "activations/layer6_attention_weight_max": 60.78266525268555,
      "activations/layer6_attention_weight_min": -57.07807540893555,
      "activations/layer7_attention_weight_max": 80.57078552246094,
      "activations/layer7_attention_weight_min": -75.68486785888672,
      "activations/layer8_attention_weight_max": 50.41777420043945,
      "activations/layer8_attention_weight_min": -49.10173034667969,
      "activations/layer9_attention_weight_max": 54.45741653442383,
      "activations/layer9_attention_weight_min": -51.26372146606445,
      "epoch": 7.79,
      "learning_rate": 0.00010078522727272727,
      "loss": 2.8098,
      "step": 134050
    },
    {
      "activations/layer0_attention_weight_max": 14.956344604492188,
      "activations/layer0_attention_weight_min": -14.804430961608887,
      "activations/layer10_attention_weight_max": 37.582096099853516,
      "activations/layer10_attention_weight_min": -34.02936935424805,
      "activations/layer11_attention_weight_max": 30.880531311035156,
      "activations/layer11_attention_weight_min": -29.170866012573242,
      "activations/layer12_attention_weight_max": 19.927581787109375,
      "activations/layer12_attention_weight_min": -30.587814331054688,
      "activations/layer13_attention_weight_max": 49.29659652709961,
      "activations/layer13_attention_weight_min": -34.9766960144043,
      "activations/layer14_attention_weight_max": 38.75524139404297,
      "activations/layer14_attention_weight_min": -30.159805297851562,
      "activations/layer15_attention_weight_max": 40.71772766113281,
      "activations/layer15_attention_weight_min": -30.964921951293945,
      "activations/layer16_attention_weight_max": 33.66678237915039,
      "activations/layer16_attention_weight_min": -25.965417861938477,
      "activations/layer17_attention_weight_max": 53.662696838378906,
      "activations/layer17_attention_weight_min": -41.595062255859375,
      "activations/layer18_attention_weight_max": 44.8814582824707,
      "activations/layer18_attention_weight_min": -36.88419723510742,
      "activations/layer19_attention_weight_max": 19.428434371948242,
      "activations/layer19_attention_weight_min": -20.92284393310547,
      "activations/layer1_attention_weight_max": 17.006877899169922,
      "activations/layer1_attention_weight_min": -15.041874885559082,
      "activations/layer20_attention_weight_max": 20.047521591186523,
      "activations/layer20_attention_weight_min": -23.23317527770996,
      "activations/layer21_attention_weight_max": 31.52724838256836,
      "activations/layer21_attention_weight_min": -24.008895874023438,
      "activations/layer22_attention_weight_max": 32.258338928222656,
      "activations/layer22_attention_weight_min": -24.89578628540039,
      "activations/layer23_attention_weight_max": 34.38980484008789,
      "activations/layer23_attention_weight_min": -20.908645629882812,
      "activations/layer2_attention_weight_max": 32.458770751953125,
      "activations/layer2_attention_weight_min": -29.630155563354492,
      "activations/layer3_attention_weight_max": 85.63887023925781,
      "activations/layer3_attention_weight_min": -87.80770111083984,
      "activations/layer4_attention_weight_max": 96.06910705566406,
      "activations/layer4_attention_weight_min": -92.77206420898438,
      "activations/layer5_attention_weight_max": 82.1999282836914,
      "activations/layer5_attention_weight_min": -74.53947448730469,
      "activations/layer6_attention_weight_max": 52.37042999267578,
      "activations/layer6_attention_weight_min": -48.96409606933594,
      "activations/layer7_attention_weight_max": 73.97210693359375,
      "activations/layer7_attention_weight_min": -71.56956481933594,
      "activations/layer8_attention_weight_max": 44.792484283447266,
      "activations/layer8_attention_weight_min": -43.36043167114258,
      "activations/layer9_attention_weight_max": 49.00851058959961,
      "activations/layer9_attention_weight_min": -48.04187774658203,
      "epoch": 7.79,
      "learning_rate": 0.00010076628787878787,
      "loss": 2.8145,
      "step": 134100
    },
    {
      "activations/layer0_attention_weight_max": 15.879279136657715,
      "activations/layer0_attention_weight_min": -14.262003898620605,
      "activations/layer10_attention_weight_max": 39.879093170166016,
      "activations/layer10_attention_weight_min": -34.92274475097656,
      "activations/layer11_attention_weight_max": 34.4122200012207,
      "activations/layer11_attention_weight_min": -31.199050903320312,
      "activations/layer12_attention_weight_max": 20.99192237854004,
      "activations/layer12_attention_weight_min": -26.557382583618164,
      "activations/layer13_attention_weight_max": 46.71455383300781,
      "activations/layer13_attention_weight_min": -30.17636489868164,
      "activations/layer14_attention_weight_max": 51.87030029296875,
      "activations/layer14_attention_weight_min": -30.3109130859375,
      "activations/layer15_attention_weight_max": 39.687889099121094,
      "activations/layer15_attention_weight_min": -29.558794021606445,
      "activations/layer16_attention_weight_max": 30.86840057373047,
      "activations/layer16_attention_weight_min": -28.51124382019043,
      "activations/layer17_attention_weight_max": 56.387062072753906,
      "activations/layer17_attention_weight_min": -41.06890869140625,
      "activations/layer18_attention_weight_max": 48.51881408691406,
      "activations/layer18_attention_weight_min": -34.866085052490234,
      "activations/layer19_attention_weight_max": 22.088092803955078,
      "activations/layer19_attention_weight_min": -19.116188049316406,
      "activations/layer1_attention_weight_max": 17.084508895874023,
      "activations/layer1_attention_weight_min": -18.702919006347656,
      "activations/layer20_attention_weight_max": 20.806846618652344,
      "activations/layer20_attention_weight_min": -26.897371292114258,
      "activations/layer21_attention_weight_max": 35.469398498535156,
      "activations/layer21_attention_weight_min": -24.8557071685791,
      "activations/layer22_attention_weight_max": 33.53500747680664,
      "activations/layer22_attention_weight_min": -23.895200729370117,
      "activations/layer23_attention_weight_max": 34.825191497802734,
      "activations/layer23_attention_weight_min": -22.373149871826172,
      "activations/layer2_attention_weight_max": 33.24812316894531,
      "activations/layer2_attention_weight_min": -31.481670379638672,
      "activations/layer3_attention_weight_max": 93.16156768798828,
      "activations/layer3_attention_weight_min": -91.71833801269531,
      "activations/layer4_attention_weight_max": 106.6109848022461,
      "activations/layer4_attention_weight_min": -97.83154296875,
      "activations/layer5_attention_weight_max": 83.05874633789062,
      "activations/layer5_attention_weight_min": -74.01248931884766,
      "activations/layer6_attention_weight_max": 54.47386169433594,
      "activations/layer6_attention_weight_min": -50.78428649902344,
      "activations/layer7_attention_weight_max": 82.4925765991211,
      "activations/layer7_attention_weight_min": -69.5886459350586,
      "activations/layer8_attention_weight_max": 46.48219299316406,
      "activations/layer8_attention_weight_min": -44.054420471191406,
      "activations/layer9_attention_weight_max": 56.8446044921875,
      "activations/layer9_attention_weight_min": -51.53753662109375,
      "epoch": 7.79,
      "learning_rate": 0.00010074734848484848,
      "loss": 2.8404,
      "step": 134150
    },
    {
      "activations/layer0_attention_weight_max": 15.375712394714355,
      "activations/layer0_attention_weight_min": -13.898353576660156,
      "activations/layer10_attention_weight_max": 40.7518196105957,
      "activations/layer10_attention_weight_min": -41.39221954345703,
      "activations/layer11_attention_weight_max": 34.24595260620117,
      "activations/layer11_attention_weight_min": -30.93958282470703,
      "activations/layer12_attention_weight_max": 20.232318878173828,
      "activations/layer12_attention_weight_min": -25.2625675201416,
      "activations/layer13_attention_weight_max": 40.58674621582031,
      "activations/layer13_attention_weight_min": -29.04128646850586,
      "activations/layer14_attention_weight_max": 36.78030014038086,
      "activations/layer14_attention_weight_min": -27.967037200927734,
      "activations/layer15_attention_weight_max": 35.800636291503906,
      "activations/layer15_attention_weight_min": -28.31266975402832,
      "activations/layer16_attention_weight_max": 32.086341857910156,
      "activations/layer16_attention_weight_min": -29.003650665283203,
      "activations/layer17_attention_weight_max": 62.01376724243164,
      "activations/layer17_attention_weight_min": -41.31538772583008,
      "activations/layer18_attention_weight_max": 48.774723052978516,
      "activations/layer18_attention_weight_min": -36.184852600097656,
      "activations/layer19_attention_weight_max": 21.914012908935547,
      "activations/layer19_attention_weight_min": -20.34256362915039,
      "activations/layer1_attention_weight_max": 18.206100463867188,
      "activations/layer1_attention_weight_min": -16.413341522216797,
      "activations/layer20_attention_weight_max": 21.38994026184082,
      "activations/layer20_attention_weight_min": -20.204072952270508,
      "activations/layer21_attention_weight_max": 40.212928771972656,
      "activations/layer21_attention_weight_min": -25.153032302856445,
      "activations/layer22_attention_weight_max": 32.878623962402344,
      "activations/layer22_attention_weight_min": -23.709497451782227,
      "activations/layer23_attention_weight_max": 33.8593864440918,
      "activations/layer23_attention_weight_min": -22.245342254638672,
      "activations/layer2_attention_weight_max": 32.57157897949219,
      "activations/layer2_attention_weight_min": -32.35328674316406,
      "activations/layer3_attention_weight_max": 89.09381866455078,
      "activations/layer3_attention_weight_min": -91.72906494140625,
      "activations/layer4_attention_weight_max": 100.65803527832031,
      "activations/layer4_attention_weight_min": -97.51078796386719,
      "activations/layer5_attention_weight_max": 83.82353973388672,
      "activations/layer5_attention_weight_min": -81.36515808105469,
      "activations/layer6_attention_weight_max": 53.506290435791016,
      "activations/layer6_attention_weight_min": -55.15626525878906,
      "activations/layer7_attention_weight_max": 82.15179443359375,
      "activations/layer7_attention_weight_min": -73.56249237060547,
      "activations/layer8_attention_weight_max": 46.321937561035156,
      "activations/layer8_attention_weight_min": -45.47832107543945,
      "activations/layer9_attention_weight_max": 59.26607131958008,
      "activations/layer9_attention_weight_min": -55.435462951660156,
      "epoch": 7.8,
      "learning_rate": 0.00010072840909090909,
      "loss": 2.8209,
      "step": 134200
    },
    {
      "activations/layer0_attention_weight_max": 14.806913375854492,
      "activations/layer0_attention_weight_min": -14.158587455749512,
      "activations/layer10_attention_weight_max": 36.961055755615234,
      "activations/layer10_attention_weight_min": -35.59915542602539,
      "activations/layer11_attention_weight_max": 29.040693283081055,
      "activations/layer11_attention_weight_min": -29.5244140625,
      "activations/layer12_attention_weight_max": 25.353839874267578,
      "activations/layer12_attention_weight_min": -24.599952697753906,
      "activations/layer13_attention_weight_max": 42.52242660522461,
      "activations/layer13_attention_weight_min": -29.859909057617188,
      "activations/layer14_attention_weight_max": 36.85907745361328,
      "activations/layer14_attention_weight_min": -29.738170623779297,
      "activations/layer15_attention_weight_max": 33.820167541503906,
      "activations/layer15_attention_weight_min": -30.82158851623535,
      "activations/layer16_attention_weight_max": 29.478715896606445,
      "activations/layer16_attention_weight_min": -25.918149948120117,
      "activations/layer17_attention_weight_max": 51.61320495605469,
      "activations/layer17_attention_weight_min": -39.42660903930664,
      "activations/layer18_attention_weight_max": 46.162193298339844,
      "activations/layer18_attention_weight_min": -32.95671463012695,
      "activations/layer19_attention_weight_max": 20.452680587768555,
      "activations/layer19_attention_weight_min": -19.92400360107422,
      "activations/layer1_attention_weight_max": 18.156858444213867,
      "activations/layer1_attention_weight_min": -17.07691192626953,
      "activations/layer20_attention_weight_max": 21.300622940063477,
      "activations/layer20_attention_weight_min": -21.41128921508789,
      "activations/layer21_attention_weight_max": 34.47284698486328,
      "activations/layer21_attention_weight_min": -22.111268997192383,
      "activations/layer22_attention_weight_max": 30.862688064575195,
      "activations/layer22_attention_weight_min": -23.741230010986328,
      "activations/layer23_attention_weight_max": 29.28647232055664,
      "activations/layer23_attention_weight_min": -23.65963363647461,
      "activations/layer2_attention_weight_max": 33.70517349243164,
      "activations/layer2_attention_weight_min": -30.999893188476562,
      "activations/layer3_attention_weight_max": 86.90625,
      "activations/layer3_attention_weight_min": -93.0247573852539,
      "activations/layer4_attention_weight_max": 97.87811279296875,
      "activations/layer4_attention_weight_min": -94.38438415527344,
      "activations/layer5_attention_weight_max": 81.30963134765625,
      "activations/layer5_attention_weight_min": -77.42356872558594,
      "activations/layer6_attention_weight_max": 53.2564811706543,
      "activations/layer6_attention_weight_min": -51.18025588989258,
      "activations/layer7_attention_weight_max": 72.82649230957031,
      "activations/layer7_attention_weight_min": -66.81848907470703,
      "activations/layer8_attention_weight_max": 44.43916702270508,
      "activations/layer8_attention_weight_min": -42.95487976074219,
      "activations/layer9_attention_weight_max": 50.80537033081055,
      "activations/layer9_attention_weight_min": -47.26702880859375,
      "epoch": 7.8,
      "learning_rate": 0.00010070946969696969,
      "loss": 2.811,
      "step": 134250
    },
    {
      "activations/layer0_attention_weight_max": 16.174991607666016,
      "activations/layer0_attention_weight_min": -14.530098915100098,
      "activations/layer10_attention_weight_max": 40.75370407104492,
      "activations/layer10_attention_weight_min": -36.773250579833984,
      "activations/layer11_attention_weight_max": 33.689674377441406,
      "activations/layer11_attention_weight_min": -30.887210845947266,
      "activations/layer12_attention_weight_max": 19.73805809020996,
      "activations/layer12_attention_weight_min": -23.36607551574707,
      "activations/layer13_attention_weight_max": 46.98069381713867,
      "activations/layer13_attention_weight_min": -32.09038543701172,
      "activations/layer14_attention_weight_max": 43.860328674316406,
      "activations/layer14_attention_weight_min": -30.766328811645508,
      "activations/layer15_attention_weight_max": 37.03989791870117,
      "activations/layer15_attention_weight_min": -30.966941833496094,
      "activations/layer16_attention_weight_max": 31.602428436279297,
      "activations/layer16_attention_weight_min": -28.04954719543457,
      "activations/layer17_attention_weight_max": 58.97765350341797,
      "activations/layer17_attention_weight_min": -42.92673110961914,
      "activations/layer18_attention_weight_max": 48.44091033935547,
      "activations/layer18_attention_weight_min": -36.399776458740234,
      "activations/layer19_attention_weight_max": 22.419946670532227,
      "activations/layer19_attention_weight_min": -21.040802001953125,
      "activations/layer1_attention_weight_max": 17.842220306396484,
      "activations/layer1_attention_weight_min": -17.96942901611328,
      "activations/layer20_attention_weight_max": 21.415111541748047,
      "activations/layer20_attention_weight_min": -20.43105125427246,
      "activations/layer21_attention_weight_max": 40.306941986083984,
      "activations/layer21_attention_weight_min": -25.163890838623047,
      "activations/layer22_attention_weight_max": 34.1987190246582,
      "activations/layer22_attention_weight_min": -26.055274963378906,
      "activations/layer23_attention_weight_max": 36.671329498291016,
      "activations/layer23_attention_weight_min": -25.783077239990234,
      "activations/layer2_attention_weight_max": 31.249908447265625,
      "activations/layer2_attention_weight_min": -29.671981811523438,
      "activations/layer3_attention_weight_max": 82.8659439086914,
      "activations/layer3_attention_weight_min": -87.58230590820312,
      "activations/layer4_attention_weight_max": 101.922607421875,
      "activations/layer4_attention_weight_min": -97.24347686767578,
      "activations/layer5_attention_weight_max": 89.06437683105469,
      "activations/layer5_attention_weight_min": -83.11820983886719,
      "activations/layer6_attention_weight_max": 54.840309143066406,
      "activations/layer6_attention_weight_min": -54.31593704223633,
      "activations/layer7_attention_weight_max": 74.25918579101562,
      "activations/layer7_attention_weight_min": -79.04481506347656,
      "activations/layer8_attention_weight_max": 44.29888153076172,
      "activations/layer8_attention_weight_min": -44.694786071777344,
      "activations/layer9_attention_weight_max": 52.68915557861328,
      "activations/layer9_attention_weight_min": -52.21311569213867,
      "epoch": 7.8,
      "learning_rate": 0.0001006905303030303,
      "loss": 2.8084,
      "step": 134300
    },
    {
      "activations/layer0_attention_weight_max": 15.594548225402832,
      "activations/layer0_attention_weight_min": -14.69477367401123,
      "activations/layer10_attention_weight_max": 36.8421516418457,
      "activations/layer10_attention_weight_min": -36.85158920288086,
      "activations/layer11_attention_weight_max": 30.065425872802734,
      "activations/layer11_attention_weight_min": -30.366641998291016,
      "activations/layer12_attention_weight_max": 21.41359519958496,
      "activations/layer12_attention_weight_min": -29.819778442382812,
      "activations/layer13_attention_weight_max": 37.90911865234375,
      "activations/layer13_attention_weight_min": -26.830066680908203,
      "activations/layer14_attention_weight_max": 35.53919219970703,
      "activations/layer14_attention_weight_min": -29.29598617553711,
      "activations/layer15_attention_weight_max": 36.582847595214844,
      "activations/layer15_attention_weight_min": -28.9040584564209,
      "activations/layer16_attention_weight_max": 33.46588897705078,
      "activations/layer16_attention_weight_min": -27.39683723449707,
      "activations/layer17_attention_weight_max": 54.36762237548828,
      "activations/layer17_attention_weight_min": -43.175758361816406,
      "activations/layer18_attention_weight_max": 50.58954620361328,
      "activations/layer18_attention_weight_min": -38.86814880371094,
      "activations/layer19_attention_weight_max": 22.568096160888672,
      "activations/layer19_attention_weight_min": -24.466527938842773,
      "activations/layer1_attention_weight_max": 18.244787216186523,
      "activations/layer1_attention_weight_min": -15.747841835021973,
      "activations/layer20_attention_weight_max": 20.341856002807617,
      "activations/layer20_attention_weight_min": -24.58474349975586,
      "activations/layer21_attention_weight_max": 37.775821685791016,
      "activations/layer21_attention_weight_min": -22.50565528869629,
      "activations/layer22_attention_weight_max": 31.7972412109375,
      "activations/layer22_attention_weight_min": -26.427928924560547,
      "activations/layer23_attention_weight_max": 36.67170333862305,
      "activations/layer23_attention_weight_min": -23.979595184326172,
      "activations/layer2_attention_weight_max": 32.06375503540039,
      "activations/layer2_attention_weight_min": -32.07776641845703,
      "activations/layer3_attention_weight_max": 89.62501525878906,
      "activations/layer3_attention_weight_min": -90.58792877197266,
      "activations/layer4_attention_weight_max": 101.75272369384766,
      "activations/layer4_attention_weight_min": -95.96061706542969,
      "activations/layer5_attention_weight_max": 86.87175750732422,
      "activations/layer5_attention_weight_min": -80.49824523925781,
      "activations/layer6_attention_weight_max": 54.13504409790039,
      "activations/layer6_attention_weight_min": -54.34711456298828,
      "activations/layer7_attention_weight_max": 73.20132446289062,
      "activations/layer7_attention_weight_min": -69.86021423339844,
      "activations/layer8_attention_weight_max": 42.68027877807617,
      "activations/layer8_attention_weight_min": -45.31515884399414,
      "activations/layer9_attention_weight_max": 52.69646072387695,
      "activations/layer9_attention_weight_min": -56.18842315673828,
      "epoch": 7.81,
      "learning_rate": 0.00010067159090909089,
      "loss": 2.8301,
      "step": 134350
    },
    {
      "activations/layer0_attention_weight_max": 15.752370834350586,
      "activations/layer0_attention_weight_min": -15.130351066589355,
      "activations/layer10_attention_weight_max": 35.258663177490234,
      "activations/layer10_attention_weight_min": -31.785778045654297,
      "activations/layer11_attention_weight_max": 30.246795654296875,
      "activations/layer11_attention_weight_min": -27.591915130615234,
      "activations/layer12_attention_weight_max": 18.04269027709961,
      "activations/layer12_attention_weight_min": -25.394773483276367,
      "activations/layer13_attention_weight_max": 45.48288345336914,
      "activations/layer13_attention_weight_min": -34.1280517578125,
      "activations/layer14_attention_weight_max": 40.236106872558594,
      "activations/layer14_attention_weight_min": -30.075761795043945,
      "activations/layer15_attention_weight_max": 33.21073913574219,
      "activations/layer15_attention_weight_min": -29.981332778930664,
      "activations/layer16_attention_weight_max": 31.9248046875,
      "activations/layer16_attention_weight_min": -27.510923385620117,
      "activations/layer17_attention_weight_max": 54.48466491699219,
      "activations/layer17_attention_weight_min": -40.86015701293945,
      "activations/layer18_attention_weight_max": 45.109107971191406,
      "activations/layer18_attention_weight_min": -35.35931396484375,
      "activations/layer19_attention_weight_max": 22.613733291625977,
      "activations/layer19_attention_weight_min": -22.000728607177734,
      "activations/layer1_attention_weight_max": 17.65620994567871,
      "activations/layer1_attention_weight_min": -16.83220672607422,
      "activations/layer20_attention_weight_max": 21.156627655029297,
      "activations/layer20_attention_weight_min": -20.032743453979492,
      "activations/layer21_attention_weight_max": 40.02393341064453,
      "activations/layer21_attention_weight_min": -25.929277420043945,
      "activations/layer22_attention_weight_max": 31.300996780395508,
      "activations/layer22_attention_weight_min": -24.792139053344727,
      "activations/layer23_attention_weight_max": 33.486019134521484,
      "activations/layer23_attention_weight_min": -20.24370574951172,
      "activations/layer2_attention_weight_max": 31.165592193603516,
      "activations/layer2_attention_weight_min": -29.156368255615234,
      "activations/layer3_attention_weight_max": 85.83003997802734,
      "activations/layer3_attention_weight_min": -87.34173583984375,
      "activations/layer4_attention_weight_max": 96.29107666015625,
      "activations/layer4_attention_weight_min": -93.46236419677734,
      "activations/layer5_attention_weight_max": 81.06787109375,
      "activations/layer5_attention_weight_min": -75.04460144042969,
      "activations/layer6_attention_weight_max": 51.718772888183594,
      "activations/layer6_attention_weight_min": -50.18259811401367,
      "activations/layer7_attention_weight_max": 75.0246810913086,
      "activations/layer7_attention_weight_min": -69.07987976074219,
      "activations/layer8_attention_weight_max": 41.68614196777344,
      "activations/layer8_attention_weight_min": -42.27273941040039,
      "activations/layer9_attention_weight_max": 49.395503997802734,
      "activations/layer9_attention_weight_min": -46.47923278808594,
      "epoch": 7.81,
      "learning_rate": 0.0001006526515151515,
      "loss": 2.8102,
      "step": 134400
    },
    {
      "activations/layer0_attention_weight_max": 14.922008514404297,
      "activations/layer0_attention_weight_min": -14.221733093261719,
      "activations/layer10_attention_weight_max": 39.18524932861328,
      "activations/layer10_attention_weight_min": -36.43824005126953,
      "activations/layer11_attention_weight_max": 33.281307220458984,
      "activations/layer11_attention_weight_min": -31.907400131225586,
      "activations/layer12_attention_weight_max": 24.356916427612305,
      "activations/layer12_attention_weight_min": -27.645536422729492,
      "activations/layer13_attention_weight_max": 45.73607635498047,
      "activations/layer13_attention_weight_min": -31.426624298095703,
      "activations/layer14_attention_weight_max": 48.296348571777344,
      "activations/layer14_attention_weight_min": -31.405284881591797,
      "activations/layer15_attention_weight_max": 40.80064392089844,
      "activations/layer15_attention_weight_min": -28.62912940979004,
      "activations/layer16_attention_weight_max": 37.983097076416016,
      "activations/layer16_attention_weight_min": -30.429365158081055,
      "activations/layer17_attention_weight_max": 58.06938552856445,
      "activations/layer17_attention_weight_min": -44.984439849853516,
      "activations/layer18_attention_weight_max": 56.28420639038086,
      "activations/layer18_attention_weight_min": -38.712982177734375,
      "activations/layer19_attention_weight_max": 29.865140914916992,
      "activations/layer19_attention_weight_min": -22.466468811035156,
      "activations/layer1_attention_weight_max": 17.59674072265625,
      "activations/layer1_attention_weight_min": -17.713422775268555,
      "activations/layer20_attention_weight_max": 30.764385223388672,
      "activations/layer20_attention_weight_min": -21.911149978637695,
      "activations/layer21_attention_weight_max": 48.20524978637695,
      "activations/layer21_attention_weight_min": -25.448759078979492,
      "activations/layer22_attention_weight_max": 39.420127868652344,
      "activations/layer22_attention_weight_min": -25.582626342773438,
      "activations/layer23_attention_weight_max": 40.90884780883789,
      "activations/layer23_attention_weight_min": -21.594709396362305,
      "activations/layer2_attention_weight_max": 32.25849914550781,
      "activations/layer2_attention_weight_min": -29.4671630859375,
      "activations/layer3_attention_weight_max": 87.86862182617188,
      "activations/layer3_attention_weight_min": -91.5711669921875,
      "activations/layer4_attention_weight_max": 101.29376983642578,
      "activations/layer4_attention_weight_min": -94.6695327758789,
      "activations/layer5_attention_weight_max": 86.77471923828125,
      "activations/layer5_attention_weight_min": -79.14495086669922,
      "activations/layer6_attention_weight_max": 57.76061248779297,
      "activations/layer6_attention_weight_min": -53.340904235839844,
      "activations/layer7_attention_weight_max": 75.18743896484375,
      "activations/layer7_attention_weight_min": -73.82635498046875,
      "activations/layer8_attention_weight_max": 44.40606689453125,
      "activations/layer8_attention_weight_min": -47.18474578857422,
      "activations/layer9_attention_weight_max": 52.58966827392578,
      "activations/layer9_attention_weight_min": -48.1637077331543,
      "epoch": 7.81,
      "learning_rate": 0.00010063371212121212,
      "loss": 2.8178,
      "step": 134450
    },
    {
      "activations/layer0_attention_weight_max": 15.229357719421387,
      "activations/layer0_attention_weight_min": -14.194732666015625,
      "activations/layer10_attention_weight_max": 39.996490478515625,
      "activations/layer10_attention_weight_min": -36.65709686279297,
      "activations/layer11_attention_weight_max": 34.31134796142578,
      "activations/layer11_attention_weight_min": -31.472881317138672,
      "activations/layer12_attention_weight_max": 20.579267501831055,
      "activations/layer12_attention_weight_min": -23.600561141967773,
      "activations/layer13_attention_weight_max": 44.7237663269043,
      "activations/layer13_attention_weight_min": -33.83859634399414,
      "activations/layer14_attention_weight_max": 42.612693786621094,
      "activations/layer14_attention_weight_min": -32.292057037353516,
      "activations/layer15_attention_weight_max": 38.362239837646484,
      "activations/layer15_attention_weight_min": -30.520811080932617,
      "activations/layer16_attention_weight_max": 36.88418197631836,
      "activations/layer16_attention_weight_min": -29.114212036132812,
      "activations/layer17_attention_weight_max": 56.12199783325195,
      "activations/layer17_attention_weight_min": -42.54676055908203,
      "activations/layer18_attention_weight_max": 49.70555114746094,
      "activations/layer18_attention_weight_min": -37.55115509033203,
      "activations/layer19_attention_weight_max": 21.187353134155273,
      "activations/layer19_attention_weight_min": -20.994192123413086,
      "activations/layer1_attention_weight_max": 17.704395294189453,
      "activations/layer1_attention_weight_min": -16.251930236816406,
      "activations/layer20_attention_weight_max": 22.000085830688477,
      "activations/layer20_attention_weight_min": -21.688209533691406,
      "activations/layer21_attention_weight_max": 40.71747589111328,
      "activations/layer21_attention_weight_min": -24.806198120117188,
      "activations/layer22_attention_weight_max": 31.15644645690918,
      "activations/layer22_attention_weight_min": -24.3790225982666,
      "activations/layer23_attention_weight_max": 34.942604064941406,
      "activations/layer23_attention_weight_min": -20.997615814208984,
      "activations/layer2_attention_weight_max": 31.334569931030273,
      "activations/layer2_attention_weight_min": -30.413022994995117,
      "activations/layer3_attention_weight_max": 87.82261657714844,
      "activations/layer3_attention_weight_min": -92.05704498291016,
      "activations/layer4_attention_weight_max": 103.95013427734375,
      "activations/layer4_attention_weight_min": -93.28582763671875,
      "activations/layer5_attention_weight_max": 87.07463836669922,
      "activations/layer5_attention_weight_min": -79.98457336425781,
      "activations/layer6_attention_weight_max": 54.484962463378906,
      "activations/layer6_attention_weight_min": -51.792057037353516,
      "activations/layer7_attention_weight_max": 77.09696197509766,
      "activations/layer7_attention_weight_min": -71.51207733154297,
      "activations/layer8_attention_weight_max": 45.472408294677734,
      "activations/layer8_attention_weight_min": -46.771217346191406,
      "activations/layer9_attention_weight_max": 51.4682502746582,
      "activations/layer9_attention_weight_min": -50.04026412963867,
      "epoch": 7.82,
      "learning_rate": 0.00010061477272727271,
      "loss": 2.8184,
      "step": 134500
    },
    {
      "activations/layer0_attention_weight_max": 15.944783210754395,
      "activations/layer0_attention_weight_min": -14.24769115447998,
      "activations/layer10_attention_weight_max": 43.25986099243164,
      "activations/layer10_attention_weight_min": -37.280914306640625,
      "activations/layer11_attention_weight_max": 33.930274963378906,
      "activations/layer11_attention_weight_min": -31.75135040283203,
      "activations/layer12_attention_weight_max": 20.886764526367188,
      "activations/layer12_attention_weight_min": -24.875579833984375,
      "activations/layer13_attention_weight_max": 47.41988754272461,
      "activations/layer13_attention_weight_min": -37.753639221191406,
      "activations/layer14_attention_weight_max": 39.24184036254883,
      "activations/layer14_attention_weight_min": -34.106136322021484,
      "activations/layer15_attention_weight_max": 36.16604232788086,
      "activations/layer15_attention_weight_min": -31.372501373291016,
      "activations/layer16_attention_weight_max": 32.942535400390625,
      "activations/layer16_attention_weight_min": -29.451318740844727,
      "activations/layer17_attention_weight_max": 55.69913864135742,
      "activations/layer17_attention_weight_min": -44.40618896484375,
      "activations/layer18_attention_weight_max": 47.41495132446289,
      "activations/layer18_attention_weight_min": -36.04664611816406,
      "activations/layer19_attention_weight_max": 23.27992057800293,
      "activations/layer19_attention_weight_min": -21.180158615112305,
      "activations/layer1_attention_weight_max": 17.22842788696289,
      "activations/layer1_attention_weight_min": -15.36624526977539,
      "activations/layer20_attention_weight_max": 23.170621871948242,
      "activations/layer20_attention_weight_min": -22.937108993530273,
      "activations/layer21_attention_weight_max": 40.12229919433594,
      "activations/layer21_attention_weight_min": -21.948383331298828,
      "activations/layer22_attention_weight_max": 34.18940734863281,
      "activations/layer22_attention_weight_min": -24.68335723876953,
      "activations/layer23_attention_weight_max": 34.28857421875,
      "activations/layer23_attention_weight_min": -20.676513671875,
      "activations/layer2_attention_weight_max": 32.27287673950195,
      "activations/layer2_attention_weight_min": -29.355022430419922,
      "activations/layer3_attention_weight_max": 89.9402847290039,
      "activations/layer3_attention_weight_min": -92.93875885009766,
      "activations/layer4_attention_weight_max": 106.46822357177734,
      "activations/layer4_attention_weight_min": -100.01007843017578,
      "activations/layer5_attention_weight_max": 90.38235473632812,
      "activations/layer5_attention_weight_min": -85.6441650390625,
      "activations/layer6_attention_weight_max": 58.14239501953125,
      "activations/layer6_attention_weight_min": -55.011112213134766,
      "activations/layer7_attention_weight_max": 76.60110473632812,
      "activations/layer7_attention_weight_min": -77.04448699951172,
      "activations/layer8_attention_weight_max": 48.37691116333008,
      "activations/layer8_attention_weight_min": -49.00801467895508,
      "activations/layer9_attention_weight_max": 50.848167419433594,
      "activations/layer9_attention_weight_min": -48.37582015991211,
      "epoch": 7.82,
      "learning_rate": 0.00010059583333333332,
      "loss": 2.8196,
      "step": 134550
    },
    {
      "activations/layer0_attention_weight_max": 15.298775672912598,
      "activations/layer0_attention_weight_min": -14.275069236755371,
      "activations/layer10_attention_weight_max": 43.72863006591797,
      "activations/layer10_attention_weight_min": -37.67566680908203,
      "activations/layer11_attention_weight_max": 33.607723236083984,
      "activations/layer11_attention_weight_min": -31.593387603759766,
      "activations/layer12_attention_weight_max": 20.66878890991211,
      "activations/layer12_attention_weight_min": -26.260068893432617,
      "activations/layer13_attention_weight_max": 48.17155075073242,
      "activations/layer13_attention_weight_min": -32.74711608886719,
      "activations/layer14_attention_weight_max": 40.59028625488281,
      "activations/layer14_attention_weight_min": -31.722957611083984,
      "activations/layer15_attention_weight_max": 37.28822326660156,
      "activations/layer15_attention_weight_min": -29.376787185668945,
      "activations/layer16_attention_weight_max": 29.689556121826172,
      "activations/layer16_attention_weight_min": -29.221527099609375,
      "activations/layer17_attention_weight_max": 50.83460998535156,
      "activations/layer17_attention_weight_min": -42.47169494628906,
      "activations/layer18_attention_weight_max": 44.81645202636719,
      "activations/layer18_attention_weight_min": -37.6368408203125,
      "activations/layer19_attention_weight_max": 21.1682186126709,
      "activations/layer19_attention_weight_min": -22.560468673706055,
      "activations/layer1_attention_weight_max": 17.249013900756836,
      "activations/layer1_attention_weight_min": -15.600366592407227,
      "activations/layer20_attention_weight_max": 21.999608993530273,
      "activations/layer20_attention_weight_min": -21.69682502746582,
      "activations/layer21_attention_weight_max": 39.53647232055664,
      "activations/layer21_attention_weight_min": -23.154144287109375,
      "activations/layer22_attention_weight_max": 37.147300720214844,
      "activations/layer22_attention_weight_min": -27.83877944946289,
      "activations/layer23_attention_weight_max": 35.38306427001953,
      "activations/layer23_attention_weight_min": -22.44050407409668,
      "activations/layer2_attention_weight_max": 33.59264373779297,
      "activations/layer2_attention_weight_min": -30.600635528564453,
      "activations/layer3_attention_weight_max": 89.9505386352539,
      "activations/layer3_attention_weight_min": -93.75677490234375,
      "activations/layer4_attention_weight_max": 105.48518371582031,
      "activations/layer4_attention_weight_min": -99.71551513671875,
      "activations/layer5_attention_weight_max": 88.06861877441406,
      "activations/layer5_attention_weight_min": -83.4017562866211,
      "activations/layer6_attention_weight_max": 58.69307327270508,
      "activations/layer6_attention_weight_min": -55.742225646972656,
      "activations/layer7_attention_weight_max": 82.0892105102539,
      "activations/layer7_attention_weight_min": -77.15533447265625,
      "activations/layer8_attention_weight_max": 49.74267578125,
      "activations/layer8_attention_weight_min": -49.94198226928711,
      "activations/layer9_attention_weight_max": 53.8231086730957,
      "activations/layer9_attention_weight_min": -50.02238845825195,
      "epoch": 7.82,
      "learning_rate": 0.00010057689393939394,
      "loss": 2.8048,
      "step": 134600
    },
    {
      "activations/layer0_attention_weight_max": 16.301416397094727,
      "activations/layer0_attention_weight_min": -14.139121055603027,
      "activations/layer10_attention_weight_max": 41.28688430786133,
      "activations/layer10_attention_weight_min": -38.08590316772461,
      "activations/layer11_attention_weight_max": 35.386619567871094,
      "activations/layer11_attention_weight_min": -31.35127067565918,
      "activations/layer12_attention_weight_max": 20.740121841430664,
      "activations/layer12_attention_weight_min": -25.94251823425293,
      "activations/layer13_attention_weight_max": 66.44474792480469,
      "activations/layer13_attention_weight_min": -38.03095245361328,
      "activations/layer14_attention_weight_max": 65.5708236694336,
      "activations/layer14_attention_weight_min": -38.31351852416992,
      "activations/layer15_attention_weight_max": 54.01194381713867,
      "activations/layer15_attention_weight_min": -29.398088455200195,
      "activations/layer16_attention_weight_max": 38.91792678833008,
      "activations/layer16_attention_weight_min": -29.44339942932129,
      "activations/layer17_attention_weight_max": 56.12861251831055,
      "activations/layer17_attention_weight_min": -44.31319046020508,
      "activations/layer18_attention_weight_max": 49.66923522949219,
      "activations/layer18_attention_weight_min": -38.61761474609375,
      "activations/layer19_attention_weight_max": 23.32810401916504,
      "activations/layer19_attention_weight_min": -20.51805305480957,
      "activations/layer1_attention_weight_max": 16.961301803588867,
      "activations/layer1_attention_weight_min": -14.390633583068848,
      "activations/layer20_attention_weight_max": 26.246843338012695,
      "activations/layer20_attention_weight_min": -20.257652282714844,
      "activations/layer21_attention_weight_max": 49.8073616027832,
      "activations/layer21_attention_weight_min": -25.021684646606445,
      "activations/layer22_attention_weight_max": 34.95630645751953,
      "activations/layer22_attention_weight_min": -28.809446334838867,
      "activations/layer23_attention_weight_max": 35.21649932861328,
      "activations/layer23_attention_weight_min": -19.447528839111328,
      "activations/layer2_attention_weight_max": 31.375524520874023,
      "activations/layer2_attention_weight_min": -29.745716094970703,
      "activations/layer3_attention_weight_max": 92.8297348022461,
      "activations/layer3_attention_weight_min": -94.61668395996094,
      "activations/layer4_attention_weight_max": 108.43757629394531,
      "activations/layer4_attention_weight_min": -99.26375579833984,
      "activations/layer5_attention_weight_max": 86.9211654663086,
      "activations/layer5_attention_weight_min": -78.20413208007812,
      "activations/layer6_attention_weight_max": 56.6370849609375,
      "activations/layer6_attention_weight_min": -53.252140045166016,
      "activations/layer7_attention_weight_max": 84.9062271118164,
      "activations/layer7_attention_weight_min": -78.15753173828125,
      "activations/layer8_attention_weight_max": 48.95014190673828,
      "activations/layer8_attention_weight_min": -45.38541793823242,
      "activations/layer9_attention_weight_max": 62.645240783691406,
      "activations/layer9_attention_weight_min": -49.548973083496094,
      "epoch": 7.82,
      "learning_rate": 0.00010055795454545454,
      "loss": 2.8021,
      "step": 134650
    },
    {
      "activations/layer0_attention_weight_max": 15.807257652282715,
      "activations/layer0_attention_weight_min": -13.881467819213867,
      "activations/layer10_attention_weight_max": 37.196834564208984,
      "activations/layer10_attention_weight_min": -33.32728576660156,
      "activations/layer11_attention_weight_max": 31.452693939208984,
      "activations/layer11_attention_weight_min": -32.85676956176758,
      "activations/layer12_attention_weight_max": 19.49518585205078,
      "activations/layer12_attention_weight_min": -27.752973556518555,
      "activations/layer13_attention_weight_max": 42.86234664916992,
      "activations/layer13_attention_weight_min": -29.51003074645996,
      "activations/layer14_attention_weight_max": 45.90174865722656,
      "activations/layer14_attention_weight_min": -29.900680541992188,
      "activations/layer15_attention_weight_max": 33.4482536315918,
      "activations/layer15_attention_weight_min": -29.11009407043457,
      "activations/layer16_attention_weight_max": 30.881778717041016,
      "activations/layer16_attention_weight_min": -27.093318939208984,
      "activations/layer17_attention_weight_max": 50.73292922973633,
      "activations/layer17_attention_weight_min": -42.95146942138672,
      "activations/layer18_attention_weight_max": 46.59563446044922,
      "activations/layer18_attention_weight_min": -35.925689697265625,
      "activations/layer19_attention_weight_max": 21.9715633392334,
      "activations/layer19_attention_weight_min": -20.61659049987793,
      "activations/layer1_attention_weight_max": 17.717275619506836,
      "activations/layer1_attention_weight_min": -14.767328262329102,
      "activations/layer20_attention_weight_max": 22.178241729736328,
      "activations/layer20_attention_weight_min": -24.121252059936523,
      "activations/layer21_attention_weight_max": 44.71503829956055,
      "activations/layer21_attention_weight_min": -24.680431365966797,
      "activations/layer22_attention_weight_max": 35.28873062133789,
      "activations/layer22_attention_weight_min": -24.88259506225586,
      "activations/layer23_attention_weight_max": 36.79509735107422,
      "activations/layer23_attention_weight_min": -23.383403778076172,
      "activations/layer2_attention_weight_max": 31.23719596862793,
      "activations/layer2_attention_weight_min": -30.545730590820312,
      "activations/layer3_attention_weight_max": 88.23603057861328,
      "activations/layer3_attention_weight_min": -93.37590026855469,
      "activations/layer4_attention_weight_max": 102.24903106689453,
      "activations/layer4_attention_weight_min": -100.95430755615234,
      "activations/layer5_attention_weight_max": 85.39019775390625,
      "activations/layer5_attention_weight_min": -83.2743148803711,
      "activations/layer6_attention_weight_max": 53.8858757019043,
      "activations/layer6_attention_weight_min": -55.63877868652344,
      "activations/layer7_attention_weight_max": 75.75174713134766,
      "activations/layer7_attention_weight_min": -70.58902740478516,
      "activations/layer8_attention_weight_max": 43.3399543762207,
      "activations/layer8_attention_weight_min": -43.35149002075195,
      "activations/layer9_attention_weight_max": 49.02484130859375,
      "activations/layer9_attention_weight_min": -46.09578323364258,
      "epoch": 7.83,
      "learning_rate": 0.00010053901515151514,
      "loss": 2.8142,
      "step": 134700
    },
    {
      "activations/layer0_attention_weight_max": 16.549997329711914,
      "activations/layer0_attention_weight_min": -13.834030151367188,
      "activations/layer10_attention_weight_max": 37.777164459228516,
      "activations/layer10_attention_weight_min": -36.80223846435547,
      "activations/layer11_attention_weight_max": 30.48025131225586,
      "activations/layer11_attention_weight_min": -32.78693771362305,
      "activations/layer12_attention_weight_max": 20.30731964111328,
      "activations/layer12_attention_weight_min": -26.964563369750977,
      "activations/layer13_attention_weight_max": 47.59139633178711,
      "activations/layer13_attention_weight_min": -31.479923248291016,
      "activations/layer14_attention_weight_max": 40.53804016113281,
      "activations/layer14_attention_weight_min": -31.514860153198242,
      "activations/layer15_attention_weight_max": 35.545166015625,
      "activations/layer15_attention_weight_min": -30.432172775268555,
      "activations/layer16_attention_weight_max": 37.023651123046875,
      "activations/layer16_attention_weight_min": -26.67421531677246,
      "activations/layer17_attention_weight_max": 59.85670852661133,
      "activations/layer17_attention_weight_min": -42.32466125488281,
      "activations/layer18_attention_weight_max": 50.4310417175293,
      "activations/layer18_attention_weight_min": -35.127777099609375,
      "activations/layer19_attention_weight_max": 25.08802604675293,
      "activations/layer19_attention_weight_min": -21.015254974365234,
      "activations/layer1_attention_weight_max": 16.300357818603516,
      "activations/layer1_attention_weight_min": -15.55932331085205,
      "activations/layer20_attention_weight_max": 25.284944534301758,
      "activations/layer20_attention_weight_min": -26.600500106811523,
      "activations/layer21_attention_weight_max": 42.82186508178711,
      "activations/layer21_attention_weight_min": -24.092519760131836,
      "activations/layer22_attention_weight_max": 34.82170486450195,
      "activations/layer22_attention_weight_min": -22.871885299682617,
      "activations/layer23_attention_weight_max": 39.80015182495117,
      "activations/layer23_attention_weight_min": -21.172170639038086,
      "activations/layer2_attention_weight_max": 30.150962829589844,
      "activations/layer2_attention_weight_min": -28.193191528320312,
      "activations/layer3_attention_weight_max": 85.11715698242188,
      "activations/layer3_attention_weight_min": -86.9598388671875,
      "activations/layer4_attention_weight_max": 96.88054656982422,
      "activations/layer4_attention_weight_min": -96.7716064453125,
      "activations/layer5_attention_weight_max": 85.10140991210938,
      "activations/layer5_attention_weight_min": -80.66517639160156,
      "activations/layer6_attention_weight_max": 56.59044647216797,
      "activations/layer6_attention_weight_min": -56.283329010009766,
      "activations/layer7_attention_weight_max": 81.95413970947266,
      "activations/layer7_attention_weight_min": -77.72132873535156,
      "activations/layer8_attention_weight_max": 46.57672119140625,
      "activations/layer8_attention_weight_min": -47.46844482421875,
      "activations/layer9_attention_weight_max": 53.220611572265625,
      "activations/layer9_attention_weight_min": -52.031471252441406,
      "epoch": 7.83,
      "learning_rate": 0.00010052007575757574,
      "loss": 2.8251,
      "step": 134750
    },
    {
      "activations/layer0_attention_weight_max": 15.889386177062988,
      "activations/layer0_attention_weight_min": -14.45095157623291,
      "activations/layer10_attention_weight_max": 36.35077667236328,
      "activations/layer10_attention_weight_min": -35.01487731933594,
      "activations/layer11_attention_weight_max": 32.161712646484375,
      "activations/layer11_attention_weight_min": -30.93341636657715,
      "activations/layer12_attention_weight_max": 18.411115646362305,
      "activations/layer12_attention_weight_min": -26.160959243774414,
      "activations/layer13_attention_weight_max": 60.095184326171875,
      "activations/layer13_attention_weight_min": -36.7822380065918,
      "activations/layer14_attention_weight_max": 68.67583465576172,
      "activations/layer14_attention_weight_min": -40.56069564819336,
      "activations/layer15_attention_weight_max": 46.09178924560547,
      "activations/layer15_attention_weight_min": -31.642793655395508,
      "activations/layer16_attention_weight_max": 32.12046432495117,
      "activations/layer16_attention_weight_min": -29.581003189086914,
      "activations/layer17_attention_weight_max": 54.58675003051758,
      "activations/layer17_attention_weight_min": -43.23163986206055,
      "activations/layer18_attention_weight_max": 48.49381637573242,
      "activations/layer18_attention_weight_min": -37.77886199951172,
      "activations/layer19_attention_weight_max": 24.3504638671875,
      "activations/layer19_attention_weight_min": -20.607820510864258,
      "activations/layer1_attention_weight_max": 18.218225479125977,
      "activations/layer1_attention_weight_min": -16.378345489501953,
      "activations/layer20_attention_weight_max": 22.538049697875977,
      "activations/layer20_attention_weight_min": -19.757017135620117,
      "activations/layer21_attention_weight_max": 40.96178436279297,
      "activations/layer21_attention_weight_min": -24.938688278198242,
      "activations/layer22_attention_weight_max": 31.04853057861328,
      "activations/layer22_attention_weight_min": -25.056129455566406,
      "activations/layer23_attention_weight_max": 32.93989944458008,
      "activations/layer23_attention_weight_min": -22.673452377319336,
      "activations/layer2_attention_weight_max": 34.00342559814453,
      "activations/layer2_attention_weight_min": -31.27943992614746,
      "activations/layer3_attention_weight_max": 88.65702056884766,
      "activations/layer3_attention_weight_min": -87.88314819335938,
      "activations/layer4_attention_weight_max": 103.1388168334961,
      "activations/layer4_attention_weight_min": -94.42828369140625,
      "activations/layer5_attention_weight_max": 87.4107666015625,
      "activations/layer5_attention_weight_min": -80.33207702636719,
      "activations/layer6_attention_weight_max": 57.51970672607422,
      "activations/layer6_attention_weight_min": -52.714115142822266,
      "activations/layer7_attention_weight_max": 77.28031921386719,
      "activations/layer7_attention_weight_min": -74.5878677368164,
      "activations/layer8_attention_weight_max": 44.53144073486328,
      "activations/layer8_attention_weight_min": -44.689476013183594,
      "activations/layer9_attention_weight_max": 50.98375701904297,
      "activations/layer9_attention_weight_min": -47.75872802734375,
      "epoch": 7.83,
      "learning_rate": 0.00010050151515151515,
      "loss": 2.8162,
      "step": 134800
    },
    {
      "activations/layer0_attention_weight_max": 15.17953872680664,
      "activations/layer0_attention_weight_min": -14.137892723083496,
      "activations/layer10_attention_weight_max": 36.283504486083984,
      "activations/layer10_attention_weight_min": -35.53863525390625,
      "activations/layer11_attention_weight_max": 30.37889862060547,
      "activations/layer11_attention_weight_min": -30.311012268066406,
      "activations/layer12_attention_weight_max": 22.59402084350586,
      "activations/layer12_attention_weight_min": -29.204694747924805,
      "activations/layer13_attention_weight_max": 42.586753845214844,
      "activations/layer13_attention_weight_min": -32.13569259643555,
      "activations/layer14_attention_weight_max": 36.813697814941406,
      "activations/layer14_attention_weight_min": -30.523984909057617,
      "activations/layer15_attention_weight_max": 33.142459869384766,
      "activations/layer15_attention_weight_min": -27.929515838623047,
      "activations/layer16_attention_weight_max": 32.17662048339844,
      "activations/layer16_attention_weight_min": -27.545894622802734,
      "activations/layer17_attention_weight_max": 54.36040115356445,
      "activations/layer17_attention_weight_min": -43.505496978759766,
      "activations/layer18_attention_weight_max": 47.5499153137207,
      "activations/layer18_attention_weight_min": -36.491695404052734,
      "activations/layer19_attention_weight_max": 24.716461181640625,
      "activations/layer19_attention_weight_min": -19.576045989990234,
      "activations/layer1_attention_weight_max": 17.959308624267578,
      "activations/layer1_attention_weight_min": -14.723767280578613,
      "activations/layer20_attention_weight_max": 21.01450538635254,
      "activations/layer20_attention_weight_min": -22.187179565429688,
      "activations/layer21_attention_weight_max": 37.70160675048828,
      "activations/layer21_attention_weight_min": -24.686330795288086,
      "activations/layer22_attention_weight_max": 31.080076217651367,
      "activations/layer22_attention_weight_min": -25.348100662231445,
      "activations/layer23_attention_weight_max": 39.612056732177734,
      "activations/layer23_attention_weight_min": -23.188453674316406,
      "activations/layer2_attention_weight_max": 33.66251754760742,
      "activations/layer2_attention_weight_min": -30.34813690185547,
      "activations/layer3_attention_weight_max": 83.69174194335938,
      "activations/layer3_attention_weight_min": -86.74053955078125,
      "activations/layer4_attention_weight_max": 95.75589752197266,
      "activations/layer4_attention_weight_min": -93.09184265136719,
      "activations/layer5_attention_weight_max": 82.31729125976562,
      "activations/layer5_attention_weight_min": -77.4911880493164,
      "activations/layer6_attention_weight_max": 55.017311096191406,
      "activations/layer6_attention_weight_min": -53.03065872192383,
      "activations/layer7_attention_weight_max": 80.81220245361328,
      "activations/layer7_attention_weight_min": -73.05162811279297,
      "activations/layer8_attention_weight_max": 43.87685012817383,
      "activations/layer8_attention_weight_min": -44.428619384765625,
      "activations/layer9_attention_weight_max": 55.83474349975586,
      "activations/layer9_attention_weight_min": -49.51313781738281,
      "epoch": 7.84,
      "learning_rate": 0.00010048257575757575,
      "loss": 2.8213,
      "step": 134850
    },
    {
      "activations/layer0_attention_weight_max": 15.166359901428223,
      "activations/layer0_attention_weight_min": -13.921113014221191,
      "activations/layer10_attention_weight_max": 38.75703048706055,
      "activations/layer10_attention_weight_min": -36.2325325012207,
      "activations/layer11_attention_weight_max": 32.46159362792969,
      "activations/layer11_attention_weight_min": -31.617597579956055,
      "activations/layer12_attention_weight_max": 24.962926864624023,
      "activations/layer12_attention_weight_min": -25.29485321044922,
      "activations/layer13_attention_weight_max": 40.18333435058594,
      "activations/layer13_attention_weight_min": -31.975072860717773,
      "activations/layer14_attention_weight_max": 39.02542495727539,
      "activations/layer14_attention_weight_min": -30.048866271972656,
      "activations/layer15_attention_weight_max": 34.37019348144531,
      "activations/layer15_attention_weight_min": -29.505971908569336,
      "activations/layer16_attention_weight_max": 31.117395401000977,
      "activations/layer16_attention_weight_min": -29.16304588317871,
      "activations/layer17_attention_weight_max": 57.67861557006836,
      "activations/layer17_attention_weight_min": -43.023651123046875,
      "activations/layer18_attention_weight_max": 44.72883224487305,
      "activations/layer18_attention_weight_min": -36.77022933959961,
      "activations/layer19_attention_weight_max": 21.5020694732666,
      "activations/layer19_attention_weight_min": -20.8348445892334,
      "activations/layer1_attention_weight_max": 17.15083122253418,
      "activations/layer1_attention_weight_min": -15.51215934753418,
      "activations/layer20_attention_weight_max": 22.714275360107422,
      "activations/layer20_attention_weight_min": -23.122303009033203,
      "activations/layer21_attention_weight_max": 39.54894256591797,
      "activations/layer21_attention_weight_min": -23.261903762817383,
      "activations/layer22_attention_weight_max": 30.22743797302246,
      "activations/layer22_attention_weight_min": -25.80681610107422,
      "activations/layer23_attention_weight_max": 33.56795120239258,
      "activations/layer23_attention_weight_min": -22.942684173583984,
      "activations/layer2_attention_weight_max": 31.542800903320312,
      "activations/layer2_attention_weight_min": -29.223533630371094,
      "activations/layer3_attention_weight_max": 92.34487915039062,
      "activations/layer3_attention_weight_min": -92.43988800048828,
      "activations/layer4_attention_weight_max": 105.04571533203125,
      "activations/layer4_attention_weight_min": -94.66654205322266,
      "activations/layer5_attention_weight_max": 87.98503112792969,
      "activations/layer5_attention_weight_min": -86.37657165527344,
      "activations/layer6_attention_weight_max": 56.18840026855469,
      "activations/layer6_attention_weight_min": -55.08765411376953,
      "activations/layer7_attention_weight_max": 76.94088745117188,
      "activations/layer7_attention_weight_min": -74.00479888916016,
      "activations/layer8_attention_weight_max": 45.93303680419922,
      "activations/layer8_attention_weight_min": -46.55398178100586,
      "activations/layer9_attention_weight_max": 51.07440185546875,
      "activations/layer9_attention_weight_min": -51.062313079833984,
      "epoch": 7.84,
      "learning_rate": 0.00010046363636363636,
      "loss": 2.7997,
      "step": 134900
    },
    {
      "activations/layer0_attention_weight_max": 14.845449447631836,
      "activations/layer0_attention_weight_min": -14.025251388549805,
      "activations/layer10_attention_weight_max": 36.919105529785156,
      "activations/layer10_attention_weight_min": -36.52153778076172,
      "activations/layer11_attention_weight_max": 31.789167404174805,
      "activations/layer11_attention_weight_min": -34.16954803466797,
      "activations/layer12_attention_weight_max": 20.58414077758789,
      "activations/layer12_attention_weight_min": -22.695411682128906,
      "activations/layer13_attention_weight_max": 41.503108978271484,
      "activations/layer13_attention_weight_min": -34.984161376953125,
      "activations/layer14_attention_weight_max": 39.460670471191406,
      "activations/layer14_attention_weight_min": -31.669422149658203,
      "activations/layer15_attention_weight_max": 38.84703063964844,
      "activations/layer15_attention_weight_min": -29.86855125427246,
      "activations/layer16_attention_weight_max": 30.75288963317871,
      "activations/layer16_attention_weight_min": -29.12347412109375,
      "activations/layer17_attention_weight_max": 60.51673889160156,
      "activations/layer17_attention_weight_min": -44.86240005493164,
      "activations/layer18_attention_weight_max": 50.74162673950195,
      "activations/layer18_attention_weight_min": -37.14902877807617,
      "activations/layer19_attention_weight_max": 22.294904708862305,
      "activations/layer19_attention_weight_min": -20.61669921875,
      "activations/layer1_attention_weight_max": 16.198740005493164,
      "activations/layer1_attention_weight_min": -16.402833938598633,
      "activations/layer20_attention_weight_max": 24.633289337158203,
      "activations/layer20_attention_weight_min": -24.15612030029297,
      "activations/layer21_attention_weight_max": 39.46274185180664,
      "activations/layer21_attention_weight_min": -23.970190048217773,
      "activations/layer22_attention_weight_max": 34.258365631103516,
      "activations/layer22_attention_weight_min": -25.429595947265625,
      "activations/layer23_attention_weight_max": 38.24961471557617,
      "activations/layer23_attention_weight_min": -21.09886932373047,
      "activations/layer2_attention_weight_max": 32.08610916137695,
      "activations/layer2_attention_weight_min": -30.71907615661621,
      "activations/layer3_attention_weight_max": 86.16813659667969,
      "activations/layer3_attention_weight_min": -90.9293212890625,
      "activations/layer4_attention_weight_max": 102.10759735107422,
      "activations/layer4_attention_weight_min": -99.58685302734375,
      "activations/layer5_attention_weight_max": 85.97291564941406,
      "activations/layer5_attention_weight_min": -80.67782592773438,
      "activations/layer6_attention_weight_max": 57.12656021118164,
      "activations/layer6_attention_weight_min": -56.0775146484375,
      "activations/layer7_attention_weight_max": 75.01197814941406,
      "activations/layer7_attention_weight_min": -70.50643920898438,
      "activations/layer8_attention_weight_max": 45.173927307128906,
      "activations/layer8_attention_weight_min": -44.8472785949707,
      "activations/layer9_attention_weight_max": 50.94792938232422,
      "activations/layer9_attention_weight_min": -49.07835006713867,
      "epoch": 7.84,
      "learning_rate": 0.00010044469696969695,
      "loss": 2.8149,
      "step": 134950
    },
    {
      "activations/layer0_attention_weight_max": 15.223992347717285,
      "activations/layer0_attention_weight_min": -13.916044235229492,
      "activations/layer10_attention_weight_max": 42.867340087890625,
      "activations/layer10_attention_weight_min": -41.476898193359375,
      "activations/layer11_attention_weight_max": 30.66466522216797,
      "activations/layer11_attention_weight_min": -31.415912628173828,
      "activations/layer12_attention_weight_max": 28.40460968017578,
      "activations/layer12_attention_weight_min": -22.169326782226562,
      "activations/layer13_attention_weight_max": 43.039302825927734,
      "activations/layer13_attention_weight_min": -31.7749080657959,
      "activations/layer14_attention_weight_max": 39.08285140991211,
      "activations/layer14_attention_weight_min": -29.843982696533203,
      "activations/layer15_attention_weight_max": 39.32485580444336,
      "activations/layer15_attention_weight_min": -29.537109375,
      "activations/layer16_attention_weight_max": 33.50986862182617,
      "activations/layer16_attention_weight_min": -27.70732879638672,
      "activations/layer17_attention_weight_max": 51.46426010131836,
      "activations/layer17_attention_weight_min": -42.49782943725586,
      "activations/layer18_attention_weight_max": 43.97361755371094,
      "activations/layer18_attention_weight_min": -35.61444091796875,
      "activations/layer19_attention_weight_max": 26.594345092773438,
      "activations/layer19_attention_weight_min": -25.91986656188965,
      "activations/layer1_attention_weight_max": 17.330585479736328,
      "activations/layer1_attention_weight_min": -13.56190299987793,
      "activations/layer20_attention_weight_max": 23.558223724365234,
      "activations/layer20_attention_weight_min": -22.371809005737305,
      "activations/layer21_attention_weight_max": 33.61894607543945,
      "activations/layer21_attention_weight_min": -21.970529556274414,
      "activations/layer22_attention_weight_max": 34.41421127319336,
      "activations/layer22_attention_weight_min": -24.93258285522461,
      "activations/layer23_attention_weight_max": 36.90565490722656,
      "activations/layer23_attention_weight_min": -23.01734733581543,
      "activations/layer2_attention_weight_max": 29.017728805541992,
      "activations/layer2_attention_weight_min": -28.30471420288086,
      "activations/layer3_attention_weight_max": 85.70772552490234,
      "activations/layer3_attention_weight_min": -90.68135070800781,
      "activations/layer4_attention_weight_max": 100.7800521850586,
      "activations/layer4_attention_weight_min": -97.0392074584961,
      "activations/layer5_attention_weight_max": 83.87213134765625,
      "activations/layer5_attention_weight_min": -77.32272338867188,
      "activations/layer6_attention_weight_max": 52.65742874145508,
      "activations/layer6_attention_weight_min": -51.00148391723633,
      "activations/layer7_attention_weight_max": 72.28713989257812,
      "activations/layer7_attention_weight_min": -70.09075164794922,
      "activations/layer8_attention_weight_max": 42.32771682739258,
      "activations/layer8_attention_weight_min": -43.277950286865234,
      "activations/layer9_attention_weight_max": 50.674800872802734,
      "activations/layer9_attention_weight_min": -48.34243392944336,
      "epoch": 7.84,
      "learning_rate": 0.00010042575757575757,
      "loss": 2.8129,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_loss": 2.765625,
      "eval_runtime": 8.5084,
      "eval_samples_per_second": 504.678,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_openwebtext_loss": 2.765625,
      "eval_openwebtext_ppl": 15.888967492629165,
      "eval_openwebtext_runtime": 8.5084,
      "eval_openwebtext_samples_per_second": 504.678,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_wikitext_loss": 3.001953125,
      "eval_wikitext_ppl": 20.124804822565828,
      "eval_wikitext_runtime": 1.9894,
      "eval_wikitext_samples_per_second": 229.218,
      "step": 135000
    },
    {
      "epoch": 7.84,
      "eval_lambada_loss": 2.76171875,
      "eval_lambada_ppl": 15.827022278874091,
      "eval_lambada_runtime": 9.6296,
      "eval_lambada_samples_per_second": 505.626,
      "step": 135000
    },
    {
      "activations/layer0_attention_weight_max": 16.216367721557617,
      "activations/layer0_attention_weight_min": -14.351487159729004,
      "activations/layer10_attention_weight_max": 36.71324157714844,
      "activations/layer10_attention_weight_min": -35.869972229003906,
      "activations/layer11_attention_weight_max": 31.23713493347168,
      "activations/layer11_attention_weight_min": -29.712238311767578,
      "activations/layer12_attention_weight_max": 21.21036720275879,
      "activations/layer12_attention_weight_min": -25.997602462768555,
      "activations/layer13_attention_weight_max": 42.71385955810547,
      "activations/layer13_attention_weight_min": -28.711517333984375,
      "activations/layer14_attention_weight_max": 45.59300994873047,
      "activations/layer14_attention_weight_min": -29.859132766723633,
      "activations/layer15_attention_weight_max": 34.123199462890625,
      "activations/layer15_attention_weight_min": -28.953182220458984,
      "activations/layer16_attention_weight_max": 30.293811798095703,
      "activations/layer16_attention_weight_min": -27.91222381591797,
      "activations/layer17_attention_weight_max": 54.512752532958984,
      "activations/layer17_attention_weight_min": -41.42095184326172,
      "activations/layer18_attention_weight_max": 46.97686004638672,
      "activations/layer18_attention_weight_min": -34.271400451660156,
      "activations/layer19_attention_weight_max": 25.012638092041016,
      "activations/layer19_attention_weight_min": -20.601917266845703,
      "activations/layer1_attention_weight_max": 16.611225128173828,
      "activations/layer1_attention_weight_min": -16.73261833190918,
      "activations/layer20_attention_weight_max": 22.78192138671875,
      "activations/layer20_attention_weight_min": -19.964622497558594,
      "activations/layer21_attention_weight_max": 38.066184997558594,
      "activations/layer21_attention_weight_min": -24.195520401000977,
      "activations/layer22_attention_weight_max": 34.747676849365234,
      "activations/layer22_attention_weight_min": -24.434730529785156,
      "activations/layer23_attention_weight_max": 37.89365768432617,
      "activations/layer23_attention_weight_min": -20.87137222290039,
      "activations/layer2_attention_weight_max": 32.85688400268555,
      "activations/layer2_attention_weight_min": -29.089866638183594,
      "activations/layer3_attention_weight_max": 90.55415344238281,
      "activations/layer3_attention_weight_min": -89.04750061035156,
      "activations/layer4_attention_weight_max": 106.1475830078125,
      "activations/layer4_attention_weight_min": -96.13677215576172,
      "activations/layer5_attention_weight_max": 89.41536712646484,
      "activations/layer5_attention_weight_min": -77.22506713867188,
      "activations/layer6_attention_weight_max": 54.335086822509766,
      "activations/layer6_attention_weight_min": -52.236122131347656,
      "activations/layer7_attention_weight_max": 73.85353088378906,
      "activations/layer7_attention_weight_min": -73.39397430419922,
      "activations/layer8_attention_weight_max": 45.2961311340332,
      "activations/layer8_attention_weight_min": -47.226104736328125,
      "activations/layer9_attention_weight_max": 53.9930419921875,
      "activations/layer9_attention_weight_min": -46.76530075073242,
      "epoch": 7.85,
      "learning_rate": 0.00010040681818181818,
      "loss": 2.8224,
      "step": 135050
    },
    {
      "activations/layer0_attention_weight_max": 15.241939544677734,
      "activations/layer0_attention_weight_min": -14.186202049255371,
      "activations/layer10_attention_weight_max": 39.24646759033203,
      "activations/layer10_attention_weight_min": -37.62998580932617,
      "activations/layer11_attention_weight_max": 29.455472946166992,
      "activations/layer11_attention_weight_min": -33.25920867919922,
      "activations/layer12_attention_weight_max": 18.96636390686035,
      "activations/layer12_attention_weight_min": -27.74354362487793,
      "activations/layer13_attention_weight_max": 39.0295524597168,
      "activations/layer13_attention_weight_min": -31.387310028076172,
      "activations/layer14_attention_weight_max": 38.50477981567383,
      "activations/layer14_attention_weight_min": -31.51092529296875,
      "activations/layer15_attention_weight_max": 39.95951461791992,
      "activations/layer15_attention_weight_min": -28.88456153869629,
      "activations/layer16_attention_weight_max": 34.04835891723633,
      "activations/layer16_attention_weight_min": -28.271303176879883,
      "activations/layer17_attention_weight_max": 58.83684539794922,
      "activations/layer17_attention_weight_min": -42.538360595703125,
      "activations/layer18_attention_weight_max": 49.79793167114258,
      "activations/layer18_attention_weight_min": -35.711387634277344,
      "activations/layer19_attention_weight_max": 21.79743194580078,
      "activations/layer19_attention_weight_min": -19.597665786743164,
      "activations/layer1_attention_weight_max": 16.48149299621582,
      "activations/layer1_attention_weight_min": -15.399323463439941,
      "activations/layer20_attention_weight_max": 20.612634658813477,
      "activations/layer20_attention_weight_min": -20.98015594482422,
      "activations/layer21_attention_weight_max": 38.49751663208008,
      "activations/layer21_attention_weight_min": -24.770763397216797,
      "activations/layer22_attention_weight_max": 31.535152435302734,
      "activations/layer22_attention_weight_min": -24.78624725341797,
      "activations/layer23_attention_weight_max": 34.66975402832031,
      "activations/layer23_attention_weight_min": -22.089567184448242,
      "activations/layer2_attention_weight_max": 31.928340911865234,
      "activations/layer2_attention_weight_min": -30.64095687866211,
      "activations/layer3_attention_weight_max": 87.4306411743164,
      "activations/layer3_attention_weight_min": -89.42496490478516,
      "activations/layer4_attention_weight_max": 101.98762512207031,
      "activations/layer4_attention_weight_min": -92.8564224243164,
      "activations/layer5_attention_weight_max": 82.03396606445312,
      "activations/layer5_attention_weight_min": -81.18826293945312,
      "activations/layer6_attention_weight_max": 54.388694763183594,
      "activations/layer6_attention_weight_min": -53.7967414855957,
      "activations/layer7_attention_weight_max": 77.877685546875,
      "activations/layer7_attention_weight_min": -70.78563690185547,
      "activations/layer8_attention_weight_max": 43.1882209777832,
      "activations/layer8_attention_weight_min": -47.777042388916016,
      "activations/layer9_attention_weight_max": 52.81229782104492,
      "activations/layer9_attention_weight_min": -48.815025329589844,
      "epoch": 7.85,
      "learning_rate": 0.00010038787878787877,
      "loss": 2.8176,
      "step": 135100
    },
    {
      "activations/layer0_attention_weight_max": 15.683331489562988,
      "activations/layer0_attention_weight_min": -14.470986366271973,
      "activations/layer10_attention_weight_max": 37.35447692871094,
      "activations/layer10_attention_weight_min": -35.316226959228516,
      "activations/layer11_attention_weight_max": 32.13605499267578,
      "activations/layer11_attention_weight_min": -31.294448852539062,
      "activations/layer12_attention_weight_max": 20.8142032623291,
      "activations/layer12_attention_weight_min": -30.67704963684082,
      "activations/layer13_attention_weight_max": 56.92844009399414,
      "activations/layer13_attention_weight_min": -45.44839096069336,
      "activations/layer14_attention_weight_max": 51.72734832763672,
      "activations/layer14_attention_weight_min": -38.73637390136719,
      "activations/layer15_attention_weight_max": 39.710575103759766,
      "activations/layer15_attention_weight_min": -30.724199295043945,
      "activations/layer16_attention_weight_max": 35.73984146118164,
      "activations/layer16_attention_weight_min": -27.2182559967041,
      "activations/layer17_attention_weight_max": 54.3745002746582,
      "activations/layer17_attention_weight_min": -44.277076721191406,
      "activations/layer18_attention_weight_max": 49.95844650268555,
      "activations/layer18_attention_weight_min": -40.12398910522461,
      "activations/layer19_attention_weight_max": 23.076705932617188,
      "activations/layer19_attention_weight_min": -23.50733757019043,
      "activations/layer1_attention_weight_max": 16.50237464904785,
      "activations/layer1_attention_weight_min": -16.821645736694336,
      "activations/layer20_attention_weight_max": 25.445859909057617,
      "activations/layer20_attention_weight_min": -30.64409828186035,
      "activations/layer21_attention_weight_max": 43.9427375793457,
      "activations/layer21_attention_weight_min": -24.320459365844727,
      "activations/layer22_attention_weight_max": 37.7940559387207,
      "activations/layer22_attention_weight_min": -25.89500617980957,
      "activations/layer23_attention_weight_max": 37.311302185058594,
      "activations/layer23_attention_weight_min": -21.963241577148438,
      "activations/layer2_attention_weight_max": 34.134666442871094,
      "activations/layer2_attention_weight_min": -31.35650634765625,
      "activations/layer3_attention_weight_max": 89.3738021850586,
      "activations/layer3_attention_weight_min": -91.67291259765625,
      "activations/layer4_attention_weight_max": 94.95037841796875,
      "activations/layer4_attention_weight_min": -96.347900390625,
      "activations/layer5_attention_weight_max": 78.63236999511719,
      "activations/layer5_attention_weight_min": -76.87205505371094,
      "activations/layer6_attention_weight_max": 51.86240005493164,
      "activations/layer6_attention_weight_min": -49.763572692871094,
      "activations/layer7_attention_weight_max": 73.6337661743164,
      "activations/layer7_attention_weight_min": -70.53583526611328,
      "activations/layer8_attention_weight_max": 42.6866455078125,
      "activations/layer8_attention_weight_min": -44.34550094604492,
      "activations/layer9_attention_weight_max": 53.8458251953125,
      "activations/layer9_attention_weight_min": -49.873287200927734,
      "epoch": 7.85,
      "learning_rate": 0.00010036893939393938,
      "loss": 2.8111,
      "step": 135150
    },
    {
      "activations/layer0_attention_weight_max": 15.968794822692871,
      "activations/layer0_attention_weight_min": -14.190492630004883,
      "activations/layer10_attention_weight_max": 40.581275939941406,
      "activations/layer10_attention_weight_min": -38.000789642333984,
      "activations/layer11_attention_weight_max": 33.8241081237793,
      "activations/layer11_attention_weight_min": -31.327978134155273,
      "activations/layer12_attention_weight_max": 21.13003921508789,
      "activations/layer12_attention_weight_min": -24.379173278808594,
      "activations/layer13_attention_weight_max": 57.03192901611328,
      "activations/layer13_attention_weight_min": -43.251094818115234,
      "activations/layer14_attention_weight_max": 49.41705322265625,
      "activations/layer14_attention_weight_min": -34.14552307128906,
      "activations/layer15_attention_weight_max": 36.657554626464844,
      "activations/layer15_attention_weight_min": -30.452417373657227,
      "activations/layer16_attention_weight_max": 31.43805694580078,
      "activations/layer16_attention_weight_min": -27.4190616607666,
      "activations/layer17_attention_weight_max": 57.62066650390625,
      "activations/layer17_attention_weight_min": -45.72139358520508,
      "activations/layer18_attention_weight_max": 49.30303192138672,
      "activations/layer18_attention_weight_min": -36.98868942260742,
      "activations/layer19_attention_weight_max": 22.546537399291992,
      "activations/layer19_attention_weight_min": -22.2738037109375,
      "activations/layer1_attention_weight_max": 16.7147274017334,
      "activations/layer1_attention_weight_min": -15.033007621765137,
      "activations/layer20_attention_weight_max": 23.921981811523438,
      "activations/layer20_attention_weight_min": -23.256145477294922,
      "activations/layer21_attention_weight_max": 39.95733642578125,
      "activations/layer21_attention_weight_min": -22.965499877929688,
      "activations/layer22_attention_weight_max": 35.3486213684082,
      "activations/layer22_attention_weight_min": -26.06178855895996,
      "activations/layer23_attention_weight_max": 37.24320983886719,
      "activations/layer23_attention_weight_min": -20.2877140045166,
      "activations/layer2_attention_weight_max": 30.92813491821289,
      "activations/layer2_attention_weight_min": -28.82006072998047,
      "activations/layer3_attention_weight_max": 88.29131317138672,
      "activations/layer3_attention_weight_min": -87.01555633544922,
      "activations/layer4_attention_weight_max": 106.52174377441406,
      "activations/layer4_attention_weight_min": -96.62238311767578,
      "activations/layer5_attention_weight_max": 90.30061340332031,
      "activations/layer5_attention_weight_min": -82.6604995727539,
      "activations/layer6_attention_weight_max": 59.21723556518555,
      "activations/layer6_attention_weight_min": -57.22117614746094,
      "activations/layer7_attention_weight_max": 80.16159057617188,
      "activations/layer7_attention_weight_min": -83.00418853759766,
      "activations/layer8_attention_weight_max": 47.62697219848633,
      "activations/layer8_attention_weight_min": -47.836856842041016,
      "activations/layer9_attention_weight_max": 56.72549057006836,
      "activations/layer9_attention_weight_min": -53.061702728271484,
      "epoch": 7.86,
      "learning_rate": 0.00010035,
      "loss": 2.7974,
      "step": 135200
    },
    {
      "activations/layer0_attention_weight_max": 15.027670860290527,
      "activations/layer0_attention_weight_min": -14.16408634185791,
      "activations/layer10_attention_weight_max": 36.11723709106445,
      "activations/layer10_attention_weight_min": -35.08938980102539,
      "activations/layer11_attention_weight_max": 31.371456146240234,
      "activations/layer11_attention_weight_min": -30.469398498535156,
      "activations/layer12_attention_weight_max": 18.906620025634766,
      "activations/layer12_attention_weight_min": -25.16862678527832,
      "activations/layer13_attention_weight_max": 43.81626510620117,
      "activations/layer13_attention_weight_min": -38.980735778808594,
      "activations/layer14_attention_weight_max": 46.8027229309082,
      "activations/layer14_attention_weight_min": -32.925636291503906,
      "activations/layer15_attention_weight_max": 35.73442459106445,
      "activations/layer15_attention_weight_min": -30.81612205505371,
      "activations/layer16_attention_weight_max": 33.42667007446289,
      "activations/layer16_attention_weight_min": -27.41832160949707,
      "activations/layer17_attention_weight_max": 56.68714141845703,
      "activations/layer17_attention_weight_min": -44.81919860839844,
      "activations/layer18_attention_weight_max": 48.442710876464844,
      "activations/layer18_attention_weight_min": -38.520118713378906,
      "activations/layer19_attention_weight_max": 20.579416275024414,
      "activations/layer19_attention_weight_min": -20.639692306518555,
      "activations/layer1_attention_weight_max": 16.988510131835938,
      "activations/layer1_attention_weight_min": -15.816972732543945,
      "activations/layer20_attention_weight_max": 23.539480209350586,
      "activations/layer20_attention_weight_min": -23.1782169342041,
      "activations/layer21_attention_weight_max": 40.859561920166016,
      "activations/layer21_attention_weight_min": -25.289804458618164,
      "activations/layer22_attention_weight_max": 32.77387619018555,
      "activations/layer22_attention_weight_min": -25.3793888092041,
      "activations/layer23_attention_weight_max": 36.673561096191406,
      "activations/layer23_attention_weight_min": -22.348907470703125,
      "activations/layer2_attention_weight_max": 32.83065414428711,
      "activations/layer2_attention_weight_min": -30.142324447631836,
      "activations/layer3_attention_weight_max": 94.10758209228516,
      "activations/layer3_attention_weight_min": -92.9485855102539,
      "activations/layer4_attention_weight_max": 105.13429260253906,
      "activations/layer4_attention_weight_min": -97.1185531616211,
      "activations/layer5_attention_weight_max": 86.56233978271484,
      "activations/layer5_attention_weight_min": -82.38896179199219,
      "activations/layer6_attention_weight_max": 56.5684928894043,
      "activations/layer6_attention_weight_min": -52.44681930541992,
      "activations/layer7_attention_weight_max": 78.63101196289062,
      "activations/layer7_attention_weight_min": -71.5911636352539,
      "activations/layer8_attention_weight_max": 44.86082077026367,
      "activations/layer8_attention_weight_min": -45.19685745239258,
      "activations/layer9_attention_weight_max": 51.835418701171875,
      "activations/layer9_attention_weight_min": -50.40924072265625,
      "epoch": 7.86,
      "learning_rate": 0.00010033106060606059,
      "loss": 2.8203,
      "step": 135250
    },
    {
      "activations/layer0_attention_weight_max": 15.150803565979004,
      "activations/layer0_attention_weight_min": -14.405759811401367,
      "activations/layer10_attention_weight_max": 36.00777816772461,
      "activations/layer10_attention_weight_min": -34.219970703125,
      "activations/layer11_attention_weight_max": 31.09254264831543,
      "activations/layer11_attention_weight_min": -32.191463470458984,
      "activations/layer12_attention_weight_max": 20.996070861816406,
      "activations/layer12_attention_weight_min": -26.467025756835938,
      "activations/layer13_attention_weight_max": 34.338600158691406,
      "activations/layer13_attention_weight_min": -34.132450103759766,
      "activations/layer14_attention_weight_max": 32.41444778442383,
      "activations/layer14_attention_weight_min": -29.30030059814453,
      "activations/layer15_attention_weight_max": 32.04833221435547,
      "activations/layer15_attention_weight_min": -31.08492660522461,
      "activations/layer16_attention_weight_max": 31.87611961364746,
      "activations/layer16_attention_weight_min": -28.597036361694336,
      "activations/layer17_attention_weight_max": 49.99824905395508,
      "activations/layer17_attention_weight_min": -43.13246536254883,
      "activations/layer18_attention_weight_max": 43.8516731262207,
      "activations/layer18_attention_weight_min": -35.56100845336914,
      "activations/layer19_attention_weight_max": 18.576213836669922,
      "activations/layer19_attention_weight_min": -20.23139762878418,
      "activations/layer1_attention_weight_max": 18.17970085144043,
      "activations/layer1_attention_weight_min": -16.411266326904297,
      "activations/layer20_attention_weight_max": 20.780662536621094,
      "activations/layer20_attention_weight_min": -20.979047775268555,
      "activations/layer21_attention_weight_max": 36.44225311279297,
      "activations/layer21_attention_weight_min": -22.050485610961914,
      "activations/layer22_attention_weight_max": 30.51558494567871,
      "activations/layer22_attention_weight_min": -23.27937889099121,
      "activations/layer23_attention_weight_max": 29.502696990966797,
      "activations/layer23_attention_weight_min": -21.112611770629883,
      "activations/layer2_attention_weight_max": 32.325584411621094,
      "activations/layer2_attention_weight_min": -28.511821746826172,
      "activations/layer3_attention_weight_max": 88.36180114746094,
      "activations/layer3_attention_weight_min": -88.03788757324219,
      "activations/layer4_attention_weight_max": 99.6373291015625,
      "activations/layer4_attention_weight_min": -97.91228485107422,
      "activations/layer5_attention_weight_max": 82.05085754394531,
      "activations/layer5_attention_weight_min": -78.25880432128906,
      "activations/layer6_attention_weight_max": 53.76515579223633,
      "activations/layer6_attention_weight_min": -51.05858612060547,
      "activations/layer7_attention_weight_max": 76.249267578125,
      "activations/layer7_attention_weight_min": -72.54338073730469,
      "activations/layer8_attention_weight_max": 43.057212829589844,
      "activations/layer8_attention_weight_min": -47.50764083862305,
      "activations/layer9_attention_weight_max": 48.69059753417969,
      "activations/layer9_attention_weight_min": -48.30576705932617,
      "epoch": 7.86,
      "learning_rate": 0.0001003121212121212,
      "loss": 2.8175,
      "step": 135300
    },
    {
      "activations/layer0_attention_weight_max": 15.614885330200195,
      "activations/layer0_attention_weight_min": -14.740917205810547,
      "activations/layer10_attention_weight_max": 37.46464920043945,
      "activations/layer10_attention_weight_min": -37.27800369262695,
      "activations/layer11_attention_weight_max": 33.703163146972656,
      "activations/layer11_attention_weight_min": -30.864133834838867,
      "activations/layer12_attention_weight_max": 28.169038772583008,
      "activations/layer12_attention_weight_min": -26.369226455688477,
      "activations/layer13_attention_weight_max": 45.87517547607422,
      "activations/layer13_attention_weight_min": -32.198665618896484,
      "activations/layer14_attention_weight_max": 40.773521423339844,
      "activations/layer14_attention_weight_min": -29.754823684692383,
      "activations/layer15_attention_weight_max": 39.02964401245117,
      "activations/layer15_attention_weight_min": -30.2827205657959,
      "activations/layer16_attention_weight_max": 38.13484191894531,
      "activations/layer16_attention_weight_min": -28.220304489135742,
      "activations/layer17_attention_weight_max": 58.66853332519531,
      "activations/layer17_attention_weight_min": -44.025142669677734,
      "activations/layer18_attention_weight_max": 49.41286849975586,
      "activations/layer18_attention_weight_min": -37.08536148071289,
      "activations/layer19_attention_weight_max": 24.300365447998047,
      "activations/layer19_attention_weight_min": -20.04372215270996,
      "activations/layer1_attention_weight_max": 16.58736801147461,
      "activations/layer1_attention_weight_min": -15.598572731018066,
      "activations/layer20_attention_weight_max": 26.465896606445312,
      "activations/layer20_attention_weight_min": -21.218961715698242,
      "activations/layer21_attention_weight_max": 42.692100524902344,
      "activations/layer21_attention_weight_min": -23.232505798339844,
      "activations/layer22_attention_weight_max": 33.41281509399414,
      "activations/layer22_attention_weight_min": -26.16424560546875,
      "activations/layer23_attention_weight_max": 36.40416717529297,
      "activations/layer23_attention_weight_min": -20.624523162841797,
      "activations/layer2_attention_weight_max": 32.016517639160156,
      "activations/layer2_attention_weight_min": -29.398193359375,
      "activations/layer3_attention_weight_max": 89.486083984375,
      "activations/layer3_attention_weight_min": -90.24298095703125,
      "activations/layer4_attention_weight_max": 106.2473373413086,
      "activations/layer4_attention_weight_min": -98.1665267944336,
      "activations/layer5_attention_weight_max": 91.01907348632812,
      "activations/layer5_attention_weight_min": -81.6898422241211,
      "activations/layer6_attention_weight_max": 56.815208435058594,
      "activations/layer6_attention_weight_min": -55.92997360229492,
      "activations/layer7_attention_weight_max": 78.91230010986328,
      "activations/layer7_attention_weight_min": -80.52689361572266,
      "activations/layer8_attention_weight_max": 47.86911392211914,
      "activations/layer8_attention_weight_min": -47.11322784423828,
      "activations/layer9_attention_weight_max": 55.50699996948242,
      "activations/layer9_attention_weight_min": -52.25189971923828,
      "epoch": 7.86,
      "learning_rate": 0.00010029318181818182,
      "loss": 2.8167,
      "step": 135350
    },
    {
      "activations/layer0_attention_weight_max": 15.59782600402832,
      "activations/layer0_attention_weight_min": -14.332361221313477,
      "activations/layer10_attention_weight_max": 35.99440383911133,
      "activations/layer10_attention_weight_min": -35.38497543334961,
      "activations/layer11_attention_weight_max": 30.76827621459961,
      "activations/layer11_attention_weight_min": -31.694072723388672,
      "activations/layer12_attention_weight_max": 20.63368797302246,
      "activations/layer12_attention_weight_min": -23.587541580200195,
      "activations/layer13_attention_weight_max": 41.77320098876953,
      "activations/layer13_attention_weight_min": -30.597991943359375,
      "activations/layer14_attention_weight_max": 44.44643783569336,
      "activations/layer14_attention_weight_min": -29.712961196899414,
      "activations/layer15_attention_weight_max": 35.15617752075195,
      "activations/layer15_attention_weight_min": -27.968280792236328,
      "activations/layer16_attention_weight_max": 31.820585250854492,
      "activations/layer16_attention_weight_min": -26.341073989868164,
      "activations/layer17_attention_weight_max": 52.68158721923828,
      "activations/layer17_attention_weight_min": -40.70242691040039,
      "activations/layer18_attention_weight_max": 46.32484436035156,
      "activations/layer18_attention_weight_min": -35.231910705566406,
      "activations/layer19_attention_weight_max": 23.214933395385742,
      "activations/layer19_attention_weight_min": -21.13178062438965,
      "activations/layer1_attention_weight_max": 17.375783920288086,
      "activations/layer1_attention_weight_min": -16.26563835144043,
      "activations/layer20_attention_weight_max": 24.447458267211914,
      "activations/layer20_attention_weight_min": -21.469449996948242,
      "activations/layer21_attention_weight_max": 37.16030502319336,
      "activations/layer21_attention_weight_min": -24.291128158569336,
      "activations/layer22_attention_weight_max": 29.280975341796875,
      "activations/layer22_attention_weight_min": -26.21402359008789,
      "activations/layer23_attention_weight_max": 31.773880004882812,
      "activations/layer23_attention_weight_min": -22.450435638427734,
      "activations/layer2_attention_weight_max": 30.67316246032715,
      "activations/layer2_attention_weight_min": -29.049407958984375,
      "activations/layer3_attention_weight_max": 86.11001586914062,
      "activations/layer3_attention_weight_min": -85.75907135009766,
      "activations/layer4_attention_weight_max": 96.5528564453125,
      "activations/layer4_attention_weight_min": -90.02796936035156,
      "activations/layer5_attention_weight_max": 82.89643859863281,
      "activations/layer5_attention_weight_min": -80.23368835449219,
      "activations/layer6_attention_weight_max": 54.06055450439453,
      "activations/layer6_attention_weight_min": -50.95515060424805,
      "activations/layer7_attention_weight_max": 77.40447998046875,
      "activations/layer7_attention_weight_min": -69.00740051269531,
      "activations/layer8_attention_weight_max": 41.49522399902344,
      "activations/layer8_attention_weight_min": -42.9600715637207,
      "activations/layer9_attention_weight_max": 49.79640579223633,
      "activations/layer9_attention_weight_min": -49.78485107421875,
      "epoch": 7.87,
      "learning_rate": 0.00010027424242424242,
      "loss": 2.8031,
      "step": 135400
    },
    {
      "activations/layer0_attention_weight_max": 16.065521240234375,
      "activations/layer0_attention_weight_min": -14.190706253051758,
      "activations/layer10_attention_weight_max": 38.446250915527344,
      "activations/layer10_attention_weight_min": -34.55689239501953,
      "activations/layer11_attention_weight_max": 30.552722930908203,
      "activations/layer11_attention_weight_min": -30.859580993652344,
      "activations/layer12_attention_weight_max": 20.45608901977539,
      "activations/layer12_attention_weight_min": -25.21278190612793,
      "activations/layer13_attention_weight_max": 39.16328430175781,
      "activations/layer13_attention_weight_min": -30.2664737701416,
      "activations/layer14_attention_weight_max": 37.60488510131836,
      "activations/layer14_attention_weight_min": -31.3607234954834,
      "activations/layer15_attention_weight_max": 37.039466857910156,
      "activations/layer15_attention_weight_min": -30.51738739013672,
      "activations/layer16_attention_weight_max": 36.357704162597656,
      "activations/layer16_attention_weight_min": -29.29473114013672,
      "activations/layer17_attention_weight_max": 58.70311737060547,
      "activations/layer17_attention_weight_min": -45.991817474365234,
      "activations/layer18_attention_weight_max": 49.970703125,
      "activations/layer18_attention_weight_min": -37.6866340637207,
      "activations/layer19_attention_weight_max": 22.97194480895996,
      "activations/layer19_attention_weight_min": -21.385984420776367,
      "activations/layer1_attention_weight_max": 16.523653030395508,
      "activations/layer1_attention_weight_min": -14.568695068359375,
      "activations/layer20_attention_weight_max": 22.23654556274414,
      "activations/layer20_attention_weight_min": -23.32377815246582,
      "activations/layer21_attention_weight_max": 39.70793533325195,
      "activations/layer21_attention_weight_min": -23.105106353759766,
      "activations/layer22_attention_weight_max": 34.1291618347168,
      "activations/layer22_attention_weight_min": -25.02164077758789,
      "activations/layer23_attention_weight_max": 37.23384475708008,
      "activations/layer23_attention_weight_min": -23.906225204467773,
      "activations/layer2_attention_weight_max": 30.497669219970703,
      "activations/layer2_attention_weight_min": -28.546798706054688,
      "activations/layer3_attention_weight_max": 84.96177673339844,
      "activations/layer3_attention_weight_min": -82.55059814453125,
      "activations/layer4_attention_weight_max": 100.36315155029297,
      "activations/layer4_attention_weight_min": -94.53582000732422,
      "activations/layer5_attention_weight_max": 84.09190368652344,
      "activations/layer5_attention_weight_min": -79.12198638916016,
      "activations/layer6_attention_weight_max": 53.69397735595703,
      "activations/layer6_attention_weight_min": -53.4857063293457,
      "activations/layer7_attention_weight_max": 76.3036880493164,
      "activations/layer7_attention_weight_min": -78.86367797851562,
      "activations/layer8_attention_weight_max": 44.40118408203125,
      "activations/layer8_attention_weight_min": -47.09540939331055,
      "activations/layer9_attention_weight_max": 50.28554916381836,
      "activations/layer9_attention_weight_min": -49.52748489379883,
      "epoch": 7.87,
      "learning_rate": 0.00010025530303030302,
      "loss": 2.8089,
      "step": 135450
    },
    {
      "activations/layer0_attention_weight_max": 15.293486595153809,
      "activations/layer0_attention_weight_min": -14.358551979064941,
      "activations/layer10_attention_weight_max": 40.04225540161133,
      "activations/layer10_attention_weight_min": -37.860382080078125,
      "activations/layer11_attention_weight_max": 31.065731048583984,
      "activations/layer11_attention_weight_min": -30.17938995361328,
      "activations/layer12_attention_weight_max": 26.530405044555664,
      "activations/layer12_attention_weight_min": -26.742719650268555,
      "activations/layer13_attention_weight_max": 44.29682540893555,
      "activations/layer13_attention_weight_min": -32.42582702636719,
      "activations/layer14_attention_weight_max": 37.340354919433594,
      "activations/layer14_attention_weight_min": -31.98737335205078,
      "activations/layer15_attention_weight_max": 35.29686737060547,
      "activations/layer15_attention_weight_min": -30.268198013305664,
      "activations/layer16_attention_weight_max": 33.256797790527344,
      "activations/layer16_attention_weight_min": -27.311662673950195,
      "activations/layer17_attention_weight_max": 51.83769989013672,
      "activations/layer17_attention_weight_min": -41.80643081665039,
      "activations/layer18_attention_weight_max": 47.96300506591797,
      "activations/layer18_attention_weight_min": -36.316062927246094,
      "activations/layer19_attention_weight_max": 22.479032516479492,
      "activations/layer19_attention_weight_min": -21.202360153198242,
      "activations/layer1_attention_weight_max": 16.87740707397461,
      "activations/layer1_attention_weight_min": -14.641429901123047,
      "activations/layer20_attention_weight_max": 26.102018356323242,
      "activations/layer20_attention_weight_min": -18.71681785583496,
      "activations/layer21_attention_weight_max": 37.280494689941406,
      "activations/layer21_attention_weight_min": -21.77475929260254,
      "activations/layer22_attention_weight_max": 29.920001983642578,
      "activations/layer22_attention_weight_min": -22.455078125,
      "activations/layer23_attention_weight_max": 33.01787185668945,
      "activations/layer23_attention_weight_min": -20.752544403076172,
      "activations/layer2_attention_weight_max": 32.486087799072266,
      "activations/layer2_attention_weight_min": -29.960920333862305,
      "activations/layer3_attention_weight_max": 91.36692810058594,
      "activations/layer3_attention_weight_min": -91.88827514648438,
      "activations/layer4_attention_weight_max": 108.7652816772461,
      "activations/layer4_attention_weight_min": -101.58489227294922,
      "activations/layer5_attention_weight_max": 93.84563446044922,
      "activations/layer5_attention_weight_min": -83.30030059814453,
      "activations/layer6_attention_weight_max": 58.740440368652344,
      "activations/layer6_attention_weight_min": -52.31732940673828,
      "activations/layer7_attention_weight_max": 78.62894439697266,
      "activations/layer7_attention_weight_min": -70.1417465209961,
      "activations/layer8_attention_weight_max": 47.3058967590332,
      "activations/layer8_attention_weight_min": -43.65311813354492,
      "activations/layer9_attention_weight_max": 51.49446487426758,
      "activations/layer9_attention_weight_min": -50.03445816040039,
      "epoch": 7.87,
      "learning_rate": 0.00010023636363636362,
      "loss": 2.8039,
      "step": 135500
    },
    {
      "activations/layer0_attention_weight_max": 14.816729545593262,
      "activations/layer0_attention_weight_min": -13.978592872619629,
      "activations/layer10_attention_weight_max": 37.039588928222656,
      "activations/layer10_attention_weight_min": -34.89643096923828,
      "activations/layer11_attention_weight_max": 31.96887969970703,
      "activations/layer11_attention_weight_min": -30.488616943359375,
      "activations/layer12_attention_weight_max": 21.40965461730957,
      "activations/layer12_attention_weight_min": -27.22242546081543,
      "activations/layer13_attention_weight_max": 43.4550895690918,
      "activations/layer13_attention_weight_min": -32.98447799682617,
      "activations/layer14_attention_weight_max": 47.448184967041016,
      "activations/layer14_attention_weight_min": -31.08787727355957,
      "activations/layer15_attention_weight_max": 38.713741302490234,
      "activations/layer15_attention_weight_min": -30.084522247314453,
      "activations/layer16_attention_weight_max": 36.52456283569336,
      "activations/layer16_attention_weight_min": -29.308399200439453,
      "activations/layer17_attention_weight_max": 56.050140380859375,
      "activations/layer17_attention_weight_min": -42.05624008178711,
      "activations/layer18_attention_weight_max": 52.02139663696289,
      "activations/layer18_attention_weight_min": -36.24103546142578,
      "activations/layer19_attention_weight_max": 24.641878128051758,
      "activations/layer19_attention_weight_min": -22.28706169128418,
      "activations/layer1_attention_weight_max": 17.053733825683594,
      "activations/layer1_attention_weight_min": -14.056861877441406,
      "activations/layer20_attention_weight_max": 30.513626098632812,
      "activations/layer20_attention_weight_min": -20.42040252685547,
      "activations/layer21_attention_weight_max": 51.35104751586914,
      "activations/layer21_attention_weight_min": -27.83356285095215,
      "activations/layer22_attention_weight_max": 36.27879333496094,
      "activations/layer22_attention_weight_min": -26.212026596069336,
      "activations/layer23_attention_weight_max": 42.449363708496094,
      "activations/layer23_attention_weight_min": -24.2308349609375,
      "activations/layer2_attention_weight_max": 29.200223922729492,
      "activations/layer2_attention_weight_min": -28.12912368774414,
      "activations/layer3_attention_weight_max": 82.97393798828125,
      "activations/layer3_attention_weight_min": -85.57657623291016,
      "activations/layer4_attention_weight_max": 93.30599212646484,
      "activations/layer4_attention_weight_min": -91.63628387451172,
      "activations/layer5_attention_weight_max": 78.79370880126953,
      "activations/layer5_attention_weight_min": -76.11077117919922,
      "activations/layer6_attention_weight_max": 52.63471984863281,
      "activations/layer6_attention_weight_min": -51.8087272644043,
      "activations/layer7_attention_weight_max": 73.97249603271484,
      "activations/layer7_attention_weight_min": -70.71221923828125,
      "activations/layer8_attention_weight_max": 43.39170455932617,
      "activations/layer8_attention_weight_min": -43.68228530883789,
      "activations/layer9_attention_weight_max": 49.99418258666992,
      "activations/layer9_attention_weight_min": -47.196834564208984,
      "epoch": 7.88,
      "learning_rate": 0.00010021780303030303,
      "loss": 2.7955,
      "step": 135550
    },
    {
      "activations/layer0_attention_weight_max": 15.276345252990723,
      "activations/layer0_attention_weight_min": -14.196030616760254,
      "activations/layer10_attention_weight_max": 36.49434280395508,
      "activations/layer10_attention_weight_min": -36.16281509399414,
      "activations/layer11_attention_weight_max": 29.629728317260742,
      "activations/layer11_attention_weight_min": -31.526302337646484,
      "activations/layer12_attention_weight_max": 20.632057189941406,
      "activations/layer12_attention_weight_min": -26.234004974365234,
      "activations/layer13_attention_weight_max": 38.67371368408203,
      "activations/layer13_attention_weight_min": -29.186275482177734,
      "activations/layer14_attention_weight_max": 33.88671112060547,
      "activations/layer14_attention_weight_min": -29.916244506835938,
      "activations/layer15_attention_weight_max": 36.00299835205078,
      "activations/layer15_attention_weight_min": -29.528213500976562,
      "activations/layer16_attention_weight_max": 29.8648738861084,
      "activations/layer16_attention_weight_min": -28.454572677612305,
      "activations/layer17_attention_weight_max": 49.13154602050781,
      "activations/layer17_attention_weight_min": -44.97597122192383,
      "activations/layer18_attention_weight_max": 42.66613006591797,
      "activations/layer18_attention_weight_min": -39.88462829589844,
      "activations/layer19_attention_weight_max": 20.831205368041992,
      "activations/layer19_attention_weight_min": -18.27332305908203,
      "activations/layer1_attention_weight_max": 16.802814483642578,
      "activations/layer1_attention_weight_min": -13.752362251281738,
      "activations/layer20_attention_weight_max": 21.7034912109375,
      "activations/layer20_attention_weight_min": -22.04096031188965,
      "activations/layer21_attention_weight_max": 34.80379104614258,
      "activations/layer21_attention_weight_min": -23.717039108276367,
      "activations/layer22_attention_weight_max": 29.613361358642578,
      "activations/layer22_attention_weight_min": -25.99049949645996,
      "activations/layer23_attention_weight_max": 29.5642147064209,
      "activations/layer23_attention_weight_min": -20.785335540771484,
      "activations/layer2_attention_weight_max": 30.586746215820312,
      "activations/layer2_attention_weight_min": -29.283193588256836,
      "activations/layer3_attention_weight_max": 88.0168228149414,
      "activations/layer3_attention_weight_min": -84.0243911743164,
      "activations/layer4_attention_weight_max": 98.80897521972656,
      "activations/layer4_attention_weight_min": -91.35375213623047,
      "activations/layer5_attention_weight_max": 82.53616333007812,
      "activations/layer5_attention_weight_min": -80.51150512695312,
      "activations/layer6_attention_weight_max": 54.91108322143555,
      "activations/layer6_attention_weight_min": -52.317237854003906,
      "activations/layer7_attention_weight_max": 80.25398254394531,
      "activations/layer7_attention_weight_min": -73.92024993896484,
      "activations/layer8_attention_weight_max": 45.68462371826172,
      "activations/layer8_attention_weight_min": -46.618133544921875,
      "activations/layer9_attention_weight_max": 52.00972366333008,
      "activations/layer9_attention_weight_min": -47.66118621826172,
      "epoch": 7.88,
      "learning_rate": 0.00010019886363636363,
      "loss": 2.8077,
      "step": 135600
    },
    {
      "activations/layer0_attention_weight_max": 14.91405963897705,
      "activations/layer0_attention_weight_min": -14.350773811340332,
      "activations/layer10_attention_weight_max": 34.870235443115234,
      "activations/layer10_attention_weight_min": -34.094173431396484,
      "activations/layer11_attention_weight_max": 30.072837829589844,
      "activations/layer11_attention_weight_min": -29.46868133544922,
      "activations/layer12_attention_weight_max": 24.406572341918945,
      "activations/layer12_attention_weight_min": -28.7158260345459,
      "activations/layer13_attention_weight_max": 38.496639251708984,
      "activations/layer13_attention_weight_min": -29.96813201904297,
      "activations/layer14_attention_weight_max": 42.962486267089844,
      "activations/layer14_attention_weight_min": -30.68778419494629,
      "activations/layer15_attention_weight_max": 38.531307220458984,
      "activations/layer15_attention_weight_min": -28.999107360839844,
      "activations/layer16_attention_weight_max": 32.976966857910156,
      "activations/layer16_attention_weight_min": -26.60481071472168,
      "activations/layer17_attention_weight_max": 54.88446807861328,
      "activations/layer17_attention_weight_min": -43.52101516723633,
      "activations/layer18_attention_weight_max": 46.07164001464844,
      "activations/layer18_attention_weight_min": -35.3128776550293,
      "activations/layer19_attention_weight_max": 20.544837951660156,
      "activations/layer19_attention_weight_min": -20.826906204223633,
      "activations/layer1_attention_weight_max": 17.13005256652832,
      "activations/layer1_attention_weight_min": -15.888297080993652,
      "activations/layer20_attention_weight_max": 21.750328063964844,
      "activations/layer20_attention_weight_min": -29.151100158691406,
      "activations/layer21_attention_weight_max": 43.43149948120117,
      "activations/layer21_attention_weight_min": -26.131772994995117,
      "activations/layer22_attention_weight_max": 29.408588409423828,
      "activations/layer22_attention_weight_min": -27.65039825439453,
      "activations/layer23_attention_weight_max": 32.112728118896484,
      "activations/layer23_attention_weight_min": -22.694433212280273,
      "activations/layer2_attention_weight_max": 31.800567626953125,
      "activations/layer2_attention_weight_min": -30.549949645996094,
      "activations/layer3_attention_weight_max": 89.00204467773438,
      "activations/layer3_attention_weight_min": -87.88168334960938,
      "activations/layer4_attention_weight_max": 96.17437744140625,
      "activations/layer4_attention_weight_min": -90.27629852294922,
      "activations/layer5_attention_weight_max": 82.33250427246094,
      "activations/layer5_attention_weight_min": -76.01667785644531,
      "activations/layer6_attention_weight_max": 53.32744598388672,
      "activations/layer6_attention_weight_min": -50.73540115356445,
      "activations/layer7_attention_weight_max": 75.07905578613281,
      "activations/layer7_attention_weight_min": -70.4254150390625,
      "activations/layer8_attention_weight_max": 41.79225540161133,
      "activations/layer8_attention_weight_min": -43.83757781982422,
      "activations/layer9_attention_weight_max": 51.33366012573242,
      "activations/layer9_attention_weight_min": -47.74515151977539,
      "epoch": 7.88,
      "learning_rate": 0.00010017992424242424,
      "loss": 2.809,
      "step": 135650
    },
    {
      "activations/layer0_attention_weight_max": 14.345246315002441,
      "activations/layer0_attention_weight_min": -14.317633628845215,
      "activations/layer10_attention_weight_max": 37.544776916503906,
      "activations/layer10_attention_weight_min": -35.036094665527344,
      "activations/layer11_attention_weight_max": 31.199886322021484,
      "activations/layer11_attention_weight_min": -30.99344253540039,
      "activations/layer12_attention_weight_max": 22.061140060424805,
      "activations/layer12_attention_weight_min": -23.348072052001953,
      "activations/layer13_attention_weight_max": 43.22480010986328,
      "activations/layer13_attention_weight_min": -31.66536521911621,
      "activations/layer14_attention_weight_max": 40.11202621459961,
      "activations/layer14_attention_weight_min": -29.613309860229492,
      "activations/layer15_attention_weight_max": 37.70479202270508,
      "activations/layer15_attention_weight_min": -27.7473087310791,
      "activations/layer16_attention_weight_max": 36.55476379394531,
      "activations/layer16_attention_weight_min": -28.046728134155273,
      "activations/layer17_attention_weight_max": 55.49913787841797,
      "activations/layer17_attention_weight_min": -42.55775833129883,
      "activations/layer18_attention_weight_max": 52.89255142211914,
      "activations/layer18_attention_weight_min": -35.437252044677734,
      "activations/layer19_attention_weight_max": 28.724729537963867,
      "activations/layer19_attention_weight_min": -19.997684478759766,
      "activations/layer1_attention_weight_max": 17.78816032409668,
      "activations/layer1_attention_weight_min": -14.341962814331055,
      "activations/layer20_attention_weight_max": 24.980939865112305,
      "activations/layer20_attention_weight_min": -21.930042266845703,
      "activations/layer21_attention_weight_max": 44.8126106262207,
      "activations/layer21_attention_weight_min": -25.084331512451172,
      "activations/layer22_attention_weight_max": 34.6197395324707,
      "activations/layer22_attention_weight_min": -24.528209686279297,
      "activations/layer23_attention_weight_max": 40.027076721191406,
      "activations/layer23_attention_weight_min": -23.789581298828125,
      "activations/layer2_attention_weight_max": 32.27930450439453,
      "activations/layer2_attention_weight_min": -29.49578857421875,
      "activations/layer3_attention_weight_max": 82.9327392578125,
      "activations/layer3_attention_weight_min": -86.25141906738281,
      "activations/layer4_attention_weight_max": 96.28829193115234,
      "activations/layer4_attention_weight_min": -92.55772399902344,
      "activations/layer5_attention_weight_max": 81.64559936523438,
      "activations/layer5_attention_weight_min": -77.06379699707031,
      "activations/layer6_attention_weight_max": 54.13820266723633,
      "activations/layer6_attention_weight_min": -54.69038772583008,
      "activations/layer7_attention_weight_max": 73.48960876464844,
      "activations/layer7_attention_weight_min": -74.50182342529297,
      "activations/layer8_attention_weight_max": 43.3463134765625,
      "activations/layer8_attention_weight_min": -44.24305725097656,
      "activations/layer9_attention_weight_max": 48.71149826049805,
      "activations/layer9_attention_weight_min": -49.13238525390625,
      "epoch": 7.88,
      "learning_rate": 0.00010016098484848483,
      "loss": 2.824,
      "step": 135700
    },
    {
      "activations/layer0_attention_weight_max": 14.980730056762695,
      "activations/layer0_attention_weight_min": -14.202200889587402,
      "activations/layer10_attention_weight_max": 36.70901870727539,
      "activations/layer10_attention_weight_min": -35.384422302246094,
      "activations/layer11_attention_weight_max": 30.601911544799805,
      "activations/layer11_attention_weight_min": -30.982284545898438,
      "activations/layer12_attention_weight_max": 24.042232513427734,
      "activations/layer12_attention_weight_min": -23.039234161376953,
      "activations/layer13_attention_weight_max": 40.01900100708008,
      "activations/layer13_attention_weight_min": -32.371456146240234,
      "activations/layer14_attention_weight_max": 39.86890411376953,
      "activations/layer14_attention_weight_min": -29.810562133789062,
      "activations/layer15_attention_weight_max": 36.353302001953125,
      "activations/layer15_attention_weight_min": -29.77798080444336,
      "activations/layer16_attention_weight_max": 30.994787216186523,
      "activations/layer16_attention_weight_min": -29.04438018798828,
      "activations/layer17_attention_weight_max": 57.63807678222656,
      "activations/layer17_attention_weight_min": -43.38334655761719,
      "activations/layer18_attention_weight_max": 48.727813720703125,
      "activations/layer18_attention_weight_min": -38.09703826904297,
      "activations/layer19_attention_weight_max": 20.76725959777832,
      "activations/layer19_attention_weight_min": -20.555265426635742,
      "activations/layer1_attention_weight_max": 18.368568420410156,
      "activations/layer1_attention_weight_min": -18.207609176635742,
      "activations/layer20_attention_weight_max": 20.50039291381836,
      "activations/layer20_attention_weight_min": -23.989030838012695,
      "activations/layer21_attention_weight_max": 43.51527786254883,
      "activations/layer21_attention_weight_min": -24.9466495513916,
      "activations/layer22_attention_weight_max": 32.004425048828125,
      "activations/layer22_attention_weight_min": -24.59683609008789,
      "activations/layer23_attention_weight_max": 39.02467727661133,
      "activations/layer23_attention_weight_min": -19.91254234313965,
      "activations/layer2_attention_weight_max": 31.979270935058594,
      "activations/layer2_attention_weight_min": -30.10535430908203,
      "activations/layer3_attention_weight_max": 88.95935821533203,
      "activations/layer3_attention_weight_min": -94.78890991210938,
      "activations/layer4_attention_weight_max": 99.96581268310547,
      "activations/layer4_attention_weight_min": -96.06971740722656,
      "activations/layer5_attention_weight_max": 87.84378051757812,
      "activations/layer5_attention_weight_min": -82.86471557617188,
      "activations/layer6_attention_weight_max": 56.87293243408203,
      "activations/layer6_attention_weight_min": -54.21327209472656,
      "activations/layer7_attention_weight_max": 73.03517150878906,
      "activations/layer7_attention_weight_min": -71.51936340332031,
      "activations/layer8_attention_weight_max": 45.5561408996582,
      "activations/layer8_attention_weight_min": -46.07600784301758,
      "activations/layer9_attention_weight_max": 51.35203170776367,
      "activations/layer9_attention_weight_min": -48.32233810424805,
      "epoch": 7.89,
      "learning_rate": 0.00010014204545454545,
      "loss": 2.8151,
      "step": 135750
    },
    {
      "activations/layer0_attention_weight_max": 14.643952369689941,
      "activations/layer0_attention_weight_min": -14.490132331848145,
      "activations/layer10_attention_weight_max": 38.52820587158203,
      "activations/layer10_attention_weight_min": -35.12990951538086,
      "activations/layer11_attention_weight_max": 30.293785095214844,
      "activations/layer11_attention_weight_min": -30.307632446289062,
      "activations/layer12_attention_weight_max": 21.191198348999023,
      "activations/layer12_attention_weight_min": -26.378009796142578,
      "activations/layer13_attention_weight_max": 50.74519348144531,
      "activations/layer13_attention_weight_min": -36.74470138549805,
      "activations/layer14_attention_weight_max": 52.03232192993164,
      "activations/layer14_attention_weight_min": -36.274253845214844,
      "activations/layer15_attention_weight_max": 37.63410949707031,
      "activations/layer15_attention_weight_min": -29.76763343811035,
      "activations/layer16_attention_weight_max": 34.12627029418945,
      "activations/layer16_attention_weight_min": -27.168302536010742,
      "activations/layer17_attention_weight_max": 56.38279724121094,
      "activations/layer17_attention_weight_min": -43.49215316772461,
      "activations/layer18_attention_weight_max": 49.04731369018555,
      "activations/layer18_attention_weight_min": -35.6696662902832,
      "activations/layer19_attention_weight_max": 22.649343490600586,
      "activations/layer19_attention_weight_min": -20.029130935668945,
      "activations/layer1_attention_weight_max": 17.956727981567383,
      "activations/layer1_attention_weight_min": -15.520867347717285,
      "activations/layer20_attention_weight_max": 22.41918182373047,
      "activations/layer20_attention_weight_min": -21.210651397705078,
      "activations/layer21_attention_weight_max": 35.97441864013672,
      "activations/layer21_attention_weight_min": -23.459861755371094,
      "activations/layer22_attention_weight_max": 33.150413513183594,
      "activations/layer22_attention_weight_min": -22.559595108032227,
      "activations/layer23_attention_weight_max": 36.616546630859375,
      "activations/layer23_attention_weight_min": -21.03815460205078,
      "activations/layer2_attention_weight_max": 30.77309799194336,
      "activations/layer2_attention_weight_min": -29.812763214111328,
      "activations/layer3_attention_weight_max": 84.31800842285156,
      "activations/layer3_attention_weight_min": -87.95323944091797,
      "activations/layer4_attention_weight_max": 97.37249755859375,
      "activations/layer4_attention_weight_min": -93.90059661865234,
      "activations/layer5_attention_weight_max": 84.51351928710938,
      "activations/layer5_attention_weight_min": -79.68165588378906,
      "activations/layer6_attention_weight_max": 57.307220458984375,
      "activations/layer6_attention_weight_min": -53.87794494628906,
      "activations/layer7_attention_weight_max": 80.31775665283203,
      "activations/layer7_attention_weight_min": -76.18173217773438,
      "activations/layer8_attention_weight_max": 45.112709045410156,
      "activations/layer8_attention_weight_min": -45.33891677856445,
      "activations/layer9_attention_weight_max": 61.67301559448242,
      "activations/layer9_attention_weight_min": -47.29164505004883,
      "epoch": 7.89,
      "learning_rate": 0.00010012310606060606,
      "loss": 2.8107,
      "step": 135800
    },
    {
      "activations/layer0_attention_weight_max": 14.640355110168457,
      "activations/layer0_attention_weight_min": -14.404327392578125,
      "activations/layer10_attention_weight_max": 35.86358642578125,
      "activations/layer10_attention_weight_min": -33.76836013793945,
      "activations/layer11_attention_weight_max": 30.528858184814453,
      "activations/layer11_attention_weight_min": -32.18182373046875,
      "activations/layer12_attention_weight_max": 20.548751831054688,
      "activations/layer12_attention_weight_min": -26.651185989379883,
      "activations/layer13_attention_weight_max": 47.278053283691406,
      "activations/layer13_attention_weight_min": -29.532012939453125,
      "activations/layer14_attention_weight_max": 40.033485412597656,
      "activations/layer14_attention_weight_min": -29.53378677368164,
      "activations/layer15_attention_weight_max": 35.74623489379883,
      "activations/layer15_attention_weight_min": -28.991382598876953,
      "activations/layer16_attention_weight_max": 30.249170303344727,
      "activations/layer16_attention_weight_min": -27.940685272216797,
      "activations/layer17_attention_weight_max": 53.80452346801758,
      "activations/layer17_attention_weight_min": -42.11540603637695,
      "activations/layer18_attention_weight_max": 46.63371658325195,
      "activations/layer18_attention_weight_min": -33.53624725341797,
      "activations/layer19_attention_weight_max": 20.923398971557617,
      "activations/layer19_attention_weight_min": -21.269563674926758,
      "activations/layer1_attention_weight_max": 17.008516311645508,
      "activations/layer1_attention_weight_min": -14.50340747833252,
      "activations/layer20_attention_weight_max": 21.565120697021484,
      "activations/layer20_attention_weight_min": -21.084503173828125,
      "activations/layer21_attention_weight_max": 38.80538558959961,
      "activations/layer21_attention_weight_min": -22.883197784423828,
      "activations/layer22_attention_weight_max": 34.346580505371094,
      "activations/layer22_attention_weight_min": -26.35087776184082,
      "activations/layer23_attention_weight_max": 37.54952621459961,
      "activations/layer23_attention_weight_min": -23.030353546142578,
      "activations/layer2_attention_weight_max": 30.53169822692871,
      "activations/layer2_attention_weight_min": -28.42432975769043,
      "activations/layer3_attention_weight_max": 82.37077331542969,
      "activations/layer3_attention_weight_min": -85.9473876953125,
      "activations/layer4_attention_weight_max": 98.10295867919922,
      "activations/layer4_attention_weight_min": -92.2823257446289,
      "activations/layer5_attention_weight_max": 85.54155731201172,
      "activations/layer5_attention_weight_min": -76.88461303710938,
      "activations/layer6_attention_weight_max": 56.69669723510742,
      "activations/layer6_attention_weight_min": -54.932254791259766,
      "activations/layer7_attention_weight_max": 77.06434631347656,
      "activations/layer7_attention_weight_min": -69.80482482910156,
      "activations/layer8_attention_weight_max": 44.82960510253906,
      "activations/layer8_attention_weight_min": -43.89839172363281,
      "activations/layer9_attention_weight_max": 48.863014221191406,
      "activations/layer9_attention_weight_min": -48.78553771972656,
      "epoch": 7.89,
      "learning_rate": 0.00010010416666666665,
      "loss": 2.8044,
      "step": 135850
    },
    {
      "activations/layer0_attention_weight_max": 15.669037818908691,
      "activations/layer0_attention_weight_min": -14.102197647094727,
      "activations/layer10_attention_weight_max": 35.04149627685547,
      "activations/layer10_attention_weight_min": -36.87608337402344,
      "activations/layer11_attention_weight_max": 30.611501693725586,
      "activations/layer11_attention_weight_min": -32.169002532958984,
      "activations/layer12_attention_weight_max": 18.2300968170166,
      "activations/layer12_attention_weight_min": -30.81214141845703,
      "activations/layer13_attention_weight_max": 40.19755172729492,
      "activations/layer13_attention_weight_min": -30.105600357055664,
      "activations/layer14_attention_weight_max": 34.91946029663086,
      "activations/layer14_attention_weight_min": -31.512495040893555,
      "activations/layer15_attention_weight_max": 32.1924934387207,
      "activations/layer15_attention_weight_min": -29.896360397338867,
      "activations/layer16_attention_weight_max": 32.02604675292969,
      "activations/layer16_attention_weight_min": -27.138063430786133,
      "activations/layer17_attention_weight_max": 52.80385208129883,
      "activations/layer17_attention_weight_min": -45.73006820678711,
      "activations/layer18_attention_weight_max": 43.91918182373047,
      "activations/layer18_attention_weight_min": -39.07563400268555,
      "activations/layer19_attention_weight_max": 19.959936141967773,
      "activations/layer19_attention_weight_min": -22.18103790283203,
      "activations/layer1_attention_weight_max": 18.529003143310547,
      "activations/layer1_attention_weight_min": -18.318384170532227,
      "activations/layer20_attention_weight_max": 20.782825469970703,
      "activations/layer20_attention_weight_min": -23.99803924560547,
      "activations/layer21_attention_weight_max": 35.88985824584961,
      "activations/layer21_attention_weight_min": -24.73455810546875,
      "activations/layer22_attention_weight_max": 30.52262306213379,
      "activations/layer22_attention_weight_min": -25.812002182006836,
      "activations/layer23_attention_weight_max": 34.485137939453125,
      "activations/layer23_attention_weight_min": -22.944259643554688,
      "activations/layer2_attention_weight_max": 31.55813217163086,
      "activations/layer2_attention_weight_min": -27.767108917236328,
      "activations/layer3_attention_weight_max": 85.5721206665039,
      "activations/layer3_attention_weight_min": -91.93397521972656,
      "activations/layer4_attention_weight_max": 97.17642974853516,
      "activations/layer4_attention_weight_min": -97.49423217773438,
      "activations/layer5_attention_weight_max": 81.31704711914062,
      "activations/layer5_attention_weight_min": -78.15065002441406,
      "activations/layer6_attention_weight_max": 53.82732391357422,
      "activations/layer6_attention_weight_min": -51.593711853027344,
      "activations/layer7_attention_weight_max": 74.20533752441406,
      "activations/layer7_attention_weight_min": -72.18492126464844,
      "activations/layer8_attention_weight_max": 43.984249114990234,
      "activations/layer8_attention_weight_min": -44.73641586303711,
      "activations/layer9_attention_weight_max": 52.828155517578125,
      "activations/layer9_attention_weight_min": -50.68629837036133,
      "epoch": 7.9,
      "learning_rate": 0.00010008522727272726,
      "loss": 2.8101,
      "step": 135900
    },
    {
      "activations/layer0_attention_weight_max": 15.20323657989502,
      "activations/layer0_attention_weight_min": -14.129256248474121,
      "activations/layer10_attention_weight_max": 35.11945343017578,
      "activations/layer10_attention_weight_min": -37.31116485595703,
      "activations/layer11_attention_weight_max": 29.814456939697266,
      "activations/layer11_attention_weight_min": -30.932132720947266,
      "activations/layer12_attention_weight_max": 21.998859405517578,
      "activations/layer12_attention_weight_min": -25.350961685180664,
      "activations/layer13_attention_weight_max": 41.93937301635742,
      "activations/layer13_attention_weight_min": -32.5916862487793,
      "activations/layer14_attention_weight_max": 37.920780181884766,
      "activations/layer14_attention_weight_min": -30.393722534179688,
      "activations/layer15_attention_weight_max": 34.99649429321289,
      "activations/layer15_attention_weight_min": -28.954744338989258,
      "activations/layer16_attention_weight_max": 34.245079040527344,
      "activations/layer16_attention_weight_min": -28.98927879333496,
      "activations/layer17_attention_weight_max": 54.0614128112793,
      "activations/layer17_attention_weight_min": -41.95894241333008,
      "activations/layer18_attention_weight_max": 49.548011779785156,
      "activations/layer18_attention_weight_min": -39.43406295776367,
      "activations/layer19_attention_weight_max": 22.47467803955078,
      "activations/layer19_attention_weight_min": -21.238981246948242,
      "activations/layer1_attention_weight_max": 17.486167907714844,
      "activations/layer1_attention_weight_min": -16.300674438476562,
      "activations/layer20_attention_weight_max": 25.147624969482422,
      "activations/layer20_attention_weight_min": -22.238037109375,
      "activations/layer21_attention_weight_max": 45.201595306396484,
      "activations/layer21_attention_weight_min": -25.63694953918457,
      "activations/layer22_attention_weight_max": 36.5215950012207,
      "activations/layer22_attention_weight_min": -27.13674545288086,
      "activations/layer23_attention_weight_max": 37.10515594482422,
      "activations/layer23_attention_weight_min": -23.06122589111328,
      "activations/layer2_attention_weight_max": 31.560537338256836,
      "activations/layer2_attention_weight_min": -28.44574546813965,
      "activations/layer3_attention_weight_max": 85.6387710571289,
      "activations/layer3_attention_weight_min": -87.52800750732422,
      "activations/layer4_attention_weight_max": 102.34850311279297,
      "activations/layer4_attention_weight_min": -95.53003692626953,
      "activations/layer5_attention_weight_max": 84.39695739746094,
      "activations/layer5_attention_weight_min": -77.42801666259766,
      "activations/layer6_attention_weight_max": 53.876304626464844,
      "activations/layer6_attention_weight_min": -52.76789855957031,
      "activations/layer7_attention_weight_max": 74.564208984375,
      "activations/layer7_attention_weight_min": -75.4347152709961,
      "activations/layer8_attention_weight_max": 43.73767852783203,
      "activations/layer8_attention_weight_min": -44.999656677246094,
      "activations/layer9_attention_weight_max": 47.771446228027344,
      "activations/layer9_attention_weight_min": -49.74045944213867,
      "epoch": 7.9,
      "learning_rate": 0.00010006628787878788,
      "loss": 2.8095,
      "step": 135950
    },
    {
      "activations/layer0_attention_weight_max": 15.23625659942627,
      "activations/layer0_attention_weight_min": -14.318131446838379,
      "activations/layer10_attention_weight_max": 43.047996520996094,
      "activations/layer10_attention_weight_min": -39.7049446105957,
      "activations/layer11_attention_weight_max": 38.08942413330078,
      "activations/layer11_attention_weight_min": -31.24267578125,
      "activations/layer12_attention_weight_max": 22.970298767089844,
      "activations/layer12_attention_weight_min": -26.96862030029297,
      "activations/layer13_attention_weight_max": 50.51008224487305,
      "activations/layer13_attention_weight_min": -34.119571685791016,
      "activations/layer14_attention_weight_max": 64.58106231689453,
      "activations/layer14_attention_weight_min": -37.60677719116211,
      "activations/layer15_attention_weight_max": 48.535308837890625,
      "activations/layer15_attention_weight_min": -31.263046264648438,
      "activations/layer16_attention_weight_max": 31.887117385864258,
      "activations/layer16_attention_weight_min": -27.384244918823242,
      "activations/layer17_attention_weight_max": 56.19731521606445,
      "activations/layer17_attention_weight_min": -43.9479866027832,
      "activations/layer18_attention_weight_max": 49.2318000793457,
      "activations/layer18_attention_weight_min": -37.7657585144043,
      "activations/layer19_attention_weight_max": 21.62899398803711,
      "activations/layer19_attention_weight_min": -20.797714233398438,
      "activations/layer1_attention_weight_max": 17.137189865112305,
      "activations/layer1_attention_weight_min": -15.332905769348145,
      "activations/layer20_attention_weight_max": 23.190860748291016,
      "activations/layer20_attention_weight_min": -24.1204891204834,
      "activations/layer21_attention_weight_max": 45.10459899902344,
      "activations/layer21_attention_weight_min": -23.6455135345459,
      "activations/layer22_attention_weight_max": 32.19043731689453,
      "activations/layer22_attention_weight_min": -24.73029899597168,
      "activations/layer23_attention_weight_max": 37.46469497680664,
      "activations/layer23_attention_weight_min": -20.978477478027344,
      "activations/layer2_attention_weight_max": 31.84552574157715,
      "activations/layer2_attention_weight_min": -30.92630386352539,
      "activations/layer3_attention_weight_max": 87.64458465576172,
      "activations/layer3_attention_weight_min": -87.41681671142578,
      "activations/layer4_attention_weight_max": 101.57594299316406,
      "activations/layer4_attention_weight_min": -96.426513671875,
      "activations/layer5_attention_weight_max": 89.95973205566406,
      "activations/layer5_attention_weight_min": -81.38963317871094,
      "activations/layer6_attention_weight_max": 57.74574279785156,
      "activations/layer6_attention_weight_min": -54.87516403198242,
      "activations/layer7_attention_weight_max": 89.05184936523438,
      "activations/layer7_attention_weight_min": -79.89154815673828,
      "activations/layer8_attention_weight_max": 54.12321472167969,
      "activations/layer8_attention_weight_min": -48.976905822753906,
      "activations/layer9_attention_weight_max": 69.00721740722656,
      "activations/layer9_attention_weight_min": -51.650062561035156,
      "epoch": 7.9,
      "learning_rate": 0.00010004734848484847,
      "loss": 2.8239,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.5764,
      "eval_samples_per_second": 500.677,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.5764,
      "eval_openwebtext_samples_per_second": 500.677,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 1.9853,
      "eval_wikitext_samples_per_second": 229.683,
      "step": 136000
    },
    {
      "epoch": 7.9,
      "eval_lambada_loss": 2.81640625,
      "eval_lambada_ppl": 16.716667072215767,
      "eval_lambada_runtime": 9.6133,
      "eval_lambada_samples_per_second": 506.484,
      "step": 136000
    },
    {
      "activations/layer0_attention_weight_max": 15.251405715942383,
      "activations/layer0_attention_weight_min": -14.036380767822266,
      "activations/layer10_attention_weight_max": 37.11745071411133,
      "activations/layer10_attention_weight_min": -35.716217041015625,
      "activations/layer11_attention_weight_max": 32.371971130371094,
      "activations/layer11_attention_weight_min": -31.50766372680664,
      "activations/layer12_attention_weight_max": 26.5882511138916,
      "activations/layer12_attention_weight_min": -24.79638671875,
      "activations/layer13_attention_weight_max": 43.51355743408203,
      "activations/layer13_attention_weight_min": -33.114173889160156,
      "activations/layer14_attention_weight_max": 45.2393684387207,
      "activations/layer14_attention_weight_min": -30.115419387817383,
      "activations/layer15_attention_weight_max": 36.92723083496094,
      "activations/layer15_attention_weight_min": -29.23090934753418,
      "activations/layer16_attention_weight_max": 33.40454864501953,
      "activations/layer16_attention_weight_min": -26.94287109375,
      "activations/layer17_attention_weight_max": 54.52254867553711,
      "activations/layer17_attention_weight_min": -44.45233917236328,
      "activations/layer18_attention_weight_max": 47.11840057373047,
      "activations/layer18_attention_weight_min": -37.591583251953125,
      "activations/layer19_attention_weight_max": 22.201873779296875,
      "activations/layer19_attention_weight_min": -20.60853385925293,
      "activations/layer1_attention_weight_max": 17.151798248291016,
      "activations/layer1_attention_weight_min": -13.918057441711426,
      "activations/layer20_attention_weight_max": 24.89293098449707,
      "activations/layer20_attention_weight_min": -20.918230056762695,
      "activations/layer21_attention_weight_max": 44.553558349609375,
      "activations/layer21_attention_weight_min": -23.124675750732422,
      "activations/layer22_attention_weight_max": 34.49813461303711,
      "activations/layer22_attention_weight_min": -26.00522232055664,
      "activations/layer23_attention_weight_max": 37.96697998046875,
      "activations/layer23_attention_weight_min": -23.019580841064453,
      "activations/layer2_attention_weight_max": 31.10280990600586,
      "activations/layer2_attention_weight_min": -28.742076873779297,
      "activations/layer3_attention_weight_max": 84.07792663574219,
      "activations/layer3_attention_weight_min": -87.16588592529297,
      "activations/layer4_attention_weight_max": 97.44792938232422,
      "activations/layer4_attention_weight_min": -94.33195495605469,
      "activations/layer5_attention_weight_max": 81.24330139160156,
      "activations/layer5_attention_weight_min": -77.30027770996094,
      "activations/layer6_attention_weight_max": 54.44451141357422,
      "activations/layer6_attention_weight_min": -51.52497100830078,
      "activations/layer7_attention_weight_max": 77.44409942626953,
      "activations/layer7_attention_weight_min": -74.27311706542969,
      "activations/layer8_attention_weight_max": 44.72461700439453,
      "activations/layer8_attention_weight_min": -44.05834197998047,
      "activations/layer9_attention_weight_max": 53.99359893798828,
      "activations/layer9_attention_weight_min": -49.881004333496094,
      "epoch": 7.91,
      "learning_rate": 0.00010002840909090908,
      "loss": 2.8201,
      "step": 136050
    },
    {
      "activations/layer0_attention_weight_max": 14.981682777404785,
      "activations/layer0_attention_weight_min": -13.976128578186035,
      "activations/layer10_attention_weight_max": 38.15237808227539,
      "activations/layer10_attention_weight_min": -34.89595031738281,
      "activations/layer11_attention_weight_max": 30.993507385253906,
      "activations/layer11_attention_weight_min": -32.6922607421875,
      "activations/layer12_attention_weight_max": 20.35902976989746,
      "activations/layer12_attention_weight_min": -27.567005157470703,
      "activations/layer13_attention_weight_max": 47.986167907714844,
      "activations/layer13_attention_weight_min": -32.42350769042969,
      "activations/layer14_attention_weight_max": 48.89026641845703,
      "activations/layer14_attention_weight_min": -32.844505310058594,
      "activations/layer15_attention_weight_max": 37.307464599609375,
      "activations/layer15_attention_weight_min": -30.754671096801758,
      "activations/layer16_attention_weight_max": 32.6438102722168,
      "activations/layer16_attention_weight_min": -29.663982391357422,
      "activations/layer17_attention_weight_max": 57.1993408203125,
      "activations/layer17_attention_weight_min": -47.73992156982422,
      "activations/layer18_attention_weight_max": 51.20090103149414,
      "activations/layer18_attention_weight_min": -37.6201057434082,
      "activations/layer19_attention_weight_max": 23.09484100341797,
      "activations/layer19_attention_weight_min": -21.08926010131836,
      "activations/layer1_attention_weight_max": 17.628650665283203,
      "activations/layer1_attention_weight_min": -14.221521377563477,
      "activations/layer20_attention_weight_max": 21.58966827392578,
      "activations/layer20_attention_weight_min": -20.550418853759766,
      "activations/layer21_attention_weight_max": 39.70227813720703,
      "activations/layer21_attention_weight_min": -23.309499740600586,
      "activations/layer22_attention_weight_max": 33.71554183959961,
      "activations/layer22_attention_weight_min": -24.34087371826172,
      "activations/layer23_attention_weight_max": 41.18782043457031,
      "activations/layer23_attention_weight_min": -21.40652847290039,
      "activations/layer2_attention_weight_max": 30.527969360351562,
      "activations/layer2_attention_weight_min": -27.593626022338867,
      "activations/layer3_attention_weight_max": 84.92402648925781,
      "activations/layer3_attention_weight_min": -86.20094299316406,
      "activations/layer4_attention_weight_max": 99.49848175048828,
      "activations/layer4_attention_weight_min": -93.49129486083984,
      "activations/layer5_attention_weight_max": 84.7562484741211,
      "activations/layer5_attention_weight_min": -77.40182495117188,
      "activations/layer6_attention_weight_max": 53.89230728149414,
      "activations/layer6_attention_weight_min": -52.787254333496094,
      "activations/layer7_attention_weight_max": 76.76483917236328,
      "activations/layer7_attention_weight_min": -72.8573226928711,
      "activations/layer8_attention_weight_max": 43.81440734863281,
      "activations/layer8_attention_weight_min": -44.67237854003906,
      "activations/layer9_attention_weight_max": 49.963523864746094,
      "activations/layer9_attention_weight_min": -49.80950927734375,
      "epoch": 7.91,
      "learning_rate": 0.0001000094696969697,
      "loss": 2.8163,
      "step": 136100
    },
    {
      "activations/layer0_attention_weight_max": 15.528980255126953,
      "activations/layer0_attention_weight_min": -14.294727325439453,
      "activations/layer10_attention_weight_max": 40.46179962158203,
      "activations/layer10_attention_weight_min": -36.2784309387207,
      "activations/layer11_attention_weight_max": 32.052978515625,
      "activations/layer11_attention_weight_min": -33.62055969238281,
      "activations/layer12_attention_weight_max": 21.50901985168457,
      "activations/layer12_attention_weight_min": -23.034040451049805,
      "activations/layer13_attention_weight_max": 45.96596908569336,
      "activations/layer13_attention_weight_min": -34.087650299072266,
      "activations/layer14_attention_weight_max": 46.60251235961914,
      "activations/layer14_attention_weight_min": -30.20041275024414,
      "activations/layer15_attention_weight_max": 36.62784957885742,
      "activations/layer15_attention_weight_min": -30.266427993774414,
      "activations/layer16_attention_weight_max": 31.163541793823242,
      "activations/layer16_attention_weight_min": -27.1834774017334,
      "activations/layer17_attention_weight_max": 54.0477180480957,
      "activations/layer17_attention_weight_min": -41.24460220336914,
      "activations/layer18_attention_weight_max": 48.155921936035156,
      "activations/layer18_attention_weight_min": -35.23194885253906,
      "activations/layer19_attention_weight_max": 20.034439086914062,
      "activations/layer19_attention_weight_min": -20.41668701171875,
      "activations/layer1_attention_weight_max": 17.060453414916992,
      "activations/layer1_attention_weight_min": -15.253110885620117,
      "activations/layer20_attention_weight_max": 22.884469985961914,
      "activations/layer20_attention_weight_min": -21.720436096191406,
      "activations/layer21_attention_weight_max": 38.20817947387695,
      "activations/layer21_attention_weight_min": -21.92533302307129,
      "activations/layer22_attention_weight_max": 30.83555030822754,
      "activations/layer22_attention_weight_min": -25.194604873657227,
      "activations/layer23_attention_weight_max": 38.26936721801758,
      "activations/layer23_attention_weight_min": -22.02620506286621,
      "activations/layer2_attention_weight_max": 31.36201286315918,
      "activations/layer2_attention_weight_min": -29.539405822753906,
      "activations/layer3_attention_weight_max": 89.23986053466797,
      "activations/layer3_attention_weight_min": -87.25144958496094,
      "activations/layer4_attention_weight_max": 99.30384063720703,
      "activations/layer4_attention_weight_min": -95.6603012084961,
      "activations/layer5_attention_weight_max": 85.4372787475586,
      "activations/layer5_attention_weight_min": -81.07930755615234,
      "activations/layer6_attention_weight_max": 55.4880485534668,
      "activations/layer6_attention_weight_min": -52.613075256347656,
      "activations/layer7_attention_weight_max": 77.94811248779297,
      "activations/layer7_attention_weight_min": -75.27648162841797,
      "activations/layer8_attention_weight_max": 46.68764114379883,
      "activations/layer8_attention_weight_min": -44.62678909301758,
      "activations/layer9_attention_weight_max": 53.8974609375,
      "activations/layer9_attention_weight_min": -49.52668380737305,
      "epoch": 7.91,
      "learning_rate": 9.999053030303029e-05,
      "loss": 2.8267,
      "step": 136150
    },
    {
      "activations/layer0_attention_weight_max": 14.713388442993164,
      "activations/layer0_attention_weight_min": -13.99492359161377,
      "activations/layer10_attention_weight_max": 37.84390640258789,
      "activations/layer10_attention_weight_min": -36.567474365234375,
      "activations/layer11_attention_weight_max": 30.503700256347656,
      "activations/layer11_attention_weight_min": -33.14088439941406,
      "activations/layer12_attention_weight_max": 18.26130485534668,
      "activations/layer12_attention_weight_min": -23.509246826171875,
      "activations/layer13_attention_weight_max": 42.85272979736328,
      "activations/layer13_attention_weight_min": -31.38074493408203,
      "activations/layer14_attention_weight_max": 34.93408203125,
      "activations/layer14_attention_weight_min": -31.84339141845703,
      "activations/layer15_attention_weight_max": 36.91176223754883,
      "activations/layer15_attention_weight_min": -29.535842895507812,
      "activations/layer16_attention_weight_max": 37.54034423828125,
      "activations/layer16_attention_weight_min": -27.114761352539062,
      "activations/layer17_attention_weight_max": 53.02713394165039,
      "activations/layer17_attention_weight_min": -42.977806091308594,
      "activations/layer18_attention_weight_max": 48.88189697265625,
      "activations/layer18_attention_weight_min": -35.45613098144531,
      "activations/layer19_attention_weight_max": 25.303430557250977,
      "activations/layer19_attention_weight_min": -21.674650192260742,
      "activations/layer1_attention_weight_max": 17.042619705200195,
      "activations/layer1_attention_weight_min": -14.085805892944336,
      "activations/layer20_attention_weight_max": 20.952014923095703,
      "activations/layer20_attention_weight_min": -19.856021881103516,
      "activations/layer21_attention_weight_max": 35.21636962890625,
      "activations/layer21_attention_weight_min": -22.498992919921875,
      "activations/layer22_attention_weight_max": 34.4141960144043,
      "activations/layer22_attention_weight_min": -24.244121551513672,
      "activations/layer23_attention_weight_max": 31.556005477905273,
      "activations/layer23_attention_weight_min": -21.315189361572266,
      "activations/layer2_attention_weight_max": 30.510860443115234,
      "activations/layer2_attention_weight_min": -28.319128036499023,
      "activations/layer3_attention_weight_max": 87.37974548339844,
      "activations/layer3_attention_weight_min": -89.35040283203125,
      "activations/layer4_attention_weight_max": 103.4920654296875,
      "activations/layer4_attention_weight_min": -101.65132141113281,
      "activations/layer5_attention_weight_max": 86.57677459716797,
      "activations/layer5_attention_weight_min": -82.87899780273438,
      "activations/layer6_attention_weight_max": 57.63482666015625,
      "activations/layer6_attention_weight_min": -52.559078216552734,
      "activations/layer7_attention_weight_max": 81.22222900390625,
      "activations/layer7_attention_weight_min": -75.21229553222656,
      "activations/layer8_attention_weight_max": 44.469146728515625,
      "activations/layer8_attention_weight_min": -45.94963455200195,
      "activations/layer9_attention_weight_max": 50.10793685913086,
      "activations/layer9_attention_weight_min": -48.65541458129883,
      "epoch": 7.91,
      "learning_rate": 9.99715909090909e-05,
      "loss": 2.8057,
      "step": 136200
    },
    {
      "activations/layer0_attention_weight_max": 14.231667518615723,
      "activations/layer0_attention_weight_min": -14.33548641204834,
      "activations/layer10_attention_weight_max": 38.18495559692383,
      "activations/layer10_attention_weight_min": -34.96680450439453,
      "activations/layer11_attention_weight_max": 32.30859375,
      "activations/layer11_attention_weight_min": -32.02021026611328,
      "activations/layer12_attention_weight_max": 24.348007202148438,
      "activations/layer12_attention_weight_min": -24.44640350341797,
      "activations/layer13_attention_weight_max": 45.89424133300781,
      "activations/layer13_attention_weight_min": -33.49483871459961,
      "activations/layer14_attention_weight_max": 39.740848541259766,
      "activations/layer14_attention_weight_min": -32.09666442871094,
      "activations/layer15_attention_weight_max": 41.007598876953125,
      "activations/layer15_attention_weight_min": -29.58696937561035,
      "activations/layer16_attention_weight_max": 35.419559478759766,
      "activations/layer16_attention_weight_min": -28.489015579223633,
      "activations/layer17_attention_weight_max": 57.26183319091797,
      "activations/layer17_attention_weight_min": -43.618473052978516,
      "activations/layer18_attention_weight_max": 51.007728576660156,
      "activations/layer18_attention_weight_min": -37.90651321411133,
      "activations/layer19_attention_weight_max": 29.774198532104492,
      "activations/layer19_attention_weight_min": -22.051328659057617,
      "activations/layer1_attention_weight_max": 16.889278411865234,
      "activations/layer1_attention_weight_min": -15.487590789794922,
      "activations/layer20_attention_weight_max": 24.258405685424805,
      "activations/layer20_attention_weight_min": -20.57445526123047,
      "activations/layer21_attention_weight_max": 46.91945266723633,
      "activations/layer21_attention_weight_min": -24.65073585510254,
      "activations/layer22_attention_weight_max": 33.6147346496582,
      "activations/layer22_attention_weight_min": -26.36338233947754,
      "activations/layer23_attention_weight_max": 38.769371032714844,
      "activations/layer23_attention_weight_min": -21.48196792602539,
      "activations/layer2_attention_weight_max": 33.227516174316406,
      "activations/layer2_attention_weight_min": -30.904674530029297,
      "activations/layer3_attention_weight_max": 89.92115783691406,
      "activations/layer3_attention_weight_min": -92.51338958740234,
      "activations/layer4_attention_weight_max": 103.83074951171875,
      "activations/layer4_attention_weight_min": -103.09857177734375,
      "activations/layer5_attention_weight_max": 90.4171142578125,
      "activations/layer5_attention_weight_min": -84.15493774414062,
      "activations/layer6_attention_weight_max": 58.797706604003906,
      "activations/layer6_attention_weight_min": -56.59559631347656,
      "activations/layer7_attention_weight_max": 77.65238952636719,
      "activations/layer7_attention_weight_min": -78.45861053466797,
      "activations/layer8_attention_weight_max": 48.243953704833984,
      "activations/layer8_attention_weight_min": -47.74492263793945,
      "activations/layer9_attention_weight_max": 50.85850143432617,
      "activations/layer9_attention_weight_min": -51.761775970458984,
      "epoch": 7.92,
      "learning_rate": 9.99526515151515e-05,
      "loss": 2.8197,
      "step": 136250
    },
    {
      "activations/layer0_attention_weight_max": 16.25006866455078,
      "activations/layer0_attention_weight_min": -13.794681549072266,
      "activations/layer10_attention_weight_max": 40.24383544921875,
      "activations/layer10_attention_weight_min": -37.426063537597656,
      "activations/layer11_attention_weight_max": 31.29103660583496,
      "activations/layer11_attention_weight_min": -33.48704147338867,
      "activations/layer12_attention_weight_max": 22.705604553222656,
      "activations/layer12_attention_weight_min": -27.807968139648438,
      "activations/layer13_attention_weight_max": 50.16330337524414,
      "activations/layer13_attention_weight_min": -35.607643127441406,
      "activations/layer14_attention_weight_max": 39.37997817993164,
      "activations/layer14_attention_weight_min": -31.3988094329834,
      "activations/layer15_attention_weight_max": 35.22356414794922,
      "activations/layer15_attention_weight_min": -31.26191520690918,
      "activations/layer16_attention_weight_max": 33.36288070678711,
      "activations/layer16_attention_weight_min": -28.828115463256836,
      "activations/layer17_attention_weight_max": 58.57438278198242,
      "activations/layer17_attention_weight_min": -45.7877311706543,
      "activations/layer18_attention_weight_max": 50.43021011352539,
      "activations/layer18_attention_weight_min": -39.75961685180664,
      "activations/layer19_attention_weight_max": 23.555377960205078,
      "activations/layer19_attention_weight_min": -21.166751861572266,
      "activations/layer1_attention_weight_max": 17.102554321289062,
      "activations/layer1_attention_weight_min": -16.6458683013916,
      "activations/layer20_attention_weight_max": 24.51302719116211,
      "activations/layer20_attention_weight_min": -18.709856033325195,
      "activations/layer21_attention_weight_max": 36.18434524536133,
      "activations/layer21_attention_weight_min": -22.867395401000977,
      "activations/layer22_attention_weight_max": 34.87320327758789,
      "activations/layer22_attention_weight_min": -24.3156795501709,
      "activations/layer23_attention_weight_max": 39.42125701904297,
      "activations/layer23_attention_weight_min": -21.344999313354492,
      "activations/layer2_attention_weight_max": 31.481657028198242,
      "activations/layer2_attention_weight_min": -31.242416381835938,
      "activations/layer3_attention_weight_max": 86.87084197998047,
      "activations/layer3_attention_weight_min": -88.39046478271484,
      "activations/layer4_attention_weight_max": 99.95610809326172,
      "activations/layer4_attention_weight_min": -96.43924713134766,
      "activations/layer5_attention_weight_max": 87.01344299316406,
      "activations/layer5_attention_weight_min": -84.39957427978516,
      "activations/layer6_attention_weight_max": 57.69300079345703,
      "activations/layer6_attention_weight_min": -56.586875915527344,
      "activations/layer7_attention_weight_max": 79.74867248535156,
      "activations/layer7_attention_weight_min": -74.14527130126953,
      "activations/layer8_attention_weight_max": 46.163299560546875,
      "activations/layer8_attention_weight_min": -49.145225524902344,
      "activations/layer9_attention_weight_max": 53.53129959106445,
      "activations/layer9_attention_weight_min": -51.01694869995117,
      "epoch": 7.92,
      "learning_rate": 9.993371212121212e-05,
      "loss": 2.8115,
      "step": 136300
    },
    {
      "activations/layer0_attention_weight_max": 14.379496574401855,
      "activations/layer0_attention_weight_min": -14.539712905883789,
      "activations/layer10_attention_weight_max": 37.23100280761719,
      "activations/layer10_attention_weight_min": -35.294036865234375,
      "activations/layer11_attention_weight_max": 31.926433563232422,
      "activations/layer11_attention_weight_min": -29.825992584228516,
      "activations/layer12_attention_weight_max": 24.231857299804688,
      "activations/layer12_attention_weight_min": -24.256393432617188,
      "activations/layer13_attention_weight_max": 35.18811798095703,
      "activations/layer13_attention_weight_min": -28.712318420410156,
      "activations/layer14_attention_weight_max": 38.18990707397461,
      "activations/layer14_attention_weight_min": -30.780717849731445,
      "activations/layer15_attention_weight_max": 34.93841552734375,
      "activations/layer15_attention_weight_min": -28.112850189208984,
      "activations/layer16_attention_weight_max": 32.32291030883789,
      "activations/layer16_attention_weight_min": -26.411632537841797,
      "activations/layer17_attention_weight_max": 54.63644027709961,
      "activations/layer17_attention_weight_min": -42.246822357177734,
      "activations/layer18_attention_weight_max": 41.659912109375,
      "activations/layer18_attention_weight_min": -33.57854461669922,
      "activations/layer19_attention_weight_max": 24.033750534057617,
      "activations/layer19_attention_weight_min": -20.73274803161621,
      "activations/layer1_attention_weight_max": 16.933216094970703,
      "activations/layer1_attention_weight_min": -16.840618133544922,
      "activations/layer20_attention_weight_max": 22.314579010009766,
      "activations/layer20_attention_weight_min": -22.01154136657715,
      "activations/layer21_attention_weight_max": 33.618629455566406,
      "activations/layer21_attention_weight_min": -22.249691009521484,
      "activations/layer22_attention_weight_max": 32.33949279785156,
      "activations/layer22_attention_weight_min": -22.666725158691406,
      "activations/layer23_attention_weight_max": 32.601863861083984,
      "activations/layer23_attention_weight_min": -21.20020866394043,
      "activations/layer2_attention_weight_max": 31.320663452148438,
      "activations/layer2_attention_weight_min": -28.603038787841797,
      "activations/layer3_attention_weight_max": 86.20039367675781,
      "activations/layer3_attention_weight_min": -85.84514617919922,
      "activations/layer4_attention_weight_max": 101.68465423583984,
      "activations/layer4_attention_weight_min": -92.6467056274414,
      "activations/layer5_attention_weight_max": 82.6568603515625,
      "activations/layer5_attention_weight_min": -77.56017303466797,
      "activations/layer6_attention_weight_max": 53.49552536010742,
      "activations/layer6_attention_weight_min": -51.29438400268555,
      "activations/layer7_attention_weight_max": 73.03433227539062,
      "activations/layer7_attention_weight_min": -70.46178436279297,
      "activations/layer8_attention_weight_max": 42.921573638916016,
      "activations/layer8_attention_weight_min": -43.106815338134766,
      "activations/layer9_attention_weight_max": 49.26709747314453,
      "activations/layer9_attention_weight_min": -46.73684310913086,
      "epoch": 7.92,
      "learning_rate": 9.991477272727272e-05,
      "loss": 2.8227,
      "step": 136350
    },
    {
      "activations/layer0_attention_weight_max": 14.6404390335083,
      "activations/layer0_attention_weight_min": -14.422626495361328,
      "activations/layer10_attention_weight_max": 39.9080810546875,
      "activations/layer10_attention_weight_min": -36.274566650390625,
      "activations/layer11_attention_weight_max": 32.04490280151367,
      "activations/layer11_attention_weight_min": -30.79523468017578,
      "activations/layer12_attention_weight_max": 20.869028091430664,
      "activations/layer12_attention_weight_min": -25.499101638793945,
      "activations/layer13_attention_weight_max": 52.10795593261719,
      "activations/layer13_attention_weight_min": -33.275291442871094,
      "activations/layer14_attention_weight_max": 50.361873626708984,
      "activations/layer14_attention_weight_min": -31.161762237548828,
      "activations/layer15_attention_weight_max": 39.96424865722656,
      "activations/layer15_attention_weight_min": -30.03005027770996,
      "activations/layer16_attention_weight_max": 36.5966682434082,
      "activations/layer16_attention_weight_min": -28.57120132446289,
      "activations/layer17_attention_weight_max": 57.58245849609375,
      "activations/layer17_attention_weight_min": -47.23882293701172,
      "activations/layer18_attention_weight_max": 52.07099151611328,
      "activations/layer18_attention_weight_min": -36.920379638671875,
      "activations/layer19_attention_weight_max": 26.173797607421875,
      "activations/layer19_attention_weight_min": -21.100299835205078,
      "activations/layer1_attention_weight_max": 17.172138214111328,
      "activations/layer1_attention_weight_min": -15.542159080505371,
      "activations/layer20_attention_weight_max": 22.245319366455078,
      "activations/layer20_attention_weight_min": -22.578990936279297,
      "activations/layer21_attention_weight_max": 50.84897232055664,
      "activations/layer21_attention_weight_min": -26.11456298828125,
      "activations/layer22_attention_weight_max": 35.8587532043457,
      "activations/layer22_attention_weight_min": -26.46207618713379,
      "activations/layer23_attention_weight_max": 39.4275016784668,
      "activations/layer23_attention_weight_min": -20.44298553466797,
      "activations/layer2_attention_weight_max": 32.43408966064453,
      "activations/layer2_attention_weight_min": -29.514455795288086,
      "activations/layer3_attention_weight_max": 87.86491394042969,
      "activations/layer3_attention_weight_min": -85.22425079345703,
      "activations/layer4_attention_weight_max": 101.16061401367188,
      "activations/layer4_attention_weight_min": -93.81168365478516,
      "activations/layer5_attention_weight_max": 88.14939880371094,
      "activations/layer5_attention_weight_min": -79.53620910644531,
      "activations/layer6_attention_weight_max": 56.93062210083008,
      "activations/layer6_attention_weight_min": -54.22682189941406,
      "activations/layer7_attention_weight_max": 77.84896087646484,
      "activations/layer7_attention_weight_min": -73.64456939697266,
      "activations/layer8_attention_weight_max": 49.11506652832031,
      "activations/layer8_attention_weight_min": -46.57316589355469,
      "activations/layer9_attention_weight_max": 55.831947326660156,
      "activations/layer9_attention_weight_min": -49.83672332763672,
      "epoch": 7.93,
      "learning_rate": 9.989583333333332e-05,
      "loss": 2.8001,
      "step": 136400
    },
    {
      "activations/layer0_attention_weight_max": 14.503971099853516,
      "activations/layer0_attention_weight_min": -14.383039474487305,
      "activations/layer10_attention_weight_max": 38.24061584472656,
      "activations/layer10_attention_weight_min": -37.20424270629883,
      "activations/layer11_attention_weight_max": 32.710906982421875,
      "activations/layer11_attention_weight_min": -30.20973777770996,
      "activations/layer12_attention_weight_max": 20.96023178100586,
      "activations/layer12_attention_weight_min": -24.59975814819336,
      "activations/layer13_attention_weight_max": 54.03175354003906,
      "activations/layer13_attention_weight_min": -37.81475830078125,
      "activations/layer14_attention_weight_max": 57.60690689086914,
      "activations/layer14_attention_weight_min": -31.16718292236328,
      "activations/layer15_attention_weight_max": 43.75947952270508,
      "activations/layer15_attention_weight_min": -29.680627822875977,
      "activations/layer16_attention_weight_max": 34.723655700683594,
      "activations/layer16_attention_weight_min": -27.7935848236084,
      "activations/layer17_attention_weight_max": 58.86126708984375,
      "activations/layer17_attention_weight_min": -47.098716735839844,
      "activations/layer18_attention_weight_max": 54.97402572631836,
      "activations/layer18_attention_weight_min": -38.582576751708984,
      "activations/layer19_attention_weight_max": 24.04356575012207,
      "activations/layer19_attention_weight_min": -22.05470085144043,
      "activations/layer1_attention_weight_max": 17.840957641601562,
      "activations/layer1_attention_weight_min": -17.54559898376465,
      "activations/layer20_attention_weight_max": 23.56438446044922,
      "activations/layer20_attention_weight_min": -20.912826538085938,
      "activations/layer21_attention_weight_max": 52.50459671020508,
      "activations/layer21_attention_weight_min": -23.36717414855957,
      "activations/layer22_attention_weight_max": 39.4647216796875,
      "activations/layer22_attention_weight_min": -25.523530960083008,
      "activations/layer23_attention_weight_max": 37.26445770263672,
      "activations/layer23_attention_weight_min": -21.939939498901367,
      "activations/layer2_attention_weight_max": 31.16266632080078,
      "activations/layer2_attention_weight_min": -28.983585357666016,
      "activations/layer3_attention_weight_max": 86.90404510498047,
      "activations/layer3_attention_weight_min": -84.95331573486328,
      "activations/layer4_attention_weight_max": 107.208984375,
      "activations/layer4_attention_weight_min": -96.2625503540039,
      "activations/layer5_attention_weight_max": 90.47499084472656,
      "activations/layer5_attention_weight_min": -80.38570404052734,
      "activations/layer6_attention_weight_max": 57.4472770690918,
      "activations/layer6_attention_weight_min": -54.641353607177734,
      "activations/layer7_attention_weight_max": 77.86680603027344,
      "activations/layer7_attention_weight_min": -77.8347396850586,
      "activations/layer8_attention_weight_max": 50.417789459228516,
      "activations/layer8_attention_weight_min": -47.63114929199219,
      "activations/layer9_attention_weight_max": 59.07577133178711,
      "activations/layer9_attention_weight_min": -53.219940185546875,
      "epoch": 7.93,
      "learning_rate": 9.987689393939393e-05,
      "loss": 2.816,
      "step": 136450
    },
    {
      "activations/layer0_attention_weight_max": 15.19271469116211,
      "activations/layer0_attention_weight_min": -14.313520431518555,
      "activations/layer10_attention_weight_max": 36.40904235839844,
      "activations/layer10_attention_weight_min": -34.203372955322266,
      "activations/layer11_attention_weight_max": 28.737773895263672,
      "activations/layer11_attention_weight_min": -30.63759994506836,
      "activations/layer12_attention_weight_max": 24.704694747924805,
      "activations/layer12_attention_weight_min": -23.546110153198242,
      "activations/layer13_attention_weight_max": 41.54854202270508,
      "activations/layer13_attention_weight_min": -31.651477813720703,
      "activations/layer14_attention_weight_max": 42.55501937866211,
      "activations/layer14_attention_weight_min": -30.966228485107422,
      "activations/layer15_attention_weight_max": 33.61463928222656,
      "activations/layer15_attention_weight_min": -27.409730911254883,
      "activations/layer16_attention_weight_max": 30.748661041259766,
      "activations/layer16_attention_weight_min": -26.86261749267578,
      "activations/layer17_attention_weight_max": 53.859535217285156,
      "activations/layer17_attention_weight_min": -41.58570861816406,
      "activations/layer18_attention_weight_max": 47.508636474609375,
      "activations/layer18_attention_weight_min": -35.11796951293945,
      "activations/layer19_attention_weight_max": 21.060150146484375,
      "activations/layer19_attention_weight_min": -21.18828773498535,
      "activations/layer1_attention_weight_max": 16.26585578918457,
      "activations/layer1_attention_weight_min": -15.737656593322754,
      "activations/layer20_attention_weight_max": 22.516420364379883,
      "activations/layer20_attention_weight_min": -20.94487762451172,
      "activations/layer21_attention_weight_max": 33.56443786621094,
      "activations/layer21_attention_weight_min": -23.99491310119629,
      "activations/layer22_attention_weight_max": 31.4044246673584,
      "activations/layer22_attention_weight_min": -25.376890182495117,
      "activations/layer23_attention_weight_max": 36.118186950683594,
      "activations/layer23_attention_weight_min": -21.36404800415039,
      "activations/layer2_attention_weight_max": 30.645164489746094,
      "activations/layer2_attention_weight_min": -28.217191696166992,
      "activations/layer3_attention_weight_max": 85.39591217041016,
      "activations/layer3_attention_weight_min": -83.61508178710938,
      "activations/layer4_attention_weight_max": 97.79254913330078,
      "activations/layer4_attention_weight_min": -92.71102142333984,
      "activations/layer5_attention_weight_max": 79.36761474609375,
      "activations/layer5_attention_weight_min": -75.62860107421875,
      "activations/layer6_attention_weight_max": 55.045188903808594,
      "activations/layer6_attention_weight_min": -51.82802963256836,
      "activations/layer7_attention_weight_max": 74.13115692138672,
      "activations/layer7_attention_weight_min": -73.17555236816406,
      "activations/layer8_attention_weight_max": 42.98273468017578,
      "activations/layer8_attention_weight_min": -44.617645263671875,
      "activations/layer9_attention_weight_max": 51.0307731628418,
      "activations/layer9_attention_weight_min": -48.95270538330078,
      "epoch": 7.93,
      "learning_rate": 9.985795454545455e-05,
      "loss": 2.7911,
      "step": 136500
    },
    {
      "activations/layer0_attention_weight_max": 14.599762916564941,
      "activations/layer0_attention_weight_min": -15.063368797302246,
      "activations/layer10_attention_weight_max": 38.99285888671875,
      "activations/layer10_attention_weight_min": -35.711910247802734,
      "activations/layer11_attention_weight_max": 34.5867919921875,
      "activations/layer11_attention_weight_min": -31.663516998291016,
      "activations/layer12_attention_weight_max": 22.537948608398438,
      "activations/layer12_attention_weight_min": -24.340065002441406,
      "activations/layer13_attention_weight_max": 58.606895446777344,
      "activations/layer13_attention_weight_min": -35.67305374145508,
      "activations/layer14_attention_weight_max": 68.39457702636719,
      "activations/layer14_attention_weight_min": -34.860435485839844,
      "activations/layer15_attention_weight_max": 48.296058654785156,
      "activations/layer15_attention_weight_min": -30.52436637878418,
      "activations/layer16_attention_weight_max": 41.034488677978516,
      "activations/layer16_attention_weight_min": -29.035871505737305,
      "activations/layer17_attention_weight_max": 68.0641860961914,
      "activations/layer17_attention_weight_min": -47.22574996948242,
      "activations/layer18_attention_weight_max": 56.98928451538086,
      "activations/layer18_attention_weight_min": -40.71036148071289,
      "activations/layer19_attention_weight_max": 25.613779067993164,
      "activations/layer19_attention_weight_min": -23.020448684692383,
      "activations/layer1_attention_weight_max": 18.21851921081543,
      "activations/layer1_attention_weight_min": -15.100874900817871,
      "activations/layer20_attention_weight_max": 29.784019470214844,
      "activations/layer20_attention_weight_min": -21.26697540283203,
      "activations/layer21_attention_weight_max": 53.5393180847168,
      "activations/layer21_attention_weight_min": -26.136417388916016,
      "activations/layer22_attention_weight_max": 44.735687255859375,
      "activations/layer22_attention_weight_min": -25.43265151977539,
      "activations/layer23_attention_weight_max": 50.592735290527344,
      "activations/layer23_attention_weight_min": -21.204797744750977,
      "activations/layer2_attention_weight_max": 33.10055923461914,
      "activations/layer2_attention_weight_min": -30.003433227539062,
      "activations/layer3_attention_weight_max": 87.06737518310547,
      "activations/layer3_attention_weight_min": -87.1179428100586,
      "activations/layer4_attention_weight_max": 101.01739501953125,
      "activations/layer4_attention_weight_min": -95.66423034667969,
      "activations/layer5_attention_weight_max": 84.76994323730469,
      "activations/layer5_attention_weight_min": -80.35926818847656,
      "activations/layer6_attention_weight_max": 55.89121627807617,
      "activations/layer6_attention_weight_min": -53.21821212768555,
      "activations/layer7_attention_weight_max": 74.27656555175781,
      "activations/layer7_attention_weight_min": -76.85655975341797,
      "activations/layer8_attention_weight_max": 46.68412780761719,
      "activations/layer8_attention_weight_min": -46.47210693359375,
      "activations/layer9_attention_weight_max": 58.26089096069336,
      "activations/layer9_attention_weight_min": -48.559356689453125,
      "epoch": 7.93,
      "learning_rate": 9.983901515151514e-05,
      "loss": 2.8067,
      "step": 136550
    },
    {
      "activations/layer0_attention_weight_max": 14.756951332092285,
      "activations/layer0_attention_weight_min": -14.718159675598145,
      "activations/layer10_attention_weight_max": 36.32023239135742,
      "activations/layer10_attention_weight_min": -34.89352798461914,
      "activations/layer11_attention_weight_max": 30.932132720947266,
      "activations/layer11_attention_weight_min": -31.142620086669922,
      "activations/layer12_attention_weight_max": 21.701740264892578,
      "activations/layer12_attention_weight_min": -26.167844772338867,
      "activations/layer13_attention_weight_max": 39.693153381347656,
      "activations/layer13_attention_weight_min": -31.19742774963379,
      "activations/layer14_attention_weight_max": 37.73400115966797,
      "activations/layer14_attention_weight_min": -29.11709213256836,
      "activations/layer15_attention_weight_max": 35.669952392578125,
      "activations/layer15_attention_weight_min": -28.266305923461914,
      "activations/layer16_attention_weight_max": 33.8388786315918,
      "activations/layer16_attention_weight_min": -27.6826229095459,
      "activations/layer17_attention_weight_max": 55.87632751464844,
      "activations/layer17_attention_weight_min": -41.1634521484375,
      "activations/layer18_attention_weight_max": 53.78081512451172,
      "activations/layer18_attention_weight_min": -36.953102111816406,
      "activations/layer19_attention_weight_max": 22.46535301208496,
      "activations/layer19_attention_weight_min": -21.158369064331055,
      "activations/layer1_attention_weight_max": 16.893512725830078,
      "activations/layer1_attention_weight_min": -15.609784126281738,
      "activations/layer20_attention_weight_max": 21.19846534729004,
      "activations/layer20_attention_weight_min": -22.136035919189453,
      "activations/layer21_attention_weight_max": 35.866249084472656,
      "activations/layer21_attention_weight_min": -22.980817794799805,
      "activations/layer22_attention_weight_max": 29.383943557739258,
      "activations/layer22_attention_weight_min": -26.442243576049805,
      "activations/layer23_attention_weight_max": 36.9842643737793,
      "activations/layer23_attention_weight_min": -22.182571411132812,
      "activations/layer2_attention_weight_max": 32.6440315246582,
      "activations/layer2_attention_weight_min": -30.246511459350586,
      "activations/layer3_attention_weight_max": 92.02177429199219,
      "activations/layer3_attention_weight_min": -97.45198059082031,
      "activations/layer4_attention_weight_max": 103.59979248046875,
      "activations/layer4_attention_weight_min": -100.31692504882812,
      "activations/layer5_attention_weight_max": 86.1435546875,
      "activations/layer5_attention_weight_min": -80.24618530273438,
      "activations/layer6_attention_weight_max": 55.72787857055664,
      "activations/layer6_attention_weight_min": -52.49649429321289,
      "activations/layer7_attention_weight_max": 75.9525375366211,
      "activations/layer7_attention_weight_min": -70.2641372680664,
      "activations/layer8_attention_weight_max": 44.78456497192383,
      "activations/layer8_attention_weight_min": -45.58796310424805,
      "activations/layer9_attention_weight_max": 47.4962043762207,
      "activations/layer9_attention_weight_min": -50.966548919677734,
      "epoch": 7.94,
      "learning_rate": 9.982007575757575e-05,
      "loss": 2.816,
      "step": 136600
    },
    {
      "activations/layer0_attention_weight_max": 15.86885929107666,
      "activations/layer0_attention_weight_min": -14.770971298217773,
      "activations/layer10_attention_weight_max": 35.610107421875,
      "activations/layer10_attention_weight_min": -35.42192840576172,
      "activations/layer11_attention_weight_max": 30.08490753173828,
      "activations/layer11_attention_weight_min": -31.174663543701172,
      "activations/layer12_attention_weight_max": 16.15906524658203,
      "activations/layer12_attention_weight_min": -26.176591873168945,
      "activations/layer13_attention_weight_max": 37.93183135986328,
      "activations/layer13_attention_weight_min": -31.987234115600586,
      "activations/layer14_attention_weight_max": 37.22278594970703,
      "activations/layer14_attention_weight_min": -30.065927505493164,
      "activations/layer15_attention_weight_max": 32.70295715332031,
      "activations/layer15_attention_weight_min": -29.248641967773438,
      "activations/layer16_attention_weight_max": 31.33518409729004,
      "activations/layer16_attention_weight_min": -28.06519889831543,
      "activations/layer17_attention_weight_max": 51.85831832885742,
      "activations/layer17_attention_weight_min": -44.5678825378418,
      "activations/layer18_attention_weight_max": 44.570255279541016,
      "activations/layer18_attention_weight_min": -35.663421630859375,
      "activations/layer19_attention_weight_max": 22.487682342529297,
      "activations/layer19_attention_weight_min": -21.38539695739746,
      "activations/layer1_attention_weight_max": 17.195396423339844,
      "activations/layer1_attention_weight_min": -15.158543586730957,
      "activations/layer20_attention_weight_max": 20.649051666259766,
      "activations/layer20_attention_weight_min": -26.57561492919922,
      "activations/layer21_attention_weight_max": 40.14154815673828,
      "activations/layer21_attention_weight_min": -23.49523162841797,
      "activations/layer22_attention_weight_max": 33.45503234863281,
      "activations/layer22_attention_weight_min": -26.416664123535156,
      "activations/layer23_attention_weight_max": 33.29247283935547,
      "activations/layer23_attention_weight_min": -23.13835906982422,
      "activations/layer2_attention_weight_max": 31.725543975830078,
      "activations/layer2_attention_weight_min": -29.38997459411621,
      "activations/layer3_attention_weight_max": 85.45045471191406,
      "activations/layer3_attention_weight_min": -85.8102035522461,
      "activations/layer4_attention_weight_max": 97.3453598022461,
      "activations/layer4_attention_weight_min": -94.80293273925781,
      "activations/layer5_attention_weight_max": 81.22322082519531,
      "activations/layer5_attention_weight_min": -80.1557846069336,
      "activations/layer6_attention_weight_max": 52.760536193847656,
      "activations/layer6_attention_weight_min": -51.350337982177734,
      "activations/layer7_attention_weight_max": 75.59051513671875,
      "activations/layer7_attention_weight_min": -72.26465606689453,
      "activations/layer8_attention_weight_max": 44.00189971923828,
      "activations/layer8_attention_weight_min": -47.57841873168945,
      "activations/layer9_attention_weight_max": 48.7120246887207,
      "activations/layer9_attention_weight_min": -47.15081024169922,
      "epoch": 7.94,
      "learning_rate": 9.980113636363634e-05,
      "loss": 2.8213,
      "step": 136650
    },
    {
      "activations/layer0_attention_weight_max": 14.798750877380371,
      "activations/layer0_attention_weight_min": -14.736015319824219,
      "activations/layer10_attention_weight_max": 41.650726318359375,
      "activations/layer10_attention_weight_min": -37.64797592163086,
      "activations/layer11_attention_weight_max": 34.261878967285156,
      "activations/layer11_attention_weight_min": -32.83551788330078,
      "activations/layer12_attention_weight_max": 22.55675506591797,
      "activations/layer12_attention_weight_min": -26.887351989746094,
      "activations/layer13_attention_weight_max": 43.81806182861328,
      "activations/layer13_attention_weight_min": -34.971580505371094,
      "activations/layer14_attention_weight_max": 41.18681716918945,
      "activations/layer14_attention_weight_min": -31.605470657348633,
      "activations/layer15_attention_weight_max": 37.66736602783203,
      "activations/layer15_attention_weight_min": -30.17349624633789,
      "activations/layer16_attention_weight_max": 36.43132400512695,
      "activations/layer16_attention_weight_min": -28.08026123046875,
      "activations/layer17_attention_weight_max": 61.39891815185547,
      "activations/layer17_attention_weight_min": -44.39149856567383,
      "activations/layer18_attention_weight_max": 53.90442657470703,
      "activations/layer18_attention_weight_min": -36.5485725402832,
      "activations/layer19_attention_weight_max": 24.898252487182617,
      "activations/layer19_attention_weight_min": -20.75689125061035,
      "activations/layer1_attention_weight_max": 17.928722381591797,
      "activations/layer1_attention_weight_min": -18.76357650756836,
      "activations/layer20_attention_weight_max": 26.1011905670166,
      "activations/layer20_attention_weight_min": -23.202796936035156,
      "activations/layer21_attention_weight_max": 47.09821701049805,
      "activations/layer21_attention_weight_min": -24.507152557373047,
      "activations/layer22_attention_weight_max": 35.4246711730957,
      "activations/layer22_attention_weight_min": -23.413829803466797,
      "activations/layer23_attention_weight_max": 39.55403137207031,
      "activations/layer23_attention_weight_min": -21.01917266845703,
      "activations/layer2_attention_weight_max": 32.387794494628906,
      "activations/layer2_attention_weight_min": -30.1805477142334,
      "activations/layer3_attention_weight_max": 89.97040557861328,
      "activations/layer3_attention_weight_min": -90.62747192382812,
      "activations/layer4_attention_weight_max": 108.78035736083984,
      "activations/layer4_attention_weight_min": -102.53826904296875,
      "activations/layer5_attention_weight_max": 90.51113891601562,
      "activations/layer5_attention_weight_min": -85.984619140625,
      "activations/layer6_attention_weight_max": 59.372154235839844,
      "activations/layer6_attention_weight_min": -57.45812225341797,
      "activations/layer7_attention_weight_max": 79.07704162597656,
      "activations/layer7_attention_weight_min": -79.98280334472656,
      "activations/layer8_attention_weight_max": 47.31587600708008,
      "activations/layer8_attention_weight_min": -48.99216842651367,
      "activations/layer9_attention_weight_max": 57.54929733276367,
      "activations/layer9_attention_weight_min": -52.450164794921875,
      "epoch": 7.94,
      "learning_rate": 9.978219696969696e-05,
      "loss": 2.8134,
      "step": 136700
    },
    {
      "activations/layer0_attention_weight_max": 15.369795799255371,
      "activations/layer0_attention_weight_min": -14.377226829528809,
      "activations/layer10_attention_weight_max": 39.4449462890625,
      "activations/layer10_attention_weight_min": -39.55731964111328,
      "activations/layer11_attention_weight_max": 32.858394622802734,
      "activations/layer11_attention_weight_min": -31.02389907836914,
      "activations/layer12_attention_weight_max": 24.649017333984375,
      "activations/layer12_attention_weight_min": -27.41485023498535,
      "activations/layer13_attention_weight_max": 54.353946685791016,
      "activations/layer13_attention_weight_min": -38.36555099487305,
      "activations/layer14_attention_weight_max": 42.30143737792969,
      "activations/layer14_attention_weight_min": -31.629837036132812,
      "activations/layer15_attention_weight_max": 42.66465377807617,
      "activations/layer15_attention_weight_min": -29.363773345947266,
      "activations/layer16_attention_weight_max": 32.10727310180664,
      "activations/layer16_attention_weight_min": -27.854019165039062,
      "activations/layer17_attention_weight_max": 55.72439193725586,
      "activations/layer17_attention_weight_min": -48.70418167114258,
      "activations/layer18_attention_weight_max": 53.21543884277344,
      "activations/layer18_attention_weight_min": -37.523216247558594,
      "activations/layer19_attention_weight_max": 21.793970108032227,
      "activations/layer19_attention_weight_min": -19.178953170776367,
      "activations/layer1_attention_weight_max": 18.34023094177246,
      "activations/layer1_attention_weight_min": -15.17750072479248,
      "activations/layer20_attention_weight_max": 22.554773330688477,
      "activations/layer20_attention_weight_min": -21.730514526367188,
      "activations/layer21_attention_weight_max": 37.896141052246094,
      "activations/layer21_attention_weight_min": -22.98302459716797,
      "activations/layer22_attention_weight_max": 30.61798667907715,
      "activations/layer22_attention_weight_min": -25.23260498046875,
      "activations/layer23_attention_weight_max": 34.49695587158203,
      "activations/layer23_attention_weight_min": -22.187788009643555,
      "activations/layer2_attention_weight_max": 30.799156188964844,
      "activations/layer2_attention_weight_min": -27.87325668334961,
      "activations/layer3_attention_weight_max": 89.31346893310547,
      "activations/layer3_attention_weight_min": -91.25382232666016,
      "activations/layer4_attention_weight_max": 98.44435119628906,
      "activations/layer4_attention_weight_min": -99.41027069091797,
      "activations/layer5_attention_weight_max": 83.80300903320312,
      "activations/layer5_attention_weight_min": -78.26864624023438,
      "activations/layer6_attention_weight_max": 54.65266799926758,
      "activations/layer6_attention_weight_min": -52.922367095947266,
      "activations/layer7_attention_weight_max": 83.92816925048828,
      "activations/layer7_attention_weight_min": -72.59930419921875,
      "activations/layer8_attention_weight_max": 44.038299560546875,
      "activations/layer8_attention_weight_min": -47.403018951416016,
      "activations/layer9_attention_weight_max": 50.421165466308594,
      "activations/layer9_attention_weight_min": -51.9244499206543,
      "epoch": 7.95,
      "learning_rate": 9.976325757575757e-05,
      "loss": 2.8191,
      "step": 136750
    },
    {
      "activations/layer0_attention_weight_max": 14.33935546875,
      "activations/layer0_attention_weight_min": -14.88044548034668,
      "activations/layer10_attention_weight_max": 39.58439254760742,
      "activations/layer10_attention_weight_min": -39.825340270996094,
      "activations/layer11_attention_weight_max": 32.544673919677734,
      "activations/layer11_attention_weight_min": -32.577178955078125,
      "activations/layer12_attention_weight_max": 19.782867431640625,
      "activations/layer12_attention_weight_min": -25.37430763244629,
      "activations/layer13_attention_weight_max": 43.79133987426758,
      "activations/layer13_attention_weight_min": -32.859806060791016,
      "activations/layer14_attention_weight_max": 41.14679718017578,
      "activations/layer14_attention_weight_min": -29.97769546508789,
      "activations/layer15_attention_weight_max": 35.67540740966797,
      "activations/layer15_attention_weight_min": -29.545978546142578,
      "activations/layer16_attention_weight_max": 31.95406150817871,
      "activations/layer16_attention_weight_min": -29.065690994262695,
      "activations/layer17_attention_weight_max": 53.33277893066406,
      "activations/layer17_attention_weight_min": -43.931941986083984,
      "activations/layer18_attention_weight_max": 48.408729553222656,
      "activations/layer18_attention_weight_min": -37.28119659423828,
      "activations/layer19_attention_weight_max": 21.213054656982422,
      "activations/layer19_attention_weight_min": -20.145009994506836,
      "activations/layer1_attention_weight_max": 17.817489624023438,
      "activations/layer1_attention_weight_min": -15.600844383239746,
      "activations/layer20_attention_weight_max": 22.62714195251465,
      "activations/layer20_attention_weight_min": -21.145442962646484,
      "activations/layer21_attention_weight_max": 39.28910446166992,
      "activations/layer21_attention_weight_min": -26.009597778320312,
      "activations/layer22_attention_weight_max": 30.389368057250977,
      "activations/layer22_attention_weight_min": -25.19190216064453,
      "activations/layer23_attention_weight_max": 37.238121032714844,
      "activations/layer23_attention_weight_min": -21.905637741088867,
      "activations/layer2_attention_weight_max": 34.16120910644531,
      "activations/layer2_attention_weight_min": -30.843915939331055,
      "activations/layer3_attention_weight_max": 88.28035736083984,
      "activations/layer3_attention_weight_min": -92.58139038085938,
      "activations/layer4_attention_weight_max": 104.53443908691406,
      "activations/layer4_attention_weight_min": -101.63143157958984,
      "activations/layer5_attention_weight_max": 87.73238372802734,
      "activations/layer5_attention_weight_min": -82.52831268310547,
      "activations/layer6_attention_weight_max": 57.54527282714844,
      "activations/layer6_attention_weight_min": -54.74388885498047,
      "activations/layer7_attention_weight_max": 87.8256607055664,
      "activations/layer7_attention_weight_min": -75.7528305053711,
      "activations/layer8_attention_weight_max": 46.93399429321289,
      "activations/layer8_attention_weight_min": -53.83150100708008,
      "activations/layer9_attention_weight_max": 66.80570220947266,
      "activations/layer9_attention_weight_min": -52.20301055908203,
      "epoch": 7.95,
      "learning_rate": 9.974431818181817e-05,
      "loss": 2.8153,
      "step": 136800
    },
    {
      "activations/layer0_attention_weight_max": 15.198869705200195,
      "activations/layer0_attention_weight_min": -14.541380882263184,
      "activations/layer10_attention_weight_max": 38.93879699707031,
      "activations/layer10_attention_weight_min": -36.19414138793945,
      "activations/layer11_attention_weight_max": 32.134986877441406,
      "activations/layer11_attention_weight_min": -31.305234909057617,
      "activations/layer12_attention_weight_max": 16.76389503479004,
      "activations/layer12_attention_weight_min": -27.662511825561523,
      "activations/layer13_attention_weight_max": 40.49235534667969,
      "activations/layer13_attention_weight_min": -32.805294036865234,
      "activations/layer14_attention_weight_max": 33.79166793823242,
      "activations/layer14_attention_weight_min": -31.892654418945312,
      "activations/layer15_attention_weight_max": 35.798614501953125,
      "activations/layer15_attention_weight_min": -29.58450698852539,
      "activations/layer16_attention_weight_max": 31.731977462768555,
      "activations/layer16_attention_weight_min": -26.64375114440918,
      "activations/layer17_attention_weight_max": 54.2557258605957,
      "activations/layer17_attention_weight_min": -41.99169158935547,
      "activations/layer18_attention_weight_max": 48.70291519165039,
      "activations/layer18_attention_weight_min": -37.60647201538086,
      "activations/layer19_attention_weight_max": 22.825035095214844,
      "activations/layer19_attention_weight_min": -21.847248077392578,
      "activations/layer1_attention_weight_max": 17.733423233032227,
      "activations/layer1_attention_weight_min": -17.07527732849121,
      "activations/layer20_attention_weight_max": 22.254608154296875,
      "activations/layer20_attention_weight_min": -23.90278434753418,
      "activations/layer21_attention_weight_max": 43.36466598510742,
      "activations/layer21_attention_weight_min": -25.585901260375977,
      "activations/layer22_attention_weight_max": 31.08516502380371,
      "activations/layer22_attention_weight_min": -26.32378578186035,
      "activations/layer23_attention_weight_max": 35.657814025878906,
      "activations/layer23_attention_weight_min": -22.288848876953125,
      "activations/layer2_attention_weight_max": 32.95622634887695,
      "activations/layer2_attention_weight_min": -29.955806732177734,
      "activations/layer3_attention_weight_max": 89.95506286621094,
      "activations/layer3_attention_weight_min": -92.78164672851562,
      "activations/layer4_attention_weight_max": 104.1537094116211,
      "activations/layer4_attention_weight_min": -98.10335540771484,
      "activations/layer5_attention_weight_max": 86.97100830078125,
      "activations/layer5_attention_weight_min": -83.09884643554688,
      "activations/layer6_attention_weight_max": 57.25795364379883,
      "activations/layer6_attention_weight_min": -55.34702682495117,
      "activations/layer7_attention_weight_max": 78.99617767333984,
      "activations/layer7_attention_weight_min": -75.15274810791016,
      "activations/layer8_attention_weight_max": 45.22285842895508,
      "activations/layer8_attention_weight_min": -46.34745407104492,
      "activations/layer9_attention_weight_max": 51.916473388671875,
      "activations/layer9_attention_weight_min": -49.84505844116211,
      "epoch": 7.95,
      "learning_rate": 9.972537878787877e-05,
      "loss": 2.7996,
      "step": 136850
    },
    {
      "activations/layer0_attention_weight_max": 14.872373580932617,
      "activations/layer0_attention_weight_min": -14.603004455566406,
      "activations/layer10_attention_weight_max": 46.66410827636719,
      "activations/layer10_attention_weight_min": -42.07834243774414,
      "activations/layer11_attention_weight_max": 41.46758270263672,
      "activations/layer11_attention_weight_min": -37.20026397705078,
      "activations/layer12_attention_weight_max": 19.81499671936035,
      "activations/layer12_attention_weight_min": -24.865182876586914,
      "activations/layer13_attention_weight_max": 41.19282531738281,
      "activations/layer13_attention_weight_min": -30.99630355834961,
      "activations/layer14_attention_weight_max": 41.4710693359375,
      "activations/layer14_attention_weight_min": -30.245254516601562,
      "activations/layer15_attention_weight_max": 36.26468276977539,
      "activations/layer15_attention_weight_min": -28.73721694946289,
      "activations/layer16_attention_weight_max": 32.74470901489258,
      "activations/layer16_attention_weight_min": -26.75864028930664,
      "activations/layer17_attention_weight_max": 56.23933410644531,
      "activations/layer17_attention_weight_min": -41.407989501953125,
      "activations/layer18_attention_weight_max": 47.532291412353516,
      "activations/layer18_attention_weight_min": -35.28974151611328,
      "activations/layer19_attention_weight_max": 20.344196319580078,
      "activations/layer19_attention_weight_min": -21.38016700744629,
      "activations/layer1_attention_weight_max": 17.62998390197754,
      "activations/layer1_attention_weight_min": -14.664530754089355,
      "activations/layer20_attention_weight_max": 21.013837814331055,
      "activations/layer20_attention_weight_min": -20.111289978027344,
      "activations/layer21_attention_weight_max": 39.24811553955078,
      "activations/layer21_attention_weight_min": -25.407949447631836,
      "activations/layer22_attention_weight_max": 30.045238494873047,
      "activations/layer22_attention_weight_min": -24.919391632080078,
      "activations/layer23_attention_weight_max": 35.20991516113281,
      "activations/layer23_attention_weight_min": -23.01593589782715,
      "activations/layer2_attention_weight_max": 36.18205261230469,
      "activations/layer2_attention_weight_min": -33.788856506347656,
      "activations/layer3_attention_weight_max": 94.25004577636719,
      "activations/layer3_attention_weight_min": -94.42476654052734,
      "activations/layer4_attention_weight_max": 105.86756896972656,
      "activations/layer4_attention_weight_min": -101.161376953125,
      "activations/layer5_attention_weight_max": 89.6144027709961,
      "activations/layer5_attention_weight_min": -80.9441146850586,
      "activations/layer6_attention_weight_max": 56.0589714050293,
      "activations/layer6_attention_weight_min": -56.01366424560547,
      "activations/layer7_attention_weight_max": 91.42594909667969,
      "activations/layer7_attention_weight_min": -80.8695068359375,
      "activations/layer8_attention_weight_max": 56.02679443359375,
      "activations/layer8_attention_weight_min": -49.875850677490234,
      "activations/layer9_attention_weight_max": 77.34452056884766,
      "activations/layer9_attention_weight_min": -60.80714797973633,
      "epoch": 7.95,
      "learning_rate": 9.970643939393939e-05,
      "loss": 2.8181,
      "step": 136900
    },
    {
      "activations/layer0_attention_weight_max": 15.891140937805176,
      "activations/layer0_attention_weight_min": -14.733141899108887,
      "activations/layer10_attention_weight_max": 37.32007598876953,
      "activations/layer10_attention_weight_min": -35.526702880859375,
      "activations/layer11_attention_weight_max": 31.564023971557617,
      "activations/layer11_attention_weight_min": -30.324949264526367,
      "activations/layer12_attention_weight_max": 17.196453094482422,
      "activations/layer12_attention_weight_min": -24.392927169799805,
      "activations/layer13_attention_weight_max": 43.23389434814453,
      "activations/layer13_attention_weight_min": -32.203792572021484,
      "activations/layer14_attention_weight_max": 38.368492126464844,
      "activations/layer14_attention_weight_min": -30.4153995513916,
      "activations/layer15_attention_weight_max": 37.80350112915039,
      "activations/layer15_attention_weight_min": -28.584274291992188,
      "activations/layer16_attention_weight_max": 31.635269165039062,
      "activations/layer16_attention_weight_min": -27.313961029052734,
      "activations/layer17_attention_weight_max": 58.58551025390625,
      "activations/layer17_attention_weight_min": -46.46492004394531,
      "activations/layer18_attention_weight_max": 48.97124481201172,
      "activations/layer18_attention_weight_min": -36.60091018676758,
      "activations/layer19_attention_weight_max": 25.084720611572266,
      "activations/layer19_attention_weight_min": -21.58107566833496,
      "activations/layer1_attention_weight_max": 16.974531173706055,
      "activations/layer1_attention_weight_min": -14.584059715270996,
      "activations/layer20_attention_weight_max": 23.360485076904297,
      "activations/layer20_attention_weight_min": -23.459674835205078,
      "activations/layer21_attention_weight_max": 42.69932174682617,
      "activations/layer21_attention_weight_min": -23.20356559753418,
      "activations/layer22_attention_weight_max": 34.864742279052734,
      "activations/layer22_attention_weight_min": -24.014354705810547,
      "activations/layer23_attention_weight_max": 41.44047927856445,
      "activations/layer23_attention_weight_min": -22.600643157958984,
      "activations/layer2_attention_weight_max": 32.533485412597656,
      "activations/layer2_attention_weight_min": -29.626737594604492,
      "activations/layer3_attention_weight_max": 92.15660858154297,
      "activations/layer3_attention_weight_min": -92.95840454101562,
      "activations/layer4_attention_weight_max": 102.39754486083984,
      "activations/layer4_attention_weight_min": -97.60113525390625,
      "activations/layer5_attention_weight_max": 88.32918548583984,
      "activations/layer5_attention_weight_min": -81.30421447753906,
      "activations/layer6_attention_weight_max": 56.87566375732422,
      "activations/layer6_attention_weight_min": -54.78491973876953,
      "activations/layer7_attention_weight_max": 77.67280578613281,
      "activations/layer7_attention_weight_min": -72.9019775390625,
      "activations/layer8_attention_weight_max": 46.93426513671875,
      "activations/layer8_attention_weight_min": -46.646209716796875,
      "activations/layer9_attention_weight_max": 51.3895149230957,
      "activations/layer9_attention_weight_min": -53.21741485595703,
      "epoch": 7.96,
      "learning_rate": 9.968749999999999e-05,
      "loss": 2.8239,
      "step": 136950
    },
    {
      "activations/layer0_attention_weight_max": 15.029824256896973,
      "activations/layer0_attention_weight_min": -14.775179862976074,
      "activations/layer10_attention_weight_max": 39.231937408447266,
      "activations/layer10_attention_weight_min": -38.52101516723633,
      "activations/layer11_attention_weight_max": 31.053882598876953,
      "activations/layer11_attention_weight_min": -32.155975341796875,
      "activations/layer12_attention_weight_max": 19.887094497680664,
      "activations/layer12_attention_weight_min": -29.160131454467773,
      "activations/layer13_attention_weight_max": 61.99360275268555,
      "activations/layer13_attention_weight_min": -36.003456115722656,
      "activations/layer14_attention_weight_max": 52.36050796508789,
      "activations/layer14_attention_weight_min": -32.756744384765625,
      "activations/layer15_attention_weight_max": 41.48151397705078,
      "activations/layer15_attention_weight_min": -28.889842987060547,
      "activations/layer16_attention_weight_max": 36.704586029052734,
      "activations/layer16_attention_weight_min": -25.778703689575195,
      "activations/layer17_attention_weight_max": 57.07061004638672,
      "activations/layer17_attention_weight_min": -43.076751708984375,
      "activations/layer18_attention_weight_max": 47.55580520629883,
      "activations/layer18_attention_weight_min": -35.88445281982422,
      "activations/layer19_attention_weight_max": 19.662382125854492,
      "activations/layer19_attention_weight_min": -24.82457733154297,
      "activations/layer1_attention_weight_max": 17.11762046813965,
      "activations/layer1_attention_weight_min": -15.152770042419434,
      "activations/layer20_attention_weight_max": 20.711326599121094,
      "activations/layer20_attention_weight_min": -23.106595993041992,
      "activations/layer21_attention_weight_max": 39.67927169799805,
      "activations/layer21_attention_weight_min": -24.382671356201172,
      "activations/layer22_attention_weight_max": 28.436113357543945,
      "activations/layer22_attention_weight_min": -23.66664695739746,
      "activations/layer23_attention_weight_max": 33.786773681640625,
      "activations/layer23_attention_weight_min": -21.030311584472656,
      "activations/layer2_attention_weight_max": 34.357208251953125,
      "activations/layer2_attention_weight_min": -29.641460418701172,
      "activations/layer3_attention_weight_max": 89.39340209960938,
      "activations/layer3_attention_weight_min": -88.28501892089844,
      "activations/layer4_attention_weight_max": 104.66168212890625,
      "activations/layer4_attention_weight_min": -97.50984191894531,
      "activations/layer5_attention_weight_max": 84.26554107666016,
      "activations/layer5_attention_weight_min": -79.18617248535156,
      "activations/layer6_attention_weight_max": 54.29283905029297,
      "activations/layer6_attention_weight_min": -53.805274963378906,
      "activations/layer7_attention_weight_max": 78.50897216796875,
      "activations/layer7_attention_weight_min": -74.78461456298828,
      "activations/layer8_attention_weight_max": 47.03639221191406,
      "activations/layer8_attention_weight_min": -46.04812240600586,
      "activations/layer9_attention_weight_max": 53.069679260253906,
      "activations/layer9_attention_weight_min": -50.896949768066406,
      "epoch": 7.96,
      "learning_rate": 9.96685606060606e-05,
      "loss": 2.8238,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.5202,
      "eval_samples_per_second": 503.976,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.5202,
      "eval_openwebtext_samples_per_second": 503.976,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 2.0328,
      "eval_wikitext_samples_per_second": 224.32,
      "step": 137000
    },
    {
      "epoch": 7.96,
      "eval_lambada_loss": 2.703125,
      "eval_lambada_ppl": 14.926303617194517,
      "eval_lambada_runtime": 9.627,
      "eval_lambada_samples_per_second": 505.767,
      "step": 137000
    },
    {
      "activations/layer0_attention_weight_max": 16.586936950683594,
      "activations/layer0_attention_weight_min": -14.62277603149414,
      "activations/layer10_attention_weight_max": 37.657020568847656,
      "activations/layer10_attention_weight_min": -38.112396240234375,
      "activations/layer11_attention_weight_max": 32.82965087890625,
      "activations/layer11_attention_weight_min": -29.815326690673828,
      "activations/layer12_attention_weight_max": 22.14309310913086,
      "activations/layer12_attention_weight_min": -27.97734832763672,
      "activations/layer13_attention_weight_max": 57.54874038696289,
      "activations/layer13_attention_weight_min": -38.769798278808594,
      "activations/layer14_attention_weight_max": 53.905460357666016,
      "activations/layer14_attention_weight_min": -34.548377990722656,
      "activations/layer15_attention_weight_max": 36.51247787475586,
      "activations/layer15_attention_weight_min": -29.56922721862793,
      "activations/layer16_attention_weight_max": 29.804584503173828,
      "activations/layer16_attention_weight_min": -28.46303367614746,
      "activations/layer17_attention_weight_max": 56.05127716064453,
      "activations/layer17_attention_weight_min": -41.57637405395508,
      "activations/layer18_attention_weight_max": 44.13813781738281,
      "activations/layer18_attention_weight_min": -35.014892578125,
      "activations/layer19_attention_weight_max": 19.715864181518555,
      "activations/layer19_attention_weight_min": -20.167749404907227,
      "activations/layer1_attention_weight_max": 17.495264053344727,
      "activations/layer1_attention_weight_min": -16.59110450744629,
      "activations/layer20_attention_weight_max": 21.072254180908203,
      "activations/layer20_attention_weight_min": -21.77171516418457,
      "activations/layer21_attention_weight_max": 34.840518951416016,
      "activations/layer21_attention_weight_min": -22.677169799804688,
      "activations/layer22_attention_weight_max": 28.124969482421875,
      "activations/layer22_attention_weight_min": -24.830984115600586,
      "activations/layer23_attention_weight_max": 34.87018585205078,
      "activations/layer23_attention_weight_min": -20.205448150634766,
      "activations/layer2_attention_weight_max": 33.8453254699707,
      "activations/layer2_attention_weight_min": -31.950523376464844,
      "activations/layer3_attention_weight_max": 91.94031524658203,
      "activations/layer3_attention_weight_min": -91.98193359375,
      "activations/layer4_attention_weight_max": 104.58928680419922,
      "activations/layer4_attention_weight_min": -98.94335174560547,
      "activations/layer5_attention_weight_max": 86.63433837890625,
      "activations/layer5_attention_weight_min": -83.53694915771484,
      "activations/layer6_attention_weight_max": 57.40520477294922,
      "activations/layer6_attention_weight_min": -55.98081588745117,
      "activations/layer7_attention_weight_max": 75.29977416992188,
      "activations/layer7_attention_weight_min": -74.69448852539062,
      "activations/layer8_attention_weight_max": 45.718170166015625,
      "activations/layer8_attention_weight_min": -48.91379928588867,
      "activations/layer9_attention_weight_max": 57.270721435546875,
      "activations/layer9_attention_weight_min": -51.993080139160156,
      "epoch": 7.96,
      "learning_rate": 9.964962121212121e-05,
      "loss": 2.8038,
      "step": 137050
    },
    {
      "activations/layer0_attention_weight_max": 14.75527286529541,
      "activations/layer0_attention_weight_min": -14.220050811767578,
      "activations/layer10_attention_weight_max": 38.526039123535156,
      "activations/layer10_attention_weight_min": -35.637332916259766,
      "activations/layer11_attention_weight_max": 32.6691780090332,
      "activations/layer11_attention_weight_min": -32.200843811035156,
      "activations/layer12_attention_weight_max": 21.927597045898438,
      "activations/layer12_attention_weight_min": -27.777297973632812,
      "activations/layer13_attention_weight_max": 43.140724182128906,
      "activations/layer13_attention_weight_min": -29.57623863220215,
      "activations/layer14_attention_weight_max": 61.151084899902344,
      "activations/layer14_attention_weight_min": -36.595462799072266,
      "activations/layer15_attention_weight_max": 47.80376052856445,
      "activations/layer15_attention_weight_min": -29.853540420532227,
      "activations/layer16_attention_weight_max": 30.182722091674805,
      "activations/layer16_attention_weight_min": -27.714111328125,
      "activations/layer17_attention_weight_max": 51.49215316772461,
      "activations/layer17_attention_weight_min": -42.565757751464844,
      "activations/layer18_attention_weight_max": 43.00548553466797,
      "activations/layer18_attention_weight_min": -35.780372619628906,
      "activations/layer19_attention_weight_max": 21.213973999023438,
      "activations/layer19_attention_weight_min": -21.69332504272461,
      "activations/layer1_attention_weight_max": 16.844812393188477,
      "activations/layer1_attention_weight_min": -14.447582244873047,
      "activations/layer20_attention_weight_max": 22.268037796020508,
      "activations/layer20_attention_weight_min": -22.24505043029785,
      "activations/layer21_attention_weight_max": 41.04848098754883,
      "activations/layer21_attention_weight_min": -24.286937713623047,
      "activations/layer22_attention_weight_max": 31.97817611694336,
      "activations/layer22_attention_weight_min": -25.76321029663086,
      "activations/layer23_attention_weight_max": 32.811256408691406,
      "activations/layer23_attention_weight_min": -23.162517547607422,
      "activations/layer2_attention_weight_max": 35.65114212036133,
      "activations/layer2_attention_weight_min": -31.18105697631836,
      "activations/layer3_attention_weight_max": 93.71988677978516,
      "activations/layer3_attention_weight_min": -94.73802947998047,
      "activations/layer4_attention_weight_max": 105.33878326416016,
      "activations/layer4_attention_weight_min": -97.75079345703125,
      "activations/layer5_attention_weight_max": 84.6512451171875,
      "activations/layer5_attention_weight_min": -78.33480834960938,
      "activations/layer6_attention_weight_max": 57.388214111328125,
      "activations/layer6_attention_weight_min": -52.87370681762695,
      "activations/layer7_attention_weight_max": 75.36813354492188,
      "activations/layer7_attention_weight_min": -72.81156921386719,
      "activations/layer8_attention_weight_max": 43.0157356262207,
      "activations/layer8_attention_weight_min": -47.119354248046875,
      "activations/layer9_attention_weight_max": 56.227821350097656,
      "activations/layer9_attention_weight_min": -48.001930236816406,
      "epoch": 7.97,
      "learning_rate": 9.963068181818181e-05,
      "loss": 2.8262,
      "step": 137100
    },
    {
      "activations/layer0_attention_weight_max": 15.84521198272705,
      "activations/layer0_attention_weight_min": -14.096068382263184,
      "activations/layer10_attention_weight_max": 39.56690216064453,
      "activations/layer10_attention_weight_min": -37.153350830078125,
      "activations/layer11_attention_weight_max": 34.42182159423828,
      "activations/layer11_attention_weight_min": -31.462438583374023,
      "activations/layer12_attention_weight_max": 21.29660415649414,
      "activations/layer12_attention_weight_min": -27.00956916809082,
      "activations/layer13_attention_weight_max": 51.49303436279297,
      "activations/layer13_attention_weight_min": -30.53119468688965,
      "activations/layer14_attention_weight_max": 43.94685363769531,
      "activations/layer14_attention_weight_min": -34.121273040771484,
      "activations/layer15_attention_weight_max": 38.70542526245117,
      "activations/layer15_attention_weight_min": -30.49614715576172,
      "activations/layer16_attention_weight_max": 35.04460906982422,
      "activations/layer16_attention_weight_min": -26.877286911010742,
      "activations/layer17_attention_weight_max": 53.96232986450195,
      "activations/layer17_attention_weight_min": -45.3182258605957,
      "activations/layer18_attention_weight_max": 50.38620376586914,
      "activations/layer18_attention_weight_min": -40.03639221191406,
      "activations/layer19_attention_weight_max": 22.214702606201172,
      "activations/layer19_attention_weight_min": -23.097322463989258,
      "activations/layer1_attention_weight_max": 17.20064353942871,
      "activations/layer1_attention_weight_min": -14.759809494018555,
      "activations/layer20_attention_weight_max": 22.200963973999023,
      "activations/layer20_attention_weight_min": -23.731365203857422,
      "activations/layer21_attention_weight_max": 44.71565246582031,
      "activations/layer21_attention_weight_min": -25.099254608154297,
      "activations/layer22_attention_weight_max": 32.5411376953125,
      "activations/layer22_attention_weight_min": -26.82956886291504,
      "activations/layer23_attention_weight_max": 35.67658615112305,
      "activations/layer23_attention_weight_min": -22.14993667602539,
      "activations/layer2_attention_weight_max": 33.99329376220703,
      "activations/layer2_attention_weight_min": -30.125411987304688,
      "activations/layer3_attention_weight_max": 91.59613037109375,
      "activations/layer3_attention_weight_min": -97.14408111572266,
      "activations/layer4_attention_weight_max": 99.53079223632812,
      "activations/layer4_attention_weight_min": -94.5801773071289,
      "activations/layer5_attention_weight_max": 84.89693450927734,
      "activations/layer5_attention_weight_min": -79.83448791503906,
      "activations/layer6_attention_weight_max": 57.007362365722656,
      "activations/layer6_attention_weight_min": -56.15045928955078,
      "activations/layer7_attention_weight_max": 80.72492218017578,
      "activations/layer7_attention_weight_min": -74.16778564453125,
      "activations/layer8_attention_weight_max": 45.80255126953125,
      "activations/layer8_attention_weight_min": -46.5571174621582,
      "activations/layer9_attention_weight_max": 54.12697219848633,
      "activations/layer9_attention_weight_min": -49.64973068237305,
      "epoch": 7.97,
      "learning_rate": 9.961174242424242e-05,
      "loss": 2.797,
      "step": 137150
    },
    {
      "activations/layer0_attention_weight_max": 14.827603340148926,
      "activations/layer0_attention_weight_min": -14.212875366210938,
      "activations/layer10_attention_weight_max": 38.812408447265625,
      "activations/layer10_attention_weight_min": -36.80126190185547,
      "activations/layer11_attention_weight_max": 33.58234786987305,
      "activations/layer11_attention_weight_min": -31.402130126953125,
      "activations/layer12_attention_weight_max": 20.242822647094727,
      "activations/layer12_attention_weight_min": -25.4021053314209,
      "activations/layer13_attention_weight_max": 42.12923049926758,
      "activations/layer13_attention_weight_min": -31.400671005249023,
      "activations/layer14_attention_weight_max": 37.36811447143555,
      "activations/layer14_attention_weight_min": -31.482206344604492,
      "activations/layer15_attention_weight_max": 35.08723449707031,
      "activations/layer15_attention_weight_min": -29.154027938842773,
      "activations/layer16_attention_weight_max": 32.39120864868164,
      "activations/layer16_attention_weight_min": -27.884506225585938,
      "activations/layer17_attention_weight_max": 53.69918441772461,
      "activations/layer17_attention_weight_min": -42.608131408691406,
      "activations/layer18_attention_weight_max": 48.44120788574219,
      "activations/layer18_attention_weight_min": -36.89226531982422,
      "activations/layer19_attention_weight_max": 24.097322463989258,
      "activations/layer19_attention_weight_min": -22.889801025390625,
      "activations/layer1_attention_weight_max": 16.580232620239258,
      "activations/layer1_attention_weight_min": -15.353859901428223,
      "activations/layer20_attention_weight_max": 22.228532791137695,
      "activations/layer20_attention_weight_min": -21.100217819213867,
      "activations/layer21_attention_weight_max": 36.97149658203125,
      "activations/layer21_attention_weight_min": -23.36174774169922,
      "activations/layer22_attention_weight_max": 31.19470977783203,
      "activations/layer22_attention_weight_min": -25.32651138305664,
      "activations/layer23_attention_weight_max": 36.110382080078125,
      "activations/layer23_attention_weight_min": -22.68288230895996,
      "activations/layer2_attention_weight_max": 31.83177947998047,
      "activations/layer2_attention_weight_min": -30.802282333374023,
      "activations/layer3_attention_weight_max": 91.34254455566406,
      "activations/layer3_attention_weight_min": -92.46041107177734,
      "activations/layer4_attention_weight_max": 105.7982406616211,
      "activations/layer4_attention_weight_min": -103.4682388305664,
      "activations/layer5_attention_weight_max": 86.62332153320312,
      "activations/layer5_attention_weight_min": -84.00686645507812,
      "activations/layer6_attention_weight_max": 57.07047653198242,
      "activations/layer6_attention_weight_min": -54.85222244262695,
      "activations/layer7_attention_weight_max": 77.78759002685547,
      "activations/layer7_attention_weight_min": -78.80792999267578,
      "activations/layer8_attention_weight_max": 47.61402130126953,
      "activations/layer8_attention_weight_min": -47.13685607910156,
      "activations/layer9_attention_weight_max": 49.676605224609375,
      "activations/layer9_attention_weight_min": -49.21282196044922,
      "epoch": 7.97,
      "learning_rate": 9.959280303030301e-05,
      "loss": 2.81,
      "step": 137200
    },
    {
      "activations/layer0_attention_weight_max": 15.284168243408203,
      "activations/layer0_attention_weight_min": -14.590620994567871,
      "activations/layer10_attention_weight_max": 34.75474548339844,
      "activations/layer10_attention_weight_min": -33.24995803833008,
      "activations/layer11_attention_weight_max": 30.170583724975586,
      "activations/layer11_attention_weight_min": -29.546857833862305,
      "activations/layer12_attention_weight_max": 18.79999351501465,
      "activations/layer12_attention_weight_min": -24.909019470214844,
      "activations/layer13_attention_weight_max": 40.40303421020508,
      "activations/layer13_attention_weight_min": -29.25775146484375,
      "activations/layer14_attention_weight_max": 37.78002166748047,
      "activations/layer14_attention_weight_min": -32.9968376159668,
      "activations/layer15_attention_weight_max": 37.1761589050293,
      "activations/layer15_attention_weight_min": -29.35262680053711,
      "activations/layer16_attention_weight_max": 32.84172439575195,
      "activations/layer16_attention_weight_min": -29.992294311523438,
      "activations/layer17_attention_weight_max": 54.38079071044922,
      "activations/layer17_attention_weight_min": -46.63279342651367,
      "activations/layer18_attention_weight_max": 52.033470153808594,
      "activations/layer18_attention_weight_min": -39.23076629638672,
      "activations/layer19_attention_weight_max": 25.473459243774414,
      "activations/layer19_attention_weight_min": -23.442832946777344,
      "activations/layer1_attention_weight_max": 16.61012840270996,
      "activations/layer1_attention_weight_min": -15.696086883544922,
      "activations/layer20_attention_weight_max": 25.27626609802246,
      "activations/layer20_attention_weight_min": -24.19144630432129,
      "activations/layer21_attention_weight_max": 49.99048614501953,
      "activations/layer21_attention_weight_min": -28.10590171813965,
      "activations/layer22_attention_weight_max": 32.71323776245117,
      "activations/layer22_attention_weight_min": -26.68855857849121,
      "activations/layer23_attention_weight_max": 37.689266204833984,
      "activations/layer23_attention_weight_min": -23.773494720458984,
      "activations/layer2_attention_weight_max": 32.206478118896484,
      "activations/layer2_attention_weight_min": -29.532135009765625,
      "activations/layer3_attention_weight_max": 85.659423828125,
      "activations/layer3_attention_weight_min": -87.70659637451172,
      "activations/layer4_attention_weight_max": 99.61369323730469,
      "activations/layer4_attention_weight_min": -93.10613250732422,
      "activations/layer5_attention_weight_max": 85.60408020019531,
      "activations/layer5_attention_weight_min": -77.60380554199219,
      "activations/layer6_attention_weight_max": 55.14876174926758,
      "activations/layer6_attention_weight_min": -54.59714889526367,
      "activations/layer7_attention_weight_max": 78.45235443115234,
      "activations/layer7_attention_weight_min": -70.42786407470703,
      "activations/layer8_attention_weight_max": 44.69375228881836,
      "activations/layer8_attention_weight_min": -44.89856719970703,
      "activations/layer9_attention_weight_max": 50.15293502807617,
      "activations/layer9_attention_weight_min": -49.13674545288086,
      "epoch": 7.97,
      "learning_rate": 9.957386363636363e-05,
      "loss": 2.7968,
      "step": 137250
    },
    {
      "activations/layer0_attention_weight_max": 14.800335884094238,
      "activations/layer0_attention_weight_min": -14.48919677734375,
      "activations/layer10_attention_weight_max": 38.40937042236328,
      "activations/layer10_attention_weight_min": -35.306968688964844,
      "activations/layer11_attention_weight_max": 32.47290802001953,
      "activations/layer11_attention_weight_min": -31.83314323425293,
      "activations/layer12_attention_weight_max": 19.304229736328125,
      "activations/layer12_attention_weight_min": -32.043514251708984,
      "activations/layer13_attention_weight_max": 38.475643157958984,
      "activations/layer13_attention_weight_min": -29.44153594970703,
      "activations/layer14_attention_weight_max": 40.99504470825195,
      "activations/layer14_attention_weight_min": -31.60106658935547,
      "activations/layer15_attention_weight_max": 34.116024017333984,
      "activations/layer15_attention_weight_min": -28.9208984375,
      "activations/layer16_attention_weight_max": 31.34052276611328,
      "activations/layer16_attention_weight_min": -28.792724609375,
      "activations/layer17_attention_weight_max": 50.25105285644531,
      "activations/layer17_attention_weight_min": -43.01152420043945,
      "activations/layer18_attention_weight_max": 45.33896255493164,
      "activations/layer18_attention_weight_min": -36.5692253112793,
      "activations/layer19_attention_weight_max": 23.023630142211914,
      "activations/layer19_attention_weight_min": -21.567914962768555,
      "activations/layer1_attention_weight_max": 16.94353485107422,
      "activations/layer1_attention_weight_min": -16.76351547241211,
      "activations/layer20_attention_weight_max": 20.985063552856445,
      "activations/layer20_attention_weight_min": -21.330541610717773,
      "activations/layer21_attention_weight_max": 39.43783950805664,
      "activations/layer21_attention_weight_min": -24.769372940063477,
      "activations/layer22_attention_weight_max": 36.65705490112305,
      "activations/layer22_attention_weight_min": -26.399011611938477,
      "activations/layer23_attention_weight_max": 39.363033294677734,
      "activations/layer23_attention_weight_min": -20.77814292907715,
      "activations/layer2_attention_weight_max": 33.626365661621094,
      "activations/layer2_attention_weight_min": -30.372482299804688,
      "activations/layer3_attention_weight_max": 92.47604370117188,
      "activations/layer3_attention_weight_min": -87.19012451171875,
      "activations/layer4_attention_weight_max": 100.38536834716797,
      "activations/layer4_attention_weight_min": -95.89771270751953,
      "activations/layer5_attention_weight_max": 84.06683349609375,
      "activations/layer5_attention_weight_min": -77.7914047241211,
      "activations/layer6_attention_weight_max": 54.2221565246582,
      "activations/layer6_attention_weight_min": -51.86052322387695,
      "activations/layer7_attention_weight_max": 71.3849868774414,
      "activations/layer7_attention_weight_min": -73.43109893798828,
      "activations/layer8_attention_weight_max": 42.53125,
      "activations/layer8_attention_weight_min": -42.83781814575195,
      "activations/layer9_attention_weight_max": 49.23542022705078,
      "activations/layer9_attention_weight_min": -47.31019973754883,
      "epoch": 7.98,
      "learning_rate": 9.955492424242424e-05,
      "loss": 2.8161,
      "step": 137300
    },
    {
      "activations/layer0_attention_weight_max": 15.807875633239746,
      "activations/layer0_attention_weight_min": -14.72331714630127,
      "activations/layer10_attention_weight_max": 40.10562515258789,
      "activations/layer10_attention_weight_min": -35.86589431762695,
      "activations/layer11_attention_weight_max": 31.589038848876953,
      "activations/layer11_attention_weight_min": -30.692838668823242,
      "activations/layer12_attention_weight_max": 18.911712646484375,
      "activations/layer12_attention_weight_min": -25.457239151000977,
      "activations/layer13_attention_weight_max": 58.10401153564453,
      "activations/layer13_attention_weight_min": -40.75111770629883,
      "activations/layer14_attention_weight_max": 46.08878707885742,
      "activations/layer14_attention_weight_min": -32.685028076171875,
      "activations/layer15_attention_weight_max": 42.99851989746094,
      "activations/layer15_attention_weight_min": -30.51895523071289,
      "activations/layer16_attention_weight_max": 32.08387756347656,
      "activations/layer16_attention_weight_min": -27.454557418823242,
      "activations/layer17_attention_weight_max": 55.98143768310547,
      "activations/layer17_attention_weight_min": -41.61543273925781,
      "activations/layer18_attention_weight_max": 47.61970901489258,
      "activations/layer18_attention_weight_min": -36.518341064453125,
      "activations/layer19_attention_weight_max": 20.702728271484375,
      "activations/layer19_attention_weight_min": -20.891576766967773,
      "activations/layer1_attention_weight_max": 16.993478775024414,
      "activations/layer1_attention_weight_min": -15.104327201843262,
      "activations/layer20_attention_weight_max": 23.676088333129883,
      "activations/layer20_attention_weight_min": -21.221330642700195,
      "activations/layer21_attention_weight_max": 45.12343978881836,
      "activations/layer21_attention_weight_min": -23.577537536621094,
      "activations/layer22_attention_weight_max": 31.718381881713867,
      "activations/layer22_attention_weight_min": -23.768823623657227,
      "activations/layer23_attention_weight_max": 37.64175796508789,
      "activations/layer23_attention_weight_min": -22.62361717224121,
      "activations/layer2_attention_weight_max": 34.433189392089844,
      "activations/layer2_attention_weight_min": -31.67292022705078,
      "activations/layer3_attention_weight_max": 88.47805786132812,
      "activations/layer3_attention_weight_min": -89.1003646850586,
      "activations/layer4_attention_weight_max": 105.1844482421875,
      "activations/layer4_attention_weight_min": -100.49726104736328,
      "activations/layer5_attention_weight_max": 89.8900146484375,
      "activations/layer5_attention_weight_min": -81.2219467163086,
      "activations/layer6_attention_weight_max": 56.2921257019043,
      "activations/layer6_attention_weight_min": -55.60845947265625,
      "activations/layer7_attention_weight_max": 77.30622863769531,
      "activations/layer7_attention_weight_min": -73.44567108154297,
      "activations/layer8_attention_weight_max": 46.54270553588867,
      "activations/layer8_attention_weight_min": -51.429256439208984,
      "activations/layer9_attention_weight_max": 50.66188049316406,
      "activations/layer9_attention_weight_min": -50.34922409057617,
      "epoch": 7.98,
      "learning_rate": 9.953598484848483e-05,
      "loss": 2.8194,
      "step": 137350
    },
    {
      "activations/layer0_attention_weight_max": 16.438674926757812,
      "activations/layer0_attention_weight_min": -15.052421569824219,
      "activations/layer10_attention_weight_max": 36.440650939941406,
      "activations/layer10_attention_weight_min": -37.70658493041992,
      "activations/layer11_attention_weight_max": 31.398427963256836,
      "activations/layer11_attention_weight_min": -33.02600860595703,
      "activations/layer12_attention_weight_max": 23.872953414916992,
      "activations/layer12_attention_weight_min": -26.431861877441406,
      "activations/layer13_attention_weight_max": 43.34601974487305,
      "activations/layer13_attention_weight_min": -33.31846237182617,
      "activations/layer14_attention_weight_max": 45.30742645263672,
      "activations/layer14_attention_weight_min": -32.924339294433594,
      "activations/layer15_attention_weight_max": 36.66789627075195,
      "activations/layer15_attention_weight_min": -29.986980438232422,
      "activations/layer16_attention_weight_max": 35.23025131225586,
      "activations/layer16_attention_weight_min": -29.05055046081543,
      "activations/layer17_attention_weight_max": 57.0226936340332,
      "activations/layer17_attention_weight_min": -46.42426681518555,
      "activations/layer18_attention_weight_max": 50.1862678527832,
      "activations/layer18_attention_weight_min": -38.53019332885742,
      "activations/layer19_attention_weight_max": 22.118682861328125,
      "activations/layer19_attention_weight_min": -23.281536102294922,
      "activations/layer1_attention_weight_max": 17.146942138671875,
      "activations/layer1_attention_weight_min": -16.629005432128906,
      "activations/layer20_attention_weight_max": 21.7242431640625,
      "activations/layer20_attention_weight_min": -19.922924041748047,
      "activations/layer21_attention_weight_max": 40.159217834472656,
      "activations/layer21_attention_weight_min": -25.141855239868164,
      "activations/layer22_attention_weight_max": 36.4967155456543,
      "activations/layer22_attention_weight_min": -26.91916275024414,
      "activations/layer23_attention_weight_max": 35.57469940185547,
      "activations/layer23_attention_weight_min": -20.60682487487793,
      "activations/layer2_attention_weight_max": 32.288490295410156,
      "activations/layer2_attention_weight_min": -30.653484344482422,
      "activations/layer3_attention_weight_max": 88.05562591552734,
      "activations/layer3_attention_weight_min": -92.89883422851562,
      "activations/layer4_attention_weight_max": 103.97965240478516,
      "activations/layer4_attention_weight_min": -96.24642181396484,
      "activations/layer5_attention_weight_max": 88.76204681396484,
      "activations/layer5_attention_weight_min": -81.71833801269531,
      "activations/layer6_attention_weight_max": 59.004676818847656,
      "activations/layer6_attention_weight_min": -55.25284194946289,
      "activations/layer7_attention_weight_max": 77.05233001708984,
      "activations/layer7_attention_weight_min": -77.4106216430664,
      "activations/layer8_attention_weight_max": 45.457969665527344,
      "activations/layer8_attention_weight_min": -46.323795318603516,
      "activations/layer9_attention_weight_max": 50.986392974853516,
      "activations/layer9_attention_weight_min": -50.7695426940918,
      "epoch": 7.98,
      "learning_rate": 9.951704545454544e-05,
      "loss": 2.8247,
      "step": 137400
    },
    {
      "activations/layer0_attention_weight_max": 14.338199615478516,
      "activations/layer0_attention_weight_min": -15.247749328613281,
      "activations/layer10_attention_weight_max": 39.4658203125,
      "activations/layer10_attention_weight_min": -35.901344299316406,
      "activations/layer11_attention_weight_max": 32.882286071777344,
      "activations/layer11_attention_weight_min": -31.143569946289062,
      "activations/layer12_attention_weight_max": 20.003986358642578,
      "activations/layer12_attention_weight_min": -25.728269577026367,
      "activations/layer13_attention_weight_max": 41.804847717285156,
      "activations/layer13_attention_weight_min": -31.998498916625977,
      "activations/layer14_attention_weight_max": 37.0671272277832,
      "activations/layer14_attention_weight_min": -31.08586311340332,
      "activations/layer15_attention_weight_max": 35.49665832519531,
      "activations/layer15_attention_weight_min": -28.693586349487305,
      "activations/layer16_attention_weight_max": 29.33579444885254,
      "activations/layer16_attention_weight_min": -27.074417114257812,
      "activations/layer17_attention_weight_max": 52.5526123046875,
      "activations/layer17_attention_weight_min": -41.25780487060547,
      "activations/layer18_attention_weight_max": 45.19833755493164,
      "activations/layer18_attention_weight_min": -35.36842727661133,
      "activations/layer19_attention_weight_max": 21.881589889526367,
      "activations/layer19_attention_weight_min": -21.079086303710938,
      "activations/layer1_attention_weight_max": 16.93219757080078,
      "activations/layer1_attention_weight_min": -17.064329147338867,
      "activations/layer20_attention_weight_max": 22.670394897460938,
      "activations/layer20_attention_weight_min": -20.473922729492188,
      "activations/layer21_attention_weight_max": 39.79189682006836,
      "activations/layer21_attention_weight_min": -24.50748634338379,
      "activations/layer22_attention_weight_max": 30.171520233154297,
      "activations/layer22_attention_weight_min": -24.21811866760254,
      "activations/layer23_attention_weight_max": 34.018653869628906,
      "activations/layer23_attention_weight_min": -21.464475631713867,
      "activations/layer2_attention_weight_max": 33.782684326171875,
      "activations/layer2_attention_weight_min": -32.66110610961914,
      "activations/layer3_attention_weight_max": 91.42070007324219,
      "activations/layer3_attention_weight_min": -92.30297088623047,
      "activations/layer4_attention_weight_max": 103.42909240722656,
      "activations/layer4_attention_weight_min": -97.12336730957031,
      "activations/layer5_attention_weight_max": 85.98345947265625,
      "activations/layer5_attention_weight_min": -79.47868347167969,
      "activations/layer6_attention_weight_max": 55.12784957885742,
      "activations/layer6_attention_weight_min": -53.86023712158203,
      "activations/layer7_attention_weight_max": 76.4599609375,
      "activations/layer7_attention_weight_min": -73.91534423828125,
      "activations/layer8_attention_weight_max": 44.71391296386719,
      "activations/layer8_attention_weight_min": -47.11809158325195,
      "activations/layer9_attention_weight_max": 48.402732849121094,
      "activations/layer9_attention_weight_min": -48.51241683959961,
      "epoch": 7.99,
      "learning_rate": 9.949810606060606e-05,
      "loss": 2.8311,
      "step": 137450
    },
    {
      "activations/layer0_attention_weight_max": 15.60464096069336,
      "activations/layer0_attention_weight_min": -14.785484313964844,
      "activations/layer10_attention_weight_max": 38.92226791381836,
      "activations/layer10_attention_weight_min": -38.04630661010742,
      "activations/layer11_attention_weight_max": 32.13433837890625,
      "activations/layer11_attention_weight_min": -31.982206344604492,
      "activations/layer12_attention_weight_max": 18.947927474975586,
      "activations/layer12_attention_weight_min": -27.981990814208984,
      "activations/layer13_attention_weight_max": 56.540733337402344,
      "activations/layer13_attention_weight_min": -31.602014541625977,
      "activations/layer14_attention_weight_max": 61.58675003051758,
      "activations/layer14_attention_weight_min": -34.466514587402344,
      "activations/layer15_attention_weight_max": 43.67422103881836,
      "activations/layer15_attention_weight_min": -30.034332275390625,
      "activations/layer16_attention_weight_max": 37.72673034667969,
      "activations/layer16_attention_weight_min": -29.637022018432617,
      "activations/layer17_attention_weight_max": 64.69327545166016,
      "activations/layer17_attention_weight_min": -49.55667495727539,
      "activations/layer18_attention_weight_max": 56.99806594848633,
      "activations/layer18_attention_weight_min": -39.6702995300293,
      "activations/layer19_attention_weight_max": 25.428552627563477,
      "activations/layer19_attention_weight_min": -22.654315948486328,
      "activations/layer1_attention_weight_max": 16.58388328552246,
      "activations/layer1_attention_weight_min": -16.765710830688477,
      "activations/layer20_attention_weight_max": 30.05440902709961,
      "activations/layer20_attention_weight_min": -21.8792667388916,
      "activations/layer21_attention_weight_max": 54.233726501464844,
      "activations/layer21_attention_weight_min": -24.425962448120117,
      "activations/layer22_attention_weight_max": 38.621517181396484,
      "activations/layer22_attention_weight_min": -25.7760009765625,
      "activations/layer23_attention_weight_max": 47.4698600769043,
      "activations/layer23_attention_weight_min": -21.340238571166992,
      "activations/layer2_attention_weight_max": 32.47770690917969,
      "activations/layer2_attention_weight_min": -31.680652618408203,
      "activations/layer3_attention_weight_max": 86.88697052001953,
      "activations/layer3_attention_weight_min": -88.59310150146484,
      "activations/layer4_attention_weight_max": 100.58304595947266,
      "activations/layer4_attention_weight_min": -93.93701934814453,
      "activations/layer5_attention_weight_max": 83.94719696044922,
      "activations/layer5_attention_weight_min": -79.24073791503906,
      "activations/layer6_attention_weight_max": 54.100181579589844,
      "activations/layer6_attention_weight_min": -51.75595474243164,
      "activations/layer7_attention_weight_max": 78.34474182128906,
      "activations/layer7_attention_weight_min": -71.96978759765625,
      "activations/layer8_attention_weight_max": 42.17012023925781,
      "activations/layer8_attention_weight_min": -46.465721130371094,
      "activations/layer9_attention_weight_max": 55.23310470581055,
      "activations/layer9_attention_weight_min": -50.251869201660156,
      "epoch": 7.99,
      "learning_rate": 9.947916666666665e-05,
      "loss": 2.8063,
      "step": 137500
    },
    {
      "activations/layer0_attention_weight_max": 15.230534553527832,
      "activations/layer0_attention_weight_min": -14.438403129577637,
      "activations/layer10_attention_weight_max": 48.69596481323242,
      "activations/layer10_attention_weight_min": -42.63541793823242,
      "activations/layer11_attention_weight_max": 41.61286926269531,
      "activations/layer11_attention_weight_min": -36.38201904296875,
      "activations/layer12_attention_weight_max": 18.939584732055664,
      "activations/layer12_attention_weight_min": -23.803272247314453,
      "activations/layer13_attention_weight_max": 74.6995620727539,
      "activations/layer13_attention_weight_min": -38.798500061035156,
      "activations/layer14_attention_weight_max": 76.18865966796875,
      "activations/layer14_attention_weight_min": -36.518043518066406,
      "activations/layer15_attention_weight_max": 64.04557037353516,
      "activations/layer15_attention_weight_min": -31.024049758911133,
      "activations/layer16_attention_weight_max": 37.90153121948242,
      "activations/layer16_attention_weight_min": -28.32297706604004,
      "activations/layer17_attention_weight_max": 56.1169548034668,
      "activations/layer17_attention_weight_min": -45.49416732788086,
      "activations/layer18_attention_weight_max": 53.39220428466797,
      "activations/layer18_attention_weight_min": -38.22077178955078,
      "activations/layer19_attention_weight_max": 24.180919647216797,
      "activations/layer19_attention_weight_min": -23.105674743652344,
      "activations/layer1_attention_weight_max": 17.7044620513916,
      "activations/layer1_attention_weight_min": -17.16803550720215,
      "activations/layer20_attention_weight_max": 26.030864715576172,
      "activations/layer20_attention_weight_min": -22.299545288085938,
      "activations/layer21_attention_weight_max": 43.23240280151367,
      "activations/layer21_attention_weight_min": -24.081424713134766,
      "activations/layer22_attention_weight_max": 33.682899475097656,
      "activations/layer22_attention_weight_min": -26.85712242126465,
      "activations/layer23_attention_weight_max": 38.77482604980469,
      "activations/layer23_attention_weight_min": -21.884765625,
      "activations/layer2_attention_weight_max": 33.61053466796875,
      "activations/layer2_attention_weight_min": -32.94743347167969,
      "activations/layer3_attention_weight_max": 89.83831787109375,
      "activations/layer3_attention_weight_min": -88.83570098876953,
      "activations/layer4_attention_weight_max": 103.4364242553711,
      "activations/layer4_attention_weight_min": -98.25984191894531,
      "activations/layer5_attention_weight_max": 87.03868865966797,
      "activations/layer5_attention_weight_min": -83.052734375,
      "activations/layer6_attention_weight_max": 55.62055969238281,
      "activations/layer6_attention_weight_min": -55.554019927978516,
      "activations/layer7_attention_weight_max": 91.91101837158203,
      "activations/layer7_attention_weight_min": -80.59551239013672,
      "activations/layer8_attention_weight_max": 56.86960220336914,
      "activations/layer8_attention_weight_min": -51.90364074707031,
      "activations/layer9_attention_weight_max": 77.09272766113281,
      "activations/layer9_attention_weight_min": -57.14364242553711,
      "epoch": 7.99,
      "learning_rate": 9.946022727272726e-05,
      "loss": 2.8014,
      "step": 137550
    },
    {
      "activations/layer0_attention_weight_max": 15.082160949707031,
      "activations/layer0_attention_weight_min": -14.522967338562012,
      "activations/layer10_attention_weight_max": 35.741371154785156,
      "activations/layer10_attention_weight_min": -34.516273498535156,
      "activations/layer11_attention_weight_max": 32.2793083190918,
      "activations/layer11_attention_weight_min": -29.994672775268555,
      "activations/layer12_attention_weight_max": 17.806900024414062,
      "activations/layer12_attention_weight_min": -25.527976989746094,
      "activations/layer13_attention_weight_max": 38.010154724121094,
      "activations/layer13_attention_weight_min": -28.443208694458008,
      "activations/layer14_attention_weight_max": 34.70698928833008,
      "activations/layer14_attention_weight_min": -31.24172592163086,
      "activations/layer15_attention_weight_max": 31.496057510375977,
      "activations/layer15_attention_weight_min": -31.851755142211914,
      "activations/layer16_attention_weight_max": 29.599685668945312,
      "activations/layer16_attention_weight_min": -29.458879470825195,
      "activations/layer17_attention_weight_max": 53.11060333251953,
      "activations/layer17_attention_weight_min": -41.6611213684082,
      "activations/layer18_attention_weight_max": 43.49679946899414,
      "activations/layer18_attention_weight_min": -36.6146240234375,
      "activations/layer19_attention_weight_max": 21.5622615814209,
      "activations/layer19_attention_weight_min": -21.47004508972168,
      "activations/layer1_attention_weight_max": 17.006086349487305,
      "activations/layer1_attention_weight_min": -15.216967582702637,
      "activations/layer20_attention_weight_max": 20.01100730895996,
      "activations/layer20_attention_weight_min": -24.32298469543457,
      "activations/layer21_attention_weight_max": 38.2958869934082,
      "activations/layer21_attention_weight_min": -22.502521514892578,
      "activations/layer22_attention_weight_max": 32.93317794799805,
      "activations/layer22_attention_weight_min": -25.735355377197266,
      "activations/layer23_attention_weight_max": 34.287017822265625,
      "activations/layer23_attention_weight_min": -21.341567993164062,
      "activations/layer2_attention_weight_max": 33.158103942871094,
      "activations/layer2_attention_weight_min": -31.81922149658203,
      "activations/layer3_attention_weight_max": 93.18402862548828,
      "activations/layer3_attention_weight_min": -93.86386108398438,
      "activations/layer4_attention_weight_max": 103.24727630615234,
      "activations/layer4_attention_weight_min": -98.0163803100586,
      "activations/layer5_attention_weight_max": 87.08784484863281,
      "activations/layer5_attention_weight_min": -78.90921783447266,
      "activations/layer6_attention_weight_max": 56.549827575683594,
      "activations/layer6_attention_weight_min": -50.27064514160156,
      "activations/layer7_attention_weight_max": 77.65931701660156,
      "activations/layer7_attention_weight_min": -68.55406188964844,
      "activations/layer8_attention_weight_max": 42.29582595825195,
      "activations/layer8_attention_weight_min": -44.392303466796875,
      "activations/layer9_attention_weight_max": 54.80485916137695,
      "activations/layer9_attention_weight_min": -49.30312728881836,
      "epoch": 8.0,
      "learning_rate": 9.944128787878786e-05,
      "loss": 2.8292,
      "step": 137600
    },
    {
      "activations/layer0_attention_weight_max": 14.411206245422363,
      "activations/layer0_attention_weight_min": -14.246432304382324,
      "activations/layer10_attention_weight_max": 36.194541931152344,
      "activations/layer10_attention_weight_min": -35.521148681640625,
      "activations/layer11_attention_weight_max": 29.13300323486328,
      "activations/layer11_attention_weight_min": -30.678009033203125,
      "activations/layer12_attention_weight_max": 17.691238403320312,
      "activations/layer12_attention_weight_min": -24.429758071899414,
      "activations/layer13_attention_weight_max": 38.086769104003906,
      "activations/layer13_attention_weight_min": -29.699302673339844,
      "activations/layer14_attention_weight_max": 40.96815490722656,
      "activations/layer14_attention_weight_min": -29.931116104125977,
      "activations/layer15_attention_weight_max": 33.73491668701172,
      "activations/layer15_attention_weight_min": -29.99700927734375,
      "activations/layer16_attention_weight_max": 32.71502685546875,
      "activations/layer16_attention_weight_min": -28.4726505279541,
      "activations/layer17_attention_weight_max": 53.07749557495117,
      "activations/layer17_attention_weight_min": -43.45186996459961,
      "activations/layer18_attention_weight_max": 45.5072135925293,
      "activations/layer18_attention_weight_min": -37.61775207519531,
      "activations/layer19_attention_weight_max": 22.602291107177734,
      "activations/layer19_attention_weight_min": -20.93321990966797,
      "activations/layer1_attention_weight_max": 17.08783721923828,
      "activations/layer1_attention_weight_min": -17.05805015563965,
      "activations/layer20_attention_weight_max": 21.100969314575195,
      "activations/layer20_attention_weight_min": -22.659658432006836,
      "activations/layer21_attention_weight_max": 39.8575325012207,
      "activations/layer21_attention_weight_min": -25.272045135498047,
      "activations/layer22_attention_weight_max": 29.99170684814453,
      "activations/layer22_attention_weight_min": -24.210054397583008,
      "activations/layer23_attention_weight_max": 34.008304595947266,
      "activations/layer23_attention_weight_min": -19.003246307373047,
      "activations/layer2_attention_weight_max": 31.52932357788086,
      "activations/layer2_attention_weight_min": -30.061664581298828,
      "activations/layer3_attention_weight_max": 89.55391693115234,
      "activations/layer3_attention_weight_min": -92.09883880615234,
      "activations/layer4_attention_weight_max": 104.892333984375,
      "activations/layer4_attention_weight_min": -103.9805679321289,
      "activations/layer5_attention_weight_max": 85.87164306640625,
      "activations/layer5_attention_weight_min": -83.45082092285156,
      "activations/layer6_attention_weight_max": 57.58351516723633,
      "activations/layer6_attention_weight_min": -55.84028244018555,
      "activations/layer7_attention_weight_max": 78.95299530029297,
      "activations/layer7_attention_weight_min": -72.6858901977539,
      "activations/layer8_attention_weight_max": 45.467979431152344,
      "activations/layer8_attention_weight_min": -46.95824432373047,
      "activations/layer9_attention_weight_max": 49.741580963134766,
      "activations/layer9_attention_weight_min": -48.58492660522461,
      "epoch": 8.0,
      "learning_rate": 9.942234848484848e-05,
      "loss": 2.8007,
      "step": 137650
    },
    {
      "activations/layer0_attention_weight_max": 14.665323257446289,
      "activations/layer0_attention_weight_min": -14.957405090332031,
      "activations/layer10_attention_weight_max": 38.19684982299805,
      "activations/layer10_attention_weight_min": -35.75592041015625,
      "activations/layer11_attention_weight_max": 33.097049713134766,
      "activations/layer11_attention_weight_min": -30.590810775756836,
      "activations/layer12_attention_weight_max": 18.385099411010742,
      "activations/layer12_attention_weight_min": -26.60910987854004,
      "activations/layer13_attention_weight_max": 33.370235443115234,
      "activations/layer13_attention_weight_min": -27.638757705688477,
      "activations/layer14_attention_weight_max": 34.70309066772461,
      "activations/layer14_attention_weight_min": -30.30984878540039,
      "activations/layer15_attention_weight_max": 29.652801513671875,
      "activations/layer15_attention_weight_min": -28.65222930908203,
      "activations/layer16_attention_weight_max": 29.316953659057617,
      "activations/layer16_attention_weight_min": -28.30830192565918,
      "activations/layer17_attention_weight_max": 50.54996871948242,
      "activations/layer17_attention_weight_min": -43.29132080078125,
      "activations/layer18_attention_weight_max": 42.67000198364258,
      "activations/layer18_attention_weight_min": -39.26939010620117,
      "activations/layer19_attention_weight_max": 20.66071128845215,
      "activations/layer19_attention_weight_min": -21.01973533630371,
      "activations/layer1_attention_weight_max": 16.44651985168457,
      "activations/layer1_attention_weight_min": -14.560131072998047,
      "activations/layer20_attention_weight_max": 21.41854476928711,
      "activations/layer20_attention_weight_min": -21.7374210357666,
      "activations/layer21_attention_weight_max": 35.498146057128906,
      "activations/layer21_attention_weight_min": -26.91881561279297,
      "activations/layer22_attention_weight_max": 30.35490608215332,
      "activations/layer22_attention_weight_min": -26.139238357543945,
      "activations/layer23_attention_weight_max": 34.008792877197266,
      "activations/layer23_attention_weight_min": -21.563621520996094,
      "activations/layer2_attention_weight_max": 32.16415023803711,
      "activations/layer2_attention_weight_min": -29.769298553466797,
      "activations/layer3_attention_weight_max": 89.18578338623047,
      "activations/layer3_attention_weight_min": -91.767822265625,
      "activations/layer4_attention_weight_max": 101.70134735107422,
      "activations/layer4_attention_weight_min": -95.25729370117188,
      "activations/layer5_attention_weight_max": 82.52590942382812,
      "activations/layer5_attention_weight_min": -77.64842987060547,
      "activations/layer6_attention_weight_max": 52.780128479003906,
      "activations/layer6_attention_weight_min": -52.698726654052734,
      "activations/layer7_attention_weight_max": 74.71964263916016,
      "activations/layer7_attention_weight_min": -73.24172973632812,
      "activations/layer8_attention_weight_max": 43.32352828979492,
      "activations/layer8_attention_weight_min": -46.25345230102539,
      "activations/layer9_attention_weight_max": 48.60661697387695,
      "activations/layer9_attention_weight_min": -48.92292404174805,
      "epoch": 8.0,
      "learning_rate": 9.940303030303029e-05,
      "loss": 2.8601,
      "step": 137700
    },
    {
      "activations/layer0_attention_weight_max": 14.493273735046387,
      "activations/layer0_attention_weight_min": -15.322681427001953,
      "activations/layer10_attention_weight_max": 40.742828369140625,
      "activations/layer10_attention_weight_min": -37.00642395019531,
      "activations/layer11_attention_weight_max": 32.37779235839844,
      "activations/layer11_attention_weight_min": -31.55877685546875,
      "activations/layer12_attention_weight_max": 18.979171752929688,
      "activations/layer12_attention_weight_min": -27.765474319458008,
      "activations/layer13_attention_weight_max": 40.87483596801758,
      "activations/layer13_attention_weight_min": -34.05051040649414,
      "activations/layer14_attention_weight_max": 38.34996795654297,
      "activations/layer14_attention_weight_min": -32.42522048950195,
      "activations/layer15_attention_weight_max": 34.336978912353516,
      "activations/layer15_attention_weight_min": -32.084869384765625,
      "activations/layer16_attention_weight_max": 31.739896774291992,
      "activations/layer16_attention_weight_min": -27.88542366027832,
      "activations/layer17_attention_weight_max": 51.13479232788086,
      "activations/layer17_attention_weight_min": -41.251888275146484,
      "activations/layer18_attention_weight_max": 44.30886459350586,
      "activations/layer18_attention_weight_min": -36.0643424987793,
      "activations/layer19_attention_weight_max": 22.778242111206055,
      "activations/layer19_attention_weight_min": -20.81717300415039,
      "activations/layer1_attention_weight_max": 16.402193069458008,
      "activations/layer1_attention_weight_min": -15.971563339233398,
      "activations/layer20_attention_weight_max": 20.27338218688965,
      "activations/layer20_attention_weight_min": -21.53289031982422,
      "activations/layer21_attention_weight_max": 37.04302978515625,
      "activations/layer21_attention_weight_min": -25.547632217407227,
      "activations/layer22_attention_weight_max": 30.835391998291016,
      "activations/layer22_attention_weight_min": -25.67171859741211,
      "activations/layer23_attention_weight_max": 39.4806022644043,
      "activations/layer23_attention_weight_min": -23.623493194580078,
      "activations/layer2_attention_weight_max": 33.363502502441406,
      "activations/layer2_attention_weight_min": -30.547163009643555,
      "activations/layer3_attention_weight_max": 93.7015380859375,
      "activations/layer3_attention_weight_min": -90.39665222167969,
      "activations/layer4_attention_weight_max": 105.91548919677734,
      "activations/layer4_attention_weight_min": -97.28206634521484,
      "activations/layer5_attention_weight_max": 89.9277572631836,
      "activations/layer5_attention_weight_min": -77.9123764038086,
      "activations/layer6_attention_weight_max": 55.76871871948242,
      "activations/layer6_attention_weight_min": -52.42498016357422,
      "activations/layer7_attention_weight_max": 77.69209289550781,
      "activations/layer7_attention_weight_min": -74.56360626220703,
      "activations/layer8_attention_weight_max": 48.16401290893555,
      "activations/layer8_attention_weight_min": -45.6362419128418,
      "activations/layer9_attention_weight_max": 53.182411193847656,
      "activations/layer9_attention_weight_min": -48.85013961791992,
      "epoch": 8.0,
      "learning_rate": 9.93840909090909e-05,
      "loss": 2.809,
      "step": 137750
    },
    {
      "activations/layer0_attention_weight_max": 15.567821502685547,
      "activations/layer0_attention_weight_min": -14.867325782775879,
      "activations/layer10_attention_weight_max": 34.95979309082031,
      "activations/layer10_attention_weight_min": -36.17534255981445,
      "activations/layer11_attention_weight_max": 32.06718063354492,
      "activations/layer11_attention_weight_min": -32.879615783691406,
      "activations/layer12_attention_weight_max": 21.609867095947266,
      "activations/layer12_attention_weight_min": -25.085344314575195,
      "activations/layer13_attention_weight_max": 44.82646560668945,
      "activations/layer13_attention_weight_min": -33.20417785644531,
      "activations/layer14_attention_weight_max": 49.2170524597168,
      "activations/layer14_attention_weight_min": -31.23118782043457,
      "activations/layer15_attention_weight_max": 36.931034088134766,
      "activations/layer15_attention_weight_min": -29.709857940673828,
      "activations/layer16_attention_weight_max": 37.49374771118164,
      "activations/layer16_attention_weight_min": -28.332542419433594,
      "activations/layer17_attention_weight_max": 56.8790283203125,
      "activations/layer17_attention_weight_min": -45.57217025756836,
      "activations/layer18_attention_weight_max": 55.20734786987305,
      "activations/layer18_attention_weight_min": -39.190460205078125,
      "activations/layer19_attention_weight_max": 26.761524200439453,
      "activations/layer19_attention_weight_min": -21.513200759887695,
      "activations/layer1_attention_weight_max": 16.878488540649414,
      "activations/layer1_attention_weight_min": -14.898059844970703,
      "activations/layer20_attention_weight_max": 25.256502151489258,
      "activations/layer20_attention_weight_min": -20.123071670532227,
      "activations/layer21_attention_weight_max": 42.483211517333984,
      "activations/layer21_attention_weight_min": -24.94795799255371,
      "activations/layer22_attention_weight_max": 36.299442291259766,
      "activations/layer22_attention_weight_min": -25.2673397064209,
      "activations/layer23_attention_weight_max": 39.083412170410156,
      "activations/layer23_attention_weight_min": -26.521671295166016,
      "activations/layer2_attention_weight_max": 34.655574798583984,
      "activations/layer2_attention_weight_min": -30.884796142578125,
      "activations/layer3_attention_weight_max": 90.65877532958984,
      "activations/layer3_attention_weight_min": -90.17571258544922,
      "activations/layer4_attention_weight_max": 104.95720672607422,
      "activations/layer4_attention_weight_min": -100.24557495117188,
      "activations/layer5_attention_weight_max": 85.44450378417969,
      "activations/layer5_attention_weight_min": -83.87260437011719,
      "activations/layer6_attention_weight_max": 54.50288391113281,
      "activations/layer6_attention_weight_min": -54.72219467163086,
      "activations/layer7_attention_weight_max": 76.9077377319336,
      "activations/layer7_attention_weight_min": -75.27715301513672,
      "activations/layer8_attention_weight_max": 45.03495407104492,
      "activations/layer8_attention_weight_min": -46.0079460144043,
      "activations/layer9_attention_weight_max": 59.436683654785156,
      "activations/layer9_attention_weight_min": -50.029205322265625,
      "epoch": 8.01,
      "learning_rate": 9.936515151515151e-05,
      "loss": 2.8023,
      "step": 137800
    },
    {
      "activations/layer0_attention_weight_max": 16.30590057373047,
      "activations/layer0_attention_weight_min": -15.215620040893555,
      "activations/layer10_attention_weight_max": 38.96565628051758,
      "activations/layer10_attention_weight_min": -38.4460334777832,
      "activations/layer11_attention_weight_max": 34.21015930175781,
      "activations/layer11_attention_weight_min": -32.26511001586914,
      "activations/layer12_attention_weight_max": 21.055673599243164,
      "activations/layer12_attention_weight_min": -24.715452194213867,
      "activations/layer13_attention_weight_max": 48.461204528808594,
      "activations/layer13_attention_weight_min": -31.400150299072266,
      "activations/layer14_attention_weight_max": 42.18653869628906,
      "activations/layer14_attention_weight_min": -30.070791244506836,
      "activations/layer15_attention_weight_max": 37.24706268310547,
      "activations/layer15_attention_weight_min": -28.361988067626953,
      "activations/layer16_attention_weight_max": 30.03575325012207,
      "activations/layer16_attention_weight_min": -28.305461883544922,
      "activations/layer17_attention_weight_max": 58.38022994995117,
      "activations/layer17_attention_weight_min": -42.187992095947266,
      "activations/layer18_attention_weight_max": 51.68239974975586,
      "activations/layer18_attention_weight_min": -35.019561767578125,
      "activations/layer19_attention_weight_max": 20.633411407470703,
      "activations/layer19_attention_weight_min": -23.309938430786133,
      "activations/layer1_attention_weight_max": 18.460033416748047,
      "activations/layer1_attention_weight_min": -15.519257545471191,
      "activations/layer20_attention_weight_max": 22.8493709564209,
      "activations/layer20_attention_weight_min": -21.293407440185547,
      "activations/layer21_attention_weight_max": 38.603668212890625,
      "activations/layer21_attention_weight_min": -24.57823944091797,
      "activations/layer22_attention_weight_max": 31.389619827270508,
      "activations/layer22_attention_weight_min": -25.460704803466797,
      "activations/layer23_attention_weight_max": 33.7567138671875,
      "activations/layer23_attention_weight_min": -21.490192413330078,
      "activations/layer2_attention_weight_max": 33.48529052734375,
      "activations/layer2_attention_weight_min": -31.010107040405273,
      "activations/layer3_attention_weight_max": 94.67340087890625,
      "activations/layer3_attention_weight_min": -95.2958755493164,
      "activations/layer4_attention_weight_max": 102.8532485961914,
      "activations/layer4_attention_weight_min": -97.3897933959961,
      "activations/layer5_attention_weight_max": 84.3035659790039,
      "activations/layer5_attention_weight_min": -78.5654525756836,
      "activations/layer6_attention_weight_max": 55.34364700317383,
      "activations/layer6_attention_weight_min": -53.90243148803711,
      "activations/layer7_attention_weight_max": 78.14684295654297,
      "activations/layer7_attention_weight_min": -80.90302276611328,
      "activations/layer8_attention_weight_max": 47.19374465942383,
      "activations/layer8_attention_weight_min": -49.02300262451172,
      "activations/layer9_attention_weight_max": 55.9388427734375,
      "activations/layer9_attention_weight_min": -61.3113899230957,
      "epoch": 8.01,
      "learning_rate": 9.934659090909091e-05,
      "loss": 2.8094,
      "step": 137850
    },
    {
      "activations/layer0_attention_weight_max": 15.92289924621582,
      "activations/layer0_attention_weight_min": -15.535798072814941,
      "activations/layer10_attention_weight_max": 38.947486877441406,
      "activations/layer10_attention_weight_min": -39.083072662353516,
      "activations/layer11_attention_weight_max": 32.941673278808594,
      "activations/layer11_attention_weight_min": -31.925846099853516,
      "activations/layer12_attention_weight_max": 18.70663833618164,
      "activations/layer12_attention_weight_min": -25.439489364624023,
      "activations/layer13_attention_weight_max": 46.59993362426758,
      "activations/layer13_attention_weight_min": -30.729503631591797,
      "activations/layer14_attention_weight_max": 39.723331451416016,
      "activations/layer14_attention_weight_min": -32.294918060302734,
      "activations/layer15_attention_weight_max": 37.82148742675781,
      "activations/layer15_attention_weight_min": -29.773420333862305,
      "activations/layer16_attention_weight_max": 33.402015686035156,
      "activations/layer16_attention_weight_min": -28.06629180908203,
      "activations/layer17_attention_weight_max": 56.55780029296875,
      "activations/layer17_attention_weight_min": -41.77522277832031,
      "activations/layer18_attention_weight_max": 49.14619827270508,
      "activations/layer18_attention_weight_min": -37.55232238769531,
      "activations/layer19_attention_weight_max": 28.712182998657227,
      "activations/layer19_attention_weight_min": -21.307268142700195,
      "activations/layer1_attention_weight_max": 17.186656951904297,
      "activations/layer1_attention_weight_min": -14.61461353302002,
      "activations/layer20_attention_weight_max": 24.695926666259766,
      "activations/layer20_attention_weight_min": -19.883098602294922,
      "activations/layer21_attention_weight_max": 40.163639068603516,
      "activations/layer21_attention_weight_min": -22.01190757751465,
      "activations/layer22_attention_weight_max": 36.17573547363281,
      "activations/layer22_attention_weight_min": -28.08976936340332,
      "activations/layer23_attention_weight_max": 33.49553298950195,
      "activations/layer23_attention_weight_min": -20.507320404052734,
      "activations/layer2_attention_weight_max": 33.85840606689453,
      "activations/layer2_attention_weight_min": -30.256160736083984,
      "activations/layer3_attention_weight_max": 90.70476531982422,
      "activations/layer3_attention_weight_min": -91.4616470336914,
      "activations/layer4_attention_weight_max": 110.67784881591797,
      "activations/layer4_attention_weight_min": -105.33287811279297,
      "activations/layer5_attention_weight_max": 91.36703491210938,
      "activations/layer5_attention_weight_min": -88.16789245605469,
      "activations/layer6_attention_weight_max": 60.34589385986328,
      "activations/layer6_attention_weight_min": -57.59265899658203,
      "activations/layer7_attention_weight_max": 86.83183288574219,
      "activations/layer7_attention_weight_min": -82.18154907226562,
      "activations/layer8_attention_weight_max": 46.7501220703125,
      "activations/layer8_attention_weight_min": -48.292022705078125,
      "activations/layer9_attention_weight_max": 56.125030517578125,
      "activations/layer9_attention_weight_min": -51.97007369995117,
      "epoch": 8.01,
      "learning_rate": 9.93280303030303e-05,
      "loss": 2.7962,
      "step": 137900
    },
    {
      "activations/layer0_attention_weight_max": 15.035219192504883,
      "activations/layer0_attention_weight_min": -15.180938720703125,
      "activations/layer10_attention_weight_max": 42.854530334472656,
      "activations/layer10_attention_weight_min": -37.77999496459961,
      "activations/layer11_attention_weight_max": 34.810546875,
      "activations/layer11_attention_weight_min": -32.525115966796875,
      "activations/layer12_attention_weight_max": 22.124984741210938,
      "activations/layer12_attention_weight_min": -25.71318244934082,
      "activations/layer13_attention_weight_max": 55.90839385986328,
      "activations/layer13_attention_weight_min": -31.10403060913086,
      "activations/layer14_attention_weight_max": 64.98260498046875,
      "activations/layer14_attention_weight_min": -33.838653564453125,
      "activations/layer15_attention_weight_max": 51.0250358581543,
      "activations/layer15_attention_weight_min": -30.672767639160156,
      "activations/layer16_attention_weight_max": 36.2325439453125,
      "activations/layer16_attention_weight_min": -29.383865356445312,
      "activations/layer17_attention_weight_max": 62.90270233154297,
      "activations/layer17_attention_weight_min": -47.451072692871094,
      "activations/layer18_attention_weight_max": 57.56266784667969,
      "activations/layer18_attention_weight_min": -37.72684860229492,
      "activations/layer19_attention_weight_max": 26.756345748901367,
      "activations/layer19_attention_weight_min": -22.250301361083984,
      "activations/layer1_attention_weight_max": 18.13269805908203,
      "activations/layer1_attention_weight_min": -14.657707214355469,
      "activations/layer20_attention_weight_max": 28.074493408203125,
      "activations/layer20_attention_weight_min": -20.930173873901367,
      "activations/layer21_attention_weight_max": 41.44282150268555,
      "activations/layer21_attention_weight_min": -23.617759704589844,
      "activations/layer22_attention_weight_max": 34.34638214111328,
      "activations/layer22_attention_weight_min": -24.713443756103516,
      "activations/layer23_attention_weight_max": 36.14299774169922,
      "activations/layer23_attention_weight_min": -21.368160247802734,
      "activations/layer2_attention_weight_max": 35.05297088623047,
      "activations/layer2_attention_weight_min": -31.08507537841797,
      "activations/layer3_attention_weight_max": 93.38687896728516,
      "activations/layer3_attention_weight_min": -95.80548095703125,
      "activations/layer4_attention_weight_max": 105.75911712646484,
      "activations/layer4_attention_weight_min": -102.6550064086914,
      "activations/layer5_attention_weight_max": 86.45750427246094,
      "activations/layer5_attention_weight_min": -82.27290344238281,
      "activations/layer6_attention_weight_max": 57.14330291748047,
      "activations/layer6_attention_weight_min": -56.23194885253906,
      "activations/layer7_attention_weight_max": 89.80855560302734,
      "activations/layer7_attention_weight_min": -78.1233139038086,
      "activations/layer8_attention_weight_max": 50.91584396362305,
      "activations/layer8_attention_weight_min": -47.50199890136719,
      "activations/layer9_attention_weight_max": 64.08851623535156,
      "activations/layer9_attention_weight_min": -53.32294464111328,
      "epoch": 8.02,
      "learning_rate": 9.930909090909089e-05,
      "loss": 2.8012,
      "step": 137950
    },
    {
      "activations/layer0_attention_weight_max": 15.03382396697998,
      "activations/layer0_attention_weight_min": -15.201332092285156,
      "activations/layer10_attention_weight_max": 35.72370147705078,
      "activations/layer10_attention_weight_min": -34.114845275878906,
      "activations/layer11_attention_weight_max": 30.176969528198242,
      "activations/layer11_attention_weight_min": -30.383275985717773,
      "activations/layer12_attention_weight_max": 19.294992446899414,
      "activations/layer12_attention_weight_min": -24.677732467651367,
      "activations/layer13_attention_weight_max": 39.251644134521484,
      "activations/layer13_attention_weight_min": -29.22064971923828,
      "activations/layer14_attention_weight_max": 35.377830505371094,
      "activations/layer14_attention_weight_min": -29.617164611816406,
      "activations/layer15_attention_weight_max": 34.09947204589844,
      "activations/layer15_attention_weight_min": -27.598567962646484,
      "activations/layer16_attention_weight_max": 30.859407424926758,
      "activations/layer16_attention_weight_min": -27.38503646850586,
      "activations/layer17_attention_weight_max": 51.47420120239258,
      "activations/layer17_attention_weight_min": -41.80690383911133,
      "activations/layer18_attention_weight_max": 47.72677230834961,
      "activations/layer18_attention_weight_min": -36.36986541748047,
      "activations/layer19_attention_weight_max": 20.714670181274414,
      "activations/layer19_attention_weight_min": -19.969636917114258,
      "activations/layer1_attention_weight_max": 16.648365020751953,
      "activations/layer1_attention_weight_min": -14.366451263427734,
      "activations/layer20_attention_weight_max": 20.892789840698242,
      "activations/layer20_attention_weight_min": -22.059791564941406,
      "activations/layer21_attention_weight_max": 34.77764129638672,
      "activations/layer21_attention_weight_min": -23.184011459350586,
      "activations/layer22_attention_weight_max": 28.65750503540039,
      "activations/layer22_attention_weight_min": -27.910053253173828,
      "activations/layer23_attention_weight_max": 32.84449005126953,
      "activations/layer23_attention_weight_min": -21.751272201538086,
      "activations/layer2_attention_weight_max": 31.569599151611328,
      "activations/layer2_attention_weight_min": -29.157339096069336,
      "activations/layer3_attention_weight_max": 85.08291625976562,
      "activations/layer3_attention_weight_min": -90.8583984375,
      "activations/layer4_attention_weight_max": 97.1598892211914,
      "activations/layer4_attention_weight_min": -91.92986297607422,
      "activations/layer5_attention_weight_max": 82.66547393798828,
      "activations/layer5_attention_weight_min": -76.67998504638672,
      "activations/layer6_attention_weight_max": 55.43273162841797,
      "activations/layer6_attention_weight_min": -52.13892364501953,
      "activations/layer7_attention_weight_max": 77.53231048583984,
      "activations/layer7_attention_weight_min": -70.73197174072266,
      "activations/layer8_attention_weight_max": 42.712337493896484,
      "activations/layer8_attention_weight_min": -45.612701416015625,
      "activations/layer9_attention_weight_max": 47.00502014160156,
      "activations/layer9_attention_weight_min": -47.626529693603516,
      "epoch": 8.02,
      "learning_rate": 9.92901515151515e-05,
      "loss": 2.8121,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5983,
      "eval_samples_per_second": 499.403,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5983,
      "eval_openwebtext_samples_per_second": 499.403,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_wikitext_loss": 2.998046875,
      "eval_wikitext_ppl": 20.046345644076645,
      "eval_wikitext_runtime": 2.0292,
      "eval_wikitext_samples_per_second": 224.719,
      "step": 138000
    },
    {
      "epoch": 8.02,
      "eval_lambada_loss": 2.720703125,
      "eval_lambada_ppl": 15.190999662372045,
      "eval_lambada_runtime": 9.6526,
      "eval_lambada_samples_per_second": 504.425,
      "step": 138000
    },
    {
      "activations/layer0_attention_weight_max": 15.731218338012695,
      "activations/layer0_attention_weight_min": -15.032254219055176,
      "activations/layer10_attention_weight_max": 39.04365158081055,
      "activations/layer10_attention_weight_min": -35.539371490478516,
      "activations/layer11_attention_weight_max": 34.34806823730469,
      "activations/layer11_attention_weight_min": -30.294790267944336,
      "activations/layer12_attention_weight_max": 22.642465591430664,
      "activations/layer12_attention_weight_min": -25.14094352722168,
      "activations/layer13_attention_weight_max": 56.11934280395508,
      "activations/layer13_attention_weight_min": -33.38618469238281,
      "activations/layer14_attention_weight_max": 50.8396110534668,
      "activations/layer14_attention_weight_min": -30.41477394104004,
      "activations/layer15_attention_weight_max": 40.32984161376953,
      "activations/layer15_attention_weight_min": -28.285293579101562,
      "activations/layer16_attention_weight_max": 31.886693954467773,
      "activations/layer16_attention_weight_min": -28.133974075317383,
      "activations/layer17_attention_weight_max": 53.299232482910156,
      "activations/layer17_attention_weight_min": -42.84934997558594,
      "activations/layer18_attention_weight_max": 45.60432815551758,
      "activations/layer18_attention_weight_min": -34.94196701049805,
      "activations/layer19_attention_weight_max": 20.876129150390625,
      "activations/layer19_attention_weight_min": -21.672266006469727,
      "activations/layer1_attention_weight_max": 16.56525993347168,
      "activations/layer1_attention_weight_min": -15.1939058303833,
      "activations/layer20_attention_weight_max": 23.42875862121582,
      "activations/layer20_attention_weight_min": -19.318849563598633,
      "activations/layer21_attention_weight_max": 37.1437873840332,
      "activations/layer21_attention_weight_min": -23.886266708374023,
      "activations/layer22_attention_weight_max": 34.60540771484375,
      "activations/layer22_attention_weight_min": -24.810546875,
      "activations/layer23_attention_weight_max": 36.743263244628906,
      "activations/layer23_attention_weight_min": -21.577003479003906,
      "activations/layer2_attention_weight_max": 32.04900360107422,
      "activations/layer2_attention_weight_min": -29.81401824951172,
      "activations/layer3_attention_weight_max": 84.37461853027344,
      "activations/layer3_attention_weight_min": -85.73628997802734,
      "activations/layer4_attention_weight_max": 104.02213287353516,
      "activations/layer4_attention_weight_min": -95.07442474365234,
      "activations/layer5_attention_weight_max": 88.82345581054688,
      "activations/layer5_attention_weight_min": -81.63900756835938,
      "activations/layer6_attention_weight_max": 56.858489990234375,
      "activations/layer6_attention_weight_min": -55.44260025024414,
      "activations/layer7_attention_weight_max": 75.20736694335938,
      "activations/layer7_attention_weight_min": -82.78955841064453,
      "activations/layer8_attention_weight_max": 45.587345123291016,
      "activations/layer8_attention_weight_min": -49.96210861206055,
      "activations/layer9_attention_weight_max": 52.074615478515625,
      "activations/layer9_attention_weight_min": -50.84505081176758,
      "epoch": 8.02,
      "learning_rate": 9.927121212121212e-05,
      "loss": 2.8149,
      "step": 138050
    },
    {
      "activations/layer0_attention_weight_max": 14.85332202911377,
      "activations/layer0_attention_weight_min": -14.197418212890625,
      "activations/layer10_attention_weight_max": 46.7701301574707,
      "activations/layer10_attention_weight_min": -43.162540435791016,
      "activations/layer11_attention_weight_max": 39.42354965209961,
      "activations/layer11_attention_weight_min": -34.69068908691406,
      "activations/layer12_attention_weight_max": 22.268314361572266,
      "activations/layer12_attention_weight_min": -27.868562698364258,
      "activations/layer13_attention_weight_max": 45.6944465637207,
      "activations/layer13_attention_weight_min": -31.18425178527832,
      "activations/layer14_attention_weight_max": 49.17459487915039,
      "activations/layer14_attention_weight_min": -33.437007904052734,
      "activations/layer15_attention_weight_max": 39.13894271850586,
      "activations/layer15_attention_weight_min": -29.0369930267334,
      "activations/layer16_attention_weight_max": 35.032684326171875,
      "activations/layer16_attention_weight_min": -28.621793746948242,
      "activations/layer17_attention_weight_max": 61.05300521850586,
      "activations/layer17_attention_weight_min": -44.23265838623047,
      "activations/layer18_attention_weight_max": 55.99387741088867,
      "activations/layer18_attention_weight_min": -36.2967529296875,
      "activations/layer19_attention_weight_max": 20.301128387451172,
      "activations/layer19_attention_weight_min": -21.594053268432617,
      "activations/layer1_attention_weight_max": 19.64748764038086,
      "activations/layer1_attention_weight_min": -17.083518981933594,
      "activations/layer20_attention_weight_max": 19.338571548461914,
      "activations/layer20_attention_weight_min": -22.326683044433594,
      "activations/layer21_attention_weight_max": 39.072227478027344,
      "activations/layer21_attention_weight_min": -23.55010414123535,
      "activations/layer22_attention_weight_max": 30.76464080810547,
      "activations/layer22_attention_weight_min": -25.13252067565918,
      "activations/layer23_attention_weight_max": 37.1062126159668,
      "activations/layer23_attention_weight_min": -21.530532836914062,
      "activations/layer2_attention_weight_max": 34.634361267089844,
      "activations/layer2_attention_weight_min": -32.28844451904297,
      "activations/layer3_attention_weight_max": 90.61872100830078,
      "activations/layer3_attention_weight_min": -91.37161254882812,
      "activations/layer4_attention_weight_max": 99.96443939208984,
      "activations/layer4_attention_weight_min": -96.39126586914062,
      "activations/layer5_attention_weight_max": 84.62513732910156,
      "activations/layer5_attention_weight_min": -81.87476348876953,
      "activations/layer6_attention_weight_max": 59.7098274230957,
      "activations/layer6_attention_weight_min": -55.48479461669922,
      "activations/layer7_attention_weight_max": 91.2010498046875,
      "activations/layer7_attention_weight_min": -81.10567474365234,
      "activations/layer8_attention_weight_max": 55.86311721801758,
      "activations/layer8_attention_weight_min": -52.420127868652344,
      "activations/layer9_attention_weight_max": 67.16370391845703,
      "activations/layer9_attention_weight_min": -54.826080322265625,
      "epoch": 8.02,
      "learning_rate": 9.925227272727271e-05,
      "loss": 2.7909,
      "step": 138100
    },
    {
      "activations/layer0_attention_weight_max": 16.465396881103516,
      "activations/layer0_attention_weight_min": -14.643900871276855,
      "activations/layer10_attention_weight_max": 43.031951904296875,
      "activations/layer10_attention_weight_min": -36.48191452026367,
      "activations/layer11_attention_weight_max": 32.4093017578125,
      "activations/layer11_attention_weight_min": -31.085533142089844,
      "activations/layer12_attention_weight_max": 23.762935638427734,
      "activations/layer12_attention_weight_min": -31.03425407409668,
      "activations/layer13_attention_weight_max": 66.10466766357422,
      "activations/layer13_attention_weight_min": -40.88294219970703,
      "activations/layer14_attention_weight_max": 54.07310104370117,
      "activations/layer14_attention_weight_min": -32.32341766357422,
      "activations/layer15_attention_weight_max": 47.0705451965332,
      "activations/layer15_attention_weight_min": -30.134658813476562,
      "activations/layer16_attention_weight_max": 38.55222702026367,
      "activations/layer16_attention_weight_min": -28.7895450592041,
      "activations/layer17_attention_weight_max": 57.78947830200195,
      "activations/layer17_attention_weight_min": -43.02669143676758,
      "activations/layer18_attention_weight_max": 51.31528854370117,
      "activations/layer18_attention_weight_min": -37.141483306884766,
      "activations/layer19_attention_weight_max": 25.5797176361084,
      "activations/layer19_attention_weight_min": -23.086210250854492,
      "activations/layer1_attention_weight_max": 17.992502212524414,
      "activations/layer1_attention_weight_min": -16.717449188232422,
      "activations/layer20_attention_weight_max": 26.872264862060547,
      "activations/layer20_attention_weight_min": -21.11191177368164,
      "activations/layer21_attention_weight_max": 45.64122009277344,
      "activations/layer21_attention_weight_min": -25.82199478149414,
      "activations/layer22_attention_weight_max": 33.74394989013672,
      "activations/layer22_attention_weight_min": -25.133102416992188,
      "activations/layer23_attention_weight_max": 38.72227096557617,
      "activations/layer23_attention_weight_min": -23.631866455078125,
      "activations/layer2_attention_weight_max": 31.38360595703125,
      "activations/layer2_attention_weight_min": -28.07454490661621,
      "activations/layer3_attention_weight_max": 85.48419952392578,
      "activations/layer3_attention_weight_min": -87.7335433959961,
      "activations/layer4_attention_weight_max": 103.05183410644531,
      "activations/layer4_attention_weight_min": -95.5678939819336,
      "activations/layer5_attention_weight_max": 86.54320526123047,
      "activations/layer5_attention_weight_min": -81.43309783935547,
      "activations/layer6_attention_weight_max": 56.68944549560547,
      "activations/layer6_attention_weight_min": -51.75202560424805,
      "activations/layer7_attention_weight_max": 74.6154556274414,
      "activations/layer7_attention_weight_min": -72.13700866699219,
      "activations/layer8_attention_weight_max": 44.607120513916016,
      "activations/layer8_attention_weight_min": -44.61066818237305,
      "activations/layer9_attention_weight_max": 49.5311279296875,
      "activations/layer9_attention_weight_min": -48.303951263427734,
      "epoch": 8.03,
      "learning_rate": 9.923333333333332e-05,
      "loss": 2.7998,
      "step": 138150
    },
    {
      "activations/layer0_attention_weight_max": 15.675480842590332,
      "activations/layer0_attention_weight_min": -14.587014198303223,
      "activations/layer10_attention_weight_max": 36.85545349121094,
      "activations/layer10_attention_weight_min": -36.64550018310547,
      "activations/layer11_attention_weight_max": 31.8288631439209,
      "activations/layer11_attention_weight_min": -31.226774215698242,
      "activations/layer12_attention_weight_max": 23.378759384155273,
      "activations/layer12_attention_weight_min": -25.075971603393555,
      "activations/layer13_attention_weight_max": 41.11658477783203,
      "activations/layer13_attention_weight_min": -30.951784133911133,
      "activations/layer14_attention_weight_max": 46.052001953125,
      "activations/layer14_attention_weight_min": -30.894899368286133,
      "activations/layer15_attention_weight_max": 37.530311584472656,
      "activations/layer15_attention_weight_min": -29.43352508544922,
      "activations/layer16_attention_weight_max": 35.69241714477539,
      "activations/layer16_attention_weight_min": -28.619789123535156,
      "activations/layer17_attention_weight_max": 52.58298110961914,
      "activations/layer17_attention_weight_min": -44.39244842529297,
      "activations/layer18_attention_weight_max": 52.26128387451172,
      "activations/layer18_attention_weight_min": -37.01268768310547,
      "activations/layer19_attention_weight_max": 22.540496826171875,
      "activations/layer19_attention_weight_min": -21.840810775756836,
      "activations/layer1_attention_weight_max": 18.63620376586914,
      "activations/layer1_attention_weight_min": -14.256418228149414,
      "activations/layer20_attention_weight_max": 25.34221839904785,
      "activations/layer20_attention_weight_min": -22.17096519470215,
      "activations/layer21_attention_weight_max": 44.82301712036133,
      "activations/layer21_attention_weight_min": -25.382488250732422,
      "activations/layer22_attention_weight_max": 31.933286666870117,
      "activations/layer22_attention_weight_min": -25.219831466674805,
      "activations/layer23_attention_weight_max": 36.53028869628906,
      "activations/layer23_attention_weight_min": -21.518346786499023,
      "activations/layer2_attention_weight_max": 31.310909271240234,
      "activations/layer2_attention_weight_min": -28.458049774169922,
      "activations/layer3_attention_weight_max": 89.36094665527344,
      "activations/layer3_attention_weight_min": -90.33592224121094,
      "activations/layer4_attention_weight_max": 102.65350341796875,
      "activations/layer4_attention_weight_min": -97.41602325439453,
      "activations/layer5_attention_weight_max": 86.37443542480469,
      "activations/layer5_attention_weight_min": -80.64083099365234,
      "activations/layer6_attention_weight_max": 53.65459060668945,
      "activations/layer6_attention_weight_min": -52.2160530090332,
      "activations/layer7_attention_weight_max": 73.68806457519531,
      "activations/layer7_attention_weight_min": -73.50798034667969,
      "activations/layer8_attention_weight_max": 44.03476333618164,
      "activations/layer8_attention_weight_min": -47.22908401489258,
      "activations/layer9_attention_weight_max": 51.82944869995117,
      "activations/layer9_attention_weight_min": -48.71110916137695,
      "epoch": 8.03,
      "learning_rate": 9.921439393939394e-05,
      "loss": 2.8077,
      "step": 138200
    },
    {
      "activations/layer0_attention_weight_max": 15.690404891967773,
      "activations/layer0_attention_weight_min": -15.061417579650879,
      "activations/layer10_attention_weight_max": 34.078346252441406,
      "activations/layer10_attention_weight_min": -34.2306022644043,
      "activations/layer11_attention_weight_max": 30.572158813476562,
      "activations/layer11_attention_weight_min": -29.378646850585938,
      "activations/layer12_attention_weight_max": 16.948619842529297,
      "activations/layer12_attention_weight_min": -24.564109802246094,
      "activations/layer13_attention_weight_max": 36.8953971862793,
      "activations/layer13_attention_weight_min": -30.22734832763672,
      "activations/layer14_attention_weight_max": 43.403934478759766,
      "activations/layer14_attention_weight_min": -31.640233993530273,
      "activations/layer15_attention_weight_max": 37.194210052490234,
      "activations/layer15_attention_weight_min": -28.169782638549805,
      "activations/layer16_attention_weight_max": 31.948184967041016,
      "activations/layer16_attention_weight_min": -28.018444061279297,
      "activations/layer17_attention_weight_max": 51.4617919921875,
      "activations/layer17_attention_weight_min": -42.12474060058594,
      "activations/layer18_attention_weight_max": 45.70793151855469,
      "activations/layer18_attention_weight_min": -35.44646072387695,
      "activations/layer19_attention_weight_max": 21.568464279174805,
      "activations/layer19_attention_weight_min": -22.11896324157715,
      "activations/layer1_attention_weight_max": 17.298410415649414,
      "activations/layer1_attention_weight_min": -15.134056091308594,
      "activations/layer20_attention_weight_max": 19.979692459106445,
      "activations/layer20_attention_weight_min": -22.025127410888672,
      "activations/layer21_attention_weight_max": 38.19791793823242,
      "activations/layer21_attention_weight_min": -24.530460357666016,
      "activations/layer22_attention_weight_max": 31.433368682861328,
      "activations/layer22_attention_weight_min": -26.604747772216797,
      "activations/layer23_attention_weight_max": 32.22099304199219,
      "activations/layer23_attention_weight_min": -22.78468132019043,
      "activations/layer2_attention_weight_max": 31.289627075195312,
      "activations/layer2_attention_weight_min": -27.568946838378906,
      "activations/layer3_attention_weight_max": 81.2901382446289,
      "activations/layer3_attention_weight_min": -80.91107177734375,
      "activations/layer4_attention_weight_max": 97.07183074951172,
      "activations/layer4_attention_weight_min": -94.44684600830078,
      "activations/layer5_attention_weight_max": 84.474365234375,
      "activations/layer5_attention_weight_min": -79.08308410644531,
      "activations/layer6_attention_weight_max": 55.38768768310547,
      "activations/layer6_attention_weight_min": -54.001182556152344,
      "activations/layer7_attention_weight_max": 75.23954010009766,
      "activations/layer7_attention_weight_min": -73.16813659667969,
      "activations/layer8_attention_weight_max": 44.1740608215332,
      "activations/layer8_attention_weight_min": -44.13581085205078,
      "activations/layer9_attention_weight_max": 48.56032943725586,
      "activations/layer9_attention_weight_min": -48.18094253540039,
      "epoch": 8.03,
      "learning_rate": 9.919545454545453e-05,
      "loss": 2.8246,
      "step": 138250
    },
    {
      "activations/layer0_attention_weight_max": 14.943682670593262,
      "activations/layer0_attention_weight_min": -15.422554016113281,
      "activations/layer10_attention_weight_max": 37.141109466552734,
      "activations/layer10_attention_weight_min": -35.5987548828125,
      "activations/layer11_attention_weight_max": 31.567779541015625,
      "activations/layer11_attention_weight_min": -29.995519638061523,
      "activations/layer12_attention_weight_max": 18.249221801757812,
      "activations/layer12_attention_weight_min": -25.068443298339844,
      "activations/layer13_attention_weight_max": 41.89790725708008,
      "activations/layer13_attention_weight_min": -30.353242874145508,
      "activations/layer14_attention_weight_max": 38.398284912109375,
      "activations/layer14_attention_weight_min": -33.908843994140625,
      "activations/layer15_attention_weight_max": 36.43606948852539,
      "activations/layer15_attention_weight_min": -30.281965255737305,
      "activations/layer16_attention_weight_max": 30.347896575927734,
      "activations/layer16_attention_weight_min": -27.688936233520508,
      "activations/layer17_attention_weight_max": 55.977813720703125,
      "activations/layer17_attention_weight_min": -44.1165771484375,
      "activations/layer18_attention_weight_max": 51.14371109008789,
      "activations/layer18_attention_weight_min": -35.9107780456543,
      "activations/layer19_attention_weight_max": 22.429555892944336,
      "activations/layer19_attention_weight_min": -21.956825256347656,
      "activations/layer1_attention_weight_max": 17.081127166748047,
      "activations/layer1_attention_weight_min": -13.43760871887207,
      "activations/layer20_attention_weight_max": 24.843753814697266,
      "activations/layer20_attention_weight_min": -21.366472244262695,
      "activations/layer21_attention_weight_max": 41.39688491821289,
      "activations/layer21_attention_weight_min": -24.264490127563477,
      "activations/layer22_attention_weight_max": 33.73396682739258,
      "activations/layer22_attention_weight_min": -25.519664764404297,
      "activations/layer23_attention_weight_max": 37.22605895996094,
      "activations/layer23_attention_weight_min": -23.634740829467773,
      "activations/layer2_attention_weight_max": 29.77791976928711,
      "activations/layer2_attention_weight_min": -27.80911636352539,
      "activations/layer3_attention_weight_max": 81.01224517822266,
      "activations/layer3_attention_weight_min": -84.6650161743164,
      "activations/layer4_attention_weight_max": 96.71673583984375,
      "activations/layer4_attention_weight_min": -95.15331268310547,
      "activations/layer5_attention_weight_max": 82.31897735595703,
      "activations/layer5_attention_weight_min": -79.10843658447266,
      "activations/layer6_attention_weight_max": 54.330013275146484,
      "activations/layer6_attention_weight_min": -51.667259216308594,
      "activations/layer7_attention_weight_max": 78.36015319824219,
      "activations/layer7_attention_weight_min": -72.77037811279297,
      "activations/layer8_attention_weight_max": 46.56978225708008,
      "activations/layer8_attention_weight_min": -47.94419479370117,
      "activations/layer9_attention_weight_max": 51.094173431396484,
      "activations/layer9_attention_weight_min": -48.33466720581055,
      "epoch": 8.04,
      "learning_rate": 9.917651515151514e-05,
      "loss": 2.7942,
      "step": 138300
    },
    {
      "activations/layer0_attention_weight_max": 15.255492210388184,
      "activations/layer0_attention_weight_min": -14.87045669555664,
      "activations/layer10_attention_weight_max": 38.82918167114258,
      "activations/layer10_attention_weight_min": -37.20528030395508,
      "activations/layer11_attention_weight_max": 30.931724548339844,
      "activations/layer11_attention_weight_min": -32.25675964355469,
      "activations/layer12_attention_weight_max": 19.48903465270996,
      "activations/layer12_attention_weight_min": -25.8533878326416,
      "activations/layer13_attention_weight_max": 41.963478088378906,
      "activations/layer13_attention_weight_min": -32.41242218017578,
      "activations/layer14_attention_weight_max": 39.09433364868164,
      "activations/layer14_attention_weight_min": -29.817453384399414,
      "activations/layer15_attention_weight_max": 36.04177474975586,
      "activations/layer15_attention_weight_min": -29.275447845458984,
      "activations/layer16_attention_weight_max": 32.99759292602539,
      "activations/layer16_attention_weight_min": -28.48992347717285,
      "activations/layer17_attention_weight_max": 56.507625579833984,
      "activations/layer17_attention_weight_min": -45.73828125,
      "activations/layer18_attention_weight_max": 57.24285125732422,
      "activations/layer18_attention_weight_min": -36.33201599121094,
      "activations/layer19_attention_weight_max": 22.02136993408203,
      "activations/layer19_attention_weight_min": -21.61203956604004,
      "activations/layer1_attention_weight_max": 16.16744613647461,
      "activations/layer1_attention_weight_min": -14.943324089050293,
      "activations/layer20_attention_weight_max": 23.127147674560547,
      "activations/layer20_attention_weight_min": -23.561389923095703,
      "activations/layer21_attention_weight_max": 35.84290313720703,
      "activations/layer21_attention_weight_min": -24.235424041748047,
      "activations/layer22_attention_weight_max": 32.72293472290039,
      "activations/layer22_attention_weight_min": -26.63828468322754,
      "activations/layer23_attention_weight_max": 36.293968200683594,
      "activations/layer23_attention_weight_min": -24.11402130126953,
      "activations/layer2_attention_weight_max": 31.736125946044922,
      "activations/layer2_attention_weight_min": -29.78015899658203,
      "activations/layer3_attention_weight_max": 83.03388214111328,
      "activations/layer3_attention_weight_min": -88.3841323852539,
      "activations/layer4_attention_weight_max": 99.24805450439453,
      "activations/layer4_attention_weight_min": -93.95234680175781,
      "activations/layer5_attention_weight_max": 83.58468627929688,
      "activations/layer5_attention_weight_min": -77.25765991210938,
      "activations/layer6_attention_weight_max": 55.32422637939453,
      "activations/layer6_attention_weight_min": -57.04216766357422,
      "activations/layer7_attention_weight_max": 78.66537475585938,
      "activations/layer7_attention_weight_min": -72.74152374267578,
      "activations/layer8_attention_weight_max": 43.415462493896484,
      "activations/layer8_attention_weight_min": -44.38026809692383,
      "activations/layer9_attention_weight_max": 52.3173713684082,
      "activations/layer9_attention_weight_min": -48.707515716552734,
      "epoch": 8.04,
      "learning_rate": 9.915757575757574e-05,
      "loss": 2.7962,
      "step": 138350
    },
    {
      "activations/layer0_attention_weight_max": 15.663383483886719,
      "activations/layer0_attention_weight_min": -14.652542114257812,
      "activations/layer10_attention_weight_max": 36.615020751953125,
      "activations/layer10_attention_weight_min": -34.646907806396484,
      "activations/layer11_attention_weight_max": 30.647090911865234,
      "activations/layer11_attention_weight_min": -31.038040161132812,
      "activations/layer12_attention_weight_max": 18.340923309326172,
      "activations/layer12_attention_weight_min": -23.709121704101562,
      "activations/layer13_attention_weight_max": 43.34669494628906,
      "activations/layer13_attention_weight_min": -39.21710968017578,
      "activations/layer14_attention_weight_max": 41.93736267089844,
      "activations/layer14_attention_weight_min": -30.337024688720703,
      "activations/layer15_attention_weight_max": 34.37519836425781,
      "activations/layer15_attention_weight_min": -30.58893585205078,
      "activations/layer16_attention_weight_max": 32.41453552246094,
      "activations/layer16_attention_weight_min": -26.647911071777344,
      "activations/layer17_attention_weight_max": 54.881412506103516,
      "activations/layer17_attention_weight_min": -44.43803787231445,
      "activations/layer18_attention_weight_max": 43.586822509765625,
      "activations/layer18_attention_weight_min": -37.69419860839844,
      "activations/layer19_attention_weight_max": 22.048490524291992,
      "activations/layer19_attention_weight_min": -24.53370475769043,
      "activations/layer1_attention_weight_max": 18.939220428466797,
      "activations/layer1_attention_weight_min": -17.349933624267578,
      "activations/layer20_attention_weight_max": 20.79121208190918,
      "activations/layer20_attention_weight_min": -20.404727935791016,
      "activations/layer21_attention_weight_max": 35.049041748046875,
      "activations/layer21_attention_weight_min": -22.929401397705078,
      "activations/layer22_attention_weight_max": 31.725915908813477,
      "activations/layer22_attention_weight_min": -24.71296501159668,
      "activations/layer23_attention_weight_max": 34.65472412109375,
      "activations/layer23_attention_weight_min": -24.177059173583984,
      "activations/layer2_attention_weight_max": 32.61482238769531,
      "activations/layer2_attention_weight_min": -29.841161727905273,
      "activations/layer3_attention_weight_max": 89.828125,
      "activations/layer3_attention_weight_min": -91.25586700439453,
      "activations/layer4_attention_weight_max": 102.77886962890625,
      "activations/layer4_attention_weight_min": -93.38677215576172,
      "activations/layer5_attention_weight_max": 85.50249481201172,
      "activations/layer5_attention_weight_min": -78.48869323730469,
      "activations/layer6_attention_weight_max": 55.39173126220703,
      "activations/layer6_attention_weight_min": -52.80888748168945,
      "activations/layer7_attention_weight_max": 75.44658660888672,
      "activations/layer7_attention_weight_min": -75.20410919189453,
      "activations/layer8_attention_weight_max": 45.294498443603516,
      "activations/layer8_attention_weight_min": -46.340030670166016,
      "activations/layer9_attention_weight_max": 48.16853332519531,
      "activations/layer9_attention_weight_min": -48.86865997314453,
      "epoch": 8.04,
      "learning_rate": 9.913863636363636e-05,
      "loss": 2.805,
      "step": 138400
    },
    {
      "activations/layer0_attention_weight_max": 15.78275203704834,
      "activations/layer0_attention_weight_min": -15.204718589782715,
      "activations/layer10_attention_weight_max": 40.069278717041016,
      "activations/layer10_attention_weight_min": -36.84172439575195,
      "activations/layer11_attention_weight_max": 32.29136657714844,
      "activations/layer11_attention_weight_min": -31.428213119506836,
      "activations/layer12_attention_weight_max": 18.900388717651367,
      "activations/layer12_attention_weight_min": -26.43815803527832,
      "activations/layer13_attention_weight_max": 47.103885650634766,
      "activations/layer13_attention_weight_min": -33.0747184753418,
      "activations/layer14_attention_weight_max": 52.09831237792969,
      "activations/layer14_attention_weight_min": -32.314170837402344,
      "activations/layer15_attention_weight_max": 40.07351303100586,
      "activations/layer15_attention_weight_min": -30.459550857543945,
      "activations/layer16_attention_weight_max": 34.50592041015625,
      "activations/layer16_attention_weight_min": -29.44022560119629,
      "activations/layer17_attention_weight_max": 61.69597244262695,
      "activations/layer17_attention_weight_min": -44.77113723754883,
      "activations/layer18_attention_weight_max": 55.00456237792969,
      "activations/layer18_attention_weight_min": -38.735862731933594,
      "activations/layer19_attention_weight_max": 24.10685920715332,
      "activations/layer19_attention_weight_min": -22.09890365600586,
      "activations/layer1_attention_weight_max": 16.962783813476562,
      "activations/layer1_attention_weight_min": -18.75992774963379,
      "activations/layer20_attention_weight_max": 28.637617111206055,
      "activations/layer20_attention_weight_min": -22.390047073364258,
      "activations/layer21_attention_weight_max": 55.57257080078125,
      "activations/layer21_attention_weight_min": -27.31822967529297,
      "activations/layer22_attention_weight_max": 34.8273811340332,
      "activations/layer22_attention_weight_min": -27.453922271728516,
      "activations/layer23_attention_weight_max": 41.09344482421875,
      "activations/layer23_attention_weight_min": -21.443706512451172,
      "activations/layer2_attention_weight_max": 33.32318115234375,
      "activations/layer2_attention_weight_min": -30.54523468017578,
      "activations/layer3_attention_weight_max": 88.05419921875,
      "activations/layer3_attention_weight_min": -89.10781860351562,
      "activations/layer4_attention_weight_max": 96.6443862915039,
      "activations/layer4_attention_weight_min": -96.37889099121094,
      "activations/layer5_attention_weight_max": 85.00579833984375,
      "activations/layer5_attention_weight_min": -79.18863677978516,
      "activations/layer6_attention_weight_max": 54.86625671386719,
      "activations/layer6_attention_weight_min": -55.266422271728516,
      "activations/layer7_attention_weight_max": 78.72776794433594,
      "activations/layer7_attention_weight_min": -76.78839111328125,
      "activations/layer8_attention_weight_max": 46.84248733520508,
      "activations/layer8_attention_weight_min": -46.309669494628906,
      "activations/layer9_attention_weight_max": 49.95418167114258,
      "activations/layer9_attention_weight_min": -50.79403305053711,
      "epoch": 8.04,
      "learning_rate": 9.911969696969696e-05,
      "loss": 2.8147,
      "step": 138450
    },
    {
      "activations/layer0_attention_weight_max": 15.688400268554688,
      "activations/layer0_attention_weight_min": -15.057604789733887,
      "activations/layer10_attention_weight_max": 40.16024398803711,
      "activations/layer10_attention_weight_min": -34.928558349609375,
      "activations/layer11_attention_weight_max": 33.83756637573242,
      "activations/layer11_attention_weight_min": -32.36909484863281,
      "activations/layer12_attention_weight_max": 22.8721923828125,
      "activations/layer12_attention_weight_min": -27.902841567993164,
      "activations/layer13_attention_weight_max": 46.192134857177734,
      "activations/layer13_attention_weight_min": -30.253053665161133,
      "activations/layer14_attention_weight_max": 52.69239044189453,
      "activations/layer14_attention_weight_min": -33.160457611083984,
      "activations/layer15_attention_weight_max": 49.00849533081055,
      "activations/layer15_attention_weight_min": -28.57541847229004,
      "activations/layer16_attention_weight_max": 37.34917449951172,
      "activations/layer16_attention_weight_min": -27.768220901489258,
      "activations/layer17_attention_weight_max": 64.75069427490234,
      "activations/layer17_attention_weight_min": -42.87538528442383,
      "activations/layer18_attention_weight_max": 58.085994720458984,
      "activations/layer18_attention_weight_min": -35.67223358154297,
      "activations/layer19_attention_weight_max": 27.152151107788086,
      "activations/layer19_attention_weight_min": -22.20937728881836,
      "activations/layer1_attention_weight_max": 17.441322326660156,
      "activations/layer1_attention_weight_min": -14.414236068725586,
      "activations/layer20_attention_weight_max": 27.790414810180664,
      "activations/layer20_attention_weight_min": -23.09783172607422,
      "activations/layer21_attention_weight_max": 58.953731536865234,
      "activations/layer21_attention_weight_min": -24.195903778076172,
      "activations/layer22_attention_weight_max": 40.37017059326172,
      "activations/layer22_attention_weight_min": -24.508813858032227,
      "activations/layer23_attention_weight_max": 42.456024169921875,
      "activations/layer23_attention_weight_min": -19.511558532714844,
      "activations/layer2_attention_weight_max": 31.44207000732422,
      "activations/layer2_attention_weight_min": -29.147438049316406,
      "activations/layer3_attention_weight_max": 90.00618743896484,
      "activations/layer3_attention_weight_min": -93.42411041259766,
      "activations/layer4_attention_weight_max": 102.95401763916016,
      "activations/layer4_attention_weight_min": -97.07499694824219,
      "activations/layer5_attention_weight_max": 86.24501037597656,
      "activations/layer5_attention_weight_min": -78.02595520019531,
      "activations/layer6_attention_weight_max": 56.1761589050293,
      "activations/layer6_attention_weight_min": -53.25086212158203,
      "activations/layer7_attention_weight_max": 79.55047607421875,
      "activations/layer7_attention_weight_min": -71.78993225097656,
      "activations/layer8_attention_weight_max": 45.7308349609375,
      "activations/layer8_attention_weight_min": -44.71485137939453,
      "activations/layer9_attention_weight_max": 63.604976654052734,
      "activations/layer9_attention_weight_min": -52.10471725463867,
      "epoch": 8.05,
      "learning_rate": 9.910075757575756e-05,
      "loss": 2.8105,
      "step": 138500
    },
    {
      "activations/layer0_attention_weight_max": 14.321914672851562,
      "activations/layer0_attention_weight_min": -14.959969520568848,
      "activations/layer10_attention_weight_max": 36.10494613647461,
      "activations/layer10_attention_weight_min": -36.0687370300293,
      "activations/layer11_attention_weight_max": 30.81182861328125,
      "activations/layer11_attention_weight_min": -29.961563110351562,
      "activations/layer12_attention_weight_max": 17.042728424072266,
      "activations/layer12_attention_weight_min": -26.51127815246582,
      "activations/layer13_attention_weight_max": 44.37910461425781,
      "activations/layer13_attention_weight_min": -32.129310607910156,
      "activations/layer14_attention_weight_max": 36.57903289794922,
      "activations/layer14_attention_weight_min": -28.812685012817383,
      "activations/layer15_attention_weight_max": 32.78048324584961,
      "activations/layer15_attention_weight_min": -28.75179672241211,
      "activations/layer16_attention_weight_max": 31.994611740112305,
      "activations/layer16_attention_weight_min": -29.579734802246094,
      "activations/layer17_attention_weight_max": 51.96792221069336,
      "activations/layer17_attention_weight_min": -42.219444274902344,
      "activations/layer18_attention_weight_max": 42.96025085449219,
      "activations/layer18_attention_weight_min": -34.956241607666016,
      "activations/layer19_attention_weight_max": 20.54427146911621,
      "activations/layer19_attention_weight_min": -20.649160385131836,
      "activations/layer1_attention_weight_max": 17.304981231689453,
      "activations/layer1_attention_weight_min": -15.678338050842285,
      "activations/layer20_attention_weight_max": 20.911724090576172,
      "activations/layer20_attention_weight_min": -22.709203720092773,
      "activations/layer21_attention_weight_max": 36.25291061401367,
      "activations/layer21_attention_weight_min": -23.522817611694336,
      "activations/layer22_attention_weight_max": 29.51844024658203,
      "activations/layer22_attention_weight_min": -27.227815628051758,
      "activations/layer23_attention_weight_max": 32.406124114990234,
      "activations/layer23_attention_weight_min": -23.217330932617188,
      "activations/layer2_attention_weight_max": 29.952009201049805,
      "activations/layer2_attention_weight_min": -28.458770751953125,
      "activations/layer3_attention_weight_max": 84.68431854248047,
      "activations/layer3_attention_weight_min": -85.86437225341797,
      "activations/layer4_attention_weight_max": 96.57508850097656,
      "activations/layer4_attention_weight_min": -91.07949829101562,
      "activations/layer5_attention_weight_max": 85.38652801513672,
      "activations/layer5_attention_weight_min": -78.23094177246094,
      "activations/layer6_attention_weight_max": 55.45358657836914,
      "activations/layer6_attention_weight_min": -50.87189483642578,
      "activations/layer7_attention_weight_max": 74.2973403930664,
      "activations/layer7_attention_weight_min": -70.89938354492188,
      "activations/layer8_attention_weight_max": 44.84142303466797,
      "activations/layer8_attention_weight_min": -43.862449645996094,
      "activations/layer9_attention_weight_max": 48.4117431640625,
      "activations/layer9_attention_weight_min": -48.640838623046875,
      "epoch": 8.05,
      "learning_rate": 9.908181818181818e-05,
      "loss": 2.8165,
      "step": 138550
    },
    {
      "activations/layer0_attention_weight_max": 15.433576583862305,
      "activations/layer0_attention_weight_min": -15.072151184082031,
      "activations/layer10_attention_weight_max": 35.51438903808594,
      "activations/layer10_attention_weight_min": -34.26171112060547,
      "activations/layer11_attention_weight_max": 30.174406051635742,
      "activations/layer11_attention_weight_min": -29.602188110351562,
      "activations/layer12_attention_weight_max": 16.989665985107422,
      "activations/layer12_attention_weight_min": -27.38545036315918,
      "activations/layer13_attention_weight_max": 41.22960662841797,
      "activations/layer13_attention_weight_min": -33.48143768310547,
      "activations/layer14_attention_weight_max": 35.76850891113281,
      "activations/layer14_attention_weight_min": -30.454418182373047,
      "activations/layer15_attention_weight_max": 32.3338623046875,
      "activations/layer15_attention_weight_min": -29.112869262695312,
      "activations/layer16_attention_weight_max": 29.722951889038086,
      "activations/layer16_attention_weight_min": -29.38214111328125,
      "activations/layer17_attention_weight_max": 49.890037536621094,
      "activations/layer17_attention_weight_min": -45.6972541809082,
      "activations/layer18_attention_weight_max": 43.2593879699707,
      "activations/layer18_attention_weight_min": -39.589271545410156,
      "activations/layer19_attention_weight_max": 19.094419479370117,
      "activations/layer19_attention_weight_min": -20.824087142944336,
      "activations/layer1_attention_weight_max": 17.953332901000977,
      "activations/layer1_attention_weight_min": -16.06748390197754,
      "activations/layer20_attention_weight_max": 19.396991729736328,
      "activations/layer20_attention_weight_min": -24.3681697845459,
      "activations/layer21_attention_weight_max": 33.21678161621094,
      "activations/layer21_attention_weight_min": -26.815624237060547,
      "activations/layer22_attention_weight_max": 29.846025466918945,
      "activations/layer22_attention_weight_min": -26.376495361328125,
      "activations/layer23_attention_weight_max": 33.8840446472168,
      "activations/layer23_attention_weight_min": -21.5844783782959,
      "activations/layer2_attention_weight_max": 31.79982566833496,
      "activations/layer2_attention_weight_min": -29.16120719909668,
      "activations/layer3_attention_weight_max": 84.59608459472656,
      "activations/layer3_attention_weight_min": -89.42045593261719,
      "activations/layer4_attention_weight_max": 95.85318756103516,
      "activations/layer4_attention_weight_min": -93.2922592163086,
      "activations/layer5_attention_weight_max": 80.71173095703125,
      "activations/layer5_attention_weight_min": -75.71536254882812,
      "activations/layer6_attention_weight_max": 52.93457794189453,
      "activations/layer6_attention_weight_min": -50.30599594116211,
      "activations/layer7_attention_weight_max": 73.1891098022461,
      "activations/layer7_attention_weight_min": -72.40998077392578,
      "activations/layer8_attention_weight_max": 42.18571472167969,
      "activations/layer8_attention_weight_min": -43.19209289550781,
      "activations/layer9_attention_weight_max": 47.88239669799805,
      "activations/layer9_attention_weight_min": -46.966468811035156,
      "epoch": 8.05,
      "learning_rate": 9.906287878787879e-05,
      "loss": 2.8031,
      "step": 138600
    },
    {
      "activations/layer0_attention_weight_max": 15.131606101989746,
      "activations/layer0_attention_weight_min": -14.991544723510742,
      "activations/layer10_attention_weight_max": 47.61780548095703,
      "activations/layer10_attention_weight_min": -45.507266998291016,
      "activations/layer11_attention_weight_max": 38.79544448852539,
      "activations/layer11_attention_weight_min": -34.41400909423828,
      "activations/layer12_attention_weight_max": 22.050474166870117,
      "activations/layer12_attention_weight_min": -29.77926254272461,
      "activations/layer13_attention_weight_max": 97.34663391113281,
      "activations/layer13_attention_weight_min": -45.26667404174805,
      "activations/layer14_attention_weight_max": 100.5497055053711,
      "activations/layer14_attention_weight_min": -43.270294189453125,
      "activations/layer15_attention_weight_max": 73.71369171142578,
      "activations/layer15_attention_weight_min": -33.83689498901367,
      "activations/layer16_attention_weight_max": 38.4559211730957,
      "activations/layer16_attention_weight_min": -29.016389846801758,
      "activations/layer17_attention_weight_max": 54.933868408203125,
      "activations/layer17_attention_weight_min": -45.24541091918945,
      "activations/layer18_attention_weight_max": 48.59504318237305,
      "activations/layer18_attention_weight_min": -38.404964447021484,
      "activations/layer19_attention_weight_max": 21.404951095581055,
      "activations/layer19_attention_weight_min": -22.486164093017578,
      "activations/layer1_attention_weight_max": 17.219745635986328,
      "activations/layer1_attention_weight_min": -14.490554809570312,
      "activations/layer20_attention_weight_max": 26.45884895324707,
      "activations/layer20_attention_weight_min": -22.774799346923828,
      "activations/layer21_attention_weight_max": 48.33643341064453,
      "activations/layer21_attention_weight_min": -24.797521591186523,
      "activations/layer22_attention_weight_max": 38.01926040649414,
      "activations/layer22_attention_weight_min": -24.007802963256836,
      "activations/layer23_attention_weight_max": 36.94322967529297,
      "activations/layer23_attention_weight_min": -21.77559471130371,
      "activations/layer2_attention_weight_max": 32.91315460205078,
      "activations/layer2_attention_weight_min": -29.171161651611328,
      "activations/layer3_attention_weight_max": 96.47515106201172,
      "activations/layer3_attention_weight_min": -94.1763687133789,
      "activations/layer4_attention_weight_max": 113.91107177734375,
      "activations/layer4_attention_weight_min": -101.3124008178711,
      "activations/layer5_attention_weight_max": 88.66780853271484,
      "activations/layer5_attention_weight_min": -81.37958526611328,
      "activations/layer6_attention_weight_max": 57.82365417480469,
      "activations/layer6_attention_weight_min": -54.61524200439453,
      "activations/layer7_attention_weight_max": 90.22013854980469,
      "activations/layer7_attention_weight_min": -81.74177551269531,
      "activations/layer8_attention_weight_max": 56.22284698486328,
      "activations/layer8_attention_weight_min": -52.03279113769531,
      "activations/layer9_attention_weight_max": 71.9878158569336,
      "activations/layer9_attention_weight_min": -64.6771469116211,
      "epoch": 8.06,
      "learning_rate": 9.904393939393938e-05,
      "loss": 2.801,
      "step": 138650
    },
    {
      "activations/layer0_attention_weight_max": 15.852540016174316,
      "activations/layer0_attention_weight_min": -15.074584007263184,
      "activations/layer10_attention_weight_max": 38.39665222167969,
      "activations/layer10_attention_weight_min": -37.75151824951172,
      "activations/layer11_attention_weight_max": 30.118534088134766,
      "activations/layer11_attention_weight_min": -31.20321273803711,
      "activations/layer12_attention_weight_max": 21.8237247467041,
      "activations/layer12_attention_weight_min": -26.961580276489258,
      "activations/layer13_attention_weight_max": 49.5793571472168,
      "activations/layer13_attention_weight_min": -33.6458625793457,
      "activations/layer14_attention_weight_max": 50.00068283081055,
      "activations/layer14_attention_weight_min": -32.14126205444336,
      "activations/layer15_attention_weight_max": 36.169803619384766,
      "activations/layer15_attention_weight_min": -30.08856773376465,
      "activations/layer16_attention_weight_max": 34.17420196533203,
      "activations/layer16_attention_weight_min": -28.991477966308594,
      "activations/layer17_attention_weight_max": 55.67812728881836,
      "activations/layer17_attention_weight_min": -45.27618408203125,
      "activations/layer18_attention_weight_max": 51.127017974853516,
      "activations/layer18_attention_weight_min": -37.9624137878418,
      "activations/layer19_attention_weight_max": 27.05812644958496,
      "activations/layer19_attention_weight_min": -24.534942626953125,
      "activations/layer1_attention_weight_max": 17.30656623840332,
      "activations/layer1_attention_weight_min": -14.552733421325684,
      "activations/layer20_attention_weight_max": 27.378948211669922,
      "activations/layer20_attention_weight_min": -22.97374725341797,
      "activations/layer21_attention_weight_max": 48.49856948852539,
      "activations/layer21_attention_weight_min": -25.135839462280273,
      "activations/layer22_attention_weight_max": 39.17051315307617,
      "activations/layer22_attention_weight_min": -27.333337783813477,
      "activations/layer23_attention_weight_max": 43.02248764038086,
      "activations/layer23_attention_weight_min": -22.72553253173828,
      "activations/layer2_attention_weight_max": 32.66908645629883,
      "activations/layer2_attention_weight_min": -30.18391227722168,
      "activations/layer3_attention_weight_max": 86.18231964111328,
      "activations/layer3_attention_weight_min": -88.911865234375,
      "activations/layer4_attention_weight_max": 97.24966430664062,
      "activations/layer4_attention_weight_min": -95.89146423339844,
      "activations/layer5_attention_weight_max": 85.06204223632812,
      "activations/layer5_attention_weight_min": -81.74341583251953,
      "activations/layer6_attention_weight_max": 54.66875076293945,
      "activations/layer6_attention_weight_min": -54.18755340576172,
      "activations/layer7_attention_weight_max": 83.97898864746094,
      "activations/layer7_attention_weight_min": -72.29735565185547,
      "activations/layer8_attention_weight_max": 45.92030334472656,
      "activations/layer8_attention_weight_min": -48.31563186645508,
      "activations/layer9_attention_weight_max": 54.65397262573242,
      "activations/layer9_attention_weight_min": -53.439796447753906,
      "epoch": 8.06,
      "learning_rate": 9.9025e-05,
      "loss": 2.8068,
      "step": 138700
    },
    {
      "activations/layer0_attention_weight_max": 15.741382598876953,
      "activations/layer0_attention_weight_min": -14.99727725982666,
      "activations/layer10_attention_weight_max": 37.37125778198242,
      "activations/layer10_attention_weight_min": -35.592262268066406,
      "activations/layer11_attention_weight_max": 31.573179244995117,
      "activations/layer11_attention_weight_min": -31.116697311401367,
      "activations/layer12_attention_weight_max": 22.993980407714844,
      "activations/layer12_attention_weight_min": -23.821578979492188,
      "activations/layer13_attention_weight_max": 38.7431755065918,
      "activations/layer13_attention_weight_min": -29.464981079101562,
      "activations/layer14_attention_weight_max": 36.27172088623047,
      "activations/layer14_attention_weight_min": -30.44326400756836,
      "activations/layer15_attention_weight_max": 36.340057373046875,
      "activations/layer15_attention_weight_min": -29.175491333007812,
      "activations/layer16_attention_weight_max": 31.641149520874023,
      "activations/layer16_attention_weight_min": -28.189977645874023,
      "activations/layer17_attention_weight_max": 53.56801223754883,
      "activations/layer17_attention_weight_min": -41.16138458251953,
      "activations/layer18_attention_weight_max": 48.35094451904297,
      "activations/layer18_attention_weight_min": -37.57843780517578,
      "activations/layer19_attention_weight_max": 22.03440284729004,
      "activations/layer19_attention_weight_min": -21.814167022705078,
      "activations/layer1_attention_weight_max": 17.005868911743164,
      "activations/layer1_attention_weight_min": -15.185912132263184,
      "activations/layer20_attention_weight_max": 22.80805015563965,
      "activations/layer20_attention_weight_min": -20.6341552734375,
      "activations/layer21_attention_weight_max": 37.03681945800781,
      "activations/layer21_attention_weight_min": -24.455312728881836,
      "activations/layer22_attention_weight_max": 32.56427764892578,
      "activations/layer22_attention_weight_min": -23.775415420532227,
      "activations/layer23_attention_weight_max": 34.24909973144531,
      "activations/layer23_attention_weight_min": -22.608108520507812,
      "activations/layer2_attention_weight_max": 31.6988582611084,
      "activations/layer2_attention_weight_min": -29.314876556396484,
      "activations/layer3_attention_weight_max": 89.5319595336914,
      "activations/layer3_attention_weight_min": -92.12528228759766,
      "activations/layer4_attention_weight_max": 104.87596893310547,
      "activations/layer4_attention_weight_min": -101.54595184326172,
      "activations/layer5_attention_weight_max": 84.6583023071289,
      "activations/layer5_attention_weight_min": -82.11605072021484,
      "activations/layer6_attention_weight_max": 56.7177619934082,
      "activations/layer6_attention_weight_min": -54.16173553466797,
      "activations/layer7_attention_weight_max": 79.5523681640625,
      "activations/layer7_attention_weight_min": -74.16934967041016,
      "activations/layer8_attention_weight_max": 45.265960693359375,
      "activations/layer8_attention_weight_min": -46.738773345947266,
      "activations/layer9_attention_weight_max": 54.68909454345703,
      "activations/layer9_attention_weight_min": -49.750301361083984,
      "epoch": 8.06,
      "learning_rate": 9.900606060606061e-05,
      "loss": 2.7892,
      "step": 138750
    },
    {
      "activations/layer0_attention_weight_max": 15.159132957458496,
      "activations/layer0_attention_weight_min": -15.217774391174316,
      "activations/layer10_attention_weight_max": 40.159027099609375,
      "activations/layer10_attention_weight_min": -38.53945541381836,
      "activations/layer11_attention_weight_max": 34.983463287353516,
      "activations/layer11_attention_weight_min": -32.04057312011719,
      "activations/layer12_attention_weight_max": 23.04311752319336,
      "activations/layer12_attention_weight_min": -26.119943618774414,
      "activations/layer13_attention_weight_max": 50.521324157714844,
      "activations/layer13_attention_weight_min": -36.48834991455078,
      "activations/layer14_attention_weight_max": 53.120662689208984,
      "activations/layer14_attention_weight_min": -35.57106399536133,
      "activations/layer15_attention_weight_max": 44.44275665283203,
      "activations/layer15_attention_weight_min": -29.514360427856445,
      "activations/layer16_attention_weight_max": 32.589744567871094,
      "activations/layer16_attention_weight_min": -28.544658660888672,
      "activations/layer17_attention_weight_max": 59.18702697753906,
      "activations/layer17_attention_weight_min": -44.48472213745117,
      "activations/layer18_attention_weight_max": 47.66745376586914,
      "activations/layer18_attention_weight_min": -37.903202056884766,
      "activations/layer19_attention_weight_max": 22.056171417236328,
      "activations/layer19_attention_weight_min": -23.41346549987793,
      "activations/layer1_attention_weight_max": 17.11495590209961,
      "activations/layer1_attention_weight_min": -14.928121566772461,
      "activations/layer20_attention_weight_max": 23.828948974609375,
      "activations/layer20_attention_weight_min": -23.478368759155273,
      "activations/layer21_attention_weight_max": 41.86185836791992,
      "activations/layer21_attention_weight_min": -22.69356918334961,
      "activations/layer22_attention_weight_max": 38.30289077758789,
      "activations/layer22_attention_weight_min": -26.035982131958008,
      "activations/layer23_attention_weight_max": 35.94611358642578,
      "activations/layer23_attention_weight_min": -21.610942840576172,
      "activations/layer2_attention_weight_max": 31.820148468017578,
      "activations/layer2_attention_weight_min": -29.94000816345215,
      "activations/layer3_attention_weight_max": 85.90035247802734,
      "activations/layer3_attention_weight_min": -90.88501739501953,
      "activations/layer4_attention_weight_max": 99.73230743408203,
      "activations/layer4_attention_weight_min": -95.37678527832031,
      "activations/layer5_attention_weight_max": 83.94465637207031,
      "activations/layer5_attention_weight_min": -78.84933471679688,
      "activations/layer6_attention_weight_max": 53.740482330322266,
      "activations/layer6_attention_weight_min": -52.84340286254883,
      "activations/layer7_attention_weight_max": 79.56742858886719,
      "activations/layer7_attention_weight_min": -72.72285461425781,
      "activations/layer8_attention_weight_max": 45.44818878173828,
      "activations/layer8_attention_weight_min": -48.90782165527344,
      "activations/layer9_attention_weight_max": 59.907466888427734,
      "activations/layer9_attention_weight_min": -50.953792572021484,
      "epoch": 8.07,
      "learning_rate": 9.89871212121212e-05,
      "loss": 2.7868,
      "step": 138800
    },
    {
      "activations/layer0_attention_weight_max": 16.026700973510742,
      "activations/layer0_attention_weight_min": -15.443477630615234,
      "activations/layer10_attention_weight_max": 40.85957336425781,
      "activations/layer10_attention_weight_min": -36.80299758911133,
      "activations/layer11_attention_weight_max": 32.14828109741211,
      "activations/layer11_attention_weight_min": -33.137184143066406,
      "activations/layer12_attention_weight_max": 16.76504135131836,
      "activations/layer12_attention_weight_min": -23.801733016967773,
      "activations/layer13_attention_weight_max": 41.01490783691406,
      "activations/layer13_attention_weight_min": -32.08543395996094,
      "activations/layer14_attention_weight_max": 38.761253356933594,
      "activations/layer14_attention_weight_min": -31.271026611328125,
      "activations/layer15_attention_weight_max": 34.5986442565918,
      "activations/layer15_attention_weight_min": -29.437721252441406,
      "activations/layer16_attention_weight_max": 29.996721267700195,
      "activations/layer16_attention_weight_min": -28.114660263061523,
      "activations/layer17_attention_weight_max": 49.403785705566406,
      "activations/layer17_attention_weight_min": -41.92147445678711,
      "activations/layer18_attention_weight_max": 40.977256774902344,
      "activations/layer18_attention_weight_min": -35.66771697998047,
      "activations/layer19_attention_weight_max": 21.649511337280273,
      "activations/layer19_attention_weight_min": -22.126617431640625,
      "activations/layer1_attention_weight_max": 16.66187858581543,
      "activations/layer1_attention_weight_min": -14.849510192871094,
      "activations/layer20_attention_weight_max": 21.43621063232422,
      "activations/layer20_attention_weight_min": -22.2807674407959,
      "activations/layer21_attention_weight_max": 37.98686218261719,
      "activations/layer21_attention_weight_min": -23.505788803100586,
      "activations/layer22_attention_weight_max": 30.19104766845703,
      "activations/layer22_attention_weight_min": -25.796920776367188,
      "activations/layer23_attention_weight_max": 37.34947967529297,
      "activations/layer23_attention_weight_min": -23.233413696289062,
      "activations/layer2_attention_weight_max": 30.24066734313965,
      "activations/layer2_attention_weight_min": -28.708738327026367,
      "activations/layer3_attention_weight_max": 84.12615966796875,
      "activations/layer3_attention_weight_min": -83.34117126464844,
      "activations/layer4_attention_weight_max": 98.27720642089844,
      "activations/layer4_attention_weight_min": -92.4801254272461,
      "activations/layer5_attention_weight_max": 84.70985412597656,
      "activations/layer5_attention_weight_min": -78.82292175292969,
      "activations/layer6_attention_weight_max": 55.85226821899414,
      "activations/layer6_attention_weight_min": -51.367530822753906,
      "activations/layer7_attention_weight_max": 75.33435821533203,
      "activations/layer7_attention_weight_min": -71.98023986816406,
      "activations/layer8_attention_weight_max": 44.86250305175781,
      "activations/layer8_attention_weight_min": -45.57658767700195,
      "activations/layer9_attention_weight_max": 49.94361877441406,
      "activations/layer9_attention_weight_min": -51.85606002807617,
      "epoch": 8.07,
      "learning_rate": 9.896818181818181e-05,
      "loss": 2.795,
      "step": 138850
    },
    {
      "activations/layer0_attention_weight_max": 15.717702865600586,
      "activations/layer0_attention_weight_min": -15.423972129821777,
      "activations/layer10_attention_weight_max": 36.98887252807617,
      "activations/layer10_attention_weight_min": -39.12188720703125,
      "activations/layer11_attention_weight_max": 30.1954345703125,
      "activations/layer11_attention_weight_min": -31.808425903320312,
      "activations/layer12_attention_weight_max": 24.577157974243164,
      "activations/layer12_attention_weight_min": -34.75183868408203,
      "activations/layer13_attention_weight_max": 55.0479736328125,
      "activations/layer13_attention_weight_min": -32.724483489990234,
      "activations/layer14_attention_weight_max": 51.337894439697266,
      "activations/layer14_attention_weight_min": -29.861448287963867,
      "activations/layer15_attention_weight_max": 44.42317199707031,
      "activations/layer15_attention_weight_min": -30.158523559570312,
      "activations/layer16_attention_weight_max": 33.63011169433594,
      "activations/layer16_attention_weight_min": -27.657859802246094,
      "activations/layer17_attention_weight_max": 53.05763626098633,
      "activations/layer17_attention_weight_min": -45.23048782348633,
      "activations/layer18_attention_weight_max": 45.66090393066406,
      "activations/layer18_attention_weight_min": -39.25481033325195,
      "activations/layer19_attention_weight_max": 19.944177627563477,
      "activations/layer19_attention_weight_min": -22.638517379760742,
      "activations/layer1_attention_weight_max": 17.599925994873047,
      "activations/layer1_attention_weight_min": -13.574982643127441,
      "activations/layer20_attention_weight_max": 21.550682067871094,
      "activations/layer20_attention_weight_min": -21.37181854248047,
      "activations/layer21_attention_weight_max": 38.666954040527344,
      "activations/layer21_attention_weight_min": -23.318937301635742,
      "activations/layer22_attention_weight_max": 34.61236572265625,
      "activations/layer22_attention_weight_min": -23.6900577545166,
      "activations/layer23_attention_weight_max": 36.97708511352539,
      "activations/layer23_attention_weight_min": -25.439136505126953,
      "activations/layer2_attention_weight_max": 30.375816345214844,
      "activations/layer2_attention_weight_min": -28.466365814208984,
      "activations/layer3_attention_weight_max": 87.87523651123047,
      "activations/layer3_attention_weight_min": -90.1376724243164,
      "activations/layer4_attention_weight_max": 99.35143280029297,
      "activations/layer4_attention_weight_min": -95.15672302246094,
      "activations/layer5_attention_weight_max": 82.88322448730469,
      "activations/layer5_attention_weight_min": -82.45753479003906,
      "activations/layer6_attention_weight_max": 54.2828483581543,
      "activations/layer6_attention_weight_min": -53.297752380371094,
      "activations/layer7_attention_weight_max": 75.99654388427734,
      "activations/layer7_attention_weight_min": -76.86406707763672,
      "activations/layer8_attention_weight_max": 45.74687576293945,
      "activations/layer8_attention_weight_min": -47.29170608520508,
      "activations/layer9_attention_weight_max": 59.59128952026367,
      "activations/layer9_attention_weight_min": -49.90560531616211,
      "epoch": 8.07,
      "learning_rate": 9.89496212121212e-05,
      "loss": 2.8119,
      "step": 138900
    },
    {
      "activations/layer0_attention_weight_max": 16.75315284729004,
      "activations/layer0_attention_weight_min": -14.853902816772461,
      "activations/layer10_attention_weight_max": 38.347137451171875,
      "activations/layer10_attention_weight_min": -36.01944351196289,
      "activations/layer11_attention_weight_max": 32.33860778808594,
      "activations/layer11_attention_weight_min": -31.490339279174805,
      "activations/layer12_attention_weight_max": 21.980064392089844,
      "activations/layer12_attention_weight_min": -24.690872192382812,
      "activations/layer13_attention_weight_max": 47.05271530151367,
      "activations/layer13_attention_weight_min": -34.14751434326172,
      "activations/layer14_attention_weight_max": 43.32400894165039,
      "activations/layer14_attention_weight_min": -30.949827194213867,
      "activations/layer15_attention_weight_max": 34.07492446899414,
      "activations/layer15_attention_weight_min": -31.1614990234375,
      "activations/layer16_attention_weight_max": 33.22084426879883,
      "activations/layer16_attention_weight_min": -27.2582950592041,
      "activations/layer17_attention_weight_max": 57.57626724243164,
      "activations/layer17_attention_weight_min": -44.569644927978516,
      "activations/layer18_attention_weight_max": 48.65620040893555,
      "activations/layer18_attention_weight_min": -36.62311553955078,
      "activations/layer19_attention_weight_max": 22.883941650390625,
      "activations/layer19_attention_weight_min": -20.591766357421875,
      "activations/layer1_attention_weight_max": 16.69988250732422,
      "activations/layer1_attention_weight_min": -13.522075653076172,
      "activations/layer20_attention_weight_max": 21.782804489135742,
      "activations/layer20_attention_weight_min": -24.006357192993164,
      "activations/layer21_attention_weight_max": 43.269287109375,
      "activations/layer21_attention_weight_min": -24.97654914855957,
      "activations/layer22_attention_weight_max": 33.725830078125,
      "activations/layer22_attention_weight_min": -26.12055015563965,
      "activations/layer23_attention_weight_max": 35.519317626953125,
      "activations/layer23_attention_weight_min": -19.792964935302734,
      "activations/layer2_attention_weight_max": 30.90847396850586,
      "activations/layer2_attention_weight_min": -28.714359283447266,
      "activations/layer3_attention_weight_max": 87.4788818359375,
      "activations/layer3_attention_weight_min": -89.75811004638672,
      "activations/layer4_attention_weight_max": 100.03504943847656,
      "activations/layer4_attention_weight_min": -96.9541244506836,
      "activations/layer5_attention_weight_max": 82.33829498291016,
      "activations/layer5_attention_weight_min": -78.06267547607422,
      "activations/layer6_attention_weight_max": 51.715511322021484,
      "activations/layer6_attention_weight_min": -51.96712875366211,
      "activations/layer7_attention_weight_max": 75.4081039428711,
      "activations/layer7_attention_weight_min": -73.83260345458984,
      "activations/layer8_attention_weight_max": 44.75624084472656,
      "activations/layer8_attention_weight_min": -45.2851676940918,
      "activations/layer9_attention_weight_max": 53.57355880737305,
      "activations/layer9_attention_weight_min": -52.78670120239258,
      "epoch": 8.07,
      "learning_rate": 9.893068181818182e-05,
      "loss": 2.7883,
      "step": 138950
    },
    {
      "activations/layer0_attention_weight_max": 15.52285385131836,
      "activations/layer0_attention_weight_min": -15.210071563720703,
      "activations/layer10_attention_weight_max": 38.59645462036133,
      "activations/layer10_attention_weight_min": -38.618961334228516,
      "activations/layer11_attention_weight_max": 32.28248596191406,
      "activations/layer11_attention_weight_min": -32.6875114440918,
      "activations/layer12_attention_weight_max": 20.36969566345215,
      "activations/layer12_attention_weight_min": -23.738309860229492,
      "activations/layer13_attention_weight_max": 45.408164978027344,
      "activations/layer13_attention_weight_min": -30.117572784423828,
      "activations/layer14_attention_weight_max": 46.587005615234375,
      "activations/layer14_attention_weight_min": -31.588594436645508,
      "activations/layer15_attention_weight_max": 38.108543395996094,
      "activations/layer15_attention_weight_min": -30.61577033996582,
      "activations/layer16_attention_weight_max": 37.46543502807617,
      "activations/layer16_attention_weight_min": -27.871414184570312,
      "activations/layer17_attention_weight_max": 56.35021209716797,
      "activations/layer17_attention_weight_min": -44.384944915771484,
      "activations/layer18_attention_weight_max": 52.273197174072266,
      "activations/layer18_attention_weight_min": -35.55567169189453,
      "activations/layer19_attention_weight_max": 21.44038200378418,
      "activations/layer19_attention_weight_min": -24.28167152404785,
      "activations/layer1_attention_weight_max": 16.280895233154297,
      "activations/layer1_attention_weight_min": -13.508939743041992,
      "activations/layer20_attention_weight_max": 24.27562141418457,
      "activations/layer20_attention_weight_min": -26.55599594116211,
      "activations/layer21_attention_weight_max": 39.45025634765625,
      "activations/layer21_attention_weight_min": -24.684995651245117,
      "activations/layer22_attention_weight_max": 31.68004608154297,
      "activations/layer22_attention_weight_min": -26.21988868713379,
      "activations/layer23_attention_weight_max": 36.594444274902344,
      "activations/layer23_attention_weight_min": -22.25033187866211,
      "activations/layer2_attention_weight_max": 32.0857048034668,
      "activations/layer2_attention_weight_min": -30.72986602783203,
      "activations/layer3_attention_weight_max": 90.66962432861328,
      "activations/layer3_attention_weight_min": -91.92668151855469,
      "activations/layer4_attention_weight_max": 102.23693084716797,
      "activations/layer4_attention_weight_min": -96.48416900634766,
      "activations/layer5_attention_weight_max": 86.6534423828125,
      "activations/layer5_attention_weight_min": -77.97547912597656,
      "activations/layer6_attention_weight_max": 55.763492584228516,
      "activations/layer6_attention_weight_min": -53.42759323120117,
      "activations/layer7_attention_weight_max": 77.8017807006836,
      "activations/layer7_attention_weight_min": -76.66888427734375,
      "activations/layer8_attention_weight_max": 48.03157043457031,
      "activations/layer8_attention_weight_min": -49.45207977294922,
      "activations/layer9_attention_weight_max": 55.630157470703125,
      "activations/layer9_attention_weight_min": -50.4287109375,
      "epoch": 8.08,
      "learning_rate": 9.89117424242424e-05,
      "loss": 2.8002,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_loss": 2.76171875,
      "eval_runtime": 8.5778,
      "eval_samples_per_second": 500.595,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_openwebtext_loss": 2.76171875,
      "eval_openwebtext_ppl": 15.827022278874091,
      "eval_openwebtext_runtime": 8.5778,
      "eval_openwebtext_samples_per_second": 500.595,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 2.0307,
      "eval_wikitext_samples_per_second": 224.558,
      "step": 139000
    },
    {
      "epoch": 8.08,
      "eval_lambada_loss": 2.765625,
      "eval_lambada_ppl": 15.888967492629165,
      "eval_lambada_runtime": 9.6306,
      "eval_lambada_samples_per_second": 505.574,
      "step": 139000
    },
    {
      "activations/layer0_attention_weight_max": 14.14484691619873,
      "activations/layer0_attention_weight_min": -15.189051628112793,
      "activations/layer10_attention_weight_max": 38.59468460083008,
      "activations/layer10_attention_weight_min": -34.623985290527344,
      "activations/layer11_attention_weight_max": 33.81212615966797,
      "activations/layer11_attention_weight_min": -33.21142578125,
      "activations/layer12_attention_weight_max": 20.943017959594727,
      "activations/layer12_attention_weight_min": -23.75135040283203,
      "activations/layer13_attention_weight_max": 41.77007293701172,
      "activations/layer13_attention_weight_min": -30.914539337158203,
      "activations/layer14_attention_weight_max": 36.712738037109375,
      "activations/layer14_attention_weight_min": -29.9718017578125,
      "activations/layer15_attention_weight_max": 32.908470153808594,
      "activations/layer15_attention_weight_min": -28.7978572845459,
      "activations/layer16_attention_weight_max": 29.647022247314453,
      "activations/layer16_attention_weight_min": -25.42445182800293,
      "activations/layer17_attention_weight_max": 51.85699462890625,
      "activations/layer17_attention_weight_min": -42.340946197509766,
      "activations/layer18_attention_weight_max": 47.00937271118164,
      "activations/layer18_attention_weight_min": -34.056182861328125,
      "activations/layer19_attention_weight_max": 19.763504028320312,
      "activations/layer19_attention_weight_min": -20.965335845947266,
      "activations/layer1_attention_weight_max": 18.121028900146484,
      "activations/layer1_attention_weight_min": -15.017340660095215,
      "activations/layer20_attention_weight_max": 20.399404525756836,
      "activations/layer20_attention_weight_min": -21.813186645507812,
      "activations/layer21_attention_weight_max": 36.35481643676758,
      "activations/layer21_attention_weight_min": -24.3949031829834,
      "activations/layer22_attention_weight_max": 30.05537986755371,
      "activations/layer22_attention_weight_min": -24.48337173461914,
      "activations/layer23_attention_weight_max": 35.334938049316406,
      "activations/layer23_attention_weight_min": -25.579002380371094,
      "activations/layer2_attention_weight_max": 31.310199737548828,
      "activations/layer2_attention_weight_min": -29.289981842041016,
      "activations/layer3_attention_weight_max": 87.87918090820312,
      "activations/layer3_attention_weight_min": -88.39569854736328,
      "activations/layer4_attention_weight_max": 100.2801513671875,
      "activations/layer4_attention_weight_min": -96.15323638916016,
      "activations/layer5_attention_weight_max": 84.13227844238281,
      "activations/layer5_attention_weight_min": -82.25696563720703,
      "activations/layer6_attention_weight_max": 55.44438171386719,
      "activations/layer6_attention_weight_min": -53.796146392822266,
      "activations/layer7_attention_weight_max": 78.66363525390625,
      "activations/layer7_attention_weight_min": -74.8118896484375,
      "activations/layer8_attention_weight_max": 45.016719818115234,
      "activations/layer8_attention_weight_min": -48.267234802246094,
      "activations/layer9_attention_weight_max": 49.95738983154297,
      "activations/layer9_attention_weight_min": -48.79679489135742,
      "epoch": 8.08,
      "learning_rate": 9.889280303030302e-05,
      "loss": 2.81,
      "step": 139050
    },
    {
      "activations/layer0_attention_weight_max": 15.550189971923828,
      "activations/layer0_attention_weight_min": -14.810341835021973,
      "activations/layer10_attention_weight_max": 37.02420425415039,
      "activations/layer10_attention_weight_min": -35.537147521972656,
      "activations/layer11_attention_weight_max": 30.545568466186523,
      "activations/layer11_attention_weight_min": -32.02987289428711,
      "activations/layer12_attention_weight_max": 17.878326416015625,
      "activations/layer12_attention_weight_min": -25.05939483642578,
      "activations/layer13_attention_weight_max": 36.18434143066406,
      "activations/layer13_attention_weight_min": -29.757118225097656,
      "activations/layer14_attention_weight_max": 36.849327087402344,
      "activations/layer14_attention_weight_min": -30.003061294555664,
      "activations/layer15_attention_weight_max": 33.33843994140625,
      "activations/layer15_attention_weight_min": -30.309249877929688,
      "activations/layer16_attention_weight_max": 30.149381637573242,
      "activations/layer16_attention_weight_min": -30.109390258789062,
      "activations/layer17_attention_weight_max": 54.25630569458008,
      "activations/layer17_attention_weight_min": -45.863250732421875,
      "activations/layer18_attention_weight_max": 47.32439422607422,
      "activations/layer18_attention_weight_min": -36.18680953979492,
      "activations/layer19_attention_weight_max": 24.179824829101562,
      "activations/layer19_attention_weight_min": -21.076885223388672,
      "activations/layer1_attention_weight_max": 16.645963668823242,
      "activations/layer1_attention_weight_min": -16.75145721435547,
      "activations/layer20_attention_weight_max": 24.220874786376953,
      "activations/layer20_attention_weight_min": -22.144485473632812,
      "activations/layer21_attention_weight_max": 36.83433151245117,
      "activations/layer21_attention_weight_min": -23.770658493041992,
      "activations/layer22_attention_weight_max": 32.3503303527832,
      "activations/layer22_attention_weight_min": -25.627330780029297,
      "activations/layer23_attention_weight_max": 34.4004020690918,
      "activations/layer23_attention_weight_min": -22.920276641845703,
      "activations/layer2_attention_weight_max": 32.912994384765625,
      "activations/layer2_attention_weight_min": -29.463422775268555,
      "activations/layer3_attention_weight_max": 86.90792083740234,
      "activations/layer3_attention_weight_min": -88.69224548339844,
      "activations/layer4_attention_weight_max": 99.0174789428711,
      "activations/layer4_attention_weight_min": -93.03939056396484,
      "activations/layer5_attention_weight_max": 85.44969940185547,
      "activations/layer5_attention_weight_min": -79.39535522460938,
      "activations/layer6_attention_weight_max": 54.52287673950195,
      "activations/layer6_attention_weight_min": -54.26359176635742,
      "activations/layer7_attention_weight_max": 73.88980865478516,
      "activations/layer7_attention_weight_min": -71.32852172851562,
      "activations/layer8_attention_weight_max": 43.94404983520508,
      "activations/layer8_attention_weight_min": -45.5212287902832,
      "activations/layer9_attention_weight_max": 49.993770599365234,
      "activations/layer9_attention_weight_min": -47.06745529174805,
      "epoch": 8.08,
      "learning_rate": 9.887386363636362e-05,
      "loss": 2.7891,
      "step": 139100
    },
    {
      "activations/layer0_attention_weight_max": 15.198877334594727,
      "activations/layer0_attention_weight_min": -14.807598114013672,
      "activations/layer10_attention_weight_max": 35.430702209472656,
      "activations/layer10_attention_weight_min": -33.02397537231445,
      "activations/layer11_attention_weight_max": 31.1820125579834,
      "activations/layer11_attention_weight_min": -29.848655700683594,
      "activations/layer12_attention_weight_max": 17.440006256103516,
      "activations/layer12_attention_weight_min": -25.710908889770508,
      "activations/layer13_attention_weight_max": 36.75225830078125,
      "activations/layer13_attention_weight_min": -31.482219696044922,
      "activations/layer14_attention_weight_max": 34.751373291015625,
      "activations/layer14_attention_weight_min": -29.69806480407715,
      "activations/layer15_attention_weight_max": 34.35323715209961,
      "activations/layer15_attention_weight_min": -28.408390045166016,
      "activations/layer16_attention_weight_max": 28.067190170288086,
      "activations/layer16_attention_weight_min": -26.68160057067871,
      "activations/layer17_attention_weight_max": 54.06157302856445,
      "activations/layer17_attention_weight_min": -42.00921630859375,
      "activations/layer18_attention_weight_max": 43.830894470214844,
      "activations/layer18_attention_weight_min": -35.79250717163086,
      "activations/layer19_attention_weight_max": 21.40029525756836,
      "activations/layer19_attention_weight_min": -21.144336700439453,
      "activations/layer1_attention_weight_max": 17.611087799072266,
      "activations/layer1_attention_weight_min": -14.745558738708496,
      "activations/layer20_attention_weight_max": 20.94208335876465,
      "activations/layer20_attention_weight_min": -25.351457595825195,
      "activations/layer21_attention_weight_max": 34.338966369628906,
      "activations/layer21_attention_weight_min": -22.050472259521484,
      "activations/layer22_attention_weight_max": 27.432018280029297,
      "activations/layer22_attention_weight_min": -24.29205322265625,
      "activations/layer23_attention_weight_max": 33.258846282958984,
      "activations/layer23_attention_weight_min": -22.822904586791992,
      "activations/layer2_attention_weight_max": 30.558029174804688,
      "activations/layer2_attention_weight_min": -29.98451042175293,
      "activations/layer3_attention_weight_max": 84.13369750976562,
      "activations/layer3_attention_weight_min": -89.56818389892578,
      "activations/layer4_attention_weight_max": 97.24915313720703,
      "activations/layer4_attention_weight_min": -93.27620697021484,
      "activations/layer5_attention_weight_max": 81.90774536132812,
      "activations/layer5_attention_weight_min": -76.69157409667969,
      "activations/layer6_attention_weight_max": 52.208335876464844,
      "activations/layer6_attention_weight_min": -53.55830383300781,
      "activations/layer7_attention_weight_max": 72.99153900146484,
      "activations/layer7_attention_weight_min": -75.76604461669922,
      "activations/layer8_attention_weight_max": 43.67342758178711,
      "activations/layer8_attention_weight_min": -44.29313659667969,
      "activations/layer9_attention_weight_max": 48.5490837097168,
      "activations/layer9_attention_weight_min": -47.34267807006836,
      "epoch": 8.09,
      "learning_rate": 9.885492424242424e-05,
      "loss": 2.8151,
      "step": 139150
    },
    {
      "activations/layer0_attention_weight_max": 14.998971939086914,
      "activations/layer0_attention_weight_min": -14.841726303100586,
      "activations/layer10_attention_weight_max": 37.584747314453125,
      "activations/layer10_attention_weight_min": -34.71736145019531,
      "activations/layer11_attention_weight_max": 33.633174896240234,
      "activations/layer11_attention_weight_min": -31.160037994384766,
      "activations/layer12_attention_weight_max": 17.654529571533203,
      "activations/layer12_attention_weight_min": -22.561660766601562,
      "activations/layer13_attention_weight_max": 45.35525894165039,
      "activations/layer13_attention_weight_min": -31.134849548339844,
      "activations/layer14_attention_weight_max": 46.7597541809082,
      "activations/layer14_attention_weight_min": -33.25871276855469,
      "activations/layer15_attention_weight_max": 40.307369232177734,
      "activations/layer15_attention_weight_min": -28.64305877685547,
      "activations/layer16_attention_weight_max": 30.146703720092773,
      "activations/layer16_attention_weight_min": -28.69464683532715,
      "activations/layer17_attention_weight_max": 54.751468658447266,
      "activations/layer17_attention_weight_min": -44.175594329833984,
      "activations/layer18_attention_weight_max": 48.20466613769531,
      "activations/layer18_attention_weight_min": -38.69187545776367,
      "activations/layer19_attention_weight_max": 23.060733795166016,
      "activations/layer19_attention_weight_min": -20.468425750732422,
      "activations/layer1_attention_weight_max": 17.522178649902344,
      "activations/layer1_attention_weight_min": -15.105385780334473,
      "activations/layer20_attention_weight_max": 24.198184967041016,
      "activations/layer20_attention_weight_min": -22.900867462158203,
      "activations/layer21_attention_weight_max": 36.240333557128906,
      "activations/layer21_attention_weight_min": -25.941062927246094,
      "activations/layer22_attention_weight_max": 34.33756637573242,
      "activations/layer22_attention_weight_min": -25.24414825439453,
      "activations/layer23_attention_weight_max": 37.756202697753906,
      "activations/layer23_attention_weight_min": -21.50185203552246,
      "activations/layer2_attention_weight_max": 31.312236785888672,
      "activations/layer2_attention_weight_min": -29.750410079956055,
      "activations/layer3_attention_weight_max": 85.89395141601562,
      "activations/layer3_attention_weight_min": -88.81492614746094,
      "activations/layer4_attention_weight_max": 98.61263275146484,
      "activations/layer4_attention_weight_min": -95.82571411132812,
      "activations/layer5_attention_weight_max": 81.66238403320312,
      "activations/layer5_attention_weight_min": -77.73049926757812,
      "activations/layer6_attention_weight_max": 57.18849563598633,
      "activations/layer6_attention_weight_min": -53.136226654052734,
      "activations/layer7_attention_weight_max": 81.17281341552734,
      "activations/layer7_attention_weight_min": -74.09593200683594,
      "activations/layer8_attention_weight_max": 47.490234375,
      "activations/layer8_attention_weight_min": -46.52970504760742,
      "activations/layer9_attention_weight_max": 50.39036560058594,
      "activations/layer9_attention_weight_min": -51.45430374145508,
      "epoch": 8.09,
      "learning_rate": 9.883598484848484e-05,
      "loss": 2.817,
      "step": 139200
    },
    {
      "activations/layer0_attention_weight_max": 14.649478912353516,
      "activations/layer0_attention_weight_min": -14.734443664550781,
      "activations/layer10_attention_weight_max": 39.763614654541016,
      "activations/layer10_attention_weight_min": -37.952117919921875,
      "activations/layer11_attention_weight_max": 31.574466705322266,
      "activations/layer11_attention_weight_min": -33.294639587402344,
      "activations/layer12_attention_weight_max": 24.482093811035156,
      "activations/layer12_attention_weight_min": -25.52067756652832,
      "activations/layer13_attention_weight_max": 45.36055374145508,
      "activations/layer13_attention_weight_min": -34.986759185791016,
      "activations/layer14_attention_weight_max": 39.180049896240234,
      "activations/layer14_attention_weight_min": -32.28704071044922,
      "activations/layer15_attention_weight_max": 36.15291976928711,
      "activations/layer15_attention_weight_min": -30.699745178222656,
      "activations/layer16_attention_weight_max": 34.419776916503906,
      "activations/layer16_attention_weight_min": -28.59908103942871,
      "activations/layer17_attention_weight_max": 57.67335510253906,
      "activations/layer17_attention_weight_min": -44.13819122314453,
      "activations/layer18_attention_weight_max": 49.72386932373047,
      "activations/layer18_attention_weight_min": -38.48362350463867,
      "activations/layer19_attention_weight_max": 20.29759979248047,
      "activations/layer19_attention_weight_min": -20.72040367126465,
      "activations/layer1_attention_weight_max": 16.959407806396484,
      "activations/layer1_attention_weight_min": -15.11394214630127,
      "activations/layer20_attention_weight_max": 22.984521865844727,
      "activations/layer20_attention_weight_min": -20.25539207458496,
      "activations/layer21_attention_weight_max": 40.50705337524414,
      "activations/layer21_attention_weight_min": -24.698400497436523,
      "activations/layer22_attention_weight_max": 34.82428741455078,
      "activations/layer22_attention_weight_min": -26.06513786315918,
      "activations/layer23_attention_weight_max": 34.947898864746094,
      "activations/layer23_attention_weight_min": -20.99811553955078,
      "activations/layer2_attention_weight_max": 31.539806365966797,
      "activations/layer2_attention_weight_min": -28.68661880493164,
      "activations/layer3_attention_weight_max": 87.9151840209961,
      "activations/layer3_attention_weight_min": -90.38117980957031,
      "activations/layer4_attention_weight_max": 106.07258605957031,
      "activations/layer4_attention_weight_min": -97.2350845336914,
      "activations/layer5_attention_weight_max": 92.50833129882812,
      "activations/layer5_attention_weight_min": -86.7686767578125,
      "activations/layer6_attention_weight_max": 60.69285202026367,
      "activations/layer6_attention_weight_min": -57.819427490234375,
      "activations/layer7_attention_weight_max": 88.4888687133789,
      "activations/layer7_attention_weight_min": -75.63409423828125,
      "activations/layer8_attention_weight_max": 49.89602279663086,
      "activations/layer8_attention_weight_min": -48.85518264770508,
      "activations/layer9_attention_weight_max": 55.1011962890625,
      "activations/layer9_attention_weight_min": -51.78348922729492,
      "epoch": 8.09,
      "learning_rate": 9.881704545454544e-05,
      "loss": 2.8082,
      "step": 139250
    },
    {
      "activations/layer0_attention_weight_max": 15.16740608215332,
      "activations/layer0_attention_weight_min": -15.36991024017334,
      "activations/layer10_attention_weight_max": 37.17255401611328,
      "activations/layer10_attention_weight_min": -34.49236297607422,
      "activations/layer11_attention_weight_max": 31.210647583007812,
      "activations/layer11_attention_weight_min": -32.241214752197266,
      "activations/layer12_attention_weight_max": 19.821115493774414,
      "activations/layer12_attention_weight_min": -24.8823299407959,
      "activations/layer13_attention_weight_max": 45.63872146606445,
      "activations/layer13_attention_weight_min": -30.23961067199707,
      "activations/layer14_attention_weight_max": 38.07683563232422,
      "activations/layer14_attention_weight_min": -29.235937118530273,
      "activations/layer15_attention_weight_max": 37.92887878417969,
      "activations/layer15_attention_weight_min": -28.088924407958984,
      "activations/layer16_attention_weight_max": 32.20222091674805,
      "activations/layer16_attention_weight_min": -26.711381912231445,
      "activations/layer17_attention_weight_max": 51.557518005371094,
      "activations/layer17_attention_weight_min": -41.892398834228516,
      "activations/layer18_attention_weight_max": 44.89965057373047,
      "activations/layer18_attention_weight_min": -35.86113739013672,
      "activations/layer19_attention_weight_max": 22.434370040893555,
      "activations/layer19_attention_weight_min": -19.85757827758789,
      "activations/layer1_attention_weight_max": 17.358806610107422,
      "activations/layer1_attention_weight_min": -15.702591896057129,
      "activations/layer20_attention_weight_max": 23.605710983276367,
      "activations/layer20_attention_weight_min": -26.250459671020508,
      "activations/layer21_attention_weight_max": 36.7801513671875,
      "activations/layer21_attention_weight_min": -22.28862953186035,
      "activations/layer22_attention_weight_max": 31.063081741333008,
      "activations/layer22_attention_weight_min": -24.95115089416504,
      "activations/layer23_attention_weight_max": 38.85272216796875,
      "activations/layer23_attention_weight_min": -20.527984619140625,
      "activations/layer2_attention_weight_max": 31.458724975585938,
      "activations/layer2_attention_weight_min": -28.620590209960938,
      "activations/layer3_attention_weight_max": 88.8627700805664,
      "activations/layer3_attention_weight_min": -93.36450958251953,
      "activations/layer4_attention_weight_max": 102.2882308959961,
      "activations/layer4_attention_weight_min": -95.93107604980469,
      "activations/layer5_attention_weight_max": 85.64602661132812,
      "activations/layer5_attention_weight_min": -80.61135864257812,
      "activations/layer6_attention_weight_max": 56.662994384765625,
      "activations/layer6_attention_weight_min": -51.59368133544922,
      "activations/layer7_attention_weight_max": 77.08741760253906,
      "activations/layer7_attention_weight_min": -72.9339828491211,
      "activations/layer8_attention_weight_max": 44.933326721191406,
      "activations/layer8_attention_weight_min": -46.675601959228516,
      "activations/layer9_attention_weight_max": 50.87523651123047,
      "activations/layer9_attention_weight_min": -48.019744873046875,
      "epoch": 8.09,
      "learning_rate": 9.879810606060606e-05,
      "loss": 2.8135,
      "step": 139300
    },
    {
      "activations/layer0_attention_weight_max": 14.640914916992188,
      "activations/layer0_attention_weight_min": -14.6104097366333,
      "activations/layer10_attention_weight_max": 37.71831512451172,
      "activations/layer10_attention_weight_min": -34.73033142089844,
      "activations/layer11_attention_weight_max": 33.690399169921875,
      "activations/layer11_attention_weight_min": -32.30652618408203,
      "activations/layer12_attention_weight_max": 16.462024688720703,
      "activations/layer12_attention_weight_min": -25.009077072143555,
      "activations/layer13_attention_weight_max": 37.521453857421875,
      "activations/layer13_attention_weight_min": -30.847450256347656,
      "activations/layer14_attention_weight_max": 34.73318862915039,
      "activations/layer14_attention_weight_min": -30.634597778320312,
      "activations/layer15_attention_weight_max": 33.643768310546875,
      "activations/layer15_attention_weight_min": -29.66506576538086,
      "activations/layer16_attention_weight_max": 30.39117431640625,
      "activations/layer16_attention_weight_min": -27.515214920043945,
      "activations/layer17_attention_weight_max": 52.513710021972656,
      "activations/layer17_attention_weight_min": -45.262664794921875,
      "activations/layer18_attention_weight_max": 45.65804672241211,
      "activations/layer18_attention_weight_min": -36.51239776611328,
      "activations/layer19_attention_weight_max": 19.00348472595215,
      "activations/layer19_attention_weight_min": -19.924169540405273,
      "activations/layer1_attention_weight_max": 17.35047721862793,
      "activations/layer1_attention_weight_min": -14.526174545288086,
      "activations/layer20_attention_weight_max": 22.838476181030273,
      "activations/layer20_attention_weight_min": -27.923614501953125,
      "activations/layer21_attention_weight_max": 37.509185791015625,
      "activations/layer21_attention_weight_min": -23.90330696105957,
      "activations/layer22_attention_weight_max": 28.645418167114258,
      "activations/layer22_attention_weight_min": -26.371776580810547,
      "activations/layer23_attention_weight_max": 34.58620071411133,
      "activations/layer23_attention_weight_min": -22.510215759277344,
      "activations/layer2_attention_weight_max": 31.24604606628418,
      "activations/layer2_attention_weight_min": -27.304845809936523,
      "activations/layer3_attention_weight_max": 85.78821563720703,
      "activations/layer3_attention_weight_min": -93.03821563720703,
      "activations/layer4_attention_weight_max": 105.04613494873047,
      "activations/layer4_attention_weight_min": -98.20338439941406,
      "activations/layer5_attention_weight_max": 87.67986297607422,
      "activations/layer5_attention_weight_min": -81.90586853027344,
      "activations/layer6_attention_weight_max": 56.46049880981445,
      "activations/layer6_attention_weight_min": -56.1056022644043,
      "activations/layer7_attention_weight_max": 75.94515991210938,
      "activations/layer7_attention_weight_min": -75.03681182861328,
      "activations/layer8_attention_weight_max": 43.513370513916016,
      "activations/layer8_attention_weight_min": -46.655426025390625,
      "activations/layer9_attention_weight_max": 51.01483917236328,
      "activations/layer9_attention_weight_min": -51.325355529785156,
      "epoch": 8.1,
      "learning_rate": 9.877916666666667e-05,
      "loss": 2.7967,
      "step": 139350
    },
    {
      "activations/layer0_attention_weight_max": 15.27749252319336,
      "activations/layer0_attention_weight_min": -14.691530227661133,
      "activations/layer10_attention_weight_max": 39.44568634033203,
      "activations/layer10_attention_weight_min": -35.79918670654297,
      "activations/layer11_attention_weight_max": 32.788612365722656,
      "activations/layer11_attention_weight_min": -33.01176071166992,
      "activations/layer12_attention_weight_max": 24.496280670166016,
      "activations/layer12_attention_weight_min": -24.329803466796875,
      "activations/layer13_attention_weight_max": 41.07190704345703,
      "activations/layer13_attention_weight_min": -30.695140838623047,
      "activations/layer14_attention_weight_max": 40.1535530090332,
      "activations/layer14_attention_weight_min": -30.22454261779785,
      "activations/layer15_attention_weight_max": 37.12760543823242,
      "activations/layer15_attention_weight_min": -30.098649978637695,
      "activations/layer16_attention_weight_max": 35.47929382324219,
      "activations/layer16_attention_weight_min": -29.34550666809082,
      "activations/layer17_attention_weight_max": 59.62833023071289,
      "activations/layer17_attention_weight_min": -44.53087615966797,
      "activations/layer18_attention_weight_max": 49.33469009399414,
      "activations/layer18_attention_weight_min": -36.6278076171875,
      "activations/layer19_attention_weight_max": 22.97273063659668,
      "activations/layer19_attention_weight_min": -21.77080535888672,
      "activations/layer1_attention_weight_max": 17.873741149902344,
      "activations/layer1_attention_weight_min": -15.772438049316406,
      "activations/layer20_attention_weight_max": 23.91315269470215,
      "activations/layer20_attention_weight_min": -25.258100509643555,
      "activations/layer21_attention_weight_max": 40.567962646484375,
      "activations/layer21_attention_weight_min": -23.60826873779297,
      "activations/layer22_attention_weight_max": 30.391048431396484,
      "activations/layer22_attention_weight_min": -24.173330307006836,
      "activations/layer23_attention_weight_max": 37.617820739746094,
      "activations/layer23_attention_weight_min": -21.90863037109375,
      "activations/layer2_attention_weight_max": 32.07495880126953,
      "activations/layer2_attention_weight_min": -31.56311798095703,
      "activations/layer3_attention_weight_max": 88.8411865234375,
      "activations/layer3_attention_weight_min": -91.68144226074219,
      "activations/layer4_attention_weight_max": 102.49267578125,
      "activations/layer4_attention_weight_min": -97.70899200439453,
      "activations/layer5_attention_weight_max": 85.7783432006836,
      "activations/layer5_attention_weight_min": -82.25379180908203,
      "activations/layer6_attention_weight_max": 57.037933349609375,
      "activations/layer6_attention_weight_min": -55.13356018066406,
      "activations/layer7_attention_weight_max": 78.00990295410156,
      "activations/layer7_attention_weight_min": -73.2605972290039,
      "activations/layer8_attention_weight_max": 44.85797882080078,
      "activations/layer8_attention_weight_min": -45.36000061035156,
      "activations/layer9_attention_weight_max": 51.914424896240234,
      "activations/layer9_attention_weight_min": -48.51029586791992,
      "epoch": 8.1,
      "learning_rate": 9.876022727272726e-05,
      "loss": 2.8004,
      "step": 139400
    },
    {
      "activations/layer0_attention_weight_max": 16.686309814453125,
      "activations/layer0_attention_weight_min": -14.290746688842773,
      "activations/layer10_attention_weight_max": 39.04094314575195,
      "activations/layer10_attention_weight_min": -35.74667739868164,
      "activations/layer11_attention_weight_max": 35.118019104003906,
      "activations/layer11_attention_weight_min": -35.26439666748047,
      "activations/layer12_attention_weight_max": 21.6995849609375,
      "activations/layer12_attention_weight_min": -23.603208541870117,
      "activations/layer13_attention_weight_max": 38.98261642456055,
      "activations/layer13_attention_weight_min": -33.13951873779297,
      "activations/layer14_attention_weight_max": 47.03787612915039,
      "activations/layer14_attention_weight_min": -33.324440002441406,
      "activations/layer15_attention_weight_max": 42.98104476928711,
      "activations/layer15_attention_weight_min": -30.67793846130371,
      "activations/layer16_attention_weight_max": 31.629348754882812,
      "activations/layer16_attention_weight_min": -29.49663734436035,
      "activations/layer17_attention_weight_max": 56.62691116333008,
      "activations/layer17_attention_weight_min": -45.42354202270508,
      "activations/layer18_attention_weight_max": 48.956321716308594,
      "activations/layer18_attention_weight_min": -38.401981353759766,
      "activations/layer19_attention_weight_max": 24.54119110107422,
      "activations/layer19_attention_weight_min": -22.17664909362793,
      "activations/layer1_attention_weight_max": 17.272802352905273,
      "activations/layer1_attention_weight_min": -14.356841087341309,
      "activations/layer20_attention_weight_max": 23.933021545410156,
      "activations/layer20_attention_weight_min": -22.797378540039062,
      "activations/layer21_attention_weight_max": 43.52741622924805,
      "activations/layer21_attention_weight_min": -25.706222534179688,
      "activations/layer22_attention_weight_max": 31.882198333740234,
      "activations/layer22_attention_weight_min": -25.49987030029297,
      "activations/layer23_attention_weight_max": 40.0657958984375,
      "activations/layer23_attention_weight_min": -24.720273971557617,
      "activations/layer2_attention_weight_max": 31.11219596862793,
      "activations/layer2_attention_weight_min": -28.689390182495117,
      "activations/layer3_attention_weight_max": 87.84436798095703,
      "activations/layer3_attention_weight_min": -90.59455871582031,
      "activations/layer4_attention_weight_max": 105.7165298461914,
      "activations/layer4_attention_weight_min": -100.49665069580078,
      "activations/layer5_attention_weight_max": 90.51025390625,
      "activations/layer5_attention_weight_min": -81.22306060791016,
      "activations/layer6_attention_weight_max": 58.47588348388672,
      "activations/layer6_attention_weight_min": -54.485191345214844,
      "activations/layer7_attention_weight_max": 83.62305450439453,
      "activations/layer7_attention_weight_min": -75.0467758178711,
      "activations/layer8_attention_weight_max": 46.39656066894531,
      "activations/layer8_attention_weight_min": -48.029518127441406,
      "activations/layer9_attention_weight_max": 54.2953987121582,
      "activations/layer9_attention_weight_min": -49.262210845947266,
      "epoch": 8.1,
      "learning_rate": 9.874128787878787e-05,
      "loss": 2.8242,
      "step": 139450
    },
    {
      "activations/layer0_attention_weight_max": 14.528487205505371,
      "activations/layer0_attention_weight_min": -14.576910018920898,
      "activations/layer10_attention_weight_max": 39.03121566772461,
      "activations/layer10_attention_weight_min": -37.104007720947266,
      "activations/layer11_attention_weight_max": 33.272972106933594,
      "activations/layer11_attention_weight_min": -34.22196578979492,
      "activations/layer12_attention_weight_max": 18.39731216430664,
      "activations/layer12_attention_weight_min": -23.736103057861328,
      "activations/layer13_attention_weight_max": 42.3745231628418,
      "activations/layer13_attention_weight_min": -28.662410736083984,
      "activations/layer14_attention_weight_max": 38.931480407714844,
      "activations/layer14_attention_weight_min": -29.906509399414062,
      "activations/layer15_attention_weight_max": 32.64640426635742,
      "activations/layer15_attention_weight_min": -29.670812606811523,
      "activations/layer16_attention_weight_max": 33.039527893066406,
      "activations/layer16_attention_weight_min": -27.863239288330078,
      "activations/layer17_attention_weight_max": 54.528717041015625,
      "activations/layer17_attention_weight_min": -43.23801040649414,
      "activations/layer18_attention_weight_max": 48.39159393310547,
      "activations/layer18_attention_weight_min": -36.87953567504883,
      "activations/layer19_attention_weight_max": 20.785587310791016,
      "activations/layer19_attention_weight_min": -20.15399742126465,
      "activations/layer1_attention_weight_max": 16.623598098754883,
      "activations/layer1_attention_weight_min": -14.85827350616455,
      "activations/layer20_attention_weight_max": 22.412315368652344,
      "activations/layer20_attention_weight_min": -21.56399154663086,
      "activations/layer21_attention_weight_max": 36.57330322265625,
      "activations/layer21_attention_weight_min": -23.47727394104004,
      "activations/layer22_attention_weight_max": 35.614505767822266,
      "activations/layer22_attention_weight_min": -25.852436065673828,
      "activations/layer23_attention_weight_max": 39.89897155761719,
      "activations/layer23_attention_weight_min": -21.380809783935547,
      "activations/layer2_attention_weight_max": 30.4514217376709,
      "activations/layer2_attention_weight_min": -27.621688842773438,
      "activations/layer3_attention_weight_max": 82.05203247070312,
      "activations/layer3_attention_weight_min": -83.45699310302734,
      "activations/layer4_attention_weight_max": 100.41497039794922,
      "activations/layer4_attention_weight_min": -94.49138641357422,
      "activations/layer5_attention_weight_max": 86.53115844726562,
      "activations/layer5_attention_weight_min": -79.79083251953125,
      "activations/layer6_attention_weight_max": 57.37137222290039,
      "activations/layer6_attention_weight_min": -54.47260284423828,
      "activations/layer7_attention_weight_max": 79.85518646240234,
      "activations/layer7_attention_weight_min": -73.46160125732422,
      "activations/layer8_attention_weight_max": 44.585296630859375,
      "activations/layer8_attention_weight_min": -45.1049690246582,
      "activations/layer9_attention_weight_max": 52.4243278503418,
      "activations/layer9_attention_weight_min": -50.46766662597656,
      "epoch": 8.11,
      "learning_rate": 9.872234848484846e-05,
      "loss": 2.8118,
      "step": 139500
    },
    {
      "activations/layer0_attention_weight_max": 16.164600372314453,
      "activations/layer0_attention_weight_min": -14.528074264526367,
      "activations/layer10_attention_weight_max": 35.05427551269531,
      "activations/layer10_attention_weight_min": -34.15336227416992,
      "activations/layer11_attention_weight_max": 29.363121032714844,
      "activations/layer11_attention_weight_min": -31.743724822998047,
      "activations/layer12_attention_weight_max": 16.124605178833008,
      "activations/layer12_attention_weight_min": -26.49293327331543,
      "activations/layer13_attention_weight_max": 42.245330810546875,
      "activations/layer13_attention_weight_min": -32.071250915527344,
      "activations/layer14_attention_weight_max": 38.001338958740234,
      "activations/layer14_attention_weight_min": -30.782251358032227,
      "activations/layer15_attention_weight_max": 33.11096954345703,
      "activations/layer15_attention_weight_min": -31.081928253173828,
      "activations/layer16_attention_weight_max": 30.03140640258789,
      "activations/layer16_attention_weight_min": -28.950281143188477,
      "activations/layer17_attention_weight_max": 55.95640182495117,
      "activations/layer17_attention_weight_min": -44.482337951660156,
      "activations/layer18_attention_weight_max": 42.51663589477539,
      "activations/layer18_attention_weight_min": -36.54033660888672,
      "activations/layer19_attention_weight_max": 21.331525802612305,
      "activations/layer19_attention_weight_min": -22.21005630493164,
      "activations/layer1_attention_weight_max": 16.75499725341797,
      "activations/layer1_attention_weight_min": -15.530476570129395,
      "activations/layer20_attention_weight_max": 20.06420135498047,
      "activations/layer20_attention_weight_min": -27.622724533081055,
      "activations/layer21_attention_weight_max": 32.1328010559082,
      "activations/layer21_attention_weight_min": -23.185150146484375,
      "activations/layer22_attention_weight_max": 30.606233596801758,
      "activations/layer22_attention_weight_min": -24.237756729125977,
      "activations/layer23_attention_weight_max": 34.806312561035156,
      "activations/layer23_attention_weight_min": -22.383934020996094,
      "activations/layer2_attention_weight_max": 30.65089988708496,
      "activations/layer2_attention_weight_min": -27.51511001586914,
      "activations/layer3_attention_weight_max": 85.05862426757812,
      "activations/layer3_attention_weight_min": -87.45845031738281,
      "activations/layer4_attention_weight_max": 99.13431549072266,
      "activations/layer4_attention_weight_min": -92.31194305419922,
      "activations/layer5_attention_weight_max": 83.93991088867188,
      "activations/layer5_attention_weight_min": -77.42090606689453,
      "activations/layer6_attention_weight_max": 52.76652526855469,
      "activations/layer6_attention_weight_min": -50.51793670654297,
      "activations/layer7_attention_weight_max": 74.7758560180664,
      "activations/layer7_attention_weight_min": -68.99915313720703,
      "activations/layer8_attention_weight_max": 42.84231185913086,
      "activations/layer8_attention_weight_min": -44.25230407714844,
      "activations/layer9_attention_weight_max": 48.37419509887695,
      "activations/layer9_attention_weight_min": -48.406646728515625,
      "epoch": 8.11,
      "learning_rate": 9.870340909090908e-05,
      "loss": 2.8115,
      "step": 139550
    },
    {
      "activations/layer0_attention_weight_max": 14.708261489868164,
      "activations/layer0_attention_weight_min": -14.45369815826416,
      "activations/layer10_attention_weight_max": 40.66217803955078,
      "activations/layer10_attention_weight_min": -36.78252029418945,
      "activations/layer11_attention_weight_max": 34.026023864746094,
      "activations/layer11_attention_weight_min": -31.377830505371094,
      "activations/layer12_attention_weight_max": 18.14093780517578,
      "activations/layer12_attention_weight_min": -27.674320220947266,
      "activations/layer13_attention_weight_max": 43.08573532104492,
      "activations/layer13_attention_weight_min": -32.838409423828125,
      "activations/layer14_attention_weight_max": 38.520198822021484,
      "activations/layer14_attention_weight_min": -30.94396209716797,
      "activations/layer15_attention_weight_max": 35.923030853271484,
      "activations/layer15_attention_weight_min": -30.294490814208984,
      "activations/layer16_attention_weight_max": 35.517967224121094,
      "activations/layer16_attention_weight_min": -27.544113159179688,
      "activations/layer17_attention_weight_max": 52.65913009643555,
      "activations/layer17_attention_weight_min": -42.34914016723633,
      "activations/layer18_attention_weight_max": 49.38148880004883,
      "activations/layer18_attention_weight_min": -36.619869232177734,
      "activations/layer19_attention_weight_max": 21.486352920532227,
      "activations/layer19_attention_weight_min": -22.2386531829834,
      "activations/layer1_attention_weight_max": 17.95661735534668,
      "activations/layer1_attention_weight_min": -16.995466232299805,
      "activations/layer20_attention_weight_max": 22.280601501464844,
      "activations/layer20_attention_weight_min": -24.6960391998291,
      "activations/layer21_attention_weight_max": 42.314842224121094,
      "activations/layer21_attention_weight_min": -24.548521041870117,
      "activations/layer22_attention_weight_max": 32.13654327392578,
      "activations/layer22_attention_weight_min": -25.831607818603516,
      "activations/layer23_attention_weight_max": 34.52463150024414,
      "activations/layer23_attention_weight_min": -21.46612548828125,
      "activations/layer2_attention_weight_max": 32.327545166015625,
      "activations/layer2_attention_weight_min": -29.345396041870117,
      "activations/layer3_attention_weight_max": 84.25079345703125,
      "activations/layer3_attention_weight_min": -84.55384063720703,
      "activations/layer4_attention_weight_max": 100.54796600341797,
      "activations/layer4_attention_weight_min": -90.83328247070312,
      "activations/layer5_attention_weight_max": 85.97557067871094,
      "activations/layer5_attention_weight_min": -77.8155517578125,
      "activations/layer6_attention_weight_max": 55.92793655395508,
      "activations/layer6_attention_weight_min": -53.10942840576172,
      "activations/layer7_attention_weight_max": 78.08836364746094,
      "activations/layer7_attention_weight_min": -71.44572448730469,
      "activations/layer8_attention_weight_max": 47.380191802978516,
      "activations/layer8_attention_weight_min": -45.961700439453125,
      "activations/layer9_attention_weight_max": 54.40855026245117,
      "activations/layer9_attention_weight_min": -49.92642593383789,
      "epoch": 8.11,
      "learning_rate": 9.868446969696969e-05,
      "loss": 2.8005,
      "step": 139600
    },
    {
      "activations/layer0_attention_weight_max": 15.105916976928711,
      "activations/layer0_attention_weight_min": -14.217329978942871,
      "activations/layer10_attention_weight_max": 38.505123138427734,
      "activations/layer10_attention_weight_min": -35.99615478515625,
      "activations/layer11_attention_weight_max": 31.11951446533203,
      "activations/layer11_attention_weight_min": -32.71906280517578,
      "activations/layer12_attention_weight_max": 20.476131439208984,
      "activations/layer12_attention_weight_min": -25.023353576660156,
      "activations/layer13_attention_weight_max": 40.644996643066406,
      "activations/layer13_attention_weight_min": -30.196382522583008,
      "activations/layer14_attention_weight_max": 37.3659553527832,
      "activations/layer14_attention_weight_min": -29.18885040283203,
      "activations/layer15_attention_weight_max": 34.21478271484375,
      "activations/layer15_attention_weight_min": -30.035032272338867,
      "activations/layer16_attention_weight_max": 28.896238327026367,
      "activations/layer16_attention_weight_min": -27.469383239746094,
      "activations/layer17_attention_weight_max": 56.14336395263672,
      "activations/layer17_attention_weight_min": -42.568912506103516,
      "activations/layer18_attention_weight_max": 44.52841567993164,
      "activations/layer18_attention_weight_min": -35.825225830078125,
      "activations/layer19_attention_weight_max": 21.81291961669922,
      "activations/layer19_attention_weight_min": -21.641447067260742,
      "activations/layer1_attention_weight_max": 17.28367805480957,
      "activations/layer1_attention_weight_min": -16.74600601196289,
      "activations/layer20_attention_weight_max": 21.325286865234375,
      "activations/layer20_attention_weight_min": -21.64349365234375,
      "activations/layer21_attention_weight_max": 42.17290496826172,
      "activations/layer21_attention_weight_min": -25.110824584960938,
      "activations/layer22_attention_weight_max": 30.074642181396484,
      "activations/layer22_attention_weight_min": -25.26754379272461,
      "activations/layer23_attention_weight_max": 35.87518310546875,
      "activations/layer23_attention_weight_min": -21.259939193725586,
      "activations/layer2_attention_weight_max": 31.104942321777344,
      "activations/layer2_attention_weight_min": -28.532466888427734,
      "activations/layer3_attention_weight_max": 84.04768371582031,
      "activations/layer3_attention_weight_min": -87.45936584472656,
      "activations/layer4_attention_weight_max": 98.59513092041016,
      "activations/layer4_attention_weight_min": -94.22510528564453,
      "activations/layer5_attention_weight_max": 83.03012084960938,
      "activations/layer5_attention_weight_min": -74.54005432128906,
      "activations/layer6_attention_weight_max": 59.34025192260742,
      "activations/layer6_attention_weight_min": -54.701908111572266,
      "activations/layer7_attention_weight_max": 76.84273529052734,
      "activations/layer7_attention_weight_min": -73.66661071777344,
      "activations/layer8_attention_weight_max": 45.858097076416016,
      "activations/layer8_attention_weight_min": -48.299110412597656,
      "activations/layer9_attention_weight_max": 54.225032806396484,
      "activations/layer9_attention_weight_min": -50.724666595458984,
      "epoch": 8.11,
      "learning_rate": 9.86655303030303e-05,
      "loss": 2.8028,
      "step": 139650
    },
    {
      "activations/layer0_attention_weight_max": 14.135307312011719,
      "activations/layer0_attention_weight_min": -14.60239028930664,
      "activations/layer10_attention_weight_max": 39.225379943847656,
      "activations/layer10_attention_weight_min": -37.8929443359375,
      "activations/layer11_attention_weight_max": 32.053409576416016,
      "activations/layer11_attention_weight_min": -31.609947204589844,
      "activations/layer12_attention_weight_max": 19.46826171875,
      "activations/layer12_attention_weight_min": -23.572349548339844,
      "activations/layer13_attention_weight_max": 44.79587936401367,
      "activations/layer13_attention_weight_min": -32.21327590942383,
      "activations/layer14_attention_weight_max": 40.50550079345703,
      "activations/layer14_attention_weight_min": -29.97999382019043,
      "activations/layer15_attention_weight_max": 36.21647644042969,
      "activations/layer15_attention_weight_min": -30.36667823791504,
      "activations/layer16_attention_weight_max": 32.38119125366211,
      "activations/layer16_attention_weight_min": -27.5998592376709,
      "activations/layer17_attention_weight_max": 53.951263427734375,
      "activations/layer17_attention_weight_min": -42.670894622802734,
      "activations/layer18_attention_weight_max": 45.69597625732422,
      "activations/layer18_attention_weight_min": -35.21503829956055,
      "activations/layer19_attention_weight_max": 21.058555603027344,
      "activations/layer19_attention_weight_min": -20.773542404174805,
      "activations/layer1_attention_weight_max": 16.929704666137695,
      "activations/layer1_attention_weight_min": -15.437409400939941,
      "activations/layer20_attention_weight_max": 22.351612091064453,
      "activations/layer20_attention_weight_min": -21.593341827392578,
      "activations/layer21_attention_weight_max": 38.06499099731445,
      "activations/layer21_attention_weight_min": -26.761394500732422,
      "activations/layer22_attention_weight_max": 29.10347557067871,
      "activations/layer22_attention_weight_min": -25.162996292114258,
      "activations/layer23_attention_weight_max": 35.1694221496582,
      "activations/layer23_attention_weight_min": -23.047760009765625,
      "activations/layer2_attention_weight_max": 29.22511100769043,
      "activations/layer2_attention_weight_min": -27.25507354736328,
      "activations/layer3_attention_weight_max": 82.99016571044922,
      "activations/layer3_attention_weight_min": -85.05846405029297,
      "activations/layer4_attention_weight_max": 99.92769622802734,
      "activations/layer4_attention_weight_min": -96.22777557373047,
      "activations/layer5_attention_weight_max": 81.64219665527344,
      "activations/layer5_attention_weight_min": -78.93831634521484,
      "activations/layer6_attention_weight_max": 55.00273132324219,
      "activations/layer6_attention_weight_min": -53.8326530456543,
      "activations/layer7_attention_weight_max": 75.70438385009766,
      "activations/layer7_attention_weight_min": -73.23934173583984,
      "activations/layer8_attention_weight_max": 46.371543884277344,
      "activations/layer8_attention_weight_min": -48.28314971923828,
      "activations/layer9_attention_weight_max": 51.328948974609375,
      "activations/layer9_attention_weight_min": -49.55385208129883,
      "epoch": 8.12,
      "learning_rate": 9.86465909090909e-05,
      "loss": 2.8225,
      "step": 139700
    },
    {
      "activations/layer0_attention_weight_max": 15.401540756225586,
      "activations/layer0_attention_weight_min": -13.7496919631958,
      "activations/layer10_attention_weight_max": 37.98867416381836,
      "activations/layer10_attention_weight_min": -37.516700744628906,
      "activations/layer11_attention_weight_max": 31.260549545288086,
      "activations/layer11_attention_weight_min": -30.85354995727539,
      "activations/layer12_attention_weight_max": 18.75943946838379,
      "activations/layer12_attention_weight_min": -25.41937828063965,
      "activations/layer13_attention_weight_max": 49.220252990722656,
      "activations/layer13_attention_weight_min": -34.17793273925781,
      "activations/layer14_attention_weight_max": 45.104156494140625,
      "activations/layer14_attention_weight_min": -31.553781509399414,
      "activations/layer15_attention_weight_max": 39.14923095703125,
      "activations/layer15_attention_weight_min": -29.132898330688477,
      "activations/layer16_attention_weight_max": 33.7686767578125,
      "activations/layer16_attention_weight_min": -28.5264835357666,
      "activations/layer17_attention_weight_max": 57.59198760986328,
      "activations/layer17_attention_weight_min": -44.590850830078125,
      "activations/layer18_attention_weight_max": 53.9354362487793,
      "activations/layer18_attention_weight_min": -36.37276077270508,
      "activations/layer19_attention_weight_max": 23.12067985534668,
      "activations/layer19_attention_weight_min": -22.306032180786133,
      "activations/layer1_attention_weight_max": 16.378034591674805,
      "activations/layer1_attention_weight_min": -15.2614164352417,
      "activations/layer20_attention_weight_max": 25.9410457611084,
      "activations/layer20_attention_weight_min": -21.065128326416016,
      "activations/layer21_attention_weight_max": 42.934139251708984,
      "activations/layer21_attention_weight_min": -23.435497283935547,
      "activations/layer22_attention_weight_max": 35.00190353393555,
      "activations/layer22_attention_weight_min": -25.793378829956055,
      "activations/layer23_attention_weight_max": 36.07472229003906,
      "activations/layer23_attention_weight_min": -23.51263999938965,
      "activations/layer2_attention_weight_max": 30.3232479095459,
      "activations/layer2_attention_weight_min": -28.41937828063965,
      "activations/layer3_attention_weight_max": 82.42766571044922,
      "activations/layer3_attention_weight_min": -87.31367492675781,
      "activations/layer4_attention_weight_max": 97.86730194091797,
      "activations/layer4_attention_weight_min": -95.49434661865234,
      "activations/layer5_attention_weight_max": 82.64778137207031,
      "activations/layer5_attention_weight_min": -77.61760711669922,
      "activations/layer6_attention_weight_max": 53.99409866333008,
      "activations/layer6_attention_weight_min": -51.69400405883789,
      "activations/layer7_attention_weight_max": 74.94422149658203,
      "activations/layer7_attention_weight_min": -73.03832244873047,
      "activations/layer8_attention_weight_max": 44.53235626220703,
      "activations/layer8_attention_weight_min": -46.67816162109375,
      "activations/layer9_attention_weight_max": 49.50061798095703,
      "activations/layer9_attention_weight_min": -50.41086196899414,
      "epoch": 8.12,
      "learning_rate": 9.862765151515151e-05,
      "loss": 2.7961,
      "step": 139750
    },
    {
      "activations/layer0_attention_weight_max": 14.064497947692871,
      "activations/layer0_attention_weight_min": -14.5516939163208,
      "activations/layer10_attention_weight_max": 42.23955154418945,
      "activations/layer10_attention_weight_min": -39.23357009887695,
      "activations/layer11_attention_weight_max": 33.63138198852539,
      "activations/layer11_attention_weight_min": -32.164127349853516,
      "activations/layer12_attention_weight_max": 23.14066505432129,
      "activations/layer12_attention_weight_min": -28.39446258544922,
      "activations/layer13_attention_weight_max": 61.131473541259766,
      "activations/layer13_attention_weight_min": -38.178932189941406,
      "activations/layer14_attention_weight_max": 59.182674407958984,
      "activations/layer14_attention_weight_min": -37.127052307128906,
      "activations/layer15_attention_weight_max": 44.550071716308594,
      "activations/layer15_attention_weight_min": -30.38134765625,
      "activations/layer16_attention_weight_max": 34.819007873535156,
      "activations/layer16_attention_weight_min": -30.77687644958496,
      "activations/layer17_attention_weight_max": 58.65967559814453,
      "activations/layer17_attention_weight_min": -47.81052017211914,
      "activations/layer18_attention_weight_max": 55.97793960571289,
      "activations/layer18_attention_weight_min": -39.69363021850586,
      "activations/layer19_attention_weight_max": 23.19561004638672,
      "activations/layer19_attention_weight_min": -21.402973175048828,
      "activations/layer1_attention_weight_max": 17.895437240600586,
      "activations/layer1_attention_weight_min": -19.752708435058594,
      "activations/layer20_attention_weight_max": 27.926401138305664,
      "activations/layer20_attention_weight_min": -24.98086166381836,
      "activations/layer21_attention_weight_max": 45.8594970703125,
      "activations/layer21_attention_weight_min": -25.212020874023438,
      "activations/layer22_attention_weight_max": 36.802757263183594,
      "activations/layer22_attention_weight_min": -26.66124725341797,
      "activations/layer23_attention_weight_max": 40.029022216796875,
      "activations/layer23_attention_weight_min": -25.195587158203125,
      "activations/layer2_attention_weight_max": 32.248252868652344,
      "activations/layer2_attention_weight_min": -30.288536071777344,
      "activations/layer3_attention_weight_max": 86.75389862060547,
      "activations/layer3_attention_weight_min": -90.86962890625,
      "activations/layer4_attention_weight_max": 106.83210754394531,
      "activations/layer4_attention_weight_min": -104.1718521118164,
      "activations/layer5_attention_weight_max": 90.35213470458984,
      "activations/layer5_attention_weight_min": -87.09873962402344,
      "activations/layer6_attention_weight_max": 58.186893463134766,
      "activations/layer6_attention_weight_min": -59.647216796875,
      "activations/layer7_attention_weight_max": 82.39766693115234,
      "activations/layer7_attention_weight_min": -81.07305145263672,
      "activations/layer8_attention_weight_max": 53.140602111816406,
      "activations/layer8_attention_weight_min": -49.43423080444336,
      "activations/layer9_attention_weight_max": 62.933589935302734,
      "activations/layer9_attention_weight_min": -52.6954345703125,
      "epoch": 8.12,
      "learning_rate": 9.860871212121211e-05,
      "loss": 2.8186,
      "step": 139800
    },
    {
      "activations/layer0_attention_weight_max": 14.596405982971191,
      "activations/layer0_attention_weight_min": -14.49814224243164,
      "activations/layer10_attention_weight_max": 39.46299743652344,
      "activations/layer10_attention_weight_min": -37.632137298583984,
      "activations/layer11_attention_weight_max": 31.41014862060547,
      "activations/layer11_attention_weight_min": -33.33798599243164,
      "activations/layer12_attention_weight_max": 21.15302848815918,
      "activations/layer12_attention_weight_min": -24.927183151245117,
      "activations/layer13_attention_weight_max": 47.90718078613281,
      "activations/layer13_attention_weight_min": -33.4472541809082,
      "activations/layer14_attention_weight_max": 48.47844314575195,
      "activations/layer14_attention_weight_min": -33.00799560546875,
      "activations/layer15_attention_weight_max": 39.73856735229492,
      "activations/layer15_attention_weight_min": -30.214719772338867,
      "activations/layer16_attention_weight_max": 35.402976989746094,
      "activations/layer16_attention_weight_min": -30.19631004333496,
      "activations/layer17_attention_weight_max": 60.21942901611328,
      "activations/layer17_attention_weight_min": -49.14102554321289,
      "activations/layer18_attention_weight_max": 52.71343231201172,
      "activations/layer18_attention_weight_min": -38.96839141845703,
      "activations/layer19_attention_weight_max": 24.41461181640625,
      "activations/layer19_attention_weight_min": -23.87754249572754,
      "activations/layer1_attention_weight_max": 16.858137130737305,
      "activations/layer1_attention_weight_min": -15.894538879394531,
      "activations/layer20_attention_weight_max": 28.383792877197266,
      "activations/layer20_attention_weight_min": -21.056011199951172,
      "activations/layer21_attention_weight_max": 43.3935546875,
      "activations/layer21_attention_weight_min": -24.13152313232422,
      "activations/layer22_attention_weight_max": 32.880619049072266,
      "activations/layer22_attention_weight_min": -24.330333709716797,
      "activations/layer23_attention_weight_max": 41.479644775390625,
      "activations/layer23_attention_weight_min": -21.93682861328125,
      "activations/layer2_attention_weight_max": 30.028213500976562,
      "activations/layer2_attention_weight_min": -27.472684860229492,
      "activations/layer3_attention_weight_max": 85.45732116699219,
      "activations/layer3_attention_weight_min": -82.7690200805664,
      "activations/layer4_attention_weight_max": 93.8604507446289,
      "activations/layer4_attention_weight_min": -89.46810150146484,
      "activations/layer5_attention_weight_max": 77.57025146484375,
      "activations/layer5_attention_weight_min": -74.63565063476562,
      "activations/layer6_attention_weight_max": 53.26559066772461,
      "activations/layer6_attention_weight_min": -49.852115631103516,
      "activations/layer7_attention_weight_max": 73.17933654785156,
      "activations/layer7_attention_weight_min": -71.7672348022461,
      "activations/layer8_attention_weight_max": 46.15829849243164,
      "activations/layer8_attention_weight_min": -44.08656311035156,
      "activations/layer9_attention_weight_max": 53.48919677734375,
      "activations/layer9_attention_weight_min": -49.92922592163086,
      "epoch": 8.13,
      "learning_rate": 9.858977272727273e-05,
      "loss": 2.8116,
      "step": 139850
    },
    {
      "activations/layer0_attention_weight_max": 15.256587982177734,
      "activations/layer0_attention_weight_min": -14.080431938171387,
      "activations/layer10_attention_weight_max": 38.58401107788086,
      "activations/layer10_attention_weight_min": -36.33948516845703,
      "activations/layer11_attention_weight_max": 31.155149459838867,
      "activations/layer11_attention_weight_min": -31.732587814331055,
      "activations/layer12_attention_weight_max": 19.282291412353516,
      "activations/layer12_attention_weight_min": -24.693622589111328,
      "activations/layer13_attention_weight_max": 37.8653678894043,
      "activations/layer13_attention_weight_min": -31.449729919433594,
      "activations/layer14_attention_weight_max": 35.17807388305664,
      "activations/layer14_attention_weight_min": -30.594892501831055,
      "activations/layer15_attention_weight_max": 33.72351837158203,
      "activations/layer15_attention_weight_min": -30.11897087097168,
      "activations/layer16_attention_weight_max": 28.76024055480957,
      "activations/layer16_attention_weight_min": -27.47751235961914,
      "activations/layer17_attention_weight_max": 50.58911895751953,
      "activations/layer17_attention_weight_min": -43.19681167602539,
      "activations/layer18_attention_weight_max": 44.6385498046875,
      "activations/layer18_attention_weight_min": -37.09654998779297,
      "activations/layer19_attention_weight_max": 26.678327560424805,
      "activations/layer19_attention_weight_min": -22.126449584960938,
      "activations/layer1_attention_weight_max": 17.156478881835938,
      "activations/layer1_attention_weight_min": -15.687314987182617,
      "activations/layer20_attention_weight_max": 22.93903160095215,
      "activations/layer20_attention_weight_min": -23.49782943725586,
      "activations/layer21_attention_weight_max": 33.076080322265625,
      "activations/layer21_attention_weight_min": -24.326416015625,
      "activations/layer22_attention_weight_max": 28.53334617614746,
      "activations/layer22_attention_weight_min": -24.08622932434082,
      "activations/layer23_attention_weight_max": 34.4329948425293,
      "activations/layer23_attention_weight_min": -22.04220962524414,
      "activations/layer2_attention_weight_max": 31.727737426757812,
      "activations/layer2_attention_weight_min": -29.28118133544922,
      "activations/layer3_attention_weight_max": 84.04813385009766,
      "activations/layer3_attention_weight_min": -84.1666488647461,
      "activations/layer4_attention_weight_max": 98.75870513916016,
      "activations/layer4_attention_weight_min": -90.70687103271484,
      "activations/layer5_attention_weight_max": 80.18817138671875,
      "activations/layer5_attention_weight_min": -75.1954116821289,
      "activations/layer6_attention_weight_max": 55.16699981689453,
      "activations/layer6_attention_weight_min": -51.70850372314453,
      "activations/layer7_attention_weight_max": 78.38658905029297,
      "activations/layer7_attention_weight_min": -71.42327117919922,
      "activations/layer8_attention_weight_max": 45.71629333496094,
      "activations/layer8_attention_weight_min": -45.489479064941406,
      "activations/layer9_attention_weight_max": 51.31385040283203,
      "activations/layer9_attention_weight_min": -49.31252670288086,
      "epoch": 8.13,
      "learning_rate": 9.857083333333333e-05,
      "loss": 2.8132,
      "step": 139900
    },
    {
      "activations/layer0_attention_weight_max": 15.338027000427246,
      "activations/layer0_attention_weight_min": -14.413928031921387,
      "activations/layer10_attention_weight_max": 43.31367111206055,
      "activations/layer10_attention_weight_min": -38.384368896484375,
      "activations/layer11_attention_weight_max": 37.09680938720703,
      "activations/layer11_attention_weight_min": -32.20452117919922,
      "activations/layer12_attention_weight_max": 24.5262393951416,
      "activations/layer12_attention_weight_min": -25.190921783447266,
      "activations/layer13_attention_weight_max": 56.95280456542969,
      "activations/layer13_attention_weight_min": -38.22224044799805,
      "activations/layer14_attention_weight_max": 50.94911575317383,
      "activations/layer14_attention_weight_min": -29.93018341064453,
      "activations/layer15_attention_weight_max": 41.01868438720703,
      "activations/layer15_attention_weight_min": -29.7822208404541,
      "activations/layer16_attention_weight_max": 39.8739128112793,
      "activations/layer16_attention_weight_min": -28.72686195373535,
      "activations/layer17_attention_weight_max": 61.42379379272461,
      "activations/layer17_attention_weight_min": -45.29600524902344,
      "activations/layer18_attention_weight_max": 58.494911193847656,
      "activations/layer18_attention_weight_min": -40.38987350463867,
      "activations/layer19_attention_weight_max": 27.276540756225586,
      "activations/layer19_attention_weight_min": -23.71079444885254,
      "activations/layer1_attention_weight_max": 16.565040588378906,
      "activations/layer1_attention_weight_min": -15.486833572387695,
      "activations/layer20_attention_weight_max": 29.060821533203125,
      "activations/layer20_attention_weight_min": -22.27814292907715,
      "activations/layer21_attention_weight_max": 51.86801528930664,
      "activations/layer21_attention_weight_min": -26.631120681762695,
      "activations/layer22_attention_weight_max": 34.478111267089844,
      "activations/layer22_attention_weight_min": -23.70362091064453,
      "activations/layer23_attention_weight_max": 37.73363494873047,
      "activations/layer23_attention_weight_min": -23.46705436706543,
      "activations/layer2_attention_weight_max": 32.792236328125,
      "activations/layer2_attention_weight_min": -30.32477569580078,
      "activations/layer3_attention_weight_max": 84.17102813720703,
      "activations/layer3_attention_weight_min": -88.09803009033203,
      "activations/layer4_attention_weight_max": 98.84175872802734,
      "activations/layer4_attention_weight_min": -99.30644989013672,
      "activations/layer5_attention_weight_max": 84.44866943359375,
      "activations/layer5_attention_weight_min": -79.24876403808594,
      "activations/layer6_attention_weight_max": 55.2977180480957,
      "activations/layer6_attention_weight_min": -54.664146423339844,
      "activations/layer7_attention_weight_max": 77.53166198730469,
      "activations/layer7_attention_weight_min": -75.31854248046875,
      "activations/layer8_attention_weight_max": 47.822364807128906,
      "activations/layer8_attention_weight_min": -47.75575637817383,
      "activations/layer9_attention_weight_max": 56.88728713989258,
      "activations/layer9_attention_weight_min": -51.057010650634766,
      "epoch": 8.13,
      "learning_rate": 9.855189393939393e-05,
      "loss": 2.8138,
      "step": 139950
    },
    {
      "activations/layer0_attention_weight_max": 15.561485290527344,
      "activations/layer0_attention_weight_min": -14.256818771362305,
      "activations/layer10_attention_weight_max": 38.80605697631836,
      "activations/layer10_attention_weight_min": -37.52674865722656,
      "activations/layer11_attention_weight_max": 31.024768829345703,
      "activations/layer11_attention_weight_min": -30.4338436126709,
      "activations/layer12_attention_weight_max": 17.95786476135254,
      "activations/layer12_attention_weight_min": -25.830524444580078,
      "activations/layer13_attention_weight_max": 37.76811599731445,
      "activations/layer13_attention_weight_min": -28.6636905670166,
      "activations/layer14_attention_weight_max": 36.22393798828125,
      "activations/layer14_attention_weight_min": -31.410844802856445,
      "activations/layer15_attention_weight_max": 32.25103759765625,
      "activations/layer15_attention_weight_min": -31.05961799621582,
      "activations/layer16_attention_weight_max": 29.12148666381836,
      "activations/layer16_attention_weight_min": -27.896026611328125,
      "activations/layer17_attention_weight_max": 53.58133316040039,
      "activations/layer17_attention_weight_min": -43.3851203918457,
      "activations/layer18_attention_weight_max": 47.9412841796875,
      "activations/layer18_attention_weight_min": -35.46849822998047,
      "activations/layer19_attention_weight_max": 21.601215362548828,
      "activations/layer19_attention_weight_min": -23.050020217895508,
      "activations/layer1_attention_weight_max": 17.04877471923828,
      "activations/layer1_attention_weight_min": -14.795516014099121,
      "activations/layer20_attention_weight_max": 20.999425888061523,
      "activations/layer20_attention_weight_min": -24.1987361907959,
      "activations/layer21_attention_weight_max": 34.250160217285156,
      "activations/layer21_attention_weight_min": -23.8948917388916,
      "activations/layer22_attention_weight_max": 33.83498764038086,
      "activations/layer22_attention_weight_min": -25.78606414794922,
      "activations/layer23_attention_weight_max": 32.98809051513672,
      "activations/layer23_attention_weight_min": -23.019786834716797,
      "activations/layer2_attention_weight_max": 30.5484676361084,
      "activations/layer2_attention_weight_min": -28.139934539794922,
      "activations/layer3_attention_weight_max": 84.48432159423828,
      "activations/layer3_attention_weight_min": -87.74418640136719,
      "activations/layer4_attention_weight_max": 101.91275787353516,
      "activations/layer4_attention_weight_min": -94.95832061767578,
      "activations/layer5_attention_weight_max": 80.97065734863281,
      "activations/layer5_attention_weight_min": -79.84263610839844,
      "activations/layer6_attention_weight_max": 56.47142028808594,
      "activations/layer6_attention_weight_min": -54.38007354736328,
      "activations/layer7_attention_weight_max": 75.19598388671875,
      "activations/layer7_attention_weight_min": -73.54663848876953,
      "activations/layer8_attention_weight_max": 45.7451057434082,
      "activations/layer8_attention_weight_min": -45.0021858215332,
      "activations/layer9_attention_weight_max": 51.06242752075195,
      "activations/layer9_attention_weight_min": -49.104251861572266,
      "epoch": 8.13,
      "learning_rate": 9.853295454545454e-05,
      "loss": 2.8166,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.575,
      "eval_samples_per_second": 500.76,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.575,
      "eval_openwebtext_samples_per_second": 500.76,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 2.0339,
      "eval_wikitext_samples_per_second": 224.199,
      "step": 140000
    },
    {
      "epoch": 8.13,
      "eval_lambada_loss": 2.7109375,
      "eval_lambada_ppl": 15.04337206739713,
      "eval_lambada_runtime": 9.5919,
      "eval_lambada_samples_per_second": 507.615,
      "step": 140000
    },
    {
      "activations/layer0_attention_weight_max": 13.965557098388672,
      "activations/layer0_attention_weight_min": -14.45364761352539,
      "activations/layer10_attention_weight_max": 37.56044006347656,
      "activations/layer10_attention_weight_min": -35.41851806640625,
      "activations/layer11_attention_weight_max": 30.49611473083496,
      "activations/layer11_attention_weight_min": -31.47450828552246,
      "activations/layer12_attention_weight_max": 22.070453643798828,
      "activations/layer12_attention_weight_min": -25.977046966552734,
      "activations/layer13_attention_weight_max": 37.68437957763672,
      "activations/layer13_attention_weight_min": -32.146263122558594,
      "activations/layer14_attention_weight_max": 35.87220001220703,
      "activations/layer14_attention_weight_min": -30.682064056396484,
      "activations/layer15_attention_weight_max": 35.01694869995117,
      "activations/layer15_attention_weight_min": -28.54990005493164,
      "activations/layer16_attention_weight_max": 35.942291259765625,
      "activations/layer16_attention_weight_min": -27.704395294189453,
      "activations/layer17_attention_weight_max": 51.97874450683594,
      "activations/layer17_attention_weight_min": -40.31749725341797,
      "activations/layer18_attention_weight_max": 46.14238739013672,
      "activations/layer18_attention_weight_min": -35.307064056396484,
      "activations/layer19_attention_weight_max": 19.747594833374023,
      "activations/layer19_attention_weight_min": -20.76859474182129,
      "activations/layer1_attention_weight_max": 16.297170639038086,
      "activations/layer1_attention_weight_min": -14.534627914428711,
      "activations/layer20_attention_weight_max": 21.648021697998047,
      "activations/layer20_attention_weight_min": -21.192827224731445,
      "activations/layer21_attention_weight_max": 41.9453239440918,
      "activations/layer21_attention_weight_min": -24.467153549194336,
      "activations/layer22_attention_weight_max": 32.5116081237793,
      "activations/layer22_attention_weight_min": -24.8383846282959,
      "activations/layer23_attention_weight_max": 35.038002014160156,
      "activations/layer23_attention_weight_min": -23.073413848876953,
      "activations/layer2_attention_weight_max": 32.77751159667969,
      "activations/layer2_attention_weight_min": -29.923797607421875,
      "activations/layer3_attention_weight_max": 85.6937026977539,
      "activations/layer3_attention_weight_min": -91.38154602050781,
      "activations/layer4_attention_weight_max": 100.77169799804688,
      "activations/layer4_attention_weight_min": -94.09068298339844,
      "activations/layer5_attention_weight_max": 82.33058166503906,
      "activations/layer5_attention_weight_min": -78.7144546508789,
      "activations/layer6_attention_weight_max": 57.162025451660156,
      "activations/layer6_attention_weight_min": -53.337730407714844,
      "activations/layer7_attention_weight_max": 79.17204284667969,
      "activations/layer7_attention_weight_min": -75.18354034423828,
      "activations/layer8_attention_weight_max": 48.346717834472656,
      "activations/layer8_attention_weight_min": -45.83049392700195,
      "activations/layer9_attention_weight_max": 49.07368469238281,
      "activations/layer9_attention_weight_min": -50.136878967285156,
      "epoch": 8.14,
      "learning_rate": 9.851401515151513e-05,
      "loss": 2.8036,
      "step": 140050
    },
    {
      "activations/layer0_attention_weight_max": 15.59666633605957,
      "activations/layer0_attention_weight_min": -14.0576810836792,
      "activations/layer10_attention_weight_max": 37.996124267578125,
      "activations/layer10_attention_weight_min": -36.81629180908203,
      "activations/layer11_attention_weight_max": 31.5672607421875,
      "activations/layer11_attention_weight_min": -31.567197799682617,
      "activations/layer12_attention_weight_max": 25.810571670532227,
      "activations/layer12_attention_weight_min": -29.52368927001953,
      "activations/layer13_attention_weight_max": 41.569488525390625,
      "activations/layer13_attention_weight_min": -31.656265258789062,
      "activations/layer14_attention_weight_max": 39.160282135009766,
      "activations/layer14_attention_weight_min": -31.141271591186523,
      "activations/layer15_attention_weight_max": 36.45661544799805,
      "activations/layer15_attention_weight_min": -31.76713752746582,
      "activations/layer16_attention_weight_max": 34.1239128112793,
      "activations/layer16_attention_weight_min": -29.106082916259766,
      "activations/layer17_attention_weight_max": 53.6329345703125,
      "activations/layer17_attention_weight_min": -45.50442886352539,
      "activations/layer18_attention_weight_max": 49.826438903808594,
      "activations/layer18_attention_weight_min": -37.673301696777344,
      "activations/layer19_attention_weight_max": 20.08574104309082,
      "activations/layer19_attention_weight_min": -25.473825454711914,
      "activations/layer1_attention_weight_max": 17.540355682373047,
      "activations/layer1_attention_weight_min": -16.458894729614258,
      "activations/layer20_attention_weight_max": 20.996402740478516,
      "activations/layer20_attention_weight_min": -25.986234664916992,
      "activations/layer21_attention_weight_max": 37.477882385253906,
      "activations/layer21_attention_weight_min": -24.487146377563477,
      "activations/layer22_attention_weight_max": 37.3715705871582,
      "activations/layer22_attention_weight_min": -26.31500244140625,
      "activations/layer23_attention_weight_max": 34.00367736816406,
      "activations/layer23_attention_weight_min": -23.329750061035156,
      "activations/layer2_attention_weight_max": 30.12257194519043,
      "activations/layer2_attention_weight_min": -29.589330673217773,
      "activations/layer3_attention_weight_max": 83.14531707763672,
      "activations/layer3_attention_weight_min": -90.25232696533203,
      "activations/layer4_attention_weight_max": 100.09269714355469,
      "activations/layer4_attention_weight_min": -97.24225616455078,
      "activations/layer5_attention_weight_max": 82.99470520019531,
      "activations/layer5_attention_weight_min": -79.70723724365234,
      "activations/layer6_attention_weight_max": 56.60456848144531,
      "activations/layer6_attention_weight_min": -52.18926239013672,
      "activations/layer7_attention_weight_max": 78.218994140625,
      "activations/layer7_attention_weight_min": -75.40377044677734,
      "activations/layer8_attention_weight_max": 45.21923828125,
      "activations/layer8_attention_weight_min": -43.786598205566406,
      "activations/layer9_attention_weight_max": 53.991817474365234,
      "activations/layer9_attention_weight_min": -49.56210708618164,
      "epoch": 8.14,
      "learning_rate": 9.849507575757575e-05,
      "loss": 2.808,
      "step": 140100
    },
    {
      "activations/layer0_attention_weight_max": 15.850902557373047,
      "activations/layer0_attention_weight_min": -14.730912208557129,
      "activations/layer10_attention_weight_max": 37.737518310546875,
      "activations/layer10_attention_weight_min": -36.51942825317383,
      "activations/layer11_attention_weight_max": 30.271642684936523,
      "activations/layer11_attention_weight_min": -30.43341636657715,
      "activations/layer12_attention_weight_max": 21.50140953063965,
      "activations/layer12_attention_weight_min": -28.749475479125977,
      "activations/layer13_attention_weight_max": 44.251461029052734,
      "activations/layer13_attention_weight_min": -32.2352409362793,
      "activations/layer14_attention_weight_max": 37.120567321777344,
      "activations/layer14_attention_weight_min": -31.56886100769043,
      "activations/layer15_attention_weight_max": 31.661609649658203,
      "activations/layer15_attention_weight_min": -29.23101043701172,
      "activations/layer16_attention_weight_max": 31.358625411987305,
      "activations/layer16_attention_weight_min": -28.829944610595703,
      "activations/layer17_attention_weight_max": 53.219173431396484,
      "activations/layer17_attention_weight_min": -43.22492218017578,
      "activations/layer18_attention_weight_max": 48.189659118652344,
      "activations/layer18_attention_weight_min": -35.75753402709961,
      "activations/layer19_attention_weight_max": 22.359657287597656,
      "activations/layer19_attention_weight_min": -22.103025436401367,
      "activations/layer1_attention_weight_max": 17.206995010375977,
      "activations/layer1_attention_weight_min": -16.173988342285156,
      "activations/layer20_attention_weight_max": 23.328338623046875,
      "activations/layer20_attention_weight_min": -26.6203670501709,
      "activations/layer21_attention_weight_max": 38.72014236450195,
      "activations/layer21_attention_weight_min": -24.276153564453125,
      "activations/layer22_attention_weight_max": 35.13604736328125,
      "activations/layer22_attention_weight_min": -25.201801300048828,
      "activations/layer23_attention_weight_max": 37.23261260986328,
      "activations/layer23_attention_weight_min": -21.671428680419922,
      "activations/layer2_attention_weight_max": 32.5224609375,
      "activations/layer2_attention_weight_min": -29.96615982055664,
      "activations/layer3_attention_weight_max": 86.9429702758789,
      "activations/layer3_attention_weight_min": -85.58174133300781,
      "activations/layer4_attention_weight_max": 99.62296295166016,
      "activations/layer4_attention_weight_min": -94.1794204711914,
      "activations/layer5_attention_weight_max": 85.21878051757812,
      "activations/layer5_attention_weight_min": -79.36791229248047,
      "activations/layer6_attention_weight_max": 57.9001579284668,
      "activations/layer6_attention_weight_min": -54.49226379394531,
      "activations/layer7_attention_weight_max": 80.44092559814453,
      "activations/layer7_attention_weight_min": -73.13069915771484,
      "activations/layer8_attention_weight_max": 46.90985107421875,
      "activations/layer8_attention_weight_min": -46.59857940673828,
      "activations/layer9_attention_weight_max": 51.38758087158203,
      "activations/layer9_attention_weight_min": -47.03153610229492,
      "epoch": 8.14,
      "learning_rate": 9.847613636363636e-05,
      "loss": 2.8023,
      "step": 140150
    },
    {
      "activations/layer0_attention_weight_max": 14.594844818115234,
      "activations/layer0_attention_weight_min": -14.901174545288086,
      "activations/layer10_attention_weight_max": 37.79328918457031,
      "activations/layer10_attention_weight_min": -34.909881591796875,
      "activations/layer11_attention_weight_max": 31.79024887084961,
      "activations/layer11_attention_weight_min": -32.33946228027344,
      "activations/layer12_attention_weight_max": 21.38015365600586,
      "activations/layer12_attention_weight_min": -25.30394744873047,
      "activations/layer13_attention_weight_max": 50.37592697143555,
      "activations/layer13_attention_weight_min": -30.343730926513672,
      "activations/layer14_attention_weight_max": 55.41074752807617,
      "activations/layer14_attention_weight_min": -31.53611946105957,
      "activations/layer15_attention_weight_max": 37.747718811035156,
      "activations/layer15_attention_weight_min": -29.16100311279297,
      "activations/layer16_attention_weight_max": 33.504554748535156,
      "activations/layer16_attention_weight_min": -25.61753273010254,
      "activations/layer17_attention_weight_max": 60.34759521484375,
      "activations/layer17_attention_weight_min": -41.885066986083984,
      "activations/layer18_attention_weight_max": 48.89167022705078,
      "activations/layer18_attention_weight_min": -36.27277755737305,
      "activations/layer19_attention_weight_max": 25.074003219604492,
      "activations/layer19_attention_weight_min": -19.917951583862305,
      "activations/layer1_attention_weight_max": 16.898311614990234,
      "activations/layer1_attention_weight_min": -14.523055076599121,
      "activations/layer20_attention_weight_max": 23.21987533569336,
      "activations/layer20_attention_weight_min": -21.207714080810547,
      "activations/layer21_attention_weight_max": 42.359222412109375,
      "activations/layer21_attention_weight_min": -23.06022834777832,
      "activations/layer22_attention_weight_max": 32.05482482910156,
      "activations/layer22_attention_weight_min": -23.60056495666504,
      "activations/layer23_attention_weight_max": 34.98707580566406,
      "activations/layer23_attention_weight_min": -22.534086227416992,
      "activations/layer2_attention_weight_max": 34.389549255371094,
      "activations/layer2_attention_weight_min": -33.67891311645508,
      "activations/layer3_attention_weight_max": 94.39137268066406,
      "activations/layer3_attention_weight_min": -101.20599365234375,
      "activations/layer4_attention_weight_max": 108.95379638671875,
      "activations/layer4_attention_weight_min": -106.0771484375,
      "activations/layer5_attention_weight_max": 81.33096313476562,
      "activations/layer5_attention_weight_min": -76.75037384033203,
      "activations/layer6_attention_weight_max": 54.73830795288086,
      "activations/layer6_attention_weight_min": -52.97624206542969,
      "activations/layer7_attention_weight_max": 77.17213439941406,
      "activations/layer7_attention_weight_min": -72.6548080444336,
      "activations/layer8_attention_weight_max": 44.32396697998047,
      "activations/layer8_attention_weight_min": -46.42293167114258,
      "activations/layer9_attention_weight_max": 56.83639144897461,
      "activations/layer9_attention_weight_min": -50.03212356567383,
      "epoch": 8.15,
      "learning_rate": 9.845719696969695e-05,
      "loss": 2.8135,
      "step": 140200
    },
    {
      "activations/layer0_attention_weight_max": 17.65308380126953,
      "activations/layer0_attention_weight_min": -14.886114120483398,
      "activations/layer10_attention_weight_max": 39.49354934692383,
      "activations/layer10_attention_weight_min": -34.667236328125,
      "activations/layer11_attention_weight_max": 33.614349365234375,
      "activations/layer11_attention_weight_min": -32.946128845214844,
      "activations/layer12_attention_weight_max": 20.28183364868164,
      "activations/layer12_attention_weight_min": -24.059350967407227,
      "activations/layer13_attention_weight_max": 44.21824645996094,
      "activations/layer13_attention_weight_min": -34.20577621459961,
      "activations/layer14_attention_weight_max": 38.327816009521484,
      "activations/layer14_attention_weight_min": -30.271692276000977,
      "activations/layer15_attention_weight_max": 35.54031753540039,
      "activations/layer15_attention_weight_min": -29.641618728637695,
      "activations/layer16_attention_weight_max": 33.65656661987305,
      "activations/layer16_attention_weight_min": -27.404293060302734,
      "activations/layer17_attention_weight_max": 50.25510025024414,
      "activations/layer17_attention_weight_min": -39.75590896606445,
      "activations/layer18_attention_weight_max": 45.448673248291016,
      "activations/layer18_attention_weight_min": -35.74164962768555,
      "activations/layer19_attention_weight_max": 21.26013946533203,
      "activations/layer19_attention_weight_min": -18.892475128173828,
      "activations/layer1_attention_weight_max": 17.826839447021484,
      "activations/layer1_attention_weight_min": -13.832976341247559,
      "activations/layer20_attention_weight_max": 22.896289825439453,
      "activations/layer20_attention_weight_min": -20.448772430419922,
      "activations/layer21_attention_weight_max": 36.61198043823242,
      "activations/layer21_attention_weight_min": -23.844995498657227,
      "activations/layer22_attention_weight_max": 31.81415367126465,
      "activations/layer22_attention_weight_min": -23.96760368347168,
      "activations/layer23_attention_weight_max": 35.149269104003906,
      "activations/layer23_attention_weight_min": -21.752349853515625,
      "activations/layer2_attention_weight_max": 29.748350143432617,
      "activations/layer2_attention_weight_min": -27.955562591552734,
      "activations/layer3_attention_weight_max": 82.12308502197266,
      "activations/layer3_attention_weight_min": -81.4312515258789,
      "activations/layer4_attention_weight_max": 93.90284729003906,
      "activations/layer4_attention_weight_min": -88.31575012207031,
      "activations/layer5_attention_weight_max": 80.84602355957031,
      "activations/layer5_attention_weight_min": -75.56682586669922,
      "activations/layer6_attention_weight_max": 55.732688903808594,
      "activations/layer6_attention_weight_min": -55.10426712036133,
      "activations/layer7_attention_weight_max": 76.83958435058594,
      "activations/layer7_attention_weight_min": -70.48140716552734,
      "activations/layer8_attention_weight_max": 43.52747344970703,
      "activations/layer8_attention_weight_min": -43.83445739746094,
      "activations/layer9_attention_weight_max": 47.77643966674805,
      "activations/layer9_attention_weight_min": -45.89097213745117,
      "epoch": 8.15,
      "learning_rate": 9.843825757575757e-05,
      "loss": 2.7947,
      "step": 140250
    },
    {
      "activations/layer0_attention_weight_max": 16.647165298461914,
      "activations/layer0_attention_weight_min": -15.12392807006836,
      "activations/layer10_attention_weight_max": 41.495304107666016,
      "activations/layer10_attention_weight_min": -37.57123565673828,
      "activations/layer11_attention_weight_max": 35.14139938354492,
      "activations/layer11_attention_weight_min": -34.16038513183594,
      "activations/layer12_attention_weight_max": 21.5540828704834,
      "activations/layer12_attention_weight_min": -29.394760131835938,
      "activations/layer13_attention_weight_max": 52.95951843261719,
      "activations/layer13_attention_weight_min": -34.8148307800293,
      "activations/layer14_attention_weight_max": 50.384864807128906,
      "activations/layer14_attention_weight_min": -32.9949836730957,
      "activations/layer15_attention_weight_max": 43.757957458496094,
      "activations/layer15_attention_weight_min": -31.193058013916016,
      "activations/layer16_attention_weight_max": 35.282588958740234,
      "activations/layer16_attention_weight_min": -31.405689239501953,
      "activations/layer17_attention_weight_max": 62.09345245361328,
      "activations/layer17_attention_weight_min": -47.56742477416992,
      "activations/layer18_attention_weight_max": 51.77302169799805,
      "activations/layer18_attention_weight_min": -40.53318786621094,
      "activations/layer19_attention_weight_max": 22.554094314575195,
      "activations/layer19_attention_weight_min": -23.434152603149414,
      "activations/layer1_attention_weight_max": 17.138713836669922,
      "activations/layer1_attention_weight_min": -16.12291717529297,
      "activations/layer20_attention_weight_max": 25.964738845825195,
      "activations/layer20_attention_weight_min": -21.472858428955078,
      "activations/layer21_attention_weight_max": 52.88002395629883,
      "activations/layer21_attention_weight_min": -28.04738998413086,
      "activations/layer22_attention_weight_max": 38.309226989746094,
      "activations/layer22_attention_weight_min": -27.69017219543457,
      "activations/layer23_attention_weight_max": 40.44049835205078,
      "activations/layer23_attention_weight_min": -26.873594284057617,
      "activations/layer2_attention_weight_max": 32.22734069824219,
      "activations/layer2_attention_weight_min": -28.98375701904297,
      "activations/layer3_attention_weight_max": 90.39800262451172,
      "activations/layer3_attention_weight_min": -87.5516586303711,
      "activations/layer4_attention_weight_max": 100.50906372070312,
      "activations/layer4_attention_weight_min": -97.18165588378906,
      "activations/layer5_attention_weight_max": 88.0331039428711,
      "activations/layer5_attention_weight_min": -79.36187744140625,
      "activations/layer6_attention_weight_max": 56.057716369628906,
      "activations/layer6_attention_weight_min": -57.17095184326172,
      "activations/layer7_attention_weight_max": 79.82218170166016,
      "activations/layer7_attention_weight_min": -80.12734985351562,
      "activations/layer8_attention_weight_max": 49.56509780883789,
      "activations/layer8_attention_weight_min": -48.355709075927734,
      "activations/layer9_attention_weight_max": 55.583351135253906,
      "activations/layer9_attention_weight_min": -53.53791046142578,
      "epoch": 8.15,
      "learning_rate": 9.841931818181818e-05,
      "loss": 2.7996,
      "step": 140300
    },
    {
      "activations/layer0_attention_weight_max": 14.345481872558594,
      "activations/layer0_attention_weight_min": -14.664568901062012,
      "activations/layer10_attention_weight_max": 42.67731857299805,
      "activations/layer10_attention_weight_min": -37.160186767578125,
      "activations/layer11_attention_weight_max": 33.42224884033203,
      "activations/layer11_attention_weight_min": -32.741371154785156,
      "activations/layer12_attention_weight_max": 17.28888511657715,
      "activations/layer12_attention_weight_min": -28.981203079223633,
      "activations/layer13_attention_weight_max": 51.733768463134766,
      "activations/layer13_attention_weight_min": -32.42672348022461,
      "activations/layer14_attention_weight_max": 47.86659622192383,
      "activations/layer14_attention_weight_min": -29.4361629486084,
      "activations/layer15_attention_weight_max": 37.109352111816406,
      "activations/layer15_attention_weight_min": -29.173263549804688,
      "activations/layer16_attention_weight_max": 30.787500381469727,
      "activations/layer16_attention_weight_min": -28.444564819335938,
      "activations/layer17_attention_weight_max": 51.90205383300781,
      "activations/layer17_attention_weight_min": -44.85065460205078,
      "activations/layer18_attention_weight_max": 43.795352935791016,
      "activations/layer18_attention_weight_min": -38.76750564575195,
      "activations/layer19_attention_weight_max": 23.1816463470459,
      "activations/layer19_attention_weight_min": -25.853940963745117,
      "activations/layer1_attention_weight_max": 16.67572593688965,
      "activations/layer1_attention_weight_min": -15.969584465026855,
      "activations/layer20_attention_weight_max": 23.885190963745117,
      "activations/layer20_attention_weight_min": -21.904830932617188,
      "activations/layer21_attention_weight_max": 36.762939453125,
      "activations/layer21_attention_weight_min": -24.765615463256836,
      "activations/layer22_attention_weight_max": 30.747968673706055,
      "activations/layer22_attention_weight_min": -25.240333557128906,
      "activations/layer23_attention_weight_max": 32.09268569946289,
      "activations/layer23_attention_weight_min": -24.469078063964844,
      "activations/layer2_attention_weight_max": 31.78183364868164,
      "activations/layer2_attention_weight_min": -30.184738159179688,
      "activations/layer3_attention_weight_max": 89.78594207763672,
      "activations/layer3_attention_weight_min": -90.18537139892578,
      "activations/layer4_attention_weight_max": 98.59508514404297,
      "activations/layer4_attention_weight_min": -93.13297271728516,
      "activations/layer5_attention_weight_max": 80.42914581298828,
      "activations/layer5_attention_weight_min": -75.94733428955078,
      "activations/layer6_attention_weight_max": 55.34581756591797,
      "activations/layer6_attention_weight_min": -51.04600524902344,
      "activations/layer7_attention_weight_max": 81.1084976196289,
      "activations/layer7_attention_weight_min": -75.14181518554688,
      "activations/layer8_attention_weight_max": 47.89955139160156,
      "activations/layer8_attention_weight_min": -45.743526458740234,
      "activations/layer9_attention_weight_max": 53.05488204956055,
      "activations/layer9_attention_weight_min": -51.55537033081055,
      "epoch": 8.16,
      "learning_rate": 9.840037878787877e-05,
      "loss": 2.8204,
      "step": 140350
    },
    {
      "activations/layer0_attention_weight_max": 15.34455680847168,
      "activations/layer0_attention_weight_min": -14.38359260559082,
      "activations/layer10_attention_weight_max": 41.17546081542969,
      "activations/layer10_attention_weight_min": -41.133331298828125,
      "activations/layer11_attention_weight_max": 36.287689208984375,
      "activations/layer11_attention_weight_min": -35.690006256103516,
      "activations/layer12_attention_weight_max": 18.05118179321289,
      "activations/layer12_attention_weight_min": -26.823566436767578,
      "activations/layer13_attention_weight_max": 42.1193733215332,
      "activations/layer13_attention_weight_min": -33.444786071777344,
      "activations/layer14_attention_weight_max": 40.19216537475586,
      "activations/layer14_attention_weight_min": -30.178640365600586,
      "activations/layer15_attention_weight_max": 36.36981964111328,
      "activations/layer15_attention_weight_min": -29.22128677368164,
      "activations/layer16_attention_weight_max": 31.75588035583496,
      "activations/layer16_attention_weight_min": -27.506893157958984,
      "activations/layer17_attention_weight_max": 55.2995491027832,
      "activations/layer17_attention_weight_min": -43.86052322387695,
      "activations/layer18_attention_weight_max": 48.49435043334961,
      "activations/layer18_attention_weight_min": -36.15532302856445,
      "activations/layer19_attention_weight_max": 24.783536911010742,
      "activations/layer19_attention_weight_min": -20.739248275756836,
      "activations/layer1_attention_weight_max": 18.603466033935547,
      "activations/layer1_attention_weight_min": -13.659208297729492,
      "activations/layer20_attention_weight_max": 22.728553771972656,
      "activations/layer20_attention_weight_min": -24.82891845703125,
      "activations/layer21_attention_weight_max": 39.96963882446289,
      "activations/layer21_attention_weight_min": -23.359603881835938,
      "activations/layer22_attention_weight_max": 30.97372055053711,
      "activations/layer22_attention_weight_min": -26.280563354492188,
      "activations/layer23_attention_weight_max": 36.04464340209961,
      "activations/layer23_attention_weight_min": -21.305822372436523,
      "activations/layer2_attention_weight_max": 31.435455322265625,
      "activations/layer2_attention_weight_min": -27.47016716003418,
      "activations/layer3_attention_weight_max": 83.1642074584961,
      "activations/layer3_attention_weight_min": -86.74767303466797,
      "activations/layer4_attention_weight_max": 102.9957504272461,
      "activations/layer4_attention_weight_min": -97.37862396240234,
      "activations/layer5_attention_weight_max": 85.99596405029297,
      "activations/layer5_attention_weight_min": -78.08506774902344,
      "activations/layer6_attention_weight_max": 55.24240493774414,
      "activations/layer6_attention_weight_min": -54.22234344482422,
      "activations/layer7_attention_weight_max": 78.71375274658203,
      "activations/layer7_attention_weight_min": -74.29024505615234,
      "activations/layer8_attention_weight_max": 47.79061508178711,
      "activations/layer8_attention_weight_min": -49.01942443847656,
      "activations/layer9_attention_weight_max": 56.85332107543945,
      "activations/layer9_attention_weight_min": -53.448970794677734,
      "epoch": 8.16,
      "learning_rate": 9.838143939393938e-05,
      "loss": 2.7971,
      "step": 140400
    },
    {
      "activations/layer0_attention_weight_max": 16.194339752197266,
      "activations/layer0_attention_weight_min": -14.272578239440918,
      "activations/layer10_attention_weight_max": 47.63758087158203,
      "activations/layer10_attention_weight_min": -42.19114685058594,
      "activations/layer11_attention_weight_max": 40.27077865600586,
      "activations/layer11_attention_weight_min": -35.34068298339844,
      "activations/layer12_attention_weight_max": 26.659330368041992,
      "activations/layer12_attention_weight_min": -25.890661239624023,
      "activations/layer13_attention_weight_max": 89.72596740722656,
      "activations/layer13_attention_weight_min": -43.67060089111328,
      "activations/layer14_attention_weight_max": 77.58728790283203,
      "activations/layer14_attention_weight_min": -39.65412902832031,
      "activations/layer15_attention_weight_max": 57.278900146484375,
      "activations/layer15_attention_weight_min": -31.06258201599121,
      "activations/layer16_attention_weight_max": 37.96542739868164,
      "activations/layer16_attention_weight_min": -28.241933822631836,
      "activations/layer17_attention_weight_max": 57.03309631347656,
      "activations/layer17_attention_weight_min": -44.42323684692383,
      "activations/layer18_attention_weight_max": 51.56143569946289,
      "activations/layer18_attention_weight_min": -39.79584884643555,
      "activations/layer19_attention_weight_max": 28.604440689086914,
      "activations/layer19_attention_weight_min": -24.089847564697266,
      "activations/layer1_attention_weight_max": 17.598054885864258,
      "activations/layer1_attention_weight_min": -14.255859375,
      "activations/layer20_attention_weight_max": 28.071969985961914,
      "activations/layer20_attention_weight_min": -22.298261642456055,
      "activations/layer21_attention_weight_max": 53.9923210144043,
      "activations/layer21_attention_weight_min": -24.317447662353516,
      "activations/layer22_attention_weight_max": 36.39368438720703,
      "activations/layer22_attention_weight_min": -25.0681095123291,
      "activations/layer23_attention_weight_max": 38.40866470336914,
      "activations/layer23_attention_weight_min": -28.971996307373047,
      "activations/layer2_attention_weight_max": 31.494693756103516,
      "activations/layer2_attention_weight_min": -29.183788299560547,
      "activations/layer3_attention_weight_max": 86.56449127197266,
      "activations/layer3_attention_weight_min": -86.06211853027344,
      "activations/layer4_attention_weight_max": 98.78446197509766,
      "activations/layer4_attention_weight_min": -93.74111938476562,
      "activations/layer5_attention_weight_max": 83.34716796875,
      "activations/layer5_attention_weight_min": -80.78147888183594,
      "activations/layer6_attention_weight_max": 57.68342590332031,
      "activations/layer6_attention_weight_min": -56.14543151855469,
      "activations/layer7_attention_weight_max": 81.08873748779297,
      "activations/layer7_attention_weight_min": -89.467529296875,
      "activations/layer8_attention_weight_max": 51.96796417236328,
      "activations/layer8_attention_weight_min": -53.97500228881836,
      "activations/layer9_attention_weight_max": 63.50541305541992,
      "activations/layer9_attention_weight_min": -60.55403518676758,
      "epoch": 8.16,
      "learning_rate": 9.83625e-05,
      "loss": 2.8097,
      "step": 140450
    },
    {
      "activations/layer0_attention_weight_max": 14.417994499206543,
      "activations/layer0_attention_weight_min": -14.6843900680542,
      "activations/layer10_attention_weight_max": 35.778045654296875,
      "activations/layer10_attention_weight_min": -33.626712799072266,
      "activations/layer11_attention_weight_max": 30.986783981323242,
      "activations/layer11_attention_weight_min": -30.32364273071289,
      "activations/layer12_attention_weight_max": 27.321138381958008,
      "activations/layer12_attention_weight_min": -25.241966247558594,
      "activations/layer13_attention_weight_max": 47.831607818603516,
      "activations/layer13_attention_weight_min": -31.50397300720215,
      "activations/layer14_attention_weight_max": 40.824806213378906,
      "activations/layer14_attention_weight_min": -31.34893035888672,
      "activations/layer15_attention_weight_max": 34.108558654785156,
      "activations/layer15_attention_weight_min": -29.359386444091797,
      "activations/layer16_attention_weight_max": 30.733413696289062,
      "activations/layer16_attention_weight_min": -27.897445678710938,
      "activations/layer17_attention_weight_max": 54.70090866088867,
      "activations/layer17_attention_weight_min": -43.404911041259766,
      "activations/layer18_attention_weight_max": 46.391632080078125,
      "activations/layer18_attention_weight_min": -35.7702522277832,
      "activations/layer19_attention_weight_max": 23.653989791870117,
      "activations/layer19_attention_weight_min": -20.234479904174805,
      "activations/layer1_attention_weight_max": 17.45811653137207,
      "activations/layer1_attention_weight_min": -17.09311866760254,
      "activations/layer20_attention_weight_max": 22.675817489624023,
      "activations/layer20_attention_weight_min": -21.521141052246094,
      "activations/layer21_attention_weight_max": 40.85709762573242,
      "activations/layer21_attention_weight_min": -25.1419677734375,
      "activations/layer22_attention_weight_max": 35.92947006225586,
      "activations/layer22_attention_weight_min": -25.9993896484375,
      "activations/layer23_attention_weight_max": 37.860748291015625,
      "activations/layer23_attention_weight_min": -24.612211227416992,
      "activations/layer2_attention_weight_max": 31.435514450073242,
      "activations/layer2_attention_weight_min": -29.178253173828125,
      "activations/layer3_attention_weight_max": 84.4102783203125,
      "activations/layer3_attention_weight_min": -88.8359146118164,
      "activations/layer4_attention_weight_max": 99.95563507080078,
      "activations/layer4_attention_weight_min": -92.08441925048828,
      "activations/layer5_attention_weight_max": 80.21781921386719,
      "activations/layer5_attention_weight_min": -76.72915649414062,
      "activations/layer6_attention_weight_max": 54.55115509033203,
      "activations/layer6_attention_weight_min": -50.1939811706543,
      "activations/layer7_attention_weight_max": 75.42394256591797,
      "activations/layer7_attention_weight_min": -79.14169311523438,
      "activations/layer8_attention_weight_max": 43.19721984863281,
      "activations/layer8_attention_weight_min": -47.37855911254883,
      "activations/layer9_attention_weight_max": 48.732826232910156,
      "activations/layer9_attention_weight_min": -48.2275505065918,
      "epoch": 8.16,
      "learning_rate": 9.83435606060606e-05,
      "loss": 2.8065,
      "step": 140500
    },
    {
      "activations/layer0_attention_weight_max": 14.242881774902344,
      "activations/layer0_attention_weight_min": -14.794432640075684,
      "activations/layer10_attention_weight_max": 37.148921966552734,
      "activations/layer10_attention_weight_min": -34.125274658203125,
      "activations/layer11_attention_weight_max": 31.967327117919922,
      "activations/layer11_attention_weight_min": -30.368499755859375,
      "activations/layer12_attention_weight_max": 18.483530044555664,
      "activations/layer12_attention_weight_min": -30.404708862304688,
      "activations/layer13_attention_weight_max": 39.36301803588867,
      "activations/layer13_attention_weight_min": -28.15366554260254,
      "activations/layer14_attention_weight_max": 47.828678131103516,
      "activations/layer14_attention_weight_min": -30.967132568359375,
      "activations/layer15_attention_weight_max": 37.210533142089844,
      "activations/layer15_attention_weight_min": -29.009794235229492,
      "activations/layer16_attention_weight_max": 33.86924362182617,
      "activations/layer16_attention_weight_min": -27.827592849731445,
      "activations/layer17_attention_weight_max": 55.47188186645508,
      "activations/layer17_attention_weight_min": -43.379173278808594,
      "activations/layer18_attention_weight_max": 49.23411178588867,
      "activations/layer18_attention_weight_min": -38.35350036621094,
      "activations/layer19_attention_weight_max": 22.09162712097168,
      "activations/layer19_attention_weight_min": -23.824405670166016,
      "activations/layer1_attention_weight_max": 16.636157989501953,
      "activations/layer1_attention_weight_min": -13.895650863647461,
      "activations/layer20_attention_weight_max": 21.26766586303711,
      "activations/layer20_attention_weight_min": -20.540367126464844,
      "activations/layer21_attention_weight_max": 45.101314544677734,
      "activations/layer21_attention_weight_min": -24.016708374023438,
      "activations/layer22_attention_weight_max": 32.658790588378906,
      "activations/layer22_attention_weight_min": -26.178138732910156,
      "activations/layer23_attention_weight_max": 36.20929718017578,
      "activations/layer23_attention_weight_min": -22.056259155273438,
      "activations/layer2_attention_weight_max": 33.34843826293945,
      "activations/layer2_attention_weight_min": -31.83456802368164,
      "activations/layer3_attention_weight_max": 89.97357177734375,
      "activations/layer3_attention_weight_min": -95.64221954345703,
      "activations/layer4_attention_weight_max": 100.12975311279297,
      "activations/layer4_attention_weight_min": -98.37126159667969,
      "activations/layer5_attention_weight_max": 80.66363525390625,
      "activations/layer5_attention_weight_min": -74.6798324584961,
      "activations/layer6_attention_weight_max": 58.072227478027344,
      "activations/layer6_attention_weight_min": -51.261409759521484,
      "activations/layer7_attention_weight_max": 81.0618667602539,
      "activations/layer7_attention_weight_min": -73.06049346923828,
      "activations/layer8_attention_weight_max": 43.48752975463867,
      "activations/layer8_attention_weight_min": -44.99899673461914,
      "activations/layer9_attention_weight_max": 52.315547943115234,
      "activations/layer9_attention_weight_min": -48.33626174926758,
      "epoch": 8.17,
      "learning_rate": 9.83246212121212e-05,
      "loss": 2.8176,
      "step": 140550
    },
    {
      "activations/layer0_attention_weight_max": 14.493066787719727,
      "activations/layer0_attention_weight_min": -14.88276195526123,
      "activations/layer10_attention_weight_max": 35.7611083984375,
      "activations/layer10_attention_weight_min": -33.641937255859375,
      "activations/layer11_attention_weight_max": 31.47031021118164,
      "activations/layer11_attention_weight_min": -30.62805938720703,
      "activations/layer12_attention_weight_max": 26.12594223022461,
      "activations/layer12_attention_weight_min": -24.88808822631836,
      "activations/layer13_attention_weight_max": 38.98995590209961,
      "activations/layer13_attention_weight_min": -31.354265213012695,
      "activations/layer14_attention_weight_max": 37.15726089477539,
      "activations/layer14_attention_weight_min": -28.106414794921875,
      "activations/layer15_attention_weight_max": 37.86330032348633,
      "activations/layer15_attention_weight_min": -28.16933822631836,
      "activations/layer16_attention_weight_max": 32.769752502441406,
      "activations/layer16_attention_weight_min": -26.256427764892578,
      "activations/layer17_attention_weight_max": 57.52105712890625,
      "activations/layer17_attention_weight_min": -43.0367317199707,
      "activations/layer18_attention_weight_max": 51.24899673461914,
      "activations/layer18_attention_weight_min": -34.319740295410156,
      "activations/layer19_attention_weight_max": 25.143672943115234,
      "activations/layer19_attention_weight_min": -21.577045440673828,
      "activations/layer1_attention_weight_max": 16.555835723876953,
      "activations/layer1_attention_weight_min": -15.181573867797852,
      "activations/layer20_attention_weight_max": 24.312145233154297,
      "activations/layer20_attention_weight_min": -26.563364028930664,
      "activations/layer21_attention_weight_max": 34.78364181518555,
      "activations/layer21_attention_weight_min": -19.74463653564453,
      "activations/layer22_attention_weight_max": 30.92613983154297,
      "activations/layer22_attention_weight_min": -22.72727394104004,
      "activations/layer23_attention_weight_max": 39.455169677734375,
      "activations/layer23_attention_weight_min": -21.0325927734375,
      "activations/layer2_attention_weight_max": 31.63568878173828,
      "activations/layer2_attention_weight_min": -30.460081100463867,
      "activations/layer3_attention_weight_max": 86.95648956298828,
      "activations/layer3_attention_weight_min": -88.46183013916016,
      "activations/layer4_attention_weight_max": 100.52471160888672,
      "activations/layer4_attention_weight_min": -98.18187713623047,
      "activations/layer5_attention_weight_max": 80.82392883300781,
      "activations/layer5_attention_weight_min": -76.30339050292969,
      "activations/layer6_attention_weight_max": 57.37991714477539,
      "activations/layer6_attention_weight_min": -54.90864181518555,
      "activations/layer7_attention_weight_max": 76.4408187866211,
      "activations/layer7_attention_weight_min": -72.19692993164062,
      "activations/layer8_attention_weight_max": 43.50624465942383,
      "activations/layer8_attention_weight_min": -44.884033203125,
      "activations/layer9_attention_weight_max": 47.83473587036133,
      "activations/layer9_attention_weight_min": -46.81249237060547,
      "epoch": 8.17,
      "learning_rate": 9.83056818181818e-05,
      "loss": 2.7929,
      "step": 140600
    },
    {
      "activations/layer0_attention_weight_max": 14.60863971710205,
      "activations/layer0_attention_weight_min": -14.415243148803711,
      "activations/layer10_attention_weight_max": 37.721099853515625,
      "activations/layer10_attention_weight_min": -39.00146484375,
      "activations/layer11_attention_weight_max": 32.98460388183594,
      "activations/layer11_attention_weight_min": -31.423179626464844,
      "activations/layer12_attention_weight_max": 23.030872344970703,
      "activations/layer12_attention_weight_min": -25.02580451965332,
      "activations/layer13_attention_weight_max": 47.37015151977539,
      "activations/layer13_attention_weight_min": -30.084413528442383,
      "activations/layer14_attention_weight_max": 38.25520324707031,
      "activations/layer14_attention_weight_min": -32.4738883972168,
      "activations/layer15_attention_weight_max": 36.16508102416992,
      "activations/layer15_attention_weight_min": -29.714128494262695,
      "activations/layer16_attention_weight_max": 32.26106262207031,
      "activations/layer16_attention_weight_min": -28.018136978149414,
      "activations/layer17_attention_weight_max": 54.2658576965332,
      "activations/layer17_attention_weight_min": -44.783721923828125,
      "activations/layer18_attention_weight_max": 49.69847869873047,
      "activations/layer18_attention_weight_min": -36.781959533691406,
      "activations/layer19_attention_weight_max": 23.4041805267334,
      "activations/layer19_attention_weight_min": -20.478986740112305,
      "activations/layer1_attention_weight_max": 18.609474182128906,
      "activations/layer1_attention_weight_min": -17.01044273376465,
      "activations/layer20_attention_weight_max": 23.267181396484375,
      "activations/layer20_attention_weight_min": -22.438535690307617,
      "activations/layer21_attention_weight_max": 40.51480484008789,
      "activations/layer21_attention_weight_min": -24.896848678588867,
      "activations/layer22_attention_weight_max": 33.474849700927734,
      "activations/layer22_attention_weight_min": -28.43851661682129,
      "activations/layer23_attention_weight_max": 36.41733169555664,
      "activations/layer23_attention_weight_min": -21.7806453704834,
      "activations/layer2_attention_weight_max": 31.249073028564453,
      "activations/layer2_attention_weight_min": -29.806386947631836,
      "activations/layer3_attention_weight_max": 86.5276870727539,
      "activations/layer3_attention_weight_min": -91.088623046875,
      "activations/layer4_attention_weight_max": 104.11976623535156,
      "activations/layer4_attention_weight_min": -97.75151824951172,
      "activations/layer5_attention_weight_max": 84.9730224609375,
      "activations/layer5_attention_weight_min": -79.99555969238281,
      "activations/layer6_attention_weight_max": 57.34341812133789,
      "activations/layer6_attention_weight_min": -56.32731246948242,
      "activations/layer7_attention_weight_max": 79.23087310791016,
      "activations/layer7_attention_weight_min": -79.79031372070312,
      "activations/layer8_attention_weight_max": 45.91435623168945,
      "activations/layer8_attention_weight_min": -48.323631286621094,
      "activations/layer9_attention_weight_max": 51.46173095703125,
      "activations/layer9_attention_weight_min": -50.358882904052734,
      "epoch": 8.17,
      "learning_rate": 9.828674242424242e-05,
      "loss": 2.8168,
      "step": 140650
    },
    {
      "activations/layer0_attention_weight_max": 14.704538345336914,
      "activations/layer0_attention_weight_min": -14.393779754638672,
      "activations/layer10_attention_weight_max": 38.12991714477539,
      "activations/layer10_attention_weight_min": -35.2381591796875,
      "activations/layer11_attention_weight_max": 29.916988372802734,
      "activations/layer11_attention_weight_min": -30.581151962280273,
      "activations/layer12_attention_weight_max": 22.16672134399414,
      "activations/layer12_attention_weight_min": -24.392881393432617,
      "activations/layer13_attention_weight_max": 41.690162658691406,
      "activations/layer13_attention_weight_min": -29.80150032043457,
      "activations/layer14_attention_weight_max": 45.75910568237305,
      "activations/layer14_attention_weight_min": -29.590654373168945,
      "activations/layer15_attention_weight_max": 38.590232849121094,
      "activations/layer15_attention_weight_min": -29.948673248291016,
      "activations/layer16_attention_weight_max": 29.83851432800293,
      "activations/layer16_attention_weight_min": -29.46938133239746,
      "activations/layer17_attention_weight_max": 55.358863830566406,
      "activations/layer17_attention_weight_min": -45.39836883544922,
      "activations/layer18_attention_weight_max": 47.30490493774414,
      "activations/layer18_attention_weight_min": -36.193603515625,
      "activations/layer19_attention_weight_max": 23.047908782958984,
      "activations/layer19_attention_weight_min": -22.205183029174805,
      "activations/layer1_attention_weight_max": 16.735248565673828,
      "activations/layer1_attention_weight_min": -15.161665916442871,
      "activations/layer20_attention_weight_max": 25.021894454956055,
      "activations/layer20_attention_weight_min": -21.526891708374023,
      "activations/layer21_attention_weight_max": 45.70673751831055,
      "activations/layer21_attention_weight_min": -23.334903717041016,
      "activations/layer22_attention_weight_max": 34.92122268676758,
      "activations/layer22_attention_weight_min": -26.649831771850586,
      "activations/layer23_attention_weight_max": 35.56507873535156,
      "activations/layer23_attention_weight_min": -23.97689437866211,
      "activations/layer2_attention_weight_max": 30.526901245117188,
      "activations/layer2_attention_weight_min": -29.030996322631836,
      "activations/layer3_attention_weight_max": 86.25180053710938,
      "activations/layer3_attention_weight_min": -91.37979125976562,
      "activations/layer4_attention_weight_max": 99.19083404541016,
      "activations/layer4_attention_weight_min": -95.8343734741211,
      "activations/layer5_attention_weight_max": 79.15985107421875,
      "activations/layer5_attention_weight_min": -75.43852233886719,
      "activations/layer6_attention_weight_max": 56.04822540283203,
      "activations/layer6_attention_weight_min": -53.3746452331543,
      "activations/layer7_attention_weight_max": 73.75543212890625,
      "activations/layer7_attention_weight_min": -76.59529113769531,
      "activations/layer8_attention_weight_max": 43.1234016418457,
      "activations/layer8_attention_weight_min": -43.5831184387207,
      "activations/layer9_attention_weight_max": 52.10343551635742,
      "activations/layer9_attention_weight_min": -49.15266418457031,
      "epoch": 8.18,
      "learning_rate": 9.826780303030303e-05,
      "loss": 2.8038,
      "step": 140700
    },
    {
      "activations/layer0_attention_weight_max": 14.266573905944824,
      "activations/layer0_attention_weight_min": -14.544668197631836,
      "activations/layer10_attention_weight_max": 57.76024627685547,
      "activations/layer10_attention_weight_min": -52.775177001953125,
      "activations/layer11_attention_weight_max": 51.52445983886719,
      "activations/layer11_attention_weight_min": -43.60398483276367,
      "activations/layer12_attention_weight_max": 21.761524200439453,
      "activations/layer12_attention_weight_min": -24.94768524169922,
      "activations/layer13_attention_weight_max": 70.71253967285156,
      "activations/layer13_attention_weight_min": -36.046356201171875,
      "activations/layer14_attention_weight_max": 101.27993774414062,
      "activations/layer14_attention_weight_min": -40.30093765258789,
      "activations/layer15_attention_weight_max": 78.34009552001953,
      "activations/layer15_attention_weight_min": -34.09854507446289,
      "activations/layer16_attention_weight_max": 35.32974624633789,
      "activations/layer16_attention_weight_min": -31.87952423095703,
      "activations/layer17_attention_weight_max": 66.8284912109375,
      "activations/layer17_attention_weight_min": -56.32093811035156,
      "activations/layer18_attention_weight_max": 58.71708679199219,
      "activations/layer18_attention_weight_min": -49.19869613647461,
      "activations/layer19_attention_weight_max": 31.56117820739746,
      "activations/layer19_attention_weight_min": -28.358129501342773,
      "activations/layer1_attention_weight_max": 17.93419075012207,
      "activations/layer1_attention_weight_min": -14.348564147949219,
      "activations/layer20_attention_weight_max": 35.228172302246094,
      "activations/layer20_attention_weight_min": -22.70488166809082,
      "activations/layer21_attention_weight_max": 64.12149810791016,
      "activations/layer21_attention_weight_min": -26.50640106201172,
      "activations/layer22_attention_weight_max": 40.47332763671875,
      "activations/layer22_attention_weight_min": -26.589998245239258,
      "activations/layer23_attention_weight_max": 38.714599609375,
      "activations/layer23_attention_weight_min": -22.839963912963867,
      "activations/layer2_attention_weight_max": 35.89825439453125,
      "activations/layer2_attention_weight_min": -33.28868103027344,
      "activations/layer3_attention_weight_max": 108.8492660522461,
      "activations/layer3_attention_weight_min": -107.07864379882812,
      "activations/layer4_attention_weight_max": 126.51075744628906,
      "activations/layer4_attention_weight_min": -115.90533447265625,
      "activations/layer5_attention_weight_max": 102.74192810058594,
      "activations/layer5_attention_weight_min": -89.33180236816406,
      "activations/layer6_attention_weight_max": 67.60977172851562,
      "activations/layer6_attention_weight_min": -60.655494689941406,
      "activations/layer7_attention_weight_max": 99.51329040527344,
      "activations/layer7_attention_weight_min": -82.38467407226562,
      "activations/layer8_attention_weight_max": 64.75021362304688,
      "activations/layer8_attention_weight_min": -55.5822639465332,
      "activations/layer9_attention_weight_max": 81.04837036132812,
      "activations/layer9_attention_weight_min": -65.57794189453125,
      "epoch": 8.18,
      "learning_rate": 9.824886363636362e-05,
      "loss": 2.81,
      "step": 140750
    },
    {
      "activations/layer0_attention_weight_max": 15.041555404663086,
      "activations/layer0_attention_weight_min": -14.651710510253906,
      "activations/layer10_attention_weight_max": 41.7435188293457,
      "activations/layer10_attention_weight_min": -35.35187530517578,
      "activations/layer11_attention_weight_max": 32.516319274902344,
      "activations/layer11_attention_weight_min": -29.819005966186523,
      "activations/layer12_attention_weight_max": 19.547754287719727,
      "activations/layer12_attention_weight_min": -25.521625518798828,
      "activations/layer13_attention_weight_max": 49.041622161865234,
      "activations/layer13_attention_weight_min": -35.51548385620117,
      "activations/layer14_attention_weight_max": 39.64475631713867,
      "activations/layer14_attention_weight_min": -31.336139678955078,
      "activations/layer15_attention_weight_max": 36.81914138793945,
      "activations/layer15_attention_weight_min": -30.59084129333496,
      "activations/layer16_attention_weight_max": 32.85505294799805,
      "activations/layer16_attention_weight_min": -29.172380447387695,
      "activations/layer17_attention_weight_max": 53.52853012084961,
      "activations/layer17_attention_weight_min": -44.991233825683594,
      "activations/layer18_attention_weight_max": 45.8228645324707,
      "activations/layer18_attention_weight_min": -36.40974807739258,
      "activations/layer19_attention_weight_max": 22.302072525024414,
      "activations/layer19_attention_weight_min": -22.0927677154541,
      "activations/layer1_attention_weight_max": 17.3485107421875,
      "activations/layer1_attention_weight_min": -14.484859466552734,
      "activations/layer20_attention_weight_max": 23.791271209716797,
      "activations/layer20_attention_weight_min": -24.6987361907959,
      "activations/layer21_attention_weight_max": 45.7436637878418,
      "activations/layer21_attention_weight_min": -25.75957489013672,
      "activations/layer22_attention_weight_max": 34.66037368774414,
      "activations/layer22_attention_weight_min": -26.74896812438965,
      "activations/layer23_attention_weight_max": 40.69536590576172,
      "activations/layer23_attention_weight_min": -24.96448516845703,
      "activations/layer2_attention_weight_max": 31.873273849487305,
      "activations/layer2_attention_weight_min": -29.279264450073242,
      "activations/layer3_attention_weight_max": 81.74755859375,
      "activations/layer3_attention_weight_min": -88.83219146728516,
      "activations/layer4_attention_weight_max": 94.63910675048828,
      "activations/layer4_attention_weight_min": -96.39434814453125,
      "activations/layer5_attention_weight_max": 78.39836120605469,
      "activations/layer5_attention_weight_min": -76.01553344726562,
      "activations/layer6_attention_weight_max": 54.41557693481445,
      "activations/layer6_attention_weight_min": -50.49303436279297,
      "activations/layer7_attention_weight_max": 80.28399658203125,
      "activations/layer7_attention_weight_min": -70.9056625366211,
      "activations/layer8_attention_weight_max": 45.71604537963867,
      "activations/layer8_attention_weight_min": -45.031002044677734,
      "activations/layer9_attention_weight_max": 50.631370544433594,
      "activations/layer9_attention_weight_min": -47.19350814819336,
      "epoch": 8.18,
      "learning_rate": 9.823030303030301e-05,
      "loss": 2.8111,
      "step": 140800
    },
    {
      "activations/layer0_attention_weight_max": 15.656740188598633,
      "activations/layer0_attention_weight_min": -14.794818878173828,
      "activations/layer10_attention_weight_max": 40.960601806640625,
      "activations/layer10_attention_weight_min": -38.80496597290039,
      "activations/layer11_attention_weight_max": 33.42180252075195,
      "activations/layer11_attention_weight_min": -33.427940368652344,
      "activations/layer12_attention_weight_max": 29.522436141967773,
      "activations/layer12_attention_weight_min": -24.820064544677734,
      "activations/layer13_attention_weight_max": 56.808189392089844,
      "activations/layer13_attention_weight_min": -39.62000274658203,
      "activations/layer14_attention_weight_max": 50.70742416381836,
      "activations/layer14_attention_weight_min": -31.91254234313965,
      "activations/layer15_attention_weight_max": 45.36283874511719,
      "activations/layer15_attention_weight_min": -30.19425392150879,
      "activations/layer16_attention_weight_max": 36.85686492919922,
      "activations/layer16_attention_weight_min": -29.094669342041016,
      "activations/layer17_attention_weight_max": 55.98630142211914,
      "activations/layer17_attention_weight_min": -43.20884704589844,
      "activations/layer18_attention_weight_max": 51.04731369018555,
      "activations/layer18_attention_weight_min": -39.0842399597168,
      "activations/layer19_attention_weight_max": 26.744598388671875,
      "activations/layer19_attention_weight_min": -21.802919387817383,
      "activations/layer1_attention_weight_max": 16.57417869567871,
      "activations/layer1_attention_weight_min": -14.992116928100586,
      "activations/layer20_attention_weight_max": 26.418039321899414,
      "activations/layer20_attention_weight_min": -22.805015563964844,
      "activations/layer21_attention_weight_max": 41.34244918823242,
      "activations/layer21_attention_weight_min": -24.82061195373535,
      "activations/layer22_attention_weight_max": 34.984153747558594,
      "activations/layer22_attention_weight_min": -26.022382736206055,
      "activations/layer23_attention_weight_max": 40.60615539550781,
      "activations/layer23_attention_weight_min": -21.941120147705078,
      "activations/layer2_attention_weight_max": 31.73407745361328,
      "activations/layer2_attention_weight_min": -30.54121971130371,
      "activations/layer3_attention_weight_max": 87.11621856689453,
      "activations/layer3_attention_weight_min": -92.37883758544922,
      "activations/layer4_attention_weight_max": 101.92010498046875,
      "activations/layer4_attention_weight_min": -99.10570526123047,
      "activations/layer5_attention_weight_max": 88.71495056152344,
      "activations/layer5_attention_weight_min": -83.51792907714844,
      "activations/layer6_attention_weight_max": 58.241275787353516,
      "activations/layer6_attention_weight_min": -55.61426544189453,
      "activations/layer7_attention_weight_max": 84.7785873413086,
      "activations/layer7_attention_weight_min": -76.29019165039062,
      "activations/layer8_attention_weight_max": 47.41432571411133,
      "activations/layer8_attention_weight_min": -46.72602081298828,
      "activations/layer9_attention_weight_max": 54.06641387939453,
      "activations/layer9_attention_weight_min": -51.22372055053711,
      "epoch": 8.18,
      "learning_rate": 9.821136363636363e-05,
      "loss": 2.8043,
      "step": 140850
    },
    {
      "activations/layer0_attention_weight_max": 15.551739692687988,
      "activations/layer0_attention_weight_min": -15.08319091796875,
      "activations/layer10_attention_weight_max": 38.84516906738281,
      "activations/layer10_attention_weight_min": -35.402225494384766,
      "activations/layer11_attention_weight_max": 29.937789916992188,
      "activations/layer11_attention_weight_min": -31.391765594482422,
      "activations/layer12_attention_weight_max": 19.485061645507812,
      "activations/layer12_attention_weight_min": -28.427736282348633,
      "activations/layer13_attention_weight_max": 38.44758224487305,
      "activations/layer13_attention_weight_min": -31.87419891357422,
      "activations/layer14_attention_weight_max": 37.30202102661133,
      "activations/layer14_attention_weight_min": -30.568574905395508,
      "activations/layer15_attention_weight_max": 33.33918380737305,
      "activations/layer15_attention_weight_min": -30.33755111694336,
      "activations/layer16_attention_weight_max": 30.323259353637695,
      "activations/layer16_attention_weight_min": -27.5628604888916,
      "activations/layer17_attention_weight_max": 51.26997756958008,
      "activations/layer17_attention_weight_min": -42.327877044677734,
      "activations/layer18_attention_weight_max": 46.39631652832031,
      "activations/layer18_attention_weight_min": -35.613426208496094,
      "activations/layer19_attention_weight_max": 21.2801570892334,
      "activations/layer19_attention_weight_min": -20.93398094177246,
      "activations/layer1_attention_weight_max": 16.929628372192383,
      "activations/layer1_attention_weight_min": -14.872015953063965,
      "activations/layer20_attention_weight_max": 21.44628143310547,
      "activations/layer20_attention_weight_min": -23.226747512817383,
      "activations/layer21_attention_weight_max": 40.183387756347656,
      "activations/layer21_attention_weight_min": -27.92902183532715,
      "activations/layer22_attention_weight_max": 32.572811126708984,
      "activations/layer22_attention_weight_min": -25.890583038330078,
      "activations/layer23_attention_weight_max": 36.808074951171875,
      "activations/layer23_attention_weight_min": -22.185470581054688,
      "activations/layer2_attention_weight_max": 30.65229034423828,
      "activations/layer2_attention_weight_min": -31.406021118164062,
      "activations/layer3_attention_weight_max": 85.10406494140625,
      "activations/layer3_attention_weight_min": -92.31234741210938,
      "activations/layer4_attention_weight_max": 97.54208374023438,
      "activations/layer4_attention_weight_min": -96.82072448730469,
      "activations/layer5_attention_weight_max": 83.01990509033203,
      "activations/layer5_attention_weight_min": -77.46165466308594,
      "activations/layer6_attention_weight_max": 55.36913299560547,
      "activations/layer6_attention_weight_min": -54.38876724243164,
      "activations/layer7_attention_weight_max": 74.9477767944336,
      "activations/layer7_attention_weight_min": -73.81959533691406,
      "activations/layer8_attention_weight_max": 42.22670364379883,
      "activations/layer8_attention_weight_min": -45.460914611816406,
      "activations/layer9_attention_weight_max": 49.51200866699219,
      "activations/layer9_attention_weight_min": -47.01397705078125,
      "epoch": 8.19,
      "learning_rate": 9.819242424242424e-05,
      "loss": 2.8043,
      "step": 140900
    },
    {
      "activations/layer0_attention_weight_max": 15.695148468017578,
      "activations/layer0_attention_weight_min": -14.779738426208496,
      "activations/layer10_attention_weight_max": 42.054290771484375,
      "activations/layer10_attention_weight_min": -38.854949951171875,
      "activations/layer11_attention_weight_max": 34.06484603881836,
      "activations/layer11_attention_weight_min": -32.319358825683594,
      "activations/layer12_attention_weight_max": 26.160356521606445,
      "activations/layer12_attention_weight_min": -26.385469436645508,
      "activations/layer13_attention_weight_max": 48.06987380981445,
      "activations/layer13_attention_weight_min": -31.737838745117188,
      "activations/layer14_attention_weight_max": 42.543609619140625,
      "activations/layer14_attention_weight_min": -32.19767379760742,
      "activations/layer15_attention_weight_max": 39.1292724609375,
      "activations/layer15_attention_weight_min": -30.90959358215332,
      "activations/layer16_attention_weight_max": 36.836334228515625,
      "activations/layer16_attention_weight_min": -30.40460968017578,
      "activations/layer17_attention_weight_max": 58.62166976928711,
      "activations/layer17_attention_weight_min": -45.7231559753418,
      "activations/layer18_attention_weight_max": 54.3321418762207,
      "activations/layer18_attention_weight_min": -39.244476318359375,
      "activations/layer19_attention_weight_max": 22.589099884033203,
      "activations/layer19_attention_weight_min": -21.92647361755371,
      "activations/layer1_attention_weight_max": 16.349777221679688,
      "activations/layer1_attention_weight_min": -16.08234214782715,
      "activations/layer20_attention_weight_max": 26.75926971435547,
      "activations/layer20_attention_weight_min": -20.71631622314453,
      "activations/layer21_attention_weight_max": 45.62980270385742,
      "activations/layer21_attention_weight_min": -24.225234985351562,
      "activations/layer22_attention_weight_max": 34.888545989990234,
      "activations/layer22_attention_weight_min": -23.966808319091797,
      "activations/layer23_attention_weight_max": 43.34040069580078,
      "activations/layer23_attention_weight_min": -20.62626838684082,
      "activations/layer2_attention_weight_max": 34.21400833129883,
      "activations/layer2_attention_weight_min": -33.637630462646484,
      "activations/layer3_attention_weight_max": 91.48153686523438,
      "activations/layer3_attention_weight_min": -93.07295227050781,
      "activations/layer4_attention_weight_max": 100.15047454833984,
      "activations/layer4_attention_weight_min": -97.58984375,
      "activations/layer5_attention_weight_max": 82.91709899902344,
      "activations/layer5_attention_weight_min": -83.86056518554688,
      "activations/layer6_attention_weight_max": 58.28879165649414,
      "activations/layer6_attention_weight_min": -53.12520217895508,
      "activations/layer7_attention_weight_max": 76.1237564086914,
      "activations/layer7_attention_weight_min": -75.84613800048828,
      "activations/layer8_attention_weight_max": 44.8834114074707,
      "activations/layer8_attention_weight_min": -46.36382293701172,
      "activations/layer9_attention_weight_max": 52.4570426940918,
      "activations/layer9_attention_weight_min": -52.38582229614258,
      "epoch": 8.19,
      "learning_rate": 9.817348484848483e-05,
      "loss": 2.8005,
      "step": 140950
    },
    {
      "activations/layer0_attention_weight_max": 15.645368576049805,
      "activations/layer0_attention_weight_min": -14.715768814086914,
      "activations/layer10_attention_weight_max": 38.464019775390625,
      "activations/layer10_attention_weight_min": -36.89580535888672,
      "activations/layer11_attention_weight_max": 32.571044921875,
      "activations/layer11_attention_weight_min": -33.051673889160156,
      "activations/layer12_attention_weight_max": 20.98113441467285,
      "activations/layer12_attention_weight_min": -28.425363540649414,
      "activations/layer13_attention_weight_max": 53.14008712768555,
      "activations/layer13_attention_weight_min": -34.28264617919922,
      "activations/layer14_attention_weight_max": 46.18758010864258,
      "activations/layer14_attention_weight_min": -31.324871063232422,
      "activations/layer15_attention_weight_max": 36.37586212158203,
      "activations/layer15_attention_weight_min": -29.53774070739746,
      "activations/layer16_attention_weight_max": 35.59292984008789,
      "activations/layer16_attention_weight_min": -27.768075942993164,
      "activations/layer17_attention_weight_max": 60.0622444152832,
      "activations/layer17_attention_weight_min": -43.361141204833984,
      "activations/layer18_attention_weight_max": 51.324920654296875,
      "activations/layer18_attention_weight_min": -36.80046844482422,
      "activations/layer19_attention_weight_max": 23.985088348388672,
      "activations/layer19_attention_weight_min": -21.46380615234375,
      "activations/layer1_attention_weight_max": 18.44732666015625,
      "activations/layer1_attention_weight_min": -14.611567497253418,
      "activations/layer20_attention_weight_max": 25.786399841308594,
      "activations/layer20_attention_weight_min": -24.908714294433594,
      "activations/layer21_attention_weight_max": 55.24192428588867,
      "activations/layer21_attention_weight_min": -28.889318466186523,
      "activations/layer22_attention_weight_max": 31.5966796875,
      "activations/layer22_attention_weight_min": -25.34349822998047,
      "activations/layer23_attention_weight_max": 41.090267181396484,
      "activations/layer23_attention_weight_min": -22.5744686126709,
      "activations/layer2_attention_weight_max": 31.310287475585938,
      "activations/layer2_attention_weight_min": -30.551212310791016,
      "activations/layer3_attention_weight_max": 92.15660858154297,
      "activations/layer3_attention_weight_min": -90.55270385742188,
      "activations/layer4_attention_weight_max": 107.33678436279297,
      "activations/layer4_attention_weight_min": -100.9143295288086,
      "activations/layer5_attention_weight_max": 88.65435028076172,
      "activations/layer5_attention_weight_min": -79.29366302490234,
      "activations/layer6_attention_weight_max": 56.96147155761719,
      "activations/layer6_attention_weight_min": -53.25862503051758,
      "activations/layer7_attention_weight_max": 79.81693267822266,
      "activations/layer7_attention_weight_min": -73.93170928955078,
      "activations/layer8_attention_weight_max": 45.52175521850586,
      "activations/layer8_attention_weight_min": -44.560611724853516,
      "activations/layer9_attention_weight_max": 57.85657501220703,
      "activations/layer9_attention_weight_min": -51.68037796020508,
      "epoch": 8.19,
      "learning_rate": 9.815454545454545e-05,
      "loss": 2.8012,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5658,
      "eval_samples_per_second": 501.296,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5658,
      "eval_openwebtext_samples_per_second": 501.296,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_wikitext_loss": 2.994140625,
      "eval_wikitext_ppl": 19.96819234893607,
      "eval_wikitext_runtime": 2.0302,
      "eval_wikitext_samples_per_second": 224.61,
      "step": 141000
    },
    {
      "epoch": 8.19,
      "eval_lambada_loss": 2.69921875,
      "eval_lambada_ppl": 14.868111474214228,
      "eval_lambada_runtime": 9.5973,
      "eval_lambada_samples_per_second": 507.33,
      "step": 141000
    },
    {
      "activations/layer0_attention_weight_max": 14.930658340454102,
      "activations/layer0_attention_weight_min": -14.702947616577148,
      "activations/layer10_attention_weight_max": 36.12457275390625,
      "activations/layer10_attention_weight_min": -36.19981384277344,
      "activations/layer11_attention_weight_max": 30.235061645507812,
      "activations/layer11_attention_weight_min": -29.749990463256836,
      "activations/layer12_attention_weight_max": 19.373289108276367,
      "activations/layer12_attention_weight_min": -28.321142196655273,
      "activations/layer13_attention_weight_max": 40.7599983215332,
      "activations/layer13_attention_weight_min": -29.665740966796875,
      "activations/layer14_attention_weight_max": 35.85195541381836,
      "activations/layer14_attention_weight_min": -29.48617935180664,
      "activations/layer15_attention_weight_max": 32.73756408691406,
      "activations/layer15_attention_weight_min": -29.098602294921875,
      "activations/layer16_attention_weight_max": 31.979032516479492,
      "activations/layer16_attention_weight_min": -27.58531379699707,
      "activations/layer17_attention_weight_max": 51.66960144042969,
      "activations/layer17_attention_weight_min": -40.778568267822266,
      "activations/layer18_attention_weight_max": 44.06950759887695,
      "activations/layer18_attention_weight_min": -34.56685256958008,
      "activations/layer19_attention_weight_max": 19.983993530273438,
      "activations/layer19_attention_weight_min": -21.07198143005371,
      "activations/layer1_attention_weight_max": 17.436487197875977,
      "activations/layer1_attention_weight_min": -14.159969329833984,
      "activations/layer20_attention_weight_max": 20.386165618896484,
      "activations/layer20_attention_weight_min": -26.972332000732422,
      "activations/layer21_attention_weight_max": 36.89482879638672,
      "activations/layer21_attention_weight_min": -22.570241928100586,
      "activations/layer22_attention_weight_max": 35.12932586669922,
      "activations/layer22_attention_weight_min": -25.009925842285156,
      "activations/layer23_attention_weight_max": 34.53132629394531,
      "activations/layer23_attention_weight_min": -19.848068237304688,
      "activations/layer2_attention_weight_max": 31.645627975463867,
      "activations/layer2_attention_weight_min": -29.995872497558594,
      "activations/layer3_attention_weight_max": 89.00668334960938,
      "activations/layer3_attention_weight_min": -88.97833251953125,
      "activations/layer4_attention_weight_max": 101.08406066894531,
      "activations/layer4_attention_weight_min": -92.20609283447266,
      "activations/layer5_attention_weight_max": 84.54481506347656,
      "activations/layer5_attention_weight_min": -77.1446304321289,
      "activations/layer6_attention_weight_max": 56.04435729980469,
      "activations/layer6_attention_weight_min": -52.94873809814453,
      "activations/layer7_attention_weight_max": 74.04496765136719,
      "activations/layer7_attention_weight_min": -71.85155487060547,
      "activations/layer8_attention_weight_max": 44.305381774902344,
      "activations/layer8_attention_weight_min": -44.941158294677734,
      "activations/layer9_attention_weight_max": 49.093345642089844,
      "activations/layer9_attention_weight_min": -49.37889099121094,
      "epoch": 8.2,
      "learning_rate": 9.813560606060606e-05,
      "loss": 2.8128,
      "step": 141050
    },
    {
      "activations/layer0_attention_weight_max": 15.146036148071289,
      "activations/layer0_attention_weight_min": -14.592840194702148,
      "activations/layer10_attention_weight_max": 34.19209289550781,
      "activations/layer10_attention_weight_min": -36.349910736083984,
      "activations/layer11_attention_weight_max": 29.615516662597656,
      "activations/layer11_attention_weight_min": -31.69862937927246,
      "activations/layer12_attention_weight_max": 20.3259334564209,
      "activations/layer12_attention_weight_min": -23.61904525756836,
      "activations/layer13_attention_weight_max": 40.84033203125,
      "activations/layer13_attention_weight_min": -34.87691879272461,
      "activations/layer14_attention_weight_max": 37.21005630493164,
      "activations/layer14_attention_weight_min": -30.311206817626953,
      "activations/layer15_attention_weight_max": 34.432373046875,
      "activations/layer15_attention_weight_min": -28.63398551940918,
      "activations/layer16_attention_weight_max": 31.645397186279297,
      "activations/layer16_attention_weight_min": -27.930156707763672,
      "activations/layer17_attention_weight_max": 54.551910400390625,
      "activations/layer17_attention_weight_min": -42.19686508178711,
      "activations/layer18_attention_weight_max": 46.5061149597168,
      "activations/layer18_attention_weight_min": -37.39978790283203,
      "activations/layer19_attention_weight_max": 24.088178634643555,
      "activations/layer19_attention_weight_min": -22.123157501220703,
      "activations/layer1_attention_weight_max": 16.652128219604492,
      "activations/layer1_attention_weight_min": -14.499250411987305,
      "activations/layer20_attention_weight_max": 21.68981170654297,
      "activations/layer20_attention_weight_min": -22.70845603942871,
      "activations/layer21_attention_weight_max": 36.28760528564453,
      "activations/layer21_attention_weight_min": -24.13751220703125,
      "activations/layer22_attention_weight_max": 36.485801696777344,
      "activations/layer22_attention_weight_min": -25.434587478637695,
      "activations/layer23_attention_weight_max": 33.166255950927734,
      "activations/layer23_attention_weight_min": -23.951887130737305,
      "activations/layer2_attention_weight_max": 30.01392936706543,
      "activations/layer2_attention_weight_min": -26.942249298095703,
      "activations/layer3_attention_weight_max": 83.65666961669922,
      "activations/layer3_attention_weight_min": -84.05233764648438,
      "activations/layer4_attention_weight_max": 96.72148895263672,
      "activations/layer4_attention_weight_min": -94.042724609375,
      "activations/layer5_attention_weight_max": 80.91246032714844,
      "activations/layer5_attention_weight_min": -78.86788177490234,
      "activations/layer6_attention_weight_max": 52.605369567871094,
      "activations/layer6_attention_weight_min": -49.650089263916016,
      "activations/layer7_attention_weight_max": 76.14373779296875,
      "activations/layer7_attention_weight_min": -72.6268539428711,
      "activations/layer8_attention_weight_max": 41.751895904541016,
      "activations/layer8_attention_weight_min": -42.911563873291016,
      "activations/layer9_attention_weight_max": 49.3206901550293,
      "activations/layer9_attention_weight_min": -48.40664291381836,
      "epoch": 8.2,
      "learning_rate": 9.811666666666665e-05,
      "loss": 2.7979,
      "step": 141100
    },
    {
      "activations/layer0_attention_weight_max": 16.39893913269043,
      "activations/layer0_attention_weight_min": -14.487369537353516,
      "activations/layer10_attention_weight_max": 38.929752349853516,
      "activations/layer10_attention_weight_min": -36.08057403564453,
      "activations/layer11_attention_weight_max": 34.19746398925781,
      "activations/layer11_attention_weight_min": -33.21241760253906,
      "activations/layer12_attention_weight_max": 20.99992561340332,
      "activations/layer12_attention_weight_min": -24.804107666015625,
      "activations/layer13_attention_weight_max": 45.566646575927734,
      "activations/layer13_attention_weight_min": -31.703460693359375,
      "activations/layer14_attention_weight_max": 38.70700454711914,
      "activations/layer14_attention_weight_min": -30.198280334472656,
      "activations/layer15_attention_weight_max": 37.30474853515625,
      "activations/layer15_attention_weight_min": -29.505212783813477,
      "activations/layer16_attention_weight_max": 35.81779861450195,
      "activations/layer16_attention_weight_min": -28.179466247558594,
      "activations/layer17_attention_weight_max": 55.751068115234375,
      "activations/layer17_attention_weight_min": -44.07578659057617,
      "activations/layer18_attention_weight_max": 48.06748962402344,
      "activations/layer18_attention_weight_min": -37.487548828125,
      "activations/layer19_attention_weight_max": 24.515975952148438,
      "activations/layer19_attention_weight_min": -22.909006118774414,
      "activations/layer1_attention_weight_max": 16.71678352355957,
      "activations/layer1_attention_weight_min": -13.932767868041992,
      "activations/layer20_attention_weight_max": 27.916318893432617,
      "activations/layer20_attention_weight_min": -22.218196868896484,
      "activations/layer21_attention_weight_max": 45.84593200683594,
      "activations/layer21_attention_weight_min": -24.32848358154297,
      "activations/layer22_attention_weight_max": 37.528202056884766,
      "activations/layer22_attention_weight_min": -25.416095733642578,
      "activations/layer23_attention_weight_max": 42.614967346191406,
      "activations/layer23_attention_weight_min": -22.289649963378906,
      "activations/layer2_attention_weight_max": 31.435413360595703,
      "activations/layer2_attention_weight_min": -29.399791717529297,
      "activations/layer3_attention_weight_max": 91.99653625488281,
      "activations/layer3_attention_weight_min": -90.91492462158203,
      "activations/layer4_attention_weight_max": 98.09529876708984,
      "activations/layer4_attention_weight_min": -94.33016967773438,
      "activations/layer5_attention_weight_max": 81.64158630371094,
      "activations/layer5_attention_weight_min": -77.84274291992188,
      "activations/layer6_attention_weight_max": 55.62239074707031,
      "activations/layer6_attention_weight_min": -57.96797561645508,
      "activations/layer7_attention_weight_max": 74.31688690185547,
      "activations/layer7_attention_weight_min": -73.34466552734375,
      "activations/layer8_attention_weight_max": 47.46780014038086,
      "activations/layer8_attention_weight_min": -48.59886169433594,
      "activations/layer9_attention_weight_max": 50.101158142089844,
      "activations/layer9_attention_weight_min": -51.32668685913086,
      "epoch": 8.2,
      "learning_rate": 9.809772727272726e-05,
      "loss": 2.799,
      "step": 141150
    },
    {
      "activations/layer0_attention_weight_max": 16.216466903686523,
      "activations/layer0_attention_weight_min": -14.630339622497559,
      "activations/layer10_attention_weight_max": 38.24885940551758,
      "activations/layer10_attention_weight_min": -37.705074310302734,
      "activations/layer11_attention_weight_max": 31.73726463317871,
      "activations/layer11_attention_weight_min": -31.03911590576172,
      "activations/layer12_attention_weight_max": 19.425994873046875,
      "activations/layer12_attention_weight_min": -27.601465225219727,
      "activations/layer13_attention_weight_max": 39.60758590698242,
      "activations/layer13_attention_weight_min": -34.93196487426758,
      "activations/layer14_attention_weight_max": 38.19926452636719,
      "activations/layer14_attention_weight_min": -31.140382766723633,
      "activations/layer15_attention_weight_max": 37.12841033935547,
      "activations/layer15_attention_weight_min": -29.297321319580078,
      "activations/layer16_attention_weight_max": 32.57102966308594,
      "activations/layer16_attention_weight_min": -29.017192840576172,
      "activations/layer17_attention_weight_max": 55.122474670410156,
      "activations/layer17_attention_weight_min": -44.70743179321289,
      "activations/layer18_attention_weight_max": 49.222747802734375,
      "activations/layer18_attention_weight_min": -36.10161590576172,
      "activations/layer19_attention_weight_max": 23.302988052368164,
      "activations/layer19_attention_weight_min": -20.772680282592773,
      "activations/layer1_attention_weight_max": 17.3284969329834,
      "activations/layer1_attention_weight_min": -14.281124114990234,
      "activations/layer20_attention_weight_max": 25.03229331970215,
      "activations/layer20_attention_weight_min": -25.293254852294922,
      "activations/layer21_attention_weight_max": 37.495758056640625,
      "activations/layer21_attention_weight_min": -25.52615737915039,
      "activations/layer22_attention_weight_max": 33.949588775634766,
      "activations/layer22_attention_weight_min": -23.680240631103516,
      "activations/layer23_attention_weight_max": 35.038673400878906,
      "activations/layer23_attention_weight_min": -22.72238540649414,
      "activations/layer2_attention_weight_max": 30.303691864013672,
      "activations/layer2_attention_weight_min": -31.65961265563965,
      "activations/layer3_attention_weight_max": 86.67527770996094,
      "activations/layer3_attention_weight_min": -91.77505493164062,
      "activations/layer4_attention_weight_max": 100.01140594482422,
      "activations/layer4_attention_weight_min": -94.8883056640625,
      "activations/layer5_attention_weight_max": 81.88838958740234,
      "activations/layer5_attention_weight_min": -75.81246948242188,
      "activations/layer6_attention_weight_max": 53.866641998291016,
      "activations/layer6_attention_weight_min": -53.190120697021484,
      "activations/layer7_attention_weight_max": 73.67377471923828,
      "activations/layer7_attention_weight_min": -71.70359802246094,
      "activations/layer8_attention_weight_max": 47.56599807739258,
      "activations/layer8_attention_weight_min": -48.80217361450195,
      "activations/layer9_attention_weight_max": 55.178524017333984,
      "activations/layer9_attention_weight_min": -47.71549987792969,
      "epoch": 8.2,
      "learning_rate": 9.807878787878786e-05,
      "loss": 2.8249,
      "step": 141200
    },
    {
      "activations/layer0_attention_weight_max": 13.835190773010254,
      "activations/layer0_attention_weight_min": -15.08385944366455,
      "activations/layer10_attention_weight_max": 41.56767654418945,
      "activations/layer10_attention_weight_min": -39.93115234375,
      "activations/layer11_attention_weight_max": 36.97205352783203,
      "activations/layer11_attention_weight_min": -33.959693908691406,
      "activations/layer12_attention_weight_max": 20.282663345336914,
      "activations/layer12_attention_weight_min": -25.551294326782227,
      "activations/layer13_attention_weight_max": 44.02657699584961,
      "activations/layer13_attention_weight_min": -31.595073699951172,
      "activations/layer14_attention_weight_max": 53.196468353271484,
      "activations/layer14_attention_weight_min": -32.038204193115234,
      "activations/layer15_attention_weight_max": 41.971248626708984,
      "activations/layer15_attention_weight_min": -32.96491622924805,
      "activations/layer16_attention_weight_max": 29.53130531311035,
      "activations/layer16_attention_weight_min": -29.340166091918945,
      "activations/layer17_attention_weight_max": 54.04208755493164,
      "activations/layer17_attention_weight_min": -44.79879379272461,
      "activations/layer18_attention_weight_max": 45.74408721923828,
      "activations/layer18_attention_weight_min": -39.41048812866211,
      "activations/layer19_attention_weight_max": 22.272876739501953,
      "activations/layer19_attention_weight_min": -23.77931022644043,
      "activations/layer1_attention_weight_max": 18.187280654907227,
      "activations/layer1_attention_weight_min": -17.713659286499023,
      "activations/layer20_attention_weight_max": 22.1002254486084,
      "activations/layer20_attention_weight_min": -25.984668731689453,
      "activations/layer21_attention_weight_max": 39.78374099731445,
      "activations/layer21_attention_weight_min": -25.272403717041016,
      "activations/layer22_attention_weight_max": 30.402544021606445,
      "activations/layer22_attention_weight_min": -26.338653564453125,
      "activations/layer23_attention_weight_max": 35.69183349609375,
      "activations/layer23_attention_weight_min": -23.027076721191406,
      "activations/layer2_attention_weight_max": 32.05833435058594,
      "activations/layer2_attention_weight_min": -30.426258087158203,
      "activations/layer3_attention_weight_max": 92.1129150390625,
      "activations/layer3_attention_weight_min": -90.99353790283203,
      "activations/layer4_attention_weight_max": 102.67584228515625,
      "activations/layer4_attention_weight_min": -97.7164535522461,
      "activations/layer5_attention_weight_max": 86.65843200683594,
      "activations/layer5_attention_weight_min": -80.96134948730469,
      "activations/layer6_attention_weight_max": 56.84565353393555,
      "activations/layer6_attention_weight_min": -55.97694778442383,
      "activations/layer7_attention_weight_max": 78.85626220703125,
      "activations/layer7_attention_weight_min": -78.48268127441406,
      "activations/layer8_attention_weight_max": 47.578651428222656,
      "activations/layer8_attention_weight_min": -49.38768005371094,
      "activations/layer9_attention_weight_max": 57.6418571472168,
      "activations/layer9_attention_weight_min": -55.5306396484375,
      "epoch": 8.21,
      "learning_rate": 9.805984848484848e-05,
      "loss": 2.8158,
      "step": 141250
    },
    {
      "activations/layer0_attention_weight_max": 14.205698013305664,
      "activations/layer0_attention_weight_min": -14.937488555908203,
      "activations/layer10_attention_weight_max": 48.18442916870117,
      "activations/layer10_attention_weight_min": -42.23179626464844,
      "activations/layer11_attention_weight_max": 40.39103698730469,
      "activations/layer11_attention_weight_min": -32.782493591308594,
      "activations/layer12_attention_weight_max": 20.462482452392578,
      "activations/layer12_attention_weight_min": -26.860496520996094,
      "activations/layer13_attention_weight_max": 72.3829574584961,
      "activations/layer13_attention_weight_min": -36.5084342956543,
      "activations/layer14_attention_weight_max": 84.70321655273438,
      "activations/layer14_attention_weight_min": -35.55255889892578,
      "activations/layer15_attention_weight_max": 56.09474182128906,
      "activations/layer15_attention_weight_min": -29.3819522857666,
      "activations/layer16_attention_weight_max": 33.24934005737305,
      "activations/layer16_attention_weight_min": -26.300559997558594,
      "activations/layer17_attention_weight_max": 53.2869873046875,
      "activations/layer17_attention_weight_min": -44.520973205566406,
      "activations/layer18_attention_weight_max": 47.83226776123047,
      "activations/layer18_attention_weight_min": -39.9084587097168,
      "activations/layer19_attention_weight_max": 25.6688175201416,
      "activations/layer19_attention_weight_min": -22.22026252746582,
      "activations/layer1_attention_weight_max": 19.004735946655273,
      "activations/layer1_attention_weight_min": -16.06293487548828,
      "activations/layer20_attention_weight_max": 29.283428192138672,
      "activations/layer20_attention_weight_min": -21.748878479003906,
      "activations/layer21_attention_weight_max": 48.8343505859375,
      "activations/layer21_attention_weight_min": -23.88627052307129,
      "activations/layer22_attention_weight_max": 35.593833923339844,
      "activations/layer22_attention_weight_min": -25.47884750366211,
      "activations/layer23_attention_weight_max": 40.33137512207031,
      "activations/layer23_attention_weight_min": -23.987159729003906,
      "activations/layer2_attention_weight_max": 35.18473434448242,
      "activations/layer2_attention_weight_min": -31.77814292907715,
      "activations/layer3_attention_weight_max": 103.54613494873047,
      "activations/layer3_attention_weight_min": -96.86613464355469,
      "activations/layer4_attention_weight_max": 113.69640350341797,
      "activations/layer4_attention_weight_min": -104.4071273803711,
      "activations/layer5_attention_weight_max": 86.49822235107422,
      "activations/layer5_attention_weight_min": -83.13268280029297,
      "activations/layer6_attention_weight_max": 60.30048751831055,
      "activations/layer6_attention_weight_min": -54.95543670654297,
      "activations/layer7_attention_weight_max": 87.31908416748047,
      "activations/layer7_attention_weight_min": -80.31584930419922,
      "activations/layer8_attention_weight_max": 53.4299201965332,
      "activations/layer8_attention_weight_min": -49.08655548095703,
      "activations/layer9_attention_weight_max": 66.23272705078125,
      "activations/layer9_attention_weight_min": -52.947669982910156,
      "epoch": 8.21,
      "learning_rate": 9.804090909090908e-05,
      "loss": 2.8141,
      "step": 141300
    },
    {
      "activations/layer0_attention_weight_max": 14.458641052246094,
      "activations/layer0_attention_weight_min": -15.0286226272583,
      "activations/layer10_attention_weight_max": 32.981964111328125,
      "activations/layer10_attention_weight_min": -33.15616989135742,
      "activations/layer11_attention_weight_max": 27.762142181396484,
      "activations/layer11_attention_weight_min": -30.39679527282715,
      "activations/layer12_attention_weight_max": 18.348966598510742,
      "activations/layer12_attention_weight_min": -24.1478214263916,
      "activations/layer13_attention_weight_max": 39.67052459716797,
      "activations/layer13_attention_weight_min": -28.933212280273438,
      "activations/layer14_attention_weight_max": 40.914451599121094,
      "activations/layer14_attention_weight_min": -31.33824348449707,
      "activations/layer15_attention_weight_max": 33.817283630371094,
      "activations/layer15_attention_weight_min": -31.233341217041016,
      "activations/layer16_attention_weight_max": 31.075056076049805,
      "activations/layer16_attention_weight_min": -29.230304718017578,
      "activations/layer17_attention_weight_max": 54.79470443725586,
      "activations/layer17_attention_weight_min": -43.21138000488281,
      "activations/layer18_attention_weight_max": 44.93793869018555,
      "activations/layer18_attention_weight_min": -35.7159538269043,
      "activations/layer19_attention_weight_max": 20.916597366333008,
      "activations/layer19_attention_weight_min": -20.565937042236328,
      "activations/layer1_attention_weight_max": 16.335643768310547,
      "activations/layer1_attention_weight_min": -15.95427417755127,
      "activations/layer20_attention_weight_max": 20.43439483642578,
      "activations/layer20_attention_weight_min": -22.893638610839844,
      "activations/layer21_attention_weight_max": 33.64273452758789,
      "activations/layer21_attention_weight_min": -24.457481384277344,
      "activations/layer22_attention_weight_max": 27.83601951599121,
      "activations/layer22_attention_weight_min": -25.57996368408203,
      "activations/layer23_attention_weight_max": 34.270362854003906,
      "activations/layer23_attention_weight_min": -25.14724349975586,
      "activations/layer2_attention_weight_max": 30.73004150390625,
      "activations/layer2_attention_weight_min": -29.832054138183594,
      "activations/layer3_attention_weight_max": 86.09146118164062,
      "activations/layer3_attention_weight_min": -88.85891723632812,
      "activations/layer4_attention_weight_max": 99.1872787475586,
      "activations/layer4_attention_weight_min": -95.84003448486328,
      "activations/layer5_attention_weight_max": 82.0933837890625,
      "activations/layer5_attention_weight_min": -76.11946105957031,
      "activations/layer6_attention_weight_max": 54.34482192993164,
      "activations/layer6_attention_weight_min": -50.6403923034668,
      "activations/layer7_attention_weight_max": 73.85289001464844,
      "activations/layer7_attention_weight_min": -72.62495422363281,
      "activations/layer8_attention_weight_max": 43.711814880371094,
      "activations/layer8_attention_weight_min": -43.06768798828125,
      "activations/layer9_attention_weight_max": 50.25315856933594,
      "activations/layer9_attention_weight_min": -46.75675964355469,
      "epoch": 8.21,
      "learning_rate": 9.802196969696968e-05,
      "loss": 2.8019,
      "step": 141350
    },
    {
      "activations/layer0_attention_weight_max": 13.768418312072754,
      "activations/layer0_attention_weight_min": -15.138875961303711,
      "activations/layer10_attention_weight_max": 42.76591110229492,
      "activations/layer10_attention_weight_min": -39.51304244995117,
      "activations/layer11_attention_weight_max": 34.95499038696289,
      "activations/layer11_attention_weight_min": -33.38685607910156,
      "activations/layer12_attention_weight_max": 22.03304672241211,
      "activations/layer12_attention_weight_min": -36.09114456176758,
      "activations/layer13_attention_weight_max": 41.89153289794922,
      "activations/layer13_attention_weight_min": -33.53943634033203,
      "activations/layer14_attention_weight_max": 41.37825012207031,
      "activations/layer14_attention_weight_min": -31.06805992126465,
      "activations/layer15_attention_weight_max": 38.26554489135742,
      "activations/layer15_attention_weight_min": -32.403865814208984,
      "activations/layer16_attention_weight_max": 34.84818649291992,
      "activations/layer16_attention_weight_min": -28.06475067138672,
      "activations/layer17_attention_weight_max": 59.52717590332031,
      "activations/layer17_attention_weight_min": -43.536869049072266,
      "activations/layer18_attention_weight_max": 49.4149055480957,
      "activations/layer18_attention_weight_min": -37.60139083862305,
      "activations/layer19_attention_weight_max": 22.42276954650879,
      "activations/layer19_attention_weight_min": -23.77931022644043,
      "activations/layer1_attention_weight_max": 16.945476531982422,
      "activations/layer1_attention_weight_min": -16.426315307617188,
      "activations/layer20_attention_weight_max": 21.704151153564453,
      "activations/layer20_attention_weight_min": -22.77008628845215,
      "activations/layer21_attention_weight_max": 38.747283935546875,
      "activations/layer21_attention_weight_min": -23.672204971313477,
      "activations/layer22_attention_weight_max": 31.555086135864258,
      "activations/layer22_attention_weight_min": -25.54937744140625,
      "activations/layer23_attention_weight_max": 35.51133728027344,
      "activations/layer23_attention_weight_min": -22.637779235839844,
      "activations/layer2_attention_weight_max": 32.759925842285156,
      "activations/layer2_attention_weight_min": -31.12030792236328,
      "activations/layer3_attention_weight_max": 89.07720947265625,
      "activations/layer3_attention_weight_min": -94.24886322021484,
      "activations/layer4_attention_weight_max": 102.45252990722656,
      "activations/layer4_attention_weight_min": -98.42149353027344,
      "activations/layer5_attention_weight_max": 84.32362365722656,
      "activations/layer5_attention_weight_min": -81.08765411376953,
      "activations/layer6_attention_weight_max": 56.62297821044922,
      "activations/layer6_attention_weight_min": -54.36135482788086,
      "activations/layer7_attention_weight_max": 83.3327865600586,
      "activations/layer7_attention_weight_min": -78.89531707763672,
      "activations/layer8_attention_weight_max": 46.731483459472656,
      "activations/layer8_attention_weight_min": -47.50754165649414,
      "activations/layer9_attention_weight_max": 52.5567741394043,
      "activations/layer9_attention_weight_min": -53.13979721069336,
      "epoch": 8.22,
      "learning_rate": 9.80030303030303e-05,
      "loss": 2.812,
      "step": 141400
    },
    {
      "activations/layer0_attention_weight_max": 13.888599395751953,
      "activations/layer0_attention_weight_min": -15.042427062988281,
      "activations/layer10_attention_weight_max": 40.35888671875,
      "activations/layer10_attention_weight_min": -37.80237579345703,
      "activations/layer11_attention_weight_max": 34.8723258972168,
      "activations/layer11_attention_weight_min": -33.63721466064453,
      "activations/layer12_attention_weight_max": 20.56669044494629,
      "activations/layer12_attention_weight_min": -23.283554077148438,
      "activations/layer13_attention_weight_max": 43.8074836730957,
      "activations/layer13_attention_weight_min": -35.088993072509766,
      "activations/layer14_attention_weight_max": 42.87232208251953,
      "activations/layer14_attention_weight_min": -34.06504440307617,
      "activations/layer15_attention_weight_max": 45.59453201293945,
      "activations/layer15_attention_weight_min": -30.023435592651367,
      "activations/layer16_attention_weight_max": 40.10001754760742,
      "activations/layer16_attention_weight_min": -32.565101623535156,
      "activations/layer17_attention_weight_max": 72.02043151855469,
      "activations/layer17_attention_weight_min": -52.45407485961914,
      "activations/layer18_attention_weight_max": 59.97200012207031,
      "activations/layer18_attention_weight_min": -46.069007873535156,
      "activations/layer19_attention_weight_max": 31.014860153198242,
      "activations/layer19_attention_weight_min": -23.84265899658203,
      "activations/layer1_attention_weight_max": 16.611848831176758,
      "activations/layer1_attention_weight_min": -14.437374114990234,
      "activations/layer20_attention_weight_max": 28.106243133544922,
      "activations/layer20_attention_weight_min": -22.2377872467041,
      "activations/layer21_attention_weight_max": 49.6517448425293,
      "activations/layer21_attention_weight_min": -22.706649780273438,
      "activations/layer22_attention_weight_max": 41.49986267089844,
      "activations/layer22_attention_weight_min": -26.551862716674805,
      "activations/layer23_attention_weight_max": 46.14521408081055,
      "activations/layer23_attention_weight_min": -19.95231056213379,
      "activations/layer2_attention_weight_max": 32.137611389160156,
      "activations/layer2_attention_weight_min": -30.84416961669922,
      "activations/layer3_attention_weight_max": 89.01707458496094,
      "activations/layer3_attention_weight_min": -94.13121795654297,
      "activations/layer4_attention_weight_max": 107.18721771240234,
      "activations/layer4_attention_weight_min": -99.94322967529297,
      "activations/layer5_attention_weight_max": 88.14376831054688,
      "activations/layer5_attention_weight_min": -81.01888275146484,
      "activations/layer6_attention_weight_max": 58.091373443603516,
      "activations/layer6_attention_weight_min": -55.07280349731445,
      "activations/layer7_attention_weight_max": 77.99394226074219,
      "activations/layer7_attention_weight_min": -79.4820556640625,
      "activations/layer8_attention_weight_max": 46.97650146484375,
      "activations/layer8_attention_weight_min": -48.43860626220703,
      "activations/layer9_attention_weight_max": 53.5311279296875,
      "activations/layer9_attention_weight_min": -54.368255615234375,
      "epoch": 8.22,
      "learning_rate": 9.798409090909091e-05,
      "loss": 2.807,
      "step": 141450
    },
    {
      "activations/layer0_attention_weight_max": 13.952940940856934,
      "activations/layer0_attention_weight_min": -15.394061088562012,
      "activations/layer10_attention_weight_max": 35.48632049560547,
      "activations/layer10_attention_weight_min": -34.39838409423828,
      "activations/layer11_attention_weight_max": 30.638629913330078,
      "activations/layer11_attention_weight_min": -29.2937068939209,
      "activations/layer12_attention_weight_max": 18.947839736938477,
      "activations/layer12_attention_weight_min": -22.476587295532227,
      "activations/layer13_attention_weight_max": 35.91353988647461,
      "activations/layer13_attention_weight_min": -30.782127380371094,
      "activations/layer14_attention_weight_max": 38.539119720458984,
      "activations/layer14_attention_weight_min": -32.12653350830078,
      "activations/layer15_attention_weight_max": 35.32755661010742,
      "activations/layer15_attention_weight_min": -30.371068954467773,
      "activations/layer16_attention_weight_max": 31.428377151489258,
      "activations/layer16_attention_weight_min": -27.41338348388672,
      "activations/layer17_attention_weight_max": 55.2613525390625,
      "activations/layer17_attention_weight_min": -42.47113800048828,
      "activations/layer18_attention_weight_max": 56.14246368408203,
      "activations/layer18_attention_weight_min": -37.10730743408203,
      "activations/layer19_attention_weight_max": 23.91611671447754,
      "activations/layer19_attention_weight_min": -19.67914581298828,
      "activations/layer1_attention_weight_max": 17.08463478088379,
      "activations/layer1_attention_weight_min": -15.163156509399414,
      "activations/layer20_attention_weight_max": 20.812686920166016,
      "activations/layer20_attention_weight_min": -22.247041702270508,
      "activations/layer21_attention_weight_max": 36.699954986572266,
      "activations/layer21_attention_weight_min": -28.42654037475586,
      "activations/layer22_attention_weight_max": 31.335535049438477,
      "activations/layer22_attention_weight_min": -28.892751693725586,
      "activations/layer23_attention_weight_max": 36.176002502441406,
      "activations/layer23_attention_weight_min": -26.532989501953125,
      "activations/layer2_attention_weight_max": 32.92510986328125,
      "activations/layer2_attention_weight_min": -31.552860260009766,
      "activations/layer3_attention_weight_max": 88.71041870117188,
      "activations/layer3_attention_weight_min": -94.23399353027344,
      "activations/layer4_attention_weight_max": 99.65750122070312,
      "activations/layer4_attention_weight_min": -99.2641372680664,
      "activations/layer5_attention_weight_max": 80.52645111083984,
      "activations/layer5_attention_weight_min": -79.51245880126953,
      "activations/layer6_attention_weight_max": 55.55656433105469,
      "activations/layer6_attention_weight_min": -54.02755355834961,
      "activations/layer7_attention_weight_max": 79.52233123779297,
      "activations/layer7_attention_weight_min": -72.89651489257812,
      "activations/layer8_attention_weight_max": 42.45230484008789,
      "activations/layer8_attention_weight_min": -46.882057189941406,
      "activations/layer9_attention_weight_max": 48.45280838012695,
      "activations/layer9_attention_weight_min": -45.91313934326172,
      "epoch": 8.22,
      "learning_rate": 9.79651515151515e-05,
      "loss": 2.7973,
      "step": 141500
    },
    {
      "activations/layer0_attention_weight_max": 13.602071762084961,
      "activations/layer0_attention_weight_min": -14.877291679382324,
      "activations/layer10_attention_weight_max": 39.49909210205078,
      "activations/layer10_attention_weight_min": -34.05565643310547,
      "activations/layer11_attention_weight_max": 29.736873626708984,
      "activations/layer11_attention_weight_min": -30.978839874267578,
      "activations/layer12_attention_weight_max": 20.321090698242188,
      "activations/layer12_attention_weight_min": -23.317747116088867,
      "activations/layer13_attention_weight_max": 43.765811920166016,
      "activations/layer13_attention_weight_min": -31.161048889160156,
      "activations/layer14_attention_weight_max": 39.524871826171875,
      "activations/layer14_attention_weight_min": -30.95843505859375,
      "activations/layer15_attention_weight_max": 35.50385284423828,
      "activations/layer15_attention_weight_min": -29.102083206176758,
      "activations/layer16_attention_weight_max": 32.35729217529297,
      "activations/layer16_attention_weight_min": -28.273481369018555,
      "activations/layer17_attention_weight_max": 57.8535041809082,
      "activations/layer17_attention_weight_min": -41.945556640625,
      "activations/layer18_attention_weight_max": 49.57378387451172,
      "activations/layer18_attention_weight_min": -36.70009994506836,
      "activations/layer19_attention_weight_max": 22.179288864135742,
      "activations/layer19_attention_weight_min": -20.98400115966797,
      "activations/layer1_attention_weight_max": 17.975997924804688,
      "activations/layer1_attention_weight_min": -14.70620059967041,
      "activations/layer20_attention_weight_max": 24.78093910217285,
      "activations/layer20_attention_weight_min": -22.70481300354004,
      "activations/layer21_attention_weight_max": 41.15336227416992,
      "activations/layer21_attention_weight_min": -22.431589126586914,
      "activations/layer22_attention_weight_max": 33.646522521972656,
      "activations/layer22_attention_weight_min": -25.19684600830078,
      "activations/layer23_attention_weight_max": 42.265113830566406,
      "activations/layer23_attention_weight_min": -21.92176055908203,
      "activations/layer2_attention_weight_max": 32.48291015625,
      "activations/layer2_attention_weight_min": -31.117862701416016,
      "activations/layer3_attention_weight_max": 88.42855072021484,
      "activations/layer3_attention_weight_min": -94.237060546875,
      "activations/layer4_attention_weight_max": 99.64044952392578,
      "activations/layer4_attention_weight_min": -96.2059555053711,
      "activations/layer5_attention_weight_max": 77.94094848632812,
      "activations/layer5_attention_weight_min": -76.96867370605469,
      "activations/layer6_attention_weight_max": 53.29936599731445,
      "activations/layer6_attention_weight_min": -52.24063491821289,
      "activations/layer7_attention_weight_max": 75.25689697265625,
      "activations/layer7_attention_weight_min": -76.16033172607422,
      "activations/layer8_attention_weight_max": 46.239288330078125,
      "activations/layer8_attention_weight_min": -44.32321548461914,
      "activations/layer9_attention_weight_max": 47.141056060791016,
      "activations/layer9_attention_weight_min": -50.16877746582031,
      "epoch": 8.22,
      "learning_rate": 9.794621212121212e-05,
      "loss": 2.8169,
      "step": 141550
    },
    {
      "activations/layer0_attention_weight_max": 14.928186416625977,
      "activations/layer0_attention_weight_min": -14.801103591918945,
      "activations/layer10_attention_weight_max": 40.23324203491211,
      "activations/layer10_attention_weight_min": -35.52533721923828,
      "activations/layer11_attention_weight_max": 31.4232177734375,
      "activations/layer11_attention_weight_min": -33.36640930175781,
      "activations/layer12_attention_weight_max": 27.792537689208984,
      "activations/layer12_attention_weight_min": -27.190710067749023,
      "activations/layer13_attention_weight_max": 40.86387252807617,
      "activations/layer13_attention_weight_min": -33.937225341796875,
      "activations/layer14_attention_weight_max": 38.738555908203125,
      "activations/layer14_attention_weight_min": -30.557390213012695,
      "activations/layer15_attention_weight_max": 36.13055419921875,
      "activations/layer15_attention_weight_min": -31.698986053466797,
      "activations/layer16_attention_weight_max": 30.802017211914062,
      "activations/layer16_attention_weight_min": -26.728525161743164,
      "activations/layer17_attention_weight_max": 55.76307678222656,
      "activations/layer17_attention_weight_min": -41.079776763916016,
      "activations/layer18_attention_weight_max": 45.54198455810547,
      "activations/layer18_attention_weight_min": -36.223175048828125,
      "activations/layer19_attention_weight_max": 22.896631240844727,
      "activations/layer19_attention_weight_min": -23.388835906982422,
      "activations/layer1_attention_weight_max": 16.37131118774414,
      "activations/layer1_attention_weight_min": -13.556450843811035,
      "activations/layer20_attention_weight_max": 21.842924118041992,
      "activations/layer20_attention_weight_min": -23.16643524169922,
      "activations/layer21_attention_weight_max": 34.243431091308594,
      "activations/layer21_attention_weight_min": -24.77924156188965,
      "activations/layer22_attention_weight_max": 30.3609676361084,
      "activations/layer22_attention_weight_min": -24.11325454711914,
      "activations/layer23_attention_weight_max": 35.4161376953125,
      "activations/layer23_attention_weight_min": -22.426349639892578,
      "activations/layer2_attention_weight_max": 32.66313171386719,
      "activations/layer2_attention_weight_min": -31.124794006347656,
      "activations/layer3_attention_weight_max": 89.1037826538086,
      "activations/layer3_attention_weight_min": -90.44586181640625,
      "activations/layer4_attention_weight_max": 102.5604248046875,
      "activations/layer4_attention_weight_min": -95.3498764038086,
      "activations/layer5_attention_weight_max": 81.10246276855469,
      "activations/layer5_attention_weight_min": -80.32261657714844,
      "activations/layer6_attention_weight_max": 57.157936096191406,
      "activations/layer6_attention_weight_min": -51.677555084228516,
      "activations/layer7_attention_weight_max": 75.82994842529297,
      "activations/layer7_attention_weight_min": -73.65825653076172,
      "activations/layer8_attention_weight_max": 45.01923370361328,
      "activations/layer8_attention_weight_min": -46.66453552246094,
      "activations/layer9_attention_weight_max": 48.894290924072266,
      "activations/layer9_attention_weight_min": -49.3355598449707,
      "epoch": 8.23,
      "learning_rate": 9.792727272727273e-05,
      "loss": 2.8094,
      "step": 141600
    },
    {
      "activations/layer0_attention_weight_max": 14.039060592651367,
      "activations/layer0_attention_weight_min": -14.60897445678711,
      "activations/layer10_attention_weight_max": 38.14435577392578,
      "activations/layer10_attention_weight_min": -36.52149200439453,
      "activations/layer11_attention_weight_max": 33.39765548706055,
      "activations/layer11_attention_weight_min": -32.21157455444336,
      "activations/layer12_attention_weight_max": 21.14520835876465,
      "activations/layer12_attention_weight_min": -25.706911087036133,
      "activations/layer13_attention_weight_max": 47.41582107543945,
      "activations/layer13_attention_weight_min": -31.125818252563477,
      "activations/layer14_attention_weight_max": 40.847164154052734,
      "activations/layer14_attention_weight_min": -31.385677337646484,
      "activations/layer15_attention_weight_max": 35.87323760986328,
      "activations/layer15_attention_weight_min": -30.396419525146484,
      "activations/layer16_attention_weight_max": 34.782657623291016,
      "activations/layer16_attention_weight_min": -29.18235969543457,
      "activations/layer17_attention_weight_max": 55.284759521484375,
      "activations/layer17_attention_weight_min": -44.45865249633789,
      "activations/layer18_attention_weight_max": 52.24409484863281,
      "activations/layer18_attention_weight_min": -38.46586990356445,
      "activations/layer19_attention_weight_max": 21.014408111572266,
      "activations/layer19_attention_weight_min": -22.26976203918457,
      "activations/layer1_attention_weight_max": 17.449451446533203,
      "activations/layer1_attention_weight_min": -16.27606201171875,
      "activations/layer20_attention_weight_max": 23.461137771606445,
      "activations/layer20_attention_weight_min": -25.853370666503906,
      "activations/layer21_attention_weight_max": 41.99692153930664,
      "activations/layer21_attention_weight_min": -26.987394332885742,
      "activations/layer22_attention_weight_max": 31.859102249145508,
      "activations/layer22_attention_weight_min": -26.5505313873291,
      "activations/layer23_attention_weight_max": 35.45684051513672,
      "activations/layer23_attention_weight_min": -24.90042495727539,
      "activations/layer2_attention_weight_max": 31.818479537963867,
      "activations/layer2_attention_weight_min": -30.3262996673584,
      "activations/layer3_attention_weight_max": 89.07076263427734,
      "activations/layer3_attention_weight_min": -91.31985473632812,
      "activations/layer4_attention_weight_max": 99.67508697509766,
      "activations/layer4_attention_weight_min": -96.68878936767578,
      "activations/layer5_attention_weight_max": 81.4158935546875,
      "activations/layer5_attention_weight_min": -76.3133544921875,
      "activations/layer6_attention_weight_max": 56.99946594238281,
      "activations/layer6_attention_weight_min": -54.82573318481445,
      "activations/layer7_attention_weight_max": 78.31634521484375,
      "activations/layer7_attention_weight_min": -75.35970306396484,
      "activations/layer8_attention_weight_max": 45.96649169921875,
      "activations/layer8_attention_weight_min": -48.210289001464844,
      "activations/layer9_attention_weight_max": 52.32763671875,
      "activations/layer9_attention_weight_min": -51.04884719848633,
      "epoch": 8.23,
      "learning_rate": 9.790833333333332e-05,
      "loss": 2.8077,
      "step": 141650
    },
    {
      "activations/layer0_attention_weight_max": 14.595593452453613,
      "activations/layer0_attention_weight_min": -14.810029983520508,
      "activations/layer10_attention_weight_max": 39.245155334472656,
      "activations/layer10_attention_weight_min": -39.02634048461914,
      "activations/layer11_attention_weight_max": 33.81916427612305,
      "activations/layer11_attention_weight_min": -35.45575714111328,
      "activations/layer12_attention_weight_max": 26.564682006835938,
      "activations/layer12_attention_weight_min": -25.243494033813477,
      "activations/layer13_attention_weight_max": 55.10292434692383,
      "activations/layer13_attention_weight_min": -33.32236099243164,
      "activations/layer14_attention_weight_max": 44.534915924072266,
      "activations/layer14_attention_weight_min": -30.116024017333984,
      "activations/layer15_attention_weight_max": 42.105712890625,
      "activations/layer15_attention_weight_min": -30.0386962890625,
      "activations/layer16_attention_weight_max": 29.707111358642578,
      "activations/layer16_attention_weight_min": -27.98821258544922,
      "activations/layer17_attention_weight_max": 57.42755889892578,
      "activations/layer17_attention_weight_min": -43.65682601928711,
      "activations/layer18_attention_weight_max": 47.72089767456055,
      "activations/layer18_attention_weight_min": -37.8634147644043,
      "activations/layer19_attention_weight_max": 24.808677673339844,
      "activations/layer19_attention_weight_min": -22.105628967285156,
      "activations/layer1_attention_weight_max": 16.401121139526367,
      "activations/layer1_attention_weight_min": -16.844911575317383,
      "activations/layer20_attention_weight_max": 27.03227996826172,
      "activations/layer20_attention_weight_min": -23.08028793334961,
      "activations/layer21_attention_weight_max": 40.018165588378906,
      "activations/layer21_attention_weight_min": -24.31892967224121,
      "activations/layer22_attention_weight_max": 32.87809371948242,
      "activations/layer22_attention_weight_min": -25.76969337463379,
      "activations/layer23_attention_weight_max": 35.26652145385742,
      "activations/layer23_attention_weight_min": -22.463417053222656,
      "activations/layer2_attention_weight_max": 32.89705276489258,
      "activations/layer2_attention_weight_min": -30.89809799194336,
      "activations/layer3_attention_weight_max": 93.25978088378906,
      "activations/layer3_attention_weight_min": -99.01566314697266,
      "activations/layer4_attention_weight_max": 105.59437561035156,
      "activations/layer4_attention_weight_min": -100.93185424804688,
      "activations/layer5_attention_weight_max": 85.85704040527344,
      "activations/layer5_attention_weight_min": -79.11254119873047,
      "activations/layer6_attention_weight_max": 55.58820343017578,
      "activations/layer6_attention_weight_min": -55.665992736816406,
      "activations/layer7_attention_weight_max": 77.09700775146484,
      "activations/layer7_attention_weight_min": -77.51048278808594,
      "activations/layer8_attention_weight_max": 47.06831359863281,
      "activations/layer8_attention_weight_min": -46.369937896728516,
      "activations/layer9_attention_weight_max": 54.3101806640625,
      "activations/layer9_attention_weight_min": -54.24216842651367,
      "epoch": 8.23,
      "learning_rate": 9.788939393939393e-05,
      "loss": 2.8052,
      "step": 141700
    },
    {
      "activations/layer0_attention_weight_max": 15.785709381103516,
      "activations/layer0_attention_weight_min": -14.947610855102539,
      "activations/layer10_attention_weight_max": 39.12006378173828,
      "activations/layer10_attention_weight_min": -40.62677001953125,
      "activations/layer11_attention_weight_max": 35.851898193359375,
      "activations/layer11_attention_weight_min": -34.590179443359375,
      "activations/layer12_attention_weight_max": 19.177682876586914,
      "activations/layer12_attention_weight_min": -26.940719604492188,
      "activations/layer13_attention_weight_max": 47.86369323730469,
      "activations/layer13_attention_weight_min": -32.29668426513672,
      "activations/layer14_attention_weight_max": 43.98958969116211,
      "activations/layer14_attention_weight_min": -31.285232543945312,
      "activations/layer15_attention_weight_max": 38.56629943847656,
      "activations/layer15_attention_weight_min": -29.944053649902344,
      "activations/layer16_attention_weight_max": 35.08464050292969,
      "activations/layer16_attention_weight_min": -29.20066261291504,
      "activations/layer17_attention_weight_max": 53.68739700317383,
      "activations/layer17_attention_weight_min": -42.466487884521484,
      "activations/layer18_attention_weight_max": 50.470943450927734,
      "activations/layer18_attention_weight_min": -36.70604705810547,
      "activations/layer19_attention_weight_max": 21.93657684326172,
      "activations/layer19_attention_weight_min": -20.460861206054688,
      "activations/layer1_attention_weight_max": 17.823150634765625,
      "activations/layer1_attention_weight_min": -15.428654670715332,
      "activations/layer20_attention_weight_max": 23.01567840576172,
      "activations/layer20_attention_weight_min": -20.71578025817871,
      "activations/layer21_attention_weight_max": 39.89237594604492,
      "activations/layer21_attention_weight_min": -23.686412811279297,
      "activations/layer22_attention_weight_max": 36.328060150146484,
      "activations/layer22_attention_weight_min": -26.325336456298828,
      "activations/layer23_attention_weight_max": 37.80567169189453,
      "activations/layer23_attention_weight_min": -20.97081756591797,
      "activations/layer2_attention_weight_max": 30.638540267944336,
      "activations/layer2_attention_weight_min": -31.235599517822266,
      "activations/layer3_attention_weight_max": 87.414794921875,
      "activations/layer3_attention_weight_min": -87.62474060058594,
      "activations/layer4_attention_weight_max": 103.80753326416016,
      "activations/layer4_attention_weight_min": -98.24903869628906,
      "activations/layer5_attention_weight_max": 85.54290008544922,
      "activations/layer5_attention_weight_min": -85.85366821289062,
      "activations/layer6_attention_weight_max": 60.563446044921875,
      "activations/layer6_attention_weight_min": -56.970096588134766,
      "activations/layer7_attention_weight_max": 88.3410415649414,
      "activations/layer7_attention_weight_min": -79.89295196533203,
      "activations/layer8_attention_weight_max": 50.83351516723633,
      "activations/layer8_attention_weight_min": -50.95735549926758,
      "activations/layer9_attention_weight_max": 59.308834075927734,
      "activations/layer9_attention_weight_min": -53.518245697021484,
      "epoch": 8.24,
      "learning_rate": 9.787045454545454e-05,
      "loss": 2.8039,
      "step": 141750
    },
    {
      "activations/layer0_attention_weight_max": 14.275066375732422,
      "activations/layer0_attention_weight_min": -15.124923706054688,
      "activations/layer10_attention_weight_max": 39.710330963134766,
      "activations/layer10_attention_weight_min": -38.3430290222168,
      "activations/layer11_attention_weight_max": 33.851524353027344,
      "activations/layer11_attention_weight_min": -34.30743408203125,
      "activations/layer12_attention_weight_max": 20.014198303222656,
      "activations/layer12_attention_weight_min": -26.37079429626465,
      "activations/layer13_attention_weight_max": 49.00028610229492,
      "activations/layer13_attention_weight_min": -32.412044525146484,
      "activations/layer14_attention_weight_max": 45.904685974121094,
      "activations/layer14_attention_weight_min": -32.72507858276367,
      "activations/layer15_attention_weight_max": 40.061065673828125,
      "activations/layer15_attention_weight_min": -31.534252166748047,
      "activations/layer16_attention_weight_max": 33.57472229003906,
      "activations/layer16_attention_weight_min": -29.442127227783203,
      "activations/layer17_attention_weight_max": 54.89272689819336,
      "activations/layer17_attention_weight_min": -45.20634841918945,
      "activations/layer18_attention_weight_max": 53.77787780761719,
      "activations/layer18_attention_weight_min": -38.34819412231445,
      "activations/layer19_attention_weight_max": 25.496427536010742,
      "activations/layer19_attention_weight_min": -21.423898696899414,
      "activations/layer1_attention_weight_max": 19.0531005859375,
      "activations/layer1_attention_weight_min": -17.302152633666992,
      "activations/layer20_attention_weight_max": 24.910860061645508,
      "activations/layer20_attention_weight_min": -22.825294494628906,
      "activations/layer21_attention_weight_max": 42.24137878417969,
      "activations/layer21_attention_weight_min": -26.073814392089844,
      "activations/layer22_attention_weight_max": 35.80967712402344,
      "activations/layer22_attention_weight_min": -25.09649085998535,
      "activations/layer23_attention_weight_max": 37.90833282470703,
      "activations/layer23_attention_weight_min": -20.35427474975586,
      "activations/layer2_attention_weight_max": 33.13356399536133,
      "activations/layer2_attention_weight_min": -30.690784454345703,
      "activations/layer3_attention_weight_max": 92.28659057617188,
      "activations/layer3_attention_weight_min": -97.8190689086914,
      "activations/layer4_attention_weight_max": 105.69886779785156,
      "activations/layer4_attention_weight_min": -99.55095672607422,
      "activations/layer5_attention_weight_max": 84.84593963623047,
      "activations/layer5_attention_weight_min": -83.36945343017578,
      "activations/layer6_attention_weight_max": 59.72183609008789,
      "activations/layer6_attention_weight_min": -57.41874694824219,
      "activations/layer7_attention_weight_max": 80.79251861572266,
      "activations/layer7_attention_weight_min": -84.24588775634766,
      "activations/layer8_attention_weight_max": 47.07435989379883,
      "activations/layer8_attention_weight_min": -50.50454330444336,
      "activations/layer9_attention_weight_max": 61.74201583862305,
      "activations/layer9_attention_weight_min": -52.1751594543457,
      "epoch": 8.24,
      "learning_rate": 9.785151515151514e-05,
      "loss": 2.7936,
      "step": 141800
    },
    {
      "activations/layer0_attention_weight_max": 14.493551254272461,
      "activations/layer0_attention_weight_min": -14.718992233276367,
      "activations/layer10_attention_weight_max": 39.76283645629883,
      "activations/layer10_attention_weight_min": -38.59410858154297,
      "activations/layer11_attention_weight_max": 32.43859100341797,
      "activations/layer11_attention_weight_min": -32.963199615478516,
      "activations/layer12_attention_weight_max": 20.226606369018555,
      "activations/layer12_attention_weight_min": -26.276277542114258,
      "activations/layer13_attention_weight_max": 43.80411911010742,
      "activations/layer13_attention_weight_min": -33.08570098876953,
      "activations/layer14_attention_weight_max": 39.03697967529297,
      "activations/layer14_attention_weight_min": -29.58030128479004,
      "activations/layer15_attention_weight_max": 34.410247802734375,
      "activations/layer15_attention_weight_min": -29.032039642333984,
      "activations/layer16_attention_weight_max": 32.59292221069336,
      "activations/layer16_attention_weight_min": -27.282325744628906,
      "activations/layer17_attention_weight_max": 50.467159271240234,
      "activations/layer17_attention_weight_min": -42.3560905456543,
      "activations/layer18_attention_weight_max": 45.622440338134766,
      "activations/layer18_attention_weight_min": -37.34878158569336,
      "activations/layer19_attention_weight_max": 24.085391998291016,
      "activations/layer19_attention_weight_min": -23.68637466430664,
      "activations/layer1_attention_weight_max": 17.392549514770508,
      "activations/layer1_attention_weight_min": -15.16749382019043,
      "activations/layer20_attention_weight_max": 23.712953567504883,
      "activations/layer20_attention_weight_min": -24.804014205932617,
      "activations/layer21_attention_weight_max": 38.31084442138672,
      "activations/layer21_attention_weight_min": -25.84099769592285,
      "activations/layer22_attention_weight_max": 33.02372741699219,
      "activations/layer22_attention_weight_min": -29.97314453125,
      "activations/layer23_attention_weight_max": 32.1887092590332,
      "activations/layer23_attention_weight_min": -23.961105346679688,
      "activations/layer2_attention_weight_max": 32.131961822509766,
      "activations/layer2_attention_weight_min": -29.807910919189453,
      "activations/layer3_attention_weight_max": 88.27272033691406,
      "activations/layer3_attention_weight_min": -89.01380920410156,
      "activations/layer4_attention_weight_max": 99.856201171875,
      "activations/layer4_attention_weight_min": -94.10478210449219,
      "activations/layer5_attention_weight_max": 80.95499420166016,
      "activations/layer5_attention_weight_min": -76.24121856689453,
      "activations/layer6_attention_weight_max": 55.825172424316406,
      "activations/layer6_attention_weight_min": -55.08038330078125,
      "activations/layer7_attention_weight_max": 74.06476593017578,
      "activations/layer7_attention_weight_min": -71.80253601074219,
      "activations/layer8_attention_weight_max": 45.5562629699707,
      "activations/layer8_attention_weight_min": -46.82984161376953,
      "activations/layer9_attention_weight_max": 50.826297760009766,
      "activations/layer9_attention_weight_min": -48.04658508300781,
      "epoch": 8.24,
      "learning_rate": 9.783257575757575e-05,
      "loss": 2.7917,
      "step": 141850
    },
    {
      "activations/layer0_attention_weight_max": 15.02569580078125,
      "activations/layer0_attention_weight_min": -15.029044151306152,
      "activations/layer10_attention_weight_max": 37.66358947753906,
      "activations/layer10_attention_weight_min": -35.53194046020508,
      "activations/layer11_attention_weight_max": 32.113494873046875,
      "activations/layer11_attention_weight_min": -32.04899215698242,
      "activations/layer12_attention_weight_max": 17.05216407775879,
      "activations/layer12_attention_weight_min": -28.44327735900879,
      "activations/layer13_attention_weight_max": 42.64065170288086,
      "activations/layer13_attention_weight_min": -31.4161319732666,
      "activations/layer14_attention_weight_max": 37.61103439331055,
      "activations/layer14_attention_weight_min": -29.841060638427734,
      "activations/layer15_attention_weight_max": 34.845359802246094,
      "activations/layer15_attention_weight_min": -28.941875457763672,
      "activations/layer16_attention_weight_max": 30.92902374267578,
      "activations/layer16_attention_weight_min": -27.022462844848633,
      "activations/layer17_attention_weight_max": 53.25579833984375,
      "activations/layer17_attention_weight_min": -43.25453186035156,
      "activations/layer18_attention_weight_max": 48.47895431518555,
      "activations/layer18_attention_weight_min": -36.9529914855957,
      "activations/layer19_attention_weight_max": 21.065528869628906,
      "activations/layer19_attention_weight_min": -21.669994354248047,
      "activations/layer1_attention_weight_max": 16.88819122314453,
      "activations/layer1_attention_weight_min": -16.130355834960938,
      "activations/layer20_attention_weight_max": 21.04446029663086,
      "activations/layer20_attention_weight_min": -21.030025482177734,
      "activations/layer21_attention_weight_max": 38.954349517822266,
      "activations/layer21_attention_weight_min": -28.180253982543945,
      "activations/layer22_attention_weight_max": 25.96813201904297,
      "activations/layer22_attention_weight_min": -24.170146942138672,
      "activations/layer23_attention_weight_max": 32.092567443847656,
      "activations/layer23_attention_weight_min": -21.636234283447266,
      "activations/layer2_attention_weight_max": 37.09435272216797,
      "activations/layer2_attention_weight_min": -32.263710021972656,
      "activations/layer3_attention_weight_max": 93.67095184326172,
      "activations/layer3_attention_weight_min": -98.96426391601562,
      "activations/layer4_attention_weight_max": 106.70819091796875,
      "activations/layer4_attention_weight_min": -105.7298812866211,
      "activations/layer5_attention_weight_max": 84.77791595458984,
      "activations/layer5_attention_weight_min": -80.22064208984375,
      "activations/layer6_attention_weight_max": 55.12498092651367,
      "activations/layer6_attention_weight_min": -54.60757064819336,
      "activations/layer7_attention_weight_max": 80.79971313476562,
      "activations/layer7_attention_weight_min": -75.4386215209961,
      "activations/layer8_attention_weight_max": 48.86155700683594,
      "activations/layer8_attention_weight_min": -46.47451400756836,
      "activations/layer9_attention_weight_max": 60.7025146484375,
      "activations/layer9_attention_weight_min": -50.84891128540039,
      "epoch": 8.25,
      "learning_rate": 9.781363636363635e-05,
      "loss": 2.8065,
      "step": 141900
    },
    {
      "activations/layer0_attention_weight_max": 14.810853004455566,
      "activations/layer0_attention_weight_min": -14.806368827819824,
      "activations/layer10_attention_weight_max": 33.242919921875,
      "activations/layer10_attention_weight_min": -34.476165771484375,
      "activations/layer11_attention_weight_max": 29.8399715423584,
      "activations/layer11_attention_weight_min": -31.27644920349121,
      "activations/layer12_attention_weight_max": 22.232446670532227,
      "activations/layer12_attention_weight_min": -29.201303482055664,
      "activations/layer13_attention_weight_max": 36.68122100830078,
      "activations/layer13_attention_weight_min": -27.919189453125,
      "activations/layer14_attention_weight_max": 39.46147918701172,
      "activations/layer14_attention_weight_min": -29.529983520507812,
      "activations/layer15_attention_weight_max": 31.083881378173828,
      "activations/layer15_attention_weight_min": -28.471031188964844,
      "activations/layer16_attention_weight_max": 33.41252136230469,
      "activations/layer16_attention_weight_min": -26.455711364746094,
      "activations/layer17_attention_weight_max": 52.66481399536133,
      "activations/layer17_attention_weight_min": -39.85859298706055,
      "activations/layer18_attention_weight_max": 46.9156608581543,
      "activations/layer18_attention_weight_min": -33.9281005859375,
      "activations/layer19_attention_weight_max": 20.08864974975586,
      "activations/layer19_attention_weight_min": -23.491662979125977,
      "activations/layer1_attention_weight_max": 16.834667205810547,
      "activations/layer1_attention_weight_min": -14.419350624084473,
      "activations/layer20_attention_weight_max": 22.089542388916016,
      "activations/layer20_attention_weight_min": -25.0195255279541,
      "activations/layer21_attention_weight_max": 34.079769134521484,
      "activations/layer21_attention_weight_min": -22.901023864746094,
      "activations/layer22_attention_weight_max": 30.465717315673828,
      "activations/layer22_attention_weight_min": -26.20465087890625,
      "activations/layer23_attention_weight_max": 36.43978500366211,
      "activations/layer23_attention_weight_min": -23.591718673706055,
      "activations/layer2_attention_weight_max": 33.000240325927734,
      "activations/layer2_attention_weight_min": -30.916515350341797,
      "activations/layer3_attention_weight_max": 93.06063079833984,
      "activations/layer3_attention_weight_min": -91.27059936523438,
      "activations/layer4_attention_weight_max": 102.74610900878906,
      "activations/layer4_attention_weight_min": -99.4300308227539,
      "activations/layer5_attention_weight_max": 81.6780014038086,
      "activations/layer5_attention_weight_min": -79.0385971069336,
      "activations/layer6_attention_weight_max": 56.8194580078125,
      "activations/layer6_attention_weight_min": -54.53493118286133,
      "activations/layer7_attention_weight_max": 75.81632995605469,
      "activations/layer7_attention_weight_min": -72.69709014892578,
      "activations/layer8_attention_weight_max": 42.39106369018555,
      "activations/layer8_attention_weight_min": -42.15818405151367,
      "activations/layer9_attention_weight_max": 47.84723663330078,
      "activations/layer9_attention_weight_min": -46.552337646484375,
      "epoch": 8.25,
      "learning_rate": 9.779469696969697e-05,
      "loss": 2.7889,
      "step": 141950
    },
    {
      "activations/layer0_attention_weight_max": 15.365973472595215,
      "activations/layer0_attention_weight_min": -14.87380313873291,
      "activations/layer10_attention_weight_max": 35.57041931152344,
      "activations/layer10_attention_weight_min": -32.80585479736328,
      "activations/layer11_attention_weight_max": 29.284475326538086,
      "activations/layer11_attention_weight_min": -29.592710494995117,
      "activations/layer12_attention_weight_max": 20.494070053100586,
      "activations/layer12_attention_weight_min": -32.4303092956543,
      "activations/layer13_attention_weight_max": 39.519588470458984,
      "activations/layer13_attention_weight_min": -28.287681579589844,
      "activations/layer14_attention_weight_max": 39.089664459228516,
      "activations/layer14_attention_weight_min": -33.002288818359375,
      "activations/layer15_attention_weight_max": 35.483726501464844,
      "activations/layer15_attention_weight_min": -30.241788864135742,
      "activations/layer16_attention_weight_max": 32.29605484008789,
      "activations/layer16_attention_weight_min": -28.769989013671875,
      "activations/layer17_attention_weight_max": 56.32438659667969,
      "activations/layer17_attention_weight_min": -43.300899505615234,
      "activations/layer18_attention_weight_max": 51.21837615966797,
      "activations/layer18_attention_weight_min": -36.25006866455078,
      "activations/layer19_attention_weight_max": 24.356834411621094,
      "activations/layer19_attention_weight_min": -21.368276596069336,
      "activations/layer1_attention_weight_max": 17.079954147338867,
      "activations/layer1_attention_weight_min": -15.073022842407227,
      "activations/layer20_attention_weight_max": 25.162900924682617,
      "activations/layer20_attention_weight_min": -22.801729202270508,
      "activations/layer21_attention_weight_max": 41.9987907409668,
      "activations/layer21_attention_weight_min": -24.76500129699707,
      "activations/layer22_attention_weight_max": 33.97478103637695,
      "activations/layer22_attention_weight_min": -24.414283752441406,
      "activations/layer23_attention_weight_max": 37.403228759765625,
      "activations/layer23_attention_weight_min": -26.59282684326172,
      "activations/layer2_attention_weight_max": 30.997312545776367,
      "activations/layer2_attention_weight_min": -29.677955627441406,
      "activations/layer3_attention_weight_max": 85.81562042236328,
      "activations/layer3_attention_weight_min": -83.8846664428711,
      "activations/layer4_attention_weight_max": 100.87166595458984,
      "activations/layer4_attention_weight_min": -91.98807525634766,
      "activations/layer5_attention_weight_max": 80.4532470703125,
      "activations/layer5_attention_weight_min": -73.85542297363281,
      "activations/layer6_attention_weight_max": 52.432960510253906,
      "activations/layer6_attention_weight_min": -50.976036071777344,
      "activations/layer7_attention_weight_max": 74.70552062988281,
      "activations/layer7_attention_weight_min": -70.06198120117188,
      "activations/layer8_attention_weight_max": 40.543724060058594,
      "activations/layer8_attention_weight_min": -43.44462585449219,
      "activations/layer9_attention_weight_max": 48.19056701660156,
      "activations/layer9_attention_weight_min": -49.904109954833984,
      "epoch": 8.25,
      "learning_rate": 9.777575757575757e-05,
      "loss": 2.7957,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.6353,
      "eval_samples_per_second": 497.259,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.6353,
      "eval_openwebtext_samples_per_second": 497.259,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_wikitext_loss": 2.9921875,
      "eval_wikitext_ppl": 19.929230034775262,
      "eval_wikitext_runtime": 2.0711,
      "eval_wikitext_samples_per_second": 220.169,
      "step": 142000
    },
    {
      "epoch": 8.25,
      "eval_lambada_loss": 2.6875,
      "eval_lambada_ppl": 14.694892728788941,
      "eval_lambada_runtime": 9.6271,
      "eval_lambada_samples_per_second": 505.76,
      "step": 142000
    },
    {
      "activations/layer0_attention_weight_max": 14.698373794555664,
      "activations/layer0_attention_weight_min": -15.097122192382812,
      "activations/layer10_attention_weight_max": 38.06185531616211,
      "activations/layer10_attention_weight_min": -36.40053176879883,
      "activations/layer11_attention_weight_max": 32.25404739379883,
      "activations/layer11_attention_weight_min": -30.40035629272461,
      "activations/layer12_attention_weight_max": 18.483522415161133,
      "activations/layer12_attention_weight_min": -24.568204879760742,
      "activations/layer13_attention_weight_max": 46.686424255371094,
      "activations/layer13_attention_weight_min": -32.746337890625,
      "activations/layer14_attention_weight_max": 44.908206939697266,
      "activations/layer14_attention_weight_min": -32.536590576171875,
      "activations/layer15_attention_weight_max": 38.52179718017578,
      "activations/layer15_attention_weight_min": -28.70458221435547,
      "activations/layer16_attention_weight_max": 33.90863800048828,
      "activations/layer16_attention_weight_min": -27.829805374145508,
      "activations/layer17_attention_weight_max": 55.2551155090332,
      "activations/layer17_attention_weight_min": -45.961647033691406,
      "activations/layer18_attention_weight_max": 51.94770812988281,
      "activations/layer18_attention_weight_min": -40.74663162231445,
      "activations/layer19_attention_weight_max": 26.109088897705078,
      "activations/layer19_attention_weight_min": -28.601200103759766,
      "activations/layer1_attention_weight_max": 24.184423446655273,
      "activations/layer1_attention_weight_min": -17.845848083496094,
      "activations/layer20_attention_weight_max": 28.53911590576172,
      "activations/layer20_attention_weight_min": -23.683786392211914,
      "activations/layer21_attention_weight_max": 43.46257400512695,
      "activations/layer21_attention_weight_min": -28.350826263427734,
      "activations/layer22_attention_weight_max": 39.977561950683594,
      "activations/layer22_attention_weight_min": -26.99517059326172,
      "activations/layer23_attention_weight_max": 41.528133392333984,
      "activations/layer23_attention_weight_min": -24.388259887695312,
      "activations/layer2_attention_weight_max": 29.51752471923828,
      "activations/layer2_attention_weight_min": -28.961650848388672,
      "activations/layer3_attention_weight_max": 92.21688842773438,
      "activations/layer3_attention_weight_min": -93.01995849609375,
      "activations/layer4_attention_weight_max": 111.74051666259766,
      "activations/layer4_attention_weight_min": -104.9089126586914,
      "activations/layer5_attention_weight_max": 89.62337493896484,
      "activations/layer5_attention_weight_min": -87.46295166015625,
      "activations/layer6_attention_weight_max": 57.99257278442383,
      "activations/layer6_attention_weight_min": -54.230960845947266,
      "activations/layer7_attention_weight_max": 82.7258529663086,
      "activations/layer7_attention_weight_min": -75.22541046142578,
      "activations/layer8_attention_weight_max": 46.44773483276367,
      "activations/layer8_attention_weight_min": -47.03429412841797,
      "activations/layer9_attention_weight_max": 49.93428421020508,
      "activations/layer9_attention_weight_min": -48.80321502685547,
      "epoch": 8.25,
      "learning_rate": 9.775681818181817e-05,
      "loss": 2.8171,
      "step": 142050
    },
    {
      "activations/layer0_attention_weight_max": 14.654563903808594,
      "activations/layer0_attention_weight_min": -14.943315505981445,
      "activations/layer10_attention_weight_max": 38.49116516113281,
      "activations/layer10_attention_weight_min": -36.37571334838867,
      "activations/layer11_attention_weight_max": 31.549650192260742,
      "activations/layer11_attention_weight_min": -32.65369415283203,
      "activations/layer12_attention_weight_max": 18.971481323242188,
      "activations/layer12_attention_weight_min": -32.31517791748047,
      "activations/layer13_attention_weight_max": 44.77798843383789,
      "activations/layer13_attention_weight_min": -33.67226028442383,
      "activations/layer14_attention_weight_max": 45.081153869628906,
      "activations/layer14_attention_weight_min": -32.370513916015625,
      "activations/layer15_attention_weight_max": 37.82487487792969,
      "activations/layer15_attention_weight_min": -30.623050689697266,
      "activations/layer16_attention_weight_max": 31.629552841186523,
      "activations/layer16_attention_weight_min": -28.27208137512207,
      "activations/layer17_attention_weight_max": 54.97357177734375,
      "activations/layer17_attention_weight_min": -42.37294006347656,
      "activations/layer18_attention_weight_max": 52.2197265625,
      "activations/layer18_attention_weight_min": -38.829769134521484,
      "activations/layer19_attention_weight_max": 24.545063018798828,
      "activations/layer19_attention_weight_min": -22.981569290161133,
      "activations/layer1_attention_weight_max": 17.03081512451172,
      "activations/layer1_attention_weight_min": -14.679569244384766,
      "activations/layer20_attention_weight_max": 23.041427612304688,
      "activations/layer20_attention_weight_min": -21.850597381591797,
      "activations/layer21_attention_weight_max": 39.65944290161133,
      "activations/layer21_attention_weight_min": -23.64511489868164,
      "activations/layer22_attention_weight_max": 34.01951599121094,
      "activations/layer22_attention_weight_min": -26.04985809326172,
      "activations/layer23_attention_weight_max": 41.556854248046875,
      "activations/layer23_attention_weight_min": -23.107412338256836,
      "activations/layer2_attention_weight_max": 32.486480712890625,
      "activations/layer2_attention_weight_min": -29.860992431640625,
      "activations/layer3_attention_weight_max": 89.77256774902344,
      "activations/layer3_attention_weight_min": -90.38973236083984,
      "activations/layer4_attention_weight_max": 101.9477767944336,
      "activations/layer4_attention_weight_min": -95.75724029541016,
      "activations/layer5_attention_weight_max": 88.06419372558594,
      "activations/layer5_attention_weight_min": -81.29039764404297,
      "activations/layer6_attention_weight_max": 57.243412017822266,
      "activations/layer6_attention_weight_min": -57.35173797607422,
      "activations/layer7_attention_weight_max": 76.56597900390625,
      "activations/layer7_attention_weight_min": -75.94688415527344,
      "activations/layer8_attention_weight_max": 45.848114013671875,
      "activations/layer8_attention_weight_min": -49.54984664916992,
      "activations/layer9_attention_weight_max": 49.55683517456055,
      "activations/layer9_attention_weight_min": -50.536651611328125,
      "epoch": 8.26,
      "learning_rate": 9.773787878787879e-05,
      "loss": 2.8065,
      "step": 142100
    },
    {
      "activations/layer0_attention_weight_max": 14.57034969329834,
      "activations/layer0_attention_weight_min": -14.772611618041992,
      "activations/layer10_attention_weight_max": 36.58423614501953,
      "activations/layer10_attention_weight_min": -34.75189971923828,
      "activations/layer11_attention_weight_max": 31.5212459564209,
      "activations/layer11_attention_weight_min": -32.46795654296875,
      "activations/layer12_attention_weight_max": 18.253597259521484,
      "activations/layer12_attention_weight_min": -24.88725471496582,
      "activations/layer13_attention_weight_max": 42.89671325683594,
      "activations/layer13_attention_weight_min": -31.179033279418945,
      "activations/layer14_attention_weight_max": 35.274139404296875,
      "activations/layer14_attention_weight_min": -30.435230255126953,
      "activations/layer15_attention_weight_max": 33.62638854980469,
      "activations/layer15_attention_weight_min": -29.48110580444336,
      "activations/layer16_attention_weight_max": 29.052637100219727,
      "activations/layer16_attention_weight_min": -29.2420654296875,
      "activations/layer17_attention_weight_max": 50.76478958129883,
      "activations/layer17_attention_weight_min": -42.50321960449219,
      "activations/layer18_attention_weight_max": 45.56612777709961,
      "activations/layer18_attention_weight_min": -35.87995910644531,
      "activations/layer19_attention_weight_max": 22.510059356689453,
      "activations/layer19_attention_weight_min": -21.146562576293945,
      "activations/layer1_attention_weight_max": 16.53377914428711,
      "activations/layer1_attention_weight_min": -14.108988761901855,
      "activations/layer20_attention_weight_max": 20.411020278930664,
      "activations/layer20_attention_weight_min": -21.81342887878418,
      "activations/layer21_attention_weight_max": 33.784461975097656,
      "activations/layer21_attention_weight_min": -21.44936752319336,
      "activations/layer22_attention_weight_max": 31.53359603881836,
      "activations/layer22_attention_weight_min": -24.8502197265625,
      "activations/layer23_attention_weight_max": 35.76374435424805,
      "activations/layer23_attention_weight_min": -20.761035919189453,
      "activations/layer2_attention_weight_max": 31.63580322265625,
      "activations/layer2_attention_weight_min": -29.334789276123047,
      "activations/layer3_attention_weight_max": 84.75057220458984,
      "activations/layer3_attention_weight_min": -92.0125503540039,
      "activations/layer4_attention_weight_max": 101.3192367553711,
      "activations/layer4_attention_weight_min": -95.03107452392578,
      "activations/layer5_attention_weight_max": 79.39079284667969,
      "activations/layer5_attention_weight_min": -75.36528778076172,
      "activations/layer6_attention_weight_max": 52.60106658935547,
      "activations/layer6_attention_weight_min": -50.28359603881836,
      "activations/layer7_attention_weight_max": 76.02044677734375,
      "activations/layer7_attention_weight_min": -71.11336517333984,
      "activations/layer8_attention_weight_max": 44.25759506225586,
      "activations/layer8_attention_weight_min": -45.861568450927734,
      "activations/layer9_attention_weight_max": 52.114051818847656,
      "activations/layer9_attention_weight_min": -46.59600830078125,
      "epoch": 8.26,
      "learning_rate": 9.771893939393937e-05,
      "loss": 2.8049,
      "step": 142150
    },
    {
      "activations/layer0_attention_weight_max": 15.518117904663086,
      "activations/layer0_attention_weight_min": -14.863550186157227,
      "activations/layer10_attention_weight_max": 39.57276153564453,
      "activations/layer10_attention_weight_min": -37.5152587890625,
      "activations/layer11_attention_weight_max": 32.797119140625,
      "activations/layer11_attention_weight_min": -31.53950309753418,
      "activations/layer12_attention_weight_max": 22.124500274658203,
      "activations/layer12_attention_weight_min": -22.427160263061523,
      "activations/layer13_attention_weight_max": 49.8967170715332,
      "activations/layer13_attention_weight_min": -32.48836135864258,
      "activations/layer14_attention_weight_max": 43.81513595581055,
      "activations/layer14_attention_weight_min": -31.322406768798828,
      "activations/layer15_attention_weight_max": 39.59181213378906,
      "activations/layer15_attention_weight_min": -29.716413497924805,
      "activations/layer16_attention_weight_max": 34.15222930908203,
      "activations/layer16_attention_weight_min": -28.55188751220703,
      "activations/layer17_attention_weight_max": 57.6363525390625,
      "activations/layer17_attention_weight_min": -41.906288146972656,
      "activations/layer18_attention_weight_max": 48.191383361816406,
      "activations/layer18_attention_weight_min": -35.668907165527344,
      "activations/layer19_attention_weight_max": 23.53721046447754,
      "activations/layer19_attention_weight_min": -22.24748420715332,
      "activations/layer1_attention_weight_max": 16.380849838256836,
      "activations/layer1_attention_weight_min": -15.099985122680664,
      "activations/layer20_attention_weight_max": 23.901142120361328,
      "activations/layer20_attention_weight_min": -22.871944427490234,
      "activations/layer21_attention_weight_max": 35.23175811767578,
      "activations/layer21_attention_weight_min": -22.964763641357422,
      "activations/layer22_attention_weight_max": 31.773401260375977,
      "activations/layer22_attention_weight_min": -25.335840225219727,
      "activations/layer23_attention_weight_max": 37.5206184387207,
      "activations/layer23_attention_weight_min": -20.74808120727539,
      "activations/layer2_attention_weight_max": 30.988065719604492,
      "activations/layer2_attention_weight_min": -28.63510513305664,
      "activations/layer3_attention_weight_max": 85.78378295898438,
      "activations/layer3_attention_weight_min": -90.78968811035156,
      "activations/layer4_attention_weight_max": 99.28655242919922,
      "activations/layer4_attention_weight_min": -94.26570892333984,
      "activations/layer5_attention_weight_max": 79.93765258789062,
      "activations/layer5_attention_weight_min": -76.35014343261719,
      "activations/layer6_attention_weight_max": 54.976444244384766,
      "activations/layer6_attention_weight_min": -56.930999755859375,
      "activations/layer7_attention_weight_max": 75.37466430664062,
      "activations/layer7_attention_weight_min": -71.77348327636719,
      "activations/layer8_attention_weight_max": 43.5398063659668,
      "activations/layer8_attention_weight_min": -46.76984405517578,
      "activations/layer9_attention_weight_max": 50.02465057373047,
      "activations/layer9_attention_weight_min": -49.31292724609375,
      "epoch": 8.26,
      "learning_rate": 9.769999999999999e-05,
      "loss": 2.8219,
      "step": 142200
    },
    {
      "activations/layer0_attention_weight_max": 14.23807430267334,
      "activations/layer0_attention_weight_min": -14.695324897766113,
      "activations/layer10_attention_weight_max": 39.217071533203125,
      "activations/layer10_attention_weight_min": -37.940834045410156,
      "activations/layer11_attention_weight_max": 34.68826675415039,
      "activations/layer11_attention_weight_min": -32.803123474121094,
      "activations/layer12_attention_weight_max": 20.304218292236328,
      "activations/layer12_attention_weight_min": -31.38323402404785,
      "activations/layer13_attention_weight_max": 46.30098342895508,
      "activations/layer13_attention_weight_min": -37.67509841918945,
      "activations/layer14_attention_weight_max": 38.83497619628906,
      "activations/layer14_attention_weight_min": -31.215675354003906,
      "activations/layer15_attention_weight_max": 36.558685302734375,
      "activations/layer15_attention_weight_min": -28.242393493652344,
      "activations/layer16_attention_weight_max": 34.335819244384766,
      "activations/layer16_attention_weight_min": -29.880300521850586,
      "activations/layer17_attention_weight_max": 58.34719467163086,
      "activations/layer17_attention_weight_min": -44.6141357421875,
      "activations/layer18_attention_weight_max": 51.62724685668945,
      "activations/layer18_attention_weight_min": -40.24985885620117,
      "activations/layer19_attention_weight_max": 23.373449325561523,
      "activations/layer19_attention_weight_min": -24.178586959838867,
      "activations/layer1_attention_weight_max": 16.930171966552734,
      "activations/layer1_attention_weight_min": -14.9830904006958,
      "activations/layer20_attention_weight_max": 24.425798416137695,
      "activations/layer20_attention_weight_min": -20.760570526123047,
      "activations/layer21_attention_weight_max": 39.4923095703125,
      "activations/layer21_attention_weight_min": -24.053369522094727,
      "activations/layer22_attention_weight_max": 32.12471389770508,
      "activations/layer22_attention_weight_min": -26.74717140197754,
      "activations/layer23_attention_weight_max": 39.83283996582031,
      "activations/layer23_attention_weight_min": -24.36899185180664,
      "activations/layer2_attention_weight_max": 32.54045104980469,
      "activations/layer2_attention_weight_min": -30.578115463256836,
      "activations/layer3_attention_weight_max": 92.9793701171875,
      "activations/layer3_attention_weight_min": -92.3465347290039,
      "activations/layer4_attention_weight_max": 102.0581283569336,
      "activations/layer4_attention_weight_min": -98.86540985107422,
      "activations/layer5_attention_weight_max": 84.04267883300781,
      "activations/layer5_attention_weight_min": -79.73806762695312,
      "activations/layer6_attention_weight_max": 57.540122985839844,
      "activations/layer6_attention_weight_min": -53.837337493896484,
      "activations/layer7_attention_weight_max": 80.6841049194336,
      "activations/layer7_attention_weight_min": -75.3932876586914,
      "activations/layer8_attention_weight_max": 46.6592903137207,
      "activations/layer8_attention_weight_min": -46.984886169433594,
      "activations/layer9_attention_weight_max": 61.113494873046875,
      "activations/layer9_attention_weight_min": -52.03400802612305,
      "epoch": 8.27,
      "learning_rate": 9.76810606060606e-05,
      "loss": 2.7883,
      "step": 142250
    },
    {
      "activations/layer0_attention_weight_max": 14.634087562561035,
      "activations/layer0_attention_weight_min": -14.88330078125,
      "activations/layer10_attention_weight_max": 40.37261962890625,
      "activations/layer10_attention_weight_min": -37.137237548828125,
      "activations/layer11_attention_weight_max": 34.318626403808594,
      "activations/layer11_attention_weight_min": -31.78573989868164,
      "activations/layer12_attention_weight_max": 18.51358985900879,
      "activations/layer12_attention_weight_min": -25.895145416259766,
      "activations/layer13_attention_weight_max": 43.529029846191406,
      "activations/layer13_attention_weight_min": -34.05426788330078,
      "activations/layer14_attention_weight_max": 37.51211166381836,
      "activations/layer14_attention_weight_min": -31.576595306396484,
      "activations/layer15_attention_weight_max": 34.493770599365234,
      "activations/layer15_attention_weight_min": -30.147857666015625,
      "activations/layer16_attention_weight_max": 32.72530746459961,
      "activations/layer16_attention_weight_min": -27.913278579711914,
      "activations/layer17_attention_weight_max": 52.94144058227539,
      "activations/layer17_attention_weight_min": -43.333152770996094,
      "activations/layer18_attention_weight_max": 48.63970947265625,
      "activations/layer18_attention_weight_min": -36.545467376708984,
      "activations/layer19_attention_weight_max": 20.872684478759766,
      "activations/layer19_attention_weight_min": -22.01666259765625,
      "activations/layer1_attention_weight_max": 16.601526260375977,
      "activations/layer1_attention_weight_min": -14.095346450805664,
      "activations/layer20_attention_weight_max": 20.11051368713379,
      "activations/layer20_attention_weight_min": -19.82326316833496,
      "activations/layer21_attention_weight_max": 37.454689025878906,
      "activations/layer21_attention_weight_min": -24.9029541015625,
      "activations/layer22_attention_weight_max": 34.96052551269531,
      "activations/layer22_attention_weight_min": -24.24703025817871,
      "activations/layer23_attention_weight_max": 35.277435302734375,
      "activations/layer23_attention_weight_min": -23.148040771484375,
      "activations/layer2_attention_weight_max": 31.3703670501709,
      "activations/layer2_attention_weight_min": -28.202098846435547,
      "activations/layer3_attention_weight_max": 84.92494201660156,
      "activations/layer3_attention_weight_min": -84.10244750976562,
      "activations/layer4_attention_weight_max": 103.85577392578125,
      "activations/layer4_attention_weight_min": -93.85649871826172,
      "activations/layer5_attention_weight_max": 85.64463806152344,
      "activations/layer5_attention_weight_min": -77.1878662109375,
      "activations/layer6_attention_weight_max": 56.69059753417969,
      "activations/layer6_attention_weight_min": -52.257301330566406,
      "activations/layer7_attention_weight_max": 74.02777099609375,
      "activations/layer7_attention_weight_min": -70.31523895263672,
      "activations/layer8_attention_weight_max": 44.239707946777344,
      "activations/layer8_attention_weight_min": -44.01679611206055,
      "activations/layer9_attention_weight_max": 51.49052047729492,
      "activations/layer9_attention_weight_min": -47.654972076416016,
      "epoch": 8.27,
      "learning_rate": 9.766212121212119e-05,
      "loss": 2.8004,
      "step": 142300
    },
    {
      "activations/layer0_attention_weight_max": 15.42603588104248,
      "activations/layer0_attention_weight_min": -14.800399780273438,
      "activations/layer10_attention_weight_max": 37.26188659667969,
      "activations/layer10_attention_weight_min": -34.982887268066406,
      "activations/layer11_attention_weight_max": 30.994831085205078,
      "activations/layer11_attention_weight_min": -33.920021057128906,
      "activations/layer12_attention_weight_max": 18.32227897644043,
      "activations/layer12_attention_weight_min": -29.681638717651367,
      "activations/layer13_attention_weight_max": 45.523250579833984,
      "activations/layer13_attention_weight_min": -30.560649871826172,
      "activations/layer14_attention_weight_max": 39.130096435546875,
      "activations/layer14_attention_weight_min": -31.67557716369629,
      "activations/layer15_attention_weight_max": 35.04277038574219,
      "activations/layer15_attention_weight_min": -29.525196075439453,
      "activations/layer16_attention_weight_max": 31.518497467041016,
      "activations/layer16_attention_weight_min": -28.024930953979492,
      "activations/layer17_attention_weight_max": 54.41154479980469,
      "activations/layer17_attention_weight_min": -43.900814056396484,
      "activations/layer18_attention_weight_max": 47.385643005371094,
      "activations/layer18_attention_weight_min": -37.88597869873047,
      "activations/layer19_attention_weight_max": 24.241886138916016,
      "activations/layer19_attention_weight_min": -25.53441619873047,
      "activations/layer1_attention_weight_max": 17.362499237060547,
      "activations/layer1_attention_weight_min": -15.83416748046875,
      "activations/layer20_attention_weight_max": 20.60774040222168,
      "activations/layer20_attention_weight_min": -24.579296112060547,
      "activations/layer21_attention_weight_max": 38.001991271972656,
      "activations/layer21_attention_weight_min": -24.984716415405273,
      "activations/layer22_attention_weight_max": 32.053184509277344,
      "activations/layer22_attention_weight_min": -25.751558303833008,
      "activations/layer23_attention_weight_max": 36.084228515625,
      "activations/layer23_attention_weight_min": -26.8914852142334,
      "activations/layer2_attention_weight_max": 32.91017150878906,
      "activations/layer2_attention_weight_min": -30.601470947265625,
      "activations/layer3_attention_weight_max": 88.62539672851562,
      "activations/layer3_attention_weight_min": -84.15673828125,
      "activations/layer4_attention_weight_max": 93.59584045410156,
      "activations/layer4_attention_weight_min": -90.87090301513672,
      "activations/layer5_attention_weight_max": 77.63471984863281,
      "activations/layer5_attention_weight_min": -76.23260498046875,
      "activations/layer6_attention_weight_max": 53.275840759277344,
      "activations/layer6_attention_weight_min": -53.06825637817383,
      "activations/layer7_attention_weight_max": 75.52613067626953,
      "activations/layer7_attention_weight_min": -71.92656707763672,
      "activations/layer8_attention_weight_max": 46.75300598144531,
      "activations/layer8_attention_weight_min": -45.3583869934082,
      "activations/layer9_attention_weight_max": 53.82380294799805,
      "activations/layer9_attention_weight_min": -46.74348831176758,
      "epoch": 8.27,
      "learning_rate": 9.764318181818181e-05,
      "loss": 2.8214,
      "step": 142350
    },
    {
      "activations/layer0_attention_weight_max": 14.864903450012207,
      "activations/layer0_attention_weight_min": -14.840131759643555,
      "activations/layer10_attention_weight_max": 43.05751419067383,
      "activations/layer10_attention_weight_min": -39.767581939697266,
      "activations/layer11_attention_weight_max": 33.7818603515625,
      "activations/layer11_attention_weight_min": -34.3095703125,
      "activations/layer12_attention_weight_max": 23.42401885986328,
      "activations/layer12_attention_weight_min": -23.53397560119629,
      "activations/layer13_attention_weight_max": 41.74531936645508,
      "activations/layer13_attention_weight_min": -31.74628257751465,
      "activations/layer14_attention_weight_max": 38.3138542175293,
      "activations/layer14_attention_weight_min": -31.97960090637207,
      "activations/layer15_attention_weight_max": 35.84153366088867,
      "activations/layer15_attention_weight_min": -30.867544174194336,
      "activations/layer16_attention_weight_max": 30.784833908081055,
      "activations/layer16_attention_weight_min": -28.448810577392578,
      "activations/layer17_attention_weight_max": 55.14076614379883,
      "activations/layer17_attention_weight_min": -41.562461853027344,
      "activations/layer18_attention_weight_max": 48.2557258605957,
      "activations/layer18_attention_weight_min": -37.5787467956543,
      "activations/layer19_attention_weight_max": 23.655471801757812,
      "activations/layer19_attention_weight_min": -22.200164794921875,
      "activations/layer1_attention_weight_max": 16.652902603149414,
      "activations/layer1_attention_weight_min": -14.559320449829102,
      "activations/layer20_attention_weight_max": 22.766807556152344,
      "activations/layer20_attention_weight_min": -25.552383422851562,
      "activations/layer21_attention_weight_max": 37.5645751953125,
      "activations/layer21_attention_weight_min": -22.671789169311523,
      "activations/layer22_attention_weight_max": 35.23622512817383,
      "activations/layer22_attention_weight_min": -24.42254066467285,
      "activations/layer23_attention_weight_max": 38.719783782958984,
      "activations/layer23_attention_weight_min": -21.85736656188965,
      "activations/layer2_attention_weight_max": 31.067684173583984,
      "activations/layer2_attention_weight_min": -32.09730529785156,
      "activations/layer3_attention_weight_max": 88.68920135498047,
      "activations/layer3_attention_weight_min": -89.40357208251953,
      "activations/layer4_attention_weight_max": 100.2816162109375,
      "activations/layer4_attention_weight_min": -97.67711639404297,
      "activations/layer5_attention_weight_max": 79.90507507324219,
      "activations/layer5_attention_weight_min": -78.05348205566406,
      "activations/layer6_attention_weight_max": 54.97377014160156,
      "activations/layer6_attention_weight_min": -53.76447677612305,
      "activations/layer7_attention_weight_max": 78.26323699951172,
      "activations/layer7_attention_weight_min": -77.14701080322266,
      "activations/layer8_attention_weight_max": 50.82063293457031,
      "activations/layer8_attention_weight_min": -49.702552795410156,
      "activations/layer9_attention_weight_max": 58.16643142700195,
      "activations/layer9_attention_weight_min": -52.560787200927734,
      "epoch": 8.27,
      "learning_rate": 9.762424242424242e-05,
      "loss": 2.811,
      "step": 142400
    },
    {
      "activations/layer0_attention_weight_max": 14.610993385314941,
      "activations/layer0_attention_weight_min": -14.528244018554688,
      "activations/layer10_attention_weight_max": 43.15547561645508,
      "activations/layer10_attention_weight_min": -37.80748748779297,
      "activations/layer11_attention_weight_max": 34.97734451293945,
      "activations/layer11_attention_weight_min": -35.23371887207031,
      "activations/layer12_attention_weight_max": 21.344253540039062,
      "activations/layer12_attention_weight_min": -24.881406784057617,
      "activations/layer13_attention_weight_max": 47.62973403930664,
      "activations/layer13_attention_weight_min": -31.449708938598633,
      "activations/layer14_attention_weight_max": 43.40522384643555,
      "activations/layer14_attention_weight_min": -31.293880462646484,
      "activations/layer15_attention_weight_max": 41.9283561706543,
      "activations/layer15_attention_weight_min": -32.342037200927734,
      "activations/layer16_attention_weight_max": 38.00908279418945,
      "activations/layer16_attention_weight_min": -29.277793884277344,
      "activations/layer17_attention_weight_max": 61.47471237182617,
      "activations/layer17_attention_weight_min": -43.608516693115234,
      "activations/layer18_attention_weight_max": 55.576576232910156,
      "activations/layer18_attention_weight_min": -37.95074462890625,
      "activations/layer19_attention_weight_max": 25.111955642700195,
      "activations/layer19_attention_weight_min": -21.756258010864258,
      "activations/layer1_attention_weight_max": 17.8729190826416,
      "activations/layer1_attention_weight_min": -15.03563404083252,
      "activations/layer20_attention_weight_max": 27.039043426513672,
      "activations/layer20_attention_weight_min": -20.651445388793945,
      "activations/layer21_attention_weight_max": 42.5602912902832,
      "activations/layer21_attention_weight_min": -23.20484161376953,
      "activations/layer22_attention_weight_max": 36.88187789916992,
      "activations/layer22_attention_weight_min": -25.086490631103516,
      "activations/layer23_attention_weight_max": 41.248023986816406,
      "activations/layer23_attention_weight_min": -20.47150230407715,
      "activations/layer2_attention_weight_max": 30.92641258239746,
      "activations/layer2_attention_weight_min": -28.687742233276367,
      "activations/layer3_attention_weight_max": 87.52091217041016,
      "activations/layer3_attention_weight_min": -86.23091888427734,
      "activations/layer4_attention_weight_max": 101.31513214111328,
      "activations/layer4_attention_weight_min": -98.35433197021484,
      "activations/layer5_attention_weight_max": 86.09524536132812,
      "activations/layer5_attention_weight_min": -80.31201171875,
      "activations/layer6_attention_weight_max": 57.5018196105957,
      "activations/layer6_attention_weight_min": -57.31553649902344,
      "activations/layer7_attention_weight_max": 80.8274917602539,
      "activations/layer7_attention_weight_min": -80.27806091308594,
      "activations/layer8_attention_weight_max": 49.9852294921875,
      "activations/layer8_attention_weight_min": -50.46337127685547,
      "activations/layer9_attention_weight_max": 56.27812576293945,
      "activations/layer9_attention_weight_min": -52.391414642333984,
      "epoch": 8.28,
      "learning_rate": 9.760530303030301e-05,
      "loss": 2.7874,
      "step": 142450
    },
    {
      "activations/layer0_attention_weight_max": 13.8117094039917,
      "activations/layer0_attention_weight_min": -14.696869850158691,
      "activations/layer10_attention_weight_max": 37.59381866455078,
      "activations/layer10_attention_weight_min": -37.048362731933594,
      "activations/layer11_attention_weight_max": 31.432363510131836,
      "activations/layer11_attention_weight_min": -33.100502014160156,
      "activations/layer12_attention_weight_max": 19.51038360595703,
      "activations/layer12_attention_weight_min": -24.52056312561035,
      "activations/layer13_attention_weight_max": 41.29848861694336,
      "activations/layer13_attention_weight_min": -33.93972396850586,
      "activations/layer14_attention_weight_max": 34.91547775268555,
      "activations/layer14_attention_weight_min": -30.184383392333984,
      "activations/layer15_attention_weight_max": 35.19181823730469,
      "activations/layer15_attention_weight_min": -30.86867904663086,
      "activations/layer16_attention_weight_max": 29.702068328857422,
      "activations/layer16_attention_weight_min": -29.17157554626465,
      "activations/layer17_attention_weight_max": 53.73922348022461,
      "activations/layer17_attention_weight_min": -41.67483139038086,
      "activations/layer18_attention_weight_max": 46.37075424194336,
      "activations/layer18_attention_weight_min": -35.08153533935547,
      "activations/layer19_attention_weight_max": 24.16420555114746,
      "activations/layer19_attention_weight_min": -21.549116134643555,
      "activations/layer1_attention_weight_max": 17.61861801147461,
      "activations/layer1_attention_weight_min": -14.067258834838867,
      "activations/layer20_attention_weight_max": 20.698806762695312,
      "activations/layer20_attention_weight_min": -19.495609283447266,
      "activations/layer21_attention_weight_max": 35.18898391723633,
      "activations/layer21_attention_weight_min": -22.243846893310547,
      "activations/layer22_attention_weight_max": 29.466218948364258,
      "activations/layer22_attention_weight_min": -23.39224624633789,
      "activations/layer23_attention_weight_max": 30.487272262573242,
      "activations/layer23_attention_weight_min": -21.98871421813965,
      "activations/layer2_attention_weight_max": 31.59430503845215,
      "activations/layer2_attention_weight_min": -31.934226989746094,
      "activations/layer3_attention_weight_max": 89.40860748291016,
      "activations/layer3_attention_weight_min": -89.61273193359375,
      "activations/layer4_attention_weight_max": 102.32548522949219,
      "activations/layer4_attention_weight_min": -99.7474136352539,
      "activations/layer5_attention_weight_max": 82.9420166015625,
      "activations/layer5_attention_weight_min": -79.17742156982422,
      "activations/layer6_attention_weight_max": 57.492279052734375,
      "activations/layer6_attention_weight_min": -55.564720153808594,
      "activations/layer7_attention_weight_max": 77.62873077392578,
      "activations/layer7_attention_weight_min": -74.16336822509766,
      "activations/layer8_attention_weight_max": 46.19663619995117,
      "activations/layer8_attention_weight_min": -46.3529052734375,
      "activations/layer9_attention_weight_max": 49.08191680908203,
      "activations/layer9_attention_weight_min": -51.61357498168945,
      "epoch": 8.28,
      "learning_rate": 9.758636363636363e-05,
      "loss": 2.7998,
      "step": 142500
    },
    {
      "activations/layer0_attention_weight_max": 15.415121078491211,
      "activations/layer0_attention_weight_min": -14.586690902709961,
      "activations/layer10_attention_weight_max": 38.23279571533203,
      "activations/layer10_attention_weight_min": -34.97398376464844,
      "activations/layer11_attention_weight_max": 32.64739990234375,
      "activations/layer11_attention_weight_min": -31.194473266601562,
      "activations/layer12_attention_weight_max": 24.99407958984375,
      "activations/layer12_attention_weight_min": -26.240144729614258,
      "activations/layer13_attention_weight_max": 44.83486557006836,
      "activations/layer13_attention_weight_min": -32.58635330200195,
      "activations/layer14_attention_weight_max": 35.95677185058594,
      "activations/layer14_attention_weight_min": -30.534669876098633,
      "activations/layer15_attention_weight_max": 33.317779541015625,
      "activations/layer15_attention_weight_min": -30.002750396728516,
      "activations/layer16_attention_weight_max": 31.247390747070312,
      "activations/layer16_attention_weight_min": -28.103273391723633,
      "activations/layer17_attention_weight_max": 54.579490661621094,
      "activations/layer17_attention_weight_min": -41.3814582824707,
      "activations/layer18_attention_weight_max": 46.32126235961914,
      "activations/layer18_attention_weight_min": -35.07927703857422,
      "activations/layer19_attention_weight_max": 21.491304397583008,
      "activations/layer19_attention_weight_min": -21.55561637878418,
      "activations/layer1_attention_weight_max": 16.986146926879883,
      "activations/layer1_attention_weight_min": -15.593539237976074,
      "activations/layer20_attention_weight_max": 22.429502487182617,
      "activations/layer20_attention_weight_min": -23.245885848999023,
      "activations/layer21_attention_weight_max": 38.112953186035156,
      "activations/layer21_attention_weight_min": -23.880168914794922,
      "activations/layer22_attention_weight_max": 30.56753921508789,
      "activations/layer22_attention_weight_min": -26.91072654724121,
      "activations/layer23_attention_weight_max": 34.05764389038086,
      "activations/layer23_attention_weight_min": -21.003246307373047,
      "activations/layer2_attention_weight_max": 33.04621887207031,
      "activations/layer2_attention_weight_min": -31.39107322692871,
      "activations/layer3_attention_weight_max": 86.29598236083984,
      "activations/layer3_attention_weight_min": -84.90009307861328,
      "activations/layer4_attention_weight_max": 98.95390319824219,
      "activations/layer4_attention_weight_min": -96.14046478271484,
      "activations/layer5_attention_weight_max": 83.07335662841797,
      "activations/layer5_attention_weight_min": -74.91110229492188,
      "activations/layer6_attention_weight_max": 57.656517028808594,
      "activations/layer6_attention_weight_min": -53.951419830322266,
      "activations/layer7_attention_weight_max": 76.0999984741211,
      "activations/layer7_attention_weight_min": -73.47570037841797,
      "activations/layer8_attention_weight_max": 44.5352668762207,
      "activations/layer8_attention_weight_min": -45.393672943115234,
      "activations/layer9_attention_weight_max": 49.17673873901367,
      "activations/layer9_attention_weight_min": -48.39707946777344,
      "epoch": 8.28,
      "learning_rate": 9.756742424242424e-05,
      "loss": 2.8026,
      "step": 142550
    },
    {
      "activations/layer0_attention_weight_max": 14.406946182250977,
      "activations/layer0_attention_weight_min": -14.53343391418457,
      "activations/layer10_attention_weight_max": 38.53662872314453,
      "activations/layer10_attention_weight_min": -37.78887939453125,
      "activations/layer11_attention_weight_max": 31.666034698486328,
      "activations/layer11_attention_weight_min": -32.952964782714844,
      "activations/layer12_attention_weight_max": 20.42572021484375,
      "activations/layer12_attention_weight_min": -26.099811553955078,
      "activations/layer13_attention_weight_max": 46.8002815246582,
      "activations/layer13_attention_weight_min": -33.86622619628906,
      "activations/layer14_attention_weight_max": 38.154693603515625,
      "activations/layer14_attention_weight_min": -30.665252685546875,
      "activations/layer15_attention_weight_max": 35.23725128173828,
      "activations/layer15_attention_weight_min": -31.84688377380371,
      "activations/layer16_attention_weight_max": 31.317447662353516,
      "activations/layer16_attention_weight_min": -27.677942276000977,
      "activations/layer17_attention_weight_max": 57.1179084777832,
      "activations/layer17_attention_weight_min": -44.32905578613281,
      "activations/layer18_attention_weight_max": 44.04508590698242,
      "activations/layer18_attention_weight_min": -35.239017486572266,
      "activations/layer19_attention_weight_max": 21.145313262939453,
      "activations/layer19_attention_weight_min": -19.009565353393555,
      "activations/layer1_attention_weight_max": 17.391399383544922,
      "activations/layer1_attention_weight_min": -17.254680633544922,
      "activations/layer20_attention_weight_max": 22.280046463012695,
      "activations/layer20_attention_weight_min": -19.714282989501953,
      "activations/layer21_attention_weight_max": 37.95683288574219,
      "activations/layer21_attention_weight_min": -22.04129981994629,
      "activations/layer22_attention_weight_max": 33.70036697387695,
      "activations/layer22_attention_weight_min": -24.6516170501709,
      "activations/layer23_attention_weight_max": 39.35811233520508,
      "activations/layer23_attention_weight_min": -22.25605010986328,
      "activations/layer2_attention_weight_max": 30.770360946655273,
      "activations/layer2_attention_weight_min": -29.152027130126953,
      "activations/layer3_attention_weight_max": 88.586181640625,
      "activations/layer3_attention_weight_min": -93.08744812011719,
      "activations/layer4_attention_weight_max": 99.5855712890625,
      "activations/layer4_attention_weight_min": -93.06524658203125,
      "activations/layer5_attention_weight_max": 83.72085571289062,
      "activations/layer5_attention_weight_min": -75.06813049316406,
      "activations/layer6_attention_weight_max": 60.485939025878906,
      "activations/layer6_attention_weight_min": -54.105648040771484,
      "activations/layer7_attention_weight_max": 81.75804138183594,
      "activations/layer7_attention_weight_min": -76.56511688232422,
      "activations/layer8_attention_weight_max": 47.378055572509766,
      "activations/layer8_attention_weight_min": -46.716148376464844,
      "activations/layer9_attention_weight_max": 57.744686126708984,
      "activations/layer9_attention_weight_min": -51.94135665893555,
      "epoch": 8.29,
      "learning_rate": 9.754848484848484e-05,
      "loss": 2.807,
      "step": 142600
    },
    {
      "activations/layer0_attention_weight_max": 14.638001441955566,
      "activations/layer0_attention_weight_min": -14.187919616699219,
      "activations/layer10_attention_weight_max": 41.4105339050293,
      "activations/layer10_attention_weight_min": -37.66792678833008,
      "activations/layer11_attention_weight_max": 34.96784210205078,
      "activations/layer11_attention_weight_min": -32.81654739379883,
      "activations/layer12_attention_weight_max": 19.609140396118164,
      "activations/layer12_attention_weight_min": -27.51132583618164,
      "activations/layer13_attention_weight_max": 41.68235397338867,
      "activations/layer13_attention_weight_min": -32.728755950927734,
      "activations/layer14_attention_weight_max": 35.82901382446289,
      "activations/layer14_attention_weight_min": -33.29130554199219,
      "activations/layer15_attention_weight_max": 32.00886917114258,
      "activations/layer15_attention_weight_min": -30.879045486450195,
      "activations/layer16_attention_weight_max": 31.4586181640625,
      "activations/layer16_attention_weight_min": -28.49517250061035,
      "activations/layer17_attention_weight_max": 49.62915802001953,
      "activations/layer17_attention_weight_min": -46.20378494262695,
      "activations/layer18_attention_weight_max": 46.6406135559082,
      "activations/layer18_attention_weight_min": -39.561466217041016,
      "activations/layer19_attention_weight_max": 21.79588508605957,
      "activations/layer19_attention_weight_min": -21.7890625,
      "activations/layer1_attention_weight_max": 17.56557846069336,
      "activations/layer1_attention_weight_min": -15.177347183227539,
      "activations/layer20_attention_weight_max": 24.211318969726562,
      "activations/layer20_attention_weight_min": -20.969362258911133,
      "activations/layer21_attention_weight_max": 38.184478759765625,
      "activations/layer21_attention_weight_min": -23.910568237304688,
      "activations/layer22_attention_weight_max": 29.221399307250977,
      "activations/layer22_attention_weight_min": -27.170774459838867,
      "activations/layer23_attention_weight_max": 34.41741180419922,
      "activations/layer23_attention_weight_min": -22.14578628540039,
      "activations/layer2_attention_weight_max": 32.10645294189453,
      "activations/layer2_attention_weight_min": -31.994991302490234,
      "activations/layer3_attention_weight_max": 91.37125396728516,
      "activations/layer3_attention_weight_min": -89.787841796875,
      "activations/layer4_attention_weight_max": 108.6089859008789,
      "activations/layer4_attention_weight_min": -96.72838592529297,
      "activations/layer5_attention_weight_max": 90.02693939208984,
      "activations/layer5_attention_weight_min": -76.84878540039062,
      "activations/layer6_attention_weight_max": 60.327293395996094,
      "activations/layer6_attention_weight_min": -54.28990173339844,
      "activations/layer7_attention_weight_max": 87.09345245361328,
      "activations/layer7_attention_weight_min": -77.73847198486328,
      "activations/layer8_attention_weight_max": 49.734798431396484,
      "activations/layer8_attention_weight_min": -47.77815628051758,
      "activations/layer9_attention_weight_max": 54.501251220703125,
      "activations/layer9_attention_weight_min": -53.075103759765625,
      "epoch": 8.29,
      "learning_rate": 9.752954545454544e-05,
      "loss": 2.8161,
      "step": 142650
    },
    {
      "activations/layer0_attention_weight_max": 15.100728034973145,
      "activations/layer0_attention_weight_min": -14.890380859375,
      "activations/layer10_attention_weight_max": 36.12406539916992,
      "activations/layer10_attention_weight_min": -35.16109085083008,
      "activations/layer11_attention_weight_max": 31.98810577392578,
      "activations/layer11_attention_weight_min": -32.52349090576172,
      "activations/layer12_attention_weight_max": 18.61383056640625,
      "activations/layer12_attention_weight_min": -25.56592559814453,
      "activations/layer13_attention_weight_max": 43.18327713012695,
      "activations/layer13_attention_weight_min": -35.45305252075195,
      "activations/layer14_attention_weight_max": 39.56264877319336,
      "activations/layer14_attention_weight_min": -29.9472713470459,
      "activations/layer15_attention_weight_max": 35.8559684753418,
      "activations/layer15_attention_weight_min": -29.374755859375,
      "activations/layer16_attention_weight_max": 32.0137825012207,
      "activations/layer16_attention_weight_min": -28.624584197998047,
      "activations/layer17_attention_weight_max": 54.830501556396484,
      "activations/layer17_attention_weight_min": -45.3161506652832,
      "activations/layer18_attention_weight_max": 45.9329833984375,
      "activations/layer18_attention_weight_min": -37.274898529052734,
      "activations/layer19_attention_weight_max": 20.63596534729004,
      "activations/layer19_attention_weight_min": -20.692646026611328,
      "activations/layer1_attention_weight_max": 16.969541549682617,
      "activations/layer1_attention_weight_min": -14.227201461791992,
      "activations/layer20_attention_weight_max": 21.07090950012207,
      "activations/layer20_attention_weight_min": -22.937604904174805,
      "activations/layer21_attention_weight_max": 37.240882873535156,
      "activations/layer21_attention_weight_min": -24.022563934326172,
      "activations/layer22_attention_weight_max": 31.411855697631836,
      "activations/layer22_attention_weight_min": -26.795455932617188,
      "activations/layer23_attention_weight_max": 36.36542510986328,
      "activations/layer23_attention_weight_min": -21.53510093688965,
      "activations/layer2_attention_weight_max": 30.86639404296875,
      "activations/layer2_attention_weight_min": -31.385574340820312,
      "activations/layer3_attention_weight_max": 90.67568969726562,
      "activations/layer3_attention_weight_min": -96.19021606445312,
      "activations/layer4_attention_weight_max": 105.37660217285156,
      "activations/layer4_attention_weight_min": -99.69062042236328,
      "activations/layer5_attention_weight_max": 86.78504943847656,
      "activations/layer5_attention_weight_min": -84.48372650146484,
      "activations/layer6_attention_weight_max": 54.42472457885742,
      "activations/layer6_attention_weight_min": -53.68947982788086,
      "activations/layer7_attention_weight_max": 76.35966491699219,
      "activations/layer7_attention_weight_min": -82.32572174072266,
      "activations/layer8_attention_weight_max": 45.3853759765625,
      "activations/layer8_attention_weight_min": -46.01758575439453,
      "activations/layer9_attention_weight_max": 51.457733154296875,
      "activations/layer9_attention_weight_min": -52.80134201049805,
      "epoch": 8.29,
      "learning_rate": 9.751060606060605e-05,
      "loss": 2.7959,
      "step": 142700
    },
    {
      "activations/layer0_attention_weight_max": 15.35153579711914,
      "activations/layer0_attention_weight_min": -14.692489624023438,
      "activations/layer10_attention_weight_max": 37.098548889160156,
      "activations/layer10_attention_weight_min": -35.03175735473633,
      "activations/layer11_attention_weight_max": 29.880508422851562,
      "activations/layer11_attention_weight_min": -32.733543395996094,
      "activations/layer12_attention_weight_max": 23.611499786376953,
      "activations/layer12_attention_weight_min": -28.5471248626709,
      "activations/layer13_attention_weight_max": 37.104286193847656,
      "activations/layer13_attention_weight_min": -30.618051528930664,
      "activations/layer14_attention_weight_max": 38.90703582763672,
      "activations/layer14_attention_weight_min": -30.999208450317383,
      "activations/layer15_attention_weight_max": 35.77518844604492,
      "activations/layer15_attention_weight_min": -29.785430908203125,
      "activations/layer16_attention_weight_max": 31.013837814331055,
      "activations/layer16_attention_weight_min": -29.384519577026367,
      "activations/layer17_attention_weight_max": 55.30392074584961,
      "activations/layer17_attention_weight_min": -45.44411849975586,
      "activations/layer18_attention_weight_max": 48.876522064208984,
      "activations/layer18_attention_weight_min": -37.452857971191406,
      "activations/layer19_attention_weight_max": 21.782119750976562,
      "activations/layer19_attention_weight_min": -20.002456665039062,
      "activations/layer1_attention_weight_max": 16.438079833984375,
      "activations/layer1_attention_weight_min": -13.053232192993164,
      "activations/layer20_attention_weight_max": 21.736291885375977,
      "activations/layer20_attention_weight_min": -20.576560974121094,
      "activations/layer21_attention_weight_max": 33.278411865234375,
      "activations/layer21_attention_weight_min": -23.060989379882812,
      "activations/layer22_attention_weight_max": 32.43620681762695,
      "activations/layer22_attention_weight_min": -25.61488914489746,
      "activations/layer23_attention_weight_max": 34.73014450073242,
      "activations/layer23_attention_weight_min": -24.448598861694336,
      "activations/layer2_attention_weight_max": 29.321304321289062,
      "activations/layer2_attention_weight_min": -28.03594970703125,
      "activations/layer3_attention_weight_max": 87.39575958251953,
      "activations/layer3_attention_weight_min": -84.35887145996094,
      "activations/layer4_attention_weight_max": 96.3545150756836,
      "activations/layer4_attention_weight_min": -89.29625701904297,
      "activations/layer5_attention_weight_max": 82.28067016601562,
      "activations/layer5_attention_weight_min": -75.1456527709961,
      "activations/layer6_attention_weight_max": 52.094261169433594,
      "activations/layer6_attention_weight_min": -50.322410583496094,
      "activations/layer7_attention_weight_max": 78.45354461669922,
      "activations/layer7_attention_weight_min": -73.86758422851562,
      "activations/layer8_attention_weight_max": 44.06604766845703,
      "activations/layer8_attention_weight_min": -44.43608474731445,
      "activations/layer9_attention_weight_max": 51.35280990600586,
      "activations/layer9_attention_weight_min": -49.74575424194336,
      "epoch": 8.29,
      "learning_rate": 9.749166666666666e-05,
      "loss": 2.7942,
      "step": 142750
    },
    {
      "activations/layer0_attention_weight_max": 14.812435150146484,
      "activations/layer0_attention_weight_min": -14.686274528503418,
      "activations/layer10_attention_weight_max": 39.395233154296875,
      "activations/layer10_attention_weight_min": -36.95619583129883,
      "activations/layer11_attention_weight_max": 35.3607177734375,
      "activations/layer11_attention_weight_min": -36.366668701171875,
      "activations/layer12_attention_weight_max": 28.454341888427734,
      "activations/layer12_attention_weight_min": -25.14862632751465,
      "activations/layer13_attention_weight_max": 51.38829803466797,
      "activations/layer13_attention_weight_min": -36.600128173828125,
      "activations/layer14_attention_weight_max": 49.971309661865234,
      "activations/layer14_attention_weight_min": -32.004371643066406,
      "activations/layer15_attention_weight_max": 40.59980773925781,
      "activations/layer15_attention_weight_min": -28.85106086730957,
      "activations/layer16_attention_weight_max": 37.563026428222656,
      "activations/layer16_attention_weight_min": -30.33888816833496,
      "activations/layer17_attention_weight_max": 59.39154815673828,
      "activations/layer17_attention_weight_min": -45.98079299926758,
      "activations/layer18_attention_weight_max": 56.16194152832031,
      "activations/layer18_attention_weight_min": -38.23726272583008,
      "activations/layer19_attention_weight_max": 23.234302520751953,
      "activations/layer19_attention_weight_min": -20.21454429626465,
      "activations/layer1_attention_weight_max": 16.52562141418457,
      "activations/layer1_attention_weight_min": -16.066192626953125,
      "activations/layer20_attention_weight_max": 23.385038375854492,
      "activations/layer20_attention_weight_min": -22.658897399902344,
      "activations/layer21_attention_weight_max": 37.86261749267578,
      "activations/layer21_attention_weight_min": -25.481590270996094,
      "activations/layer22_attention_weight_max": 36.83900451660156,
      "activations/layer22_attention_weight_min": -24.982107162475586,
      "activations/layer23_attention_weight_max": 37.50906753540039,
      "activations/layer23_attention_weight_min": -22.26952362060547,
      "activations/layer2_attention_weight_max": 35.140323638916016,
      "activations/layer2_attention_weight_min": -32.47172546386719,
      "activations/layer3_attention_weight_max": 95.53178405761719,
      "activations/layer3_attention_weight_min": -97.2435531616211,
      "activations/layer4_attention_weight_max": 112.02800750732422,
      "activations/layer4_attention_weight_min": -103.3878173828125,
      "activations/layer5_attention_weight_max": 85.67107391357422,
      "activations/layer5_attention_weight_min": -78.86528778076172,
      "activations/layer6_attention_weight_max": 58.59136962890625,
      "activations/layer6_attention_weight_min": -55.549442291259766,
      "activations/layer7_attention_weight_max": 79.2388916015625,
      "activations/layer7_attention_weight_min": -77.3733139038086,
      "activations/layer8_attention_weight_max": 49.76399612426758,
      "activations/layer8_attention_weight_min": -50.79875564575195,
      "activations/layer9_attention_weight_max": 55.57542037963867,
      "activations/layer9_attention_weight_min": -50.57614517211914,
      "epoch": 8.3,
      "learning_rate": 9.747272727272728e-05,
      "loss": 2.8047,
      "step": 142800
    },
    {
      "activations/layer0_attention_weight_max": 14.490145683288574,
      "activations/layer0_attention_weight_min": -14.989405632019043,
      "activations/layer10_attention_weight_max": 37.744384765625,
      "activations/layer10_attention_weight_min": -37.06868362426758,
      "activations/layer11_attention_weight_max": 29.58953094482422,
      "activations/layer11_attention_weight_min": -33.51839828491211,
      "activations/layer12_attention_weight_max": 21.403316497802734,
      "activations/layer12_attention_weight_min": -22.31554412841797,
      "activations/layer13_attention_weight_max": 40.78630828857422,
      "activations/layer13_attention_weight_min": -33.27719497680664,
      "activations/layer14_attention_weight_max": 38.340362548828125,
      "activations/layer14_attention_weight_min": -33.24764633178711,
      "activations/layer15_attention_weight_max": 33.137813568115234,
      "activations/layer15_attention_weight_min": -30.826282501220703,
      "activations/layer16_attention_weight_max": 30.05933380126953,
      "activations/layer16_attention_weight_min": -29.812557220458984,
      "activations/layer17_attention_weight_max": 54.89160919189453,
      "activations/layer17_attention_weight_min": -45.5610237121582,
      "activations/layer18_attention_weight_max": 45.609344482421875,
      "activations/layer18_attention_weight_min": -39.81995391845703,
      "activations/layer19_attention_weight_max": 22.169862747192383,
      "activations/layer19_attention_weight_min": -21.33363151550293,
      "activations/layer1_attention_weight_max": 17.427284240722656,
      "activations/layer1_attention_weight_min": -15.506244659423828,
      "activations/layer20_attention_weight_max": 21.19841194152832,
      "activations/layer20_attention_weight_min": -22.544353485107422,
      "activations/layer21_attention_weight_max": 39.099212646484375,
      "activations/layer21_attention_weight_min": -25.967138290405273,
      "activations/layer22_attention_weight_max": 31.490747451782227,
      "activations/layer22_attention_weight_min": -27.5333251953125,
      "activations/layer23_attention_weight_max": 34.342552185058594,
      "activations/layer23_attention_weight_min": -22.160030364990234,
      "activations/layer2_attention_weight_max": 31.188491821289062,
      "activations/layer2_attention_weight_min": -31.426563262939453,
      "activations/layer3_attention_weight_max": 81.94951629638672,
      "activations/layer3_attention_weight_min": -82.4906005859375,
      "activations/layer4_attention_weight_max": 94.06633758544922,
      "activations/layer4_attention_weight_min": -90.69628143310547,
      "activations/layer5_attention_weight_max": 80.8974380493164,
      "activations/layer5_attention_weight_min": -77.6330795288086,
      "activations/layer6_attention_weight_max": 53.77193832397461,
      "activations/layer6_attention_weight_min": -53.853702545166016,
      "activations/layer7_attention_weight_max": 74.60062408447266,
      "activations/layer7_attention_weight_min": -80.20162963867188,
      "activations/layer8_attention_weight_max": 44.13212966918945,
      "activations/layer8_attention_weight_min": -45.63483428955078,
      "activations/layer9_attention_weight_max": 48.989810943603516,
      "activations/layer9_attention_weight_min": -54.1343879699707,
      "epoch": 8.3,
      "learning_rate": 9.745378787878786e-05,
      "loss": 2.7999,
      "step": 142850
    },
    {
      "activations/layer0_attention_weight_max": 14.266118049621582,
      "activations/layer0_attention_weight_min": -15.1959810256958,
      "activations/layer10_attention_weight_max": 36.39707565307617,
      "activations/layer10_attention_weight_min": -36.26478958129883,
      "activations/layer11_attention_weight_max": 31.232282638549805,
      "activations/layer11_attention_weight_min": -31.473079681396484,
      "activations/layer12_attention_weight_max": 17.900793075561523,
      "activations/layer12_attention_weight_min": -28.077882766723633,
      "activations/layer13_attention_weight_max": 50.29087448120117,
      "activations/layer13_attention_weight_min": -33.527618408203125,
      "activations/layer14_attention_weight_max": 42.624847412109375,
      "activations/layer14_attention_weight_min": -33.50355911254883,
      "activations/layer15_attention_weight_max": 36.9411506652832,
      "activations/layer15_attention_weight_min": -34.15806579589844,
      "activations/layer16_attention_weight_max": 32.15274429321289,
      "activations/layer16_attention_weight_min": -31.188735961914062,
      "activations/layer17_attention_weight_max": 54.342437744140625,
      "activations/layer17_attention_weight_min": -48.564964294433594,
      "activations/layer18_attention_weight_max": 52.212188720703125,
      "activations/layer18_attention_weight_min": -43.71797561645508,
      "activations/layer19_attention_weight_max": 23.996095657348633,
      "activations/layer19_attention_weight_min": -24.597700119018555,
      "activations/layer1_attention_weight_max": 16.799007415771484,
      "activations/layer1_attention_weight_min": -14.530776977539062,
      "activations/layer20_attention_weight_max": 27.764524459838867,
      "activations/layer20_attention_weight_min": -23.47274398803711,
      "activations/layer21_attention_weight_max": 40.81681823730469,
      "activations/layer21_attention_weight_min": -24.724468231201172,
      "activations/layer22_attention_weight_max": 32.1494026184082,
      "activations/layer22_attention_weight_min": -25.706663131713867,
      "activations/layer23_attention_weight_max": 38.16741943359375,
      "activations/layer23_attention_weight_min": -21.06783676147461,
      "activations/layer2_attention_weight_max": 30.96365737915039,
      "activations/layer2_attention_weight_min": -30.1212215423584,
      "activations/layer3_attention_weight_max": 85.9795913696289,
      "activations/layer3_attention_weight_min": -94.74954986572266,
      "activations/layer4_attention_weight_max": 97.56231689453125,
      "activations/layer4_attention_weight_min": -94.2254409790039,
      "activations/layer5_attention_weight_max": 79.97927856445312,
      "activations/layer5_attention_weight_min": -80.30777740478516,
      "activations/layer6_attention_weight_max": 54.286495208740234,
      "activations/layer6_attention_weight_min": -55.077476501464844,
      "activations/layer7_attention_weight_max": 74.44841003417969,
      "activations/layer7_attention_weight_min": -73.69695281982422,
      "activations/layer8_attention_weight_max": 42.77987289428711,
      "activations/layer8_attention_weight_min": -44.71833419799805,
      "activations/layer9_attention_weight_max": 50.813507080078125,
      "activations/layer9_attention_weight_min": -48.19387435913086,
      "epoch": 8.3,
      "learning_rate": 9.743484848484848e-05,
      "loss": 2.7974,
      "step": 142900
    },
    {
      "activations/layer0_attention_weight_max": 14.944323539733887,
      "activations/layer0_attention_weight_min": -14.790860176086426,
      "activations/layer10_attention_weight_max": 39.445037841796875,
      "activations/layer10_attention_weight_min": -37.866600036621094,
      "activations/layer11_attention_weight_max": 32.436275482177734,
      "activations/layer11_attention_weight_min": -32.200660705566406,
      "activations/layer12_attention_weight_max": 20.547067642211914,
      "activations/layer12_attention_weight_min": -26.102609634399414,
      "activations/layer13_attention_weight_max": 41.93720245361328,
      "activations/layer13_attention_weight_min": -31.13208770751953,
      "activations/layer14_attention_weight_max": 37.29243850708008,
      "activations/layer14_attention_weight_min": -31.31926155090332,
      "activations/layer15_attention_weight_max": 35.322418212890625,
      "activations/layer15_attention_weight_min": -28.83696174621582,
      "activations/layer16_attention_weight_max": 32.47246170043945,
      "activations/layer16_attention_weight_min": -27.00135612487793,
      "activations/layer17_attention_weight_max": 53.7939453125,
      "activations/layer17_attention_weight_min": -43.82167434692383,
      "activations/layer18_attention_weight_max": 47.23807144165039,
      "activations/layer18_attention_weight_min": -38.286590576171875,
      "activations/layer19_attention_weight_max": 22.095388412475586,
      "activations/layer19_attention_weight_min": -21.05868148803711,
      "activations/layer1_attention_weight_max": 16.68093490600586,
      "activations/layer1_attention_weight_min": -16.655004501342773,
      "activations/layer20_attention_weight_max": 20.638328552246094,
      "activations/layer20_attention_weight_min": -23.762983322143555,
      "activations/layer21_attention_weight_max": 37.0371208190918,
      "activations/layer21_attention_weight_min": -25.8521785736084,
      "activations/layer22_attention_weight_max": 34.86626434326172,
      "activations/layer22_attention_weight_min": -27.04050636291504,
      "activations/layer23_attention_weight_max": 33.13007736206055,
      "activations/layer23_attention_weight_min": -22.99488067626953,
      "activations/layer2_attention_weight_max": 31.993680953979492,
      "activations/layer2_attention_weight_min": -30.48177719116211,
      "activations/layer3_attention_weight_max": 93.15239715576172,
      "activations/layer3_attention_weight_min": -96.31986999511719,
      "activations/layer4_attention_weight_max": 104.07275390625,
      "activations/layer4_attention_weight_min": -101.33975219726562,
      "activations/layer5_attention_weight_max": 82.78330993652344,
      "activations/layer5_attention_weight_min": -80.72723388671875,
      "activations/layer6_attention_weight_max": 56.36655807495117,
      "activations/layer6_attention_weight_min": -54.20981216430664,
      "activations/layer7_attention_weight_max": 79.79449462890625,
      "activations/layer7_attention_weight_min": -74.97150421142578,
      "activations/layer8_attention_weight_max": 46.89607238769531,
      "activations/layer8_attention_weight_min": -47.61076736450195,
      "activations/layer9_attention_weight_max": 62.942481994628906,
      "activations/layer9_attention_weight_min": -50.423641204833984,
      "epoch": 8.31,
      "learning_rate": 9.741628787878787e-05,
      "loss": 2.8006,
      "step": 142950
    },
    {
      "activations/layer0_attention_weight_max": 14.826828002929688,
      "activations/layer0_attention_weight_min": -14.436104774475098,
      "activations/layer10_attention_weight_max": 41.5144157409668,
      "activations/layer10_attention_weight_min": -36.81496047973633,
      "activations/layer11_attention_weight_max": 35.81004333496094,
      "activations/layer11_attention_weight_min": -33.59715270996094,
      "activations/layer12_attention_weight_max": 21.77530288696289,
      "activations/layer12_attention_weight_min": -26.83896255493164,
      "activations/layer13_attention_weight_max": 37.77463912963867,
      "activations/layer13_attention_weight_min": -31.605920791625977,
      "activations/layer14_attention_weight_max": 39.26822280883789,
      "activations/layer14_attention_weight_min": -30.886327743530273,
      "activations/layer15_attention_weight_max": 33.051883697509766,
      "activations/layer15_attention_weight_min": -30.14217185974121,
      "activations/layer16_attention_weight_max": 30.573091506958008,
      "activations/layer16_attention_weight_min": -27.221569061279297,
      "activations/layer17_attention_weight_max": 52.86082458496094,
      "activations/layer17_attention_weight_min": -42.954349517822266,
      "activations/layer18_attention_weight_max": 44.52385330200195,
      "activations/layer18_attention_weight_min": -37.298919677734375,
      "activations/layer19_attention_weight_max": 22.28657341003418,
      "activations/layer19_attention_weight_min": -21.59143829345703,
      "activations/layer1_attention_weight_max": 17.056934356689453,
      "activations/layer1_attention_weight_min": -13.784812927246094,
      "activations/layer20_attention_weight_max": 22.11266326904297,
      "activations/layer20_attention_weight_min": -22.271373748779297,
      "activations/layer21_attention_weight_max": 41.9306755065918,
      "activations/layer21_attention_weight_min": -24.551345825195312,
      "activations/layer22_attention_weight_max": 35.58416748046875,
      "activations/layer22_attention_weight_min": -29.16594696044922,
      "activations/layer23_attention_weight_max": 38.944129943847656,
      "activations/layer23_attention_weight_min": -21.30368423461914,
      "activations/layer2_attention_weight_max": 31.41164779663086,
      "activations/layer2_attention_weight_min": -28.759794235229492,
      "activations/layer3_attention_weight_max": 83.59687042236328,
      "activations/layer3_attention_weight_min": -86.63966369628906,
      "activations/layer4_attention_weight_max": 97.56255340576172,
      "activations/layer4_attention_weight_min": -94.16094970703125,
      "activations/layer5_attention_weight_max": 78.88285064697266,
      "activations/layer5_attention_weight_min": -77.35130310058594,
      "activations/layer6_attention_weight_max": 53.13675308227539,
      "activations/layer6_attention_weight_min": -51.93770980834961,
      "activations/layer7_attention_weight_max": 75.113037109375,
      "activations/layer7_attention_weight_min": -72.0454330444336,
      "activations/layer8_attention_weight_max": 47.34706497192383,
      "activations/layer8_attention_weight_min": -45.333702087402344,
      "activations/layer9_attention_weight_max": 57.46245193481445,
      "activations/layer9_attention_weight_min": -48.796531677246094,
      "epoch": 8.31,
      "learning_rate": 9.739734848484848e-05,
      "loss": 2.8017,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_loss": 2.759765625,
      "eval_runtime": 8.5757,
      "eval_samples_per_second": 500.715,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_openwebtext_loss": 2.759765625,
      "eval_openwebtext_ppl": 15.796140293991145,
      "eval_openwebtext_runtime": 8.5757,
      "eval_openwebtext_samples_per_second": 500.715,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 2.0525,
      "eval_wikitext_samples_per_second": 222.165,
      "step": 143000
    },
    {
      "epoch": 8.31,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.6261,
      "eval_lambada_samples_per_second": 505.813,
      "step": 143000
    },
    {
      "activations/layer0_attention_weight_max": 14.806563377380371,
      "activations/layer0_attention_weight_min": -14.2892484664917,
      "activations/layer10_attention_weight_max": 38.43400192260742,
      "activations/layer10_attention_weight_min": -36.4716796875,
      "activations/layer11_attention_weight_max": 32.82386779785156,
      "activations/layer11_attention_weight_min": -32.75489807128906,
      "activations/layer12_attention_weight_max": 17.598621368408203,
      "activations/layer12_attention_weight_min": -26.664621353149414,
      "activations/layer13_attention_weight_max": 41.23073959350586,
      "activations/layer13_attention_weight_min": -32.576271057128906,
      "activations/layer14_attention_weight_max": 39.7558708190918,
      "activations/layer14_attention_weight_min": -33.72218704223633,
      "activations/layer15_attention_weight_max": 34.03615951538086,
      "activations/layer15_attention_weight_min": -31.637269973754883,
      "activations/layer16_attention_weight_max": 30.645381927490234,
      "activations/layer16_attention_weight_min": -29.853496551513672,
      "activations/layer17_attention_weight_max": 54.23823928833008,
      "activations/layer17_attention_weight_min": -48.09953689575195,
      "activations/layer18_attention_weight_max": 47.006683349609375,
      "activations/layer18_attention_weight_min": -41.17430877685547,
      "activations/layer19_attention_weight_max": 23.315645217895508,
      "activations/layer19_attention_weight_min": -23.55612564086914,
      "activations/layer1_attention_weight_max": 17.012378692626953,
      "activations/layer1_attention_weight_min": -14.7304105758667,
      "activations/layer20_attention_weight_max": 25.60002899169922,
      "activations/layer20_attention_weight_min": -21.943286895751953,
      "activations/layer21_attention_weight_max": 40.355220794677734,
      "activations/layer21_attention_weight_min": -23.062789916992188,
      "activations/layer22_attention_weight_max": 32.10658264160156,
      "activations/layer22_attention_weight_min": -26.729278564453125,
      "activations/layer23_attention_weight_max": 34.78004455566406,
      "activations/layer23_attention_weight_min": -23.33955955505371,
      "activations/layer2_attention_weight_max": 32.230064392089844,
      "activations/layer2_attention_weight_min": -31.113262176513672,
      "activations/layer3_attention_weight_max": 88.72311401367188,
      "activations/layer3_attention_weight_min": -90.9395980834961,
      "activations/layer4_attention_weight_max": 101.28849792480469,
      "activations/layer4_attention_weight_min": -98.383056640625,
      "activations/layer5_attention_weight_max": 82.74143981933594,
      "activations/layer5_attention_weight_min": -77.32833862304688,
      "activations/layer6_attention_weight_max": 54.13787078857422,
      "activations/layer6_attention_weight_min": -54.03813171386719,
      "activations/layer7_attention_weight_max": 81.18353271484375,
      "activations/layer7_attention_weight_min": -73.14617919921875,
      "activations/layer8_attention_weight_max": 43.904537200927734,
      "activations/layer8_attention_weight_min": -47.05387878417969,
      "activations/layer9_attention_weight_max": 49.380653381347656,
      "activations/layer9_attention_weight_min": -48.842281341552734,
      "epoch": 8.31,
      "learning_rate": 9.737840909090907e-05,
      "loss": 2.8033,
      "step": 143050
    },
    {
      "activations/layer0_attention_weight_max": 15.201363563537598,
      "activations/layer0_attention_weight_min": -14.490682601928711,
      "activations/layer10_attention_weight_max": 38.46284866333008,
      "activations/layer10_attention_weight_min": -35.94736862182617,
      "activations/layer11_attention_weight_max": 31.621837615966797,
      "activations/layer11_attention_weight_min": -31.897010803222656,
      "activations/layer12_attention_weight_max": 19.045974731445312,
      "activations/layer12_attention_weight_min": -26.009078979492188,
      "activations/layer13_attention_weight_max": 44.19888687133789,
      "activations/layer13_attention_weight_min": -31.55710792541504,
      "activations/layer14_attention_weight_max": 39.365692138671875,
      "activations/layer14_attention_weight_min": -30.5325927734375,
      "activations/layer15_attention_weight_max": 36.572471618652344,
      "activations/layer15_attention_weight_min": -28.152496337890625,
      "activations/layer16_attention_weight_max": 31.766651153564453,
      "activations/layer16_attention_weight_min": -27.722057342529297,
      "activations/layer17_attention_weight_max": 55.54171371459961,
      "activations/layer17_attention_weight_min": -43.497169494628906,
      "activations/layer18_attention_weight_max": 53.27275848388672,
      "activations/layer18_attention_weight_min": -37.46009826660156,
      "activations/layer19_attention_weight_max": 27.447656631469727,
      "activations/layer19_attention_weight_min": -24.604246139526367,
      "activations/layer1_attention_weight_max": 16.80386734008789,
      "activations/layer1_attention_weight_min": -14.244391441345215,
      "activations/layer20_attention_weight_max": 28.436588287353516,
      "activations/layer20_attention_weight_min": -24.058862686157227,
      "activations/layer21_attention_weight_max": 42.2227668762207,
      "activations/layer21_attention_weight_min": -21.786359786987305,
      "activations/layer22_attention_weight_max": 32.55281066894531,
      "activations/layer22_attention_weight_min": -23.990978240966797,
      "activations/layer23_attention_weight_max": 36.40620422363281,
      "activations/layer23_attention_weight_min": -21.325942993164062,
      "activations/layer2_attention_weight_max": 31.951688766479492,
      "activations/layer2_attention_weight_min": -29.743267059326172,
      "activations/layer3_attention_weight_max": 88.50665283203125,
      "activations/layer3_attention_weight_min": -92.55879974365234,
      "activations/layer4_attention_weight_max": 104.53108215332031,
      "activations/layer4_attention_weight_min": -95.28153228759766,
      "activations/layer5_attention_weight_max": 82.22843933105469,
      "activations/layer5_attention_weight_min": -77.90157318115234,
      "activations/layer6_attention_weight_max": 53.96529769897461,
      "activations/layer6_attention_weight_min": -52.977108001708984,
      "activations/layer7_attention_weight_max": 83.74750518798828,
      "activations/layer7_attention_weight_min": -77.88993835449219,
      "activations/layer8_attention_weight_max": 46.23689651489258,
      "activations/layer8_attention_weight_min": -47.21371841430664,
      "activations/layer9_attention_weight_max": 51.52095413208008,
      "activations/layer9_attention_weight_min": -50.331214904785156,
      "epoch": 8.31,
      "learning_rate": 9.735946969696969e-05,
      "loss": 2.8082,
      "step": 143100
    },
    {
      "activations/layer0_attention_weight_max": 15.276403427124023,
      "activations/layer0_attention_weight_min": -14.817008018493652,
      "activations/layer10_attention_weight_max": 39.1440544128418,
      "activations/layer10_attention_weight_min": -39.0947265625,
      "activations/layer11_attention_weight_max": 32.95802307128906,
      "activations/layer11_attention_weight_min": -34.961341857910156,
      "activations/layer12_attention_weight_max": 22.66404151916504,
      "activations/layer12_attention_weight_min": -20.099103927612305,
      "activations/layer13_attention_weight_max": 43.24277114868164,
      "activations/layer13_attention_weight_min": -32.329063415527344,
      "activations/layer14_attention_weight_max": 45.44664001464844,
      "activations/layer14_attention_weight_min": -31.95419692993164,
      "activations/layer15_attention_weight_max": 36.20917510986328,
      "activations/layer15_attention_weight_min": -31.91944694519043,
      "activations/layer16_attention_weight_max": 33.54904556274414,
      "activations/layer16_attention_weight_min": -28.670581817626953,
      "activations/layer17_attention_weight_max": 58.11098098754883,
      "activations/layer17_attention_weight_min": -43.83855438232422,
      "activations/layer18_attention_weight_max": 56.69841003417969,
      "activations/layer18_attention_weight_min": -36.839324951171875,
      "activations/layer19_attention_weight_max": 24.41922378540039,
      "activations/layer19_attention_weight_min": -21.63391876220703,
      "activations/layer1_attention_weight_max": 17.63170623779297,
      "activations/layer1_attention_weight_min": -14.478763580322266,
      "activations/layer20_attention_weight_max": 23.974754333496094,
      "activations/layer20_attention_weight_min": -22.46220588684082,
      "activations/layer21_attention_weight_max": 37.06624221801758,
      "activations/layer21_attention_weight_min": -23.48453712463379,
      "activations/layer22_attention_weight_max": 34.032649993896484,
      "activations/layer22_attention_weight_min": -25.369443893432617,
      "activations/layer23_attention_weight_max": 37.76362609863281,
      "activations/layer23_attention_weight_min": -22.208087921142578,
      "activations/layer2_attention_weight_max": 30.919219970703125,
      "activations/layer2_attention_weight_min": -29.593053817749023,
      "activations/layer3_attention_weight_max": 92.11909484863281,
      "activations/layer3_attention_weight_min": -90.64900207519531,
      "activations/layer4_attention_weight_max": 107.26903533935547,
      "activations/layer4_attention_weight_min": -101.17998504638672,
      "activations/layer5_attention_weight_max": 87.56981658935547,
      "activations/layer5_attention_weight_min": -84.01953125,
      "activations/layer6_attention_weight_max": 56.68025207519531,
      "activations/layer6_attention_weight_min": -54.62758255004883,
      "activations/layer7_attention_weight_max": 81.85746002197266,
      "activations/layer7_attention_weight_min": -84.16991424560547,
      "activations/layer8_attention_weight_max": 48.48793411254883,
      "activations/layer8_attention_weight_min": -49.87179183959961,
      "activations/layer9_attention_weight_max": 52.80670166015625,
      "activations/layer9_attention_weight_min": -54.5078239440918,
      "epoch": 8.32,
      "learning_rate": 9.73405303030303e-05,
      "loss": 2.7937,
      "step": 143150
    },
    {
      "activations/layer0_attention_weight_max": 15.055686950683594,
      "activations/layer0_attention_weight_min": -14.748896598815918,
      "activations/layer10_attention_weight_max": 39.320255279541016,
      "activations/layer10_attention_weight_min": -36.896148681640625,
      "activations/layer11_attention_weight_max": 32.720619201660156,
      "activations/layer11_attention_weight_min": -31.925987243652344,
      "activations/layer12_attention_weight_max": 20.10760498046875,
      "activations/layer12_attention_weight_min": -26.436094284057617,
      "activations/layer13_attention_weight_max": 42.608924865722656,
      "activations/layer13_attention_weight_min": -31.9677791595459,
      "activations/layer14_attention_weight_max": 38.32256317138672,
      "activations/layer14_attention_weight_min": -29.996532440185547,
      "activations/layer15_attention_weight_max": 36.94599151611328,
      "activations/layer15_attention_weight_min": -30.49919319152832,
      "activations/layer16_attention_weight_max": 36.30551528930664,
      "activations/layer16_attention_weight_min": -27.4537410736084,
      "activations/layer17_attention_weight_max": 53.688575744628906,
      "activations/layer17_attention_weight_min": -40.465370178222656,
      "activations/layer18_attention_weight_max": 44.10184860229492,
      "activations/layer18_attention_weight_min": -35.21086502075195,
      "activations/layer19_attention_weight_max": 20.747854232788086,
      "activations/layer19_attention_weight_min": -22.525836944580078,
      "activations/layer1_attention_weight_max": 16.45846939086914,
      "activations/layer1_attention_weight_min": -14.927153587341309,
      "activations/layer20_attention_weight_max": 20.692752838134766,
      "activations/layer20_attention_weight_min": -22.0972957611084,
      "activations/layer21_attention_weight_max": 35.8935661315918,
      "activations/layer21_attention_weight_min": -23.266672134399414,
      "activations/layer22_attention_weight_max": 31.40267562866211,
      "activations/layer22_attention_weight_min": -26.276521682739258,
      "activations/layer23_attention_weight_max": 33.364219665527344,
      "activations/layer23_attention_weight_min": -21.797597885131836,
      "activations/layer2_attention_weight_max": 34.11683654785156,
      "activations/layer2_attention_weight_min": -31.973295211791992,
      "activations/layer3_attention_weight_max": 94.35757446289062,
      "activations/layer3_attention_weight_min": -97.44515228271484,
      "activations/layer4_attention_weight_max": 101.61791229248047,
      "activations/layer4_attention_weight_min": -99.0473861694336,
      "activations/layer5_attention_weight_max": 82.21566772460938,
      "activations/layer5_attention_weight_min": -83.230712890625,
      "activations/layer6_attention_weight_max": 56.75147247314453,
      "activations/layer6_attention_weight_min": -54.189292907714844,
      "activations/layer7_attention_weight_max": 79.40167999267578,
      "activations/layer7_attention_weight_min": -73.32534790039062,
      "activations/layer8_attention_weight_max": 48.13270950317383,
      "activations/layer8_attention_weight_min": -47.6950569152832,
      "activations/layer9_attention_weight_max": 52.836883544921875,
      "activations/layer9_attention_weight_min": -52.03059768676758,
      "epoch": 8.32,
      "learning_rate": 9.732159090909089e-05,
      "loss": 2.7905,
      "step": 143200
    },
    {
      "activations/layer0_attention_weight_max": 14.17834186553955,
      "activations/layer0_attention_weight_min": -14.342574119567871,
      "activations/layer10_attention_weight_max": 39.2822151184082,
      "activations/layer10_attention_weight_min": -37.26002502441406,
      "activations/layer11_attention_weight_max": 33.19169235229492,
      "activations/layer11_attention_weight_min": -32.35641098022461,
      "activations/layer12_attention_weight_max": 19.17026138305664,
      "activations/layer12_attention_weight_min": -32.73716354370117,
      "activations/layer13_attention_weight_max": 40.38275909423828,
      "activations/layer13_attention_weight_min": -28.819658279418945,
      "activations/layer14_attention_weight_max": 36.7075309753418,
      "activations/layer14_attention_weight_min": -30.627901077270508,
      "activations/layer15_attention_weight_max": 33.6009635925293,
      "activations/layer15_attention_weight_min": -28.689987182617188,
      "activations/layer16_attention_weight_max": 30.28029441833496,
      "activations/layer16_attention_weight_min": -27.58648681640625,
      "activations/layer17_attention_weight_max": 55.67241287231445,
      "activations/layer17_attention_weight_min": -43.36403274536133,
      "activations/layer18_attention_weight_max": 51.54059600830078,
      "activations/layer18_attention_weight_min": -35.7236442565918,
      "activations/layer19_attention_weight_max": 20.675397872924805,
      "activations/layer19_attention_weight_min": -22.113954544067383,
      "activations/layer1_attention_weight_max": 19.16216278076172,
      "activations/layer1_attention_weight_min": -17.290348052978516,
      "activations/layer20_attention_weight_max": 21.358522415161133,
      "activations/layer20_attention_weight_min": -24.431074142456055,
      "activations/layer21_attention_weight_max": 35.07194137573242,
      "activations/layer21_attention_weight_min": -22.70722198486328,
      "activations/layer22_attention_weight_max": 31.580421447753906,
      "activations/layer22_attention_weight_min": -24.754060745239258,
      "activations/layer23_attention_weight_max": 33.38676452636719,
      "activations/layer23_attention_weight_min": -21.803295135498047,
      "activations/layer2_attention_weight_max": 31.478290557861328,
      "activations/layer2_attention_weight_min": -29.10134506225586,
      "activations/layer3_attention_weight_max": 92.32108306884766,
      "activations/layer3_attention_weight_min": -89.73123931884766,
      "activations/layer4_attention_weight_max": 104.7837905883789,
      "activations/layer4_attention_weight_min": -98.1167221069336,
      "activations/layer5_attention_weight_max": 84.19174194335938,
      "activations/layer5_attention_weight_min": -79.59970092773438,
      "activations/layer6_attention_weight_max": 57.47177505493164,
      "activations/layer6_attention_weight_min": -55.252777099609375,
      "activations/layer7_attention_weight_max": 81.76079559326172,
      "activations/layer7_attention_weight_min": -73.32150268554688,
      "activations/layer8_attention_weight_max": 47.78178787231445,
      "activations/layer8_attention_weight_min": -46.71550369262695,
      "activations/layer9_attention_weight_max": 57.61806106567383,
      "activations/layer9_attention_weight_min": -56.4152946472168,
      "epoch": 8.32,
      "learning_rate": 9.73026515151515e-05,
      "loss": 2.7825,
      "step": 143250
    },
    {
      "activations/layer0_attention_weight_max": 15.735193252563477,
      "activations/layer0_attention_weight_min": -14.389838218688965,
      "activations/layer10_attention_weight_max": 39.714111328125,
      "activations/layer10_attention_weight_min": -36.048057556152344,
      "activations/layer11_attention_weight_max": 33.22076416015625,
      "activations/layer11_attention_weight_min": -31.612831115722656,
      "activations/layer12_attention_weight_max": 23.750497817993164,
      "activations/layer12_attention_weight_min": -26.165170669555664,
      "activations/layer13_attention_weight_max": 43.92958068847656,
      "activations/layer13_attention_weight_min": -32.909400939941406,
      "activations/layer14_attention_weight_max": 41.0789680480957,
      "activations/layer14_attention_weight_min": -32.31173324584961,
      "activations/layer15_attention_weight_max": 36.30534744262695,
      "activations/layer15_attention_weight_min": -33.083499908447266,
      "activations/layer16_attention_weight_max": 35.617340087890625,
      "activations/layer16_attention_weight_min": -28.188613891601562,
      "activations/layer17_attention_weight_max": 57.30342483520508,
      "activations/layer17_attention_weight_min": -43.4257698059082,
      "activations/layer18_attention_weight_max": 56.30065155029297,
      "activations/layer18_attention_weight_min": -37.47615051269531,
      "activations/layer19_attention_weight_max": 24.915328979492188,
      "activations/layer19_attention_weight_min": -20.423063278198242,
      "activations/layer1_attention_weight_max": 17.30356216430664,
      "activations/layer1_attention_weight_min": -14.384223937988281,
      "activations/layer20_attention_weight_max": 23.961740493774414,
      "activations/layer20_attention_weight_min": -23.128416061401367,
      "activations/layer21_attention_weight_max": 42.258018493652344,
      "activations/layer21_attention_weight_min": -22.710939407348633,
      "activations/layer22_attention_weight_max": 35.295555114746094,
      "activations/layer22_attention_weight_min": -26.306325912475586,
      "activations/layer23_attention_weight_max": 37.144134521484375,
      "activations/layer23_attention_weight_min": -20.98761749267578,
      "activations/layer2_attention_weight_max": 32.22491455078125,
      "activations/layer2_attention_weight_min": -31.04538345336914,
      "activations/layer3_attention_weight_max": 90.054931640625,
      "activations/layer3_attention_weight_min": -99.47894287109375,
      "activations/layer4_attention_weight_max": 104.312744140625,
      "activations/layer4_attention_weight_min": -101.1589584350586,
      "activations/layer5_attention_weight_max": 83.31182861328125,
      "activations/layer5_attention_weight_min": -77.56317138671875,
      "activations/layer6_attention_weight_max": 54.10251998901367,
      "activations/layer6_attention_weight_min": -52.63907241821289,
      "activations/layer7_attention_weight_max": 78.9345703125,
      "activations/layer7_attention_weight_min": -73.16567993164062,
      "activations/layer8_attention_weight_max": 44.78355026245117,
      "activations/layer8_attention_weight_min": -45.01656723022461,
      "activations/layer9_attention_weight_max": 52.2291145324707,
      "activations/layer9_attention_weight_min": -49.7097282409668,
      "epoch": 8.33,
      "learning_rate": 9.728371212121212e-05,
      "loss": 2.7927,
      "step": 143300
    },
    {
      "activations/layer0_attention_weight_max": 14.73610782623291,
      "activations/layer0_attention_weight_min": -14.941886901855469,
      "activations/layer10_attention_weight_max": 46.318931579589844,
      "activations/layer10_attention_weight_min": -42.350555419921875,
      "activations/layer11_attention_weight_max": 41.880496978759766,
      "activations/layer11_attention_weight_min": -34.90568923950195,
      "activations/layer12_attention_weight_max": 21.174495697021484,
      "activations/layer12_attention_weight_min": -22.94396209716797,
      "activations/layer13_attention_weight_max": 80.4337387084961,
      "activations/layer13_attention_weight_min": -40.174373626708984,
      "activations/layer14_attention_weight_max": 106.53539276123047,
      "activations/layer14_attention_weight_min": -43.475616455078125,
      "activations/layer15_attention_weight_max": 83.9752426147461,
      "activations/layer15_attention_weight_min": -36.325679779052734,
      "activations/layer16_attention_weight_max": 43.540870666503906,
      "activations/layer16_attention_weight_min": -30.752391815185547,
      "activations/layer17_attention_weight_max": 67.61182403564453,
      "activations/layer17_attention_weight_min": -52.20642852783203,
      "activations/layer18_attention_weight_max": 62.54145431518555,
      "activations/layer18_attention_weight_min": -47.71831512451172,
      "activations/layer19_attention_weight_max": 38.777557373046875,
      "activations/layer19_attention_weight_min": -27.195722579956055,
      "activations/layer1_attention_weight_max": 16.493345260620117,
      "activations/layer1_attention_weight_min": -14.793676376342773,
      "activations/layer20_attention_weight_max": 41.26072692871094,
      "activations/layer20_attention_weight_min": -21.870054244995117,
      "activations/layer21_attention_weight_max": 75.73054504394531,
      "activations/layer21_attention_weight_min": -28.278030395507812,
      "activations/layer22_attention_weight_max": 48.353607177734375,
      "activations/layer22_attention_weight_min": -25.614004135131836,
      "activations/layer23_attention_weight_max": 41.34695053100586,
      "activations/layer23_attention_weight_min": -20.453380584716797,
      "activations/layer2_attention_weight_max": 33.092350006103516,
      "activations/layer2_attention_weight_min": -31.204694747924805,
      "activations/layer3_attention_weight_max": 93.63168334960938,
      "activations/layer3_attention_weight_min": -94.86109161376953,
      "activations/layer4_attention_weight_max": 109.77351379394531,
      "activations/layer4_attention_weight_min": -103.26973724365234,
      "activations/layer5_attention_weight_max": 87.74104309082031,
      "activations/layer5_attention_weight_min": -84.21055603027344,
      "activations/layer6_attention_weight_max": 54.82602310180664,
      "activations/layer6_attention_weight_min": -52.53559875488281,
      "activations/layer7_attention_weight_max": 94.25493621826172,
      "activations/layer7_attention_weight_min": -77.76519012451172,
      "activations/layer8_attention_weight_max": 53.107666015625,
      "activations/layer8_attention_weight_min": -46.17238235473633,
      "activations/layer9_attention_weight_max": 73.1454086303711,
      "activations/layer9_attention_weight_min": -53.58530807495117,
      "epoch": 8.33,
      "learning_rate": 9.726477272727272e-05,
      "loss": 2.7898,
      "step": 143350
    },
    {
      "activations/layer0_attention_weight_max": 14.601099967956543,
      "activations/layer0_attention_weight_min": -14.69129467010498,
      "activations/layer10_attention_weight_max": 39.076454162597656,
      "activations/layer10_attention_weight_min": -37.82931900024414,
      "activations/layer11_attention_weight_max": 32.82642364501953,
      "activations/layer11_attention_weight_min": -32.62386703491211,
      "activations/layer12_attention_weight_max": 21.100662231445312,
      "activations/layer12_attention_weight_min": -30.75133514404297,
      "activations/layer13_attention_weight_max": 52.486881256103516,
      "activations/layer13_attention_weight_min": -35.762332916259766,
      "activations/layer14_attention_weight_max": 52.311771392822266,
      "activations/layer14_attention_weight_min": -35.27297592163086,
      "activations/layer15_attention_weight_max": 40.495662689208984,
      "activations/layer15_attention_weight_min": -31.511390686035156,
      "activations/layer16_attention_weight_max": 35.56303405761719,
      "activations/layer16_attention_weight_min": -30.913850784301758,
      "activations/layer17_attention_weight_max": 60.9923095703125,
      "activations/layer17_attention_weight_min": -48.26251983642578,
      "activations/layer18_attention_weight_max": 50.39826583862305,
      "activations/layer18_attention_weight_min": -40.0576057434082,
      "activations/layer19_attention_weight_max": 24.0752010345459,
      "activations/layer19_attention_weight_min": -22.88140106201172,
      "activations/layer1_attention_weight_max": 18.97201919555664,
      "activations/layer1_attention_weight_min": -17.274486541748047,
      "activations/layer20_attention_weight_max": 21.655115127563477,
      "activations/layer20_attention_weight_min": -23.908977508544922,
      "activations/layer21_attention_weight_max": 42.75782012939453,
      "activations/layer21_attention_weight_min": -26.48341941833496,
      "activations/layer22_attention_weight_max": 35.209434509277344,
      "activations/layer22_attention_weight_min": -28.336627960205078,
      "activations/layer23_attention_weight_max": 36.431541442871094,
      "activations/layer23_attention_weight_min": -23.319541931152344,
      "activations/layer2_attention_weight_max": 32.75066375732422,
      "activations/layer2_attention_weight_min": -29.668195724487305,
      "activations/layer3_attention_weight_max": 90.44171142578125,
      "activations/layer3_attention_weight_min": -93.16651153564453,
      "activations/layer4_attention_weight_max": 106.75374603271484,
      "activations/layer4_attention_weight_min": -99.45806884765625,
      "activations/layer5_attention_weight_max": 86.84869384765625,
      "activations/layer5_attention_weight_min": -81.98507690429688,
      "activations/layer6_attention_weight_max": 55.92401123046875,
      "activations/layer6_attention_weight_min": -57.018890380859375,
      "activations/layer7_attention_weight_max": 80.61359405517578,
      "activations/layer7_attention_weight_min": -76.63307189941406,
      "activations/layer8_attention_weight_max": 50.05382537841797,
      "activations/layer8_attention_weight_min": -51.420352935791016,
      "activations/layer9_attention_weight_max": 56.36670684814453,
      "activations/layer9_attention_weight_min": -50.6639518737793,
      "epoch": 8.33,
      "learning_rate": 9.724583333333332e-05,
      "loss": 2.7878,
      "step": 143400
    },
    {
      "activations/layer0_attention_weight_max": 16.058042526245117,
      "activations/layer0_attention_weight_min": -14.682019233703613,
      "activations/layer10_attention_weight_max": 39.7078742980957,
      "activations/layer10_attention_weight_min": -38.228515625,
      "activations/layer11_attention_weight_max": 34.216453552246094,
      "activations/layer11_attention_weight_min": -32.00971603393555,
      "activations/layer12_attention_weight_max": 18.59995460510254,
      "activations/layer12_attention_weight_min": -28.772708892822266,
      "activations/layer13_attention_weight_max": 40.39276123046875,
      "activations/layer13_attention_weight_min": -33.010353088378906,
      "activations/layer14_attention_weight_max": 42.02766418457031,
      "activations/layer14_attention_weight_min": -31.342374801635742,
      "activations/layer15_attention_weight_max": 32.7360954284668,
      "activations/layer15_attention_weight_min": -29.78803253173828,
      "activations/layer16_attention_weight_max": 32.67422103881836,
      "activations/layer16_attention_weight_min": -26.919878005981445,
      "activations/layer17_attention_weight_max": 50.2745361328125,
      "activations/layer17_attention_weight_min": -40.62562561035156,
      "activations/layer18_attention_weight_max": 44.78498458862305,
      "activations/layer18_attention_weight_min": -34.28589630126953,
      "activations/layer19_attention_weight_max": 22.433835983276367,
      "activations/layer19_attention_weight_min": -21.48422622680664,
      "activations/layer1_attention_weight_max": 16.98708724975586,
      "activations/layer1_attention_weight_min": -16.501272201538086,
      "activations/layer20_attention_weight_max": 21.145549774169922,
      "activations/layer20_attention_weight_min": -25.208372116088867,
      "activations/layer21_attention_weight_max": 43.83141326904297,
      "activations/layer21_attention_weight_min": -22.607065200805664,
      "activations/layer22_attention_weight_max": 32.42240524291992,
      "activations/layer22_attention_weight_min": -23.17511558532715,
      "activations/layer23_attention_weight_max": 35.13633728027344,
      "activations/layer23_attention_weight_min": -22.669906616210938,
      "activations/layer2_attention_weight_max": 34.72795867919922,
      "activations/layer2_attention_weight_min": -32.16771697998047,
      "activations/layer3_attention_weight_max": 94.98442077636719,
      "activations/layer3_attention_weight_min": -91.8328857421875,
      "activations/layer4_attention_weight_max": 107.50740051269531,
      "activations/layer4_attention_weight_min": -100.57913970947266,
      "activations/layer5_attention_weight_max": 84.68016052246094,
      "activations/layer5_attention_weight_min": -81.39324188232422,
      "activations/layer6_attention_weight_max": 58.77139663696289,
      "activations/layer6_attention_weight_min": -56.63864517211914,
      "activations/layer7_attention_weight_max": 82.63277435302734,
      "activations/layer7_attention_weight_min": -76.21113586425781,
      "activations/layer8_attention_weight_max": 48.729209899902344,
      "activations/layer8_attention_weight_min": -49.65338897705078,
      "activations/layer9_attention_weight_max": 52.24567413330078,
      "activations/layer9_attention_weight_min": -51.4334716796875,
      "epoch": 8.34,
      "learning_rate": 9.722689393939392e-05,
      "loss": 2.8127,
      "step": 143450
    },
    {
      "activations/layer0_attention_weight_max": 14.896499633789062,
      "activations/layer0_attention_weight_min": -14.621514320373535,
      "activations/layer10_attention_weight_max": 40.469207763671875,
      "activations/layer10_attention_weight_min": -38.223907470703125,
      "activations/layer11_attention_weight_max": 36.383811950683594,
      "activations/layer11_attention_weight_min": -32.3393669128418,
      "activations/layer12_attention_weight_max": 19.485660552978516,
      "activations/layer12_attention_weight_min": -24.87217903137207,
      "activations/layer13_attention_weight_max": 47.04421615600586,
      "activations/layer13_attention_weight_min": -32.681129455566406,
      "activations/layer14_attention_weight_max": 47.21095657348633,
      "activations/layer14_attention_weight_min": -31.876218795776367,
      "activations/layer15_attention_weight_max": 40.60073471069336,
      "activations/layer15_attention_weight_min": -29.795835494995117,
      "activations/layer16_attention_weight_max": 34.39752197265625,
      "activations/layer16_attention_weight_min": -27.507568359375,
      "activations/layer17_attention_weight_max": 55.648685455322266,
      "activations/layer17_attention_weight_min": -44.88523483276367,
      "activations/layer18_attention_weight_max": 50.94457244873047,
      "activations/layer18_attention_weight_min": -38.5283203125,
      "activations/layer19_attention_weight_max": 23.967592239379883,
      "activations/layer19_attention_weight_min": -23.094223022460938,
      "activations/layer1_attention_weight_max": 17.04655647277832,
      "activations/layer1_attention_weight_min": -16.212430953979492,
      "activations/layer20_attention_weight_max": 25.80475425720215,
      "activations/layer20_attention_weight_min": -21.114131927490234,
      "activations/layer21_attention_weight_max": 45.10707473754883,
      "activations/layer21_attention_weight_min": -25.350027084350586,
      "activations/layer22_attention_weight_max": 32.74641036987305,
      "activations/layer22_attention_weight_min": -26.157442092895508,
      "activations/layer23_attention_weight_max": 38.12273025512695,
      "activations/layer23_attention_weight_min": -24.238859176635742,
      "activations/layer2_attention_weight_max": 33.350704193115234,
      "activations/layer2_attention_weight_min": -31.19428062438965,
      "activations/layer3_attention_weight_max": 98.25666046142578,
      "activations/layer3_attention_weight_min": -97.37660217285156,
      "activations/layer4_attention_weight_max": 108.3036117553711,
      "activations/layer4_attention_weight_min": -99.6775131225586,
      "activations/layer5_attention_weight_max": 86.85442352294922,
      "activations/layer5_attention_weight_min": -81.83238983154297,
      "activations/layer6_attention_weight_max": 57.2350959777832,
      "activations/layer6_attention_weight_min": -54.02629852294922,
      "activations/layer7_attention_weight_max": 81.3641128540039,
      "activations/layer7_attention_weight_min": -76.06733703613281,
      "activations/layer8_attention_weight_max": 49.44647216796875,
      "activations/layer8_attention_weight_min": -46.914085388183594,
      "activations/layer9_attention_weight_max": 63.14426803588867,
      "activations/layer9_attention_weight_min": -53.66750717163086,
      "epoch": 8.34,
      "learning_rate": 9.720795454545454e-05,
      "loss": 2.7794,
      "step": 143500
    },
    {
      "activations/layer0_attention_weight_max": 14.346094131469727,
      "activations/layer0_attention_weight_min": -14.55115795135498,
      "activations/layer10_attention_weight_max": 40.61929702758789,
      "activations/layer10_attention_weight_min": -39.48200988769531,
      "activations/layer11_attention_weight_max": 34.732452392578125,
      "activations/layer11_attention_weight_min": -34.469482421875,
      "activations/layer12_attention_weight_max": 17.681852340698242,
      "activations/layer12_attention_weight_min": -29.78672981262207,
      "activations/layer13_attention_weight_max": 48.1887092590332,
      "activations/layer13_attention_weight_min": -32.95797348022461,
      "activations/layer14_attention_weight_max": 61.681400299072266,
      "activations/layer14_attention_weight_min": -37.94837188720703,
      "activations/layer15_attention_weight_max": 46.00401306152344,
      "activations/layer15_attention_weight_min": -30.893423080444336,
      "activations/layer16_attention_weight_max": 32.85892105102539,
      "activations/layer16_attention_weight_min": -30.550235748291016,
      "activations/layer17_attention_weight_max": 59.20187759399414,
      "activations/layer17_attention_weight_min": -50.567115783691406,
      "activations/layer18_attention_weight_max": 49.9968376159668,
      "activations/layer18_attention_weight_min": -40.49652862548828,
      "activations/layer19_attention_weight_max": 23.357969284057617,
      "activations/layer19_attention_weight_min": -23.12273406982422,
      "activations/layer1_attention_weight_max": 17.43927001953125,
      "activations/layer1_attention_weight_min": -15.650331497192383,
      "activations/layer20_attention_weight_max": 25.696184158325195,
      "activations/layer20_attention_weight_min": -22.295909881591797,
      "activations/layer21_attention_weight_max": 46.2076530456543,
      "activations/layer21_attention_weight_min": -25.020397186279297,
      "activations/layer22_attention_weight_max": 31.562259674072266,
      "activations/layer22_attention_weight_min": -25.4383602142334,
      "activations/layer23_attention_weight_max": 34.16456604003906,
      "activations/layer23_attention_weight_min": -21.542720794677734,
      "activations/layer2_attention_weight_max": 32.59178161621094,
      "activations/layer2_attention_weight_min": -30.87194061279297,
      "activations/layer3_attention_weight_max": 92.415771484375,
      "activations/layer3_attention_weight_min": -93.36355590820312,
      "activations/layer4_attention_weight_max": 106.46244049072266,
      "activations/layer4_attention_weight_min": -102.74855041503906,
      "activations/layer5_attention_weight_max": 82.10299682617188,
      "activations/layer5_attention_weight_min": -83.27448272705078,
      "activations/layer6_attention_weight_max": 57.36137008666992,
      "activations/layer6_attention_weight_min": -57.798709869384766,
      "activations/layer7_attention_weight_max": 83.19944763183594,
      "activations/layer7_attention_weight_min": -81.1051025390625,
      "activations/layer8_attention_weight_max": 51.246299743652344,
      "activations/layer8_attention_weight_min": -54.85752487182617,
      "activations/layer9_attention_weight_max": 62.59162521362305,
      "activations/layer9_attention_weight_min": -58.1100959777832,
      "epoch": 8.34,
      "learning_rate": 9.718901515151515e-05,
      "loss": 2.7875,
      "step": 143550
    },
    {
      "activations/layer0_attention_weight_max": 14.757134437561035,
      "activations/layer0_attention_weight_min": -14.504329681396484,
      "activations/layer10_attention_weight_max": 42.21380615234375,
      "activations/layer10_attention_weight_min": -40.53308868408203,
      "activations/layer11_attention_weight_max": 34.8005485534668,
      "activations/layer11_attention_weight_min": -34.794776916503906,
      "activations/layer12_attention_weight_max": 24.271806716918945,
      "activations/layer12_attention_weight_min": -25.11560821533203,
      "activations/layer13_attention_weight_max": 52.492671966552734,
      "activations/layer13_attention_weight_min": -37.84712219238281,
      "activations/layer14_attention_weight_max": 44.05948257446289,
      "activations/layer14_attention_weight_min": -31.90713882446289,
      "activations/layer15_attention_weight_max": 39.36391830444336,
      "activations/layer15_attention_weight_min": -31.702648162841797,
      "activations/layer16_attention_weight_max": 38.41409683227539,
      "activations/layer16_attention_weight_min": -27.223493576049805,
      "activations/layer17_attention_weight_max": 59.5463752746582,
      "activations/layer17_attention_weight_min": -41.90748977661133,
      "activations/layer18_attention_weight_max": 51.63932800292969,
      "activations/layer18_attention_weight_min": -35.979549407958984,
      "activations/layer19_attention_weight_max": 25.848892211914062,
      "activations/layer19_attention_weight_min": -22.48895835876465,
      "activations/layer1_attention_weight_max": 16.41315269470215,
      "activations/layer1_attention_weight_min": -14.335172653198242,
      "activations/layer20_attention_weight_max": 24.38921356201172,
      "activations/layer20_attention_weight_min": -22.726993560791016,
      "activations/layer21_attention_weight_max": 43.856300354003906,
      "activations/layer21_attention_weight_min": -24.25107765197754,
      "activations/layer22_attention_weight_max": 33.37952423095703,
      "activations/layer22_attention_weight_min": -23.0495548248291,
      "activations/layer23_attention_weight_max": 38.039276123046875,
      "activations/layer23_attention_weight_min": -21.876256942749023,
      "activations/layer2_attention_weight_max": 31.735607147216797,
      "activations/layer2_attention_weight_min": -31.250829696655273,
      "activations/layer3_attention_weight_max": 90.0006332397461,
      "activations/layer3_attention_weight_min": -91.02912902832031,
      "activations/layer4_attention_weight_max": 106.72692108154297,
      "activations/layer4_attention_weight_min": -105.52983856201172,
      "activations/layer5_attention_weight_max": 93.52647399902344,
      "activations/layer5_attention_weight_min": -85.97753143310547,
      "activations/layer6_attention_weight_max": 57.863346099853516,
      "activations/layer6_attention_weight_min": -56.662357330322266,
      "activations/layer7_attention_weight_max": 86.32088470458984,
      "activations/layer7_attention_weight_min": -76.95845031738281,
      "activations/layer8_attention_weight_max": 51.93989181518555,
      "activations/layer8_attention_weight_min": -49.751155853271484,
      "activations/layer9_attention_weight_max": 59.698036193847656,
      "activations/layer9_attention_weight_min": -51.601654052734375,
      "epoch": 8.34,
      "learning_rate": 9.717045454545455e-05,
      "loss": 2.7935,
      "step": 143600
    },
    {
      "activations/layer0_attention_weight_max": 15.245644569396973,
      "activations/layer0_attention_weight_min": -14.413032531738281,
      "activations/layer10_attention_weight_max": 36.031246185302734,
      "activations/layer10_attention_weight_min": -34.8709831237793,
      "activations/layer11_attention_weight_max": 31.20789337158203,
      "activations/layer11_attention_weight_min": -32.24488067626953,
      "activations/layer12_attention_weight_max": 19.5684814453125,
      "activations/layer12_attention_weight_min": -26.890409469604492,
      "activations/layer13_attention_weight_max": 42.600555419921875,
      "activations/layer13_attention_weight_min": -32.41240692138672,
      "activations/layer14_attention_weight_max": 38.879642486572266,
      "activations/layer14_attention_weight_min": -31.27011489868164,
      "activations/layer15_attention_weight_max": 35.96057891845703,
      "activations/layer15_attention_weight_min": -29.61807632446289,
      "activations/layer16_attention_weight_max": 30.632156372070312,
      "activations/layer16_attention_weight_min": -27.99413299560547,
      "activations/layer17_attention_weight_max": 56.85323715209961,
      "activations/layer17_attention_weight_min": -43.920597076416016,
      "activations/layer18_attention_weight_max": 47.31407165527344,
      "activations/layer18_attention_weight_min": -35.972206115722656,
      "activations/layer19_attention_weight_max": 22.765111923217773,
      "activations/layer19_attention_weight_min": -19.742183685302734,
      "activations/layer1_attention_weight_max": 16.018308639526367,
      "activations/layer1_attention_weight_min": -14.833724975585938,
      "activations/layer20_attention_weight_max": 22.795021057128906,
      "activations/layer20_attention_weight_min": -19.98200225830078,
      "activations/layer21_attention_weight_max": 38.102752685546875,
      "activations/layer21_attention_weight_min": -23.55923843383789,
      "activations/layer22_attention_weight_max": 31.221223831176758,
      "activations/layer22_attention_weight_min": -24.541791915893555,
      "activations/layer23_attention_weight_max": 35.291015625,
      "activations/layer23_attention_weight_min": -20.52759552001953,
      "activations/layer2_attention_weight_max": 33.169883728027344,
      "activations/layer2_attention_weight_min": -29.219924926757812,
      "activations/layer3_attention_weight_max": 90.14960479736328,
      "activations/layer3_attention_weight_min": -93.2159423828125,
      "activations/layer4_attention_weight_max": 102.34944915771484,
      "activations/layer4_attention_weight_min": -99.76361846923828,
      "activations/layer5_attention_weight_max": 83.31027221679688,
      "activations/layer5_attention_weight_min": -79.90508270263672,
      "activations/layer6_attention_weight_max": 55.99873352050781,
      "activations/layer6_attention_weight_min": -51.95322036743164,
      "activations/layer7_attention_weight_max": 83.02705383300781,
      "activations/layer7_attention_weight_min": -70.96626281738281,
      "activations/layer8_attention_weight_max": 46.447269439697266,
      "activations/layer8_attention_weight_min": -45.93242263793945,
      "activations/layer9_attention_weight_max": 55.68740463256836,
      "activations/layer9_attention_weight_min": -49.48448944091797,
      "epoch": 8.35,
      "learning_rate": 9.715151515151513e-05,
      "loss": 2.7888,
      "step": 143650
    },
    {
      "activations/layer0_attention_weight_max": 14.268950462341309,
      "activations/layer0_attention_weight_min": -14.116536140441895,
      "activations/layer10_attention_weight_max": 35.90513229370117,
      "activations/layer10_attention_weight_min": -35.049560546875,
      "activations/layer11_attention_weight_max": 30.246707916259766,
      "activations/layer11_attention_weight_min": -31.399517059326172,
      "activations/layer12_attention_weight_max": 19.401775360107422,
      "activations/layer12_attention_weight_min": -28.873212814331055,
      "activations/layer13_attention_weight_max": 41.173301696777344,
      "activations/layer13_attention_weight_min": -36.14535903930664,
      "activations/layer14_attention_weight_max": 36.00365447998047,
      "activations/layer14_attention_weight_min": -29.280534744262695,
      "activations/layer15_attention_weight_max": 33.38957214355469,
      "activations/layer15_attention_weight_min": -28.80117416381836,
      "activations/layer16_attention_weight_max": 30.964323043823242,
      "activations/layer16_attention_weight_min": -27.090831756591797,
      "activations/layer17_attention_weight_max": 50.93523406982422,
      "activations/layer17_attention_weight_min": -40.66056823730469,
      "activations/layer18_attention_weight_max": 46.23231506347656,
      "activations/layer18_attention_weight_min": -32.93558883666992,
      "activations/layer19_attention_weight_max": 22.835988998413086,
      "activations/layer19_attention_weight_min": -18.361677169799805,
      "activations/layer1_attention_weight_max": 17.352745056152344,
      "activations/layer1_attention_weight_min": -15.253217697143555,
      "activations/layer20_attention_weight_max": 19.180259704589844,
      "activations/layer20_attention_weight_min": -20.936344146728516,
      "activations/layer21_attention_weight_max": 34.11923599243164,
      "activations/layer21_attention_weight_min": -23.07831382751465,
      "activations/layer22_attention_weight_max": 29.18059539794922,
      "activations/layer22_attention_weight_min": -22.660079956054688,
      "activations/layer23_attention_weight_max": 32.49959182739258,
      "activations/layer23_attention_weight_min": -23.506811141967773,
      "activations/layer2_attention_weight_max": 32.2817268371582,
      "activations/layer2_attention_weight_min": -29.706558227539062,
      "activations/layer3_attention_weight_max": 89.704833984375,
      "activations/layer3_attention_weight_min": -91.42875671386719,
      "activations/layer4_attention_weight_max": 104.92203521728516,
      "activations/layer4_attention_weight_min": -98.30699157714844,
      "activations/layer5_attention_weight_max": 82.35491943359375,
      "activations/layer5_attention_weight_min": -79.4218978881836,
      "activations/layer6_attention_weight_max": 56.3420524597168,
      "activations/layer6_attention_weight_min": -53.95566177368164,
      "activations/layer7_attention_weight_max": 80.31755065917969,
      "activations/layer7_attention_weight_min": -75.36331939697266,
      "activations/layer8_attention_weight_max": 43.9183464050293,
      "activations/layer8_attention_weight_min": -46.12420654296875,
      "activations/layer9_attention_weight_max": 51.24201202392578,
      "activations/layer9_attention_weight_min": -47.78790283203125,
      "epoch": 8.35,
      "learning_rate": 9.713257575757575e-05,
      "loss": 2.7889,
      "step": 143700
    },
    {
      "activations/layer0_attention_weight_max": 14.433358192443848,
      "activations/layer0_attention_weight_min": -14.564371109008789,
      "activations/layer10_attention_weight_max": 37.920021057128906,
      "activations/layer10_attention_weight_min": -35.3807487487793,
      "activations/layer11_attention_weight_max": 31.66585922241211,
      "activations/layer11_attention_weight_min": -31.679176330566406,
      "activations/layer12_attention_weight_max": 21.615575790405273,
      "activations/layer12_attention_weight_min": -25.02601432800293,
      "activations/layer13_attention_weight_max": 45.68180465698242,
      "activations/layer13_attention_weight_min": -33.48145294189453,
      "activations/layer14_attention_weight_max": 45.54441833496094,
      "activations/layer14_attention_weight_min": -33.21632766723633,
      "activations/layer15_attention_weight_max": 38.39641571044922,
      "activations/layer15_attention_weight_min": -29.489608764648438,
      "activations/layer16_attention_weight_max": 30.27234649658203,
      "activations/layer16_attention_weight_min": -27.442590713500977,
      "activations/layer17_attention_weight_max": 53.40276336669922,
      "activations/layer17_attention_weight_min": -41.39381408691406,
      "activations/layer18_attention_weight_max": 46.39218521118164,
      "activations/layer18_attention_weight_min": -36.5137825012207,
      "activations/layer19_attention_weight_max": 23.193754196166992,
      "activations/layer19_attention_weight_min": -20.140850067138672,
      "activations/layer1_attention_weight_max": 16.15687370300293,
      "activations/layer1_attention_weight_min": -14.925015449523926,
      "activations/layer20_attention_weight_max": 21.33942985534668,
      "activations/layer20_attention_weight_min": -22.441287994384766,
      "activations/layer21_attention_weight_max": 42.4078369140625,
      "activations/layer21_attention_weight_min": -24.847375869750977,
      "activations/layer22_attention_weight_max": 34.098209381103516,
      "activations/layer22_attention_weight_min": -25.569660186767578,
      "activations/layer23_attention_weight_max": 37.006072998046875,
      "activations/layer23_attention_weight_min": -22.85340690612793,
      "activations/layer2_attention_weight_max": 33.386817932128906,
      "activations/layer2_attention_weight_min": -32.000667572021484,
      "activations/layer3_attention_weight_max": 91.1841049194336,
      "activations/layer3_attention_weight_min": -92.51683807373047,
      "activations/layer4_attention_weight_max": 105.55274963378906,
      "activations/layer4_attention_weight_min": -100.83462524414062,
      "activations/layer5_attention_weight_max": 83.32280731201172,
      "activations/layer5_attention_weight_min": -77.79906463623047,
      "activations/layer6_attention_weight_max": 55.89969253540039,
      "activations/layer6_attention_weight_min": -54.13633728027344,
      "activations/layer7_attention_weight_max": 75.25216674804688,
      "activations/layer7_attention_weight_min": -76.82257080078125,
      "activations/layer8_attention_weight_max": 48.3826789855957,
      "activations/layer8_attention_weight_min": -46.85690689086914,
      "activations/layer9_attention_weight_max": 51.318824768066406,
      "activations/layer9_attention_weight_min": -52.17535400390625,
      "epoch": 8.35,
      "learning_rate": 9.711363636363636e-05,
      "loss": 2.7977,
      "step": 143750
    },
    {
      "activations/layer0_attention_weight_max": 13.883857727050781,
      "activations/layer0_attention_weight_min": -14.29779052734375,
      "activations/layer10_attention_weight_max": 43.23967361450195,
      "activations/layer10_attention_weight_min": -39.5871467590332,
      "activations/layer11_attention_weight_max": 37.572322845458984,
      "activations/layer11_attention_weight_min": -33.033958435058594,
      "activations/layer12_attention_weight_max": 22.67169189453125,
      "activations/layer12_attention_weight_min": -28.278976440429688,
      "activations/layer13_attention_weight_max": 46.42093276977539,
      "activations/layer13_attention_weight_min": -34.8575439453125,
      "activations/layer14_attention_weight_max": 46.956382751464844,
      "activations/layer14_attention_weight_min": -31.993118286132812,
      "activations/layer15_attention_weight_max": 39.45127868652344,
      "activations/layer15_attention_weight_min": -29.866844177246094,
      "activations/layer16_attention_weight_max": 34.741432189941406,
      "activations/layer16_attention_weight_min": -28.671247482299805,
      "activations/layer17_attention_weight_max": 55.02724075317383,
      "activations/layer17_attention_weight_min": -47.2327995300293,
      "activations/layer18_attention_weight_max": 56.06550598144531,
      "activations/layer18_attention_weight_min": -40.457176208496094,
      "activations/layer19_attention_weight_max": 21.63810920715332,
      "activations/layer19_attention_weight_min": -21.48727035522461,
      "activations/layer1_attention_weight_max": 17.693452835083008,
      "activations/layer1_attention_weight_min": -14.868288040161133,
      "activations/layer20_attention_weight_max": 22.907337188720703,
      "activations/layer20_attention_weight_min": -22.411882400512695,
      "activations/layer21_attention_weight_max": 48.409847259521484,
      "activations/layer21_attention_weight_min": -25.211315155029297,
      "activations/layer22_attention_weight_max": 31.498912811279297,
      "activations/layer22_attention_weight_min": -24.75858497619629,
      "activations/layer23_attention_weight_max": 36.10557556152344,
      "activations/layer23_attention_weight_min": -21.39679718017578,
      "activations/layer2_attention_weight_max": 33.59192657470703,
      "activations/layer2_attention_weight_min": -30.589275360107422,
      "activations/layer3_attention_weight_max": 90.28395080566406,
      "activations/layer3_attention_weight_min": -95.82386016845703,
      "activations/layer4_attention_weight_max": 100.92601776123047,
      "activations/layer4_attention_weight_min": -98.96439361572266,
      "activations/layer5_attention_weight_max": 80.26788330078125,
      "activations/layer5_attention_weight_min": -77.90669250488281,
      "activations/layer6_attention_weight_max": 54.91050720214844,
      "activations/layer6_attention_weight_min": -53.59310531616211,
      "activations/layer7_attention_weight_max": 90.51109313964844,
      "activations/layer7_attention_weight_min": -76.9095230102539,
      "activations/layer8_attention_weight_max": 50.15104675292969,
      "activations/layer8_attention_weight_min": -45.47673797607422,
      "activations/layer9_attention_weight_max": 65.6017837524414,
      "activations/layer9_attention_weight_min": -53.10390090942383,
      "epoch": 8.36,
      "learning_rate": 9.709469696969695e-05,
      "loss": 2.797,
      "step": 143800
    },
    {
      "activations/layer0_attention_weight_max": 13.859139442443848,
      "activations/layer0_attention_weight_min": -14.61230754852295,
      "activations/layer10_attention_weight_max": 41.88264846801758,
      "activations/layer10_attention_weight_min": -36.64842987060547,
      "activations/layer11_attention_weight_max": 30.981603622436523,
      "activations/layer11_attention_weight_min": -31.980716705322266,
      "activations/layer12_attention_weight_max": 23.014585494995117,
      "activations/layer12_attention_weight_min": -24.775726318359375,
      "activations/layer13_attention_weight_max": 44.10749816894531,
      "activations/layer13_attention_weight_min": -30.353782653808594,
      "activations/layer14_attention_weight_max": 46.95882034301758,
      "activations/layer14_attention_weight_min": -33.595970153808594,
      "activations/layer15_attention_weight_max": 37.1605110168457,
      "activations/layer15_attention_weight_min": -31.015588760375977,
      "activations/layer16_attention_weight_max": 36.790164947509766,
      "activations/layer16_attention_weight_min": -28.669450759887695,
      "activations/layer17_attention_weight_max": 54.16584777832031,
      "activations/layer17_attention_weight_min": -42.66889190673828,
      "activations/layer18_attention_weight_max": 49.306434631347656,
      "activations/layer18_attention_weight_min": -37.3824577331543,
      "activations/layer19_attention_weight_max": 21.011404037475586,
      "activations/layer19_attention_weight_min": -22.304180145263672,
      "activations/layer1_attention_weight_max": 15.761838912963867,
      "activations/layer1_attention_weight_min": -16.020641326904297,
      "activations/layer20_attention_weight_max": 22.508275985717773,
      "activations/layer20_attention_weight_min": -27.7884464263916,
      "activations/layer21_attention_weight_max": 43.966190338134766,
      "activations/layer21_attention_weight_min": -26.19374656677246,
      "activations/layer22_attention_weight_max": 34.37607192993164,
      "activations/layer22_attention_weight_min": -24.916223526000977,
      "activations/layer23_attention_weight_max": 39.39930725097656,
      "activations/layer23_attention_weight_min": -22.338489532470703,
      "activations/layer2_attention_weight_max": 32.171363830566406,
      "activations/layer2_attention_weight_min": -31.57811737060547,
      "activations/layer3_attention_weight_max": 92.45325469970703,
      "activations/layer3_attention_weight_min": -92.68441772460938,
      "activations/layer4_attention_weight_max": 104.7873306274414,
      "activations/layer4_attention_weight_min": -99.62451171875,
      "activations/layer5_attention_weight_max": 88.363037109375,
      "activations/layer5_attention_weight_min": -81.22665405273438,
      "activations/layer6_attention_weight_max": 55.93680191040039,
      "activations/layer6_attention_weight_min": -55.67034149169922,
      "activations/layer7_attention_weight_max": 80.92633056640625,
      "activations/layer7_attention_weight_min": -77.2802963256836,
      "activations/layer8_attention_weight_max": 47.958335876464844,
      "activations/layer8_attention_weight_min": -51.38932800292969,
      "activations/layer9_attention_weight_max": 52.70484161376953,
      "activations/layer9_attention_weight_min": -50.69930648803711,
      "epoch": 8.36,
      "learning_rate": 9.707575757575757e-05,
      "loss": 2.7781,
      "step": 143850
    },
    {
      "activations/layer0_attention_weight_max": 15.522229194641113,
      "activations/layer0_attention_weight_min": -14.48211669921875,
      "activations/layer10_attention_weight_max": 41.51913070678711,
      "activations/layer10_attention_weight_min": -39.718544006347656,
      "activations/layer11_attention_weight_max": 34.451515197753906,
      "activations/layer11_attention_weight_min": -31.740184783935547,
      "activations/layer12_attention_weight_max": 23.050800323486328,
      "activations/layer12_attention_weight_min": -28.28952980041504,
      "activations/layer13_attention_weight_max": 63.838706970214844,
      "activations/layer13_attention_weight_min": -37.64840316772461,
      "activations/layer14_attention_weight_max": 38.72526931762695,
      "activations/layer14_attention_weight_min": -31.195178985595703,
      "activations/layer15_attention_weight_max": 38.21890640258789,
      "activations/layer15_attention_weight_min": -29.343053817749023,
      "activations/layer16_attention_weight_max": 33.354347229003906,
      "activations/layer16_attention_weight_min": -28.206283569335938,
      "activations/layer17_attention_weight_max": 57.38637161254883,
      "activations/layer17_attention_weight_min": -46.38298416137695,
      "activations/layer18_attention_weight_max": 45.265785217285156,
      "activations/layer18_attention_weight_min": -37.14924240112305,
      "activations/layer19_attention_weight_max": 26.024885177612305,
      "activations/layer19_attention_weight_min": -20.84965705871582,
      "activations/layer1_attention_weight_max": 16.081403732299805,
      "activations/layer1_attention_weight_min": -14.319037437438965,
      "activations/layer20_attention_weight_max": 22.23900604248047,
      "activations/layer20_attention_weight_min": -22.047388076782227,
      "activations/layer21_attention_weight_max": 43.217041015625,
      "activations/layer21_attention_weight_min": -26.588237762451172,
      "activations/layer22_attention_weight_max": 33.63776397705078,
      "activations/layer22_attention_weight_min": -24.95355224609375,
      "activations/layer23_attention_weight_max": 35.579322814941406,
      "activations/layer23_attention_weight_min": -22.386625289916992,
      "activations/layer2_attention_weight_max": 32.13658905029297,
      "activations/layer2_attention_weight_min": -30.611988067626953,
      "activations/layer3_attention_weight_max": 93.0998764038086,
      "activations/layer3_attention_weight_min": -87.9956283569336,
      "activations/layer4_attention_weight_max": 103.85897064208984,
      "activations/layer4_attention_weight_min": -96.62510681152344,
      "activations/layer5_attention_weight_max": 86.87821960449219,
      "activations/layer5_attention_weight_min": -79.31822967529297,
      "activations/layer6_attention_weight_max": 58.871978759765625,
      "activations/layer6_attention_weight_min": -54.931182861328125,
      "activations/layer7_attention_weight_max": 80.64620971679688,
      "activations/layer7_attention_weight_min": -78.10474395751953,
      "activations/layer8_attention_weight_max": 49.54912567138672,
      "activations/layer8_attention_weight_min": -48.708526611328125,
      "activations/layer9_attention_weight_max": 57.34580612182617,
      "activations/layer9_attention_weight_min": -54.875545501708984,
      "epoch": 8.36,
      "learning_rate": 9.705681818181818e-05,
      "loss": 2.8055,
      "step": 143900
    },
    {
      "activations/layer0_attention_weight_max": 14.788323402404785,
      "activations/layer0_attention_weight_min": -14.613946914672852,
      "activations/layer10_attention_weight_max": 38.64295959472656,
      "activations/layer10_attention_weight_min": -36.91438674926758,
      "activations/layer11_attention_weight_max": 32.44176483154297,
      "activations/layer11_attention_weight_min": -32.351505279541016,
      "activations/layer12_attention_weight_max": 22.79983139038086,
      "activations/layer12_attention_weight_min": -27.45388412475586,
      "activations/layer13_attention_weight_max": 47.75706481933594,
      "activations/layer13_attention_weight_min": -36.860687255859375,
      "activations/layer14_attention_weight_max": 40.61359786987305,
      "activations/layer14_attention_weight_min": -31.946556091308594,
      "activations/layer15_attention_weight_max": 34.72105026245117,
      "activations/layer15_attention_weight_min": -32.19245910644531,
      "activations/layer16_attention_weight_max": 32.27940368652344,
      "activations/layer16_attention_weight_min": -28.543167114257812,
      "activations/layer17_attention_weight_max": 55.16514205932617,
      "activations/layer17_attention_weight_min": -46.4518928527832,
      "activations/layer18_attention_weight_max": 48.595726013183594,
      "activations/layer18_attention_weight_min": -38.29753875732422,
      "activations/layer19_attention_weight_max": 22.50225067138672,
      "activations/layer19_attention_weight_min": -19.725446701049805,
      "activations/layer1_attention_weight_max": 18.064746856689453,
      "activations/layer1_attention_weight_min": -15.176473617553711,
      "activations/layer20_attention_weight_max": 23.858945846557617,
      "activations/layer20_attention_weight_min": -20.96200942993164,
      "activations/layer21_attention_weight_max": 36.04579544067383,
      "activations/layer21_attention_weight_min": -22.11818504333496,
      "activations/layer22_attention_weight_max": 30.83061408996582,
      "activations/layer22_attention_weight_min": -24.428796768188477,
      "activations/layer23_attention_weight_max": 35.507652282714844,
      "activations/layer23_attention_weight_min": -22.789236068725586,
      "activations/layer2_attention_weight_max": 32.03092956542969,
      "activations/layer2_attention_weight_min": -30.280927658081055,
      "activations/layer3_attention_weight_max": 89.63023376464844,
      "activations/layer3_attention_weight_min": -90.0830078125,
      "activations/layer4_attention_weight_max": 106.2036361694336,
      "activations/layer4_attention_weight_min": -100.80500793457031,
      "activations/layer5_attention_weight_max": 86.04962158203125,
      "activations/layer5_attention_weight_min": -81.63497161865234,
      "activations/layer6_attention_weight_max": 56.93968200683594,
      "activations/layer6_attention_weight_min": -54.92298889160156,
      "activations/layer7_attention_weight_max": 78.95357513427734,
      "activations/layer7_attention_weight_min": -77.99983978271484,
      "activations/layer8_attention_weight_max": 46.37250518798828,
      "activations/layer8_attention_weight_min": -46.79438400268555,
      "activations/layer9_attention_weight_max": 52.5028076171875,
      "activations/layer9_attention_weight_min": -51.81757736206055,
      "epoch": 8.36,
      "learning_rate": 9.703787878787877e-05,
      "loss": 2.7925,
      "step": 143950
    },
    {
      "activations/layer0_attention_weight_max": 15.421452522277832,
      "activations/layer0_attention_weight_min": -14.493123054504395,
      "activations/layer10_attention_weight_max": 39.85653305053711,
      "activations/layer10_attention_weight_min": -37.432350158691406,
      "activations/layer11_attention_weight_max": 33.747684478759766,
      "activations/layer11_attention_weight_min": -31.939075469970703,
      "activations/layer12_attention_weight_max": 20.27490997314453,
      "activations/layer12_attention_weight_min": -34.72377395629883,
      "activations/layer13_attention_weight_max": 60.81598663330078,
      "activations/layer13_attention_weight_min": -35.23618698120117,
      "activations/layer14_attention_weight_max": 40.15452194213867,
      "activations/layer14_attention_weight_min": -31.379152297973633,
      "activations/layer15_attention_weight_max": 40.429019927978516,
      "activations/layer15_attention_weight_min": -29.705642700195312,
      "activations/layer16_attention_weight_max": 32.160213470458984,
      "activations/layer16_attention_weight_min": -28.382186889648438,
      "activations/layer17_attention_weight_max": 52.98706817626953,
      "activations/layer17_attention_weight_min": -42.80936050415039,
      "activations/layer18_attention_weight_max": 49.41969680786133,
      "activations/layer18_attention_weight_min": -37.63142395019531,
      "activations/layer19_attention_weight_max": 24.429412841796875,
      "activations/layer19_attention_weight_min": -21.116432189941406,
      "activations/layer1_attention_weight_max": 18.7019100189209,
      "activations/layer1_attention_weight_min": -14.37110710144043,
      "activations/layer20_attention_weight_max": 25.1835994720459,
      "activations/layer20_attention_weight_min": -19.874542236328125,
      "activations/layer21_attention_weight_max": 42.18769454956055,
      "activations/layer21_attention_weight_min": -24.571697235107422,
      "activations/layer22_attention_weight_max": 30.754474639892578,
      "activations/layer22_attention_weight_min": -23.94394302368164,
      "activations/layer23_attention_weight_max": 40.14509582519531,
      "activations/layer23_attention_weight_min": -26.563915252685547,
      "activations/layer2_attention_weight_max": 34.37667465209961,
      "activations/layer2_attention_weight_min": -31.331878662109375,
      "activations/layer3_attention_weight_max": 92.30152893066406,
      "activations/layer3_attention_weight_min": -91.33749389648438,
      "activations/layer4_attention_weight_max": 105.6298599243164,
      "activations/layer4_attention_weight_min": -100.49189758300781,
      "activations/layer5_attention_weight_max": 86.26948547363281,
      "activations/layer5_attention_weight_min": -79.84153747558594,
      "activations/layer6_attention_weight_max": 58.308631896972656,
      "activations/layer6_attention_weight_min": -55.22264862060547,
      "activations/layer7_attention_weight_max": 84.16776275634766,
      "activations/layer7_attention_weight_min": -73.96200561523438,
      "activations/layer8_attention_weight_max": 52.62981033325195,
      "activations/layer8_attention_weight_min": -50.735870361328125,
      "activations/layer9_attention_weight_max": 58.74357223510742,
      "activations/layer9_attention_weight_min": -53.84055709838867,
      "epoch": 8.37,
      "learning_rate": 9.701893939393938e-05,
      "loss": 2.812,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_loss": 2.75,
      "eval_runtime": 8.5651,
      "eval_samples_per_second": 501.338,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.5651,
      "eval_openwebtext_samples_per_second": 501.338,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 1.9906,
      "eval_wikitext_samples_per_second": 229.076,
      "step": 144000
    },
    {
      "epoch": 8.37,
      "eval_lambada_loss": 2.78125,
      "eval_lambada_ppl": 16.139182322446413,
      "eval_lambada_runtime": 9.7116,
      "eval_lambada_samples_per_second": 501.359,
      "step": 144000
    },
    {
      "activations/layer0_attention_weight_max": 15.528882026672363,
      "activations/layer0_attention_weight_min": -14.862786293029785,
      "activations/layer10_attention_weight_max": 36.617897033691406,
      "activations/layer10_attention_weight_min": -35.51927185058594,
      "activations/layer11_attention_weight_max": 31.020782470703125,
      "activations/layer11_attention_weight_min": -31.137462615966797,
      "activations/layer12_attention_weight_max": 18.32313346862793,
      "activations/layer12_attention_weight_min": -24.6306095123291,
      "activations/layer13_attention_weight_max": 43.31779098510742,
      "activations/layer13_attention_weight_min": -33.28025436401367,
      "activations/layer14_attention_weight_max": 39.435516357421875,
      "activations/layer14_attention_weight_min": -32.507568359375,
      "activations/layer15_attention_weight_max": 33.89632034301758,
      "activations/layer15_attention_weight_min": -28.852476119995117,
      "activations/layer16_attention_weight_max": 30.10723114013672,
      "activations/layer16_attention_weight_min": -27.833375930786133,
      "activations/layer17_attention_weight_max": 57.78952407836914,
      "activations/layer17_attention_weight_min": -44.29972457885742,
      "activations/layer18_attention_weight_max": 48.91307830810547,
      "activations/layer18_attention_weight_min": -35.53428268432617,
      "activations/layer19_attention_weight_max": 24.13933563232422,
      "activations/layer19_attention_weight_min": -19.399677276611328,
      "activations/layer1_attention_weight_max": 17.665990829467773,
      "activations/layer1_attention_weight_min": -15.734423637390137,
      "activations/layer20_attention_weight_max": 25.331880569458008,
      "activations/layer20_attention_weight_min": -20.1094913482666,
      "activations/layer21_attention_weight_max": 35.390140533447266,
      "activations/layer21_attention_weight_min": -22.292705535888672,
      "activations/layer22_attention_weight_max": 30.701841354370117,
      "activations/layer22_attention_weight_min": -25.0434627532959,
      "activations/layer23_attention_weight_max": 36.7225341796875,
      "activations/layer23_attention_weight_min": -22.336769104003906,
      "activations/layer2_attention_weight_max": 31.730438232421875,
      "activations/layer2_attention_weight_min": -30.13922882080078,
      "activations/layer3_attention_weight_max": 86.309814453125,
      "activations/layer3_attention_weight_min": -90.65423583984375,
      "activations/layer4_attention_weight_max": 100.8613510131836,
      "activations/layer4_attention_weight_min": -99.36436462402344,
      "activations/layer5_attention_weight_max": 84.17343139648438,
      "activations/layer5_attention_weight_min": -79.66322326660156,
      "activations/layer6_attention_weight_max": 56.01057815551758,
      "activations/layer6_attention_weight_min": -56.66469192504883,
      "activations/layer7_attention_weight_max": 77.68434143066406,
      "activations/layer7_attention_weight_min": -71.61753845214844,
      "activations/layer8_attention_weight_max": 45.33535385131836,
      "activations/layer8_attention_weight_min": -44.8484992980957,
      "activations/layer9_attention_weight_max": 46.5177116394043,
      "activations/layer9_attention_weight_min": -48.6307373046875,
      "epoch": 8.37,
      "learning_rate": 9.699999999999999e-05,
      "loss": 2.8196,
      "step": 144050
    },
    {
      "activations/layer0_attention_weight_max": 15.218533515930176,
      "activations/layer0_attention_weight_min": -14.773496627807617,
      "activations/layer10_attention_weight_max": 39.64506912231445,
      "activations/layer10_attention_weight_min": -37.910606384277344,
      "activations/layer11_attention_weight_max": 33.611019134521484,
      "activations/layer11_attention_weight_min": -34.50151062011719,
      "activations/layer12_attention_weight_max": 18.488279342651367,
      "activations/layer12_attention_weight_min": -24.48870277404785,
      "activations/layer13_attention_weight_max": 36.23271179199219,
      "activations/layer13_attention_weight_min": -28.628353118896484,
      "activations/layer14_attention_weight_max": 37.76744842529297,
      "activations/layer14_attention_weight_min": -29.417104721069336,
      "activations/layer15_attention_weight_max": 32.65959930419922,
      "activations/layer15_attention_weight_min": -29.584604263305664,
      "activations/layer16_attention_weight_max": 30.31993293762207,
      "activations/layer16_attention_weight_min": -30.19878387451172,
      "activations/layer17_attention_weight_max": 56.095157623291016,
      "activations/layer17_attention_weight_min": -42.36406326293945,
      "activations/layer18_attention_weight_max": 47.87715530395508,
      "activations/layer18_attention_weight_min": -36.88454818725586,
      "activations/layer19_attention_weight_max": 24.650663375854492,
      "activations/layer19_attention_weight_min": -21.679555892944336,
      "activations/layer1_attention_weight_max": 16.342716217041016,
      "activations/layer1_attention_weight_min": -13.77475357055664,
      "activations/layer20_attention_weight_max": 21.856273651123047,
      "activations/layer20_attention_weight_min": -27.244640350341797,
      "activations/layer21_attention_weight_max": 40.6805419921875,
      "activations/layer21_attention_weight_min": -25.234018325805664,
      "activations/layer22_attention_weight_max": 29.00845718383789,
      "activations/layer22_attention_weight_min": -25.08470344543457,
      "activations/layer23_attention_weight_max": 35.343833923339844,
      "activations/layer23_attention_weight_min": -24.35829734802246,
      "activations/layer2_attention_weight_max": 31.943279266357422,
      "activations/layer2_attention_weight_min": -32.39369201660156,
      "activations/layer3_attention_weight_max": 90.54524230957031,
      "activations/layer3_attention_weight_min": -98.88257598876953,
      "activations/layer4_attention_weight_max": 106.70246887207031,
      "activations/layer4_attention_weight_min": -103.8045654296875,
      "activations/layer5_attention_weight_max": 88.69215393066406,
      "activations/layer5_attention_weight_min": -86.4714584350586,
      "activations/layer6_attention_weight_max": 59.28535461425781,
      "activations/layer6_attention_weight_min": -57.10057830810547,
      "activations/layer7_attention_weight_max": 78.71817016601562,
      "activations/layer7_attention_weight_min": -78.67742156982422,
      "activations/layer8_attention_weight_max": 47.60182571411133,
      "activations/layer8_attention_weight_min": -47.91048049926758,
      "activations/layer9_attention_weight_max": 50.75791549682617,
      "activations/layer9_attention_weight_min": -50.987003326416016,
      "epoch": 8.37,
      "learning_rate": 9.69810606060606e-05,
      "loss": 2.7946,
      "step": 144100
    },
    {
      "activations/layer0_attention_weight_max": 15.407171249389648,
      "activations/layer0_attention_weight_min": -14.636046409606934,
      "activations/layer10_attention_weight_max": 45.90824508666992,
      "activations/layer10_attention_weight_min": -44.15007781982422,
      "activations/layer11_attention_weight_max": 40.504364013671875,
      "activations/layer11_attention_weight_min": -36.648494720458984,
      "activations/layer12_attention_weight_max": 21.61686134338379,
      "activations/layer12_attention_weight_min": -23.13011932373047,
      "activations/layer13_attention_weight_max": 51.57648849487305,
      "activations/layer13_attention_weight_min": -32.412147521972656,
      "activations/layer14_attention_weight_max": 50.129085540771484,
      "activations/layer14_attention_weight_min": -31.93443489074707,
      "activations/layer15_attention_weight_max": 41.17356491088867,
      "activations/layer15_attention_weight_min": -30.626161575317383,
      "activations/layer16_attention_weight_max": 35.82870864868164,
      "activations/layer16_attention_weight_min": -29.714031219482422,
      "activations/layer17_attention_weight_max": 58.80724334716797,
      "activations/layer17_attention_weight_min": -45.5117301940918,
      "activations/layer18_attention_weight_max": 56.52589797973633,
      "activations/layer18_attention_weight_min": -39.855812072753906,
      "activations/layer19_attention_weight_max": 24.787553787231445,
      "activations/layer19_attention_weight_min": -22.088531494140625,
      "activations/layer1_attention_weight_max": 16.641361236572266,
      "activations/layer1_attention_weight_min": -16.213558197021484,
      "activations/layer20_attention_weight_max": 26.903419494628906,
      "activations/layer20_attention_weight_min": -21.77193260192871,
      "activations/layer21_attention_weight_max": 49.30426025390625,
      "activations/layer21_attention_weight_min": -25.757402420043945,
      "activations/layer22_attention_weight_max": 34.64835739135742,
      "activations/layer22_attention_weight_min": -24.172893524169922,
      "activations/layer23_attention_weight_max": 40.40262985229492,
      "activations/layer23_attention_weight_min": -24.093122482299805,
      "activations/layer2_attention_weight_max": 29.856525421142578,
      "activations/layer2_attention_weight_min": -31.032363891601562,
      "activations/layer3_attention_weight_max": 83.81267547607422,
      "activations/layer3_attention_weight_min": -90.58543395996094,
      "activations/layer4_attention_weight_max": 99.67733764648438,
      "activations/layer4_attention_weight_min": -98.74956512451172,
      "activations/layer5_attention_weight_max": 84.28361511230469,
      "activations/layer5_attention_weight_min": -77.26575469970703,
      "activations/layer6_attention_weight_max": 56.62871170043945,
      "activations/layer6_attention_weight_min": -52.59785842895508,
      "activations/layer7_attention_weight_max": 89.36730194091797,
      "activations/layer7_attention_weight_min": -78.02892303466797,
      "activations/layer8_attention_weight_max": 54.49415588378906,
      "activations/layer8_attention_weight_min": -50.603946685791016,
      "activations/layer9_attention_weight_max": 75.32173919677734,
      "activations/layer9_attention_weight_min": -54.63077926635742,
      "epoch": 8.38,
      "learning_rate": 9.69621212121212e-05,
      "loss": 2.8015,
      "step": 144150
    },
    {
      "activations/layer0_attention_weight_max": 14.73915958404541,
      "activations/layer0_attention_weight_min": -14.71558952331543,
      "activations/layer10_attention_weight_max": 39.156822204589844,
      "activations/layer10_attention_weight_min": -39.10526657104492,
      "activations/layer11_attention_weight_max": 35.1173095703125,
      "activations/layer11_attention_weight_min": -32.216285705566406,
      "activations/layer12_attention_weight_max": 20.167495727539062,
      "activations/layer12_attention_weight_min": -23.517038345336914,
      "activations/layer13_attention_weight_max": 48.84244918823242,
      "activations/layer13_attention_weight_min": -29.42014503479004,
      "activations/layer14_attention_weight_max": 44.84531021118164,
      "activations/layer14_attention_weight_min": -30.111003875732422,
      "activations/layer15_attention_weight_max": 39.16856384277344,
      "activations/layer15_attention_weight_min": -27.8853759765625,
      "activations/layer16_attention_weight_max": 34.167816162109375,
      "activations/layer16_attention_weight_min": -26.97245979309082,
      "activations/layer17_attention_weight_max": 57.14754867553711,
      "activations/layer17_attention_weight_min": -42.07025909423828,
      "activations/layer18_attention_weight_max": 48.70786666870117,
      "activations/layer18_attention_weight_min": -34.9669189453125,
      "activations/layer19_attention_weight_max": 24.192827224731445,
      "activations/layer19_attention_weight_min": -20.681333541870117,
      "activations/layer1_attention_weight_max": 17.478721618652344,
      "activations/layer1_attention_weight_min": -15.610806465148926,
      "activations/layer20_attention_weight_max": 23.83132553100586,
      "activations/layer20_attention_weight_min": -19.285381317138672,
      "activations/layer21_attention_weight_max": 40.28135299682617,
      "activations/layer21_attention_weight_min": -23.526437759399414,
      "activations/layer22_attention_weight_max": 35.62934875488281,
      "activations/layer22_attention_weight_min": -26.31879997253418,
      "activations/layer23_attention_weight_max": 45.03984069824219,
      "activations/layer23_attention_weight_min": -23.818836212158203,
      "activations/layer2_attention_weight_max": 31.816696166992188,
      "activations/layer2_attention_weight_min": -31.714216232299805,
      "activations/layer3_attention_weight_max": 92.26164245605469,
      "activations/layer3_attention_weight_min": -95.26957702636719,
      "activations/layer4_attention_weight_max": 111.160888671875,
      "activations/layer4_attention_weight_min": -104.11952209472656,
      "activations/layer5_attention_weight_max": 93.73316955566406,
      "activations/layer5_attention_weight_min": -86.20320892333984,
      "activations/layer6_attention_weight_max": 60.11103820800781,
      "activations/layer6_attention_weight_min": -57.9672966003418,
      "activations/layer7_attention_weight_max": 83.70993041992188,
      "activations/layer7_attention_weight_min": -81.80257415771484,
      "activations/layer8_attention_weight_max": 48.20357131958008,
      "activations/layer8_attention_weight_min": -53.09721755981445,
      "activations/layer9_attention_weight_max": 54.77079391479492,
      "activations/layer9_attention_weight_min": -53.8338623046875,
      "epoch": 8.38,
      "learning_rate": 9.69431818181818e-05,
      "loss": 2.7888,
      "step": 144200
    },
    {
      "activations/layer0_attention_weight_max": 15.100202560424805,
      "activations/layer0_attention_weight_min": -14.845865249633789,
      "activations/layer10_attention_weight_max": 35.899925231933594,
      "activations/layer10_attention_weight_min": -34.32582092285156,
      "activations/layer11_attention_weight_max": 30.508079528808594,
      "activations/layer11_attention_weight_min": -30.841266632080078,
      "activations/layer12_attention_weight_max": 21.219173431396484,
      "activations/layer12_attention_weight_min": -20.85013198852539,
      "activations/layer13_attention_weight_max": 46.39008712768555,
      "activations/layer13_attention_weight_min": -34.52178955078125,
      "activations/layer14_attention_weight_max": 36.326324462890625,
      "activations/layer14_attention_weight_min": -30.024168014526367,
      "activations/layer15_attention_weight_max": 33.610599517822266,
      "activations/layer15_attention_weight_min": -28.084787368774414,
      "activations/layer16_attention_weight_max": 32.183433532714844,
      "activations/layer16_attention_weight_min": -29.3485050201416,
      "activations/layer17_attention_weight_max": 58.21746063232422,
      "activations/layer17_attention_weight_min": -45.66917037963867,
      "activations/layer18_attention_weight_max": 48.877254486083984,
      "activations/layer18_attention_weight_min": -38.30216598510742,
      "activations/layer19_attention_weight_max": 25.41319465637207,
      "activations/layer19_attention_weight_min": -25.002689361572266,
      "activations/layer1_attention_weight_max": 16.125221252441406,
      "activations/layer1_attention_weight_min": -15.370359420776367,
      "activations/layer20_attention_weight_max": 25.657663345336914,
      "activations/layer20_attention_weight_min": -22.112272262573242,
      "activations/layer21_attention_weight_max": 38.97475051879883,
      "activations/layer21_attention_weight_min": -25.739646911621094,
      "activations/layer22_attention_weight_max": 33.49221420288086,
      "activations/layer22_attention_weight_min": -27.00261116027832,
      "activations/layer23_attention_weight_max": 41.776893615722656,
      "activations/layer23_attention_weight_min": -22.471397399902344,
      "activations/layer2_attention_weight_max": 31.3966064453125,
      "activations/layer2_attention_weight_min": -29.359281539916992,
      "activations/layer3_attention_weight_max": 90.8677749633789,
      "activations/layer3_attention_weight_min": -90.56221771240234,
      "activations/layer4_attention_weight_max": 102.00289916992188,
      "activations/layer4_attention_weight_min": -96.97917175292969,
      "activations/layer5_attention_weight_max": 83.48100280761719,
      "activations/layer5_attention_weight_min": -77.75048065185547,
      "activations/layer6_attention_weight_max": 54.91520690917969,
      "activations/layer6_attention_weight_min": -54.22068405151367,
      "activations/layer7_attention_weight_max": 75.4092788696289,
      "activations/layer7_attention_weight_min": -72.35433959960938,
      "activations/layer8_attention_weight_max": 44.25127029418945,
      "activations/layer8_attention_weight_min": -43.59839630126953,
      "activations/layer9_attention_weight_max": 51.464942932128906,
      "activations/layer9_attention_weight_min": -49.669681549072266,
      "epoch": 8.38,
      "learning_rate": 9.692424242424242e-05,
      "loss": 2.7989,
      "step": 144250
    },
    {
      "activations/layer0_attention_weight_max": 14.78736400604248,
      "activations/layer0_attention_weight_min": -14.790545463562012,
      "activations/layer10_attention_weight_max": 34.8682746887207,
      "activations/layer10_attention_weight_min": -33.372833251953125,
      "activations/layer11_attention_weight_max": 28.949934005737305,
      "activations/layer11_attention_weight_min": -29.787643432617188,
      "activations/layer12_attention_weight_max": 17.873289108276367,
      "activations/layer12_attention_weight_min": -23.38253402709961,
      "activations/layer13_attention_weight_max": 34.36337661743164,
      "activations/layer13_attention_weight_min": -30.731090545654297,
      "activations/layer14_attention_weight_max": 35.178863525390625,
      "activations/layer14_attention_weight_min": -30.645122528076172,
      "activations/layer15_attention_weight_max": 30.860214233398438,
      "activations/layer15_attention_weight_min": -29.020687103271484,
      "activations/layer16_attention_weight_max": 28.831872940063477,
      "activations/layer16_attention_weight_min": -27.436527252197266,
      "activations/layer17_attention_weight_max": 50.89079666137695,
      "activations/layer17_attention_weight_min": -43.64326095581055,
      "activations/layer18_attention_weight_max": 44.52324676513672,
      "activations/layer18_attention_weight_min": -35.89640808105469,
      "activations/layer19_attention_weight_max": 23.330989837646484,
      "activations/layer19_attention_weight_min": -20.51032066345215,
      "activations/layer1_attention_weight_max": 16.31631851196289,
      "activations/layer1_attention_weight_min": -15.117304801940918,
      "activations/layer20_attention_weight_max": 21.073305130004883,
      "activations/layer20_attention_weight_min": -24.3541316986084,
      "activations/layer21_attention_weight_max": 34.462257385253906,
      "activations/layer21_attention_weight_min": -23.786943435668945,
      "activations/layer22_attention_weight_max": 34.57782745361328,
      "activations/layer22_attention_weight_min": -26.99131965637207,
      "activations/layer23_attention_weight_max": 31.923053741455078,
      "activations/layer23_attention_weight_min": -23.231779098510742,
      "activations/layer2_attention_weight_max": 31.580486297607422,
      "activations/layer2_attention_weight_min": -29.801488876342773,
      "activations/layer3_attention_weight_max": 87.82120513916016,
      "activations/layer3_attention_weight_min": -90.9281234741211,
      "activations/layer4_attention_weight_max": 99.91016387939453,
      "activations/layer4_attention_weight_min": -96.68793487548828,
      "activations/layer5_attention_weight_max": 82.01717376708984,
      "activations/layer5_attention_weight_min": -80.28263854980469,
      "activations/layer6_attention_weight_max": 52.46807098388672,
      "activations/layer6_attention_weight_min": -50.06455993652344,
      "activations/layer7_attention_weight_max": 74.70899200439453,
      "activations/layer7_attention_weight_min": -74.65666198730469,
      "activations/layer8_attention_weight_max": 45.8183479309082,
      "activations/layer8_attention_weight_min": -44.96138381958008,
      "activations/layer9_attention_weight_max": 52.45631790161133,
      "activations/layer9_attention_weight_min": -46.90854263305664,
      "epoch": 8.38,
      "learning_rate": 9.690530303030303e-05,
      "loss": 2.7933,
      "step": 144300
    },
    {
      "activations/layer0_attention_weight_max": 14.163182258605957,
      "activations/layer0_attention_weight_min": -15.3079195022583,
      "activations/layer10_attention_weight_max": 48.59516143798828,
      "activations/layer10_attention_weight_min": -40.42572021484375,
      "activations/layer11_attention_weight_max": 38.20131301879883,
      "activations/layer11_attention_weight_min": -33.040863037109375,
      "activations/layer12_attention_weight_max": 18.55327796936035,
      "activations/layer12_attention_weight_min": -25.745882034301758,
      "activations/layer13_attention_weight_max": 46.9095573425293,
      "activations/layer13_attention_weight_min": -38.164398193359375,
      "activations/layer14_attention_weight_max": 41.39414596557617,
      "activations/layer14_attention_weight_min": -30.38783836364746,
      "activations/layer15_attention_weight_max": 35.08184051513672,
      "activations/layer15_attention_weight_min": -29.042484283447266,
      "activations/layer16_attention_weight_max": 29.85325050354004,
      "activations/layer16_attention_weight_min": -28.65825653076172,
      "activations/layer17_attention_weight_max": 55.646549224853516,
      "activations/layer17_attention_weight_min": -44.53288650512695,
      "activations/layer18_attention_weight_max": 46.488975524902344,
      "activations/layer18_attention_weight_min": -38.85862731933594,
      "activations/layer19_attention_weight_max": 22.744230270385742,
      "activations/layer19_attention_weight_min": -23.671388626098633,
      "activations/layer1_attention_weight_max": 16.086990356445312,
      "activations/layer1_attention_weight_min": -16.04709243774414,
      "activations/layer20_attention_weight_max": 25.712139129638672,
      "activations/layer20_attention_weight_min": -23.906322479248047,
      "activations/layer21_attention_weight_max": 43.254581451416016,
      "activations/layer21_attention_weight_min": -26.554418563842773,
      "activations/layer22_attention_weight_max": 31.90452003479004,
      "activations/layer22_attention_weight_min": -25.71658706665039,
      "activations/layer23_attention_weight_max": 38.85893249511719,
      "activations/layer23_attention_weight_min": -23.145339965820312,
      "activations/layer2_attention_weight_max": 33.700313568115234,
      "activations/layer2_attention_weight_min": -31.138160705566406,
      "activations/layer3_attention_weight_max": 92.85909271240234,
      "activations/layer3_attention_weight_min": -100.66903686523438,
      "activations/layer4_attention_weight_max": 111.07109832763672,
      "activations/layer4_attention_weight_min": -104.0634994506836,
      "activations/layer5_attention_weight_max": 94.38230895996094,
      "activations/layer5_attention_weight_min": -85.20419311523438,
      "activations/layer6_attention_weight_max": 61.220035552978516,
      "activations/layer6_attention_weight_min": -57.34938049316406,
      "activations/layer7_attention_weight_max": 84.08382415771484,
      "activations/layer7_attention_weight_min": -80.36780548095703,
      "activations/layer8_attention_weight_max": 54.134517669677734,
      "activations/layer8_attention_weight_min": -47.8293571472168,
      "activations/layer9_attention_weight_max": 65.49022674560547,
      "activations/layer9_attention_weight_min": -52.4038200378418,
      "epoch": 8.39,
      "learning_rate": 9.688636363636362e-05,
      "loss": 2.7957,
      "step": 144350
    },
    {
      "activations/layer0_attention_weight_max": 14.733823776245117,
      "activations/layer0_attention_weight_min": -14.999106407165527,
      "activations/layer10_attention_weight_max": 39.3775634765625,
      "activations/layer10_attention_weight_min": -37.652870178222656,
      "activations/layer11_attention_weight_max": 33.045005798339844,
      "activations/layer11_attention_weight_min": -31.202930450439453,
      "activations/layer12_attention_weight_max": 22.461856842041016,
      "activations/layer12_attention_weight_min": -28.894485473632812,
      "activations/layer13_attention_weight_max": 43.672794342041016,
      "activations/layer13_attention_weight_min": -30.884511947631836,
      "activations/layer14_attention_weight_max": 39.49931716918945,
      "activations/layer14_attention_weight_min": -30.733665466308594,
      "activations/layer15_attention_weight_max": 37.40431213378906,
      "activations/layer15_attention_weight_min": -30.183053970336914,
      "activations/layer16_attention_weight_max": 34.20606231689453,
      "activations/layer16_attention_weight_min": -28.400718688964844,
      "activations/layer17_attention_weight_max": 55.58007049560547,
      "activations/layer17_attention_weight_min": -42.879329681396484,
      "activations/layer18_attention_weight_max": 50.75944519042969,
      "activations/layer18_attention_weight_min": -37.5096549987793,
      "activations/layer19_attention_weight_max": 22.495168685913086,
      "activations/layer19_attention_weight_min": -20.845205307006836,
      "activations/layer1_attention_weight_max": 16.44957160949707,
      "activations/layer1_attention_weight_min": -15.330845832824707,
      "activations/layer20_attention_weight_max": 23.950706481933594,
      "activations/layer20_attention_weight_min": -22.321203231811523,
      "activations/layer21_attention_weight_max": 42.34135818481445,
      "activations/layer21_attention_weight_min": -25.850322723388672,
      "activations/layer22_attention_weight_max": 34.79492950439453,
      "activations/layer22_attention_weight_min": -25.832897186279297,
      "activations/layer23_attention_weight_max": 38.33483123779297,
      "activations/layer23_attention_weight_min": -25.432294845581055,
      "activations/layer2_attention_weight_max": 31.52313995361328,
      "activations/layer2_attention_weight_min": -29.679182052612305,
      "activations/layer3_attention_weight_max": 87.12969207763672,
      "activations/layer3_attention_weight_min": -92.93195343017578,
      "activations/layer4_attention_weight_max": 100.99798583984375,
      "activations/layer4_attention_weight_min": -98.81810760498047,
      "activations/layer5_attention_weight_max": 82.20547485351562,
      "activations/layer5_attention_weight_min": -80.10792541503906,
      "activations/layer6_attention_weight_max": 55.5040168762207,
      "activations/layer6_attention_weight_min": -54.11521530151367,
      "activations/layer7_attention_weight_max": 76.98059844970703,
      "activations/layer7_attention_weight_min": -72.57102966308594,
      "activations/layer8_attention_weight_max": 44.0762939453125,
      "activations/layer8_attention_weight_min": -46.36068344116211,
      "activations/layer9_attention_weight_max": 52.28297805786133,
      "activations/layer9_attention_weight_min": -48.183433532714844,
      "epoch": 8.39,
      "learning_rate": 9.686742424242424e-05,
      "loss": 2.7876,
      "step": 144400
    },
    {
      "activations/layer0_attention_weight_max": 15.472841262817383,
      "activations/layer0_attention_weight_min": -15.529025077819824,
      "activations/layer10_attention_weight_max": 38.48505401611328,
      "activations/layer10_attention_weight_min": -37.082725524902344,
      "activations/layer11_attention_weight_max": 33.117393493652344,
      "activations/layer11_attention_weight_min": -30.94184684753418,
      "activations/layer12_attention_weight_max": 21.569963455200195,
      "activations/layer12_attention_weight_min": -23.417142868041992,
      "activations/layer13_attention_weight_max": 51.0227165222168,
      "activations/layer13_attention_weight_min": -31.80954360961914,
      "activations/layer14_attention_weight_max": 45.86099624633789,
      "activations/layer14_attention_weight_min": -30.77225685119629,
      "activations/layer15_attention_weight_max": 50.94770431518555,
      "activations/layer15_attention_weight_min": -29.669025421142578,
      "activations/layer16_attention_weight_max": 31.600650787353516,
      "activations/layer16_attention_weight_min": -27.031185150146484,
      "activations/layer17_attention_weight_max": 59.22002410888672,
      "activations/layer17_attention_weight_min": -43.71645736694336,
      "activations/layer18_attention_weight_max": 44.12919998168945,
      "activations/layer18_attention_weight_min": -36.6180534362793,
      "activations/layer19_attention_weight_max": 20.965784072875977,
      "activations/layer19_attention_weight_min": -22.711620330810547,
      "activations/layer1_attention_weight_max": 16.888717651367188,
      "activations/layer1_attention_weight_min": -14.621777534484863,
      "activations/layer20_attention_weight_max": 20.55522918701172,
      "activations/layer20_attention_weight_min": -26.55373764038086,
      "activations/layer21_attention_weight_max": 45.527137756347656,
      "activations/layer21_attention_weight_min": -24.689592361450195,
      "activations/layer22_attention_weight_max": 29.841611862182617,
      "activations/layer22_attention_weight_min": -25.261194229125977,
      "activations/layer23_attention_weight_max": 33.61469650268555,
      "activations/layer23_attention_weight_min": -21.041431427001953,
      "activations/layer2_attention_weight_max": 30.5023250579834,
      "activations/layer2_attention_weight_min": -29.97232437133789,
      "activations/layer3_attention_weight_max": 89.28436279296875,
      "activations/layer3_attention_weight_min": -88.37257385253906,
      "activations/layer4_attention_weight_max": 98.55216217041016,
      "activations/layer4_attention_weight_min": -93.885009765625,
      "activations/layer5_attention_weight_max": 79.69514465332031,
      "activations/layer5_attention_weight_min": -76.3100357055664,
      "activations/layer6_attention_weight_max": 53.376155853271484,
      "activations/layer6_attention_weight_min": -51.82337951660156,
      "activations/layer7_attention_weight_max": 75.46424102783203,
      "activations/layer7_attention_weight_min": -72.10772705078125,
      "activations/layer8_attention_weight_max": 46.0103645324707,
      "activations/layer8_attention_weight_min": -44.8517951965332,
      "activations/layer9_attention_weight_max": 50.13684844970703,
      "activations/layer9_attention_weight_min": -47.827510833740234,
      "epoch": 8.39,
      "learning_rate": 9.684848484848485e-05,
      "loss": 2.7908,
      "step": 144450
    },
    {
      "activations/layer0_attention_weight_max": 14.705248832702637,
      "activations/layer0_attention_weight_min": -15.53108024597168,
      "activations/layer10_attention_weight_max": 41.82246017456055,
      "activations/layer10_attention_weight_min": -39.500274658203125,
      "activations/layer11_attention_weight_max": 34.94831466674805,
      "activations/layer11_attention_weight_min": -33.100685119628906,
      "activations/layer12_attention_weight_max": 19.3266658782959,
      "activations/layer12_attention_weight_min": -23.81690788269043,
      "activations/layer13_attention_weight_max": 42.795021057128906,
      "activations/layer13_attention_weight_min": -34.55325698852539,
      "activations/layer14_attention_weight_max": 38.58967208862305,
      "activations/layer14_attention_weight_min": -30.561399459838867,
      "activations/layer15_attention_weight_max": 36.88859558105469,
      "activations/layer15_attention_weight_min": -30.90292739868164,
      "activations/layer16_attention_weight_max": 31.667423248291016,
      "activations/layer16_attention_weight_min": -26.023731231689453,
      "activations/layer17_attention_weight_max": 53.669715881347656,
      "activations/layer17_attention_weight_min": -39.46356964111328,
      "activations/layer18_attention_weight_max": 44.19626998901367,
      "activations/layer18_attention_weight_min": -33.24448013305664,
      "activations/layer19_attention_weight_max": 23.255788803100586,
      "activations/layer19_attention_weight_min": -20.91842269897461,
      "activations/layer1_attention_weight_max": 16.953914642333984,
      "activations/layer1_attention_weight_min": -14.125260353088379,
      "activations/layer20_attention_weight_max": 22.104389190673828,
      "activations/layer20_attention_weight_min": -23.612228393554688,
      "activations/layer21_attention_weight_max": 39.779178619384766,
      "activations/layer21_attention_weight_min": -23.134624481201172,
      "activations/layer22_attention_weight_max": 28.739364624023438,
      "activations/layer22_attention_weight_min": -24.54865837097168,
      "activations/layer23_attention_weight_max": 33.75872039794922,
      "activations/layer23_attention_weight_min": -21.538616180419922,
      "activations/layer2_attention_weight_max": 31.212589263916016,
      "activations/layer2_attention_weight_min": -30.332975387573242,
      "activations/layer3_attention_weight_max": 88.46522521972656,
      "activations/layer3_attention_weight_min": -91.82284545898438,
      "activations/layer4_attention_weight_max": 101.948486328125,
      "activations/layer4_attention_weight_min": -97.52915954589844,
      "activations/layer5_attention_weight_max": 80.74163055419922,
      "activations/layer5_attention_weight_min": -77.74270629882812,
      "activations/layer6_attention_weight_max": 55.24845886230469,
      "activations/layer6_attention_weight_min": -55.14009094238281,
      "activations/layer7_attention_weight_max": 80.32052612304688,
      "activations/layer7_attention_weight_min": -77.40782165527344,
      "activations/layer8_attention_weight_max": 48.11709213256836,
      "activations/layer8_attention_weight_min": -48.30885696411133,
      "activations/layer9_attention_weight_max": 51.31522750854492,
      "activations/layer9_attention_weight_min": -53.0446662902832,
      "epoch": 8.4,
      "learning_rate": 9.682954545454544e-05,
      "loss": 2.8001,
      "step": 144500
    },
    {
      "activations/layer0_attention_weight_max": 16.080074310302734,
      "activations/layer0_attention_weight_min": -15.204364776611328,
      "activations/layer10_attention_weight_max": 36.678627014160156,
      "activations/layer10_attention_weight_min": -35.426918029785156,
      "activations/layer11_attention_weight_max": 33.391075134277344,
      "activations/layer11_attention_weight_min": -31.244123458862305,
      "activations/layer12_attention_weight_max": 19.267086029052734,
      "activations/layer12_attention_weight_min": -26.35028648376465,
      "activations/layer13_attention_weight_max": 43.668601989746094,
      "activations/layer13_attention_weight_min": -32.29851150512695,
      "activations/layer14_attention_weight_max": 43.81389617919922,
      "activations/layer14_attention_weight_min": -30.79947280883789,
      "activations/layer15_attention_weight_max": 37.05815124511719,
      "activations/layer15_attention_weight_min": -29.408647537231445,
      "activations/layer16_attention_weight_max": 31.404279708862305,
      "activations/layer16_attention_weight_min": -27.120498657226562,
      "activations/layer17_attention_weight_max": 51.72565841674805,
      "activations/layer17_attention_weight_min": -43.01002502441406,
      "activations/layer18_attention_weight_max": 45.29131317138672,
      "activations/layer18_attention_weight_min": -36.43537521362305,
      "activations/layer19_attention_weight_max": 22.836231231689453,
      "activations/layer19_attention_weight_min": -21.34046173095703,
      "activations/layer1_attention_weight_max": 16.97292709350586,
      "activations/layer1_attention_weight_min": -15.761260032653809,
      "activations/layer20_attention_weight_max": 24.532560348510742,
      "activations/layer20_attention_weight_min": -18.676877975463867,
      "activations/layer21_attention_weight_max": 43.17038345336914,
      "activations/layer21_attention_weight_min": -24.357707977294922,
      "activations/layer22_attention_weight_max": 34.18451690673828,
      "activations/layer22_attention_weight_min": -26.242164611816406,
      "activations/layer23_attention_weight_max": 35.58464050292969,
      "activations/layer23_attention_weight_min": -20.068145751953125,
      "activations/layer2_attention_weight_max": 33.70246887207031,
      "activations/layer2_attention_weight_min": -30.70868682861328,
      "activations/layer3_attention_weight_max": 89.7096939086914,
      "activations/layer3_attention_weight_min": -93.99972534179688,
      "activations/layer4_attention_weight_max": 100.87278747558594,
      "activations/layer4_attention_weight_min": -97.10021209716797,
      "activations/layer5_attention_weight_max": 83.51813507080078,
      "activations/layer5_attention_weight_min": -77.40599822998047,
      "activations/layer6_attention_weight_max": 56.32121658325195,
      "activations/layer6_attention_weight_min": -54.83113098144531,
      "activations/layer7_attention_weight_max": 82.13043212890625,
      "activations/layer7_attention_weight_min": -75.7643814086914,
      "activations/layer8_attention_weight_max": 47.30461502075195,
      "activations/layer8_attention_weight_min": -46.766151428222656,
      "activations/layer9_attention_weight_max": 55.28446578979492,
      "activations/layer9_attention_weight_min": -51.972042083740234,
      "epoch": 8.4,
      "learning_rate": 9.681060606060606e-05,
      "loss": 2.7907,
      "step": 144550
    },
    {
      "activations/layer0_attention_weight_max": 15.314294815063477,
      "activations/layer0_attention_weight_min": -15.112451553344727,
      "activations/layer10_attention_weight_max": 38.375640869140625,
      "activations/layer10_attention_weight_min": -37.10407257080078,
      "activations/layer11_attention_weight_max": 31.368122100830078,
      "activations/layer11_attention_weight_min": -32.53509521484375,
      "activations/layer12_attention_weight_max": 27.861629486083984,
      "activations/layer12_attention_weight_min": -23.626399993896484,
      "activations/layer13_attention_weight_max": 41.394775390625,
      "activations/layer13_attention_weight_min": -32.47016143798828,
      "activations/layer14_attention_weight_max": 37.66852951049805,
      "activations/layer14_attention_weight_min": -30.941261291503906,
      "activations/layer15_attention_weight_max": 33.9344482421875,
      "activations/layer15_attention_weight_min": -30.059179306030273,
      "activations/layer16_attention_weight_max": 30.31122589111328,
      "activations/layer16_attention_weight_min": -27.776350021362305,
      "activations/layer17_attention_weight_max": 55.46149444580078,
      "activations/layer17_attention_weight_min": -43.90761184692383,
      "activations/layer18_attention_weight_max": 49.84128189086914,
      "activations/layer18_attention_weight_min": -38.02252960205078,
      "activations/layer19_attention_weight_max": 24.037412643432617,
      "activations/layer19_attention_weight_min": -19.822921752929688,
      "activations/layer1_attention_weight_max": 17.05846405029297,
      "activations/layer1_attention_weight_min": -14.678099632263184,
      "activations/layer20_attention_weight_max": 22.764324188232422,
      "activations/layer20_attention_weight_min": -24.645048141479492,
      "activations/layer21_attention_weight_max": 35.792545318603516,
      "activations/layer21_attention_weight_min": -22.651412963867188,
      "activations/layer22_attention_weight_max": 34.922698974609375,
      "activations/layer22_attention_weight_min": -27.78801918029785,
      "activations/layer23_attention_weight_max": 35.99385452270508,
      "activations/layer23_attention_weight_min": -20.742610931396484,
      "activations/layer2_attention_weight_max": 31.663440704345703,
      "activations/layer2_attention_weight_min": -29.60903549194336,
      "activations/layer3_attention_weight_max": 90.642578125,
      "activations/layer3_attention_weight_min": -91.29776763916016,
      "activations/layer4_attention_weight_max": 105.01183319091797,
      "activations/layer4_attention_weight_min": -99.0431137084961,
      "activations/layer5_attention_weight_max": 86.01840209960938,
      "activations/layer5_attention_weight_min": -85.7931137084961,
      "activations/layer6_attention_weight_max": 58.90391540527344,
      "activations/layer6_attention_weight_min": -55.44581985473633,
      "activations/layer7_attention_weight_max": 83.69498443603516,
      "activations/layer7_attention_weight_min": -80.1080093383789,
      "activations/layer8_attention_weight_max": 49.192508697509766,
      "activations/layer8_attention_weight_min": -48.053009033203125,
      "activations/layer9_attention_weight_max": 50.17440414428711,
      "activations/layer9_attention_weight_min": -53.256561279296875,
      "epoch": 8.4,
      "learning_rate": 9.679166666666666e-05,
      "loss": 2.8052,
      "step": 144600
    },
    {
      "activations/layer0_attention_weight_max": 14.845047950744629,
      "activations/layer0_attention_weight_min": -15.100740432739258,
      "activations/layer10_attention_weight_max": 37.84589385986328,
      "activations/layer10_attention_weight_min": -36.977455139160156,
      "activations/layer11_attention_weight_max": 32.18812942504883,
      "activations/layer11_attention_weight_min": -31.686767578125,
      "activations/layer12_attention_weight_max": 20.234094619750977,
      "activations/layer12_attention_weight_min": -24.861732482910156,
      "activations/layer13_attention_weight_max": 47.62570571899414,
      "activations/layer13_attention_weight_min": -33.529911041259766,
      "activations/layer14_attention_weight_max": 36.68297576904297,
      "activations/layer14_attention_weight_min": -32.02961730957031,
      "activations/layer15_attention_weight_max": 33.501399993896484,
      "activations/layer15_attention_weight_min": -28.086877822875977,
      "activations/layer16_attention_weight_max": 32.663246154785156,
      "activations/layer16_attention_weight_min": -26.585893630981445,
      "activations/layer17_attention_weight_max": 54.04103469848633,
      "activations/layer17_attention_weight_min": -42.48189163208008,
      "activations/layer18_attention_weight_max": 43.07803726196289,
      "activations/layer18_attention_weight_min": -33.95008087158203,
      "activations/layer19_attention_weight_max": 22.291576385498047,
      "activations/layer19_attention_weight_min": -19.790882110595703,
      "activations/layer1_attention_weight_max": 17.254297256469727,
      "activations/layer1_attention_weight_min": -15.115432739257812,
      "activations/layer20_attention_weight_max": 24.62740707397461,
      "activations/layer20_attention_weight_min": -20.464189529418945,
      "activations/layer21_attention_weight_max": 38.74768829345703,
      "activations/layer21_attention_weight_min": -21.759929656982422,
      "activations/layer22_attention_weight_max": 36.124847412109375,
      "activations/layer22_attention_weight_min": -25.24366569519043,
      "activations/layer23_attention_weight_max": 35.462440490722656,
      "activations/layer23_attention_weight_min": -22.828994750976562,
      "activations/layer2_attention_weight_max": 33.12522506713867,
      "activations/layer2_attention_weight_min": -30.72991943359375,
      "activations/layer3_attention_weight_max": 92.10752868652344,
      "activations/layer3_attention_weight_min": -96.31632232666016,
      "activations/layer4_attention_weight_max": 105.29671478271484,
      "activations/layer4_attention_weight_min": -100.20915222167969,
      "activations/layer5_attention_weight_max": 85.382568359375,
      "activations/layer5_attention_weight_min": -80.8235092163086,
      "activations/layer6_attention_weight_max": 60.08195877075195,
      "activations/layer6_attention_weight_min": -60.1597785949707,
      "activations/layer7_attention_weight_max": 78.1812973022461,
      "activations/layer7_attention_weight_min": -79.1328353881836,
      "activations/layer8_attention_weight_max": 46.82227325439453,
      "activations/layer8_attention_weight_min": -48.865970611572266,
      "activations/layer9_attention_weight_max": 55.87217330932617,
      "activations/layer9_attention_weight_min": -50.312339782714844,
      "epoch": 8.4,
      "learning_rate": 9.677272727272726e-05,
      "loss": 2.7871,
      "step": 144650
    },
    {
      "activations/layer0_attention_weight_max": 14.591904640197754,
      "activations/layer0_attention_weight_min": -15.408029556274414,
      "activations/layer10_attention_weight_max": 39.914127349853516,
      "activations/layer10_attention_weight_min": -39.33162307739258,
      "activations/layer11_attention_weight_max": 33.37393569946289,
      "activations/layer11_attention_weight_min": -36.10359191894531,
      "activations/layer12_attention_weight_max": 21.519067764282227,
      "activations/layer12_attention_weight_min": -23.473554611206055,
      "activations/layer13_attention_weight_max": 44.874168395996094,
      "activations/layer13_attention_weight_min": -30.866615295410156,
      "activations/layer14_attention_weight_max": 43.17000198364258,
      "activations/layer14_attention_weight_min": -32.931758880615234,
      "activations/layer15_attention_weight_max": 37.973697662353516,
      "activations/layer15_attention_weight_min": -30.92929458618164,
      "activations/layer16_attention_weight_max": 31.323266983032227,
      "activations/layer16_attention_weight_min": -28.53734588623047,
      "activations/layer17_attention_weight_max": 52.780879974365234,
      "activations/layer17_attention_weight_min": -42.29243850708008,
      "activations/layer18_attention_weight_max": 46.8717155456543,
      "activations/layer18_attention_weight_min": -37.5053825378418,
      "activations/layer19_attention_weight_max": 22.511213302612305,
      "activations/layer19_attention_weight_min": -21.79793357849121,
      "activations/layer1_attention_weight_max": 16.72821044921875,
      "activations/layer1_attention_weight_min": -15.918733596801758,
      "activations/layer20_attention_weight_max": 25.290006637573242,
      "activations/layer20_attention_weight_min": -20.279800415039062,
      "activations/layer21_attention_weight_max": 48.90878677368164,
      "activations/layer21_attention_weight_min": -28.868270874023438,
      "activations/layer22_attention_weight_max": 30.358898162841797,
      "activations/layer22_attention_weight_min": -24.1630802154541,
      "activations/layer23_attention_weight_max": 35.44224548339844,
      "activations/layer23_attention_weight_min": -25.35361099243164,
      "activations/layer2_attention_weight_max": 30.171133041381836,
      "activations/layer2_attention_weight_min": -28.85878562927246,
      "activations/layer3_attention_weight_max": 89.17005920410156,
      "activations/layer3_attention_weight_min": -92.15291595458984,
      "activations/layer4_attention_weight_max": 101.42644500732422,
      "activations/layer4_attention_weight_min": -96.44247436523438,
      "activations/layer5_attention_weight_max": 82.37789916992188,
      "activations/layer5_attention_weight_min": -78.78475189208984,
      "activations/layer6_attention_weight_max": 54.50072479248047,
      "activations/layer6_attention_weight_min": -56.710845947265625,
      "activations/layer7_attention_weight_max": 77.36361694335938,
      "activations/layer7_attention_weight_min": -78.0207748413086,
      "activations/layer8_attention_weight_max": 46.8224983215332,
      "activations/layer8_attention_weight_min": -46.6212272644043,
      "activations/layer9_attention_weight_max": 52.03487777709961,
      "activations/layer9_attention_weight_min": -55.635860443115234,
      "epoch": 8.41,
      "learning_rate": 9.675378787878787e-05,
      "loss": 2.788,
      "step": 144700
    },
    {
      "activations/layer0_attention_weight_max": 15.789544105529785,
      "activations/layer0_attention_weight_min": -14.992873191833496,
      "activations/layer10_attention_weight_max": 39.017852783203125,
      "activations/layer10_attention_weight_min": -36.721988677978516,
      "activations/layer11_attention_weight_max": 35.07233428955078,
      "activations/layer11_attention_weight_min": -31.478923797607422,
      "activations/layer12_attention_weight_max": 17.40624237060547,
      "activations/layer12_attention_weight_min": -26.046871185302734,
      "activations/layer13_attention_weight_max": 47.71065139770508,
      "activations/layer13_attention_weight_min": -35.39867401123047,
      "activations/layer14_attention_weight_max": 44.09968948364258,
      "activations/layer14_attention_weight_min": -31.401458740234375,
      "activations/layer15_attention_weight_max": 36.8778190612793,
      "activations/layer15_attention_weight_min": -31.90753173828125,
      "activations/layer16_attention_weight_max": 30.236738204956055,
      "activations/layer16_attention_weight_min": -28.121492385864258,
      "activations/layer17_attention_weight_max": 53.704105377197266,
      "activations/layer17_attention_weight_min": -43.43553924560547,
      "activations/layer18_attention_weight_max": 47.27165985107422,
      "activations/layer18_attention_weight_min": -36.3979606628418,
      "activations/layer19_attention_weight_max": 22.147687911987305,
      "activations/layer19_attention_weight_min": -22.519765853881836,
      "activations/layer1_attention_weight_max": 18.597312927246094,
      "activations/layer1_attention_weight_min": -16.13654327392578,
      "activations/layer20_attention_weight_max": 22.280241012573242,
      "activations/layer20_attention_weight_min": -23.950679779052734,
      "activations/layer21_attention_weight_max": 37.852622985839844,
      "activations/layer21_attention_weight_min": -23.429391860961914,
      "activations/layer22_attention_weight_max": 30.34927749633789,
      "activations/layer22_attention_weight_min": -24.379093170166016,
      "activations/layer23_attention_weight_max": 33.58458709716797,
      "activations/layer23_attention_weight_min": -23.69863510131836,
      "activations/layer2_attention_weight_max": 30.086593627929688,
      "activations/layer2_attention_weight_min": -30.08320426940918,
      "activations/layer3_attention_weight_max": 86.59339904785156,
      "activations/layer3_attention_weight_min": -95.87014770507812,
      "activations/layer4_attention_weight_max": 99.9010238647461,
      "activations/layer4_attention_weight_min": -97.93702697753906,
      "activations/layer5_attention_weight_max": 83.46122741699219,
      "activations/layer5_attention_weight_min": -81.31585693359375,
      "activations/layer6_attention_weight_max": 58.39108657836914,
      "activations/layer6_attention_weight_min": -56.08149719238281,
      "activations/layer7_attention_weight_max": 84.46438598632812,
      "activations/layer7_attention_weight_min": -74.38179016113281,
      "activations/layer8_attention_weight_max": 47.17076873779297,
      "activations/layer8_attention_weight_min": -47.21428298950195,
      "activations/layer9_attention_weight_max": 63.24342727661133,
      "activations/layer9_attention_weight_min": -50.8207893371582,
      "epoch": 8.41,
      "learning_rate": 9.673484848484847e-05,
      "loss": 2.7904,
      "step": 144750
    },
    {
      "activations/layer0_attention_weight_max": 16.25763511657715,
      "activations/layer0_attention_weight_min": -15.32066822052002,
      "activations/layer10_attention_weight_max": 36.0378303527832,
      "activations/layer10_attention_weight_min": -34.472930908203125,
      "activations/layer11_attention_weight_max": 29.927997589111328,
      "activations/layer11_attention_weight_min": -31.71102523803711,
      "activations/layer12_attention_weight_max": 18.8563289642334,
      "activations/layer12_attention_weight_min": -26.18071746826172,
      "activations/layer13_attention_weight_max": 38.56464385986328,
      "activations/layer13_attention_weight_min": -30.844684600830078,
      "activations/layer14_attention_weight_max": 36.141719818115234,
      "activations/layer14_attention_weight_min": -31.363143920898438,
      "activations/layer15_attention_weight_max": 33.0558967590332,
      "activations/layer15_attention_weight_min": -29.64191436767578,
      "activations/layer16_attention_weight_max": 31.005138397216797,
      "activations/layer16_attention_weight_min": -30.07327651977539,
      "activations/layer17_attention_weight_max": 54.66352462768555,
      "activations/layer17_attention_weight_min": -44.81866455078125,
      "activations/layer18_attention_weight_max": 46.94917678833008,
      "activations/layer18_attention_weight_min": -37.17610168457031,
      "activations/layer19_attention_weight_max": 20.379850387573242,
      "activations/layer19_attention_weight_min": -21.054401397705078,
      "activations/layer1_attention_weight_max": 17.611614227294922,
      "activations/layer1_attention_weight_min": -14.594157218933105,
      "activations/layer20_attention_weight_max": 22.946971893310547,
      "activations/layer20_attention_weight_min": -26.31427001953125,
      "activations/layer21_attention_weight_max": 41.43659591674805,
      "activations/layer21_attention_weight_min": -28.937973022460938,
      "activations/layer22_attention_weight_max": 32.444427490234375,
      "activations/layer22_attention_weight_min": -24.91242790222168,
      "activations/layer23_attention_weight_max": 32.37653732299805,
      "activations/layer23_attention_weight_min": -22.03107261657715,
      "activations/layer2_attention_weight_max": 30.853824615478516,
      "activations/layer2_attention_weight_min": -29.129470825195312,
      "activations/layer3_attention_weight_max": 87.579345703125,
      "activations/layer3_attention_weight_min": -90.60697174072266,
      "activations/layer4_attention_weight_max": 97.24940490722656,
      "activations/layer4_attention_weight_min": -92.54046630859375,
      "activations/layer5_attention_weight_max": 79.40786743164062,
      "activations/layer5_attention_weight_min": -74.27851104736328,
      "activations/layer6_attention_weight_max": 53.44043731689453,
      "activations/layer6_attention_weight_min": -49.957862854003906,
      "activations/layer7_attention_weight_max": 75.0447006225586,
      "activations/layer7_attention_weight_min": -71.28860473632812,
      "activations/layer8_attention_weight_max": 42.22356033325195,
      "activations/layer8_attention_weight_min": -44.464622497558594,
      "activations/layer9_attention_weight_max": 49.818546295166016,
      "activations/layer9_attention_weight_min": -50.32108688354492,
      "epoch": 8.41,
      "learning_rate": 9.671590909090908e-05,
      "loss": 2.8049,
      "step": 144800
    },
    {
      "activations/layer0_attention_weight_max": 14.6886625289917,
      "activations/layer0_attention_weight_min": -14.952192306518555,
      "activations/layer10_attention_weight_max": 45.5764045715332,
      "activations/layer10_attention_weight_min": -42.57685470581055,
      "activations/layer11_attention_weight_max": 38.228607177734375,
      "activations/layer11_attention_weight_min": -33.87337112426758,
      "activations/layer12_attention_weight_max": 20.032672882080078,
      "activations/layer12_attention_weight_min": -22.66339111328125,
      "activations/layer13_attention_weight_max": 61.810768127441406,
      "activations/layer13_attention_weight_min": -34.82807922363281,
      "activations/layer14_attention_weight_max": 83.41194152832031,
      "activations/layer14_attention_weight_min": -40.396080017089844,
      "activations/layer15_attention_weight_max": 64.40440368652344,
      "activations/layer15_attention_weight_min": -31.670063018798828,
      "activations/layer16_attention_weight_max": 32.35042953491211,
      "activations/layer16_attention_weight_min": -28.202062606811523,
      "activations/layer17_attention_weight_max": 56.51588439941406,
      "activations/layer17_attention_weight_min": -47.60202407836914,
      "activations/layer18_attention_weight_max": 57.102359771728516,
      "activations/layer18_attention_weight_min": -43.08245086669922,
      "activations/layer19_attention_weight_max": 23.494670867919922,
      "activations/layer19_attention_weight_min": -23.340553283691406,
      "activations/layer1_attention_weight_max": 17.309005737304688,
      "activations/layer1_attention_weight_min": -15.843563079833984,
      "activations/layer20_attention_weight_max": 27.621498107910156,
      "activations/layer20_attention_weight_min": -21.49839973449707,
      "activations/layer21_attention_weight_max": 50.53971481323242,
      "activations/layer21_attention_weight_min": -26.661779403686523,
      "activations/layer22_attention_weight_max": 39.3671760559082,
      "activations/layer22_attention_weight_min": -25.066165924072266,
      "activations/layer23_attention_weight_max": 41.96350860595703,
      "activations/layer23_attention_weight_min": -22.29494285583496,
      "activations/layer2_attention_weight_max": 31.77339744567871,
      "activations/layer2_attention_weight_min": -29.210819244384766,
      "activations/layer3_attention_weight_max": 90.75865173339844,
      "activations/layer3_attention_weight_min": -94.60546875,
      "activations/layer4_attention_weight_max": 102.24859619140625,
      "activations/layer4_attention_weight_min": -99.19515991210938,
      "activations/layer5_attention_weight_max": 84.86288452148438,
      "activations/layer5_attention_weight_min": -80.67423248291016,
      "activations/layer6_attention_weight_max": 58.66218185424805,
      "activations/layer6_attention_weight_min": -55.64881134033203,
      "activations/layer7_attention_weight_max": 91.5409164428711,
      "activations/layer7_attention_weight_min": -77.1966552734375,
      "activations/layer8_attention_weight_max": 54.383636474609375,
      "activations/layer8_attention_weight_min": -46.8297119140625,
      "activations/layer9_attention_weight_max": 74.07177734375,
      "activations/layer9_attention_weight_min": -52.2806510925293,
      "epoch": 8.42,
      "learning_rate": 9.669696969696969e-05,
      "loss": 2.8146,
      "step": 144850
    },
    {
      "activations/layer0_attention_weight_max": 16.100788116455078,
      "activations/layer0_attention_weight_min": -14.896500587463379,
      "activations/layer10_attention_weight_max": 38.9595947265625,
      "activations/layer10_attention_weight_min": -39.02237319946289,
      "activations/layer11_attention_weight_max": 32.37305450439453,
      "activations/layer11_attention_weight_min": -32.08726501464844,
      "activations/layer12_attention_weight_max": 17.41773796081543,
      "activations/layer12_attention_weight_min": -27.1474552154541,
      "activations/layer13_attention_weight_max": 36.147560119628906,
      "activations/layer13_attention_weight_min": -30.191898345947266,
      "activations/layer14_attention_weight_max": 48.373680114746094,
      "activations/layer14_attention_weight_min": -32.392295837402344,
      "activations/layer15_attention_weight_max": 38.53314208984375,
      "activations/layer15_attention_weight_min": -29.711101531982422,
      "activations/layer16_attention_weight_max": 29.610092163085938,
      "activations/layer16_attention_weight_min": -27.40028190612793,
      "activations/layer17_attention_weight_max": 51.71769714355469,
      "activations/layer17_attention_weight_min": -45.22849655151367,
      "activations/layer18_attention_weight_max": 44.111324310302734,
      "activations/layer18_attention_weight_min": -40.02281951904297,
      "activations/layer19_attention_weight_max": 22.24044418334961,
      "activations/layer19_attention_weight_min": -21.606338500976562,
      "activations/layer1_attention_weight_max": 17.466033935546875,
      "activations/layer1_attention_weight_min": -17.698026657104492,
      "activations/layer20_attention_weight_max": 21.37977409362793,
      "activations/layer20_attention_weight_min": -21.42011260986328,
      "activations/layer21_attention_weight_max": 37.0015869140625,
      "activations/layer21_attention_weight_min": -24.719377517700195,
      "activations/layer22_attention_weight_max": 27.696945190429688,
      "activations/layer22_attention_weight_min": -23.90821647644043,
      "activations/layer23_attention_weight_max": 32.143577575683594,
      "activations/layer23_attention_weight_min": -21.457958221435547,
      "activations/layer2_attention_weight_max": 32.34974670410156,
      "activations/layer2_attention_weight_min": -30.88818359375,
      "activations/layer3_attention_weight_max": 90.52960968017578,
      "activations/layer3_attention_weight_min": -93.34446716308594,
      "activations/layer4_attention_weight_max": 102.13863372802734,
      "activations/layer4_attention_weight_min": -97.5224380493164,
      "activations/layer5_attention_weight_max": 83.38920593261719,
      "activations/layer5_attention_weight_min": -78.45579528808594,
      "activations/layer6_attention_weight_max": 56.114959716796875,
      "activations/layer6_attention_weight_min": -50.70769119262695,
      "activations/layer7_attention_weight_max": 75.72058868408203,
      "activations/layer7_attention_weight_min": -72.80156707763672,
      "activations/layer8_attention_weight_max": 45.802249908447266,
      "activations/layer8_attention_weight_min": -48.5799674987793,
      "activations/layer9_attention_weight_max": 56.09975814819336,
      "activations/layer9_attention_weight_min": -51.9403076171875,
      "epoch": 8.42,
      "learning_rate": 9.667803030303029e-05,
      "loss": 2.8077,
      "step": 144900
    },
    {
      "activations/layer0_attention_weight_max": 14.300954818725586,
      "activations/layer0_attention_weight_min": -15.435233116149902,
      "activations/layer10_attention_weight_max": 34.6957893371582,
      "activations/layer10_attention_weight_min": -34.56535720825195,
      "activations/layer11_attention_weight_max": 28.213722229003906,
      "activations/layer11_attention_weight_min": -31.1331844329834,
      "activations/layer12_attention_weight_max": 16.449434280395508,
      "activations/layer12_attention_weight_min": -25.48618507385254,
      "activations/layer13_attention_weight_max": 39.73775863647461,
      "activations/layer13_attention_weight_min": -27.278667449951172,
      "activations/layer14_attention_weight_max": 36.30988693237305,
      "activations/layer14_attention_weight_min": -32.00391387939453,
      "activations/layer15_attention_weight_max": 34.11874008178711,
      "activations/layer15_attention_weight_min": -28.879724502563477,
      "activations/layer16_attention_weight_max": 28.397781372070312,
      "activations/layer16_attention_weight_min": -27.114906311035156,
      "activations/layer17_attention_weight_max": 52.158634185791016,
      "activations/layer17_attention_weight_min": -42.154327392578125,
      "activations/layer18_attention_weight_max": 43.77220916748047,
      "activations/layer18_attention_weight_min": -37.92283248901367,
      "activations/layer19_attention_weight_max": 19.653709411621094,
      "activations/layer19_attention_weight_min": -21.85609245300293,
      "activations/layer1_attention_weight_max": 16.610397338867188,
      "activations/layer1_attention_weight_min": -16.824983596801758,
      "activations/layer20_attention_weight_max": 21.285781860351562,
      "activations/layer20_attention_weight_min": -27.01354217529297,
      "activations/layer21_attention_weight_max": 34.262474060058594,
      "activations/layer21_attention_weight_min": -22.5481014251709,
      "activations/layer22_attention_weight_max": 29.928749084472656,
      "activations/layer22_attention_weight_min": -24.830718994140625,
      "activations/layer23_attention_weight_max": 35.257591247558594,
      "activations/layer23_attention_weight_min": -20.464933395385742,
      "activations/layer2_attention_weight_max": 30.23463249206543,
      "activations/layer2_attention_weight_min": -29.723041534423828,
      "activations/layer3_attention_weight_max": 95.37753295898438,
      "activations/layer3_attention_weight_min": -91.27310943603516,
      "activations/layer4_attention_weight_max": 101.7846908569336,
      "activations/layer4_attention_weight_min": -96.16246032714844,
      "activations/layer5_attention_weight_max": 85.29827880859375,
      "activations/layer5_attention_weight_min": -77.35792541503906,
      "activations/layer6_attention_weight_max": 55.618980407714844,
      "activations/layer6_attention_weight_min": -53.37171173095703,
      "activations/layer7_attention_weight_max": 76.90078735351562,
      "activations/layer7_attention_weight_min": -73.3780517578125,
      "activations/layer8_attention_weight_max": 42.474220275878906,
      "activations/layer8_attention_weight_min": -43.975830078125,
      "activations/layer9_attention_weight_max": 48.67194747924805,
      "activations/layer9_attention_weight_min": -47.8663215637207,
      "epoch": 8.42,
      "learning_rate": 9.665909090909091e-05,
      "loss": 2.7973,
      "step": 144950
    },
    {
      "activations/layer0_attention_weight_max": 15.744222640991211,
      "activations/layer0_attention_weight_min": -14.933978080749512,
      "activations/layer10_attention_weight_max": 40.98664093017578,
      "activations/layer10_attention_weight_min": -37.49993896484375,
      "activations/layer11_attention_weight_max": 32.17702102661133,
      "activations/layer11_attention_weight_min": -32.20500183105469,
      "activations/layer12_attention_weight_max": 21.685623168945312,
      "activations/layer12_attention_weight_min": -25.203205108642578,
      "activations/layer13_attention_weight_max": 45.23666763305664,
      "activations/layer13_attention_weight_min": -33.53453826904297,
      "activations/layer14_attention_weight_max": 46.51335144042969,
      "activations/layer14_attention_weight_min": -30.02362060546875,
      "activations/layer15_attention_weight_max": 37.61763381958008,
      "activations/layer15_attention_weight_min": -29.0642147064209,
      "activations/layer16_attention_weight_max": 35.95832061767578,
      "activations/layer16_attention_weight_min": -28.98503303527832,
      "activations/layer17_attention_weight_max": 57.14458465576172,
      "activations/layer17_attention_weight_min": -44.82344055175781,
      "activations/layer18_attention_weight_max": 54.34549331665039,
      "activations/layer18_attention_weight_min": -37.86137390136719,
      "activations/layer19_attention_weight_max": 21.9742431640625,
      "activations/layer19_attention_weight_min": -20.713308334350586,
      "activations/layer1_attention_weight_max": 17.72884750366211,
      "activations/layer1_attention_weight_min": -15.082275390625,
      "activations/layer20_attention_weight_max": 25.52094078063965,
      "activations/layer20_attention_weight_min": -20.21955680847168,
      "activations/layer21_attention_weight_max": 39.806724548339844,
      "activations/layer21_attention_weight_min": -21.460407257080078,
      "activations/layer22_attention_weight_max": 33.62405014038086,
      "activations/layer22_attention_weight_min": -23.37151336669922,
      "activations/layer23_attention_weight_max": 40.219276428222656,
      "activations/layer23_attention_weight_min": -22.385101318359375,
      "activations/layer2_attention_weight_max": 31.47457504272461,
      "activations/layer2_attention_weight_min": -29.396377563476562,
      "activations/layer3_attention_weight_max": 88.8060531616211,
      "activations/layer3_attention_weight_min": -89.74320983886719,
      "activations/layer4_attention_weight_max": 100.83197021484375,
      "activations/layer4_attention_weight_min": -94.40953826904297,
      "activations/layer5_attention_weight_max": 80.56694030761719,
      "activations/layer5_attention_weight_min": -77.50604248046875,
      "activations/layer6_attention_weight_max": 54.8929328918457,
      "activations/layer6_attention_weight_min": -52.246620178222656,
      "activations/layer7_attention_weight_max": 75.17833709716797,
      "activations/layer7_attention_weight_min": -72.87339782714844,
      "activations/layer8_attention_weight_max": 46.95381546020508,
      "activations/layer8_attention_weight_min": -47.131473541259766,
      "activations/layer9_attention_weight_max": 52.44456100463867,
      "activations/layer9_attention_weight_min": -53.04075241088867,
      "epoch": 8.43,
      "learning_rate": 9.664015151515151e-05,
      "loss": 2.8033,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.5144,
      "eval_samples_per_second": 504.323,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.5144,
      "eval_openwebtext_samples_per_second": 504.323,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 1.9624,
      "eval_wikitext_samples_per_second": 232.372,
      "step": 145000
    },
    {
      "epoch": 8.43,
      "eval_lambada_loss": 2.66015625,
      "eval_lambada_ppl": 14.298523068373607,
      "eval_lambada_runtime": 9.5396,
      "eval_lambada_samples_per_second": 510.399,
      "step": 145000
    },
    {
      "activations/layer0_attention_weight_max": 14.682269096374512,
      "activations/layer0_attention_weight_min": -14.916037559509277,
      "activations/layer10_attention_weight_max": 37.18000793457031,
      "activations/layer10_attention_weight_min": -36.1541748046875,
      "activations/layer11_attention_weight_max": 33.503517150878906,
      "activations/layer11_attention_weight_min": -31.74575424194336,
      "activations/layer12_attention_weight_max": 23.76727867126465,
      "activations/layer12_attention_weight_min": -22.49972152709961,
      "activations/layer13_attention_weight_max": 49.993980407714844,
      "activations/layer13_attention_weight_min": -34.09758758544922,
      "activations/layer14_attention_weight_max": 51.673213958740234,
      "activations/layer14_attention_weight_min": -31.51458740234375,
      "activations/layer15_attention_weight_max": 39.63682174682617,
      "activations/layer15_attention_weight_min": -28.568214416503906,
      "activations/layer16_attention_weight_max": 34.46683120727539,
      "activations/layer16_attention_weight_min": -28.207012176513672,
      "activations/layer17_attention_weight_max": 58.58427810668945,
      "activations/layer17_attention_weight_min": -43.65974426269531,
      "activations/layer18_attention_weight_max": 55.64020919799805,
      "activations/layer18_attention_weight_min": -37.96022033691406,
      "activations/layer19_attention_weight_max": 23.11034393310547,
      "activations/layer19_attention_weight_min": -21.884998321533203,
      "activations/layer1_attention_weight_max": 17.593278884887695,
      "activations/layer1_attention_weight_min": -15.252476692199707,
      "activations/layer20_attention_weight_max": 25.139652252197266,
      "activations/layer20_attention_weight_min": -21.10934829711914,
      "activations/layer21_attention_weight_max": 46.365028381347656,
      "activations/layer21_attention_weight_min": -21.386972427368164,
      "activations/layer22_attention_weight_max": 36.92210006713867,
      "activations/layer22_attention_weight_min": -27.576616287231445,
      "activations/layer23_attention_weight_max": 41.12693405151367,
      "activations/layer23_attention_weight_min": -21.718727111816406,
      "activations/layer2_attention_weight_max": 32.083465576171875,
      "activations/layer2_attention_weight_min": -30.24803924560547,
      "activations/layer3_attention_weight_max": 89.50216674804688,
      "activations/layer3_attention_weight_min": -91.52144622802734,
      "activations/layer4_attention_weight_max": 104.41108703613281,
      "activations/layer4_attention_weight_min": -100.00848388671875,
      "activations/layer5_attention_weight_max": 84.9830551147461,
      "activations/layer5_attention_weight_min": -80.60926818847656,
      "activations/layer6_attention_weight_max": 56.17289352416992,
      "activations/layer6_attention_weight_min": -54.607872009277344,
      "activations/layer7_attention_weight_max": 78.61469268798828,
      "activations/layer7_attention_weight_min": -71.9522476196289,
      "activations/layer8_attention_weight_max": 45.06010437011719,
      "activations/layer8_attention_weight_min": -48.91057205200195,
      "activations/layer9_attention_weight_max": 51.183834075927734,
      "activations/layer9_attention_weight_min": -51.533416748046875,
      "epoch": 8.43,
      "learning_rate": 9.662121212121211e-05,
      "loss": 2.7907,
      "step": 145050
    },
    {
      "activations/layer0_attention_weight_max": 14.886697769165039,
      "activations/layer0_attention_weight_min": -15.096318244934082,
      "activations/layer10_attention_weight_max": 40.28841781616211,
      "activations/layer10_attention_weight_min": -40.0185432434082,
      "activations/layer11_attention_weight_max": 32.108516693115234,
      "activations/layer11_attention_weight_min": -31.802579879760742,
      "activations/layer12_attention_weight_max": 19.84543228149414,
      "activations/layer12_attention_weight_min": -27.19852638244629,
      "activations/layer13_attention_weight_max": 42.03887176513672,
      "activations/layer13_attention_weight_min": -29.289573669433594,
      "activations/layer14_attention_weight_max": 40.190696716308594,
      "activations/layer14_attention_weight_min": -31.0833683013916,
      "activations/layer15_attention_weight_max": 37.44236755371094,
      "activations/layer15_attention_weight_min": -29.613723754882812,
      "activations/layer16_attention_weight_max": 34.52964782714844,
      "activations/layer16_attention_weight_min": -27.17790412902832,
      "activations/layer17_attention_weight_max": 52.070831298828125,
      "activations/layer17_attention_weight_min": -42.11733627319336,
      "activations/layer18_attention_weight_max": 49.62909698486328,
      "activations/layer18_attention_weight_min": -37.80903625488281,
      "activations/layer19_attention_weight_max": 19.594114303588867,
      "activations/layer19_attention_weight_min": -20.384164810180664,
      "activations/layer1_attention_weight_max": 16.00103759765625,
      "activations/layer1_attention_weight_min": -15.952136039733887,
      "activations/layer20_attention_weight_max": 23.095855712890625,
      "activations/layer20_attention_weight_min": -19.61296844482422,
      "activations/layer21_attention_weight_max": 42.17105484008789,
      "activations/layer21_attention_weight_min": -22.57264518737793,
      "activations/layer22_attention_weight_max": 31.872636795043945,
      "activations/layer22_attention_weight_min": -24.97603416442871,
      "activations/layer23_attention_weight_max": 34.98081588745117,
      "activations/layer23_attention_weight_min": -21.98853302001953,
      "activations/layer2_attention_weight_max": 29.712501525878906,
      "activations/layer2_attention_weight_min": -28.579465866088867,
      "activations/layer3_attention_weight_max": 91.71864318847656,
      "activations/layer3_attention_weight_min": -90.24329376220703,
      "activations/layer4_attention_weight_max": 107.89448547363281,
      "activations/layer4_attention_weight_min": -105.13407135009766,
      "activations/layer5_attention_weight_max": 85.56105041503906,
      "activations/layer5_attention_weight_min": -85.58220672607422,
      "activations/layer6_attention_weight_max": 58.876182556152344,
      "activations/layer6_attention_weight_min": -61.22904586791992,
      "activations/layer7_attention_weight_max": 81.95601654052734,
      "activations/layer7_attention_weight_min": -79.44256591796875,
      "activations/layer8_attention_weight_max": 45.292327880859375,
      "activations/layer8_attention_weight_min": -48.68193054199219,
      "activations/layer9_attention_weight_max": 52.16477584838867,
      "activations/layer9_attention_weight_min": -52.17683029174805,
      "epoch": 8.43,
      "learning_rate": 9.660227272727273e-05,
      "loss": 2.8071,
      "step": 145100
    },
    {
      "activations/layer0_attention_weight_max": 14.825409889221191,
      "activations/layer0_attention_weight_min": -14.823156356811523,
      "activations/layer10_attention_weight_max": 39.2957649230957,
      "activations/layer10_attention_weight_min": -37.07582092285156,
      "activations/layer11_attention_weight_max": 32.38264465332031,
      "activations/layer11_attention_weight_min": -30.600486755371094,
      "activations/layer12_attention_weight_max": 23.577808380126953,
      "activations/layer12_attention_weight_min": -23.98391342163086,
      "activations/layer13_attention_weight_max": 41.40882110595703,
      "activations/layer13_attention_weight_min": -32.887935638427734,
      "activations/layer14_attention_weight_max": 45.074378967285156,
      "activations/layer14_attention_weight_min": -31.478696823120117,
      "activations/layer15_attention_weight_max": 38.48616027832031,
      "activations/layer15_attention_weight_min": -29.482988357543945,
      "activations/layer16_attention_weight_max": 28.38283920288086,
      "activations/layer16_attention_weight_min": -27.44512367248535,
      "activations/layer17_attention_weight_max": 61.47795867919922,
      "activations/layer17_attention_weight_min": -43.8249397277832,
      "activations/layer18_attention_weight_max": 48.6638298034668,
      "activations/layer18_attention_weight_min": -37.92350769042969,
      "activations/layer19_attention_weight_max": 24.36698341369629,
      "activations/layer19_attention_weight_min": -21.94343376159668,
      "activations/layer1_attention_weight_max": 16.483211517333984,
      "activations/layer1_attention_weight_min": -15.357407569885254,
      "activations/layer20_attention_weight_max": 23.278715133666992,
      "activations/layer20_attention_weight_min": -19.431072235107422,
      "activations/layer21_attention_weight_max": 39.800010681152344,
      "activations/layer21_attention_weight_min": -22.600177764892578,
      "activations/layer22_attention_weight_max": 30.450748443603516,
      "activations/layer22_attention_weight_min": -25.366533279418945,
      "activations/layer23_attention_weight_max": 36.3568115234375,
      "activations/layer23_attention_weight_min": -21.67469596862793,
      "activations/layer2_attention_weight_max": 29.59439468383789,
      "activations/layer2_attention_weight_min": -29.493289947509766,
      "activations/layer3_attention_weight_max": 85.107421875,
      "activations/layer3_attention_weight_min": -89.77296447753906,
      "activations/layer4_attention_weight_max": 100.63849639892578,
      "activations/layer4_attention_weight_min": -95.88518524169922,
      "activations/layer5_attention_weight_max": 80.49203491210938,
      "activations/layer5_attention_weight_min": -77.09281921386719,
      "activations/layer6_attention_weight_max": 54.47907257080078,
      "activations/layer6_attention_weight_min": -54.99827194213867,
      "activations/layer7_attention_weight_max": 78.14645385742188,
      "activations/layer7_attention_weight_min": -75.80911254882812,
      "activations/layer8_attention_weight_max": 44.72269058227539,
      "activations/layer8_attention_weight_min": -49.282264709472656,
      "activations/layer9_attention_weight_max": 48.63823318481445,
      "activations/layer9_attention_weight_min": -51.27296829223633,
      "epoch": 8.43,
      "learning_rate": 9.658371212121212e-05,
      "loss": 2.8003,
      "step": 145150
    },
    {
      "activations/layer0_attention_weight_max": 15.362009048461914,
      "activations/layer0_attention_weight_min": -15.159524917602539,
      "activations/layer10_attention_weight_max": 34.89053726196289,
      "activations/layer10_attention_weight_min": -34.15510559082031,
      "activations/layer11_attention_weight_max": 30.31563377380371,
      "activations/layer11_attention_weight_min": -30.468551635742188,
      "activations/layer12_attention_weight_max": 17.54458236694336,
      "activations/layer12_attention_weight_min": -23.88788414001465,
      "activations/layer13_attention_weight_max": 39.88762664794922,
      "activations/layer13_attention_weight_min": -32.59272384643555,
      "activations/layer14_attention_weight_max": 42.73258590698242,
      "activations/layer14_attention_weight_min": -30.074172973632812,
      "activations/layer15_attention_weight_max": 31.299358367919922,
      "activations/layer15_attention_weight_min": -28.1849308013916,
      "activations/layer16_attention_weight_max": 30.78386688232422,
      "activations/layer16_attention_weight_min": -27.715927124023438,
      "activations/layer17_attention_weight_max": 50.58121109008789,
      "activations/layer17_attention_weight_min": -44.24642562866211,
      "activations/layer18_attention_weight_max": 42.475624084472656,
      "activations/layer18_attention_weight_min": -37.50944900512695,
      "activations/layer19_attention_weight_max": 24.453039169311523,
      "activations/layer19_attention_weight_min": -22.974706649780273,
      "activations/layer1_attention_weight_max": 17.160036087036133,
      "activations/layer1_attention_weight_min": -16.512290954589844,
      "activations/layer20_attention_weight_max": 20.364288330078125,
      "activations/layer20_attention_weight_min": -23.764127731323242,
      "activations/layer21_attention_weight_max": 34.700687408447266,
      "activations/layer21_attention_weight_min": -24.479949951171875,
      "activations/layer22_attention_weight_max": 30.230833053588867,
      "activations/layer22_attention_weight_min": -26.5667724609375,
      "activations/layer23_attention_weight_max": 34.93915557861328,
      "activations/layer23_attention_weight_min": -24.910388946533203,
      "activations/layer2_attention_weight_max": 30.749263763427734,
      "activations/layer2_attention_weight_min": -29.93378257751465,
      "activations/layer3_attention_weight_max": 86.99638366699219,
      "activations/layer3_attention_weight_min": -91.36380767822266,
      "activations/layer4_attention_weight_max": 100.25331115722656,
      "activations/layer4_attention_weight_min": -101.63835906982422,
      "activations/layer5_attention_weight_max": 83.54428100585938,
      "activations/layer5_attention_weight_min": -82.21405792236328,
      "activations/layer6_attention_weight_max": 59.881473541259766,
      "activations/layer6_attention_weight_min": -56.09882354736328,
      "activations/layer7_attention_weight_max": 77.99281311035156,
      "activations/layer7_attention_weight_min": -81.15850830078125,
      "activations/layer8_attention_weight_max": 44.031227111816406,
      "activations/layer8_attention_weight_min": -46.92701721191406,
      "activations/layer9_attention_weight_max": 53.91273880004883,
      "activations/layer9_attention_weight_min": -50.90364074707031,
      "epoch": 8.44,
      "learning_rate": 9.656477272727273e-05,
      "loss": 2.8039,
      "step": 145200
    },
    {
      "activations/layer0_attention_weight_max": 14.449518203735352,
      "activations/layer0_attention_weight_min": -15.013647079467773,
      "activations/layer10_attention_weight_max": 40.13050079345703,
      "activations/layer10_attention_weight_min": -38.73001480102539,
      "activations/layer11_attention_weight_max": 34.200164794921875,
      "activations/layer11_attention_weight_min": -34.94682693481445,
      "activations/layer12_attention_weight_max": 20.419864654541016,
      "activations/layer12_attention_weight_min": -25.338050842285156,
      "activations/layer13_attention_weight_max": 36.67621612548828,
      "activations/layer13_attention_weight_min": -28.49437141418457,
      "activations/layer14_attention_weight_max": 36.4475212097168,
      "activations/layer14_attention_weight_min": -29.989389419555664,
      "activations/layer15_attention_weight_max": 33.79084396362305,
      "activations/layer15_attention_weight_min": -30.461288452148438,
      "activations/layer16_attention_weight_max": 33.41178894042969,
      "activations/layer16_attention_weight_min": -27.762767791748047,
      "activations/layer17_attention_weight_max": 51.85170364379883,
      "activations/layer17_attention_weight_min": -43.98038101196289,
      "activations/layer18_attention_weight_max": 44.520355224609375,
      "activations/layer18_attention_weight_min": -35.881160736083984,
      "activations/layer19_attention_weight_max": 19.925188064575195,
      "activations/layer19_attention_weight_min": -21.41987419128418,
      "activations/layer1_attention_weight_max": 16.49629783630371,
      "activations/layer1_attention_weight_min": -16.348012924194336,
      "activations/layer20_attention_weight_max": 20.98525047302246,
      "activations/layer20_attention_weight_min": -21.058683395385742,
      "activations/layer21_attention_weight_max": 38.39381790161133,
      "activations/layer21_attention_weight_min": -23.201095581054688,
      "activations/layer22_attention_weight_max": 33.56898498535156,
      "activations/layer22_attention_weight_min": -24.98125457763672,
      "activations/layer23_attention_weight_max": 36.795711517333984,
      "activations/layer23_attention_weight_min": -22.521940231323242,
      "activations/layer2_attention_weight_max": 32.352684020996094,
      "activations/layer2_attention_weight_min": -29.864185333251953,
      "activations/layer3_attention_weight_max": 94.30518341064453,
      "activations/layer3_attention_weight_min": -92.28495025634766,
      "activations/layer4_attention_weight_max": 102.2667236328125,
      "activations/layer4_attention_weight_min": -94.0262680053711,
      "activations/layer5_attention_weight_max": 82.4367446899414,
      "activations/layer5_attention_weight_min": -76.51422119140625,
      "activations/layer6_attention_weight_max": 55.295074462890625,
      "activations/layer6_attention_weight_min": -52.34785842895508,
      "activations/layer7_attention_weight_max": 81.55538940429688,
      "activations/layer7_attention_weight_min": -79.18067169189453,
      "activations/layer8_attention_weight_max": 47.10989761352539,
      "activations/layer8_attention_weight_min": -49.351993560791016,
      "activations/layer9_attention_weight_max": 58.43894577026367,
      "activations/layer9_attention_weight_min": -54.373931884765625,
      "epoch": 8.44,
      "learning_rate": 9.654583333333332e-05,
      "loss": 2.8014,
      "step": 145250
    },
    {
      "activations/layer0_attention_weight_max": 15.271430015563965,
      "activations/layer0_attention_weight_min": -14.805069923400879,
      "activations/layer10_attention_weight_max": 33.596641540527344,
      "activations/layer10_attention_weight_min": -35.152435302734375,
      "activations/layer11_attention_weight_max": 28.628782272338867,
      "activations/layer11_attention_weight_min": -30.375967025756836,
      "activations/layer12_attention_weight_max": 18.749343872070312,
      "activations/layer12_attention_weight_min": -24.827253341674805,
      "activations/layer13_attention_weight_max": 41.27616500854492,
      "activations/layer13_attention_weight_min": -31.971721649169922,
      "activations/layer14_attention_weight_max": 36.943782806396484,
      "activations/layer14_attention_weight_min": -31.68641471862793,
      "activations/layer15_attention_weight_max": 36.49006271362305,
      "activations/layer15_attention_weight_min": -29.040910720825195,
      "activations/layer16_attention_weight_max": 32.046329498291016,
      "activations/layer16_attention_weight_min": -28.615633010864258,
      "activations/layer17_attention_weight_max": 51.575923919677734,
      "activations/layer17_attention_weight_min": -44.83043670654297,
      "activations/layer18_attention_weight_max": 48.23347091674805,
      "activations/layer18_attention_weight_min": -37.041744232177734,
      "activations/layer19_attention_weight_max": 22.886316299438477,
      "activations/layer19_attention_weight_min": -22.534637451171875,
      "activations/layer1_attention_weight_max": 17.633255004882812,
      "activations/layer1_attention_weight_min": -16.018329620361328,
      "activations/layer20_attention_weight_max": 21.934528350830078,
      "activations/layer20_attention_weight_min": -24.215696334838867,
      "activations/layer21_attention_weight_max": 40.94009017944336,
      "activations/layer21_attention_weight_min": -22.725561141967773,
      "activations/layer22_attention_weight_max": 36.021297454833984,
      "activations/layer22_attention_weight_min": -26.76431655883789,
      "activations/layer23_attention_weight_max": 41.1245231628418,
      "activations/layer23_attention_weight_min": -22.163360595703125,
      "activations/layer2_attention_weight_max": 30.74056053161621,
      "activations/layer2_attention_weight_min": -28.697011947631836,
      "activations/layer3_attention_weight_max": 85.76166534423828,
      "activations/layer3_attention_weight_min": -86.41218566894531,
      "activations/layer4_attention_weight_max": 94.94658660888672,
      "activations/layer4_attention_weight_min": -93.95140838623047,
      "activations/layer5_attention_weight_max": 78.31258392333984,
      "activations/layer5_attention_weight_min": -74.79811096191406,
      "activations/layer6_attention_weight_max": 51.46331024169922,
      "activations/layer6_attention_weight_min": -52.43582534790039,
      "activations/layer7_attention_weight_max": 71.98020935058594,
      "activations/layer7_attention_weight_min": -70.22570037841797,
      "activations/layer8_attention_weight_max": 40.69734573364258,
      "activations/layer8_attention_weight_min": -42.41090774536133,
      "activations/layer9_attention_weight_max": 46.62986373901367,
      "activations/layer9_attention_weight_min": -47.965816497802734,
      "epoch": 8.44,
      "learning_rate": 9.652689393939393e-05,
      "loss": 2.8129,
      "step": 145300
    },
    {
      "activations/layer0_attention_weight_max": 14.867152214050293,
      "activations/layer0_attention_weight_min": -15.173617362976074,
      "activations/layer10_attention_weight_max": 34.404415130615234,
      "activations/layer10_attention_weight_min": -34.801727294921875,
      "activations/layer11_attention_weight_max": 30.080860137939453,
      "activations/layer11_attention_weight_min": -30.83869171142578,
      "activations/layer12_attention_weight_max": 18.654052734375,
      "activations/layer12_attention_weight_min": -22.115821838378906,
      "activations/layer13_attention_weight_max": 42.5739860534668,
      "activations/layer13_attention_weight_min": -38.09856414794922,
      "activations/layer14_attention_weight_max": 41.00651931762695,
      "activations/layer14_attention_weight_min": -30.561338424682617,
      "activations/layer15_attention_weight_max": 32.99055480957031,
      "activations/layer15_attention_weight_min": -31.356969833374023,
      "activations/layer16_attention_weight_max": 32.52421951293945,
      "activations/layer16_attention_weight_min": -28.218976974487305,
      "activations/layer17_attention_weight_max": 53.92998504638672,
      "activations/layer17_attention_weight_min": -44.21316146850586,
      "activations/layer18_attention_weight_max": 45.7307243347168,
      "activations/layer18_attention_weight_min": -37.10248947143555,
      "activations/layer19_attention_weight_max": 23.805219650268555,
      "activations/layer19_attention_weight_min": -22.092561721801758,
      "activations/layer1_attention_weight_max": 16.648258209228516,
      "activations/layer1_attention_weight_min": -14.33779525756836,
      "activations/layer20_attention_weight_max": 24.175739288330078,
      "activations/layer20_attention_weight_min": -21.64112663269043,
      "activations/layer21_attention_weight_max": 45.472373962402344,
      "activations/layer21_attention_weight_min": -23.08466148376465,
      "activations/layer22_attention_weight_max": 30.51776885986328,
      "activations/layer22_attention_weight_min": -26.87691307067871,
      "activations/layer23_attention_weight_max": 37.29396057128906,
      "activations/layer23_attention_weight_min": -23.782806396484375,
      "activations/layer2_attention_weight_max": 30.962158203125,
      "activations/layer2_attention_weight_min": -28.412633895874023,
      "activations/layer3_attention_weight_max": 89.07408142089844,
      "activations/layer3_attention_weight_min": -88.91377258300781,
      "activations/layer4_attention_weight_max": 100.81596374511719,
      "activations/layer4_attention_weight_min": -90.21846771240234,
      "activations/layer5_attention_weight_max": 81.08853912353516,
      "activations/layer5_attention_weight_min": -76.54845428466797,
      "activations/layer6_attention_weight_max": 54.28639221191406,
      "activations/layer6_attention_weight_min": -52.79652786254883,
      "activations/layer7_attention_weight_max": 75.4625473022461,
      "activations/layer7_attention_weight_min": -70.103759765625,
      "activations/layer8_attention_weight_max": 42.5072135925293,
      "activations/layer8_attention_weight_min": -46.45694351196289,
      "activations/layer9_attention_weight_max": 51.103939056396484,
      "activations/layer9_attention_weight_min": -46.73041915893555,
      "epoch": 8.45,
      "learning_rate": 9.650795454545454e-05,
      "loss": 2.8076,
      "step": 145350
    },
    {
      "activations/layer0_attention_weight_max": 14.977848052978516,
      "activations/layer0_attention_weight_min": -14.738141059875488,
      "activations/layer10_attention_weight_max": 36.423152923583984,
      "activations/layer10_attention_weight_min": -34.03813552856445,
      "activations/layer11_attention_weight_max": 30.441755294799805,
      "activations/layer11_attention_weight_min": -31.57484245300293,
      "activations/layer12_attention_weight_max": 16.259035110473633,
      "activations/layer12_attention_weight_min": -23.78718376159668,
      "activations/layer13_attention_weight_max": 39.64107894897461,
      "activations/layer13_attention_weight_min": -29.829776763916016,
      "activations/layer14_attention_weight_max": 38.48221206665039,
      "activations/layer14_attention_weight_min": -30.250890731811523,
      "activations/layer15_attention_weight_max": 33.503719329833984,
      "activations/layer15_attention_weight_min": -28.6453857421875,
      "activations/layer16_attention_weight_max": 31.623653411865234,
      "activations/layer16_attention_weight_min": -26.503681182861328,
      "activations/layer17_attention_weight_max": 50.58233642578125,
      "activations/layer17_attention_weight_min": -42.66527557373047,
      "activations/layer18_attention_weight_max": 46.687015533447266,
      "activations/layer18_attention_weight_min": -36.0954704284668,
      "activations/layer19_attention_weight_max": 23.814743041992188,
      "activations/layer19_attention_weight_min": -21.45663833618164,
      "activations/layer1_attention_weight_max": 16.409290313720703,
      "activations/layer1_attention_weight_min": -14.590323448181152,
      "activations/layer20_attention_weight_max": 20.313297271728516,
      "activations/layer20_attention_weight_min": -21.88369369506836,
      "activations/layer21_attention_weight_max": 38.359474182128906,
      "activations/layer21_attention_weight_min": -22.58399200439453,
      "activations/layer22_attention_weight_max": 32.02043533325195,
      "activations/layer22_attention_weight_min": -25.128131866455078,
      "activations/layer23_attention_weight_max": 30.092369079589844,
      "activations/layer23_attention_weight_min": -20.436655044555664,
      "activations/layer2_attention_weight_max": 31.653743743896484,
      "activations/layer2_attention_weight_min": -30.926055908203125,
      "activations/layer3_attention_weight_max": 82.13771057128906,
      "activations/layer3_attention_weight_min": -85.98656463623047,
      "activations/layer4_attention_weight_max": 94.93728637695312,
      "activations/layer4_attention_weight_min": -89.08728790283203,
      "activations/layer5_attention_weight_max": 78.58880615234375,
      "activations/layer5_attention_weight_min": -77.02465057373047,
      "activations/layer6_attention_weight_max": 55.08544158935547,
      "activations/layer6_attention_weight_min": -51.902992248535156,
      "activations/layer7_attention_weight_max": 71.96319580078125,
      "activations/layer7_attention_weight_min": -67.82093048095703,
      "activations/layer8_attention_weight_max": 42.62733459472656,
      "activations/layer8_attention_weight_min": -46.016902923583984,
      "activations/layer9_attention_weight_max": 47.80076217651367,
      "activations/layer9_attention_weight_min": -45.911739349365234,
      "epoch": 8.45,
      "learning_rate": 9.648901515151514e-05,
      "loss": 2.8198,
      "step": 145400
    },
    {
      "activations/layer0_attention_weight_max": 14.985135078430176,
      "activations/layer0_attention_weight_min": -14.41171646118164,
      "activations/layer10_attention_weight_max": 41.810176849365234,
      "activations/layer10_attention_weight_min": -38.65725326538086,
      "activations/layer11_attention_weight_max": 35.533287048339844,
      "activations/layer11_attention_weight_min": -32.69221878051758,
      "activations/layer12_attention_weight_max": 24.67687225341797,
      "activations/layer12_attention_weight_min": -25.462806701660156,
      "activations/layer13_attention_weight_max": 44.94832992553711,
      "activations/layer13_attention_weight_min": -31.513668060302734,
      "activations/layer14_attention_weight_max": 41.85902786254883,
      "activations/layer14_attention_weight_min": -31.259035110473633,
      "activations/layer15_attention_weight_max": 39.59292984008789,
      "activations/layer15_attention_weight_min": -30.542085647583008,
      "activations/layer16_attention_weight_max": 30.01480484008789,
      "activations/layer16_attention_weight_min": -28.46336555480957,
      "activations/layer17_attention_weight_max": 58.93650436401367,
      "activations/layer17_attention_weight_min": -45.049808502197266,
      "activations/layer18_attention_weight_max": 51.54827117919922,
      "activations/layer18_attention_weight_min": -36.701778411865234,
      "activations/layer19_attention_weight_max": 25.21967124938965,
      "activations/layer19_attention_weight_min": -22.61675262451172,
      "activations/layer1_attention_weight_max": 16.17835235595703,
      "activations/layer1_attention_weight_min": -15.532209396362305,
      "activations/layer20_attention_weight_max": 24.572973251342773,
      "activations/layer20_attention_weight_min": -22.450700759887695,
      "activations/layer21_attention_weight_max": 37.89802932739258,
      "activations/layer21_attention_weight_min": -23.005298614501953,
      "activations/layer22_attention_weight_max": 32.31007385253906,
      "activations/layer22_attention_weight_min": -25.33660125732422,
      "activations/layer23_attention_weight_max": 36.18603515625,
      "activations/layer23_attention_weight_min": -22.502155303955078,
      "activations/layer2_attention_weight_max": 32.41190719604492,
      "activations/layer2_attention_weight_min": -30.269804000854492,
      "activations/layer3_attention_weight_max": 89.40135192871094,
      "activations/layer3_attention_weight_min": -93.1142349243164,
      "activations/layer4_attention_weight_max": 100.14899444580078,
      "activations/layer4_attention_weight_min": -96.04312133789062,
      "activations/layer5_attention_weight_max": 81.4393081665039,
      "activations/layer5_attention_weight_min": -79.31515502929688,
      "activations/layer6_attention_weight_max": 54.64412307739258,
      "activations/layer6_attention_weight_min": -55.6488151550293,
      "activations/layer7_attention_weight_max": 77.20223236083984,
      "activations/layer7_attention_weight_min": -76.09648132324219,
      "activations/layer8_attention_weight_max": 45.72084045410156,
      "activations/layer8_attention_weight_min": -48.32438278198242,
      "activations/layer9_attention_weight_max": 51.056549072265625,
      "activations/layer9_attention_weight_min": -54.8511848449707,
      "epoch": 8.45,
      "learning_rate": 9.647007575757575e-05,
      "loss": 2.7973,
      "step": 145450
    },
    {
      "activations/layer0_attention_weight_max": 15.29086685180664,
      "activations/layer0_attention_weight_min": -14.506040573120117,
      "activations/layer10_attention_weight_max": 38.663421630859375,
      "activations/layer10_attention_weight_min": -39.2192497253418,
      "activations/layer11_attention_weight_max": 32.67329406738281,
      "activations/layer11_attention_weight_min": -32.16804885864258,
      "activations/layer12_attention_weight_max": 18.687955856323242,
      "activations/layer12_attention_weight_min": -20.995702743530273,
      "activations/layer13_attention_weight_max": 41.6840705871582,
      "activations/layer13_attention_weight_min": -31.96395492553711,
      "activations/layer14_attention_weight_max": 42.68581771850586,
      "activations/layer14_attention_weight_min": -30.564424514770508,
      "activations/layer15_attention_weight_max": 37.28840255737305,
      "activations/layer15_attention_weight_min": -31.94212532043457,
      "activations/layer16_attention_weight_max": 35.15052032470703,
      "activations/layer16_attention_weight_min": -29.728580474853516,
      "activations/layer17_attention_weight_max": 59.78281021118164,
      "activations/layer17_attention_weight_min": -44.67699432373047,
      "activations/layer18_attention_weight_max": 49.41387176513672,
      "activations/layer18_attention_weight_min": -40.601898193359375,
      "activations/layer19_attention_weight_max": 26.896535873413086,
      "activations/layer19_attention_weight_min": -22.641563415527344,
      "activations/layer1_attention_weight_max": 18.437532424926758,
      "activations/layer1_attention_weight_min": -14.08129596710205,
      "activations/layer20_attention_weight_max": 24.088993072509766,
      "activations/layer20_attention_weight_min": -24.607898712158203,
      "activations/layer21_attention_weight_max": 40.38319778442383,
      "activations/layer21_attention_weight_min": -24.3934383392334,
      "activations/layer22_attention_weight_max": 33.26682662963867,
      "activations/layer22_attention_weight_min": -27.066791534423828,
      "activations/layer23_attention_weight_max": 39.200469970703125,
      "activations/layer23_attention_weight_min": -27.736709594726562,
      "activations/layer2_attention_weight_max": 31.098037719726562,
      "activations/layer2_attention_weight_min": -27.76937484741211,
      "activations/layer3_attention_weight_max": 86.03042602539062,
      "activations/layer3_attention_weight_min": -88.51100158691406,
      "activations/layer4_attention_weight_max": 99.73743438720703,
      "activations/layer4_attention_weight_min": -95.3417739868164,
      "activations/layer5_attention_weight_max": 82.5015640258789,
      "activations/layer5_attention_weight_min": -79.30357360839844,
      "activations/layer6_attention_weight_max": 55.0736083984375,
      "activations/layer6_attention_weight_min": -53.617855072021484,
      "activations/layer7_attention_weight_max": 80.5531005859375,
      "activations/layer7_attention_weight_min": -76.68939971923828,
      "activations/layer8_attention_weight_max": 46.053070068359375,
      "activations/layer8_attention_weight_min": -48.33247375488281,
      "activations/layer9_attention_weight_max": 50.38225555419922,
      "activations/layer9_attention_weight_min": -50.04794692993164,
      "epoch": 8.45,
      "learning_rate": 9.645113636363635e-05,
      "loss": 2.8231,
      "step": 145500
    },
    {
      "activations/layer0_attention_weight_max": 15.315058708190918,
      "activations/layer0_attention_weight_min": -14.4533109664917,
      "activations/layer10_attention_weight_max": 33.85385513305664,
      "activations/layer10_attention_weight_min": -34.18471908569336,
      "activations/layer11_attention_weight_max": 31.487613677978516,
      "activations/layer11_attention_weight_min": -31.11107635498047,
      "activations/layer12_attention_weight_max": 17.600128173828125,
      "activations/layer12_attention_weight_min": -24.572635650634766,
      "activations/layer13_attention_weight_max": 37.62830352783203,
      "activations/layer13_attention_weight_min": -30.91146469116211,
      "activations/layer14_attention_weight_max": 37.66469192504883,
      "activations/layer14_attention_weight_min": -30.46753692626953,
      "activations/layer15_attention_weight_max": 33.61018753051758,
      "activations/layer15_attention_weight_min": -29.292747497558594,
      "activations/layer16_attention_weight_max": 30.969356536865234,
      "activations/layer16_attention_weight_min": -27.104209899902344,
      "activations/layer17_attention_weight_max": 51.80384063720703,
      "activations/layer17_attention_weight_min": -42.163604736328125,
      "activations/layer18_attention_weight_max": 48.57184982299805,
      "activations/layer18_attention_weight_min": -35.28431701660156,
      "activations/layer19_attention_weight_max": 20.85590934753418,
      "activations/layer19_attention_weight_min": -20.59198760986328,
      "activations/layer1_attention_weight_max": 16.832975387573242,
      "activations/layer1_attention_weight_min": -14.502092361450195,
      "activations/layer20_attention_weight_max": 22.91259002685547,
      "activations/layer20_attention_weight_min": -20.080350875854492,
      "activations/layer21_attention_weight_max": 38.909034729003906,
      "activations/layer21_attention_weight_min": -24.511621475219727,
      "activations/layer22_attention_weight_max": 29.716814041137695,
      "activations/layer22_attention_weight_min": -24.070371627807617,
      "activations/layer23_attention_weight_max": 34.092193603515625,
      "activations/layer23_attention_weight_min": -22.082008361816406,
      "activations/layer2_attention_weight_max": 31.404205322265625,
      "activations/layer2_attention_weight_min": -28.713911056518555,
      "activations/layer3_attention_weight_max": 85.49072265625,
      "activations/layer3_attention_weight_min": -92.07918548583984,
      "activations/layer4_attention_weight_max": 97.91505432128906,
      "activations/layer4_attention_weight_min": -94.95318603515625,
      "activations/layer5_attention_weight_max": 85.66891479492188,
      "activations/layer5_attention_weight_min": -75.80753326416016,
      "activations/layer6_attention_weight_max": 56.13573455810547,
      "activations/layer6_attention_weight_min": -51.71365737915039,
      "activations/layer7_attention_weight_max": 77.64994049072266,
      "activations/layer7_attention_weight_min": -72.48902130126953,
      "activations/layer8_attention_weight_max": 41.86447525024414,
      "activations/layer8_attention_weight_min": -44.203617095947266,
      "activations/layer9_attention_weight_max": 51.512733459472656,
      "activations/layer9_attention_weight_min": -49.08958053588867,
      "epoch": 8.46,
      "learning_rate": 9.643219696969696e-05,
      "loss": 2.8246,
      "step": 145550
    },
    {
      "activations/layer0_attention_weight_max": 17.814208984375,
      "activations/layer0_attention_weight_min": -14.720789909362793,
      "activations/layer10_attention_weight_max": 38.92073440551758,
      "activations/layer10_attention_weight_min": -34.34929275512695,
      "activations/layer11_attention_weight_max": 30.833026885986328,
      "activations/layer11_attention_weight_min": -31.0648250579834,
      "activations/layer12_attention_weight_max": 20.194801330566406,
      "activations/layer12_attention_weight_min": -27.47366714477539,
      "activations/layer13_attention_weight_max": 42.339012145996094,
      "activations/layer13_attention_weight_min": -32.02731704711914,
      "activations/layer14_attention_weight_max": 38.873435974121094,
      "activations/layer14_attention_weight_min": -30.275543212890625,
      "activations/layer15_attention_weight_max": 37.043365478515625,
      "activations/layer15_attention_weight_min": -30.046369552612305,
      "activations/layer16_attention_weight_max": 31.531627655029297,
      "activations/layer16_attention_weight_min": -26.695852279663086,
      "activations/layer17_attention_weight_max": 54.82942581176758,
      "activations/layer17_attention_weight_min": -39.95957565307617,
      "activations/layer18_attention_weight_max": 45.90418243408203,
      "activations/layer18_attention_weight_min": -33.86199188232422,
      "activations/layer19_attention_weight_max": 25.456912994384766,
      "activations/layer19_attention_weight_min": -21.305706024169922,
      "activations/layer1_attention_weight_max": 16.350276947021484,
      "activations/layer1_attention_weight_min": -13.81386661529541,
      "activations/layer20_attention_weight_max": 21.91794776916504,
      "activations/layer20_attention_weight_min": -21.369741439819336,
      "activations/layer21_attention_weight_max": 39.97882080078125,
      "activations/layer21_attention_weight_min": -24.555809020996094,
      "activations/layer22_attention_weight_max": 33.16038513183594,
      "activations/layer22_attention_weight_min": -25.916889190673828,
      "activations/layer23_attention_weight_max": 36.258056640625,
      "activations/layer23_attention_weight_min": -24.543766021728516,
      "activations/layer2_attention_weight_max": 29.74745750427246,
      "activations/layer2_attention_weight_min": -28.903820037841797,
      "activations/layer3_attention_weight_max": 86.22006225585938,
      "activations/layer3_attention_weight_min": -86.25776672363281,
      "activations/layer4_attention_weight_max": 99.55906677246094,
      "activations/layer4_attention_weight_min": -92.96527862548828,
      "activations/layer5_attention_weight_max": 83.5206298828125,
      "activations/layer5_attention_weight_min": -77.38187408447266,
      "activations/layer6_attention_weight_max": 53.40465545654297,
      "activations/layer6_attention_weight_min": -53.63106918334961,
      "activations/layer7_attention_weight_max": 75.3802490234375,
      "activations/layer7_attention_weight_min": -77.68263244628906,
      "activations/layer8_attention_weight_max": 43.82388687133789,
      "activations/layer8_attention_weight_min": -44.4090576171875,
      "activations/layer9_attention_weight_max": 49.48384475708008,
      "activations/layer9_attention_weight_min": -49.811119079589844,
      "epoch": 8.46,
      "learning_rate": 9.641325757575757e-05,
      "loss": 2.7914,
      "step": 145600
    },
    {
      "activations/layer0_attention_weight_max": 16.875879287719727,
      "activations/layer0_attention_weight_min": -14.597386360168457,
      "activations/layer10_attention_weight_max": 37.42091369628906,
      "activations/layer10_attention_weight_min": -36.676082611083984,
      "activations/layer11_attention_weight_max": 33.09038543701172,
      "activations/layer11_attention_weight_min": -33.23215103149414,
      "activations/layer12_attention_weight_max": 20.149927139282227,
      "activations/layer12_attention_weight_min": -25.237985610961914,
      "activations/layer13_attention_weight_max": 41.611595153808594,
      "activations/layer13_attention_weight_min": -31.377761840820312,
      "activations/layer14_attention_weight_max": 37.56222152709961,
      "activations/layer14_attention_weight_min": -30.169998168945312,
      "activations/layer15_attention_weight_max": 43.505104064941406,
      "activations/layer15_attention_weight_min": -30.22210693359375,
      "activations/layer16_attention_weight_max": 34.61835861206055,
      "activations/layer16_attention_weight_min": -28.25111961364746,
      "activations/layer17_attention_weight_max": 55.4598388671875,
      "activations/layer17_attention_weight_min": -46.54503631591797,
      "activations/layer18_attention_weight_max": 49.96830368041992,
      "activations/layer18_attention_weight_min": -39.91788864135742,
      "activations/layer19_attention_weight_max": 35.08988571166992,
      "activations/layer19_attention_weight_min": -22.69694709777832,
      "activations/layer1_attention_weight_max": 16.984277725219727,
      "activations/layer1_attention_weight_min": -15.439467430114746,
      "activations/layer20_attention_weight_max": 27.630615234375,
      "activations/layer20_attention_weight_min": -24.69643211364746,
      "activations/layer21_attention_weight_max": 39.61080551147461,
      "activations/layer21_attention_weight_min": -25.360198974609375,
      "activations/layer22_attention_weight_max": 35.515647888183594,
      "activations/layer22_attention_weight_min": -26.900346755981445,
      "activations/layer23_attention_weight_max": 36.80186462402344,
      "activations/layer23_attention_weight_min": -23.659757614135742,
      "activations/layer2_attention_weight_max": 29.249670028686523,
      "activations/layer2_attention_weight_min": -28.045209884643555,
      "activations/layer3_attention_weight_max": 86.14151000976562,
      "activations/layer3_attention_weight_min": -86.68802642822266,
      "activations/layer4_attention_weight_max": 100.37760925292969,
      "activations/layer4_attention_weight_min": -93.46019744873047,
      "activations/layer5_attention_weight_max": 82.12431335449219,
      "activations/layer5_attention_weight_min": -75.33113098144531,
      "activations/layer6_attention_weight_max": 55.5118293762207,
      "activations/layer6_attention_weight_min": -52.23454666137695,
      "activations/layer7_attention_weight_max": 77.87985229492188,
      "activations/layer7_attention_weight_min": -73.35136413574219,
      "activations/layer8_attention_weight_max": 45.66447830200195,
      "activations/layer8_attention_weight_min": -45.120731353759766,
      "activations/layer9_attention_weight_max": 52.98521041870117,
      "activations/layer9_attention_weight_min": -54.1473503112793,
      "epoch": 8.46,
      "learning_rate": 9.639431818181817e-05,
      "loss": 2.7869,
      "step": 145650
    },
    {
      "activations/layer0_attention_weight_max": 16.192853927612305,
      "activations/layer0_attention_weight_min": -14.308320045471191,
      "activations/layer10_attention_weight_max": 34.85767364501953,
      "activations/layer10_attention_weight_min": -34.95879364013672,
      "activations/layer11_attention_weight_max": 29.032262802124023,
      "activations/layer11_attention_weight_min": -30.24106216430664,
      "activations/layer12_attention_weight_max": 19.36187744140625,
      "activations/layer12_attention_weight_min": -27.24779510498047,
      "activations/layer13_attention_weight_max": 34.055789947509766,
      "activations/layer13_attention_weight_min": -29.226078033447266,
      "activations/layer14_attention_weight_max": 35.7753791809082,
      "activations/layer14_attention_weight_min": -31.084335327148438,
      "activations/layer15_attention_weight_max": 31.99948501586914,
      "activations/layer15_attention_weight_min": -28.656728744506836,
      "activations/layer16_attention_weight_max": 31.858596801757812,
      "activations/layer16_attention_weight_min": -26.918394088745117,
      "activations/layer17_attention_weight_max": 55.06924819946289,
      "activations/layer17_attention_weight_min": -43.21981430053711,
      "activations/layer18_attention_weight_max": 45.48857879638672,
      "activations/layer18_attention_weight_min": -36.156314849853516,
      "activations/layer19_attention_weight_max": 20.15380096435547,
      "activations/layer19_attention_weight_min": -21.011796951293945,
      "activations/layer1_attention_weight_max": 17.40106773376465,
      "activations/layer1_attention_weight_min": -13.66111946105957,
      "activations/layer20_attention_weight_max": 20.91938591003418,
      "activations/layer20_attention_weight_min": -19.935514450073242,
      "activations/layer21_attention_weight_max": 32.88640213012695,
      "activations/layer21_attention_weight_min": -23.763925552368164,
      "activations/layer22_attention_weight_max": 32.729209899902344,
      "activations/layer22_attention_weight_min": -26.60538673400879,
      "activations/layer23_attention_weight_max": 32.52206039428711,
      "activations/layer23_attention_weight_min": -23.246448516845703,
      "activations/layer2_attention_weight_max": 29.74637222290039,
      "activations/layer2_attention_weight_min": -27.61702537536621,
      "activations/layer3_attention_weight_max": 82.55924224853516,
      "activations/layer3_attention_weight_min": -88.22637939453125,
      "activations/layer4_attention_weight_max": 96.34822845458984,
      "activations/layer4_attention_weight_min": -93.84117889404297,
      "activations/layer5_attention_weight_max": 80.43324279785156,
      "activations/layer5_attention_weight_min": -78.38511657714844,
      "activations/layer6_attention_weight_max": 52.157779693603516,
      "activations/layer6_attention_weight_min": -51.51365661621094,
      "activations/layer7_attention_weight_max": 71.45519256591797,
      "activations/layer7_attention_weight_min": -71.06291961669922,
      "activations/layer8_attention_weight_max": 41.786067962646484,
      "activations/layer8_attention_weight_min": -45.326820373535156,
      "activations/layer9_attention_weight_max": 49.725894927978516,
      "activations/layer9_attention_weight_min": -50.054073333740234,
      "epoch": 8.47,
      "learning_rate": 9.637537878787879e-05,
      "loss": 2.8047,
      "step": 145700
    },
    {
      "activations/layer0_attention_weight_max": 15.5239896774292,
      "activations/layer0_attention_weight_min": -15.598012924194336,
      "activations/layer10_attention_weight_max": 39.42816925048828,
      "activations/layer10_attention_weight_min": -35.29965591430664,
      "activations/layer11_attention_weight_max": 31.963741302490234,
      "activations/layer11_attention_weight_min": -33.30680847167969,
      "activations/layer12_attention_weight_max": 21.139251708984375,
      "activations/layer12_attention_weight_min": -23.789962768554688,
      "activations/layer13_attention_weight_max": 44.23134994506836,
      "activations/layer13_attention_weight_min": -31.436847686767578,
      "activations/layer14_attention_weight_max": 42.37862014770508,
      "activations/layer14_attention_weight_min": -31.418426513671875,
      "activations/layer15_attention_weight_max": 39.86309814453125,
      "activations/layer15_attention_weight_min": -30.49982452392578,
      "activations/layer16_attention_weight_max": 34.239253997802734,
      "activations/layer16_attention_weight_min": -30.82122802734375,
      "activations/layer17_attention_weight_max": 57.58486557006836,
      "activations/layer17_attention_weight_min": -46.61182403564453,
      "activations/layer18_attention_weight_max": 53.47614669799805,
      "activations/layer18_attention_weight_min": -37.59389114379883,
      "activations/layer19_attention_weight_max": 25.625089645385742,
      "activations/layer19_attention_weight_min": -21.207218170166016,
      "activations/layer1_attention_weight_max": 17.377384185791016,
      "activations/layer1_attention_weight_min": -14.693909645080566,
      "activations/layer20_attention_weight_max": 28.827136993408203,
      "activations/layer20_attention_weight_min": -19.543285369873047,
      "activations/layer21_attention_weight_max": 50.56584548950195,
      "activations/layer21_attention_weight_min": -25.404491424560547,
      "activations/layer22_attention_weight_max": 34.99513626098633,
      "activations/layer22_attention_weight_min": -25.977386474609375,
      "activations/layer23_attention_weight_max": 43.093345642089844,
      "activations/layer23_attention_weight_min": -21.292156219482422,
      "activations/layer2_attention_weight_max": 29.34095001220703,
      "activations/layer2_attention_weight_min": -27.649330139160156,
      "activations/layer3_attention_weight_max": 87.74274444580078,
      "activations/layer3_attention_weight_min": -88.53933715820312,
      "activations/layer4_attention_weight_max": 100.05683135986328,
      "activations/layer4_attention_weight_min": -93.7877426147461,
      "activations/layer5_attention_weight_max": 84.48494720458984,
      "activations/layer5_attention_weight_min": -77.74284362792969,
      "activations/layer6_attention_weight_max": 57.156803131103516,
      "activations/layer6_attention_weight_min": -54.72701644897461,
      "activations/layer7_attention_weight_max": 78.49932098388672,
      "activations/layer7_attention_weight_min": -74.76900482177734,
      "activations/layer8_attention_weight_max": 47.174049377441406,
      "activations/layer8_attention_weight_min": -45.44780349731445,
      "activations/layer9_attention_weight_max": 52.2822151184082,
      "activations/layer9_attention_weight_min": -52.01685333251953,
      "epoch": 8.47,
      "learning_rate": 9.635643939393938e-05,
      "loss": 2.7993,
      "step": 145750
    },
    {
      "activations/layer0_attention_weight_max": 16.042814254760742,
      "activations/layer0_attention_weight_min": -13.688570022583008,
      "activations/layer10_attention_weight_max": 39.39094924926758,
      "activations/layer10_attention_weight_min": -36.54483413696289,
      "activations/layer11_attention_weight_max": 33.008575439453125,
      "activations/layer11_attention_weight_min": -33.22502136230469,
      "activations/layer12_attention_weight_max": 21.44041633605957,
      "activations/layer12_attention_weight_min": -25.956262588500977,
      "activations/layer13_attention_weight_max": 43.78929901123047,
      "activations/layer13_attention_weight_min": -32.425086975097656,
      "activations/layer14_attention_weight_max": 37.19617462158203,
      "activations/layer14_attention_weight_min": -29.5218563079834,
      "activations/layer15_attention_weight_max": 33.897132873535156,
      "activations/layer15_attention_weight_min": -30.430326461791992,
      "activations/layer16_attention_weight_max": 31.33849334716797,
      "activations/layer16_attention_weight_min": -27.823570251464844,
      "activations/layer17_attention_weight_max": 52.49287796020508,
      "activations/layer17_attention_weight_min": -43.31608963012695,
      "activations/layer18_attention_weight_max": 49.7756462097168,
      "activations/layer18_attention_weight_min": -37.201229095458984,
      "activations/layer19_attention_weight_max": 21.97829246520996,
      "activations/layer19_attention_weight_min": -20.481307983398438,
      "activations/layer1_attention_weight_max": 18.1861629486084,
      "activations/layer1_attention_weight_min": -13.357647895812988,
      "activations/layer20_attention_weight_max": 24.377023696899414,
      "activations/layer20_attention_weight_min": -20.84825325012207,
      "activations/layer21_attention_weight_max": 45.44111251831055,
      "activations/layer21_attention_weight_min": -24.798385620117188,
      "activations/layer22_attention_weight_max": 33.52031326293945,
      "activations/layer22_attention_weight_min": -22.82623291015625,
      "activations/layer23_attention_weight_max": 34.702911376953125,
      "activations/layer23_attention_weight_min": -22.597936630249023,
      "activations/layer2_attention_weight_max": 31.009193420410156,
      "activations/layer2_attention_weight_min": -30.620262145996094,
      "activations/layer3_attention_weight_max": 88.67403411865234,
      "activations/layer3_attention_weight_min": -86.04012298583984,
      "activations/layer4_attention_weight_max": 108.25689697265625,
      "activations/layer4_attention_weight_min": -104.05513763427734,
      "activations/layer5_attention_weight_max": 88.55663299560547,
      "activations/layer5_attention_weight_min": -86.22004699707031,
      "activations/layer6_attention_weight_max": 59.59415817260742,
      "activations/layer6_attention_weight_min": -60.52313232421875,
      "activations/layer7_attention_weight_max": 80.87960052490234,
      "activations/layer7_attention_weight_min": -78.26116180419922,
      "activations/layer8_attention_weight_max": 48.33087921142578,
      "activations/layer8_attention_weight_min": -49.325077056884766,
      "activations/layer9_attention_weight_max": 49.335872650146484,
      "activations/layer9_attention_weight_min": -50.321224212646484,
      "epoch": 8.47,
      "learning_rate": 9.633749999999999e-05,
      "loss": 2.8062,
      "step": 145800
    },
    {
      "activations/layer0_attention_weight_max": 15.809724807739258,
      "activations/layer0_attention_weight_min": -13.847485542297363,
      "activations/layer10_attention_weight_max": 36.43002700805664,
      "activations/layer10_attention_weight_min": -34.54960250854492,
      "activations/layer11_attention_weight_max": 30.339933395385742,
      "activations/layer11_attention_weight_min": -31.88824462890625,
      "activations/layer12_attention_weight_max": 16.6326847076416,
      "activations/layer12_attention_weight_min": -26.815900802612305,
      "activations/layer13_attention_weight_max": 39.39140319824219,
      "activations/layer13_attention_weight_min": -29.781497955322266,
      "activations/layer14_attention_weight_max": 34.805015563964844,
      "activations/layer14_attention_weight_min": -32.52021408081055,
      "activations/layer15_attention_weight_max": 33.496097564697266,
      "activations/layer15_attention_weight_min": -30.684465408325195,
      "activations/layer16_attention_weight_max": 30.151016235351562,
      "activations/layer16_attention_weight_min": -29.68902015686035,
      "activations/layer17_attention_weight_max": 54.31727600097656,
      "activations/layer17_attention_weight_min": -47.21262741088867,
      "activations/layer18_attention_weight_max": 43.391902923583984,
      "activations/layer18_attention_weight_min": -38.07098388671875,
      "activations/layer19_attention_weight_max": 19.9659481048584,
      "activations/layer19_attention_weight_min": -22.14868927001953,
      "activations/layer1_attention_weight_max": 16.68905258178711,
      "activations/layer1_attention_weight_min": -14.969281196594238,
      "activations/layer20_attention_weight_max": 24.00421714782715,
      "activations/layer20_attention_weight_min": -23.721580505371094,
      "activations/layer21_attention_weight_max": 34.64691162109375,
      "activations/layer21_attention_weight_min": -22.70941162109375,
      "activations/layer22_attention_weight_max": 27.917369842529297,
      "activations/layer22_attention_weight_min": -25.855236053466797,
      "activations/layer23_attention_weight_max": 36.60392761230469,
      "activations/layer23_attention_weight_min": -22.36776351928711,
      "activations/layer2_attention_weight_max": 31.077407836914062,
      "activations/layer2_attention_weight_min": -30.461326599121094,
      "activations/layer3_attention_weight_max": 89.86772918701172,
      "activations/layer3_attention_weight_min": -91.69723510742188,
      "activations/layer4_attention_weight_max": 102.2089614868164,
      "activations/layer4_attention_weight_min": -96.64071655273438,
      "activations/layer5_attention_weight_max": 85.27293395996094,
      "activations/layer5_attention_weight_min": -76.4299087524414,
      "activations/layer6_attention_weight_max": 57.97655487060547,
      "activations/layer6_attention_weight_min": -51.88747024536133,
      "activations/layer7_attention_weight_max": 77.58739471435547,
      "activations/layer7_attention_weight_min": -72.37103271484375,
      "activations/layer8_attention_weight_max": 44.44344711303711,
      "activations/layer8_attention_weight_min": -44.98895263671875,
      "activations/layer9_attention_weight_max": 46.8614501953125,
      "activations/layer9_attention_weight_min": -46.14860153198242,
      "epoch": 8.47,
      "learning_rate": 9.63185606060606e-05,
      "loss": 2.8084,
      "step": 145850
    },
    {
      "activations/layer0_attention_weight_max": 14.962404251098633,
      "activations/layer0_attention_weight_min": -13.811586380004883,
      "activations/layer10_attention_weight_max": 35.25694274902344,
      "activations/layer10_attention_weight_min": -34.212677001953125,
      "activations/layer11_attention_weight_max": 30.617788314819336,
      "activations/layer11_attention_weight_min": -30.67294692993164,
      "activations/layer12_attention_weight_max": 18.023319244384766,
      "activations/layer12_attention_weight_min": -27.60041618347168,
      "activations/layer13_attention_weight_max": 40.07320785522461,
      "activations/layer13_attention_weight_min": -33.005130767822266,
      "activations/layer14_attention_weight_max": 42.9096794128418,
      "activations/layer14_attention_weight_min": -30.810728073120117,
      "activations/layer15_attention_weight_max": 35.44831466674805,
      "activations/layer15_attention_weight_min": -29.847366333007812,
      "activations/layer16_attention_weight_max": 34.09516906738281,
      "activations/layer16_attention_weight_min": -28.765506744384766,
      "activations/layer17_attention_weight_max": 52.926204681396484,
      "activations/layer17_attention_weight_min": -45.74563217163086,
      "activations/layer18_attention_weight_max": 46.19940185546875,
      "activations/layer18_attention_weight_min": -39.07633590698242,
      "activations/layer19_attention_weight_max": 19.773170471191406,
      "activations/layer19_attention_weight_min": -20.643348693847656,
      "activations/layer1_attention_weight_max": 17.519088745117188,
      "activations/layer1_attention_weight_min": -15.161490440368652,
      "activations/layer20_attention_weight_max": 22.075117111206055,
      "activations/layer20_attention_weight_min": -20.800067901611328,
      "activations/layer21_attention_weight_max": 43.914276123046875,
      "activations/layer21_attention_weight_min": -24.360729217529297,
      "activations/layer22_attention_weight_max": 29.69699478149414,
      "activations/layer22_attention_weight_min": -29.277448654174805,
      "activations/layer23_attention_weight_max": 34.473262786865234,
      "activations/layer23_attention_weight_min": -24.49110221862793,
      "activations/layer2_attention_weight_max": 30.3602352142334,
      "activations/layer2_attention_weight_min": -30.395732879638672,
      "activations/layer3_attention_weight_max": 91.046630859375,
      "activations/layer3_attention_weight_min": -93.08867645263672,
      "activations/layer4_attention_weight_max": 100.3370590209961,
      "activations/layer4_attention_weight_min": -96.04278564453125,
      "activations/layer5_attention_weight_max": 79.78361511230469,
      "activations/layer5_attention_weight_min": -76.70817565917969,
      "activations/layer6_attention_weight_max": 55.15073776245117,
      "activations/layer6_attention_weight_min": -51.98463821411133,
      "activations/layer7_attention_weight_max": 81.15330505371094,
      "activations/layer7_attention_weight_min": -73.04839324951172,
      "activations/layer8_attention_weight_max": 47.437252044677734,
      "activations/layer8_attention_weight_min": -48.5241584777832,
      "activations/layer9_attention_weight_max": 55.88753128051758,
      "activations/layer9_attention_weight_min": -51.22600173950195,
      "epoch": 8.48,
      "learning_rate": 9.62996212121212e-05,
      "loss": 2.7899,
      "step": 145900
    },
    {
      "activations/layer0_attention_weight_max": 15.467262268066406,
      "activations/layer0_attention_weight_min": -14.342913627624512,
      "activations/layer10_attention_weight_max": 36.61671447753906,
      "activations/layer10_attention_weight_min": -36.19325637817383,
      "activations/layer11_attention_weight_max": 31.312246322631836,
      "activations/layer11_attention_weight_min": -32.816192626953125,
      "activations/layer12_attention_weight_max": 22.671716690063477,
      "activations/layer12_attention_weight_min": -22.146259307861328,
      "activations/layer13_attention_weight_max": 56.8375244140625,
      "activations/layer13_attention_weight_min": -38.49565505981445,
      "activations/layer14_attention_weight_max": 43.1434440612793,
      "activations/layer14_attention_weight_min": -30.618816375732422,
      "activations/layer15_attention_weight_max": 37.06086349487305,
      "activations/layer15_attention_weight_min": -29.585914611816406,
      "activations/layer16_attention_weight_max": 34.941261291503906,
      "activations/layer16_attention_weight_min": -27.90999412536621,
      "activations/layer17_attention_weight_max": 56.757938385009766,
      "activations/layer17_attention_weight_min": -42.83203887939453,
      "activations/layer18_attention_weight_max": 57.84233856201172,
      "activations/layer18_attention_weight_min": -37.77992248535156,
      "activations/layer19_attention_weight_max": 24.14723014831543,
      "activations/layer19_attention_weight_min": -21.208694458007812,
      "activations/layer1_attention_weight_max": 17.1911678314209,
      "activations/layer1_attention_weight_min": -17.186046600341797,
      "activations/layer20_attention_weight_max": 25.81081771850586,
      "activations/layer20_attention_weight_min": -21.380399703979492,
      "activations/layer21_attention_weight_max": 44.29745101928711,
      "activations/layer21_attention_weight_min": -22.681682586669922,
      "activations/layer22_attention_weight_max": 37.659393310546875,
      "activations/layer22_attention_weight_min": -28.74309730529785,
      "activations/layer23_attention_weight_max": 41.37702941894531,
      "activations/layer23_attention_weight_min": -26.131866455078125,
      "activations/layer2_attention_weight_max": 28.92339324951172,
      "activations/layer2_attention_weight_min": -28.49838638305664,
      "activations/layer3_attention_weight_max": 85.22673797607422,
      "activations/layer3_attention_weight_min": -88.00138092041016,
      "activations/layer4_attention_weight_max": 100.05801391601562,
      "activations/layer4_attention_weight_min": -95.23455810546875,
      "activations/layer5_attention_weight_max": 82.19725036621094,
      "activations/layer5_attention_weight_min": -77.14119720458984,
      "activations/layer6_attention_weight_max": 53.48550033569336,
      "activations/layer6_attention_weight_min": -54.56342315673828,
      "activations/layer7_attention_weight_max": 74.9896469116211,
      "activations/layer7_attention_weight_min": -75.1263198852539,
      "activations/layer8_attention_weight_max": 47.031578063964844,
      "activations/layer8_attention_weight_min": -48.60820007324219,
      "activations/layer9_attention_weight_max": 49.64550018310547,
      "activations/layer9_attention_weight_min": -51.3342170715332,
      "epoch": 8.48,
      "learning_rate": 9.628068181818181e-05,
      "loss": 2.784,
      "step": 145950
    },
    {
      "activations/layer0_attention_weight_max": 16.799802780151367,
      "activations/layer0_attention_weight_min": -14.312911033630371,
      "activations/layer10_attention_weight_max": 35.558349609375,
      "activations/layer10_attention_weight_min": -36.02266311645508,
      "activations/layer11_attention_weight_max": 30.0281982421875,
      "activations/layer11_attention_weight_min": -31.889385223388672,
      "activations/layer12_attention_weight_max": 20.14206314086914,
      "activations/layer12_attention_weight_min": -25.736635208129883,
      "activations/layer13_attention_weight_max": 39.80094528198242,
      "activations/layer13_attention_weight_min": -29.256023406982422,
      "activations/layer14_attention_weight_max": 39.13847732543945,
      "activations/layer14_attention_weight_min": -29.31283950805664,
      "activations/layer15_attention_weight_max": 34.65969467163086,
      "activations/layer15_attention_weight_min": -29.22372055053711,
      "activations/layer16_attention_weight_max": 32.16451644897461,
      "activations/layer16_attention_weight_min": -27.620277404785156,
      "activations/layer17_attention_weight_max": 53.39923095703125,
      "activations/layer17_attention_weight_min": -43.36468505859375,
      "activations/layer18_attention_weight_max": 46.393009185791016,
      "activations/layer18_attention_weight_min": -35.30735397338867,
      "activations/layer19_attention_weight_max": 19.73426055908203,
      "activations/layer19_attention_weight_min": -20.31354522705078,
      "activations/layer1_attention_weight_max": 16.780885696411133,
      "activations/layer1_attention_weight_min": -15.638175964355469,
      "activations/layer20_attention_weight_max": 18.48790740966797,
      "activations/layer20_attention_weight_min": -21.60356903076172,
      "activations/layer21_attention_weight_max": 33.93811798095703,
      "activations/layer21_attention_weight_min": -22.14095115661621,
      "activations/layer22_attention_weight_max": 30.84782600402832,
      "activations/layer22_attention_weight_min": -24.008033752441406,
      "activations/layer23_attention_weight_max": 33.1712646484375,
      "activations/layer23_attention_weight_min": -22.51931381225586,
      "activations/layer2_attention_weight_max": 31.0710506439209,
      "activations/layer2_attention_weight_min": -31.456588745117188,
      "activations/layer3_attention_weight_max": 85.53475952148438,
      "activations/layer3_attention_weight_min": -86.01665496826172,
      "activations/layer4_attention_weight_max": 101.30136108398438,
      "activations/layer4_attention_weight_min": -89.09728240966797,
      "activations/layer5_attention_weight_max": 79.7840347290039,
      "activations/layer5_attention_weight_min": -77.26271057128906,
      "activations/layer6_attention_weight_max": 58.27082061767578,
      "activations/layer6_attention_weight_min": -53.29729080200195,
      "activations/layer7_attention_weight_max": 81.293701171875,
      "activations/layer7_attention_weight_min": -71.26493835449219,
      "activations/layer8_attention_weight_max": 43.39461898803711,
      "activations/layer8_attention_weight_min": -44.5488395690918,
      "activations/layer9_attention_weight_max": 50.27883529663086,
      "activations/layer9_attention_weight_min": -49.7712287902832,
      "epoch": 8.48,
      "learning_rate": 9.626174242424242e-05,
      "loss": 2.8132,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.501,
      "eval_samples_per_second": 505.114,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.501,
      "eval_openwebtext_samples_per_second": 505.114,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 1.9999,
      "eval_wikitext_samples_per_second": 228.009,
      "step": 146000
    },
    {
      "epoch": 8.48,
      "eval_lambada_loss": 2.619140625,
      "eval_lambada_ppl": 13.72392451825968,
      "eval_lambada_runtime": 9.5611,
      "eval_lambada_samples_per_second": 509.249,
      "step": 146000
    },
    {
      "activations/layer0_attention_weight_max": 16.632827758789062,
      "activations/layer0_attention_weight_min": -14.525375366210938,
      "activations/layer10_attention_weight_max": 37.920814514160156,
      "activations/layer10_attention_weight_min": -37.657379150390625,
      "activations/layer11_attention_weight_max": 33.28190994262695,
      "activations/layer11_attention_weight_min": -30.78310203552246,
      "activations/layer12_attention_weight_max": 20.219186782836914,
      "activations/layer12_attention_weight_min": -25.38615608215332,
      "activations/layer13_attention_weight_max": 49.04820251464844,
      "activations/layer13_attention_weight_min": -34.029266357421875,
      "activations/layer14_attention_weight_max": 39.28150939941406,
      "activations/layer14_attention_weight_min": -30.785825729370117,
      "activations/layer15_attention_weight_max": 35.375885009765625,
      "activations/layer15_attention_weight_min": -29.81064224243164,
      "activations/layer16_attention_weight_max": 29.866201400756836,
      "activations/layer16_attention_weight_min": -27.630216598510742,
      "activations/layer17_attention_weight_max": 53.959556579589844,
      "activations/layer17_attention_weight_min": -43.986148834228516,
      "activations/layer18_attention_weight_max": 47.66006088256836,
      "activations/layer18_attention_weight_min": -36.67152786254883,
      "activations/layer19_attention_weight_max": 19.831037521362305,
      "activations/layer19_attention_weight_min": -20.863588333129883,
      "activations/layer1_attention_weight_max": 18.728260040283203,
      "activations/layer1_attention_weight_min": -18.467418670654297,
      "activations/layer20_attention_weight_max": 21.32279396057129,
      "activations/layer20_attention_weight_min": -21.943187713623047,
      "activations/layer21_attention_weight_max": 38.38311767578125,
      "activations/layer21_attention_weight_min": -25.269407272338867,
      "activations/layer22_attention_weight_max": 31.957984924316406,
      "activations/layer22_attention_weight_min": -26.821334838867188,
      "activations/layer23_attention_weight_max": 37.44512176513672,
      "activations/layer23_attention_weight_min": -26.12710189819336,
      "activations/layer2_attention_weight_max": 30.431289672851562,
      "activations/layer2_attention_weight_min": -29.703100204467773,
      "activations/layer3_attention_weight_max": 87.86937713623047,
      "activations/layer3_attention_weight_min": -90.3628158569336,
      "activations/layer4_attention_weight_max": 98.54193878173828,
      "activations/layer4_attention_weight_min": -97.88103485107422,
      "activations/layer5_attention_weight_max": 78.49885559082031,
      "activations/layer5_attention_weight_min": -74.61137390136719,
      "activations/layer6_attention_weight_max": 56.05137634277344,
      "activations/layer6_attention_weight_min": -53.37691879272461,
      "activations/layer7_attention_weight_max": 78.53121185302734,
      "activations/layer7_attention_weight_min": -78.93621063232422,
      "activations/layer8_attention_weight_max": 47.61051940917969,
      "activations/layer8_attention_weight_min": -49.68684768676758,
      "activations/layer9_attention_weight_max": 57.114559173583984,
      "activations/layer9_attention_weight_min": -49.393287658691406,
      "epoch": 8.49,
      "learning_rate": 9.624280303030301e-05,
      "loss": 2.8235,
      "step": 146050
    },
    {
      "activations/layer0_attention_weight_max": 15.515314102172852,
      "activations/layer0_attention_weight_min": -14.78412914276123,
      "activations/layer10_attention_weight_max": 35.01723098754883,
      "activations/layer10_attention_weight_min": -33.3078727722168,
      "activations/layer11_attention_weight_max": 29.6294002532959,
      "activations/layer11_attention_weight_min": -29.801860809326172,
      "activations/layer12_attention_weight_max": 20.501432418823242,
      "activations/layer12_attention_weight_min": -25.761430740356445,
      "activations/layer13_attention_weight_max": 39.36477279663086,
      "activations/layer13_attention_weight_min": -30.042455673217773,
      "activations/layer14_attention_weight_max": 34.58675765991211,
      "activations/layer14_attention_weight_min": -29.399356842041016,
      "activations/layer15_attention_weight_max": 33.74048614501953,
      "activations/layer15_attention_weight_min": -28.064937591552734,
      "activations/layer16_attention_weight_max": 29.237266540527344,
      "activations/layer16_attention_weight_min": -27.930692672729492,
      "activations/layer17_attention_weight_max": 52.6719856262207,
      "activations/layer17_attention_weight_min": -42.47650909423828,
      "activations/layer18_attention_weight_max": 42.52582931518555,
      "activations/layer18_attention_weight_min": -36.04610824584961,
      "activations/layer19_attention_weight_max": 20.075712203979492,
      "activations/layer19_attention_weight_min": -21.502172470092773,
      "activations/layer1_attention_weight_max": 17.90276527404785,
      "activations/layer1_attention_weight_min": -13.898770332336426,
      "activations/layer20_attention_weight_max": 19.70419692993164,
      "activations/layer20_attention_weight_min": -20.943979263305664,
      "activations/layer21_attention_weight_max": 33.766666412353516,
      "activations/layer21_attention_weight_min": -22.168560028076172,
      "activations/layer22_attention_weight_max": 29.75994873046875,
      "activations/layer22_attention_weight_min": -23.2830753326416,
      "activations/layer23_attention_weight_max": 31.789907455444336,
      "activations/layer23_attention_weight_min": -22.29414176940918,
      "activations/layer2_attention_weight_max": 29.850540161132812,
      "activations/layer2_attention_weight_min": -29.106239318847656,
      "activations/layer3_attention_weight_max": 87.6623764038086,
      "activations/layer3_attention_weight_min": -87.2225112915039,
      "activations/layer4_attention_weight_max": 99.15171813964844,
      "activations/layer4_attention_weight_min": -93.1954116821289,
      "activations/layer5_attention_weight_max": 77.31533813476562,
      "activations/layer5_attention_weight_min": -78.01358032226562,
      "activations/layer6_attention_weight_max": 51.76619338989258,
      "activations/layer6_attention_weight_min": -51.278446197509766,
      "activations/layer7_attention_weight_max": 73.05126190185547,
      "activations/layer7_attention_weight_min": -70.47759246826172,
      "activations/layer8_attention_weight_max": 44.0189094543457,
      "activations/layer8_attention_weight_min": -45.148406982421875,
      "activations/layer9_attention_weight_max": 50.06519317626953,
      "activations/layer9_attention_weight_min": -46.48215866088867,
      "epoch": 8.49,
      "learning_rate": 9.622386363636363e-05,
      "loss": 2.8081,
      "step": 146100
    },
    {
      "activations/layer0_attention_weight_max": 14.703447341918945,
      "activations/layer0_attention_weight_min": -14.521380424499512,
      "activations/layer10_attention_weight_max": 49.72222900390625,
      "activations/layer10_attention_weight_min": -43.112876892089844,
      "activations/layer11_attention_weight_max": 41.347938537597656,
      "activations/layer11_attention_weight_min": -36.97868347167969,
      "activations/layer12_attention_weight_max": 20.117446899414062,
      "activations/layer12_attention_weight_min": -23.097055435180664,
      "activations/layer13_attention_weight_max": 60.20259475708008,
      "activations/layer13_attention_weight_min": -36.1878662109375,
      "activations/layer14_attention_weight_max": 77.18189239501953,
      "activations/layer14_attention_weight_min": -39.40867614746094,
      "activations/layer15_attention_weight_max": 59.544612884521484,
      "activations/layer15_attention_weight_min": -33.87865447998047,
      "activations/layer16_attention_weight_max": 35.718299865722656,
      "activations/layer16_attention_weight_min": -28.551143646240234,
      "activations/layer17_attention_weight_max": 56.3540153503418,
      "activations/layer17_attention_weight_min": -44.039146423339844,
      "activations/layer18_attention_weight_max": 46.175174713134766,
      "activations/layer18_attention_weight_min": -36.593238830566406,
      "activations/layer19_attention_weight_max": 23.576574325561523,
      "activations/layer19_attention_weight_min": -20.968639373779297,
      "activations/layer1_attention_weight_max": 16.557037353515625,
      "activations/layer1_attention_weight_min": -14.750421524047852,
      "activations/layer20_attention_weight_max": 24.560340881347656,
      "activations/layer20_attention_weight_min": -21.56015968322754,
      "activations/layer21_attention_weight_max": 45.927276611328125,
      "activations/layer21_attention_weight_min": -23.020620346069336,
      "activations/layer22_attention_weight_max": 34.43407440185547,
      "activations/layer22_attention_weight_min": -25.36516571044922,
      "activations/layer23_attention_weight_max": 33.51890182495117,
      "activations/layer23_attention_weight_min": -23.040115356445312,
      "activations/layer2_attention_weight_max": 32.30236053466797,
      "activations/layer2_attention_weight_min": -30.814300537109375,
      "activations/layer3_attention_weight_max": 99.87112426757812,
      "activations/layer3_attention_weight_min": -98.58507537841797,
      "activations/layer4_attention_weight_max": 114.18983459472656,
      "activations/layer4_attention_weight_min": -104.81829071044922,
      "activations/layer5_attention_weight_max": 90.13604736328125,
      "activations/layer5_attention_weight_min": -86.8584213256836,
      "activations/layer6_attention_weight_max": 59.07686233520508,
      "activations/layer6_attention_weight_min": -53.67319869995117,
      "activations/layer7_attention_weight_max": 99.02733612060547,
      "activations/layer7_attention_weight_min": -83.69641876220703,
      "activations/layer8_attention_weight_max": 55.70249938964844,
      "activations/layer8_attention_weight_min": -49.9278678894043,
      "activations/layer9_attention_weight_max": 70.11054229736328,
      "activations/layer9_attention_weight_min": -56.975685119628906,
      "epoch": 8.49,
      "learning_rate": 9.620492424242424e-05,
      "loss": 2.7997,
      "step": 146150
    },
    {
      "activations/layer0_attention_weight_max": 15.172216415405273,
      "activations/layer0_attention_weight_min": -14.684259414672852,
      "activations/layer10_attention_weight_max": 37.688880920410156,
      "activations/layer10_attention_weight_min": -35.94996643066406,
      "activations/layer11_attention_weight_max": 32.20166015625,
      "activations/layer11_attention_weight_min": -32.022789001464844,
      "activations/layer12_attention_weight_max": 24.926313400268555,
      "activations/layer12_attention_weight_min": -23.76784896850586,
      "activations/layer13_attention_weight_max": 45.454978942871094,
      "activations/layer13_attention_weight_min": -33.856964111328125,
      "activations/layer14_attention_weight_max": 38.940799713134766,
      "activations/layer14_attention_weight_min": -32.291683197021484,
      "activations/layer15_attention_weight_max": 35.71084976196289,
      "activations/layer15_attention_weight_min": -29.944759368896484,
      "activations/layer16_attention_weight_max": 35.07467269897461,
      "activations/layer16_attention_weight_min": -28.537656784057617,
      "activations/layer17_attention_weight_max": 54.050743103027344,
      "activations/layer17_attention_weight_min": -43.0026741027832,
      "activations/layer18_attention_weight_max": 47.33998107910156,
      "activations/layer18_attention_weight_min": -36.82765579223633,
      "activations/layer19_attention_weight_max": 22.640832901000977,
      "activations/layer19_attention_weight_min": -22.65638542175293,
      "activations/layer1_attention_weight_max": 16.36102294921875,
      "activations/layer1_attention_weight_min": -14.43489933013916,
      "activations/layer20_attention_weight_max": 23.353199005126953,
      "activations/layer20_attention_weight_min": -20.64344024658203,
      "activations/layer21_attention_weight_max": 39.628639221191406,
      "activations/layer21_attention_weight_min": -24.919614791870117,
      "activations/layer22_attention_weight_max": 31.02863883972168,
      "activations/layer22_attention_weight_min": -25.179433822631836,
      "activations/layer23_attention_weight_max": 36.85470962524414,
      "activations/layer23_attention_weight_min": -25.137561798095703,
      "activations/layer2_attention_weight_max": 30.121259689331055,
      "activations/layer2_attention_weight_min": -29.497467041015625,
      "activations/layer3_attention_weight_max": 85.76277160644531,
      "activations/layer3_attention_weight_min": -86.51033020019531,
      "activations/layer4_attention_weight_max": 99.0053939819336,
      "activations/layer4_attention_weight_min": -93.13335418701172,
      "activations/layer5_attention_weight_max": 84.1797103881836,
      "activations/layer5_attention_weight_min": -77.52882385253906,
      "activations/layer6_attention_weight_max": 56.63618469238281,
      "activations/layer6_attention_weight_min": -52.22159194946289,
      "activations/layer7_attention_weight_max": 80.20360565185547,
      "activations/layer7_attention_weight_min": -72.85441589355469,
      "activations/layer8_attention_weight_max": 43.98543167114258,
      "activations/layer8_attention_weight_min": -46.180397033691406,
      "activations/layer9_attention_weight_max": 49.71990966796875,
      "activations/layer9_attention_weight_min": -48.58141326904297,
      "epoch": 8.5,
      "learning_rate": 9.618598484848484e-05,
      "loss": 2.8094,
      "step": 146200
    },
    {
      "activations/layer0_attention_weight_max": 14.688196182250977,
      "activations/layer0_attention_weight_min": -14.501640319824219,
      "activations/layer10_attention_weight_max": 39.24616622924805,
      "activations/layer10_attention_weight_min": -37.93050003051758,
      "activations/layer11_attention_weight_max": 31.759765625,
      "activations/layer11_attention_weight_min": -33.75954055786133,
      "activations/layer12_attention_weight_max": 22.939029693603516,
      "activations/layer12_attention_weight_min": -24.622982025146484,
      "activations/layer13_attention_weight_max": 39.48946762084961,
      "activations/layer13_attention_weight_min": -31.737262725830078,
      "activations/layer14_attention_weight_max": 45.22473907470703,
      "activations/layer14_attention_weight_min": -32.050209045410156,
      "activations/layer15_attention_weight_max": 38.00527572631836,
      "activations/layer15_attention_weight_min": -29.91133689880371,
      "activations/layer16_attention_weight_max": 34.62504959106445,
      "activations/layer16_attention_weight_min": -28.483976364135742,
      "activations/layer17_attention_weight_max": 55.32476043701172,
      "activations/layer17_attention_weight_min": -44.423892974853516,
      "activations/layer18_attention_weight_max": 49.38318634033203,
      "activations/layer18_attention_weight_min": -37.79658889770508,
      "activations/layer19_attention_weight_max": 21.996570587158203,
      "activations/layer19_attention_weight_min": -21.54773712158203,
      "activations/layer1_attention_weight_max": 17.695632934570312,
      "activations/layer1_attention_weight_min": -13.218002319335938,
      "activations/layer20_attention_weight_max": 23.257925033569336,
      "activations/layer20_attention_weight_min": -21.961429595947266,
      "activations/layer21_attention_weight_max": 39.74919509887695,
      "activations/layer21_attention_weight_min": -24.710464477539062,
      "activations/layer22_attention_weight_max": 34.80203628540039,
      "activations/layer22_attention_weight_min": -25.0701904296875,
      "activations/layer23_attention_weight_max": 39.29262161254883,
      "activations/layer23_attention_weight_min": -22.398792266845703,
      "activations/layer2_attention_weight_max": 30.021339416503906,
      "activations/layer2_attention_weight_min": -28.171890258789062,
      "activations/layer3_attention_weight_max": 85.83171081542969,
      "activations/layer3_attention_weight_min": -92.67401885986328,
      "activations/layer4_attention_weight_max": 95.41921997070312,
      "activations/layer4_attention_weight_min": -98.37035369873047,
      "activations/layer5_attention_weight_max": 78.04456329345703,
      "activations/layer5_attention_weight_min": -76.7081069946289,
      "activations/layer6_attention_weight_max": 53.30929946899414,
      "activations/layer6_attention_weight_min": -52.97394561767578,
      "activations/layer7_attention_weight_max": 79.03661346435547,
      "activations/layer7_attention_weight_min": -72.39584350585938,
      "activations/layer8_attention_weight_max": 45.673545837402344,
      "activations/layer8_attention_weight_min": -46.97169876098633,
      "activations/layer9_attention_weight_max": 54.54346466064453,
      "activations/layer9_attention_weight_min": -49.38737487792969,
      "epoch": 8.5,
      "learning_rate": 9.616704545454544e-05,
      "loss": 2.7947,
      "step": 146250
    },
    {
      "activations/layer0_attention_weight_max": 15.384424209594727,
      "activations/layer0_attention_weight_min": -14.580881118774414,
      "activations/layer10_attention_weight_max": 40.40039825439453,
      "activations/layer10_attention_weight_min": -38.03226089477539,
      "activations/layer11_attention_weight_max": 33.09913635253906,
      "activations/layer11_attention_weight_min": -32.5180778503418,
      "activations/layer12_attention_weight_max": 32.57633590698242,
      "activations/layer12_attention_weight_min": -26.512239456176758,
      "activations/layer13_attention_weight_max": 47.34963607788086,
      "activations/layer13_attention_weight_min": -33.53550720214844,
      "activations/layer14_attention_weight_max": 38.74140930175781,
      "activations/layer14_attention_weight_min": -32.47300338745117,
      "activations/layer15_attention_weight_max": 38.51034927368164,
      "activations/layer15_attention_weight_min": -28.52302360534668,
      "activations/layer16_attention_weight_max": 32.64566421508789,
      "activations/layer16_attention_weight_min": -26.80296516418457,
      "activations/layer17_attention_weight_max": 57.7293701171875,
      "activations/layer17_attention_weight_min": -42.103668212890625,
      "activations/layer18_attention_weight_max": 48.95259475708008,
      "activations/layer18_attention_weight_min": -34.87339782714844,
      "activations/layer19_attention_weight_max": 23.491897583007812,
      "activations/layer19_attention_weight_min": -21.664875030517578,
      "activations/layer1_attention_weight_max": 16.851415634155273,
      "activations/layer1_attention_weight_min": -16.010581970214844,
      "activations/layer20_attention_weight_max": 21.061389923095703,
      "activations/layer20_attention_weight_min": -22.914630889892578,
      "activations/layer21_attention_weight_max": 41.610836029052734,
      "activations/layer21_attention_weight_min": -24.627090454101562,
      "activations/layer22_attention_weight_max": 31.223283767700195,
      "activations/layer22_attention_weight_min": -23.364770889282227,
      "activations/layer23_attention_weight_max": 35.890228271484375,
      "activations/layer23_attention_weight_min": -22.777441024780273,
      "activations/layer2_attention_weight_max": 30.39594078063965,
      "activations/layer2_attention_weight_min": -28.838232040405273,
      "activations/layer3_attention_weight_max": 88.07389068603516,
      "activations/layer3_attention_weight_min": -91.87627410888672,
      "activations/layer4_attention_weight_max": 97.8609848022461,
      "activations/layer4_attention_weight_min": -97.21881866455078,
      "activations/layer5_attention_weight_max": 79.26472473144531,
      "activations/layer5_attention_weight_min": -77.76518249511719,
      "activations/layer6_attention_weight_max": 55.70240783691406,
      "activations/layer6_attention_weight_min": -54.303646087646484,
      "activations/layer7_attention_weight_max": 78.56806182861328,
      "activations/layer7_attention_weight_min": -77.4272689819336,
      "activations/layer8_attention_weight_max": 45.489315032958984,
      "activations/layer8_attention_weight_min": -46.30946350097656,
      "activations/layer9_attention_weight_max": 50.38497543334961,
      "activations/layer9_attention_weight_min": -52.529048919677734,
      "epoch": 8.5,
      "learning_rate": 9.614810606060605e-05,
      "loss": 2.8127,
      "step": 146300
    },
    {
      "activations/layer0_attention_weight_max": 15.956390380859375,
      "activations/layer0_attention_weight_min": -14.118223190307617,
      "activations/layer10_attention_weight_max": 35.72125244140625,
      "activations/layer10_attention_weight_min": -36.5889778137207,
      "activations/layer11_attention_weight_max": 29.693862915039062,
      "activations/layer11_attention_weight_min": -32.451988220214844,
      "activations/layer12_attention_weight_max": 19.62531089782715,
      "activations/layer12_attention_weight_min": -25.340051651000977,
      "activations/layer13_attention_weight_max": 43.357154846191406,
      "activations/layer13_attention_weight_min": -31.142745971679688,
      "activations/layer14_attention_weight_max": 38.81858825683594,
      "activations/layer14_attention_weight_min": -29.40543556213379,
      "activations/layer15_attention_weight_max": 33.357662200927734,
      "activations/layer15_attention_weight_min": -28.715425491333008,
      "activations/layer16_attention_weight_max": 33.09492492675781,
      "activations/layer16_attention_weight_min": -27.60980987548828,
      "activations/layer17_attention_weight_max": 52.65483093261719,
      "activations/layer17_attention_weight_min": -41.380714416503906,
      "activations/layer18_attention_weight_max": 48.91094970703125,
      "activations/layer18_attention_weight_min": -34.454620361328125,
      "activations/layer19_attention_weight_max": 25.951923370361328,
      "activations/layer19_attention_weight_min": -22.319263458251953,
      "activations/layer1_attention_weight_max": 16.69220542907715,
      "activations/layer1_attention_weight_min": -15.938008308410645,
      "activations/layer20_attention_weight_max": 22.611431121826172,
      "activations/layer20_attention_weight_min": -23.63190269470215,
      "activations/layer21_attention_weight_max": 41.29231643676758,
      "activations/layer21_attention_weight_min": -24.77427864074707,
      "activations/layer22_attention_weight_max": 35.191402435302734,
      "activations/layer22_attention_weight_min": -26.21605682373047,
      "activations/layer23_attention_weight_max": 37.40155029296875,
      "activations/layer23_attention_weight_min": -22.673290252685547,
      "activations/layer2_attention_weight_max": 30.047100067138672,
      "activations/layer2_attention_weight_min": -28.380966186523438,
      "activations/layer3_attention_weight_max": 86.56201171875,
      "activations/layer3_attention_weight_min": -93.34968566894531,
      "activations/layer4_attention_weight_max": 98.40365600585938,
      "activations/layer4_attention_weight_min": -99.8492202758789,
      "activations/layer5_attention_weight_max": 77.17290496826172,
      "activations/layer5_attention_weight_min": -75.83759307861328,
      "activations/layer6_attention_weight_max": 53.238502502441406,
      "activations/layer6_attention_weight_min": -52.84347915649414,
      "activations/layer7_attention_weight_max": 74.22330474853516,
      "activations/layer7_attention_weight_min": -74.79531860351562,
      "activations/layer8_attention_weight_max": 43.94842529296875,
      "activations/layer8_attention_weight_min": -46.62512969970703,
      "activations/layer9_attention_weight_max": 48.44214630126953,
      "activations/layer9_attention_weight_min": -50.52721405029297,
      "epoch": 8.5,
      "learning_rate": 9.612916666666666e-05,
      "loss": 2.8122,
      "step": 146350
    },
    {
      "activations/layer0_attention_weight_max": 15.17863655090332,
      "activations/layer0_attention_weight_min": -14.383612632751465,
      "activations/layer10_attention_weight_max": 36.51887130737305,
      "activations/layer10_attention_weight_min": -35.77277374267578,
      "activations/layer11_attention_weight_max": 32.49430847167969,
      "activations/layer11_attention_weight_min": -30.357410430908203,
      "activations/layer12_attention_weight_max": 20.75189971923828,
      "activations/layer12_attention_weight_min": -27.29766273498535,
      "activations/layer13_attention_weight_max": 40.32744598388672,
      "activations/layer13_attention_weight_min": -33.2000846862793,
      "activations/layer14_attention_weight_max": 39.766239166259766,
      "activations/layer14_attention_weight_min": -28.637977600097656,
      "activations/layer15_attention_weight_max": 33.60565948486328,
      "activations/layer15_attention_weight_min": -28.47786521911621,
      "activations/layer16_attention_weight_max": 29.473085403442383,
      "activations/layer16_attention_weight_min": -27.847986221313477,
      "activations/layer17_attention_weight_max": 53.11111831665039,
      "activations/layer17_attention_weight_min": -41.40876388549805,
      "activations/layer18_attention_weight_max": 46.32803726196289,
      "activations/layer18_attention_weight_min": -36.3794059753418,
      "activations/layer19_attention_weight_max": 26.167829513549805,
      "activations/layer19_attention_weight_min": -21.48328399658203,
      "activations/layer1_attention_weight_max": 16.757770538330078,
      "activations/layer1_attention_weight_min": -16.74372673034668,
      "activations/layer20_attention_weight_max": 22.06625747680664,
      "activations/layer20_attention_weight_min": -22.701658248901367,
      "activations/layer21_attention_weight_max": 38.510616302490234,
      "activations/layer21_attention_weight_min": -24.077159881591797,
      "activations/layer22_attention_weight_max": 29.40555763244629,
      "activations/layer22_attention_weight_min": -24.245866775512695,
      "activations/layer23_attention_weight_max": 36.29863357543945,
      "activations/layer23_attention_weight_min": -23.54891014099121,
      "activations/layer2_attention_weight_max": 34.19593048095703,
      "activations/layer2_attention_weight_min": -31.67566680908203,
      "activations/layer3_attention_weight_max": 91.86503601074219,
      "activations/layer3_attention_weight_min": -98.2252426147461,
      "activations/layer4_attention_weight_max": 102.27896881103516,
      "activations/layer4_attention_weight_min": -99.19869232177734,
      "activations/layer5_attention_weight_max": 81.94651794433594,
      "activations/layer5_attention_weight_min": -76.9122543334961,
      "activations/layer6_attention_weight_max": 55.100563049316406,
      "activations/layer6_attention_weight_min": -53.298072814941406,
      "activations/layer7_attention_weight_max": 77.30337524414062,
      "activations/layer7_attention_weight_min": -73.0399169921875,
      "activations/layer8_attention_weight_max": 45.39453125,
      "activations/layer8_attention_weight_min": -44.568355560302734,
      "activations/layer9_attention_weight_max": 54.84786605834961,
      "activations/layer9_attention_weight_min": -49.69920349121094,
      "epoch": 8.51,
      "learning_rate": 9.611022727272728e-05,
      "loss": 2.805,
      "step": 146400
    },
    {
      "activations/layer0_attention_weight_max": 15.929532051086426,
      "activations/layer0_attention_weight_min": -14.518651008605957,
      "activations/layer10_attention_weight_max": 38.744972229003906,
      "activations/layer10_attention_weight_min": -36.75141143798828,
      "activations/layer11_attention_weight_max": 32.58161926269531,
      "activations/layer11_attention_weight_min": -31.67517852783203,
      "activations/layer12_attention_weight_max": 22.558496475219727,
      "activations/layer12_attention_weight_min": -25.40109634399414,
      "activations/layer13_attention_weight_max": 62.32891845703125,
      "activations/layer13_attention_weight_min": -32.6310920715332,
      "activations/layer14_attention_weight_max": 81.91024017333984,
      "activations/layer14_attention_weight_min": -37.009483337402344,
      "activations/layer15_attention_weight_max": 61.55512237548828,
      "activations/layer15_attention_weight_min": -29.566734313964844,
      "activations/layer16_attention_weight_max": 35.8979377746582,
      "activations/layer16_attention_weight_min": -28.707534790039062,
      "activations/layer17_attention_weight_max": 61.5169563293457,
      "activations/layer17_attention_weight_min": -44.47527313232422,
      "activations/layer18_attention_weight_max": 56.25959777832031,
      "activations/layer18_attention_weight_min": -37.621604919433594,
      "activations/layer19_attention_weight_max": 25.33474349975586,
      "activations/layer19_attention_weight_min": -22.641155242919922,
      "activations/layer1_attention_weight_max": 16.004499435424805,
      "activations/layer1_attention_weight_min": -15.144552230834961,
      "activations/layer20_attention_weight_max": 24.340856552124023,
      "activations/layer20_attention_weight_min": -20.57420539855957,
      "activations/layer21_attention_weight_max": 52.14714431762695,
      "activations/layer21_attention_weight_min": -22.264541625976562,
      "activations/layer22_attention_weight_max": 34.175682067871094,
      "activations/layer22_attention_weight_min": -25.125852584838867,
      "activations/layer23_attention_weight_max": 42.05821228027344,
      "activations/layer23_attention_weight_min": -23.13458824157715,
      "activations/layer2_attention_weight_max": 30.445087432861328,
      "activations/layer2_attention_weight_min": -28.81726837158203,
      "activations/layer3_attention_weight_max": 88.36050415039062,
      "activations/layer3_attention_weight_min": -87.72279357910156,
      "activations/layer4_attention_weight_max": 102.69132232666016,
      "activations/layer4_attention_weight_min": -93.53968048095703,
      "activations/layer5_attention_weight_max": 80.63508605957031,
      "activations/layer5_attention_weight_min": -77.00579833984375,
      "activations/layer6_attention_weight_max": 55.13248062133789,
      "activations/layer6_attention_weight_min": -50.57571792602539,
      "activations/layer7_attention_weight_max": 77.51118469238281,
      "activations/layer7_attention_weight_min": -73.41053009033203,
      "activations/layer8_attention_weight_max": 45.52424621582031,
      "activations/layer8_attention_weight_min": -45.818416595458984,
      "activations/layer9_attention_weight_max": 50.77988815307617,
      "activations/layer9_attention_weight_min": -48.79946517944336,
      "epoch": 8.51,
      "learning_rate": 9.609128787878786e-05,
      "loss": 2.8018,
      "step": 146450
    },
    {
      "activations/layer0_attention_weight_max": 14.887887954711914,
      "activations/layer0_attention_weight_min": -14.665163040161133,
      "activations/layer10_attention_weight_max": 40.32741165161133,
      "activations/layer10_attention_weight_min": -34.6050910949707,
      "activations/layer11_attention_weight_max": 31.39137077331543,
      "activations/layer11_attention_weight_min": -30.376583099365234,
      "activations/layer12_attention_weight_max": 24.28436279296875,
      "activations/layer12_attention_weight_min": -24.749263763427734,
      "activations/layer13_attention_weight_max": 41.86650466918945,
      "activations/layer13_attention_weight_min": -28.85702133178711,
      "activations/layer14_attention_weight_max": 36.91325378417969,
      "activations/layer14_attention_weight_min": -28.630990982055664,
      "activations/layer15_attention_weight_max": 36.164794921875,
      "activations/layer15_attention_weight_min": -29.157012939453125,
      "activations/layer16_attention_weight_max": 34.160614013671875,
      "activations/layer16_attention_weight_min": -28.195186614990234,
      "activations/layer17_attention_weight_max": 54.94353103637695,
      "activations/layer17_attention_weight_min": -42.98585891723633,
      "activations/layer18_attention_weight_max": 47.11209487915039,
      "activations/layer18_attention_weight_min": -36.1975212097168,
      "activations/layer19_attention_weight_max": 24.895421981811523,
      "activations/layer19_attention_weight_min": -21.282541275024414,
      "activations/layer1_attention_weight_max": 16.902034759521484,
      "activations/layer1_attention_weight_min": -15.440625190734863,
      "activations/layer20_attention_weight_max": 24.658756256103516,
      "activations/layer20_attention_weight_min": -22.14731216430664,
      "activations/layer21_attention_weight_max": 40.21802520751953,
      "activations/layer21_attention_weight_min": -23.84207534790039,
      "activations/layer22_attention_weight_max": 32.30651092529297,
      "activations/layer22_attention_weight_min": -25.975406646728516,
      "activations/layer23_attention_weight_max": 38.31488800048828,
      "activations/layer23_attention_weight_min": -25.76020622253418,
      "activations/layer2_attention_weight_max": 31.062572479248047,
      "activations/layer2_attention_weight_min": -29.59627342224121,
      "activations/layer3_attention_weight_max": 89.57257080078125,
      "activations/layer3_attention_weight_min": -91.88239288330078,
      "activations/layer4_attention_weight_max": 100.06591033935547,
      "activations/layer4_attention_weight_min": -95.7974624633789,
      "activations/layer5_attention_weight_max": 82.19937133789062,
      "activations/layer5_attention_weight_min": -77.68159484863281,
      "activations/layer6_attention_weight_max": 54.52389144897461,
      "activations/layer6_attention_weight_min": -53.02732467651367,
      "activations/layer7_attention_weight_max": 76.92694091796875,
      "activations/layer7_attention_weight_min": -75.54590606689453,
      "activations/layer8_attention_weight_max": 46.7764778137207,
      "activations/layer8_attention_weight_min": -46.921321868896484,
      "activations/layer9_attention_weight_max": 51.41744613647461,
      "activations/layer9_attention_weight_min": -47.860572814941406,
      "epoch": 8.51,
      "learning_rate": 9.607234848484848e-05,
      "loss": 2.8008,
      "step": 146500
    },
    {
      "activations/layer0_attention_weight_max": 14.493720054626465,
      "activations/layer0_attention_weight_min": -14.308895111083984,
      "activations/layer10_attention_weight_max": 44.2497673034668,
      "activations/layer10_attention_weight_min": -40.821895599365234,
      "activations/layer11_attention_weight_max": 36.117340087890625,
      "activations/layer11_attention_weight_min": -34.87789535522461,
      "activations/layer12_attention_weight_max": 21.899534225463867,
      "activations/layer12_attention_weight_min": -26.135393142700195,
      "activations/layer13_attention_weight_max": 43.83870315551758,
      "activations/layer13_attention_weight_min": -31.62640953063965,
      "activations/layer14_attention_weight_max": 42.18247985839844,
      "activations/layer14_attention_weight_min": -31.220691680908203,
      "activations/layer15_attention_weight_max": 39.78045654296875,
      "activations/layer15_attention_weight_min": -30.37106704711914,
      "activations/layer16_attention_weight_max": 34.891929626464844,
      "activations/layer16_attention_weight_min": -29.276140213012695,
      "activations/layer17_attention_weight_max": 58.17415237426758,
      "activations/layer17_attention_weight_min": -48.21358108520508,
      "activations/layer18_attention_weight_max": 53.49004364013672,
      "activations/layer18_attention_weight_min": -42.14313888549805,
      "activations/layer19_attention_weight_max": 23.967782974243164,
      "activations/layer19_attention_weight_min": -22.6761531829834,
      "activations/layer1_attention_weight_max": 16.257369995117188,
      "activations/layer1_attention_weight_min": -15.77745532989502,
      "activations/layer20_attention_weight_max": 24.30048179626465,
      "activations/layer20_attention_weight_min": -23.60101890563965,
      "activations/layer21_attention_weight_max": 49.64009475708008,
      "activations/layer21_attention_weight_min": -29.29977798461914,
      "activations/layer22_attention_weight_max": 32.98967361450195,
      "activations/layer22_attention_weight_min": -28.605310440063477,
      "activations/layer23_attention_weight_max": 38.66612243652344,
      "activations/layer23_attention_weight_min": -26.72882652282715,
      "activations/layer2_attention_weight_max": 29.99309539794922,
      "activations/layer2_attention_weight_min": -27.49374008178711,
      "activations/layer3_attention_weight_max": 88.45235443115234,
      "activations/layer3_attention_weight_min": -90.66815185546875,
      "activations/layer4_attention_weight_max": 103.21207427978516,
      "activations/layer4_attention_weight_min": -98.1891098022461,
      "activations/layer5_attention_weight_max": 82.68141174316406,
      "activations/layer5_attention_weight_min": -78.06041717529297,
      "activations/layer6_attention_weight_max": 58.544273376464844,
      "activations/layer6_attention_weight_min": -54.86864471435547,
      "activations/layer7_attention_weight_max": 84.59148406982422,
      "activations/layer7_attention_weight_min": -80.2670669555664,
      "activations/layer8_attention_weight_max": 48.929969787597656,
      "activations/layer8_attention_weight_min": -51.717437744140625,
      "activations/layer9_attention_weight_max": 55.2463264465332,
      "activations/layer9_attention_weight_min": -54.06032943725586,
      "epoch": 8.52,
      "learning_rate": 9.60534090909091e-05,
      "loss": 2.8091,
      "step": 146550
    },
    {
      "activations/layer0_attention_weight_max": 15.285365104675293,
      "activations/layer0_attention_weight_min": -15.100855827331543,
      "activations/layer10_attention_weight_max": 37.836734771728516,
      "activations/layer10_attention_weight_min": -36.3284797668457,
      "activations/layer11_attention_weight_max": 31.423221588134766,
      "activations/layer11_attention_weight_min": -31.94495391845703,
      "activations/layer12_attention_weight_max": 20.615699768066406,
      "activations/layer12_attention_weight_min": -27.105833053588867,
      "activations/layer13_attention_weight_max": 51.37648391723633,
      "activations/layer13_attention_weight_min": -36.63486862182617,
      "activations/layer14_attention_weight_max": 41.60926055908203,
      "activations/layer14_attention_weight_min": -31.24077033996582,
      "activations/layer15_attention_weight_max": 34.801795959472656,
      "activations/layer15_attention_weight_min": -29.39125633239746,
      "activations/layer16_attention_weight_max": 31.712247848510742,
      "activations/layer16_attention_weight_min": -30.173978805541992,
      "activations/layer17_attention_weight_max": 53.10466384887695,
      "activations/layer17_attention_weight_min": -42.50583267211914,
      "activations/layer18_attention_weight_max": 46.335567474365234,
      "activations/layer18_attention_weight_min": -36.36004638671875,
      "activations/layer19_attention_weight_max": 22.20029067993164,
      "activations/layer19_attention_weight_min": -21.108449935913086,
      "activations/layer1_attention_weight_max": 18.06553077697754,
      "activations/layer1_attention_weight_min": -17.11676025390625,
      "activations/layer20_attention_weight_max": 25.80629539489746,
      "activations/layer20_attention_weight_min": -23.813262939453125,
      "activations/layer21_attention_weight_max": 40.671356201171875,
      "activations/layer21_attention_weight_min": -25.92450523376465,
      "activations/layer22_attention_weight_max": 33.503665924072266,
      "activations/layer22_attention_weight_min": -25.689159393310547,
      "activations/layer23_attention_weight_max": 38.04866027832031,
      "activations/layer23_attention_weight_min": -24.315702438354492,
      "activations/layer2_attention_weight_max": 29.832542419433594,
      "activations/layer2_attention_weight_min": -28.946348190307617,
      "activations/layer3_attention_weight_max": 88.34937286376953,
      "activations/layer3_attention_weight_min": -88.49044799804688,
      "activations/layer4_attention_weight_max": 97.83251190185547,
      "activations/layer4_attention_weight_min": -91.32460021972656,
      "activations/layer5_attention_weight_max": 79.76936340332031,
      "activations/layer5_attention_weight_min": -80.39287567138672,
      "activations/layer6_attention_weight_max": 53.4236946105957,
      "activations/layer6_attention_weight_min": -52.23067855834961,
      "activations/layer7_attention_weight_max": 77.9659423828125,
      "activations/layer7_attention_weight_min": -74.29411315917969,
      "activations/layer8_attention_weight_max": 45.20631790161133,
      "activations/layer8_attention_weight_min": -47.873260498046875,
      "activations/layer9_attention_weight_max": 50.350460052490234,
      "activations/layer9_attention_weight_min": -50.41157150268555,
      "epoch": 8.52,
      "learning_rate": 9.603446969696968e-05,
      "loss": 2.8195,
      "step": 146600
    },
    {
      "activations/layer0_attention_weight_max": 14.941162109375,
      "activations/layer0_attention_weight_min": -14.43315315246582,
      "activations/layer10_attention_weight_max": 37.71843719482422,
      "activations/layer10_attention_weight_min": -35.686920166015625,
      "activations/layer11_attention_weight_max": 32.943580627441406,
      "activations/layer11_attention_weight_min": -31.829299926757812,
      "activations/layer12_attention_weight_max": 21.18599510192871,
      "activations/layer12_attention_weight_min": -25.29473114013672,
      "activations/layer13_attention_weight_max": 57.27800369262695,
      "activations/layer13_attention_weight_min": -32.518428802490234,
      "activations/layer14_attention_weight_max": 48.00028991699219,
      "activations/layer14_attention_weight_min": -31.540855407714844,
      "activations/layer15_attention_weight_max": 36.78973388671875,
      "activations/layer15_attention_weight_min": -30.787446975708008,
      "activations/layer16_attention_weight_max": 33.25314712524414,
      "activations/layer16_attention_weight_min": -27.785491943359375,
      "activations/layer17_attention_weight_max": 54.67305374145508,
      "activations/layer17_attention_weight_min": -45.1871337890625,
      "activations/layer18_attention_weight_max": 46.24750900268555,
      "activations/layer18_attention_weight_min": -36.635135650634766,
      "activations/layer19_attention_weight_max": 21.39235496520996,
      "activations/layer19_attention_weight_min": -20.440555572509766,
      "activations/layer1_attention_weight_max": 18.795574188232422,
      "activations/layer1_attention_weight_min": -14.66933822631836,
      "activations/layer20_attention_weight_max": 24.244871139526367,
      "activations/layer20_attention_weight_min": -20.15750503540039,
      "activations/layer21_attention_weight_max": 43.78801345825195,
      "activations/layer21_attention_weight_min": -22.5790958404541,
      "activations/layer22_attention_weight_max": 33.22902297973633,
      "activations/layer22_attention_weight_min": -27.46108055114746,
      "activations/layer23_attention_weight_max": 41.590370178222656,
      "activations/layer23_attention_weight_min": -20.30535316467285,
      "activations/layer2_attention_weight_max": 29.96942901611328,
      "activations/layer2_attention_weight_min": -27.73232650756836,
      "activations/layer3_attention_weight_max": 87.76575469970703,
      "activations/layer3_attention_weight_min": -89.73662567138672,
      "activations/layer4_attention_weight_max": 97.658447265625,
      "activations/layer4_attention_weight_min": -96.90975189208984,
      "activations/layer5_attention_weight_max": 77.2431869506836,
      "activations/layer5_attention_weight_min": -75.45447540283203,
      "activations/layer6_attention_weight_max": 53.817344665527344,
      "activations/layer6_attention_weight_min": -51.15548324584961,
      "activations/layer7_attention_weight_max": 84.85567474365234,
      "activations/layer7_attention_weight_min": -73.37740325927734,
      "activations/layer8_attention_weight_max": 48.94440460205078,
      "activations/layer8_attention_weight_min": -46.75293731689453,
      "activations/layer9_attention_weight_max": 63.25199508666992,
      "activations/layer9_attention_weight_min": -49.8495979309082,
      "epoch": 8.52,
      "learning_rate": 9.601590909090907e-05,
      "loss": 2.8092,
      "step": 146650
    },
    {
      "activations/layer0_attention_weight_max": 16.215665817260742,
      "activations/layer0_attention_weight_min": -14.637630462646484,
      "activations/layer10_attention_weight_max": 34.11923599243164,
      "activations/layer10_attention_weight_min": -32.11188507080078,
      "activations/layer11_attention_weight_max": 28.986387252807617,
      "activations/layer11_attention_weight_min": -30.115726470947266,
      "activations/layer12_attention_weight_max": 18.589632034301758,
      "activations/layer12_attention_weight_min": -26.534189224243164,
      "activations/layer13_attention_weight_max": 37.61314392089844,
      "activations/layer13_attention_weight_min": -29.892236709594727,
      "activations/layer14_attention_weight_max": 35.3165168762207,
      "activations/layer14_attention_weight_min": -28.888822555541992,
      "activations/layer15_attention_weight_max": 31.231914520263672,
      "activations/layer15_attention_weight_min": -29.80209732055664,
      "activations/layer16_attention_weight_max": 29.417137145996094,
      "activations/layer16_attention_weight_min": -27.636743545532227,
      "activations/layer17_attention_weight_max": 49.2415885925293,
      "activations/layer17_attention_weight_min": -40.66907501220703,
      "activations/layer18_attention_weight_max": 42.418392181396484,
      "activations/layer18_attention_weight_min": -35.307647705078125,
      "activations/layer19_attention_weight_max": 20.222604751586914,
      "activations/layer19_attention_weight_min": -22.167844772338867,
      "activations/layer1_attention_weight_max": 16.51502227783203,
      "activations/layer1_attention_weight_min": -13.512645721435547,
      "activations/layer20_attention_weight_max": 19.993093490600586,
      "activations/layer20_attention_weight_min": -23.356719970703125,
      "activations/layer21_attention_weight_max": 36.19809341430664,
      "activations/layer21_attention_weight_min": -23.85099220275879,
      "activations/layer22_attention_weight_max": 28.679658889770508,
      "activations/layer22_attention_weight_min": -26.36168670654297,
      "activations/layer23_attention_weight_max": 34.12738800048828,
      "activations/layer23_attention_weight_min": -25.1412410736084,
      "activations/layer2_attention_weight_max": 27.3718318939209,
      "activations/layer2_attention_weight_min": -24.519634246826172,
      "activations/layer3_attention_weight_max": 80.06037139892578,
      "activations/layer3_attention_weight_min": -80.73937225341797,
      "activations/layer4_attention_weight_max": 97.60192108154297,
      "activations/layer4_attention_weight_min": -91.58753204345703,
      "activations/layer5_attention_weight_max": 75.118896484375,
      "activations/layer5_attention_weight_min": -74.07085418701172,
      "activations/layer6_attention_weight_max": 53.11425018310547,
      "activations/layer6_attention_weight_min": -51.02119827270508,
      "activations/layer7_attention_weight_max": 70.48046112060547,
      "activations/layer7_attention_weight_min": -68.5675277709961,
      "activations/layer8_attention_weight_max": 42.20772171020508,
      "activations/layer8_attention_weight_min": -44.183189392089844,
      "activations/layer9_attention_weight_max": 47.754356384277344,
      "activations/layer9_attention_weight_min": -45.07023239135742,
      "epoch": 8.52,
      "learning_rate": 9.599696969696969e-05,
      "loss": 2.8175,
      "step": 146700
    },
    {
      "activations/layer0_attention_weight_max": 15.713449478149414,
      "activations/layer0_attention_weight_min": -13.540743827819824,
      "activations/layer10_attention_weight_max": 39.21196365356445,
      "activations/layer10_attention_weight_min": -35.57050323486328,
      "activations/layer11_attention_weight_max": 32.166893005371094,
      "activations/layer11_attention_weight_min": -31.553569793701172,
      "activations/layer12_attention_weight_max": 19.94339370727539,
      "activations/layer12_attention_weight_min": -26.377378463745117,
      "activations/layer13_attention_weight_max": 45.69646072387695,
      "activations/layer13_attention_weight_min": -34.38841247558594,
      "activations/layer14_attention_weight_max": 38.48973846435547,
      "activations/layer14_attention_weight_min": -30.419559478759766,
      "activations/layer15_attention_weight_max": 35.60768127441406,
      "activations/layer15_attention_weight_min": -31.446714401245117,
      "activations/layer16_attention_weight_max": 31.23063850402832,
      "activations/layer16_attention_weight_min": -28.330310821533203,
      "activations/layer17_attention_weight_max": 60.25821304321289,
      "activations/layer17_attention_weight_min": -44.99969482421875,
      "activations/layer18_attention_weight_max": 51.496116638183594,
      "activations/layer18_attention_weight_min": -37.93492126464844,
      "activations/layer19_attention_weight_max": 22.10521697998047,
      "activations/layer19_attention_weight_min": -22.352245330810547,
      "activations/layer1_attention_weight_max": 17.045143127441406,
      "activations/layer1_attention_weight_min": -15.596829414367676,
      "activations/layer20_attention_weight_max": 23.940366744995117,
      "activations/layer20_attention_weight_min": -21.317358016967773,
      "activations/layer21_attention_weight_max": 38.219268798828125,
      "activations/layer21_attention_weight_min": -24.395116806030273,
      "activations/layer22_attention_weight_max": 32.02948760986328,
      "activations/layer22_attention_weight_min": -25.91920280456543,
      "activations/layer23_attention_weight_max": 37.82471466064453,
      "activations/layer23_attention_weight_min": -23.072044372558594,
      "activations/layer2_attention_weight_max": 30.246810913085938,
      "activations/layer2_attention_weight_min": -29.31704330444336,
      "activations/layer3_attention_weight_max": 86.96200561523438,
      "activations/layer3_attention_weight_min": -92.56196594238281,
      "activations/layer4_attention_weight_max": 98.81281280517578,
      "activations/layer4_attention_weight_min": -93.09422302246094,
      "activations/layer5_attention_weight_max": 80.76942443847656,
      "activations/layer5_attention_weight_min": -74.39311218261719,
      "activations/layer6_attention_weight_max": 54.76948547363281,
      "activations/layer6_attention_weight_min": -51.25960922241211,
      "activations/layer7_attention_weight_max": 75.20211791992188,
      "activations/layer7_attention_weight_min": -75.20693969726562,
      "activations/layer8_attention_weight_max": 45.27595520019531,
      "activations/layer8_attention_weight_min": -46.345298767089844,
      "activations/layer9_attention_weight_max": 49.18490219116211,
      "activations/layer9_attention_weight_min": -48.784908294677734,
      "epoch": 8.53,
      "learning_rate": 9.59780303030303e-05,
      "loss": 2.7998,
      "step": 146750
    },
    {
      "activations/layer0_attention_weight_max": 14.928656578063965,
      "activations/layer0_attention_weight_min": -14.28467082977295,
      "activations/layer10_attention_weight_max": 35.59130859375,
      "activations/layer10_attention_weight_min": -34.31474685668945,
      "activations/layer11_attention_weight_max": 28.77696990966797,
      "activations/layer11_attention_weight_min": -28.875659942626953,
      "activations/layer12_attention_weight_max": 16.187747955322266,
      "activations/layer12_attention_weight_min": -26.029109954833984,
      "activations/layer13_attention_weight_max": 38.997703552246094,
      "activations/layer13_attention_weight_min": -28.93930435180664,
      "activations/layer14_attention_weight_max": 34.13604736328125,
      "activations/layer14_attention_weight_min": -28.634267807006836,
      "activations/layer15_attention_weight_max": 32.176937103271484,
      "activations/layer15_attention_weight_min": -28.472129821777344,
      "activations/layer16_attention_weight_max": 27.03151512145996,
      "activations/layer16_attention_weight_min": -28.681982040405273,
      "activations/layer17_attention_weight_max": 48.67831802368164,
      "activations/layer17_attention_weight_min": -41.16092300415039,
      "activations/layer18_attention_weight_max": 41.40565872192383,
      "activations/layer18_attention_weight_min": -34.98444747924805,
      "activations/layer19_attention_weight_max": 19.01018524169922,
      "activations/layer19_attention_weight_min": -20.453598022460938,
      "activations/layer1_attention_weight_max": 16.57063102722168,
      "activations/layer1_attention_weight_min": -16.316688537597656,
      "activations/layer20_attention_weight_max": 19.118913650512695,
      "activations/layer20_attention_weight_min": -22.21467399597168,
      "activations/layer21_attention_weight_max": 31.943954467773438,
      "activations/layer21_attention_weight_min": -22.177881240844727,
      "activations/layer22_attention_weight_max": 27.119518280029297,
      "activations/layer22_attention_weight_min": -24.177507400512695,
      "activations/layer23_attention_weight_max": 31.163618087768555,
      "activations/layer23_attention_weight_min": -23.631183624267578,
      "activations/layer2_attention_weight_max": 29.880218505859375,
      "activations/layer2_attention_weight_min": -28.372739791870117,
      "activations/layer3_attention_weight_max": 93.15426635742188,
      "activations/layer3_attention_weight_min": -93.13678741455078,
      "activations/layer4_attention_weight_max": 101.99536895751953,
      "activations/layer4_attention_weight_min": -96.99830627441406,
      "activations/layer5_attention_weight_max": 82.64479064941406,
      "activations/layer5_attention_weight_min": -76.57234954833984,
      "activations/layer6_attention_weight_max": 56.77885055541992,
      "activations/layer6_attention_weight_min": -54.85520553588867,
      "activations/layer7_attention_weight_max": 75.16207885742188,
      "activations/layer7_attention_weight_min": -79.9608383178711,
      "activations/layer8_attention_weight_max": 45.55552673339844,
      "activations/layer8_attention_weight_min": -46.266841888427734,
      "activations/layer9_attention_weight_max": 49.92985534667969,
      "activations/layer9_attention_weight_min": -50.22581481933594,
      "epoch": 8.53,
      "learning_rate": 9.595909090909089e-05,
      "loss": 2.8143,
      "step": 146800
    },
    {
      "activations/layer0_attention_weight_max": 15.15095329284668,
      "activations/layer0_attention_weight_min": -14.120279312133789,
      "activations/layer10_attention_weight_max": 38.97529220581055,
      "activations/layer10_attention_weight_min": -35.682640075683594,
      "activations/layer11_attention_weight_max": 32.73558044433594,
      "activations/layer11_attention_weight_min": -32.65675354003906,
      "activations/layer12_attention_weight_max": 20.642948150634766,
      "activations/layer12_attention_weight_min": -27.907499313354492,
      "activations/layer13_attention_weight_max": 48.184959411621094,
      "activations/layer13_attention_weight_min": -33.15705871582031,
      "activations/layer14_attention_weight_max": 56.55061340332031,
      "activations/layer14_attention_weight_min": -31.808395385742188,
      "activations/layer15_attention_weight_max": 38.85921096801758,
      "activations/layer15_attention_weight_min": -30.142614364624023,
      "activations/layer16_attention_weight_max": 34.178279876708984,
      "activations/layer16_attention_weight_min": -27.94137191772461,
      "activations/layer17_attention_weight_max": 61.23480224609375,
      "activations/layer17_attention_weight_min": -46.1613883972168,
      "activations/layer18_attention_weight_max": 53.7103385925293,
      "activations/layer18_attention_weight_min": -37.200260162353516,
      "activations/layer19_attention_weight_max": 26.03714942932129,
      "activations/layer19_attention_weight_min": -22.389509201049805,
      "activations/layer1_attention_weight_max": 16.295194625854492,
      "activations/layer1_attention_weight_min": -16.364032745361328,
      "activations/layer20_attention_weight_max": 24.20879554748535,
      "activations/layer20_attention_weight_min": -20.86859703063965,
      "activations/layer21_attention_weight_max": 42.7529411315918,
      "activations/layer21_attention_weight_min": -25.333831787109375,
      "activations/layer22_attention_weight_max": 33.902244567871094,
      "activations/layer22_attention_weight_min": -25.524900436401367,
      "activations/layer23_attention_weight_max": 40.88106918334961,
      "activations/layer23_attention_weight_min": -22.301315307617188,
      "activations/layer2_attention_weight_max": 30.420364379882812,
      "activations/layer2_attention_weight_min": -29.437482833862305,
      "activations/layer3_attention_weight_max": 87.02515411376953,
      "activations/layer3_attention_weight_min": -90.39459991455078,
      "activations/layer4_attention_weight_max": 103.26683807373047,
      "activations/layer4_attention_weight_min": -94.36851501464844,
      "activations/layer5_attention_weight_max": 78.397705078125,
      "activations/layer5_attention_weight_min": -80.9076919555664,
      "activations/layer6_attention_weight_max": 55.302650451660156,
      "activations/layer6_attention_weight_min": -53.4578971862793,
      "activations/layer7_attention_weight_max": 77.62730407714844,
      "activations/layer7_attention_weight_min": -73.86891174316406,
      "activations/layer8_attention_weight_max": 44.84720993041992,
      "activations/layer8_attention_weight_min": -45.3624153137207,
      "activations/layer9_attention_weight_max": 51.97575759887695,
      "activations/layer9_attention_weight_min": -48.956363677978516,
      "epoch": 8.53,
      "learning_rate": 9.59401515151515e-05,
      "loss": 2.7976,
      "step": 146850
    },
    {
      "activations/layer0_attention_weight_max": 15.922602653503418,
      "activations/layer0_attention_weight_min": -14.47598934173584,
      "activations/layer10_attention_weight_max": 38.564903259277344,
      "activations/layer10_attention_weight_min": -36.947265625,
      "activations/layer11_attention_weight_max": 30.217702865600586,
      "activations/layer11_attention_weight_min": -29.99837875366211,
      "activations/layer12_attention_weight_max": 19.8955078125,
      "activations/layer12_attention_weight_min": -26.07011604309082,
      "activations/layer13_attention_weight_max": 39.065269470214844,
      "activations/layer13_attention_weight_min": -29.529788970947266,
      "activations/layer14_attention_weight_max": 37.556480407714844,
      "activations/layer14_attention_weight_min": -29.746973037719727,
      "activations/layer15_attention_weight_max": 35.58692169189453,
      "activations/layer15_attention_weight_min": -29.404447555541992,
      "activations/layer16_attention_weight_max": 34.55220413208008,
      "activations/layer16_attention_weight_min": -27.092004776000977,
      "activations/layer17_attention_weight_max": 59.01966094970703,
      "activations/layer17_attention_weight_min": -44.3514518737793,
      "activations/layer18_attention_weight_max": 50.48985290527344,
      "activations/layer18_attention_weight_min": -39.26264572143555,
      "activations/layer19_attention_weight_max": 24.235557556152344,
      "activations/layer19_attention_weight_min": -21.204072952270508,
      "activations/layer1_attention_weight_max": 17.147613525390625,
      "activations/layer1_attention_weight_min": -14.320577621459961,
      "activations/layer20_attention_weight_max": 27.068485260009766,
      "activations/layer20_attention_weight_min": -20.298185348510742,
      "activations/layer21_attention_weight_max": 40.806182861328125,
      "activations/layer21_attention_weight_min": -22.85819435119629,
      "activations/layer22_attention_weight_max": 33.28868865966797,
      "activations/layer22_attention_weight_min": -26.6025390625,
      "activations/layer23_attention_weight_max": 41.37421798706055,
      "activations/layer23_attention_weight_min": -22.099416732788086,
      "activations/layer2_attention_weight_max": 28.87319564819336,
      "activations/layer2_attention_weight_min": -27.13785171508789,
      "activations/layer3_attention_weight_max": 86.77135467529297,
      "activations/layer3_attention_weight_min": -87.92198181152344,
      "activations/layer4_attention_weight_max": 100.03453826904297,
      "activations/layer4_attention_weight_min": -93.90697479248047,
      "activations/layer5_attention_weight_max": 80.18702697753906,
      "activations/layer5_attention_weight_min": -76.08269500732422,
      "activations/layer6_attention_weight_max": 54.94275665283203,
      "activations/layer6_attention_weight_min": -53.9518928527832,
      "activations/layer7_attention_weight_max": 78.81815338134766,
      "activations/layer7_attention_weight_min": -75.572265625,
      "activations/layer8_attention_weight_max": 47.734947204589844,
      "activations/layer8_attention_weight_min": -51.57426834106445,
      "activations/layer9_attention_weight_max": 50.63056182861328,
      "activations/layer9_attention_weight_min": -49.06414031982422,
      "epoch": 8.54,
      "learning_rate": 9.592121212121212e-05,
      "loss": 2.8028,
      "step": 146900
    },
    {
      "activations/layer0_attention_weight_max": 16.216655731201172,
      "activations/layer0_attention_weight_min": -14.842948913574219,
      "activations/layer10_attention_weight_max": 44.5371208190918,
      "activations/layer10_attention_weight_min": -39.149070739746094,
      "activations/layer11_attention_weight_max": 36.82213592529297,
      "activations/layer11_attention_weight_min": -36.448211669921875,
      "activations/layer12_attention_weight_max": 24.03056526184082,
      "activations/layer12_attention_weight_min": -25.408485412597656,
      "activations/layer13_attention_weight_max": 45.20018005371094,
      "activations/layer13_attention_weight_min": -32.89139938354492,
      "activations/layer14_attention_weight_max": 44.21807861328125,
      "activations/layer14_attention_weight_min": -30.399364471435547,
      "activations/layer15_attention_weight_max": 39.58388137817383,
      "activations/layer15_attention_weight_min": -28.724777221679688,
      "activations/layer16_attention_weight_max": 34.24070739746094,
      "activations/layer16_attention_weight_min": -28.049880981445312,
      "activations/layer17_attention_weight_max": 61.58050537109375,
      "activations/layer17_attention_weight_min": -45.27925491333008,
      "activations/layer18_attention_weight_max": 54.883705139160156,
      "activations/layer18_attention_weight_min": -38.09557342529297,
      "activations/layer19_attention_weight_max": 23.953601837158203,
      "activations/layer19_attention_weight_min": -20.981374740600586,
      "activations/layer1_attention_weight_max": 17.334522247314453,
      "activations/layer1_attention_weight_min": -14.064229011535645,
      "activations/layer20_attention_weight_max": 25.83656883239746,
      "activations/layer20_attention_weight_min": -23.790782928466797,
      "activations/layer21_attention_weight_max": 47.88197708129883,
      "activations/layer21_attention_weight_min": -25.10489273071289,
      "activations/layer22_attention_weight_max": 34.96623229980469,
      "activations/layer22_attention_weight_min": -26.9173526763916,
      "activations/layer23_attention_weight_max": 40.074195861816406,
      "activations/layer23_attention_weight_min": -21.63178825378418,
      "activations/layer2_attention_weight_max": 30.88275146484375,
      "activations/layer2_attention_weight_min": -29.64887237548828,
      "activations/layer3_attention_weight_max": 91.3416748046875,
      "activations/layer3_attention_weight_min": -98.3452377319336,
      "activations/layer4_attention_weight_max": 105.90431213378906,
      "activations/layer4_attention_weight_min": -98.54541778564453,
      "activations/layer5_attention_weight_max": 82.7808609008789,
      "activations/layer5_attention_weight_min": -79.56837463378906,
      "activations/layer6_attention_weight_max": 58.4456672668457,
      "activations/layer6_attention_weight_min": -58.713539123535156,
      "activations/layer7_attention_weight_max": 80.95475769042969,
      "activations/layer7_attention_weight_min": -75.20818328857422,
      "activations/layer8_attention_weight_max": 46.80154037475586,
      "activations/layer8_attention_weight_min": -47.11931610107422,
      "activations/layer9_attention_weight_max": 53.71305465698242,
      "activations/layer9_attention_weight_min": -51.96113204956055,
      "epoch": 8.54,
      "learning_rate": 9.590227272727272e-05,
      "loss": 2.7926,
      "step": 146950
    },
    {
      "activations/layer0_attention_weight_max": 15.519079208374023,
      "activations/layer0_attention_weight_min": -14.292579650878906,
      "activations/layer10_attention_weight_max": 35.89876174926758,
      "activations/layer10_attention_weight_min": -33.93169021606445,
      "activations/layer11_attention_weight_max": 31.094066619873047,
      "activations/layer11_attention_weight_min": -32.04764938354492,
      "activations/layer12_attention_weight_max": 21.101337432861328,
      "activations/layer12_attention_weight_min": -23.56951141357422,
      "activations/layer13_attention_weight_max": 41.77100372314453,
      "activations/layer13_attention_weight_min": -32.52314376831055,
      "activations/layer14_attention_weight_max": 45.62778091430664,
      "activations/layer14_attention_weight_min": -28.433815002441406,
      "activations/layer15_attention_weight_max": 40.72499465942383,
      "activations/layer15_attention_weight_min": -29.035110473632812,
      "activations/layer16_attention_weight_max": 33.30086898803711,
      "activations/layer16_attention_weight_min": -27.269609451293945,
      "activations/layer17_attention_weight_max": 59.90150833129883,
      "activations/layer17_attention_weight_min": -43.3272819519043,
      "activations/layer18_attention_weight_max": 44.98594284057617,
      "activations/layer18_attention_weight_min": -36.10334777832031,
      "activations/layer19_attention_weight_max": 24.22178840637207,
      "activations/layer19_attention_weight_min": -20.75731658935547,
      "activations/layer1_attention_weight_max": 17.062299728393555,
      "activations/layer1_attention_weight_min": -15.500624656677246,
      "activations/layer20_attention_weight_max": 23.147031784057617,
      "activations/layer20_attention_weight_min": -24.020971298217773,
      "activations/layer21_attention_weight_max": 37.26091003417969,
      "activations/layer21_attention_weight_min": -21.33428382873535,
      "activations/layer22_attention_weight_max": 34.83172607421875,
      "activations/layer22_attention_weight_min": -24.07155990600586,
      "activations/layer23_attention_weight_max": 33.195499420166016,
      "activations/layer23_attention_weight_min": -21.31848907470703,
      "activations/layer2_attention_weight_max": 32.30019760131836,
      "activations/layer2_attention_weight_min": -30.61174774169922,
      "activations/layer3_attention_weight_max": 89.18966674804688,
      "activations/layer3_attention_weight_min": -91.56378173828125,
      "activations/layer4_attention_weight_max": 99.84561157226562,
      "activations/layer4_attention_weight_min": -95.51226806640625,
      "activations/layer5_attention_weight_max": 79.34233856201172,
      "activations/layer5_attention_weight_min": -76.8614730834961,
      "activations/layer6_attention_weight_max": 52.75421905517578,
      "activations/layer6_attention_weight_min": -51.36040496826172,
      "activations/layer7_attention_weight_max": 77.2548599243164,
      "activations/layer7_attention_weight_min": -71.91089630126953,
      "activations/layer8_attention_weight_max": 43.0634880065918,
      "activations/layer8_attention_weight_min": -44.176513671875,
      "activations/layer9_attention_weight_max": 57.51417922973633,
      "activations/layer9_attention_weight_min": -48.07415771484375,
      "epoch": 8.54,
      "learning_rate": 9.588333333333332e-05,
      "loss": 2.7871,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.4985,
      "eval_samples_per_second": 505.264,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.4985,
      "eval_openwebtext_samples_per_second": 505.264,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 1.9992,
      "eval_wikitext_samples_per_second": 228.086,
      "step": 147000
    },
    {
      "epoch": 8.54,
      "eval_lambada_loss": 2.6640625,
      "eval_lambada_ppl": 14.354485905365031,
      "eval_lambada_runtime": 9.6041,
      "eval_lambada_samples_per_second": 506.971,
      "step": 147000
    },
    {
      "activations/layer0_attention_weight_max": 14.70541000366211,
      "activations/layer0_attention_weight_min": -14.492138862609863,
      "activations/layer10_attention_weight_max": 38.27173614501953,
      "activations/layer10_attention_weight_min": -35.01612091064453,
      "activations/layer11_attention_weight_max": 33.0987434387207,
      "activations/layer11_attention_weight_min": -32.1224365234375,
      "activations/layer12_attention_weight_max": 18.517168045043945,
      "activations/layer12_attention_weight_min": -32.9070930480957,
      "activations/layer13_attention_weight_max": 39.93775177001953,
      "activations/layer13_attention_weight_min": -29.614675521850586,
      "activations/layer14_attention_weight_max": 37.26911163330078,
      "activations/layer14_attention_weight_min": -30.628864288330078,
      "activations/layer15_attention_weight_max": 32.78425979614258,
      "activations/layer15_attention_weight_min": -28.55988121032715,
      "activations/layer16_attention_weight_max": 30.163110733032227,
      "activations/layer16_attention_weight_min": -27.684680938720703,
      "activations/layer17_attention_weight_max": 58.01478958129883,
      "activations/layer17_attention_weight_min": -41.18124008178711,
      "activations/layer18_attention_weight_max": 48.79981994628906,
      "activations/layer18_attention_weight_min": -36.99872970581055,
      "activations/layer19_attention_weight_max": 20.928789138793945,
      "activations/layer19_attention_weight_min": -20.848922729492188,
      "activations/layer1_attention_weight_max": 17.8107852935791,
      "activations/layer1_attention_weight_min": -14.786648750305176,
      "activations/layer20_attention_weight_max": 21.285367965698242,
      "activations/layer20_attention_weight_min": -20.428424835205078,
      "activations/layer21_attention_weight_max": 39.59238052368164,
      "activations/layer21_attention_weight_min": -24.37455177307129,
      "activations/layer22_attention_weight_max": 28.937816619873047,
      "activations/layer22_attention_weight_min": -25.640262603759766,
      "activations/layer23_attention_weight_max": 32.3198127746582,
      "activations/layer23_attention_weight_min": -22.714000701904297,
      "activations/layer2_attention_weight_max": 30.948246002197266,
      "activations/layer2_attention_weight_min": -30.09207534790039,
      "activations/layer3_attention_weight_max": 90.72725677490234,
      "activations/layer3_attention_weight_min": -93.00269317626953,
      "activations/layer4_attention_weight_max": 103.88858795166016,
      "activations/layer4_attention_weight_min": -97.67117309570312,
      "activations/layer5_attention_weight_max": 79.61201477050781,
      "activations/layer5_attention_weight_min": -75.91221618652344,
      "activations/layer6_attention_weight_max": 54.31493377685547,
      "activations/layer6_attention_weight_min": -51.86083221435547,
      "activations/layer7_attention_weight_max": 84.49044799804688,
      "activations/layer7_attention_weight_min": -73.94314575195312,
      "activations/layer8_attention_weight_max": 45.23819351196289,
      "activations/layer8_attention_weight_min": -47.558284759521484,
      "activations/layer9_attention_weight_max": 60.76655197143555,
      "activations/layer9_attention_weight_min": -55.06531524658203,
      "epoch": 8.54,
      "learning_rate": 9.586439393939393e-05,
      "loss": 2.8047,
      "step": 147050
    },
    {
      "activations/layer0_attention_weight_max": 15.62624740600586,
      "activations/layer0_attention_weight_min": -14.366003036499023,
      "activations/layer10_attention_weight_max": 36.363136291503906,
      "activations/layer10_attention_weight_min": -33.65301513671875,
      "activations/layer11_attention_weight_max": 30.12678337097168,
      "activations/layer11_attention_weight_min": -32.25858688354492,
      "activations/layer12_attention_weight_max": 16.61862564086914,
      "activations/layer12_attention_weight_min": -23.934141159057617,
      "activations/layer13_attention_weight_max": 39.88727951049805,
      "activations/layer13_attention_weight_min": -29.74457550048828,
      "activations/layer14_attention_weight_max": 37.25932312011719,
      "activations/layer14_attention_weight_min": -31.383556365966797,
      "activations/layer15_attention_weight_max": 33.14875411987305,
      "activations/layer15_attention_weight_min": -30.132335662841797,
      "activations/layer16_attention_weight_max": 28.73785972595215,
      "activations/layer16_attention_weight_min": -28.584712982177734,
      "activations/layer17_attention_weight_max": 54.2717170715332,
      "activations/layer17_attention_weight_min": -44.93513870239258,
      "activations/layer18_attention_weight_max": 48.597625732421875,
      "activations/layer18_attention_weight_min": -38.264678955078125,
      "activations/layer19_attention_weight_max": 22.50201988220215,
      "activations/layer19_attention_weight_min": -20.785202026367188,
      "activations/layer1_attention_weight_max": 16.25090217590332,
      "activations/layer1_attention_weight_min": -14.238079071044922,
      "activations/layer20_attention_weight_max": 22.207765579223633,
      "activations/layer20_attention_weight_min": -20.61629867553711,
      "activations/layer21_attention_weight_max": 38.341766357421875,
      "activations/layer21_attention_weight_min": -21.09278106689453,
      "activations/layer22_attention_weight_max": 32.27898025512695,
      "activations/layer22_attention_weight_min": -26.81266212463379,
      "activations/layer23_attention_weight_max": 34.78494644165039,
      "activations/layer23_attention_weight_min": -23.86981964111328,
      "activations/layer2_attention_weight_max": 29.33526611328125,
      "activations/layer2_attention_weight_min": -28.691099166870117,
      "activations/layer3_attention_weight_max": 86.63508605957031,
      "activations/layer3_attention_weight_min": -90.93314361572266,
      "activations/layer4_attention_weight_max": 101.04313659667969,
      "activations/layer4_attention_weight_min": -98.66936492919922,
      "activations/layer5_attention_weight_max": 81.78170013427734,
      "activations/layer5_attention_weight_min": -78.33456420898438,
      "activations/layer6_attention_weight_max": 53.56502914428711,
      "activations/layer6_attention_weight_min": -52.56272888183594,
      "activations/layer7_attention_weight_max": 79.87086486816406,
      "activations/layer7_attention_weight_min": -74.25748443603516,
      "activations/layer8_attention_weight_max": 45.79340362548828,
      "activations/layer8_attention_weight_min": -44.97453689575195,
      "activations/layer9_attention_weight_max": 48.82874298095703,
      "activations/layer9_attention_weight_min": -48.56095886230469,
      "epoch": 8.55,
      "learning_rate": 9.584545454545454e-05,
      "loss": 2.8099,
      "step": 147100
    },
    {
      "activations/layer0_attention_weight_max": 14.855819702148438,
      "activations/layer0_attention_weight_min": -14.305088996887207,
      "activations/layer10_attention_weight_max": 38.318092346191406,
      "activations/layer10_attention_weight_min": -38.50388717651367,
      "activations/layer11_attention_weight_max": 30.87759017944336,
      "activations/layer11_attention_weight_min": -33.489078521728516,
      "activations/layer12_attention_weight_max": 18.579166412353516,
      "activations/layer12_attention_weight_min": -24.58549690246582,
      "activations/layer13_attention_weight_max": 40.349700927734375,
      "activations/layer13_attention_weight_min": -32.99169921875,
      "activations/layer14_attention_weight_max": 42.62454605102539,
      "activations/layer14_attention_weight_min": -30.631450653076172,
      "activations/layer15_attention_weight_max": 31.7755069732666,
      "activations/layer15_attention_weight_min": -31.16022491455078,
      "activations/layer16_attention_weight_max": 31.311063766479492,
      "activations/layer16_attention_weight_min": -28.497844696044922,
      "activations/layer17_attention_weight_max": 51.72069549560547,
      "activations/layer17_attention_weight_min": -42.384700775146484,
      "activations/layer18_attention_weight_max": 44.67219161987305,
      "activations/layer18_attention_weight_min": -37.18715286254883,
      "activations/layer19_attention_weight_max": 21.612443923950195,
      "activations/layer19_attention_weight_min": -26.910446166992188,
      "activations/layer1_attention_weight_max": 16.406057357788086,
      "activations/layer1_attention_weight_min": -14.913653373718262,
      "activations/layer20_attention_weight_max": 21.51711082458496,
      "activations/layer20_attention_weight_min": -22.95169448852539,
      "activations/layer21_attention_weight_max": 37.88612747192383,
      "activations/layer21_attention_weight_min": -23.511638641357422,
      "activations/layer22_attention_weight_max": 32.08824157714844,
      "activations/layer22_attention_weight_min": -28.62469482421875,
      "activations/layer23_attention_weight_max": 35.69296646118164,
      "activations/layer23_attention_weight_min": -22.823013305664062,
      "activations/layer2_attention_weight_max": 30.55588150024414,
      "activations/layer2_attention_weight_min": -28.539291381835938,
      "activations/layer3_attention_weight_max": 88.68345642089844,
      "activations/layer3_attention_weight_min": -89.41292572021484,
      "activations/layer4_attention_weight_max": 99.64300537109375,
      "activations/layer4_attention_weight_min": -94.09607696533203,
      "activations/layer5_attention_weight_max": 81.8126220703125,
      "activations/layer5_attention_weight_min": -77.91424560546875,
      "activations/layer6_attention_weight_max": 54.71625518798828,
      "activations/layer6_attention_weight_min": -53.13103485107422,
      "activations/layer7_attention_weight_max": 80.87647247314453,
      "activations/layer7_attention_weight_min": -71.78705596923828,
      "activations/layer8_attention_weight_max": 43.69350051879883,
      "activations/layer8_attention_weight_min": -46.1548957824707,
      "activations/layer9_attention_weight_max": 48.99901580810547,
      "activations/layer9_attention_weight_min": -48.18495178222656,
      "epoch": 8.55,
      "learning_rate": 9.582651515151514e-05,
      "loss": 2.8181,
      "step": 147150
    },
    {
      "activations/layer0_attention_weight_max": 14.388849258422852,
      "activations/layer0_attention_weight_min": -14.192124366760254,
      "activations/layer10_attention_weight_max": 37.515235900878906,
      "activations/layer10_attention_weight_min": -34.27068328857422,
      "activations/layer11_attention_weight_max": 30.845075607299805,
      "activations/layer11_attention_weight_min": -31.745136260986328,
      "activations/layer12_attention_weight_max": 18.674388885498047,
      "activations/layer12_attention_weight_min": -25.8391170501709,
      "activations/layer13_attention_weight_max": 37.46839141845703,
      "activations/layer13_attention_weight_min": -28.304092407226562,
      "activations/layer14_attention_weight_max": 38.14596176147461,
      "activations/layer14_attention_weight_min": -29.557342529296875,
      "activations/layer15_attention_weight_max": 33.64656066894531,
      "activations/layer15_attention_weight_min": -28.04725456237793,
      "activations/layer16_attention_weight_max": 30.161441802978516,
      "activations/layer16_attention_weight_min": -28.255191802978516,
      "activations/layer17_attention_weight_max": 53.24531555175781,
      "activations/layer17_attention_weight_min": -43.0218505859375,
      "activations/layer18_attention_weight_max": 47.17648696899414,
      "activations/layer18_attention_weight_min": -37.32863998413086,
      "activations/layer19_attention_weight_max": 21.709800720214844,
      "activations/layer19_attention_weight_min": -21.163249969482422,
      "activations/layer1_attention_weight_max": 16.30674934387207,
      "activations/layer1_attention_weight_min": -13.742561340332031,
      "activations/layer20_attention_weight_max": 21.798477172851562,
      "activations/layer20_attention_weight_min": -20.76305389404297,
      "activations/layer21_attention_weight_max": 43.23991775512695,
      "activations/layer21_attention_weight_min": -26.063255310058594,
      "activations/layer22_attention_weight_max": 29.897315979003906,
      "activations/layer22_attention_weight_min": -26.658302307128906,
      "activations/layer23_attention_weight_max": 35.329811096191406,
      "activations/layer23_attention_weight_min": -21.925296783447266,
      "activations/layer2_attention_weight_max": 29.581741333007812,
      "activations/layer2_attention_weight_min": -28.439308166503906,
      "activations/layer3_attention_weight_max": 88.57609558105469,
      "activations/layer3_attention_weight_min": -90.82308959960938,
      "activations/layer4_attention_weight_max": 99.16102600097656,
      "activations/layer4_attention_weight_min": -94.52962493896484,
      "activations/layer5_attention_weight_max": 83.3481674194336,
      "activations/layer5_attention_weight_min": -76.35919189453125,
      "activations/layer6_attention_weight_max": 55.3564453125,
      "activations/layer6_attention_weight_min": -52.37800216674805,
      "activations/layer7_attention_weight_max": 79.0605239868164,
      "activations/layer7_attention_weight_min": -72.60237884521484,
      "activations/layer8_attention_weight_max": 48.905277252197266,
      "activations/layer8_attention_weight_min": -45.44719314575195,
      "activations/layer9_attention_weight_max": 51.35831832885742,
      "activations/layer9_attention_weight_min": -49.77252197265625,
      "epoch": 8.55,
      "learning_rate": 9.580757575757574e-05,
      "loss": 2.7924,
      "step": 147200
    },
    {
      "activations/layer0_attention_weight_max": 14.84965991973877,
      "activations/layer0_attention_weight_min": -14.767860412597656,
      "activations/layer10_attention_weight_max": 36.51548385620117,
      "activations/layer10_attention_weight_min": -34.2123908996582,
      "activations/layer11_attention_weight_max": 30.095836639404297,
      "activations/layer11_attention_weight_min": -32.4066047668457,
      "activations/layer12_attention_weight_max": 24.10236167907715,
      "activations/layer12_attention_weight_min": -24.987937927246094,
      "activations/layer13_attention_weight_max": 42.06287384033203,
      "activations/layer13_attention_weight_min": -29.64559555053711,
      "activations/layer14_attention_weight_max": 47.92080307006836,
      "activations/layer14_attention_weight_min": -30.59246063232422,
      "activations/layer15_attention_weight_max": 38.39695358276367,
      "activations/layer15_attention_weight_min": -29.03927993774414,
      "activations/layer16_attention_weight_max": 35.33535385131836,
      "activations/layer16_attention_weight_min": -29.229223251342773,
      "activations/layer17_attention_weight_max": 59.55865478515625,
      "activations/layer17_attention_weight_min": -44.731143951416016,
      "activations/layer18_attention_weight_max": 54.14224624633789,
      "activations/layer18_attention_weight_min": -37.671749114990234,
      "activations/layer19_attention_weight_max": 23.942975997924805,
      "activations/layer19_attention_weight_min": -20.746328353881836,
      "activations/layer1_attention_weight_max": 16.919435501098633,
      "activations/layer1_attention_weight_min": -13.86097240447998,
      "activations/layer20_attention_weight_max": 24.242399215698242,
      "activations/layer20_attention_weight_min": -24.763811111450195,
      "activations/layer21_attention_weight_max": 36.731719970703125,
      "activations/layer21_attention_weight_min": -21.84427261352539,
      "activations/layer22_attention_weight_max": 40.604881286621094,
      "activations/layer22_attention_weight_min": -23.78806495666504,
      "activations/layer23_attention_weight_max": 45.5999870300293,
      "activations/layer23_attention_weight_min": -25.073749542236328,
      "activations/layer2_attention_weight_max": 30.769813537597656,
      "activations/layer2_attention_weight_min": -29.134794235229492,
      "activations/layer3_attention_weight_max": 94.31135559082031,
      "activations/layer3_attention_weight_min": -93.2367935180664,
      "activations/layer4_attention_weight_max": 101.048095703125,
      "activations/layer4_attention_weight_min": -95.22350311279297,
      "activations/layer5_attention_weight_max": 79.85698699951172,
      "activations/layer5_attention_weight_min": -75.91559600830078,
      "activations/layer6_attention_weight_max": 53.874183654785156,
      "activations/layer6_attention_weight_min": -52.44233322143555,
      "activations/layer7_attention_weight_max": 71.95130920410156,
      "activations/layer7_attention_weight_min": -71.43147277832031,
      "activations/layer8_attention_weight_max": 44.232364654541016,
      "activations/layer8_attention_weight_min": -47.104774475097656,
      "activations/layer9_attention_weight_max": 50.80180358886719,
      "activations/layer9_attention_weight_min": -47.67782211303711,
      "epoch": 8.56,
      "learning_rate": 9.578863636363636e-05,
      "loss": 2.8178,
      "step": 147250
    },
    {
      "activations/layer0_attention_weight_max": 14.666719436645508,
      "activations/layer0_attention_weight_min": -14.28100299835205,
      "activations/layer10_attention_weight_max": 36.79705047607422,
      "activations/layer10_attention_weight_min": -34.7441520690918,
      "activations/layer11_attention_weight_max": 33.39228820800781,
      "activations/layer11_attention_weight_min": -31.231298446655273,
      "activations/layer12_attention_weight_max": 19.40475082397461,
      "activations/layer12_attention_weight_min": -22.271320343017578,
      "activations/layer13_attention_weight_max": 43.20162582397461,
      "activations/layer13_attention_weight_min": -32.3001823425293,
      "activations/layer14_attention_weight_max": 37.8190803527832,
      "activations/layer14_attention_weight_min": -31.633167266845703,
      "activations/layer15_attention_weight_max": 37.04380798339844,
      "activations/layer15_attention_weight_min": -28.514245986938477,
      "activations/layer16_attention_weight_max": 30.422439575195312,
      "activations/layer16_attention_weight_min": -29.31424331665039,
      "activations/layer17_attention_weight_max": 53.46289825439453,
      "activations/layer17_attention_weight_min": -41.64948272705078,
      "activations/layer18_attention_weight_max": 45.06993865966797,
      "activations/layer18_attention_weight_min": -37.6428337097168,
      "activations/layer19_attention_weight_max": 23.653034210205078,
      "activations/layer19_attention_weight_min": -21.680343627929688,
      "activations/layer1_attention_weight_max": 17.12726593017578,
      "activations/layer1_attention_weight_min": -14.39209270477295,
      "activations/layer20_attention_weight_max": 22.999309539794922,
      "activations/layer20_attention_weight_min": -22.086450576782227,
      "activations/layer21_attention_weight_max": 36.208396911621094,
      "activations/layer21_attention_weight_min": -23.387527465820312,
      "activations/layer22_attention_weight_max": 32.36459732055664,
      "activations/layer22_attention_weight_min": -26.071687698364258,
      "activations/layer23_attention_weight_max": 38.18531036376953,
      "activations/layer23_attention_weight_min": -23.998523712158203,
      "activations/layer2_attention_weight_max": 30.238300323486328,
      "activations/layer2_attention_weight_min": -29.711423873901367,
      "activations/layer3_attention_weight_max": 85.09465789794922,
      "activations/layer3_attention_weight_min": -93.6899185180664,
      "activations/layer4_attention_weight_max": 103.24488830566406,
      "activations/layer4_attention_weight_min": -95.89922332763672,
      "activations/layer5_attention_weight_max": 79.49614715576172,
      "activations/layer5_attention_weight_min": -76.5605697631836,
      "activations/layer6_attention_weight_max": 52.82912826538086,
      "activations/layer6_attention_weight_min": -51.969505310058594,
      "activations/layer7_attention_weight_max": 75.6739501953125,
      "activations/layer7_attention_weight_min": -74.63355255126953,
      "activations/layer8_attention_weight_max": 42.10673904418945,
      "activations/layer8_attention_weight_min": -45.1912727355957,
      "activations/layer9_attention_weight_max": 45.63694381713867,
      "activations/layer9_attention_weight_min": -47.400489807128906,
      "epoch": 8.56,
      "learning_rate": 9.576969696969697e-05,
      "loss": 2.8206,
      "step": 147300
    },
    {
      "activations/layer0_attention_weight_max": 15.303470611572266,
      "activations/layer0_attention_weight_min": -13.95536994934082,
      "activations/layer10_attention_weight_max": 35.59511947631836,
      "activations/layer10_attention_weight_min": -35.73298263549805,
      "activations/layer11_attention_weight_max": 30.967453002929688,
      "activations/layer11_attention_weight_min": -30.84986686706543,
      "activations/layer12_attention_weight_max": 20.341081619262695,
      "activations/layer12_attention_weight_min": -23.589195251464844,
      "activations/layer13_attention_weight_max": 41.42391586303711,
      "activations/layer13_attention_weight_min": -33.529998779296875,
      "activations/layer14_attention_weight_max": 36.31315612792969,
      "activations/layer14_attention_weight_min": -31.366321563720703,
      "activations/layer15_attention_weight_max": 35.35236740112305,
      "activations/layer15_attention_weight_min": -28.340991973876953,
      "activations/layer16_attention_weight_max": 31.87357521057129,
      "activations/layer16_attention_weight_min": -28.795963287353516,
      "activations/layer17_attention_weight_max": 53.20315170288086,
      "activations/layer17_attention_weight_min": -44.20990753173828,
      "activations/layer18_attention_weight_max": 47.67439651489258,
      "activations/layer18_attention_weight_min": -38.5279541015625,
      "activations/layer19_attention_weight_max": 21.368305206298828,
      "activations/layer19_attention_weight_min": -22.790369033813477,
      "activations/layer1_attention_weight_max": 16.523550033569336,
      "activations/layer1_attention_weight_min": -13.980110168457031,
      "activations/layer20_attention_weight_max": 24.326366424560547,
      "activations/layer20_attention_weight_min": -22.221458435058594,
      "activations/layer21_attention_weight_max": 38.12739181518555,
      "activations/layer21_attention_weight_min": -22.62813949584961,
      "activations/layer22_attention_weight_max": 34.02079391479492,
      "activations/layer22_attention_weight_min": -25.8758602142334,
      "activations/layer23_attention_weight_max": 37.468833923339844,
      "activations/layer23_attention_weight_min": -24.509624481201172,
      "activations/layer2_attention_weight_max": 29.335378646850586,
      "activations/layer2_attention_weight_min": -28.784934997558594,
      "activations/layer3_attention_weight_max": 89.26570892333984,
      "activations/layer3_attention_weight_min": -88.4693832397461,
      "activations/layer4_attention_weight_max": 100.92807006835938,
      "activations/layer4_attention_weight_min": -93.40409851074219,
      "activations/layer5_attention_weight_max": 75.77703857421875,
      "activations/layer5_attention_weight_min": -74.01722717285156,
      "activations/layer6_attention_weight_max": 53.92890548706055,
      "activations/layer6_attention_weight_min": -49.96415710449219,
      "activations/layer7_attention_weight_max": 70.76490783691406,
      "activations/layer7_attention_weight_min": -67.47027587890625,
      "activations/layer8_attention_weight_max": 44.3301887512207,
      "activations/layer8_attention_weight_min": -42.11240005493164,
      "activations/layer9_attention_weight_max": 47.569732666015625,
      "activations/layer9_attention_weight_min": -47.51264190673828,
      "epoch": 8.56,
      "learning_rate": 9.575075757575756e-05,
      "loss": 2.7929,
      "step": 147350
    },
    {
      "activations/layer0_attention_weight_max": 15.059362411499023,
      "activations/layer0_attention_weight_min": -14.4242525100708,
      "activations/layer10_attention_weight_max": 35.91751480102539,
      "activations/layer10_attention_weight_min": -33.60785675048828,
      "activations/layer11_attention_weight_max": 30.278976440429688,
      "activations/layer11_attention_weight_min": -30.018741607666016,
      "activations/layer12_attention_weight_max": 18.3118839263916,
      "activations/layer12_attention_weight_min": -27.53237533569336,
      "activations/layer13_attention_weight_max": 43.261356353759766,
      "activations/layer13_attention_weight_min": -30.27701187133789,
      "activations/layer14_attention_weight_max": 37.64461898803711,
      "activations/layer14_attention_weight_min": -28.80911636352539,
      "activations/layer15_attention_weight_max": 33.65105438232422,
      "activations/layer15_attention_weight_min": -28.38736915588379,
      "activations/layer16_attention_weight_max": 30.539827346801758,
      "activations/layer16_attention_weight_min": -27.6943359375,
      "activations/layer17_attention_weight_max": 51.53288650512695,
      "activations/layer17_attention_weight_min": -41.20595169067383,
      "activations/layer18_attention_weight_max": 49.16775894165039,
      "activations/layer18_attention_weight_min": -35.489959716796875,
      "activations/layer19_attention_weight_max": 24.86098861694336,
      "activations/layer19_attention_weight_min": -20.832754135131836,
      "activations/layer1_attention_weight_max": 17.377994537353516,
      "activations/layer1_attention_weight_min": -15.10229778289795,
      "activations/layer20_attention_weight_max": 21.510787963867188,
      "activations/layer20_attention_weight_min": -22.4145450592041,
      "activations/layer21_attention_weight_max": 39.513736724853516,
      "activations/layer21_attention_weight_min": -21.938100814819336,
      "activations/layer22_attention_weight_max": 33.75128173828125,
      "activations/layer22_attention_weight_min": -26.01789093017578,
      "activations/layer23_attention_weight_max": 35.432640075683594,
      "activations/layer23_attention_weight_min": -22.7299861907959,
      "activations/layer2_attention_weight_max": 30.413917541503906,
      "activations/layer2_attention_weight_min": -29.572744369506836,
      "activations/layer3_attention_weight_max": 89.86610412597656,
      "activations/layer3_attention_weight_min": -91.83992004394531,
      "activations/layer4_attention_weight_max": 101.53707122802734,
      "activations/layer4_attention_weight_min": -94.2444076538086,
      "activations/layer5_attention_weight_max": 76.9516372680664,
      "activations/layer5_attention_weight_min": -75.37071228027344,
      "activations/layer6_attention_weight_max": 54.22992706298828,
      "activations/layer6_attention_weight_min": -50.39146041870117,
      "activations/layer7_attention_weight_max": 76.77779388427734,
      "activations/layer7_attention_weight_min": -70.66390991210938,
      "activations/layer8_attention_weight_max": 42.37976837158203,
      "activations/layer8_attention_weight_min": -41.652923583984375,
      "activations/layer9_attention_weight_max": 49.60422134399414,
      "activations/layer9_attention_weight_min": -46.23940658569336,
      "epoch": 8.56,
      "learning_rate": 9.573181818181818e-05,
      "loss": 2.8084,
      "step": 147400
    },
    {
      "activations/layer0_attention_weight_max": 15.30277156829834,
      "activations/layer0_attention_weight_min": -14.206329345703125,
      "activations/layer10_attention_weight_max": 36.6549072265625,
      "activations/layer10_attention_weight_min": -35.562774658203125,
      "activations/layer11_attention_weight_max": 30.84497833251953,
      "activations/layer11_attention_weight_min": -30.029212951660156,
      "activations/layer12_attention_weight_max": 23.266265869140625,
      "activations/layer12_attention_weight_min": -24.845548629760742,
      "activations/layer13_attention_weight_max": 47.02344512939453,
      "activations/layer13_attention_weight_min": -32.82220458984375,
      "activations/layer14_attention_weight_max": 37.659202575683594,
      "activations/layer14_attention_weight_min": -29.856945037841797,
      "activations/layer15_attention_weight_max": 34.41636657714844,
      "activations/layer15_attention_weight_min": -29.582563400268555,
      "activations/layer16_attention_weight_max": 32.62541580200195,
      "activations/layer16_attention_weight_min": -27.753150939941406,
      "activations/layer17_attention_weight_max": 54.95940399169922,
      "activations/layer17_attention_weight_min": -42.64192581176758,
      "activations/layer18_attention_weight_max": 48.48339080810547,
      "activations/layer18_attention_weight_min": -34.47311019897461,
      "activations/layer19_attention_weight_max": 22.359411239624023,
      "activations/layer19_attention_weight_min": -20.228940963745117,
      "activations/layer1_attention_weight_max": 17.30504035949707,
      "activations/layer1_attention_weight_min": -15.455066680908203,
      "activations/layer20_attention_weight_max": 22.974143981933594,
      "activations/layer20_attention_weight_min": -23.877830505371094,
      "activations/layer21_attention_weight_max": 43.17854690551758,
      "activations/layer21_attention_weight_min": -26.56307601928711,
      "activations/layer22_attention_weight_max": 31.771352767944336,
      "activations/layer22_attention_weight_min": -26.179367065429688,
      "activations/layer23_attention_weight_max": 38.7440071105957,
      "activations/layer23_attention_weight_min": -20.40736198425293,
      "activations/layer2_attention_weight_max": 33.0686149597168,
      "activations/layer2_attention_weight_min": -29.23273468017578,
      "activations/layer3_attention_weight_max": 94.37615966796875,
      "activations/layer3_attention_weight_min": -98.77922821044922,
      "activations/layer4_attention_weight_max": 99.07561492919922,
      "activations/layer4_attention_weight_min": -97.59468078613281,
      "activations/layer5_attention_weight_max": 78.2786865234375,
      "activations/layer5_attention_weight_min": -75.49443817138672,
      "activations/layer6_attention_weight_max": 53.70069122314453,
      "activations/layer6_attention_weight_min": -53.17621994018555,
      "activations/layer7_attention_weight_max": 74.37417602539062,
      "activations/layer7_attention_weight_min": -70.31287384033203,
      "activations/layer8_attention_weight_max": 45.65180206298828,
      "activations/layer8_attention_weight_min": -46.13614273071289,
      "activations/layer9_attention_weight_max": 49.794464111328125,
      "activations/layer9_attention_weight_min": -49.620521545410156,
      "epoch": 8.57,
      "learning_rate": 9.571287878787878e-05,
      "loss": 2.8251,
      "step": 147450
    },
    {
      "activations/layer0_attention_weight_max": 14.869656562805176,
      "activations/layer0_attention_weight_min": -14.662543296813965,
      "activations/layer10_attention_weight_max": 38.4600830078125,
      "activations/layer10_attention_weight_min": -35.925167083740234,
      "activations/layer11_attention_weight_max": 33.058990478515625,
      "activations/layer11_attention_weight_min": -30.648038864135742,
      "activations/layer12_attention_weight_max": 19.888750076293945,
      "activations/layer12_attention_weight_min": -27.030452728271484,
      "activations/layer13_attention_weight_max": 49.01274490356445,
      "activations/layer13_attention_weight_min": -33.04478454589844,
      "activations/layer14_attention_weight_max": 40.19279098510742,
      "activations/layer14_attention_weight_min": -31.25313949584961,
      "activations/layer15_attention_weight_max": 36.850921630859375,
      "activations/layer15_attention_weight_min": -29.839473724365234,
      "activations/layer16_attention_weight_max": 35.87103271484375,
      "activations/layer16_attention_weight_min": -29.347227096557617,
      "activations/layer17_attention_weight_max": 54.4736213684082,
      "activations/layer17_attention_weight_min": -44.9854850769043,
      "activations/layer18_attention_weight_max": 50.69459915161133,
      "activations/layer18_attention_weight_min": -37.564273834228516,
      "activations/layer19_attention_weight_max": 22.164836883544922,
      "activations/layer19_attention_weight_min": -21.974477767944336,
      "activations/layer1_attention_weight_max": 17.37078857421875,
      "activations/layer1_attention_weight_min": -14.64248275756836,
      "activations/layer20_attention_weight_max": 23.41196060180664,
      "activations/layer20_attention_weight_min": -22.30880355834961,
      "activations/layer21_attention_weight_max": 43.59136199951172,
      "activations/layer21_attention_weight_min": -27.896928787231445,
      "activations/layer22_attention_weight_max": 30.016544342041016,
      "activations/layer22_attention_weight_min": -25.448375701904297,
      "activations/layer23_attention_weight_max": 34.41075897216797,
      "activations/layer23_attention_weight_min": -20.894617080688477,
      "activations/layer2_attention_weight_max": 31.403959274291992,
      "activations/layer2_attention_weight_min": -29.938045501708984,
      "activations/layer3_attention_weight_max": 91.88516235351562,
      "activations/layer3_attention_weight_min": -91.54082489013672,
      "activations/layer4_attention_weight_max": 100.19088745117188,
      "activations/layer4_attention_weight_min": -97.72264862060547,
      "activations/layer5_attention_weight_max": 82.9764404296875,
      "activations/layer5_attention_weight_min": -75.38131713867188,
      "activations/layer6_attention_weight_max": 54.04975128173828,
      "activations/layer6_attention_weight_min": -55.21891784667969,
      "activations/layer7_attention_weight_max": 76.56438446044922,
      "activations/layer7_attention_weight_min": -76.10248565673828,
      "activations/layer8_attention_weight_max": 45.5359001159668,
      "activations/layer8_attention_weight_min": -48.358009338378906,
      "activations/layer9_attention_weight_max": 53.54657745361328,
      "activations/layer9_attention_weight_min": -51.26580810546875,
      "epoch": 8.57,
      "learning_rate": 9.569393939393938e-05,
      "loss": 2.8105,
      "step": 147500
    },
    {
      "activations/layer0_attention_weight_max": 15.09999942779541,
      "activations/layer0_attention_weight_min": -14.134876251220703,
      "activations/layer10_attention_weight_max": 36.428680419921875,
      "activations/layer10_attention_weight_min": -35.897830963134766,
      "activations/layer11_attention_weight_max": 30.99427604675293,
      "activations/layer11_attention_weight_min": -31.51589012145996,
      "activations/layer12_attention_weight_max": 18.067514419555664,
      "activations/layer12_attention_weight_min": -25.726076126098633,
      "activations/layer13_attention_weight_max": 39.29734802246094,
      "activations/layer13_attention_weight_min": -27.354999542236328,
      "activations/layer14_attention_weight_max": 36.80260467529297,
      "activations/layer14_attention_weight_min": -32.98200988769531,
      "activations/layer15_attention_weight_max": 33.71662902832031,
      "activations/layer15_attention_weight_min": -29.958341598510742,
      "activations/layer16_attention_weight_max": 31.419214248657227,
      "activations/layer16_attention_weight_min": -29.132808685302734,
      "activations/layer17_attention_weight_max": 54.2747802734375,
      "activations/layer17_attention_weight_min": -43.90983581542969,
      "activations/layer18_attention_weight_max": 45.69612503051758,
      "activations/layer18_attention_weight_min": -34.713436126708984,
      "activations/layer19_attention_weight_max": 22.159042358398438,
      "activations/layer19_attention_weight_min": -20.39483070373535,
      "activations/layer1_attention_weight_max": 16.374238967895508,
      "activations/layer1_attention_weight_min": -15.43513011932373,
      "activations/layer20_attention_weight_max": 23.12128257751465,
      "activations/layer20_attention_weight_min": -20.897581100463867,
      "activations/layer21_attention_weight_max": 39.73276138305664,
      "activations/layer21_attention_weight_min": -21.546037673950195,
      "activations/layer22_attention_weight_max": 33.1944694519043,
      "activations/layer22_attention_weight_min": -27.20487403869629,
      "activations/layer23_attention_weight_max": 36.13445281982422,
      "activations/layer23_attention_weight_min": -21.04745864868164,
      "activations/layer2_attention_weight_max": 30.844411849975586,
      "activations/layer2_attention_weight_min": -28.87129020690918,
      "activations/layer3_attention_weight_max": 88.52012634277344,
      "activations/layer3_attention_weight_min": -93.2823715209961,
      "activations/layer4_attention_weight_max": 100.05520629882812,
      "activations/layer4_attention_weight_min": -90.57681274414062,
      "activations/layer5_attention_weight_max": 78.4388427734375,
      "activations/layer5_attention_weight_min": -73.64448547363281,
      "activations/layer6_attention_weight_max": 53.762184143066406,
      "activations/layer6_attention_weight_min": -49.83122634887695,
      "activations/layer7_attention_weight_max": 76.87227630615234,
      "activations/layer7_attention_weight_min": -72.11666107177734,
      "activations/layer8_attention_weight_max": 42.6591682434082,
      "activations/layer8_attention_weight_min": -45.15580368041992,
      "activations/layer9_attention_weight_max": 47.441707611083984,
      "activations/layer9_attention_weight_min": -50.29228591918945,
      "epoch": 8.57,
      "learning_rate": 9.5675e-05,
      "loss": 2.8069,
      "step": 147550
    },
    {
      "activations/layer0_attention_weight_max": 15.90654182434082,
      "activations/layer0_attention_weight_min": -14.570971488952637,
      "activations/layer10_attention_weight_max": 38.18952941894531,
      "activations/layer10_attention_weight_min": -34.895511627197266,
      "activations/layer11_attention_weight_max": 31.720502853393555,
      "activations/layer11_attention_weight_min": -31.209148406982422,
      "activations/layer12_attention_weight_max": 20.49278450012207,
      "activations/layer12_attention_weight_min": -26.805198669433594,
      "activations/layer13_attention_weight_max": 43.318519592285156,
      "activations/layer13_attention_weight_min": -33.6719970703125,
      "activations/layer14_attention_weight_max": 38.675540924072266,
      "activations/layer14_attention_weight_min": -32.87931442260742,
      "activations/layer15_attention_weight_max": 32.80946731567383,
      "activations/layer15_attention_weight_min": -31.24980354309082,
      "activations/layer16_attention_weight_max": 29.419437408447266,
      "activations/layer16_attention_weight_min": -27.736621856689453,
      "activations/layer17_attention_weight_max": 50.3980827331543,
      "activations/layer17_attention_weight_min": -42.22361755371094,
      "activations/layer18_attention_weight_max": 44.94591522216797,
      "activations/layer18_attention_weight_min": -35.31697463989258,
      "activations/layer19_attention_weight_max": 20.846332550048828,
      "activations/layer19_attention_weight_min": -21.589275360107422,
      "activations/layer1_attention_weight_max": 16.426740646362305,
      "activations/layer1_attention_weight_min": -15.80285358428955,
      "activations/layer20_attention_weight_max": 21.58156967163086,
      "activations/layer20_attention_weight_min": -20.25067710876465,
      "activations/layer21_attention_weight_max": 34.235477447509766,
      "activations/layer21_attention_weight_min": -22.797380447387695,
      "activations/layer22_attention_weight_max": 31.762531280517578,
      "activations/layer22_attention_weight_min": -25.24283790588379,
      "activations/layer23_attention_weight_max": 32.71689987182617,
      "activations/layer23_attention_weight_min": -20.553573608398438,
      "activations/layer2_attention_weight_max": 30.24268913269043,
      "activations/layer2_attention_weight_min": -28.567596435546875,
      "activations/layer3_attention_weight_max": 90.3851089477539,
      "activations/layer3_attention_weight_min": -91.34447479248047,
      "activations/layer4_attention_weight_max": 101.9493408203125,
      "activations/layer4_attention_weight_min": -98.19658660888672,
      "activations/layer5_attention_weight_max": 84.38821411132812,
      "activations/layer5_attention_weight_min": -78.65998077392578,
      "activations/layer6_attention_weight_max": 57.224578857421875,
      "activations/layer6_attention_weight_min": -55.38227462768555,
      "activations/layer7_attention_weight_max": 80.25150299072266,
      "activations/layer7_attention_weight_min": -71.9084243774414,
      "activations/layer8_attention_weight_max": 45.94139862060547,
      "activations/layer8_attention_weight_min": -49.96596145629883,
      "activations/layer9_attention_weight_max": 52.76564407348633,
      "activations/layer9_attention_weight_min": -48.768218994140625,
      "epoch": 8.58,
      "learning_rate": 9.56560606060606e-05,
      "loss": 2.8095,
      "step": 147600
    },
    {
      "activations/layer0_attention_weight_max": 17.46230125427246,
      "activations/layer0_attention_weight_min": -14.145038604736328,
      "activations/layer10_attention_weight_max": 38.587223052978516,
      "activations/layer10_attention_weight_min": -36.18217849731445,
      "activations/layer11_attention_weight_max": 33.279197692871094,
      "activations/layer11_attention_weight_min": -31.936325073242188,
      "activations/layer12_attention_weight_max": 22.90677833557129,
      "activations/layer12_attention_weight_min": -26.933332443237305,
      "activations/layer13_attention_weight_max": 45.729618072509766,
      "activations/layer13_attention_weight_min": -32.67333221435547,
      "activations/layer14_attention_weight_max": 45.82980728149414,
      "activations/layer14_attention_weight_min": -32.10464859008789,
      "activations/layer15_attention_weight_max": 36.34562683105469,
      "activations/layer15_attention_weight_min": -29.964813232421875,
      "activations/layer16_attention_weight_max": 34.49552917480469,
      "activations/layer16_attention_weight_min": -29.492612838745117,
      "activations/layer17_attention_weight_max": 58.5977783203125,
      "activations/layer17_attention_weight_min": -47.115882873535156,
      "activations/layer18_attention_weight_max": 52.6519889831543,
      "activations/layer18_attention_weight_min": -37.993953704833984,
      "activations/layer19_attention_weight_max": 23.13532829284668,
      "activations/layer19_attention_weight_min": -21.460996627807617,
      "activations/layer1_attention_weight_max": 17.23079490661621,
      "activations/layer1_attention_weight_min": -15.037938117980957,
      "activations/layer20_attention_weight_max": 27.811628341674805,
      "activations/layer20_attention_weight_min": -23.727163314819336,
      "activations/layer21_attention_weight_max": 46.5926628112793,
      "activations/layer21_attention_weight_min": -24.889249801635742,
      "activations/layer22_attention_weight_max": 37.36500930786133,
      "activations/layer22_attention_weight_min": -25.8502140045166,
      "activations/layer23_attention_weight_max": 43.19370651245117,
      "activations/layer23_attention_weight_min": -20.44204330444336,
      "activations/layer2_attention_weight_max": 31.443492889404297,
      "activations/layer2_attention_weight_min": -30.848499298095703,
      "activations/layer3_attention_weight_max": 93.99222564697266,
      "activations/layer3_attention_weight_min": -92.1368637084961,
      "activations/layer4_attention_weight_max": 105.68446350097656,
      "activations/layer4_attention_weight_min": -100.97004699707031,
      "activations/layer5_attention_weight_max": 81.27778625488281,
      "activations/layer5_attention_weight_min": -81.86418914794922,
      "activations/layer6_attention_weight_max": 57.478179931640625,
      "activations/layer6_attention_weight_min": -53.78921127319336,
      "activations/layer7_attention_weight_max": 77.7564697265625,
      "activations/layer7_attention_weight_min": -74.34393310546875,
      "activations/layer8_attention_weight_max": 48.70258331298828,
      "activations/layer8_attention_weight_min": -45.91035461425781,
      "activations/layer9_attention_weight_max": 54.31968307495117,
      "activations/layer9_attention_weight_min": -48.483253479003906,
      "epoch": 8.58,
      "learning_rate": 9.56371212121212e-05,
      "loss": 2.8032,
      "step": 147650
    },
    {
      "activations/layer0_attention_weight_max": 15.456610679626465,
      "activations/layer0_attention_weight_min": -14.588380813598633,
      "activations/layer10_attention_weight_max": 35.322975158691406,
      "activations/layer10_attention_weight_min": -34.97225570678711,
      "activations/layer11_attention_weight_max": 30.735416412353516,
      "activations/layer11_attention_weight_min": -31.705768585205078,
      "activations/layer12_attention_weight_max": 19.55442237854004,
      "activations/layer12_attention_weight_min": -22.226924896240234,
      "activations/layer13_attention_weight_max": 43.011661529541016,
      "activations/layer13_attention_weight_min": -30.152048110961914,
      "activations/layer14_attention_weight_max": 37.98926544189453,
      "activations/layer14_attention_weight_min": -29.52853012084961,
      "activations/layer15_attention_weight_max": 32.94036865234375,
      "activations/layer15_attention_weight_min": -27.975067138671875,
      "activations/layer16_attention_weight_max": 31.26586151123047,
      "activations/layer16_attention_weight_min": -26.010549545288086,
      "activations/layer17_attention_weight_max": 50.415443420410156,
      "activations/layer17_attention_weight_min": -42.32380676269531,
      "activations/layer18_attention_weight_max": 44.9531364440918,
      "activations/layer18_attention_weight_min": -35.69548034667969,
      "activations/layer19_attention_weight_max": 21.733083724975586,
      "activations/layer19_attention_weight_min": -20.533044815063477,
      "activations/layer1_attention_weight_max": 18.500314712524414,
      "activations/layer1_attention_weight_min": -17.371068954467773,
      "activations/layer20_attention_weight_max": 23.4146785736084,
      "activations/layer20_attention_weight_min": -22.318706512451172,
      "activations/layer21_attention_weight_max": 32.96432876586914,
      "activations/layer21_attention_weight_min": -21.775110244750977,
      "activations/layer22_attention_weight_max": 33.63275909423828,
      "activations/layer22_attention_weight_min": -24.6055908203125,
      "activations/layer23_attention_weight_max": 37.73735809326172,
      "activations/layer23_attention_weight_min": -21.453704833984375,
      "activations/layer2_attention_weight_max": 31.59380340576172,
      "activations/layer2_attention_weight_min": -30.794326782226562,
      "activations/layer3_attention_weight_max": 89.96907806396484,
      "activations/layer3_attention_weight_min": -103.22697448730469,
      "activations/layer4_attention_weight_max": 100.65691375732422,
      "activations/layer4_attention_weight_min": -99.46283721923828,
      "activations/layer5_attention_weight_max": 79.58006286621094,
      "activations/layer5_attention_weight_min": -77.86787414550781,
      "activations/layer6_attention_weight_max": 55.31007766723633,
      "activations/layer6_attention_weight_min": -53.367942810058594,
      "activations/layer7_attention_weight_max": 81.89556121826172,
      "activations/layer7_attention_weight_min": -77.11820220947266,
      "activations/layer8_attention_weight_max": 43.293617248535156,
      "activations/layer8_attention_weight_min": -47.55013656616211,
      "activations/layer9_attention_weight_max": 50.75613784790039,
      "activations/layer9_attention_weight_min": -48.85212326049805,
      "epoch": 8.58,
      "learning_rate": 9.561818181818181e-05,
      "loss": 2.7868,
      "step": 147700
    },
    {
      "activations/layer0_attention_weight_max": 15.665189743041992,
      "activations/layer0_attention_weight_min": -14.465361595153809,
      "activations/layer10_attention_weight_max": 36.25053787231445,
      "activations/layer10_attention_weight_min": -34.02238082885742,
      "activations/layer11_attention_weight_max": 31.096969604492188,
      "activations/layer11_attention_weight_min": -32.3101921081543,
      "activations/layer12_attention_weight_max": 19.72500228881836,
      "activations/layer12_attention_weight_min": -25.01642608642578,
      "activations/layer13_attention_weight_max": 37.39649200439453,
      "activations/layer13_attention_weight_min": -30.257890701293945,
      "activations/layer14_attention_weight_max": 35.86778259277344,
      "activations/layer14_attention_weight_min": -29.86586570739746,
      "activations/layer15_attention_weight_max": 31.61159896850586,
      "activations/layer15_attention_weight_min": -30.23908805847168,
      "activations/layer16_attention_weight_max": 29.636425018310547,
      "activations/layer16_attention_weight_min": -27.73173713684082,
      "activations/layer17_attention_weight_max": 50.87391662597656,
      "activations/layer17_attention_weight_min": -44.5443000793457,
      "activations/layer18_attention_weight_max": 46.69356918334961,
      "activations/layer18_attention_weight_min": -36.27668762207031,
      "activations/layer19_attention_weight_max": 21.51839256286621,
      "activations/layer19_attention_weight_min": -22.30685806274414,
      "activations/layer1_attention_weight_max": 16.71178436279297,
      "activations/layer1_attention_weight_min": -14.234397888183594,
      "activations/layer20_attention_weight_max": 18.61993980407715,
      "activations/layer20_attention_weight_min": -19.84158706665039,
      "activations/layer21_attention_weight_max": 36.622432708740234,
      "activations/layer21_attention_weight_min": -22.478300094604492,
      "activations/layer22_attention_weight_max": 30.858930587768555,
      "activations/layer22_attention_weight_min": -26.599172592163086,
      "activations/layer23_attention_weight_max": 35.25944137573242,
      "activations/layer23_attention_weight_min": -22.779151916503906,
      "activations/layer2_attention_weight_max": 30.985586166381836,
      "activations/layer2_attention_weight_min": -30.056438446044922,
      "activations/layer3_attention_weight_max": 87.07584381103516,
      "activations/layer3_attention_weight_min": -92.36287689208984,
      "activations/layer4_attention_weight_max": 98.68415832519531,
      "activations/layer4_attention_weight_min": -94.52911376953125,
      "activations/layer5_attention_weight_max": 78.0316390991211,
      "activations/layer5_attention_weight_min": -77.97827911376953,
      "activations/layer6_attention_weight_max": 52.80561828613281,
      "activations/layer6_attention_weight_min": -52.450252532958984,
      "activations/layer7_attention_weight_max": 74.914794921875,
      "activations/layer7_attention_weight_min": -69.64627838134766,
      "activations/layer8_attention_weight_max": 47.182212829589844,
      "activations/layer8_attention_weight_min": -44.229339599609375,
      "activations/layer9_attention_weight_max": 49.219120025634766,
      "activations/layer9_attention_weight_min": -47.38344955444336,
      "epoch": 8.59,
      "learning_rate": 9.559924242424241e-05,
      "loss": 2.8168,
      "step": 147750
    },
    {
      "activations/layer0_attention_weight_max": 15.308370590209961,
      "activations/layer0_attention_weight_min": -14.618175506591797,
      "activations/layer10_attention_weight_max": 39.3643913269043,
      "activations/layer10_attention_weight_min": -36.723426818847656,
      "activations/layer11_attention_weight_max": 35.300045013427734,
      "activations/layer11_attention_weight_min": -32.979496002197266,
      "activations/layer12_attention_weight_max": 31.788280487060547,
      "activations/layer12_attention_weight_min": -25.856281280517578,
      "activations/layer13_attention_weight_max": 45.040245056152344,
      "activations/layer13_attention_weight_min": -38.398536682128906,
      "activations/layer14_attention_weight_max": 40.58415985107422,
      "activations/layer14_attention_weight_min": -30.28717803955078,
      "activations/layer15_attention_weight_max": 37.87980651855469,
      "activations/layer15_attention_weight_min": -30.209304809570312,
      "activations/layer16_attention_weight_max": 34.19511413574219,
      "activations/layer16_attention_weight_min": -26.996074676513672,
      "activations/layer17_attention_weight_max": 56.89076614379883,
      "activations/layer17_attention_weight_min": -44.799102783203125,
      "activations/layer18_attention_weight_max": 47.668922424316406,
      "activations/layer18_attention_weight_min": -37.309234619140625,
      "activations/layer19_attention_weight_max": 24.70929527282715,
      "activations/layer19_attention_weight_min": -22.268022537231445,
      "activations/layer1_attention_weight_max": 16.630128860473633,
      "activations/layer1_attention_weight_min": -15.79818058013916,
      "activations/layer20_attention_weight_max": 25.429136276245117,
      "activations/layer20_attention_weight_min": -22.88870620727539,
      "activations/layer21_attention_weight_max": 37.59682846069336,
      "activations/layer21_attention_weight_min": -23.034109115600586,
      "activations/layer22_attention_weight_max": 33.33485412597656,
      "activations/layer22_attention_weight_min": -25.325641632080078,
      "activations/layer23_attention_weight_max": 41.054718017578125,
      "activations/layer23_attention_weight_min": -26.524311065673828,
      "activations/layer2_attention_weight_max": 31.351177215576172,
      "activations/layer2_attention_weight_min": -29.66596794128418,
      "activations/layer3_attention_weight_max": 93.22684478759766,
      "activations/layer3_attention_weight_min": -94.24872589111328,
      "activations/layer4_attention_weight_max": 103.36595916748047,
      "activations/layer4_attention_weight_min": -100.2125473022461,
      "activations/layer5_attention_weight_max": 83.80476379394531,
      "activations/layer5_attention_weight_min": -80.48335266113281,
      "activations/layer6_attention_weight_max": 54.97028732299805,
      "activations/layer6_attention_weight_min": -53.52718734741211,
      "activations/layer7_attention_weight_max": 79.54199981689453,
      "activations/layer7_attention_weight_min": -74.72114562988281,
      "activations/layer8_attention_weight_max": 45.7746696472168,
      "activations/layer8_attention_weight_min": -45.68635940551758,
      "activations/layer9_attention_weight_max": 51.20484161376953,
      "activations/layer9_attention_weight_min": -49.080745697021484,
      "epoch": 8.59,
      "learning_rate": 9.558030303030303e-05,
      "loss": 2.809,
      "step": 147800
    },
    {
      "activations/layer0_attention_weight_max": 15.588415145874023,
      "activations/layer0_attention_weight_min": -14.154519081115723,
      "activations/layer10_attention_weight_max": 41.454593658447266,
      "activations/layer10_attention_weight_min": -38.055023193359375,
      "activations/layer11_attention_weight_max": 32.37497329711914,
      "activations/layer11_attention_weight_min": -33.867271423339844,
      "activations/layer12_attention_weight_max": 20.213979721069336,
      "activations/layer12_attention_weight_min": -24.54816436767578,
      "activations/layer13_attention_weight_max": 39.887413024902344,
      "activations/layer13_attention_weight_min": -30.37314224243164,
      "activations/layer14_attention_weight_max": 36.41509246826172,
      "activations/layer14_attention_weight_min": -30.035682678222656,
      "activations/layer15_attention_weight_max": 34.4609260559082,
      "activations/layer15_attention_weight_min": -28.996177673339844,
      "activations/layer16_attention_weight_max": 30.726831436157227,
      "activations/layer16_attention_weight_min": -30.20816993713379,
      "activations/layer17_attention_weight_max": 55.31740188598633,
      "activations/layer17_attention_weight_min": -42.272098541259766,
      "activations/layer18_attention_weight_max": 51.380191802978516,
      "activations/layer18_attention_weight_min": -36.059234619140625,
      "activations/layer19_attention_weight_max": 19.39237403869629,
      "activations/layer19_attention_weight_min": -22.59954261779785,
      "activations/layer1_attention_weight_max": 16.805408477783203,
      "activations/layer1_attention_weight_min": -16.79590606689453,
      "activations/layer20_attention_weight_max": 21.988731384277344,
      "activations/layer20_attention_weight_min": -19.93838882446289,
      "activations/layer21_attention_weight_max": 34.83807373046875,
      "activations/layer21_attention_weight_min": -23.477346420288086,
      "activations/layer22_attention_weight_max": 34.25086975097656,
      "activations/layer22_attention_weight_min": -23.268449783325195,
      "activations/layer23_attention_weight_max": 36.63909912109375,
      "activations/layer23_attention_weight_min": -22.03815269470215,
      "activations/layer2_attention_weight_max": 32.22966766357422,
      "activations/layer2_attention_weight_min": -28.778820037841797,
      "activations/layer3_attention_weight_max": 92.07133483886719,
      "activations/layer3_attention_weight_min": -92.39380645751953,
      "activations/layer4_attention_weight_max": 104.67388153076172,
      "activations/layer4_attention_weight_min": -98.94329071044922,
      "activations/layer5_attention_weight_max": 80.53257751464844,
      "activations/layer5_attention_weight_min": -75.2107925415039,
      "activations/layer6_attention_weight_max": 55.744632720947266,
      "activations/layer6_attention_weight_min": -55.08045196533203,
      "activations/layer7_attention_weight_max": 76.67668914794922,
      "activations/layer7_attention_weight_min": -73.49715423583984,
      "activations/layer8_attention_weight_max": 44.17705154418945,
      "activations/layer8_attention_weight_min": -45.75441360473633,
      "activations/layer9_attention_weight_max": 51.18056869506836,
      "activations/layer9_attention_weight_min": -48.32954406738281,
      "epoch": 8.59,
      "learning_rate": 9.556174242424242e-05,
      "loss": 2.8099,
      "step": 147850
    },
    {
      "activations/layer0_attention_weight_max": 16.798444747924805,
      "activations/layer0_attention_weight_min": -14.292837142944336,
      "activations/layer10_attention_weight_max": 37.043067932128906,
      "activations/layer10_attention_weight_min": -36.92292404174805,
      "activations/layer11_attention_weight_max": 33.286048889160156,
      "activations/layer11_attention_weight_min": -31.740331649780273,
      "activations/layer12_attention_weight_max": 22.81352996826172,
      "activations/layer12_attention_weight_min": -25.118892669677734,
      "activations/layer13_attention_weight_max": 51.56093215942383,
      "activations/layer13_attention_weight_min": -33.68644332885742,
      "activations/layer14_attention_weight_max": 43.13801956176758,
      "activations/layer14_attention_weight_min": -30.287755966186523,
      "activations/layer15_attention_weight_max": 35.979610443115234,
      "activations/layer15_attention_weight_min": -29.369152069091797,
      "activations/layer16_attention_weight_max": 31.59141731262207,
      "activations/layer16_attention_weight_min": -28.156272888183594,
      "activations/layer17_attention_weight_max": 53.99367141723633,
      "activations/layer17_attention_weight_min": -43.22963333129883,
      "activations/layer18_attention_weight_max": 49.252227783203125,
      "activations/layer18_attention_weight_min": -36.74916076660156,
      "activations/layer19_attention_weight_max": 22.53415298461914,
      "activations/layer19_attention_weight_min": -19.64409828186035,
      "activations/layer1_attention_weight_max": 16.427200317382812,
      "activations/layer1_attention_weight_min": -13.892816543579102,
      "activations/layer20_attention_weight_max": 22.4251766204834,
      "activations/layer20_attention_weight_min": -22.817607879638672,
      "activations/layer21_attention_weight_max": 37.5303955078125,
      "activations/layer21_attention_weight_min": -24.785367965698242,
      "activations/layer22_attention_weight_max": 33.617218017578125,
      "activations/layer22_attention_weight_min": -24.93766975402832,
      "activations/layer23_attention_weight_max": 37.8093147277832,
      "activations/layer23_attention_weight_min": -22.23464584350586,
      "activations/layer2_attention_weight_max": 32.871299743652344,
      "activations/layer2_attention_weight_min": -31.979368209838867,
      "activations/layer3_attention_weight_max": 93.78054809570312,
      "activations/layer3_attention_weight_min": -96.02043914794922,
      "activations/layer4_attention_weight_max": 104.28187561035156,
      "activations/layer4_attention_weight_min": -99.42375946044922,
      "activations/layer5_attention_weight_max": 82.40949249267578,
      "activations/layer5_attention_weight_min": -79.29680633544922,
      "activations/layer6_attention_weight_max": 57.78662872314453,
      "activations/layer6_attention_weight_min": -55.94313049316406,
      "activations/layer7_attention_weight_max": 80.15625762939453,
      "activations/layer7_attention_weight_min": -73.78800201416016,
      "activations/layer8_attention_weight_max": 48.375213623046875,
      "activations/layer8_attention_weight_min": -47.96040725708008,
      "activations/layer9_attention_weight_max": 57.55448532104492,
      "activations/layer9_attention_weight_min": -49.64802932739258,
      "epoch": 8.59,
      "learning_rate": 9.554280303030302e-05,
      "loss": 2.8053,
      "step": 147900
    },
    {
      "activations/layer0_attention_weight_max": 15.192462921142578,
      "activations/layer0_attention_weight_min": -14.333189964294434,
      "activations/layer10_attention_weight_max": 37.38992691040039,
      "activations/layer10_attention_weight_min": -36.44025421142578,
      "activations/layer11_attention_weight_max": 31.09039306640625,
      "activations/layer11_attention_weight_min": -32.03114318847656,
      "activations/layer12_attention_weight_max": 20.070215225219727,
      "activations/layer12_attention_weight_min": -25.63905143737793,
      "activations/layer13_attention_weight_max": 46.01695251464844,
      "activations/layer13_attention_weight_min": -34.56926727294922,
      "activations/layer14_attention_weight_max": 41.001251220703125,
      "activations/layer14_attention_weight_min": -32.69589614868164,
      "activations/layer15_attention_weight_max": 40.367462158203125,
      "activations/layer15_attention_weight_min": -31.837017059326172,
      "activations/layer16_attention_weight_max": 36.30559158325195,
      "activations/layer16_attention_weight_min": -28.974044799804688,
      "activations/layer17_attention_weight_max": 54.71766662597656,
      "activations/layer17_attention_weight_min": -42.4847526550293,
      "activations/layer18_attention_weight_max": 45.20136260986328,
      "activations/layer18_attention_weight_min": -34.96702575683594,
      "activations/layer19_attention_weight_max": 21.291316986083984,
      "activations/layer19_attention_weight_min": -20.22846221923828,
      "activations/layer1_attention_weight_max": 17.33112144470215,
      "activations/layer1_attention_weight_min": -16.39596176147461,
      "activations/layer20_attention_weight_max": 24.181076049804688,
      "activations/layer20_attention_weight_min": -19.871583938598633,
      "activations/layer21_attention_weight_max": 43.992000579833984,
      "activations/layer21_attention_weight_min": -26.242685317993164,
      "activations/layer22_attention_weight_max": 33.209678649902344,
      "activations/layer22_attention_weight_min": -24.184804916381836,
      "activations/layer23_attention_weight_max": 40.67894744873047,
      "activations/layer23_attention_weight_min": -24.101760864257812,
      "activations/layer2_attention_weight_max": 32.144287109375,
      "activations/layer2_attention_weight_min": -30.312002182006836,
      "activations/layer3_attention_weight_max": 93.51600646972656,
      "activations/layer3_attention_weight_min": -94.06965637207031,
      "activations/layer4_attention_weight_max": 108.99820709228516,
      "activations/layer4_attention_weight_min": -100.77542114257812,
      "activations/layer5_attention_weight_max": 85.42181396484375,
      "activations/layer5_attention_weight_min": -76.21382141113281,
      "activations/layer6_attention_weight_max": 58.258209228515625,
      "activations/layer6_attention_weight_min": -53.46868133544922,
      "activations/layer7_attention_weight_max": 79.90567779541016,
      "activations/layer7_attention_weight_min": -78.53168487548828,
      "activations/layer8_attention_weight_max": 45.38667678833008,
      "activations/layer8_attention_weight_min": -45.36167907714844,
      "activations/layer9_attention_weight_max": 47.72010040283203,
      "activations/layer9_attention_weight_min": -48.58245849609375,
      "epoch": 8.6,
      "learning_rate": 9.552386363636362e-05,
      "loss": 2.8019,
      "step": 147950
    },
    {
      "activations/layer0_attention_weight_max": 15.42734146118164,
      "activations/layer0_attention_weight_min": -14.219493865966797,
      "activations/layer10_attention_weight_max": 43.924617767333984,
      "activations/layer10_attention_weight_min": -40.012046813964844,
      "activations/layer11_attention_weight_max": 38.65480041503906,
      "activations/layer11_attention_weight_min": -34.78492736816406,
      "activations/layer12_attention_weight_max": 32.93838119506836,
      "activations/layer12_attention_weight_min": -26.893938064575195,
      "activations/layer13_attention_weight_max": 51.7114143371582,
      "activations/layer13_attention_weight_min": -36.56584167480469,
      "activations/layer14_attention_weight_max": 54.010135650634766,
      "activations/layer14_attention_weight_min": -31.550315856933594,
      "activations/layer15_attention_weight_max": 41.22935485839844,
      "activations/layer15_attention_weight_min": -30.36962890625,
      "activations/layer16_attention_weight_max": 32.31757354736328,
      "activations/layer16_attention_weight_min": -28.767518997192383,
      "activations/layer17_attention_weight_max": 58.24956130981445,
      "activations/layer17_attention_weight_min": -48.68217849731445,
      "activations/layer18_attention_weight_max": 51.143333435058594,
      "activations/layer18_attention_weight_min": -38.469879150390625,
      "activations/layer19_attention_weight_max": 26.238834381103516,
      "activations/layer19_attention_weight_min": -21.474531173706055,
      "activations/layer1_attention_weight_max": 16.26764488220215,
      "activations/layer1_attention_weight_min": -13.945549964904785,
      "activations/layer20_attention_weight_max": 25.103797912597656,
      "activations/layer20_attention_weight_min": -22.81102752685547,
      "activations/layer21_attention_weight_max": 41.47678756713867,
      "activations/layer21_attention_weight_min": -24.55137825012207,
      "activations/layer22_attention_weight_max": 29.524473190307617,
      "activations/layer22_attention_weight_min": -24.848230361938477,
      "activations/layer23_attention_weight_max": 34.83929443359375,
      "activations/layer23_attention_weight_min": -20.472305297851562,
      "activations/layer2_attention_weight_max": 31.271297454833984,
      "activations/layer2_attention_weight_min": -30.05187225341797,
      "activations/layer3_attention_weight_max": 93.585205078125,
      "activations/layer3_attention_weight_min": -94.23055267333984,
      "activations/layer4_attention_weight_max": 106.58642578125,
      "activations/layer4_attention_weight_min": -106.6259536743164,
      "activations/layer5_attention_weight_max": 87.34867858886719,
      "activations/layer5_attention_weight_min": -79.10261535644531,
      "activations/layer6_attention_weight_max": 58.16966247558594,
      "activations/layer6_attention_weight_min": -56.31114196777344,
      "activations/layer7_attention_weight_max": 85.36176300048828,
      "activations/layer7_attention_weight_min": -78.68513488769531,
      "activations/layer8_attention_weight_max": 53.79861068725586,
      "activations/layer8_attention_weight_min": -51.35445022583008,
      "activations/layer9_attention_weight_max": 65.2387924194336,
      "activations/layer9_attention_weight_min": -55.78199005126953,
      "epoch": 8.6,
      "learning_rate": 9.550492424242424e-05,
      "loss": 2.7949,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.5084,
      "eval_samples_per_second": 504.677,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.5084,
      "eval_openwebtext_samples_per_second": 504.677,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 1.9755,
      "eval_wikitext_samples_per_second": 230.824,
      "step": 148000
    },
    {
      "epoch": 8.6,
      "eval_lambada_loss": 2.603515625,
      "eval_lambada_ppl": 13.511154789957821,
      "eval_lambada_runtime": 9.5532,
      "eval_lambada_samples_per_second": 509.67,
      "step": 148000
    },
    {
      "activations/layer0_attention_weight_max": 15.679511070251465,
      "activations/layer0_attention_weight_min": -14.480989456176758,
      "activations/layer10_attention_weight_max": 42.23952865600586,
      "activations/layer10_attention_weight_min": -37.895469665527344,
      "activations/layer11_attention_weight_max": 34.34734344482422,
      "activations/layer11_attention_weight_min": -34.28748321533203,
      "activations/layer12_attention_weight_max": 21.803728103637695,
      "activations/layer12_attention_weight_min": -27.04501724243164,
      "activations/layer13_attention_weight_max": 62.148502349853516,
      "activations/layer13_attention_weight_min": -36.512577056884766,
      "activations/layer14_attention_weight_max": 45.19274139404297,
      "activations/layer14_attention_weight_min": -32.628395080566406,
      "activations/layer15_attention_weight_max": 34.29389190673828,
      "activations/layer15_attention_weight_min": -29.688968658447266,
      "activations/layer16_attention_weight_max": 32.81035614013672,
      "activations/layer16_attention_weight_min": -28.090578079223633,
      "activations/layer17_attention_weight_max": 51.853275299072266,
      "activations/layer17_attention_weight_min": -43.942501068115234,
      "activations/layer18_attention_weight_max": 47.728111267089844,
      "activations/layer18_attention_weight_min": -35.648136138916016,
      "activations/layer19_attention_weight_max": 22.310508728027344,
      "activations/layer19_attention_weight_min": -19.656965255737305,
      "activations/layer1_attention_weight_max": 15.619402885437012,
      "activations/layer1_attention_weight_min": -14.262974739074707,
      "activations/layer20_attention_weight_max": 24.439401626586914,
      "activations/layer20_attention_weight_min": -20.586978912353516,
      "activations/layer21_attention_weight_max": 38.62954330444336,
      "activations/layer21_attention_weight_min": -23.517019271850586,
      "activations/layer22_attention_weight_max": 32.04599380493164,
      "activations/layer22_attention_weight_min": -25.784223556518555,
      "activations/layer23_attention_weight_max": 34.41374206542969,
      "activations/layer23_attention_weight_min": -20.274734497070312,
      "activations/layer2_attention_weight_max": 31.733810424804688,
      "activations/layer2_attention_weight_min": -29.139122009277344,
      "activations/layer3_attention_weight_max": 90.97715759277344,
      "activations/layer3_attention_weight_min": -92.78899383544922,
      "activations/layer4_attention_weight_max": 103.30374908447266,
      "activations/layer4_attention_weight_min": -100.82955169677734,
      "activations/layer5_attention_weight_max": 82.3644027709961,
      "activations/layer5_attention_weight_min": -79.99852752685547,
      "activations/layer6_attention_weight_max": 58.56154251098633,
      "activations/layer6_attention_weight_min": -56.97398376464844,
      "activations/layer7_attention_weight_max": 78.61498260498047,
      "activations/layer7_attention_weight_min": -75.5945816040039,
      "activations/layer8_attention_weight_max": 47.41627502441406,
      "activations/layer8_attention_weight_min": -47.41687774658203,
      "activations/layer9_attention_weight_max": 54.31978225708008,
      "activations/layer9_attention_weight_min": -48.71527862548828,
      "epoch": 8.6,
      "learning_rate": 9.548598484848485e-05,
      "loss": 2.8118,
      "step": 148050
    },
    {
      "activations/layer0_attention_weight_max": 15.72795295715332,
      "activations/layer0_attention_weight_min": -14.563342094421387,
      "activations/layer10_attention_weight_max": 38.2802734375,
      "activations/layer10_attention_weight_min": -35.65169143676758,
      "activations/layer11_attention_weight_max": 32.52489471435547,
      "activations/layer11_attention_weight_min": -32.16722106933594,
      "activations/layer12_attention_weight_max": 20.03963851928711,
      "activations/layer12_attention_weight_min": -26.49270248413086,
      "activations/layer13_attention_weight_max": 47.16590881347656,
      "activations/layer13_attention_weight_min": -31.981958389282227,
      "activations/layer14_attention_weight_max": 37.7445182800293,
      "activations/layer14_attention_weight_min": -30.750377655029297,
      "activations/layer15_attention_weight_max": 34.76106262207031,
      "activations/layer15_attention_weight_min": -31.34407615661621,
      "activations/layer16_attention_weight_max": 29.77086639404297,
      "activations/layer16_attention_weight_min": -28.554561614990234,
      "activations/layer17_attention_weight_max": 54.33076477050781,
      "activations/layer17_attention_weight_min": -42.67998123168945,
      "activations/layer18_attention_weight_max": 44.887168884277344,
      "activations/layer18_attention_weight_min": -37.568119049072266,
      "activations/layer19_attention_weight_max": 24.858911514282227,
      "activations/layer19_attention_weight_min": -23.25812339782715,
      "activations/layer1_attention_weight_max": 16.63078498840332,
      "activations/layer1_attention_weight_min": -17.050378799438477,
      "activations/layer20_attention_weight_max": 21.900951385498047,
      "activations/layer20_attention_weight_min": -28.129308700561523,
      "activations/layer21_attention_weight_max": 37.09403991699219,
      "activations/layer21_attention_weight_min": -23.03213119506836,
      "activations/layer22_attention_weight_max": 29.97927474975586,
      "activations/layer22_attention_weight_min": -27.443819046020508,
      "activations/layer23_attention_weight_max": 34.97610092163086,
      "activations/layer23_attention_weight_min": -22.68981170654297,
      "activations/layer2_attention_weight_max": 29.508525848388672,
      "activations/layer2_attention_weight_min": -28.387828826904297,
      "activations/layer3_attention_weight_max": 84.11223602294922,
      "activations/layer3_attention_weight_min": -86.06571197509766,
      "activations/layer4_attention_weight_max": 97.07024383544922,
      "activations/layer4_attention_weight_min": -91.22078704833984,
      "activations/layer5_attention_weight_max": 74.58622741699219,
      "activations/layer5_attention_weight_min": -72.69551086425781,
      "activations/layer6_attention_weight_max": 55.104705810546875,
      "activations/layer6_attention_weight_min": -54.20319747924805,
      "activations/layer7_attention_weight_max": 74.08679962158203,
      "activations/layer7_attention_weight_min": -72.11911010742188,
      "activations/layer8_attention_weight_max": 45.83945083618164,
      "activations/layer8_attention_weight_min": -48.932518005371094,
      "activations/layer9_attention_weight_max": 50.875144958496094,
      "activations/layer9_attention_weight_min": -48.950862884521484,
      "epoch": 8.61,
      "learning_rate": 9.546704545454544e-05,
      "loss": 2.8111,
      "step": 148100
    },
    {
      "activations/layer0_attention_weight_max": 16.063039779663086,
      "activations/layer0_attention_weight_min": -14.62227725982666,
      "activations/layer10_attention_weight_max": 39.1249885559082,
      "activations/layer10_attention_weight_min": -36.62469482421875,
      "activations/layer11_attention_weight_max": 33.76070022583008,
      "activations/layer11_attention_weight_min": -31.714550018310547,
      "activations/layer12_attention_weight_max": 20.872817993164062,
      "activations/layer12_attention_weight_min": -25.6467227935791,
      "activations/layer13_attention_weight_max": 38.09184265136719,
      "activations/layer13_attention_weight_min": -29.80753517150879,
      "activations/layer14_attention_weight_max": 45.823753356933594,
      "activations/layer14_attention_weight_min": -31.17484474182129,
      "activations/layer15_attention_weight_max": 40.124717712402344,
      "activations/layer15_attention_weight_min": -28.981708526611328,
      "activations/layer16_attention_weight_max": 29.22819709777832,
      "activations/layer16_attention_weight_min": -26.922000885009766,
      "activations/layer17_attention_weight_max": 52.97800064086914,
      "activations/layer17_attention_weight_min": -41.44941711425781,
      "activations/layer18_attention_weight_max": 47.891937255859375,
      "activations/layer18_attention_weight_min": -34.51512908935547,
      "activations/layer19_attention_weight_max": 20.165315628051758,
      "activations/layer19_attention_weight_min": -23.118839263916016,
      "activations/layer1_attention_weight_max": 16.60755729675293,
      "activations/layer1_attention_weight_min": -14.510069847106934,
      "activations/layer20_attention_weight_max": 21.344013214111328,
      "activations/layer20_attention_weight_min": -18.883647918701172,
      "activations/layer21_attention_weight_max": 36.9393310546875,
      "activations/layer21_attention_weight_min": -25.40204429626465,
      "activations/layer22_attention_weight_max": 30.10297393798828,
      "activations/layer22_attention_weight_min": -23.91791534423828,
      "activations/layer23_attention_weight_max": 34.26157760620117,
      "activations/layer23_attention_weight_min": -22.73737335205078,
      "activations/layer2_attention_weight_max": 30.6075496673584,
      "activations/layer2_attention_weight_min": -28.55994415283203,
      "activations/layer3_attention_weight_max": 92.13452911376953,
      "activations/layer3_attention_weight_min": -91.72701263427734,
      "activations/layer4_attention_weight_max": 102.27645874023438,
      "activations/layer4_attention_weight_min": -98.60980987548828,
      "activations/layer5_attention_weight_max": 80.00480651855469,
      "activations/layer5_attention_weight_min": -80.76763916015625,
      "activations/layer6_attention_weight_max": 55.10374450683594,
      "activations/layer6_attention_weight_min": -55.561424255371094,
      "activations/layer7_attention_weight_max": 78.96035766601562,
      "activations/layer7_attention_weight_min": -77.1377944946289,
      "activations/layer8_attention_weight_max": 47.19239807128906,
      "activations/layer8_attention_weight_min": -48.83155822753906,
      "activations/layer9_attention_weight_max": 54.37150192260742,
      "activations/layer9_attention_weight_min": -54.18253707885742,
      "epoch": 8.61,
      "learning_rate": 9.544810606060606e-05,
      "loss": 2.7907,
      "step": 148150
    },
    {
      "activations/layer0_attention_weight_max": 15.168445587158203,
      "activations/layer0_attention_weight_min": -14.039322853088379,
      "activations/layer10_attention_weight_max": 37.40216827392578,
      "activations/layer10_attention_weight_min": -35.212772369384766,
      "activations/layer11_attention_weight_max": 33.889915466308594,
      "activations/layer11_attention_weight_min": -31.34723663330078,
      "activations/layer12_attention_weight_max": 19.187068939208984,
      "activations/layer12_attention_weight_min": -27.921518325805664,
      "activations/layer13_attention_weight_max": 46.94834899902344,
      "activations/layer13_attention_weight_min": -29.870342254638672,
      "activations/layer14_attention_weight_max": 41.042747497558594,
      "activations/layer14_attention_weight_min": -32.995357513427734,
      "activations/layer15_attention_weight_max": 39.14533996582031,
      "activations/layer15_attention_weight_min": -30.68627166748047,
      "activations/layer16_attention_weight_max": 33.1143913269043,
      "activations/layer16_attention_weight_min": -30.97829246520996,
      "activations/layer17_attention_weight_max": 57.48054885864258,
      "activations/layer17_attention_weight_min": -42.78091049194336,
      "activations/layer18_attention_weight_max": 52.02960205078125,
      "activations/layer18_attention_weight_min": -38.836326599121094,
      "activations/layer19_attention_weight_max": 25.97626304626465,
      "activations/layer19_attention_weight_min": -20.722713470458984,
      "activations/layer1_attention_weight_max": 16.729042053222656,
      "activations/layer1_attention_weight_min": -14.270956993103027,
      "activations/layer20_attention_weight_max": 23.52047348022461,
      "activations/layer20_attention_weight_min": -23.640239715576172,
      "activations/layer21_attention_weight_max": 38.96985626220703,
      "activations/layer21_attention_weight_min": -21.914901733398438,
      "activations/layer22_attention_weight_max": 36.24120330810547,
      "activations/layer22_attention_weight_min": -24.69635009765625,
      "activations/layer23_attention_weight_max": 35.64678192138672,
      "activations/layer23_attention_weight_min": -20.114526748657227,
      "activations/layer2_attention_weight_max": 30.25201988220215,
      "activations/layer2_attention_weight_min": -27.541088104248047,
      "activations/layer3_attention_weight_max": 88.56202697753906,
      "activations/layer3_attention_weight_min": -90.80398559570312,
      "activations/layer4_attention_weight_max": 101.7302017211914,
      "activations/layer4_attention_weight_min": -95.48619842529297,
      "activations/layer5_attention_weight_max": 82.59585571289062,
      "activations/layer5_attention_weight_min": -82.3166732788086,
      "activations/layer6_attention_weight_max": 56.393985748291016,
      "activations/layer6_attention_weight_min": -57.46595764160156,
      "activations/layer7_attention_weight_max": 79.41999816894531,
      "activations/layer7_attention_weight_min": -76.53948974609375,
      "activations/layer8_attention_weight_max": 47.45189666748047,
      "activations/layer8_attention_weight_min": -49.98870849609375,
      "activations/layer9_attention_weight_max": 52.02077102661133,
      "activations/layer9_attention_weight_min": -51.49015426635742,
      "epoch": 8.61,
      "learning_rate": 9.542916666666666e-05,
      "loss": 2.7897,
      "step": 148200
    },
    {
      "activations/layer0_attention_weight_max": 15.2416410446167,
      "activations/layer0_attention_weight_min": -14.356230735778809,
      "activations/layer10_attention_weight_max": 40.79936981201172,
      "activations/layer10_attention_weight_min": -37.43289566040039,
      "activations/layer11_attention_weight_max": 32.68487548828125,
      "activations/layer11_attention_weight_min": -32.580101013183594,
      "activations/layer12_attention_weight_max": 19.472280502319336,
      "activations/layer12_attention_weight_min": -25.83537483215332,
      "activations/layer13_attention_weight_max": 45.749141693115234,
      "activations/layer13_attention_weight_min": -32.38431167602539,
      "activations/layer14_attention_weight_max": 52.17223358154297,
      "activations/layer14_attention_weight_min": -32.63936233520508,
      "activations/layer15_attention_weight_max": 42.40810775756836,
      "activations/layer15_attention_weight_min": -29.11552619934082,
      "activations/layer16_attention_weight_max": 37.348045349121094,
      "activations/layer16_attention_weight_min": -30.05450439453125,
      "activations/layer17_attention_weight_max": 59.51976776123047,
      "activations/layer17_attention_weight_min": -46.53408432006836,
      "activations/layer18_attention_weight_max": 53.146785736083984,
      "activations/layer18_attention_weight_min": -40.159725189208984,
      "activations/layer19_attention_weight_max": 23.6377010345459,
      "activations/layer19_attention_weight_min": -22.465919494628906,
      "activations/layer1_attention_weight_max": 17.138011932373047,
      "activations/layer1_attention_weight_min": -14.687647819519043,
      "activations/layer20_attention_weight_max": 27.70318603515625,
      "activations/layer20_attention_weight_min": -22.4445858001709,
      "activations/layer21_attention_weight_max": 46.02918243408203,
      "activations/layer21_attention_weight_min": -23.817724227905273,
      "activations/layer22_attention_weight_max": 34.21201705932617,
      "activations/layer22_attention_weight_min": -25.717817306518555,
      "activations/layer23_attention_weight_max": 39.789024353027344,
      "activations/layer23_attention_weight_min": -21.025691986083984,
      "activations/layer2_attention_weight_max": 32.916900634765625,
      "activations/layer2_attention_weight_min": -31.139158248901367,
      "activations/layer3_attention_weight_max": 89.93427276611328,
      "activations/layer3_attention_weight_min": -92.29074096679688,
      "activations/layer4_attention_weight_max": 101.75298309326172,
      "activations/layer4_attention_weight_min": -99.89200592041016,
      "activations/layer5_attention_weight_max": 81.50453186035156,
      "activations/layer5_attention_weight_min": -78.46553039550781,
      "activations/layer6_attention_weight_max": 59.74852752685547,
      "activations/layer6_attention_weight_min": -54.27161407470703,
      "activations/layer7_attention_weight_max": 81.0069580078125,
      "activations/layer7_attention_weight_min": -77.91377258300781,
      "activations/layer8_attention_weight_max": 45.676334381103516,
      "activations/layer8_attention_weight_min": -48.16777801513672,
      "activations/layer9_attention_weight_max": 54.10683059692383,
      "activations/layer9_attention_weight_min": -55.30469512939453,
      "epoch": 8.61,
      "learning_rate": 9.541022727272726e-05,
      "loss": 2.8196,
      "step": 148250
    },
    {
      "activations/layer0_attention_weight_max": 14.737343788146973,
      "activations/layer0_attention_weight_min": -14.223845481872559,
      "activations/layer10_attention_weight_max": 37.72819519042969,
      "activations/layer10_attention_weight_min": -37.36201858520508,
      "activations/layer11_attention_weight_max": 31.207210540771484,
      "activations/layer11_attention_weight_min": -32.73858642578125,
      "activations/layer12_attention_weight_max": 19.634185791015625,
      "activations/layer12_attention_weight_min": -26.51002311706543,
      "activations/layer13_attention_weight_max": 37.40319061279297,
      "activations/layer13_attention_weight_min": -30.99301528930664,
      "activations/layer14_attention_weight_max": 41.993656158447266,
      "activations/layer14_attention_weight_min": -30.358566284179688,
      "activations/layer15_attention_weight_max": 32.66600799560547,
      "activations/layer15_attention_weight_min": -30.93260955810547,
      "activations/layer16_attention_weight_max": 28.7950439453125,
      "activations/layer16_attention_weight_min": -28.032533645629883,
      "activations/layer17_attention_weight_max": 53.695823669433594,
      "activations/layer17_attention_weight_min": -41.44691467285156,
      "activations/layer18_attention_weight_max": 45.29132080078125,
      "activations/layer18_attention_weight_min": -34.46340560913086,
      "activations/layer19_attention_weight_max": 21.437604904174805,
      "activations/layer19_attention_weight_min": -20.0640869140625,
      "activations/layer1_attention_weight_max": 17.683746337890625,
      "activations/layer1_attention_weight_min": -14.199137687683105,
      "activations/layer20_attention_weight_max": 22.81169891357422,
      "activations/layer20_attention_weight_min": -21.310672760009766,
      "activations/layer21_attention_weight_max": 33.1423225402832,
      "activations/layer21_attention_weight_min": -22.566402435302734,
      "activations/layer22_attention_weight_max": 34.4637336730957,
      "activations/layer22_attention_weight_min": -24.47180938720703,
      "activations/layer23_attention_weight_max": 37.117576599121094,
      "activations/layer23_attention_weight_min": -22.850643157958984,
      "activations/layer2_attention_weight_max": 29.59215545654297,
      "activations/layer2_attention_weight_min": -27.961881637573242,
      "activations/layer3_attention_weight_max": 85.87462615966797,
      "activations/layer3_attention_weight_min": -90.55894470214844,
      "activations/layer4_attention_weight_max": 100.8624496459961,
      "activations/layer4_attention_weight_min": -91.05864715576172,
      "activations/layer5_attention_weight_max": 79.1910629272461,
      "activations/layer5_attention_weight_min": -74.69561004638672,
      "activations/layer6_attention_weight_max": 53.020423889160156,
      "activations/layer6_attention_weight_min": -53.03792190551758,
      "activations/layer7_attention_weight_max": 76.74111938476562,
      "activations/layer7_attention_weight_min": -75.05796813964844,
      "activations/layer8_attention_weight_max": 43.79160690307617,
      "activations/layer8_attention_weight_min": -45.39558792114258,
      "activations/layer9_attention_weight_max": 53.6259880065918,
      "activations/layer9_attention_weight_min": -51.0359001159668,
      "epoch": 8.62,
      "learning_rate": 9.539128787878787e-05,
      "loss": 2.8097,
      "step": 148300
    },
    {
      "activations/layer0_attention_weight_max": 15.568669319152832,
      "activations/layer0_attention_weight_min": -13.996047973632812,
      "activations/layer10_attention_weight_max": 40.761417388916016,
      "activations/layer10_attention_weight_min": -38.774410247802734,
      "activations/layer11_attention_weight_max": 31.73476219177246,
      "activations/layer11_attention_weight_min": -32.29458999633789,
      "activations/layer12_attention_weight_max": 25.405893325805664,
      "activations/layer12_attention_weight_min": -25.80856704711914,
      "activations/layer13_attention_weight_max": 47.959590911865234,
      "activations/layer13_attention_weight_min": -33.795841217041016,
      "activations/layer14_attention_weight_max": 46.54050064086914,
      "activations/layer14_attention_weight_min": -34.00029754638672,
      "activations/layer15_attention_weight_max": 36.67816162109375,
      "activations/layer15_attention_weight_min": -32.94766616821289,
      "activations/layer16_attention_weight_max": 35.994815826416016,
      "activations/layer16_attention_weight_min": -28.95838737487793,
      "activations/layer17_attention_weight_max": 56.665122985839844,
      "activations/layer17_attention_weight_min": -46.252296447753906,
      "activations/layer18_attention_weight_max": 51.55590057373047,
      "activations/layer18_attention_weight_min": -38.91539001464844,
      "activations/layer19_attention_weight_max": 21.651212692260742,
      "activations/layer19_attention_weight_min": -21.685407638549805,
      "activations/layer1_attention_weight_max": 16.336734771728516,
      "activations/layer1_attention_weight_min": -15.007240295410156,
      "activations/layer20_attention_weight_max": 25.108959197998047,
      "activations/layer20_attention_weight_min": -21.091676712036133,
      "activations/layer21_attention_weight_max": 39.89963912963867,
      "activations/layer21_attention_weight_min": -24.886642456054688,
      "activations/layer22_attention_weight_max": 34.88850402832031,
      "activations/layer22_attention_weight_min": -26.448932647705078,
      "activations/layer23_attention_weight_max": 40.47722625732422,
      "activations/layer23_attention_weight_min": -23.73652458190918,
      "activations/layer2_attention_weight_max": 30.621299743652344,
      "activations/layer2_attention_weight_min": -30.284955978393555,
      "activations/layer3_attention_weight_max": 89.6635513305664,
      "activations/layer3_attention_weight_min": -91.60067749023438,
      "activations/layer4_attention_weight_max": 101.64501190185547,
      "activations/layer4_attention_weight_min": -98.34767150878906,
      "activations/layer5_attention_weight_max": 84.45172882080078,
      "activations/layer5_attention_weight_min": -84.36229705810547,
      "activations/layer6_attention_weight_max": 58.304649353027344,
      "activations/layer6_attention_weight_min": -54.550411224365234,
      "activations/layer7_attention_weight_max": 78.85736846923828,
      "activations/layer7_attention_weight_min": -75.32452392578125,
      "activations/layer8_attention_weight_max": 47.6409912109375,
      "activations/layer8_attention_weight_min": -49.49611282348633,
      "activations/layer9_attention_weight_max": 53.2841796875,
      "activations/layer9_attention_weight_min": -51.182315826416016,
      "epoch": 8.62,
      "learning_rate": 9.537234848484848e-05,
      "loss": 2.8051,
      "step": 148350
    },
    {
      "activations/layer0_attention_weight_max": 14.899115562438965,
      "activations/layer0_attention_weight_min": -14.261479377746582,
      "activations/layer10_attention_weight_max": 37.59612274169922,
      "activations/layer10_attention_weight_min": -36.85474395751953,
      "activations/layer11_attention_weight_max": 32.36079406738281,
      "activations/layer11_attention_weight_min": -33.42185974121094,
      "activations/layer12_attention_weight_max": 20.427963256835938,
      "activations/layer12_attention_weight_min": -27.334470748901367,
      "activations/layer13_attention_weight_max": 41.13103103637695,
      "activations/layer13_attention_weight_min": -28.16944122314453,
      "activations/layer14_attention_weight_max": 41.60554122924805,
      "activations/layer14_attention_weight_min": -28.776159286499023,
      "activations/layer15_attention_weight_max": 41.06859588623047,
      "activations/layer15_attention_weight_min": -26.7830867767334,
      "activations/layer16_attention_weight_max": 35.89896011352539,
      "activations/layer16_attention_weight_min": -26.068620681762695,
      "activations/layer17_attention_weight_max": 57.90245819091797,
      "activations/layer17_attention_weight_min": -41.97401428222656,
      "activations/layer18_attention_weight_max": 50.92202377319336,
      "activations/layer18_attention_weight_min": -34.85084915161133,
      "activations/layer19_attention_weight_max": 24.493507385253906,
      "activations/layer19_attention_weight_min": -21.44765281677246,
      "activations/layer1_attention_weight_max": 16.653833389282227,
      "activations/layer1_attention_weight_min": -17.528793334960938,
      "activations/layer20_attention_weight_max": 22.936065673828125,
      "activations/layer20_attention_weight_min": -21.25253677368164,
      "activations/layer21_attention_weight_max": 46.72977828979492,
      "activations/layer21_attention_weight_min": -23.86533546447754,
      "activations/layer22_attention_weight_max": 35.670196533203125,
      "activations/layer22_attention_weight_min": -27.445812225341797,
      "activations/layer23_attention_weight_max": 40.016998291015625,
      "activations/layer23_attention_weight_min": -22.183319091796875,
      "activations/layer2_attention_weight_max": 31.62299919128418,
      "activations/layer2_attention_weight_min": -29.78241729736328,
      "activations/layer3_attention_weight_max": 91.2899398803711,
      "activations/layer3_attention_weight_min": -93.24885559082031,
      "activations/layer4_attention_weight_max": 106.16680145263672,
      "activations/layer4_attention_weight_min": -100.26398468017578,
      "activations/layer5_attention_weight_max": 82.17185974121094,
      "activations/layer5_attention_weight_min": -81.68380737304688,
      "activations/layer6_attention_weight_max": 55.441162109375,
      "activations/layer6_attention_weight_min": -54.36497116088867,
      "activations/layer7_attention_weight_max": 79.7940444946289,
      "activations/layer7_attention_weight_min": -75.40421295166016,
      "activations/layer8_attention_weight_max": 48.84312057495117,
      "activations/layer8_attention_weight_min": -50.38401794433594,
      "activations/layer9_attention_weight_max": 51.414947509765625,
      "activations/layer9_attention_weight_min": -48.80912399291992,
      "epoch": 8.62,
      "learning_rate": 9.535340909090908e-05,
      "loss": 2.7959,
      "step": 148400
    },
    {
      "activations/layer0_attention_weight_max": 16.489593505859375,
      "activations/layer0_attention_weight_min": -14.34614086151123,
      "activations/layer10_attention_weight_max": 35.835243225097656,
      "activations/layer10_attention_weight_min": -36.84893035888672,
      "activations/layer11_attention_weight_max": 32.378623962402344,
      "activations/layer11_attention_weight_min": -31.24909210205078,
      "activations/layer12_attention_weight_max": 19.934932708740234,
      "activations/layer12_attention_weight_min": -24.315555572509766,
      "activations/layer13_attention_weight_max": 43.910057067871094,
      "activations/layer13_attention_weight_min": -31.839075088500977,
      "activations/layer14_attention_weight_max": 34.576759338378906,
      "activations/layer14_attention_weight_min": -30.508329391479492,
      "activations/layer15_attention_weight_max": 36.6784782409668,
      "activations/layer15_attention_weight_min": -29.473384857177734,
      "activations/layer16_attention_weight_max": 35.68212127685547,
      "activations/layer16_attention_weight_min": -29.797807693481445,
      "activations/layer17_attention_weight_max": 51.55946350097656,
      "activations/layer17_attention_weight_min": -42.44990158081055,
      "activations/layer18_attention_weight_max": 46.085391998291016,
      "activations/layer18_attention_weight_min": -36.85211944580078,
      "activations/layer19_attention_weight_max": 20.971839904785156,
      "activations/layer19_attention_weight_min": -20.78290367126465,
      "activations/layer1_attention_weight_max": 16.66761589050293,
      "activations/layer1_attention_weight_min": -15.185705184936523,
      "activations/layer20_attention_weight_max": 23.550151824951172,
      "activations/layer20_attention_weight_min": -21.12862777709961,
      "activations/layer21_attention_weight_max": 35.276493072509766,
      "activations/layer21_attention_weight_min": -24.255783081054688,
      "activations/layer22_attention_weight_max": 31.952266693115234,
      "activations/layer22_attention_weight_min": -25.124340057373047,
      "activations/layer23_attention_weight_max": 38.522117614746094,
      "activations/layer23_attention_weight_min": -22.916698455810547,
      "activations/layer2_attention_weight_max": 29.786544799804688,
      "activations/layer2_attention_weight_min": -28.13638687133789,
      "activations/layer3_attention_weight_max": 89.11293029785156,
      "activations/layer3_attention_weight_min": -90.77867889404297,
      "activations/layer4_attention_weight_max": 98.3644790649414,
      "activations/layer4_attention_weight_min": -93.03681182861328,
      "activations/layer5_attention_weight_max": 80.1146240234375,
      "activations/layer5_attention_weight_min": -75.7342529296875,
      "activations/layer6_attention_weight_max": 54.131168365478516,
      "activations/layer6_attention_weight_min": -52.72755432128906,
      "activations/layer7_attention_weight_max": 76.19794464111328,
      "activations/layer7_attention_weight_min": -74.61225128173828,
      "activations/layer8_attention_weight_max": 44.11358642578125,
      "activations/layer8_attention_weight_min": -47.34364700317383,
      "activations/layer9_attention_weight_max": 49.79972839355469,
      "activations/layer9_attention_weight_min": -51.95642852783203,
      "epoch": 8.63,
      "learning_rate": 9.533446969696969e-05,
      "loss": 2.803,
      "step": 148450
    },
    {
      "activations/layer0_attention_weight_max": 15.231732368469238,
      "activations/layer0_attention_weight_min": -14.039762496948242,
      "activations/layer10_attention_weight_max": 35.36604309082031,
      "activations/layer10_attention_weight_min": -36.65996551513672,
      "activations/layer11_attention_weight_max": 29.158004760742188,
      "activations/layer11_attention_weight_min": -32.522850036621094,
      "activations/layer12_attention_weight_max": 21.153854370117188,
      "activations/layer12_attention_weight_min": -25.694293975830078,
      "activations/layer13_attention_weight_max": 38.77504348754883,
      "activations/layer13_attention_weight_min": -29.23980712890625,
      "activations/layer14_attention_weight_max": 39.57326126098633,
      "activations/layer14_attention_weight_min": -29.555294036865234,
      "activations/layer15_attention_weight_max": 34.68717956542969,
      "activations/layer15_attention_weight_min": -29.853267669677734,
      "activations/layer16_attention_weight_max": 34.67479705810547,
      "activations/layer16_attention_weight_min": -27.83993911743164,
      "activations/layer17_attention_weight_max": 51.878929138183594,
      "activations/layer17_attention_weight_min": -46.04855728149414,
      "activations/layer18_attention_weight_max": 47.390071868896484,
      "activations/layer18_attention_weight_min": -37.62419891357422,
      "activations/layer19_attention_weight_max": 20.584184646606445,
      "activations/layer19_attention_weight_min": -20.916479110717773,
      "activations/layer1_attention_weight_max": 16.219676971435547,
      "activations/layer1_attention_weight_min": -16.267942428588867,
      "activations/layer20_attention_weight_max": 20.94756317138672,
      "activations/layer20_attention_weight_min": -20.98921012878418,
      "activations/layer21_attention_weight_max": 36.57490158081055,
      "activations/layer21_attention_weight_min": -24.900482177734375,
      "activations/layer22_attention_weight_max": 30.519338607788086,
      "activations/layer22_attention_weight_min": -25.90137481689453,
      "activations/layer23_attention_weight_max": 36.812416076660156,
      "activations/layer23_attention_weight_min": -24.019506454467773,
      "activations/layer2_attention_weight_max": 30.828670501708984,
      "activations/layer2_attention_weight_min": -28.80109977722168,
      "activations/layer3_attention_weight_max": 80.64171600341797,
      "activations/layer3_attention_weight_min": -84.94121551513672,
      "activations/layer4_attention_weight_max": 94.26946258544922,
      "activations/layer4_attention_weight_min": -89.09709930419922,
      "activations/layer5_attention_weight_max": 77.50224304199219,
      "activations/layer5_attention_weight_min": -75.54252624511719,
      "activations/layer6_attention_weight_max": 52.58853530883789,
      "activations/layer6_attention_weight_min": -52.5990104675293,
      "activations/layer7_attention_weight_max": 69.4494400024414,
      "activations/layer7_attention_weight_min": -73.91414642333984,
      "activations/layer8_attention_weight_max": 45.47120666503906,
      "activations/layer8_attention_weight_min": -45.58183288574219,
      "activations/layer9_attention_weight_max": 49.98883056640625,
      "activations/layer9_attention_weight_min": -52.152462005615234,
      "epoch": 8.63,
      "learning_rate": 9.53155303030303e-05,
      "loss": 2.8043,
      "step": 148500
    },
    {
      "activations/layer0_attention_weight_max": 15.08930492401123,
      "activations/layer0_attention_weight_min": -14.5557861328125,
      "activations/layer10_attention_weight_max": 35.67848587036133,
      "activations/layer10_attention_weight_min": -34.589412689208984,
      "activations/layer11_attention_weight_max": 29.654836654663086,
      "activations/layer11_attention_weight_min": -31.208189010620117,
      "activations/layer12_attention_weight_max": 15.39084529876709,
      "activations/layer12_attention_weight_min": -22.56583023071289,
      "activations/layer13_attention_weight_max": 36.292625427246094,
      "activations/layer13_attention_weight_min": -31.89024543762207,
      "activations/layer14_attention_weight_max": 37.44328308105469,
      "activations/layer14_attention_weight_min": -32.00185012817383,
      "activations/layer15_attention_weight_max": 32.766414642333984,
      "activations/layer15_attention_weight_min": -31.24759864807129,
      "activations/layer16_attention_weight_max": 29.37493133544922,
      "activations/layer16_attention_weight_min": -28.985952377319336,
      "activations/layer17_attention_weight_max": 55.90119171142578,
      "activations/layer17_attention_weight_min": -47.55792999267578,
      "activations/layer18_attention_weight_max": 48.52156066894531,
      "activations/layer18_attention_weight_min": -40.62881851196289,
      "activations/layer19_attention_weight_max": 22.735639572143555,
      "activations/layer19_attention_weight_min": -21.77425193786621,
      "activations/layer1_attention_weight_max": 16.141826629638672,
      "activations/layer1_attention_weight_min": -14.261645317077637,
      "activations/layer20_attention_weight_max": 21.557003021240234,
      "activations/layer20_attention_weight_min": -26.125486373901367,
      "activations/layer21_attention_weight_max": 40.31183624267578,
      "activations/layer21_attention_weight_min": -26.16697883605957,
      "activations/layer22_attention_weight_max": 32.20156478881836,
      "activations/layer22_attention_weight_min": -27.479333877563477,
      "activations/layer23_attention_weight_max": 37.44645690917969,
      "activations/layer23_attention_weight_min": -23.319843292236328,
      "activations/layer2_attention_weight_max": 29.316877365112305,
      "activations/layer2_attention_weight_min": -28.577863693237305,
      "activations/layer3_attention_weight_max": 79.1390609741211,
      "activations/layer3_attention_weight_min": -89.72129821777344,
      "activations/layer4_attention_weight_max": 95.01261901855469,
      "activations/layer4_attention_weight_min": -95.6929702758789,
      "activations/layer5_attention_weight_max": 76.5328140258789,
      "activations/layer5_attention_weight_min": -73.3694076538086,
      "activations/layer6_attention_weight_max": 53.792335510253906,
      "activations/layer6_attention_weight_min": -51.82044219970703,
      "activations/layer7_attention_weight_max": 73.60098266601562,
      "activations/layer7_attention_weight_min": -72.69253540039062,
      "activations/layer8_attention_weight_max": 42.36691665649414,
      "activations/layer8_attention_weight_min": -43.62777328491211,
      "activations/layer9_attention_weight_max": 46.9382209777832,
      "activations/layer9_attention_weight_min": -48.54233932495117,
      "epoch": 8.63,
      "learning_rate": 9.529659090909091e-05,
      "loss": 2.8092,
      "step": 148550
    },
    {
      "activations/layer0_attention_weight_max": 15.689458847045898,
      "activations/layer0_attention_weight_min": -14.202214241027832,
      "activations/layer10_attention_weight_max": 36.614036560058594,
      "activations/layer10_attention_weight_min": -33.9359245300293,
      "activations/layer11_attention_weight_max": 30.1380615234375,
      "activations/layer11_attention_weight_min": -31.00006103515625,
      "activations/layer12_attention_weight_max": 19.044282913208008,
      "activations/layer12_attention_weight_min": -29.798307418823242,
      "activations/layer13_attention_weight_max": 43.85797882080078,
      "activations/layer13_attention_weight_min": -30.306739807128906,
      "activations/layer14_attention_weight_max": 36.73319625854492,
      "activations/layer14_attention_weight_min": -29.91754913330078,
      "activations/layer15_attention_weight_max": 32.19402313232422,
      "activations/layer15_attention_weight_min": -28.284454345703125,
      "activations/layer16_attention_weight_max": 30.023021697998047,
      "activations/layer16_attention_weight_min": -26.948505401611328,
      "activations/layer17_attention_weight_max": 48.27629852294922,
      "activations/layer17_attention_weight_min": -41.714534759521484,
      "activations/layer18_attention_weight_max": 43.76889419555664,
      "activations/layer18_attention_weight_min": -36.30173873901367,
      "activations/layer19_attention_weight_max": 19.18137550354004,
      "activations/layer19_attention_weight_min": -19.919601440429688,
      "activations/layer1_attention_weight_max": 17.850046157836914,
      "activations/layer1_attention_weight_min": -16.738920211791992,
      "activations/layer20_attention_weight_max": 21.000953674316406,
      "activations/layer20_attention_weight_min": -19.30095672607422,
      "activations/layer21_attention_weight_max": 32.30718231201172,
      "activations/layer21_attention_weight_min": -23.34255027770996,
      "activations/layer22_attention_weight_max": 30.54879379272461,
      "activations/layer22_attention_weight_min": -25.453338623046875,
      "activations/layer23_attention_weight_max": 37.230796813964844,
      "activations/layer23_attention_weight_min": -23.698667526245117,
      "activations/layer2_attention_weight_max": 29.68524742126465,
      "activations/layer2_attention_weight_min": -28.908735275268555,
      "activations/layer3_attention_weight_max": 86.77484893798828,
      "activations/layer3_attention_weight_min": -94.03506469726562,
      "activations/layer4_attention_weight_max": 98.21136474609375,
      "activations/layer4_attention_weight_min": -88.9219970703125,
      "activations/layer5_attention_weight_max": 77.25918579101562,
      "activations/layer5_attention_weight_min": -73.87220764160156,
      "activations/layer6_attention_weight_max": 56.2990837097168,
      "activations/layer6_attention_weight_min": -50.019004821777344,
      "activations/layer7_attention_weight_max": 72.8885726928711,
      "activations/layer7_attention_weight_min": -68.50865173339844,
      "activations/layer8_attention_weight_max": 42.744056701660156,
      "activations/layer8_attention_weight_min": -44.728267669677734,
      "activations/layer9_attention_weight_max": 49.17280197143555,
      "activations/layer9_attention_weight_min": -47.332645416259766,
      "epoch": 8.63,
      "learning_rate": 9.52776515151515e-05,
      "loss": 2.8083,
      "step": 148600
    },
    {
      "activations/layer0_attention_weight_max": 14.639236450195312,
      "activations/layer0_attention_weight_min": -14.513545036315918,
      "activations/layer10_attention_weight_max": 40.80208969116211,
      "activations/layer10_attention_weight_min": -37.331268310546875,
      "activations/layer11_attention_weight_max": 34.48686218261719,
      "activations/layer11_attention_weight_min": -32.124916076660156,
      "activations/layer12_attention_weight_max": 27.07451820373535,
      "activations/layer12_attention_weight_min": -24.78289222717285,
      "activations/layer13_attention_weight_max": 47.212894439697266,
      "activations/layer13_attention_weight_min": -33.207427978515625,
      "activations/layer14_attention_weight_max": 39.223533630371094,
      "activations/layer14_attention_weight_min": -30.474506378173828,
      "activations/layer15_attention_weight_max": 38.82997512817383,
      "activations/layer15_attention_weight_min": -30.15190315246582,
      "activations/layer16_attention_weight_max": 33.49256896972656,
      "activations/layer16_attention_weight_min": -29.42402458190918,
      "activations/layer17_attention_weight_max": 54.69392395019531,
      "activations/layer17_attention_weight_min": -45.15616989135742,
      "activations/layer18_attention_weight_max": 48.12018585205078,
      "activations/layer18_attention_weight_min": -37.21113967895508,
      "activations/layer19_attention_weight_max": 20.86715316772461,
      "activations/layer19_attention_weight_min": -21.74372673034668,
      "activations/layer1_attention_weight_max": 17.50226593017578,
      "activations/layer1_attention_weight_min": -14.579813957214355,
      "activations/layer20_attention_weight_max": 23.28260040283203,
      "activations/layer20_attention_weight_min": -23.814273834228516,
      "activations/layer21_attention_weight_max": 42.20957565307617,
      "activations/layer21_attention_weight_min": -24.456762313842773,
      "activations/layer22_attention_weight_max": 30.237939834594727,
      "activations/layer22_attention_weight_min": -25.321821212768555,
      "activations/layer23_attention_weight_max": 36.723567962646484,
      "activations/layer23_attention_weight_min": -22.32712173461914,
      "activations/layer2_attention_weight_max": 30.392629623413086,
      "activations/layer2_attention_weight_min": -28.252517700195312,
      "activations/layer3_attention_weight_max": 88.3270263671875,
      "activations/layer3_attention_weight_min": -90.82907104492188,
      "activations/layer4_attention_weight_max": 101.99172973632812,
      "activations/layer4_attention_weight_min": -96.81627655029297,
      "activations/layer5_attention_weight_max": 80.88691711425781,
      "activations/layer5_attention_weight_min": -79.3418197631836,
      "activations/layer6_attention_weight_max": 56.18022918701172,
      "activations/layer6_attention_weight_min": -53.96241760253906,
      "activations/layer7_attention_weight_max": 79.74571990966797,
      "activations/layer7_attention_weight_min": -73.38568115234375,
      "activations/layer8_attention_weight_max": 46.88302230834961,
      "activations/layer8_attention_weight_min": -48.557010650634766,
      "activations/layer9_attention_weight_max": 52.05988693237305,
      "activations/layer9_attention_weight_min": -50.86686706542969,
      "epoch": 8.64,
      "learning_rate": 9.525871212121211e-05,
      "loss": 2.7904,
      "step": 148650
    },
    {
      "activations/layer0_attention_weight_max": 14.462285041809082,
      "activations/layer0_attention_weight_min": -14.267260551452637,
      "activations/layer10_attention_weight_max": 40.38533401489258,
      "activations/layer10_attention_weight_min": -35.90056228637695,
      "activations/layer11_attention_weight_max": 33.517269134521484,
      "activations/layer11_attention_weight_min": -32.016849517822266,
      "activations/layer12_attention_weight_max": 20.90180778503418,
      "activations/layer12_attention_weight_min": -24.154502868652344,
      "activations/layer13_attention_weight_max": 45.19697952270508,
      "activations/layer13_attention_weight_min": -30.551578521728516,
      "activations/layer14_attention_weight_max": 40.00700759887695,
      "activations/layer14_attention_weight_min": -30.776817321777344,
      "activations/layer15_attention_weight_max": 35.61367416381836,
      "activations/layer15_attention_weight_min": -29.018186569213867,
      "activations/layer16_attention_weight_max": 35.51172637939453,
      "activations/layer16_attention_weight_min": -27.915037155151367,
      "activations/layer17_attention_weight_max": 57.11078643798828,
      "activations/layer17_attention_weight_min": -46.49592208862305,
      "activations/layer18_attention_weight_max": 52.17089080810547,
      "activations/layer18_attention_weight_min": -38.20486068725586,
      "activations/layer19_attention_weight_max": 22.274517059326172,
      "activations/layer19_attention_weight_min": -20.27214813232422,
      "activations/layer1_attention_weight_max": 16.570186614990234,
      "activations/layer1_attention_weight_min": -14.437461853027344,
      "activations/layer20_attention_weight_max": 25.008981704711914,
      "activations/layer20_attention_weight_min": -21.432491302490234,
      "activations/layer21_attention_weight_max": 44.01725387573242,
      "activations/layer21_attention_weight_min": -24.236520767211914,
      "activations/layer22_attention_weight_max": 36.12824249267578,
      "activations/layer22_attention_weight_min": -27.0908203125,
      "activations/layer23_attention_weight_max": 43.05710983276367,
      "activations/layer23_attention_weight_min": -24.33717918395996,
      "activations/layer2_attention_weight_max": 30.084936141967773,
      "activations/layer2_attention_weight_min": -27.51832389831543,
      "activations/layer3_attention_weight_max": 82.70970153808594,
      "activations/layer3_attention_weight_min": -85.99691772460938,
      "activations/layer4_attention_weight_max": 99.35395812988281,
      "activations/layer4_attention_weight_min": -91.26280975341797,
      "activations/layer5_attention_weight_max": 79.39866638183594,
      "activations/layer5_attention_weight_min": -73.66778564453125,
      "activations/layer6_attention_weight_max": 57.700321197509766,
      "activations/layer6_attention_weight_min": -52.885257720947266,
      "activations/layer7_attention_weight_max": 83.72677612304688,
      "activations/layer7_attention_weight_min": -75.32613372802734,
      "activations/layer8_attention_weight_max": 50.156558990478516,
      "activations/layer8_attention_weight_min": -49.78569412231445,
      "activations/layer9_attention_weight_max": 52.441200256347656,
      "activations/layer9_attention_weight_min": -49.09825134277344,
      "epoch": 8.64,
      "learning_rate": 9.523977272727273e-05,
      "loss": 2.8109,
      "step": 148700
    },
    {
      "activations/layer0_attention_weight_max": 15.292210578918457,
      "activations/layer0_attention_weight_min": -14.32729721069336,
      "activations/layer10_attention_weight_max": 35.352230072021484,
      "activations/layer10_attention_weight_min": -38.556644439697266,
      "activations/layer11_attention_weight_max": 29.273025512695312,
      "activations/layer11_attention_weight_min": -34.297080993652344,
      "activations/layer12_attention_weight_max": 20.759092330932617,
      "activations/layer12_attention_weight_min": -24.851104736328125,
      "activations/layer13_attention_weight_max": 41.44189453125,
      "activations/layer13_attention_weight_min": -34.41367721557617,
      "activations/layer14_attention_weight_max": 38.57665252685547,
      "activations/layer14_attention_weight_min": -30.041671752929688,
      "activations/layer15_attention_weight_max": 36.25893783569336,
      "activations/layer15_attention_weight_min": -28.37748146057129,
      "activations/layer16_attention_weight_max": 32.279876708984375,
      "activations/layer16_attention_weight_min": -26.936429977416992,
      "activations/layer17_attention_weight_max": 59.87599563598633,
      "activations/layer17_attention_weight_min": -42.97377014160156,
      "activations/layer18_attention_weight_max": 52.78008270263672,
      "activations/layer18_attention_weight_min": -35.07988739013672,
      "activations/layer19_attention_weight_max": 23.184324264526367,
      "activations/layer19_attention_weight_min": -20.426877975463867,
      "activations/layer1_attention_weight_max": 16.953718185424805,
      "activations/layer1_attention_weight_min": -16.055797576904297,
      "activations/layer20_attention_weight_max": 22.957063674926758,
      "activations/layer20_attention_weight_min": -24.82823371887207,
      "activations/layer21_attention_weight_max": 39.14439010620117,
      "activations/layer21_attention_weight_min": -22.45191764831543,
      "activations/layer22_attention_weight_max": 32.412593841552734,
      "activations/layer22_attention_weight_min": -23.75937843322754,
      "activations/layer23_attention_weight_max": 38.618507385253906,
      "activations/layer23_attention_weight_min": -23.588546752929688,
      "activations/layer2_attention_weight_max": 28.46758270263672,
      "activations/layer2_attention_weight_min": -27.245969772338867,
      "activations/layer3_attention_weight_max": 85.24823760986328,
      "activations/layer3_attention_weight_min": -87.42975616455078,
      "activations/layer4_attention_weight_max": 98.1828842163086,
      "activations/layer4_attention_weight_min": -93.82074737548828,
      "activations/layer5_attention_weight_max": 77.61454772949219,
      "activations/layer5_attention_weight_min": -72.64637756347656,
      "activations/layer6_attention_weight_max": 54.8616828918457,
      "activations/layer6_attention_weight_min": -52.85157775878906,
      "activations/layer7_attention_weight_max": 77.39330291748047,
      "activations/layer7_attention_weight_min": -74.06439208984375,
      "activations/layer8_attention_weight_max": 43.204681396484375,
      "activations/layer8_attention_weight_min": -46.99879837036133,
      "activations/layer9_attention_weight_max": 49.608238220214844,
      "activations/layer9_attention_weight_min": -51.33856964111328,
      "epoch": 8.64,
      "learning_rate": 9.522083333333331e-05,
      "loss": 2.8003,
      "step": 148750
    },
    {
      "activations/layer0_attention_weight_max": 16.72112274169922,
      "activations/layer0_attention_weight_min": -14.079371452331543,
      "activations/layer10_attention_weight_max": 34.062313079833984,
      "activations/layer10_attention_weight_min": -35.32948303222656,
      "activations/layer11_attention_weight_max": 29.56940460205078,
      "activations/layer11_attention_weight_min": -30.93560028076172,
      "activations/layer12_attention_weight_max": 22.694175720214844,
      "activations/layer12_attention_weight_min": -27.160480499267578,
      "activations/layer13_attention_weight_max": 41.6428108215332,
      "activations/layer13_attention_weight_min": -33.37063980102539,
      "activations/layer14_attention_weight_max": 38.76523971557617,
      "activations/layer14_attention_weight_min": -34.2938232421875,
      "activations/layer15_attention_weight_max": 35.076168060302734,
      "activations/layer15_attention_weight_min": -33.486053466796875,
      "activations/layer16_attention_weight_max": 33.02280807495117,
      "activations/layer16_attention_weight_min": -30.194229125976562,
      "activations/layer17_attention_weight_max": 58.13028335571289,
      "activations/layer17_attention_weight_min": -47.37144470214844,
      "activations/layer18_attention_weight_max": 49.73734664916992,
      "activations/layer18_attention_weight_min": -42.05964660644531,
      "activations/layer19_attention_weight_max": 22.93027114868164,
      "activations/layer19_attention_weight_min": -23.510108947753906,
      "activations/layer1_attention_weight_max": 16.046945571899414,
      "activations/layer1_attention_weight_min": -15.214518547058105,
      "activations/layer20_attention_weight_max": 23.472196578979492,
      "activations/layer20_attention_weight_min": -20.71226692199707,
      "activations/layer21_attention_weight_max": 42.85544967651367,
      "activations/layer21_attention_weight_min": -23.202512741088867,
      "activations/layer22_attention_weight_max": 33.163978576660156,
      "activations/layer22_attention_weight_min": -25.0688419342041,
      "activations/layer23_attention_weight_max": 38.09210205078125,
      "activations/layer23_attention_weight_min": -24.642560958862305,
      "activations/layer2_attention_weight_max": 29.740341186523438,
      "activations/layer2_attention_weight_min": -28.30487060546875,
      "activations/layer3_attention_weight_max": 84.0203628540039,
      "activations/layer3_attention_weight_min": -83.49650573730469,
      "activations/layer4_attention_weight_max": 95.94658660888672,
      "activations/layer4_attention_weight_min": -89.84947967529297,
      "activations/layer5_attention_weight_max": 77.80453491210938,
      "activations/layer5_attention_weight_min": -77.37157440185547,
      "activations/layer6_attention_weight_max": 54.153316497802734,
      "activations/layer6_attention_weight_min": -50.42646408081055,
      "activations/layer7_attention_weight_max": 75.87675476074219,
      "activations/layer7_attention_weight_min": -71.13033294677734,
      "activations/layer8_attention_weight_max": 41.77592849731445,
      "activations/layer8_attention_weight_min": -44.33599090576172,
      "activations/layer9_attention_weight_max": 52.315059661865234,
      "activations/layer9_attention_weight_min": -48.64994812011719,
      "epoch": 8.65,
      "learning_rate": 9.520189393939393e-05,
      "loss": 2.7939,
      "step": 148800
    },
    {
      "activations/layer0_attention_weight_max": 14.463910102844238,
      "activations/layer0_attention_weight_min": -14.032310485839844,
      "activations/layer10_attention_weight_max": 36.157447814941406,
      "activations/layer10_attention_weight_min": -34.1082649230957,
      "activations/layer11_attention_weight_max": 29.339096069335938,
      "activations/layer11_attention_weight_min": -32.506710052490234,
      "activations/layer12_attention_weight_max": 17.827878952026367,
      "activations/layer12_attention_weight_min": -28.283348083496094,
      "activations/layer13_attention_weight_max": 39.14139938354492,
      "activations/layer13_attention_weight_min": -31.177021026611328,
      "activations/layer14_attention_weight_max": 38.902042388916016,
      "activations/layer14_attention_weight_min": -30.38967514038086,
      "activations/layer15_attention_weight_max": 35.43404006958008,
      "activations/layer15_attention_weight_min": -28.488380432128906,
      "activations/layer16_attention_weight_max": 31.25414276123047,
      "activations/layer16_attention_weight_min": -27.92613410949707,
      "activations/layer17_attention_weight_max": 53.248924255371094,
      "activations/layer17_attention_weight_min": -43.20246124267578,
      "activations/layer18_attention_weight_max": 48.009395599365234,
      "activations/layer18_attention_weight_min": -35.83272933959961,
      "activations/layer19_attention_weight_max": 21.973876953125,
      "activations/layer19_attention_weight_min": -22.079946517944336,
      "activations/layer1_attention_weight_max": 16.488765716552734,
      "activations/layer1_attention_weight_min": -18.050443649291992,
      "activations/layer20_attention_weight_max": 23.60464096069336,
      "activations/layer20_attention_weight_min": -20.439058303833008,
      "activations/layer21_attention_weight_max": 41.42924118041992,
      "activations/layer21_attention_weight_min": -21.739707946777344,
      "activations/layer22_attention_weight_max": 31.89267349243164,
      "activations/layer22_attention_weight_min": -25.255239486694336,
      "activations/layer23_attention_weight_max": 37.54067611694336,
      "activations/layer23_attention_weight_min": -22.071056365966797,
      "activations/layer2_attention_weight_max": 30.33863067626953,
      "activations/layer2_attention_weight_min": -30.93878746032715,
      "activations/layer3_attention_weight_max": 86.34880828857422,
      "activations/layer3_attention_weight_min": -90.17494201660156,
      "activations/layer4_attention_weight_max": 96.7435531616211,
      "activations/layer4_attention_weight_min": -91.62661743164062,
      "activations/layer5_attention_weight_max": 75.17764282226562,
      "activations/layer5_attention_weight_min": -77.68462371826172,
      "activations/layer6_attention_weight_max": 51.84581756591797,
      "activations/layer6_attention_weight_min": -48.81526565551758,
      "activations/layer7_attention_weight_max": 77.01969909667969,
      "activations/layer7_attention_weight_min": -72.06944274902344,
      "activations/layer8_attention_weight_max": 43.79052734375,
      "activations/layer8_attention_weight_min": -43.465091705322266,
      "activations/layer9_attention_weight_max": 49.01713180541992,
      "activations/layer9_attention_weight_min": -49.04637908935547,
      "epoch": 8.65,
      "learning_rate": 9.518295454545454e-05,
      "loss": 2.8071,
      "step": 148850
    },
    {
      "activations/layer0_attention_weight_max": 14.919486045837402,
      "activations/layer0_attention_weight_min": -13.944632530212402,
      "activations/layer10_attention_weight_max": 37.20245361328125,
      "activations/layer10_attention_weight_min": -34.90755081176758,
      "activations/layer11_attention_weight_max": 30.687397003173828,
      "activations/layer11_attention_weight_min": -31.004840850830078,
      "activations/layer12_attention_weight_max": 19.048171997070312,
      "activations/layer12_attention_weight_min": -21.784561157226562,
      "activations/layer13_attention_weight_max": 41.970123291015625,
      "activations/layer13_attention_weight_min": -28.387781143188477,
      "activations/layer14_attention_weight_max": 36.15091323852539,
      "activations/layer14_attention_weight_min": -30.56157684326172,
      "activations/layer15_attention_weight_max": 34.04029846191406,
      "activations/layer15_attention_weight_min": -28.829504013061523,
      "activations/layer16_attention_weight_max": 27.752294540405273,
      "activations/layer16_attention_weight_min": -27.185176849365234,
      "activations/layer17_attention_weight_max": 51.81489181518555,
      "activations/layer17_attention_weight_min": -44.3243293762207,
      "activations/layer18_attention_weight_max": 45.32769012451172,
      "activations/layer18_attention_weight_min": -35.765804290771484,
      "activations/layer19_attention_weight_max": 24.725317001342773,
      "activations/layer19_attention_weight_min": -21.850934982299805,
      "activations/layer1_attention_weight_max": 16.989587783813477,
      "activations/layer1_attention_weight_min": -15.595292091369629,
      "activations/layer20_attention_weight_max": 21.49688720703125,
      "activations/layer20_attention_weight_min": -22.732696533203125,
      "activations/layer21_attention_weight_max": 38.872501373291016,
      "activations/layer21_attention_weight_min": -23.85739517211914,
      "activations/layer22_attention_weight_max": 30.05438995361328,
      "activations/layer22_attention_weight_min": -23.966936111450195,
      "activations/layer23_attention_weight_max": 36.009307861328125,
      "activations/layer23_attention_weight_min": -20.671470642089844,
      "activations/layer2_attention_weight_max": 30.330810546875,
      "activations/layer2_attention_weight_min": -30.506383895874023,
      "activations/layer3_attention_weight_max": 86.1923599243164,
      "activations/layer3_attention_weight_min": -92.23049926757812,
      "activations/layer4_attention_weight_max": 98.17596435546875,
      "activations/layer4_attention_weight_min": -93.73074340820312,
      "activations/layer5_attention_weight_max": 81.32612609863281,
      "activations/layer5_attention_weight_min": -79.47576904296875,
      "activations/layer6_attention_weight_max": 52.088809967041016,
      "activations/layer6_attention_weight_min": -53.64688491821289,
      "activations/layer7_attention_weight_max": 75.17047882080078,
      "activations/layer7_attention_weight_min": -72.62003326416016,
      "activations/layer8_attention_weight_max": 43.30284881591797,
      "activations/layer8_attention_weight_min": -43.378379821777344,
      "activations/layer9_attention_weight_max": 49.26459884643555,
      "activations/layer9_attention_weight_min": -45.4688720703125,
      "epoch": 8.65,
      "learning_rate": 9.516401515151513e-05,
      "loss": 2.8079,
      "step": 148900
    },
    {
      "activations/layer0_attention_weight_max": 14.773240089416504,
      "activations/layer0_attention_weight_min": -14.064115524291992,
      "activations/layer10_attention_weight_max": 34.61026382446289,
      "activations/layer10_attention_weight_min": -35.00718688964844,
      "activations/layer11_attention_weight_max": 29.216358184814453,
      "activations/layer11_attention_weight_min": -29.71019744873047,
      "activations/layer12_attention_weight_max": 20.036319732666016,
      "activations/layer12_attention_weight_min": -24.12564468383789,
      "activations/layer13_attention_weight_max": 42.34931564331055,
      "activations/layer13_attention_weight_min": -33.938865661621094,
      "activations/layer14_attention_weight_max": 38.82155990600586,
      "activations/layer14_attention_weight_min": -30.302845001220703,
      "activations/layer15_attention_weight_max": 36.7109260559082,
      "activations/layer15_attention_weight_min": -29.815664291381836,
      "activations/layer16_attention_weight_max": 32.17543029785156,
      "activations/layer16_attention_weight_min": -29.521381378173828,
      "activations/layer17_attention_weight_max": 58.19294738769531,
      "activations/layer17_attention_weight_min": -45.27500534057617,
      "activations/layer18_attention_weight_max": 52.85272979736328,
      "activations/layer18_attention_weight_min": -38.914546966552734,
      "activations/layer19_attention_weight_max": 23.018110275268555,
      "activations/layer19_attention_weight_min": -22.355314254760742,
      "activations/layer1_attention_weight_max": 16.58806037902832,
      "activations/layer1_attention_weight_min": -14.828367233276367,
      "activations/layer20_attention_weight_max": 27.802478790283203,
      "activations/layer20_attention_weight_min": -21.889366149902344,
      "activations/layer21_attention_weight_max": 40.13409423828125,
      "activations/layer21_attention_weight_min": -24.394695281982422,
      "activations/layer22_attention_weight_max": 33.002925872802734,
      "activations/layer22_attention_weight_min": -26.19303321838379,
      "activations/layer23_attention_weight_max": 36.84819412231445,
      "activations/layer23_attention_weight_min": -21.833660125732422,
      "activations/layer2_attention_weight_max": 27.71858024597168,
      "activations/layer2_attention_weight_min": -27.828474044799805,
      "activations/layer3_attention_weight_max": 81.11748504638672,
      "activations/layer3_attention_weight_min": -87.09013366699219,
      "activations/layer4_attention_weight_max": 97.84941864013672,
      "activations/layer4_attention_weight_min": -90.1700668334961,
      "activations/layer5_attention_weight_max": 78.09259033203125,
      "activations/layer5_attention_weight_min": -74.84276580810547,
      "activations/layer6_attention_weight_max": 54.8259162902832,
      "activations/layer6_attention_weight_min": -50.82017517089844,
      "activations/layer7_attention_weight_max": 80.13041687011719,
      "activations/layer7_attention_weight_min": -67.77254486083984,
      "activations/layer8_attention_weight_max": 45.64139938354492,
      "activations/layer8_attention_weight_min": -44.00067901611328,
      "activations/layer9_attention_weight_max": 50.7647705078125,
      "activations/layer9_attention_weight_min": -45.71010208129883,
      "epoch": 8.65,
      "learning_rate": 9.514507575757575e-05,
      "loss": 2.8163,
      "step": 148950
    },
    {
      "activations/layer0_attention_weight_max": 17.872615814208984,
      "activations/layer0_attention_weight_min": -14.062505722045898,
      "activations/layer10_attention_weight_max": 38.7559814453125,
      "activations/layer10_attention_weight_min": -36.994232177734375,
      "activations/layer11_attention_weight_max": 31.790603637695312,
      "activations/layer11_attention_weight_min": -33.72996520996094,
      "activations/layer12_attention_weight_max": 17.300710678100586,
      "activations/layer12_attention_weight_min": -27.569406509399414,
      "activations/layer13_attention_weight_max": 41.96342849731445,
      "activations/layer13_attention_weight_min": -33.81880187988281,
      "activations/layer14_attention_weight_max": 34.428619384765625,
      "activations/layer14_attention_weight_min": -28.479568481445312,
      "activations/layer15_attention_weight_max": 30.995784759521484,
      "activations/layer15_attention_weight_min": -28.58221435546875,
      "activations/layer16_attention_weight_max": 31.444013595581055,
      "activations/layer16_attention_weight_min": -26.979639053344727,
      "activations/layer17_attention_weight_max": 48.824951171875,
      "activations/layer17_attention_weight_min": -40.68305969238281,
      "activations/layer18_attention_weight_max": 44.35469436645508,
      "activations/layer18_attention_weight_min": -34.293121337890625,
      "activations/layer19_attention_weight_max": 20.14729118347168,
      "activations/layer19_attention_weight_min": -21.441301345825195,
      "activations/layer1_attention_weight_max": 16.73279571533203,
      "activations/layer1_attention_weight_min": -16.980241775512695,
      "activations/layer20_attention_weight_max": 21.26738166809082,
      "activations/layer20_attention_weight_min": -22.58631706237793,
      "activations/layer21_attention_weight_max": 35.6072998046875,
      "activations/layer21_attention_weight_min": -23.40375328063965,
      "activations/layer22_attention_weight_max": 30.921751022338867,
      "activations/layer22_attention_weight_min": -25.564403533935547,
      "activations/layer23_attention_weight_max": 35.727291107177734,
      "activations/layer23_attention_weight_min": -23.83339500427246,
      "activations/layer2_attention_weight_max": 30.061573028564453,
      "activations/layer2_attention_weight_min": -27.797245025634766,
      "activations/layer3_attention_weight_max": 83.80120086669922,
      "activations/layer3_attention_weight_min": -86.142578125,
      "activations/layer4_attention_weight_max": 98.5656509399414,
      "activations/layer4_attention_weight_min": -91.5357666015625,
      "activations/layer5_attention_weight_max": 76.29923248291016,
      "activations/layer5_attention_weight_min": -76.16110229492188,
      "activations/layer6_attention_weight_max": 54.24161911010742,
      "activations/layer6_attention_weight_min": -52.765830993652344,
      "activations/layer7_attention_weight_max": 77.20068359375,
      "activations/layer7_attention_weight_min": -75.7344970703125,
      "activations/layer8_attention_weight_max": 46.558441162109375,
      "activations/layer8_attention_weight_min": -49.22163391113281,
      "activations/layer9_attention_weight_max": 50.11260986328125,
      "activations/layer9_attention_weight_min": -48.04071044921875,
      "epoch": 8.66,
      "learning_rate": 9.512613636363636e-05,
      "loss": 2.7879,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.4942,
      "eval_samples_per_second": 505.521,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.4942,
      "eval_openwebtext_samples_per_second": 505.521,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 1.9821,
      "eval_wikitext_samples_per_second": 230.063,
      "step": 149000
    },
    {
      "epoch": 8.66,
      "eval_lambada_loss": 2.693359375,
      "eval_lambada_ppl": 14.781248363830914,
      "eval_lambada_runtime": 9.5784,
      "eval_lambada_samples_per_second": 508.332,
      "step": 149000
    },
    {
      "activations/layer0_attention_weight_max": 15.078170776367188,
      "activations/layer0_attention_weight_min": -13.989498138427734,
      "activations/layer10_attention_weight_max": 39.13885498046875,
      "activations/layer10_attention_weight_min": -35.517486572265625,
      "activations/layer11_attention_weight_max": 31.96185874938965,
      "activations/layer11_attention_weight_min": -31.786590576171875,
      "activations/layer12_attention_weight_max": 21.423540115356445,
      "activations/layer12_attention_weight_min": -23.187618255615234,
      "activations/layer13_attention_weight_max": 44.43760681152344,
      "activations/layer13_attention_weight_min": -31.21240234375,
      "activations/layer14_attention_weight_max": 41.358943939208984,
      "activations/layer14_attention_weight_min": -30.683069229125977,
      "activations/layer15_attention_weight_max": 41.987606048583984,
      "activations/layer15_attention_weight_min": -30.433826446533203,
      "activations/layer16_attention_weight_max": 37.08181381225586,
      "activations/layer16_attention_weight_min": -29.63285255432129,
      "activations/layer17_attention_weight_max": 64.0128173828125,
      "activations/layer17_attention_weight_min": -48.5869026184082,
      "activations/layer18_attention_weight_max": 57.11503601074219,
      "activations/layer18_attention_weight_min": -38.96734619140625,
      "activations/layer19_attention_weight_max": 25.583105087280273,
      "activations/layer19_attention_weight_min": -21.356679916381836,
      "activations/layer1_attention_weight_max": 16.959379196166992,
      "activations/layer1_attention_weight_min": -14.60039234161377,
      "activations/layer20_attention_weight_max": 23.103679656982422,
      "activations/layer20_attention_weight_min": -20.000526428222656,
      "activations/layer21_attention_weight_max": 41.910926818847656,
      "activations/layer21_attention_weight_min": -24.560178756713867,
      "activations/layer22_attention_weight_max": 40.0469856262207,
      "activations/layer22_attention_weight_min": -26.07010269165039,
      "activations/layer23_attention_weight_max": 35.7462272644043,
      "activations/layer23_attention_weight_min": -21.208160400390625,
      "activations/layer2_attention_weight_max": 29.46603775024414,
      "activations/layer2_attention_weight_min": -28.729263305664062,
      "activations/layer3_attention_weight_max": 87.77886199951172,
      "activations/layer3_attention_weight_min": -92.10124206542969,
      "activations/layer4_attention_weight_max": 104.1225357055664,
      "activations/layer4_attention_weight_min": -99.37494659423828,
      "activations/layer5_attention_weight_max": 85.30060577392578,
      "activations/layer5_attention_weight_min": -83.54093933105469,
      "activations/layer6_attention_weight_max": 57.82207489013672,
      "activations/layer6_attention_weight_min": -56.058265686035156,
      "activations/layer7_attention_weight_max": 75.44048309326172,
      "activations/layer7_attention_weight_min": -78.47348022460938,
      "activations/layer8_attention_weight_max": 45.62297058105469,
      "activations/layer8_attention_weight_min": -46.07761001586914,
      "activations/layer9_attention_weight_max": 52.551666259765625,
      "activations/layer9_attention_weight_min": -51.48383712768555,
      "epoch": 8.66,
      "learning_rate": 9.510719696969696e-05,
      "loss": 2.793,
      "step": 149050
    },
    {
      "activations/layer0_attention_weight_max": 14.819345474243164,
      "activations/layer0_attention_weight_min": -14.275869369506836,
      "activations/layer10_attention_weight_max": 35.509464263916016,
      "activations/layer10_attention_weight_min": -35.450279235839844,
      "activations/layer11_attention_weight_max": 32.13799285888672,
      "activations/layer11_attention_weight_min": -31.057464599609375,
      "activations/layer12_attention_weight_max": 20.094629287719727,
      "activations/layer12_attention_weight_min": -25.64232063293457,
      "activations/layer13_attention_weight_max": 41.31629180908203,
      "activations/layer13_attention_weight_min": -34.19275665283203,
      "activations/layer14_attention_weight_max": 38.29438400268555,
      "activations/layer14_attention_weight_min": -32.274295806884766,
      "activations/layer15_attention_weight_max": 35.92859649658203,
      "activations/layer15_attention_weight_min": -29.572002410888672,
      "activations/layer16_attention_weight_max": 32.95047378540039,
      "activations/layer16_attention_weight_min": -30.201126098632812,
      "activations/layer17_attention_weight_max": 51.457218170166016,
      "activations/layer17_attention_weight_min": -45.06013870239258,
      "activations/layer18_attention_weight_max": 46.400657653808594,
      "activations/layer18_attention_weight_min": -35.71548843383789,
      "activations/layer19_attention_weight_max": 20.46937370300293,
      "activations/layer19_attention_weight_min": -21.173879623413086,
      "activations/layer1_attention_weight_max": 17.072277069091797,
      "activations/layer1_attention_weight_min": -14.635074615478516,
      "activations/layer20_attention_weight_max": 22.172527313232422,
      "activations/layer20_attention_weight_min": -21.637187957763672,
      "activations/layer21_attention_weight_max": 42.83537292480469,
      "activations/layer21_attention_weight_min": -23.322887420654297,
      "activations/layer22_attention_weight_max": 30.60555076599121,
      "activations/layer22_attention_weight_min": -24.747652053833008,
      "activations/layer23_attention_weight_max": 35.264835357666016,
      "activations/layer23_attention_weight_min": -23.009292602539062,
      "activations/layer2_attention_weight_max": 30.55982208251953,
      "activations/layer2_attention_weight_min": -28.67919921875,
      "activations/layer3_attention_weight_max": 90.1458511352539,
      "activations/layer3_attention_weight_min": -93.40619659423828,
      "activations/layer4_attention_weight_max": 99.47101593017578,
      "activations/layer4_attention_weight_min": -94.5556411743164,
      "activations/layer5_attention_weight_max": 81.135009765625,
      "activations/layer5_attention_weight_min": -75.3833999633789,
      "activations/layer6_attention_weight_max": 53.421974182128906,
      "activations/layer6_attention_weight_min": -52.50107955932617,
      "activations/layer7_attention_weight_max": 79.86176300048828,
      "activations/layer7_attention_weight_min": -75.36872100830078,
      "activations/layer8_attention_weight_max": 45.68098068237305,
      "activations/layer8_attention_weight_min": -43.850975036621094,
      "activations/layer9_attention_weight_max": 56.90180206298828,
      "activations/layer9_attention_weight_min": -47.39244079589844,
      "epoch": 8.66,
      "learning_rate": 9.508825757575757e-05,
      "loss": 2.8117,
      "step": 149100
    },
    {
      "activations/layer0_attention_weight_max": 15.103890419006348,
      "activations/layer0_attention_weight_min": -14.314753532409668,
      "activations/layer10_attention_weight_max": 40.79426193237305,
      "activations/layer10_attention_weight_min": -40.23811721801758,
      "activations/layer11_attention_weight_max": 33.85490417480469,
      "activations/layer11_attention_weight_min": -32.325584411621094,
      "activations/layer12_attention_weight_max": 19.799983978271484,
      "activations/layer12_attention_weight_min": -27.20555877685547,
      "activations/layer13_attention_weight_max": 45.34733581542969,
      "activations/layer13_attention_weight_min": -32.05054473876953,
      "activations/layer14_attention_weight_max": 43.29225540161133,
      "activations/layer14_attention_weight_min": -31.622970581054688,
      "activations/layer15_attention_weight_max": 36.577552795410156,
      "activations/layer15_attention_weight_min": -30.139562606811523,
      "activations/layer16_attention_weight_max": 32.202144622802734,
      "activations/layer16_attention_weight_min": -29.193862915039062,
      "activations/layer17_attention_weight_max": 57.61565017700195,
      "activations/layer17_attention_weight_min": -45.51191711425781,
      "activations/layer18_attention_weight_max": 50.706573486328125,
      "activations/layer18_attention_weight_min": -37.650291442871094,
      "activations/layer19_attention_weight_max": 22.90059471130371,
      "activations/layer19_attention_weight_min": -20.90764808654785,
      "activations/layer1_attention_weight_max": 17.36519432067871,
      "activations/layer1_attention_weight_min": -15.948820114135742,
      "activations/layer20_attention_weight_max": 27.317319869995117,
      "activations/layer20_attention_weight_min": -22.190078735351562,
      "activations/layer21_attention_weight_max": 47.30153274536133,
      "activations/layer21_attention_weight_min": -24.078201293945312,
      "activations/layer22_attention_weight_max": 36.09366989135742,
      "activations/layer22_attention_weight_min": -26.135164260864258,
      "activations/layer23_attention_weight_max": 40.50938415527344,
      "activations/layer23_attention_weight_min": -25.11240005493164,
      "activations/layer2_attention_weight_max": 31.032705307006836,
      "activations/layer2_attention_weight_min": -28.894668579101562,
      "activations/layer3_attention_weight_max": 89.81513214111328,
      "activations/layer3_attention_weight_min": -92.79600524902344,
      "activations/layer4_attention_weight_max": 103.15289306640625,
      "activations/layer4_attention_weight_min": -97.31713104248047,
      "activations/layer5_attention_weight_max": 83.67323303222656,
      "activations/layer5_attention_weight_min": -79.43313598632812,
      "activations/layer6_attention_weight_max": 57.293670654296875,
      "activations/layer6_attention_weight_min": -55.558013916015625,
      "activations/layer7_attention_weight_max": 83.1820068359375,
      "activations/layer7_attention_weight_min": -81.51026916503906,
      "activations/layer8_attention_weight_max": 46.232566833496094,
      "activations/layer8_attention_weight_min": -52.10162353515625,
      "activations/layer9_attention_weight_max": 54.07721710205078,
      "activations/layer9_attention_weight_min": -55.64398956298828,
      "epoch": 8.67,
      "learning_rate": 9.506931818181817e-05,
      "loss": 2.8093,
      "step": 149150
    },
    {
      "activations/layer0_attention_weight_max": 15.479482650756836,
      "activations/layer0_attention_weight_min": -13.807551383972168,
      "activations/layer10_attention_weight_max": 35.67642593383789,
      "activations/layer10_attention_weight_min": -33.264041900634766,
      "activations/layer11_attention_weight_max": 31.597240447998047,
      "activations/layer11_attention_weight_min": -29.977188110351562,
      "activations/layer12_attention_weight_max": 17.840740203857422,
      "activations/layer12_attention_weight_min": -26.976465225219727,
      "activations/layer13_attention_weight_max": 44.273033142089844,
      "activations/layer13_attention_weight_min": -30.4344482421875,
      "activations/layer14_attention_weight_max": 47.43852615356445,
      "activations/layer14_attention_weight_min": -30.236562728881836,
      "activations/layer15_attention_weight_max": 42.885433197021484,
      "activations/layer15_attention_weight_min": -28.133363723754883,
      "activations/layer16_attention_weight_max": 37.25298309326172,
      "activations/layer16_attention_weight_min": -28.522356033325195,
      "activations/layer17_attention_weight_max": 58.01211929321289,
      "activations/layer17_attention_weight_min": -43.972076416015625,
      "activations/layer18_attention_weight_max": 53.05255889892578,
      "activations/layer18_attention_weight_min": -37.95808029174805,
      "activations/layer19_attention_weight_max": 28.70033836364746,
      "activations/layer19_attention_weight_min": -23.51205062866211,
      "activations/layer1_attention_weight_max": 16.7569637298584,
      "activations/layer1_attention_weight_min": -15.133413314819336,
      "activations/layer20_attention_weight_max": 29.99858283996582,
      "activations/layer20_attention_weight_min": -21.566068649291992,
      "activations/layer21_attention_weight_max": 54.904991149902344,
      "activations/layer21_attention_weight_min": -28.38848876953125,
      "activations/layer22_attention_weight_max": 41.14982986450195,
      "activations/layer22_attention_weight_min": -26.98887062072754,
      "activations/layer23_attention_weight_max": 42.4404182434082,
      "activations/layer23_attention_weight_min": -25.645185470581055,
      "activations/layer2_attention_weight_max": 33.17420196533203,
      "activations/layer2_attention_weight_min": -30.349544525146484,
      "activations/layer3_attention_weight_max": 92.80571746826172,
      "activations/layer3_attention_weight_min": -97.43623352050781,
      "activations/layer4_attention_weight_max": 102.31996154785156,
      "activations/layer4_attention_weight_min": -98.60311126708984,
      "activations/layer5_attention_weight_max": 77.8512954711914,
      "activations/layer5_attention_weight_min": -72.47700500488281,
      "activations/layer6_attention_weight_max": 53.29621124267578,
      "activations/layer6_attention_weight_min": -53.90399932861328,
      "activations/layer7_attention_weight_max": 76.51809692382812,
      "activations/layer7_attention_weight_min": -72.11579895019531,
      "activations/layer8_attention_weight_max": 43.92314529418945,
      "activations/layer8_attention_weight_min": -47.1826057434082,
      "activations/layer9_attention_weight_max": 48.15632629394531,
      "activations/layer9_attention_weight_min": -50.0080451965332,
      "epoch": 8.67,
      "learning_rate": 9.505037878787878e-05,
      "loss": 2.787,
      "step": 149200
    },
    {
      "activations/layer0_attention_weight_max": 15.239496231079102,
      "activations/layer0_attention_weight_min": -14.37667179107666,
      "activations/layer10_attention_weight_max": 35.41750717163086,
      "activations/layer10_attention_weight_min": -35.08794403076172,
      "activations/layer11_attention_weight_max": 31.08697509765625,
      "activations/layer11_attention_weight_min": -30.977582931518555,
      "activations/layer12_attention_weight_max": 29.86029624938965,
      "activations/layer12_attention_weight_min": -25.806867599487305,
      "activations/layer13_attention_weight_max": 41.2888069152832,
      "activations/layer13_attention_weight_min": -33.17179489135742,
      "activations/layer14_attention_weight_max": 40.653160095214844,
      "activations/layer14_attention_weight_min": -31.02916145324707,
      "activations/layer15_attention_weight_max": 36.28505325317383,
      "activations/layer15_attention_weight_min": -29.02570915222168,
      "activations/layer16_attention_weight_max": 37.22915267944336,
      "activations/layer16_attention_weight_min": -27.945772171020508,
      "activations/layer17_attention_weight_max": 54.79165267944336,
      "activations/layer17_attention_weight_min": -42.2481689453125,
      "activations/layer18_attention_weight_max": 46.45707321166992,
      "activations/layer18_attention_weight_min": -38.75365447998047,
      "activations/layer19_attention_weight_max": 21.236661911010742,
      "activations/layer19_attention_weight_min": -20.915363311767578,
      "activations/layer1_attention_weight_max": 17.302194595336914,
      "activations/layer1_attention_weight_min": -15.78885269165039,
      "activations/layer20_attention_weight_max": 21.274982452392578,
      "activations/layer20_attention_weight_min": -23.645999908447266,
      "activations/layer21_attention_weight_max": 38.07415008544922,
      "activations/layer21_attention_weight_min": -23.855514526367188,
      "activations/layer22_attention_weight_max": 32.1281852722168,
      "activations/layer22_attention_weight_min": -25.03121566772461,
      "activations/layer23_attention_weight_max": 35.364959716796875,
      "activations/layer23_attention_weight_min": -22.485687255859375,
      "activations/layer2_attention_weight_max": 30.795438766479492,
      "activations/layer2_attention_weight_min": -29.64374351501465,
      "activations/layer3_attention_weight_max": 86.50992584228516,
      "activations/layer3_attention_weight_min": -90.65779876708984,
      "activations/layer4_attention_weight_max": 98.06439208984375,
      "activations/layer4_attention_weight_min": -93.69276428222656,
      "activations/layer5_attention_weight_max": 76.56678771972656,
      "activations/layer5_attention_weight_min": -78.97467803955078,
      "activations/layer6_attention_weight_max": 53.36174774169922,
      "activations/layer6_attention_weight_min": -51.807228088378906,
      "activations/layer7_attention_weight_max": 74.08551788330078,
      "activations/layer7_attention_weight_min": -70.79484558105469,
      "activations/layer8_attention_weight_max": 43.10034942626953,
      "activations/layer8_attention_weight_min": -44.346221923828125,
      "activations/layer9_attention_weight_max": 49.101844787597656,
      "activations/layer9_attention_weight_min": -48.773292541503906,
      "epoch": 8.67,
      "learning_rate": 9.503143939393938e-05,
      "loss": 2.7954,
      "step": 149250
    },
    {
      "activations/layer0_attention_weight_max": 14.608039855957031,
      "activations/layer0_attention_weight_min": -14.568397521972656,
      "activations/layer10_attention_weight_max": 38.32463836669922,
      "activations/layer10_attention_weight_min": -36.35200881958008,
      "activations/layer11_attention_weight_max": 33.04779815673828,
      "activations/layer11_attention_weight_min": -33.120933532714844,
      "activations/layer12_attention_weight_max": 21.160539627075195,
      "activations/layer12_attention_weight_min": -25.14935302734375,
      "activations/layer13_attention_weight_max": 39.655548095703125,
      "activations/layer13_attention_weight_min": -28.369375228881836,
      "activations/layer14_attention_weight_max": 37.64606857299805,
      "activations/layer14_attention_weight_min": -30.419921875,
      "activations/layer15_attention_weight_max": 37.49949264526367,
      "activations/layer15_attention_weight_min": -28.28566551208496,
      "activations/layer16_attention_weight_max": 33.86538314819336,
      "activations/layer16_attention_weight_min": -29.419313430786133,
      "activations/layer17_attention_weight_max": 57.71059036254883,
      "activations/layer17_attention_weight_min": -42.85146713256836,
      "activations/layer18_attention_weight_max": 49.71171951293945,
      "activations/layer18_attention_weight_min": -36.94745635986328,
      "activations/layer19_attention_weight_max": 22.730989456176758,
      "activations/layer19_attention_weight_min": -20.834636688232422,
      "activations/layer1_attention_weight_max": 17.85105323791504,
      "activations/layer1_attention_weight_min": -15.766505241394043,
      "activations/layer20_attention_weight_max": 20.86117172241211,
      "activations/layer20_attention_weight_min": -24.055971145629883,
      "activations/layer21_attention_weight_max": 36.42109298706055,
      "activations/layer21_attention_weight_min": -22.02251434326172,
      "activations/layer22_attention_weight_max": 33.008140563964844,
      "activations/layer22_attention_weight_min": -27.496620178222656,
      "activations/layer23_attention_weight_max": 36.32250213623047,
      "activations/layer23_attention_weight_min": -22.98889923095703,
      "activations/layer2_attention_weight_max": 30.73011589050293,
      "activations/layer2_attention_weight_min": -29.57103729248047,
      "activations/layer3_attention_weight_max": 88.17745971679688,
      "activations/layer3_attention_weight_min": -89.5023193359375,
      "activations/layer4_attention_weight_max": 97.61273193359375,
      "activations/layer4_attention_weight_min": -93.70066833496094,
      "activations/layer5_attention_weight_max": 79.78199005126953,
      "activations/layer5_attention_weight_min": -78.00711059570312,
      "activations/layer6_attention_weight_max": 55.254791259765625,
      "activations/layer6_attention_weight_min": -53.490352630615234,
      "activations/layer7_attention_weight_max": 74.08412170410156,
      "activations/layer7_attention_weight_min": -74.76786041259766,
      "activations/layer8_attention_weight_max": 46.42403030395508,
      "activations/layer8_attention_weight_min": -45.53687286376953,
      "activations/layer9_attention_weight_max": 49.277915954589844,
      "activations/layer9_attention_weight_min": -49.77688980102539,
      "epoch": 8.68,
      "learning_rate": 9.501249999999999e-05,
      "loss": 2.7999,
      "step": 149300
    },
    {
      "activations/layer0_attention_weight_max": 15.153352737426758,
      "activations/layer0_attention_weight_min": -14.09877872467041,
      "activations/layer10_attention_weight_max": 39.446075439453125,
      "activations/layer10_attention_weight_min": -35.55720138549805,
      "activations/layer11_attention_weight_max": 31.99578857421875,
      "activations/layer11_attention_weight_min": -31.509984970092773,
      "activations/layer12_attention_weight_max": 24.649765014648438,
      "activations/layer12_attention_weight_min": -28.417381286621094,
      "activations/layer13_attention_weight_max": 43.21641159057617,
      "activations/layer13_attention_weight_min": -30.58428192138672,
      "activations/layer14_attention_weight_max": 41.380802154541016,
      "activations/layer14_attention_weight_min": -32.82606506347656,
      "activations/layer15_attention_weight_max": 36.053466796875,
      "activations/layer15_attention_weight_min": -30.433879852294922,
      "activations/layer16_attention_weight_max": 33.38341522216797,
      "activations/layer16_attention_weight_min": -29.19649887084961,
      "activations/layer17_attention_weight_max": 52.79424285888672,
      "activations/layer17_attention_weight_min": -45.15267562866211,
      "activations/layer18_attention_weight_max": 45.899234771728516,
      "activations/layer18_attention_weight_min": -37.10257339477539,
      "activations/layer19_attention_weight_max": 22.9151668548584,
      "activations/layer19_attention_weight_min": -21.251232147216797,
      "activations/layer1_attention_weight_max": 16.72096061706543,
      "activations/layer1_attention_weight_min": -14.775777816772461,
      "activations/layer20_attention_weight_max": 25.259010314941406,
      "activations/layer20_attention_weight_min": -23.510637283325195,
      "activations/layer21_attention_weight_max": 43.39043045043945,
      "activations/layer21_attention_weight_min": -23.49108123779297,
      "activations/layer22_attention_weight_max": 30.523714065551758,
      "activations/layer22_attention_weight_min": -26.857297897338867,
      "activations/layer23_attention_weight_max": 38.09423065185547,
      "activations/layer23_attention_weight_min": -23.570632934570312,
      "activations/layer2_attention_weight_max": 28.672748565673828,
      "activations/layer2_attention_weight_min": -30.288305282592773,
      "activations/layer3_attention_weight_max": 85.4250717163086,
      "activations/layer3_attention_weight_min": -89.68495178222656,
      "activations/layer4_attention_weight_max": 99.30350494384766,
      "activations/layer4_attention_weight_min": -91.9834213256836,
      "activations/layer5_attention_weight_max": 78.88253021240234,
      "activations/layer5_attention_weight_min": -76.33924102783203,
      "activations/layer6_attention_weight_max": 54.636775970458984,
      "activations/layer6_attention_weight_min": -50.42673873901367,
      "activations/layer7_attention_weight_max": 74.2868423461914,
      "activations/layer7_attention_weight_min": -70.88603210449219,
      "activations/layer8_attention_weight_max": 46.00395584106445,
      "activations/layer8_attention_weight_min": -44.3288459777832,
      "activations/layer9_attention_weight_max": 52.35213088989258,
      "activations/layer9_attention_weight_min": -47.35189437866211,
      "epoch": 8.68,
      "learning_rate": 9.49935606060606e-05,
      "loss": 2.8088,
      "step": 149350
    },
    {
      "activations/layer0_attention_weight_max": 15.129526138305664,
      "activations/layer0_attention_weight_min": -14.545859336853027,
      "activations/layer10_attention_weight_max": 42.700408935546875,
      "activations/layer10_attention_weight_min": -39.68817138671875,
      "activations/layer11_attention_weight_max": 35.845069885253906,
      "activations/layer11_attention_weight_min": -35.79185104370117,
      "activations/layer12_attention_weight_max": 20.074798583984375,
      "activations/layer12_attention_weight_min": -25.087753295898438,
      "activations/layer13_attention_weight_max": 40.94371032714844,
      "activations/layer13_attention_weight_min": -34.42662048339844,
      "activations/layer14_attention_weight_max": 43.75347900390625,
      "activations/layer14_attention_weight_min": -31.529922485351562,
      "activations/layer15_attention_weight_max": 37.8259391784668,
      "activations/layer15_attention_weight_min": -30.01337242126465,
      "activations/layer16_attention_weight_max": 32.400997161865234,
      "activations/layer16_attention_weight_min": -30.194042205810547,
      "activations/layer17_attention_weight_max": 57.44245147705078,
      "activations/layer17_attention_weight_min": -45.49812698364258,
      "activations/layer18_attention_weight_max": 47.20895004272461,
      "activations/layer18_attention_weight_min": -39.740055084228516,
      "activations/layer19_attention_weight_max": 21.59369468688965,
      "activations/layer19_attention_weight_min": -21.115327835083008,
      "activations/layer1_attention_weight_max": 16.765378952026367,
      "activations/layer1_attention_weight_min": -16.302021026611328,
      "activations/layer20_attention_weight_max": 23.00128936767578,
      "activations/layer20_attention_weight_min": -20.544410705566406,
      "activations/layer21_attention_weight_max": 39.99295425415039,
      "activations/layer21_attention_weight_min": -23.44870948791504,
      "activations/layer22_attention_weight_max": 33.30911636352539,
      "activations/layer22_attention_weight_min": -25.58389663696289,
      "activations/layer23_attention_weight_max": 37.01026916503906,
      "activations/layer23_attention_weight_min": -22.125595092773438,
      "activations/layer2_attention_weight_max": 31.349149703979492,
      "activations/layer2_attention_weight_min": -28.394832611083984,
      "activations/layer3_attention_weight_max": 86.7024154663086,
      "activations/layer3_attention_weight_min": -91.884765625,
      "activations/layer4_attention_weight_max": 104.4225082397461,
      "activations/layer4_attention_weight_min": -100.04906463623047,
      "activations/layer5_attention_weight_max": 83.98196411132812,
      "activations/layer5_attention_weight_min": -79.45443725585938,
      "activations/layer6_attention_weight_max": 57.97483825683594,
      "activations/layer6_attention_weight_min": -54.312381744384766,
      "activations/layer7_attention_weight_max": 82.01797485351562,
      "activations/layer7_attention_weight_min": -74.54203796386719,
      "activations/layer8_attention_weight_max": 49.684326171875,
      "activations/layer8_attention_weight_min": -50.947940826416016,
      "activations/layer9_attention_weight_max": 54.368892669677734,
      "activations/layer9_attention_weight_min": -51.29889678955078,
      "epoch": 8.68,
      "learning_rate": 9.497462121212122e-05,
      "loss": 2.798,
      "step": 149400
    },
    {
      "activations/layer0_attention_weight_max": 15.937339782714844,
      "activations/layer0_attention_weight_min": -14.556556701660156,
      "activations/layer10_attention_weight_max": 38.432044982910156,
      "activations/layer10_attention_weight_min": -37.38813018798828,
      "activations/layer11_attention_weight_max": 34.74382781982422,
      "activations/layer11_attention_weight_min": -33.24098205566406,
      "activations/layer12_attention_weight_max": 20.530044555664062,
      "activations/layer12_attention_weight_min": -27.45867156982422,
      "activations/layer13_attention_weight_max": 52.35149383544922,
      "activations/layer13_attention_weight_min": -32.45948028564453,
      "activations/layer14_attention_weight_max": 37.73318099975586,
      "activations/layer14_attention_weight_min": -30.596647262573242,
      "activations/layer15_attention_weight_max": 43.05616760253906,
      "activations/layer15_attention_weight_min": -28.79309844970703,
      "activations/layer16_attention_weight_max": 29.812089920043945,
      "activations/layer16_attention_weight_min": -28.679466247558594,
      "activations/layer17_attention_weight_max": 57.57185745239258,
      "activations/layer17_attention_weight_min": -44.77431106567383,
      "activations/layer18_attention_weight_max": 47.84538269042969,
      "activations/layer18_attention_weight_min": -38.3859748840332,
      "activations/layer19_attention_weight_max": 22.781705856323242,
      "activations/layer19_attention_weight_min": -20.084142684936523,
      "activations/layer1_attention_weight_max": 16.95469093322754,
      "activations/layer1_attention_weight_min": -14.79985237121582,
      "activations/layer20_attention_weight_max": 21.51051139831543,
      "activations/layer20_attention_weight_min": -21.311016082763672,
      "activations/layer21_attention_weight_max": 39.597923278808594,
      "activations/layer21_attention_weight_min": -22.39058494567871,
      "activations/layer22_attention_weight_max": 32.232391357421875,
      "activations/layer22_attention_weight_min": -24.509891510009766,
      "activations/layer23_attention_weight_max": 33.72430419921875,
      "activations/layer23_attention_weight_min": -25.09410858154297,
      "activations/layer2_attention_weight_max": 33.82673645019531,
      "activations/layer2_attention_weight_min": -30.379074096679688,
      "activations/layer3_attention_weight_max": 96.4156265258789,
      "activations/layer3_attention_weight_min": -97.09576416015625,
      "activations/layer4_attention_weight_max": 107.9549560546875,
      "activations/layer4_attention_weight_min": -99.13672637939453,
      "activations/layer5_attention_weight_max": 87.76838684082031,
      "activations/layer5_attention_weight_min": -86.59773254394531,
      "activations/layer6_attention_weight_max": 58.25836181640625,
      "activations/layer6_attention_weight_min": -56.128700256347656,
      "activations/layer7_attention_weight_max": 79.51846313476562,
      "activations/layer7_attention_weight_min": -80.43689727783203,
      "activations/layer8_attention_weight_max": 47.52389907836914,
      "activations/layer8_attention_weight_min": -50.660545349121094,
      "activations/layer9_attention_weight_max": 52.07349395751953,
      "activations/layer9_attention_weight_min": -51.78413009643555,
      "epoch": 8.68,
      "learning_rate": 9.49556818181818e-05,
      "loss": 2.8058,
      "step": 149450
    },
    {
      "activations/layer0_attention_weight_max": 15.47114086151123,
      "activations/layer0_attention_weight_min": -13.94760799407959,
      "activations/layer10_attention_weight_max": 38.9149169921875,
      "activations/layer10_attention_weight_min": -37.9800910949707,
      "activations/layer11_attention_weight_max": 36.18434143066406,
      "activations/layer11_attention_weight_min": -33.04998779296875,
      "activations/layer12_attention_weight_max": 21.98838233947754,
      "activations/layer12_attention_weight_min": -27.919387817382812,
      "activations/layer13_attention_weight_max": 48.880672454833984,
      "activations/layer13_attention_weight_min": -32.05638885498047,
      "activations/layer14_attention_weight_max": 52.84407043457031,
      "activations/layer14_attention_weight_min": -32.1032600402832,
      "activations/layer15_attention_weight_max": 40.12090301513672,
      "activations/layer15_attention_weight_min": -29.313520431518555,
      "activations/layer16_attention_weight_max": 29.028995513916016,
      "activations/layer16_attention_weight_min": -27.69276237487793,
      "activations/layer17_attention_weight_max": 53.77666091918945,
      "activations/layer17_attention_weight_min": -43.083274841308594,
      "activations/layer18_attention_weight_max": 46.5679931640625,
      "activations/layer18_attention_weight_min": -37.67921447753906,
      "activations/layer19_attention_weight_max": 21.035696029663086,
      "activations/layer19_attention_weight_min": -21.045011520385742,
      "activations/layer1_attention_weight_max": 17.34329605102539,
      "activations/layer1_attention_weight_min": -13.605908393859863,
      "activations/layer20_attention_weight_max": 22.56865692138672,
      "activations/layer20_attention_weight_min": -21.191408157348633,
      "activations/layer21_attention_weight_max": 40.263790130615234,
      "activations/layer21_attention_weight_min": -23.04392433166504,
      "activations/layer22_attention_weight_max": 30.186256408691406,
      "activations/layer22_attention_weight_min": -24.571001052856445,
      "activations/layer23_attention_weight_max": 36.31952667236328,
      "activations/layer23_attention_weight_min": -24.808948516845703,
      "activations/layer2_attention_weight_max": 31.757047653198242,
      "activations/layer2_attention_weight_min": -29.97957992553711,
      "activations/layer3_attention_weight_max": 97.0118179321289,
      "activations/layer3_attention_weight_min": -95.05332946777344,
      "activations/layer4_attention_weight_max": 106.5392074584961,
      "activations/layer4_attention_weight_min": -100.2835922241211,
      "activations/layer5_attention_weight_max": 81.92448425292969,
      "activations/layer5_attention_weight_min": -75.68792724609375,
      "activations/layer6_attention_weight_max": 57.655548095703125,
      "activations/layer6_attention_weight_min": -52.73128890991211,
      "activations/layer7_attention_weight_max": 78.33416748046875,
      "activations/layer7_attention_weight_min": -71.89989471435547,
      "activations/layer8_attention_weight_max": 46.008323669433594,
      "activations/layer8_attention_weight_min": -44.75615692138672,
      "activations/layer9_attention_weight_max": 56.97263717651367,
      "activations/layer9_attention_weight_min": -50.30680847167969,
      "epoch": 8.69,
      "learning_rate": 9.493674242424242e-05,
      "loss": 2.7957,
      "step": 149500
    },
    {
      "activations/layer0_attention_weight_max": 14.842538833618164,
      "activations/layer0_attention_weight_min": -14.065768241882324,
      "activations/layer10_attention_weight_max": 41.93175506591797,
      "activations/layer10_attention_weight_min": -35.854835510253906,
      "activations/layer11_attention_weight_max": 34.70200729370117,
      "activations/layer11_attention_weight_min": -33.669898986816406,
      "activations/layer12_attention_weight_max": 21.251815795898438,
      "activations/layer12_attention_weight_min": -25.07413673400879,
      "activations/layer13_attention_weight_max": 48.83613586425781,
      "activations/layer13_attention_weight_min": -32.91459274291992,
      "activations/layer14_attention_weight_max": 43.18547439575195,
      "activations/layer14_attention_weight_min": -30.814420700073242,
      "activations/layer15_attention_weight_max": 40.30386734008789,
      "activations/layer15_attention_weight_min": -30.367809295654297,
      "activations/layer16_attention_weight_max": 35.402748107910156,
      "activations/layer16_attention_weight_min": -28.88249969482422,
      "activations/layer17_attention_weight_max": 56.646358489990234,
      "activations/layer17_attention_weight_min": -42.23136520385742,
      "activations/layer18_attention_weight_max": 49.86301803588867,
      "activations/layer18_attention_weight_min": -36.89653015136719,
      "activations/layer19_attention_weight_max": 21.531984329223633,
      "activations/layer19_attention_weight_min": -22.66451072692871,
      "activations/layer1_attention_weight_max": 16.596101760864258,
      "activations/layer1_attention_weight_min": -16.26242446899414,
      "activations/layer20_attention_weight_max": 21.287797927856445,
      "activations/layer20_attention_weight_min": -20.362939834594727,
      "activations/layer21_attention_weight_max": 37.6749267578125,
      "activations/layer21_attention_weight_min": -22.883737564086914,
      "activations/layer22_attention_weight_max": 32.09180450439453,
      "activations/layer22_attention_weight_min": -25.940929412841797,
      "activations/layer23_attention_weight_max": 41.425254821777344,
      "activations/layer23_attention_weight_min": -22.35798454284668,
      "activations/layer2_attention_weight_max": 32.172462463378906,
      "activations/layer2_attention_weight_min": -30.578718185424805,
      "activations/layer3_attention_weight_max": 94.26385498046875,
      "activations/layer3_attention_weight_min": -90.69596099853516,
      "activations/layer4_attention_weight_max": 100.91438293457031,
      "activations/layer4_attention_weight_min": -97.3001937866211,
      "activations/layer5_attention_weight_max": 82.2474365234375,
      "activations/layer5_attention_weight_min": -77.1685791015625,
      "activations/layer6_attention_weight_max": 57.18680191040039,
      "activations/layer6_attention_weight_min": -53.78343963623047,
      "activations/layer7_attention_weight_max": 77.63858795166016,
      "activations/layer7_attention_weight_min": -73.46129608154297,
      "activations/layer8_attention_weight_max": 48.73503494262695,
      "activations/layer8_attention_weight_min": -47.35066604614258,
      "activations/layer9_attention_weight_max": 50.48270034790039,
      "activations/layer9_attention_weight_min": -48.56254196166992,
      "epoch": 8.69,
      "learning_rate": 9.491780303030303e-05,
      "loss": 2.8044,
      "step": 149550
    },
    {
      "activations/layer0_attention_weight_max": 15.598442077636719,
      "activations/layer0_attention_weight_min": -13.963150978088379,
      "activations/layer10_attention_weight_max": 48.69984817504883,
      "activations/layer10_attention_weight_min": -44.296390533447266,
      "activations/layer11_attention_weight_max": 42.136741638183594,
      "activations/layer11_attention_weight_min": -36.326576232910156,
      "activations/layer12_attention_weight_max": 20.448699951171875,
      "activations/layer12_attention_weight_min": -24.260623931884766,
      "activations/layer13_attention_weight_max": 48.53690719604492,
      "activations/layer13_attention_weight_min": -30.15069580078125,
      "activations/layer14_attention_weight_max": 52.332847595214844,
      "activations/layer14_attention_weight_min": -31.53769874572754,
      "activations/layer15_attention_weight_max": 49.91238784790039,
      "activations/layer15_attention_weight_min": -29.803159713745117,
      "activations/layer16_attention_weight_max": 31.780269622802734,
      "activations/layer16_attention_weight_min": -28.048946380615234,
      "activations/layer17_attention_weight_max": 55.85243606567383,
      "activations/layer17_attention_weight_min": -46.46226119995117,
      "activations/layer18_attention_weight_max": 46.05218505859375,
      "activations/layer18_attention_weight_min": -37.70380401611328,
      "activations/layer19_attention_weight_max": 21.716909408569336,
      "activations/layer19_attention_weight_min": -22.98402214050293,
      "activations/layer1_attention_weight_max": 17.110004425048828,
      "activations/layer1_attention_weight_min": -14.123536109924316,
      "activations/layer20_attention_weight_max": 21.55107307434082,
      "activations/layer20_attention_weight_min": -20.466087341308594,
      "activations/layer21_attention_weight_max": 41.95882797241211,
      "activations/layer21_attention_weight_min": -23.573833465576172,
      "activations/layer22_attention_weight_max": 30.815595626831055,
      "activations/layer22_attention_weight_min": -26.05730438232422,
      "activations/layer23_attention_weight_max": 34.98748779296875,
      "activations/layer23_attention_weight_min": -19.979557037353516,
      "activations/layer2_attention_weight_max": 31.607654571533203,
      "activations/layer2_attention_weight_min": -30.04865264892578,
      "activations/layer3_attention_weight_max": 89.7341537475586,
      "activations/layer3_attention_weight_min": -87.37833404541016,
      "activations/layer4_attention_weight_max": 100.9181137084961,
      "activations/layer4_attention_weight_min": -95.56489562988281,
      "activations/layer5_attention_weight_max": 86.93673706054688,
      "activations/layer5_attention_weight_min": -77.23361206054688,
      "activations/layer6_attention_weight_max": 58.72306442260742,
      "activations/layer6_attention_weight_min": -53.862972259521484,
      "activations/layer7_attention_weight_max": 91.85556030273438,
      "activations/layer7_attention_weight_min": -77.86466217041016,
      "activations/layer8_attention_weight_max": 59.84433364868164,
      "activations/layer8_attention_weight_min": -53.93368911743164,
      "activations/layer9_attention_weight_max": 77.69895935058594,
      "activations/layer9_attention_weight_min": -54.0056037902832,
      "epoch": 8.69,
      "learning_rate": 9.489886363636362e-05,
      "loss": 2.7931,
      "step": 149600
    },
    {
      "activations/layer0_attention_weight_max": 14.742534637451172,
      "activations/layer0_attention_weight_min": -13.762057304382324,
      "activations/layer10_attention_weight_max": 36.10308837890625,
      "activations/layer10_attention_weight_min": -35.01172637939453,
      "activations/layer11_attention_weight_max": 31.400226593017578,
      "activations/layer11_attention_weight_min": -29.868377685546875,
      "activations/layer12_attention_weight_max": 16.9766845703125,
      "activations/layer12_attention_weight_min": -25.325660705566406,
      "activations/layer13_attention_weight_max": 41.868621826171875,
      "activations/layer13_attention_weight_min": -31.27837371826172,
      "activations/layer14_attention_weight_max": 35.500972747802734,
      "activations/layer14_attention_weight_min": -31.309911727905273,
      "activations/layer15_attention_weight_max": 32.98771667480469,
      "activations/layer15_attention_weight_min": -33.74422073364258,
      "activations/layer16_attention_weight_max": 30.058094024658203,
      "activations/layer16_attention_weight_min": -29.56667137145996,
      "activations/layer17_attention_weight_max": 49.16400909423828,
      "activations/layer17_attention_weight_min": -42.870052337646484,
      "activations/layer18_attention_weight_max": 43.39043045043945,
      "activations/layer18_attention_weight_min": -37.80937957763672,
      "activations/layer19_attention_weight_max": 21.104772567749023,
      "activations/layer19_attention_weight_min": -19.211254119873047,
      "activations/layer1_attention_weight_max": 16.385971069335938,
      "activations/layer1_attention_weight_min": -15.559778213500977,
      "activations/layer20_attention_weight_max": 20.073991775512695,
      "activations/layer20_attention_weight_min": -21.095300674438477,
      "activations/layer21_attention_weight_max": 37.55495071411133,
      "activations/layer21_attention_weight_min": -22.668703079223633,
      "activations/layer22_attention_weight_max": 31.34284782409668,
      "activations/layer22_attention_weight_min": -23.69112205505371,
      "activations/layer23_attention_weight_max": 35.05289077758789,
      "activations/layer23_attention_weight_min": -23.737943649291992,
      "activations/layer2_attention_weight_max": 30.695295333862305,
      "activations/layer2_attention_weight_min": -29.572629928588867,
      "activations/layer3_attention_weight_max": 89.359619140625,
      "activations/layer3_attention_weight_min": -89.04129028320312,
      "activations/layer4_attention_weight_max": 102.03385162353516,
      "activations/layer4_attention_weight_min": -97.1963882446289,
      "activations/layer5_attention_weight_max": 80.8910140991211,
      "activations/layer5_attention_weight_min": -74.95286560058594,
      "activations/layer6_attention_weight_max": 55.88698196411133,
      "activations/layer6_attention_weight_min": -52.568851470947266,
      "activations/layer7_attention_weight_max": 75.95484161376953,
      "activations/layer7_attention_weight_min": -74.19180297851562,
      "activations/layer8_attention_weight_max": 45.755977630615234,
      "activations/layer8_attention_weight_min": -46.64112854003906,
      "activations/layer9_attention_weight_max": 52.3340950012207,
      "activations/layer9_attention_weight_min": -46.046634674072266,
      "epoch": 8.7,
      "learning_rate": 9.487992424242424e-05,
      "loss": 2.7969,
      "step": 149650
    },
    {
      "activations/layer0_attention_weight_max": 16.32265853881836,
      "activations/layer0_attention_weight_min": -14.202981948852539,
      "activations/layer10_attention_weight_max": 37.26605224609375,
      "activations/layer10_attention_weight_min": -36.458831787109375,
      "activations/layer11_attention_weight_max": 33.60359191894531,
      "activations/layer11_attention_weight_min": -33.59449005126953,
      "activations/layer12_attention_weight_max": 20.930973052978516,
      "activations/layer12_attention_weight_min": -25.031919479370117,
      "activations/layer13_attention_weight_max": 45.280799865722656,
      "activations/layer13_attention_weight_min": -30.672136306762695,
      "activations/layer14_attention_weight_max": 43.08648681640625,
      "activations/layer14_attention_weight_min": -32.7690544128418,
      "activations/layer15_attention_weight_max": 37.15044021606445,
      "activations/layer15_attention_weight_min": -30.555864334106445,
      "activations/layer16_attention_weight_max": 34.915470123291016,
      "activations/layer16_attention_weight_min": -30.14235496520996,
      "activations/layer17_attention_weight_max": 61.45308303833008,
      "activations/layer17_attention_weight_min": -46.0462760925293,
      "activations/layer18_attention_weight_max": 50.59891128540039,
      "activations/layer18_attention_weight_min": -37.78364181518555,
      "activations/layer19_attention_weight_max": 22.978757858276367,
      "activations/layer19_attention_weight_min": -21.407520294189453,
      "activations/layer1_attention_weight_max": 17.68284034729004,
      "activations/layer1_attention_weight_min": -17.918909072875977,
      "activations/layer20_attention_weight_max": 23.711942672729492,
      "activations/layer20_attention_weight_min": -20.129384994506836,
      "activations/layer21_attention_weight_max": 45.466888427734375,
      "activations/layer21_attention_weight_min": -21.880949020385742,
      "activations/layer22_attention_weight_max": 33.89878463745117,
      "activations/layer22_attention_weight_min": -24.314857482910156,
      "activations/layer23_attention_weight_max": 40.36735534667969,
      "activations/layer23_attention_weight_min": -20.94635009765625,
      "activations/layer2_attention_weight_max": 31.936969757080078,
      "activations/layer2_attention_weight_min": -29.82982635498047,
      "activations/layer3_attention_weight_max": 86.31439208984375,
      "activations/layer3_attention_weight_min": -91.7215805053711,
      "activations/layer4_attention_weight_max": 99.599609375,
      "activations/layer4_attention_weight_min": -95.15383911132812,
      "activations/layer5_attention_weight_max": 81.79483795166016,
      "activations/layer5_attention_weight_min": -79.09036254882812,
      "activations/layer6_attention_weight_max": 57.735450744628906,
      "activations/layer6_attention_weight_min": -53.9946174621582,
      "activations/layer7_attention_weight_max": 83.21261596679688,
      "activations/layer7_attention_weight_min": -77.50257873535156,
      "activations/layer8_attention_weight_max": 46.83205795288086,
      "activations/layer8_attention_weight_min": -46.241416931152344,
      "activations/layer9_attention_weight_max": 61.50751876831055,
      "activations/layer9_attention_weight_min": -48.00863265991211,
      "epoch": 8.7,
      "learning_rate": 9.486098484848484e-05,
      "loss": 2.8035,
      "step": 149700
    },
    {
      "activations/layer0_attention_weight_max": 14.760208129882812,
      "activations/layer0_attention_weight_min": -13.92277717590332,
      "activations/layer10_attention_weight_max": 37.75775146484375,
      "activations/layer10_attention_weight_min": -36.15922927856445,
      "activations/layer11_attention_weight_max": 32.64055252075195,
      "activations/layer11_attention_weight_min": -33.049034118652344,
      "activations/layer12_attention_weight_max": 30.647951126098633,
      "activations/layer12_attention_weight_min": -23.422636032104492,
      "activations/layer13_attention_weight_max": 42.95014953613281,
      "activations/layer13_attention_weight_min": -31.48261260986328,
      "activations/layer14_attention_weight_max": 39.94414520263672,
      "activations/layer14_attention_weight_min": -30.6026611328125,
      "activations/layer15_attention_weight_max": 36.797119140625,
      "activations/layer15_attention_weight_min": -29.084177017211914,
      "activations/layer16_attention_weight_max": 33.99570846557617,
      "activations/layer16_attention_weight_min": -29.053802490234375,
      "activations/layer17_attention_weight_max": 55.66764831542969,
      "activations/layer17_attention_weight_min": -44.065494537353516,
      "activations/layer18_attention_weight_max": 52.614601135253906,
      "activations/layer18_attention_weight_min": -38.497039794921875,
      "activations/layer19_attention_weight_max": 23.88302993774414,
      "activations/layer19_attention_weight_min": -20.732099533081055,
      "activations/layer1_attention_weight_max": 16.71544647216797,
      "activations/layer1_attention_weight_min": -16.101179122924805,
      "activations/layer20_attention_weight_max": 22.912504196166992,
      "activations/layer20_attention_weight_min": -21.18684196472168,
      "activations/layer21_attention_weight_max": 43.686492919921875,
      "activations/layer21_attention_weight_min": -24.911691665649414,
      "activations/layer22_attention_weight_max": 32.920265197753906,
      "activations/layer22_attention_weight_min": -26.19618797302246,
      "activations/layer23_attention_weight_max": 42.08174133300781,
      "activations/layer23_attention_weight_min": -20.941112518310547,
      "activations/layer2_attention_weight_max": 33.750823974609375,
      "activations/layer2_attention_weight_min": -30.782331466674805,
      "activations/layer3_attention_weight_max": 94.0396957397461,
      "activations/layer3_attention_weight_min": -92.8270492553711,
      "activations/layer4_attention_weight_max": 104.9803695678711,
      "activations/layer4_attention_weight_min": -95.64478302001953,
      "activations/layer5_attention_weight_max": 82.45405578613281,
      "activations/layer5_attention_weight_min": -78.0313720703125,
      "activations/layer6_attention_weight_max": 54.69823455810547,
      "activations/layer6_attention_weight_min": -56.366111755371094,
      "activations/layer7_attention_weight_max": 77.48247528076172,
      "activations/layer7_attention_weight_min": -75.4541244506836,
      "activations/layer8_attention_weight_max": 44.31511306762695,
      "activations/layer8_attention_weight_min": -44.87477493286133,
      "activations/layer9_attention_weight_max": 47.77274703979492,
      "activations/layer9_attention_weight_min": -52.425758361816406,
      "epoch": 8.7,
      "learning_rate": 9.484204545454544e-05,
      "loss": 2.7993,
      "step": 149750
    },
    {
      "activations/layer0_attention_weight_max": 16.31218147277832,
      "activations/layer0_attention_weight_min": -14.227723121643066,
      "activations/layer10_attention_weight_max": 46.42864990234375,
      "activations/layer10_attention_weight_min": -40.263885498046875,
      "activations/layer11_attention_weight_max": 37.342018127441406,
      "activations/layer11_attention_weight_min": -32.819522857666016,
      "activations/layer12_attention_weight_max": 17.538381576538086,
      "activations/layer12_attention_weight_min": -27.27192497253418,
      "activations/layer13_attention_weight_max": 62.20268249511719,
      "activations/layer13_attention_weight_min": -36.74981689453125,
      "activations/layer14_attention_weight_max": 72.62979888916016,
      "activations/layer14_attention_weight_min": -32.802978515625,
      "activations/layer15_attention_weight_max": 65.25312042236328,
      "activations/layer15_attention_weight_min": -28.125139236450195,
      "activations/layer16_attention_weight_max": 51.126277923583984,
      "activations/layer16_attention_weight_min": -27.26508140563965,
      "activations/layer17_attention_weight_max": 66.05621337890625,
      "activations/layer17_attention_weight_min": -45.9328498840332,
      "activations/layer18_attention_weight_max": 53.28626251220703,
      "activations/layer18_attention_weight_min": -39.52911376953125,
      "activations/layer19_attention_weight_max": 22.299116134643555,
      "activations/layer19_attention_weight_min": -22.692686080932617,
      "activations/layer1_attention_weight_max": 17.61351203918457,
      "activations/layer1_attention_weight_min": -15.578094482421875,
      "activations/layer20_attention_weight_max": 25.83620834350586,
      "activations/layer20_attention_weight_min": -23.63023567199707,
      "activations/layer21_attention_weight_max": 63.434566497802734,
      "activations/layer21_attention_weight_min": -26.434110641479492,
      "activations/layer22_attention_weight_max": 32.84084701538086,
      "activations/layer22_attention_weight_min": -23.81848907470703,
      "activations/layer23_attention_weight_max": 39.88420104980469,
      "activations/layer23_attention_weight_min": -26.398853302001953,
      "activations/layer2_attention_weight_max": 33.26961898803711,
      "activations/layer2_attention_weight_min": -29.905048370361328,
      "activations/layer3_attention_weight_max": 89.98497009277344,
      "activations/layer3_attention_weight_min": -94.07476806640625,
      "activations/layer4_attention_weight_max": 104.17866516113281,
      "activations/layer4_attention_weight_min": -99.79399871826172,
      "activations/layer5_attention_weight_max": 84.15839385986328,
      "activations/layer5_attention_weight_min": -75.97286224365234,
      "activations/layer6_attention_weight_max": 54.05788803100586,
      "activations/layer6_attention_weight_min": -51.12909698486328,
      "activations/layer7_attention_weight_max": 84.95050811767578,
      "activations/layer7_attention_weight_min": -74.6369400024414,
      "activations/layer8_attention_weight_max": 53.4235725402832,
      "activations/layer8_attention_weight_min": -49.076507568359375,
      "activations/layer9_attention_weight_max": 64.33855438232422,
      "activations/layer9_attention_weight_min": -53.17861557006836,
      "epoch": 8.7,
      "learning_rate": 9.482310606060605e-05,
      "loss": 2.8064,
      "step": 149800
    },
    {
      "activations/layer0_attention_weight_max": 15.588363647460938,
      "activations/layer0_attention_weight_min": -14.050814628601074,
      "activations/layer10_attention_weight_max": 34.8995361328125,
      "activations/layer10_attention_weight_min": -34.57992172241211,
      "activations/layer11_attention_weight_max": 31.982975006103516,
      "activations/layer11_attention_weight_min": -31.509199142456055,
      "activations/layer12_attention_weight_max": 23.416683197021484,
      "activations/layer12_attention_weight_min": -25.776065826416016,
      "activations/layer13_attention_weight_max": 37.28922653198242,
      "activations/layer13_attention_weight_min": -28.190853118896484,
      "activations/layer14_attention_weight_max": 35.60423278808594,
      "activations/layer14_attention_weight_min": -31.206470489501953,
      "activations/layer15_attention_weight_max": 31.951778411865234,
      "activations/layer15_attention_weight_min": -29.655202865600586,
      "activations/layer16_attention_weight_max": 28.00470733642578,
      "activations/layer16_attention_weight_min": -28.001354217529297,
      "activations/layer17_attention_weight_max": 49.16408157348633,
      "activations/layer17_attention_weight_min": -40.9337272644043,
      "activations/layer18_attention_weight_max": 42.44496154785156,
      "activations/layer18_attention_weight_min": -36.1966667175293,
      "activations/layer19_attention_weight_max": 20.727800369262695,
      "activations/layer19_attention_weight_min": -19.928133010864258,
      "activations/layer1_attention_weight_max": 16.535430908203125,
      "activations/layer1_attention_weight_min": -15.139455795288086,
      "activations/layer20_attention_weight_max": 20.18842315673828,
      "activations/layer20_attention_weight_min": -22.720230102539062,
      "activations/layer21_attention_weight_max": 32.78694534301758,
      "activations/layer21_attention_weight_min": -23.07722282409668,
      "activations/layer22_attention_weight_max": 29.698564529418945,
      "activations/layer22_attention_weight_min": -25.819320678710938,
      "activations/layer23_attention_weight_max": 33.71675109863281,
      "activations/layer23_attention_weight_min": -23.264488220214844,
      "activations/layer2_attention_weight_max": 29.821426391601562,
      "activations/layer2_attention_weight_min": -28.98436737060547,
      "activations/layer3_attention_weight_max": 91.08668518066406,
      "activations/layer3_attention_weight_min": -90.52851104736328,
      "activations/layer4_attention_weight_max": 101.1992416381836,
      "activations/layer4_attention_weight_min": -95.3451156616211,
      "activations/layer5_attention_weight_max": 79.74562072753906,
      "activations/layer5_attention_weight_min": -80.85545349121094,
      "activations/layer6_attention_weight_max": 55.07921600341797,
      "activations/layer6_attention_weight_min": -53.06450271606445,
      "activations/layer7_attention_weight_max": 78.73789978027344,
      "activations/layer7_attention_weight_min": -72.0792465209961,
      "activations/layer8_attention_weight_max": 44.97433853149414,
      "activations/layer8_attention_weight_min": -46.231048583984375,
      "activations/layer9_attention_weight_max": 49.01118469238281,
      "activations/layer9_attention_weight_min": -48.1043815612793,
      "epoch": 8.71,
      "learning_rate": 9.480416666666666e-05,
      "loss": 2.8183,
      "step": 149850
    },
    {
      "activations/layer0_attention_weight_max": 14.638470649719238,
      "activations/layer0_attention_weight_min": -13.811517715454102,
      "activations/layer10_attention_weight_max": 36.525203704833984,
      "activations/layer10_attention_weight_min": -37.700592041015625,
      "activations/layer11_attention_weight_max": 31.621274948120117,
      "activations/layer11_attention_weight_min": -33.58020782470703,
      "activations/layer12_attention_weight_max": 21.092803955078125,
      "activations/layer12_attention_weight_min": -27.292137145996094,
      "activations/layer13_attention_weight_max": 37.83156967163086,
      "activations/layer13_attention_weight_min": -29.017078399658203,
      "activations/layer14_attention_weight_max": 37.76411437988281,
      "activations/layer14_attention_weight_min": -32.55815887451172,
      "activations/layer15_attention_weight_max": 35.780174255371094,
      "activations/layer15_attention_weight_min": -29.7509708404541,
      "activations/layer16_attention_weight_max": 33.13190841674805,
      "activations/layer16_attention_weight_min": -29.365556716918945,
      "activations/layer17_attention_weight_max": 60.33537673950195,
      "activations/layer17_attention_weight_min": -46.0775146484375,
      "activations/layer18_attention_weight_max": 52.94955825805664,
      "activations/layer18_attention_weight_min": -39.34843826293945,
      "activations/layer19_attention_weight_max": 21.680994033813477,
      "activations/layer19_attention_weight_min": -22.975759506225586,
      "activations/layer1_attention_weight_max": 15.977341651916504,
      "activations/layer1_attention_weight_min": -14.073103904724121,
      "activations/layer20_attention_weight_max": 23.070819854736328,
      "activations/layer20_attention_weight_min": -21.318626403808594,
      "activations/layer21_attention_weight_max": 43.34126663208008,
      "activations/layer21_attention_weight_min": -26.138015747070312,
      "activations/layer22_attention_weight_max": 32.41055679321289,
      "activations/layer22_attention_weight_min": -26.713783264160156,
      "activations/layer23_attention_weight_max": 34.807125091552734,
      "activations/layer23_attention_weight_min": -24.536222457885742,
      "activations/layer2_attention_weight_max": 30.467777252197266,
      "activations/layer2_attention_weight_min": -28.777992248535156,
      "activations/layer3_attention_weight_max": 89.42223358154297,
      "activations/layer3_attention_weight_min": -84.4465103149414,
      "activations/layer4_attention_weight_max": 96.62922668457031,
      "activations/layer4_attention_weight_min": -89.91461181640625,
      "activations/layer5_attention_weight_max": 76.34844970703125,
      "activations/layer5_attention_weight_min": -73.31085968017578,
      "activations/layer6_attention_weight_max": 51.89033126831055,
      "activations/layer6_attention_weight_min": -52.204673767089844,
      "activations/layer7_attention_weight_max": 72.6806640625,
      "activations/layer7_attention_weight_min": -69.7954330444336,
      "activations/layer8_attention_weight_max": 43.42313003540039,
      "activations/layer8_attention_weight_min": -46.01734161376953,
      "activations/layer9_attention_weight_max": 49.523399353027344,
      "activations/layer9_attention_weight_min": -48.77061080932617,
      "epoch": 8.71,
      "learning_rate": 9.478522727272727e-05,
      "loss": 2.7994,
      "step": 149900
    },
    {
      "activations/layer0_attention_weight_max": 15.023201942443848,
      "activations/layer0_attention_weight_min": -14.116691589355469,
      "activations/layer10_attention_weight_max": 36.290771484375,
      "activations/layer10_attention_weight_min": -35.08400344848633,
      "activations/layer11_attention_weight_max": 29.851848602294922,
      "activations/layer11_attention_weight_min": -31.46774673461914,
      "activations/layer12_attention_weight_max": 21.106870651245117,
      "activations/layer12_attention_weight_min": -24.618412017822266,
      "activations/layer13_attention_weight_max": 45.38666915893555,
      "activations/layer13_attention_weight_min": -29.152339935302734,
      "activations/layer14_attention_weight_max": 46.72272491455078,
      "activations/layer14_attention_weight_min": -30.7467098236084,
      "activations/layer15_attention_weight_max": 37.08814239501953,
      "activations/layer15_attention_weight_min": -29.512447357177734,
      "activations/layer16_attention_weight_max": 33.02177047729492,
      "activations/layer16_attention_weight_min": -30.320934295654297,
      "activations/layer17_attention_weight_max": 55.79639434814453,
      "activations/layer17_attention_weight_min": -46.47930145263672,
      "activations/layer18_attention_weight_max": 50.55464553833008,
      "activations/layer18_attention_weight_min": -36.833839416503906,
      "activations/layer19_attention_weight_max": 22.67201042175293,
      "activations/layer19_attention_weight_min": -20.877134323120117,
      "activations/layer1_attention_weight_max": 15.98071575164795,
      "activations/layer1_attention_weight_min": -15.817109107971191,
      "activations/layer20_attention_weight_max": 23.0682315826416,
      "activations/layer20_attention_weight_min": -21.141006469726562,
      "activations/layer21_attention_weight_max": 47.228546142578125,
      "activations/layer21_attention_weight_min": -23.20635414123535,
      "activations/layer22_attention_weight_max": 33.85908508300781,
      "activations/layer22_attention_weight_min": -27.09756088256836,
      "activations/layer23_attention_weight_max": 37.73109436035156,
      "activations/layer23_attention_weight_min": -20.43492317199707,
      "activations/layer2_attention_weight_max": 32.888607025146484,
      "activations/layer2_attention_weight_min": -30.71845245361328,
      "activations/layer3_attention_weight_max": 89.90509033203125,
      "activations/layer3_attention_weight_min": -91.69081115722656,
      "activations/layer4_attention_weight_max": 98.78740692138672,
      "activations/layer4_attention_weight_min": -98.03203582763672,
      "activations/layer5_attention_weight_max": 78.75463104248047,
      "activations/layer5_attention_weight_min": -80.09284973144531,
      "activations/layer6_attention_weight_max": 54.76889419555664,
      "activations/layer6_attention_weight_min": -54.85520553588867,
      "activations/layer7_attention_weight_max": 75.50768280029297,
      "activations/layer7_attention_weight_min": -73.70232391357422,
      "activations/layer8_attention_weight_max": 45.514041900634766,
      "activations/layer8_attention_weight_min": -44.60503387451172,
      "activations/layer9_attention_weight_max": 50.356929779052734,
      "activations/layer9_attention_weight_min": -50.702884674072266,
      "epoch": 8.71,
      "learning_rate": 9.476628787878787e-05,
      "loss": 2.7896,
      "step": 149950
    },
    {
      "activations/layer0_attention_weight_max": 15.57520866394043,
      "activations/layer0_attention_weight_min": -13.809289932250977,
      "activations/layer10_attention_weight_max": 37.68422317504883,
      "activations/layer10_attention_weight_min": -35.54541778564453,
      "activations/layer11_attention_weight_max": 30.601848602294922,
      "activations/layer11_attention_weight_min": -30.266136169433594,
      "activations/layer12_attention_weight_max": 20.05977439880371,
      "activations/layer12_attention_weight_min": -23.028564453125,
      "activations/layer13_attention_weight_max": 48.9878044128418,
      "activations/layer13_attention_weight_min": -32.54658508300781,
      "activations/layer14_attention_weight_max": 38.586971282958984,
      "activations/layer14_attention_weight_min": -29.204206466674805,
      "activations/layer15_attention_weight_max": 36.14595413208008,
      "activations/layer15_attention_weight_min": -26.683523178100586,
      "activations/layer16_attention_weight_max": 31.63498878479004,
      "activations/layer16_attention_weight_min": -27.639909744262695,
      "activations/layer17_attention_weight_max": 53.88808822631836,
      "activations/layer17_attention_weight_min": -43.17451858520508,
      "activations/layer18_attention_weight_max": 47.15501403808594,
      "activations/layer18_attention_weight_min": -37.26752853393555,
      "activations/layer19_attention_weight_max": 23.16274070739746,
      "activations/layer19_attention_weight_min": -22.16089630126953,
      "activations/layer1_attention_weight_max": 17.336366653442383,
      "activations/layer1_attention_weight_min": -15.172183990478516,
      "activations/layer20_attention_weight_max": 21.762126922607422,
      "activations/layer20_attention_weight_min": -20.61420249938965,
      "activations/layer21_attention_weight_max": 37.696876525878906,
      "activations/layer21_attention_weight_min": -22.086767196655273,
      "activations/layer22_attention_weight_max": 32.60296630859375,
      "activations/layer22_attention_weight_min": -25.45687484741211,
      "activations/layer23_attention_weight_max": 34.933738708496094,
      "activations/layer23_attention_weight_min": -22.633487701416016,
      "activations/layer2_attention_weight_max": 32.86907196044922,
      "activations/layer2_attention_weight_min": -31.804473876953125,
      "activations/layer3_attention_weight_max": 87.77545928955078,
      "activations/layer3_attention_weight_min": -92.90770721435547,
      "activations/layer4_attention_weight_max": 99.5606460571289,
      "activations/layer4_attention_weight_min": -94.30628967285156,
      "activations/layer5_attention_weight_max": 80.07470703125,
      "activations/layer5_attention_weight_min": -78.820556640625,
      "activations/layer6_attention_weight_max": 53.589473724365234,
      "activations/layer6_attention_weight_min": -50.99871063232422,
      "activations/layer7_attention_weight_max": 73.93522644042969,
      "activations/layer7_attention_weight_min": -71.56006622314453,
      "activations/layer8_attention_weight_max": 45.39449691772461,
      "activations/layer8_attention_weight_min": -46.23543167114258,
      "activations/layer9_attention_weight_max": 49.580867767333984,
      "activations/layer9_attention_weight_min": -48.98399353027344,
      "epoch": 8.72,
      "learning_rate": 9.474734848484847e-05,
      "loss": 2.7986,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_loss": 2.755859375,
      "eval_runtime": 8.4798,
      "eval_samples_per_second": 506.379,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_openwebtext_loss": 2.755859375,
      "eval_openwebtext_ppl": 15.734556979186685,
      "eval_openwebtext_runtime": 8.4798,
      "eval_openwebtext_samples_per_second": 506.379,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_wikitext_loss": 2.994140625,
      "eval_wikitext_ppl": 19.96819234893607,
      "eval_wikitext_runtime": 2.0073,
      "eval_wikitext_samples_per_second": 227.173,
      "step": 150000
    },
    {
      "epoch": 8.72,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.5921,
      "eval_lambada_samples_per_second": 507.603,
      "step": 150000
    },
    {
      "activations/layer0_attention_weight_max": 15.36512279510498,
      "activations/layer0_attention_weight_min": -14.23259449005127,
      "activations/layer10_attention_weight_max": 35.29385757446289,
      "activations/layer10_attention_weight_min": -34.39580154418945,
      "activations/layer11_attention_weight_max": 32.07664489746094,
      "activations/layer11_attention_weight_min": -31.817813873291016,
      "activations/layer12_attention_weight_max": 20.294036865234375,
      "activations/layer12_attention_weight_min": -25.5830078125,
      "activations/layer13_attention_weight_max": 44.035396575927734,
      "activations/layer13_attention_weight_min": -31.180015563964844,
      "activations/layer14_attention_weight_max": 39.888404846191406,
      "activations/layer14_attention_weight_min": -31.791088104248047,
      "activations/layer15_attention_weight_max": 35.175235748291016,
      "activations/layer15_attention_weight_min": -29.282936096191406,
      "activations/layer16_attention_weight_max": 32.72218704223633,
      "activations/layer16_attention_weight_min": -29.07160758972168,
      "activations/layer17_attention_weight_max": 52.76361083984375,
      "activations/layer17_attention_weight_min": -45.9567985534668,
      "activations/layer18_attention_weight_max": 52.89107131958008,
      "activations/layer18_attention_weight_min": -37.21638870239258,
      "activations/layer19_attention_weight_max": 24.50413703918457,
      "activations/layer19_attention_weight_min": -21.033761978149414,
      "activations/layer1_attention_weight_max": 16.741313934326172,
      "activations/layer1_attention_weight_min": -14.28599739074707,
      "activations/layer20_attention_weight_max": 25.617881774902344,
      "activations/layer20_attention_weight_min": -23.81523895263672,
      "activations/layer21_attention_weight_max": 42.03726577758789,
      "activations/layer21_attention_weight_min": -22.21329689025879,
      "activations/layer22_attention_weight_max": 32.65536117553711,
      "activations/layer22_attention_weight_min": -25.82491683959961,
      "activations/layer23_attention_weight_max": 38.87126159667969,
      "activations/layer23_attention_weight_min": -22.721363067626953,
      "activations/layer2_attention_weight_max": 32.65129852294922,
      "activations/layer2_attention_weight_min": -32.36044692993164,
      "activations/layer3_attention_weight_max": 88.98353576660156,
      "activations/layer3_attention_weight_min": -96.88811492919922,
      "activations/layer4_attention_weight_max": 99.01354217529297,
      "activations/layer4_attention_weight_min": -99.00482177734375,
      "activations/layer5_attention_weight_max": 76.50192260742188,
      "activations/layer5_attention_weight_min": -74.8021240234375,
      "activations/layer6_attention_weight_max": 53.935340881347656,
      "activations/layer6_attention_weight_min": -53.6875114440918,
      "activations/layer7_attention_weight_max": 71.24882507324219,
      "activations/layer7_attention_weight_min": -73.25883483886719,
      "activations/layer8_attention_weight_max": 43.70670700073242,
      "activations/layer8_attention_weight_min": -48.31608963012695,
      "activations/layer9_attention_weight_max": 46.95374298095703,
      "activations/layer9_attention_weight_min": -47.59577560424805,
      "epoch": 8.72,
      "learning_rate": 9.472840909090909e-05,
      "loss": 2.8016,
      "step": 150050
    },
    {
      "activations/layer0_attention_weight_max": 16.093130111694336,
      "activations/layer0_attention_weight_min": -14.24086856842041,
      "activations/layer10_attention_weight_max": 42.14237594604492,
      "activations/layer10_attention_weight_min": -36.35600280761719,
      "activations/layer11_attention_weight_max": 34.67176818847656,
      "activations/layer11_attention_weight_min": -34.02450180053711,
      "activations/layer12_attention_weight_max": 24.0146484375,
      "activations/layer12_attention_weight_min": -26.775861740112305,
      "activations/layer13_attention_weight_max": 44.06502914428711,
      "activations/layer13_attention_weight_min": -31.878480911254883,
      "activations/layer14_attention_weight_max": 42.474979400634766,
      "activations/layer14_attention_weight_min": -31.55901336669922,
      "activations/layer15_attention_weight_max": 34.10530471801758,
      "activations/layer15_attention_weight_min": -31.829355239868164,
      "activations/layer16_attention_weight_max": 33.79059600830078,
      "activations/layer16_attention_weight_min": -28.756061553955078,
      "activations/layer17_attention_weight_max": 57.06087112426758,
      "activations/layer17_attention_weight_min": -44.38041687011719,
      "activations/layer18_attention_weight_max": 52.7681999206543,
      "activations/layer18_attention_weight_min": -38.920867919921875,
      "activations/layer19_attention_weight_max": 21.5740966796875,
      "activations/layer19_attention_weight_min": -22.32781410217285,
      "activations/layer1_attention_weight_max": 16.414783477783203,
      "activations/layer1_attention_weight_min": -15.683146476745605,
      "activations/layer20_attention_weight_max": 24.51769256591797,
      "activations/layer20_attention_weight_min": -23.064376831054688,
      "activations/layer21_attention_weight_max": 41.90782165527344,
      "activations/layer21_attention_weight_min": -24.954912185668945,
      "activations/layer22_attention_weight_max": 35.066123962402344,
      "activations/layer22_attention_weight_min": -25.44078826904297,
      "activations/layer23_attention_weight_max": 41.35120391845703,
      "activations/layer23_attention_weight_min": -25.571706771850586,
      "activations/layer2_attention_weight_max": 30.83930206298828,
      "activations/layer2_attention_weight_min": -28.966245651245117,
      "activations/layer3_attention_weight_max": 91.64866638183594,
      "activations/layer3_attention_weight_min": -92.41924285888672,
      "activations/layer4_attention_weight_max": 102.11605834960938,
      "activations/layer4_attention_weight_min": -98.04216766357422,
      "activations/layer5_attention_weight_max": 81.4151611328125,
      "activations/layer5_attention_weight_min": -78.59300231933594,
      "activations/layer6_attention_weight_max": 56.136356353759766,
      "activations/layer6_attention_weight_min": -54.422340393066406,
      "activations/layer7_attention_weight_max": 79.56842803955078,
      "activations/layer7_attention_weight_min": -76.18384552001953,
      "activations/layer8_attention_weight_max": 47.46049499511719,
      "activations/layer8_attention_weight_min": -45.833309173583984,
      "activations/layer9_attention_weight_max": 53.599605560302734,
      "activations/layer9_attention_weight_min": -49.43517303466797,
      "epoch": 8.72,
      "learning_rate": 9.470946969696968e-05,
      "loss": 2.8106,
      "step": 150100
    },
    {
      "activations/layer0_attention_weight_max": 15.046406745910645,
      "activations/layer0_attention_weight_min": -13.795672416687012,
      "activations/layer10_attention_weight_max": 39.123390197753906,
      "activations/layer10_attention_weight_min": -36.52177810668945,
      "activations/layer11_attention_weight_max": 34.167579650878906,
      "activations/layer11_attention_weight_min": -32.188880920410156,
      "activations/layer12_attention_weight_max": 33.73566818237305,
      "activations/layer12_attention_weight_min": -30.967164993286133,
      "activations/layer13_attention_weight_max": 47.5319709777832,
      "activations/layer13_attention_weight_min": -34.8604850769043,
      "activations/layer14_attention_weight_max": 55.626773834228516,
      "activations/layer14_attention_weight_min": -34.426509857177734,
      "activations/layer15_attention_weight_max": 42.85787582397461,
      "activations/layer15_attention_weight_min": -29.043777465820312,
      "activations/layer16_attention_weight_max": 33.9483642578125,
      "activations/layer16_attention_weight_min": -27.483943939208984,
      "activations/layer17_attention_weight_max": 55.03923034667969,
      "activations/layer17_attention_weight_min": -44.30760192871094,
      "activations/layer18_attention_weight_max": 48.15163803100586,
      "activations/layer18_attention_weight_min": -37.9380989074707,
      "activations/layer19_attention_weight_max": 22.667455673217773,
      "activations/layer19_attention_weight_min": -19.59969139099121,
      "activations/layer1_attention_weight_max": 17.840717315673828,
      "activations/layer1_attention_weight_min": -14.834739685058594,
      "activations/layer20_attention_weight_max": 22.63895606994629,
      "activations/layer20_attention_weight_min": -22.586423873901367,
      "activations/layer21_attention_weight_max": 35.71612548828125,
      "activations/layer21_attention_weight_min": -20.613811492919922,
      "activations/layer22_attention_weight_max": 34.78565216064453,
      "activations/layer22_attention_weight_min": -23.747295379638672,
      "activations/layer23_attention_weight_max": 39.644283294677734,
      "activations/layer23_attention_weight_min": -21.715322494506836,
      "activations/layer2_attention_weight_max": 33.48219299316406,
      "activations/layer2_attention_weight_min": -31.483013153076172,
      "activations/layer3_attention_weight_max": 97.11865997314453,
      "activations/layer3_attention_weight_min": -97.34053039550781,
      "activations/layer4_attention_weight_max": 108.83219146728516,
      "activations/layer4_attention_weight_min": -100.70822143554688,
      "activations/layer5_attention_weight_max": 88.41513061523438,
      "activations/layer5_attention_weight_min": -78.60880279541016,
      "activations/layer6_attention_weight_max": 59.19429397583008,
      "activations/layer6_attention_weight_min": -56.57807159423828,
      "activations/layer7_attention_weight_max": 77.96212768554688,
      "activations/layer7_attention_weight_min": -79.4461669921875,
      "activations/layer8_attention_weight_max": 46.587825775146484,
      "activations/layer8_attention_weight_min": -48.11246871948242,
      "activations/layer9_attention_weight_max": 51.1483268737793,
      "activations/layer9_attention_weight_min": -50.49491500854492,
      "epoch": 8.72,
      "learning_rate": 9.469053030303029e-05,
      "loss": 2.823,
      "step": 150150
    },
    {
      "activations/layer0_attention_weight_max": 15.99207878112793,
      "activations/layer0_attention_weight_min": -14.261207580566406,
      "activations/layer10_attention_weight_max": 35.114898681640625,
      "activations/layer10_attention_weight_min": -35.12932205200195,
      "activations/layer11_attention_weight_max": 29.45345115661621,
      "activations/layer11_attention_weight_min": -31.555463790893555,
      "activations/layer12_attention_weight_max": 17.977245330810547,
      "activations/layer12_attention_weight_min": -27.82575225830078,
      "activations/layer13_attention_weight_max": 37.82160568237305,
      "activations/layer13_attention_weight_min": -30.219905853271484,
      "activations/layer14_attention_weight_max": 37.06511688232422,
      "activations/layer14_attention_weight_min": -30.142030715942383,
      "activations/layer15_attention_weight_max": 33.191829681396484,
      "activations/layer15_attention_weight_min": -30.655738830566406,
      "activations/layer16_attention_weight_max": 34.8314094543457,
      "activations/layer16_attention_weight_min": -27.711429595947266,
      "activations/layer17_attention_weight_max": 52.296714782714844,
      "activations/layer17_attention_weight_min": -44.178749084472656,
      "activations/layer18_attention_weight_max": 49.86723709106445,
      "activations/layer18_attention_weight_min": -37.670135498046875,
      "activations/layer19_attention_weight_max": 22.942276000976562,
      "activations/layer19_attention_weight_min": -21.7198429107666,
      "activations/layer1_attention_weight_max": 16.514890670776367,
      "activations/layer1_attention_weight_min": -16.063993453979492,
      "activations/layer20_attention_weight_max": 21.693401336669922,
      "activations/layer20_attention_weight_min": -20.683269500732422,
      "activations/layer21_attention_weight_max": 40.06462478637695,
      "activations/layer21_attention_weight_min": -22.412622451782227,
      "activations/layer22_attention_weight_max": 32.470821380615234,
      "activations/layer22_attention_weight_min": -25.317489624023438,
      "activations/layer23_attention_weight_max": 36.041534423828125,
      "activations/layer23_attention_weight_min": -22.303028106689453,
      "activations/layer2_attention_weight_max": 29.71968650817871,
      "activations/layer2_attention_weight_min": -31.81075668334961,
      "activations/layer3_attention_weight_max": 87.559326171875,
      "activations/layer3_attention_weight_min": -94.84618377685547,
      "activations/layer4_attention_weight_max": 102.5242691040039,
      "activations/layer4_attention_weight_min": -103.50560760498047,
      "activations/layer5_attention_weight_max": 79.94747924804688,
      "activations/layer5_attention_weight_min": -83.87773132324219,
      "activations/layer6_attention_weight_max": 56.0018310546875,
      "activations/layer6_attention_weight_min": -55.50030517578125,
      "activations/layer7_attention_weight_max": 76.74928283691406,
      "activations/layer7_attention_weight_min": -73.76683807373047,
      "activations/layer8_attention_weight_max": 42.30542755126953,
      "activations/layer8_attention_weight_min": -44.19377899169922,
      "activations/layer9_attention_weight_max": 47.488555908203125,
      "activations/layer9_attention_weight_min": -48.369964599609375,
      "epoch": 8.73,
      "learning_rate": 9.467159090909091e-05,
      "loss": 2.7891,
      "step": 150200
    },
    {
      "activations/layer0_attention_weight_max": 15.295856475830078,
      "activations/layer0_attention_weight_min": -13.704577445983887,
      "activations/layer10_attention_weight_max": 35.06847381591797,
      "activations/layer10_attention_weight_min": -34.387596130371094,
      "activations/layer11_attention_weight_max": 29.748546600341797,
      "activations/layer11_attention_weight_min": -28.894916534423828,
      "activations/layer12_attention_weight_max": 25.079086303710938,
      "activations/layer12_attention_weight_min": -27.460769653320312,
      "activations/layer13_attention_weight_max": 39.075172424316406,
      "activations/layer13_attention_weight_min": -29.44426155090332,
      "activations/layer14_attention_weight_max": 36.183494567871094,
      "activations/layer14_attention_weight_min": -29.947935104370117,
      "activations/layer15_attention_weight_max": 37.07410430908203,
      "activations/layer15_attention_weight_min": -28.690706253051758,
      "activations/layer16_attention_weight_max": 32.6290283203125,
      "activations/layer16_attention_weight_min": -28.68828010559082,
      "activations/layer17_attention_weight_max": 53.68550491333008,
      "activations/layer17_attention_weight_min": -43.69846725463867,
      "activations/layer18_attention_weight_max": 46.642242431640625,
      "activations/layer18_attention_weight_min": -36.5710563659668,
      "activations/layer19_attention_weight_max": 25.60999870300293,
      "activations/layer19_attention_weight_min": -21.6492862701416,
      "activations/layer1_attention_weight_max": 17.117769241333008,
      "activations/layer1_attention_weight_min": -14.894475936889648,
      "activations/layer20_attention_weight_max": 24.0803279876709,
      "activations/layer20_attention_weight_min": -21.60178565979004,
      "activations/layer21_attention_weight_max": 34.51685333251953,
      "activations/layer21_attention_weight_min": -23.9146728515625,
      "activations/layer22_attention_weight_max": 35.756492614746094,
      "activations/layer22_attention_weight_min": -27.73411750793457,
      "activations/layer23_attention_weight_max": 38.50233459472656,
      "activations/layer23_attention_weight_min": -22.9559326171875,
      "activations/layer2_attention_weight_max": 32.23573684692383,
      "activations/layer2_attention_weight_min": -31.291967391967773,
      "activations/layer3_attention_weight_max": 86.62625885009766,
      "activations/layer3_attention_weight_min": -93.80824279785156,
      "activations/layer4_attention_weight_max": 95.80956268310547,
      "activations/layer4_attention_weight_min": -92.41845703125,
      "activations/layer5_attention_weight_max": 75.65542602539062,
      "activations/layer5_attention_weight_min": -77.47539520263672,
      "activations/layer6_attention_weight_max": 54.09434127807617,
      "activations/layer6_attention_weight_min": -49.97024917602539,
      "activations/layer7_attention_weight_max": 78.68412780761719,
      "activations/layer7_attention_weight_min": -70.66641998291016,
      "activations/layer8_attention_weight_max": 43.000003814697266,
      "activations/layer8_attention_weight_min": -44.15842819213867,
      "activations/layer9_attention_weight_max": 50.6663932800293,
      "activations/layer9_attention_weight_min": -46.550071716308594,
      "epoch": 8.73,
      "learning_rate": 9.46526515151515e-05,
      "loss": 2.8002,
      "step": 150250
    },
    {
      "activations/layer0_attention_weight_max": 15.371956825256348,
      "activations/layer0_attention_weight_min": -13.726189613342285,
      "activations/layer10_attention_weight_max": 37.00379943847656,
      "activations/layer10_attention_weight_min": -35.551212310791016,
      "activations/layer11_attention_weight_max": 34.94937515258789,
      "activations/layer11_attention_weight_min": -32.36082458496094,
      "activations/layer12_attention_weight_max": 25.677162170410156,
      "activations/layer12_attention_weight_min": -24.210067749023438,
      "activations/layer13_attention_weight_max": 42.62970733642578,
      "activations/layer13_attention_weight_min": -32.849769592285156,
      "activations/layer14_attention_weight_max": 38.06112289428711,
      "activations/layer14_attention_weight_min": -31.12790298461914,
      "activations/layer15_attention_weight_max": 34.132530212402344,
      "activations/layer15_attention_weight_min": -28.877017974853516,
      "activations/layer16_attention_weight_max": 37.38632583618164,
      "activations/layer16_attention_weight_min": -26.740724563598633,
      "activations/layer17_attention_weight_max": 54.61605453491211,
      "activations/layer17_attention_weight_min": -42.51411819458008,
      "activations/layer18_attention_weight_max": 46.68346405029297,
      "activations/layer18_attention_weight_min": -35.439674377441406,
      "activations/layer19_attention_weight_max": 21.182485580444336,
      "activations/layer19_attention_weight_min": -20.315391540527344,
      "activations/layer1_attention_weight_max": 17.130651473999023,
      "activations/layer1_attention_weight_min": -15.84461498260498,
      "activations/layer20_attention_weight_max": 21.34859848022461,
      "activations/layer20_attention_weight_min": -22.870882034301758,
      "activations/layer21_attention_weight_max": 43.41859817504883,
      "activations/layer21_attention_weight_min": -25.8280086517334,
      "activations/layer22_attention_weight_max": 32.79224395751953,
      "activations/layer22_attention_weight_min": -25.50646209716797,
      "activations/layer23_attention_weight_max": 37.8210563659668,
      "activations/layer23_attention_weight_min": -20.93846893310547,
      "activations/layer2_attention_weight_max": 31.24257469177246,
      "activations/layer2_attention_weight_min": -30.780616760253906,
      "activations/layer3_attention_weight_max": 84.2353744506836,
      "activations/layer3_attention_weight_min": -92.35191345214844,
      "activations/layer4_attention_weight_max": 96.87317657470703,
      "activations/layer4_attention_weight_min": -90.8558578491211,
      "activations/layer5_attention_weight_max": 79.78020477294922,
      "activations/layer5_attention_weight_min": -75.848876953125,
      "activations/layer6_attention_weight_max": 54.151615142822266,
      "activations/layer6_attention_weight_min": -54.931785583496094,
      "activations/layer7_attention_weight_max": 72.4028549194336,
      "activations/layer7_attention_weight_min": -73.49564361572266,
      "activations/layer8_attention_weight_max": 45.626102447509766,
      "activations/layer8_attention_weight_min": -50.61708450317383,
      "activations/layer9_attention_weight_max": 53.63424301147461,
      "activations/layer9_attention_weight_min": -48.390621185302734,
      "epoch": 8.73,
      "learning_rate": 9.463371212121211e-05,
      "loss": 2.8045,
      "step": 150300
    },
    {
      "activations/layer0_attention_weight_max": 15.439621925354004,
      "activations/layer0_attention_weight_min": -13.815059661865234,
      "activations/layer10_attention_weight_max": 39.06476974487305,
      "activations/layer10_attention_weight_min": -37.246795654296875,
      "activations/layer11_attention_weight_max": 32.87506103515625,
      "activations/layer11_attention_weight_min": -31.793994903564453,
      "activations/layer12_attention_weight_max": 19.963918685913086,
      "activations/layer12_attention_weight_min": -27.364337921142578,
      "activations/layer13_attention_weight_max": 48.04696273803711,
      "activations/layer13_attention_weight_min": -33.71355438232422,
      "activations/layer14_attention_weight_max": 40.261043548583984,
      "activations/layer14_attention_weight_min": -30.794713973999023,
      "activations/layer15_attention_weight_max": 40.37380599975586,
      "activations/layer15_attention_weight_min": -30.32575035095215,
      "activations/layer16_attention_weight_max": 35.31975555419922,
      "activations/layer16_attention_weight_min": -28.202960968017578,
      "activations/layer17_attention_weight_max": 55.4761962890625,
      "activations/layer17_attention_weight_min": -45.341957092285156,
      "activations/layer18_attention_weight_max": 51.17032241821289,
      "activations/layer18_attention_weight_min": -39.649993896484375,
      "activations/layer19_attention_weight_max": 26.88064956665039,
      "activations/layer19_attention_weight_min": -21.523767471313477,
      "activations/layer1_attention_weight_max": 15.892335891723633,
      "activations/layer1_attention_weight_min": -15.395818710327148,
      "activations/layer20_attention_weight_max": 24.774852752685547,
      "activations/layer20_attention_weight_min": -20.756105422973633,
      "activations/layer21_attention_weight_max": 40.571319580078125,
      "activations/layer21_attention_weight_min": -22.274810791015625,
      "activations/layer22_attention_weight_max": 34.46337127685547,
      "activations/layer22_attention_weight_min": -25.41307830810547,
      "activations/layer23_attention_weight_max": 40.32423400878906,
      "activations/layer23_attention_weight_min": -23.825117111206055,
      "activations/layer2_attention_weight_max": 30.366958618164062,
      "activations/layer2_attention_weight_min": -28.93305778503418,
      "activations/layer3_attention_weight_max": 88.4351806640625,
      "activations/layer3_attention_weight_min": -89.78219604492188,
      "activations/layer4_attention_weight_max": 99.42028045654297,
      "activations/layer4_attention_weight_min": -99.20096588134766,
      "activations/layer5_attention_weight_max": 81.95466613769531,
      "activations/layer5_attention_weight_min": -84.2852783203125,
      "activations/layer6_attention_weight_max": 57.009769439697266,
      "activations/layer6_attention_weight_min": -54.890174865722656,
      "activations/layer7_attention_weight_max": 77.17607879638672,
      "activations/layer7_attention_weight_min": -76.6415786743164,
      "activations/layer8_attention_weight_max": 45.00442123413086,
      "activations/layer8_attention_weight_min": -47.363677978515625,
      "activations/layer9_attention_weight_max": 53.35007858276367,
      "activations/layer9_attention_weight_min": -54.312721252441406,
      "epoch": 8.74,
      "learning_rate": 9.461477272727272e-05,
      "loss": 2.8038,
      "step": 150350
    },
    {
      "activations/layer0_attention_weight_max": 15.163801193237305,
      "activations/layer0_attention_weight_min": -13.232120513916016,
      "activations/layer10_attention_weight_max": 36.97496795654297,
      "activations/layer10_attention_weight_min": -33.83724594116211,
      "activations/layer11_attention_weight_max": 29.502256393432617,
      "activations/layer11_attention_weight_min": -29.779396057128906,
      "activations/layer12_attention_weight_max": 19.290430068969727,
      "activations/layer12_attention_weight_min": -26.026939392089844,
      "activations/layer13_attention_weight_max": 39.8110237121582,
      "activations/layer13_attention_weight_min": -30.488502502441406,
      "activations/layer14_attention_weight_max": 35.69853210449219,
      "activations/layer14_attention_weight_min": -30.01771354675293,
      "activations/layer15_attention_weight_max": 33.22811508178711,
      "activations/layer15_attention_weight_min": -29.376590728759766,
      "activations/layer16_attention_weight_max": 29.898353576660156,
      "activations/layer16_attention_weight_min": -27.327083587646484,
      "activations/layer17_attention_weight_max": 52.29350662231445,
      "activations/layer17_attention_weight_min": -42.63657760620117,
      "activations/layer18_attention_weight_max": 47.45951843261719,
      "activations/layer18_attention_weight_min": -36.3133430480957,
      "activations/layer19_attention_weight_max": 22.405431747436523,
      "activations/layer19_attention_weight_min": -19.34287452697754,
      "activations/layer1_attention_weight_max": 16.0222110748291,
      "activations/layer1_attention_weight_min": -13.899531364440918,
      "activations/layer20_attention_weight_max": 24.235782623291016,
      "activations/layer20_attention_weight_min": -21.28563117980957,
      "activations/layer21_attention_weight_max": 37.275611877441406,
      "activations/layer21_attention_weight_min": -23.640371322631836,
      "activations/layer22_attention_weight_max": 30.667661666870117,
      "activations/layer22_attention_weight_min": -26.40160369873047,
      "activations/layer23_attention_weight_max": 37.2763786315918,
      "activations/layer23_attention_weight_min": -23.36353302001953,
      "activations/layer2_attention_weight_max": 32.900115966796875,
      "activations/layer2_attention_weight_min": -29.695117950439453,
      "activations/layer3_attention_weight_max": 88.18656921386719,
      "activations/layer3_attention_weight_min": -85.25198364257812,
      "activations/layer4_attention_weight_max": 95.7770004272461,
      "activations/layer4_attention_weight_min": -89.85094451904297,
      "activations/layer5_attention_weight_max": 77.3459701538086,
      "activations/layer5_attention_weight_min": -78.74578094482422,
      "activations/layer6_attention_weight_max": 53.333072662353516,
      "activations/layer6_attention_weight_min": -51.82883071899414,
      "activations/layer7_attention_weight_max": 74.41584014892578,
      "activations/layer7_attention_weight_min": -74.22720336914062,
      "activations/layer8_attention_weight_max": 43.8188362121582,
      "activations/layer8_attention_weight_min": -44.774253845214844,
      "activations/layer9_attention_weight_max": 50.39332580566406,
      "activations/layer9_attention_weight_min": -49.06846618652344,
      "epoch": 8.74,
      "learning_rate": 9.459621212121212e-05,
      "loss": 2.8039,
      "step": 150400
    },
    {
      "activations/layer0_attention_weight_max": 14.860259056091309,
      "activations/layer0_attention_weight_min": -13.333460807800293,
      "activations/layer10_attention_weight_max": 34.98396301269531,
      "activations/layer10_attention_weight_min": -34.7225341796875,
      "activations/layer11_attention_weight_max": 29.061641693115234,
      "activations/layer11_attention_weight_min": -30.639249801635742,
      "activations/layer12_attention_weight_max": 18.1524600982666,
      "activations/layer12_attention_weight_min": -28.244388580322266,
      "activations/layer13_attention_weight_max": 43.453712463378906,
      "activations/layer13_attention_weight_min": -29.539779663085938,
      "activations/layer14_attention_weight_max": 54.86033630371094,
      "activations/layer14_attention_weight_min": -33.50983428955078,
      "activations/layer15_attention_weight_max": 37.02407455444336,
      "activations/layer15_attention_weight_min": -32.65058898925781,
      "activations/layer16_attention_weight_max": 34.85263442993164,
      "activations/layer16_attention_weight_min": -29.677692413330078,
      "activations/layer17_attention_weight_max": 55.826194763183594,
      "activations/layer17_attention_weight_min": -43.57327651977539,
      "activations/layer18_attention_weight_max": 47.95930862426758,
      "activations/layer18_attention_weight_min": -36.59645462036133,
      "activations/layer19_attention_weight_max": 24.32740020751953,
      "activations/layer19_attention_weight_min": -21.404890060424805,
      "activations/layer1_attention_weight_max": 16.356836318969727,
      "activations/layer1_attention_weight_min": -15.1106538772583,
      "activations/layer20_attention_weight_max": 21.94270133972168,
      "activations/layer20_attention_weight_min": -21.690073013305664,
      "activations/layer21_attention_weight_max": 37.7928352355957,
      "activations/layer21_attention_weight_min": -23.649747848510742,
      "activations/layer22_attention_weight_max": 33.24100875854492,
      "activations/layer22_attention_weight_min": -24.625825881958008,
      "activations/layer23_attention_weight_max": 37.139041900634766,
      "activations/layer23_attention_weight_min": -21.343435287475586,
      "activations/layer2_attention_weight_max": 31.306297302246094,
      "activations/layer2_attention_weight_min": -29.724327087402344,
      "activations/layer3_attention_weight_max": 88.97244262695312,
      "activations/layer3_attention_weight_min": -89.5103530883789,
      "activations/layer4_attention_weight_max": 101.0277328491211,
      "activations/layer4_attention_weight_min": -96.83446502685547,
      "activations/layer5_attention_weight_max": 82.87932586669922,
      "activations/layer5_attention_weight_min": -78.22958374023438,
      "activations/layer6_attention_weight_max": 55.25081253051758,
      "activations/layer6_attention_weight_min": -53.81294250488281,
      "activations/layer7_attention_weight_max": 78.41517639160156,
      "activations/layer7_attention_weight_min": -71.70608520507812,
      "activations/layer8_attention_weight_max": 44.03528594970703,
      "activations/layer8_attention_weight_min": -46.85725784301758,
      "activations/layer9_attention_weight_max": 46.23808670043945,
      "activations/layer9_attention_weight_min": -47.87679672241211,
      "epoch": 8.74,
      "learning_rate": 9.457727272727272e-05,
      "loss": 2.8112,
      "step": 150450
    },
    {
      "activations/layer0_attention_weight_max": 15.192279815673828,
      "activations/layer0_attention_weight_min": -13.32346248626709,
      "activations/layer10_attention_weight_max": 37.20859146118164,
      "activations/layer10_attention_weight_min": -38.3221321105957,
      "activations/layer11_attention_weight_max": 30.217687606811523,
      "activations/layer11_attention_weight_min": -33.66237258911133,
      "activations/layer12_attention_weight_max": 22.944095611572266,
      "activations/layer12_attention_weight_min": -23.848548889160156,
      "activations/layer13_attention_weight_max": 42.5212287902832,
      "activations/layer13_attention_weight_min": -31.255863189697266,
      "activations/layer14_attention_weight_max": 38.504737854003906,
      "activations/layer14_attention_weight_min": -31.299270629882812,
      "activations/layer15_attention_weight_max": 35.49741744995117,
      "activations/layer15_attention_weight_min": -29.544279098510742,
      "activations/layer16_attention_weight_max": 31.61005401611328,
      "activations/layer16_attention_weight_min": -28.387451171875,
      "activations/layer17_attention_weight_max": 55.47209548950195,
      "activations/layer17_attention_weight_min": -45.34153366088867,
      "activations/layer18_attention_weight_max": 47.798194885253906,
      "activations/layer18_attention_weight_min": -36.45829772949219,
      "activations/layer19_attention_weight_max": 24.546497344970703,
      "activations/layer19_attention_weight_min": -21.033964157104492,
      "activations/layer1_attention_weight_max": 17.269725799560547,
      "activations/layer1_attention_weight_min": -14.758110046386719,
      "activations/layer20_attention_weight_max": 23.72692108154297,
      "activations/layer20_attention_weight_min": -23.438640594482422,
      "activations/layer21_attention_weight_max": 40.30952453613281,
      "activations/layer21_attention_weight_min": -24.286422729492188,
      "activations/layer22_attention_weight_max": 31.652490615844727,
      "activations/layer22_attention_weight_min": -26.0736026763916,
      "activations/layer23_attention_weight_max": 36.97148895263672,
      "activations/layer23_attention_weight_min": -22.676666259765625,
      "activations/layer2_attention_weight_max": 32.090980529785156,
      "activations/layer2_attention_weight_min": -30.073776245117188,
      "activations/layer3_attention_weight_max": 85.82621765136719,
      "activations/layer3_attention_weight_min": -88.8293228149414,
      "activations/layer4_attention_weight_max": 102.82103729248047,
      "activations/layer4_attention_weight_min": -95.44792175292969,
      "activations/layer5_attention_weight_max": 85.16934204101562,
      "activations/layer5_attention_weight_min": -82.11138916015625,
      "activations/layer6_attention_weight_max": 54.78327178955078,
      "activations/layer6_attention_weight_min": -52.95103073120117,
      "activations/layer7_attention_weight_max": 79.2420883178711,
      "activations/layer7_attention_weight_min": -76.62948608398438,
      "activations/layer8_attention_weight_max": 46.073143005371094,
      "activations/layer8_attention_weight_min": -47.96075439453125,
      "activations/layer9_attention_weight_max": 50.86933517456055,
      "activations/layer9_attention_weight_min": -51.90301513671875,
      "epoch": 8.74,
      "learning_rate": 9.455833333333332e-05,
      "loss": 2.8075,
      "step": 150500
    },
    {
      "activations/layer0_attention_weight_max": 15.431483268737793,
      "activations/layer0_attention_weight_min": -13.711029052734375,
      "activations/layer10_attention_weight_max": 40.82182693481445,
      "activations/layer10_attention_weight_min": -39.190738677978516,
      "activations/layer11_attention_weight_max": 34.319618225097656,
      "activations/layer11_attention_weight_min": -33.324283599853516,
      "activations/layer12_attention_weight_max": 19.806486129760742,
      "activations/layer12_attention_weight_min": -24.410356521606445,
      "activations/layer13_attention_weight_max": 47.21364212036133,
      "activations/layer13_attention_weight_min": -35.01312255859375,
      "activations/layer14_attention_weight_max": 40.8610725402832,
      "activations/layer14_attention_weight_min": -30.28407096862793,
      "activations/layer15_attention_weight_max": 35.83199691772461,
      "activations/layer15_attention_weight_min": -29.342330932617188,
      "activations/layer16_attention_weight_max": 32.57889175415039,
      "activations/layer16_attention_weight_min": -29.32419204711914,
      "activations/layer17_attention_weight_max": 56.8214111328125,
      "activations/layer17_attention_weight_min": -44.52827835083008,
      "activations/layer18_attention_weight_max": 51.07292938232422,
      "activations/layer18_attention_weight_min": -35.0919075012207,
      "activations/layer19_attention_weight_max": 20.58506965637207,
      "activations/layer19_attention_weight_min": -20.933547973632812,
      "activations/layer1_attention_weight_max": 16.78590202331543,
      "activations/layer1_attention_weight_min": -16.446895599365234,
      "activations/layer20_attention_weight_max": 23.162519454956055,
      "activations/layer20_attention_weight_min": -22.668132781982422,
      "activations/layer21_attention_weight_max": 39.37023162841797,
      "activations/layer21_attention_weight_min": -23.875730514526367,
      "activations/layer22_attention_weight_max": 35.37687683105469,
      "activations/layer22_attention_weight_min": -25.844892501831055,
      "activations/layer23_attention_weight_max": 37.93606185913086,
      "activations/layer23_attention_weight_min": -23.490215301513672,
      "activations/layer2_attention_weight_max": 31.255781173706055,
      "activations/layer2_attention_weight_min": -31.194927215576172,
      "activations/layer3_attention_weight_max": 90.95980834960938,
      "activations/layer3_attention_weight_min": -91.78123474121094,
      "activations/layer4_attention_weight_max": 104.5529556274414,
      "activations/layer4_attention_weight_min": -99.16940307617188,
      "activations/layer5_attention_weight_max": 88.08085632324219,
      "activations/layer5_attention_weight_min": -78.45867156982422,
      "activations/layer6_attention_weight_max": 58.66367721557617,
      "activations/layer6_attention_weight_min": -56.353294372558594,
      "activations/layer7_attention_weight_max": 80.18270874023438,
      "activations/layer7_attention_weight_min": -74.26602935791016,
      "activations/layer8_attention_weight_max": 48.943695068359375,
      "activations/layer8_attention_weight_min": -49.53522872924805,
      "activations/layer9_attention_weight_max": 51.29114532470703,
      "activations/layer9_attention_weight_min": -50.99380874633789,
      "epoch": 8.75,
      "learning_rate": 9.453939393939393e-05,
      "loss": 2.8139,
      "step": 150550
    },
    {
      "activations/layer0_attention_weight_max": 15.01482105255127,
      "activations/layer0_attention_weight_min": -14.050458908081055,
      "activations/layer10_attention_weight_max": 34.70015335083008,
      "activations/layer10_attention_weight_min": -34.294830322265625,
      "activations/layer11_attention_weight_max": 29.074546813964844,
      "activations/layer11_attention_weight_min": -29.958724975585938,
      "activations/layer12_attention_weight_max": 18.002742767333984,
      "activations/layer12_attention_weight_min": -25.52256965637207,
      "activations/layer13_attention_weight_max": 48.712669372558594,
      "activations/layer13_attention_weight_min": -33.90275192260742,
      "activations/layer14_attention_weight_max": 36.27175521850586,
      "activations/layer14_attention_weight_min": -31.528512954711914,
      "activations/layer15_attention_weight_max": 35.84355926513672,
      "activations/layer15_attention_weight_min": -29.51942253112793,
      "activations/layer16_attention_weight_max": 29.55789566040039,
      "activations/layer16_attention_weight_min": -27.244924545288086,
      "activations/layer17_attention_weight_max": 50.964500427246094,
      "activations/layer17_attention_weight_min": -43.66029739379883,
      "activations/layer18_attention_weight_max": 47.27854537963867,
      "activations/layer18_attention_weight_min": -36.82469177246094,
      "activations/layer19_attention_weight_max": 22.027366638183594,
      "activations/layer19_attention_weight_min": -20.342348098754883,
      "activations/layer1_attention_weight_max": 16.38753318786621,
      "activations/layer1_attention_weight_min": -13.393905639648438,
      "activations/layer20_attention_weight_max": 23.39263343811035,
      "activations/layer20_attention_weight_min": -23.094961166381836,
      "activations/layer21_attention_weight_max": 39.08346939086914,
      "activations/layer21_attention_weight_min": -21.465179443359375,
      "activations/layer22_attention_weight_max": 30.856233596801758,
      "activations/layer22_attention_weight_min": -24.763771057128906,
      "activations/layer23_attention_weight_max": 34.42897415161133,
      "activations/layer23_attention_weight_min": -21.947948455810547,
      "activations/layer2_attention_weight_max": 28.855178833007812,
      "activations/layer2_attention_weight_min": -27.348730087280273,
      "activations/layer3_attention_weight_max": 82.37041473388672,
      "activations/layer3_attention_weight_min": -85.24812316894531,
      "activations/layer4_attention_weight_max": 99.38420104980469,
      "activations/layer4_attention_weight_min": -92.72787475585938,
      "activations/layer5_attention_weight_max": 78.9692153930664,
      "activations/layer5_attention_weight_min": -79.2098388671875,
      "activations/layer6_attention_weight_max": 59.05470657348633,
      "activations/layer6_attention_weight_min": -52.36654281616211,
      "activations/layer7_attention_weight_max": 80.50285339355469,
      "activations/layer7_attention_weight_min": -73.34064483642578,
      "activations/layer8_attention_weight_max": 43.449527740478516,
      "activations/layer8_attention_weight_min": -45.440818786621094,
      "activations/layer9_attention_weight_max": 49.58826446533203,
      "activations/layer9_attention_weight_min": -48.865962982177734,
      "epoch": 8.75,
      "learning_rate": 9.452083333333332e-05,
      "loss": 2.8116,
      "step": 150600
    },
    {
      "activations/layer0_attention_weight_max": 16.41718292236328,
      "activations/layer0_attention_weight_min": -14.237110137939453,
      "activations/layer10_attention_weight_max": 45.53118896484375,
      "activations/layer10_attention_weight_min": -42.48048782348633,
      "activations/layer11_attention_weight_max": 40.68421936035156,
      "activations/layer11_attention_weight_min": -34.5236930847168,
      "activations/layer12_attention_weight_max": 25.122726440429688,
      "activations/layer12_attention_weight_min": -28.585453033447266,
      "activations/layer13_attention_weight_max": 45.28144454956055,
      "activations/layer13_attention_weight_min": -35.16011047363281,
      "activations/layer14_attention_weight_max": 49.50886154174805,
      "activations/layer14_attention_weight_min": -39.20109939575195,
      "activations/layer15_attention_weight_max": 41.00969314575195,
      "activations/layer15_attention_weight_min": -35.10203170776367,
      "activations/layer16_attention_weight_max": 34.944698333740234,
      "activations/layer16_attention_weight_min": -30.514102935791016,
      "activations/layer17_attention_weight_max": 53.188026428222656,
      "activations/layer17_attention_weight_min": -44.50059127807617,
      "activations/layer18_attention_weight_max": 47.62247085571289,
      "activations/layer18_attention_weight_min": -39.361785888671875,
      "activations/layer19_attention_weight_max": 21.470266342163086,
      "activations/layer19_attention_weight_min": -22.500484466552734,
      "activations/layer1_attention_weight_max": 17.567697525024414,
      "activations/layer1_attention_weight_min": -13.890584945678711,
      "activations/layer20_attention_weight_max": 21.017696380615234,
      "activations/layer20_attention_weight_min": -22.87421417236328,
      "activations/layer21_attention_weight_max": 34.14459228515625,
      "activations/layer21_attention_weight_min": -22.866867065429688,
      "activations/layer22_attention_weight_max": 32.44723129272461,
      "activations/layer22_attention_weight_min": -23.13185691833496,
      "activations/layer23_attention_weight_max": 38.37970733642578,
      "activations/layer23_attention_weight_min": -23.499103546142578,
      "activations/layer2_attention_weight_max": 32.137821197509766,
      "activations/layer2_attention_weight_min": -32.1456184387207,
      "activations/layer3_attention_weight_max": 87.79320526123047,
      "activations/layer3_attention_weight_min": -95.41827392578125,
      "activations/layer4_attention_weight_max": 99.45470428466797,
      "activations/layer4_attention_weight_min": -97.79247283935547,
      "activations/layer5_attention_weight_max": 82.44607543945312,
      "activations/layer5_attention_weight_min": -86.11344909667969,
      "activations/layer6_attention_weight_max": 56.457786560058594,
      "activations/layer6_attention_weight_min": -54.959434509277344,
      "activations/layer7_attention_weight_max": 90.76739501953125,
      "activations/layer7_attention_weight_min": -77.34999084472656,
      "activations/layer8_attention_weight_max": 55.10011291503906,
      "activations/layer8_attention_weight_min": -46.37287139892578,
      "activations/layer9_attention_weight_max": 71.55493927001953,
      "activations/layer9_attention_weight_min": -59.12541580200195,
      "epoch": 8.75,
      "learning_rate": 9.450189393939393e-05,
      "loss": 2.8131,
      "step": 150650
    },
    {
      "activations/layer0_attention_weight_max": 15.540002822875977,
      "activations/layer0_attention_weight_min": -13.807205200195312,
      "activations/layer10_attention_weight_max": 51.365108489990234,
      "activations/layer10_attention_weight_min": -44.90463638305664,
      "activations/layer11_attention_weight_max": 43.38977813720703,
      "activations/layer11_attention_weight_min": -37.86644744873047,
      "activations/layer12_attention_weight_max": 18.71350860595703,
      "activations/layer12_attention_weight_min": -26.324438095092773,
      "activations/layer13_attention_weight_max": 52.8569450378418,
      "activations/layer13_attention_weight_min": -32.097076416015625,
      "activations/layer14_attention_weight_max": 74.55387115478516,
      "activations/layer14_attention_weight_min": -37.056766510009766,
      "activations/layer15_attention_weight_max": 67.08231353759766,
      "activations/layer15_attention_weight_min": -32.42340087890625,
      "activations/layer16_attention_weight_max": 42.8419303894043,
      "activations/layer16_attention_weight_min": -34.375125885009766,
      "activations/layer17_attention_weight_max": 62.9532356262207,
      "activations/layer17_attention_weight_min": -56.73443603515625,
      "activations/layer18_attention_weight_max": 53.99504852294922,
      "activations/layer18_attention_weight_min": -47.05880355834961,
      "activations/layer19_attention_weight_max": 22.02987289428711,
      "activations/layer19_attention_weight_min": -23.948240280151367,
      "activations/layer1_attention_weight_max": 17.14979362487793,
      "activations/layer1_attention_weight_min": -14.420004844665527,
      "activations/layer20_attention_weight_max": 25.459074020385742,
      "activations/layer20_attention_weight_min": -19.993932723999023,
      "activations/layer21_attention_weight_max": 41.49464797973633,
      "activations/layer21_attention_weight_min": -22.416015625,
      "activations/layer22_attention_weight_max": 36.671634674072266,
      "activations/layer22_attention_weight_min": -23.675273895263672,
      "activations/layer23_attention_weight_max": 34.122562408447266,
      "activations/layer23_attention_weight_min": -20.636775970458984,
      "activations/layer2_attention_weight_max": 34.47261047363281,
      "activations/layer2_attention_weight_min": -31.619979858398438,
      "activations/layer3_attention_weight_max": 102.86774444580078,
      "activations/layer3_attention_weight_min": -101.00362396240234,
      "activations/layer4_attention_weight_max": 116.17646789550781,
      "activations/layer4_attention_weight_min": -105.0387191772461,
      "activations/layer5_attention_weight_max": 90.31889343261719,
      "activations/layer5_attention_weight_min": -80.38753509521484,
      "activations/layer6_attention_weight_max": 58.470008850097656,
      "activations/layer6_attention_weight_min": -55.25601577758789,
      "activations/layer7_attention_weight_max": 95.98900604248047,
      "activations/layer7_attention_weight_min": -80.92005920410156,
      "activations/layer8_attention_weight_max": 56.81114196777344,
      "activations/layer8_attention_weight_min": -52.47795486450195,
      "activations/layer9_attention_weight_max": 70.55296325683594,
      "activations/layer9_attention_weight_min": -59.504390716552734,
      "epoch": 8.76,
      "learning_rate": 9.448295454545454e-05,
      "loss": 2.7954,
      "step": 150700
    },
    {
      "activations/layer0_attention_weight_max": 14.950238227844238,
      "activations/layer0_attention_weight_min": -14.182429313659668,
      "activations/layer10_attention_weight_max": 37.527183532714844,
      "activations/layer10_attention_weight_min": -35.63043975830078,
      "activations/layer11_attention_weight_max": 32.921409606933594,
      "activations/layer11_attention_weight_min": -32.921875,
      "activations/layer12_attention_weight_max": 21.41697120666504,
      "activations/layer12_attention_weight_min": -26.40019989013672,
      "activations/layer13_attention_weight_max": 48.328208923339844,
      "activations/layer13_attention_weight_min": -32.08621597290039,
      "activations/layer14_attention_weight_max": 49.83238220214844,
      "activations/layer14_attention_weight_min": -32.43269348144531,
      "activations/layer15_attention_weight_max": 40.9217414855957,
      "activations/layer15_attention_weight_min": -29.278873443603516,
      "activations/layer16_attention_weight_max": 31.807527542114258,
      "activations/layer16_attention_weight_min": -28.66557502746582,
      "activations/layer17_attention_weight_max": 53.21931076049805,
      "activations/layer17_attention_weight_min": -45.8090705871582,
      "activations/layer18_attention_weight_max": 47.06310272216797,
      "activations/layer18_attention_weight_min": -40.69413375854492,
      "activations/layer19_attention_weight_max": 22.48090934753418,
      "activations/layer19_attention_weight_min": -21.742368698120117,
      "activations/layer1_attention_weight_max": 17.45199203491211,
      "activations/layer1_attention_weight_min": -15.634126663208008,
      "activations/layer20_attention_weight_max": 26.116249084472656,
      "activations/layer20_attention_weight_min": -24.13435173034668,
      "activations/layer21_attention_weight_max": 40.63948440551758,
      "activations/layer21_attention_weight_min": -21.219057083129883,
      "activations/layer22_attention_weight_max": 32.58268356323242,
      "activations/layer22_attention_weight_min": -24.918922424316406,
      "activations/layer23_attention_weight_max": 41.064144134521484,
      "activations/layer23_attention_weight_min": -22.012962341308594,
      "activations/layer2_attention_weight_max": 31.65045928955078,
      "activations/layer2_attention_weight_min": -29.411527633666992,
      "activations/layer3_attention_weight_max": 87.85871887207031,
      "activations/layer3_attention_weight_min": -98.52689361572266,
      "activations/layer4_attention_weight_max": 99.60175323486328,
      "activations/layer4_attention_weight_min": -98.45924377441406,
      "activations/layer5_attention_weight_max": 82.32646942138672,
      "activations/layer5_attention_weight_min": -77.83392333984375,
      "activations/layer6_attention_weight_max": 54.50435256958008,
      "activations/layer6_attention_weight_min": -53.94443130493164,
      "activations/layer7_attention_weight_max": 78.8244857788086,
      "activations/layer7_attention_weight_min": -75.979248046875,
      "activations/layer8_attention_weight_max": 44.63169479370117,
      "activations/layer8_attention_weight_min": -48.12884521484375,
      "activations/layer9_attention_weight_max": 55.25543975830078,
      "activations/layer9_attention_weight_min": -52.1571159362793,
      "epoch": 8.76,
      "learning_rate": 9.446401515151514e-05,
      "loss": 2.8069,
      "step": 150750
    },
    {
      "activations/layer0_attention_weight_max": 15.590631484985352,
      "activations/layer0_attention_weight_min": -13.621722221374512,
      "activations/layer10_attention_weight_max": 39.45643997192383,
      "activations/layer10_attention_weight_min": -39.37417984008789,
      "activations/layer11_attention_weight_max": 32.7460823059082,
      "activations/layer11_attention_weight_min": -33.82556915283203,
      "activations/layer12_attention_weight_max": 20.412269592285156,
      "activations/layer12_attention_weight_min": -28.714820861816406,
      "activations/layer13_attention_weight_max": 41.79775619506836,
      "activations/layer13_attention_weight_min": -31.143014907836914,
      "activations/layer14_attention_weight_max": 37.575782775878906,
      "activations/layer14_attention_weight_min": -29.188474655151367,
      "activations/layer15_attention_weight_max": 34.15834045410156,
      "activations/layer15_attention_weight_min": -28.2377872467041,
      "activations/layer16_attention_weight_max": 33.880550384521484,
      "activations/layer16_attention_weight_min": -26.817243576049805,
      "activations/layer17_attention_weight_max": 51.54331970214844,
      "activations/layer17_attention_weight_min": -42.28826904296875,
      "activations/layer18_attention_weight_max": 50.77433395385742,
      "activations/layer18_attention_weight_min": -35.679588317871094,
      "activations/layer19_attention_weight_max": 21.479406356811523,
      "activations/layer19_attention_weight_min": -22.951292037963867,
      "activations/layer1_attention_weight_max": 17.239765167236328,
      "activations/layer1_attention_weight_min": -15.042869567871094,
      "activations/layer20_attention_weight_max": 25.211448669433594,
      "activations/layer20_attention_weight_min": -20.07315444946289,
      "activations/layer21_attention_weight_max": 38.77946853637695,
      "activations/layer21_attention_weight_min": -23.791839599609375,
      "activations/layer22_attention_weight_max": 37.08192443847656,
      "activations/layer22_attention_weight_min": -24.890384674072266,
      "activations/layer23_attention_weight_max": 44.08161926269531,
      "activations/layer23_attention_weight_min": -22.786819458007812,
      "activations/layer2_attention_weight_max": 32.68263244628906,
      "activations/layer2_attention_weight_min": -31.746068954467773,
      "activations/layer3_attention_weight_max": 92.25547790527344,
      "activations/layer3_attention_weight_min": -94.31237030029297,
      "activations/layer4_attention_weight_max": 106.73479461669922,
      "activations/layer4_attention_weight_min": -100.35301971435547,
      "activations/layer5_attention_weight_max": 85.93536376953125,
      "activations/layer5_attention_weight_min": -83.63642883300781,
      "activations/layer6_attention_weight_max": 62.555213928222656,
      "activations/layer6_attention_weight_min": -59.72871780395508,
      "activations/layer7_attention_weight_max": 79.49346160888672,
      "activations/layer7_attention_weight_min": -84.8213119506836,
      "activations/layer8_attention_weight_max": 46.9770393371582,
      "activations/layer8_attention_weight_min": -47.841766357421875,
      "activations/layer9_attention_weight_max": 54.0806999206543,
      "activations/layer9_attention_weight_min": -53.26180648803711,
      "epoch": 8.76,
      "learning_rate": 9.444507575757574e-05,
      "loss": 2.8024,
      "step": 150800
    },
    {
      "activations/layer0_attention_weight_max": 15.4508638381958,
      "activations/layer0_attention_weight_min": -13.720094680786133,
      "activations/layer10_attention_weight_max": 37.194393157958984,
      "activations/layer10_attention_weight_min": -34.44041442871094,
      "activations/layer11_attention_weight_max": 33.27553939819336,
      "activations/layer11_attention_weight_min": -32.062068939208984,
      "activations/layer12_attention_weight_max": 19.553335189819336,
      "activations/layer12_attention_weight_min": -25.616548538208008,
      "activations/layer13_attention_weight_max": 41.41807556152344,
      "activations/layer13_attention_weight_min": -28.667505264282227,
      "activations/layer14_attention_weight_max": 38.542503356933594,
      "activations/layer14_attention_weight_min": -29.233341217041016,
      "activations/layer15_attention_weight_max": 33.832908630371094,
      "activations/layer15_attention_weight_min": -29.180246353149414,
      "activations/layer16_attention_weight_max": 35.85636520385742,
      "activations/layer16_attention_weight_min": -27.958385467529297,
      "activations/layer17_attention_weight_max": 58.4407958984375,
      "activations/layer17_attention_weight_min": -43.38111114501953,
      "activations/layer18_attention_weight_max": 48.251739501953125,
      "activations/layer18_attention_weight_min": -37.947113037109375,
      "activations/layer19_attention_weight_max": 21.893583297729492,
      "activations/layer19_attention_weight_min": -21.74623680114746,
      "activations/layer1_attention_weight_max": 15.877402305603027,
      "activations/layer1_attention_weight_min": -13.827823638916016,
      "activations/layer20_attention_weight_max": 26.204877853393555,
      "activations/layer20_attention_weight_min": -20.528573989868164,
      "activations/layer21_attention_weight_max": 46.00019454956055,
      "activations/layer21_attention_weight_min": -23.88187599182129,
      "activations/layer22_attention_weight_max": 35.98762130737305,
      "activations/layer22_attention_weight_min": -25.178133010864258,
      "activations/layer23_attention_weight_max": 39.215171813964844,
      "activations/layer23_attention_weight_min": -22.89608383178711,
      "activations/layer2_attention_weight_max": 31.017927169799805,
      "activations/layer2_attention_weight_min": -29.507614135742188,
      "activations/layer3_attention_weight_max": 92.03314971923828,
      "activations/layer3_attention_weight_min": -90.74810791015625,
      "activations/layer4_attention_weight_max": 103.15325164794922,
      "activations/layer4_attention_weight_min": -94.8715591430664,
      "activations/layer5_attention_weight_max": 79.25463104248047,
      "activations/layer5_attention_weight_min": -80.90139770507812,
      "activations/layer6_attention_weight_max": 54.02656173706055,
      "activations/layer6_attention_weight_min": -53.19847869873047,
      "activations/layer7_attention_weight_max": 78.11984252929688,
      "activations/layer7_attention_weight_min": -73.4985122680664,
      "activations/layer8_attention_weight_max": 43.69565963745117,
      "activations/layer8_attention_weight_min": -46.33879470825195,
      "activations/layer9_attention_weight_max": 50.52134704589844,
      "activations/layer9_attention_weight_min": -48.61934280395508,
      "epoch": 8.77,
      "learning_rate": 9.442613636363636e-05,
      "loss": 2.7956,
      "step": 150850
    },
    {
      "activations/layer0_attention_weight_max": 15.368088722229004,
      "activations/layer0_attention_weight_min": -13.67049503326416,
      "activations/layer10_attention_weight_max": 37.96413040161133,
      "activations/layer10_attention_weight_min": -36.278358459472656,
      "activations/layer11_attention_weight_max": 32.85160827636719,
      "activations/layer11_attention_weight_min": -31.49103546142578,
      "activations/layer12_attention_weight_max": 20.166452407836914,
      "activations/layer12_attention_weight_min": -22.552539825439453,
      "activations/layer13_attention_weight_max": 49.16895294189453,
      "activations/layer13_attention_weight_min": -35.03321075439453,
      "activations/layer14_attention_weight_max": 47.03853225708008,
      "activations/layer14_attention_weight_min": -31.565349578857422,
      "activations/layer15_attention_weight_max": 35.969852447509766,
      "activations/layer15_attention_weight_min": -30.355533599853516,
      "activations/layer16_attention_weight_max": 33.12615966796875,
      "activations/layer16_attention_weight_min": -30.336870193481445,
      "activations/layer17_attention_weight_max": 54.54952621459961,
      "activations/layer17_attention_weight_min": -46.87320327758789,
      "activations/layer18_attention_weight_max": 47.73976135253906,
      "activations/layer18_attention_weight_min": -41.15509796142578,
      "activations/layer19_attention_weight_max": 23.170574188232422,
      "activations/layer19_attention_weight_min": -21.899938583374023,
      "activations/layer1_attention_weight_max": 16.349327087402344,
      "activations/layer1_attention_weight_min": -15.55280876159668,
      "activations/layer20_attention_weight_max": 23.728435516357422,
      "activations/layer20_attention_weight_min": -27.794189453125,
      "activations/layer21_attention_weight_max": 40.40044403076172,
      "activations/layer21_attention_weight_min": -26.428342819213867,
      "activations/layer22_attention_weight_max": 32.5361442565918,
      "activations/layer22_attention_weight_min": -26.52437400817871,
      "activations/layer23_attention_weight_max": 39.07685852050781,
      "activations/layer23_attention_weight_min": -23.159996032714844,
      "activations/layer2_attention_weight_max": 30.301307678222656,
      "activations/layer2_attention_weight_min": -27.792526245117188,
      "activations/layer3_attention_weight_max": 82.56438446044922,
      "activations/layer3_attention_weight_min": -84.31990814208984,
      "activations/layer4_attention_weight_max": 98.61624908447266,
      "activations/layer4_attention_weight_min": -96.082763671875,
      "activations/layer5_attention_weight_max": 80.89663696289062,
      "activations/layer5_attention_weight_min": -78.6713638305664,
      "activations/layer6_attention_weight_max": 54.459354400634766,
      "activations/layer6_attention_weight_min": -53.08224868774414,
      "activations/layer7_attention_weight_max": 75.6604232788086,
      "activations/layer7_attention_weight_min": -76.08894348144531,
      "activations/layer8_attention_weight_max": 49.15900421142578,
      "activations/layer8_attention_weight_min": -50.278072357177734,
      "activations/layer9_attention_weight_max": 51.12778091430664,
      "activations/layer9_attention_weight_min": -50.99812316894531,
      "epoch": 8.77,
      "learning_rate": 9.440719696969697e-05,
      "loss": 2.8231,
      "step": 150900
    },
    {
      "activations/layer0_attention_weight_max": 15.628988265991211,
      "activations/layer0_attention_weight_min": -13.026568412780762,
      "activations/layer10_attention_weight_max": 39.39418411254883,
      "activations/layer10_attention_weight_min": -37.63167190551758,
      "activations/layer11_attention_weight_max": 34.094974517822266,
      "activations/layer11_attention_weight_min": -33.59668731689453,
      "activations/layer12_attention_weight_max": 19.310916900634766,
      "activations/layer12_attention_weight_min": -24.640390396118164,
      "activations/layer13_attention_weight_max": 42.76608657836914,
      "activations/layer13_attention_weight_min": -32.459075927734375,
      "activations/layer14_attention_weight_max": 36.95949172973633,
      "activations/layer14_attention_weight_min": -30.626258850097656,
      "activations/layer15_attention_weight_max": 33.81875991821289,
      "activations/layer15_attention_weight_min": -30.52096939086914,
      "activations/layer16_attention_weight_max": 35.31575012207031,
      "activations/layer16_attention_weight_min": -28.130033493041992,
      "activations/layer17_attention_weight_max": 54.569114685058594,
      "activations/layer17_attention_weight_min": -42.161407470703125,
      "activations/layer18_attention_weight_max": 51.783016204833984,
      "activations/layer18_attention_weight_min": -35.222965240478516,
      "activations/layer19_attention_weight_max": 25.278217315673828,
      "activations/layer19_attention_weight_min": -21.391416549682617,
      "activations/layer1_attention_weight_max": 16.640785217285156,
      "activations/layer1_attention_weight_min": -14.789592742919922,
      "activations/layer20_attention_weight_max": 23.800155639648438,
      "activations/layer20_attention_weight_min": -22.16963768005371,
      "activations/layer21_attention_weight_max": 39.340850830078125,
      "activations/layer21_attention_weight_min": -22.075735092163086,
      "activations/layer22_attention_weight_max": 34.23814010620117,
      "activations/layer22_attention_weight_min": -26.029722213745117,
      "activations/layer23_attention_weight_max": 41.762229919433594,
      "activations/layer23_attention_weight_min": -22.166881561279297,
      "activations/layer2_attention_weight_max": 31.814884185791016,
      "activations/layer2_attention_weight_min": -30.47262191772461,
      "activations/layer3_attention_weight_max": 91.28852081298828,
      "activations/layer3_attention_weight_min": -94.32391357421875,
      "activations/layer4_attention_weight_max": 101.73021697998047,
      "activations/layer4_attention_weight_min": -98.31956481933594,
      "activations/layer5_attention_weight_max": 79.62213134765625,
      "activations/layer5_attention_weight_min": -78.1388931274414,
      "activations/layer6_attention_weight_max": 55.217262268066406,
      "activations/layer6_attention_weight_min": -54.744327545166016,
      "activations/layer7_attention_weight_max": 78.44178771972656,
      "activations/layer7_attention_weight_min": -78.33267974853516,
      "activations/layer8_attention_weight_max": 47.45766067504883,
      "activations/layer8_attention_weight_min": -47.267757415771484,
      "activations/layer9_attention_weight_max": 51.72760772705078,
      "activations/layer9_attention_weight_min": -49.27231979370117,
      "epoch": 8.77,
      "learning_rate": 9.438825757575756e-05,
      "loss": 2.8105,
      "step": 150950
    },
    {
      "activations/layer0_attention_weight_max": 15.294633865356445,
      "activations/layer0_attention_weight_min": -13.693629264831543,
      "activations/layer10_attention_weight_max": 35.932167053222656,
      "activations/layer10_attention_weight_min": -35.76544952392578,
      "activations/layer11_attention_weight_max": 33.39820098876953,
      "activations/layer11_attention_weight_min": -30.531665802001953,
      "activations/layer12_attention_weight_max": 17.179153442382812,
      "activations/layer12_attention_weight_min": -26.88319206237793,
      "activations/layer13_attention_weight_max": 38.17940139770508,
      "activations/layer13_attention_weight_min": -30.23984718322754,
      "activations/layer14_attention_weight_max": 37.37704086303711,
      "activations/layer14_attention_weight_min": -29.465042114257812,
      "activations/layer15_attention_weight_max": 33.031795501708984,
      "activations/layer15_attention_weight_min": -30.09284019470215,
      "activations/layer16_attention_weight_max": 31.4329833984375,
      "activations/layer16_attention_weight_min": -27.310691833496094,
      "activations/layer17_attention_weight_max": 57.0613899230957,
      "activations/layer17_attention_weight_min": -44.53654861450195,
      "activations/layer18_attention_weight_max": 44.10749053955078,
      "activations/layer18_attention_weight_min": -36.016239166259766,
      "activations/layer19_attention_weight_max": 19.877681732177734,
      "activations/layer19_attention_weight_min": -19.272083282470703,
      "activations/layer1_attention_weight_max": 16.598779678344727,
      "activations/layer1_attention_weight_min": -13.907546997070312,
      "activations/layer20_attention_weight_max": 20.137157440185547,
      "activations/layer20_attention_weight_min": -25.531957626342773,
      "activations/layer21_attention_weight_max": 37.86937713623047,
      "activations/layer21_attention_weight_min": -23.994199752807617,
      "activations/layer22_attention_weight_max": 29.172571182250977,
      "activations/layer22_attention_weight_min": -24.24428367614746,
      "activations/layer23_attention_weight_max": 34.88233947753906,
      "activations/layer23_attention_weight_min": -21.9807186126709,
      "activations/layer2_attention_weight_max": 29.98715591430664,
      "activations/layer2_attention_weight_min": -28.045791625976562,
      "activations/layer3_attention_weight_max": 86.041015625,
      "activations/layer3_attention_weight_min": -86.81822204589844,
      "activations/layer4_attention_weight_max": 96.94239044189453,
      "activations/layer4_attention_weight_min": -90.98360443115234,
      "activations/layer5_attention_weight_max": 79.65220642089844,
      "activations/layer5_attention_weight_min": -80.14593505859375,
      "activations/layer6_attention_weight_max": 52.522762298583984,
      "activations/layer6_attention_weight_min": -52.30629348754883,
      "activations/layer7_attention_weight_max": 77.17412567138672,
      "activations/layer7_attention_weight_min": -71.31289672851562,
      "activations/layer8_attention_weight_max": 43.477691650390625,
      "activations/layer8_attention_weight_min": -44.97319412231445,
      "activations/layer9_attention_weight_max": 49.68589401245117,
      "activations/layer9_attention_weight_min": -50.298885345458984,
      "epoch": 8.77,
      "learning_rate": 9.436931818181818e-05,
      "loss": 2.7994,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.4747,
      "eval_samples_per_second": 506.682,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.4747,
      "eval_openwebtext_samples_per_second": 506.682,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 1.9836,
      "eval_wikitext_samples_per_second": 229.88,
      "step": 151000
    },
    {
      "epoch": 8.77,
      "eval_lambada_loss": 2.630859375,
      "eval_lambada_ppl": 13.885697797673076,
      "eval_lambada_runtime": 9.5601,
      "eval_lambada_samples_per_second": 509.307,
      "step": 151000
    },
    {
      "activations/layer0_attention_weight_max": 13.998669624328613,
      "activations/layer0_attention_weight_min": -13.980514526367188,
      "activations/layer10_attention_weight_max": 41.967681884765625,
      "activations/layer10_attention_weight_min": -38.46680450439453,
      "activations/layer11_attention_weight_max": 35.52374267578125,
      "activations/layer11_attention_weight_min": -33.889068603515625,
      "activations/layer12_attention_weight_max": 22.520126342773438,
      "activations/layer12_attention_weight_min": -25.701814651489258,
      "activations/layer13_attention_weight_max": 48.3106803894043,
      "activations/layer13_attention_weight_min": -35.071067810058594,
      "activations/layer14_attention_weight_max": 40.493587493896484,
      "activations/layer14_attention_weight_min": -30.032001495361328,
      "activations/layer15_attention_weight_max": 36.90732192993164,
      "activations/layer15_attention_weight_min": -27.63728141784668,
      "activations/layer16_attention_weight_max": 37.22955322265625,
      "activations/layer16_attention_weight_min": -28.616546630859375,
      "activations/layer17_attention_weight_max": 61.57403564453125,
      "activations/layer17_attention_weight_min": -42.86335754394531,
      "activations/layer18_attention_weight_max": 55.35805130004883,
      "activations/layer18_attention_weight_min": -38.47877883911133,
      "activations/layer19_attention_weight_max": 22.762725830078125,
      "activations/layer19_attention_weight_min": -21.38273048400879,
      "activations/layer1_attention_weight_max": 16.851024627685547,
      "activations/layer1_attention_weight_min": -15.659981727600098,
      "activations/layer20_attention_weight_max": 26.425243377685547,
      "activations/layer20_attention_weight_min": -19.730606079101562,
      "activations/layer21_attention_weight_max": 47.20908737182617,
      "activations/layer21_attention_weight_min": -22.240434646606445,
      "activations/layer22_attention_weight_max": 38.00388717651367,
      "activations/layer22_attention_weight_min": -25.450180053710938,
      "activations/layer23_attention_weight_max": 45.72288513183594,
      "activations/layer23_attention_weight_min": -20.97303009033203,
      "activations/layer2_attention_weight_max": 31.432363510131836,
      "activations/layer2_attention_weight_min": -29.377256393432617,
      "activations/layer3_attention_weight_max": 87.56871032714844,
      "activations/layer3_attention_weight_min": -91.07994842529297,
      "activations/layer4_attention_weight_max": 101.47896575927734,
      "activations/layer4_attention_weight_min": -96.92858123779297,
      "activations/layer5_attention_weight_max": 82.98792266845703,
      "activations/layer5_attention_weight_min": -79.5776596069336,
      "activations/layer6_attention_weight_max": 56.8649787902832,
      "activations/layer6_attention_weight_min": -53.23764419555664,
      "activations/layer7_attention_weight_max": 89.46411895751953,
      "activations/layer7_attention_weight_min": -76.5940933227539,
      "activations/layer8_attention_weight_max": 50.4260139465332,
      "activations/layer8_attention_weight_min": -48.02949523925781,
      "activations/layer9_attention_weight_max": 60.97115707397461,
      "activations/layer9_attention_weight_min": -51.6725959777832,
      "epoch": 8.78,
      "learning_rate": 9.435037878787878e-05,
      "loss": 2.7995,
      "step": 151050
    },
    {
      "activations/layer0_attention_weight_max": 17.248716354370117,
      "activations/layer0_attention_weight_min": -13.953912734985352,
      "activations/layer10_attention_weight_max": 43.21962356567383,
      "activations/layer10_attention_weight_min": -40.366878509521484,
      "activations/layer11_attention_weight_max": 37.4914436340332,
      "activations/layer11_attention_weight_min": -36.2198371887207,
      "activations/layer12_attention_weight_max": 18.064729690551758,
      "activations/layer12_attention_weight_min": -30.592758178710938,
      "activations/layer13_attention_weight_max": 40.80427932739258,
      "activations/layer13_attention_weight_min": -30.818832397460938,
      "activations/layer14_attention_weight_max": 41.776771545410156,
      "activations/layer14_attention_weight_min": -29.85225486755371,
      "activations/layer15_attention_weight_max": 35.38874435424805,
      "activations/layer15_attention_weight_min": -28.9816837310791,
      "activations/layer16_attention_weight_max": 31.01066017150879,
      "activations/layer16_attention_weight_min": -27.657135009765625,
      "activations/layer17_attention_weight_max": 57.02216339111328,
      "activations/layer17_attention_weight_min": -44.417720794677734,
      "activations/layer18_attention_weight_max": 46.3679313659668,
      "activations/layer18_attention_weight_min": -35.474456787109375,
      "activations/layer19_attention_weight_max": 22.921476364135742,
      "activations/layer19_attention_weight_min": -21.78190803527832,
      "activations/layer1_attention_weight_max": 18.263315200805664,
      "activations/layer1_attention_weight_min": -14.64964485168457,
      "activations/layer20_attention_weight_max": 23.493471145629883,
      "activations/layer20_attention_weight_min": -23.17926597595215,
      "activations/layer21_attention_weight_max": 43.66459274291992,
      "activations/layer21_attention_weight_min": -23.692678451538086,
      "activations/layer22_attention_weight_max": 31.21454429626465,
      "activations/layer22_attention_weight_min": -24.817646026611328,
      "activations/layer23_attention_weight_max": 37.45941925048828,
      "activations/layer23_attention_weight_min": -22.17184066772461,
      "activations/layer2_attention_weight_max": 32.17146301269531,
      "activations/layer2_attention_weight_min": -31.029281616210938,
      "activations/layer3_attention_weight_max": 86.97418975830078,
      "activations/layer3_attention_weight_min": -98.00917053222656,
      "activations/layer4_attention_weight_max": 100.67678833007812,
      "activations/layer4_attention_weight_min": -97.12267303466797,
      "activations/layer5_attention_weight_max": 80.71235656738281,
      "activations/layer5_attention_weight_min": -76.7147216796875,
      "activations/layer6_attention_weight_max": 53.853763580322266,
      "activations/layer6_attention_weight_min": -51.71413803100586,
      "activations/layer7_attention_weight_max": 79.20524597167969,
      "activations/layer7_attention_weight_min": -75.33462524414062,
      "activations/layer8_attention_weight_max": 46.2354621887207,
      "activations/layer8_attention_weight_min": -46.89584732055664,
      "activations/layer9_attention_weight_max": 51.9239387512207,
      "activations/layer9_attention_weight_min": -51.61515426635742,
      "epoch": 8.78,
      "learning_rate": 9.433143939393938e-05,
      "loss": 2.81,
      "step": 151100
    },
    {
      "activations/layer0_attention_weight_max": 14.88796329498291,
      "activations/layer0_attention_weight_min": -14.070903778076172,
      "activations/layer10_attention_weight_max": 37.224369049072266,
      "activations/layer10_attention_weight_min": -36.20302200317383,
      "activations/layer11_attention_weight_max": 31.57648468017578,
      "activations/layer11_attention_weight_min": -31.841175079345703,
      "activations/layer12_attention_weight_max": 17.336645126342773,
      "activations/layer12_attention_weight_min": -24.63945960998535,
      "activations/layer13_attention_weight_max": 42.45890808105469,
      "activations/layer13_attention_weight_min": -37.29876708984375,
      "activations/layer14_attention_weight_max": 37.20378112792969,
      "activations/layer14_attention_weight_min": -30.960208892822266,
      "activations/layer15_attention_weight_max": 32.55964279174805,
      "activations/layer15_attention_weight_min": -30.241222381591797,
      "activations/layer16_attention_weight_max": 31.28971290588379,
      "activations/layer16_attention_weight_min": -28.29408073425293,
      "activations/layer17_attention_weight_max": 51.778812408447266,
      "activations/layer17_attention_weight_min": -45.73543930053711,
      "activations/layer18_attention_weight_max": 46.74845886230469,
      "activations/layer18_attention_weight_min": -37.81769943237305,
      "activations/layer19_attention_weight_max": 20.302261352539062,
      "activations/layer19_attention_weight_min": -21.195117950439453,
      "activations/layer1_attention_weight_max": 17.020570755004883,
      "activations/layer1_attention_weight_min": -19.14563751220703,
      "activations/layer20_attention_weight_max": 21.58556365966797,
      "activations/layer20_attention_weight_min": -23.683422088623047,
      "activations/layer21_attention_weight_max": 35.63563919067383,
      "activations/layer21_attention_weight_min": -23.657798767089844,
      "activations/layer22_attention_weight_max": 32.51789474487305,
      "activations/layer22_attention_weight_min": -25.112892150878906,
      "activations/layer23_attention_weight_max": 39.190731048583984,
      "activations/layer23_attention_weight_min": -21.328689575195312,
      "activations/layer2_attention_weight_max": 31.240795135498047,
      "activations/layer2_attention_weight_min": -29.054584503173828,
      "activations/layer3_attention_weight_max": 86.23440551757812,
      "activations/layer3_attention_weight_min": -89.7110595703125,
      "activations/layer4_attention_weight_max": 96.56849670410156,
      "activations/layer4_attention_weight_min": -95.21940612792969,
      "activations/layer5_attention_weight_max": 78.97040557861328,
      "activations/layer5_attention_weight_min": -74.39696502685547,
      "activations/layer6_attention_weight_max": 53.22718048095703,
      "activations/layer6_attention_weight_min": -51.82322311401367,
      "activations/layer7_attention_weight_max": 74.81794738769531,
      "activations/layer7_attention_weight_min": -71.8265151977539,
      "activations/layer8_attention_weight_max": 44.85004425048828,
      "activations/layer8_attention_weight_min": -49.88290023803711,
      "activations/layer9_attention_weight_max": 48.19659423828125,
      "activations/layer9_attention_weight_min": -50.29445266723633,
      "epoch": 8.78,
      "learning_rate": 9.43125e-05,
      "loss": 2.8108,
      "step": 151150
    },
    {
      "activations/layer0_attention_weight_max": 14.495270729064941,
      "activations/layer0_attention_weight_min": -13.8091459274292,
      "activations/layer10_attention_weight_max": 40.78481674194336,
      "activations/layer10_attention_weight_min": -37.418270111083984,
      "activations/layer11_attention_weight_max": 33.951133728027344,
      "activations/layer11_attention_weight_min": -32.61033248901367,
      "activations/layer12_attention_weight_max": 20.38643455505371,
      "activations/layer12_attention_weight_min": -24.709989547729492,
      "activations/layer13_attention_weight_max": 39.12055587768555,
      "activations/layer13_attention_weight_min": -32.12885665893555,
      "activations/layer14_attention_weight_max": 41.786136627197266,
      "activations/layer14_attention_weight_min": -32.20600128173828,
      "activations/layer15_attention_weight_max": 37.74618911743164,
      "activations/layer15_attention_weight_min": -30.186738967895508,
      "activations/layer16_attention_weight_max": 32.438751220703125,
      "activations/layer16_attention_weight_min": -27.150047302246094,
      "activations/layer17_attention_weight_max": 51.99699783325195,
      "activations/layer17_attention_weight_min": -42.46029281616211,
      "activations/layer18_attention_weight_max": 47.99938201904297,
      "activations/layer18_attention_weight_min": -38.01934814453125,
      "activations/layer19_attention_weight_max": 24.887802124023438,
      "activations/layer19_attention_weight_min": -21.262939453125,
      "activations/layer1_attention_weight_max": 17.378450393676758,
      "activations/layer1_attention_weight_min": -15.083145141601562,
      "activations/layer20_attention_weight_max": 22.70323371887207,
      "activations/layer20_attention_weight_min": -22.160024642944336,
      "activations/layer21_attention_weight_max": 38.31692886352539,
      "activations/layer21_attention_weight_min": -25.523143768310547,
      "activations/layer22_attention_weight_max": 33.296817779541016,
      "activations/layer22_attention_weight_min": -25.3282527923584,
      "activations/layer23_attention_weight_max": 33.80049133300781,
      "activations/layer23_attention_weight_min": -21.161766052246094,
      "activations/layer2_attention_weight_max": 30.976394653320312,
      "activations/layer2_attention_weight_min": -28.83356475830078,
      "activations/layer3_attention_weight_max": 88.95214080810547,
      "activations/layer3_attention_weight_min": -90.50096130371094,
      "activations/layer4_attention_weight_max": 101.19401550292969,
      "activations/layer4_attention_weight_min": -98.4769515991211,
      "activations/layer5_attention_weight_max": 80.57572174072266,
      "activations/layer5_attention_weight_min": -79.1100082397461,
      "activations/layer6_attention_weight_max": 54.964881896972656,
      "activations/layer6_attention_weight_min": -52.48060607910156,
      "activations/layer7_attention_weight_max": 80.00440216064453,
      "activations/layer7_attention_weight_min": -79.30879211425781,
      "activations/layer8_attention_weight_max": 46.02089309692383,
      "activations/layer8_attention_weight_min": -46.25286865234375,
      "activations/layer9_attention_weight_max": 52.9348258972168,
      "activations/layer9_attention_weight_min": -52.1102180480957,
      "epoch": 8.79,
      "learning_rate": 9.42935606060606e-05,
      "loss": 2.808,
      "step": 151200
    },
    {
      "activations/layer0_attention_weight_max": 14.692133903503418,
      "activations/layer0_attention_weight_min": -13.623819351196289,
      "activations/layer10_attention_weight_max": 33.83281707763672,
      "activations/layer10_attention_weight_min": -33.802284240722656,
      "activations/layer11_attention_weight_max": 31.018545150756836,
      "activations/layer11_attention_weight_min": -29.86844253540039,
      "activations/layer12_attention_weight_max": 18.141666412353516,
      "activations/layer12_attention_weight_min": -31.397640228271484,
      "activations/layer13_attention_weight_max": 47.615455627441406,
      "activations/layer13_attention_weight_min": -30.947601318359375,
      "activations/layer14_attention_weight_max": 42.14493942260742,
      "activations/layer14_attention_weight_min": -32.35698318481445,
      "activations/layer15_attention_weight_max": 35.834224700927734,
      "activations/layer15_attention_weight_min": -30.871109008789062,
      "activations/layer16_attention_weight_max": 34.19706344604492,
      "activations/layer16_attention_weight_min": -30.139673233032227,
      "activations/layer17_attention_weight_max": 55.244102478027344,
      "activations/layer17_attention_weight_min": -46.24116897583008,
      "activations/layer18_attention_weight_max": 47.094703674316406,
      "activations/layer18_attention_weight_min": -37.078487396240234,
      "activations/layer19_attention_weight_max": 26.540908813476562,
      "activations/layer19_attention_weight_min": -21.453628540039062,
      "activations/layer1_attention_weight_max": 17.160463333129883,
      "activations/layer1_attention_weight_min": -13.596620559692383,
      "activations/layer20_attention_weight_max": 23.07648277282715,
      "activations/layer20_attention_weight_min": -19.805864334106445,
      "activations/layer21_attention_weight_max": 40.3051643371582,
      "activations/layer21_attention_weight_min": -21.99485206604004,
      "activations/layer22_attention_weight_max": 32.24030303955078,
      "activations/layer22_attention_weight_min": -23.77745246887207,
      "activations/layer23_attention_weight_max": 34.8968505859375,
      "activations/layer23_attention_weight_min": -22.442874908447266,
      "activations/layer2_attention_weight_max": 32.01548385620117,
      "activations/layer2_attention_weight_min": -29.623945236206055,
      "activations/layer3_attention_weight_max": 94.92747497558594,
      "activations/layer3_attention_weight_min": -92.83491516113281,
      "activations/layer4_attention_weight_max": 102.0276870727539,
      "activations/layer4_attention_weight_min": -91.91950988769531,
      "activations/layer5_attention_weight_max": 77.41402435302734,
      "activations/layer5_attention_weight_min": -74.63327026367188,
      "activations/layer6_attention_weight_max": 54.04996871948242,
      "activations/layer6_attention_weight_min": -50.46475601196289,
      "activations/layer7_attention_weight_max": 77.41593170166016,
      "activations/layer7_attention_weight_min": -71.53431701660156,
      "activations/layer8_attention_weight_max": 42.73563766479492,
      "activations/layer8_attention_weight_min": -45.52413558959961,
      "activations/layer9_attention_weight_max": 55.99479293823242,
      "activations/layer9_attention_weight_min": -46.76611328125,
      "epoch": 8.79,
      "learning_rate": 9.42746212121212e-05,
      "loss": 2.784,
      "step": 151250
    },
    {
      "activations/layer0_attention_weight_max": 14.702546119689941,
      "activations/layer0_attention_weight_min": -13.90597915649414,
      "activations/layer10_attention_weight_max": 38.06902313232422,
      "activations/layer10_attention_weight_min": -34.28269958496094,
      "activations/layer11_attention_weight_max": 32.356014251708984,
      "activations/layer11_attention_weight_min": -30.33772850036621,
      "activations/layer12_attention_weight_max": 17.443187713623047,
      "activations/layer12_attention_weight_min": -25.4258975982666,
      "activations/layer13_attention_weight_max": 37.544654846191406,
      "activations/layer13_attention_weight_min": -28.968997955322266,
      "activations/layer14_attention_weight_max": 38.29448699951172,
      "activations/layer14_attention_weight_min": -28.685789108276367,
      "activations/layer15_attention_weight_max": 35.18341827392578,
      "activations/layer15_attention_weight_min": -27.80344009399414,
      "activations/layer16_attention_weight_max": 31.4810791015625,
      "activations/layer16_attention_weight_min": -27.444011688232422,
      "activations/layer17_attention_weight_max": 49.7001953125,
      "activations/layer17_attention_weight_min": -40.980464935302734,
      "activations/layer18_attention_weight_max": 44.10015106201172,
      "activations/layer18_attention_weight_min": -36.229583740234375,
      "activations/layer19_attention_weight_max": 20.40989112854004,
      "activations/layer19_attention_weight_min": -20.155019760131836,
      "activations/layer1_attention_weight_max": 17.465959548950195,
      "activations/layer1_attention_weight_min": -15.021855354309082,
      "activations/layer20_attention_weight_max": 20.527591705322266,
      "activations/layer20_attention_weight_min": -21.95576286315918,
      "activations/layer21_attention_weight_max": 35.9206657409668,
      "activations/layer21_attention_weight_min": -22.191802978515625,
      "activations/layer22_attention_weight_max": 29.725095748901367,
      "activations/layer22_attention_weight_min": -25.73273468017578,
      "activations/layer23_attention_weight_max": 34.096588134765625,
      "activations/layer23_attention_weight_min": -24.551738739013672,
      "activations/layer2_attention_weight_max": 30.019378662109375,
      "activations/layer2_attention_weight_min": -29.42642593383789,
      "activations/layer3_attention_weight_max": 91.03703308105469,
      "activations/layer3_attention_weight_min": -91.44043731689453,
      "activations/layer4_attention_weight_max": 96.0340576171875,
      "activations/layer4_attention_weight_min": -91.72444152832031,
      "activations/layer5_attention_weight_max": 77.65995788574219,
      "activations/layer5_attention_weight_min": -77.46082305908203,
      "activations/layer6_attention_weight_max": 52.84151077270508,
      "activations/layer6_attention_weight_min": -49.742061614990234,
      "activations/layer7_attention_weight_max": 76.4527816772461,
      "activations/layer7_attention_weight_min": -71.55939483642578,
      "activations/layer8_attention_weight_max": 46.5206298828125,
      "activations/layer8_attention_weight_min": -43.30397033691406,
      "activations/layer9_attention_weight_max": 51.24562454223633,
      "activations/layer9_attention_weight_min": -47.28423309326172,
      "epoch": 8.79,
      "learning_rate": 9.425568181818181e-05,
      "loss": 2.8148,
      "step": 151300
    },
    {
      "activations/layer0_attention_weight_max": 15.25191593170166,
      "activations/layer0_attention_weight_min": -14.28481388092041,
      "activations/layer10_attention_weight_max": 39.195587158203125,
      "activations/layer10_attention_weight_min": -35.50797653198242,
      "activations/layer11_attention_weight_max": 33.363704681396484,
      "activations/layer11_attention_weight_min": -30.863292694091797,
      "activations/layer12_attention_weight_max": 20.91822052001953,
      "activations/layer12_attention_weight_min": -26.681446075439453,
      "activations/layer13_attention_weight_max": 42.76078796386719,
      "activations/layer13_attention_weight_min": -32.434104919433594,
      "activations/layer14_attention_weight_max": 43.0838737487793,
      "activations/layer14_attention_weight_min": -31.766315460205078,
      "activations/layer15_attention_weight_max": 37.027198791503906,
      "activations/layer15_attention_weight_min": -31.683507919311523,
      "activations/layer16_attention_weight_max": 30.626476287841797,
      "activations/layer16_attention_weight_min": -30.466495513916016,
      "activations/layer17_attention_weight_max": 55.09175491333008,
      "activations/layer17_attention_weight_min": -44.38264846801758,
      "activations/layer18_attention_weight_max": 50.67959976196289,
      "activations/layer18_attention_weight_min": -37.366180419921875,
      "activations/layer19_attention_weight_max": 21.704275131225586,
      "activations/layer19_attention_weight_min": -22.905553817749023,
      "activations/layer1_attention_weight_max": 16.23879623413086,
      "activations/layer1_attention_weight_min": -14.252588272094727,
      "activations/layer20_attention_weight_max": 21.578563690185547,
      "activations/layer20_attention_weight_min": -24.36334228515625,
      "activations/layer21_attention_weight_max": 34.5460090637207,
      "activations/layer21_attention_weight_min": -23.85496711730957,
      "activations/layer22_attention_weight_max": 30.1407470703125,
      "activations/layer22_attention_weight_min": -26.67822265625,
      "activations/layer23_attention_weight_max": 38.05281066894531,
      "activations/layer23_attention_weight_min": -26.378599166870117,
      "activations/layer2_attention_weight_max": 30.98241424560547,
      "activations/layer2_attention_weight_min": -30.179540634155273,
      "activations/layer3_attention_weight_max": 89.16903686523438,
      "activations/layer3_attention_weight_min": -92.51078796386719,
      "activations/layer4_attention_weight_max": 98.73148345947266,
      "activations/layer4_attention_weight_min": -98.85140991210938,
      "activations/layer5_attention_weight_max": 80.80850219726562,
      "activations/layer5_attention_weight_min": -77.13644409179688,
      "activations/layer6_attention_weight_max": 59.141197204589844,
      "activations/layer6_attention_weight_min": -52.88296127319336,
      "activations/layer7_attention_weight_max": 75.71054077148438,
      "activations/layer7_attention_weight_min": -75.61173248291016,
      "activations/layer8_attention_weight_max": 44.56077194213867,
      "activations/layer8_attention_weight_min": -44.80900192260742,
      "activations/layer9_attention_weight_max": 50.420997619628906,
      "activations/layer9_attention_weight_min": -50.98124694824219,
      "epoch": 8.79,
      "learning_rate": 9.423674242424241e-05,
      "loss": 2.8087,
      "step": 151350
    },
    {
      "activations/layer0_attention_weight_max": 14.40961742401123,
      "activations/layer0_attention_weight_min": -13.671035766601562,
      "activations/layer10_attention_weight_max": 38.647125244140625,
      "activations/layer10_attention_weight_min": -35.015350341796875,
      "activations/layer11_attention_weight_max": 30.481794357299805,
      "activations/layer11_attention_weight_min": -29.469188690185547,
      "activations/layer12_attention_weight_max": 19.112089157104492,
      "activations/layer12_attention_weight_min": -22.526363372802734,
      "activations/layer13_attention_weight_max": 38.526641845703125,
      "activations/layer13_attention_weight_min": -30.229766845703125,
      "activations/layer14_attention_weight_max": 36.108795166015625,
      "activations/layer14_attention_weight_min": -29.20904541015625,
      "activations/layer15_attention_weight_max": 32.41484069824219,
      "activations/layer15_attention_weight_min": -28.998083114624023,
      "activations/layer16_attention_weight_max": 29.15505599975586,
      "activations/layer16_attention_weight_min": -26.426963806152344,
      "activations/layer17_attention_weight_max": 52.1476936340332,
      "activations/layer17_attention_weight_min": -41.067012786865234,
      "activations/layer18_attention_weight_max": 46.86761474609375,
      "activations/layer18_attention_weight_min": -36.291500091552734,
      "activations/layer19_attention_weight_max": 21.344667434692383,
      "activations/layer19_attention_weight_min": -21.88706398010254,
      "activations/layer1_attention_weight_max": 16.741783142089844,
      "activations/layer1_attention_weight_min": -13.847305297851562,
      "activations/layer20_attention_weight_max": 24.197608947753906,
      "activations/layer20_attention_weight_min": -21.41031837463379,
      "activations/layer21_attention_weight_max": 37.13195037841797,
      "activations/layer21_attention_weight_min": -23.189725875854492,
      "activations/layer22_attention_weight_max": 30.627296447753906,
      "activations/layer22_attention_weight_min": -24.509201049804688,
      "activations/layer23_attention_weight_max": 36.559051513671875,
      "activations/layer23_attention_weight_min": -22.344886779785156,
      "activations/layer2_attention_weight_max": 30.382038116455078,
      "activations/layer2_attention_weight_min": -29.138973236083984,
      "activations/layer3_attention_weight_max": 84.57215881347656,
      "activations/layer3_attention_weight_min": -87.1853256225586,
      "activations/layer4_attention_weight_max": 99.8735580444336,
      "activations/layer4_attention_weight_min": -96.00879669189453,
      "activations/layer5_attention_weight_max": 79.5372314453125,
      "activations/layer5_attention_weight_min": -77.99699401855469,
      "activations/layer6_attention_weight_max": 55.327857971191406,
      "activations/layer6_attention_weight_min": -52.72185516357422,
      "activations/layer7_attention_weight_max": 73.94166564941406,
      "activations/layer7_attention_weight_min": -74.4200668334961,
      "activations/layer8_attention_weight_max": 44.607398986816406,
      "activations/layer8_attention_weight_min": -45.0181770324707,
      "activations/layer9_attention_weight_max": 46.764198303222656,
      "activations/layer9_attention_weight_min": -48.19210433959961,
      "epoch": 8.8,
      "learning_rate": 9.421780303030303e-05,
      "loss": 2.8152,
      "step": 151400
    },
    {
      "activations/layer0_attention_weight_max": 14.165191650390625,
      "activations/layer0_attention_weight_min": -13.629695892333984,
      "activations/layer10_attention_weight_max": 35.894527435302734,
      "activations/layer10_attention_weight_min": -34.495689392089844,
      "activations/layer11_attention_weight_max": 32.883888244628906,
      "activations/layer11_attention_weight_min": -30.46164321899414,
      "activations/layer12_attention_weight_max": 22.89527130126953,
      "activations/layer12_attention_weight_min": -24.948078155517578,
      "activations/layer13_attention_weight_max": 45.189273834228516,
      "activations/layer13_attention_weight_min": -33.44398498535156,
      "activations/layer14_attention_weight_max": 38.42632293701172,
      "activations/layer14_attention_weight_min": -30.280258178710938,
      "activations/layer15_attention_weight_max": 35.317832946777344,
      "activations/layer15_attention_weight_min": -27.874530792236328,
      "activations/layer16_attention_weight_max": 32.808536529541016,
      "activations/layer16_attention_weight_min": -28.59052848815918,
      "activations/layer17_attention_weight_max": 55.78251647949219,
      "activations/layer17_attention_weight_min": -41.53457260131836,
      "activations/layer18_attention_weight_max": 49.83110809326172,
      "activations/layer18_attention_weight_min": -34.34838104248047,
      "activations/layer19_attention_weight_max": 22.63977813720703,
      "activations/layer19_attention_weight_min": -22.371017456054688,
      "activations/layer1_attention_weight_max": 17.81540870666504,
      "activations/layer1_attention_weight_min": -14.739431381225586,
      "activations/layer20_attention_weight_max": 24.961647033691406,
      "activations/layer20_attention_weight_min": -28.3241024017334,
      "activations/layer21_attention_weight_max": 37.51503372192383,
      "activations/layer21_attention_weight_min": -21.982479095458984,
      "activations/layer22_attention_weight_max": 36.416255950927734,
      "activations/layer22_attention_weight_min": -24.8255615234375,
      "activations/layer23_attention_weight_max": 45.90140914916992,
      "activations/layer23_attention_weight_min": -23.603635787963867,
      "activations/layer2_attention_weight_max": 29.811201095581055,
      "activations/layer2_attention_weight_min": -29.800764083862305,
      "activations/layer3_attention_weight_max": 86.21068572998047,
      "activations/layer3_attention_weight_min": -88.66105651855469,
      "activations/layer4_attention_weight_max": 96.72747039794922,
      "activations/layer4_attention_weight_min": -92.7265396118164,
      "activations/layer5_attention_weight_max": 80.27582550048828,
      "activations/layer5_attention_weight_min": -79.46830749511719,
      "activations/layer6_attention_weight_max": 52.80693054199219,
      "activations/layer6_attention_weight_min": -52.04981231689453,
      "activations/layer7_attention_weight_max": 75.7166519165039,
      "activations/layer7_attention_weight_min": -72.9613037109375,
      "activations/layer8_attention_weight_max": 44.11469268798828,
      "activations/layer8_attention_weight_min": -45.13523483276367,
      "activations/layer9_attention_weight_max": 56.015235900878906,
      "activations/layer9_attention_weight_min": -52.1472053527832,
      "epoch": 8.8,
      "learning_rate": 9.419886363636363e-05,
      "loss": 2.804,
      "step": 151450
    },
    {
      "activations/layer0_attention_weight_max": 17.35361671447754,
      "activations/layer0_attention_weight_min": -13.558587074279785,
      "activations/layer10_attention_weight_max": 36.9008674621582,
      "activations/layer10_attention_weight_min": -38.62474060058594,
      "activations/layer11_attention_weight_max": 30.895978927612305,
      "activations/layer11_attention_weight_min": -31.901939392089844,
      "activations/layer12_attention_weight_max": 19.905927658081055,
      "activations/layer12_attention_weight_min": -24.60993194580078,
      "activations/layer13_attention_weight_max": 41.67841720581055,
      "activations/layer13_attention_weight_min": -29.7342529296875,
      "activations/layer14_attention_weight_max": 36.874874114990234,
      "activations/layer14_attention_weight_min": -31.713512420654297,
      "activations/layer15_attention_weight_max": 35.096126556396484,
      "activations/layer15_attention_weight_min": -28.352176666259766,
      "activations/layer16_attention_weight_max": 38.45915985107422,
      "activations/layer16_attention_weight_min": -29.3543758392334,
      "activations/layer17_attention_weight_max": 54.49298095703125,
      "activations/layer17_attention_weight_min": -43.83163070678711,
      "activations/layer18_attention_weight_max": 47.81285858154297,
      "activations/layer18_attention_weight_min": -36.89990997314453,
      "activations/layer19_attention_weight_max": 21.413311004638672,
      "activations/layer19_attention_weight_min": -20.803579330444336,
      "activations/layer1_attention_weight_max": 16.48858070373535,
      "activations/layer1_attention_weight_min": -14.465639114379883,
      "activations/layer20_attention_weight_max": 22.54446792602539,
      "activations/layer20_attention_weight_min": -19.839031219482422,
      "activations/layer21_attention_weight_max": 36.838134765625,
      "activations/layer21_attention_weight_min": -23.317142486572266,
      "activations/layer22_attention_weight_max": 32.94486618041992,
      "activations/layer22_attention_weight_min": -26.266111373901367,
      "activations/layer23_attention_weight_max": 38.041500091552734,
      "activations/layer23_attention_weight_min": -22.089004516601562,
      "activations/layer2_attention_weight_max": 29.68712615966797,
      "activations/layer2_attention_weight_min": -29.34930419921875,
      "activations/layer3_attention_weight_max": 88.9447250366211,
      "activations/layer3_attention_weight_min": -86.85529327392578,
      "activations/layer4_attention_weight_max": 102.22989654541016,
      "activations/layer4_attention_weight_min": -95.79750061035156,
      "activations/layer5_attention_weight_max": 79.9166030883789,
      "activations/layer5_attention_weight_min": -77.66751861572266,
      "activations/layer6_attention_weight_max": 54.77339553833008,
      "activations/layer6_attention_weight_min": -53.618202209472656,
      "activations/layer7_attention_weight_max": 78.855712890625,
      "activations/layer7_attention_weight_min": -74.1547622680664,
      "activations/layer8_attention_weight_max": 44.47536849975586,
      "activations/layer8_attention_weight_min": -45.18612289428711,
      "activations/layer9_attention_weight_max": 50.75747299194336,
      "activations/layer9_attention_weight_min": -47.54941940307617,
      "epoch": 8.8,
      "learning_rate": 9.417992424242423e-05,
      "loss": 2.8096,
      "step": 151500
    },
    {
      "activations/layer0_attention_weight_max": 17.017995834350586,
      "activations/layer0_attention_weight_min": -13.59268856048584,
      "activations/layer10_attention_weight_max": 41.17352294921875,
      "activations/layer10_attention_weight_min": -36.804779052734375,
      "activations/layer11_attention_weight_max": 32.08207321166992,
      "activations/layer11_attention_weight_min": -31.184188842773438,
      "activations/layer12_attention_weight_max": 30.969829559326172,
      "activations/layer12_attention_weight_min": -30.77168083190918,
      "activations/layer13_attention_weight_max": 40.93540954589844,
      "activations/layer13_attention_weight_min": -28.946128845214844,
      "activations/layer14_attention_weight_max": 38.4603385925293,
      "activations/layer14_attention_weight_min": -29.252925872802734,
      "activations/layer15_attention_weight_max": 34.463951110839844,
      "activations/layer15_attention_weight_min": -31.044702529907227,
      "activations/layer16_attention_weight_max": 32.591678619384766,
      "activations/layer16_attention_weight_min": -28.068729400634766,
      "activations/layer17_attention_weight_max": 52.62592315673828,
      "activations/layer17_attention_weight_min": -41.093013763427734,
      "activations/layer18_attention_weight_max": 48.15465545654297,
      "activations/layer18_attention_weight_min": -35.093379974365234,
      "activations/layer19_attention_weight_max": 20.946826934814453,
      "activations/layer19_attention_weight_min": -21.148319244384766,
      "activations/layer1_attention_weight_max": 18.283811569213867,
      "activations/layer1_attention_weight_min": -15.405909538269043,
      "activations/layer20_attention_weight_max": 22.240198135375977,
      "activations/layer20_attention_weight_min": -20.445043563842773,
      "activations/layer21_attention_weight_max": 40.644264221191406,
      "activations/layer21_attention_weight_min": -23.38779640197754,
      "activations/layer22_attention_weight_max": 31.845619201660156,
      "activations/layer22_attention_weight_min": -22.690855026245117,
      "activations/layer23_attention_weight_max": 35.91004943847656,
      "activations/layer23_attention_weight_min": -23.65201187133789,
      "activations/layer2_attention_weight_max": 32.906455993652344,
      "activations/layer2_attention_weight_min": -31.330211639404297,
      "activations/layer3_attention_weight_max": 92.65027618408203,
      "activations/layer3_attention_weight_min": -94.54762268066406,
      "activations/layer4_attention_weight_max": 101.67537689208984,
      "activations/layer4_attention_weight_min": -100.15489959716797,
      "activations/layer5_attention_weight_max": 79.44889068603516,
      "activations/layer5_attention_weight_min": -78.990478515625,
      "activations/layer6_attention_weight_max": 55.14440155029297,
      "activations/layer6_attention_weight_min": -53.87645721435547,
      "activations/layer7_attention_weight_max": 85.3386459350586,
      "activations/layer7_attention_weight_min": -74.96601104736328,
      "activations/layer8_attention_weight_max": 47.21084976196289,
      "activations/layer8_attention_weight_min": -46.770111083984375,
      "activations/layer9_attention_weight_max": 57.124305725097656,
      "activations/layer9_attention_weight_min": -47.91240692138672,
      "epoch": 8.81,
      "learning_rate": 9.416098484848485e-05,
      "loss": 2.7986,
      "step": 151550
    },
    {
      "activations/layer0_attention_weight_max": 14.94595718383789,
      "activations/layer0_attention_weight_min": -13.495258331298828,
      "activations/layer10_attention_weight_max": 36.36729049682617,
      "activations/layer10_attention_weight_min": -34.4881591796875,
      "activations/layer11_attention_weight_max": 30.642698287963867,
      "activations/layer11_attention_weight_min": -30.738065719604492,
      "activations/layer12_attention_weight_max": 19.774810791015625,
      "activations/layer12_attention_weight_min": -29.234678268432617,
      "activations/layer13_attention_weight_max": 48.75297927856445,
      "activations/layer13_attention_weight_min": -35.22479248046875,
      "activations/layer14_attention_weight_max": 42.2293586730957,
      "activations/layer14_attention_weight_min": -30.433696746826172,
      "activations/layer15_attention_weight_max": 34.108524322509766,
      "activations/layer15_attention_weight_min": -28.92898178100586,
      "activations/layer16_attention_weight_max": 36.90607452392578,
      "activations/layer16_attention_weight_min": -27.44247055053711,
      "activations/layer17_attention_weight_max": 51.01264572143555,
      "activations/layer17_attention_weight_min": -46.1343879699707,
      "activations/layer18_attention_weight_max": 43.8717155456543,
      "activations/layer18_attention_weight_min": -38.58013153076172,
      "activations/layer19_attention_weight_max": 21.703929901123047,
      "activations/layer19_attention_weight_min": -20.951858520507812,
      "activations/layer1_attention_weight_max": 16.97963523864746,
      "activations/layer1_attention_weight_min": -15.29360294342041,
      "activations/layer20_attention_weight_max": 21.471101760864258,
      "activations/layer20_attention_weight_min": -25.25155258178711,
      "activations/layer21_attention_weight_max": 35.01564407348633,
      "activations/layer21_attention_weight_min": -23.144939422607422,
      "activations/layer22_attention_weight_max": 31.05270767211914,
      "activations/layer22_attention_weight_min": -24.766632080078125,
      "activations/layer23_attention_weight_max": 40.75562286376953,
      "activations/layer23_attention_weight_min": -24.343902587890625,
      "activations/layer2_attention_weight_max": 29.819419860839844,
      "activations/layer2_attention_weight_min": -29.419300079345703,
      "activations/layer3_attention_weight_max": 86.79112243652344,
      "activations/layer3_attention_weight_min": -86.02738189697266,
      "activations/layer4_attention_weight_max": 98.67401123046875,
      "activations/layer4_attention_weight_min": -92.96459197998047,
      "activations/layer5_attention_weight_max": 77.44283294677734,
      "activations/layer5_attention_weight_min": -77.1783447265625,
      "activations/layer6_attention_weight_max": 54.03364944458008,
      "activations/layer6_attention_weight_min": -52.68252944946289,
      "activations/layer7_attention_weight_max": 79.6722412109375,
      "activations/layer7_attention_weight_min": -71.73624420166016,
      "activations/layer8_attention_weight_max": 44.34530258178711,
      "activations/layer8_attention_weight_min": -46.5068244934082,
      "activations/layer9_attention_weight_max": 52.58866500854492,
      "activations/layer9_attention_weight_min": -45.7713737487793,
      "epoch": 8.81,
      "learning_rate": 9.414204545454544e-05,
      "loss": 2.8048,
      "step": 151600
    },
    {
      "activations/layer0_attention_weight_max": 15.008186340332031,
      "activations/layer0_attention_weight_min": -13.993244171142578,
      "activations/layer10_attention_weight_max": 38.68913269042969,
      "activations/layer10_attention_weight_min": -35.4678955078125,
      "activations/layer11_attention_weight_max": 33.13026809692383,
      "activations/layer11_attention_weight_min": -31.867664337158203,
      "activations/layer12_attention_weight_max": 20.145185470581055,
      "activations/layer12_attention_weight_min": -22.447906494140625,
      "activations/layer13_attention_weight_max": 48.387699127197266,
      "activations/layer13_attention_weight_min": -35.11589813232422,
      "activations/layer14_attention_weight_max": 42.21371078491211,
      "activations/layer14_attention_weight_min": -32.707122802734375,
      "activations/layer15_attention_weight_max": 36.60905838012695,
      "activations/layer15_attention_weight_min": -31.890581130981445,
      "activations/layer16_attention_weight_max": 35.72880935668945,
      "activations/layer16_attention_weight_min": -31.079118728637695,
      "activations/layer17_attention_weight_max": 59.06161117553711,
      "activations/layer17_attention_weight_min": -49.670406341552734,
      "activations/layer18_attention_weight_max": 55.0888786315918,
      "activations/layer18_attention_weight_min": -42.07738494873047,
      "activations/layer19_attention_weight_max": 23.763713836669922,
      "activations/layer19_attention_weight_min": -23.437957763671875,
      "activations/layer1_attention_weight_max": 16.36745834350586,
      "activations/layer1_attention_weight_min": -16.711265563964844,
      "activations/layer20_attention_weight_max": 25.179216384887695,
      "activations/layer20_attention_weight_min": -21.001340866088867,
      "activations/layer21_attention_weight_max": 39.90382385253906,
      "activations/layer21_attention_weight_min": -24.490459442138672,
      "activations/layer22_attention_weight_max": 38.14667510986328,
      "activations/layer22_attention_weight_min": -25.950544357299805,
      "activations/layer23_attention_weight_max": 44.07207489013672,
      "activations/layer23_attention_weight_min": -23.697494506835938,
      "activations/layer2_attention_weight_max": 30.275569915771484,
      "activations/layer2_attention_weight_min": -28.804466247558594,
      "activations/layer3_attention_weight_max": 84.72477722167969,
      "activations/layer3_attention_weight_min": -88.44171142578125,
      "activations/layer4_attention_weight_max": 100.32763671875,
      "activations/layer4_attention_weight_min": -91.93021392822266,
      "activations/layer5_attention_weight_max": 80.2189712524414,
      "activations/layer5_attention_weight_min": -77.22306060791016,
      "activations/layer6_attention_weight_max": 54.04197311401367,
      "activations/layer6_attention_weight_min": -55.86616516113281,
      "activations/layer7_attention_weight_max": 77.01295471191406,
      "activations/layer7_attention_weight_min": -73.9552001953125,
      "activations/layer8_attention_weight_max": 45.18896484375,
      "activations/layer8_attention_weight_min": -45.39943313598633,
      "activations/layer9_attention_weight_max": 51.5594596862793,
      "activations/layer9_attention_weight_min": -54.23016357421875,
      "epoch": 8.81,
      "learning_rate": 9.412310606060605e-05,
      "loss": 2.7994,
      "step": 151650
    },
    {
      "activations/layer0_attention_weight_max": 14.729384422302246,
      "activations/layer0_attention_weight_min": -13.524894714355469,
      "activations/layer10_attention_weight_max": 36.96251678466797,
      "activations/layer10_attention_weight_min": -37.553470611572266,
      "activations/layer11_attention_weight_max": 31.85395622253418,
      "activations/layer11_attention_weight_min": -35.646488189697266,
      "activations/layer12_attention_weight_max": 18.563329696655273,
      "activations/layer12_attention_weight_min": -23.42965316772461,
      "activations/layer13_attention_weight_max": 40.01555252075195,
      "activations/layer13_attention_weight_min": -32.33610534667969,
      "activations/layer14_attention_weight_max": 39.97799301147461,
      "activations/layer14_attention_weight_min": -31.549150466918945,
      "activations/layer15_attention_weight_max": 34.425743103027344,
      "activations/layer15_attention_weight_min": -28.316118240356445,
      "activations/layer16_attention_weight_max": 30.646743774414062,
      "activations/layer16_attention_weight_min": -30.35437774658203,
      "activations/layer17_attention_weight_max": 54.111534118652344,
      "activations/layer17_attention_weight_min": -45.5369873046875,
      "activations/layer18_attention_weight_max": 47.61390686035156,
      "activations/layer18_attention_weight_min": -34.835445404052734,
      "activations/layer19_attention_weight_max": 21.878629684448242,
      "activations/layer19_attention_weight_min": -23.374797821044922,
      "activations/layer1_attention_weight_max": 17.743553161621094,
      "activations/layer1_attention_weight_min": -16.282594680786133,
      "activations/layer20_attention_weight_max": 22.918224334716797,
      "activations/layer20_attention_weight_min": -25.960336685180664,
      "activations/layer21_attention_weight_max": 36.12038803100586,
      "activations/layer21_attention_weight_min": -23.955106735229492,
      "activations/layer22_attention_weight_max": 31.147871017456055,
      "activations/layer22_attention_weight_min": -26.460126876831055,
      "activations/layer23_attention_weight_max": 36.21297073364258,
      "activations/layer23_attention_weight_min": -21.808441162109375,
      "activations/layer2_attention_weight_max": 31.990894317626953,
      "activations/layer2_attention_weight_min": -31.28522300720215,
      "activations/layer3_attention_weight_max": 89.41671752929688,
      "activations/layer3_attention_weight_min": -91.65855407714844,
      "activations/layer4_attention_weight_max": 106.74043273925781,
      "activations/layer4_attention_weight_min": -99.68071746826172,
      "activations/layer5_attention_weight_max": 84.99662780761719,
      "activations/layer5_attention_weight_min": -79.07389831542969,
      "activations/layer6_attention_weight_max": 56.256431579589844,
      "activations/layer6_attention_weight_min": -55.240264892578125,
      "activations/layer7_attention_weight_max": 78.39788818359375,
      "activations/layer7_attention_weight_min": -75.68765258789062,
      "activations/layer8_attention_weight_max": 47.768333435058594,
      "activations/layer8_attention_weight_min": -47.28049850463867,
      "activations/layer9_attention_weight_max": 51.55775833129883,
      "activations/layer9_attention_weight_min": -51.46287155151367,
      "epoch": 8.81,
      "learning_rate": 9.410416666666667e-05,
      "loss": 2.7992,
      "step": 151700
    },
    {
      "activations/layer0_attention_weight_max": 15.999825477600098,
      "activations/layer0_attention_weight_min": -13.446602821350098,
      "activations/layer10_attention_weight_max": 41.7569465637207,
      "activations/layer10_attention_weight_min": -42.17137908935547,
      "activations/layer11_attention_weight_max": 32.356536865234375,
      "activations/layer11_attention_weight_min": -32.60453796386719,
      "activations/layer12_attention_weight_max": 18.65277099609375,
      "activations/layer12_attention_weight_min": -24.880048751831055,
      "activations/layer13_attention_weight_max": 41.862876892089844,
      "activations/layer13_attention_weight_min": -27.901748657226562,
      "activations/layer14_attention_weight_max": 50.03135681152344,
      "activations/layer14_attention_weight_min": -32.676979064941406,
      "activations/layer15_attention_weight_max": 34.74557876586914,
      "activations/layer15_attention_weight_min": -29.58588218688965,
      "activations/layer16_attention_weight_max": 32.11958312988281,
      "activations/layer16_attention_weight_min": -28.92022705078125,
      "activations/layer17_attention_weight_max": 54.2675895690918,
      "activations/layer17_attention_weight_min": -47.094886779785156,
      "activations/layer18_attention_weight_max": 49.66585159301758,
      "activations/layer18_attention_weight_min": -40.63249969482422,
      "activations/layer19_attention_weight_max": 22.7572021484375,
      "activations/layer19_attention_weight_min": -22.627182006835938,
      "activations/layer1_attention_weight_max": 18.079713821411133,
      "activations/layer1_attention_weight_min": -15.865523338317871,
      "activations/layer20_attention_weight_max": 26.460819244384766,
      "activations/layer20_attention_weight_min": -21.829421997070312,
      "activations/layer21_attention_weight_max": 39.889549255371094,
      "activations/layer21_attention_weight_min": -24.68987464904785,
      "activations/layer22_attention_weight_max": 34.532691955566406,
      "activations/layer22_attention_weight_min": -27.357709884643555,
      "activations/layer23_attention_weight_max": 37.550437927246094,
      "activations/layer23_attention_weight_min": -23.031055450439453,
      "activations/layer2_attention_weight_max": 31.345596313476562,
      "activations/layer2_attention_weight_min": -31.09054183959961,
      "activations/layer3_attention_weight_max": 89.64569091796875,
      "activations/layer3_attention_weight_min": -93.4998779296875,
      "activations/layer4_attention_weight_max": 101.43633270263672,
      "activations/layer4_attention_weight_min": -99.03889465332031,
      "activations/layer5_attention_weight_max": 82.61492919921875,
      "activations/layer5_attention_weight_min": -76.42729187011719,
      "activations/layer6_attention_weight_max": 54.938438415527344,
      "activations/layer6_attention_weight_min": -54.32107925415039,
      "activations/layer7_attention_weight_max": 79.63052368164062,
      "activations/layer7_attention_weight_min": -78.70448303222656,
      "activations/layer8_attention_weight_max": 49.63161849975586,
      "activations/layer8_attention_weight_min": -47.9357795715332,
      "activations/layer9_attention_weight_max": 53.184566497802734,
      "activations/layer9_attention_weight_min": -51.26401901245117,
      "epoch": 8.82,
      "learning_rate": 9.408522727272725e-05,
      "loss": 2.7984,
      "step": 151750
    },
    {
      "activations/layer0_attention_weight_max": 15.551921844482422,
      "activations/layer0_attention_weight_min": -13.463430404663086,
      "activations/layer10_attention_weight_max": 45.56189727783203,
      "activations/layer10_attention_weight_min": -42.68877029418945,
      "activations/layer11_attention_weight_max": 38.149192810058594,
      "activations/layer11_attention_weight_min": -38.2436408996582,
      "activations/layer12_attention_weight_max": 20.1773681640625,
      "activations/layer12_attention_weight_min": -25.2742919921875,
      "activations/layer13_attention_weight_max": 42.76191711425781,
      "activations/layer13_attention_weight_min": -32.670345306396484,
      "activations/layer14_attention_weight_max": 35.9058837890625,
      "activations/layer14_attention_weight_min": -30.003849029541016,
      "activations/layer15_attention_weight_max": 33.45637893676758,
      "activations/layer15_attention_weight_min": -29.26323127746582,
      "activations/layer16_attention_weight_max": 33.21920394897461,
      "activations/layer16_attention_weight_min": -29.536439895629883,
      "activations/layer17_attention_weight_max": 55.48430252075195,
      "activations/layer17_attention_weight_min": -44.36152648925781,
      "activations/layer18_attention_weight_max": 47.56048583984375,
      "activations/layer18_attention_weight_min": -36.69430923461914,
      "activations/layer19_attention_weight_max": 21.66685676574707,
      "activations/layer19_attention_weight_min": -21.057708740234375,
      "activations/layer1_attention_weight_max": 17.006101608276367,
      "activations/layer1_attention_weight_min": -15.902542114257812,
      "activations/layer20_attention_weight_max": 23.50038719177246,
      "activations/layer20_attention_weight_min": -21.510501861572266,
      "activations/layer21_attention_weight_max": 40.848758697509766,
      "activations/layer21_attention_weight_min": -24.63872718811035,
      "activations/layer22_attention_weight_max": 34.35115051269531,
      "activations/layer22_attention_weight_min": -26.14202308654785,
      "activations/layer23_attention_weight_max": 33.78248596191406,
      "activations/layer23_attention_weight_min": -21.927534103393555,
      "activations/layer2_attention_weight_max": 32.4511604309082,
      "activations/layer2_attention_weight_min": -31.02460289001465,
      "activations/layer3_attention_weight_max": 94.427490234375,
      "activations/layer3_attention_weight_min": -92.06076049804688,
      "activations/layer4_attention_weight_max": 105.94266510009766,
      "activations/layer4_attention_weight_min": -106.423583984375,
      "activations/layer5_attention_weight_max": 85.06620788574219,
      "activations/layer5_attention_weight_min": -82.41348266601562,
      "activations/layer6_attention_weight_max": 56.9056510925293,
      "activations/layer6_attention_weight_min": -54.70994186401367,
      "activations/layer7_attention_weight_max": 84.6754150390625,
      "activations/layer7_attention_weight_min": -80.62483215332031,
      "activations/layer8_attention_weight_max": 49.73696517944336,
      "activations/layer8_attention_weight_min": -49.450157165527344,
      "activations/layer9_attention_weight_max": 60.284645080566406,
      "activations/layer9_attention_weight_min": -58.103919982910156,
      "epoch": 8.82,
      "learning_rate": 9.406628787878787e-05,
      "loss": 2.7979,
      "step": 151800
    },
    {
      "activations/layer0_attention_weight_max": 14.747729301452637,
      "activations/layer0_attention_weight_min": -13.665411949157715,
      "activations/layer10_attention_weight_max": 37.93816375732422,
      "activations/layer10_attention_weight_min": -37.20219039916992,
      "activations/layer11_attention_weight_max": 30.669559478759766,
      "activations/layer11_attention_weight_min": -31.3867244720459,
      "activations/layer12_attention_weight_max": 17.565628051757812,
      "activations/layer12_attention_weight_min": -23.631383895874023,
      "activations/layer13_attention_weight_max": 42.69863510131836,
      "activations/layer13_attention_weight_min": -34.30485534667969,
      "activations/layer14_attention_weight_max": 37.97417449951172,
      "activations/layer14_attention_weight_min": -32.30892562866211,
      "activations/layer15_attention_weight_max": 37.056541442871094,
      "activations/layer15_attention_weight_min": -30.747629165649414,
      "activations/layer16_attention_weight_max": 36.30020523071289,
      "activations/layer16_attention_weight_min": -30.879858016967773,
      "activations/layer17_attention_weight_max": 55.63396072387695,
      "activations/layer17_attention_weight_min": -43.26825714111328,
      "activations/layer18_attention_weight_max": 46.8372917175293,
      "activations/layer18_attention_weight_min": -38.558685302734375,
      "activations/layer19_attention_weight_max": 20.86872100830078,
      "activations/layer19_attention_weight_min": -23.47408103942871,
      "activations/layer1_attention_weight_max": 17.14678955078125,
      "activations/layer1_attention_weight_min": -14.535008430480957,
      "activations/layer20_attention_weight_max": 22.193105697631836,
      "activations/layer20_attention_weight_min": -21.786527633666992,
      "activations/layer21_attention_weight_max": 34.90898513793945,
      "activations/layer21_attention_weight_min": -23.98367691040039,
      "activations/layer22_attention_weight_max": 28.24057388305664,
      "activations/layer22_attention_weight_min": -23.89653205871582,
      "activations/layer23_attention_weight_max": 36.58512496948242,
      "activations/layer23_attention_weight_min": -23.47827911376953,
      "activations/layer2_attention_weight_max": 33.67258071899414,
      "activations/layer2_attention_weight_min": -29.58045196533203,
      "activations/layer3_attention_weight_max": 96.17567443847656,
      "activations/layer3_attention_weight_min": -95.39185333251953,
      "activations/layer4_attention_weight_max": 106.2844009399414,
      "activations/layer4_attention_weight_min": -99.06228637695312,
      "activations/layer5_attention_weight_max": 84.29388427734375,
      "activations/layer5_attention_weight_min": -81.86198425292969,
      "activations/layer6_attention_weight_max": 57.45964050292969,
      "activations/layer6_attention_weight_min": -56.25868225097656,
      "activations/layer7_attention_weight_max": 77.41554260253906,
      "activations/layer7_attention_weight_min": -80.7233657836914,
      "activations/layer8_attention_weight_max": 46.644508361816406,
      "activations/layer8_attention_weight_min": -48.11699295043945,
      "activations/layer9_attention_weight_max": 51.75129318237305,
      "activations/layer9_attention_weight_min": -52.41422653198242,
      "epoch": 8.82,
      "learning_rate": 9.404734848484848e-05,
      "loss": 2.8019,
      "step": 151850
    },
    {
      "activations/layer0_attention_weight_max": 15.4487943649292,
      "activations/layer0_attention_weight_min": -13.659342765808105,
      "activations/layer10_attention_weight_max": 37.166229248046875,
      "activations/layer10_attention_weight_min": -36.80459213256836,
      "activations/layer11_attention_weight_max": 31.346996307373047,
      "activations/layer11_attention_weight_min": -31.29947280883789,
      "activations/layer12_attention_weight_max": 20.44158363342285,
      "activations/layer12_attention_weight_min": -21.13406753540039,
      "activations/layer13_attention_weight_max": 35.32113265991211,
      "activations/layer13_attention_weight_min": -30.231170654296875,
      "activations/layer14_attention_weight_max": 35.087257385253906,
      "activations/layer14_attention_weight_min": -30.498695373535156,
      "activations/layer15_attention_weight_max": 30.784912109375,
      "activations/layer15_attention_weight_min": -31.077518463134766,
      "activations/layer16_attention_weight_max": 28.491497039794922,
      "activations/layer16_attention_weight_min": -26.995941162109375,
      "activations/layer17_attention_weight_max": 49.651912689208984,
      "activations/layer17_attention_weight_min": -42.09343338012695,
      "activations/layer18_attention_weight_max": 44.83949279785156,
      "activations/layer18_attention_weight_min": -37.25068283081055,
      "activations/layer19_attention_weight_max": 21.79030418395996,
      "activations/layer19_attention_weight_min": -21.43644905090332,
      "activations/layer1_attention_weight_max": 16.270315170288086,
      "activations/layer1_attention_weight_min": -15.162233352661133,
      "activations/layer20_attention_weight_max": 23.235544204711914,
      "activations/layer20_attention_weight_min": -27.50747299194336,
      "activations/layer21_attention_weight_max": 36.724853515625,
      "activations/layer21_attention_weight_min": -24.110477447509766,
      "activations/layer22_attention_weight_max": 32.730953216552734,
      "activations/layer22_attention_weight_min": -25.179935455322266,
      "activations/layer23_attention_weight_max": 36.07734680175781,
      "activations/layer23_attention_weight_min": -24.339397430419922,
      "activations/layer2_attention_weight_max": 32.94895935058594,
      "activations/layer2_attention_weight_min": -31.803518295288086,
      "activations/layer3_attention_weight_max": 91.16960144042969,
      "activations/layer3_attention_weight_min": -94.90702056884766,
      "activations/layer4_attention_weight_max": 97.60408782958984,
      "activations/layer4_attention_weight_min": -95.2383041381836,
      "activations/layer5_attention_weight_max": 77.63047790527344,
      "activations/layer5_attention_weight_min": -75.96571350097656,
      "activations/layer6_attention_weight_max": 54.43403244018555,
      "activations/layer6_attention_weight_min": -53.6894645690918,
      "activations/layer7_attention_weight_max": 79.42425537109375,
      "activations/layer7_attention_weight_min": -76.10430145263672,
      "activations/layer8_attention_weight_max": 43.7067985534668,
      "activations/layer8_attention_weight_min": -45.892887115478516,
      "activations/layer9_attention_weight_max": 46.68465805053711,
      "activations/layer9_attention_weight_min": -54.72124099731445,
      "epoch": 8.83,
      "learning_rate": 9.402840909090909e-05,
      "loss": 2.7979,
      "step": 151900
    },
    {
      "activations/layer0_attention_weight_max": 14.618566513061523,
      "activations/layer0_attention_weight_min": -13.202340126037598,
      "activations/layer10_attention_weight_max": 45.13374328613281,
      "activations/layer10_attention_weight_min": -42.90280532836914,
      "activations/layer11_attention_weight_max": 38.100990295410156,
      "activations/layer11_attention_weight_min": -32.531097412109375,
      "activations/layer12_attention_weight_max": 21.69862174987793,
      "activations/layer12_attention_weight_min": -25.39105224609375,
      "activations/layer13_attention_weight_max": 57.05327224731445,
      "activations/layer13_attention_weight_min": -34.82277297973633,
      "activations/layer14_attention_weight_max": 45.1436767578125,
      "activations/layer14_attention_weight_min": -31.57787322998047,
      "activations/layer15_attention_weight_max": 44.643821716308594,
      "activations/layer15_attention_weight_min": -32.47138214111328,
      "activations/layer16_attention_weight_max": 32.48966598510742,
      "activations/layer16_attention_weight_min": -28.385528564453125,
      "activations/layer17_attention_weight_max": 56.1429557800293,
      "activations/layer17_attention_weight_min": -47.83537292480469,
      "activations/layer18_attention_weight_max": 46.61767578125,
      "activations/layer18_attention_weight_min": -39.15835189819336,
      "activations/layer19_attention_weight_max": 22.264326095581055,
      "activations/layer19_attention_weight_min": -21.360069274902344,
      "activations/layer1_attention_weight_max": 18.053667068481445,
      "activations/layer1_attention_weight_min": -16.430580139160156,
      "activations/layer20_attention_weight_max": 24.265968322753906,
      "activations/layer20_attention_weight_min": -22.42007064819336,
      "activations/layer21_attention_weight_max": 39.02759552001953,
      "activations/layer21_attention_weight_min": -23.587244033813477,
      "activations/layer22_attention_weight_max": 35.48097229003906,
      "activations/layer22_attention_weight_min": -26.496458053588867,
      "activations/layer23_attention_weight_max": 39.455230712890625,
      "activations/layer23_attention_weight_min": -22.992992401123047,
      "activations/layer2_attention_weight_max": 31.59925651550293,
      "activations/layer2_attention_weight_min": -32.255367279052734,
      "activations/layer3_attention_weight_max": 85.40916442871094,
      "activations/layer3_attention_weight_min": -92.06718444824219,
      "activations/layer4_attention_weight_max": 99.91151428222656,
      "activations/layer4_attention_weight_min": -98.09110260009766,
      "activations/layer5_attention_weight_max": 80.53569030761719,
      "activations/layer5_attention_weight_min": -78.07164001464844,
      "activations/layer6_attention_weight_max": 55.13299560546875,
      "activations/layer6_attention_weight_min": -53.54582595825195,
      "activations/layer7_attention_weight_max": 80.73651885986328,
      "activations/layer7_attention_weight_min": -74.13606262207031,
      "activations/layer8_attention_weight_max": 53.03129196166992,
      "activations/layer8_attention_weight_min": -51.48381805419922,
      "activations/layer9_attention_weight_max": 61.79539108276367,
      "activations/layer9_attention_weight_min": -51.657875061035156,
      "epoch": 8.83,
      "learning_rate": 9.400946969696969e-05,
      "loss": 2.7834,
      "step": 151950
    },
    {
      "activations/layer0_attention_weight_max": 14.478304862976074,
      "activations/layer0_attention_weight_min": -13.135990142822266,
      "activations/layer10_attention_weight_max": 33.832088470458984,
      "activations/layer10_attention_weight_min": -33.17716979980469,
      "activations/layer11_attention_weight_max": 29.902013778686523,
      "activations/layer11_attention_weight_min": -29.926061630249023,
      "activations/layer12_attention_weight_max": 21.564960479736328,
      "activations/layer12_attention_weight_min": -25.77952766418457,
      "activations/layer13_attention_weight_max": 42.69007110595703,
      "activations/layer13_attention_weight_min": -32.41025924682617,
      "activations/layer14_attention_weight_max": 37.033241271972656,
      "activations/layer14_attention_weight_min": -33.149539947509766,
      "activations/layer15_attention_weight_max": 35.25343704223633,
      "activations/layer15_attention_weight_min": -29.569313049316406,
      "activations/layer16_attention_weight_max": 32.55514907836914,
      "activations/layer16_attention_weight_min": -28.584766387939453,
      "activations/layer17_attention_weight_max": 52.95338821411133,
      "activations/layer17_attention_weight_min": -44.24287796020508,
      "activations/layer18_attention_weight_max": 48.445892333984375,
      "activations/layer18_attention_weight_min": -36.74090576171875,
      "activations/layer19_attention_weight_max": 24.965993881225586,
      "activations/layer19_attention_weight_min": -22.540061950683594,
      "activations/layer1_attention_weight_max": 17.274816513061523,
      "activations/layer1_attention_weight_min": -17.21431541442871,
      "activations/layer20_attention_weight_max": 24.058635711669922,
      "activations/layer20_attention_weight_min": -23.228912353515625,
      "activations/layer21_attention_weight_max": 48.37845993041992,
      "activations/layer21_attention_weight_min": -27.73947525024414,
      "activations/layer22_attention_weight_max": 33.48069381713867,
      "activations/layer22_attention_weight_min": -25.68077850341797,
      "activations/layer23_attention_weight_max": 37.53586196899414,
      "activations/layer23_attention_weight_min": -23.68975067138672,
      "activations/layer2_attention_weight_max": 30.59100341796875,
      "activations/layer2_attention_weight_min": -29.53559112548828,
      "activations/layer3_attention_weight_max": 85.23236083984375,
      "activations/layer3_attention_weight_min": -87.65121459960938,
      "activations/layer4_attention_weight_max": 94.35535430908203,
      "activations/layer4_attention_weight_min": -87.6894760131836,
      "activations/layer5_attention_weight_max": 79.3416748046875,
      "activations/layer5_attention_weight_min": -74.13638305664062,
      "activations/layer6_attention_weight_max": 53.69253158569336,
      "activations/layer6_attention_weight_min": -51.5478630065918,
      "activations/layer7_attention_weight_max": 74.27436065673828,
      "activations/layer7_attention_weight_min": -71.9237060546875,
      "activations/layer8_attention_weight_max": 44.78687286376953,
      "activations/layer8_attention_weight_min": -44.016700744628906,
      "activations/layer9_attention_weight_max": 46.886898040771484,
      "activations/layer9_attention_weight_min": -46.30834197998047,
      "epoch": 8.83,
      "learning_rate": 9.399053030303029e-05,
      "loss": 2.8005,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.5257,
      "eval_samples_per_second": 503.654,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.5257,
      "eval_openwebtext_samples_per_second": 503.654,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_wikitext_loss": 2.994140625,
      "eval_wikitext_ppl": 19.96819234893607,
      "eval_wikitext_runtime": 1.9769,
      "eval_wikitext_samples_per_second": 230.66,
      "step": 152000
    },
    {
      "epoch": 8.83,
      "eval_lambada_loss": 2.755859375,
      "eval_lambada_ppl": 15.734556979186685,
      "eval_lambada_runtime": 9.5935,
      "eval_lambada_samples_per_second": 507.53,
      "step": 152000
    },
    {
      "activations/layer0_attention_weight_max": 15.167202949523926,
      "activations/layer0_attention_weight_min": -12.848104476928711,
      "activations/layer10_attention_weight_max": 37.57086944580078,
      "activations/layer10_attention_weight_min": -37.117923736572266,
      "activations/layer11_attention_weight_max": 32.6268310546875,
      "activations/layer11_attention_weight_min": -32.85528564453125,
      "activations/layer12_attention_weight_max": 21.098621368408203,
      "activations/layer12_attention_weight_min": -25.36651611328125,
      "activations/layer13_attention_weight_max": 41.28031921386719,
      "activations/layer13_attention_weight_min": -29.927066802978516,
      "activations/layer14_attention_weight_max": 39.018898010253906,
      "activations/layer14_attention_weight_min": -29.916032791137695,
      "activations/layer15_attention_weight_max": 35.98817825317383,
      "activations/layer15_attention_weight_min": -30.23484230041504,
      "activations/layer16_attention_weight_max": 35.05771255493164,
      "activations/layer16_attention_weight_min": -27.362924575805664,
      "activations/layer17_attention_weight_max": 60.34083557128906,
      "activations/layer17_attention_weight_min": -44.9098014831543,
      "activations/layer18_attention_weight_max": 54.949867248535156,
      "activations/layer18_attention_weight_min": -38.945125579833984,
      "activations/layer19_attention_weight_max": 21.690053939819336,
      "activations/layer19_attention_weight_min": -21.42591667175293,
      "activations/layer1_attention_weight_max": 17.754854202270508,
      "activations/layer1_attention_weight_min": -14.505465507507324,
      "activations/layer20_attention_weight_max": 24.989543914794922,
      "activations/layer20_attention_weight_min": -20.36124038696289,
      "activations/layer21_attention_weight_max": 46.80537033081055,
      "activations/layer21_attention_weight_min": -24.044464111328125,
      "activations/layer22_attention_weight_max": 38.878562927246094,
      "activations/layer22_attention_weight_min": -25.33802604675293,
      "activations/layer23_attention_weight_max": 42.26123809814453,
      "activations/layer23_attention_weight_min": -21.60239028930664,
      "activations/layer2_attention_weight_max": 30.52108383178711,
      "activations/layer2_attention_weight_min": -30.68910026550293,
      "activations/layer3_attention_weight_max": 87.95460510253906,
      "activations/layer3_attention_weight_min": -93.79500579833984,
      "activations/layer4_attention_weight_max": 102.9287109375,
      "activations/layer4_attention_weight_min": -95.85485076904297,
      "activations/layer5_attention_weight_max": 81.5274658203125,
      "activations/layer5_attention_weight_min": -79.38339233398438,
      "activations/layer6_attention_weight_max": 56.26964569091797,
      "activations/layer6_attention_weight_min": -53.58620834350586,
      "activations/layer7_attention_weight_max": 82.33910369873047,
      "activations/layer7_attention_weight_min": -77.22850799560547,
      "activations/layer8_attention_weight_max": 49.1750373840332,
      "activations/layer8_attention_weight_min": -48.721004486083984,
      "activations/layer9_attention_weight_max": 52.985897064208984,
      "activations/layer9_attention_weight_min": -52.866024017333984,
      "epoch": 8.83,
      "learning_rate": 9.39715909090909e-05,
      "loss": 2.799,
      "step": 152050
    },
    {
      "activations/layer0_attention_weight_max": 14.720516204833984,
      "activations/layer0_attention_weight_min": -13.207119941711426,
      "activations/layer10_attention_weight_max": 36.18710708618164,
      "activations/layer10_attention_weight_min": -36.18445587158203,
      "activations/layer11_attention_weight_max": 32.25567626953125,
      "activations/layer11_attention_weight_min": -32.96812438964844,
      "activations/layer12_attention_weight_max": 17.60628890991211,
      "activations/layer12_attention_weight_min": -28.688045501708984,
      "activations/layer13_attention_weight_max": 44.563785552978516,
      "activations/layer13_attention_weight_min": -36.42662048339844,
      "activations/layer14_attention_weight_max": 54.443416595458984,
      "activations/layer14_attention_weight_min": -40.08988952636719,
      "activations/layer15_attention_weight_max": 38.24491882324219,
      "activations/layer15_attention_weight_min": -31.286102294921875,
      "activations/layer16_attention_weight_max": 33.49161148071289,
      "activations/layer16_attention_weight_min": -27.256790161132812,
      "activations/layer17_attention_weight_max": 56.18711471557617,
      "activations/layer17_attention_weight_min": -45.68983459472656,
      "activations/layer18_attention_weight_max": 53.62618637084961,
      "activations/layer18_attention_weight_min": -40.883975982666016,
      "activations/layer19_attention_weight_max": 22.948579788208008,
      "activations/layer19_attention_weight_min": -23.146526336669922,
      "activations/layer1_attention_weight_max": 16.585159301757812,
      "activations/layer1_attention_weight_min": -15.132204055786133,
      "activations/layer20_attention_weight_max": 27.760059356689453,
      "activations/layer20_attention_weight_min": -23.500953674316406,
      "activations/layer21_attention_weight_max": 45.21244430541992,
      "activations/layer21_attention_weight_min": -26.103927612304688,
      "activations/layer22_attention_weight_max": 30.82282829284668,
      "activations/layer22_attention_weight_min": -24.5064697265625,
      "activations/layer23_attention_weight_max": 36.85688781738281,
      "activations/layer23_attention_weight_min": -22.16774559020996,
      "activations/layer2_attention_weight_max": 31.849889755249023,
      "activations/layer2_attention_weight_min": -30.763870239257812,
      "activations/layer3_attention_weight_max": 93.22296142578125,
      "activations/layer3_attention_weight_min": -97.98321533203125,
      "activations/layer4_attention_weight_max": 99.6279525756836,
      "activations/layer4_attention_weight_min": -94.54593658447266,
      "activations/layer5_attention_weight_max": 79.37937927246094,
      "activations/layer5_attention_weight_min": -73.6854019165039,
      "activations/layer6_attention_weight_max": 53.27659606933594,
      "activations/layer6_attention_weight_min": -50.883113861083984,
      "activations/layer7_attention_weight_max": 78.39666748046875,
      "activations/layer7_attention_weight_min": -70.66397094726562,
      "activations/layer8_attention_weight_max": 44.8026008605957,
      "activations/layer8_attention_weight_min": -44.81343078613281,
      "activations/layer9_attention_weight_max": 48.173099517822266,
      "activations/layer9_attention_weight_min": -48.78517150878906,
      "epoch": 8.84,
      "learning_rate": 9.39526515151515e-05,
      "loss": 2.8088,
      "step": 152100
    },
    {
      "activations/layer0_attention_weight_max": 14.925248146057129,
      "activations/layer0_attention_weight_min": -13.318597793579102,
      "activations/layer10_attention_weight_max": 41.79533004760742,
      "activations/layer10_attention_weight_min": -40.35695266723633,
      "activations/layer11_attention_weight_max": 36.34138488769531,
      "activations/layer11_attention_weight_min": -34.35700988769531,
      "activations/layer12_attention_weight_max": 18.44308853149414,
      "activations/layer12_attention_weight_min": -26.466110229492188,
      "activations/layer13_attention_weight_max": 43.90797805786133,
      "activations/layer13_attention_weight_min": -32.38045883178711,
      "activations/layer14_attention_weight_max": 36.86558151245117,
      "activations/layer14_attention_weight_min": -33.0467414855957,
      "activations/layer15_attention_weight_max": 35.18727493286133,
      "activations/layer15_attention_weight_min": -31.564550399780273,
      "activations/layer16_attention_weight_max": 36.399776458740234,
      "activations/layer16_attention_weight_min": -30.602970123291016,
      "activations/layer17_attention_weight_max": 57.22736358642578,
      "activations/layer17_attention_weight_min": -49.46791458129883,
      "activations/layer18_attention_weight_max": 50.37234878540039,
      "activations/layer18_attention_weight_min": -41.71247482299805,
      "activations/layer19_attention_weight_max": 25.712387084960938,
      "activations/layer19_attention_weight_min": -22.265798568725586,
      "activations/layer1_attention_weight_max": 16.0584774017334,
      "activations/layer1_attention_weight_min": -14.220778465270996,
      "activations/layer20_attention_weight_max": 24.410503387451172,
      "activations/layer20_attention_weight_min": -21.08909797668457,
      "activations/layer21_attention_weight_max": 46.19840621948242,
      "activations/layer21_attention_weight_min": -25.9051456451416,
      "activations/layer22_attention_weight_max": 34.872318267822266,
      "activations/layer22_attention_weight_min": -26.123899459838867,
      "activations/layer23_attention_weight_max": 42.910377502441406,
      "activations/layer23_attention_weight_min": -22.96845817565918,
      "activations/layer2_attention_weight_max": 32.064903259277344,
      "activations/layer2_attention_weight_min": -29.775012969970703,
      "activations/layer3_attention_weight_max": 99.61154174804688,
      "activations/layer3_attention_weight_min": -94.15309143066406,
      "activations/layer4_attention_weight_max": 112.84419250488281,
      "activations/layer4_attention_weight_min": -105.06996154785156,
      "activations/layer5_attention_weight_max": 89.67913055419922,
      "activations/layer5_attention_weight_min": -80.77229309082031,
      "activations/layer6_attention_weight_max": 57.84633255004883,
      "activations/layer6_attention_weight_min": -55.227134704589844,
      "activations/layer7_attention_weight_max": 80.58740234375,
      "activations/layer7_attention_weight_min": -80.74626159667969,
      "activations/layer8_attention_weight_max": 47.86980438232422,
      "activations/layer8_attention_weight_min": -47.66323471069336,
      "activations/layer9_attention_weight_max": 58.242835998535156,
      "activations/layer9_attention_weight_min": -55.04268264770508,
      "epoch": 8.84,
      "learning_rate": 9.39337121212121e-05,
      "loss": 2.8156,
      "step": 152150
    },
    {
      "activations/layer0_attention_weight_max": 14.668246269226074,
      "activations/layer0_attention_weight_min": -13.203145980834961,
      "activations/layer10_attention_weight_max": 45.55028533935547,
      "activations/layer10_attention_weight_min": -38.74513626098633,
      "activations/layer11_attention_weight_max": 41.58057403564453,
      "activations/layer11_attention_weight_min": -34.342567443847656,
      "activations/layer12_attention_weight_max": 17.6610050201416,
      "activations/layer12_attention_weight_min": -28.044906616210938,
      "activations/layer13_attention_weight_max": 46.64640426635742,
      "activations/layer13_attention_weight_min": -30.72075843811035,
      "activations/layer14_attention_weight_max": 54.69717788696289,
      "activations/layer14_attention_weight_min": -32.54508972167969,
      "activations/layer15_attention_weight_max": 44.625450134277344,
      "activations/layer15_attention_weight_min": -26.649370193481445,
      "activations/layer16_attention_weight_max": 30.357906341552734,
      "activations/layer16_attention_weight_min": -27.465694427490234,
      "activations/layer17_attention_weight_max": 51.72892379760742,
      "activations/layer17_attention_weight_min": -41.31200408935547,
      "activations/layer18_attention_weight_max": 46.76108932495117,
      "activations/layer18_attention_weight_min": -35.49668502807617,
      "activations/layer19_attention_weight_max": 20.458707809448242,
      "activations/layer19_attention_weight_min": -20.62253761291504,
      "activations/layer1_attention_weight_max": 17.648319244384766,
      "activations/layer1_attention_weight_min": -15.598257064819336,
      "activations/layer20_attention_weight_max": 20.38798713684082,
      "activations/layer20_attention_weight_min": -24.468486785888672,
      "activations/layer21_attention_weight_max": 35.3536491394043,
      "activations/layer21_attention_weight_min": -23.78141975402832,
      "activations/layer22_attention_weight_max": 33.387290954589844,
      "activations/layer22_attention_weight_min": -25.424421310424805,
      "activations/layer23_attention_weight_max": 35.254310607910156,
      "activations/layer23_attention_weight_min": -21.263174057006836,
      "activations/layer2_attention_weight_max": 33.667457580566406,
      "activations/layer2_attention_weight_min": -32.46695327758789,
      "activations/layer3_attention_weight_max": 94.99877166748047,
      "activations/layer3_attention_weight_min": -95.220947265625,
      "activations/layer4_attention_weight_max": 103.58597564697266,
      "activations/layer4_attention_weight_min": -96.20857238769531,
      "activations/layer5_attention_weight_max": 84.75752258300781,
      "activations/layer5_attention_weight_min": -76.80553436279297,
      "activations/layer6_attention_weight_max": 59.0048713684082,
      "activations/layer6_attention_weight_min": -57.55179214477539,
      "activations/layer7_attention_weight_max": 89.39291381835938,
      "activations/layer7_attention_weight_min": -82.217529296875,
      "activations/layer8_attention_weight_max": 59.50286865234375,
      "activations/layer8_attention_weight_min": -49.195350646972656,
      "activations/layer9_attention_weight_max": 71.67198944091797,
      "activations/layer9_attention_weight_min": -55.00080490112305,
      "epoch": 8.84,
      "learning_rate": 9.391477272727272e-05,
      "loss": 2.7835,
      "step": 152200
    },
    {
      "activations/layer0_attention_weight_max": 15.885688781738281,
      "activations/layer0_attention_weight_min": -13.594152450561523,
      "activations/layer10_attention_weight_max": 37.52854919433594,
      "activations/layer10_attention_weight_min": -35.6275634765625,
      "activations/layer11_attention_weight_max": 33.180908203125,
      "activations/layer11_attention_weight_min": -32.29230880737305,
      "activations/layer12_attention_weight_max": 18.218551635742188,
      "activations/layer12_attention_weight_min": -27.108095169067383,
      "activations/layer13_attention_weight_max": 49.45877456665039,
      "activations/layer13_attention_weight_min": -32.640411376953125,
      "activations/layer14_attention_weight_max": 41.55808639526367,
      "activations/layer14_attention_weight_min": -31.247129440307617,
      "activations/layer15_attention_weight_max": 36.52348327636719,
      "activations/layer15_attention_weight_min": -28.700485229492188,
      "activations/layer16_attention_weight_max": 31.95581817626953,
      "activations/layer16_attention_weight_min": -27.88714027404785,
      "activations/layer17_attention_weight_max": 56.214271545410156,
      "activations/layer17_attention_weight_min": -45.09373474121094,
      "activations/layer18_attention_weight_max": 48.05332946777344,
      "activations/layer18_attention_weight_min": -38.2024040222168,
      "activations/layer19_attention_weight_max": 22.351598739624023,
      "activations/layer19_attention_weight_min": -21.423744201660156,
      "activations/layer1_attention_weight_max": 17.614248275756836,
      "activations/layer1_attention_weight_min": -15.907686233520508,
      "activations/layer20_attention_weight_max": 22.940990447998047,
      "activations/layer20_attention_weight_min": -21.186738967895508,
      "activations/layer21_attention_weight_max": 38.40521240234375,
      "activations/layer21_attention_weight_min": -22.891149520874023,
      "activations/layer22_attention_weight_max": 33.260032653808594,
      "activations/layer22_attention_weight_min": -24.498600006103516,
      "activations/layer23_attention_weight_max": 38.11078643798828,
      "activations/layer23_attention_weight_min": -22.684019088745117,
      "activations/layer2_attention_weight_max": 32.34090805053711,
      "activations/layer2_attention_weight_min": -31.904464721679688,
      "activations/layer3_attention_weight_max": 87.89707946777344,
      "activations/layer3_attention_weight_min": -91.21253967285156,
      "activations/layer4_attention_weight_max": 105.74478912353516,
      "activations/layer4_attention_weight_min": -101.94470977783203,
      "activations/layer5_attention_weight_max": 84.15505981445312,
      "activations/layer5_attention_weight_min": -79.55815124511719,
      "activations/layer6_attention_weight_max": 54.834373474121094,
      "activations/layer6_attention_weight_min": -54.55458068847656,
      "activations/layer7_attention_weight_max": 78.6319351196289,
      "activations/layer7_attention_weight_min": -80.3929672241211,
      "activations/layer8_attention_weight_max": 48.06801223754883,
      "activations/layer8_attention_weight_min": -46.84749221801758,
      "activations/layer9_attention_weight_max": 55.447914123535156,
      "activations/layer9_attention_weight_min": -50.53089904785156,
      "epoch": 8.85,
      "learning_rate": 9.389583333333334e-05,
      "loss": 2.8009,
      "step": 152250
    },
    {
      "activations/layer0_attention_weight_max": 15.36992359161377,
      "activations/layer0_attention_weight_min": -13.570363998413086,
      "activations/layer10_attention_weight_max": 40.461055755615234,
      "activations/layer10_attention_weight_min": -37.75046920776367,
      "activations/layer11_attention_weight_max": 31.123441696166992,
      "activations/layer11_attention_weight_min": -31.011093139648438,
      "activations/layer12_attention_weight_max": 16.589065551757812,
      "activations/layer12_attention_weight_min": -24.332275390625,
      "activations/layer13_attention_weight_max": 37.118003845214844,
      "activations/layer13_attention_weight_min": -32.23314666748047,
      "activations/layer14_attention_weight_max": 35.63772964477539,
      "activations/layer14_attention_weight_min": -30.490537643432617,
      "activations/layer15_attention_weight_max": 32.76906204223633,
      "activations/layer15_attention_weight_min": -29.815216064453125,
      "activations/layer16_attention_weight_max": 28.56081199645996,
      "activations/layer16_attention_weight_min": -29.52305793762207,
      "activations/layer17_attention_weight_max": 50.25278091430664,
      "activations/layer17_attention_weight_min": -41.66015625,
      "activations/layer18_attention_weight_max": 41.107177734375,
      "activations/layer18_attention_weight_min": -36.54662322998047,
      "activations/layer19_attention_weight_max": 20.342632293701172,
      "activations/layer19_attention_weight_min": -20.480573654174805,
      "activations/layer1_attention_weight_max": 17.2738037109375,
      "activations/layer1_attention_weight_min": -15.316154479980469,
      "activations/layer20_attention_weight_max": 20.1698055267334,
      "activations/layer20_attention_weight_min": -22.686750411987305,
      "activations/layer21_attention_weight_max": 31.35868263244629,
      "activations/layer21_attention_weight_min": -21.721803665161133,
      "activations/layer22_attention_weight_max": 29.930660247802734,
      "activations/layer22_attention_weight_min": -26.77525520324707,
      "activations/layer23_attention_weight_max": 32.81660461425781,
      "activations/layer23_attention_weight_min": -22.559696197509766,
      "activations/layer2_attention_weight_max": 31.446666717529297,
      "activations/layer2_attention_weight_min": -31.953174591064453,
      "activations/layer3_attention_weight_max": 85.49932861328125,
      "activations/layer3_attention_weight_min": -94.68119812011719,
      "activations/layer4_attention_weight_max": 94.90608215332031,
      "activations/layer4_attention_weight_min": -99.1039047241211,
      "activations/layer5_attention_weight_max": 81.04377746582031,
      "activations/layer5_attention_weight_min": -78.39307403564453,
      "activations/layer6_attention_weight_max": 53.02985763549805,
      "activations/layer6_attention_weight_min": -52.9323844909668,
      "activations/layer7_attention_weight_max": 77.17066192626953,
      "activations/layer7_attention_weight_min": -70.44305419921875,
      "activations/layer8_attention_weight_max": 43.3338737487793,
      "activations/layer8_attention_weight_min": -47.34054183959961,
      "activations/layer9_attention_weight_max": 48.125118255615234,
      "activations/layer9_attention_weight_min": -50.06538009643555,
      "epoch": 8.85,
      "learning_rate": 9.387689393939392e-05,
      "loss": 2.8142,
      "step": 152300
    },
    {
      "activations/layer0_attention_weight_max": 17.433374404907227,
      "activations/layer0_attention_weight_min": -13.240520477294922,
      "activations/layer10_attention_weight_max": 38.82655334472656,
      "activations/layer10_attention_weight_min": -36.03388214111328,
      "activations/layer11_attention_weight_max": 31.70630645751953,
      "activations/layer11_attention_weight_min": -32.13078308105469,
      "activations/layer12_attention_weight_max": 18.885894775390625,
      "activations/layer12_attention_weight_min": -24.971847534179688,
      "activations/layer13_attention_weight_max": 39.81129837036133,
      "activations/layer13_attention_weight_min": -31.828697204589844,
      "activations/layer14_attention_weight_max": 38.78739929199219,
      "activations/layer14_attention_weight_min": -28.619455337524414,
      "activations/layer15_attention_weight_max": 34.638893127441406,
      "activations/layer15_attention_weight_min": -28.612958908081055,
      "activations/layer16_attention_weight_max": 32.179054260253906,
      "activations/layer16_attention_weight_min": -29.8156681060791,
      "activations/layer17_attention_weight_max": 57.93383026123047,
      "activations/layer17_attention_weight_min": -46.19095993041992,
      "activations/layer18_attention_weight_max": 48.03232192993164,
      "activations/layer18_attention_weight_min": -35.56931686401367,
      "activations/layer19_attention_weight_max": 20.733739852905273,
      "activations/layer19_attention_weight_min": -20.704084396362305,
      "activations/layer1_attention_weight_max": 18.618717193603516,
      "activations/layer1_attention_weight_min": -16.22394371032715,
      "activations/layer20_attention_weight_max": 22.322025299072266,
      "activations/layer20_attention_weight_min": -20.908103942871094,
      "activations/layer21_attention_weight_max": 34.74223709106445,
      "activations/layer21_attention_weight_min": -24.78354835510254,
      "activations/layer22_attention_weight_max": 31.804622650146484,
      "activations/layer22_attention_weight_min": -26.632001876831055,
      "activations/layer23_attention_weight_max": 37.303897857666016,
      "activations/layer23_attention_weight_min": -24.228443145751953,
      "activations/layer2_attention_weight_max": 31.066463470458984,
      "activations/layer2_attention_weight_min": -28.379194259643555,
      "activations/layer3_attention_weight_max": 91.37125396728516,
      "activations/layer3_attention_weight_min": -89.66371154785156,
      "activations/layer4_attention_weight_max": 102.0331802368164,
      "activations/layer4_attention_weight_min": -98.04265594482422,
      "activations/layer5_attention_weight_max": 81.21049499511719,
      "activations/layer5_attention_weight_min": -78.36407470703125,
      "activations/layer6_attention_weight_max": 57.21856689453125,
      "activations/layer6_attention_weight_min": -54.178348541259766,
      "activations/layer7_attention_weight_max": 79.60833740234375,
      "activations/layer7_attention_weight_min": -77.43756866455078,
      "activations/layer8_attention_weight_max": 47.0451774597168,
      "activations/layer8_attention_weight_min": -49.97640609741211,
      "activations/layer9_attention_weight_max": 48.44670867919922,
      "activations/layer9_attention_weight_min": -55.341468811035156,
      "epoch": 8.85,
      "learning_rate": 9.385795454545454e-05,
      "loss": 2.8009,
      "step": 152350
    },
    {
      "activations/layer0_attention_weight_max": 14.880141258239746,
      "activations/layer0_attention_weight_min": -13.545693397521973,
      "activations/layer10_attention_weight_max": 35.7508430480957,
      "activations/layer10_attention_weight_min": -33.48276138305664,
      "activations/layer11_attention_weight_max": 30.012372970581055,
      "activations/layer11_attention_weight_min": -30.227230072021484,
      "activations/layer12_attention_weight_max": 16.85318946838379,
      "activations/layer12_attention_weight_min": -24.86965560913086,
      "activations/layer13_attention_weight_max": 36.49775695800781,
      "activations/layer13_attention_weight_min": -28.290151596069336,
      "activations/layer14_attention_weight_max": 37.28308868408203,
      "activations/layer14_attention_weight_min": -31.5697021484375,
      "activations/layer15_attention_weight_max": 33.16191864013672,
      "activations/layer15_attention_weight_min": -29.55415916442871,
      "activations/layer16_attention_weight_max": 29.936723709106445,
      "activations/layer16_attention_weight_min": -28.32632827758789,
      "activations/layer17_attention_weight_max": 51.64895248413086,
      "activations/layer17_attention_weight_min": -43.20905685424805,
      "activations/layer18_attention_weight_max": 47.898963928222656,
      "activations/layer18_attention_weight_min": -37.45051193237305,
      "activations/layer19_attention_weight_max": 20.89851188659668,
      "activations/layer19_attention_weight_min": -20.027856826782227,
      "activations/layer1_attention_weight_max": 17.963455200195312,
      "activations/layer1_attention_weight_min": -14.953155517578125,
      "activations/layer20_attention_weight_max": 23.583152770996094,
      "activations/layer20_attention_weight_min": -22.80162239074707,
      "activations/layer21_attention_weight_max": 36.739410400390625,
      "activations/layer21_attention_weight_min": -24.43975830078125,
      "activations/layer22_attention_weight_max": 31.753416061401367,
      "activations/layer22_attention_weight_min": -24.62306022644043,
      "activations/layer23_attention_weight_max": 36.797218322753906,
      "activations/layer23_attention_weight_min": -20.906909942626953,
      "activations/layer2_attention_weight_max": 35.006736755371094,
      "activations/layer2_attention_weight_min": -33.043357849121094,
      "activations/layer3_attention_weight_max": 90.77494049072266,
      "activations/layer3_attention_weight_min": -97.05147552490234,
      "activations/layer4_attention_weight_max": 101.29802703857422,
      "activations/layer4_attention_weight_min": -94.64744567871094,
      "activations/layer5_attention_weight_max": 79.2567367553711,
      "activations/layer5_attention_weight_min": -74.67501831054688,
      "activations/layer6_attention_weight_max": 56.04515075683594,
      "activations/layer6_attention_weight_min": -56.1334114074707,
      "activations/layer7_attention_weight_max": 74.93575286865234,
      "activations/layer7_attention_weight_min": -76.23491668701172,
      "activations/layer8_attention_weight_max": 43.331153869628906,
      "activations/layer8_attention_weight_min": -47.09450149536133,
      "activations/layer9_attention_weight_max": 51.443843841552734,
      "activations/layer9_attention_weight_min": -47.74543762207031,
      "epoch": 8.86,
      "learning_rate": 9.383939393939393e-05,
      "loss": 2.7951,
      "step": 152400
    },
    {
      "activations/layer0_attention_weight_max": 14.784645080566406,
      "activations/layer0_attention_weight_min": -13.324047088623047,
      "activations/layer10_attention_weight_max": 38.8486442565918,
      "activations/layer10_attention_weight_min": -35.195587158203125,
      "activations/layer11_attention_weight_max": 31.585302352905273,
      "activations/layer11_attention_weight_min": -31.803028106689453,
      "activations/layer12_attention_weight_max": 19.973676681518555,
      "activations/layer12_attention_weight_min": -24.702966690063477,
      "activations/layer13_attention_weight_max": 49.1266975402832,
      "activations/layer13_attention_weight_min": -31.709993362426758,
      "activations/layer14_attention_weight_max": 53.900142669677734,
      "activations/layer14_attention_weight_min": -33.24021911621094,
      "activations/layer15_attention_weight_max": 47.214515686035156,
      "activations/layer15_attention_weight_min": -32.14165115356445,
      "activations/layer16_attention_weight_max": 32.2374153137207,
      "activations/layer16_attention_weight_min": -29.27350425720215,
      "activations/layer17_attention_weight_max": 55.616939544677734,
      "activations/layer17_attention_weight_min": -43.704715728759766,
      "activations/layer18_attention_weight_max": 52.84968185424805,
      "activations/layer18_attention_weight_min": -38.854766845703125,
      "activations/layer19_attention_weight_max": 24.216411590576172,
      "activations/layer19_attention_weight_min": -21.73655128479004,
      "activations/layer1_attention_weight_max": 17.01848793029785,
      "activations/layer1_attention_weight_min": -15.812708854675293,
      "activations/layer20_attention_weight_max": 28.42228889465332,
      "activations/layer20_attention_weight_min": -20.83750343322754,
      "activations/layer21_attention_weight_max": 46.89125442504883,
      "activations/layer21_attention_weight_min": -24.1678524017334,
      "activations/layer22_attention_weight_max": 36.09452819824219,
      "activations/layer22_attention_weight_min": -25.186975479125977,
      "activations/layer23_attention_weight_max": 41.073211669921875,
      "activations/layer23_attention_weight_min": -24.262248992919922,
      "activations/layer2_attention_weight_max": 31.984882354736328,
      "activations/layer2_attention_weight_min": -30.261404037475586,
      "activations/layer3_attention_weight_max": 87.75331115722656,
      "activations/layer3_attention_weight_min": -87.78865051269531,
      "activations/layer4_attention_weight_max": 96.5901870727539,
      "activations/layer4_attention_weight_min": -94.93922424316406,
      "activations/layer5_attention_weight_max": 78.36483001708984,
      "activations/layer5_attention_weight_min": -79.70487976074219,
      "activations/layer6_attention_weight_max": 56.056434631347656,
      "activations/layer6_attention_weight_min": -52.188663482666016,
      "activations/layer7_attention_weight_max": 76.30490112304688,
      "activations/layer7_attention_weight_min": -69.54557037353516,
      "activations/layer8_attention_weight_max": 45.08346939086914,
      "activations/layer8_attention_weight_min": -44.0276985168457,
      "activations/layer9_attention_weight_max": 48.000518798828125,
      "activations/layer9_attention_weight_min": -46.225006103515625,
      "epoch": 8.86,
      "learning_rate": 9.382045454545455e-05,
      "loss": 2.8011,
      "step": 152450
    },
    {
      "activations/layer0_attention_weight_max": 14.976316452026367,
      "activations/layer0_attention_weight_min": -13.388967514038086,
      "activations/layer10_attention_weight_max": 41.9782829284668,
      "activations/layer10_attention_weight_min": -37.03287124633789,
      "activations/layer11_attention_weight_max": 37.66750717163086,
      "activations/layer11_attention_weight_min": -32.92577362060547,
      "activations/layer12_attention_weight_max": 20.64374351501465,
      "activations/layer12_attention_weight_min": -23.457698822021484,
      "activations/layer13_attention_weight_max": 38.5791015625,
      "activations/layer13_attention_weight_min": -29.427745819091797,
      "activations/layer14_attention_weight_max": 37.524208068847656,
      "activations/layer14_attention_weight_min": -31.80356788635254,
      "activations/layer15_attention_weight_max": 33.15056228637695,
      "activations/layer15_attention_weight_min": -30.52248764038086,
      "activations/layer16_attention_weight_max": 30.639522552490234,
      "activations/layer16_attention_weight_min": -28.352415084838867,
      "activations/layer17_attention_weight_max": 55.416015625,
      "activations/layer17_attention_weight_min": -44.64809799194336,
      "activations/layer18_attention_weight_max": 50.0287971496582,
      "activations/layer18_attention_weight_min": -38.940528869628906,
      "activations/layer19_attention_weight_max": 24.632902145385742,
      "activations/layer19_attention_weight_min": -22.733625411987305,
      "activations/layer1_attention_weight_max": 16.89480209350586,
      "activations/layer1_attention_weight_min": -15.820427894592285,
      "activations/layer20_attention_weight_max": 24.533336639404297,
      "activations/layer20_attention_weight_min": -21.91130256652832,
      "activations/layer21_attention_weight_max": 44.3803596496582,
      "activations/layer21_attention_weight_min": -23.956497192382812,
      "activations/layer22_attention_weight_max": 31.94786834716797,
      "activations/layer22_attention_weight_min": -26.74566078186035,
      "activations/layer23_attention_weight_max": 39.868309020996094,
      "activations/layer23_attention_weight_min": -22.774864196777344,
      "activations/layer2_attention_weight_max": 33.5067138671875,
      "activations/layer2_attention_weight_min": -30.064908981323242,
      "activations/layer3_attention_weight_max": 92.03385925292969,
      "activations/layer3_attention_weight_min": -92.3876953125,
      "activations/layer4_attention_weight_max": 103.58556365966797,
      "activations/layer4_attention_weight_min": -96.08911895751953,
      "activations/layer5_attention_weight_max": 88.26274108886719,
      "activations/layer5_attention_weight_min": -80.68746948242188,
      "activations/layer6_attention_weight_max": 60.13203430175781,
      "activations/layer6_attention_weight_min": -53.77052307128906,
      "activations/layer7_attention_weight_max": 79.08745574951172,
      "activations/layer7_attention_weight_min": -79.04844665527344,
      "activations/layer8_attention_weight_max": 51.88199234008789,
      "activations/layer8_attention_weight_min": -49.44609832763672,
      "activations/layer9_attention_weight_max": 51.43587875366211,
      "activations/layer9_attention_weight_min": -52.99309158325195,
      "epoch": 8.86,
      "learning_rate": 9.380151515151513e-05,
      "loss": 2.7963,
      "step": 152500
    },
    {
      "activations/layer0_attention_weight_max": 14.707324028015137,
      "activations/layer0_attention_weight_min": -13.102109909057617,
      "activations/layer10_attention_weight_max": 40.52058029174805,
      "activations/layer10_attention_weight_min": -37.066619873046875,
      "activations/layer11_attention_weight_max": 32.33324432373047,
      "activations/layer11_attention_weight_min": -32.32916259765625,
      "activations/layer12_attention_weight_max": 23.647600173950195,
      "activations/layer12_attention_weight_min": -29.253149032592773,
      "activations/layer13_attention_weight_max": 47.715721130371094,
      "activations/layer13_attention_weight_min": -33.331398010253906,
      "activations/layer14_attention_weight_max": 39.67082214355469,
      "activations/layer14_attention_weight_min": -31.204280853271484,
      "activations/layer15_attention_weight_max": 40.07894515991211,
      "activations/layer15_attention_weight_min": -29.567825317382812,
      "activations/layer16_attention_weight_max": 31.269563674926758,
      "activations/layer16_attention_weight_min": -29.09019660949707,
      "activations/layer17_attention_weight_max": 54.405643463134766,
      "activations/layer17_attention_weight_min": -45.16421127319336,
      "activations/layer18_attention_weight_max": 47.08666229248047,
      "activations/layer18_attention_weight_min": -36.810550689697266,
      "activations/layer19_attention_weight_max": 24.23565673828125,
      "activations/layer19_attention_weight_min": -21.339725494384766,
      "activations/layer1_attention_weight_max": 17.58016014099121,
      "activations/layer1_attention_weight_min": -14.248023986816406,
      "activations/layer20_attention_weight_max": 25.50334358215332,
      "activations/layer20_attention_weight_min": -19.019546508789062,
      "activations/layer21_attention_weight_max": 37.242916107177734,
      "activations/layer21_attention_weight_min": -22.100622177124023,
      "activations/layer22_attention_weight_max": 34.68937301635742,
      "activations/layer22_attention_weight_min": -25.93537712097168,
      "activations/layer23_attention_weight_max": 37.50312042236328,
      "activations/layer23_attention_weight_min": -20.278160095214844,
      "activations/layer2_attention_weight_max": 33.92393112182617,
      "activations/layer2_attention_weight_min": -31.583974838256836,
      "activations/layer3_attention_weight_max": 90.60466766357422,
      "activations/layer3_attention_weight_min": -91.80196380615234,
      "activations/layer4_attention_weight_max": 99.86071014404297,
      "activations/layer4_attention_weight_min": -95.84773254394531,
      "activations/layer5_attention_weight_max": 84.74404907226562,
      "activations/layer5_attention_weight_min": -81.54396057128906,
      "activations/layer6_attention_weight_max": 57.68109893798828,
      "activations/layer6_attention_weight_min": -55.19450378417969,
      "activations/layer7_attention_weight_max": 78.21465301513672,
      "activations/layer7_attention_weight_min": -77.71357727050781,
      "activations/layer8_attention_weight_max": 47.45877456665039,
      "activations/layer8_attention_weight_min": -51.180152893066406,
      "activations/layer9_attention_weight_max": 56.843624114990234,
      "activations/layer9_attention_weight_min": -50.2137565612793,
      "epoch": 8.86,
      "learning_rate": 9.378257575757575e-05,
      "loss": 2.7961,
      "step": 152550
    },
    {
      "activations/layer0_attention_weight_max": 14.68077278137207,
      "activations/layer0_attention_weight_min": -13.575735092163086,
      "activations/layer10_attention_weight_max": 41.01183319091797,
      "activations/layer10_attention_weight_min": -35.878902435302734,
      "activations/layer11_attention_weight_max": 33.755577087402344,
      "activations/layer11_attention_weight_min": -33.169044494628906,
      "activations/layer12_attention_weight_max": 19.881425857543945,
      "activations/layer12_attention_weight_min": -26.71117401123047,
      "activations/layer13_attention_weight_max": 52.47203063964844,
      "activations/layer13_attention_weight_min": -35.92472839355469,
      "activations/layer14_attention_weight_max": 54.33692932128906,
      "activations/layer14_attention_weight_min": -30.961650848388672,
      "activations/layer15_attention_weight_max": 43.38785934448242,
      "activations/layer15_attention_weight_min": -30.99777603149414,
      "activations/layer16_attention_weight_max": 37.52825164794922,
      "activations/layer16_attention_weight_min": -31.832365036010742,
      "activations/layer17_attention_weight_max": 64.43545532226562,
      "activations/layer17_attention_weight_min": -46.94679260253906,
      "activations/layer18_attention_weight_max": 55.60546112060547,
      "activations/layer18_attention_weight_min": -39.32875442504883,
      "activations/layer19_attention_weight_max": 26.460172653198242,
      "activations/layer19_attention_weight_min": -21.40005874633789,
      "activations/layer1_attention_weight_max": 17.701448440551758,
      "activations/layer1_attention_weight_min": -15.864557266235352,
      "activations/layer20_attention_weight_max": 25.878938674926758,
      "activations/layer20_attention_weight_min": -21.419885635375977,
      "activations/layer21_attention_weight_max": 45.51481246948242,
      "activations/layer21_attention_weight_min": -25.301525115966797,
      "activations/layer22_attention_weight_max": 37.35813522338867,
      "activations/layer22_attention_weight_min": -26.952125549316406,
      "activations/layer23_attention_weight_max": 45.254486083984375,
      "activations/layer23_attention_weight_min": -26.69007110595703,
      "activations/layer2_attention_weight_max": 31.767589569091797,
      "activations/layer2_attention_weight_min": -31.047374725341797,
      "activations/layer3_attention_weight_max": 86.28056335449219,
      "activations/layer3_attention_weight_min": -90.0724868774414,
      "activations/layer4_attention_weight_max": 98.69658660888672,
      "activations/layer4_attention_weight_min": -95.93685150146484,
      "activations/layer5_attention_weight_max": 78.23158264160156,
      "activations/layer5_attention_weight_min": -77.48003387451172,
      "activations/layer6_attention_weight_max": 53.89494323730469,
      "activations/layer6_attention_weight_min": -52.40550231933594,
      "activations/layer7_attention_weight_max": 78.63385009765625,
      "activations/layer7_attention_weight_min": -77.16071319580078,
      "activations/layer8_attention_weight_max": 46.95785140991211,
      "activations/layer8_attention_weight_min": -48.038150787353516,
      "activations/layer9_attention_weight_max": 51.959571838378906,
      "activations/layer9_attention_weight_min": -48.37885665893555,
      "epoch": 8.87,
      "learning_rate": 9.376363636363636e-05,
      "loss": 2.7868,
      "step": 152600
    },
    {
      "activations/layer0_attention_weight_max": 15.377392768859863,
      "activations/layer0_attention_weight_min": -13.424151420593262,
      "activations/layer10_attention_weight_max": 37.584922790527344,
      "activations/layer10_attention_weight_min": -36.22468948364258,
      "activations/layer11_attention_weight_max": 30.640172958374023,
      "activations/layer11_attention_weight_min": -31.08142852783203,
      "activations/layer12_attention_weight_max": 20.25559425354004,
      "activations/layer12_attention_weight_min": -25.653419494628906,
      "activations/layer13_attention_weight_max": 40.178977966308594,
      "activations/layer13_attention_weight_min": -28.128875732421875,
      "activations/layer14_attention_weight_max": 37.605873107910156,
      "activations/layer14_attention_weight_min": -29.66408920288086,
      "activations/layer15_attention_weight_max": 32.37907028198242,
      "activations/layer15_attention_weight_min": -28.753389358520508,
      "activations/layer16_attention_weight_max": 32.95929718017578,
      "activations/layer16_attention_weight_min": -27.796831130981445,
      "activations/layer17_attention_weight_max": 51.92823028564453,
      "activations/layer17_attention_weight_min": -43.363555908203125,
      "activations/layer18_attention_weight_max": 47.960113525390625,
      "activations/layer18_attention_weight_min": -39.147491455078125,
      "activations/layer19_attention_weight_max": 24.255414962768555,
      "activations/layer19_attention_weight_min": -21.102205276489258,
      "activations/layer1_attention_weight_max": 17.666629791259766,
      "activations/layer1_attention_weight_min": -15.95516586303711,
      "activations/layer20_attention_weight_max": 25.042375564575195,
      "activations/layer20_attention_weight_min": -24.009946823120117,
      "activations/layer21_attention_weight_max": 37.42469024658203,
      "activations/layer21_attention_weight_min": -22.47280502319336,
      "activations/layer22_attention_weight_max": 34.85746383666992,
      "activations/layer22_attention_weight_min": -25.835121154785156,
      "activations/layer23_attention_weight_max": 42.81255340576172,
      "activations/layer23_attention_weight_min": -24.049522399902344,
      "activations/layer2_attention_weight_max": 31.381786346435547,
      "activations/layer2_attention_weight_min": -31.294042587280273,
      "activations/layer3_attention_weight_max": 91.04988098144531,
      "activations/layer3_attention_weight_min": -96.70714569091797,
      "activations/layer4_attention_weight_max": 102.12939453125,
      "activations/layer4_attention_weight_min": -106.69087982177734,
      "activations/layer5_attention_weight_max": 81.14723205566406,
      "activations/layer5_attention_weight_min": -81.51728820800781,
      "activations/layer6_attention_weight_max": 53.24435806274414,
      "activations/layer6_attention_weight_min": -55.24224090576172,
      "activations/layer7_attention_weight_max": 76.0255126953125,
      "activations/layer7_attention_weight_min": -75.32180786132812,
      "activations/layer8_attention_weight_max": 43.5662956237793,
      "activations/layer8_attention_weight_min": -44.11821365356445,
      "activations/layer9_attention_weight_max": 51.37846755981445,
      "activations/layer9_attention_weight_min": -49.688720703125,
      "epoch": 8.87,
      "learning_rate": 9.374469696969696e-05,
      "loss": 2.8055,
      "step": 152650
    },
    {
      "activations/layer0_attention_weight_max": 16.26901626586914,
      "activations/layer0_attention_weight_min": -13.617138862609863,
      "activations/layer10_attention_weight_max": 38.12471008300781,
      "activations/layer10_attention_weight_min": -35.534358978271484,
      "activations/layer11_attention_weight_max": 32.727745056152344,
      "activations/layer11_attention_weight_min": -32.227848052978516,
      "activations/layer12_attention_weight_max": 21.594764709472656,
      "activations/layer12_attention_weight_min": -26.51179313659668,
      "activations/layer13_attention_weight_max": 39.17658996582031,
      "activations/layer13_attention_weight_min": -29.244714736938477,
      "activations/layer14_attention_weight_max": 38.09866714477539,
      "activations/layer14_attention_weight_min": -29.507131576538086,
      "activations/layer15_attention_weight_max": 33.05887222290039,
      "activations/layer15_attention_weight_min": -29.313766479492188,
      "activations/layer16_attention_weight_max": 30.688846588134766,
      "activations/layer16_attention_weight_min": -29.741609573364258,
      "activations/layer17_attention_weight_max": 54.97768020629883,
      "activations/layer17_attention_weight_min": -42.87635040283203,
      "activations/layer18_attention_weight_max": 49.21440887451172,
      "activations/layer18_attention_weight_min": -36.48472213745117,
      "activations/layer19_attention_weight_max": 23.42601203918457,
      "activations/layer19_attention_weight_min": -21.094417572021484,
      "activations/layer1_attention_weight_max": 17.138776779174805,
      "activations/layer1_attention_weight_min": -15.697739601135254,
      "activations/layer20_attention_weight_max": 22.584678649902344,
      "activations/layer20_attention_weight_min": -21.85881233215332,
      "activations/layer21_attention_weight_max": 39.60247802734375,
      "activations/layer21_attention_weight_min": -24.19051170349121,
      "activations/layer22_attention_weight_max": 33.271976470947266,
      "activations/layer22_attention_weight_min": -26.35588264465332,
      "activations/layer23_attention_weight_max": 35.94853973388672,
      "activations/layer23_attention_weight_min": -24.77405548095703,
      "activations/layer2_attention_weight_max": 31.514265060424805,
      "activations/layer2_attention_weight_min": -31.396574020385742,
      "activations/layer3_attention_weight_max": 93.23595428466797,
      "activations/layer3_attention_weight_min": -98.3926010131836,
      "activations/layer4_attention_weight_max": 102.76131439208984,
      "activations/layer4_attention_weight_min": -101.25228118896484,
      "activations/layer5_attention_weight_max": 83.27960968017578,
      "activations/layer5_attention_weight_min": -76.72978210449219,
      "activations/layer6_attention_weight_max": 58.63836669921875,
      "activations/layer6_attention_weight_min": -54.717227935791016,
      "activations/layer7_attention_weight_max": 85.01699829101562,
      "activations/layer7_attention_weight_min": -72.17778778076172,
      "activations/layer8_attention_weight_max": 48.25264358520508,
      "activations/layer8_attention_weight_min": -47.01645278930664,
      "activations/layer9_attention_weight_max": 50.01809310913086,
      "activations/layer9_attention_weight_min": -48.93264389038086,
      "epoch": 8.87,
      "learning_rate": 9.372575757575757e-05,
      "loss": 2.7885,
      "step": 152700
    },
    {
      "activations/layer0_attention_weight_max": 15.560099601745605,
      "activations/layer0_attention_weight_min": -14.050029754638672,
      "activations/layer10_attention_weight_max": 38.54121780395508,
      "activations/layer10_attention_weight_min": -37.04091262817383,
      "activations/layer11_attention_weight_max": 32.46563720703125,
      "activations/layer11_attention_weight_min": -32.152435302734375,
      "activations/layer12_attention_weight_max": 21.685026168823242,
      "activations/layer12_attention_weight_min": -27.714195251464844,
      "activations/layer13_attention_weight_max": 42.16999053955078,
      "activations/layer13_attention_weight_min": -30.593923568725586,
      "activations/layer14_attention_weight_max": 43.79530334472656,
      "activations/layer14_attention_weight_min": -30.1937198638916,
      "activations/layer15_attention_weight_max": 37.55399703979492,
      "activations/layer15_attention_weight_min": -28.534435272216797,
      "activations/layer16_attention_weight_max": 34.65559387207031,
      "activations/layer16_attention_weight_min": -27.02234649658203,
      "activations/layer17_attention_weight_max": 60.58320999145508,
      "activations/layer17_attention_weight_min": -46.34339141845703,
      "activations/layer18_attention_weight_max": 48.896751403808594,
      "activations/layer18_attention_weight_min": -38.6099739074707,
      "activations/layer19_attention_weight_max": 24.232961654663086,
      "activations/layer19_attention_weight_min": -21.766029357910156,
      "activations/layer1_attention_weight_max": 18.247783660888672,
      "activations/layer1_attention_weight_min": -18.462142944335938,
      "activations/layer20_attention_weight_max": 28.065279006958008,
      "activations/layer20_attention_weight_min": -21.0003719329834,
      "activations/layer21_attention_weight_max": 41.86329650878906,
      "activations/layer21_attention_weight_min": -25.86073875427246,
      "activations/layer22_attention_weight_max": 37.9454231262207,
      "activations/layer22_attention_weight_min": -25.31305503845215,
      "activations/layer23_attention_weight_max": 35.94014358520508,
      "activations/layer23_attention_weight_min": -22.220632553100586,
      "activations/layer2_attention_weight_max": 32.64509582519531,
      "activations/layer2_attention_weight_min": -30.422466278076172,
      "activations/layer3_attention_weight_max": 92.81005096435547,
      "activations/layer3_attention_weight_min": -90.873291015625,
      "activations/layer4_attention_weight_max": 102.14717102050781,
      "activations/layer4_attention_weight_min": -99.16352844238281,
      "activations/layer5_attention_weight_max": 86.52754211425781,
      "activations/layer5_attention_weight_min": -77.45808410644531,
      "activations/layer6_attention_weight_max": 56.038536071777344,
      "activations/layer6_attention_weight_min": -53.534400939941406,
      "activations/layer7_attention_weight_max": 79.02339935302734,
      "activations/layer7_attention_weight_min": -76.2222900390625,
      "activations/layer8_attention_weight_max": 47.63902282714844,
      "activations/layer8_attention_weight_min": -47.17796325683594,
      "activations/layer9_attention_weight_max": 54.4913330078125,
      "activations/layer9_attention_weight_min": -50.48652267456055,
      "epoch": 8.88,
      "learning_rate": 9.370681818181817e-05,
      "loss": 2.8091,
      "step": 152750
    },
    {
      "activations/layer0_attention_weight_max": 16.19060516357422,
      "activations/layer0_attention_weight_min": -13.824227333068848,
      "activations/layer10_attention_weight_max": 43.94623565673828,
      "activations/layer10_attention_weight_min": -38.04899597167969,
      "activations/layer11_attention_weight_max": 34.22348403930664,
      "activations/layer11_attention_weight_min": -33.096656799316406,
      "activations/layer12_attention_weight_max": 17.201520919799805,
      "activations/layer12_attention_weight_min": -26.452335357666016,
      "activations/layer13_attention_weight_max": 39.529212951660156,
      "activations/layer13_attention_weight_min": -32.699398040771484,
      "activations/layer14_attention_weight_max": 39.63079071044922,
      "activations/layer14_attention_weight_min": -29.047197341918945,
      "activations/layer15_attention_weight_max": 36.77177429199219,
      "activations/layer15_attention_weight_min": -28.214801788330078,
      "activations/layer16_attention_weight_max": 29.652420043945312,
      "activations/layer16_attention_weight_min": -27.310096740722656,
      "activations/layer17_attention_weight_max": 50.21554183959961,
      "activations/layer17_attention_weight_min": -40.945255279541016,
      "activations/layer18_attention_weight_max": 47.917118072509766,
      "activations/layer18_attention_weight_min": -33.77614974975586,
      "activations/layer19_attention_weight_max": 21.329431533813477,
      "activations/layer19_attention_weight_min": -21.646554946899414,
      "activations/layer1_attention_weight_max": 16.1629695892334,
      "activations/layer1_attention_weight_min": -14.37837028503418,
      "activations/layer20_attention_weight_max": 20.887876510620117,
      "activations/layer20_attention_weight_min": -26.422151565551758,
      "activations/layer21_attention_weight_max": 38.5322265625,
      "activations/layer21_attention_weight_min": -27.081621170043945,
      "activations/layer22_attention_weight_max": 28.6937198638916,
      "activations/layer22_attention_weight_min": -25.52415657043457,
      "activations/layer23_attention_weight_max": 36.05253982543945,
      "activations/layer23_attention_weight_min": -24.611865997314453,
      "activations/layer2_attention_weight_max": 31.402990341186523,
      "activations/layer2_attention_weight_min": -29.622173309326172,
      "activations/layer3_attention_weight_max": 88.03376007080078,
      "activations/layer3_attention_weight_min": -93.5115737915039,
      "activations/layer4_attention_weight_max": 95.68595123291016,
      "activations/layer4_attention_weight_min": -92.89373016357422,
      "activations/layer5_attention_weight_max": 80.8079833984375,
      "activations/layer5_attention_weight_min": -81.04590606689453,
      "activations/layer6_attention_weight_max": 54.82090377807617,
      "activations/layer6_attention_weight_min": -51.41568374633789,
      "activations/layer7_attention_weight_max": 74.45292663574219,
      "activations/layer7_attention_weight_min": -76.45740509033203,
      "activations/layer8_attention_weight_max": 50.237613677978516,
      "activations/layer8_attention_weight_min": -48.07577896118164,
      "activations/layer9_attention_weight_max": 54.44011306762695,
      "activations/layer9_attention_weight_min": -49.07624053955078,
      "epoch": 8.88,
      "learning_rate": 9.368787878787878e-05,
      "loss": 2.8112,
      "step": 152800
    },
    {
      "activations/layer0_attention_weight_max": 15.645801544189453,
      "activations/layer0_attention_weight_min": -13.407392501831055,
      "activations/layer10_attention_weight_max": 37.25217819213867,
      "activations/layer10_attention_weight_min": -35.934059143066406,
      "activations/layer11_attention_weight_max": 34.07454299926758,
      "activations/layer11_attention_weight_min": -31.406578063964844,
      "activations/layer12_attention_weight_max": 18.39080238342285,
      "activations/layer12_attention_weight_min": -25.588232040405273,
      "activations/layer13_attention_weight_max": 52.017051696777344,
      "activations/layer13_attention_weight_min": -39.213130950927734,
      "activations/layer14_attention_weight_max": 40.92911911010742,
      "activations/layer14_attention_weight_min": -30.856834411621094,
      "activations/layer15_attention_weight_max": 33.015716552734375,
      "activations/layer15_attention_weight_min": -28.790307998657227,
      "activations/layer16_attention_weight_max": 30.988529205322266,
      "activations/layer16_attention_weight_min": -28.031803131103516,
      "activations/layer17_attention_weight_max": 53.36800003051758,
      "activations/layer17_attention_weight_min": -41.63722229003906,
      "activations/layer18_attention_weight_max": 46.380210876464844,
      "activations/layer18_attention_weight_min": -36.877220153808594,
      "activations/layer19_attention_weight_max": 20.120630264282227,
      "activations/layer19_attention_weight_min": -20.58997917175293,
      "activations/layer1_attention_weight_max": 17.395517349243164,
      "activations/layer1_attention_weight_min": -14.285425186157227,
      "activations/layer20_attention_weight_max": 22.935789108276367,
      "activations/layer20_attention_weight_min": -21.657169342041016,
      "activations/layer21_attention_weight_max": 37.4919548034668,
      "activations/layer21_attention_weight_min": -23.035633087158203,
      "activations/layer22_attention_weight_max": 36.10819625854492,
      "activations/layer22_attention_weight_min": -25.365436553955078,
      "activations/layer23_attention_weight_max": 38.924049377441406,
      "activations/layer23_attention_weight_min": -24.70107078552246,
      "activations/layer2_attention_weight_max": 32.57493209838867,
      "activations/layer2_attention_weight_min": -29.69814109802246,
      "activations/layer3_attention_weight_max": 92.11470794677734,
      "activations/layer3_attention_weight_min": -93.0709228515625,
      "activations/layer4_attention_weight_max": 103.5226058959961,
      "activations/layer4_attention_weight_min": -97.40731048583984,
      "activations/layer5_attention_weight_max": 87.06380462646484,
      "activations/layer5_attention_weight_min": -80.50072479248047,
      "activations/layer6_attention_weight_max": 60.15494918823242,
      "activations/layer6_attention_weight_min": -58.249610900878906,
      "activations/layer7_attention_weight_max": 80.55259704589844,
      "activations/layer7_attention_weight_min": -77.66146850585938,
      "activations/layer8_attention_weight_max": 46.89524459838867,
      "activations/layer8_attention_weight_min": -49.25099563598633,
      "activations/layer9_attention_weight_max": 55.261226654052734,
      "activations/layer9_attention_weight_min": -50.32944869995117,
      "epoch": 8.88,
      "learning_rate": 9.366893939393938e-05,
      "loss": 2.792,
      "step": 152850
    },
    {
      "activations/layer0_attention_weight_max": 16.384075164794922,
      "activations/layer0_attention_weight_min": -13.63056755065918,
      "activations/layer10_attention_weight_max": 39.29126739501953,
      "activations/layer10_attention_weight_min": -38.818359375,
      "activations/layer11_attention_weight_max": 32.512428283691406,
      "activations/layer11_attention_weight_min": -33.42243194580078,
      "activations/layer12_attention_weight_max": 22.763126373291016,
      "activations/layer12_attention_weight_min": -23.7957706451416,
      "activations/layer13_attention_weight_max": 51.29867935180664,
      "activations/layer13_attention_weight_min": -35.1143798828125,
      "activations/layer14_attention_weight_max": 42.455238342285156,
      "activations/layer14_attention_weight_min": -32.27274703979492,
      "activations/layer15_attention_weight_max": 34.41265869140625,
      "activations/layer15_attention_weight_min": -31.6143856048584,
      "activations/layer16_attention_weight_max": 34.18819046020508,
      "activations/layer16_attention_weight_min": -28.883121490478516,
      "activations/layer17_attention_weight_max": 55.22616958618164,
      "activations/layer17_attention_weight_min": -43.014404296875,
      "activations/layer18_attention_weight_max": 55.528865814208984,
      "activations/layer18_attention_weight_min": -37.80078887939453,
      "activations/layer19_attention_weight_max": 25.0333194732666,
      "activations/layer19_attention_weight_min": -22.028858184814453,
      "activations/layer1_attention_weight_max": 17.9770450592041,
      "activations/layer1_attention_weight_min": -14.821283340454102,
      "activations/layer20_attention_weight_max": 25.325420379638672,
      "activations/layer20_attention_weight_min": -23.20965003967285,
      "activations/layer21_attention_weight_max": 44.70096969604492,
      "activations/layer21_attention_weight_min": -23.787113189697266,
      "activations/layer22_attention_weight_max": 35.21358108520508,
      "activations/layer22_attention_weight_min": -25.02676010131836,
      "activations/layer23_attention_weight_max": 35.4995231628418,
      "activations/layer23_attention_weight_min": -21.143251419067383,
      "activations/layer2_attention_weight_max": 32.37997055053711,
      "activations/layer2_attention_weight_min": -31.287214279174805,
      "activations/layer3_attention_weight_max": 96.2645034790039,
      "activations/layer3_attention_weight_min": -97.14665985107422,
      "activations/layer4_attention_weight_max": 105.02828216552734,
      "activations/layer4_attention_weight_min": -99.99044036865234,
      "activations/layer5_attention_weight_max": 82.53746795654297,
      "activations/layer5_attention_weight_min": -80.9797134399414,
      "activations/layer6_attention_weight_max": 56.6284294128418,
      "activations/layer6_attention_weight_min": -54.43276596069336,
      "activations/layer7_attention_weight_max": 78.30424499511719,
      "activations/layer7_attention_weight_min": -74.43169403076172,
      "activations/layer8_attention_weight_max": 48.3266487121582,
      "activations/layer8_attention_weight_min": -48.11451721191406,
      "activations/layer9_attention_weight_max": 55.22553634643555,
      "activations/layer9_attention_weight_min": -52.68320846557617,
      "epoch": 8.88,
      "learning_rate": 9.364999999999999e-05,
      "loss": 2.7958,
      "step": 152900
    },
    {
      "activations/layer0_attention_weight_max": 15.107341766357422,
      "activations/layer0_attention_weight_min": -13.635293006896973,
      "activations/layer10_attention_weight_max": 41.748329162597656,
      "activations/layer10_attention_weight_min": -42.353240966796875,
      "activations/layer11_attention_weight_max": 35.68230438232422,
      "activations/layer11_attention_weight_min": -35.57790756225586,
      "activations/layer12_attention_weight_max": 19.769813537597656,
      "activations/layer12_attention_weight_min": -26.985315322875977,
      "activations/layer13_attention_weight_max": 43.65361785888672,
      "activations/layer13_attention_weight_min": -31.476463317871094,
      "activations/layer14_attention_weight_max": 58.1260871887207,
      "activations/layer14_attention_weight_min": -31.832353591918945,
      "activations/layer15_attention_weight_max": 48.70863723754883,
      "activations/layer15_attention_weight_min": -27.816741943359375,
      "activations/layer16_attention_weight_max": 32.35093307495117,
      "activations/layer16_attention_weight_min": -27.734548568725586,
      "activations/layer17_attention_weight_max": 53.52769088745117,
      "activations/layer17_attention_weight_min": -42.86423873901367,
      "activations/layer18_attention_weight_max": 48.77058029174805,
      "activations/layer18_attention_weight_min": -37.952301025390625,
      "activations/layer19_attention_weight_max": 22.59455680847168,
      "activations/layer19_attention_weight_min": -23.801753997802734,
      "activations/layer1_attention_weight_max": 18.21016502380371,
      "activations/layer1_attention_weight_min": -15.514165878295898,
      "activations/layer20_attention_weight_max": 24.759082794189453,
      "activations/layer20_attention_weight_min": -21.999671936035156,
      "activations/layer21_attention_weight_max": 41.98299026489258,
      "activations/layer21_attention_weight_min": -23.19827651977539,
      "activations/layer22_attention_weight_max": 31.22409439086914,
      "activations/layer22_attention_weight_min": -28.21550750732422,
      "activations/layer23_attention_weight_max": 37.848968505859375,
      "activations/layer23_attention_weight_min": -24.737403869628906,
      "activations/layer2_attention_weight_max": 32.720298767089844,
      "activations/layer2_attention_weight_min": -30.185827255249023,
      "activations/layer3_attention_weight_max": 93.8808364868164,
      "activations/layer3_attention_weight_min": -91.34825897216797,
      "activations/layer4_attention_weight_max": 104.87451171875,
      "activations/layer4_attention_weight_min": -95.80272674560547,
      "activations/layer5_attention_weight_max": 79.81519317626953,
      "activations/layer5_attention_weight_min": -75.0218734741211,
      "activations/layer6_attention_weight_max": 55.090152740478516,
      "activations/layer6_attention_weight_min": -52.46806716918945,
      "activations/layer7_attention_weight_max": 80.8138198852539,
      "activations/layer7_attention_weight_min": -72.04344940185547,
      "activations/layer8_attention_weight_max": 47.61494064331055,
      "activations/layer8_attention_weight_min": -50.45138168334961,
      "activations/layer9_attention_weight_max": 56.00691604614258,
      "activations/layer9_attention_weight_min": -55.13546371459961,
      "epoch": 8.89,
      "learning_rate": 9.36310606060606e-05,
      "loss": 2.7911,
      "step": 152950
    },
    {
      "activations/layer0_attention_weight_max": 16.72488784790039,
      "activations/layer0_attention_weight_min": -13.399471282958984,
      "activations/layer10_attention_weight_max": 46.94731521606445,
      "activations/layer10_attention_weight_min": -42.84819793701172,
      "activations/layer11_attention_weight_max": 41.86991882324219,
      "activations/layer11_attention_weight_min": -37.20486068725586,
      "activations/layer12_attention_weight_max": 22.51841163635254,
      "activations/layer12_attention_weight_min": -27.934709548950195,
      "activations/layer13_attention_weight_max": 46.30484390258789,
      "activations/layer13_attention_weight_min": -33.491546630859375,
      "activations/layer14_attention_weight_max": 47.958251953125,
      "activations/layer14_attention_weight_min": -33.4675407409668,
      "activations/layer15_attention_weight_max": 41.452171325683594,
      "activations/layer15_attention_weight_min": -28.48373031616211,
      "activations/layer16_attention_weight_max": 36.873207092285156,
      "activations/layer16_attention_weight_min": -30.811904907226562,
      "activations/layer17_attention_weight_max": 60.07701873779297,
      "activations/layer17_attention_weight_min": -47.78826141357422,
      "activations/layer18_attention_weight_max": 52.155025482177734,
      "activations/layer18_attention_weight_min": -40.05546951293945,
      "activations/layer19_attention_weight_max": 28.3680419921875,
      "activations/layer19_attention_weight_min": -22.541288375854492,
      "activations/layer1_attention_weight_max": 16.976055145263672,
      "activations/layer1_attention_weight_min": -16.451923370361328,
      "activations/layer20_attention_weight_max": 29.505409240722656,
      "activations/layer20_attention_weight_min": -22.823360443115234,
      "activations/layer21_attention_weight_max": 42.98227310180664,
      "activations/layer21_attention_weight_min": -23.648046493530273,
      "activations/layer22_attention_weight_max": 38.78977966308594,
      "activations/layer22_attention_weight_min": -26.496612548828125,
      "activations/layer23_attention_weight_max": 36.921993255615234,
      "activations/layer23_attention_weight_min": -23.329471588134766,
      "activations/layer2_attention_weight_max": 31.92066764831543,
      "activations/layer2_attention_weight_min": -31.46889305114746,
      "activations/layer3_attention_weight_max": 87.61982727050781,
      "activations/layer3_attention_weight_min": -91.57281494140625,
      "activations/layer4_attention_weight_max": 100.66410064697266,
      "activations/layer4_attention_weight_min": -95.0737075805664,
      "activations/layer5_attention_weight_max": 85.88191223144531,
      "activations/layer5_attention_weight_min": -76.72777557373047,
      "activations/layer6_attention_weight_max": 55.808841705322266,
      "activations/layer6_attention_weight_min": -53.043983459472656,
      "activations/layer7_attention_weight_max": 88.41401672363281,
      "activations/layer7_attention_weight_min": -92.36123657226562,
      "activations/layer8_attention_weight_max": 54.61516571044922,
      "activations/layer8_attention_weight_min": -53.05927658081055,
      "activations/layer9_attention_weight_max": 69.45549774169922,
      "activations/layer9_attention_weight_min": -61.543190002441406,
      "epoch": 8.89,
      "learning_rate": 9.361212121212122e-05,
      "loss": 2.8087,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.5336,
      "eval_samples_per_second": 503.189,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.5336,
      "eval_openwebtext_samples_per_second": 503.189,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 1.9848,
      "eval_wikitext_samples_per_second": 229.747,
      "step": 153000
    },
    {
      "epoch": 8.89,
      "eval_lambada_loss": 2.67578125,
      "eval_lambada_ppl": 14.523692042874359,
      "eval_lambada_runtime": 9.562,
      "eval_lambada_samples_per_second": 509.201,
      "step": 153000
    },
    {
      "activations/layer0_attention_weight_max": 15.067030906677246,
      "activations/layer0_attention_weight_min": -13.425201416015625,
      "activations/layer10_attention_weight_max": 36.32206726074219,
      "activations/layer10_attention_weight_min": -38.750335693359375,
      "activations/layer11_attention_weight_max": 30.66822052001953,
      "activations/layer11_attention_weight_min": -31.631694793701172,
      "activations/layer12_attention_weight_max": 21.566946029663086,
      "activations/layer12_attention_weight_min": -24.40504264831543,
      "activations/layer13_attention_weight_max": 37.592002868652344,
      "activations/layer13_attention_weight_min": -30.15419578552246,
      "activations/layer14_attention_weight_max": 37.82417297363281,
      "activations/layer14_attention_weight_min": -32.22972869873047,
      "activations/layer15_attention_weight_max": 33.56026077270508,
      "activations/layer15_attention_weight_min": -31.74545669555664,
      "activations/layer16_attention_weight_max": 32.76169204711914,
      "activations/layer16_attention_weight_min": -30.353853225708008,
      "activations/layer17_attention_weight_max": 57.33557891845703,
      "activations/layer17_attention_weight_min": -45.58720779418945,
      "activations/layer18_attention_weight_max": 46.33290100097656,
      "activations/layer18_attention_weight_min": -39.13087844848633,
      "activations/layer19_attention_weight_max": 20.46282196044922,
      "activations/layer19_attention_weight_min": -20.776384353637695,
      "activations/layer1_attention_weight_max": 16.326810836791992,
      "activations/layer1_attention_weight_min": -14.6537504196167,
      "activations/layer20_attention_weight_max": 22.838640213012695,
      "activations/layer20_attention_weight_min": -22.244050979614258,
      "activations/layer21_attention_weight_max": 42.28990936279297,
      "activations/layer21_attention_weight_min": -24.93657112121582,
      "activations/layer22_attention_weight_max": 29.62344741821289,
      "activations/layer22_attention_weight_min": -27.74518394470215,
      "activations/layer23_attention_weight_max": 35.5732307434082,
      "activations/layer23_attention_weight_min": -21.737180709838867,
      "activations/layer2_attention_weight_max": 33.66671371459961,
      "activations/layer2_attention_weight_min": -29.749988555908203,
      "activations/layer3_attention_weight_max": 92.52507781982422,
      "activations/layer3_attention_weight_min": -88.23922729492188,
      "activations/layer4_attention_weight_max": 103.17822265625,
      "activations/layer4_attention_weight_min": -94.92535400390625,
      "activations/layer5_attention_weight_max": 82.18891906738281,
      "activations/layer5_attention_weight_min": -81.30677795410156,
      "activations/layer6_attention_weight_max": 56.21435546875,
      "activations/layer6_attention_weight_min": -56.81829833984375,
      "activations/layer7_attention_weight_max": 79.9700927734375,
      "activations/layer7_attention_weight_min": -73.32410430908203,
      "activations/layer8_attention_weight_max": 44.494842529296875,
      "activations/layer8_attention_weight_min": -47.191078186035156,
      "activations/layer9_attention_weight_max": 57.3000373840332,
      "activations/layer9_attention_weight_min": -53.95548629760742,
      "epoch": 8.89,
      "learning_rate": 9.35931818181818e-05,
      "loss": 2.7984,
      "step": 153050
    },
    {
      "activations/layer0_attention_weight_max": 14.789997100830078,
      "activations/layer0_attention_weight_min": -13.961104393005371,
      "activations/layer10_attention_weight_max": 35.270790100097656,
      "activations/layer10_attention_weight_min": -35.959686279296875,
      "activations/layer11_attention_weight_max": 30.73351287841797,
      "activations/layer11_attention_weight_min": -31.91492462158203,
      "activations/layer12_attention_weight_max": 16.478097915649414,
      "activations/layer12_attention_weight_min": -25.54681968688965,
      "activations/layer13_attention_weight_max": 41.5154914855957,
      "activations/layer13_attention_weight_min": -32.89201354980469,
      "activations/layer14_attention_weight_max": 39.215145111083984,
      "activations/layer14_attention_weight_min": -32.14113998413086,
      "activations/layer15_attention_weight_max": 32.71435546875,
      "activations/layer15_attention_weight_min": -30.245508193969727,
      "activations/layer16_attention_weight_max": 28.72565460205078,
      "activations/layer16_attention_weight_min": -27.21472930908203,
      "activations/layer17_attention_weight_max": 55.0040283203125,
      "activations/layer17_attention_weight_min": -41.86701965332031,
      "activations/layer18_attention_weight_max": 46.437808990478516,
      "activations/layer18_attention_weight_min": -36.35667419433594,
      "activations/layer19_attention_weight_max": 20.15383529663086,
      "activations/layer19_attention_weight_min": -21.54669761657715,
      "activations/layer1_attention_weight_max": 17.283845901489258,
      "activations/layer1_attention_weight_min": -18.108123779296875,
      "activations/layer20_attention_weight_max": 21.036008834838867,
      "activations/layer20_attention_weight_min": -22.882219314575195,
      "activations/layer21_attention_weight_max": 36.65114974975586,
      "activations/layer21_attention_weight_min": -23.443267822265625,
      "activations/layer22_attention_weight_max": 30.53738784790039,
      "activations/layer22_attention_weight_min": -26.920183181762695,
      "activations/layer23_attention_weight_max": 34.05669403076172,
      "activations/layer23_attention_weight_min": -23.476398468017578,
      "activations/layer2_attention_weight_max": 31.87312126159668,
      "activations/layer2_attention_weight_min": -30.577369689941406,
      "activations/layer3_attention_weight_max": 94.9828109741211,
      "activations/layer3_attention_weight_min": -91.64524841308594,
      "activations/layer4_attention_weight_max": 102.46537017822266,
      "activations/layer4_attention_weight_min": -94.65445709228516,
      "activations/layer5_attention_weight_max": 79.15142822265625,
      "activations/layer5_attention_weight_min": -77.07552337646484,
      "activations/layer6_attention_weight_max": 52.90605926513672,
      "activations/layer6_attention_weight_min": -51.67518615722656,
      "activations/layer7_attention_weight_max": 70.68678283691406,
      "activations/layer7_attention_weight_min": -73.69447326660156,
      "activations/layer8_attention_weight_max": 44.316749572753906,
      "activations/layer8_attention_weight_min": -45.646366119384766,
      "activations/layer9_attention_weight_max": 53.34505081176758,
      "activations/layer9_attention_weight_min": -51.951080322265625,
      "epoch": 8.9,
      "learning_rate": 9.357424242424242e-05,
      "loss": 2.7852,
      "step": 153100
    },
    {
      "activations/layer0_attention_weight_max": 13.968714714050293,
      "activations/layer0_attention_weight_min": -13.65000057220459,
      "activations/layer10_attention_weight_max": 40.098289489746094,
      "activations/layer10_attention_weight_min": -37.079463958740234,
      "activations/layer11_attention_weight_max": 33.59918975830078,
      "activations/layer11_attention_weight_min": -34.233863830566406,
      "activations/layer12_attention_weight_max": 26.486953735351562,
      "activations/layer12_attention_weight_min": -26.443252563476562,
      "activations/layer13_attention_weight_max": 43.47665023803711,
      "activations/layer13_attention_weight_min": -30.1496639251709,
      "activations/layer14_attention_weight_max": 40.69284439086914,
      "activations/layer14_attention_weight_min": -29.567651748657227,
      "activations/layer15_attention_weight_max": 40.74196243286133,
      "activations/layer15_attention_weight_min": -29.757553100585938,
      "activations/layer16_attention_weight_max": 35.332637786865234,
      "activations/layer16_attention_weight_min": -28.799701690673828,
      "activations/layer17_attention_weight_max": 57.73253631591797,
      "activations/layer17_attention_weight_min": -44.99605178833008,
      "activations/layer18_attention_weight_max": 55.828163146972656,
      "activations/layer18_attention_weight_min": -35.92331314086914,
      "activations/layer19_attention_weight_max": 22.504087448120117,
      "activations/layer19_attention_weight_min": -20.98959732055664,
      "activations/layer1_attention_weight_max": 17.283735275268555,
      "activations/layer1_attention_weight_min": -16.019695281982422,
      "activations/layer20_attention_weight_max": 22.760282516479492,
      "activations/layer20_attention_weight_min": -23.622526168823242,
      "activations/layer21_attention_weight_max": 40.156944274902344,
      "activations/layer21_attention_weight_min": -24.603025436401367,
      "activations/layer22_attention_weight_max": 32.88612747192383,
      "activations/layer22_attention_weight_min": -25.747901916503906,
      "activations/layer23_attention_weight_max": 38.84144592285156,
      "activations/layer23_attention_weight_min": -21.968902587890625,
      "activations/layer2_attention_weight_max": 33.912376403808594,
      "activations/layer2_attention_weight_min": -30.711441040039062,
      "activations/layer3_attention_weight_max": 94.70185852050781,
      "activations/layer3_attention_weight_min": -94.51374816894531,
      "activations/layer4_attention_weight_max": 107.93999481201172,
      "activations/layer4_attention_weight_min": -101.43502044677734,
      "activations/layer5_attention_weight_max": 85.75988006591797,
      "activations/layer5_attention_weight_min": -82.79267120361328,
      "activations/layer6_attention_weight_max": 57.849613189697266,
      "activations/layer6_attention_weight_min": -59.532413482666016,
      "activations/layer7_attention_weight_max": 84.80000305175781,
      "activations/layer7_attention_weight_min": -79.02457427978516,
      "activations/layer8_attention_weight_max": 49.811248779296875,
      "activations/layer8_attention_weight_min": -50.33168411254883,
      "activations/layer9_attention_weight_max": 53.82000732421875,
      "activations/layer9_attention_weight_min": -50.718570709228516,
      "epoch": 8.9,
      "learning_rate": 9.355530303030303e-05,
      "loss": 2.8243,
      "step": 153150
    },
    {
      "activations/layer0_attention_weight_max": 14.847146034240723,
      "activations/layer0_attention_weight_min": -13.504558563232422,
      "activations/layer10_attention_weight_max": 40.50346755981445,
      "activations/layer10_attention_weight_min": -34.381465911865234,
      "activations/layer11_attention_weight_max": 34.35405731201172,
      "activations/layer11_attention_weight_min": -30.737045288085938,
      "activations/layer12_attention_weight_max": 36.050392150878906,
      "activations/layer12_attention_weight_min": -29.3719539642334,
      "activations/layer13_attention_weight_max": 40.53816223144531,
      "activations/layer13_attention_weight_min": -32.368019104003906,
      "activations/layer14_attention_weight_max": 40.198184967041016,
      "activations/layer14_attention_weight_min": -31.018442153930664,
      "activations/layer15_attention_weight_max": 33.87269592285156,
      "activations/layer15_attention_weight_min": -29.879308700561523,
      "activations/layer16_attention_weight_max": 30.659212112426758,
      "activations/layer16_attention_weight_min": -27.603513717651367,
      "activations/layer17_attention_weight_max": 55.919158935546875,
      "activations/layer17_attention_weight_min": -44.9292106628418,
      "activations/layer18_attention_weight_max": 46.75444030761719,
      "activations/layer18_attention_weight_min": -35.3868293762207,
      "activations/layer19_attention_weight_max": 22.6997127532959,
      "activations/layer19_attention_weight_min": -20.049489974975586,
      "activations/layer1_attention_weight_max": 17.28389549255371,
      "activations/layer1_attention_weight_min": -14.04666805267334,
      "activations/layer20_attention_weight_max": 19.263593673706055,
      "activations/layer20_attention_weight_min": -19.997623443603516,
      "activations/layer21_attention_weight_max": 35.2069091796875,
      "activations/layer21_attention_weight_min": -20.88959312438965,
      "activations/layer22_attention_weight_max": 27.531776428222656,
      "activations/layer22_attention_weight_min": -26.644134521484375,
      "activations/layer23_attention_weight_max": 38.59380340576172,
      "activations/layer23_attention_weight_min": -26.380199432373047,
      "activations/layer2_attention_weight_max": 32.285423278808594,
      "activations/layer2_attention_weight_min": -29.931865692138672,
      "activations/layer3_attention_weight_max": 92.20367431640625,
      "activations/layer3_attention_weight_min": -95.79582214355469,
      "activations/layer4_attention_weight_max": 101.91043090820312,
      "activations/layer4_attention_weight_min": -94.81184387207031,
      "activations/layer5_attention_weight_max": 79.94451141357422,
      "activations/layer5_attention_weight_min": -76.9313735961914,
      "activations/layer6_attention_weight_max": 52.9607048034668,
      "activations/layer6_attention_weight_min": -52.38700866699219,
      "activations/layer7_attention_weight_max": 77.25264739990234,
      "activations/layer7_attention_weight_min": -72.46709442138672,
      "activations/layer8_attention_weight_max": 44.1897087097168,
      "activations/layer8_attention_weight_min": -49.431800842285156,
      "activations/layer9_attention_weight_max": 49.71867752075195,
      "activations/layer9_attention_weight_min": -48.23682403564453,
      "epoch": 8.9,
      "learning_rate": 9.353636363636362e-05,
      "loss": 2.8024,
      "step": 153200
    },
    {
      "activations/layer0_attention_weight_max": 15.502334594726562,
      "activations/layer0_attention_weight_min": -13.712743759155273,
      "activations/layer10_attention_weight_max": 41.08815383911133,
      "activations/layer10_attention_weight_min": -38.77117919921875,
      "activations/layer11_attention_weight_max": 34.82505798339844,
      "activations/layer11_attention_weight_min": -33.41542053222656,
      "activations/layer12_attention_weight_max": 20.331607818603516,
      "activations/layer12_attention_weight_min": -25.31523323059082,
      "activations/layer13_attention_weight_max": 54.147525787353516,
      "activations/layer13_attention_weight_min": -38.37663269042969,
      "activations/layer14_attention_weight_max": 51.3910026550293,
      "activations/layer14_attention_weight_min": -34.36948013305664,
      "activations/layer15_attention_weight_max": 41.612815856933594,
      "activations/layer15_attention_weight_min": -29.998098373413086,
      "activations/layer16_attention_weight_max": 33.285484313964844,
      "activations/layer16_attention_weight_min": -29.17566680908203,
      "activations/layer17_attention_weight_max": 51.13652420043945,
      "activations/layer17_attention_weight_min": -43.15625762939453,
      "activations/layer18_attention_weight_max": 45.8438835144043,
      "activations/layer18_attention_weight_min": -36.96796798706055,
      "activations/layer19_attention_weight_max": 20.63888168334961,
      "activations/layer19_attention_weight_min": -20.258377075195312,
      "activations/layer1_attention_weight_max": 16.75728416442871,
      "activations/layer1_attention_weight_min": -15.107438087463379,
      "activations/layer20_attention_weight_max": 21.200519561767578,
      "activations/layer20_attention_weight_min": -22.871238708496094,
      "activations/layer21_attention_weight_max": 36.43805694580078,
      "activations/layer21_attention_weight_min": -24.897197723388672,
      "activations/layer22_attention_weight_max": 35.991188049316406,
      "activations/layer22_attention_weight_min": -25.297069549560547,
      "activations/layer23_attention_weight_max": 34.643585205078125,
      "activations/layer23_attention_weight_min": -21.591289520263672,
      "activations/layer2_attention_weight_max": 32.00173568725586,
      "activations/layer2_attention_weight_min": -32.73661422729492,
      "activations/layer3_attention_weight_max": 95.60279083251953,
      "activations/layer3_attention_weight_min": -103.545654296875,
      "activations/layer4_attention_weight_max": 106.70623779296875,
      "activations/layer4_attention_weight_min": -107.9786605834961,
      "activations/layer5_attention_weight_max": 84.66789245605469,
      "activations/layer5_attention_weight_min": -81.16692352294922,
      "activations/layer6_attention_weight_max": 58.711181640625,
      "activations/layer6_attention_weight_min": -57.404151916503906,
      "activations/layer7_attention_weight_max": 76.97220611572266,
      "activations/layer7_attention_weight_min": -77.50472259521484,
      "activations/layer8_attention_weight_max": 46.15734100341797,
      "activations/layer8_attention_weight_min": -49.376888275146484,
      "activations/layer9_attention_weight_max": 50.937461853027344,
      "activations/layer9_attention_weight_min": -52.654415130615234,
      "epoch": 8.9,
      "learning_rate": 9.351742424242424e-05,
      "loss": 2.8202,
      "step": 153250
    },
    {
      "activations/layer0_attention_weight_max": 14.251324653625488,
      "activations/layer0_attention_weight_min": -13.586809158325195,
      "activations/layer10_attention_weight_max": 40.39863586425781,
      "activations/layer10_attention_weight_min": -37.29716491699219,
      "activations/layer11_attention_weight_max": 31.960472106933594,
      "activations/layer11_attention_weight_min": -32.710601806640625,
      "activations/layer12_attention_weight_max": 24.257402420043945,
      "activations/layer12_attention_weight_min": -24.890491485595703,
      "activations/layer13_attention_weight_max": 44.7680549621582,
      "activations/layer13_attention_weight_min": -32.40043640136719,
      "activations/layer14_attention_weight_max": 39.24198913574219,
      "activations/layer14_attention_weight_min": -30.73096466064453,
      "activations/layer15_attention_weight_max": 35.25492858886719,
      "activations/layer15_attention_weight_min": -30.220552444458008,
      "activations/layer16_attention_weight_max": 34.767555236816406,
      "activations/layer16_attention_weight_min": -28.01285171508789,
      "activations/layer17_attention_weight_max": 58.44931411743164,
      "activations/layer17_attention_weight_min": -44.69137954711914,
      "activations/layer18_attention_weight_max": 51.854183197021484,
      "activations/layer18_attention_weight_min": -36.42760467529297,
      "activations/layer19_attention_weight_max": 21.895353317260742,
      "activations/layer19_attention_weight_min": -19.776880264282227,
      "activations/layer1_attention_weight_max": 16.23789405822754,
      "activations/layer1_attention_weight_min": -16.227352142333984,
      "activations/layer20_attention_weight_max": 22.71505355834961,
      "activations/layer20_attention_weight_min": -19.507463455200195,
      "activations/layer21_attention_weight_max": 37.15239715576172,
      "activations/layer21_attention_weight_min": -23.33749771118164,
      "activations/layer22_attention_weight_max": 32.293949127197266,
      "activations/layer22_attention_weight_min": -24.968782424926758,
      "activations/layer23_attention_weight_max": 38.57878875732422,
      "activations/layer23_attention_weight_min": -20.19131851196289,
      "activations/layer2_attention_weight_max": 32.23554992675781,
      "activations/layer2_attention_weight_min": -30.885847091674805,
      "activations/layer3_attention_weight_max": 90.87529754638672,
      "activations/layer3_attention_weight_min": -95.20941162109375,
      "activations/layer4_attention_weight_max": 103.71573638916016,
      "activations/layer4_attention_weight_min": -97.48800659179688,
      "activations/layer5_attention_weight_max": 82.86006164550781,
      "activations/layer5_attention_weight_min": -80.66080474853516,
      "activations/layer6_attention_weight_max": 55.46476364135742,
      "activations/layer6_attention_weight_min": -54.800533294677734,
      "activations/layer7_attention_weight_max": 78.44841766357422,
      "activations/layer7_attention_weight_min": -73.66626739501953,
      "activations/layer8_attention_weight_max": 45.20209503173828,
      "activations/layer8_attention_weight_min": -48.74605178833008,
      "activations/layer9_attention_weight_max": 51.54768753051758,
      "activations/layer9_attention_weight_min": -50.59279251098633,
      "epoch": 8.91,
      "learning_rate": 9.349848484848484e-05,
      "loss": 2.7967,
      "step": 153300
    },
    {
      "activations/layer0_attention_weight_max": 14.140748977661133,
      "activations/layer0_attention_weight_min": -14.091580390930176,
      "activations/layer10_attention_weight_max": 43.68972396850586,
      "activations/layer10_attention_weight_min": -41.36934280395508,
      "activations/layer11_attention_weight_max": 36.86357116699219,
      "activations/layer11_attention_weight_min": -34.39250183105469,
      "activations/layer12_attention_weight_max": 22.750322341918945,
      "activations/layer12_attention_weight_min": -26.12647247314453,
      "activations/layer13_attention_weight_max": 47.01078414916992,
      "activations/layer13_attention_weight_min": -32.23463439941406,
      "activations/layer14_attention_weight_max": 67.26776123046875,
      "activations/layer14_attention_weight_min": -34.636695861816406,
      "activations/layer15_attention_weight_max": 54.18430709838867,
      "activations/layer15_attention_weight_min": -31.469467163085938,
      "activations/layer16_attention_weight_max": 37.885826110839844,
      "activations/layer16_attention_weight_min": -29.8829288482666,
      "activations/layer17_attention_weight_max": 57.1201171875,
      "activations/layer17_attention_weight_min": -44.99455261230469,
      "activations/layer18_attention_weight_max": 52.289451599121094,
      "activations/layer18_attention_weight_min": -38.19807815551758,
      "activations/layer19_attention_weight_max": 30.216201782226562,
      "activations/layer19_attention_weight_min": -22.44762420654297,
      "activations/layer1_attention_weight_max": 17.60988426208496,
      "activations/layer1_attention_weight_min": -14.503302574157715,
      "activations/layer20_attention_weight_max": 27.659677505493164,
      "activations/layer20_attention_weight_min": -25.3812198638916,
      "activations/layer21_attention_weight_max": 41.79167175292969,
      "activations/layer21_attention_weight_min": -23.28351402282715,
      "activations/layer22_attention_weight_max": 37.39888381958008,
      "activations/layer22_attention_weight_min": -24.653278350830078,
      "activations/layer23_attention_weight_max": 41.65613555908203,
      "activations/layer23_attention_weight_min": -21.405792236328125,
      "activations/layer2_attention_weight_max": 34.3896484375,
      "activations/layer2_attention_weight_min": -32.99919891357422,
      "activations/layer3_attention_weight_max": 104.27782440185547,
      "activations/layer3_attention_weight_min": -109.32987976074219,
      "activations/layer4_attention_weight_max": 113.86272430419922,
      "activations/layer4_attention_weight_min": -113.68731689453125,
      "activations/layer5_attention_weight_max": 91.29365539550781,
      "activations/layer5_attention_weight_min": -84.631591796875,
      "activations/layer6_attention_weight_max": 61.23881912231445,
      "activations/layer6_attention_weight_min": -62.251808166503906,
      "activations/layer7_attention_weight_max": 86.1771240234375,
      "activations/layer7_attention_weight_min": -80.7397232055664,
      "activations/layer8_attention_weight_max": 50.82918930053711,
      "activations/layer8_attention_weight_min": -53.30091094970703,
      "activations/layer9_attention_weight_max": 56.303768157958984,
      "activations/layer9_attention_weight_min": -54.367706298828125,
      "epoch": 8.91,
      "learning_rate": 9.347954545454544e-05,
      "loss": 2.8031,
      "step": 153350
    },
    {
      "activations/layer0_attention_weight_max": 14.745397567749023,
      "activations/layer0_attention_weight_min": -13.29448413848877,
      "activations/layer10_attention_weight_max": 38.021602630615234,
      "activations/layer10_attention_weight_min": -35.565185546875,
      "activations/layer11_attention_weight_max": 30.78230094909668,
      "activations/layer11_attention_weight_min": -32.035518646240234,
      "activations/layer12_attention_weight_max": 20.536813735961914,
      "activations/layer12_attention_weight_min": -26.767200469970703,
      "activations/layer13_attention_weight_max": 46.72077178955078,
      "activations/layer13_attention_weight_min": -29.300600051879883,
      "activations/layer14_attention_weight_max": 37.8940315246582,
      "activations/layer14_attention_weight_min": -31.4821720123291,
      "activations/layer15_attention_weight_max": 36.13499069213867,
      "activations/layer15_attention_weight_min": -30.010820388793945,
      "activations/layer16_attention_weight_max": 33.44776153564453,
      "activations/layer16_attention_weight_min": -28.899555206298828,
      "activations/layer17_attention_weight_max": 53.04429626464844,
      "activations/layer17_attention_weight_min": -43.06596374511719,
      "activations/layer18_attention_weight_max": 48.64472961425781,
      "activations/layer18_attention_weight_min": -36.22443771362305,
      "activations/layer19_attention_weight_max": 24.66866111755371,
      "activations/layer19_attention_weight_min": -20.78763771057129,
      "activations/layer1_attention_weight_max": 17.189958572387695,
      "activations/layer1_attention_weight_min": -18.51337432861328,
      "activations/layer20_attention_weight_max": 25.68242835998535,
      "activations/layer20_attention_weight_min": -21.443925857543945,
      "activations/layer21_attention_weight_max": 36.21189498901367,
      "activations/layer21_attention_weight_min": -25.548154830932617,
      "activations/layer22_attention_weight_max": 33.20280838012695,
      "activations/layer22_attention_weight_min": -25.008779525756836,
      "activations/layer23_attention_weight_max": 38.19621658325195,
      "activations/layer23_attention_weight_min": -21.063156127929688,
      "activations/layer2_attention_weight_max": 31.3100643157959,
      "activations/layer2_attention_weight_min": -29.43097496032715,
      "activations/layer3_attention_weight_max": 91.12078094482422,
      "activations/layer3_attention_weight_min": -94.89176177978516,
      "activations/layer4_attention_weight_max": 102.5552749633789,
      "activations/layer4_attention_weight_min": -97.0008316040039,
      "activations/layer5_attention_weight_max": 82.26114654541016,
      "activations/layer5_attention_weight_min": -79.2867431640625,
      "activations/layer6_attention_weight_max": 54.5731086730957,
      "activations/layer6_attention_weight_min": -52.0267219543457,
      "activations/layer7_attention_weight_max": 80.58601379394531,
      "activations/layer7_attention_weight_min": -73.64075469970703,
      "activations/layer8_attention_weight_max": 45.85877227783203,
      "activations/layer8_attention_weight_min": -45.31651306152344,
      "activations/layer9_attention_weight_max": 48.929901123046875,
      "activations/layer9_attention_weight_min": -48.845977783203125,
      "epoch": 8.91,
      "learning_rate": 9.346060606060606e-05,
      "loss": 2.8103,
      "step": 153400
    },
    {
      "activations/layer0_attention_weight_max": 14.377850532531738,
      "activations/layer0_attention_weight_min": -13.473605155944824,
      "activations/layer10_attention_weight_max": 39.4656982421875,
      "activations/layer10_attention_weight_min": -38.30890655517578,
      "activations/layer11_attention_weight_max": 36.686866760253906,
      "activations/layer11_attention_weight_min": -36.024227142333984,
      "activations/layer12_attention_weight_max": 17.909393310546875,
      "activations/layer12_attention_weight_min": -30.956275939941406,
      "activations/layer13_attention_weight_max": 43.00217819213867,
      "activations/layer13_attention_weight_min": -33.79955291748047,
      "activations/layer14_attention_weight_max": 39.644004821777344,
      "activations/layer14_attention_weight_min": -31.16179656982422,
      "activations/layer15_attention_weight_max": 33.19779586791992,
      "activations/layer15_attention_weight_min": -29.882801055908203,
      "activations/layer16_attention_weight_max": 28.775922775268555,
      "activations/layer16_attention_weight_min": -27.469158172607422,
      "activations/layer17_attention_weight_max": 51.544708251953125,
      "activations/layer17_attention_weight_min": -43.50432205200195,
      "activations/layer18_attention_weight_max": 45.608402252197266,
      "activations/layer18_attention_weight_min": -36.434608459472656,
      "activations/layer19_attention_weight_max": 19.895435333251953,
      "activations/layer19_attention_weight_min": -20.007444381713867,
      "activations/layer1_attention_weight_max": 16.584705352783203,
      "activations/layer1_attention_weight_min": -14.901378631591797,
      "activations/layer20_attention_weight_max": 20.442569732666016,
      "activations/layer20_attention_weight_min": -21.883438110351562,
      "activations/layer21_attention_weight_max": 35.8212890625,
      "activations/layer21_attention_weight_min": -21.639673233032227,
      "activations/layer22_attention_weight_max": 28.568546295166016,
      "activations/layer22_attention_weight_min": -26.0898494720459,
      "activations/layer23_attention_weight_max": 35.283546447753906,
      "activations/layer23_attention_weight_min": -21.57269287109375,
      "activations/layer2_attention_weight_max": 32.288238525390625,
      "activations/layer2_attention_weight_min": -33.05374526977539,
      "activations/layer3_attention_weight_max": 92.26319885253906,
      "activations/layer3_attention_weight_min": -106.08013916015625,
      "activations/layer4_attention_weight_max": 105.84132385253906,
      "activations/layer4_attention_weight_min": -103.05095672607422,
      "activations/layer5_attention_weight_max": 86.21532440185547,
      "activations/layer5_attention_weight_min": -81.37996673583984,
      "activations/layer6_attention_weight_max": 58.36820983886719,
      "activations/layer6_attention_weight_min": -60.27372741699219,
      "activations/layer7_attention_weight_max": 84.33631134033203,
      "activations/layer7_attention_weight_min": -78.70819854736328,
      "activations/layer8_attention_weight_max": 48.71525955200195,
      "activations/layer8_attention_weight_min": -51.44685745239258,
      "activations/layer9_attention_weight_max": 51.87092208862305,
      "activations/layer9_attention_weight_min": -52.32645797729492,
      "epoch": 8.92,
      "learning_rate": 9.344166666666666e-05,
      "loss": 2.8159,
      "step": 153450
    },
    {
      "activations/layer0_attention_weight_max": 14.775935173034668,
      "activations/layer0_attention_weight_min": -13.702301979064941,
      "activations/layer10_attention_weight_max": 39.379425048828125,
      "activations/layer10_attention_weight_min": -37.860816955566406,
      "activations/layer11_attention_weight_max": 35.79473876953125,
      "activations/layer11_attention_weight_min": -33.894344329833984,
      "activations/layer12_attention_weight_max": 18.056346893310547,
      "activations/layer12_attention_weight_min": -27.852951049804688,
      "activations/layer13_attention_weight_max": 39.32611083984375,
      "activations/layer13_attention_weight_min": -29.80701446533203,
      "activations/layer14_attention_weight_max": 36.979942321777344,
      "activations/layer14_attention_weight_min": -30.532821655273438,
      "activations/layer15_attention_weight_max": 34.23625183105469,
      "activations/layer15_attention_weight_min": -29.03639030456543,
      "activations/layer16_attention_weight_max": 28.366748809814453,
      "activations/layer16_attention_weight_min": -26.991151809692383,
      "activations/layer17_attention_weight_max": 53.5682373046875,
      "activations/layer17_attention_weight_min": -41.55329132080078,
      "activations/layer18_attention_weight_max": 43.7451057434082,
      "activations/layer18_attention_weight_min": -35.57496643066406,
      "activations/layer19_attention_weight_max": 18.955015182495117,
      "activations/layer19_attention_weight_min": -20.806169509887695,
      "activations/layer1_attention_weight_max": 16.745576858520508,
      "activations/layer1_attention_weight_min": -15.766934394836426,
      "activations/layer20_attention_weight_max": 21.02599334716797,
      "activations/layer20_attention_weight_min": -22.84791374206543,
      "activations/layer21_attention_weight_max": 31.084890365600586,
      "activations/layer21_attention_weight_min": -22.6104793548584,
      "activations/layer22_attention_weight_max": 29.010854721069336,
      "activations/layer22_attention_weight_min": -24.76266860961914,
      "activations/layer23_attention_weight_max": 34.50485610961914,
      "activations/layer23_attention_weight_min": -21.376697540283203,
      "activations/layer2_attention_weight_max": 31.38361930847168,
      "activations/layer2_attention_weight_min": -30.987140655517578,
      "activations/layer3_attention_weight_max": 88.55859375,
      "activations/layer3_attention_weight_min": -95.49104309082031,
      "activations/layer4_attention_weight_max": 99.4942855834961,
      "activations/layer4_attention_weight_min": -97.17040252685547,
      "activations/layer5_attention_weight_max": 81.27311706542969,
      "activations/layer5_attention_weight_min": -78.29094696044922,
      "activations/layer6_attention_weight_max": 57.436683654785156,
      "activations/layer6_attention_weight_min": -54.395084381103516,
      "activations/layer7_attention_weight_max": 78.12867736816406,
      "activations/layer7_attention_weight_min": -76.18641662597656,
      "activations/layer8_attention_weight_max": 46.90525436401367,
      "activations/layer8_attention_weight_min": -52.388389587402344,
      "activations/layer9_attention_weight_max": 50.523372650146484,
      "activations/layer9_attention_weight_min": -52.1900749206543,
      "epoch": 8.92,
      "learning_rate": 9.342272727272727e-05,
      "loss": 2.8046,
      "step": 153500
    },
    {
      "activations/layer0_attention_weight_max": 14.253887176513672,
      "activations/layer0_attention_weight_min": -13.944144248962402,
      "activations/layer10_attention_weight_max": 37.17285919189453,
      "activations/layer10_attention_weight_min": -36.169307708740234,
      "activations/layer11_attention_weight_max": 30.617080688476562,
      "activations/layer11_attention_weight_min": -35.26256561279297,
      "activations/layer12_attention_weight_max": 23.372705459594727,
      "activations/layer12_attention_weight_min": -23.22450065612793,
      "activations/layer13_attention_weight_max": 42.77725601196289,
      "activations/layer13_attention_weight_min": -30.970666885375977,
      "activations/layer14_attention_weight_max": 39.020469665527344,
      "activations/layer14_attention_weight_min": -30.859725952148438,
      "activations/layer15_attention_weight_max": 33.573116302490234,
      "activations/layer15_attention_weight_min": -30.875362396240234,
      "activations/layer16_attention_weight_max": 30.08674430847168,
      "activations/layer16_attention_weight_min": -27.90916633605957,
      "activations/layer17_attention_weight_max": 53.06424331665039,
      "activations/layer17_attention_weight_min": -44.28080368041992,
      "activations/layer18_attention_weight_max": 44.001041412353516,
      "activations/layer18_attention_weight_min": -36.917449951171875,
      "activations/layer19_attention_weight_max": 21.689367294311523,
      "activations/layer19_attention_weight_min": -22.62413787841797,
      "activations/layer1_attention_weight_max": 16.873119354248047,
      "activations/layer1_attention_weight_min": -15.7362699508667,
      "activations/layer20_attention_weight_max": 21.49852752685547,
      "activations/layer20_attention_weight_min": -27.284440994262695,
      "activations/layer21_attention_weight_max": 35.50111389160156,
      "activations/layer21_attention_weight_min": -23.370128631591797,
      "activations/layer22_attention_weight_max": 30.645559310913086,
      "activations/layer22_attention_weight_min": -26.304903030395508,
      "activations/layer23_attention_weight_max": 35.295562744140625,
      "activations/layer23_attention_weight_min": -22.574390411376953,
      "activations/layer2_attention_weight_max": 29.361251831054688,
      "activations/layer2_attention_weight_min": -28.9155330657959,
      "activations/layer3_attention_weight_max": 89.12622833251953,
      "activations/layer3_attention_weight_min": -90.34587860107422,
      "activations/layer4_attention_weight_max": 101.37852478027344,
      "activations/layer4_attention_weight_min": -99.70060729980469,
      "activations/layer5_attention_weight_max": 82.7159652709961,
      "activations/layer5_attention_weight_min": -78.99147033691406,
      "activations/layer6_attention_weight_max": 53.91460418701172,
      "activations/layer6_attention_weight_min": -58.00630187988281,
      "activations/layer7_attention_weight_max": 78.1575927734375,
      "activations/layer7_attention_weight_min": -76.77676391601562,
      "activations/layer8_attention_weight_max": 45.54033279418945,
      "activations/layer8_attention_weight_min": -46.145381927490234,
      "activations/layer9_attention_weight_max": 48.417179107666016,
      "activations/layer9_attention_weight_min": -49.85908889770508,
      "epoch": 8.92,
      "learning_rate": 9.340378787878787e-05,
      "loss": 2.8109,
      "step": 153550
    },
    {
      "activations/layer0_attention_weight_max": 15.420899391174316,
      "activations/layer0_attention_weight_min": -14.313441276550293,
      "activations/layer10_attention_weight_max": 36.92293167114258,
      "activations/layer10_attention_weight_min": -34.771461486816406,
      "activations/layer11_attention_weight_max": 30.620986938476562,
      "activations/layer11_attention_weight_min": -31.838285446166992,
      "activations/layer12_attention_weight_max": 16.856719970703125,
      "activations/layer12_attention_weight_min": -25.26437759399414,
      "activations/layer13_attention_weight_max": 37.67692947387695,
      "activations/layer13_attention_weight_min": -33.347251892089844,
      "activations/layer14_attention_weight_max": 38.07283401489258,
      "activations/layer14_attention_weight_min": -28.695161819458008,
      "activations/layer15_attention_weight_max": 32.44917297363281,
      "activations/layer15_attention_weight_min": -27.409059524536133,
      "activations/layer16_attention_weight_max": 31.729646682739258,
      "activations/layer16_attention_weight_min": -27.185258865356445,
      "activations/layer17_attention_weight_max": 52.80888366699219,
      "activations/layer17_attention_weight_min": -41.01407241821289,
      "activations/layer18_attention_weight_max": 46.81026840209961,
      "activations/layer18_attention_weight_min": -35.47689437866211,
      "activations/layer19_attention_weight_max": 22.7828369140625,
      "activations/layer19_attention_weight_min": -22.261592864990234,
      "activations/layer1_attention_weight_max": 17.17789649963379,
      "activations/layer1_attention_weight_min": -13.8899564743042,
      "activations/layer20_attention_weight_max": 23.226930618286133,
      "activations/layer20_attention_weight_min": -22.33734130859375,
      "activations/layer21_attention_weight_max": 41.298973083496094,
      "activations/layer21_attention_weight_min": -23.749608993530273,
      "activations/layer22_attention_weight_max": 30.617565155029297,
      "activations/layer22_attention_weight_min": -25.982379913330078,
      "activations/layer23_attention_weight_max": 37.675193786621094,
      "activations/layer23_attention_weight_min": -20.845714569091797,
      "activations/layer2_attention_weight_max": 32.8870735168457,
      "activations/layer2_attention_weight_min": -30.877696990966797,
      "activations/layer3_attention_weight_max": 81.39431762695312,
      "activations/layer3_attention_weight_min": -82.16407012939453,
      "activations/layer4_attention_weight_max": 99.20513153076172,
      "activations/layer4_attention_weight_min": -94.8310546875,
      "activations/layer5_attention_weight_max": 77.2945785522461,
      "activations/layer5_attention_weight_min": -75.36100769042969,
      "activations/layer6_attention_weight_max": 52.049137115478516,
      "activations/layer6_attention_weight_min": -53.61451721191406,
      "activations/layer7_attention_weight_max": 75.90827178955078,
      "activations/layer7_attention_weight_min": -68.5654067993164,
      "activations/layer8_attention_weight_max": 40.55485916137695,
      "activations/layer8_attention_weight_min": -43.7110595703125,
      "activations/layer9_attention_weight_max": 45.443336486816406,
      "activations/layer9_attention_weight_min": -47.47119140625,
      "epoch": 8.92,
      "learning_rate": 9.338484848484847e-05,
      "loss": 2.8058,
      "step": 153600
    },
    {
      "activations/layer0_attention_weight_max": 16.39655303955078,
      "activations/layer0_attention_weight_min": -13.70635986328125,
      "activations/layer10_attention_weight_max": 38.446861267089844,
      "activations/layer10_attention_weight_min": -39.144386291503906,
      "activations/layer11_attention_weight_max": 33.13496780395508,
      "activations/layer11_attention_weight_min": -35.3566780090332,
      "activations/layer12_attention_weight_max": 22.733304977416992,
      "activations/layer12_attention_weight_min": -28.664321899414062,
      "activations/layer13_attention_weight_max": 47.422359466552734,
      "activations/layer13_attention_weight_min": -31.643213272094727,
      "activations/layer14_attention_weight_max": 38.28517150878906,
      "activations/layer14_attention_weight_min": -29.13790512084961,
      "activations/layer15_attention_weight_max": 33.435543060302734,
      "activations/layer15_attention_weight_min": -29.73177146911621,
      "activations/layer16_attention_weight_max": 31.664478302001953,
      "activations/layer16_attention_weight_min": -27.144357681274414,
      "activations/layer17_attention_weight_max": 55.74428939819336,
      "activations/layer17_attention_weight_min": -43.67018127441406,
      "activations/layer18_attention_weight_max": 47.01326370239258,
      "activations/layer18_attention_weight_min": -36.32404708862305,
      "activations/layer19_attention_weight_max": 27.009069442749023,
      "activations/layer19_attention_weight_min": -22.906768798828125,
      "activations/layer1_attention_weight_max": 16.85474967956543,
      "activations/layer1_attention_weight_min": -14.945860862731934,
      "activations/layer20_attention_weight_max": 21.747987747192383,
      "activations/layer20_attention_weight_min": -21.03480339050293,
      "activations/layer21_attention_weight_max": 39.232051849365234,
      "activations/layer21_attention_weight_min": -21.157150268554688,
      "activations/layer22_attention_weight_max": 29.21183204650879,
      "activations/layer22_attention_weight_min": -24.920461654663086,
      "activations/layer23_attention_weight_max": 36.991004943847656,
      "activations/layer23_attention_weight_min": -20.473495483398438,
      "activations/layer2_attention_weight_max": 31.50629425048828,
      "activations/layer2_attention_weight_min": -31.296689987182617,
      "activations/layer3_attention_weight_max": 89.04452514648438,
      "activations/layer3_attention_weight_min": -94.33271026611328,
      "activations/layer4_attention_weight_max": 102.22762298583984,
      "activations/layer4_attention_weight_min": -100.07881927490234,
      "activations/layer5_attention_weight_max": 82.32086181640625,
      "activations/layer5_attention_weight_min": -79.10407257080078,
      "activations/layer6_attention_weight_max": 54.62069320678711,
      "activations/layer6_attention_weight_min": -53.935218811035156,
      "activations/layer7_attention_weight_max": 78.81768035888672,
      "activations/layer7_attention_weight_min": -79.4332504272461,
      "activations/layer8_attention_weight_max": 47.08731460571289,
      "activations/layer8_attention_weight_min": -48.89409255981445,
      "activations/layer9_attention_weight_max": 51.45914840698242,
      "activations/layer9_attention_weight_min": -52.32403564453125,
      "epoch": 8.93,
      "learning_rate": 9.336628787878787e-05,
      "loss": 2.8098,
      "step": 153650
    },
    {
      "activations/layer0_attention_weight_max": 14.984374046325684,
      "activations/layer0_attention_weight_min": -14.408306121826172,
      "activations/layer10_attention_weight_max": 34.39814758300781,
      "activations/layer10_attention_weight_min": -34.77974319458008,
      "activations/layer11_attention_weight_max": 31.302886962890625,
      "activations/layer11_attention_weight_min": -31.246915817260742,
      "activations/layer12_attention_weight_max": 17.27963638305664,
      "activations/layer12_attention_weight_min": -23.6490421295166,
      "activations/layer13_attention_weight_max": 37.58199691772461,
      "activations/layer13_attention_weight_min": -31.36675453186035,
      "activations/layer14_attention_weight_max": 36.404388427734375,
      "activations/layer14_attention_weight_min": -29.936771392822266,
      "activations/layer15_attention_weight_max": 32.98971939086914,
      "activations/layer15_attention_weight_min": -27.733097076416016,
      "activations/layer16_attention_weight_max": 28.689111709594727,
      "activations/layer16_attention_weight_min": -27.56454849243164,
      "activations/layer17_attention_weight_max": 52.87720489501953,
      "activations/layer17_attention_weight_min": -41.97014236450195,
      "activations/layer18_attention_weight_max": 45.25809097290039,
      "activations/layer18_attention_weight_min": -36.30828094482422,
      "activations/layer19_attention_weight_max": 21.077054977416992,
      "activations/layer19_attention_weight_min": -21.508708953857422,
      "activations/layer1_attention_weight_max": 16.98796844482422,
      "activations/layer1_attention_weight_min": -14.0957670211792,
      "activations/layer20_attention_weight_max": 21.148517608642578,
      "activations/layer20_attention_weight_min": -21.541399002075195,
      "activations/layer21_attention_weight_max": 36.02283477783203,
      "activations/layer21_attention_weight_min": -23.201866149902344,
      "activations/layer22_attention_weight_max": 30.836700439453125,
      "activations/layer22_attention_weight_min": -24.68507957458496,
      "activations/layer23_attention_weight_max": 35.000816345214844,
      "activations/layer23_attention_weight_min": -22.82333755493164,
      "activations/layer2_attention_weight_max": 30.852928161621094,
      "activations/layer2_attention_weight_min": -30.14694595336914,
      "activations/layer3_attention_weight_max": 86.9402084350586,
      "activations/layer3_attention_weight_min": -92.1341323852539,
      "activations/layer4_attention_weight_max": 98.18004608154297,
      "activations/layer4_attention_weight_min": -93.19281005859375,
      "activations/layer5_attention_weight_max": 80.23609924316406,
      "activations/layer5_attention_weight_min": -76.51762390136719,
      "activations/layer6_attention_weight_max": 56.33208084106445,
      "activations/layer6_attention_weight_min": -53.03355407714844,
      "activations/layer7_attention_weight_max": 75.21942901611328,
      "activations/layer7_attention_weight_min": -71.88573455810547,
      "activations/layer8_attention_weight_max": 43.37324142456055,
      "activations/layer8_attention_weight_min": -43.689231872558594,
      "activations/layer9_attention_weight_max": 47.309814453125,
      "activations/layer9_attention_weight_min": -46.36928939819336,
      "epoch": 8.93,
      "learning_rate": 9.334734848484848e-05,
      "loss": 2.8095,
      "step": 153700
    },
    {
      "activations/layer0_attention_weight_max": 14.34322738647461,
      "activations/layer0_attention_weight_min": -14.244329452514648,
      "activations/layer10_attention_weight_max": 37.05113983154297,
      "activations/layer10_attention_weight_min": -34.97031021118164,
      "activations/layer11_attention_weight_max": 32.17334747314453,
      "activations/layer11_attention_weight_min": -31.492704391479492,
      "activations/layer12_attention_weight_max": 21.91445541381836,
      "activations/layer12_attention_weight_min": -30.49174690246582,
      "activations/layer13_attention_weight_max": 43.02927017211914,
      "activations/layer13_attention_weight_min": -30.801177978515625,
      "activations/layer14_attention_weight_max": 39.200321197509766,
      "activations/layer14_attention_weight_min": -29.49251365661621,
      "activations/layer15_attention_weight_max": 34.90225601196289,
      "activations/layer15_attention_weight_min": -29.363555908203125,
      "activations/layer16_attention_weight_max": 33.078853607177734,
      "activations/layer16_attention_weight_min": -28.46274757385254,
      "activations/layer17_attention_weight_max": 59.22838592529297,
      "activations/layer17_attention_weight_min": -47.43013381958008,
      "activations/layer18_attention_weight_max": 52.41625213623047,
      "activations/layer18_attention_weight_min": -39.83187484741211,
      "activations/layer19_attention_weight_max": 21.332489013671875,
      "activations/layer19_attention_weight_min": -24.547122955322266,
      "activations/layer1_attention_weight_max": 17.43208122253418,
      "activations/layer1_attention_weight_min": -16.358579635620117,
      "activations/layer20_attention_weight_max": 22.906545639038086,
      "activations/layer20_attention_weight_min": -21.747705459594727,
      "activations/layer21_attention_weight_max": 44.08332443237305,
      "activations/layer21_attention_weight_min": -24.4169864654541,
      "activations/layer22_attention_weight_max": 30.93965721130371,
      "activations/layer22_attention_weight_min": -26.312213897705078,
      "activations/layer23_attention_weight_max": 36.16986083984375,
      "activations/layer23_attention_weight_min": -22.71520233154297,
      "activations/layer2_attention_weight_max": 31.7707576751709,
      "activations/layer2_attention_weight_min": -30.938739776611328,
      "activations/layer3_attention_weight_max": 87.25749969482422,
      "activations/layer3_attention_weight_min": -93.16661071777344,
      "activations/layer4_attention_weight_max": 99.48892211914062,
      "activations/layer4_attention_weight_min": -101.25994873046875,
      "activations/layer5_attention_weight_max": 79.55838012695312,
      "activations/layer5_attention_weight_min": -80.00617980957031,
      "activations/layer6_attention_weight_max": 56.1038818359375,
      "activations/layer6_attention_weight_min": -56.0090446472168,
      "activations/layer7_attention_weight_max": 86.714599609375,
      "activations/layer7_attention_weight_min": -75.33553314208984,
      "activations/layer8_attention_weight_max": 46.98827362060547,
      "activations/layer8_attention_weight_min": -45.004119873046875,
      "activations/layer9_attention_weight_max": 60.79109573364258,
      "activations/layer9_attention_weight_min": -48.332611083984375,
      "epoch": 8.93,
      "learning_rate": 9.33284090909091e-05,
      "loss": 2.7944,
      "step": 153750
    },
    {
      "activations/layer0_attention_weight_max": 15.72325325012207,
      "activations/layer0_attention_weight_min": -14.787455558776855,
      "activations/layer10_attention_weight_max": 34.31889343261719,
      "activations/layer10_attention_weight_min": -33.83013916015625,
      "activations/layer11_attention_weight_max": 29.591886520385742,
      "activations/layer11_attention_weight_min": -29.724788665771484,
      "activations/layer12_attention_weight_max": 22.25090217590332,
      "activations/layer12_attention_weight_min": -26.87153434753418,
      "activations/layer13_attention_weight_max": 42.580116271972656,
      "activations/layer13_attention_weight_min": -31.38759994506836,
      "activations/layer14_attention_weight_max": 37.65601348876953,
      "activations/layer14_attention_weight_min": -30.261425018310547,
      "activations/layer15_attention_weight_max": 33.55490493774414,
      "activations/layer15_attention_weight_min": -29.191410064697266,
      "activations/layer16_attention_weight_max": 32.73754119873047,
      "activations/layer16_attention_weight_min": -29.72376823425293,
      "activations/layer17_attention_weight_max": 55.4840202331543,
      "activations/layer17_attention_weight_min": -45.04146957397461,
      "activations/layer18_attention_weight_max": 50.29605484008789,
      "activations/layer18_attention_weight_min": -35.78940963745117,
      "activations/layer19_attention_weight_max": 21.84358024597168,
      "activations/layer19_attention_weight_min": -19.993106842041016,
      "activations/layer1_attention_weight_max": 16.25941276550293,
      "activations/layer1_attention_weight_min": -14.546693801879883,
      "activations/layer20_attention_weight_max": 22.77347183227539,
      "activations/layer20_attention_weight_min": -20.905242919921875,
      "activations/layer21_attention_weight_max": 37.67906188964844,
      "activations/layer21_attention_weight_min": -22.307302474975586,
      "activations/layer22_attention_weight_max": 31.722869873046875,
      "activations/layer22_attention_weight_min": -23.484020233154297,
      "activations/layer23_attention_weight_max": 38.039180755615234,
      "activations/layer23_attention_weight_min": -23.463172912597656,
      "activations/layer2_attention_weight_max": 29.05950927734375,
      "activations/layer2_attention_weight_min": -29.803144454956055,
      "activations/layer3_attention_weight_max": 90.85302734375,
      "activations/layer3_attention_weight_min": -94.81409454345703,
      "activations/layer4_attention_weight_max": 100.10338592529297,
      "activations/layer4_attention_weight_min": -97.20012664794922,
      "activations/layer5_attention_weight_max": 79.91838073730469,
      "activations/layer5_attention_weight_min": -77.0023422241211,
      "activations/layer6_attention_weight_max": 55.893104553222656,
      "activations/layer6_attention_weight_min": -51.756553649902344,
      "activations/layer7_attention_weight_max": 83.4513168334961,
      "activations/layer7_attention_weight_min": -77.0511703491211,
      "activations/layer8_attention_weight_max": 45.70223617553711,
      "activations/layer8_attention_weight_min": -46.173545837402344,
      "activations/layer9_attention_weight_max": 51.931392669677734,
      "activations/layer9_attention_weight_min": -48.318546295166016,
      "epoch": 8.94,
      "learning_rate": 9.330946969696968e-05,
      "loss": 2.7999,
      "step": 153800
    },
    {
      "activations/layer0_attention_weight_max": 15.771858215332031,
      "activations/layer0_attention_weight_min": -14.247087478637695,
      "activations/layer10_attention_weight_max": 37.63482666015625,
      "activations/layer10_attention_weight_min": -35.907623291015625,
      "activations/layer11_attention_weight_max": 30.79119873046875,
      "activations/layer11_attention_weight_min": -33.02259063720703,
      "activations/layer12_attention_weight_max": 17.851150512695312,
      "activations/layer12_attention_weight_min": -26.131990432739258,
      "activations/layer13_attention_weight_max": 37.06849670410156,
      "activations/layer13_attention_weight_min": -30.827709197998047,
      "activations/layer14_attention_weight_max": 50.77847671508789,
      "activations/layer14_attention_weight_min": -31.137393951416016,
      "activations/layer15_attention_weight_max": 41.19854736328125,
      "activations/layer15_attention_weight_min": -28.420883178710938,
      "activations/layer16_attention_weight_max": 28.81907081604004,
      "activations/layer16_attention_weight_min": -26.832292556762695,
      "activations/layer17_attention_weight_max": 51.962223052978516,
      "activations/layer17_attention_weight_min": -42.690059661865234,
      "activations/layer18_attention_weight_max": 47.10123062133789,
      "activations/layer18_attention_weight_min": -35.80998992919922,
      "activations/layer19_attention_weight_max": 24.067968368530273,
      "activations/layer19_attention_weight_min": -21.242542266845703,
      "activations/layer1_attention_weight_max": 17.17905616760254,
      "activations/layer1_attention_weight_min": -14.709304809570312,
      "activations/layer20_attention_weight_max": 24.774852752685547,
      "activations/layer20_attention_weight_min": -21.123151779174805,
      "activations/layer21_attention_weight_max": 48.125946044921875,
      "activations/layer21_attention_weight_min": -28.107656478881836,
      "activations/layer22_attention_weight_max": 30.130558013916016,
      "activations/layer22_attention_weight_min": -28.46558952331543,
      "activations/layer23_attention_weight_max": 33.61724090576172,
      "activations/layer23_attention_weight_min": -25.277725219726562,
      "activations/layer2_attention_weight_max": 30.266477584838867,
      "activations/layer2_attention_weight_min": -29.435649871826172,
      "activations/layer3_attention_weight_max": 86.74578094482422,
      "activations/layer3_attention_weight_min": -94.86243438720703,
      "activations/layer4_attention_weight_max": 100.51079559326172,
      "activations/layer4_attention_weight_min": -94.33088684082031,
      "activations/layer5_attention_weight_max": 78.16488647460938,
      "activations/layer5_attention_weight_min": -73.90048217773438,
      "activations/layer6_attention_weight_max": 54.21846008300781,
      "activations/layer6_attention_weight_min": -52.28456115722656,
      "activations/layer7_attention_weight_max": 73.90017700195312,
      "activations/layer7_attention_weight_min": -73.32911682128906,
      "activations/layer8_attention_weight_max": 43.0081901550293,
      "activations/layer8_attention_weight_min": -43.219051361083984,
      "activations/layer9_attention_weight_max": 46.40638732910156,
      "activations/layer9_attention_weight_min": -46.605186462402344,
      "epoch": 8.94,
      "learning_rate": 9.32905303030303e-05,
      "loss": 2.7899,
      "step": 153850
    },
    {
      "activations/layer0_attention_weight_max": 14.771871566772461,
      "activations/layer0_attention_weight_min": -14.192092895507812,
      "activations/layer10_attention_weight_max": 39.59953689575195,
      "activations/layer10_attention_weight_min": -37.34611892700195,
      "activations/layer11_attention_weight_max": 32.744544982910156,
      "activations/layer11_attention_weight_min": -32.750732421875,
      "activations/layer12_attention_weight_max": 18.80416488647461,
      "activations/layer12_attention_weight_min": -24.91268539428711,
      "activations/layer13_attention_weight_max": 40.79480743408203,
      "activations/layer13_attention_weight_min": -31.91707420349121,
      "activations/layer14_attention_weight_max": 39.641597747802734,
      "activations/layer14_attention_weight_min": -30.432899475097656,
      "activations/layer15_attention_weight_max": 39.27003860473633,
      "activations/layer15_attention_weight_min": -27.9810733795166,
      "activations/layer16_attention_weight_max": 37.1685676574707,
      "activations/layer16_attention_weight_min": -27.753063201904297,
      "activations/layer17_attention_weight_max": 57.80799102783203,
      "activations/layer17_attention_weight_min": -43.73761749267578,
      "activations/layer18_attention_weight_max": 50.58424758911133,
      "activations/layer18_attention_weight_min": -37.6130485534668,
      "activations/layer19_attention_weight_max": 20.668590545654297,
      "activations/layer19_attention_weight_min": -22.13831901550293,
      "activations/layer1_attention_weight_max": 16.760887145996094,
      "activations/layer1_attention_weight_min": -15.830567359924316,
      "activations/layer20_attention_weight_max": 23.144309997558594,
      "activations/layer20_attention_weight_min": -22.886194229125977,
      "activations/layer21_attention_weight_max": 42.163597106933594,
      "activations/layer21_attention_weight_min": -27.86851692199707,
      "activations/layer22_attention_weight_max": 32.93391036987305,
      "activations/layer22_attention_weight_min": -26.72127342224121,
      "activations/layer23_attention_weight_max": 41.139957427978516,
      "activations/layer23_attention_weight_min": -22.701095581054688,
      "activations/layer2_attention_weight_max": 30.385894775390625,
      "activations/layer2_attention_weight_min": -30.36001205444336,
      "activations/layer3_attention_weight_max": 91.99954986572266,
      "activations/layer3_attention_weight_min": -94.77425384521484,
      "activations/layer4_attention_weight_max": 105.1148910522461,
      "activations/layer4_attention_weight_min": -101.3886947631836,
      "activations/layer5_attention_weight_max": 85.0416259765625,
      "activations/layer5_attention_weight_min": -82.55235290527344,
      "activations/layer6_attention_weight_max": 57.61693572998047,
      "activations/layer6_attention_weight_min": -57.93230438232422,
      "activations/layer7_attention_weight_max": 78.78076934814453,
      "activations/layer7_attention_weight_min": -73.6497802734375,
      "activations/layer8_attention_weight_max": 46.13457107543945,
      "activations/layer8_attention_weight_min": -48.74812316894531,
      "activations/layer9_attention_weight_max": 50.485572814941406,
      "activations/layer9_attention_weight_min": -51.31026077270508,
      "epoch": 8.94,
      "learning_rate": 9.327159090909089e-05,
      "loss": 2.7948,
      "step": 153900
    },
    {
      "activations/layer0_attention_weight_max": 15.725762367248535,
      "activations/layer0_attention_weight_min": -13.423606872558594,
      "activations/layer10_attention_weight_max": 43.93516159057617,
      "activations/layer10_attention_weight_min": -41.58989334106445,
      "activations/layer11_attention_weight_max": 35.53423309326172,
      "activations/layer11_attention_weight_min": -33.6198844909668,
      "activations/layer12_attention_weight_max": 31.262659072875977,
      "activations/layer12_attention_weight_min": -25.596567153930664,
      "activations/layer13_attention_weight_max": 44.556190490722656,
      "activations/layer13_attention_weight_min": -34.53602981567383,
      "activations/layer14_attention_weight_max": 35.820762634277344,
      "activations/layer14_attention_weight_min": -29.7270565032959,
      "activations/layer15_attention_weight_max": 34.788814544677734,
      "activations/layer15_attention_weight_min": -28.545394897460938,
      "activations/layer16_attention_weight_max": 30.395227432250977,
      "activations/layer16_attention_weight_min": -27.21661949157715,
      "activations/layer17_attention_weight_max": 54.36785888671875,
      "activations/layer17_attention_weight_min": -40.89200973510742,
      "activations/layer18_attention_weight_max": 47.5258903503418,
      "activations/layer18_attention_weight_min": -34.60404968261719,
      "activations/layer19_attention_weight_max": 22.129987716674805,
      "activations/layer19_attention_weight_min": -21.183448791503906,
      "activations/layer1_attention_weight_max": 16.797449111938477,
      "activations/layer1_attention_weight_min": -15.742144584655762,
      "activations/layer20_attention_weight_max": 21.59323501586914,
      "activations/layer20_attention_weight_min": -24.70360565185547,
      "activations/layer21_attention_weight_max": 33.30878448486328,
      "activations/layer21_attention_weight_min": -25.01836395263672,
      "activations/layer22_attention_weight_max": 30.17591094970703,
      "activations/layer22_attention_weight_min": -25.05765724182129,
      "activations/layer23_attention_weight_max": 34.545310974121094,
      "activations/layer23_attention_weight_min": -21.210498809814453,
      "activations/layer2_attention_weight_max": 31.091808319091797,
      "activations/layer2_attention_weight_min": -30.48381805419922,
      "activations/layer3_attention_weight_max": 89.33759307861328,
      "activations/layer3_attention_weight_min": -92.44575500488281,
      "activations/layer4_attention_weight_max": 102.07396697998047,
      "activations/layer4_attention_weight_min": -100.35291290283203,
      "activations/layer5_attention_weight_max": 83.88695526123047,
      "activations/layer5_attention_weight_min": -83.62814331054688,
      "activations/layer6_attention_weight_max": 55.6529426574707,
      "activations/layer6_attention_weight_min": -56.159637451171875,
      "activations/layer7_attention_weight_max": 77.6809310913086,
      "activations/layer7_attention_weight_min": -77.17515563964844,
      "activations/layer8_attention_weight_max": 50.84257125854492,
      "activations/layer8_attention_weight_min": -50.26538848876953,
      "activations/layer9_attention_weight_max": 52.061492919921875,
      "activations/layer9_attention_weight_min": -54.55226516723633,
      "epoch": 8.95,
      "learning_rate": 9.32526515151515e-05,
      "loss": 2.8007,
      "step": 153950
    },
    {
      "activations/layer0_attention_weight_max": 15.130661010742188,
      "activations/layer0_attention_weight_min": -13.613568305969238,
      "activations/layer10_attention_weight_max": 36.75313186645508,
      "activations/layer10_attention_weight_min": -36.80686950683594,
      "activations/layer11_attention_weight_max": 31.25305938720703,
      "activations/layer11_attention_weight_min": -32.094844818115234,
      "activations/layer12_attention_weight_max": 23.70468521118164,
      "activations/layer12_attention_weight_min": -22.204240798950195,
      "activations/layer13_attention_weight_max": 41.26133346557617,
      "activations/layer13_attention_weight_min": -33.29979705810547,
      "activations/layer14_attention_weight_max": 36.93647766113281,
      "activations/layer14_attention_weight_min": -31.213924407958984,
      "activations/layer15_attention_weight_max": 34.251564025878906,
      "activations/layer15_attention_weight_min": -30.85276985168457,
      "activations/layer16_attention_weight_max": 31.38115119934082,
      "activations/layer16_attention_weight_min": -30.204105377197266,
      "activations/layer17_attention_weight_max": 52.30669021606445,
      "activations/layer17_attention_weight_min": -41.66429138183594,
      "activations/layer18_attention_weight_max": 47.72722625732422,
      "activations/layer18_attention_weight_min": -36.0908317565918,
      "activations/layer19_attention_weight_max": 23.916074752807617,
      "activations/layer19_attention_weight_min": -21.72739601135254,
      "activations/layer1_attention_weight_max": 16.9653377532959,
      "activations/layer1_attention_weight_min": -15.24860668182373,
      "activations/layer20_attention_weight_max": 23.073728561401367,
      "activations/layer20_attention_weight_min": -20.957839965820312,
      "activations/layer21_attention_weight_max": 36.71698760986328,
      "activations/layer21_attention_weight_min": -23.215776443481445,
      "activations/layer22_attention_weight_max": 28.76221466064453,
      "activations/layer22_attention_weight_min": -25.223283767700195,
      "activations/layer23_attention_weight_max": 32.067569732666016,
      "activations/layer23_attention_weight_min": -23.587804794311523,
      "activations/layer2_attention_weight_max": 31.56672477722168,
      "activations/layer2_attention_weight_min": -30.416580200195312,
      "activations/layer3_attention_weight_max": 91.56425476074219,
      "activations/layer3_attention_weight_min": -93.84735107421875,
      "activations/layer4_attention_weight_max": 98.84284973144531,
      "activations/layer4_attention_weight_min": -98.39299011230469,
      "activations/layer5_attention_weight_max": 77.31107330322266,
      "activations/layer5_attention_weight_min": -76.98773193359375,
      "activations/layer6_attention_weight_max": 51.19990539550781,
      "activations/layer6_attention_weight_min": -50.788936614990234,
      "activations/layer7_attention_weight_max": 79.00878143310547,
      "activations/layer7_attention_weight_min": -73.94017791748047,
      "activations/layer8_attention_weight_max": 43.65990447998047,
      "activations/layer8_attention_weight_min": -45.542640686035156,
      "activations/layer9_attention_weight_max": 46.8609733581543,
      "activations/layer9_attention_weight_min": -51.35384750366211,
      "epoch": 8.95,
      "learning_rate": 9.323371212121212e-05,
      "loss": 2.8225,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.4844,
      "eval_samples_per_second": 506.108,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.4844,
      "eval_openwebtext_samples_per_second": 506.108,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_wikitext_loss": 2.990234375,
      "eval_wikitext_ppl": 19.89034374461794,
      "eval_wikitext_runtime": 2.0094,
      "eval_wikitext_samples_per_second": 226.93,
      "step": 154000
    },
    {
      "epoch": 8.95,
      "eval_lambada_loss": 2.74609375,
      "eval_lambada_ppl": 15.581647041956762,
      "eval_lambada_runtime": 9.5513,
      "eval_lambada_samples_per_second": 509.771,
      "step": 154000
    },
    {
      "activations/layer0_attention_weight_max": 14.926556587219238,
      "activations/layer0_attention_weight_min": -13.886993408203125,
      "activations/layer10_attention_weight_max": 38.22201156616211,
      "activations/layer10_attention_weight_min": -36.88886260986328,
      "activations/layer11_attention_weight_max": 31.792774200439453,
      "activations/layer11_attention_weight_min": -29.991344451904297,
      "activations/layer12_attention_weight_max": 17.642305374145508,
      "activations/layer12_attention_weight_min": -26.391271591186523,
      "activations/layer13_attention_weight_max": 40.02767562866211,
      "activations/layer13_attention_weight_min": -33.49102783203125,
      "activations/layer14_attention_weight_max": 41.065155029296875,
      "activations/layer14_attention_weight_min": -31.509414672851562,
      "activations/layer15_attention_weight_max": 32.47452926635742,
      "activations/layer15_attention_weight_min": -29.727718353271484,
      "activations/layer16_attention_weight_max": 30.974781036376953,
      "activations/layer16_attention_weight_min": -27.865785598754883,
      "activations/layer17_attention_weight_max": 52.37892532348633,
      "activations/layer17_attention_weight_min": -45.890159606933594,
      "activations/layer18_attention_weight_max": 45.70164108276367,
      "activations/layer18_attention_weight_min": -40.048370361328125,
      "activations/layer19_attention_weight_max": 20.93788719177246,
      "activations/layer19_attention_weight_min": -21.953123092651367,
      "activations/layer1_attention_weight_max": 16.312183380126953,
      "activations/layer1_attention_weight_min": -16.728126525878906,
      "activations/layer20_attention_weight_max": 20.93316078186035,
      "activations/layer20_attention_weight_min": -20.55826759338379,
      "activations/layer21_attention_weight_max": 34.9339485168457,
      "activations/layer21_attention_weight_min": -24.024160385131836,
      "activations/layer22_attention_weight_max": 30.409482955932617,
      "activations/layer22_attention_weight_min": -25.305126190185547,
      "activations/layer23_attention_weight_max": 36.25544738769531,
      "activations/layer23_attention_weight_min": -23.0356502532959,
      "activations/layer2_attention_weight_max": 32.57243347167969,
      "activations/layer2_attention_weight_min": -30.739212036132812,
      "activations/layer3_attention_weight_max": 94.02538299560547,
      "activations/layer3_attention_weight_min": -97.99459838867188,
      "activations/layer4_attention_weight_max": 102.88153076171875,
      "activations/layer4_attention_weight_min": -98.4176254272461,
      "activations/layer5_attention_weight_max": 82.19358825683594,
      "activations/layer5_attention_weight_min": -79.06462097167969,
      "activations/layer6_attention_weight_max": 55.47831726074219,
      "activations/layer6_attention_weight_min": -56.39611053466797,
      "activations/layer7_attention_weight_max": 81.57034301757812,
      "activations/layer7_attention_weight_min": -72.99720001220703,
      "activations/layer8_attention_weight_max": 45.60223388671875,
      "activations/layer8_attention_weight_min": -46.42667770385742,
      "activations/layer9_attention_weight_max": 57.82416534423828,
      "activations/layer9_attention_weight_min": -52.006195068359375,
      "epoch": 8.95,
      "learning_rate": 9.321477272727272e-05,
      "loss": 2.7955,
      "step": 154050
    },
    {
      "activations/layer0_attention_weight_max": 15.403304100036621,
      "activations/layer0_attention_weight_min": -13.940926551818848,
      "activations/layer10_attention_weight_max": 40.4940299987793,
      "activations/layer10_attention_weight_min": -35.372230529785156,
      "activations/layer11_attention_weight_max": 36.311248779296875,
      "activations/layer11_attention_weight_min": -31.34429931640625,
      "activations/layer12_attention_weight_max": 20.356897354125977,
      "activations/layer12_attention_weight_min": -24.954687118530273,
      "activations/layer13_attention_weight_max": 46.0998420715332,
      "activations/layer13_attention_weight_min": -32.817108154296875,
      "activations/layer14_attention_weight_max": 40.246192932128906,
      "activations/layer14_attention_weight_min": -31.228050231933594,
      "activations/layer15_attention_weight_max": 39.361080169677734,
      "activations/layer15_attention_weight_min": -27.950706481933594,
      "activations/layer16_attention_weight_max": 33.03297805786133,
      "activations/layer16_attention_weight_min": -29.548046112060547,
      "activations/layer17_attention_weight_max": 57.50374221801758,
      "activations/layer17_attention_weight_min": -43.81645584106445,
      "activations/layer18_attention_weight_max": 55.471397399902344,
      "activations/layer18_attention_weight_min": -35.72576141357422,
      "activations/layer19_attention_weight_max": 26.764013290405273,
      "activations/layer19_attention_weight_min": -21.369953155517578,
      "activations/layer1_attention_weight_max": 16.452587127685547,
      "activations/layer1_attention_weight_min": -14.804448127746582,
      "activations/layer20_attention_weight_max": 25.963388442993164,
      "activations/layer20_attention_weight_min": -19.504661560058594,
      "activations/layer21_attention_weight_max": 50.78594970703125,
      "activations/layer21_attention_weight_min": -22.8292293548584,
      "activations/layer22_attention_weight_max": 39.72621536254883,
      "activations/layer22_attention_weight_min": -23.459991455078125,
      "activations/layer23_attention_weight_max": 46.742279052734375,
      "activations/layer23_attention_weight_min": -21.609725952148438,
      "activations/layer2_attention_weight_max": 31.22079849243164,
      "activations/layer2_attention_weight_min": -30.264219284057617,
      "activations/layer3_attention_weight_max": 91.54935455322266,
      "activations/layer3_attention_weight_min": -93.588134765625,
      "activations/layer4_attention_weight_max": 102.13719940185547,
      "activations/layer4_attention_weight_min": -96.00654602050781,
      "activations/layer5_attention_weight_max": 82.99954986572266,
      "activations/layer5_attention_weight_min": -79.00735473632812,
      "activations/layer6_attention_weight_max": 58.72613525390625,
      "activations/layer6_attention_weight_min": -53.123538970947266,
      "activations/layer7_attention_weight_max": 79.31829833984375,
      "activations/layer7_attention_weight_min": -73.39253997802734,
      "activations/layer8_attention_weight_max": 45.99413299560547,
      "activations/layer8_attention_weight_min": -45.732818603515625,
      "activations/layer9_attention_weight_max": 51.70708084106445,
      "activations/layer9_attention_weight_min": -48.31550979614258,
      "epoch": 8.95,
      "learning_rate": 9.319583333333332e-05,
      "loss": 2.8161,
      "step": 154100
    },
    {
      "activations/layer0_attention_weight_max": 15.176263809204102,
      "activations/layer0_attention_weight_min": -13.790950775146484,
      "activations/layer10_attention_weight_max": 36.17757034301758,
      "activations/layer10_attention_weight_min": -33.567138671875,
      "activations/layer11_attention_weight_max": 31.907672882080078,
      "activations/layer11_attention_weight_min": -31.433887481689453,
      "activations/layer12_attention_weight_max": 19.050344467163086,
      "activations/layer12_attention_weight_min": -25.48697280883789,
      "activations/layer13_attention_weight_max": 39.78582763671875,
      "activations/layer13_attention_weight_min": -29.434158325195312,
      "activations/layer14_attention_weight_max": 37.66011047363281,
      "activations/layer14_attention_weight_min": -29.863380432128906,
      "activations/layer15_attention_weight_max": 38.857521057128906,
      "activations/layer15_attention_weight_min": -27.9642391204834,
      "activations/layer16_attention_weight_max": 32.79694366455078,
      "activations/layer16_attention_weight_min": -27.635099411010742,
      "activations/layer17_attention_weight_max": 52.52351379394531,
      "activations/layer17_attention_weight_min": -40.89957046508789,
      "activations/layer18_attention_weight_max": 50.6223258972168,
      "activations/layer18_attention_weight_min": -37.29010009765625,
      "activations/layer19_attention_weight_max": 23.608728408813477,
      "activations/layer19_attention_weight_min": -22.336383819580078,
      "activations/layer1_attention_weight_max": 17.606842041015625,
      "activations/layer1_attention_weight_min": -14.358431816101074,
      "activations/layer20_attention_weight_max": 24.511423110961914,
      "activations/layer20_attention_weight_min": -24.95909881591797,
      "activations/layer21_attention_weight_max": 41.60858917236328,
      "activations/layer21_attention_weight_min": -23.254161834716797,
      "activations/layer22_attention_weight_max": 33.000244140625,
      "activations/layer22_attention_weight_min": -25.81828498840332,
      "activations/layer23_attention_weight_max": 38.03712463378906,
      "activations/layer23_attention_weight_min": -23.706134796142578,
      "activations/layer2_attention_weight_max": 31.21332359313965,
      "activations/layer2_attention_weight_min": -30.374595642089844,
      "activations/layer3_attention_weight_max": 92.6249771118164,
      "activations/layer3_attention_weight_min": -91.7716064453125,
      "activations/layer4_attention_weight_max": 98.98052215576172,
      "activations/layer4_attention_weight_min": -95.98269653320312,
      "activations/layer5_attention_weight_max": 76.4622802734375,
      "activations/layer5_attention_weight_min": -74.02891540527344,
      "activations/layer6_attention_weight_max": 51.809452056884766,
      "activations/layer6_attention_weight_min": -56.424076080322266,
      "activations/layer7_attention_weight_max": 74.60154724121094,
      "activations/layer7_attention_weight_min": -73.39359283447266,
      "activations/layer8_attention_weight_max": 45.22099685668945,
      "activations/layer8_attention_weight_min": -45.59376525878906,
      "activations/layer9_attention_weight_max": 49.4074821472168,
      "activations/layer9_attention_weight_min": -47.2264404296875,
      "epoch": 8.96,
      "learning_rate": 9.317689393939393e-05,
      "loss": 2.8127,
      "step": 154150
    },
    {
      "activations/layer0_attention_weight_max": 15.966922760009766,
      "activations/layer0_attention_weight_min": -14.183884620666504,
      "activations/layer10_attention_weight_max": 34.50428009033203,
      "activations/layer10_attention_weight_min": -36.74555206298828,
      "activations/layer11_attention_weight_max": 30.234466552734375,
      "activations/layer11_attention_weight_min": -31.990516662597656,
      "activations/layer12_attention_weight_max": 18.381662368774414,
      "activations/layer12_attention_weight_min": -24.143329620361328,
      "activations/layer13_attention_weight_max": 39.607322692871094,
      "activations/layer13_attention_weight_min": -33.55337142944336,
      "activations/layer14_attention_weight_max": 35.4780387878418,
      "activations/layer14_attention_weight_min": -31.888916015625,
      "activations/layer15_attention_weight_max": 34.954307556152344,
      "activations/layer15_attention_weight_min": -30.212129592895508,
      "activations/layer16_attention_weight_max": 29.685787200927734,
      "activations/layer16_attention_weight_min": -28.80211067199707,
      "activations/layer17_attention_weight_max": 49.35209274291992,
      "activations/layer17_attention_weight_min": -45.80415725708008,
      "activations/layer18_attention_weight_max": 40.9923210144043,
      "activations/layer18_attention_weight_min": -36.672325134277344,
      "activations/layer19_attention_weight_max": 19.55826187133789,
      "activations/layer19_attention_weight_min": -21.007335662841797,
      "activations/layer1_attention_weight_max": 17.002967834472656,
      "activations/layer1_attention_weight_min": -13.84561824798584,
      "activations/layer20_attention_weight_max": 19.247257232666016,
      "activations/layer20_attention_weight_min": -21.892724990844727,
      "activations/layer21_attention_weight_max": 31.109800338745117,
      "activations/layer21_attention_weight_min": -25.01914405822754,
      "activations/layer22_attention_weight_max": 30.089908599853516,
      "activations/layer22_attention_weight_min": -27.6187744140625,
      "activations/layer23_attention_weight_max": 33.82935333251953,
      "activations/layer23_attention_weight_min": -23.921079635620117,
      "activations/layer2_attention_weight_max": 28.638172149658203,
      "activations/layer2_attention_weight_min": -24.852094650268555,
      "activations/layer3_attention_weight_max": 80.83521270751953,
      "activations/layer3_attention_weight_min": -86.59286499023438,
      "activations/layer4_attention_weight_max": 99.64319610595703,
      "activations/layer4_attention_weight_min": -96.21141815185547,
      "activations/layer5_attention_weight_max": 79.49337768554688,
      "activations/layer5_attention_weight_min": -76.30534362792969,
      "activations/layer6_attention_weight_max": 51.39232635498047,
      "activations/layer6_attention_weight_min": -53.83540725708008,
      "activations/layer7_attention_weight_max": 78.3416519165039,
      "activations/layer7_attention_weight_min": -74.94416809082031,
      "activations/layer8_attention_weight_max": 43.255271911621094,
      "activations/layer8_attention_weight_min": -43.950050354003906,
      "activations/layer9_attention_weight_max": 47.88436508178711,
      "activations/layer9_attention_weight_min": -50.025543212890625,
      "epoch": 8.96,
      "learning_rate": 9.315795454545454e-05,
      "loss": 2.8106,
      "step": 154200
    },
    {
      "activations/layer0_attention_weight_max": 15.03749942779541,
      "activations/layer0_attention_weight_min": -13.795900344848633,
      "activations/layer10_attention_weight_max": 36.54319763183594,
      "activations/layer10_attention_weight_min": -35.537601470947266,
      "activations/layer11_attention_weight_max": 31.631269454956055,
      "activations/layer11_attention_weight_min": -29.65570831298828,
      "activations/layer12_attention_weight_max": 18.650211334228516,
      "activations/layer12_attention_weight_min": -23.058944702148438,
      "activations/layer13_attention_weight_max": 47.2024040222168,
      "activations/layer13_attention_weight_min": -32.1345100402832,
      "activations/layer14_attention_weight_max": 38.08450698852539,
      "activations/layer14_attention_weight_min": -31.391992568969727,
      "activations/layer15_attention_weight_max": 38.140621185302734,
      "activations/layer15_attention_weight_min": -30.037229537963867,
      "activations/layer16_attention_weight_max": 31.883459091186523,
      "activations/layer16_attention_weight_min": -27.854272842407227,
      "activations/layer17_attention_weight_max": 53.086097717285156,
      "activations/layer17_attention_weight_min": -43.02531814575195,
      "activations/layer18_attention_weight_max": 46.2443962097168,
      "activations/layer18_attention_weight_min": -37.77824783325195,
      "activations/layer19_attention_weight_max": 21.594717025756836,
      "activations/layer19_attention_weight_min": -20.598222732543945,
      "activations/layer1_attention_weight_max": 17.6016902923584,
      "activations/layer1_attention_weight_min": -14.758973121643066,
      "activations/layer20_attention_weight_max": 21.039752960205078,
      "activations/layer20_attention_weight_min": -20.676353454589844,
      "activations/layer21_attention_weight_max": 44.33927917480469,
      "activations/layer21_attention_weight_min": -28.125,
      "activations/layer22_attention_weight_max": 31.93097496032715,
      "activations/layer22_attention_weight_min": -25.602062225341797,
      "activations/layer23_attention_weight_max": 39.87699890136719,
      "activations/layer23_attention_weight_min": -23.943958282470703,
      "activations/layer2_attention_weight_max": 30.462535858154297,
      "activations/layer2_attention_weight_min": -28.550216674804688,
      "activations/layer3_attention_weight_max": 86.50838470458984,
      "activations/layer3_attention_weight_min": -92.01438903808594,
      "activations/layer4_attention_weight_max": 95.13335418701172,
      "activations/layer4_attention_weight_min": -93.61984252929688,
      "activations/layer5_attention_weight_max": 76.3721694946289,
      "activations/layer5_attention_weight_min": -74.73355865478516,
      "activations/layer6_attention_weight_max": 51.275291442871094,
      "activations/layer6_attention_weight_min": -55.04269790649414,
      "activations/layer7_attention_weight_max": 73.46675872802734,
      "activations/layer7_attention_weight_min": -72.31087493896484,
      "activations/layer8_attention_weight_max": 42.44337844848633,
      "activations/layer8_attention_weight_min": -48.057891845703125,
      "activations/layer9_attention_weight_max": 48.35612869262695,
      "activations/layer9_attention_weight_min": -49.52314758300781,
      "epoch": 8.96,
      "learning_rate": 9.313901515151515e-05,
      "loss": 2.8183,
      "step": 154250
    },
    {
      "activations/layer0_attention_weight_max": 16.413103103637695,
      "activations/layer0_attention_weight_min": -12.839909553527832,
      "activations/layer10_attention_weight_max": 36.8392219543457,
      "activations/layer10_attention_weight_min": -33.84720230102539,
      "activations/layer11_attention_weight_max": 31.19522476196289,
      "activations/layer11_attention_weight_min": -33.245521545410156,
      "activations/layer12_attention_weight_max": 20.83693504333496,
      "activations/layer12_attention_weight_min": -26.685134887695312,
      "activations/layer13_attention_weight_max": 41.992286682128906,
      "activations/layer13_attention_weight_min": -27.304872512817383,
      "activations/layer14_attention_weight_max": 38.640750885009766,
      "activations/layer14_attention_weight_min": -31.21637535095215,
      "activations/layer15_attention_weight_max": 35.36029815673828,
      "activations/layer15_attention_weight_min": -29.715303421020508,
      "activations/layer16_attention_weight_max": 42.82945251464844,
      "activations/layer16_attention_weight_min": -31.516210556030273,
      "activations/layer17_attention_weight_max": 54.92051315307617,
      "activations/layer17_attention_weight_min": -45.710533142089844,
      "activations/layer18_attention_weight_max": 48.11844253540039,
      "activations/layer18_attention_weight_min": -37.75801467895508,
      "activations/layer19_attention_weight_max": 20.693710327148438,
      "activations/layer19_attention_weight_min": -20.69115447998047,
      "activations/layer1_attention_weight_max": 16.944948196411133,
      "activations/layer1_attention_weight_min": -13.436249732971191,
      "activations/layer20_attention_weight_max": 23.567148208618164,
      "activations/layer20_attention_weight_min": -19.609941482543945,
      "activations/layer21_attention_weight_max": 41.80147933959961,
      "activations/layer21_attention_weight_min": -22.84795570373535,
      "activations/layer22_attention_weight_max": 29.94220542907715,
      "activations/layer22_attention_weight_min": -24.668903350830078,
      "activations/layer23_attention_weight_max": 38.00172424316406,
      "activations/layer23_attention_weight_min": -22.972517013549805,
      "activations/layer2_attention_weight_max": 28.786542892456055,
      "activations/layer2_attention_weight_min": -27.083906173706055,
      "activations/layer3_attention_weight_max": 86.58573913574219,
      "activations/layer3_attention_weight_min": -87.48335266113281,
      "activations/layer4_attention_weight_max": 98.84366607666016,
      "activations/layer4_attention_weight_min": -91.34803771972656,
      "activations/layer5_attention_weight_max": 79.7608871459961,
      "activations/layer5_attention_weight_min": -78.99907684326172,
      "activations/layer6_attention_weight_max": 54.21936798095703,
      "activations/layer6_attention_weight_min": -52.111019134521484,
      "activations/layer7_attention_weight_max": 78.86891174316406,
      "activations/layer7_attention_weight_min": -73.38164520263672,
      "activations/layer8_attention_weight_max": 49.06050491333008,
      "activations/layer8_attention_weight_min": -46.26961135864258,
      "activations/layer9_attention_weight_max": 50.90507125854492,
      "activations/layer9_attention_weight_min": -47.58744430541992,
      "epoch": 8.97,
      "learning_rate": 9.312007575757575e-05,
      "loss": 2.7995,
      "step": 154300
    },
    {
      "activations/layer0_attention_weight_max": 14.722308158874512,
      "activations/layer0_attention_weight_min": -13.272613525390625,
      "activations/layer10_attention_weight_max": 33.78329086303711,
      "activations/layer10_attention_weight_min": -34.49212646484375,
      "activations/layer11_attention_weight_max": 29.583478927612305,
      "activations/layer11_attention_weight_min": -29.28049087524414,
      "activations/layer12_attention_weight_max": 18.91483497619629,
      "activations/layer12_attention_weight_min": -24.33405113220215,
      "activations/layer13_attention_weight_max": 38.9042854309082,
      "activations/layer13_attention_weight_min": -28.091175079345703,
      "activations/layer14_attention_weight_max": 37.24300765991211,
      "activations/layer14_attention_weight_min": -29.817882537841797,
      "activations/layer15_attention_weight_max": 33.248260498046875,
      "activations/layer15_attention_weight_min": -28.96078109741211,
      "activations/layer16_attention_weight_max": 30.356449127197266,
      "activations/layer16_attention_weight_min": -26.557235717773438,
      "activations/layer17_attention_weight_max": 53.7741584777832,
      "activations/layer17_attention_weight_min": -40.6237907409668,
      "activations/layer18_attention_weight_max": 43.27239990234375,
      "activations/layer18_attention_weight_min": -37.542293548583984,
      "activations/layer19_attention_weight_max": 21.96221160888672,
      "activations/layer19_attention_weight_min": -21.90486717224121,
      "activations/layer1_attention_weight_max": 17.222726821899414,
      "activations/layer1_attention_weight_min": -15.700857162475586,
      "activations/layer20_attention_weight_max": 21.290502548217773,
      "activations/layer20_attention_weight_min": -20.764551162719727,
      "activations/layer21_attention_weight_max": 38.94495391845703,
      "activations/layer21_attention_weight_min": -27.502641677856445,
      "activations/layer22_attention_weight_max": 29.63704490661621,
      "activations/layer22_attention_weight_min": -23.832862854003906,
      "activations/layer23_attention_weight_max": 35.14894104003906,
      "activations/layer23_attention_weight_min": -29.664358139038086,
      "activations/layer2_attention_weight_max": 29.996387481689453,
      "activations/layer2_attention_weight_min": -28.636381149291992,
      "activations/layer3_attention_weight_max": 83.26158905029297,
      "activations/layer3_attention_weight_min": -93.46980285644531,
      "activations/layer4_attention_weight_max": 96.99726104736328,
      "activations/layer4_attention_weight_min": -94.7314682006836,
      "activations/layer5_attention_weight_max": 81.65736389160156,
      "activations/layer5_attention_weight_min": -75.27061462402344,
      "activations/layer6_attention_weight_max": 54.44889450073242,
      "activations/layer6_attention_weight_min": -51.25273132324219,
      "activations/layer7_attention_weight_max": 73.2007064819336,
      "activations/layer7_attention_weight_min": -71.68621063232422,
      "activations/layer8_attention_weight_max": 43.34293746948242,
      "activations/layer8_attention_weight_min": -44.05305099487305,
      "activations/layer9_attention_weight_max": 48.47981262207031,
      "activations/layer9_attention_weight_min": -48.8809928894043,
      "epoch": 8.97,
      "learning_rate": 9.310113636363635e-05,
      "loss": 2.8051,
      "step": 154350
    },
    {
      "activations/layer0_attention_weight_max": 16.256492614746094,
      "activations/layer0_attention_weight_min": -13.401408195495605,
      "activations/layer10_attention_weight_max": 36.150238037109375,
      "activations/layer10_attention_weight_min": -35.70275115966797,
      "activations/layer11_attention_weight_max": 29.76491928100586,
      "activations/layer11_attention_weight_min": -33.18836975097656,
      "activations/layer12_attention_weight_max": 18.0545654296875,
      "activations/layer12_attention_weight_min": -25.664627075195312,
      "activations/layer13_attention_weight_max": 37.194149017333984,
      "activations/layer13_attention_weight_min": -30.43767738342285,
      "activations/layer14_attention_weight_max": 37.08064651489258,
      "activations/layer14_attention_weight_min": -29.767314910888672,
      "activations/layer15_attention_weight_max": 33.917572021484375,
      "activations/layer15_attention_weight_min": -29.010652542114258,
      "activations/layer16_attention_weight_max": 33.2427978515625,
      "activations/layer16_attention_weight_min": -29.241228103637695,
      "activations/layer17_attention_weight_max": 53.90952682495117,
      "activations/layer17_attention_weight_min": -44.04595184326172,
      "activations/layer18_attention_weight_max": 49.74716567993164,
      "activations/layer18_attention_weight_min": -38.842952728271484,
      "activations/layer19_attention_weight_max": 22.24384117126465,
      "activations/layer19_attention_weight_min": -22.462726593017578,
      "activations/layer1_attention_weight_max": 17.285131454467773,
      "activations/layer1_attention_weight_min": -17.60784339904785,
      "activations/layer20_attention_weight_max": 22.153491973876953,
      "activations/layer20_attention_weight_min": -26.57001304626465,
      "activations/layer21_attention_weight_max": 36.029640197753906,
      "activations/layer21_attention_weight_min": -25.884733200073242,
      "activations/layer22_attention_weight_max": 30.139253616333008,
      "activations/layer22_attention_weight_min": -26.080230712890625,
      "activations/layer23_attention_weight_max": 36.63711929321289,
      "activations/layer23_attention_weight_min": -25.56220245361328,
      "activations/layer2_attention_weight_max": 31.353328704833984,
      "activations/layer2_attention_weight_min": -29.42963981628418,
      "activations/layer3_attention_weight_max": 87.29994201660156,
      "activations/layer3_attention_weight_min": -89.90552520751953,
      "activations/layer4_attention_weight_max": 98.99124145507812,
      "activations/layer4_attention_weight_min": -92.17324829101562,
      "activations/layer5_attention_weight_max": 81.17290496826172,
      "activations/layer5_attention_weight_min": -78.36602020263672,
      "activations/layer6_attention_weight_max": 54.687435150146484,
      "activations/layer6_attention_weight_min": -52.681949615478516,
      "activations/layer7_attention_weight_max": 74.17900085449219,
      "activations/layer7_attention_weight_min": -78.39665985107422,
      "activations/layer8_attention_weight_max": 41.79785919189453,
      "activations/layer8_attention_weight_min": -47.47720718383789,
      "activations/layer9_attention_weight_max": 46.62777328491211,
      "activations/layer9_attention_weight_min": -47.23176574707031,
      "epoch": 8.97,
      "learning_rate": 9.308219696969697e-05,
      "loss": 2.7903,
      "step": 154400
    },
    {
      "activations/layer0_attention_weight_max": 14.806963920593262,
      "activations/layer0_attention_weight_min": -13.390689849853516,
      "activations/layer10_attention_weight_max": 36.4791374206543,
      "activations/layer10_attention_weight_min": -35.56684112548828,
      "activations/layer11_attention_weight_max": 30.982389450073242,
      "activations/layer11_attention_weight_min": -33.76959228515625,
      "activations/layer12_attention_weight_max": 21.89923667907715,
      "activations/layer12_attention_weight_min": -22.767993927001953,
      "activations/layer13_attention_weight_max": 41.03429412841797,
      "activations/layer13_attention_weight_min": -28.508800506591797,
      "activations/layer14_attention_weight_max": 43.26140213012695,
      "activations/layer14_attention_weight_min": -32.05315017700195,
      "activations/layer15_attention_weight_max": 36.49020767211914,
      "activations/layer15_attention_weight_min": -30.341331481933594,
      "activations/layer16_attention_weight_max": 32.123085021972656,
      "activations/layer16_attention_weight_min": -29.932207107543945,
      "activations/layer17_attention_weight_max": 56.22001266479492,
      "activations/layer17_attention_weight_min": -42.57891082763672,
      "activations/layer18_attention_weight_max": 49.45144271850586,
      "activations/layer18_attention_weight_min": -37.71385955810547,
      "activations/layer19_attention_weight_max": 20.572355270385742,
      "activations/layer19_attention_weight_min": -20.80818748474121,
      "activations/layer1_attention_weight_max": 18.45747947692871,
      "activations/layer1_attention_weight_min": -15.560694694519043,
      "activations/layer20_attention_weight_max": 21.15998649597168,
      "activations/layer20_attention_weight_min": -26.204988479614258,
      "activations/layer21_attention_weight_max": 38.700408935546875,
      "activations/layer21_attention_weight_min": -23.322080612182617,
      "activations/layer22_attention_weight_max": 28.987546920776367,
      "activations/layer22_attention_weight_min": -26.086193084716797,
      "activations/layer23_attention_weight_max": 41.69646453857422,
      "activations/layer23_attention_weight_min": -27.929733276367188,
      "activations/layer2_attention_weight_max": 31.464942932128906,
      "activations/layer2_attention_weight_min": -29.811767578125,
      "activations/layer3_attention_weight_max": 97.10694885253906,
      "activations/layer3_attention_weight_min": -97.8532485961914,
      "activations/layer4_attention_weight_max": 104.75548553466797,
      "activations/layer4_attention_weight_min": -100.13619232177734,
      "activations/layer5_attention_weight_max": 81.36161804199219,
      "activations/layer5_attention_weight_min": -78.84815216064453,
      "activations/layer6_attention_weight_max": 56.43517303466797,
      "activations/layer6_attention_weight_min": -54.74810028076172,
      "activations/layer7_attention_weight_max": 82.53333282470703,
      "activations/layer7_attention_weight_min": -75.11680603027344,
      "activations/layer8_attention_weight_max": 46.92875671386719,
      "activations/layer8_attention_weight_min": -49.1854362487793,
      "activations/layer9_attention_weight_max": 53.76383590698242,
      "activations/layer9_attention_weight_min": -50.85958480834961,
      "epoch": 8.97,
      "learning_rate": 9.306325757575756e-05,
      "loss": 2.8043,
      "step": 154450
    },
    {
      "activations/layer0_attention_weight_max": 15.263232231140137,
      "activations/layer0_attention_weight_min": -13.570990562438965,
      "activations/layer10_attention_weight_max": 37.91136169433594,
      "activations/layer10_attention_weight_min": -34.9124755859375,
      "activations/layer11_attention_weight_max": 30.560842514038086,
      "activations/layer11_attention_weight_min": -31.783641815185547,
      "activations/layer12_attention_weight_max": 16.856292724609375,
      "activations/layer12_attention_weight_min": -26.158079147338867,
      "activations/layer13_attention_weight_max": 43.69459533691406,
      "activations/layer13_attention_weight_min": -37.59403991699219,
      "activations/layer14_attention_weight_max": 46.76255798339844,
      "activations/layer14_attention_weight_min": -36.591094970703125,
      "activations/layer15_attention_weight_max": 35.174293518066406,
      "activations/layer15_attention_weight_min": -30.981794357299805,
      "activations/layer16_attention_weight_max": 27.852312088012695,
      "activations/layer16_attention_weight_min": -29.156511306762695,
      "activations/layer17_attention_weight_max": 49.08346176147461,
      "activations/layer17_attention_weight_min": -43.15806198120117,
      "activations/layer18_attention_weight_max": 45.29422378540039,
      "activations/layer18_attention_weight_min": -36.6217155456543,
      "activations/layer19_attention_weight_max": 20.68950843811035,
      "activations/layer19_attention_weight_min": -21.126184463500977,
      "activations/layer1_attention_weight_max": 18.188579559326172,
      "activations/layer1_attention_weight_min": -14.822327613830566,
      "activations/layer20_attention_weight_max": 22.619932174682617,
      "activations/layer20_attention_weight_min": -25.639564514160156,
      "activations/layer21_attention_weight_max": 34.40424346923828,
      "activations/layer21_attention_weight_min": -24.10039710998535,
      "activations/layer22_attention_weight_max": 29.285419464111328,
      "activations/layer22_attention_weight_min": -24.462135314941406,
      "activations/layer23_attention_weight_max": 34.07477569580078,
      "activations/layer23_attention_weight_min": -27.855621337890625,
      "activations/layer2_attention_weight_max": 30.110183715820312,
      "activations/layer2_attention_weight_min": -29.471141815185547,
      "activations/layer3_attention_weight_max": 88.93560028076172,
      "activations/layer3_attention_weight_min": -93.80716705322266,
      "activations/layer4_attention_weight_max": 100.79422760009766,
      "activations/layer4_attention_weight_min": -96.80269622802734,
      "activations/layer5_attention_weight_max": 77.84817504882812,
      "activations/layer5_attention_weight_min": -73.0404281616211,
      "activations/layer6_attention_weight_max": 53.02253341674805,
      "activations/layer6_attention_weight_min": -53.09329605102539,
      "activations/layer7_attention_weight_max": 78.20177459716797,
      "activations/layer7_attention_weight_min": -71.07325744628906,
      "activations/layer8_attention_weight_max": 45.98860549926758,
      "activations/layer8_attention_weight_min": -43.43400955200195,
      "activations/layer9_attention_weight_max": 51.475582122802734,
      "activations/layer9_attention_weight_min": -50.07533264160156,
      "epoch": 8.98,
      "learning_rate": 9.304431818181817e-05,
      "loss": 2.7984,
      "step": 154500
    },
    {
      "activations/layer0_attention_weight_max": 15.538581848144531,
      "activations/layer0_attention_weight_min": -13.769596099853516,
      "activations/layer10_attention_weight_max": 37.7736701965332,
      "activations/layer10_attention_weight_min": -36.49998092651367,
      "activations/layer11_attention_weight_max": 30.944717407226562,
      "activations/layer11_attention_weight_min": -31.155731201171875,
      "activations/layer12_attention_weight_max": 19.282793045043945,
      "activations/layer12_attention_weight_min": -23.137041091918945,
      "activations/layer13_attention_weight_max": 41.20383834838867,
      "activations/layer13_attention_weight_min": -32.13210678100586,
      "activations/layer14_attention_weight_max": 42.99045181274414,
      "activations/layer14_attention_weight_min": -32.04816436767578,
      "activations/layer15_attention_weight_max": 40.0007438659668,
      "activations/layer15_attention_weight_min": -30.111726760864258,
      "activations/layer16_attention_weight_max": 31.63132667541504,
      "activations/layer16_attention_weight_min": -28.52657699584961,
      "activations/layer17_attention_weight_max": 52.65452194213867,
      "activations/layer17_attention_weight_min": -43.50244903564453,
      "activations/layer18_attention_weight_max": 46.77540588378906,
      "activations/layer18_attention_weight_min": -37.788902282714844,
      "activations/layer19_attention_weight_max": 21.293031692504883,
      "activations/layer19_attention_weight_min": -22.573108673095703,
      "activations/layer1_attention_weight_max": 16.860897064208984,
      "activations/layer1_attention_weight_min": -15.481481552124023,
      "activations/layer20_attention_weight_max": 21.98411750793457,
      "activations/layer20_attention_weight_min": -22.366456985473633,
      "activations/layer21_attention_weight_max": 40.02192306518555,
      "activations/layer21_attention_weight_min": -26.248239517211914,
      "activations/layer22_attention_weight_max": 30.027639389038086,
      "activations/layer22_attention_weight_min": -26.43581199645996,
      "activations/layer23_attention_weight_max": 36.515220642089844,
      "activations/layer23_attention_weight_min": -25.676647186279297,
      "activations/layer2_attention_weight_max": 28.901477813720703,
      "activations/layer2_attention_weight_min": -27.874099731445312,
      "activations/layer3_attention_weight_max": 87.42461395263672,
      "activations/layer3_attention_weight_min": -91.56044006347656,
      "activations/layer4_attention_weight_max": 97.00523376464844,
      "activations/layer4_attention_weight_min": -90.10092163085938,
      "activations/layer5_attention_weight_max": 79.31605529785156,
      "activations/layer5_attention_weight_min": -77.91838836669922,
      "activations/layer6_attention_weight_max": 53.2374153137207,
      "activations/layer6_attention_weight_min": -51.664852142333984,
      "activations/layer7_attention_weight_max": 71.87190246582031,
      "activations/layer7_attention_weight_min": -78.6165771484375,
      "activations/layer8_attention_weight_max": 46.07713317871094,
      "activations/layer8_attention_weight_min": -47.63687515258789,
      "activations/layer9_attention_weight_max": 49.23436737060547,
      "activations/layer9_attention_weight_min": -49.059722900390625,
      "epoch": 8.98,
      "learning_rate": 9.302575757575756e-05,
      "loss": 2.8041,
      "step": 154550
    },
    {
      "activations/layer0_attention_weight_max": 15.366884231567383,
      "activations/layer0_attention_weight_min": -13.818058967590332,
      "activations/layer10_attention_weight_max": 45.00269317626953,
      "activations/layer10_attention_weight_min": -44.47967529296875,
      "activations/layer11_attention_weight_max": 37.68858337402344,
      "activations/layer11_attention_weight_min": -36.6900634765625,
      "activations/layer12_attention_weight_max": 19.710405349731445,
      "activations/layer12_attention_weight_min": -27.4107723236084,
      "activations/layer13_attention_weight_max": 53.70088577270508,
      "activations/layer13_attention_weight_min": -35.51271057128906,
      "activations/layer14_attention_weight_max": 63.33815383911133,
      "activations/layer14_attention_weight_min": -35.49462890625,
      "activations/layer15_attention_weight_max": 53.4902229309082,
      "activations/layer15_attention_weight_min": -30.073627471923828,
      "activations/layer16_attention_weight_max": 30.503170013427734,
      "activations/layer16_attention_weight_min": -28.65814971923828,
      "activations/layer17_attention_weight_max": 51.12666320800781,
      "activations/layer17_attention_weight_min": -44.880340576171875,
      "activations/layer18_attention_weight_max": 44.98062515258789,
      "activations/layer18_attention_weight_min": -37.88323974609375,
      "activations/layer19_attention_weight_max": 22.56495475769043,
      "activations/layer19_attention_weight_min": -21.77019691467285,
      "activations/layer1_attention_weight_max": 17.414913177490234,
      "activations/layer1_attention_weight_min": -16.022539138793945,
      "activations/layer20_attention_weight_max": 23.31079864501953,
      "activations/layer20_attention_weight_min": -22.85897445678711,
      "activations/layer21_attention_weight_max": 40.57646560668945,
      "activations/layer21_attention_weight_min": -22.540489196777344,
      "activations/layer22_attention_weight_max": 31.12800407409668,
      "activations/layer22_attention_weight_min": -24.327512741088867,
      "activations/layer23_attention_weight_max": 38.79669952392578,
      "activations/layer23_attention_weight_min": -24.071022033691406,
      "activations/layer2_attention_weight_max": 32.51795196533203,
      "activations/layer2_attention_weight_min": -30.615772247314453,
      "activations/layer3_attention_weight_max": 95.82003021240234,
      "activations/layer3_attention_weight_min": -96.57477569580078,
      "activations/layer4_attention_weight_max": 110.72994232177734,
      "activations/layer4_attention_weight_min": -104.0154037475586,
      "activations/layer5_attention_weight_max": 88.57499694824219,
      "activations/layer5_attention_weight_min": -83.44950866699219,
      "activations/layer6_attention_weight_max": 58.400672912597656,
      "activations/layer6_attention_weight_min": -57.581825256347656,
      "activations/layer7_attention_weight_max": 82.62700653076172,
      "activations/layer7_attention_weight_min": -80.43354034423828,
      "activations/layer8_attention_weight_max": 52.61504364013672,
      "activations/layer8_attention_weight_min": -50.27200698852539,
      "activations/layer9_attention_weight_max": 65.2398910522461,
      "activations/layer9_attention_weight_min": -60.04838180541992,
      "epoch": 8.98,
      "learning_rate": 9.300681818181818e-05,
      "loss": 2.7998,
      "step": 154600
    },
    {
      "activations/layer0_attention_weight_max": 15.617491722106934,
      "activations/layer0_attention_weight_min": -13.578585624694824,
      "activations/layer10_attention_weight_max": 35.81807327270508,
      "activations/layer10_attention_weight_min": -36.05553436279297,
      "activations/layer11_attention_weight_max": 33.56610870361328,
      "activations/layer11_attention_weight_min": -30.661548614501953,
      "activations/layer12_attention_weight_max": 22.96158790588379,
      "activations/layer12_attention_weight_min": -26.847623825073242,
      "activations/layer13_attention_weight_max": 56.390995025634766,
      "activations/layer13_attention_weight_min": -34.46205139160156,
      "activations/layer14_attention_weight_max": 48.62770462036133,
      "activations/layer14_attention_weight_min": -30.79153060913086,
      "activations/layer15_attention_weight_max": 48.855533599853516,
      "activations/layer15_attention_weight_min": -30.27257537841797,
      "activations/layer16_attention_weight_max": 30.66752815246582,
      "activations/layer16_attention_weight_min": -27.8636417388916,
      "activations/layer17_attention_weight_max": 56.13965606689453,
      "activations/layer17_attention_weight_min": -43.67549514770508,
      "activations/layer18_attention_weight_max": 47.907508850097656,
      "activations/layer18_attention_weight_min": -38.212093353271484,
      "activations/layer19_attention_weight_max": 22.62062644958496,
      "activations/layer19_attention_weight_min": -22.540937423706055,
      "activations/layer1_attention_weight_max": 18.43485450744629,
      "activations/layer1_attention_weight_min": -14.815872192382812,
      "activations/layer20_attention_weight_max": 21.912382125854492,
      "activations/layer20_attention_weight_min": -20.295207977294922,
      "activations/layer21_attention_weight_max": 43.66633605957031,
      "activations/layer21_attention_weight_min": -22.741809844970703,
      "activations/layer22_attention_weight_max": 32.52459716796875,
      "activations/layer22_attention_weight_min": -25.642250061035156,
      "activations/layer23_attention_weight_max": 37.615394592285156,
      "activations/layer23_attention_weight_min": -25.40163803100586,
      "activations/layer2_attention_weight_max": 29.296512603759766,
      "activations/layer2_attention_weight_min": -28.94825553894043,
      "activations/layer3_attention_weight_max": 86.3275375366211,
      "activations/layer3_attention_weight_min": -88.78138732910156,
      "activations/layer4_attention_weight_max": 93.41462707519531,
      "activations/layer4_attention_weight_min": -93.13934326171875,
      "activations/layer5_attention_weight_max": 75.85552978515625,
      "activations/layer5_attention_weight_min": -79.62075805664062,
      "activations/layer6_attention_weight_max": 49.66497039794922,
      "activations/layer6_attention_weight_min": -51.022056579589844,
      "activations/layer7_attention_weight_max": 73.99634552001953,
      "activations/layer7_attention_weight_min": -74.0683364868164,
      "activations/layer8_attention_weight_max": 46.72285842895508,
      "activations/layer8_attention_weight_min": -43.84873962402344,
      "activations/layer9_attention_weight_max": 56.441650390625,
      "activations/layer9_attention_weight_min": -50.01522445678711,
      "epoch": 8.99,
      "learning_rate": 9.298787878787877e-05,
      "loss": 2.8182,
      "step": 154650
    },
    {
      "activations/layer0_attention_weight_max": 14.314316749572754,
      "activations/layer0_attention_weight_min": -13.780248641967773,
      "activations/layer10_attention_weight_max": 45.519630432128906,
      "activations/layer10_attention_weight_min": -38.435787200927734,
      "activations/layer11_attention_weight_max": 38.58983612060547,
      "activations/layer11_attention_weight_min": -33.77874755859375,
      "activations/layer12_attention_weight_max": 18.357145309448242,
      "activations/layer12_attention_weight_min": -31.51841926574707,
      "activations/layer13_attention_weight_max": 59.066673278808594,
      "activations/layer13_attention_weight_min": -33.422698974609375,
      "activations/layer14_attention_weight_max": 53.36646270751953,
      "activations/layer14_attention_weight_min": -31.52439308166504,
      "activations/layer15_attention_weight_max": 39.95859146118164,
      "activations/layer15_attention_weight_min": -29.46147346496582,
      "activations/layer16_attention_weight_max": 37.14883804321289,
      "activations/layer16_attention_weight_min": -28.64592170715332,
      "activations/layer17_attention_weight_max": 52.426422119140625,
      "activations/layer17_attention_weight_min": -43.56082534790039,
      "activations/layer18_attention_weight_max": 47.44490432739258,
      "activations/layer18_attention_weight_min": -35.469215393066406,
      "activations/layer19_attention_weight_max": 22.04107666015625,
      "activations/layer19_attention_weight_min": -20.864540100097656,
      "activations/layer1_attention_weight_max": 18.812101364135742,
      "activations/layer1_attention_weight_min": -19.126955032348633,
      "activations/layer20_attention_weight_max": 24.51647186279297,
      "activations/layer20_attention_weight_min": -20.83601951599121,
      "activations/layer21_attention_weight_max": 38.90013885498047,
      "activations/layer21_attention_weight_min": -21.650083541870117,
      "activations/layer22_attention_weight_max": 34.4976692199707,
      "activations/layer22_attention_weight_min": -25.443500518798828,
      "activations/layer23_attention_weight_max": 35.58509063720703,
      "activations/layer23_attention_weight_min": -22.91632843017578,
      "activations/layer2_attention_weight_max": 32.93681716918945,
      "activations/layer2_attention_weight_min": -29.84469223022461,
      "activations/layer3_attention_weight_max": 99.47914123535156,
      "activations/layer3_attention_weight_min": -99.16455078125,
      "activations/layer4_attention_weight_max": 103.92813873291016,
      "activations/layer4_attention_weight_min": -100.69043731689453,
      "activations/layer5_attention_weight_max": 78.33961486816406,
      "activations/layer5_attention_weight_min": -77.66301727294922,
      "activations/layer6_attention_weight_max": 54.85794448852539,
      "activations/layer6_attention_weight_min": -54.44230270385742,
      "activations/layer7_attention_weight_max": 86.82270050048828,
      "activations/layer7_attention_weight_min": -87.38640594482422,
      "activations/layer8_attention_weight_max": 55.82585906982422,
      "activations/layer8_attention_weight_min": -54.718353271484375,
      "activations/layer9_attention_weight_max": 65.92549133300781,
      "activations/layer9_attention_weight_min": -58.13326644897461,
      "epoch": 8.99,
      "learning_rate": 9.296893939393938e-05,
      "loss": 2.7967,
      "step": 154700
    },
    {
      "activations/layer0_attention_weight_max": 14.428383827209473,
      "activations/layer0_attention_weight_min": -13.654157638549805,
      "activations/layer10_attention_weight_max": 37.33648681640625,
      "activations/layer10_attention_weight_min": -36.00248718261719,
      "activations/layer11_attention_weight_max": 30.84805679321289,
      "activations/layer11_attention_weight_min": -31.11111831665039,
      "activations/layer12_attention_weight_max": 21.866790771484375,
      "activations/layer12_attention_weight_min": -25.803035736083984,
      "activations/layer13_attention_weight_max": 41.82464599609375,
      "activations/layer13_attention_weight_min": -30.737993240356445,
      "activations/layer14_attention_weight_max": 36.0498161315918,
      "activations/layer14_attention_weight_min": -31.003711700439453,
      "activations/layer15_attention_weight_max": 33.696895599365234,
      "activations/layer15_attention_weight_min": -30.191417694091797,
      "activations/layer16_attention_weight_max": 31.546817779541016,
      "activations/layer16_attention_weight_min": -31.24635887145996,
      "activations/layer17_attention_weight_max": 53.08580780029297,
      "activations/layer17_attention_weight_min": -45.511314392089844,
      "activations/layer18_attention_weight_max": 48.5666618347168,
      "activations/layer18_attention_weight_min": -37.630550384521484,
      "activations/layer19_attention_weight_max": 24.340145111083984,
      "activations/layer19_attention_weight_min": -22.734275817871094,
      "activations/layer1_attention_weight_max": 17.279191970825195,
      "activations/layer1_attention_weight_min": -15.345664978027344,
      "activations/layer20_attention_weight_max": 22.923912048339844,
      "activations/layer20_attention_weight_min": -22.243303298950195,
      "activations/layer21_attention_weight_max": 37.07305908203125,
      "activations/layer21_attention_weight_min": -24.481163024902344,
      "activations/layer22_attention_weight_max": 37.33683395385742,
      "activations/layer22_attention_weight_min": -27.005693435668945,
      "activations/layer23_attention_weight_max": 39.43324661254883,
      "activations/layer23_attention_weight_min": -22.308399200439453,
      "activations/layer2_attention_weight_max": 29.797636032104492,
      "activations/layer2_attention_weight_min": -28.22820281982422,
      "activations/layer3_attention_weight_max": 90.79379272460938,
      "activations/layer3_attention_weight_min": -91.89884185791016,
      "activations/layer4_attention_weight_max": 100.79437255859375,
      "activations/layer4_attention_weight_min": -99.90839385986328,
      "activations/layer5_attention_weight_max": 83.6988525390625,
      "activations/layer5_attention_weight_min": -81.00325775146484,
      "activations/layer6_attention_weight_max": 56.71244812011719,
      "activations/layer6_attention_weight_min": -52.423622131347656,
      "activations/layer7_attention_weight_max": 78.10028076171875,
      "activations/layer7_attention_weight_min": -74.26219177246094,
      "activations/layer8_attention_weight_max": 46.93772506713867,
      "activations/layer8_attention_weight_min": -45.331912994384766,
      "activations/layer9_attention_weight_max": 52.80933380126953,
      "activations/layer9_attention_weight_min": -49.532901763916016,
      "epoch": 8.99,
      "learning_rate": 9.295e-05,
      "loss": 2.7867,
      "step": 154750
    },
    {
      "activations/layer0_attention_weight_max": 14.913140296936035,
      "activations/layer0_attention_weight_min": -13.954156875610352,
      "activations/layer10_attention_weight_max": 40.194313049316406,
      "activations/layer10_attention_weight_min": -37.23783493041992,
      "activations/layer11_attention_weight_max": 35.34296798706055,
      "activations/layer11_attention_weight_min": -33.94066619873047,
      "activations/layer12_attention_weight_max": 22.50603485107422,
      "activations/layer12_attention_weight_min": -22.98127555847168,
      "activations/layer13_attention_weight_max": 70.94487762451172,
      "activations/layer13_attention_weight_min": -38.989471435546875,
      "activations/layer14_attention_weight_max": 82.39200592041016,
      "activations/layer14_attention_weight_min": -35.35340118408203,
      "activations/layer15_attention_weight_max": 57.923580169677734,
      "activations/layer15_attention_weight_min": -30.528377532958984,
      "activations/layer16_attention_weight_max": 38.695274353027344,
      "activations/layer16_attention_weight_min": -30.790088653564453,
      "activations/layer17_attention_weight_max": 62.19947052001953,
      "activations/layer17_attention_weight_min": -46.62344741821289,
      "activations/layer18_attention_weight_max": 57.6592903137207,
      "activations/layer18_attention_weight_min": -38.42082977294922,
      "activations/layer19_attention_weight_max": 25.692855834960938,
      "activations/layer19_attention_weight_min": -21.508573532104492,
      "activations/layer1_attention_weight_max": 18.683353424072266,
      "activations/layer1_attention_weight_min": -16.844223022460938,
      "activations/layer20_attention_weight_max": 26.814435958862305,
      "activations/layer20_attention_weight_min": -21.659290313720703,
      "activations/layer21_attention_weight_max": 46.48101806640625,
      "activations/layer21_attention_weight_min": -23.956066131591797,
      "activations/layer22_attention_weight_max": 35.8221435546875,
      "activations/layer22_attention_weight_min": -23.784137725830078,
      "activations/layer23_attention_weight_max": 40.12049865722656,
      "activations/layer23_attention_weight_min": -22.645889282226562,
      "activations/layer2_attention_weight_max": 30.75122833251953,
      "activations/layer2_attention_weight_min": -30.296085357666016,
      "activations/layer3_attention_weight_max": 91.66739654541016,
      "activations/layer3_attention_weight_min": -92.37415313720703,
      "activations/layer4_attention_weight_max": 102.08366394042969,
      "activations/layer4_attention_weight_min": -99.42467498779297,
      "activations/layer5_attention_weight_max": 83.59170532226562,
      "activations/layer5_attention_weight_min": -80.39253234863281,
      "activations/layer6_attention_weight_max": 56.4001350402832,
      "activations/layer6_attention_weight_min": -57.5758171081543,
      "activations/layer7_attention_weight_max": 78.13279724121094,
      "activations/layer7_attention_weight_min": -76.55413818359375,
      "activations/layer8_attention_weight_max": 46.82611083984375,
      "activations/layer8_attention_weight_min": -45.98528289794922,
      "activations/layer9_attention_weight_max": 51.418312072753906,
      "activations/layer9_attention_weight_min": -49.682796478271484,
      "epoch": 8.99,
      "learning_rate": 9.29310606060606e-05,
      "loss": 2.7991,
      "step": 154800
    },
    {
      "activations/layer0_attention_weight_max": 15.145586013793945,
      "activations/layer0_attention_weight_min": -13.815557479858398,
      "activations/layer10_attention_weight_max": 35.20077133178711,
      "activations/layer10_attention_weight_min": -34.054710388183594,
      "activations/layer11_attention_weight_max": 28.800601959228516,
      "activations/layer11_attention_weight_min": -32.722633361816406,
      "activations/layer12_attention_weight_max": 19.52434730529785,
      "activations/layer12_attention_weight_min": -24.348873138427734,
      "activations/layer13_attention_weight_max": 39.3392219543457,
      "activations/layer13_attention_weight_min": -28.73886489868164,
      "activations/layer14_attention_weight_max": 37.121490478515625,
      "activations/layer14_attention_weight_min": -30.860851287841797,
      "activations/layer15_attention_weight_max": 32.60645294189453,
      "activations/layer15_attention_weight_min": -29.586780548095703,
      "activations/layer16_attention_weight_max": 30.433597564697266,
      "activations/layer16_attention_weight_min": -28.163753509521484,
      "activations/layer17_attention_weight_max": 52.94289016723633,
      "activations/layer17_attention_weight_min": -43.459293365478516,
      "activations/layer18_attention_weight_max": 45.95585250854492,
      "activations/layer18_attention_weight_min": -36.997440338134766,
      "activations/layer19_attention_weight_max": 23.896108627319336,
      "activations/layer19_attention_weight_min": -20.87653350830078,
      "activations/layer1_attention_weight_max": 16.603164672851562,
      "activations/layer1_attention_weight_min": -15.496291160583496,
      "activations/layer20_attention_weight_max": 22.179533004760742,
      "activations/layer20_attention_weight_min": -20.875816345214844,
      "activations/layer21_attention_weight_max": 38.07649612426758,
      "activations/layer21_attention_weight_min": -23.005149841308594,
      "activations/layer22_attention_weight_max": 32.307579040527344,
      "activations/layer22_attention_weight_min": -24.20318603515625,
      "activations/layer23_attention_weight_max": 36.57787322998047,
      "activations/layer23_attention_weight_min": -21.81678581237793,
      "activations/layer2_attention_weight_max": 28.563949584960938,
      "activations/layer2_attention_weight_min": -29.580692291259766,
      "activations/layer3_attention_weight_max": 89.95223999023438,
      "activations/layer3_attention_weight_min": -90.9029541015625,
      "activations/layer4_attention_weight_max": 98.86351776123047,
      "activations/layer4_attention_weight_min": -93.09703063964844,
      "activations/layer5_attention_weight_max": 80.4323501586914,
      "activations/layer5_attention_weight_min": -77.87683868408203,
      "activations/layer6_attention_weight_max": 55.99738311767578,
      "activations/layer6_attention_weight_min": -53.82722091674805,
      "activations/layer7_attention_weight_max": 73.92005920410156,
      "activations/layer7_attention_weight_min": -74.45594024658203,
      "activations/layer8_attention_weight_max": 46.154693603515625,
      "activations/layer8_attention_weight_min": -46.063499450683594,
      "activations/layer9_attention_weight_max": 48.03731918334961,
      "activations/layer9_attention_weight_min": -48.06930160522461,
      "epoch": 9.0,
      "learning_rate": 9.29121212121212e-05,
      "loss": 2.7971,
      "step": 154850
    },
    {
      "activations/layer0_attention_weight_max": 16.87030029296875,
      "activations/layer0_attention_weight_min": -13.599311828613281,
      "activations/layer10_attention_weight_max": 40.02178955078125,
      "activations/layer10_attention_weight_min": -38.10427474975586,
      "activations/layer11_attention_weight_max": 35.33622360229492,
      "activations/layer11_attention_weight_min": -29.826871871948242,
      "activations/layer12_attention_weight_max": 17.982725143432617,
      "activations/layer12_attention_weight_min": -28.76923942565918,
      "activations/layer13_attention_weight_max": 41.97664260864258,
      "activations/layer13_attention_weight_min": -32.236873626708984,
      "activations/layer14_attention_weight_max": 38.37403106689453,
      "activations/layer14_attention_weight_min": -32.458251953125,
      "activations/layer15_attention_weight_max": 34.44438934326172,
      "activations/layer15_attention_weight_min": -29.458415985107422,
      "activations/layer16_attention_weight_max": 32.81583023071289,
      "activations/layer16_attention_weight_min": -28.70970344543457,
      "activations/layer17_attention_weight_max": 54.03761672973633,
      "activations/layer17_attention_weight_min": -46.0245361328125,
      "activations/layer18_attention_weight_max": 45.98151779174805,
      "activations/layer18_attention_weight_min": -38.656028747558594,
      "activations/layer19_attention_weight_max": 23.006689071655273,
      "activations/layer19_attention_weight_min": -22.24106788635254,
      "activations/layer1_attention_weight_max": 17.97245216369629,
      "activations/layer1_attention_weight_min": -14.800704002380371,
      "activations/layer20_attention_weight_max": 23.501361846923828,
      "activations/layer20_attention_weight_min": -23.133079528808594,
      "activations/layer21_attention_weight_max": 44.819969177246094,
      "activations/layer21_attention_weight_min": -25.44196128845215,
      "activations/layer22_attention_weight_max": 33.95612335205078,
      "activations/layer22_attention_weight_min": -27.27509880065918,
      "activations/layer23_attention_weight_max": 36.839622497558594,
      "activations/layer23_attention_weight_min": -21.489368438720703,
      "activations/layer2_attention_weight_max": 29.344383239746094,
      "activations/layer2_attention_weight_min": -28.89190673828125,
      "activations/layer3_attention_weight_max": 91.72394561767578,
      "activations/layer3_attention_weight_min": -87.46399688720703,
      "activations/layer4_attention_weight_max": 93.53781127929688,
      "activations/layer4_attention_weight_min": -90.68566131591797,
      "activations/layer5_attention_weight_max": 74.96406555175781,
      "activations/layer5_attention_weight_min": -71.88166809082031,
      "activations/layer6_attention_weight_max": 54.37467575073242,
      "activations/layer6_attention_weight_min": -51.56563949584961,
      "activations/layer7_attention_weight_max": 79.09007263183594,
      "activations/layer7_attention_weight_min": -75.03954315185547,
      "activations/layer8_attention_weight_max": 49.54826736450195,
      "activations/layer8_attention_weight_min": -44.96531295776367,
      "activations/layer9_attention_weight_max": 57.42409133911133,
      "activations/layer9_attention_weight_min": -48.49016571044922,
      "epoch": 9.0,
      "learning_rate": 9.289318181818181e-05,
      "loss": 2.8414,
      "step": 154900
    },
    {
      "activations/layer0_attention_weight_max": 15.261013984680176,
      "activations/layer0_attention_weight_min": -14.315752029418945,
      "activations/layer10_attention_weight_max": 39.291015625,
      "activations/layer10_attention_weight_min": -38.848148345947266,
      "activations/layer11_attention_weight_max": 35.458595275878906,
      "activations/layer11_attention_weight_min": -34.9173583984375,
      "activations/layer12_attention_weight_max": 19.616735458374023,
      "activations/layer12_attention_weight_min": -35.9379768371582,
      "activations/layer13_attention_weight_max": 48.42631149291992,
      "activations/layer13_attention_weight_min": -33.25419616699219,
      "activations/layer14_attention_weight_max": 51.57938003540039,
      "activations/layer14_attention_weight_min": -34.75981140136719,
      "activations/layer15_attention_weight_max": 40.76818084716797,
      "activations/layer15_attention_weight_min": -31.729000091552734,
      "activations/layer16_attention_weight_max": 42.41206359863281,
      "activations/layer16_attention_weight_min": -32.341861724853516,
      "activations/layer17_attention_weight_max": 64.96590423583984,
      "activations/layer17_attention_weight_min": -50.86748123168945,
      "activations/layer18_attention_weight_max": 57.4757080078125,
      "activations/layer18_attention_weight_min": -45.1474494934082,
      "activations/layer19_attention_weight_max": 28.23952293395996,
      "activations/layer19_attention_weight_min": -25.51845932006836,
      "activations/layer1_attention_weight_max": 16.7469482421875,
      "activations/layer1_attention_weight_min": -13.760307312011719,
      "activations/layer20_attention_weight_max": 30.815372467041016,
      "activations/layer20_attention_weight_min": -23.91667938232422,
      "activations/layer21_attention_weight_max": 58.68988800048828,
      "activations/layer21_attention_weight_min": -26.68386459350586,
      "activations/layer22_attention_weight_max": 40.16975402832031,
      "activations/layer22_attention_weight_min": -28.718002319335938,
      "activations/layer23_attention_weight_max": 50.88636016845703,
      "activations/layer23_attention_weight_min": -21.903152465820312,
      "activations/layer2_attention_weight_max": 31.892250061035156,
      "activations/layer2_attention_weight_min": -29.8863582611084,
      "activations/layer3_attention_weight_max": 97.47991943359375,
      "activations/layer3_attention_weight_min": -97.04137420654297,
      "activations/layer4_attention_weight_max": 109.17266845703125,
      "activations/layer4_attention_weight_min": -101.93810272216797,
      "activations/layer5_attention_weight_max": 90.80868530273438,
      "activations/layer5_attention_weight_min": -79.75321960449219,
      "activations/layer6_attention_weight_max": 58.377166748046875,
      "activations/layer6_attention_weight_min": -53.0839729309082,
      "activations/layer7_attention_weight_max": 77.11030578613281,
      "activations/layer7_attention_weight_min": -76.36753845214844,
      "activations/layer8_attention_weight_max": 45.76028823852539,
      "activations/layer8_attention_weight_min": -46.254669189453125,
      "activations/layer9_attention_weight_max": 56.52478790283203,
      "activations/layer9_attention_weight_min": -56.434322357177734,
      "epoch": 9.0,
      "learning_rate": 9.287424242424242e-05,
      "loss": 2.8087,
      "step": 154950
    },
    {
      "activations/layer0_attention_weight_max": 14.495880126953125,
      "activations/layer0_attention_weight_min": -14.221898078918457,
      "activations/layer10_attention_weight_max": 38.791297912597656,
      "activations/layer10_attention_weight_min": -35.84687805175781,
      "activations/layer11_attention_weight_max": 31.019208908081055,
      "activations/layer11_attention_weight_min": -31.66529083251953,
      "activations/layer12_attention_weight_max": 19.76613998413086,
      "activations/layer12_attention_weight_min": -25.84172821044922,
      "activations/layer13_attention_weight_max": 37.44076156616211,
      "activations/layer13_attention_weight_min": -32.114906311035156,
      "activations/layer14_attention_weight_max": 38.827999114990234,
      "activations/layer14_attention_weight_min": -29.581369400024414,
      "activations/layer15_attention_weight_max": 36.67325973510742,
      "activations/layer15_attention_weight_min": -28.33115005493164,
      "activations/layer16_attention_weight_max": 32.612178802490234,
      "activations/layer16_attention_weight_min": -27.900781631469727,
      "activations/layer17_attention_weight_max": 51.50141525268555,
      "activations/layer17_attention_weight_min": -43.156429290771484,
      "activations/layer18_attention_weight_max": 46.9723014831543,
      "activations/layer18_attention_weight_min": -36.799903869628906,
      "activations/layer19_attention_weight_max": 21.382993698120117,
      "activations/layer19_attention_weight_min": -20.97234535217285,
      "activations/layer1_attention_weight_max": 16.32358169555664,
      "activations/layer1_attention_weight_min": -16.280691146850586,
      "activations/layer20_attention_weight_max": 24.343292236328125,
      "activations/layer20_attention_weight_min": -21.698375701904297,
      "activations/layer21_attention_weight_max": 37.066551208496094,
      "activations/layer21_attention_weight_min": -23.594249725341797,
      "activations/layer22_attention_weight_max": 29.995643615722656,
      "activations/layer22_attention_weight_min": -24.8603458404541,
      "activations/layer23_attention_weight_max": 35.6473388671875,
      "activations/layer23_attention_weight_min": -22.249412536621094,
      "activations/layer2_attention_weight_max": 30.797332763671875,
      "activations/layer2_attention_weight_min": -30.649185180664062,
      "activations/layer3_attention_weight_max": 92.10591125488281,
      "activations/layer3_attention_weight_min": -97.72562408447266,
      "activations/layer4_attention_weight_max": 105.8010482788086,
      "activations/layer4_attention_weight_min": -101.78140258789062,
      "activations/layer5_attention_weight_max": 83.88007354736328,
      "activations/layer5_attention_weight_min": -84.19966888427734,
      "activations/layer6_attention_weight_max": 56.0471305847168,
      "activations/layer6_attention_weight_min": -55.6380729675293,
      "activations/layer7_attention_weight_max": 75.848388671875,
      "activations/layer7_attention_weight_min": -76.23445892333984,
      "activations/layer8_attention_weight_max": 47.634456634521484,
      "activations/layer8_attention_weight_min": -49.712318420410156,
      "activations/layer9_attention_weight_max": 50.4298210144043,
      "activations/layer9_attention_weight_min": -50.862266540527344,
      "epoch": 9.01,
      "learning_rate": 9.285530303030303e-05,
      "loss": 2.7844,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.5448,
      "eval_samples_per_second": 502.526,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.5448,
      "eval_openwebtext_samples_per_second": 502.526,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_wikitext_loss": 2.9921875,
      "eval_wikitext_ppl": 19.929230034775262,
      "eval_wikitext_runtime": 2.0054,
      "eval_wikitext_samples_per_second": 227.389,
      "step": 155000
    },
    {
      "epoch": 9.01,
      "eval_lambada_loss": 2.7578125,
      "eval_lambada_ppl": 15.765318566620541,
      "eval_lambada_runtime": 9.6198,
      "eval_lambada_samples_per_second": 506.145,
      "step": 155000
    },
    {
      "activations/layer0_attention_weight_max": 15.171979904174805,
      "activations/layer0_attention_weight_min": -14.547740936279297,
      "activations/layer10_attention_weight_max": 34.729347229003906,
      "activations/layer10_attention_weight_min": -33.51077651977539,
      "activations/layer11_attention_weight_max": 29.588186264038086,
      "activations/layer11_attention_weight_min": -31.68653678894043,
      "activations/layer12_attention_weight_max": 18.802017211914062,
      "activations/layer12_attention_weight_min": -25.489702224731445,
      "activations/layer13_attention_weight_max": 55.52140426635742,
      "activations/layer13_attention_weight_min": -37.15606689453125,
      "activations/layer14_attention_weight_max": 42.229637145996094,
      "activations/layer14_attention_weight_min": -30.05967903137207,
      "activations/layer15_attention_weight_max": 38.7687873840332,
      "activations/layer15_attention_weight_min": -28.68384552001953,
      "activations/layer16_attention_weight_max": 37.022151947021484,
      "activations/layer16_attention_weight_min": -30.2977294921875,
      "activations/layer17_attention_weight_max": 53.42402267456055,
      "activations/layer17_attention_weight_min": -44.592899322509766,
      "activations/layer18_attention_weight_max": 48.886131286621094,
      "activations/layer18_attention_weight_min": -38.77155685424805,
      "activations/layer19_attention_weight_max": 25.205663681030273,
      "activations/layer19_attention_weight_min": -24.321012496948242,
      "activations/layer1_attention_weight_max": 18.64131736755371,
      "activations/layer1_attention_weight_min": -13.398394584655762,
      "activations/layer20_attention_weight_max": 24.341896057128906,
      "activations/layer20_attention_weight_min": -22.63228416442871,
      "activations/layer21_attention_weight_max": 46.93775177001953,
      "activations/layer21_attention_weight_min": -23.52713394165039,
      "activations/layer22_attention_weight_max": 34.46523666381836,
      "activations/layer22_attention_weight_min": -28.691003799438477,
      "activations/layer23_attention_weight_max": 39.920433044433594,
      "activations/layer23_attention_weight_min": -24.737350463867188,
      "activations/layer2_attention_weight_max": 29.604969024658203,
      "activations/layer2_attention_weight_min": -28.97866439819336,
      "activations/layer3_attention_weight_max": 86.9859848022461,
      "activations/layer3_attention_weight_min": -90.88798522949219,
      "activations/layer4_attention_weight_max": 95.37981414794922,
      "activations/layer4_attention_weight_min": -94.34412384033203,
      "activations/layer5_attention_weight_max": 81.71881103515625,
      "activations/layer5_attention_weight_min": -75.86234283447266,
      "activations/layer6_attention_weight_max": 53.681888580322266,
      "activations/layer6_attention_weight_min": -53.75221633911133,
      "activations/layer7_attention_weight_max": 76.89054107666016,
      "activations/layer7_attention_weight_min": -75.20091247558594,
      "activations/layer8_attention_weight_max": 46.255767822265625,
      "activations/layer8_attention_weight_min": -45.36159133911133,
      "activations/layer9_attention_weight_max": 46.70895004272461,
      "activations/layer9_attention_weight_min": -49.1197509765625,
      "epoch": 9.01,
      "learning_rate": 9.283636363636363e-05,
      "loss": 2.7759,
      "step": 155050
    },
    {
      "activations/layer0_attention_weight_max": 15.158439636230469,
      "activations/layer0_attention_weight_min": -14.173986434936523,
      "activations/layer10_attention_weight_max": 35.42051315307617,
      "activations/layer10_attention_weight_min": -33.78411865234375,
      "activations/layer11_attention_weight_max": 29.93077850341797,
      "activations/layer11_attention_weight_min": -30.393638610839844,
      "activations/layer12_attention_weight_max": 19.629167556762695,
      "activations/layer12_attention_weight_min": -25.27800178527832,
      "activations/layer13_attention_weight_max": 40.04171371459961,
      "activations/layer13_attention_weight_min": -29.92485237121582,
      "activations/layer14_attention_weight_max": 36.174652099609375,
      "activations/layer14_attention_weight_min": -32.66752243041992,
      "activations/layer15_attention_weight_max": 35.636573791503906,
      "activations/layer15_attention_weight_min": -29.093313217163086,
      "activations/layer16_attention_weight_max": 31.294660568237305,
      "activations/layer16_attention_weight_min": -28.616838455200195,
      "activations/layer17_attention_weight_max": 55.29643630981445,
      "activations/layer17_attention_weight_min": -45.3712272644043,
      "activations/layer18_attention_weight_max": 48.600196838378906,
      "activations/layer18_attention_weight_min": -39.4509391784668,
      "activations/layer19_attention_weight_max": 24.166648864746094,
      "activations/layer19_attention_weight_min": -19.741003036499023,
      "activations/layer1_attention_weight_max": 17.484928131103516,
      "activations/layer1_attention_weight_min": -14.156620025634766,
      "activations/layer20_attention_weight_max": 24.066150665283203,
      "activations/layer20_attention_weight_min": -21.424785614013672,
      "activations/layer21_attention_weight_max": 33.65901184082031,
      "activations/layer21_attention_weight_min": -24.73801612854004,
      "activations/layer22_attention_weight_max": 32.55179977416992,
      "activations/layer22_attention_weight_min": -26.51910972595215,
      "activations/layer23_attention_weight_max": 37.95287322998047,
      "activations/layer23_attention_weight_min": -22.93643569946289,
      "activations/layer2_attention_weight_max": 31.354400634765625,
      "activations/layer2_attention_weight_min": -30.81134796142578,
      "activations/layer3_attention_weight_max": 89.24893188476562,
      "activations/layer3_attention_weight_min": -94.42428588867188,
      "activations/layer4_attention_weight_max": 100.06632232666016,
      "activations/layer4_attention_weight_min": -92.92647552490234,
      "activations/layer5_attention_weight_max": 77.71845245361328,
      "activations/layer5_attention_weight_min": -72.90315246582031,
      "activations/layer6_attention_weight_max": 51.26645278930664,
      "activations/layer6_attention_weight_min": -51.00825119018555,
      "activations/layer7_attention_weight_max": 72.8907470703125,
      "activations/layer7_attention_weight_min": -68.18144226074219,
      "activations/layer8_attention_weight_max": 43.70615005493164,
      "activations/layer8_attention_weight_min": -50.03053283691406,
      "activations/layer9_attention_weight_max": 47.115055084228516,
      "activations/layer9_attention_weight_min": -45.296470642089844,
      "epoch": 9.01,
      "learning_rate": 9.281742424242423e-05,
      "loss": 2.7849,
      "step": 155100
    },
    {
      "activations/layer0_attention_weight_max": 15.770895004272461,
      "activations/layer0_attention_weight_min": -13.822256088256836,
      "activations/layer10_attention_weight_max": 34.39741897583008,
      "activations/layer10_attention_weight_min": -32.80805206298828,
      "activations/layer11_attention_weight_max": 28.12923812866211,
      "activations/layer11_attention_weight_min": -31.092138290405273,
      "activations/layer12_attention_weight_max": 18.882287979125977,
      "activations/layer12_attention_weight_min": -27.552391052246094,
      "activations/layer13_attention_weight_max": 35.54478073120117,
      "activations/layer13_attention_weight_min": -31.6406192779541,
      "activations/layer14_attention_weight_max": 35.86989974975586,
      "activations/layer14_attention_weight_min": -30.81770133972168,
      "activations/layer15_attention_weight_max": 33.849342346191406,
      "activations/layer15_attention_weight_min": -28.779346466064453,
      "activations/layer16_attention_weight_max": 28.39169692993164,
      "activations/layer16_attention_weight_min": -27.06143569946289,
      "activations/layer17_attention_weight_max": 50.51667022705078,
      "activations/layer17_attention_weight_min": -42.86643981933594,
      "activations/layer18_attention_weight_max": 44.73583984375,
      "activations/layer18_attention_weight_min": -36.406097412109375,
      "activations/layer19_attention_weight_max": 20.810302734375,
      "activations/layer19_attention_weight_min": -20.289161682128906,
      "activations/layer1_attention_weight_max": 15.820961952209473,
      "activations/layer1_attention_weight_min": -15.71888542175293,
      "activations/layer20_attention_weight_max": 22.0941162109375,
      "activations/layer20_attention_weight_min": -24.026750564575195,
      "activations/layer21_attention_weight_max": 34.015647888183594,
      "activations/layer21_attention_weight_min": -22.266530990600586,
      "activations/layer22_attention_weight_max": 30.635339736938477,
      "activations/layer22_attention_weight_min": -24.074420928955078,
      "activations/layer23_attention_weight_max": 35.283233642578125,
      "activations/layer23_attention_weight_min": -23.18463897705078,
      "activations/layer2_attention_weight_max": 30.224315643310547,
      "activations/layer2_attention_weight_min": -27.448850631713867,
      "activations/layer3_attention_weight_max": 78.37945556640625,
      "activations/layer3_attention_weight_min": -81.33789825439453,
      "activations/layer4_attention_weight_max": 87.66680908203125,
      "activations/layer4_attention_weight_min": -87.93909454345703,
      "activations/layer5_attention_weight_max": 72.36914825439453,
      "activations/layer5_attention_weight_min": -75.76589965820312,
      "activations/layer6_attention_weight_max": 49.8221321105957,
      "activations/layer6_attention_weight_min": -49.343223571777344,
      "activations/layer7_attention_weight_max": 73.73274993896484,
      "activations/layer7_attention_weight_min": -71.71895599365234,
      "activations/layer8_attention_weight_max": 42.231407165527344,
      "activations/layer8_attention_weight_min": -43.61220169067383,
      "activations/layer9_attention_weight_max": 45.524391174316406,
      "activations/layer9_attention_weight_min": -44.141807556152344,
      "epoch": 9.02,
      "learning_rate": 9.279848484848485e-05,
      "loss": 2.7968,
      "step": 155150
    },
    {
      "activations/layer0_attention_weight_max": 15.153220176696777,
      "activations/layer0_attention_weight_min": -13.961329460144043,
      "activations/layer10_attention_weight_max": 39.101356506347656,
      "activations/layer10_attention_weight_min": -34.65202713012695,
      "activations/layer11_attention_weight_max": 32.30134582519531,
      "activations/layer11_attention_weight_min": -31.325946807861328,
      "activations/layer12_attention_weight_max": 33.06293869018555,
      "activations/layer12_attention_weight_min": -25.53415298461914,
      "activations/layer13_attention_weight_max": 53.95431137084961,
      "activations/layer13_attention_weight_min": -30.516830444335938,
      "activations/layer14_attention_weight_max": 59.740413665771484,
      "activations/layer14_attention_weight_min": -32.23904037475586,
      "activations/layer15_attention_weight_max": 44.977787017822266,
      "activations/layer15_attention_weight_min": -30.137231826782227,
      "activations/layer16_attention_weight_max": 33.528385162353516,
      "activations/layer16_attention_weight_min": -29.970048904418945,
      "activations/layer17_attention_weight_max": 58.25105285644531,
      "activations/layer17_attention_weight_min": -45.25901794433594,
      "activations/layer18_attention_weight_max": 49.00999069213867,
      "activations/layer18_attention_weight_min": -37.74468994140625,
      "activations/layer19_attention_weight_max": 22.693090438842773,
      "activations/layer19_attention_weight_min": -21.502132415771484,
      "activations/layer1_attention_weight_max": 22.31096649169922,
      "activations/layer1_attention_weight_min": -14.133692741394043,
      "activations/layer20_attention_weight_max": 23.83387565612793,
      "activations/layer20_attention_weight_min": -22.40692901611328,
      "activations/layer21_attention_weight_max": 38.073890686035156,
      "activations/layer21_attention_weight_min": -22.228553771972656,
      "activations/layer22_attention_weight_max": 33.216636657714844,
      "activations/layer22_attention_weight_min": -25.28464126586914,
      "activations/layer23_attention_weight_max": 38.63322448730469,
      "activations/layer23_attention_weight_min": -22.37502098083496,
      "activations/layer2_attention_weight_max": 32.58120346069336,
      "activations/layer2_attention_weight_min": -29.55743980407715,
      "activations/layer3_attention_weight_max": 87.30354309082031,
      "activations/layer3_attention_weight_min": -93.3422622680664,
      "activations/layer4_attention_weight_max": 98.89978790283203,
      "activations/layer4_attention_weight_min": -94.1576919555664,
      "activations/layer5_attention_weight_max": 77.78489685058594,
      "activations/layer5_attention_weight_min": -75.5421142578125,
      "activations/layer6_attention_weight_max": 52.460689544677734,
      "activations/layer6_attention_weight_min": -51.91162872314453,
      "activations/layer7_attention_weight_max": 80.98759460449219,
      "activations/layer7_attention_weight_min": -74.59902954101562,
      "activations/layer8_attention_weight_max": 47.095458984375,
      "activations/layer8_attention_weight_min": -47.808650970458984,
      "activations/layer9_attention_weight_max": 57.95598220825195,
      "activations/layer9_attention_weight_min": -48.23286819458008,
      "epoch": 9.02,
      "learning_rate": 9.277954545454544e-05,
      "loss": 2.7922,
      "step": 155200
    },
    {
      "activations/layer0_attention_weight_max": 14.582036972045898,
      "activations/layer0_attention_weight_min": -14.197714805603027,
      "activations/layer10_attention_weight_max": 34.93488693237305,
      "activations/layer10_attention_weight_min": -34.44279098510742,
      "activations/layer11_attention_weight_max": 29.863805770874023,
      "activations/layer11_attention_weight_min": -33.02075958251953,
      "activations/layer12_attention_weight_max": 20.784427642822266,
      "activations/layer12_attention_weight_min": -24.22039031982422,
      "activations/layer13_attention_weight_max": 45.497684478759766,
      "activations/layer13_attention_weight_min": -30.80011558532715,
      "activations/layer14_attention_weight_max": 49.2634391784668,
      "activations/layer14_attention_weight_min": -32.44369125366211,
      "activations/layer15_attention_weight_max": 44.18647003173828,
      "activations/layer15_attention_weight_min": -30.99753189086914,
      "activations/layer16_attention_weight_max": 35.186981201171875,
      "activations/layer16_attention_weight_min": -29.75338363647461,
      "activations/layer17_attention_weight_max": 58.61639404296875,
      "activations/layer17_attention_weight_min": -44.676700592041016,
      "activations/layer18_attention_weight_max": 54.93012619018555,
      "activations/layer18_attention_weight_min": -37.886287689208984,
      "activations/layer19_attention_weight_max": 24.360578536987305,
      "activations/layer19_attention_weight_min": -23.800125122070312,
      "activations/layer1_attention_weight_max": 16.827024459838867,
      "activations/layer1_attention_weight_min": -14.236702919006348,
      "activations/layer20_attention_weight_max": 26.57179832458496,
      "activations/layer20_attention_weight_min": -23.953147888183594,
      "activations/layer21_attention_weight_max": 52.10906982421875,
      "activations/layer21_attention_weight_min": -25.345720291137695,
      "activations/layer22_attention_weight_max": 36.29545593261719,
      "activations/layer22_attention_weight_min": -26.54940414428711,
      "activations/layer23_attention_weight_max": 40.35933303833008,
      "activations/layer23_attention_weight_min": -24.52324676513672,
      "activations/layer2_attention_weight_max": 29.743024826049805,
      "activations/layer2_attention_weight_min": -28.08937644958496,
      "activations/layer3_attention_weight_max": 84.18598937988281,
      "activations/layer3_attention_weight_min": -87.16700744628906,
      "activations/layer4_attention_weight_max": 98.04315185546875,
      "activations/layer4_attention_weight_min": -92.71614074707031,
      "activations/layer5_attention_weight_max": 81.16534423828125,
      "activations/layer5_attention_weight_min": -75.11259460449219,
      "activations/layer6_attention_weight_max": 54.397098541259766,
      "activations/layer6_attention_weight_min": -52.66762924194336,
      "activations/layer7_attention_weight_max": 76.68527221679688,
      "activations/layer7_attention_weight_min": -72.63153076171875,
      "activations/layer8_attention_weight_max": 42.334110260009766,
      "activations/layer8_attention_weight_min": -44.78445053100586,
      "activations/layer9_attention_weight_max": 48.676387786865234,
      "activations/layer9_attention_weight_min": -47.09464645385742,
      "epoch": 9.02,
      "learning_rate": 9.276060606060605e-05,
      "loss": 2.7818,
      "step": 155250
    },
    {
      "activations/layer0_attention_weight_max": 14.662468910217285,
      "activations/layer0_attention_weight_min": -12.495055198669434,
      "activations/layer10_attention_weight_max": 41.220733642578125,
      "activations/layer10_attention_weight_min": -40.0433235168457,
      "activations/layer11_attention_weight_max": 36.394752502441406,
      "activations/layer11_attention_weight_min": -34.067230224609375,
      "activations/layer12_attention_weight_max": 22.591581344604492,
      "activations/layer12_attention_weight_min": -25.200340270996094,
      "activations/layer13_attention_weight_max": 38.939300537109375,
      "activations/layer13_attention_weight_min": -30.485393524169922,
      "activations/layer14_attention_weight_max": 40.35820388793945,
      "activations/layer14_attention_weight_min": -29.204179763793945,
      "activations/layer15_attention_weight_max": 35.907737731933594,
      "activations/layer15_attention_weight_min": -27.39234733581543,
      "activations/layer16_attention_weight_max": 32.1391487121582,
      "activations/layer16_attention_weight_min": -28.447425842285156,
      "activations/layer17_attention_weight_max": 55.06087112426758,
      "activations/layer17_attention_weight_min": -41.654850006103516,
      "activations/layer18_attention_weight_max": 45.81000900268555,
      "activations/layer18_attention_weight_min": -36.354217529296875,
      "activations/layer19_attention_weight_max": 27.22017478942871,
      "activations/layer19_attention_weight_min": -22.82147789001465,
      "activations/layer1_attention_weight_max": 16.830821990966797,
      "activations/layer1_attention_weight_min": -14.394484519958496,
      "activations/layer20_attention_weight_max": 22.661136627197266,
      "activations/layer20_attention_weight_min": -21.73680877685547,
      "activations/layer21_attention_weight_max": 45.6690559387207,
      "activations/layer21_attention_weight_min": -29.383983612060547,
      "activations/layer22_attention_weight_max": 31.216699600219727,
      "activations/layer22_attention_weight_min": -25.726776123046875,
      "activations/layer23_attention_weight_max": 36.41850662231445,
      "activations/layer23_attention_weight_min": -23.773252487182617,
      "activations/layer2_attention_weight_max": 29.58251953125,
      "activations/layer2_attention_weight_min": -28.807828903198242,
      "activations/layer3_attention_weight_max": 85.1729736328125,
      "activations/layer3_attention_weight_min": -83.21346282958984,
      "activations/layer4_attention_weight_max": 97.13984680175781,
      "activations/layer4_attention_weight_min": -88.3451919555664,
      "activations/layer5_attention_weight_max": 81.42990112304688,
      "activations/layer5_attention_weight_min": -74.58598327636719,
      "activations/layer6_attention_weight_max": 56.333580017089844,
      "activations/layer6_attention_weight_min": -51.74843978881836,
      "activations/layer7_attention_weight_max": 82.84579467773438,
      "activations/layer7_attention_weight_min": -73.50061798095703,
      "activations/layer8_attention_weight_max": 48.978118896484375,
      "activations/layer8_attention_weight_min": -48.710609436035156,
      "activations/layer9_attention_weight_max": 55.67897415161133,
      "activations/layer9_attention_weight_min": -51.46833419799805,
      "epoch": 9.02,
      "learning_rate": 9.274166666666667e-05,
      "loss": 2.8189,
      "step": 155300
    },
    {
      "activations/layer0_attention_weight_max": 15.296608924865723,
      "activations/layer0_attention_weight_min": -12.50696086883545,
      "activations/layer10_attention_weight_max": 36.976741790771484,
      "activations/layer10_attention_weight_min": -35.945045471191406,
      "activations/layer11_attention_weight_max": 34.234642028808594,
      "activations/layer11_attention_weight_min": -32.13132858276367,
      "activations/layer12_attention_weight_max": 17.788909912109375,
      "activations/layer12_attention_weight_min": -24.699588775634766,
      "activations/layer13_attention_weight_max": 50.54800033569336,
      "activations/layer13_attention_weight_min": -35.29029846191406,
      "activations/layer14_attention_weight_max": 53.872493743896484,
      "activations/layer14_attention_weight_min": -33.09410858154297,
      "activations/layer15_attention_weight_max": 44.251712799072266,
      "activations/layer15_attention_weight_min": -31.851884841918945,
      "activations/layer16_attention_weight_max": 33.199974060058594,
      "activations/layer16_attention_weight_min": -29.335142135620117,
      "activations/layer17_attention_weight_max": 55.222591400146484,
      "activations/layer17_attention_weight_min": -44.689186096191406,
      "activations/layer18_attention_weight_max": 51.81560516357422,
      "activations/layer18_attention_weight_min": -39.36053466796875,
      "activations/layer19_attention_weight_max": 24.011016845703125,
      "activations/layer19_attention_weight_min": -22.972881317138672,
      "activations/layer1_attention_weight_max": 16.964214324951172,
      "activations/layer1_attention_weight_min": -14.591336250305176,
      "activations/layer20_attention_weight_max": 24.556673049926758,
      "activations/layer20_attention_weight_min": -22.165752410888672,
      "activations/layer21_attention_weight_max": 46.88960647583008,
      "activations/layer21_attention_weight_min": -23.47209358215332,
      "activations/layer22_attention_weight_max": 33.052799224853516,
      "activations/layer22_attention_weight_min": -25.68515396118164,
      "activations/layer23_attention_weight_max": 36.44012451171875,
      "activations/layer23_attention_weight_min": -23.460485458374023,
      "activations/layer2_attention_weight_max": 30.22699737548828,
      "activations/layer2_attention_weight_min": -28.62103843688965,
      "activations/layer3_attention_weight_max": 80.85038757324219,
      "activations/layer3_attention_weight_min": -87.90064239501953,
      "activations/layer4_attention_weight_max": 93.6315689086914,
      "activations/layer4_attention_weight_min": -90.7817611694336,
      "activations/layer5_attention_weight_max": 81.26553344726562,
      "activations/layer5_attention_weight_min": -72.99848937988281,
      "activations/layer6_attention_weight_max": 56.984352111816406,
      "activations/layer6_attention_weight_min": -52.73396682739258,
      "activations/layer7_attention_weight_max": 76.41785430908203,
      "activations/layer7_attention_weight_min": -74.49569702148438,
      "activations/layer8_attention_weight_max": 45.72136306762695,
      "activations/layer8_attention_weight_min": -45.088905334472656,
      "activations/layer9_attention_weight_max": 50.27433776855469,
      "activations/layer9_attention_weight_min": -49.731231689453125,
      "epoch": 9.03,
      "learning_rate": 9.272272727272725e-05,
      "loss": 2.8063,
      "step": 155350
    },
    {
      "activations/layer0_attention_weight_max": 15.93273639678955,
      "activations/layer0_attention_weight_min": -13.528529167175293,
      "activations/layer10_attention_weight_max": 34.516082763671875,
      "activations/layer10_attention_weight_min": -35.38735580444336,
      "activations/layer11_attention_weight_max": 29.046993255615234,
      "activations/layer11_attention_weight_min": -32.8948860168457,
      "activations/layer12_attention_weight_max": 18.014436721801758,
      "activations/layer12_attention_weight_min": -24.233505249023438,
      "activations/layer13_attention_weight_max": 46.42609405517578,
      "activations/layer13_attention_weight_min": -30.73577117919922,
      "activations/layer14_attention_weight_max": 47.224185943603516,
      "activations/layer14_attention_weight_min": -32.93966293334961,
      "activations/layer15_attention_weight_max": 38.04153823852539,
      "activations/layer15_attention_weight_min": -31.181991577148438,
      "activations/layer16_attention_weight_max": 33.61213684082031,
      "activations/layer16_attention_weight_min": -29.65828514099121,
      "activations/layer17_attention_weight_max": 57.29792022705078,
      "activations/layer17_attention_weight_min": -45.58744430541992,
      "activations/layer18_attention_weight_max": 53.36393737792969,
      "activations/layer18_attention_weight_min": -38.41214370727539,
      "activations/layer19_attention_weight_max": 23.191999435424805,
      "activations/layer19_attention_weight_min": -21.664091110229492,
      "activations/layer1_attention_weight_max": 16.338768005371094,
      "activations/layer1_attention_weight_min": -14.999913215637207,
      "activations/layer20_attention_weight_max": 24.058025360107422,
      "activations/layer20_attention_weight_min": -21.715490341186523,
      "activations/layer21_attention_weight_max": 39.611061096191406,
      "activations/layer21_attention_weight_min": -24.837528228759766,
      "activations/layer22_attention_weight_max": 32.890132904052734,
      "activations/layer22_attention_weight_min": -26.16802978515625,
      "activations/layer23_attention_weight_max": 43.459266662597656,
      "activations/layer23_attention_weight_min": -22.46538543701172,
      "activations/layer2_attention_weight_max": 28.391666412353516,
      "activations/layer2_attention_weight_min": -27.982330322265625,
      "activations/layer3_attention_weight_max": 85.02564239501953,
      "activations/layer3_attention_weight_min": -86.8777847290039,
      "activations/layer4_attention_weight_max": 97.37234497070312,
      "activations/layer4_attention_weight_min": -93.6505355834961,
      "activations/layer5_attention_weight_max": 78.83344268798828,
      "activations/layer5_attention_weight_min": -76.76884460449219,
      "activations/layer6_attention_weight_max": 55.48219680786133,
      "activations/layer6_attention_weight_min": -53.536312103271484,
      "activations/layer7_attention_weight_max": 81.50257873535156,
      "activations/layer7_attention_weight_min": -74.38046264648438,
      "activations/layer8_attention_weight_max": 44.0520133972168,
      "activations/layer8_attention_weight_min": -45.26963424682617,
      "activations/layer9_attention_weight_max": 51.7178955078125,
      "activations/layer9_attention_weight_min": -47.21985626220703,
      "epoch": 9.03,
      "learning_rate": 9.270378787878787e-05,
      "loss": 2.7936,
      "step": 155400
    },
    {
      "activations/layer0_attention_weight_max": 15.126521110534668,
      "activations/layer0_attention_weight_min": -13.873830795288086,
      "activations/layer10_attention_weight_max": 38.14674758911133,
      "activations/layer10_attention_weight_min": -37.1380615234375,
      "activations/layer11_attention_weight_max": 31.157773971557617,
      "activations/layer11_attention_weight_min": -34.498435974121094,
      "activations/layer12_attention_weight_max": 18.636016845703125,
      "activations/layer12_attention_weight_min": -25.39246940612793,
      "activations/layer13_attention_weight_max": 46.48379135131836,
      "activations/layer13_attention_weight_min": -30.865703582763672,
      "activations/layer14_attention_weight_max": 35.546356201171875,
      "activations/layer14_attention_weight_min": -30.900161743164062,
      "activations/layer15_attention_weight_max": 33.564697265625,
      "activations/layer15_attention_weight_min": -28.329545974731445,
      "activations/layer16_attention_weight_max": 30.736209869384766,
      "activations/layer16_attention_weight_min": -27.247249603271484,
      "activations/layer17_attention_weight_max": 54.81660842895508,
      "activations/layer17_attention_weight_min": -44.242244720458984,
      "activations/layer18_attention_weight_max": 49.116233825683594,
      "activations/layer18_attention_weight_min": -37.294734954833984,
      "activations/layer19_attention_weight_max": 21.281089782714844,
      "activations/layer19_attention_weight_min": -21.083532333374023,
      "activations/layer1_attention_weight_max": 16.609338760375977,
      "activations/layer1_attention_weight_min": -14.30089282989502,
      "activations/layer20_attention_weight_max": 21.928308486938477,
      "activations/layer20_attention_weight_min": -23.075056076049805,
      "activations/layer21_attention_weight_max": 42.29711151123047,
      "activations/layer21_attention_weight_min": -23.663108825683594,
      "activations/layer22_attention_weight_max": 33.900657653808594,
      "activations/layer22_attention_weight_min": -23.996248245239258,
      "activations/layer23_attention_weight_max": 34.10623550415039,
      "activations/layer23_attention_weight_min": -21.585472106933594,
      "activations/layer2_attention_weight_max": 30.643070220947266,
      "activations/layer2_attention_weight_min": -30.151594161987305,
      "activations/layer3_attention_weight_max": 91.56725311279297,
      "activations/layer3_attention_weight_min": -92.63893127441406,
      "activations/layer4_attention_weight_max": 97.72225189208984,
      "activations/layer4_attention_weight_min": -92.53717041015625,
      "activations/layer5_attention_weight_max": 77.11161804199219,
      "activations/layer5_attention_weight_min": -77.4847183227539,
      "activations/layer6_attention_weight_max": 54.93354034423828,
      "activations/layer6_attention_weight_min": -54.06412124633789,
      "activations/layer7_attention_weight_max": 81.37696838378906,
      "activations/layer7_attention_weight_min": -77.69552612304688,
      "activations/layer8_attention_weight_max": 44.78489303588867,
      "activations/layer8_attention_weight_min": -49.69171905517578,
      "activations/layer9_attention_weight_max": 49.08613204956055,
      "activations/layer9_attention_weight_min": -50.28609085083008,
      "epoch": 9.03,
      "learning_rate": 9.268484848484848e-05,
      "loss": 2.7914,
      "step": 155450
    },
    {
      "activations/layer0_attention_weight_max": 15.853549003601074,
      "activations/layer0_attention_weight_min": -13.687936782836914,
      "activations/layer10_attention_weight_max": 40.599308013916016,
      "activations/layer10_attention_weight_min": -38.468902587890625,
      "activations/layer11_attention_weight_max": 35.410194396972656,
      "activations/layer11_attention_weight_min": -32.521942138671875,
      "activations/layer12_attention_weight_max": 20.997249603271484,
      "activations/layer12_attention_weight_min": -22.99237823486328,
      "activations/layer13_attention_weight_max": 47.29098892211914,
      "activations/layer13_attention_weight_min": -37.12837600708008,
      "activations/layer14_attention_weight_max": 54.195777893066406,
      "activations/layer14_attention_weight_min": -36.31802749633789,
      "activations/layer15_attention_weight_max": 41.483428955078125,
      "activations/layer15_attention_weight_min": -28.548995971679688,
      "activations/layer16_attention_weight_max": 34.46189498901367,
      "activations/layer16_attention_weight_min": -29.992475509643555,
      "activations/layer17_attention_weight_max": 56.356075286865234,
      "activations/layer17_attention_weight_min": -47.86395263671875,
      "activations/layer18_attention_weight_max": 52.42184066772461,
      "activations/layer18_attention_weight_min": -38.3822021484375,
      "activations/layer19_attention_weight_max": 23.131877899169922,
      "activations/layer19_attention_weight_min": -21.590351104736328,
      "activations/layer1_attention_weight_max": 16.61870002746582,
      "activations/layer1_attention_weight_min": -13.49201774597168,
      "activations/layer20_attention_weight_max": 26.53890609741211,
      "activations/layer20_attention_weight_min": -27.679826736450195,
      "activations/layer21_attention_weight_max": 46.13306427001953,
      "activations/layer21_attention_weight_min": -24.164628982543945,
      "activations/layer22_attention_weight_max": 33.90018081665039,
      "activations/layer22_attention_weight_min": -24.59063720703125,
      "activations/layer23_attention_weight_max": 38.44316101074219,
      "activations/layer23_attention_weight_min": -25.097023010253906,
      "activations/layer2_attention_weight_max": 29.60707664489746,
      "activations/layer2_attention_weight_min": -32.69203186035156,
      "activations/layer3_attention_weight_max": 84.51976776123047,
      "activations/layer3_attention_weight_min": -90.28913879394531,
      "activations/layer4_attention_weight_max": 98.45720672607422,
      "activations/layer4_attention_weight_min": -94.55059051513672,
      "activations/layer5_attention_weight_max": 80.830078125,
      "activations/layer5_attention_weight_min": -77.86196899414062,
      "activations/layer6_attention_weight_max": 53.39053726196289,
      "activations/layer6_attention_weight_min": -55.35297775268555,
      "activations/layer7_attention_weight_max": 79.48695373535156,
      "activations/layer7_attention_weight_min": -82.6390609741211,
      "activations/layer8_attention_weight_max": 45.46154022216797,
      "activations/layer8_attention_weight_min": -47.94922637939453,
      "activations/layer9_attention_weight_max": 50.892330169677734,
      "activations/layer9_attention_weight_min": -54.16670608520508,
      "epoch": 9.04,
      "learning_rate": 9.266590909090909e-05,
      "loss": 2.8105,
      "step": 155500
    },
    {
      "activations/layer0_attention_weight_max": 14.561805725097656,
      "activations/layer0_attention_weight_min": -13.752174377441406,
      "activations/layer10_attention_weight_max": 37.83052444458008,
      "activations/layer10_attention_weight_min": -36.88072967529297,
      "activations/layer11_attention_weight_max": 33.33717346191406,
      "activations/layer11_attention_weight_min": -33.09890365600586,
      "activations/layer12_attention_weight_max": 19.276884078979492,
      "activations/layer12_attention_weight_min": -25.617094039916992,
      "activations/layer13_attention_weight_max": 46.425437927246094,
      "activations/layer13_attention_weight_min": -35.0172004699707,
      "activations/layer14_attention_weight_max": 37.35770797729492,
      "activations/layer14_attention_weight_min": -30.904245376586914,
      "activations/layer15_attention_weight_max": 33.5228157043457,
      "activations/layer15_attention_weight_min": -30.095130920410156,
      "activations/layer16_attention_weight_max": 30.973905563354492,
      "activations/layer16_attention_weight_min": -28.213701248168945,
      "activations/layer17_attention_weight_max": 56.238304138183594,
      "activations/layer17_attention_weight_min": -43.54957580566406,
      "activations/layer18_attention_weight_max": 45.99602508544922,
      "activations/layer18_attention_weight_min": -37.21438217163086,
      "activations/layer19_attention_weight_max": 22.972654342651367,
      "activations/layer19_attention_weight_min": -21.03034782409668,
      "activations/layer1_attention_weight_max": 17.14887237548828,
      "activations/layer1_attention_weight_min": -16.662273406982422,
      "activations/layer20_attention_weight_max": 22.614707946777344,
      "activations/layer20_attention_weight_min": -22.201446533203125,
      "activations/layer21_attention_weight_max": 42.81644821166992,
      "activations/layer21_attention_weight_min": -24.060104370117188,
      "activations/layer22_attention_weight_max": 34.449378967285156,
      "activations/layer22_attention_weight_min": -24.708757400512695,
      "activations/layer23_attention_weight_max": 36.20525360107422,
      "activations/layer23_attention_weight_min": -23.60694122314453,
      "activations/layer2_attention_weight_max": 30.701154708862305,
      "activations/layer2_attention_weight_min": -31.498001098632812,
      "activations/layer3_attention_weight_max": 89.68043518066406,
      "activations/layer3_attention_weight_min": -88.00606536865234,
      "activations/layer4_attention_weight_max": 97.35484313964844,
      "activations/layer4_attention_weight_min": -99.95172882080078,
      "activations/layer5_attention_weight_max": 80.74305725097656,
      "activations/layer5_attention_weight_min": -81.68406677246094,
      "activations/layer6_attention_weight_max": 56.98707962036133,
      "activations/layer6_attention_weight_min": -53.86381912231445,
      "activations/layer7_attention_weight_max": 82.63389587402344,
      "activations/layer7_attention_weight_min": -76.70947265625,
      "activations/layer8_attention_weight_max": 46.25102615356445,
      "activations/layer8_attention_weight_min": -46.959327697753906,
      "activations/layer9_attention_weight_max": 54.37942123413086,
      "activations/layer9_attention_weight_min": -52.096920013427734,
      "epoch": 9.04,
      "learning_rate": 9.264696969696969e-05,
      "loss": 2.8028,
      "step": 155550
    },
    {
      "activations/layer0_attention_weight_max": 15.135546684265137,
      "activations/layer0_attention_weight_min": -13.739969253540039,
      "activations/layer10_attention_weight_max": 37.547054290771484,
      "activations/layer10_attention_weight_min": -37.702850341796875,
      "activations/layer11_attention_weight_max": 30.27503204345703,
      "activations/layer11_attention_weight_min": -32.58125305175781,
      "activations/layer12_attention_weight_max": 20.78001594543457,
      "activations/layer12_attention_weight_min": -25.216161727905273,
      "activations/layer13_attention_weight_max": 42.80598068237305,
      "activations/layer13_attention_weight_min": -31.713268280029297,
      "activations/layer14_attention_weight_max": 38.91385269165039,
      "activations/layer14_attention_weight_min": -30.98398208618164,
      "activations/layer15_attention_weight_max": 35.6511116027832,
      "activations/layer15_attention_weight_min": -30.255939483642578,
      "activations/layer16_attention_weight_max": 31.502796173095703,
      "activations/layer16_attention_weight_min": -30.289201736450195,
      "activations/layer17_attention_weight_max": 58.32974624633789,
      "activations/layer17_attention_weight_min": -44.96529006958008,
      "activations/layer18_attention_weight_max": 50.692352294921875,
      "activations/layer18_attention_weight_min": -38.92644500732422,
      "activations/layer19_attention_weight_max": 24.960596084594727,
      "activations/layer19_attention_weight_min": -22.04647445678711,
      "activations/layer1_attention_weight_max": 15.7888822555542,
      "activations/layer1_attention_weight_min": -15.02227783203125,
      "activations/layer20_attention_weight_max": 25.525146484375,
      "activations/layer20_attention_weight_min": -24.289634704589844,
      "activations/layer21_attention_weight_max": 42.3101692199707,
      "activations/layer21_attention_weight_min": -24.731937408447266,
      "activations/layer22_attention_weight_max": 34.38664627075195,
      "activations/layer22_attention_weight_min": -25.39130973815918,
      "activations/layer23_attention_weight_max": 42.48980712890625,
      "activations/layer23_attention_weight_min": -21.169692993164062,
      "activations/layer2_attention_weight_max": 29.9366512298584,
      "activations/layer2_attention_weight_min": -29.043991088867188,
      "activations/layer3_attention_weight_max": 88.81238555908203,
      "activations/layer3_attention_weight_min": -91.43925476074219,
      "activations/layer4_attention_weight_max": 97.66594696044922,
      "activations/layer4_attention_weight_min": -95.54949951171875,
      "activations/layer5_attention_weight_max": 79.32887268066406,
      "activations/layer5_attention_weight_min": -83.94625854492188,
      "activations/layer6_attention_weight_max": 55.58109664916992,
      "activations/layer6_attention_weight_min": -54.72034454345703,
      "activations/layer7_attention_weight_max": 77.13736724853516,
      "activations/layer7_attention_weight_min": -74.20531463623047,
      "activations/layer8_attention_weight_max": 43.06526565551758,
      "activations/layer8_attention_weight_min": -46.92842102050781,
      "activations/layer9_attention_weight_max": 47.479225158691406,
      "activations/layer9_attention_weight_min": -49.48401641845703,
      "epoch": 9.04,
      "learning_rate": 9.262803030303029e-05,
      "loss": 2.8001,
      "step": 155600
    },
    {
      "activations/layer0_attention_weight_max": 14.871742248535156,
      "activations/layer0_attention_weight_min": -13.47016429901123,
      "activations/layer10_attention_weight_max": 38.757686614990234,
      "activations/layer10_attention_weight_min": -36.54450225830078,
      "activations/layer11_attention_weight_max": 34.67985153198242,
      "activations/layer11_attention_weight_min": -34.632232666015625,
      "activations/layer12_attention_weight_max": 18.853960037231445,
      "activations/layer12_attention_weight_min": -23.93107032775879,
      "activations/layer13_attention_weight_max": 42.946006774902344,
      "activations/layer13_attention_weight_min": -32.56363296508789,
      "activations/layer14_attention_weight_max": 38.055057525634766,
      "activations/layer14_attention_weight_min": -29.68583106994629,
      "activations/layer15_attention_weight_max": 35.3017463684082,
      "activations/layer15_attention_weight_min": -30.118391036987305,
      "activations/layer16_attention_weight_max": 31.64760398864746,
      "activations/layer16_attention_weight_min": -28.274145126342773,
      "activations/layer17_attention_weight_max": 58.06694412231445,
      "activations/layer17_attention_weight_min": -45.53754425048828,
      "activations/layer18_attention_weight_max": 47.52432632446289,
      "activations/layer18_attention_weight_min": -38.117591857910156,
      "activations/layer19_attention_weight_max": 22.527624130249023,
      "activations/layer19_attention_weight_min": -21.220991134643555,
      "activations/layer1_attention_weight_max": 16.01530647277832,
      "activations/layer1_attention_weight_min": -13.58881950378418,
      "activations/layer20_attention_weight_max": 21.590831756591797,
      "activations/layer20_attention_weight_min": -21.859575271606445,
      "activations/layer21_attention_weight_max": 36.75200271606445,
      "activations/layer21_attention_weight_min": -23.5790958404541,
      "activations/layer22_attention_weight_max": 30.28812599182129,
      "activations/layer22_attention_weight_min": -23.81365203857422,
      "activations/layer23_attention_weight_max": 38.39506149291992,
      "activations/layer23_attention_weight_min": -22.907514572143555,
      "activations/layer2_attention_weight_max": 29.461376190185547,
      "activations/layer2_attention_weight_min": -29.278614044189453,
      "activations/layer3_attention_weight_max": 83.62716674804688,
      "activations/layer3_attention_weight_min": -89.72050476074219,
      "activations/layer4_attention_weight_max": 95.7330551147461,
      "activations/layer4_attention_weight_min": -92.38160705566406,
      "activations/layer5_attention_weight_max": 81.266357421875,
      "activations/layer5_attention_weight_min": -78.52171325683594,
      "activations/layer6_attention_weight_max": 53.42930221557617,
      "activations/layer6_attention_weight_min": -53.84134292602539,
      "activations/layer7_attention_weight_max": 80.25894165039062,
      "activations/layer7_attention_weight_min": -77.23820495605469,
      "activations/layer8_attention_weight_max": 48.2202033996582,
      "activations/layer8_attention_weight_min": -47.86381530761719,
      "activations/layer9_attention_weight_max": 48.54884719848633,
      "activations/layer9_attention_weight_min": -49.28998947143555,
      "epoch": 9.04,
      "learning_rate": 9.26090909090909e-05,
      "loss": 2.8042,
      "step": 155650
    },
    {
      "activations/layer0_attention_weight_max": 15.640729904174805,
      "activations/layer0_attention_weight_min": -14.482553482055664,
      "activations/layer10_attention_weight_max": 36.87979507446289,
      "activations/layer10_attention_weight_min": -37.54621505737305,
      "activations/layer11_attention_weight_max": 31.679319381713867,
      "activations/layer11_attention_weight_min": -32.21991729736328,
      "activations/layer12_attention_weight_max": 19.931188583374023,
      "activations/layer12_attention_weight_min": -24.588760375976562,
      "activations/layer13_attention_weight_max": 40.486419677734375,
      "activations/layer13_attention_weight_min": -28.54242706298828,
      "activations/layer14_attention_weight_max": 42.775718688964844,
      "activations/layer14_attention_weight_min": -30.616268157958984,
      "activations/layer15_attention_weight_max": 35.41951370239258,
      "activations/layer15_attention_weight_min": -29.563615798950195,
      "activations/layer16_attention_weight_max": 35.72136306762695,
      "activations/layer16_attention_weight_min": -28.883792877197266,
      "activations/layer17_attention_weight_max": 54.06345748901367,
      "activations/layer17_attention_weight_min": -46.413177490234375,
      "activations/layer18_attention_weight_max": 52.282920837402344,
      "activations/layer18_attention_weight_min": -39.48765563964844,
      "activations/layer19_attention_weight_max": 23.118511199951172,
      "activations/layer19_attention_weight_min": -22.286623001098633,
      "activations/layer1_attention_weight_max": 17.71393394470215,
      "activations/layer1_attention_weight_min": -13.695523262023926,
      "activations/layer20_attention_weight_max": 25.821537017822266,
      "activations/layer20_attention_weight_min": -21.531103134155273,
      "activations/layer21_attention_weight_max": 49.87617111206055,
      "activations/layer21_attention_weight_min": -24.354232788085938,
      "activations/layer22_attention_weight_max": 34.62971496582031,
      "activations/layer22_attention_weight_min": -25.32339859008789,
      "activations/layer23_attention_weight_max": 39.40665054321289,
      "activations/layer23_attention_weight_min": -23.16101837158203,
      "activations/layer2_attention_weight_max": 29.917877197265625,
      "activations/layer2_attention_weight_min": -30.27715301513672,
      "activations/layer3_attention_weight_max": 85.25541687011719,
      "activations/layer3_attention_weight_min": -90.53761291503906,
      "activations/layer4_attention_weight_max": 95.72825622558594,
      "activations/layer4_attention_weight_min": -91.67811584472656,
      "activations/layer5_attention_weight_max": 79.65039825439453,
      "activations/layer5_attention_weight_min": -78.42428588867188,
      "activations/layer6_attention_weight_max": 57.419654846191406,
      "activations/layer6_attention_weight_min": -54.79832458496094,
      "activations/layer7_attention_weight_max": 78.12831115722656,
      "activations/layer7_attention_weight_min": -75.13569641113281,
      "activations/layer8_attention_weight_max": 45.81343460083008,
      "activations/layer8_attention_weight_min": -47.48091506958008,
      "activations/layer9_attention_weight_max": 46.6507568359375,
      "activations/layer9_attention_weight_min": -48.98763656616211,
      "epoch": 9.05,
      "learning_rate": 9.25901515151515e-05,
      "loss": 2.7921,
      "step": 155700
    },
    {
      "activations/layer0_attention_weight_max": 15.778693199157715,
      "activations/layer0_attention_weight_min": -14.154621124267578,
      "activations/layer10_attention_weight_max": 36.442386627197266,
      "activations/layer10_attention_weight_min": -35.20933151245117,
      "activations/layer11_attention_weight_max": 32.09191131591797,
      "activations/layer11_attention_weight_min": -32.339820861816406,
      "activations/layer12_attention_weight_max": 19.898344039916992,
      "activations/layer12_attention_weight_min": -26.82242774963379,
      "activations/layer13_attention_weight_max": 44.52634048461914,
      "activations/layer13_attention_weight_min": -30.358734130859375,
      "activations/layer14_attention_weight_max": 49.0578498840332,
      "activations/layer14_attention_weight_min": -31.257354736328125,
      "activations/layer15_attention_weight_max": 38.46095657348633,
      "activations/layer15_attention_weight_min": -30.377336502075195,
      "activations/layer16_attention_weight_max": 34.13031768798828,
      "activations/layer16_attention_weight_min": -29.25343132019043,
      "activations/layer17_attention_weight_max": 56.37932586669922,
      "activations/layer17_attention_weight_min": -44.023956298828125,
      "activations/layer18_attention_weight_max": 47.77460479736328,
      "activations/layer18_attention_weight_min": -37.44271469116211,
      "activations/layer19_attention_weight_max": 19.911579132080078,
      "activations/layer19_attention_weight_min": -20.695329666137695,
      "activations/layer1_attention_weight_max": 16.447072982788086,
      "activations/layer1_attention_weight_min": -13.367775917053223,
      "activations/layer20_attention_weight_max": 20.998985290527344,
      "activations/layer20_attention_weight_min": -19.930686950683594,
      "activations/layer21_attention_weight_max": 35.83287811279297,
      "activations/layer21_attention_weight_min": -21.905635833740234,
      "activations/layer22_attention_weight_max": 32.32268142700195,
      "activations/layer22_attention_weight_min": -24.901832580566406,
      "activations/layer23_attention_weight_max": 34.51694869995117,
      "activations/layer23_attention_weight_min": -21.0703125,
      "activations/layer2_attention_weight_max": 30.480392456054688,
      "activations/layer2_attention_weight_min": -30.209341049194336,
      "activations/layer3_attention_weight_max": 87.80126953125,
      "activations/layer3_attention_weight_min": -90.61418151855469,
      "activations/layer4_attention_weight_max": 95.99059295654297,
      "activations/layer4_attention_weight_min": -95.94243621826172,
      "activations/layer5_attention_weight_max": 78.83209228515625,
      "activations/layer5_attention_weight_min": -76.60047149658203,
      "activations/layer6_attention_weight_max": 55.289981842041016,
      "activations/layer6_attention_weight_min": -54.35874938964844,
      "activations/layer7_attention_weight_max": 78.0478286743164,
      "activations/layer7_attention_weight_min": -76.9837646484375,
      "activations/layer8_attention_weight_max": 44.466278076171875,
      "activations/layer8_attention_weight_min": -46.56108856201172,
      "activations/layer9_attention_weight_max": 51.25386428833008,
      "activations/layer9_attention_weight_min": -51.661067962646484,
      "epoch": 9.05,
      "learning_rate": 9.257121212121211e-05,
      "loss": 2.7954,
      "step": 155750
    },
    {
      "activations/layer0_attention_weight_max": 15.579278945922852,
      "activations/layer0_attention_weight_min": -14.143805503845215,
      "activations/layer10_attention_weight_max": 38.37604522705078,
      "activations/layer10_attention_weight_min": -35.93336868286133,
      "activations/layer11_attention_weight_max": 34.895904541015625,
      "activations/layer11_attention_weight_min": -32.87245559692383,
      "activations/layer12_attention_weight_max": 20.933557510375977,
      "activations/layer12_attention_weight_min": -26.41825294494629,
      "activations/layer13_attention_weight_max": 46.36804962158203,
      "activations/layer13_attention_weight_min": -29.44704818725586,
      "activations/layer14_attention_weight_max": 40.93687438964844,
      "activations/layer14_attention_weight_min": -29.614910125732422,
      "activations/layer15_attention_weight_max": 36.93738555908203,
      "activations/layer15_attention_weight_min": -28.41649055480957,
      "activations/layer16_attention_weight_max": 36.535274505615234,
      "activations/layer16_attention_weight_min": -28.465286254882812,
      "activations/layer17_attention_weight_max": 57.1299934387207,
      "activations/layer17_attention_weight_min": -42.842403411865234,
      "activations/layer18_attention_weight_max": 52.030113220214844,
      "activations/layer18_attention_weight_min": -34.743309020996094,
      "activations/layer19_attention_weight_max": 22.540380477905273,
      "activations/layer19_attention_weight_min": -20.249637603759766,
      "activations/layer1_attention_weight_max": 17.995912551879883,
      "activations/layer1_attention_weight_min": -13.939443588256836,
      "activations/layer20_attention_weight_max": 22.264007568359375,
      "activations/layer20_attention_weight_min": -19.88536262512207,
      "activations/layer21_attention_weight_max": 43.563262939453125,
      "activations/layer21_attention_weight_min": -26.0697021484375,
      "activations/layer22_attention_weight_max": 30.981138229370117,
      "activations/layer22_attention_weight_min": -25.795501708984375,
      "activations/layer23_attention_weight_max": 36.10143280029297,
      "activations/layer23_attention_weight_min": -21.383037567138672,
      "activations/layer2_attention_weight_max": 30.16286849975586,
      "activations/layer2_attention_weight_min": -26.855945587158203,
      "activations/layer3_attention_weight_max": 82.50723266601562,
      "activations/layer3_attention_weight_min": -88.16393280029297,
      "activations/layer4_attention_weight_max": 96.71875762939453,
      "activations/layer4_attention_weight_min": -92.68962860107422,
      "activations/layer5_attention_weight_max": 81.48658752441406,
      "activations/layer5_attention_weight_min": -78.85977172851562,
      "activations/layer6_attention_weight_max": 52.96334457397461,
      "activations/layer6_attention_weight_min": -54.75472640991211,
      "activations/layer7_attention_weight_max": 76.632080078125,
      "activations/layer7_attention_weight_min": -77.1198501586914,
      "activations/layer8_attention_weight_max": 45.47929000854492,
      "activations/layer8_attention_weight_min": -47.57387924194336,
      "activations/layer9_attention_weight_max": 49.29105758666992,
      "activations/layer9_attention_weight_min": -51.29243850708008,
      "epoch": 9.05,
      "learning_rate": 9.255227272727272e-05,
      "loss": 2.7987,
      "step": 155800
    },
    {
      "activations/layer0_attention_weight_max": 14.781034469604492,
      "activations/layer0_attention_weight_min": -14.093209266662598,
      "activations/layer10_attention_weight_max": 40.933349609375,
      "activations/layer10_attention_weight_min": -35.351680755615234,
      "activations/layer11_attention_weight_max": 34.22945785522461,
      "activations/layer11_attention_weight_min": -32.554420471191406,
      "activations/layer12_attention_weight_max": 37.26497268676758,
      "activations/layer12_attention_weight_min": -25.53860855102539,
      "activations/layer13_attention_weight_max": 55.951934814453125,
      "activations/layer13_attention_weight_min": -35.70401382446289,
      "activations/layer14_attention_weight_max": 40.68735885620117,
      "activations/layer14_attention_weight_min": -30.514476776123047,
      "activations/layer15_attention_weight_max": 43.0158805847168,
      "activations/layer15_attention_weight_min": -29.845399856567383,
      "activations/layer16_attention_weight_max": 34.357582092285156,
      "activations/layer16_attention_weight_min": -28.842308044433594,
      "activations/layer17_attention_weight_max": 51.800018310546875,
      "activations/layer17_attention_weight_min": -43.76579284667969,
      "activations/layer18_attention_weight_max": 46.47333526611328,
      "activations/layer18_attention_weight_min": -34.5908203125,
      "activations/layer19_attention_weight_max": 24.39089012145996,
      "activations/layer19_attention_weight_min": -20.174823760986328,
      "activations/layer1_attention_weight_max": 16.790538787841797,
      "activations/layer1_attention_weight_min": -14.470205307006836,
      "activations/layer20_attention_weight_max": 22.625812530517578,
      "activations/layer20_attention_weight_min": -21.78075408935547,
      "activations/layer21_attention_weight_max": 34.36983871459961,
      "activations/layer21_attention_weight_min": -22.059667587280273,
      "activations/layer22_attention_weight_max": 30.278793334960938,
      "activations/layer22_attention_weight_min": -25.369544982910156,
      "activations/layer23_attention_weight_max": 34.31923294067383,
      "activations/layer23_attention_weight_min": -22.7421932220459,
      "activations/layer2_attention_weight_max": 30.542789459228516,
      "activations/layer2_attention_weight_min": -29.410017013549805,
      "activations/layer3_attention_weight_max": 87.4714584350586,
      "activations/layer3_attention_weight_min": -89.03941345214844,
      "activations/layer4_attention_weight_max": 105.49144744873047,
      "activations/layer4_attention_weight_min": -95.45941925048828,
      "activations/layer5_attention_weight_max": 86.85848999023438,
      "activations/layer5_attention_weight_min": -77.800048828125,
      "activations/layer6_attention_weight_max": 59.744144439697266,
      "activations/layer6_attention_weight_min": -53.91295623779297,
      "activations/layer7_attention_weight_max": 80.20521545410156,
      "activations/layer7_attention_weight_min": -76.78890228271484,
      "activations/layer8_attention_weight_max": 48.94064712524414,
      "activations/layer8_attention_weight_min": -50.423583984375,
      "activations/layer9_attention_weight_max": 54.98696517944336,
      "activations/layer9_attention_weight_min": -52.54354476928711,
      "epoch": 9.06,
      "learning_rate": 9.253333333333334e-05,
      "loss": 2.805,
      "step": 155850
    },
    {
      "activations/layer0_attention_weight_max": 15.260313034057617,
      "activations/layer0_attention_weight_min": -14.103618621826172,
      "activations/layer10_attention_weight_max": 39.342247009277344,
      "activations/layer10_attention_weight_min": -38.72285842895508,
      "activations/layer11_attention_weight_max": 33.874908447265625,
      "activations/layer11_attention_weight_min": -32.77493667602539,
      "activations/layer12_attention_weight_max": 24.741077423095703,
      "activations/layer12_attention_weight_min": -24.875202178955078,
      "activations/layer13_attention_weight_max": 44.29349136352539,
      "activations/layer13_attention_weight_min": -31.724884033203125,
      "activations/layer14_attention_weight_max": 46.94095993041992,
      "activations/layer14_attention_weight_min": -31.357349395751953,
      "activations/layer15_attention_weight_max": 39.673683166503906,
      "activations/layer15_attention_weight_min": -31.918743133544922,
      "activations/layer16_attention_weight_max": 33.107540130615234,
      "activations/layer16_attention_weight_min": -28.67957305908203,
      "activations/layer17_attention_weight_max": 51.11201477050781,
      "activations/layer17_attention_weight_min": -45.5247917175293,
      "activations/layer18_attention_weight_max": 46.72238540649414,
      "activations/layer18_attention_weight_min": -37.92439270019531,
      "activations/layer19_attention_weight_max": 22.796377182006836,
      "activations/layer19_attention_weight_min": -20.512083053588867,
      "activations/layer1_attention_weight_max": 18.33318519592285,
      "activations/layer1_attention_weight_min": -15.57935619354248,
      "activations/layer20_attention_weight_max": 25.399229049682617,
      "activations/layer20_attention_weight_min": -22.332263946533203,
      "activations/layer21_attention_weight_max": 44.977088928222656,
      "activations/layer21_attention_weight_min": -24.222171783447266,
      "activations/layer22_attention_weight_max": 33.82551574707031,
      "activations/layer22_attention_weight_min": -26.74778175354004,
      "activations/layer23_attention_weight_max": 38.43093490600586,
      "activations/layer23_attention_weight_min": -22.599769592285156,
      "activations/layer2_attention_weight_max": 31.314128875732422,
      "activations/layer2_attention_weight_min": -28.414207458496094,
      "activations/layer3_attention_weight_max": 85.98258209228516,
      "activations/layer3_attention_weight_min": -87.13745880126953,
      "activations/layer4_attention_weight_max": 98.9482421875,
      "activations/layer4_attention_weight_min": -92.8991928100586,
      "activations/layer5_attention_weight_max": 78.80716705322266,
      "activations/layer5_attention_weight_min": -75.63138580322266,
      "activations/layer6_attention_weight_max": 52.89213562011719,
      "activations/layer6_attention_weight_min": -52.41469955444336,
      "activations/layer7_attention_weight_max": 73.72611236572266,
      "activations/layer7_attention_weight_min": -73.36558532714844,
      "activations/layer8_attention_weight_max": 48.819828033447266,
      "activations/layer8_attention_weight_min": -48.838470458984375,
      "activations/layer9_attention_weight_max": 49.63450241088867,
      "activations/layer9_attention_weight_min": -50.029151916503906,
      "epoch": 9.06,
      "learning_rate": 9.251477272727273e-05,
      "loss": 2.7992,
      "step": 155900
    },
    {
      "activations/layer0_attention_weight_max": 14.676400184631348,
      "activations/layer0_attention_weight_min": -14.067214965820312,
      "activations/layer10_attention_weight_max": 35.53028869628906,
      "activations/layer10_attention_weight_min": -35.50627136230469,
      "activations/layer11_attention_weight_max": 29.30262565612793,
      "activations/layer11_attention_weight_min": -32.39862060546875,
      "activations/layer12_attention_weight_max": 17.53035545349121,
      "activations/layer12_attention_weight_min": -24.04116439819336,
      "activations/layer13_attention_weight_max": 38.80860900878906,
      "activations/layer13_attention_weight_min": -30.539897918701172,
      "activations/layer14_attention_weight_max": 35.99601364135742,
      "activations/layer14_attention_weight_min": -31.7185001373291,
      "activations/layer15_attention_weight_max": 31.74154281616211,
      "activations/layer15_attention_weight_min": -29.969375610351562,
      "activations/layer16_attention_weight_max": 28.737384796142578,
      "activations/layer16_attention_weight_min": -27.98430633544922,
      "activations/layer17_attention_weight_max": 50.733123779296875,
      "activations/layer17_attention_weight_min": -42.98631286621094,
      "activations/layer18_attention_weight_max": 41.27003479003906,
      "activations/layer18_attention_weight_min": -38.11005783081055,
      "activations/layer19_attention_weight_max": 19.899492263793945,
      "activations/layer19_attention_weight_min": -23.59035301208496,
      "activations/layer1_attention_weight_max": 16.40492057800293,
      "activations/layer1_attention_weight_min": -15.197957038879395,
      "activations/layer20_attention_weight_max": 20.75128173828125,
      "activations/layer20_attention_weight_min": -25.661325454711914,
      "activations/layer21_attention_weight_max": 35.78973388671875,
      "activations/layer21_attention_weight_min": -23.25865936279297,
      "activations/layer22_attention_weight_max": 29.262428283691406,
      "activations/layer22_attention_weight_min": -28.401052474975586,
      "activations/layer23_attention_weight_max": 35.589820861816406,
      "activations/layer23_attention_weight_min": -25.352346420288086,
      "activations/layer2_attention_weight_max": 29.451610565185547,
      "activations/layer2_attention_weight_min": -29.996694564819336,
      "activations/layer3_attention_weight_max": 83.34585571289062,
      "activations/layer3_attention_weight_min": -85.4029541015625,
      "activations/layer4_attention_weight_max": 92.08004760742188,
      "activations/layer4_attention_weight_min": -91.21782684326172,
      "activations/layer5_attention_weight_max": 74.97706604003906,
      "activations/layer5_attention_weight_min": -78.41859436035156,
      "activations/layer6_attention_weight_max": 53.29789352416992,
      "activations/layer6_attention_weight_min": -47.978721618652344,
      "activations/layer7_attention_weight_max": 76.14403533935547,
      "activations/layer7_attention_weight_min": -69.19894409179688,
      "activations/layer8_attention_weight_max": 43.0687141418457,
      "activations/layer8_attention_weight_min": -42.7429084777832,
      "activations/layer9_attention_weight_max": 47.458744049072266,
      "activations/layer9_attention_weight_min": -49.112300872802734,
      "epoch": 9.06,
      "learning_rate": 9.249583333333332e-05,
      "loss": 2.7966,
      "step": 155950
    },
    {
      "activations/layer0_attention_weight_max": 15.925387382507324,
      "activations/layer0_attention_weight_min": -14.415396690368652,
      "activations/layer10_attention_weight_max": 34.81740188598633,
      "activations/layer10_attention_weight_min": -37.448997497558594,
      "activations/layer11_attention_weight_max": 32.827247619628906,
      "activations/layer11_attention_weight_min": -31.919654846191406,
      "activations/layer12_attention_weight_max": 18.261369705200195,
      "activations/layer12_attention_weight_min": -34.035430908203125,
      "activations/layer13_attention_weight_max": 44.21657943725586,
      "activations/layer13_attention_weight_min": -35.731117248535156,
      "activations/layer14_attention_weight_max": 59.6357536315918,
      "activations/layer14_attention_weight_min": -38.89631271362305,
      "activations/layer15_attention_weight_max": 47.52067565917969,
      "activations/layer15_attention_weight_min": -31.723590850830078,
      "activations/layer16_attention_weight_max": 27.942821502685547,
      "activations/layer16_attention_weight_min": -27.28081512451172,
      "activations/layer17_attention_weight_max": 52.02933883666992,
      "activations/layer17_attention_weight_min": -41.69436264038086,
      "activations/layer18_attention_weight_max": 41.9045524597168,
      "activations/layer18_attention_weight_min": -35.908843994140625,
      "activations/layer19_attention_weight_max": 18.97206687927246,
      "activations/layer19_attention_weight_min": -19.98935317993164,
      "activations/layer1_attention_weight_max": 18.933298110961914,
      "activations/layer1_attention_weight_min": -15.147401809692383,
      "activations/layer20_attention_weight_max": 21.48666763305664,
      "activations/layer20_attention_weight_min": -21.75041389465332,
      "activations/layer21_attention_weight_max": 32.01789093017578,
      "activations/layer21_attention_weight_min": -24.413372039794922,
      "activations/layer22_attention_weight_max": 26.862621307373047,
      "activations/layer22_attention_weight_min": -25.638072967529297,
      "activations/layer23_attention_weight_max": 30.24574851989746,
      "activations/layer23_attention_weight_min": -25.01790428161621,
      "activations/layer2_attention_weight_max": 32.251365661621094,
      "activations/layer2_attention_weight_min": -31.63290023803711,
      "activations/layer3_attention_weight_max": 88.3017578125,
      "activations/layer3_attention_weight_min": -89.82347869873047,
      "activations/layer4_attention_weight_max": 93.9184799194336,
      "activations/layer4_attention_weight_min": -92.91234588623047,
      "activations/layer5_attention_weight_max": 72.94170379638672,
      "activations/layer5_attention_weight_min": -76.55120849609375,
      "activations/layer6_attention_weight_max": 51.38230895996094,
      "activations/layer6_attention_weight_min": -51.50826644897461,
      "activations/layer7_attention_weight_max": 80.65618896484375,
      "activations/layer7_attention_weight_min": -76.05919647216797,
      "activations/layer8_attention_weight_max": 44.68876266479492,
      "activations/layer8_attention_weight_min": -43.55485916137695,
      "activations/layer9_attention_weight_max": 61.88962936401367,
      "activations/layer9_attention_weight_min": -53.27236557006836,
      "epoch": 9.06,
      "learning_rate": 9.247689393939393e-05,
      "loss": 2.7975,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_loss": 2.75390625,
      "eval_runtime": 8.5171,
      "eval_samples_per_second": 504.16,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_openwebtext_loss": 2.75390625,
      "eval_openwebtext_ppl": 15.703855414343392,
      "eval_openwebtext_runtime": 8.5171,
      "eval_openwebtext_samples_per_second": 504.16,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_wikitext_loss": 2.982421875,
      "eval_wikitext_ppl": 19.735555861572347,
      "eval_wikitext_runtime": 2.0149,
      "eval_wikitext_samples_per_second": 226.318,
      "step": 156000
    },
    {
      "epoch": 9.06,
      "eval_lambada_loss": 2.712890625,
      "eval_lambada_ppl": 15.072782365110845,
      "eval_lambada_runtime": 9.6165,
      "eval_lambada_samples_per_second": 506.318,
      "step": 156000
    },
    {
      "activations/layer0_attention_weight_max": 15.68367862701416,
      "activations/layer0_attention_weight_min": -13.97088623046875,
      "activations/layer10_attention_weight_max": 37.41853713989258,
      "activations/layer10_attention_weight_min": -37.05970764160156,
      "activations/layer11_attention_weight_max": 32.398826599121094,
      "activations/layer11_attention_weight_min": -33.39010238647461,
      "activations/layer12_attention_weight_max": 19.4442195892334,
      "activations/layer12_attention_weight_min": -26.645198822021484,
      "activations/layer13_attention_weight_max": 41.3732795715332,
      "activations/layer13_attention_weight_min": -30.785585403442383,
      "activations/layer14_attention_weight_max": 41.27946472167969,
      "activations/layer14_attention_weight_min": -30.73981475830078,
      "activations/layer15_attention_weight_max": 34.357295989990234,
      "activations/layer15_attention_weight_min": -28.626327514648438,
      "activations/layer16_attention_weight_max": 31.334285736083984,
      "activations/layer16_attention_weight_min": -29.259918212890625,
      "activations/layer17_attention_weight_max": 53.363399505615234,
      "activations/layer17_attention_weight_min": -44.34243392944336,
      "activations/layer18_attention_weight_max": 46.62035369873047,
      "activations/layer18_attention_weight_min": -38.32685470581055,
      "activations/layer19_attention_weight_max": 23.251802444458008,
      "activations/layer19_attention_weight_min": -21.05811309814453,
      "activations/layer1_attention_weight_max": 17.09418296813965,
      "activations/layer1_attention_weight_min": -14.753554344177246,
      "activations/layer20_attention_weight_max": 21.025114059448242,
      "activations/layer20_attention_weight_min": -23.1138973236084,
      "activations/layer21_attention_weight_max": 40.32536697387695,
      "activations/layer21_attention_weight_min": -25.923748016357422,
      "activations/layer22_attention_weight_max": 30.42867088317871,
      "activations/layer22_attention_weight_min": -25.578386306762695,
      "activations/layer23_attention_weight_max": 37.24754333496094,
      "activations/layer23_attention_weight_min": -27.378629684448242,
      "activations/layer2_attention_weight_max": 30.017990112304688,
      "activations/layer2_attention_weight_min": -28.90106201171875,
      "activations/layer3_attention_weight_max": 87.88472747802734,
      "activations/layer3_attention_weight_min": -89.04126739501953,
      "activations/layer4_attention_weight_max": 101.43123626708984,
      "activations/layer4_attention_weight_min": -93.26528930664062,
      "activations/layer5_attention_weight_max": 79.74728393554688,
      "activations/layer5_attention_weight_min": -76.98094940185547,
      "activations/layer6_attention_weight_max": 55.951927185058594,
      "activations/layer6_attention_weight_min": -53.71767044067383,
      "activations/layer7_attention_weight_max": 81.639404296875,
      "activations/layer7_attention_weight_min": -74.6139144897461,
      "activations/layer8_attention_weight_max": 44.95930480957031,
      "activations/layer8_attention_weight_min": -45.44184494018555,
      "activations/layer9_attention_weight_max": 54.247589111328125,
      "activations/layer9_attention_weight_min": -51.142269134521484,
      "epoch": 9.07,
      "learning_rate": 9.245795454545455e-05,
      "loss": 2.7883,
      "step": 156050
    },
    {
      "activations/layer0_attention_weight_max": 15.044940948486328,
      "activations/layer0_attention_weight_min": -14.458552360534668,
      "activations/layer10_attention_weight_max": 37.80863571166992,
      "activations/layer10_attention_weight_min": -36.17387008666992,
      "activations/layer11_attention_weight_max": 32.17693328857422,
      "activations/layer11_attention_weight_min": -32.99908447265625,
      "activations/layer12_attention_weight_max": 16.345457077026367,
      "activations/layer12_attention_weight_min": -25.198108673095703,
      "activations/layer13_attention_weight_max": 50.22890853881836,
      "activations/layer13_attention_weight_min": -37.824825286865234,
      "activations/layer14_attention_weight_max": 58.57101058959961,
      "activations/layer14_attention_weight_min": -39.53789520263672,
      "activations/layer15_attention_weight_max": 45.25847625732422,
      "activations/layer15_attention_weight_min": -31.355369567871094,
      "activations/layer16_attention_weight_max": 29.113908767700195,
      "activations/layer16_attention_weight_min": -29.52361488342285,
      "activations/layer17_attention_weight_max": 52.20923614501953,
      "activations/layer17_attention_weight_min": -43.71400451660156,
      "activations/layer18_attention_weight_max": 44.03328323364258,
      "activations/layer18_attention_weight_min": -38.625640869140625,
      "activations/layer19_attention_weight_max": 22.00824546813965,
      "activations/layer19_attention_weight_min": -22.230815887451172,
      "activations/layer1_attention_weight_max": 17.09604835510254,
      "activations/layer1_attention_weight_min": -15.209275245666504,
      "activations/layer20_attention_weight_max": 23.07893943786621,
      "activations/layer20_attention_weight_min": -23.38686180114746,
      "activations/layer21_attention_weight_max": 41.08341598510742,
      "activations/layer21_attention_weight_min": -26.40185546875,
      "activations/layer22_attention_weight_max": 36.0939826965332,
      "activations/layer22_attention_weight_min": -26.37236213684082,
      "activations/layer23_attention_weight_max": 39.51709747314453,
      "activations/layer23_attention_weight_min": -21.65744400024414,
      "activations/layer2_attention_weight_max": 30.306392669677734,
      "activations/layer2_attention_weight_min": -29.041807174682617,
      "activations/layer3_attention_weight_max": 91.07070922851562,
      "activations/layer3_attention_weight_min": -93.94956970214844,
      "activations/layer4_attention_weight_max": 102.94884490966797,
      "activations/layer4_attention_weight_min": -98.08190155029297,
      "activations/layer5_attention_weight_max": 82.69366455078125,
      "activations/layer5_attention_weight_min": -78.1064224243164,
      "activations/layer6_attention_weight_max": 56.97732925415039,
      "activations/layer6_attention_weight_min": -53.59028625488281,
      "activations/layer7_attention_weight_max": 78.22345733642578,
      "activations/layer7_attention_weight_min": -77.54666900634766,
      "activations/layer8_attention_weight_max": 45.487586975097656,
      "activations/layer8_attention_weight_min": -47.71797180175781,
      "activations/layer9_attention_weight_max": 55.8458137512207,
      "activations/layer9_attention_weight_min": -54.53829574584961,
      "epoch": 9.07,
      "learning_rate": 9.243901515151513e-05,
      "loss": 2.8036,
      "step": 156100
    },
    {
      "activations/layer0_attention_weight_max": 16.484895706176758,
      "activations/layer0_attention_weight_min": -13.758331298828125,
      "activations/layer10_attention_weight_max": 38.90670394897461,
      "activations/layer10_attention_weight_min": -37.079837799072266,
      "activations/layer11_attention_weight_max": 32.712913513183594,
      "activations/layer11_attention_weight_min": -35.29338836669922,
      "activations/layer12_attention_weight_max": 22.340599060058594,
      "activations/layer12_attention_weight_min": -25.01421546936035,
      "activations/layer13_attention_weight_max": 41.49968338012695,
      "activations/layer13_attention_weight_min": -31.52123260498047,
      "activations/layer14_attention_weight_max": 36.60449981689453,
      "activations/layer14_attention_weight_min": -31.225126266479492,
      "activations/layer15_attention_weight_max": 37.722496032714844,
      "activations/layer15_attention_weight_min": -28.982505798339844,
      "activations/layer16_attention_weight_max": 35.15442657470703,
      "activations/layer16_attention_weight_min": -26.809011459350586,
      "activations/layer17_attention_weight_max": 56.74074935913086,
      "activations/layer17_attention_weight_min": -44.411170959472656,
      "activations/layer18_attention_weight_max": 49.096107482910156,
      "activations/layer18_attention_weight_min": -36.10841751098633,
      "activations/layer19_attention_weight_max": 26.610584259033203,
      "activations/layer19_attention_weight_min": -21.471506118774414,
      "activations/layer1_attention_weight_max": 17.19495391845703,
      "activations/layer1_attention_weight_min": -16.375003814697266,
      "activations/layer20_attention_weight_max": 22.988309860229492,
      "activations/layer20_attention_weight_min": -20.1060791015625,
      "activations/layer21_attention_weight_max": 36.002197265625,
      "activations/layer21_attention_weight_min": -23.5720272064209,
      "activations/layer22_attention_weight_max": 34.18256378173828,
      "activations/layer22_attention_weight_min": -24.5841121673584,
      "activations/layer23_attention_weight_max": 39.03561782836914,
      "activations/layer23_attention_weight_min": -24.10256576538086,
      "activations/layer2_attention_weight_max": 29.257030487060547,
      "activations/layer2_attention_weight_min": -28.781110763549805,
      "activations/layer3_attention_weight_max": 85.76569366455078,
      "activations/layer3_attention_weight_min": -86.88752746582031,
      "activations/layer4_attention_weight_max": 97.20439910888672,
      "activations/layer4_attention_weight_min": -92.17928314208984,
      "activations/layer5_attention_weight_max": 78.42040252685547,
      "activations/layer5_attention_weight_min": -80.59767150878906,
      "activations/layer6_attention_weight_max": 55.7842903137207,
      "activations/layer6_attention_weight_min": -52.00190734863281,
      "activations/layer7_attention_weight_max": 76.82503509521484,
      "activations/layer7_attention_weight_min": -73.06452941894531,
      "activations/layer8_attention_weight_max": 46.125465393066406,
      "activations/layer8_attention_weight_min": -45.925811767578125,
      "activations/layer9_attention_weight_max": 51.85551071166992,
      "activations/layer9_attention_weight_min": -47.611045837402344,
      "epoch": 9.07,
      "learning_rate": 9.242007575757575e-05,
      "loss": 2.8104,
      "step": 156150
    },
    {
      "activations/layer0_attention_weight_max": 15.695052146911621,
      "activations/layer0_attention_weight_min": -14.27829647064209,
      "activations/layer10_attention_weight_max": 35.71070861816406,
      "activations/layer10_attention_weight_min": -34.97601318359375,
      "activations/layer11_attention_weight_max": 31.24988555908203,
      "activations/layer11_attention_weight_min": -31.217777252197266,
      "activations/layer12_attention_weight_max": 20.1434383392334,
      "activations/layer12_attention_weight_min": -25.496150970458984,
      "activations/layer13_attention_weight_max": 40.99585723876953,
      "activations/layer13_attention_weight_min": -32.05684280395508,
      "activations/layer14_attention_weight_max": 41.95601272583008,
      "activations/layer14_attention_weight_min": -32.44139862060547,
      "activations/layer15_attention_weight_max": 35.43768310546875,
      "activations/layer15_attention_weight_min": -29.335500717163086,
      "activations/layer16_attention_weight_max": 33.91480255126953,
      "activations/layer16_attention_weight_min": -29.273406982421875,
      "activations/layer17_attention_weight_max": 56.466983795166016,
      "activations/layer17_attention_weight_min": -46.354793548583984,
      "activations/layer18_attention_weight_max": 51.71066665649414,
      "activations/layer18_attention_weight_min": -36.70206069946289,
      "activations/layer19_attention_weight_max": 28.456296920776367,
      "activations/layer19_attention_weight_min": -20.2882022857666,
      "activations/layer1_attention_weight_max": 16.976438522338867,
      "activations/layer1_attention_weight_min": -14.956938743591309,
      "activations/layer20_attention_weight_max": 24.914159774780273,
      "activations/layer20_attention_weight_min": -22.591724395751953,
      "activations/layer21_attention_weight_max": 40.912357330322266,
      "activations/layer21_attention_weight_min": -26.66360855102539,
      "activations/layer22_attention_weight_max": 35.15024948120117,
      "activations/layer22_attention_weight_min": -25.12411880493164,
      "activations/layer23_attention_weight_max": 40.40305709838867,
      "activations/layer23_attention_weight_min": -20.875293731689453,
      "activations/layer2_attention_weight_max": 29.35487937927246,
      "activations/layer2_attention_weight_min": -27.247671127319336,
      "activations/layer3_attention_weight_max": 84.38135528564453,
      "activations/layer3_attention_weight_min": -84.20895385742188,
      "activations/layer4_attention_weight_max": 94.83882904052734,
      "activations/layer4_attention_weight_min": -90.57567596435547,
      "activations/layer5_attention_weight_max": 78.7322998046875,
      "activations/layer5_attention_weight_min": -76.01576232910156,
      "activations/layer6_attention_weight_max": 53.155799865722656,
      "activations/layer6_attention_weight_min": -52.093360900878906,
      "activations/layer7_attention_weight_max": 78.30804443359375,
      "activations/layer7_attention_weight_min": -77.27740478515625,
      "activations/layer8_attention_weight_max": 44.49946975708008,
      "activations/layer8_attention_weight_min": -46.68987274169922,
      "activations/layer9_attention_weight_max": 47.56755828857422,
      "activations/layer9_attention_weight_min": -48.4965934753418,
      "epoch": 9.08,
      "learning_rate": 9.240113636363636e-05,
      "loss": 2.7981,
      "step": 156200
    },
    {
      "activations/layer0_attention_weight_max": 16.104915618896484,
      "activations/layer0_attention_weight_min": -14.2057523727417,
      "activations/layer10_attention_weight_max": 37.530738830566406,
      "activations/layer10_attention_weight_min": -39.22314453125,
      "activations/layer11_attention_weight_max": 33.01584243774414,
      "activations/layer11_attention_weight_min": -35.31476593017578,
      "activations/layer12_attention_weight_max": 21.271329879760742,
      "activations/layer12_attention_weight_min": -25.28766632080078,
      "activations/layer13_attention_weight_max": 40.746761322021484,
      "activations/layer13_attention_weight_min": -28.879852294921875,
      "activations/layer14_attention_weight_max": 38.91023254394531,
      "activations/layer14_attention_weight_min": -29.57478141784668,
      "activations/layer15_attention_weight_max": 33.66060256958008,
      "activations/layer15_attention_weight_min": -28.787837982177734,
      "activations/layer16_attention_weight_max": 32.758209228515625,
      "activations/layer16_attention_weight_min": -28.96179962158203,
      "activations/layer17_attention_weight_max": 53.94914627075195,
      "activations/layer17_attention_weight_min": -42.765995025634766,
      "activations/layer18_attention_weight_max": 47.652259826660156,
      "activations/layer18_attention_weight_min": -37.378387451171875,
      "activations/layer19_attention_weight_max": 24.997282028198242,
      "activations/layer19_attention_weight_min": -22.234474182128906,
      "activations/layer1_attention_weight_max": 16.613475799560547,
      "activations/layer1_attention_weight_min": -14.578274726867676,
      "activations/layer20_attention_weight_max": 21.303194046020508,
      "activations/layer20_attention_weight_min": -19.979345321655273,
      "activations/layer21_attention_weight_max": 36.08491897583008,
      "activations/layer21_attention_weight_min": -23.897329330444336,
      "activations/layer22_attention_weight_max": 32.349830627441406,
      "activations/layer22_attention_weight_min": -24.05884552001953,
      "activations/layer23_attention_weight_max": 38.0053825378418,
      "activations/layer23_attention_weight_min": -24.808002471923828,
      "activations/layer2_attention_weight_max": 29.828262329101562,
      "activations/layer2_attention_weight_min": -30.02142333984375,
      "activations/layer3_attention_weight_max": 84.67808532714844,
      "activations/layer3_attention_weight_min": -87.00015258789062,
      "activations/layer4_attention_weight_max": 98.17278289794922,
      "activations/layer4_attention_weight_min": -94.4411392211914,
      "activations/layer5_attention_weight_max": 80.9859619140625,
      "activations/layer5_attention_weight_min": -79.16539001464844,
      "activations/layer6_attention_weight_max": 54.80944061279297,
      "activations/layer6_attention_weight_min": -53.80118942260742,
      "activations/layer7_attention_weight_max": 76.05823516845703,
      "activations/layer7_attention_weight_min": -79.75982666015625,
      "activations/layer8_attention_weight_max": 45.85972213745117,
      "activations/layer8_attention_weight_min": -49.1497802734375,
      "activations/layer9_attention_weight_max": 56.13656234741211,
      "activations/layer9_attention_weight_min": -53.6939811706543,
      "epoch": 9.08,
      "learning_rate": 9.238219696969695e-05,
      "loss": 2.7952,
      "step": 156250
    },
    {
      "activations/layer0_attention_weight_max": 14.94577407836914,
      "activations/layer0_attention_weight_min": -14.527119636535645,
      "activations/layer10_attention_weight_max": 39.06498718261719,
      "activations/layer10_attention_weight_min": -37.451416015625,
      "activations/layer11_attention_weight_max": 37.49805450439453,
      "activations/layer11_attention_weight_min": -33.892093658447266,
      "activations/layer12_attention_weight_max": 20.83953285217285,
      "activations/layer12_attention_weight_min": -25.63888931274414,
      "activations/layer13_attention_weight_max": 39.349456787109375,
      "activations/layer13_attention_weight_min": -28.966651916503906,
      "activations/layer14_attention_weight_max": 43.557403564453125,
      "activations/layer14_attention_weight_min": -32.44447708129883,
      "activations/layer15_attention_weight_max": 45.09968185424805,
      "activations/layer15_attention_weight_min": -31.140640258789062,
      "activations/layer16_attention_weight_max": 32.6917839050293,
      "activations/layer16_attention_weight_min": -29.834978103637695,
      "activations/layer17_attention_weight_max": 55.252986907958984,
      "activations/layer17_attention_weight_min": -46.14666748046875,
      "activations/layer18_attention_weight_max": 51.11961364746094,
      "activations/layer18_attention_weight_min": -39.72550582885742,
      "activations/layer19_attention_weight_max": 21.5421199798584,
      "activations/layer19_attention_weight_min": -21.555965423583984,
      "activations/layer1_attention_weight_max": 16.665695190429688,
      "activations/layer1_attention_weight_min": -15.938544273376465,
      "activations/layer20_attention_weight_max": 23.198009490966797,
      "activations/layer20_attention_weight_min": -21.345746994018555,
      "activations/layer21_attention_weight_max": 44.22997283935547,
      "activations/layer21_attention_weight_min": -24.947479248046875,
      "activations/layer22_attention_weight_max": 33.76700210571289,
      "activations/layer22_attention_weight_min": -27.198579788208008,
      "activations/layer23_attention_weight_max": 35.534095764160156,
      "activations/layer23_attention_weight_min": -23.678874969482422,
      "activations/layer2_attention_weight_max": 31.28188705444336,
      "activations/layer2_attention_weight_min": -30.078243255615234,
      "activations/layer3_attention_weight_max": 88.43196868896484,
      "activations/layer3_attention_weight_min": -86.70153045654297,
      "activations/layer4_attention_weight_max": 100.3062973022461,
      "activations/layer4_attention_weight_min": -97.66236877441406,
      "activations/layer5_attention_weight_max": 84.8233642578125,
      "activations/layer5_attention_weight_min": -80.10317993164062,
      "activations/layer6_attention_weight_max": 56.915348052978516,
      "activations/layer6_attention_weight_min": -58.8619270324707,
      "activations/layer7_attention_weight_max": 83.92388916015625,
      "activations/layer7_attention_weight_min": -74.40925598144531,
      "activations/layer8_attention_weight_max": 44.25789260864258,
      "activations/layer8_attention_weight_min": -47.03435134887695,
      "activations/layer9_attention_weight_max": 54.32941436767578,
      "activations/layer9_attention_weight_min": -48.27640914916992,
      "epoch": 9.08,
      "learning_rate": 9.236325757575757e-05,
      "loss": 2.8156,
      "step": 156300
    },
    {
      "activations/layer0_attention_weight_max": 15.726187705993652,
      "activations/layer0_attention_weight_min": -14.079978942871094,
      "activations/layer10_attention_weight_max": 42.6125602722168,
      "activations/layer10_attention_weight_min": -36.19865798950195,
      "activations/layer11_attention_weight_max": 36.85858154296875,
      "activations/layer11_attention_weight_min": -32.407630920410156,
      "activations/layer12_attention_weight_max": 16.940099716186523,
      "activations/layer12_attention_weight_min": -26.60566520690918,
      "activations/layer13_attention_weight_max": 46.577884674072266,
      "activations/layer13_attention_weight_min": -32.32839584350586,
      "activations/layer14_attention_weight_max": 45.15132141113281,
      "activations/layer14_attention_weight_min": -29.75110626220703,
      "activations/layer15_attention_weight_max": 40.00383758544922,
      "activations/layer15_attention_weight_min": -29.679407119750977,
      "activations/layer16_attention_weight_max": 36.37228775024414,
      "activations/layer16_attention_weight_min": -27.72739028930664,
      "activations/layer17_attention_weight_max": 53.39237594604492,
      "activations/layer17_attention_weight_min": -43.09670639038086,
      "activations/layer18_attention_weight_max": 49.149635314941406,
      "activations/layer18_attention_weight_min": -37.15142059326172,
      "activations/layer19_attention_weight_max": 23.786352157592773,
      "activations/layer19_attention_weight_min": -21.356924057006836,
      "activations/layer1_attention_weight_max": 17.294286727905273,
      "activations/layer1_attention_weight_min": -14.695943832397461,
      "activations/layer20_attention_weight_max": 24.4658145904541,
      "activations/layer20_attention_weight_min": -22.709596633911133,
      "activations/layer21_attention_weight_max": 43.53340530395508,
      "activations/layer21_attention_weight_min": -24.826889038085938,
      "activations/layer22_attention_weight_max": 35.0267448425293,
      "activations/layer22_attention_weight_min": -24.46856117248535,
      "activations/layer23_attention_weight_max": 34.75231170654297,
      "activations/layer23_attention_weight_min": -22.351215362548828,
      "activations/layer2_attention_weight_max": 30.985036849975586,
      "activations/layer2_attention_weight_min": -29.406063079833984,
      "activations/layer3_attention_weight_max": 85.2872543334961,
      "activations/layer3_attention_weight_min": -88.31951141357422,
      "activations/layer4_attention_weight_max": 96.30998229980469,
      "activations/layer4_attention_weight_min": -95.34305572509766,
      "activations/layer5_attention_weight_max": 78.80286407470703,
      "activations/layer5_attention_weight_min": -77.185546875,
      "activations/layer6_attention_weight_max": 55.05110549926758,
      "activations/layer6_attention_weight_min": -53.36275863647461,
      "activations/layer7_attention_weight_max": 88.2171401977539,
      "activations/layer7_attention_weight_min": -77.93122100830078,
      "activations/layer8_attention_weight_max": 45.92926788330078,
      "activations/layer8_attention_weight_min": -49.31852340698242,
      "activations/layer9_attention_weight_max": 59.09583282470703,
      "activations/layer9_attention_weight_min": -50.63792037963867,
      "epoch": 9.08,
      "learning_rate": 9.234431818181817e-05,
      "loss": 2.8113,
      "step": 156350
    },
    {
      "activations/layer0_attention_weight_max": 15.215540885925293,
      "activations/layer0_attention_weight_min": -13.897374153137207,
      "activations/layer10_attention_weight_max": 40.594818115234375,
      "activations/layer10_attention_weight_min": -38.95058822631836,
      "activations/layer11_attention_weight_max": 33.840911865234375,
      "activations/layer11_attention_weight_min": -34.678218841552734,
      "activations/layer12_attention_weight_max": 18.46770668029785,
      "activations/layer12_attention_weight_min": -23.62566566467285,
      "activations/layer13_attention_weight_max": 46.89219284057617,
      "activations/layer13_attention_weight_min": -33.19106674194336,
      "activations/layer14_attention_weight_max": 44.05678176879883,
      "activations/layer14_attention_weight_min": -32.28446960449219,
      "activations/layer15_attention_weight_max": 35.9392204284668,
      "activations/layer15_attention_weight_min": -32.24472427368164,
      "activations/layer16_attention_weight_max": 35.017242431640625,
      "activations/layer16_attention_weight_min": -30.40104103088379,
      "activations/layer17_attention_weight_max": 56.85966873168945,
      "activations/layer17_attention_weight_min": -47.808040618896484,
      "activations/layer18_attention_weight_max": 55.96082305908203,
      "activations/layer18_attention_weight_min": -39.222015380859375,
      "activations/layer19_attention_weight_max": 23.703237533569336,
      "activations/layer19_attention_weight_min": -23.640167236328125,
      "activations/layer1_attention_weight_max": 16.930561065673828,
      "activations/layer1_attention_weight_min": -13.375256538391113,
      "activations/layer20_attention_weight_max": 23.65235710144043,
      "activations/layer20_attention_weight_min": -22.0335750579834,
      "activations/layer21_attention_weight_max": 47.25209426879883,
      "activations/layer21_attention_weight_min": -23.45683479309082,
      "activations/layer22_attention_weight_max": 34.33763885498047,
      "activations/layer22_attention_weight_min": -27.904804229736328,
      "activations/layer23_attention_weight_max": 37.463680267333984,
      "activations/layer23_attention_weight_min": -26.165016174316406,
      "activations/layer2_attention_weight_max": 29.54481315612793,
      "activations/layer2_attention_weight_min": -28.561508178710938,
      "activations/layer3_attention_weight_max": 84.70402526855469,
      "activations/layer3_attention_weight_min": -87.1440200805664,
      "activations/layer4_attention_weight_max": 98.23988342285156,
      "activations/layer4_attention_weight_min": -91.95828247070312,
      "activations/layer5_attention_weight_max": 83.0992431640625,
      "activations/layer5_attention_weight_min": -78.86425018310547,
      "activations/layer6_attention_weight_max": 57.085697174072266,
      "activations/layer6_attention_weight_min": -58.53986358642578,
      "activations/layer7_attention_weight_max": 79.8782958984375,
      "activations/layer7_attention_weight_min": -82.35009002685547,
      "activations/layer8_attention_weight_max": 47.07295227050781,
      "activations/layer8_attention_weight_min": -48.62108612060547,
      "activations/layer9_attention_weight_max": 55.056453704833984,
      "activations/layer9_attention_weight_min": -55.15613555908203,
      "epoch": 9.09,
      "learning_rate": 9.232537878787878e-05,
      "loss": 2.7995,
      "step": 156400
    },
    {
      "activations/layer0_attention_weight_max": 15.823530197143555,
      "activations/layer0_attention_weight_min": -13.791265487670898,
      "activations/layer10_attention_weight_max": 36.25444030761719,
      "activations/layer10_attention_weight_min": -35.94844436645508,
      "activations/layer11_attention_weight_max": 30.822141647338867,
      "activations/layer11_attention_weight_min": -31.126150131225586,
      "activations/layer12_attention_weight_max": 19.112619400024414,
      "activations/layer12_attention_weight_min": -29.27214813232422,
      "activations/layer13_attention_weight_max": 37.18603515625,
      "activations/layer13_attention_weight_min": -30.503124237060547,
      "activations/layer14_attention_weight_max": 37.790138244628906,
      "activations/layer14_attention_weight_min": -30.026453018188477,
      "activations/layer15_attention_weight_max": 34.726036071777344,
      "activations/layer15_attention_weight_min": -29.239688873291016,
      "activations/layer16_attention_weight_max": 33.401573181152344,
      "activations/layer16_attention_weight_min": -29.99216079711914,
      "activations/layer17_attention_weight_max": 52.86561965942383,
      "activations/layer17_attention_weight_min": -44.84181213378906,
      "activations/layer18_attention_weight_max": 50.507659912109375,
      "activations/layer18_attention_weight_min": -37.857421875,
      "activations/layer19_attention_weight_max": 24.09206199645996,
      "activations/layer19_attention_weight_min": -22.318479537963867,
      "activations/layer1_attention_weight_max": 16.933345794677734,
      "activations/layer1_attention_weight_min": -15.933972358703613,
      "activations/layer20_attention_weight_max": 23.9101619720459,
      "activations/layer20_attention_weight_min": -19.596281051635742,
      "activations/layer21_attention_weight_max": 40.75920104980469,
      "activations/layer21_attention_weight_min": -28.27374267578125,
      "activations/layer22_attention_weight_max": 32.32698059082031,
      "activations/layer22_attention_weight_min": -23.753742218017578,
      "activations/layer23_attention_weight_max": 36.51054763793945,
      "activations/layer23_attention_weight_min": -25.97539520263672,
      "activations/layer2_attention_weight_max": 31.56080436706543,
      "activations/layer2_attention_weight_min": -29.653358459472656,
      "activations/layer3_attention_weight_max": 90.42581939697266,
      "activations/layer3_attention_weight_min": -90.8123550415039,
      "activations/layer4_attention_weight_max": 105.64366912841797,
      "activations/layer4_attention_weight_min": -98.16416931152344,
      "activations/layer5_attention_weight_max": 84.03469848632812,
      "activations/layer5_attention_weight_min": -78.54096221923828,
      "activations/layer6_attention_weight_max": 58.19303512573242,
      "activations/layer6_attention_weight_min": -52.83715057373047,
      "activations/layer7_attention_weight_max": 79.65362548828125,
      "activations/layer7_attention_weight_min": -74.59566497802734,
      "activations/layer8_attention_weight_max": 44.634281158447266,
      "activations/layer8_attention_weight_min": -46.63322067260742,
      "activations/layer9_attention_weight_max": 57.943450927734375,
      "activations/layer9_attention_weight_min": -49.665401458740234,
      "epoch": 9.09,
      "learning_rate": 9.230643939393939e-05,
      "loss": 2.7926,
      "step": 156450
    },
    {
      "activations/layer0_attention_weight_max": 15.501251220703125,
      "activations/layer0_attention_weight_min": -14.14033031463623,
      "activations/layer10_attention_weight_max": 34.45350646972656,
      "activations/layer10_attention_weight_min": -34.495643615722656,
      "activations/layer11_attention_weight_max": 30.455734252929688,
      "activations/layer11_attention_weight_min": -30.245262145996094,
      "activations/layer12_attention_weight_max": 17.07938575744629,
      "activations/layer12_attention_weight_min": -25.85901641845703,
      "activations/layer13_attention_weight_max": 34.09249496459961,
      "activations/layer13_attention_weight_min": -29.844823837280273,
      "activations/layer14_attention_weight_max": 33.60776138305664,
      "activations/layer14_attention_weight_min": -31.52768898010254,
      "activations/layer15_attention_weight_max": 31.776432037353516,
      "activations/layer15_attention_weight_min": -30.07928466796875,
      "activations/layer16_attention_weight_max": 28.023618698120117,
      "activations/layer16_attention_weight_min": -26.669294357299805,
      "activations/layer17_attention_weight_max": 49.88078689575195,
      "activations/layer17_attention_weight_min": -40.46227264404297,
      "activations/layer18_attention_weight_max": 41.70854949951172,
      "activations/layer18_attention_weight_min": -34.72163772583008,
      "activations/layer19_attention_weight_max": 19.88698387145996,
      "activations/layer19_attention_weight_min": -19.703344345092773,
      "activations/layer1_attention_weight_max": 17.72208595275879,
      "activations/layer1_attention_weight_min": -14.393575668334961,
      "activations/layer20_attention_weight_max": 18.96714973449707,
      "activations/layer20_attention_weight_min": -22.75481414794922,
      "activations/layer21_attention_weight_max": 34.248775482177734,
      "activations/layer21_attention_weight_min": -23.067138671875,
      "activations/layer22_attention_weight_max": 31.645925521850586,
      "activations/layer22_attention_weight_min": -23.86622428894043,
      "activations/layer23_attention_weight_max": 31.629301071166992,
      "activations/layer23_attention_weight_min": -22.941356658935547,
      "activations/layer2_attention_weight_max": 31.495811462402344,
      "activations/layer2_attention_weight_min": -30.284011840820312,
      "activations/layer3_attention_weight_max": 87.35716247558594,
      "activations/layer3_attention_weight_min": -91.40985107421875,
      "activations/layer4_attention_weight_max": 96.10618591308594,
      "activations/layer4_attention_weight_min": -90.64167022705078,
      "activations/layer5_attention_weight_max": 78.56598663330078,
      "activations/layer5_attention_weight_min": -76.41746520996094,
      "activations/layer6_attention_weight_max": 51.48306655883789,
      "activations/layer6_attention_weight_min": -50.1568717956543,
      "activations/layer7_attention_weight_max": 76.16485595703125,
      "activations/layer7_attention_weight_min": -71.46654510498047,
      "activations/layer8_attention_weight_max": 44.149288177490234,
      "activations/layer8_attention_weight_min": -43.861244201660156,
      "activations/layer9_attention_weight_max": 44.35192108154297,
      "activations/layer9_attention_weight_min": -47.88622283935547,
      "epoch": 9.09,
      "learning_rate": 9.228749999999999e-05,
      "loss": 2.8102,
      "step": 156500
    },
    {
      "activations/layer0_attention_weight_max": 15.132716178894043,
      "activations/layer0_attention_weight_min": -14.211483001708984,
      "activations/layer10_attention_weight_max": 36.015968322753906,
      "activations/layer10_attention_weight_min": -34.56174087524414,
      "activations/layer11_attention_weight_max": 30.84786033630371,
      "activations/layer11_attention_weight_min": -33.00861358642578,
      "activations/layer12_attention_weight_max": 17.114534378051758,
      "activations/layer12_attention_weight_min": -28.81036949157715,
      "activations/layer13_attention_weight_max": 37.31676483154297,
      "activations/layer13_attention_weight_min": -30.147754669189453,
      "activations/layer14_attention_weight_max": 34.90243911743164,
      "activations/layer14_attention_weight_min": -29.75611686706543,
      "activations/layer15_attention_weight_max": 30.191007614135742,
      "activations/layer15_attention_weight_min": -27.693038940429688,
      "activations/layer16_attention_weight_max": 29.421119689941406,
      "activations/layer16_attention_weight_min": -30.05567169189453,
      "activations/layer17_attention_weight_max": 49.58889389038086,
      "activations/layer17_attention_weight_min": -42.889591217041016,
      "activations/layer18_attention_weight_max": 45.12085723876953,
      "activations/layer18_attention_weight_min": -36.22862243652344,
      "activations/layer19_attention_weight_max": 20.742408752441406,
      "activations/layer19_attention_weight_min": -22.48801040649414,
      "activations/layer1_attention_weight_max": 16.702123641967773,
      "activations/layer1_attention_weight_min": -15.942747116088867,
      "activations/layer20_attention_weight_max": 20.891813278198242,
      "activations/layer20_attention_weight_min": -21.3963680267334,
      "activations/layer21_attention_weight_max": 36.719520568847656,
      "activations/layer21_attention_weight_min": -22.437833786010742,
      "activations/layer22_attention_weight_max": 28.96405792236328,
      "activations/layer22_attention_weight_min": -25.09192657470703,
      "activations/layer23_attention_weight_max": 33.13465881347656,
      "activations/layer23_attention_weight_min": -22.077590942382812,
      "activations/layer2_attention_weight_max": 33.05493927001953,
      "activations/layer2_attention_weight_min": -31.164274215698242,
      "activations/layer3_attention_weight_max": 87.66910552978516,
      "activations/layer3_attention_weight_min": -88.9749526977539,
      "activations/layer4_attention_weight_max": 96.54570770263672,
      "activations/layer4_attention_weight_min": -93.37041473388672,
      "activations/layer5_attention_weight_max": 82.75395202636719,
      "activations/layer5_attention_weight_min": -78.5760498046875,
      "activations/layer6_attention_weight_max": 52.870662689208984,
      "activations/layer6_attention_weight_min": -53.357826232910156,
      "activations/layer7_attention_weight_max": 77.30325317382812,
      "activations/layer7_attention_weight_min": -71.9228286743164,
      "activations/layer8_attention_weight_max": 45.91020202636719,
      "activations/layer8_attention_weight_min": -47.92802810668945,
      "activations/layer9_attention_weight_max": 51.60206985473633,
      "activations/layer9_attention_weight_min": -48.60441970825195,
      "epoch": 9.1,
      "learning_rate": 9.22685606060606e-05,
      "loss": 2.8109,
      "step": 156550
    },
    {
      "activations/layer0_attention_weight_max": 15.883024215698242,
      "activations/layer0_attention_weight_min": -13.610082626342773,
      "activations/layer10_attention_weight_max": 39.171756744384766,
      "activations/layer10_attention_weight_min": -37.06519317626953,
      "activations/layer11_attention_weight_max": 31.597064971923828,
      "activations/layer11_attention_weight_min": -31.344526290893555,
      "activations/layer12_attention_weight_max": 20.465381622314453,
      "activations/layer12_attention_weight_min": -23.189470291137695,
      "activations/layer13_attention_weight_max": 39.41798782348633,
      "activations/layer13_attention_weight_min": -29.77971076965332,
      "activations/layer14_attention_weight_max": 41.29509353637695,
      "activations/layer14_attention_weight_min": -31.592937469482422,
      "activations/layer15_attention_weight_max": 35.40182876586914,
      "activations/layer15_attention_weight_min": -30.419727325439453,
      "activations/layer16_attention_weight_max": 32.643280029296875,
      "activations/layer16_attention_weight_min": -27.755292892456055,
      "activations/layer17_attention_weight_max": 54.77547073364258,
      "activations/layer17_attention_weight_min": -41.05584716796875,
      "activations/layer18_attention_weight_max": 46.028175354003906,
      "activations/layer18_attention_weight_min": -35.366634368896484,
      "activations/layer19_attention_weight_max": 22.091495513916016,
      "activations/layer19_attention_weight_min": -21.621932983398438,
      "activations/layer1_attention_weight_max": 17.626575469970703,
      "activations/layer1_attention_weight_min": -15.15764331817627,
      "activations/layer20_attention_weight_max": 20.902721405029297,
      "activations/layer20_attention_weight_min": -20.826297760009766,
      "activations/layer21_attention_weight_max": 36.328704833984375,
      "activations/layer21_attention_weight_min": -22.7891845703125,
      "activations/layer22_attention_weight_max": 33.49473571777344,
      "activations/layer22_attention_weight_min": -25.61298179626465,
      "activations/layer23_attention_weight_max": 35.735511779785156,
      "activations/layer23_attention_weight_min": -23.759445190429688,
      "activations/layer2_attention_weight_max": 32.12224578857422,
      "activations/layer2_attention_weight_min": -29.208660125732422,
      "activations/layer3_attention_weight_max": 89.40702819824219,
      "activations/layer3_attention_weight_min": -87.54910278320312,
      "activations/layer4_attention_weight_max": 99.18663787841797,
      "activations/layer4_attention_weight_min": -94.11557006835938,
      "activations/layer5_attention_weight_max": 81.05287170410156,
      "activations/layer5_attention_weight_min": -80.16159057617188,
      "activations/layer6_attention_weight_max": 54.97325897216797,
      "activations/layer6_attention_weight_min": -54.314456939697266,
      "activations/layer7_attention_weight_max": 83.10611724853516,
      "activations/layer7_attention_weight_min": -76.37740325927734,
      "activations/layer8_attention_weight_max": 47.990482330322266,
      "activations/layer8_attention_weight_min": -47.100772857666016,
      "activations/layer9_attention_weight_max": 51.993568420410156,
      "activations/layer9_attention_weight_min": -48.758235931396484,
      "epoch": 9.1,
      "learning_rate": 9.224962121212122e-05,
      "loss": 2.7768,
      "step": 156600
    },
    {
      "activations/layer0_attention_weight_max": 15.135905265808105,
      "activations/layer0_attention_weight_min": -13.90602970123291,
      "activations/layer10_attention_weight_max": 36.978614807128906,
      "activations/layer10_attention_weight_min": -36.427520751953125,
      "activations/layer11_attention_weight_max": 31.575138092041016,
      "activations/layer11_attention_weight_min": -32.88948059082031,
      "activations/layer12_attention_weight_max": 18.530786514282227,
      "activations/layer12_attention_weight_min": -25.813804626464844,
      "activations/layer13_attention_weight_max": 47.11356735229492,
      "activations/layer13_attention_weight_min": -30.252521514892578,
      "activations/layer14_attention_weight_max": 48.19183349609375,
      "activations/layer14_attention_weight_min": -31.326141357421875,
      "activations/layer15_attention_weight_max": 38.395381927490234,
      "activations/layer15_attention_weight_min": -32.50247573852539,
      "activations/layer16_attention_weight_max": 32.011539459228516,
      "activations/layer16_attention_weight_min": -29.169889450073242,
      "activations/layer17_attention_weight_max": 56.36882019042969,
      "activations/layer17_attention_weight_min": -47.69187927246094,
      "activations/layer18_attention_weight_max": 47.04894256591797,
      "activations/layer18_attention_weight_min": -39.72919845581055,
      "activations/layer19_attention_weight_max": 21.080421447753906,
      "activations/layer19_attention_weight_min": -21.44089698791504,
      "activations/layer1_attention_weight_max": 16.801755905151367,
      "activations/layer1_attention_weight_min": -14.653546333312988,
      "activations/layer20_attention_weight_max": 22.670555114746094,
      "activations/layer20_attention_weight_min": -21.820144653320312,
      "activations/layer21_attention_weight_max": 42.30998611450195,
      "activations/layer21_attention_weight_min": -24.007444381713867,
      "activations/layer22_attention_weight_max": 31.654706954956055,
      "activations/layer22_attention_weight_min": -25.60386085510254,
      "activations/layer23_attention_weight_max": 35.84616470336914,
      "activations/layer23_attention_weight_min": -22.78112030029297,
      "activations/layer2_attention_weight_max": 30.16290855407715,
      "activations/layer2_attention_weight_min": -28.61572265625,
      "activations/layer3_attention_weight_max": 86.10307312011719,
      "activations/layer3_attention_weight_min": -84.55467987060547,
      "activations/layer4_attention_weight_max": 99.92819213867188,
      "activations/layer4_attention_weight_min": -93.519287109375,
      "activations/layer5_attention_weight_max": 81.77725219726562,
      "activations/layer5_attention_weight_min": -78.052490234375,
      "activations/layer6_attention_weight_max": 56.146766662597656,
      "activations/layer6_attention_weight_min": -54.99604797363281,
      "activations/layer7_attention_weight_max": 86.2687759399414,
      "activations/layer7_attention_weight_min": -76.68325805664062,
      "activations/layer8_attention_weight_max": 43.86079025268555,
      "activations/layer8_attention_weight_min": -45.006717681884766,
      "activations/layer9_attention_weight_max": 48.8852653503418,
      "activations/layer9_attention_weight_min": -49.80678939819336,
      "epoch": 9.1,
      "learning_rate": 9.22306818181818e-05,
      "loss": 2.7876,
      "step": 156650
    },
    {
      "activations/layer0_attention_weight_max": 16.0414981842041,
      "activations/layer0_attention_weight_min": -14.072629928588867,
      "activations/layer10_attention_weight_max": 39.63762283325195,
      "activations/layer10_attention_weight_min": -39.052162170410156,
      "activations/layer11_attention_weight_max": 35.869667053222656,
      "activations/layer11_attention_weight_min": -35.71318817138672,
      "activations/layer12_attention_weight_max": 18.887163162231445,
      "activations/layer12_attention_weight_min": -24.94992446899414,
      "activations/layer13_attention_weight_max": 39.885128021240234,
      "activations/layer13_attention_weight_min": -28.005355834960938,
      "activations/layer14_attention_weight_max": 46.3168830871582,
      "activations/layer14_attention_weight_min": -30.58231544494629,
      "activations/layer15_attention_weight_max": 37.517425537109375,
      "activations/layer15_attention_weight_min": -29.30289649963379,
      "activations/layer16_attention_weight_max": 36.21100997924805,
      "activations/layer16_attention_weight_min": -28.352052688598633,
      "activations/layer17_attention_weight_max": 59.33686828613281,
      "activations/layer17_attention_weight_min": -46.29228591918945,
      "activations/layer18_attention_weight_max": 48.677886962890625,
      "activations/layer18_attention_weight_min": -41.240509033203125,
      "activations/layer19_attention_weight_max": 21.524728775024414,
      "activations/layer19_attention_weight_min": -21.218870162963867,
      "activations/layer1_attention_weight_max": 18.348796844482422,
      "activations/layer1_attention_weight_min": -14.634936332702637,
      "activations/layer20_attention_weight_max": 22.686140060424805,
      "activations/layer20_attention_weight_min": -20.175031661987305,
      "activations/layer21_attention_weight_max": 39.48423767089844,
      "activations/layer21_attention_weight_min": -23.181360244750977,
      "activations/layer22_attention_weight_max": 34.19451141357422,
      "activations/layer22_attention_weight_min": -26.823593139648438,
      "activations/layer23_attention_weight_max": 38.17381286621094,
      "activations/layer23_attention_weight_min": -21.776447296142578,
      "activations/layer2_attention_weight_max": 32.514076232910156,
      "activations/layer2_attention_weight_min": -30.80849266052246,
      "activations/layer3_attention_weight_max": 95.32189178466797,
      "activations/layer3_attention_weight_min": -96.48814392089844,
      "activations/layer4_attention_weight_max": 109.28063201904297,
      "activations/layer4_attention_weight_min": -101.5845718383789,
      "activations/layer5_attention_weight_max": 88.48649597167969,
      "activations/layer5_attention_weight_min": -82.5836410522461,
      "activations/layer6_attention_weight_max": 60.737403869628906,
      "activations/layer6_attention_weight_min": -59.05585479736328,
      "activations/layer7_attention_weight_max": 82.72765350341797,
      "activations/layer7_attention_weight_min": -81.00020599365234,
      "activations/layer8_attention_weight_max": 49.58772659301758,
      "activations/layer8_attention_weight_min": -53.7062873840332,
      "activations/layer9_attention_weight_max": 59.529964447021484,
      "activations/layer9_attention_weight_min": -52.98299026489258,
      "epoch": 9.11,
      "learning_rate": 9.221174242424242e-05,
      "loss": 2.8061,
      "step": 156700
    },
    {
      "activations/layer0_attention_weight_max": 14.375667572021484,
      "activations/layer0_attention_weight_min": -13.902454376220703,
      "activations/layer10_attention_weight_max": 39.80553436279297,
      "activations/layer10_attention_weight_min": -37.58943176269531,
      "activations/layer11_attention_weight_max": 36.361289978027344,
      "activations/layer11_attention_weight_min": -34.858070373535156,
      "activations/layer12_attention_weight_max": 18.807554244995117,
      "activations/layer12_attention_weight_min": -31.690929412841797,
      "activations/layer13_attention_weight_max": 54.7507438659668,
      "activations/layer13_attention_weight_min": -39.57719421386719,
      "activations/layer14_attention_weight_max": 66.36852264404297,
      "activations/layer14_attention_weight_min": -33.04262924194336,
      "activations/layer15_attention_weight_max": 53.42036437988281,
      "activations/layer15_attention_weight_min": -31.31252670288086,
      "activations/layer16_attention_weight_max": 32.89609146118164,
      "activations/layer16_attention_weight_min": -30.05126190185547,
      "activations/layer17_attention_weight_max": 59.31452178955078,
      "activations/layer17_attention_weight_min": -51.80266189575195,
      "activations/layer18_attention_weight_max": 49.025936126708984,
      "activations/layer18_attention_weight_min": -43.60798645019531,
      "activations/layer19_attention_weight_max": 24.318029403686523,
      "activations/layer19_attention_weight_min": -23.017105102539062,
      "activations/layer1_attention_weight_max": 17.367263793945312,
      "activations/layer1_attention_weight_min": -14.538132667541504,
      "activations/layer20_attention_weight_max": 28.264366149902344,
      "activations/layer20_attention_weight_min": -21.115747451782227,
      "activations/layer21_attention_weight_max": 59.47679901123047,
      "activations/layer21_attention_weight_min": -24.36974334716797,
      "activations/layer22_attention_weight_max": 37.80268859863281,
      "activations/layer22_attention_weight_min": -25.744253158569336,
      "activations/layer23_attention_weight_max": 36.35248947143555,
      "activations/layer23_attention_weight_min": -25.634578704833984,
      "activations/layer2_attention_weight_max": 32.20954895019531,
      "activations/layer2_attention_weight_min": -31.606184005737305,
      "activations/layer3_attention_weight_max": 88.35897064208984,
      "activations/layer3_attention_weight_min": -90.5076904296875,
      "activations/layer4_attention_weight_max": 100.38162231445312,
      "activations/layer4_attention_weight_min": -93.1043930053711,
      "activations/layer5_attention_weight_max": 80.26468658447266,
      "activations/layer5_attention_weight_min": -77.49427795410156,
      "activations/layer6_attention_weight_max": 54.73197937011719,
      "activations/layer6_attention_weight_min": -51.971107482910156,
      "activations/layer7_attention_weight_max": 82.1189956665039,
      "activations/layer7_attention_weight_min": -75.79483032226562,
      "activations/layer8_attention_weight_max": 45.2000846862793,
      "activations/layer8_attention_weight_min": -49.26521682739258,
      "activations/layer9_attention_weight_max": 65.01298522949219,
      "activations/layer9_attention_weight_min": -56.07145309448242,
      "epoch": 9.11,
      "learning_rate": 9.219280303030301e-05,
      "loss": 2.7953,
      "step": 156750
    },
    {
      "activations/layer0_attention_weight_max": 14.668211936950684,
      "activations/layer0_attention_weight_min": -13.597847938537598,
      "activations/layer10_attention_weight_max": 36.91195297241211,
      "activations/layer10_attention_weight_min": -36.52857971191406,
      "activations/layer11_attention_weight_max": 32.148292541503906,
      "activations/layer11_attention_weight_min": -31.028684616088867,
      "activations/layer12_attention_weight_max": 20.447357177734375,
      "activations/layer12_attention_weight_min": -26.605817794799805,
      "activations/layer13_attention_weight_max": 50.43588638305664,
      "activations/layer13_attention_weight_min": -31.407695770263672,
      "activations/layer14_attention_weight_max": 42.96559524536133,
      "activations/layer14_attention_weight_min": -30.5513916015625,
      "activations/layer15_attention_weight_max": 38.46916961669922,
      "activations/layer15_attention_weight_min": -30.989023208618164,
      "activations/layer16_attention_weight_max": 37.41619110107422,
      "activations/layer16_attention_weight_min": -28.52946662902832,
      "activations/layer17_attention_weight_max": 62.15676498413086,
      "activations/layer17_attention_weight_min": -46.62862014770508,
      "activations/layer18_attention_weight_max": 53.73276138305664,
      "activations/layer18_attention_weight_min": -39.299224853515625,
      "activations/layer19_attention_weight_max": 27.10530662536621,
      "activations/layer19_attention_weight_min": -21.823156356811523,
      "activations/layer1_attention_weight_max": 16.212902069091797,
      "activations/layer1_attention_weight_min": -15.05593490600586,
      "activations/layer20_attention_weight_max": 27.18367576599121,
      "activations/layer20_attention_weight_min": -26.008220672607422,
      "activations/layer21_attention_weight_max": 52.81085205078125,
      "activations/layer21_attention_weight_min": -25.911361694335938,
      "activations/layer22_attention_weight_max": 39.003997802734375,
      "activations/layer22_attention_weight_min": -25.410236358642578,
      "activations/layer23_attention_weight_max": 49.261287689208984,
      "activations/layer23_attention_weight_min": -26.795743942260742,
      "activations/layer2_attention_weight_max": 32.017303466796875,
      "activations/layer2_attention_weight_min": -29.153980255126953,
      "activations/layer3_attention_weight_max": 91.43592834472656,
      "activations/layer3_attention_weight_min": -93.03960418701172,
      "activations/layer4_attention_weight_max": 100.15699005126953,
      "activations/layer4_attention_weight_min": -96.53155517578125,
      "activations/layer5_attention_weight_max": 83.8602523803711,
      "activations/layer5_attention_weight_min": -77.82701110839844,
      "activations/layer6_attention_weight_max": 55.34252166748047,
      "activations/layer6_attention_weight_min": -53.49937438964844,
      "activations/layer7_attention_weight_max": 77.12445831298828,
      "activations/layer7_attention_weight_min": -73.74156951904297,
      "activations/layer8_attention_weight_max": 42.716796875,
      "activations/layer8_attention_weight_min": -44.95372009277344,
      "activations/layer9_attention_weight_max": 49.60943603515625,
      "activations/layer9_attention_weight_min": -48.80526351928711,
      "epoch": 9.11,
      "learning_rate": 9.217386363636362e-05,
      "loss": 2.7957,
      "step": 156800
    },
    {
      "activations/layer0_attention_weight_max": 15.658438682556152,
      "activations/layer0_attention_weight_min": -13.842083930969238,
      "activations/layer10_attention_weight_max": 39.40299606323242,
      "activations/layer10_attention_weight_min": -36.255035400390625,
      "activations/layer11_attention_weight_max": 37.447940826416016,
      "activations/layer11_attention_weight_min": -31.112356185913086,
      "activations/layer12_attention_weight_max": 19.292688369750977,
      "activations/layer12_attention_weight_min": -25.299354553222656,
      "activations/layer13_attention_weight_max": 38.67426681518555,
      "activations/layer13_attention_weight_min": -29.738658905029297,
      "activations/layer14_attention_weight_max": 37.561771392822266,
      "activations/layer14_attention_weight_min": -31.300514221191406,
      "activations/layer15_attention_weight_max": 37.11547088623047,
      "activations/layer15_attention_weight_min": -31.193527221679688,
      "activations/layer16_attention_weight_max": 30.052541732788086,
      "activations/layer16_attention_weight_min": -27.680788040161133,
      "activations/layer17_attention_weight_max": 54.8078498840332,
      "activations/layer17_attention_weight_min": -47.95725631713867,
      "activations/layer18_attention_weight_max": 50.05195236206055,
      "activations/layer18_attention_weight_min": -42.703548431396484,
      "activations/layer19_attention_weight_max": 20.235929489135742,
      "activations/layer19_attention_weight_min": -24.127967834472656,
      "activations/layer1_attention_weight_max": 16.409948348999023,
      "activations/layer1_attention_weight_min": -15.957201957702637,
      "activations/layer20_attention_weight_max": 22.728729248046875,
      "activations/layer20_attention_weight_min": -22.641769409179688,
      "activations/layer21_attention_weight_max": 39.76531982421875,
      "activations/layer21_attention_weight_min": -21.652318954467773,
      "activations/layer22_attention_weight_max": 29.71381378173828,
      "activations/layer22_attention_weight_min": -26.510164260864258,
      "activations/layer23_attention_weight_max": 37.066627502441406,
      "activations/layer23_attention_weight_min": -23.340797424316406,
      "activations/layer2_attention_weight_max": 32.5719108581543,
      "activations/layer2_attention_weight_min": -28.123130798339844,
      "activations/layer3_attention_weight_max": 89.22201538085938,
      "activations/layer3_attention_weight_min": -87.19290924072266,
      "activations/layer4_attention_weight_max": 97.04847717285156,
      "activations/layer4_attention_weight_min": -94.62776947021484,
      "activations/layer5_attention_weight_max": 76.35826110839844,
      "activations/layer5_attention_weight_min": -78.61060333251953,
      "activations/layer6_attention_weight_max": 50.245887756347656,
      "activations/layer6_attention_weight_min": -51.4793586730957,
      "activations/layer7_attention_weight_max": 80.83812713623047,
      "activations/layer7_attention_weight_min": -72.86282348632812,
      "activations/layer8_attention_weight_max": 48.82492446899414,
      "activations/layer8_attention_weight_min": -44.15278625488281,
      "activations/layer9_attention_weight_max": 62.10004806518555,
      "activations/layer9_attention_weight_min": -50.278526306152344,
      "epoch": 9.11,
      "learning_rate": 9.215492424242424e-05,
      "loss": 2.7952,
      "step": 156850
    },
    {
      "activations/layer0_attention_weight_max": 14.904682159423828,
      "activations/layer0_attention_weight_min": -13.93222713470459,
      "activations/layer10_attention_weight_max": 34.710609436035156,
      "activations/layer10_attention_weight_min": -33.70319366455078,
      "activations/layer11_attention_weight_max": 30.47376251220703,
      "activations/layer11_attention_weight_min": -31.60445785522461,
      "activations/layer12_attention_weight_max": 19.834218978881836,
      "activations/layer12_attention_weight_min": -22.619583129882812,
      "activations/layer13_attention_weight_max": 43.07157516479492,
      "activations/layer13_attention_weight_min": -34.30180358886719,
      "activations/layer14_attention_weight_max": 37.74345397949219,
      "activations/layer14_attention_weight_min": -31.559879302978516,
      "activations/layer15_attention_weight_max": 35.39567565917969,
      "activations/layer15_attention_weight_min": -32.579734802246094,
      "activations/layer16_attention_weight_max": 32.93729019165039,
      "activations/layer16_attention_weight_min": -30.255428314208984,
      "activations/layer17_attention_weight_max": 54.38243865966797,
      "activations/layer17_attention_weight_min": -43.993568420410156,
      "activations/layer18_attention_weight_max": 45.65428924560547,
      "activations/layer18_attention_weight_min": -37.50571060180664,
      "activations/layer19_attention_weight_max": 21.550949096679688,
      "activations/layer19_attention_weight_min": -21.444612503051758,
      "activations/layer1_attention_weight_max": 17.90134620666504,
      "activations/layer1_attention_weight_min": -15.158340454101562,
      "activations/layer20_attention_weight_max": 22.614463806152344,
      "activations/layer20_attention_weight_min": -21.96373748779297,
      "activations/layer21_attention_weight_max": 36.61508560180664,
      "activations/layer21_attention_weight_min": -23.845775604248047,
      "activations/layer22_attention_weight_max": 32.88214111328125,
      "activations/layer22_attention_weight_min": -26.095266342163086,
      "activations/layer23_attention_weight_max": 38.47950744628906,
      "activations/layer23_attention_weight_min": -24.659568786621094,
      "activations/layer2_attention_weight_max": 30.805387496948242,
      "activations/layer2_attention_weight_min": -28.52449607849121,
      "activations/layer3_attention_weight_max": 84.73865509033203,
      "activations/layer3_attention_weight_min": -83.24957275390625,
      "activations/layer4_attention_weight_max": 94.9674301147461,
      "activations/layer4_attention_weight_min": -89.0012435913086,
      "activations/layer5_attention_weight_max": 79.80130767822266,
      "activations/layer5_attention_weight_min": -76.08187866210938,
      "activations/layer6_attention_weight_max": 57.92119216918945,
      "activations/layer6_attention_weight_min": -51.579444885253906,
      "activations/layer7_attention_weight_max": 77.69371032714844,
      "activations/layer7_attention_weight_min": -71.67213439941406,
      "activations/layer8_attention_weight_max": 45.62559509277344,
      "activations/layer8_attention_weight_min": -49.0872802734375,
      "activations/layer9_attention_weight_max": 53.67289352416992,
      "activations/layer9_attention_weight_min": -47.982086181640625,
      "epoch": 9.12,
      "learning_rate": 9.213598484848484e-05,
      "loss": 2.7911,
      "step": 156900
    },
    {
      "activations/layer0_attention_weight_max": 14.368230819702148,
      "activations/layer0_attention_weight_min": -13.818581581115723,
      "activations/layer10_attention_weight_max": 42.6568603515625,
      "activations/layer10_attention_weight_min": -37.9897575378418,
      "activations/layer11_attention_weight_max": 32.97468185424805,
      "activations/layer11_attention_weight_min": -33.02790069580078,
      "activations/layer12_attention_weight_max": 20.1915340423584,
      "activations/layer12_attention_weight_min": -25.60479164123535,
      "activations/layer13_attention_weight_max": 47.02586364746094,
      "activations/layer13_attention_weight_min": -33.683162689208984,
      "activations/layer14_attention_weight_max": 37.913150787353516,
      "activations/layer14_attention_weight_min": -30.96876335144043,
      "activations/layer15_attention_weight_max": 35.586036682128906,
      "activations/layer15_attention_weight_min": -29.7359676361084,
      "activations/layer16_attention_weight_max": 33.773902893066406,
      "activations/layer16_attention_weight_min": -29.29924774169922,
      "activations/layer17_attention_weight_max": 53.49414825439453,
      "activations/layer17_attention_weight_min": -42.32756805419922,
      "activations/layer18_attention_weight_max": 49.865421295166016,
      "activations/layer18_attention_weight_min": -38.015933990478516,
      "activations/layer19_attention_weight_max": 23.984100341796875,
      "activations/layer19_attention_weight_min": -23.355131149291992,
      "activations/layer1_attention_weight_max": 16.694448471069336,
      "activations/layer1_attention_weight_min": -16.470922470092773,
      "activations/layer20_attention_weight_max": 23.224109649658203,
      "activations/layer20_attention_weight_min": -22.105854034423828,
      "activations/layer21_attention_weight_max": 43.98600387573242,
      "activations/layer21_attention_weight_min": -22.39107894897461,
      "activations/layer22_attention_weight_max": 31.37464714050293,
      "activations/layer22_attention_weight_min": -24.65886116027832,
      "activations/layer23_attention_weight_max": 37.397377014160156,
      "activations/layer23_attention_weight_min": -24.111793518066406,
      "activations/layer2_attention_weight_max": 30.852407455444336,
      "activations/layer2_attention_weight_min": -29.838939666748047,
      "activations/layer3_attention_weight_max": 86.89686584472656,
      "activations/layer3_attention_weight_min": -90.46647644042969,
      "activations/layer4_attention_weight_max": 99.79134368896484,
      "activations/layer4_attention_weight_min": -97.7954330444336,
      "activations/layer5_attention_weight_max": 82.58370971679688,
      "activations/layer5_attention_weight_min": -82.32134246826172,
      "activations/layer6_attention_weight_max": 57.920291900634766,
      "activations/layer6_attention_weight_min": -57.37167739868164,
      "activations/layer7_attention_weight_max": 78.8144760131836,
      "activations/layer7_attention_weight_min": -76.729248046875,
      "activations/layer8_attention_weight_max": 49.738922119140625,
      "activations/layer8_attention_weight_min": -51.44883728027344,
      "activations/layer9_attention_weight_max": 51.89459991455078,
      "activations/layer9_attention_weight_min": -51.9386100769043,
      "epoch": 9.12,
      "learning_rate": 9.211704545454544e-05,
      "loss": 2.7981,
      "step": 156950
    },
    {
      "activations/layer0_attention_weight_max": 15.494095802307129,
      "activations/layer0_attention_weight_min": -14.467245101928711,
      "activations/layer10_attention_weight_max": 37.39772033691406,
      "activations/layer10_attention_weight_min": -34.80371856689453,
      "activations/layer11_attention_weight_max": 34.51030731201172,
      "activations/layer11_attention_weight_min": -33.65616989135742,
      "activations/layer12_attention_weight_max": 19.44233512878418,
      "activations/layer12_attention_weight_min": -25.93789291381836,
      "activations/layer13_attention_weight_max": 36.59719467163086,
      "activations/layer13_attention_weight_min": -27.352563858032227,
      "activations/layer14_attention_weight_max": 38.19691848754883,
      "activations/layer14_attention_weight_min": -29.933088302612305,
      "activations/layer15_attention_weight_max": 34.45964813232422,
      "activations/layer15_attention_weight_min": -29.962646484375,
      "activations/layer16_attention_weight_max": 28.88231086730957,
      "activations/layer16_attention_weight_min": -28.75956916809082,
      "activations/layer17_attention_weight_max": 59.01849365234375,
      "activations/layer17_attention_weight_min": -44.61664581298828,
      "activations/layer18_attention_weight_max": 46.97560119628906,
      "activations/layer18_attention_weight_min": -39.721954345703125,
      "activations/layer19_attention_weight_max": 21.619932174682617,
      "activations/layer19_attention_weight_min": -22.43556022644043,
      "activations/layer1_attention_weight_max": 16.105419158935547,
      "activations/layer1_attention_weight_min": -15.950101852416992,
      "activations/layer20_attention_weight_max": 21.834487915039062,
      "activations/layer20_attention_weight_min": -22.665653228759766,
      "activations/layer21_attention_weight_max": 34.694190979003906,
      "activations/layer21_attention_weight_min": -24.010704040527344,
      "activations/layer22_attention_weight_max": 29.955190658569336,
      "activations/layer22_attention_weight_min": -28.56827163696289,
      "activations/layer23_attention_weight_max": 36.521568298339844,
      "activations/layer23_attention_weight_min": -23.40278434753418,
      "activations/layer2_attention_weight_max": 30.671920776367188,
      "activations/layer2_attention_weight_min": -28.93427276611328,
      "activations/layer3_attention_weight_max": 86.26854705810547,
      "activations/layer3_attention_weight_min": -88.33374786376953,
      "activations/layer4_attention_weight_max": 97.48148345947266,
      "activations/layer4_attention_weight_min": -93.27806854248047,
      "activations/layer5_attention_weight_max": 79.06818389892578,
      "activations/layer5_attention_weight_min": -76.26394653320312,
      "activations/layer6_attention_weight_max": 53.68153381347656,
      "activations/layer6_attention_weight_min": -52.37335968017578,
      "activations/layer7_attention_weight_max": 79.4395751953125,
      "activations/layer7_attention_weight_min": -74.27719116210938,
      "activations/layer8_attention_weight_max": 44.54376983642578,
      "activations/layer8_attention_weight_min": -49.11057662963867,
      "activations/layer9_attention_weight_max": 48.53321075439453,
      "activations/layer9_attention_weight_min": -47.365745544433594,
      "epoch": 9.12,
      "learning_rate": 9.209810606060606e-05,
      "loss": 2.8073,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_loss": 2.751953125,
      "eval_runtime": 8.5237,
      "eval_samples_per_second": 503.77,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_openwebtext_loss": 2.751953125,
      "eval_openwebtext_ppl": 15.673213754973451,
      "eval_openwebtext_runtime": 8.5237,
      "eval_openwebtext_samples_per_second": 503.77,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 2.0038,
      "eval_wikitext_samples_per_second": 227.571,
      "step": 157000
    },
    {
      "epoch": 9.12,
      "eval_lambada_loss": 2.685546875,
      "eval_lambada_ppl": 14.666219776472907,
      "eval_lambada_runtime": 9.5731,
      "eval_lambada_samples_per_second": 508.615,
      "step": 157000
    },
    {
      "activations/layer0_attention_weight_max": 15.685750961303711,
      "activations/layer0_attention_weight_min": -14.374491691589355,
      "activations/layer10_attention_weight_max": 42.7769889831543,
      "activations/layer10_attention_weight_min": -38.542415618896484,
      "activations/layer11_attention_weight_max": 36.15226745605469,
      "activations/layer11_attention_weight_min": -34.65938949584961,
      "activations/layer12_attention_weight_max": 20.469236373901367,
      "activations/layer12_attention_weight_min": -27.115169525146484,
      "activations/layer13_attention_weight_max": 49.98324203491211,
      "activations/layer13_attention_weight_min": -32.78329849243164,
      "activations/layer14_attention_weight_max": 44.68192672729492,
      "activations/layer14_attention_weight_min": -30.825281143188477,
      "activations/layer15_attention_weight_max": 39.82754135131836,
      "activations/layer15_attention_weight_min": -28.93763542175293,
      "activations/layer16_attention_weight_max": 35.208553314208984,
      "activations/layer16_attention_weight_min": -29.0577449798584,
      "activations/layer17_attention_weight_max": 62.1574592590332,
      "activations/layer17_attention_weight_min": -44.9879150390625,
      "activations/layer18_attention_weight_max": 54.82986831665039,
      "activations/layer18_attention_weight_min": -39.6714973449707,
      "activations/layer19_attention_weight_max": 26.86578941345215,
      "activations/layer19_attention_weight_min": -21.3690185546875,
      "activations/layer1_attention_weight_max": 16.523900985717773,
      "activations/layer1_attention_weight_min": -14.718804359436035,
      "activations/layer20_attention_weight_max": 26.370372772216797,
      "activations/layer20_attention_weight_min": -24.907180786132812,
      "activations/layer21_attention_weight_max": 44.016109466552734,
      "activations/layer21_attention_weight_min": -21.650794982910156,
      "activations/layer22_attention_weight_max": 32.6033821105957,
      "activations/layer22_attention_weight_min": -24.28263282775879,
      "activations/layer23_attention_weight_max": 43.83221435546875,
      "activations/layer23_attention_weight_min": -23.553442001342773,
      "activations/layer2_attention_weight_max": 31.585941314697266,
      "activations/layer2_attention_weight_min": -29.732519149780273,
      "activations/layer3_attention_weight_max": 87.28002166748047,
      "activations/layer3_attention_weight_min": -90.32566833496094,
      "activations/layer4_attention_weight_max": 101.95983123779297,
      "activations/layer4_attention_weight_min": -97.95484924316406,
      "activations/layer5_attention_weight_max": 85.75447845458984,
      "activations/layer5_attention_weight_min": -81.24877166748047,
      "activations/layer6_attention_weight_max": 60.40632247924805,
      "activations/layer6_attention_weight_min": -56.301944732666016,
      "activations/layer7_attention_weight_max": 83.4054946899414,
      "activations/layer7_attention_weight_min": -75.45548248291016,
      "activations/layer8_attention_weight_max": 50.733028411865234,
      "activations/layer8_attention_weight_min": -50.70591735839844,
      "activations/layer9_attention_weight_max": 58.04232406616211,
      "activations/layer9_attention_weight_min": -51.614864349365234,
      "epoch": 9.13,
      "learning_rate": 9.207916666666666e-05,
      "loss": 2.8086,
      "step": 157050
    },
    {
      "activations/layer0_attention_weight_max": 16.83711051940918,
      "activations/layer0_attention_weight_min": -14.134169578552246,
      "activations/layer10_attention_weight_max": 36.55438232421875,
      "activations/layer10_attention_weight_min": -37.078060150146484,
      "activations/layer11_attention_weight_max": 30.413753509521484,
      "activations/layer11_attention_weight_min": -31.159103393554688,
      "activations/layer12_attention_weight_max": 19.13951873779297,
      "activations/layer12_attention_weight_min": -29.662460327148438,
      "activations/layer13_attention_weight_max": 37.76921463012695,
      "activations/layer13_attention_weight_min": -31.571908950805664,
      "activations/layer14_attention_weight_max": 37.82730484008789,
      "activations/layer14_attention_weight_min": -30.891704559326172,
      "activations/layer15_attention_weight_max": 34.57534408569336,
      "activations/layer15_attention_weight_min": -31.364648818969727,
      "activations/layer16_attention_weight_max": 32.564796447753906,
      "activations/layer16_attention_weight_min": -30.463838577270508,
      "activations/layer17_attention_weight_max": 56.53695297241211,
      "activations/layer17_attention_weight_min": -47.184967041015625,
      "activations/layer18_attention_weight_max": 47.1865234375,
      "activations/layer18_attention_weight_min": -38.72517013549805,
      "activations/layer19_attention_weight_max": 22.464052200317383,
      "activations/layer19_attention_weight_min": -22.996936798095703,
      "activations/layer1_attention_weight_max": 16.3209285736084,
      "activations/layer1_attention_weight_min": -16.197559356689453,
      "activations/layer20_attention_weight_max": 24.702980041503906,
      "activations/layer20_attention_weight_min": -21.81262969970703,
      "activations/layer21_attention_weight_max": 42.71406173706055,
      "activations/layer21_attention_weight_min": -24.083099365234375,
      "activations/layer22_attention_weight_max": 33.52168655395508,
      "activations/layer22_attention_weight_min": -25.414047241210938,
      "activations/layer23_attention_weight_max": 41.494903564453125,
      "activations/layer23_attention_weight_min": -23.13933563232422,
      "activations/layer2_attention_weight_max": 31.450241088867188,
      "activations/layer2_attention_weight_min": -30.047292709350586,
      "activations/layer3_attention_weight_max": 86.91564178466797,
      "activations/layer3_attention_weight_min": -89.99885559082031,
      "activations/layer4_attention_weight_max": 99.2313003540039,
      "activations/layer4_attention_weight_min": -95.97603607177734,
      "activations/layer5_attention_weight_max": 81.22110748291016,
      "activations/layer5_attention_weight_min": -79.45427703857422,
      "activations/layer6_attention_weight_max": 55.50578308105469,
      "activations/layer6_attention_weight_min": -56.699317932128906,
      "activations/layer7_attention_weight_max": 78.36652374267578,
      "activations/layer7_attention_weight_min": -73.8858871459961,
      "activations/layer8_attention_weight_max": 45.743186950683594,
      "activations/layer8_attention_weight_min": -47.03427505493164,
      "activations/layer9_attention_weight_max": 47.61954879760742,
      "activations/layer9_attention_weight_min": -48.40249252319336,
      "epoch": 9.13,
      "learning_rate": 9.206022727272727e-05,
      "loss": 2.8098,
      "step": 157100
    },
    {
      "activations/layer0_attention_weight_max": 15.511924743652344,
      "activations/layer0_attention_weight_min": -14.011625289916992,
      "activations/layer10_attention_weight_max": 37.66498565673828,
      "activations/layer10_attention_weight_min": -36.276187896728516,
      "activations/layer11_attention_weight_max": 32.442481994628906,
      "activations/layer11_attention_weight_min": -35.573326110839844,
      "activations/layer12_attention_weight_max": 18.64350128173828,
      "activations/layer12_attention_weight_min": -23.691627502441406,
      "activations/layer13_attention_weight_max": 48.59636306762695,
      "activations/layer13_attention_weight_min": -35.141902923583984,
      "activations/layer14_attention_weight_max": 38.959232330322266,
      "activations/layer14_attention_weight_min": -32.67281723022461,
      "activations/layer15_attention_weight_max": 35.036197662353516,
      "activations/layer15_attention_weight_min": -30.868589401245117,
      "activations/layer16_attention_weight_max": 31.49585723876953,
      "activations/layer16_attention_weight_min": -31.37913703918457,
      "activations/layer17_attention_weight_max": 55.714561462402344,
      "activations/layer17_attention_weight_min": -47.95005416870117,
      "activations/layer18_attention_weight_max": 50.971717834472656,
      "activations/layer18_attention_weight_min": -40.9770622253418,
      "activations/layer19_attention_weight_max": 22.94139862060547,
      "activations/layer19_attention_weight_min": -25.278785705566406,
      "activations/layer1_attention_weight_max": 15.662049293518066,
      "activations/layer1_attention_weight_min": -16.12664794921875,
      "activations/layer20_attention_weight_max": 23.611074447631836,
      "activations/layer20_attention_weight_min": -23.02477264404297,
      "activations/layer21_attention_weight_max": 39.88057327270508,
      "activations/layer21_attention_weight_min": -24.70573616027832,
      "activations/layer22_attention_weight_max": 34.208553314208984,
      "activations/layer22_attention_weight_min": -29.852094650268555,
      "activations/layer23_attention_weight_max": 38.53919982910156,
      "activations/layer23_attention_weight_min": -22.51152801513672,
      "activations/layer2_attention_weight_max": 31.207435607910156,
      "activations/layer2_attention_weight_min": -29.46019744873047,
      "activations/layer3_attention_weight_max": 89.34636688232422,
      "activations/layer3_attention_weight_min": -90.46857452392578,
      "activations/layer4_attention_weight_max": 102.00765228271484,
      "activations/layer4_attention_weight_min": -96.3004379272461,
      "activations/layer5_attention_weight_max": 81.84139251708984,
      "activations/layer5_attention_weight_min": -78.46810150146484,
      "activations/layer6_attention_weight_max": 55.95415115356445,
      "activations/layer6_attention_weight_min": -55.7253532409668,
      "activations/layer7_attention_weight_max": 77.71509552001953,
      "activations/layer7_attention_weight_min": -75.37318420410156,
      "activations/layer8_attention_weight_max": 47.0594367980957,
      "activations/layer8_attention_weight_min": -48.30421829223633,
      "activations/layer9_attention_weight_max": 49.378177642822266,
      "activations/layer9_attention_weight_min": -50.177459716796875,
      "epoch": 9.13,
      "learning_rate": 9.204128787878787e-05,
      "loss": 2.7947,
      "step": 157150
    },
    {
      "activations/layer0_attention_weight_max": 14.667534828186035,
      "activations/layer0_attention_weight_min": -13.616412162780762,
      "activations/layer10_attention_weight_max": 41.39193344116211,
      "activations/layer10_attention_weight_min": -36.794551849365234,
      "activations/layer11_attention_weight_max": 33.477962493896484,
      "activations/layer11_attention_weight_min": -32.652984619140625,
      "activations/layer12_attention_weight_max": 17.7390079498291,
      "activations/layer12_attention_weight_min": -24.487934112548828,
      "activations/layer13_attention_weight_max": 36.16522216796875,
      "activations/layer13_attention_weight_min": -28.336856842041016,
      "activations/layer14_attention_weight_max": 35.521392822265625,
      "activations/layer14_attention_weight_min": -29.097810745239258,
      "activations/layer15_attention_weight_max": 32.97209930419922,
      "activations/layer15_attention_weight_min": -29.27834701538086,
      "activations/layer16_attention_weight_max": 29.746084213256836,
      "activations/layer16_attention_weight_min": -28.15458869934082,
      "activations/layer17_attention_weight_max": 51.71711730957031,
      "activations/layer17_attention_weight_min": -44.45040512084961,
      "activations/layer18_attention_weight_max": 45.67688751220703,
      "activations/layer18_attention_weight_min": -34.192108154296875,
      "activations/layer19_attention_weight_max": 21.41182518005371,
      "activations/layer19_attention_weight_min": -21.390613555908203,
      "activations/layer1_attention_weight_max": 17.657957077026367,
      "activations/layer1_attention_weight_min": -14.396336555480957,
      "activations/layer20_attention_weight_max": 21.884122848510742,
      "activations/layer20_attention_weight_min": -24.43492889404297,
      "activations/layer21_attention_weight_max": 33.209163665771484,
      "activations/layer21_attention_weight_min": -22.650850296020508,
      "activations/layer22_attention_weight_max": 30.055837631225586,
      "activations/layer22_attention_weight_min": -26.301362991333008,
      "activations/layer23_attention_weight_max": 32.988624572753906,
      "activations/layer23_attention_weight_min": -23.521957397460938,
      "activations/layer2_attention_weight_max": 33.21760177612305,
      "activations/layer2_attention_weight_min": -29.6788387298584,
      "activations/layer3_attention_weight_max": 95.7197265625,
      "activations/layer3_attention_weight_min": -93.01233673095703,
      "activations/layer4_attention_weight_max": 107.78498840332031,
      "activations/layer4_attention_weight_min": -100.20296478271484,
      "activations/layer5_attention_weight_max": 88.19798278808594,
      "activations/layer5_attention_weight_min": -80.77281951904297,
      "activations/layer6_attention_weight_max": 57.7260856628418,
      "activations/layer6_attention_weight_min": -55.7896842956543,
      "activations/layer7_attention_weight_max": 81.5592269897461,
      "activations/layer7_attention_weight_min": -72.06536865234375,
      "activations/layer8_attention_weight_max": 48.22361755371094,
      "activations/layer8_attention_weight_min": -45.385108947753906,
      "activations/layer9_attention_weight_max": 52.54072189331055,
      "activations/layer9_attention_weight_min": -48.85228729248047,
      "epoch": 9.13,
      "learning_rate": 9.202234848484848e-05,
      "loss": 2.7945,
      "step": 157200
    },
    {
      "activations/layer0_attention_weight_max": 15.710617065429688,
      "activations/layer0_attention_weight_min": -13.948493003845215,
      "activations/layer10_attention_weight_max": 35.942264556884766,
      "activations/layer10_attention_weight_min": -36.28766632080078,
      "activations/layer11_attention_weight_max": 33.666709899902344,
      "activations/layer11_attention_weight_min": -32.5006103515625,
      "activations/layer12_attention_weight_max": 22.046411514282227,
      "activations/layer12_attention_weight_min": -25.585269927978516,
      "activations/layer13_attention_weight_max": 55.8774299621582,
      "activations/layer13_attention_weight_min": -36.05020523071289,
      "activations/layer14_attention_weight_max": 71.13300323486328,
      "activations/layer14_attention_weight_min": -40.0124626159668,
      "activations/layer15_attention_weight_max": 57.13087844848633,
      "activations/layer15_attention_weight_min": -32.32051467895508,
      "activations/layer16_attention_weight_max": 30.343843460083008,
      "activations/layer16_attention_weight_min": -27.486064910888672,
      "activations/layer17_attention_weight_max": 52.70759963989258,
      "activations/layer17_attention_weight_min": -44.29496765136719,
      "activations/layer18_attention_weight_max": 44.516536712646484,
      "activations/layer18_attention_weight_min": -40.84164810180664,
      "activations/layer19_attention_weight_max": 20.78303337097168,
      "activations/layer19_attention_weight_min": -21.652481079101562,
      "activations/layer1_attention_weight_max": 17.236068725585938,
      "activations/layer1_attention_weight_min": -14.290177345275879,
      "activations/layer20_attention_weight_max": 22.421693801879883,
      "activations/layer20_attention_weight_min": -22.245342254638672,
      "activations/layer21_attention_weight_max": 37.92805099487305,
      "activations/layer21_attention_weight_min": -23.727209091186523,
      "activations/layer22_attention_weight_max": 28.88568687438965,
      "activations/layer22_attention_weight_min": -26.356674194335938,
      "activations/layer23_attention_weight_max": 37.445167541503906,
      "activations/layer23_attention_weight_min": -23.58696746826172,
      "activations/layer2_attention_weight_max": 30.464033126831055,
      "activations/layer2_attention_weight_min": -29.17822265625,
      "activations/layer3_attention_weight_max": 90.12041473388672,
      "activations/layer3_attention_weight_min": -96.69666290283203,
      "activations/layer4_attention_weight_max": 100.41813659667969,
      "activations/layer4_attention_weight_min": -98.58247375488281,
      "activations/layer5_attention_weight_max": 81.9366455078125,
      "activations/layer5_attention_weight_min": -79.61839294433594,
      "activations/layer6_attention_weight_max": 53.52175521850586,
      "activations/layer6_attention_weight_min": -56.24308395385742,
      "activations/layer7_attention_weight_max": 77.60505676269531,
      "activations/layer7_attention_weight_min": -73.06190490722656,
      "activations/layer8_attention_weight_max": 45.77525329589844,
      "activations/layer8_attention_weight_min": -50.24037551879883,
      "activations/layer9_attention_weight_max": 54.72457504272461,
      "activations/layer9_attention_weight_min": -50.797569274902344,
      "epoch": 9.14,
      "learning_rate": 9.200340909090909e-05,
      "loss": 2.7979,
      "step": 157250
    },
    {
      "activations/layer0_attention_weight_max": 15.366354942321777,
      "activations/layer0_attention_weight_min": -13.399584770202637,
      "activations/layer10_attention_weight_max": 40.60462188720703,
      "activations/layer10_attention_weight_min": -36.76611328125,
      "activations/layer11_attention_weight_max": 33.488800048828125,
      "activations/layer11_attention_weight_min": -32.78850555419922,
      "activations/layer12_attention_weight_max": 17.883333206176758,
      "activations/layer12_attention_weight_min": -23.427284240722656,
      "activations/layer13_attention_weight_max": 42.10890579223633,
      "activations/layer13_attention_weight_min": -27.935853958129883,
      "activations/layer14_attention_weight_max": 37.08604431152344,
      "activations/layer14_attention_weight_min": -29.321359634399414,
      "activations/layer15_attention_weight_max": 34.11054992675781,
      "activations/layer15_attention_weight_min": -28.696487426757812,
      "activations/layer16_attention_weight_max": 34.3005256652832,
      "activations/layer16_attention_weight_min": -26.777732849121094,
      "activations/layer17_attention_weight_max": 53.02785110473633,
      "activations/layer17_attention_weight_min": -41.49968719482422,
      "activations/layer18_attention_weight_max": 48.40502166748047,
      "activations/layer18_attention_weight_min": -35.15123748779297,
      "activations/layer19_attention_weight_max": 22.76645851135254,
      "activations/layer19_attention_weight_min": -20.405784606933594,
      "activations/layer1_attention_weight_max": 16.182960510253906,
      "activations/layer1_attention_weight_min": -15.762831687927246,
      "activations/layer20_attention_weight_max": 21.59074592590332,
      "activations/layer20_attention_weight_min": -21.002927780151367,
      "activations/layer21_attention_weight_max": 38.1192512512207,
      "activations/layer21_attention_weight_min": -22.616596221923828,
      "activations/layer22_attention_weight_max": 32.24732971191406,
      "activations/layer22_attention_weight_min": -26.02052879333496,
      "activations/layer23_attention_weight_max": 35.06847381591797,
      "activations/layer23_attention_weight_min": -22.223630905151367,
      "activations/layer2_attention_weight_max": 32.2903938293457,
      "activations/layer2_attention_weight_min": -31.99081039428711,
      "activations/layer3_attention_weight_max": 87.20563507080078,
      "activations/layer3_attention_weight_min": -91.79544830322266,
      "activations/layer4_attention_weight_max": 103.13607025146484,
      "activations/layer4_attention_weight_min": -95.11705780029297,
      "activations/layer5_attention_weight_max": 83.45462036132812,
      "activations/layer5_attention_weight_min": -78.5913314819336,
      "activations/layer6_attention_weight_max": 57.83856964111328,
      "activations/layer6_attention_weight_min": -55.01799774169922,
      "activations/layer7_attention_weight_max": 78.20838165283203,
      "activations/layer7_attention_weight_min": -71.91522979736328,
      "activations/layer8_attention_weight_max": 45.46072769165039,
      "activations/layer8_attention_weight_min": -45.25252151489258,
      "activations/layer9_attention_weight_max": 50.50071334838867,
      "activations/layer9_attention_weight_min": -45.57817077636719,
      "epoch": 9.14,
      "learning_rate": 9.198446969696968e-05,
      "loss": 2.7865,
      "step": 157300
    },
    {
      "activations/layer0_attention_weight_max": 14.972227096557617,
      "activations/layer0_attention_weight_min": -13.721230506896973,
      "activations/layer10_attention_weight_max": 36.22340774536133,
      "activations/layer10_attention_weight_min": -36.973976135253906,
      "activations/layer11_attention_weight_max": 30.104658126831055,
      "activations/layer11_attention_weight_min": -31.27589988708496,
      "activations/layer12_attention_weight_max": 17.496448516845703,
      "activations/layer12_attention_weight_min": -25.39525604248047,
      "activations/layer13_attention_weight_max": 40.136993408203125,
      "activations/layer13_attention_weight_min": -30.913089752197266,
      "activations/layer14_attention_weight_max": 47.903167724609375,
      "activations/layer14_attention_weight_min": -31.8095645904541,
      "activations/layer15_attention_weight_max": 35.8597412109375,
      "activations/layer15_attention_weight_min": -28.47408103942871,
      "activations/layer16_attention_weight_max": 28.591678619384766,
      "activations/layer16_attention_weight_min": -28.356473922729492,
      "activations/layer17_attention_weight_max": 52.15536117553711,
      "activations/layer17_attention_weight_min": -48.20656204223633,
      "activations/layer18_attention_weight_max": 42.42924118041992,
      "activations/layer18_attention_weight_min": -41.43056869506836,
      "activations/layer19_attention_weight_max": 19.698495864868164,
      "activations/layer19_attention_weight_min": -21.960630416870117,
      "activations/layer1_attention_weight_max": 17.56962776184082,
      "activations/layer1_attention_weight_min": -16.29872703552246,
      "activations/layer20_attention_weight_max": 20.859539031982422,
      "activations/layer20_attention_weight_min": -20.865493774414062,
      "activations/layer21_attention_weight_max": 32.10089111328125,
      "activations/layer21_attention_weight_min": -23.027921676635742,
      "activations/layer22_attention_weight_max": 29.919912338256836,
      "activations/layer22_attention_weight_min": -25.51317596435547,
      "activations/layer23_attention_weight_max": 34.181705474853516,
      "activations/layer23_attention_weight_min": -24.320148468017578,
      "activations/layer2_attention_weight_max": 31.982038497924805,
      "activations/layer2_attention_weight_min": -30.407575607299805,
      "activations/layer3_attention_weight_max": 87.2962417602539,
      "activations/layer3_attention_weight_min": -95.89067077636719,
      "activations/layer4_attention_weight_max": 96.33190155029297,
      "activations/layer4_attention_weight_min": -96.75182342529297,
      "activations/layer5_attention_weight_max": 80.66755676269531,
      "activations/layer5_attention_weight_min": -78.29901885986328,
      "activations/layer6_attention_weight_max": 51.73197555541992,
      "activations/layer6_attention_weight_min": -51.21516418457031,
      "activations/layer7_attention_weight_max": 71.4712142944336,
      "activations/layer7_attention_weight_min": -70.5478515625,
      "activations/layer8_attention_weight_max": 42.34003448486328,
      "activations/layer8_attention_weight_min": -43.16286087036133,
      "activations/layer9_attention_weight_max": 48.74201965332031,
      "activations/layer9_attention_weight_min": -47.26036071777344,
      "epoch": 9.14,
      "learning_rate": 9.19655303030303e-05,
      "loss": 2.7901,
      "step": 157350
    },
    {
      "activations/layer0_attention_weight_max": 15.980862617492676,
      "activations/layer0_attention_weight_min": -13.766925811767578,
      "activations/layer10_attention_weight_max": 37.56565856933594,
      "activations/layer10_attention_weight_min": -35.51362991333008,
      "activations/layer11_attention_weight_max": 32.530609130859375,
      "activations/layer11_attention_weight_min": -33.004783630371094,
      "activations/layer12_attention_weight_max": 19.18927574157715,
      "activations/layer12_attention_weight_min": -23.799713134765625,
      "activations/layer13_attention_weight_max": 39.306148529052734,
      "activations/layer13_attention_weight_min": -30.68119239807129,
      "activations/layer14_attention_weight_max": 42.68263626098633,
      "activations/layer14_attention_weight_min": -29.864477157592773,
      "activations/layer15_attention_weight_max": 34.68333435058594,
      "activations/layer15_attention_weight_min": -29.84563636779785,
      "activations/layer16_attention_weight_max": 31.976024627685547,
      "activations/layer16_attention_weight_min": -28.61627960205078,
      "activations/layer17_attention_weight_max": 57.17366027832031,
      "activations/layer17_attention_weight_min": -45.914432525634766,
      "activations/layer18_attention_weight_max": 50.46078872680664,
      "activations/layer18_attention_weight_min": -37.93956756591797,
      "activations/layer19_attention_weight_max": 22.60757827758789,
      "activations/layer19_attention_weight_min": -21.448789596557617,
      "activations/layer1_attention_weight_max": 16.275283813476562,
      "activations/layer1_attention_weight_min": -15.94635009765625,
      "activations/layer20_attention_weight_max": 24.928821563720703,
      "activations/layer20_attention_weight_min": -21.471935272216797,
      "activations/layer21_attention_weight_max": 43.199462890625,
      "activations/layer21_attention_weight_min": -25.318140029907227,
      "activations/layer22_attention_weight_max": 35.45392608642578,
      "activations/layer22_attention_weight_min": -25.458499908447266,
      "activations/layer23_attention_weight_max": 44.735877990722656,
      "activations/layer23_attention_weight_min": -24.665197372436523,
      "activations/layer2_attention_weight_max": 32.313377380371094,
      "activations/layer2_attention_weight_min": -29.701431274414062,
      "activations/layer3_attention_weight_max": 90.13262939453125,
      "activations/layer3_attention_weight_min": -90.41043853759766,
      "activations/layer4_attention_weight_max": 98.77272033691406,
      "activations/layer4_attention_weight_min": -94.2876205444336,
      "activations/layer5_attention_weight_max": 80.72856140136719,
      "activations/layer5_attention_weight_min": -78.73735046386719,
      "activations/layer6_attention_weight_max": 55.33551788330078,
      "activations/layer6_attention_weight_min": -52.7547492980957,
      "activations/layer7_attention_weight_max": 78.3030014038086,
      "activations/layer7_attention_weight_min": -75.6528549194336,
      "activations/layer8_attention_weight_max": 44.550750732421875,
      "activations/layer8_attention_weight_min": -45.467071533203125,
      "activations/layer9_attention_weight_max": 51.78486251831055,
      "activations/layer9_attention_weight_min": -50.7302131652832,
      "epoch": 9.15,
      "learning_rate": 9.194659090909091e-05,
      "loss": 2.8005,
      "step": 157400
    },
    {
      "activations/layer0_attention_weight_max": 14.90832233428955,
      "activations/layer0_attention_weight_min": -14.311159133911133,
      "activations/layer10_attention_weight_max": 36.57963180541992,
      "activations/layer10_attention_weight_min": -35.796138763427734,
      "activations/layer11_attention_weight_max": 31.283063888549805,
      "activations/layer11_attention_weight_min": -32.21709060668945,
      "activations/layer12_attention_weight_max": 18.86383819580078,
      "activations/layer12_attention_weight_min": -25.219945907592773,
      "activations/layer13_attention_weight_max": 44.487770080566406,
      "activations/layer13_attention_weight_min": -37.714317321777344,
      "activations/layer14_attention_weight_max": 36.828792572021484,
      "activations/layer14_attention_weight_min": -31.608776092529297,
      "activations/layer15_attention_weight_max": 34.23146057128906,
      "activations/layer15_attention_weight_min": -30.044992446899414,
      "activations/layer16_attention_weight_max": 32.73234558105469,
      "activations/layer16_attention_weight_min": -30.691511154174805,
      "activations/layer17_attention_weight_max": 55.39043045043945,
      "activations/layer17_attention_weight_min": -46.139469146728516,
      "activations/layer18_attention_weight_max": 46.273109436035156,
      "activations/layer18_attention_weight_min": -40.283138275146484,
      "activations/layer19_attention_weight_max": 22.209362030029297,
      "activations/layer19_attention_weight_min": -22.823135375976562,
      "activations/layer1_attention_weight_max": 16.461380004882812,
      "activations/layer1_attention_weight_min": -15.693746566772461,
      "activations/layer20_attention_weight_max": 23.67667007446289,
      "activations/layer20_attention_weight_min": -22.020883560180664,
      "activations/layer21_attention_weight_max": 46.73578643798828,
      "activations/layer21_attention_weight_min": -25.92144012451172,
      "activations/layer22_attention_weight_max": 32.64277648925781,
      "activations/layer22_attention_weight_min": -26.17945671081543,
      "activations/layer23_attention_weight_max": 39.93114471435547,
      "activations/layer23_attention_weight_min": -22.473379135131836,
      "activations/layer2_attention_weight_max": 33.03180694580078,
      "activations/layer2_attention_weight_min": -30.426488876342773,
      "activations/layer3_attention_weight_max": 89.04460144042969,
      "activations/layer3_attention_weight_min": -94.582275390625,
      "activations/layer4_attention_weight_max": 99.31370544433594,
      "activations/layer4_attention_weight_min": -97.67357635498047,
      "activations/layer5_attention_weight_max": 78.57441711425781,
      "activations/layer5_attention_weight_min": -75.83736419677734,
      "activations/layer6_attention_weight_max": 53.90927505493164,
      "activations/layer6_attention_weight_min": -53.82857894897461,
      "activations/layer7_attention_weight_max": 77.74165344238281,
      "activations/layer7_attention_weight_min": -74.83365631103516,
      "activations/layer8_attention_weight_max": 43.1878547668457,
      "activations/layer8_attention_weight_min": -44.94409942626953,
      "activations/layer9_attention_weight_max": 51.83112335205078,
      "activations/layer9_attention_weight_min": -50.9305534362793,
      "epoch": 9.15,
      "learning_rate": 9.19276515151515e-05,
      "loss": 2.8037,
      "step": 157450
    },
    {
      "activations/layer0_attention_weight_max": 14.34084701538086,
      "activations/layer0_attention_weight_min": -14.217262268066406,
      "activations/layer10_attention_weight_max": 35.98621368408203,
      "activations/layer10_attention_weight_min": -34.85947036743164,
      "activations/layer11_attention_weight_max": 29.725461959838867,
      "activations/layer11_attention_weight_min": -33.21236801147461,
      "activations/layer12_attention_weight_max": 19.211349487304688,
      "activations/layer12_attention_weight_min": -25.42329216003418,
      "activations/layer13_attention_weight_max": 41.309959411621094,
      "activations/layer13_attention_weight_min": -27.57022476196289,
      "activations/layer14_attention_weight_max": 36.21197509765625,
      "activations/layer14_attention_weight_min": -30.7191104888916,
      "activations/layer15_attention_weight_max": 35.44427490234375,
      "activations/layer15_attention_weight_min": -31.136459350585938,
      "activations/layer16_attention_weight_max": 31.28398323059082,
      "activations/layer16_attention_weight_min": -29.01300048828125,
      "activations/layer17_attention_weight_max": 51.59337615966797,
      "activations/layer17_attention_weight_min": -41.21481704711914,
      "activations/layer18_attention_weight_max": 45.386497497558594,
      "activations/layer18_attention_weight_min": -35.777069091796875,
      "activations/layer19_attention_weight_max": 23.01891326904297,
      "activations/layer19_attention_weight_min": -21.148435592651367,
      "activations/layer1_attention_weight_max": 17.03573989868164,
      "activations/layer1_attention_weight_min": -16.746374130249023,
      "activations/layer20_attention_weight_max": 22.26829719543457,
      "activations/layer20_attention_weight_min": -23.5335636138916,
      "activations/layer21_attention_weight_max": 37.225128173828125,
      "activations/layer21_attention_weight_min": -23.099151611328125,
      "activations/layer22_attention_weight_max": 31.644224166870117,
      "activations/layer22_attention_weight_min": -25.927345275878906,
      "activations/layer23_attention_weight_max": 34.484283447265625,
      "activations/layer23_attention_weight_min": -22.236083984375,
      "activations/layer2_attention_weight_max": 31.14126205444336,
      "activations/layer2_attention_weight_min": -29.721342086791992,
      "activations/layer3_attention_weight_max": 88.18553924560547,
      "activations/layer3_attention_weight_min": -92.66230773925781,
      "activations/layer4_attention_weight_max": 100.66731262207031,
      "activations/layer4_attention_weight_min": -95.1903305053711,
      "activations/layer5_attention_weight_max": 81.23687744140625,
      "activations/layer5_attention_weight_min": -74.57054138183594,
      "activations/layer6_attention_weight_max": 53.112388610839844,
      "activations/layer6_attention_weight_min": -54.466087341308594,
      "activations/layer7_attention_weight_max": 75.84051513671875,
      "activations/layer7_attention_weight_min": -71.3451156616211,
      "activations/layer8_attention_weight_max": 43.68793487548828,
      "activations/layer8_attention_weight_min": -44.04755783081055,
      "activations/layer9_attention_weight_max": 51.6351203918457,
      "activations/layer9_attention_weight_min": -48.5887336730957,
      "epoch": 9.15,
      "learning_rate": 9.190871212121211e-05,
      "loss": 2.7855,
      "step": 157500
    },
    {
      "activations/layer0_attention_weight_max": 15.108270645141602,
      "activations/layer0_attention_weight_min": -13.969242095947266,
      "activations/layer10_attention_weight_max": 35.675567626953125,
      "activations/layer10_attention_weight_min": -34.69825744628906,
      "activations/layer11_attention_weight_max": 30.346792221069336,
      "activations/layer11_attention_weight_min": -32.23979568481445,
      "activations/layer12_attention_weight_max": 19.388769149780273,
      "activations/layer12_attention_weight_min": -26.16206169128418,
      "activations/layer13_attention_weight_max": 39.46177673339844,
      "activations/layer13_attention_weight_min": -27.982807159423828,
      "activations/layer14_attention_weight_max": 38.27208709716797,
      "activations/layer14_attention_weight_min": -32.666290283203125,
      "activations/layer15_attention_weight_max": 33.42053985595703,
      "activations/layer15_attention_weight_min": -29.796878814697266,
      "activations/layer16_attention_weight_max": 32.4610595703125,
      "activations/layer16_attention_weight_min": -27.13525390625,
      "activations/layer17_attention_weight_max": 54.109989166259766,
      "activations/layer17_attention_weight_min": -42.94585037231445,
      "activations/layer18_attention_weight_max": 55.47537612915039,
      "activations/layer18_attention_weight_min": -37.68131637573242,
      "activations/layer19_attention_weight_max": 21.3824462890625,
      "activations/layer19_attention_weight_min": -22.236364364624023,
      "activations/layer1_attention_weight_max": 17.607940673828125,
      "activations/layer1_attention_weight_min": -16.702421188354492,
      "activations/layer20_attention_weight_max": 23.301395416259766,
      "activations/layer20_attention_weight_min": -24.995384216308594,
      "activations/layer21_attention_weight_max": 39.90260696411133,
      "activations/layer21_attention_weight_min": -22.89287757873535,
      "activations/layer22_attention_weight_max": 35.804317474365234,
      "activations/layer22_attention_weight_min": -24.85230827331543,
      "activations/layer23_attention_weight_max": 38.725101470947266,
      "activations/layer23_attention_weight_min": -21.764204025268555,
      "activations/layer2_attention_weight_max": 32.66266632080078,
      "activations/layer2_attention_weight_min": -30.553518295288086,
      "activations/layer3_attention_weight_max": 89.16400146484375,
      "activations/layer3_attention_weight_min": -90.79566955566406,
      "activations/layer4_attention_weight_max": 98.87627410888672,
      "activations/layer4_attention_weight_min": -95.06420135498047,
      "activations/layer5_attention_weight_max": 78.91986083984375,
      "activations/layer5_attention_weight_min": -75.90260314941406,
      "activations/layer6_attention_weight_max": 55.087738037109375,
      "activations/layer6_attention_weight_min": -52.6690788269043,
      "activations/layer7_attention_weight_max": 78.91346740722656,
      "activations/layer7_attention_weight_min": -74.03681182861328,
      "activations/layer8_attention_weight_max": 44.32879638671875,
      "activations/layer8_attention_weight_min": -46.83210754394531,
      "activations/layer9_attention_weight_max": 49.7331657409668,
      "activations/layer9_attention_weight_min": -45.841453552246094,
      "epoch": 9.15,
      "learning_rate": 9.188977272727273e-05,
      "loss": 2.7813,
      "step": 157550
    },
    {
      "activations/layer0_attention_weight_max": 16.052152633666992,
      "activations/layer0_attention_weight_min": -13.95780086517334,
      "activations/layer10_attention_weight_max": 35.49897384643555,
      "activations/layer10_attention_weight_min": -34.42238235473633,
      "activations/layer11_attention_weight_max": 31.93796157836914,
      "activations/layer11_attention_weight_min": -30.511150360107422,
      "activations/layer12_attention_weight_max": 21.041894912719727,
      "activations/layer12_attention_weight_min": -22.794858932495117,
      "activations/layer13_attention_weight_max": 58.88551712036133,
      "activations/layer13_attention_weight_min": -35.32522201538086,
      "activations/layer14_attention_weight_max": 42.4543342590332,
      "activations/layer14_attention_weight_min": -29.959693908691406,
      "activations/layer15_attention_weight_max": 36.21780776977539,
      "activations/layer15_attention_weight_min": -30.744770050048828,
      "activations/layer16_attention_weight_max": 34.509056091308594,
      "activations/layer16_attention_weight_min": -28.046525955200195,
      "activations/layer17_attention_weight_max": 53.38486099243164,
      "activations/layer17_attention_weight_min": -41.53818893432617,
      "activations/layer18_attention_weight_max": 45.00223159790039,
      "activations/layer18_attention_weight_min": -34.99028396606445,
      "activations/layer19_attention_weight_max": 23.446043014526367,
      "activations/layer19_attention_weight_min": -19.06755256652832,
      "activations/layer1_attention_weight_max": 16.874147415161133,
      "activations/layer1_attention_weight_min": -15.53210163116455,
      "activations/layer20_attention_weight_max": 22.517873764038086,
      "activations/layer20_attention_weight_min": -20.341520309448242,
      "activations/layer21_attention_weight_max": 34.149253845214844,
      "activations/layer21_attention_weight_min": -22.19076156616211,
      "activations/layer22_attention_weight_max": 30.470632553100586,
      "activations/layer22_attention_weight_min": -23.94293975830078,
      "activations/layer23_attention_weight_max": 39.65612030029297,
      "activations/layer23_attention_weight_min": -22.811355590820312,
      "activations/layer2_attention_weight_max": 32.32324981689453,
      "activations/layer2_attention_weight_min": -29.533740997314453,
      "activations/layer3_attention_weight_max": 88.57725524902344,
      "activations/layer3_attention_weight_min": -90.60629272460938,
      "activations/layer4_attention_weight_max": 95.46014404296875,
      "activations/layer4_attention_weight_min": -92.68458557128906,
      "activations/layer5_attention_weight_max": 80.06434631347656,
      "activations/layer5_attention_weight_min": -75.34905242919922,
      "activations/layer6_attention_weight_max": 52.63520812988281,
      "activations/layer6_attention_weight_min": -50.18421173095703,
      "activations/layer7_attention_weight_max": 78.7744140625,
      "activations/layer7_attention_weight_min": -72.90723419189453,
      "activations/layer8_attention_weight_max": 46.18045425415039,
      "activations/layer8_attention_weight_min": -46.310176849365234,
      "activations/layer9_attention_weight_max": 49.77947235107422,
      "activations/layer9_attention_weight_min": -49.69879913330078,
      "epoch": 9.16,
      "learning_rate": 9.187083333333333e-05,
      "loss": 2.8022,
      "step": 157600
    },
    {
      "activations/layer0_attention_weight_max": 15.06613540649414,
      "activations/layer0_attention_weight_min": -14.245283126831055,
      "activations/layer10_attention_weight_max": 37.41593933105469,
      "activations/layer10_attention_weight_min": -35.756412506103516,
      "activations/layer11_attention_weight_max": 31.241973876953125,
      "activations/layer11_attention_weight_min": -31.304824829101562,
      "activations/layer12_attention_weight_max": 18.386987686157227,
      "activations/layer12_attention_weight_min": -25.062620162963867,
      "activations/layer13_attention_weight_max": 39.10600662231445,
      "activations/layer13_attention_weight_min": -30.463932037353516,
      "activations/layer14_attention_weight_max": 35.64311218261719,
      "activations/layer14_attention_weight_min": -31.703895568847656,
      "activations/layer15_attention_weight_max": 33.83919143676758,
      "activations/layer15_attention_weight_min": -31.022960662841797,
      "activations/layer16_attention_weight_max": 30.643054962158203,
      "activations/layer16_attention_weight_min": -29.859081268310547,
      "activations/layer17_attention_weight_max": 49.39906692504883,
      "activations/layer17_attention_weight_min": -43.674781799316406,
      "activations/layer18_attention_weight_max": 46.51315689086914,
      "activations/layer18_attention_weight_min": -36.0517692565918,
      "activations/layer19_attention_weight_max": 20.23090934753418,
      "activations/layer19_attention_weight_min": -20.540508270263672,
      "activations/layer1_attention_weight_max": 16.384769439697266,
      "activations/layer1_attention_weight_min": -15.647690773010254,
      "activations/layer20_attention_weight_max": 21.77815055847168,
      "activations/layer20_attention_weight_min": -20.925765991210938,
      "activations/layer21_attention_weight_max": 34.95148468017578,
      "activations/layer21_attention_weight_min": -21.30117416381836,
      "activations/layer22_attention_weight_max": 31.53641128540039,
      "activations/layer22_attention_weight_min": -26.242156982421875,
      "activations/layer23_attention_weight_max": 36.592491149902344,
      "activations/layer23_attention_weight_min": -21.879865646362305,
      "activations/layer2_attention_weight_max": 33.397830963134766,
      "activations/layer2_attention_weight_min": -32.89070129394531,
      "activations/layer3_attention_weight_max": 90.16561889648438,
      "activations/layer3_attention_weight_min": -91.60643768310547,
      "activations/layer4_attention_weight_max": 102.43592834472656,
      "activations/layer4_attention_weight_min": -98.58126068115234,
      "activations/layer5_attention_weight_max": 81.43241119384766,
      "activations/layer5_attention_weight_min": -77.09913635253906,
      "activations/layer6_attention_weight_max": 53.45168685913086,
      "activations/layer6_attention_weight_min": -53.45527267456055,
      "activations/layer7_attention_weight_max": 75.65214538574219,
      "activations/layer7_attention_weight_min": -76.61619567871094,
      "activations/layer8_attention_weight_max": 44.728614807128906,
      "activations/layer8_attention_weight_min": -46.78208541870117,
      "activations/layer9_attention_weight_max": 51.63245391845703,
      "activations/layer9_attention_weight_min": -49.8437385559082,
      "epoch": 9.16,
      "learning_rate": 9.185189393939393e-05,
      "loss": 2.7963,
      "step": 157650
    },
    {
      "activations/layer0_attention_weight_max": 15.995274543762207,
      "activations/layer0_attention_weight_min": -13.959744453430176,
      "activations/layer10_attention_weight_max": 34.340152740478516,
      "activations/layer10_attention_weight_min": -35.107906341552734,
      "activations/layer11_attention_weight_max": 30.180273056030273,
      "activations/layer11_attention_weight_min": -31.713958740234375,
      "activations/layer12_attention_weight_max": 17.76327133178711,
      "activations/layer12_attention_weight_min": -27.46477508544922,
      "activations/layer13_attention_weight_max": 39.85215377807617,
      "activations/layer13_attention_weight_min": -30.3848934173584,
      "activations/layer14_attention_weight_max": 36.1807746887207,
      "activations/layer14_attention_weight_min": -31.507095336914062,
      "activations/layer15_attention_weight_max": 34.62899398803711,
      "activations/layer15_attention_weight_min": -31.862762451171875,
      "activations/layer16_attention_weight_max": 30.620824813842773,
      "activations/layer16_attention_weight_min": -27.78635597229004,
      "activations/layer17_attention_weight_max": 51.40058135986328,
      "activations/layer17_attention_weight_min": -43.99736785888672,
      "activations/layer18_attention_weight_max": 46.13364028930664,
      "activations/layer18_attention_weight_min": -36.647464752197266,
      "activations/layer19_attention_weight_max": 20.560028076171875,
      "activations/layer19_attention_weight_min": -22.066450119018555,
      "activations/layer1_attention_weight_max": 17.092998504638672,
      "activations/layer1_attention_weight_min": -15.834074974060059,
      "activations/layer20_attention_weight_max": 21.443336486816406,
      "activations/layer20_attention_weight_min": -20.67938995361328,
      "activations/layer21_attention_weight_max": 35.80448913574219,
      "activations/layer21_attention_weight_min": -22.590490341186523,
      "activations/layer22_attention_weight_max": 34.010677337646484,
      "activations/layer22_attention_weight_min": -27.795137405395508,
      "activations/layer23_attention_weight_max": 38.99952697753906,
      "activations/layer23_attention_weight_min": -24.524642944335938,
      "activations/layer2_attention_weight_max": 32.26961898803711,
      "activations/layer2_attention_weight_min": -31.155019760131836,
      "activations/layer3_attention_weight_max": 84.07659149169922,
      "activations/layer3_attention_weight_min": -88.34294128417969,
      "activations/layer4_attention_weight_max": 96.53868865966797,
      "activations/layer4_attention_weight_min": -91.10137176513672,
      "activations/layer5_attention_weight_max": 77.081787109375,
      "activations/layer5_attention_weight_min": -83.22395324707031,
      "activations/layer6_attention_weight_max": 52.43558883666992,
      "activations/layer6_attention_weight_min": -52.30995559692383,
      "activations/layer7_attention_weight_max": 77.85987091064453,
      "activations/layer7_attention_weight_min": -73.52764129638672,
      "activations/layer8_attention_weight_max": 42.418453216552734,
      "activations/layer8_attention_weight_min": -44.475101470947266,
      "activations/layer9_attention_weight_max": 47.0313606262207,
      "activations/layer9_attention_weight_min": -45.535423278808594,
      "epoch": 9.16,
      "learning_rate": 9.183295454545454e-05,
      "loss": 2.7803,
      "step": 157700
    },
    {
      "activations/layer0_attention_weight_max": 15.22437572479248,
      "activations/layer0_attention_weight_min": -14.357569694519043,
      "activations/layer10_attention_weight_max": 36.121849060058594,
      "activations/layer10_attention_weight_min": -36.21514129638672,
      "activations/layer11_attention_weight_max": 31.590208053588867,
      "activations/layer11_attention_weight_min": -32.18960189819336,
      "activations/layer12_attention_weight_max": 18.552465438842773,
      "activations/layer12_attention_weight_min": -27.295473098754883,
      "activations/layer13_attention_weight_max": 42.786170959472656,
      "activations/layer13_attention_weight_min": -33.041805267333984,
      "activations/layer14_attention_weight_max": 38.87960433959961,
      "activations/layer14_attention_weight_min": -31.227764129638672,
      "activations/layer15_attention_weight_max": 36.575416564941406,
      "activations/layer15_attention_weight_min": -28.772966384887695,
      "activations/layer16_attention_weight_max": 33.83781814575195,
      "activations/layer16_attention_weight_min": -29.86800765991211,
      "activations/layer17_attention_weight_max": 56.39696502685547,
      "activations/layer17_attention_weight_min": -46.06596374511719,
      "activations/layer18_attention_weight_max": 46.534507751464844,
      "activations/layer18_attention_weight_min": -38.067779541015625,
      "activations/layer19_attention_weight_max": 23.537425994873047,
      "activations/layer19_attention_weight_min": -23.8226261138916,
      "activations/layer1_attention_weight_max": 16.74985122680664,
      "activations/layer1_attention_weight_min": -14.111126899719238,
      "activations/layer20_attention_weight_max": 25.39822006225586,
      "activations/layer20_attention_weight_min": -22.592151641845703,
      "activations/layer21_attention_weight_max": 43.7075309753418,
      "activations/layer21_attention_weight_min": -23.096237182617188,
      "activations/layer22_attention_weight_max": 32.281394958496094,
      "activations/layer22_attention_weight_min": -25.52610206604004,
      "activations/layer23_attention_weight_max": 42.56668472290039,
      "activations/layer23_attention_weight_min": -24.407611846923828,
      "activations/layer2_attention_weight_max": 30.994733810424805,
      "activations/layer2_attention_weight_min": -28.897541046142578,
      "activations/layer3_attention_weight_max": 85.77902221679688,
      "activations/layer3_attention_weight_min": -88.18103790283203,
      "activations/layer4_attention_weight_max": 97.6533432006836,
      "activations/layer4_attention_weight_min": -92.9091567993164,
      "activations/layer5_attention_weight_max": 84.42864227294922,
      "activations/layer5_attention_weight_min": -78.09440612792969,
      "activations/layer6_attention_weight_max": 55.90888977050781,
      "activations/layer6_attention_weight_min": -55.12588882446289,
      "activations/layer7_attention_weight_max": 81.42273712158203,
      "activations/layer7_attention_weight_min": -74.67053985595703,
      "activations/layer8_attention_weight_max": 46.11024856567383,
      "activations/layer8_attention_weight_min": -44.17125701904297,
      "activations/layer9_attention_weight_max": 51.046669006347656,
      "activations/layer9_attention_weight_min": -48.347198486328125,
      "epoch": 9.17,
      "learning_rate": 9.181401515151515e-05,
      "loss": 2.788,
      "step": 157750
    },
    {
      "activations/layer0_attention_weight_max": 15.298863410949707,
      "activations/layer0_attention_weight_min": -14.21564769744873,
      "activations/layer10_attention_weight_max": 35.06207275390625,
      "activations/layer10_attention_weight_min": -34.503028869628906,
      "activations/layer11_attention_weight_max": 30.997093200683594,
      "activations/layer11_attention_weight_min": -30.558530807495117,
      "activations/layer12_attention_weight_max": 21.773359298706055,
      "activations/layer12_attention_weight_min": -25.570205688476562,
      "activations/layer13_attention_weight_max": 45.60746765136719,
      "activations/layer13_attention_weight_min": -33.3326416015625,
      "activations/layer14_attention_weight_max": 44.82241439819336,
      "activations/layer14_attention_weight_min": -32.690948486328125,
      "activations/layer15_attention_weight_max": 41.0733528137207,
      "activations/layer15_attention_weight_min": -29.925994873046875,
      "activations/layer16_attention_weight_max": 36.97261428833008,
      "activations/layer16_attention_weight_min": -30.04011344909668,
      "activations/layer17_attention_weight_max": 58.06062316894531,
      "activations/layer17_attention_weight_min": -46.828857421875,
      "activations/layer18_attention_weight_max": 53.61669921875,
      "activations/layer18_attention_weight_min": -38.522193908691406,
      "activations/layer19_attention_weight_max": 23.452878952026367,
      "activations/layer19_attention_weight_min": -22.118492126464844,
      "activations/layer1_attention_weight_max": 16.737266540527344,
      "activations/layer1_attention_weight_min": -15.730106353759766,
      "activations/layer20_attention_weight_max": 24.627037048339844,
      "activations/layer20_attention_weight_min": -21.03171730041504,
      "activations/layer21_attention_weight_max": 42.178306579589844,
      "activations/layer21_attention_weight_min": -22.924795150756836,
      "activations/layer22_attention_weight_max": 32.99470901489258,
      "activations/layer22_attention_weight_min": -23.966686248779297,
      "activations/layer23_attention_weight_max": 42.28173828125,
      "activations/layer23_attention_weight_min": -22.71534538269043,
      "activations/layer2_attention_weight_max": 31.884803771972656,
      "activations/layer2_attention_weight_min": -31.010009765625,
      "activations/layer3_attention_weight_max": 88.40874481201172,
      "activations/layer3_attention_weight_min": -92.26502990722656,
      "activations/layer4_attention_weight_max": 101.58629608154297,
      "activations/layer4_attention_weight_min": -94.46041107177734,
      "activations/layer5_attention_weight_max": 80.61713409423828,
      "activations/layer5_attention_weight_min": -77.94035339355469,
      "activations/layer6_attention_weight_max": 57.7911262512207,
      "activations/layer6_attention_weight_min": -54.97450637817383,
      "activations/layer7_attention_weight_max": 78.33032989501953,
      "activations/layer7_attention_weight_min": -71.32780456542969,
      "activations/layer8_attention_weight_max": 43.06132125854492,
      "activations/layer8_attention_weight_min": -44.271236419677734,
      "activations/layer9_attention_weight_max": 48.9783935546875,
      "activations/layer9_attention_weight_min": -46.45318603515625,
      "epoch": 9.17,
      "learning_rate": 9.179507575757575e-05,
      "loss": 2.8032,
      "step": 157800
    },
    {
      "activations/layer0_attention_weight_max": 15.03512954711914,
      "activations/layer0_attention_weight_min": -14.069589614868164,
      "activations/layer10_attention_weight_max": 38.862709045410156,
      "activations/layer10_attention_weight_min": -36.48535919189453,
      "activations/layer11_attention_weight_max": 33.563846588134766,
      "activations/layer11_attention_weight_min": -32.409507751464844,
      "activations/layer12_attention_weight_max": 22.06846046447754,
      "activations/layer12_attention_weight_min": -24.476642608642578,
      "activations/layer13_attention_weight_max": 43.325355529785156,
      "activations/layer13_attention_weight_min": -27.286413192749023,
      "activations/layer14_attention_weight_max": 38.36415100097656,
      "activations/layer14_attention_weight_min": -29.738082885742188,
      "activations/layer15_attention_weight_max": 35.792110443115234,
      "activations/layer15_attention_weight_min": -28.615995407104492,
      "activations/layer16_attention_weight_max": 37.38132095336914,
      "activations/layer16_attention_weight_min": -30.57288360595703,
      "activations/layer17_attention_weight_max": 56.64357376098633,
      "activations/layer17_attention_weight_min": -45.663116455078125,
      "activations/layer18_attention_weight_max": 49.93989562988281,
      "activations/layer18_attention_weight_min": -36.219539642333984,
      "activations/layer19_attention_weight_max": 24.03340721130371,
      "activations/layer19_attention_weight_min": -22.72576904296875,
      "activations/layer1_attention_weight_max": 16.66884422302246,
      "activations/layer1_attention_weight_min": -15.404657363891602,
      "activations/layer20_attention_weight_max": 23.230897903442383,
      "activations/layer20_attention_weight_min": -26.854759216308594,
      "activations/layer21_attention_weight_max": 43.87388610839844,
      "activations/layer21_attention_weight_min": -26.47745132446289,
      "activations/layer22_attention_weight_max": 36.41883087158203,
      "activations/layer22_attention_weight_min": -25.9670467376709,
      "activations/layer23_attention_weight_max": 46.983192443847656,
      "activations/layer23_attention_weight_min": -23.468353271484375,
      "activations/layer2_attention_weight_max": 32.982566833496094,
      "activations/layer2_attention_weight_min": -31.255802154541016,
      "activations/layer3_attention_weight_max": 95.1622314453125,
      "activations/layer3_attention_weight_min": -94.20480346679688,
      "activations/layer4_attention_weight_max": 102.85980224609375,
      "activations/layer4_attention_weight_min": -95.06636810302734,
      "activations/layer5_attention_weight_max": 81.909912109375,
      "activations/layer5_attention_weight_min": -79.08079528808594,
      "activations/layer6_attention_weight_max": 56.268829345703125,
      "activations/layer6_attention_weight_min": -55.81147003173828,
      "activations/layer7_attention_weight_max": 79.01127624511719,
      "activations/layer7_attention_weight_min": -77.82035064697266,
      "activations/layer8_attention_weight_max": 47.2025146484375,
      "activations/layer8_attention_weight_min": -47.689056396484375,
      "activations/layer9_attention_weight_max": 49.886287689208984,
      "activations/layer9_attention_weight_min": -55.3343391418457,
      "epoch": 9.17,
      "learning_rate": 9.177613636363635e-05,
      "loss": 2.7844,
      "step": 157850
    },
    {
      "activations/layer0_attention_weight_max": 16.437511444091797,
      "activations/layer0_attention_weight_min": -13.74957275390625,
      "activations/layer10_attention_weight_max": 44.78004455566406,
      "activations/layer10_attention_weight_min": -39.86311721801758,
      "activations/layer11_attention_weight_max": 37.55427551269531,
      "activations/layer11_attention_weight_min": -34.879451751708984,
      "activations/layer12_attention_weight_max": 19.453659057617188,
      "activations/layer12_attention_weight_min": -23.511884689331055,
      "activations/layer13_attention_weight_max": 42.81022644042969,
      "activations/layer13_attention_weight_min": -30.401033401489258,
      "activations/layer14_attention_weight_max": 49.811527252197266,
      "activations/layer14_attention_weight_min": -31.65997314453125,
      "activations/layer15_attention_weight_max": 37.934715270996094,
      "activations/layer15_attention_weight_min": -30.906343460083008,
      "activations/layer16_attention_weight_max": 34.81535720825195,
      "activations/layer16_attention_weight_min": -29.265735626220703,
      "activations/layer17_attention_weight_max": 60.96263885498047,
      "activations/layer17_attention_weight_min": -48.21268844604492,
      "activations/layer18_attention_weight_max": 51.70690155029297,
      "activations/layer18_attention_weight_min": -41.33689498901367,
      "activations/layer19_attention_weight_max": 26.176502227783203,
      "activations/layer19_attention_weight_min": -24.301599502563477,
      "activations/layer1_attention_weight_max": 17.29594612121582,
      "activations/layer1_attention_weight_min": -15.524714469909668,
      "activations/layer20_attention_weight_max": 25.789106369018555,
      "activations/layer20_attention_weight_min": -21.719905853271484,
      "activations/layer21_attention_weight_max": 44.09783172607422,
      "activations/layer21_attention_weight_min": -24.762571334838867,
      "activations/layer22_attention_weight_max": 35.439937591552734,
      "activations/layer22_attention_weight_min": -25.291303634643555,
      "activations/layer23_attention_weight_max": 44.74795150756836,
      "activations/layer23_attention_weight_min": -24.092731475830078,
      "activations/layer2_attention_weight_max": 30.967618942260742,
      "activations/layer2_attention_weight_min": -28.825315475463867,
      "activations/layer3_attention_weight_max": 88.58335876464844,
      "activations/layer3_attention_weight_min": -91.96842193603516,
      "activations/layer4_attention_weight_max": 100.89611053466797,
      "activations/layer4_attention_weight_min": -97.04131317138672,
      "activations/layer5_attention_weight_max": 82.55316162109375,
      "activations/layer5_attention_weight_min": -76.21695709228516,
      "activations/layer6_attention_weight_max": 64.04566955566406,
      "activations/layer6_attention_weight_min": -58.498695373535156,
      "activations/layer7_attention_weight_max": 85.83313751220703,
      "activations/layer7_attention_weight_min": -74.41731262207031,
      "activations/layer8_attention_weight_max": 50.4837760925293,
      "activations/layer8_attention_weight_min": -49.00930404663086,
      "activations/layer9_attention_weight_max": 56.25214385986328,
      "activations/layer9_attention_weight_min": -50.926414489746094,
      "epoch": 9.17,
      "learning_rate": 9.175719696969696e-05,
      "loss": 2.7799,
      "step": 157900
    },
    {
      "activations/layer0_attention_weight_max": 15.598459243774414,
      "activations/layer0_attention_weight_min": -13.945764541625977,
      "activations/layer10_attention_weight_max": 39.97242736816406,
      "activations/layer10_attention_weight_min": -36.80802917480469,
      "activations/layer11_attention_weight_max": 34.942989349365234,
      "activations/layer11_attention_weight_min": -32.67131042480469,
      "activations/layer12_attention_weight_max": 20.20563316345215,
      "activations/layer12_attention_weight_min": -28.03492546081543,
      "activations/layer13_attention_weight_max": 39.4337043762207,
      "activations/layer13_attention_weight_min": -27.48309898376465,
      "activations/layer14_attention_weight_max": 40.03837966918945,
      "activations/layer14_attention_weight_min": -28.090402603149414,
      "activations/layer15_attention_weight_max": 32.26556396484375,
      "activations/layer15_attention_weight_min": -26.95647430419922,
      "activations/layer16_attention_weight_max": 30.22731590270996,
      "activations/layer16_attention_weight_min": -26.69822883605957,
      "activations/layer17_attention_weight_max": 55.459999084472656,
      "activations/layer17_attention_weight_min": -40.07382583618164,
      "activations/layer18_attention_weight_max": 49.04082489013672,
      "activations/layer18_attention_weight_min": -35.901161193847656,
      "activations/layer19_attention_weight_max": 18.42609405517578,
      "activations/layer19_attention_weight_min": -20.262996673583984,
      "activations/layer1_attention_weight_max": 17.414306640625,
      "activations/layer1_attention_weight_min": -15.885599136352539,
      "activations/layer20_attention_weight_max": 20.58995246887207,
      "activations/layer20_attention_weight_min": -21.648508071899414,
      "activations/layer21_attention_weight_max": 32.7056770324707,
      "activations/layer21_attention_weight_min": -22.303340911865234,
      "activations/layer22_attention_weight_max": 28.838037490844727,
      "activations/layer22_attention_weight_min": -25.890270233154297,
      "activations/layer23_attention_weight_max": 34.80305480957031,
      "activations/layer23_attention_weight_min": -21.04421615600586,
      "activations/layer2_attention_weight_max": 31.198684692382812,
      "activations/layer2_attention_weight_min": -29.536224365234375,
      "activations/layer3_attention_weight_max": 88.2313461303711,
      "activations/layer3_attention_weight_min": -92.22013854980469,
      "activations/layer4_attention_weight_max": 98.91490936279297,
      "activations/layer4_attention_weight_min": -93.20439910888672,
      "activations/layer5_attention_weight_max": 79.47366333007812,
      "activations/layer5_attention_weight_min": -73.63906860351562,
      "activations/layer6_attention_weight_max": 55.55046844482422,
      "activations/layer6_attention_weight_min": -53.799320220947266,
      "activations/layer7_attention_weight_max": 81.586181640625,
      "activations/layer7_attention_weight_min": -71.88672637939453,
      "activations/layer8_attention_weight_max": 47.44919967651367,
      "activations/layer8_attention_weight_min": -46.084190368652344,
      "activations/layer9_attention_weight_max": 53.52983474731445,
      "activations/layer9_attention_weight_min": -48.90407180786133,
      "epoch": 9.18,
      "learning_rate": 9.173825757575758e-05,
      "loss": 2.8031,
      "step": 157950
    },
    {
      "activations/layer0_attention_weight_max": 15.675469398498535,
      "activations/layer0_attention_weight_min": -13.889877319335938,
      "activations/layer10_attention_weight_max": 38.59505081176758,
      "activations/layer10_attention_weight_min": -34.29289627075195,
      "activations/layer11_attention_weight_max": 32.69541931152344,
      "activations/layer11_attention_weight_min": -30.910221099853516,
      "activations/layer12_attention_weight_max": 22.967924118041992,
      "activations/layer12_attention_weight_min": -23.23922348022461,
      "activations/layer13_attention_weight_max": 43.33582305908203,
      "activations/layer13_attention_weight_min": -29.69359588623047,
      "activations/layer14_attention_weight_max": 50.87261962890625,
      "activations/layer14_attention_weight_min": -29.92830467224121,
      "activations/layer15_attention_weight_max": 38.7039794921875,
      "activations/layer15_attention_weight_min": -28.41282844543457,
      "activations/layer16_attention_weight_max": 32.08932876586914,
      "activations/layer16_attention_weight_min": -28.26850128173828,
      "activations/layer17_attention_weight_max": 55.52051544189453,
      "activations/layer17_attention_weight_min": -42.74941635131836,
      "activations/layer18_attention_weight_max": 51.483638763427734,
      "activations/layer18_attention_weight_min": -35.60811996459961,
      "activations/layer19_attention_weight_max": 27.30555534362793,
      "activations/layer19_attention_weight_min": -22.619539260864258,
      "activations/layer1_attention_weight_max": 17.390825271606445,
      "activations/layer1_attention_weight_min": -14.74675178527832,
      "activations/layer20_attention_weight_max": 26.486488342285156,
      "activations/layer20_attention_weight_min": -20.749038696289062,
      "activations/layer21_attention_weight_max": 49.057003021240234,
      "activations/layer21_attention_weight_min": -23.486125946044922,
      "activations/layer22_attention_weight_max": 33.73653030395508,
      "activations/layer22_attention_weight_min": -23.110410690307617,
      "activations/layer23_attention_weight_max": 37.71785354614258,
      "activations/layer23_attention_weight_min": -23.337438583374023,
      "activations/layer2_attention_weight_max": 29.927188873291016,
      "activations/layer2_attention_weight_min": -29.199705123901367,
      "activations/layer3_attention_weight_max": 86.71466064453125,
      "activations/layer3_attention_weight_min": -87.6125259399414,
      "activations/layer4_attention_weight_max": 96.8193130493164,
      "activations/layer4_attention_weight_min": -91.2158203125,
      "activations/layer5_attention_weight_max": 78.8543472290039,
      "activations/layer5_attention_weight_min": -75.22017669677734,
      "activations/layer6_attention_weight_max": 56.13117980957031,
      "activations/layer6_attention_weight_min": -52.12679672241211,
      "activations/layer7_attention_weight_max": 82.18035125732422,
      "activations/layer7_attention_weight_min": -72.03529357910156,
      "activations/layer8_attention_weight_max": 46.69640350341797,
      "activations/layer8_attention_weight_min": -44.96013641357422,
      "activations/layer9_attention_weight_max": 48.27059555053711,
      "activations/layer9_attention_weight_min": -47.71844482421875,
      "epoch": 9.18,
      "learning_rate": 9.171931818181817e-05,
      "loss": 2.7996,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_loss": 2.75,
      "eval_runtime": 8.5177,
      "eval_samples_per_second": 504.128,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.5177,
      "eval_openwebtext_samples_per_second": 504.128,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_wikitext_loss": 2.986328125,
      "eval_wikitext_ppl": 19.812798643245316,
      "eval_wikitext_runtime": 1.9803,
      "eval_wikitext_samples_per_second": 230.273,
      "step": 158000
    },
    {
      "epoch": 9.18,
      "eval_lambada_loss": 2.662109375,
      "eval_lambada_ppl": 14.326477161274006,
      "eval_lambada_runtime": 9.5978,
      "eval_lambada_samples_per_second": 507.301,
      "step": 158000
    },
    {
      "activations/layer0_attention_weight_max": 15.22169017791748,
      "activations/layer0_attention_weight_min": -14.128365516662598,
      "activations/layer10_attention_weight_max": 38.5972900390625,
      "activations/layer10_attention_weight_min": -33.5919075012207,
      "activations/layer11_attention_weight_max": 30.67744255065918,
      "activations/layer11_attention_weight_min": -32.000335693359375,
      "activations/layer12_attention_weight_max": 25.979896545410156,
      "activations/layer12_attention_weight_min": -26.605587005615234,
      "activations/layer13_attention_weight_max": 40.15668869018555,
      "activations/layer13_attention_weight_min": -31.958200454711914,
      "activations/layer14_attention_weight_max": 39.50587844848633,
      "activations/layer14_attention_weight_min": -31.71548843383789,
      "activations/layer15_attention_weight_max": 37.284427642822266,
      "activations/layer15_attention_weight_min": -30.52566146850586,
      "activations/layer16_attention_weight_max": 31.513566970825195,
      "activations/layer16_attention_weight_min": -29.071205139160156,
      "activations/layer17_attention_weight_max": 59.38909912109375,
      "activations/layer17_attention_weight_min": -45.8135871887207,
      "activations/layer18_attention_weight_max": 52.58967971801758,
      "activations/layer18_attention_weight_min": -39.830692291259766,
      "activations/layer19_attention_weight_max": 22.68547821044922,
      "activations/layer19_attention_weight_min": -22.664033889770508,
      "activations/layer1_attention_weight_max": 16.64935302734375,
      "activations/layer1_attention_weight_min": -14.842937469482422,
      "activations/layer20_attention_weight_max": 24.622337341308594,
      "activations/layer20_attention_weight_min": -20.961368560791016,
      "activations/layer21_attention_weight_max": 42.65412521362305,
      "activations/layer21_attention_weight_min": -23.796220779418945,
      "activations/layer22_attention_weight_max": 33.86219787597656,
      "activations/layer22_attention_weight_min": -25.57228660583496,
      "activations/layer23_attention_weight_max": 38.5972900390625,
      "activations/layer23_attention_weight_min": -23.686803817749023,
      "activations/layer2_attention_weight_max": 31.329082489013672,
      "activations/layer2_attention_weight_min": -30.29150390625,
      "activations/layer3_attention_weight_max": 87.05540466308594,
      "activations/layer3_attention_weight_min": -93.29666900634766,
      "activations/layer4_attention_weight_max": 95.88021087646484,
      "activations/layer4_attention_weight_min": -90.29869842529297,
      "activations/layer5_attention_weight_max": 77.50611114501953,
      "activations/layer5_attention_weight_min": -79.23133087158203,
      "activations/layer6_attention_weight_max": 53.285179138183594,
      "activations/layer6_attention_weight_min": -53.26045227050781,
      "activations/layer7_attention_weight_max": 74.03128051757812,
      "activations/layer7_attention_weight_min": -70.00314331054688,
      "activations/layer8_attention_weight_max": 41.68406295776367,
      "activations/layer8_attention_weight_min": -44.96473693847656,
      "activations/layer9_attention_weight_max": 49.176631927490234,
      "activations/layer9_attention_weight_min": -46.3031120300293,
      "epoch": 9.18,
      "learning_rate": 9.170037878787878e-05,
      "loss": 2.8035,
      "step": 158050
    },
    {
      "activations/layer0_attention_weight_max": 15.65988540649414,
      "activations/layer0_attention_weight_min": -13.964019775390625,
      "activations/layer10_attention_weight_max": 36.24135208129883,
      "activations/layer10_attention_weight_min": -34.202880859375,
      "activations/layer11_attention_weight_max": 31.17768669128418,
      "activations/layer11_attention_weight_min": -31.243452072143555,
      "activations/layer12_attention_weight_max": 16.608930587768555,
      "activations/layer12_attention_weight_min": -32.50654220581055,
      "activations/layer13_attention_weight_max": 38.66770935058594,
      "activations/layer13_attention_weight_min": -29.034015655517578,
      "activations/layer14_attention_weight_max": 34.575496673583984,
      "activations/layer14_attention_weight_min": -29.954179763793945,
      "activations/layer15_attention_weight_max": 32.26866912841797,
      "activations/layer15_attention_weight_min": -29.343008041381836,
      "activations/layer16_attention_weight_max": 31.99578857421875,
      "activations/layer16_attention_weight_min": -26.945039749145508,
      "activations/layer17_attention_weight_max": 50.0048713684082,
      "activations/layer17_attention_weight_min": -41.551116943359375,
      "activations/layer18_attention_weight_max": 42.361228942871094,
      "activations/layer18_attention_weight_min": -37.96631622314453,
      "activations/layer19_attention_weight_max": 20.233083724975586,
      "activations/layer19_attention_weight_min": -22.100828170776367,
      "activations/layer1_attention_weight_max": 16.656648635864258,
      "activations/layer1_attention_weight_min": -13.61435604095459,
      "activations/layer20_attention_weight_max": 20.764881134033203,
      "activations/layer20_attention_weight_min": -22.740333557128906,
      "activations/layer21_attention_weight_max": 34.77579116821289,
      "activations/layer21_attention_weight_min": -22.783672332763672,
      "activations/layer22_attention_weight_max": 30.088186264038086,
      "activations/layer22_attention_weight_min": -25.68667221069336,
      "activations/layer23_attention_weight_max": 36.173805236816406,
      "activations/layer23_attention_weight_min": -24.73598861694336,
      "activations/layer2_attention_weight_max": 34.38569641113281,
      "activations/layer2_attention_weight_min": -31.836952209472656,
      "activations/layer3_attention_weight_max": 101.81046295166016,
      "activations/layer3_attention_weight_min": -103.03740692138672,
      "activations/layer4_attention_weight_max": 107.69840240478516,
      "activations/layer4_attention_weight_min": -99.3017349243164,
      "activations/layer5_attention_weight_max": 82.46154022216797,
      "activations/layer5_attention_weight_min": -78.79281616210938,
      "activations/layer6_attention_weight_max": 55.90666580200195,
      "activations/layer6_attention_weight_min": -52.26958084106445,
      "activations/layer7_attention_weight_max": 77.32221221923828,
      "activations/layer7_attention_weight_min": -73.3978042602539,
      "activations/layer8_attention_weight_max": 43.46834182739258,
      "activations/layer8_attention_weight_min": -45.183807373046875,
      "activations/layer9_attention_weight_max": 49.1218376159668,
      "activations/layer9_attention_weight_min": -46.492557525634766,
      "epoch": 9.19,
      "learning_rate": 9.16814393939394e-05,
      "loss": 2.78,
      "step": 158100
    },
    {
      "activations/layer0_attention_weight_max": 15.191315650939941,
      "activations/layer0_attention_weight_min": -14.143655776977539,
      "activations/layer10_attention_weight_max": 38.84144592285156,
      "activations/layer10_attention_weight_min": -36.04535675048828,
      "activations/layer11_attention_weight_max": 32.41080093383789,
      "activations/layer11_attention_weight_min": -30.388019561767578,
      "activations/layer12_attention_weight_max": 22.31245231628418,
      "activations/layer12_attention_weight_min": -26.717411041259766,
      "activations/layer13_attention_weight_max": 42.02756881713867,
      "activations/layer13_attention_weight_min": -33.50579071044922,
      "activations/layer14_attention_weight_max": 37.04534149169922,
      "activations/layer14_attention_weight_min": -29.953168869018555,
      "activations/layer15_attention_weight_max": 35.0303955078125,
      "activations/layer15_attention_weight_min": -29.520009994506836,
      "activations/layer16_attention_weight_max": 33.9842414855957,
      "activations/layer16_attention_weight_min": -28.657798767089844,
      "activations/layer17_attention_weight_max": 57.59251022338867,
      "activations/layer17_attention_weight_min": -41.030147552490234,
      "activations/layer18_attention_weight_max": 51.12678909301758,
      "activations/layer18_attention_weight_min": -36.229583740234375,
      "activations/layer19_attention_weight_max": 26.542875289916992,
      "activations/layer19_attention_weight_min": -22.14654541015625,
      "activations/layer1_attention_weight_max": 17.2989444732666,
      "activations/layer1_attention_weight_min": -15.473599433898926,
      "activations/layer20_attention_weight_max": 27.90245819091797,
      "activations/layer20_attention_weight_min": -22.633190155029297,
      "activations/layer21_attention_weight_max": 46.3187141418457,
      "activations/layer21_attention_weight_min": -24.998098373413086,
      "activations/layer22_attention_weight_max": 34.41798400878906,
      "activations/layer22_attention_weight_min": -26.476703643798828,
      "activations/layer23_attention_weight_max": 38.40650177001953,
      "activations/layer23_attention_weight_min": -23.118467330932617,
      "activations/layer2_attention_weight_max": 31.984731674194336,
      "activations/layer2_attention_weight_min": -30.536243438720703,
      "activations/layer3_attention_weight_max": 92.65882110595703,
      "activations/layer3_attention_weight_min": -94.29324340820312,
      "activations/layer4_attention_weight_max": 100.13005828857422,
      "activations/layer4_attention_weight_min": -96.5667495727539,
      "activations/layer5_attention_weight_max": 81.02998352050781,
      "activations/layer5_attention_weight_min": -84.72673034667969,
      "activations/layer6_attention_weight_max": 57.40239334106445,
      "activations/layer6_attention_weight_min": -58.90272521972656,
      "activations/layer7_attention_weight_max": 76.00463104248047,
      "activations/layer7_attention_weight_min": -81.61084747314453,
      "activations/layer8_attention_weight_max": 49.37223815917969,
      "activations/layer8_attention_weight_min": -49.422393798828125,
      "activations/layer9_attention_weight_max": 50.46857833862305,
      "activations/layer9_attention_weight_min": -55.510231018066406,
      "epoch": 9.19,
      "learning_rate": 9.166249999999999e-05,
      "loss": 2.7931,
      "step": 158150
    },
    {
      "activations/layer0_attention_weight_max": 15.97290325164795,
      "activations/layer0_attention_weight_min": -13.848219871520996,
      "activations/layer10_attention_weight_max": 45.0729866027832,
      "activations/layer10_attention_weight_min": -42.34650421142578,
      "activations/layer11_attention_weight_max": 39.622642517089844,
      "activations/layer11_attention_weight_min": -34.57270812988281,
      "activations/layer12_attention_weight_max": 18.845874786376953,
      "activations/layer12_attention_weight_min": -27.766075134277344,
      "activations/layer13_attention_weight_max": 48.442928314208984,
      "activations/layer13_attention_weight_min": -35.14861297607422,
      "activations/layer14_attention_weight_max": 47.33639144897461,
      "activations/layer14_attention_weight_min": -30.774805068969727,
      "activations/layer15_attention_weight_max": 40.13235855102539,
      "activations/layer15_attention_weight_min": -29.669696807861328,
      "activations/layer16_attention_weight_max": 33.327144622802734,
      "activations/layer16_attention_weight_min": -27.7490177154541,
      "activations/layer17_attention_weight_max": 52.570682525634766,
      "activations/layer17_attention_weight_min": -44.33744430541992,
      "activations/layer18_attention_weight_max": 45.84729766845703,
      "activations/layer18_attention_weight_min": -35.73106384277344,
      "activations/layer19_attention_weight_max": 20.656545639038086,
      "activations/layer19_attention_weight_min": -20.939006805419922,
      "activations/layer1_attention_weight_max": 17.321922302246094,
      "activations/layer1_attention_weight_min": -15.229872703552246,
      "activations/layer20_attention_weight_max": 24.29677391052246,
      "activations/layer20_attention_weight_min": -23.36473274230957,
      "activations/layer21_attention_weight_max": 51.79547119140625,
      "activations/layer21_attention_weight_min": -26.775041580200195,
      "activations/layer22_attention_weight_max": 29.65998649597168,
      "activations/layer22_attention_weight_min": -24.04129409790039,
      "activations/layer23_attention_weight_max": 35.45163345336914,
      "activations/layer23_attention_weight_min": -22.42336654663086,
      "activations/layer2_attention_weight_max": 34.052345275878906,
      "activations/layer2_attention_weight_min": -31.053373336791992,
      "activations/layer3_attention_weight_max": 95.25862121582031,
      "activations/layer3_attention_weight_min": -95.40802764892578,
      "activations/layer4_attention_weight_max": 103.76326751708984,
      "activations/layer4_attention_weight_min": -99.23665618896484,
      "activations/layer5_attention_weight_max": 87.63567352294922,
      "activations/layer5_attention_weight_min": -76.35076141357422,
      "activations/layer6_attention_weight_max": 59.80915451049805,
      "activations/layer6_attention_weight_min": -57.72020721435547,
      "activations/layer7_attention_weight_max": 89.2078857421875,
      "activations/layer7_attention_weight_min": -84.60093688964844,
      "activations/layer8_attention_weight_max": 54.862266540527344,
      "activations/layer8_attention_weight_min": -52.792964935302734,
      "activations/layer9_attention_weight_max": 64.35369110107422,
      "activations/layer9_attention_weight_min": -56.544063568115234,
      "epoch": 9.19,
      "learning_rate": 9.164393939393938e-05,
      "loss": 2.7876,
      "step": 158200
    },
    {
      "activations/layer0_attention_weight_max": 14.934611320495605,
      "activations/layer0_attention_weight_min": -13.48488712310791,
      "activations/layer10_attention_weight_max": 39.28084182739258,
      "activations/layer10_attention_weight_min": -38.52935028076172,
      "activations/layer11_attention_weight_max": 33.094825744628906,
      "activations/layer11_attention_weight_min": -31.96868896484375,
      "activations/layer12_attention_weight_max": 20.822463989257812,
      "activations/layer12_attention_weight_min": -25.14597511291504,
      "activations/layer13_attention_weight_max": 44.42007827758789,
      "activations/layer13_attention_weight_min": -29.537384033203125,
      "activations/layer14_attention_weight_max": 39.234703063964844,
      "activations/layer14_attention_weight_min": -31.00729751586914,
      "activations/layer15_attention_weight_max": 36.08883285522461,
      "activations/layer15_attention_weight_min": -28.772735595703125,
      "activations/layer16_attention_weight_max": 33.80799865722656,
      "activations/layer16_attention_weight_min": -29.215347290039062,
      "activations/layer17_attention_weight_max": 58.26185989379883,
      "activations/layer17_attention_weight_min": -42.1525993347168,
      "activations/layer18_attention_weight_max": 52.51152801513672,
      "activations/layer18_attention_weight_min": -35.864288330078125,
      "activations/layer19_attention_weight_max": 25.25331687927246,
      "activations/layer19_attention_weight_min": -21.601943969726562,
      "activations/layer1_attention_weight_max": 16.59965705871582,
      "activations/layer1_attention_weight_min": -15.202457427978516,
      "activations/layer20_attention_weight_max": 27.950105667114258,
      "activations/layer20_attention_weight_min": -20.134050369262695,
      "activations/layer21_attention_weight_max": 41.38679504394531,
      "activations/layer21_attention_weight_min": -22.87952995300293,
      "activations/layer22_attention_weight_max": 34.63650894165039,
      "activations/layer22_attention_weight_min": -25.458486557006836,
      "activations/layer23_attention_weight_max": 42.95721435546875,
      "activations/layer23_attention_weight_min": -23.18411636352539,
      "activations/layer2_attention_weight_max": 32.900508880615234,
      "activations/layer2_attention_weight_min": -31.223506927490234,
      "activations/layer3_attention_weight_max": 89.1512680053711,
      "activations/layer3_attention_weight_min": -93.63462829589844,
      "activations/layer4_attention_weight_max": 100.2383041381836,
      "activations/layer4_attention_weight_min": -98.87638092041016,
      "activations/layer5_attention_weight_max": 82.72411346435547,
      "activations/layer5_attention_weight_min": -82.96607208251953,
      "activations/layer6_attention_weight_max": 59.17432403564453,
      "activations/layer6_attention_weight_min": -54.8275260925293,
      "activations/layer7_attention_weight_max": 81.17951202392578,
      "activations/layer7_attention_weight_min": -80.64224243164062,
      "activations/layer8_attention_weight_max": 50.92966079711914,
      "activations/layer8_attention_weight_min": -51.79617691040039,
      "activations/layer9_attention_weight_max": 55.35834884643555,
      "activations/layer9_attention_weight_min": -52.82767868041992,
      "epoch": 9.2,
      "learning_rate": 9.162499999999999e-05,
      "loss": 2.796,
      "step": 158250
    },
    {
      "activations/layer0_attention_weight_max": 16.288822174072266,
      "activations/layer0_attention_weight_min": -13.721528053283691,
      "activations/layer10_attention_weight_max": 33.89310073852539,
      "activations/layer10_attention_weight_min": -33.22588348388672,
      "activations/layer11_attention_weight_max": 29.66374969482422,
      "activations/layer11_attention_weight_min": -30.95242691040039,
      "activations/layer12_attention_weight_max": 17.292015075683594,
      "activations/layer12_attention_weight_min": -26.0048770904541,
      "activations/layer13_attention_weight_max": 38.746517181396484,
      "activations/layer13_attention_weight_min": -30.191837310791016,
      "activations/layer14_attention_weight_max": 35.884273529052734,
      "activations/layer14_attention_weight_min": -31.761022567749023,
      "activations/layer15_attention_weight_max": 34.03695297241211,
      "activations/layer15_attention_weight_min": -31.123929977416992,
      "activations/layer16_attention_weight_max": 31.432132720947266,
      "activations/layer16_attention_weight_min": -28.20231819152832,
      "activations/layer17_attention_weight_max": 48.694461822509766,
      "activations/layer17_attention_weight_min": -40.104312896728516,
      "activations/layer18_attention_weight_max": 43.9846305847168,
      "activations/layer18_attention_weight_min": -35.5129508972168,
      "activations/layer19_attention_weight_max": 23.437673568725586,
      "activations/layer19_attention_weight_min": -21.22772216796875,
      "activations/layer1_attention_weight_max": 17.05550193786621,
      "activations/layer1_attention_weight_min": -16.250751495361328,
      "activations/layer20_attention_weight_max": 22.025774002075195,
      "activations/layer20_attention_weight_min": -21.47630500793457,
      "activations/layer21_attention_weight_max": 36.91792678833008,
      "activations/layer21_attention_weight_min": -22.33595848083496,
      "activations/layer22_attention_weight_max": 32.04066467285156,
      "activations/layer22_attention_weight_min": -25.54613494873047,
      "activations/layer23_attention_weight_max": 35.12212371826172,
      "activations/layer23_attention_weight_min": -23.364974975585938,
      "activations/layer2_attention_weight_max": 31.191648483276367,
      "activations/layer2_attention_weight_min": -28.39171600341797,
      "activations/layer3_attention_weight_max": 83.01236724853516,
      "activations/layer3_attention_weight_min": -91.4767074584961,
      "activations/layer4_attention_weight_max": 99.92361450195312,
      "activations/layer4_attention_weight_min": -95.4752426147461,
      "activations/layer5_attention_weight_max": 78.05905151367188,
      "activations/layer5_attention_weight_min": -77.03514099121094,
      "activations/layer6_attention_weight_max": 53.01117706298828,
      "activations/layer6_attention_weight_min": -51.82964324951172,
      "activations/layer7_attention_weight_max": 75.51203918457031,
      "activations/layer7_attention_weight_min": -76.01862335205078,
      "activations/layer8_attention_weight_max": 42.60099792480469,
      "activations/layer8_attention_weight_min": -45.95740509033203,
      "activations/layer9_attention_weight_max": 46.36073303222656,
      "activations/layer9_attention_weight_min": -46.90696716308594,
      "epoch": 9.2,
      "learning_rate": 9.16060606060606e-05,
      "loss": 2.7928,
      "step": 158300
    },
    {
      "activations/layer0_attention_weight_max": 15.447064399719238,
      "activations/layer0_attention_weight_min": -13.71923542022705,
      "activations/layer10_attention_weight_max": 34.85175704956055,
      "activations/layer10_attention_weight_min": -33.868324279785156,
      "activations/layer11_attention_weight_max": 30.59540367126465,
      "activations/layer11_attention_weight_min": -30.466793060302734,
      "activations/layer12_attention_weight_max": 20.8934268951416,
      "activations/layer12_attention_weight_min": -37.0375862121582,
      "activations/layer13_attention_weight_max": 50.836814880371094,
      "activations/layer13_attention_weight_min": -34.672176361083984,
      "activations/layer14_attention_weight_max": 52.568458557128906,
      "activations/layer14_attention_weight_min": -33.572418212890625,
      "activations/layer15_attention_weight_max": 40.303714752197266,
      "activations/layer15_attention_weight_min": -30.19965934753418,
      "activations/layer16_attention_weight_max": 30.572710037231445,
      "activations/layer16_attention_weight_min": -27.03810691833496,
      "activations/layer17_attention_weight_max": 52.24895477294922,
      "activations/layer17_attention_weight_min": -43.85923385620117,
      "activations/layer18_attention_weight_max": 45.75087356567383,
      "activations/layer18_attention_weight_min": -36.60870361328125,
      "activations/layer19_attention_weight_max": 23.70308494567871,
      "activations/layer19_attention_weight_min": -21.803953170776367,
      "activations/layer1_attention_weight_max": 16.351024627685547,
      "activations/layer1_attention_weight_min": -14.915300369262695,
      "activations/layer20_attention_weight_max": 22.254047393798828,
      "activations/layer20_attention_weight_min": -19.75640106201172,
      "activations/layer21_attention_weight_max": 38.19908142089844,
      "activations/layer21_attention_weight_min": -23.292333602905273,
      "activations/layer22_attention_weight_max": 31.290693283081055,
      "activations/layer22_attention_weight_min": -25.33485221862793,
      "activations/layer23_attention_weight_max": 35.641056060791016,
      "activations/layer23_attention_weight_min": -22.025930404663086,
      "activations/layer2_attention_weight_max": 29.272018432617188,
      "activations/layer2_attention_weight_min": -28.319866180419922,
      "activations/layer3_attention_weight_max": 89.13785552978516,
      "activations/layer3_attention_weight_min": -82.8582534790039,
      "activations/layer4_attention_weight_max": 98.36450958251953,
      "activations/layer4_attention_weight_min": -88.47277069091797,
      "activations/layer5_attention_weight_max": 82.35798645019531,
      "activations/layer5_attention_weight_min": -74.45709228515625,
      "activations/layer6_attention_weight_max": 53.60832214355469,
      "activations/layer6_attention_weight_min": -49.548728942871094,
      "activations/layer7_attention_weight_max": 80.31243133544922,
      "activations/layer7_attention_weight_min": -72.12084197998047,
      "activations/layer8_attention_weight_max": 44.4134407043457,
      "activations/layer8_attention_weight_min": -43.900413513183594,
      "activations/layer9_attention_weight_max": 55.46098709106445,
      "activations/layer9_attention_weight_min": -53.44901657104492,
      "epoch": 9.2,
      "learning_rate": 9.15871212121212e-05,
      "loss": 2.785,
      "step": 158350
    },
    {
      "activations/layer0_attention_weight_max": 16.429738998413086,
      "activations/layer0_attention_weight_min": -13.551567077636719,
      "activations/layer10_attention_weight_max": 39.94431686401367,
      "activations/layer10_attention_weight_min": -37.22669982910156,
      "activations/layer11_attention_weight_max": 33.916419982910156,
      "activations/layer11_attention_weight_min": -32.04866409301758,
      "activations/layer12_attention_weight_max": 19.534643173217773,
      "activations/layer12_attention_weight_min": -29.812536239624023,
      "activations/layer13_attention_weight_max": 40.419654846191406,
      "activations/layer13_attention_weight_min": -31.4346923828125,
      "activations/layer14_attention_weight_max": 38.206790924072266,
      "activations/layer14_attention_weight_min": -29.095623016357422,
      "activations/layer15_attention_weight_max": 36.790687561035156,
      "activations/layer15_attention_weight_min": -28.77503204345703,
      "activations/layer16_attention_weight_max": 31.67730712890625,
      "activations/layer16_attention_weight_min": -29.80217933654785,
      "activations/layer17_attention_weight_max": 57.91035461425781,
      "activations/layer17_attention_weight_min": -44.272308349609375,
      "activations/layer18_attention_weight_max": 51.11408996582031,
      "activations/layer18_attention_weight_min": -36.490447998046875,
      "activations/layer19_attention_weight_max": 25.78927993774414,
      "activations/layer19_attention_weight_min": -20.299240112304688,
      "activations/layer1_attention_weight_max": 17.944704055786133,
      "activations/layer1_attention_weight_min": -16.724458694458008,
      "activations/layer20_attention_weight_max": 24.41485023498535,
      "activations/layer20_attention_weight_min": -20.36671257019043,
      "activations/layer21_attention_weight_max": 38.9012336730957,
      "activations/layer21_attention_weight_min": -24.248449325561523,
      "activations/layer22_attention_weight_max": 32.73255920410156,
      "activations/layer22_attention_weight_min": -25.306476593017578,
      "activations/layer23_attention_weight_max": 40.7318115234375,
      "activations/layer23_attention_weight_min": -22.5841121673584,
      "activations/layer2_attention_weight_max": 33.987892150878906,
      "activations/layer2_attention_weight_min": -30.33810043334961,
      "activations/layer3_attention_weight_max": 96.2541275024414,
      "activations/layer3_attention_weight_min": -91.88616180419922,
      "activations/layer4_attention_weight_max": 99.47149658203125,
      "activations/layer4_attention_weight_min": -94.81786346435547,
      "activations/layer5_attention_weight_max": 80.44645690917969,
      "activations/layer5_attention_weight_min": -74.18201446533203,
      "activations/layer6_attention_weight_max": 55.32756042480469,
      "activations/layer6_attention_weight_min": -53.82658386230469,
      "activations/layer7_attention_weight_max": 79.07598876953125,
      "activations/layer7_attention_weight_min": -81.28195190429688,
      "activations/layer8_attention_weight_max": 46.04254913330078,
      "activations/layer8_attention_weight_min": -47.48518753051758,
      "activations/layer9_attention_weight_max": 58.221588134765625,
      "activations/layer9_attention_weight_min": -49.92567443847656,
      "epoch": 9.2,
      "learning_rate": 9.156818181818181e-05,
      "loss": 2.8085,
      "step": 158400
    },
    {
      "activations/layer0_attention_weight_max": 14.567919731140137,
      "activations/layer0_attention_weight_min": -13.79438304901123,
      "activations/layer10_attention_weight_max": 35.51539993286133,
      "activations/layer10_attention_weight_min": -35.12983703613281,
      "activations/layer11_attention_weight_max": 31.813133239746094,
      "activations/layer11_attention_weight_min": -32.159095764160156,
      "activations/layer12_attention_weight_max": 19.614002227783203,
      "activations/layer12_attention_weight_min": -24.64092445373535,
      "activations/layer13_attention_weight_max": 36.06477737426758,
      "activations/layer13_attention_weight_min": -27.592979431152344,
      "activations/layer14_attention_weight_max": 36.61648941040039,
      "activations/layer14_attention_weight_min": -28.484115600585938,
      "activations/layer15_attention_weight_max": 32.13743209838867,
      "activations/layer15_attention_weight_min": -27.505367279052734,
      "activations/layer16_attention_weight_max": 30.028533935546875,
      "activations/layer16_attention_weight_min": -27.63909912109375,
      "activations/layer17_attention_weight_max": 49.554229736328125,
      "activations/layer17_attention_weight_min": -42.264366149902344,
      "activations/layer18_attention_weight_max": 45.03105926513672,
      "activations/layer18_attention_weight_min": -36.70420455932617,
      "activations/layer19_attention_weight_max": 22.484365463256836,
      "activations/layer19_attention_weight_min": -20.57369041442871,
      "activations/layer1_attention_weight_max": 16.982057571411133,
      "activations/layer1_attention_weight_min": -15.750219345092773,
      "activations/layer20_attention_weight_max": 22.083660125732422,
      "activations/layer20_attention_weight_min": -24.36383628845215,
      "activations/layer21_attention_weight_max": 44.77523422241211,
      "activations/layer21_attention_weight_min": -25.585222244262695,
      "activations/layer22_attention_weight_max": 29.959716796875,
      "activations/layer22_attention_weight_min": -25.457944869995117,
      "activations/layer23_attention_weight_max": 38.24993896484375,
      "activations/layer23_attention_weight_min": -26.657329559326172,
      "activations/layer2_attention_weight_max": 30.971424102783203,
      "activations/layer2_attention_weight_min": -30.085350036621094,
      "activations/layer3_attention_weight_max": 90.02153778076172,
      "activations/layer3_attention_weight_min": -95.23292541503906,
      "activations/layer4_attention_weight_max": 99.62301635742188,
      "activations/layer4_attention_weight_min": -92.00597381591797,
      "activations/layer5_attention_weight_max": 81.16859436035156,
      "activations/layer5_attention_weight_min": -76.17927551269531,
      "activations/layer6_attention_weight_max": 54.81285858154297,
      "activations/layer6_attention_weight_min": -52.432518005371094,
      "activations/layer7_attention_weight_max": 76.64849090576172,
      "activations/layer7_attention_weight_min": -76.34357452392578,
      "activations/layer8_attention_weight_max": 45.84248733520508,
      "activations/layer8_attention_weight_min": -47.354278564453125,
      "activations/layer9_attention_weight_max": 58.15139389038086,
      "activations/layer9_attention_weight_min": -49.54854202270508,
      "epoch": 9.21,
      "learning_rate": 9.154924242424241e-05,
      "loss": 2.7973,
      "step": 158450
    },
    {
      "activations/layer0_attention_weight_max": 16.1452693939209,
      "activations/layer0_attention_weight_min": -13.901171684265137,
      "activations/layer10_attention_weight_max": 35.876808166503906,
      "activations/layer10_attention_weight_min": -35.61735534667969,
      "activations/layer11_attention_weight_max": 29.460264205932617,
      "activations/layer11_attention_weight_min": -32.45597839355469,
      "activations/layer12_attention_weight_max": 18.000123977661133,
      "activations/layer12_attention_weight_min": -25.05274200439453,
      "activations/layer13_attention_weight_max": 38.22674560546875,
      "activations/layer13_attention_weight_min": -30.30280113220215,
      "activations/layer14_attention_weight_max": 35.69720458984375,
      "activations/layer14_attention_weight_min": -31.007762908935547,
      "activations/layer15_attention_weight_max": 32.984092712402344,
      "activations/layer15_attention_weight_min": -29.00979232788086,
      "activations/layer16_attention_weight_max": 30.513643264770508,
      "activations/layer16_attention_weight_min": -27.324052810668945,
      "activations/layer17_attention_weight_max": 56.747615814208984,
      "activations/layer17_attention_weight_min": -43.283809661865234,
      "activations/layer18_attention_weight_max": 46.73904800415039,
      "activations/layer18_attention_weight_min": -36.7835807800293,
      "activations/layer19_attention_weight_max": 24.077186584472656,
      "activations/layer19_attention_weight_min": -20.461463928222656,
      "activations/layer1_attention_weight_max": 16.626272201538086,
      "activations/layer1_attention_weight_min": -15.099310874938965,
      "activations/layer20_attention_weight_max": 22.978193283081055,
      "activations/layer20_attention_weight_min": -21.437931060791016,
      "activations/layer21_attention_weight_max": 39.421241760253906,
      "activations/layer21_attention_weight_min": -22.294279098510742,
      "activations/layer22_attention_weight_max": 32.28168869018555,
      "activations/layer22_attention_weight_min": -26.14576530456543,
      "activations/layer23_attention_weight_max": 37.786956787109375,
      "activations/layer23_attention_weight_min": -23.698078155517578,
      "activations/layer2_attention_weight_max": 31.07209014892578,
      "activations/layer2_attention_weight_min": -30.746597290039062,
      "activations/layer3_attention_weight_max": 88.94667053222656,
      "activations/layer3_attention_weight_min": -92.2523422241211,
      "activations/layer4_attention_weight_max": 96.7250747680664,
      "activations/layer4_attention_weight_min": -89.86644744873047,
      "activations/layer5_attention_weight_max": 80.0806655883789,
      "activations/layer5_attention_weight_min": -79.35425567626953,
      "activations/layer6_attention_weight_max": 53.84157943725586,
      "activations/layer6_attention_weight_min": -52.09122085571289,
      "activations/layer7_attention_weight_max": 82.39141082763672,
      "activations/layer7_attention_weight_min": -69.75199127197266,
      "activations/layer8_attention_weight_max": 43.8871955871582,
      "activations/layer8_attention_weight_min": -43.986358642578125,
      "activations/layer9_attention_weight_max": 52.26870346069336,
      "activations/layer9_attention_weight_min": -47.3615837097168,
      "epoch": 9.21,
      "learning_rate": 9.153030303030303e-05,
      "loss": 2.7872,
      "step": 158500
    },
    {
      "activations/layer0_attention_weight_max": 16.41986846923828,
      "activations/layer0_attention_weight_min": -13.587265968322754,
      "activations/layer10_attention_weight_max": 36.460121154785156,
      "activations/layer10_attention_weight_min": -38.4276008605957,
      "activations/layer11_attention_weight_max": 31.204174041748047,
      "activations/layer11_attention_weight_min": -32.38496780395508,
      "activations/layer12_attention_weight_max": 17.297754287719727,
      "activations/layer12_attention_weight_min": -30.591699600219727,
      "activations/layer13_attention_weight_max": 36.45380783081055,
      "activations/layer13_attention_weight_min": -29.194805145263672,
      "activations/layer14_attention_weight_max": 34.27015686035156,
      "activations/layer14_attention_weight_min": -32.45900344848633,
      "activations/layer15_attention_weight_max": 30.09286880493164,
      "activations/layer15_attention_weight_min": -32.19578170776367,
      "activations/layer16_attention_weight_max": 27.73370361328125,
      "activations/layer16_attention_weight_min": -28.599285125732422,
      "activations/layer17_attention_weight_max": 49.57377624511719,
      "activations/layer17_attention_weight_min": -43.343502044677734,
      "activations/layer18_attention_weight_max": 41.6001091003418,
      "activations/layer18_attention_weight_min": -36.97486114501953,
      "activations/layer19_attention_weight_max": 22.070308685302734,
      "activations/layer19_attention_weight_min": -22.397748947143555,
      "activations/layer1_attention_weight_max": 16.81829071044922,
      "activations/layer1_attention_weight_min": -15.706754684448242,
      "activations/layer20_attention_weight_max": 20.883787155151367,
      "activations/layer20_attention_weight_min": -23.10895347595215,
      "activations/layer21_attention_weight_max": 35.741050720214844,
      "activations/layer21_attention_weight_min": -23.709556579589844,
      "activations/layer22_attention_weight_max": 29.95808982849121,
      "activations/layer22_attention_weight_min": -26.99313735961914,
      "activations/layer23_attention_weight_max": 35.686424255371094,
      "activations/layer23_attention_weight_min": -23.92902183532715,
      "activations/layer2_attention_weight_max": 31.684654235839844,
      "activations/layer2_attention_weight_min": -30.120147705078125,
      "activations/layer3_attention_weight_max": 91.61949157714844,
      "activations/layer3_attention_weight_min": -101.32566833496094,
      "activations/layer4_attention_weight_max": 98.84420013427734,
      "activations/layer4_attention_weight_min": -104.15196990966797,
      "activations/layer5_attention_weight_max": 82.76177215576172,
      "activations/layer5_attention_weight_min": -81.54524230957031,
      "activations/layer6_attention_weight_max": 56.772857666015625,
      "activations/layer6_attention_weight_min": -53.87404251098633,
      "activations/layer7_attention_weight_max": 78.67203521728516,
      "activations/layer7_attention_weight_min": -75.3570556640625,
      "activations/layer8_attention_weight_max": 46.09364700317383,
      "activations/layer8_attention_weight_min": -46.63620376586914,
      "activations/layer9_attention_weight_max": 48.92115783691406,
      "activations/layer9_attention_weight_min": -51.890594482421875,
      "epoch": 9.21,
      "learning_rate": 9.151136363636363e-05,
      "loss": 2.7998,
      "step": 158550
    },
    {
      "activations/layer0_attention_weight_max": 14.41310977935791,
      "activations/layer0_attention_weight_min": -14.056172370910645,
      "activations/layer10_attention_weight_max": 38.47272491455078,
      "activations/layer10_attention_weight_min": -35.70835494995117,
      "activations/layer11_attention_weight_max": 33.69816970825195,
      "activations/layer11_attention_weight_min": -32.255001068115234,
      "activations/layer12_attention_weight_max": 18.609968185424805,
      "activations/layer12_attention_weight_min": -27.761838912963867,
      "activations/layer13_attention_weight_max": 37.309608459472656,
      "activations/layer13_attention_weight_min": -29.14522933959961,
      "activations/layer14_attention_weight_max": 39.32459259033203,
      "activations/layer14_attention_weight_min": -32.80438232421875,
      "activations/layer15_attention_weight_max": 34.50283432006836,
      "activations/layer15_attention_weight_min": -30.114452362060547,
      "activations/layer16_attention_weight_max": 28.79155158996582,
      "activations/layer16_attention_weight_min": -28.27961540222168,
      "activations/layer17_attention_weight_max": 50.89483642578125,
      "activations/layer17_attention_weight_min": -42.705223083496094,
      "activations/layer18_attention_weight_max": 44.306488037109375,
      "activations/layer18_attention_weight_min": -37.915035247802734,
      "activations/layer19_attention_weight_max": 22.608915328979492,
      "activations/layer19_attention_weight_min": -21.270343780517578,
      "activations/layer1_attention_weight_max": 18.1289005279541,
      "activations/layer1_attention_weight_min": -15.736838340759277,
      "activations/layer20_attention_weight_max": 20.62247657775879,
      "activations/layer20_attention_weight_min": -21.337133407592773,
      "activations/layer21_attention_weight_max": 41.73042678833008,
      "activations/layer21_attention_weight_min": -23.00018882751465,
      "activations/layer22_attention_weight_max": 29.39777183532715,
      "activations/layer22_attention_weight_min": -29.16735076904297,
      "activations/layer23_attention_weight_max": 34.897403717041016,
      "activations/layer23_attention_weight_min": -25.402599334716797,
      "activations/layer2_attention_weight_max": 32.82372283935547,
      "activations/layer2_attention_weight_min": -31.20833969116211,
      "activations/layer3_attention_weight_max": 94.14091491699219,
      "activations/layer3_attention_weight_min": -95.6690902709961,
      "activations/layer4_attention_weight_max": 105.34935760498047,
      "activations/layer4_attention_weight_min": -100.9094009399414,
      "activations/layer5_attention_weight_max": 82.98789978027344,
      "activations/layer5_attention_weight_min": -79.38224792480469,
      "activations/layer6_attention_weight_max": 52.71532440185547,
      "activations/layer6_attention_weight_min": -52.84221649169922,
      "activations/layer7_attention_weight_max": 78.30609893798828,
      "activations/layer7_attention_weight_min": -70.94535064697266,
      "activations/layer8_attention_weight_max": 45.13106918334961,
      "activations/layer8_attention_weight_min": -44.451141357421875,
      "activations/layer9_attention_weight_max": 48.612728118896484,
      "activations/layer9_attention_weight_min": -48.27980422973633,
      "epoch": 9.22,
      "learning_rate": 9.149242424242423e-05,
      "loss": 2.8066,
      "step": 158600
    },
    {
      "activations/layer0_attention_weight_max": 14.778433799743652,
      "activations/layer0_attention_weight_min": -13.72191047668457,
      "activations/layer10_attention_weight_max": 37.00215530395508,
      "activations/layer10_attention_weight_min": -34.511207580566406,
      "activations/layer11_attention_weight_max": 31.55463409423828,
      "activations/layer11_attention_weight_min": -31.169124603271484,
      "activations/layer12_attention_weight_max": 18.033422470092773,
      "activations/layer12_attention_weight_min": -27.985679626464844,
      "activations/layer13_attention_weight_max": 40.34904479980469,
      "activations/layer13_attention_weight_min": -27.968263626098633,
      "activations/layer14_attention_weight_max": 54.542171478271484,
      "activations/layer14_attention_weight_min": -32.453269958496094,
      "activations/layer15_attention_weight_max": 42.63080596923828,
      "activations/layer15_attention_weight_min": -28.89813232421875,
      "activations/layer16_attention_weight_max": 35.4139404296875,
      "activations/layer16_attention_weight_min": -28.432828903198242,
      "activations/layer17_attention_weight_max": 49.20304489135742,
      "activations/layer17_attention_weight_min": -42.86609649658203,
      "activations/layer18_attention_weight_max": 46.86153793334961,
      "activations/layer18_attention_weight_min": -36.54753112792969,
      "activations/layer19_attention_weight_max": 23.43866539001465,
      "activations/layer19_attention_weight_min": -21.848134994506836,
      "activations/layer1_attention_weight_max": 16.080114364624023,
      "activations/layer1_attention_weight_min": -14.780930519104004,
      "activations/layer20_attention_weight_max": 22.71220588684082,
      "activations/layer20_attention_weight_min": -22.3240966796875,
      "activations/layer21_attention_weight_max": 37.51715087890625,
      "activations/layer21_attention_weight_min": -24.475616455078125,
      "activations/layer22_attention_weight_max": 33.21247482299805,
      "activations/layer22_attention_weight_min": -25.90142250061035,
      "activations/layer23_attention_weight_max": 38.56724548339844,
      "activations/layer23_attention_weight_min": -23.26819610595703,
      "activations/layer2_attention_weight_max": 30.9397029876709,
      "activations/layer2_attention_weight_min": -28.694992065429688,
      "activations/layer3_attention_weight_max": 90.7234115600586,
      "activations/layer3_attention_weight_min": -88.70126342773438,
      "activations/layer4_attention_weight_max": 100.28877258300781,
      "activations/layer4_attention_weight_min": -94.7290267944336,
      "activations/layer5_attention_weight_max": 80.22369384765625,
      "activations/layer5_attention_weight_min": -75.1421890258789,
      "activations/layer6_attention_weight_max": 53.703224182128906,
      "activations/layer6_attention_weight_min": -52.1475715637207,
      "activations/layer7_attention_weight_max": 79.41232299804688,
      "activations/layer7_attention_weight_min": -74.96227264404297,
      "activations/layer8_attention_weight_max": 46.02695846557617,
      "activations/layer8_attention_weight_min": -46.38470458984375,
      "activations/layer9_attention_weight_max": 53.141395568847656,
      "activations/layer9_attention_weight_min": -49.205902099609375,
      "epoch": 9.22,
      "learning_rate": 9.147348484848484e-05,
      "loss": 2.7984,
      "step": 158650
    },
    {
      "activations/layer0_attention_weight_max": 14.355547904968262,
      "activations/layer0_attention_weight_min": -13.488008499145508,
      "activations/layer10_attention_weight_max": 35.477787017822266,
      "activations/layer10_attention_weight_min": -34.26694869995117,
      "activations/layer11_attention_weight_max": 29.22498893737793,
      "activations/layer11_attention_weight_min": -31.87447166442871,
      "activations/layer12_attention_weight_max": 19.108747482299805,
      "activations/layer12_attention_weight_min": -24.698848724365234,
      "activations/layer13_attention_weight_max": 40.142486572265625,
      "activations/layer13_attention_weight_min": -28.7276611328125,
      "activations/layer14_attention_weight_max": 45.60527801513672,
      "activations/layer14_attention_weight_min": -31.246810913085938,
      "activations/layer15_attention_weight_max": 33.71129608154297,
      "activations/layer15_attention_weight_min": -30.33490562438965,
      "activations/layer16_attention_weight_max": 31.62294578552246,
      "activations/layer16_attention_weight_min": -28.98259735107422,
      "activations/layer17_attention_weight_max": 51.87617874145508,
      "activations/layer17_attention_weight_min": -45.576416015625,
      "activations/layer18_attention_weight_max": 47.85821533203125,
      "activations/layer18_attention_weight_min": -38.57445526123047,
      "activations/layer19_attention_weight_max": 20.59625816345215,
      "activations/layer19_attention_weight_min": -21.720314025878906,
      "activations/layer1_attention_weight_max": 17.825550079345703,
      "activations/layer1_attention_weight_min": -15.207284927368164,
      "activations/layer20_attention_weight_max": 22.54169273376465,
      "activations/layer20_attention_weight_min": -23.24457359313965,
      "activations/layer21_attention_weight_max": 43.5944938659668,
      "activations/layer21_attention_weight_min": -24.403629302978516,
      "activations/layer22_attention_weight_max": 32.1348762512207,
      "activations/layer22_attention_weight_min": -25.03234100341797,
      "activations/layer23_attention_weight_max": 36.78166961669922,
      "activations/layer23_attention_weight_min": -29.306564331054688,
      "activations/layer2_attention_weight_max": 32.443824768066406,
      "activations/layer2_attention_weight_min": -29.7620849609375,
      "activations/layer3_attention_weight_max": 86.80284118652344,
      "activations/layer3_attention_weight_min": -92.57977294921875,
      "activations/layer4_attention_weight_max": 100.15412139892578,
      "activations/layer4_attention_weight_min": -94.94499969482422,
      "activations/layer5_attention_weight_max": 79.59707641601562,
      "activations/layer5_attention_weight_min": -74.24456787109375,
      "activations/layer6_attention_weight_max": 55.94910430908203,
      "activations/layer6_attention_weight_min": -53.66962814331055,
      "activations/layer7_attention_weight_max": 79.8752212524414,
      "activations/layer7_attention_weight_min": -74.67630767822266,
      "activations/layer8_attention_weight_max": 42.8953742980957,
      "activations/layer8_attention_weight_min": -48.95796203613281,
      "activations/layer9_attention_weight_max": 50.86543273925781,
      "activations/layer9_attention_weight_min": -51.52069091796875,
      "epoch": 9.22,
      "learning_rate": 9.145454545454546e-05,
      "loss": 2.7774,
      "step": 158700
    },
    {
      "activations/layer0_attention_weight_max": 15.313796997070312,
      "activations/layer0_attention_weight_min": -13.55100154876709,
      "activations/layer10_attention_weight_max": 37.43043899536133,
      "activations/layer10_attention_weight_min": -35.58292770385742,
      "activations/layer11_attention_weight_max": 30.12991714477539,
      "activations/layer11_attention_weight_min": -32.911231994628906,
      "activations/layer12_attention_weight_max": 18.068695068359375,
      "activations/layer12_attention_weight_min": -23.62152099609375,
      "activations/layer13_attention_weight_max": 40.46715545654297,
      "activations/layer13_attention_weight_min": -31.59642219543457,
      "activations/layer14_attention_weight_max": 34.585655212402344,
      "activations/layer14_attention_weight_min": -32.5235481262207,
      "activations/layer15_attention_weight_max": 34.918792724609375,
      "activations/layer15_attention_weight_min": -32.60878372192383,
      "activations/layer16_attention_weight_max": 30.92896270751953,
      "activations/layer16_attention_weight_min": -28.4288330078125,
      "activations/layer17_attention_weight_max": 50.23527908325195,
      "activations/layer17_attention_weight_min": -43.00160598754883,
      "activations/layer18_attention_weight_max": 42.94828796386719,
      "activations/layer18_attention_weight_min": -35.26001739501953,
      "activations/layer19_attention_weight_max": 21.674314498901367,
      "activations/layer19_attention_weight_min": -19.807193756103516,
      "activations/layer1_attention_weight_max": 16.740137100219727,
      "activations/layer1_attention_weight_min": -14.561091423034668,
      "activations/layer20_attention_weight_max": 22.504959106445312,
      "activations/layer20_attention_weight_min": -22.117897033691406,
      "activations/layer21_attention_weight_max": 35.4962043762207,
      "activations/layer21_attention_weight_min": -20.71255874633789,
      "activations/layer22_attention_weight_max": 32.51498031616211,
      "activations/layer22_attention_weight_min": -25.85805320739746,
      "activations/layer23_attention_weight_max": 32.05012130737305,
      "activations/layer23_attention_weight_min": -23.0822811126709,
      "activations/layer2_attention_weight_max": 30.716026306152344,
      "activations/layer2_attention_weight_min": -31.906705856323242,
      "activations/layer3_attention_weight_max": 90.83026885986328,
      "activations/layer3_attention_weight_min": -96.72712707519531,
      "activations/layer4_attention_weight_max": 99.0511703491211,
      "activations/layer4_attention_weight_min": -96.93653869628906,
      "activations/layer5_attention_weight_max": 78.73606872558594,
      "activations/layer5_attention_weight_min": -79.88751983642578,
      "activations/layer6_attention_weight_max": 54.18838882446289,
      "activations/layer6_attention_weight_min": -54.24179458618164,
      "activations/layer7_attention_weight_max": 77.35353088378906,
      "activations/layer7_attention_weight_min": -73.72100830078125,
      "activations/layer8_attention_weight_max": 46.02320098876953,
      "activations/layer8_attention_weight_min": -46.851409912109375,
      "activations/layer9_attention_weight_max": 49.92441940307617,
      "activations/layer9_attention_weight_min": -47.7107048034668,
      "epoch": 9.22,
      "learning_rate": 9.143560606060605e-05,
      "loss": 2.7771,
      "step": 158750
    },
    {
      "activations/layer0_attention_weight_max": 14.835800170898438,
      "activations/layer0_attention_weight_min": -13.500332832336426,
      "activations/layer10_attention_weight_max": 35.29289627075195,
      "activations/layer10_attention_weight_min": -33.668434143066406,
      "activations/layer11_attention_weight_max": 32.4346923828125,
      "activations/layer11_attention_weight_min": -30.680822372436523,
      "activations/layer12_attention_weight_max": 18.138660430908203,
      "activations/layer12_attention_weight_min": -24.998672485351562,
      "activations/layer13_attention_weight_max": 38.573448181152344,
      "activations/layer13_attention_weight_min": -32.98705291748047,
      "activations/layer14_attention_weight_max": 45.60771179199219,
      "activations/layer14_attention_weight_min": -31.360530853271484,
      "activations/layer15_attention_weight_max": 35.45161819458008,
      "activations/layer15_attention_weight_min": -28.870010375976562,
      "activations/layer16_attention_weight_max": 31.530452728271484,
      "activations/layer16_attention_weight_min": -28.683242797851562,
      "activations/layer17_attention_weight_max": 57.92540740966797,
      "activations/layer17_attention_weight_min": -42.58418273925781,
      "activations/layer18_attention_weight_max": 49.9554557800293,
      "activations/layer18_attention_weight_min": -39.34041213989258,
      "activations/layer19_attention_weight_max": 22.3858699798584,
      "activations/layer19_attention_weight_min": -22.905990600585938,
      "activations/layer1_attention_weight_max": 19.064424514770508,
      "activations/layer1_attention_weight_min": -15.751508712768555,
      "activations/layer20_attention_weight_max": 21.991008758544922,
      "activations/layer20_attention_weight_min": -22.528600692749023,
      "activations/layer21_attention_weight_max": 39.7635498046875,
      "activations/layer21_attention_weight_min": -23.2844295501709,
      "activations/layer22_attention_weight_max": 33.68813705444336,
      "activations/layer22_attention_weight_min": -24.760814666748047,
      "activations/layer23_attention_weight_max": 35.17751693725586,
      "activations/layer23_attention_weight_min": -22.06800079345703,
      "activations/layer2_attention_weight_max": 30.7357234954834,
      "activations/layer2_attention_weight_min": -29.697553634643555,
      "activations/layer3_attention_weight_max": 86.28162384033203,
      "activations/layer3_attention_weight_min": -89.1627197265625,
      "activations/layer4_attention_weight_max": 94.55091094970703,
      "activations/layer4_attention_weight_min": -91.054443359375,
      "activations/layer5_attention_weight_max": 77.16429138183594,
      "activations/layer5_attention_weight_min": -75.66059112548828,
      "activations/layer6_attention_weight_max": 54.06571960449219,
      "activations/layer6_attention_weight_min": -51.202232360839844,
      "activations/layer7_attention_weight_max": 76.25922393798828,
      "activations/layer7_attention_weight_min": -74.655029296875,
      "activations/layer8_attention_weight_max": 43.413238525390625,
      "activations/layer8_attention_weight_min": -43.589115142822266,
      "activations/layer9_attention_weight_max": 48.04362869262695,
      "activations/layer9_attention_weight_min": -51.28231430053711,
      "epoch": 9.23,
      "learning_rate": 9.141666666666666e-05,
      "loss": 2.7875,
      "step": 158800
    },
    {
      "activations/layer0_attention_weight_max": 15.67076587677002,
      "activations/layer0_attention_weight_min": -14.116490364074707,
      "activations/layer10_attention_weight_max": 46.06334686279297,
      "activations/layer10_attention_weight_min": -38.482086181640625,
      "activations/layer11_attention_weight_max": 38.72170639038086,
      "activations/layer11_attention_weight_min": -33.927276611328125,
      "activations/layer12_attention_weight_max": 26.428552627563477,
      "activations/layer12_attention_weight_min": -25.245927810668945,
      "activations/layer13_attention_weight_max": 46.28938293457031,
      "activations/layer13_attention_weight_min": -31.688282012939453,
      "activations/layer14_attention_weight_max": 40.643375396728516,
      "activations/layer14_attention_weight_min": -29.9284725189209,
      "activations/layer15_attention_weight_max": 35.17141342163086,
      "activations/layer15_attention_weight_min": -28.78974151611328,
      "activations/layer16_attention_weight_max": 32.17903137207031,
      "activations/layer16_attention_weight_min": -27.527660369873047,
      "activations/layer17_attention_weight_max": 57.2883186340332,
      "activations/layer17_attention_weight_min": -43.365509033203125,
      "activations/layer18_attention_weight_max": 48.22974395751953,
      "activations/layer18_attention_weight_min": -36.764461517333984,
      "activations/layer19_attention_weight_max": 25.99338722229004,
      "activations/layer19_attention_weight_min": -23.329864501953125,
      "activations/layer1_attention_weight_max": 17.067150115966797,
      "activations/layer1_attention_weight_min": -16.688966751098633,
      "activations/layer20_attention_weight_max": 25.475839614868164,
      "activations/layer20_attention_weight_min": -20.03236198425293,
      "activations/layer21_attention_weight_max": 41.65974807739258,
      "activations/layer21_attention_weight_min": -25.730186462402344,
      "activations/layer22_attention_weight_max": 33.145042419433594,
      "activations/layer22_attention_weight_min": -26.582843780517578,
      "activations/layer23_attention_weight_max": 38.92484664916992,
      "activations/layer23_attention_weight_min": -22.27010726928711,
      "activations/layer2_attention_weight_max": 31.825063705444336,
      "activations/layer2_attention_weight_min": -29.4619197845459,
      "activations/layer3_attention_weight_max": 89.07152557373047,
      "activations/layer3_attention_weight_min": -85.50414276123047,
      "activations/layer4_attention_weight_max": 98.2381820678711,
      "activations/layer4_attention_weight_min": -92.48064422607422,
      "activations/layer5_attention_weight_max": 83.73575592041016,
      "activations/layer5_attention_weight_min": -77.18672180175781,
      "activations/layer6_attention_weight_max": 56.24904251098633,
      "activations/layer6_attention_weight_min": -53.372989654541016,
      "activations/layer7_attention_weight_max": 82.0677719116211,
      "activations/layer7_attention_weight_min": -75.36245727539062,
      "activations/layer8_attention_weight_max": 47.557159423828125,
      "activations/layer8_attention_weight_min": -50.44831466674805,
      "activations/layer9_attention_weight_max": 53.13393020629883,
      "activations/layer9_attention_weight_min": -50.42072296142578,
      "epoch": 9.23,
      "learning_rate": 9.139772727272728e-05,
      "loss": 2.8055,
      "step": 158850
    },
    {
      "activations/layer0_attention_weight_max": 15.146987915039062,
      "activations/layer0_attention_weight_min": -13.70855712890625,
      "activations/layer10_attention_weight_max": 41.8752326965332,
      "activations/layer10_attention_weight_min": -38.56398010253906,
      "activations/layer11_attention_weight_max": 37.893829345703125,
      "activations/layer11_attention_weight_min": -34.48276901245117,
      "activations/layer12_attention_weight_max": 19.850461959838867,
      "activations/layer12_attention_weight_min": -24.779727935791016,
      "activations/layer13_attention_weight_max": 43.445064544677734,
      "activations/layer13_attention_weight_min": -31.59379005432129,
      "activations/layer14_attention_weight_max": 58.155189514160156,
      "activations/layer14_attention_weight_min": -31.0152530670166,
      "activations/layer15_attention_weight_max": 42.3783073425293,
      "activations/layer15_attention_weight_min": -29.432266235351562,
      "activations/layer16_attention_weight_max": 29.743927001953125,
      "activations/layer16_attention_weight_min": -28.11737632751465,
      "activations/layer17_attention_weight_max": 53.77238082885742,
      "activations/layer17_attention_weight_min": -41.62077331542969,
      "activations/layer18_attention_weight_max": 46.97966003417969,
      "activations/layer18_attention_weight_min": -36.808387756347656,
      "activations/layer19_attention_weight_max": 23.025588989257812,
      "activations/layer19_attention_weight_min": -22.8459529876709,
      "activations/layer1_attention_weight_max": 18.44099998474121,
      "activations/layer1_attention_weight_min": -16.204452514648438,
      "activations/layer20_attention_weight_max": 25.002782821655273,
      "activations/layer20_attention_weight_min": -22.167037963867188,
      "activations/layer21_attention_weight_max": 38.72676467895508,
      "activations/layer21_attention_weight_min": -22.75710105895996,
      "activations/layer22_attention_weight_max": 29.64454460144043,
      "activations/layer22_attention_weight_min": -23.58356285095215,
      "activations/layer23_attention_weight_max": 35.971378326416016,
      "activations/layer23_attention_weight_min": -22.044742584228516,
      "activations/layer2_attention_weight_max": 33.47042465209961,
      "activations/layer2_attention_weight_min": -31.715879440307617,
      "activations/layer3_attention_weight_max": 93.32723999023438,
      "activations/layer3_attention_weight_min": -97.08472442626953,
      "activations/layer4_attention_weight_max": 105.3403549194336,
      "activations/layer4_attention_weight_min": -99.41289520263672,
      "activations/layer5_attention_weight_max": 84.40733337402344,
      "activations/layer5_attention_weight_min": -80.80506896972656,
      "activations/layer6_attention_weight_max": 54.72370529174805,
      "activations/layer6_attention_weight_min": -55.560157775878906,
      "activations/layer7_attention_weight_max": 84.9502182006836,
      "activations/layer7_attention_weight_min": -75.10938262939453,
      "activations/layer8_attention_weight_max": 51.30033874511719,
      "activations/layer8_attention_weight_min": -49.366981506347656,
      "activations/layer9_attention_weight_max": 63.42489242553711,
      "activations/layer9_attention_weight_min": -54.45722579956055,
      "epoch": 9.23,
      "learning_rate": 9.137878787878786e-05,
      "loss": 2.7785,
      "step": 158900
    },
    {
      "activations/layer0_attention_weight_max": 14.739182472229004,
      "activations/layer0_attention_weight_min": -13.920361518859863,
      "activations/layer10_attention_weight_max": 39.00648880004883,
      "activations/layer10_attention_weight_min": -36.00136184692383,
      "activations/layer11_attention_weight_max": 31.768753051757812,
      "activations/layer11_attention_weight_min": -33.84001159667969,
      "activations/layer12_attention_weight_max": 18.471309661865234,
      "activations/layer12_attention_weight_min": -22.313926696777344,
      "activations/layer13_attention_weight_max": 43.783626556396484,
      "activations/layer13_attention_weight_min": -29.448535919189453,
      "activations/layer14_attention_weight_max": 37.419010162353516,
      "activations/layer14_attention_weight_min": -29.982038497924805,
      "activations/layer15_attention_weight_max": 38.36896896362305,
      "activations/layer15_attention_weight_min": -28.988309860229492,
      "activations/layer16_attention_weight_max": 31.772558212280273,
      "activations/layer16_attention_weight_min": -27.639257431030273,
      "activations/layer17_attention_weight_max": 53.36759948730469,
      "activations/layer17_attention_weight_min": -46.606319427490234,
      "activations/layer18_attention_weight_max": 43.776248931884766,
      "activations/layer18_attention_weight_min": -38.098548889160156,
      "activations/layer19_attention_weight_max": 22.012983322143555,
      "activations/layer19_attention_weight_min": -24.3248348236084,
      "activations/layer1_attention_weight_max": 16.659086227416992,
      "activations/layer1_attention_weight_min": -15.587301254272461,
      "activations/layer20_attention_weight_max": 21.616710662841797,
      "activations/layer20_attention_weight_min": -21.568689346313477,
      "activations/layer21_attention_weight_max": 36.434181213378906,
      "activations/layer21_attention_weight_min": -23.767120361328125,
      "activations/layer22_attention_weight_max": 31.735931396484375,
      "activations/layer22_attention_weight_min": -28.68155860900879,
      "activations/layer23_attention_weight_max": 39.25944900512695,
      "activations/layer23_attention_weight_min": -26.763639450073242,
      "activations/layer2_attention_weight_max": 30.9530029296875,
      "activations/layer2_attention_weight_min": -30.625259399414062,
      "activations/layer3_attention_weight_max": 87.10234832763672,
      "activations/layer3_attention_weight_min": -90.57222747802734,
      "activations/layer4_attention_weight_max": 99.50956726074219,
      "activations/layer4_attention_weight_min": -93.39692687988281,
      "activations/layer5_attention_weight_max": 80.63864135742188,
      "activations/layer5_attention_weight_min": -76.90675354003906,
      "activations/layer6_attention_weight_max": 56.516090393066406,
      "activations/layer6_attention_weight_min": -55.287296295166016,
      "activations/layer7_attention_weight_max": 77.01010131835938,
      "activations/layer7_attention_weight_min": -71.41858673095703,
      "activations/layer8_attention_weight_max": 44.431053161621094,
      "activations/layer8_attention_weight_min": -47.37841033935547,
      "activations/layer9_attention_weight_max": 47.03105926513672,
      "activations/layer9_attention_weight_min": -48.1894645690918,
      "epoch": 9.24,
      "learning_rate": 9.135984848484848e-05,
      "loss": 2.8118,
      "step": 158950
    },
    {
      "activations/layer0_attention_weight_max": 14.41427993774414,
      "activations/layer0_attention_weight_min": -13.806296348571777,
      "activations/layer10_attention_weight_max": 36.41035461425781,
      "activations/layer10_attention_weight_min": -35.147315979003906,
      "activations/layer11_attention_weight_max": 30.826656341552734,
      "activations/layer11_attention_weight_min": -33.61326217651367,
      "activations/layer12_attention_weight_max": 22.153976440429688,
      "activations/layer12_attention_weight_min": -27.10197639465332,
      "activations/layer13_attention_weight_max": 46.99411392211914,
      "activations/layer13_attention_weight_min": -35.64760971069336,
      "activations/layer14_attention_weight_max": 38.053340911865234,
      "activations/layer14_attention_weight_min": -28.978328704833984,
      "activations/layer15_attention_weight_max": 35.32085037231445,
      "activations/layer15_attention_weight_min": -29.723413467407227,
      "activations/layer16_attention_weight_max": 32.6646614074707,
      "activations/layer16_attention_weight_min": -27.325223922729492,
      "activations/layer17_attention_weight_max": 55.7232666015625,
      "activations/layer17_attention_weight_min": -43.11215591430664,
      "activations/layer18_attention_weight_max": 48.944278717041016,
      "activations/layer18_attention_weight_min": -34.789390563964844,
      "activations/layer19_attention_weight_max": 21.178152084350586,
      "activations/layer19_attention_weight_min": -21.183034896850586,
      "activations/layer1_attention_weight_max": 17.73068618774414,
      "activations/layer1_attention_weight_min": -15.792567253112793,
      "activations/layer20_attention_weight_max": 22.310644149780273,
      "activations/layer20_attention_weight_min": -24.17254638671875,
      "activations/layer21_attention_weight_max": 33.05718994140625,
      "activations/layer21_attention_weight_min": -22.507160186767578,
      "activations/layer22_attention_weight_max": 33.313316345214844,
      "activations/layer22_attention_weight_min": -25.89186668395996,
      "activations/layer23_attention_weight_max": 40.61448287963867,
      "activations/layer23_attention_weight_min": -21.59317398071289,
      "activations/layer2_attention_weight_max": 30.31629180908203,
      "activations/layer2_attention_weight_min": -30.39081573486328,
      "activations/layer3_attention_weight_max": 87.19770050048828,
      "activations/layer3_attention_weight_min": -91.775146484375,
      "activations/layer4_attention_weight_max": 100.5935287475586,
      "activations/layer4_attention_weight_min": -97.69243621826172,
      "activations/layer5_attention_weight_max": 82.30276489257812,
      "activations/layer5_attention_weight_min": -80.90423583984375,
      "activations/layer6_attention_weight_max": 55.54845428466797,
      "activations/layer6_attention_weight_min": -53.79610824584961,
      "activations/layer7_attention_weight_max": 76.55471801757812,
      "activations/layer7_attention_weight_min": -75.14885711669922,
      "activations/layer8_attention_weight_max": 47.27327346801758,
      "activations/layer8_attention_weight_min": -51.51959991455078,
      "activations/layer9_attention_weight_max": 52.70996856689453,
      "activations/layer9_attention_weight_min": -52.77924346923828,
      "epoch": 9.24,
      "learning_rate": 9.134090909090908e-05,
      "loss": 2.8029,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_loss": 2.75,
      "eval_runtime": 8.515,
      "eval_samples_per_second": 504.289,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.515,
      "eval_openwebtext_samples_per_second": 504.289,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 2.0009,
      "eval_wikitext_samples_per_second": 227.892,
      "step": 159000
    },
    {
      "epoch": 9.24,
      "eval_lambada_loss": 2.640625,
      "eval_lambada_ppl": 14.021964597512564,
      "eval_lambada_runtime": 9.5979,
      "eval_lambada_samples_per_second": 507.297,
      "step": 159000
    },
    {
      "activations/layer0_attention_weight_max": 15.025551795959473,
      "activations/layer0_attention_weight_min": -13.808392524719238,
      "activations/layer10_attention_weight_max": 37.60491943359375,
      "activations/layer10_attention_weight_min": -36.50612258911133,
      "activations/layer11_attention_weight_max": 31.2978572845459,
      "activations/layer11_attention_weight_min": -32.90473937988281,
      "activations/layer12_attention_weight_max": 17.141366958618164,
      "activations/layer12_attention_weight_min": -25.40955352783203,
      "activations/layer13_attention_weight_max": 39.12314987182617,
      "activations/layer13_attention_weight_min": -30.200698852539062,
      "activations/layer14_attention_weight_max": 37.09897232055664,
      "activations/layer14_attention_weight_min": -31.057159423828125,
      "activations/layer15_attention_weight_max": 38.74714660644531,
      "activations/layer15_attention_weight_min": -29.4467830657959,
      "activations/layer16_attention_weight_max": 30.134754180908203,
      "activations/layer16_attention_weight_min": -28.349105834960938,
      "activations/layer17_attention_weight_max": 54.69536590576172,
      "activations/layer17_attention_weight_min": -43.425025939941406,
      "activations/layer18_attention_weight_max": 47.2666130065918,
      "activations/layer18_attention_weight_min": -35.962398529052734,
      "activations/layer19_attention_weight_max": 23.1230525970459,
      "activations/layer19_attention_weight_min": -22.53676414489746,
      "activations/layer1_attention_weight_max": 16.215862274169922,
      "activations/layer1_attention_weight_min": -15.2090482711792,
      "activations/layer20_attention_weight_max": 21.636951446533203,
      "activations/layer20_attention_weight_min": -22.314783096313477,
      "activations/layer21_attention_weight_max": 35.73356246948242,
      "activations/layer21_attention_weight_min": -22.239145278930664,
      "activations/layer22_attention_weight_max": 28.52058982849121,
      "activations/layer22_attention_weight_min": -25.16412925720215,
      "activations/layer23_attention_weight_max": 37.00496292114258,
      "activations/layer23_attention_weight_min": -23.733409881591797,
      "activations/layer2_attention_weight_max": 31.63410186767578,
      "activations/layer2_attention_weight_min": -31.26392936706543,
      "activations/layer3_attention_weight_max": 85.0451889038086,
      "activations/layer3_attention_weight_min": -88.79641723632812,
      "activations/layer4_attention_weight_max": 96.62084197998047,
      "activations/layer4_attention_weight_min": -94.8755874633789,
      "activations/layer5_attention_weight_max": 77.23814392089844,
      "activations/layer5_attention_weight_min": -78.54173278808594,
      "activations/layer6_attention_weight_max": 51.27096176147461,
      "activations/layer6_attention_weight_min": -53.48896789550781,
      "activations/layer7_attention_weight_max": 77.83963012695312,
      "activations/layer7_attention_weight_min": -76.70647430419922,
      "activations/layer8_attention_weight_max": 44.66484451293945,
      "activations/layer8_attention_weight_min": -46.0417366027832,
      "activations/layer9_attention_weight_max": 49.05185317993164,
      "activations/layer9_attention_weight_min": -48.38298797607422,
      "epoch": 9.24,
      "learning_rate": 9.132196969696968e-05,
      "loss": 2.8058,
      "step": 159050
    },
    {
      "activations/layer0_attention_weight_max": 14.61204719543457,
      "activations/layer0_attention_weight_min": -13.592403411865234,
      "activations/layer10_attention_weight_max": 39.63035583496094,
      "activations/layer10_attention_weight_min": -37.08697509765625,
      "activations/layer11_attention_weight_max": 31.184600830078125,
      "activations/layer11_attention_weight_min": -33.500972747802734,
      "activations/layer12_attention_weight_max": 17.894134521484375,
      "activations/layer12_attention_weight_min": -26.537885665893555,
      "activations/layer13_attention_weight_max": 36.260528564453125,
      "activations/layer13_attention_weight_min": -29.340145111083984,
      "activations/layer14_attention_weight_max": 36.52370071411133,
      "activations/layer14_attention_weight_min": -29.12038230895996,
      "activations/layer15_attention_weight_max": 34.35597229003906,
      "activations/layer15_attention_weight_min": -28.819101333618164,
      "activations/layer16_attention_weight_max": 30.861568450927734,
      "activations/layer16_attention_weight_min": -26.83521842956543,
      "activations/layer17_attention_weight_max": 54.54331970214844,
      "activations/layer17_attention_weight_min": -43.6235466003418,
      "activations/layer18_attention_weight_max": 47.34239196777344,
      "activations/layer18_attention_weight_min": -36.22200393676758,
      "activations/layer19_attention_weight_max": 22.148544311523438,
      "activations/layer19_attention_weight_min": -19.913494110107422,
      "activations/layer1_attention_weight_max": 16.44377899169922,
      "activations/layer1_attention_weight_min": -14.430719375610352,
      "activations/layer20_attention_weight_max": 21.595748901367188,
      "activations/layer20_attention_weight_min": -21.73908042907715,
      "activations/layer21_attention_weight_max": 35.85111999511719,
      "activations/layer21_attention_weight_min": -20.890518188476562,
      "activations/layer22_attention_weight_max": 32.981956481933594,
      "activations/layer22_attention_weight_min": -24.808349609375,
      "activations/layer23_attention_weight_max": 35.164451599121094,
      "activations/layer23_attention_weight_min": -21.385257720947266,
      "activations/layer2_attention_weight_max": 30.475364685058594,
      "activations/layer2_attention_weight_min": -30.083389282226562,
      "activations/layer3_attention_weight_max": 87.60538482666016,
      "activations/layer3_attention_weight_min": -93.17150115966797,
      "activations/layer4_attention_weight_max": 103.35359191894531,
      "activations/layer4_attention_weight_min": -99.27369689941406,
      "activations/layer5_attention_weight_max": 80.26380920410156,
      "activations/layer5_attention_weight_min": -78.40386199951172,
      "activations/layer6_attention_weight_max": 54.844642639160156,
      "activations/layer6_attention_weight_min": -55.938663482666016,
      "activations/layer7_attention_weight_max": 79.10968017578125,
      "activations/layer7_attention_weight_min": -75.71051025390625,
      "activations/layer8_attention_weight_max": 46.885501861572266,
      "activations/layer8_attention_weight_min": -48.243499755859375,
      "activations/layer9_attention_weight_max": 52.80584716796875,
      "activations/layer9_attention_weight_min": -51.42397689819336,
      "epoch": 9.24,
      "learning_rate": 9.13030303030303e-05,
      "loss": 2.8059,
      "step": 159100
    },
    {
      "activations/layer0_attention_weight_max": 15.499049186706543,
      "activations/layer0_attention_weight_min": -13.961936950683594,
      "activations/layer10_attention_weight_max": 40.85137939453125,
      "activations/layer10_attention_weight_min": -40.663787841796875,
      "activations/layer11_attention_weight_max": 34.7880744934082,
      "activations/layer11_attention_weight_min": -34.48784637451172,
      "activations/layer12_attention_weight_max": 19.418323516845703,
      "activations/layer12_attention_weight_min": -28.421228408813477,
      "activations/layer13_attention_weight_max": 40.908485412597656,
      "activations/layer13_attention_weight_min": -31.09127426147461,
      "activations/layer14_attention_weight_max": 37.718894958496094,
      "activations/layer14_attention_weight_min": -30.67049789428711,
      "activations/layer15_attention_weight_max": 34.91468811035156,
      "activations/layer15_attention_weight_min": -29.994630813598633,
      "activations/layer16_attention_weight_max": 30.474214553833008,
      "activations/layer16_attention_weight_min": -28.978147506713867,
      "activations/layer17_attention_weight_max": 52.0026741027832,
      "activations/layer17_attention_weight_min": -45.84737014770508,
      "activations/layer18_attention_weight_max": 50.14090347290039,
      "activations/layer18_attention_weight_min": -37.5929069519043,
      "activations/layer19_attention_weight_max": 29.56037712097168,
      "activations/layer19_attention_weight_min": -21.9417781829834,
      "activations/layer1_attention_weight_max": 17.078943252563477,
      "activations/layer1_attention_weight_min": -14.606435775756836,
      "activations/layer20_attention_weight_max": 24.49357795715332,
      "activations/layer20_attention_weight_min": -21.441020965576172,
      "activations/layer21_attention_weight_max": 39.1124267578125,
      "activations/layer21_attention_weight_min": -24.555803298950195,
      "activations/layer22_attention_weight_max": 34.03907775878906,
      "activations/layer22_attention_weight_min": -25.409238815307617,
      "activations/layer23_attention_weight_max": 37.68573760986328,
      "activations/layer23_attention_weight_min": -21.509929656982422,
      "activations/layer2_attention_weight_max": 31.12671661376953,
      "activations/layer2_attention_weight_min": -30.392072677612305,
      "activations/layer3_attention_weight_max": 91.27533721923828,
      "activations/layer3_attention_weight_min": -92.40963745117188,
      "activations/layer4_attention_weight_max": 98.08589935302734,
      "activations/layer4_attention_weight_min": -95.85183715820312,
      "activations/layer5_attention_weight_max": 80.76575469970703,
      "activations/layer5_attention_weight_min": -78.67262268066406,
      "activations/layer6_attention_weight_max": 56.13081359863281,
      "activations/layer6_attention_weight_min": -54.293277740478516,
      "activations/layer7_attention_weight_max": 80.58025360107422,
      "activations/layer7_attention_weight_min": -74.8381118774414,
      "activations/layer8_attention_weight_max": 47.083526611328125,
      "activations/layer8_attention_weight_min": -49.672569274902344,
      "activations/layer9_attention_weight_max": 50.107662200927734,
      "activations/layer9_attention_weight_min": -49.80449295043945,
      "epoch": 9.25,
      "learning_rate": 9.12840909090909e-05,
      "loss": 2.8047,
      "step": 159150
    },
    {
      "activations/layer0_attention_weight_max": 15.143214225769043,
      "activations/layer0_attention_weight_min": -14.242545127868652,
      "activations/layer10_attention_weight_max": 35.23276901245117,
      "activations/layer10_attention_weight_min": -36.92530059814453,
      "activations/layer11_attention_weight_max": 32.225440979003906,
      "activations/layer11_attention_weight_min": -31.182735443115234,
      "activations/layer12_attention_weight_max": 17.50282859802246,
      "activations/layer12_attention_weight_min": -27.202295303344727,
      "activations/layer13_attention_weight_max": 45.22126388549805,
      "activations/layer13_attention_weight_min": -36.10300064086914,
      "activations/layer14_attention_weight_max": 38.233192443847656,
      "activations/layer14_attention_weight_min": -30.227354049682617,
      "activations/layer15_attention_weight_max": 32.81101608276367,
      "activations/layer15_attention_weight_min": -29.665767669677734,
      "activations/layer16_attention_weight_max": 29.0943546295166,
      "activations/layer16_attention_weight_min": -27.89702606201172,
      "activations/layer17_attention_weight_max": 55.21814727783203,
      "activations/layer17_attention_weight_min": -42.8752326965332,
      "activations/layer18_attention_weight_max": 49.630733489990234,
      "activations/layer18_attention_weight_min": -37.8785400390625,
      "activations/layer19_attention_weight_max": 22.242324829101562,
      "activations/layer19_attention_weight_min": -22.023160934448242,
      "activations/layer1_attention_weight_max": 16.277477264404297,
      "activations/layer1_attention_weight_min": -14.816949844360352,
      "activations/layer20_attention_weight_max": 24.62665367126465,
      "activations/layer20_attention_weight_min": -22.784513473510742,
      "activations/layer21_attention_weight_max": 40.17247009277344,
      "activations/layer21_attention_weight_min": -25.65170669555664,
      "activations/layer22_attention_weight_max": 33.71856689453125,
      "activations/layer22_attention_weight_min": -25.96371841430664,
      "activations/layer23_attention_weight_max": 39.19651412963867,
      "activations/layer23_attention_weight_min": -24.03940200805664,
      "activations/layer2_attention_weight_max": 28.864059448242188,
      "activations/layer2_attention_weight_min": -28.63365936279297,
      "activations/layer3_attention_weight_max": 86.76905822753906,
      "activations/layer3_attention_weight_min": -87.48593139648438,
      "activations/layer4_attention_weight_max": 100.98322296142578,
      "activations/layer4_attention_weight_min": -92.7507553100586,
      "activations/layer5_attention_weight_max": 81.5658187866211,
      "activations/layer5_attention_weight_min": -77.21048736572266,
      "activations/layer6_attention_weight_max": 53.49497985839844,
      "activations/layer6_attention_weight_min": -56.18035888671875,
      "activations/layer7_attention_weight_max": 75.90385437011719,
      "activations/layer7_attention_weight_min": -78.55726623535156,
      "activations/layer8_attention_weight_max": 43.68484115600586,
      "activations/layer8_attention_weight_min": -45.933250427246094,
      "activations/layer9_attention_weight_max": 49.136573791503906,
      "activations/layer9_attention_weight_min": -49.523014068603516,
      "epoch": 9.25,
      "learning_rate": 9.126515151515151e-05,
      "loss": 2.7953,
      "step": 159200
    },
    {
      "activations/layer0_attention_weight_max": 15.725924491882324,
      "activations/layer0_attention_weight_min": -13.841047286987305,
      "activations/layer10_attention_weight_max": 34.886173248291016,
      "activations/layer10_attention_weight_min": -36.253292083740234,
      "activations/layer11_attention_weight_max": 31.991086959838867,
      "activations/layer11_attention_weight_min": -30.803592681884766,
      "activations/layer12_attention_weight_max": 18.91200065612793,
      "activations/layer12_attention_weight_min": -24.4833984375,
      "activations/layer13_attention_weight_max": 36.85455322265625,
      "activations/layer13_attention_weight_min": -29.03268051147461,
      "activations/layer14_attention_weight_max": 38.26015853881836,
      "activations/layer14_attention_weight_min": -30.109600067138672,
      "activations/layer15_attention_weight_max": 34.54702377319336,
      "activations/layer15_attention_weight_min": -29.527055740356445,
      "activations/layer16_attention_weight_max": 31.726200103759766,
      "activations/layer16_attention_weight_min": -27.74654197692871,
      "activations/layer17_attention_weight_max": 55.312278747558594,
      "activations/layer17_attention_weight_min": -42.3563117980957,
      "activations/layer18_attention_weight_max": 48.13631820678711,
      "activations/layer18_attention_weight_min": -35.79256057739258,
      "activations/layer19_attention_weight_max": 20.91417694091797,
      "activations/layer19_attention_weight_min": -21.98777198791504,
      "activations/layer1_attention_weight_max": 16.416179656982422,
      "activations/layer1_attention_weight_min": -15.367472648620605,
      "activations/layer20_attention_weight_max": 24.27688980102539,
      "activations/layer20_attention_weight_min": -21.10372543334961,
      "activations/layer21_attention_weight_max": 35.093963623046875,
      "activations/layer21_attention_weight_min": -24.121191024780273,
      "activations/layer22_attention_weight_max": 32.795284271240234,
      "activations/layer22_attention_weight_min": -26.622114181518555,
      "activations/layer23_attention_weight_max": 36.26272201538086,
      "activations/layer23_attention_weight_min": -22.844135284423828,
      "activations/layer2_attention_weight_max": 32.08212661743164,
      "activations/layer2_attention_weight_min": -30.518095016479492,
      "activations/layer3_attention_weight_max": 90.63634490966797,
      "activations/layer3_attention_weight_min": -91.61133575439453,
      "activations/layer4_attention_weight_max": 101.40332794189453,
      "activations/layer4_attention_weight_min": -94.05406951904297,
      "activations/layer5_attention_weight_max": 77.9176025390625,
      "activations/layer5_attention_weight_min": -78.33794403076172,
      "activations/layer6_attention_weight_max": 53.55070114135742,
      "activations/layer6_attention_weight_min": -53.63682556152344,
      "activations/layer7_attention_weight_max": 76.88453674316406,
      "activations/layer7_attention_weight_min": -71.82583618164062,
      "activations/layer8_attention_weight_max": 42.79671096801758,
      "activations/layer8_attention_weight_min": -45.987125396728516,
      "activations/layer9_attention_weight_max": 53.04110336303711,
      "activations/layer9_attention_weight_min": -47.8855094909668,
      "epoch": 9.25,
      "learning_rate": 9.124621212121212e-05,
      "loss": 2.7788,
      "step": 159250
    },
    {
      "activations/layer0_attention_weight_max": 15.5274658203125,
      "activations/layer0_attention_weight_min": -13.754752159118652,
      "activations/layer10_attention_weight_max": 39.16170120239258,
      "activations/layer10_attention_weight_min": -37.88608932495117,
      "activations/layer11_attention_weight_max": 33.76019287109375,
      "activations/layer11_attention_weight_min": -32.86519241333008,
      "activations/layer12_attention_weight_max": 25.772201538085938,
      "activations/layer12_attention_weight_min": -25.932205200195312,
      "activations/layer13_attention_weight_max": 42.54067611694336,
      "activations/layer13_attention_weight_min": -32.997802734375,
      "activations/layer14_attention_weight_max": 35.613216400146484,
      "activations/layer14_attention_weight_min": -29.731462478637695,
      "activations/layer15_attention_weight_max": 35.612022399902344,
      "activations/layer15_attention_weight_min": -28.73989486694336,
      "activations/layer16_attention_weight_max": 31.247655868530273,
      "activations/layer16_attention_weight_min": -27.72686195373535,
      "activations/layer17_attention_weight_max": 59.71742248535156,
      "activations/layer17_attention_weight_min": -44.381832122802734,
      "activations/layer18_attention_weight_max": 49.81928634643555,
      "activations/layer18_attention_weight_min": -37.226139068603516,
      "activations/layer19_attention_weight_max": 22.510482788085938,
      "activations/layer19_attention_weight_min": -21.801345825195312,
      "activations/layer1_attention_weight_max": 16.345762252807617,
      "activations/layer1_attention_weight_min": -15.167975425720215,
      "activations/layer20_attention_weight_max": 20.326881408691406,
      "activations/layer20_attention_weight_min": -21.75895881652832,
      "activations/layer21_attention_weight_max": 32.94319534301758,
      "activations/layer21_attention_weight_min": -23.870813369750977,
      "activations/layer22_attention_weight_max": 27.56454849243164,
      "activations/layer22_attention_weight_min": -25.6207332611084,
      "activations/layer23_attention_weight_max": 29.975269317626953,
      "activations/layer23_attention_weight_min": -23.49785614013672,
      "activations/layer2_attention_weight_max": 31.36702537536621,
      "activations/layer2_attention_weight_min": -29.410207748413086,
      "activations/layer3_attention_weight_max": 93.99536895751953,
      "activations/layer3_attention_weight_min": -92.49327850341797,
      "activations/layer4_attention_weight_max": 104.17198944091797,
      "activations/layer4_attention_weight_min": -97.60004425048828,
      "activations/layer5_attention_weight_max": 82.62510681152344,
      "activations/layer5_attention_weight_min": -80.45111846923828,
      "activations/layer6_attention_weight_max": 57.40425491333008,
      "activations/layer6_attention_weight_min": -54.76808166503906,
      "activations/layer7_attention_weight_max": 79.75464630126953,
      "activations/layer7_attention_weight_min": -74.69822692871094,
      "activations/layer8_attention_weight_max": 47.299598693847656,
      "activations/layer8_attention_weight_min": -50.25946807861328,
      "activations/layer9_attention_weight_max": 53.443603515625,
      "activations/layer9_attention_weight_min": -51.71822738647461,
      "epoch": 9.26,
      "learning_rate": 9.122727272727272e-05,
      "loss": 2.7789,
      "step": 159300
    },
    {
      "activations/layer0_attention_weight_max": 14.857043266296387,
      "activations/layer0_attention_weight_min": -13.681233406066895,
      "activations/layer10_attention_weight_max": 41.507843017578125,
      "activations/layer10_attention_weight_min": -37.67006301879883,
      "activations/layer11_attention_weight_max": 33.50973892211914,
      "activations/layer11_attention_weight_min": -33.58686065673828,
      "activations/layer12_attention_weight_max": 20.09990882873535,
      "activations/layer12_attention_weight_min": -24.35308074951172,
      "activations/layer13_attention_weight_max": 42.31879806518555,
      "activations/layer13_attention_weight_min": -30.22161102294922,
      "activations/layer14_attention_weight_max": 38.31473922729492,
      "activations/layer14_attention_weight_min": -31.620601654052734,
      "activations/layer15_attention_weight_max": 39.20707702636719,
      "activations/layer15_attention_weight_min": -30.47415542602539,
      "activations/layer16_attention_weight_max": 30.62498664855957,
      "activations/layer16_attention_weight_min": -30.20044708251953,
      "activations/layer17_attention_weight_max": 49.361297607421875,
      "activations/layer17_attention_weight_min": -41.07743835449219,
      "activations/layer18_attention_weight_max": 45.18525695800781,
      "activations/layer18_attention_weight_min": -38.48103332519531,
      "activations/layer19_attention_weight_max": 23.813932418823242,
      "activations/layer19_attention_weight_min": -22.019500732421875,
      "activations/layer1_attention_weight_max": 16.81340217590332,
      "activations/layer1_attention_weight_min": -15.51942253112793,
      "activations/layer20_attention_weight_max": 22.58277702331543,
      "activations/layer20_attention_weight_min": -22.836618423461914,
      "activations/layer21_attention_weight_max": 41.17767333984375,
      "activations/layer21_attention_weight_min": -26.639705657958984,
      "activations/layer22_attention_weight_max": 34.2731819152832,
      "activations/layer22_attention_weight_min": -27.636825561523438,
      "activations/layer23_attention_weight_max": 38.166378021240234,
      "activations/layer23_attention_weight_min": -23.1160888671875,
      "activations/layer2_attention_weight_max": 33.115318298339844,
      "activations/layer2_attention_weight_min": -30.737218856811523,
      "activations/layer3_attention_weight_max": 90.75391387939453,
      "activations/layer3_attention_weight_min": -96.70481872558594,
      "activations/layer4_attention_weight_max": 100.41515350341797,
      "activations/layer4_attention_weight_min": -96.21234130859375,
      "activations/layer5_attention_weight_max": 84.98137664794922,
      "activations/layer5_attention_weight_min": -85.01509094238281,
      "activations/layer6_attention_weight_max": 58.88323211669922,
      "activations/layer6_attention_weight_min": -56.00178909301758,
      "activations/layer7_attention_weight_max": 80.62362670898438,
      "activations/layer7_attention_weight_min": -78.794189453125,
      "activations/layer8_attention_weight_max": 47.33594512939453,
      "activations/layer8_attention_weight_min": -48.57189178466797,
      "activations/layer9_attention_weight_max": 52.08234405517578,
      "activations/layer9_attention_weight_min": -52.65290069580078,
      "epoch": 9.26,
      "learning_rate": 9.120833333333333e-05,
      "loss": 2.7939,
      "step": 159350
    },
    {
      "activations/layer0_attention_weight_max": 15.320237159729004,
      "activations/layer0_attention_weight_min": -13.810604095458984,
      "activations/layer10_attention_weight_max": 36.6802864074707,
      "activations/layer10_attention_weight_min": -36.06356430053711,
      "activations/layer11_attention_weight_max": 30.1175537109375,
      "activations/layer11_attention_weight_min": -32.04191589355469,
      "activations/layer12_attention_weight_max": 18.352659225463867,
      "activations/layer12_attention_weight_min": -24.20420265197754,
      "activations/layer13_attention_weight_max": 45.032752990722656,
      "activations/layer13_attention_weight_min": -32.288963317871094,
      "activations/layer14_attention_weight_max": 38.805538177490234,
      "activations/layer14_attention_weight_min": -29.663293838500977,
      "activations/layer15_attention_weight_max": 34.33498001098633,
      "activations/layer15_attention_weight_min": -30.279823303222656,
      "activations/layer16_attention_weight_max": 30.026559829711914,
      "activations/layer16_attention_weight_min": -27.53948211669922,
      "activations/layer17_attention_weight_max": 51.8028678894043,
      "activations/layer17_attention_weight_min": -41.69150161743164,
      "activations/layer18_attention_weight_max": 47.75082778930664,
      "activations/layer18_attention_weight_min": -35.03774642944336,
      "activations/layer19_attention_weight_max": 23.791088104248047,
      "activations/layer19_attention_weight_min": -23.25386619567871,
      "activations/layer1_attention_weight_max": 17.21474838256836,
      "activations/layer1_attention_weight_min": -16.639158248901367,
      "activations/layer20_attention_weight_max": 22.16374969482422,
      "activations/layer20_attention_weight_min": -21.63969612121582,
      "activations/layer21_attention_weight_max": 39.02403259277344,
      "activations/layer21_attention_weight_min": -22.994848251342773,
      "activations/layer22_attention_weight_max": 31.87322235107422,
      "activations/layer22_attention_weight_min": -25.49762725830078,
      "activations/layer23_attention_weight_max": 36.70564270019531,
      "activations/layer23_attention_weight_min": -22.70250701904297,
      "activations/layer2_attention_weight_max": 31.320022583007812,
      "activations/layer2_attention_weight_min": -31.57752227783203,
      "activations/layer3_attention_weight_max": 95.974853515625,
      "activations/layer3_attention_weight_min": -96.01343536376953,
      "activations/layer4_attention_weight_max": 101.03716278076172,
      "activations/layer4_attention_weight_min": -101.28064727783203,
      "activations/layer5_attention_weight_max": 82.75146484375,
      "activations/layer5_attention_weight_min": -80.20580291748047,
      "activations/layer6_attention_weight_max": 55.95055389404297,
      "activations/layer6_attention_weight_min": -55.06907272338867,
      "activations/layer7_attention_weight_max": 78.4453125,
      "activations/layer7_attention_weight_min": -73.81403350830078,
      "activations/layer8_attention_weight_max": 44.60132598876953,
      "activations/layer8_attention_weight_min": -46.82089614868164,
      "activations/layer9_attention_weight_max": 49.130184173583984,
      "activations/layer9_attention_weight_min": -48.58095932006836,
      "epoch": 9.26,
      "learning_rate": 9.118939393939392e-05,
      "loss": 2.7932,
      "step": 159400
    },
    {
      "activations/layer0_attention_weight_max": 15.272436141967773,
      "activations/layer0_attention_weight_min": -13.75239086151123,
      "activations/layer10_attention_weight_max": 37.866310119628906,
      "activations/layer10_attention_weight_min": -36.49481201171875,
      "activations/layer11_attention_weight_max": 31.879209518432617,
      "activations/layer11_attention_weight_min": -33.349952697753906,
      "activations/layer12_attention_weight_max": 26.836910247802734,
      "activations/layer12_attention_weight_min": -30.475242614746094,
      "activations/layer13_attention_weight_max": 43.61343765258789,
      "activations/layer13_attention_weight_min": -29.81321144104004,
      "activations/layer14_attention_weight_max": 38.98208999633789,
      "activations/layer14_attention_weight_min": -31.290742874145508,
      "activations/layer15_attention_weight_max": 35.993900299072266,
      "activations/layer15_attention_weight_min": -28.8687744140625,
      "activations/layer16_attention_weight_max": 31.989818572998047,
      "activations/layer16_attention_weight_min": -28.292896270751953,
      "activations/layer17_attention_weight_max": 53.92524337768555,
      "activations/layer17_attention_weight_min": -41.59215545654297,
      "activations/layer18_attention_weight_max": 46.58575439453125,
      "activations/layer18_attention_weight_min": -37.23458480834961,
      "activations/layer19_attention_weight_max": 22.462949752807617,
      "activations/layer19_attention_weight_min": -20.90348243713379,
      "activations/layer1_attention_weight_max": 17.30698013305664,
      "activations/layer1_attention_weight_min": -15.152342796325684,
      "activations/layer20_attention_weight_max": 22.630891799926758,
      "activations/layer20_attention_weight_min": -20.572105407714844,
      "activations/layer21_attention_weight_max": 38.54695129394531,
      "activations/layer21_attention_weight_min": -23.29361915588379,
      "activations/layer22_attention_weight_max": 33.09809112548828,
      "activations/layer22_attention_weight_min": -25.089244842529297,
      "activations/layer23_attention_weight_max": 41.140743255615234,
      "activations/layer23_attention_weight_min": -23.94721794128418,
      "activations/layer2_attention_weight_max": 32.360530853271484,
      "activations/layer2_attention_weight_min": -30.275833129882812,
      "activations/layer3_attention_weight_max": 91.53203582763672,
      "activations/layer3_attention_weight_min": -92.92022705078125,
      "activations/layer4_attention_weight_max": 99.40032196044922,
      "activations/layer4_attention_weight_min": -96.48854064941406,
      "activations/layer5_attention_weight_max": 80.39021301269531,
      "activations/layer5_attention_weight_min": -76.81642150878906,
      "activations/layer6_attention_weight_max": 53.8392219543457,
      "activations/layer6_attention_weight_min": -52.97700881958008,
      "activations/layer7_attention_weight_max": 80.78679656982422,
      "activations/layer7_attention_weight_min": -76.92414093017578,
      "activations/layer8_attention_weight_max": 45.234588623046875,
      "activations/layer8_attention_weight_min": -48.19730758666992,
      "activations/layer9_attention_weight_max": 55.98029708862305,
      "activations/layer9_attention_weight_min": -51.6159553527832,
      "epoch": 9.26,
      "learning_rate": 9.117045454545454e-05,
      "loss": 2.7971,
      "step": 159450
    },
    {
      "activations/layer0_attention_weight_max": 15.87096881866455,
      "activations/layer0_attention_weight_min": -14.432465553283691,
      "activations/layer10_attention_weight_max": 41.4377555847168,
      "activations/layer10_attention_weight_min": -40.366817474365234,
      "activations/layer11_attention_weight_max": 32.12556076049805,
      "activations/layer11_attention_weight_min": -32.702964782714844,
      "activations/layer12_attention_weight_max": 21.958393096923828,
      "activations/layer12_attention_weight_min": -24.269235610961914,
      "activations/layer13_attention_weight_max": 47.92729568481445,
      "activations/layer13_attention_weight_min": -36.46415328979492,
      "activations/layer14_attention_weight_max": 38.646636962890625,
      "activations/layer14_attention_weight_min": -31.955429077148438,
      "activations/layer15_attention_weight_max": 36.82038497924805,
      "activations/layer15_attention_weight_min": -29.093482971191406,
      "activations/layer16_attention_weight_max": 31.05652618408203,
      "activations/layer16_attention_weight_min": -30.137357711791992,
      "activations/layer17_attention_weight_max": 54.11064529418945,
      "activations/layer17_attention_weight_min": -46.26091766357422,
      "activations/layer18_attention_weight_max": 49.40890884399414,
      "activations/layer18_attention_weight_min": -39.05934524536133,
      "activations/layer19_attention_weight_max": 21.800451278686523,
      "activations/layer19_attention_weight_min": -21.727649688720703,
      "activations/layer1_attention_weight_max": 16.948890686035156,
      "activations/layer1_attention_weight_min": -15.37514591217041,
      "activations/layer20_attention_weight_max": 24.19605255126953,
      "activations/layer20_attention_weight_min": -25.36955451965332,
      "activations/layer21_attention_weight_max": 41.002708435058594,
      "activations/layer21_attention_weight_min": -24.807510375976562,
      "activations/layer22_attention_weight_max": 30.975923538208008,
      "activations/layer22_attention_weight_min": -26.67465591430664,
      "activations/layer23_attention_weight_max": 35.79692077636719,
      "activations/layer23_attention_weight_min": -23.35279083251953,
      "activations/layer2_attention_weight_max": 31.903484344482422,
      "activations/layer2_attention_weight_min": -30.547704696655273,
      "activations/layer3_attention_weight_max": 88.12146759033203,
      "activations/layer3_attention_weight_min": -92.33635711669922,
      "activations/layer4_attention_weight_max": 99.41024017333984,
      "activations/layer4_attention_weight_min": -96.51901245117188,
      "activations/layer5_attention_weight_max": 80.6219482421875,
      "activations/layer5_attention_weight_min": -74.89002990722656,
      "activations/layer6_attention_weight_max": 54.3087043762207,
      "activations/layer6_attention_weight_min": -53.053001403808594,
      "activations/layer7_attention_weight_max": 74.12164306640625,
      "activations/layer7_attention_weight_min": -76.03131866455078,
      "activations/layer8_attention_weight_max": 43.923187255859375,
      "activations/layer8_attention_weight_min": -48.16521072387695,
      "activations/layer9_attention_weight_max": 49.564449310302734,
      "activations/layer9_attention_weight_min": -52.7818717956543,
      "epoch": 9.27,
      "learning_rate": 9.115151515151515e-05,
      "loss": 2.8089,
      "step": 159500
    },
    {
      "activations/layer0_attention_weight_max": 14.93084716796875,
      "activations/layer0_attention_weight_min": -13.728693962097168,
      "activations/layer10_attention_weight_max": 38.45790481567383,
      "activations/layer10_attention_weight_min": -35.59007263183594,
      "activations/layer11_attention_weight_max": 34.506168365478516,
      "activations/layer11_attention_weight_min": -31.644643783569336,
      "activations/layer12_attention_weight_max": 16.933242797851562,
      "activations/layer12_attention_weight_min": -25.87681770324707,
      "activations/layer13_attention_weight_max": 40.15044403076172,
      "activations/layer13_attention_weight_min": -30.46166229248047,
      "activations/layer14_attention_weight_max": 42.62404251098633,
      "activations/layer14_attention_weight_min": -31.956817626953125,
      "activations/layer15_attention_weight_max": 34.9476432800293,
      "activations/layer15_attention_weight_min": -28.804101943969727,
      "activations/layer16_attention_weight_max": 30.699316024780273,
      "activations/layer16_attention_weight_min": -30.378934860229492,
      "activations/layer17_attention_weight_max": 52.42265701293945,
      "activations/layer17_attention_weight_min": -42.92902374267578,
      "activations/layer18_attention_weight_max": 44.320556640625,
      "activations/layer18_attention_weight_min": -37.024070739746094,
      "activations/layer19_attention_weight_max": 21.255537033081055,
      "activations/layer19_attention_weight_min": -21.794904708862305,
      "activations/layer1_attention_weight_max": 17.394733428955078,
      "activations/layer1_attention_weight_min": -16.205951690673828,
      "activations/layer20_attention_weight_max": 22.47978973388672,
      "activations/layer20_attention_weight_min": -21.60308074951172,
      "activations/layer21_attention_weight_max": 45.356712341308594,
      "activations/layer21_attention_weight_min": -24.3720760345459,
      "activations/layer22_attention_weight_max": 31.129072189331055,
      "activations/layer22_attention_weight_min": -27.331750869750977,
      "activations/layer23_attention_weight_max": 36.768028259277344,
      "activations/layer23_attention_weight_min": -24.40163803100586,
      "activations/layer2_attention_weight_max": 32.788551330566406,
      "activations/layer2_attention_weight_min": -30.458913803100586,
      "activations/layer3_attention_weight_max": 90.61126708984375,
      "activations/layer3_attention_weight_min": -89.43637084960938,
      "activations/layer4_attention_weight_max": 102.19401550292969,
      "activations/layer4_attention_weight_min": -95.26071166992188,
      "activations/layer5_attention_weight_max": 80.25053405761719,
      "activations/layer5_attention_weight_min": -75.92301940917969,
      "activations/layer6_attention_weight_max": 54.748775482177734,
      "activations/layer6_attention_weight_min": -53.489112854003906,
      "activations/layer7_attention_weight_max": 78.68605041503906,
      "activations/layer7_attention_weight_min": -77.62104034423828,
      "activations/layer8_attention_weight_max": 48.11561584472656,
      "activations/layer8_attention_weight_min": -46.8012809753418,
      "activations/layer9_attention_weight_max": 51.77213668823242,
      "activations/layer9_attention_weight_min": -51.51644515991211,
      "epoch": 9.27,
      "learning_rate": 9.113257575757574e-05,
      "loss": 2.7958,
      "step": 159550
    },
    {
      "activations/layer0_attention_weight_max": 15.315520286560059,
      "activations/layer0_attention_weight_min": -14.060969352722168,
      "activations/layer10_attention_weight_max": 56.1826286315918,
      "activations/layer10_attention_weight_min": -53.1829833984375,
      "activations/layer11_attention_weight_max": 47.67864227294922,
      "activations/layer11_attention_weight_min": -47.80921173095703,
      "activations/layer12_attention_weight_max": 21.602319717407227,
      "activations/layer12_attention_weight_min": -30.413959503173828,
      "activations/layer13_attention_weight_max": 45.45866394042969,
      "activations/layer13_attention_weight_min": -32.67552947998047,
      "activations/layer14_attention_weight_max": 56.65217971801758,
      "activations/layer14_attention_weight_min": -34.26579284667969,
      "activations/layer15_attention_weight_max": 40.53996276855469,
      "activations/layer15_attention_weight_min": -32.58434295654297,
      "activations/layer16_attention_weight_max": 39.966957092285156,
      "activations/layer16_attention_weight_min": -29.779510498046875,
      "activations/layer17_attention_weight_max": 58.85486602783203,
      "activations/layer17_attention_weight_min": -43.37826156616211,
      "activations/layer18_attention_weight_max": 52.55362319946289,
      "activations/layer18_attention_weight_min": -39.55352020263672,
      "activations/layer19_attention_weight_max": 24.600034713745117,
      "activations/layer19_attention_weight_min": -22.476552963256836,
      "activations/layer1_attention_weight_max": 16.7164249420166,
      "activations/layer1_attention_weight_min": -15.358484268188477,
      "activations/layer20_attention_weight_max": 24.432350158691406,
      "activations/layer20_attention_weight_min": -25.372697830200195,
      "activations/layer21_attention_weight_max": 45.50772476196289,
      "activations/layer21_attention_weight_min": -24.774734497070312,
      "activations/layer22_attention_weight_max": 31.137773513793945,
      "activations/layer22_attention_weight_min": -25.958141326904297,
      "activations/layer23_attention_weight_max": 38.70793151855469,
      "activations/layer23_attention_weight_min": -23.240215301513672,
      "activations/layer2_attention_weight_max": 37.79347610473633,
      "activations/layer2_attention_weight_min": -34.41150665283203,
      "activations/layer3_attention_weight_max": 102.77059936523438,
      "activations/layer3_attention_weight_min": -104.12617492675781,
      "activations/layer4_attention_weight_max": 116.92134857177734,
      "activations/layer4_attention_weight_min": -111.00422668457031,
      "activations/layer5_attention_weight_max": 86.48365783691406,
      "activations/layer5_attention_weight_min": -83.93968963623047,
      "activations/layer6_attention_weight_max": 64.34310150146484,
      "activations/layer6_attention_weight_min": -62.1640510559082,
      "activations/layer7_attention_weight_max": 112.2991714477539,
      "activations/layer7_attention_weight_min": -93.58252716064453,
      "activations/layer8_attention_weight_max": 69.12215423583984,
      "activations/layer8_attention_weight_min": -64.59683990478516,
      "activations/layer9_attention_weight_max": 83.12540435791016,
      "activations/layer9_attention_weight_min": -64.35689544677734,
      "epoch": 9.27,
      "learning_rate": 9.111363636363635e-05,
      "loss": 2.7912,
      "step": 159600
    },
    {
      "activations/layer0_attention_weight_max": 14.882984161376953,
      "activations/layer0_attention_weight_min": -13.570466995239258,
      "activations/layer10_attention_weight_max": 37.39128112792969,
      "activations/layer10_attention_weight_min": -39.39838409423828,
      "activations/layer11_attention_weight_max": 31.275739669799805,
      "activations/layer11_attention_weight_min": -35.38561248779297,
      "activations/layer12_attention_weight_max": 16.234121322631836,
      "activations/layer12_attention_weight_min": -26.21407699584961,
      "activations/layer13_attention_weight_max": 45.533958435058594,
      "activations/layer13_attention_weight_min": -33.78639221191406,
      "activations/layer14_attention_weight_max": 38.455413818359375,
      "activations/layer14_attention_weight_min": -30.25234603881836,
      "activations/layer15_attention_weight_max": 39.30191421508789,
      "activations/layer15_attention_weight_min": -31.832048416137695,
      "activations/layer16_attention_weight_max": 31.740543365478516,
      "activations/layer16_attention_weight_min": -27.383556365966797,
      "activations/layer17_attention_weight_max": 53.617034912109375,
      "activations/layer17_attention_weight_min": -41.30208206176758,
      "activations/layer18_attention_weight_max": 46.06174087524414,
      "activations/layer18_attention_weight_min": -37.66704177856445,
      "activations/layer19_attention_weight_max": 23.5197811126709,
      "activations/layer19_attention_weight_min": -21.518247604370117,
      "activations/layer1_attention_weight_max": 16.955291748046875,
      "activations/layer1_attention_weight_min": -16.30158233642578,
      "activations/layer20_attention_weight_max": 24.110015869140625,
      "activations/layer20_attention_weight_min": -22.832473754882812,
      "activations/layer21_attention_weight_max": 41.437042236328125,
      "activations/layer21_attention_weight_min": -25.7119083404541,
      "activations/layer22_attention_weight_max": 31.246232986450195,
      "activations/layer22_attention_weight_min": -25.836585998535156,
      "activations/layer23_attention_weight_max": 38.91154479980469,
      "activations/layer23_attention_weight_min": -23.682125091552734,
      "activations/layer2_attention_weight_max": 32.31927490234375,
      "activations/layer2_attention_weight_min": -29.75371551513672,
      "activations/layer3_attention_weight_max": 86.59395599365234,
      "activations/layer3_attention_weight_min": -91.14405059814453,
      "activations/layer4_attention_weight_max": 96.127685546875,
      "activations/layer4_attention_weight_min": -96.19720458984375,
      "activations/layer5_attention_weight_max": 77.80905151367188,
      "activations/layer5_attention_weight_min": -75.00227355957031,
      "activations/layer6_attention_weight_max": 53.48460388183594,
      "activations/layer6_attention_weight_min": -53.31465148925781,
      "activations/layer7_attention_weight_max": 75.00978088378906,
      "activations/layer7_attention_weight_min": -72.8121109008789,
      "activations/layer8_attention_weight_max": 44.98076248168945,
      "activations/layer8_attention_weight_min": -49.102020263671875,
      "activations/layer9_attention_weight_max": 48.11373519897461,
      "activations/layer9_attention_weight_min": -48.32433319091797,
      "epoch": 9.28,
      "learning_rate": 9.109469696969697e-05,
      "loss": 2.7894,
      "step": 159650
    },
    {
      "activations/layer0_attention_weight_max": 15.331446647644043,
      "activations/layer0_attention_weight_min": -13.586921691894531,
      "activations/layer10_attention_weight_max": 38.723182678222656,
      "activations/layer10_attention_weight_min": -37.938880920410156,
      "activations/layer11_attention_weight_max": 30.67928123474121,
      "activations/layer11_attention_weight_min": -35.53728103637695,
      "activations/layer12_attention_weight_max": 18.47492790222168,
      "activations/layer12_attention_weight_min": -24.56795883178711,
      "activations/layer13_attention_weight_max": 38.120948791503906,
      "activations/layer13_attention_weight_min": -30.420122146606445,
      "activations/layer14_attention_weight_max": 37.541175842285156,
      "activations/layer14_attention_weight_min": -30.572298049926758,
      "activations/layer15_attention_weight_max": 32.63819885253906,
      "activations/layer15_attention_weight_min": -30.056455612182617,
      "activations/layer16_attention_weight_max": 31.9232177734375,
      "activations/layer16_attention_weight_min": -28.689794540405273,
      "activations/layer17_attention_weight_max": 49.46248245239258,
      "activations/layer17_attention_weight_min": -44.575130462646484,
      "activations/layer18_attention_weight_max": 43.15117645263672,
      "activations/layer18_attention_weight_min": -35.739349365234375,
      "activations/layer19_attention_weight_max": 18.802757263183594,
      "activations/layer19_attention_weight_min": -19.639326095581055,
      "activations/layer1_attention_weight_max": 16.503263473510742,
      "activations/layer1_attention_weight_min": -15.147883415222168,
      "activations/layer20_attention_weight_max": 21.884288787841797,
      "activations/layer20_attention_weight_min": -20.390716552734375,
      "activations/layer21_attention_weight_max": 36.30595016479492,
      "activations/layer21_attention_weight_min": -23.863340377807617,
      "activations/layer22_attention_weight_max": 29.56569480895996,
      "activations/layer22_attention_weight_min": -25.342060089111328,
      "activations/layer23_attention_weight_max": 34.28669357299805,
      "activations/layer23_attention_weight_min": -23.834606170654297,
      "activations/layer2_attention_weight_max": 32.01490783691406,
      "activations/layer2_attention_weight_min": -29.818822860717773,
      "activations/layer3_attention_weight_max": 87.30329132080078,
      "activations/layer3_attention_weight_min": -87.9386215209961,
      "activations/layer4_attention_weight_max": 102.49430084228516,
      "activations/layer4_attention_weight_min": -92.8321304321289,
      "activations/layer5_attention_weight_max": 83.70032501220703,
      "activations/layer5_attention_weight_min": -78.626953125,
      "activations/layer6_attention_weight_max": 56.156593322753906,
      "activations/layer6_attention_weight_min": -55.4074821472168,
      "activations/layer7_attention_weight_max": 76.21127319335938,
      "activations/layer7_attention_weight_min": -75.52770233154297,
      "activations/layer8_attention_weight_max": 45.400115966796875,
      "activations/layer8_attention_weight_min": -47.07188034057617,
      "activations/layer9_attention_weight_max": 47.62873840332031,
      "activations/layer9_attention_weight_min": -48.46015548706055,
      "epoch": 9.28,
      "learning_rate": 9.107575757575757e-05,
      "loss": 2.8154,
      "step": 159700
    },
    {
      "activations/layer0_attention_weight_max": 14.874547004699707,
      "activations/layer0_attention_weight_min": -13.757468223571777,
      "activations/layer10_attention_weight_max": 36.77326965332031,
      "activations/layer10_attention_weight_min": -35.08327865600586,
      "activations/layer11_attention_weight_max": 30.85564422607422,
      "activations/layer11_attention_weight_min": -33.11903762817383,
      "activations/layer12_attention_weight_max": 19.590391159057617,
      "activations/layer12_attention_weight_min": -29.549659729003906,
      "activations/layer13_attention_weight_max": 51.865230560302734,
      "activations/layer13_attention_weight_min": -35.492401123046875,
      "activations/layer14_attention_weight_max": 37.84893035888672,
      "activations/layer14_attention_weight_min": -30.604869842529297,
      "activations/layer15_attention_weight_max": 37.37910461425781,
      "activations/layer15_attention_weight_min": -29.62105369567871,
      "activations/layer16_attention_weight_max": 35.585845947265625,
      "activations/layer16_attention_weight_min": -30.121919631958008,
      "activations/layer17_attention_weight_max": 55.137874603271484,
      "activations/layer17_attention_weight_min": -44.71600341796875,
      "activations/layer18_attention_weight_max": 48.59137725830078,
      "activations/layer18_attention_weight_min": -38.004981994628906,
      "activations/layer19_attention_weight_max": 25.035747528076172,
      "activations/layer19_attention_weight_min": -21.086135864257812,
      "activations/layer1_attention_weight_max": 16.670272827148438,
      "activations/layer1_attention_weight_min": -13.845402717590332,
      "activations/layer20_attention_weight_max": 23.664165496826172,
      "activations/layer20_attention_weight_min": -20.80255126953125,
      "activations/layer21_attention_weight_max": 42.34260177612305,
      "activations/layer21_attention_weight_min": -23.35228729248047,
      "activations/layer22_attention_weight_max": 32.55355453491211,
      "activations/layer22_attention_weight_min": -26.45461082458496,
      "activations/layer23_attention_weight_max": 37.85220718383789,
      "activations/layer23_attention_weight_min": -22.40357208251953,
      "activations/layer2_attention_weight_max": 30.968915939331055,
      "activations/layer2_attention_weight_min": -28.52057647705078,
      "activations/layer3_attention_weight_max": 95.87371826171875,
      "activations/layer3_attention_weight_min": -98.1698226928711,
      "activations/layer4_attention_weight_max": 103.6125259399414,
      "activations/layer4_attention_weight_min": -103.8565902709961,
      "activations/layer5_attention_weight_max": 79.09819030761719,
      "activations/layer5_attention_weight_min": -80.51214599609375,
      "activations/layer6_attention_weight_max": 54.82060241699219,
      "activations/layer6_attention_weight_min": -55.50725555419922,
      "activations/layer7_attention_weight_max": 80.83068084716797,
      "activations/layer7_attention_weight_min": -74.62996673583984,
      "activations/layer8_attention_weight_max": 46.92378616333008,
      "activations/layer8_attention_weight_min": -47.04563903808594,
      "activations/layer9_attention_weight_max": 51.88724899291992,
      "activations/layer9_attention_weight_min": -48.51705551147461,
      "epoch": 9.28,
      "learning_rate": 9.105681818181817e-05,
      "loss": 2.7933,
      "step": 159750
    },
    {
      "activations/layer0_attention_weight_max": 17.603029251098633,
      "activations/layer0_attention_weight_min": -14.088735580444336,
      "activations/layer10_attention_weight_max": 35.695098876953125,
      "activations/layer10_attention_weight_min": -34.30803680419922,
      "activations/layer11_attention_weight_max": 31.12746238708496,
      "activations/layer11_attention_weight_min": -32.22982406616211,
      "activations/layer12_attention_weight_max": 21.589841842651367,
      "activations/layer12_attention_weight_min": -25.464401245117188,
      "activations/layer13_attention_weight_max": 34.650184631347656,
      "activations/layer13_attention_weight_min": -29.498291015625,
      "activations/layer14_attention_weight_max": 36.18805694580078,
      "activations/layer14_attention_weight_min": -29.606313705444336,
      "activations/layer15_attention_weight_max": 32.77909851074219,
      "activations/layer15_attention_weight_min": -29.900920867919922,
      "activations/layer16_attention_weight_max": 32.82810974121094,
      "activations/layer16_attention_weight_min": -31.822229385375977,
      "activations/layer17_attention_weight_max": 52.64363479614258,
      "activations/layer17_attention_weight_min": -46.26760482788086,
      "activations/layer18_attention_weight_max": 47.489952087402344,
      "activations/layer18_attention_weight_min": -40.881900787353516,
      "activations/layer19_attention_weight_max": 23.134836196899414,
      "activations/layer19_attention_weight_min": -22.73841667175293,
      "activations/layer1_attention_weight_max": 16.613441467285156,
      "activations/layer1_attention_weight_min": -13.70138931274414,
      "activations/layer20_attention_weight_max": 21.69420623779297,
      "activations/layer20_attention_weight_min": -24.26202964782715,
      "activations/layer21_attention_weight_max": 35.63313674926758,
      "activations/layer21_attention_weight_min": -24.323766708374023,
      "activations/layer22_attention_weight_max": 30.73853302001953,
      "activations/layer22_attention_weight_min": -25.914413452148438,
      "activations/layer23_attention_weight_max": 37.804893493652344,
      "activations/layer23_attention_weight_min": -21.14422607421875,
      "activations/layer2_attention_weight_max": 32.27198791503906,
      "activations/layer2_attention_weight_min": -32.58061981201172,
      "activations/layer3_attention_weight_max": 94.0587387084961,
      "activations/layer3_attention_weight_min": -94.87075805664062,
      "activations/layer4_attention_weight_max": 103.68087005615234,
      "activations/layer4_attention_weight_min": -96.94580078125,
      "activations/layer5_attention_weight_max": 83.15312194824219,
      "activations/layer5_attention_weight_min": -79.89886474609375,
      "activations/layer6_attention_weight_max": 58.317352294921875,
      "activations/layer6_attention_weight_min": -56.48674392700195,
      "activations/layer7_attention_weight_max": 84.56732940673828,
      "activations/layer7_attention_weight_min": -76.55522918701172,
      "activations/layer8_attention_weight_max": 42.98268127441406,
      "activations/layer8_attention_weight_min": -45.32754898071289,
      "activations/layer9_attention_weight_max": 49.41367721557617,
      "activations/layer9_attention_weight_min": -49.83851623535156,
      "epoch": 9.29,
      "learning_rate": 9.103787878787879e-05,
      "loss": 2.7862,
      "step": 159800
    },
    {
      "activations/layer0_attention_weight_max": 15.43482780456543,
      "activations/layer0_attention_weight_min": -13.955965995788574,
      "activations/layer10_attention_weight_max": 38.767181396484375,
      "activations/layer10_attention_weight_min": -35.14947509765625,
      "activations/layer11_attention_weight_max": 33.541038513183594,
      "activations/layer11_attention_weight_min": -32.825111389160156,
      "activations/layer12_attention_weight_max": 23.76007843017578,
      "activations/layer12_attention_weight_min": -24.337562561035156,
      "activations/layer13_attention_weight_max": 37.08198547363281,
      "activations/layer13_attention_weight_min": -29.130958557128906,
      "activations/layer14_attention_weight_max": 37.31821823120117,
      "activations/layer14_attention_weight_min": -28.516931533813477,
      "activations/layer15_attention_weight_max": 34.250572204589844,
      "activations/layer15_attention_weight_min": -27.802213668823242,
      "activations/layer16_attention_weight_max": 31.63224220275879,
      "activations/layer16_attention_weight_min": -27.8334903717041,
      "activations/layer17_attention_weight_max": 54.21026611328125,
      "activations/layer17_attention_weight_min": -40.370399475097656,
      "activations/layer18_attention_weight_max": 45.219207763671875,
      "activations/layer18_attention_weight_min": -36.3760871887207,
      "activations/layer19_attention_weight_max": 22.558609008789062,
      "activations/layer19_attention_weight_min": -20.8447208404541,
      "activations/layer1_attention_weight_max": 17.246400833129883,
      "activations/layer1_attention_weight_min": -13.666581153869629,
      "activations/layer20_attention_weight_max": 23.368162155151367,
      "activations/layer20_attention_weight_min": -23.31370735168457,
      "activations/layer21_attention_weight_max": 38.56740188598633,
      "activations/layer21_attention_weight_min": -22.350004196166992,
      "activations/layer22_attention_weight_max": 28.152509689331055,
      "activations/layer22_attention_weight_min": -23.57588768005371,
      "activations/layer23_attention_weight_max": 34.85475158691406,
      "activations/layer23_attention_weight_min": -23.203754425048828,
      "activations/layer2_attention_weight_max": 32.31547546386719,
      "activations/layer2_attention_weight_min": -30.12666130065918,
      "activations/layer3_attention_weight_max": 92.07783508300781,
      "activations/layer3_attention_weight_min": -92.96955108642578,
      "activations/layer4_attention_weight_max": 105.05751037597656,
      "activations/layer4_attention_weight_min": -98.89344787597656,
      "activations/layer5_attention_weight_max": 79.78575897216797,
      "activations/layer5_attention_weight_min": -79.08943939208984,
      "activations/layer6_attention_weight_max": 52.90119934082031,
      "activations/layer6_attention_weight_min": -53.76725387573242,
      "activations/layer7_attention_weight_max": 78.02388000488281,
      "activations/layer7_attention_weight_min": -73.42867279052734,
      "activations/layer8_attention_weight_max": 46.07337951660156,
      "activations/layer8_attention_weight_min": -46.45429992675781,
      "activations/layer9_attention_weight_max": 53.072486877441406,
      "activations/layer9_attention_weight_min": -48.12558364868164,
      "epoch": 9.29,
      "learning_rate": 9.101893939393939e-05,
      "loss": 2.805,
      "step": 159850
    },
    {
      "activations/layer0_attention_weight_max": 15.641026496887207,
      "activations/layer0_attention_weight_min": -13.78892707824707,
      "activations/layer10_attention_weight_max": 35.00340270996094,
      "activations/layer10_attention_weight_min": -36.271400451660156,
      "activations/layer11_attention_weight_max": 30.658620834350586,
      "activations/layer11_attention_weight_min": -31.640426635742188,
      "activations/layer12_attention_weight_max": 22.07364845275879,
      "activations/layer12_attention_weight_min": -24.737390518188477,
      "activations/layer13_attention_weight_max": 31.826147079467773,
      "activations/layer13_attention_weight_min": -27.403779983520508,
      "activations/layer14_attention_weight_max": 33.69784164428711,
      "activations/layer14_attention_weight_min": -29.918245315551758,
      "activations/layer15_attention_weight_max": 31.955434799194336,
      "activations/layer15_attention_weight_min": -27.357765197753906,
      "activations/layer16_attention_weight_max": 27.791339874267578,
      "activations/layer16_attention_weight_min": -26.6223087310791,
      "activations/layer17_attention_weight_max": 49.51264953613281,
      "activations/layer17_attention_weight_min": -43.76911544799805,
      "activations/layer18_attention_weight_max": 44.37054443359375,
      "activations/layer18_attention_weight_min": -36.605403900146484,
      "activations/layer19_attention_weight_max": 21.868894577026367,
      "activations/layer19_attention_weight_min": -19.23302459716797,
      "activations/layer1_attention_weight_max": 17.058887481689453,
      "activations/layer1_attention_weight_min": -14.330830574035645,
      "activations/layer20_attention_weight_max": 20.004833221435547,
      "activations/layer20_attention_weight_min": -20.736169815063477,
      "activations/layer21_attention_weight_max": 35.40956497192383,
      "activations/layer21_attention_weight_min": -23.68100357055664,
      "activations/layer22_attention_weight_max": 30.67720603942871,
      "activations/layer22_attention_weight_min": -23.42393684387207,
      "activations/layer23_attention_weight_max": 34.34906768798828,
      "activations/layer23_attention_weight_min": -20.11568260192871,
      "activations/layer2_attention_weight_max": 31.015913009643555,
      "activations/layer2_attention_weight_min": -29.922882080078125,
      "activations/layer3_attention_weight_max": 86.88870239257812,
      "activations/layer3_attention_weight_min": -87.52263641357422,
      "activations/layer4_attention_weight_max": 96.82678985595703,
      "activations/layer4_attention_weight_min": -93.1677017211914,
      "activations/layer5_attention_weight_max": 78.98062133789062,
      "activations/layer5_attention_weight_min": -78.02912139892578,
      "activations/layer6_attention_weight_max": 53.63108444213867,
      "activations/layer6_attention_weight_min": -51.13737106323242,
      "activations/layer7_attention_weight_max": 77.89694213867188,
      "activations/layer7_attention_weight_min": -72.27535247802734,
      "activations/layer8_attention_weight_max": 45.60945129394531,
      "activations/layer8_attention_weight_min": -46.22175979614258,
      "activations/layer9_attention_weight_max": 51.436622619628906,
      "activations/layer9_attention_weight_min": -48.144168853759766,
      "epoch": 9.29,
      "learning_rate": 9.099999999999999e-05,
      "loss": 2.7976,
      "step": 159900
    },
    {
      "activations/layer0_attention_weight_max": 14.826648712158203,
      "activations/layer0_attention_weight_min": -14.402429580688477,
      "activations/layer10_attention_weight_max": 35.19739532470703,
      "activations/layer10_attention_weight_min": -34.420318603515625,
      "activations/layer11_attention_weight_max": 30.918575286865234,
      "activations/layer11_attention_weight_min": -31.857364654541016,
      "activations/layer12_attention_weight_max": 19.113773345947266,
      "activations/layer12_attention_weight_min": -26.91119956970215,
      "activations/layer13_attention_weight_max": 43.62712860107422,
      "activations/layer13_attention_weight_min": -35.562156677246094,
      "activations/layer14_attention_weight_max": 37.93419647216797,
      "activations/layer14_attention_weight_min": -31.266935348510742,
      "activations/layer15_attention_weight_max": 33.14650344848633,
      "activations/layer15_attention_weight_min": -30.51070213317871,
      "activations/layer16_attention_weight_max": 33.88687515258789,
      "activations/layer16_attention_weight_min": -28.38389015197754,
      "activations/layer17_attention_weight_max": 56.34965133666992,
      "activations/layer17_attention_weight_min": -44.90672302246094,
      "activations/layer18_attention_weight_max": 49.41215515136719,
      "activations/layer18_attention_weight_min": -37.236045837402344,
      "activations/layer19_attention_weight_max": 22.787548065185547,
      "activations/layer19_attention_weight_min": -22.54071807861328,
      "activations/layer1_attention_weight_max": 16.670440673828125,
      "activations/layer1_attention_weight_min": -15.463386535644531,
      "activations/layer20_attention_weight_max": 20.47637176513672,
      "activations/layer20_attention_weight_min": -22.967981338500977,
      "activations/layer21_attention_weight_max": 38.31095886230469,
      "activations/layer21_attention_weight_min": -24.107580184936523,
      "activations/layer22_attention_weight_max": 28.89232635498047,
      "activations/layer22_attention_weight_min": -25.864425659179688,
      "activations/layer23_attention_weight_max": 34.52058029174805,
      "activations/layer23_attention_weight_min": -25.708242416381836,
      "activations/layer2_attention_weight_max": 31.525699615478516,
      "activations/layer2_attention_weight_min": -28.5645809173584,
      "activations/layer3_attention_weight_max": 92.32396697998047,
      "activations/layer3_attention_weight_min": -90.90968322753906,
      "activations/layer4_attention_weight_max": 97.45641326904297,
      "activations/layer4_attention_weight_min": -91.82078552246094,
      "activations/layer5_attention_weight_max": 80.76834869384766,
      "activations/layer5_attention_weight_min": -78.76821899414062,
      "activations/layer6_attention_weight_max": 53.842647552490234,
      "activations/layer6_attention_weight_min": -50.5313606262207,
      "activations/layer7_attention_weight_max": 76.08892822265625,
      "activations/layer7_attention_weight_min": -73.2225112915039,
      "activations/layer8_attention_weight_max": 46.439659118652344,
      "activations/layer8_attention_weight_min": -45.8627815246582,
      "activations/layer9_attention_weight_max": 47.99595260620117,
      "activations/layer9_attention_weight_min": -47.174827575683594,
      "epoch": 9.29,
      "learning_rate": 9.098106060606059e-05,
      "loss": 2.8098,
      "step": 159950
    },
    {
      "activations/layer0_attention_weight_max": 15.150318145751953,
      "activations/layer0_attention_weight_min": -14.330552101135254,
      "activations/layer10_attention_weight_max": 41.775672912597656,
      "activations/layer10_attention_weight_min": -38.95043182373047,
      "activations/layer11_attention_weight_max": 37.69492721557617,
      "activations/layer11_attention_weight_min": -34.009368896484375,
      "activations/layer12_attention_weight_max": 18.020036697387695,
      "activations/layer12_attention_weight_min": -29.873498916625977,
      "activations/layer13_attention_weight_max": 42.81761169433594,
      "activations/layer13_attention_weight_min": -37.5698127746582,
      "activations/layer14_attention_weight_max": 36.20922088623047,
      "activations/layer14_attention_weight_min": -31.896953582763672,
      "activations/layer15_attention_weight_max": 32.3702507019043,
      "activations/layer15_attention_weight_min": -30.853416442871094,
      "activations/layer16_attention_weight_max": 30.958492279052734,
      "activations/layer16_attention_weight_min": -28.89154815673828,
      "activations/layer17_attention_weight_max": 50.89080810546875,
      "activations/layer17_attention_weight_min": -43.7145881652832,
      "activations/layer18_attention_weight_max": 45.08335494995117,
      "activations/layer18_attention_weight_min": -35.9058837890625,
      "activations/layer19_attention_weight_max": 22.175437927246094,
      "activations/layer19_attention_weight_min": -21.36701774597168,
      "activations/layer1_attention_weight_max": 16.702377319335938,
      "activations/layer1_attention_weight_min": -14.37741756439209,
      "activations/layer20_attention_weight_max": 20.508941650390625,
      "activations/layer20_attention_weight_min": -22.066661834716797,
      "activations/layer21_attention_weight_max": 38.911109924316406,
      "activations/layer21_attention_weight_min": -25.29012107849121,
      "activations/layer22_attention_weight_max": 33.1463508605957,
      "activations/layer22_attention_weight_min": -27.110986709594727,
      "activations/layer23_attention_weight_max": 36.64883041381836,
      "activations/layer23_attention_weight_min": -23.592609405517578,
      "activations/layer2_attention_weight_max": 34.137962341308594,
      "activations/layer2_attention_weight_min": -32.84373474121094,
      "activations/layer3_attention_weight_max": 94.21269226074219,
      "activations/layer3_attention_weight_min": -97.39337921142578,
      "activations/layer4_attention_weight_max": 109.65826416015625,
      "activations/layer4_attention_weight_min": -103.76873016357422,
      "activations/layer5_attention_weight_max": 83.24949645996094,
      "activations/layer5_attention_weight_min": -85.36746215820312,
      "activations/layer6_attention_weight_max": 59.5283203125,
      "activations/layer6_attention_weight_min": -59.16044616699219,
      "activations/layer7_attention_weight_max": 81.34480285644531,
      "activations/layer7_attention_weight_min": -82.22908020019531,
      "activations/layer8_attention_weight_max": 51.7763671875,
      "activations/layer8_attention_weight_min": -54.37717819213867,
      "activations/layer9_attention_weight_max": 54.876190185546875,
      "activations/layer9_attention_weight_min": -56.302345275878906,
      "epoch": 9.3,
      "learning_rate": 9.09625e-05,
      "loss": 2.7985,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_loss": 2.75,
      "eval_runtime": 8.5171,
      "eval_samples_per_second": 504.162,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.5171,
      "eval_openwebtext_samples_per_second": 504.162,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_wikitext_loss": 2.98828125,
      "eval_wikitext_ppl": 19.851533330124635,
      "eval_wikitext_runtime": 2.0026,
      "eval_wikitext_samples_per_second": 227.701,
      "step": 160000
    },
    {
      "epoch": 9.3,
      "eval_lambada_loss": 2.69140625,
      "eval_lambada_ppl": 14.75240691276831,
      "eval_lambada_runtime": 9.6185,
      "eval_lambada_samples_per_second": 506.213,
      "step": 160000
    },
    {
      "activations/layer0_attention_weight_max": 16.81181526184082,
      "activations/layer0_attention_weight_min": -13.54903793334961,
      "activations/layer10_attention_weight_max": 39.0255126953125,
      "activations/layer10_attention_weight_min": -36.55424880981445,
      "activations/layer11_attention_weight_max": 35.379234313964844,
      "activations/layer11_attention_weight_min": -31.332090377807617,
      "activations/layer12_attention_weight_max": 21.243478775024414,
      "activations/layer12_attention_weight_min": -25.838218688964844,
      "activations/layer13_attention_weight_max": 41.09934997558594,
      "activations/layer13_attention_weight_min": -31.12504768371582,
      "activations/layer14_attention_weight_max": 36.84845733642578,
      "activations/layer14_attention_weight_min": -28.635234832763672,
      "activations/layer15_attention_weight_max": 34.12221145629883,
      "activations/layer15_attention_weight_min": -28.04631805419922,
      "activations/layer16_attention_weight_max": 33.895668029785156,
      "activations/layer16_attention_weight_min": -27.270889282226562,
      "activations/layer17_attention_weight_max": 56.56638717651367,
      "activations/layer17_attention_weight_min": -42.7590217590332,
      "activations/layer18_attention_weight_max": 47.61100769042969,
      "activations/layer18_attention_weight_min": -38.98416519165039,
      "activations/layer19_attention_weight_max": 22.970935821533203,
      "activations/layer19_attention_weight_min": -24.571365356445312,
      "activations/layer1_attention_weight_max": 18.18440818786621,
      "activations/layer1_attention_weight_min": -15.378150939941406,
      "activations/layer20_attention_weight_max": 23.094139099121094,
      "activations/layer20_attention_weight_min": -23.88749885559082,
      "activations/layer21_attention_weight_max": 34.57759094238281,
      "activations/layer21_attention_weight_min": -24.432754516601562,
      "activations/layer22_attention_weight_max": 31.38467788696289,
      "activations/layer22_attention_weight_min": -23.558340072631836,
      "activations/layer23_attention_weight_max": 38.536781311035156,
      "activations/layer23_attention_weight_min": -22.585716247558594,
      "activations/layer2_attention_weight_max": 32.601776123046875,
      "activations/layer2_attention_weight_min": -30.59066390991211,
      "activations/layer3_attention_weight_max": 86.68541717529297,
      "activations/layer3_attention_weight_min": -92.8046646118164,
      "activations/layer4_attention_weight_max": 98.27488708496094,
      "activations/layer4_attention_weight_min": -93.78485107421875,
      "activations/layer5_attention_weight_max": 79.13838195800781,
      "activations/layer5_attention_weight_min": -77.4012451171875,
      "activations/layer6_attention_weight_max": 57.11201477050781,
      "activations/layer6_attention_weight_min": -53.89094543457031,
      "activations/layer7_attention_weight_max": 82.58556365966797,
      "activations/layer7_attention_weight_min": -78.20947265625,
      "activations/layer8_attention_weight_max": 47.2266845703125,
      "activations/layer8_attention_weight_min": -45.15740203857422,
      "activations/layer9_attention_weight_max": 60.22764205932617,
      "activations/layer9_attention_weight_min": -47.68653106689453,
      "epoch": 9.3,
      "learning_rate": 9.09435606060606e-05,
      "loss": 2.7643,
      "step": 160050
    },
    {
      "activations/layer0_attention_weight_max": 15.735353469848633,
      "activations/layer0_attention_weight_min": -13.471916198730469,
      "activations/layer10_attention_weight_max": 38.18820571899414,
      "activations/layer10_attention_weight_min": -37.8657112121582,
      "activations/layer11_attention_weight_max": 35.032752990722656,
      "activations/layer11_attention_weight_min": -34.3089485168457,
      "activations/layer12_attention_weight_max": 21.065256118774414,
      "activations/layer12_attention_weight_min": -25.787817001342773,
      "activations/layer13_attention_weight_max": 39.970394134521484,
      "activations/layer13_attention_weight_min": -34.53963851928711,
      "activations/layer14_attention_weight_max": 36.4912109375,
      "activations/layer14_attention_weight_min": -30.55435562133789,
      "activations/layer15_attention_weight_max": 31.627145767211914,
      "activations/layer15_attention_weight_min": -30.08142852783203,
      "activations/layer16_attention_weight_max": 29.432218551635742,
      "activations/layer16_attention_weight_min": -29.55915069580078,
      "activations/layer17_attention_weight_max": 52.03225326538086,
      "activations/layer17_attention_weight_min": -46.883853912353516,
      "activations/layer18_attention_weight_max": 44.87806701660156,
      "activations/layer18_attention_weight_min": -37.85861587524414,
      "activations/layer19_attention_weight_max": 21.103759765625,
      "activations/layer19_attention_weight_min": -22.25470733642578,
      "activations/layer1_attention_weight_max": 16.089500427246094,
      "activations/layer1_attention_weight_min": -13.776105880737305,
      "activations/layer20_attention_weight_max": 21.262592315673828,
      "activations/layer20_attention_weight_min": -25.91014862060547,
      "activations/layer21_attention_weight_max": 35.270103454589844,
      "activations/layer21_attention_weight_min": -23.048023223876953,
      "activations/layer22_attention_weight_max": 29.19296646118164,
      "activations/layer22_attention_weight_min": -26.862411499023438,
      "activations/layer23_attention_weight_max": 36.45115280151367,
      "activations/layer23_attention_weight_min": -24.200958251953125,
      "activations/layer2_attention_weight_max": 32.119651794433594,
      "activations/layer2_attention_weight_min": -31.45429229736328,
      "activations/layer3_attention_weight_max": 87.78240966796875,
      "activations/layer3_attention_weight_min": -92.4902114868164,
      "activations/layer4_attention_weight_max": 100.35039520263672,
      "activations/layer4_attention_weight_min": -97.01316833496094,
      "activations/layer5_attention_weight_max": 81.67066955566406,
      "activations/layer5_attention_weight_min": -83.24886322021484,
      "activations/layer6_attention_weight_max": 56.39142608642578,
      "activations/layer6_attention_weight_min": -56.528480529785156,
      "activations/layer7_attention_weight_max": 80.71366882324219,
      "activations/layer7_attention_weight_min": -79.8534927368164,
      "activations/layer8_attention_weight_max": 48.90465545654297,
      "activations/layer8_attention_weight_min": -49.7795524597168,
      "activations/layer9_attention_weight_max": 50.18818283081055,
      "activations/layer9_attention_weight_min": -50.91770553588867,
      "epoch": 9.3,
      "learning_rate": 9.092462121212121e-05,
      "loss": 2.7953,
      "step": 160100
    },
    {
      "activations/layer0_attention_weight_max": 15.431011199951172,
      "activations/layer0_attention_weight_min": -13.451120376586914,
      "activations/layer10_attention_weight_max": 49.36822509765625,
      "activations/layer10_attention_weight_min": -44.50129318237305,
      "activations/layer11_attention_weight_max": 43.75951385498047,
      "activations/layer11_attention_weight_min": -39.229736328125,
      "activations/layer12_attention_weight_max": 18.541349411010742,
      "activations/layer12_attention_weight_min": -31.364253997802734,
      "activations/layer13_attention_weight_max": 48.290077209472656,
      "activations/layer13_attention_weight_min": -33.15568542480469,
      "activations/layer14_attention_weight_max": 58.74631118774414,
      "activations/layer14_attention_weight_min": -30.848657608032227,
      "activations/layer15_attention_weight_max": 39.29951477050781,
      "activations/layer15_attention_weight_min": -29.645301818847656,
      "activations/layer16_attention_weight_max": 32.28790283203125,
      "activations/layer16_attention_weight_min": -27.95955467224121,
      "activations/layer17_attention_weight_max": 59.12646484375,
      "activations/layer17_attention_weight_min": -47.20158004760742,
      "activations/layer18_attention_weight_max": 48.59440612792969,
      "activations/layer18_attention_weight_min": -37.20096206665039,
      "activations/layer19_attention_weight_max": 23.327146530151367,
      "activations/layer19_attention_weight_min": -21.340627670288086,
      "activations/layer1_attention_weight_max": 16.95174789428711,
      "activations/layer1_attention_weight_min": -16.05780792236328,
      "activations/layer20_attention_weight_max": 24.554365158081055,
      "activations/layer20_attention_weight_min": -20.928295135498047,
      "activations/layer21_attention_weight_max": 47.9200439453125,
      "activations/layer21_attention_weight_min": -22.87591552734375,
      "activations/layer22_attention_weight_max": 31.7359619140625,
      "activations/layer22_attention_weight_min": -23.49357032775879,
      "activations/layer23_attention_weight_max": 40.765159606933594,
      "activations/layer23_attention_weight_min": -28.89028549194336,
      "activations/layer2_attention_weight_max": 34.1986083984375,
      "activations/layer2_attention_weight_min": -30.71184539794922,
      "activations/layer3_attention_weight_max": 93.2171630859375,
      "activations/layer3_attention_weight_min": -92.73198699951172,
      "activations/layer4_attention_weight_max": 102.77301025390625,
      "activations/layer4_attention_weight_min": -94.73889923095703,
      "activations/layer5_attention_weight_max": 82.29196166992188,
      "activations/layer5_attention_weight_min": -76.35679626464844,
      "activations/layer6_attention_weight_max": 60.96640396118164,
      "activations/layer6_attention_weight_min": -54.960418701171875,
      "activations/layer7_attention_weight_max": 90.39669036865234,
      "activations/layer7_attention_weight_min": -82.94648742675781,
      "activations/layer8_attention_weight_max": 53.05928421020508,
      "activations/layer8_attention_weight_min": -50.8046760559082,
      "activations/layer9_attention_weight_max": 64.60021209716797,
      "activations/layer9_attention_weight_min": -51.75639724731445,
      "epoch": 9.31,
      "learning_rate": 9.09056818181818e-05,
      "loss": 2.801,
      "step": 160150
    },
    {
      "activations/layer0_attention_weight_max": 14.254400253295898,
      "activations/layer0_attention_weight_min": -13.880108833312988,
      "activations/layer10_attention_weight_max": 41.700164794921875,
      "activations/layer10_attention_weight_min": -40.81296157836914,
      "activations/layer11_attention_weight_max": 34.34185028076172,
      "activations/layer11_attention_weight_min": -34.57302474975586,
      "activations/layer12_attention_weight_max": 23.650753021240234,
      "activations/layer12_attention_weight_min": -22.68386459350586,
      "activations/layer13_attention_weight_max": 51.5601692199707,
      "activations/layer13_attention_weight_min": -33.25077819824219,
      "activations/layer14_attention_weight_max": 42.65631103515625,
      "activations/layer14_attention_weight_min": -31.763402938842773,
      "activations/layer15_attention_weight_max": 37.43345260620117,
      "activations/layer15_attention_weight_min": -28.48060417175293,
      "activations/layer16_attention_weight_max": 36.85153579711914,
      "activations/layer16_attention_weight_min": -30.432926177978516,
      "activations/layer17_attention_weight_max": 59.72733688354492,
      "activations/layer17_attention_weight_min": -45.9799690246582,
      "activations/layer18_attention_weight_max": 51.87664794921875,
      "activations/layer18_attention_weight_min": -36.61703109741211,
      "activations/layer19_attention_weight_max": 25.615985870361328,
      "activations/layer19_attention_weight_min": -20.292814254760742,
      "activations/layer1_attention_weight_max": 17.7799015045166,
      "activations/layer1_attention_weight_min": -14.122522354125977,
      "activations/layer20_attention_weight_max": 23.13649559020996,
      "activations/layer20_attention_weight_min": -20.485258102416992,
      "activations/layer21_attention_weight_max": 43.09373474121094,
      "activations/layer21_attention_weight_min": -21.720184326171875,
      "activations/layer22_attention_weight_max": 34.42766571044922,
      "activations/layer22_attention_weight_min": -25.58320426940918,
      "activations/layer23_attention_weight_max": 48.668556213378906,
      "activations/layer23_attention_weight_min": -23.5282039642334,
      "activations/layer2_attention_weight_max": 31.531890869140625,
      "activations/layer2_attention_weight_min": -30.051284790039062,
      "activations/layer3_attention_weight_max": 90.95439147949219,
      "activations/layer3_attention_weight_min": -97.97418212890625,
      "activations/layer4_attention_weight_max": 101.43175506591797,
      "activations/layer4_attention_weight_min": -99.99102020263672,
      "activations/layer5_attention_weight_max": 83.32568359375,
      "activations/layer5_attention_weight_min": -79.97178649902344,
      "activations/layer6_attention_weight_max": 57.88301467895508,
      "activations/layer6_attention_weight_min": -58.85144805908203,
      "activations/layer7_attention_weight_max": 83.91832733154297,
      "activations/layer7_attention_weight_min": -83.74195861816406,
      "activations/layer8_attention_weight_max": 49.91661071777344,
      "activations/layer8_attention_weight_min": -49.26951217651367,
      "activations/layer9_attention_weight_max": 54.24333572387695,
      "activations/layer9_attention_weight_min": -53.982398986816406,
      "epoch": 9.31,
      "learning_rate": 9.088674242424241e-05,
      "loss": 2.7918,
      "step": 160200
    },
    {
      "activations/layer0_attention_weight_max": 16.008670806884766,
      "activations/layer0_attention_weight_min": -13.73486328125,
      "activations/layer10_attention_weight_max": 44.65957260131836,
      "activations/layer10_attention_weight_min": -39.547218322753906,
      "activations/layer11_attention_weight_max": 35.69843673706055,
      "activations/layer11_attention_weight_min": -33.51683044433594,
      "activations/layer12_attention_weight_max": 23.200315475463867,
      "activations/layer12_attention_weight_min": -27.070627212524414,
      "activations/layer13_attention_weight_max": 49.47056198120117,
      "activations/layer13_attention_weight_min": -35.540626525878906,
      "activations/layer14_attention_weight_max": 53.30481719970703,
      "activations/layer14_attention_weight_min": -32.001625061035156,
      "activations/layer15_attention_weight_max": 42.277164459228516,
      "activations/layer15_attention_weight_min": -32.30582809448242,
      "activations/layer16_attention_weight_max": 37.56486511230469,
      "activations/layer16_attention_weight_min": -31.67027473449707,
      "activations/layer17_attention_weight_max": 65.94717407226562,
      "activations/layer17_attention_weight_min": -48.33941650390625,
      "activations/layer18_attention_weight_max": 62.30097579956055,
      "activations/layer18_attention_weight_min": -39.393341064453125,
      "activations/layer19_attention_weight_max": 25.959854125976562,
      "activations/layer19_attention_weight_min": -21.158405303955078,
      "activations/layer1_attention_weight_max": 17.15402603149414,
      "activations/layer1_attention_weight_min": -15.12188720703125,
      "activations/layer20_attention_weight_max": 26.271345138549805,
      "activations/layer20_attention_weight_min": -20.348350524902344,
      "activations/layer21_attention_weight_max": 45.9035530090332,
      "activations/layer21_attention_weight_min": -28.137529373168945,
      "activations/layer22_attention_weight_max": 34.61439514160156,
      "activations/layer22_attention_weight_min": -25.2616024017334,
      "activations/layer23_attention_weight_max": 46.80592346191406,
      "activations/layer23_attention_weight_min": -24.47344207763672,
      "activations/layer2_attention_weight_max": 31.108474731445312,
      "activations/layer2_attention_weight_min": -29.67139434814453,
      "activations/layer3_attention_weight_max": 88.36234283447266,
      "activations/layer3_attention_weight_min": -89.6219482421875,
      "activations/layer4_attention_weight_max": 98.9602279663086,
      "activations/layer4_attention_weight_min": -97.37734985351562,
      "activations/layer5_attention_weight_max": 82.98341369628906,
      "activations/layer5_attention_weight_min": -79.24163055419922,
      "activations/layer6_attention_weight_max": 56.32830810546875,
      "activations/layer6_attention_weight_min": -55.6454963684082,
      "activations/layer7_attention_weight_max": 81.25930786132812,
      "activations/layer7_attention_weight_min": -78.4493637084961,
      "activations/layer8_attention_weight_max": 47.634498596191406,
      "activations/layer8_attention_weight_min": -49.13620376586914,
      "activations/layer9_attention_weight_max": 54.151668548583984,
      "activations/layer9_attention_weight_min": -53.588043212890625,
      "epoch": 9.31,
      "learning_rate": 9.086780303030303e-05,
      "loss": 2.8037,
      "step": 160250
    },
    {
      "activations/layer0_attention_weight_max": 15.610965728759766,
      "activations/layer0_attention_weight_min": -14.013535499572754,
      "activations/layer10_attention_weight_max": 36.35921859741211,
      "activations/layer10_attention_weight_min": -37.04801940917969,
      "activations/layer11_attention_weight_max": 32.10853958129883,
      "activations/layer11_attention_weight_min": -33.02430725097656,
      "activations/layer12_attention_weight_max": 18.255788803100586,
      "activations/layer12_attention_weight_min": -27.089990615844727,
      "activations/layer13_attention_weight_max": 37.14235305786133,
      "activations/layer13_attention_weight_min": -31.07807159423828,
      "activations/layer14_attention_weight_max": 36.49258804321289,
      "activations/layer14_attention_weight_min": -30.634883880615234,
      "activations/layer15_attention_weight_max": 32.89625549316406,
      "activations/layer15_attention_weight_min": -28.238855361938477,
      "activations/layer16_attention_weight_max": 29.49873161315918,
      "activations/layer16_attention_weight_min": -26.71293067932129,
      "activations/layer17_attention_weight_max": 49.31911087036133,
      "activations/layer17_attention_weight_min": -41.844520568847656,
      "activations/layer18_attention_weight_max": 46.51690673828125,
      "activations/layer18_attention_weight_min": -37.45701217651367,
      "activations/layer19_attention_weight_max": 19.906091690063477,
      "activations/layer19_attention_weight_min": -22.056989669799805,
      "activations/layer1_attention_weight_max": 17.534881591796875,
      "activations/layer1_attention_weight_min": -15.419598579406738,
      "activations/layer20_attention_weight_max": 22.312299728393555,
      "activations/layer20_attention_weight_min": -23.19240951538086,
      "activations/layer21_attention_weight_max": 36.86214065551758,
      "activations/layer21_attention_weight_min": -22.331506729125977,
      "activations/layer22_attention_weight_max": 31.32278060913086,
      "activations/layer22_attention_weight_min": -28.648910522460938,
      "activations/layer23_attention_weight_max": 35.98736572265625,
      "activations/layer23_attention_weight_min": -21.571210861206055,
      "activations/layer2_attention_weight_max": 32.41188049316406,
      "activations/layer2_attention_weight_min": -30.03474235534668,
      "activations/layer3_attention_weight_max": 89.48221588134766,
      "activations/layer3_attention_weight_min": -95.26155853271484,
      "activations/layer4_attention_weight_max": 98.1742935180664,
      "activations/layer4_attention_weight_min": -94.3838119506836,
      "activations/layer5_attention_weight_max": 81.21307373046875,
      "activations/layer5_attention_weight_min": -79.04287719726562,
      "activations/layer6_attention_weight_max": 56.28327560424805,
      "activations/layer6_attention_weight_min": -56.30480194091797,
      "activations/layer7_attention_weight_max": 75.99686431884766,
      "activations/layer7_attention_weight_min": -75.56845092773438,
      "activations/layer8_attention_weight_max": 46.43510818481445,
      "activations/layer8_attention_weight_min": -47.75780487060547,
      "activations/layer9_attention_weight_max": 52.742244720458984,
      "activations/layer9_attention_weight_min": -51.64702224731445,
      "epoch": 9.31,
      "learning_rate": 9.084886363636362e-05,
      "loss": 2.7856,
      "step": 160300
    },
    {
      "activations/layer0_attention_weight_max": 15.23864459991455,
      "activations/layer0_attention_weight_min": -13.74445629119873,
      "activations/layer10_attention_weight_max": 36.88434600830078,
      "activations/layer10_attention_weight_min": -36.347984313964844,
      "activations/layer11_attention_weight_max": 33.290618896484375,
      "activations/layer11_attention_weight_min": -31.760229110717773,
      "activations/layer12_attention_weight_max": 19.686418533325195,
      "activations/layer12_attention_weight_min": -27.174976348876953,
      "activations/layer13_attention_weight_max": 37.05241012573242,
      "activations/layer13_attention_weight_min": -28.402568817138672,
      "activations/layer14_attention_weight_max": 38.376949310302734,
      "activations/layer14_attention_weight_min": -30.938350677490234,
      "activations/layer15_attention_weight_max": 35.05238342285156,
      "activations/layer15_attention_weight_min": -31.264184951782227,
      "activations/layer16_attention_weight_max": 32.35652542114258,
      "activations/layer16_attention_weight_min": -32.540000915527344,
      "activations/layer17_attention_weight_max": 52.894439697265625,
      "activations/layer17_attention_weight_min": -45.08992385864258,
      "activations/layer18_attention_weight_max": 47.63534927368164,
      "activations/layer18_attention_weight_min": -38.498146057128906,
      "activations/layer19_attention_weight_max": 23.514331817626953,
      "activations/layer19_attention_weight_min": -20.8896484375,
      "activations/layer1_attention_weight_max": 16.182727813720703,
      "activations/layer1_attention_weight_min": -13.275153160095215,
      "activations/layer20_attention_weight_max": 21.97738265991211,
      "activations/layer20_attention_weight_min": -23.21499252319336,
      "activations/layer21_attention_weight_max": 36.89326095581055,
      "activations/layer21_attention_weight_min": -23.223901748657227,
      "activations/layer22_attention_weight_max": 32.791385650634766,
      "activations/layer22_attention_weight_min": -26.95771026611328,
      "activations/layer23_attention_weight_max": 37.93840026855469,
      "activations/layer23_attention_weight_min": -21.136917114257812,
      "activations/layer2_attention_weight_max": 30.174488067626953,
      "activations/layer2_attention_weight_min": -28.690624237060547,
      "activations/layer3_attention_weight_max": 87.55256652832031,
      "activations/layer3_attention_weight_min": -91.48211669921875,
      "activations/layer4_attention_weight_max": 99.99372100830078,
      "activations/layer4_attention_weight_min": -97.20634460449219,
      "activations/layer5_attention_weight_max": 78.87158966064453,
      "activations/layer5_attention_weight_min": -78.70111083984375,
      "activations/layer6_attention_weight_max": 54.233741760253906,
      "activations/layer6_attention_weight_min": -53.58518600463867,
      "activations/layer7_attention_weight_max": 78.04853820800781,
      "activations/layer7_attention_weight_min": -78.1507797241211,
      "activations/layer8_attention_weight_max": 45.04068374633789,
      "activations/layer8_attention_weight_min": -46.09360885620117,
      "activations/layer9_attention_weight_max": 51.3447380065918,
      "activations/layer9_attention_weight_min": -51.62443923950195,
      "epoch": 9.32,
      "learning_rate": 9.082992424242423e-05,
      "loss": 2.8047,
      "step": 160350
    },
    {
      "activations/layer0_attention_weight_max": 14.846924781799316,
      "activations/layer0_attention_weight_min": -13.741129875183105,
      "activations/layer10_attention_weight_max": 51.54096221923828,
      "activations/layer10_attention_weight_min": -47.73926544189453,
      "activations/layer11_attention_weight_max": 45.08100128173828,
      "activations/layer11_attention_weight_min": -40.51361846923828,
      "activations/layer12_attention_weight_max": 35.767616271972656,
      "activations/layer12_attention_weight_min": -29.24713706970215,
      "activations/layer13_attention_weight_max": 86.04926300048828,
      "activations/layer13_attention_weight_min": -38.845069885253906,
      "activations/layer14_attention_weight_max": 77.42255401611328,
      "activations/layer14_attention_weight_min": -39.21984100341797,
      "activations/layer15_attention_weight_max": 63.2608528137207,
      "activations/layer15_attention_weight_min": -32.20502853393555,
      "activations/layer16_attention_weight_max": 37.54275894165039,
      "activations/layer16_attention_weight_min": -33.49845886230469,
      "activations/layer17_attention_weight_max": 65.89222717285156,
      "activations/layer17_attention_weight_min": -51.94990158081055,
      "activations/layer18_attention_weight_max": 54.13615417480469,
      "activations/layer18_attention_weight_min": -43.26522445678711,
      "activations/layer19_attention_weight_max": 26.145750045776367,
      "activations/layer19_attention_weight_min": -24.48064422607422,
      "activations/layer1_attention_weight_max": 18.33087921142578,
      "activations/layer1_attention_weight_min": -15.074108123779297,
      "activations/layer20_attention_weight_max": 26.982492446899414,
      "activations/layer20_attention_weight_min": -21.565324783325195,
      "activations/layer21_attention_weight_max": 51.8870849609375,
      "activations/layer21_attention_weight_min": -25.631162643432617,
      "activations/layer22_attention_weight_max": 35.75057601928711,
      "activations/layer22_attention_weight_min": -25.567787170410156,
      "activations/layer23_attention_weight_max": 39.36786651611328,
      "activations/layer23_attention_weight_min": -24.777677536010742,
      "activations/layer2_attention_weight_max": 34.938507080078125,
      "activations/layer2_attention_weight_min": -30.793148040771484,
      "activations/layer3_attention_weight_max": 98.11653137207031,
      "activations/layer3_attention_weight_min": -92.71815490722656,
      "activations/layer4_attention_weight_max": 108.29388427734375,
      "activations/layer4_attention_weight_min": -101.84004974365234,
      "activations/layer5_attention_weight_max": 86.51656341552734,
      "activations/layer5_attention_weight_min": -83.4771957397461,
      "activations/layer6_attention_weight_max": 60.76979064941406,
      "activations/layer6_attention_weight_min": -57.43445587158203,
      "activations/layer7_attention_weight_max": 96.84139251708984,
      "activations/layer7_attention_weight_min": -91.51740264892578,
      "activations/layer8_attention_weight_max": 55.85483932495117,
      "activations/layer8_attention_weight_min": -56.78082275390625,
      "activations/layer9_attention_weight_max": 67.58953094482422,
      "activations/layer9_attention_weight_min": -58.35316848754883,
      "epoch": 9.32,
      "learning_rate": 9.081098484848485e-05,
      "loss": 2.7973,
      "step": 160400
    },
    {
      "activations/layer0_attention_weight_max": 15.955659866333008,
      "activations/layer0_attention_weight_min": -13.46032428741455,
      "activations/layer10_attention_weight_max": 35.2935791015625,
      "activations/layer10_attention_weight_min": -35.773223876953125,
      "activations/layer11_attention_weight_max": 31.12936019897461,
      "activations/layer11_attention_weight_min": -31.030752182006836,
      "activations/layer12_attention_weight_max": 16.822296142578125,
      "activations/layer12_attention_weight_min": -31.489809036254883,
      "activations/layer13_attention_weight_max": 35.197757720947266,
      "activations/layer13_attention_weight_min": -30.355173110961914,
      "activations/layer14_attention_weight_max": 34.10061264038086,
      "activations/layer14_attention_weight_min": -31.068084716796875,
      "activations/layer15_attention_weight_max": 30.672958374023438,
      "activations/layer15_attention_weight_min": -29.4031925201416,
      "activations/layer16_attention_weight_max": 29.208372116088867,
      "activations/layer16_attention_weight_min": -30.260087966918945,
      "activations/layer17_attention_weight_max": 50.16394805908203,
      "activations/layer17_attention_weight_min": -41.92835235595703,
      "activations/layer18_attention_weight_max": 42.68318176269531,
      "activations/layer18_attention_weight_min": -36.76268768310547,
      "activations/layer19_attention_weight_max": 21.516592025756836,
      "activations/layer19_attention_weight_min": -20.752077102661133,
      "activations/layer1_attention_weight_max": 17.47728729248047,
      "activations/layer1_attention_weight_min": -15.029186248779297,
      "activations/layer20_attention_weight_max": 21.181211471557617,
      "activations/layer20_attention_weight_min": -21.94706916809082,
      "activations/layer21_attention_weight_max": 35.01136779785156,
      "activations/layer21_attention_weight_min": -22.564577102661133,
      "activations/layer22_attention_weight_max": 28.10626792907715,
      "activations/layer22_attention_weight_min": -26.06397247314453,
      "activations/layer23_attention_weight_max": 33.69198226928711,
      "activations/layer23_attention_weight_min": -24.323522567749023,
      "activations/layer2_attention_weight_max": 32.17285919189453,
      "activations/layer2_attention_weight_min": -29.666194915771484,
      "activations/layer3_attention_weight_max": 94.83496856689453,
      "activations/layer3_attention_weight_min": -91.6789779663086,
      "activations/layer4_attention_weight_max": 105.14397430419922,
      "activations/layer4_attention_weight_min": -96.7858657836914,
      "activations/layer5_attention_weight_max": 76.25627136230469,
      "activations/layer5_attention_weight_min": -79.22455596923828,
      "activations/layer6_attention_weight_max": 51.76072311401367,
      "activations/layer6_attention_weight_min": -51.74009704589844,
      "activations/layer7_attention_weight_max": 79.47415161132812,
      "activations/layer7_attention_weight_min": -75.4695053100586,
      "activations/layer8_attention_weight_max": 45.312747955322266,
      "activations/layer8_attention_weight_min": -43.32987976074219,
      "activations/layer9_attention_weight_max": 58.3228759765625,
      "activations/layer9_attention_weight_min": -53.76591873168945,
      "epoch": 9.32,
      "learning_rate": 9.079204545454544e-05,
      "loss": 2.8141,
      "step": 160450
    },
    {
      "activations/layer0_attention_weight_max": 15.046490669250488,
      "activations/layer0_attention_weight_min": -14.10778522491455,
      "activations/layer10_attention_weight_max": 37.34645080566406,
      "activations/layer10_attention_weight_min": -34.21562576293945,
      "activations/layer11_attention_weight_max": 33.99772262573242,
      "activations/layer11_attention_weight_min": -31.19679069519043,
      "activations/layer12_attention_weight_max": 17.882610321044922,
      "activations/layer12_attention_weight_min": -22.935388565063477,
      "activations/layer13_attention_weight_max": 41.47964096069336,
      "activations/layer13_attention_weight_min": -29.485139846801758,
      "activations/layer14_attention_weight_max": 34.787200927734375,
      "activations/layer14_attention_weight_min": -31.217811584472656,
      "activations/layer15_attention_weight_max": 34.17826843261719,
      "activations/layer15_attention_weight_min": -30.039505004882812,
      "activations/layer16_attention_weight_max": 28.88909912109375,
      "activations/layer16_attention_weight_min": -29.40669059753418,
      "activations/layer17_attention_weight_max": 48.11658477783203,
      "activations/layer17_attention_weight_min": -43.55533981323242,
      "activations/layer18_attention_weight_max": 46.73664474487305,
      "activations/layer18_attention_weight_min": -37.433616638183594,
      "activations/layer19_attention_weight_max": 27.725284576416016,
      "activations/layer19_attention_weight_min": -23.506505966186523,
      "activations/layer1_attention_weight_max": 16.089075088500977,
      "activations/layer1_attention_weight_min": -15.230960845947266,
      "activations/layer20_attention_weight_max": 23.29722785949707,
      "activations/layer20_attention_weight_min": -26.70806884765625,
      "activations/layer21_attention_weight_max": 41.58359909057617,
      "activations/layer21_attention_weight_min": -23.262990951538086,
      "activations/layer22_attention_weight_max": 31.98634910583496,
      "activations/layer22_attention_weight_min": -26.868677139282227,
      "activations/layer23_attention_weight_max": 37.029727935791016,
      "activations/layer23_attention_weight_min": -24.578060150146484,
      "activations/layer2_attention_weight_max": 31.31711196899414,
      "activations/layer2_attention_weight_min": -30.85232162475586,
      "activations/layer3_attention_weight_max": 90.02379608154297,
      "activations/layer3_attention_weight_min": -92.73815155029297,
      "activations/layer4_attention_weight_max": 97.83563995361328,
      "activations/layer4_attention_weight_min": -93.63270568847656,
      "activations/layer5_attention_weight_max": 79.46426391601562,
      "activations/layer5_attention_weight_min": -77.51007843017578,
      "activations/layer6_attention_weight_max": 53.19788360595703,
      "activations/layer6_attention_weight_min": -55.54557418823242,
      "activations/layer7_attention_weight_max": 79.2645034790039,
      "activations/layer7_attention_weight_min": -72.87421417236328,
      "activations/layer8_attention_weight_max": 42.593177795410156,
      "activations/layer8_attention_weight_min": -44.18971633911133,
      "activations/layer9_attention_weight_max": 47.989131927490234,
      "activations/layer9_attention_weight_min": -47.25914764404297,
      "epoch": 9.33,
      "learning_rate": 9.077310606060605e-05,
      "loss": 2.798,
      "step": 160500
    },
    {
      "activations/layer0_attention_weight_max": 14.6622953414917,
      "activations/layer0_attention_weight_min": -13.864973068237305,
      "activations/layer10_attention_weight_max": 35.253326416015625,
      "activations/layer10_attention_weight_min": -34.259788513183594,
      "activations/layer11_attention_weight_max": 30.417499542236328,
      "activations/layer11_attention_weight_min": -30.61224937438965,
      "activations/layer12_attention_weight_max": 22.091339111328125,
      "activations/layer12_attention_weight_min": -25.357105255126953,
      "activations/layer13_attention_weight_max": 46.953006744384766,
      "activations/layer13_attention_weight_min": -31.455821990966797,
      "activations/layer14_attention_weight_max": 34.00289535522461,
      "activations/layer14_attention_weight_min": -30.38132667541504,
      "activations/layer15_attention_weight_max": 33.40927505493164,
      "activations/layer15_attention_weight_min": -29.50917625427246,
      "activations/layer16_attention_weight_max": 31.738094329833984,
      "activations/layer16_attention_weight_min": -28.173538208007812,
      "activations/layer17_attention_weight_max": 49.47848892211914,
      "activations/layer17_attention_weight_min": -41.62513732910156,
      "activations/layer18_attention_weight_max": 44.038787841796875,
      "activations/layer18_attention_weight_min": -35.20980453491211,
      "activations/layer19_attention_weight_max": 22.078886032104492,
      "activations/layer19_attention_weight_min": -21.202632904052734,
      "activations/layer1_attention_weight_max": 16.270109176635742,
      "activations/layer1_attention_weight_min": -16.50678825378418,
      "activations/layer20_attention_weight_max": 20.793601989746094,
      "activations/layer20_attention_weight_min": -20.407291412353516,
      "activations/layer21_attention_weight_max": 40.828712463378906,
      "activations/layer21_attention_weight_min": -22.769929885864258,
      "activations/layer22_attention_weight_max": 35.12067413330078,
      "activations/layer22_attention_weight_min": -25.51389503479004,
      "activations/layer23_attention_weight_max": 36.449405670166016,
      "activations/layer23_attention_weight_min": -24.068862915039062,
      "activations/layer2_attention_weight_max": 33.754676818847656,
      "activations/layer2_attention_weight_min": -30.80055809020996,
      "activations/layer3_attention_weight_max": 92.39104461669922,
      "activations/layer3_attention_weight_min": -90.15520477294922,
      "activations/layer4_attention_weight_max": 97.36119079589844,
      "activations/layer4_attention_weight_min": -92.02403259277344,
      "activations/layer5_attention_weight_max": 78.82063293457031,
      "activations/layer5_attention_weight_min": -75.93923950195312,
      "activations/layer6_attention_weight_max": 53.26029968261719,
      "activations/layer6_attention_weight_min": -49.68379592895508,
      "activations/layer7_attention_weight_max": 80.47103118896484,
      "activations/layer7_attention_weight_min": -72.68049621582031,
      "activations/layer8_attention_weight_max": 44.89301300048828,
      "activations/layer8_attention_weight_min": -45.353389739990234,
      "activations/layer9_attention_weight_max": 50.06086349487305,
      "activations/layer9_attention_weight_min": -44.899131774902344,
      "epoch": 9.33,
      "learning_rate": 9.075416666666667e-05,
      "loss": 2.7911,
      "step": 160550
    },
    {
      "activations/layer0_attention_weight_max": 15.09742546081543,
      "activations/layer0_attention_weight_min": -13.973552703857422,
      "activations/layer10_attention_weight_max": 39.25910186767578,
      "activations/layer10_attention_weight_min": -36.03911209106445,
      "activations/layer11_attention_weight_max": 33.73280715942383,
      "activations/layer11_attention_weight_min": -31.524839401245117,
      "activations/layer12_attention_weight_max": 20.359094619750977,
      "activations/layer12_attention_weight_min": -25.203189849853516,
      "activations/layer13_attention_weight_max": 40.24607849121094,
      "activations/layer13_attention_weight_min": -32.67622756958008,
      "activations/layer14_attention_weight_max": 39.594139099121094,
      "activations/layer14_attention_weight_min": -31.961034774780273,
      "activations/layer15_attention_weight_max": 34.13387680053711,
      "activations/layer15_attention_weight_min": -29.36159324645996,
      "activations/layer16_attention_weight_max": 32.92483901977539,
      "activations/layer16_attention_weight_min": -29.33076286315918,
      "activations/layer17_attention_weight_max": 55.44129180908203,
      "activations/layer17_attention_weight_min": -47.028831481933594,
      "activations/layer18_attention_weight_max": 53.83563232421875,
      "activations/layer18_attention_weight_min": -39.363243103027344,
      "activations/layer19_attention_weight_max": 22.886320114135742,
      "activations/layer19_attention_weight_min": -21.618173599243164,
      "activations/layer1_attention_weight_max": 16.773880004882812,
      "activations/layer1_attention_weight_min": -14.836267471313477,
      "activations/layer20_attention_weight_max": 23.116785049438477,
      "activations/layer20_attention_weight_min": -19.852231979370117,
      "activations/layer21_attention_weight_max": 43.40970993041992,
      "activations/layer21_attention_weight_min": -23.460874557495117,
      "activations/layer22_attention_weight_max": 32.44327163696289,
      "activations/layer22_attention_weight_min": -26.76276969909668,
      "activations/layer23_attention_weight_max": 38.22686767578125,
      "activations/layer23_attention_weight_min": -22.965450286865234,
      "activations/layer2_attention_weight_max": 34.749446868896484,
      "activations/layer2_attention_weight_min": -31.701034545898438,
      "activations/layer3_attention_weight_max": 87.6093978881836,
      "activations/layer3_attention_weight_min": -88.07792663574219,
      "activations/layer4_attention_weight_max": 98.0782241821289,
      "activations/layer4_attention_weight_min": -91.67244720458984,
      "activations/layer5_attention_weight_max": 79.3642807006836,
      "activations/layer5_attention_weight_min": -84.8532943725586,
      "activations/layer6_attention_weight_max": 54.478187561035156,
      "activations/layer6_attention_weight_min": -52.394065856933594,
      "activations/layer7_attention_weight_max": 76.53584289550781,
      "activations/layer7_attention_weight_min": -73.45714569091797,
      "activations/layer8_attention_weight_max": 45.01580047607422,
      "activations/layer8_attention_weight_min": -47.67869186401367,
      "activations/layer9_attention_weight_max": 50.26941680908203,
      "activations/layer9_attention_weight_min": -49.07725143432617,
      "epoch": 9.33,
      "learning_rate": 9.073522727272727e-05,
      "loss": 2.793,
      "step": 160600
    },
    {
      "activations/layer0_attention_weight_max": 15.755338668823242,
      "activations/layer0_attention_weight_min": -13.815202713012695,
      "activations/layer10_attention_weight_max": 43.8365364074707,
      "activations/layer10_attention_weight_min": -41.163726806640625,
      "activations/layer11_attention_weight_max": 39.72148895263672,
      "activations/layer11_attention_weight_min": -36.163055419921875,
      "activations/layer12_attention_weight_max": 17.8093318939209,
      "activations/layer12_attention_weight_min": -27.25605010986328,
      "activations/layer13_attention_weight_max": 46.058834075927734,
      "activations/layer13_attention_weight_min": -38.56816864013672,
      "activations/layer14_attention_weight_max": 50.81121826171875,
      "activations/layer14_attention_weight_min": -38.58604049682617,
      "activations/layer15_attention_weight_max": 41.90857696533203,
      "activations/layer15_attention_weight_min": -30.752199172973633,
      "activations/layer16_attention_weight_max": 29.717859268188477,
      "activations/layer16_attention_weight_min": -28.0573787689209,
      "activations/layer17_attention_weight_max": 50.72080993652344,
      "activations/layer17_attention_weight_min": -47.23381042480469,
      "activations/layer18_attention_weight_max": 45.44050979614258,
      "activations/layer18_attention_weight_min": -40.5190315246582,
      "activations/layer19_attention_weight_max": 24.036884307861328,
      "activations/layer19_attention_weight_min": -23.570302963256836,
      "activations/layer1_attention_weight_max": 17.144617080688477,
      "activations/layer1_attention_weight_min": -14.53115463256836,
      "activations/layer20_attention_weight_max": 25.552989959716797,
      "activations/layer20_attention_weight_min": -21.377140045166016,
      "activations/layer21_attention_weight_max": 41.6921501159668,
      "activations/layer21_attention_weight_min": -24.446941375732422,
      "activations/layer22_attention_weight_max": 31.21868133544922,
      "activations/layer22_attention_weight_min": -24.026466369628906,
      "activations/layer23_attention_weight_max": 37.783260345458984,
      "activations/layer23_attention_weight_min": -22.200153350830078,
      "activations/layer2_attention_weight_max": 34.24871063232422,
      "activations/layer2_attention_weight_min": -31.788341522216797,
      "activations/layer3_attention_weight_max": 94.33409118652344,
      "activations/layer3_attention_weight_min": -91.62335205078125,
      "activations/layer4_attention_weight_max": 105.89044189453125,
      "activations/layer4_attention_weight_min": -99.19121551513672,
      "activations/layer5_attention_weight_max": 80.4762191772461,
      "activations/layer5_attention_weight_min": -75.10592651367188,
      "activations/layer6_attention_weight_max": 56.1501579284668,
      "activations/layer6_attention_weight_min": -52.28261947631836,
      "activations/layer7_attention_weight_max": 80.3118896484375,
      "activations/layer7_attention_weight_min": -75.27056884765625,
      "activations/layer8_attention_weight_max": 51.41981887817383,
      "activations/layer8_attention_weight_min": -47.57659149169922,
      "activations/layer9_attention_weight_max": 65.83135986328125,
      "activations/layer9_attention_weight_min": -54.565948486328125,
      "epoch": 9.33,
      "learning_rate": 9.071628787878787e-05,
      "loss": 2.7981,
      "step": 160650
    },
    {
      "activations/layer0_attention_weight_max": 14.610766410827637,
      "activations/layer0_attention_weight_min": -13.929859161376953,
      "activations/layer10_attention_weight_max": 47.3289680480957,
      "activations/layer10_attention_weight_min": -40.25693130493164,
      "activations/layer11_attention_weight_max": 39.253822326660156,
      "activations/layer11_attention_weight_min": -33.5128059387207,
      "activations/layer12_attention_weight_max": 22.656421661376953,
      "activations/layer12_attention_weight_min": -28.35787582397461,
      "activations/layer13_attention_weight_max": 51.84629821777344,
      "activations/layer13_attention_weight_min": -31.791179656982422,
      "activations/layer14_attention_weight_max": 54.20889663696289,
      "activations/layer14_attention_weight_min": -33.07941818237305,
      "activations/layer15_attention_weight_max": 48.12565612792969,
      "activations/layer15_attention_weight_min": -30.434091567993164,
      "activations/layer16_attention_weight_max": 34.1883544921875,
      "activations/layer16_attention_weight_min": -30.234790802001953,
      "activations/layer17_attention_weight_max": 62.47933578491211,
      "activations/layer17_attention_weight_min": -49.719120025634766,
      "activations/layer18_attention_weight_max": 52.65998458862305,
      "activations/layer18_attention_weight_min": -37.99192810058594,
      "activations/layer19_attention_weight_max": 26.539779663085938,
      "activations/layer19_attention_weight_min": -21.273252487182617,
      "activations/layer1_attention_weight_max": 17.84606170654297,
      "activations/layer1_attention_weight_min": -14.706239700317383,
      "activations/layer20_attention_weight_max": 26.194135665893555,
      "activations/layer20_attention_weight_min": -21.249500274658203,
      "activations/layer21_attention_weight_max": 48.90738296508789,
      "activations/layer21_attention_weight_min": -25.980003356933594,
      "activations/layer22_attention_weight_max": 35.899314880371094,
      "activations/layer22_attention_weight_min": -25.611494064331055,
      "activations/layer23_attention_weight_max": 44.94424057006836,
      "activations/layer23_attention_weight_min": -25.362638473510742,
      "activations/layer2_attention_weight_max": 33.61820983886719,
      "activations/layer2_attention_weight_min": -32.77587890625,
      "activations/layer3_attention_weight_max": 92.85059356689453,
      "activations/layer3_attention_weight_min": -95.11758422851562,
      "activations/layer4_attention_weight_max": 101.72948455810547,
      "activations/layer4_attention_weight_min": -95.072998046875,
      "activations/layer5_attention_weight_max": 79.6384048461914,
      "activations/layer5_attention_weight_min": -81.5712890625,
      "activations/layer6_attention_weight_max": 54.19929885864258,
      "activations/layer6_attention_weight_min": -54.37493133544922,
      "activations/layer7_attention_weight_max": 84.9352798461914,
      "activations/layer7_attention_weight_min": -79.45167541503906,
      "activations/layer8_attention_weight_max": 53.01017379760742,
      "activations/layer8_attention_weight_min": -48.69060134887695,
      "activations/layer9_attention_weight_max": 65.53668975830078,
      "activations/layer9_attention_weight_min": -54.78487014770508,
      "epoch": 9.34,
      "learning_rate": 9.069734848484847e-05,
      "loss": 2.8021,
      "step": 160700
    },
    {
      "activations/layer0_attention_weight_max": 17.785497665405273,
      "activations/layer0_attention_weight_min": -13.654507637023926,
      "activations/layer10_attention_weight_max": 37.18404006958008,
      "activations/layer10_attention_weight_min": -35.85163116455078,
      "activations/layer11_attention_weight_max": 33.84642028808594,
      "activations/layer11_attention_weight_min": -30.907094955444336,
      "activations/layer12_attention_weight_max": 21.09804344177246,
      "activations/layer12_attention_weight_min": -31.574020385742188,
      "activations/layer13_attention_weight_max": 43.01261901855469,
      "activations/layer13_attention_weight_min": -28.969762802124023,
      "activations/layer14_attention_weight_max": 42.20197296142578,
      "activations/layer14_attention_weight_min": -31.539783477783203,
      "activations/layer15_attention_weight_max": 41.393680572509766,
      "activations/layer15_attention_weight_min": -31.170135498046875,
      "activations/layer16_attention_weight_max": 32.91813659667969,
      "activations/layer16_attention_weight_min": -29.322986602783203,
      "activations/layer17_attention_weight_max": 53.76652526855469,
      "activations/layer17_attention_weight_min": -43.11103439331055,
      "activations/layer18_attention_weight_max": 43.88356018066406,
      "activations/layer18_attention_weight_min": -35.990718841552734,
      "activations/layer19_attention_weight_max": 20.516878128051758,
      "activations/layer19_attention_weight_min": -20.545976638793945,
      "activations/layer1_attention_weight_max": 17.29564094543457,
      "activations/layer1_attention_weight_min": -15.43998908996582,
      "activations/layer20_attention_weight_max": 20.788761138916016,
      "activations/layer20_attention_weight_min": -20.77530860900879,
      "activations/layer21_attention_weight_max": 39.63504409790039,
      "activations/layer21_attention_weight_min": -22.706579208374023,
      "activations/layer22_attention_weight_max": 30.62348747253418,
      "activations/layer22_attention_weight_min": -23.22719955444336,
      "activations/layer23_attention_weight_max": 36.27728271484375,
      "activations/layer23_attention_weight_min": -22.98683738708496,
      "activations/layer2_attention_weight_max": 33.26289367675781,
      "activations/layer2_attention_weight_min": -30.871597290039062,
      "activations/layer3_attention_weight_max": 90.19796752929688,
      "activations/layer3_attention_weight_min": -93.94225311279297,
      "activations/layer4_attention_weight_max": 103.1470718383789,
      "activations/layer4_attention_weight_min": -98.71302795410156,
      "activations/layer5_attention_weight_max": 81.95527648925781,
      "activations/layer5_attention_weight_min": -79.29360961914062,
      "activations/layer6_attention_weight_max": 55.47265625,
      "activations/layer6_attention_weight_min": -55.38725280761719,
      "activations/layer7_attention_weight_max": 86.39627838134766,
      "activations/layer7_attention_weight_min": -76.47480773925781,
      "activations/layer8_attention_weight_max": 48.078163146972656,
      "activations/layer8_attention_weight_min": -49.29815673828125,
      "activations/layer9_attention_weight_max": 62.78887939453125,
      "activations/layer9_attention_weight_min": -51.97587203979492,
      "epoch": 9.34,
      "learning_rate": 9.067840909090909e-05,
      "loss": 2.8005,
      "step": 160750
    },
    {
      "activations/layer0_attention_weight_max": 15.783608436584473,
      "activations/layer0_attention_weight_min": -13.977242469787598,
      "activations/layer10_attention_weight_max": 48.03738784790039,
      "activations/layer10_attention_weight_min": -46.39111328125,
      "activations/layer11_attention_weight_max": 41.94110107421875,
      "activations/layer11_attention_weight_min": -38.16762924194336,
      "activations/layer12_attention_weight_max": 19.66075325012207,
      "activations/layer12_attention_weight_min": -26.965126037597656,
      "activations/layer13_attention_weight_max": 56.94196701049805,
      "activations/layer13_attention_weight_min": -34.19282531738281,
      "activations/layer14_attention_weight_max": 69.91039276123047,
      "activations/layer14_attention_weight_min": -33.735198974609375,
      "activations/layer15_attention_weight_max": 55.56953811645508,
      "activations/layer15_attention_weight_min": -29.26442527770996,
      "activations/layer16_attention_weight_max": 47.52128219604492,
      "activations/layer16_attention_weight_min": -31.68596649169922,
      "activations/layer17_attention_weight_max": 63.07330322265625,
      "activations/layer17_attention_weight_min": -51.1181755065918,
      "activations/layer18_attention_weight_max": 56.26514434814453,
      "activations/layer18_attention_weight_min": -44.48492431640625,
      "activations/layer19_attention_weight_max": 24.28030014038086,
      "activations/layer19_attention_weight_min": -24.066129684448242,
      "activations/layer1_attention_weight_max": 17.5703182220459,
      "activations/layer1_attention_weight_min": -14.863076210021973,
      "activations/layer20_attention_weight_max": 30.717864990234375,
      "activations/layer20_attention_weight_min": -22.213096618652344,
      "activations/layer21_attention_weight_max": 61.85669708251953,
      "activations/layer21_attention_weight_min": -27.371501922607422,
      "activations/layer22_attention_weight_max": 36.56626892089844,
      "activations/layer22_attention_weight_min": -24.38011932373047,
      "activations/layer23_attention_weight_max": 38.85654830932617,
      "activations/layer23_attention_weight_min": -21.44998550415039,
      "activations/layer2_attention_weight_max": 34.9528694152832,
      "activations/layer2_attention_weight_min": -34.132408142089844,
      "activations/layer3_attention_weight_max": 97.21387481689453,
      "activations/layer3_attention_weight_min": -95.74876403808594,
      "activations/layer4_attention_weight_max": 107.38685607910156,
      "activations/layer4_attention_weight_min": -101.88719177246094,
      "activations/layer5_attention_weight_max": 91.86802673339844,
      "activations/layer5_attention_weight_min": -83.98001861572266,
      "activations/layer6_attention_weight_max": 65.78103637695312,
      "activations/layer6_attention_weight_min": -62.050262451171875,
      "activations/layer7_attention_weight_max": 92.55538940429688,
      "activations/layer7_attention_weight_min": -89.41423034667969,
      "activations/layer8_attention_weight_max": 59.10310363769531,
      "activations/layer8_attention_weight_min": -54.07413101196289,
      "activations/layer9_attention_weight_max": 69.23897552490234,
      "activations/layer9_attention_weight_min": -56.660091400146484,
      "epoch": 9.34,
      "learning_rate": 9.06594696969697e-05,
      "loss": 2.805,
      "step": 160800
    },
    {
      "activations/layer0_attention_weight_max": 15.9957275390625,
      "activations/layer0_attention_weight_min": -14.461960792541504,
      "activations/layer10_attention_weight_max": 35.398834228515625,
      "activations/layer10_attention_weight_min": -34.534446716308594,
      "activations/layer11_attention_weight_max": 30.227195739746094,
      "activations/layer11_attention_weight_min": -31.93682098388672,
      "activations/layer12_attention_weight_max": 17.353652954101562,
      "activations/layer12_attention_weight_min": -27.58877182006836,
      "activations/layer13_attention_weight_max": 35.69403076171875,
      "activations/layer13_attention_weight_min": -29.7954158782959,
      "activations/layer14_attention_weight_max": 34.26531982421875,
      "activations/layer14_attention_weight_min": -30.212688446044922,
      "activations/layer15_attention_weight_max": 32.30361557006836,
      "activations/layer15_attention_weight_min": -28.94788932800293,
      "activations/layer16_attention_weight_max": 33.58837127685547,
      "activations/layer16_attention_weight_min": -27.705469131469727,
      "activations/layer17_attention_weight_max": 52.847103118896484,
      "activations/layer17_attention_weight_min": -41.81896209716797,
      "activations/layer18_attention_weight_max": 43.73244857788086,
      "activations/layer18_attention_weight_min": -37.25233840942383,
      "activations/layer19_attention_weight_max": 21.10042381286621,
      "activations/layer19_attention_weight_min": -22.83779525756836,
      "activations/layer1_attention_weight_max": 15.557537078857422,
      "activations/layer1_attention_weight_min": -14.543801307678223,
      "activations/layer20_attention_weight_max": 20.90973663330078,
      "activations/layer20_attention_weight_min": -25.056312561035156,
      "activations/layer21_attention_weight_max": 37.371315002441406,
      "activations/layer21_attention_weight_min": -22.48441505432129,
      "activations/layer22_attention_weight_max": 30.34861946105957,
      "activations/layer22_attention_weight_min": -26.047212600708008,
      "activations/layer23_attention_weight_max": 40.63423156738281,
      "activations/layer23_attention_weight_min": -22.222328186035156,
      "activations/layer2_attention_weight_max": 32.016029357910156,
      "activations/layer2_attention_weight_min": -30.347328186035156,
      "activations/layer3_attention_weight_max": 89.55338287353516,
      "activations/layer3_attention_weight_min": -89.96780395507812,
      "activations/layer4_attention_weight_max": 96.96217346191406,
      "activations/layer4_attention_weight_min": -94.57295227050781,
      "activations/layer5_attention_weight_max": 77.50167846679688,
      "activations/layer5_attention_weight_min": -80.81681060791016,
      "activations/layer6_attention_weight_max": 52.92090606689453,
      "activations/layer6_attention_weight_min": -52.08738327026367,
      "activations/layer7_attention_weight_max": 76.5919189453125,
      "activations/layer7_attention_weight_min": -73.24514770507812,
      "activations/layer8_attention_weight_max": 43.72171401977539,
      "activations/layer8_attention_weight_min": -45.32697296142578,
      "activations/layer9_attention_weight_max": 48.72913360595703,
      "activations/layer9_attention_weight_min": -50.15718460083008,
      "epoch": 9.35,
      "learning_rate": 9.064053030303029e-05,
      "loss": 2.8162,
      "step": 160850
    },
    {
      "activations/layer0_attention_weight_max": 15.394539833068848,
      "activations/layer0_attention_weight_min": -14.081989288330078,
      "activations/layer10_attention_weight_max": 42.80100631713867,
      "activations/layer10_attention_weight_min": -38.32392120361328,
      "activations/layer11_attention_weight_max": 34.91584777832031,
      "activations/layer11_attention_weight_min": -33.56942367553711,
      "activations/layer12_attention_weight_max": 21.324649810791016,
      "activations/layer12_attention_weight_min": -24.364646911621094,
      "activations/layer13_attention_weight_max": 45.46312713623047,
      "activations/layer13_attention_weight_min": -33.55046844482422,
      "activations/layer14_attention_weight_max": 38.716827392578125,
      "activations/layer14_attention_weight_min": -29.121103286743164,
      "activations/layer15_attention_weight_max": 35.30768585205078,
      "activations/layer15_attention_weight_min": -29.130929946899414,
      "activations/layer16_attention_weight_max": 31.49058723449707,
      "activations/layer16_attention_weight_min": -27.68462562561035,
      "activations/layer17_attention_weight_max": 56.29405212402344,
      "activations/layer17_attention_weight_min": -41.70183181762695,
      "activations/layer18_attention_weight_max": 46.80042266845703,
      "activations/layer18_attention_weight_min": -35.811832427978516,
      "activations/layer19_attention_weight_max": 22.74344253540039,
      "activations/layer19_attention_weight_min": -20.516679763793945,
      "activations/layer1_attention_weight_max": 17.244054794311523,
      "activations/layer1_attention_weight_min": -14.717438697814941,
      "activations/layer20_attention_weight_max": 21.621410369873047,
      "activations/layer20_attention_weight_min": -24.563697814941406,
      "activations/layer21_attention_weight_max": 37.62462615966797,
      "activations/layer21_attention_weight_min": -24.305334091186523,
      "activations/layer22_attention_weight_max": 30.516645431518555,
      "activations/layer22_attention_weight_min": -26.26706314086914,
      "activations/layer23_attention_weight_max": 35.0167236328125,
      "activations/layer23_attention_weight_min": -23.25434684753418,
      "activations/layer2_attention_weight_max": 30.677345275878906,
      "activations/layer2_attention_weight_min": -30.505348205566406,
      "activations/layer3_attention_weight_max": 93.74465942382812,
      "activations/layer3_attention_weight_min": -99.03578186035156,
      "activations/layer4_attention_weight_max": 102.4568862915039,
      "activations/layer4_attention_weight_min": -96.86465454101562,
      "activations/layer5_attention_weight_max": 84.4575424194336,
      "activations/layer5_attention_weight_min": -76.37521362304688,
      "activations/layer6_attention_weight_max": 58.683197021484375,
      "activations/layer6_attention_weight_min": -54.883182525634766,
      "activations/layer7_attention_weight_max": 82.71610260009766,
      "activations/layer7_attention_weight_min": -77.24857330322266,
      "activations/layer8_attention_weight_max": 48.572750091552734,
      "activations/layer8_attention_weight_min": -47.51216506958008,
      "activations/layer9_attention_weight_max": 52.200836181640625,
      "activations/layer9_attention_weight_min": -51.954376220703125,
      "epoch": 9.35,
      "learning_rate": 9.06215909090909e-05,
      "loss": 2.8048,
      "step": 160900
    },
    {
      "activations/layer0_attention_weight_max": 15.586613655090332,
      "activations/layer0_attention_weight_min": -14.142281532287598,
      "activations/layer10_attention_weight_max": 40.3463020324707,
      "activations/layer10_attention_weight_min": -37.965999603271484,
      "activations/layer11_attention_weight_max": 32.9002799987793,
      "activations/layer11_attention_weight_min": -34.00415802001953,
      "activations/layer12_attention_weight_max": 19.689435958862305,
      "activations/layer12_attention_weight_min": -25.988996505737305,
      "activations/layer13_attention_weight_max": 50.87135314941406,
      "activations/layer13_attention_weight_min": -33.868980407714844,
      "activations/layer14_attention_weight_max": 46.24299240112305,
      "activations/layer14_attention_weight_min": -32.25568389892578,
      "activations/layer15_attention_weight_max": 40.46106719970703,
      "activations/layer15_attention_weight_min": -30.773513793945312,
      "activations/layer16_attention_weight_max": 34.10479736328125,
      "activations/layer16_attention_weight_min": -29.595457077026367,
      "activations/layer17_attention_weight_max": 58.8668098449707,
      "activations/layer17_attention_weight_min": -44.23506546020508,
      "activations/layer18_attention_weight_max": 52.57392120361328,
      "activations/layer18_attention_weight_min": -36.009422302246094,
      "activations/layer19_attention_weight_max": 24.328575134277344,
      "activations/layer19_attention_weight_min": -21.30577850341797,
      "activations/layer1_attention_weight_max": 16.856243133544922,
      "activations/layer1_attention_weight_min": -14.799838066101074,
      "activations/layer20_attention_weight_max": 24.92964744567871,
      "activations/layer20_attention_weight_min": -21.520830154418945,
      "activations/layer21_attention_weight_max": 44.5669059753418,
      "activations/layer21_attention_weight_min": -24.87587547302246,
      "activations/layer22_attention_weight_max": 33.39265441894531,
      "activations/layer22_attention_weight_min": -24.164962768554688,
      "activations/layer23_attention_weight_max": 46.475929260253906,
      "activations/layer23_attention_weight_min": -24.44338035583496,
      "activations/layer2_attention_weight_max": 31.219268798828125,
      "activations/layer2_attention_weight_min": -29.101743698120117,
      "activations/layer3_attention_weight_max": 88.38008117675781,
      "activations/layer3_attention_weight_min": -100.73368835449219,
      "activations/layer4_attention_weight_max": 98.32601165771484,
      "activations/layer4_attention_weight_min": -94.76522827148438,
      "activations/layer5_attention_weight_max": 78.2646484375,
      "activations/layer5_attention_weight_min": -76.86422729492188,
      "activations/layer6_attention_weight_max": 56.244873046875,
      "activations/layer6_attention_weight_min": -52.89629364013672,
      "activations/layer7_attention_weight_max": 76.88003540039062,
      "activations/layer7_attention_weight_min": -74.98932647705078,
      "activations/layer8_attention_weight_max": 45.3671760559082,
      "activations/layer8_attention_weight_min": -45.38275146484375,
      "activations/layer9_attention_weight_max": 52.17340850830078,
      "activations/layer9_attention_weight_min": -52.336544036865234,
      "epoch": 9.35,
      "learning_rate": 9.060265151515152e-05,
      "loss": 2.7867,
      "step": 160950
    },
    {
      "activations/layer0_attention_weight_max": 15.496136665344238,
      "activations/layer0_attention_weight_min": -14.276034355163574,
      "activations/layer10_attention_weight_max": 38.63876724243164,
      "activations/layer10_attention_weight_min": -37.18679428100586,
      "activations/layer11_attention_weight_max": 32.63266372680664,
      "activations/layer11_attention_weight_min": -33.32300567626953,
      "activations/layer12_attention_weight_max": 22.51026725769043,
      "activations/layer12_attention_weight_min": -24.485157012939453,
      "activations/layer13_attention_weight_max": 38.82414245605469,
      "activations/layer13_attention_weight_min": -30.87321662902832,
      "activations/layer14_attention_weight_max": 35.764068603515625,
      "activations/layer14_attention_weight_min": -30.570566177368164,
      "activations/layer15_attention_weight_max": 32.24601745605469,
      "activations/layer15_attention_weight_min": -29.22162437438965,
      "activations/layer16_attention_weight_max": 29.49345588684082,
      "activations/layer16_attention_weight_min": -28.502513885498047,
      "activations/layer17_attention_weight_max": 51.4881706237793,
      "activations/layer17_attention_weight_min": -40.8565673828125,
      "activations/layer18_attention_weight_max": 47.051475524902344,
      "activations/layer18_attention_weight_min": -35.081703186035156,
      "activations/layer19_attention_weight_max": 21.32162857055664,
      "activations/layer19_attention_weight_min": -21.1427001953125,
      "activations/layer1_attention_weight_max": 17.17888069152832,
      "activations/layer1_attention_weight_min": -15.877435684204102,
      "activations/layer20_attention_weight_max": 24.558731079101562,
      "activations/layer20_attention_weight_min": -20.75522804260254,
      "activations/layer21_attention_weight_max": 42.57499313354492,
      "activations/layer21_attention_weight_min": -23.785980224609375,
      "activations/layer22_attention_weight_max": 31.97506332397461,
      "activations/layer22_attention_weight_min": -24.58605194091797,
      "activations/layer23_attention_weight_max": 35.91203689575195,
      "activations/layer23_attention_weight_min": -20.908260345458984,
      "activations/layer2_attention_weight_max": 28.538787841796875,
      "activations/layer2_attention_weight_min": -27.886211395263672,
      "activations/layer3_attention_weight_max": 85.31484985351562,
      "activations/layer3_attention_weight_min": -86.31111907958984,
      "activations/layer4_attention_weight_max": 98.61227416992188,
      "activations/layer4_attention_weight_min": -95.39329528808594,
      "activations/layer5_attention_weight_max": 78.98451232910156,
      "activations/layer5_attention_weight_min": -76.55828857421875,
      "activations/layer6_attention_weight_max": 54.68630599975586,
      "activations/layer6_attention_weight_min": -54.09862518310547,
      "activations/layer7_attention_weight_max": 77.8677749633789,
      "activations/layer7_attention_weight_min": -77.08145141601562,
      "activations/layer8_attention_weight_max": 46.76268768310547,
      "activations/layer8_attention_weight_min": -46.7257194519043,
      "activations/layer9_attention_weight_max": 49.435630798339844,
      "activations/layer9_attention_weight_min": -50.51259994506836,
      "epoch": 9.36,
      "learning_rate": 9.05837121212121e-05,
      "loss": 2.7852,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_loss": 2.75,
      "eval_runtime": 8.5197,
      "eval_samples_per_second": 504.011,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_openwebtext_loss": 2.75,
      "eval_openwebtext_ppl": 15.642631884188171,
      "eval_openwebtext_runtime": 8.5197,
      "eval_openwebtext_samples_per_second": 504.011,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 1.9788,
      "eval_wikitext_samples_per_second": 230.444,
      "step": 161000
    },
    {
      "epoch": 9.36,
      "eval_lambada_loss": 2.638671875,
      "eval_lambada_ppl": 13.994604675279588,
      "eval_lambada_runtime": 9.5687,
      "eval_lambada_samples_per_second": 508.844,
      "step": 161000
    },
    {
      "activations/layer0_attention_weight_max": 16.235349655151367,
      "activations/layer0_attention_weight_min": -13.885470390319824,
      "activations/layer10_attention_weight_max": 39.09238815307617,
      "activations/layer10_attention_weight_min": -35.46272659301758,
      "activations/layer11_attention_weight_max": 30.18611717224121,
      "activations/layer11_attention_weight_min": -31.38091278076172,
      "activations/layer12_attention_weight_max": 20.132247924804688,
      "activations/layer12_attention_weight_min": -28.190134048461914,
      "activations/layer13_attention_weight_max": 34.96034622192383,
      "activations/layer13_attention_weight_min": -28.508811950683594,
      "activations/layer14_attention_weight_max": 38.76905059814453,
      "activations/layer14_attention_weight_min": -29.85523223876953,
      "activations/layer15_attention_weight_max": 35.37894058227539,
      "activations/layer15_attention_weight_min": -28.29816246032715,
      "activations/layer16_attention_weight_max": 34.85258483886719,
      "activations/layer16_attention_weight_min": -30.149776458740234,
      "activations/layer17_attention_weight_max": 54.34747314453125,
      "activations/layer17_attention_weight_min": -43.518985748291016,
      "activations/layer18_attention_weight_max": 49.099952697753906,
      "activations/layer18_attention_weight_min": -35.79718017578125,
      "activations/layer19_attention_weight_max": 21.80645751953125,
      "activations/layer19_attention_weight_min": -21.63058853149414,
      "activations/layer1_attention_weight_max": 17.621458053588867,
      "activations/layer1_attention_weight_min": -13.682389259338379,
      "activations/layer20_attention_weight_max": 21.42816734313965,
      "activations/layer20_attention_weight_min": -19.960750579833984,
      "activations/layer21_attention_weight_max": 37.5760612487793,
      "activations/layer21_attention_weight_min": -21.62885856628418,
      "activations/layer22_attention_weight_max": 33.817352294921875,
      "activations/layer22_attention_weight_min": -25.473155975341797,
      "activations/layer23_attention_weight_max": 39.77135467529297,
      "activations/layer23_attention_weight_min": -23.40768814086914,
      "activations/layer2_attention_weight_max": 30.049474716186523,
      "activations/layer2_attention_weight_min": -29.077259063720703,
      "activations/layer3_attention_weight_max": 87.07386779785156,
      "activations/layer3_attention_weight_min": -91.77823638916016,
      "activations/layer4_attention_weight_max": 100.9485092163086,
      "activations/layer4_attention_weight_min": -95.2399673461914,
      "activations/layer5_attention_weight_max": 80.37867736816406,
      "activations/layer5_attention_weight_min": -77.88388061523438,
      "activations/layer6_attention_weight_max": 54.12040328979492,
      "activations/layer6_attention_weight_min": -53.628910064697266,
      "activations/layer7_attention_weight_max": 77.28785705566406,
      "activations/layer7_attention_weight_min": -74.41728973388672,
      "activations/layer8_attention_weight_max": 46.14226531982422,
      "activations/layer8_attention_weight_min": -45.35050964355469,
      "activations/layer9_attention_weight_max": 49.078758239746094,
      "activations/layer9_attention_weight_min": -47.7101936340332,
      "epoch": 9.36,
      "learning_rate": 9.056477272727272e-05,
      "loss": 2.7867,
      "step": 161050
    },
    {
      "activations/layer0_attention_weight_max": 15.85412883758545,
      "activations/layer0_attention_weight_min": -14.21054744720459,
      "activations/layer10_attention_weight_max": 38.760154724121094,
      "activations/layer10_attention_weight_min": -35.0300178527832,
      "activations/layer11_attention_weight_max": 33.23597717285156,
      "activations/layer11_attention_weight_min": -31.066478729248047,
      "activations/layer12_attention_weight_max": 21.417516708374023,
      "activations/layer12_attention_weight_min": -26.433887481689453,
      "activations/layer13_attention_weight_max": 39.00223922729492,
      "activations/layer13_attention_weight_min": -30.772308349609375,
      "activations/layer14_attention_weight_max": 37.69780731201172,
      "activations/layer14_attention_weight_min": -30.921905517578125,
      "activations/layer15_attention_weight_max": 32.85426712036133,
      "activations/layer15_attention_weight_min": -29.6496524810791,
      "activations/layer16_attention_weight_max": 28.62250328063965,
      "activations/layer16_attention_weight_min": -28.32066535949707,
      "activations/layer17_attention_weight_max": 53.13798141479492,
      "activations/layer17_attention_weight_min": -42.8548469543457,
      "activations/layer18_attention_weight_max": 46.477783203125,
      "activations/layer18_attention_weight_min": -35.514984130859375,
      "activations/layer19_attention_weight_max": 23.323802947998047,
      "activations/layer19_attention_weight_min": -21.306838989257812,
      "activations/layer1_attention_weight_max": 16.6010684967041,
      "activations/layer1_attention_weight_min": -15.257492065429688,
      "activations/layer20_attention_weight_max": 21.87645721435547,
      "activations/layer20_attention_weight_min": -22.531517028808594,
      "activations/layer21_attention_weight_max": 37.112884521484375,
      "activations/layer21_attention_weight_min": -23.903324127197266,
      "activations/layer22_attention_weight_max": 30.982114791870117,
      "activations/layer22_attention_weight_min": -24.185659408569336,
      "activations/layer23_attention_weight_max": 40.374969482421875,
      "activations/layer23_attention_weight_min": -24.375865936279297,
      "activations/layer2_attention_weight_max": 30.41119956970215,
      "activations/layer2_attention_weight_min": -30.587858200073242,
      "activations/layer3_attention_weight_max": 88.01725006103516,
      "activations/layer3_attention_weight_min": -92.7347412109375,
      "activations/layer4_attention_weight_max": 100.40403747558594,
      "activations/layer4_attention_weight_min": -94.85255432128906,
      "activations/layer5_attention_weight_max": 79.60108947753906,
      "activations/layer5_attention_weight_min": -80.17181396484375,
      "activations/layer6_attention_weight_max": 55.26316452026367,
      "activations/layer6_attention_weight_min": -52.92688751220703,
      "activations/layer7_attention_weight_max": 76.45753479003906,
      "activations/layer7_attention_weight_min": -72.4406967163086,
      "activations/layer8_attention_weight_max": 45.9858512878418,
      "activations/layer8_attention_weight_min": -45.56913375854492,
      "activations/layer9_attention_weight_max": 49.359031677246094,
      "activations/layer9_attention_weight_min": -50.775596618652344,
      "epoch": 9.36,
      "learning_rate": 9.054583333333332e-05,
      "loss": 2.8105,
      "step": 161100
    },
    {
      "activations/layer0_attention_weight_max": 16.109302520751953,
      "activations/layer0_attention_weight_min": -14.204225540161133,
      "activations/layer10_attention_weight_max": 35.56377410888672,
      "activations/layer10_attention_weight_min": -33.91424560546875,
      "activations/layer11_attention_weight_max": 31.659732818603516,
      "activations/layer11_attention_weight_min": -32.4891471862793,
      "activations/layer12_attention_weight_max": 19.146747589111328,
      "activations/layer12_attention_weight_min": -25.597217559814453,
      "activations/layer13_attention_weight_max": 39.75829315185547,
      "activations/layer13_attention_weight_min": -29.63292121887207,
      "activations/layer14_attention_weight_max": 35.035797119140625,
      "activations/layer14_attention_weight_min": -29.943397521972656,
      "activations/layer15_attention_weight_max": 32.03408432006836,
      "activations/layer15_attention_weight_min": -28.331933975219727,
      "activations/layer16_attention_weight_max": 29.025611877441406,
      "activations/layer16_attention_weight_min": -28.03466033935547,
      "activations/layer17_attention_weight_max": 54.89921951293945,
      "activations/layer17_attention_weight_min": -43.01483154296875,
      "activations/layer18_attention_weight_max": 47.23344039916992,
      "activations/layer18_attention_weight_min": -37.180511474609375,
      "activations/layer19_attention_weight_max": 20.161544799804688,
      "activations/layer19_attention_weight_min": -19.853126525878906,
      "activations/layer1_attention_weight_max": 17.552291870117188,
      "activations/layer1_attention_weight_min": -14.500626564025879,
      "activations/layer20_attention_weight_max": 20.5997371673584,
      "activations/layer20_attention_weight_min": -21.17860221862793,
      "activations/layer21_attention_weight_max": 37.025386810302734,
      "activations/layer21_attention_weight_min": -25.31032371520996,
      "activations/layer22_attention_weight_max": 29.68480682373047,
      "activations/layer22_attention_weight_min": -22.820337295532227,
      "activations/layer23_attention_weight_max": 36.04120635986328,
      "activations/layer23_attention_weight_min": -22.84327507019043,
      "activations/layer2_attention_weight_max": 30.194189071655273,
      "activations/layer2_attention_weight_min": -29.068273544311523,
      "activations/layer3_attention_weight_max": 85.00040435791016,
      "activations/layer3_attention_weight_min": -85.41417694091797,
      "activations/layer4_attention_weight_max": 95.87623596191406,
      "activations/layer4_attention_weight_min": -90.3567123413086,
      "activations/layer5_attention_weight_max": 75.89859771728516,
      "activations/layer5_attention_weight_min": -75.64881134033203,
      "activations/layer6_attention_weight_max": 52.38896560668945,
      "activations/layer6_attention_weight_min": -51.773555755615234,
      "activations/layer7_attention_weight_max": 77.27193450927734,
      "activations/layer7_attention_weight_min": -72.97830963134766,
      "activations/layer8_attention_weight_max": 42.85910415649414,
      "activations/layer8_attention_weight_min": -45.07027053833008,
      "activations/layer9_attention_weight_max": 47.01626968383789,
      "activations/layer9_attention_weight_min": -48.74735641479492,
      "epoch": 9.36,
      "learning_rate": 9.052727272727273e-05,
      "loss": 2.8006,
      "step": 161150
    },
    {
      "activations/layer0_attention_weight_max": 15.199119567871094,
      "activations/layer0_attention_weight_min": -13.962459564208984,
      "activations/layer10_attention_weight_max": 38.03639602661133,
      "activations/layer10_attention_weight_min": -35.44976806640625,
      "activations/layer11_attention_weight_max": 31.53958511352539,
      "activations/layer11_attention_weight_min": -31.154632568359375,
      "activations/layer12_attention_weight_max": 21.131643295288086,
      "activations/layer12_attention_weight_min": -27.448104858398438,
      "activations/layer13_attention_weight_max": 48.253177642822266,
      "activations/layer13_attention_weight_min": -33.1543083190918,
      "activations/layer14_attention_weight_max": 42.91377258300781,
      "activations/layer14_attention_weight_min": -32.11801528930664,
      "activations/layer15_attention_weight_max": 33.12508010864258,
      "activations/layer15_attention_weight_min": -28.465513229370117,
      "activations/layer16_attention_weight_max": 29.894227981567383,
      "activations/layer16_attention_weight_min": -27.787776947021484,
      "activations/layer17_attention_weight_max": 53.51094436645508,
      "activations/layer17_attention_weight_min": -43.21335983276367,
      "activations/layer18_attention_weight_max": 44.78811264038086,
      "activations/layer18_attention_weight_min": -37.081695556640625,
      "activations/layer19_attention_weight_max": 19.637449264526367,
      "activations/layer19_attention_weight_min": -21.348310470581055,
      "activations/layer1_attention_weight_max": 16.627805709838867,
      "activations/layer1_attention_weight_min": -14.028721809387207,
      "activations/layer20_attention_weight_max": 21.309795379638672,
      "activations/layer20_attention_weight_min": -21.446653366088867,
      "activations/layer21_attention_weight_max": 36.88190841674805,
      "activations/layer21_attention_weight_min": -24.484331130981445,
      "activations/layer22_attention_weight_max": 30.99077606201172,
      "activations/layer22_attention_weight_min": -25.90511131286621,
      "activations/layer23_attention_weight_max": 42.158226013183594,
      "activations/layer23_attention_weight_min": -24.327571868896484,
      "activations/layer2_attention_weight_max": 28.88758659362793,
      "activations/layer2_attention_weight_min": -28.184616088867188,
      "activations/layer3_attention_weight_max": 84.57132720947266,
      "activations/layer3_attention_weight_min": -86.81233215332031,
      "activations/layer4_attention_weight_max": 96.59256744384766,
      "activations/layer4_attention_weight_min": -94.1111831665039,
      "activations/layer5_attention_weight_max": 77.61622619628906,
      "activations/layer5_attention_weight_min": -79.61652374267578,
      "activations/layer6_attention_weight_max": 53.683616638183594,
      "activations/layer6_attention_weight_min": -53.746376037597656,
      "activations/layer7_attention_weight_max": 76.34896850585938,
      "activations/layer7_attention_weight_min": -79.78671264648438,
      "activations/layer8_attention_weight_max": 43.7919807434082,
      "activations/layer8_attention_weight_min": -45.658775329589844,
      "activations/layer9_attention_weight_max": 48.239479064941406,
      "activations/layer9_attention_weight_min": -51.159732818603516,
      "epoch": 9.37,
      "learning_rate": 9.050833333333332e-05,
      "loss": 2.802,
      "step": 161200
    },
    {
      "activations/layer0_attention_weight_max": 15.393845558166504,
      "activations/layer0_attention_weight_min": -14.177291870117188,
      "activations/layer10_attention_weight_max": 45.70017623901367,
      "activations/layer10_attention_weight_min": -42.694190979003906,
      "activations/layer11_attention_weight_max": 38.23244094848633,
      "activations/layer11_attention_weight_min": -33.18302917480469,
      "activations/layer12_attention_weight_max": 21.263975143432617,
      "activations/layer12_attention_weight_min": -24.77701759338379,
      "activations/layer13_attention_weight_max": 66.63483428955078,
      "activations/layer13_attention_weight_min": -36.45827102661133,
      "activations/layer14_attention_weight_max": 52.65079116821289,
      "activations/layer14_attention_weight_min": -30.967891693115234,
      "activations/layer15_attention_weight_max": 41.0239372253418,
      "activations/layer15_attention_weight_min": -28.345314025878906,
      "activations/layer16_attention_weight_max": 33.11712646484375,
      "activations/layer16_attention_weight_min": -28.11895751953125,
      "activations/layer17_attention_weight_max": 51.3630256652832,
      "activations/layer17_attention_weight_min": -41.67721939086914,
      "activations/layer18_attention_weight_max": 45.10884094238281,
      "activations/layer18_attention_weight_min": -35.791996002197266,
      "activations/layer19_attention_weight_max": 20.391326904296875,
      "activations/layer19_attention_weight_min": -20.65411949157715,
      "activations/layer1_attention_weight_max": 17.599668502807617,
      "activations/layer1_attention_weight_min": -14.20814037322998,
      "activations/layer20_attention_weight_max": 25.38676643371582,
      "activations/layer20_attention_weight_min": -24.8905029296875,
      "activations/layer21_attention_weight_max": 37.001678466796875,
      "activations/layer21_attention_weight_min": -21.70020294189453,
      "activations/layer22_attention_weight_max": 34.51006317138672,
      "activations/layer22_attention_weight_min": -24.377201080322266,
      "activations/layer23_attention_weight_max": 43.268829345703125,
      "activations/layer23_attention_weight_min": -23.26668930053711,
      "activations/layer2_attention_weight_max": 30.480854034423828,
      "activations/layer2_attention_weight_min": -30.0383358001709,
      "activations/layer3_attention_weight_max": 84.07320404052734,
      "activations/layer3_attention_weight_min": -90.4162368774414,
      "activations/layer4_attention_weight_max": 92.79026794433594,
      "activations/layer4_attention_weight_min": -92.0528564453125,
      "activations/layer5_attention_weight_max": 74.51359558105469,
      "activations/layer5_attention_weight_min": -77.95924377441406,
      "activations/layer6_attention_weight_max": 52.25163269042969,
      "activations/layer6_attention_weight_min": -50.80268478393555,
      "activations/layer7_attention_weight_max": 81.23602294921875,
      "activations/layer7_attention_weight_min": -75.64000701904297,
      "activations/layer8_attention_weight_max": 52.22807693481445,
      "activations/layer8_attention_weight_min": -52.34283447265625,
      "activations/layer9_attention_weight_max": 60.838775634765625,
      "activations/layer9_attention_weight_min": -53.90522384643555,
      "epoch": 9.37,
      "learning_rate": 9.048939393939393e-05,
      "loss": 2.7709,
      "step": 161250
    },
    {
      "activations/layer0_attention_weight_max": 14.953710556030273,
      "activations/layer0_attention_weight_min": -14.915450096130371,
      "activations/layer10_attention_weight_max": 36.457313537597656,
      "activations/layer10_attention_weight_min": -35.67033767700195,
      "activations/layer11_attention_weight_max": 31.128353118896484,
      "activations/layer11_attention_weight_min": -30.80563735961914,
      "activations/layer12_attention_weight_max": 21.288711547851562,
      "activations/layer12_attention_weight_min": -25.281339645385742,
      "activations/layer13_attention_weight_max": 42.286895751953125,
      "activations/layer13_attention_weight_min": -31.990434646606445,
      "activations/layer14_attention_weight_max": 42.18604278564453,
      "activations/layer14_attention_weight_min": -32.67226791381836,
      "activations/layer15_attention_weight_max": 36.76199722290039,
      "activations/layer15_attention_weight_min": -29.345216751098633,
      "activations/layer16_attention_weight_max": 34.0866813659668,
      "activations/layer16_attention_weight_min": -29.462968826293945,
      "activations/layer17_attention_weight_max": 60.01763916015625,
      "activations/layer17_attention_weight_min": -45.924285888671875,
      "activations/layer18_attention_weight_max": 51.632896423339844,
      "activations/layer18_attention_weight_min": -37.937313079833984,
      "activations/layer19_attention_weight_max": 21.128036499023438,
      "activations/layer19_attention_weight_min": -22.027990341186523,
      "activations/layer1_attention_weight_max": 16.593273162841797,
      "activations/layer1_attention_weight_min": -15.314716339111328,
      "activations/layer20_attention_weight_max": 21.256603240966797,
      "activations/layer20_attention_weight_min": -23.61606216430664,
      "activations/layer21_attention_weight_max": 38.77753829956055,
      "activations/layer21_attention_weight_min": -27.51921844482422,
      "activations/layer22_attention_weight_max": 32.6436767578125,
      "activations/layer22_attention_weight_min": -26.41205596923828,
      "activations/layer23_attention_weight_max": 35.71312713623047,
      "activations/layer23_attention_weight_min": -23.91333770751953,
      "activations/layer2_attention_weight_max": 29.18154525756836,
      "activations/layer2_attention_weight_min": -30.214597702026367,
      "activations/layer3_attention_weight_max": 87.66312408447266,
      "activations/layer3_attention_weight_min": -93.22039031982422,
      "activations/layer4_attention_weight_max": 99.09734344482422,
      "activations/layer4_attention_weight_min": -95.90433502197266,
      "activations/layer5_attention_weight_max": 76.00189208984375,
      "activations/layer5_attention_weight_min": -79.4078140258789,
      "activations/layer6_attention_weight_max": 51.80986022949219,
      "activations/layer6_attention_weight_min": -51.705928802490234,
      "activations/layer7_attention_weight_max": 76.58142852783203,
      "activations/layer7_attention_weight_min": -70.47582244873047,
      "activations/layer8_attention_weight_max": 42.34067916870117,
      "activations/layer8_attention_weight_min": -46.296875,
      "activations/layer9_attention_weight_max": 48.24565887451172,
      "activations/layer9_attention_weight_min": -47.3101921081543,
      "epoch": 9.37,
      "learning_rate": 9.047045454545453e-05,
      "loss": 2.7934,
      "step": 161300
    },
    {
      "activations/layer0_attention_weight_max": 15.069147109985352,
      "activations/layer0_attention_weight_min": -14.57634162902832,
      "activations/layer10_attention_weight_max": 35.05270767211914,
      "activations/layer10_attention_weight_min": -33.96857452392578,
      "activations/layer11_attention_weight_max": 29.087905883789062,
      "activations/layer11_attention_weight_min": -30.734588623046875,
      "activations/layer12_attention_weight_max": 16.701229095458984,
      "activations/layer12_attention_weight_min": -23.064176559448242,
      "activations/layer13_attention_weight_max": 35.551422119140625,
      "activations/layer13_attention_weight_min": -28.853836059570312,
      "activations/layer14_attention_weight_max": 34.992637634277344,
      "activations/layer14_attention_weight_min": -29.62705421447754,
      "activations/layer15_attention_weight_max": 35.114593505859375,
      "activations/layer15_attention_weight_min": -28.76690673828125,
      "activations/layer16_attention_weight_max": 29.212881088256836,
      "activations/layer16_attention_weight_min": -27.712814331054688,
      "activations/layer17_attention_weight_max": 53.609161376953125,
      "activations/layer17_attention_weight_min": -42.94700622558594,
      "activations/layer18_attention_weight_max": 44.71869659423828,
      "activations/layer18_attention_weight_min": -37.937347412109375,
      "activations/layer19_attention_weight_max": 20.392210006713867,
      "activations/layer19_attention_weight_min": -21.981033325195312,
      "activations/layer1_attention_weight_max": 16.514266967773438,
      "activations/layer1_attention_weight_min": -15.431663513183594,
      "activations/layer20_attention_weight_max": 23.54271697998047,
      "activations/layer20_attention_weight_min": -26.45387840270996,
      "activations/layer21_attention_weight_max": 38.4991455078125,
      "activations/layer21_attention_weight_min": -23.858642578125,
      "activations/layer22_attention_weight_max": 27.63938331604004,
      "activations/layer22_attention_weight_min": -26.74079704284668,
      "activations/layer23_attention_weight_max": 37.28289031982422,
      "activations/layer23_attention_weight_min": -24.67629051208496,
      "activations/layer2_attention_weight_max": 28.57925033569336,
      "activations/layer2_attention_weight_min": -28.219045639038086,
      "activations/layer3_attention_weight_max": 82.23533630371094,
      "activations/layer3_attention_weight_min": -87.43486022949219,
      "activations/layer4_attention_weight_max": 97.39982604980469,
      "activations/layer4_attention_weight_min": -89.42070007324219,
      "activations/layer5_attention_weight_max": 75.75717163085938,
      "activations/layer5_attention_weight_min": -75.90319061279297,
      "activations/layer6_attention_weight_max": 54.36613082885742,
      "activations/layer6_attention_weight_min": -52.71316909790039,
      "activations/layer7_attention_weight_max": 79.74262237548828,
      "activations/layer7_attention_weight_min": -72.89867401123047,
      "activations/layer8_attention_weight_max": 43.370792388916016,
      "activations/layer8_attention_weight_min": -43.85896682739258,
      "activations/layer9_attention_weight_max": 47.080509185791016,
      "activations/layer9_attention_weight_min": -49.5064582824707,
      "epoch": 9.38,
      "learning_rate": 9.045151515151515e-05,
      "loss": 2.7897,
      "step": 161350
    },
    {
      "activations/layer0_attention_weight_max": 15.859275817871094,
      "activations/layer0_attention_weight_min": -13.849031448364258,
      "activations/layer10_attention_weight_max": 38.914031982421875,
      "activations/layer10_attention_weight_min": -38.081787109375,
      "activations/layer11_attention_weight_max": 35.02920150756836,
      "activations/layer11_attention_weight_min": -31.891216278076172,
      "activations/layer12_attention_weight_max": 17.688302993774414,
      "activations/layer12_attention_weight_min": -27.502965927124023,
      "activations/layer13_attention_weight_max": 37.85883331298828,
      "activations/layer13_attention_weight_min": -29.036815643310547,
      "activations/layer14_attention_weight_max": 36.603729248046875,
      "activations/layer14_attention_weight_min": -31.477420806884766,
      "activations/layer15_attention_weight_max": 34.41999435424805,
      "activations/layer15_attention_weight_min": -28.70003890991211,
      "activations/layer16_attention_weight_max": 30.753902435302734,
      "activations/layer16_attention_weight_min": -29.157873153686523,
      "activations/layer17_attention_weight_max": 50.612709045410156,
      "activations/layer17_attention_weight_min": -43.70621109008789,
      "activations/layer18_attention_weight_max": 48.42848205566406,
      "activations/layer18_attention_weight_min": -36.268211364746094,
      "activations/layer19_attention_weight_max": 24.44024085998535,
      "activations/layer19_attention_weight_min": -21.847501754760742,
      "activations/layer1_attention_weight_max": 17.304235458374023,
      "activations/layer1_attention_weight_min": -17.3877010345459,
      "activations/layer20_attention_weight_max": 21.982988357543945,
      "activations/layer20_attention_weight_min": -20.553327560424805,
      "activations/layer21_attention_weight_max": 40.127105712890625,
      "activations/layer21_attention_weight_min": -23.708663940429688,
      "activations/layer22_attention_weight_max": 32.58616638183594,
      "activations/layer22_attention_weight_min": -23.608518600463867,
      "activations/layer23_attention_weight_max": 34.135498046875,
      "activations/layer23_attention_weight_min": -25.087146759033203,
      "activations/layer2_attention_weight_max": 29.04248809814453,
      "activations/layer2_attention_weight_min": -28.53920555114746,
      "activations/layer3_attention_weight_max": 84.97142791748047,
      "activations/layer3_attention_weight_min": -92.86922454833984,
      "activations/layer4_attention_weight_max": 98.72857666015625,
      "activations/layer4_attention_weight_min": -94.67418670654297,
      "activations/layer5_attention_weight_max": 85.52340698242188,
      "activations/layer5_attention_weight_min": -73.81918334960938,
      "activations/layer6_attention_weight_max": 56.68724822998047,
      "activations/layer6_attention_weight_min": -53.854732513427734,
      "activations/layer7_attention_weight_max": 79.22897338867188,
      "activations/layer7_attention_weight_min": -78.43063354492188,
      "activations/layer8_attention_weight_max": 47.67766571044922,
      "activations/layer8_attention_weight_min": -47.809547424316406,
      "activations/layer9_attention_weight_max": 62.80463409423828,
      "activations/layer9_attention_weight_min": -51.99287033081055,
      "epoch": 9.38,
      "learning_rate": 9.043257575757575e-05,
      "loss": 2.8004,
      "step": 161400
    },
    {
      "activations/layer0_attention_weight_max": 14.499375343322754,
      "activations/layer0_attention_weight_min": -13.554300308227539,
      "activations/layer10_attention_weight_max": 42.572357177734375,
      "activations/layer10_attention_weight_min": -40.70712661743164,
      "activations/layer11_attention_weight_max": 40.09425354003906,
      "activations/layer11_attention_weight_min": -34.618568420410156,
      "activations/layer12_attention_weight_max": 19.333194732666016,
      "activations/layer12_attention_weight_min": -26.294696807861328,
      "activations/layer13_attention_weight_max": 34.93342590332031,
      "activations/layer13_attention_weight_min": -28.32523536682129,
      "activations/layer14_attention_weight_max": 36.54228591918945,
      "activations/layer14_attention_weight_min": -29.621261596679688,
      "activations/layer15_attention_weight_max": 31.37232780456543,
      "activations/layer15_attention_weight_min": -28.682300567626953,
      "activations/layer16_attention_weight_max": 27.84589385986328,
      "activations/layer16_attention_weight_min": -28.19110870361328,
      "activations/layer17_attention_weight_max": 52.53279113769531,
      "activations/layer17_attention_weight_min": -44.578857421875,
      "activations/layer18_attention_weight_max": 47.766990661621094,
      "activations/layer18_attention_weight_min": -38.308441162109375,
      "activations/layer19_attention_weight_max": 19.3360652923584,
      "activations/layer19_attention_weight_min": -22.628835678100586,
      "activations/layer1_attention_weight_max": 17.878894805908203,
      "activations/layer1_attention_weight_min": -13.529458999633789,
      "activations/layer20_attention_weight_max": 22.63526153564453,
      "activations/layer20_attention_weight_min": -21.82010269165039,
      "activations/layer21_attention_weight_max": 33.662105560302734,
      "activations/layer21_attention_weight_min": -23.4285945892334,
      "activations/layer22_attention_weight_max": 30.247020721435547,
      "activations/layer22_attention_weight_min": -24.712270736694336,
      "activations/layer23_attention_weight_max": 35.926673889160156,
      "activations/layer23_attention_weight_min": -23.218135833740234,
      "activations/layer2_attention_weight_max": 31.76308250427246,
      "activations/layer2_attention_weight_min": -31.5839900970459,
      "activations/layer3_attention_weight_max": 93.61869812011719,
      "activations/layer3_attention_weight_min": -97.58146667480469,
      "activations/layer4_attention_weight_max": 104.99928283691406,
      "activations/layer4_attention_weight_min": -98.29072570800781,
      "activations/layer5_attention_weight_max": 82.02818298339844,
      "activations/layer5_attention_weight_min": -79.28746032714844,
      "activations/layer6_attention_weight_max": 56.01606369018555,
      "activations/layer6_attention_weight_min": -54.42459487915039,
      "activations/layer7_attention_weight_max": 102.29668426513672,
      "activations/layer7_attention_weight_min": -78.22798919677734,
      "activations/layer8_attention_weight_max": 56.46535873413086,
      "activations/layer8_attention_weight_min": -49.941036224365234,
      "activations/layer9_attention_weight_max": 71.22911071777344,
      "activations/layer9_attention_weight_min": -55.16634750366211,
      "epoch": 9.38,
      "learning_rate": 9.041363636363635e-05,
      "loss": 2.8015,
      "step": 161450
    },
    {
      "activations/layer0_attention_weight_max": 15.979351043701172,
      "activations/layer0_attention_weight_min": -13.51865291595459,
      "activations/layer10_attention_weight_max": 37.636871337890625,
      "activations/layer10_attention_weight_min": -38.178871154785156,
      "activations/layer11_attention_weight_max": 31.640087127685547,
      "activations/layer11_attention_weight_min": -32.845428466796875,
      "activations/layer12_attention_weight_max": 17.76780128479004,
      "activations/layer12_attention_weight_min": -23.27239418029785,
      "activations/layer13_attention_weight_max": 41.12969207763672,
      "activations/layer13_attention_weight_min": -33.56755065917969,
      "activations/layer14_attention_weight_max": 36.81766128540039,
      "activations/layer14_attention_weight_min": -31.231292724609375,
      "activations/layer15_attention_weight_max": 31.599069595336914,
      "activations/layer15_attention_weight_min": -29.292102813720703,
      "activations/layer16_attention_weight_max": 35.00871276855469,
      "activations/layer16_attention_weight_min": -28.92119026184082,
      "activations/layer17_attention_weight_max": 53.07392501831055,
      "activations/layer17_attention_weight_min": -46.66596221923828,
      "activations/layer18_attention_weight_max": 47.64679718017578,
      "activations/layer18_attention_weight_min": -41.93433380126953,
      "activations/layer19_attention_weight_max": 22.539960861206055,
      "activations/layer19_attention_weight_min": -23.505285263061523,
      "activations/layer1_attention_weight_max": 16.85936737060547,
      "activations/layer1_attention_weight_min": -13.44253921508789,
      "activations/layer20_attention_weight_max": 23.679916381835938,
      "activations/layer20_attention_weight_min": -23.575834274291992,
      "activations/layer21_attention_weight_max": 36.99678421020508,
      "activations/layer21_attention_weight_min": -23.91985511779785,
      "activations/layer22_attention_weight_max": 32.69356155395508,
      "activations/layer22_attention_weight_min": -26.7605037689209,
      "activations/layer23_attention_weight_max": 42.90806579589844,
      "activations/layer23_attention_weight_min": -21.702497482299805,
      "activations/layer2_attention_weight_max": 30.938396453857422,
      "activations/layer2_attention_weight_min": -29.58718490600586,
      "activations/layer3_attention_weight_max": 85.74493408203125,
      "activations/layer3_attention_weight_min": -92.78849029541016,
      "activations/layer4_attention_weight_max": 93.49625396728516,
      "activations/layer4_attention_weight_min": -93.80242156982422,
      "activations/layer5_attention_weight_max": 75.35014343261719,
      "activations/layer5_attention_weight_min": -75.24693298339844,
      "activations/layer6_attention_weight_max": 51.3294563293457,
      "activations/layer6_attention_weight_min": -52.227542877197266,
      "activations/layer7_attention_weight_max": 79.84762573242188,
      "activations/layer7_attention_weight_min": -78.10306549072266,
      "activations/layer8_attention_weight_max": 45.921119689941406,
      "activations/layer8_attention_weight_min": -47.599971771240234,
      "activations/layer9_attention_weight_max": 53.600196838378906,
      "activations/layer9_attention_weight_min": -49.84596633911133,
      "epoch": 9.38,
      "learning_rate": 9.039469696969696e-05,
      "loss": 2.7995,
      "step": 161500
    },
    {
      "activations/layer0_attention_weight_max": 14.667038917541504,
      "activations/layer0_attention_weight_min": -14.223039627075195,
      "activations/layer10_attention_weight_max": 36.7425537109375,
      "activations/layer10_attention_weight_min": -36.754150390625,
      "activations/layer11_attention_weight_max": 31.312597274780273,
      "activations/layer11_attention_weight_min": -34.015106201171875,
      "activations/layer12_attention_weight_max": 19.143600463867188,
      "activations/layer12_attention_weight_min": -24.128273010253906,
      "activations/layer13_attention_weight_max": 38.78173828125,
      "activations/layer13_attention_weight_min": -29.6134090423584,
      "activations/layer14_attention_weight_max": 37.33818054199219,
      "activations/layer14_attention_weight_min": -29.39422607421875,
      "activations/layer15_attention_weight_max": 34.522525787353516,
      "activations/layer15_attention_weight_min": -28.898061752319336,
      "activations/layer16_attention_weight_max": 32.978206634521484,
      "activations/layer16_attention_weight_min": -27.39057731628418,
      "activations/layer17_attention_weight_max": 49.165218353271484,
      "activations/layer17_attention_weight_min": -42.70920181274414,
      "activations/layer18_attention_weight_max": 46.915218353271484,
      "activations/layer18_attention_weight_min": -36.60894012451172,
      "activations/layer19_attention_weight_max": 22.111127853393555,
      "activations/layer19_attention_weight_min": -22.686080932617188,
      "activations/layer1_attention_weight_max": 16.288713455200195,
      "activations/layer1_attention_weight_min": -13.303563117980957,
      "activations/layer20_attention_weight_max": 22.62371253967285,
      "activations/layer20_attention_weight_min": -23.074840545654297,
      "activations/layer21_attention_weight_max": 35.72968673706055,
      "activations/layer21_attention_weight_min": -25.828500747680664,
      "activations/layer22_attention_weight_max": 31.554840087890625,
      "activations/layer22_attention_weight_min": -25.463552474975586,
      "activations/layer23_attention_weight_max": 36.49011993408203,
      "activations/layer23_attention_weight_min": -24.850223541259766,
      "activations/layer2_attention_weight_max": 30.123119354248047,
      "activations/layer2_attention_weight_min": -28.224658966064453,
      "activations/layer3_attention_weight_max": 83.76324462890625,
      "activations/layer3_attention_weight_min": -87.65550231933594,
      "activations/layer4_attention_weight_max": 96.50177764892578,
      "activations/layer4_attention_weight_min": -93.00360870361328,
      "activations/layer5_attention_weight_max": 74.10931396484375,
      "activations/layer5_attention_weight_min": -77.17829895019531,
      "activations/layer6_attention_weight_max": 51.738155364990234,
      "activations/layer6_attention_weight_min": -51.75851058959961,
      "activations/layer7_attention_weight_max": 75.4011001586914,
      "activations/layer7_attention_weight_min": -76.0562515258789,
      "activations/layer8_attention_weight_max": 46.55278015136719,
      "activations/layer8_attention_weight_min": -46.79936218261719,
      "activations/layer9_attention_weight_max": 49.27177810668945,
      "activations/layer9_attention_weight_min": -49.659393310546875,
      "epoch": 9.39,
      "learning_rate": 9.037575757575758e-05,
      "loss": 2.8049,
      "step": 161550
    },
    {
      "activations/layer0_attention_weight_max": 14.747061729431152,
      "activations/layer0_attention_weight_min": -14.129170417785645,
      "activations/layer10_attention_weight_max": 35.45673751831055,
      "activations/layer10_attention_weight_min": -34.200782775878906,
      "activations/layer11_attention_weight_max": 30.970563888549805,
      "activations/layer11_attention_weight_min": -30.818578720092773,
      "activations/layer12_attention_weight_max": 18.774677276611328,
      "activations/layer12_attention_weight_min": -29.103166580200195,
      "activations/layer13_attention_weight_max": 37.10276794433594,
      "activations/layer13_attention_weight_min": -30.253023147583008,
      "activations/layer14_attention_weight_max": 35.17072677612305,
      "activations/layer14_attention_weight_min": -30.342082977294922,
      "activations/layer15_attention_weight_max": 33.2238883972168,
      "activations/layer15_attention_weight_min": -30.435941696166992,
      "activations/layer16_attention_weight_max": 34.88746643066406,
      "activations/layer16_attention_weight_min": -26.857927322387695,
      "activations/layer17_attention_weight_max": 49.77972412109375,
      "activations/layer17_attention_weight_min": -40.23280715942383,
      "activations/layer18_attention_weight_max": 43.622230529785156,
      "activations/layer18_attention_weight_min": -35.54452896118164,
      "activations/layer19_attention_weight_max": 21.949607849121094,
      "activations/layer19_attention_weight_min": -22.282140731811523,
      "activations/layer1_attention_weight_max": 16.279735565185547,
      "activations/layer1_attention_weight_min": -14.798866271972656,
      "activations/layer20_attention_weight_max": 24.468826293945312,
      "activations/layer20_attention_weight_min": -24.610340118408203,
      "activations/layer21_attention_weight_max": 38.45221710205078,
      "activations/layer21_attention_weight_min": -24.324121475219727,
      "activations/layer22_attention_weight_max": 33.78873062133789,
      "activations/layer22_attention_weight_min": -25.92386817932129,
      "activations/layer23_attention_weight_max": 38.97929763793945,
      "activations/layer23_attention_weight_min": -25.84225845336914,
      "activations/layer2_attention_weight_max": 29.33302116394043,
      "activations/layer2_attention_weight_min": -28.934810638427734,
      "activations/layer3_attention_weight_max": 83.89340209960938,
      "activations/layer3_attention_weight_min": -90.41630554199219,
      "activations/layer4_attention_weight_max": 90.51351165771484,
      "activations/layer4_attention_weight_min": -91.00447845458984,
      "activations/layer5_attention_weight_max": 75.29625701904297,
      "activations/layer5_attention_weight_min": -74.6895523071289,
      "activations/layer6_attention_weight_max": 53.44871520996094,
      "activations/layer6_attention_weight_min": -55.08201599121094,
      "activations/layer7_attention_weight_max": 75.34457397460938,
      "activations/layer7_attention_weight_min": -69.20149993896484,
      "activations/layer8_attention_weight_max": 43.61976623535156,
      "activations/layer8_attention_weight_min": -43.7776985168457,
      "activations/layer9_attention_weight_max": 45.954689025878906,
      "activations/layer9_attention_weight_min": -44.40876388549805,
      "epoch": 9.39,
      "learning_rate": 9.035681818181817e-05,
      "loss": 2.7985,
      "step": 161600
    },
    {
      "activations/layer0_attention_weight_max": 16.88508415222168,
      "activations/layer0_attention_weight_min": -13.59825325012207,
      "activations/layer10_attention_weight_max": 34.56966018676758,
      "activations/layer10_attention_weight_min": -35.721168518066406,
      "activations/layer11_attention_weight_max": 31.072675704956055,
      "activations/layer11_attention_weight_min": -32.075706481933594,
      "activations/layer12_attention_weight_max": 18.44833755493164,
      "activations/layer12_attention_weight_min": -25.510595321655273,
      "activations/layer13_attention_weight_max": 37.75200271606445,
      "activations/layer13_attention_weight_min": -31.985130310058594,
      "activations/layer14_attention_weight_max": 36.41261291503906,
      "activations/layer14_attention_weight_min": -30.043277740478516,
      "activations/layer15_attention_weight_max": 31.8636531829834,
      "activations/layer15_attention_weight_min": -28.843408584594727,
      "activations/layer16_attention_weight_max": 28.50145149230957,
      "activations/layer16_attention_weight_min": -26.851655960083008,
      "activations/layer17_attention_weight_max": 53.90165710449219,
      "activations/layer17_attention_weight_min": -43.39557647705078,
      "activations/layer18_attention_weight_max": 45.95315170288086,
      "activations/layer18_attention_weight_min": -36.92433166503906,
      "activations/layer19_attention_weight_max": 21.407665252685547,
      "activations/layer19_attention_weight_min": -19.9742488861084,
      "activations/layer1_attention_weight_max": 16.832550048828125,
      "activations/layer1_attention_weight_min": -14.058943748474121,
      "activations/layer20_attention_weight_max": 20.148595809936523,
      "activations/layer20_attention_weight_min": -21.809717178344727,
      "activations/layer21_attention_weight_max": 35.963497161865234,
      "activations/layer21_attention_weight_min": -23.164648056030273,
      "activations/layer22_attention_weight_max": 29.919189453125,
      "activations/layer22_attention_weight_min": -25.283954620361328,
      "activations/layer23_attention_weight_max": 34.77676010131836,
      "activations/layer23_attention_weight_min": -21.820526123046875,
      "activations/layer2_attention_weight_max": 30.909778594970703,
      "activations/layer2_attention_weight_min": -29.296611785888672,
      "activations/layer3_attention_weight_max": 89.92132568359375,
      "activations/layer3_attention_weight_min": -88.70228576660156,
      "activations/layer4_attention_weight_max": 100.39470672607422,
      "activations/layer4_attention_weight_min": -100.28202056884766,
      "activations/layer5_attention_weight_max": 79.10333251953125,
      "activations/layer5_attention_weight_min": -80.22001647949219,
      "activations/layer6_attention_weight_max": 54.06425857543945,
      "activations/layer6_attention_weight_min": -54.14137268066406,
      "activations/layer7_attention_weight_max": 79.10010528564453,
      "activations/layer7_attention_weight_min": -73.935546875,
      "activations/layer8_attention_weight_max": 44.58900833129883,
      "activations/layer8_attention_weight_min": -47.06181335449219,
      "activations/layer9_attention_weight_max": 54.94929122924805,
      "activations/layer9_attention_weight_min": -50.74013900756836,
      "epoch": 9.39,
      "learning_rate": 9.033787878787878e-05,
      "loss": 2.7958,
      "step": 161650
    },
    {
      "activations/layer0_attention_weight_max": 16.025611877441406,
      "activations/layer0_attention_weight_min": -13.766046524047852,
      "activations/layer10_attention_weight_max": 34.44635009765625,
      "activations/layer10_attention_weight_min": -34.03450393676758,
      "activations/layer11_attention_weight_max": 29.001728057861328,
      "activations/layer11_attention_weight_min": -31.199596405029297,
      "activations/layer12_attention_weight_max": 17.469104766845703,
      "activations/layer12_attention_weight_min": -25.220605850219727,
      "activations/layer13_attention_weight_max": 52.20737075805664,
      "activations/layer13_attention_weight_min": -31.653812408447266,
      "activations/layer14_attention_weight_max": 47.036766052246094,
      "activations/layer14_attention_weight_min": -32.78260803222656,
      "activations/layer15_attention_weight_max": 35.94104766845703,
      "activations/layer15_attention_weight_min": -30.110551834106445,
      "activations/layer16_attention_weight_max": 30.033018112182617,
      "activations/layer16_attention_weight_min": -30.809818267822266,
      "activations/layer17_attention_weight_max": 52.61696243286133,
      "activations/layer17_attention_weight_min": -48.9918098449707,
      "activations/layer18_attention_weight_max": 52.012367248535156,
      "activations/layer18_attention_weight_min": -37.7677001953125,
      "activations/layer19_attention_weight_max": 19.9934024810791,
      "activations/layer19_attention_weight_min": -22.895488739013672,
      "activations/layer1_attention_weight_max": 17.806011199951172,
      "activations/layer1_attention_weight_min": -15.345484733581543,
      "activations/layer20_attention_weight_max": 21.89740753173828,
      "activations/layer20_attention_weight_min": -22.920331954956055,
      "activations/layer21_attention_weight_max": 38.08986282348633,
      "activations/layer21_attention_weight_min": -24.20290184020996,
      "activations/layer22_attention_weight_max": 30.004451751708984,
      "activations/layer22_attention_weight_min": -27.49787139892578,
      "activations/layer23_attention_weight_max": 37.532623291015625,
      "activations/layer23_attention_weight_min": -27.671688079833984,
      "activations/layer2_attention_weight_max": 30.310758590698242,
      "activations/layer2_attention_weight_min": -29.86275863647461,
      "activations/layer3_attention_weight_max": 84.2400894165039,
      "activations/layer3_attention_weight_min": -88.61308288574219,
      "activations/layer4_attention_weight_max": 96.12169647216797,
      "activations/layer4_attention_weight_min": -91.13683319091797,
      "activations/layer5_attention_weight_max": 75.42727661132812,
      "activations/layer5_attention_weight_min": -79.73396301269531,
      "activations/layer6_attention_weight_max": 51.94977569580078,
      "activations/layer6_attention_weight_min": -50.61994171142578,
      "activations/layer7_attention_weight_max": 77.69573974609375,
      "activations/layer7_attention_weight_min": -69.18758392333984,
      "activations/layer8_attention_weight_max": 43.31343078613281,
      "activations/layer8_attention_weight_min": -42.99244689941406,
      "activations/layer9_attention_weight_max": 46.39988327026367,
      "activations/layer9_attention_weight_min": -47.507869720458984,
      "epoch": 9.4,
      "learning_rate": 9.03189393939394e-05,
      "loss": 2.7929,
      "step": 161700
    },
    {
      "activations/layer0_attention_weight_max": 14.93689250946045,
      "activations/layer0_attention_weight_min": -13.887262344360352,
      "activations/layer10_attention_weight_max": 33.51875686645508,
      "activations/layer10_attention_weight_min": -34.141944885253906,
      "activations/layer11_attention_weight_max": 30.4594783782959,
      "activations/layer11_attention_weight_min": -30.618879318237305,
      "activations/layer12_attention_weight_max": 19.131145477294922,
      "activations/layer12_attention_weight_min": -24.027318954467773,
      "activations/layer13_attention_weight_max": 38.2034912109375,
      "activations/layer13_attention_weight_min": -31.03147315979004,
      "activations/layer14_attention_weight_max": 34.92866516113281,
      "activations/layer14_attention_weight_min": -29.223907470703125,
      "activations/layer15_attention_weight_max": 33.021141052246094,
      "activations/layer15_attention_weight_min": -28.20439338684082,
      "activations/layer16_attention_weight_max": 29.87906837463379,
      "activations/layer16_attention_weight_min": -27.2724552154541,
      "activations/layer17_attention_weight_max": 52.2713508605957,
      "activations/layer17_attention_weight_min": -41.40327072143555,
      "activations/layer18_attention_weight_max": 45.11260223388672,
      "activations/layer18_attention_weight_min": -34.892364501953125,
      "activations/layer19_attention_weight_max": 21.639114379882812,
      "activations/layer19_attention_weight_min": -23.69815444946289,
      "activations/layer1_attention_weight_max": 16.461170196533203,
      "activations/layer1_attention_weight_min": -15.908394813537598,
      "activations/layer20_attention_weight_max": 21.348949432373047,
      "activations/layer20_attention_weight_min": -22.298166275024414,
      "activations/layer21_attention_weight_max": 38.301002502441406,
      "activations/layer21_attention_weight_min": -22.822189331054688,
      "activations/layer22_attention_weight_max": 29.918691635131836,
      "activations/layer22_attention_weight_min": -25.75164031982422,
      "activations/layer23_attention_weight_max": 37.094058990478516,
      "activations/layer23_attention_weight_min": -23.697612762451172,
      "activations/layer2_attention_weight_max": 29.14095687866211,
      "activations/layer2_attention_weight_min": -28.295467376708984,
      "activations/layer3_attention_weight_max": 85.12440490722656,
      "activations/layer3_attention_weight_min": -93.60808563232422,
      "activations/layer4_attention_weight_max": 96.7356185913086,
      "activations/layer4_attention_weight_min": -91.5929183959961,
      "activations/layer5_attention_weight_max": 77.52000427246094,
      "activations/layer5_attention_weight_min": -74.23776245117188,
      "activations/layer6_attention_weight_max": 51.971702575683594,
      "activations/layer6_attention_weight_min": -50.96406555175781,
      "activations/layer7_attention_weight_max": 75.53167724609375,
      "activations/layer7_attention_weight_min": -72.50245666503906,
      "activations/layer8_attention_weight_max": 44.426368713378906,
      "activations/layer8_attention_weight_min": -45.200279235839844,
      "activations/layer9_attention_weight_max": 46.9436149597168,
      "activations/layer9_attention_weight_min": -47.66498947143555,
      "epoch": 9.4,
      "learning_rate": 9.029999999999999e-05,
      "loss": 2.7966,
      "step": 161750
    },
    {
      "activations/layer0_attention_weight_max": 17.488168716430664,
      "activations/layer0_attention_weight_min": -14.146953582763672,
      "activations/layer10_attention_weight_max": 44.08677291870117,
      "activations/layer10_attention_weight_min": -37.231266021728516,
      "activations/layer11_attention_weight_max": 41.598480224609375,
      "activations/layer11_attention_weight_min": -32.328773498535156,
      "activations/layer12_attention_weight_max": 23.451881408691406,
      "activations/layer12_attention_weight_min": -30.13703727722168,
      "activations/layer13_attention_weight_max": 38.8345832824707,
      "activations/layer13_attention_weight_min": -28.480905532836914,
      "activations/layer14_attention_weight_max": 37.37620162963867,
      "activations/layer14_attention_weight_min": -30.786540985107422,
      "activations/layer15_attention_weight_max": 32.201908111572266,
      "activations/layer15_attention_weight_min": -29.56609344482422,
      "activations/layer16_attention_weight_max": 31.38963508605957,
      "activations/layer16_attention_weight_min": -29.822063446044922,
      "activations/layer17_attention_weight_max": 55.951541900634766,
      "activations/layer17_attention_weight_min": -45.45999526977539,
      "activations/layer18_attention_weight_max": 46.74767303466797,
      "activations/layer18_attention_weight_min": -39.89324951171875,
      "activations/layer19_attention_weight_max": 23.68404197692871,
      "activations/layer19_attention_weight_min": -20.152854919433594,
      "activations/layer1_attention_weight_max": 17.15094757080078,
      "activations/layer1_attention_weight_min": -15.102352142333984,
      "activations/layer20_attention_weight_max": 21.650646209716797,
      "activations/layer20_attention_weight_min": -20.82796859741211,
      "activations/layer21_attention_weight_max": 33.30656051635742,
      "activations/layer21_attention_weight_min": -22.332956314086914,
      "activations/layer22_attention_weight_max": 31.642507553100586,
      "activations/layer22_attention_weight_min": -26.40894317626953,
      "activations/layer23_attention_weight_max": 36.944557189941406,
      "activations/layer23_attention_weight_min": -23.10650634765625,
      "activations/layer2_attention_weight_max": 32.15348815917969,
      "activations/layer2_attention_weight_min": -31.01272201538086,
      "activations/layer3_attention_weight_max": 97.59201049804688,
      "activations/layer3_attention_weight_min": -93.38633728027344,
      "activations/layer4_attention_weight_max": 103.14739990234375,
      "activations/layer4_attention_weight_min": -96.99156188964844,
      "activations/layer5_attention_weight_max": 77.59988403320312,
      "activations/layer5_attention_weight_min": -83.250732421875,
      "activations/layer6_attention_weight_max": 54.31321334838867,
      "activations/layer6_attention_weight_min": -52.89305877685547,
      "activations/layer7_attention_weight_max": 87.1309814453125,
      "activations/layer7_attention_weight_min": -73.42265319824219,
      "activations/layer8_attention_weight_max": 51.36857223510742,
      "activations/layer8_attention_weight_min": -44.61405563354492,
      "activations/layer9_attention_weight_max": 65.04316711425781,
      "activations/layer9_attention_weight_min": -53.46160888671875,
      "epoch": 9.4,
      "learning_rate": 9.02810606060606e-05,
      "loss": 2.7959,
      "step": 161800
    },
    {
      "activations/layer0_attention_weight_max": 14.733652114868164,
      "activations/layer0_attention_weight_min": -13.845367431640625,
      "activations/layer10_attention_weight_max": 39.711585998535156,
      "activations/layer10_attention_weight_min": -38.00783157348633,
      "activations/layer11_attention_weight_max": 34.540260314941406,
      "activations/layer11_attention_weight_min": -32.848358154296875,
      "activations/layer12_attention_weight_max": 20.632858276367188,
      "activations/layer12_attention_weight_min": -23.489519119262695,
      "activations/layer13_attention_weight_max": 38.87397003173828,
      "activations/layer13_attention_weight_min": -32.60240173339844,
      "activations/layer14_attention_weight_max": 39.404876708984375,
      "activations/layer14_attention_weight_min": -30.69174575805664,
      "activations/layer15_attention_weight_max": 36.10251998901367,
      "activations/layer15_attention_weight_min": -30.508464813232422,
      "activations/layer16_attention_weight_max": 31.951980590820312,
      "activations/layer16_attention_weight_min": -29.262956619262695,
      "activations/layer17_attention_weight_max": 54.39624786376953,
      "activations/layer17_attention_weight_min": -44.411216735839844,
      "activations/layer18_attention_weight_max": 49.3489990234375,
      "activations/layer18_attention_weight_min": -38.1207160949707,
      "activations/layer19_attention_weight_max": 23.404212951660156,
      "activations/layer19_attention_weight_min": -24.612140655517578,
      "activations/layer1_attention_weight_max": 16.636642456054688,
      "activations/layer1_attention_weight_min": -15.00464916229248,
      "activations/layer20_attention_weight_max": 25.348543167114258,
      "activations/layer20_attention_weight_min": -20.92888641357422,
      "activations/layer21_attention_weight_max": 42.25552749633789,
      "activations/layer21_attention_weight_min": -23.99944496154785,
      "activations/layer22_attention_weight_max": 34.76376724243164,
      "activations/layer22_attention_weight_min": -25.93760871887207,
      "activations/layer23_attention_weight_max": 39.600433349609375,
      "activations/layer23_attention_weight_min": -21.725507736206055,
      "activations/layer2_attention_weight_max": 30.9719295501709,
      "activations/layer2_attention_weight_min": -29.993892669677734,
      "activations/layer3_attention_weight_max": 90.59309387207031,
      "activations/layer3_attention_weight_min": -91.32452392578125,
      "activations/layer4_attention_weight_max": 104.06807708740234,
      "activations/layer4_attention_weight_min": -97.70677185058594,
      "activations/layer5_attention_weight_max": 82.51548767089844,
      "activations/layer5_attention_weight_min": -82.17249298095703,
      "activations/layer6_attention_weight_max": 56.20355224609375,
      "activations/layer6_attention_weight_min": -56.407928466796875,
      "activations/layer7_attention_weight_max": 83.41759490966797,
      "activations/layer7_attention_weight_min": -76.87442779541016,
      "activations/layer8_attention_weight_max": 48.4891357421875,
      "activations/layer8_attention_weight_min": -51.72987365722656,
      "activations/layer9_attention_weight_max": 52.49744415283203,
      "activations/layer9_attention_weight_min": -53.87421798706055,
      "epoch": 9.4,
      "learning_rate": 9.02621212121212e-05,
      "loss": 2.7946,
      "step": 161850
    },
    {
      "activations/layer0_attention_weight_max": 14.90407657623291,
      "activations/layer0_attention_weight_min": -13.757511138916016,
      "activations/layer10_attention_weight_max": 37.11322784423828,
      "activations/layer10_attention_weight_min": -35.055416107177734,
      "activations/layer11_attention_weight_max": 30.818389892578125,
      "activations/layer11_attention_weight_min": -31.286582946777344,
      "activations/layer12_attention_weight_max": 19.342803955078125,
      "activations/layer12_attention_weight_min": -32.31763458251953,
      "activations/layer13_attention_weight_max": 35.51608657836914,
      "activations/layer13_attention_weight_min": -29.968505859375,
      "activations/layer14_attention_weight_max": 37.38864517211914,
      "activations/layer14_attention_weight_min": -29.691055297851562,
      "activations/layer15_attention_weight_max": 33.21195983886719,
      "activations/layer15_attention_weight_min": -27.307865142822266,
      "activations/layer16_attention_weight_max": 29.61635971069336,
      "activations/layer16_attention_weight_min": -28.240768432617188,
      "activations/layer17_attention_weight_max": 52.9959602355957,
      "activations/layer17_attention_weight_min": -42.70225143432617,
      "activations/layer18_attention_weight_max": 48.573856353759766,
      "activations/layer18_attention_weight_min": -36.59272384643555,
      "activations/layer19_attention_weight_max": 22.19670867919922,
      "activations/layer19_attention_weight_min": -21.53232192993164,
      "activations/layer1_attention_weight_max": 16.953393936157227,
      "activations/layer1_attention_weight_min": -15.578947067260742,
      "activations/layer20_attention_weight_max": 23.621599197387695,
      "activations/layer20_attention_weight_min": -22.73934555053711,
      "activations/layer21_attention_weight_max": 37.548255920410156,
      "activations/layer21_attention_weight_min": -21.29175567626953,
      "activations/layer22_attention_weight_max": 32.51921081542969,
      "activations/layer22_attention_weight_min": -24.8557186126709,
      "activations/layer23_attention_weight_max": 34.19927978515625,
      "activations/layer23_attention_weight_min": -22.67646598815918,
      "activations/layer2_attention_weight_max": 31.416866302490234,
      "activations/layer2_attention_weight_min": -30.475914001464844,
      "activations/layer3_attention_weight_max": 94.43083190917969,
      "activations/layer3_attention_weight_min": -96.56127166748047,
      "activations/layer4_attention_weight_max": 99.16596221923828,
      "activations/layer4_attention_weight_min": -95.79829406738281,
      "activations/layer5_attention_weight_max": 77.4414291381836,
      "activations/layer5_attention_weight_min": -74.330078125,
      "activations/layer6_attention_weight_max": 54.510501861572266,
      "activations/layer6_attention_weight_min": -52.08443069458008,
      "activations/layer7_attention_weight_max": 81.6921157836914,
      "activations/layer7_attention_weight_min": -71.7373046875,
      "activations/layer8_attention_weight_max": 47.555198669433594,
      "activations/layer8_attention_weight_min": -44.4815788269043,
      "activations/layer9_attention_weight_max": 51.14692687988281,
      "activations/layer9_attention_weight_min": -46.30514144897461,
      "epoch": 9.41,
      "learning_rate": 9.02431818181818e-05,
      "loss": 2.7981,
      "step": 161900
    },
    {
      "activations/layer0_attention_weight_max": 15.324671745300293,
      "activations/layer0_attention_weight_min": -13.126220703125,
      "activations/layer10_attention_weight_max": 36.88990783691406,
      "activations/layer10_attention_weight_min": -38.265625,
      "activations/layer11_attention_weight_max": 33.98719024658203,
      "activations/layer11_attention_weight_min": -32.88709259033203,
      "activations/layer12_attention_weight_max": 27.795063018798828,
      "activations/layer12_attention_weight_min": -25.070117950439453,
      "activations/layer13_attention_weight_max": 43.08625793457031,
      "activations/layer13_attention_weight_min": -30.12540054321289,
      "activations/layer14_attention_weight_max": 43.04872131347656,
      "activations/layer14_attention_weight_min": -32.079254150390625,
      "activations/layer15_attention_weight_max": 36.95604705810547,
      "activations/layer15_attention_weight_min": -29.94081687927246,
      "activations/layer16_attention_weight_max": 33.88636779785156,
      "activations/layer16_attention_weight_min": -29.767301559448242,
      "activations/layer17_attention_weight_max": 57.5981330871582,
      "activations/layer17_attention_weight_min": -53.25364685058594,
      "activations/layer18_attention_weight_max": 53.418922424316406,
      "activations/layer18_attention_weight_min": -44.8558235168457,
      "activations/layer19_attention_weight_max": 25.28931427001953,
      "activations/layer19_attention_weight_min": -24.11481475830078,
      "activations/layer1_attention_weight_max": 17.752525329589844,
      "activations/layer1_attention_weight_min": -14.776236534118652,
      "activations/layer20_attention_weight_max": 27.464365005493164,
      "activations/layer20_attention_weight_min": -23.8101749420166,
      "activations/layer21_attention_weight_max": 51.45847702026367,
      "activations/layer21_attention_weight_min": -24.911449432373047,
      "activations/layer22_attention_weight_max": 32.39844512939453,
      "activations/layer22_attention_weight_min": -25.656566619873047,
      "activations/layer23_attention_weight_max": 35.711063385009766,
      "activations/layer23_attention_weight_min": -22.651840209960938,
      "activations/layer2_attention_weight_max": 31.540462493896484,
      "activations/layer2_attention_weight_min": -29.90530776977539,
      "activations/layer3_attention_weight_max": 88.26876068115234,
      "activations/layer3_attention_weight_min": -85.41512298583984,
      "activations/layer4_attention_weight_max": 94.1735610961914,
      "activations/layer4_attention_weight_min": -91.55431365966797,
      "activations/layer5_attention_weight_max": 78.98209381103516,
      "activations/layer5_attention_weight_min": -76.90899658203125,
      "activations/layer6_attention_weight_max": 52.539390563964844,
      "activations/layer6_attention_weight_min": -50.41749954223633,
      "activations/layer7_attention_weight_max": 78.29501342773438,
      "activations/layer7_attention_weight_min": -71.00654602050781,
      "activations/layer8_attention_weight_max": 45.46002197265625,
      "activations/layer8_attention_weight_min": -46.49517059326172,
      "activations/layer9_attention_weight_max": 58.38450241088867,
      "activations/layer9_attention_weight_min": -48.76138687133789,
      "epoch": 9.41,
      "learning_rate": 9.022424242424242e-05,
      "loss": 2.7969,
      "step": 161950
    },
    {
      "activations/layer0_attention_weight_max": 16.141572952270508,
      "activations/layer0_attention_weight_min": -13.589532852172852,
      "activations/layer10_attention_weight_max": 41.50682067871094,
      "activations/layer10_attention_weight_min": -39.95914077758789,
      "activations/layer11_attention_weight_max": 35.45860290527344,
      "activations/layer11_attention_weight_min": -33.170677185058594,
      "activations/layer12_attention_weight_max": 17.5152587890625,
      "activations/layer12_attention_weight_min": -31.01544952392578,
      "activations/layer13_attention_weight_max": 38.98219299316406,
      "activations/layer13_attention_weight_min": -30.395946502685547,
      "activations/layer14_attention_weight_max": 36.4211311340332,
      "activations/layer14_attention_weight_min": -32.347145080566406,
      "activations/layer15_attention_weight_max": 33.288856506347656,
      "activations/layer15_attention_weight_min": -31.789993286132812,
      "activations/layer16_attention_weight_max": 28.745222091674805,
      "activations/layer16_attention_weight_min": -29.823787689208984,
      "activations/layer17_attention_weight_max": 53.764400482177734,
      "activations/layer17_attention_weight_min": -44.767051696777344,
      "activations/layer18_attention_weight_max": 48.39200210571289,
      "activations/layer18_attention_weight_min": -36.8784065246582,
      "activations/layer19_attention_weight_max": 20.531736373901367,
      "activations/layer19_attention_weight_min": -20.95944595336914,
      "activations/layer1_attention_weight_max": 18.2469539642334,
      "activations/layer1_attention_weight_min": -16.747665405273438,
      "activations/layer20_attention_weight_max": 23.245960235595703,
      "activations/layer20_attention_weight_min": -20.690231323242188,
      "activations/layer21_attention_weight_max": 35.0694580078125,
      "activations/layer21_attention_weight_min": -23.58848762512207,
      "activations/layer22_attention_weight_max": 29.95406150817871,
      "activations/layer22_attention_weight_min": -26.700952529907227,
      "activations/layer23_attention_weight_max": 37.46141052246094,
      "activations/layer23_attention_weight_min": -22.63922119140625,
      "activations/layer2_attention_weight_max": 31.37897300720215,
      "activations/layer2_attention_weight_min": -30.678115844726562,
      "activations/layer3_attention_weight_max": 100.34142303466797,
      "activations/layer3_attention_weight_min": -94.87743377685547,
      "activations/layer4_attention_weight_max": 111.44219207763672,
      "activations/layer4_attention_weight_min": -105.30213928222656,
      "activations/layer5_attention_weight_max": 80.7820053100586,
      "activations/layer5_attention_weight_min": -79.26515197753906,
      "activations/layer6_attention_weight_max": 56.671714782714844,
      "activations/layer6_attention_weight_min": -55.39809036254883,
      "activations/layer7_attention_weight_max": 81.07998657226562,
      "activations/layer7_attention_weight_min": -75.91138458251953,
      "activations/layer8_attention_weight_max": 45.76263427734375,
      "activations/layer8_attention_weight_min": -48.032081604003906,
      "activations/layer9_attention_weight_max": 59.13508987426758,
      "activations/layer9_attention_weight_min": -56.08242416381836,
      "epoch": 9.41,
      "learning_rate": 9.020530303030302e-05,
      "loss": 2.7904,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.5515,
      "eval_samples_per_second": 502.133,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.5515,
      "eval_openwebtext_samples_per_second": 502.133,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 2.0722,
      "eval_wikitext_samples_per_second": 220.053,
      "step": 162000
    },
    {
      "epoch": 9.41,
      "eval_lambada_loss": 2.625,
      "eval_lambada_ppl": 13.804574186067095,
      "eval_lambada_runtime": 9.6406,
      "eval_lambada_samples_per_second": 505.052,
      "step": 162000
    },
    {
      "activations/layer0_attention_weight_max": 14.935195922851562,
      "activations/layer0_attention_weight_min": -14.262683868408203,
      "activations/layer10_attention_weight_max": 39.041053771972656,
      "activations/layer10_attention_weight_min": -36.33344268798828,
      "activations/layer11_attention_weight_max": 31.852354049682617,
      "activations/layer11_attention_weight_min": -32.348392486572266,
      "activations/layer12_attention_weight_max": 22.347835540771484,
      "activations/layer12_attention_weight_min": -26.40007972717285,
      "activations/layer13_attention_weight_max": 46.1048698425293,
      "activations/layer13_attention_weight_min": -34.17265319824219,
      "activations/layer14_attention_weight_max": 38.79326248168945,
      "activations/layer14_attention_weight_min": -30.683242797851562,
      "activations/layer15_attention_weight_max": 36.97528839111328,
      "activations/layer15_attention_weight_min": -29.504629135131836,
      "activations/layer16_attention_weight_max": 36.58164596557617,
      "activations/layer16_attention_weight_min": -30.84636116027832,
      "activations/layer17_attention_weight_max": 59.960205078125,
      "activations/layer17_attention_weight_min": -48.436222076416016,
      "activations/layer18_attention_weight_max": 50.892974853515625,
      "activations/layer18_attention_weight_min": -41.208221435546875,
      "activations/layer19_attention_weight_max": 24.55575180053711,
      "activations/layer19_attention_weight_min": -21.895259857177734,
      "activations/layer1_attention_weight_max": 16.774473190307617,
      "activations/layer1_attention_weight_min": -14.27790641784668,
      "activations/layer20_attention_weight_max": 26.03102684020996,
      "activations/layer20_attention_weight_min": -21.890111923217773,
      "activations/layer21_attention_weight_max": 37.97703552246094,
      "activations/layer21_attention_weight_min": -22.857906341552734,
      "activations/layer22_attention_weight_max": 35.42084503173828,
      "activations/layer22_attention_weight_min": -25.49469757080078,
      "activations/layer23_attention_weight_max": 36.784725189208984,
      "activations/layer23_attention_weight_min": -24.200218200683594,
      "activations/layer2_attention_weight_max": 30.022520065307617,
      "activations/layer2_attention_weight_min": -28.62804412841797,
      "activations/layer3_attention_weight_max": 83.58495330810547,
      "activations/layer3_attention_weight_min": -85.38490295410156,
      "activations/layer4_attention_weight_max": 96.92354583740234,
      "activations/layer4_attention_weight_min": -93.2281723022461,
      "activations/layer5_attention_weight_max": 78.48241424560547,
      "activations/layer5_attention_weight_min": -79.79376220703125,
      "activations/layer6_attention_weight_max": 53.670066833496094,
      "activations/layer6_attention_weight_min": -53.47297668457031,
      "activations/layer7_attention_weight_max": 77.97660827636719,
      "activations/layer7_attention_weight_min": -72.76520538330078,
      "activations/layer8_attention_weight_max": 43.96638107299805,
      "activations/layer8_attention_weight_min": -46.7403450012207,
      "activations/layer9_attention_weight_max": 49.223262786865234,
      "activations/layer9_attention_weight_min": -50.03350067138672,
      "epoch": 9.42,
      "learning_rate": 9.018636363636364e-05,
      "loss": 2.7975,
      "step": 162050
    },
    {
      "activations/layer0_attention_weight_max": 15.110527038574219,
      "activations/layer0_attention_weight_min": -13.684231758117676,
      "activations/layer10_attention_weight_max": 35.631866455078125,
      "activations/layer10_attention_weight_min": -35.782752990722656,
      "activations/layer11_attention_weight_max": 30.348432540893555,
      "activations/layer11_attention_weight_min": -32.58979034423828,
      "activations/layer12_attention_weight_max": 20.777278900146484,
      "activations/layer12_attention_weight_min": -27.781347274780273,
      "activations/layer13_attention_weight_max": 38.750694274902344,
      "activations/layer13_attention_weight_min": -30.325801849365234,
      "activations/layer14_attention_weight_max": 37.99229431152344,
      "activations/layer14_attention_weight_min": -30.828262329101562,
      "activations/layer15_attention_weight_max": 35.29281234741211,
      "activations/layer15_attention_weight_min": -28.69084358215332,
      "activations/layer16_attention_weight_max": 35.14955139160156,
      "activations/layer16_attention_weight_min": -28.26892852783203,
      "activations/layer17_attention_weight_max": 55.23674392700195,
      "activations/layer17_attention_weight_min": -42.21929931640625,
      "activations/layer18_attention_weight_max": 48.841651916503906,
      "activations/layer18_attention_weight_min": -36.27915954589844,
      "activations/layer19_attention_weight_max": 22.302457809448242,
      "activations/layer19_attention_weight_min": -19.734880447387695,
      "activations/layer1_attention_weight_max": 17.92766761779785,
      "activations/layer1_attention_weight_min": -14.298978805541992,
      "activations/layer20_attention_weight_max": 25.964088439941406,
      "activations/layer20_attention_weight_min": -24.4304141998291,
      "activations/layer21_attention_weight_max": 37.035865783691406,
      "activations/layer21_attention_weight_min": -22.312942504882812,
      "activations/layer22_attention_weight_max": 33.70474624633789,
      "activations/layer22_attention_weight_min": -26.516077041625977,
      "activations/layer23_attention_weight_max": 40.06470489501953,
      "activations/layer23_attention_weight_min": -22.593891143798828,
      "activations/layer2_attention_weight_max": 29.403112411499023,
      "activations/layer2_attention_weight_min": -27.23300552368164,
      "activations/layer3_attention_weight_max": 83.39106750488281,
      "activations/layer3_attention_weight_min": -82.13556671142578,
      "activations/layer4_attention_weight_max": 95.6021957397461,
      "activations/layer4_attention_weight_min": -87.57696533203125,
      "activations/layer5_attention_weight_max": 76.51946258544922,
      "activations/layer5_attention_weight_min": -75.19831848144531,
      "activations/layer6_attention_weight_max": 53.18616485595703,
      "activations/layer6_attention_weight_min": -50.945899963378906,
      "activations/layer7_attention_weight_max": 79.29312133789062,
      "activations/layer7_attention_weight_min": -71.7885513305664,
      "activations/layer8_attention_weight_max": 44.317413330078125,
      "activations/layer8_attention_weight_min": -47.977272033691406,
      "activations/layer9_attention_weight_max": 48.684200286865234,
      "activations/layer9_attention_weight_min": -49.3293342590332,
      "epoch": 9.42,
      "learning_rate": 9.016742424242424e-05,
      "loss": 2.7928,
      "step": 162100
    },
    {
      "activations/layer0_attention_weight_max": 14.595064163208008,
      "activations/layer0_attention_weight_min": -14.386839866638184,
      "activations/layer10_attention_weight_max": 36.1798095703125,
      "activations/layer10_attention_weight_min": -34.139869689941406,
      "activations/layer11_attention_weight_max": 29.527982711791992,
      "activations/layer11_attention_weight_min": -32.52051544189453,
      "activations/layer12_attention_weight_max": 18.554428100585938,
      "activations/layer12_attention_weight_min": -25.96927261352539,
      "activations/layer13_attention_weight_max": 42.09794998168945,
      "activations/layer13_attention_weight_min": -33.64490509033203,
      "activations/layer14_attention_weight_max": 36.720054626464844,
      "activations/layer14_attention_weight_min": -30.550979614257812,
      "activations/layer15_attention_weight_max": 35.788211822509766,
      "activations/layer15_attention_weight_min": -28.79833221435547,
      "activations/layer16_attention_weight_max": 30.881013870239258,
      "activations/layer16_attention_weight_min": -27.75886344909668,
      "activations/layer17_attention_weight_max": 55.53001403808594,
      "activations/layer17_attention_weight_min": -44.522586822509766,
      "activations/layer18_attention_weight_max": 50.280784606933594,
      "activations/layer18_attention_weight_min": -36.859764099121094,
      "activations/layer19_attention_weight_max": 23.44982147216797,
      "activations/layer19_attention_weight_min": -22.416173934936523,
      "activations/layer1_attention_weight_max": 16.567659378051758,
      "activations/layer1_attention_weight_min": -14.902666091918945,
      "activations/layer20_attention_weight_max": 24.055273056030273,
      "activations/layer20_attention_weight_min": -21.281034469604492,
      "activations/layer21_attention_weight_max": 36.5127067565918,
      "activations/layer21_attention_weight_min": -21.738210678100586,
      "activations/layer22_attention_weight_max": 32.650386810302734,
      "activations/layer22_attention_weight_min": -26.363624572753906,
      "activations/layer23_attention_weight_max": 37.14479064941406,
      "activations/layer23_attention_weight_min": -23.11916160583496,
      "activations/layer2_attention_weight_max": 29.84072494506836,
      "activations/layer2_attention_weight_min": -28.236331939697266,
      "activations/layer3_attention_weight_max": 84.0855941772461,
      "activations/layer3_attention_weight_min": -88.26229095458984,
      "activations/layer4_attention_weight_max": 94.20344543457031,
      "activations/layer4_attention_weight_min": -91.25328063964844,
      "activations/layer5_attention_weight_max": 75.38492584228516,
      "activations/layer5_attention_weight_min": -75.50271606445312,
      "activations/layer6_attention_weight_max": 51.459815979003906,
      "activations/layer6_attention_weight_min": -49.06288528442383,
      "activations/layer7_attention_weight_max": 77.2493896484375,
      "activations/layer7_attention_weight_min": -75.6207504272461,
      "activations/layer8_attention_weight_max": 44.691123962402344,
      "activations/layer8_attention_weight_min": -45.12810134887695,
      "activations/layer9_attention_weight_max": 48.42898178100586,
      "activations/layer9_attention_weight_min": -50.590389251708984,
      "epoch": 9.42,
      "learning_rate": 9.014848484848484e-05,
      "loss": 2.8006,
      "step": 162150
    },
    {
      "activations/layer0_attention_weight_max": 15.3145112991333,
      "activations/layer0_attention_weight_min": -14.156678199768066,
      "activations/layer10_attention_weight_max": 42.10288619995117,
      "activations/layer10_attention_weight_min": -41.06679916381836,
      "activations/layer11_attention_weight_max": 35.52184295654297,
      "activations/layer11_attention_weight_min": -34.03802490234375,
      "activations/layer12_attention_weight_max": 25.661954879760742,
      "activations/layer12_attention_weight_min": -23.925933837890625,
      "activations/layer13_attention_weight_max": 43.01688003540039,
      "activations/layer13_attention_weight_min": -31.97992706298828,
      "activations/layer14_attention_weight_max": 35.7956428527832,
      "activations/layer14_attention_weight_min": -29.76567268371582,
      "activations/layer15_attention_weight_max": 33.27371597290039,
      "activations/layer15_attention_weight_min": -29.598567962646484,
      "activations/layer16_attention_weight_max": 29.956329345703125,
      "activations/layer16_attention_weight_min": -29.69340705871582,
      "activations/layer17_attention_weight_max": 55.2844123840332,
      "activations/layer17_attention_weight_min": -45.2159423828125,
      "activations/layer18_attention_weight_max": 46.876888275146484,
      "activations/layer18_attention_weight_min": -37.07374954223633,
      "activations/layer19_attention_weight_max": 22.71472930908203,
      "activations/layer19_attention_weight_min": -20.895252227783203,
      "activations/layer1_attention_weight_max": 16.0421085357666,
      "activations/layer1_attention_weight_min": -13.212685585021973,
      "activations/layer20_attention_weight_max": 22.031124114990234,
      "activations/layer20_attention_weight_min": -23.062475204467773,
      "activations/layer21_attention_weight_max": 37.90911102294922,
      "activations/layer21_attention_weight_min": -25.996780395507812,
      "activations/layer22_attention_weight_max": 30.32001495361328,
      "activations/layer22_attention_weight_min": -25.568750381469727,
      "activations/layer23_attention_weight_max": 32.36222457885742,
      "activations/layer23_attention_weight_min": -23.206172943115234,
      "activations/layer2_attention_weight_max": 31.871564865112305,
      "activations/layer2_attention_weight_min": -31.295568466186523,
      "activations/layer3_attention_weight_max": 91.90697479248047,
      "activations/layer3_attention_weight_min": -95.8973388671875,
      "activations/layer4_attention_weight_max": 106.25091552734375,
      "activations/layer4_attention_weight_min": -99.95902252197266,
      "activations/layer5_attention_weight_max": 84.95233917236328,
      "activations/layer5_attention_weight_min": -80.81913757324219,
      "activations/layer6_attention_weight_max": 60.07518768310547,
      "activations/layer6_attention_weight_min": -58.79304504394531,
      "activations/layer7_attention_weight_max": 84.03067016601562,
      "activations/layer7_attention_weight_min": -78.67952728271484,
      "activations/layer8_attention_weight_max": 47.5911865234375,
      "activations/layer8_attention_weight_min": -49.91377258300781,
      "activations/layer9_attention_weight_max": 52.322933197021484,
      "activations/layer9_attention_weight_min": -52.411590576171875,
      "epoch": 9.42,
      "learning_rate": 9.012954545454545e-05,
      "loss": 2.7959,
      "step": 162200
    },
    {
      "activations/layer0_attention_weight_max": 15.285701751708984,
      "activations/layer0_attention_weight_min": -14.440194129943848,
      "activations/layer10_attention_weight_max": 35.308414459228516,
      "activations/layer10_attention_weight_min": -34.227081298828125,
      "activations/layer11_attention_weight_max": 31.335325241088867,
      "activations/layer11_attention_weight_min": -30.910978317260742,
      "activations/layer12_attention_weight_max": 21.066186904907227,
      "activations/layer12_attention_weight_min": -25.444110870361328,
      "activations/layer13_attention_weight_max": 55.445560455322266,
      "activations/layer13_attention_weight_min": -37.023414611816406,
      "activations/layer14_attention_weight_max": 62.82710266113281,
      "activations/layer14_attention_weight_min": -36.63481521606445,
      "activations/layer15_attention_weight_max": 42.477134704589844,
      "activations/layer15_attention_weight_min": -29.492679595947266,
      "activations/layer16_attention_weight_max": 28.67341423034668,
      "activations/layer16_attention_weight_min": -28.360851287841797,
      "activations/layer17_attention_weight_max": 47.60192108154297,
      "activations/layer17_attention_weight_min": -43.06947708129883,
      "activations/layer18_attention_weight_max": 43.14918899536133,
      "activations/layer18_attention_weight_min": -36.19072341918945,
      "activations/layer19_attention_weight_max": 20.352174758911133,
      "activations/layer19_attention_weight_min": -21.350507736206055,
      "activations/layer1_attention_weight_max": 17.084030151367188,
      "activations/layer1_attention_weight_min": -14.422835350036621,
      "activations/layer20_attention_weight_max": 21.851741790771484,
      "activations/layer20_attention_weight_min": -20.87454605102539,
      "activations/layer21_attention_weight_max": 34.85049057006836,
      "activations/layer21_attention_weight_min": -23.88947105407715,
      "activations/layer22_attention_weight_max": 29.36390495300293,
      "activations/layer22_attention_weight_min": -28.317285537719727,
      "activations/layer23_attention_weight_max": 34.966609954833984,
      "activations/layer23_attention_weight_min": -22.488929748535156,
      "activations/layer2_attention_weight_max": 30.653831481933594,
      "activations/layer2_attention_weight_min": -29.473737716674805,
      "activations/layer3_attention_weight_max": 86.56219482421875,
      "activations/layer3_attention_weight_min": -89.52008819580078,
      "activations/layer4_attention_weight_max": 96.03072357177734,
      "activations/layer4_attention_weight_min": -90.06529235839844,
      "activations/layer5_attention_weight_max": 77.26979064941406,
      "activations/layer5_attention_weight_min": -78.31268310546875,
      "activations/layer6_attention_weight_max": 54.443077087402344,
      "activations/layer6_attention_weight_min": -50.67978286743164,
      "activations/layer7_attention_weight_max": 77.7633056640625,
      "activations/layer7_attention_weight_min": -71.60466766357422,
      "activations/layer8_attention_weight_max": 42.1175537109375,
      "activations/layer8_attention_weight_min": -45.56560516357422,
      "activations/layer9_attention_weight_max": 51.80652618408203,
      "activations/layer9_attention_weight_min": -49.1956901550293,
      "epoch": 9.43,
      "learning_rate": 9.011060606060605e-05,
      "loss": 2.8072,
      "step": 162250
    },
    {
      "activations/layer0_attention_weight_max": 14.543844223022461,
      "activations/layer0_attention_weight_min": -14.318119049072266,
      "activations/layer10_attention_weight_max": 42.807411193847656,
      "activations/layer10_attention_weight_min": -41.2678337097168,
      "activations/layer11_attention_weight_max": 33.64210891723633,
      "activations/layer11_attention_weight_min": -33.145084381103516,
      "activations/layer12_attention_weight_max": 19.51912498474121,
      "activations/layer12_attention_weight_min": -25.502296447753906,
      "activations/layer13_attention_weight_max": 48.37742614746094,
      "activations/layer13_attention_weight_min": -33.305824279785156,
      "activations/layer14_attention_weight_max": 38.37971115112305,
      "activations/layer14_attention_weight_min": -31.330488204956055,
      "activations/layer15_attention_weight_max": 34.517311096191406,
      "activations/layer15_attention_weight_min": -29.17617416381836,
      "activations/layer16_attention_weight_max": 32.05850601196289,
      "activations/layer16_attention_weight_min": -29.795364379882812,
      "activations/layer17_attention_weight_max": 57.24678421020508,
      "activations/layer17_attention_weight_min": -48.72893142700195,
      "activations/layer18_attention_weight_max": 52.17425537109375,
      "activations/layer18_attention_weight_min": -41.070621490478516,
      "activations/layer19_attention_weight_max": 23.658660888671875,
      "activations/layer19_attention_weight_min": -22.87518882751465,
      "activations/layer1_attention_weight_max": 16.584949493408203,
      "activations/layer1_attention_weight_min": -15.92372989654541,
      "activations/layer20_attention_weight_max": 25.97459602355957,
      "activations/layer20_attention_weight_min": -24.024066925048828,
      "activations/layer21_attention_weight_max": 46.47809982299805,
      "activations/layer21_attention_weight_min": -22.878623962402344,
      "activations/layer22_attention_weight_max": 37.05440139770508,
      "activations/layer22_attention_weight_min": -26.451528549194336,
      "activations/layer23_attention_weight_max": 40.49761962890625,
      "activations/layer23_attention_weight_min": -23.61524772644043,
      "activations/layer2_attention_weight_max": 33.64253616333008,
      "activations/layer2_attention_weight_min": -30.610294342041016,
      "activations/layer3_attention_weight_max": 90.22543334960938,
      "activations/layer3_attention_weight_min": -91.93526458740234,
      "activations/layer4_attention_weight_max": 99.9954833984375,
      "activations/layer4_attention_weight_min": -96.39806365966797,
      "activations/layer5_attention_weight_max": 77.77040100097656,
      "activations/layer5_attention_weight_min": -75.9766616821289,
      "activations/layer6_attention_weight_max": 54.90106201171875,
      "activations/layer6_attention_weight_min": -56.7574462890625,
      "activations/layer7_attention_weight_max": 81.1832046508789,
      "activations/layer7_attention_weight_min": -76.4397201538086,
      "activations/layer8_attention_weight_max": 49.86064529418945,
      "activations/layer8_attention_weight_min": -51.51618576049805,
      "activations/layer9_attention_weight_max": 53.037391662597656,
      "activations/layer9_attention_weight_min": -52.74631881713867,
      "epoch": 9.43,
      "learning_rate": 9.009166666666666e-05,
      "loss": 2.7876,
      "step": 162300
    },
    {
      "activations/layer0_attention_weight_max": 13.677412033081055,
      "activations/layer0_attention_weight_min": -14.46873664855957,
      "activations/layer10_attention_weight_max": 35.46033477783203,
      "activations/layer10_attention_weight_min": -34.57495880126953,
      "activations/layer11_attention_weight_max": 30.520652770996094,
      "activations/layer11_attention_weight_min": -32.330528259277344,
      "activations/layer12_attention_weight_max": 17.123804092407227,
      "activations/layer12_attention_weight_min": -27.45921516418457,
      "activations/layer13_attention_weight_max": 36.59272384643555,
      "activations/layer13_attention_weight_min": -30.29789924621582,
      "activations/layer14_attention_weight_max": 45.833805084228516,
      "activations/layer14_attention_weight_min": -35.195613861083984,
      "activations/layer15_attention_weight_max": 37.801429748535156,
      "activations/layer15_attention_weight_min": -29.413818359375,
      "activations/layer16_attention_weight_max": 27.289180755615234,
      "activations/layer16_attention_weight_min": -26.396530151367188,
      "activations/layer17_attention_weight_max": 48.12507247924805,
      "activations/layer17_attention_weight_min": -39.72648620605469,
      "activations/layer18_attention_weight_max": 40.304840087890625,
      "activations/layer18_attention_weight_min": -34.357330322265625,
      "activations/layer19_attention_weight_max": 21.310176849365234,
      "activations/layer19_attention_weight_min": -20.25892448425293,
      "activations/layer1_attention_weight_max": 16.997417449951172,
      "activations/layer1_attention_weight_min": -13.900396347045898,
      "activations/layer20_attention_weight_max": 23.764429092407227,
      "activations/layer20_attention_weight_min": -21.318357467651367,
      "activations/layer21_attention_weight_max": 39.047454833984375,
      "activations/layer21_attention_weight_min": -22.409713745117188,
      "activations/layer22_attention_weight_max": 27.355541229248047,
      "activations/layer22_attention_weight_min": -25.74298095703125,
      "activations/layer23_attention_weight_max": 32.64789581298828,
      "activations/layer23_attention_weight_min": -25.035076141357422,
      "activations/layer2_attention_weight_max": 29.87702751159668,
      "activations/layer2_attention_weight_min": -28.13159942626953,
      "activations/layer3_attention_weight_max": 87.58096313476562,
      "activations/layer3_attention_weight_min": -90.74869537353516,
      "activations/layer4_attention_weight_max": 97.0362548828125,
      "activations/layer4_attention_weight_min": -95.2246322631836,
      "activations/layer5_attention_weight_max": 75.8301773071289,
      "activations/layer5_attention_weight_min": -77.01008605957031,
      "activations/layer6_attention_weight_max": 52.49891662597656,
      "activations/layer6_attention_weight_min": -52.59403610229492,
      "activations/layer7_attention_weight_max": 75.9834976196289,
      "activations/layer7_attention_weight_min": -70.4218521118164,
      "activations/layer8_attention_weight_max": 44.63497543334961,
      "activations/layer8_attention_weight_min": -47.723838806152344,
      "activations/layer9_attention_weight_max": 49.13018035888672,
      "activations/layer9_attention_weight_min": -49.134708404541016,
      "epoch": 9.43,
      "learning_rate": 9.007272727272727e-05,
      "loss": 2.7967,
      "step": 162350
    },
    {
      "activations/layer0_attention_weight_max": 13.702988624572754,
      "activations/layer0_attention_weight_min": -14.173497200012207,
      "activations/layer10_attention_weight_max": 38.55543518066406,
      "activations/layer10_attention_weight_min": -36.10310745239258,
      "activations/layer11_attention_weight_max": 31.785160064697266,
      "activations/layer11_attention_weight_min": -32.591243743896484,
      "activations/layer12_attention_weight_max": 19.938940048217773,
      "activations/layer12_attention_weight_min": -24.75263214111328,
      "activations/layer13_attention_weight_max": 44.80055236816406,
      "activations/layer13_attention_weight_min": -34.458866119384766,
      "activations/layer14_attention_weight_max": 41.49934005737305,
      "activations/layer14_attention_weight_min": -30.626508712768555,
      "activations/layer15_attention_weight_max": 31.486665725708008,
      "activations/layer15_attention_weight_min": -29.278545379638672,
      "activations/layer16_attention_weight_max": 30.039138793945312,
      "activations/layer16_attention_weight_min": -28.121435165405273,
      "activations/layer17_attention_weight_max": 54.09584426879883,
      "activations/layer17_attention_weight_min": -45.7442512512207,
      "activations/layer18_attention_weight_max": 43.842071533203125,
      "activations/layer18_attention_weight_min": -38.795528411865234,
      "activations/layer19_attention_weight_max": 24.308067321777344,
      "activations/layer19_attention_weight_min": -21.888574600219727,
      "activations/layer1_attention_weight_max": 16.085542678833008,
      "activations/layer1_attention_weight_min": -15.216309547424316,
      "activations/layer20_attention_weight_max": 21.616697311401367,
      "activations/layer20_attention_weight_min": -23.859785079956055,
      "activations/layer21_attention_weight_max": 33.75749588012695,
      "activations/layer21_attention_weight_min": -23.230178833007812,
      "activations/layer22_attention_weight_max": 33.040672302246094,
      "activations/layer22_attention_weight_min": -25.198917388916016,
      "activations/layer23_attention_weight_max": 34.5689811706543,
      "activations/layer23_attention_weight_min": -22.49950408935547,
      "activations/layer2_attention_weight_max": 32.345096588134766,
      "activations/layer2_attention_weight_min": -30.420438766479492,
      "activations/layer3_attention_weight_max": 89.45923614501953,
      "activations/layer3_attention_weight_min": -89.52396392822266,
      "activations/layer4_attention_weight_max": 99.501220703125,
      "activations/layer4_attention_weight_min": -95.4252700805664,
      "activations/layer5_attention_weight_max": 79.84677124023438,
      "activations/layer5_attention_weight_min": -77.96392059326172,
      "activations/layer6_attention_weight_max": 55.43574905395508,
      "activations/layer6_attention_weight_min": -55.35628128051758,
      "activations/layer7_attention_weight_max": 79.4751968383789,
      "activations/layer7_attention_weight_min": -74.44706726074219,
      "activations/layer8_attention_weight_max": 46.59640121459961,
      "activations/layer8_attention_weight_min": -47.48866653442383,
      "activations/layer9_attention_weight_max": 52.699989318847656,
      "activations/layer9_attention_weight_min": -47.8775749206543,
      "epoch": 9.44,
      "learning_rate": 9.005378787878786e-05,
      "loss": 2.7894,
      "step": 162400
    },
    {
      "activations/layer0_attention_weight_max": 14.554610252380371,
      "activations/layer0_attention_weight_min": -14.478166580200195,
      "activations/layer10_attention_weight_max": 41.788970947265625,
      "activations/layer10_attention_weight_min": -36.704933166503906,
      "activations/layer11_attention_weight_max": 34.65264892578125,
      "activations/layer11_attention_weight_min": -33.27146911621094,
      "activations/layer12_attention_weight_max": 18.794870376586914,
      "activations/layer12_attention_weight_min": -27.287546157836914,
      "activations/layer13_attention_weight_max": 42.79317092895508,
      "activations/layer13_attention_weight_min": -29.157379150390625,
      "activations/layer14_attention_weight_max": 43.56360626220703,
      "activations/layer14_attention_weight_min": -31.24997901916504,
      "activations/layer15_attention_weight_max": 37.32052230834961,
      "activations/layer15_attention_weight_min": -30.44135093688965,
      "activations/layer16_attention_weight_max": 32.848876953125,
      "activations/layer16_attention_weight_min": -28.079633712768555,
      "activations/layer17_attention_weight_max": 54.652931213378906,
      "activations/layer17_attention_weight_min": -46.73407745361328,
      "activations/layer18_attention_weight_max": 45.952762603759766,
      "activations/layer18_attention_weight_min": -37.22278594970703,
      "activations/layer19_attention_weight_max": 21.33332061767578,
      "activations/layer19_attention_weight_min": -22.650617599487305,
      "activations/layer1_attention_weight_max": 16.711931228637695,
      "activations/layer1_attention_weight_min": -15.344022750854492,
      "activations/layer20_attention_weight_max": 24.756633758544922,
      "activations/layer20_attention_weight_min": -20.71808433532715,
      "activations/layer21_attention_weight_max": 46.2447624206543,
      "activations/layer21_attention_weight_min": -21.626815795898438,
      "activations/layer22_attention_weight_max": 33.04250717163086,
      "activations/layer22_attention_weight_min": -24.581466674804688,
      "activations/layer23_attention_weight_max": 34.451744079589844,
      "activations/layer23_attention_weight_min": -25.324642181396484,
      "activations/layer2_attention_weight_max": 32.58381271362305,
      "activations/layer2_attention_weight_min": -30.637428283691406,
      "activations/layer3_attention_weight_max": 89.21966552734375,
      "activations/layer3_attention_weight_min": -88.75819396972656,
      "activations/layer4_attention_weight_max": 100.95289611816406,
      "activations/layer4_attention_weight_min": -99.59776306152344,
      "activations/layer5_attention_weight_max": 82.38687133789062,
      "activations/layer5_attention_weight_min": -79.4886245727539,
      "activations/layer6_attention_weight_max": 54.3135871887207,
      "activations/layer6_attention_weight_min": -54.73744583129883,
      "activations/layer7_attention_weight_max": 79.6572265625,
      "activations/layer7_attention_weight_min": -76.124267578125,
      "activations/layer8_attention_weight_max": 46.70332717895508,
      "activations/layer8_attention_weight_min": -46.0375862121582,
      "activations/layer9_attention_weight_max": 52.94369125366211,
      "activations/layer9_attention_weight_min": -49.44566345214844,
      "epoch": 9.44,
      "learning_rate": 9.003484848484847e-05,
      "loss": 2.8016,
      "step": 162450
    },
    {
      "activations/layer0_attention_weight_max": 15.823338508605957,
      "activations/layer0_attention_weight_min": -14.41469669342041,
      "activations/layer10_attention_weight_max": 38.95172119140625,
      "activations/layer10_attention_weight_min": -38.489837646484375,
      "activations/layer11_attention_weight_max": 31.88197135925293,
      "activations/layer11_attention_weight_min": -32.08180618286133,
      "activations/layer12_attention_weight_max": 19.72328758239746,
      "activations/layer12_attention_weight_min": -26.268144607543945,
      "activations/layer13_attention_weight_max": 43.27340316772461,
      "activations/layer13_attention_weight_min": -30.371782302856445,
      "activations/layer14_attention_weight_max": 41.1619987487793,
      "activations/layer14_attention_weight_min": -30.160846710205078,
      "activations/layer15_attention_weight_max": 36.160831451416016,
      "activations/layer15_attention_weight_min": -28.673324584960938,
      "activations/layer16_attention_weight_max": 31.336807250976562,
      "activations/layer16_attention_weight_min": -29.007158279418945,
      "activations/layer17_attention_weight_max": 53.63851547241211,
      "activations/layer17_attention_weight_min": -43.52849197387695,
      "activations/layer18_attention_weight_max": 45.982852935791016,
      "activations/layer18_attention_weight_min": -38.25988006591797,
      "activations/layer19_attention_weight_max": 22.603355407714844,
      "activations/layer19_attention_weight_min": -21.759939193725586,
      "activations/layer1_attention_weight_max": 16.592159271240234,
      "activations/layer1_attention_weight_min": -15.161495208740234,
      "activations/layer20_attention_weight_max": 24.087549209594727,
      "activations/layer20_attention_weight_min": -22.91842269897461,
      "activations/layer21_attention_weight_max": 44.68611145019531,
      "activations/layer21_attention_weight_min": -27.81172752380371,
      "activations/layer22_attention_weight_max": 32.344398498535156,
      "activations/layer22_attention_weight_min": -25.45830726623535,
      "activations/layer23_attention_weight_max": 41.086143493652344,
      "activations/layer23_attention_weight_min": -23.795516967773438,
      "activations/layer2_attention_weight_max": 31.494491577148438,
      "activations/layer2_attention_weight_min": -30.195358276367188,
      "activations/layer3_attention_weight_max": 93.57081604003906,
      "activations/layer3_attention_weight_min": -100.3681640625,
      "activations/layer4_attention_weight_max": 100.7625503540039,
      "activations/layer4_attention_weight_min": -98.64598846435547,
      "activations/layer5_attention_weight_max": 79.573974609375,
      "activations/layer5_attention_weight_min": -78.37358093261719,
      "activations/layer6_attention_weight_max": 55.98655319213867,
      "activations/layer6_attention_weight_min": -52.91559600830078,
      "activations/layer7_attention_weight_max": 79.1195297241211,
      "activations/layer7_attention_weight_min": -70.87898254394531,
      "activations/layer8_attention_weight_max": 49.265079498291016,
      "activations/layer8_attention_weight_min": -52.15874099731445,
      "activations/layer9_attention_weight_max": 51.31824493408203,
      "activations/layer9_attention_weight_min": -52.9924430847168,
      "epoch": 9.44,
      "learning_rate": 9.001590909090909e-05,
      "loss": 2.7985,
      "step": 162500
    },
    {
      "activations/layer0_attention_weight_max": 15.318483352661133,
      "activations/layer0_attention_weight_min": -13.98563289642334,
      "activations/layer10_attention_weight_max": 37.64071273803711,
      "activations/layer10_attention_weight_min": -36.7771110534668,
      "activations/layer11_attention_weight_max": 32.699161529541016,
      "activations/layer11_attention_weight_min": -32.886383056640625,
      "activations/layer12_attention_weight_max": 19.45631980895996,
      "activations/layer12_attention_weight_min": -25.730382919311523,
      "activations/layer13_attention_weight_max": 39.04210662841797,
      "activations/layer13_attention_weight_min": -30.643680572509766,
      "activations/layer14_attention_weight_max": 53.074710845947266,
      "activations/layer14_attention_weight_min": -32.385169982910156,
      "activations/layer15_attention_weight_max": 42.670108795166016,
      "activations/layer15_attention_weight_min": -29.524213790893555,
      "activations/layer16_attention_weight_max": 31.941967010498047,
      "activations/layer16_attention_weight_min": -28.24324607849121,
      "activations/layer17_attention_weight_max": 55.31178665161133,
      "activations/layer17_attention_weight_min": -43.53104019165039,
      "activations/layer18_attention_weight_max": 46.52032470703125,
      "activations/layer18_attention_weight_min": -36.648414611816406,
      "activations/layer19_attention_weight_max": 20.94430160522461,
      "activations/layer19_attention_weight_min": -20.593400955200195,
      "activations/layer1_attention_weight_max": 16.776451110839844,
      "activations/layer1_attention_weight_min": -14.917258262634277,
      "activations/layer20_attention_weight_max": 21.195486068725586,
      "activations/layer20_attention_weight_min": -21.566373825073242,
      "activations/layer21_attention_weight_max": 38.795166015625,
      "activations/layer21_attention_weight_min": -24.72551155090332,
      "activations/layer22_attention_weight_max": 30.20838165283203,
      "activations/layer22_attention_weight_min": -24.96869659423828,
      "activations/layer23_attention_weight_max": 34.49018859863281,
      "activations/layer23_attention_weight_min": -22.742218017578125,
      "activations/layer2_attention_weight_max": 31.991268157958984,
      "activations/layer2_attention_weight_min": -28.982755661010742,
      "activations/layer3_attention_weight_max": 90.20204162597656,
      "activations/layer3_attention_weight_min": -91.192138671875,
      "activations/layer4_attention_weight_max": 102.98909759521484,
      "activations/layer4_attention_weight_min": -97.86319732666016,
      "activations/layer5_attention_weight_max": 78.08135986328125,
      "activations/layer5_attention_weight_min": -75.93802642822266,
      "activations/layer6_attention_weight_max": 58.237003326416016,
      "activations/layer6_attention_weight_min": -54.871952056884766,
      "activations/layer7_attention_weight_max": 82.02533721923828,
      "activations/layer7_attention_weight_min": -72.41386413574219,
      "activations/layer8_attention_weight_max": 48.04506301879883,
      "activations/layer8_attention_weight_min": -46.48711013793945,
      "activations/layer9_attention_weight_max": 52.4893798828125,
      "activations/layer9_attention_weight_min": -48.770782470703125,
      "epoch": 9.45,
      "learning_rate": 8.999696969696968e-05,
      "loss": 2.7876,
      "step": 162550
    },
    {
      "activations/layer0_attention_weight_max": 15.400433540344238,
      "activations/layer0_attention_weight_min": -14.542409896850586,
      "activations/layer10_attention_weight_max": 35.587581634521484,
      "activations/layer10_attention_weight_min": -35.49660110473633,
      "activations/layer11_attention_weight_max": 30.880783081054688,
      "activations/layer11_attention_weight_min": -31.701353073120117,
      "activations/layer12_attention_weight_max": 19.76519203186035,
      "activations/layer12_attention_weight_min": -27.037919998168945,
      "activations/layer13_attention_weight_max": 46.22893524169922,
      "activations/layer13_attention_weight_min": -33.57663345336914,
      "activations/layer14_attention_weight_max": 54.360984802246094,
      "activations/layer14_attention_weight_min": -33.98080062866211,
      "activations/layer15_attention_weight_max": 40.55961608886719,
      "activations/layer15_attention_weight_min": -30.210113525390625,
      "activations/layer16_attention_weight_max": 35.9671745300293,
      "activations/layer16_attention_weight_min": -31.03096580505371,
      "activations/layer17_attention_weight_max": 57.13907241821289,
      "activations/layer17_attention_weight_min": -46.468536376953125,
      "activations/layer18_attention_weight_max": 51.12104415893555,
      "activations/layer18_attention_weight_min": -39.39242935180664,
      "activations/layer19_attention_weight_max": 22.55323028564453,
      "activations/layer19_attention_weight_min": -23.375301361083984,
      "activations/layer1_attention_weight_max": 17.671396255493164,
      "activations/layer1_attention_weight_min": -14.848983764648438,
      "activations/layer20_attention_weight_max": 23.866331100463867,
      "activations/layer20_attention_weight_min": -22.375621795654297,
      "activations/layer21_attention_weight_max": 54.750022888183594,
      "activations/layer21_attention_weight_min": -27.717422485351562,
      "activations/layer22_attention_weight_max": 34.65778732299805,
      "activations/layer22_attention_weight_min": -25.11914825439453,
      "activations/layer23_attention_weight_max": 39.714744567871094,
      "activations/layer23_attention_weight_min": -24.036582946777344,
      "activations/layer2_attention_weight_max": 31.704587936401367,
      "activations/layer2_attention_weight_min": -29.88311767578125,
      "activations/layer3_attention_weight_max": 88.51434326171875,
      "activations/layer3_attention_weight_min": -92.62042999267578,
      "activations/layer4_attention_weight_max": 99.80461883544922,
      "activations/layer4_attention_weight_min": -92.24275970458984,
      "activations/layer5_attention_weight_max": 81.71350860595703,
      "activations/layer5_attention_weight_min": -75.93372344970703,
      "activations/layer6_attention_weight_max": 55.56614685058594,
      "activations/layer6_attention_weight_min": -53.43279266357422,
      "activations/layer7_attention_weight_max": 83.01490020751953,
      "activations/layer7_attention_weight_min": -73.64299011230469,
      "activations/layer8_attention_weight_max": 43.366085052490234,
      "activations/layer8_attention_weight_min": -45.12576675415039,
      "activations/layer9_attention_weight_max": 46.085655212402344,
      "activations/layer9_attention_weight_min": -47.327449798583984,
      "epoch": 9.45,
      "learning_rate": 8.997803030303029e-05,
      "loss": 2.81,
      "step": 162600
    },
    {
      "activations/layer0_attention_weight_max": 15.353737831115723,
      "activations/layer0_attention_weight_min": -13.522891998291016,
      "activations/layer10_attention_weight_max": 37.450538635253906,
      "activations/layer10_attention_weight_min": -39.418418884277344,
      "activations/layer11_attention_weight_max": 33.50734329223633,
      "activations/layer11_attention_weight_min": -33.686302185058594,
      "activations/layer12_attention_weight_max": 22.691083908081055,
      "activations/layer12_attention_weight_min": -28.4223690032959,
      "activations/layer13_attention_weight_max": 48.08510208129883,
      "activations/layer13_attention_weight_min": -31.737844467163086,
      "activations/layer14_attention_weight_max": 62.708797454833984,
      "activations/layer14_attention_weight_min": -32.57318878173828,
      "activations/layer15_attention_weight_max": 46.150516510009766,
      "activations/layer15_attention_weight_min": -29.790246963500977,
      "activations/layer16_attention_weight_max": 35.040382385253906,
      "activations/layer16_attention_weight_min": -30.344011306762695,
      "activations/layer17_attention_weight_max": 60.28774642944336,
      "activations/layer17_attention_weight_min": -48.52942657470703,
      "activations/layer18_attention_weight_max": 56.852012634277344,
      "activations/layer18_attention_weight_min": -40.18184280395508,
      "activations/layer19_attention_weight_max": 23.227033615112305,
      "activations/layer19_attention_weight_min": -23.747894287109375,
      "activations/layer1_attention_weight_max": 16.521224975585938,
      "activations/layer1_attention_weight_min": -14.022104263305664,
      "activations/layer20_attention_weight_max": 24.78363609313965,
      "activations/layer20_attention_weight_min": -21.59072494506836,
      "activations/layer21_attention_weight_max": 56.39058303833008,
      "activations/layer21_attention_weight_min": -22.756423950195312,
      "activations/layer22_attention_weight_max": 31.943098068237305,
      "activations/layer22_attention_weight_min": -26.142398834228516,
      "activations/layer23_attention_weight_max": 41.30302047729492,
      "activations/layer23_attention_weight_min": -26.233671188354492,
      "activations/layer2_attention_weight_max": 32.91351318359375,
      "activations/layer2_attention_weight_min": -31.88091278076172,
      "activations/layer3_attention_weight_max": 93.9097671508789,
      "activations/layer3_attention_weight_min": -97.40814971923828,
      "activations/layer4_attention_weight_max": 103.35658264160156,
      "activations/layer4_attention_weight_min": -103.45654296875,
      "activations/layer5_attention_weight_max": 80.73355865478516,
      "activations/layer5_attention_weight_min": -83.4447021484375,
      "activations/layer6_attention_weight_max": 54.81765365600586,
      "activations/layer6_attention_weight_min": -56.284732818603516,
      "activations/layer7_attention_weight_max": 80.38773345947266,
      "activations/layer7_attention_weight_min": -75.93941497802734,
      "activations/layer8_attention_weight_max": 45.59053421020508,
      "activations/layer8_attention_weight_min": -47.73372268676758,
      "activations/layer9_attention_weight_max": 54.996517181396484,
      "activations/layer9_attention_weight_min": -49.49641799926758,
      "epoch": 9.45,
      "learning_rate": 8.995909090909091e-05,
      "loss": 2.7823,
      "step": 162650
    },
    {
      "activations/layer0_attention_weight_max": 14.807554244995117,
      "activations/layer0_attention_weight_min": -13.638172149658203,
      "activations/layer10_attention_weight_max": 34.85213088989258,
      "activations/layer10_attention_weight_min": -34.398963928222656,
      "activations/layer11_attention_weight_max": 31.964590072631836,
      "activations/layer11_attention_weight_min": -32.30211639404297,
      "activations/layer12_attention_weight_max": 18.900346755981445,
      "activations/layer12_attention_weight_min": -26.397371292114258,
      "activations/layer13_attention_weight_max": 37.44501876831055,
      "activations/layer13_attention_weight_min": -28.990825653076172,
      "activations/layer14_attention_weight_max": 36.443763732910156,
      "activations/layer14_attention_weight_min": -29.89435386657715,
      "activations/layer15_attention_weight_max": 36.54283142089844,
      "activations/layer15_attention_weight_min": -28.471147537231445,
      "activations/layer16_attention_weight_max": 31.800382614135742,
      "activations/layer16_attention_weight_min": -28.23236846923828,
      "activations/layer17_attention_weight_max": 52.25545120239258,
      "activations/layer17_attention_weight_min": -45.09614562988281,
      "activations/layer18_attention_weight_max": 43.234188079833984,
      "activations/layer18_attention_weight_min": -36.48076629638672,
      "activations/layer19_attention_weight_max": 19.494775772094727,
      "activations/layer19_attention_weight_min": -20.14676284790039,
      "activations/layer1_attention_weight_max": 16.623851776123047,
      "activations/layer1_attention_weight_min": -14.434106826782227,
      "activations/layer20_attention_weight_max": 21.620378494262695,
      "activations/layer20_attention_weight_min": -20.363340377807617,
      "activations/layer21_attention_weight_max": 36.190284729003906,
      "activations/layer21_attention_weight_min": -23.453826904296875,
      "activations/layer22_attention_weight_max": 33.2368049621582,
      "activations/layer22_attention_weight_min": -26.37636947631836,
      "activations/layer23_attention_weight_max": 37.146812438964844,
      "activations/layer23_attention_weight_min": -22.299434661865234,
      "activations/layer2_attention_weight_max": 32.448570251464844,
      "activations/layer2_attention_weight_min": -28.133647918701172,
      "activations/layer3_attention_weight_max": 87.5794906616211,
      "activations/layer3_attention_weight_min": -87.20003509521484,
      "activations/layer4_attention_weight_max": 98.35465240478516,
      "activations/layer4_attention_weight_min": -93.41886138916016,
      "activations/layer5_attention_weight_max": 78.73107147216797,
      "activations/layer5_attention_weight_min": -74.9942398071289,
      "activations/layer6_attention_weight_max": 55.95935821533203,
      "activations/layer6_attention_weight_min": -52.100685119628906,
      "activations/layer7_attention_weight_max": 86.8280029296875,
      "activations/layer7_attention_weight_min": -74.79170989990234,
      "activations/layer8_attention_weight_max": 45.790809631347656,
      "activations/layer8_attention_weight_min": -46.549560546875,
      "activations/layer9_attention_weight_max": 54.511817932128906,
      "activations/layer9_attention_weight_min": -49.88741683959961,
      "epoch": 9.45,
      "learning_rate": 8.994015151515151e-05,
      "loss": 2.7919,
      "step": 162700
    },
    {
      "activations/layer0_attention_weight_max": 14.837199211120605,
      "activations/layer0_attention_weight_min": -13.974876403808594,
      "activations/layer10_attention_weight_max": 36.008975982666016,
      "activations/layer10_attention_weight_min": -33.9322395324707,
      "activations/layer11_attention_weight_max": 30.796667098999023,
      "activations/layer11_attention_weight_min": -31.764028549194336,
      "activations/layer12_attention_weight_max": 18.328298568725586,
      "activations/layer12_attention_weight_min": -23.893436431884766,
      "activations/layer13_attention_weight_max": 38.94137191772461,
      "activations/layer13_attention_weight_min": -29.834558486938477,
      "activations/layer14_attention_weight_max": 35.09400939941406,
      "activations/layer14_attention_weight_min": -28.908084869384766,
      "activations/layer15_attention_weight_max": 33.26454544067383,
      "activations/layer15_attention_weight_min": -28.770065307617188,
      "activations/layer16_attention_weight_max": 30.926651000976562,
      "activations/layer16_attention_weight_min": -29.417865753173828,
      "activations/layer17_attention_weight_max": 52.32248306274414,
      "activations/layer17_attention_weight_min": -45.84205627441406,
      "activations/layer18_attention_weight_max": 48.268497467041016,
      "activations/layer18_attention_weight_min": -40.09148406982422,
      "activations/layer19_attention_weight_max": 21.63581657409668,
      "activations/layer19_attention_weight_min": -22.612485885620117,
      "activations/layer1_attention_weight_max": 16.442371368408203,
      "activations/layer1_attention_weight_min": -14.25529956817627,
      "activations/layer20_attention_weight_max": 23.632169723510742,
      "activations/layer20_attention_weight_min": -21.798664093017578,
      "activations/layer21_attention_weight_max": 36.27117919921875,
      "activations/layer21_attention_weight_min": -24.876516342163086,
      "activations/layer22_attention_weight_max": 33.83152389526367,
      "activations/layer22_attention_weight_min": -26.532726287841797,
      "activations/layer23_attention_weight_max": 37.504234313964844,
      "activations/layer23_attention_weight_min": -23.54461669921875,
      "activations/layer2_attention_weight_max": 32.86290740966797,
      "activations/layer2_attention_weight_min": -30.086517333984375,
      "activations/layer3_attention_weight_max": 89.74629211425781,
      "activations/layer3_attention_weight_min": -93.21232604980469,
      "activations/layer4_attention_weight_max": 99.28352355957031,
      "activations/layer4_attention_weight_min": -98.1334457397461,
      "activations/layer5_attention_weight_max": 81.2641830444336,
      "activations/layer5_attention_weight_min": -81.28402709960938,
      "activations/layer6_attention_weight_max": 54.40280532836914,
      "activations/layer6_attention_weight_min": -52.95170593261719,
      "activations/layer7_attention_weight_max": 76.76892852783203,
      "activations/layer7_attention_weight_min": -70.86349487304688,
      "activations/layer8_attention_weight_max": 46.05360794067383,
      "activations/layer8_attention_weight_min": -47.72858810424805,
      "activations/layer9_attention_weight_max": 48.15910720825195,
      "activations/layer9_attention_weight_min": -50.812625885009766,
      "epoch": 9.46,
      "learning_rate": 8.992121212121211e-05,
      "loss": 2.7827,
      "step": 162750
    },
    {
      "activations/layer0_attention_weight_max": 16.250598907470703,
      "activations/layer0_attention_weight_min": -14.264410972595215,
      "activations/layer10_attention_weight_max": 36.54619216918945,
      "activations/layer10_attention_weight_min": -37.486473083496094,
      "activations/layer11_attention_weight_max": 31.692678451538086,
      "activations/layer11_attention_weight_min": -33.67646026611328,
      "activations/layer12_attention_weight_max": 18.557788848876953,
      "activations/layer12_attention_weight_min": -24.259414672851562,
      "activations/layer13_attention_weight_max": 36.10869598388672,
      "activations/layer13_attention_weight_min": -31.112838745117188,
      "activations/layer14_attention_weight_max": 35.26052474975586,
      "activations/layer14_attention_weight_min": -32.47736740112305,
      "activations/layer15_attention_weight_max": 31.92582893371582,
      "activations/layer15_attention_weight_min": -29.072721481323242,
      "activations/layer16_attention_weight_max": 28.24558448791504,
      "activations/layer16_attention_weight_min": -27.803043365478516,
      "activations/layer17_attention_weight_max": 50.10786819458008,
      "activations/layer17_attention_weight_min": -42.48689651489258,
      "activations/layer18_attention_weight_max": 46.43700408935547,
      "activations/layer18_attention_weight_min": -36.37698745727539,
      "activations/layer19_attention_weight_max": 21.83275604248047,
      "activations/layer19_attention_weight_min": -21.253114700317383,
      "activations/layer1_attention_weight_max": 16.63829803466797,
      "activations/layer1_attention_weight_min": -14.500653266906738,
      "activations/layer20_attention_weight_max": 20.862730026245117,
      "activations/layer20_attention_weight_min": -23.94217300415039,
      "activations/layer21_attention_weight_max": 33.24658966064453,
      "activations/layer21_attention_weight_min": -22.14497947692871,
      "activations/layer22_attention_weight_max": 28.50726890563965,
      "activations/layer22_attention_weight_min": -24.00528335571289,
      "activations/layer23_attention_weight_max": 34.36307907104492,
      "activations/layer23_attention_weight_min": -23.865211486816406,
      "activations/layer2_attention_weight_max": 32.124664306640625,
      "activations/layer2_attention_weight_min": -32.79761505126953,
      "activations/layer3_attention_weight_max": 94.91901397705078,
      "activations/layer3_attention_weight_min": -96.18639373779297,
      "activations/layer4_attention_weight_max": 99.66254425048828,
      "activations/layer4_attention_weight_min": -94.74191284179688,
      "activations/layer5_attention_weight_max": 78.89539337158203,
      "activations/layer5_attention_weight_min": -76.23686218261719,
      "activations/layer6_attention_weight_max": 54.683326721191406,
      "activations/layer6_attention_weight_min": -54.78536605834961,
      "activations/layer7_attention_weight_max": 77.24574279785156,
      "activations/layer7_attention_weight_min": -72.95189666748047,
      "activations/layer8_attention_weight_max": 46.43977737426758,
      "activations/layer8_attention_weight_min": -47.00319290161133,
      "activations/layer9_attention_weight_max": 48.44049072265625,
      "activations/layer9_attention_weight_min": -48.66035079956055,
      "epoch": 9.46,
      "learning_rate": 8.990227272727271e-05,
      "loss": 2.7929,
      "step": 162800
    },
    {
      "activations/layer0_attention_weight_max": 14.66126537322998,
      "activations/layer0_attention_weight_min": -14.490592956542969,
      "activations/layer10_attention_weight_max": 35.926116943359375,
      "activations/layer10_attention_weight_min": -35.13408660888672,
      "activations/layer11_attention_weight_max": 32.92969512939453,
      "activations/layer11_attention_weight_min": -32.245758056640625,
      "activations/layer12_attention_weight_max": 21.111642837524414,
      "activations/layer12_attention_weight_min": -26.010339736938477,
      "activations/layer13_attention_weight_max": 41.123077392578125,
      "activations/layer13_attention_weight_min": -33.31043243408203,
      "activations/layer14_attention_weight_max": 37.44346618652344,
      "activations/layer14_attention_weight_min": -30.683242797851562,
      "activations/layer15_attention_weight_max": 33.07585906982422,
      "activations/layer15_attention_weight_min": -28.59794044494629,
      "activations/layer16_attention_weight_max": 31.69774627685547,
      "activations/layer16_attention_weight_min": -28.923322677612305,
      "activations/layer17_attention_weight_max": 53.31313705444336,
      "activations/layer17_attention_weight_min": -45.430423736572266,
      "activations/layer18_attention_weight_max": 47.27042007446289,
      "activations/layer18_attention_weight_min": -36.142799377441406,
      "activations/layer19_attention_weight_max": 23.164505004882812,
      "activations/layer19_attention_weight_min": -21.128236770629883,
      "activations/layer1_attention_weight_max": 17.322389602661133,
      "activations/layer1_attention_weight_min": -14.701275825500488,
      "activations/layer20_attention_weight_max": 24.794015884399414,
      "activations/layer20_attention_weight_min": -23.147470474243164,
      "activations/layer21_attention_weight_max": 41.880615234375,
      "activations/layer21_attention_weight_min": -24.369888305664062,
      "activations/layer22_attention_weight_max": 30.357723236083984,
      "activations/layer22_attention_weight_min": -29.70145034790039,
      "activations/layer23_attention_weight_max": 35.08140182495117,
      "activations/layer23_attention_weight_min": -24.083608627319336,
      "activations/layer2_attention_weight_max": 31.79669761657715,
      "activations/layer2_attention_weight_min": -30.595855712890625,
      "activations/layer3_attention_weight_max": 90.89693450927734,
      "activations/layer3_attention_weight_min": -95.2271957397461,
      "activations/layer4_attention_weight_max": 98.146484375,
      "activations/layer4_attention_weight_min": -99.93119049072266,
      "activations/layer5_attention_weight_max": 77.04704284667969,
      "activations/layer5_attention_weight_min": -80.6457290649414,
      "activations/layer6_attention_weight_max": 52.98066329956055,
      "activations/layer6_attention_weight_min": -56.470497131347656,
      "activations/layer7_attention_weight_max": 77.15977478027344,
      "activations/layer7_attention_weight_min": -72.1573486328125,
      "activations/layer8_attention_weight_max": 43.019866943359375,
      "activations/layer8_attention_weight_min": -46.10367965698242,
      "activations/layer9_attention_weight_max": 50.28800582885742,
      "activations/layer9_attention_weight_min": -48.118255615234375,
      "epoch": 9.46,
      "learning_rate": 8.988333333333333e-05,
      "loss": 2.8036,
      "step": 162850
    },
    {
      "activations/layer0_attention_weight_max": 16.12302017211914,
      "activations/layer0_attention_weight_min": -14.412398338317871,
      "activations/layer10_attention_weight_max": 39.959083557128906,
      "activations/layer10_attention_weight_min": -38.47956085205078,
      "activations/layer11_attention_weight_max": 34.264068603515625,
      "activations/layer11_attention_weight_min": -34.66135787963867,
      "activations/layer12_attention_weight_max": 22.55939292907715,
      "activations/layer12_attention_weight_min": -26.59440803527832,
      "activations/layer13_attention_weight_max": 43.28331756591797,
      "activations/layer13_attention_weight_min": -30.179052352905273,
      "activations/layer14_attention_weight_max": 40.30308151245117,
      "activations/layer14_attention_weight_min": -30.090373992919922,
      "activations/layer15_attention_weight_max": 33.97603988647461,
      "activations/layer15_attention_weight_min": -29.3026180267334,
      "activations/layer16_attention_weight_max": 31.601009368896484,
      "activations/layer16_attention_weight_min": -28.44146156311035,
      "activations/layer17_attention_weight_max": 55.32673263549805,
      "activations/layer17_attention_weight_min": -43.281028747558594,
      "activations/layer18_attention_weight_max": 47.38837432861328,
      "activations/layer18_attention_weight_min": -36.19324493408203,
      "activations/layer19_attention_weight_max": 20.233367919921875,
      "activations/layer19_attention_weight_min": -19.84888458251953,
      "activations/layer1_attention_weight_max": 16.78354835510254,
      "activations/layer1_attention_weight_min": -15.631032943725586,
      "activations/layer20_attention_weight_max": 21.120512008666992,
      "activations/layer20_attention_weight_min": -24.778335571289062,
      "activations/layer21_attention_weight_max": 35.292388916015625,
      "activations/layer21_attention_weight_min": -21.675378799438477,
      "activations/layer22_attention_weight_max": 30.899335861206055,
      "activations/layer22_attention_weight_min": -25.85407257080078,
      "activations/layer23_attention_weight_max": 36.091705322265625,
      "activations/layer23_attention_weight_min": -25.03537368774414,
      "activations/layer2_attention_weight_max": 31.884864807128906,
      "activations/layer2_attention_weight_min": -30.26556968688965,
      "activations/layer3_attention_weight_max": 95.24047088623047,
      "activations/layer3_attention_weight_min": -98.5467758178711,
      "activations/layer4_attention_weight_max": 103.27925109863281,
      "activations/layer4_attention_weight_min": -102.03691864013672,
      "activations/layer5_attention_weight_max": 80.89314270019531,
      "activations/layer5_attention_weight_min": -78.78832244873047,
      "activations/layer6_attention_weight_max": 56.918426513671875,
      "activations/layer6_attention_weight_min": -54.794857025146484,
      "activations/layer7_attention_weight_max": 82.64595794677734,
      "activations/layer7_attention_weight_min": -79.5327377319336,
      "activations/layer8_attention_weight_max": 50.48384475708008,
      "activations/layer8_attention_weight_min": -51.80057144165039,
      "activations/layer9_attention_weight_max": 52.1871223449707,
      "activations/layer9_attention_weight_min": -50.197547912597656,
      "epoch": 9.47,
      "learning_rate": 8.986439393939394e-05,
      "loss": 2.8019,
      "step": 162900
    },
    {
      "activations/layer0_attention_weight_max": 16.1265811920166,
      "activations/layer0_attention_weight_min": -14.597423553466797,
      "activations/layer10_attention_weight_max": 36.22514724731445,
      "activations/layer10_attention_weight_min": -37.0717887878418,
      "activations/layer11_attention_weight_max": 30.692611694335938,
      "activations/layer11_attention_weight_min": -31.911144256591797,
      "activations/layer12_attention_weight_max": 18.758955001831055,
      "activations/layer12_attention_weight_min": -26.204622268676758,
      "activations/layer13_attention_weight_max": 42.126190185546875,
      "activations/layer13_attention_weight_min": -33.01961135864258,
      "activations/layer14_attention_weight_max": 38.89152145385742,
      "activations/layer14_attention_weight_min": -30.87985610961914,
      "activations/layer15_attention_weight_max": 34.222633361816406,
      "activations/layer15_attention_weight_min": -30.68414306640625,
      "activations/layer16_attention_weight_max": 32.14725875854492,
      "activations/layer16_attention_weight_min": -27.7071533203125,
      "activations/layer17_attention_weight_max": 55.14909362792969,
      "activations/layer17_attention_weight_min": -42.892433166503906,
      "activations/layer18_attention_weight_max": 47.00756072998047,
      "activations/layer18_attention_weight_min": -37.546852111816406,
      "activations/layer19_attention_weight_max": 25.09174919128418,
      "activations/layer19_attention_weight_min": -21.409231185913086,
      "activations/layer1_attention_weight_max": 17.076040267944336,
      "activations/layer1_attention_weight_min": -15.717880249023438,
      "activations/layer20_attention_weight_max": 23.579801559448242,
      "activations/layer20_attention_weight_min": -24.748502731323242,
      "activations/layer21_attention_weight_max": 40.93031692504883,
      "activations/layer21_attention_weight_min": -25.3300724029541,
      "activations/layer22_attention_weight_max": 31.23265838623047,
      "activations/layer22_attention_weight_min": -25.735641479492188,
      "activations/layer23_attention_weight_max": 41.42189025878906,
      "activations/layer23_attention_weight_min": -23.899078369140625,
      "activations/layer2_attention_weight_max": 32.54621887207031,
      "activations/layer2_attention_weight_min": -30.644451141357422,
      "activations/layer3_attention_weight_max": 96.34178161621094,
      "activations/layer3_attention_weight_min": -100.78966522216797,
      "activations/layer4_attention_weight_max": 100.59371185302734,
      "activations/layer4_attention_weight_min": -101.17296600341797,
      "activations/layer5_attention_weight_max": 79.30792236328125,
      "activations/layer5_attention_weight_min": -79.68310546875,
      "activations/layer6_attention_weight_max": 54.08781051635742,
      "activations/layer6_attention_weight_min": -53.22663879394531,
      "activations/layer7_attention_weight_max": 82.61646270751953,
      "activations/layer7_attention_weight_min": -73.15127563476562,
      "activations/layer8_attention_weight_max": 46.6932258605957,
      "activations/layer8_attention_weight_min": -47.380184173583984,
      "activations/layer9_attention_weight_max": 47.66128921508789,
      "activations/layer9_attention_weight_min": -48.414920806884766,
      "epoch": 9.47,
      "learning_rate": 8.984545454545453e-05,
      "loss": 2.7928,
      "step": 162950
    },
    {
      "activations/layer0_attention_weight_max": 14.594449996948242,
      "activations/layer0_attention_weight_min": -14.399548530578613,
      "activations/layer10_attention_weight_max": 37.951690673828125,
      "activations/layer10_attention_weight_min": -36.67734146118164,
      "activations/layer11_attention_weight_max": 30.81917953491211,
      "activations/layer11_attention_weight_min": -31.281688690185547,
      "activations/layer12_attention_weight_max": 18.14885711669922,
      "activations/layer12_attention_weight_min": -24.83695411682129,
      "activations/layer13_attention_weight_max": 36.86593246459961,
      "activations/layer13_attention_weight_min": -30.92870330810547,
      "activations/layer14_attention_weight_max": 38.80207061767578,
      "activations/layer14_attention_weight_min": -30.06488609313965,
      "activations/layer15_attention_weight_max": 35.17247772216797,
      "activations/layer15_attention_weight_min": -29.640857696533203,
      "activations/layer16_attention_weight_max": 29.80537223815918,
      "activations/layer16_attention_weight_min": -28.520875930786133,
      "activations/layer17_attention_weight_max": 56.753509521484375,
      "activations/layer17_attention_weight_min": -42.88068389892578,
      "activations/layer18_attention_weight_max": 48.04584503173828,
      "activations/layer18_attention_weight_min": -39.13097381591797,
      "activations/layer19_attention_weight_max": 22.052623748779297,
      "activations/layer19_attention_weight_min": -20.89735984802246,
      "activations/layer1_attention_weight_max": 16.472585678100586,
      "activations/layer1_attention_weight_min": -15.289908409118652,
      "activations/layer20_attention_weight_max": 22.19754981994629,
      "activations/layer20_attention_weight_min": -21.70115089416504,
      "activations/layer21_attention_weight_max": 36.56109619140625,
      "activations/layer21_attention_weight_min": -24.2003231048584,
      "activations/layer22_attention_weight_max": 33.29835891723633,
      "activations/layer22_attention_weight_min": -26.540273666381836,
      "activations/layer23_attention_weight_max": 38.205806732177734,
      "activations/layer23_attention_weight_min": -25.000038146972656,
      "activations/layer2_attention_weight_max": 31.56536102294922,
      "activations/layer2_attention_weight_min": -30.112730026245117,
      "activations/layer3_attention_weight_max": 92.35014343261719,
      "activations/layer3_attention_weight_min": -89.85426330566406,
      "activations/layer4_attention_weight_max": 99.14849853515625,
      "activations/layer4_attention_weight_min": -99.2297592163086,
      "activations/layer5_attention_weight_max": 77.46401977539062,
      "activations/layer5_attention_weight_min": -76.79087829589844,
      "activations/layer6_attention_weight_max": 52.69601058959961,
      "activations/layer6_attention_weight_min": -51.85175704956055,
      "activations/layer7_attention_weight_max": 74.791015625,
      "activations/layer7_attention_weight_min": -74.67395782470703,
      "activations/layer8_attention_weight_max": 42.710182189941406,
      "activations/layer8_attention_weight_min": -45.522457122802734,
      "activations/layer9_attention_weight_max": 47.46268844604492,
      "activations/layer9_attention_weight_min": -50.78242492675781,
      "epoch": 9.47,
      "learning_rate": 8.982651515151515e-05,
      "loss": 2.8042,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.5052,
      "eval_samples_per_second": 504.87,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.5052,
      "eval_openwebtext_samples_per_second": 504.87,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_wikitext_loss": 2.984375,
      "eval_wikitext_ppl": 19.774139536218833,
      "eval_wikitext_runtime": 1.986,
      "eval_wikitext_samples_per_second": 229.608,
      "step": 163000
    },
    {
      "epoch": 9.47,
      "eval_lambada_loss": 2.650390625,
      "eval_lambada_ppl": 14.159568646729724,
      "eval_lambada_runtime": 9.6395,
      "eval_lambada_samples_per_second": 505.107,
      "step": 163000
    },
    {
      "activations/layer0_attention_weight_max": 15.448040008544922,
      "activations/layer0_attention_weight_min": -14.222122192382812,
      "activations/layer10_attention_weight_max": 35.64087677001953,
      "activations/layer10_attention_weight_min": -32.88959884643555,
      "activations/layer11_attention_weight_max": 30.89821434020996,
      "activations/layer11_attention_weight_min": -28.743282318115234,
      "activations/layer12_attention_weight_max": 18.59531021118164,
      "activations/layer12_attention_weight_min": -24.20046615600586,
      "activations/layer13_attention_weight_max": 48.579856872558594,
      "activations/layer13_attention_weight_min": -29.343482971191406,
      "activations/layer14_attention_weight_max": 36.84999084472656,
      "activations/layer14_attention_weight_min": -30.24275779724121,
      "activations/layer15_attention_weight_max": 38.192359924316406,
      "activations/layer15_attention_weight_min": -27.875545501708984,
      "activations/layer16_attention_weight_max": 31.103076934814453,
      "activations/layer16_attention_weight_min": -28.94468116760254,
      "activations/layer17_attention_weight_max": 55.66461944580078,
      "activations/layer17_attention_weight_min": -45.417724609375,
      "activations/layer18_attention_weight_max": 48.42333221435547,
      "activations/layer18_attention_weight_min": -37.27748107910156,
      "activations/layer19_attention_weight_max": 23.455345153808594,
      "activations/layer19_attention_weight_min": -22.49220085144043,
      "activations/layer1_attention_weight_max": 17.07879638671875,
      "activations/layer1_attention_weight_min": -13.421310424804688,
      "activations/layer20_attention_weight_max": 25.847604751586914,
      "activations/layer20_attention_weight_min": -21.7438907623291,
      "activations/layer21_attention_weight_max": 44.36893844604492,
      "activations/layer21_attention_weight_min": -24.159751892089844,
      "activations/layer22_attention_weight_max": 31.528108596801758,
      "activations/layer22_attention_weight_min": -29.197086334228516,
      "activations/layer23_attention_weight_max": 40.86771011352539,
      "activations/layer23_attention_weight_min": -23.432716369628906,
      "activations/layer2_attention_weight_max": 31.501663208007812,
      "activations/layer2_attention_weight_min": -29.98644256591797,
      "activations/layer3_attention_weight_max": 91.39527893066406,
      "activations/layer3_attention_weight_min": -92.51312255859375,
      "activations/layer4_attention_weight_max": 96.61339569091797,
      "activations/layer4_attention_weight_min": -95.44863891601562,
      "activations/layer5_attention_weight_max": 77.33660888671875,
      "activations/layer5_attention_weight_min": -78.02173614501953,
      "activations/layer6_attention_weight_max": 55.746742248535156,
      "activations/layer6_attention_weight_min": -52.46488952636719,
      "activations/layer7_attention_weight_max": 74.45523834228516,
      "activations/layer7_attention_weight_min": -69.46058654785156,
      "activations/layer8_attention_weight_max": 42.2231559753418,
      "activations/layer8_attention_weight_min": -42.7374267578125,
      "activations/layer9_attention_weight_max": 47.309654235839844,
      "activations/layer9_attention_weight_min": -46.28431701660156,
      "epoch": 9.47,
      "learning_rate": 8.980757575757576e-05,
      "loss": 2.7954,
      "step": 163050
    },
    {
      "activations/layer0_attention_weight_max": 14.836840629577637,
      "activations/layer0_attention_weight_min": -14.376075744628906,
      "activations/layer10_attention_weight_max": 35.810791015625,
      "activations/layer10_attention_weight_min": -33.763465881347656,
      "activations/layer11_attention_weight_max": 31.815784454345703,
      "activations/layer11_attention_weight_min": -30.48110580444336,
      "activations/layer12_attention_weight_max": 21.20926856994629,
      "activations/layer12_attention_weight_min": -24.654220581054688,
      "activations/layer13_attention_weight_max": 42.923885345458984,
      "activations/layer13_attention_weight_min": -31.054153442382812,
      "activations/layer14_attention_weight_max": 39.365272521972656,
      "activations/layer14_attention_weight_min": -31.255863189697266,
      "activations/layer15_attention_weight_max": 38.86101531982422,
      "activations/layer15_attention_weight_min": -28.760805130004883,
      "activations/layer16_attention_weight_max": 35.73823928833008,
      "activations/layer16_attention_weight_min": -30.152156829833984,
      "activations/layer17_attention_weight_max": 56.1878662109375,
      "activations/layer17_attention_weight_min": -46.02943801879883,
      "activations/layer18_attention_weight_max": 48.69807434082031,
      "activations/layer18_attention_weight_min": -38.45989227294922,
      "activations/layer19_attention_weight_max": 23.21083641052246,
      "activations/layer19_attention_weight_min": -24.05659294128418,
      "activations/layer1_attention_weight_max": 16.841581344604492,
      "activations/layer1_attention_weight_min": -17.296117782592773,
      "activations/layer20_attention_weight_max": 23.622798919677734,
      "activations/layer20_attention_weight_min": -25.45913314819336,
      "activations/layer21_attention_weight_max": 41.349544525146484,
      "activations/layer21_attention_weight_min": -25.63582420349121,
      "activations/layer22_attention_weight_max": 33.236141204833984,
      "activations/layer22_attention_weight_min": -24.84075355529785,
      "activations/layer23_attention_weight_max": 40.12362289428711,
      "activations/layer23_attention_weight_min": -23.148815155029297,
      "activations/layer2_attention_weight_max": 31.52634048461914,
      "activations/layer2_attention_weight_min": -28.773860931396484,
      "activations/layer3_attention_weight_max": 93.25000762939453,
      "activations/layer3_attention_weight_min": -98.25326538085938,
      "activations/layer4_attention_weight_max": 104.13655090332031,
      "activations/layer4_attention_weight_min": -95.62643432617188,
      "activations/layer5_attention_weight_max": 78.53544616699219,
      "activations/layer5_attention_weight_min": -77.86707305908203,
      "activations/layer6_attention_weight_max": 54.53761672973633,
      "activations/layer6_attention_weight_min": -50.21050262451172,
      "activations/layer7_attention_weight_max": 80.34048461914062,
      "activations/layer7_attention_weight_min": -69.48177337646484,
      "activations/layer8_attention_weight_max": 45.055641174316406,
      "activations/layer8_attention_weight_min": -46.642303466796875,
      "activations/layer9_attention_weight_max": 47.982975006103516,
      "activations/layer9_attention_weight_min": -47.12183380126953,
      "epoch": 9.48,
      "learning_rate": 8.978863636363635e-05,
      "loss": 2.7909,
      "step": 163100
    },
    {
      "activations/layer0_attention_weight_max": 16.18248176574707,
      "activations/layer0_attention_weight_min": -14.087475776672363,
      "activations/layer10_attention_weight_max": 36.65864944458008,
      "activations/layer10_attention_weight_min": -37.40728759765625,
      "activations/layer11_attention_weight_max": 29.826595306396484,
      "activations/layer11_attention_weight_min": -31.10358238220215,
      "activations/layer12_attention_weight_max": 19.124099731445312,
      "activations/layer12_attention_weight_min": -27.161739349365234,
      "activations/layer13_attention_weight_max": 40.65584182739258,
      "activations/layer13_attention_weight_min": -35.229652404785156,
      "activations/layer14_attention_weight_max": 42.88539123535156,
      "activations/layer14_attention_weight_min": -33.48476791381836,
      "activations/layer15_attention_weight_max": 44.60614776611328,
      "activations/layer15_attention_weight_min": -30.592884063720703,
      "activations/layer16_attention_weight_max": 42.133522033691406,
      "activations/layer16_attention_weight_min": -33.133602142333984,
      "activations/layer17_attention_weight_max": 71.29598236083984,
      "activations/layer17_attention_weight_min": -51.08549499511719,
      "activations/layer18_attention_weight_max": 55.01753234863281,
      "activations/layer18_attention_weight_min": -41.4512825012207,
      "activations/layer19_attention_weight_max": 27.607908248901367,
      "activations/layer19_attention_weight_min": -25.5114803314209,
      "activations/layer1_attention_weight_max": 17.70834732055664,
      "activations/layer1_attention_weight_min": -14.913419723510742,
      "activations/layer20_attention_weight_max": 32.28746795654297,
      "activations/layer20_attention_weight_min": -23.208324432373047,
      "activations/layer21_attention_weight_max": 48.6302375793457,
      "activations/layer21_attention_weight_min": -23.95962905883789,
      "activations/layer22_attention_weight_max": 42.65813446044922,
      "activations/layer22_attention_weight_min": -27.964950561523438,
      "activations/layer23_attention_weight_max": 40.50459289550781,
      "activations/layer23_attention_weight_min": -23.965124130249023,
      "activations/layer2_attention_weight_max": 32.770538330078125,
      "activations/layer2_attention_weight_min": -32.376224517822266,
      "activations/layer3_attention_weight_max": 97.01941680908203,
      "activations/layer3_attention_weight_min": -96.54586791992188,
      "activations/layer4_attention_weight_max": 98.82203674316406,
      "activations/layer4_attention_weight_min": -96.50208282470703,
      "activations/layer5_attention_weight_max": 78.04229736328125,
      "activations/layer5_attention_weight_min": -80.87136840820312,
      "activations/layer6_attention_weight_max": 53.62002944946289,
      "activations/layer6_attention_weight_min": -53.12079620361328,
      "activations/layer7_attention_weight_max": 74.64672088623047,
      "activations/layer7_attention_weight_min": -73.08959197998047,
      "activations/layer8_attention_weight_max": 43.85031509399414,
      "activations/layer8_attention_weight_min": -49.29890060424805,
      "activations/layer9_attention_weight_max": 48.850337982177734,
      "activations/layer9_attention_weight_min": -52.05463409423828,
      "epoch": 9.48,
      "learning_rate": 8.976969696969696e-05,
      "loss": 2.7839,
      "step": 163150
    },
    {
      "activations/layer0_attention_weight_max": 15.55722427368164,
      "activations/layer0_attention_weight_min": -14.406267166137695,
      "activations/layer10_attention_weight_max": 35.215599060058594,
      "activations/layer10_attention_weight_min": -34.61879348754883,
      "activations/layer11_attention_weight_max": 29.167591094970703,
      "activations/layer11_attention_weight_min": -30.922950744628906,
      "activations/layer12_attention_weight_max": 17.320337295532227,
      "activations/layer12_attention_weight_min": -25.21524429321289,
      "activations/layer13_attention_weight_max": 36.873836517333984,
      "activations/layer13_attention_weight_min": -29.47068214416504,
      "activations/layer14_attention_weight_max": 35.04277420043945,
      "activations/layer14_attention_weight_min": -31.564746856689453,
      "activations/layer15_attention_weight_max": 32.68375778198242,
      "activations/layer15_attention_weight_min": -29.526512145996094,
      "activations/layer16_attention_weight_max": 31.1553955078125,
      "activations/layer16_attention_weight_min": -29.196025848388672,
      "activations/layer17_attention_weight_max": 54.5450553894043,
      "activations/layer17_attention_weight_min": -43.28042221069336,
      "activations/layer18_attention_weight_max": 45.683658599853516,
      "activations/layer18_attention_weight_min": -34.99944305419922,
      "activations/layer19_attention_weight_max": 20.015361785888672,
      "activations/layer19_attention_weight_min": -21.477886199951172,
      "activations/layer1_attention_weight_max": 16.590770721435547,
      "activations/layer1_attention_weight_min": -14.589704513549805,
      "activations/layer20_attention_weight_max": 21.673534393310547,
      "activations/layer20_attention_weight_min": -19.71383285522461,
      "activations/layer21_attention_weight_max": 35.13368225097656,
      "activations/layer21_attention_weight_min": -21.486724853515625,
      "activations/layer22_attention_weight_max": 28.847742080688477,
      "activations/layer22_attention_weight_min": -23.033769607543945,
      "activations/layer23_attention_weight_max": 35.51030349731445,
      "activations/layer23_attention_weight_min": -24.371259689331055,
      "activations/layer2_attention_weight_max": 31.89252281188965,
      "activations/layer2_attention_weight_min": -30.076284408569336,
      "activations/layer3_attention_weight_max": 88.59819793701172,
      "activations/layer3_attention_weight_min": -94.60594177246094,
      "activations/layer4_attention_weight_max": 100.40666961669922,
      "activations/layer4_attention_weight_min": -96.17878723144531,
      "activations/layer5_attention_weight_max": 76.83544921875,
      "activations/layer5_attention_weight_min": -80.60952758789062,
      "activations/layer6_attention_weight_max": 55.417381286621094,
      "activations/layer6_attention_weight_min": -55.73648452758789,
      "activations/layer7_attention_weight_max": 75.65782928466797,
      "activations/layer7_attention_weight_min": -73.6974868774414,
      "activations/layer8_attention_weight_max": 42.08641815185547,
      "activations/layer8_attention_weight_min": -44.3159294128418,
      "activations/layer9_attention_weight_max": 44.00259017944336,
      "activations/layer9_attention_weight_min": -46.729427337646484,
      "epoch": 9.48,
      "learning_rate": 8.975075757575758e-05,
      "loss": 2.8042,
      "step": 163200
    },
    {
      "activations/layer0_attention_weight_max": 16.94110870361328,
      "activations/layer0_attention_weight_min": -14.886510848999023,
      "activations/layer10_attention_weight_max": 35.97381591796875,
      "activations/layer10_attention_weight_min": -34.435325622558594,
      "activations/layer11_attention_weight_max": 32.540775299072266,
      "activations/layer11_attention_weight_min": -31.11416244506836,
      "activations/layer12_attention_weight_max": 22.731992721557617,
      "activations/layer12_attention_weight_min": -21.96086883544922,
      "activations/layer13_attention_weight_max": 42.78702163696289,
      "activations/layer13_attention_weight_min": -29.292133331298828,
      "activations/layer14_attention_weight_max": 46.70529556274414,
      "activations/layer14_attention_weight_min": -29.868513107299805,
      "activations/layer15_attention_weight_max": 39.611228942871094,
      "activations/layer15_attention_weight_min": -28.15755271911621,
      "activations/layer16_attention_weight_max": 35.54217529296875,
      "activations/layer16_attention_weight_min": -27.582181930541992,
      "activations/layer17_attention_weight_max": 56.851295471191406,
      "activations/layer17_attention_weight_min": -45.1463623046875,
      "activations/layer18_attention_weight_max": 55.99387741088867,
      "activations/layer18_attention_weight_min": -36.4935302734375,
      "activations/layer19_attention_weight_max": 20.947782516479492,
      "activations/layer19_attention_weight_min": -19.685062408447266,
      "activations/layer1_attention_weight_max": 16.758403778076172,
      "activations/layer1_attention_weight_min": -15.818045616149902,
      "activations/layer20_attention_weight_max": 24.587013244628906,
      "activations/layer20_attention_weight_min": -20.260446548461914,
      "activations/layer21_attention_weight_max": 36.565040588378906,
      "activations/layer21_attention_weight_min": -21.655670166015625,
      "activations/layer22_attention_weight_max": 34.82545471191406,
      "activations/layer22_attention_weight_min": -26.24100112915039,
      "activations/layer23_attention_weight_max": 38.39532470703125,
      "activations/layer23_attention_weight_min": -20.931556701660156,
      "activations/layer2_attention_weight_max": 32.731117248535156,
      "activations/layer2_attention_weight_min": -29.824081420898438,
      "activations/layer3_attention_weight_max": 90.12915802001953,
      "activations/layer3_attention_weight_min": -91.94019317626953,
      "activations/layer4_attention_weight_max": 99.91063690185547,
      "activations/layer4_attention_weight_min": -95.08915710449219,
      "activations/layer5_attention_weight_max": 78.17488098144531,
      "activations/layer5_attention_weight_min": -75.72015380859375,
      "activations/layer6_attention_weight_max": 51.78819274902344,
      "activations/layer6_attention_weight_min": -52.538978576660156,
      "activations/layer7_attention_weight_max": 77.14099884033203,
      "activations/layer7_attention_weight_min": -77.00264739990234,
      "activations/layer8_attention_weight_max": 42.736515045166016,
      "activations/layer8_attention_weight_min": -46.385562896728516,
      "activations/layer9_attention_weight_max": 50.389827728271484,
      "activations/layer9_attention_weight_min": -49.395790100097656,
      "epoch": 9.49,
      "learning_rate": 8.973181818181817e-05,
      "loss": 2.8162,
      "step": 163250
    },
    {
      "activations/layer0_attention_weight_max": 14.723611831665039,
      "activations/layer0_attention_weight_min": -14.482915878295898,
      "activations/layer10_attention_weight_max": 38.25200271606445,
      "activations/layer10_attention_weight_min": -36.34832763671875,
      "activations/layer11_attention_weight_max": 35.14955139160156,
      "activations/layer11_attention_weight_min": -32.73322677612305,
      "activations/layer12_attention_weight_max": 18.555896759033203,
      "activations/layer12_attention_weight_min": -23.39708137512207,
      "activations/layer13_attention_weight_max": 38.51287841796875,
      "activations/layer13_attention_weight_min": -29.820680618286133,
      "activations/layer14_attention_weight_max": 37.899044036865234,
      "activations/layer14_attention_weight_min": -32.84151840209961,
      "activations/layer15_attention_weight_max": 34.39447021484375,
      "activations/layer15_attention_weight_min": -30.647689819335938,
      "activations/layer16_attention_weight_max": 29.90652084350586,
      "activations/layer16_attention_weight_min": -27.140710830688477,
      "activations/layer17_attention_weight_max": 56.90542984008789,
      "activations/layer17_attention_weight_min": -44.077293395996094,
      "activations/layer18_attention_weight_max": 46.28229904174805,
      "activations/layer18_attention_weight_min": -35.946739196777344,
      "activations/layer19_attention_weight_max": 24.438053131103516,
      "activations/layer19_attention_weight_min": -21.781930923461914,
      "activations/layer1_attention_weight_max": 18.773998260498047,
      "activations/layer1_attention_weight_min": -16.328048706054688,
      "activations/layer20_attention_weight_max": 22.73724365234375,
      "activations/layer20_attention_weight_min": -21.274930953979492,
      "activations/layer21_attention_weight_max": 41.703125,
      "activations/layer21_attention_weight_min": -22.809558868408203,
      "activations/layer22_attention_weight_max": 30.929643630981445,
      "activations/layer22_attention_weight_min": -25.62108612060547,
      "activations/layer23_attention_weight_max": 36.208595275878906,
      "activations/layer23_attention_weight_min": -22.739439010620117,
      "activations/layer2_attention_weight_max": 34.84967041015625,
      "activations/layer2_attention_weight_min": -33.68264389038086,
      "activations/layer3_attention_weight_max": 100.0693130493164,
      "activations/layer3_attention_weight_min": -99.9122543334961,
      "activations/layer4_attention_weight_max": 102.68607330322266,
      "activations/layer4_attention_weight_min": -104.40030670166016,
      "activations/layer5_attention_weight_max": 82.31666564941406,
      "activations/layer5_attention_weight_min": -79.62741088867188,
      "activations/layer6_attention_weight_max": 56.81412887573242,
      "activations/layer6_attention_weight_min": -58.298004150390625,
      "activations/layer7_attention_weight_max": 74.98091888427734,
      "activations/layer7_attention_weight_min": -75.63490295410156,
      "activations/layer8_attention_weight_max": 47.96889114379883,
      "activations/layer8_attention_weight_min": -49.21579360961914,
      "activations/layer9_attention_weight_max": 50.00257110595703,
      "activations/layer9_attention_weight_min": -49.10976791381836,
      "epoch": 9.49,
      "learning_rate": 8.971287878787878e-05,
      "loss": 2.8016,
      "step": 163300
    },
    {
      "activations/layer0_attention_weight_max": 15.858134269714355,
      "activations/layer0_attention_weight_min": -14.26531982421875,
      "activations/layer10_attention_weight_max": 40.45695877075195,
      "activations/layer10_attention_weight_min": -36.50000762939453,
      "activations/layer11_attention_weight_max": 35.50572204589844,
      "activations/layer11_attention_weight_min": -32.49737548828125,
      "activations/layer12_attention_weight_max": 24.725847244262695,
      "activations/layer12_attention_weight_min": -24.4647274017334,
      "activations/layer13_attention_weight_max": 44.01096725463867,
      "activations/layer13_attention_weight_min": -35.30909729003906,
      "activations/layer14_attention_weight_max": 48.73445129394531,
      "activations/layer14_attention_weight_min": -31.0052547454834,
      "activations/layer15_attention_weight_max": 38.523746490478516,
      "activations/layer15_attention_weight_min": -29.611034393310547,
      "activations/layer16_attention_weight_max": 39.66106033325195,
      "activations/layer16_attention_weight_min": -30.46654510498047,
      "activations/layer17_attention_weight_max": 65.86043548583984,
      "activations/layer17_attention_weight_min": -47.58533477783203,
      "activations/layer18_attention_weight_max": 59.83149337768555,
      "activations/layer18_attention_weight_min": -39.011131286621094,
      "activations/layer19_attention_weight_max": 25.049962997436523,
      "activations/layer19_attention_weight_min": -21.9334774017334,
      "activations/layer1_attention_weight_max": 17.103918075561523,
      "activations/layer1_attention_weight_min": -14.772062301635742,
      "activations/layer20_attention_weight_max": 28.395294189453125,
      "activations/layer20_attention_weight_min": -22.90319061279297,
      "activations/layer21_attention_weight_max": 48.44205856323242,
      "activations/layer21_attention_weight_min": -25.34702491760254,
      "activations/layer22_attention_weight_max": 33.47495651245117,
      "activations/layer22_attention_weight_min": -25.861541748046875,
      "activations/layer23_attention_weight_max": 46.470298767089844,
      "activations/layer23_attention_weight_min": -24.06865692138672,
      "activations/layer2_attention_weight_max": 34.15464782714844,
      "activations/layer2_attention_weight_min": -30.153484344482422,
      "activations/layer3_attention_weight_max": 92.48526000976562,
      "activations/layer3_attention_weight_min": -97.4684066772461,
      "activations/layer4_attention_weight_max": 102.6796875,
      "activations/layer4_attention_weight_min": -102.10009765625,
      "activations/layer5_attention_weight_max": 80.96339416503906,
      "activations/layer5_attention_weight_min": -78.45572662353516,
      "activations/layer6_attention_weight_max": 56.70756149291992,
      "activations/layer6_attention_weight_min": -54.87117004394531,
      "activations/layer7_attention_weight_max": 80.56549072265625,
      "activations/layer7_attention_weight_min": -71.80339813232422,
      "activations/layer8_attention_weight_max": 48.01251983642578,
      "activations/layer8_attention_weight_min": -49.84444808959961,
      "activations/layer9_attention_weight_max": 53.08386993408203,
      "activations/layer9_attention_weight_min": -51.0108528137207,
      "epoch": 9.49,
      "learning_rate": 8.969393939393938e-05,
      "loss": 2.7961,
      "step": 163350
    },
    {
      "activations/layer0_attention_weight_max": 15.032448768615723,
      "activations/layer0_attention_weight_min": -14.465713500976562,
      "activations/layer10_attention_weight_max": 37.531681060791016,
      "activations/layer10_attention_weight_min": -35.88990020751953,
      "activations/layer11_attention_weight_max": 32.68861770629883,
      "activations/layer11_attention_weight_min": -33.32023620605469,
      "activations/layer12_attention_weight_max": 21.760116577148438,
      "activations/layer12_attention_weight_min": -23.385221481323242,
      "activations/layer13_attention_weight_max": 46.53929901123047,
      "activations/layer13_attention_weight_min": -33.05473327636719,
      "activations/layer14_attention_weight_max": 39.57960891723633,
      "activations/layer14_attention_weight_min": -30.101545333862305,
      "activations/layer15_attention_weight_max": 35.71643829345703,
      "activations/layer15_attention_weight_min": -31.20513153076172,
      "activations/layer16_attention_weight_max": 36.320770263671875,
      "activations/layer16_attention_weight_min": -31.52286720275879,
      "activations/layer17_attention_weight_max": 53.01463317871094,
      "activations/layer17_attention_weight_min": -50.47403335571289,
      "activations/layer18_attention_weight_max": 49.034297943115234,
      "activations/layer18_attention_weight_min": -42.537559509277344,
      "activations/layer19_attention_weight_max": 22.119474411010742,
      "activations/layer19_attention_weight_min": -21.496028900146484,
      "activations/layer1_attention_weight_max": 16.853355407714844,
      "activations/layer1_attention_weight_min": -15.275249481201172,
      "activations/layer20_attention_weight_max": 26.307592391967773,
      "activations/layer20_attention_weight_min": -22.716930389404297,
      "activations/layer21_attention_weight_max": 36.93819046020508,
      "activations/layer21_attention_weight_min": -23.125905990600586,
      "activations/layer22_attention_weight_max": 33.874473571777344,
      "activations/layer22_attention_weight_min": -27.16645050048828,
      "activations/layer23_attention_weight_max": 44.06376266479492,
      "activations/layer23_attention_weight_min": -27.903396606445312,
      "activations/layer2_attention_weight_max": 31.136341094970703,
      "activations/layer2_attention_weight_min": -31.359832763671875,
      "activations/layer3_attention_weight_max": 89.44518280029297,
      "activations/layer3_attention_weight_min": -98.14695739746094,
      "activations/layer4_attention_weight_max": 100.82025909423828,
      "activations/layer4_attention_weight_min": -97.12199401855469,
      "activations/layer5_attention_weight_max": 79.68434143066406,
      "activations/layer5_attention_weight_min": -79.44429016113281,
      "activations/layer6_attention_weight_max": 54.36992263793945,
      "activations/layer6_attention_weight_min": -53.64637756347656,
      "activations/layer7_attention_weight_max": 80.11017608642578,
      "activations/layer7_attention_weight_min": -77.23710632324219,
      "activations/layer8_attention_weight_max": 47.19007110595703,
      "activations/layer8_attention_weight_min": -48.04975128173828,
      "activations/layer9_attention_weight_max": 55.021053314208984,
      "activations/layer9_attention_weight_min": -51.34553146362305,
      "epoch": 9.49,
      "learning_rate": 8.9675e-05,
      "loss": 2.7941,
      "step": 163400
    },
    {
      "activations/layer0_attention_weight_max": 15.220696449279785,
      "activations/layer0_attention_weight_min": -14.099344253540039,
      "activations/layer10_attention_weight_max": 35.39385986328125,
      "activations/layer10_attention_weight_min": -33.63730239868164,
      "activations/layer11_attention_weight_max": 30.748069763183594,
      "activations/layer11_attention_weight_min": -32.2132682800293,
      "activations/layer12_attention_weight_max": 21.348011016845703,
      "activations/layer12_attention_weight_min": -27.2432918548584,
      "activations/layer13_attention_weight_max": 38.24384689331055,
      "activations/layer13_attention_weight_min": -29.136995315551758,
      "activations/layer14_attention_weight_max": 40.02796936035156,
      "activations/layer14_attention_weight_min": -30.061630249023438,
      "activations/layer15_attention_weight_max": 33.07868194580078,
      "activations/layer15_attention_weight_min": -30.248323440551758,
      "activations/layer16_attention_weight_max": 30.873424530029297,
      "activations/layer16_attention_weight_min": -28.825592041015625,
      "activations/layer17_attention_weight_max": 54.59477615356445,
      "activations/layer17_attention_weight_min": -46.08061981201172,
      "activations/layer18_attention_weight_max": 46.40646743774414,
      "activations/layer18_attention_weight_min": -38.335350036621094,
      "activations/layer19_attention_weight_max": 20.074460983276367,
      "activations/layer19_attention_weight_min": -22.07840919494629,
      "activations/layer1_attention_weight_max": 16.859115600585938,
      "activations/layer1_attention_weight_min": -16.02861976623535,
      "activations/layer20_attention_weight_max": 21.491321563720703,
      "activations/layer20_attention_weight_min": -21.03517723083496,
      "activations/layer21_attention_weight_max": 34.02693557739258,
      "activations/layer21_attention_weight_min": -23.665483474731445,
      "activations/layer22_attention_weight_max": 30.933671951293945,
      "activations/layer22_attention_weight_min": -25.8789119720459,
      "activations/layer23_attention_weight_max": 35.97066879272461,
      "activations/layer23_attention_weight_min": -25.997840881347656,
      "activations/layer2_attention_weight_max": 32.447593688964844,
      "activations/layer2_attention_weight_min": -29.57027816772461,
      "activations/layer3_attention_weight_max": 93.48285675048828,
      "activations/layer3_attention_weight_min": -95.09246063232422,
      "activations/layer4_attention_weight_max": 99.1423568725586,
      "activations/layer4_attention_weight_min": -94.30724334716797,
      "activations/layer5_attention_weight_max": 79.49107360839844,
      "activations/layer5_attention_weight_min": -78.32554626464844,
      "activations/layer6_attention_weight_max": 52.05864715576172,
      "activations/layer6_attention_weight_min": -53.170738220214844,
      "activations/layer7_attention_weight_max": 71.60767364501953,
      "activations/layer7_attention_weight_min": -74.02266693115234,
      "activations/layer8_attention_weight_max": 45.17963409423828,
      "activations/layer8_attention_weight_min": -46.63685989379883,
      "activations/layer9_attention_weight_max": 46.610107421875,
      "activations/layer9_attention_weight_min": -48.24699020385742,
      "epoch": 9.5,
      "learning_rate": 8.965643939393939e-05,
      "loss": 2.7968,
      "step": 163450
    },
    {
      "activations/layer0_attention_weight_max": 15.559794425964355,
      "activations/layer0_attention_weight_min": -13.93848991394043,
      "activations/layer10_attention_weight_max": 35.70379638671875,
      "activations/layer10_attention_weight_min": -35.62236404418945,
      "activations/layer11_attention_weight_max": 31.124439239501953,
      "activations/layer11_attention_weight_min": -32.792388916015625,
      "activations/layer12_attention_weight_max": 20.759984970092773,
      "activations/layer12_attention_weight_min": -28.16804313659668,
      "activations/layer13_attention_weight_max": 37.17674255371094,
      "activations/layer13_attention_weight_min": -30.639122009277344,
      "activations/layer14_attention_weight_max": 38.58679962158203,
      "activations/layer14_attention_weight_min": -31.822431564331055,
      "activations/layer15_attention_weight_max": 36.57682800292969,
      "activations/layer15_attention_weight_min": -30.70683479309082,
      "activations/layer16_attention_weight_max": 31.017093658447266,
      "activations/layer16_attention_weight_min": -31.197689056396484,
      "activations/layer17_attention_weight_max": 54.751888275146484,
      "activations/layer17_attention_weight_min": -46.24661636352539,
      "activations/layer18_attention_weight_max": 47.728084564208984,
      "activations/layer18_attention_weight_min": -37.00439453125,
      "activations/layer19_attention_weight_max": 20.16884994506836,
      "activations/layer19_attention_weight_min": -21.18452262878418,
      "activations/layer1_attention_weight_max": 17.08629608154297,
      "activations/layer1_attention_weight_min": -16.225189208984375,
      "activations/layer20_attention_weight_max": 21.40612030029297,
      "activations/layer20_attention_weight_min": -23.357301712036133,
      "activations/layer21_attention_weight_max": 36.40803527832031,
      "activations/layer21_attention_weight_min": -23.594823837280273,
      "activations/layer22_attention_weight_max": 31.36444854736328,
      "activations/layer22_attention_weight_min": -24.918657302856445,
      "activations/layer23_attention_weight_max": 35.09550476074219,
      "activations/layer23_attention_weight_min": -21.833402633666992,
      "activations/layer2_attention_weight_max": 32.09877395629883,
      "activations/layer2_attention_weight_min": -30.47754669189453,
      "activations/layer3_attention_weight_max": 90.89492797851562,
      "activations/layer3_attention_weight_min": -95.63513946533203,
      "activations/layer4_attention_weight_max": 100.4452133178711,
      "activations/layer4_attention_weight_min": -97.86922454833984,
      "activations/layer5_attention_weight_max": 80.17245483398438,
      "activations/layer5_attention_weight_min": -79.61076354980469,
      "activations/layer6_attention_weight_max": 52.946292877197266,
      "activations/layer6_attention_weight_min": -55.10911560058594,
      "activations/layer7_attention_weight_max": 75.55738067626953,
      "activations/layer7_attention_weight_min": -79.47798156738281,
      "activations/layer8_attention_weight_max": 42.654850006103516,
      "activations/layer8_attention_weight_min": -45.16873550415039,
      "activations/layer9_attention_weight_max": 48.36915588378906,
      "activations/layer9_attention_weight_min": -48.51935577392578,
      "epoch": 9.5,
      "learning_rate": 8.963749999999999e-05,
      "loss": 2.7832,
      "step": 163500
    },
    {
      "activations/layer0_attention_weight_max": 15.757920265197754,
      "activations/layer0_attention_weight_min": -13.902278900146484,
      "activations/layer10_attention_weight_max": 35.224029541015625,
      "activations/layer10_attention_weight_min": -33.37953567504883,
      "activations/layer11_attention_weight_max": 31.725250244140625,
      "activations/layer11_attention_weight_min": -29.388975143432617,
      "activations/layer12_attention_weight_max": 19.712820053100586,
      "activations/layer12_attention_weight_min": -29.307981491088867,
      "activations/layer13_attention_weight_max": 41.718990325927734,
      "activations/layer13_attention_weight_min": -29.209245681762695,
      "activations/layer14_attention_weight_max": 36.677528381347656,
      "activations/layer14_attention_weight_min": -29.903980255126953,
      "activations/layer15_attention_weight_max": 33.53493881225586,
      "activations/layer15_attention_weight_min": -28.656932830810547,
      "activations/layer16_attention_weight_max": 32.26472854614258,
      "activations/layer16_attention_weight_min": -27.15481948852539,
      "activations/layer17_attention_weight_max": 57.58736038208008,
      "activations/layer17_attention_weight_min": -41.86518096923828,
      "activations/layer18_attention_weight_max": 52.80167007446289,
      "activations/layer18_attention_weight_min": -36.196258544921875,
      "activations/layer19_attention_weight_max": 23.853744506835938,
      "activations/layer19_attention_weight_min": -21.87617301940918,
      "activations/layer1_attention_weight_max": 17.324111938476562,
      "activations/layer1_attention_weight_min": -14.324525833129883,
      "activations/layer20_attention_weight_max": 21.856237411499023,
      "activations/layer20_attention_weight_min": -24.218748092651367,
      "activations/layer21_attention_weight_max": 38.4129638671875,
      "activations/layer21_attention_weight_min": -22.01315689086914,
      "activations/layer22_attention_weight_max": 32.03025817871094,
      "activations/layer22_attention_weight_min": -27.54372215270996,
      "activations/layer23_attention_weight_max": 36.332847595214844,
      "activations/layer23_attention_weight_min": -24.734182357788086,
      "activations/layer2_attention_weight_max": 35.05830001831055,
      "activations/layer2_attention_weight_min": -31.46880340576172,
      "activations/layer3_attention_weight_max": 94.00164794921875,
      "activations/layer3_attention_weight_min": -99.92753601074219,
      "activations/layer4_attention_weight_max": 101.48775482177734,
      "activations/layer4_attention_weight_min": -104.33026123046875,
      "activations/layer5_attention_weight_max": 76.43682098388672,
      "activations/layer5_attention_weight_min": -78.79350280761719,
      "activations/layer6_attention_weight_max": 51.580787658691406,
      "activations/layer6_attention_weight_min": -52.29601287841797,
      "activations/layer7_attention_weight_max": 75.21702575683594,
      "activations/layer7_attention_weight_min": -75.13339233398438,
      "activations/layer8_attention_weight_max": 43.637630462646484,
      "activations/layer8_attention_weight_min": -44.11399841308594,
      "activations/layer9_attention_weight_max": 50.29751205444336,
      "activations/layer9_attention_weight_min": -52.35404586791992,
      "epoch": 9.5,
      "learning_rate": 8.961856060606059e-05,
      "loss": 2.7962,
      "step": 163550
    },
    {
      "activations/layer0_attention_weight_max": 15.123417854309082,
      "activations/layer0_attention_weight_min": -14.557463645935059,
      "activations/layer10_attention_weight_max": 39.06165313720703,
      "activations/layer10_attention_weight_min": -36.20073318481445,
      "activations/layer11_attention_weight_max": 32.454463958740234,
      "activations/layer11_attention_weight_min": -32.797340393066406,
      "activations/layer12_attention_weight_max": 20.41529655456543,
      "activations/layer12_attention_weight_min": -23.984554290771484,
      "activations/layer13_attention_weight_max": 43.125648498535156,
      "activations/layer13_attention_weight_min": -30.508464813232422,
      "activations/layer14_attention_weight_max": 38.32993698120117,
      "activations/layer14_attention_weight_min": -32.6136589050293,
      "activations/layer15_attention_weight_max": 33.70737075805664,
      "activations/layer15_attention_weight_min": -30.253597259521484,
      "activations/layer16_attention_weight_max": 32.49201965332031,
      "activations/layer16_attention_weight_min": -27.83481216430664,
      "activations/layer17_attention_weight_max": 51.474727630615234,
      "activations/layer17_attention_weight_min": -44.109825134277344,
      "activations/layer18_attention_weight_max": 45.66856002807617,
      "activations/layer18_attention_weight_min": -39.59328842163086,
      "activations/layer19_attention_weight_max": 21.326810836791992,
      "activations/layer19_attention_weight_min": -22.417932510375977,
      "activations/layer1_attention_weight_max": 17.280078887939453,
      "activations/layer1_attention_weight_min": -17.44110107421875,
      "activations/layer20_attention_weight_max": 21.30289077758789,
      "activations/layer20_attention_weight_min": -25.594152450561523,
      "activations/layer21_attention_weight_max": 35.49962615966797,
      "activations/layer21_attention_weight_min": -23.701749801635742,
      "activations/layer22_attention_weight_max": 28.620874404907227,
      "activations/layer22_attention_weight_min": -26.32457733154297,
      "activations/layer23_attention_weight_max": 35.381919860839844,
      "activations/layer23_attention_weight_min": -26.03700065612793,
      "activations/layer2_attention_weight_max": 32.91632843017578,
      "activations/layer2_attention_weight_min": -30.735885620117188,
      "activations/layer3_attention_weight_max": 91.08522033691406,
      "activations/layer3_attention_weight_min": -97.63977813720703,
      "activations/layer4_attention_weight_max": 99.09983825683594,
      "activations/layer4_attention_weight_min": -97.07635498046875,
      "activations/layer5_attention_weight_max": 79.7157211303711,
      "activations/layer5_attention_weight_min": -77.16151428222656,
      "activations/layer6_attention_weight_max": 53.78341293334961,
      "activations/layer6_attention_weight_min": -54.26276397705078,
      "activations/layer7_attention_weight_max": 78.4378662109375,
      "activations/layer7_attention_weight_min": -72.82608032226562,
      "activations/layer8_attention_weight_max": 44.73502731323242,
      "activations/layer8_attention_weight_min": -46.00213623046875,
      "activations/layer9_attention_weight_max": 58.19026565551758,
      "activations/layer9_attention_weight_min": -49.60673141479492,
      "epoch": 9.51,
      "learning_rate": 8.95996212121212e-05,
      "loss": 2.7998,
      "step": 163600
    },
    {
      "activations/layer0_attention_weight_max": 15.352164268493652,
      "activations/layer0_attention_weight_min": -13.848316192626953,
      "activations/layer10_attention_weight_max": 33.6927490234375,
      "activations/layer10_attention_weight_min": -34.156707763671875,
      "activations/layer11_attention_weight_max": 31.21636962890625,
      "activations/layer11_attention_weight_min": -32.28257751464844,
      "activations/layer12_attention_weight_max": 19.307958602905273,
      "activations/layer12_attention_weight_min": -26.80453872680664,
      "activations/layer13_attention_weight_max": 42.1932373046875,
      "activations/layer13_attention_weight_min": -37.79655838012695,
      "activations/layer14_attention_weight_max": 38.5489387512207,
      "activations/layer14_attention_weight_min": -29.19870376586914,
      "activations/layer15_attention_weight_max": 32.6401481628418,
      "activations/layer15_attention_weight_min": -27.234859466552734,
      "activations/layer16_attention_weight_max": 27.321575164794922,
      "activations/layer16_attention_weight_min": -28.22940444946289,
      "activations/layer17_attention_weight_max": 56.87626647949219,
      "activations/layer17_attention_weight_min": -42.93513870239258,
      "activations/layer18_attention_weight_max": 45.36365509033203,
      "activations/layer18_attention_weight_min": -36.2169303894043,
      "activations/layer19_attention_weight_max": 20.9379940032959,
      "activations/layer19_attention_weight_min": -20.77242088317871,
      "activations/layer1_attention_weight_max": 16.753589630126953,
      "activations/layer1_attention_weight_min": -17.314733505249023,
      "activations/layer20_attention_weight_max": 21.91006088256836,
      "activations/layer20_attention_weight_min": -24.31498908996582,
      "activations/layer21_attention_weight_max": 36.65272521972656,
      "activations/layer21_attention_weight_min": -22.91929054260254,
      "activations/layer22_attention_weight_max": 29.381439208984375,
      "activations/layer22_attention_weight_min": -26.48026466369629,
      "activations/layer23_attention_weight_max": 32.54290008544922,
      "activations/layer23_attention_weight_min": -24.23949432373047,
      "activations/layer2_attention_weight_max": 31.788814544677734,
      "activations/layer2_attention_weight_min": -29.70132064819336,
      "activations/layer3_attention_weight_max": 94.06404876708984,
      "activations/layer3_attention_weight_min": -101.55125427246094,
      "activations/layer4_attention_weight_max": 99.64070892333984,
      "activations/layer4_attention_weight_min": -97.90144348144531,
      "activations/layer5_attention_weight_max": 81.5642318725586,
      "activations/layer5_attention_weight_min": -78.90757751464844,
      "activations/layer6_attention_weight_max": 53.37924575805664,
      "activations/layer6_attention_weight_min": -55.13090133666992,
      "activations/layer7_attention_weight_max": 73.44820404052734,
      "activations/layer7_attention_weight_min": -75.91826629638672,
      "activations/layer8_attention_weight_max": 43.783145904541016,
      "activations/layer8_attention_weight_min": -46.636566162109375,
      "activations/layer9_attention_weight_max": 46.896060943603516,
      "activations/layer9_attention_weight_min": -50.61875534057617,
      "epoch": 9.51,
      "learning_rate": 8.958068181818182e-05,
      "loss": 2.7912,
      "step": 163650
    },
    {
      "activations/layer0_attention_weight_max": 15.350536346435547,
      "activations/layer0_attention_weight_min": -14.328071594238281,
      "activations/layer10_attention_weight_max": 38.554473876953125,
      "activations/layer10_attention_weight_min": -36.06354522705078,
      "activations/layer11_attention_weight_max": 32.484901428222656,
      "activations/layer11_attention_weight_min": -33.23615646362305,
      "activations/layer12_attention_weight_max": 19.568004608154297,
      "activations/layer12_attention_weight_min": -25.497711181640625,
      "activations/layer13_attention_weight_max": 39.415138244628906,
      "activations/layer13_attention_weight_min": -31.831111907958984,
      "activations/layer14_attention_weight_max": 37.737754821777344,
      "activations/layer14_attention_weight_min": -29.85190773010254,
      "activations/layer15_attention_weight_max": 36.99569320678711,
      "activations/layer15_attention_weight_min": -30.890226364135742,
      "activations/layer16_attention_weight_max": 32.918006896972656,
      "activations/layer16_attention_weight_min": -28.51253890991211,
      "activations/layer17_attention_weight_max": 58.4870491027832,
      "activations/layer17_attention_weight_min": -44.404136657714844,
      "activations/layer18_attention_weight_max": 46.78763961791992,
      "activations/layer18_attention_weight_min": -37.582889556884766,
      "activations/layer19_attention_weight_max": 25.129737854003906,
      "activations/layer19_attention_weight_min": -22.04799461364746,
      "activations/layer1_attention_weight_max": 16.703392028808594,
      "activations/layer1_attention_weight_min": -17.42635726928711,
      "activations/layer20_attention_weight_max": 22.97671890258789,
      "activations/layer20_attention_weight_min": -22.410051345825195,
      "activations/layer21_attention_weight_max": 38.30355453491211,
      "activations/layer21_attention_weight_min": -23.488351821899414,
      "activations/layer22_attention_weight_max": 32.51742935180664,
      "activations/layer22_attention_weight_min": -29.148767471313477,
      "activations/layer23_attention_weight_max": 45.37650680541992,
      "activations/layer23_attention_weight_min": -26.697345733642578,
      "activations/layer2_attention_weight_max": 31.484046936035156,
      "activations/layer2_attention_weight_min": -29.93287467956543,
      "activations/layer3_attention_weight_max": 96.10472869873047,
      "activations/layer3_attention_weight_min": -95.00977325439453,
      "activations/layer4_attention_weight_max": 101.5562744140625,
      "activations/layer4_attention_weight_min": -100.63085174560547,
      "activations/layer5_attention_weight_max": 80.30241394042969,
      "activations/layer5_attention_weight_min": -83.68731689453125,
      "activations/layer6_attention_weight_max": 57.06251907348633,
      "activations/layer6_attention_weight_min": -54.171730041503906,
      "activations/layer7_attention_weight_max": 80.22988891601562,
      "activations/layer7_attention_weight_min": -74.22998809814453,
      "activations/layer8_attention_weight_max": 48.30182647705078,
      "activations/layer8_attention_weight_min": -50.20698928833008,
      "activations/layer9_attention_weight_max": 54.47111892700195,
      "activations/layer9_attention_weight_min": -48.1676139831543,
      "epoch": 9.51,
      "learning_rate": 8.956174242424241e-05,
      "loss": 2.7994,
      "step": 163700
    },
    {
      "activations/layer0_attention_weight_max": 15.936558723449707,
      "activations/layer0_attention_weight_min": -14.356260299682617,
      "activations/layer10_attention_weight_max": 41.64032745361328,
      "activations/layer10_attention_weight_min": -39.26384353637695,
      "activations/layer11_attention_weight_max": 35.22420120239258,
      "activations/layer11_attention_weight_min": -34.2326774597168,
      "activations/layer12_attention_weight_max": 18.42848014831543,
      "activations/layer12_attention_weight_min": -33.51719284057617,
      "activations/layer13_attention_weight_max": 62.32794952392578,
      "activations/layer13_attention_weight_min": -33.66288375854492,
      "activations/layer14_attention_weight_max": 71.7879409790039,
      "activations/layer14_attention_weight_min": -34.61573791503906,
      "activations/layer15_attention_weight_max": 58.42986297607422,
      "activations/layer15_attention_weight_min": -31.43634033203125,
      "activations/layer16_attention_weight_max": 32.66748809814453,
      "activations/layer16_attention_weight_min": -30.540685653686523,
      "activations/layer17_attention_weight_max": 52.9228401184082,
      "activations/layer17_attention_weight_min": -45.183780670166016,
      "activations/layer18_attention_weight_max": 49.70250701904297,
      "activations/layer18_attention_weight_min": -38.10331726074219,
      "activations/layer19_attention_weight_max": 20.453794479370117,
      "activations/layer19_attention_weight_min": -22.635534286499023,
      "activations/layer1_attention_weight_max": 16.73880386352539,
      "activations/layer1_attention_weight_min": -15.569840431213379,
      "activations/layer20_attention_weight_max": 21.108417510986328,
      "activations/layer20_attention_weight_min": -20.53680992126465,
      "activations/layer21_attention_weight_max": 34.2991943359375,
      "activations/layer21_attention_weight_min": -23.290544509887695,
      "activations/layer22_attention_weight_max": 32.59419631958008,
      "activations/layer22_attention_weight_min": -25.583398818969727,
      "activations/layer23_attention_weight_max": 39.041419982910156,
      "activations/layer23_attention_weight_min": -20.77596664428711,
      "activations/layer2_attention_weight_max": 34.266727447509766,
      "activations/layer2_attention_weight_min": -29.51720428466797,
      "activations/layer3_attention_weight_max": 98.01178741455078,
      "activations/layer3_attention_weight_min": -95.51276397705078,
      "activations/layer4_attention_weight_max": 103.40213775634766,
      "activations/layer4_attention_weight_min": -95.21007537841797,
      "activations/layer5_attention_weight_max": 77.16766357421875,
      "activations/layer5_attention_weight_min": -76.78009033203125,
      "activations/layer6_attention_weight_max": 54.085655212402344,
      "activations/layer6_attention_weight_min": -54.57245635986328,
      "activations/layer7_attention_weight_max": 83.2264404296875,
      "activations/layer7_attention_weight_min": -74.4293212890625,
      "activations/layer8_attention_weight_max": 45.51225280761719,
      "activations/layer8_attention_weight_min": -48.87623596191406,
      "activations/layer9_attention_weight_max": 58.29267501831055,
      "activations/layer9_attention_weight_min": -51.28717803955078,
      "epoch": 9.51,
      "learning_rate": 8.954280303030302e-05,
      "loss": 2.8179,
      "step": 163750
    },
    {
      "activations/layer0_attention_weight_max": 14.601612091064453,
      "activations/layer0_attention_weight_min": -14.41970157623291,
      "activations/layer10_attention_weight_max": 37.73122024536133,
      "activations/layer10_attention_weight_min": -34.110435485839844,
      "activations/layer11_attention_weight_max": 31.618711471557617,
      "activations/layer11_attention_weight_min": -30.330556869506836,
      "activations/layer12_attention_weight_max": 23.66965103149414,
      "activations/layer12_attention_weight_min": -24.572132110595703,
      "activations/layer13_attention_weight_max": 43.21213912963867,
      "activations/layer13_attention_weight_min": -31.986543655395508,
      "activations/layer14_attention_weight_max": 35.70820999145508,
      "activations/layer14_attention_weight_min": -30.55875587463379,
      "activations/layer15_attention_weight_max": 35.606204986572266,
      "activations/layer15_attention_weight_min": -30.26392364501953,
      "activations/layer16_attention_weight_max": 29.18775177001953,
      "activations/layer16_attention_weight_min": -28.09685707092285,
      "activations/layer17_attention_weight_max": 51.88114929199219,
      "activations/layer17_attention_weight_min": -42.06466293334961,
      "activations/layer18_attention_weight_max": 46.59788131713867,
      "activations/layer18_attention_weight_min": -36.35197830200195,
      "activations/layer19_attention_weight_max": 22.161725997924805,
      "activations/layer19_attention_weight_min": -20.980669021606445,
      "activations/layer1_attention_weight_max": 16.317567825317383,
      "activations/layer1_attention_weight_min": -16.03380012512207,
      "activations/layer20_attention_weight_max": 24.19754981994629,
      "activations/layer20_attention_weight_min": -21.095802307128906,
      "activations/layer21_attention_weight_max": 34.46503448486328,
      "activations/layer21_attention_weight_min": -23.702850341796875,
      "activations/layer22_attention_weight_max": 31.672237396240234,
      "activations/layer22_attention_weight_min": -25.81081199645996,
      "activations/layer23_attention_weight_max": 36.614749908447266,
      "activations/layer23_attention_weight_min": -23.996013641357422,
      "activations/layer2_attention_weight_max": 32.279300689697266,
      "activations/layer2_attention_weight_min": -29.211570739746094,
      "activations/layer3_attention_weight_max": 90.64537811279297,
      "activations/layer3_attention_weight_min": -94.70073699951172,
      "activations/layer4_attention_weight_max": 98.69721221923828,
      "activations/layer4_attention_weight_min": -95.87596893310547,
      "activations/layer5_attention_weight_max": 78.49046325683594,
      "activations/layer5_attention_weight_min": -79.46772766113281,
      "activations/layer6_attention_weight_max": 54.30034637451172,
      "activations/layer6_attention_weight_min": -53.07107925415039,
      "activations/layer7_attention_weight_max": 78.90052032470703,
      "activations/layer7_attention_weight_min": -72.50896453857422,
      "activations/layer8_attention_weight_max": 45.64501190185547,
      "activations/layer8_attention_weight_min": -45.83887481689453,
      "activations/layer9_attention_weight_max": 48.639583587646484,
      "activations/layer9_attention_weight_min": -46.43360137939453,
      "epoch": 9.52,
      "learning_rate": 8.952386363636364e-05,
      "loss": 2.7972,
      "step": 163800
    },
    {
      "activations/layer0_attention_weight_max": 14.95390796661377,
      "activations/layer0_attention_weight_min": -14.013700485229492,
      "activations/layer10_attention_weight_max": 34.202606201171875,
      "activations/layer10_attention_weight_min": -33.017669677734375,
      "activations/layer11_attention_weight_max": 30.7229061126709,
      "activations/layer11_attention_weight_min": -30.283893585205078,
      "activations/layer12_attention_weight_max": 20.722732543945312,
      "activations/layer12_attention_weight_min": -27.168678283691406,
      "activations/layer13_attention_weight_max": 40.116573333740234,
      "activations/layer13_attention_weight_min": -33.18815612792969,
      "activations/layer14_attention_weight_max": 38.245296478271484,
      "activations/layer14_attention_weight_min": -32.11124801635742,
      "activations/layer15_attention_weight_max": 35.4559326171875,
      "activations/layer15_attention_weight_min": -29.98040199279785,
      "activations/layer16_attention_weight_max": 32.18000411987305,
      "activations/layer16_attention_weight_min": -30.40302276611328,
      "activations/layer17_attention_weight_max": 53.665828704833984,
      "activations/layer17_attention_weight_min": -45.36555862426758,
      "activations/layer18_attention_weight_max": 46.498291015625,
      "activations/layer18_attention_weight_min": -39.50970458984375,
      "activations/layer19_attention_weight_max": 25.07660484313965,
      "activations/layer19_attention_weight_min": -20.689680099487305,
      "activations/layer1_attention_weight_max": 17.456571578979492,
      "activations/layer1_attention_weight_min": -19.335140228271484,
      "activations/layer20_attention_weight_max": 22.222307205200195,
      "activations/layer20_attention_weight_min": -21.738601684570312,
      "activations/layer21_attention_weight_max": 37.88618850708008,
      "activations/layer21_attention_weight_min": -23.26854133605957,
      "activations/layer22_attention_weight_max": 31.19940185546875,
      "activations/layer22_attention_weight_min": -26.894941329956055,
      "activations/layer23_attention_weight_max": 38.194828033447266,
      "activations/layer23_attention_weight_min": -24.531415939331055,
      "activations/layer2_attention_weight_max": 32.56032180786133,
      "activations/layer2_attention_weight_min": -30.756210327148438,
      "activations/layer3_attention_weight_max": 90.41215515136719,
      "activations/layer3_attention_weight_min": -95.66313171386719,
      "activations/layer4_attention_weight_max": 100.80757904052734,
      "activations/layer4_attention_weight_min": -98.218505859375,
      "activations/layer5_attention_weight_max": 79.3123779296875,
      "activations/layer5_attention_weight_min": -80.3450927734375,
      "activations/layer6_attention_weight_max": 56.34540557861328,
      "activations/layer6_attention_weight_min": -57.13762664794922,
      "activations/layer7_attention_weight_max": 77.1283950805664,
      "activations/layer7_attention_weight_min": -75.61559295654297,
      "activations/layer8_attention_weight_max": 44.13050079345703,
      "activations/layer8_attention_weight_min": -46.548316955566406,
      "activations/layer9_attention_weight_max": 46.98597717285156,
      "activations/layer9_attention_weight_min": -47.525108337402344,
      "epoch": 9.52,
      "learning_rate": 8.950492424242423e-05,
      "loss": 2.7965,
      "step": 163850
    },
    {
      "activations/layer0_attention_weight_max": 15.721258163452148,
      "activations/layer0_attention_weight_min": -14.136161804199219,
      "activations/layer10_attention_weight_max": 37.65049362182617,
      "activations/layer10_attention_weight_min": -36.195526123046875,
      "activations/layer11_attention_weight_max": 33.929237365722656,
      "activations/layer11_attention_weight_min": -33.399696350097656,
      "activations/layer12_attention_weight_max": 24.555524826049805,
      "activations/layer12_attention_weight_min": -28.13181495666504,
      "activations/layer13_attention_weight_max": 49.917720794677734,
      "activations/layer13_attention_weight_min": -32.47218322753906,
      "activations/layer14_attention_weight_max": 48.34634780883789,
      "activations/layer14_attention_weight_min": -31.274808883666992,
      "activations/layer15_attention_weight_max": 43.17033767700195,
      "activations/layer15_attention_weight_min": -30.559417724609375,
      "activations/layer16_attention_weight_max": 38.9039192199707,
      "activations/layer16_attention_weight_min": -31.497013092041016,
      "activations/layer17_attention_weight_max": 59.7370491027832,
      "activations/layer17_attention_weight_min": -48.081077575683594,
      "activations/layer18_attention_weight_max": 55.494197845458984,
      "activations/layer18_attention_weight_min": -40.38692092895508,
      "activations/layer19_attention_weight_max": 26.520856857299805,
      "activations/layer19_attention_weight_min": -22.408000946044922,
      "activations/layer1_attention_weight_max": 16.84452247619629,
      "activations/layer1_attention_weight_min": -16.252269744873047,
      "activations/layer20_attention_weight_max": 28.168865203857422,
      "activations/layer20_attention_weight_min": -21.795085906982422,
      "activations/layer21_attention_weight_max": 50.535457611083984,
      "activations/layer21_attention_weight_min": -22.605859756469727,
      "activations/layer22_attention_weight_max": 38.46488571166992,
      "activations/layer22_attention_weight_min": -25.1370906829834,
      "activations/layer23_attention_weight_max": 56.57234191894531,
      "activations/layer23_attention_weight_min": -25.39080810546875,
      "activations/layer2_attention_weight_max": 31.753402709960938,
      "activations/layer2_attention_weight_min": -29.607906341552734,
      "activations/layer3_attention_weight_max": 91.1098861694336,
      "activations/layer3_attention_weight_min": -95.59619140625,
      "activations/layer4_attention_weight_max": 103.11988830566406,
      "activations/layer4_attention_weight_min": -99.37196350097656,
      "activations/layer5_attention_weight_max": 78.6788330078125,
      "activations/layer5_attention_weight_min": -79.77957153320312,
      "activations/layer6_attention_weight_max": 58.34153366088867,
      "activations/layer6_attention_weight_min": -53.615882873535156,
      "activations/layer7_attention_weight_max": 86.12849426269531,
      "activations/layer7_attention_weight_min": -76.27511596679688,
      "activations/layer8_attention_weight_max": 45.9122200012207,
      "activations/layer8_attention_weight_min": -47.93379211425781,
      "activations/layer9_attention_weight_max": 62.02467727661133,
      "activations/layer9_attention_weight_min": -48.79027557373047,
      "epoch": 9.52,
      "learning_rate": 8.948598484848484e-05,
      "loss": 2.7881,
      "step": 163900
    },
    {
      "activations/layer0_attention_weight_max": 15.377507209777832,
      "activations/layer0_attention_weight_min": -13.98526668548584,
      "activations/layer10_attention_weight_max": 43.63126754760742,
      "activations/layer10_attention_weight_min": -41.801753997802734,
      "activations/layer11_attention_weight_max": 36.84468078613281,
      "activations/layer11_attention_weight_min": -36.59697723388672,
      "activations/layer12_attention_weight_max": 21.61712074279785,
      "activations/layer12_attention_weight_min": -24.71303939819336,
      "activations/layer13_attention_weight_max": 45.555625915527344,
      "activations/layer13_attention_weight_min": -31.0737361907959,
      "activations/layer14_attention_weight_max": 40.05562210083008,
      "activations/layer14_attention_weight_min": -31.98529052734375,
      "activations/layer15_attention_weight_max": 37.73942565917969,
      "activations/layer15_attention_weight_min": -28.882646560668945,
      "activations/layer16_attention_weight_max": 32.070556640625,
      "activations/layer16_attention_weight_min": -29.38860511779785,
      "activations/layer17_attention_weight_max": 62.68918228149414,
      "activations/layer17_attention_weight_min": -45.89117431640625,
      "activations/layer18_attention_weight_max": 56.34915542602539,
      "activations/layer18_attention_weight_min": -38.190391540527344,
      "activations/layer19_attention_weight_max": 24.106393814086914,
      "activations/layer19_attention_weight_min": -22.05274772644043,
      "activations/layer1_attention_weight_max": 17.4960994720459,
      "activations/layer1_attention_weight_min": -16.177228927612305,
      "activations/layer20_attention_weight_max": 23.56195831298828,
      "activations/layer20_attention_weight_min": -22.852161407470703,
      "activations/layer21_attention_weight_max": 41.62213897705078,
      "activations/layer21_attention_weight_min": -24.6239070892334,
      "activations/layer22_attention_weight_max": 34.854103088378906,
      "activations/layer22_attention_weight_min": -26.02056121826172,
      "activations/layer23_attention_weight_max": 41.838565826416016,
      "activations/layer23_attention_weight_min": -24.283262252807617,
      "activations/layer2_attention_weight_max": 30.538650512695312,
      "activations/layer2_attention_weight_min": -29.216787338256836,
      "activations/layer3_attention_weight_max": 90.22046661376953,
      "activations/layer3_attention_weight_min": -90.88821411132812,
      "activations/layer4_attention_weight_max": 100.37796783447266,
      "activations/layer4_attention_weight_min": -96.28507232666016,
      "activations/layer5_attention_weight_max": 82.85099029541016,
      "activations/layer5_attention_weight_min": -85.62852478027344,
      "activations/layer6_attention_weight_max": 57.82270431518555,
      "activations/layer6_attention_weight_min": -58.19065475463867,
      "activations/layer7_attention_weight_max": 90.04048156738281,
      "activations/layer7_attention_weight_min": -80.370849609375,
      "activations/layer8_attention_weight_max": 50.109535217285156,
      "activations/layer8_attention_weight_min": -50.77669143676758,
      "activations/layer9_attention_weight_max": 56.973480224609375,
      "activations/layer9_attention_weight_min": -50.92699432373047,
      "epoch": 9.53,
      "learning_rate": 8.946704545454544e-05,
      "loss": 2.8104,
      "step": 163950
    },
    {
      "activations/layer0_attention_weight_max": 13.86681079864502,
      "activations/layer0_attention_weight_min": -14.6259126663208,
      "activations/layer10_attention_weight_max": 41.015106201171875,
      "activations/layer10_attention_weight_min": -38.01728057861328,
      "activations/layer11_attention_weight_max": 35.128257751464844,
      "activations/layer11_attention_weight_min": -33.29524230957031,
      "activations/layer12_attention_weight_max": 21.043352127075195,
      "activations/layer12_attention_weight_min": -26.83873748779297,
      "activations/layer13_attention_weight_max": 42.6454963684082,
      "activations/layer13_attention_weight_min": -30.227041244506836,
      "activations/layer14_attention_weight_max": 40.42111587524414,
      "activations/layer14_attention_weight_min": -32.179405212402344,
      "activations/layer15_attention_weight_max": 36.70792007446289,
      "activations/layer15_attention_weight_min": -29.37021255493164,
      "activations/layer16_attention_weight_max": 34.458919525146484,
      "activations/layer16_attention_weight_min": -29.413238525390625,
      "activations/layer17_attention_weight_max": 55.45899963378906,
      "activations/layer17_attention_weight_min": -46.33088302612305,
      "activations/layer18_attention_weight_max": 52.379207611083984,
      "activations/layer18_attention_weight_min": -38.21877670288086,
      "activations/layer19_attention_weight_max": 27.1263484954834,
      "activations/layer19_attention_weight_min": -20.87215805053711,
      "activations/layer1_attention_weight_max": 16.667457580566406,
      "activations/layer1_attention_weight_min": -15.332982063293457,
      "activations/layer20_attention_weight_max": 27.885412216186523,
      "activations/layer20_attention_weight_min": -22.293010711669922,
      "activations/layer21_attention_weight_max": 44.35648727416992,
      "activations/layer21_attention_weight_min": -25.66495704650879,
      "activations/layer22_attention_weight_max": 32.85322189331055,
      "activations/layer22_attention_weight_min": -27.199811935424805,
      "activations/layer23_attention_weight_max": 38.53886795043945,
      "activations/layer23_attention_weight_min": -25.608253479003906,
      "activations/layer2_attention_weight_max": 32.508792877197266,
      "activations/layer2_attention_weight_min": -30.131637573242188,
      "activations/layer3_attention_weight_max": 96.51362609863281,
      "activations/layer3_attention_weight_min": -94.64230346679688,
      "activations/layer4_attention_weight_max": 106.64603424072266,
      "activations/layer4_attention_weight_min": -97.56920623779297,
      "activations/layer5_attention_weight_max": 79.48593139648438,
      "activations/layer5_attention_weight_min": -79.17269897460938,
      "activations/layer6_attention_weight_max": 57.98833465576172,
      "activations/layer6_attention_weight_min": -55.869354248046875,
      "activations/layer7_attention_weight_max": 78.41612243652344,
      "activations/layer7_attention_weight_min": -74.99696350097656,
      "activations/layer8_attention_weight_max": 48.428932189941406,
      "activations/layer8_attention_weight_min": -51.23951721191406,
      "activations/layer9_attention_weight_max": 51.97880172729492,
      "activations/layer9_attention_weight_min": -50.95178985595703,
      "epoch": 9.53,
      "learning_rate": 8.944810606060605e-05,
      "loss": 2.7981,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.4344,
      "eval_samples_per_second": 509.105,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.4344,
      "eval_openwebtext_samples_per_second": 509.105,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_wikitext_loss": 2.982421875,
      "eval_wikitext_ppl": 19.735555861572347,
      "eval_wikitext_runtime": 1.9934,
      "eval_wikitext_samples_per_second": 228.758,
      "step": 164000
    },
    {
      "epoch": 9.53,
      "eval_lambada_loss": 2.748046875,
      "eval_lambada_ppl": 15.612109685326937,
      "eval_lambada_runtime": 9.6139,
      "eval_lambada_samples_per_second": 506.452,
      "step": 164000
    },
    {
      "activations/layer0_attention_weight_max": 15.292431831359863,
      "activations/layer0_attention_weight_min": -14.048606872558594,
      "activations/layer10_attention_weight_max": 35.550655364990234,
      "activations/layer10_attention_weight_min": -35.4102897644043,
      "activations/layer11_attention_weight_max": 30.43000602722168,
      "activations/layer11_attention_weight_min": -33.53276824951172,
      "activations/layer12_attention_weight_max": 18.31916618347168,
      "activations/layer12_attention_weight_min": -26.2852725982666,
      "activations/layer13_attention_weight_max": 37.77806854248047,
      "activations/layer13_attention_weight_min": -29.076923370361328,
      "activations/layer14_attention_weight_max": 38.99796676635742,
      "activations/layer14_attention_weight_min": -29.925146102905273,
      "activations/layer15_attention_weight_max": 34.809410095214844,
      "activations/layer15_attention_weight_min": -27.825952529907227,
      "activations/layer16_attention_weight_max": 32.2323112487793,
      "activations/layer16_attention_weight_min": -28.2679443359375,
      "activations/layer17_attention_weight_max": 55.2143669128418,
      "activations/layer17_attention_weight_min": -43.183231353759766,
      "activations/layer18_attention_weight_max": 48.2108039855957,
      "activations/layer18_attention_weight_min": -38.88593292236328,
      "activations/layer19_attention_weight_max": 23.212305068969727,
      "activations/layer19_attention_weight_min": -20.838298797607422,
      "activations/layer1_attention_weight_max": 16.286561965942383,
      "activations/layer1_attention_weight_min": -13.600383758544922,
      "activations/layer20_attention_weight_max": 24.627973556518555,
      "activations/layer20_attention_weight_min": -20.63286018371582,
      "activations/layer21_attention_weight_max": 40.549373626708984,
      "activations/layer21_attention_weight_min": -21.679370880126953,
      "activations/layer22_attention_weight_max": 31.659563064575195,
      "activations/layer22_attention_weight_min": -24.35818099975586,
      "activations/layer23_attention_weight_max": 39.961219787597656,
      "activations/layer23_attention_weight_min": -24.063034057617188,
      "activations/layer2_attention_weight_max": 32.45567321777344,
      "activations/layer2_attention_weight_min": -31.52703094482422,
      "activations/layer3_attention_weight_max": 92.47035217285156,
      "activations/layer3_attention_weight_min": -97.17105102539062,
      "activations/layer4_attention_weight_max": 104.59698486328125,
      "activations/layer4_attention_weight_min": -99.08269500732422,
      "activations/layer5_attention_weight_max": 84.9432373046875,
      "activations/layer5_attention_weight_min": -83.07301330566406,
      "activations/layer6_attention_weight_max": 56.83604049682617,
      "activations/layer6_attention_weight_min": -57.45228958129883,
      "activations/layer7_attention_weight_max": 85.93682861328125,
      "activations/layer7_attention_weight_min": -76.2631607055664,
      "activations/layer8_attention_weight_max": 47.61623764038086,
      "activations/layer8_attention_weight_min": -51.15678024291992,
      "activations/layer9_attention_weight_max": 50.565895080566406,
      "activations/layer9_attention_weight_min": -51.92463302612305,
      "epoch": 9.53,
      "learning_rate": 8.942916666666666e-05,
      "loss": 2.8031,
      "step": 164050
    },
    {
      "activations/layer0_attention_weight_max": 15.457563400268555,
      "activations/layer0_attention_weight_min": -14.075630187988281,
      "activations/layer10_attention_weight_max": 40.952632904052734,
      "activations/layer10_attention_weight_min": -38.062740325927734,
      "activations/layer11_attention_weight_max": 33.30029296875,
      "activations/layer11_attention_weight_min": -32.80381393432617,
      "activations/layer12_attention_weight_max": 20.2834415435791,
      "activations/layer12_attention_weight_min": -28.593442916870117,
      "activations/layer13_attention_weight_max": 48.7643928527832,
      "activations/layer13_attention_weight_min": -35.05186462402344,
      "activations/layer14_attention_weight_max": 38.224449157714844,
      "activations/layer14_attention_weight_min": -29.524333953857422,
      "activations/layer15_attention_weight_max": 33.06820297241211,
      "activations/layer15_attention_weight_min": -29.698165893554688,
      "activations/layer16_attention_weight_max": 31.052600860595703,
      "activations/layer16_attention_weight_min": -27.107954025268555,
      "activations/layer17_attention_weight_max": 56.33573913574219,
      "activations/layer17_attention_weight_min": -46.455318450927734,
      "activations/layer18_attention_weight_max": 47.89011764526367,
      "activations/layer18_attention_weight_min": -39.7972297668457,
      "activations/layer19_attention_weight_max": 22.084314346313477,
      "activations/layer19_attention_weight_min": -22.080509185791016,
      "activations/layer1_attention_weight_max": 17.200937271118164,
      "activations/layer1_attention_weight_min": -15.747574806213379,
      "activations/layer20_attention_weight_max": 23.455114364624023,
      "activations/layer20_attention_weight_min": -21.39442253112793,
      "activations/layer21_attention_weight_max": 33.130821228027344,
      "activations/layer21_attention_weight_min": -22.83860969543457,
      "activations/layer22_attention_weight_max": 32.92856979370117,
      "activations/layer22_attention_weight_min": -26.01117515563965,
      "activations/layer23_attention_weight_max": 36.895263671875,
      "activations/layer23_attention_weight_min": -23.543027877807617,
      "activations/layer2_attention_weight_max": 33.39986038208008,
      "activations/layer2_attention_weight_min": -31.126262664794922,
      "activations/layer3_attention_weight_max": 96.81390380859375,
      "activations/layer3_attention_weight_min": -94.05984497070312,
      "activations/layer4_attention_weight_max": 102.18490600585938,
      "activations/layer4_attention_weight_min": -97.93701934814453,
      "activations/layer5_attention_weight_max": 79.81159973144531,
      "activations/layer5_attention_weight_min": -79.27581787109375,
      "activations/layer6_attention_weight_max": 54.19607162475586,
      "activations/layer6_attention_weight_min": -52.76072311401367,
      "activations/layer7_attention_weight_max": 78.71349334716797,
      "activations/layer7_attention_weight_min": -78.36165618896484,
      "activations/layer8_attention_weight_max": 50.86655807495117,
      "activations/layer8_attention_weight_min": -48.71565246582031,
      "activations/layer9_attention_weight_max": 52.553829193115234,
      "activations/layer9_attention_weight_min": -49.23234176635742,
      "epoch": 9.54,
      "learning_rate": 8.941022727272726e-05,
      "loss": 2.8003,
      "step": 164100
    },
    {
      "activations/layer0_attention_weight_max": 15.101484298706055,
      "activations/layer0_attention_weight_min": -14.453472137451172,
      "activations/layer10_attention_weight_max": 39.364158630371094,
      "activations/layer10_attention_weight_min": -39.23836898803711,
      "activations/layer11_attention_weight_max": 33.07120132446289,
      "activations/layer11_attention_weight_min": -32.41001892089844,
      "activations/layer12_attention_weight_max": 19.98236846923828,
      "activations/layer12_attention_weight_min": -26.159589767456055,
      "activations/layer13_attention_weight_max": 46.10267639160156,
      "activations/layer13_attention_weight_min": -31.684307098388672,
      "activations/layer14_attention_weight_max": 36.76701736450195,
      "activations/layer14_attention_weight_min": -31.29589080810547,
      "activations/layer15_attention_weight_max": 35.32965850830078,
      "activations/layer15_attention_weight_min": -29.916236877441406,
      "activations/layer16_attention_weight_max": 31.18192481994629,
      "activations/layer16_attention_weight_min": -29.489572525024414,
      "activations/layer17_attention_weight_max": 54.684364318847656,
      "activations/layer17_attention_weight_min": -42.24324417114258,
      "activations/layer18_attention_weight_max": 48.98218536376953,
      "activations/layer18_attention_weight_min": -38.285888671875,
      "activations/layer19_attention_weight_max": 25.42949867248535,
      "activations/layer19_attention_weight_min": -20.889936447143555,
      "activations/layer1_attention_weight_max": 16.903976440429688,
      "activations/layer1_attention_weight_min": -14.944507598876953,
      "activations/layer20_attention_weight_max": 23.027935028076172,
      "activations/layer20_attention_weight_min": -21.627214431762695,
      "activations/layer21_attention_weight_max": 41.83621597290039,
      "activations/layer21_attention_weight_min": -25.830259323120117,
      "activations/layer22_attention_weight_max": 34.85218811035156,
      "activations/layer22_attention_weight_min": -27.96146011352539,
      "activations/layer23_attention_weight_max": 36.73170471191406,
      "activations/layer23_attention_weight_min": -24.92365264892578,
      "activations/layer2_attention_weight_max": 33.29841613769531,
      "activations/layer2_attention_weight_min": -30.642417907714844,
      "activations/layer3_attention_weight_max": 93.71408081054688,
      "activations/layer3_attention_weight_min": -95.5580825805664,
      "activations/layer4_attention_weight_max": 102.33682250976562,
      "activations/layer4_attention_weight_min": -96.46508026123047,
      "activations/layer5_attention_weight_max": 82.9517593383789,
      "activations/layer5_attention_weight_min": -81.68810272216797,
      "activations/layer6_attention_weight_max": 57.28746032714844,
      "activations/layer6_attention_weight_min": -56.30223083496094,
      "activations/layer7_attention_weight_max": 80.2125015258789,
      "activations/layer7_attention_weight_min": -76.24920654296875,
      "activations/layer8_attention_weight_max": 48.50188446044922,
      "activations/layer8_attention_weight_min": -50.597496032714844,
      "activations/layer9_attention_weight_max": 52.38877487182617,
      "activations/layer9_attention_weight_min": -52.38859176635742,
      "epoch": 9.54,
      "learning_rate": 8.939128787878788e-05,
      "loss": 2.8033,
      "step": 164150
    },
    {
      "activations/layer0_attention_weight_max": 16.62670135498047,
      "activations/layer0_attention_weight_min": -14.083914756774902,
      "activations/layer10_attention_weight_max": 34.2102165222168,
      "activations/layer10_attention_weight_min": -34.57307815551758,
      "activations/layer11_attention_weight_max": 31.04671287536621,
      "activations/layer11_attention_weight_min": -32.9567985534668,
      "activations/layer12_attention_weight_max": 19.425678253173828,
      "activations/layer12_attention_weight_min": -26.976295471191406,
      "activations/layer13_attention_weight_max": 35.783329010009766,
      "activations/layer13_attention_weight_min": -29.55113983154297,
      "activations/layer14_attention_weight_max": 35.78025436401367,
      "activations/layer14_attention_weight_min": -29.372352600097656,
      "activations/layer15_attention_weight_max": 37.02106475830078,
      "activations/layer15_attention_weight_min": -30.22430419921875,
      "activations/layer16_attention_weight_max": 32.87100601196289,
      "activations/layer16_attention_weight_min": -28.294879913330078,
      "activations/layer17_attention_weight_max": 56.51829147338867,
      "activations/layer17_attention_weight_min": -44.41277313232422,
      "activations/layer18_attention_weight_max": 49.98344421386719,
      "activations/layer18_attention_weight_min": -37.89605712890625,
      "activations/layer19_attention_weight_max": 23.82131004333496,
      "activations/layer19_attention_weight_min": -21.070165634155273,
      "activations/layer1_attention_weight_max": 17.90283203125,
      "activations/layer1_attention_weight_min": -17.084943771362305,
      "activations/layer20_attention_weight_max": 22.681785583496094,
      "activations/layer20_attention_weight_min": -21.853120803833008,
      "activations/layer21_attention_weight_max": 36.76384353637695,
      "activations/layer21_attention_weight_min": -23.50851821899414,
      "activations/layer22_attention_weight_max": 32.93626403808594,
      "activations/layer22_attention_weight_min": -24.787689208984375,
      "activations/layer23_attention_weight_max": 39.080963134765625,
      "activations/layer23_attention_weight_min": -22.691120147705078,
      "activations/layer2_attention_weight_max": 33.61996841430664,
      "activations/layer2_attention_weight_min": -33.135101318359375,
      "activations/layer3_attention_weight_max": 94.53189086914062,
      "activations/layer3_attention_weight_min": -96.12857055664062,
      "activations/layer4_attention_weight_max": 105.14705657958984,
      "activations/layer4_attention_weight_min": -97.82844543457031,
      "activations/layer5_attention_weight_max": 81.72488403320312,
      "activations/layer5_attention_weight_min": -81.53787231445312,
      "activations/layer6_attention_weight_max": 57.72157669067383,
      "activations/layer6_attention_weight_min": -54.3209114074707,
      "activations/layer7_attention_weight_max": 80.54008483886719,
      "activations/layer7_attention_weight_min": -74.99833679199219,
      "activations/layer8_attention_weight_max": 44.18107986450195,
      "activations/layer8_attention_weight_min": -46.91205978393555,
      "activations/layer9_attention_weight_max": 48.126224517822266,
      "activations/layer9_attention_weight_min": -51.15481185913086,
      "epoch": 9.54,
      "learning_rate": 8.937234848484848e-05,
      "loss": 2.8054,
      "step": 164200
    },
    {
      "activations/layer0_attention_weight_max": 14.79627513885498,
      "activations/layer0_attention_weight_min": -14.120444297790527,
      "activations/layer10_attention_weight_max": 39.61919021606445,
      "activations/layer10_attention_weight_min": -34.977230072021484,
      "activations/layer11_attention_weight_max": 31.57046127319336,
      "activations/layer11_attention_weight_min": -32.12232208251953,
      "activations/layer12_attention_weight_max": 16.833295822143555,
      "activations/layer12_attention_weight_min": -25.062475204467773,
      "activations/layer13_attention_weight_max": 40.4887580871582,
      "activations/layer13_attention_weight_min": -31.862234115600586,
      "activations/layer14_attention_weight_max": 35.1542854309082,
      "activations/layer14_attention_weight_min": -30.388423919677734,
      "activations/layer15_attention_weight_max": 30.66189193725586,
      "activations/layer15_attention_weight_min": -29.840730667114258,
      "activations/layer16_attention_weight_max": 29.312397003173828,
      "activations/layer16_attention_weight_min": -29.41150665283203,
      "activations/layer17_attention_weight_max": 49.25593566894531,
      "activations/layer17_attention_weight_min": -45.08052444458008,
      "activations/layer18_attention_weight_max": 43.17551040649414,
      "activations/layer18_attention_weight_min": -37.686805725097656,
      "activations/layer19_attention_weight_max": 20.285356521606445,
      "activations/layer19_attention_weight_min": -21.7713680267334,
      "activations/layer1_attention_weight_max": 16.711490631103516,
      "activations/layer1_attention_weight_min": -17.17650032043457,
      "activations/layer20_attention_weight_max": 21.30780601501465,
      "activations/layer20_attention_weight_min": -23.464908599853516,
      "activations/layer21_attention_weight_max": 39.71238327026367,
      "activations/layer21_attention_weight_min": -23.749114990234375,
      "activations/layer22_attention_weight_max": 29.391252517700195,
      "activations/layer22_attention_weight_min": -26.50843048095703,
      "activations/layer23_attention_weight_max": 31.916568756103516,
      "activations/layer23_attention_weight_min": -24.753721237182617,
      "activations/layer2_attention_weight_max": 31.348918914794922,
      "activations/layer2_attention_weight_min": -30.649917602539062,
      "activations/layer3_attention_weight_max": 91.3625717163086,
      "activations/layer3_attention_weight_min": -95.87687683105469,
      "activations/layer4_attention_weight_max": 100.82757568359375,
      "activations/layer4_attention_weight_min": -97.78543853759766,
      "activations/layer5_attention_weight_max": 78.62657165527344,
      "activations/layer5_attention_weight_min": -79.3864517211914,
      "activations/layer6_attention_weight_max": 55.952613830566406,
      "activations/layer6_attention_weight_min": -53.382057189941406,
      "activations/layer7_attention_weight_max": 78.12137603759766,
      "activations/layer7_attention_weight_min": -73.97947692871094,
      "activations/layer8_attention_weight_max": 45.66168975830078,
      "activations/layer8_attention_weight_min": -48.24050521850586,
      "activations/layer9_attention_weight_max": 49.88642501831055,
      "activations/layer9_attention_weight_min": -50.34257125854492,
      "epoch": 9.54,
      "learning_rate": 8.935340909090908e-05,
      "loss": 2.7932,
      "step": 164250
    },
    {
      "activations/layer0_attention_weight_max": 14.592880249023438,
      "activations/layer0_attention_weight_min": -13.945277214050293,
      "activations/layer10_attention_weight_max": 41.54069900512695,
      "activations/layer10_attention_weight_min": -39.444156646728516,
      "activations/layer11_attention_weight_max": 35.75172805786133,
      "activations/layer11_attention_weight_min": -33.8792610168457,
      "activations/layer12_attention_weight_max": 20.15182113647461,
      "activations/layer12_attention_weight_min": -26.71857261657715,
      "activations/layer13_attention_weight_max": 39.502098083496094,
      "activations/layer13_attention_weight_min": -32.12274932861328,
      "activations/layer14_attention_weight_max": 38.63056182861328,
      "activations/layer14_attention_weight_min": -31.853591918945312,
      "activations/layer15_attention_weight_max": 34.660640716552734,
      "activations/layer15_attention_weight_min": -30.58958625793457,
      "activations/layer16_attention_weight_max": 34.46402359008789,
      "activations/layer16_attention_weight_min": -30.060039520263672,
      "activations/layer17_attention_weight_max": 55.34798049926758,
      "activations/layer17_attention_weight_min": -44.230438232421875,
      "activations/layer18_attention_weight_max": 55.470184326171875,
      "activations/layer18_attention_weight_min": -38.793060302734375,
      "activations/layer19_attention_weight_max": 22.34959602355957,
      "activations/layer19_attention_weight_min": -22.863285064697266,
      "activations/layer1_attention_weight_max": 17.182018280029297,
      "activations/layer1_attention_weight_min": -14.840269088745117,
      "activations/layer20_attention_weight_max": 24.180070877075195,
      "activations/layer20_attention_weight_min": -22.54277801513672,
      "activations/layer21_attention_weight_max": 37.92719268798828,
      "activations/layer21_attention_weight_min": -28.24759292602539,
      "activations/layer22_attention_weight_max": 35.44813919067383,
      "activations/layer22_attention_weight_min": -27.461912155151367,
      "activations/layer23_attention_weight_max": 40.27923583984375,
      "activations/layer23_attention_weight_min": -22.957487106323242,
      "activations/layer2_attention_weight_max": 31.112396240234375,
      "activations/layer2_attention_weight_min": -29.092222213745117,
      "activations/layer3_attention_weight_max": 89.64505004882812,
      "activations/layer3_attention_weight_min": -92.39421081542969,
      "activations/layer4_attention_weight_max": 100.8089599609375,
      "activations/layer4_attention_weight_min": -97.99635314941406,
      "activations/layer5_attention_weight_max": 80.00938415527344,
      "activations/layer5_attention_weight_min": -77.1632080078125,
      "activations/layer6_attention_weight_max": 55.690704345703125,
      "activations/layer6_attention_weight_min": -54.73111343383789,
      "activations/layer7_attention_weight_max": 80.17696380615234,
      "activations/layer7_attention_weight_min": -78.02287292480469,
      "activations/layer8_attention_weight_max": 48.554046630859375,
      "activations/layer8_attention_weight_min": -50.0653076171875,
      "activations/layer9_attention_weight_max": 51.58136749267578,
      "activations/layer9_attention_weight_min": -50.47138595581055,
      "epoch": 9.55,
      "learning_rate": 8.93344696969697e-05,
      "loss": 2.8093,
      "step": 164300
    },
    {
      "activations/layer0_attention_weight_max": 15.100407600402832,
      "activations/layer0_attention_weight_min": -14.722935676574707,
      "activations/layer10_attention_weight_max": 38.22956085205078,
      "activations/layer10_attention_weight_min": -34.965370178222656,
      "activations/layer11_attention_weight_max": 32.61619186401367,
      "activations/layer11_attention_weight_min": -31.929746627807617,
      "activations/layer12_attention_weight_max": 21.266326904296875,
      "activations/layer12_attention_weight_min": -24.1512393951416,
      "activations/layer13_attention_weight_max": 49.28287124633789,
      "activations/layer13_attention_weight_min": -34.347442626953125,
      "activations/layer14_attention_weight_max": 39.88511276245117,
      "activations/layer14_attention_weight_min": -31.166507720947266,
      "activations/layer15_attention_weight_max": 39.82376480102539,
      "activations/layer15_attention_weight_min": -28.38360023498535,
      "activations/layer16_attention_weight_max": 34.951698303222656,
      "activations/layer16_attention_weight_min": -29.14728355407715,
      "activations/layer17_attention_weight_max": 52.89415740966797,
      "activations/layer17_attention_weight_min": -44.61481475830078,
      "activations/layer18_attention_weight_max": 47.810298919677734,
      "activations/layer18_attention_weight_min": -39.74302291870117,
      "activations/layer19_attention_weight_max": 25.382497787475586,
      "activations/layer19_attention_weight_min": -22.325414657592773,
      "activations/layer1_attention_weight_max": 16.710628509521484,
      "activations/layer1_attention_weight_min": -16.5249080657959,
      "activations/layer20_attention_weight_max": 26.981332778930664,
      "activations/layer20_attention_weight_min": -21.436145782470703,
      "activations/layer21_attention_weight_max": 41.96352005004883,
      "activations/layer21_attention_weight_min": -24.07775115966797,
      "activations/layer22_attention_weight_max": 35.61121368408203,
      "activations/layer22_attention_weight_min": -25.56338882446289,
      "activations/layer23_attention_weight_max": 42.70931625366211,
      "activations/layer23_attention_weight_min": -26.700956344604492,
      "activations/layer2_attention_weight_max": 31.784414291381836,
      "activations/layer2_attention_weight_min": -30.345176696777344,
      "activations/layer3_attention_weight_max": 91.31507873535156,
      "activations/layer3_attention_weight_min": -87.81787872314453,
      "activations/layer4_attention_weight_max": 101.9922103881836,
      "activations/layer4_attention_weight_min": -96.27525329589844,
      "activations/layer5_attention_weight_max": 76.83053588867188,
      "activations/layer5_attention_weight_min": -79.67353057861328,
      "activations/layer6_attention_weight_max": 54.332401275634766,
      "activations/layer6_attention_weight_min": -56.02082443237305,
      "activations/layer7_attention_weight_max": 82.14051055908203,
      "activations/layer7_attention_weight_min": -75.22808837890625,
      "activations/layer8_attention_weight_max": 44.80959701538086,
      "activations/layer8_attention_weight_min": -45.38992691040039,
      "activations/layer9_attention_weight_max": 50.74563980102539,
      "activations/layer9_attention_weight_min": -51.859527587890625,
      "epoch": 9.55,
      "learning_rate": 8.93155303030303e-05,
      "loss": 2.8012,
      "step": 164350
    },
    {
      "activations/layer0_attention_weight_max": 14.947037696838379,
      "activations/layer0_attention_weight_min": -14.241881370544434,
      "activations/layer10_attention_weight_max": 37.293067932128906,
      "activations/layer10_attention_weight_min": -34.514461517333984,
      "activations/layer11_attention_weight_max": 32.91607666015625,
      "activations/layer11_attention_weight_min": -31.13619613647461,
      "activations/layer12_attention_weight_max": 20.406490325927734,
      "activations/layer12_attention_weight_min": -27.9212703704834,
      "activations/layer13_attention_weight_max": 37.18263244628906,
      "activations/layer13_attention_weight_min": -31.47043800354004,
      "activations/layer14_attention_weight_max": 39.58805847167969,
      "activations/layer14_attention_weight_min": -30.640283584594727,
      "activations/layer15_attention_weight_max": 34.429840087890625,
      "activations/layer15_attention_weight_min": -28.11554718017578,
      "activations/layer16_attention_weight_max": 31.58354949951172,
      "activations/layer16_attention_weight_min": -28.5061092376709,
      "activations/layer17_attention_weight_max": 49.49126052856445,
      "activations/layer17_attention_weight_min": -44.83403396606445,
      "activations/layer18_attention_weight_max": 45.436485290527344,
      "activations/layer18_attention_weight_min": -38.37346649169922,
      "activations/layer19_attention_weight_max": 21.758319854736328,
      "activations/layer19_attention_weight_min": -23.014326095581055,
      "activations/layer1_attention_weight_max": 17.15648651123047,
      "activations/layer1_attention_weight_min": -16.50518226623535,
      "activations/layer20_attention_weight_max": 22.314207077026367,
      "activations/layer20_attention_weight_min": -21.935453414916992,
      "activations/layer21_attention_weight_max": 39.808204650878906,
      "activations/layer21_attention_weight_min": -25.31100845336914,
      "activations/layer22_attention_weight_max": 32.49168014526367,
      "activations/layer22_attention_weight_min": -25.680675506591797,
      "activations/layer23_attention_weight_max": 39.17401123046875,
      "activations/layer23_attention_weight_min": -23.054981231689453,
      "activations/layer2_attention_weight_max": 32.268653869628906,
      "activations/layer2_attention_weight_min": -30.1234188079834,
      "activations/layer3_attention_weight_max": 92.34300994873047,
      "activations/layer3_attention_weight_min": -87.86201477050781,
      "activations/layer4_attention_weight_max": 99.91722106933594,
      "activations/layer4_attention_weight_min": -91.98650360107422,
      "activations/layer5_attention_weight_max": 76.15387725830078,
      "activations/layer5_attention_weight_min": -75.31429290771484,
      "activations/layer6_attention_weight_max": 54.931007385253906,
      "activations/layer6_attention_weight_min": -53.404762268066406,
      "activations/layer7_attention_weight_max": 76.32118225097656,
      "activations/layer7_attention_weight_min": -73.85174560546875,
      "activations/layer8_attention_weight_max": 48.123939514160156,
      "activations/layer8_attention_weight_min": -45.82277297973633,
      "activations/layer9_attention_weight_max": 49.9011116027832,
      "activations/layer9_attention_weight_min": -47.437740325927734,
      "epoch": 9.55,
      "learning_rate": 8.92965909090909e-05,
      "loss": 2.7909,
      "step": 164400
    },
    {
      "activations/layer0_attention_weight_max": 16.023300170898438,
      "activations/layer0_attention_weight_min": -13.931526184082031,
      "activations/layer10_attention_weight_max": 34.87813949584961,
      "activations/layer10_attention_weight_min": -34.31439971923828,
      "activations/layer11_attention_weight_max": 30.440610885620117,
      "activations/layer11_attention_weight_min": -31.84284782409668,
      "activations/layer12_attention_weight_max": 19.982940673828125,
      "activations/layer12_attention_weight_min": -26.91863250732422,
      "activations/layer13_attention_weight_max": 41.808509826660156,
      "activations/layer13_attention_weight_min": -31.47909927368164,
      "activations/layer14_attention_weight_max": 36.93102264404297,
      "activations/layer14_attention_weight_min": -30.446325302124023,
      "activations/layer15_attention_weight_max": 34.737266540527344,
      "activations/layer15_attention_weight_min": -28.286523818969727,
      "activations/layer16_attention_weight_max": 33.587772369384766,
      "activations/layer16_attention_weight_min": -29.072772979736328,
      "activations/layer17_attention_weight_max": 50.84711456298828,
      "activations/layer17_attention_weight_min": -45.534969329833984,
      "activations/layer18_attention_weight_max": 49.08142852783203,
      "activations/layer18_attention_weight_min": -39.38003158569336,
      "activations/layer19_attention_weight_max": 22.027603149414062,
      "activations/layer19_attention_weight_min": -23.113924026489258,
      "activations/layer1_attention_weight_max": 17.106298446655273,
      "activations/layer1_attention_weight_min": -14.909762382507324,
      "activations/layer20_attention_weight_max": 21.931913375854492,
      "activations/layer20_attention_weight_min": -20.25812339782715,
      "activations/layer21_attention_weight_max": 40.07537841796875,
      "activations/layer21_attention_weight_min": -22.93899154663086,
      "activations/layer22_attention_weight_max": 32.698665618896484,
      "activations/layer22_attention_weight_min": -27.887557983398438,
      "activations/layer23_attention_weight_max": 37.98304748535156,
      "activations/layer23_attention_weight_min": -23.499135971069336,
      "activations/layer2_attention_weight_max": 31.483413696289062,
      "activations/layer2_attention_weight_min": -30.366573333740234,
      "activations/layer3_attention_weight_max": 89.02728271484375,
      "activations/layer3_attention_weight_min": -90.22709655761719,
      "activations/layer4_attention_weight_max": 100.28437805175781,
      "activations/layer4_attention_weight_min": -92.77032470703125,
      "activations/layer5_attention_weight_max": 76.5837173461914,
      "activations/layer5_attention_weight_min": -76.51399230957031,
      "activations/layer6_attention_weight_max": 54.30781936645508,
      "activations/layer6_attention_weight_min": -51.1528205871582,
      "activations/layer7_attention_weight_max": 79.88203430175781,
      "activations/layer7_attention_weight_min": -72.46551513671875,
      "activations/layer8_attention_weight_max": 42.96595764160156,
      "activations/layer8_attention_weight_min": -43.770904541015625,
      "activations/layer9_attention_weight_max": 48.82963562011719,
      "activations/layer9_attention_weight_min": -45.98875427246094,
      "epoch": 9.56,
      "learning_rate": 8.927765151515151e-05,
      "loss": 2.8009,
      "step": 164450
    },
    {
      "activations/layer0_attention_weight_max": 15.033814430236816,
      "activations/layer0_attention_weight_min": -14.328377723693848,
      "activations/layer10_attention_weight_max": 34.940486907958984,
      "activations/layer10_attention_weight_min": -34.797637939453125,
      "activations/layer11_attention_weight_max": 31.906597137451172,
      "activations/layer11_attention_weight_min": -31.80989646911621,
      "activations/layer12_attention_weight_max": 20.138486862182617,
      "activations/layer12_attention_weight_min": -24.652246475219727,
      "activations/layer13_attention_weight_max": 50.782474517822266,
      "activations/layer13_attention_weight_min": -37.093746185302734,
      "activations/layer14_attention_weight_max": 40.39030838012695,
      "activations/layer14_attention_weight_min": -30.467512130737305,
      "activations/layer15_attention_weight_max": 34.057411193847656,
      "activations/layer15_attention_weight_min": -30.606216430664062,
      "activations/layer16_attention_weight_max": 30.720842361450195,
      "activations/layer16_attention_weight_min": -28.616939544677734,
      "activations/layer17_attention_weight_max": 50.8804817199707,
      "activations/layer17_attention_weight_min": -44.59125518798828,
      "activations/layer18_attention_weight_max": 46.961517333984375,
      "activations/layer18_attention_weight_min": -37.81330871582031,
      "activations/layer19_attention_weight_max": 22.379566192626953,
      "activations/layer19_attention_weight_min": -23.85051727294922,
      "activations/layer1_attention_weight_max": 17.447649002075195,
      "activations/layer1_attention_weight_min": -14.667104721069336,
      "activations/layer20_attention_weight_max": 22.0543270111084,
      "activations/layer20_attention_weight_min": -24.3856201171875,
      "activations/layer21_attention_weight_max": 42.803253173828125,
      "activations/layer21_attention_weight_min": -24.351177215576172,
      "activations/layer22_attention_weight_max": 30.048887252807617,
      "activations/layer22_attention_weight_min": -26.803510665893555,
      "activations/layer23_attention_weight_max": 36.052894592285156,
      "activations/layer23_attention_weight_min": -26.108840942382812,
      "activations/layer2_attention_weight_max": 34.99066925048828,
      "activations/layer2_attention_weight_min": -31.834529876708984,
      "activations/layer3_attention_weight_max": 94.38356018066406,
      "activations/layer3_attention_weight_min": -93.6702651977539,
      "activations/layer4_attention_weight_max": 104.4865951538086,
      "activations/layer4_attention_weight_min": -94.5076675415039,
      "activations/layer5_attention_weight_max": 78.07542419433594,
      "activations/layer5_attention_weight_min": -79.00080871582031,
      "activations/layer6_attention_weight_max": 55.068809509277344,
      "activations/layer6_attention_weight_min": -54.375423431396484,
      "activations/layer7_attention_weight_max": 78.22905731201172,
      "activations/layer7_attention_weight_min": -75.21736145019531,
      "activations/layer8_attention_weight_max": 43.55085754394531,
      "activations/layer8_attention_weight_min": -45.55973815917969,
      "activations/layer9_attention_weight_max": 46.778053283691406,
      "activations/layer9_attention_weight_min": -47.68317413330078,
      "epoch": 9.56,
      "learning_rate": 8.92587121212121e-05,
      "loss": 2.7969,
      "step": 164500
    },
    {
      "activations/layer0_attention_weight_max": 14.719680786132812,
      "activations/layer0_attention_weight_min": -14.321989059448242,
      "activations/layer10_attention_weight_max": 37.94955062866211,
      "activations/layer10_attention_weight_min": -37.611549377441406,
      "activations/layer11_attention_weight_max": 32.3060188293457,
      "activations/layer11_attention_weight_min": -32.96996307373047,
      "activations/layer12_attention_weight_max": 18.09234046936035,
      "activations/layer12_attention_weight_min": -25.231189727783203,
      "activations/layer13_attention_weight_max": 48.581363677978516,
      "activations/layer13_attention_weight_min": -35.066802978515625,
      "activations/layer14_attention_weight_max": 36.019927978515625,
      "activations/layer14_attention_weight_min": -31.497020721435547,
      "activations/layer15_attention_weight_max": 33.02365493774414,
      "activations/layer15_attention_weight_min": -29.130359649658203,
      "activations/layer16_attention_weight_max": 31.818986892700195,
      "activations/layer16_attention_weight_min": -28.995332717895508,
      "activations/layer17_attention_weight_max": 50.85726547241211,
      "activations/layer17_attention_weight_min": -43.26009750366211,
      "activations/layer18_attention_weight_max": 43.289146423339844,
      "activations/layer18_attention_weight_min": -37.51886749267578,
      "activations/layer19_attention_weight_max": 22.69142723083496,
      "activations/layer19_attention_weight_min": -21.133920669555664,
      "activations/layer1_attention_weight_max": 17.71625518798828,
      "activations/layer1_attention_weight_min": -15.759928703308105,
      "activations/layer20_attention_weight_max": 24.638343811035156,
      "activations/layer20_attention_weight_min": -20.437808990478516,
      "activations/layer21_attention_weight_max": 34.14387130737305,
      "activations/layer21_attention_weight_min": -24.196420669555664,
      "activations/layer22_attention_weight_max": 31.421382904052734,
      "activations/layer22_attention_weight_min": -25.38701057434082,
      "activations/layer23_attention_weight_max": 35.82563781738281,
      "activations/layer23_attention_weight_min": -23.00035858154297,
      "activations/layer2_attention_weight_max": 32.75608444213867,
      "activations/layer2_attention_weight_min": -30.68621826171875,
      "activations/layer3_attention_weight_max": 94.7690658569336,
      "activations/layer3_attention_weight_min": -95.36668395996094,
      "activations/layer4_attention_weight_max": 102.21586608886719,
      "activations/layer4_attention_weight_min": -100.61256408691406,
      "activations/layer5_attention_weight_max": 80.18771362304688,
      "activations/layer5_attention_weight_min": -84.49375915527344,
      "activations/layer6_attention_weight_max": 54.90497970581055,
      "activations/layer6_attention_weight_min": -55.98109817504883,
      "activations/layer7_attention_weight_max": 79.68133544921875,
      "activations/layer7_attention_weight_min": -83.55786895751953,
      "activations/layer8_attention_weight_max": 48.49735641479492,
      "activations/layer8_attention_weight_min": -49.30498123168945,
      "activations/layer9_attention_weight_max": 50.65266418457031,
      "activations/layer9_attention_weight_min": -53.658870697021484,
      "epoch": 9.56,
      "learning_rate": 8.923977272727272e-05,
      "loss": 2.8109,
      "step": 164550
    },
    {
      "activations/layer0_attention_weight_max": 14.608977317810059,
      "activations/layer0_attention_weight_min": -14.514140129089355,
      "activations/layer10_attention_weight_max": 35.884498596191406,
      "activations/layer10_attention_weight_min": -36.77949905395508,
      "activations/layer11_attention_weight_max": 30.86879539489746,
      "activations/layer11_attention_weight_min": -33.220191955566406,
      "activations/layer12_attention_weight_max": 18.970613479614258,
      "activations/layer12_attention_weight_min": -23.62921905517578,
      "activations/layer13_attention_weight_max": 49.23068618774414,
      "activations/layer13_attention_weight_min": -39.58369445800781,
      "activations/layer14_attention_weight_max": 38.916194915771484,
      "activations/layer14_attention_weight_min": -31.391647338867188,
      "activations/layer15_attention_weight_max": 33.76690673828125,
      "activations/layer15_attention_weight_min": -30.094221115112305,
      "activations/layer16_attention_weight_max": 31.30592918395996,
      "activations/layer16_attention_weight_min": -28.964468002319336,
      "activations/layer17_attention_weight_max": 48.802005767822266,
      "activations/layer17_attention_weight_min": -45.96216583251953,
      "activations/layer18_attention_weight_max": 44.61747741699219,
      "activations/layer18_attention_weight_min": -38.695953369140625,
      "activations/layer19_attention_weight_max": 22.229759216308594,
      "activations/layer19_attention_weight_min": -23.838598251342773,
      "activations/layer1_attention_weight_max": 17.644588470458984,
      "activations/layer1_attention_weight_min": -18.149709701538086,
      "activations/layer20_attention_weight_max": 22.898235321044922,
      "activations/layer20_attention_weight_min": -26.678632736206055,
      "activations/layer21_attention_weight_max": 41.80647277832031,
      "activations/layer21_attention_weight_min": -30.10526466369629,
      "activations/layer22_attention_weight_max": 30.514007568359375,
      "activations/layer22_attention_weight_min": -29.08247947692871,
      "activations/layer23_attention_weight_max": 35.898406982421875,
      "activations/layer23_attention_weight_min": -27.82805633544922,
      "activations/layer2_attention_weight_max": 34.13212966918945,
      "activations/layer2_attention_weight_min": -31.30713653564453,
      "activations/layer3_attention_weight_max": 96.25285339355469,
      "activations/layer3_attention_weight_min": -101.91376495361328,
      "activations/layer4_attention_weight_max": 102.29630279541016,
      "activations/layer4_attention_weight_min": -97.7445297241211,
      "activations/layer5_attention_weight_max": 79.26100158691406,
      "activations/layer5_attention_weight_min": -79.5133285522461,
      "activations/layer6_attention_weight_max": 57.6214599609375,
      "activations/layer6_attention_weight_min": -51.40870666503906,
      "activations/layer7_attention_weight_max": 77.1634521484375,
      "activations/layer7_attention_weight_min": -81.3503189086914,
      "activations/layer8_attention_weight_max": 45.10438537597656,
      "activations/layer8_attention_weight_min": -46.3378791809082,
      "activations/layer9_attention_weight_max": 52.128170013427734,
      "activations/layer9_attention_weight_min": -50.21493911743164,
      "epoch": 9.56,
      "learning_rate": 8.922083333333333e-05,
      "loss": 2.7812,
      "step": 164600
    },
    {
      "activations/layer0_attention_weight_max": 14.10597038269043,
      "activations/layer0_attention_weight_min": -13.710554122924805,
      "activations/layer10_attention_weight_max": 40.36159133911133,
      "activations/layer10_attention_weight_min": -37.35670852661133,
      "activations/layer11_attention_weight_max": 36.58555221557617,
      "activations/layer11_attention_weight_min": -33.568965911865234,
      "activations/layer12_attention_weight_max": 18.294559478759766,
      "activations/layer12_attention_weight_min": -22.54895782470703,
      "activations/layer13_attention_weight_max": 50.33487319946289,
      "activations/layer13_attention_weight_min": -37.052696228027344,
      "activations/layer14_attention_weight_max": 76.01498413085938,
      "activations/layer14_attention_weight_min": -37.13136672973633,
      "activations/layer15_attention_weight_max": 61.0632209777832,
      "activations/layer15_attention_weight_min": -33.39002990722656,
      "activations/layer16_attention_weight_max": 34.41221237182617,
      "activations/layer16_attention_weight_min": -28.72180938720703,
      "activations/layer17_attention_weight_max": 59.6400032043457,
      "activations/layer17_attention_weight_min": -44.78848648071289,
      "activations/layer18_attention_weight_max": 47.277252197265625,
      "activations/layer18_attention_weight_min": -41.7757682800293,
      "activations/layer19_attention_weight_max": 26.376220703125,
      "activations/layer19_attention_weight_min": -23.55662727355957,
      "activations/layer1_attention_weight_max": 17.97627067565918,
      "activations/layer1_attention_weight_min": -14.00292682647705,
      "activations/layer20_attention_weight_max": 26.968664169311523,
      "activations/layer20_attention_weight_min": -24.045610427856445,
      "activations/layer21_attention_weight_max": 53.246028900146484,
      "activations/layer21_attention_weight_min": -24.833158493041992,
      "activations/layer22_attention_weight_max": 33.66077423095703,
      "activations/layer22_attention_weight_min": -27.43453025817871,
      "activations/layer23_attention_weight_max": 35.04419708251953,
      "activations/layer23_attention_weight_min": -25.856948852539062,
      "activations/layer2_attention_weight_max": 31.49756622314453,
      "activations/layer2_attention_weight_min": -30.15766716003418,
      "activations/layer3_attention_weight_max": 93.21537780761719,
      "activations/layer3_attention_weight_min": -97.54486846923828,
      "activations/layer4_attention_weight_max": 101.99211883544922,
      "activations/layer4_attention_weight_min": -100.64665985107422,
      "activations/layer5_attention_weight_max": 81.34784698486328,
      "activations/layer5_attention_weight_min": -81.84585571289062,
      "activations/layer6_attention_weight_max": 55.73003387451172,
      "activations/layer6_attention_weight_min": -53.354454040527344,
      "activations/layer7_attention_weight_max": 79.59298706054688,
      "activations/layer7_attention_weight_min": -77.67742156982422,
      "activations/layer8_attention_weight_max": 53.684078216552734,
      "activations/layer8_attention_weight_min": -48.25465774536133,
      "activations/layer9_attention_weight_max": 60.19985580444336,
      "activations/layer9_attention_weight_min": -52.96143341064453,
      "epoch": 9.57,
      "learning_rate": 8.920227272727272e-05,
      "loss": 2.7929,
      "step": 164650
    },
    {
      "activations/layer0_attention_weight_max": 16.21001434326172,
      "activations/layer0_attention_weight_min": -13.149354934692383,
      "activations/layer10_attention_weight_max": 38.46236038208008,
      "activations/layer10_attention_weight_min": -36.76591873168945,
      "activations/layer11_attention_weight_max": 31.57863998413086,
      "activations/layer11_attention_weight_min": -32.195068359375,
      "activations/layer12_attention_weight_max": 19.61826515197754,
      "activations/layer12_attention_weight_min": -23.014469146728516,
      "activations/layer13_attention_weight_max": 39.086029052734375,
      "activations/layer13_attention_weight_min": -28.95354461669922,
      "activations/layer14_attention_weight_max": 35.75797653198242,
      "activations/layer14_attention_weight_min": -31.49606704711914,
      "activations/layer15_attention_weight_max": 34.235313415527344,
      "activations/layer15_attention_weight_min": -28.341251373291016,
      "activations/layer16_attention_weight_max": 28.75399398803711,
      "activations/layer16_attention_weight_min": -27.73999786376953,
      "activations/layer17_attention_weight_max": 51.11235427856445,
      "activations/layer17_attention_weight_min": -43.6507453918457,
      "activations/layer18_attention_weight_max": 50.26949691772461,
      "activations/layer18_attention_weight_min": -40.49327087402344,
      "activations/layer19_attention_weight_max": 21.81683349609375,
      "activations/layer19_attention_weight_min": -21.40681266784668,
      "activations/layer1_attention_weight_max": 16.182334899902344,
      "activations/layer1_attention_weight_min": -14.625456809997559,
      "activations/layer20_attention_weight_max": 22.143991470336914,
      "activations/layer20_attention_weight_min": -23.361047744750977,
      "activations/layer21_attention_weight_max": 32.70857238769531,
      "activations/layer21_attention_weight_min": -23.630266189575195,
      "activations/layer22_attention_weight_max": 31.407245635986328,
      "activations/layer22_attention_weight_min": -25.316789627075195,
      "activations/layer23_attention_weight_max": 35.12359619140625,
      "activations/layer23_attention_weight_min": -20.852127075195312,
      "activations/layer2_attention_weight_max": 31.813297271728516,
      "activations/layer2_attention_weight_min": -28.892698287963867,
      "activations/layer3_attention_weight_max": 85.55387115478516,
      "activations/layer3_attention_weight_min": -91.95707702636719,
      "activations/layer4_attention_weight_max": 95.47573852539062,
      "activations/layer4_attention_weight_min": -92.18233489990234,
      "activations/layer5_attention_weight_max": 76.88053131103516,
      "activations/layer5_attention_weight_min": -77.894775390625,
      "activations/layer6_attention_weight_max": 52.70729446411133,
      "activations/layer6_attention_weight_min": -54.0480842590332,
      "activations/layer7_attention_weight_max": 75.8532485961914,
      "activations/layer7_attention_weight_min": -76.76837921142578,
      "activations/layer8_attention_weight_max": 49.60234832763672,
      "activations/layer8_attention_weight_min": -50.78596115112305,
      "activations/layer9_attention_weight_max": 55.363128662109375,
      "activations/layer9_attention_weight_min": -51.31031036376953,
      "epoch": 9.57,
      "learning_rate": 8.918333333333332e-05,
      "loss": 2.7863,
      "step": 164700
    },
    {
      "activations/layer0_attention_weight_max": 16.108854293823242,
      "activations/layer0_attention_weight_min": -14.011958122253418,
      "activations/layer10_attention_weight_max": 38.6476936340332,
      "activations/layer10_attention_weight_min": -36.49073028564453,
      "activations/layer11_attention_weight_max": 32.973777770996094,
      "activations/layer11_attention_weight_min": -31.80233383178711,
      "activations/layer12_attention_weight_max": 19.053049087524414,
      "activations/layer12_attention_weight_min": -25.827665328979492,
      "activations/layer13_attention_weight_max": 44.759220123291016,
      "activations/layer13_attention_weight_min": -32.779151916503906,
      "activations/layer14_attention_weight_max": 39.20941925048828,
      "activations/layer14_attention_weight_min": -30.9510555267334,
      "activations/layer15_attention_weight_max": 34.40118408203125,
      "activations/layer15_attention_weight_min": -30.427400588989258,
      "activations/layer16_attention_weight_max": 29.697731018066406,
      "activations/layer16_attention_weight_min": -29.047060012817383,
      "activations/layer17_attention_weight_max": 53.189903259277344,
      "activations/layer17_attention_weight_min": -44.73683166503906,
      "activations/layer18_attention_weight_max": 45.03034591674805,
      "activations/layer18_attention_weight_min": -38.23322677612305,
      "activations/layer19_attention_weight_max": 21.807462692260742,
      "activations/layer19_attention_weight_min": -22.181894302368164,
      "activations/layer1_attention_weight_max": 16.570175170898438,
      "activations/layer1_attention_weight_min": -15.03148078918457,
      "activations/layer20_attention_weight_max": 23.50309944152832,
      "activations/layer20_attention_weight_min": -24.188518524169922,
      "activations/layer21_attention_weight_max": 39.172889709472656,
      "activations/layer21_attention_weight_min": -22.174198150634766,
      "activations/layer22_attention_weight_max": 30.958101272583008,
      "activations/layer22_attention_weight_min": -26.28797149658203,
      "activations/layer23_attention_weight_max": 37.262245178222656,
      "activations/layer23_attention_weight_min": -22.343307495117188,
      "activations/layer2_attention_weight_max": 31.673320770263672,
      "activations/layer2_attention_weight_min": -29.328184127807617,
      "activations/layer3_attention_weight_max": 96.88057708740234,
      "activations/layer3_attention_weight_min": -101.45735931396484,
      "activations/layer4_attention_weight_max": 105.74468231201172,
      "activations/layer4_attention_weight_min": -99.85686492919922,
      "activations/layer5_attention_weight_max": 78.20782470703125,
      "activations/layer5_attention_weight_min": -85.43699645996094,
      "activations/layer6_attention_weight_max": 53.03535842895508,
      "activations/layer6_attention_weight_min": -52.939151763916016,
      "activations/layer7_attention_weight_max": 79.51014709472656,
      "activations/layer7_attention_weight_min": -73.52344512939453,
      "activations/layer8_attention_weight_max": 44.71466827392578,
      "activations/layer8_attention_weight_min": -47.19153594970703,
      "activations/layer9_attention_weight_max": 48.03563690185547,
      "activations/layer9_attention_weight_min": -49.7717399597168,
      "epoch": 9.57,
      "learning_rate": 8.916439393939393e-05,
      "loss": 2.7928,
      "step": 164750
    },
    {
      "activations/layer0_attention_weight_max": 15.661346435546875,
      "activations/layer0_attention_weight_min": -13.537490844726562,
      "activations/layer10_attention_weight_max": 38.680763244628906,
      "activations/layer10_attention_weight_min": -35.69521713256836,
      "activations/layer11_attention_weight_max": 33.896583557128906,
      "activations/layer11_attention_weight_min": -32.985877990722656,
      "activations/layer12_attention_weight_max": 19.626482009887695,
      "activations/layer12_attention_weight_min": -25.71855926513672,
      "activations/layer13_attention_weight_max": 40.57830047607422,
      "activations/layer13_attention_weight_min": -32.24552536010742,
      "activations/layer14_attention_weight_max": 43.83781814575195,
      "activations/layer14_attention_weight_min": -30.30587387084961,
      "activations/layer15_attention_weight_max": 37.264259338378906,
      "activations/layer15_attention_weight_min": -27.433000564575195,
      "activations/layer16_attention_weight_max": 32.94029998779297,
      "activations/layer16_attention_weight_min": -31.032466888427734,
      "activations/layer17_attention_weight_max": 51.80630874633789,
      "activations/layer17_attention_weight_min": -45.03219985961914,
      "activations/layer18_attention_weight_max": 48.28897476196289,
      "activations/layer18_attention_weight_min": -36.38441848754883,
      "activations/layer19_attention_weight_max": 23.40981101989746,
      "activations/layer19_attention_weight_min": -21.324600219726562,
      "activations/layer1_attention_weight_max": 16.75337791442871,
      "activations/layer1_attention_weight_min": -15.912470817565918,
      "activations/layer20_attention_weight_max": 22.267070770263672,
      "activations/layer20_attention_weight_min": -20.092378616333008,
      "activations/layer21_attention_weight_max": 42.87042236328125,
      "activations/layer21_attention_weight_min": -24.816368103027344,
      "activations/layer22_attention_weight_max": 31.67098617553711,
      "activations/layer22_attention_weight_min": -25.092548370361328,
      "activations/layer23_attention_weight_max": 37.707401275634766,
      "activations/layer23_attention_weight_min": -23.05712890625,
      "activations/layer2_attention_weight_max": 32.155113220214844,
      "activations/layer2_attention_weight_min": -30.500904083251953,
      "activations/layer3_attention_weight_max": 96.50537872314453,
      "activations/layer3_attention_weight_min": -98.22234344482422,
      "activations/layer4_attention_weight_max": 102.61541748046875,
      "activations/layer4_attention_weight_min": -98.5346908569336,
      "activations/layer5_attention_weight_max": 79.75996398925781,
      "activations/layer5_attention_weight_min": -82.19520568847656,
      "activations/layer6_attention_weight_max": 56.57324981689453,
      "activations/layer6_attention_weight_min": -54.190799713134766,
      "activations/layer7_attention_weight_max": 80.58428955078125,
      "activations/layer7_attention_weight_min": -75.12468719482422,
      "activations/layer8_attention_weight_max": 43.56698989868164,
      "activations/layer8_attention_weight_min": -43.655094146728516,
      "activations/layer9_attention_weight_max": 54.01057815551758,
      "activations/layer9_attention_weight_min": -49.1900749206543,
      "epoch": 9.58,
      "learning_rate": 8.914545454545454e-05,
      "loss": 2.7996,
      "step": 164800
    },
    {
      "activations/layer0_attention_weight_max": 17.628259658813477,
      "activations/layer0_attention_weight_min": -13.464245796203613,
      "activations/layer10_attention_weight_max": 34.600677490234375,
      "activations/layer10_attention_weight_min": -33.9785270690918,
      "activations/layer11_attention_weight_max": 30.4925594329834,
      "activations/layer11_attention_weight_min": -29.917768478393555,
      "activations/layer12_attention_weight_max": 19.578039169311523,
      "activations/layer12_attention_weight_min": -26.397342681884766,
      "activations/layer13_attention_weight_max": 39.038360595703125,
      "activations/layer13_attention_weight_min": -29.37162971496582,
      "activations/layer14_attention_weight_max": 38.25752258300781,
      "activations/layer14_attention_weight_min": -32.767677307128906,
      "activations/layer15_attention_weight_max": 35.75251388549805,
      "activations/layer15_attention_weight_min": -31.59810447692871,
      "activations/layer16_attention_weight_max": 32.507049560546875,
      "activations/layer16_attention_weight_min": -28.307767868041992,
      "activations/layer17_attention_weight_max": 52.18269729614258,
      "activations/layer17_attention_weight_min": -42.171470642089844,
      "activations/layer18_attention_weight_max": 49.53704833984375,
      "activations/layer18_attention_weight_min": -36.88652801513672,
      "activations/layer19_attention_weight_max": 24.486448287963867,
      "activations/layer19_attention_weight_min": -22.414913177490234,
      "activations/layer1_attention_weight_max": 16.993120193481445,
      "activations/layer1_attention_weight_min": -15.630220413208008,
      "activations/layer20_attention_weight_max": 23.773963928222656,
      "activations/layer20_attention_weight_min": -19.902597427368164,
      "activations/layer21_attention_weight_max": 39.07398986816406,
      "activations/layer21_attention_weight_min": -22.083677291870117,
      "activations/layer22_attention_weight_max": 33.14951705932617,
      "activations/layer22_attention_weight_min": -26.179555892944336,
      "activations/layer23_attention_weight_max": 40.83658218383789,
      "activations/layer23_attention_weight_min": -23.532773971557617,
      "activations/layer2_attention_weight_max": 30.64719009399414,
      "activations/layer2_attention_weight_min": -31.595718383789062,
      "activations/layer3_attention_weight_max": 88.40938568115234,
      "activations/layer3_attention_weight_min": -95.4256820678711,
      "activations/layer4_attention_weight_max": 96.60635375976562,
      "activations/layer4_attention_weight_min": -93.53317260742188,
      "activations/layer5_attention_weight_max": 78.02690124511719,
      "activations/layer5_attention_weight_min": -77.27767944335938,
      "activations/layer6_attention_weight_max": 51.72568130493164,
      "activations/layer6_attention_weight_min": -50.13695526123047,
      "activations/layer7_attention_weight_max": 71.94957733154297,
      "activations/layer7_attention_weight_min": -76.09264373779297,
      "activations/layer8_attention_weight_max": 47.18739700317383,
      "activations/layer8_attention_weight_min": -49.14461135864258,
      "activations/layer9_attention_weight_max": 47.379966735839844,
      "activations/layer9_attention_weight_min": -45.37788772583008,
      "epoch": 9.58,
      "learning_rate": 8.912651515151514e-05,
      "loss": 2.7995,
      "step": 164850
    },
    {
      "activations/layer0_attention_weight_max": 15.194976806640625,
      "activations/layer0_attention_weight_min": -13.994694709777832,
      "activations/layer10_attention_weight_max": 39.12921142578125,
      "activations/layer10_attention_weight_min": -38.06309127807617,
      "activations/layer11_attention_weight_max": 34.78285217285156,
      "activations/layer11_attention_weight_min": -36.03058624267578,
      "activations/layer12_attention_weight_max": 23.231231689453125,
      "activations/layer12_attention_weight_min": -27.846830368041992,
      "activations/layer13_attention_weight_max": 39.624671936035156,
      "activations/layer13_attention_weight_min": -31.672283172607422,
      "activations/layer14_attention_weight_max": 38.87154769897461,
      "activations/layer14_attention_weight_min": -33.84303665161133,
      "activations/layer15_attention_weight_max": 35.531410217285156,
      "activations/layer15_attention_weight_min": -31.964868545532227,
      "activations/layer16_attention_weight_max": 32.30421447753906,
      "activations/layer16_attention_weight_min": -29.322837829589844,
      "activations/layer17_attention_weight_max": 56.120155334472656,
      "activations/layer17_attention_weight_min": -48.26080322265625,
      "activations/layer18_attention_weight_max": 52.11042404174805,
      "activations/layer18_attention_weight_min": -37.898807525634766,
      "activations/layer19_attention_weight_max": 23.196012496948242,
      "activations/layer19_attention_weight_min": -22.01197052001953,
      "activations/layer1_attention_weight_max": 17.695642471313477,
      "activations/layer1_attention_weight_min": -15.499547958374023,
      "activations/layer20_attention_weight_max": 25.116836547851562,
      "activations/layer20_attention_weight_min": -20.666643142700195,
      "activations/layer21_attention_weight_max": 43.260189056396484,
      "activations/layer21_attention_weight_min": -25.935836791992188,
      "activations/layer22_attention_weight_max": 32.26750183105469,
      "activations/layer22_attention_weight_min": -27.40506362915039,
      "activations/layer23_attention_weight_max": 41.1978645324707,
      "activations/layer23_attention_weight_min": -24.88711929321289,
      "activations/layer2_attention_weight_max": 31.550573348999023,
      "activations/layer2_attention_weight_min": -30.585920333862305,
      "activations/layer3_attention_weight_max": 91.89306640625,
      "activations/layer3_attention_weight_min": -93.60932159423828,
      "activations/layer4_attention_weight_max": 105.5656509399414,
      "activations/layer4_attention_weight_min": -97.91565704345703,
      "activations/layer5_attention_weight_max": 80.24166870117188,
      "activations/layer5_attention_weight_min": -79.98318481445312,
      "activations/layer6_attention_weight_max": 55.23767852783203,
      "activations/layer6_attention_weight_min": -55.00923156738281,
      "activations/layer7_attention_weight_max": 78.77178955078125,
      "activations/layer7_attention_weight_min": -81.46793365478516,
      "activations/layer8_attention_weight_max": 48.96714782714844,
      "activations/layer8_attention_weight_min": -52.841976165771484,
      "activations/layer9_attention_weight_max": 52.64298629760742,
      "activations/layer9_attention_weight_min": -50.87148666381836,
      "epoch": 9.58,
      "learning_rate": 8.910757575757574e-05,
      "loss": 2.8019,
      "step": 164900
    },
    {
      "activations/layer0_attention_weight_max": 16.389554977416992,
      "activations/layer0_attention_weight_min": -13.761251449584961,
      "activations/layer10_attention_weight_max": 39.65967559814453,
      "activations/layer10_attention_weight_min": -37.48324966430664,
      "activations/layer11_attention_weight_max": 34.66046142578125,
      "activations/layer11_attention_weight_min": -34.887123107910156,
      "activations/layer12_attention_weight_max": 19.30791664123535,
      "activations/layer12_attention_weight_min": -26.26177406311035,
      "activations/layer13_attention_weight_max": 56.99384689331055,
      "activations/layer13_attention_weight_min": -42.289207458496094,
      "activations/layer14_attention_weight_max": 56.05670928955078,
      "activations/layer14_attention_weight_min": -39.719818115234375,
      "activations/layer15_attention_weight_max": 45.9245719909668,
      "activations/layer15_attention_weight_min": -31.527482986450195,
      "activations/layer16_attention_weight_max": 31.66847801208496,
      "activations/layer16_attention_weight_min": -28.50873565673828,
      "activations/layer17_attention_weight_max": 52.91051483154297,
      "activations/layer17_attention_weight_min": -44.3378791809082,
      "activations/layer18_attention_weight_max": 46.608375549316406,
      "activations/layer18_attention_weight_min": -37.11628341674805,
      "activations/layer19_attention_weight_max": 22.47445297241211,
      "activations/layer19_attention_weight_min": -21.381086349487305,
      "activations/layer1_attention_weight_max": 17.10755729675293,
      "activations/layer1_attention_weight_min": -16.838008880615234,
      "activations/layer20_attention_weight_max": 22.295795440673828,
      "activations/layer20_attention_weight_min": -20.955669403076172,
      "activations/layer21_attention_weight_max": 35.62724685668945,
      "activations/layer21_attention_weight_min": -24.168350219726562,
      "activations/layer22_attention_weight_max": 28.569971084594727,
      "activations/layer22_attention_weight_min": -26.11134147644043,
      "activations/layer23_attention_weight_max": 35.041709899902344,
      "activations/layer23_attention_weight_min": -22.89105796813965,
      "activations/layer2_attention_weight_max": 30.843502044677734,
      "activations/layer2_attention_weight_min": -31.492717742919922,
      "activations/layer3_attention_weight_max": 88.61151123046875,
      "activations/layer3_attention_weight_min": -92.6371078491211,
      "activations/layer4_attention_weight_max": 103.2646255493164,
      "activations/layer4_attention_weight_min": -99.51607513427734,
      "activations/layer5_attention_weight_max": 84.40570068359375,
      "activations/layer5_attention_weight_min": -83.88153839111328,
      "activations/layer6_attention_weight_max": 56.0151252746582,
      "activations/layer6_attention_weight_min": -55.27894973754883,
      "activations/layer7_attention_weight_max": 83.56678009033203,
      "activations/layer7_attention_weight_min": -77.0794906616211,
      "activations/layer8_attention_weight_max": 46.25910949707031,
      "activations/layer8_attention_weight_min": -47.238643646240234,
      "activations/layer9_attention_weight_max": 54.820674896240234,
      "activations/layer9_attention_weight_min": -52.51300048828125,
      "epoch": 9.58,
      "learning_rate": 8.908863636363636e-05,
      "loss": 2.7938,
      "step": 164950
    },
    {
      "activations/layer0_attention_weight_max": 15.257251739501953,
      "activations/layer0_attention_weight_min": -14.319635391235352,
      "activations/layer10_attention_weight_max": 39.19536209106445,
      "activations/layer10_attention_weight_min": -36.87787628173828,
      "activations/layer11_attention_weight_max": 32.76762771606445,
      "activations/layer11_attention_weight_min": -34.88249206542969,
      "activations/layer12_attention_weight_max": 20.242916107177734,
      "activations/layer12_attention_weight_min": -22.635372161865234,
      "activations/layer13_attention_weight_max": 35.11567306518555,
      "activations/layer13_attention_weight_min": -28.684078216552734,
      "activations/layer14_attention_weight_max": 38.14940643310547,
      "activations/layer14_attention_weight_min": -32.690006256103516,
      "activations/layer15_attention_weight_max": 32.15945816040039,
      "activations/layer15_attention_weight_min": -29.95569610595703,
      "activations/layer16_attention_weight_max": 31.492952346801758,
      "activations/layer16_attention_weight_min": -29.827980041503906,
      "activations/layer17_attention_weight_max": 52.22976303100586,
      "activations/layer17_attention_weight_min": -46.17750930786133,
      "activations/layer18_attention_weight_max": 47.06673812866211,
      "activations/layer18_attention_weight_min": -36.08867645263672,
      "activations/layer19_attention_weight_max": 22.354841232299805,
      "activations/layer19_attention_weight_min": -23.335432052612305,
      "activations/layer1_attention_weight_max": 17.514423370361328,
      "activations/layer1_attention_weight_min": -15.831520080566406,
      "activations/layer20_attention_weight_max": 23.147918701171875,
      "activations/layer20_attention_weight_min": -26.435888290405273,
      "activations/layer21_attention_weight_max": 38.86091995239258,
      "activations/layer21_attention_weight_min": -21.988542556762695,
      "activations/layer22_attention_weight_max": 28.397459030151367,
      "activations/layer22_attention_weight_min": -24.435565948486328,
      "activations/layer23_attention_weight_max": 35.130149841308594,
      "activations/layer23_attention_weight_min": -26.388410568237305,
      "activations/layer2_attention_weight_max": 33.010719299316406,
      "activations/layer2_attention_weight_min": -30.081787109375,
      "activations/layer3_attention_weight_max": 99.38983154296875,
      "activations/layer3_attention_weight_min": -100.20011138916016,
      "activations/layer4_attention_weight_max": 111.11571502685547,
      "activations/layer4_attention_weight_min": -101.71544647216797,
      "activations/layer5_attention_weight_max": 86.50218200683594,
      "activations/layer5_attention_weight_min": -83.04268646240234,
      "activations/layer6_attention_weight_max": 58.4840202331543,
      "activations/layer6_attention_weight_min": -54.8122444152832,
      "activations/layer7_attention_weight_max": 77.62618255615234,
      "activations/layer7_attention_weight_min": -75.42164611816406,
      "activations/layer8_attention_weight_max": 49.48436737060547,
      "activations/layer8_attention_weight_min": -50.38674545288086,
      "activations/layer9_attention_weight_max": 53.46026611328125,
      "activations/layer9_attention_weight_min": -51.50769805908203,
      "epoch": 9.59,
      "learning_rate": 8.906969696969696e-05,
      "loss": 2.8109,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_loss": 2.748046875,
      "eval_runtime": 8.5141,
      "eval_samples_per_second": 504.338,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_openwebtext_loss": 2.748046875,
      "eval_openwebtext_ppl": 15.612109685326937,
      "eval_openwebtext_runtime": 8.5141,
      "eval_openwebtext_samples_per_second": 504.338,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 2.0019,
      "eval_wikitext_samples_per_second": 227.779,
      "step": 165000
    },
    {
      "epoch": 9.59,
      "eval_lambada_loss": 2.63671875,
      "eval_lambada_ppl": 13.96729813824377,
      "eval_lambada_runtime": 9.6111,
      "eval_lambada_samples_per_second": 506.604,
      "step": 165000
    },
    {
      "activations/layer0_attention_weight_max": 15.717178344726562,
      "activations/layer0_attention_weight_min": -14.129828453063965,
      "activations/layer10_attention_weight_max": 38.601375579833984,
      "activations/layer10_attention_weight_min": -35.065128326416016,
      "activations/layer11_attention_weight_max": 31.244773864746094,
      "activations/layer11_attention_weight_min": -32.44322967529297,
      "activations/layer12_attention_weight_max": 22.62032699584961,
      "activations/layer12_attention_weight_min": -25.4776611328125,
      "activations/layer13_attention_weight_max": 39.34407043457031,
      "activations/layer13_attention_weight_min": -27.946168899536133,
      "activations/layer14_attention_weight_max": 35.97978591918945,
      "activations/layer14_attention_weight_min": -29.02937126159668,
      "activations/layer15_attention_weight_max": 35.129146575927734,
      "activations/layer15_attention_weight_min": -29.6436710357666,
      "activations/layer16_attention_weight_max": 34.99460983276367,
      "activations/layer16_attention_weight_min": -27.65354347229004,
      "activations/layer17_attention_weight_max": 55.88192367553711,
      "activations/layer17_attention_weight_min": -45.21391296386719,
      "activations/layer18_attention_weight_max": 47.14363479614258,
      "activations/layer18_attention_weight_min": -38.38189697265625,
      "activations/layer19_attention_weight_max": 26.06801414489746,
      "activations/layer19_attention_weight_min": -19.655576705932617,
      "activations/layer1_attention_weight_max": 16.46599006652832,
      "activations/layer1_attention_weight_min": -14.040775299072266,
      "activations/layer20_attention_weight_max": 22.73735237121582,
      "activations/layer20_attention_weight_min": -21.078237533569336,
      "activations/layer21_attention_weight_max": 40.41697692871094,
      "activations/layer21_attention_weight_min": -24.24340057373047,
      "activations/layer22_attention_weight_max": 30.487913131713867,
      "activations/layer22_attention_weight_min": -23.331275939941406,
      "activations/layer23_attention_weight_max": 39.22468948364258,
      "activations/layer23_attention_weight_min": -23.5076961517334,
      "activations/layer2_attention_weight_max": 32.21503829956055,
      "activations/layer2_attention_weight_min": -29.88024139404297,
      "activations/layer3_attention_weight_max": 91.742919921875,
      "activations/layer3_attention_weight_min": -97.51934814453125,
      "activations/layer4_attention_weight_max": 99.46892547607422,
      "activations/layer4_attention_weight_min": -97.04515075683594,
      "activations/layer5_attention_weight_max": 79.14949035644531,
      "activations/layer5_attention_weight_min": -77.94055938720703,
      "activations/layer6_attention_weight_max": 55.50163269042969,
      "activations/layer6_attention_weight_min": -53.23322296142578,
      "activations/layer7_attention_weight_max": 76.23506164550781,
      "activations/layer7_attention_weight_min": -71.43730926513672,
      "activations/layer8_attention_weight_max": 44.810672760009766,
      "activations/layer8_attention_weight_min": -47.15410232543945,
      "activations/layer9_attention_weight_max": 50.49245834350586,
      "activations/layer9_attention_weight_min": -49.23499298095703,
      "epoch": 9.59,
      "learning_rate": 8.905075757575757e-05,
      "loss": 2.7827,
      "step": 165050
    },
    {
      "activations/layer0_attention_weight_max": 15.736443519592285,
      "activations/layer0_attention_weight_min": -13.989075660705566,
      "activations/layer10_attention_weight_max": 36.855003356933594,
      "activations/layer10_attention_weight_min": -36.19645690917969,
      "activations/layer11_attention_weight_max": 32.831180572509766,
      "activations/layer11_attention_weight_min": -31.811941146850586,
      "activations/layer12_attention_weight_max": 20.267541885375977,
      "activations/layer12_attention_weight_min": -24.939041137695312,
      "activations/layer13_attention_weight_max": 44.40761947631836,
      "activations/layer13_attention_weight_min": -30.42620086669922,
      "activations/layer14_attention_weight_max": 48.951297760009766,
      "activations/layer14_attention_weight_min": -31.142780303955078,
      "activations/layer15_attention_weight_max": 40.544002532958984,
      "activations/layer15_attention_weight_min": -32.79370880126953,
      "activations/layer16_attention_weight_max": 32.58864974975586,
      "activations/layer16_attention_weight_min": -29.45901107788086,
      "activations/layer17_attention_weight_max": 55.16297149658203,
      "activations/layer17_attention_weight_min": -43.89054870605469,
      "activations/layer18_attention_weight_max": 50.1764030456543,
      "activations/layer18_attention_weight_min": -38.03053283691406,
      "activations/layer19_attention_weight_max": 22.250211715698242,
      "activations/layer19_attention_weight_min": -21.115463256835938,
      "activations/layer1_attention_weight_max": 17.082426071166992,
      "activations/layer1_attention_weight_min": -15.046950340270996,
      "activations/layer20_attention_weight_max": 24.594209671020508,
      "activations/layer20_attention_weight_min": -21.61585807800293,
      "activations/layer21_attention_weight_max": 46.49909591674805,
      "activations/layer21_attention_weight_min": -23.322141647338867,
      "activations/layer22_attention_weight_max": 37.51025390625,
      "activations/layer22_attention_weight_min": -26.028060913085938,
      "activations/layer23_attention_weight_max": 40.09637451171875,
      "activations/layer23_attention_weight_min": -23.218910217285156,
      "activations/layer2_attention_weight_max": 32.58135986328125,
      "activations/layer2_attention_weight_min": -29.92578125,
      "activations/layer3_attention_weight_max": 93.94434356689453,
      "activations/layer3_attention_weight_min": -91.53032684326172,
      "activations/layer4_attention_weight_max": 101.13951873779297,
      "activations/layer4_attention_weight_min": -96.21090698242188,
      "activations/layer5_attention_weight_max": 80.3756103515625,
      "activations/layer5_attention_weight_min": -77.23295593261719,
      "activations/layer6_attention_weight_max": 53.88076400756836,
      "activations/layer6_attention_weight_min": -51.81459426879883,
      "activations/layer7_attention_weight_max": 76.13704681396484,
      "activations/layer7_attention_weight_min": -74.4844970703125,
      "activations/layer8_attention_weight_max": 44.815467834472656,
      "activations/layer8_attention_weight_min": -45.952735900878906,
      "activations/layer9_attention_weight_max": 54.00957107543945,
      "activations/layer9_attention_weight_min": -47.054386138916016,
      "epoch": 9.59,
      "learning_rate": 8.903181818181818e-05,
      "loss": 2.7979,
      "step": 165100
    },
    {
      "activations/layer0_attention_weight_max": 14.667644500732422,
      "activations/layer0_attention_weight_min": -14.170451164245605,
      "activations/layer10_attention_weight_max": 37.02764892578125,
      "activations/layer10_attention_weight_min": -39.40857696533203,
      "activations/layer11_attention_weight_max": 32.67707061767578,
      "activations/layer11_attention_weight_min": -32.665409088134766,
      "activations/layer12_attention_weight_max": 20.985748291015625,
      "activations/layer12_attention_weight_min": -24.879728317260742,
      "activations/layer13_attention_weight_max": 42.42010498046875,
      "activations/layer13_attention_weight_min": -28.611854553222656,
      "activations/layer14_attention_weight_max": 35.39107894897461,
      "activations/layer14_attention_weight_min": -29.89103889465332,
      "activations/layer15_attention_weight_max": 35.07529067993164,
      "activations/layer15_attention_weight_min": -29.73121452331543,
      "activations/layer16_attention_weight_max": 36.116966247558594,
      "activations/layer16_attention_weight_min": -28.4052734375,
      "activations/layer17_attention_weight_max": 56.97367477416992,
      "activations/layer17_attention_weight_min": -41.30324172973633,
      "activations/layer18_attention_weight_max": 44.91091537475586,
      "activations/layer18_attention_weight_min": -34.11366271972656,
      "activations/layer19_attention_weight_max": 19.444232940673828,
      "activations/layer19_attention_weight_min": -19.695980072021484,
      "activations/layer1_attention_weight_max": 17.02641487121582,
      "activations/layer1_attention_weight_min": -12.945439338684082,
      "activations/layer20_attention_weight_max": 21.7696475982666,
      "activations/layer20_attention_weight_min": -21.704795837402344,
      "activations/layer21_attention_weight_max": 32.88037872314453,
      "activations/layer21_attention_weight_min": -21.491609573364258,
      "activations/layer22_attention_weight_max": 33.87804412841797,
      "activations/layer22_attention_weight_min": -26.10346031188965,
      "activations/layer23_attention_weight_max": 39.951698303222656,
      "activations/layer23_attention_weight_min": -24.721294403076172,
      "activations/layer2_attention_weight_max": 31.691198348999023,
      "activations/layer2_attention_weight_min": -31.20001983642578,
      "activations/layer3_attention_weight_max": 92.06414794921875,
      "activations/layer3_attention_weight_min": -99.74055480957031,
      "activations/layer4_attention_weight_max": 101.81090545654297,
      "activations/layer4_attention_weight_min": -94.83636474609375,
      "activations/layer5_attention_weight_max": 79.4383316040039,
      "activations/layer5_attention_weight_min": -79.41152954101562,
      "activations/layer6_attention_weight_max": 55.29852294921875,
      "activations/layer6_attention_weight_min": -53.13014602661133,
      "activations/layer7_attention_weight_max": 83.82429504394531,
      "activations/layer7_attention_weight_min": -70.87522888183594,
      "activations/layer8_attention_weight_max": 44.4482536315918,
      "activations/layer8_attention_weight_min": -45.07027053833008,
      "activations/layer9_attention_weight_max": 53.184261322021484,
      "activations/layer9_attention_weight_min": -49.618247985839844,
      "epoch": 9.6,
      "learning_rate": 8.901287878787878e-05,
      "loss": 2.8024,
      "step": 165150
    },
    {
      "activations/layer0_attention_weight_max": 16.14424705505371,
      "activations/layer0_attention_weight_min": -13.991856575012207,
      "activations/layer10_attention_weight_max": 36.19873046875,
      "activations/layer10_attention_weight_min": -37.51780700683594,
      "activations/layer11_attention_weight_max": 31.893569946289062,
      "activations/layer11_attention_weight_min": -32.33445358276367,
      "activations/layer12_attention_weight_max": 19.1660099029541,
      "activations/layer12_attention_weight_min": -21.662940979003906,
      "activations/layer13_attention_weight_max": 38.919822692871094,
      "activations/layer13_attention_weight_min": -29.888662338256836,
      "activations/layer14_attention_weight_max": 36.978607177734375,
      "activations/layer14_attention_weight_min": -31.856704711914062,
      "activations/layer15_attention_weight_max": 33.49703598022461,
      "activations/layer15_attention_weight_min": -30.91341209411621,
      "activations/layer16_attention_weight_max": 31.679460525512695,
      "activations/layer16_attention_weight_min": -28.320981979370117,
      "activations/layer17_attention_weight_max": 51.56748580932617,
      "activations/layer17_attention_weight_min": -45.79408645629883,
      "activations/layer18_attention_weight_max": 44.65620803833008,
      "activations/layer18_attention_weight_min": -37.52326583862305,
      "activations/layer19_attention_weight_max": 26.17606544494629,
      "activations/layer19_attention_weight_min": -22.287994384765625,
      "activations/layer1_attention_weight_max": 16.664491653442383,
      "activations/layer1_attention_weight_min": -15.393125534057617,
      "activations/layer20_attention_weight_max": 22.78006362915039,
      "activations/layer20_attention_weight_min": -22.5942325592041,
      "activations/layer21_attention_weight_max": 40.80891036987305,
      "activations/layer21_attention_weight_min": -23.748844146728516,
      "activations/layer22_attention_weight_max": 31.54574966430664,
      "activations/layer22_attention_weight_min": -25.789474487304688,
      "activations/layer23_attention_weight_max": 35.518184661865234,
      "activations/layer23_attention_weight_min": -23.516212463378906,
      "activations/layer2_attention_weight_max": 32.23537826538086,
      "activations/layer2_attention_weight_min": -32.17536926269531,
      "activations/layer3_attention_weight_max": 91.55521392822266,
      "activations/layer3_attention_weight_min": -97.21063995361328,
      "activations/layer4_attention_weight_max": 103.40840148925781,
      "activations/layer4_attention_weight_min": -101.33501434326172,
      "activations/layer5_attention_weight_max": 84.19339752197266,
      "activations/layer5_attention_weight_min": -84.8880844116211,
      "activations/layer6_attention_weight_max": 56.98601150512695,
      "activations/layer6_attention_weight_min": -55.17127227783203,
      "activations/layer7_attention_weight_max": 84.3898696899414,
      "activations/layer7_attention_weight_min": -79.15816497802734,
      "activations/layer8_attention_weight_max": 47.09661865234375,
      "activations/layer8_attention_weight_min": -48.18031692504883,
      "activations/layer9_attention_weight_max": 52.9662971496582,
      "activations/layer9_attention_weight_min": -52.09342575073242,
      "epoch": 9.6,
      "learning_rate": 8.899393939393939e-05,
      "loss": 2.7893,
      "step": 165200
    },
    {
      "activations/layer0_attention_weight_max": 15.594618797302246,
      "activations/layer0_attention_weight_min": -14.384035110473633,
      "activations/layer10_attention_weight_max": 35.815731048583984,
      "activations/layer10_attention_weight_min": -34.91182327270508,
      "activations/layer11_attention_weight_max": 31.70248794555664,
      "activations/layer11_attention_weight_min": -32.64035415649414,
      "activations/layer12_attention_weight_max": 18.85930061340332,
      "activations/layer12_attention_weight_min": -26.725303649902344,
      "activations/layer13_attention_weight_max": 41.366416931152344,
      "activations/layer13_attention_weight_min": -34.193397521972656,
      "activations/layer14_attention_weight_max": 37.475215911865234,
      "activations/layer14_attention_weight_min": -31.68996810913086,
      "activations/layer15_attention_weight_max": 31.551733016967773,
      "activations/layer15_attention_weight_min": -31.395605087280273,
      "activations/layer16_attention_weight_max": 29.99003791809082,
      "activations/layer16_attention_weight_min": -29.599992752075195,
      "activations/layer17_attention_weight_max": 51.73435592651367,
      "activations/layer17_attention_weight_min": -46.35019302368164,
      "activations/layer18_attention_weight_max": 44.293975830078125,
      "activations/layer18_attention_weight_min": -37.389862060546875,
      "activations/layer19_attention_weight_max": 22.619417190551758,
      "activations/layer19_attention_weight_min": -21.893234252929688,
      "activations/layer1_attention_weight_max": 16.53401756286621,
      "activations/layer1_attention_weight_min": -13.947564125061035,
      "activations/layer20_attention_weight_max": 22.569255828857422,
      "activations/layer20_attention_weight_min": -21.684619903564453,
      "activations/layer21_attention_weight_max": 40.99061965942383,
      "activations/layer21_attention_weight_min": -23.867889404296875,
      "activations/layer22_attention_weight_max": 32.957035064697266,
      "activations/layer22_attention_weight_min": -28.586843490600586,
      "activations/layer23_attention_weight_max": 35.57844543457031,
      "activations/layer23_attention_weight_min": -24.76944351196289,
      "activations/layer2_attention_weight_max": 30.669174194335938,
      "activations/layer2_attention_weight_min": -28.439071655273438,
      "activations/layer3_attention_weight_max": 92.08963775634766,
      "activations/layer3_attention_weight_min": -93.07401275634766,
      "activations/layer4_attention_weight_max": 97.77747344970703,
      "activations/layer4_attention_weight_min": -91.42952728271484,
      "activations/layer5_attention_weight_max": 75.93621826171875,
      "activations/layer5_attention_weight_min": -72.87833404541016,
      "activations/layer6_attention_weight_max": 52.96400833129883,
      "activations/layer6_attention_weight_min": -49.81404113769531,
      "activations/layer7_attention_weight_max": 77.9523696899414,
      "activations/layer7_attention_weight_min": -74.19366455078125,
      "activations/layer8_attention_weight_max": 43.74715805053711,
      "activations/layer8_attention_weight_min": -43.89278030395508,
      "activations/layer9_attention_weight_max": 46.62752151489258,
      "activations/layer9_attention_weight_min": -48.7857780456543,
      "epoch": 9.6,
      "learning_rate": 8.897499999999998e-05,
      "loss": 2.8044,
      "step": 165250
    },
    {
      "activations/layer0_attention_weight_max": 15.455669403076172,
      "activations/layer0_attention_weight_min": -14.14293384552002,
      "activations/layer10_attention_weight_max": 34.63160705566406,
      "activations/layer10_attention_weight_min": -37.80983352661133,
      "activations/layer11_attention_weight_max": 31.993900299072266,
      "activations/layer11_attention_weight_min": -30.787948608398438,
      "activations/layer12_attention_weight_max": 18.296720504760742,
      "activations/layer12_attention_weight_min": -40.11592102050781,
      "activations/layer13_attention_weight_max": 38.11091995239258,
      "activations/layer13_attention_weight_min": -29.90525245666504,
      "activations/layer14_attention_weight_max": 38.743614196777344,
      "activations/layer14_attention_weight_min": -29.721691131591797,
      "activations/layer15_attention_weight_max": 35.11861038208008,
      "activations/layer15_attention_weight_min": -28.704084396362305,
      "activations/layer16_attention_weight_max": 33.752174377441406,
      "activations/layer16_attention_weight_min": -29.44999885559082,
      "activations/layer17_attention_weight_max": 61.30132293701172,
      "activations/layer17_attention_weight_min": -50.00534439086914,
      "activations/layer18_attention_weight_max": 50.7497444152832,
      "activations/layer18_attention_weight_min": -40.036502838134766,
      "activations/layer19_attention_weight_max": 22.95189666748047,
      "activations/layer19_attention_weight_min": -21.710336685180664,
      "activations/layer1_attention_weight_max": 16.546098709106445,
      "activations/layer1_attention_weight_min": -14.646574020385742,
      "activations/layer20_attention_weight_max": 22.159488677978516,
      "activations/layer20_attention_weight_min": -19.917545318603516,
      "activations/layer21_attention_weight_max": 38.26123046875,
      "activations/layer21_attention_weight_min": -22.844629287719727,
      "activations/layer22_attention_weight_max": 32.34611511230469,
      "activations/layer22_attention_weight_min": -25.266836166381836,
      "activations/layer23_attention_weight_max": 37.4847412109375,
      "activations/layer23_attention_weight_min": -21.120044708251953,
      "activations/layer2_attention_weight_max": 33.54182052612305,
      "activations/layer2_attention_weight_min": -30.143354415893555,
      "activations/layer3_attention_weight_max": 90.29788208007812,
      "activations/layer3_attention_weight_min": -95.2981185913086,
      "activations/layer4_attention_weight_max": 100.1238784790039,
      "activations/layer4_attention_weight_min": -96.71012878417969,
      "activations/layer5_attention_weight_max": 78.1628646850586,
      "activations/layer5_attention_weight_min": -77.67940521240234,
      "activations/layer6_attention_weight_max": 53.929420471191406,
      "activations/layer6_attention_weight_min": -55.797019958496094,
      "activations/layer7_attention_weight_max": 82.60948944091797,
      "activations/layer7_attention_weight_min": -76.09281921386719,
      "activations/layer8_attention_weight_max": 46.71784591674805,
      "activations/layer8_attention_weight_min": -46.64418411254883,
      "activations/layer9_attention_weight_max": 60.22698974609375,
      "activations/layer9_attention_weight_min": -58.7320556640625,
      "epoch": 9.6,
      "learning_rate": 8.89560606060606e-05,
      "loss": 2.7923,
      "step": 165300
    },
    {
      "activations/layer0_attention_weight_max": 15.281708717346191,
      "activations/layer0_attention_weight_min": -14.850679397583008,
      "activations/layer10_attention_weight_max": 37.177120208740234,
      "activations/layer10_attention_weight_min": -37.00944900512695,
      "activations/layer11_attention_weight_max": 31.06654167175293,
      "activations/layer11_attention_weight_min": -33.60567855834961,
      "activations/layer12_attention_weight_max": 19.490997314453125,
      "activations/layer12_attention_weight_min": -24.625150680541992,
      "activations/layer13_attention_weight_max": 48.915592193603516,
      "activations/layer13_attention_weight_min": -34.48221969604492,
      "activations/layer14_attention_weight_max": 39.70295333862305,
      "activations/layer14_attention_weight_min": -31.413034439086914,
      "activations/layer15_attention_weight_max": 36.54916000366211,
      "activations/layer15_attention_weight_min": -30.760330200195312,
      "activations/layer16_attention_weight_max": 31.60835838317871,
      "activations/layer16_attention_weight_min": -29.810977935791016,
      "activations/layer17_attention_weight_max": 54.492515563964844,
      "activations/layer17_attention_weight_min": -46.483917236328125,
      "activations/layer18_attention_weight_max": 52.808719635009766,
      "activations/layer18_attention_weight_min": -39.753509521484375,
      "activations/layer19_attention_weight_max": 21.96614646911621,
      "activations/layer19_attention_weight_min": -23.20026969909668,
      "activations/layer1_attention_weight_max": 17.246902465820312,
      "activations/layer1_attention_weight_min": -14.00693130493164,
      "activations/layer20_attention_weight_max": 22.48729705810547,
      "activations/layer20_attention_weight_min": -20.83009147644043,
      "activations/layer21_attention_weight_max": 41.94388961791992,
      "activations/layer21_attention_weight_min": -27.052127838134766,
      "activations/layer22_attention_weight_max": 36.27072525024414,
      "activations/layer22_attention_weight_min": -26.68705177307129,
      "activations/layer23_attention_weight_max": 37.50163269042969,
      "activations/layer23_attention_weight_min": -25.97481346130371,
      "activations/layer2_attention_weight_max": 31.659473419189453,
      "activations/layer2_attention_weight_min": -29.32128143310547,
      "activations/layer3_attention_weight_max": 90.23007202148438,
      "activations/layer3_attention_weight_min": -88.12122344970703,
      "activations/layer4_attention_weight_max": 102.02513885498047,
      "activations/layer4_attention_weight_min": -95.9473648071289,
      "activations/layer5_attention_weight_max": 79.93151092529297,
      "activations/layer5_attention_weight_min": -78.69502258300781,
      "activations/layer6_attention_weight_max": 56.6881217956543,
      "activations/layer6_attention_weight_min": -55.15400695800781,
      "activations/layer7_attention_weight_max": 80.09866333007812,
      "activations/layer7_attention_weight_min": -77.28567504882812,
      "activations/layer8_attention_weight_max": 48.43864059448242,
      "activations/layer8_attention_weight_min": -50.74568557739258,
      "activations/layer9_attention_weight_max": 52.953651428222656,
      "activations/layer9_attention_weight_min": -53.6146125793457,
      "epoch": 9.61,
      "learning_rate": 8.893712121212121e-05,
      "loss": 2.7879,
      "step": 165350
    },
    {
      "activations/layer0_attention_weight_max": 15.278942108154297,
      "activations/layer0_attention_weight_min": -14.02889633178711,
      "activations/layer10_attention_weight_max": 41.8165283203125,
      "activations/layer10_attention_weight_min": -37.097900390625,
      "activations/layer11_attention_weight_max": 34.080772399902344,
      "activations/layer11_attention_weight_min": -34.07595443725586,
      "activations/layer12_attention_weight_max": 25.87818717956543,
      "activations/layer12_attention_weight_min": -23.4195556640625,
      "activations/layer13_attention_weight_max": 41.28887939453125,
      "activations/layer13_attention_weight_min": -29.841766357421875,
      "activations/layer14_attention_weight_max": 37.39961624145508,
      "activations/layer14_attention_weight_min": -29.549488067626953,
      "activations/layer15_attention_weight_max": 34.98831558227539,
      "activations/layer15_attention_weight_min": -30.269922256469727,
      "activations/layer16_attention_weight_max": 35.25983428955078,
      "activations/layer16_attention_weight_min": -29.49382781982422,
      "activations/layer17_attention_weight_max": 52.687889099121094,
      "activations/layer17_attention_weight_min": -46.36762237548828,
      "activations/layer18_attention_weight_max": 50.66599655151367,
      "activations/layer18_attention_weight_min": -36.72575378417969,
      "activations/layer19_attention_weight_max": 23.613054275512695,
      "activations/layer19_attention_weight_min": -21.03078269958496,
      "activations/layer1_attention_weight_max": 17.009756088256836,
      "activations/layer1_attention_weight_min": -15.010831832885742,
      "activations/layer20_attention_weight_max": 22.025785446166992,
      "activations/layer20_attention_weight_min": -21.63563346862793,
      "activations/layer21_attention_weight_max": 40.38477325439453,
      "activations/layer21_attention_weight_min": -25.04994773864746,
      "activations/layer22_attention_weight_max": 30.612071990966797,
      "activations/layer22_attention_weight_min": -24.89409065246582,
      "activations/layer23_attention_weight_max": 38.45099639892578,
      "activations/layer23_attention_weight_min": -23.683074951171875,
      "activations/layer2_attention_weight_max": 33.16508102416992,
      "activations/layer2_attention_weight_min": -30.621509552001953,
      "activations/layer3_attention_weight_max": 97.03827667236328,
      "activations/layer3_attention_weight_min": -98.67691040039062,
      "activations/layer4_attention_weight_max": 102.833740234375,
      "activations/layer4_attention_weight_min": -97.45268249511719,
      "activations/layer5_attention_weight_max": 78.44773864746094,
      "activations/layer5_attention_weight_min": -79.07240295410156,
      "activations/layer6_attention_weight_max": 55.982887268066406,
      "activations/layer6_attention_weight_min": -54.607181549072266,
      "activations/layer7_attention_weight_max": 81.86649322509766,
      "activations/layer7_attention_weight_min": -77.80644989013672,
      "activations/layer8_attention_weight_max": 48.71247863769531,
      "activations/layer8_attention_weight_min": -48.833580017089844,
      "activations/layer9_attention_weight_max": 53.416988372802734,
      "activations/layer9_attention_weight_min": -49.58333206176758,
      "epoch": 9.61,
      "learning_rate": 8.89181818181818e-05,
      "loss": 2.8059,
      "step": 165400
    },
    {
      "activations/layer0_attention_weight_max": 16.195262908935547,
      "activations/layer0_attention_weight_min": -14.097132682800293,
      "activations/layer10_attention_weight_max": 38.98204040527344,
      "activations/layer10_attention_weight_min": -38.74504852294922,
      "activations/layer11_attention_weight_max": 32.950843811035156,
      "activations/layer11_attention_weight_min": -32.387786865234375,
      "activations/layer12_attention_weight_max": 16.846839904785156,
      "activations/layer12_attention_weight_min": -26.41667366027832,
      "activations/layer13_attention_weight_max": 43.41902542114258,
      "activations/layer13_attention_weight_min": -34.33184051513672,
      "activations/layer14_attention_weight_max": 34.82316970825195,
      "activations/layer14_attention_weight_min": -30.80036735534668,
      "activations/layer15_attention_weight_max": 32.07469177246094,
      "activations/layer15_attention_weight_min": -29.611927032470703,
      "activations/layer16_attention_weight_max": 28.022388458251953,
      "activations/layer16_attention_weight_min": -27.89500617980957,
      "activations/layer17_attention_weight_max": 48.492977142333984,
      "activations/layer17_attention_weight_min": -41.653419494628906,
      "activations/layer18_attention_weight_max": 45.09101486206055,
      "activations/layer18_attention_weight_min": -35.48577117919922,
      "activations/layer19_attention_weight_max": 21.67770004272461,
      "activations/layer19_attention_weight_min": -23.107646942138672,
      "activations/layer1_attention_weight_max": 16.909387588500977,
      "activations/layer1_attention_weight_min": -16.381690979003906,
      "activations/layer20_attention_weight_max": 19.731548309326172,
      "activations/layer20_attention_weight_min": -26.364452362060547,
      "activations/layer21_attention_weight_max": 35.01328659057617,
      "activations/layer21_attention_weight_min": -23.753734588623047,
      "activations/layer22_attention_weight_max": 33.9888801574707,
      "activations/layer22_attention_weight_min": -25.70262908935547,
      "activations/layer23_attention_weight_max": 36.545284271240234,
      "activations/layer23_attention_weight_min": -26.974782943725586,
      "activations/layer2_attention_weight_max": 31.35146713256836,
      "activations/layer2_attention_weight_min": -29.53030776977539,
      "activations/layer3_attention_weight_max": 89.3078384399414,
      "activations/layer3_attention_weight_min": -91.11217498779297,
      "activations/layer4_attention_weight_max": 98.72718048095703,
      "activations/layer4_attention_weight_min": -100.57648468017578,
      "activations/layer5_attention_weight_max": 77.33753204345703,
      "activations/layer5_attention_weight_min": -77.24925231933594,
      "activations/layer6_attention_weight_max": 54.54561233520508,
      "activations/layer6_attention_weight_min": -53.04883575439453,
      "activations/layer7_attention_weight_max": 76.52529907226562,
      "activations/layer7_attention_weight_min": -73.19161224365234,
      "activations/layer8_attention_weight_max": 45.88078689575195,
      "activations/layer8_attention_weight_min": -46.59176254272461,
      "activations/layer9_attention_weight_max": 48.949485778808594,
      "activations/layer9_attention_weight_min": -49.99638366699219,
      "epoch": 9.61,
      "learning_rate": 8.889924242424241e-05,
      "loss": 2.8004,
      "step": 165450
    },
    {
      "activations/layer0_attention_weight_max": 16.0307674407959,
      "activations/layer0_attention_weight_min": -14.29881763458252,
      "activations/layer10_attention_weight_max": 39.578582763671875,
      "activations/layer10_attention_weight_min": -38.05308532714844,
      "activations/layer11_attention_weight_max": 34.180992126464844,
      "activations/layer11_attention_weight_min": -32.469261169433594,
      "activations/layer12_attention_weight_max": 21.9685001373291,
      "activations/layer12_attention_weight_min": -27.82435417175293,
      "activations/layer13_attention_weight_max": 47.37691116333008,
      "activations/layer13_attention_weight_min": -29.65706443786621,
      "activations/layer14_attention_weight_max": 40.38050079345703,
      "activations/layer14_attention_weight_min": -32.77918243408203,
      "activations/layer15_attention_weight_max": 34.61674880981445,
      "activations/layer15_attention_weight_min": -31.167373657226562,
      "activations/layer16_attention_weight_max": 30.272045135498047,
      "activations/layer16_attention_weight_min": -28.120031356811523,
      "activations/layer17_attention_weight_max": 48.411781311035156,
      "activations/layer17_attention_weight_min": -42.08933639526367,
      "activations/layer18_attention_weight_max": 45.512081146240234,
      "activations/layer18_attention_weight_min": -35.953243255615234,
      "activations/layer19_attention_weight_max": 21.660436630249023,
      "activations/layer19_attention_weight_min": -22.606176376342773,
      "activations/layer1_attention_weight_max": 16.06077766418457,
      "activations/layer1_attention_weight_min": -16.201980590820312,
      "activations/layer20_attention_weight_max": 22.605907440185547,
      "activations/layer20_attention_weight_min": -23.090126037597656,
      "activations/layer21_attention_weight_max": 34.39225769042969,
      "activations/layer21_attention_weight_min": -23.42428970336914,
      "activations/layer22_attention_weight_max": 29.574460983276367,
      "activations/layer22_attention_weight_min": -26.31720733642578,
      "activations/layer23_attention_weight_max": 34.48711395263672,
      "activations/layer23_attention_weight_min": -22.688199996948242,
      "activations/layer2_attention_weight_max": 34.29681396484375,
      "activations/layer2_attention_weight_min": -33.049861907958984,
      "activations/layer3_attention_weight_max": 96.29679107666016,
      "activations/layer3_attention_weight_min": -101.5976333618164,
      "activations/layer4_attention_weight_max": 108.02909088134766,
      "activations/layer4_attention_weight_min": -112.20592498779297,
      "activations/layer5_attention_weight_max": 84.1785888671875,
      "activations/layer5_attention_weight_min": -82.77723693847656,
      "activations/layer6_attention_weight_max": 61.24420928955078,
      "activations/layer6_attention_weight_min": -58.338584899902344,
      "activations/layer7_attention_weight_max": 81.3647232055664,
      "activations/layer7_attention_weight_min": -79.2022933959961,
      "activations/layer8_attention_weight_max": 51.58415603637695,
      "activations/layer8_attention_weight_min": -52.15340042114258,
      "activations/layer9_attention_weight_max": 53.365142822265625,
      "activations/layer9_attention_weight_min": -52.835174560546875,
      "epoch": 9.62,
      "learning_rate": 8.888030303030303e-05,
      "loss": 2.7939,
      "step": 165500
    },
    {
      "activations/layer0_attention_weight_max": 14.76002311706543,
      "activations/layer0_attention_weight_min": -14.311823844909668,
      "activations/layer10_attention_weight_max": 36.777103424072266,
      "activations/layer10_attention_weight_min": -36.99406814575195,
      "activations/layer11_attention_weight_max": 31.552120208740234,
      "activations/layer11_attention_weight_min": -31.33485221862793,
      "activations/layer12_attention_weight_max": 19.2147159576416,
      "activations/layer12_attention_weight_min": -26.593849182128906,
      "activations/layer13_attention_weight_max": 42.36791229248047,
      "activations/layer13_attention_weight_min": -32.2789421081543,
      "activations/layer14_attention_weight_max": 37.02829360961914,
      "activations/layer14_attention_weight_min": -30.675439834594727,
      "activations/layer15_attention_weight_max": 33.75715637207031,
      "activations/layer15_attention_weight_min": -29.427499771118164,
      "activations/layer16_attention_weight_max": 31.23059844970703,
      "activations/layer16_attention_weight_min": -29.242753982543945,
      "activations/layer17_attention_weight_max": 54.03145217895508,
      "activations/layer17_attention_weight_min": -43.69551467895508,
      "activations/layer18_attention_weight_max": 49.27659225463867,
      "activations/layer18_attention_weight_min": -40.27749252319336,
      "activations/layer19_attention_weight_max": 22.490440368652344,
      "activations/layer19_attention_weight_min": -24.25454330444336,
      "activations/layer1_attention_weight_max": 17.731950759887695,
      "activations/layer1_attention_weight_min": -15.80471420288086,
      "activations/layer20_attention_weight_max": 21.773832321166992,
      "activations/layer20_attention_weight_min": -22.439735412597656,
      "activations/layer21_attention_weight_max": 40.172950744628906,
      "activations/layer21_attention_weight_min": -26.020891189575195,
      "activations/layer22_attention_weight_max": 30.61859703063965,
      "activations/layer22_attention_weight_min": -24.990266799926758,
      "activations/layer23_attention_weight_max": 39.691429138183594,
      "activations/layer23_attention_weight_min": -26.52413558959961,
      "activations/layer2_attention_weight_max": 33.250709533691406,
      "activations/layer2_attention_weight_min": -31.71912384033203,
      "activations/layer3_attention_weight_max": 93.36988067626953,
      "activations/layer3_attention_weight_min": -98.63623809814453,
      "activations/layer4_attention_weight_max": 106.44771575927734,
      "activations/layer4_attention_weight_min": -105.04095458984375,
      "activations/layer5_attention_weight_max": 82.87007141113281,
      "activations/layer5_attention_weight_min": -82.98020935058594,
      "activations/layer6_attention_weight_max": 58.78649139404297,
      "activations/layer6_attention_weight_min": -57.20610046386719,
      "activations/layer7_attention_weight_max": 76.87491607666016,
      "activations/layer7_attention_weight_min": -80.7259292602539,
      "activations/layer8_attention_weight_max": 46.03961944580078,
      "activations/layer8_attention_weight_min": -49.388092041015625,
      "activations/layer9_attention_weight_max": 49.166744232177734,
      "activations/layer9_attention_weight_min": -52.1543083190918,
      "epoch": 9.62,
      "learning_rate": 8.886136363636363e-05,
      "loss": 2.7994,
      "step": 165550
    },
    {
      "activations/layer0_attention_weight_max": 16.780059814453125,
      "activations/layer0_attention_weight_min": -14.034873008728027,
      "activations/layer10_attention_weight_max": 36.455291748046875,
      "activations/layer10_attention_weight_min": -36.70367431640625,
      "activations/layer11_attention_weight_max": 29.62085723876953,
      "activations/layer11_attention_weight_min": -31.8487606048584,
      "activations/layer12_attention_weight_max": 18.7430419921875,
      "activations/layer12_attention_weight_min": -30.660058975219727,
      "activations/layer13_attention_weight_max": 48.08810043334961,
      "activations/layer13_attention_weight_min": -34.75300598144531,
      "activations/layer14_attention_weight_max": 40.10160446166992,
      "activations/layer14_attention_weight_min": -31.696582794189453,
      "activations/layer15_attention_weight_max": 36.340293884277344,
      "activations/layer15_attention_weight_min": -33.007118225097656,
      "activations/layer16_attention_weight_max": 36.46755599975586,
      "activations/layer16_attention_weight_min": -30.22278594970703,
      "activations/layer17_attention_weight_max": 53.73109436035156,
      "activations/layer17_attention_weight_min": -45.40766525268555,
      "activations/layer18_attention_weight_max": 50.71744155883789,
      "activations/layer18_attention_weight_min": -37.338584899902344,
      "activations/layer19_attention_weight_max": 22.097280502319336,
      "activations/layer19_attention_weight_min": -22.151626586914062,
      "activations/layer1_attention_weight_max": 17.616533279418945,
      "activations/layer1_attention_weight_min": -14.01476764678955,
      "activations/layer20_attention_weight_max": 23.521486282348633,
      "activations/layer20_attention_weight_min": -21.06148910522461,
      "activations/layer21_attention_weight_max": 46.050018310546875,
      "activations/layer21_attention_weight_min": -25.08165740966797,
      "activations/layer22_attention_weight_max": 32.89750289916992,
      "activations/layer22_attention_weight_min": -25.977569580078125,
      "activations/layer23_attention_weight_max": 37.60060501098633,
      "activations/layer23_attention_weight_min": -21.367023468017578,
      "activations/layer2_attention_weight_max": 30.65038299560547,
      "activations/layer2_attention_weight_min": -29.653575897216797,
      "activations/layer3_attention_weight_max": 91.55712890625,
      "activations/layer3_attention_weight_min": -98.5010986328125,
      "activations/layer4_attention_weight_max": 105.17860412597656,
      "activations/layer4_attention_weight_min": -98.35076141357422,
      "activations/layer5_attention_weight_max": 81.01608276367188,
      "activations/layer5_attention_weight_min": -78.46826934814453,
      "activations/layer6_attention_weight_max": 55.612205505371094,
      "activations/layer6_attention_weight_min": -54.01425552368164,
      "activations/layer7_attention_weight_max": 78.83917999267578,
      "activations/layer7_attention_weight_min": -80.98812103271484,
      "activations/layer8_attention_weight_max": 50.31865310668945,
      "activations/layer8_attention_weight_min": -48.013702392578125,
      "activations/layer9_attention_weight_max": 50.42436981201172,
      "activations/layer9_attention_weight_min": -51.690608978271484,
      "epoch": 9.62,
      "learning_rate": 8.884242424242423e-05,
      "loss": 2.7976,
      "step": 165600
    },
    {
      "activations/layer0_attention_weight_max": 16.387407302856445,
      "activations/layer0_attention_weight_min": -13.608007431030273,
      "activations/layer10_attention_weight_max": 37.308380126953125,
      "activations/layer10_attention_weight_min": -37.55224609375,
      "activations/layer11_attention_weight_max": 33.91910171508789,
      "activations/layer11_attention_weight_min": -33.782875061035156,
      "activations/layer12_attention_weight_max": 17.2202091217041,
      "activations/layer12_attention_weight_min": -32.81597137451172,
      "activations/layer13_attention_weight_max": 37.02492141723633,
      "activations/layer13_attention_weight_min": -29.316864013671875,
      "activations/layer14_attention_weight_max": 37.76835632324219,
      "activations/layer14_attention_weight_min": -31.092361450195312,
      "activations/layer15_attention_weight_max": 31.469621658325195,
      "activations/layer15_attention_weight_min": -29.907455444335938,
      "activations/layer16_attention_weight_max": 29.397829055786133,
      "activations/layer16_attention_weight_min": -28.075061798095703,
      "activations/layer17_attention_weight_max": 49.41162109375,
      "activations/layer17_attention_weight_min": -41.99691390991211,
      "activations/layer18_attention_weight_max": 43.13310241699219,
      "activations/layer18_attention_weight_min": -35.44099044799805,
      "activations/layer19_attention_weight_max": 21.197431564331055,
      "activations/layer19_attention_weight_min": -19.036113739013672,
      "activations/layer1_attention_weight_max": 17.010143280029297,
      "activations/layer1_attention_weight_min": -16.682737350463867,
      "activations/layer20_attention_weight_max": 20.86009407043457,
      "activations/layer20_attention_weight_min": -19.78537368774414,
      "activations/layer21_attention_weight_max": 36.961997985839844,
      "activations/layer21_attention_weight_min": -20.77289390563965,
      "activations/layer22_attention_weight_max": 30.916439056396484,
      "activations/layer22_attention_weight_min": -23.681381225585938,
      "activations/layer23_attention_weight_max": 31.8635311126709,
      "activations/layer23_attention_weight_min": -21.386089324951172,
      "activations/layer2_attention_weight_max": 32.27756881713867,
      "activations/layer2_attention_weight_min": -29.491437911987305,
      "activations/layer3_attention_weight_max": 95.44921112060547,
      "activations/layer3_attention_weight_min": -94.0853500366211,
      "activations/layer4_attention_weight_max": 105.00318908691406,
      "activations/layer4_attention_weight_min": -96.44164276123047,
      "activations/layer5_attention_weight_max": 74.51303100585938,
      "activations/layer5_attention_weight_min": -81.64190673828125,
      "activations/layer6_attention_weight_max": 55.04716873168945,
      "activations/layer6_attention_weight_min": -55.94445037841797,
      "activations/layer7_attention_weight_max": 80.52446746826172,
      "activations/layer7_attention_weight_min": -77.01760864257812,
      "activations/layer8_attention_weight_max": 44.897762298583984,
      "activations/layer8_attention_weight_min": -45.96121597290039,
      "activations/layer9_attention_weight_max": 56.72761917114258,
      "activations/layer9_attention_weight_min": -55.03225326538086,
      "epoch": 9.63,
      "learning_rate": 8.882386363636362e-05,
      "loss": 2.7921,
      "step": 165650
    },
    {
      "activations/layer0_attention_weight_max": 15.266300201416016,
      "activations/layer0_attention_weight_min": -14.331195831298828,
      "activations/layer10_attention_weight_max": 34.589385986328125,
      "activations/layer10_attention_weight_min": -34.665122985839844,
      "activations/layer11_attention_weight_max": 31.155668258666992,
      "activations/layer11_attention_weight_min": -30.873226165771484,
      "activations/layer12_attention_weight_max": 18.13532066345215,
      "activations/layer12_attention_weight_min": -25.11444854736328,
      "activations/layer13_attention_weight_max": 37.64716720581055,
      "activations/layer13_attention_weight_min": -30.5128116607666,
      "activations/layer14_attention_weight_max": 35.49741744995117,
      "activations/layer14_attention_weight_min": -28.8063907623291,
      "activations/layer15_attention_weight_max": 31.766963958740234,
      "activations/layer15_attention_weight_min": -28.223787307739258,
      "activations/layer16_attention_weight_max": 31.511302947998047,
      "activations/layer16_attention_weight_min": -28.677804946899414,
      "activations/layer17_attention_weight_max": 50.90336990356445,
      "activations/layer17_attention_weight_min": -42.19359588623047,
      "activations/layer18_attention_weight_max": 47.08521270751953,
      "activations/layer18_attention_weight_min": -35.8123664855957,
      "activations/layer19_attention_weight_max": 21.61513328552246,
      "activations/layer19_attention_weight_min": -21.669315338134766,
      "activations/layer1_attention_weight_max": 17.122400283813477,
      "activations/layer1_attention_weight_min": -16.64990234375,
      "activations/layer20_attention_weight_max": 21.490276336669922,
      "activations/layer20_attention_weight_min": -22.029272079467773,
      "activations/layer21_attention_weight_max": 38.31873321533203,
      "activations/layer21_attention_weight_min": -23.65338134765625,
      "activations/layer22_attention_weight_max": 32.37179946899414,
      "activations/layer22_attention_weight_min": -24.56134033203125,
      "activations/layer23_attention_weight_max": 37.24946594238281,
      "activations/layer23_attention_weight_min": -23.00796890258789,
      "activations/layer2_attention_weight_max": 33.392181396484375,
      "activations/layer2_attention_weight_min": -32.37397003173828,
      "activations/layer3_attention_weight_max": 94.01241302490234,
      "activations/layer3_attention_weight_min": -101.32551574707031,
      "activations/layer4_attention_weight_max": 99.88896942138672,
      "activations/layer4_attention_weight_min": -96.2301254272461,
      "activations/layer5_attention_weight_max": 78.53462219238281,
      "activations/layer5_attention_weight_min": -78.18824768066406,
      "activations/layer6_attention_weight_max": 54.92298889160156,
      "activations/layer6_attention_weight_min": -51.76653289794922,
      "activations/layer7_attention_weight_max": 78.12113952636719,
      "activations/layer7_attention_weight_min": -72.337158203125,
      "activations/layer8_attention_weight_max": 44.85261917114258,
      "activations/layer8_attention_weight_min": -46.73677444458008,
      "activations/layer9_attention_weight_max": 47.54995346069336,
      "activations/layer9_attention_weight_min": -46.371036529541016,
      "epoch": 9.63,
      "learning_rate": 8.880492424242424e-05,
      "loss": 2.8087,
      "step": 165700
    },
    {
      "activations/layer0_attention_weight_max": 15.195564270019531,
      "activations/layer0_attention_weight_min": -14.168514251708984,
      "activations/layer10_attention_weight_max": 40.846397399902344,
      "activations/layer10_attention_weight_min": -36.904335021972656,
      "activations/layer11_attention_weight_max": 35.14284896850586,
      "activations/layer11_attention_weight_min": -33.742794036865234,
      "activations/layer12_attention_weight_max": 21.704750061035156,
      "activations/layer12_attention_weight_min": -26.52487564086914,
      "activations/layer13_attention_weight_max": 40.322044372558594,
      "activations/layer13_attention_weight_min": -33.71112060546875,
      "activations/layer14_attention_weight_max": 44.34788513183594,
      "activations/layer14_attention_weight_min": -31.481489181518555,
      "activations/layer15_attention_weight_max": 36.26156234741211,
      "activations/layer15_attention_weight_min": -32.32758712768555,
      "activations/layer16_attention_weight_max": 35.8121337890625,
      "activations/layer16_attention_weight_min": -28.00385284423828,
      "activations/layer17_attention_weight_max": 52.95298767089844,
      "activations/layer17_attention_weight_min": -46.43021011352539,
      "activations/layer18_attention_weight_max": 49.60997772216797,
      "activations/layer18_attention_weight_min": -40.12324142456055,
      "activations/layer19_attention_weight_max": 26.003637313842773,
      "activations/layer19_attention_weight_min": -25.833585739135742,
      "activations/layer1_attention_weight_max": 16.839981079101562,
      "activations/layer1_attention_weight_min": -16.335651397705078,
      "activations/layer20_attention_weight_max": 25.15871810913086,
      "activations/layer20_attention_weight_min": -24.079174041748047,
      "activations/layer21_attention_weight_max": 42.65060043334961,
      "activations/layer21_attention_weight_min": -23.9565372467041,
      "activations/layer22_attention_weight_max": 34.3315315246582,
      "activations/layer22_attention_weight_min": -28.01177406311035,
      "activations/layer23_attention_weight_max": 51.198890686035156,
      "activations/layer23_attention_weight_min": -23.832082748413086,
      "activations/layer2_attention_weight_max": 32.604576110839844,
      "activations/layer2_attention_weight_min": -30.519390106201172,
      "activations/layer3_attention_weight_max": 93.55238342285156,
      "activations/layer3_attention_weight_min": -99.49222564697266,
      "activations/layer4_attention_weight_max": 106.08562469482422,
      "activations/layer4_attention_weight_min": -105.1713638305664,
      "activations/layer5_attention_weight_max": 81.293701171875,
      "activations/layer5_attention_weight_min": -78.42204284667969,
      "activations/layer6_attention_weight_max": 56.664939880371094,
      "activations/layer6_attention_weight_min": -55.96924591064453,
      "activations/layer7_attention_weight_max": 85.1376953125,
      "activations/layer7_attention_weight_min": -78.2197494506836,
      "activations/layer8_attention_weight_max": 51.83354568481445,
      "activations/layer8_attention_weight_min": -48.59727096557617,
      "activations/layer9_attention_weight_max": 58.48958969116211,
      "activations/layer9_attention_weight_min": -50.16575241088867,
      "epoch": 9.63,
      "learning_rate": 8.878598484848484e-05,
      "loss": 2.7969,
      "step": 165750
    },
    {
      "activations/layer0_attention_weight_max": 14.68023681640625,
      "activations/layer0_attention_weight_min": -14.255176544189453,
      "activations/layer10_attention_weight_max": 39.849910736083984,
      "activations/layer10_attention_weight_min": -38.906673431396484,
      "activations/layer11_attention_weight_max": 33.399330139160156,
      "activations/layer11_attention_weight_min": -35.20614242553711,
      "activations/layer12_attention_weight_max": 17.43337631225586,
      "activations/layer12_attention_weight_min": -26.391279220581055,
      "activations/layer13_attention_weight_max": 37.387969970703125,
      "activations/layer13_attention_weight_min": -30.579988479614258,
      "activations/layer14_attention_weight_max": 37.52772903442383,
      "activations/layer14_attention_weight_min": -31.460329055786133,
      "activations/layer15_attention_weight_max": 35.50227355957031,
      "activations/layer15_attention_weight_min": -31.013792037963867,
      "activations/layer16_attention_weight_max": 33.23884201049805,
      "activations/layer16_attention_weight_min": -28.548255920410156,
      "activations/layer17_attention_weight_max": 55.63075637817383,
      "activations/layer17_attention_weight_min": -42.70790100097656,
      "activations/layer18_attention_weight_max": 44.770206451416016,
      "activations/layer18_attention_weight_min": -38.92047119140625,
      "activations/layer19_attention_weight_max": 21.57452392578125,
      "activations/layer19_attention_weight_min": -22.54008674621582,
      "activations/layer1_attention_weight_max": 16.448841094970703,
      "activations/layer1_attention_weight_min": -13.69414234161377,
      "activations/layer20_attention_weight_max": 20.68291664123535,
      "activations/layer20_attention_weight_min": -21.872270584106445,
      "activations/layer21_attention_weight_max": 34.9649543762207,
      "activations/layer21_attention_weight_min": -21.751928329467773,
      "activations/layer22_attention_weight_max": 31.408538818359375,
      "activations/layer22_attention_weight_min": -25.677045822143555,
      "activations/layer23_attention_weight_max": 38.14875030517578,
      "activations/layer23_attention_weight_min": -22.756465911865234,
      "activations/layer2_attention_weight_max": 32.0582160949707,
      "activations/layer2_attention_weight_min": -29.657485961914062,
      "activations/layer3_attention_weight_max": 99.07003784179688,
      "activations/layer3_attention_weight_min": -95.52452087402344,
      "activations/layer4_attention_weight_max": 106.01335906982422,
      "activations/layer4_attention_weight_min": -100.47235107421875,
      "activations/layer5_attention_weight_max": 85.23049926757812,
      "activations/layer5_attention_weight_min": -82.17581176757812,
      "activations/layer6_attention_weight_max": 56.69115447998047,
      "activations/layer6_attention_weight_min": -58.37065124511719,
      "activations/layer7_attention_weight_max": 77.60990142822266,
      "activations/layer7_attention_weight_min": -83.67884063720703,
      "activations/layer8_attention_weight_max": 49.99226760864258,
      "activations/layer8_attention_weight_min": -50.451133728027344,
      "activations/layer9_attention_weight_max": 54.1872444152832,
      "activations/layer9_attention_weight_min": -51.26430130004883,
      "epoch": 9.63,
      "learning_rate": 8.876704545454545e-05,
      "loss": 2.7875,
      "step": 165800
    },
    {
      "activations/layer0_attention_weight_max": 14.880845069885254,
      "activations/layer0_attention_weight_min": -14.089635848999023,
      "activations/layer10_attention_weight_max": 36.15326690673828,
      "activations/layer10_attention_weight_min": -35.103126525878906,
      "activations/layer11_attention_weight_max": 31.79011344909668,
      "activations/layer11_attention_weight_min": -30.73166275024414,
      "activations/layer12_attention_weight_max": 17.33174705505371,
      "activations/layer12_attention_weight_min": -25.45040512084961,
      "activations/layer13_attention_weight_max": 44.8110237121582,
      "activations/layer13_attention_weight_min": -32.36683654785156,
      "activations/layer14_attention_weight_max": 37.74713897705078,
      "activations/layer14_attention_weight_min": -31.892410278320312,
      "activations/layer15_attention_weight_max": 34.80558395385742,
      "activations/layer15_attention_weight_min": -30.88530158996582,
      "activations/layer16_attention_weight_max": 31.537248611450195,
      "activations/layer16_attention_weight_min": -31.663068771362305,
      "activations/layer17_attention_weight_max": 54.3200569152832,
      "activations/layer17_attention_weight_min": -48.40349197387695,
      "activations/layer18_attention_weight_max": 44.576595306396484,
      "activations/layer18_attention_weight_min": -37.83599090576172,
      "activations/layer19_attention_weight_max": 21.462629318237305,
      "activations/layer19_attention_weight_min": -23.192554473876953,
      "activations/layer1_attention_weight_max": 17.01382064819336,
      "activations/layer1_attention_weight_min": -19.341880798339844,
      "activations/layer20_attention_weight_max": 20.990327835083008,
      "activations/layer20_attention_weight_min": -20.759262084960938,
      "activations/layer21_attention_weight_max": 41.38142776489258,
      "activations/layer21_attention_weight_min": -27.170196533203125,
      "activations/layer22_attention_weight_max": 29.494998931884766,
      "activations/layer22_attention_weight_min": -30.511507034301758,
      "activations/layer23_attention_weight_max": 34.61473083496094,
      "activations/layer23_attention_weight_min": -24.23830795288086,
      "activations/layer2_attention_weight_max": 33.30241775512695,
      "activations/layer2_attention_weight_min": -31.095924377441406,
      "activations/layer3_attention_weight_max": 94.9870834350586,
      "activations/layer3_attention_weight_min": -101.02395629882812,
      "activations/layer4_attention_weight_max": 105.88179016113281,
      "activations/layer4_attention_weight_min": -101.66259002685547,
      "activations/layer5_attention_weight_max": 79.82089233398438,
      "activations/layer5_attention_weight_min": -86.34041595458984,
      "activations/layer6_attention_weight_max": 56.38420486450195,
      "activations/layer6_attention_weight_min": -57.58439636230469,
      "activations/layer7_attention_weight_max": 80.49361419677734,
      "activations/layer7_attention_weight_min": -75.60100555419922,
      "activations/layer8_attention_weight_max": 45.02870559692383,
      "activations/layer8_attention_weight_min": -47.506065368652344,
      "activations/layer9_attention_weight_max": 48.65256118774414,
      "activations/layer9_attention_weight_min": -48.9848518371582,
      "epoch": 9.64,
      "learning_rate": 8.874810606060604e-05,
      "loss": 2.7999,
      "step": 165850
    },
    {
      "activations/layer0_attention_weight_max": 14.404389381408691,
      "activations/layer0_attention_weight_min": -14.296953201293945,
      "activations/layer10_attention_weight_max": 38.53917694091797,
      "activations/layer10_attention_weight_min": -36.52202224731445,
      "activations/layer11_attention_weight_max": 32.060516357421875,
      "activations/layer11_attention_weight_min": -32.954742431640625,
      "activations/layer12_attention_weight_max": 17.602802276611328,
      "activations/layer12_attention_weight_min": -23.57939910888672,
      "activations/layer13_attention_weight_max": 51.28401565551758,
      "activations/layer13_attention_weight_min": -34.16870880126953,
      "activations/layer14_attention_weight_max": 48.700557708740234,
      "activations/layer14_attention_weight_min": -31.20195960998535,
      "activations/layer15_attention_weight_max": 38.07938766479492,
      "activations/layer15_attention_weight_min": -31.00087547302246,
      "activations/layer16_attention_weight_max": 34.204429626464844,
      "activations/layer16_attention_weight_min": -29.23505973815918,
      "activations/layer17_attention_weight_max": 55.91877365112305,
      "activations/layer17_attention_weight_min": -43.970237731933594,
      "activations/layer18_attention_weight_max": 47.427974700927734,
      "activations/layer18_attention_weight_min": -37.56904220581055,
      "activations/layer19_attention_weight_max": 23.6494197845459,
      "activations/layer19_attention_weight_min": -23.58212661743164,
      "activations/layer1_attention_weight_max": 18.012948989868164,
      "activations/layer1_attention_weight_min": -16.38731575012207,
      "activations/layer20_attention_weight_max": 21.349239349365234,
      "activations/layer20_attention_weight_min": -20.690738677978516,
      "activations/layer21_attention_weight_max": 42.762935638427734,
      "activations/layer21_attention_weight_min": -22.108144760131836,
      "activations/layer22_attention_weight_max": 32.84386444091797,
      "activations/layer22_attention_weight_min": -25.70125961303711,
      "activations/layer23_attention_weight_max": 41.39677429199219,
      "activations/layer23_attention_weight_min": -22.678348541259766,
      "activations/layer2_attention_weight_max": 34.50714111328125,
      "activations/layer2_attention_weight_min": -32.246788024902344,
      "activations/layer3_attention_weight_max": 98.0997543334961,
      "activations/layer3_attention_weight_min": -97.81425476074219,
      "activations/layer4_attention_weight_max": 107.75933837890625,
      "activations/layer4_attention_weight_min": -104.8283920288086,
      "activations/layer5_attention_weight_max": 80.89984130859375,
      "activations/layer5_attention_weight_min": -82.84886169433594,
      "activations/layer6_attention_weight_max": 57.18330764770508,
      "activations/layer6_attention_weight_min": -55.17512893676758,
      "activations/layer7_attention_weight_max": 80.00157928466797,
      "activations/layer7_attention_weight_min": -78.10823822021484,
      "activations/layer8_attention_weight_max": 46.899757385253906,
      "activations/layer8_attention_weight_min": -48.756954193115234,
      "activations/layer9_attention_weight_max": 52.098140716552734,
      "activations/layer9_attention_weight_min": -51.221588134765625,
      "epoch": 9.64,
      "learning_rate": 8.872916666666666e-05,
      "loss": 2.7929,
      "step": 165900
    },
    {
      "activations/layer0_attention_weight_max": 14.6251802444458,
      "activations/layer0_attention_weight_min": -14.730518341064453,
      "activations/layer10_attention_weight_max": 37.9913330078125,
      "activations/layer10_attention_weight_min": -34.818241119384766,
      "activations/layer11_attention_weight_max": 31.57900047302246,
      "activations/layer11_attention_weight_min": -32.422767639160156,
      "activations/layer12_attention_weight_max": 17.253498077392578,
      "activations/layer12_attention_weight_min": -30.052196502685547,
      "activations/layer13_attention_weight_max": 44.983760833740234,
      "activations/layer13_attention_weight_min": -34.46997833251953,
      "activations/layer14_attention_weight_max": 53.67465591430664,
      "activations/layer14_attention_weight_min": -37.78004837036133,
      "activations/layer15_attention_weight_max": 44.921321868896484,
      "activations/layer15_attention_weight_min": -31.676664352416992,
      "activations/layer16_attention_weight_max": 30.61714744567871,
      "activations/layer16_attention_weight_min": -28.057361602783203,
      "activations/layer17_attention_weight_max": 52.43859100341797,
      "activations/layer17_attention_weight_min": -41.265106201171875,
      "activations/layer18_attention_weight_max": 43.9859504699707,
      "activations/layer18_attention_weight_min": -35.649173736572266,
      "activations/layer19_attention_weight_max": 23.252294540405273,
      "activations/layer19_attention_weight_min": -22.46385955810547,
      "activations/layer1_attention_weight_max": 16.257699966430664,
      "activations/layer1_attention_weight_min": -17.622291564941406,
      "activations/layer20_attention_weight_max": 23.809282302856445,
      "activations/layer20_attention_weight_min": -22.767559051513672,
      "activations/layer21_attention_weight_max": 40.81817626953125,
      "activations/layer21_attention_weight_min": -23.27007293701172,
      "activations/layer22_attention_weight_max": 31.86052703857422,
      "activations/layer22_attention_weight_min": -24.710710525512695,
      "activations/layer23_attention_weight_max": 37.50875473022461,
      "activations/layer23_attention_weight_min": -22.13715934753418,
      "activations/layer2_attention_weight_max": 32.126731872558594,
      "activations/layer2_attention_weight_min": -31.198657989501953,
      "activations/layer3_attention_weight_max": 98.27642059326172,
      "activations/layer3_attention_weight_min": -98.15814208984375,
      "activations/layer4_attention_weight_max": 104.65184783935547,
      "activations/layer4_attention_weight_min": -99.15126037597656,
      "activations/layer5_attention_weight_max": 78.93653869628906,
      "activations/layer5_attention_weight_min": -81.57205200195312,
      "activations/layer6_attention_weight_max": 55.24921417236328,
      "activations/layer6_attention_weight_min": -55.6689453125,
      "activations/layer7_attention_weight_max": 77.3741226196289,
      "activations/layer7_attention_weight_min": -76.6347427368164,
      "activations/layer8_attention_weight_max": 44.65780258178711,
      "activations/layer8_attention_weight_min": -47.968238830566406,
      "activations/layer9_attention_weight_max": 48.89597702026367,
      "activations/layer9_attention_weight_min": -47.74921417236328,
      "epoch": 9.64,
      "learning_rate": 8.871022727272727e-05,
      "loss": 2.7971,
      "step": 165950
    },
    {
      "activations/layer0_attention_weight_max": 15.762022972106934,
      "activations/layer0_attention_weight_min": -14.179545402526855,
      "activations/layer10_attention_weight_max": 37.308204650878906,
      "activations/layer10_attention_weight_min": -36.90570068359375,
      "activations/layer11_attention_weight_max": 32.407474517822266,
      "activations/layer11_attention_weight_min": -33.42058563232422,
      "activations/layer12_attention_weight_max": 19.83104133605957,
      "activations/layer12_attention_weight_min": -27.394010543823242,
      "activations/layer13_attention_weight_max": 48.240562438964844,
      "activations/layer13_attention_weight_min": -37.347103118896484,
      "activations/layer14_attention_weight_max": 51.26537322998047,
      "activations/layer14_attention_weight_min": -37.829261779785156,
      "activations/layer15_attention_weight_max": 39.997982025146484,
      "activations/layer15_attention_weight_min": -30.841455459594727,
      "activations/layer16_attention_weight_max": 31.458940505981445,
      "activations/layer16_attention_weight_min": -30.866947174072266,
      "activations/layer17_attention_weight_max": 55.5034065246582,
      "activations/layer17_attention_weight_min": -46.034584045410156,
      "activations/layer18_attention_weight_max": 48.187164306640625,
      "activations/layer18_attention_weight_min": -41.25043869018555,
      "activations/layer19_attention_weight_max": 22.342453002929688,
      "activations/layer19_attention_weight_min": -25.030136108398438,
      "activations/layer1_attention_weight_max": 17.037202835083008,
      "activations/layer1_attention_weight_min": -16.10715103149414,
      "activations/layer20_attention_weight_max": 23.471614837646484,
      "activations/layer20_attention_weight_min": -24.704252243041992,
      "activations/layer21_attention_weight_max": 40.659759521484375,
      "activations/layer21_attention_weight_min": -24.88957977294922,
      "activations/layer22_attention_weight_max": 34.773231506347656,
      "activations/layer22_attention_weight_min": -24.104448318481445,
      "activations/layer23_attention_weight_max": 45.487327575683594,
      "activations/layer23_attention_weight_min": -26.97743034362793,
      "activations/layer2_attention_weight_max": 32.462833404541016,
      "activations/layer2_attention_weight_min": -31.390056610107422,
      "activations/layer3_attention_weight_max": 92.27742004394531,
      "activations/layer3_attention_weight_min": -92.2112045288086,
      "activations/layer4_attention_weight_max": 104.69339752197266,
      "activations/layer4_attention_weight_min": -98.97149658203125,
      "activations/layer5_attention_weight_max": 81.67472839355469,
      "activations/layer5_attention_weight_min": -79.64765167236328,
      "activations/layer6_attention_weight_max": 56.16607666015625,
      "activations/layer6_attention_weight_min": -54.13698959350586,
      "activations/layer7_attention_weight_max": 80.04818725585938,
      "activations/layer7_attention_weight_min": -78.7118148803711,
      "activations/layer8_attention_weight_max": 46.25373458862305,
      "activations/layer8_attention_weight_min": -49.199092864990234,
      "activations/layer9_attention_weight_max": 51.42499923706055,
      "activations/layer9_attention_weight_min": -49.512203216552734,
      "epoch": 9.65,
      "learning_rate": 8.869128787878786e-05,
      "loss": 2.7734,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.5023,
      "eval_samples_per_second": 505.043,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.5023,
      "eval_openwebtext_samples_per_second": 505.043,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_wikitext_loss": 2.982421875,
      "eval_wikitext_ppl": 19.735555861572347,
      "eval_wikitext_runtime": 2.0161,
      "eval_wikitext_samples_per_second": 226.178,
      "step": 166000
    },
    {
      "epoch": 9.65,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.5819,
      "eval_lambada_samples_per_second": 508.146,
      "step": 166000
    },
    {
      "activations/layer0_attention_weight_max": 14.941959381103516,
      "activations/layer0_attention_weight_min": -14.198932647705078,
      "activations/layer10_attention_weight_max": 44.46831130981445,
      "activations/layer10_attention_weight_min": -41.550418853759766,
      "activations/layer11_attention_weight_max": 41.40452194213867,
      "activations/layer11_attention_weight_min": -35.24980163574219,
      "activations/layer12_attention_weight_max": 20.95833969116211,
      "activations/layer12_attention_weight_min": -26.962099075317383,
      "activations/layer13_attention_weight_max": 40.966670989990234,
      "activations/layer13_attention_weight_min": -28.96834373474121,
      "activations/layer14_attention_weight_max": 40.20806121826172,
      "activations/layer14_attention_weight_min": -32.24029541015625,
      "activations/layer15_attention_weight_max": 31.773439407348633,
      "activations/layer15_attention_weight_min": -28.054723739624023,
      "activations/layer16_attention_weight_max": 31.217199325561523,
      "activations/layer16_attention_weight_min": -31.064123153686523,
      "activations/layer17_attention_weight_max": 49.88092803955078,
      "activations/layer17_attention_weight_min": -46.97246170043945,
      "activations/layer18_attention_weight_max": 44.63228988647461,
      "activations/layer18_attention_weight_min": -40.32917785644531,
      "activations/layer19_attention_weight_max": 20.77777671813965,
      "activations/layer19_attention_weight_min": -21.275501251220703,
      "activations/layer1_attention_weight_max": 17.687030792236328,
      "activations/layer1_attention_weight_min": -15.941739082336426,
      "activations/layer20_attention_weight_max": 20.887676239013672,
      "activations/layer20_attention_weight_min": -20.63846778869629,
      "activations/layer21_attention_weight_max": 35.90202713012695,
      "activations/layer21_attention_weight_min": -22.24030113220215,
      "activations/layer22_attention_weight_max": 31.121492385864258,
      "activations/layer22_attention_weight_min": -25.027423858642578,
      "activations/layer23_attention_weight_max": 33.16551971435547,
      "activations/layer23_attention_weight_min": -23.2791805267334,
      "activations/layer2_attention_weight_max": 33.91005325317383,
      "activations/layer2_attention_weight_min": -30.724098205566406,
      "activations/layer3_attention_weight_max": 91.49467468261719,
      "activations/layer3_attention_weight_min": -94.3786392211914,
      "activations/layer4_attention_weight_max": 99.06841278076172,
      "activations/layer4_attention_weight_min": -94.84725189208984,
      "activations/layer5_attention_weight_max": 77.27903747558594,
      "activations/layer5_attention_weight_min": -78.89014434814453,
      "activations/layer6_attention_weight_max": 56.19150161743164,
      "activations/layer6_attention_weight_min": -53.81997299194336,
      "activations/layer7_attention_weight_max": 93.65927124023438,
      "activations/layer7_attention_weight_min": -77.83516693115234,
      "activations/layer8_attention_weight_max": 55.597808837890625,
      "activations/layer8_attention_weight_min": -49.55036163330078,
      "activations/layer9_attention_weight_max": 70.30686950683594,
      "activations/layer9_attention_weight_min": -54.66780471801758,
      "epoch": 9.65,
      "learning_rate": 8.867234848484848e-05,
      "loss": 2.7966,
      "step": 166050
    },
    {
      "activations/layer0_attention_weight_max": 14.670713424682617,
      "activations/layer0_attention_weight_min": -14.173665046691895,
      "activations/layer10_attention_weight_max": 39.774906158447266,
      "activations/layer10_attention_weight_min": -39.11314010620117,
      "activations/layer11_attention_weight_max": 35.286163330078125,
      "activations/layer11_attention_weight_min": -32.587745666503906,
      "activations/layer12_attention_weight_max": 19.578895568847656,
      "activations/layer12_attention_weight_min": -25.558603286743164,
      "activations/layer13_attention_weight_max": 36.990989685058594,
      "activations/layer13_attention_weight_min": -28.50339126586914,
      "activations/layer14_attention_weight_max": 35.97048568725586,
      "activations/layer14_attention_weight_min": -29.814754486083984,
      "activations/layer15_attention_weight_max": 33.00567626953125,
      "activations/layer15_attention_weight_min": -28.305757522583008,
      "activations/layer16_attention_weight_max": 33.23211669921875,
      "activations/layer16_attention_weight_min": -28.465648651123047,
      "activations/layer17_attention_weight_max": 50.00829315185547,
      "activations/layer17_attention_weight_min": -43.452552795410156,
      "activations/layer18_attention_weight_max": 45.197235107421875,
      "activations/layer18_attention_weight_min": -37.504547119140625,
      "activations/layer19_attention_weight_max": 20.278221130371094,
      "activations/layer19_attention_weight_min": -23.304208755493164,
      "activations/layer1_attention_weight_max": 17.796537399291992,
      "activations/layer1_attention_weight_min": -16.53757095336914,
      "activations/layer20_attention_weight_max": 21.07244110107422,
      "activations/layer20_attention_weight_min": -21.056825637817383,
      "activations/layer21_attention_weight_max": 38.883975982666016,
      "activations/layer21_attention_weight_min": -21.967309951782227,
      "activations/layer22_attention_weight_max": 28.735904693603516,
      "activations/layer22_attention_weight_min": -25.45987892150879,
      "activations/layer23_attention_weight_max": 34.58031463623047,
      "activations/layer23_attention_weight_min": -23.380802154541016,
      "activations/layer2_attention_weight_max": 35.75373840332031,
      "activations/layer2_attention_weight_min": -32.3422737121582,
      "activations/layer3_attention_weight_max": 99.78042602539062,
      "activations/layer3_attention_weight_min": -107.21780395507812,
      "activations/layer4_attention_weight_max": 110.89039611816406,
      "activations/layer4_attention_weight_min": -106.83160400390625,
      "activations/layer5_attention_weight_max": 79.86973571777344,
      "activations/layer5_attention_weight_min": -77.6123046875,
      "activations/layer6_attention_weight_max": 56.362701416015625,
      "activations/layer6_attention_weight_min": -54.1932487487793,
      "activations/layer7_attention_weight_max": 83.70016479492188,
      "activations/layer7_attention_weight_min": -80.92543029785156,
      "activations/layer8_attention_weight_max": 49.10301971435547,
      "activations/layer8_attention_weight_min": -47.98128128051758,
      "activations/layer9_attention_weight_max": 65.69100952148438,
      "activations/layer9_attention_weight_min": -59.850093841552734,
      "epoch": 9.65,
      "learning_rate": 8.865340909090909e-05,
      "loss": 2.794,
      "step": 166100
    },
    {
      "activations/layer0_attention_weight_max": 16.10126304626465,
      "activations/layer0_attention_weight_min": -14.382404327392578,
      "activations/layer10_attention_weight_max": 48.4215202331543,
      "activations/layer10_attention_weight_min": -41.97254943847656,
      "activations/layer11_attention_weight_max": 38.7145881652832,
      "activations/layer11_attention_weight_min": -37.31053161621094,
      "activations/layer12_attention_weight_max": 19.970829010009766,
      "activations/layer12_attention_weight_min": -26.46963119506836,
      "activations/layer13_attention_weight_max": 53.724674224853516,
      "activations/layer13_attention_weight_min": -34.97214126586914,
      "activations/layer14_attention_weight_max": 62.32161331176758,
      "activations/layer14_attention_weight_min": -32.668521881103516,
      "activations/layer15_attention_weight_max": 47.78000259399414,
      "activations/layer15_attention_weight_min": -30.77399253845215,
      "activations/layer16_attention_weight_max": 40.19081497192383,
      "activations/layer16_attention_weight_min": -29.371736526489258,
      "activations/layer17_attention_weight_max": 62.13198471069336,
      "activations/layer17_attention_weight_min": -45.55347442626953,
      "activations/layer18_attention_weight_max": 53.640438079833984,
      "activations/layer18_attention_weight_min": -38.30192565917969,
      "activations/layer19_attention_weight_max": 24.917692184448242,
      "activations/layer19_attention_weight_min": -22.853689193725586,
      "activations/layer1_attention_weight_max": 17.3615779876709,
      "activations/layer1_attention_weight_min": -15.79655933380127,
      "activations/layer20_attention_weight_max": 30.008466720581055,
      "activations/layer20_attention_weight_min": -22.010845184326172,
      "activations/layer21_attention_weight_max": 52.579063415527344,
      "activations/layer21_attention_weight_min": -24.583768844604492,
      "activations/layer22_attention_weight_max": 36.737918853759766,
      "activations/layer22_attention_weight_min": -24.876937866210938,
      "activations/layer23_attention_weight_max": 43.462730407714844,
      "activations/layer23_attention_weight_min": -25.995159149169922,
      "activations/layer2_attention_weight_max": 32.21131896972656,
      "activations/layer2_attention_weight_min": -30.27882194519043,
      "activations/layer3_attention_weight_max": 96.95670318603516,
      "activations/layer3_attention_weight_min": -95.12290954589844,
      "activations/layer4_attention_weight_max": 104.01246643066406,
      "activations/layer4_attention_weight_min": -95.11550903320312,
      "activations/layer5_attention_weight_max": 80.30960083007812,
      "activations/layer5_attention_weight_min": -78.90963745117188,
      "activations/layer6_attention_weight_max": 56.91761016845703,
      "activations/layer6_attention_weight_min": -55.06794738769531,
      "activations/layer7_attention_weight_max": 77.28641510009766,
      "activations/layer7_attention_weight_min": -84.18602752685547,
      "activations/layer8_attention_weight_max": 51.433345794677734,
      "activations/layer8_attention_weight_min": -49.762203216552734,
      "activations/layer9_attention_weight_max": 63.475433349609375,
      "activations/layer9_attention_weight_min": -56.3046989440918,
      "epoch": 9.65,
      "learning_rate": 8.863446969696968e-05,
      "loss": 2.7978,
      "step": 166150
    },
    {
      "activations/layer0_attention_weight_max": 14.942737579345703,
      "activations/layer0_attention_weight_min": -14.276976585388184,
      "activations/layer10_attention_weight_max": 35.01414108276367,
      "activations/layer10_attention_weight_min": -34.437339782714844,
      "activations/layer11_attention_weight_max": 32.28507614135742,
      "activations/layer11_attention_weight_min": -30.674232482910156,
      "activations/layer12_attention_weight_max": 19.122922897338867,
      "activations/layer12_attention_weight_min": -26.180973052978516,
      "activations/layer13_attention_weight_max": 39.00673294067383,
      "activations/layer13_attention_weight_min": -31.999971389770508,
      "activations/layer14_attention_weight_max": 35.993995666503906,
      "activations/layer14_attention_weight_min": -30.11602783203125,
      "activations/layer15_attention_weight_max": 35.00043487548828,
      "activations/layer15_attention_weight_min": -29.036325454711914,
      "activations/layer16_attention_weight_max": 30.161266326904297,
      "activations/layer16_attention_weight_min": -28.67020606994629,
      "activations/layer17_attention_weight_max": 50.10601043701172,
      "activations/layer17_attention_weight_min": -43.765445709228516,
      "activations/layer18_attention_weight_max": 43.05662536621094,
      "activations/layer18_attention_weight_min": -33.60019302368164,
      "activations/layer19_attention_weight_max": 21.037504196166992,
      "activations/layer19_attention_weight_min": -20.72052574157715,
      "activations/layer1_attention_weight_max": 16.34195899963379,
      "activations/layer1_attention_weight_min": -16.254934310913086,
      "activations/layer20_attention_weight_max": 23.36929702758789,
      "activations/layer20_attention_weight_min": -22.85600471496582,
      "activations/layer21_attention_weight_max": 38.95301055908203,
      "activations/layer21_attention_weight_min": -22.32927131652832,
      "activations/layer22_attention_weight_max": 32.54495620727539,
      "activations/layer22_attention_weight_min": -25.833332061767578,
      "activations/layer23_attention_weight_max": 34.700958251953125,
      "activations/layer23_attention_weight_min": -24.70697784423828,
      "activations/layer2_attention_weight_max": 31.83822250366211,
      "activations/layer2_attention_weight_min": -30.0166015625,
      "activations/layer3_attention_weight_max": 90.37630462646484,
      "activations/layer3_attention_weight_min": -88.822021484375,
      "activations/layer4_attention_weight_max": 100.29722595214844,
      "activations/layer4_attention_weight_min": -97.01415252685547,
      "activations/layer5_attention_weight_max": 78.73684692382812,
      "activations/layer5_attention_weight_min": -75.24525451660156,
      "activations/layer6_attention_weight_max": 55.851646423339844,
      "activations/layer6_attention_weight_min": -52.38871383666992,
      "activations/layer7_attention_weight_max": 78.38346099853516,
      "activations/layer7_attention_weight_min": -75.79039001464844,
      "activations/layer8_attention_weight_max": 43.84709930419922,
      "activations/layer8_attention_weight_min": -46.4322395324707,
      "activations/layer9_attention_weight_max": 46.10498809814453,
      "activations/layer9_attention_weight_min": -46.47064971923828,
      "epoch": 9.66,
      "learning_rate": 8.86155303030303e-05,
      "loss": 2.7971,
      "step": 166200
    },
    {
      "activations/layer0_attention_weight_max": 15.966021537780762,
      "activations/layer0_attention_weight_min": -14.016812324523926,
      "activations/layer10_attention_weight_max": 41.17243576049805,
      "activations/layer10_attention_weight_min": -36.518028259277344,
      "activations/layer11_attention_weight_max": 34.346092224121094,
      "activations/layer11_attention_weight_min": -33.33964920043945,
      "activations/layer12_attention_weight_max": 22.512874603271484,
      "activations/layer12_attention_weight_min": -22.936567306518555,
      "activations/layer13_attention_weight_max": 47.372581481933594,
      "activations/layer13_attention_weight_min": -29.90912628173828,
      "activations/layer14_attention_weight_max": 45.7587890625,
      "activations/layer14_attention_weight_min": -30.395130157470703,
      "activations/layer15_attention_weight_max": 40.17763137817383,
      "activations/layer15_attention_weight_min": -31.8944091796875,
      "activations/layer16_attention_weight_max": 36.763832092285156,
      "activations/layer16_attention_weight_min": -30.634899139404297,
      "activations/layer17_attention_weight_max": 57.42219543457031,
      "activations/layer17_attention_weight_min": -44.378292083740234,
      "activations/layer18_attention_weight_max": 50.26704025268555,
      "activations/layer18_attention_weight_min": -35.854068756103516,
      "activations/layer19_attention_weight_max": 23.966014862060547,
      "activations/layer19_attention_weight_min": -20.83320426940918,
      "activations/layer1_attention_weight_max": 17.33213996887207,
      "activations/layer1_attention_weight_min": -16.269649505615234,
      "activations/layer20_attention_weight_max": 23.676097869873047,
      "activations/layer20_attention_weight_min": -20.53036880493164,
      "activations/layer21_attention_weight_max": 42.419830322265625,
      "activations/layer21_attention_weight_min": -23.46086883544922,
      "activations/layer22_attention_weight_max": 35.3317756652832,
      "activations/layer22_attention_weight_min": -22.749359130859375,
      "activations/layer23_attention_weight_max": 43.36872482299805,
      "activations/layer23_attention_weight_min": -21.243043899536133,
      "activations/layer2_attention_weight_max": 33.13127136230469,
      "activations/layer2_attention_weight_min": -33.1452522277832,
      "activations/layer3_attention_weight_max": 99.24755096435547,
      "activations/layer3_attention_weight_min": -104.5660400390625,
      "activations/layer4_attention_weight_max": 108.75052642822266,
      "activations/layer4_attention_weight_min": -102.8357925415039,
      "activations/layer5_attention_weight_max": 80.63343811035156,
      "activations/layer5_attention_weight_min": -78.40819549560547,
      "activations/layer6_attention_weight_max": 56.823768615722656,
      "activations/layer6_attention_weight_min": -54.2942008972168,
      "activations/layer7_attention_weight_max": 82.58419799804688,
      "activations/layer7_attention_weight_min": -76.18875122070312,
      "activations/layer8_attention_weight_max": 47.15707778930664,
      "activations/layer8_attention_weight_min": -47.53268814086914,
      "activations/layer9_attention_weight_max": 51.33313751220703,
      "activations/layer9_attention_weight_min": -50.711368560791016,
      "epoch": 9.66,
      "learning_rate": 8.859659090909091e-05,
      "loss": 2.7827,
      "step": 166250
    },
    {
      "activations/layer0_attention_weight_max": 14.588851928710938,
      "activations/layer0_attention_weight_min": -14.12763786315918,
      "activations/layer10_attention_weight_max": 44.917877197265625,
      "activations/layer10_attention_weight_min": -40.6938591003418,
      "activations/layer11_attention_weight_max": 34.02229690551758,
      "activations/layer11_attention_weight_min": -34.2412109375,
      "activations/layer12_attention_weight_max": 17.3986759185791,
      "activations/layer12_attention_weight_min": -29.029560089111328,
      "activations/layer13_attention_weight_max": 46.37004470825195,
      "activations/layer13_attention_weight_min": -36.54327392578125,
      "activations/layer14_attention_weight_max": 44.04224395751953,
      "activations/layer14_attention_weight_min": -31.951095581054688,
      "activations/layer15_attention_weight_max": 42.538047790527344,
      "activations/layer15_attention_weight_min": -30.4918155670166,
      "activations/layer16_attention_weight_max": 31.385587692260742,
      "activations/layer16_attention_weight_min": -27.948444366455078,
      "activations/layer17_attention_weight_max": 57.97432327270508,
      "activations/layer17_attention_weight_min": -42.5850715637207,
      "activations/layer18_attention_weight_max": 47.2891960144043,
      "activations/layer18_attention_weight_min": -39.429386138916016,
      "activations/layer19_attention_weight_max": 24.06675148010254,
      "activations/layer19_attention_weight_min": -22.990503311157227,
      "activations/layer1_attention_weight_max": 17.57944107055664,
      "activations/layer1_attention_weight_min": -15.647336959838867,
      "activations/layer20_attention_weight_max": 26.35424041748047,
      "activations/layer20_attention_weight_min": -21.763046264648438,
      "activations/layer21_attention_weight_max": 44.5750617980957,
      "activations/layer21_attention_weight_min": -23.000734329223633,
      "activations/layer22_attention_weight_max": 33.783870697021484,
      "activations/layer22_attention_weight_min": -25.6826171875,
      "activations/layer23_attention_weight_max": 44.7511100769043,
      "activations/layer23_attention_weight_min": -24.427017211914062,
      "activations/layer2_attention_weight_max": 34.38379669189453,
      "activations/layer2_attention_weight_min": -32.90095520019531,
      "activations/layer3_attention_weight_max": 94.65388488769531,
      "activations/layer3_attention_weight_min": -95.83922576904297,
      "activations/layer4_attention_weight_max": 103.94002532958984,
      "activations/layer4_attention_weight_min": -98.98442077636719,
      "activations/layer5_attention_weight_max": 85.64596557617188,
      "activations/layer5_attention_weight_min": -80.67155456542969,
      "activations/layer6_attention_weight_max": 61.81329345703125,
      "activations/layer6_attention_weight_min": -58.41244888305664,
      "activations/layer7_attention_weight_max": 82.09545135498047,
      "activations/layer7_attention_weight_min": -81.57795715332031,
      "activations/layer8_attention_weight_max": 52.83772659301758,
      "activations/layer8_attention_weight_min": -52.07451248168945,
      "activations/layer9_attention_weight_max": 59.29433822631836,
      "activations/layer9_attention_weight_min": -56.21842575073242,
      "epoch": 9.66,
      "learning_rate": 8.85780303030303e-05,
      "loss": 2.7977,
      "step": 166300
    },
    {
      "activations/layer0_attention_weight_max": 15.546438217163086,
      "activations/layer0_attention_weight_min": -13.992278099060059,
      "activations/layer10_attention_weight_max": 37.855796813964844,
      "activations/layer10_attention_weight_min": -37.12841796875,
      "activations/layer11_attention_weight_max": 32.99711608886719,
      "activations/layer11_attention_weight_min": -36.11198806762695,
      "activations/layer12_attention_weight_max": 18.65711784362793,
      "activations/layer12_attention_weight_min": -23.757898330688477,
      "activations/layer13_attention_weight_max": 36.664268493652344,
      "activations/layer13_attention_weight_min": -29.142383575439453,
      "activations/layer14_attention_weight_max": 37.64006805419922,
      "activations/layer14_attention_weight_min": -31.106565475463867,
      "activations/layer15_attention_weight_max": 33.30613327026367,
      "activations/layer15_attention_weight_min": -31.057373046875,
      "activations/layer16_attention_weight_max": 31.11765480041504,
      "activations/layer16_attention_weight_min": -29.50579261779785,
      "activations/layer17_attention_weight_max": 50.944847106933594,
      "activations/layer17_attention_weight_min": -44.059749603271484,
      "activations/layer18_attention_weight_max": 46.676395416259766,
      "activations/layer18_attention_weight_min": -36.07781982421875,
      "activations/layer19_attention_weight_max": 24.406604766845703,
      "activations/layer19_attention_weight_min": -21.17046546936035,
      "activations/layer1_attention_weight_max": 16.736600875854492,
      "activations/layer1_attention_weight_min": -15.650839805603027,
      "activations/layer20_attention_weight_max": 20.200214385986328,
      "activations/layer20_attention_weight_min": -20.832860946655273,
      "activations/layer21_attention_weight_max": 38.926612854003906,
      "activations/layer21_attention_weight_min": -26.099416732788086,
      "activations/layer22_attention_weight_max": 32.92677688598633,
      "activations/layer22_attention_weight_min": -25.15534019470215,
      "activations/layer23_attention_weight_max": 39.37082290649414,
      "activations/layer23_attention_weight_min": -23.950925827026367,
      "activations/layer2_attention_weight_max": 32.2295036315918,
      "activations/layer2_attention_weight_min": -31.005752563476562,
      "activations/layer3_attention_weight_max": 93.83235168457031,
      "activations/layer3_attention_weight_min": -97.2317886352539,
      "activations/layer4_attention_weight_max": 110.56900787353516,
      "activations/layer4_attention_weight_min": -102.94561004638672,
      "activations/layer5_attention_weight_max": 83.32270812988281,
      "activations/layer5_attention_weight_min": -79.64114379882812,
      "activations/layer6_attention_weight_max": 55.2827033996582,
      "activations/layer6_attention_weight_min": -56.380615234375,
      "activations/layer7_attention_weight_max": 80.47003936767578,
      "activations/layer7_attention_weight_min": -76.07767486572266,
      "activations/layer8_attention_weight_max": 44.87592697143555,
      "activations/layer8_attention_weight_min": -45.117149353027344,
      "activations/layer9_attention_weight_max": 54.7995719909668,
      "activations/layer9_attention_weight_min": -51.15073013305664,
      "epoch": 9.67,
      "learning_rate": 8.85590909090909e-05,
      "loss": 2.7814,
      "step": 166350
    },
    {
      "activations/layer0_attention_weight_max": 16.176530838012695,
      "activations/layer0_attention_weight_min": -13.868460655212402,
      "activations/layer10_attention_weight_max": 37.7628173828125,
      "activations/layer10_attention_weight_min": -36.580265045166016,
      "activations/layer11_attention_weight_max": 32.053314208984375,
      "activations/layer11_attention_weight_min": -33.28434753417969,
      "activations/layer12_attention_weight_max": 21.631113052368164,
      "activations/layer12_attention_weight_min": -26.279924392700195,
      "activations/layer13_attention_weight_max": 42.66181182861328,
      "activations/layer13_attention_weight_min": -31.47829246520996,
      "activations/layer14_attention_weight_max": 42.27082824707031,
      "activations/layer14_attention_weight_min": -32.2913818359375,
      "activations/layer15_attention_weight_max": 38.41398239135742,
      "activations/layer15_attention_weight_min": -29.062807083129883,
      "activations/layer16_attention_weight_max": 42.5752067565918,
      "activations/layer16_attention_weight_min": -30.349624633789062,
      "activations/layer17_attention_weight_max": 59.34059143066406,
      "activations/layer17_attention_weight_min": -44.30131149291992,
      "activations/layer18_attention_weight_max": 51.776756286621094,
      "activations/layer18_attention_weight_min": -38.07219696044922,
      "activations/layer19_attention_weight_max": 26.878915786743164,
      "activations/layer19_attention_weight_min": -20.703142166137695,
      "activations/layer1_attention_weight_max": 16.23289680480957,
      "activations/layer1_attention_weight_min": -12.874460220336914,
      "activations/layer20_attention_weight_max": 24.16914939880371,
      "activations/layer20_attention_weight_min": -21.436662673950195,
      "activations/layer21_attention_weight_max": 41.96455001831055,
      "activations/layer21_attention_weight_min": -23.669145584106445,
      "activations/layer22_attention_weight_max": 34.92952346801758,
      "activations/layer22_attention_weight_min": -26.550262451171875,
      "activations/layer23_attention_weight_max": 40.27833938598633,
      "activations/layer23_attention_weight_min": -21.866886138916016,
      "activations/layer2_attention_weight_max": 28.72669219970703,
      "activations/layer2_attention_weight_min": -29.548240661621094,
      "activations/layer3_attention_weight_max": 86.86863708496094,
      "activations/layer3_attention_weight_min": -91.71073913574219,
      "activations/layer4_attention_weight_max": 99.73576354980469,
      "activations/layer4_attention_weight_min": -95.20487213134766,
      "activations/layer5_attention_weight_max": 76.05941009521484,
      "activations/layer5_attention_weight_min": -79.81181335449219,
      "activations/layer6_attention_weight_max": 57.96766662597656,
      "activations/layer6_attention_weight_min": -55.37269592285156,
      "activations/layer7_attention_weight_max": 77.05177307128906,
      "activations/layer7_attention_weight_min": -78.7379379272461,
      "activations/layer8_attention_weight_max": 45.810401916503906,
      "activations/layer8_attention_weight_min": -48.09245681762695,
      "activations/layer9_attention_weight_max": 52.283355712890625,
      "activations/layer9_attention_weight_min": -52.33793258666992,
      "epoch": 9.67,
      "learning_rate": 8.85401515151515e-05,
      "loss": 2.7959,
      "step": 166400
    },
    {
      "activations/layer0_attention_weight_max": 16.675174713134766,
      "activations/layer0_attention_weight_min": -13.835273742675781,
      "activations/layer10_attention_weight_max": 36.34688186645508,
      "activations/layer10_attention_weight_min": -36.280128479003906,
      "activations/layer11_attention_weight_max": 31.864673614501953,
      "activations/layer11_attention_weight_min": -32.777835845947266,
      "activations/layer12_attention_weight_max": 18.643753051757812,
      "activations/layer12_attention_weight_min": -27.09135627746582,
      "activations/layer13_attention_weight_max": 42.40202713012695,
      "activations/layer13_attention_weight_min": -29.702808380126953,
      "activations/layer14_attention_weight_max": 35.837059020996094,
      "activations/layer14_attention_weight_min": -30.94300651550293,
      "activations/layer15_attention_weight_max": 33.654815673828125,
      "activations/layer15_attention_weight_min": -30.694473266601562,
      "activations/layer16_attention_weight_max": 32.00881576538086,
      "activations/layer16_attention_weight_min": -29.234664916992188,
      "activations/layer17_attention_weight_max": 51.60978698730469,
      "activations/layer17_attention_weight_min": -46.26713943481445,
      "activations/layer18_attention_weight_max": 46.29621887207031,
      "activations/layer18_attention_weight_min": -39.752891540527344,
      "activations/layer19_attention_weight_max": 24.76889419555664,
      "activations/layer19_attention_weight_min": -20.793704986572266,
      "activations/layer1_attention_weight_max": 17.34158706665039,
      "activations/layer1_attention_weight_min": -14.417485237121582,
      "activations/layer20_attention_weight_max": 23.558427810668945,
      "activations/layer20_attention_weight_min": -22.443225860595703,
      "activations/layer21_attention_weight_max": 38.55290222167969,
      "activations/layer21_attention_weight_min": -24.136220932006836,
      "activations/layer22_attention_weight_max": 32.063961029052734,
      "activations/layer22_attention_weight_min": -22.80681037902832,
      "activations/layer23_attention_weight_max": 41.23172378540039,
      "activations/layer23_attention_weight_min": -21.960708618164062,
      "activations/layer2_attention_weight_max": 31.369251251220703,
      "activations/layer2_attention_weight_min": -31.434293746948242,
      "activations/layer3_attention_weight_max": 89.16929626464844,
      "activations/layer3_attention_weight_min": -89.07599639892578,
      "activations/layer4_attention_weight_max": 93.68879699707031,
      "activations/layer4_attention_weight_min": -92.7032699584961,
      "activations/layer5_attention_weight_max": 74.56475067138672,
      "activations/layer5_attention_weight_min": -77.89866638183594,
      "activations/layer6_attention_weight_max": 54.675392150878906,
      "activations/layer6_attention_weight_min": -55.01015090942383,
      "activations/layer7_attention_weight_max": 77.187744140625,
      "activations/layer7_attention_weight_min": -75.56695556640625,
      "activations/layer8_attention_weight_max": 43.930931091308594,
      "activations/layer8_attention_weight_min": -46.860382080078125,
      "activations/layer9_attention_weight_max": 48.674049377441406,
      "activations/layer9_attention_weight_min": -51.13318634033203,
      "epoch": 9.67,
      "learning_rate": 8.852121212121212e-05,
      "loss": 2.8038,
      "step": 166450
    },
    {
      "activations/layer0_attention_weight_max": 14.626809120178223,
      "activations/layer0_attention_weight_min": -13.892484664916992,
      "activations/layer10_attention_weight_max": 36.92049026489258,
      "activations/layer10_attention_weight_min": -36.63897705078125,
      "activations/layer11_attention_weight_max": 31.33646011352539,
      "activations/layer11_attention_weight_min": -31.037994384765625,
      "activations/layer12_attention_weight_max": 23.21233367919922,
      "activations/layer12_attention_weight_min": -23.76736068725586,
      "activations/layer13_attention_weight_max": 47.19835662841797,
      "activations/layer13_attention_weight_min": -28.88357162475586,
      "activations/layer14_attention_weight_max": 43.21599197387695,
      "activations/layer14_attention_weight_min": -31.734933853149414,
      "activations/layer15_attention_weight_max": 37.23941421508789,
      "activations/layer15_attention_weight_min": -30.914188385009766,
      "activations/layer16_attention_weight_max": 35.56742858886719,
      "activations/layer16_attention_weight_min": -29.115251541137695,
      "activations/layer17_attention_weight_max": 66.04505920410156,
      "activations/layer17_attention_weight_min": -48.64017105102539,
      "activations/layer18_attention_weight_max": 52.3048210144043,
      "activations/layer18_attention_weight_min": -39.858219146728516,
      "activations/layer19_attention_weight_max": 25.200468063354492,
      "activations/layer19_attention_weight_min": -21.38755226135254,
      "activations/layer1_attention_weight_max": 17.869651794433594,
      "activations/layer1_attention_weight_min": -15.61622142791748,
      "activations/layer20_attention_weight_max": 25.9693660736084,
      "activations/layer20_attention_weight_min": -21.137989044189453,
      "activations/layer21_attention_weight_max": 44.29063034057617,
      "activations/layer21_attention_weight_min": -23.78290367126465,
      "activations/layer22_attention_weight_max": 35.53755187988281,
      "activations/layer22_attention_weight_min": -25.13189697265625,
      "activations/layer23_attention_weight_max": 41.884361267089844,
      "activations/layer23_attention_weight_min": -21.651321411132812,
      "activations/layer2_attention_weight_max": 32.9598388671875,
      "activations/layer2_attention_weight_min": -31.915937423706055,
      "activations/layer3_attention_weight_max": 90.58867645263672,
      "activations/layer3_attention_weight_min": -89.51402282714844,
      "activations/layer4_attention_weight_max": 99.56211853027344,
      "activations/layer4_attention_weight_min": -95.5496826171875,
      "activations/layer5_attention_weight_max": 78.54153442382812,
      "activations/layer5_attention_weight_min": -82.73414611816406,
      "activations/layer6_attention_weight_max": 54.15442657470703,
      "activations/layer6_attention_weight_min": -53.55805587768555,
      "activations/layer7_attention_weight_max": 72.92726135253906,
      "activations/layer7_attention_weight_min": -75.01714324951172,
      "activations/layer8_attention_weight_max": 45.678688049316406,
      "activations/layer8_attention_weight_min": -47.51960372924805,
      "activations/layer9_attention_weight_max": 49.58842086791992,
      "activations/layer9_attention_weight_min": -49.18074417114258,
      "epoch": 9.67,
      "learning_rate": 8.850227272727272e-05,
      "loss": 2.7883,
      "step": 166500
    },
    {
      "activations/layer0_attention_weight_max": 16.221054077148438,
      "activations/layer0_attention_weight_min": -14.52664566040039,
      "activations/layer10_attention_weight_max": 35.42559814453125,
      "activations/layer10_attention_weight_min": -33.82366943359375,
      "activations/layer11_attention_weight_max": 31.737079620361328,
      "activations/layer11_attention_weight_min": -31.947574615478516,
      "activations/layer12_attention_weight_max": 17.196292877197266,
      "activations/layer12_attention_weight_min": -26.899765014648438,
      "activations/layer13_attention_weight_max": 33.73252487182617,
      "activations/layer13_attention_weight_min": -29.398174285888672,
      "activations/layer14_attention_weight_max": 35.039710998535156,
      "activations/layer14_attention_weight_min": -30.73954200744629,
      "activations/layer15_attention_weight_max": 32.57052230834961,
      "activations/layer15_attention_weight_min": -29.940628051757812,
      "activations/layer16_attention_weight_max": 29.735462188720703,
      "activations/layer16_attention_weight_min": -27.719970703125,
      "activations/layer17_attention_weight_max": 50.24176025390625,
      "activations/layer17_attention_weight_min": -43.37297058105469,
      "activations/layer18_attention_weight_max": 42.63650894165039,
      "activations/layer18_attention_weight_min": -37.958351135253906,
      "activations/layer19_attention_weight_max": 19.696191787719727,
      "activations/layer19_attention_weight_min": -23.102136611938477,
      "activations/layer1_attention_weight_max": 16.292652130126953,
      "activations/layer1_attention_weight_min": -16.10934066772461,
      "activations/layer20_attention_weight_max": 21.55911636352539,
      "activations/layer20_attention_weight_min": -20.501312255859375,
      "activations/layer21_attention_weight_max": 40.7515754699707,
      "activations/layer21_attention_weight_min": -22.70426368713379,
      "activations/layer22_attention_weight_max": 31.73308563232422,
      "activations/layer22_attention_weight_min": -25.007343292236328,
      "activations/layer23_attention_weight_max": 35.06187438964844,
      "activations/layer23_attention_weight_min": -22.474760055541992,
      "activations/layer2_attention_weight_max": 32.97975540161133,
      "activations/layer2_attention_weight_min": -29.07845115661621,
      "activations/layer3_attention_weight_max": 86.5304946899414,
      "activations/layer3_attention_weight_min": -89.51683807373047,
      "activations/layer4_attention_weight_max": 99.32518768310547,
      "activations/layer4_attention_weight_min": -96.40342712402344,
      "activations/layer5_attention_weight_max": 79.54940795898438,
      "activations/layer5_attention_weight_min": -77.81658172607422,
      "activations/layer6_attention_weight_max": 55.52924728393555,
      "activations/layer6_attention_weight_min": -53.90441131591797,
      "activations/layer7_attention_weight_max": 76.43264770507812,
      "activations/layer7_attention_weight_min": -70.46539306640625,
      "activations/layer8_attention_weight_max": 46.3256721496582,
      "activations/layer8_attention_weight_min": -48.73079299926758,
      "activations/layer9_attention_weight_max": 51.89899444580078,
      "activations/layer9_attention_weight_min": -49.69068908691406,
      "epoch": 9.68,
      "learning_rate": 8.848333333333333e-05,
      "loss": 2.8106,
      "step": 166550
    },
    {
      "activations/layer0_attention_weight_max": 14.598005294799805,
      "activations/layer0_attention_weight_min": -14.072870254516602,
      "activations/layer10_attention_weight_max": 41.6354866027832,
      "activations/layer10_attention_weight_min": -36.7040901184082,
      "activations/layer11_attention_weight_max": 36.253501892089844,
      "activations/layer11_attention_weight_min": -34.505638122558594,
      "activations/layer12_attention_weight_max": 26.269941329956055,
      "activations/layer12_attention_weight_min": -24.51279067993164,
      "activations/layer13_attention_weight_max": 41.90525436401367,
      "activations/layer13_attention_weight_min": -31.63029670715332,
      "activations/layer14_attention_weight_max": 39.760746002197266,
      "activations/layer14_attention_weight_min": -29.60503578186035,
      "activations/layer15_attention_weight_max": 36.22849655151367,
      "activations/layer15_attention_weight_min": -30.04330062866211,
      "activations/layer16_attention_weight_max": 34.098716735839844,
      "activations/layer16_attention_weight_min": -28.789020538330078,
      "activations/layer17_attention_weight_max": 61.04225540161133,
      "activations/layer17_attention_weight_min": -45.195064544677734,
      "activations/layer18_attention_weight_max": 47.07469940185547,
      "activations/layer18_attention_weight_min": -36.389068603515625,
      "activations/layer19_attention_weight_max": 25.799036026000977,
      "activations/layer19_attention_weight_min": -21.657838821411133,
      "activations/layer1_attention_weight_max": 16.603843688964844,
      "activations/layer1_attention_weight_min": -14.476579666137695,
      "activations/layer20_attention_weight_max": 24.30201530456543,
      "activations/layer20_attention_weight_min": -21.148271560668945,
      "activations/layer21_attention_weight_max": 43.034454345703125,
      "activations/layer21_attention_weight_min": -23.32744026184082,
      "activations/layer22_attention_weight_max": 31.458412170410156,
      "activations/layer22_attention_weight_min": -25.048725128173828,
      "activations/layer23_attention_weight_max": 42.24907684326172,
      "activations/layer23_attention_weight_min": -29.56987762451172,
      "activations/layer2_attention_weight_max": 31.53018569946289,
      "activations/layer2_attention_weight_min": -30.16312026977539,
      "activations/layer3_attention_weight_max": 87.55970001220703,
      "activations/layer3_attention_weight_min": -95.33058166503906,
      "activations/layer4_attention_weight_max": 100.98635864257812,
      "activations/layer4_attention_weight_min": -104.93660736083984,
      "activations/layer5_attention_weight_max": 78.82017517089844,
      "activations/layer5_attention_weight_min": -82.69148254394531,
      "activations/layer6_attention_weight_max": 56.137062072753906,
      "activations/layer6_attention_weight_min": -55.04850387573242,
      "activations/layer7_attention_weight_max": 82.15385437011719,
      "activations/layer7_attention_weight_min": -76.33393859863281,
      "activations/layer8_attention_weight_max": 45.800052642822266,
      "activations/layer8_attention_weight_min": -49.3433723449707,
      "activations/layer9_attention_weight_max": 50.0587272644043,
      "activations/layer9_attention_weight_min": -51.13351058959961,
      "epoch": 9.68,
      "learning_rate": 8.846439393939392e-05,
      "loss": 2.7884,
      "step": 166600
    },
    {
      "activations/layer0_attention_weight_max": 14.5474271774292,
      "activations/layer0_attention_weight_min": -13.821636199951172,
      "activations/layer10_attention_weight_max": 40.569725036621094,
      "activations/layer10_attention_weight_min": -37.442806243896484,
      "activations/layer11_attention_weight_max": 35.779903411865234,
      "activations/layer11_attention_weight_min": -33.04676818847656,
      "activations/layer12_attention_weight_max": 27.40094566345215,
      "activations/layer12_attention_weight_min": -26.60840606689453,
      "activations/layer13_attention_weight_max": 63.518436431884766,
      "activations/layer13_attention_weight_min": -33.85450744628906,
      "activations/layer14_attention_weight_max": 74.68059539794922,
      "activations/layer14_attention_weight_min": -32.7166633605957,
      "activations/layer15_attention_weight_max": 59.7781867980957,
      "activations/layer15_attention_weight_min": -30.569791793823242,
      "activations/layer16_attention_weight_max": 35.83456802368164,
      "activations/layer16_attention_weight_min": -29.450307846069336,
      "activations/layer17_attention_weight_max": 58.92756652832031,
      "activations/layer17_attention_weight_min": -49.17976379394531,
      "activations/layer18_attention_weight_max": 52.628116607666016,
      "activations/layer18_attention_weight_min": -42.5605583190918,
      "activations/layer19_attention_weight_max": 27.1874942779541,
      "activations/layer19_attention_weight_min": -23.23604393005371,
      "activations/layer1_attention_weight_max": 17.066471099853516,
      "activations/layer1_attention_weight_min": -15.618511199951172,
      "activations/layer20_attention_weight_max": 25.592151641845703,
      "activations/layer20_attention_weight_min": -22.65166473388672,
      "activations/layer21_attention_weight_max": 46.91056442260742,
      "activations/layer21_attention_weight_min": -22.353317260742188,
      "activations/layer22_attention_weight_max": 36.37351989746094,
      "activations/layer22_attention_weight_min": -25.71664047241211,
      "activations/layer23_attention_weight_max": 39.4246826171875,
      "activations/layer23_attention_weight_min": -24.74036407470703,
      "activations/layer2_attention_weight_max": 30.95828628540039,
      "activations/layer2_attention_weight_min": -32.537147521972656,
      "activations/layer3_attention_weight_max": 90.07072448730469,
      "activations/layer3_attention_weight_min": -93.50667572021484,
      "activations/layer4_attention_weight_max": 99.21236419677734,
      "activations/layer4_attention_weight_min": -96.63915252685547,
      "activations/layer5_attention_weight_max": 78.95865631103516,
      "activations/layer5_attention_weight_min": -76.84523010253906,
      "activations/layer6_attention_weight_max": 54.04315948486328,
      "activations/layer6_attention_weight_min": -52.556217193603516,
      "activations/layer7_attention_weight_max": 78.74212646484375,
      "activations/layer7_attention_weight_min": -78.77664184570312,
      "activations/layer8_attention_weight_max": 49.69508743286133,
      "activations/layer8_attention_weight_min": -50.969608306884766,
      "activations/layer9_attention_weight_max": 59.18400192260742,
      "activations/layer9_attention_weight_min": -51.3186149597168,
      "epoch": 9.68,
      "learning_rate": 8.844545454545454e-05,
      "loss": 2.8014,
      "step": 166650
    },
    {
      "activations/layer0_attention_weight_max": 14.905010223388672,
      "activations/layer0_attention_weight_min": -14.120015144348145,
      "activations/layer10_attention_weight_max": 37.66864013671875,
      "activations/layer10_attention_weight_min": -37.23183059692383,
      "activations/layer11_attention_weight_max": 36.245914459228516,
      "activations/layer11_attention_weight_min": -32.726768493652344,
      "activations/layer12_attention_weight_max": 17.242109298706055,
      "activations/layer12_attention_weight_min": -32.99152755737305,
      "activations/layer13_attention_weight_max": 38.69282913208008,
      "activations/layer13_attention_weight_min": -34.238521575927734,
      "activations/layer14_attention_weight_max": 36.25946044921875,
      "activations/layer14_attention_weight_min": -30.627050399780273,
      "activations/layer15_attention_weight_max": 30.138507843017578,
      "activations/layer15_attention_weight_min": -29.37179946899414,
      "activations/layer16_attention_weight_max": 28.555988311767578,
      "activations/layer16_attention_weight_min": -27.09803009033203,
      "activations/layer17_attention_weight_max": 50.881874084472656,
      "activations/layer17_attention_weight_min": -43.69760513305664,
      "activations/layer18_attention_weight_max": 43.798980712890625,
      "activations/layer18_attention_weight_min": -36.031463623046875,
      "activations/layer19_attention_weight_max": 24.98562240600586,
      "activations/layer19_attention_weight_min": -21.799955368041992,
      "activations/layer1_attention_weight_max": 17.610750198364258,
      "activations/layer1_attention_weight_min": -15.094941139221191,
      "activations/layer20_attention_weight_max": 20.533910751342773,
      "activations/layer20_attention_weight_min": -22.295522689819336,
      "activations/layer21_attention_weight_max": 35.87262725830078,
      "activations/layer21_attention_weight_min": -24.34694480895996,
      "activations/layer22_attention_weight_max": 33.069671630859375,
      "activations/layer22_attention_weight_min": -27.56564712524414,
      "activations/layer23_attention_weight_max": 34.73896026611328,
      "activations/layer23_attention_weight_min": -25.400611877441406,
      "activations/layer2_attention_weight_max": 31.964941024780273,
      "activations/layer2_attention_weight_min": -29.20947265625,
      "activations/layer3_attention_weight_max": 89.38829803466797,
      "activations/layer3_attention_weight_min": -92.74919128417969,
      "activations/layer4_attention_weight_max": 105.74161529541016,
      "activations/layer4_attention_weight_min": -99.70905303955078,
      "activations/layer5_attention_weight_max": 79.98941802978516,
      "activations/layer5_attention_weight_min": -80.11202239990234,
      "activations/layer6_attention_weight_max": 53.881736755371094,
      "activations/layer6_attention_weight_min": -52.348716735839844,
      "activations/layer7_attention_weight_max": 87.47654724121094,
      "activations/layer7_attention_weight_min": -75.50210571289062,
      "activations/layer8_attention_weight_max": 49.779014587402344,
      "activations/layer8_attention_weight_min": -45.53441619873047,
      "activations/layer9_attention_weight_max": 67.55956268310547,
      "activations/layer9_attention_weight_min": -60.64138412475586,
      "epoch": 9.69,
      "learning_rate": 8.842651515151515e-05,
      "loss": 2.8061,
      "step": 166700
    },
    {
      "activations/layer0_attention_weight_max": 15.763158798217773,
      "activations/layer0_attention_weight_min": -13.946065902709961,
      "activations/layer10_attention_weight_max": 35.488502502441406,
      "activations/layer10_attention_weight_min": -34.73748779296875,
      "activations/layer11_attention_weight_max": 31.56468963623047,
      "activations/layer11_attention_weight_min": -30.763633728027344,
      "activations/layer12_attention_weight_max": 17.35518455505371,
      "activations/layer12_attention_weight_min": -27.62053108215332,
      "activations/layer13_attention_weight_max": 40.28130340576172,
      "activations/layer13_attention_weight_min": -30.569509506225586,
      "activations/layer14_attention_weight_max": 36.2123908996582,
      "activations/layer14_attention_weight_min": -29.739715576171875,
      "activations/layer15_attention_weight_max": 33.78202819824219,
      "activations/layer15_attention_weight_min": -28.550315856933594,
      "activations/layer16_attention_weight_max": 31.473644256591797,
      "activations/layer16_attention_weight_min": -28.338911056518555,
      "activations/layer17_attention_weight_max": 51.89177703857422,
      "activations/layer17_attention_weight_min": -43.23024368286133,
      "activations/layer18_attention_weight_max": 46.21181869506836,
      "activations/layer18_attention_weight_min": -37.38431167602539,
      "activations/layer19_attention_weight_max": 23.02395248413086,
      "activations/layer19_attention_weight_min": -23.463899612426758,
      "activations/layer1_attention_weight_max": 17.135822296142578,
      "activations/layer1_attention_weight_min": -15.538322448730469,
      "activations/layer20_attention_weight_max": 22.854469299316406,
      "activations/layer20_attention_weight_min": -22.66910171508789,
      "activations/layer21_attention_weight_max": 37.274085998535156,
      "activations/layer21_attention_weight_min": -24.0786190032959,
      "activations/layer22_attention_weight_max": 28.24679183959961,
      "activations/layer22_attention_weight_min": -26.09450340270996,
      "activations/layer23_attention_weight_max": 34.64482879638672,
      "activations/layer23_attention_weight_min": -21.456480026245117,
      "activations/layer2_attention_weight_max": 36.89307403564453,
      "activations/layer2_attention_weight_min": -32.449501037597656,
      "activations/layer3_attention_weight_max": 99.93611907958984,
      "activations/layer3_attention_weight_min": -97.6034164428711,
      "activations/layer4_attention_weight_max": 101.58892059326172,
      "activations/layer4_attention_weight_min": -99.46293640136719,
      "activations/layer5_attention_weight_max": 81.20556640625,
      "activations/layer5_attention_weight_min": -80.52857208251953,
      "activations/layer6_attention_weight_max": 55.16250991821289,
      "activations/layer6_attention_weight_min": -56.23371124267578,
      "activations/layer7_attention_weight_max": 80.51988220214844,
      "activations/layer7_attention_weight_min": -77.62701416015625,
      "activations/layer8_attention_weight_max": 49.97251510620117,
      "activations/layer8_attention_weight_min": -47.59013748168945,
      "activations/layer9_attention_weight_max": 52.242828369140625,
      "activations/layer9_attention_weight_min": -49.03675079345703,
      "epoch": 9.69,
      "learning_rate": 8.840757575757574e-05,
      "loss": 2.8046,
      "step": 166750
    },
    {
      "activations/layer0_attention_weight_max": 15.727459907531738,
      "activations/layer0_attention_weight_min": -13.676362991333008,
      "activations/layer10_attention_weight_max": 38.87175750732422,
      "activations/layer10_attention_weight_min": -37.22115707397461,
      "activations/layer11_attention_weight_max": 33.21562194824219,
      "activations/layer11_attention_weight_min": -33.095497131347656,
      "activations/layer12_attention_weight_max": 20.85013771057129,
      "activations/layer12_attention_weight_min": -24.865503311157227,
      "activations/layer13_attention_weight_max": 46.60960006713867,
      "activations/layer13_attention_weight_min": -34.61662292480469,
      "activations/layer14_attention_weight_max": 38.041168212890625,
      "activations/layer14_attention_weight_min": -31.144620895385742,
      "activations/layer15_attention_weight_max": 36.5424690246582,
      "activations/layer15_attention_weight_min": -30.78060531616211,
      "activations/layer16_attention_weight_max": 32.30064010620117,
      "activations/layer16_attention_weight_min": -29.43474769592285,
      "activations/layer17_attention_weight_max": 57.5477294921875,
      "activations/layer17_attention_weight_min": -45.30160903930664,
      "activations/layer18_attention_weight_max": 53.8000373840332,
      "activations/layer18_attention_weight_min": -38.434085845947266,
      "activations/layer19_attention_weight_max": 22.56812858581543,
      "activations/layer19_attention_weight_min": -22.211524963378906,
      "activations/layer1_attention_weight_max": 16.668136596679688,
      "activations/layer1_attention_weight_min": -14.545353889465332,
      "activations/layer20_attention_weight_max": 22.591928482055664,
      "activations/layer20_attention_weight_min": -22.211769104003906,
      "activations/layer21_attention_weight_max": 42.644927978515625,
      "activations/layer21_attention_weight_min": -23.769378662109375,
      "activations/layer22_attention_weight_max": 32.60954666137695,
      "activations/layer22_attention_weight_min": -27.90273094177246,
      "activations/layer23_attention_weight_max": 45.601661682128906,
      "activations/layer23_attention_weight_min": -24.44051742553711,
      "activations/layer2_attention_weight_max": 30.96796989440918,
      "activations/layer2_attention_weight_min": -30.745235443115234,
      "activations/layer3_attention_weight_max": 87.64509582519531,
      "activations/layer3_attention_weight_min": -95.7856216430664,
      "activations/layer4_attention_weight_max": 100.69989776611328,
      "activations/layer4_attention_weight_min": -99.32685089111328,
      "activations/layer5_attention_weight_max": 78.22767639160156,
      "activations/layer5_attention_weight_min": -80.89111328125,
      "activations/layer6_attention_weight_max": 53.82835006713867,
      "activations/layer6_attention_weight_min": -55.68964385986328,
      "activations/layer7_attention_weight_max": 77.43791198730469,
      "activations/layer7_attention_weight_min": -74.98529815673828,
      "activations/layer8_attention_weight_max": 45.410179138183594,
      "activations/layer8_attention_weight_min": -48.3934211730957,
      "activations/layer9_attention_weight_max": 56.19586944580078,
      "activations/layer9_attention_weight_min": -48.97312545776367,
      "epoch": 9.69,
      "learning_rate": 8.838863636363635e-05,
      "loss": 2.8145,
      "step": 166800
    },
    {
      "activations/layer0_attention_weight_max": 16.28055191040039,
      "activations/layer0_attention_weight_min": -13.750089645385742,
      "activations/layer10_attention_weight_max": 47.285972595214844,
      "activations/layer10_attention_weight_min": -40.55666732788086,
      "activations/layer11_attention_weight_max": 40.57176208496094,
      "activations/layer11_attention_weight_min": -35.44788360595703,
      "activations/layer12_attention_weight_max": 22.27419662475586,
      "activations/layer12_attention_weight_min": -27.201221466064453,
      "activations/layer13_attention_weight_max": 63.54943084716797,
      "activations/layer13_attention_weight_min": -31.933277130126953,
      "activations/layer14_attention_weight_max": 66.08744812011719,
      "activations/layer14_attention_weight_min": -35.32004928588867,
      "activations/layer15_attention_weight_max": 51.8143196105957,
      "activations/layer15_attention_weight_min": -28.78584861755371,
      "activations/layer16_attention_weight_max": 34.83894729614258,
      "activations/layer16_attention_weight_min": -28.53240966796875,
      "activations/layer17_attention_weight_max": 60.08656692504883,
      "activations/layer17_attention_weight_min": -47.22850799560547,
      "activations/layer18_attention_weight_max": 48.250282287597656,
      "activations/layer18_attention_weight_min": -39.40633010864258,
      "activations/layer19_attention_weight_max": 23.187108993530273,
      "activations/layer19_attention_weight_min": -22.802125930786133,
      "activations/layer1_attention_weight_max": 17.117347717285156,
      "activations/layer1_attention_weight_min": -15.615706443786621,
      "activations/layer20_attention_weight_max": 26.893497467041016,
      "activations/layer20_attention_weight_min": -20.4091854095459,
      "activations/layer21_attention_weight_max": 45.77150344848633,
      "activations/layer21_attention_weight_min": -21.291196823120117,
      "activations/layer22_attention_weight_max": 32.86946105957031,
      "activations/layer22_attention_weight_min": -25.596757888793945,
      "activations/layer23_attention_weight_max": 36.581398010253906,
      "activations/layer23_attention_weight_min": -20.841808319091797,
      "activations/layer2_attention_weight_max": 34.35191345214844,
      "activations/layer2_attention_weight_min": -32.868797302246094,
      "activations/layer3_attention_weight_max": 94.67866516113281,
      "activations/layer3_attention_weight_min": -92.95890808105469,
      "activations/layer4_attention_weight_max": 100.46717071533203,
      "activations/layer4_attention_weight_min": -100.1076889038086,
      "activations/layer5_attention_weight_max": 78.94046020507812,
      "activations/layer5_attention_weight_min": -79.38313293457031,
      "activations/layer6_attention_weight_max": 57.775577545166016,
      "activations/layer6_attention_weight_min": -54.78959274291992,
      "activations/layer7_attention_weight_max": 88.80721282958984,
      "activations/layer7_attention_weight_min": -85.67872619628906,
      "activations/layer8_attention_weight_max": 58.28134536743164,
      "activations/layer8_attention_weight_min": -51.16062545776367,
      "activations/layer9_attention_weight_max": 67.69925689697266,
      "activations/layer9_attention_weight_min": -58.43220901489258,
      "epoch": 9.69,
      "learning_rate": 8.836969696969697e-05,
      "loss": 2.7957,
      "step": 166850
    },
    {
      "activations/layer0_attention_weight_max": 14.93067741394043,
      "activations/layer0_attention_weight_min": -13.779487609863281,
      "activations/layer10_attention_weight_max": 36.234962463378906,
      "activations/layer10_attention_weight_min": -35.34534454345703,
      "activations/layer11_attention_weight_max": 32.740386962890625,
      "activations/layer11_attention_weight_min": -33.155906677246094,
      "activations/layer12_attention_weight_max": 21.684995651245117,
      "activations/layer12_attention_weight_min": -29.46283531188965,
      "activations/layer13_attention_weight_max": 49.78739929199219,
      "activations/layer13_attention_weight_min": -31.26043701171875,
      "activations/layer14_attention_weight_max": 46.805355072021484,
      "activations/layer14_attention_weight_min": -30.314945220947266,
      "activations/layer15_attention_weight_max": 43.246891021728516,
      "activations/layer15_attention_weight_min": -29.550508499145508,
      "activations/layer16_attention_weight_max": 32.92546463012695,
      "activations/layer16_attention_weight_min": -27.84192657470703,
      "activations/layer17_attention_weight_max": 54.974449157714844,
      "activations/layer17_attention_weight_min": -44.37675857543945,
      "activations/layer18_attention_weight_max": 47.5748291015625,
      "activations/layer18_attention_weight_min": -39.451751708984375,
      "activations/layer19_attention_weight_max": 23.492435455322266,
      "activations/layer19_attention_weight_min": -24.30830955505371,
      "activations/layer1_attention_weight_max": 17.4088191986084,
      "activations/layer1_attention_weight_min": -15.864587783813477,
      "activations/layer20_attention_weight_max": 24.789731979370117,
      "activations/layer20_attention_weight_min": -22.069854736328125,
      "activations/layer21_attention_weight_max": 43.27962112426758,
      "activations/layer21_attention_weight_min": -28.851993560791016,
      "activations/layer22_attention_weight_max": 34.992366790771484,
      "activations/layer22_attention_weight_min": -26.047210693359375,
      "activations/layer23_attention_weight_max": 41.606964111328125,
      "activations/layer23_attention_weight_min": -26.41378402709961,
      "activations/layer2_attention_weight_max": 31.824735641479492,
      "activations/layer2_attention_weight_min": -30.94959259033203,
      "activations/layer3_attention_weight_max": 90.93959045410156,
      "activations/layer3_attention_weight_min": -93.42221069335938,
      "activations/layer4_attention_weight_max": 101.27376556396484,
      "activations/layer4_attention_weight_min": -96.54460144042969,
      "activations/layer5_attention_weight_max": 81.55726623535156,
      "activations/layer5_attention_weight_min": -80.78919219970703,
      "activations/layer6_attention_weight_max": 56.2103271484375,
      "activations/layer6_attention_weight_min": -55.256370544433594,
      "activations/layer7_attention_weight_max": 78.90135192871094,
      "activations/layer7_attention_weight_min": -77.09986114501953,
      "activations/layer8_attention_weight_max": 44.59876251220703,
      "activations/layer8_attention_weight_min": -46.77729415893555,
      "activations/layer9_attention_weight_max": 53.993377685546875,
      "activations/layer9_attention_weight_min": -55.19346237182617,
      "epoch": 9.7,
      "learning_rate": 8.835075757575756e-05,
      "loss": 2.792,
      "step": 166900
    },
    {
      "activations/layer0_attention_weight_max": 14.795440673828125,
      "activations/layer0_attention_weight_min": -14.166369438171387,
      "activations/layer10_attention_weight_max": 35.74951934814453,
      "activations/layer10_attention_weight_min": -35.18791580200195,
      "activations/layer11_attention_weight_max": 31.56099510192871,
      "activations/layer11_attention_weight_min": -31.704145431518555,
      "activations/layer12_attention_weight_max": 17.72892189025879,
      "activations/layer12_attention_weight_min": -20.892715454101562,
      "activations/layer13_attention_weight_max": 37.852516174316406,
      "activations/layer13_attention_weight_min": -30.0809383392334,
      "activations/layer14_attention_weight_max": 38.22450256347656,
      "activations/layer14_attention_weight_min": -32.61153030395508,
      "activations/layer15_attention_weight_max": 35.03142547607422,
      "activations/layer15_attention_weight_min": -31.78499984741211,
      "activations/layer16_attention_weight_max": 31.487064361572266,
      "activations/layer16_attention_weight_min": -31.659061431884766,
      "activations/layer17_attention_weight_max": 54.388004302978516,
      "activations/layer17_attention_weight_min": -46.034515380859375,
      "activations/layer18_attention_weight_max": 52.152191162109375,
      "activations/layer18_attention_weight_min": -39.706146240234375,
      "activations/layer19_attention_weight_max": 22.30544662475586,
      "activations/layer19_attention_weight_min": -23.210630416870117,
      "activations/layer1_attention_weight_max": 16.593490600585938,
      "activations/layer1_attention_weight_min": -14.467329978942871,
      "activations/layer20_attention_weight_max": 23.138818740844727,
      "activations/layer20_attention_weight_min": -21.63519859313965,
      "activations/layer21_attention_weight_max": 46.4467887878418,
      "activations/layer21_attention_weight_min": -24.753787994384766,
      "activations/layer22_attention_weight_max": 34.874019622802734,
      "activations/layer22_attention_weight_min": -26.97120475769043,
      "activations/layer23_attention_weight_max": 40.100303649902344,
      "activations/layer23_attention_weight_min": -23.509796142578125,
      "activations/layer2_attention_weight_max": 34.09009552001953,
      "activations/layer2_attention_weight_min": -30.51660919189453,
      "activations/layer3_attention_weight_max": 92.1397476196289,
      "activations/layer3_attention_weight_min": -90.92483520507812,
      "activations/layer4_attention_weight_max": 102.30526733398438,
      "activations/layer4_attention_weight_min": -96.16683197021484,
      "activations/layer5_attention_weight_max": 82.22601318359375,
      "activations/layer5_attention_weight_min": -79.47956848144531,
      "activations/layer6_attention_weight_max": 57.06259536743164,
      "activations/layer6_attention_weight_min": -52.93901062011719,
      "activations/layer7_attention_weight_max": 77.6110610961914,
      "activations/layer7_attention_weight_min": -77.66948699951172,
      "activations/layer8_attention_weight_max": 46.21292495727539,
      "activations/layer8_attention_weight_min": -46.7762336730957,
      "activations/layer9_attention_weight_max": 48.991207122802734,
      "activations/layer9_attention_weight_min": -49.817264556884766,
      "epoch": 9.7,
      "learning_rate": 8.833181818181817e-05,
      "loss": 2.786,
      "step": 166950
    },
    {
      "activations/layer0_attention_weight_max": 16.256696701049805,
      "activations/layer0_attention_weight_min": -14.127188682556152,
      "activations/layer10_attention_weight_max": 39.201263427734375,
      "activations/layer10_attention_weight_min": -39.71900177001953,
      "activations/layer11_attention_weight_max": 32.572933197021484,
      "activations/layer11_attention_weight_min": -34.24519348144531,
      "activations/layer12_attention_weight_max": 24.451705932617188,
      "activations/layer12_attention_weight_min": -23.839067459106445,
      "activations/layer13_attention_weight_max": 39.1428108215332,
      "activations/layer13_attention_weight_min": -29.315889358520508,
      "activations/layer14_attention_weight_max": 36.536312103271484,
      "activations/layer14_attention_weight_min": -30.721065521240234,
      "activations/layer15_attention_weight_max": 33.225669860839844,
      "activations/layer15_attention_weight_min": -29.204801559448242,
      "activations/layer16_attention_weight_max": 33.27546691894531,
      "activations/layer16_attention_weight_min": -29.070478439331055,
      "activations/layer17_attention_weight_max": 53.84315872192383,
      "activations/layer17_attention_weight_min": -43.360172271728516,
      "activations/layer18_attention_weight_max": 46.14033126831055,
      "activations/layer18_attention_weight_min": -37.624881744384766,
      "activations/layer19_attention_weight_max": 24.037588119506836,
      "activations/layer19_attention_weight_min": -20.89628791809082,
      "activations/layer1_attention_weight_max": 16.093353271484375,
      "activations/layer1_attention_weight_min": -14.380078315734863,
      "activations/layer20_attention_weight_max": 22.698333740234375,
      "activations/layer20_attention_weight_min": -23.311389923095703,
      "activations/layer21_attention_weight_max": 34.773841857910156,
      "activations/layer21_attention_weight_min": -22.951435089111328,
      "activations/layer22_attention_weight_max": 33.18543243408203,
      "activations/layer22_attention_weight_min": -23.78073501586914,
      "activations/layer23_attention_weight_max": 39.784278869628906,
      "activations/layer23_attention_weight_min": -20.728713989257812,
      "activations/layer2_attention_weight_max": 32.020111083984375,
      "activations/layer2_attention_weight_min": -30.915931701660156,
      "activations/layer3_attention_weight_max": 90.57721710205078,
      "activations/layer3_attention_weight_min": -89.61682891845703,
      "activations/layer4_attention_weight_max": 101.32035064697266,
      "activations/layer4_attention_weight_min": -96.0914535522461,
      "activations/layer5_attention_weight_max": 77.86885070800781,
      "activations/layer5_attention_weight_min": -80.6590347290039,
      "activations/layer6_attention_weight_max": 56.28084182739258,
      "activations/layer6_attention_weight_min": -53.50070571899414,
      "activations/layer7_attention_weight_max": 79.44493103027344,
      "activations/layer7_attention_weight_min": -76.76050567626953,
      "activations/layer8_attention_weight_max": 47.70738220214844,
      "activations/layer8_attention_weight_min": -48.81795120239258,
      "activations/layer9_attention_weight_max": 50.082759857177734,
      "activations/layer9_attention_weight_min": -49.0557975769043,
      "epoch": 9.7,
      "learning_rate": 8.831287878787879e-05,
      "loss": 2.7935,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.5277,
      "eval_samples_per_second": 503.537,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.5277,
      "eval_openwebtext_samples_per_second": 503.537,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_wikitext_loss": 2.982421875,
      "eval_wikitext_ppl": 19.735555861572347,
      "eval_wikitext_runtime": 1.9649,
      "eval_wikitext_samples_per_second": 232.07,
      "step": 167000
    },
    {
      "epoch": 9.7,
      "eval_lambada_loss": 2.59375,
      "eval_lambada_ppl": 13.379852088930456,
      "eval_lambada_runtime": 9.5663,
      "eval_lambada_samples_per_second": 508.975,
      "step": 167000
    },
    {
      "activations/layer0_attention_weight_max": 14.663678169250488,
      "activations/layer0_attention_weight_min": -13.985970497131348,
      "activations/layer10_attention_weight_max": 39.17838668823242,
      "activations/layer10_attention_weight_min": -36.3981819152832,
      "activations/layer11_attention_weight_max": 33.398948669433594,
      "activations/layer11_attention_weight_min": -31.639179229736328,
      "activations/layer12_attention_weight_max": 19.497262954711914,
      "activations/layer12_attention_weight_min": -27.338041305541992,
      "activations/layer13_attention_weight_max": 40.86811828613281,
      "activations/layer13_attention_weight_min": -32.28951644897461,
      "activations/layer14_attention_weight_max": 44.0748176574707,
      "activations/layer14_attention_weight_min": -31.841999053955078,
      "activations/layer15_attention_weight_max": 35.93412780761719,
      "activations/layer15_attention_weight_min": -30.906524658203125,
      "activations/layer16_attention_weight_max": 31.867338180541992,
      "activations/layer16_attention_weight_min": -31.63519287109375,
      "activations/layer17_attention_weight_max": 52.592041015625,
      "activations/layer17_attention_weight_min": -44.736839294433594,
      "activations/layer18_attention_weight_max": 52.36909103393555,
      "activations/layer18_attention_weight_min": -39.22571563720703,
      "activations/layer19_attention_weight_max": 25.552289962768555,
      "activations/layer19_attention_weight_min": -22.866788864135742,
      "activations/layer1_attention_weight_max": 17.417911529541016,
      "activations/layer1_attention_weight_min": -15.193222045898438,
      "activations/layer20_attention_weight_max": 22.82882308959961,
      "activations/layer20_attention_weight_min": -22.732725143432617,
      "activations/layer21_attention_weight_max": 40.00533676147461,
      "activations/layer21_attention_weight_min": -25.137588500976562,
      "activations/layer22_attention_weight_max": 34.459590911865234,
      "activations/layer22_attention_weight_min": -27.281709671020508,
      "activations/layer23_attention_weight_max": 39.23046875,
      "activations/layer23_attention_weight_min": -22.771461486816406,
      "activations/layer2_attention_weight_max": 31.565284729003906,
      "activations/layer2_attention_weight_min": -30.78554344177246,
      "activations/layer3_attention_weight_max": 96.29381561279297,
      "activations/layer3_attention_weight_min": -97.48240661621094,
      "activations/layer4_attention_weight_max": 97.93107604980469,
      "activations/layer4_attention_weight_min": -99.02655792236328,
      "activations/layer5_attention_weight_max": 76.54174041748047,
      "activations/layer5_attention_weight_min": -77.18434143066406,
      "activations/layer6_attention_weight_max": 54.126041412353516,
      "activations/layer6_attention_weight_min": -51.60464859008789,
      "activations/layer7_attention_weight_max": 75.50711822509766,
      "activations/layer7_attention_weight_min": -74.17169189453125,
      "activations/layer8_attention_weight_max": 46.5677604675293,
      "activations/layer8_attention_weight_min": -50.16670608520508,
      "activations/layer9_attention_weight_max": 49.9858283996582,
      "activations/layer9_attention_weight_min": -50.33621597290039,
      "epoch": 9.71,
      "learning_rate": 8.829393939393939e-05,
      "loss": 2.8077,
      "step": 167050
    },
    {
      "activations/layer0_attention_weight_max": 15.346774101257324,
      "activations/layer0_attention_weight_min": -14.03827953338623,
      "activations/layer10_attention_weight_max": 40.953006744384766,
      "activations/layer10_attention_weight_min": -37.87314987182617,
      "activations/layer11_attention_weight_max": 34.175537109375,
      "activations/layer11_attention_weight_min": -34.767723083496094,
      "activations/layer12_attention_weight_max": 18.80362319946289,
      "activations/layer12_attention_weight_min": -25.033720016479492,
      "activations/layer13_attention_weight_max": 41.579429626464844,
      "activations/layer13_attention_weight_min": -29.962846755981445,
      "activations/layer14_attention_weight_max": 39.32450485229492,
      "activations/layer14_attention_weight_min": -31.55462646484375,
      "activations/layer15_attention_weight_max": 35.17439270019531,
      "activations/layer15_attention_weight_min": -31.783693313598633,
      "activations/layer16_attention_weight_max": 30.152099609375,
      "activations/layer16_attention_weight_min": -29.517141342163086,
      "activations/layer17_attention_weight_max": 56.2985725402832,
      "activations/layer17_attention_weight_min": -44.35525894165039,
      "activations/layer18_attention_weight_max": 48.56782150268555,
      "activations/layer18_attention_weight_min": -37.060546875,
      "activations/layer19_attention_weight_max": 21.911928176879883,
      "activations/layer19_attention_weight_min": -21.46733856201172,
      "activations/layer1_attention_weight_max": 17.30759048461914,
      "activations/layer1_attention_weight_min": -14.71580982208252,
      "activations/layer20_attention_weight_max": 29.649669647216797,
      "activations/layer20_attention_weight_min": -22.02659034729004,
      "activations/layer21_attention_weight_max": 34.8645133972168,
      "activations/layer21_attention_weight_min": -22.3706111907959,
      "activations/layer22_attention_weight_max": 34.9938850402832,
      "activations/layer22_attention_weight_min": -24.321533203125,
      "activations/layer23_attention_weight_max": 40.63792037963867,
      "activations/layer23_attention_weight_min": -25.023174285888672,
      "activations/layer2_attention_weight_max": 32.45063781738281,
      "activations/layer2_attention_weight_min": -29.40907096862793,
      "activations/layer3_attention_weight_max": 90.48577117919922,
      "activations/layer3_attention_weight_min": -95.57453918457031,
      "activations/layer4_attention_weight_max": 100.64738464355469,
      "activations/layer4_attention_weight_min": -97.10514068603516,
      "activations/layer5_attention_weight_max": 79.35521697998047,
      "activations/layer5_attention_weight_min": -81.54986572265625,
      "activations/layer6_attention_weight_max": 55.02138137817383,
      "activations/layer6_attention_weight_min": -55.68896484375,
      "activations/layer7_attention_weight_max": 77.82027435302734,
      "activations/layer7_attention_weight_min": -74.04046630859375,
      "activations/layer8_attention_weight_max": 46.58848190307617,
      "activations/layer8_attention_weight_min": -49.20233917236328,
      "activations/layer9_attention_weight_max": 50.447486877441406,
      "activations/layer9_attention_weight_min": -50.77808380126953,
      "epoch": 9.71,
      "learning_rate": 8.827499999999999e-05,
      "loss": 2.7954,
      "step": 167100
    },
    {
      "activations/layer0_attention_weight_max": 15.42768669128418,
      "activations/layer0_attention_weight_min": -13.746977806091309,
      "activations/layer10_attention_weight_max": 40.8675537109375,
      "activations/layer10_attention_weight_min": -37.857173919677734,
      "activations/layer11_attention_weight_max": 34.520416259765625,
      "activations/layer11_attention_weight_min": -32.6820182800293,
      "activations/layer12_attention_weight_max": 32.24656295776367,
      "activations/layer12_attention_weight_min": -26.46381378173828,
      "activations/layer13_attention_weight_max": 40.07378005981445,
      "activations/layer13_attention_weight_min": -31.420751571655273,
      "activations/layer14_attention_weight_max": 35.91727066040039,
      "activations/layer14_attention_weight_min": -31.510313034057617,
      "activations/layer15_attention_weight_max": 37.3042106628418,
      "activations/layer15_attention_weight_min": -29.849754333496094,
      "activations/layer16_attention_weight_max": 30.11376190185547,
      "activations/layer16_attention_weight_min": -28.207124710083008,
      "activations/layer17_attention_weight_max": 55.80409622192383,
      "activations/layer17_attention_weight_min": -44.962100982666016,
      "activations/layer18_attention_weight_max": 46.45433807373047,
      "activations/layer18_attention_weight_min": -35.441959381103516,
      "activations/layer19_attention_weight_max": 23.32478904724121,
      "activations/layer19_attention_weight_min": -21.26498794555664,
      "activations/layer1_attention_weight_max": 16.376808166503906,
      "activations/layer1_attention_weight_min": -15.566498756408691,
      "activations/layer20_attention_weight_max": 22.03135871887207,
      "activations/layer20_attention_weight_min": -25.317005157470703,
      "activations/layer21_attention_weight_max": 38.55746078491211,
      "activations/layer21_attention_weight_min": -25.23627281188965,
      "activations/layer22_attention_weight_max": 35.5234489440918,
      "activations/layer22_attention_weight_min": -25.06092643737793,
      "activations/layer23_attention_weight_max": 37.67535400390625,
      "activations/layer23_attention_weight_min": -21.627241134643555,
      "activations/layer2_attention_weight_max": 34.25790786743164,
      "activations/layer2_attention_weight_min": -33.56559753417969,
      "activations/layer3_attention_weight_max": 93.56720733642578,
      "activations/layer3_attention_weight_min": -95.29450988769531,
      "activations/layer4_attention_weight_max": 100.03667449951172,
      "activations/layer4_attention_weight_min": -101.5667953491211,
      "activations/layer5_attention_weight_max": 80.11369323730469,
      "activations/layer5_attention_weight_min": -81.73175048828125,
      "activations/layer6_attention_weight_max": 56.16259765625,
      "activations/layer6_attention_weight_min": -56.06181716918945,
      "activations/layer7_attention_weight_max": 82.44697570800781,
      "activations/layer7_attention_weight_min": -76.82085418701172,
      "activations/layer8_attention_weight_max": 46.414730072021484,
      "activations/layer8_attention_weight_min": -47.44742202758789,
      "activations/layer9_attention_weight_max": 53.87713623046875,
      "activations/layer9_attention_weight_min": -50.086421966552734,
      "epoch": 9.71,
      "learning_rate": 8.825606060606059e-05,
      "loss": 2.7879,
      "step": 167150
    },
    {
      "activations/layer0_attention_weight_max": 18.057846069335938,
      "activations/layer0_attention_weight_min": -14.318246841430664,
      "activations/layer10_attention_weight_max": 37.333770751953125,
      "activations/layer10_attention_weight_min": -37.22625732421875,
      "activations/layer11_attention_weight_max": 33.21562194824219,
      "activations/layer11_attention_weight_min": -31.58120346069336,
      "activations/layer12_attention_weight_max": 17.920278549194336,
      "activations/layer12_attention_weight_min": -27.31895637512207,
      "activations/layer13_attention_weight_max": 48.67900848388672,
      "activations/layer13_attention_weight_min": -33.440696716308594,
      "activations/layer14_attention_weight_max": 39.144325256347656,
      "activations/layer14_attention_weight_min": -33.000282287597656,
      "activations/layer15_attention_weight_max": 33.577781677246094,
      "activations/layer15_attention_weight_min": -31.54050636291504,
      "activations/layer16_attention_weight_max": 31.719701766967773,
      "activations/layer16_attention_weight_min": -29.79918670654297,
      "activations/layer17_attention_weight_max": 55.59920120239258,
      "activations/layer17_attention_weight_min": -46.242374420166016,
      "activations/layer18_attention_weight_max": 52.94463348388672,
      "activations/layer18_attention_weight_min": -37.725765228271484,
      "activations/layer19_attention_weight_max": 24.600440979003906,
      "activations/layer19_attention_weight_min": -22.63881492614746,
      "activations/layer1_attention_weight_max": 16.872013092041016,
      "activations/layer1_attention_weight_min": -15.054232597351074,
      "activations/layer20_attention_weight_max": 24.685874938964844,
      "activations/layer20_attention_weight_min": -22.742658615112305,
      "activations/layer21_attention_weight_max": 44.32707214355469,
      "activations/layer21_attention_weight_min": -26.65413475036621,
      "activations/layer22_attention_weight_max": 32.75423812866211,
      "activations/layer22_attention_weight_min": -24.967527389526367,
      "activations/layer23_attention_weight_max": 40.66206359863281,
      "activations/layer23_attention_weight_min": -27.152156829833984,
      "activations/layer2_attention_weight_max": 32.50025939941406,
      "activations/layer2_attention_weight_min": -29.48379898071289,
      "activations/layer3_attention_weight_max": 91.65263366699219,
      "activations/layer3_attention_weight_min": -92.39876556396484,
      "activations/layer4_attention_weight_max": 97.351806640625,
      "activations/layer4_attention_weight_min": -94.99346160888672,
      "activations/layer5_attention_weight_max": 76.34376525878906,
      "activations/layer5_attention_weight_min": -77.21267700195312,
      "activations/layer6_attention_weight_max": 55.27101516723633,
      "activations/layer6_attention_weight_min": -51.19163513183594,
      "activations/layer7_attention_weight_max": 77.92359924316406,
      "activations/layer7_attention_weight_min": -74.22608184814453,
      "activations/layer8_attention_weight_max": 46.1484375,
      "activations/layer8_attention_weight_min": -46.19942092895508,
      "activations/layer9_attention_weight_max": 53.98177719116211,
      "activations/layer9_attention_weight_min": -48.72513961791992,
      "epoch": 9.72,
      "learning_rate": 8.823712121212121e-05,
      "loss": 2.789,
      "step": 167200
    },
    {
      "activations/layer0_attention_weight_max": 15.10818862915039,
      "activations/layer0_attention_weight_min": -13.545479774475098,
      "activations/layer10_attention_weight_max": 41.02963638305664,
      "activations/layer10_attention_weight_min": -37.47669982910156,
      "activations/layer11_attention_weight_max": 36.678741455078125,
      "activations/layer11_attention_weight_min": -33.416908264160156,
      "activations/layer12_attention_weight_max": 19.133085250854492,
      "activations/layer12_attention_weight_min": -27.531490325927734,
      "activations/layer13_attention_weight_max": 37.82101058959961,
      "activations/layer13_attention_weight_min": -30.996387481689453,
      "activations/layer14_attention_weight_max": 38.30295944213867,
      "activations/layer14_attention_weight_min": -30.354204177856445,
      "activations/layer15_attention_weight_max": 37.05381393432617,
      "activations/layer15_attention_weight_min": -28.3839054107666,
      "activations/layer16_attention_weight_max": 34.65351867675781,
      "activations/layer16_attention_weight_min": -28.902345657348633,
      "activations/layer17_attention_weight_max": 54.37395477294922,
      "activations/layer17_attention_weight_min": -44.09065628051758,
      "activations/layer18_attention_weight_max": 48.74661636352539,
      "activations/layer18_attention_weight_min": -38.243499755859375,
      "activations/layer19_attention_weight_max": 20.502010345458984,
      "activations/layer19_attention_weight_min": -20.07426643371582,
      "activations/layer1_attention_weight_max": 16.38237953186035,
      "activations/layer1_attention_weight_min": -15.256897926330566,
      "activations/layer20_attention_weight_max": 25.409954071044922,
      "activations/layer20_attention_weight_min": -22.356590270996094,
      "activations/layer21_attention_weight_max": 38.24180603027344,
      "activations/layer21_attention_weight_min": -22.52128028869629,
      "activations/layer22_attention_weight_max": 31.478391647338867,
      "activations/layer22_attention_weight_min": -26.42409324645996,
      "activations/layer23_attention_weight_max": 37.74625015258789,
      "activations/layer23_attention_weight_min": -26.314712524414062,
      "activations/layer2_attention_weight_max": 34.16939926147461,
      "activations/layer2_attention_weight_min": -30.748615264892578,
      "activations/layer3_attention_weight_max": 90.62877655029297,
      "activations/layer3_attention_weight_min": -93.20661163330078,
      "activations/layer4_attention_weight_max": 101.82683563232422,
      "activations/layer4_attention_weight_min": -99.2138900756836,
      "activations/layer5_attention_weight_max": 79.35462951660156,
      "activations/layer5_attention_weight_min": -80.19686889648438,
      "activations/layer6_attention_weight_max": 55.24457931518555,
      "activations/layer6_attention_weight_min": -52.26152801513672,
      "activations/layer7_attention_weight_max": 77.64635467529297,
      "activations/layer7_attention_weight_min": -74.97418212890625,
      "activations/layer8_attention_weight_max": 49.8170166015625,
      "activations/layer8_attention_weight_min": -46.13173294067383,
      "activations/layer9_attention_weight_max": 55.97825241088867,
      "activations/layer9_attention_weight_min": -48.99916458129883,
      "epoch": 9.72,
      "learning_rate": 8.821818181818181e-05,
      "loss": 2.7853,
      "step": 167250
    },
    {
      "activations/layer0_attention_weight_max": 15.368399620056152,
      "activations/layer0_attention_weight_min": -13.778877258300781,
      "activations/layer10_attention_weight_max": 36.92148208618164,
      "activations/layer10_attention_weight_min": -36.739768981933594,
      "activations/layer11_attention_weight_max": 30.94601058959961,
      "activations/layer11_attention_weight_min": -31.867717742919922,
      "activations/layer12_attention_weight_max": 19.433530807495117,
      "activations/layer12_attention_weight_min": -22.56014633178711,
      "activations/layer13_attention_weight_max": 38.15690612792969,
      "activations/layer13_attention_weight_min": -26.74704360961914,
      "activations/layer14_attention_weight_max": 39.738380432128906,
      "activations/layer14_attention_weight_min": -30.249553680419922,
      "activations/layer15_attention_weight_max": 32.38229751586914,
      "activations/layer15_attention_weight_min": -29.034130096435547,
      "activations/layer16_attention_weight_max": 29.359683990478516,
      "activations/layer16_attention_weight_min": -27.897533416748047,
      "activations/layer17_attention_weight_max": 51.93122863769531,
      "activations/layer17_attention_weight_min": -41.92589569091797,
      "activations/layer18_attention_weight_max": 45.047447204589844,
      "activations/layer18_attention_weight_min": -36.18065643310547,
      "activations/layer19_attention_weight_max": 23.39925193786621,
      "activations/layer19_attention_weight_min": -20.676612854003906,
      "activations/layer1_attention_weight_max": 18.18598747253418,
      "activations/layer1_attention_weight_min": -16.43741226196289,
      "activations/layer20_attention_weight_max": 21.245498657226562,
      "activations/layer20_attention_weight_min": -21.459848403930664,
      "activations/layer21_attention_weight_max": 36.32634735107422,
      "activations/layer21_attention_weight_min": -22.28185272216797,
      "activations/layer22_attention_weight_max": 29.352998733520508,
      "activations/layer22_attention_weight_min": -25.006696701049805,
      "activations/layer23_attention_weight_max": 35.66423034667969,
      "activations/layer23_attention_weight_min": -23.21210289001465,
      "activations/layer2_attention_weight_max": 32.28065490722656,
      "activations/layer2_attention_weight_min": -31.832660675048828,
      "activations/layer3_attention_weight_max": 94.23502349853516,
      "activations/layer3_attention_weight_min": -92.0008316040039,
      "activations/layer4_attention_weight_max": 103.98052978515625,
      "activations/layer4_attention_weight_min": -100.83930206298828,
      "activations/layer5_attention_weight_max": 79.4326171875,
      "activations/layer5_attention_weight_min": -78.49253845214844,
      "activations/layer6_attention_weight_max": 56.718910217285156,
      "activations/layer6_attention_weight_min": -54.62525939941406,
      "activations/layer7_attention_weight_max": 79.41281127929688,
      "activations/layer7_attention_weight_min": -73.908203125,
      "activations/layer8_attention_weight_max": 47.221858978271484,
      "activations/layer8_attention_weight_min": -48.6463737487793,
      "activations/layer9_attention_weight_max": 53.478328704833984,
      "activations/layer9_attention_weight_min": -51.70558547973633,
      "epoch": 9.72,
      "learning_rate": 8.819924242424241e-05,
      "loss": 2.8014,
      "step": 167300
    },
    {
      "activations/layer0_attention_weight_max": 15.437641143798828,
      "activations/layer0_attention_weight_min": -14.160582542419434,
      "activations/layer10_attention_weight_max": 37.95029830932617,
      "activations/layer10_attention_weight_min": -36.27407455444336,
      "activations/layer11_attention_weight_max": 34.91725158691406,
      "activations/layer11_attention_weight_min": -34.866943359375,
      "activations/layer12_attention_weight_max": 21.6136417388916,
      "activations/layer12_attention_weight_min": -27.777454376220703,
      "activations/layer13_attention_weight_max": 44.241275787353516,
      "activations/layer13_attention_weight_min": -31.07432746887207,
      "activations/layer14_attention_weight_max": 37.39179992675781,
      "activations/layer14_attention_weight_min": -30.662015914916992,
      "activations/layer15_attention_weight_max": 37.51073455810547,
      "activations/layer15_attention_weight_min": -30.197528839111328,
      "activations/layer16_attention_weight_max": 34.00523376464844,
      "activations/layer16_attention_weight_min": -28.349035263061523,
      "activations/layer17_attention_weight_max": 54.02788543701172,
      "activations/layer17_attention_weight_min": -44.07697296142578,
      "activations/layer18_attention_weight_max": 45.900909423828125,
      "activations/layer18_attention_weight_min": -37.11295700073242,
      "activations/layer19_attention_weight_max": 21.601560592651367,
      "activations/layer19_attention_weight_min": -22.06769371032715,
      "activations/layer1_attention_weight_max": 17.81874656677246,
      "activations/layer1_attention_weight_min": -17.583486557006836,
      "activations/layer20_attention_weight_max": 24.817398071289062,
      "activations/layer20_attention_weight_min": -22.479169845581055,
      "activations/layer21_attention_weight_max": 40.26165771484375,
      "activations/layer21_attention_weight_min": -22.898141860961914,
      "activations/layer22_attention_weight_max": 33.17655563354492,
      "activations/layer22_attention_weight_min": -27.511600494384766,
      "activations/layer23_attention_weight_max": 38.694244384765625,
      "activations/layer23_attention_weight_min": -24.51998519897461,
      "activations/layer2_attention_weight_max": 32.74867630004883,
      "activations/layer2_attention_weight_min": -30.897602081298828,
      "activations/layer3_attention_weight_max": 88.91495513916016,
      "activations/layer3_attention_weight_min": -94.93507385253906,
      "activations/layer4_attention_weight_max": 102.0191879272461,
      "activations/layer4_attention_weight_min": -101.44002532958984,
      "activations/layer5_attention_weight_max": 79.20785522460938,
      "activations/layer5_attention_weight_min": -79.61805725097656,
      "activations/layer6_attention_weight_max": 54.23231506347656,
      "activations/layer6_attention_weight_min": -52.54130172729492,
      "activations/layer7_attention_weight_max": 79.66398620605469,
      "activations/layer7_attention_weight_min": -76.09983825683594,
      "activations/layer8_attention_weight_max": 48.55064392089844,
      "activations/layer8_attention_weight_min": -48.81924819946289,
      "activations/layer9_attention_weight_max": 51.09635543823242,
      "activations/layer9_attention_weight_min": -49.73320007324219,
      "epoch": 9.72,
      "learning_rate": 8.818030303030303e-05,
      "loss": 2.7791,
      "step": 167350
    },
    {
      "activations/layer0_attention_weight_max": 15.7550630569458,
      "activations/layer0_attention_weight_min": -14.142827033996582,
      "activations/layer10_attention_weight_max": 36.14009094238281,
      "activations/layer10_attention_weight_min": -34.55131912231445,
      "activations/layer11_attention_weight_max": 31.586074829101562,
      "activations/layer11_attention_weight_min": -32.812828063964844,
      "activations/layer12_attention_weight_max": 20.035978317260742,
      "activations/layer12_attention_weight_min": -27.28999137878418,
      "activations/layer13_attention_weight_max": 40.39073944091797,
      "activations/layer13_attention_weight_min": -33.146522521972656,
      "activations/layer14_attention_weight_max": 37.72926330566406,
      "activations/layer14_attention_weight_min": -32.71876907348633,
      "activations/layer15_attention_weight_max": 33.52368927001953,
      "activations/layer15_attention_weight_min": -30.82823371887207,
      "activations/layer16_attention_weight_max": 31.377761840820312,
      "activations/layer16_attention_weight_min": -29.458019256591797,
      "activations/layer17_attention_weight_max": 55.18566131591797,
      "activations/layer17_attention_weight_min": -45.04829788208008,
      "activations/layer18_attention_weight_max": 45.668235778808594,
      "activations/layer18_attention_weight_min": -37.3060417175293,
      "activations/layer19_attention_weight_max": 21.696273803710938,
      "activations/layer19_attention_weight_min": -23.060258865356445,
      "activations/layer1_attention_weight_max": 16.85016632080078,
      "activations/layer1_attention_weight_min": -15.093677520751953,
      "activations/layer20_attention_weight_max": 25.39217185974121,
      "activations/layer20_attention_weight_min": -23.35110092163086,
      "activations/layer21_attention_weight_max": 37.59956359863281,
      "activations/layer21_attention_weight_min": -25.557462692260742,
      "activations/layer22_attention_weight_max": 31.973209381103516,
      "activations/layer22_attention_weight_min": -26.80217742919922,
      "activations/layer23_attention_weight_max": 36.61760711669922,
      "activations/layer23_attention_weight_min": -25.485576629638672,
      "activations/layer2_attention_weight_max": 33.98554229736328,
      "activations/layer2_attention_weight_min": -30.536365509033203,
      "activations/layer3_attention_weight_max": 90.20413970947266,
      "activations/layer3_attention_weight_min": -97.54322814941406,
      "activations/layer4_attention_weight_max": 103.52961730957031,
      "activations/layer4_attention_weight_min": -100.49893188476562,
      "activations/layer5_attention_weight_max": 80.11282348632812,
      "activations/layer5_attention_weight_min": -79.96989440917969,
      "activations/layer6_attention_weight_max": 56.58536148071289,
      "activations/layer6_attention_weight_min": -55.826847076416016,
      "activations/layer7_attention_weight_max": 82.91586303710938,
      "activations/layer7_attention_weight_min": -74.22166442871094,
      "activations/layer8_attention_weight_max": 46.75044631958008,
      "activations/layer8_attention_weight_min": -46.44938659667969,
      "activations/layer9_attention_weight_max": 51.94618606567383,
      "activations/layer9_attention_weight_min": -48.03114318847656,
      "epoch": 9.73,
      "learning_rate": 8.816136363636364e-05,
      "loss": 2.7904,
      "step": 167400
    },
    {
      "activations/layer0_attention_weight_max": 15.57504653930664,
      "activations/layer0_attention_weight_min": -14.216113090515137,
      "activations/layer10_attention_weight_max": 39.29026794433594,
      "activations/layer10_attention_weight_min": -35.83871841430664,
      "activations/layer11_attention_weight_max": 33.22957229614258,
      "activations/layer11_attention_weight_min": -30.935741424560547,
      "activations/layer12_attention_weight_max": 19.957353591918945,
      "activations/layer12_attention_weight_min": -28.035072326660156,
      "activations/layer13_attention_weight_max": 45.628475189208984,
      "activations/layer13_attention_weight_min": -33.224544525146484,
      "activations/layer14_attention_weight_max": 38.84894561767578,
      "activations/layer14_attention_weight_min": -29.38194465637207,
      "activations/layer15_attention_weight_max": 39.625423431396484,
      "activations/layer15_attention_weight_min": -31.21268081665039,
      "activations/layer16_attention_weight_max": 35.67970657348633,
      "activations/layer16_attention_weight_min": -28.849628448486328,
      "activations/layer17_attention_weight_max": 55.238616943359375,
      "activations/layer17_attention_weight_min": -43.38017654418945,
      "activations/layer18_attention_weight_max": 51.597660064697266,
      "activations/layer18_attention_weight_min": -37.77463912963867,
      "activations/layer19_attention_weight_max": 24.565217971801758,
      "activations/layer19_attention_weight_min": -23.267942428588867,
      "activations/layer1_attention_weight_max": 16.86224937438965,
      "activations/layer1_attention_weight_min": -15.031658172607422,
      "activations/layer20_attention_weight_max": 26.725173950195312,
      "activations/layer20_attention_weight_min": -23.3510799407959,
      "activations/layer21_attention_weight_max": 47.0562744140625,
      "activations/layer21_attention_weight_min": -22.224634170532227,
      "activations/layer22_attention_weight_max": 37.717891693115234,
      "activations/layer22_attention_weight_min": -24.332775115966797,
      "activations/layer23_attention_weight_max": 45.969085693359375,
      "activations/layer23_attention_weight_min": -22.400123596191406,
      "activations/layer2_attention_weight_max": 34.08482360839844,
      "activations/layer2_attention_weight_min": -31.810016632080078,
      "activations/layer3_attention_weight_max": 95.80086517333984,
      "activations/layer3_attention_weight_min": -95.12250518798828,
      "activations/layer4_attention_weight_max": 100.71409606933594,
      "activations/layer4_attention_weight_min": -95.63912963867188,
      "activations/layer5_attention_weight_max": 78.77162170410156,
      "activations/layer5_attention_weight_min": -79.39348602294922,
      "activations/layer6_attention_weight_max": 53.647586822509766,
      "activations/layer6_attention_weight_min": -53.00330352783203,
      "activations/layer7_attention_weight_max": 75.53709411621094,
      "activations/layer7_attention_weight_min": -76.30989837646484,
      "activations/layer8_attention_weight_max": 45.85182189941406,
      "activations/layer8_attention_weight_min": -46.62518310546875,
      "activations/layer9_attention_weight_max": 47.70241928100586,
      "activations/layer9_attention_weight_min": -46.72845458984375,
      "epoch": 9.73,
      "learning_rate": 8.814242424242423e-05,
      "loss": 2.8022,
      "step": 167450
    },
    {
      "activations/layer0_attention_weight_max": 15.107158660888672,
      "activations/layer0_attention_weight_min": -14.114890098571777,
      "activations/layer10_attention_weight_max": 36.60377883911133,
      "activations/layer10_attention_weight_min": -34.61361312866211,
      "activations/layer11_attention_weight_max": 32.045143127441406,
      "activations/layer11_attention_weight_min": -31.779861450195312,
      "activations/layer12_attention_weight_max": 18.676786422729492,
      "activations/layer12_attention_weight_min": -25.68606948852539,
      "activations/layer13_attention_weight_max": 32.570289611816406,
      "activations/layer13_attention_weight_min": -30.898921966552734,
      "activations/layer14_attention_weight_max": 35.474090576171875,
      "activations/layer14_attention_weight_min": -30.561542510986328,
      "activations/layer15_attention_weight_max": 30.78313636779785,
      "activations/layer15_attention_weight_min": -30.210670471191406,
      "activations/layer16_attention_weight_max": 28.098651885986328,
      "activations/layer16_attention_weight_min": -28.377986907958984,
      "activations/layer17_attention_weight_max": 49.27603530883789,
      "activations/layer17_attention_weight_min": -42.240474700927734,
      "activations/layer18_attention_weight_max": 42.21632385253906,
      "activations/layer18_attention_weight_min": -36.94716262817383,
      "activations/layer19_attention_weight_max": 21.459749221801758,
      "activations/layer19_attention_weight_min": -23.19379234313965,
      "activations/layer1_attention_weight_max": 16.679628372192383,
      "activations/layer1_attention_weight_min": -17.593647003173828,
      "activations/layer20_attention_weight_max": 23.623424530029297,
      "activations/layer20_attention_weight_min": -22.393007278442383,
      "activations/layer21_attention_weight_max": 36.606082916259766,
      "activations/layer21_attention_weight_min": -25.3846492767334,
      "activations/layer22_attention_weight_max": 33.17723846435547,
      "activations/layer22_attention_weight_min": -25.279483795166016,
      "activations/layer23_attention_weight_max": 35.46466827392578,
      "activations/layer23_attention_weight_min": -21.717260360717773,
      "activations/layer2_attention_weight_max": 34.17288589477539,
      "activations/layer2_attention_weight_min": -33.78977966308594,
      "activations/layer3_attention_weight_max": 93.27946472167969,
      "activations/layer3_attention_weight_min": -93.192138671875,
      "activations/layer4_attention_weight_max": 100.42304229736328,
      "activations/layer4_attention_weight_min": -96.45276641845703,
      "activations/layer5_attention_weight_max": 78.9388656616211,
      "activations/layer5_attention_weight_min": -73.07012939453125,
      "activations/layer6_attention_weight_max": 52.247398376464844,
      "activations/layer6_attention_weight_min": -52.39683151245117,
      "activations/layer7_attention_weight_max": 76.47908020019531,
      "activations/layer7_attention_weight_min": -75.60939025878906,
      "activations/layer8_attention_weight_max": 44.61002731323242,
      "activations/layer8_attention_weight_min": -46.71590042114258,
      "activations/layer9_attention_weight_max": 48.407371520996094,
      "activations/layer9_attention_weight_min": -47.58852767944336,
      "epoch": 9.73,
      "learning_rate": 8.812386363636362e-05,
      "loss": 2.7959,
      "step": 167500
    },
    {
      "activations/layer0_attention_weight_max": 14.931963920593262,
      "activations/layer0_attention_weight_min": -14.405292510986328,
      "activations/layer10_attention_weight_max": 34.31804275512695,
      "activations/layer10_attention_weight_min": -36.907630920410156,
      "activations/layer11_attention_weight_max": 28.553993225097656,
      "activations/layer11_attention_weight_min": -30.298114776611328,
      "activations/layer12_attention_weight_max": 18.53875732421875,
      "activations/layer12_attention_weight_min": -22.781326293945312,
      "activations/layer13_attention_weight_max": 40.51337432861328,
      "activations/layer13_attention_weight_min": -32.58713150024414,
      "activations/layer14_attention_weight_max": 33.80318069458008,
      "activations/layer14_attention_weight_min": -29.947858810424805,
      "activations/layer15_attention_weight_max": 36.210693359375,
      "activations/layer15_attention_weight_min": -28.134227752685547,
      "activations/layer16_attention_weight_max": 30.631567001342773,
      "activations/layer16_attention_weight_min": -29.135480880737305,
      "activations/layer17_attention_weight_max": 50.37963104248047,
      "activations/layer17_attention_weight_min": -42.23485565185547,
      "activations/layer18_attention_weight_max": 43.50551986694336,
      "activations/layer18_attention_weight_min": -36.962154388427734,
      "activations/layer19_attention_weight_max": 23.071619033813477,
      "activations/layer19_attention_weight_min": -20.298236846923828,
      "activations/layer1_attention_weight_max": 17.040054321289062,
      "activations/layer1_attention_weight_min": -15.204200744628906,
      "activations/layer20_attention_weight_max": 23.241514205932617,
      "activations/layer20_attention_weight_min": -22.326047897338867,
      "activations/layer21_attention_weight_max": 39.760433197021484,
      "activations/layer21_attention_weight_min": -22.515846252441406,
      "activations/layer22_attention_weight_max": 28.53546905517578,
      "activations/layer22_attention_weight_min": -27.948755264282227,
      "activations/layer23_attention_weight_max": 35.862770080566406,
      "activations/layer23_attention_weight_min": -24.64528465270996,
      "activations/layer2_attention_weight_max": 31.82375144958496,
      "activations/layer2_attention_weight_min": -30.94251251220703,
      "activations/layer3_attention_weight_max": 89.99459075927734,
      "activations/layer3_attention_weight_min": -91.54293060302734,
      "activations/layer4_attention_weight_max": 95.27232360839844,
      "activations/layer4_attention_weight_min": -97.02354431152344,
      "activations/layer5_attention_weight_max": 73.51972961425781,
      "activations/layer5_attention_weight_min": -77.45506286621094,
      "activations/layer6_attention_weight_max": 52.40082931518555,
      "activations/layer6_attention_weight_min": -51.51306915283203,
      "activations/layer7_attention_weight_max": 72.98489379882812,
      "activations/layer7_attention_weight_min": -71.06354522705078,
      "activations/layer8_attention_weight_max": 43.16328048706055,
      "activations/layer8_attention_weight_min": -43.42710876464844,
      "activations/layer9_attention_weight_max": 47.137359619140625,
      "activations/layer9_attention_weight_min": -47.64520263671875,
      "epoch": 9.74,
      "learning_rate": 8.810492424242423e-05,
      "loss": 2.7941,
      "step": 167550
    },
    {
      "activations/layer0_attention_weight_max": 15.328782081604004,
      "activations/layer0_attention_weight_min": -13.638771057128906,
      "activations/layer10_attention_weight_max": 37.096466064453125,
      "activations/layer10_attention_weight_min": -35.93104553222656,
      "activations/layer11_attention_weight_max": 31.499807357788086,
      "activations/layer11_attention_weight_min": -32.25108337402344,
      "activations/layer12_attention_weight_max": 20.580867767333984,
      "activations/layer12_attention_weight_min": -24.095333099365234,
      "activations/layer13_attention_weight_max": 44.663536071777344,
      "activations/layer13_attention_weight_min": -33.35768127441406,
      "activations/layer14_attention_weight_max": 37.165340423583984,
      "activations/layer14_attention_weight_min": -31.579526901245117,
      "activations/layer15_attention_weight_max": 33.955631256103516,
      "activations/layer15_attention_weight_min": -32.5244255065918,
      "activations/layer16_attention_weight_max": 29.13456153869629,
      "activations/layer16_attention_weight_min": -28.195091247558594,
      "activations/layer17_attention_weight_max": 53.55891799926758,
      "activations/layer17_attention_weight_min": -44.51657485961914,
      "activations/layer18_attention_weight_max": 47.043155670166016,
      "activations/layer18_attention_weight_min": -36.109588623046875,
      "activations/layer19_attention_weight_max": 22.243993759155273,
      "activations/layer19_attention_weight_min": -20.568695068359375,
      "activations/layer1_attention_weight_max": 16.597206115722656,
      "activations/layer1_attention_weight_min": -15.27249813079834,
      "activations/layer20_attention_weight_max": 22.684274673461914,
      "activations/layer20_attention_weight_min": -22.433866500854492,
      "activations/layer21_attention_weight_max": 34.73637390136719,
      "activations/layer21_attention_weight_min": -22.693143844604492,
      "activations/layer22_attention_weight_max": 32.80167007446289,
      "activations/layer22_attention_weight_min": -27.077192306518555,
      "activations/layer23_attention_weight_max": 38.925350189208984,
      "activations/layer23_attention_weight_min": -22.69980239868164,
      "activations/layer2_attention_weight_max": 32.85340881347656,
      "activations/layer2_attention_weight_min": -29.879934310913086,
      "activations/layer3_attention_weight_max": 94.04725646972656,
      "activations/layer3_attention_weight_min": -95.52288818359375,
      "activations/layer4_attention_weight_max": 105.59935760498047,
      "activations/layer4_attention_weight_min": -99.05730438232422,
      "activations/layer5_attention_weight_max": 79.49712371826172,
      "activations/layer5_attention_weight_min": -80.14787292480469,
      "activations/layer6_attention_weight_max": 54.35434341430664,
      "activations/layer6_attention_weight_min": -52.686214447021484,
      "activations/layer7_attention_weight_max": 80.03169250488281,
      "activations/layer7_attention_weight_min": -76.7516098022461,
      "activations/layer8_attention_weight_max": 45.28568649291992,
      "activations/layer8_attention_weight_min": -46.48908996582031,
      "activations/layer9_attention_weight_max": 48.692195892333984,
      "activations/layer9_attention_weight_min": -50.60708999633789,
      "epoch": 9.74,
      "learning_rate": 8.808598484848485e-05,
      "loss": 2.8051,
      "step": 167600
    },
    {
      "activations/layer0_attention_weight_max": 15.354126930236816,
      "activations/layer0_attention_weight_min": -14.185382843017578,
      "activations/layer10_attention_weight_max": 48.63499069213867,
      "activations/layer10_attention_weight_min": -46.746456146240234,
      "activations/layer11_attention_weight_max": 41.03730010986328,
      "activations/layer11_attention_weight_min": -39.879486083984375,
      "activations/layer12_attention_weight_max": 27.563343048095703,
      "activations/layer12_attention_weight_min": -23.353910446166992,
      "activations/layer13_attention_weight_max": 46.21464538574219,
      "activations/layer13_attention_weight_min": -30.948854446411133,
      "activations/layer14_attention_weight_max": 38.902828216552734,
      "activations/layer14_attention_weight_min": -31.817827224731445,
      "activations/layer15_attention_weight_max": 36.20136260986328,
      "activations/layer15_attention_weight_min": -30.423709869384766,
      "activations/layer16_attention_weight_max": 30.684825897216797,
      "activations/layer16_attention_weight_min": -31.044872283935547,
      "activations/layer17_attention_weight_max": 53.503509521484375,
      "activations/layer17_attention_weight_min": -45.08912658691406,
      "activations/layer18_attention_weight_max": 44.71359634399414,
      "activations/layer18_attention_weight_min": -36.737693786621094,
      "activations/layer19_attention_weight_max": 22.76715850830078,
      "activations/layer19_attention_weight_min": -22.003459930419922,
      "activations/layer1_attention_weight_max": 17.098770141601562,
      "activations/layer1_attention_weight_min": -17.51355743408203,
      "activations/layer20_attention_weight_max": 22.617650985717773,
      "activations/layer20_attention_weight_min": -29.29442596435547,
      "activations/layer21_attention_weight_max": 34.27116775512695,
      "activations/layer21_attention_weight_min": -21.080652236938477,
      "activations/layer22_attention_weight_max": 29.903793334960938,
      "activations/layer22_attention_weight_min": -26.812347412109375,
      "activations/layer23_attention_weight_max": 33.75006866455078,
      "activations/layer23_attention_weight_min": -22.390470504760742,
      "activations/layer2_attention_weight_max": 35.40484619140625,
      "activations/layer2_attention_weight_min": -32.88319396972656,
      "activations/layer3_attention_weight_max": 102.79309844970703,
      "activations/layer3_attention_weight_min": -104.81414031982422,
      "activations/layer4_attention_weight_max": 119.18963623046875,
      "activations/layer4_attention_weight_min": -112.8559341430664,
      "activations/layer5_attention_weight_max": 90.46183776855469,
      "activations/layer5_attention_weight_min": -84.9651107788086,
      "activations/layer6_attention_weight_max": 66.14785766601562,
      "activations/layer6_attention_weight_min": -62.15062713623047,
      "activations/layer7_attention_weight_max": 96.6224365234375,
      "activations/layer7_attention_weight_min": -87.1441879272461,
      "activations/layer8_attention_weight_max": 60.86345291137695,
      "activations/layer8_attention_weight_min": -57.17793273925781,
      "activations/layer9_attention_weight_max": 62.35002517700195,
      "activations/layer9_attention_weight_min": -55.68587112426758,
      "epoch": 9.74,
      "learning_rate": 8.806704545454544e-05,
      "loss": 2.7782,
      "step": 167650
    },
    {
      "activations/layer0_attention_weight_max": 14.498811721801758,
      "activations/layer0_attention_weight_min": -13.661117553710938,
      "activations/layer10_attention_weight_max": 36.90719223022461,
      "activations/layer10_attention_weight_min": -38.55861282348633,
      "activations/layer11_attention_weight_max": 33.361724853515625,
      "activations/layer11_attention_weight_min": -34.20787048339844,
      "activations/layer12_attention_weight_max": 18.36003303527832,
      "activations/layer12_attention_weight_min": -26.068836212158203,
      "activations/layer13_attention_weight_max": 43.27510452270508,
      "activations/layer13_attention_weight_min": -31.172096252441406,
      "activations/layer14_attention_weight_max": 44.38496398925781,
      "activations/layer14_attention_weight_min": -30.452993392944336,
      "activations/layer15_attention_weight_max": 36.89712905883789,
      "activations/layer15_attention_weight_min": -29.8128719329834,
      "activations/layer16_attention_weight_max": 34.35820007324219,
      "activations/layer16_attention_weight_min": -28.336055755615234,
      "activations/layer17_attention_weight_max": 53.9376106262207,
      "activations/layer17_attention_weight_min": -42.4621696472168,
      "activations/layer18_attention_weight_max": 47.10445785522461,
      "activations/layer18_attention_weight_min": -35.14950942993164,
      "activations/layer19_attention_weight_max": 21.419401168823242,
      "activations/layer19_attention_weight_min": -22.48838996887207,
      "activations/layer1_attention_weight_max": 17.1960506439209,
      "activations/layer1_attention_weight_min": -16.202939987182617,
      "activations/layer20_attention_weight_max": 24.8154239654541,
      "activations/layer20_attention_weight_min": -22.06896209716797,
      "activations/layer21_attention_weight_max": 43.095733642578125,
      "activations/layer21_attention_weight_min": -24.73744773864746,
      "activations/layer22_attention_weight_max": 33.5708122253418,
      "activations/layer22_attention_weight_min": -25.13593101501465,
      "activations/layer23_attention_weight_max": 39.47211456298828,
      "activations/layer23_attention_weight_min": -22.707807540893555,
      "activations/layer2_attention_weight_max": 32.70647048950195,
      "activations/layer2_attention_weight_min": -31.275333404541016,
      "activations/layer3_attention_weight_max": 91.26168060302734,
      "activations/layer3_attention_weight_min": -92.10852813720703,
      "activations/layer4_attention_weight_max": 105.58241271972656,
      "activations/layer4_attention_weight_min": -100.43855285644531,
      "activations/layer5_attention_weight_max": 80.67214965820312,
      "activations/layer5_attention_weight_min": -84.77532958984375,
      "activations/layer6_attention_weight_max": 57.99704360961914,
      "activations/layer6_attention_weight_min": -56.240074157714844,
      "activations/layer7_attention_weight_max": 83.02994537353516,
      "activations/layer7_attention_weight_min": -84.47721862792969,
      "activations/layer8_attention_weight_max": 47.939266204833984,
      "activations/layer8_attention_weight_min": -49.763427734375,
      "activations/layer9_attention_weight_max": 53.93983840942383,
      "activations/layer9_attention_weight_min": -51.329345703125,
      "epoch": 9.74,
      "learning_rate": 8.804810606060605e-05,
      "loss": 2.8115,
      "step": 167700
    },
    {
      "activations/layer0_attention_weight_max": 15.433597564697266,
      "activations/layer0_attention_weight_min": -13.776060104370117,
      "activations/layer10_attention_weight_max": 37.70355987548828,
      "activations/layer10_attention_weight_min": -35.146156311035156,
      "activations/layer11_attention_weight_max": 32.96656799316406,
      "activations/layer11_attention_weight_min": -32.80595397949219,
      "activations/layer12_attention_weight_max": 18.79416847229004,
      "activations/layer12_attention_weight_min": -25.11766815185547,
      "activations/layer13_attention_weight_max": 37.42564010620117,
      "activations/layer13_attention_weight_min": -30.428476333618164,
      "activations/layer14_attention_weight_max": 35.80241012573242,
      "activations/layer14_attention_weight_min": -35.105194091796875,
      "activations/layer15_attention_weight_max": 31.406076431274414,
      "activations/layer15_attention_weight_min": -31.24982261657715,
      "activations/layer16_attention_weight_max": 29.30962562561035,
      "activations/layer16_attention_weight_min": -29.37066650390625,
      "activations/layer17_attention_weight_max": 52.59598922729492,
      "activations/layer17_attention_weight_min": -45.02443313598633,
      "activations/layer18_attention_weight_max": 45.68306350708008,
      "activations/layer18_attention_weight_min": -40.955039978027344,
      "activations/layer19_attention_weight_max": 23.805219650268555,
      "activations/layer19_attention_weight_min": -25.19689178466797,
      "activations/layer1_attention_weight_max": 17.943971633911133,
      "activations/layer1_attention_weight_min": -15.487339973449707,
      "activations/layer20_attention_weight_max": 24.22068214416504,
      "activations/layer20_attention_weight_min": -23.521514892578125,
      "activations/layer21_attention_weight_max": 37.14763259887695,
      "activations/layer21_attention_weight_min": -23.43800926208496,
      "activations/layer22_attention_weight_max": 28.87236213684082,
      "activations/layer22_attention_weight_min": -25.020851135253906,
      "activations/layer23_attention_weight_max": 35.45994567871094,
      "activations/layer23_attention_weight_min": -24.800352096557617,
      "activations/layer2_attention_weight_max": 32.19850540161133,
      "activations/layer2_attention_weight_min": -30.441326141357422,
      "activations/layer3_attention_weight_max": 89.78951263427734,
      "activations/layer3_attention_weight_min": -91.32186889648438,
      "activations/layer4_attention_weight_max": 99.85331726074219,
      "activations/layer4_attention_weight_min": -99.20700073242188,
      "activations/layer5_attention_weight_max": 78.40350341796875,
      "activations/layer5_attention_weight_min": -79.05413818359375,
      "activations/layer6_attention_weight_max": 54.43149185180664,
      "activations/layer6_attention_weight_min": -55.498199462890625,
      "activations/layer7_attention_weight_max": 77.04085540771484,
      "activations/layer7_attention_weight_min": -75.34471893310547,
      "activations/layer8_attention_weight_max": 43.68632888793945,
      "activations/layer8_attention_weight_min": -47.034549713134766,
      "activations/layer9_attention_weight_max": 51.368953704833984,
      "activations/layer9_attention_weight_min": -47.61796188354492,
      "epoch": 9.75,
      "learning_rate": 8.802916666666665e-05,
      "loss": 2.7936,
      "step": 167750
    },
    {
      "activations/layer0_attention_weight_max": 15.50363826751709,
      "activations/layer0_attention_weight_min": -14.549513816833496,
      "activations/layer10_attention_weight_max": 39.54631423950195,
      "activations/layer10_attention_weight_min": -37.218475341796875,
      "activations/layer11_attention_weight_max": 36.10492706298828,
      "activations/layer11_attention_weight_min": -33.006473541259766,
      "activations/layer12_attention_weight_max": 19.025314331054688,
      "activations/layer12_attention_weight_min": -25.636903762817383,
      "activations/layer13_attention_weight_max": 45.90938186645508,
      "activations/layer13_attention_weight_min": -32.849430084228516,
      "activations/layer14_attention_weight_max": 42.78630065917969,
      "activations/layer14_attention_weight_min": -33.07708740234375,
      "activations/layer15_attention_weight_max": 33.90845489501953,
      "activations/layer15_attention_weight_min": -30.647031784057617,
      "activations/layer16_attention_weight_max": 29.58907699584961,
      "activations/layer16_attention_weight_min": -30.009389877319336,
      "activations/layer17_attention_weight_max": 52.09489822387695,
      "activations/layer17_attention_weight_min": -46.13835144042969,
      "activations/layer18_attention_weight_max": 48.505550384521484,
      "activations/layer18_attention_weight_min": -38.33407211303711,
      "activations/layer19_attention_weight_max": 23.004671096801758,
      "activations/layer19_attention_weight_min": -22.961292266845703,
      "activations/layer1_attention_weight_max": 16.609130859375,
      "activations/layer1_attention_weight_min": -15.708014488220215,
      "activations/layer20_attention_weight_max": 23.013669967651367,
      "activations/layer20_attention_weight_min": -22.410377502441406,
      "activations/layer21_attention_weight_max": 40.397865295410156,
      "activations/layer21_attention_weight_min": -23.249055862426758,
      "activations/layer22_attention_weight_max": 30.70890998840332,
      "activations/layer22_attention_weight_min": -25.4081974029541,
      "activations/layer23_attention_weight_max": 38.161903381347656,
      "activations/layer23_attention_weight_min": -22.6677303314209,
      "activations/layer2_attention_weight_max": 33.92548370361328,
      "activations/layer2_attention_weight_min": -32.487701416015625,
      "activations/layer3_attention_weight_max": 97.87335968017578,
      "activations/layer3_attention_weight_min": -93.0196304321289,
      "activations/layer4_attention_weight_max": 99.4471435546875,
      "activations/layer4_attention_weight_min": -95.90355682373047,
      "activations/layer5_attention_weight_max": 76.6227798461914,
      "activations/layer5_attention_weight_min": -75.01205444335938,
      "activations/layer6_attention_weight_max": 53.392723083496094,
      "activations/layer6_attention_weight_min": -56.59369659423828,
      "activations/layer7_attention_weight_max": 82.59877014160156,
      "activations/layer7_attention_weight_min": -71.4813003540039,
      "activations/layer8_attention_weight_max": 46.35975646972656,
      "activations/layer8_attention_weight_min": -46.01422119140625,
      "activations/layer9_attention_weight_max": 59.16118240356445,
      "activations/layer9_attention_weight_min": -49.81187438964844,
      "epoch": 9.75,
      "learning_rate": 8.801022727272727e-05,
      "loss": 2.793,
      "step": 167800
    },
    {
      "activations/layer0_attention_weight_max": 16.25909423828125,
      "activations/layer0_attention_weight_min": -14.133526802062988,
      "activations/layer10_attention_weight_max": 37.1118049621582,
      "activations/layer10_attention_weight_min": -34.69782257080078,
      "activations/layer11_attention_weight_max": 31.084333419799805,
      "activations/layer11_attention_weight_min": -35.01496124267578,
      "activations/layer12_attention_weight_max": 20.21426010131836,
      "activations/layer12_attention_weight_min": -25.97842788696289,
      "activations/layer13_attention_weight_max": 43.24077224731445,
      "activations/layer13_attention_weight_min": -30.865163803100586,
      "activations/layer14_attention_weight_max": 41.28742980957031,
      "activations/layer14_attention_weight_min": -29.010705947875977,
      "activations/layer15_attention_weight_max": 32.7851448059082,
      "activations/layer15_attention_weight_min": -28.04341697692871,
      "activations/layer16_attention_weight_max": 31.871854782104492,
      "activations/layer16_attention_weight_min": -29.06767463684082,
      "activations/layer17_attention_weight_max": 52.09800338745117,
      "activations/layer17_attention_weight_min": -44.38957595825195,
      "activations/layer18_attention_weight_max": 45.4111442565918,
      "activations/layer18_attention_weight_min": -38.916404724121094,
      "activations/layer19_attention_weight_max": 23.27752113342285,
      "activations/layer19_attention_weight_min": -23.734272003173828,
      "activations/layer1_attention_weight_max": 17.563922882080078,
      "activations/layer1_attention_weight_min": -14.641194343566895,
      "activations/layer20_attention_weight_max": 24.859268188476562,
      "activations/layer20_attention_weight_min": -23.03730010986328,
      "activations/layer21_attention_weight_max": 41.26347732543945,
      "activations/layer21_attention_weight_min": -23.27825355529785,
      "activations/layer22_attention_weight_max": 34.42061233520508,
      "activations/layer22_attention_weight_min": -25.757829666137695,
      "activations/layer23_attention_weight_max": 42.80776596069336,
      "activations/layer23_attention_weight_min": -24.128318786621094,
      "activations/layer2_attention_weight_max": 32.24205780029297,
      "activations/layer2_attention_weight_min": -31.541440963745117,
      "activations/layer3_attention_weight_max": 91.44168853759766,
      "activations/layer3_attention_weight_min": -96.0213394165039,
      "activations/layer4_attention_weight_max": 105.53958892822266,
      "activations/layer4_attention_weight_min": -101.91636657714844,
      "activations/layer5_attention_weight_max": 79.17649841308594,
      "activations/layer5_attention_weight_min": -82.96742248535156,
      "activations/layer6_attention_weight_max": 53.99832534790039,
      "activations/layer6_attention_weight_min": -56.203956604003906,
      "activations/layer7_attention_weight_max": 79.7872085571289,
      "activations/layer7_attention_weight_min": -75.50643920898438,
      "activations/layer8_attention_weight_max": 43.77262496948242,
      "activations/layer8_attention_weight_min": -46.3282470703125,
      "activations/layer9_attention_weight_max": 49.37018966674805,
      "activations/layer9_attention_weight_min": -48.81521987915039,
      "epoch": 9.75,
      "learning_rate": 8.799128787878787e-05,
      "loss": 2.7805,
      "step": 167850
    },
    {
      "activations/layer0_attention_weight_max": 16.384653091430664,
      "activations/layer0_attention_weight_min": -14.113607406616211,
      "activations/layer10_attention_weight_max": 46.3756217956543,
      "activations/layer10_attention_weight_min": -42.218936920166016,
      "activations/layer11_attention_weight_max": 41.54948425292969,
      "activations/layer11_attention_weight_min": -34.50533676147461,
      "activations/layer12_attention_weight_max": 21.695276260375977,
      "activations/layer12_attention_weight_min": -23.538175582885742,
      "activations/layer13_attention_weight_max": 49.73258590698242,
      "activations/layer13_attention_weight_min": -33.562496185302734,
      "activations/layer14_attention_weight_max": 65.9693832397461,
      "activations/layer14_attention_weight_min": -35.71424865722656,
      "activations/layer15_attention_weight_max": 55.03765869140625,
      "activations/layer15_attention_weight_min": -33.05689239501953,
      "activations/layer16_attention_weight_max": 35.48649215698242,
      "activations/layer16_attention_weight_min": -28.864086151123047,
      "activations/layer17_attention_weight_max": 56.81114196777344,
      "activations/layer17_attention_weight_min": -44.51395797729492,
      "activations/layer18_attention_weight_max": 50.12453079223633,
      "activations/layer18_attention_weight_min": -42.59611892700195,
      "activations/layer19_attention_weight_max": 23.20436668395996,
      "activations/layer19_attention_weight_min": -25.167287826538086,
      "activations/layer1_attention_weight_max": 16.54802894592285,
      "activations/layer1_attention_weight_min": -15.585384368896484,
      "activations/layer20_attention_weight_max": 27.29330062866211,
      "activations/layer20_attention_weight_min": -23.222511291503906,
      "activations/layer21_attention_weight_max": 44.46270751953125,
      "activations/layer21_attention_weight_min": -27.557851791381836,
      "activations/layer22_attention_weight_max": 34.52079772949219,
      "activations/layer22_attention_weight_min": -26.868316650390625,
      "activations/layer23_attention_weight_max": 40.826454162597656,
      "activations/layer23_attention_weight_min": -22.64812469482422,
      "activations/layer2_attention_weight_max": 37.411888122558594,
      "activations/layer2_attention_weight_min": -35.04437255859375,
      "activations/layer3_attention_weight_max": 93.39817810058594,
      "activations/layer3_attention_weight_min": -100.21334838867188,
      "activations/layer4_attention_weight_max": 99.56310272216797,
      "activations/layer4_attention_weight_min": -97.14392852783203,
      "activations/layer5_attention_weight_max": 77.62734985351562,
      "activations/layer5_attention_weight_min": -81.60697937011719,
      "activations/layer6_attention_weight_max": 55.91010284423828,
      "activations/layer6_attention_weight_min": -52.61692810058594,
      "activations/layer7_attention_weight_max": 89.4366455078125,
      "activations/layer7_attention_weight_min": -75.65579223632812,
      "activations/layer8_attention_weight_max": 55.739219665527344,
      "activations/layer8_attention_weight_min": -48.671142578125,
      "activations/layer9_attention_weight_max": 68.63655090332031,
      "activations/layer9_attention_weight_min": -51.48133087158203,
      "epoch": 9.76,
      "learning_rate": 8.797234848484847e-05,
      "loss": 2.7858,
      "step": 167900
    },
    {
      "activations/layer0_attention_weight_max": 15.3381986618042,
      "activations/layer0_attention_weight_min": -14.179437637329102,
      "activations/layer10_attention_weight_max": 38.98933792114258,
      "activations/layer10_attention_weight_min": -38.38188171386719,
      "activations/layer11_attention_weight_max": 33.90196228027344,
      "activations/layer11_attention_weight_min": -33.59532928466797,
      "activations/layer12_attention_weight_max": 24.408077239990234,
      "activations/layer12_attention_weight_min": -23.837173461914062,
      "activations/layer13_attention_weight_max": 41.18488693237305,
      "activations/layer13_attention_weight_min": -32.642513275146484,
      "activations/layer14_attention_weight_max": 38.4712028503418,
      "activations/layer14_attention_weight_min": -31.17923927307129,
      "activations/layer15_attention_weight_max": 37.575225830078125,
      "activations/layer15_attention_weight_min": -30.50634002685547,
      "activations/layer16_attention_weight_max": 32.280799865722656,
      "activations/layer16_attention_weight_min": -31.45635223388672,
      "activations/layer17_attention_weight_max": 59.9549560546875,
      "activations/layer17_attention_weight_min": -49.27762985229492,
      "activations/layer18_attention_weight_max": 51.203548431396484,
      "activations/layer18_attention_weight_min": -40.339073181152344,
      "activations/layer19_attention_weight_max": 23.7038631439209,
      "activations/layer19_attention_weight_min": -21.14497184753418,
      "activations/layer1_attention_weight_max": 17.348388671875,
      "activations/layer1_attention_weight_min": -13.536964416503906,
      "activations/layer20_attention_weight_max": 20.994840621948242,
      "activations/layer20_attention_weight_min": -20.094118118286133,
      "activations/layer21_attention_weight_max": 40.18287658691406,
      "activations/layer21_attention_weight_min": -24.868528366088867,
      "activations/layer22_attention_weight_max": 29.75484848022461,
      "activations/layer22_attention_weight_min": -25.412195205688477,
      "activations/layer23_attention_weight_max": 38.3382453918457,
      "activations/layer23_attention_weight_min": -23.77895736694336,
      "activations/layer2_attention_weight_max": 32.410850524902344,
      "activations/layer2_attention_weight_min": -32.22303771972656,
      "activations/layer3_attention_weight_max": 92.35574340820312,
      "activations/layer3_attention_weight_min": -100.51129150390625,
      "activations/layer4_attention_weight_max": 106.0112075805664,
      "activations/layer4_attention_weight_min": -100.16490936279297,
      "activations/layer5_attention_weight_max": 82.46150207519531,
      "activations/layer5_attention_weight_min": -79.89386749267578,
      "activations/layer6_attention_weight_max": 56.817779541015625,
      "activations/layer6_attention_weight_min": -55.409812927246094,
      "activations/layer7_attention_weight_max": 80.760986328125,
      "activations/layer7_attention_weight_min": -76.55841827392578,
      "activations/layer8_attention_weight_max": 45.2333984375,
      "activations/layer8_attention_weight_min": -48.037620544433594,
      "activations/layer9_attention_weight_max": 53.08732223510742,
      "activations/layer9_attention_weight_min": -50.72562789916992,
      "epoch": 9.76,
      "learning_rate": 8.795340909090909e-05,
      "loss": 2.7838,
      "step": 167950
    },
    {
      "activations/layer0_attention_weight_max": 16.23810386657715,
      "activations/layer0_attention_weight_min": -14.034263610839844,
      "activations/layer10_attention_weight_max": 34.44401931762695,
      "activations/layer10_attention_weight_min": -34.437355041503906,
      "activations/layer11_attention_weight_max": 30.148509979248047,
      "activations/layer11_attention_weight_min": -31.81551742553711,
      "activations/layer12_attention_weight_max": 17.26259994506836,
      "activations/layer12_attention_weight_min": -23.3071231842041,
      "activations/layer13_attention_weight_max": 40.57231140136719,
      "activations/layer13_attention_weight_min": -30.151737213134766,
      "activations/layer14_attention_weight_max": 42.25590896606445,
      "activations/layer14_attention_weight_min": -30.405517578125,
      "activations/layer15_attention_weight_max": 33.6965446472168,
      "activations/layer15_attention_weight_min": -31.0930118560791,
      "activations/layer16_attention_weight_max": 30.818538665771484,
      "activations/layer16_attention_weight_min": -31.046552658081055,
      "activations/layer17_attention_weight_max": 51.02969741821289,
      "activations/layer17_attention_weight_min": -43.81879425048828,
      "activations/layer18_attention_weight_max": 47.14955139160156,
      "activations/layer18_attention_weight_min": -37.10206604003906,
      "activations/layer19_attention_weight_max": 23.219003677368164,
      "activations/layer19_attention_weight_min": -23.935087203979492,
      "activations/layer1_attention_weight_max": 18.40555763244629,
      "activations/layer1_attention_weight_min": -15.585442543029785,
      "activations/layer20_attention_weight_max": 22.219236373901367,
      "activations/layer20_attention_weight_min": -22.011241912841797,
      "activations/layer21_attention_weight_max": 36.927162170410156,
      "activations/layer21_attention_weight_min": -22.041236877441406,
      "activations/layer22_attention_weight_max": 29.97861671447754,
      "activations/layer22_attention_weight_min": -26.966829299926758,
      "activations/layer23_attention_weight_max": 39.39569091796875,
      "activations/layer23_attention_weight_min": -22.53151512145996,
      "activations/layer2_attention_weight_max": 33.16304016113281,
      "activations/layer2_attention_weight_min": -32.76800537109375,
      "activations/layer3_attention_weight_max": 92.0549545288086,
      "activations/layer3_attention_weight_min": -102.10935974121094,
      "activations/layer4_attention_weight_max": 99.2728042602539,
      "activations/layer4_attention_weight_min": -96.1583480834961,
      "activations/layer5_attention_weight_max": 78.90971374511719,
      "activations/layer5_attention_weight_min": -79.4626235961914,
      "activations/layer6_attention_weight_max": 54.844764709472656,
      "activations/layer6_attention_weight_min": -52.22469711303711,
      "activations/layer7_attention_weight_max": 75.78410339355469,
      "activations/layer7_attention_weight_min": -72.40190887451172,
      "activations/layer8_attention_weight_max": 45.165802001953125,
      "activations/layer8_attention_weight_min": -46.205142974853516,
      "activations/layer9_attention_weight_max": 46.32816696166992,
      "activations/layer9_attention_weight_min": -47.52467346191406,
      "epoch": 9.76,
      "learning_rate": 8.793446969696969e-05,
      "loss": 2.7865,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.5274,
      "eval_samples_per_second": 503.555,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.5274,
      "eval_openwebtext_samples_per_second": 503.555,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 1.9737,
      "eval_wikitext_samples_per_second": 231.041,
      "step": 168000
    },
    {
      "epoch": 9.76,
      "eval_lambada_loss": 2.654296875,
      "eval_lambada_ppl": 14.214987631491761,
      "eval_lambada_runtime": 9.6166,
      "eval_lambada_samples_per_second": 506.311,
      "step": 168000
    },
    {
      "activations/layer0_attention_weight_max": 14.864392280578613,
      "activations/layer0_attention_weight_min": -14.39584732055664,
      "activations/layer10_attention_weight_max": 37.72800064086914,
      "activations/layer10_attention_weight_min": -35.054595947265625,
      "activations/layer11_attention_weight_max": 32.057708740234375,
      "activations/layer11_attention_weight_min": -33.494686126708984,
      "activations/layer12_attention_weight_max": 22.420499801635742,
      "activations/layer12_attention_weight_min": -26.198183059692383,
      "activations/layer13_attention_weight_max": 41.18373489379883,
      "activations/layer13_attention_weight_min": -28.70336151123047,
      "activations/layer14_attention_weight_max": 37.21005630493164,
      "activations/layer14_attention_weight_min": -28.772693634033203,
      "activations/layer15_attention_weight_max": 33.21928024291992,
      "activations/layer15_attention_weight_min": -28.375585556030273,
      "activations/layer16_attention_weight_max": 31.44484519958496,
      "activations/layer16_attention_weight_min": -27.241436004638672,
      "activations/layer17_attention_weight_max": 51.59339904785156,
      "activations/layer17_attention_weight_min": -41.366390228271484,
      "activations/layer18_attention_weight_max": 47.4670524597168,
      "activations/layer18_attention_weight_min": -34.62184143066406,
      "activations/layer19_attention_weight_max": 21.452329635620117,
      "activations/layer19_attention_weight_min": -22.028676986694336,
      "activations/layer1_attention_weight_max": 17.156089782714844,
      "activations/layer1_attention_weight_min": -14.958192825317383,
      "activations/layer20_attention_weight_max": 21.529735565185547,
      "activations/layer20_attention_weight_min": -21.503610610961914,
      "activations/layer21_attention_weight_max": 36.60279846191406,
      "activations/layer21_attention_weight_min": -21.238698959350586,
      "activations/layer22_attention_weight_max": 32.221824645996094,
      "activations/layer22_attention_weight_min": -24.08392906188965,
      "activations/layer23_attention_weight_max": 35.55439376831055,
      "activations/layer23_attention_weight_min": -21.92644500732422,
      "activations/layer2_attention_weight_max": 33.331626892089844,
      "activations/layer2_attention_weight_min": -30.534658432006836,
      "activations/layer3_attention_weight_max": 92.9297866821289,
      "activations/layer3_attention_weight_min": -95.5281982421875,
      "activations/layer4_attention_weight_max": 100.8787841796875,
      "activations/layer4_attention_weight_min": -98.08912658691406,
      "activations/layer5_attention_weight_max": 76.4548110961914,
      "activations/layer5_attention_weight_min": -77.1926040649414,
      "activations/layer6_attention_weight_max": 53.897605895996094,
      "activations/layer6_attention_weight_min": -53.378963470458984,
      "activations/layer7_attention_weight_max": 73.93270111083984,
      "activations/layer7_attention_weight_min": -72.97394561767578,
      "activations/layer8_attention_weight_max": 46.27210235595703,
      "activations/layer8_attention_weight_min": -46.31570053100586,
      "activations/layer9_attention_weight_max": 49.03144454956055,
      "activations/layer9_attention_weight_min": -48.02687072753906,
      "epoch": 9.76,
      "learning_rate": 8.791553030303029e-05,
      "loss": 2.811,
      "step": 168050
    },
    {
      "activations/layer0_attention_weight_max": 14.72020435333252,
      "activations/layer0_attention_weight_min": -14.180843353271484,
      "activations/layer10_attention_weight_max": 35.10490036010742,
      "activations/layer10_attention_weight_min": -36.50281524658203,
      "activations/layer11_attention_weight_max": 30.66060447692871,
      "activations/layer11_attention_weight_min": -35.16168212890625,
      "activations/layer12_attention_weight_max": 18.53445053100586,
      "activations/layer12_attention_weight_min": -27.733930587768555,
      "activations/layer13_attention_weight_max": 37.36125183105469,
      "activations/layer13_attention_weight_min": -31.979610443115234,
      "activations/layer14_attention_weight_max": 36.99095916748047,
      "activations/layer14_attention_weight_min": -30.700815200805664,
      "activations/layer15_attention_weight_max": 34.548614501953125,
      "activations/layer15_attention_weight_min": -28.555662155151367,
      "activations/layer16_attention_weight_max": 34.38242721557617,
      "activations/layer16_attention_weight_min": -29.734825134277344,
      "activations/layer17_attention_weight_max": 59.59145736694336,
      "activations/layer17_attention_weight_min": -47.37551498413086,
      "activations/layer18_attention_weight_max": 52.146461486816406,
      "activations/layer18_attention_weight_min": -38.359832763671875,
      "activations/layer19_attention_weight_max": 22.354202270507812,
      "activations/layer19_attention_weight_min": -21.32463836669922,
      "activations/layer1_attention_weight_max": 16.04570198059082,
      "activations/layer1_attention_weight_min": -13.312862396240234,
      "activations/layer20_attention_weight_max": 23.26120376586914,
      "activations/layer20_attention_weight_min": -20.697723388671875,
      "activations/layer21_attention_weight_max": 33.75799560546875,
      "activations/layer21_attention_weight_min": -21.35272216796875,
      "activations/layer22_attention_weight_max": 37.12273406982422,
      "activations/layer22_attention_weight_min": -24.97938346862793,
      "activations/layer23_attention_weight_max": 38.92841339111328,
      "activations/layer23_attention_weight_min": -23.251300811767578,
      "activations/layer2_attention_weight_max": 33.406471252441406,
      "activations/layer2_attention_weight_min": -31.435745239257812,
      "activations/layer3_attention_weight_max": 93.11167907714844,
      "activations/layer3_attention_weight_min": -95.50144958496094,
      "activations/layer4_attention_weight_max": 102.6396713256836,
      "activations/layer4_attention_weight_min": -97.80248260498047,
      "activations/layer5_attention_weight_max": 78.92017364501953,
      "activations/layer5_attention_weight_min": -84.39009094238281,
      "activations/layer6_attention_weight_max": 56.592376708984375,
      "activations/layer6_attention_weight_min": -55.41462326049805,
      "activations/layer7_attention_weight_max": 86.79401397705078,
      "activations/layer7_attention_weight_min": -77.96731567382812,
      "activations/layer8_attention_weight_max": 47.63466262817383,
      "activations/layer8_attention_weight_min": -47.11702346801758,
      "activations/layer9_attention_weight_max": 48.88405227661133,
      "activations/layer9_attention_weight_min": -52.051082611083984,
      "epoch": 9.77,
      "learning_rate": 8.78965909090909e-05,
      "loss": 2.7874,
      "step": 168100
    },
    {
      "activations/layer0_attention_weight_max": 14.636187553405762,
      "activations/layer0_attention_weight_min": -14.41946029663086,
      "activations/layer10_attention_weight_max": 39.122413635253906,
      "activations/layer10_attention_weight_min": -40.92819595336914,
      "activations/layer11_attention_weight_max": 32.544456481933594,
      "activations/layer11_attention_weight_min": -32.99433135986328,
      "activations/layer12_attention_weight_max": 18.995807647705078,
      "activations/layer12_attention_weight_min": -27.92223358154297,
      "activations/layer13_attention_weight_max": 46.3338737487793,
      "activations/layer13_attention_weight_min": -33.558311462402344,
      "activations/layer14_attention_weight_max": 44.63669967651367,
      "activations/layer14_attention_weight_min": -32.23655319213867,
      "activations/layer15_attention_weight_max": 38.77528381347656,
      "activations/layer15_attention_weight_min": -28.65205955505371,
      "activations/layer16_attention_weight_max": 34.81283187866211,
      "activations/layer16_attention_weight_min": -29.448362350463867,
      "activations/layer17_attention_weight_max": 56.35633087158203,
      "activations/layer17_attention_weight_min": -44.16997146606445,
      "activations/layer18_attention_weight_max": 49.35782241821289,
      "activations/layer18_attention_weight_min": -37.34527587890625,
      "activations/layer19_attention_weight_max": 23.006193161010742,
      "activations/layer19_attention_weight_min": -23.696765899658203,
      "activations/layer1_attention_weight_max": 17.019786834716797,
      "activations/layer1_attention_weight_min": -14.331513404846191,
      "activations/layer20_attention_weight_max": 24.11212730407715,
      "activations/layer20_attention_weight_min": -26.569225311279297,
      "activations/layer21_attention_weight_max": 35.27166748046875,
      "activations/layer21_attention_weight_min": -22.43695068359375,
      "activations/layer22_attention_weight_max": 34.02553939819336,
      "activations/layer22_attention_weight_min": -26.96415138244629,
      "activations/layer23_attention_weight_max": 37.696205139160156,
      "activations/layer23_attention_weight_min": -23.380521774291992,
      "activations/layer2_attention_weight_max": 34.15045166015625,
      "activations/layer2_attention_weight_min": -32.26820373535156,
      "activations/layer3_attention_weight_max": 91.89091491699219,
      "activations/layer3_attention_weight_min": -95.75830841064453,
      "activations/layer4_attention_weight_max": 103.23284912109375,
      "activations/layer4_attention_weight_min": -100.1918716430664,
      "activations/layer5_attention_weight_max": 77.9908447265625,
      "activations/layer5_attention_weight_min": -76.37019348144531,
      "activations/layer6_attention_weight_max": 56.72930145263672,
      "activations/layer6_attention_weight_min": -56.21117401123047,
      "activations/layer7_attention_weight_max": 76.09557342529297,
      "activations/layer7_attention_weight_min": -79.58707427978516,
      "activations/layer8_attention_weight_max": 47.80873489379883,
      "activations/layer8_attention_weight_min": -49.35073471069336,
      "activations/layer9_attention_weight_max": 50.173824310302734,
      "activations/layer9_attention_weight_min": -50.40400314331055,
      "epoch": 9.77,
      "learning_rate": 8.787765151515152e-05,
      "loss": 2.7767,
      "step": 168150
    },
    {
      "activations/layer0_attention_weight_max": 15.305886268615723,
      "activations/layer0_attention_weight_min": -14.18812370300293,
      "activations/layer10_attention_weight_max": 39.96388244628906,
      "activations/layer10_attention_weight_min": -38.4091682434082,
      "activations/layer11_attention_weight_max": 37.340824127197266,
      "activations/layer11_attention_weight_min": -32.25835418701172,
      "activations/layer12_attention_weight_max": 19.80182647705078,
      "activations/layer12_attention_weight_min": -28.897825241088867,
      "activations/layer13_attention_weight_max": 51.09629821777344,
      "activations/layer13_attention_weight_min": -34.91130065917969,
      "activations/layer14_attention_weight_max": 44.27538299560547,
      "activations/layer14_attention_weight_min": -33.15047073364258,
      "activations/layer15_attention_weight_max": 39.442806243896484,
      "activations/layer15_attention_weight_min": -32.419490814208984,
      "activations/layer16_attention_weight_max": 32.36607360839844,
      "activations/layer16_attention_weight_min": -32.729087829589844,
      "activations/layer17_attention_weight_max": 54.32040786743164,
      "activations/layer17_attention_weight_min": -51.36967086791992,
      "activations/layer18_attention_weight_max": 47.623321533203125,
      "activations/layer18_attention_weight_min": -41.82758712768555,
      "activations/layer19_attention_weight_max": 20.722393035888672,
      "activations/layer19_attention_weight_min": -23.62156867980957,
      "activations/layer1_attention_weight_max": 17.476516723632812,
      "activations/layer1_attention_weight_min": -14.476242065429688,
      "activations/layer20_attention_weight_max": 21.84351348876953,
      "activations/layer20_attention_weight_min": -21.543317794799805,
      "activations/layer21_attention_weight_max": 50.48234176635742,
      "activations/layer21_attention_weight_min": -28.34374237060547,
      "activations/layer22_attention_weight_max": 31.02366828918457,
      "activations/layer22_attention_weight_min": -26.45853614807129,
      "activations/layer23_attention_weight_max": 42.05217361450195,
      "activations/layer23_attention_weight_min": -23.695287704467773,
      "activations/layer2_attention_weight_max": 32.67710494995117,
      "activations/layer2_attention_weight_min": -31.574237823486328,
      "activations/layer3_attention_weight_max": 94.08525085449219,
      "activations/layer3_attention_weight_min": -93.4010238647461,
      "activations/layer4_attention_weight_max": 101.0262451171875,
      "activations/layer4_attention_weight_min": -95.4257583618164,
      "activations/layer5_attention_weight_max": 80.89231872558594,
      "activations/layer5_attention_weight_min": -81.97047424316406,
      "activations/layer6_attention_weight_max": 57.14012145996094,
      "activations/layer6_attention_weight_min": -53.45079040527344,
      "activations/layer7_attention_weight_max": 86.18148040771484,
      "activations/layer7_attention_weight_min": -73.49805450439453,
      "activations/layer8_attention_weight_max": 52.852691650390625,
      "activations/layer8_attention_weight_min": -45.51836013793945,
      "activations/layer9_attention_weight_max": 65.05184936523438,
      "activations/layer9_attention_weight_min": -49.76704788208008,
      "epoch": 9.77,
      "learning_rate": 8.785871212121211e-05,
      "loss": 2.7758,
      "step": 168200
    },
    {
      "activations/layer0_attention_weight_max": 14.880373001098633,
      "activations/layer0_attention_weight_min": -14.10007381439209,
      "activations/layer10_attention_weight_max": 44.97309112548828,
      "activations/layer10_attention_weight_min": -40.75192642211914,
      "activations/layer11_attention_weight_max": 39.28985595703125,
      "activations/layer11_attention_weight_min": -34.82350158691406,
      "activations/layer12_attention_weight_max": 38.03953170776367,
      "activations/layer12_attention_weight_min": -27.867996215820312,
      "activations/layer13_attention_weight_max": 45.77727508544922,
      "activations/layer13_attention_weight_min": -33.291015625,
      "activations/layer14_attention_weight_max": 37.136329650878906,
      "activations/layer14_attention_weight_min": -30.738237380981445,
      "activations/layer15_attention_weight_max": 35.25590133666992,
      "activations/layer15_attention_weight_min": -30.007076263427734,
      "activations/layer16_attention_weight_max": 31.36578369140625,
      "activations/layer16_attention_weight_min": -28.697587966918945,
      "activations/layer17_attention_weight_max": 58.69943618774414,
      "activations/layer17_attention_weight_min": -45.45417022705078,
      "activations/layer18_attention_weight_max": 45.13614273071289,
      "activations/layer18_attention_weight_min": -39.893394470214844,
      "activations/layer19_attention_weight_max": 27.2148380279541,
      "activations/layer19_attention_weight_min": -22.62464141845703,
      "activations/layer1_attention_weight_max": 17.40142059326172,
      "activations/layer1_attention_weight_min": -16.436399459838867,
      "activations/layer20_attention_weight_max": 22.721912384033203,
      "activations/layer20_attention_weight_min": -23.66270637512207,
      "activations/layer21_attention_weight_max": 44.31331253051758,
      "activations/layer21_attention_weight_min": -25.94379234313965,
      "activations/layer22_attention_weight_max": 28.803810119628906,
      "activations/layer22_attention_weight_min": -26.36040687561035,
      "activations/layer23_attention_weight_max": 37.62550354003906,
      "activations/layer23_attention_weight_min": -22.66097068786621,
      "activations/layer2_attention_weight_max": 35.507110595703125,
      "activations/layer2_attention_weight_min": -33.5885124206543,
      "activations/layer3_attention_weight_max": 96.99861907958984,
      "activations/layer3_attention_weight_min": -100.15522003173828,
      "activations/layer4_attention_weight_max": 104.11412811279297,
      "activations/layer4_attention_weight_min": -100.86864471435547,
      "activations/layer5_attention_weight_max": 81.01261901855469,
      "activations/layer5_attention_weight_min": -79.5229721069336,
      "activations/layer6_attention_weight_max": 58.969242095947266,
      "activations/layer6_attention_weight_min": -55.96685791015625,
      "activations/layer7_attention_weight_max": 81.0504379272461,
      "activations/layer7_attention_weight_min": -78.33897399902344,
      "activations/layer8_attention_weight_max": 49.71367645263672,
      "activations/layer8_attention_weight_min": -49.38872146606445,
      "activations/layer9_attention_weight_max": 59.00049591064453,
      "activations/layer9_attention_weight_min": -51.15208435058594,
      "epoch": 9.78,
      "learning_rate": 8.783977272727272e-05,
      "loss": 2.7973,
      "step": 168250
    },
    {
      "activations/layer0_attention_weight_max": 15.307646751403809,
      "activations/layer0_attention_weight_min": -14.041936874389648,
      "activations/layer10_attention_weight_max": 44.16372299194336,
      "activations/layer10_attention_weight_min": -39.05645751953125,
      "activations/layer11_attention_weight_max": 38.63350296020508,
      "activations/layer11_attention_weight_min": -32.926692962646484,
      "activations/layer12_attention_weight_max": 23.55780792236328,
      "activations/layer12_attention_weight_min": -25.6862850189209,
      "activations/layer13_attention_weight_max": 63.44895553588867,
      "activations/layer13_attention_weight_min": -32.80448913574219,
      "activations/layer14_attention_weight_max": 72.59234619140625,
      "activations/layer14_attention_weight_min": -34.837242126464844,
      "activations/layer15_attention_weight_max": 57.27757263183594,
      "activations/layer15_attention_weight_min": -31.306983947753906,
      "activations/layer16_attention_weight_max": 37.46601486206055,
      "activations/layer16_attention_weight_min": -30.049760818481445,
      "activations/layer17_attention_weight_max": 52.85586166381836,
      "activations/layer17_attention_weight_min": -47.31458282470703,
      "activations/layer18_attention_weight_max": 49.44901657104492,
      "activations/layer18_attention_weight_min": -39.816959381103516,
      "activations/layer19_attention_weight_max": 24.251638412475586,
      "activations/layer19_attention_weight_min": -23.184837341308594,
      "activations/layer1_attention_weight_max": 17.037588119506836,
      "activations/layer1_attention_weight_min": -14.852384567260742,
      "activations/layer20_attention_weight_max": 25.864011764526367,
      "activations/layer20_attention_weight_min": -22.114770889282227,
      "activations/layer21_attention_weight_max": 47.029930114746094,
      "activations/layer21_attention_weight_min": -22.55137062072754,
      "activations/layer22_attention_weight_max": 32.04719161987305,
      "activations/layer22_attention_weight_min": -24.265380859375,
      "activations/layer23_attention_weight_max": 38.380035400390625,
      "activations/layer23_attention_weight_min": -23.113540649414062,
      "activations/layer2_attention_weight_max": 32.97224426269531,
      "activations/layer2_attention_weight_min": -30.456024169921875,
      "activations/layer3_attention_weight_max": 96.12037658691406,
      "activations/layer3_attention_weight_min": -95.62055969238281,
      "activations/layer4_attention_weight_max": 98.82488250732422,
      "activations/layer4_attention_weight_min": -93.22105407714844,
      "activations/layer5_attention_weight_max": 76.0908203125,
      "activations/layer5_attention_weight_min": -81.82000732421875,
      "activations/layer6_attention_weight_max": 52.977176666259766,
      "activations/layer6_attention_weight_min": -53.066097259521484,
      "activations/layer7_attention_weight_max": 82.6698226928711,
      "activations/layer7_attention_weight_min": -70.3571548461914,
      "activations/layer8_attention_weight_max": 51.64079666137695,
      "activations/layer8_attention_weight_min": -45.01642608642578,
      "activations/layer9_attention_weight_max": 63.6174430847168,
      "activations/layer9_attention_weight_min": -46.55873107910156,
      "epoch": 9.78,
      "learning_rate": 8.782083333333332e-05,
      "loss": 2.8022,
      "step": 168300
    },
    {
      "activations/layer0_attention_weight_max": 15.441588401794434,
      "activations/layer0_attention_weight_min": -14.113353729248047,
      "activations/layer10_attention_weight_max": 39.185733795166016,
      "activations/layer10_attention_weight_min": -36.528324127197266,
      "activations/layer11_attention_weight_max": 36.233924865722656,
      "activations/layer11_attention_weight_min": -33.57880783081055,
      "activations/layer12_attention_weight_max": 21.593095779418945,
      "activations/layer12_attention_weight_min": -26.518888473510742,
      "activations/layer13_attention_weight_max": 42.69126510620117,
      "activations/layer13_attention_weight_min": -31.083436965942383,
      "activations/layer14_attention_weight_max": 42.65972900390625,
      "activations/layer14_attention_weight_min": -31.309019088745117,
      "activations/layer15_attention_weight_max": 37.7263069152832,
      "activations/layer15_attention_weight_min": -30.614194869995117,
      "activations/layer16_attention_weight_max": 32.2515754699707,
      "activations/layer16_attention_weight_min": -31.826534271240234,
      "activations/layer17_attention_weight_max": 53.47377395629883,
      "activations/layer17_attention_weight_min": -45.45619583129883,
      "activations/layer18_attention_weight_max": 48.48744583129883,
      "activations/layer18_attention_weight_min": -38.54545211791992,
      "activations/layer19_attention_weight_max": 23.81610679626465,
      "activations/layer19_attention_weight_min": -21.352380752563477,
      "activations/layer1_attention_weight_max": 18.44344139099121,
      "activations/layer1_attention_weight_min": -14.671036720275879,
      "activations/layer20_attention_weight_max": 25.847867965698242,
      "activations/layer20_attention_weight_min": -22.47665786743164,
      "activations/layer21_attention_weight_max": 42.871883392333984,
      "activations/layer21_attention_weight_min": -24.96348762512207,
      "activations/layer22_attention_weight_max": 30.72910499572754,
      "activations/layer22_attention_weight_min": -25.443092346191406,
      "activations/layer23_attention_weight_max": 41.124568939208984,
      "activations/layer23_attention_weight_min": -23.379108428955078,
      "activations/layer2_attention_weight_max": 33.555294036865234,
      "activations/layer2_attention_weight_min": -32.70185852050781,
      "activations/layer3_attention_weight_max": 94.66254425048828,
      "activations/layer3_attention_weight_min": -97.71388244628906,
      "activations/layer4_attention_weight_max": 103.8117904663086,
      "activations/layer4_attention_weight_min": -102.8714828491211,
      "activations/layer5_attention_weight_max": 79.25675964355469,
      "activations/layer5_attention_weight_min": -84.6885757446289,
      "activations/layer6_attention_weight_max": 54.71900939941406,
      "activations/layer6_attention_weight_min": -55.5755729675293,
      "activations/layer7_attention_weight_max": 78.35514831542969,
      "activations/layer7_attention_weight_min": -73.36068725585938,
      "activations/layer8_attention_weight_max": 45.18254470825195,
      "activations/layer8_attention_weight_min": -46.72882843017578,
      "activations/layer9_attention_weight_max": 51.919429779052734,
      "activations/layer9_attention_weight_min": -49.74128723144531,
      "epoch": 9.78,
      "learning_rate": 8.780189393939393e-05,
      "loss": 2.7909,
      "step": 168350
    },
    {
      "activations/layer0_attention_weight_max": 15.03520679473877,
      "activations/layer0_attention_weight_min": -14.051229476928711,
      "activations/layer10_attention_weight_max": 34.70960998535156,
      "activations/layer10_attention_weight_min": -35.11592102050781,
      "activations/layer11_attention_weight_max": 29.662612915039062,
      "activations/layer11_attention_weight_min": -31.040355682373047,
      "activations/layer12_attention_weight_max": 19.48137664794922,
      "activations/layer12_attention_weight_min": -24.820871353149414,
      "activations/layer13_attention_weight_max": 42.09160232543945,
      "activations/layer13_attention_weight_min": -32.46217346191406,
      "activations/layer14_attention_weight_max": 39.664615631103516,
      "activations/layer14_attention_weight_min": -30.817432403564453,
      "activations/layer15_attention_weight_max": 35.85594940185547,
      "activations/layer15_attention_weight_min": -28.58479881286621,
      "activations/layer16_attention_weight_max": 32.99563980102539,
      "activations/layer16_attention_weight_min": -30.99965476989746,
      "activations/layer17_attention_weight_max": 51.86071014404297,
      "activations/layer17_attention_weight_min": -43.04112243652344,
      "activations/layer18_attention_weight_max": 44.40593338012695,
      "activations/layer18_attention_weight_min": -35.19904327392578,
      "activations/layer19_attention_weight_max": 21.463655471801758,
      "activations/layer19_attention_weight_min": -21.131025314331055,
      "activations/layer1_attention_weight_max": 16.886730194091797,
      "activations/layer1_attention_weight_min": -13.964226722717285,
      "activations/layer20_attention_weight_max": 23.107627868652344,
      "activations/layer20_attention_weight_min": -22.521373748779297,
      "activations/layer21_attention_weight_max": 36.86365509033203,
      "activations/layer21_attention_weight_min": -23.18301010131836,
      "activations/layer22_attention_weight_max": 32.75703811645508,
      "activations/layer22_attention_weight_min": -25.00726318359375,
      "activations/layer23_attention_weight_max": 37.114410400390625,
      "activations/layer23_attention_weight_min": -23.40936279296875,
      "activations/layer2_attention_weight_max": 32.79751968383789,
      "activations/layer2_attention_weight_min": -31.502864837646484,
      "activations/layer3_attention_weight_max": 91.21121978759766,
      "activations/layer3_attention_weight_min": -98.97588348388672,
      "activations/layer4_attention_weight_max": 99.00064086914062,
      "activations/layer4_attention_weight_min": -94.18746185302734,
      "activations/layer5_attention_weight_max": 77.49784851074219,
      "activations/layer5_attention_weight_min": -81.54055786132812,
      "activations/layer6_attention_weight_max": 54.55827713012695,
      "activations/layer6_attention_weight_min": -52.83866882324219,
      "activations/layer7_attention_weight_max": 80.17301177978516,
      "activations/layer7_attention_weight_min": -76.15155029296875,
      "activations/layer8_attention_weight_max": 44.6075325012207,
      "activations/layer8_attention_weight_min": -44.850440979003906,
      "activations/layer9_attention_weight_max": 47.99563980102539,
      "activations/layer9_attention_weight_min": -54.62215805053711,
      "epoch": 9.78,
      "learning_rate": 8.778295454545454e-05,
      "loss": 2.79,
      "step": 168400
    },
    {
      "activations/layer0_attention_weight_max": 18.27626609802246,
      "activations/layer0_attention_weight_min": -14.214913368225098,
      "activations/layer10_attention_weight_max": 52.707672119140625,
      "activations/layer10_attention_weight_min": -49.35938262939453,
      "activations/layer11_attention_weight_max": 46.614410400390625,
      "activations/layer11_attention_weight_min": -42.1083869934082,
      "activations/layer12_attention_weight_max": 21.080535888671875,
      "activations/layer12_attention_weight_min": -27.52094268798828,
      "activations/layer13_attention_weight_max": 79.91087341308594,
      "activations/layer13_attention_weight_min": -42.46229553222656,
      "activations/layer14_attention_weight_max": 92.84809875488281,
      "activations/layer14_attention_weight_min": -42.52517318725586,
      "activations/layer15_attention_weight_max": 66.25873565673828,
      "activations/layer15_attention_weight_min": -32.778648376464844,
      "activations/layer16_attention_weight_max": 35.208683013916016,
      "activations/layer16_attention_weight_min": -32.048583984375,
      "activations/layer17_attention_weight_max": 60.64608383178711,
      "activations/layer17_attention_weight_min": -50.89754867553711,
      "activations/layer18_attention_weight_max": 49.24720001220703,
      "activations/layer18_attention_weight_min": -40.83005142211914,
      "activations/layer19_attention_weight_max": 23.546743392944336,
      "activations/layer19_attention_weight_min": -22.45684051513672,
      "activations/layer1_attention_weight_max": 16.868684768676758,
      "activations/layer1_attention_weight_min": -15.882107734680176,
      "activations/layer20_attention_weight_max": 24.686567306518555,
      "activations/layer20_attention_weight_min": -25.488203048706055,
      "activations/layer21_attention_weight_max": 44.77155685424805,
      "activations/layer21_attention_weight_min": -21.50694465637207,
      "activations/layer22_attention_weight_max": 30.212799072265625,
      "activations/layer22_attention_weight_min": -24.124223709106445,
      "activations/layer23_attention_weight_max": 35.42164611816406,
      "activations/layer23_attention_weight_min": -25.492033004760742,
      "activations/layer2_attention_weight_max": 35.057090759277344,
      "activations/layer2_attention_weight_min": -33.220855712890625,
      "activations/layer3_attention_weight_max": 102.20491027832031,
      "activations/layer3_attention_weight_min": -103.53146362304688,
      "activations/layer4_attention_weight_max": 109.14739990234375,
      "activations/layer4_attention_weight_min": -102.52928924560547,
      "activations/layer5_attention_weight_max": 82.3408203125,
      "activations/layer5_attention_weight_min": -79.10369873046875,
      "activations/layer6_attention_weight_max": 62.26111602783203,
      "activations/layer6_attention_weight_min": -60.175331115722656,
      "activations/layer7_attention_weight_max": 93.15319061279297,
      "activations/layer7_attention_weight_min": -88.61305236816406,
      "activations/layer8_attention_weight_max": 60.24031066894531,
      "activations/layer8_attention_weight_min": -58.67546844482422,
      "activations/layer9_attention_weight_max": 71.21376037597656,
      "activations/layer9_attention_weight_min": -66.26493072509766,
      "epoch": 9.79,
      "learning_rate": 8.776401515151514e-05,
      "loss": 2.7968,
      "step": 168450
    },
    {
      "activations/layer0_attention_weight_max": 14.838669776916504,
      "activations/layer0_attention_weight_min": -14.429113388061523,
      "activations/layer10_attention_weight_max": 36.117584228515625,
      "activations/layer10_attention_weight_min": -34.89851379394531,
      "activations/layer11_attention_weight_max": 32.84613037109375,
      "activations/layer11_attention_weight_min": -31.92909049987793,
      "activations/layer12_attention_weight_max": 21.69249725341797,
      "activations/layer12_attention_weight_min": -27.979398727416992,
      "activations/layer13_attention_weight_max": 50.722896575927734,
      "activations/layer13_attention_weight_min": -36.97735595703125,
      "activations/layer14_attention_weight_max": 49.29117965698242,
      "activations/layer14_attention_weight_min": -33.71315002441406,
      "activations/layer15_attention_weight_max": 41.71003341674805,
      "activations/layer15_attention_weight_min": -30.2499942779541,
      "activations/layer16_attention_weight_max": 32.5751838684082,
      "activations/layer16_attention_weight_min": -30.80940818786621,
      "activations/layer17_attention_weight_max": 56.95073699951172,
      "activations/layer17_attention_weight_min": -45.168540954589844,
      "activations/layer18_attention_weight_max": 46.33815002441406,
      "activations/layer18_attention_weight_min": -38.99740982055664,
      "activations/layer19_attention_weight_max": 24.82512855529785,
      "activations/layer19_attention_weight_min": -23.234643936157227,
      "activations/layer1_attention_weight_max": 16.865915298461914,
      "activations/layer1_attention_weight_min": -14.240301132202148,
      "activations/layer20_attention_weight_max": 23.560710906982422,
      "activations/layer20_attention_weight_min": -21.815006256103516,
      "activations/layer21_attention_weight_max": 40.551971435546875,
      "activations/layer21_attention_weight_min": -23.378698348999023,
      "activations/layer22_attention_weight_max": 29.66260528564453,
      "activations/layer22_attention_weight_min": -25.98207664489746,
      "activations/layer23_attention_weight_max": 37.691368103027344,
      "activations/layer23_attention_weight_min": -23.686721801757812,
      "activations/layer2_attention_weight_max": 35.325225830078125,
      "activations/layer2_attention_weight_min": -32.389156341552734,
      "activations/layer3_attention_weight_max": 99.06949615478516,
      "activations/layer3_attention_weight_min": -95.38236999511719,
      "activations/layer4_attention_weight_max": 99.67288208007812,
      "activations/layer4_attention_weight_min": -102.24637603759766,
      "activations/layer5_attention_weight_max": 78.39110565185547,
      "activations/layer5_attention_weight_min": -80.26908874511719,
      "activations/layer6_attention_weight_max": 55.77898406982422,
      "activations/layer6_attention_weight_min": -53.57011032104492,
      "activations/layer7_attention_weight_max": 74.19277954101562,
      "activations/layer7_attention_weight_min": -74.49116516113281,
      "activations/layer8_attention_weight_max": 44.91438674926758,
      "activations/layer8_attention_weight_min": -45.80488967895508,
      "activations/layer9_attention_weight_max": 54.03989791870117,
      "activations/layer9_attention_weight_min": -48.85993576049805,
      "epoch": 9.79,
      "learning_rate": 8.774507575757574e-05,
      "loss": 2.7947,
      "step": 168500
    },
    {
      "activations/layer0_attention_weight_max": 15.506250381469727,
      "activations/layer0_attention_weight_min": -14.353327751159668,
      "activations/layer10_attention_weight_max": 42.240455627441406,
      "activations/layer10_attention_weight_min": -37.70586395263672,
      "activations/layer11_attention_weight_max": 35.604225158691406,
      "activations/layer11_attention_weight_min": -33.484031677246094,
      "activations/layer12_attention_weight_max": 19.28807830810547,
      "activations/layer12_attention_weight_min": -28.797931671142578,
      "activations/layer13_attention_weight_max": 50.97921371459961,
      "activations/layer13_attention_weight_min": -37.01462936401367,
      "activations/layer14_attention_weight_max": 52.520660400390625,
      "activations/layer14_attention_weight_min": -35.47122573852539,
      "activations/layer15_attention_weight_max": 48.68562316894531,
      "activations/layer15_attention_weight_min": -32.4495964050293,
      "activations/layer16_attention_weight_max": 31.169771194458008,
      "activations/layer16_attention_weight_min": -30.32970428466797,
      "activations/layer17_attention_weight_max": 57.652099609375,
      "activations/layer17_attention_weight_min": -48.81640625,
      "activations/layer18_attention_weight_max": 49.2460823059082,
      "activations/layer18_attention_weight_min": -38.87922286987305,
      "activations/layer19_attention_weight_max": 22.939973831176758,
      "activations/layer19_attention_weight_min": -21.988771438598633,
      "activations/layer1_attention_weight_max": 16.148197174072266,
      "activations/layer1_attention_weight_min": -14.773112297058105,
      "activations/layer20_attention_weight_max": 22.28191375732422,
      "activations/layer20_attention_weight_min": -23.06572723388672,
      "activations/layer21_attention_weight_max": 44.428653717041016,
      "activations/layer21_attention_weight_min": -23.50446128845215,
      "activations/layer22_attention_weight_max": 31.93915557861328,
      "activations/layer22_attention_weight_min": -26.244266510009766,
      "activations/layer23_attention_weight_max": 36.059837341308594,
      "activations/layer23_attention_weight_min": -24.470291137695312,
      "activations/layer2_attention_weight_max": 32.98712158203125,
      "activations/layer2_attention_weight_min": -30.8675594329834,
      "activations/layer3_attention_weight_max": 96.12837982177734,
      "activations/layer3_attention_weight_min": -92.3644790649414,
      "activations/layer4_attention_weight_max": 104.47985076904297,
      "activations/layer4_attention_weight_min": -99.90779876708984,
      "activations/layer5_attention_weight_max": 78.25032043457031,
      "activations/layer5_attention_weight_min": -77.68067932128906,
      "activations/layer6_attention_weight_max": 57.21809387207031,
      "activations/layer6_attention_weight_min": -54.534297943115234,
      "activations/layer7_attention_weight_max": 84.21733856201172,
      "activations/layer7_attention_weight_min": -79.21990966796875,
      "activations/layer8_attention_weight_max": 51.5662956237793,
      "activations/layer8_attention_weight_min": -54.20894241333008,
      "activations/layer9_attention_weight_max": 52.563270568847656,
      "activations/layer9_attention_weight_min": -51.818572998046875,
      "epoch": 9.79,
      "learning_rate": 8.772613636363636e-05,
      "loss": 2.7812,
      "step": 168550
    },
    {
      "activations/layer0_attention_weight_max": 15.524090766906738,
      "activations/layer0_attention_weight_min": -14.496630668640137,
      "activations/layer10_attention_weight_max": 38.17414093017578,
      "activations/layer10_attention_weight_min": -39.480018615722656,
      "activations/layer11_attention_weight_max": 31.13289451599121,
      "activations/layer11_attention_weight_min": -32.47843933105469,
      "activations/layer12_attention_weight_max": 18.572425842285156,
      "activations/layer12_attention_weight_min": -26.48353385925293,
      "activations/layer13_attention_weight_max": 49.80105209350586,
      "activations/layer13_attention_weight_min": -33.01814270019531,
      "activations/layer14_attention_weight_max": 58.16600799560547,
      "activations/layer14_attention_weight_min": -32.118350982666016,
      "activations/layer15_attention_weight_max": 42.90394973754883,
      "activations/layer15_attention_weight_min": -29.807832717895508,
      "activations/layer16_attention_weight_max": 32.312992095947266,
      "activations/layer16_attention_weight_min": -30.346036911010742,
      "activations/layer17_attention_weight_max": 56.61021041870117,
      "activations/layer17_attention_weight_min": -45.41748046875,
      "activations/layer18_attention_weight_max": 46.34105682373047,
      "activations/layer18_attention_weight_min": -39.09782791137695,
      "activations/layer19_attention_weight_max": 26.1538143157959,
      "activations/layer19_attention_weight_min": -23.495166778564453,
      "activations/layer1_attention_weight_max": 17.499406814575195,
      "activations/layer1_attention_weight_min": -14.960250854492188,
      "activations/layer20_attention_weight_max": 22.46805763244629,
      "activations/layer20_attention_weight_min": -22.299535751342773,
      "activations/layer21_attention_weight_max": 44.93144989013672,
      "activations/layer21_attention_weight_min": -26.58591651916504,
      "activations/layer22_attention_weight_max": 37.63043975830078,
      "activations/layer22_attention_weight_min": -25.418603897094727,
      "activations/layer23_attention_weight_max": 43.76696014404297,
      "activations/layer23_attention_weight_min": -22.723602294921875,
      "activations/layer2_attention_weight_max": 32.699562072753906,
      "activations/layer2_attention_weight_min": -29.855575561523438,
      "activations/layer3_attention_weight_max": 93.22721862792969,
      "activations/layer3_attention_weight_min": -92.34991455078125,
      "activations/layer4_attention_weight_max": 97.98226928710938,
      "activations/layer4_attention_weight_min": -94.47172546386719,
      "activations/layer5_attention_weight_max": 77.28762817382812,
      "activations/layer5_attention_weight_min": -81.11021423339844,
      "activations/layer6_attention_weight_max": 54.67399978637695,
      "activations/layer6_attention_weight_min": -54.486839294433594,
      "activations/layer7_attention_weight_max": 76.19140625,
      "activations/layer7_attention_weight_min": -77.41619873046875,
      "activations/layer8_attention_weight_max": 45.23616409301758,
      "activations/layer8_attention_weight_min": -47.95350646972656,
      "activations/layer9_attention_weight_max": 54.92904281616211,
      "activations/layer9_attention_weight_min": -50.39406204223633,
      "epoch": 9.8,
      "learning_rate": 8.770719696969696e-05,
      "loss": 2.7831,
      "step": 168600
    },
    {
      "activations/layer0_attention_weight_max": 16.16244888305664,
      "activations/layer0_attention_weight_min": -13.997757911682129,
      "activations/layer10_attention_weight_max": 39.514259338378906,
      "activations/layer10_attention_weight_min": -35.81706237792969,
      "activations/layer11_attention_weight_max": 30.31745719909668,
      "activations/layer11_attention_weight_min": -30.224231719970703,
      "activations/layer12_attention_weight_max": 19.693151473999023,
      "activations/layer12_attention_weight_min": -27.145462036132812,
      "activations/layer13_attention_weight_max": 42.76595687866211,
      "activations/layer13_attention_weight_min": -30.398998260498047,
      "activations/layer14_attention_weight_max": 35.834259033203125,
      "activations/layer14_attention_weight_min": -30.52032470703125,
      "activations/layer15_attention_weight_max": 33.25336456298828,
      "activations/layer15_attention_weight_min": -30.492725372314453,
      "activations/layer16_attention_weight_max": 33.82188415527344,
      "activations/layer16_attention_weight_min": -29.647552490234375,
      "activations/layer17_attention_weight_max": 52.936134338378906,
      "activations/layer17_attention_weight_min": -46.01932907104492,
      "activations/layer18_attention_weight_max": 46.2261962890625,
      "activations/layer18_attention_weight_min": -37.98051452636719,
      "activations/layer19_attention_weight_max": 23.979732513427734,
      "activations/layer19_attention_weight_min": -20.273447036743164,
      "activations/layer1_attention_weight_max": 17.896739959716797,
      "activations/layer1_attention_weight_min": -16.223146438598633,
      "activations/layer20_attention_weight_max": 22.873149871826172,
      "activations/layer20_attention_weight_min": -20.35686492919922,
      "activations/layer21_attention_weight_max": 36.3846435546875,
      "activations/layer21_attention_weight_min": -22.12176513671875,
      "activations/layer22_attention_weight_max": 31.247543334960938,
      "activations/layer22_attention_weight_min": -25.583637237548828,
      "activations/layer23_attention_weight_max": 37.70121383666992,
      "activations/layer23_attention_weight_min": -25.352834701538086,
      "activations/layer2_attention_weight_max": 33.40422058105469,
      "activations/layer2_attention_weight_min": -33.232913970947266,
      "activations/layer3_attention_weight_max": 90.05016326904297,
      "activations/layer3_attention_weight_min": -90.53450775146484,
      "activations/layer4_attention_weight_max": 99.87591552734375,
      "activations/layer4_attention_weight_min": -93.74458312988281,
      "activations/layer5_attention_weight_max": 74.3072738647461,
      "activations/layer5_attention_weight_min": -77.35220336914062,
      "activations/layer6_attention_weight_max": 52.519691467285156,
      "activations/layer6_attention_weight_min": -51.345821380615234,
      "activations/layer7_attention_weight_max": 77.69808197021484,
      "activations/layer7_attention_weight_min": -71.4210433959961,
      "activations/layer8_attention_weight_max": 44.99150466918945,
      "activations/layer8_attention_weight_min": -43.53834533691406,
      "activations/layer9_attention_weight_max": 50.17679977416992,
      "activations/layer9_attention_weight_min": -46.76072311401367,
      "epoch": 9.8,
      "learning_rate": 8.768825757575758e-05,
      "loss": 2.7954,
      "step": 168650
    },
    {
      "activations/layer0_attention_weight_max": 15.398957252502441,
      "activations/layer0_attention_weight_min": -14.177788734436035,
      "activations/layer10_attention_weight_max": 40.69952392578125,
      "activations/layer10_attention_weight_min": -36.00077819824219,
      "activations/layer11_attention_weight_max": 33.64865493774414,
      "activations/layer11_attention_weight_min": -34.152626037597656,
      "activations/layer12_attention_weight_max": 26.181493759155273,
      "activations/layer12_attention_weight_min": -24.36355972290039,
      "activations/layer13_attention_weight_max": 45.65559768676758,
      "activations/layer13_attention_weight_min": -32.3681526184082,
      "activations/layer14_attention_weight_max": 38.71733856201172,
      "activations/layer14_attention_weight_min": -30.429046630859375,
      "activations/layer15_attention_weight_max": 37.75757598876953,
      "activations/layer15_attention_weight_min": -28.1789608001709,
      "activations/layer16_attention_weight_max": 32.6478385925293,
      "activations/layer16_attention_weight_min": -29.047300338745117,
      "activations/layer17_attention_weight_max": 53.788116455078125,
      "activations/layer17_attention_weight_min": -43.30131912231445,
      "activations/layer18_attention_weight_max": 46.54523468017578,
      "activations/layer18_attention_weight_min": -37.96927261352539,
      "activations/layer19_attention_weight_max": 22.519819259643555,
      "activations/layer19_attention_weight_min": -23.77382469177246,
      "activations/layer1_attention_weight_max": 16.6268253326416,
      "activations/layer1_attention_weight_min": -14.145130157470703,
      "activations/layer20_attention_weight_max": 23.272485733032227,
      "activations/layer20_attention_weight_min": -22.555912017822266,
      "activations/layer21_attention_weight_max": 37.63666534423828,
      "activations/layer21_attention_weight_min": -23.784238815307617,
      "activations/layer22_attention_weight_max": 30.876619338989258,
      "activations/layer22_attention_weight_min": -24.93739128112793,
      "activations/layer23_attention_weight_max": 33.88608932495117,
      "activations/layer23_attention_weight_min": -23.682445526123047,
      "activations/layer2_attention_weight_max": 32.53652572631836,
      "activations/layer2_attention_weight_min": -30.53619384765625,
      "activations/layer3_attention_weight_max": 87.27938842773438,
      "activations/layer3_attention_weight_min": -91.89945983886719,
      "activations/layer4_attention_weight_max": 96.73743438720703,
      "activations/layer4_attention_weight_min": -95.49415588378906,
      "activations/layer5_attention_weight_max": 77.5311279296875,
      "activations/layer5_attention_weight_min": -77.18293762207031,
      "activations/layer6_attention_weight_max": 55.174163818359375,
      "activations/layer6_attention_weight_min": -52.22164535522461,
      "activations/layer7_attention_weight_max": 80.92123413085938,
      "activations/layer7_attention_weight_min": -75.85691833496094,
      "activations/layer8_attention_weight_max": 48.56120681762695,
      "activations/layer8_attention_weight_min": -47.990325927734375,
      "activations/layer9_attention_weight_max": 51.13283157348633,
      "activations/layer9_attention_weight_min": -47.83182907104492,
      "epoch": 9.8,
      "learning_rate": 8.766931818181818e-05,
      "loss": 2.7975,
      "step": 168700
    },
    {
      "activations/layer0_attention_weight_max": 15.427156448364258,
      "activations/layer0_attention_weight_min": -14.241695404052734,
      "activations/layer10_attention_weight_max": 36.14997482299805,
      "activations/layer10_attention_weight_min": -34.42253494262695,
      "activations/layer11_attention_weight_max": 34.05683135986328,
      "activations/layer11_attention_weight_min": -31.4982967376709,
      "activations/layer12_attention_weight_max": 20.721784591674805,
      "activations/layer12_attention_weight_min": -25.759675979614258,
      "activations/layer13_attention_weight_max": 48.77762985229492,
      "activations/layer13_attention_weight_min": -33.458133697509766,
      "activations/layer14_attention_weight_max": 36.915321350097656,
      "activations/layer14_attention_weight_min": -30.597665786743164,
      "activations/layer15_attention_weight_max": 37.32134246826172,
      "activations/layer15_attention_weight_min": -31.01188850402832,
      "activations/layer16_attention_weight_max": 31.27996063232422,
      "activations/layer16_attention_weight_min": -28.780378341674805,
      "activations/layer17_attention_weight_max": 55.03340148925781,
      "activations/layer17_attention_weight_min": -44.939064025878906,
      "activations/layer18_attention_weight_max": 47.599971771240234,
      "activations/layer18_attention_weight_min": -39.597049713134766,
      "activations/layer19_attention_weight_max": 21.130834579467773,
      "activations/layer19_attention_weight_min": -22.615503311157227,
      "activations/layer1_attention_weight_max": 17.7613525390625,
      "activations/layer1_attention_weight_min": -17.597415924072266,
      "activations/layer20_attention_weight_max": 22.63326072692871,
      "activations/layer20_attention_weight_min": -24.751895904541016,
      "activations/layer21_attention_weight_max": 39.2787971496582,
      "activations/layer21_attention_weight_min": -22.5136775970459,
      "activations/layer22_attention_weight_max": 30.16166114807129,
      "activations/layer22_attention_weight_min": -24.559329986572266,
      "activations/layer23_attention_weight_max": 36.700740814208984,
      "activations/layer23_attention_weight_min": -24.574657440185547,
      "activations/layer2_attention_weight_max": 35.564334869384766,
      "activations/layer2_attention_weight_min": -32.983158111572266,
      "activations/layer3_attention_weight_max": 94.60907745361328,
      "activations/layer3_attention_weight_min": -90.05635070800781,
      "activations/layer4_attention_weight_max": 100.05318450927734,
      "activations/layer4_attention_weight_min": -93.50794982910156,
      "activations/layer5_attention_weight_max": 77.18345642089844,
      "activations/layer5_attention_weight_min": -77.82127380371094,
      "activations/layer6_attention_weight_max": 54.656776428222656,
      "activations/layer6_attention_weight_min": -51.8822135925293,
      "activations/layer7_attention_weight_max": 79.9173812866211,
      "activations/layer7_attention_weight_min": -83.00074768066406,
      "activations/layer8_attention_weight_max": 45.80171203613281,
      "activations/layer8_attention_weight_min": -46.41594314575195,
      "activations/layer9_attention_weight_max": 49.89635467529297,
      "activations/layer9_attention_weight_min": -48.907981872558594,
      "epoch": 9.81,
      "learning_rate": 8.765037878787878e-05,
      "loss": 2.7909,
      "step": 168750
    },
    {
      "activations/layer0_attention_weight_max": 15.706411361694336,
      "activations/layer0_attention_weight_min": -14.833754539489746,
      "activations/layer10_attention_weight_max": 38.77800369262695,
      "activations/layer10_attention_weight_min": -37.112850189208984,
      "activations/layer11_attention_weight_max": 35.84876251220703,
      "activations/layer11_attention_weight_min": -32.44354248046875,
      "activations/layer12_attention_weight_max": 22.002832412719727,
      "activations/layer12_attention_weight_min": -25.975793838500977,
      "activations/layer13_attention_weight_max": 46.161155700683594,
      "activations/layer13_attention_weight_min": -32.40847396850586,
      "activations/layer14_attention_weight_max": 39.2502326965332,
      "activations/layer14_attention_weight_min": -31.386545181274414,
      "activations/layer15_attention_weight_max": 36.1507568359375,
      "activations/layer15_attention_weight_min": -30.354909896850586,
      "activations/layer16_attention_weight_max": 36.79473114013672,
      "activations/layer16_attention_weight_min": -30.438785552978516,
      "activations/layer17_attention_weight_max": 57.81983184814453,
      "activations/layer17_attention_weight_min": -45.1703987121582,
      "activations/layer18_attention_weight_max": 52.38695526123047,
      "activations/layer18_attention_weight_min": -37.092674255371094,
      "activations/layer19_attention_weight_max": 21.53957176208496,
      "activations/layer19_attention_weight_min": -24.564266204833984,
      "activations/layer1_attention_weight_max": 17.422595977783203,
      "activations/layer1_attention_weight_min": -14.361639976501465,
      "activations/layer20_attention_weight_max": 22.429494857788086,
      "activations/layer20_attention_weight_min": -23.788713455200195,
      "activations/layer21_attention_weight_max": 35.1229133605957,
      "activations/layer21_attention_weight_min": -22.58216094970703,
      "activations/layer22_attention_weight_max": 35.06141662597656,
      "activations/layer22_attention_weight_min": -25.915870666503906,
      "activations/layer23_attention_weight_max": 43.992820739746094,
      "activations/layer23_attention_weight_min": -23.479408264160156,
      "activations/layer2_attention_weight_max": 32.64593505859375,
      "activations/layer2_attention_weight_min": -31.33315658569336,
      "activations/layer3_attention_weight_max": 97.43455505371094,
      "activations/layer3_attention_weight_min": -93.63473510742188,
      "activations/layer4_attention_weight_max": 99.84972381591797,
      "activations/layer4_attention_weight_min": -96.44894409179688,
      "activations/layer5_attention_weight_max": 78.17022705078125,
      "activations/layer5_attention_weight_min": -79.98593139648438,
      "activations/layer6_attention_weight_max": 56.220909118652344,
      "activations/layer6_attention_weight_min": -53.57027816772461,
      "activations/layer7_attention_weight_max": 82.20609283447266,
      "activations/layer7_attention_weight_min": -76.6240463256836,
      "activations/layer8_attention_weight_max": 47.39149856567383,
      "activations/layer8_attention_weight_min": -46.0348014831543,
      "activations/layer9_attention_weight_max": 60.67731857299805,
      "activations/layer9_attention_weight_min": -50.58381652832031,
      "epoch": 9.81,
      "learning_rate": 8.76314393939394e-05,
      "loss": 2.7904,
      "step": 168800
    },
    {
      "activations/layer0_attention_weight_max": 15.195876121520996,
      "activations/layer0_attention_weight_min": -13.613519668579102,
      "activations/layer10_attention_weight_max": 36.545169830322266,
      "activations/layer10_attention_weight_min": -34.622100830078125,
      "activations/layer11_attention_weight_max": 30.59968376159668,
      "activations/layer11_attention_weight_min": -32.64080810546875,
      "activations/layer12_attention_weight_max": 20.136449813842773,
      "activations/layer12_attention_weight_min": -25.686527252197266,
      "activations/layer13_attention_weight_max": 39.11854934692383,
      "activations/layer13_attention_weight_min": -28.773584365844727,
      "activations/layer14_attention_weight_max": 39.92961120605469,
      "activations/layer14_attention_weight_min": -30.20341682434082,
      "activations/layer15_attention_weight_max": 36.44626998901367,
      "activations/layer15_attention_weight_min": -27.74308967590332,
      "activations/layer16_attention_weight_max": 30.47171974182129,
      "activations/layer16_attention_weight_min": -30.02037239074707,
      "activations/layer17_attention_weight_max": 54.16603469848633,
      "activations/layer17_attention_weight_min": -45.24161911010742,
      "activations/layer18_attention_weight_max": 50.48531723022461,
      "activations/layer18_attention_weight_min": -37.24602127075195,
      "activations/layer19_attention_weight_max": 20.432666778564453,
      "activations/layer19_attention_weight_min": -21.441938400268555,
      "activations/layer1_attention_weight_max": 16.772920608520508,
      "activations/layer1_attention_weight_min": -15.436176300048828,
      "activations/layer20_attention_weight_max": 20.846216201782227,
      "activations/layer20_attention_weight_min": -22.833641052246094,
      "activations/layer21_attention_weight_max": 38.71498107910156,
      "activations/layer21_attention_weight_min": -21.83949851989746,
      "activations/layer22_attention_weight_max": 29.455068588256836,
      "activations/layer22_attention_weight_min": -25.440584182739258,
      "activations/layer23_attention_weight_max": 36.9937744140625,
      "activations/layer23_attention_weight_min": -22.965274810791016,
      "activations/layer2_attention_weight_max": 33.49897003173828,
      "activations/layer2_attention_weight_min": -31.81423568725586,
      "activations/layer3_attention_weight_max": 95.67281341552734,
      "activations/layer3_attention_weight_min": -91.88786315917969,
      "activations/layer4_attention_weight_max": 102.34912109375,
      "activations/layer4_attention_weight_min": -96.32821655273438,
      "activations/layer5_attention_weight_max": 78.06126403808594,
      "activations/layer5_attention_weight_min": -81.79728698730469,
      "activations/layer6_attention_weight_max": 52.74370193481445,
      "activations/layer6_attention_weight_min": -51.65115737915039,
      "activations/layer7_attention_weight_max": 74.07914733886719,
      "activations/layer7_attention_weight_min": -74.16149139404297,
      "activations/layer8_attention_weight_max": 43.42111587524414,
      "activations/layer8_attention_weight_min": -44.26727294921875,
      "activations/layer9_attention_weight_max": 46.306983947753906,
      "activations/layer9_attention_weight_min": -46.096275329589844,
      "epoch": 9.81,
      "learning_rate": 8.761249999999998e-05,
      "loss": 2.8027,
      "step": 168850
    },
    {
      "activations/layer0_attention_weight_max": 17.191829681396484,
      "activations/layer0_attention_weight_min": -14.077186584472656,
      "activations/layer10_attention_weight_max": 36.956878662109375,
      "activations/layer10_attention_weight_min": -35.20989227294922,
      "activations/layer11_attention_weight_max": 30.883956909179688,
      "activations/layer11_attention_weight_min": -33.21014404296875,
      "activations/layer12_attention_weight_max": 18.813806533813477,
      "activations/layer12_attention_weight_min": -25.157880783081055,
      "activations/layer13_attention_weight_max": 41.77010726928711,
      "activations/layer13_attention_weight_min": -31.850242614746094,
      "activations/layer14_attention_weight_max": 37.26376724243164,
      "activations/layer14_attention_weight_min": -30.281978607177734,
      "activations/layer15_attention_weight_max": 34.72222137451172,
      "activations/layer15_attention_weight_min": -29.617212295532227,
      "activations/layer16_attention_weight_max": 32.47695541381836,
      "activations/layer16_attention_weight_min": -28.3675479888916,
      "activations/layer17_attention_weight_max": 55.86497497558594,
      "activations/layer17_attention_weight_min": -45.5264892578125,
      "activations/layer18_attention_weight_max": 54.64615249633789,
      "activations/layer18_attention_weight_min": -39.13334655761719,
      "activations/layer19_attention_weight_max": 22.828298568725586,
      "activations/layer19_attention_weight_min": -21.356529235839844,
      "activations/layer1_attention_weight_max": 17.694028854370117,
      "activations/layer1_attention_weight_min": -13.809412956237793,
      "activations/layer20_attention_weight_max": 23.361886978149414,
      "activations/layer20_attention_weight_min": -20.838346481323242,
      "activations/layer21_attention_weight_max": 36.17134094238281,
      "activations/layer21_attention_weight_min": -21.98613929748535,
      "activations/layer22_attention_weight_max": 28.83721923828125,
      "activations/layer22_attention_weight_min": -24.95499610900879,
      "activations/layer23_attention_weight_max": 37.729225158691406,
      "activations/layer23_attention_weight_min": -22.011449813842773,
      "activations/layer2_attention_weight_max": 32.59596633911133,
      "activations/layer2_attention_weight_min": -30.31305503845215,
      "activations/layer3_attention_weight_max": 93.36840057373047,
      "activations/layer3_attention_weight_min": -97.24427795410156,
      "activations/layer4_attention_weight_max": 96.9898910522461,
      "activations/layer4_attention_weight_min": -96.65721130371094,
      "activations/layer5_attention_weight_max": 74.67780303955078,
      "activations/layer5_attention_weight_min": -79.16168212890625,
      "activations/layer6_attention_weight_max": 52.784828186035156,
      "activations/layer6_attention_weight_min": -54.14353942871094,
      "activations/layer7_attention_weight_max": 77.07688903808594,
      "activations/layer7_attention_weight_min": -77.57528686523438,
      "activations/layer8_attention_weight_max": 41.5292854309082,
      "activations/layer8_attention_weight_min": -43.95721435546875,
      "activations/layer9_attention_weight_max": 46.551448822021484,
      "activations/layer9_attention_weight_min": -46.97542953491211,
      "epoch": 9.81,
      "learning_rate": 8.75935606060606e-05,
      "loss": 2.8074,
      "step": 168900
    },
    {
      "activations/layer0_attention_weight_max": 15.850536346435547,
      "activations/layer0_attention_weight_min": -14.186807632446289,
      "activations/layer10_attention_weight_max": 36.201210021972656,
      "activations/layer10_attention_weight_min": -35.43036651611328,
      "activations/layer11_attention_weight_max": 30.260974884033203,
      "activations/layer11_attention_weight_min": -30.542463302612305,
      "activations/layer12_attention_weight_max": 18.928495407104492,
      "activations/layer12_attention_weight_min": -22.28716468811035,
      "activations/layer13_attention_weight_max": 35.75543212890625,
      "activations/layer13_attention_weight_min": -31.273977279663086,
      "activations/layer14_attention_weight_max": 35.754234313964844,
      "activations/layer14_attention_weight_min": -29.57843017578125,
      "activations/layer15_attention_weight_max": 30.214784622192383,
      "activations/layer15_attention_weight_min": -28.65420150756836,
      "activations/layer16_attention_weight_max": 29.512313842773438,
      "activations/layer16_attention_weight_min": -27.339130401611328,
      "activations/layer17_attention_weight_max": 53.25772476196289,
      "activations/layer17_attention_weight_min": -41.499778747558594,
      "activations/layer18_attention_weight_max": 43.40790939331055,
      "activations/layer18_attention_weight_min": -34.01014709472656,
      "activations/layer19_attention_weight_max": 21.406341552734375,
      "activations/layer19_attention_weight_min": -20.902212142944336,
      "activations/layer1_attention_weight_max": 16.756359100341797,
      "activations/layer1_attention_weight_min": -17.71641731262207,
      "activations/layer20_attention_weight_max": 20.957027435302734,
      "activations/layer20_attention_weight_min": -22.89524269104004,
      "activations/layer21_attention_weight_max": 30.093900680541992,
      "activations/layer21_attention_weight_min": -21.50638771057129,
      "activations/layer22_attention_weight_max": 28.39129638671875,
      "activations/layer22_attention_weight_min": -25.808305740356445,
      "activations/layer23_attention_weight_max": 37.593162536621094,
      "activations/layer23_attention_weight_min": -23.95741081237793,
      "activations/layer2_attention_weight_max": 31.623798370361328,
      "activations/layer2_attention_weight_min": -31.089691162109375,
      "activations/layer3_attention_weight_max": 93.50154876708984,
      "activations/layer3_attention_weight_min": -98.60794830322266,
      "activations/layer4_attention_weight_max": 99.90270233154297,
      "activations/layer4_attention_weight_min": -98.97663116455078,
      "activations/layer5_attention_weight_max": 80.14910125732422,
      "activations/layer5_attention_weight_min": -85.89763641357422,
      "activations/layer6_attention_weight_max": 52.64887619018555,
      "activations/layer6_attention_weight_min": -53.553409576416016,
      "activations/layer7_attention_weight_max": 76.5071029663086,
      "activations/layer7_attention_weight_min": -74.6596908569336,
      "activations/layer8_attention_weight_max": 44.88037109375,
      "activations/layer8_attention_weight_min": -45.2152099609375,
      "activations/layer9_attention_weight_max": 48.40613555908203,
      "activations/layer9_attention_weight_min": -49.1658935546875,
      "epoch": 9.82,
      "learning_rate": 8.757462121212121e-05,
      "loss": 2.783,
      "step": 168950
    },
    {
      "activations/layer0_attention_weight_max": 15.080901145935059,
      "activations/layer0_attention_weight_min": -14.142631530761719,
      "activations/layer10_attention_weight_max": 39.24477767944336,
      "activations/layer10_attention_weight_min": -36.17017364501953,
      "activations/layer11_attention_weight_max": 34.05330276489258,
      "activations/layer11_attention_weight_min": -33.72324752807617,
      "activations/layer12_attention_weight_max": 17.878753662109375,
      "activations/layer12_attention_weight_min": -25.915000915527344,
      "activations/layer13_attention_weight_max": 43.14003372192383,
      "activations/layer13_attention_weight_min": -29.1976261138916,
      "activations/layer14_attention_weight_max": 36.990875244140625,
      "activations/layer14_attention_weight_min": -31.174299240112305,
      "activations/layer15_attention_weight_max": 34.744197845458984,
      "activations/layer15_attention_weight_min": -29.64815330505371,
      "activations/layer16_attention_weight_max": 29.52088165283203,
      "activations/layer16_attention_weight_min": -28.43621253967285,
      "activations/layer17_attention_weight_max": 52.45711135864258,
      "activations/layer17_attention_weight_min": -44.07908630371094,
      "activations/layer18_attention_weight_max": 46.15978240966797,
      "activations/layer18_attention_weight_min": -35.63294219970703,
      "activations/layer19_attention_weight_max": 21.80411148071289,
      "activations/layer19_attention_weight_min": -21.080961227416992,
      "activations/layer1_attention_weight_max": 17.3045597076416,
      "activations/layer1_attention_weight_min": -16.927581787109375,
      "activations/layer20_attention_weight_max": 21.211605072021484,
      "activations/layer20_attention_weight_min": -20.983806610107422,
      "activations/layer21_attention_weight_max": 37.38430404663086,
      "activations/layer21_attention_weight_min": -23.809776306152344,
      "activations/layer22_attention_weight_max": 30.70082664489746,
      "activations/layer22_attention_weight_min": -25.86815643310547,
      "activations/layer23_attention_weight_max": 32.86411666870117,
      "activations/layer23_attention_weight_min": -24.866064071655273,
      "activations/layer2_attention_weight_max": 33.32070541381836,
      "activations/layer2_attention_weight_min": -30.557613372802734,
      "activations/layer3_attention_weight_max": 95.45372009277344,
      "activations/layer3_attention_weight_min": -95.20221710205078,
      "activations/layer4_attention_weight_max": 108.62126922607422,
      "activations/layer4_attention_weight_min": -104.08778381347656,
      "activations/layer5_attention_weight_max": 83.86898803710938,
      "activations/layer5_attention_weight_min": -81.59700775146484,
      "activations/layer6_attention_weight_max": 56.939273834228516,
      "activations/layer6_attention_weight_min": -56.1263427734375,
      "activations/layer7_attention_weight_max": 78.13030242919922,
      "activations/layer7_attention_weight_min": -75.16877746582031,
      "activations/layer8_attention_weight_max": 47.072879791259766,
      "activations/layer8_attention_weight_min": -49.94438552856445,
      "activations/layer9_attention_weight_max": 51.58582305908203,
      "activations/layer9_attention_weight_min": -50.3455696105957,
      "epoch": 9.82,
      "learning_rate": 8.75556818181818e-05,
      "loss": 2.7988,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_loss": 2.74609375,
      "eval_runtime": 8.5031,
      "eval_samples_per_second": 504.989,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_openwebtext_loss": 2.74609375,
      "eval_openwebtext_ppl": 15.581647041956762,
      "eval_openwebtext_runtime": 8.5031,
      "eval_openwebtext_samples_per_second": 504.989,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_wikitext_loss": 2.978515625,
      "eval_wikitext_ppl": 19.658614220966218,
      "eval_wikitext_runtime": 2.0006,
      "eval_wikitext_samples_per_second": 227.928,
      "step": 169000
    },
    {
      "epoch": 9.82,
      "eval_lambada_loss": 2.654296875,
      "eval_lambada_ppl": 14.214987631491761,
      "eval_lambada_runtime": 9.5969,
      "eval_lambada_samples_per_second": 507.35,
      "step": 169000
    },
    {
      "activations/layer0_attention_weight_max": 15.962904930114746,
      "activations/layer0_attention_weight_min": -14.386040687561035,
      "activations/layer10_attention_weight_max": 34.86336135864258,
      "activations/layer10_attention_weight_min": -33.80424118041992,
      "activations/layer11_attention_weight_max": 32.03412628173828,
      "activations/layer11_attention_weight_min": -31.34767723083496,
      "activations/layer12_attention_weight_max": 19.226306915283203,
      "activations/layer12_attention_weight_min": -30.837427139282227,
      "activations/layer13_attention_weight_max": 41.46979522705078,
      "activations/layer13_attention_weight_min": -30.588777542114258,
      "activations/layer14_attention_weight_max": 37.656333923339844,
      "activations/layer14_attention_weight_min": -31.39151382446289,
      "activations/layer15_attention_weight_max": 35.27363204956055,
      "activations/layer15_attention_weight_min": -29.73137664794922,
      "activations/layer16_attention_weight_max": 35.74356460571289,
      "activations/layer16_attention_weight_min": -30.537715911865234,
      "activations/layer17_attention_weight_max": 51.493202209472656,
      "activations/layer17_attention_weight_min": -43.66926956176758,
      "activations/layer18_attention_weight_max": 48.10935974121094,
      "activations/layer18_attention_weight_min": -38.98662567138672,
      "activations/layer19_attention_weight_max": 22.82337188720703,
      "activations/layer19_attention_weight_min": -22.60451889038086,
      "activations/layer1_attention_weight_max": 16.991191864013672,
      "activations/layer1_attention_weight_min": -15.866323471069336,
      "activations/layer20_attention_weight_max": 22.203567504882812,
      "activations/layer20_attention_weight_min": -21.413984298706055,
      "activations/layer21_attention_weight_max": 36.40595245361328,
      "activations/layer21_attention_weight_min": -24.242258071899414,
      "activations/layer22_attention_weight_max": 33.096763610839844,
      "activations/layer22_attention_weight_min": -28.375804901123047,
      "activations/layer23_attention_weight_max": 34.54038619995117,
      "activations/layer23_attention_weight_min": -23.96420669555664,
      "activations/layer2_attention_weight_max": 32.93962097167969,
      "activations/layer2_attention_weight_min": -30.599599838256836,
      "activations/layer3_attention_weight_max": 90.8072280883789,
      "activations/layer3_attention_weight_min": -87.94280242919922,
      "activations/layer4_attention_weight_max": 101.86558532714844,
      "activations/layer4_attention_weight_min": -92.66705322265625,
      "activations/layer5_attention_weight_max": 78.15142822265625,
      "activations/layer5_attention_weight_min": -76.42894744873047,
      "activations/layer6_attention_weight_max": 54.58412551879883,
      "activations/layer6_attention_weight_min": -52.5884895324707,
      "activations/layer7_attention_weight_max": 75.23027038574219,
      "activations/layer7_attention_weight_min": -70.71852111816406,
      "activations/layer8_attention_weight_max": 43.588314056396484,
      "activations/layer8_attention_weight_min": -46.261810302734375,
      "activations/layer9_attention_weight_max": 54.525020599365234,
      "activations/layer9_attention_weight_min": -51.68425369262695,
      "epoch": 9.82,
      "learning_rate": 8.753674242424241e-05,
      "loss": 2.8012,
      "step": 169050
    },
    {
      "activations/layer0_attention_weight_max": 15.374713897705078,
      "activations/layer0_attention_weight_min": -14.226460456848145,
      "activations/layer10_attention_weight_max": 36.22744369506836,
      "activations/layer10_attention_weight_min": -34.52384567260742,
      "activations/layer11_attention_weight_max": 31.785781860351562,
      "activations/layer11_attention_weight_min": -31.10165786743164,
      "activations/layer12_attention_weight_max": 19.61488914489746,
      "activations/layer12_attention_weight_min": -26.552997589111328,
      "activations/layer13_attention_weight_max": 52.006752014160156,
      "activations/layer13_attention_weight_min": -34.23946762084961,
      "activations/layer14_attention_weight_max": 52.33848571777344,
      "activations/layer14_attention_weight_min": -33.02784729003906,
      "activations/layer15_attention_weight_max": 43.52057647705078,
      "activations/layer15_attention_weight_min": -32.1546630859375,
      "activations/layer16_attention_weight_max": 30.64154815673828,
      "activations/layer16_attention_weight_min": -29.105424880981445,
      "activations/layer17_attention_weight_max": 49.39003372192383,
      "activations/layer17_attention_weight_min": -47.14775848388672,
      "activations/layer18_attention_weight_max": 45.60523986816406,
      "activations/layer18_attention_weight_min": -38.177547454833984,
      "activations/layer19_attention_weight_max": 20.787357330322266,
      "activations/layer19_attention_weight_min": -20.152223587036133,
      "activations/layer1_attention_weight_max": 17.615121841430664,
      "activations/layer1_attention_weight_min": -15.345609664916992,
      "activations/layer20_attention_weight_max": 21.237472534179688,
      "activations/layer20_attention_weight_min": -21.79912567138672,
      "activations/layer21_attention_weight_max": 36.699745178222656,
      "activations/layer21_attention_weight_min": -24.54628562927246,
      "activations/layer22_attention_weight_max": 34.184600830078125,
      "activations/layer22_attention_weight_min": -24.562240600585938,
      "activations/layer23_attention_weight_max": 38.93536376953125,
      "activations/layer23_attention_weight_min": -24.602672576904297,
      "activations/layer2_attention_weight_max": 32.49989318847656,
      "activations/layer2_attention_weight_min": -31.157773971557617,
      "activations/layer3_attention_weight_max": 96.21469116210938,
      "activations/layer3_attention_weight_min": -96.4516830444336,
      "activations/layer4_attention_weight_max": 102.95548248291016,
      "activations/layer4_attention_weight_min": -97.50729370117188,
      "activations/layer5_attention_weight_max": 76.20291137695312,
      "activations/layer5_attention_weight_min": -81.66876220703125,
      "activations/layer6_attention_weight_max": 54.57231521606445,
      "activations/layer6_attention_weight_min": -53.528011322021484,
      "activations/layer7_attention_weight_max": 76.67548370361328,
      "activations/layer7_attention_weight_min": -75.95137786865234,
      "activations/layer8_attention_weight_max": 43.8419189453125,
      "activations/layer8_attention_weight_min": -46.12767028808594,
      "activations/layer9_attention_weight_max": 53.634735107421875,
      "activations/layer9_attention_weight_min": -48.51839828491211,
      "epoch": 9.83,
      "learning_rate": 8.751780303030303e-05,
      "loss": 2.7869,
      "step": 169100
    },
    {
      "activations/layer0_attention_weight_max": 15.5625,
      "activations/layer0_attention_weight_min": -14.376465797424316,
      "activations/layer10_attention_weight_max": 47.911983489990234,
      "activations/layer10_attention_weight_min": -43.624412536621094,
      "activations/layer11_attention_weight_max": 42.566253662109375,
      "activations/layer11_attention_weight_min": -38.99652099609375,
      "activations/layer12_attention_weight_max": 20.511621475219727,
      "activations/layer12_attention_weight_min": -27.09617805480957,
      "activations/layer13_attention_weight_max": 59.340232849121094,
      "activations/layer13_attention_weight_min": -32.83175277709961,
      "activations/layer14_attention_weight_max": 80.54362487792969,
      "activations/layer14_attention_weight_min": -34.35673141479492,
      "activations/layer15_attention_weight_max": 61.36228561401367,
      "activations/layer15_attention_weight_min": -32.00646209716797,
      "activations/layer16_attention_weight_max": 36.65229034423828,
      "activations/layer16_attention_weight_min": -31.94928741455078,
      "activations/layer17_attention_weight_max": 63.93281936645508,
      "activations/layer17_attention_weight_min": -51.22829818725586,
      "activations/layer18_attention_weight_max": 58.27176284790039,
      "activations/layer18_attention_weight_min": -44.45450210571289,
      "activations/layer19_attention_weight_max": 25.644140243530273,
      "activations/layer19_attention_weight_min": -24.606204986572266,
      "activations/layer1_attention_weight_max": 17.345516204833984,
      "activations/layer1_attention_weight_min": -14.246487617492676,
      "activations/layer20_attention_weight_max": 29.093788146972656,
      "activations/layer20_attention_weight_min": -20.373157501220703,
      "activations/layer21_attention_weight_max": 49.253719329833984,
      "activations/layer21_attention_weight_min": -23.903764724731445,
      "activations/layer22_attention_weight_max": 34.515769958496094,
      "activations/layer22_attention_weight_min": -26.20570182800293,
      "activations/layer23_attention_weight_max": 40.393775939941406,
      "activations/layer23_attention_weight_min": -23.621877670288086,
      "activations/layer2_attention_weight_max": 36.02309799194336,
      "activations/layer2_attention_weight_min": -32.009002685546875,
      "activations/layer3_attention_weight_max": 99.86822509765625,
      "activations/layer3_attention_weight_min": -103.60897827148438,
      "activations/layer4_attention_weight_max": 107.81416320800781,
      "activations/layer4_attention_weight_min": -104.18122863769531,
      "activations/layer5_attention_weight_max": 80.6884765625,
      "activations/layer5_attention_weight_min": -82.02386474609375,
      "activations/layer6_attention_weight_max": 55.27094650268555,
      "activations/layer6_attention_weight_min": -54.99028396606445,
      "activations/layer7_attention_weight_max": 86.65487670898438,
      "activations/layer7_attention_weight_min": -85.62991333007812,
      "activations/layer8_attention_weight_max": 52.42936706542969,
      "activations/layer8_attention_weight_min": -52.167694091796875,
      "activations/layer9_attention_weight_max": 65.9023208618164,
      "activations/layer9_attention_weight_min": -54.80646896362305,
      "epoch": 9.83,
      "learning_rate": 8.749886363636363e-05,
      "loss": 2.7947,
      "step": 169150
    },
    {
      "activations/layer0_attention_weight_max": 14.829185485839844,
      "activations/layer0_attention_weight_min": -14.096196174621582,
      "activations/layer10_attention_weight_max": 36.65531921386719,
      "activations/layer10_attention_weight_min": -34.35693359375,
      "activations/layer11_attention_weight_max": 31.18761444091797,
      "activations/layer11_attention_weight_min": -32.70442199707031,
      "activations/layer12_attention_weight_max": 18.4207820892334,
      "activations/layer12_attention_weight_min": -25.491378784179688,
      "activations/layer13_attention_weight_max": 49.22737121582031,
      "activations/layer13_attention_weight_min": -31.85500717163086,
      "activations/layer14_attention_weight_max": 52.35132598876953,
      "activations/layer14_attention_weight_min": -32.206687927246094,
      "activations/layer15_attention_weight_max": 43.45387649536133,
      "activations/layer15_attention_weight_min": -28.85873794555664,
      "activations/layer16_attention_weight_max": 30.784147262573242,
      "activations/layer16_attention_weight_min": -29.175397872924805,
      "activations/layer17_attention_weight_max": 52.80437469482422,
      "activations/layer17_attention_weight_min": -41.764244079589844,
      "activations/layer18_attention_weight_max": 48.75490951538086,
      "activations/layer18_attention_weight_min": -36.42058181762695,
      "activations/layer19_attention_weight_max": 20.01972770690918,
      "activations/layer19_attention_weight_min": -21.152048110961914,
      "activations/layer1_attention_weight_max": 17.183805465698242,
      "activations/layer1_attention_weight_min": -13.73197078704834,
      "activations/layer20_attention_weight_max": 23.240751266479492,
      "activations/layer20_attention_weight_min": -21.54669189453125,
      "activations/layer21_attention_weight_max": 38.9485969543457,
      "activations/layer21_attention_weight_min": -24.231529235839844,
      "activations/layer22_attention_weight_max": 31.63602066040039,
      "activations/layer22_attention_weight_min": -24.550386428833008,
      "activations/layer23_attention_weight_max": 35.590511322021484,
      "activations/layer23_attention_weight_min": -23.204151153564453,
      "activations/layer2_attention_weight_max": 30.856834411621094,
      "activations/layer2_attention_weight_min": -28.84493637084961,
      "activations/layer3_attention_weight_max": 88.3555908203125,
      "activations/layer3_attention_weight_min": -89.626953125,
      "activations/layer4_attention_weight_max": 105.22570037841797,
      "activations/layer4_attention_weight_min": -102.011962890625,
      "activations/layer5_attention_weight_max": 78.1032943725586,
      "activations/layer5_attention_weight_min": -82.54804992675781,
      "activations/layer6_attention_weight_max": 55.13983917236328,
      "activations/layer6_attention_weight_min": -53.04256820678711,
      "activations/layer7_attention_weight_max": 78.78272247314453,
      "activations/layer7_attention_weight_min": -73.35423278808594,
      "activations/layer8_attention_weight_max": 44.394264221191406,
      "activations/layer8_attention_weight_min": -45.28358840942383,
      "activations/layer9_attention_weight_max": 50.68056869506836,
      "activations/layer9_attention_weight_min": -48.958255767822266,
      "epoch": 9.83,
      "learning_rate": 8.747992424242423e-05,
      "loss": 2.8044,
      "step": 169200
    },
    {
      "activations/layer0_attention_weight_max": 16.133289337158203,
      "activations/layer0_attention_weight_min": -14.123370170593262,
      "activations/layer10_attention_weight_max": 35.0869026184082,
      "activations/layer10_attention_weight_min": -36.50604248046875,
      "activations/layer11_attention_weight_max": 30.61224365234375,
      "activations/layer11_attention_weight_min": -31.478599548339844,
      "activations/layer12_attention_weight_max": 19.776138305664062,
      "activations/layer12_attention_weight_min": -25.50914764404297,
      "activations/layer13_attention_weight_max": 44.992027282714844,
      "activations/layer13_attention_weight_min": -32.57194519042969,
      "activations/layer14_attention_weight_max": 38.31630325317383,
      "activations/layer14_attention_weight_min": -30.596454620361328,
      "activations/layer15_attention_weight_max": 37.10638427734375,
      "activations/layer15_attention_weight_min": -29.546470642089844,
      "activations/layer16_attention_weight_max": 38.28821563720703,
      "activations/layer16_attention_weight_min": -29.18109130859375,
      "activations/layer17_attention_weight_max": 56.45887756347656,
      "activations/layer17_attention_weight_min": -45.77812576293945,
      "activations/layer18_attention_weight_max": 50.65748977661133,
      "activations/layer18_attention_weight_min": -39.3961296081543,
      "activations/layer19_attention_weight_max": 23.778409957885742,
      "activations/layer19_attention_weight_min": -21.454246520996094,
      "activations/layer1_attention_weight_max": 16.247535705566406,
      "activations/layer1_attention_weight_min": -16.501222610473633,
      "activations/layer20_attention_weight_max": 24.43328857421875,
      "activations/layer20_attention_weight_min": -22.16534423828125,
      "activations/layer21_attention_weight_max": 41.370697021484375,
      "activations/layer21_attention_weight_min": -24.470029830932617,
      "activations/layer22_attention_weight_max": 32.41959762573242,
      "activations/layer22_attention_weight_min": -25.43096351623535,
      "activations/layer23_attention_weight_max": 38.412010192871094,
      "activations/layer23_attention_weight_min": -22.586231231689453,
      "activations/layer2_attention_weight_max": 32.294960021972656,
      "activations/layer2_attention_weight_min": -29.99972152709961,
      "activations/layer3_attention_weight_max": 92.77572631835938,
      "activations/layer3_attention_weight_min": -90.00892639160156,
      "activations/layer4_attention_weight_max": 102.6593246459961,
      "activations/layer4_attention_weight_min": -95.40740203857422,
      "activations/layer5_attention_weight_max": 77.62384033203125,
      "activations/layer5_attention_weight_min": -76.66141510009766,
      "activations/layer6_attention_weight_max": 53.36692428588867,
      "activations/layer6_attention_weight_min": -50.24156951904297,
      "activations/layer7_attention_weight_max": 74.17266082763672,
      "activations/layer7_attention_weight_min": -72.27577209472656,
      "activations/layer8_attention_weight_max": 44.34200668334961,
      "activations/layer8_attention_weight_min": -43.82181167602539,
      "activations/layer9_attention_weight_max": 48.51735305786133,
      "activations/layer9_attention_weight_min": -45.36457061767578,
      "epoch": 9.83,
      "learning_rate": 8.746098484848483e-05,
      "loss": 2.8024,
      "step": 169250
    },
    {
      "activations/layer0_attention_weight_max": 15.444395065307617,
      "activations/layer0_attention_weight_min": -14.091765403747559,
      "activations/layer10_attention_weight_max": 35.55195236206055,
      "activations/layer10_attention_weight_min": -36.38026809692383,
      "activations/layer11_attention_weight_max": 29.634796142578125,
      "activations/layer11_attention_weight_min": -33.11628341674805,
      "activations/layer12_attention_weight_max": 21.399166107177734,
      "activations/layer12_attention_weight_min": -26.562732696533203,
      "activations/layer13_attention_weight_max": 44.93361282348633,
      "activations/layer13_attention_weight_min": -33.878726959228516,
      "activations/layer14_attention_weight_max": 41.56045150756836,
      "activations/layer14_attention_weight_min": -34.42321014404297,
      "activations/layer15_attention_weight_max": 33.6756477355957,
      "activations/layer15_attention_weight_min": -30.93128204345703,
      "activations/layer16_attention_weight_max": 31.956632614135742,
      "activations/layer16_attention_weight_min": -31.134693145751953,
      "activations/layer17_attention_weight_max": 55.448299407958984,
      "activations/layer17_attention_weight_min": -47.23262405395508,
      "activations/layer18_attention_weight_max": 52.99057388305664,
      "activations/layer18_attention_weight_min": -39.93274688720703,
      "activations/layer19_attention_weight_max": 22.55390739440918,
      "activations/layer19_attention_weight_min": -22.169326782226562,
      "activations/layer1_attention_weight_max": 17.29597282409668,
      "activations/layer1_attention_weight_min": -15.324398040771484,
      "activations/layer20_attention_weight_max": 26.832029342651367,
      "activations/layer20_attention_weight_min": -21.99111557006836,
      "activations/layer21_attention_weight_max": 42.41245651245117,
      "activations/layer21_attention_weight_min": -24.45083999633789,
      "activations/layer22_attention_weight_max": 34.80460739135742,
      "activations/layer22_attention_weight_min": -26.4617977142334,
      "activations/layer23_attention_weight_max": 45.038265228271484,
      "activations/layer23_attention_weight_min": -24.041929244995117,
      "activations/layer2_attention_weight_max": 32.900779724121094,
      "activations/layer2_attention_weight_min": -31.143774032592773,
      "activations/layer3_attention_weight_max": 88.24710083007812,
      "activations/layer3_attention_weight_min": -96.86940002441406,
      "activations/layer4_attention_weight_max": 99.21448516845703,
      "activations/layer4_attention_weight_min": -97.71922302246094,
      "activations/layer5_attention_weight_max": 76.75358581542969,
      "activations/layer5_attention_weight_min": -79.01921081542969,
      "activations/layer6_attention_weight_max": 53.10179901123047,
      "activations/layer6_attention_weight_min": -51.930213928222656,
      "activations/layer7_attention_weight_max": 80.371826171875,
      "activations/layer7_attention_weight_min": -74.55471801757812,
      "activations/layer8_attention_weight_max": 45.41252899169922,
      "activations/layer8_attention_weight_min": -45.84286880493164,
      "activations/layer9_attention_weight_max": 48.375755310058594,
      "activations/layer9_attention_weight_min": -47.63913345336914,
      "epoch": 9.84,
      "learning_rate": 8.744204545454545e-05,
      "loss": 2.7812,
      "step": 169300
    },
    {
      "activations/layer0_attention_weight_max": 15.947983741760254,
      "activations/layer0_attention_weight_min": -14.067900657653809,
      "activations/layer10_attention_weight_max": 34.11304473876953,
      "activations/layer10_attention_weight_min": -34.334632873535156,
      "activations/layer11_attention_weight_max": 30.31251335144043,
      "activations/layer11_attention_weight_min": -33.090885162353516,
      "activations/layer12_attention_weight_max": 16.683958053588867,
      "activations/layer12_attention_weight_min": -22.371679306030273,
      "activations/layer13_attention_weight_max": 34.88238525390625,
      "activations/layer13_attention_weight_min": -33.26277160644531,
      "activations/layer14_attention_weight_max": 32.54204559326172,
      "activations/layer14_attention_weight_min": -31.277942657470703,
      "activations/layer15_attention_weight_max": 32.62801742553711,
      "activations/layer15_attention_weight_min": -27.988910675048828,
      "activations/layer16_attention_weight_max": 29.88109016418457,
      "activations/layer16_attention_weight_min": -27.755754470825195,
      "activations/layer17_attention_weight_max": 49.942710876464844,
      "activations/layer17_attention_weight_min": -42.18579864501953,
      "activations/layer18_attention_weight_max": 44.84355926513672,
      "activations/layer18_attention_weight_min": -36.3454704284668,
      "activations/layer19_attention_weight_max": 21.66978645324707,
      "activations/layer19_attention_weight_min": -22.06081199645996,
      "activations/layer1_attention_weight_max": 18.714855194091797,
      "activations/layer1_attention_weight_min": -18.271331787109375,
      "activations/layer20_attention_weight_max": 21.863706588745117,
      "activations/layer20_attention_weight_min": -22.6969051361084,
      "activations/layer21_attention_weight_max": 33.359405517578125,
      "activations/layer21_attention_weight_min": -22.284854888916016,
      "activations/layer22_attention_weight_max": 30.193119049072266,
      "activations/layer22_attention_weight_min": -24.244647979736328,
      "activations/layer23_attention_weight_max": 40.23225402832031,
      "activations/layer23_attention_weight_min": -25.7254695892334,
      "activations/layer2_attention_weight_max": 32.796661376953125,
      "activations/layer2_attention_weight_min": -31.184240341186523,
      "activations/layer3_attention_weight_max": 88.71878051757812,
      "activations/layer3_attention_weight_min": -90.97261047363281,
      "activations/layer4_attention_weight_max": 92.91130828857422,
      "activations/layer4_attention_weight_min": -95.41523742675781,
      "activations/layer5_attention_weight_max": 72.20489501953125,
      "activations/layer5_attention_weight_min": -78.57437133789062,
      "activations/layer6_attention_weight_max": 53.48066711425781,
      "activations/layer6_attention_weight_min": -54.260616302490234,
      "activations/layer7_attention_weight_max": 74.49258422851562,
      "activations/layer7_attention_weight_min": -71.55482482910156,
      "activations/layer8_attention_weight_max": 43.1762580871582,
      "activations/layer8_attention_weight_min": -46.39908218383789,
      "activations/layer9_attention_weight_max": 46.41211700439453,
      "activations/layer9_attention_weight_min": -46.44639587402344,
      "epoch": 9.84,
      "learning_rate": 8.742310606060605e-05,
      "loss": 2.7966,
      "step": 169350
    },
    {
      "activations/layer0_attention_weight_max": 15.830735206604004,
      "activations/layer0_attention_weight_min": -14.466347694396973,
      "activations/layer10_attention_weight_max": 43.51470184326172,
      "activations/layer10_attention_weight_min": -42.117095947265625,
      "activations/layer11_attention_weight_max": 37.91486740112305,
      "activations/layer11_attention_weight_min": -38.75684356689453,
      "activations/layer12_attention_weight_max": 24.163251876831055,
      "activations/layer12_attention_weight_min": -24.14365005493164,
      "activations/layer13_attention_weight_max": 39.470245361328125,
      "activations/layer13_attention_weight_min": -31.996431350708008,
      "activations/layer14_attention_weight_max": 38.956199645996094,
      "activations/layer14_attention_weight_min": -31.907381057739258,
      "activations/layer15_attention_weight_max": 36.42169189453125,
      "activations/layer15_attention_weight_min": -30.247982025146484,
      "activations/layer16_attention_weight_max": 34.62920379638672,
      "activations/layer16_attention_weight_min": -29.34379768371582,
      "activations/layer17_attention_weight_max": 52.54880905151367,
      "activations/layer17_attention_weight_min": -43.71558380126953,
      "activations/layer18_attention_weight_max": 47.660926818847656,
      "activations/layer18_attention_weight_min": -37.2891960144043,
      "activations/layer19_attention_weight_max": 22.260650634765625,
      "activations/layer19_attention_weight_min": -23.157543182373047,
      "activations/layer1_attention_weight_max": 16.750465393066406,
      "activations/layer1_attention_weight_min": -14.406285285949707,
      "activations/layer20_attention_weight_max": 23.570383071899414,
      "activations/layer20_attention_weight_min": -22.273223876953125,
      "activations/layer21_attention_weight_max": 38.01506423950195,
      "activations/layer21_attention_weight_min": -23.784202575683594,
      "activations/layer22_attention_weight_max": 34.03598403930664,
      "activations/layer22_attention_weight_min": -26.129629135131836,
      "activations/layer23_attention_weight_max": 41.36455535888672,
      "activations/layer23_attention_weight_min": -24.680049896240234,
      "activations/layer2_attention_weight_max": 31.414226531982422,
      "activations/layer2_attention_weight_min": -29.4378662109375,
      "activations/layer3_attention_weight_max": 91.04939270019531,
      "activations/layer3_attention_weight_min": -95.50946807861328,
      "activations/layer4_attention_weight_max": 104.10262298583984,
      "activations/layer4_attention_weight_min": -99.1751708984375,
      "activations/layer5_attention_weight_max": 80.06196594238281,
      "activations/layer5_attention_weight_min": -83.56538391113281,
      "activations/layer6_attention_weight_max": 58.66763687133789,
      "activations/layer6_attention_weight_min": -55.61665344238281,
      "activations/layer7_attention_weight_max": 88.34013366699219,
      "activations/layer7_attention_weight_min": -79.19912719726562,
      "activations/layer8_attention_weight_max": 48.29301834106445,
      "activations/layer8_attention_weight_min": -48.97214889526367,
      "activations/layer9_attention_weight_max": 56.454803466796875,
      "activations/layer9_attention_weight_min": -52.99341583251953,
      "epoch": 9.84,
      "learning_rate": 8.740454545454546e-05,
      "loss": 2.8005,
      "step": 169400
    },
    {
      "activations/layer0_attention_weight_max": 14.682859420776367,
      "activations/layer0_attention_weight_min": -13.936257362365723,
      "activations/layer10_attention_weight_max": 37.822139739990234,
      "activations/layer10_attention_weight_min": -38.397544860839844,
      "activations/layer11_attention_weight_max": 32.45716857910156,
      "activations/layer11_attention_weight_min": -33.373783111572266,
      "activations/layer12_attention_weight_max": 23.60149574279785,
      "activations/layer12_attention_weight_min": -23.097562789916992,
      "activations/layer13_attention_weight_max": 40.470489501953125,
      "activations/layer13_attention_weight_min": -31.789806365966797,
      "activations/layer14_attention_weight_max": 36.72734069824219,
      "activations/layer14_attention_weight_min": -31.669034957885742,
      "activations/layer15_attention_weight_max": 33.19204330444336,
      "activations/layer15_attention_weight_min": -28.433420181274414,
      "activations/layer16_attention_weight_max": 31.441951751708984,
      "activations/layer16_attention_weight_min": -28.410871505737305,
      "activations/layer17_attention_weight_max": 50.275848388671875,
      "activations/layer17_attention_weight_min": -43.64744567871094,
      "activations/layer18_attention_weight_max": 44.96833419799805,
      "activations/layer18_attention_weight_min": -37.46389389038086,
      "activations/layer19_attention_weight_max": 25.373788833618164,
      "activations/layer19_attention_weight_min": -20.554412841796875,
      "activations/layer1_attention_weight_max": 18.754980087280273,
      "activations/layer1_attention_weight_min": -13.55178451538086,
      "activations/layer20_attention_weight_max": 21.420589447021484,
      "activations/layer20_attention_weight_min": -21.287561416625977,
      "activations/layer21_attention_weight_max": 38.851112365722656,
      "activations/layer21_attention_weight_min": -25.157907485961914,
      "activations/layer22_attention_weight_max": 32.38190841674805,
      "activations/layer22_attention_weight_min": -31.58622932434082,
      "activations/layer23_attention_weight_max": 38.873931884765625,
      "activations/layer23_attention_weight_min": -23.97566795349121,
      "activations/layer2_attention_weight_max": 30.119518280029297,
      "activations/layer2_attention_weight_min": -30.69972038269043,
      "activations/layer3_attention_weight_max": 88.69310760498047,
      "activations/layer3_attention_weight_min": -89.35733795166016,
      "activations/layer4_attention_weight_max": 98.64241027832031,
      "activations/layer4_attention_weight_min": -96.80803680419922,
      "activations/layer5_attention_weight_max": 78.89082336425781,
      "activations/layer5_attention_weight_min": -80.68045043945312,
      "activations/layer6_attention_weight_max": 54.38214874267578,
      "activations/layer6_attention_weight_min": -52.41905975341797,
      "activations/layer7_attention_weight_max": 84.83690643310547,
      "activations/layer7_attention_weight_min": -78.53067779541016,
      "activations/layer8_attention_weight_max": 46.34031677246094,
      "activations/layer8_attention_weight_min": -48.6351432800293,
      "activations/layer9_attention_weight_max": 48.841880798339844,
      "activations/layer9_attention_weight_min": -49.31079864501953,
      "epoch": 9.85,
      "learning_rate": 8.738560606060604e-05,
      "loss": 2.8081,
      "step": 169450
    },
    {
      "activations/layer0_attention_weight_max": 16.198673248291016,
      "activations/layer0_attention_weight_min": -14.137860298156738,
      "activations/layer10_attention_weight_max": 41.77817916870117,
      "activations/layer10_attention_weight_min": -39.16429138183594,
      "activations/layer11_attention_weight_max": 37.19092559814453,
      "activations/layer11_attention_weight_min": -32.99933624267578,
      "activations/layer12_attention_weight_max": 18.21653175354004,
      "activations/layer12_attention_weight_min": -23.873186111450195,
      "activations/layer13_attention_weight_max": 43.968997955322266,
      "activations/layer13_attention_weight_min": -30.64922332763672,
      "activations/layer14_attention_weight_max": 35.85456848144531,
      "activations/layer14_attention_weight_min": -28.700407028198242,
      "activations/layer15_attention_weight_max": 35.566131591796875,
      "activations/layer15_attention_weight_min": -29.033626556396484,
      "activations/layer16_attention_weight_max": 31.63323974609375,
      "activations/layer16_attention_weight_min": -28.88547134399414,
      "activations/layer17_attention_weight_max": 57.01751708984375,
      "activations/layer17_attention_weight_min": -45.993568420410156,
      "activations/layer18_attention_weight_max": 48.19585037231445,
      "activations/layer18_attention_weight_min": -38.84989547729492,
      "activations/layer19_attention_weight_max": 22.135379791259766,
      "activations/layer19_attention_weight_min": -21.971433639526367,
      "activations/layer1_attention_weight_max": 16.327320098876953,
      "activations/layer1_attention_weight_min": -16.056230545043945,
      "activations/layer20_attention_weight_max": 22.1760196685791,
      "activations/layer20_attention_weight_min": -21.10690689086914,
      "activations/layer21_attention_weight_max": 46.25083923339844,
      "activations/layer21_attention_weight_min": -25.030338287353516,
      "activations/layer22_attention_weight_max": 30.16482162475586,
      "activations/layer22_attention_weight_min": -23.87908363342285,
      "activations/layer23_attention_weight_max": 41.9423713684082,
      "activations/layer23_attention_weight_min": -21.478870391845703,
      "activations/layer2_attention_weight_max": 35.639984130859375,
      "activations/layer2_attention_weight_min": -33.960548400878906,
      "activations/layer3_attention_weight_max": 91.26664733886719,
      "activations/layer3_attention_weight_min": -97.01509857177734,
      "activations/layer4_attention_weight_max": 104.47759246826172,
      "activations/layer4_attention_weight_min": -99.27265930175781,
      "activations/layer5_attention_weight_max": 79.8475112915039,
      "activations/layer5_attention_weight_min": -81.26271057128906,
      "activations/layer6_attention_weight_max": 59.33259963989258,
      "activations/layer6_attention_weight_min": -53.63599395751953,
      "activations/layer7_attention_weight_max": 77.8193588256836,
      "activations/layer7_attention_weight_min": -74.44763946533203,
      "activations/layer8_attention_weight_max": 51.42121505737305,
      "activations/layer8_attention_weight_min": -47.89562225341797,
      "activations/layer9_attention_weight_max": 51.738197326660156,
      "activations/layer9_attention_weight_min": -47.76808166503906,
      "epoch": 9.85,
      "learning_rate": 8.736666666666666e-05,
      "loss": 2.7879,
      "step": 169500
    },
    {
      "activations/layer0_attention_weight_max": 15.459792137145996,
      "activations/layer0_attention_weight_min": -12.999417304992676,
      "activations/layer10_attention_weight_max": 37.2288932800293,
      "activations/layer10_attention_weight_min": -36.6180305480957,
      "activations/layer11_attention_weight_max": 31.765579223632812,
      "activations/layer11_attention_weight_min": -32.91065216064453,
      "activations/layer12_attention_weight_max": 19.636077880859375,
      "activations/layer12_attention_weight_min": -24.689359664916992,
      "activations/layer13_attention_weight_max": 41.64176559448242,
      "activations/layer13_attention_weight_min": -30.68459701538086,
      "activations/layer14_attention_weight_max": 39.474700927734375,
      "activations/layer14_attention_weight_min": -30.63214683532715,
      "activations/layer15_attention_weight_max": 33.36359786987305,
      "activations/layer15_attention_weight_min": -30.770719528198242,
      "activations/layer16_attention_weight_max": 31.381196975708008,
      "activations/layer16_attention_weight_min": -28.33402442932129,
      "activations/layer17_attention_weight_max": 56.589962005615234,
      "activations/layer17_attention_weight_min": -42.61910629272461,
      "activations/layer18_attention_weight_max": 53.05366134643555,
      "activations/layer18_attention_weight_min": -38.14006423950195,
      "activations/layer19_attention_weight_max": 23.87038803100586,
      "activations/layer19_attention_weight_min": -22.071941375732422,
      "activations/layer1_attention_weight_max": 17.029876708984375,
      "activations/layer1_attention_weight_min": -16.76991081237793,
      "activations/layer20_attention_weight_max": 26.361345291137695,
      "activations/layer20_attention_weight_min": -21.269428253173828,
      "activations/layer21_attention_weight_max": 42.94093704223633,
      "activations/layer21_attention_weight_min": -21.218475341796875,
      "activations/layer22_attention_weight_max": 34.63435363769531,
      "activations/layer22_attention_weight_min": -26.104511260986328,
      "activations/layer23_attention_weight_max": 40.82177734375,
      "activations/layer23_attention_weight_min": -22.772306442260742,
      "activations/layer2_attention_weight_max": 33.73841094970703,
      "activations/layer2_attention_weight_min": -30.396453857421875,
      "activations/layer3_attention_weight_max": 90.51653289794922,
      "activations/layer3_attention_weight_min": -91.69530487060547,
      "activations/layer4_attention_weight_max": 99.6173095703125,
      "activations/layer4_attention_weight_min": -93.17727661132812,
      "activations/layer5_attention_weight_max": 80.1437759399414,
      "activations/layer5_attention_weight_min": -77.83173370361328,
      "activations/layer6_attention_weight_max": 58.53085708618164,
      "activations/layer6_attention_weight_min": -54.20231628417969,
      "activations/layer7_attention_weight_max": 82.14258575439453,
      "activations/layer7_attention_weight_min": -76.19979858398438,
      "activations/layer8_attention_weight_max": 46.33993911743164,
      "activations/layer8_attention_weight_min": -49.121829986572266,
      "activations/layer9_attention_weight_max": 49.155818939208984,
      "activations/layer9_attention_weight_min": -50.28007125854492,
      "epoch": 9.85,
      "learning_rate": 8.734772727272727e-05,
      "loss": 2.8026,
      "step": 169550
    },
    {
      "activations/layer0_attention_weight_max": 15.794861793518066,
      "activations/layer0_attention_weight_min": -13.320182800292969,
      "activations/layer10_attention_weight_max": 38.446136474609375,
      "activations/layer10_attention_weight_min": -35.891395568847656,
      "activations/layer11_attention_weight_max": 35.501564025878906,
      "activations/layer11_attention_weight_min": -32.75672912597656,
      "activations/layer12_attention_weight_max": 19.862098693847656,
      "activations/layer12_attention_weight_min": -25.19809913635254,
      "activations/layer13_attention_weight_max": 36.81739807128906,
      "activations/layer13_attention_weight_min": -32.398956298828125,
      "activations/layer14_attention_weight_max": 38.64204025268555,
      "activations/layer14_attention_weight_min": -31.868558883666992,
      "activations/layer15_attention_weight_max": 32.51002883911133,
      "activations/layer15_attention_weight_min": -28.507652282714844,
      "activations/layer16_attention_weight_max": 31.116682052612305,
      "activations/layer16_attention_weight_min": -29.11361312866211,
      "activations/layer17_attention_weight_max": 52.141212463378906,
      "activations/layer17_attention_weight_min": -42.813621520996094,
      "activations/layer18_attention_weight_max": 50.432090759277344,
      "activations/layer18_attention_weight_min": -37.8082389831543,
      "activations/layer19_attention_weight_max": 21.931610107421875,
      "activations/layer19_attention_weight_min": -20.226261138916016,
      "activations/layer1_attention_weight_max": 17.267202377319336,
      "activations/layer1_attention_weight_min": -15.739516258239746,
      "activations/layer20_attention_weight_max": 21.868581771850586,
      "activations/layer20_attention_weight_min": -22.66869354248047,
      "activations/layer21_attention_weight_max": 35.14995193481445,
      "activations/layer21_attention_weight_min": -23.2163028717041,
      "activations/layer22_attention_weight_max": 33.57147979736328,
      "activations/layer22_attention_weight_min": -23.756534576416016,
      "activations/layer23_attention_weight_max": 38.47412109375,
      "activations/layer23_attention_weight_min": -25.279958724975586,
      "activations/layer2_attention_weight_max": 32.90998077392578,
      "activations/layer2_attention_weight_min": -30.921384811401367,
      "activations/layer3_attention_weight_max": 92.00674438476562,
      "activations/layer3_attention_weight_min": -90.51283264160156,
      "activations/layer4_attention_weight_max": 101.12627410888672,
      "activations/layer4_attention_weight_min": -98.13726043701172,
      "activations/layer5_attention_weight_max": 83.33384704589844,
      "activations/layer5_attention_weight_min": -79.113037109375,
      "activations/layer6_attention_weight_max": 55.41029739379883,
      "activations/layer6_attention_weight_min": -52.97626876831055,
      "activations/layer7_attention_weight_max": 79.19034576416016,
      "activations/layer7_attention_weight_min": -77.8649673461914,
      "activations/layer8_attention_weight_max": 46.11825942993164,
      "activations/layer8_attention_weight_min": -45.683807373046875,
      "activations/layer9_attention_weight_max": 52.955745697021484,
      "activations/layer9_attention_weight_min": -50.68832015991211,
      "epoch": 9.85,
      "learning_rate": 8.732878787878786e-05,
      "loss": 2.7873,
      "step": 169600
    },
    {
      "activations/layer0_attention_weight_max": 15.64712905883789,
      "activations/layer0_attention_weight_min": -14.10781478881836,
      "activations/layer10_attention_weight_max": 37.367271423339844,
      "activations/layer10_attention_weight_min": -34.84210968017578,
      "activations/layer11_attention_weight_max": 30.297908782958984,
      "activations/layer11_attention_weight_min": -31.525524139404297,
      "activations/layer12_attention_weight_max": 18.61122703552246,
      "activations/layer12_attention_weight_min": -26.1766414642334,
      "activations/layer13_attention_weight_max": 40.783416748046875,
      "activations/layer13_attention_weight_min": -31.436983108520508,
      "activations/layer14_attention_weight_max": 38.15670394897461,
      "activations/layer14_attention_weight_min": -32.054725646972656,
      "activations/layer15_attention_weight_max": 35.9411506652832,
      "activations/layer15_attention_weight_min": -29.479965209960938,
      "activations/layer16_attention_weight_max": 31.767772674560547,
      "activations/layer16_attention_weight_min": -28.59626579284668,
      "activations/layer17_attention_weight_max": 50.44588088989258,
      "activations/layer17_attention_weight_min": -44.02323913574219,
      "activations/layer18_attention_weight_max": 48.1407585144043,
      "activations/layer18_attention_weight_min": -36.90373611450195,
      "activations/layer19_attention_weight_max": 23.2106990814209,
      "activations/layer19_attention_weight_min": -22.251888275146484,
      "activations/layer1_attention_weight_max": 17.453107833862305,
      "activations/layer1_attention_weight_min": -14.999845504760742,
      "activations/layer20_attention_weight_max": 23.384233474731445,
      "activations/layer20_attention_weight_min": -20.841346740722656,
      "activations/layer21_attention_weight_max": 39.026344299316406,
      "activations/layer21_attention_weight_min": -24.110729217529297,
      "activations/layer22_attention_weight_max": 32.11671447753906,
      "activations/layer22_attention_weight_min": -25.760753631591797,
      "activations/layer23_attention_weight_max": 41.596309661865234,
      "activations/layer23_attention_weight_min": -27.404464721679688,
      "activations/layer2_attention_weight_max": 30.5479793548584,
      "activations/layer2_attention_weight_min": -29.713239669799805,
      "activations/layer3_attention_weight_max": 84.63335418701172,
      "activations/layer3_attention_weight_min": -89.81181335449219,
      "activations/layer4_attention_weight_max": 94.04801940917969,
      "activations/layer4_attention_weight_min": -87.65144348144531,
      "activations/layer5_attention_weight_max": 74.08100891113281,
      "activations/layer5_attention_weight_min": -79.68525695800781,
      "activations/layer6_attention_weight_max": 52.125179290771484,
      "activations/layer6_attention_weight_min": -49.95368194580078,
      "activations/layer7_attention_weight_max": 72.73442840576172,
      "activations/layer7_attention_weight_min": -72.4292221069336,
      "activations/layer8_attention_weight_max": 43.264835357666016,
      "activations/layer8_attention_weight_min": -46.270816802978516,
      "activations/layer9_attention_weight_max": 46.05012130737305,
      "activations/layer9_attention_weight_min": -46.78419494628906,
      "epoch": 9.86,
      "learning_rate": 8.730984848484848e-05,
      "loss": 2.7881,
      "step": 169650
    },
    {
      "activations/layer0_attention_weight_max": 15.59644889831543,
      "activations/layer0_attention_weight_min": -14.131717681884766,
      "activations/layer10_attention_weight_max": 36.42692947387695,
      "activations/layer10_attention_weight_min": -35.77766799926758,
      "activations/layer11_attention_weight_max": 33.68437957763672,
      "activations/layer11_attention_weight_min": -30.846548080444336,
      "activations/layer12_attention_weight_max": 18.903148651123047,
      "activations/layer12_attention_weight_min": -29.779664993286133,
      "activations/layer13_attention_weight_max": 43.28573226928711,
      "activations/layer13_attention_weight_min": -32.20813751220703,
      "activations/layer14_attention_weight_max": 42.85264587402344,
      "activations/layer14_attention_weight_min": -30.754098892211914,
      "activations/layer15_attention_weight_max": 39.30048751831055,
      "activations/layer15_attention_weight_min": -28.718780517578125,
      "activations/layer16_attention_weight_max": 31.364294052124023,
      "activations/layer16_attention_weight_min": -28.706085205078125,
      "activations/layer17_attention_weight_max": 56.6641845703125,
      "activations/layer17_attention_weight_min": -43.78053665161133,
      "activations/layer18_attention_weight_max": 51.720157623291016,
      "activations/layer18_attention_weight_min": -36.89865493774414,
      "activations/layer19_attention_weight_max": 22.144577026367188,
      "activations/layer19_attention_weight_min": -19.70470428466797,
      "activations/layer1_attention_weight_max": 16.937782287597656,
      "activations/layer1_attention_weight_min": -14.895103454589844,
      "activations/layer20_attention_weight_max": 23.314189910888672,
      "activations/layer20_attention_weight_min": -20.583871841430664,
      "activations/layer21_attention_weight_max": 35.161285400390625,
      "activations/layer21_attention_weight_min": -21.074636459350586,
      "activations/layer22_attention_weight_max": 32.0344123840332,
      "activations/layer22_attention_weight_min": -23.513967514038086,
      "activations/layer23_attention_weight_max": 37.99762725830078,
      "activations/layer23_attention_weight_min": -22.993728637695312,
      "activations/layer2_attention_weight_max": 32.7794303894043,
      "activations/layer2_attention_weight_min": -30.330989837646484,
      "activations/layer3_attention_weight_max": 94.97834777832031,
      "activations/layer3_attention_weight_min": -93.91849517822266,
      "activations/layer4_attention_weight_max": 102.80278778076172,
      "activations/layer4_attention_weight_min": -99.43275451660156,
      "activations/layer5_attention_weight_max": 78.18568420410156,
      "activations/layer5_attention_weight_min": -77.77227783203125,
      "activations/layer6_attention_weight_max": 53.603538513183594,
      "activations/layer6_attention_weight_min": -55.54713439941406,
      "activations/layer7_attention_weight_max": 76.5175552368164,
      "activations/layer7_attention_weight_min": -75.03936767578125,
      "activations/layer8_attention_weight_max": 44.71053695678711,
      "activations/layer8_attention_weight_min": -50.28356170654297,
      "activations/layer9_attention_weight_max": 52.8891487121582,
      "activations/layer9_attention_weight_min": -48.682865142822266,
      "epoch": 9.86,
      "learning_rate": 8.729090909090909e-05,
      "loss": 2.8087,
      "step": 169700
    },
    {
      "activations/layer0_attention_weight_max": 15.955860137939453,
      "activations/layer0_attention_weight_min": -13.793201446533203,
      "activations/layer10_attention_weight_max": 36.80961608886719,
      "activations/layer10_attention_weight_min": -36.31038284301758,
      "activations/layer11_attention_weight_max": 31.840444564819336,
      "activations/layer11_attention_weight_min": -32.28228759765625,
      "activations/layer12_attention_weight_max": 18.002628326416016,
      "activations/layer12_attention_weight_min": -25.113174438476562,
      "activations/layer13_attention_weight_max": 48.25164031982422,
      "activations/layer13_attention_weight_min": -33.80520248413086,
      "activations/layer14_attention_weight_max": 36.944427490234375,
      "activations/layer14_attention_weight_min": -29.640396118164062,
      "activations/layer15_attention_weight_max": 34.945674896240234,
      "activations/layer15_attention_weight_min": -29.414146423339844,
      "activations/layer16_attention_weight_max": 35.07017517089844,
      "activations/layer16_attention_weight_min": -27.121755599975586,
      "activations/layer17_attention_weight_max": 53.13825607299805,
      "activations/layer17_attention_weight_min": -39.660884857177734,
      "activations/layer18_attention_weight_max": 48.26628875732422,
      "activations/layer18_attention_weight_min": -34.62656021118164,
      "activations/layer19_attention_weight_max": 23.9879150390625,
      "activations/layer19_attention_weight_min": -20.146116256713867,
      "activations/layer1_attention_weight_max": 16.24437713623047,
      "activations/layer1_attention_weight_min": -15.94110107421875,
      "activations/layer20_attention_weight_max": 21.948619842529297,
      "activations/layer20_attention_weight_min": -19.985963821411133,
      "activations/layer21_attention_weight_max": 37.347625732421875,
      "activations/layer21_attention_weight_min": -23.0505428314209,
      "activations/layer22_attention_weight_max": 31.818452835083008,
      "activations/layer22_attention_weight_min": -25.22524070739746,
      "activations/layer23_attention_weight_max": 37.647308349609375,
      "activations/layer23_attention_weight_min": -22.520605087280273,
      "activations/layer2_attention_weight_max": 31.857877731323242,
      "activations/layer2_attention_weight_min": -29.62865447998047,
      "activations/layer3_attention_weight_max": 91.47393798828125,
      "activations/layer3_attention_weight_min": -92.71852111816406,
      "activations/layer4_attention_weight_max": 101.22248077392578,
      "activations/layer4_attention_weight_min": -98.50054168701172,
      "activations/layer5_attention_weight_max": 77.27125549316406,
      "activations/layer5_attention_weight_min": -80.52923583984375,
      "activations/layer6_attention_weight_max": 56.08161926269531,
      "activations/layer6_attention_weight_min": -55.13818359375,
      "activations/layer7_attention_weight_max": 80.16556549072266,
      "activations/layer7_attention_weight_min": -78.23251342773438,
      "activations/layer8_attention_weight_max": 44.0687141418457,
      "activations/layer8_attention_weight_min": -47.21211624145508,
      "activations/layer9_attention_weight_max": 47.5346565246582,
      "activations/layer9_attention_weight_min": -47.854007720947266,
      "epoch": 9.86,
      "learning_rate": 8.727196969696968e-05,
      "loss": 2.7945,
      "step": 169750
    },
    {
      "activations/layer0_attention_weight_max": 16.154088973999023,
      "activations/layer0_attention_weight_min": -13.862386703491211,
      "activations/layer10_attention_weight_max": 39.430397033691406,
      "activations/layer10_attention_weight_min": -36.606754302978516,
      "activations/layer11_attention_weight_max": 34.3741569519043,
      "activations/layer11_attention_weight_min": -32.70749282836914,
      "activations/layer12_attention_weight_max": 19.041933059692383,
      "activations/layer12_attention_weight_min": -25.6815242767334,
      "activations/layer13_attention_weight_max": 43.73973083496094,
      "activations/layer13_attention_weight_min": -31.06831932067871,
      "activations/layer14_attention_weight_max": 36.27215576171875,
      "activations/layer14_attention_weight_min": -31.442564010620117,
      "activations/layer15_attention_weight_max": 34.799659729003906,
      "activations/layer15_attention_weight_min": -30.69285011291504,
      "activations/layer16_attention_weight_max": 33.66001510620117,
      "activations/layer16_attention_weight_min": -29.48349380493164,
      "activations/layer17_attention_weight_max": 55.61446762084961,
      "activations/layer17_attention_weight_min": -48.33485794067383,
      "activations/layer18_attention_weight_max": 48.40463638305664,
      "activations/layer18_attention_weight_min": -41.1031608581543,
      "activations/layer19_attention_weight_max": 21.779296875,
      "activations/layer19_attention_weight_min": -23.69232749938965,
      "activations/layer1_attention_weight_max": 17.61224937438965,
      "activations/layer1_attention_weight_min": -16.236469268798828,
      "activations/layer20_attention_weight_max": 23.499181747436523,
      "activations/layer20_attention_weight_min": -22.20574188232422,
      "activations/layer21_attention_weight_max": 38.30413055419922,
      "activations/layer21_attention_weight_min": -22.587718963623047,
      "activations/layer22_attention_weight_max": 31.76068687438965,
      "activations/layer22_attention_weight_min": -25.102460861206055,
      "activations/layer23_attention_weight_max": 40.13409423828125,
      "activations/layer23_attention_weight_min": -21.65582275390625,
      "activations/layer2_attention_weight_max": 33.07319259643555,
      "activations/layer2_attention_weight_min": -31.244586944580078,
      "activations/layer3_attention_weight_max": 96.81904602050781,
      "activations/layer3_attention_weight_min": -99.37317657470703,
      "activations/layer4_attention_weight_max": 106.4356689453125,
      "activations/layer4_attention_weight_min": -101.37447357177734,
      "activations/layer5_attention_weight_max": 78.34352111816406,
      "activations/layer5_attention_weight_min": -78.35569763183594,
      "activations/layer6_attention_weight_max": 54.14954376220703,
      "activations/layer6_attention_weight_min": -55.89101028442383,
      "activations/layer7_attention_weight_max": 81.54379272460938,
      "activations/layer7_attention_weight_min": -74.26898193359375,
      "activations/layer8_attention_weight_max": 48.34406661987305,
      "activations/layer8_attention_weight_min": -48.19580841064453,
      "activations/layer9_attention_weight_max": 57.689300537109375,
      "activations/layer9_attention_weight_min": -50.67215347290039,
      "epoch": 9.87,
      "learning_rate": 8.72530303030303e-05,
      "loss": 2.7844,
      "step": 169800
    },
    {
      "activations/layer0_attention_weight_max": 16.15873146057129,
      "activations/layer0_attention_weight_min": -14.391497611999512,
      "activations/layer10_attention_weight_max": 39.93975067138672,
      "activations/layer10_attention_weight_min": -37.84971237182617,
      "activations/layer11_attention_weight_max": 34.07942199707031,
      "activations/layer11_attention_weight_min": -36.701026916503906,
      "activations/layer12_attention_weight_max": 24.66169548034668,
      "activations/layer12_attention_weight_min": -27.01414680480957,
      "activations/layer13_attention_weight_max": 46.32938766479492,
      "activations/layer13_attention_weight_min": -32.27627944946289,
      "activations/layer14_attention_weight_max": 40.429962158203125,
      "activations/layer14_attention_weight_min": -30.393278121948242,
      "activations/layer15_attention_weight_max": 38.11173629760742,
      "activations/layer15_attention_weight_min": -29.03853416442871,
      "activations/layer16_attention_weight_max": 33.29340744018555,
      "activations/layer16_attention_weight_min": -29.925159454345703,
      "activations/layer17_attention_weight_max": 55.549591064453125,
      "activations/layer17_attention_weight_min": -44.80562210083008,
      "activations/layer18_attention_weight_max": 49.2999382019043,
      "activations/layer18_attention_weight_min": -36.576412200927734,
      "activations/layer19_attention_weight_max": 23.465118408203125,
      "activations/layer19_attention_weight_min": -20.63312339782715,
      "activations/layer1_attention_weight_max": 16.381793975830078,
      "activations/layer1_attention_weight_min": -17.54450225830078,
      "activations/layer20_attention_weight_max": 29.694700241088867,
      "activations/layer20_attention_weight_min": -20.587926864624023,
      "activations/layer21_attention_weight_max": 38.11910629272461,
      "activations/layer21_attention_weight_min": -23.425493240356445,
      "activations/layer22_attention_weight_max": 31.73122787475586,
      "activations/layer22_attention_weight_min": -25.725515365600586,
      "activations/layer23_attention_weight_max": 38.03326416015625,
      "activations/layer23_attention_weight_min": -21.935901641845703,
      "activations/layer2_attention_weight_max": 34.03478240966797,
      "activations/layer2_attention_weight_min": -32.454345703125,
      "activations/layer3_attention_weight_max": 91.83931732177734,
      "activations/layer3_attention_weight_min": -96.80934143066406,
      "activations/layer4_attention_weight_max": 102.6125717163086,
      "activations/layer4_attention_weight_min": -100.76426696777344,
      "activations/layer5_attention_weight_max": 78.16719818115234,
      "activations/layer5_attention_weight_min": -81.78436279296875,
      "activations/layer6_attention_weight_max": 54.66862869262695,
      "activations/layer6_attention_weight_min": -55.922672271728516,
      "activations/layer7_attention_weight_max": 81.45680236816406,
      "activations/layer7_attention_weight_min": -76.22662353515625,
      "activations/layer8_attention_weight_max": 48.53392028808594,
      "activations/layer8_attention_weight_min": -48.15505599975586,
      "activations/layer9_attention_weight_max": 50.383399963378906,
      "activations/layer9_attention_weight_min": -49.172706604003906,
      "epoch": 9.87,
      "learning_rate": 8.723409090909091e-05,
      "loss": 2.797,
      "step": 169850
    },
    {
      "activations/layer0_attention_weight_max": 16.51387596130371,
      "activations/layer0_attention_weight_min": -13.984625816345215,
      "activations/layer10_attention_weight_max": 39.97344207763672,
      "activations/layer10_attention_weight_min": -36.64502716064453,
      "activations/layer11_attention_weight_max": 34.8929328918457,
      "activations/layer11_attention_weight_min": -33.81241989135742,
      "activations/layer12_attention_weight_max": 18.299711227416992,
      "activations/layer12_attention_weight_min": -24.93622398376465,
      "activations/layer13_attention_weight_max": 44.79574966430664,
      "activations/layer13_attention_weight_min": -33.32859802246094,
      "activations/layer14_attention_weight_max": 35.63567352294922,
      "activations/layer14_attention_weight_min": -30.610855102539062,
      "activations/layer15_attention_weight_max": 36.18622970581055,
      "activations/layer15_attention_weight_min": -29.33539390563965,
      "activations/layer16_attention_weight_max": 30.576236724853516,
      "activations/layer16_attention_weight_min": -27.84477424621582,
      "activations/layer17_attention_weight_max": 50.48927688598633,
      "activations/layer17_attention_weight_min": -41.54450607299805,
      "activations/layer18_attention_weight_max": 45.01664352416992,
      "activations/layer18_attention_weight_min": -36.41367721557617,
      "activations/layer19_attention_weight_max": 20.786867141723633,
      "activations/layer19_attention_weight_min": -21.41792106628418,
      "activations/layer1_attention_weight_max": 15.761445045471191,
      "activations/layer1_attention_weight_min": -15.277023315429688,
      "activations/layer20_attention_weight_max": 23.287256240844727,
      "activations/layer20_attention_weight_min": -22.84160804748535,
      "activations/layer21_attention_weight_max": 34.941246032714844,
      "activations/layer21_attention_weight_min": -24.299161911010742,
      "activations/layer22_attention_weight_max": 33.81623077392578,
      "activations/layer22_attention_weight_min": -25.7985782623291,
      "activations/layer23_attention_weight_max": 34.772438049316406,
      "activations/layer23_attention_weight_min": -24.206098556518555,
      "activations/layer2_attention_weight_max": 32.34471893310547,
      "activations/layer2_attention_weight_min": -30.780406951904297,
      "activations/layer3_attention_weight_max": 91.66371154785156,
      "activations/layer3_attention_weight_min": -92.0708999633789,
      "activations/layer4_attention_weight_max": 100.90050506591797,
      "activations/layer4_attention_weight_min": -94.58570861816406,
      "activations/layer5_attention_weight_max": 80.95600891113281,
      "activations/layer5_attention_weight_min": -77.38275909423828,
      "activations/layer6_attention_weight_max": 56.826995849609375,
      "activations/layer6_attention_weight_min": -53.49895477294922,
      "activations/layer7_attention_weight_max": 82.71649169921875,
      "activations/layer7_attention_weight_min": -71.71353149414062,
      "activations/layer8_attention_weight_max": 47.0477409362793,
      "activations/layer8_attention_weight_min": -48.31938171386719,
      "activations/layer9_attention_weight_max": 50.68061065673828,
      "activations/layer9_attention_weight_min": -49.3771858215332,
      "epoch": 9.87,
      "learning_rate": 8.721515151515151e-05,
      "loss": 2.7932,
      "step": 169900
    },
    {
      "activations/layer0_attention_weight_max": 16.030010223388672,
      "activations/layer0_attention_weight_min": -13.88153076171875,
      "activations/layer10_attention_weight_max": 36.92826461791992,
      "activations/layer10_attention_weight_min": -36.750030517578125,
      "activations/layer11_attention_weight_max": 31.3657283782959,
      "activations/layer11_attention_weight_min": -34.77587127685547,
      "activations/layer12_attention_weight_max": 19.499835968017578,
      "activations/layer12_attention_weight_min": -22.90003204345703,
      "activations/layer13_attention_weight_max": 38.90965270996094,
      "activations/layer13_attention_weight_min": -29.951356887817383,
      "activations/layer14_attention_weight_max": 38.93751525878906,
      "activations/layer14_attention_weight_min": -30.580842971801758,
      "activations/layer15_attention_weight_max": 33.076805114746094,
      "activations/layer15_attention_weight_min": -30.381921768188477,
      "activations/layer16_attention_weight_max": 30.838048934936523,
      "activations/layer16_attention_weight_min": -28.733638763427734,
      "activations/layer17_attention_weight_max": 53.718910217285156,
      "activations/layer17_attention_weight_min": -50.68135070800781,
      "activations/layer18_attention_weight_max": 50.02070236206055,
      "activations/layer18_attention_weight_min": -44.064727783203125,
      "activations/layer19_attention_weight_max": 23.25782585144043,
      "activations/layer19_attention_weight_min": -22.44342041015625,
      "activations/layer1_attention_weight_max": 16.705970764160156,
      "activations/layer1_attention_weight_min": -16.593467712402344,
      "activations/layer20_attention_weight_max": 23.94091033935547,
      "activations/layer20_attention_weight_min": -23.992889404296875,
      "activations/layer21_attention_weight_max": 41.17264938354492,
      "activations/layer21_attention_weight_min": -26.698537826538086,
      "activations/layer22_attention_weight_max": 30.902477264404297,
      "activations/layer22_attention_weight_min": -27.11020278930664,
      "activations/layer23_attention_weight_max": 39.608497619628906,
      "activations/layer23_attention_weight_min": -24.520709991455078,
      "activations/layer2_attention_weight_max": 32.53913116455078,
      "activations/layer2_attention_weight_min": -29.42243194580078,
      "activations/layer3_attention_weight_max": 90.47405242919922,
      "activations/layer3_attention_weight_min": -91.62957763671875,
      "activations/layer4_attention_weight_max": 101.3629379272461,
      "activations/layer4_attention_weight_min": -97.55944061279297,
      "activations/layer5_attention_weight_max": 77.03048706054688,
      "activations/layer5_attention_weight_min": -79.08812713623047,
      "activations/layer6_attention_weight_max": 54.45298385620117,
      "activations/layer6_attention_weight_min": -54.210575103759766,
      "activations/layer7_attention_weight_max": 77.30323791503906,
      "activations/layer7_attention_weight_min": -75.27701568603516,
      "activations/layer8_attention_weight_max": 47.0691032409668,
      "activations/layer8_attention_weight_min": -47.98448944091797,
      "activations/layer9_attention_weight_max": 54.848915100097656,
      "activations/layer9_attention_weight_min": -48.77784729003906,
      "epoch": 9.88,
      "learning_rate": 8.719621212121211e-05,
      "loss": 2.793,
      "step": 169950
    },
    {
      "activations/layer0_attention_weight_max": 14.852356910705566,
      "activations/layer0_attention_weight_min": -14.105666160583496,
      "activations/layer10_attention_weight_max": 39.67844772338867,
      "activations/layer10_attention_weight_min": -35.91383743286133,
      "activations/layer11_attention_weight_max": 33.71335983276367,
      "activations/layer11_attention_weight_min": -33.56078338623047,
      "activations/layer12_attention_weight_max": 24.732065200805664,
      "activations/layer12_attention_weight_min": -22.798994064331055,
      "activations/layer13_attention_weight_max": 36.698875427246094,
      "activations/layer13_attention_weight_min": -28.750640869140625,
      "activations/layer14_attention_weight_max": 36.213653564453125,
      "activations/layer14_attention_weight_min": -30.70530128479004,
      "activations/layer15_attention_weight_max": 33.82078552246094,
      "activations/layer15_attention_weight_min": -30.015647888183594,
      "activations/layer16_attention_weight_max": 32.7189826965332,
      "activations/layer16_attention_weight_min": -28.446117401123047,
      "activations/layer17_attention_weight_max": 58.36789321899414,
      "activations/layer17_attention_weight_min": -45.107025146484375,
      "activations/layer18_attention_weight_max": 50.40137481689453,
      "activations/layer18_attention_weight_min": -38.13554000854492,
      "activations/layer19_attention_weight_max": 25.102048873901367,
      "activations/layer19_attention_weight_min": -22.857629776000977,
      "activations/layer1_attention_weight_max": 16.982173919677734,
      "activations/layer1_attention_weight_min": -15.770002365112305,
      "activations/layer20_attention_weight_max": 21.432268142700195,
      "activations/layer20_attention_weight_min": -22.557109832763672,
      "activations/layer21_attention_weight_max": 37.06859588623047,
      "activations/layer21_attention_weight_min": -26.581756591796875,
      "activations/layer22_attention_weight_max": 31.852739334106445,
      "activations/layer22_attention_weight_min": -25.255266189575195,
      "activations/layer23_attention_weight_max": 36.368446350097656,
      "activations/layer23_attention_weight_min": -24.096769332885742,
      "activations/layer2_attention_weight_max": 33.94095993041992,
      "activations/layer2_attention_weight_min": -31.840248107910156,
      "activations/layer3_attention_weight_max": 93.0294189453125,
      "activations/layer3_attention_weight_min": -94.84829711914062,
      "activations/layer4_attention_weight_max": 100.6982421875,
      "activations/layer4_attention_weight_min": -98.09294128417969,
      "activations/layer5_attention_weight_max": 80.02735137939453,
      "activations/layer5_attention_weight_min": -81.77462768554688,
      "activations/layer6_attention_weight_max": 57.5700798034668,
      "activations/layer6_attention_weight_min": -55.859588623046875,
      "activations/layer7_attention_weight_max": 82.85531616210938,
      "activations/layer7_attention_weight_min": -75.38729095458984,
      "activations/layer8_attention_weight_max": 46.294532775878906,
      "activations/layer8_attention_weight_min": -48.1528205871582,
      "activations/layer9_attention_weight_max": 51.15798568725586,
      "activations/layer9_attention_weight_min": -50.3143310546875,
      "epoch": 9.88,
      "learning_rate": 8.717727272727271e-05,
      "loss": 2.8014,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.544,
      "eval_samples_per_second": 502.575,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.544,
      "eval_openwebtext_samples_per_second": 502.575,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_wikitext_loss": 2.98046875,
      "eval_wikitext_ppl": 19.697047472120772,
      "eval_wikitext_runtime": 2.0185,
      "eval_wikitext_samples_per_second": 225.911,
      "step": 170000
    },
    {
      "epoch": 9.88,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.6073,
      "eval_lambada_samples_per_second": 506.802,
      "step": 170000
    },
    {
      "activations/layer0_attention_weight_max": 16.217361450195312,
      "activations/layer0_attention_weight_min": -13.546013832092285,
      "activations/layer10_attention_weight_max": 37.5911865234375,
      "activations/layer10_attention_weight_min": -36.1987190246582,
      "activations/layer11_attention_weight_max": 32.35166931152344,
      "activations/layer11_attention_weight_min": -32.3968391418457,
      "activations/layer12_attention_weight_max": 18.735607147216797,
      "activations/layer12_attention_weight_min": -24.99628257751465,
      "activations/layer13_attention_weight_max": 40.27893829345703,
      "activations/layer13_attention_weight_min": -29.06460952758789,
      "activations/layer14_attention_weight_max": 38.67232131958008,
      "activations/layer14_attention_weight_min": -33.51594924926758,
      "activations/layer15_attention_weight_max": 35.06340026855469,
      "activations/layer15_attention_weight_min": -32.04201889038086,
      "activations/layer16_attention_weight_max": 31.09636116027832,
      "activations/layer16_attention_weight_min": -30.147058486938477,
      "activations/layer17_attention_weight_max": 56.19468688964844,
      "activations/layer17_attention_weight_min": -49.463958740234375,
      "activations/layer18_attention_weight_max": 47.4748649597168,
      "activations/layer18_attention_weight_min": -42.02648162841797,
      "activations/layer19_attention_weight_max": 24.393844604492188,
      "activations/layer19_attention_weight_min": -24.430166244506836,
      "activations/layer1_attention_weight_max": 16.816865921020508,
      "activations/layer1_attention_weight_min": -15.81621265411377,
      "activations/layer20_attention_weight_max": 23.68581771850586,
      "activations/layer20_attention_weight_min": -21.973581314086914,
      "activations/layer21_attention_weight_max": 44.7824592590332,
      "activations/layer21_attention_weight_min": -27.859439849853516,
      "activations/layer22_attention_weight_max": 32.82512283325195,
      "activations/layer22_attention_weight_min": -27.479705810546875,
      "activations/layer23_attention_weight_max": 37.66962814331055,
      "activations/layer23_attention_weight_min": -24.227237701416016,
      "activations/layer2_attention_weight_max": 32.51690673828125,
      "activations/layer2_attention_weight_min": -31.31351089477539,
      "activations/layer3_attention_weight_max": 92.34917449951172,
      "activations/layer3_attention_weight_min": -92.6590347290039,
      "activations/layer4_attention_weight_max": 102.20073699951172,
      "activations/layer4_attention_weight_min": -97.68873596191406,
      "activations/layer5_attention_weight_max": 80.38752746582031,
      "activations/layer5_attention_weight_min": -78.3165512084961,
      "activations/layer6_attention_weight_max": 54.66789245605469,
      "activations/layer6_attention_weight_min": -52.600181579589844,
      "activations/layer7_attention_weight_max": 76.44620513916016,
      "activations/layer7_attention_weight_min": -78.1407241821289,
      "activations/layer8_attention_weight_max": 46.02138900756836,
      "activations/layer8_attention_weight_min": -45.12433624267578,
      "activations/layer9_attention_weight_max": 51.055519104003906,
      "activations/layer9_attention_weight_min": -49.34648132324219,
      "epoch": 9.88,
      "learning_rate": 8.715833333333333e-05,
      "loss": 2.7889,
      "step": 170050
    },
    {
      "activations/layer0_attention_weight_max": 15.754191398620605,
      "activations/layer0_attention_weight_min": -13.206046104431152,
      "activations/layer10_attention_weight_max": 41.31017303466797,
      "activations/layer10_attention_weight_min": -36.452430725097656,
      "activations/layer11_attention_weight_max": 36.18681335449219,
      "activations/layer11_attention_weight_min": -33.84097671508789,
      "activations/layer12_attention_weight_max": 20.57769203186035,
      "activations/layer12_attention_weight_min": -26.442567825317383,
      "activations/layer13_attention_weight_max": 40.776405334472656,
      "activations/layer13_attention_weight_min": -29.519277572631836,
      "activations/layer14_attention_weight_max": 41.53174591064453,
      "activations/layer14_attention_weight_min": -30.62411880493164,
      "activations/layer15_attention_weight_max": 38.00291442871094,
      "activations/layer15_attention_weight_min": -32.56587219238281,
      "activations/layer16_attention_weight_max": 41.03697967529297,
      "activations/layer16_attention_weight_min": -33.62715148925781,
      "activations/layer17_attention_weight_max": 66.8193588256836,
      "activations/layer17_attention_weight_min": -54.5380744934082,
      "activations/layer18_attention_weight_max": 62.38474655151367,
      "activations/layer18_attention_weight_min": -42.11396408081055,
      "activations/layer19_attention_weight_max": 25.271120071411133,
      "activations/layer19_attention_weight_min": -23.740962982177734,
      "activations/layer1_attention_weight_max": 16.664487838745117,
      "activations/layer1_attention_weight_min": -15.728805541992188,
      "activations/layer20_attention_weight_max": 24.645200729370117,
      "activations/layer20_attention_weight_min": -23.511381149291992,
      "activations/layer21_attention_weight_max": 38.77533721923828,
      "activations/layer21_attention_weight_min": -22.833892822265625,
      "activations/layer22_attention_weight_max": 33.15171432495117,
      "activations/layer22_attention_weight_min": -27.545673370361328,
      "activations/layer23_attention_weight_max": 38.41530990600586,
      "activations/layer23_attention_weight_min": -22.632442474365234,
      "activations/layer2_attention_weight_max": 30.470897674560547,
      "activations/layer2_attention_weight_min": -29.340560913085938,
      "activations/layer3_attention_weight_max": 89.55655670166016,
      "activations/layer3_attention_weight_min": -89.9662857055664,
      "activations/layer4_attention_weight_max": 99.35839080810547,
      "activations/layer4_attention_weight_min": -93.50434875488281,
      "activations/layer5_attention_weight_max": 75.93802642822266,
      "activations/layer5_attention_weight_min": -79.94657897949219,
      "activations/layer6_attention_weight_max": 54.72410202026367,
      "activations/layer6_attention_weight_min": -51.20030975341797,
      "activations/layer7_attention_weight_max": 77.34809875488281,
      "activations/layer7_attention_weight_min": -73.76309204101562,
      "activations/layer8_attention_weight_max": 45.71133804321289,
      "activations/layer8_attention_weight_min": -45.958961486816406,
      "activations/layer9_attention_weight_max": 50.72932434082031,
      "activations/layer9_attention_weight_min": -50.730613708496094,
      "epoch": 9.88,
      "learning_rate": 8.713939393939393e-05,
      "loss": 2.7983,
      "step": 170100
    },
    {
      "activations/layer0_attention_weight_max": 16.060091018676758,
      "activations/layer0_attention_weight_min": -13.962753295898438,
      "activations/layer10_attention_weight_max": 41.9382438659668,
      "activations/layer10_attention_weight_min": -39.665863037109375,
      "activations/layer11_attention_weight_max": 41.542327880859375,
      "activations/layer11_attention_weight_min": -35.805572509765625,
      "activations/layer12_attention_weight_max": 19.183198928833008,
      "activations/layer12_attention_weight_min": -29.806015014648438,
      "activations/layer13_attention_weight_max": 42.50202941894531,
      "activations/layer13_attention_weight_min": -35.542354583740234,
      "activations/layer14_attention_weight_max": 44.48874282836914,
      "activations/layer14_attention_weight_min": -31.165019989013672,
      "activations/layer15_attention_weight_max": 33.14261245727539,
      "activations/layer15_attention_weight_min": -29.047077178955078,
      "activations/layer16_attention_weight_max": 33.535099029541016,
      "activations/layer16_attention_weight_min": -26.820636749267578,
      "activations/layer17_attention_weight_max": 48.02171325683594,
      "activations/layer17_attention_weight_min": -43.52307891845703,
      "activations/layer18_attention_weight_max": 45.6601676940918,
      "activations/layer18_attention_weight_min": -37.70599365234375,
      "activations/layer19_attention_weight_max": 21.570402145385742,
      "activations/layer19_attention_weight_min": -21.396764755249023,
      "activations/layer1_attention_weight_max": 17.502235412597656,
      "activations/layer1_attention_weight_min": -16.036962509155273,
      "activations/layer20_attention_weight_max": 21.117006301879883,
      "activations/layer20_attention_weight_min": -20.92500114440918,
      "activations/layer21_attention_weight_max": 38.535945892333984,
      "activations/layer21_attention_weight_min": -25.311491012573242,
      "activations/layer22_attention_weight_max": 29.043149948120117,
      "activations/layer22_attention_weight_min": -25.48601531982422,
      "activations/layer23_attention_weight_max": 34.33094024658203,
      "activations/layer23_attention_weight_min": -24.600540161132812,
      "activations/layer2_attention_weight_max": 35.530662536621094,
      "activations/layer2_attention_weight_min": -31.259191513061523,
      "activations/layer3_attention_weight_max": 98.54048156738281,
      "activations/layer3_attention_weight_min": -93.88594818115234,
      "activations/layer4_attention_weight_max": 105.82970428466797,
      "activations/layer4_attention_weight_min": -96.7804183959961,
      "activations/layer5_attention_weight_max": 81.297607421875,
      "activations/layer5_attention_weight_min": -78.77225494384766,
      "activations/layer6_attention_weight_max": 53.319847106933594,
      "activations/layer6_attention_weight_min": -53.47850036621094,
      "activations/layer7_attention_weight_max": 86.52664947509766,
      "activations/layer7_attention_weight_min": -73.91421508789062,
      "activations/layer8_attention_weight_max": 55.235435485839844,
      "activations/layer8_attention_weight_min": -47.81568908691406,
      "activations/layer9_attention_weight_max": 63.61690139770508,
      "activations/layer9_attention_weight_min": -51.3713264465332,
      "epoch": 9.89,
      "learning_rate": 8.712045454545453e-05,
      "loss": 2.8004,
      "step": 170150
    },
    {
      "activations/layer0_attention_weight_max": 15.433277130126953,
      "activations/layer0_attention_weight_min": -13.69704532623291,
      "activations/layer10_attention_weight_max": 35.154876708984375,
      "activations/layer10_attention_weight_min": -34.422088623046875,
      "activations/layer11_attention_weight_max": 29.862815856933594,
      "activations/layer11_attention_weight_min": -31.09238052368164,
      "activations/layer12_attention_weight_max": 17.80156707763672,
      "activations/layer12_attention_weight_min": -26.2097110748291,
      "activations/layer13_attention_weight_max": 41.41673278808594,
      "activations/layer13_attention_weight_min": -30.24880027770996,
      "activations/layer14_attention_weight_max": 40.16459655761719,
      "activations/layer14_attention_weight_min": -30.281055450439453,
      "activations/layer15_attention_weight_max": 34.9364128112793,
      "activations/layer15_attention_weight_min": -28.232128143310547,
      "activations/layer16_attention_weight_max": 30.31805419921875,
      "activations/layer16_attention_weight_min": -28.35019874572754,
      "activations/layer17_attention_weight_max": 54.801856994628906,
      "activations/layer17_attention_weight_min": -42.39442825317383,
      "activations/layer18_attention_weight_max": 50.86201095581055,
      "activations/layer18_attention_weight_min": -35.938690185546875,
      "activations/layer19_attention_weight_max": 22.91927146911621,
      "activations/layer19_attention_weight_min": -19.926977157592773,
      "activations/layer1_attention_weight_max": 16.94449234008789,
      "activations/layer1_attention_weight_min": -16.134490966796875,
      "activations/layer20_attention_weight_max": 26.169050216674805,
      "activations/layer20_attention_weight_min": -19.947233200073242,
      "activations/layer21_attention_weight_max": 40.0889892578125,
      "activations/layer21_attention_weight_min": -23.40250587463379,
      "activations/layer22_attention_weight_max": 32.91408920288086,
      "activations/layer22_attention_weight_min": -24.276081085205078,
      "activations/layer23_attention_weight_max": 39.30229949951172,
      "activations/layer23_attention_weight_min": -26.08230209350586,
      "activations/layer2_attention_weight_max": 33.064903259277344,
      "activations/layer2_attention_weight_min": -32.124637603759766,
      "activations/layer3_attention_weight_max": 94.16008758544922,
      "activations/layer3_attention_weight_min": -93.34258270263672,
      "activations/layer4_attention_weight_max": 99.47685241699219,
      "activations/layer4_attention_weight_min": -96.05524444580078,
      "activations/layer5_attention_weight_max": 77.63015747070312,
      "activations/layer5_attention_weight_min": -80.6371078491211,
      "activations/layer6_attention_weight_max": 52.43583297729492,
      "activations/layer6_attention_weight_min": -53.21109390258789,
      "activations/layer7_attention_weight_max": 76.177978515625,
      "activations/layer7_attention_weight_min": -73.45085906982422,
      "activations/layer8_attention_weight_max": 43.3560676574707,
      "activations/layer8_attention_weight_min": -46.249271392822266,
      "activations/layer9_attention_weight_max": 48.27199935913086,
      "activations/layer9_attention_weight_min": -47.010894775390625,
      "epoch": 9.89,
      "learning_rate": 8.710151515151515e-05,
      "loss": 2.8009,
      "step": 170200
    },
    {
      "activations/layer0_attention_weight_max": 15.649589538574219,
      "activations/layer0_attention_weight_min": -13.54887866973877,
      "activations/layer10_attention_weight_max": 36.06502151489258,
      "activations/layer10_attention_weight_min": -34.08898162841797,
      "activations/layer11_attention_weight_max": 30.81277847290039,
      "activations/layer11_attention_weight_min": -31.862930297851562,
      "activations/layer12_attention_weight_max": 22.91366958618164,
      "activations/layer12_attention_weight_min": -23.749053955078125,
      "activations/layer13_attention_weight_max": 36.43244934082031,
      "activations/layer13_attention_weight_min": -29.10121726989746,
      "activations/layer14_attention_weight_max": 35.83338165283203,
      "activations/layer14_attention_weight_min": -30.190547943115234,
      "activations/layer15_attention_weight_max": 34.22454833984375,
      "activations/layer15_attention_weight_min": -29.929044723510742,
      "activations/layer16_attention_weight_max": 28.745513916015625,
      "activations/layer16_attention_weight_min": -28.159378051757812,
      "activations/layer17_attention_weight_max": 51.30076217651367,
      "activations/layer17_attention_weight_min": -42.66559982299805,
      "activations/layer18_attention_weight_max": 43.93092346191406,
      "activations/layer18_attention_weight_min": -35.4081916809082,
      "activations/layer19_attention_weight_max": 23.338117599487305,
      "activations/layer19_attention_weight_min": -21.003137588500977,
      "activations/layer1_attention_weight_max": 17.345842361450195,
      "activations/layer1_attention_weight_min": -14.927701950073242,
      "activations/layer20_attention_weight_max": 22.65702247619629,
      "activations/layer20_attention_weight_min": -22.28318977355957,
      "activations/layer21_attention_weight_max": 34.464786529541016,
      "activations/layer21_attention_weight_min": -20.062868118286133,
      "activations/layer22_attention_weight_max": 30.612018585205078,
      "activations/layer22_attention_weight_min": -24.43501091003418,
      "activations/layer23_attention_weight_max": 38.87535095214844,
      "activations/layer23_attention_weight_min": -21.81734848022461,
      "activations/layer2_attention_weight_max": 31.550743103027344,
      "activations/layer2_attention_weight_min": -31.27540397644043,
      "activations/layer3_attention_weight_max": 89.71090698242188,
      "activations/layer3_attention_weight_min": -95.72410583496094,
      "activations/layer4_attention_weight_max": 98.55046844482422,
      "activations/layer4_attention_weight_min": -98.1932144165039,
      "activations/layer5_attention_weight_max": 74.22994995117188,
      "activations/layer5_attention_weight_min": -79.58348846435547,
      "activations/layer6_attention_weight_max": 54.435081481933594,
      "activations/layer6_attention_weight_min": -54.458370208740234,
      "activations/layer7_attention_weight_max": 78.48139953613281,
      "activations/layer7_attention_weight_min": -72.24187469482422,
      "activations/layer8_attention_weight_max": 42.521018981933594,
      "activations/layer8_attention_weight_min": -43.457210540771484,
      "activations/layer9_attention_weight_max": 49.61199188232422,
      "activations/layer9_attention_weight_min": -47.73923110961914,
      "epoch": 9.89,
      "learning_rate": 8.708257575757576e-05,
      "loss": 2.8009,
      "step": 170250
    },
    {
      "activations/layer0_attention_weight_max": 15.080796241760254,
      "activations/layer0_attention_weight_min": -13.524975776672363,
      "activations/layer10_attention_weight_max": 35.228546142578125,
      "activations/layer10_attention_weight_min": -35.89341354370117,
      "activations/layer11_attention_weight_max": 31.125930786132812,
      "activations/layer11_attention_weight_min": -34.522071838378906,
      "activations/layer12_attention_weight_max": 18.23299217224121,
      "activations/layer12_attention_weight_min": -26.716697692871094,
      "activations/layer13_attention_weight_max": 44.24386215209961,
      "activations/layer13_attention_weight_min": -37.063514709472656,
      "activations/layer14_attention_weight_max": 39.99219512939453,
      "activations/layer14_attention_weight_min": -31.207042694091797,
      "activations/layer15_attention_weight_max": 36.26091384887695,
      "activations/layer15_attention_weight_min": -30.09370231628418,
      "activations/layer16_attention_weight_max": 32.74900817871094,
      "activations/layer16_attention_weight_min": -28.837427139282227,
      "activations/layer17_attention_weight_max": 56.03999328613281,
      "activations/layer17_attention_weight_min": -44.28822708129883,
      "activations/layer18_attention_weight_max": 55.45975112915039,
      "activations/layer18_attention_weight_min": -38.788509368896484,
      "activations/layer19_attention_weight_max": 23.11419105529785,
      "activations/layer19_attention_weight_min": -23.384334564208984,
      "activations/layer1_attention_weight_max": 16.84479522705078,
      "activations/layer1_attention_weight_min": -15.1439208984375,
      "activations/layer20_attention_weight_max": 25.423410415649414,
      "activations/layer20_attention_weight_min": -21.59156608581543,
      "activations/layer21_attention_weight_max": 41.893768310546875,
      "activations/layer21_attention_weight_min": -25.336748123168945,
      "activations/layer22_attention_weight_max": 37.595829010009766,
      "activations/layer22_attention_weight_min": -26.83296012878418,
      "activations/layer23_attention_weight_max": 39.75117111206055,
      "activations/layer23_attention_weight_min": -23.451152801513672,
      "activations/layer2_attention_weight_max": 31.39092254638672,
      "activations/layer2_attention_weight_min": -29.866531372070312,
      "activations/layer3_attention_weight_max": 86.82655334472656,
      "activations/layer3_attention_weight_min": -91.16522979736328,
      "activations/layer4_attention_weight_max": 96.42098236083984,
      "activations/layer4_attention_weight_min": -95.03804779052734,
      "activations/layer5_attention_weight_max": 73.85926055908203,
      "activations/layer5_attention_weight_min": -79.65805053710938,
      "activations/layer6_attention_weight_max": 53.97768020629883,
      "activations/layer6_attention_weight_min": -52.47238540649414,
      "activations/layer7_attention_weight_max": 77.05670928955078,
      "activations/layer7_attention_weight_min": -73.07711029052734,
      "activations/layer8_attention_weight_max": 43.9648551940918,
      "activations/layer8_attention_weight_min": -46.21638870239258,
      "activations/layer9_attention_weight_max": 50.582454681396484,
      "activations/layer9_attention_weight_min": -49.07768249511719,
      "epoch": 9.9,
      "learning_rate": 8.706363636363635e-05,
      "loss": 2.7878,
      "step": 170300
    },
    {
      "activations/layer0_attention_weight_max": 15.806231498718262,
      "activations/layer0_attention_weight_min": -13.900083541870117,
      "activations/layer10_attention_weight_max": 37.53776931762695,
      "activations/layer10_attention_weight_min": -37.90472412109375,
      "activations/layer11_attention_weight_max": 33.92646026611328,
      "activations/layer11_attention_weight_min": -34.65733337402344,
      "activations/layer12_attention_weight_max": 21.10578155517578,
      "activations/layer12_attention_weight_min": -26.341703414916992,
      "activations/layer13_attention_weight_max": 39.9925537109375,
      "activations/layer13_attention_weight_min": -30.39420509338379,
      "activations/layer14_attention_weight_max": 37.85067367553711,
      "activations/layer14_attention_weight_min": -32.4572639465332,
      "activations/layer15_attention_weight_max": 33.93760681152344,
      "activations/layer15_attention_weight_min": -29.70911979675293,
      "activations/layer16_attention_weight_max": 31.826143264770508,
      "activations/layer16_attention_weight_min": -30.082794189453125,
      "activations/layer17_attention_weight_max": 55.69230651855469,
      "activations/layer17_attention_weight_min": -45.169498443603516,
      "activations/layer18_attention_weight_max": 47.31704330444336,
      "activations/layer18_attention_weight_min": -41.27886962890625,
      "activations/layer19_attention_weight_max": 22.826576232910156,
      "activations/layer19_attention_weight_min": -24.317533493041992,
      "activations/layer1_attention_weight_max": 17.739919662475586,
      "activations/layer1_attention_weight_min": -15.956951141357422,
      "activations/layer20_attention_weight_max": 22.27829360961914,
      "activations/layer20_attention_weight_min": -21.03847885131836,
      "activations/layer21_attention_weight_max": 36.15559005737305,
      "activations/layer21_attention_weight_min": -23.81819725036621,
      "activations/layer22_attention_weight_max": 29.824352264404297,
      "activations/layer22_attention_weight_min": -26.40215301513672,
      "activations/layer23_attention_weight_max": 40.6807975769043,
      "activations/layer23_attention_weight_min": -22.33910369873047,
      "activations/layer2_attention_weight_max": 33.453346252441406,
      "activations/layer2_attention_weight_min": -31.512210845947266,
      "activations/layer3_attention_weight_max": 91.51984405517578,
      "activations/layer3_attention_weight_min": -90.91140747070312,
      "activations/layer4_attention_weight_max": 100.00392150878906,
      "activations/layer4_attention_weight_min": -98.48291015625,
      "activations/layer5_attention_weight_max": 77.26620483398438,
      "activations/layer5_attention_weight_min": -80.13897705078125,
      "activations/layer6_attention_weight_max": 53.91407012939453,
      "activations/layer6_attention_weight_min": -52.846004486083984,
      "activations/layer7_attention_weight_max": 79.37994384765625,
      "activations/layer7_attention_weight_min": -79.72290802001953,
      "activations/layer8_attention_weight_max": 47.364715576171875,
      "activations/layer8_attention_weight_min": -50.08918762207031,
      "activations/layer9_attention_weight_max": 56.506893157958984,
      "activations/layer9_attention_weight_min": -57.109901428222656,
      "epoch": 9.9,
      "learning_rate": 8.704469696969696e-05,
      "loss": 2.8009,
      "step": 170350
    },
    {
      "activations/layer0_attention_weight_max": 15.081611633300781,
      "activations/layer0_attention_weight_min": -13.987066268920898,
      "activations/layer10_attention_weight_max": 38.54341506958008,
      "activations/layer10_attention_weight_min": -35.02877426147461,
      "activations/layer11_attention_weight_max": 32.60934066772461,
      "activations/layer11_attention_weight_min": -31.223968505859375,
      "activations/layer12_attention_weight_max": 17.945829391479492,
      "activations/layer12_attention_weight_min": -25.051692962646484,
      "activations/layer13_attention_weight_max": 37.740196228027344,
      "activations/layer13_attention_weight_min": -30.449363708496094,
      "activations/layer14_attention_weight_max": 35.4172248840332,
      "activations/layer14_attention_weight_min": -28.75355339050293,
      "activations/layer15_attention_weight_max": 33.14762496948242,
      "activations/layer15_attention_weight_min": -27.850574493408203,
      "activations/layer16_attention_weight_max": 30.37846565246582,
      "activations/layer16_attention_weight_min": -28.315692901611328,
      "activations/layer17_attention_weight_max": 52.3475456237793,
      "activations/layer17_attention_weight_min": -41.85006332397461,
      "activations/layer18_attention_weight_max": 43.42329025268555,
      "activations/layer18_attention_weight_min": -34.555259704589844,
      "activations/layer19_attention_weight_max": 20.459259033203125,
      "activations/layer19_attention_weight_min": -19.639738082885742,
      "activations/layer1_attention_weight_max": 17.545217514038086,
      "activations/layer1_attention_weight_min": -16.847124099731445,
      "activations/layer20_attention_weight_max": 21.460975646972656,
      "activations/layer20_attention_weight_min": -20.052776336669922,
      "activations/layer21_attention_weight_max": 39.323612213134766,
      "activations/layer21_attention_weight_min": -21.123872756958008,
      "activations/layer22_attention_weight_max": 29.278968811035156,
      "activations/layer22_attention_weight_min": -23.24102020263672,
      "activations/layer23_attention_weight_max": 35.61186981201172,
      "activations/layer23_attention_weight_min": -22.784719467163086,
      "activations/layer2_attention_weight_max": 32.03016662597656,
      "activations/layer2_attention_weight_min": -30.865150451660156,
      "activations/layer3_attention_weight_max": 91.3152084350586,
      "activations/layer3_attention_weight_min": -90.75728607177734,
      "activations/layer4_attention_weight_max": 103.829345703125,
      "activations/layer4_attention_weight_min": -97.49987030029297,
      "activations/layer5_attention_weight_max": 80.75904083251953,
      "activations/layer5_attention_weight_min": -77.6396484375,
      "activations/layer6_attention_weight_max": 55.956512451171875,
      "activations/layer6_attention_weight_min": -53.988216400146484,
      "activations/layer7_attention_weight_max": 81.45272064208984,
      "activations/layer7_attention_weight_min": -78.21015930175781,
      "activations/layer8_attention_weight_max": 47.97322082519531,
      "activations/layer8_attention_weight_min": -45.744083404541016,
      "activations/layer9_attention_weight_max": 50.445701599121094,
      "activations/layer9_attention_weight_min": -51.1660270690918,
      "epoch": 9.9,
      "learning_rate": 8.702575757575758e-05,
      "loss": 2.8008,
      "step": 170400
    },
    {
      "activations/layer0_attention_weight_max": 14.997612953186035,
      "activations/layer0_attention_weight_min": -13.499397277832031,
      "activations/layer10_attention_weight_max": 35.042579650878906,
      "activations/layer10_attention_weight_min": -34.723121643066406,
      "activations/layer11_attention_weight_max": 29.76386070251465,
      "activations/layer11_attention_weight_min": -32.515106201171875,
      "activations/layer12_attention_weight_max": 21.198158264160156,
      "activations/layer12_attention_weight_min": -26.986658096313477,
      "activations/layer13_attention_weight_max": 42.49127960205078,
      "activations/layer13_attention_weight_min": -29.41668701171875,
      "activations/layer14_attention_weight_max": 44.8542594909668,
      "activations/layer14_attention_weight_min": -31.656166076660156,
      "activations/layer15_attention_weight_max": 34.673988342285156,
      "activations/layer15_attention_weight_min": -29.877378463745117,
      "activations/layer16_attention_weight_max": 31.180150985717773,
      "activations/layer16_attention_weight_min": -28.937301635742188,
      "activations/layer17_attention_weight_max": 50.92560958862305,
      "activations/layer17_attention_weight_min": -44.04136657714844,
      "activations/layer18_attention_weight_max": 47.514888763427734,
      "activations/layer18_attention_weight_min": -37.052894592285156,
      "activations/layer19_attention_weight_max": 21.50946807861328,
      "activations/layer19_attention_weight_min": -23.418806076049805,
      "activations/layer1_attention_weight_max": 17.10426139831543,
      "activations/layer1_attention_weight_min": -17.248268127441406,
      "activations/layer20_attention_weight_max": 23.47859001159668,
      "activations/layer20_attention_weight_min": -23.526792526245117,
      "activations/layer21_attention_weight_max": 40.540889739990234,
      "activations/layer21_attention_weight_min": -24.961931228637695,
      "activations/layer22_attention_weight_max": 33.19447708129883,
      "activations/layer22_attention_weight_min": -28.276752471923828,
      "activations/layer23_attention_weight_max": 40.84196853637695,
      "activations/layer23_attention_weight_min": -24.068288803100586,
      "activations/layer2_attention_weight_max": 30.871784210205078,
      "activations/layer2_attention_weight_min": -28.668163299560547,
      "activations/layer3_attention_weight_max": 89.29103088378906,
      "activations/layer3_attention_weight_min": -90.36759948730469,
      "activations/layer4_attention_weight_max": 97.91952514648438,
      "activations/layer4_attention_weight_min": -94.06846618652344,
      "activations/layer5_attention_weight_max": 74.60896301269531,
      "activations/layer5_attention_weight_min": -75.1688232421875,
      "activations/layer6_attention_weight_max": 51.453365325927734,
      "activations/layer6_attention_weight_min": -50.395301818847656,
      "activations/layer7_attention_weight_max": 77.10575866699219,
      "activations/layer7_attention_weight_min": -71.32767486572266,
      "activations/layer8_attention_weight_max": 43.9853630065918,
      "activations/layer8_attention_weight_min": -43.81170654296875,
      "activations/layer9_attention_weight_max": 48.27145767211914,
      "activations/layer9_attention_weight_min": -45.59656524658203,
      "epoch": 9.9,
      "learning_rate": 8.700681818181817e-05,
      "loss": 2.7783,
      "step": 170450
    },
    {
      "activations/layer0_attention_weight_max": 16.098846435546875,
      "activations/layer0_attention_weight_min": -13.549443244934082,
      "activations/layer10_attention_weight_max": 36.45726013183594,
      "activations/layer10_attention_weight_min": -34.99666213989258,
      "activations/layer11_attention_weight_max": 30.813220977783203,
      "activations/layer11_attention_weight_min": -31.85098648071289,
      "activations/layer12_attention_weight_max": 17.107309341430664,
      "activations/layer12_attention_weight_min": -25.62867546081543,
      "activations/layer13_attention_weight_max": 36.30180740356445,
      "activations/layer13_attention_weight_min": -31.071189880371094,
      "activations/layer14_attention_weight_max": 37.36085891723633,
      "activations/layer14_attention_weight_min": -31.13735580444336,
      "activations/layer15_attention_weight_max": 35.96179962158203,
      "activations/layer15_attention_weight_min": -29.92721939086914,
      "activations/layer16_attention_weight_max": 31.8328857421875,
      "activations/layer16_attention_weight_min": -30.282686233520508,
      "activations/layer17_attention_weight_max": 54.13351058959961,
      "activations/layer17_attention_weight_min": -43.48871994018555,
      "activations/layer18_attention_weight_max": 45.575904846191406,
      "activations/layer18_attention_weight_min": -38.326045989990234,
      "activations/layer19_attention_weight_max": 22.107126235961914,
      "activations/layer19_attention_weight_min": -23.205467224121094,
      "activations/layer1_attention_weight_max": 16.786518096923828,
      "activations/layer1_attention_weight_min": -14.819629669189453,
      "activations/layer20_attention_weight_max": 23.193544387817383,
      "activations/layer20_attention_weight_min": -23.08051872253418,
      "activations/layer21_attention_weight_max": 43.77391815185547,
      "activations/layer21_attention_weight_min": -27.773080825805664,
      "activations/layer22_attention_weight_max": 31.07440185546875,
      "activations/layer22_attention_weight_min": -24.88397789001465,
      "activations/layer23_attention_weight_max": 37.47860336303711,
      "activations/layer23_attention_weight_min": -24.82122039794922,
      "activations/layer2_attention_weight_max": 32.439918518066406,
      "activations/layer2_attention_weight_min": -29.443805694580078,
      "activations/layer3_attention_weight_max": 88.70084381103516,
      "activations/layer3_attention_weight_min": -94.00213623046875,
      "activations/layer4_attention_weight_max": 100.12047576904297,
      "activations/layer4_attention_weight_min": -98.56108856201172,
      "activations/layer5_attention_weight_max": 75.75044250488281,
      "activations/layer5_attention_weight_min": -80.03848266601562,
      "activations/layer6_attention_weight_max": 52.99152374267578,
      "activations/layer6_attention_weight_min": -50.67298889160156,
      "activations/layer7_attention_weight_max": 75.1293716430664,
      "activations/layer7_attention_weight_min": -72.24407196044922,
      "activations/layer8_attention_weight_max": 44.336692810058594,
      "activations/layer8_attention_weight_min": -45.65225601196289,
      "activations/layer9_attention_weight_max": 47.58963394165039,
      "activations/layer9_attention_weight_min": -47.2008171081543,
      "epoch": 9.91,
      "learning_rate": 8.698787878787878e-05,
      "loss": 2.7816,
      "step": 170500
    },
    {
      "activations/layer0_attention_weight_max": 14.63376522064209,
      "activations/layer0_attention_weight_min": -13.733227729797363,
      "activations/layer10_attention_weight_max": 35.742156982421875,
      "activations/layer10_attention_weight_min": -34.71942138671875,
      "activations/layer11_attention_weight_max": 32.8675537109375,
      "activations/layer11_attention_weight_min": -32.644100189208984,
      "activations/layer12_attention_weight_max": 17.40329360961914,
      "activations/layer12_attention_weight_min": -30.870237350463867,
      "activations/layer13_attention_weight_max": 45.23158264160156,
      "activations/layer13_attention_weight_min": -34.022979736328125,
      "activations/layer14_attention_weight_max": 41.9632568359375,
      "activations/layer14_attention_weight_min": -32.68252944946289,
      "activations/layer15_attention_weight_max": 34.91987991333008,
      "activations/layer15_attention_weight_min": -29.42304229736328,
      "activations/layer16_attention_weight_max": 32.29335403442383,
      "activations/layer16_attention_weight_min": -30.40570831298828,
      "activations/layer17_attention_weight_max": 56.37257385253906,
      "activations/layer17_attention_weight_min": -44.62849807739258,
      "activations/layer18_attention_weight_max": 45.72677993774414,
      "activations/layer18_attention_weight_min": -37.73623275756836,
      "activations/layer19_attention_weight_max": 22.533018112182617,
      "activations/layer19_attention_weight_min": -22.733116149902344,
      "activations/layer1_attention_weight_max": 17.521472930908203,
      "activations/layer1_attention_weight_min": -15.929285049438477,
      "activations/layer20_attention_weight_max": 24.365673065185547,
      "activations/layer20_attention_weight_min": -22.324716567993164,
      "activations/layer21_attention_weight_max": 37.298011779785156,
      "activations/layer21_attention_weight_min": -27.161540985107422,
      "activations/layer22_attention_weight_max": 31.343685150146484,
      "activations/layer22_attention_weight_min": -26.29408073425293,
      "activations/layer23_attention_weight_max": 37.77174377441406,
      "activations/layer23_attention_weight_min": -23.38715362548828,
      "activations/layer2_attention_weight_max": 33.09748840332031,
      "activations/layer2_attention_weight_min": -31.290489196777344,
      "activations/layer3_attention_weight_max": 89.5132827758789,
      "activations/layer3_attention_weight_min": -88.553955078125,
      "activations/layer4_attention_weight_max": 101.36543273925781,
      "activations/layer4_attention_weight_min": -95.01647186279297,
      "activations/layer5_attention_weight_max": 79.69779205322266,
      "activations/layer5_attention_weight_min": -82.13119506835938,
      "activations/layer6_attention_weight_max": 55.862464904785156,
      "activations/layer6_attention_weight_min": -54.887271881103516,
      "activations/layer7_attention_weight_max": 81.39567565917969,
      "activations/layer7_attention_weight_min": -74.5599365234375,
      "activations/layer8_attention_weight_max": 50.27012252807617,
      "activations/layer8_attention_weight_min": -50.187103271484375,
      "activations/layer9_attention_weight_max": 50.005882263183594,
      "activations/layer9_attention_weight_min": -48.58206558227539,
      "epoch": 9.91,
      "learning_rate": 8.696893939393938e-05,
      "loss": 2.7918,
      "step": 170550
    },
    {
      "activations/layer0_attention_weight_max": 15.988335609436035,
      "activations/layer0_attention_weight_min": -13.694950103759766,
      "activations/layer10_attention_weight_max": 39.30607986450195,
      "activations/layer10_attention_weight_min": -36.29621124267578,
      "activations/layer11_attention_weight_max": 34.40025329589844,
      "activations/layer11_attention_weight_min": -33.61607360839844,
      "activations/layer12_attention_weight_max": 18.6189022064209,
      "activations/layer12_attention_weight_min": -28.166547775268555,
      "activations/layer13_attention_weight_max": 44.7124137878418,
      "activations/layer13_attention_weight_min": -30.202373504638672,
      "activations/layer14_attention_weight_max": 41.8044548034668,
      "activations/layer14_attention_weight_min": -33.24751663208008,
      "activations/layer15_attention_weight_max": 33.2496452331543,
      "activations/layer15_attention_weight_min": -29.895078659057617,
      "activations/layer16_attention_weight_max": 31.178028106689453,
      "activations/layer16_attention_weight_min": -29.1206111907959,
      "activations/layer17_attention_weight_max": 50.499961853027344,
      "activations/layer17_attention_weight_min": -43.255863189697266,
      "activations/layer18_attention_weight_max": 44.67424774169922,
      "activations/layer18_attention_weight_min": -36.762962341308594,
      "activations/layer19_attention_weight_max": 22.140056610107422,
      "activations/layer19_attention_weight_min": -21.71674919128418,
      "activations/layer1_attention_weight_max": 18.221054077148438,
      "activations/layer1_attention_weight_min": -15.902345657348633,
      "activations/layer20_attention_weight_max": 21.89247703552246,
      "activations/layer20_attention_weight_min": -23.370969772338867,
      "activations/layer21_attention_weight_max": 39.07229995727539,
      "activations/layer21_attention_weight_min": -23.973411560058594,
      "activations/layer22_attention_weight_max": 30.927961349487305,
      "activations/layer22_attention_weight_min": -24.772167205810547,
      "activations/layer23_attention_weight_max": 35.59397888183594,
      "activations/layer23_attention_weight_min": -23.135879516601562,
      "activations/layer2_attention_weight_max": 34.53941345214844,
      "activations/layer2_attention_weight_min": -32.2049560546875,
      "activations/layer3_attention_weight_max": 94.3131103515625,
      "activations/layer3_attention_weight_min": -92.61245727539062,
      "activations/layer4_attention_weight_max": 107.44659423828125,
      "activations/layer4_attention_weight_min": -99.23014068603516,
      "activations/layer5_attention_weight_max": 81.93738555908203,
      "activations/layer5_attention_weight_min": -77.44660186767578,
      "activations/layer6_attention_weight_max": 60.60731887817383,
      "activations/layer6_attention_weight_min": -59.055763244628906,
      "activations/layer7_attention_weight_max": 84.48948669433594,
      "activations/layer7_attention_weight_min": -78.18196868896484,
      "activations/layer8_attention_weight_max": 50.87326431274414,
      "activations/layer8_attention_weight_min": -50.556663513183594,
      "activations/layer9_attention_weight_max": 52.53125,
      "activations/layer9_attention_weight_min": -50.35572814941406,
      "epoch": 9.91,
      "learning_rate": 8.695037878787879e-05,
      "loss": 2.7969,
      "step": 170600
    },
    {
      "activations/layer0_attention_weight_max": 16.490503311157227,
      "activations/layer0_attention_weight_min": -13.572407722473145,
      "activations/layer10_attention_weight_max": 37.362823486328125,
      "activations/layer10_attention_weight_min": -35.0875244140625,
      "activations/layer11_attention_weight_max": 32.663604736328125,
      "activations/layer11_attention_weight_min": -32.279109954833984,
      "activations/layer12_attention_weight_max": 19.029333114624023,
      "activations/layer12_attention_weight_min": -26.61809730529785,
      "activations/layer13_attention_weight_max": 38.39805603027344,
      "activations/layer13_attention_weight_min": -32.27310562133789,
      "activations/layer14_attention_weight_max": 33.478633880615234,
      "activations/layer14_attention_weight_min": -29.135549545288086,
      "activations/layer15_attention_weight_max": 30.51940155029297,
      "activations/layer15_attention_weight_min": -30.77897071838379,
      "activations/layer16_attention_weight_max": 28.48683738708496,
      "activations/layer16_attention_weight_min": -27.66867446899414,
      "activations/layer17_attention_weight_max": 48.8068733215332,
      "activations/layer17_attention_weight_min": -41.73720169067383,
      "activations/layer18_attention_weight_max": 40.586631774902344,
      "activations/layer18_attention_weight_min": -35.39268493652344,
      "activations/layer19_attention_weight_max": 20.409927368164062,
      "activations/layer19_attention_weight_min": -21.984495162963867,
      "activations/layer1_attention_weight_max": 17.912763595581055,
      "activations/layer1_attention_weight_min": -17.247682571411133,
      "activations/layer20_attention_weight_max": 22.849809646606445,
      "activations/layer20_attention_weight_min": -20.44933319091797,
      "activations/layer21_attention_weight_max": 33.464664459228516,
      "activations/layer21_attention_weight_min": -24.779544830322266,
      "activations/layer22_attention_weight_max": 28.695621490478516,
      "activations/layer22_attention_weight_min": -25.09612464904785,
      "activations/layer23_attention_weight_max": 35.54212951660156,
      "activations/layer23_attention_weight_min": -22.94274139404297,
      "activations/layer2_attention_weight_max": 31.801067352294922,
      "activations/layer2_attention_weight_min": -32.21368408203125,
      "activations/layer3_attention_weight_max": 89.6303482055664,
      "activations/layer3_attention_weight_min": -91.8179702758789,
      "activations/layer4_attention_weight_max": 107.0106201171875,
      "activations/layer4_attention_weight_min": -96.7059555053711,
      "activations/layer5_attention_weight_max": 81.37947845458984,
      "activations/layer5_attention_weight_min": -80.74945831298828,
      "activations/layer6_attention_weight_max": 58.0478630065918,
      "activations/layer6_attention_weight_min": -54.52902603149414,
      "activations/layer7_attention_weight_max": 80.54840850830078,
      "activations/layer7_attention_weight_min": -77.353759765625,
      "activations/layer8_attention_weight_max": 46.97004318237305,
      "activations/layer8_attention_weight_min": -47.7486457824707,
      "activations/layer9_attention_weight_max": 49.990901947021484,
      "activations/layer9_attention_weight_min": -47.08720016479492,
      "epoch": 9.92,
      "learning_rate": 8.693143939393939e-05,
      "loss": 2.786,
      "step": 170650
    },
    {
      "activations/layer0_attention_weight_max": 16.013729095458984,
      "activations/layer0_attention_weight_min": -13.57927417755127,
      "activations/layer10_attention_weight_max": 38.74864196777344,
      "activations/layer10_attention_weight_min": -36.17222595214844,
      "activations/layer11_attention_weight_max": 32.771217346191406,
      "activations/layer11_attention_weight_min": -32.41277313232422,
      "activations/layer12_attention_weight_max": 28.28348731994629,
      "activations/layer12_attention_weight_min": -23.749479293823242,
      "activations/layer13_attention_weight_max": 46.84306335449219,
      "activations/layer13_attention_weight_min": -34.10054397583008,
      "activations/layer14_attention_weight_max": 39.732643127441406,
      "activations/layer14_attention_weight_min": -31.03622817993164,
      "activations/layer15_attention_weight_max": 35.920745849609375,
      "activations/layer15_attention_weight_min": -29.978124618530273,
      "activations/layer16_attention_weight_max": 31.973730087280273,
      "activations/layer16_attention_weight_min": -28.600175857543945,
      "activations/layer17_attention_weight_max": 55.101749420166016,
      "activations/layer17_attention_weight_min": -43.78215789794922,
      "activations/layer18_attention_weight_max": 49.04463195800781,
      "activations/layer18_attention_weight_min": -38.08392333984375,
      "activations/layer19_attention_weight_max": 22.572998046875,
      "activations/layer19_attention_weight_min": -21.412221908569336,
      "activations/layer1_attention_weight_max": 16.49246597290039,
      "activations/layer1_attention_weight_min": -15.832216262817383,
      "activations/layer20_attention_weight_max": 25.310489654541016,
      "activations/layer20_attention_weight_min": -20.261564254760742,
      "activations/layer21_attention_weight_max": 37.12787628173828,
      "activations/layer21_attention_weight_min": -22.3897762298584,
      "activations/layer22_attention_weight_max": 34.62104415893555,
      "activations/layer22_attention_weight_min": -25.06446075439453,
      "activations/layer23_attention_weight_max": 39.71281814575195,
      "activations/layer23_attention_weight_min": -23.881038665771484,
      "activations/layer2_attention_weight_max": 33.11424255371094,
      "activations/layer2_attention_weight_min": -31.16168785095215,
      "activations/layer3_attention_weight_max": 94.07502746582031,
      "activations/layer3_attention_weight_min": -94.15078735351562,
      "activations/layer4_attention_weight_max": 102.4966812133789,
      "activations/layer4_attention_weight_min": -95.39225006103516,
      "activations/layer5_attention_weight_max": 75.34390258789062,
      "activations/layer5_attention_weight_min": -77.25749969482422,
      "activations/layer6_attention_weight_max": 53.827701568603516,
      "activations/layer6_attention_weight_min": -53.00480270385742,
      "activations/layer7_attention_weight_max": 81.46662139892578,
      "activations/layer7_attention_weight_min": -72.38896942138672,
      "activations/layer8_attention_weight_max": 48.544960021972656,
      "activations/layer8_attention_weight_min": -47.1551513671875,
      "activations/layer9_attention_weight_max": 49.66205978393555,
      "activations/layer9_attention_weight_min": -50.015743255615234,
      "epoch": 9.92,
      "learning_rate": 8.691249999999999e-05,
      "loss": 2.7908,
      "step": 170700
    },
    {
      "activations/layer0_attention_weight_max": 16.7470760345459,
      "activations/layer0_attention_weight_min": -13.580216407775879,
      "activations/layer10_attention_weight_max": 37.10540771484375,
      "activations/layer10_attention_weight_min": -35.700984954833984,
      "activations/layer11_attention_weight_max": 32.080108642578125,
      "activations/layer11_attention_weight_min": -32.778385162353516,
      "activations/layer12_attention_weight_max": 19.59510612487793,
      "activations/layer12_attention_weight_min": -25.33733558654785,
      "activations/layer13_attention_weight_max": 41.86381912231445,
      "activations/layer13_attention_weight_min": -29.32565689086914,
      "activations/layer14_attention_weight_max": 38.22211837768555,
      "activations/layer14_attention_weight_min": -34.42233657836914,
      "activations/layer15_attention_weight_max": 34.141448974609375,
      "activations/layer15_attention_weight_min": -29.53270721435547,
      "activations/layer16_attention_weight_max": 33.42737579345703,
      "activations/layer16_attention_weight_min": -31.66449737548828,
      "activations/layer17_attention_weight_max": 54.020816802978516,
      "activations/layer17_attention_weight_min": -47.17120361328125,
      "activations/layer18_attention_weight_max": 46.518089294433594,
      "activations/layer18_attention_weight_min": -36.51831817626953,
      "activations/layer19_attention_weight_max": 22.390127182006836,
      "activations/layer19_attention_weight_min": -21.48347282409668,
      "activations/layer1_attention_weight_max": 17.091306686401367,
      "activations/layer1_attention_weight_min": -14.444458961486816,
      "activations/layer20_attention_weight_max": 22.672767639160156,
      "activations/layer20_attention_weight_min": -22.999868392944336,
      "activations/layer21_attention_weight_max": 34.049354553222656,
      "activations/layer21_attention_weight_min": -23.26571273803711,
      "activations/layer22_attention_weight_max": 31.4741268157959,
      "activations/layer22_attention_weight_min": -25.44809341430664,
      "activations/layer23_attention_weight_max": 36.3693733215332,
      "activations/layer23_attention_weight_min": -23.25313377380371,
      "activations/layer2_attention_weight_max": 33.17704772949219,
      "activations/layer2_attention_weight_min": -31.466650009155273,
      "activations/layer3_attention_weight_max": 95.40707397460938,
      "activations/layer3_attention_weight_min": -91.39717102050781,
      "activations/layer4_attention_weight_max": 103.00768280029297,
      "activations/layer4_attention_weight_min": -96.5434341430664,
      "activations/layer5_attention_weight_max": 81.72445678710938,
      "activations/layer5_attention_weight_min": -84.36898803710938,
      "activations/layer6_attention_weight_max": 55.8273811340332,
      "activations/layer6_attention_weight_min": -53.752681732177734,
      "activations/layer7_attention_weight_max": 76.33390045166016,
      "activations/layer7_attention_weight_min": -75.34221649169922,
      "activations/layer8_attention_weight_max": 45.27214431762695,
      "activations/layer8_attention_weight_min": -49.61359786987305,
      "activations/layer9_attention_weight_max": 51.920082092285156,
      "activations/layer9_attention_weight_min": -49.81135177612305,
      "epoch": 9.92,
      "learning_rate": 8.689356060606059e-05,
      "loss": 2.7962,
      "step": 170750
    },
    {
      "activations/layer0_attention_weight_max": 15.968647003173828,
      "activations/layer0_attention_weight_min": -13.650551795959473,
      "activations/layer10_attention_weight_max": 36.54130172729492,
      "activations/layer10_attention_weight_min": -36.310707092285156,
      "activations/layer11_attention_weight_max": 32.10882568359375,
      "activations/layer11_attention_weight_min": -30.98042106628418,
      "activations/layer12_attention_weight_max": 19.91592788696289,
      "activations/layer12_attention_weight_min": -24.00681495666504,
      "activations/layer13_attention_weight_max": 40.198909759521484,
      "activations/layer13_attention_weight_min": -31.824710845947266,
      "activations/layer14_attention_weight_max": 35.83523178100586,
      "activations/layer14_attention_weight_min": -30.361400604248047,
      "activations/layer15_attention_weight_max": 33.06571578979492,
      "activations/layer15_attention_weight_min": -31.8421688079834,
      "activations/layer16_attention_weight_max": 31.15271759033203,
      "activations/layer16_attention_weight_min": -29.005746841430664,
      "activations/layer17_attention_weight_max": 52.37242126464844,
      "activations/layer17_attention_weight_min": -45.1381950378418,
      "activations/layer18_attention_weight_max": 46.0190544128418,
      "activations/layer18_attention_weight_min": -39.13876724243164,
      "activations/layer19_attention_weight_max": 21.45917320251465,
      "activations/layer19_attention_weight_min": -22.43523406982422,
      "activations/layer1_attention_weight_max": 17.124902725219727,
      "activations/layer1_attention_weight_min": -14.17801570892334,
      "activations/layer20_attention_weight_max": 20.45182991027832,
      "activations/layer20_attention_weight_min": -20.398345947265625,
      "activations/layer21_attention_weight_max": 37.536468505859375,
      "activations/layer21_attention_weight_min": -21.90753173828125,
      "activations/layer22_attention_weight_max": 31.779111862182617,
      "activations/layer22_attention_weight_min": -25.095375061035156,
      "activations/layer23_attention_weight_max": 35.065982818603516,
      "activations/layer23_attention_weight_min": -25.595413208007812,
      "activations/layer2_attention_weight_max": 32.16553497314453,
      "activations/layer2_attention_weight_min": -30.10127830505371,
      "activations/layer3_attention_weight_max": 92.6232681274414,
      "activations/layer3_attention_weight_min": -91.53156280517578,
      "activations/layer4_attention_weight_max": 103.17536926269531,
      "activations/layer4_attention_weight_min": -95.4806900024414,
      "activations/layer5_attention_weight_max": 76.94392395019531,
      "activations/layer5_attention_weight_min": -80.52971649169922,
      "activations/layer6_attention_weight_max": 56.578819274902344,
      "activations/layer6_attention_weight_min": -52.05057144165039,
      "activations/layer7_attention_weight_max": 77.81461334228516,
      "activations/layer7_attention_weight_min": -69.60047149658203,
      "activations/layer8_attention_weight_max": 44.744789123535156,
      "activations/layer8_attention_weight_min": -44.61427307128906,
      "activations/layer9_attention_weight_max": 47.73967742919922,
      "activations/layer9_attention_weight_min": -47.17573165893555,
      "epoch": 9.92,
      "learning_rate": 8.687462121212121e-05,
      "loss": 2.7843,
      "step": 170800
    },
    {
      "activations/layer0_attention_weight_max": 14.300943374633789,
      "activations/layer0_attention_weight_min": -13.940938949584961,
      "activations/layer10_attention_weight_max": 40.42562484741211,
      "activations/layer10_attention_weight_min": -37.48788070678711,
      "activations/layer11_attention_weight_max": 34.63275909423828,
      "activations/layer11_attention_weight_min": -32.86930465698242,
      "activations/layer12_attention_weight_max": 18.62453269958496,
      "activations/layer12_attention_weight_min": -26.518110275268555,
      "activations/layer13_attention_weight_max": 36.307220458984375,
      "activations/layer13_attention_weight_min": -31.472862243652344,
      "activations/layer14_attention_weight_max": 34.66630935668945,
      "activations/layer14_attention_weight_min": -30.339292526245117,
      "activations/layer15_attention_weight_max": 33.09974670410156,
      "activations/layer15_attention_weight_min": -28.6590633392334,
      "activations/layer16_attention_weight_max": 32.27602005004883,
      "activations/layer16_attention_weight_min": -29.545955657958984,
      "activations/layer17_attention_weight_max": 52.993412017822266,
      "activations/layer17_attention_weight_min": -41.820674896240234,
      "activations/layer18_attention_weight_max": 46.74078369140625,
      "activations/layer18_attention_weight_min": -37.441123962402344,
      "activations/layer19_attention_weight_max": 22.174680709838867,
      "activations/layer19_attention_weight_min": -21.589895248413086,
      "activations/layer1_attention_weight_max": 16.690425872802734,
      "activations/layer1_attention_weight_min": -15.404593467712402,
      "activations/layer20_attention_weight_max": 20.912662506103516,
      "activations/layer20_attention_weight_min": -19.396970748901367,
      "activations/layer21_attention_weight_max": 36.73875045776367,
      "activations/layer21_attention_weight_min": -25.799850463867188,
      "activations/layer22_attention_weight_max": 31.465974807739258,
      "activations/layer22_attention_weight_min": -24.01447105407715,
      "activations/layer23_attention_weight_max": 34.15135955810547,
      "activations/layer23_attention_weight_min": -26.244115829467773,
      "activations/layer2_attention_weight_max": 35.384117126464844,
      "activations/layer2_attention_weight_min": -33.780433654785156,
      "activations/layer3_attention_weight_max": 95.20921325683594,
      "activations/layer3_attention_weight_min": -91.67727661132812,
      "activations/layer4_attention_weight_max": 103.4367904663086,
      "activations/layer4_attention_weight_min": -97.61817169189453,
      "activations/layer5_attention_weight_max": 81.9607162475586,
      "activations/layer5_attention_weight_min": -79.41571044921875,
      "activations/layer6_attention_weight_max": 56.86455154418945,
      "activations/layer6_attention_weight_min": -56.42005920410156,
      "activations/layer7_attention_weight_max": 79.45042419433594,
      "activations/layer7_attention_weight_min": -76.0151138305664,
      "activations/layer8_attention_weight_max": 48.418495178222656,
      "activations/layer8_attention_weight_min": -47.8118896484375,
      "activations/layer9_attention_weight_max": 52.95534896850586,
      "activations/layer9_attention_weight_min": -54.27396774291992,
      "epoch": 9.93,
      "learning_rate": 8.685568181818181e-05,
      "loss": 2.7954,
      "step": 170850
    },
    {
      "activations/layer0_attention_weight_max": 15.27945613861084,
      "activations/layer0_attention_weight_min": -14.18464183807373,
      "activations/layer10_attention_weight_max": 34.904354095458984,
      "activations/layer10_attention_weight_min": -34.99644470214844,
      "activations/layer11_attention_weight_max": 30.619802474975586,
      "activations/layer11_attention_weight_min": -31.343753814697266,
      "activations/layer12_attention_weight_max": 19.9367618560791,
      "activations/layer12_attention_weight_min": -26.764875411987305,
      "activations/layer13_attention_weight_max": 36.681793212890625,
      "activations/layer13_attention_weight_min": -29.557077407836914,
      "activations/layer14_attention_weight_max": 33.72065734863281,
      "activations/layer14_attention_weight_min": -30.81645965576172,
      "activations/layer15_attention_weight_max": 32.891109466552734,
      "activations/layer15_attention_weight_min": -29.275291442871094,
      "activations/layer16_attention_weight_max": 28.38788414001465,
      "activations/layer16_attention_weight_min": -28.46199607849121,
      "activations/layer17_attention_weight_max": 53.2801399230957,
      "activations/layer17_attention_weight_min": -45.340789794921875,
      "activations/layer18_attention_weight_max": 43.33992385864258,
      "activations/layer18_attention_weight_min": -37.807044982910156,
      "activations/layer19_attention_weight_max": 21.295543670654297,
      "activations/layer19_attention_weight_min": -20.66718864440918,
      "activations/layer1_attention_weight_max": 16.61284065246582,
      "activations/layer1_attention_weight_min": -14.347031593322754,
      "activations/layer20_attention_weight_max": 22.334508895874023,
      "activations/layer20_attention_weight_min": -21.582542419433594,
      "activations/layer21_attention_weight_max": 39.60849380493164,
      "activations/layer21_attention_weight_min": -27.035354614257812,
      "activations/layer22_attention_weight_max": 29.117870330810547,
      "activations/layer22_attention_weight_min": -26.530195236206055,
      "activations/layer23_attention_weight_max": 36.40058898925781,
      "activations/layer23_attention_weight_min": -23.637760162353516,
      "activations/layer2_attention_weight_max": 33.32444381713867,
      "activations/layer2_attention_weight_min": -31.525875091552734,
      "activations/layer3_attention_weight_max": 91.98809814453125,
      "activations/layer3_attention_weight_min": -97.80398559570312,
      "activations/layer4_attention_weight_max": 102.2865219116211,
      "activations/layer4_attention_weight_min": -100.80939483642578,
      "activations/layer5_attention_weight_max": 82.56251525878906,
      "activations/layer5_attention_weight_min": -82.05824279785156,
      "activations/layer6_attention_weight_max": 54.342769622802734,
      "activations/layer6_attention_weight_min": -53.325679779052734,
      "activations/layer7_attention_weight_max": 81.03762817382812,
      "activations/layer7_attention_weight_min": -77.42008209228516,
      "activations/layer8_attention_weight_max": 43.63527297973633,
      "activations/layer8_attention_weight_min": -48.27192306518555,
      "activations/layer9_attention_weight_max": 49.38949203491211,
      "activations/layer9_attention_weight_min": -48.17522048950195,
      "epoch": 9.93,
      "learning_rate": 8.683674242424241e-05,
      "loss": 2.7849,
      "step": 170900
    },
    {
      "activations/layer0_attention_weight_max": 16.808687210083008,
      "activations/layer0_attention_weight_min": -14.42693042755127,
      "activations/layer10_attention_weight_max": 35.859779357910156,
      "activations/layer10_attention_weight_min": -34.22536849975586,
      "activations/layer11_attention_weight_max": 32.418006896972656,
      "activations/layer11_attention_weight_min": -31.52985382080078,
      "activations/layer12_attention_weight_max": 21.66156768798828,
      "activations/layer12_attention_weight_min": -32.42067337036133,
      "activations/layer13_attention_weight_max": 37.551063537597656,
      "activations/layer13_attention_weight_min": -29.080690383911133,
      "activations/layer14_attention_weight_max": 40.83101272583008,
      "activations/layer14_attention_weight_min": -29.4532470703125,
      "activations/layer15_attention_weight_max": 35.97604751586914,
      "activations/layer15_attention_weight_min": -29.410358428955078,
      "activations/layer16_attention_weight_max": 29.354475021362305,
      "activations/layer16_attention_weight_min": -27.740978240966797,
      "activations/layer17_attention_weight_max": 53.19532775878906,
      "activations/layer17_attention_weight_min": -45.4463005065918,
      "activations/layer18_attention_weight_max": 48.84169006347656,
      "activations/layer18_attention_weight_min": -36.33294677734375,
      "activations/layer19_attention_weight_max": 22.264984130859375,
      "activations/layer19_attention_weight_min": -22.72352409362793,
      "activations/layer1_attention_weight_max": 17.04458999633789,
      "activations/layer1_attention_weight_min": -15.30898380279541,
      "activations/layer20_attention_weight_max": 23.5167236328125,
      "activations/layer20_attention_weight_min": -22.306766510009766,
      "activations/layer21_attention_weight_max": 43.432769775390625,
      "activations/layer21_attention_weight_min": -26.065549850463867,
      "activations/layer22_attention_weight_max": 32.73175811767578,
      "activations/layer22_attention_weight_min": -24.55352020263672,
      "activations/layer23_attention_weight_max": 35.44367980957031,
      "activations/layer23_attention_weight_min": -23.434112548828125,
      "activations/layer2_attention_weight_max": 33.2708854675293,
      "activations/layer2_attention_weight_min": -30.928403854370117,
      "activations/layer3_attention_weight_max": 92.60630798339844,
      "activations/layer3_attention_weight_min": -92.51959991455078,
      "activations/layer4_attention_weight_max": 98.87216186523438,
      "activations/layer4_attention_weight_min": -95.4636459350586,
      "activations/layer5_attention_weight_max": 76.85665893554688,
      "activations/layer5_attention_weight_min": -75.9540786743164,
      "activations/layer6_attention_weight_max": 54.796546936035156,
      "activations/layer6_attention_weight_min": -52.326168060302734,
      "activations/layer7_attention_weight_max": 78.7158203125,
      "activations/layer7_attention_weight_min": -70.16950988769531,
      "activations/layer8_attention_weight_max": 45.11696243286133,
      "activations/layer8_attention_weight_min": -45.97198486328125,
      "activations/layer9_attention_weight_max": 50.32720184326172,
      "activations/layer9_attention_weight_min": -51.31663131713867,
      "epoch": 9.93,
      "learning_rate": 8.681780303030303e-05,
      "loss": 2.7801,
      "step": 170950
    },
    {
      "activations/layer0_attention_weight_max": 16.85118293762207,
      "activations/layer0_attention_weight_min": -13.798379898071289,
      "activations/layer10_attention_weight_max": 34.610374450683594,
      "activations/layer10_attention_weight_min": -32.60639190673828,
      "activations/layer11_attention_weight_max": 28.69019317626953,
      "activations/layer11_attention_weight_min": -29.240371704101562,
      "activations/layer12_attention_weight_max": 19.181312561035156,
      "activations/layer12_attention_weight_min": -25.541839599609375,
      "activations/layer13_attention_weight_max": 38.02791213989258,
      "activations/layer13_attention_weight_min": -29.89646339416504,
      "activations/layer14_attention_weight_max": 36.267005920410156,
      "activations/layer14_attention_weight_min": -30.446788787841797,
      "activations/layer15_attention_weight_max": 32.64019012451172,
      "activations/layer15_attention_weight_min": -27.79640007019043,
      "activations/layer16_attention_weight_max": 31.129541397094727,
      "activations/layer16_attention_weight_min": -31.834680557250977,
      "activations/layer17_attention_weight_max": 49.791595458984375,
      "activations/layer17_attention_weight_min": -49.30842971801758,
      "activations/layer18_attention_weight_max": 44.46365737915039,
      "activations/layer18_attention_weight_min": -39.131378173828125,
      "activations/layer19_attention_weight_max": 21.77625274658203,
      "activations/layer19_attention_weight_min": -21.169660568237305,
      "activations/layer1_attention_weight_max": 17.441715240478516,
      "activations/layer1_attention_weight_min": -17.006555557250977,
      "activations/layer20_attention_weight_max": 21.92449378967285,
      "activations/layer20_attention_weight_min": -20.361555099487305,
      "activations/layer21_attention_weight_max": 36.45435333251953,
      "activations/layer21_attention_weight_min": -21.213626861572266,
      "activations/layer22_attention_weight_max": 28.93223762512207,
      "activations/layer22_attention_weight_min": -26.480453491210938,
      "activations/layer23_attention_weight_max": 34.00135803222656,
      "activations/layer23_attention_weight_min": -24.822141647338867,
      "activations/layer2_attention_weight_max": 32.190826416015625,
      "activations/layer2_attention_weight_min": -29.3642578125,
      "activations/layer3_attention_weight_max": 86.81830596923828,
      "activations/layer3_attention_weight_min": -92.39964294433594,
      "activations/layer4_attention_weight_max": 95.85765075683594,
      "activations/layer4_attention_weight_min": -94.64640808105469,
      "activations/layer5_attention_weight_max": 73.0638198852539,
      "activations/layer5_attention_weight_min": -77.280029296875,
      "activations/layer6_attention_weight_max": 52.30705261230469,
      "activations/layer6_attention_weight_min": -51.89012908935547,
      "activations/layer7_attention_weight_max": 74.6819076538086,
      "activations/layer7_attention_weight_min": -70.99250793457031,
      "activations/layer8_attention_weight_max": 42.64567947387695,
      "activations/layer8_attention_weight_min": -42.92033004760742,
      "activations/layer9_attention_weight_max": 46.18446350097656,
      "activations/layer9_attention_weight_min": -45.07029342651367,
      "epoch": 9.94,
      "learning_rate": 8.679886363636364e-05,
      "loss": 2.796,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_loss": 2.744140625,
      "eval_runtime": 8.5664,
      "eval_samples_per_second": 501.258,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_openwebtext_loss": 2.744140625,
      "eval_openwebtext_ppl": 15.551243837871848,
      "eval_openwebtext_runtime": 8.5664,
      "eval_openwebtext_samples_per_second": 501.258,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0218,
      "eval_wikitext_samples_per_second": 225.544,
      "step": 171000
    },
    {
      "epoch": 9.94,
      "eval_lambada_loss": 2.64453125,
      "eval_lambada_ppl": 14.076845015253456,
      "eval_lambada_runtime": 9.6616,
      "eval_lambada_samples_per_second": 503.952,
      "step": 171000
    },
    {
      "activations/layer0_attention_weight_max": 15.69287395477295,
      "activations/layer0_attention_weight_min": -14.199206352233887,
      "activations/layer10_attention_weight_max": 40.30124282836914,
      "activations/layer10_attention_weight_min": -37.110267639160156,
      "activations/layer11_attention_weight_max": 34.794254302978516,
      "activations/layer11_attention_weight_min": -35.91623306274414,
      "activations/layer12_attention_weight_max": 23.27922248840332,
      "activations/layer12_attention_weight_min": -26.81255531311035,
      "activations/layer13_attention_weight_max": 36.006370544433594,
      "activations/layer13_attention_weight_min": -27.899124145507812,
      "activations/layer14_attention_weight_max": 43.703208923339844,
      "activations/layer14_attention_weight_min": -29.91947364807129,
      "activations/layer15_attention_weight_max": 33.022117614746094,
      "activations/layer15_attention_weight_min": -28.472612380981445,
      "activations/layer16_attention_weight_max": 32.514610290527344,
      "activations/layer16_attention_weight_min": -29.57610321044922,
      "activations/layer17_attention_weight_max": 53.41550827026367,
      "activations/layer17_attention_weight_min": -44.82402801513672,
      "activations/layer18_attention_weight_max": 46.73111343383789,
      "activations/layer18_attention_weight_min": -35.83918380737305,
      "activations/layer19_attention_weight_max": 22.02918243408203,
      "activations/layer19_attention_weight_min": -22.248857498168945,
      "activations/layer1_attention_weight_max": 17.226057052612305,
      "activations/layer1_attention_weight_min": -14.46784496307373,
      "activations/layer20_attention_weight_max": 22.529611587524414,
      "activations/layer20_attention_weight_min": -23.52997589111328,
      "activations/layer21_attention_weight_max": 38.86871337890625,
      "activations/layer21_attention_weight_min": -25.359094619750977,
      "activations/layer22_attention_weight_max": 33.07819366455078,
      "activations/layer22_attention_weight_min": -24.026391983032227,
      "activations/layer23_attention_weight_max": 35.75532531738281,
      "activations/layer23_attention_weight_min": -22.07423973083496,
      "activations/layer2_attention_weight_max": 33.55903625488281,
      "activations/layer2_attention_weight_min": -30.804370880126953,
      "activations/layer3_attention_weight_max": 91.54965209960938,
      "activations/layer3_attention_weight_min": -95.03944396972656,
      "activations/layer4_attention_weight_max": 105.98432922363281,
      "activations/layer4_attention_weight_min": -97.16664123535156,
      "activations/layer5_attention_weight_max": 79.22900390625,
      "activations/layer5_attention_weight_min": -83.23242950439453,
      "activations/layer6_attention_weight_max": 54.94677734375,
      "activations/layer6_attention_weight_min": -56.69453811645508,
      "activations/layer7_attention_weight_max": 78.13836669921875,
      "activations/layer7_attention_weight_min": -75.7003402709961,
      "activations/layer8_attention_weight_max": 46.176143646240234,
      "activations/layer8_attention_weight_min": -46.82499313354492,
      "activations/layer9_attention_weight_max": 51.9228515625,
      "activations/layer9_attention_weight_min": -50.0388069152832,
      "epoch": 9.94,
      "learning_rate": 8.677992424242423e-05,
      "loss": 2.786,
      "step": 171050
    },
    {
      "activations/layer0_attention_weight_max": 16.0250301361084,
      "activations/layer0_attention_weight_min": -13.94029712677002,
      "activations/layer10_attention_weight_max": 34.429691314697266,
      "activations/layer10_attention_weight_min": -35.7311897277832,
      "activations/layer11_attention_weight_max": 30.351417541503906,
      "activations/layer11_attention_weight_min": -31.317523956298828,
      "activations/layer12_attention_weight_max": 19.768539428710938,
      "activations/layer12_attention_weight_min": -26.426692962646484,
      "activations/layer13_attention_weight_max": 43.20085525512695,
      "activations/layer13_attention_weight_min": -34.388736724853516,
      "activations/layer14_attention_weight_max": 38.063804626464844,
      "activations/layer14_attention_weight_min": -29.051504135131836,
      "activations/layer15_attention_weight_max": 33.856964111328125,
      "activations/layer15_attention_weight_min": -29.711023330688477,
      "activations/layer16_attention_weight_max": 30.512466430664062,
      "activations/layer16_attention_weight_min": -28.17344093322754,
      "activations/layer17_attention_weight_max": 52.907501220703125,
      "activations/layer17_attention_weight_min": -42.929359436035156,
      "activations/layer18_attention_weight_max": 45.77546691894531,
      "activations/layer18_attention_weight_min": -35.953041076660156,
      "activations/layer19_attention_weight_max": 23.827970504760742,
      "activations/layer19_attention_weight_min": -22.109092712402344,
      "activations/layer1_attention_weight_max": 17.699291229248047,
      "activations/layer1_attention_weight_min": -14.135808944702148,
      "activations/layer20_attention_weight_max": 22.239904403686523,
      "activations/layer20_attention_weight_min": -24.759992599487305,
      "activations/layer21_attention_weight_max": 40.54545211791992,
      "activations/layer21_attention_weight_min": -27.634456634521484,
      "activations/layer22_attention_weight_max": 30.186967849731445,
      "activations/layer22_attention_weight_min": -27.59215545654297,
      "activations/layer23_attention_weight_max": 34.8021240234375,
      "activations/layer23_attention_weight_min": -21.3520565032959,
      "activations/layer2_attention_weight_max": 32.54082107543945,
      "activations/layer2_attention_weight_min": -31.368873596191406,
      "activations/layer3_attention_weight_max": 95.6697769165039,
      "activations/layer3_attention_weight_min": -94.78006744384766,
      "activations/layer4_attention_weight_max": 99.01290130615234,
      "activations/layer4_attention_weight_min": -93.91302490234375,
      "activations/layer5_attention_weight_max": 74.76364135742188,
      "activations/layer5_attention_weight_min": -79.05091094970703,
      "activations/layer6_attention_weight_max": 55.15788269042969,
      "activations/layer6_attention_weight_min": -54.27997970581055,
      "activations/layer7_attention_weight_max": 74.63985443115234,
      "activations/layer7_attention_weight_min": -68.7453842163086,
      "activations/layer8_attention_weight_max": 43.25358963012695,
      "activations/layer8_attention_weight_min": -46.86513900756836,
      "activations/layer9_attention_weight_max": 49.051727294921875,
      "activations/layer9_attention_weight_min": -50.45475387573242,
      "epoch": 9.94,
      "learning_rate": 8.676098484848484e-05,
      "loss": 2.785,
      "step": 171100
    },
    {
      "activations/layer0_attention_weight_max": 15.339089393615723,
      "activations/layer0_attention_weight_min": -14.325285911560059,
      "activations/layer10_attention_weight_max": 36.43808364868164,
      "activations/layer10_attention_weight_min": -34.35990905761719,
      "activations/layer11_attention_weight_max": 31.439739227294922,
      "activations/layer11_attention_weight_min": -32.61517333984375,
      "activations/layer12_attention_weight_max": 18.719011306762695,
      "activations/layer12_attention_weight_min": -25.015743255615234,
      "activations/layer13_attention_weight_max": 43.950477600097656,
      "activations/layer13_attention_weight_min": -31.787811279296875,
      "activations/layer14_attention_weight_max": 38.23149108886719,
      "activations/layer14_attention_weight_min": -30.451961517333984,
      "activations/layer15_attention_weight_max": 33.74249267578125,
      "activations/layer15_attention_weight_min": -29.674020767211914,
      "activations/layer16_attention_weight_max": 30.894485473632812,
      "activations/layer16_attention_weight_min": -29.18285369873047,
      "activations/layer17_attention_weight_max": 52.4091682434082,
      "activations/layer17_attention_weight_min": -43.62371826171875,
      "activations/layer18_attention_weight_max": 46.35660934448242,
      "activations/layer18_attention_weight_min": -35.971797943115234,
      "activations/layer19_attention_weight_max": 20.075620651245117,
      "activations/layer19_attention_weight_min": -19.83627700805664,
      "activations/layer1_attention_weight_max": 17.363788604736328,
      "activations/layer1_attention_weight_min": -17.691486358642578,
      "activations/layer20_attention_weight_max": 24.438413619995117,
      "activations/layer20_attention_weight_min": -19.065868377685547,
      "activations/layer21_attention_weight_max": 34.536964416503906,
      "activations/layer21_attention_weight_min": -24.28167152404785,
      "activations/layer22_attention_weight_max": 32.51515579223633,
      "activations/layer22_attention_weight_min": -24.15247917175293,
      "activations/layer23_attention_weight_max": 39.94117736816406,
      "activations/layer23_attention_weight_min": -23.892820358276367,
      "activations/layer2_attention_weight_max": 32.54515075683594,
      "activations/layer2_attention_weight_min": -31.41046905517578,
      "activations/layer3_attention_weight_max": 97.13924407958984,
      "activations/layer3_attention_weight_min": -94.71951293945312,
      "activations/layer4_attention_weight_max": 101.66497039794922,
      "activations/layer4_attention_weight_min": -94.22969818115234,
      "activations/layer5_attention_weight_max": 77.17562103271484,
      "activations/layer5_attention_weight_min": -78.18677520751953,
      "activations/layer6_attention_weight_max": 55.60063934326172,
      "activations/layer6_attention_weight_min": -50.90079879760742,
      "activations/layer7_attention_weight_max": 80.02667999267578,
      "activations/layer7_attention_weight_min": -69.42472076416016,
      "activations/layer8_attention_weight_max": 44.678741455078125,
      "activations/layer8_attention_weight_min": -44.79185104370117,
      "activations/layer9_attention_weight_max": 52.477088928222656,
      "activations/layer9_attention_weight_min": -47.22170639038086,
      "epoch": 9.94,
      "learning_rate": 8.674204545454545e-05,
      "loss": 2.8036,
      "step": 171150
    },
    {
      "activations/layer0_attention_weight_max": 15.341289520263672,
      "activations/layer0_attention_weight_min": -14.520584106445312,
      "activations/layer10_attention_weight_max": 45.432533264160156,
      "activations/layer10_attention_weight_min": -41.87994384765625,
      "activations/layer11_attention_weight_max": 41.01187515258789,
      "activations/layer11_attention_weight_min": -36.67728042602539,
      "activations/layer12_attention_weight_max": 20.15960121154785,
      "activations/layer12_attention_weight_min": -26.527589797973633,
      "activations/layer13_attention_weight_max": 43.16700744628906,
      "activations/layer13_attention_weight_min": -32.5302734375,
      "activations/layer14_attention_weight_max": 67.55660247802734,
      "activations/layer14_attention_weight_min": -31.21864128112793,
      "activations/layer15_attention_weight_max": 52.642333984375,
      "activations/layer15_attention_weight_min": -30.242050170898438,
      "activations/layer16_attention_weight_max": 33.018768310546875,
      "activations/layer16_attention_weight_min": -29.432647705078125,
      "activations/layer17_attention_weight_max": 56.48469924926758,
      "activations/layer17_attention_weight_min": -46.82472610473633,
      "activations/layer18_attention_weight_max": 48.04983901977539,
      "activations/layer18_attention_weight_min": -39.53319549560547,
      "activations/layer19_attention_weight_max": 25.788986206054688,
      "activations/layer19_attention_weight_min": -23.78819465637207,
      "activations/layer1_attention_weight_max": 18.091405868530273,
      "activations/layer1_attention_weight_min": -16.732460021972656,
      "activations/layer20_attention_weight_max": 28.122844696044922,
      "activations/layer20_attention_weight_min": -22.88951873779297,
      "activations/layer21_attention_weight_max": 45.82716751098633,
      "activations/layer21_attention_weight_min": -23.14437484741211,
      "activations/layer22_attention_weight_max": 33.49238586425781,
      "activations/layer22_attention_weight_min": -25.99153709411621,
      "activations/layer23_attention_weight_max": 40.422393798828125,
      "activations/layer23_attention_weight_min": -22.559383392333984,
      "activations/layer2_attention_weight_max": 38.72532653808594,
      "activations/layer2_attention_weight_min": -35.5809326171875,
      "activations/layer3_attention_weight_max": 102.28364562988281,
      "activations/layer3_attention_weight_min": -103.97547149658203,
      "activations/layer4_attention_weight_max": 106.4471664428711,
      "activations/layer4_attention_weight_min": -100.71977996826172,
      "activations/layer5_attention_weight_max": 81.22087097167969,
      "activations/layer5_attention_weight_min": -77.97016906738281,
      "activations/layer6_attention_weight_max": 56.12114715576172,
      "activations/layer6_attention_weight_min": -56.106754302978516,
      "activations/layer7_attention_weight_max": 91.63361358642578,
      "activations/layer7_attention_weight_min": -78.14122009277344,
      "activations/layer8_attention_weight_max": 54.88808822631836,
      "activations/layer8_attention_weight_min": -53.81688690185547,
      "activations/layer9_attention_weight_max": 66.91007995605469,
      "activations/layer9_attention_weight_min": -54.918617248535156,
      "epoch": 9.95,
      "learning_rate": 8.672310606060605e-05,
      "loss": 2.8019,
      "step": 171200
    },
    {
      "activations/layer0_attention_weight_max": 16.099281311035156,
      "activations/layer0_attention_weight_min": -14.270423889160156,
      "activations/layer10_attention_weight_max": 38.778724670410156,
      "activations/layer10_attention_weight_min": -38.28091812133789,
      "activations/layer11_attention_weight_max": 34.23030090332031,
      "activations/layer11_attention_weight_min": -35.16873550415039,
      "activations/layer12_attention_weight_max": 19.432096481323242,
      "activations/layer12_attention_weight_min": -25.541767120361328,
      "activations/layer13_attention_weight_max": 46.969703674316406,
      "activations/layer13_attention_weight_min": -35.846927642822266,
      "activations/layer14_attention_weight_max": 38.841224670410156,
      "activations/layer14_attention_weight_min": -31.264158248901367,
      "activations/layer15_attention_weight_max": 33.89208221435547,
      "activations/layer15_attention_weight_min": -28.630399703979492,
      "activations/layer16_attention_weight_max": 31.23980712890625,
      "activations/layer16_attention_weight_min": -28.255786895751953,
      "activations/layer17_attention_weight_max": 51.46763229370117,
      "activations/layer17_attention_weight_min": -43.41069412231445,
      "activations/layer18_attention_weight_max": 46.47224426269531,
      "activations/layer18_attention_weight_min": -37.356658935546875,
      "activations/layer19_attention_weight_max": 21.902263641357422,
      "activations/layer19_attention_weight_min": -21.676504135131836,
      "activations/layer1_attention_weight_max": 16.362268447875977,
      "activations/layer1_attention_weight_min": -15.685887336730957,
      "activations/layer20_attention_weight_max": 21.682680130004883,
      "activations/layer20_attention_weight_min": -22.683940887451172,
      "activations/layer21_attention_weight_max": 36.49808883666992,
      "activations/layer21_attention_weight_min": -21.831512451171875,
      "activations/layer22_attention_weight_max": 31.304338455200195,
      "activations/layer22_attention_weight_min": -25.215648651123047,
      "activations/layer23_attention_weight_max": 38.39381408691406,
      "activations/layer23_attention_weight_min": -21.346195220947266,
      "activations/layer2_attention_weight_max": 34.70702362060547,
      "activations/layer2_attention_weight_min": -33.17858123779297,
      "activations/layer3_attention_weight_max": 94.69220733642578,
      "activations/layer3_attention_weight_min": -94.41838836669922,
      "activations/layer4_attention_weight_max": 102.7278823852539,
      "activations/layer4_attention_weight_min": -100.62252044677734,
      "activations/layer5_attention_weight_max": 81.4801025390625,
      "activations/layer5_attention_weight_min": -78.97434997558594,
      "activations/layer6_attention_weight_max": 55.4697265625,
      "activations/layer6_attention_weight_min": -55.20584487915039,
      "activations/layer7_attention_weight_max": 80.26165008544922,
      "activations/layer7_attention_weight_min": -80.8482894897461,
      "activations/layer8_attention_weight_max": 48.33372116088867,
      "activations/layer8_attention_weight_min": -50.831871032714844,
      "activations/layer9_attention_weight_max": 53.71478271484375,
      "activations/layer9_attention_weight_min": -52.286163330078125,
      "epoch": 9.95,
      "learning_rate": 8.670416666666666e-05,
      "loss": 2.7744,
      "step": 171250
    },
    {
      "activations/layer0_attention_weight_max": 16.55636978149414,
      "activations/layer0_attention_weight_min": -14.177953720092773,
      "activations/layer10_attention_weight_max": 39.610321044921875,
      "activations/layer10_attention_weight_min": -36.421138763427734,
      "activations/layer11_attention_weight_max": 34.00239562988281,
      "activations/layer11_attention_weight_min": -33.02668762207031,
      "activations/layer12_attention_weight_max": 20.39787483215332,
      "activations/layer12_attention_weight_min": -24.837175369262695,
      "activations/layer13_attention_weight_max": 45.61439514160156,
      "activations/layer13_attention_weight_min": -31.248096466064453,
      "activations/layer14_attention_weight_max": 38.387447357177734,
      "activations/layer14_attention_weight_min": -30.966825485229492,
      "activations/layer15_attention_weight_max": 34.752410888671875,
      "activations/layer15_attention_weight_min": -29.807466506958008,
      "activations/layer16_attention_weight_max": 32.051448822021484,
      "activations/layer16_attention_weight_min": -29.546043395996094,
      "activations/layer17_attention_weight_max": 55.80873489379883,
      "activations/layer17_attention_weight_min": -45.846370697021484,
      "activations/layer18_attention_weight_max": 48.19418716430664,
      "activations/layer18_attention_weight_min": -38.88958740234375,
      "activations/layer19_attention_weight_max": 25.056135177612305,
      "activations/layer19_attention_weight_min": -20.881017684936523,
      "activations/layer1_attention_weight_max": 16.04012680053711,
      "activations/layer1_attention_weight_min": -15.846519470214844,
      "activations/layer20_attention_weight_max": 24.893468856811523,
      "activations/layer20_attention_weight_min": -22.463621139526367,
      "activations/layer21_attention_weight_max": 39.78914260864258,
      "activations/layer21_attention_weight_min": -25.439647674560547,
      "activations/layer22_attention_weight_max": 34.15612030029297,
      "activations/layer22_attention_weight_min": -25.241012573242188,
      "activations/layer23_attention_weight_max": 40.363101959228516,
      "activations/layer23_attention_weight_min": -22.335540771484375,
      "activations/layer2_attention_weight_max": 33.18608856201172,
      "activations/layer2_attention_weight_min": -34.754112243652344,
      "activations/layer3_attention_weight_max": 92.10733032226562,
      "activations/layer3_attention_weight_min": -93.68292999267578,
      "activations/layer4_attention_weight_max": 98.72742462158203,
      "activations/layer4_attention_weight_min": -97.7964096069336,
      "activations/layer5_attention_weight_max": 75.01913452148438,
      "activations/layer5_attention_weight_min": -78.78944396972656,
      "activations/layer6_attention_weight_max": 54.14231491088867,
      "activations/layer6_attention_weight_min": -54.439781188964844,
      "activations/layer7_attention_weight_max": 78.29747772216797,
      "activations/layer7_attention_weight_min": -72.31781005859375,
      "activations/layer8_attention_weight_max": 46.00629806518555,
      "activations/layer8_attention_weight_min": -48.80140686035156,
      "activations/layer9_attention_weight_max": 50.50991439819336,
      "activations/layer9_attention_weight_min": -48.020694732666016,
      "epoch": 9.95,
      "learning_rate": 8.668522727272726e-05,
      "loss": 2.7912,
      "step": 171300
    },
    {
      "activations/layer0_attention_weight_max": 15.235851287841797,
      "activations/layer0_attention_weight_min": -13.868329048156738,
      "activations/layer10_attention_weight_max": 37.6932258605957,
      "activations/layer10_attention_weight_min": -35.82747268676758,
      "activations/layer11_attention_weight_max": 32.405548095703125,
      "activations/layer11_attention_weight_min": -32.71192932128906,
      "activations/layer12_attention_weight_max": 18.53467559814453,
      "activations/layer12_attention_weight_min": -24.4455509185791,
      "activations/layer13_attention_weight_max": 39.3096809387207,
      "activations/layer13_attention_weight_min": -29.359880447387695,
      "activations/layer14_attention_weight_max": 33.95169448852539,
      "activations/layer14_attention_weight_min": -28.633092880249023,
      "activations/layer15_attention_weight_max": 31.953968048095703,
      "activations/layer15_attention_weight_min": -27.473487854003906,
      "activations/layer16_attention_weight_max": 28.735822677612305,
      "activations/layer16_attention_weight_min": -27.422168731689453,
      "activations/layer17_attention_weight_max": 52.816673278808594,
      "activations/layer17_attention_weight_min": -42.31172561645508,
      "activations/layer18_attention_weight_max": 43.03982925415039,
      "activations/layer18_attention_weight_min": -36.73973846435547,
      "activations/layer19_attention_weight_max": 22.659456253051758,
      "activations/layer19_attention_weight_min": -20.139616012573242,
      "activations/layer1_attention_weight_max": 16.300140380859375,
      "activations/layer1_attention_weight_min": -14.354697227478027,
      "activations/layer20_attention_weight_max": 20.7443790435791,
      "activations/layer20_attention_weight_min": -23.37395668029785,
      "activations/layer21_attention_weight_max": 34.79497528076172,
      "activations/layer21_attention_weight_min": -23.33360481262207,
      "activations/layer22_attention_weight_max": 29.498090744018555,
      "activations/layer22_attention_weight_min": -24.31300163269043,
      "activations/layer23_attention_weight_max": 34.495235443115234,
      "activations/layer23_attention_weight_min": -22.760944366455078,
      "activations/layer2_attention_weight_max": 30.858997344970703,
      "activations/layer2_attention_weight_min": -31.057819366455078,
      "activations/layer3_attention_weight_max": 89.69883728027344,
      "activations/layer3_attention_weight_min": -94.18736267089844,
      "activations/layer4_attention_weight_max": 95.91484832763672,
      "activations/layer4_attention_weight_min": -95.8338851928711,
      "activations/layer5_attention_weight_max": 75.45079803466797,
      "activations/layer5_attention_weight_min": -76.88111877441406,
      "activations/layer6_attention_weight_max": 52.73407745361328,
      "activations/layer6_attention_weight_min": -50.91570281982422,
      "activations/layer7_attention_weight_max": 85.30681610107422,
      "activations/layer7_attention_weight_min": -71.48492431640625,
      "activations/layer8_attention_weight_max": 48.38052749633789,
      "activations/layer8_attention_weight_min": -44.96824264526367,
      "activations/layer9_attention_weight_max": 60.4951057434082,
      "activations/layer9_attention_weight_min": -47.682559967041016,
      "epoch": 9.96,
      "learning_rate": 8.666628787878787e-05,
      "loss": 2.7882,
      "step": 171350
    },
    {
      "activations/layer0_attention_weight_max": 15.345458984375,
      "activations/layer0_attention_weight_min": -13.9220609664917,
      "activations/layer10_attention_weight_max": 39.732666015625,
      "activations/layer10_attention_weight_min": -35.1119270324707,
      "activations/layer11_attention_weight_max": 33.16560363769531,
      "activations/layer11_attention_weight_min": -33.359928131103516,
      "activations/layer12_attention_weight_max": 19.377893447875977,
      "activations/layer12_attention_weight_min": -28.19346809387207,
      "activations/layer13_attention_weight_max": 45.55949783325195,
      "activations/layer13_attention_weight_min": -29.586471557617188,
      "activations/layer14_attention_weight_max": 38.388633728027344,
      "activations/layer14_attention_weight_min": -29.87839126586914,
      "activations/layer15_attention_weight_max": 34.312259674072266,
      "activations/layer15_attention_weight_min": -29.29915428161621,
      "activations/layer16_attention_weight_max": 29.935272216796875,
      "activations/layer16_attention_weight_min": -29.17615509033203,
      "activations/layer17_attention_weight_max": 52.29652786254883,
      "activations/layer17_attention_weight_min": -42.448097229003906,
      "activations/layer18_attention_weight_max": 48.91440200805664,
      "activations/layer18_attention_weight_min": -37.26915740966797,
      "activations/layer19_attention_weight_max": 21.954025268554688,
      "activations/layer19_attention_weight_min": -21.97487449645996,
      "activations/layer1_attention_weight_max": 16.15422821044922,
      "activations/layer1_attention_weight_min": -14.622645378112793,
      "activations/layer20_attention_weight_max": 22.7341365814209,
      "activations/layer20_attention_weight_min": -20.480817794799805,
      "activations/layer21_attention_weight_max": 36.2977409362793,
      "activations/layer21_attention_weight_min": -22.628890991210938,
      "activations/layer22_attention_weight_max": 31.147705078125,
      "activations/layer22_attention_weight_min": -25.780752182006836,
      "activations/layer23_attention_weight_max": 39.69184875488281,
      "activations/layer23_attention_weight_min": -24.86539077758789,
      "activations/layer2_attention_weight_max": 31.156362533569336,
      "activations/layer2_attention_weight_min": -30.65619659423828,
      "activations/layer3_attention_weight_max": 94.81336975097656,
      "activations/layer3_attention_weight_min": -94.9657974243164,
      "activations/layer4_attention_weight_max": 99.3768081665039,
      "activations/layer4_attention_weight_min": -95.50584411621094,
      "activations/layer5_attention_weight_max": 75.15272521972656,
      "activations/layer5_attention_weight_min": -77.43789672851562,
      "activations/layer6_attention_weight_max": 56.588401794433594,
      "activations/layer6_attention_weight_min": -54.945926666259766,
      "activations/layer7_attention_weight_max": 78.00472259521484,
      "activations/layer7_attention_weight_min": -76.0225830078125,
      "activations/layer8_attention_weight_max": 46.19852828979492,
      "activations/layer8_attention_weight_min": -48.55821990966797,
      "activations/layer9_attention_weight_max": 47.67820358276367,
      "activations/layer9_attention_weight_min": -46.644588470458984,
      "epoch": 9.96,
      "learning_rate": 8.664734848484848e-05,
      "loss": 2.7867,
      "step": 171400
    },
    {
      "activations/layer0_attention_weight_max": 15.501853942871094,
      "activations/layer0_attention_weight_min": -13.820303916931152,
      "activations/layer10_attention_weight_max": 36.60203170776367,
      "activations/layer10_attention_weight_min": -36.040283203125,
      "activations/layer11_attention_weight_max": 31.279531478881836,
      "activations/layer11_attention_weight_min": -31.865842819213867,
      "activations/layer12_attention_weight_max": 18.053449630737305,
      "activations/layer12_attention_weight_min": -27.214195251464844,
      "activations/layer13_attention_weight_max": 40.30219268798828,
      "activations/layer13_attention_weight_min": -28.9100341796875,
      "activations/layer14_attention_weight_max": 37.20795440673828,
      "activations/layer14_attention_weight_min": -30.09379005432129,
      "activations/layer15_attention_weight_max": 33.301700592041016,
      "activations/layer15_attention_weight_min": -29.69694709777832,
      "activations/layer16_attention_weight_max": 31.34813690185547,
      "activations/layer16_attention_weight_min": -28.567546844482422,
      "activations/layer17_attention_weight_max": 53.44511413574219,
      "activations/layer17_attention_weight_min": -43.930233001708984,
      "activations/layer18_attention_weight_max": 46.72468566894531,
      "activations/layer18_attention_weight_min": -37.0470085144043,
      "activations/layer19_attention_weight_max": 22.557464599609375,
      "activations/layer19_attention_weight_min": -21.247770309448242,
      "activations/layer1_attention_weight_max": 16.66173553466797,
      "activations/layer1_attention_weight_min": -16.092456817626953,
      "activations/layer20_attention_weight_max": 22.006309509277344,
      "activations/layer20_attention_weight_min": -22.76544761657715,
      "activations/layer21_attention_weight_max": 35.647300720214844,
      "activations/layer21_attention_weight_min": -22.240976333618164,
      "activations/layer22_attention_weight_max": 32.17246627807617,
      "activations/layer22_attention_weight_min": -26.52602767944336,
      "activations/layer23_attention_weight_max": 34.6937370300293,
      "activations/layer23_attention_weight_min": -23.4185791015625,
      "activations/layer2_attention_weight_max": 32.60616683959961,
      "activations/layer2_attention_weight_min": -32.35040283203125,
      "activations/layer3_attention_weight_max": 94.88367462158203,
      "activations/layer3_attention_weight_min": -93.76465606689453,
      "activations/layer4_attention_weight_max": 103.99449920654297,
      "activations/layer4_attention_weight_min": -96.0680160522461,
      "activations/layer5_attention_weight_max": 80.28895568847656,
      "activations/layer5_attention_weight_min": -77.3246841430664,
      "activations/layer6_attention_weight_max": 55.618492126464844,
      "activations/layer6_attention_weight_min": -53.990291595458984,
      "activations/layer7_attention_weight_max": 77.3405532836914,
      "activations/layer7_attention_weight_min": -71.5979232788086,
      "activations/layer8_attention_weight_max": 44.94894027709961,
      "activations/layer8_attention_weight_min": -46.49810028076172,
      "activations/layer9_attention_weight_max": 47.93551254272461,
      "activations/layer9_attention_weight_min": -49.17544174194336,
      "epoch": 9.96,
      "learning_rate": 8.662840909090908e-05,
      "loss": 2.7952,
      "step": 171450
    },
    {
      "activations/layer0_attention_weight_max": 14.475162506103516,
      "activations/layer0_attention_weight_min": -13.669334411621094,
      "activations/layer10_attention_weight_max": 38.42375183105469,
      "activations/layer10_attention_weight_min": -37.76442337036133,
      "activations/layer11_attention_weight_max": 31.55379295349121,
      "activations/layer11_attention_weight_min": -32.459014892578125,
      "activations/layer12_attention_weight_max": 18.13108253479004,
      "activations/layer12_attention_weight_min": -26.322057723999023,
      "activations/layer13_attention_weight_max": 38.312843322753906,
      "activations/layer13_attention_weight_min": -29.36078453063965,
      "activations/layer14_attention_weight_max": 36.63279724121094,
      "activations/layer14_attention_weight_min": -29.36517333984375,
      "activations/layer15_attention_weight_max": 31.436885833740234,
      "activations/layer15_attention_weight_min": -28.518190383911133,
      "activations/layer16_attention_weight_max": 29.715961456298828,
      "activations/layer16_attention_weight_min": -26.938282012939453,
      "activations/layer17_attention_weight_max": 51.92702865600586,
      "activations/layer17_attention_weight_min": -47.312469482421875,
      "activations/layer18_attention_weight_max": 44.23318099975586,
      "activations/layer18_attention_weight_min": -38.4497184753418,
      "activations/layer19_attention_weight_max": 22.058446884155273,
      "activations/layer19_attention_weight_min": -22.006549835205078,
      "activations/layer1_attention_weight_max": 16.60077667236328,
      "activations/layer1_attention_weight_min": -15.179116249084473,
      "activations/layer20_attention_weight_max": 22.313579559326172,
      "activations/layer20_attention_weight_min": -24.723766326904297,
      "activations/layer21_attention_weight_max": 38.870361328125,
      "activations/layer21_attention_weight_min": -24.414011001586914,
      "activations/layer22_attention_weight_max": 29.78173828125,
      "activations/layer22_attention_weight_min": -26.039262771606445,
      "activations/layer23_attention_weight_max": 37.39021301269531,
      "activations/layer23_attention_weight_min": -28.610008239746094,
      "activations/layer2_attention_weight_max": 32.76835632324219,
      "activations/layer2_attention_weight_min": -31.442707061767578,
      "activations/layer3_attention_weight_max": 95.04057312011719,
      "activations/layer3_attention_weight_min": -96.58950805664062,
      "activations/layer4_attention_weight_max": 104.89894104003906,
      "activations/layer4_attention_weight_min": -97.53980255126953,
      "activations/layer5_attention_weight_max": 81.09357452392578,
      "activations/layer5_attention_weight_min": -78.7903060913086,
      "activations/layer6_attention_weight_max": 56.263916015625,
      "activations/layer6_attention_weight_min": -55.56077575683594,
      "activations/layer7_attention_weight_max": 77.16368865966797,
      "activations/layer7_attention_weight_min": -75.90801239013672,
      "activations/layer8_attention_weight_max": 49.42234420776367,
      "activations/layer8_attention_weight_min": -50.89778137207031,
      "activations/layer9_attention_weight_max": 48.40180587768555,
      "activations/layer9_attention_weight_min": -51.17190170288086,
      "epoch": 9.97,
      "learning_rate": 8.66094696969697e-05,
      "loss": 2.7981,
      "step": 171500
    },
    {
      "activations/layer0_attention_weight_max": 16.322425842285156,
      "activations/layer0_attention_weight_min": -14.070928573608398,
      "activations/layer10_attention_weight_max": 37.4733772277832,
      "activations/layer10_attention_weight_min": -35.67499923706055,
      "activations/layer11_attention_weight_max": 33.311344146728516,
      "activations/layer11_attention_weight_min": -32.6994514465332,
      "activations/layer12_attention_weight_max": 17.820159912109375,
      "activations/layer12_attention_weight_min": -28.69597816467285,
      "activations/layer13_attention_weight_max": 41.0473518371582,
      "activations/layer13_attention_weight_min": -31.12030792236328,
      "activations/layer14_attention_weight_max": 35.82904815673828,
      "activations/layer14_attention_weight_min": -30.99956512451172,
      "activations/layer15_attention_weight_max": 33.728233337402344,
      "activations/layer15_attention_weight_min": -30.037824630737305,
      "activations/layer16_attention_weight_max": 33.120521545410156,
      "activations/layer16_attention_weight_min": -27.65325927734375,
      "activations/layer17_attention_weight_max": 54.186607360839844,
      "activations/layer17_attention_weight_min": -45.94743347167969,
      "activations/layer18_attention_weight_max": 45.890201568603516,
      "activations/layer18_attention_weight_min": -36.368778228759766,
      "activations/layer19_attention_weight_max": 20.992158889770508,
      "activations/layer19_attention_weight_min": -21.595388412475586,
      "activations/layer1_attention_weight_max": 18.20708656311035,
      "activations/layer1_attention_weight_min": -14.701132774353027,
      "activations/layer20_attention_weight_max": 20.975509643554688,
      "activations/layer20_attention_weight_min": -19.453153610229492,
      "activations/layer21_attention_weight_max": 35.81248092651367,
      "activations/layer21_attention_weight_min": -24.067718505859375,
      "activations/layer22_attention_weight_max": 30.73631477355957,
      "activations/layer22_attention_weight_min": -24.387157440185547,
      "activations/layer23_attention_weight_max": 33.15482711791992,
      "activations/layer23_attention_weight_min": -22.86618423461914,
      "activations/layer2_attention_weight_max": 32.23899841308594,
      "activations/layer2_attention_weight_min": -31.390850067138672,
      "activations/layer3_attention_weight_max": 92.76045227050781,
      "activations/layer3_attention_weight_min": -97.34553527832031,
      "activations/layer4_attention_weight_max": 106.16899871826172,
      "activations/layer4_attention_weight_min": -103.656494140625,
      "activations/layer5_attention_weight_max": 81.29829406738281,
      "activations/layer5_attention_weight_min": -85.15640258789062,
      "activations/layer6_attention_weight_max": 56.49779510498047,
      "activations/layer6_attention_weight_min": -58.06144332885742,
      "activations/layer7_attention_weight_max": 80.80865478515625,
      "activations/layer7_attention_weight_min": -77.06077575683594,
      "activations/layer8_attention_weight_max": 48.1231689453125,
      "activations/layer8_attention_weight_min": -47.74003219604492,
      "activations/layer9_attention_weight_max": 51.40778350830078,
      "activations/layer9_attention_weight_min": -50.25571823120117,
      "epoch": 9.97,
      "learning_rate": 8.65905303030303e-05,
      "loss": 2.7907,
      "step": 171550
    },
    {
      "activations/layer0_attention_weight_max": 14.640088081359863,
      "activations/layer0_attention_weight_min": -13.94955062866211,
      "activations/layer10_attention_weight_max": 36.13541793823242,
      "activations/layer10_attention_weight_min": -37.275230407714844,
      "activations/layer11_attention_weight_max": 30.08966636657715,
      "activations/layer11_attention_weight_min": -32.673866271972656,
      "activations/layer12_attention_weight_max": 20.352046966552734,
      "activations/layer12_attention_weight_min": -24.26002311706543,
      "activations/layer13_attention_weight_max": 40.26160430908203,
      "activations/layer13_attention_weight_min": -32.438682556152344,
      "activations/layer14_attention_weight_max": 44.4282341003418,
      "activations/layer14_attention_weight_min": -30.826005935668945,
      "activations/layer15_attention_weight_max": 36.6507568359375,
      "activations/layer15_attention_weight_min": -28.425748825073242,
      "activations/layer16_attention_weight_max": 28.22925567626953,
      "activations/layer16_attention_weight_min": -27.65376853942871,
      "activations/layer17_attention_weight_max": 46.927879333496094,
      "activations/layer17_attention_weight_min": -42.33479309082031,
      "activations/layer18_attention_weight_max": 46.129180908203125,
      "activations/layer18_attention_weight_min": -36.95983123779297,
      "activations/layer19_attention_weight_max": 22.50296401977539,
      "activations/layer19_attention_weight_min": -22.521726608276367,
      "activations/layer1_attention_weight_max": 17.30641746520996,
      "activations/layer1_attention_weight_min": -14.30554485321045,
      "activations/layer20_attention_weight_max": 20.814577102661133,
      "activations/layer20_attention_weight_min": -20.13835334777832,
      "activations/layer21_attention_weight_max": 36.81824493408203,
      "activations/layer21_attention_weight_min": -22.565696716308594,
      "activations/layer22_attention_weight_max": 30.917940139770508,
      "activations/layer22_attention_weight_min": -24.385467529296875,
      "activations/layer23_attention_weight_max": 35.95708465576172,
      "activations/layer23_attention_weight_min": -22.2593994140625,
      "activations/layer2_attention_weight_max": 32.96641540527344,
      "activations/layer2_attention_weight_min": -31.3719482421875,
      "activations/layer3_attention_weight_max": 91.70701599121094,
      "activations/layer3_attention_weight_min": -93.44353485107422,
      "activations/layer4_attention_weight_max": 101.41444396972656,
      "activations/layer4_attention_weight_min": -95.19058227539062,
      "activations/layer5_attention_weight_max": 77.72862243652344,
      "activations/layer5_attention_weight_min": -75.52676391601562,
      "activations/layer6_attention_weight_max": 53.26755905151367,
      "activations/layer6_attention_weight_min": -52.8939094543457,
      "activations/layer7_attention_weight_max": 74.34819793701172,
      "activations/layer7_attention_weight_min": -74.2797622680664,
      "activations/layer8_attention_weight_max": 47.466552734375,
      "activations/layer8_attention_weight_min": -49.610069274902344,
      "activations/layer9_attention_weight_max": 58.231422424316406,
      "activations/layer9_attention_weight_min": -50.96097946166992,
      "epoch": 9.97,
      "learning_rate": 8.65715909090909e-05,
      "loss": 2.7875,
      "step": 171600
    },
    {
      "activations/layer0_attention_weight_max": 15.234850883483887,
      "activations/layer0_attention_weight_min": -14.241016387939453,
      "activations/layer10_attention_weight_max": 39.12383270263672,
      "activations/layer10_attention_weight_min": -37.869102478027344,
      "activations/layer11_attention_weight_max": 32.298805236816406,
      "activations/layer11_attention_weight_min": -33.48169708251953,
      "activations/layer12_attention_weight_max": 19.66338348388672,
      "activations/layer12_attention_weight_min": -23.971923828125,
      "activations/layer13_attention_weight_max": 38.449066162109375,
      "activations/layer13_attention_weight_min": -30.74449920654297,
      "activations/layer14_attention_weight_max": 40.106693267822266,
      "activations/layer14_attention_weight_min": -30.500186920166016,
      "activations/layer15_attention_weight_max": 33.085426330566406,
      "activations/layer15_attention_weight_min": -28.510053634643555,
      "activations/layer16_attention_weight_max": 33.8086051940918,
      "activations/layer16_attention_weight_min": -29.710830688476562,
      "activations/layer17_attention_weight_max": 54.7189826965332,
      "activations/layer17_attention_weight_min": -46.00577926635742,
      "activations/layer18_attention_weight_max": 46.06245422363281,
      "activations/layer18_attention_weight_min": -38.52472686767578,
      "activations/layer19_attention_weight_max": 22.983001708984375,
      "activations/layer19_attention_weight_min": -20.527742385864258,
      "activations/layer1_attention_weight_max": 17.036306381225586,
      "activations/layer1_attention_weight_min": -15.688750267028809,
      "activations/layer20_attention_weight_max": 21.760326385498047,
      "activations/layer20_attention_weight_min": -22.398340225219727,
      "activations/layer21_attention_weight_max": 34.62395095825195,
      "activations/layer21_attention_weight_min": -22.549474716186523,
      "activations/layer22_attention_weight_max": 28.934370040893555,
      "activations/layer22_attention_weight_min": -25.292613983154297,
      "activations/layer23_attention_weight_max": 40.27035903930664,
      "activations/layer23_attention_weight_min": -26.090965270996094,
      "activations/layer2_attention_weight_max": 31.85208511352539,
      "activations/layer2_attention_weight_min": -30.810428619384766,
      "activations/layer3_attention_weight_max": 89.45282745361328,
      "activations/layer3_attention_weight_min": -91.54485321044922,
      "activations/layer4_attention_weight_max": 102.29615783691406,
      "activations/layer4_attention_weight_min": -95.58879852294922,
      "activations/layer5_attention_weight_max": 80.57965087890625,
      "activations/layer5_attention_weight_min": -76.61886596679688,
      "activations/layer6_attention_weight_max": 57.03953170776367,
      "activations/layer6_attention_weight_min": -53.56209182739258,
      "activations/layer7_attention_weight_max": 78.49832153320312,
      "activations/layer7_attention_weight_min": -69.77753448486328,
      "activations/layer8_attention_weight_max": 44.73733139038086,
      "activations/layer8_attention_weight_min": -44.89305114746094,
      "activations/layer9_attention_weight_max": 47.8669319152832,
      "activations/layer9_attention_weight_min": -49.060115814208984,
      "epoch": 9.97,
      "learning_rate": 8.655265151515151e-05,
      "loss": 2.795,
      "step": 171650
    },
    {
      "activations/layer0_attention_weight_max": 15.667740821838379,
      "activations/layer0_attention_weight_min": -13.850595474243164,
      "activations/layer10_attention_weight_max": 37.061431884765625,
      "activations/layer10_attention_weight_min": -35.7645149230957,
      "activations/layer11_attention_weight_max": 31.62991714477539,
      "activations/layer11_attention_weight_min": -34.146141052246094,
      "activations/layer12_attention_weight_max": 17.13230323791504,
      "activations/layer12_attention_weight_min": -24.9891414642334,
      "activations/layer13_attention_weight_max": 44.51985168457031,
      "activations/layer13_attention_weight_min": -30.169586181640625,
      "activations/layer14_attention_weight_max": 37.46699142456055,
      "activations/layer14_attention_weight_min": -31.70466423034668,
      "activations/layer15_attention_weight_max": 34.55445098876953,
      "activations/layer15_attention_weight_min": -30.64598274230957,
      "activations/layer16_attention_weight_max": 31.361669540405273,
      "activations/layer16_attention_weight_min": -30.924665451049805,
      "activations/layer17_attention_weight_max": 51.66072463989258,
      "activations/layer17_attention_weight_min": -44.260372161865234,
      "activations/layer18_attention_weight_max": 45.71677017211914,
      "activations/layer18_attention_weight_min": -38.71765899658203,
      "activations/layer19_attention_weight_max": 20.024877548217773,
      "activations/layer19_attention_weight_min": -23.786195755004883,
      "activations/layer1_attention_weight_max": 16.556283950805664,
      "activations/layer1_attention_weight_min": -15.184128761291504,
      "activations/layer20_attention_weight_max": 21.665950775146484,
      "activations/layer20_attention_weight_min": -22.080551147460938,
      "activations/layer21_attention_weight_max": 40.484737396240234,
      "activations/layer21_attention_weight_min": -25.450725555419922,
      "activations/layer22_attention_weight_max": 29.020219802856445,
      "activations/layer22_attention_weight_min": -27.226268768310547,
      "activations/layer23_attention_weight_max": 36.37159729003906,
      "activations/layer23_attention_weight_min": -24.307111740112305,
      "activations/layer2_attention_weight_max": 31.169782638549805,
      "activations/layer2_attention_weight_min": -30.025718688964844,
      "activations/layer3_attention_weight_max": 91.5008316040039,
      "activations/layer3_attention_weight_min": -90.09870147705078,
      "activations/layer4_attention_weight_max": 103.96553802490234,
      "activations/layer4_attention_weight_min": -97.86580657958984,
      "activations/layer5_attention_weight_max": 75.75550842285156,
      "activations/layer5_attention_weight_min": -79.93816375732422,
      "activations/layer6_attention_weight_max": 55.326438903808594,
      "activations/layer6_attention_weight_min": -55.78190231323242,
      "activations/layer7_attention_weight_max": 85.39601135253906,
      "activations/layer7_attention_weight_min": -73.39983367919922,
      "activations/layer8_attention_weight_max": 50.618412017822266,
      "activations/layer8_attention_weight_min": -49.64422607421875,
      "activations/layer9_attention_weight_max": 51.9840202331543,
      "activations/layer9_attention_weight_min": -51.566802978515625,
      "epoch": 9.98,
      "learning_rate": 8.65337121212121e-05,
      "loss": 2.7847,
      "step": 171700
    },
    {
      "activations/layer0_attention_weight_max": 14.34608268737793,
      "activations/layer0_attention_weight_min": -13.896552085876465,
      "activations/layer10_attention_weight_max": 37.551605224609375,
      "activations/layer10_attention_weight_min": -37.18098449707031,
      "activations/layer11_attention_weight_max": 32.23631286621094,
      "activations/layer11_attention_weight_min": -34.0819091796875,
      "activations/layer12_attention_weight_max": 21.323339462280273,
      "activations/layer12_attention_weight_min": -28.436927795410156,
      "activations/layer13_attention_weight_max": 38.90247344970703,
      "activations/layer13_attention_weight_min": -28.064668655395508,
      "activations/layer14_attention_weight_max": 39.98857116699219,
      "activations/layer14_attention_weight_min": -29.72957992553711,
      "activations/layer15_attention_weight_max": 35.65159606933594,
      "activations/layer15_attention_weight_min": -28.27803611755371,
      "activations/layer16_attention_weight_max": 31.899446487426758,
      "activations/layer16_attention_weight_min": -27.78204917907715,
      "activations/layer17_attention_weight_max": 52.09839630126953,
      "activations/layer17_attention_weight_min": -41.986995697021484,
      "activations/layer18_attention_weight_max": 46.47593307495117,
      "activations/layer18_attention_weight_min": -36.49113082885742,
      "activations/layer19_attention_weight_max": 20.866901397705078,
      "activations/layer19_attention_weight_min": -21.791210174560547,
      "activations/layer1_attention_weight_max": 17.481792449951172,
      "activations/layer1_attention_weight_min": -15.599120140075684,
      "activations/layer20_attention_weight_max": 22.27630615234375,
      "activations/layer20_attention_weight_min": -24.667001724243164,
      "activations/layer21_attention_weight_max": 38.87201690673828,
      "activations/layer21_attention_weight_min": -21.030797958374023,
      "activations/layer22_attention_weight_max": 31.780916213989258,
      "activations/layer22_attention_weight_min": -26.631372451782227,
      "activations/layer23_attention_weight_max": 40.45408248901367,
      "activations/layer23_attention_weight_min": -23.54703140258789,
      "activations/layer2_attention_weight_max": 32.41278076171875,
      "activations/layer2_attention_weight_min": -31.782901763916016,
      "activations/layer3_attention_weight_max": 93.12171936035156,
      "activations/layer3_attention_weight_min": -94.7820816040039,
      "activations/layer4_attention_weight_max": 101.95940399169922,
      "activations/layer4_attention_weight_min": -95.3899154663086,
      "activations/layer5_attention_weight_max": 78.296875,
      "activations/layer5_attention_weight_min": -77.56446838378906,
      "activations/layer6_attention_weight_max": 59.172340393066406,
      "activations/layer6_attention_weight_min": -55.9557991027832,
      "activations/layer7_attention_weight_max": 90.14002990722656,
      "activations/layer7_attention_weight_min": -72.819091796875,
      "activations/layer8_attention_weight_max": 48.46175003051758,
      "activations/layer8_attention_weight_min": -47.66889572143555,
      "activations/layer9_attention_weight_max": 51.407649993896484,
      "activations/layer9_attention_weight_min": -50.205039978027344,
      "epoch": 9.98,
      "learning_rate": 8.651477272727272e-05,
      "loss": 2.7853,
      "step": 171750
    },
    {
      "activations/layer0_attention_weight_max": 14.730141639709473,
      "activations/layer0_attention_weight_min": -14.046109199523926,
      "activations/layer10_attention_weight_max": 37.96985626220703,
      "activations/layer10_attention_weight_min": -36.04904556274414,
      "activations/layer11_attention_weight_max": 34.834049224853516,
      "activations/layer11_attention_weight_min": -32.383541107177734,
      "activations/layer12_attention_weight_max": 19.892995834350586,
      "activations/layer12_attention_weight_min": -28.261415481567383,
      "activations/layer13_attention_weight_max": 37.249427795410156,
      "activations/layer13_attention_weight_min": -27.350252151489258,
      "activations/layer14_attention_weight_max": 37.913536071777344,
      "activations/layer14_attention_weight_min": -31.223987579345703,
      "activations/layer15_attention_weight_max": 32.85072708129883,
      "activations/layer15_attention_weight_min": -29.958730697631836,
      "activations/layer16_attention_weight_max": 31.71062469482422,
      "activations/layer16_attention_weight_min": -27.904279708862305,
      "activations/layer17_attention_weight_max": 53.17235565185547,
      "activations/layer17_attention_weight_min": -42.87428665161133,
      "activations/layer18_attention_weight_max": 44.90414047241211,
      "activations/layer18_attention_weight_min": -35.7769889831543,
      "activations/layer19_attention_weight_max": 22.706567764282227,
      "activations/layer19_attention_weight_min": -21.197086334228516,
      "activations/layer1_attention_weight_max": 16.757020950317383,
      "activations/layer1_attention_weight_min": -15.399626731872559,
      "activations/layer20_attention_weight_max": 21.987791061401367,
      "activations/layer20_attention_weight_min": -21.3372745513916,
      "activations/layer21_attention_weight_max": 39.726837158203125,
      "activations/layer21_attention_weight_min": -23.680126190185547,
      "activations/layer22_attention_weight_max": 30.385862350463867,
      "activations/layer22_attention_weight_min": -26.106584548950195,
      "activations/layer23_attention_weight_max": 35.86696243286133,
      "activations/layer23_attention_weight_min": -24.865581512451172,
      "activations/layer2_attention_weight_max": 34.41908645629883,
      "activations/layer2_attention_weight_min": -32.224693298339844,
      "activations/layer3_attention_weight_max": 93.61113739013672,
      "activations/layer3_attention_weight_min": -91.56021881103516,
      "activations/layer4_attention_weight_max": 99.67140197753906,
      "activations/layer4_attention_weight_min": -93.86125946044922,
      "activations/layer5_attention_weight_max": 77.89701843261719,
      "activations/layer5_attention_weight_min": -78.11427307128906,
      "activations/layer6_attention_weight_max": 54.72877883911133,
      "activations/layer6_attention_weight_min": -52.81478500366211,
      "activations/layer7_attention_weight_max": 77.08245086669922,
      "activations/layer7_attention_weight_min": -76.37664031982422,
      "activations/layer8_attention_weight_max": 47.99009704589844,
      "activations/layer8_attention_weight_min": -50.269386291503906,
      "activations/layer9_attention_weight_max": 48.31956100463867,
      "activations/layer9_attention_weight_min": -50.442928314208984,
      "epoch": 9.98,
      "learning_rate": 8.649583333333333e-05,
      "loss": 2.7913,
      "step": 171800
    },
    {
      "activations/layer0_attention_weight_max": 15.830633163452148,
      "activations/layer0_attention_weight_min": -14.428102493286133,
      "activations/layer10_attention_weight_max": 34.535160064697266,
      "activations/layer10_attention_weight_min": -34.569854736328125,
      "activations/layer11_attention_weight_max": 31.03195571899414,
      "activations/layer11_attention_weight_min": -32.41278076171875,
      "activations/layer12_attention_weight_max": 18.156064987182617,
      "activations/layer12_attention_weight_min": -22.955184936523438,
      "activations/layer13_attention_weight_max": 38.21651840209961,
      "activations/layer13_attention_weight_min": -31.7329044342041,
      "activations/layer14_attention_weight_max": 35.20198059082031,
      "activations/layer14_attention_weight_min": -31.189313888549805,
      "activations/layer15_attention_weight_max": 31.889387130737305,
      "activations/layer15_attention_weight_min": -30.796199798583984,
      "activations/layer16_attention_weight_max": 31.671098709106445,
      "activations/layer16_attention_weight_min": -30.753156661987305,
      "activations/layer17_attention_weight_max": 51.79582214355469,
      "activations/layer17_attention_weight_min": -45.03382110595703,
      "activations/layer18_attention_weight_max": 44.17832565307617,
      "activations/layer18_attention_weight_min": -39.656105041503906,
      "activations/layer19_attention_weight_max": 21.8879337310791,
      "activations/layer19_attention_weight_min": -22.508638381958008,
      "activations/layer1_attention_weight_max": 17.95435905456543,
      "activations/layer1_attention_weight_min": -14.035420417785645,
      "activations/layer20_attention_weight_max": 21.46388816833496,
      "activations/layer20_attention_weight_min": -22.658273696899414,
      "activations/layer21_attention_weight_max": 31.919986724853516,
      "activations/layer21_attention_weight_min": -24.705936431884766,
      "activations/layer22_attention_weight_max": 30.351383209228516,
      "activations/layer22_attention_weight_min": -28.3783016204834,
      "activations/layer23_attention_weight_max": 36.12949752807617,
      "activations/layer23_attention_weight_min": -27.896343231201172,
      "activations/layer2_attention_weight_max": 32.47733688354492,
      "activations/layer2_attention_weight_min": -31.398229598999023,
      "activations/layer3_attention_weight_max": 95.83052062988281,
      "activations/layer3_attention_weight_min": -95.82550811767578,
      "activations/layer4_attention_weight_max": 103.30289459228516,
      "activations/layer4_attention_weight_min": -95.9449462890625,
      "activations/layer5_attention_weight_max": 86.027587890625,
      "activations/layer5_attention_weight_min": -78.81541442871094,
      "activations/layer6_attention_weight_max": 58.72816848754883,
      "activations/layer6_attention_weight_min": -53.40810012817383,
      "activations/layer7_attention_weight_max": 84.88565826416016,
      "activations/layer7_attention_weight_min": -74.68090057373047,
      "activations/layer8_attention_weight_max": 45.39790725708008,
      "activations/layer8_attention_weight_min": -46.55368423461914,
      "activations/layer9_attention_weight_max": 50.876312255859375,
      "activations/layer9_attention_weight_min": -47.88705062866211,
      "epoch": 9.99,
      "learning_rate": 8.647689393939392e-05,
      "loss": 2.7785,
      "step": 171850
    },
    {
      "activations/layer0_attention_weight_max": 15.307893753051758,
      "activations/layer0_attention_weight_min": -14.170950889587402,
      "activations/layer10_attention_weight_max": 40.0521354675293,
      "activations/layer10_attention_weight_min": -37.5467414855957,
      "activations/layer11_attention_weight_max": 34.76453399658203,
      "activations/layer11_attention_weight_min": -32.25522994995117,
      "activations/layer12_attention_weight_max": 22.32452392578125,
      "activations/layer12_attention_weight_min": -28.45209503173828,
      "activations/layer13_attention_weight_max": 45.59529495239258,
      "activations/layer13_attention_weight_min": -32.75755310058594,
      "activations/layer14_attention_weight_max": 39.879940032958984,
      "activations/layer14_attention_weight_min": -29.071256637573242,
      "activations/layer15_attention_weight_max": 32.40163803100586,
      "activations/layer15_attention_weight_min": -28.506092071533203,
      "activations/layer16_attention_weight_max": 30.10676383972168,
      "activations/layer16_attention_weight_min": -28.158597946166992,
      "activations/layer17_attention_weight_max": 51.43529510498047,
      "activations/layer17_attention_weight_min": -41.108665466308594,
      "activations/layer18_attention_weight_max": 43.82746505737305,
      "activations/layer18_attention_weight_min": -34.973575592041016,
      "activations/layer19_attention_weight_max": 20.884305953979492,
      "activations/layer19_attention_weight_min": -21.685850143432617,
      "activations/layer1_attention_weight_max": 17.435819625854492,
      "activations/layer1_attention_weight_min": -14.932581901550293,
      "activations/layer20_attention_weight_max": 20.54857063293457,
      "activations/layer20_attention_weight_min": -21.590787887573242,
      "activations/layer21_attention_weight_max": 35.63385009765625,
      "activations/layer21_attention_weight_min": -22.208457946777344,
      "activations/layer22_attention_weight_max": 29.53639030456543,
      "activations/layer22_attention_weight_min": -24.134490966796875,
      "activations/layer23_attention_weight_max": 37.502586364746094,
      "activations/layer23_attention_weight_min": -23.185272216796875,
      "activations/layer2_attention_weight_max": 34.423675537109375,
      "activations/layer2_attention_weight_min": -32.491172790527344,
      "activations/layer3_attention_weight_max": 95.546875,
      "activations/layer3_attention_weight_min": -96.64482879638672,
      "activations/layer4_attention_weight_max": 103.98336029052734,
      "activations/layer4_attention_weight_min": -97.58402252197266,
      "activations/layer5_attention_weight_max": 84.09651947021484,
      "activations/layer5_attention_weight_min": -82.41316223144531,
      "activations/layer6_attention_weight_max": 55.78700637817383,
      "activations/layer6_attention_weight_min": -54.44214630126953,
      "activations/layer7_attention_weight_max": 93.2396011352539,
      "activations/layer7_attention_weight_min": -77.67194366455078,
      "activations/layer8_attention_weight_max": 54.1876106262207,
      "activations/layer8_attention_weight_min": -48.958003997802734,
      "activations/layer9_attention_weight_max": 63.86181640625,
      "activations/layer9_attention_weight_min": -50.24553298950195,
      "epoch": 9.99,
      "learning_rate": 8.645795454545454e-05,
      "loss": 2.7797,
      "step": 171900
    },
    {
      "activations/layer0_attention_weight_max": 14.792501449584961,
      "activations/layer0_attention_weight_min": -14.016304969787598,
      "activations/layer10_attention_weight_max": 38.098411560058594,
      "activations/layer10_attention_weight_min": -37.243595123291016,
      "activations/layer11_attention_weight_max": 33.21773147583008,
      "activations/layer11_attention_weight_min": -33.01963424682617,
      "activations/layer12_attention_weight_max": 18.61711883544922,
      "activations/layer12_attention_weight_min": -26.083826065063477,
      "activations/layer13_attention_weight_max": 41.217742919921875,
      "activations/layer13_attention_weight_min": -30.116573333740234,
      "activations/layer14_attention_weight_max": 37.6461296081543,
      "activations/layer14_attention_weight_min": -30.305362701416016,
      "activations/layer15_attention_weight_max": 33.61258316040039,
      "activations/layer15_attention_weight_min": -29.91759490966797,
      "activations/layer16_attention_weight_max": 32.2814826965332,
      "activations/layer16_attention_weight_min": -29.141334533691406,
      "activations/layer17_attention_weight_max": 56.02223587036133,
      "activations/layer17_attention_weight_min": -45.56705856323242,
      "activations/layer18_attention_weight_max": 47.93341827392578,
      "activations/layer18_attention_weight_min": -36.281558990478516,
      "activations/layer19_attention_weight_max": 22.889291763305664,
      "activations/layer19_attention_weight_min": -22.320419311523438,
      "activations/layer1_attention_weight_max": 16.714237213134766,
      "activations/layer1_attention_weight_min": -17.233675003051758,
      "activations/layer20_attention_weight_max": 21.459617614746094,
      "activations/layer20_attention_weight_min": -22.52532386779785,
      "activations/layer21_attention_weight_max": 40.46332931518555,
      "activations/layer21_attention_weight_min": -23.858795166015625,
      "activations/layer22_attention_weight_max": 30.752315521240234,
      "activations/layer22_attention_weight_min": -25.066789627075195,
      "activations/layer23_attention_weight_max": 41.67300796508789,
      "activations/layer23_attention_weight_min": -23.17635154724121,
      "activations/layer2_attention_weight_max": 33.021697998046875,
      "activations/layer2_attention_weight_min": -33.66496276855469,
      "activations/layer3_attention_weight_max": 102.05425262451172,
      "activations/layer3_attention_weight_min": -100.75884246826172,
      "activations/layer4_attention_weight_max": 105.18035888671875,
      "activations/layer4_attention_weight_min": -98.47037506103516,
      "activations/layer5_attention_weight_max": 80.41310119628906,
      "activations/layer5_attention_weight_min": -83.07612609863281,
      "activations/layer6_attention_weight_max": 55.23810577392578,
      "activations/layer6_attention_weight_min": -53.99808883666992,
      "activations/layer7_attention_weight_max": 78.96556091308594,
      "activations/layer7_attention_weight_min": -72.1093978881836,
      "activations/layer8_attention_weight_max": 45.17783737182617,
      "activations/layer8_attention_weight_min": -47.881229400634766,
      "activations/layer9_attention_weight_max": 54.73360061645508,
      "activations/layer9_attention_weight_min": -49.4100456237793,
      "epoch": 9.99,
      "learning_rate": 8.643901515151515e-05,
      "loss": 2.8138,
      "step": 171950
    },
    {
      "activations/layer0_attention_weight_max": 15.426905632019043,
      "activations/layer0_attention_weight_min": -14.657185554504395,
      "activations/layer10_attention_weight_max": 40.34111022949219,
      "activations/layer10_attention_weight_min": -39.68035888671875,
      "activations/layer11_attention_weight_max": 31.898717880249023,
      "activations/layer11_attention_weight_min": -32.74600601196289,
      "activations/layer12_attention_weight_max": 21.896080017089844,
      "activations/layer12_attention_weight_min": -25.30777931213379,
      "activations/layer13_attention_weight_max": 42.66791915893555,
      "activations/layer13_attention_weight_min": -31.483999252319336,
      "activations/layer14_attention_weight_max": 37.46817398071289,
      "activations/layer14_attention_weight_min": -32.38457107543945,
      "activations/layer15_attention_weight_max": 35.40089416503906,
      "activations/layer15_attention_weight_min": -30.45476722717285,
      "activations/layer16_attention_weight_max": 30.77652931213379,
      "activations/layer16_attention_weight_min": -30.50566291809082,
      "activations/layer17_attention_weight_max": 56.77174758911133,
      "activations/layer17_attention_weight_min": -47.567535400390625,
      "activations/layer18_attention_weight_max": 49.30868148803711,
      "activations/layer18_attention_weight_min": -40.67423629760742,
      "activations/layer19_attention_weight_max": 22.78020477294922,
      "activations/layer19_attention_weight_min": -23.4647216796875,
      "activations/layer1_attention_weight_max": 17.385414123535156,
      "activations/layer1_attention_weight_min": -15.755668640136719,
      "activations/layer20_attention_weight_max": 21.786357879638672,
      "activations/layer20_attention_weight_min": -23.371339797973633,
      "activations/layer21_attention_weight_max": 40.94105911254883,
      "activations/layer21_attention_weight_min": -27.048625946044922,
      "activations/layer22_attention_weight_max": 29.8881893157959,
      "activations/layer22_attention_weight_min": -25.696517944335938,
      "activations/layer23_attention_weight_max": 37.49472427368164,
      "activations/layer23_attention_weight_min": -24.273555755615234,
      "activations/layer2_attention_weight_max": 32.74348068237305,
      "activations/layer2_attention_weight_min": -30.824081420898438,
      "activations/layer3_attention_weight_max": 94.8933334350586,
      "activations/layer3_attention_weight_min": -94.55176544189453,
      "activations/layer4_attention_weight_max": 102.9862060546875,
      "activations/layer4_attention_weight_min": -99.1982421875,
      "activations/layer5_attention_weight_max": 79.3733139038086,
      "activations/layer5_attention_weight_min": -80.3003158569336,
      "activations/layer6_attention_weight_max": 55.77013397216797,
      "activations/layer6_attention_weight_min": -55.80112075805664,
      "activations/layer7_attention_weight_max": 83.97290802001953,
      "activations/layer7_attention_weight_min": -77.14006805419922,
      "activations/layer8_attention_weight_max": 48.73796463012695,
      "activations/layer8_attention_weight_min": -50.15399932861328,
      "activations/layer9_attention_weight_max": 53.0744743347168,
      "activations/layer9_attention_weight_min": -53.04775619506836,
      "epoch": 9.99,
      "learning_rate": 8.642007575757575e-05,
      "loss": 2.775,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.5201,
      "eval_samples_per_second": 503.985,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.5201,
      "eval_openwebtext_samples_per_second": 503.985,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0096,
      "eval_wikitext_samples_per_second": 226.911,
      "step": 172000
    },
    {
      "epoch": 9.99,
      "eval_lambada_loss": 2.6875,
      "eval_lambada_ppl": 14.694892728788941,
      "eval_lambada_runtime": 9.6136,
      "eval_lambada_samples_per_second": 506.471,
      "step": 172000
    },
    {
      "activations/layer0_attention_weight_max": 15.44904613494873,
      "activations/layer0_attention_weight_min": -14.092097282409668,
      "activations/layer10_attention_weight_max": 37.35199737548828,
      "activations/layer10_attention_weight_min": -36.160404205322266,
      "activations/layer11_attention_weight_max": 34.85974884033203,
      "activations/layer11_attention_weight_min": -31.58742332458496,
      "activations/layer12_attention_weight_max": 20.196895599365234,
      "activations/layer12_attention_weight_min": -27.517553329467773,
      "activations/layer13_attention_weight_max": 45.9878044128418,
      "activations/layer13_attention_weight_min": -31.79608154296875,
      "activations/layer14_attention_weight_max": 39.50359344482422,
      "activations/layer14_attention_weight_min": -31.324718475341797,
      "activations/layer15_attention_weight_max": 41.80084991455078,
      "activations/layer15_attention_weight_min": -30.996068954467773,
      "activations/layer16_attention_weight_max": 36.542152404785156,
      "activations/layer16_attention_weight_min": -29.54589080810547,
      "activations/layer17_attention_weight_max": 59.64968490600586,
      "activations/layer17_attention_weight_min": -47.3318977355957,
      "activations/layer18_attention_weight_max": 50.97705078125,
      "activations/layer18_attention_weight_min": -41.29839324951172,
      "activations/layer19_attention_weight_max": 25.317731857299805,
      "activations/layer19_attention_weight_min": -22.245426177978516,
      "activations/layer1_attention_weight_max": 16.852981567382812,
      "activations/layer1_attention_weight_min": -15.624736785888672,
      "activations/layer20_attention_weight_max": 27.75969886779785,
      "activations/layer20_attention_weight_min": -21.87115478515625,
      "activations/layer21_attention_weight_max": 41.79541778564453,
      "activations/layer21_attention_weight_min": -23.500797271728516,
      "activations/layer22_attention_weight_max": 33.756683349609375,
      "activations/layer22_attention_weight_min": -26.2622127532959,
      "activations/layer23_attention_weight_max": 42.23372268676758,
      "activations/layer23_attention_weight_min": -22.13136100769043,
      "activations/layer2_attention_weight_max": 35.51026916503906,
      "activations/layer2_attention_weight_min": -31.646862030029297,
      "activations/layer3_attention_weight_max": 98.92868041992188,
      "activations/layer3_attention_weight_min": -96.976806640625,
      "activations/layer4_attention_weight_max": 105.41165924072266,
      "activations/layer4_attention_weight_min": -102.63677978515625,
      "activations/layer5_attention_weight_max": 79.87223815917969,
      "activations/layer5_attention_weight_min": -83.64765930175781,
      "activations/layer6_attention_weight_max": 54.01210021972656,
      "activations/layer6_attention_weight_min": -56.348819732666016,
      "activations/layer7_attention_weight_max": 87.12989044189453,
      "activations/layer7_attention_weight_min": -75.90486907958984,
      "activations/layer8_attention_weight_max": 47.038543701171875,
      "activations/layer8_attention_weight_min": -51.316673278808594,
      "activations/layer9_attention_weight_max": 58.120967864990234,
      "activations/layer9_attention_weight_min": -48.68757629394531,
      "epoch": 10.0,
      "learning_rate": 8.640113636363635e-05,
      "loss": 2.7828,
      "step": 172050
    },
    {
      "activations/layer0_attention_weight_max": 18.029582977294922,
      "activations/layer0_attention_weight_min": -14.149932861328125,
      "activations/layer10_attention_weight_max": 37.73146057128906,
      "activations/layer10_attention_weight_min": -36.684486389160156,
      "activations/layer11_attention_weight_max": 31.701353073120117,
      "activations/layer11_attention_weight_min": -35.08991622924805,
      "activations/layer12_attention_weight_max": 18.301610946655273,
      "activations/layer12_attention_weight_min": -32.313785552978516,
      "activations/layer13_attention_weight_max": 42.404335021972656,
      "activations/layer13_attention_weight_min": -32.35662841796875,
      "activations/layer14_attention_weight_max": 45.105960845947266,
      "activations/layer14_attention_weight_min": -33.95580291748047,
      "activations/layer15_attention_weight_max": 36.352081298828125,
      "activations/layer15_attention_weight_min": -31.170942306518555,
      "activations/layer16_attention_weight_max": 36.48933410644531,
      "activations/layer16_attention_weight_min": -33.371803283691406,
      "activations/layer17_attention_weight_max": 59.98091125488281,
      "activations/layer17_attention_weight_min": -48.427040100097656,
      "activations/layer18_attention_weight_max": 54.95143508911133,
      "activations/layer18_attention_weight_min": -40.17568588256836,
      "activations/layer19_attention_weight_max": 27.178125381469727,
      "activations/layer19_attention_weight_min": -23.61533546447754,
      "activations/layer1_attention_weight_max": 17.532426834106445,
      "activations/layer1_attention_weight_min": -15.75629711151123,
      "activations/layer20_attention_weight_max": 28.19354248046875,
      "activations/layer20_attention_weight_min": -21.4686336517334,
      "activations/layer21_attention_weight_max": 46.73329544067383,
      "activations/layer21_attention_weight_min": -26.229806900024414,
      "activations/layer22_attention_weight_max": 41.39720916748047,
      "activations/layer22_attention_weight_min": -27.95243263244629,
      "activations/layer23_attention_weight_max": 44.15751647949219,
      "activations/layer23_attention_weight_min": -22.472593307495117,
      "activations/layer2_attention_weight_max": 33.60285949707031,
      "activations/layer2_attention_weight_min": -32.35060501098633,
      "activations/layer3_attention_weight_max": 94.76898193359375,
      "activations/layer3_attention_weight_min": -96.2294921875,
      "activations/layer4_attention_weight_max": 100.94161224365234,
      "activations/layer4_attention_weight_min": -97.2093276977539,
      "activations/layer5_attention_weight_max": 78.77926635742188,
      "activations/layer5_attention_weight_min": -78.70880126953125,
      "activations/layer6_attention_weight_max": 57.35694885253906,
      "activations/layer6_attention_weight_min": -57.90106201171875,
      "activations/layer7_attention_weight_max": 78.8271484375,
      "activations/layer7_attention_weight_min": -76.32640838623047,
      "activations/layer8_attention_weight_max": 47.65824890136719,
      "activations/layer8_attention_weight_min": -47.550567626953125,
      "activations/layer9_attention_weight_max": 50.34869384765625,
      "activations/layer9_attention_weight_min": -51.3497428894043,
      "epoch": 10.0,
      "learning_rate": 8.638219696969696e-05,
      "loss": 2.7795,
      "step": 172100
    },
    {
      "activations/layer0_attention_weight_max": 15.883184432983398,
      "activations/layer0_attention_weight_min": -14.056863784790039,
      "activations/layer10_attention_weight_max": 36.47038650512695,
      "activations/layer10_attention_weight_min": -37.143760681152344,
      "activations/layer11_attention_weight_max": 31.848838806152344,
      "activations/layer11_attention_weight_min": -34.03115463256836,
      "activations/layer12_attention_weight_max": 18.3276309967041,
      "activations/layer12_attention_weight_min": -23.056751251220703,
      "activations/layer13_attention_weight_max": 41.614593505859375,
      "activations/layer13_attention_weight_min": -31.153182983398438,
      "activations/layer14_attention_weight_max": 35.603485107421875,
      "activations/layer14_attention_weight_min": -31.39897918701172,
      "activations/layer15_attention_weight_max": 35.546451568603516,
      "activations/layer15_attention_weight_min": -30.068214416503906,
      "activations/layer16_attention_weight_max": 29.610294342041016,
      "activations/layer16_attention_weight_min": -28.98552703857422,
      "activations/layer17_attention_weight_max": 51.85994338989258,
      "activations/layer17_attention_weight_min": -43.651275634765625,
      "activations/layer18_attention_weight_max": 45.61154556274414,
      "activations/layer18_attention_weight_min": -36.401405334472656,
      "activations/layer19_attention_weight_max": 20.772626876831055,
      "activations/layer19_attention_weight_min": -21.70374870300293,
      "activations/layer1_attention_weight_max": 16.8624210357666,
      "activations/layer1_attention_weight_min": -16.550413131713867,
      "activations/layer20_attention_weight_max": 23.060993194580078,
      "activations/layer20_attention_weight_min": -24.059579849243164,
      "activations/layer21_attention_weight_max": 35.662681579589844,
      "activations/layer21_attention_weight_min": -23.93605613708496,
      "activations/layer22_attention_weight_max": 30.216827392578125,
      "activations/layer22_attention_weight_min": -25.71234130859375,
      "activations/layer23_attention_weight_max": 35.14588165283203,
      "activations/layer23_attention_weight_min": -24.39675521850586,
      "activations/layer2_attention_weight_max": 32.415313720703125,
      "activations/layer2_attention_weight_min": -32.459625244140625,
      "activations/layer3_attention_weight_max": 93.10487365722656,
      "activations/layer3_attention_weight_min": -100.78592681884766,
      "activations/layer4_attention_weight_max": 103.42036437988281,
      "activations/layer4_attention_weight_min": -98.33013153076172,
      "activations/layer5_attention_weight_max": 78.00979614257812,
      "activations/layer5_attention_weight_min": -79.35123443603516,
      "activations/layer6_attention_weight_max": 55.25190353393555,
      "activations/layer6_attention_weight_min": -54.19709396362305,
      "activations/layer7_attention_weight_max": 78.5527114868164,
      "activations/layer7_attention_weight_min": -78.9722900390625,
      "activations/layer8_attention_weight_max": 48.64570617675781,
      "activations/layer8_attention_weight_min": -50.27244567871094,
      "activations/layer9_attention_weight_max": 49.02161407470703,
      "activations/layer9_attention_weight_min": -50.10068893432617,
      "epoch": 10.0,
      "learning_rate": 8.636325757575757e-05,
      "loss": 2.8039,
      "step": 172150
    },
    {
      "activations/layer0_attention_weight_max": 16.839757919311523,
      "activations/layer0_attention_weight_min": -14.134581565856934,
      "activations/layer10_attention_weight_max": 56.272029876708984,
      "activations/layer10_attention_weight_min": -51.122520446777344,
      "activations/layer11_attention_weight_max": 46.068458557128906,
      "activations/layer11_attention_weight_min": -42.8966064453125,
      "activations/layer12_attention_weight_max": 18.367591857910156,
      "activations/layer12_attention_weight_min": -27.81053352355957,
      "activations/layer13_attention_weight_max": 52.22904586791992,
      "activations/layer13_attention_weight_min": -41.70256423950195,
      "activations/layer14_attention_weight_max": 57.062652587890625,
      "activations/layer14_attention_weight_min": -39.32777404785156,
      "activations/layer15_attention_weight_max": 53.2530632019043,
      "activations/layer15_attention_weight_min": -36.89170455932617,
      "activations/layer16_attention_weight_max": 41.46977615356445,
      "activations/layer16_attention_weight_min": -34.285621643066406,
      "activations/layer17_attention_weight_max": 57.75385284423828,
      "activations/layer17_attention_weight_min": -51.775978088378906,
      "activations/layer18_attention_weight_max": 47.947418212890625,
      "activations/layer18_attention_weight_min": -42.29114532470703,
      "activations/layer19_attention_weight_max": 22.39314079284668,
      "activations/layer19_attention_weight_min": -23.79860496520996,
      "activations/layer1_attention_weight_max": 16.6846923828125,
      "activations/layer1_attention_weight_min": -16.43570327758789,
      "activations/layer20_attention_weight_max": 24.488929748535156,
      "activations/layer20_attention_weight_min": -24.170372009277344,
      "activations/layer21_attention_weight_max": 43.66990280151367,
      "activations/layer21_attention_weight_min": -27.965105056762695,
      "activations/layer22_attention_weight_max": 33.40604019165039,
      "activations/layer22_attention_weight_min": -28.25606918334961,
      "activations/layer23_attention_weight_max": 39.46739196777344,
      "activations/layer23_attention_weight_min": -23.98495101928711,
      "activations/layer2_attention_weight_max": 36.48223114013672,
      "activations/layer2_attention_weight_min": -36.851051330566406,
      "activations/layer3_attention_weight_max": 102.53850555419922,
      "activations/layer3_attention_weight_min": -107.30890655517578,
      "activations/layer4_attention_weight_max": 116.49055480957031,
      "activations/layer4_attention_weight_min": -110.60511779785156,
      "activations/layer5_attention_weight_max": 88.93257904052734,
      "activations/layer5_attention_weight_min": -79.25064086914062,
      "activations/layer6_attention_weight_max": 64.83090209960938,
      "activations/layer6_attention_weight_min": -63.01551818847656,
      "activations/layer7_attention_weight_max": 104.76778411865234,
      "activations/layer7_attention_weight_min": -94.8371810913086,
      "activations/layer8_attention_weight_max": 70.48526000976562,
      "activations/layer8_attention_weight_min": -65.96945190429688,
      "activations/layer9_attention_weight_max": 81.18502044677734,
      "activations/layer9_attention_weight_min": -65.9923095703125,
      "epoch": 10.01,
      "learning_rate": 8.634431818181817e-05,
      "loss": 2.7613,
      "step": 172200
    },
    {
      "activations/layer0_attention_weight_max": 15.053824424743652,
      "activations/layer0_attention_weight_min": -13.94357967376709,
      "activations/layer10_attention_weight_max": 39.927947998046875,
      "activations/layer10_attention_weight_min": -38.933372497558594,
      "activations/layer11_attention_weight_max": 34.677398681640625,
      "activations/layer11_attention_weight_min": -35.36592102050781,
      "activations/layer12_attention_weight_max": 20.86619758605957,
      "activations/layer12_attention_weight_min": -24.601993560791016,
      "activations/layer13_attention_weight_max": 44.50983428955078,
      "activations/layer13_attention_weight_min": -34.997554779052734,
      "activations/layer14_attention_weight_max": 41.266536712646484,
      "activations/layer14_attention_weight_min": -32.975521087646484,
      "activations/layer15_attention_weight_max": 34.7812614440918,
      "activations/layer15_attention_weight_min": -31.387292861938477,
      "activations/layer16_attention_weight_max": 35.568931579589844,
      "activations/layer16_attention_weight_min": -29.125808715820312,
      "activations/layer17_attention_weight_max": 54.32522964477539,
      "activations/layer17_attention_weight_min": -45.83914566040039,
      "activations/layer18_attention_weight_max": 48.26700973510742,
      "activations/layer18_attention_weight_min": -39.557647705078125,
      "activations/layer19_attention_weight_max": 24.0617733001709,
      "activations/layer19_attention_weight_min": -22.91327476501465,
      "activations/layer1_attention_weight_max": 17.010164260864258,
      "activations/layer1_attention_weight_min": -16.10331153869629,
      "activations/layer20_attention_weight_max": 25.933555603027344,
      "activations/layer20_attention_weight_min": -23.073469161987305,
      "activations/layer21_attention_weight_max": 43.87056350708008,
      "activations/layer21_attention_weight_min": -29.564855575561523,
      "activations/layer22_attention_weight_max": 38.28202438354492,
      "activations/layer22_attention_weight_min": -26.126136779785156,
      "activations/layer23_attention_weight_max": 43.226680755615234,
      "activations/layer23_attention_weight_min": -28.37673568725586,
      "activations/layer2_attention_weight_max": 34.4375,
      "activations/layer2_attention_weight_min": -32.87730407714844,
      "activations/layer3_attention_weight_max": 96.63396453857422,
      "activations/layer3_attention_weight_min": -95.12188720703125,
      "activations/layer4_attention_weight_max": 104.44254302978516,
      "activations/layer4_attention_weight_min": -102.64210510253906,
      "activations/layer5_attention_weight_max": 80.3828125,
      "activations/layer5_attention_weight_min": -79.25995635986328,
      "activations/layer6_attention_weight_max": 56.59101486206055,
      "activations/layer6_attention_weight_min": -56.16007614135742,
      "activations/layer7_attention_weight_max": 80.26519775390625,
      "activations/layer7_attention_weight_min": -76.77662658691406,
      "activations/layer8_attention_weight_max": 47.837955474853516,
      "activations/layer8_attention_weight_min": -48.10080337524414,
      "activations/layer9_attention_weight_max": 54.244232177734375,
      "activations/layer9_attention_weight_min": -52.19987869262695,
      "epoch": 10.01,
      "learning_rate": 8.632537878787877e-05,
      "loss": 2.7536,
      "step": 172250
    },
    {
      "activations/layer0_attention_weight_max": 14.484837532043457,
      "activations/layer0_attention_weight_min": -13.82975959777832,
      "activations/layer10_attention_weight_max": 41.60503005981445,
      "activations/layer10_attention_weight_min": -38.043487548828125,
      "activations/layer11_attention_weight_max": 33.27967071533203,
      "activations/layer11_attention_weight_min": -33.312339782714844,
      "activations/layer12_attention_weight_max": 17.781702041625977,
      "activations/layer12_attention_weight_min": -24.54020881652832,
      "activations/layer13_attention_weight_max": 44.38563919067383,
      "activations/layer13_attention_weight_min": -29.577932357788086,
      "activations/layer14_attention_weight_max": 37.97312545776367,
      "activations/layer14_attention_weight_min": -31.057649612426758,
      "activations/layer15_attention_weight_max": 38.6845703125,
      "activations/layer15_attention_weight_min": -30.19614028930664,
      "activations/layer16_attention_weight_max": 33.47161865234375,
      "activations/layer16_attention_weight_min": -29.970996856689453,
      "activations/layer17_attention_weight_max": 55.0340461730957,
      "activations/layer17_attention_weight_min": -44.159202575683594,
      "activations/layer18_attention_weight_max": 48.61429977416992,
      "activations/layer18_attention_weight_min": -37.1690788269043,
      "activations/layer19_attention_weight_max": 26.706151962280273,
      "activations/layer19_attention_weight_min": -21.205123901367188,
      "activations/layer1_attention_weight_max": 16.292558670043945,
      "activations/layer1_attention_weight_min": -15.02046012878418,
      "activations/layer20_attention_weight_max": 24.3917236328125,
      "activations/layer20_attention_weight_min": -22.779714584350586,
      "activations/layer21_attention_weight_max": 37.50175857543945,
      "activations/layer21_attention_weight_min": -25.064531326293945,
      "activations/layer22_attention_weight_max": 33.326499938964844,
      "activations/layer22_attention_weight_min": -24.17374610900879,
      "activations/layer23_attention_weight_max": 36.748573303222656,
      "activations/layer23_attention_weight_min": -23.745258331298828,
      "activations/layer2_attention_weight_max": 31.66144561767578,
      "activations/layer2_attention_weight_min": -31.178274154663086,
      "activations/layer3_attention_weight_max": 95.49685668945312,
      "activations/layer3_attention_weight_min": -96.62345123291016,
      "activations/layer4_attention_weight_max": 108.49785614013672,
      "activations/layer4_attention_weight_min": -100.1502914428711,
      "activations/layer5_attention_weight_max": 83.20600891113281,
      "activations/layer5_attention_weight_min": -82.12385559082031,
      "activations/layer6_attention_weight_max": 60.82221603393555,
      "activations/layer6_attention_weight_min": -56.234092712402344,
      "activations/layer7_attention_weight_max": 85.27594757080078,
      "activations/layer7_attention_weight_min": -78.4185562133789,
      "activations/layer8_attention_weight_max": 47.47305679321289,
      "activations/layer8_attention_weight_min": -49.253273010253906,
      "activations/layer9_attention_weight_max": 50.13961410522461,
      "activations/layer9_attention_weight_min": -51.231201171875,
      "epoch": 10.01,
      "learning_rate": 8.630643939393939e-05,
      "loss": 2.7703,
      "step": 172300
    },
    {
      "activations/layer0_attention_weight_max": 15.679947853088379,
      "activations/layer0_attention_weight_min": -13.889183044433594,
      "activations/layer10_attention_weight_max": 40.84348678588867,
      "activations/layer10_attention_weight_min": -40.94681167602539,
      "activations/layer11_attention_weight_max": 36.084102630615234,
      "activations/layer11_attention_weight_min": -38.44601058959961,
      "activations/layer12_attention_weight_max": 23.690933227539062,
      "activations/layer12_attention_weight_min": -25.892396926879883,
      "activations/layer13_attention_weight_max": 36.24201583862305,
      "activations/layer13_attention_weight_min": -31.131120681762695,
      "activations/layer14_attention_weight_max": 36.07486343383789,
      "activations/layer14_attention_weight_min": -32.3448486328125,
      "activations/layer15_attention_weight_max": 32.07489776611328,
      "activations/layer15_attention_weight_min": -30.194433212280273,
      "activations/layer16_attention_weight_max": 29.69091796875,
      "activations/layer16_attention_weight_min": -27.80251693725586,
      "activations/layer17_attention_weight_max": 50.67488479614258,
      "activations/layer17_attention_weight_min": -41.705223083496094,
      "activations/layer18_attention_weight_max": 45.68120574951172,
      "activations/layer18_attention_weight_min": -36.5374755859375,
      "activations/layer19_attention_weight_max": 23.172677993774414,
      "activations/layer19_attention_weight_min": -23.115724563598633,
      "activations/layer1_attention_weight_max": 17.860610961914062,
      "activations/layer1_attention_weight_min": -14.202855110168457,
      "activations/layer20_attention_weight_max": 23.11141586303711,
      "activations/layer20_attention_weight_min": -24.919689178466797,
      "activations/layer21_attention_weight_max": 42.567020416259766,
      "activations/layer21_attention_weight_min": -23.168222427368164,
      "activations/layer22_attention_weight_max": 30.95578956604004,
      "activations/layer22_attention_weight_min": -26.8342342376709,
      "activations/layer23_attention_weight_max": 38.01666259765625,
      "activations/layer23_attention_weight_min": -25.089649200439453,
      "activations/layer2_attention_weight_max": 33.89851379394531,
      "activations/layer2_attention_weight_min": -32.13547134399414,
      "activations/layer3_attention_weight_max": 98.26539611816406,
      "activations/layer3_attention_weight_min": -94.7796859741211,
      "activations/layer4_attention_weight_max": 109.05406951904297,
      "activations/layer4_attention_weight_min": -98.77423858642578,
      "activations/layer5_attention_weight_max": 82.5379409790039,
      "activations/layer5_attention_weight_min": -81.04388427734375,
      "activations/layer6_attention_weight_max": 57.96242141723633,
      "activations/layer6_attention_weight_min": -57.57193374633789,
      "activations/layer7_attention_weight_max": 79.27899932861328,
      "activations/layer7_attention_weight_min": -81.57225799560547,
      "activations/layer8_attention_weight_max": 46.51484298706055,
      "activations/layer8_attention_weight_min": -49.48598098754883,
      "activations/layer9_attention_weight_max": 49.123565673828125,
      "activations/layer9_attention_weight_min": -55.29291915893555,
      "epoch": 10.01,
      "learning_rate": 8.62875e-05,
      "loss": 2.7512,
      "step": 172350
    },
    {
      "activations/layer0_attention_weight_max": 15.330698013305664,
      "activations/layer0_attention_weight_min": -13.850032806396484,
      "activations/layer10_attention_weight_max": 37.59170150756836,
      "activations/layer10_attention_weight_min": -36.47911071777344,
      "activations/layer11_attention_weight_max": 31.13698387145996,
      "activations/layer11_attention_weight_min": -32.21034240722656,
      "activations/layer12_attention_weight_max": 22.118579864501953,
      "activations/layer12_attention_weight_min": -29.90619659423828,
      "activations/layer13_attention_weight_max": 41.903167724609375,
      "activations/layer13_attention_weight_min": -34.390316009521484,
      "activations/layer14_attention_weight_max": 39.41012191772461,
      "activations/layer14_attention_weight_min": -31.269201278686523,
      "activations/layer15_attention_weight_max": 36.852725982666016,
      "activations/layer15_attention_weight_min": -30.41884422302246,
      "activations/layer16_attention_weight_max": 37.366973876953125,
      "activations/layer16_attention_weight_min": -31.003442764282227,
      "activations/layer17_attention_weight_max": 53.82344436645508,
      "activations/layer17_attention_weight_min": -45.25189971923828,
      "activations/layer18_attention_weight_max": 52.40463638305664,
      "activations/layer18_attention_weight_min": -36.98883056640625,
      "activations/layer19_attention_weight_max": 21.708444595336914,
      "activations/layer19_attention_weight_min": -22.642852783203125,
      "activations/layer1_attention_weight_max": 15.967482566833496,
      "activations/layer1_attention_weight_min": -15.525131225585938,
      "activations/layer20_attention_weight_max": 22.600133895874023,
      "activations/layer20_attention_weight_min": -22.336864471435547,
      "activations/layer21_attention_weight_max": 41.86711502075195,
      "activations/layer21_attention_weight_min": -23.547239303588867,
      "activations/layer22_attention_weight_max": 30.956079483032227,
      "activations/layer22_attention_weight_min": -24.108003616333008,
      "activations/layer23_attention_weight_max": 41.385719299316406,
      "activations/layer23_attention_weight_min": -23.350849151611328,
      "activations/layer2_attention_weight_max": 31.86294937133789,
      "activations/layer2_attention_weight_min": -30.73847198486328,
      "activations/layer3_attention_weight_max": 93.19810485839844,
      "activations/layer3_attention_weight_min": -94.6873779296875,
      "activations/layer4_attention_weight_max": 102.08750915527344,
      "activations/layer4_attention_weight_min": -96.77182006835938,
      "activations/layer5_attention_weight_max": 77.50920867919922,
      "activations/layer5_attention_weight_min": -78.44442749023438,
      "activations/layer6_attention_weight_max": 55.38270950317383,
      "activations/layer6_attention_weight_min": -54.640384674072266,
      "activations/layer7_attention_weight_max": 75.31045532226562,
      "activations/layer7_attention_weight_min": -76.70691680908203,
      "activations/layer8_attention_weight_max": 45.01347732543945,
      "activations/layer8_attention_weight_min": -48.42998123168945,
      "activations/layer9_attention_weight_max": 46.8908805847168,
      "activations/layer9_attention_weight_min": -52.82662582397461,
      "epoch": 10.02,
      "learning_rate": 8.626856060606059e-05,
      "loss": 2.7718,
      "step": 172400
    },
    {
      "activations/layer0_attention_weight_max": 14.773283004760742,
      "activations/layer0_attention_weight_min": -14.455674171447754,
      "activations/layer10_attention_weight_max": 37.69280242919922,
      "activations/layer10_attention_weight_min": -34.870262145996094,
      "activations/layer11_attention_weight_max": 34.57064437866211,
      "activations/layer11_attention_weight_min": -31.388240814208984,
      "activations/layer12_attention_weight_max": 19.762008666992188,
      "activations/layer12_attention_weight_min": -25.405778884887695,
      "activations/layer13_attention_weight_max": 38.52169418334961,
      "activations/layer13_attention_weight_min": -29.45469093322754,
      "activations/layer14_attention_weight_max": 36.68790054321289,
      "activations/layer14_attention_weight_min": -30.134973526000977,
      "activations/layer15_attention_weight_max": 33.73182678222656,
      "activations/layer15_attention_weight_min": -29.729656219482422,
      "activations/layer16_attention_weight_max": 30.59618377685547,
      "activations/layer16_attention_weight_min": -29.293432235717773,
      "activations/layer17_attention_weight_max": 55.85359191894531,
      "activations/layer17_attention_weight_min": -42.637062072753906,
      "activations/layer18_attention_weight_max": 47.55763244628906,
      "activations/layer18_attention_weight_min": -36.698875427246094,
      "activations/layer19_attention_weight_max": 23.76984214782715,
      "activations/layer19_attention_weight_min": -22.622364044189453,
      "activations/layer1_attention_weight_max": 16.734766006469727,
      "activations/layer1_attention_weight_min": -14.949360847473145,
      "activations/layer20_attention_weight_max": 23.51008415222168,
      "activations/layer20_attention_weight_min": -25.071754455566406,
      "activations/layer21_attention_weight_max": 38.29018020629883,
      "activations/layer21_attention_weight_min": -21.946990966796875,
      "activations/layer22_attention_weight_max": 31.162153244018555,
      "activations/layer22_attention_weight_min": -24.662521362304688,
      "activations/layer23_attention_weight_max": 36.895870208740234,
      "activations/layer23_attention_weight_min": -23.17624282836914,
      "activations/layer2_attention_weight_max": 32.54057312011719,
      "activations/layer2_attention_weight_min": -30.976856231689453,
      "activations/layer3_attention_weight_max": 92.66542053222656,
      "activations/layer3_attention_weight_min": -92.6754150390625,
      "activations/layer4_attention_weight_max": 106.19319915771484,
      "activations/layer4_attention_weight_min": -98.1364974975586,
      "activations/layer5_attention_weight_max": 80.93038940429688,
      "activations/layer5_attention_weight_min": -78.86787414550781,
      "activations/layer6_attention_weight_max": 56.32517623901367,
      "activations/layer6_attention_weight_min": -53.607017517089844,
      "activations/layer7_attention_weight_max": 79.74714660644531,
      "activations/layer7_attention_weight_min": -72.62127685546875,
      "activations/layer8_attention_weight_max": 46.86555862426758,
      "activations/layer8_attention_weight_min": -47.78752136230469,
      "activations/layer9_attention_weight_max": 48.483970642089844,
      "activations/layer9_attention_weight_min": -48.19590377807617,
      "epoch": 10.02,
      "learning_rate": 8.62496212121212e-05,
      "loss": 2.7689,
      "step": 172450
    },
    {
      "activations/layer0_attention_weight_max": 14.703083992004395,
      "activations/layer0_attention_weight_min": -14.079381942749023,
      "activations/layer10_attention_weight_max": 36.89638137817383,
      "activations/layer10_attention_weight_min": -36.45766830444336,
      "activations/layer11_attention_weight_max": 33.5919303894043,
      "activations/layer11_attention_weight_min": -36.19221115112305,
      "activations/layer12_attention_weight_max": 18.723081588745117,
      "activations/layer12_attention_weight_min": -26.97238540649414,
      "activations/layer13_attention_weight_max": 36.736854553222656,
      "activations/layer13_attention_weight_min": -27.883575439453125,
      "activations/layer14_attention_weight_max": 37.02079772949219,
      "activations/layer14_attention_weight_min": -29.46600914001465,
      "activations/layer15_attention_weight_max": 31.6875,
      "activations/layer15_attention_weight_min": -29.236547470092773,
      "activations/layer16_attention_weight_max": 29.02914047241211,
      "activations/layer16_attention_weight_min": -27.34613800048828,
      "activations/layer17_attention_weight_max": 51.318824768066406,
      "activations/layer17_attention_weight_min": -41.96477127075195,
      "activations/layer18_attention_weight_max": 46.990684509277344,
      "activations/layer18_attention_weight_min": -35.73134231567383,
      "activations/layer19_attention_weight_max": 23.316722869873047,
      "activations/layer19_attention_weight_min": -21.76114273071289,
      "activations/layer1_attention_weight_max": 16.60211753845215,
      "activations/layer1_attention_weight_min": -16.460895538330078,
      "activations/layer20_attention_weight_max": 22.572477340698242,
      "activations/layer20_attention_weight_min": -24.029386520385742,
      "activations/layer21_attention_weight_max": 39.18848419189453,
      "activations/layer21_attention_weight_min": -24.48929214477539,
      "activations/layer22_attention_weight_max": 28.846725463867188,
      "activations/layer22_attention_weight_min": -25.380796432495117,
      "activations/layer23_attention_weight_max": 35.309146881103516,
      "activations/layer23_attention_weight_min": -24.22161865234375,
      "activations/layer2_attention_weight_max": 33.0379638671875,
      "activations/layer2_attention_weight_min": -35.091400146484375,
      "activations/layer3_attention_weight_max": 93.48064422607422,
      "activations/layer3_attention_weight_min": -103.87934112548828,
      "activations/layer4_attention_weight_max": 105.41301727294922,
      "activations/layer4_attention_weight_min": -102.80506896972656,
      "activations/layer5_attention_weight_max": 80.60942840576172,
      "activations/layer5_attention_weight_min": -82.45581817626953,
      "activations/layer6_attention_weight_max": 55.48064422607422,
      "activations/layer6_attention_weight_min": -58.191162109375,
      "activations/layer7_attention_weight_max": 84.31917572021484,
      "activations/layer7_attention_weight_min": -78.92314147949219,
      "activations/layer8_attention_weight_max": 48.57368850708008,
      "activations/layer8_attention_weight_min": -49.19282913208008,
      "activations/layer9_attention_weight_max": 52.33632278442383,
      "activations/layer9_attention_weight_min": -51.858131408691406,
      "epoch": 10.02,
      "learning_rate": 8.623068181818182e-05,
      "loss": 2.7589,
      "step": 172500
    },
    {
      "activations/layer0_attention_weight_max": 14.695061683654785,
      "activations/layer0_attention_weight_min": -14.572674751281738,
      "activations/layer10_attention_weight_max": 38.11339569091797,
      "activations/layer10_attention_weight_min": -37.32365798950195,
      "activations/layer11_attention_weight_max": 31.560697555541992,
      "activations/layer11_attention_weight_min": -32.12055587768555,
      "activations/layer12_attention_weight_max": 19.53671646118164,
      "activations/layer12_attention_weight_min": -23.903018951416016,
      "activations/layer13_attention_weight_max": 47.224056243896484,
      "activations/layer13_attention_weight_min": -34.568748474121094,
      "activations/layer14_attention_weight_max": 46.28830337524414,
      "activations/layer14_attention_weight_min": -32.62272262573242,
      "activations/layer15_attention_weight_max": 41.75849914550781,
      "activations/layer15_attention_weight_min": -28.920856475830078,
      "activations/layer16_attention_weight_max": 30.207677841186523,
      "activations/layer16_attention_weight_min": -30.49949836730957,
      "activations/layer17_attention_weight_max": 53.40928268432617,
      "activations/layer17_attention_weight_min": -45.757545471191406,
      "activations/layer18_attention_weight_max": 46.40800094604492,
      "activations/layer18_attention_weight_min": -37.183265686035156,
      "activations/layer19_attention_weight_max": 22.471092224121094,
      "activations/layer19_attention_weight_min": -21.996084213256836,
      "activations/layer1_attention_weight_max": 16.491151809692383,
      "activations/layer1_attention_weight_min": -15.406702041625977,
      "activations/layer20_attention_weight_max": 22.964759826660156,
      "activations/layer20_attention_weight_min": -21.66925811767578,
      "activations/layer21_attention_weight_max": 42.011962890625,
      "activations/layer21_attention_weight_min": -26.78683853149414,
      "activations/layer22_attention_weight_max": 29.829383850097656,
      "activations/layer22_attention_weight_min": -25.010204315185547,
      "activations/layer23_attention_weight_max": 36.36492156982422,
      "activations/layer23_attention_weight_min": -21.68947982788086,
      "activations/layer2_attention_weight_max": 33.368324279785156,
      "activations/layer2_attention_weight_min": -29.780685424804688,
      "activations/layer3_attention_weight_max": 91.799072265625,
      "activations/layer3_attention_weight_min": -94.00823211669922,
      "activations/layer4_attention_weight_max": 99.09062957763672,
      "activations/layer4_attention_weight_min": -92.99790954589844,
      "activations/layer5_attention_weight_max": 77.64856719970703,
      "activations/layer5_attention_weight_min": -81.65868377685547,
      "activations/layer6_attention_weight_max": 55.79043960571289,
      "activations/layer6_attention_weight_min": -54.223854064941406,
      "activations/layer7_attention_weight_max": 81.12055206298828,
      "activations/layer7_attention_weight_min": -77.57980346679688,
      "activations/layer8_attention_weight_max": 47.181034088134766,
      "activations/layer8_attention_weight_min": -48.85192108154297,
      "activations/layer9_attention_weight_max": 50.33782958984375,
      "activations/layer9_attention_weight_min": -50.91273880004883,
      "epoch": 10.03,
      "learning_rate": 8.621174242424241e-05,
      "loss": 2.7849,
      "step": 172550
    },
    {
      "activations/layer0_attention_weight_max": 14.95200252532959,
      "activations/layer0_attention_weight_min": -14.129650115966797,
      "activations/layer10_attention_weight_max": 39.59752655029297,
      "activations/layer10_attention_weight_min": -36.73305892944336,
      "activations/layer11_attention_weight_max": 35.70643615722656,
      "activations/layer11_attention_weight_min": -34.962371826171875,
      "activations/layer12_attention_weight_max": 21.451181411743164,
      "activations/layer12_attention_weight_min": -25.67296600341797,
      "activations/layer13_attention_weight_max": 41.27669143676758,
      "activations/layer13_attention_weight_min": -33.50616455078125,
      "activations/layer14_attention_weight_max": 39.45587921142578,
      "activations/layer14_attention_weight_min": -33.174434661865234,
      "activations/layer15_attention_weight_max": 37.1183967590332,
      "activations/layer15_attention_weight_min": -32.761268615722656,
      "activations/layer16_attention_weight_max": 38.92821502685547,
      "activations/layer16_attention_weight_min": -33.33019256591797,
      "activations/layer17_attention_weight_max": 58.61687088012695,
      "activations/layer17_attention_weight_min": -46.71266555786133,
      "activations/layer18_attention_weight_max": 52.87113571166992,
      "activations/layer18_attention_weight_min": -39.83549118041992,
      "activations/layer19_attention_weight_max": 22.32360076904297,
      "activations/layer19_attention_weight_min": -23.6011905670166,
      "activations/layer1_attention_weight_max": 17.517501831054688,
      "activations/layer1_attention_weight_min": -18.88345718383789,
      "activations/layer20_attention_weight_max": 22.60959243774414,
      "activations/layer20_attention_weight_min": -24.458415985107422,
      "activations/layer21_attention_weight_max": 40.55015563964844,
      "activations/layer21_attention_weight_min": -24.15593719482422,
      "activations/layer22_attention_weight_max": 35.69997024536133,
      "activations/layer22_attention_weight_min": -26.738374710083008,
      "activations/layer23_attention_weight_max": 40.76398849487305,
      "activations/layer23_attention_weight_min": -22.25115966796875,
      "activations/layer2_attention_weight_max": 32.973876953125,
      "activations/layer2_attention_weight_min": -31.4383602142334,
      "activations/layer3_attention_weight_max": 94.61693572998047,
      "activations/layer3_attention_weight_min": -96.00444030761719,
      "activations/layer4_attention_weight_max": 101.55088806152344,
      "activations/layer4_attention_weight_min": -96.58824157714844,
      "activations/layer5_attention_weight_max": 77.57783508300781,
      "activations/layer5_attention_weight_min": -77.05133819580078,
      "activations/layer6_attention_weight_max": 55.21095657348633,
      "activations/layer6_attention_weight_min": -54.202598571777344,
      "activations/layer7_attention_weight_max": 79.48883819580078,
      "activations/layer7_attention_weight_min": -77.25917053222656,
      "activations/layer8_attention_weight_max": 47.21440505981445,
      "activations/layer8_attention_weight_min": -47.208885192871094,
      "activations/layer9_attention_weight_max": 53.9489631652832,
      "activations/layer9_attention_weight_min": -49.65179443359375,
      "epoch": 10.03,
      "learning_rate": 8.619280303030302e-05,
      "loss": 2.7613,
      "step": 172600
    },
    {
      "activations/layer0_attention_weight_max": 15.623210906982422,
      "activations/layer0_attention_weight_min": -14.23468017578125,
      "activations/layer10_attention_weight_max": 36.71461868286133,
      "activations/layer10_attention_weight_min": -35.899173736572266,
      "activations/layer11_attention_weight_max": 35.28036117553711,
      "activations/layer11_attention_weight_min": -35.031707763671875,
      "activations/layer12_attention_weight_max": 17.685773849487305,
      "activations/layer12_attention_weight_min": -24.533971786499023,
      "activations/layer13_attention_weight_max": 42.79838562011719,
      "activations/layer13_attention_weight_min": -32.774742126464844,
      "activations/layer14_attention_weight_max": 36.47880172729492,
      "activations/layer14_attention_weight_min": -32.36860656738281,
      "activations/layer15_attention_weight_max": 33.27177810668945,
      "activations/layer15_attention_weight_min": -31.753877639770508,
      "activations/layer16_attention_weight_max": 31.79644012451172,
      "activations/layer16_attention_weight_min": -30.718786239624023,
      "activations/layer17_attention_weight_max": 50.1964111328125,
      "activations/layer17_attention_weight_min": -45.68606948852539,
      "activations/layer18_attention_weight_max": 45.94937515258789,
      "activations/layer18_attention_weight_min": -39.670108795166016,
      "activations/layer19_attention_weight_max": 22.177547454833984,
      "activations/layer19_attention_weight_min": -22.59002685546875,
      "activations/layer1_attention_weight_max": 16.980955123901367,
      "activations/layer1_attention_weight_min": -16.64838409423828,
      "activations/layer20_attention_weight_max": 25.66775131225586,
      "activations/layer20_attention_weight_min": -24.068111419677734,
      "activations/layer21_attention_weight_max": 42.580833435058594,
      "activations/layer21_attention_weight_min": -24.04594612121582,
      "activations/layer22_attention_weight_max": 31.403316497802734,
      "activations/layer22_attention_weight_min": -26.41820526123047,
      "activations/layer23_attention_weight_max": 37.46324157714844,
      "activations/layer23_attention_weight_min": -24.59139060974121,
      "activations/layer2_attention_weight_max": 33.36277389526367,
      "activations/layer2_attention_weight_min": -32.79957580566406,
      "activations/layer3_attention_weight_max": 97.55291748046875,
      "activations/layer3_attention_weight_min": -99.97840881347656,
      "activations/layer4_attention_weight_max": 106.7889404296875,
      "activations/layer4_attention_weight_min": -98.91756439208984,
      "activations/layer5_attention_weight_max": 83.16789245605469,
      "activations/layer5_attention_weight_min": -82.18582153320312,
      "activations/layer6_attention_weight_max": 57.066162109375,
      "activations/layer6_attention_weight_min": -53.98271560668945,
      "activations/layer7_attention_weight_max": 78.81248474121094,
      "activations/layer7_attention_weight_min": -76.28107452392578,
      "activations/layer8_attention_weight_max": 47.943817138671875,
      "activations/layer8_attention_weight_min": -49.8090934753418,
      "activations/layer9_attention_weight_max": 49.81332778930664,
      "activations/layer9_attention_weight_min": -50.349334716796875,
      "epoch": 10.03,
      "learning_rate": 8.617386363636363e-05,
      "loss": 2.7614,
      "step": 172650
    },
    {
      "activations/layer0_attention_weight_max": 15.518956184387207,
      "activations/layer0_attention_weight_min": -14.517930030822754,
      "activations/layer10_attention_weight_max": 40.023651123046875,
      "activations/layer10_attention_weight_min": -38.63874053955078,
      "activations/layer11_attention_weight_max": 34.62059783935547,
      "activations/layer11_attention_weight_min": -33.694610595703125,
      "activations/layer12_attention_weight_max": 21.597795486450195,
      "activations/layer12_attention_weight_min": -24.140396118164062,
      "activations/layer13_attention_weight_max": 49.93867874145508,
      "activations/layer13_attention_weight_min": -28.50531768798828,
      "activations/layer14_attention_weight_max": 37.949066162109375,
      "activations/layer14_attention_weight_min": -32.872989654541016,
      "activations/layer15_attention_weight_max": 42.50912094116211,
      "activations/layer15_attention_weight_min": -31.298368453979492,
      "activations/layer16_attention_weight_max": 31.371837615966797,
      "activations/layer16_attention_weight_min": -29.331714630126953,
      "activations/layer17_attention_weight_max": 54.262969970703125,
      "activations/layer17_attention_weight_min": -43.89374542236328,
      "activations/layer18_attention_weight_max": 50.91855239868164,
      "activations/layer18_attention_weight_min": -36.7125244140625,
      "activations/layer19_attention_weight_max": 24.671695709228516,
      "activations/layer19_attention_weight_min": -21.99266242980957,
      "activations/layer1_attention_weight_max": 16.151018142700195,
      "activations/layer1_attention_weight_min": -16.393224716186523,
      "activations/layer20_attention_weight_max": 23.568222045898438,
      "activations/layer20_attention_weight_min": -22.691749572753906,
      "activations/layer21_attention_weight_max": 41.65682601928711,
      "activations/layer21_attention_weight_min": -21.891475677490234,
      "activations/layer22_attention_weight_max": 31.095849990844727,
      "activations/layer22_attention_weight_min": -24.35457420349121,
      "activations/layer23_attention_weight_max": 38.3029670715332,
      "activations/layer23_attention_weight_min": -21.29192352294922,
      "activations/layer2_attention_weight_max": 33.667240142822266,
      "activations/layer2_attention_weight_min": -33.26520538330078,
      "activations/layer3_attention_weight_max": 94.85912322998047,
      "activations/layer3_attention_weight_min": -105.03768920898438,
      "activations/layer4_attention_weight_max": 107.85619354248047,
      "activations/layer4_attention_weight_min": -107.26039123535156,
      "activations/layer5_attention_weight_max": 80.98086547851562,
      "activations/layer5_attention_weight_min": -80.79800415039062,
      "activations/layer6_attention_weight_max": 58.963382720947266,
      "activations/layer6_attention_weight_min": -56.79777526855469,
      "activations/layer7_attention_weight_max": 79.62559509277344,
      "activations/layer7_attention_weight_min": -78.24232482910156,
      "activations/layer8_attention_weight_max": 47.999656677246094,
      "activations/layer8_attention_weight_min": -50.83682632446289,
      "activations/layer9_attention_weight_max": 52.095863342285156,
      "activations/layer9_attention_weight_min": -52.55678939819336,
      "epoch": 10.03,
      "learning_rate": 8.615492424242423e-05,
      "loss": 2.7747,
      "step": 172700
    },
    {
      "activations/layer0_attention_weight_max": 14.994890213012695,
      "activations/layer0_attention_weight_min": -14.633537292480469,
      "activations/layer10_attention_weight_max": 34.431427001953125,
      "activations/layer10_attention_weight_min": -35.186527252197266,
      "activations/layer11_attention_weight_max": 29.910886764526367,
      "activations/layer11_attention_weight_min": -32.36956024169922,
      "activations/layer12_attention_weight_max": 17.424346923828125,
      "activations/layer12_attention_weight_min": -22.70045280456543,
      "activations/layer13_attention_weight_max": 32.950096130371094,
      "activations/layer13_attention_weight_min": -29.72353172302246,
      "activations/layer14_attention_weight_max": 36.350372314453125,
      "activations/layer14_attention_weight_min": -29.84625244140625,
      "activations/layer15_attention_weight_max": 31.68526268005371,
      "activations/layer15_attention_weight_min": -29.577533721923828,
      "activations/layer16_attention_weight_max": 31.433183670043945,
      "activations/layer16_attention_weight_min": -28.16390609741211,
      "activations/layer17_attention_weight_max": 54.13956832885742,
      "activations/layer17_attention_weight_min": -46.36165237426758,
      "activations/layer18_attention_weight_max": 44.65251922607422,
      "activations/layer18_attention_weight_min": -38.87739562988281,
      "activations/layer19_attention_weight_max": 22.712800979614258,
      "activations/layer19_attention_weight_min": -22.816404342651367,
      "activations/layer1_attention_weight_max": 16.079418182373047,
      "activations/layer1_attention_weight_min": -14.087621688842773,
      "activations/layer20_attention_weight_max": 23.345115661621094,
      "activations/layer20_attention_weight_min": -22.94928550720215,
      "activations/layer21_attention_weight_max": 38.051246643066406,
      "activations/layer21_attention_weight_min": -27.001113891601562,
      "activations/layer22_attention_weight_max": 29.352054595947266,
      "activations/layer22_attention_weight_min": -27.046875,
      "activations/layer23_attention_weight_max": 35.544517517089844,
      "activations/layer23_attention_weight_min": -26.49970054626465,
      "activations/layer2_attention_weight_max": 32.53820037841797,
      "activations/layer2_attention_weight_min": -30.663814544677734,
      "activations/layer3_attention_weight_max": 93.0275650024414,
      "activations/layer3_attention_weight_min": -92.1832504272461,
      "activations/layer4_attention_weight_max": 98.75054168701172,
      "activations/layer4_attention_weight_min": -95.10945892333984,
      "activations/layer5_attention_weight_max": 76.43439483642578,
      "activations/layer5_attention_weight_min": -76.67867279052734,
      "activations/layer6_attention_weight_max": 54.413352966308594,
      "activations/layer6_attention_weight_min": -52.89353561401367,
      "activations/layer7_attention_weight_max": 73.97344207763672,
      "activations/layer7_attention_weight_min": -74.71825408935547,
      "activations/layer8_attention_weight_max": 44.1954345703125,
      "activations/layer8_attention_weight_min": -47.90736770629883,
      "activations/layer9_attention_weight_max": 46.834625244140625,
      "activations/layer9_attention_weight_min": -48.470699310302734,
      "epoch": 10.04,
      "learning_rate": 8.613598484848484e-05,
      "loss": 2.7691,
      "step": 172750
    },
    {
      "activations/layer0_attention_weight_max": 15.409357070922852,
      "activations/layer0_attention_weight_min": -13.988930702209473,
      "activations/layer10_attention_weight_max": 36.09012222290039,
      "activations/layer10_attention_weight_min": -35.359622955322266,
      "activations/layer11_attention_weight_max": 31.852209091186523,
      "activations/layer11_attention_weight_min": -31.172534942626953,
      "activations/layer12_attention_weight_max": 22.149383544921875,
      "activations/layer12_attention_weight_min": -22.77688980102539,
      "activations/layer13_attention_weight_max": 40.15439987182617,
      "activations/layer13_attention_weight_min": -27.219486236572266,
      "activations/layer14_attention_weight_max": 40.06944274902344,
      "activations/layer14_attention_weight_min": -30.641414642333984,
      "activations/layer15_attention_weight_max": 32.70970916748047,
      "activations/layer15_attention_weight_min": -28.26140594482422,
      "activations/layer16_attention_weight_max": 32.19439697265625,
      "activations/layer16_attention_weight_min": -29.3358154296875,
      "activations/layer17_attention_weight_max": 53.86327362060547,
      "activations/layer17_attention_weight_min": -42.00801467895508,
      "activations/layer18_attention_weight_max": 46.96870040893555,
      "activations/layer18_attention_weight_min": -34.78666687011719,
      "activations/layer19_attention_weight_max": 25.839859008789062,
      "activations/layer19_attention_weight_min": -20.862934112548828,
      "activations/layer1_attention_weight_max": 16.70988655090332,
      "activations/layer1_attention_weight_min": -15.198644638061523,
      "activations/layer20_attention_weight_max": 22.91148567199707,
      "activations/layer20_attention_weight_min": -20.262012481689453,
      "activations/layer21_attention_weight_max": 35.41604995727539,
      "activations/layer21_attention_weight_min": -22.068368911743164,
      "activations/layer22_attention_weight_max": 31.7369384765625,
      "activations/layer22_attention_weight_min": -26.932018280029297,
      "activations/layer23_attention_weight_max": 38.19821548461914,
      "activations/layer23_attention_weight_min": -25.801109313964844,
      "activations/layer2_attention_weight_max": 31.37133026123047,
      "activations/layer2_attention_weight_min": -29.81024932861328,
      "activations/layer3_attention_weight_max": 91.36846923828125,
      "activations/layer3_attention_weight_min": -95.84400939941406,
      "activations/layer4_attention_weight_max": 97.64405059814453,
      "activations/layer4_attention_weight_min": -96.85951232910156,
      "activations/layer5_attention_weight_max": 74.90074157714844,
      "activations/layer5_attention_weight_min": -79.48587036132812,
      "activations/layer6_attention_weight_max": 54.041622161865234,
      "activations/layer6_attention_weight_min": -54.868927001953125,
      "activations/layer7_attention_weight_max": 76.83320617675781,
      "activations/layer7_attention_weight_min": -73.14442443847656,
      "activations/layer8_attention_weight_max": 44.61567306518555,
      "activations/layer8_attention_weight_min": -48.790679931640625,
      "activations/layer9_attention_weight_max": 47.89348220825195,
      "activations/layer9_attention_weight_min": -48.9958610534668,
      "epoch": 10.04,
      "learning_rate": 8.611742424242423e-05,
      "loss": 2.7745,
      "step": 172800
    },
    {
      "activations/layer0_attention_weight_max": 15.470086097717285,
      "activations/layer0_attention_weight_min": -13.855828285217285,
      "activations/layer10_attention_weight_max": 37.23383331298828,
      "activations/layer10_attention_weight_min": -36.41571807861328,
      "activations/layer11_attention_weight_max": 31.938274383544922,
      "activations/layer11_attention_weight_min": -31.139272689819336,
      "activations/layer12_attention_weight_max": 17.74053382873535,
      "activations/layer12_attention_weight_min": -27.57499122619629,
      "activations/layer13_attention_weight_max": 40.63069534301758,
      "activations/layer13_attention_weight_min": -31.21480369567871,
      "activations/layer14_attention_weight_max": 35.44234848022461,
      "activations/layer14_attention_weight_min": -30.814855575561523,
      "activations/layer15_attention_weight_max": 31.715917587280273,
      "activations/layer15_attention_weight_min": -29.671525955200195,
      "activations/layer16_attention_weight_max": 28.729028701782227,
      "activations/layer16_attention_weight_min": -28.015796661376953,
      "activations/layer17_attention_weight_max": 52.01081085205078,
      "activations/layer17_attention_weight_min": -44.932655334472656,
      "activations/layer18_attention_weight_max": 43.09980392456055,
      "activations/layer18_attention_weight_min": -39.25994110107422,
      "activations/layer19_attention_weight_max": 20.492820739746094,
      "activations/layer19_attention_weight_min": -23.595726013183594,
      "activations/layer1_attention_weight_max": 16.44510269165039,
      "activations/layer1_attention_weight_min": -15.84704875946045,
      "activations/layer20_attention_weight_max": 22.029830932617188,
      "activations/layer20_attention_weight_min": -23.050790786743164,
      "activations/layer21_attention_weight_max": 34.35129165649414,
      "activations/layer21_attention_weight_min": -22.80866050720215,
      "activations/layer22_attention_weight_max": 29.14202880859375,
      "activations/layer22_attention_weight_min": -25.657508850097656,
      "activations/layer23_attention_weight_max": 35.8009147644043,
      "activations/layer23_attention_weight_min": -24.366331100463867,
      "activations/layer2_attention_weight_max": 31.94271469116211,
      "activations/layer2_attention_weight_min": -30.429418563842773,
      "activations/layer3_attention_weight_max": 96.61064147949219,
      "activations/layer3_attention_weight_min": -97.2961654663086,
      "activations/layer4_attention_weight_max": 100.30110931396484,
      "activations/layer4_attention_weight_min": -96.44811248779297,
      "activations/layer5_attention_weight_max": 74.61898803710938,
      "activations/layer5_attention_weight_min": -77.85546112060547,
      "activations/layer6_attention_weight_max": 53.95891189575195,
      "activations/layer6_attention_weight_min": -52.731712341308594,
      "activations/layer7_attention_weight_max": 74.81553649902344,
      "activations/layer7_attention_weight_min": -71.95289611816406,
      "activations/layer8_attention_weight_max": 46.68581008911133,
      "activations/layer8_attention_weight_min": -47.73729705810547,
      "activations/layer9_attention_weight_max": 48.52915573120117,
      "activations/layer9_attention_weight_min": -49.74738693237305,
      "epoch": 10.04,
      "learning_rate": 8.609848484848483e-05,
      "loss": 2.7738,
      "step": 172850
    },
    {
      "activations/layer0_attention_weight_max": 15.959157943725586,
      "activations/layer0_attention_weight_min": -13.78276252746582,
      "activations/layer10_attention_weight_max": 38.028541564941406,
      "activations/layer10_attention_weight_min": -36.05150604248047,
      "activations/layer11_attention_weight_max": 34.031341552734375,
      "activations/layer11_attention_weight_min": -32.307899475097656,
      "activations/layer12_attention_weight_max": 18.651609420776367,
      "activations/layer12_attention_weight_min": -24.41997528076172,
      "activations/layer13_attention_weight_max": 40.45600128173828,
      "activations/layer13_attention_weight_min": -29.946645736694336,
      "activations/layer14_attention_weight_max": 39.68242645263672,
      "activations/layer14_attention_weight_min": -29.738325119018555,
      "activations/layer15_attention_weight_max": 34.158409118652344,
      "activations/layer15_attention_weight_min": -30.53704261779785,
      "activations/layer16_attention_weight_max": 34.26169204711914,
      "activations/layer16_attention_weight_min": -29.41175651550293,
      "activations/layer17_attention_weight_max": 53.07267761230469,
      "activations/layer17_attention_weight_min": -45.99853515625,
      "activations/layer18_attention_weight_max": 49.28504180908203,
      "activations/layer18_attention_weight_min": -40.87364959716797,
      "activations/layer19_attention_weight_max": 21.609067916870117,
      "activations/layer19_attention_weight_min": -21.445486068725586,
      "activations/layer1_attention_weight_max": 15.557709693908691,
      "activations/layer1_attention_weight_min": -14.257438659667969,
      "activations/layer20_attention_weight_max": 21.39903450012207,
      "activations/layer20_attention_weight_min": -23.13762855529785,
      "activations/layer21_attention_weight_max": 35.77427291870117,
      "activations/layer21_attention_weight_min": -23.026865005493164,
      "activations/layer22_attention_weight_max": 32.74909973144531,
      "activations/layer22_attention_weight_min": -27.072664260864258,
      "activations/layer23_attention_weight_max": 41.04023742675781,
      "activations/layer23_attention_weight_min": -22.470380783081055,
      "activations/layer2_attention_weight_max": 31.636974334716797,
      "activations/layer2_attention_weight_min": -31.418289184570312,
      "activations/layer3_attention_weight_max": 90.02330017089844,
      "activations/layer3_attention_weight_min": -94.95199584960938,
      "activations/layer4_attention_weight_max": 104.4023666381836,
      "activations/layer4_attention_weight_min": -98.64293670654297,
      "activations/layer5_attention_weight_max": 79.14054870605469,
      "activations/layer5_attention_weight_min": -82.2051773071289,
      "activations/layer6_attention_weight_max": 57.49072265625,
      "activations/layer6_attention_weight_min": -56.12565231323242,
      "activations/layer7_attention_weight_max": 81.89466857910156,
      "activations/layer7_attention_weight_min": -74.45366668701172,
      "activations/layer8_attention_weight_max": 49.880126953125,
      "activations/layer8_attention_weight_min": -48.9687385559082,
      "activations/layer9_attention_weight_max": 48.44213104248047,
      "activations/layer9_attention_weight_min": -48.11996078491211,
      "epoch": 10.05,
      "learning_rate": 8.607954545454545e-05,
      "loss": 2.7795,
      "step": 172900
    },
    {
      "activations/layer0_attention_weight_max": 15.778573036193848,
      "activations/layer0_attention_weight_min": -14.096529960632324,
      "activations/layer10_attention_weight_max": 35.20643997192383,
      "activations/layer10_attention_weight_min": -36.12933349609375,
      "activations/layer11_attention_weight_max": 30.303813934326172,
      "activations/layer11_attention_weight_min": -34.184844970703125,
      "activations/layer12_attention_weight_max": 16.381465911865234,
      "activations/layer12_attention_weight_min": -26.62407875061035,
      "activations/layer13_attention_weight_max": 41.746734619140625,
      "activations/layer13_attention_weight_min": -29.64854621887207,
      "activations/layer14_attention_weight_max": 36.05379867553711,
      "activations/layer14_attention_weight_min": -29.907869338989258,
      "activations/layer15_attention_weight_max": 37.175621032714844,
      "activations/layer15_attention_weight_min": -30.46315574645996,
      "activations/layer16_attention_weight_max": 30.148311614990234,
      "activations/layer16_attention_weight_min": -31.295299530029297,
      "activations/layer17_attention_weight_max": 53.7590446472168,
      "activations/layer17_attention_weight_min": -44.42799377441406,
      "activations/layer18_attention_weight_max": 47.00931930541992,
      "activations/layer18_attention_weight_min": -37.777034759521484,
      "activations/layer19_attention_weight_max": 21.0831298828125,
      "activations/layer19_attention_weight_min": -21.353376388549805,
      "activations/layer1_attention_weight_max": 18.56090545654297,
      "activations/layer1_attention_weight_min": -15.745355606079102,
      "activations/layer20_attention_weight_max": 21.284835815429688,
      "activations/layer20_attention_weight_min": -21.3297061920166,
      "activations/layer21_attention_weight_max": 36.50928497314453,
      "activations/layer21_attention_weight_min": -23.226539611816406,
      "activations/layer22_attention_weight_max": 29.12779998779297,
      "activations/layer22_attention_weight_min": -27.190385818481445,
      "activations/layer23_attention_weight_max": 38.58655548095703,
      "activations/layer23_attention_weight_min": -24.294174194335938,
      "activations/layer2_attention_weight_max": 32.039222717285156,
      "activations/layer2_attention_weight_min": -31.744253158569336,
      "activations/layer3_attention_weight_max": 97.03327941894531,
      "activations/layer3_attention_weight_min": -99.02118682861328,
      "activations/layer4_attention_weight_max": 100.62922668457031,
      "activations/layer4_attention_weight_min": -95.44632720947266,
      "activations/layer5_attention_weight_max": 73.96424865722656,
      "activations/layer5_attention_weight_min": -77.26253509521484,
      "activations/layer6_attention_weight_max": 52.93220901489258,
      "activations/layer6_attention_weight_min": -51.55229187011719,
      "activations/layer7_attention_weight_max": 75.76568603515625,
      "activations/layer7_attention_weight_min": -72.31731414794922,
      "activations/layer8_attention_weight_max": 43.11119842529297,
      "activations/layer8_attention_weight_min": -45.90133285522461,
      "activations/layer9_attention_weight_max": 48.842586517333984,
      "activations/layer9_attention_weight_min": -50.95607376098633,
      "epoch": 10.05,
      "learning_rate": 8.606098484848484e-05,
      "loss": 2.7726,
      "step": 172950
    },
    {
      "activations/layer0_attention_weight_max": 15.733986854553223,
      "activations/layer0_attention_weight_min": -14.086106300354004,
      "activations/layer10_attention_weight_max": 37.81692123413086,
      "activations/layer10_attention_weight_min": -36.318572998046875,
      "activations/layer11_attention_weight_max": 32.371734619140625,
      "activations/layer11_attention_weight_min": -33.11357116699219,
      "activations/layer12_attention_weight_max": 20.09946632385254,
      "activations/layer12_attention_weight_min": -21.951566696166992,
      "activations/layer13_attention_weight_max": 37.35517120361328,
      "activations/layer13_attention_weight_min": -30.97067642211914,
      "activations/layer14_attention_weight_max": 38.07313537597656,
      "activations/layer14_attention_weight_min": -30.127622604370117,
      "activations/layer15_attention_weight_max": 33.68391418457031,
      "activations/layer15_attention_weight_min": -30.817750930786133,
      "activations/layer16_attention_weight_max": 32.299964904785156,
      "activations/layer16_attention_weight_min": -29.618240356445312,
      "activations/layer17_attention_weight_max": 50.47587203979492,
      "activations/layer17_attention_weight_min": -44.81147384643555,
      "activations/layer18_attention_weight_max": 45.41429901123047,
      "activations/layer18_attention_weight_min": -36.64167404174805,
      "activations/layer19_attention_weight_max": 22.871755599975586,
      "activations/layer19_attention_weight_min": -21.206939697265625,
      "activations/layer1_attention_weight_max": 17.262134552001953,
      "activations/layer1_attention_weight_min": -15.355794906616211,
      "activations/layer20_attention_weight_max": 21.025896072387695,
      "activations/layer20_attention_weight_min": -21.379764556884766,
      "activations/layer21_attention_weight_max": 34.174842834472656,
      "activations/layer21_attention_weight_min": -22.111791610717773,
      "activations/layer22_attention_weight_max": 28.974458694458008,
      "activations/layer22_attention_weight_min": -26.3975772857666,
      "activations/layer23_attention_weight_max": 35.93627166748047,
      "activations/layer23_attention_weight_min": -23.461782455444336,
      "activations/layer2_attention_weight_max": 32.3233642578125,
      "activations/layer2_attention_weight_min": -31.139760971069336,
      "activations/layer3_attention_weight_max": 93.17630767822266,
      "activations/layer3_attention_weight_min": -93.13594055175781,
      "activations/layer4_attention_weight_max": 100.8492431640625,
      "activations/layer4_attention_weight_min": -97.0132827758789,
      "activations/layer5_attention_weight_max": 74.3841552734375,
      "activations/layer5_attention_weight_min": -76.50776672363281,
      "activations/layer6_attention_weight_max": 53.91957092285156,
      "activations/layer6_attention_weight_min": -53.53130340576172,
      "activations/layer7_attention_weight_max": 78.80048370361328,
      "activations/layer7_attention_weight_min": -75.95864868164062,
      "activations/layer8_attention_weight_max": 44.99019241333008,
      "activations/layer8_attention_weight_min": -47.0778923034668,
      "activations/layer9_attention_weight_max": 47.26969528198242,
      "activations/layer9_attention_weight_min": -48.43212890625,
      "epoch": 10.05,
      "learning_rate": 8.604204545454546e-05,
      "loss": 2.763,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_loss": 2.734375,
      "eval_runtime": 8.4989,
      "eval_samples_per_second": 505.24,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.4989,
      "eval_openwebtext_samples_per_second": 505.24,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_wikitext_loss": 2.966796875,
      "eval_wikitext_ppl": 19.429584428039178,
      "eval_wikitext_runtime": 1.9439,
      "eval_wikitext_samples_per_second": 234.584,
      "step": 173000
    },
    {
      "epoch": 10.05,
      "eval_lambada_loss": 2.611328125,
      "eval_lambada_ppl": 13.617124090346826,
      "eval_lambada_runtime": 9.7817,
      "eval_lambada_samples_per_second": 497.768,
      "step": 173000
    },
    {
      "activations/layer0_attention_weight_max": 15.127530097961426,
      "activations/layer0_attention_weight_min": -14.098051071166992,
      "activations/layer10_attention_weight_max": 36.685546875,
      "activations/layer10_attention_weight_min": -37.1566276550293,
      "activations/layer11_attention_weight_max": 30.524185180664062,
      "activations/layer11_attention_weight_min": -31.326040267944336,
      "activations/layer12_attention_weight_max": 18.55628776550293,
      "activations/layer12_attention_weight_min": -22.276962280273438,
      "activations/layer13_attention_weight_max": 36.20410919189453,
      "activations/layer13_attention_weight_min": -28.083133697509766,
      "activations/layer14_attention_weight_max": 35.79752731323242,
      "activations/layer14_attention_weight_min": -29.889328002929688,
      "activations/layer15_attention_weight_max": 31.599809646606445,
      "activations/layer15_attention_weight_min": -28.99333953857422,
      "activations/layer16_attention_weight_max": 30.35506248474121,
      "activations/layer16_attention_weight_min": -28.428104400634766,
      "activations/layer17_attention_weight_max": 51.09779739379883,
      "activations/layer17_attention_weight_min": -43.599464416503906,
      "activations/layer18_attention_weight_max": 50.25289535522461,
      "activations/layer18_attention_weight_min": -38.444671630859375,
      "activations/layer19_attention_weight_max": 21.736433029174805,
      "activations/layer19_attention_weight_min": -21.5649471282959,
      "activations/layer1_attention_weight_max": 16.603694915771484,
      "activations/layer1_attention_weight_min": -17.286714553833008,
      "activations/layer20_attention_weight_max": 21.56658363342285,
      "activations/layer20_attention_weight_min": -21.24648094177246,
      "activations/layer21_attention_weight_max": 39.967342376708984,
      "activations/layer21_attention_weight_min": -22.202177047729492,
      "activations/layer22_attention_weight_max": 35.88636016845703,
      "activations/layer22_attention_weight_min": -27.361387252807617,
      "activations/layer23_attention_weight_max": 36.28425598144531,
      "activations/layer23_attention_weight_min": -25.63625144958496,
      "activations/layer2_attention_weight_max": 31.762229919433594,
      "activations/layer2_attention_weight_min": -30.537593841552734,
      "activations/layer3_attention_weight_max": 93.1158676147461,
      "activations/layer3_attention_weight_min": -91.86937713623047,
      "activations/layer4_attention_weight_max": 97.60050201416016,
      "activations/layer4_attention_weight_min": -95.08736419677734,
      "activations/layer5_attention_weight_max": 73.50949096679688,
      "activations/layer5_attention_weight_min": -79.1165771484375,
      "activations/layer6_attention_weight_max": 52.39636993408203,
      "activations/layer6_attention_weight_min": -52.48588180541992,
      "activations/layer7_attention_weight_max": 76.88343048095703,
      "activations/layer7_attention_weight_min": -72.15050506591797,
      "activations/layer8_attention_weight_max": 44.32967758178711,
      "activations/layer8_attention_weight_min": -44.98537063598633,
      "activations/layer9_attention_weight_max": 49.547828674316406,
      "activations/layer9_attention_weight_min": -45.499935150146484,
      "epoch": 10.06,
      "learning_rate": 8.602310606060604e-05,
      "loss": 2.76,
      "step": 173050
    },
    {
      "activations/layer0_attention_weight_max": 16.2546443939209,
      "activations/layer0_attention_weight_min": -14.291851997375488,
      "activations/layer10_attention_weight_max": 40.78790283203125,
      "activations/layer10_attention_weight_min": -37.22859573364258,
      "activations/layer11_attention_weight_max": 32.86433792114258,
      "activations/layer11_attention_weight_min": -33.39899444580078,
      "activations/layer12_attention_weight_max": 17.79306983947754,
      "activations/layer12_attention_weight_min": -25.220375061035156,
      "activations/layer13_attention_weight_max": 44.489967346191406,
      "activations/layer13_attention_weight_min": -31.820587158203125,
      "activations/layer14_attention_weight_max": 38.324161529541016,
      "activations/layer14_attention_weight_min": -33.39474105834961,
      "activations/layer15_attention_weight_max": 32.013092041015625,
      "activations/layer15_attention_weight_min": -28.686054229736328,
      "activations/layer16_attention_weight_max": 30.56298828125,
      "activations/layer16_attention_weight_min": -30.245052337646484,
      "activations/layer17_attention_weight_max": 51.716163635253906,
      "activations/layer17_attention_weight_min": -43.85176467895508,
      "activations/layer18_attention_weight_max": 45.423377990722656,
      "activations/layer18_attention_weight_min": -37.601497650146484,
      "activations/layer19_attention_weight_max": 20.566162109375,
      "activations/layer19_attention_weight_min": -21.43463706970215,
      "activations/layer1_attention_weight_max": 16.981088638305664,
      "activations/layer1_attention_weight_min": -16.518285751342773,
      "activations/layer20_attention_weight_max": 22.187686920166016,
      "activations/layer20_attention_weight_min": -23.730100631713867,
      "activations/layer21_attention_weight_max": 34.105350494384766,
      "activations/layer21_attention_weight_min": -21.937660217285156,
      "activations/layer22_attention_weight_max": 32.634849548339844,
      "activations/layer22_attention_weight_min": -26.08676528930664,
      "activations/layer23_attention_weight_max": 33.46549987792969,
      "activations/layer23_attention_weight_min": -22.46339225769043,
      "activations/layer2_attention_weight_max": 33.97156524658203,
      "activations/layer2_attention_weight_min": -31.182373046875,
      "activations/layer3_attention_weight_max": 95.91341400146484,
      "activations/layer3_attention_weight_min": -96.99298858642578,
      "activations/layer4_attention_weight_max": 107.24513244628906,
      "activations/layer4_attention_weight_min": -103.72106170654297,
      "activations/layer5_attention_weight_max": 82.3021011352539,
      "activations/layer5_attention_weight_min": -84.00019836425781,
      "activations/layer6_attention_weight_max": 58.58629608154297,
      "activations/layer6_attention_weight_min": -56.92681121826172,
      "activations/layer7_attention_weight_max": 83.2432632446289,
      "activations/layer7_attention_weight_min": -79.01561737060547,
      "activations/layer8_attention_weight_max": 50.0284309387207,
      "activations/layer8_attention_weight_min": -49.57514953613281,
      "activations/layer9_attention_weight_max": 54.693458557128906,
      "activations/layer9_attention_weight_min": -49.99403762817383,
      "epoch": 10.06,
      "learning_rate": 8.600416666666666e-05,
      "loss": 2.7679,
      "step": 173100
    },
    {
      "activations/layer0_attention_weight_max": 15.356266975402832,
      "activations/layer0_attention_weight_min": -14.387118339538574,
      "activations/layer10_attention_weight_max": 37.10464096069336,
      "activations/layer10_attention_weight_min": -37.11542892456055,
      "activations/layer11_attention_weight_max": 31.7532958984375,
      "activations/layer11_attention_weight_min": -33.22446060180664,
      "activations/layer12_attention_weight_max": 26.955036163330078,
      "activations/layer12_attention_weight_min": -26.84833335876465,
      "activations/layer13_attention_weight_max": 36.69753646850586,
      "activations/layer13_attention_weight_min": -28.72861099243164,
      "activations/layer14_attention_weight_max": 36.00230407714844,
      "activations/layer14_attention_weight_min": -32.657562255859375,
      "activations/layer15_attention_weight_max": 33.2740478515625,
      "activations/layer15_attention_weight_min": -29.862810134887695,
      "activations/layer16_attention_weight_max": 30.653499603271484,
      "activations/layer16_attention_weight_min": -28.332054138183594,
      "activations/layer17_attention_weight_max": 50.87849807739258,
      "activations/layer17_attention_weight_min": -43.47261428833008,
      "activations/layer18_attention_weight_max": 47.2133674621582,
      "activations/layer18_attention_weight_min": -38.321983337402344,
      "activations/layer19_attention_weight_max": 22.448820114135742,
      "activations/layer19_attention_weight_min": -21.763242721557617,
      "activations/layer1_attention_weight_max": 16.869251251220703,
      "activations/layer1_attention_weight_min": -15.68710994720459,
      "activations/layer20_attention_weight_max": 20.944637298583984,
      "activations/layer20_attention_weight_min": -21.274375915527344,
      "activations/layer21_attention_weight_max": 38.6229133605957,
      "activations/layer21_attention_weight_min": -23.56270408630371,
      "activations/layer22_attention_weight_max": 31.382009506225586,
      "activations/layer22_attention_weight_min": -26.150863647460938,
      "activations/layer23_attention_weight_max": 35.656463623046875,
      "activations/layer23_attention_weight_min": -24.970966339111328,
      "activations/layer2_attention_weight_max": 32.98305130004883,
      "activations/layer2_attention_weight_min": -32.63249969482422,
      "activations/layer3_attention_weight_max": 97.41062927246094,
      "activations/layer3_attention_weight_min": -100.62015533447266,
      "activations/layer4_attention_weight_max": 104.89631652832031,
      "activations/layer4_attention_weight_min": -102.60603332519531,
      "activations/layer5_attention_weight_max": 79.4287109375,
      "activations/layer5_attention_weight_min": -79.64190673828125,
      "activations/layer6_attention_weight_max": 57.0147705078125,
      "activations/layer6_attention_weight_min": -55.10905456542969,
      "activations/layer7_attention_weight_max": 76.4376449584961,
      "activations/layer7_attention_weight_min": -77.1717300415039,
      "activations/layer8_attention_weight_max": 45.88191223144531,
      "activations/layer8_attention_weight_min": -44.95591354370117,
      "activations/layer9_attention_weight_max": 54.72932815551758,
      "activations/layer9_attention_weight_min": -48.69483947753906,
      "epoch": 10.06,
      "learning_rate": 8.598522727272727e-05,
      "loss": 2.7797,
      "step": 173150
    },
    {
      "activations/layer0_attention_weight_max": 14.954936981201172,
      "activations/layer0_attention_weight_min": -14.034543991088867,
      "activations/layer10_attention_weight_max": 36.45726013183594,
      "activations/layer10_attention_weight_min": -36.195831298828125,
      "activations/layer11_attention_weight_max": 31.687808990478516,
      "activations/layer11_attention_weight_min": -32.04570007324219,
      "activations/layer12_attention_weight_max": 16.846721649169922,
      "activations/layer12_attention_weight_min": -27.18064308166504,
      "activations/layer13_attention_weight_max": 37.699485778808594,
      "activations/layer13_attention_weight_min": -27.34125518798828,
      "activations/layer14_attention_weight_max": 37.37848663330078,
      "activations/layer14_attention_weight_min": -32.31465530395508,
      "activations/layer15_attention_weight_max": 34.72678756713867,
      "activations/layer15_attention_weight_min": -30.550548553466797,
      "activations/layer16_attention_weight_max": 31.883472442626953,
      "activations/layer16_attention_weight_min": -28.966434478759766,
      "activations/layer17_attention_weight_max": 55.42969512939453,
      "activations/layer17_attention_weight_min": -43.04115676879883,
      "activations/layer18_attention_weight_max": 48.71458435058594,
      "activations/layer18_attention_weight_min": -36.64397430419922,
      "activations/layer19_attention_weight_max": 23.858572006225586,
      "activations/layer19_attention_weight_min": -22.572908401489258,
      "activations/layer1_attention_weight_max": 15.63359260559082,
      "activations/layer1_attention_weight_min": -15.236143112182617,
      "activations/layer20_attention_weight_max": 25.675188064575195,
      "activations/layer20_attention_weight_min": -21.779125213623047,
      "activations/layer21_attention_weight_max": 36.17425537109375,
      "activations/layer21_attention_weight_min": -23.087541580200195,
      "activations/layer22_attention_weight_max": 33.7487907409668,
      "activations/layer22_attention_weight_min": -25.69697380065918,
      "activations/layer23_attention_weight_max": 37.78619384765625,
      "activations/layer23_attention_weight_min": -23.714466094970703,
      "activations/layer2_attention_weight_max": 33.18547058105469,
      "activations/layer2_attention_weight_min": -31.02619171142578,
      "activations/layer3_attention_weight_max": 90.53646087646484,
      "activations/layer3_attention_weight_min": -92.26596069335938,
      "activations/layer4_attention_weight_max": 101.65211486816406,
      "activations/layer4_attention_weight_min": -98.51296997070312,
      "activations/layer5_attention_weight_max": 77.00943756103516,
      "activations/layer5_attention_weight_min": -82.30496978759766,
      "activations/layer6_attention_weight_max": 55.28670120239258,
      "activations/layer6_attention_weight_min": -54.478214263916016,
      "activations/layer7_attention_weight_max": 83.61844635009766,
      "activations/layer7_attention_weight_min": -74.42803955078125,
      "activations/layer8_attention_weight_max": 46.05986785888672,
      "activations/layer8_attention_weight_min": -48.68523025512695,
      "activations/layer9_attention_weight_max": 51.162044525146484,
      "activations/layer9_attention_weight_min": -51.49520492553711,
      "epoch": 10.06,
      "learning_rate": 8.596628787878786e-05,
      "loss": 2.7828,
      "step": 173200
    },
    {
      "activations/layer0_attention_weight_max": 15.596757888793945,
      "activations/layer0_attention_weight_min": -14.285950660705566,
      "activations/layer10_attention_weight_max": 39.21241760253906,
      "activations/layer10_attention_weight_min": -39.04342269897461,
      "activations/layer11_attention_weight_max": 34.64085388183594,
      "activations/layer11_attention_weight_min": -34.38422393798828,
      "activations/layer12_attention_weight_max": 21.32559585571289,
      "activations/layer12_attention_weight_min": -27.34585189819336,
      "activations/layer13_attention_weight_max": 43.45674514770508,
      "activations/layer13_attention_weight_min": -41.269676208496094,
      "activations/layer14_attention_weight_max": 40.54581069946289,
      "activations/layer14_attention_weight_min": -30.783180236816406,
      "activations/layer15_attention_weight_max": 35.74161911010742,
      "activations/layer15_attention_weight_min": -32.157684326171875,
      "activations/layer16_attention_weight_max": 31.387439727783203,
      "activations/layer16_attention_weight_min": -29.815593719482422,
      "activations/layer17_attention_weight_max": 55.19511032104492,
      "activations/layer17_attention_weight_min": -45.217552185058594,
      "activations/layer18_attention_weight_max": 51.58879089355469,
      "activations/layer18_attention_weight_min": -38.424346923828125,
      "activations/layer19_attention_weight_max": 21.781713485717773,
      "activations/layer19_attention_weight_min": -21.67943572998047,
      "activations/layer1_attention_weight_max": 16.304691314697266,
      "activations/layer1_attention_weight_min": -15.609223365783691,
      "activations/layer20_attention_weight_max": 23.788951873779297,
      "activations/layer20_attention_weight_min": -21.059398651123047,
      "activations/layer21_attention_weight_max": 40.28525161743164,
      "activations/layer21_attention_weight_min": -22.8786678314209,
      "activations/layer22_attention_weight_max": 29.233383178710938,
      "activations/layer22_attention_weight_min": -26.52765464782715,
      "activations/layer23_attention_weight_max": 35.19915008544922,
      "activations/layer23_attention_weight_min": -23.851001739501953,
      "activations/layer2_attention_weight_max": 32.20716857910156,
      "activations/layer2_attention_weight_min": -31.374507904052734,
      "activations/layer3_attention_weight_max": 96.99887084960938,
      "activations/layer3_attention_weight_min": -99.19766998291016,
      "activations/layer4_attention_weight_max": 108.81285858154297,
      "activations/layer4_attention_weight_min": -106.36363983154297,
      "activations/layer5_attention_weight_max": 82.24880981445312,
      "activations/layer5_attention_weight_min": -84.48551177978516,
      "activations/layer6_attention_weight_max": 58.56869888305664,
      "activations/layer6_attention_weight_min": -58.85416793823242,
      "activations/layer7_attention_weight_max": 81.78803253173828,
      "activations/layer7_attention_weight_min": -79.11751556396484,
      "activations/layer8_attention_weight_max": 49.99701690673828,
      "activations/layer8_attention_weight_min": -52.507781982421875,
      "activations/layer9_attention_weight_max": 51.4932746887207,
      "activations/layer9_attention_weight_min": -54.45857620239258,
      "epoch": 10.07,
      "learning_rate": 8.594734848484848e-05,
      "loss": 2.7949,
      "step": 173250
    },
    {
      "activations/layer0_attention_weight_max": 16.873332977294922,
      "activations/layer0_attention_weight_min": -14.450135231018066,
      "activations/layer10_attention_weight_max": 39.056278228759766,
      "activations/layer10_attention_weight_min": -38.99721145629883,
      "activations/layer11_attention_weight_max": 33.30732727050781,
      "activations/layer11_attention_weight_min": -33.19255447387695,
      "activations/layer12_attention_weight_max": 18.952783584594727,
      "activations/layer12_attention_weight_min": -25.904712677001953,
      "activations/layer13_attention_weight_max": 38.1362190246582,
      "activations/layer13_attention_weight_min": -29.943477630615234,
      "activations/layer14_attention_weight_max": 37.495872497558594,
      "activations/layer14_attention_weight_min": -30.929698944091797,
      "activations/layer15_attention_weight_max": 32.22451400756836,
      "activations/layer15_attention_weight_min": -30.06473731994629,
      "activations/layer16_attention_weight_max": 31.172534942626953,
      "activations/layer16_attention_weight_min": -26.756776809692383,
      "activations/layer17_attention_weight_max": 53.451942443847656,
      "activations/layer17_attention_weight_min": -41.067684173583984,
      "activations/layer18_attention_weight_max": 43.77665328979492,
      "activations/layer18_attention_weight_min": -35.05488586425781,
      "activations/layer19_attention_weight_max": 19.841215133666992,
      "activations/layer19_attention_weight_min": -21.653932571411133,
      "activations/layer1_attention_weight_max": 15.921320915222168,
      "activations/layer1_attention_weight_min": -17.046428680419922,
      "activations/layer20_attention_weight_max": 20.810192108154297,
      "activations/layer20_attention_weight_min": -21.048503875732422,
      "activations/layer21_attention_weight_max": 36.810855865478516,
      "activations/layer21_attention_weight_min": -25.808706283569336,
      "activations/layer22_attention_weight_max": 31.830251693725586,
      "activations/layer22_attention_weight_min": -25.65145492553711,
      "activations/layer23_attention_weight_max": 34.66279983520508,
      "activations/layer23_attention_weight_min": -25.241254806518555,
      "activations/layer2_attention_weight_max": 33.65277099609375,
      "activations/layer2_attention_weight_min": -31.91131591796875,
      "activations/layer3_attention_weight_max": 94.60987854003906,
      "activations/layer3_attention_weight_min": -93.60526275634766,
      "activations/layer4_attention_weight_max": 106.27302551269531,
      "activations/layer4_attention_weight_min": -100.0132827758789,
      "activations/layer5_attention_weight_max": 79.46224212646484,
      "activations/layer5_attention_weight_min": -79.37447357177734,
      "activations/layer6_attention_weight_max": 59.327545166015625,
      "activations/layer6_attention_weight_min": -58.76651382446289,
      "activations/layer7_attention_weight_max": 74.89102172851562,
      "activations/layer7_attention_weight_min": -75.25956726074219,
      "activations/layer8_attention_weight_max": 47.80611038208008,
      "activations/layer8_attention_weight_min": -48.8849983215332,
      "activations/layer9_attention_weight_max": 50.1409797668457,
      "activations/layer9_attention_weight_min": -49.07749557495117,
      "epoch": 10.07,
      "learning_rate": 8.592840909090909e-05,
      "loss": 2.7792,
      "step": 173300
    },
    {
      "activations/layer0_attention_weight_max": 15.448698043823242,
      "activations/layer0_attention_weight_min": -14.221468925476074,
      "activations/layer10_attention_weight_max": 36.927860260009766,
      "activations/layer10_attention_weight_min": -37.544708251953125,
      "activations/layer11_attention_weight_max": 31.513927459716797,
      "activations/layer11_attention_weight_min": -32.38227462768555,
      "activations/layer12_attention_weight_max": 17.30367088317871,
      "activations/layer12_attention_weight_min": -24.78962516784668,
      "activations/layer13_attention_weight_max": 34.68640899658203,
      "activations/layer13_attention_weight_min": -29.03329086303711,
      "activations/layer14_attention_weight_max": 34.48284912109375,
      "activations/layer14_attention_weight_min": -31.77627182006836,
      "activations/layer15_attention_weight_max": 31.80327606201172,
      "activations/layer15_attention_weight_min": -30.242399215698242,
      "activations/layer16_attention_weight_max": 28.14384651184082,
      "activations/layer16_attention_weight_min": -28.511009216308594,
      "activations/layer17_attention_weight_max": 51.096954345703125,
      "activations/layer17_attention_weight_min": -42.819053649902344,
      "activations/layer18_attention_weight_max": 44.14234161376953,
      "activations/layer18_attention_weight_min": -37.08256912231445,
      "activations/layer19_attention_weight_max": 21.12697410583496,
      "activations/layer19_attention_weight_min": -21.019515991210938,
      "activations/layer1_attention_weight_max": 17.091047286987305,
      "activations/layer1_attention_weight_min": -16.42120361328125,
      "activations/layer20_attention_weight_max": 20.751680374145508,
      "activations/layer20_attention_weight_min": -24.457534790039062,
      "activations/layer21_attention_weight_max": 36.838897705078125,
      "activations/layer21_attention_weight_min": -23.195266723632812,
      "activations/layer22_attention_weight_max": 30.272504806518555,
      "activations/layer22_attention_weight_min": -24.657712936401367,
      "activations/layer23_attention_weight_max": 37.36482238769531,
      "activations/layer23_attention_weight_min": -23.348392486572266,
      "activations/layer2_attention_weight_max": 32.73912048339844,
      "activations/layer2_attention_weight_min": -32.2586555480957,
      "activations/layer3_attention_weight_max": 92.7658462524414,
      "activations/layer3_attention_weight_min": -101.20728302001953,
      "activations/layer4_attention_weight_max": 100.56404113769531,
      "activations/layer4_attention_weight_min": -96.10198211669922,
      "activations/layer5_attention_weight_max": 75.73430633544922,
      "activations/layer5_attention_weight_min": -75.6467056274414,
      "activations/layer6_attention_weight_max": 54.48149490356445,
      "activations/layer6_attention_weight_min": -54.306697845458984,
      "activations/layer7_attention_weight_max": 76.88484191894531,
      "activations/layer7_attention_weight_min": -73.678466796875,
      "activations/layer8_attention_weight_max": 43.81217575073242,
      "activations/layer8_attention_weight_min": -48.08028030395508,
      "activations/layer9_attention_weight_max": 46.79620361328125,
      "activations/layer9_attention_weight_min": -49.2409782409668,
      "epoch": 10.07,
      "learning_rate": 8.590946969696968e-05,
      "loss": 2.7839,
      "step": 173350
    },
    {
      "activations/layer0_attention_weight_max": 15.716730117797852,
      "activations/layer0_attention_weight_min": -14.274873733520508,
      "activations/layer10_attention_weight_max": 37.97965621948242,
      "activations/layer10_attention_weight_min": -36.74839782714844,
      "activations/layer11_attention_weight_max": 35.597198486328125,
      "activations/layer11_attention_weight_min": -32.741050720214844,
      "activations/layer12_attention_weight_max": 19.484939575195312,
      "activations/layer12_attention_weight_min": -30.045238494873047,
      "activations/layer13_attention_weight_max": 37.892417907714844,
      "activations/layer13_attention_weight_min": -32.83802032470703,
      "activations/layer14_attention_weight_max": 37.42605972290039,
      "activations/layer14_attention_weight_min": -29.52931785583496,
      "activations/layer15_attention_weight_max": 34.98210144042969,
      "activations/layer15_attention_weight_min": -32.17353439331055,
      "activations/layer16_attention_weight_max": 34.698753356933594,
      "activations/layer16_attention_weight_min": -28.48025894165039,
      "activations/layer17_attention_weight_max": 51.444740295410156,
      "activations/layer17_attention_weight_min": -44.36830520629883,
      "activations/layer18_attention_weight_max": 44.48873519897461,
      "activations/layer18_attention_weight_min": -36.07813262939453,
      "activations/layer19_attention_weight_max": 22.180683135986328,
      "activations/layer19_attention_weight_min": -23.130102157592773,
      "activations/layer1_attention_weight_max": 17.06679344177246,
      "activations/layer1_attention_weight_min": -17.37266731262207,
      "activations/layer20_attention_weight_max": 23.464210510253906,
      "activations/layer20_attention_weight_min": -25.1904296875,
      "activations/layer21_attention_weight_max": 44.52772521972656,
      "activations/layer21_attention_weight_min": -25.512840270996094,
      "activations/layer22_attention_weight_max": 31.647132873535156,
      "activations/layer22_attention_weight_min": -27.54627799987793,
      "activations/layer23_attention_weight_max": 37.05193328857422,
      "activations/layer23_attention_weight_min": -25.47076988220215,
      "activations/layer2_attention_weight_max": 34.50107192993164,
      "activations/layer2_attention_weight_min": -33.19562530517578,
      "activations/layer3_attention_weight_max": 99.11746978759766,
      "activations/layer3_attention_weight_min": -95.03050231933594,
      "activations/layer4_attention_weight_max": 104.648681640625,
      "activations/layer4_attention_weight_min": -96.45539093017578,
      "activations/layer5_attention_weight_max": 77.64323425292969,
      "activations/layer5_attention_weight_min": -79.61686706542969,
      "activations/layer6_attention_weight_max": 57.73699188232422,
      "activations/layer6_attention_weight_min": -54.18349838256836,
      "activations/layer7_attention_weight_max": 78.5107650756836,
      "activations/layer7_attention_weight_min": -80.62531280517578,
      "activations/layer8_attention_weight_max": 48.85725784301758,
      "activations/layer8_attention_weight_min": -52.95856475830078,
      "activations/layer9_attention_weight_max": 54.724727630615234,
      "activations/layer9_attention_weight_min": -51.37762451171875,
      "epoch": 10.08,
      "learning_rate": 8.58905303030303e-05,
      "loss": 2.773,
      "step": 173400
    },
    {
      "activations/layer0_attention_weight_max": 15.653417587280273,
      "activations/layer0_attention_weight_min": -13.99693489074707,
      "activations/layer10_attention_weight_max": 38.958740234375,
      "activations/layer10_attention_weight_min": -36.82094192504883,
      "activations/layer11_attention_weight_max": 34.12749481201172,
      "activations/layer11_attention_weight_min": -32.18462371826172,
      "activations/layer12_attention_weight_max": 22.894432067871094,
      "activations/layer12_attention_weight_min": -27.469865798950195,
      "activations/layer13_attention_weight_max": 33.836936950683594,
      "activations/layer13_attention_weight_min": -29.656213760375977,
      "activations/layer14_attention_weight_max": 34.501914978027344,
      "activations/layer14_attention_weight_min": -31.060253143310547,
      "activations/layer15_attention_weight_max": 32.364959716796875,
      "activations/layer15_attention_weight_min": -29.402313232421875,
      "activations/layer16_attention_weight_max": 29.935871124267578,
      "activations/layer16_attention_weight_min": -28.624187469482422,
      "activations/layer17_attention_weight_max": 52.29269027709961,
      "activations/layer17_attention_weight_min": -44.88087463378906,
      "activations/layer18_attention_weight_max": 45.002716064453125,
      "activations/layer18_attention_weight_min": -40.57172393798828,
      "activations/layer19_attention_weight_max": 23.040761947631836,
      "activations/layer19_attention_weight_min": -23.68988609313965,
      "activations/layer1_attention_weight_max": 16.246105194091797,
      "activations/layer1_attention_weight_min": -15.319507598876953,
      "activations/layer20_attention_weight_max": 22.445905685424805,
      "activations/layer20_attention_weight_min": -20.67283058166504,
      "activations/layer21_attention_weight_max": 34.67518997192383,
      "activations/layer21_attention_weight_min": -24.90545082092285,
      "activations/layer22_attention_weight_max": 31.2392635345459,
      "activations/layer22_attention_weight_min": -26.496044158935547,
      "activations/layer23_attention_weight_max": 37.738861083984375,
      "activations/layer23_attention_weight_min": -24.490629196166992,
      "activations/layer2_attention_weight_max": 33.051475524902344,
      "activations/layer2_attention_weight_min": -31.134010314941406,
      "activations/layer3_attention_weight_max": 93.7305908203125,
      "activations/layer3_attention_weight_min": -96.44831085205078,
      "activations/layer4_attention_weight_max": 105.24137115478516,
      "activations/layer4_attention_weight_min": -100.63492584228516,
      "activations/layer5_attention_weight_max": 77.34004211425781,
      "activations/layer5_attention_weight_min": -78.75495910644531,
      "activations/layer6_attention_weight_max": 58.24346160888672,
      "activations/layer6_attention_weight_min": -55.86145782470703,
      "activations/layer7_attention_weight_max": 79.4562759399414,
      "activations/layer7_attention_weight_min": -74.72986602783203,
      "activations/layer8_attention_weight_max": 46.548709869384766,
      "activations/layer8_attention_weight_min": -47.53144836425781,
      "activations/layer9_attention_weight_max": 50.21121597290039,
      "activations/layer9_attention_weight_min": -50.876365661621094,
      "epoch": 10.08,
      "learning_rate": 8.587159090909091e-05,
      "loss": 2.7695,
      "step": 173450
    },
    {
      "activations/layer0_attention_weight_max": 15.088357925415039,
      "activations/layer0_attention_weight_min": -14.152220726013184,
      "activations/layer10_attention_weight_max": 38.62050247192383,
      "activations/layer10_attention_weight_min": -37.05371856689453,
      "activations/layer11_attention_weight_max": 31.406295776367188,
      "activations/layer11_attention_weight_min": -32.982444763183594,
      "activations/layer12_attention_weight_max": 23.59905242919922,
      "activations/layer12_attention_weight_min": -24.320520401000977,
      "activations/layer13_attention_weight_max": 42.99631881713867,
      "activations/layer13_attention_weight_min": -29.705429077148438,
      "activations/layer14_attention_weight_max": 40.412696838378906,
      "activations/layer14_attention_weight_min": -32.84015655517578,
      "activations/layer15_attention_weight_max": 35.11431884765625,
      "activations/layer15_attention_weight_min": -30.07921028137207,
      "activations/layer16_attention_weight_max": 31.122835159301758,
      "activations/layer16_attention_weight_min": -28.34453582763672,
      "activations/layer17_attention_weight_max": 53.97661590576172,
      "activations/layer17_attention_weight_min": -43.70018005371094,
      "activations/layer18_attention_weight_max": 46.99993896484375,
      "activations/layer18_attention_weight_min": -37.2412223815918,
      "activations/layer19_attention_weight_max": 22.063344955444336,
      "activations/layer19_attention_weight_min": -22.09747314453125,
      "activations/layer1_attention_weight_max": 15.993158340454102,
      "activations/layer1_attention_weight_min": -15.964532852172852,
      "activations/layer20_attention_weight_max": 21.777427673339844,
      "activations/layer20_attention_weight_min": -25.591968536376953,
      "activations/layer21_attention_weight_max": 36.05653762817383,
      "activations/layer21_attention_weight_min": -21.692312240600586,
      "activations/layer22_attention_weight_max": 31.62218475341797,
      "activations/layer22_attention_weight_min": -27.574291229248047,
      "activations/layer23_attention_weight_max": 36.56372833251953,
      "activations/layer23_attention_weight_min": -23.613174438476562,
      "activations/layer2_attention_weight_max": 33.49243927001953,
      "activations/layer2_attention_weight_min": -30.829410552978516,
      "activations/layer3_attention_weight_max": 91.99978637695312,
      "activations/layer3_attention_weight_min": -95.91732025146484,
      "activations/layer4_attention_weight_max": 101.79875946044922,
      "activations/layer4_attention_weight_min": -97.14952850341797,
      "activations/layer5_attention_weight_max": 82.15857696533203,
      "activations/layer5_attention_weight_min": -83.33123779296875,
      "activations/layer6_attention_weight_max": 55.0248908996582,
      "activations/layer6_attention_weight_min": -52.74527359008789,
      "activations/layer7_attention_weight_max": 74.13536834716797,
      "activations/layer7_attention_weight_min": -73.53355407714844,
      "activations/layer8_attention_weight_max": 47.61375045776367,
      "activations/layer8_attention_weight_min": -49.699073791503906,
      "activations/layer9_attention_weight_max": 52.67720413208008,
      "activations/layer9_attention_weight_min": -52.17232131958008,
      "epoch": 10.08,
      "learning_rate": 8.585265151515151e-05,
      "loss": 2.7729,
      "step": 173500
    },
    {
      "activations/layer0_attention_weight_max": 15.792060852050781,
      "activations/layer0_attention_weight_min": -14.158464431762695,
      "activations/layer10_attention_weight_max": 39.53305435180664,
      "activations/layer10_attention_weight_min": -38.7637939453125,
      "activations/layer11_attention_weight_max": 36.179107666015625,
      "activations/layer11_attention_weight_min": -36.27518081665039,
      "activations/layer12_attention_weight_max": 25.43010139465332,
      "activations/layer12_attention_weight_min": -22.280858993530273,
      "activations/layer13_attention_weight_max": 40.15776443481445,
      "activations/layer13_attention_weight_min": -32.183013916015625,
      "activations/layer14_attention_weight_max": 37.457855224609375,
      "activations/layer14_attention_weight_min": -29.26960563659668,
      "activations/layer15_attention_weight_max": 33.405418395996094,
      "activations/layer15_attention_weight_min": -29.384796142578125,
      "activations/layer16_attention_weight_max": 29.865915298461914,
      "activations/layer16_attention_weight_min": -28.720985412597656,
      "activations/layer17_attention_weight_max": 54.88957595825195,
      "activations/layer17_attention_weight_min": -42.160194396972656,
      "activations/layer18_attention_weight_max": 48.254966735839844,
      "activations/layer18_attention_weight_min": -35.939552307128906,
      "activations/layer19_attention_weight_max": 22.569318771362305,
      "activations/layer19_attention_weight_min": -19.964309692382812,
      "activations/layer1_attention_weight_max": 16.13591766357422,
      "activations/layer1_attention_weight_min": -16.926321029663086,
      "activations/layer20_attention_weight_max": 23.4414119720459,
      "activations/layer20_attention_weight_min": -20.516685485839844,
      "activations/layer21_attention_weight_max": 37.53138732910156,
      "activations/layer21_attention_weight_min": -22.072662353515625,
      "activations/layer22_attention_weight_max": 32.884010314941406,
      "activations/layer22_attention_weight_min": -23.569461822509766,
      "activations/layer23_attention_weight_max": 43.45089340209961,
      "activations/layer23_attention_weight_min": -22.89240074157715,
      "activations/layer2_attention_weight_max": 32.43695068359375,
      "activations/layer2_attention_weight_min": -32.68501281738281,
      "activations/layer3_attention_weight_max": 100.06260681152344,
      "activations/layer3_attention_weight_min": -105.49774169921875,
      "activations/layer4_attention_weight_max": 105.96317291259766,
      "activations/layer4_attention_weight_min": -103.77037811279297,
      "activations/layer5_attention_weight_max": 77.17143249511719,
      "activations/layer5_attention_weight_min": -83.88519287109375,
      "activations/layer6_attention_weight_max": 57.26576232910156,
      "activations/layer6_attention_weight_min": -55.64102554321289,
      "activations/layer7_attention_weight_max": 75.1337661743164,
      "activations/layer7_attention_weight_min": -74.93316650390625,
      "activations/layer8_attention_weight_max": 46.85247802734375,
      "activations/layer8_attention_weight_min": -47.16900634765625,
      "activations/layer9_attention_weight_max": 49.11404800415039,
      "activations/layer9_attention_weight_min": -49.30366897583008,
      "epoch": 10.08,
      "learning_rate": 8.583371212121211e-05,
      "loss": 2.7817,
      "step": 173550
    },
    {
      "activations/layer0_attention_weight_max": 15.077425956726074,
      "activations/layer0_attention_weight_min": -14.181710243225098,
      "activations/layer10_attention_weight_max": 40.922447204589844,
      "activations/layer10_attention_weight_min": -39.35822296142578,
      "activations/layer11_attention_weight_max": 34.32965850830078,
      "activations/layer11_attention_weight_min": -33.04649353027344,
      "activations/layer12_attention_weight_max": 18.88612937927246,
      "activations/layer12_attention_weight_min": -24.3695125579834,
      "activations/layer13_attention_weight_max": 41.14887619018555,
      "activations/layer13_attention_weight_min": -29.61065101623535,
      "activations/layer14_attention_weight_max": 37.3411750793457,
      "activations/layer14_attention_weight_min": -32.641090393066406,
      "activations/layer15_attention_weight_max": 33.49641036987305,
      "activations/layer15_attention_weight_min": -31.17775535583496,
      "activations/layer16_attention_weight_max": 35.103553771972656,
      "activations/layer16_attention_weight_min": -30.01548957824707,
      "activations/layer17_attention_weight_max": 56.35870361328125,
      "activations/layer17_attention_weight_min": -47.002445220947266,
      "activations/layer18_attention_weight_max": 50.55626678466797,
      "activations/layer18_attention_weight_min": -39.46075439453125,
      "activations/layer19_attention_weight_max": 24.058584213256836,
      "activations/layer19_attention_weight_min": -22.382755279541016,
      "activations/layer1_attention_weight_max": 16.295957565307617,
      "activations/layer1_attention_weight_min": -18.894426345825195,
      "activations/layer20_attention_weight_max": 23.84726905822754,
      "activations/layer20_attention_weight_min": -22.982547760009766,
      "activations/layer21_attention_weight_max": 39.008148193359375,
      "activations/layer21_attention_weight_min": -23.649320602416992,
      "activations/layer22_attention_weight_max": 32.85795593261719,
      "activations/layer22_attention_weight_min": -27.119903564453125,
      "activations/layer23_attention_weight_max": 38.12202453613281,
      "activations/layer23_attention_weight_min": -24.102081298828125,
      "activations/layer2_attention_weight_max": 33.836666107177734,
      "activations/layer2_attention_weight_min": -32.47683334350586,
      "activations/layer3_attention_weight_max": 97.47807312011719,
      "activations/layer3_attention_weight_min": -97.49547576904297,
      "activations/layer4_attention_weight_max": 105.29681396484375,
      "activations/layer4_attention_weight_min": -102.91695404052734,
      "activations/layer5_attention_weight_max": 80.65315246582031,
      "activations/layer5_attention_weight_min": -86.28346252441406,
      "activations/layer6_attention_weight_max": 59.51991271972656,
      "activations/layer6_attention_weight_min": -58.34811782836914,
      "activations/layer7_attention_weight_max": 78.7789077758789,
      "activations/layer7_attention_weight_min": -79.212890625,
      "activations/layer8_attention_weight_max": 48.53187942504883,
      "activations/layer8_attention_weight_min": -50.03462219238281,
      "activations/layer9_attention_weight_max": 53.660987854003906,
      "activations/layer9_attention_weight_min": -52.251197814941406,
      "epoch": 10.09,
      "learning_rate": 8.581477272727271e-05,
      "loss": 2.796,
      "step": 173600
    },
    {
      "activations/layer0_attention_weight_max": 15.637811660766602,
      "activations/layer0_attention_weight_min": -13.857938766479492,
      "activations/layer10_attention_weight_max": 35.36941146850586,
      "activations/layer10_attention_weight_min": -36.84712219238281,
      "activations/layer11_attention_weight_max": 30.58704948425293,
      "activations/layer11_attention_weight_min": -32.15176773071289,
      "activations/layer12_attention_weight_max": 17.06595802307129,
      "activations/layer12_attention_weight_min": -25.056493759155273,
      "activations/layer13_attention_weight_max": 41.22624969482422,
      "activations/layer13_attention_weight_min": -30.082050323486328,
      "activations/layer14_attention_weight_max": 36.04479217529297,
      "activations/layer14_attention_weight_min": -30.35592269897461,
      "activations/layer15_attention_weight_max": 33.51151657104492,
      "activations/layer15_attention_weight_min": -30.099021911621094,
      "activations/layer16_attention_weight_max": 29.795516967773438,
      "activations/layer16_attention_weight_min": -27.846572875976562,
      "activations/layer17_attention_weight_max": 51.046329498291016,
      "activations/layer17_attention_weight_min": -43.288761138916016,
      "activations/layer18_attention_weight_max": 47.37442398071289,
      "activations/layer18_attention_weight_min": -35.74523162841797,
      "activations/layer19_attention_weight_max": 21.752504348754883,
      "activations/layer19_attention_weight_min": -22.664770126342773,
      "activations/layer1_attention_weight_max": 17.701316833496094,
      "activations/layer1_attention_weight_min": -14.709184646606445,
      "activations/layer20_attention_weight_max": 24.169572830200195,
      "activations/layer20_attention_weight_min": -20.703855514526367,
      "activations/layer21_attention_weight_max": 39.89997482299805,
      "activations/layer21_attention_weight_min": -25.435319900512695,
      "activations/layer22_attention_weight_max": 30.538986206054688,
      "activations/layer22_attention_weight_min": -27.73996353149414,
      "activations/layer23_attention_weight_max": 38.588584899902344,
      "activations/layer23_attention_weight_min": -23.691776275634766,
      "activations/layer2_attention_weight_max": 32.0704345703125,
      "activations/layer2_attention_weight_min": -30.419950485229492,
      "activations/layer3_attention_weight_max": 89.62918853759766,
      "activations/layer3_attention_weight_min": -87.15362548828125,
      "activations/layer4_attention_weight_max": 95.42363739013672,
      "activations/layer4_attention_weight_min": -90.46643829345703,
      "activations/layer5_attention_weight_max": 75.28131103515625,
      "activations/layer5_attention_weight_min": -78.03021240234375,
      "activations/layer6_attention_weight_max": 57.16038131713867,
      "activations/layer6_attention_weight_min": -54.692256927490234,
      "activations/layer7_attention_weight_max": 76.94609832763672,
      "activations/layer7_attention_weight_min": -74.99852752685547,
      "activations/layer8_attention_weight_max": 45.769832611083984,
      "activations/layer8_attention_weight_min": -50.207366943359375,
      "activations/layer9_attention_weight_max": 50.6707649230957,
      "activations/layer9_attention_weight_min": -49.46796417236328,
      "epoch": 10.09,
      "learning_rate": 8.579583333333333e-05,
      "loss": 2.7795,
      "step": 173650
    },
    {
      "activations/layer0_attention_weight_max": 15.368695259094238,
      "activations/layer0_attention_weight_min": -14.32684326171875,
      "activations/layer10_attention_weight_max": 38.28510284423828,
      "activations/layer10_attention_weight_min": -35.438934326171875,
      "activations/layer11_attention_weight_max": 35.61866760253906,
      "activations/layer11_attention_weight_min": -34.71872329711914,
      "activations/layer12_attention_weight_max": 23.843286514282227,
      "activations/layer12_attention_weight_min": -29.746219635009766,
      "activations/layer13_attention_weight_max": 44.62101364135742,
      "activations/layer13_attention_weight_min": -30.243785858154297,
      "activations/layer14_attention_weight_max": 39.39426803588867,
      "activations/layer14_attention_weight_min": -30.662019729614258,
      "activations/layer15_attention_weight_max": 34.98711013793945,
      "activations/layer15_attention_weight_min": -30.724689483642578,
      "activations/layer16_attention_weight_max": 30.155794143676758,
      "activations/layer16_attention_weight_min": -28.834245681762695,
      "activations/layer17_attention_weight_max": 49.01779556274414,
      "activations/layer17_attention_weight_min": -45.09029769897461,
      "activations/layer18_attention_weight_max": 47.51874923706055,
      "activations/layer18_attention_weight_min": -38.376380920410156,
      "activations/layer19_attention_weight_max": 23.68848419189453,
      "activations/layer19_attention_weight_min": -21.969655990600586,
      "activations/layer1_attention_weight_max": 16.083919525146484,
      "activations/layer1_attention_weight_min": -14.574009895324707,
      "activations/layer20_attention_weight_max": 26.434091567993164,
      "activations/layer20_attention_weight_min": -27.0364990234375,
      "activations/layer21_attention_weight_max": 37.17311477661133,
      "activations/layer21_attention_weight_min": -21.60734748840332,
      "activations/layer22_attention_weight_max": 34.91828155517578,
      "activations/layer22_attention_weight_min": -23.995712280273438,
      "activations/layer23_attention_weight_max": 36.76060485839844,
      "activations/layer23_attention_weight_min": -20.7601375579834,
      "activations/layer2_attention_weight_max": 33.835426330566406,
      "activations/layer2_attention_weight_min": -32.526519775390625,
      "activations/layer3_attention_weight_max": 91.50841522216797,
      "activations/layer3_attention_weight_min": -90.82707977294922,
      "activations/layer4_attention_weight_max": 98.504638671875,
      "activations/layer4_attention_weight_min": -91.33602905273438,
      "activations/layer5_attention_weight_max": 79.41468811035156,
      "activations/layer5_attention_weight_min": -77.45931243896484,
      "activations/layer6_attention_weight_max": 55.63963317871094,
      "activations/layer6_attention_weight_min": -53.00429153442383,
      "activations/layer7_attention_weight_max": 81.32797241210938,
      "activations/layer7_attention_weight_min": -74.18753051757812,
      "activations/layer8_attention_weight_max": 48.943138122558594,
      "activations/layer8_attention_weight_min": -47.29353713989258,
      "activations/layer9_attention_weight_max": 52.09546661376953,
      "activations/layer9_attention_weight_min": -47.510860443115234,
      "epoch": 10.09,
      "learning_rate": 8.577689393939393e-05,
      "loss": 2.7928,
      "step": 173700
    },
    {
      "activations/layer0_attention_weight_max": 17.468454360961914,
      "activations/layer0_attention_weight_min": -14.58239459991455,
      "activations/layer10_attention_weight_max": 35.686378479003906,
      "activations/layer10_attention_weight_min": -34.880653381347656,
      "activations/layer11_attention_weight_max": 31.912673950195312,
      "activations/layer11_attention_weight_min": -31.398860931396484,
      "activations/layer12_attention_weight_max": 20.0234432220459,
      "activations/layer12_attention_weight_min": -23.976911544799805,
      "activations/layer13_attention_weight_max": 50.862815856933594,
      "activations/layer13_attention_weight_min": -32.226322174072266,
      "activations/layer14_attention_weight_max": 36.58564758300781,
      "activations/layer14_attention_weight_min": -28.95797348022461,
      "activations/layer15_attention_weight_max": 34.40427017211914,
      "activations/layer15_attention_weight_min": -29.305316925048828,
      "activations/layer16_attention_weight_max": 32.9681396484375,
      "activations/layer16_attention_weight_min": -28.73505401611328,
      "activations/layer17_attention_weight_max": 51.824302673339844,
      "activations/layer17_attention_weight_min": -46.298248291015625,
      "activations/layer18_attention_weight_max": 47.16655731201172,
      "activations/layer18_attention_weight_min": -39.43215560913086,
      "activations/layer19_attention_weight_max": 21.681129455566406,
      "activations/layer19_attention_weight_min": -22.475566864013672,
      "activations/layer1_attention_weight_max": 16.318845748901367,
      "activations/layer1_attention_weight_min": -16.153974533081055,
      "activations/layer20_attention_weight_max": 22.979965209960938,
      "activations/layer20_attention_weight_min": -20.99925994873047,
      "activations/layer21_attention_weight_max": 39.36200714111328,
      "activations/layer21_attention_weight_min": -22.38518524169922,
      "activations/layer22_attention_weight_max": 34.46491241455078,
      "activations/layer22_attention_weight_min": -25.479238510131836,
      "activations/layer23_attention_weight_max": 38.04515075683594,
      "activations/layer23_attention_weight_min": -23.29370880126953,
      "activations/layer2_attention_weight_max": 32.96748352050781,
      "activations/layer2_attention_weight_min": -31.3311824798584,
      "activations/layer3_attention_weight_max": 88.97615814208984,
      "activations/layer3_attention_weight_min": -89.92172241210938,
      "activations/layer4_attention_weight_max": 94.5688247680664,
      "activations/layer4_attention_weight_min": -96.20619201660156,
      "activations/layer5_attention_weight_max": 76.0139389038086,
      "activations/layer5_attention_weight_min": -78.16650390625,
      "activations/layer6_attention_weight_max": 55.475135803222656,
      "activations/layer6_attention_weight_min": -54.306556701660156,
      "activations/layer7_attention_weight_max": 78.46251678466797,
      "activations/layer7_attention_weight_min": -73.61611938476562,
      "activations/layer8_attention_weight_max": 45.07710647583008,
      "activations/layer8_attention_weight_min": -44.18008041381836,
      "activations/layer9_attention_weight_max": 51.03704833984375,
      "activations/layer9_attention_weight_min": -48.57672882080078,
      "epoch": 10.1,
      "learning_rate": 8.575795454545453e-05,
      "loss": 2.7825,
      "step": 173750
    },
    {
      "activations/layer0_attention_weight_max": 15.078387260437012,
      "activations/layer0_attention_weight_min": -14.087882041931152,
      "activations/layer10_attention_weight_max": 35.028289794921875,
      "activations/layer10_attention_weight_min": -35.158119201660156,
      "activations/layer11_attention_weight_max": 29.99372100830078,
      "activations/layer11_attention_weight_min": -31.836530685424805,
      "activations/layer12_attention_weight_max": 17.385364532470703,
      "activations/layer12_attention_weight_min": -26.831811904907227,
      "activations/layer13_attention_weight_max": 37.546958923339844,
      "activations/layer13_attention_weight_min": -29.941978454589844,
      "activations/layer14_attention_weight_max": 42.553504943847656,
      "activations/layer14_attention_weight_min": -28.885269165039062,
      "activations/layer15_attention_weight_max": 34.71575164794922,
      "activations/layer15_attention_weight_min": -27.1038818359375,
      "activations/layer16_attention_weight_max": 31.89453125,
      "activations/layer16_attention_weight_min": -29.012624740600586,
      "activations/layer17_attention_weight_max": 52.80587387084961,
      "activations/layer17_attention_weight_min": -42.39516067504883,
      "activations/layer18_attention_weight_max": 45.19540786743164,
      "activations/layer18_attention_weight_min": -36.17502212524414,
      "activations/layer19_attention_weight_max": 20.36066246032715,
      "activations/layer19_attention_weight_min": -21.54269790649414,
      "activations/layer1_attention_weight_max": 15.950130462646484,
      "activations/layer1_attention_weight_min": -14.015630722045898,
      "activations/layer20_attention_weight_max": 23.190845489501953,
      "activations/layer20_attention_weight_min": -23.486968994140625,
      "activations/layer21_attention_weight_max": 43.9470329284668,
      "activations/layer21_attention_weight_min": -25.2181339263916,
      "activations/layer22_attention_weight_max": 30.948945999145508,
      "activations/layer22_attention_weight_min": -25.64900016784668,
      "activations/layer23_attention_weight_max": 36.84694290161133,
      "activations/layer23_attention_weight_min": -22.364498138427734,
      "activations/layer2_attention_weight_max": 33.531700134277344,
      "activations/layer2_attention_weight_min": -31.03148651123047,
      "activations/layer3_attention_weight_max": 91.70906066894531,
      "activations/layer3_attention_weight_min": -94.05059051513672,
      "activations/layer4_attention_weight_max": 99.87957000732422,
      "activations/layer4_attention_weight_min": -95.08650970458984,
      "activations/layer5_attention_weight_max": 76.00037384033203,
      "activations/layer5_attention_weight_min": -81.92073059082031,
      "activations/layer6_attention_weight_max": 54.3170280456543,
      "activations/layer6_attention_weight_min": -54.317039489746094,
      "activations/layer7_attention_weight_max": 78.45320129394531,
      "activations/layer7_attention_weight_min": -75.34545135498047,
      "activations/layer8_attention_weight_max": 43.59294128417969,
      "activations/layer8_attention_weight_min": -46.6623420715332,
      "activations/layer9_attention_weight_max": 47.083885192871094,
      "activations/layer9_attention_weight_min": -49.210044860839844,
      "epoch": 10.1,
      "learning_rate": 8.573901515151515e-05,
      "loss": 2.7634,
      "step": 173800
    },
    {
      "activations/layer0_attention_weight_max": 16.812183380126953,
      "activations/layer0_attention_weight_min": -13.511150360107422,
      "activations/layer10_attention_weight_max": 35.585330963134766,
      "activations/layer10_attention_weight_min": -34.54252243041992,
      "activations/layer11_attention_weight_max": 31.95404624938965,
      "activations/layer11_attention_weight_min": -30.745590209960938,
      "activations/layer12_attention_weight_max": 30.244407653808594,
      "activations/layer12_attention_weight_min": -25.655216217041016,
      "activations/layer13_attention_weight_max": 37.498409271240234,
      "activations/layer13_attention_weight_min": -28.90390396118164,
      "activations/layer14_attention_weight_max": 38.16810607910156,
      "activations/layer14_attention_weight_min": -30.285310745239258,
      "activations/layer15_attention_weight_max": 32.44770812988281,
      "activations/layer15_attention_weight_min": -29.40203857421875,
      "activations/layer16_attention_weight_max": 33.372108459472656,
      "activations/layer16_attention_weight_min": -29.33495330810547,
      "activations/layer17_attention_weight_max": 52.83338928222656,
      "activations/layer17_attention_weight_min": -46.19647979736328,
      "activations/layer18_attention_weight_max": 43.6759147644043,
      "activations/layer18_attention_weight_min": -36.7993278503418,
      "activations/layer19_attention_weight_max": 23.310317993164062,
      "activations/layer19_attention_weight_min": -22.022390365600586,
      "activations/layer1_attention_weight_max": 16.728343963623047,
      "activations/layer1_attention_weight_min": -14.978939056396484,
      "activations/layer20_attention_weight_max": 20.72449493408203,
      "activations/layer20_attention_weight_min": -24.165008544921875,
      "activations/layer21_attention_weight_max": 37.94777297973633,
      "activations/layer21_attention_weight_min": -27.360031127929688,
      "activations/layer22_attention_weight_max": 29.436147689819336,
      "activations/layer22_attention_weight_min": -25.2288761138916,
      "activations/layer23_attention_weight_max": 33.213443756103516,
      "activations/layer23_attention_weight_min": -24.367008209228516,
      "activations/layer2_attention_weight_max": 33.53015899658203,
      "activations/layer2_attention_weight_min": -30.659055709838867,
      "activations/layer3_attention_weight_max": 93.30282592773438,
      "activations/layer3_attention_weight_min": -93.1842269897461,
      "activations/layer4_attention_weight_max": 104.5324935913086,
      "activations/layer4_attention_weight_min": -101.2322006225586,
      "activations/layer5_attention_weight_max": 79.60588836669922,
      "activations/layer5_attention_weight_min": -83.60487365722656,
      "activations/layer6_attention_weight_max": 55.0174674987793,
      "activations/layer6_attention_weight_min": -54.43289566040039,
      "activations/layer7_attention_weight_max": 78.80506134033203,
      "activations/layer7_attention_weight_min": -74.18587493896484,
      "activations/layer8_attention_weight_max": 47.36726760864258,
      "activations/layer8_attention_weight_min": -49.12286376953125,
      "activations/layer9_attention_weight_max": 48.703800201416016,
      "activations/layer9_attention_weight_min": -46.306339263916016,
      "epoch": 10.1,
      "learning_rate": 8.572007575757576e-05,
      "loss": 2.7656,
      "step": 173850
    },
    {
      "activations/layer0_attention_weight_max": 15.852392196655273,
      "activations/layer0_attention_weight_min": -13.574296951293945,
      "activations/layer10_attention_weight_max": 38.03701400756836,
      "activations/layer10_attention_weight_min": -35.93540954589844,
      "activations/layer11_attention_weight_max": 32.92726135253906,
      "activations/layer11_attention_weight_min": -33.2220573425293,
      "activations/layer12_attention_weight_max": 19.47848129272461,
      "activations/layer12_attention_weight_min": -23.35922622680664,
      "activations/layer13_attention_weight_max": 42.00297927856445,
      "activations/layer13_attention_weight_min": -31.314655303955078,
      "activations/layer14_attention_weight_max": 40.391082763671875,
      "activations/layer14_attention_weight_min": -30.40118408203125,
      "activations/layer15_attention_weight_max": 38.750972747802734,
      "activations/layer15_attention_weight_min": -28.763402938842773,
      "activations/layer16_attention_weight_max": 38.4307746887207,
      "activations/layer16_attention_weight_min": -29.22894859313965,
      "activations/layer17_attention_weight_max": 57.86642074584961,
      "activations/layer17_attention_weight_min": -43.46146011352539,
      "activations/layer18_attention_weight_max": 52.237064361572266,
      "activations/layer18_attention_weight_min": -37.46372604370117,
      "activations/layer19_attention_weight_max": 22.28414535522461,
      "activations/layer19_attention_weight_min": -21.30076789855957,
      "activations/layer1_attention_weight_max": 16.311758041381836,
      "activations/layer1_attention_weight_min": -16.841989517211914,
      "activations/layer20_attention_weight_max": 23.56546974182129,
      "activations/layer20_attention_weight_min": -23.29266357421875,
      "activations/layer21_attention_weight_max": 42.712100982666016,
      "activations/layer21_attention_weight_min": -26.252593994140625,
      "activations/layer22_attention_weight_max": 36.85145950317383,
      "activations/layer22_attention_weight_min": -28.734893798828125,
      "activations/layer23_attention_weight_max": 49.99476623535156,
      "activations/layer23_attention_weight_min": -23.082983016967773,
      "activations/layer2_attention_weight_max": 32.652374267578125,
      "activations/layer2_attention_weight_min": -31.828121185302734,
      "activations/layer3_attention_weight_max": 92.29566192626953,
      "activations/layer3_attention_weight_min": -96.81485748291016,
      "activations/layer4_attention_weight_max": 97.81199645996094,
      "activations/layer4_attention_weight_min": -93.68733978271484,
      "activations/layer5_attention_weight_max": 72.93777465820312,
      "activations/layer5_attention_weight_min": -78.05365753173828,
      "activations/layer6_attention_weight_max": 53.500892639160156,
      "activations/layer6_attention_weight_min": -51.3940315246582,
      "activations/layer7_attention_weight_max": 77.14846801757812,
      "activations/layer7_attention_weight_min": -74.3275146484375,
      "activations/layer8_attention_weight_max": 42.9937858581543,
      "activations/layer8_attention_weight_min": -45.3675537109375,
      "activations/layer9_attention_weight_max": 51.253448486328125,
      "activations/layer9_attention_weight_min": -47.854557037353516,
      "epoch": 10.1,
      "learning_rate": 8.570113636363635e-05,
      "loss": 2.791,
      "step": 173900
    },
    {
      "activations/layer0_attention_weight_max": 16.190715789794922,
      "activations/layer0_attention_weight_min": -13.80257511138916,
      "activations/layer10_attention_weight_max": 37.04926681518555,
      "activations/layer10_attention_weight_min": -36.216773986816406,
      "activations/layer11_attention_weight_max": 31.63418960571289,
      "activations/layer11_attention_weight_min": -32.03077697753906,
      "activations/layer12_attention_weight_max": 19.464340209960938,
      "activations/layer12_attention_weight_min": -23.173307418823242,
      "activations/layer13_attention_weight_max": 41.370967864990234,
      "activations/layer13_attention_weight_min": -32.2807731628418,
      "activations/layer14_attention_weight_max": 39.27530288696289,
      "activations/layer14_attention_weight_min": -29.933103561401367,
      "activations/layer15_attention_weight_max": 37.067901611328125,
      "activations/layer15_attention_weight_min": -28.969112396240234,
      "activations/layer16_attention_weight_max": 32.68478012084961,
      "activations/layer16_attention_weight_min": -29.52122688293457,
      "activations/layer17_attention_weight_max": 54.94550704956055,
      "activations/layer17_attention_weight_min": -43.702632904052734,
      "activations/layer18_attention_weight_max": 50.72604751586914,
      "activations/layer18_attention_weight_min": -36.7220573425293,
      "activations/layer19_attention_weight_max": 21.940961837768555,
      "activations/layer19_attention_weight_min": -23.481836318969727,
      "activations/layer1_attention_weight_max": 15.982185363769531,
      "activations/layer1_attention_weight_min": -14.010272979736328,
      "activations/layer20_attention_weight_max": 24.1055965423584,
      "activations/layer20_attention_weight_min": -21.350934982299805,
      "activations/layer21_attention_weight_max": 40.69144821166992,
      "activations/layer21_attention_weight_min": -22.99452781677246,
      "activations/layer22_attention_weight_max": 31.42469024658203,
      "activations/layer22_attention_weight_min": -24.15407371520996,
      "activations/layer23_attention_weight_max": 37.634613037109375,
      "activations/layer23_attention_weight_min": -24.645122528076172,
      "activations/layer2_attention_weight_max": 32.13389587402344,
      "activations/layer2_attention_weight_min": -30.998655319213867,
      "activations/layer3_attention_weight_max": 96.04205322265625,
      "activations/layer3_attention_weight_min": -94.06656646728516,
      "activations/layer4_attention_weight_max": 98.05280303955078,
      "activations/layer4_attention_weight_min": -95.2779312133789,
      "activations/layer5_attention_weight_max": 74.31053924560547,
      "activations/layer5_attention_weight_min": -77.36614227294922,
      "activations/layer6_attention_weight_max": 55.0064582824707,
      "activations/layer6_attention_weight_min": -53.10744857788086,
      "activations/layer7_attention_weight_max": 77.96212005615234,
      "activations/layer7_attention_weight_min": -73.77579498291016,
      "activations/layer8_attention_weight_max": 44.689903259277344,
      "activations/layer8_attention_weight_min": -46.654273986816406,
      "activations/layer9_attention_weight_max": 47.7961540222168,
      "activations/layer9_attention_weight_min": -48.14360046386719,
      "epoch": 10.11,
      "learning_rate": 8.568219696969697e-05,
      "loss": 2.7884,
      "step": 173950
    },
    {
      "activations/layer0_attention_weight_max": 15.799079895019531,
      "activations/layer0_attention_weight_min": -13.92753791809082,
      "activations/layer10_attention_weight_max": 40.9268798828125,
      "activations/layer10_attention_weight_min": -36.266517639160156,
      "activations/layer11_attention_weight_max": 35.417274475097656,
      "activations/layer11_attention_weight_min": -33.83082580566406,
      "activations/layer12_attention_weight_max": 19.475366592407227,
      "activations/layer12_attention_weight_min": -29.95687484741211,
      "activations/layer13_attention_weight_max": 41.90157699584961,
      "activations/layer13_attention_weight_min": -32.38827133178711,
      "activations/layer14_attention_weight_max": 37.10494613647461,
      "activations/layer14_attention_weight_min": -31.180274963378906,
      "activations/layer15_attention_weight_max": 35.24516296386719,
      "activations/layer15_attention_weight_min": -29.563812255859375,
      "activations/layer16_attention_weight_max": 31.556529998779297,
      "activations/layer16_attention_weight_min": -28.635107040405273,
      "activations/layer17_attention_weight_max": 51.442474365234375,
      "activations/layer17_attention_weight_min": -43.69751739501953,
      "activations/layer18_attention_weight_max": 47.85818862915039,
      "activations/layer18_attention_weight_min": -37.65949249267578,
      "activations/layer19_attention_weight_max": 24.313993453979492,
      "activations/layer19_attention_weight_min": -22.869977951049805,
      "activations/layer1_attention_weight_max": 16.300247192382812,
      "activations/layer1_attention_weight_min": -14.227822303771973,
      "activations/layer20_attention_weight_max": 20.734983444213867,
      "activations/layer20_attention_weight_min": -21.163908004760742,
      "activations/layer21_attention_weight_max": 34.98540496826172,
      "activations/layer21_attention_weight_min": -22.878381729125977,
      "activations/layer22_attention_weight_max": 29.701555252075195,
      "activations/layer22_attention_weight_min": -25.64655303955078,
      "activations/layer23_attention_weight_max": 37.12751388549805,
      "activations/layer23_attention_weight_min": -23.704845428466797,
      "activations/layer2_attention_weight_max": 32.28826904296875,
      "activations/layer2_attention_weight_min": -30.455638885498047,
      "activations/layer3_attention_weight_max": 93.32274627685547,
      "activations/layer3_attention_weight_min": -93.17876434326172,
      "activations/layer4_attention_weight_max": 103.5960922241211,
      "activations/layer4_attention_weight_min": -98.57990264892578,
      "activations/layer5_attention_weight_max": 77.09077453613281,
      "activations/layer5_attention_weight_min": -78.08085632324219,
      "activations/layer6_attention_weight_max": 54.56776428222656,
      "activations/layer6_attention_weight_min": -54.17878341674805,
      "activations/layer7_attention_weight_max": 75.29572296142578,
      "activations/layer7_attention_weight_min": -77.05601501464844,
      "activations/layer8_attention_weight_max": 45.32545471191406,
      "activations/layer8_attention_weight_min": -44.033897399902344,
      "activations/layer9_attention_weight_max": 49.87641906738281,
      "activations/layer9_attention_weight_min": -52.08762741088867,
      "epoch": 10.11,
      "learning_rate": 8.566325757575755e-05,
      "loss": 2.7847,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4464,
      "eval_samples_per_second": 508.382,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4464,
      "eval_openwebtext_samples_per_second": 508.382,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 1.9755,
      "eval_wikitext_samples_per_second": 230.824,
      "step": 174000
    },
    {
      "epoch": 10.11,
      "eval_lambada_loss": 2.642578125,
      "eval_lambada_ppl": 14.049378009312553,
      "eval_lambada_runtime": 9.5693,
      "eval_lambada_samples_per_second": 508.812,
      "step": 174000
    },
    {
      "activations/layer0_attention_weight_max": 15.351786613464355,
      "activations/layer0_attention_weight_min": -13.834856033325195,
      "activations/layer10_attention_weight_max": 39.04877853393555,
      "activations/layer10_attention_weight_min": -37.07258605957031,
      "activations/layer11_attention_weight_max": 33.34538650512695,
      "activations/layer11_attention_weight_min": -33.1851921081543,
      "activations/layer12_attention_weight_max": 18.34073257446289,
      "activations/layer12_attention_weight_min": -24.428504943847656,
      "activations/layer13_attention_weight_max": 44.649696350097656,
      "activations/layer13_attention_weight_min": -29.56378936767578,
      "activations/layer14_attention_weight_max": 52.61410140991211,
      "activations/layer14_attention_weight_min": -34.56180191040039,
      "activations/layer15_attention_weight_max": 42.365447998046875,
      "activations/layer15_attention_weight_min": -29.110614776611328,
      "activations/layer16_attention_weight_max": 32.822998046875,
      "activations/layer16_attention_weight_min": -29.1534366607666,
      "activations/layer17_attention_weight_max": 53.972434997558594,
      "activations/layer17_attention_weight_min": -43.49153518676758,
      "activations/layer18_attention_weight_max": 48.05665969848633,
      "activations/layer18_attention_weight_min": -38.467063903808594,
      "activations/layer19_attention_weight_max": 22.022964477539062,
      "activations/layer19_attention_weight_min": -21.886634826660156,
      "activations/layer1_attention_weight_max": 16.257936477661133,
      "activations/layer1_attention_weight_min": -14.703761100769043,
      "activations/layer20_attention_weight_max": 23.5553035736084,
      "activations/layer20_attention_weight_min": -22.01970100402832,
      "activations/layer21_attention_weight_max": 41.39527130126953,
      "activations/layer21_attention_weight_min": -22.76009178161621,
      "activations/layer22_attention_weight_max": 35.51474380493164,
      "activations/layer22_attention_weight_min": -25.309906005859375,
      "activations/layer23_attention_weight_max": 38.93355941772461,
      "activations/layer23_attention_weight_min": -23.722171783447266,
      "activations/layer2_attention_weight_max": 31.420928955078125,
      "activations/layer2_attention_weight_min": -29.372482299804688,
      "activations/layer3_attention_weight_max": 91.72288513183594,
      "activations/layer3_attention_weight_min": -95.01473999023438,
      "activations/layer4_attention_weight_max": 104.03053283691406,
      "activations/layer4_attention_weight_min": -96.34333801269531,
      "activations/layer5_attention_weight_max": 76.73226928710938,
      "activations/layer5_attention_weight_min": -76.9503402709961,
      "activations/layer6_attention_weight_max": 52.14963150024414,
      "activations/layer6_attention_weight_min": -53.8745002746582,
      "activations/layer7_attention_weight_max": 73.04717254638672,
      "activations/layer7_attention_weight_min": -75.74162292480469,
      "activations/layer8_attention_weight_max": 45.19841766357422,
      "activations/layer8_attention_weight_min": -45.80042266845703,
      "activations/layer9_attention_weight_max": 47.06472396850586,
      "activations/layer9_attention_weight_min": -48.77337646484375,
      "epoch": 10.11,
      "learning_rate": 8.564431818181817e-05,
      "loss": 2.7789,
      "step": 174050
    },
    {
      "activations/layer0_attention_weight_max": 16.292903900146484,
      "activations/layer0_attention_weight_min": -13.730226516723633,
      "activations/layer10_attention_weight_max": 35.726322174072266,
      "activations/layer10_attention_weight_min": -35.713478088378906,
      "activations/layer11_attention_weight_max": 32.317481994628906,
      "activations/layer11_attention_weight_min": -34.018310546875,
      "activations/layer12_attention_weight_max": 18.469526290893555,
      "activations/layer12_attention_weight_min": -27.52937126159668,
      "activations/layer13_attention_weight_max": 33.913211822509766,
      "activations/layer13_attention_weight_min": -32.51555633544922,
      "activations/layer14_attention_weight_max": 36.160804748535156,
      "activations/layer14_attention_weight_min": -29.91788673400879,
      "activations/layer15_attention_weight_max": 33.85295867919922,
      "activations/layer15_attention_weight_min": -30.089736938476562,
      "activations/layer16_attention_weight_max": 33.98463439941406,
      "activations/layer16_attention_weight_min": -28.624177932739258,
      "activations/layer17_attention_weight_max": 53.750911712646484,
      "activations/layer17_attention_weight_min": -43.07292556762695,
      "activations/layer18_attention_weight_max": 46.14452362060547,
      "activations/layer18_attention_weight_min": -36.95355224609375,
      "activations/layer19_attention_weight_max": 21.04071044921875,
      "activations/layer19_attention_weight_min": -21.36044692993164,
      "activations/layer1_attention_weight_max": 16.30840301513672,
      "activations/layer1_attention_weight_min": -14.517669677734375,
      "activations/layer20_attention_weight_max": 22.310131072998047,
      "activations/layer20_attention_weight_min": -24.332799911499023,
      "activations/layer21_attention_weight_max": 42.18143844604492,
      "activations/layer21_attention_weight_min": -24.770105361938477,
      "activations/layer22_attention_weight_max": 31.674346923828125,
      "activations/layer22_attention_weight_min": -25.068859100341797,
      "activations/layer23_attention_weight_max": 35.96700668334961,
      "activations/layer23_attention_weight_min": -22.370426177978516,
      "activations/layer2_attention_weight_max": 33.98200225830078,
      "activations/layer2_attention_weight_min": -32.05134582519531,
      "activations/layer3_attention_weight_max": 93.43925476074219,
      "activations/layer3_attention_weight_min": -99.77718353271484,
      "activations/layer4_attention_weight_max": 102.2221908569336,
      "activations/layer4_attention_weight_min": -97.95482635498047,
      "activations/layer5_attention_weight_max": 77.62060546875,
      "activations/layer5_attention_weight_min": -77.53038024902344,
      "activations/layer6_attention_weight_max": 55.078861236572266,
      "activations/layer6_attention_weight_min": -54.311256408691406,
      "activations/layer7_attention_weight_max": 78.0892562866211,
      "activations/layer7_attention_weight_min": -76.57196044921875,
      "activations/layer8_attention_weight_max": 44.49696731567383,
      "activations/layer8_attention_weight_min": -47.51723098754883,
      "activations/layer9_attention_weight_max": 50.77605438232422,
      "activations/layer9_attention_weight_min": -49.08531188964844,
      "epoch": 10.12,
      "learning_rate": 8.562537878787878e-05,
      "loss": 2.7778,
      "step": 174100
    },
    {
      "activations/layer0_attention_weight_max": 14.976393699645996,
      "activations/layer0_attention_weight_min": -13.647850036621094,
      "activations/layer10_attention_weight_max": 37.01184844970703,
      "activations/layer10_attention_weight_min": -37.418373107910156,
      "activations/layer11_attention_weight_max": 33.19051742553711,
      "activations/layer11_attention_weight_min": -34.37458801269531,
      "activations/layer12_attention_weight_max": 21.239002227783203,
      "activations/layer12_attention_weight_min": -27.895360946655273,
      "activations/layer13_attention_weight_max": 40.454830169677734,
      "activations/layer13_attention_weight_min": -31.200159072875977,
      "activations/layer14_attention_weight_max": 37.06293869018555,
      "activations/layer14_attention_weight_min": -30.226102828979492,
      "activations/layer15_attention_weight_max": 33.120296478271484,
      "activations/layer15_attention_weight_min": -29.776460647583008,
      "activations/layer16_attention_weight_max": 28.77303123474121,
      "activations/layer16_attention_weight_min": -29.1491641998291,
      "activations/layer17_attention_weight_max": 52.725128173828125,
      "activations/layer17_attention_weight_min": -45.52766418457031,
      "activations/layer18_attention_weight_max": 46.27078628540039,
      "activations/layer18_attention_weight_min": -37.463226318359375,
      "activations/layer19_attention_weight_max": 23.09317970275879,
      "activations/layer19_attention_weight_min": -21.75752067565918,
      "activations/layer1_attention_weight_max": 17.642702102661133,
      "activations/layer1_attention_weight_min": -15.250265121459961,
      "activations/layer20_attention_weight_max": 22.389196395874023,
      "activations/layer20_attention_weight_min": -19.94463348388672,
      "activations/layer21_attention_weight_max": 36.471614837646484,
      "activations/layer21_attention_weight_min": -23.51881980895996,
      "activations/layer22_attention_weight_max": 30.8695068359375,
      "activations/layer22_attention_weight_min": -28.378618240356445,
      "activations/layer23_attention_weight_max": 37.8481559753418,
      "activations/layer23_attention_weight_min": -24.578176498413086,
      "activations/layer2_attention_weight_max": 30.359455108642578,
      "activations/layer2_attention_weight_min": -31.206846237182617,
      "activations/layer3_attention_weight_max": 89.5895004272461,
      "activations/layer3_attention_weight_min": -90.30686950683594,
      "activations/layer4_attention_weight_max": 99.96299743652344,
      "activations/layer4_attention_weight_min": -98.6031723022461,
      "activations/layer5_attention_weight_max": 75.07852172851562,
      "activations/layer5_attention_weight_min": -80.10246276855469,
      "activations/layer6_attention_weight_max": 52.52641296386719,
      "activations/layer6_attention_weight_min": -51.832115173339844,
      "activations/layer7_attention_weight_max": 75.65139770507812,
      "activations/layer7_attention_weight_min": -73.71768188476562,
      "activations/layer8_attention_weight_max": 45.12192916870117,
      "activations/layer8_attention_weight_min": -47.69707107543945,
      "activations/layer9_attention_weight_max": 50.2999382019043,
      "activations/layer9_attention_weight_min": -50.94184875488281,
      "epoch": 10.12,
      "learning_rate": 8.560643939393939e-05,
      "loss": 2.778,
      "step": 174150
    },
    {
      "activations/layer0_attention_weight_max": 15.390905380249023,
      "activations/layer0_attention_weight_min": -13.548922538757324,
      "activations/layer10_attention_weight_max": 38.92548370361328,
      "activations/layer10_attention_weight_min": -37.22850036621094,
      "activations/layer11_attention_weight_max": 33.35002899169922,
      "activations/layer11_attention_weight_min": -33.93247604370117,
      "activations/layer12_attention_weight_max": 21.22366714477539,
      "activations/layer12_attention_weight_min": -23.24762725830078,
      "activations/layer13_attention_weight_max": 46.70473861694336,
      "activations/layer13_attention_weight_min": -31.154787063598633,
      "activations/layer14_attention_weight_max": 39.28907775878906,
      "activations/layer14_attention_weight_min": -30.152685165405273,
      "activations/layer15_attention_weight_max": 32.73912811279297,
      "activations/layer15_attention_weight_min": -29.778675079345703,
      "activations/layer16_attention_weight_max": 30.61554718017578,
      "activations/layer16_attention_weight_min": -28.211105346679688,
      "activations/layer17_attention_weight_max": 50.0928955078125,
      "activations/layer17_attention_weight_min": -43.89104080200195,
      "activations/layer18_attention_weight_max": 44.184715270996094,
      "activations/layer18_attention_weight_min": -37.51042938232422,
      "activations/layer19_attention_weight_max": 20.180505752563477,
      "activations/layer19_attention_weight_min": -20.62331199645996,
      "activations/layer1_attention_weight_max": 16.994457244873047,
      "activations/layer1_attention_weight_min": -17.3101806640625,
      "activations/layer20_attention_weight_max": 22.009443283081055,
      "activations/layer20_attention_weight_min": -21.865386962890625,
      "activations/layer21_attention_weight_max": 35.26862335205078,
      "activations/layer21_attention_weight_min": -25.0712833404541,
      "activations/layer22_attention_weight_max": 33.8631706237793,
      "activations/layer22_attention_weight_min": -26.22037124633789,
      "activations/layer23_attention_weight_max": 37.31914138793945,
      "activations/layer23_attention_weight_min": -24.584461212158203,
      "activations/layer2_attention_weight_max": 33.135597229003906,
      "activations/layer2_attention_weight_min": -31.094585418701172,
      "activations/layer3_attention_weight_max": 94.83526611328125,
      "activations/layer3_attention_weight_min": -99.31077575683594,
      "activations/layer4_attention_weight_max": 107.19673919677734,
      "activations/layer4_attention_weight_min": -97.77090454101562,
      "activations/layer5_attention_weight_max": 79.70443725585938,
      "activations/layer5_attention_weight_min": -80.55854034423828,
      "activations/layer6_attention_weight_max": 61.58732986450195,
      "activations/layer6_attention_weight_min": -56.11515808105469,
      "activations/layer7_attention_weight_max": 81.67855834960938,
      "activations/layer7_attention_weight_min": -79.32364654541016,
      "activations/layer8_attention_weight_max": 45.63733673095703,
      "activations/layer8_attention_weight_min": -49.46574783325195,
      "activations/layer9_attention_weight_max": 51.578792572021484,
      "activations/layer9_attention_weight_min": -51.00056838989258,
      "epoch": 10.12,
      "learning_rate": 8.558749999999999e-05,
      "loss": 2.7899,
      "step": 174200
    },
    {
      "activations/layer0_attention_weight_max": 15.270291328430176,
      "activations/layer0_attention_weight_min": -13.81799030303955,
      "activations/layer10_attention_weight_max": 40.79572677612305,
      "activations/layer10_attention_weight_min": -37.48622512817383,
      "activations/layer11_attention_weight_max": 34.668033599853516,
      "activations/layer11_attention_weight_min": -33.64815139770508,
      "activations/layer12_attention_weight_max": 19.571279525756836,
      "activations/layer12_attention_weight_min": -26.194446563720703,
      "activations/layer13_attention_weight_max": 42.89189910888672,
      "activations/layer13_attention_weight_min": -28.799537658691406,
      "activations/layer14_attention_weight_max": 37.84587478637695,
      "activations/layer14_attention_weight_min": -30.96379852294922,
      "activations/layer15_attention_weight_max": 33.13856887817383,
      "activations/layer15_attention_weight_min": -30.672414779663086,
      "activations/layer16_attention_weight_max": 30.747074127197266,
      "activations/layer16_attention_weight_min": -28.43035316467285,
      "activations/layer17_attention_weight_max": 53.92900466918945,
      "activations/layer17_attention_weight_min": -41.53485107421875,
      "activations/layer18_attention_weight_max": 51.9237060546875,
      "activations/layer18_attention_weight_min": -37.48493194580078,
      "activations/layer19_attention_weight_max": 22.261085510253906,
      "activations/layer19_attention_weight_min": -20.693870544433594,
      "activations/layer1_attention_weight_max": 17.008241653442383,
      "activations/layer1_attention_weight_min": -17.574111938476562,
      "activations/layer20_attention_weight_max": 24.755943298339844,
      "activations/layer20_attention_weight_min": -20.701438903808594,
      "activations/layer21_attention_weight_max": 38.298614501953125,
      "activations/layer21_attention_weight_min": -21.56119728088379,
      "activations/layer22_attention_weight_max": 31.41105842590332,
      "activations/layer22_attention_weight_min": -25.580415725708008,
      "activations/layer23_attention_weight_max": 38.668663024902344,
      "activations/layer23_attention_weight_min": -23.833913803100586,
      "activations/layer2_attention_weight_max": 32.58345031738281,
      "activations/layer2_attention_weight_min": -33.95183563232422,
      "activations/layer3_attention_weight_max": 99.75711059570312,
      "activations/layer3_attention_weight_min": -98.50285339355469,
      "activations/layer4_attention_weight_max": 112.44454193115234,
      "activations/layer4_attention_weight_min": -99.80023956298828,
      "activations/layer5_attention_weight_max": 88.56590270996094,
      "activations/layer5_attention_weight_min": -79.2154312133789,
      "activations/layer6_attention_weight_max": 62.0312385559082,
      "activations/layer6_attention_weight_min": -59.41598892211914,
      "activations/layer7_attention_weight_max": 83.9577407836914,
      "activations/layer7_attention_weight_min": -77.94580078125,
      "activations/layer8_attention_weight_max": 49.05733871459961,
      "activations/layer8_attention_weight_min": -50.743255615234375,
      "activations/layer9_attention_weight_max": 52.45379638671875,
      "activations/layer9_attention_weight_min": -51.82525634765625,
      "epoch": 10.12,
      "learning_rate": 8.55685606060606e-05,
      "loss": 2.7725,
      "step": 174250
    },
    {
      "activations/layer0_attention_weight_max": 15.462650299072266,
      "activations/layer0_attention_weight_min": -13.718474388122559,
      "activations/layer10_attention_weight_max": 35.097015380859375,
      "activations/layer10_attention_weight_min": -36.51951599121094,
      "activations/layer11_attention_weight_max": 31.213788986206055,
      "activations/layer11_attention_weight_min": -31.744359970092773,
      "activations/layer12_attention_weight_max": 17.56106185913086,
      "activations/layer12_attention_weight_min": -22.38318634033203,
      "activations/layer13_attention_weight_max": 36.02772521972656,
      "activations/layer13_attention_weight_min": -29.376497268676758,
      "activations/layer14_attention_weight_max": 35.35527420043945,
      "activations/layer14_attention_weight_min": -29.459121704101562,
      "activations/layer15_attention_weight_max": 31.801931381225586,
      "activations/layer15_attention_weight_min": -28.23932456970215,
      "activations/layer16_attention_weight_max": 30.616804122924805,
      "activations/layer16_attention_weight_min": -28.13591766357422,
      "activations/layer17_attention_weight_max": 52.52267074584961,
      "activations/layer17_attention_weight_min": -42.403297424316406,
      "activations/layer18_attention_weight_max": 44.12317657470703,
      "activations/layer18_attention_weight_min": -35.585540771484375,
      "activations/layer19_attention_weight_max": 19.862966537475586,
      "activations/layer19_attention_weight_min": -21.535215377807617,
      "activations/layer1_attention_weight_max": 16.173736572265625,
      "activations/layer1_attention_weight_min": -16.90092658996582,
      "activations/layer20_attention_weight_max": 19.423173904418945,
      "activations/layer20_attention_weight_min": -21.176042556762695,
      "activations/layer21_attention_weight_max": 34.580535888671875,
      "activations/layer21_attention_weight_min": -23.42486572265625,
      "activations/layer22_attention_weight_max": 31.049219131469727,
      "activations/layer22_attention_weight_min": -27.16998291015625,
      "activations/layer23_attention_weight_max": 39.0367431640625,
      "activations/layer23_attention_weight_min": -25.736358642578125,
      "activations/layer2_attention_weight_max": 30.21253204345703,
      "activations/layer2_attention_weight_min": -31.108915328979492,
      "activations/layer3_attention_weight_max": 87.993896484375,
      "activations/layer3_attention_weight_min": -87.25511169433594,
      "activations/layer4_attention_weight_max": 95.17310333251953,
      "activations/layer4_attention_weight_min": -93.50877380371094,
      "activations/layer5_attention_weight_max": 71.88954162597656,
      "activations/layer5_attention_weight_min": -82.05708312988281,
      "activations/layer6_attention_weight_max": 50.712013244628906,
      "activations/layer6_attention_weight_min": -51.72999954223633,
      "activations/layer7_attention_weight_max": 74.10336303710938,
      "activations/layer7_attention_weight_min": -76.2821273803711,
      "activations/layer8_attention_weight_max": 42.963706970214844,
      "activations/layer8_attention_weight_min": -45.82490539550781,
      "activations/layer9_attention_weight_max": 45.40424728393555,
      "activations/layer9_attention_weight_min": -52.21449661254883,
      "epoch": 10.13,
      "learning_rate": 8.55496212121212e-05,
      "loss": 2.7713,
      "step": 174300
    },
    {
      "activations/layer0_attention_weight_max": 14.704670906066895,
      "activations/layer0_attention_weight_min": -14.192584991455078,
      "activations/layer10_attention_weight_max": 45.056724548339844,
      "activations/layer10_attention_weight_min": -39.136348724365234,
      "activations/layer11_attention_weight_max": 38.7681884765625,
      "activations/layer11_attention_weight_min": -33.314414978027344,
      "activations/layer12_attention_weight_max": 23.3125,
      "activations/layer12_attention_weight_min": -33.78654861450195,
      "activations/layer13_attention_weight_max": 39.1394157409668,
      "activations/layer13_attention_weight_min": -34.662574768066406,
      "activations/layer14_attention_weight_max": 40.51932907104492,
      "activations/layer14_attention_weight_min": -29.852752685546875,
      "activations/layer15_attention_weight_max": 34.355709075927734,
      "activations/layer15_attention_weight_min": -29.549009323120117,
      "activations/layer16_attention_weight_max": 29.569128036499023,
      "activations/layer16_attention_weight_min": -28.279266357421875,
      "activations/layer17_attention_weight_max": 51.63773727416992,
      "activations/layer17_attention_weight_min": -43.348819732666016,
      "activations/layer18_attention_weight_max": 44.5129280090332,
      "activations/layer18_attention_weight_min": -38.123741149902344,
      "activations/layer19_attention_weight_max": 21.227235794067383,
      "activations/layer19_attention_weight_min": -23.642578125,
      "activations/layer1_attention_weight_max": 16.824604034423828,
      "activations/layer1_attention_weight_min": -15.773573875427246,
      "activations/layer20_attention_weight_max": 22.052322387695312,
      "activations/layer20_attention_weight_min": -22.46894645690918,
      "activations/layer21_attention_weight_max": 45.60967254638672,
      "activations/layer21_attention_weight_min": -32.084083557128906,
      "activations/layer22_attention_weight_max": 28.387481689453125,
      "activations/layer22_attention_weight_min": -24.284645080566406,
      "activations/layer23_attention_weight_max": 37.246421813964844,
      "activations/layer23_attention_weight_min": -23.683570861816406,
      "activations/layer2_attention_weight_max": 34.050079345703125,
      "activations/layer2_attention_weight_min": -31.51874542236328,
      "activations/layer3_attention_weight_max": 94.49298858642578,
      "activations/layer3_attention_weight_min": -97.0431137084961,
      "activations/layer4_attention_weight_max": 102.42707824707031,
      "activations/layer4_attention_weight_min": -98.57323455810547,
      "activations/layer5_attention_weight_max": 76.6007080078125,
      "activations/layer5_attention_weight_min": -76.48710632324219,
      "activations/layer6_attention_weight_max": 55.14812469482422,
      "activations/layer6_attention_weight_min": -54.15769577026367,
      "activations/layer7_attention_weight_max": 80.00643157958984,
      "activations/layer7_attention_weight_min": -71.97349548339844,
      "activations/layer8_attention_weight_max": 51.55695724487305,
      "activations/layer8_attention_weight_min": -47.41743850708008,
      "activations/layer9_attention_weight_max": 59.04279708862305,
      "activations/layer9_attention_weight_min": -51.51259994506836,
      "epoch": 10.13,
      "learning_rate": 8.553068181818182e-05,
      "loss": 2.7838,
      "step": 174350
    },
    {
      "activations/layer0_attention_weight_max": 15.488886833190918,
      "activations/layer0_attention_weight_min": -14.199883460998535,
      "activations/layer10_attention_weight_max": 36.27276611328125,
      "activations/layer10_attention_weight_min": -36.58990478515625,
      "activations/layer11_attention_weight_max": 31.84268569946289,
      "activations/layer11_attention_weight_min": -30.80677604675293,
      "activations/layer12_attention_weight_max": 18.2452449798584,
      "activations/layer12_attention_weight_min": -25.18446159362793,
      "activations/layer13_attention_weight_max": 43.7407341003418,
      "activations/layer13_attention_weight_min": -32.70928192138672,
      "activations/layer14_attention_weight_max": 42.816043853759766,
      "activations/layer14_attention_weight_min": -31.869159698486328,
      "activations/layer15_attention_weight_max": 37.420326232910156,
      "activations/layer15_attention_weight_min": -29.71566390991211,
      "activations/layer16_attention_weight_max": 35.396026611328125,
      "activations/layer16_attention_weight_min": -29.109771728515625,
      "activations/layer17_attention_weight_max": 54.58470153808594,
      "activations/layer17_attention_weight_min": -43.541412353515625,
      "activations/layer18_attention_weight_max": 47.1999397277832,
      "activations/layer18_attention_weight_min": -38.87946319580078,
      "activations/layer19_attention_weight_max": 21.039966583251953,
      "activations/layer19_attention_weight_min": -21.598142623901367,
      "activations/layer1_attention_weight_max": 17.37415885925293,
      "activations/layer1_attention_weight_min": -15.252822875976562,
      "activations/layer20_attention_weight_max": 21.354339599609375,
      "activations/layer20_attention_weight_min": -22.57546615600586,
      "activations/layer21_attention_weight_max": 39.097808837890625,
      "activations/layer21_attention_weight_min": -23.006044387817383,
      "activations/layer22_attention_weight_max": 33.972679138183594,
      "activations/layer22_attention_weight_min": -29.438257217407227,
      "activations/layer23_attention_weight_max": 37.47454071044922,
      "activations/layer23_attention_weight_min": -21.944143295288086,
      "activations/layer2_attention_weight_max": 32.15504455566406,
      "activations/layer2_attention_weight_min": -30.913528442382812,
      "activations/layer3_attention_weight_max": 89.43894958496094,
      "activations/layer3_attention_weight_min": -91.350830078125,
      "activations/layer4_attention_weight_max": 99.02217864990234,
      "activations/layer4_attention_weight_min": -93.77860260009766,
      "activations/layer5_attention_weight_max": 75.4202651977539,
      "activations/layer5_attention_weight_min": -77.91226196289062,
      "activations/layer6_attention_weight_max": 52.918373107910156,
      "activations/layer6_attention_weight_min": -53.32962417602539,
      "activations/layer7_attention_weight_max": 75.2017822265625,
      "activations/layer7_attention_weight_min": -76.61505889892578,
      "activations/layer8_attention_weight_max": 43.94648361206055,
      "activations/layer8_attention_weight_min": -43.91594696044922,
      "activations/layer9_attention_weight_max": 48.761512756347656,
      "activations/layer9_attention_weight_min": -48.43180465698242,
      "epoch": 10.13,
      "learning_rate": 8.551174242424242e-05,
      "loss": 2.7613,
      "step": 174400
    },
    {
      "activations/layer0_attention_weight_max": 14.972078323364258,
      "activations/layer0_attention_weight_min": -14.034416198730469,
      "activations/layer10_attention_weight_max": 37.37333297729492,
      "activations/layer10_attention_weight_min": -33.209136962890625,
      "activations/layer11_attention_weight_max": 31.270214080810547,
      "activations/layer11_attention_weight_min": -30.424922943115234,
      "activations/layer12_attention_weight_max": 17.204105377197266,
      "activations/layer12_attention_weight_min": -25.34136390686035,
      "activations/layer13_attention_weight_max": 38.48160934448242,
      "activations/layer13_attention_weight_min": -29.042694091796875,
      "activations/layer14_attention_weight_max": 35.461883544921875,
      "activations/layer14_attention_weight_min": -29.633508682250977,
      "activations/layer15_attention_weight_max": 32.632537841796875,
      "activations/layer15_attention_weight_min": -30.522659301757812,
      "activations/layer16_attention_weight_max": 30.50980567932129,
      "activations/layer16_attention_weight_min": -29.00084114074707,
      "activations/layer17_attention_weight_max": 51.78900909423828,
      "activations/layer17_attention_weight_min": -44.7081184387207,
      "activations/layer18_attention_weight_max": 49.155853271484375,
      "activations/layer18_attention_weight_min": -35.53293228149414,
      "activations/layer19_attention_weight_max": 21.647645950317383,
      "activations/layer19_attention_weight_min": -21.901790618896484,
      "activations/layer1_attention_weight_max": 17.43267822265625,
      "activations/layer1_attention_weight_min": -14.324252128601074,
      "activations/layer20_attention_weight_max": 22.827878952026367,
      "activations/layer20_attention_weight_min": -22.36469078063965,
      "activations/layer21_attention_weight_max": 36.50919723510742,
      "activations/layer21_attention_weight_min": -20.81646156311035,
      "activations/layer22_attention_weight_max": 33.26933288574219,
      "activations/layer22_attention_weight_min": -25.146299362182617,
      "activations/layer23_attention_weight_max": 38.41492462158203,
      "activations/layer23_attention_weight_min": -23.62189483642578,
      "activations/layer2_attention_weight_max": 33.168243408203125,
      "activations/layer2_attention_weight_min": -31.65085220336914,
      "activations/layer3_attention_weight_max": 95.5945816040039,
      "activations/layer3_attention_weight_min": -93.45135498046875,
      "activations/layer4_attention_weight_max": 99.53063201904297,
      "activations/layer4_attention_weight_min": -96.77674102783203,
      "activations/layer5_attention_weight_max": 76.47898864746094,
      "activations/layer5_attention_weight_min": -77.9134521484375,
      "activations/layer6_attention_weight_max": 55.01673126220703,
      "activations/layer6_attention_weight_min": -52.10404586791992,
      "activations/layer7_attention_weight_max": 73.14583587646484,
      "activations/layer7_attention_weight_min": -74.24585723876953,
      "activations/layer8_attention_weight_max": 46.296058654785156,
      "activations/layer8_attention_weight_min": -49.53815841674805,
      "activations/layer9_attention_weight_max": 48.55833053588867,
      "activations/layer9_attention_weight_min": -47.37894821166992,
      "epoch": 10.14,
      "learning_rate": 8.549280303030302e-05,
      "loss": 2.7737,
      "step": 174450
    },
    {
      "activations/layer0_attention_weight_max": 15.592512130737305,
      "activations/layer0_attention_weight_min": -14.26266098022461,
      "activations/layer10_attention_weight_max": 41.18507766723633,
      "activations/layer10_attention_weight_min": -37.4573860168457,
      "activations/layer11_attention_weight_max": 37.39527130126953,
      "activations/layer11_attention_weight_min": -33.8221435546875,
      "activations/layer12_attention_weight_max": 24.2365779876709,
      "activations/layer12_attention_weight_min": -27.408893585205078,
      "activations/layer13_attention_weight_max": 45.23997497558594,
      "activations/layer13_attention_weight_min": -31.211483001708984,
      "activations/layer14_attention_weight_max": 53.14072799682617,
      "activations/layer14_attention_weight_min": -29.7558650970459,
      "activations/layer15_attention_weight_max": 44.685672760009766,
      "activations/layer15_attention_weight_min": -27.917173385620117,
      "activations/layer16_attention_weight_max": 28.845983505249023,
      "activations/layer16_attention_weight_min": -27.543128967285156,
      "activations/layer17_attention_weight_max": 53.877105712890625,
      "activations/layer17_attention_weight_min": -45.1201057434082,
      "activations/layer18_attention_weight_max": 45.21627426147461,
      "activations/layer18_attention_weight_min": -36.721435546875,
      "activations/layer19_attention_weight_max": 21.349903106689453,
      "activations/layer19_attention_weight_min": -21.73402214050293,
      "activations/layer1_attention_weight_max": 16.659385681152344,
      "activations/layer1_attention_weight_min": -16.012779235839844,
      "activations/layer20_attention_weight_max": 23.044496536254883,
      "activations/layer20_attention_weight_min": -26.882240295410156,
      "activations/layer21_attention_weight_max": 44.15674591064453,
      "activations/layer21_attention_weight_min": -26.39815330505371,
      "activations/layer22_attention_weight_max": 34.611061096191406,
      "activations/layer22_attention_weight_min": -27.000967025756836,
      "activations/layer23_attention_weight_max": 35.959693908691406,
      "activations/layer23_attention_weight_min": -26.106060028076172,
      "activations/layer2_attention_weight_max": 33.93292236328125,
      "activations/layer2_attention_weight_min": -32.38231658935547,
      "activations/layer3_attention_weight_max": 96.6175308227539,
      "activations/layer3_attention_weight_min": -99.06982421875,
      "activations/layer4_attention_weight_max": 105.5786361694336,
      "activations/layer4_attention_weight_min": -97.70455169677734,
      "activations/layer5_attention_weight_max": 85.7021713256836,
      "activations/layer5_attention_weight_min": -80.3141098022461,
      "activations/layer6_attention_weight_max": 54.55828094482422,
      "activations/layer6_attention_weight_min": -55.910404205322266,
      "activations/layer7_attention_weight_max": 91.32354736328125,
      "activations/layer7_attention_weight_min": -75.97999572753906,
      "activations/layer8_attention_weight_max": 52.806358337402344,
      "activations/layer8_attention_weight_min": -45.51900100708008,
      "activations/layer9_attention_weight_max": 66.99742889404297,
      "activations/layer9_attention_weight_min": -51.702999114990234,
      "epoch": 10.14,
      "learning_rate": 8.547386363636364e-05,
      "loss": 2.7866,
      "step": 174500
    },
    {
      "activations/layer0_attention_weight_max": 16.322097778320312,
      "activations/layer0_attention_weight_min": -14.228655815124512,
      "activations/layer10_attention_weight_max": 40.5761604309082,
      "activations/layer10_attention_weight_min": -38.854522705078125,
      "activations/layer11_attention_weight_max": 35.18084716796875,
      "activations/layer11_attention_weight_min": -34.379669189453125,
      "activations/layer12_attention_weight_max": 19.2592716217041,
      "activations/layer12_attention_weight_min": -23.719635009765625,
      "activations/layer13_attention_weight_max": 38.287017822265625,
      "activations/layer13_attention_weight_min": -29.89147186279297,
      "activations/layer14_attention_weight_max": 38.71619415283203,
      "activations/layer14_attention_weight_min": -31.104097366333008,
      "activations/layer15_attention_weight_max": 34.69712448120117,
      "activations/layer15_attention_weight_min": -29.69898796081543,
      "activations/layer16_attention_weight_max": 31.188081741333008,
      "activations/layer16_attention_weight_min": -29.41168212890625,
      "activations/layer17_attention_weight_max": 49.88258743286133,
      "activations/layer17_attention_weight_min": -41.40944290161133,
      "activations/layer18_attention_weight_max": 45.884254455566406,
      "activations/layer18_attention_weight_min": -36.81517791748047,
      "activations/layer19_attention_weight_max": 21.258586883544922,
      "activations/layer19_attention_weight_min": -20.973119735717773,
      "activations/layer1_attention_weight_max": 17.575580596923828,
      "activations/layer1_attention_weight_min": -15.032323837280273,
      "activations/layer20_attention_weight_max": 21.657140731811523,
      "activations/layer20_attention_weight_min": -19.813291549682617,
      "activations/layer21_attention_weight_max": 38.08980178833008,
      "activations/layer21_attention_weight_min": -23.14289093017578,
      "activations/layer22_attention_weight_max": 32.74464416503906,
      "activations/layer22_attention_weight_min": -26.9571590423584,
      "activations/layer23_attention_weight_max": 39.971397399902344,
      "activations/layer23_attention_weight_min": -21.842815399169922,
      "activations/layer2_attention_weight_max": 32.928314208984375,
      "activations/layer2_attention_weight_min": -33.27770233154297,
      "activations/layer3_attention_weight_max": 94.1107406616211,
      "activations/layer3_attention_weight_min": -100.37092590332031,
      "activations/layer4_attention_weight_max": 101.86202239990234,
      "activations/layer4_attention_weight_min": -97.22710418701172,
      "activations/layer5_attention_weight_max": 73.30508422851562,
      "activations/layer5_attention_weight_min": -79.44166564941406,
      "activations/layer6_attention_weight_max": 55.50871276855469,
      "activations/layer6_attention_weight_min": -53.126224517822266,
      "activations/layer7_attention_weight_max": 79.68771362304688,
      "activations/layer7_attention_weight_min": -74.29053497314453,
      "activations/layer8_attention_weight_max": 47.02348709106445,
      "activations/layer8_attention_weight_min": -48.00387191772461,
      "activations/layer9_attention_weight_max": 53.21657180786133,
      "activations/layer9_attention_weight_min": -52.42802810668945,
      "epoch": 10.14,
      "learning_rate": 8.545492424242422e-05,
      "loss": 2.7829,
      "step": 174550
    },
    {
      "activations/layer0_attention_weight_max": 16.211881637573242,
      "activations/layer0_attention_weight_min": -14.191385269165039,
      "activations/layer10_attention_weight_max": 43.21782302856445,
      "activations/layer10_attention_weight_min": -40.18372344970703,
      "activations/layer11_attention_weight_max": 38.187591552734375,
      "activations/layer11_attention_weight_min": -35.91625213623047,
      "activations/layer12_attention_weight_max": 23.208295822143555,
      "activations/layer12_attention_weight_min": -25.98223304748535,
      "activations/layer13_attention_weight_max": 50.72317123413086,
      "activations/layer13_attention_weight_min": -29.53173065185547,
      "activations/layer14_attention_weight_max": 41.737815856933594,
      "activations/layer14_attention_weight_min": -33.03602981567383,
      "activations/layer15_attention_weight_max": 37.38433074951172,
      "activations/layer15_attention_weight_min": -31.3581485748291,
      "activations/layer16_attention_weight_max": 33.370967864990234,
      "activations/layer16_attention_weight_min": -30.93883514404297,
      "activations/layer17_attention_weight_max": 58.75044631958008,
      "activations/layer17_attention_weight_min": -49.133052825927734,
      "activations/layer18_attention_weight_max": 47.51331329345703,
      "activations/layer18_attention_weight_min": -39.560791015625,
      "activations/layer19_attention_weight_max": 22.209007263183594,
      "activations/layer19_attention_weight_min": -21.94392204284668,
      "activations/layer1_attention_weight_max": 16.70435333251953,
      "activations/layer1_attention_weight_min": -15.392007827758789,
      "activations/layer20_attention_weight_max": 23.251691818237305,
      "activations/layer20_attention_weight_min": -21.5401668548584,
      "activations/layer21_attention_weight_max": 42.516212463378906,
      "activations/layer21_attention_weight_min": -26.05745506286621,
      "activations/layer22_attention_weight_max": 31.012794494628906,
      "activations/layer22_attention_weight_min": -28.988563537597656,
      "activations/layer23_attention_weight_max": 33.62028121948242,
      "activations/layer23_attention_weight_min": -24.058006286621094,
      "activations/layer2_attention_weight_max": 37.31459426879883,
      "activations/layer2_attention_weight_min": -31.71332359313965,
      "activations/layer3_attention_weight_max": 104.76679992675781,
      "activations/layer3_attention_weight_min": -99.8360366821289,
      "activations/layer4_attention_weight_max": 105.12602996826172,
      "activations/layer4_attention_weight_min": -97.35002899169922,
      "activations/layer5_attention_weight_max": 75.72867584228516,
      "activations/layer5_attention_weight_min": -81.89508056640625,
      "activations/layer6_attention_weight_max": 54.86708068847656,
      "activations/layer6_attention_weight_min": -55.892906188964844,
      "activations/layer7_attention_weight_max": 81.7964096069336,
      "activations/layer7_attention_weight_min": -80.26910400390625,
      "activations/layer8_attention_weight_max": 48.9458122253418,
      "activations/layer8_attention_weight_min": -49.49506378173828,
      "activations/layer9_attention_weight_max": 55.911415100097656,
      "activations/layer9_attention_weight_min": -54.895626068115234,
      "epoch": 10.15,
      "learning_rate": 8.543598484848484e-05,
      "loss": 2.7984,
      "step": 174600
    },
    {
      "activations/layer0_attention_weight_max": 15.685379981994629,
      "activations/layer0_attention_weight_min": -14.121086120605469,
      "activations/layer10_attention_weight_max": 35.66959762573242,
      "activations/layer10_attention_weight_min": -38.04465103149414,
      "activations/layer11_attention_weight_max": 30.66364097595215,
      "activations/layer11_attention_weight_min": -31.076892852783203,
      "activations/layer12_attention_weight_max": 21.565969467163086,
      "activations/layer12_attention_weight_min": -24.50330352783203,
      "activations/layer13_attention_weight_max": 51.966880798339844,
      "activations/layer13_attention_weight_min": -31.741403579711914,
      "activations/layer14_attention_weight_max": 45.219749450683594,
      "activations/layer14_attention_weight_min": -31.782974243164062,
      "activations/layer15_attention_weight_max": 37.788360595703125,
      "activations/layer15_attention_weight_min": -31.06239891052246,
      "activations/layer16_attention_weight_max": 36.87395477294922,
      "activations/layer16_attention_weight_min": -29.049123764038086,
      "activations/layer17_attention_weight_max": 54.405372619628906,
      "activations/layer17_attention_weight_min": -43.165157318115234,
      "activations/layer18_attention_weight_max": 52.44086456298828,
      "activations/layer18_attention_weight_min": -39.67018127441406,
      "activations/layer19_attention_weight_max": 27.161718368530273,
      "activations/layer19_attention_weight_min": -22.332183837890625,
      "activations/layer1_attention_weight_max": 16.798803329467773,
      "activations/layer1_attention_weight_min": -13.494377136230469,
      "activations/layer20_attention_weight_max": 27.921113967895508,
      "activations/layer20_attention_weight_min": -23.24764633178711,
      "activations/layer21_attention_weight_max": 40.13452911376953,
      "activations/layer21_attention_weight_min": -25.366954803466797,
      "activations/layer22_attention_weight_max": 36.127201080322266,
      "activations/layer22_attention_weight_min": -26.556991577148438,
      "activations/layer23_attention_weight_max": 41.13198471069336,
      "activations/layer23_attention_weight_min": -25.59906578063965,
      "activations/layer2_attention_weight_max": 34.8721923828125,
      "activations/layer2_attention_weight_min": -31.75703239440918,
      "activations/layer3_attention_weight_max": 95.34232330322266,
      "activations/layer3_attention_weight_min": -97.73360443115234,
      "activations/layer4_attention_weight_max": 98.59486389160156,
      "activations/layer4_attention_weight_min": -94.54102325439453,
      "activations/layer5_attention_weight_max": 73.87046813964844,
      "activations/layer5_attention_weight_min": -76.24156188964844,
      "activations/layer6_attention_weight_max": 52.824832916259766,
      "activations/layer6_attention_weight_min": -52.759761810302734,
      "activations/layer7_attention_weight_max": 72.10246276855469,
      "activations/layer7_attention_weight_min": -73.50921630859375,
      "activations/layer8_attention_weight_max": 47.28935623168945,
      "activations/layer8_attention_weight_min": -46.016422271728516,
      "activations/layer9_attention_weight_max": 47.09171676635742,
      "activations/layer9_attention_weight_min": -48.20079040527344,
      "epoch": 10.15,
      "learning_rate": 8.541704545454545e-05,
      "loss": 2.7933,
      "step": 174650
    },
    {
      "activations/layer0_attention_weight_max": 16.161657333374023,
      "activations/layer0_attention_weight_min": -14.370259284973145,
      "activations/layer10_attention_weight_max": 35.54437255859375,
      "activations/layer10_attention_weight_min": -34.41878128051758,
      "activations/layer11_attention_weight_max": 31.3707275390625,
      "activations/layer11_attention_weight_min": -32.5905876159668,
      "activations/layer12_attention_weight_max": 21.300140380859375,
      "activations/layer12_attention_weight_min": -24.556697845458984,
      "activations/layer13_attention_weight_max": 47.8550910949707,
      "activations/layer13_attention_weight_min": -33.7960090637207,
      "activations/layer14_attention_weight_max": 50.804683685302734,
      "activations/layer14_attention_weight_min": -33.03768539428711,
      "activations/layer15_attention_weight_max": 39.17650604248047,
      "activations/layer15_attention_weight_min": -29.283727645874023,
      "activations/layer16_attention_weight_max": 36.4214973449707,
      "activations/layer16_attention_weight_min": -31.33414649963379,
      "activations/layer17_attention_weight_max": 54.7713623046875,
      "activations/layer17_attention_weight_min": -46.12961196899414,
      "activations/layer18_attention_weight_max": 54.12259292602539,
      "activations/layer18_attention_weight_min": -39.1559944152832,
      "activations/layer19_attention_weight_max": 24.3072509765625,
      "activations/layer19_attention_weight_min": -22.932783126831055,
      "activations/layer1_attention_weight_max": 16.611373901367188,
      "activations/layer1_attention_weight_min": -16.71585464477539,
      "activations/layer20_attention_weight_max": 29.0855712890625,
      "activations/layer20_attention_weight_min": -22.935518264770508,
      "activations/layer21_attention_weight_max": 60.89728546142578,
      "activations/layer21_attention_weight_min": -28.715932846069336,
      "activations/layer22_attention_weight_max": 35.03485870361328,
      "activations/layer22_attention_weight_min": -26.08165168762207,
      "activations/layer23_attention_weight_max": 39.719696044921875,
      "activations/layer23_attention_weight_min": -25.680774688720703,
      "activations/layer2_attention_weight_max": 33.117820739746094,
      "activations/layer2_attention_weight_min": -33.22346496582031,
      "activations/layer3_attention_weight_max": 94.78661346435547,
      "activations/layer3_attention_weight_min": -96.28707122802734,
      "activations/layer4_attention_weight_max": 101.62640380859375,
      "activations/layer4_attention_weight_min": -100.8434066772461,
      "activations/layer5_attention_weight_max": 75.72003173828125,
      "activations/layer5_attention_weight_min": -82.5025634765625,
      "activations/layer6_attention_weight_max": 54.70979690551758,
      "activations/layer6_attention_weight_min": -56.06397247314453,
      "activations/layer7_attention_weight_max": 78.98983764648438,
      "activations/layer7_attention_weight_min": -79.83979034423828,
      "activations/layer8_attention_weight_max": 42.15659713745117,
      "activations/layer8_attention_weight_min": -46.91814041137695,
      "activations/layer9_attention_weight_max": 56.368988037109375,
      "activations/layer9_attention_weight_min": -47.959293365478516,
      "epoch": 10.15,
      "learning_rate": 8.539810606060604e-05,
      "loss": 2.7764,
      "step": 174700
    },
    {
      "activations/layer0_attention_weight_max": 14.653440475463867,
      "activations/layer0_attention_weight_min": -14.560826301574707,
      "activations/layer10_attention_weight_max": 39.122859954833984,
      "activations/layer10_attention_weight_min": -36.67961120605469,
      "activations/layer11_attention_weight_max": 32.04201126098633,
      "activations/layer11_attention_weight_min": -33.33387756347656,
      "activations/layer12_attention_weight_max": 18.116323471069336,
      "activations/layer12_attention_weight_min": -26.739145278930664,
      "activations/layer13_attention_weight_max": 39.50990295410156,
      "activations/layer13_attention_weight_min": -31.22890281677246,
      "activations/layer14_attention_weight_max": 36.768218994140625,
      "activations/layer14_attention_weight_min": -29.67828941345215,
      "activations/layer15_attention_weight_max": 32.70026779174805,
      "activations/layer15_attention_weight_min": -27.318065643310547,
      "activations/layer16_attention_weight_max": 30.32427215576172,
      "activations/layer16_attention_weight_min": -28.78654670715332,
      "activations/layer17_attention_weight_max": 54.17915344238281,
      "activations/layer17_attention_weight_min": -44.45270538330078,
      "activations/layer18_attention_weight_max": 49.56024932861328,
      "activations/layer18_attention_weight_min": -35.1529541015625,
      "activations/layer19_attention_weight_max": 22.561431884765625,
      "activations/layer19_attention_weight_min": -22.497175216674805,
      "activations/layer1_attention_weight_max": 16.20610237121582,
      "activations/layer1_attention_weight_min": -15.310708045959473,
      "activations/layer20_attention_weight_max": 21.939804077148438,
      "activations/layer20_attention_weight_min": -22.656578063964844,
      "activations/layer21_attention_weight_max": 34.710121154785156,
      "activations/layer21_attention_weight_min": -22.66527557373047,
      "activations/layer22_attention_weight_max": 32.00177764892578,
      "activations/layer22_attention_weight_min": -25.675508499145508,
      "activations/layer23_attention_weight_max": 45.776458740234375,
      "activations/layer23_attention_weight_min": -29.05093765258789,
      "activations/layer2_attention_weight_max": 34.521087646484375,
      "activations/layer2_attention_weight_min": -31.21167755126953,
      "activations/layer3_attention_weight_max": 92.7337875366211,
      "activations/layer3_attention_weight_min": -92.38027954101562,
      "activations/layer4_attention_weight_max": 100.3507308959961,
      "activations/layer4_attention_weight_min": -96.8384780883789,
      "activations/layer5_attention_weight_max": 74.80215454101562,
      "activations/layer5_attention_weight_min": -78.88190460205078,
      "activations/layer6_attention_weight_max": 53.15278244018555,
      "activations/layer6_attention_weight_min": -56.648441314697266,
      "activations/layer7_attention_weight_max": 75.70677947998047,
      "activations/layer7_attention_weight_min": -71.9231185913086,
      "activations/layer8_attention_weight_max": 45.6180419921875,
      "activations/layer8_attention_weight_min": -48.214603424072266,
      "activations/layer9_attention_weight_max": 49.99477005004883,
      "activations/layer9_attention_weight_min": -50.015926361083984,
      "epoch": 10.15,
      "learning_rate": 8.537916666666666e-05,
      "loss": 2.7821,
      "step": 174750
    },
    {
      "activations/layer0_attention_weight_max": 15.069228172302246,
      "activations/layer0_attention_weight_min": -14.688329696655273,
      "activations/layer10_attention_weight_max": 37.44773483276367,
      "activations/layer10_attention_weight_min": -36.30680847167969,
      "activations/layer11_attention_weight_max": 30.858844757080078,
      "activations/layer11_attention_weight_min": -31.637317657470703,
      "activations/layer12_attention_weight_max": 22.691078186035156,
      "activations/layer12_attention_weight_min": -26.083560943603516,
      "activations/layer13_attention_weight_max": 44.47794723510742,
      "activations/layer13_attention_weight_min": -35.10125732421875,
      "activations/layer14_attention_weight_max": 48.0186767578125,
      "activations/layer14_attention_weight_min": -31.750883102416992,
      "activations/layer15_attention_weight_max": 47.464900970458984,
      "activations/layer15_attention_weight_min": -31.374919891357422,
      "activations/layer16_attention_weight_max": 37.26809310913086,
      "activations/layer16_attention_weight_min": -31.4189453125,
      "activations/layer17_attention_weight_max": 57.70439529418945,
      "activations/layer17_attention_weight_min": -49.478843688964844,
      "activations/layer18_attention_weight_max": 52.06103515625,
      "activations/layer18_attention_weight_min": -43.1493034362793,
      "activations/layer19_attention_weight_max": 24.247892379760742,
      "activations/layer19_attention_weight_min": -25.570383071899414,
      "activations/layer1_attention_weight_max": 16.878694534301758,
      "activations/layer1_attention_weight_min": -14.242637634277344,
      "activations/layer20_attention_weight_max": 24.652585983276367,
      "activations/layer20_attention_weight_min": -24.587926864624023,
      "activations/layer21_attention_weight_max": 42.817832946777344,
      "activations/layer21_attention_weight_min": -26.716026306152344,
      "activations/layer22_attention_weight_max": 35.03561782836914,
      "activations/layer22_attention_weight_min": -27.897634506225586,
      "activations/layer23_attention_weight_max": 39.14836883544922,
      "activations/layer23_attention_weight_min": -23.181610107421875,
      "activations/layer2_attention_weight_max": 31.64980125427246,
      "activations/layer2_attention_weight_min": -29.73676109313965,
      "activations/layer3_attention_weight_max": 87.7647476196289,
      "activations/layer3_attention_weight_min": -91.47496795654297,
      "activations/layer4_attention_weight_max": 96.58444213867188,
      "activations/layer4_attention_weight_min": -99.74884033203125,
      "activations/layer5_attention_weight_max": 71.59447479248047,
      "activations/layer5_attention_weight_min": -74.06938171386719,
      "activations/layer6_attention_weight_max": 52.99494171142578,
      "activations/layer6_attention_weight_min": -52.09458923339844,
      "activations/layer7_attention_weight_max": 70.50836944580078,
      "activations/layer7_attention_weight_min": -71.1742172241211,
      "activations/layer8_attention_weight_max": 45.62730407714844,
      "activations/layer8_attention_weight_min": -48.94114685058594,
      "activations/layer9_attention_weight_max": 50.726924896240234,
      "activations/layer9_attention_weight_min": -49.68745803833008,
      "epoch": 10.16,
      "learning_rate": 8.536022727272727e-05,
      "loss": 2.7832,
      "step": 174800
    },
    {
      "activations/layer0_attention_weight_max": 15.75277328491211,
      "activations/layer0_attention_weight_min": -14.42607593536377,
      "activations/layer10_attention_weight_max": 38.35062026977539,
      "activations/layer10_attention_weight_min": -38.91213607788086,
      "activations/layer11_attention_weight_max": 31.781787872314453,
      "activations/layer11_attention_weight_min": -34.34821319580078,
      "activations/layer12_attention_weight_max": 20.96370506286621,
      "activations/layer12_attention_weight_min": -26.739587783813477,
      "activations/layer13_attention_weight_max": 41.18115997314453,
      "activations/layer13_attention_weight_min": -30.059288024902344,
      "activations/layer14_attention_weight_max": 36.6171989440918,
      "activations/layer14_attention_weight_min": -31.643861770629883,
      "activations/layer15_attention_weight_max": 33.67276382446289,
      "activations/layer15_attention_weight_min": -31.027217864990234,
      "activations/layer16_attention_weight_max": 32.62603759765625,
      "activations/layer16_attention_weight_min": -28.762405395507812,
      "activations/layer17_attention_weight_max": 56.06645584106445,
      "activations/layer17_attention_weight_min": -47.821834564208984,
      "activations/layer18_attention_weight_max": 51.13463592529297,
      "activations/layer18_attention_weight_min": -41.413002014160156,
      "activations/layer19_attention_weight_max": 21.18508529663086,
      "activations/layer19_attention_weight_min": -23.170080184936523,
      "activations/layer1_attention_weight_max": 17.19428062438965,
      "activations/layer1_attention_weight_min": -14.321330070495605,
      "activations/layer20_attention_weight_max": 21.589818954467773,
      "activations/layer20_attention_weight_min": -21.174062728881836,
      "activations/layer21_attention_weight_max": 40.83308792114258,
      "activations/layer21_attention_weight_min": -21.807857513427734,
      "activations/layer22_attention_weight_max": 34.79069900512695,
      "activations/layer22_attention_weight_min": -24.749393463134766,
      "activations/layer23_attention_weight_max": 46.53928756713867,
      "activations/layer23_attention_weight_min": -24.05875015258789,
      "activations/layer2_attention_weight_max": 31.597068786621094,
      "activations/layer2_attention_weight_min": -33.17333221435547,
      "activations/layer3_attention_weight_max": 92.29467010498047,
      "activations/layer3_attention_weight_min": -100.37032318115234,
      "activations/layer4_attention_weight_max": 97.90641784667969,
      "activations/layer4_attention_weight_min": -98.29075622558594,
      "activations/layer5_attention_weight_max": 75.1152572631836,
      "activations/layer5_attention_weight_min": -76.9856948852539,
      "activations/layer6_attention_weight_max": 55.00186538696289,
      "activations/layer6_attention_weight_min": -54.02756881713867,
      "activations/layer7_attention_weight_max": 76.74339294433594,
      "activations/layer7_attention_weight_min": -75.59066772460938,
      "activations/layer8_attention_weight_max": 44.62763977050781,
      "activations/layer8_attention_weight_min": -49.31770706176758,
      "activations/layer9_attention_weight_max": 52.234344482421875,
      "activations/layer9_attention_weight_min": -53.37421417236328,
      "epoch": 10.16,
      "learning_rate": 8.534166666666666e-05,
      "loss": 2.7876,
      "step": 174850
    },
    {
      "activations/layer0_attention_weight_max": 15.19668960571289,
      "activations/layer0_attention_weight_min": -14.76033878326416,
      "activations/layer10_attention_weight_max": 36.89064407348633,
      "activations/layer10_attention_weight_min": -40.442352294921875,
      "activations/layer11_attention_weight_max": 33.09931945800781,
      "activations/layer11_attention_weight_min": -33.61023712158203,
      "activations/layer12_attention_weight_max": 19.336061477661133,
      "activations/layer12_attention_weight_min": -24.84687614440918,
      "activations/layer13_attention_weight_max": 37.96711730957031,
      "activations/layer13_attention_weight_min": -28.924766540527344,
      "activations/layer14_attention_weight_max": 35.362632751464844,
      "activations/layer14_attention_weight_min": -32.789207458496094,
      "activations/layer15_attention_weight_max": 32.50593566894531,
      "activations/layer15_attention_weight_min": -30.04793930053711,
      "activations/layer16_attention_weight_max": 31.702579498291016,
      "activations/layer16_attention_weight_min": -29.65892219543457,
      "activations/layer17_attention_weight_max": 49.37273025512695,
      "activations/layer17_attention_weight_min": -46.41559600830078,
      "activations/layer18_attention_weight_max": 46.524635314941406,
      "activations/layer18_attention_weight_min": -37.67371368408203,
      "activations/layer19_attention_weight_max": 23.13713264465332,
      "activations/layer19_attention_weight_min": -20.850296020507812,
      "activations/layer1_attention_weight_max": 17.144790649414062,
      "activations/layer1_attention_weight_min": -14.738433837890625,
      "activations/layer20_attention_weight_max": 23.785327911376953,
      "activations/layer20_attention_weight_min": -23.681194305419922,
      "activations/layer21_attention_weight_max": 35.55354690551758,
      "activations/layer21_attention_weight_min": -24.994897842407227,
      "activations/layer22_attention_weight_max": 35.352046966552734,
      "activations/layer22_attention_weight_min": -25.80754280090332,
      "activations/layer23_attention_weight_max": 37.28053283691406,
      "activations/layer23_attention_weight_min": -24.931516647338867,
      "activations/layer2_attention_weight_max": 34.213226318359375,
      "activations/layer2_attention_weight_min": -31.268733978271484,
      "activations/layer3_attention_weight_max": 94.36276245117188,
      "activations/layer3_attention_weight_min": -94.8265609741211,
      "activations/layer4_attention_weight_max": 109.61602020263672,
      "activations/layer4_attention_weight_min": -102.48995208740234,
      "activations/layer5_attention_weight_max": 86.87432861328125,
      "activations/layer5_attention_weight_min": -82.44623565673828,
      "activations/layer6_attention_weight_max": 58.81321334838867,
      "activations/layer6_attention_weight_min": -56.27295684814453,
      "activations/layer7_attention_weight_max": 79.41202545166016,
      "activations/layer7_attention_weight_min": -76.84310150146484,
      "activations/layer8_attention_weight_max": 50.3530158996582,
      "activations/layer8_attention_weight_min": -49.65388488769531,
      "activations/layer9_attention_weight_max": 50.90227508544922,
      "activations/layer9_attention_weight_min": -50.11146545410156,
      "epoch": 10.16,
      "learning_rate": 8.532272727272726e-05,
      "loss": 2.7902,
      "step": 174900
    },
    {
      "activations/layer0_attention_weight_max": 13.991283416748047,
      "activations/layer0_attention_weight_min": -14.608824729919434,
      "activations/layer10_attention_weight_max": 37.4865608215332,
      "activations/layer10_attention_weight_min": -36.002437591552734,
      "activations/layer11_attention_weight_max": 33.94708251953125,
      "activations/layer11_attention_weight_min": -31.656829833984375,
      "activations/layer12_attention_weight_max": 17.976850509643555,
      "activations/layer12_attention_weight_min": -26.719955444335938,
      "activations/layer13_attention_weight_max": 39.585323333740234,
      "activations/layer13_attention_weight_min": -28.859010696411133,
      "activations/layer14_attention_weight_max": 39.42127227783203,
      "activations/layer14_attention_weight_min": -28.520116806030273,
      "activations/layer15_attention_weight_max": 38.446922302246094,
      "activations/layer15_attention_weight_min": -27.696481704711914,
      "activations/layer16_attention_weight_max": 32.25094985961914,
      "activations/layer16_attention_weight_min": -30.371341705322266,
      "activations/layer17_attention_weight_max": 52.25511932373047,
      "activations/layer17_attention_weight_min": -44.699317932128906,
      "activations/layer18_attention_weight_max": 52.48904037475586,
      "activations/layer18_attention_weight_min": -36.714237213134766,
      "activations/layer19_attention_weight_max": 22.31418800354004,
      "activations/layer19_attention_weight_min": -20.268220901489258,
      "activations/layer1_attention_weight_max": 16.531105041503906,
      "activations/layer1_attention_weight_min": -14.786896705627441,
      "activations/layer20_attention_weight_max": 24.273784637451172,
      "activations/layer20_attention_weight_min": -21.798011779785156,
      "activations/layer21_attention_weight_max": 37.436073303222656,
      "activations/layer21_attention_weight_min": -20.63311195373535,
      "activations/layer22_attention_weight_max": 31.652904510498047,
      "activations/layer22_attention_weight_min": -25.006961822509766,
      "activations/layer23_attention_weight_max": 36.69390869140625,
      "activations/layer23_attention_weight_min": -23.36513900756836,
      "activations/layer2_attention_weight_max": 31.477855682373047,
      "activations/layer2_attention_weight_min": -30.055828094482422,
      "activations/layer3_attention_weight_max": 91.29887390136719,
      "activations/layer3_attention_weight_min": -94.08335876464844,
      "activations/layer4_attention_weight_max": 100.66170501708984,
      "activations/layer4_attention_weight_min": -95.34272766113281,
      "activations/layer5_attention_weight_max": 76.08419036865234,
      "activations/layer5_attention_weight_min": -81.39366149902344,
      "activations/layer6_attention_weight_max": 55.12177276611328,
      "activations/layer6_attention_weight_min": -54.12045669555664,
      "activations/layer7_attention_weight_max": 76.98120880126953,
      "activations/layer7_attention_weight_min": -76.49051666259766,
      "activations/layer8_attention_weight_max": 45.09452438354492,
      "activations/layer8_attention_weight_min": -46.94540023803711,
      "activations/layer9_attention_weight_max": 53.02888870239258,
      "activations/layer9_attention_weight_min": -51.25055694580078,
      "epoch": 10.17,
      "learning_rate": 8.530378787878787e-05,
      "loss": 2.7935,
      "step": 174950
    },
    {
      "activations/layer0_attention_weight_max": 15.5260009765625,
      "activations/layer0_attention_weight_min": -14.828740119934082,
      "activations/layer10_attention_weight_max": 44.33721923828125,
      "activations/layer10_attention_weight_min": -41.86229705810547,
      "activations/layer11_attention_weight_max": 37.57685852050781,
      "activations/layer11_attention_weight_min": -34.34888458251953,
      "activations/layer12_attention_weight_max": 19.022754669189453,
      "activations/layer12_attention_weight_min": -25.340049743652344,
      "activations/layer13_attention_weight_max": 34.40750503540039,
      "activations/layer13_attention_weight_min": -27.535070419311523,
      "activations/layer14_attention_weight_max": 34.478675842285156,
      "activations/layer14_attention_weight_min": -29.689640045166016,
      "activations/layer15_attention_weight_max": 29.342561721801758,
      "activations/layer15_attention_weight_min": -29.648494720458984,
      "activations/layer16_attention_weight_max": 28.892223358154297,
      "activations/layer16_attention_weight_min": -28.571428298950195,
      "activations/layer17_attention_weight_max": 55.065185546875,
      "activations/layer17_attention_weight_min": -50.44978713989258,
      "activations/layer18_attention_weight_max": 42.61882019042969,
      "activations/layer18_attention_weight_min": -39.58380126953125,
      "activations/layer19_attention_weight_max": 19.478261947631836,
      "activations/layer19_attention_weight_min": -19.849903106689453,
      "activations/layer1_attention_weight_max": 16.986238479614258,
      "activations/layer1_attention_weight_min": -13.971900939941406,
      "activations/layer20_attention_weight_max": 20.83570671081543,
      "activations/layer20_attention_weight_min": -20.79961395263672,
      "activations/layer21_attention_weight_max": 33.746055603027344,
      "activations/layer21_attention_weight_min": -22.322816848754883,
      "activations/layer22_attention_weight_max": 27.261072158813477,
      "activations/layer22_attention_weight_min": -23.620128631591797,
      "activations/layer23_attention_weight_max": 33.43394470214844,
      "activations/layer23_attention_weight_min": -24.048208236694336,
      "activations/layer2_attention_weight_max": 32.931976318359375,
      "activations/layer2_attention_weight_min": -31.554189682006836,
      "activations/layer3_attention_weight_max": 93.24285125732422,
      "activations/layer3_attention_weight_min": -93.49552154541016,
      "activations/layer4_attention_weight_max": 108.72635650634766,
      "activations/layer4_attention_weight_min": -98.0947265625,
      "activations/layer5_attention_weight_max": 82.59227752685547,
      "activations/layer5_attention_weight_min": -81.19102478027344,
      "activations/layer6_attention_weight_max": 57.01096725463867,
      "activations/layer6_attention_weight_min": -53.224178314208984,
      "activations/layer7_attention_weight_max": 86.72233581542969,
      "activations/layer7_attention_weight_min": -81.03972625732422,
      "activations/layer8_attention_weight_max": 52.162113189697266,
      "activations/layer8_attention_weight_min": -50.449974060058594,
      "activations/layer9_attention_weight_max": 64.08995819091797,
      "activations/layer9_attention_weight_min": -55.7808723449707,
      "epoch": 10.17,
      "learning_rate": 8.528484848484848e-05,
      "loss": 2.7937,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4964,
      "eval_samples_per_second": 505.388,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4964,
      "eval_openwebtext_samples_per_second": 505.388,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.9617,
      "eval_wikitext_samples_per_second": 232.456,
      "step": 175000
    },
    {
      "epoch": 10.17,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.5472,
      "eval_lambada_samples_per_second": 509.99,
      "step": 175000
    },
    {
      "activations/layer0_attention_weight_max": 15.097421646118164,
      "activations/layer0_attention_weight_min": -14.483402252197266,
      "activations/layer10_attention_weight_max": 41.774356842041016,
      "activations/layer10_attention_weight_min": -38.496028900146484,
      "activations/layer11_attention_weight_max": 35.8829460144043,
      "activations/layer11_attention_weight_min": -36.18472671508789,
      "activations/layer12_attention_weight_max": 19.148345947265625,
      "activations/layer12_attention_weight_min": -26.374494552612305,
      "activations/layer13_attention_weight_max": 43.17493438720703,
      "activations/layer13_attention_weight_min": -28.701313018798828,
      "activations/layer14_attention_weight_max": 42.0459098815918,
      "activations/layer14_attention_weight_min": -32.62282180786133,
      "activations/layer15_attention_weight_max": 39.15726089477539,
      "activations/layer15_attention_weight_min": -29.970123291015625,
      "activations/layer16_attention_weight_max": 33.34239959716797,
      "activations/layer16_attention_weight_min": -30.522838592529297,
      "activations/layer17_attention_weight_max": 57.77265930175781,
      "activations/layer17_attention_weight_min": -44.2140998840332,
      "activations/layer18_attention_weight_max": 48.55889129638672,
      "activations/layer18_attention_weight_min": -37.03961181640625,
      "activations/layer19_attention_weight_max": 21.878490447998047,
      "activations/layer19_attention_weight_min": -20.778261184692383,
      "activations/layer1_attention_weight_max": 16.389867782592773,
      "activations/layer1_attention_weight_min": -15.70440673828125,
      "activations/layer20_attention_weight_max": 22.143461227416992,
      "activations/layer20_attention_weight_min": -24.131502151489258,
      "activations/layer21_attention_weight_max": 35.693016052246094,
      "activations/layer21_attention_weight_min": -27.07059669494629,
      "activations/layer22_attention_weight_max": 30.34499740600586,
      "activations/layer22_attention_weight_min": -26.32958221435547,
      "activations/layer23_attention_weight_max": 35.9047737121582,
      "activations/layer23_attention_weight_min": -26.758525848388672,
      "activations/layer2_attention_weight_max": 31.973892211914062,
      "activations/layer2_attention_weight_min": -31.55382537841797,
      "activations/layer3_attention_weight_max": 88.93033599853516,
      "activations/layer3_attention_weight_min": -99.9853286743164,
      "activations/layer4_attention_weight_max": 100.1172103881836,
      "activations/layer4_attention_weight_min": -95.43623352050781,
      "activations/layer5_attention_weight_max": 75.42295837402344,
      "activations/layer5_attention_weight_min": -79.79190826416016,
      "activations/layer6_attention_weight_max": 55.695777893066406,
      "activations/layer6_attention_weight_min": -53.42589569091797,
      "activations/layer7_attention_weight_max": 78.55794525146484,
      "activations/layer7_attention_weight_min": -74.56560516357422,
      "activations/layer8_attention_weight_max": 51.403507232666016,
      "activations/layer8_attention_weight_min": -51.23586654663086,
      "activations/layer9_attention_weight_max": 54.110206604003906,
      "activations/layer9_attention_weight_min": -52.2552604675293,
      "epoch": 10.17,
      "learning_rate": 8.526590909090908e-05,
      "loss": 2.7827,
      "step": 175050
    },
    {
      "activations/layer0_attention_weight_max": 15.279827117919922,
      "activations/layer0_attention_weight_min": -14.780030250549316,
      "activations/layer10_attention_weight_max": 34.8455810546875,
      "activations/layer10_attention_weight_min": -35.940555572509766,
      "activations/layer11_attention_weight_max": 31.112287521362305,
      "activations/layer11_attention_weight_min": -31.85609245300293,
      "activations/layer12_attention_weight_max": 20.383193969726562,
      "activations/layer12_attention_weight_min": -23.44828987121582,
      "activations/layer13_attention_weight_max": 34.726837158203125,
      "activations/layer13_attention_weight_min": -28.792200088500977,
      "activations/layer14_attention_weight_max": 34.48558044433594,
      "activations/layer14_attention_weight_min": -32.94990158081055,
      "activations/layer15_attention_weight_max": 32.83761215209961,
      "activations/layer15_attention_weight_min": -31.795202255249023,
      "activations/layer16_attention_weight_max": 31.298351287841797,
      "activations/layer16_attention_weight_min": -31.1749267578125,
      "activations/layer17_attention_weight_max": 55.299747467041016,
      "activations/layer17_attention_weight_min": -45.97583770751953,
      "activations/layer18_attention_weight_max": 48.2768669128418,
      "activations/layer18_attention_weight_min": -38.797645568847656,
      "activations/layer19_attention_weight_max": 22.383865356445312,
      "activations/layer19_attention_weight_min": -21.5146541595459,
      "activations/layer1_attention_weight_max": 16.183225631713867,
      "activations/layer1_attention_weight_min": -14.98535442352295,
      "activations/layer20_attention_weight_max": 22.506973266601562,
      "activations/layer20_attention_weight_min": -20.987194061279297,
      "activations/layer21_attention_weight_max": 42.894775390625,
      "activations/layer21_attention_weight_min": -26.345767974853516,
      "activations/layer22_attention_weight_max": 26.56939697265625,
      "activations/layer22_attention_weight_min": -23.96864891052246,
      "activations/layer23_attention_weight_max": 36.968116760253906,
      "activations/layer23_attention_weight_min": -22.553329467773438,
      "activations/layer2_attention_weight_max": 32.60803985595703,
      "activations/layer2_attention_weight_min": -31.434486389160156,
      "activations/layer3_attention_weight_max": 92.94444274902344,
      "activations/layer3_attention_weight_min": -94.5667495727539,
      "activations/layer4_attention_weight_max": 105.05850982666016,
      "activations/layer4_attention_weight_min": -97.51824188232422,
      "activations/layer5_attention_weight_max": 75.3851089477539,
      "activations/layer5_attention_weight_min": -79.93905639648438,
      "activations/layer6_attention_weight_max": 53.734275817871094,
      "activations/layer6_attention_weight_min": -54.153072357177734,
      "activations/layer7_attention_weight_max": 75.64662170410156,
      "activations/layer7_attention_weight_min": -74.80435180664062,
      "activations/layer8_attention_weight_max": 44.577049255371094,
      "activations/layer8_attention_weight_min": -48.98113250732422,
      "activations/layer9_attention_weight_max": 49.120521545410156,
      "activations/layer9_attention_weight_min": -51.80668258666992,
      "epoch": 10.17,
      "learning_rate": 8.52469696969697e-05,
      "loss": 2.7774,
      "step": 175100
    },
    {
      "activations/layer0_attention_weight_max": 14.818513870239258,
      "activations/layer0_attention_weight_min": -14.654037475585938,
      "activations/layer10_attention_weight_max": 39.556461334228516,
      "activations/layer10_attention_weight_min": -36.675819396972656,
      "activations/layer11_attention_weight_max": 32.76770782470703,
      "activations/layer11_attention_weight_min": -32.17987060546875,
      "activations/layer12_attention_weight_max": 18.612363815307617,
      "activations/layer12_attention_weight_min": -22.496219635009766,
      "activations/layer13_attention_weight_max": 42.33744812011719,
      "activations/layer13_attention_weight_min": -32.24800109863281,
      "activations/layer14_attention_weight_max": 36.4200553894043,
      "activations/layer14_attention_weight_min": -35.55060958862305,
      "activations/layer15_attention_weight_max": 32.6674690246582,
      "activations/layer15_attention_weight_min": -31.0518856048584,
      "activations/layer16_attention_weight_max": 32.317806243896484,
      "activations/layer16_attention_weight_min": -30.410289764404297,
      "activations/layer17_attention_weight_max": 52.69151306152344,
      "activations/layer17_attention_weight_min": -42.9339599609375,
      "activations/layer18_attention_weight_max": 47.37260818481445,
      "activations/layer18_attention_weight_min": -35.799720764160156,
      "activations/layer19_attention_weight_max": 24.17911148071289,
      "activations/layer19_attention_weight_min": -21.904844284057617,
      "activations/layer1_attention_weight_max": 15.984528541564941,
      "activations/layer1_attention_weight_min": -15.337000846862793,
      "activations/layer20_attention_weight_max": 21.768518447875977,
      "activations/layer20_attention_weight_min": -21.92514991760254,
      "activations/layer21_attention_weight_max": 39.22626876831055,
      "activations/layer21_attention_weight_min": -22.746496200561523,
      "activations/layer22_attention_weight_max": 31.180898666381836,
      "activations/layer22_attention_weight_min": -26.125064849853516,
      "activations/layer23_attention_weight_max": 38.35053253173828,
      "activations/layer23_attention_weight_min": -25.978429794311523,
      "activations/layer2_attention_weight_max": 30.646087646484375,
      "activations/layer2_attention_weight_min": -29.1805419921875,
      "activations/layer3_attention_weight_max": 87.3681411743164,
      "activations/layer3_attention_weight_min": -89.13021087646484,
      "activations/layer4_attention_weight_max": 98.28376770019531,
      "activations/layer4_attention_weight_min": -94.82865142822266,
      "activations/layer5_attention_weight_max": 77.31289672851562,
      "activations/layer5_attention_weight_min": -78.7867660522461,
      "activations/layer6_attention_weight_max": 55.30778121948242,
      "activations/layer6_attention_weight_min": -54.77814483642578,
      "activations/layer7_attention_weight_max": 77.16285705566406,
      "activations/layer7_attention_weight_min": -74.35147857666016,
      "activations/layer8_attention_weight_max": 46.547019958496094,
      "activations/layer8_attention_weight_min": -50.969017028808594,
      "activations/layer9_attention_weight_max": 54.27443313598633,
      "activations/layer9_attention_weight_min": -52.87742233276367,
      "epoch": 10.18,
      "learning_rate": 8.52280303030303e-05,
      "loss": 2.7813,
      "step": 175150
    },
    {
      "activations/layer0_attention_weight_max": 15.940695762634277,
      "activations/layer0_attention_weight_min": -14.679388046264648,
      "activations/layer10_attention_weight_max": 42.65331268310547,
      "activations/layer10_attention_weight_min": -39.15057373046875,
      "activations/layer11_attention_weight_max": 35.59629821777344,
      "activations/layer11_attention_weight_min": -34.10111618041992,
      "activations/layer12_attention_weight_max": 21.605140686035156,
      "activations/layer12_attention_weight_min": -24.512386322021484,
      "activations/layer13_attention_weight_max": 44.74217224121094,
      "activations/layer13_attention_weight_min": -32.62489318847656,
      "activations/layer14_attention_weight_max": 46.9962158203125,
      "activations/layer14_attention_weight_min": -30.935026168823242,
      "activations/layer15_attention_weight_max": 39.36567687988281,
      "activations/layer15_attention_weight_min": -31.623764038085938,
      "activations/layer16_attention_weight_max": 35.34119415283203,
      "activations/layer16_attention_weight_min": -30.267444610595703,
      "activations/layer17_attention_weight_max": 60.65472412109375,
      "activations/layer17_attention_weight_min": -46.24156951904297,
      "activations/layer18_attention_weight_max": 55.07284927368164,
      "activations/layer18_attention_weight_min": -40.163116455078125,
      "activations/layer19_attention_weight_max": 28.308195114135742,
      "activations/layer19_attention_weight_min": -21.535694122314453,
      "activations/layer1_attention_weight_max": 16.1282901763916,
      "activations/layer1_attention_weight_min": -15.018632888793945,
      "activations/layer20_attention_weight_max": 25.96935272216797,
      "activations/layer20_attention_weight_min": -24.10761260986328,
      "activations/layer21_attention_weight_max": 44.55398178100586,
      "activations/layer21_attention_weight_min": -22.6644287109375,
      "activations/layer22_attention_weight_max": 33.68696594238281,
      "activations/layer22_attention_weight_min": -27.011343002319336,
      "activations/layer23_attention_weight_max": 41.50780487060547,
      "activations/layer23_attention_weight_min": -22.80188751220703,
      "activations/layer2_attention_weight_max": 30.497962951660156,
      "activations/layer2_attention_weight_min": -29.552776336669922,
      "activations/layer3_attention_weight_max": 94.46790313720703,
      "activations/layer3_attention_weight_min": -85.34412384033203,
      "activations/layer4_attention_weight_max": 103.94322967529297,
      "activations/layer4_attention_weight_min": -96.037109375,
      "activations/layer5_attention_weight_max": 81.12728881835938,
      "activations/layer5_attention_weight_min": -87.02508544921875,
      "activations/layer6_attention_weight_max": 58.620323181152344,
      "activations/layer6_attention_weight_min": -56.19565963745117,
      "activations/layer7_attention_weight_max": 85.9610824584961,
      "activations/layer7_attention_weight_min": -75.49061584472656,
      "activations/layer8_attention_weight_max": 48.27098083496094,
      "activations/layer8_attention_weight_min": -48.034183502197266,
      "activations/layer9_attention_weight_max": 55.90686798095703,
      "activations/layer9_attention_weight_min": -49.26957702636719,
      "epoch": 10.18,
      "learning_rate": 8.52090909090909e-05,
      "loss": 2.7838,
      "step": 175200
    },
    {
      "activations/layer0_attention_weight_max": 15.011116027832031,
      "activations/layer0_attention_weight_min": -14.801785469055176,
      "activations/layer10_attention_weight_max": 37.05477523803711,
      "activations/layer10_attention_weight_min": -37.53793716430664,
      "activations/layer11_attention_weight_max": 32.988182067871094,
      "activations/layer11_attention_weight_min": -32.8878288269043,
      "activations/layer12_attention_weight_max": 25.049942016601562,
      "activations/layer12_attention_weight_min": -28.072616577148438,
      "activations/layer13_attention_weight_max": 40.09403610229492,
      "activations/layer13_attention_weight_min": -31.751821517944336,
      "activations/layer14_attention_weight_max": 36.75448989868164,
      "activations/layer14_attention_weight_min": -29.026920318603516,
      "activations/layer15_attention_weight_max": 36.426082611083984,
      "activations/layer15_attention_weight_min": -28.144874572753906,
      "activations/layer16_attention_weight_max": 32.9953727722168,
      "activations/layer16_attention_weight_min": -28.705366134643555,
      "activations/layer17_attention_weight_max": 51.46170425415039,
      "activations/layer17_attention_weight_min": -42.51518249511719,
      "activations/layer18_attention_weight_max": 47.547523498535156,
      "activations/layer18_attention_weight_min": -36.84532928466797,
      "activations/layer19_attention_weight_max": 26.987985610961914,
      "activations/layer19_attention_weight_min": -25.194299697875977,
      "activations/layer1_attention_weight_max": 16.700546264648438,
      "activations/layer1_attention_weight_min": -13.811030387878418,
      "activations/layer20_attention_weight_max": 21.904481887817383,
      "activations/layer20_attention_weight_min": -21.606325149536133,
      "activations/layer21_attention_weight_max": 34.26627731323242,
      "activations/layer21_attention_weight_min": -24.87921714782715,
      "activations/layer22_attention_weight_max": 29.787494659423828,
      "activations/layer22_attention_weight_min": -25.299524307250977,
      "activations/layer23_attention_weight_max": 36.68682861328125,
      "activations/layer23_attention_weight_min": -21.714717864990234,
      "activations/layer2_attention_weight_max": 32.431339263916016,
      "activations/layer2_attention_weight_min": -30.966495513916016,
      "activations/layer3_attention_weight_max": 87.62910461425781,
      "activations/layer3_attention_weight_min": -92.06097412109375,
      "activations/layer4_attention_weight_max": 97.1513900756836,
      "activations/layer4_attention_weight_min": -94.00121307373047,
      "activations/layer5_attention_weight_max": 76.67998504638672,
      "activations/layer5_attention_weight_min": -79.41325378417969,
      "activations/layer6_attention_weight_max": 53.24558639526367,
      "activations/layer6_attention_weight_min": -54.82097625732422,
      "activations/layer7_attention_weight_max": 79.62042999267578,
      "activations/layer7_attention_weight_min": -71.55548095703125,
      "activations/layer8_attention_weight_max": 43.71237564086914,
      "activations/layer8_attention_weight_min": -46.7609748840332,
      "activations/layer9_attention_weight_max": 51.401973724365234,
      "activations/layer9_attention_weight_min": -49.50087356567383,
      "epoch": 10.18,
      "learning_rate": 8.519015151515152e-05,
      "loss": 2.7754,
      "step": 175250
    },
    {
      "activations/layer0_attention_weight_max": 14.866718292236328,
      "activations/layer0_attention_weight_min": -14.547284126281738,
      "activations/layer10_attention_weight_max": 34.77650833129883,
      "activations/layer10_attention_weight_min": -35.73549270629883,
      "activations/layer11_attention_weight_max": 29.892135620117188,
      "activations/layer11_attention_weight_min": -32.1722297668457,
      "activations/layer12_attention_weight_max": 19.63985252380371,
      "activations/layer12_attention_weight_min": -25.81605339050293,
      "activations/layer13_attention_weight_max": 39.165245056152344,
      "activations/layer13_attention_weight_min": -32.44582748413086,
      "activations/layer14_attention_weight_max": 36.79315948486328,
      "activations/layer14_attention_weight_min": -31.992467880249023,
      "activations/layer15_attention_weight_max": 32.59296798706055,
      "activations/layer15_attention_weight_min": -30.315820693969727,
      "activations/layer16_attention_weight_max": 31.84922218322754,
      "activations/layer16_attention_weight_min": -29.43102264404297,
      "activations/layer17_attention_weight_max": 49.72349548339844,
      "activations/layer17_attention_weight_min": -44.10762023925781,
      "activations/layer18_attention_weight_max": 45.883880615234375,
      "activations/layer18_attention_weight_min": -37.04523468017578,
      "activations/layer19_attention_weight_max": 19.730735778808594,
      "activations/layer19_attention_weight_min": -21.897491455078125,
      "activations/layer1_attention_weight_max": 16.478740692138672,
      "activations/layer1_attention_weight_min": -15.27653980255127,
      "activations/layer20_attention_weight_max": 20.38555145263672,
      "activations/layer20_attention_weight_min": -22.22289276123047,
      "activations/layer21_attention_weight_max": 36.646644592285156,
      "activations/layer21_attention_weight_min": -23.96756362915039,
      "activations/layer22_attention_weight_max": 30.14017105102539,
      "activations/layer22_attention_weight_min": -24.596229553222656,
      "activations/layer23_attention_weight_max": 35.329673767089844,
      "activations/layer23_attention_weight_min": -21.925413131713867,
      "activations/layer2_attention_weight_max": 32.1046142578125,
      "activations/layer2_attention_weight_min": -31.444204330444336,
      "activations/layer3_attention_weight_max": 90.60063934326172,
      "activations/layer3_attention_weight_min": -91.5147933959961,
      "activations/layer4_attention_weight_max": 95.9355697631836,
      "activations/layer4_attention_weight_min": -101.65908813476562,
      "activations/layer5_attention_weight_max": 75.34734344482422,
      "activations/layer5_attention_weight_min": -80.31438446044922,
      "activations/layer6_attention_weight_max": 52.14332962036133,
      "activations/layer6_attention_weight_min": -52.867496490478516,
      "activations/layer7_attention_weight_max": 73.19974517822266,
      "activations/layer7_attention_weight_min": -73.25552368164062,
      "activations/layer8_attention_weight_max": 43.11244201660156,
      "activations/layer8_attention_weight_min": -46.25282287597656,
      "activations/layer9_attention_weight_max": 48.256187438964844,
      "activations/layer9_attention_weight_min": -46.932151794433594,
      "epoch": 10.19,
      "learning_rate": 8.51712121212121e-05,
      "loss": 2.7863,
      "step": 175300
    },
    {
      "activations/layer0_attention_weight_max": 15.849394798278809,
      "activations/layer0_attention_weight_min": -14.883115768432617,
      "activations/layer10_attention_weight_max": 40.94609069824219,
      "activations/layer10_attention_weight_min": -35.683815002441406,
      "activations/layer11_attention_weight_max": 31.72315788269043,
      "activations/layer11_attention_weight_min": -32.893531799316406,
      "activations/layer12_attention_weight_max": 19.329505920410156,
      "activations/layer12_attention_weight_min": -25.48346519470215,
      "activations/layer13_attention_weight_max": 46.89677047729492,
      "activations/layer13_attention_weight_min": -33.51824951171875,
      "activations/layer14_attention_weight_max": 39.650413513183594,
      "activations/layer14_attention_weight_min": -31.582294464111328,
      "activations/layer15_attention_weight_max": 35.14994812011719,
      "activations/layer15_attention_weight_min": -31.451820373535156,
      "activations/layer16_attention_weight_max": 34.591609954833984,
      "activations/layer16_attention_weight_min": -30.86391258239746,
      "activations/layer17_attention_weight_max": 55.94294357299805,
      "activations/layer17_attention_weight_min": -48.12931823730469,
      "activations/layer18_attention_weight_max": 52.722740173339844,
      "activations/layer18_attention_weight_min": -39.2412223815918,
      "activations/layer19_attention_weight_max": 26.469053268432617,
      "activations/layer19_attention_weight_min": -23.227617263793945,
      "activations/layer1_attention_weight_max": 17.736392974853516,
      "activations/layer1_attention_weight_min": -14.331075668334961,
      "activations/layer20_attention_weight_max": 25.989286422729492,
      "activations/layer20_attention_weight_min": -20.573305130004883,
      "activations/layer21_attention_weight_max": 38.73875045776367,
      "activations/layer21_attention_weight_min": -24.922622680664062,
      "activations/layer22_attention_weight_max": 36.764408111572266,
      "activations/layer22_attention_weight_min": -25.050189971923828,
      "activations/layer23_attention_weight_max": 44.4342155456543,
      "activations/layer23_attention_weight_min": -24.439998626708984,
      "activations/layer2_attention_weight_max": 32.06258010864258,
      "activations/layer2_attention_weight_min": -32.916385650634766,
      "activations/layer3_attention_weight_max": 95.7376480102539,
      "activations/layer3_attention_weight_min": -99.28693389892578,
      "activations/layer4_attention_weight_max": 101.248779296875,
      "activations/layer4_attention_weight_min": -99.49800109863281,
      "activations/layer5_attention_weight_max": 76.98811340332031,
      "activations/layer5_attention_weight_min": -79.14110565185547,
      "activations/layer6_attention_weight_max": 52.63457489013672,
      "activations/layer6_attention_weight_min": -54.117671966552734,
      "activations/layer7_attention_weight_max": 76.10401916503906,
      "activations/layer7_attention_weight_min": -79.50718688964844,
      "activations/layer8_attention_weight_max": 43.27769088745117,
      "activations/layer8_attention_weight_min": -48.00899124145508,
      "activations/layer9_attention_weight_max": 49.66253662109375,
      "activations/layer9_attention_weight_min": -50.34474182128906,
      "epoch": 10.19,
      "learning_rate": 8.515227272727272e-05,
      "loss": 2.7827,
      "step": 175350
    },
    {
      "activations/layer0_attention_weight_max": 14.982434272766113,
      "activations/layer0_attention_weight_min": -14.496700286865234,
      "activations/layer10_attention_weight_max": 42.02752685546875,
      "activations/layer10_attention_weight_min": -37.57230758666992,
      "activations/layer11_attention_weight_max": 35.93476867675781,
      "activations/layer11_attention_weight_min": -33.64319610595703,
      "activations/layer12_attention_weight_max": 17.51162338256836,
      "activations/layer12_attention_weight_min": -26.99531364440918,
      "activations/layer13_attention_weight_max": 42.11042785644531,
      "activations/layer13_attention_weight_min": -34.053123474121094,
      "activations/layer14_attention_weight_max": 34.510528564453125,
      "activations/layer14_attention_weight_min": -30.071203231811523,
      "activations/layer15_attention_weight_max": 33.316551208496094,
      "activations/layer15_attention_weight_min": -29.405187606811523,
      "activations/layer16_attention_weight_max": 30.269298553466797,
      "activations/layer16_attention_weight_min": -29.629150390625,
      "activations/layer17_attention_weight_max": 52.13059616088867,
      "activations/layer17_attention_weight_min": -43.700557708740234,
      "activations/layer18_attention_weight_max": 48.840213775634766,
      "activations/layer18_attention_weight_min": -37.7938346862793,
      "activations/layer19_attention_weight_max": 19.931745529174805,
      "activations/layer19_attention_weight_min": -21.30726432800293,
      "activations/layer1_attention_weight_max": 16.97218894958496,
      "activations/layer1_attention_weight_min": -14.4784517288208,
      "activations/layer20_attention_weight_max": 21.925188064575195,
      "activations/layer20_attention_weight_min": -22.00225257873535,
      "activations/layer21_attention_weight_max": 37.19697570800781,
      "activations/layer21_attention_weight_min": -23.594446182250977,
      "activations/layer22_attention_weight_max": 33.183860778808594,
      "activations/layer22_attention_weight_min": -26.970844268798828,
      "activations/layer23_attention_weight_max": 37.70777893066406,
      "activations/layer23_attention_weight_min": -24.567848205566406,
      "activations/layer2_attention_weight_max": 33.790443420410156,
      "activations/layer2_attention_weight_min": -30.570478439331055,
      "activations/layer3_attention_weight_max": 88.73550415039062,
      "activations/layer3_attention_weight_min": -84.07522583007812,
      "activations/layer4_attention_weight_max": 93.68278503417969,
      "activations/layer4_attention_weight_min": -86.59451293945312,
      "activations/layer5_attention_weight_max": 75.15214538574219,
      "activations/layer5_attention_weight_min": -74.89503479003906,
      "activations/layer6_attention_weight_max": 56.868431091308594,
      "activations/layer6_attention_weight_min": -54.043575286865234,
      "activations/layer7_attention_weight_max": 77.73661804199219,
      "activations/layer7_attention_weight_min": -75.7280502319336,
      "activations/layer8_attention_weight_max": 49.20970916748047,
      "activations/layer8_attention_weight_min": -49.98206329345703,
      "activations/layer9_attention_weight_max": 55.4561767578125,
      "activations/layer9_attention_weight_min": -47.73141860961914,
      "epoch": 10.19,
      "learning_rate": 8.513333333333333e-05,
      "loss": 2.7897,
      "step": 175400
    },
    {
      "activations/layer0_attention_weight_max": 14.040961265563965,
      "activations/layer0_attention_weight_min": -14.626180648803711,
      "activations/layer10_attention_weight_max": 38.46021270751953,
      "activations/layer10_attention_weight_min": -35.698150634765625,
      "activations/layer11_attention_weight_max": 33.64458465576172,
      "activations/layer11_attention_weight_min": -33.56614303588867,
      "activations/layer12_attention_weight_max": 20.735029220581055,
      "activations/layer12_attention_weight_min": -22.572038650512695,
      "activations/layer13_attention_weight_max": 38.67702865600586,
      "activations/layer13_attention_weight_min": -28.82219886779785,
      "activations/layer14_attention_weight_max": 36.89994812011719,
      "activations/layer14_attention_weight_min": -29.83547019958496,
      "activations/layer15_attention_weight_max": 31.900266647338867,
      "activations/layer15_attention_weight_min": -29.082918167114258,
      "activations/layer16_attention_weight_max": 33.226383209228516,
      "activations/layer16_attention_weight_min": -28.69371223449707,
      "activations/layer17_attention_weight_max": 51.50577926635742,
      "activations/layer17_attention_weight_min": -42.75092697143555,
      "activations/layer18_attention_weight_max": 44.72356414794922,
      "activations/layer18_attention_weight_min": -35.238365173339844,
      "activations/layer19_attention_weight_max": 22.83962059020996,
      "activations/layer19_attention_weight_min": -20.618154525756836,
      "activations/layer1_attention_weight_max": 16.89591407775879,
      "activations/layer1_attention_weight_min": -14.207873344421387,
      "activations/layer20_attention_weight_max": 22.752361297607422,
      "activations/layer20_attention_weight_min": -20.961698532104492,
      "activations/layer21_attention_weight_max": 38.09511184692383,
      "activations/layer21_attention_weight_min": -22.33457374572754,
      "activations/layer22_attention_weight_max": 29.264360427856445,
      "activations/layer22_attention_weight_min": -23.756093978881836,
      "activations/layer23_attention_weight_max": 37.28844451904297,
      "activations/layer23_attention_weight_min": -23.576377868652344,
      "activations/layer2_attention_weight_max": 31.823917388916016,
      "activations/layer2_attention_weight_min": -32.70848083496094,
      "activations/layer3_attention_weight_max": 91.98249816894531,
      "activations/layer3_attention_weight_min": -95.7593002319336,
      "activations/layer4_attention_weight_max": 102.45635223388672,
      "activations/layer4_attention_weight_min": -97.9678726196289,
      "activations/layer5_attention_weight_max": 76.74979400634766,
      "activations/layer5_attention_weight_min": -77.23277282714844,
      "activations/layer6_attention_weight_max": 60.27906036376953,
      "activations/layer6_attention_weight_min": -56.23052215576172,
      "activations/layer7_attention_weight_max": 77.31512451171875,
      "activations/layer7_attention_weight_min": -72.32363891601562,
      "activations/layer8_attention_weight_max": 47.1789436340332,
      "activations/layer8_attention_weight_min": -45.64805603027344,
      "activations/layer9_attention_weight_max": 48.72057342529297,
      "activations/layer9_attention_weight_min": -48.2584114074707,
      "epoch": 10.19,
      "learning_rate": 8.511439393939392e-05,
      "loss": 2.783,
      "step": 175450
    },
    {
      "activations/layer0_attention_weight_max": 15.671991348266602,
      "activations/layer0_attention_weight_min": -14.390235900878906,
      "activations/layer10_attention_weight_max": 42.87601089477539,
      "activations/layer10_attention_weight_min": -38.20027542114258,
      "activations/layer11_attention_weight_max": 35.65748977661133,
      "activations/layer11_attention_weight_min": -33.481544494628906,
      "activations/layer12_attention_weight_max": 19.310684204101562,
      "activations/layer12_attention_weight_min": -25.28388786315918,
      "activations/layer13_attention_weight_max": 36.847625732421875,
      "activations/layer13_attention_weight_min": -30.499778747558594,
      "activations/layer14_attention_weight_max": 35.20266342163086,
      "activations/layer14_attention_weight_min": -30.711580276489258,
      "activations/layer15_attention_weight_max": 33.98971939086914,
      "activations/layer15_attention_weight_min": -29.510255813598633,
      "activations/layer16_attention_weight_max": 32.209678649902344,
      "activations/layer16_attention_weight_min": -29.39261817932129,
      "activations/layer17_attention_weight_max": 52.79058074951172,
      "activations/layer17_attention_weight_min": -48.36634826660156,
      "activations/layer18_attention_weight_max": 44.62473678588867,
      "activations/layer18_attention_weight_min": -37.90888214111328,
      "activations/layer19_attention_weight_max": 21.371700286865234,
      "activations/layer19_attention_weight_min": -20.906475067138672,
      "activations/layer1_attention_weight_max": 16.335647583007812,
      "activations/layer1_attention_weight_min": -15.87318229675293,
      "activations/layer20_attention_weight_max": 21.295612335205078,
      "activations/layer20_attention_weight_min": -22.960559844970703,
      "activations/layer21_attention_weight_max": 39.91559600830078,
      "activations/layer21_attention_weight_min": -24.10262107849121,
      "activations/layer22_attention_weight_max": 31.43409538269043,
      "activations/layer22_attention_weight_min": -26.272708892822266,
      "activations/layer23_attention_weight_max": 37.773460388183594,
      "activations/layer23_attention_weight_min": -20.96587371826172,
      "activations/layer2_attention_weight_max": 31.878149032592773,
      "activations/layer2_attention_weight_min": -32.14414978027344,
      "activations/layer3_attention_weight_max": 92.87403106689453,
      "activations/layer3_attention_weight_min": -97.21931457519531,
      "activations/layer4_attention_weight_max": 104.61891174316406,
      "activations/layer4_attention_weight_min": -103.3278579711914,
      "activations/layer5_attention_weight_max": 84.74974060058594,
      "activations/layer5_attention_weight_min": -81.01309204101562,
      "activations/layer6_attention_weight_max": 59.28059768676758,
      "activations/layer6_attention_weight_min": -61.75408172607422,
      "activations/layer7_attention_weight_max": 80.34494018554688,
      "activations/layer7_attention_weight_min": -76.51776885986328,
      "activations/layer8_attention_weight_max": 49.568450927734375,
      "activations/layer8_attention_weight_min": -50.30006790161133,
      "activations/layer9_attention_weight_max": 51.30487060546875,
      "activations/layer9_attention_weight_min": -49.14925765991211,
      "epoch": 10.2,
      "learning_rate": 8.509545454545454e-05,
      "loss": 2.7866,
      "step": 175500
    },
    {
      "activations/layer0_attention_weight_max": 15.992292404174805,
      "activations/layer0_attention_weight_min": -14.248408317565918,
      "activations/layer10_attention_weight_max": 43.48047637939453,
      "activations/layer10_attention_weight_min": -40.94440841674805,
      "activations/layer11_attention_weight_max": 36.45664596557617,
      "activations/layer11_attention_weight_min": -34.90743637084961,
      "activations/layer12_attention_weight_max": 20.386775970458984,
      "activations/layer12_attention_weight_min": -26.91275405883789,
      "activations/layer13_attention_weight_max": 44.181922912597656,
      "activations/layer13_attention_weight_min": -36.15285110473633,
      "activations/layer14_attention_weight_max": 39.01197052001953,
      "activations/layer14_attention_weight_min": -30.937467575073242,
      "activations/layer15_attention_weight_max": 34.651241302490234,
      "activations/layer15_attention_weight_min": -30.54703712463379,
      "activations/layer16_attention_weight_max": 33.249267578125,
      "activations/layer16_attention_weight_min": -29.132055282592773,
      "activations/layer17_attention_weight_max": 54.99953079223633,
      "activations/layer17_attention_weight_min": -46.473941802978516,
      "activations/layer18_attention_weight_max": 49.63270568847656,
      "activations/layer18_attention_weight_min": -39.02817153930664,
      "activations/layer19_attention_weight_max": 24.730131149291992,
      "activations/layer19_attention_weight_min": -22.530210494995117,
      "activations/layer1_attention_weight_max": 16.600196838378906,
      "activations/layer1_attention_weight_min": -14.416473388671875,
      "activations/layer20_attention_weight_max": 25.236936569213867,
      "activations/layer20_attention_weight_min": -22.523496627807617,
      "activations/layer21_attention_weight_max": 40.55231857299805,
      "activations/layer21_attention_weight_min": -22.865903854370117,
      "activations/layer22_attention_weight_max": 32.73487091064453,
      "activations/layer22_attention_weight_min": -26.546283721923828,
      "activations/layer23_attention_weight_max": 38.726898193359375,
      "activations/layer23_attention_weight_min": -23.656890869140625,
      "activations/layer2_attention_weight_max": 31.674894332885742,
      "activations/layer2_attention_weight_min": -31.851661682128906,
      "activations/layer3_attention_weight_max": 91.39594268798828,
      "activations/layer3_attention_weight_min": -94.62161254882812,
      "activations/layer4_attention_weight_max": 97.76472473144531,
      "activations/layer4_attention_weight_min": -97.70191955566406,
      "activations/layer5_attention_weight_max": 77.68577575683594,
      "activations/layer5_attention_weight_min": -78.1646728515625,
      "activations/layer6_attention_weight_max": 52.63031768798828,
      "activations/layer6_attention_weight_min": -53.60227966308594,
      "activations/layer7_attention_weight_max": 81.41326904296875,
      "activations/layer7_attention_weight_min": -78.44546508789062,
      "activations/layer8_attention_weight_max": 48.95390701293945,
      "activations/layer8_attention_weight_min": -49.39616394042969,
      "activations/layer9_attention_weight_max": 61.1893196105957,
      "activations/layer9_attention_weight_min": -53.4616584777832,
      "epoch": 10.2,
      "learning_rate": 8.507651515151515e-05,
      "loss": 2.7864,
      "step": 175550
    },
    {
      "activations/layer0_attention_weight_max": 15.33033275604248,
      "activations/layer0_attention_weight_min": -14.641688346862793,
      "activations/layer10_attention_weight_max": 40.016456604003906,
      "activations/layer10_attention_weight_min": -37.519989013671875,
      "activations/layer11_attention_weight_max": 33.78783416748047,
      "activations/layer11_attention_weight_min": -34.0532112121582,
      "activations/layer12_attention_weight_max": 21.92119789123535,
      "activations/layer12_attention_weight_min": -25.99003028869629,
      "activations/layer13_attention_weight_max": 41.77832794189453,
      "activations/layer13_attention_weight_min": -30.859638214111328,
      "activations/layer14_attention_weight_max": 41.193111419677734,
      "activations/layer14_attention_weight_min": -36.82011795043945,
      "activations/layer15_attention_weight_max": 39.947410583496094,
      "activations/layer15_attention_weight_min": -32.62711715698242,
      "activations/layer16_attention_weight_max": 35.59090805053711,
      "activations/layer16_attention_weight_min": -31.959436416625977,
      "activations/layer17_attention_weight_max": 55.69431686401367,
      "activations/layer17_attention_weight_min": -46.15750503540039,
      "activations/layer18_attention_weight_max": 51.30594253540039,
      "activations/layer18_attention_weight_min": -39.22423553466797,
      "activations/layer19_attention_weight_max": 24.99666404724121,
      "activations/layer19_attention_weight_min": -22.678930282592773,
      "activations/layer1_attention_weight_max": 17.999732971191406,
      "activations/layer1_attention_weight_min": -14.199962615966797,
      "activations/layer20_attention_weight_max": 26.526647567749023,
      "activations/layer20_attention_weight_min": -21.704286575317383,
      "activations/layer21_attention_weight_max": 51.04844665527344,
      "activations/layer21_attention_weight_min": -26.053770065307617,
      "activations/layer22_attention_weight_max": 33.29697036743164,
      "activations/layer22_attention_weight_min": -25.71767234802246,
      "activations/layer23_attention_weight_max": 41.26702117919922,
      "activations/layer23_attention_weight_min": -24.44855499267578,
      "activations/layer2_attention_weight_max": 32.01605987548828,
      "activations/layer2_attention_weight_min": -30.73493003845215,
      "activations/layer3_attention_weight_max": 89.52967834472656,
      "activations/layer3_attention_weight_min": -92.66814422607422,
      "activations/layer4_attention_weight_max": 99.53726196289062,
      "activations/layer4_attention_weight_min": -96.01872253417969,
      "activations/layer5_attention_weight_max": 73.0802230834961,
      "activations/layer5_attention_weight_min": -73.27987670898438,
      "activations/layer6_attention_weight_max": 53.15656661987305,
      "activations/layer6_attention_weight_min": -54.28464889526367,
      "activations/layer7_attention_weight_max": 76.44200134277344,
      "activations/layer7_attention_weight_min": -74.03199768066406,
      "activations/layer8_attention_weight_max": 45.71327590942383,
      "activations/layer8_attention_weight_min": -47.66427993774414,
      "activations/layer9_attention_weight_max": 51.6187629699707,
      "activations/layer9_attention_weight_min": -47.53237533569336,
      "epoch": 10.2,
      "learning_rate": 8.505757575757575e-05,
      "loss": 2.7812,
      "step": 175600
    },
    {
      "activations/layer0_attention_weight_max": 15.27886962890625,
      "activations/layer0_attention_weight_min": -14.312276840209961,
      "activations/layer10_attention_weight_max": 33.18857955932617,
      "activations/layer10_attention_weight_min": -33.720672607421875,
      "activations/layer11_attention_weight_max": 31.029769897460938,
      "activations/layer11_attention_weight_min": -31.504974365234375,
      "activations/layer12_attention_weight_max": 18.329755783081055,
      "activations/layer12_attention_weight_min": -28.913230895996094,
      "activations/layer13_attention_weight_max": 35.68241882324219,
      "activations/layer13_attention_weight_min": -28.65282440185547,
      "activations/layer14_attention_weight_max": 38.696434020996094,
      "activations/layer14_attention_weight_min": -28.434738159179688,
      "activations/layer15_attention_weight_max": 33.154178619384766,
      "activations/layer15_attention_weight_min": -28.516403198242188,
      "activations/layer16_attention_weight_max": 29.216787338256836,
      "activations/layer16_attention_weight_min": -27.28569984436035,
      "activations/layer17_attention_weight_max": 50.210906982421875,
      "activations/layer17_attention_weight_min": -41.157623291015625,
      "activations/layer18_attention_weight_max": 45.15916061401367,
      "activations/layer18_attention_weight_min": -36.26192855834961,
      "activations/layer19_attention_weight_max": 20.20778465270996,
      "activations/layer19_attention_weight_min": -21.542932510375977,
      "activations/layer1_attention_weight_max": 17.95882225036621,
      "activations/layer1_attention_weight_min": -13.042428016662598,
      "activations/layer20_attention_weight_max": 20.186155319213867,
      "activations/layer20_attention_weight_min": -20.73883819580078,
      "activations/layer21_attention_weight_max": 32.03562545776367,
      "activations/layer21_attention_weight_min": -18.869401931762695,
      "activations/layer22_attention_weight_max": 27.380268096923828,
      "activations/layer22_attention_weight_min": -25.51216697692871,
      "activations/layer23_attention_weight_max": 34.438541412353516,
      "activations/layer23_attention_weight_min": -21.438316345214844,
      "activations/layer2_attention_weight_max": 31.377300262451172,
      "activations/layer2_attention_weight_min": -30.52791404724121,
      "activations/layer3_attention_weight_max": 92.71537780761719,
      "activations/layer3_attention_weight_min": -93.98694610595703,
      "activations/layer4_attention_weight_max": 101.44004821777344,
      "activations/layer4_attention_weight_min": -91.7412338256836,
      "activations/layer5_attention_weight_max": 75.35868835449219,
      "activations/layer5_attention_weight_min": -78.39515686035156,
      "activations/layer6_attention_weight_max": 51.64549255371094,
      "activations/layer6_attention_weight_min": -53.81930160522461,
      "activations/layer7_attention_weight_max": 75.77995300292969,
      "activations/layer7_attention_weight_min": -72.09881591796875,
      "activations/layer8_attention_weight_max": 42.73393630981445,
      "activations/layer8_attention_weight_min": -43.81382751464844,
      "activations/layer9_attention_weight_max": 60.878944396972656,
      "activations/layer9_attention_weight_min": -48.17132568359375,
      "epoch": 10.21,
      "learning_rate": 8.503863636363635e-05,
      "loss": 2.7905,
      "step": 175650
    },
    {
      "activations/layer0_attention_weight_max": 15.965662002563477,
      "activations/layer0_attention_weight_min": -14.128305435180664,
      "activations/layer10_attention_weight_max": 40.518558502197266,
      "activations/layer10_attention_weight_min": -38.62923049926758,
      "activations/layer11_attention_weight_max": 36.704837799072266,
      "activations/layer11_attention_weight_min": -33.75673294067383,
      "activations/layer12_attention_weight_max": 22.369733810424805,
      "activations/layer12_attention_weight_min": -27.632198333740234,
      "activations/layer13_attention_weight_max": 46.13866424560547,
      "activations/layer13_attention_weight_min": -30.930740356445312,
      "activations/layer14_attention_weight_max": 42.01299285888672,
      "activations/layer14_attention_weight_min": -32.247314453125,
      "activations/layer15_attention_weight_max": 37.26614761352539,
      "activations/layer15_attention_weight_min": -30.018177032470703,
      "activations/layer16_attention_weight_max": 33.28114318847656,
      "activations/layer16_attention_weight_min": -30.968477249145508,
      "activations/layer17_attention_weight_max": 53.81038284301758,
      "activations/layer17_attention_weight_min": -44.90488815307617,
      "activations/layer18_attention_weight_max": 50.58238220214844,
      "activations/layer18_attention_weight_min": -37.455413818359375,
      "activations/layer19_attention_weight_max": 20.840232849121094,
      "activations/layer19_attention_weight_min": -22.247085571289062,
      "activations/layer1_attention_weight_max": 16.984832763671875,
      "activations/layer1_attention_weight_min": -15.414334297180176,
      "activations/layer20_attention_weight_max": 22.041175842285156,
      "activations/layer20_attention_weight_min": -20.999162673950195,
      "activations/layer21_attention_weight_max": 36.316505432128906,
      "activations/layer21_attention_weight_min": -23.524057388305664,
      "activations/layer22_attention_weight_max": 32.48621368408203,
      "activations/layer22_attention_weight_min": -26.51998519897461,
      "activations/layer23_attention_weight_max": 36.57086181640625,
      "activations/layer23_attention_weight_min": -20.458507537841797,
      "activations/layer2_attention_weight_max": 33.49840545654297,
      "activations/layer2_attention_weight_min": -32.35233688354492,
      "activations/layer3_attention_weight_max": 94.13595581054688,
      "activations/layer3_attention_weight_min": -97.40418243408203,
      "activations/layer4_attention_weight_max": 100.70777893066406,
      "activations/layer4_attention_weight_min": -98.22867584228516,
      "activations/layer5_attention_weight_max": 80.03178405761719,
      "activations/layer5_attention_weight_min": -77.95130920410156,
      "activations/layer6_attention_weight_max": 57.22137451171875,
      "activations/layer6_attention_weight_min": -54.28071594238281,
      "activations/layer7_attention_weight_max": 84.3214340209961,
      "activations/layer7_attention_weight_min": -76.51025390625,
      "activations/layer8_attention_weight_max": 53.86894607543945,
      "activations/layer8_attention_weight_min": -49.58243942260742,
      "activations/layer9_attention_weight_max": 65.60382080078125,
      "activations/layer9_attention_weight_min": -52.002132415771484,
      "epoch": 10.21,
      "learning_rate": 8.501969696969696e-05,
      "loss": 2.7943,
      "step": 175700
    },
    {
      "activations/layer0_attention_weight_max": 15.867070198059082,
      "activations/layer0_attention_weight_min": -14.45648193359375,
      "activations/layer10_attention_weight_max": 36.91480255126953,
      "activations/layer10_attention_weight_min": -34.48967361450195,
      "activations/layer11_attention_weight_max": 31.732891082763672,
      "activations/layer11_attention_weight_min": -32.18415832519531,
      "activations/layer12_attention_weight_max": 26.45303726196289,
      "activations/layer12_attention_weight_min": -24.467729568481445,
      "activations/layer13_attention_weight_max": 41.900413513183594,
      "activations/layer13_attention_weight_min": -31.24347496032715,
      "activations/layer14_attention_weight_max": 38.691532135009766,
      "activations/layer14_attention_weight_min": -29.83868408203125,
      "activations/layer15_attention_weight_max": 37.10536193847656,
      "activations/layer15_attention_weight_min": -29.67837142944336,
      "activations/layer16_attention_weight_max": 33.57612609863281,
      "activations/layer16_attention_weight_min": -29.58625030517578,
      "activations/layer17_attention_weight_max": 57.70235824584961,
      "activations/layer17_attention_weight_min": -44.61577224731445,
      "activations/layer18_attention_weight_max": 48.67874526977539,
      "activations/layer18_attention_weight_min": -37.57573699951172,
      "activations/layer19_attention_weight_max": 22.77322006225586,
      "activations/layer19_attention_weight_min": -23.98626136779785,
      "activations/layer1_attention_weight_max": 16.726394653320312,
      "activations/layer1_attention_weight_min": -14.36600112915039,
      "activations/layer20_attention_weight_max": 24.493206024169922,
      "activations/layer20_attention_weight_min": -24.437889099121094,
      "activations/layer21_attention_weight_max": 38.569480895996094,
      "activations/layer21_attention_weight_min": -22.220773696899414,
      "activations/layer22_attention_weight_max": 31.376569747924805,
      "activations/layer22_attention_weight_min": -25.729496002197266,
      "activations/layer23_attention_weight_max": 36.117958068847656,
      "activations/layer23_attention_weight_min": -24.174026489257812,
      "activations/layer2_attention_weight_max": 32.75016784667969,
      "activations/layer2_attention_weight_min": -30.905393600463867,
      "activations/layer3_attention_weight_max": 89.64849853515625,
      "activations/layer3_attention_weight_min": -95.96057891845703,
      "activations/layer4_attention_weight_max": 97.90421295166016,
      "activations/layer4_attention_weight_min": -96.04931640625,
      "activations/layer5_attention_weight_max": 76.87667846679688,
      "activations/layer5_attention_weight_min": -78.81515502929688,
      "activations/layer6_attention_weight_max": 55.14429473876953,
      "activations/layer6_attention_weight_min": -53.95203399658203,
      "activations/layer7_attention_weight_max": 77.70283508300781,
      "activations/layer7_attention_weight_min": -74.20878601074219,
      "activations/layer8_attention_weight_max": 47.385616302490234,
      "activations/layer8_attention_weight_min": -48.868717193603516,
      "activations/layer9_attention_weight_max": 48.82400894165039,
      "activations/layer9_attention_weight_min": -47.89236068725586,
      "epoch": 10.21,
      "learning_rate": 8.500075757575757e-05,
      "loss": 2.7888,
      "step": 175750
    },
    {
      "activations/layer0_attention_weight_max": 15.3095064163208,
      "activations/layer0_attention_weight_min": -14.669332504272461,
      "activations/layer10_attention_weight_max": 35.91691970825195,
      "activations/layer10_attention_weight_min": -35.772056579589844,
      "activations/layer11_attention_weight_max": 31.11652946472168,
      "activations/layer11_attention_weight_min": -33.148529052734375,
      "activations/layer12_attention_weight_max": 18.458799362182617,
      "activations/layer12_attention_weight_min": -25.622215270996094,
      "activations/layer13_attention_weight_max": 38.26241683959961,
      "activations/layer13_attention_weight_min": -28.66068458557129,
      "activations/layer14_attention_weight_max": 36.51130294799805,
      "activations/layer14_attention_weight_min": -32.05470275878906,
      "activations/layer15_attention_weight_max": 31.957504272460938,
      "activations/layer15_attention_weight_min": -32.92597579956055,
      "activations/layer16_attention_weight_max": 29.852109909057617,
      "activations/layer16_attention_weight_min": -28.583810806274414,
      "activations/layer17_attention_weight_max": 51.40742492675781,
      "activations/layer17_attention_weight_min": -43.76150131225586,
      "activations/layer18_attention_weight_max": 45.436912536621094,
      "activations/layer18_attention_weight_min": -36.185638427734375,
      "activations/layer19_attention_weight_max": 21.920631408691406,
      "activations/layer19_attention_weight_min": -20.814687728881836,
      "activations/layer1_attention_weight_max": 16.56911849975586,
      "activations/layer1_attention_weight_min": -17.031110763549805,
      "activations/layer20_attention_weight_max": 21.52463150024414,
      "activations/layer20_attention_weight_min": -23.07347297668457,
      "activations/layer21_attention_weight_max": 37.097408294677734,
      "activations/layer21_attention_weight_min": -26.703323364257812,
      "activations/layer22_attention_weight_max": 29.785743713378906,
      "activations/layer22_attention_weight_min": -24.863319396972656,
      "activations/layer23_attention_weight_max": 37.58702850341797,
      "activations/layer23_attention_weight_min": -27.193952560424805,
      "activations/layer2_attention_weight_max": 32.18342971801758,
      "activations/layer2_attention_weight_min": -32.439964294433594,
      "activations/layer3_attention_weight_max": 91.27733612060547,
      "activations/layer3_attention_weight_min": -97.16423797607422,
      "activations/layer4_attention_weight_max": 101.6119155883789,
      "activations/layer4_attention_weight_min": -94.60470581054688,
      "activations/layer5_attention_weight_max": 74.1589126586914,
      "activations/layer5_attention_weight_min": -81.04655456542969,
      "activations/layer6_attention_weight_max": 54.07468032836914,
      "activations/layer6_attention_weight_min": -52.90602111816406,
      "activations/layer7_attention_weight_max": 75.56414794921875,
      "activations/layer7_attention_weight_min": -74.43730926513672,
      "activations/layer8_attention_weight_max": 43.8340950012207,
      "activations/layer8_attention_weight_min": -45.16177749633789,
      "activations/layer9_attention_weight_max": 46.26612854003906,
      "activations/layer9_attention_weight_min": -46.73106002807617,
      "epoch": 10.21,
      "learning_rate": 8.498181818181817e-05,
      "loss": 2.7879,
      "step": 175800
    },
    {
      "activations/layer0_attention_weight_max": 16.4408016204834,
      "activations/layer0_attention_weight_min": -15.045951843261719,
      "activations/layer10_attention_weight_max": 36.20378494262695,
      "activations/layer10_attention_weight_min": -34.989864349365234,
      "activations/layer11_attention_weight_max": 30.9588680267334,
      "activations/layer11_attention_weight_min": -32.24601745605469,
      "activations/layer12_attention_weight_max": 16.797372817993164,
      "activations/layer12_attention_weight_min": -25.04009246826172,
      "activations/layer13_attention_weight_max": 33.61227035522461,
      "activations/layer13_attention_weight_min": -27.97489356994629,
      "activations/layer14_attention_weight_max": 31.9302921295166,
      "activations/layer14_attention_weight_min": -29.969560623168945,
      "activations/layer15_attention_weight_max": 29.22689437866211,
      "activations/layer15_attention_weight_min": -27.968000411987305,
      "activations/layer16_attention_weight_max": 29.637548446655273,
      "activations/layer16_attention_weight_min": -26.61528205871582,
      "activations/layer17_attention_weight_max": 48.43887710571289,
      "activations/layer17_attention_weight_min": -40.8691291809082,
      "activations/layer18_attention_weight_max": 41.487518310546875,
      "activations/layer18_attention_weight_min": -35.35663986206055,
      "activations/layer19_attention_weight_max": 19.495777130126953,
      "activations/layer19_attention_weight_min": -21.520021438598633,
      "activations/layer1_attention_weight_max": 16.464372634887695,
      "activations/layer1_attention_weight_min": -16.526445388793945,
      "activations/layer20_attention_weight_max": 19.83749008178711,
      "activations/layer20_attention_weight_min": -20.98018455505371,
      "activations/layer21_attention_weight_max": 34.73122787475586,
      "activations/layer21_attention_weight_min": -24.998210906982422,
      "activations/layer22_attention_weight_max": 28.29417610168457,
      "activations/layer22_attention_weight_min": -26.46223258972168,
      "activations/layer23_attention_weight_max": 39.02404022216797,
      "activations/layer23_attention_weight_min": -22.634105682373047,
      "activations/layer2_attention_weight_max": 30.816015243530273,
      "activations/layer2_attention_weight_min": -29.793956756591797,
      "activations/layer3_attention_weight_max": 91.45040130615234,
      "activations/layer3_attention_weight_min": -89.7950439453125,
      "activations/layer4_attention_weight_max": 99.20655822753906,
      "activations/layer4_attention_weight_min": -94.88610076904297,
      "activations/layer5_attention_weight_max": 75.45209503173828,
      "activations/layer5_attention_weight_min": -80.35047912597656,
      "activations/layer6_attention_weight_max": 58.66841506958008,
      "activations/layer6_attention_weight_min": -55.71585464477539,
      "activations/layer7_attention_weight_max": 78.56927490234375,
      "activations/layer7_attention_weight_min": -73.45787048339844,
      "activations/layer8_attention_weight_max": 47.32167434692383,
      "activations/layer8_attention_weight_min": -47.671104431152344,
      "activations/layer9_attention_weight_max": 47.04439163208008,
      "activations/layer9_attention_weight_min": -49.844329833984375,
      "epoch": 10.22,
      "learning_rate": 8.496287878787877e-05,
      "loss": 2.7748,
      "step": 175850
    },
    {
      "activations/layer0_attention_weight_max": 14.630350112915039,
      "activations/layer0_attention_weight_min": -14.739961624145508,
      "activations/layer10_attention_weight_max": 38.5169563293457,
      "activations/layer10_attention_weight_min": -36.20869827270508,
      "activations/layer11_attention_weight_max": 30.608718872070312,
      "activations/layer11_attention_weight_min": -30.15011978149414,
      "activations/layer12_attention_weight_max": 19.88751792907715,
      "activations/layer12_attention_weight_min": -23.039169311523438,
      "activations/layer13_attention_weight_max": 46.76997375488281,
      "activations/layer13_attention_weight_min": -33.260250091552734,
      "activations/layer14_attention_weight_max": 41.20940399169922,
      "activations/layer14_attention_weight_min": -32.43706512451172,
      "activations/layer15_attention_weight_max": 34.83867263793945,
      "activations/layer15_attention_weight_min": -29.84322166442871,
      "activations/layer16_attention_weight_max": 32.242889404296875,
      "activations/layer16_attention_weight_min": -28.282852172851562,
      "activations/layer17_attention_weight_max": 55.540489196777344,
      "activations/layer17_attention_weight_min": -43.36531066894531,
      "activations/layer18_attention_weight_max": 52.141780853271484,
      "activations/layer18_attention_weight_min": -37.852874755859375,
      "activations/layer19_attention_weight_max": 22.86225700378418,
      "activations/layer19_attention_weight_min": -21.086275100708008,
      "activations/layer1_attention_weight_max": 16.37989044189453,
      "activations/layer1_attention_weight_min": -15.738590240478516,
      "activations/layer20_attention_weight_max": 26.560340881347656,
      "activations/layer20_attention_weight_min": -22.934425354003906,
      "activations/layer21_attention_weight_max": 47.247554779052734,
      "activations/layer21_attention_weight_min": -28.362457275390625,
      "activations/layer22_attention_weight_max": 33.14246368408203,
      "activations/layer22_attention_weight_min": -25.130918502807617,
      "activations/layer23_attention_weight_max": 43.306575775146484,
      "activations/layer23_attention_weight_min": -25.06597137451172,
      "activations/layer2_attention_weight_max": 32.32380294799805,
      "activations/layer2_attention_weight_min": -30.090246200561523,
      "activations/layer3_attention_weight_max": 90.8259506225586,
      "activations/layer3_attention_weight_min": -92.92713928222656,
      "activations/layer4_attention_weight_max": 101.06472778320312,
      "activations/layer4_attention_weight_min": -95.37760925292969,
      "activations/layer5_attention_weight_max": 79.30994415283203,
      "activations/layer5_attention_weight_min": -79.80897521972656,
      "activations/layer6_attention_weight_max": 55.276519775390625,
      "activations/layer6_attention_weight_min": -55.240028381347656,
      "activations/layer7_attention_weight_max": 73.42662811279297,
      "activations/layer7_attention_weight_min": -78.05445098876953,
      "activations/layer8_attention_weight_max": 46.628414154052734,
      "activations/layer8_attention_weight_min": -45.90751647949219,
      "activations/layer9_attention_weight_max": 46.40201187133789,
      "activations/layer9_attention_weight_min": -48.3708610534668,
      "epoch": 10.22,
      "learning_rate": 8.494393939393939e-05,
      "loss": 2.782,
      "step": 175900
    },
    {
      "activations/layer0_attention_weight_max": 14.705317497253418,
      "activations/layer0_attention_weight_min": -14.863985061645508,
      "activations/layer10_attention_weight_max": 38.77434158325195,
      "activations/layer10_attention_weight_min": -35.06049346923828,
      "activations/layer11_attention_weight_max": 34.53460693359375,
      "activations/layer11_attention_weight_min": -32.85893630981445,
      "activations/layer12_attention_weight_max": 17.96427345275879,
      "activations/layer12_attention_weight_min": -26.37220001220703,
      "activations/layer13_attention_weight_max": 37.892513275146484,
      "activations/layer13_attention_weight_min": -28.777545928955078,
      "activations/layer14_attention_weight_max": 39.174495697021484,
      "activations/layer14_attention_weight_min": -31.111082077026367,
      "activations/layer15_attention_weight_max": 32.72751998901367,
      "activations/layer15_attention_weight_min": -30.27581787109375,
      "activations/layer16_attention_weight_max": 31.251367568969727,
      "activations/layer16_attention_weight_min": -28.500410079956055,
      "activations/layer17_attention_weight_max": 54.7266731262207,
      "activations/layer17_attention_weight_min": -43.74337387084961,
      "activations/layer18_attention_weight_max": 50.56003952026367,
      "activations/layer18_attention_weight_min": -37.97384262084961,
      "activations/layer19_attention_weight_max": 23.751996994018555,
      "activations/layer19_attention_weight_min": -22.724470138549805,
      "activations/layer1_attention_weight_max": 18.954647064208984,
      "activations/layer1_attention_weight_min": -17.093505859375,
      "activations/layer20_attention_weight_max": 24.5372257232666,
      "activations/layer20_attention_weight_min": -22.601261138916016,
      "activations/layer21_attention_weight_max": 41.9717903137207,
      "activations/layer21_attention_weight_min": -23.72661018371582,
      "activations/layer22_attention_weight_max": 29.998003005981445,
      "activations/layer22_attention_weight_min": -26.182357788085938,
      "activations/layer23_attention_weight_max": 39.39887237548828,
      "activations/layer23_attention_weight_min": -21.421247482299805,
      "activations/layer2_attention_weight_max": 32.66923522949219,
      "activations/layer2_attention_weight_min": -30.897613525390625,
      "activations/layer3_attention_weight_max": 88.3086929321289,
      "activations/layer3_attention_weight_min": -92.45631408691406,
      "activations/layer4_attention_weight_max": 99.10884857177734,
      "activations/layer4_attention_weight_min": -96.61360168457031,
      "activations/layer5_attention_weight_max": 74.74504089355469,
      "activations/layer5_attention_weight_min": -77.29924011230469,
      "activations/layer6_attention_weight_max": 56.13041687011719,
      "activations/layer6_attention_weight_min": -51.60313034057617,
      "activations/layer7_attention_weight_max": 76.28108978271484,
      "activations/layer7_attention_weight_min": -72.3683853149414,
      "activations/layer8_attention_weight_max": 46.17420959472656,
      "activations/layer8_attention_weight_min": -47.01200485229492,
      "activations/layer9_attention_weight_max": 48.814762115478516,
      "activations/layer9_attention_weight_min": -48.14714050292969,
      "epoch": 10.22,
      "learning_rate": 8.4925e-05,
      "loss": 2.7909,
      "step": 175950
    },
    {
      "activations/layer0_attention_weight_max": 16.020963668823242,
      "activations/layer0_attention_weight_min": -14.294148445129395,
      "activations/layer10_attention_weight_max": 41.229976654052734,
      "activations/layer10_attention_weight_min": -39.73921585083008,
      "activations/layer11_attention_weight_max": 34.984169006347656,
      "activations/layer11_attention_weight_min": -33.54559326171875,
      "activations/layer12_attention_weight_max": 20.291086196899414,
      "activations/layer12_attention_weight_min": -29.971752166748047,
      "activations/layer13_attention_weight_max": 38.92384719848633,
      "activations/layer13_attention_weight_min": -32.06951904296875,
      "activations/layer14_attention_weight_max": 35.33666229248047,
      "activations/layer14_attention_weight_min": -31.164127349853516,
      "activations/layer15_attention_weight_max": 33.085079193115234,
      "activations/layer15_attention_weight_min": -29.129240036010742,
      "activations/layer16_attention_weight_max": 30.595657348632812,
      "activations/layer16_attention_weight_min": -30.460222244262695,
      "activations/layer17_attention_weight_max": 52.89607620239258,
      "activations/layer17_attention_weight_min": -44.61428451538086,
      "activations/layer18_attention_weight_max": 46.88612747192383,
      "activations/layer18_attention_weight_min": -38.26258850097656,
      "activations/layer19_attention_weight_max": 21.24903678894043,
      "activations/layer19_attention_weight_min": -23.041425704956055,
      "activations/layer1_attention_weight_max": 17.32762336730957,
      "activations/layer1_attention_weight_min": -17.283349990844727,
      "activations/layer20_attention_weight_max": 18.865575790405273,
      "activations/layer20_attention_weight_min": -20.88566780090332,
      "activations/layer21_attention_weight_max": 39.2788200378418,
      "activations/layer21_attention_weight_min": -22.696090698242188,
      "activations/layer22_attention_weight_max": 29.599994659423828,
      "activations/layer22_attention_weight_min": -23.15325164794922,
      "activations/layer23_attention_weight_max": 33.38472366333008,
      "activations/layer23_attention_weight_min": -21.900976181030273,
      "activations/layer2_attention_weight_max": 35.75768280029297,
      "activations/layer2_attention_weight_min": -32.31898498535156,
      "activations/layer3_attention_weight_max": 101.72274017333984,
      "activations/layer3_attention_weight_min": -101.14297485351562,
      "activations/layer4_attention_weight_max": 104.48603820800781,
      "activations/layer4_attention_weight_min": -104.98164367675781,
      "activations/layer5_attention_weight_max": 81.27017211914062,
      "activations/layer5_attention_weight_min": -79.72025299072266,
      "activations/layer6_attention_weight_max": 57.93816375732422,
      "activations/layer6_attention_weight_min": -58.078861236572266,
      "activations/layer7_attention_weight_max": 80.71334075927734,
      "activations/layer7_attention_weight_min": -79.7190933227539,
      "activations/layer8_attention_weight_max": 49.56834411621094,
      "activations/layer8_attention_weight_min": -51.766536712646484,
      "activations/layer9_attention_weight_max": 53.081085205078125,
      "activations/layer9_attention_weight_min": -53.37887191772461,
      "epoch": 10.23,
      "learning_rate": 8.490606060606059e-05,
      "loss": 2.778,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4586,
      "eval_samples_per_second": 507.65,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4586,
      "eval_openwebtext_samples_per_second": 507.65,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_wikitext_loss": 2.966796875,
      "eval_wikitext_ppl": 19.429584428039178,
      "eval_wikitext_runtime": 1.9603,
      "eval_wikitext_samples_per_second": 232.622,
      "step": 176000
    },
    {
      "epoch": 10.23,
      "eval_lambada_loss": 2.728515625,
      "eval_lambada_ppl": 15.310144149387147,
      "eval_lambada_runtime": 9.54,
      "eval_lambada_samples_per_second": 510.38,
      "step": 176000
    },
    {
      "activations/layer0_attention_weight_max": 15.18541145324707,
      "activations/layer0_attention_weight_min": -14.076282501220703,
      "activations/layer10_attention_weight_max": 37.60745620727539,
      "activations/layer10_attention_weight_min": -36.488014221191406,
      "activations/layer11_attention_weight_max": 33.52497863769531,
      "activations/layer11_attention_weight_min": -32.78129577636719,
      "activations/layer12_attention_weight_max": 18.062551498413086,
      "activations/layer12_attention_weight_min": -22.567407608032227,
      "activations/layer13_attention_weight_max": 40.948219299316406,
      "activations/layer13_attention_weight_min": -29.964452743530273,
      "activations/layer14_attention_weight_max": 40.151832580566406,
      "activations/layer14_attention_weight_min": -32.565711975097656,
      "activations/layer15_attention_weight_max": 34.75767517089844,
      "activations/layer15_attention_weight_min": -30.3129940032959,
      "activations/layer16_attention_weight_max": 32.30006790161133,
      "activations/layer16_attention_weight_min": -29.679418563842773,
      "activations/layer17_attention_weight_max": 55.356998443603516,
      "activations/layer17_attention_weight_min": -43.99658203125,
      "activations/layer18_attention_weight_max": 49.31052780151367,
      "activations/layer18_attention_weight_min": -37.66891860961914,
      "activations/layer19_attention_weight_max": 25.024730682373047,
      "activations/layer19_attention_weight_min": -22.297163009643555,
      "activations/layer1_attention_weight_max": 15.633134841918945,
      "activations/layer1_attention_weight_min": -13.88061237335205,
      "activations/layer20_attention_weight_max": 25.960670471191406,
      "activations/layer20_attention_weight_min": -21.463218688964844,
      "activations/layer21_attention_weight_max": 41.49623107910156,
      "activations/layer21_attention_weight_min": -22.13907241821289,
      "activations/layer22_attention_weight_max": 33.68935012817383,
      "activations/layer22_attention_weight_min": -25.604511260986328,
      "activations/layer23_attention_weight_max": 43.66302490234375,
      "activations/layer23_attention_weight_min": -25.60662841796875,
      "activations/layer2_attention_weight_max": 30.2027645111084,
      "activations/layer2_attention_weight_min": -30.061986923217773,
      "activations/layer3_attention_weight_max": 91.2286376953125,
      "activations/layer3_attention_weight_min": -88.92965698242188,
      "activations/layer4_attention_weight_max": 100.81964874267578,
      "activations/layer4_attention_weight_min": -95.40220642089844,
      "activations/layer5_attention_weight_max": 75.31706237792969,
      "activations/layer5_attention_weight_min": -81.3908462524414,
      "activations/layer6_attention_weight_max": 56.43368911743164,
      "activations/layer6_attention_weight_min": -57.50104904174805,
      "activations/layer7_attention_weight_max": 76.77045440673828,
      "activations/layer7_attention_weight_min": -74.90740966796875,
      "activations/layer8_attention_weight_max": 45.70725631713867,
      "activations/layer8_attention_weight_min": -48.325469970703125,
      "activations/layer9_attention_weight_max": 49.48894500732422,
      "activations/layer9_attention_weight_min": -48.92940902709961,
      "epoch": 10.23,
      "learning_rate": 8.488712121212121e-05,
      "loss": 2.7614,
      "step": 176050
    },
    {
      "activations/layer0_attention_weight_max": 15.237534523010254,
      "activations/layer0_attention_weight_min": -14.55395793914795,
      "activations/layer10_attention_weight_max": 34.40476989746094,
      "activations/layer10_attention_weight_min": -35.045570373535156,
      "activations/layer11_attention_weight_max": 28.982254028320312,
      "activations/layer11_attention_weight_min": -32.3171501159668,
      "activations/layer12_attention_weight_max": 17.972139358520508,
      "activations/layer12_attention_weight_min": -27.077878952026367,
      "activations/layer13_attention_weight_max": 45.66200637817383,
      "activations/layer13_attention_weight_min": -31.001079559326172,
      "activations/layer14_attention_weight_max": 39.83180618286133,
      "activations/layer14_attention_weight_min": -31.932327270507812,
      "activations/layer15_attention_weight_max": 33.92220687866211,
      "activations/layer15_attention_weight_min": -29.46650505065918,
      "activations/layer16_attention_weight_max": 32.136600494384766,
      "activations/layer16_attention_weight_min": -29.673912048339844,
      "activations/layer17_attention_weight_max": 54.01591110229492,
      "activations/layer17_attention_weight_min": -43.1987190246582,
      "activations/layer18_attention_weight_max": 48.76963806152344,
      "activations/layer18_attention_weight_min": -37.750057220458984,
      "activations/layer19_attention_weight_max": 23.58540916442871,
      "activations/layer19_attention_weight_min": -20.82728385925293,
      "activations/layer1_attention_weight_max": 17.273197174072266,
      "activations/layer1_attention_weight_min": -15.861625671386719,
      "activations/layer20_attention_weight_max": 22.251827239990234,
      "activations/layer20_attention_weight_min": -21.773712158203125,
      "activations/layer21_attention_weight_max": 36.45878601074219,
      "activations/layer21_attention_weight_min": -24.129417419433594,
      "activations/layer22_attention_weight_max": 33.05550765991211,
      "activations/layer22_attention_weight_min": -25.942054748535156,
      "activations/layer23_attention_weight_max": 39.590946197509766,
      "activations/layer23_attention_weight_min": -23.663162231445312,
      "activations/layer2_attention_weight_max": 29.790992736816406,
      "activations/layer2_attention_weight_min": -28.85152244567871,
      "activations/layer3_attention_weight_max": 90.63871765136719,
      "activations/layer3_attention_weight_min": -93.67167663574219,
      "activations/layer4_attention_weight_max": 96.49064636230469,
      "activations/layer4_attention_weight_min": -94.41374206542969,
      "activations/layer5_attention_weight_max": 74.15753173828125,
      "activations/layer5_attention_weight_min": -84.30967712402344,
      "activations/layer6_attention_weight_max": 51.60777282714844,
      "activations/layer6_attention_weight_min": -50.55220031738281,
      "activations/layer7_attention_weight_max": 76.25228118896484,
      "activations/layer7_attention_weight_min": -72.42170715332031,
      "activations/layer8_attention_weight_max": 43.059783935546875,
      "activations/layer8_attention_weight_min": -44.0115966796875,
      "activations/layer9_attention_weight_max": 45.89640808105469,
      "activations/layer9_attention_weight_min": -45.03596115112305,
      "epoch": 10.23,
      "learning_rate": 8.486818181818182e-05,
      "loss": 2.7894,
      "step": 176100
    },
    {
      "activations/layer0_attention_weight_max": 15.20928955078125,
      "activations/layer0_attention_weight_min": -14.509885787963867,
      "activations/layer10_attention_weight_max": 37.186553955078125,
      "activations/layer10_attention_weight_min": -37.50709915161133,
      "activations/layer11_attention_weight_max": 32.508419036865234,
      "activations/layer11_attention_weight_min": -32.63822937011719,
      "activations/layer12_attention_weight_max": 19.826271057128906,
      "activations/layer12_attention_weight_min": -22.689838409423828,
      "activations/layer13_attention_weight_max": 39.3813362121582,
      "activations/layer13_attention_weight_min": -30.632705688476562,
      "activations/layer14_attention_weight_max": 37.11552810668945,
      "activations/layer14_attention_weight_min": -29.089128494262695,
      "activations/layer15_attention_weight_max": 33.5462760925293,
      "activations/layer15_attention_weight_min": -28.137144088745117,
      "activations/layer16_attention_weight_max": 32.8432731628418,
      "activations/layer16_attention_weight_min": -27.89970588684082,
      "activations/layer17_attention_weight_max": 53.26869201660156,
      "activations/layer17_attention_weight_min": -44.27581024169922,
      "activations/layer18_attention_weight_max": 46.55005645751953,
      "activations/layer18_attention_weight_min": -37.238311767578125,
      "activations/layer19_attention_weight_max": 20.91579818725586,
      "activations/layer19_attention_weight_min": -21.59184455871582,
      "activations/layer1_attention_weight_max": 17.136648178100586,
      "activations/layer1_attention_weight_min": -14.206147193908691,
      "activations/layer20_attention_weight_max": 24.175573348999023,
      "activations/layer20_attention_weight_min": -20.639183044433594,
      "activations/layer21_attention_weight_max": 40.21750259399414,
      "activations/layer21_attention_weight_min": -21.182193756103516,
      "activations/layer22_attention_weight_max": 31.378021240234375,
      "activations/layer22_attention_weight_min": -23.49605369567871,
      "activations/layer23_attention_weight_max": 44.31144332885742,
      "activations/layer23_attention_weight_min": -21.95013999938965,
      "activations/layer2_attention_weight_max": 29.795181274414062,
      "activations/layer2_attention_weight_min": -30.59236717224121,
      "activations/layer3_attention_weight_max": 89.5340805053711,
      "activations/layer3_attention_weight_min": -91.25336456298828,
      "activations/layer4_attention_weight_max": 107.21724700927734,
      "activations/layer4_attention_weight_min": -99.44998168945312,
      "activations/layer5_attention_weight_max": 82.7699203491211,
      "activations/layer5_attention_weight_min": -80.43020629882812,
      "activations/layer6_attention_weight_max": 61.13353729248047,
      "activations/layer6_attention_weight_min": -55.95729446411133,
      "activations/layer7_attention_weight_max": 78.95755004882812,
      "activations/layer7_attention_weight_min": -78.00762176513672,
      "activations/layer8_attention_weight_max": 46.888675689697266,
      "activations/layer8_attention_weight_min": -49.361053466796875,
      "activations/layer9_attention_weight_max": 54.33991241455078,
      "activations/layer9_attention_weight_min": -48.89297866821289,
      "epoch": 10.24,
      "learning_rate": 8.484924242424241e-05,
      "loss": 2.7946,
      "step": 176150
    },
    {
      "activations/layer0_attention_weight_max": 14.812296867370605,
      "activations/layer0_attention_weight_min": -14.544715881347656,
      "activations/layer10_attention_weight_max": 35.5774040222168,
      "activations/layer10_attention_weight_min": -36.237831115722656,
      "activations/layer11_attention_weight_max": 30.408985137939453,
      "activations/layer11_attention_weight_min": -31.669248580932617,
      "activations/layer12_attention_weight_max": 18.011398315429688,
      "activations/layer12_attention_weight_min": -25.803951263427734,
      "activations/layer13_attention_weight_max": 35.273582458496094,
      "activations/layer13_attention_weight_min": -28.67424201965332,
      "activations/layer14_attention_weight_max": 36.79098129272461,
      "activations/layer14_attention_weight_min": -31.38189697265625,
      "activations/layer15_attention_weight_max": 30.351184844970703,
      "activations/layer15_attention_weight_min": -30.055490493774414,
      "activations/layer16_attention_weight_max": 30.370622634887695,
      "activations/layer16_attention_weight_min": -28.52430534362793,
      "activations/layer17_attention_weight_max": 48.51553726196289,
      "activations/layer17_attention_weight_min": -47.06047821044922,
      "activations/layer18_attention_weight_max": 45.76047897338867,
      "activations/layer18_attention_weight_min": -38.462249755859375,
      "activations/layer19_attention_weight_max": 20.102375030517578,
      "activations/layer19_attention_weight_min": -22.31403160095215,
      "activations/layer1_attention_weight_max": 16.91489601135254,
      "activations/layer1_attention_weight_min": -15.984354972839355,
      "activations/layer20_attention_weight_max": 19.952329635620117,
      "activations/layer20_attention_weight_min": -24.289274215698242,
      "activations/layer21_attention_weight_max": 35.28180694580078,
      "activations/layer21_attention_weight_min": -20.971511840820312,
      "activations/layer22_attention_weight_max": 33.33015060424805,
      "activations/layer22_attention_weight_min": -26.365478515625,
      "activations/layer23_attention_weight_max": 33.29509735107422,
      "activations/layer23_attention_weight_min": -22.39236068725586,
      "activations/layer2_attention_weight_max": 31.112546920776367,
      "activations/layer2_attention_weight_min": -29.279254913330078,
      "activations/layer3_attention_weight_max": 89.01872253417969,
      "activations/layer3_attention_weight_min": -87.46829986572266,
      "activations/layer4_attention_weight_max": 97.89501953125,
      "activations/layer4_attention_weight_min": -92.32849884033203,
      "activations/layer5_attention_weight_max": 71.58185577392578,
      "activations/layer5_attention_weight_min": -77.13909912109375,
      "activations/layer6_attention_weight_max": 52.02141571044922,
      "activations/layer6_attention_weight_min": -51.655391693115234,
      "activations/layer7_attention_weight_max": 72.29592895507812,
      "activations/layer7_attention_weight_min": -67.64446258544922,
      "activations/layer8_attention_weight_max": 42.260276794433594,
      "activations/layer8_attention_weight_min": -45.88216781616211,
      "activations/layer9_attention_weight_max": 44.148319244384766,
      "activations/layer9_attention_weight_min": -46.146541595458984,
      "epoch": 10.24,
      "learning_rate": 8.48306818181818e-05,
      "loss": 2.783,
      "step": 176200
    },
    {
      "activations/layer0_attention_weight_max": 15.713919639587402,
      "activations/layer0_attention_weight_min": -15.058711051940918,
      "activations/layer10_attention_weight_max": 36.46050262451172,
      "activations/layer10_attention_weight_min": -35.65373229980469,
      "activations/layer11_attention_weight_max": 32.11301040649414,
      "activations/layer11_attention_weight_min": -32.42066192626953,
      "activations/layer12_attention_weight_max": 21.775739669799805,
      "activations/layer12_attention_weight_min": -24.8302001953125,
      "activations/layer13_attention_weight_max": 43.89183807373047,
      "activations/layer13_attention_weight_min": -32.27830505371094,
      "activations/layer14_attention_weight_max": 39.64981460571289,
      "activations/layer14_attention_weight_min": -32.4606819152832,
      "activations/layer15_attention_weight_max": 37.740386962890625,
      "activations/layer15_attention_weight_min": -30.94325828552246,
      "activations/layer16_attention_weight_max": 31.946060180664062,
      "activations/layer16_attention_weight_min": -30.311429977416992,
      "activations/layer17_attention_weight_max": 52.00830078125,
      "activations/layer17_attention_weight_min": -45.58995056152344,
      "activations/layer18_attention_weight_max": 48.58647918701172,
      "activations/layer18_attention_weight_min": -43.71709442138672,
      "activations/layer19_attention_weight_max": 25.650714874267578,
      "activations/layer19_attention_weight_min": -25.247570037841797,
      "activations/layer1_attention_weight_max": 17.59279441833496,
      "activations/layer1_attention_weight_min": -14.593220710754395,
      "activations/layer20_attention_weight_max": 25.600919723510742,
      "activations/layer20_attention_weight_min": -22.986234664916992,
      "activations/layer21_attention_weight_max": 39.989410400390625,
      "activations/layer21_attention_weight_min": -23.3236083984375,
      "activations/layer22_attention_weight_max": 33.636104583740234,
      "activations/layer22_attention_weight_min": -26.637605667114258,
      "activations/layer23_attention_weight_max": 34.781986236572266,
      "activations/layer23_attention_weight_min": -25.027538299560547,
      "activations/layer2_attention_weight_max": 34.24079895019531,
      "activations/layer2_attention_weight_min": -30.95282745361328,
      "activations/layer3_attention_weight_max": 93.18424224853516,
      "activations/layer3_attention_weight_min": -96.03209686279297,
      "activations/layer4_attention_weight_max": 102.4978256225586,
      "activations/layer4_attention_weight_min": -98.05250549316406,
      "activations/layer5_attention_weight_max": 80.2147216796875,
      "activations/layer5_attention_weight_min": -78.171875,
      "activations/layer6_attention_weight_max": 55.63270950317383,
      "activations/layer6_attention_weight_min": -52.63753890991211,
      "activations/layer7_attention_weight_max": 74.84481811523438,
      "activations/layer7_attention_weight_min": -74.12519073486328,
      "activations/layer8_attention_weight_max": 44.55377197265625,
      "activations/layer8_attention_weight_min": -47.833316802978516,
      "activations/layer9_attention_weight_max": 52.07584762573242,
      "activations/layer9_attention_weight_min": -47.89022445678711,
      "epoch": 10.24,
      "learning_rate": 8.481174242424242e-05,
      "loss": 2.7841,
      "step": 176250
    },
    {
      "activations/layer0_attention_weight_max": 14.390604972839355,
      "activations/layer0_attention_weight_min": -14.600326538085938,
      "activations/layer10_attention_weight_max": 37.37567138671875,
      "activations/layer10_attention_weight_min": -35.64794921875,
      "activations/layer11_attention_weight_max": 30.951269149780273,
      "activations/layer11_attention_weight_min": -33.577144622802734,
      "activations/layer12_attention_weight_max": 18.441892623901367,
      "activations/layer12_attention_weight_min": -25.177885055541992,
      "activations/layer13_attention_weight_max": 36.46554183959961,
      "activations/layer13_attention_weight_min": -27.5047607421875,
      "activations/layer14_attention_weight_max": 35.45820236206055,
      "activations/layer14_attention_weight_min": -28.7082576751709,
      "activations/layer15_attention_weight_max": 31.378002166748047,
      "activations/layer15_attention_weight_min": -28.084991455078125,
      "activations/layer16_attention_weight_max": 30.392959594726562,
      "activations/layer16_attention_weight_min": -28.203125,
      "activations/layer17_attention_weight_max": 51.46656036376953,
      "activations/layer17_attention_weight_min": -42.47378158569336,
      "activations/layer18_attention_weight_max": 48.81739807128906,
      "activations/layer18_attention_weight_min": -37.74485397338867,
      "activations/layer19_attention_weight_max": 22.602317810058594,
      "activations/layer19_attention_weight_min": -21.239757537841797,
      "activations/layer1_attention_weight_max": 16.930065155029297,
      "activations/layer1_attention_weight_min": -12.35508918762207,
      "activations/layer20_attention_weight_max": 22.358978271484375,
      "activations/layer20_attention_weight_min": -20.46996307373047,
      "activations/layer21_attention_weight_max": 35.968849182128906,
      "activations/layer21_attention_weight_min": -21.840730667114258,
      "activations/layer22_attention_weight_max": 31.979917526245117,
      "activations/layer22_attention_weight_min": -25.668502807617188,
      "activations/layer23_attention_weight_max": 41.32928466796875,
      "activations/layer23_attention_weight_min": -23.15681266784668,
      "activations/layer2_attention_weight_max": 27.053424835205078,
      "activations/layer2_attention_weight_min": -26.297088623046875,
      "activations/layer3_attention_weight_max": 87.61865997314453,
      "activations/layer3_attention_weight_min": -89.93975067138672,
      "activations/layer4_attention_weight_max": 100.51808166503906,
      "activations/layer4_attention_weight_min": -95.90696716308594,
      "activations/layer5_attention_weight_max": 77.52298736572266,
      "activations/layer5_attention_weight_min": -83.03822326660156,
      "activations/layer6_attention_weight_max": 56.2646369934082,
      "activations/layer6_attention_weight_min": -54.61565017700195,
      "activations/layer7_attention_weight_max": 74.02398681640625,
      "activations/layer7_attention_weight_min": -72.83415222167969,
      "activations/layer8_attention_weight_max": 45.015010833740234,
      "activations/layer8_attention_weight_min": -46.57160568237305,
      "activations/layer9_attention_weight_max": 47.665348052978516,
      "activations/layer9_attention_weight_min": -49.081600189208984,
      "epoch": 10.24,
      "learning_rate": 8.479280303030303e-05,
      "loss": 2.7722,
      "step": 176300
    },
    {
      "activations/layer0_attention_weight_max": 15.65467643737793,
      "activations/layer0_attention_weight_min": -14.199715614318848,
      "activations/layer10_attention_weight_max": 37.40114974975586,
      "activations/layer10_attention_weight_min": -37.812049865722656,
      "activations/layer11_attention_weight_max": 34.140480041503906,
      "activations/layer11_attention_weight_min": -34.024200439453125,
      "activations/layer12_attention_weight_max": 20.388322830200195,
      "activations/layer12_attention_weight_min": -24.396806716918945,
      "activations/layer13_attention_weight_max": 50.020328521728516,
      "activations/layer13_attention_weight_min": -32.08586883544922,
      "activations/layer14_attention_weight_max": 41.85932540893555,
      "activations/layer14_attention_weight_min": -33.118797302246094,
      "activations/layer15_attention_weight_max": 39.215457916259766,
      "activations/layer15_attention_weight_min": -31.6535587310791,
      "activations/layer16_attention_weight_max": 35.399173736572266,
      "activations/layer16_attention_weight_min": -32.755245208740234,
      "activations/layer17_attention_weight_max": 63.09059524536133,
      "activations/layer17_attention_weight_min": -48.02085876464844,
      "activations/layer18_attention_weight_max": 57.03605651855469,
      "activations/layer18_attention_weight_min": -38.68315887451172,
      "activations/layer19_attention_weight_max": 29.360931396484375,
      "activations/layer19_attention_weight_min": -22.088428497314453,
      "activations/layer1_attention_weight_max": 17.341209411621094,
      "activations/layer1_attention_weight_min": -14.380023002624512,
      "activations/layer20_attention_weight_max": 28.366899490356445,
      "activations/layer20_attention_weight_min": -23.919137954711914,
      "activations/layer21_attention_weight_max": 42.80084228515625,
      "activations/layer21_attention_weight_min": -23.076705932617188,
      "activations/layer22_attention_weight_max": 35.81474304199219,
      "activations/layer22_attention_weight_min": -26.64666748046875,
      "activations/layer23_attention_weight_max": 44.04458236694336,
      "activations/layer23_attention_weight_min": -26.867950439453125,
      "activations/layer2_attention_weight_max": 31.321088790893555,
      "activations/layer2_attention_weight_min": -30.037734985351562,
      "activations/layer3_attention_weight_max": 88.38056182861328,
      "activations/layer3_attention_weight_min": -91.75436401367188,
      "activations/layer4_attention_weight_max": 97.408935546875,
      "activations/layer4_attention_weight_min": -94.5906982421875,
      "activations/layer5_attention_weight_max": 77.36021423339844,
      "activations/layer5_attention_weight_min": -80.9330825805664,
      "activations/layer6_attention_weight_max": 55.47938537597656,
      "activations/layer6_attention_weight_min": -56.409034729003906,
      "activations/layer7_attention_weight_max": 80.32996368408203,
      "activations/layer7_attention_weight_min": -82.76830291748047,
      "activations/layer8_attention_weight_max": 48.31242752075195,
      "activations/layer8_attention_weight_min": -48.49595260620117,
      "activations/layer9_attention_weight_max": 58.473487854003906,
      "activations/layer9_attention_weight_min": -51.513099670410156,
      "epoch": 10.25,
      "learning_rate": 8.477386363636363e-05,
      "loss": 2.7862,
      "step": 176350
    },
    {
      "activations/layer0_attention_weight_max": 15.824804306030273,
      "activations/layer0_attention_weight_min": -14.558405876159668,
      "activations/layer10_attention_weight_max": 34.3688850402832,
      "activations/layer10_attention_weight_min": -33.99037551879883,
      "activations/layer11_attention_weight_max": 29.4405574798584,
      "activations/layer11_attention_weight_min": -31.293935775756836,
      "activations/layer12_attention_weight_max": 17.28915023803711,
      "activations/layer12_attention_weight_min": -24.505111694335938,
      "activations/layer13_attention_weight_max": 37.890281677246094,
      "activations/layer13_attention_weight_min": -29.685081481933594,
      "activations/layer14_attention_weight_max": 34.4324836730957,
      "activations/layer14_attention_weight_min": -30.626985549926758,
      "activations/layer15_attention_weight_max": 31.94249153137207,
      "activations/layer15_attention_weight_min": -29.320377349853516,
      "activations/layer16_attention_weight_max": 29.980417251586914,
      "activations/layer16_attention_weight_min": -29.851848602294922,
      "activations/layer17_attention_weight_max": 51.96375274658203,
      "activations/layer17_attention_weight_min": -44.8983039855957,
      "activations/layer18_attention_weight_max": 47.77784729003906,
      "activations/layer18_attention_weight_min": -40.341552734375,
      "activations/layer19_attention_weight_max": 21.503028869628906,
      "activations/layer19_attention_weight_min": -21.16777801513672,
      "activations/layer1_attention_weight_max": 17.14417266845703,
      "activations/layer1_attention_weight_min": -16.97233009338379,
      "activations/layer20_attention_weight_max": 21.07963752746582,
      "activations/layer20_attention_weight_min": -23.584033966064453,
      "activations/layer21_attention_weight_max": 36.02614974975586,
      "activations/layer21_attention_weight_min": -22.952234268188477,
      "activations/layer22_attention_weight_max": 31.24161148071289,
      "activations/layer22_attention_weight_min": -27.804712295532227,
      "activations/layer23_attention_weight_max": 35.32551574707031,
      "activations/layer23_attention_weight_min": -24.911962509155273,
      "activations/layer2_attention_weight_max": 30.747238159179688,
      "activations/layer2_attention_weight_min": -29.53192710876465,
      "activations/layer3_attention_weight_max": 88.63380432128906,
      "activations/layer3_attention_weight_min": -91.30292510986328,
      "activations/layer4_attention_weight_max": 100.11957550048828,
      "activations/layer4_attention_weight_min": -91.78309631347656,
      "activations/layer5_attention_weight_max": 71.70638275146484,
      "activations/layer5_attention_weight_min": -76.206787109375,
      "activations/layer6_attention_weight_max": 51.96158981323242,
      "activations/layer6_attention_weight_min": -51.85551452636719,
      "activations/layer7_attention_weight_max": 75.30622863769531,
      "activations/layer7_attention_weight_min": -68.21146392822266,
      "activations/layer8_attention_weight_max": 45.26725387573242,
      "activations/layer8_attention_weight_min": -44.30647277832031,
      "activations/layer9_attention_weight_max": 45.25396728515625,
      "activations/layer9_attention_weight_min": -47.578399658203125,
      "epoch": 10.25,
      "learning_rate": 8.475492424242423e-05,
      "loss": 2.7903,
      "step": 176400
    },
    {
      "activations/layer0_attention_weight_max": 15.617469787597656,
      "activations/layer0_attention_weight_min": -14.088789939880371,
      "activations/layer10_attention_weight_max": 36.067787170410156,
      "activations/layer10_attention_weight_min": -34.5584602355957,
      "activations/layer11_attention_weight_max": 30.65142822265625,
      "activations/layer11_attention_weight_min": -31.479644775390625,
      "activations/layer12_attention_weight_max": 17.767839431762695,
      "activations/layer12_attention_weight_min": -22.37368392944336,
      "activations/layer13_attention_weight_max": 39.89239501953125,
      "activations/layer13_attention_weight_min": -31.107946395874023,
      "activations/layer14_attention_weight_max": 36.04511260986328,
      "activations/layer14_attention_weight_min": -29.090431213378906,
      "activations/layer15_attention_weight_max": 31.921926498413086,
      "activations/layer15_attention_weight_min": -28.141279220581055,
      "activations/layer16_attention_weight_max": 29.890623092651367,
      "activations/layer16_attention_weight_min": -27.25647735595703,
      "activations/layer17_attention_weight_max": 50.49254608154297,
      "activations/layer17_attention_weight_min": -42.943267822265625,
      "activations/layer18_attention_weight_max": 44.67041015625,
      "activations/layer18_attention_weight_min": -38.1209831237793,
      "activations/layer19_attention_weight_max": 22.186153411865234,
      "activations/layer19_attention_weight_min": -23.785062789916992,
      "activations/layer1_attention_weight_max": 16.83729362487793,
      "activations/layer1_attention_weight_min": -14.735848426818848,
      "activations/layer20_attention_weight_max": 22.28440284729004,
      "activations/layer20_attention_weight_min": -28.42535972595215,
      "activations/layer21_attention_weight_max": 35.501644134521484,
      "activations/layer21_attention_weight_min": -23.781158447265625,
      "activations/layer22_attention_weight_max": 29.60720443725586,
      "activations/layer22_attention_weight_min": -26.37135124206543,
      "activations/layer23_attention_weight_max": 40.55326461791992,
      "activations/layer23_attention_weight_min": -25.12978172302246,
      "activations/layer2_attention_weight_max": 28.923324584960938,
      "activations/layer2_attention_weight_min": -28.208242416381836,
      "activations/layer3_attention_weight_max": 85.82915496826172,
      "activations/layer3_attention_weight_min": -86.19290161132812,
      "activations/layer4_attention_weight_max": 92.89260864257812,
      "activations/layer4_attention_weight_min": -87.75377655029297,
      "activations/layer5_attention_weight_max": 72.54438018798828,
      "activations/layer5_attention_weight_min": -77.45001220703125,
      "activations/layer6_attention_weight_max": 52.917198181152344,
      "activations/layer6_attention_weight_min": -51.81554412841797,
      "activations/layer7_attention_weight_max": 75.72512817382812,
      "activations/layer7_attention_weight_min": -71.4307861328125,
      "activations/layer8_attention_weight_max": 45.213233947753906,
      "activations/layer8_attention_weight_min": -46.012428283691406,
      "activations/layer9_attention_weight_max": 47.24179458618164,
      "activations/layer9_attention_weight_min": -48.95573806762695,
      "epoch": 10.25,
      "learning_rate": 8.473598484848484e-05,
      "loss": 2.7671,
      "step": 176450
    },
    {
      "activations/layer0_attention_weight_max": 15.090914726257324,
      "activations/layer0_attention_weight_min": -14.022415161132812,
      "activations/layer10_attention_weight_max": 42.84490966796875,
      "activations/layer10_attention_weight_min": -39.49510955810547,
      "activations/layer11_attention_weight_max": 35.34186553955078,
      "activations/layer11_attention_weight_min": -34.8721923828125,
      "activations/layer12_attention_weight_max": 17.995302200317383,
      "activations/layer12_attention_weight_min": -27.16469955444336,
      "activations/layer13_attention_weight_max": 41.82390213012695,
      "activations/layer13_attention_weight_min": -31.16805648803711,
      "activations/layer14_attention_weight_max": 37.63850021362305,
      "activations/layer14_attention_weight_min": -28.67102813720703,
      "activations/layer15_attention_weight_max": 34.553401947021484,
      "activations/layer15_attention_weight_min": -29.306039810180664,
      "activations/layer16_attention_weight_max": 29.481462478637695,
      "activations/layer16_attention_weight_min": -28.33353614807129,
      "activations/layer17_attention_weight_max": 54.08464050292969,
      "activations/layer17_attention_weight_min": -42.931861877441406,
      "activations/layer18_attention_weight_max": 47.066375732421875,
      "activations/layer18_attention_weight_min": -36.49811553955078,
      "activations/layer19_attention_weight_max": 19.616722106933594,
      "activations/layer19_attention_weight_min": -20.249271392822266,
      "activations/layer1_attention_weight_max": 16.860637664794922,
      "activations/layer1_attention_weight_min": -15.679474830627441,
      "activations/layer20_attention_weight_max": 21.860273361206055,
      "activations/layer20_attention_weight_min": -19.615339279174805,
      "activations/layer21_attention_weight_max": 40.2967529296875,
      "activations/layer21_attention_weight_min": -23.40362548828125,
      "activations/layer22_attention_weight_max": 29.694000244140625,
      "activations/layer22_attention_weight_min": -24.551233291625977,
      "activations/layer23_attention_weight_max": 37.97174072265625,
      "activations/layer23_attention_weight_min": -22.015249252319336,
      "activations/layer2_attention_weight_max": 30.340808868408203,
      "activations/layer2_attention_weight_min": -31.627948760986328,
      "activations/layer3_attention_weight_max": 90.21941375732422,
      "activations/layer3_attention_weight_min": -99.8212890625,
      "activations/layer4_attention_weight_max": 97.75078582763672,
      "activations/layer4_attention_weight_min": -100.9859390258789,
      "activations/layer5_attention_weight_max": 76.64127349853516,
      "activations/layer5_attention_weight_min": -80.20654296875,
      "activations/layer6_attention_weight_max": 56.97918701171875,
      "activations/layer6_attention_weight_min": -55.98028564453125,
      "activations/layer7_attention_weight_max": 76.96886444091797,
      "activations/layer7_attention_weight_min": -78.040771484375,
      "activations/layer8_attention_weight_max": 51.05677795410156,
      "activations/layer8_attention_weight_min": -53.74020767211914,
      "activations/layer9_attention_weight_max": 51.3884391784668,
      "activations/layer9_attention_weight_min": -52.23215866088867,
      "epoch": 10.26,
      "learning_rate": 8.471704545454545e-05,
      "loss": 2.7919,
      "step": 176500
    },
    {
      "activations/layer0_attention_weight_max": 15.614940643310547,
      "activations/layer0_attention_weight_min": -14.577378273010254,
      "activations/layer10_attention_weight_max": 41.4183235168457,
      "activations/layer10_attention_weight_min": -37.309242248535156,
      "activations/layer11_attention_weight_max": 36.146583557128906,
      "activations/layer11_attention_weight_min": -34.10834503173828,
      "activations/layer12_attention_weight_max": 20.025299072265625,
      "activations/layer12_attention_weight_min": -24.40806770324707,
      "activations/layer13_attention_weight_max": 41.98229217529297,
      "activations/layer13_attention_weight_min": -31.160709381103516,
      "activations/layer14_attention_weight_max": 38.10674285888672,
      "activations/layer14_attention_weight_min": -31.498193740844727,
      "activations/layer15_attention_weight_max": 35.92057800292969,
      "activations/layer15_attention_weight_min": -30.048355102539062,
      "activations/layer16_attention_weight_max": 32.361572265625,
      "activations/layer16_attention_weight_min": -29.09139633178711,
      "activations/layer17_attention_weight_max": 53.85060501098633,
      "activations/layer17_attention_weight_min": -42.58287811279297,
      "activations/layer18_attention_weight_max": 52.49550247192383,
      "activations/layer18_attention_weight_min": -39.82488250732422,
      "activations/layer19_attention_weight_max": 23.19023895263672,
      "activations/layer19_attention_weight_min": -23.13384437561035,
      "activations/layer1_attention_weight_max": 17.751548767089844,
      "activations/layer1_attention_weight_min": -15.712112426757812,
      "activations/layer20_attention_weight_max": 25.032459259033203,
      "activations/layer20_attention_weight_min": -20.474411010742188,
      "activations/layer21_attention_weight_max": 47.211822509765625,
      "activations/layer21_attention_weight_min": -24.22410774230957,
      "activations/layer22_attention_weight_max": 30.951990127563477,
      "activations/layer22_attention_weight_min": -25.82024383544922,
      "activations/layer23_attention_weight_max": 37.265716552734375,
      "activations/layer23_attention_weight_min": -25.276533126831055,
      "activations/layer2_attention_weight_max": 32.56755828857422,
      "activations/layer2_attention_weight_min": -30.427488327026367,
      "activations/layer3_attention_weight_max": 100.06649780273438,
      "activations/layer3_attention_weight_min": -95.5,
      "activations/layer4_attention_weight_max": 105.610595703125,
      "activations/layer4_attention_weight_min": -98.55879211425781,
      "activations/layer5_attention_weight_max": 79.15296173095703,
      "activations/layer5_attention_weight_min": -81.95845031738281,
      "activations/layer6_attention_weight_max": 55.58102798461914,
      "activations/layer6_attention_weight_min": -56.33555603027344,
      "activations/layer7_attention_weight_max": 75.8233413696289,
      "activations/layer7_attention_weight_min": -78.0040512084961,
      "activations/layer8_attention_weight_max": 47.805049896240234,
      "activations/layer8_attention_weight_min": -47.7213020324707,
      "activations/layer9_attention_weight_max": 50.64243698120117,
      "activations/layer9_attention_weight_min": -54.08468246459961,
      "epoch": 10.26,
      "learning_rate": 8.469810606060605e-05,
      "loss": 2.8004,
      "step": 176550
    },
    {
      "activations/layer0_attention_weight_max": 15.816827774047852,
      "activations/layer0_attention_weight_min": -14.399389266967773,
      "activations/layer10_attention_weight_max": 39.896453857421875,
      "activations/layer10_attention_weight_min": -38.51539993286133,
      "activations/layer11_attention_weight_max": 35.19334030151367,
      "activations/layer11_attention_weight_min": -34.34870529174805,
      "activations/layer12_attention_weight_max": 16.81023406982422,
      "activations/layer12_attention_weight_min": -25.04560661315918,
      "activations/layer13_attention_weight_max": 37.669822692871094,
      "activations/layer13_attention_weight_min": -29.213762283325195,
      "activations/layer14_attention_weight_max": 34.2275276184082,
      "activations/layer14_attention_weight_min": -30.306488037109375,
      "activations/layer15_attention_weight_max": 29.602121353149414,
      "activations/layer15_attention_weight_min": -27.589311599731445,
      "activations/layer16_attention_weight_max": 27.174724578857422,
      "activations/layer16_attention_weight_min": -26.379804611206055,
      "activations/layer17_attention_weight_max": 47.943450927734375,
      "activations/layer17_attention_weight_min": -42.963157653808594,
      "activations/layer18_attention_weight_max": 41.70403289794922,
      "activations/layer18_attention_weight_min": -33.017818450927734,
      "activations/layer19_attention_weight_max": 19.604354858398438,
      "activations/layer19_attention_weight_min": -20.33743667602539,
      "activations/layer1_attention_weight_max": 16.366954803466797,
      "activations/layer1_attention_weight_min": -15.476758003234863,
      "activations/layer20_attention_weight_max": 19.965255737304688,
      "activations/layer20_attention_weight_min": -21.784975051879883,
      "activations/layer21_attention_weight_max": 35.35261917114258,
      "activations/layer21_attention_weight_min": -20.90393829345703,
      "activations/layer22_attention_weight_max": 25.320465087890625,
      "activations/layer22_attention_weight_min": -24.124502182006836,
      "activations/layer23_attention_weight_max": 30.141035079956055,
      "activations/layer23_attention_weight_min": -22.235301971435547,
      "activations/layer2_attention_weight_max": 35.54941177368164,
      "activations/layer2_attention_weight_min": -34.506195068359375,
      "activations/layer3_attention_weight_max": 101.29103088378906,
      "activations/layer3_attention_weight_min": -101.75430297851562,
      "activations/layer4_attention_weight_max": 112.9852066040039,
      "activations/layer4_attention_weight_min": -107.34896087646484,
      "activations/layer5_attention_weight_max": 85.2496109008789,
      "activations/layer5_attention_weight_min": -81.39202880859375,
      "activations/layer6_attention_weight_max": 59.014434814453125,
      "activations/layer6_attention_weight_min": -56.685123443603516,
      "activations/layer7_attention_weight_max": 92.53292846679688,
      "activations/layer7_attention_weight_min": -83.7383804321289,
      "activations/layer8_attention_weight_max": 55.70654296875,
      "activations/layer8_attention_weight_min": -55.90752029418945,
      "activations/layer9_attention_weight_max": 60.49153518676758,
      "activations/layer9_attention_weight_min": -54.590023040771484,
      "epoch": 10.26,
      "learning_rate": 8.467916666666665e-05,
      "loss": 2.8007,
      "step": 176600
    },
    {
      "activations/layer0_attention_weight_max": 15.442647933959961,
      "activations/layer0_attention_weight_min": -14.080856323242188,
      "activations/layer10_attention_weight_max": 35.935367584228516,
      "activations/layer10_attention_weight_min": -37.164161682128906,
      "activations/layer11_attention_weight_max": 32.76346969604492,
      "activations/layer11_attention_weight_min": -33.67991638183594,
      "activations/layer12_attention_weight_max": 18.112363815307617,
      "activations/layer12_attention_weight_min": -25.7907657623291,
      "activations/layer13_attention_weight_max": 35.68989562988281,
      "activations/layer13_attention_weight_min": -28.439973831176758,
      "activations/layer14_attention_weight_max": 39.49258804321289,
      "activations/layer14_attention_weight_min": -31.591175079345703,
      "activations/layer15_attention_weight_max": 34.57543182373047,
      "activations/layer15_attention_weight_min": -29.665874481201172,
      "activations/layer16_attention_weight_max": 29.810176849365234,
      "activations/layer16_attention_weight_min": -29.695663452148438,
      "activations/layer17_attention_weight_max": 52.72316360473633,
      "activations/layer17_attention_weight_min": -44.87653732299805,
      "activations/layer18_attention_weight_max": 44.92253875732422,
      "activations/layer18_attention_weight_min": -38.88540267944336,
      "activations/layer19_attention_weight_max": 21.630781173706055,
      "activations/layer19_attention_weight_min": -22.547475814819336,
      "activations/layer1_attention_weight_max": 16.832294464111328,
      "activations/layer1_attention_weight_min": -14.555764198303223,
      "activations/layer20_attention_weight_max": 22.43360137939453,
      "activations/layer20_attention_weight_min": -24.97066879272461,
      "activations/layer21_attention_weight_max": 38.58590316772461,
      "activations/layer21_attention_weight_min": -23.20395851135254,
      "activations/layer22_attention_weight_max": 33.13823699951172,
      "activations/layer22_attention_weight_min": -27.85700225830078,
      "activations/layer23_attention_weight_max": 42.85736846923828,
      "activations/layer23_attention_weight_min": -24.21491813659668,
      "activations/layer2_attention_weight_max": 31.885732650756836,
      "activations/layer2_attention_weight_min": -30.49038314819336,
      "activations/layer3_attention_weight_max": 86.99594116210938,
      "activations/layer3_attention_weight_min": -89.92208099365234,
      "activations/layer4_attention_weight_max": 97.81803131103516,
      "activations/layer4_attention_weight_min": -94.94707489013672,
      "activations/layer5_attention_weight_max": 73.83674621582031,
      "activations/layer5_attention_weight_min": -78.87391662597656,
      "activations/layer6_attention_weight_max": 55.5228157043457,
      "activations/layer6_attention_weight_min": -52.2674674987793,
      "activations/layer7_attention_weight_max": 75.81393432617188,
      "activations/layer7_attention_weight_min": -73.84136199951172,
      "activations/layer8_attention_weight_max": 43.65337371826172,
      "activations/layer8_attention_weight_min": -47.436405181884766,
      "activations/layer9_attention_weight_max": 46.60676193237305,
      "activations/layer9_attention_weight_min": -50.025455474853516,
      "epoch": 10.26,
      "learning_rate": 8.466022727272727e-05,
      "loss": 2.7831,
      "step": 176650
    },
    {
      "activations/layer0_attention_weight_max": 15.251273155212402,
      "activations/layer0_attention_weight_min": -14.500540733337402,
      "activations/layer10_attention_weight_max": 43.61558151245117,
      "activations/layer10_attention_weight_min": -42.67156982421875,
      "activations/layer11_attention_weight_max": 39.407371520996094,
      "activations/layer11_attention_weight_min": -38.792015075683594,
      "activations/layer12_attention_weight_max": 19.957763671875,
      "activations/layer12_attention_weight_min": -27.364795684814453,
      "activations/layer13_attention_weight_max": 39.95335006713867,
      "activations/layer13_attention_weight_min": -30.612167358398438,
      "activations/layer14_attention_weight_max": 45.15378189086914,
      "activations/layer14_attention_weight_min": -29.78742218017578,
      "activations/layer15_attention_weight_max": 39.63124465942383,
      "activations/layer15_attention_weight_min": -28.777219772338867,
      "activations/layer16_attention_weight_max": 35.378807067871094,
      "activations/layer16_attention_weight_min": -30.003419876098633,
      "activations/layer17_attention_weight_max": 54.25677490234375,
      "activations/layer17_attention_weight_min": -44.672664642333984,
      "activations/layer18_attention_weight_max": 52.771732330322266,
      "activations/layer18_attention_weight_min": -40.25799560546875,
      "activations/layer19_attention_weight_max": 26.037748336791992,
      "activations/layer19_attention_weight_min": -25.26825714111328,
      "activations/layer1_attention_weight_max": 17.77362823486328,
      "activations/layer1_attention_weight_min": -15.796623229980469,
      "activations/layer20_attention_weight_max": 30.3704891204834,
      "activations/layer20_attention_weight_min": -23.42032241821289,
      "activations/layer21_attention_weight_max": 54.7664909362793,
      "activations/layer21_attention_weight_min": -25.59281349182129,
      "activations/layer22_attention_weight_max": 36.47771072387695,
      "activations/layer22_attention_weight_min": -25.67601203918457,
      "activations/layer23_attention_weight_max": 46.63468933105469,
      "activations/layer23_attention_weight_min": -25.75979995727539,
      "activations/layer2_attention_weight_max": 32.650238037109375,
      "activations/layer2_attention_weight_min": -32.15338897705078,
      "activations/layer3_attention_weight_max": 91.29214477539062,
      "activations/layer3_attention_weight_min": -94.5241928100586,
      "activations/layer4_attention_weight_max": 100.02584838867188,
      "activations/layer4_attention_weight_min": -102.04661560058594,
      "activations/layer5_attention_weight_max": 77.22675323486328,
      "activations/layer5_attention_weight_min": -79.31265258789062,
      "activations/layer6_attention_weight_max": 55.045570373535156,
      "activations/layer6_attention_weight_min": -57.03404998779297,
      "activations/layer7_attention_weight_max": 82.22286224365234,
      "activations/layer7_attention_weight_min": -79.93609619140625,
      "activations/layer8_attention_weight_max": 54.65165328979492,
      "activations/layer8_attention_weight_min": -53.79948043823242,
      "activations/layer9_attention_weight_max": 61.43886947631836,
      "activations/layer9_attention_weight_min": -55.79700469970703,
      "epoch": 10.27,
      "learning_rate": 8.464128787878788e-05,
      "loss": 2.7939,
      "step": 176700
    },
    {
      "activations/layer0_attention_weight_max": 15.430362701416016,
      "activations/layer0_attention_weight_min": -14.238945007324219,
      "activations/layer10_attention_weight_max": 42.4969482421875,
      "activations/layer10_attention_weight_min": -42.379180908203125,
      "activations/layer11_attention_weight_max": 36.719749450683594,
      "activations/layer11_attention_weight_min": -36.07322311401367,
      "activations/layer12_attention_weight_max": 20.426469802856445,
      "activations/layer12_attention_weight_min": -24.45440101623535,
      "activations/layer13_attention_weight_max": 43.106563568115234,
      "activations/layer13_attention_weight_min": -31.20425033569336,
      "activations/layer14_attention_weight_max": 39.902530670166016,
      "activations/layer14_attention_weight_min": -31.738540649414062,
      "activations/layer15_attention_weight_max": 35.594364166259766,
      "activations/layer15_attention_weight_min": -30.721378326416016,
      "activations/layer16_attention_weight_max": 32.82033157348633,
      "activations/layer16_attention_weight_min": -29.306964874267578,
      "activations/layer17_attention_weight_max": 57.33967590332031,
      "activations/layer17_attention_weight_min": -44.78841781616211,
      "activations/layer18_attention_weight_max": 51.37662124633789,
      "activations/layer18_attention_weight_min": -38.67159652709961,
      "activations/layer19_attention_weight_max": 27.978666305541992,
      "activations/layer19_attention_weight_min": -21.19771957397461,
      "activations/layer1_attention_weight_max": 17.35961151123047,
      "activations/layer1_attention_weight_min": -15.319478988647461,
      "activations/layer20_attention_weight_max": 25.04631996154785,
      "activations/layer20_attention_weight_min": -22.067049026489258,
      "activations/layer21_attention_weight_max": 46.721435546875,
      "activations/layer21_attention_weight_min": -24.592702865600586,
      "activations/layer22_attention_weight_max": 37.71122360229492,
      "activations/layer22_attention_weight_min": -25.49821662902832,
      "activations/layer23_attention_weight_max": 38.97357940673828,
      "activations/layer23_attention_weight_min": -22.295385360717773,
      "activations/layer2_attention_weight_max": 32.37633514404297,
      "activations/layer2_attention_weight_min": -28.71363067626953,
      "activations/layer3_attention_weight_max": 87.47970581054688,
      "activations/layer3_attention_weight_min": -86.99334716796875,
      "activations/layer4_attention_weight_max": 99.33325958251953,
      "activations/layer4_attention_weight_min": -99.0203628540039,
      "activations/layer5_attention_weight_max": 79.650634765625,
      "activations/layer5_attention_weight_min": -85.76039123535156,
      "activations/layer6_attention_weight_max": 58.36408233642578,
      "activations/layer6_attention_weight_min": -60.05572509765625,
      "activations/layer7_attention_weight_max": 79.12598419189453,
      "activations/layer7_attention_weight_min": -80.98876190185547,
      "activations/layer8_attention_weight_max": 53.80448532104492,
      "activations/layer8_attention_weight_min": -54.082332611083984,
      "activations/layer9_attention_weight_max": 54.315120697021484,
      "activations/layer9_attention_weight_min": -52.36518478393555,
      "epoch": 10.27,
      "learning_rate": 8.462234848484847e-05,
      "loss": 2.7797,
      "step": 176750
    },
    {
      "activations/layer0_attention_weight_max": 14.98134994506836,
      "activations/layer0_attention_weight_min": -14.403788566589355,
      "activations/layer10_attention_weight_max": 38.78367233276367,
      "activations/layer10_attention_weight_min": -35.93159103393555,
      "activations/layer11_attention_weight_max": 33.83116149902344,
      "activations/layer11_attention_weight_min": -33.46808624267578,
      "activations/layer12_attention_weight_max": 23.256351470947266,
      "activations/layer12_attention_weight_min": -23.48074722290039,
      "activations/layer13_attention_weight_max": 45.88630676269531,
      "activations/layer13_attention_weight_min": -32.26375961303711,
      "activations/layer14_attention_weight_max": 41.215606689453125,
      "activations/layer14_attention_weight_min": -29.484983444213867,
      "activations/layer15_attention_weight_max": 35.459503173828125,
      "activations/layer15_attention_weight_min": -28.182018280029297,
      "activations/layer16_attention_weight_max": 35.794673919677734,
      "activations/layer16_attention_weight_min": -28.017553329467773,
      "activations/layer17_attention_weight_max": 55.51312255859375,
      "activations/layer17_attention_weight_min": -41.64296340942383,
      "activations/layer18_attention_weight_max": 48.09690856933594,
      "activations/layer18_attention_weight_min": -36.60334396362305,
      "activations/layer19_attention_weight_max": 23.314809799194336,
      "activations/layer19_attention_weight_min": -21.297657012939453,
      "activations/layer1_attention_weight_max": 17.018169403076172,
      "activations/layer1_attention_weight_min": -15.279879570007324,
      "activations/layer20_attention_weight_max": 23.24764633178711,
      "activations/layer20_attention_weight_min": -24.913999557495117,
      "activations/layer21_attention_weight_max": 38.830909729003906,
      "activations/layer21_attention_weight_min": -22.759315490722656,
      "activations/layer22_attention_weight_max": 33.650936126708984,
      "activations/layer22_attention_weight_min": -26.31816291809082,
      "activations/layer23_attention_weight_max": 40.228878021240234,
      "activations/layer23_attention_weight_min": -27.460895538330078,
      "activations/layer2_attention_weight_max": 32.708106994628906,
      "activations/layer2_attention_weight_min": -30.298683166503906,
      "activations/layer3_attention_weight_max": 91.50585174560547,
      "activations/layer3_attention_weight_min": -91.75679016113281,
      "activations/layer4_attention_weight_max": 100.28216552734375,
      "activations/layer4_attention_weight_min": -91.00223541259766,
      "activations/layer5_attention_weight_max": 76.62483978271484,
      "activations/layer5_attention_weight_min": -78.82727813720703,
      "activations/layer6_attention_weight_max": 56.260379791259766,
      "activations/layer6_attention_weight_min": -55.35655212402344,
      "activations/layer7_attention_weight_max": 80.8145751953125,
      "activations/layer7_attention_weight_min": -72.0789566040039,
      "activations/layer8_attention_weight_max": 47.91611862182617,
      "activations/layer8_attention_weight_min": -49.575347900390625,
      "activations/layer9_attention_weight_max": 50.35966110229492,
      "activations/layer9_attention_weight_min": -52.396697998046875,
      "epoch": 10.27,
      "learning_rate": 8.460340909090909e-05,
      "loss": 2.7835,
      "step": 176800
    },
    {
      "activations/layer0_attention_weight_max": 14.93674087524414,
      "activations/layer0_attention_weight_min": -14.682114601135254,
      "activations/layer10_attention_weight_max": 38.35224533081055,
      "activations/layer10_attention_weight_min": -38.08045196533203,
      "activations/layer11_attention_weight_max": 31.81263542175293,
      "activations/layer11_attention_weight_min": -35.00518798828125,
      "activations/layer12_attention_weight_max": 17.71791648864746,
      "activations/layer12_attention_weight_min": -29.665416717529297,
      "activations/layer13_attention_weight_max": 36.66288757324219,
      "activations/layer13_attention_weight_min": -30.42654037475586,
      "activations/layer14_attention_weight_max": 34.491233825683594,
      "activations/layer14_attention_weight_min": -30.471294403076172,
      "activations/layer15_attention_weight_max": 34.39042282104492,
      "activations/layer15_attention_weight_min": -28.768333435058594,
      "activations/layer16_attention_weight_max": 31.386869430541992,
      "activations/layer16_attention_weight_min": -27.542797088623047,
      "activations/layer17_attention_weight_max": 49.10268020629883,
      "activations/layer17_attention_weight_min": -41.36685562133789,
      "activations/layer18_attention_weight_max": 43.91747283935547,
      "activations/layer18_attention_weight_min": -36.00475311279297,
      "activations/layer19_attention_weight_max": 19.806264877319336,
      "activations/layer19_attention_weight_min": -19.918630599975586,
      "activations/layer1_attention_weight_max": 16.851905822753906,
      "activations/layer1_attention_weight_min": -14.597099304199219,
      "activations/layer20_attention_weight_max": 22.94694709777832,
      "activations/layer20_attention_weight_min": -20.08207893371582,
      "activations/layer21_attention_weight_max": 34.1732177734375,
      "activations/layer21_attention_weight_min": -22.46597671508789,
      "activations/layer22_attention_weight_max": 33.13737487792969,
      "activations/layer22_attention_weight_min": -24.956954956054688,
      "activations/layer23_attention_weight_max": 39.08611297607422,
      "activations/layer23_attention_weight_min": -26.54213523864746,
      "activations/layer2_attention_weight_max": 32.310523986816406,
      "activations/layer2_attention_weight_min": -30.13338851928711,
      "activations/layer3_attention_weight_max": 92.1873779296875,
      "activations/layer3_attention_weight_min": -89.0302734375,
      "activations/layer4_attention_weight_max": 98.21348571777344,
      "activations/layer4_attention_weight_min": -95.72982788085938,
      "activations/layer5_attention_weight_max": 73.8861083984375,
      "activations/layer5_attention_weight_min": -81.41268920898438,
      "activations/layer6_attention_weight_max": 53.1489372253418,
      "activations/layer6_attention_weight_min": -52.60081481933594,
      "activations/layer7_attention_weight_max": 78.66879272460938,
      "activations/layer7_attention_weight_min": -72.52836608886719,
      "activations/layer8_attention_weight_max": 47.73367691040039,
      "activations/layer8_attention_weight_min": -47.05678176879883,
      "activations/layer9_attention_weight_max": 53.28676223754883,
      "activations/layer9_attention_weight_min": -49.19173812866211,
      "epoch": 10.28,
      "learning_rate": 8.45844696969697e-05,
      "loss": 2.7961,
      "step": 176850
    },
    {
      "activations/layer0_attention_weight_max": 15.488592147827148,
      "activations/layer0_attention_weight_min": -14.575925827026367,
      "activations/layer10_attention_weight_max": 38.3828239440918,
      "activations/layer10_attention_weight_min": -36.8864631652832,
      "activations/layer11_attention_weight_max": 33.62098693847656,
      "activations/layer11_attention_weight_min": -32.50006103515625,
      "activations/layer12_attention_weight_max": 17.911266326904297,
      "activations/layer12_attention_weight_min": -25.07286834716797,
      "activations/layer13_attention_weight_max": 53.926753997802734,
      "activations/layer13_attention_weight_min": -34.445655822753906,
      "activations/layer14_attention_weight_max": 46.875343322753906,
      "activations/layer14_attention_weight_min": -33.30568313598633,
      "activations/layer15_attention_weight_max": 36.91117858886719,
      "activations/layer15_attention_weight_min": -27.72228240966797,
      "activations/layer16_attention_weight_max": 30.845300674438477,
      "activations/layer16_attention_weight_min": -28.5875186920166,
      "activations/layer17_attention_weight_max": 53.55121612548828,
      "activations/layer17_attention_weight_min": -44.673282623291016,
      "activations/layer18_attention_weight_max": 48.13453674316406,
      "activations/layer18_attention_weight_min": -39.24766159057617,
      "activations/layer19_attention_weight_max": 25.426830291748047,
      "activations/layer19_attention_weight_min": -22.396974563598633,
      "activations/layer1_attention_weight_max": 17.808317184448242,
      "activations/layer1_attention_weight_min": -15.865403175354004,
      "activations/layer20_attention_weight_max": 26.70067596435547,
      "activations/layer20_attention_weight_min": -21.106491088867188,
      "activations/layer21_attention_weight_max": 40.38804244995117,
      "activations/layer21_attention_weight_min": -22.144990921020508,
      "activations/layer22_attention_weight_max": 31.277185440063477,
      "activations/layer22_attention_weight_min": -24.586715698242188,
      "activations/layer23_attention_weight_max": 37.210304260253906,
      "activations/layer23_attention_weight_min": -26.017059326171875,
      "activations/layer2_attention_weight_max": 32.5389518737793,
      "activations/layer2_attention_weight_min": -31.63327407836914,
      "activations/layer3_attention_weight_max": 91.10687255859375,
      "activations/layer3_attention_weight_min": -90.5494613647461,
      "activations/layer4_attention_weight_max": 99.43950653076172,
      "activations/layer4_attention_weight_min": -97.6078872680664,
      "activations/layer5_attention_weight_max": 75.64476776123047,
      "activations/layer5_attention_weight_min": -79.44719696044922,
      "activations/layer6_attention_weight_max": 52.23790740966797,
      "activations/layer6_attention_weight_min": -52.222938537597656,
      "activations/layer7_attention_weight_max": 84.77204895019531,
      "activations/layer7_attention_weight_min": -73.13247680664062,
      "activations/layer8_attention_weight_max": 47.75754928588867,
      "activations/layer8_attention_weight_min": -46.26799011230469,
      "activations/layer9_attention_weight_max": 59.25202178955078,
      "activations/layer9_attention_weight_min": -50.50889587402344,
      "epoch": 10.28,
      "learning_rate": 8.456553030303029e-05,
      "loss": 2.793,
      "step": 176900
    },
    {
      "activations/layer0_attention_weight_max": 14.505263328552246,
      "activations/layer0_attention_weight_min": -13.886676788330078,
      "activations/layer10_attention_weight_max": 37.5054817199707,
      "activations/layer10_attention_weight_min": -37.47522735595703,
      "activations/layer11_attention_weight_max": 31.182781219482422,
      "activations/layer11_attention_weight_min": -32.18979263305664,
      "activations/layer12_attention_weight_max": 18.039108276367188,
      "activations/layer12_attention_weight_min": -26.88604164123535,
      "activations/layer13_attention_weight_max": 36.26708221435547,
      "activations/layer13_attention_weight_min": -26.613300323486328,
      "activations/layer14_attention_weight_max": 37.26213455200195,
      "activations/layer14_attention_weight_min": -32.64207077026367,
      "activations/layer15_attention_weight_max": 36.04485321044922,
      "activations/layer15_attention_weight_min": -30.194982528686523,
      "activations/layer16_attention_weight_max": 31.056608200073242,
      "activations/layer16_attention_weight_min": -32.520450592041016,
      "activations/layer17_attention_weight_max": 54.88949966430664,
      "activations/layer17_attention_weight_min": -46.82758712768555,
      "activations/layer18_attention_weight_max": 46.48137283325195,
      "activations/layer18_attention_weight_min": -38.563751220703125,
      "activations/layer19_attention_weight_max": 20.9677734375,
      "activations/layer19_attention_weight_min": -21.677404403686523,
      "activations/layer1_attention_weight_max": 17.054365158081055,
      "activations/layer1_attention_weight_min": -14.141624450683594,
      "activations/layer20_attention_weight_max": 22.421720504760742,
      "activations/layer20_attention_weight_min": -22.703357696533203,
      "activations/layer21_attention_weight_max": 36.871883392333984,
      "activations/layer21_attention_weight_min": -23.193979263305664,
      "activations/layer22_attention_weight_max": 31.399106979370117,
      "activations/layer22_attention_weight_min": -26.09147834777832,
      "activations/layer23_attention_weight_max": 34.683868408203125,
      "activations/layer23_attention_weight_min": -21.8053035736084,
      "activations/layer2_attention_weight_max": 32.27688217163086,
      "activations/layer2_attention_weight_min": -29.095897674560547,
      "activations/layer3_attention_weight_max": 88.09646606445312,
      "activations/layer3_attention_weight_min": -87.03289794921875,
      "activations/layer4_attention_weight_max": 92.94657897949219,
      "activations/layer4_attention_weight_min": -92.44269561767578,
      "activations/layer5_attention_weight_max": 72.55885314941406,
      "activations/layer5_attention_weight_min": -78.24667358398438,
      "activations/layer6_attention_weight_max": 52.0545539855957,
      "activations/layer6_attention_weight_min": -50.34928512573242,
      "activations/layer7_attention_weight_max": 73.81481170654297,
      "activations/layer7_attention_weight_min": -72.05896759033203,
      "activations/layer8_attention_weight_max": 42.454498291015625,
      "activations/layer8_attention_weight_min": -44.67280960083008,
      "activations/layer9_attention_weight_max": 49.83527755737305,
      "activations/layer9_attention_weight_min": -47.38808822631836,
      "epoch": 10.28,
      "learning_rate": 8.45465909090909e-05,
      "loss": 2.8019,
      "step": 176950
    },
    {
      "activations/layer0_attention_weight_max": 15.482404708862305,
      "activations/layer0_attention_weight_min": -14.230889320373535,
      "activations/layer10_attention_weight_max": 34.166168212890625,
      "activations/layer10_attention_weight_min": -32.742401123046875,
      "activations/layer11_attention_weight_max": 30.214696884155273,
      "activations/layer11_attention_weight_min": -31.587873458862305,
      "activations/layer12_attention_weight_max": 19.850427627563477,
      "activations/layer12_attention_weight_min": -21.43183135986328,
      "activations/layer13_attention_weight_max": 36.958126068115234,
      "activations/layer13_attention_weight_min": -26.310583114624023,
      "activations/layer14_attention_weight_max": 35.615230560302734,
      "activations/layer14_attention_weight_min": -29.33770179748535,
      "activations/layer15_attention_weight_max": 34.2229118347168,
      "activations/layer15_attention_weight_min": -29.393056869506836,
      "activations/layer16_attention_weight_max": 31.78037452697754,
      "activations/layer16_attention_weight_min": -28.171810150146484,
      "activations/layer17_attention_weight_max": 53.01593780517578,
      "activations/layer17_attention_weight_min": -44.35093307495117,
      "activations/layer18_attention_weight_max": 45.38515853881836,
      "activations/layer18_attention_weight_min": -35.69801712036133,
      "activations/layer19_attention_weight_max": 23.591245651245117,
      "activations/layer19_attention_weight_min": -21.485727310180664,
      "activations/layer1_attention_weight_max": 17.0056095123291,
      "activations/layer1_attention_weight_min": -15.117289543151855,
      "activations/layer20_attention_weight_max": 21.055936813354492,
      "activations/layer20_attention_weight_min": -21.8585205078125,
      "activations/layer21_attention_weight_max": 35.045387268066406,
      "activations/layer21_attention_weight_min": -22.98428726196289,
      "activations/layer22_attention_weight_max": 32.982112884521484,
      "activations/layer22_attention_weight_min": -24.994592666625977,
      "activations/layer23_attention_weight_max": 37.109886169433594,
      "activations/layer23_attention_weight_min": -24.95779037475586,
      "activations/layer2_attention_weight_max": 29.291847229003906,
      "activations/layer2_attention_weight_min": -29.22133445739746,
      "activations/layer3_attention_weight_max": 82.09725952148438,
      "activations/layer3_attention_weight_min": -85.75602722167969,
      "activations/layer4_attention_weight_max": 90.17568969726562,
      "activations/layer4_attention_weight_min": -84.55393981933594,
      "activations/layer5_attention_weight_max": 72.01626586914062,
      "activations/layer5_attention_weight_min": -75.94256591796875,
      "activations/layer6_attention_weight_max": 50.64242935180664,
      "activations/layer6_attention_weight_min": -48.471473693847656,
      "activations/layer7_attention_weight_max": 71.61170196533203,
      "activations/layer7_attention_weight_min": -67.20332336425781,
      "activations/layer8_attention_weight_max": 42.21950149536133,
      "activations/layer8_attention_weight_min": -42.93024444580078,
      "activations/layer9_attention_weight_max": 44.00713348388672,
      "activations/layer9_attention_weight_min": -46.62668991088867,
      "epoch": 10.28,
      "learning_rate": 8.45276515151515e-05,
      "loss": 2.7728,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_loss": 2.7421875,
      "eval_runtime": 8.4494,
      "eval_samples_per_second": 508.204,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_openwebtext_loss": 2.7421875,
      "eval_openwebtext_ppl": 15.520899957093135,
      "eval_openwebtext_runtime": 8.4494,
      "eval_openwebtext_samples_per_second": 508.204,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 1.9612,
      "eval_wikitext_samples_per_second": 232.508,
      "step": 177000
    },
    {
      "epoch": 10.28,
      "eval_lambada_loss": 2.578125,
      "eval_lambada_ppl": 13.172416709211404,
      "eval_lambada_runtime": 9.5423,
      "eval_lambada_samples_per_second": 510.254,
      "step": 177000
    },
    {
      "activations/layer0_attention_weight_max": 14.869553565979004,
      "activations/layer0_attention_weight_min": -14.265247344970703,
      "activations/layer10_attention_weight_max": 36.2814826965332,
      "activations/layer10_attention_weight_min": -37.613525390625,
      "activations/layer11_attention_weight_max": 31.210891723632812,
      "activations/layer11_attention_weight_min": -32.05747985839844,
      "activations/layer12_attention_weight_max": 22.490110397338867,
      "activations/layer12_attention_weight_min": -26.673826217651367,
      "activations/layer13_attention_weight_max": 40.020896911621094,
      "activations/layer13_attention_weight_min": -27.8477725982666,
      "activations/layer14_attention_weight_max": 37.367862701416016,
      "activations/layer14_attention_weight_min": -32.623802185058594,
      "activations/layer15_attention_weight_max": 34.62950134277344,
      "activations/layer15_attention_weight_min": -31.457748413085938,
      "activations/layer16_attention_weight_max": 30.40703582763672,
      "activations/layer16_attention_weight_min": -30.10602378845215,
      "activations/layer17_attention_weight_max": 51.51904296875,
      "activations/layer17_attention_weight_min": -43.41520309448242,
      "activations/layer18_attention_weight_max": 45.13173294067383,
      "activations/layer18_attention_weight_min": -39.58412551879883,
      "activations/layer19_attention_weight_max": 22.70496940612793,
      "activations/layer19_attention_weight_min": -23.726940155029297,
      "activations/layer1_attention_weight_max": 17.039722442626953,
      "activations/layer1_attention_weight_min": -15.81855583190918,
      "activations/layer20_attention_weight_max": 21.640609741210938,
      "activations/layer20_attention_weight_min": -27.48501205444336,
      "activations/layer21_attention_weight_max": 36.59292221069336,
      "activations/layer21_attention_weight_min": -22.882431030273438,
      "activations/layer22_attention_weight_max": 32.610958099365234,
      "activations/layer22_attention_weight_min": -26.367347717285156,
      "activations/layer23_attention_weight_max": 42.071022033691406,
      "activations/layer23_attention_weight_min": -23.623703002929688,
      "activations/layer2_attention_weight_max": 32.766143798828125,
      "activations/layer2_attention_weight_min": -29.26580047607422,
      "activations/layer3_attention_weight_max": 90.10607147216797,
      "activations/layer3_attention_weight_min": -93.36563873291016,
      "activations/layer4_attention_weight_max": 94.15484619140625,
      "activations/layer4_attention_weight_min": -92.8672866821289,
      "activations/layer5_attention_weight_max": 73.40249633789062,
      "activations/layer5_attention_weight_min": -75.09451293945312,
      "activations/layer6_attention_weight_max": 52.32640075683594,
      "activations/layer6_attention_weight_min": -52.43396759033203,
      "activations/layer7_attention_weight_max": 72.50584411621094,
      "activations/layer7_attention_weight_min": -68.9092788696289,
      "activations/layer8_attention_weight_max": 43.302223205566406,
      "activations/layer8_attention_weight_min": -44.85786819458008,
      "activations/layer9_attention_weight_max": 49.75533676147461,
      "activations/layer9_attention_weight_min": -46.50010681152344,
      "epoch": 10.29,
      "learning_rate": 8.450871212121211e-05,
      "loss": 2.7905,
      "step": 177050
    },
    {
      "activations/layer0_attention_weight_max": 15.97676944732666,
      "activations/layer0_attention_weight_min": -14.377588272094727,
      "activations/layer10_attention_weight_max": 41.4295768737793,
      "activations/layer10_attention_weight_min": -38.45917510986328,
      "activations/layer11_attention_weight_max": 34.425724029541016,
      "activations/layer11_attention_weight_min": -33.970611572265625,
      "activations/layer12_attention_weight_max": 24.163253784179688,
      "activations/layer12_attention_weight_min": -23.26862144470215,
      "activations/layer13_attention_weight_max": 39.81480407714844,
      "activations/layer13_attention_weight_min": -29.36341094970703,
      "activations/layer14_attention_weight_max": 40.18339920043945,
      "activations/layer14_attention_weight_min": -31.934125900268555,
      "activations/layer15_attention_weight_max": 37.16273498535156,
      "activations/layer15_attention_weight_min": -30.567508697509766,
      "activations/layer16_attention_weight_max": 32.41898727416992,
      "activations/layer16_attention_weight_min": -30.239002227783203,
      "activations/layer17_attention_weight_max": 59.40740203857422,
      "activations/layer17_attention_weight_min": -47.46834182739258,
      "activations/layer18_attention_weight_max": 50.962886810302734,
      "activations/layer18_attention_weight_min": -43.75288391113281,
      "activations/layer19_attention_weight_max": 23.746763229370117,
      "activations/layer19_attention_weight_min": -22.675312042236328,
      "activations/layer1_attention_weight_max": 17.061351776123047,
      "activations/layer1_attention_weight_min": -14.256834030151367,
      "activations/layer20_attention_weight_max": 25.020816802978516,
      "activations/layer20_attention_weight_min": -20.35533905029297,
      "activations/layer21_attention_weight_max": 39.463233947753906,
      "activations/layer21_attention_weight_min": -23.226449966430664,
      "activations/layer22_attention_weight_max": 34.71982192993164,
      "activations/layer22_attention_weight_min": -28.226274490356445,
      "activations/layer23_attention_weight_max": 40.29966354370117,
      "activations/layer23_attention_weight_min": -22.85871696472168,
      "activations/layer2_attention_weight_max": 30.107215881347656,
      "activations/layer2_attention_weight_min": -31.304058074951172,
      "activations/layer3_attention_weight_max": 83.20550537109375,
      "activations/layer3_attention_weight_min": -85.7605209350586,
      "activations/layer4_attention_weight_max": 94.02123260498047,
      "activations/layer4_attention_weight_min": -95.81771087646484,
      "activations/layer5_attention_weight_max": 75.96144104003906,
      "activations/layer5_attention_weight_min": -78.84130096435547,
      "activations/layer6_attention_weight_max": 56.45941162109375,
      "activations/layer6_attention_weight_min": -53.891841888427734,
      "activations/layer7_attention_weight_max": 76.86599731445312,
      "activations/layer7_attention_weight_min": -76.63514709472656,
      "activations/layer8_attention_weight_max": 48.6544075012207,
      "activations/layer8_attention_weight_min": -49.65873336791992,
      "activations/layer9_attention_weight_max": 52.28351974487305,
      "activations/layer9_attention_weight_min": -49.1171875,
      "epoch": 10.29,
      "learning_rate": 8.448977272727272e-05,
      "loss": 2.7897,
      "step": 177100
    },
    {
      "activations/layer0_attention_weight_max": 15.037652969360352,
      "activations/layer0_attention_weight_min": -14.542831420898438,
      "activations/layer10_attention_weight_max": 34.769287109375,
      "activations/layer10_attention_weight_min": -34.84727478027344,
      "activations/layer11_attention_weight_max": 31.45509910583496,
      "activations/layer11_attention_weight_min": -32.596221923828125,
      "activations/layer12_attention_weight_max": 18.078739166259766,
      "activations/layer12_attention_weight_min": -24.406314849853516,
      "activations/layer13_attention_weight_max": 37.38799285888672,
      "activations/layer13_attention_weight_min": -28.347789764404297,
      "activations/layer14_attention_weight_max": 34.282470703125,
      "activations/layer14_attention_weight_min": -31.444677352905273,
      "activations/layer15_attention_weight_max": 31.759761810302734,
      "activations/layer15_attention_weight_min": -30.010915756225586,
      "activations/layer16_attention_weight_max": 29.96811866760254,
      "activations/layer16_attention_weight_min": -30.370494842529297,
      "activations/layer17_attention_weight_max": 52.36906051635742,
      "activations/layer17_attention_weight_min": -42.983760833740234,
      "activations/layer18_attention_weight_max": 49.601436614990234,
      "activations/layer18_attention_weight_min": -36.73996353149414,
      "activations/layer19_attention_weight_max": 21.094789505004883,
      "activations/layer19_attention_weight_min": -21.97701072692871,
      "activations/layer1_attention_weight_max": 17.401657104492188,
      "activations/layer1_attention_weight_min": -16.56541633605957,
      "activations/layer20_attention_weight_max": 22.992841720581055,
      "activations/layer20_attention_weight_min": -24.915325164794922,
      "activations/layer21_attention_weight_max": 36.66407012939453,
      "activations/layer21_attention_weight_min": -23.02012062072754,
      "activations/layer22_attention_weight_max": 32.03512954711914,
      "activations/layer22_attention_weight_min": -25.798364639282227,
      "activations/layer23_attention_weight_max": 36.252471923828125,
      "activations/layer23_attention_weight_min": -22.52355194091797,
      "activations/layer2_attention_weight_max": 31.387577056884766,
      "activations/layer2_attention_weight_min": -29.19148063659668,
      "activations/layer3_attention_weight_max": 90.24520874023438,
      "activations/layer3_attention_weight_min": -90.03797912597656,
      "activations/layer4_attention_weight_max": 94.42269897460938,
      "activations/layer4_attention_weight_min": -91.36788940429688,
      "activations/layer5_attention_weight_max": 73.95561218261719,
      "activations/layer5_attention_weight_min": -78.44485473632812,
      "activations/layer6_attention_weight_max": 53.138221740722656,
      "activations/layer6_attention_weight_min": -54.91719055175781,
      "activations/layer7_attention_weight_max": 72.49002838134766,
      "activations/layer7_attention_weight_min": -70.36717224121094,
      "activations/layer8_attention_weight_max": 45.790504455566406,
      "activations/layer8_attention_weight_min": -47.04148864746094,
      "activations/layer9_attention_weight_max": 43.0534553527832,
      "activations/layer9_attention_weight_min": -48.75029754638672,
      "epoch": 10.29,
      "learning_rate": 8.447083333333332e-05,
      "loss": 2.7889,
      "step": 177150
    },
    {
      "activations/layer0_attention_weight_max": 15.97632122039795,
      "activations/layer0_attention_weight_min": -14.202581405639648,
      "activations/layer10_attention_weight_max": 40.05820846557617,
      "activations/layer10_attention_weight_min": -36.224281311035156,
      "activations/layer11_attention_weight_max": 34.325469970703125,
      "activations/layer11_attention_weight_min": -32.47624588012695,
      "activations/layer12_attention_weight_max": 21.24151611328125,
      "activations/layer12_attention_weight_min": -30.788776397705078,
      "activations/layer13_attention_weight_max": 44.23332595825195,
      "activations/layer13_attention_weight_min": -32.28042984008789,
      "activations/layer14_attention_weight_max": 45.24213790893555,
      "activations/layer14_attention_weight_min": -30.35366439819336,
      "activations/layer15_attention_weight_max": 35.86500549316406,
      "activations/layer15_attention_weight_min": -29.8132381439209,
      "activations/layer16_attention_weight_max": 33.10187911987305,
      "activations/layer16_attention_weight_min": -28.645471572875977,
      "activations/layer17_attention_weight_max": 53.18162536621094,
      "activations/layer17_attention_weight_min": -43.51521301269531,
      "activations/layer18_attention_weight_max": 48.97462844848633,
      "activations/layer18_attention_weight_min": -39.624671936035156,
      "activations/layer19_attention_weight_max": 23.624399185180664,
      "activations/layer19_attention_weight_min": -23.416736602783203,
      "activations/layer1_attention_weight_max": 16.914756774902344,
      "activations/layer1_attention_weight_min": -16.66892433166504,
      "activations/layer20_attention_weight_max": 25.090768814086914,
      "activations/layer20_attention_weight_min": -21.51555633544922,
      "activations/layer21_attention_weight_max": 41.458892822265625,
      "activations/layer21_attention_weight_min": -24.666349411010742,
      "activations/layer22_attention_weight_max": 31.20622444152832,
      "activations/layer22_attention_weight_min": -27.055423736572266,
      "activations/layer23_attention_weight_max": 37.96250534057617,
      "activations/layer23_attention_weight_min": -25.118831634521484,
      "activations/layer2_attention_weight_max": 32.539405822753906,
      "activations/layer2_attention_weight_min": -30.184850692749023,
      "activations/layer3_attention_weight_max": 88.62958526611328,
      "activations/layer3_attention_weight_min": -88.64842987060547,
      "activations/layer4_attention_weight_max": 101.17584228515625,
      "activations/layer4_attention_weight_min": -97.7037582397461,
      "activations/layer5_attention_weight_max": 78.14964294433594,
      "activations/layer5_attention_weight_min": -81.75767517089844,
      "activations/layer6_attention_weight_max": 58.28972625732422,
      "activations/layer6_attention_weight_min": -55.3568000793457,
      "activations/layer7_attention_weight_max": 76.42304992675781,
      "activations/layer7_attention_weight_min": -73.4407958984375,
      "activations/layer8_attention_weight_max": 46.9162483215332,
      "activations/layer8_attention_weight_min": -49.76737976074219,
      "activations/layer9_attention_weight_max": 51.01436996459961,
      "activations/layer9_attention_weight_min": -52.0787467956543,
      "epoch": 10.3,
      "learning_rate": 8.445189393939394e-05,
      "loss": 2.7956,
      "step": 177200
    },
    {
      "activations/layer0_attention_weight_max": 16.4595890045166,
      "activations/layer0_attention_weight_min": -13.999845504760742,
      "activations/layer10_attention_weight_max": 35.36014938354492,
      "activations/layer10_attention_weight_min": -33.14408874511719,
      "activations/layer11_attention_weight_max": 31.02166748046875,
      "activations/layer11_attention_weight_min": -31.24114990234375,
      "activations/layer12_attention_weight_max": 18.548402786254883,
      "activations/layer12_attention_weight_min": -32.75774002075195,
      "activations/layer13_attention_weight_max": 38.19747543334961,
      "activations/layer13_attention_weight_min": -31.20713233947754,
      "activations/layer14_attention_weight_max": 36.21139144897461,
      "activations/layer14_attention_weight_min": -29.526437759399414,
      "activations/layer15_attention_weight_max": 34.920440673828125,
      "activations/layer15_attention_weight_min": -29.630069732666016,
      "activations/layer16_attention_weight_max": 31.147090911865234,
      "activations/layer16_attention_weight_min": -28.610694885253906,
      "activations/layer17_attention_weight_max": 52.383033752441406,
      "activations/layer17_attention_weight_min": -44.11014175415039,
      "activations/layer18_attention_weight_max": 46.337337493896484,
      "activations/layer18_attention_weight_min": -38.32381057739258,
      "activations/layer19_attention_weight_max": 24.96269416809082,
      "activations/layer19_attention_weight_min": -23.418424606323242,
      "activations/layer1_attention_weight_max": 16.21888542175293,
      "activations/layer1_attention_weight_min": -14.081500053405762,
      "activations/layer20_attention_weight_max": 23.28453826904297,
      "activations/layer20_attention_weight_min": -20.987394332885742,
      "activations/layer21_attention_weight_max": 36.858558654785156,
      "activations/layer21_attention_weight_min": -23.17450523376465,
      "activations/layer22_attention_weight_max": 29.610912322998047,
      "activations/layer22_attention_weight_min": -24.9609317779541,
      "activations/layer23_attention_weight_max": 38.474952697753906,
      "activations/layer23_attention_weight_min": -23.940876007080078,
      "activations/layer2_attention_weight_max": 32.8591423034668,
      "activations/layer2_attention_weight_min": -29.87856674194336,
      "activations/layer3_attention_weight_max": 95.36698913574219,
      "activations/layer3_attention_weight_min": -87.26302337646484,
      "activations/layer4_attention_weight_max": 99.75096893310547,
      "activations/layer4_attention_weight_min": -89.9684066772461,
      "activations/layer5_attention_weight_max": 70.70065307617188,
      "activations/layer5_attention_weight_min": -76.90101623535156,
      "activations/layer6_attention_weight_max": 50.48322296142578,
      "activations/layer6_attention_weight_min": -52.17929458618164,
      "activations/layer7_attention_weight_max": 76.26668548583984,
      "activations/layer7_attention_weight_min": -72.65189361572266,
      "activations/layer8_attention_weight_max": 42.44852066040039,
      "activations/layer8_attention_weight_min": -43.844051361083984,
      "activations/layer9_attention_weight_max": 54.41651153564453,
      "activations/layer9_attention_weight_min": -48.97343063354492,
      "epoch": 10.3,
      "learning_rate": 8.443295454545454e-05,
      "loss": 2.8005,
      "step": 177250
    },
    {
      "activations/layer0_attention_weight_max": 15.484231948852539,
      "activations/layer0_attention_weight_min": -14.585933685302734,
      "activations/layer10_attention_weight_max": 34.9339599609375,
      "activations/layer10_attention_weight_min": -34.13330078125,
      "activations/layer11_attention_weight_max": 29.578495025634766,
      "activations/layer11_attention_weight_min": -30.7519588470459,
      "activations/layer12_attention_weight_max": 18.44280433654785,
      "activations/layer12_attention_weight_min": -25.230222702026367,
      "activations/layer13_attention_weight_max": 39.94813919067383,
      "activations/layer13_attention_weight_min": -31.181753158569336,
      "activations/layer14_attention_weight_max": 38.447959899902344,
      "activations/layer14_attention_weight_min": -30.602527618408203,
      "activations/layer15_attention_weight_max": 36.07820510864258,
      "activations/layer15_attention_weight_min": -30.216712951660156,
      "activations/layer16_attention_weight_max": 33.33232498168945,
      "activations/layer16_attention_weight_min": -28.319177627563477,
      "activations/layer17_attention_weight_max": 50.87117385864258,
      "activations/layer17_attention_weight_min": -43.019447326660156,
      "activations/layer18_attention_weight_max": 48.86001968383789,
      "activations/layer18_attention_weight_min": -36.0634880065918,
      "activations/layer19_attention_weight_max": 22.227100372314453,
      "activations/layer19_attention_weight_min": -21.606224060058594,
      "activations/layer1_attention_weight_max": 16.775354385375977,
      "activations/layer1_attention_weight_min": -15.07877254486084,
      "activations/layer20_attention_weight_max": 24.237632751464844,
      "activations/layer20_attention_weight_min": -22.27124786376953,
      "activations/layer21_attention_weight_max": 39.967567443847656,
      "activations/layer21_attention_weight_min": -25.96060562133789,
      "activations/layer22_attention_weight_max": 32.46365737915039,
      "activations/layer22_attention_weight_min": -25.993162155151367,
      "activations/layer23_attention_weight_max": 38.212520599365234,
      "activations/layer23_attention_weight_min": -25.99200439453125,
      "activations/layer2_attention_weight_max": 29.719512939453125,
      "activations/layer2_attention_weight_min": -29.358951568603516,
      "activations/layer3_attention_weight_max": 83.72293853759766,
      "activations/layer3_attention_weight_min": -86.43428802490234,
      "activations/layer4_attention_weight_max": 92.72820281982422,
      "activations/layer4_attention_weight_min": -89.06859588623047,
      "activations/layer5_attention_weight_max": 72.31634521484375,
      "activations/layer5_attention_weight_min": -76.15669250488281,
      "activations/layer6_attention_weight_max": 50.548954010009766,
      "activations/layer6_attention_weight_min": -51.520145416259766,
      "activations/layer7_attention_weight_max": 71.42481231689453,
      "activations/layer7_attention_weight_min": -67.12511444091797,
      "activations/layer8_attention_weight_max": 43.10811233520508,
      "activations/layer8_attention_weight_min": -44.18291473388672,
      "activations/layer9_attention_weight_max": 44.31525421142578,
      "activations/layer9_attention_weight_min": -48.3111457824707,
      "epoch": 10.3,
      "learning_rate": 8.441401515151514e-05,
      "loss": 2.7721,
      "step": 177300
    },
    {
      "activations/layer0_attention_weight_max": 14.963398933410645,
      "activations/layer0_attention_weight_min": -14.445732116699219,
      "activations/layer10_attention_weight_max": 35.62180709838867,
      "activations/layer10_attention_weight_min": -35.40771484375,
      "activations/layer11_attention_weight_max": 33.18703842163086,
      "activations/layer11_attention_weight_min": -33.210880279541016,
      "activations/layer12_attention_weight_max": 17.491954803466797,
      "activations/layer12_attention_weight_min": -24.616735458374023,
      "activations/layer13_attention_weight_max": 38.46179962158203,
      "activations/layer13_attention_weight_min": -29.65201759338379,
      "activations/layer14_attention_weight_max": 40.626155853271484,
      "activations/layer14_attention_weight_min": -30.702478408813477,
      "activations/layer15_attention_weight_max": 34.463706970214844,
      "activations/layer15_attention_weight_min": -32.921791076660156,
      "activations/layer16_attention_weight_max": 33.11408615112305,
      "activations/layer16_attention_weight_min": -29.978227615356445,
      "activations/layer17_attention_weight_max": 50.7208366394043,
      "activations/layer17_attention_weight_min": -44.323978424072266,
      "activations/layer18_attention_weight_max": 46.605865478515625,
      "activations/layer18_attention_weight_min": -37.40425109863281,
      "activations/layer19_attention_weight_max": 20.96219253540039,
      "activations/layer19_attention_weight_min": -21.739233016967773,
      "activations/layer1_attention_weight_max": 17.13646697998047,
      "activations/layer1_attention_weight_min": -16.271814346313477,
      "activations/layer20_attention_weight_max": 23.276227951049805,
      "activations/layer20_attention_weight_min": -21.84305763244629,
      "activations/layer21_attention_weight_max": 44.05465316772461,
      "activations/layer21_attention_weight_min": -24.541677474975586,
      "activations/layer22_attention_weight_max": 31.827518463134766,
      "activations/layer22_attention_weight_min": -28.805885314941406,
      "activations/layer23_attention_weight_max": 35.850921630859375,
      "activations/layer23_attention_weight_min": -24.540729522705078,
      "activations/layer2_attention_weight_max": 32.40715408325195,
      "activations/layer2_attention_weight_min": -29.362545013427734,
      "activations/layer3_attention_weight_max": 92.5026626586914,
      "activations/layer3_attention_weight_min": -92.8017807006836,
      "activations/layer4_attention_weight_max": 96.58297729492188,
      "activations/layer4_attention_weight_min": -92.24504852294922,
      "activations/layer5_attention_weight_max": 74.90959930419922,
      "activations/layer5_attention_weight_min": -77.59943389892578,
      "activations/layer6_attention_weight_max": 53.27664566040039,
      "activations/layer6_attention_weight_min": -52.726234436035156,
      "activations/layer7_attention_weight_max": 71.60919189453125,
      "activations/layer7_attention_weight_min": -72.3636474609375,
      "activations/layer8_attention_weight_max": 44.131900787353516,
      "activations/layer8_attention_weight_min": -47.73332977294922,
      "activations/layer9_attention_weight_max": 46.86112594604492,
      "activations/layer9_attention_weight_min": -46.83299255371094,
      "epoch": 10.31,
      "learning_rate": 8.439545454545453e-05,
      "loss": 2.7803,
      "step": 177350
    },
    {
      "activations/layer0_attention_weight_max": 15.669242858886719,
      "activations/layer0_attention_weight_min": -14.62796401977539,
      "activations/layer10_attention_weight_max": 38.603912353515625,
      "activations/layer10_attention_weight_min": -40.7341423034668,
      "activations/layer11_attention_weight_max": 34.52116012573242,
      "activations/layer11_attention_weight_min": -32.82958221435547,
      "activations/layer12_attention_weight_max": 23.39126205444336,
      "activations/layer12_attention_weight_min": -26.234094619750977,
      "activations/layer13_attention_weight_max": 61.73443603515625,
      "activations/layer13_attention_weight_min": -35.759281158447266,
      "activations/layer14_attention_weight_max": 49.688262939453125,
      "activations/layer14_attention_weight_min": -29.967607498168945,
      "activations/layer15_attention_weight_max": 36.96577835083008,
      "activations/layer15_attention_weight_min": -28.377885818481445,
      "activations/layer16_attention_weight_max": 33.001461029052734,
      "activations/layer16_attention_weight_min": -27.761301040649414,
      "activations/layer17_attention_weight_max": 52.94485092163086,
      "activations/layer17_attention_weight_min": -46.46495056152344,
      "activations/layer18_attention_weight_max": 46.77159118652344,
      "activations/layer18_attention_weight_min": -40.34252166748047,
      "activations/layer19_attention_weight_max": 21.87651824951172,
      "activations/layer19_attention_weight_min": -22.287654876708984,
      "activations/layer1_attention_weight_max": 17.60089683532715,
      "activations/layer1_attention_weight_min": -16.162141799926758,
      "activations/layer20_attention_weight_max": 24.19042205810547,
      "activations/layer20_attention_weight_min": -24.71533203125,
      "activations/layer21_attention_weight_max": 38.88307571411133,
      "activations/layer21_attention_weight_min": -25.041606903076172,
      "activations/layer22_attention_weight_max": 30.85015869140625,
      "activations/layer22_attention_weight_min": -26.932472229003906,
      "activations/layer23_attention_weight_max": 34.631744384765625,
      "activations/layer23_attention_weight_min": -23.147449493408203,
      "activations/layer2_attention_weight_max": 32.925811767578125,
      "activations/layer2_attention_weight_min": -31.077007293701172,
      "activations/layer3_attention_weight_max": 88.75322723388672,
      "activations/layer3_attention_weight_min": -95.92323303222656,
      "activations/layer4_attention_weight_max": 99.18956756591797,
      "activations/layer4_attention_weight_min": -96.64778900146484,
      "activations/layer5_attention_weight_max": 74.96749114990234,
      "activations/layer5_attention_weight_min": -77.30728149414062,
      "activations/layer6_attention_weight_max": 52.42080307006836,
      "activations/layer6_attention_weight_min": -50.1522331237793,
      "activations/layer7_attention_weight_max": 76.04779815673828,
      "activations/layer7_attention_weight_min": -74.18995666503906,
      "activations/layer8_attention_weight_max": 46.67230987548828,
      "activations/layer8_attention_weight_min": -47.725746154785156,
      "activations/layer9_attention_weight_max": 56.97755813598633,
      "activations/layer9_attention_weight_min": -53.566280364990234,
      "epoch": 10.31,
      "learning_rate": 8.437651515151515e-05,
      "loss": 2.7953,
      "step": 177400
    },
    {
      "activations/layer0_attention_weight_max": 15.85486888885498,
      "activations/layer0_attention_weight_min": -14.421212196350098,
      "activations/layer10_attention_weight_max": 36.08473205566406,
      "activations/layer10_attention_weight_min": -35.4916877746582,
      "activations/layer11_attention_weight_max": 31.64937973022461,
      "activations/layer11_attention_weight_min": -32.68414306640625,
      "activations/layer12_attention_weight_max": 17.068893432617188,
      "activations/layer12_attention_weight_min": -26.88231086730957,
      "activations/layer13_attention_weight_max": 35.56783676147461,
      "activations/layer13_attention_weight_min": -29.931621551513672,
      "activations/layer14_attention_weight_max": 33.499488830566406,
      "activations/layer14_attention_weight_min": -31.411174774169922,
      "activations/layer15_attention_weight_max": 31.644392013549805,
      "activations/layer15_attention_weight_min": -30.159786224365234,
      "activations/layer16_attention_weight_max": 31.807085037231445,
      "activations/layer16_attention_weight_min": -29.08409309387207,
      "activations/layer17_attention_weight_max": 50.42276382446289,
      "activations/layer17_attention_weight_min": -42.47758865356445,
      "activations/layer18_attention_weight_max": 44.34681701660156,
      "activations/layer18_attention_weight_min": -37.53929138183594,
      "activations/layer19_attention_weight_max": 22.396854400634766,
      "activations/layer19_attention_weight_min": -21.045991897583008,
      "activations/layer1_attention_weight_max": 16.019821166992188,
      "activations/layer1_attention_weight_min": -16.87546730041504,
      "activations/layer20_attention_weight_max": 21.908245086669922,
      "activations/layer20_attention_weight_min": -21.782480239868164,
      "activations/layer21_attention_weight_max": 39.25641632080078,
      "activations/layer21_attention_weight_min": -25.588693618774414,
      "activations/layer22_attention_weight_max": 32.045005798339844,
      "activations/layer22_attention_weight_min": -26.14230728149414,
      "activations/layer23_attention_weight_max": 39.11908721923828,
      "activations/layer23_attention_weight_min": -24.939199447631836,
      "activations/layer2_attention_weight_max": 29.998397827148438,
      "activations/layer2_attention_weight_min": -29.073410034179688,
      "activations/layer3_attention_weight_max": 87.26608276367188,
      "activations/layer3_attention_weight_min": -89.69536590576172,
      "activations/layer4_attention_weight_max": 97.39290618896484,
      "activations/layer4_attention_weight_min": -92.63713836669922,
      "activations/layer5_attention_weight_max": 74.65320587158203,
      "activations/layer5_attention_weight_min": -84.22862243652344,
      "activations/layer6_attention_weight_max": 52.52733612060547,
      "activations/layer6_attention_weight_min": -52.74263381958008,
      "activations/layer7_attention_weight_max": 73.15248107910156,
      "activations/layer7_attention_weight_min": -71.18367004394531,
      "activations/layer8_attention_weight_max": 44.84914016723633,
      "activations/layer8_attention_weight_min": -45.38760757446289,
      "activations/layer9_attention_weight_max": 46.29661178588867,
      "activations/layer9_attention_weight_min": -51.95216751098633,
      "epoch": 10.31,
      "learning_rate": 8.435757575757576e-05,
      "loss": 2.8074,
      "step": 177450
    },
    {
      "activations/layer0_attention_weight_max": 17.940326690673828,
      "activations/layer0_attention_weight_min": -14.427018165588379,
      "activations/layer10_attention_weight_max": 43.86972427368164,
      "activations/layer10_attention_weight_min": -41.514400482177734,
      "activations/layer11_attention_weight_max": 42.00368881225586,
      "activations/layer11_attention_weight_min": -35.54610824584961,
      "activations/layer12_attention_weight_max": 22.19738006591797,
      "activations/layer12_attention_weight_min": -26.39228630065918,
      "activations/layer13_attention_weight_max": 42.85646057128906,
      "activations/layer13_attention_weight_min": -30.82132339477539,
      "activations/layer14_attention_weight_max": 54.73406219482422,
      "activations/layer14_attention_weight_min": -31.6502685546875,
      "activations/layer15_attention_weight_max": 44.191184997558594,
      "activations/layer15_attention_weight_min": -29.087726593017578,
      "activations/layer16_attention_weight_max": 34.84379196166992,
      "activations/layer16_attention_weight_min": -30.800443649291992,
      "activations/layer17_attention_weight_max": 59.3589973449707,
      "activations/layer17_attention_weight_min": -47.68622970581055,
      "activations/layer18_attention_weight_max": 49.11217498779297,
      "activations/layer18_attention_weight_min": -39.0594482421875,
      "activations/layer19_attention_weight_max": 23.23279571533203,
      "activations/layer19_attention_weight_min": -22.26154136657715,
      "activations/layer1_attention_weight_max": 17.1321964263916,
      "activations/layer1_attention_weight_min": -19.673593521118164,
      "activations/layer20_attention_weight_max": 25.080644607543945,
      "activations/layer20_attention_weight_min": -19.939708709716797,
      "activations/layer21_attention_weight_max": 47.599273681640625,
      "activations/layer21_attention_weight_min": -26.747861862182617,
      "activations/layer22_attention_weight_max": 33.863853454589844,
      "activations/layer22_attention_weight_min": -26.026941299438477,
      "activations/layer23_attention_weight_max": 40.18034744262695,
      "activations/layer23_attention_weight_min": -26.627744674682617,
      "activations/layer2_attention_weight_max": 31.255962371826172,
      "activations/layer2_attention_weight_min": -31.448341369628906,
      "activations/layer3_attention_weight_max": 93.846435546875,
      "activations/layer3_attention_weight_min": -93.43419647216797,
      "activations/layer4_attention_weight_max": 103.2542495727539,
      "activations/layer4_attention_weight_min": -101.33110809326172,
      "activations/layer5_attention_weight_max": 75.24776458740234,
      "activations/layer5_attention_weight_min": -77.06681823730469,
      "activations/layer6_attention_weight_max": 53.0757942199707,
      "activations/layer6_attention_weight_min": -52.04849624633789,
      "activations/layer7_attention_weight_max": 93.1537094116211,
      "activations/layer7_attention_weight_min": -86.529296875,
      "activations/layer8_attention_weight_max": 53.99547576904297,
      "activations/layer8_attention_weight_min": -52.055938720703125,
      "activations/layer9_attention_weight_max": 64.8489761352539,
      "activations/layer9_attention_weight_min": -55.785152435302734,
      "epoch": 10.31,
      "learning_rate": 8.433863636363635e-05,
      "loss": 2.792,
      "step": 177500
    },
    {
      "activations/layer0_attention_weight_max": 15.04147720336914,
      "activations/layer0_attention_weight_min": -14.264261245727539,
      "activations/layer10_attention_weight_max": 38.147621154785156,
      "activations/layer10_attention_weight_min": -36.384796142578125,
      "activations/layer11_attention_weight_max": 35.410465240478516,
      "activations/layer11_attention_weight_min": -32.16465377807617,
      "activations/layer12_attention_weight_max": 22.539819717407227,
      "activations/layer12_attention_weight_min": -27.895740509033203,
      "activations/layer13_attention_weight_max": 41.500301361083984,
      "activations/layer13_attention_weight_min": -33.810630798339844,
      "activations/layer14_attention_weight_max": 37.04309844970703,
      "activations/layer14_attention_weight_min": -32.855411529541016,
      "activations/layer15_attention_weight_max": 34.984527587890625,
      "activations/layer15_attention_weight_min": -33.20305633544922,
      "activations/layer16_attention_weight_max": 31.687116622924805,
      "activations/layer16_attention_weight_min": -30.395593643188477,
      "activations/layer17_attention_weight_max": 52.25861358642578,
      "activations/layer17_attention_weight_min": -46.23758316040039,
      "activations/layer18_attention_weight_max": 43.670101165771484,
      "activations/layer18_attention_weight_min": -36.49639129638672,
      "activations/layer19_attention_weight_max": 22.081457138061523,
      "activations/layer19_attention_weight_min": -21.689390182495117,
      "activations/layer1_attention_weight_max": 16.814403533935547,
      "activations/layer1_attention_weight_min": -15.7449951171875,
      "activations/layer20_attention_weight_max": 22.084125518798828,
      "activations/layer20_attention_weight_min": -21.038284301757812,
      "activations/layer21_attention_weight_max": 33.64494705200195,
      "activations/layer21_attention_weight_min": -26.11299705505371,
      "activations/layer22_attention_weight_max": 28.73541831970215,
      "activations/layer22_attention_weight_min": -27.237459182739258,
      "activations/layer23_attention_weight_max": 36.47468185424805,
      "activations/layer23_attention_weight_min": -22.659427642822266,
      "activations/layer2_attention_weight_max": 30.77387809753418,
      "activations/layer2_attention_weight_min": -30.155744552612305,
      "activations/layer3_attention_weight_max": 89.36713409423828,
      "activations/layer3_attention_weight_min": -90.35082244873047,
      "activations/layer4_attention_weight_max": 94.84809112548828,
      "activations/layer4_attention_weight_min": -89.18232727050781,
      "activations/layer5_attention_weight_max": 78.6843032836914,
      "activations/layer5_attention_weight_min": -76.63917541503906,
      "activations/layer6_attention_weight_max": 52.073524475097656,
      "activations/layer6_attention_weight_min": -51.4291877746582,
      "activations/layer7_attention_weight_max": 74.98877716064453,
      "activations/layer7_attention_weight_min": -76.6622543334961,
      "activations/layer8_attention_weight_max": 46.88488006591797,
      "activations/layer8_attention_weight_min": -47.56243133544922,
      "activations/layer9_attention_weight_max": 50.2357292175293,
      "activations/layer9_attention_weight_min": -50.1523323059082,
      "epoch": 10.32,
      "learning_rate": 8.431969696969697e-05,
      "loss": 2.7954,
      "step": 177550
    },
    {
      "activations/layer0_attention_weight_max": 15.684735298156738,
      "activations/layer0_attention_weight_min": -14.37027359008789,
      "activations/layer10_attention_weight_max": 39.9106330871582,
      "activations/layer10_attention_weight_min": -37.159515380859375,
      "activations/layer11_attention_weight_max": 38.22370529174805,
      "activations/layer11_attention_weight_min": -33.269073486328125,
      "activations/layer12_attention_weight_max": 17.810070037841797,
      "activations/layer12_attention_weight_min": -23.770442962646484,
      "activations/layer13_attention_weight_max": 43.567298889160156,
      "activations/layer13_attention_weight_min": -31.10428810119629,
      "activations/layer14_attention_weight_max": 47.18330764770508,
      "activations/layer14_attention_weight_min": -32.99597930908203,
      "activations/layer15_attention_weight_max": 44.33545684814453,
      "activations/layer15_attention_weight_min": -30.199909210205078,
      "activations/layer16_attention_weight_max": 29.497642517089844,
      "activations/layer16_attention_weight_min": -29.046632766723633,
      "activations/layer17_attention_weight_max": 51.924129486083984,
      "activations/layer17_attention_weight_min": -44.00517654418945,
      "activations/layer18_attention_weight_max": 46.96315002441406,
      "activations/layer18_attention_weight_min": -39.228607177734375,
      "activations/layer19_attention_weight_max": 22.701154708862305,
      "activations/layer19_attention_weight_min": -23.380834579467773,
      "activations/layer1_attention_weight_max": 17.250102996826172,
      "activations/layer1_attention_weight_min": -15.049127578735352,
      "activations/layer20_attention_weight_max": 22.161449432373047,
      "activations/layer20_attention_weight_min": -24.654611587524414,
      "activations/layer21_attention_weight_max": 41.17460250854492,
      "activations/layer21_attention_weight_min": -28.349655151367188,
      "activations/layer22_attention_weight_max": 28.193960189819336,
      "activations/layer22_attention_weight_min": -27.429515838623047,
      "activations/layer23_attention_weight_max": 36.46531295776367,
      "activations/layer23_attention_weight_min": -24.97602653503418,
      "activations/layer2_attention_weight_max": 32.69900131225586,
      "activations/layer2_attention_weight_min": -30.127891540527344,
      "activations/layer3_attention_weight_max": 93.85179138183594,
      "activations/layer3_attention_weight_min": -93.93487548828125,
      "activations/layer4_attention_weight_max": 105.069091796875,
      "activations/layer4_attention_weight_min": -104.7551498413086,
      "activations/layer5_attention_weight_max": 78.77876281738281,
      "activations/layer5_attention_weight_min": -77.92562103271484,
      "activations/layer6_attention_weight_max": 58.140960693359375,
      "activations/layer6_attention_weight_min": -56.55160903930664,
      "activations/layer7_attention_weight_max": 78.48653411865234,
      "activations/layer7_attention_weight_min": -74.30238342285156,
      "activations/layer8_attention_weight_max": 47.493629455566406,
      "activations/layer8_attention_weight_min": -49.296443939208984,
      "activations/layer9_attention_weight_max": 59.52179718017578,
      "activations/layer9_attention_weight_min": -52.17780303955078,
      "epoch": 10.32,
      "learning_rate": 8.430075757575755e-05,
      "loss": 2.7943,
      "step": 177600
    },
    {
      "activations/layer0_attention_weight_max": 14.76517391204834,
      "activations/layer0_attention_weight_min": -14.326579093933105,
      "activations/layer10_attention_weight_max": 36.66642761230469,
      "activations/layer10_attention_weight_min": -34.91002655029297,
      "activations/layer11_attention_weight_max": 31.603538513183594,
      "activations/layer11_attention_weight_min": -33.07041931152344,
      "activations/layer12_attention_weight_max": 18.081003189086914,
      "activations/layer12_attention_weight_min": -24.59941291809082,
      "activations/layer13_attention_weight_max": 33.36154556274414,
      "activations/layer13_attention_weight_min": -29.711162567138672,
      "activations/layer14_attention_weight_max": 37.23324203491211,
      "activations/layer14_attention_weight_min": -32.47027587890625,
      "activations/layer15_attention_weight_max": 32.48617172241211,
      "activations/layer15_attention_weight_min": -29.42337989807129,
      "activations/layer16_attention_weight_max": 30.49340057373047,
      "activations/layer16_attention_weight_min": -29.777244567871094,
      "activations/layer17_attention_weight_max": 53.4677848815918,
      "activations/layer17_attention_weight_min": -44.413475036621094,
      "activations/layer18_attention_weight_max": 47.87398147583008,
      "activations/layer18_attention_weight_min": -36.91779708862305,
      "activations/layer19_attention_weight_max": 21.328506469726562,
      "activations/layer19_attention_weight_min": -22.720876693725586,
      "activations/layer1_attention_weight_max": 16.858728408813477,
      "activations/layer1_attention_weight_min": -14.855867385864258,
      "activations/layer20_attention_weight_max": 24.517925262451172,
      "activations/layer20_attention_weight_min": -21.16710090637207,
      "activations/layer21_attention_weight_max": 41.28057098388672,
      "activations/layer21_attention_weight_min": -24.16640281677246,
      "activations/layer22_attention_weight_max": 33.737388610839844,
      "activations/layer22_attention_weight_min": -25.05044174194336,
      "activations/layer23_attention_weight_max": 47.07525634765625,
      "activations/layer23_attention_weight_min": -25.185638427734375,
      "activations/layer2_attention_weight_max": 31.11524772644043,
      "activations/layer2_attention_weight_min": -30.28858184814453,
      "activations/layer3_attention_weight_max": 94.3392562866211,
      "activations/layer3_attention_weight_min": -94.67897033691406,
      "activations/layer4_attention_weight_max": 103.40541076660156,
      "activations/layer4_attention_weight_min": -94.88658142089844,
      "activations/layer5_attention_weight_max": 77.83384704589844,
      "activations/layer5_attention_weight_min": -77.23968505859375,
      "activations/layer6_attention_weight_max": 55.39014434814453,
      "activations/layer6_attention_weight_min": -55.553123474121094,
      "activations/layer7_attention_weight_max": 80.33482360839844,
      "activations/layer7_attention_weight_min": -76.65338134765625,
      "activations/layer8_attention_weight_max": 45.01031494140625,
      "activations/layer8_attention_weight_min": -45.18818664550781,
      "activations/layer9_attention_weight_max": 46.43581771850586,
      "activations/layer9_attention_weight_min": -45.103355407714844,
      "epoch": 10.32,
      "learning_rate": 8.428181818181817e-05,
      "loss": 2.7981,
      "step": 177650
    },
    {
      "activations/layer0_attention_weight_max": 14.48715877532959,
      "activations/layer0_attention_weight_min": -14.30663013458252,
      "activations/layer10_attention_weight_max": 44.59946060180664,
      "activations/layer10_attention_weight_min": -42.252803802490234,
      "activations/layer11_attention_weight_max": 37.8633918762207,
      "activations/layer11_attention_weight_min": -37.45986557006836,
      "activations/layer12_attention_weight_max": 20.020339965820312,
      "activations/layer12_attention_weight_min": -26.599946975708008,
      "activations/layer13_attention_weight_max": 43.20706558227539,
      "activations/layer13_attention_weight_min": -30.927024841308594,
      "activations/layer14_attention_weight_max": 53.264617919921875,
      "activations/layer14_attention_weight_min": -33.462928771972656,
      "activations/layer15_attention_weight_max": 46.939876556396484,
      "activations/layer15_attention_weight_min": -29.92559242248535,
      "activations/layer16_attention_weight_max": 32.74985885620117,
      "activations/layer16_attention_weight_min": -28.558486938476562,
      "activations/layer17_attention_weight_max": 54.07246398925781,
      "activations/layer17_attention_weight_min": -44.84291076660156,
      "activations/layer18_attention_weight_max": 51.10711669921875,
      "activations/layer18_attention_weight_min": -42.50786590576172,
      "activations/layer19_attention_weight_max": 23.99913787841797,
      "activations/layer19_attention_weight_min": -23.54708480834961,
      "activations/layer1_attention_weight_max": 16.60540771484375,
      "activations/layer1_attention_weight_min": -18.356157302856445,
      "activations/layer20_attention_weight_max": 24.176183700561523,
      "activations/layer20_attention_weight_min": -22.923803329467773,
      "activations/layer21_attention_weight_max": 36.241371154785156,
      "activations/layer21_attention_weight_min": -22.349939346313477,
      "activations/layer22_attention_weight_max": 34.44927215576172,
      "activations/layer22_attention_weight_min": -25.843246459960938,
      "activations/layer23_attention_weight_max": 39.78544998168945,
      "activations/layer23_attention_weight_min": -22.255470275878906,
      "activations/layer2_attention_weight_max": 34.3312873840332,
      "activations/layer2_attention_weight_min": -32.60856628417969,
      "activations/layer3_attention_weight_max": 97.30734252929688,
      "activations/layer3_attention_weight_min": -96.45419311523438,
      "activations/layer4_attention_weight_max": 106.76661682128906,
      "activations/layer4_attention_weight_min": -100.89579772949219,
      "activations/layer5_attention_weight_max": 81.45427703857422,
      "activations/layer5_attention_weight_min": -77.25909423828125,
      "activations/layer6_attention_weight_max": 60.94427490234375,
      "activations/layer6_attention_weight_min": -56.413368225097656,
      "activations/layer7_attention_weight_max": 83.01807403564453,
      "activations/layer7_attention_weight_min": -81.53349304199219,
      "activations/layer8_attention_weight_max": 53.42483139038086,
      "activations/layer8_attention_weight_min": -51.09431457519531,
      "activations/layer9_attention_weight_max": 61.310638427734375,
      "activations/layer9_attention_weight_min": -55.77976608276367,
      "epoch": 10.33,
      "learning_rate": 8.426287878787878e-05,
      "loss": 2.7785,
      "step": 177700
    },
    {
      "activations/layer0_attention_weight_max": 14.510818481445312,
      "activations/layer0_attention_weight_min": -13.787413597106934,
      "activations/layer10_attention_weight_max": 36.67353057861328,
      "activations/layer10_attention_weight_min": -35.589534759521484,
      "activations/layer11_attention_weight_max": 32.05514907836914,
      "activations/layer11_attention_weight_min": -33.465415954589844,
      "activations/layer12_attention_weight_max": 20.67222785949707,
      "activations/layer12_attention_weight_min": -23.238481521606445,
      "activations/layer13_attention_weight_max": 52.32002258300781,
      "activations/layer13_attention_weight_min": -30.325439453125,
      "activations/layer14_attention_weight_max": 45.867095947265625,
      "activations/layer14_attention_weight_min": -32.004024505615234,
      "activations/layer15_attention_weight_max": 37.56488037109375,
      "activations/layer15_attention_weight_min": -30.833425521850586,
      "activations/layer16_attention_weight_max": 33.601741790771484,
      "activations/layer16_attention_weight_min": -31.08121681213379,
      "activations/layer17_attention_weight_max": 58.68821334838867,
      "activations/layer17_attention_weight_min": -45.97370529174805,
      "activations/layer18_attention_weight_max": 57.45766067504883,
      "activations/layer18_attention_weight_min": -41.50339889526367,
      "activations/layer19_attention_weight_max": 24.513282775878906,
      "activations/layer19_attention_weight_min": -24.217988967895508,
      "activations/layer1_attention_weight_max": 16.769264221191406,
      "activations/layer1_attention_weight_min": -15.942036628723145,
      "activations/layer20_attention_weight_max": 27.71633529663086,
      "activations/layer20_attention_weight_min": -20.66947364807129,
      "activations/layer21_attention_weight_max": 41.58369445800781,
      "activations/layer21_attention_weight_min": -22.962556838989258,
      "activations/layer22_attention_weight_max": 39.600128173828125,
      "activations/layer22_attention_weight_min": -26.234365463256836,
      "activations/layer23_attention_weight_max": 41.71209716796875,
      "activations/layer23_attention_weight_min": -21.727554321289062,
      "activations/layer2_attention_weight_max": 33.15938949584961,
      "activations/layer2_attention_weight_min": -31.41707420349121,
      "activations/layer3_attention_weight_max": 89.64704895019531,
      "activations/layer3_attention_weight_min": -91.08950805664062,
      "activations/layer4_attention_weight_max": 99.62484741210938,
      "activations/layer4_attention_weight_min": -95.82169342041016,
      "activations/layer5_attention_weight_max": 76.83116912841797,
      "activations/layer5_attention_weight_min": -80.74467468261719,
      "activations/layer6_attention_weight_max": 54.11396026611328,
      "activations/layer6_attention_weight_min": -56.1624641418457,
      "activations/layer7_attention_weight_max": 76.56275177001953,
      "activations/layer7_attention_weight_min": -72.79280090332031,
      "activations/layer8_attention_weight_max": 46.034629821777344,
      "activations/layer8_attention_weight_min": -48.681888580322266,
      "activations/layer9_attention_weight_max": 50.184722900390625,
      "activations/layer9_attention_weight_min": -48.84450149536133,
      "epoch": 10.33,
      "learning_rate": 8.424393939393939e-05,
      "loss": 2.7779,
      "step": 177750
    },
    {
      "activations/layer0_attention_weight_max": 16.081010818481445,
      "activations/layer0_attention_weight_min": -14.617900848388672,
      "activations/layer10_attention_weight_max": 41.5716667175293,
      "activations/layer10_attention_weight_min": -37.96599578857422,
      "activations/layer11_attention_weight_max": 36.99884796142578,
      "activations/layer11_attention_weight_min": -33.23099136352539,
      "activations/layer12_attention_weight_max": 20.236677169799805,
      "activations/layer12_attention_weight_min": -30.034442901611328,
      "activations/layer13_attention_weight_max": 49.17344665527344,
      "activations/layer13_attention_weight_min": -31.615083694458008,
      "activations/layer14_attention_weight_max": 52.14232635498047,
      "activations/layer14_attention_weight_min": -30.547636032104492,
      "activations/layer15_attention_weight_max": 45.62766647338867,
      "activations/layer15_attention_weight_min": -28.05402946472168,
      "activations/layer16_attention_weight_max": 32.37094497680664,
      "activations/layer16_attention_weight_min": -28.095632553100586,
      "activations/layer17_attention_weight_max": 55.43797302246094,
      "activations/layer17_attention_weight_min": -42.90019607543945,
      "activations/layer18_attention_weight_max": 49.10911178588867,
      "activations/layer18_attention_weight_min": -38.21123123168945,
      "activations/layer19_attention_weight_max": 22.101099014282227,
      "activations/layer19_attention_weight_min": -21.405040740966797,
      "activations/layer1_attention_weight_max": 16.961795806884766,
      "activations/layer1_attention_weight_min": -18.185577392578125,
      "activations/layer20_attention_weight_max": 24.561450958251953,
      "activations/layer20_attention_weight_min": -20.907817840576172,
      "activations/layer21_attention_weight_max": 44.26210021972656,
      "activations/layer21_attention_weight_min": -23.71708106994629,
      "activations/layer22_attention_weight_max": 33.81938552856445,
      "activations/layer22_attention_weight_min": -23.62420654296875,
      "activations/layer23_attention_weight_max": 37.88240432739258,
      "activations/layer23_attention_weight_min": -24.14297103881836,
      "activations/layer2_attention_weight_max": 35.734493255615234,
      "activations/layer2_attention_weight_min": -33.17875671386719,
      "activations/layer3_attention_weight_max": 98.9343490600586,
      "activations/layer3_attention_weight_min": -96.33955383300781,
      "activations/layer4_attention_weight_max": 100.80812072753906,
      "activations/layer4_attention_weight_min": -96.23986053466797,
      "activations/layer5_attention_weight_max": 76.91075134277344,
      "activations/layer5_attention_weight_min": -81.2450942993164,
      "activations/layer6_attention_weight_max": 57.05869674682617,
      "activations/layer6_attention_weight_min": -58.47629165649414,
      "activations/layer7_attention_weight_max": 81.03063201904297,
      "activations/layer7_attention_weight_min": -76.99310302734375,
      "activations/layer8_attention_weight_max": 49.35110855102539,
      "activations/layer8_attention_weight_min": -49.884361267089844,
      "activations/layer9_attention_weight_max": 56.55817794799805,
      "activations/layer9_attention_weight_min": -57.68387985229492,
      "epoch": 10.33,
      "learning_rate": 8.422499999999999e-05,
      "loss": 2.7764,
      "step": 177800
    },
    {
      "activations/layer0_attention_weight_max": 14.727795600891113,
      "activations/layer0_attention_weight_min": -14.16807746887207,
      "activations/layer10_attention_weight_max": 34.631591796875,
      "activations/layer10_attention_weight_min": -35.04443359375,
      "activations/layer11_attention_weight_max": 30.574844360351562,
      "activations/layer11_attention_weight_min": -31.984508514404297,
      "activations/layer12_attention_weight_max": 18.46103858947754,
      "activations/layer12_attention_weight_min": -26.5517635345459,
      "activations/layer13_attention_weight_max": 38.44923400878906,
      "activations/layer13_attention_weight_min": -29.102806091308594,
      "activations/layer14_attention_weight_max": 37.788902282714844,
      "activations/layer14_attention_weight_min": -28.247699737548828,
      "activations/layer15_attention_weight_max": 35.15345764160156,
      "activations/layer15_attention_weight_min": -26.748584747314453,
      "activations/layer16_attention_weight_max": 34.65622329711914,
      "activations/layer16_attention_weight_min": -28.05107879638672,
      "activations/layer17_attention_weight_max": 52.414772033691406,
      "activations/layer17_attention_weight_min": -39.906890869140625,
      "activations/layer18_attention_weight_max": 48.98853302001953,
      "activations/layer18_attention_weight_min": -34.4004020690918,
      "activations/layer19_attention_weight_max": 21.7036190032959,
      "activations/layer19_attention_weight_min": -20.485549926757812,
      "activations/layer1_attention_weight_max": 17.998104095458984,
      "activations/layer1_attention_weight_min": -14.226619720458984,
      "activations/layer20_attention_weight_max": 24.61482810974121,
      "activations/layer20_attention_weight_min": -21.54402732849121,
      "activations/layer21_attention_weight_max": 38.147926330566406,
      "activations/layer21_attention_weight_min": -22.294097900390625,
      "activations/layer22_attention_weight_max": 30.82899284362793,
      "activations/layer22_attention_weight_min": -29.652538299560547,
      "activations/layer23_attention_weight_max": 37.78892135620117,
      "activations/layer23_attention_weight_min": -27.067981719970703,
      "activations/layer2_attention_weight_max": 31.929019927978516,
      "activations/layer2_attention_weight_min": -29.412845611572266,
      "activations/layer3_attention_weight_max": 93.38223266601562,
      "activations/layer3_attention_weight_min": -90.20274353027344,
      "activations/layer4_attention_weight_max": 97.99632263183594,
      "activations/layer4_attention_weight_min": -95.05378723144531,
      "activations/layer5_attention_weight_max": 74.25814819335938,
      "activations/layer5_attention_weight_min": -74.7073974609375,
      "activations/layer6_attention_weight_max": 54.70930099487305,
      "activations/layer6_attention_weight_min": -51.24729919433594,
      "activations/layer7_attention_weight_max": 71.3331298828125,
      "activations/layer7_attention_weight_min": -70.42113494873047,
      "activations/layer8_attention_weight_max": 43.897586822509766,
      "activations/layer8_attention_weight_min": -46.10598373413086,
      "activations/layer9_attention_weight_max": 46.1367301940918,
      "activations/layer9_attention_weight_min": -46.43417739868164,
      "epoch": 10.33,
      "learning_rate": 8.42060606060606e-05,
      "loss": 2.7961,
      "step": 177850
    },
    {
      "activations/layer0_attention_weight_max": 15.083106994628906,
      "activations/layer0_attention_weight_min": -14.105318069458008,
      "activations/layer10_attention_weight_max": 37.37776565551758,
      "activations/layer10_attention_weight_min": -36.334659576416016,
      "activations/layer11_attention_weight_max": 33.2908821105957,
      "activations/layer11_attention_weight_min": -32.90330505371094,
      "activations/layer12_attention_weight_max": 19.978784561157227,
      "activations/layer12_attention_weight_min": -23.82583999633789,
      "activations/layer13_attention_weight_max": 38.1594352722168,
      "activations/layer13_attention_weight_min": -32.69000244140625,
      "activations/layer14_attention_weight_max": 38.07273483276367,
      "activations/layer14_attention_weight_min": -31.65220832824707,
      "activations/layer15_attention_weight_max": 33.02246856689453,
      "activations/layer15_attention_weight_min": -30.083229064941406,
      "activations/layer16_attention_weight_max": 31.21869659423828,
      "activations/layer16_attention_weight_min": -29.278793334960938,
      "activations/layer17_attention_weight_max": 54.49443817138672,
      "activations/layer17_attention_weight_min": -48.24907302856445,
      "activations/layer18_attention_weight_max": 48.866207122802734,
      "activations/layer18_attention_weight_min": -38.2459716796875,
      "activations/layer19_attention_weight_max": 21.59618377685547,
      "activations/layer19_attention_weight_min": -21.380935668945312,
      "activations/layer1_attention_weight_max": 15.979227066040039,
      "activations/layer1_attention_weight_min": -14.614786148071289,
      "activations/layer20_attention_weight_max": 23.7000675201416,
      "activations/layer20_attention_weight_min": -24.778697967529297,
      "activations/layer21_attention_weight_max": 39.22054672241211,
      "activations/layer21_attention_weight_min": -22.580137252807617,
      "activations/layer22_attention_weight_max": 34.70051956176758,
      "activations/layer22_attention_weight_min": -25.88427734375,
      "activations/layer23_attention_weight_max": 38.26351547241211,
      "activations/layer23_attention_weight_min": -22.300519943237305,
      "activations/layer2_attention_weight_max": 34.908424377441406,
      "activations/layer2_attention_weight_min": -30.397266387939453,
      "activations/layer3_attention_weight_max": 93.39988708496094,
      "activations/layer3_attention_weight_min": -89.8597183227539,
      "activations/layer4_attention_weight_max": 100.35912322998047,
      "activations/layer4_attention_weight_min": -95.6489028930664,
      "activations/layer5_attention_weight_max": 79.19752502441406,
      "activations/layer5_attention_weight_min": -78.88470458984375,
      "activations/layer6_attention_weight_max": 54.858375549316406,
      "activations/layer6_attention_weight_min": -53.48548126220703,
      "activations/layer7_attention_weight_max": 79.80445861816406,
      "activations/layer7_attention_weight_min": -72.16476440429688,
      "activations/layer8_attention_weight_max": 48.42836380004883,
      "activations/layer8_attention_weight_min": -48.8353157043457,
      "activations/layer9_attention_weight_max": 50.30170822143555,
      "activations/layer9_attention_weight_min": -49.08181381225586,
      "epoch": 10.34,
      "learning_rate": 8.41871212121212e-05,
      "loss": 2.7829,
      "step": 177900
    },
    {
      "activations/layer0_attention_weight_max": 15.785124778747559,
      "activations/layer0_attention_weight_min": -14.713277816772461,
      "activations/layer10_attention_weight_max": 37.60606384277344,
      "activations/layer10_attention_weight_min": -34.73637008666992,
      "activations/layer11_attention_weight_max": 32.395851135253906,
      "activations/layer11_attention_weight_min": -31.64011001586914,
      "activations/layer12_attention_weight_max": 23.189252853393555,
      "activations/layer12_attention_weight_min": -24.94150733947754,
      "activations/layer13_attention_weight_max": 42.45697021484375,
      "activations/layer13_attention_weight_min": -30.871109008789062,
      "activations/layer14_attention_weight_max": 39.36171340942383,
      "activations/layer14_attention_weight_min": -29.208723068237305,
      "activations/layer15_attention_weight_max": 35.84379959106445,
      "activations/layer15_attention_weight_min": -29.792299270629883,
      "activations/layer16_attention_weight_max": 31.202709197998047,
      "activations/layer16_attention_weight_min": -28.71220588684082,
      "activations/layer17_attention_weight_max": 57.8966064453125,
      "activations/layer17_attention_weight_min": -43.2042121887207,
      "activations/layer18_attention_weight_max": 50.03157043457031,
      "activations/layer18_attention_weight_min": -37.224300384521484,
      "activations/layer19_attention_weight_max": 23.565561294555664,
      "activations/layer19_attention_weight_min": -23.035797119140625,
      "activations/layer1_attention_weight_max": 16.67921257019043,
      "activations/layer1_attention_weight_min": -16.08736228942871,
      "activations/layer20_attention_weight_max": 23.702404022216797,
      "activations/layer20_attention_weight_min": -22.48807144165039,
      "activations/layer21_attention_weight_max": 34.30730056762695,
      "activations/layer21_attention_weight_min": -20.596473693847656,
      "activations/layer22_attention_weight_max": 29.17125701904297,
      "activations/layer22_attention_weight_min": -25.335269927978516,
      "activations/layer23_attention_weight_max": 37.826873779296875,
      "activations/layer23_attention_weight_min": -21.463071823120117,
      "activations/layer2_attention_weight_max": 32.75106430053711,
      "activations/layer2_attention_weight_min": -30.36385726928711,
      "activations/layer3_attention_weight_max": 93.48313903808594,
      "activations/layer3_attention_weight_min": -92.17253112792969,
      "activations/layer4_attention_weight_max": 99.61261749267578,
      "activations/layer4_attention_weight_min": -94.69075775146484,
      "activations/layer5_attention_weight_max": 76.38069152832031,
      "activations/layer5_attention_weight_min": -76.89580535888672,
      "activations/layer6_attention_weight_max": 55.66791534423828,
      "activations/layer6_attention_weight_min": -56.657203674316406,
      "activations/layer7_attention_weight_max": 72.36241912841797,
      "activations/layer7_attention_weight_min": -75.01671600341797,
      "activations/layer8_attention_weight_max": 45.22712707519531,
      "activations/layer8_attention_weight_min": -46.693504333496094,
      "activations/layer9_attention_weight_max": 49.6283073425293,
      "activations/layer9_attention_weight_min": -50.82023239135742,
      "epoch": 10.34,
      "learning_rate": 8.416818181818182e-05,
      "loss": 2.7863,
      "step": 177950
    },
    {
      "activations/layer0_attention_weight_max": 15.896669387817383,
      "activations/layer0_attention_weight_min": -14.242389678955078,
      "activations/layer10_attention_weight_max": 37.897987365722656,
      "activations/layer10_attention_weight_min": -37.46464157104492,
      "activations/layer11_attention_weight_max": 32.28575134277344,
      "activations/layer11_attention_weight_min": -34.436344146728516,
      "activations/layer12_attention_weight_max": 20.545928955078125,
      "activations/layer12_attention_weight_min": -29.80331802368164,
      "activations/layer13_attention_weight_max": 57.39322280883789,
      "activations/layer13_attention_weight_min": -35.61796188354492,
      "activations/layer14_attention_weight_max": 53.33235168457031,
      "activations/layer14_attention_weight_min": -32.74415969848633,
      "activations/layer15_attention_weight_max": 40.02210235595703,
      "activations/layer15_attention_weight_min": -28.01414680480957,
      "activations/layer16_attention_weight_max": 41.12377166748047,
      "activations/layer16_attention_weight_min": -28.567094802856445,
      "activations/layer17_attention_weight_max": 66.77860260009766,
      "activations/layer17_attention_weight_min": -48.052520751953125,
      "activations/layer18_attention_weight_max": 59.49189376831055,
      "activations/layer18_attention_weight_min": -38.860618591308594,
      "activations/layer19_attention_weight_max": 26.794321060180664,
      "activations/layer19_attention_weight_min": -23.2402400970459,
      "activations/layer1_attention_weight_max": 17.09984016418457,
      "activations/layer1_attention_weight_min": -16.646011352539062,
      "activations/layer20_attention_weight_max": 27.7316837310791,
      "activations/layer20_attention_weight_min": -20.130035400390625,
      "activations/layer21_attention_weight_max": 42.85383224487305,
      "activations/layer21_attention_weight_min": -21.844253540039062,
      "activations/layer22_attention_weight_max": 39.680484771728516,
      "activations/layer22_attention_weight_min": -23.585542678833008,
      "activations/layer23_attention_weight_max": 49.307395935058594,
      "activations/layer23_attention_weight_min": -22.65558624267578,
      "activations/layer2_attention_weight_max": 35.221458435058594,
      "activations/layer2_attention_weight_min": -31.41230010986328,
      "activations/layer3_attention_weight_max": 95.35806274414062,
      "activations/layer3_attention_weight_min": -96.65637969970703,
      "activations/layer4_attention_weight_max": 102.59748840332031,
      "activations/layer4_attention_weight_min": -99.39568328857422,
      "activations/layer5_attention_weight_max": 80.79322052001953,
      "activations/layer5_attention_weight_min": -77.5733413696289,
      "activations/layer6_attention_weight_max": 53.34572982788086,
      "activations/layer6_attention_weight_min": -54.002540588378906,
      "activations/layer7_attention_weight_max": 74.84880828857422,
      "activations/layer7_attention_weight_min": -77.0608901977539,
      "activations/layer8_attention_weight_max": 45.1694450378418,
      "activations/layer8_attention_weight_min": -47.03470230102539,
      "activations/layer9_attention_weight_max": 59.22147750854492,
      "activations/layer9_attention_weight_min": -50.705379486083984,
      "epoch": 10.34,
      "learning_rate": 8.414924242424242e-05,
      "loss": 2.7881,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4929,
      "eval_samples_per_second": 505.601,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4929,
      "eval_openwebtext_samples_per_second": 505.601,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 1.9626,
      "eval_wikitext_samples_per_second": 232.341,
      "step": 178000
    },
    {
      "epoch": 10.34,
      "eval_lambada_loss": 2.580078125,
      "eval_lambada_ppl": 13.198169226352599,
      "eval_lambada_runtime": 9.5625,
      "eval_lambada_samples_per_second": 509.178,
      "step": 178000
    },
    {
      "activations/layer0_attention_weight_max": 14.938871383666992,
      "activations/layer0_attention_weight_min": -14.118640899658203,
      "activations/layer10_attention_weight_max": 38.45595169067383,
      "activations/layer10_attention_weight_min": -35.3203010559082,
      "activations/layer11_attention_weight_max": 32.84279251098633,
      "activations/layer11_attention_weight_min": -33.81928634643555,
      "activations/layer12_attention_weight_max": 17.958316802978516,
      "activations/layer12_attention_weight_min": -25.16098403930664,
      "activations/layer13_attention_weight_max": 40.247581481933594,
      "activations/layer13_attention_weight_min": -27.924884796142578,
      "activations/layer14_attention_weight_max": 35.64359664916992,
      "activations/layer14_attention_weight_min": -30.458972930908203,
      "activations/layer15_attention_weight_max": 33.03385925292969,
      "activations/layer15_attention_weight_min": -31.250476837158203,
      "activations/layer16_attention_weight_max": 30.2572021484375,
      "activations/layer16_attention_weight_min": -28.71479606628418,
      "activations/layer17_attention_weight_max": 52.175498962402344,
      "activations/layer17_attention_weight_min": -45.38831329345703,
      "activations/layer18_attention_weight_max": 46.81269073486328,
      "activations/layer18_attention_weight_min": -36.59040069580078,
      "activations/layer19_attention_weight_max": 20.804458618164062,
      "activations/layer19_attention_weight_min": -20.481658935546875,
      "activations/layer1_attention_weight_max": 17.844079971313477,
      "activations/layer1_attention_weight_min": -18.279985427856445,
      "activations/layer20_attention_weight_max": 24.70155906677246,
      "activations/layer20_attention_weight_min": -24.538002014160156,
      "activations/layer21_attention_weight_max": 37.017974853515625,
      "activations/layer21_attention_weight_min": -21.240463256835938,
      "activations/layer22_attention_weight_max": 32.083492279052734,
      "activations/layer22_attention_weight_min": -26.176633834838867,
      "activations/layer23_attention_weight_max": 36.064537048339844,
      "activations/layer23_attention_weight_min": -23.75421905517578,
      "activations/layer2_attention_weight_max": 31.899791717529297,
      "activations/layer2_attention_weight_min": -30.377668380737305,
      "activations/layer3_attention_weight_max": 90.23443603515625,
      "activations/layer3_attention_weight_min": -91.9909439086914,
      "activations/layer4_attention_weight_max": 100.4457015991211,
      "activations/layer4_attention_weight_min": -96.56139373779297,
      "activations/layer5_attention_weight_max": 76.83617401123047,
      "activations/layer5_attention_weight_min": -79.53938293457031,
      "activations/layer6_attention_weight_max": 57.09318542480469,
      "activations/layer6_attention_weight_min": -54.20292282104492,
      "activations/layer7_attention_weight_max": 79.77822875976562,
      "activations/layer7_attention_weight_min": -74.38213348388672,
      "activations/layer8_attention_weight_max": 47.60221862792969,
      "activations/layer8_attention_weight_min": -47.10475158691406,
      "activations/layer9_attention_weight_max": 47.403507232666016,
      "activations/layer9_attention_weight_min": -49.15013122558594,
      "epoch": 10.35,
      "learning_rate": 8.413030303030302e-05,
      "loss": 2.7927,
      "step": 178050
    },
    {
      "activations/layer0_attention_weight_max": 15.52512264251709,
      "activations/layer0_attention_weight_min": -13.948505401611328,
      "activations/layer10_attention_weight_max": 37.65796661376953,
      "activations/layer10_attention_weight_min": -35.5556755065918,
      "activations/layer11_attention_weight_max": 31.762022018432617,
      "activations/layer11_attention_weight_min": -32.91905975341797,
      "activations/layer12_attention_weight_max": 19.55830955505371,
      "activations/layer12_attention_weight_min": -22.673072814941406,
      "activations/layer13_attention_weight_max": 39.4498405456543,
      "activations/layer13_attention_weight_min": -29.035127639770508,
      "activations/layer14_attention_weight_max": 38.487728118896484,
      "activations/layer14_attention_weight_min": -29.87241554260254,
      "activations/layer15_attention_weight_max": 34.74658966064453,
      "activations/layer15_attention_weight_min": -27.613719940185547,
      "activations/layer16_attention_weight_max": 30.658206939697266,
      "activations/layer16_attention_weight_min": -28.515254974365234,
      "activations/layer17_attention_weight_max": 55.9765510559082,
      "activations/layer17_attention_weight_min": -44.82231521606445,
      "activations/layer18_attention_weight_max": 48.25071334838867,
      "activations/layer18_attention_weight_min": -36.93333435058594,
      "activations/layer19_attention_weight_max": 24.67827606201172,
      "activations/layer19_attention_weight_min": -21.810575485229492,
      "activations/layer1_attention_weight_max": 17.697677612304688,
      "activations/layer1_attention_weight_min": -14.574864387512207,
      "activations/layer20_attention_weight_max": 26.46904945373535,
      "activations/layer20_attention_weight_min": -22.281326293945312,
      "activations/layer21_attention_weight_max": 40.4501953125,
      "activations/layer21_attention_weight_min": -23.638742446899414,
      "activations/layer22_attention_weight_max": 33.46532440185547,
      "activations/layer22_attention_weight_min": -24.84053611755371,
      "activations/layer23_attention_weight_max": 43.29825973510742,
      "activations/layer23_attention_weight_min": -25.312721252441406,
      "activations/layer2_attention_weight_max": 31.256277084350586,
      "activations/layer2_attention_weight_min": -29.207983016967773,
      "activations/layer3_attention_weight_max": 89.09815979003906,
      "activations/layer3_attention_weight_min": -91.04440307617188,
      "activations/layer4_attention_weight_max": 99.33008575439453,
      "activations/layer4_attention_weight_min": -93.88294219970703,
      "activations/layer5_attention_weight_max": 75.48667907714844,
      "activations/layer5_attention_weight_min": -79.22370147705078,
      "activations/layer6_attention_weight_max": 55.682220458984375,
      "activations/layer6_attention_weight_min": -54.84706115722656,
      "activations/layer7_attention_weight_max": 74.39139556884766,
      "activations/layer7_attention_weight_min": -73.1429443359375,
      "activations/layer8_attention_weight_max": 45.26503372192383,
      "activations/layer8_attention_weight_min": -48.41788101196289,
      "activations/layer9_attention_weight_max": 47.554073333740234,
      "activations/layer9_attention_weight_min": -50.3021354675293,
      "epoch": 10.35,
      "learning_rate": 8.411136363636364e-05,
      "loss": 2.797,
      "step": 178100
    },
    {
      "activations/layer0_attention_weight_max": 16.309892654418945,
      "activations/layer0_attention_weight_min": -13.836214065551758,
      "activations/layer10_attention_weight_max": 35.23887634277344,
      "activations/layer10_attention_weight_min": -33.44356918334961,
      "activations/layer11_attention_weight_max": 29.73154067993164,
      "activations/layer11_attention_weight_min": -29.449460983276367,
      "activations/layer12_attention_weight_max": 19.78917121887207,
      "activations/layer12_attention_weight_min": -24.606979370117188,
      "activations/layer13_attention_weight_max": 35.082847595214844,
      "activations/layer13_attention_weight_min": -27.816869735717773,
      "activations/layer14_attention_weight_max": 36.4423828125,
      "activations/layer14_attention_weight_min": -29.79029083251953,
      "activations/layer15_attention_weight_max": 32.78750228881836,
      "activations/layer15_attention_weight_min": -29.538236618041992,
      "activations/layer16_attention_weight_max": 29.717021942138672,
      "activations/layer16_attention_weight_min": -27.87126922607422,
      "activations/layer17_attention_weight_max": 51.36153793334961,
      "activations/layer17_attention_weight_min": -46.3717155456543,
      "activations/layer18_attention_weight_max": 47.22391128540039,
      "activations/layer18_attention_weight_min": -37.57666015625,
      "activations/layer19_attention_weight_max": 23.135887145996094,
      "activations/layer19_attention_weight_min": -20.60466194152832,
      "activations/layer1_attention_weight_max": 17.108722686767578,
      "activations/layer1_attention_weight_min": -14.005547523498535,
      "activations/layer20_attention_weight_max": 21.17498207092285,
      "activations/layer20_attention_weight_min": -22.193147659301758,
      "activations/layer21_attention_weight_max": 33.78382873535156,
      "activations/layer21_attention_weight_min": -21.590028762817383,
      "activations/layer22_attention_weight_max": 29.56266975402832,
      "activations/layer22_attention_weight_min": -23.047130584716797,
      "activations/layer23_attention_weight_max": 36.12971496582031,
      "activations/layer23_attention_weight_min": -21.7346248626709,
      "activations/layer2_attention_weight_max": 32.831947326660156,
      "activations/layer2_attention_weight_min": -30.727317810058594,
      "activations/layer3_attention_weight_max": 92.32485961914062,
      "activations/layer3_attention_weight_min": -94.66943359375,
      "activations/layer4_attention_weight_max": 99.00524139404297,
      "activations/layer4_attention_weight_min": -98.64510345458984,
      "activations/layer5_attention_weight_max": 77.256591796875,
      "activations/layer5_attention_weight_min": -78.80268859863281,
      "activations/layer6_attention_weight_max": 52.31775665283203,
      "activations/layer6_attention_weight_min": -52.8798942565918,
      "activations/layer7_attention_weight_max": 73.0046615600586,
      "activations/layer7_attention_weight_min": -72.14317321777344,
      "activations/layer8_attention_weight_max": 42.488868713378906,
      "activations/layer8_attention_weight_min": -45.88871765136719,
      "activations/layer9_attention_weight_max": 47.0286865234375,
      "activations/layer9_attention_weight_min": -47.779537200927734,
      "epoch": 10.35,
      "learning_rate": 8.409242424242422e-05,
      "loss": 2.779,
      "step": 178150
    },
    {
      "activations/layer0_attention_weight_max": 17.962032318115234,
      "activations/layer0_attention_weight_min": -14.522656440734863,
      "activations/layer10_attention_weight_max": 36.20618438720703,
      "activations/layer10_attention_weight_min": -35.08769607543945,
      "activations/layer11_attention_weight_max": 32.328983306884766,
      "activations/layer11_attention_weight_min": -30.90816307067871,
      "activations/layer12_attention_weight_max": 22.11493682861328,
      "activations/layer12_attention_weight_min": -27.79928970336914,
      "activations/layer13_attention_weight_max": 39.30491638183594,
      "activations/layer13_attention_weight_min": -30.002307891845703,
      "activations/layer14_attention_weight_max": 37.004695892333984,
      "activations/layer14_attention_weight_min": -34.42917251586914,
      "activations/layer15_attention_weight_max": 33.443756103515625,
      "activations/layer15_attention_weight_min": -32.641998291015625,
      "activations/layer16_attention_weight_max": 33.97574996948242,
      "activations/layer16_attention_weight_min": -32.656646728515625,
      "activations/layer17_attention_weight_max": 52.237762451171875,
      "activations/layer17_attention_weight_min": -48.012489318847656,
      "activations/layer18_attention_weight_max": 47.39506912231445,
      "activations/layer18_attention_weight_min": -42.72063446044922,
      "activations/layer19_attention_weight_max": 24.217592239379883,
      "activations/layer19_attention_weight_min": -24.469070434570312,
      "activations/layer1_attention_weight_max": 16.14154815673828,
      "activations/layer1_attention_weight_min": -15.68216609954834,
      "activations/layer20_attention_weight_max": 26.179277420043945,
      "activations/layer20_attention_weight_min": -22.547372817993164,
      "activations/layer21_attention_weight_max": 36.55781173706055,
      "activations/layer21_attention_weight_min": -23.47392463684082,
      "activations/layer22_attention_weight_max": 31.850719451904297,
      "activations/layer22_attention_weight_min": -25.225141525268555,
      "activations/layer23_attention_weight_max": 39.76322555541992,
      "activations/layer23_attention_weight_min": -22.322629928588867,
      "activations/layer2_attention_weight_max": 32.70042419433594,
      "activations/layer2_attention_weight_min": -30.778953552246094,
      "activations/layer3_attention_weight_max": 95.90995788574219,
      "activations/layer3_attention_weight_min": -101.19955444335938,
      "activations/layer4_attention_weight_max": 100.27510833740234,
      "activations/layer4_attention_weight_min": -96.67278289794922,
      "activations/layer5_attention_weight_max": 76.82444763183594,
      "activations/layer5_attention_weight_min": -76.74385070800781,
      "activations/layer6_attention_weight_max": 50.00898742675781,
      "activations/layer6_attention_weight_min": -51.248783111572266,
      "activations/layer7_attention_weight_max": 73.82487487792969,
      "activations/layer7_attention_weight_min": -72.5524673461914,
      "activations/layer8_attention_weight_max": 42.79328536987305,
      "activations/layer8_attention_weight_min": -44.49103546142578,
      "activations/layer9_attention_weight_max": 52.97779083251953,
      "activations/layer9_attention_weight_min": -46.34626388549805,
      "epoch": 10.35,
      "learning_rate": 8.407348484848484e-05,
      "loss": 2.7591,
      "step": 178200
    },
    {
      "activations/layer0_attention_weight_max": 15.29175090789795,
      "activations/layer0_attention_weight_min": -14.394177436828613,
      "activations/layer10_attention_weight_max": 35.18600845336914,
      "activations/layer10_attention_weight_min": -35.2503662109375,
      "activations/layer11_attention_weight_max": 34.3704833984375,
      "activations/layer11_attention_weight_min": -32.10658264160156,
      "activations/layer12_attention_weight_max": 18.01813507080078,
      "activations/layer12_attention_weight_min": -25.105321884155273,
      "activations/layer13_attention_weight_max": 41.185890197753906,
      "activations/layer13_attention_weight_min": -28.363948822021484,
      "activations/layer14_attention_weight_max": 38.93533706665039,
      "activations/layer14_attention_weight_min": -32.02064514160156,
      "activations/layer15_attention_weight_max": 37.2711296081543,
      "activations/layer15_attention_weight_min": -31.71371078491211,
      "activations/layer16_attention_weight_max": 36.21677780151367,
      "activations/layer16_attention_weight_min": -32.84024429321289,
      "activations/layer17_attention_weight_max": 57.77881622314453,
      "activations/layer17_attention_weight_min": -50.05632400512695,
      "activations/layer18_attention_weight_max": 54.430110931396484,
      "activations/layer18_attention_weight_min": -42.27031707763672,
      "activations/layer19_attention_weight_max": 24.407318115234375,
      "activations/layer19_attention_weight_min": -22.683170318603516,
      "activations/layer1_attention_weight_max": 15.891907691955566,
      "activations/layer1_attention_weight_min": -14.23349666595459,
      "activations/layer20_attention_weight_max": 24.341785430908203,
      "activations/layer20_attention_weight_min": -21.418224334716797,
      "activations/layer21_attention_weight_max": 39.732017517089844,
      "activations/layer21_attention_weight_min": -22.504867553710938,
      "activations/layer22_attention_weight_max": 32.50952911376953,
      "activations/layer22_attention_weight_min": -25.37354278564453,
      "activations/layer23_attention_weight_max": 41.169437408447266,
      "activations/layer23_attention_weight_min": -22.30803680419922,
      "activations/layer2_attention_weight_max": 27.020254135131836,
      "activations/layer2_attention_weight_min": -28.003009796142578,
      "activations/layer3_attention_weight_max": 79.39733123779297,
      "activations/layer3_attention_weight_min": -84.54918670654297,
      "activations/layer4_attention_weight_max": 87.14421081542969,
      "activations/layer4_attention_weight_min": -85.18741607666016,
      "activations/layer5_attention_weight_max": 67.31726837158203,
      "activations/layer5_attention_weight_min": -75.95040893554688,
      "activations/layer6_attention_weight_max": 47.43552780151367,
      "activations/layer6_attention_weight_min": -47.139339447021484,
      "activations/layer7_attention_weight_max": 68.95597839355469,
      "activations/layer7_attention_weight_min": -68.98477935791016,
      "activations/layer8_attention_weight_max": 41.60747146606445,
      "activations/layer8_attention_weight_min": -43.919246673583984,
      "activations/layer9_attention_weight_max": 45.925048828125,
      "activations/layer9_attention_weight_min": -48.13679504394531,
      "epoch": 10.36,
      "learning_rate": 8.405454545454545e-05,
      "loss": 2.7965,
      "step": 178250
    },
    {
      "activations/layer0_attention_weight_max": 16.14252471923828,
      "activations/layer0_attention_weight_min": -14.27279281616211,
      "activations/layer10_attention_weight_max": 38.0804443359375,
      "activations/layer10_attention_weight_min": -36.41781997680664,
      "activations/layer11_attention_weight_max": 33.47332000732422,
      "activations/layer11_attention_weight_min": -32.6661491394043,
      "activations/layer12_attention_weight_max": 18.304912567138672,
      "activations/layer12_attention_weight_min": -23.204524993896484,
      "activations/layer13_attention_weight_max": 35.41560363769531,
      "activations/layer13_attention_weight_min": -27.96734046936035,
      "activations/layer14_attention_weight_max": 35.350257873535156,
      "activations/layer14_attention_weight_min": -29.11648941040039,
      "activations/layer15_attention_weight_max": 33.59206771850586,
      "activations/layer15_attention_weight_min": -27.131507873535156,
      "activations/layer16_attention_weight_max": 30.018774032592773,
      "activations/layer16_attention_weight_min": -27.480777740478516,
      "activations/layer17_attention_weight_max": 51.66242218017578,
      "activations/layer17_attention_weight_min": -42.21149444580078,
      "activations/layer18_attention_weight_max": 44.98041534423828,
      "activations/layer18_attention_weight_min": -36.619972229003906,
      "activations/layer19_attention_weight_max": 20.8896541595459,
      "activations/layer19_attention_weight_min": -22.21298599243164,
      "activations/layer1_attention_weight_max": 16.66873550415039,
      "activations/layer1_attention_weight_min": -14.480573654174805,
      "activations/layer20_attention_weight_max": 21.592178344726562,
      "activations/layer20_attention_weight_min": -18.9620361328125,
      "activations/layer21_attention_weight_max": 34.09880447387695,
      "activations/layer21_attention_weight_min": -19.112714767456055,
      "activations/layer22_attention_weight_max": 33.327354431152344,
      "activations/layer22_attention_weight_min": -22.86728858947754,
      "activations/layer23_attention_weight_max": 36.997535705566406,
      "activations/layer23_attention_weight_min": -23.522960662841797,
      "activations/layer2_attention_weight_max": 31.208332061767578,
      "activations/layer2_attention_weight_min": -29.03317642211914,
      "activations/layer3_attention_weight_max": 97.22752380371094,
      "activations/layer3_attention_weight_min": -98.52296447753906,
      "activations/layer4_attention_weight_max": 101.71070861816406,
      "activations/layer4_attention_weight_min": -96.45655059814453,
      "activations/layer5_attention_weight_max": 73.46949768066406,
      "activations/layer5_attention_weight_min": -79.04092407226562,
      "activations/layer6_attention_weight_max": 53.84967803955078,
      "activations/layer6_attention_weight_min": -52.51179885864258,
      "activations/layer7_attention_weight_max": 74.29239654541016,
      "activations/layer7_attention_weight_min": -70.63209533691406,
      "activations/layer8_attention_weight_max": 47.4483528137207,
      "activations/layer8_attention_weight_min": -48.499244689941406,
      "activations/layer9_attention_weight_max": 47.40916061401367,
      "activations/layer9_attention_weight_min": -48.09965133666992,
      "epoch": 10.36,
      "learning_rate": 8.403560606060604e-05,
      "loss": 2.7845,
      "step": 178300
    },
    {
      "activations/layer0_attention_weight_max": 14.797723770141602,
      "activations/layer0_attention_weight_min": -14.344179153442383,
      "activations/layer10_attention_weight_max": 38.17890167236328,
      "activations/layer10_attention_weight_min": -36.815887451171875,
      "activations/layer11_attention_weight_max": 32.099456787109375,
      "activations/layer11_attention_weight_min": -33.1187744140625,
      "activations/layer12_attention_weight_max": 23.25528907775879,
      "activations/layer12_attention_weight_min": -24.4585018157959,
      "activations/layer13_attention_weight_max": 39.15712356567383,
      "activations/layer13_attention_weight_min": -32.07461166381836,
      "activations/layer14_attention_weight_max": 40.51108932495117,
      "activations/layer14_attention_weight_min": -28.363021850585938,
      "activations/layer15_attention_weight_max": 35.48714828491211,
      "activations/layer15_attention_weight_min": -28.08650016784668,
      "activations/layer16_attention_weight_max": 34.51364517211914,
      "activations/layer16_attention_weight_min": -27.660295486450195,
      "activations/layer17_attention_weight_max": 54.6294059753418,
      "activations/layer17_attention_weight_min": -42.66293716430664,
      "activations/layer18_attention_weight_max": 45.30313491821289,
      "activations/layer18_attention_weight_min": -36.460880279541016,
      "activations/layer19_attention_weight_max": 21.23141860961914,
      "activations/layer19_attention_weight_min": -21.8356990814209,
      "activations/layer1_attention_weight_max": 16.514610290527344,
      "activations/layer1_attention_weight_min": -15.3955078125,
      "activations/layer20_attention_weight_max": 21.85215950012207,
      "activations/layer20_attention_weight_min": -22.41918182373047,
      "activations/layer21_attention_weight_max": 40.46204376220703,
      "activations/layer21_attention_weight_min": -24.1173152923584,
      "activations/layer22_attention_weight_max": 35.361358642578125,
      "activations/layer22_attention_weight_min": -26.21786880493164,
      "activations/layer23_attention_weight_max": 39.18534469604492,
      "activations/layer23_attention_weight_min": -25.02347183227539,
      "activations/layer2_attention_weight_max": 32.732177734375,
      "activations/layer2_attention_weight_min": -30.929336547851562,
      "activations/layer3_attention_weight_max": 96.64159393310547,
      "activations/layer3_attention_weight_min": -93.60436248779297,
      "activations/layer4_attention_weight_max": 98.82125091552734,
      "activations/layer4_attention_weight_min": -96.38872528076172,
      "activations/layer5_attention_weight_max": 75.73272705078125,
      "activations/layer5_attention_weight_min": -77.98851776123047,
      "activations/layer6_attention_weight_max": 56.121055603027344,
      "activations/layer6_attention_weight_min": -54.02851104736328,
      "activations/layer7_attention_weight_max": 77.58079528808594,
      "activations/layer7_attention_weight_min": -72.96244812011719,
      "activations/layer8_attention_weight_max": 47.0562629699707,
      "activations/layer8_attention_weight_min": -48.56853103637695,
      "activations/layer9_attention_weight_max": 49.77897644042969,
      "activations/layer9_attention_weight_min": -47.82502746582031,
      "epoch": 10.36,
      "learning_rate": 8.401666666666666e-05,
      "loss": 2.7708,
      "step": 178350
    },
    {
      "activations/layer0_attention_weight_max": 14.708279609680176,
      "activations/layer0_attention_weight_min": -14.34047794342041,
      "activations/layer10_attention_weight_max": 36.906639099121094,
      "activations/layer10_attention_weight_min": -36.539249420166016,
      "activations/layer11_attention_weight_max": 31.013769149780273,
      "activations/layer11_attention_weight_min": -32.65911102294922,
      "activations/layer12_attention_weight_max": 17.338300704956055,
      "activations/layer12_attention_weight_min": -24.68362808227539,
      "activations/layer13_attention_weight_max": 37.396663665771484,
      "activations/layer13_attention_weight_min": -30.040700912475586,
      "activations/layer14_attention_weight_max": 41.816162109375,
      "activations/layer14_attention_weight_min": -31.529987335205078,
      "activations/layer15_attention_weight_max": 35.90966796875,
      "activations/layer15_attention_weight_min": -30.062416076660156,
      "activations/layer16_attention_weight_max": 33.12324523925781,
      "activations/layer16_attention_weight_min": -29.898611068725586,
      "activations/layer17_attention_weight_max": 52.55231857299805,
      "activations/layer17_attention_weight_min": -43.248077392578125,
      "activations/layer18_attention_weight_max": 47.498748779296875,
      "activations/layer18_attention_weight_min": -35.881553649902344,
      "activations/layer19_attention_weight_max": 21.55698013305664,
      "activations/layer19_attention_weight_min": -21.280826568603516,
      "activations/layer1_attention_weight_max": 16.423009872436523,
      "activations/layer1_attention_weight_min": -15.571558952331543,
      "activations/layer20_attention_weight_max": 21.556779861450195,
      "activations/layer20_attention_weight_min": -20.575942993164062,
      "activations/layer21_attention_weight_max": 36.27489471435547,
      "activations/layer21_attention_weight_min": -24.23418617248535,
      "activations/layer22_attention_weight_max": 31.66778564453125,
      "activations/layer22_attention_weight_min": -24.86810302734375,
      "activations/layer23_attention_weight_max": 34.53556823730469,
      "activations/layer23_attention_weight_min": -23.641033172607422,
      "activations/layer2_attention_weight_max": 33.22126770019531,
      "activations/layer2_attention_weight_min": -30.598203659057617,
      "activations/layer3_attention_weight_max": 94.75039672851562,
      "activations/layer3_attention_weight_min": -89.33578491210938,
      "activations/layer4_attention_weight_max": 99.22813415527344,
      "activations/layer4_attention_weight_min": -93.9122543334961,
      "activations/layer5_attention_weight_max": 76.92219543457031,
      "activations/layer5_attention_weight_min": -75.6889419555664,
      "activations/layer6_attention_weight_max": 55.327720642089844,
      "activations/layer6_attention_weight_min": -54.242061614990234,
      "activations/layer7_attention_weight_max": 75.16596984863281,
      "activations/layer7_attention_weight_min": -72.0693359375,
      "activations/layer8_attention_weight_max": 51.24299621582031,
      "activations/layer8_attention_weight_min": -51.522090911865234,
      "activations/layer9_attention_weight_max": 49.78379821777344,
      "activations/layer9_attention_weight_min": -48.78091812133789,
      "epoch": 10.37,
      "learning_rate": 8.399772727272727e-05,
      "loss": 2.7864,
      "step": 178400
    },
    {
      "activations/layer0_attention_weight_max": 15.728864669799805,
      "activations/layer0_attention_weight_min": -14.474160194396973,
      "activations/layer10_attention_weight_max": 44.752197265625,
      "activations/layer10_attention_weight_min": -41.09611129760742,
      "activations/layer11_attention_weight_max": 37.664215087890625,
      "activations/layer11_attention_weight_min": -36.3304443359375,
      "activations/layer12_attention_weight_max": 31.62079620361328,
      "activations/layer12_attention_weight_min": -23.54413604736328,
      "activations/layer13_attention_weight_max": 50.43385696411133,
      "activations/layer13_attention_weight_min": -32.40605545043945,
      "activations/layer14_attention_weight_max": 40.081756591796875,
      "activations/layer14_attention_weight_min": -30.729755401611328,
      "activations/layer15_attention_weight_max": 40.06793975830078,
      "activations/layer15_attention_weight_min": -30.723682403564453,
      "activations/layer16_attention_weight_max": 35.933231353759766,
      "activations/layer16_attention_weight_min": -31.977127075195312,
      "activations/layer17_attention_weight_max": 68.5414810180664,
      "activations/layer17_attention_weight_min": -50.84770202636719,
      "activations/layer18_attention_weight_max": 57.83046340942383,
      "activations/layer18_attention_weight_min": -40.486671447753906,
      "activations/layer19_attention_weight_max": 28.605484008789062,
      "activations/layer19_attention_weight_min": -21.70958137512207,
      "activations/layer1_attention_weight_max": 17.53561782836914,
      "activations/layer1_attention_weight_min": -15.578512191772461,
      "activations/layer20_attention_weight_max": 24.72972869873047,
      "activations/layer20_attention_weight_min": -18.70047378540039,
      "activations/layer21_attention_weight_max": 38.50343704223633,
      "activations/layer21_attention_weight_min": -20.134618759155273,
      "activations/layer22_attention_weight_max": 33.50944137573242,
      "activations/layer22_attention_weight_min": -24.85466957092285,
      "activations/layer23_attention_weight_max": 41.52587890625,
      "activations/layer23_attention_weight_min": -22.55970573425293,
      "activations/layer2_attention_weight_max": 32.31705856323242,
      "activations/layer2_attention_weight_min": -31.017887115478516,
      "activations/layer3_attention_weight_max": 92.34017181396484,
      "activations/layer3_attention_weight_min": -94.77409362792969,
      "activations/layer4_attention_weight_max": 103.0401382446289,
      "activations/layer4_attention_weight_min": -100.26525115966797,
      "activations/layer5_attention_weight_max": 81.96287536621094,
      "activations/layer5_attention_weight_min": -82.63934326171875,
      "activations/layer6_attention_weight_max": 55.138206481933594,
      "activations/layer6_attention_weight_min": -55.352821350097656,
      "activations/layer7_attention_weight_max": 77.2655029296875,
      "activations/layer7_attention_weight_min": -72.31466674804688,
      "activations/layer8_attention_weight_max": 50.2103157043457,
      "activations/layer8_attention_weight_min": -52.12079620361328,
      "activations/layer9_attention_weight_max": 55.7820930480957,
      "activations/layer9_attention_weight_min": -53.88587188720703,
      "epoch": 10.37,
      "learning_rate": 8.397878787878786e-05,
      "loss": 2.8041,
      "step": 178450
    },
    {
      "activations/layer0_attention_weight_max": 15.776646614074707,
      "activations/layer0_attention_weight_min": -14.144597053527832,
      "activations/layer10_attention_weight_max": 36.18054962158203,
      "activations/layer10_attention_weight_min": -37.37097930908203,
      "activations/layer11_attention_weight_max": 32.9779052734375,
      "activations/layer11_attention_weight_min": -34.03784942626953,
      "activations/layer12_attention_weight_max": 17.9276180267334,
      "activations/layer12_attention_weight_min": -27.22136688232422,
      "activations/layer13_attention_weight_max": 44.75178527832031,
      "activations/layer13_attention_weight_min": -33.14451217651367,
      "activations/layer14_attention_weight_max": 41.479248046875,
      "activations/layer14_attention_weight_min": -30.827043533325195,
      "activations/layer15_attention_weight_max": 35.34675216674805,
      "activations/layer15_attention_weight_min": -29.70210075378418,
      "activations/layer16_attention_weight_max": 34.88860321044922,
      "activations/layer16_attention_weight_min": -31.18328857421875,
      "activations/layer17_attention_weight_max": 55.25294494628906,
      "activations/layer17_attention_weight_min": -46.46748733520508,
      "activations/layer18_attention_weight_max": 47.76405334472656,
      "activations/layer18_attention_weight_min": -36.86648941040039,
      "activations/layer19_attention_weight_max": 23.384428024291992,
      "activations/layer19_attention_weight_min": -20.831762313842773,
      "activations/layer1_attention_weight_max": 17.65150260925293,
      "activations/layer1_attention_weight_min": -16.036174774169922,
      "activations/layer20_attention_weight_max": 25.470836639404297,
      "activations/layer20_attention_weight_min": -22.7076358795166,
      "activations/layer21_attention_weight_max": 38.28417205810547,
      "activations/layer21_attention_weight_min": -21.58785057067871,
      "activations/layer22_attention_weight_max": 37.16829299926758,
      "activations/layer22_attention_weight_min": -25.691003799438477,
      "activations/layer23_attention_weight_max": 44.38319396972656,
      "activations/layer23_attention_weight_min": -22.540502548217773,
      "activations/layer2_attention_weight_max": 35.274436950683594,
      "activations/layer2_attention_weight_min": -33.384281158447266,
      "activations/layer3_attention_weight_max": 95.12493133544922,
      "activations/layer3_attention_weight_min": -100.05696868896484,
      "activations/layer4_attention_weight_max": 100.36113739013672,
      "activations/layer4_attention_weight_min": -98.82213592529297,
      "activations/layer5_attention_weight_max": 81.21156311035156,
      "activations/layer5_attention_weight_min": -77.21476745605469,
      "activations/layer6_attention_weight_max": 55.932525634765625,
      "activations/layer6_attention_weight_min": -56.510257720947266,
      "activations/layer7_attention_weight_max": 75.55758666992188,
      "activations/layer7_attention_weight_min": -79.93231201171875,
      "activations/layer8_attention_weight_max": 45.48468017578125,
      "activations/layer8_attention_weight_min": -49.92702102661133,
      "activations/layer9_attention_weight_max": 52.46806716918945,
      "activations/layer9_attention_weight_min": -52.323463439941406,
      "epoch": 10.37,
      "learning_rate": 8.395984848484848e-05,
      "loss": 2.7818,
      "step": 178500
    },
    {
      "activations/layer0_attention_weight_max": 15.262703895568848,
      "activations/layer0_attention_weight_min": -14.17455005645752,
      "activations/layer10_attention_weight_max": 39.735084533691406,
      "activations/layer10_attention_weight_min": -36.840824127197266,
      "activations/layer11_attention_weight_max": 32.90245819091797,
      "activations/layer11_attention_weight_min": -32.40419006347656,
      "activations/layer12_attention_weight_max": 18.445951461791992,
      "activations/layer12_attention_weight_min": -24.309837341308594,
      "activations/layer13_attention_weight_max": 40.675880432128906,
      "activations/layer13_attention_weight_min": -30.18310546875,
      "activations/layer14_attention_weight_max": 36.723262786865234,
      "activations/layer14_attention_weight_min": -32.984169006347656,
      "activations/layer15_attention_weight_max": 33.91848373413086,
      "activations/layer15_attention_weight_min": -30.724639892578125,
      "activations/layer16_attention_weight_max": 30.662858963012695,
      "activations/layer16_attention_weight_min": -30.60447883605957,
      "activations/layer17_attention_weight_max": 49.230628967285156,
      "activations/layer17_attention_weight_min": -43.77372741699219,
      "activations/layer18_attention_weight_max": 44.42918395996094,
      "activations/layer18_attention_weight_min": -36.482330322265625,
      "activations/layer19_attention_weight_max": 25.85043716430664,
      "activations/layer19_attention_weight_min": -22.01146697998047,
      "activations/layer1_attention_weight_max": 17.47294044494629,
      "activations/layer1_attention_weight_min": -16.47566032409668,
      "activations/layer20_attention_weight_max": 20.64305877685547,
      "activations/layer20_attention_weight_min": -22.67762565612793,
      "activations/layer21_attention_weight_max": 37.843467712402344,
      "activations/layer21_attention_weight_min": -23.70657730102539,
      "activations/layer22_attention_weight_max": 31.803741455078125,
      "activations/layer22_attention_weight_min": -24.162046432495117,
      "activations/layer23_attention_weight_max": 37.079776763916016,
      "activations/layer23_attention_weight_min": -22.705421447753906,
      "activations/layer2_attention_weight_max": 33.310646057128906,
      "activations/layer2_attention_weight_min": -31.830839157104492,
      "activations/layer3_attention_weight_max": 94.54125213623047,
      "activations/layer3_attention_weight_min": -92.06636810302734,
      "activations/layer4_attention_weight_max": 101.65983581542969,
      "activations/layer4_attention_weight_min": -97.62368774414062,
      "activations/layer5_attention_weight_max": 81.21391296386719,
      "activations/layer5_attention_weight_min": -87.018798828125,
      "activations/layer6_attention_weight_max": 55.84885025024414,
      "activations/layer6_attention_weight_min": -57.85660171508789,
      "activations/layer7_attention_weight_max": 81.05120849609375,
      "activations/layer7_attention_weight_min": -77.04084777832031,
      "activations/layer8_attention_weight_max": 49.80472183227539,
      "activations/layer8_attention_weight_min": -50.229740142822266,
      "activations/layer9_attention_weight_max": 56.51267623901367,
      "activations/layer9_attention_weight_min": -57.45046615600586,
      "epoch": 10.37,
      "learning_rate": 8.394090909090908e-05,
      "loss": 2.7932,
      "step": 178550
    },
    {
      "activations/layer0_attention_weight_max": 15.070960998535156,
      "activations/layer0_attention_weight_min": -14.727127075195312,
      "activations/layer10_attention_weight_max": 41.32801818847656,
      "activations/layer10_attention_weight_min": -39.486751556396484,
      "activations/layer11_attention_weight_max": 35.07517623901367,
      "activations/layer11_attention_weight_min": -33.67249298095703,
      "activations/layer12_attention_weight_max": 16.615278244018555,
      "activations/layer12_attention_weight_min": -22.935317993164062,
      "activations/layer13_attention_weight_max": 43.78608703613281,
      "activations/layer13_attention_weight_min": -34.76942443847656,
      "activations/layer14_attention_weight_max": 52.16624450683594,
      "activations/layer14_attention_weight_min": -33.518009185791016,
      "activations/layer15_attention_weight_max": 42.87864685058594,
      "activations/layer15_attention_weight_min": -32.25924301147461,
      "activations/layer16_attention_weight_max": 29.94170379638672,
      "activations/layer16_attention_weight_min": -27.93724250793457,
      "activations/layer17_attention_weight_max": 51.79059982299805,
      "activations/layer17_attention_weight_min": -45.97119140625,
      "activations/layer18_attention_weight_max": 43.30521011352539,
      "activations/layer18_attention_weight_min": -39.12010955810547,
      "activations/layer19_attention_weight_max": 21.710866928100586,
      "activations/layer19_attention_weight_min": -22.299253463745117,
      "activations/layer1_attention_weight_max": 17.057777404785156,
      "activations/layer1_attention_weight_min": -15.154056549072266,
      "activations/layer20_attention_weight_max": 20.6418399810791,
      "activations/layer20_attention_weight_min": -23.721879959106445,
      "activations/layer21_attention_weight_max": 37.73194885253906,
      "activations/layer21_attention_weight_min": -21.399887084960938,
      "activations/layer22_attention_weight_max": 34.36515426635742,
      "activations/layer22_attention_weight_min": -23.980670928955078,
      "activations/layer23_attention_weight_max": 33.00357437133789,
      "activations/layer23_attention_weight_min": -22.13922882080078,
      "activations/layer2_attention_weight_max": 32.89874267578125,
      "activations/layer2_attention_weight_min": -31.64826202392578,
      "activations/layer3_attention_weight_max": 95.18212890625,
      "activations/layer3_attention_weight_min": -98.64739227294922,
      "activations/layer4_attention_weight_max": 98.67659759521484,
      "activations/layer4_attention_weight_min": -94.53836059570312,
      "activations/layer5_attention_weight_max": 75.19706726074219,
      "activations/layer5_attention_weight_min": -77.18184661865234,
      "activations/layer6_attention_weight_max": 54.411128997802734,
      "activations/layer6_attention_weight_min": -53.679683685302734,
      "activations/layer7_attention_weight_max": 76.46192169189453,
      "activations/layer7_attention_weight_min": -73.54170227050781,
      "activations/layer8_attention_weight_max": 47.596370697021484,
      "activations/layer8_attention_weight_min": -46.4665412902832,
      "activations/layer9_attention_weight_max": 50.19902420043945,
      "activations/layer9_attention_weight_min": -51.19875717163086,
      "epoch": 10.38,
      "learning_rate": 8.392234848484848e-05,
      "loss": 2.7898,
      "step": 178600
    },
    {
      "activations/layer0_attention_weight_max": 14.285898208618164,
      "activations/layer0_attention_weight_min": -14.039053916931152,
      "activations/layer10_attention_weight_max": 34.93647384643555,
      "activations/layer10_attention_weight_min": -34.705711364746094,
      "activations/layer11_attention_weight_max": 30.940298080444336,
      "activations/layer11_attention_weight_min": -32.75269317626953,
      "activations/layer12_attention_weight_max": 18.43153190612793,
      "activations/layer12_attention_weight_min": -30.044178009033203,
      "activations/layer13_attention_weight_max": 37.0871696472168,
      "activations/layer13_attention_weight_min": -27.972251892089844,
      "activations/layer14_attention_weight_max": 35.835243225097656,
      "activations/layer14_attention_weight_min": -28.105581283569336,
      "activations/layer15_attention_weight_max": 31.403966903686523,
      "activations/layer15_attention_weight_min": -29.635482788085938,
      "activations/layer16_attention_weight_max": 29.392684936523438,
      "activations/layer16_attention_weight_min": -27.607589721679688,
      "activations/layer17_attention_weight_max": 49.993858337402344,
      "activations/layer17_attention_weight_min": -42.247962951660156,
      "activations/layer18_attention_weight_max": 43.71478271484375,
      "activations/layer18_attention_weight_min": -35.07977294921875,
      "activations/layer19_attention_weight_max": 21.575063705444336,
      "activations/layer19_attention_weight_min": -20.62757110595703,
      "activations/layer1_attention_weight_max": 16.551259994506836,
      "activations/layer1_attention_weight_min": -16.11253547668457,
      "activations/layer20_attention_weight_max": 22.638593673706055,
      "activations/layer20_attention_weight_min": -22.208972930908203,
      "activations/layer21_attention_weight_max": 40.9050407409668,
      "activations/layer21_attention_weight_min": -26.813812255859375,
      "activations/layer22_attention_weight_max": 28.439435958862305,
      "activations/layer22_attention_weight_min": -24.12005615234375,
      "activations/layer23_attention_weight_max": 33.304901123046875,
      "activations/layer23_attention_weight_min": -23.781789779663086,
      "activations/layer2_attention_weight_max": 32.0834846496582,
      "activations/layer2_attention_weight_min": -30.07984161376953,
      "activations/layer3_attention_weight_max": 85.67033386230469,
      "activations/layer3_attention_weight_min": -92.1162338256836,
      "activations/layer4_attention_weight_max": 98.99910736083984,
      "activations/layer4_attention_weight_min": -97.74243927001953,
      "activations/layer5_attention_weight_max": 74.18903350830078,
      "activations/layer5_attention_weight_min": -84.87932586669922,
      "activations/layer6_attention_weight_max": 51.91872024536133,
      "activations/layer6_attention_weight_min": -56.91630172729492,
      "activations/layer7_attention_weight_max": 73.03010559082031,
      "activations/layer7_attention_weight_min": -71.4404067993164,
      "activations/layer8_attention_weight_max": 45.32097244262695,
      "activations/layer8_attention_weight_min": -46.18241882324219,
      "activations/layer9_attention_weight_max": 46.97865295410156,
      "activations/layer9_attention_weight_min": -50.169593811035156,
      "epoch": 10.38,
      "learning_rate": 8.390340909090908e-05,
      "loss": 2.7815,
      "step": 178650
    },
    {
      "activations/layer0_attention_weight_max": 15.850997924804688,
      "activations/layer0_attention_weight_min": -14.35881233215332,
      "activations/layer10_attention_weight_max": 35.73527145385742,
      "activations/layer10_attention_weight_min": -34.449363708496094,
      "activations/layer11_attention_weight_max": 30.41266632080078,
      "activations/layer11_attention_weight_min": -31.88789939880371,
      "activations/layer12_attention_weight_max": 18.920679092407227,
      "activations/layer12_attention_weight_min": -25.723148345947266,
      "activations/layer13_attention_weight_max": 38.60896301269531,
      "activations/layer13_attention_weight_min": -31.41449546813965,
      "activations/layer14_attention_weight_max": 34.62689971923828,
      "activations/layer14_attention_weight_min": -30.515254974365234,
      "activations/layer15_attention_weight_max": 33.48952865600586,
      "activations/layer15_attention_weight_min": -30.19365119934082,
      "activations/layer16_attention_weight_max": 31.704395294189453,
      "activations/layer16_attention_weight_min": -27.809457778930664,
      "activations/layer17_attention_weight_max": 48.24729537963867,
      "activations/layer17_attention_weight_min": -41.74373245239258,
      "activations/layer18_attention_weight_max": 44.338043212890625,
      "activations/layer18_attention_weight_min": -36.28748321533203,
      "activations/layer19_attention_weight_max": 25.29525375366211,
      "activations/layer19_attention_weight_min": -20.85647964477539,
      "activations/layer1_attention_weight_max": 17.438329696655273,
      "activations/layer1_attention_weight_min": -16.65901756286621,
      "activations/layer20_attention_weight_max": 20.22966766357422,
      "activations/layer20_attention_weight_min": -21.529254913330078,
      "activations/layer21_attention_weight_max": 39.817543029785156,
      "activations/layer21_attention_weight_min": -25.99112319946289,
      "activations/layer22_attention_weight_max": 30.48255729675293,
      "activations/layer22_attention_weight_min": -24.817672729492188,
      "activations/layer23_attention_weight_max": 35.826866149902344,
      "activations/layer23_attention_weight_min": -22.46767807006836,
      "activations/layer2_attention_weight_max": 33.03282928466797,
      "activations/layer2_attention_weight_min": -30.080997467041016,
      "activations/layer3_attention_weight_max": 90.61144256591797,
      "activations/layer3_attention_weight_min": -91.42414855957031,
      "activations/layer4_attention_weight_max": 99.40890502929688,
      "activations/layer4_attention_weight_min": -90.23839569091797,
      "activations/layer5_attention_weight_max": 76.9892807006836,
      "activations/layer5_attention_weight_min": -79.09354400634766,
      "activations/layer6_attention_weight_max": 53.26260757446289,
      "activations/layer6_attention_weight_min": -50.43996047973633,
      "activations/layer7_attention_weight_max": 73.129638671875,
      "activations/layer7_attention_weight_min": -69.72315979003906,
      "activations/layer8_attention_weight_max": 46.620506286621094,
      "activations/layer8_attention_weight_min": -46.71269226074219,
      "activations/layer9_attention_weight_max": 47.58146286010742,
      "activations/layer9_attention_weight_min": -46.58504867553711,
      "epoch": 10.38,
      "learning_rate": 8.38844696969697e-05,
      "loss": 2.8073,
      "step": 178700
    },
    {
      "activations/layer0_attention_weight_max": 15.16574764251709,
      "activations/layer0_attention_weight_min": -14.243901252746582,
      "activations/layer10_attention_weight_max": 36.48634338378906,
      "activations/layer10_attention_weight_min": -34.540897369384766,
      "activations/layer11_attention_weight_max": 31.375242233276367,
      "activations/layer11_attention_weight_min": -31.600675582885742,
      "activations/layer12_attention_weight_max": 17.719785690307617,
      "activations/layer12_attention_weight_min": -24.62028694152832,
      "activations/layer13_attention_weight_max": 35.230316162109375,
      "activations/layer13_attention_weight_min": -28.007211685180664,
      "activations/layer14_attention_weight_max": 33.04121398925781,
      "activations/layer14_attention_weight_min": -29.049163818359375,
      "activations/layer15_attention_weight_max": 31.402742385864258,
      "activations/layer15_attention_weight_min": -27.77147674560547,
      "activations/layer16_attention_weight_max": 26.96044921875,
      "activations/layer16_attention_weight_min": -27.025522232055664,
      "activations/layer17_attention_weight_max": 48.708885192871094,
      "activations/layer17_attention_weight_min": -41.37912368774414,
      "activations/layer18_attention_weight_max": 40.38119125366211,
      "activations/layer18_attention_weight_min": -35.25449752807617,
      "activations/layer19_attention_weight_max": 19.616735458374023,
      "activations/layer19_attention_weight_min": -21.948162078857422,
      "activations/layer1_attention_weight_max": 16.929527282714844,
      "activations/layer1_attention_weight_min": -14.927064895629883,
      "activations/layer20_attention_weight_max": 20.25988006591797,
      "activations/layer20_attention_weight_min": -20.964054107666016,
      "activations/layer21_attention_weight_max": 31.4124813079834,
      "activations/layer21_attention_weight_min": -22.418519973754883,
      "activations/layer22_attention_weight_max": 28.425649642944336,
      "activations/layer22_attention_weight_min": -26.531696319580078,
      "activations/layer23_attention_weight_max": 33.9379997253418,
      "activations/layer23_attention_weight_min": -24.14150047302246,
      "activations/layer2_attention_weight_max": 33.09993362426758,
      "activations/layer2_attention_weight_min": -32.66660690307617,
      "activations/layer3_attention_weight_max": 90.91958618164062,
      "activations/layer3_attention_weight_min": -89.62747955322266,
      "activations/layer4_attention_weight_max": 95.69859313964844,
      "activations/layer4_attention_weight_min": -90.48063659667969,
      "activations/layer5_attention_weight_max": 75.17236328125,
      "activations/layer5_attention_weight_min": -74.90933990478516,
      "activations/layer6_attention_weight_max": 54.007259368896484,
      "activations/layer6_attention_weight_min": -54.36713790893555,
      "activations/layer7_attention_weight_max": 75.61332702636719,
      "activations/layer7_attention_weight_min": -75.99793243408203,
      "activations/layer8_attention_weight_max": 44.454261779785156,
      "activations/layer8_attention_weight_min": -46.760643005371094,
      "activations/layer9_attention_weight_max": 47.196685791015625,
      "activations/layer9_attention_weight_min": -44.93522262573242,
      "epoch": 10.39,
      "learning_rate": 8.38655303030303e-05,
      "loss": 2.7835,
      "step": 178750
    },
    {
      "activations/layer0_attention_weight_max": 14.555089950561523,
      "activations/layer0_attention_weight_min": -14.412456512451172,
      "activations/layer10_attention_weight_max": 39.726497650146484,
      "activations/layer10_attention_weight_min": -38.341575622558594,
      "activations/layer11_attention_weight_max": 34.87302780151367,
      "activations/layer11_attention_weight_min": -35.06420135498047,
      "activations/layer12_attention_weight_max": 22.376256942749023,
      "activations/layer12_attention_weight_min": -25.799095153808594,
      "activations/layer13_attention_weight_max": 42.5859260559082,
      "activations/layer13_attention_weight_min": -31.03521728515625,
      "activations/layer14_attention_weight_max": 48.58930969238281,
      "activations/layer14_attention_weight_min": -32.133323669433594,
      "activations/layer15_attention_weight_max": 39.886474609375,
      "activations/layer15_attention_weight_min": -30.592987060546875,
      "activations/layer16_attention_weight_max": 34.353599548339844,
      "activations/layer16_attention_weight_min": -31.216699600219727,
      "activations/layer17_attention_weight_max": 60.77095413208008,
      "activations/layer17_attention_weight_min": -48.94813919067383,
      "activations/layer18_attention_weight_max": 51.0623664855957,
      "activations/layer18_attention_weight_min": -40.91820526123047,
      "activations/layer19_attention_weight_max": 24.700674057006836,
      "activations/layer19_attention_weight_min": -23.8466739654541,
      "activations/layer1_attention_weight_max": 18.458948135375977,
      "activations/layer1_attention_weight_min": -16.178342819213867,
      "activations/layer20_attention_weight_max": 25.96840476989746,
      "activations/layer20_attention_weight_min": -21.45596694946289,
      "activations/layer21_attention_weight_max": 40.53279113769531,
      "activations/layer21_attention_weight_min": -22.553884506225586,
      "activations/layer22_attention_weight_max": 33.940650939941406,
      "activations/layer22_attention_weight_min": -26.657630920410156,
      "activations/layer23_attention_weight_max": 42.197669982910156,
      "activations/layer23_attention_weight_min": -25.372711181640625,
      "activations/layer2_attention_weight_max": 32.92127227783203,
      "activations/layer2_attention_weight_min": -31.127614974975586,
      "activations/layer3_attention_weight_max": 94.00701141357422,
      "activations/layer3_attention_weight_min": -91.4358901977539,
      "activations/layer4_attention_weight_max": 99.84517669677734,
      "activations/layer4_attention_weight_min": -97.48187255859375,
      "activations/layer5_attention_weight_max": 81.53050994873047,
      "activations/layer5_attention_weight_min": -81.62129211425781,
      "activations/layer6_attention_weight_max": 56.934146881103516,
      "activations/layer6_attention_weight_min": -55.74208068847656,
      "activations/layer7_attention_weight_max": 78.34950256347656,
      "activations/layer7_attention_weight_min": -78.90267944335938,
      "activations/layer8_attention_weight_max": 50.565608978271484,
      "activations/layer8_attention_weight_min": -49.87495040893555,
      "activations/layer9_attention_weight_max": 58.0162353515625,
      "activations/layer9_attention_weight_min": -54.968563079833984,
      "epoch": 10.39,
      "learning_rate": 8.38465909090909e-05,
      "loss": 2.7893,
      "step": 178800
    },
    {
      "activations/layer0_attention_weight_max": 14.649247169494629,
      "activations/layer0_attention_weight_min": -14.509217262268066,
      "activations/layer10_attention_weight_max": 44.74454879760742,
      "activations/layer10_attention_weight_min": -42.32071304321289,
      "activations/layer11_attention_weight_max": 41.565250396728516,
      "activations/layer11_attention_weight_min": -37.22074508666992,
      "activations/layer12_attention_weight_max": 22.440284729003906,
      "activations/layer12_attention_weight_min": -25.26671600341797,
      "activations/layer13_attention_weight_max": 58.73163986206055,
      "activations/layer13_attention_weight_min": -36.8266716003418,
      "activations/layer14_attention_weight_max": 71.45882415771484,
      "activations/layer14_attention_weight_min": -40.35029983520508,
      "activations/layer15_attention_weight_max": 61.540836334228516,
      "activations/layer15_attention_weight_min": -38.820343017578125,
      "activations/layer16_attention_weight_max": 32.25638198852539,
      "activations/layer16_attention_weight_min": -29.881305694580078,
      "activations/layer17_attention_weight_max": 59.80999755859375,
      "activations/layer17_attention_weight_min": -51.5974006652832,
      "activations/layer18_attention_weight_max": 50.235233306884766,
      "activations/layer18_attention_weight_min": -45.00406265258789,
      "activations/layer19_attention_weight_max": 23.51405143737793,
      "activations/layer19_attention_weight_min": -22.104873657226562,
      "activations/layer1_attention_weight_max": 17.525033950805664,
      "activations/layer1_attention_weight_min": -15.097192764282227,
      "activations/layer20_attention_weight_max": 25.790056228637695,
      "activations/layer20_attention_weight_min": -20.881946563720703,
      "activations/layer21_attention_weight_max": 50.1114616394043,
      "activations/layer21_attention_weight_min": -26.605436325073242,
      "activations/layer22_attention_weight_max": 34.48345947265625,
      "activations/layer22_attention_weight_min": -23.987873077392578,
      "activations/layer23_attention_weight_max": 37.54848861694336,
      "activations/layer23_attention_weight_min": -23.10309600830078,
      "activations/layer2_attention_weight_max": 34.96864318847656,
      "activations/layer2_attention_weight_min": -31.01839828491211,
      "activations/layer3_attention_weight_max": 101.54804992675781,
      "activations/layer3_attention_weight_min": -99.20274353027344,
      "activations/layer4_attention_weight_max": 106.460205078125,
      "activations/layer4_attention_weight_min": -97.89331817626953,
      "activations/layer5_attention_weight_max": 77.03408813476562,
      "activations/layer5_attention_weight_min": -89.20585632324219,
      "activations/layer6_attention_weight_max": 60.86650848388672,
      "activations/layer6_attention_weight_min": -55.66279220581055,
      "activations/layer7_attention_weight_max": 83.01216125488281,
      "activations/layer7_attention_weight_min": -77.03641510009766,
      "activations/layer8_attention_weight_max": 52.4814567565918,
      "activations/layer8_attention_weight_min": -49.0705451965332,
      "activations/layer9_attention_weight_max": 62.90627670288086,
      "activations/layer9_attention_weight_min": -57.55869674682617,
      "epoch": 10.39,
      "learning_rate": 8.382765151515152e-05,
      "loss": 2.773,
      "step": 178850
    },
    {
      "activations/layer0_attention_weight_max": 14.905881881713867,
      "activations/layer0_attention_weight_min": -14.697233200073242,
      "activations/layer10_attention_weight_max": 36.986976623535156,
      "activations/layer10_attention_weight_min": -37.359657287597656,
      "activations/layer11_attention_weight_max": 33.17798614501953,
      "activations/layer11_attention_weight_min": -33.89478302001953,
      "activations/layer12_attention_weight_max": 19.521223068237305,
      "activations/layer12_attention_weight_min": -24.63060188293457,
      "activations/layer13_attention_weight_max": 47.08091735839844,
      "activations/layer13_attention_weight_min": -30.664230346679688,
      "activations/layer14_attention_weight_max": 38.266929626464844,
      "activations/layer14_attention_weight_min": -29.59677505493164,
      "activations/layer15_attention_weight_max": 33.50898742675781,
      "activations/layer15_attention_weight_min": -29.009445190429688,
      "activations/layer16_attention_weight_max": 29.751995086669922,
      "activations/layer16_attention_weight_min": -28.673463821411133,
      "activations/layer17_attention_weight_max": 51.522823333740234,
      "activations/layer17_attention_weight_min": -43.11880874633789,
      "activations/layer18_attention_weight_max": 47.5352668762207,
      "activations/layer18_attention_weight_min": -37.612728118896484,
      "activations/layer19_attention_weight_max": 24.744308471679688,
      "activations/layer19_attention_weight_min": -21.97857093811035,
      "activations/layer1_attention_weight_max": 16.933618545532227,
      "activations/layer1_attention_weight_min": -14.78630542755127,
      "activations/layer20_attention_weight_max": 22.542570114135742,
      "activations/layer20_attention_weight_min": -22.047311782836914,
      "activations/layer21_attention_weight_max": 40.73371505737305,
      "activations/layer21_attention_weight_min": -24.97061538696289,
      "activations/layer22_attention_weight_max": 28.124183654785156,
      "activations/layer22_attention_weight_min": -26.27841567993164,
      "activations/layer23_attention_weight_max": 35.353118896484375,
      "activations/layer23_attention_weight_min": -24.055761337280273,
      "activations/layer2_attention_weight_max": 32.50066375732422,
      "activations/layer2_attention_weight_min": -29.597469329833984,
      "activations/layer3_attention_weight_max": 94.75907135009766,
      "activations/layer3_attention_weight_min": -97.2732162475586,
      "activations/layer4_attention_weight_max": 103.71385955810547,
      "activations/layer4_attention_weight_min": -95.6729965209961,
      "activations/layer5_attention_weight_max": 82.55987548828125,
      "activations/layer5_attention_weight_min": -81.17947387695312,
      "activations/layer6_attention_weight_max": 60.09608459472656,
      "activations/layer6_attention_weight_min": -56.055580139160156,
      "activations/layer7_attention_weight_max": 75.81269073486328,
      "activations/layer7_attention_weight_min": -72.63410949707031,
      "activations/layer8_attention_weight_max": 47.3104248046875,
      "activations/layer8_attention_weight_min": -50.91994857788086,
      "activations/layer9_attention_weight_max": 51.07436752319336,
      "activations/layer9_attention_weight_min": -50.82258224487305,
      "epoch": 10.4,
      "learning_rate": 8.38087121212121e-05,
      "loss": 2.7821,
      "step": 178900
    },
    {
      "activations/layer0_attention_weight_max": 15.619789123535156,
      "activations/layer0_attention_weight_min": -14.536131858825684,
      "activations/layer10_attention_weight_max": 38.67620849609375,
      "activations/layer10_attention_weight_min": -38.09782028198242,
      "activations/layer11_attention_weight_max": 32.58660888671875,
      "activations/layer11_attention_weight_min": -33.62146759033203,
      "activations/layer12_attention_weight_max": 19.72369384765625,
      "activations/layer12_attention_weight_min": -25.71529769897461,
      "activations/layer13_attention_weight_max": 44.89202117919922,
      "activations/layer13_attention_weight_min": -32.17411422729492,
      "activations/layer14_attention_weight_max": 46.05821228027344,
      "activations/layer14_attention_weight_min": -34.340110778808594,
      "activations/layer15_attention_weight_max": 37.813453674316406,
      "activations/layer15_attention_weight_min": -30.176803588867188,
      "activations/layer16_attention_weight_max": 33.37849044799805,
      "activations/layer16_attention_weight_min": -32.94413375854492,
      "activations/layer17_attention_weight_max": 56.64430618286133,
      "activations/layer17_attention_weight_min": -45.89128494262695,
      "activations/layer18_attention_weight_max": 52.685184478759766,
      "activations/layer18_attention_weight_min": -38.56000900268555,
      "activations/layer19_attention_weight_max": 25.211315155029297,
      "activations/layer19_attention_weight_min": -21.643125534057617,
      "activations/layer1_attention_weight_max": 17.357988357543945,
      "activations/layer1_attention_weight_min": -15.214900016784668,
      "activations/layer20_attention_weight_max": 25.365188598632812,
      "activations/layer20_attention_weight_min": -21.79450225830078,
      "activations/layer21_attention_weight_max": 41.00416946411133,
      "activations/layer21_attention_weight_min": -24.77042007446289,
      "activations/layer22_attention_weight_max": 34.41554641723633,
      "activations/layer22_attention_weight_min": -26.219629287719727,
      "activations/layer23_attention_weight_max": 42.681884765625,
      "activations/layer23_attention_weight_min": -24.68523406982422,
      "activations/layer2_attention_weight_max": 30.949951171875,
      "activations/layer2_attention_weight_min": -30.740432739257812,
      "activations/layer3_attention_weight_max": 94.13709259033203,
      "activations/layer3_attention_weight_min": -96.45548248291016,
      "activations/layer4_attention_weight_max": 103.574462890625,
      "activations/layer4_attention_weight_min": -96.91316223144531,
      "activations/layer5_attention_weight_max": 77.12258911132812,
      "activations/layer5_attention_weight_min": -80.97071075439453,
      "activations/layer6_attention_weight_max": 57.11323547363281,
      "activations/layer6_attention_weight_min": -54.937232971191406,
      "activations/layer7_attention_weight_max": 80.99836730957031,
      "activations/layer7_attention_weight_min": -77.85250091552734,
      "activations/layer8_attention_weight_max": 49.09481430053711,
      "activations/layer8_attention_weight_min": -49.54473876953125,
      "activations/layer9_attention_weight_max": 50.27071762084961,
      "activations/layer9_attention_weight_min": -50.39647674560547,
      "epoch": 10.4,
      "learning_rate": 8.378977272727272e-05,
      "loss": 2.797,
      "step": 178950
    },
    {
      "activations/layer0_attention_weight_max": 15.410118103027344,
      "activations/layer0_attention_weight_min": -14.454201698303223,
      "activations/layer10_attention_weight_max": 37.91460418701172,
      "activations/layer10_attention_weight_min": -37.21675109863281,
      "activations/layer11_attention_weight_max": 30.299606323242188,
      "activations/layer11_attention_weight_min": -31.945228576660156,
      "activations/layer12_attention_weight_max": 18.877506256103516,
      "activations/layer12_attention_weight_min": -28.936569213867188,
      "activations/layer13_attention_weight_max": 40.9711799621582,
      "activations/layer13_attention_weight_min": -30.758655548095703,
      "activations/layer14_attention_weight_max": 34.56671142578125,
      "activations/layer14_attention_weight_min": -31.485639572143555,
      "activations/layer15_attention_weight_max": 32.53085708618164,
      "activations/layer15_attention_weight_min": -28.9449462890625,
      "activations/layer16_attention_weight_max": 29.044958114624023,
      "activations/layer16_attention_weight_min": -29.50710105895996,
      "activations/layer17_attention_weight_max": 52.22495651245117,
      "activations/layer17_attention_weight_min": -41.025360107421875,
      "activations/layer18_attention_weight_max": 44.079681396484375,
      "activations/layer18_attention_weight_min": -35.63640594482422,
      "activations/layer19_attention_weight_max": 25.08009147644043,
      "activations/layer19_attention_weight_min": -21.186561584472656,
      "activations/layer1_attention_weight_max": 17.503297805786133,
      "activations/layer1_attention_weight_min": -14.347963333129883,
      "activations/layer20_attention_weight_max": 21.816898345947266,
      "activations/layer20_attention_weight_min": -21.092039108276367,
      "activations/layer21_attention_weight_max": 36.01702880859375,
      "activations/layer21_attention_weight_min": -22.644195556640625,
      "activations/layer22_attention_weight_max": 33.90890121459961,
      "activations/layer22_attention_weight_min": -26.30732536315918,
      "activations/layer23_attention_weight_max": 39.42884826660156,
      "activations/layer23_attention_weight_min": -22.820653915405273,
      "activations/layer2_attention_weight_max": 30.195615768432617,
      "activations/layer2_attention_weight_min": -29.27176284790039,
      "activations/layer3_attention_weight_max": 89.71329498291016,
      "activations/layer3_attention_weight_min": -88.54173278808594,
      "activations/layer4_attention_weight_max": 98.6318588256836,
      "activations/layer4_attention_weight_min": -90.9215087890625,
      "activations/layer5_attention_weight_max": 75.61799621582031,
      "activations/layer5_attention_weight_min": -80.29849243164062,
      "activations/layer6_attention_weight_max": 52.48623275756836,
      "activations/layer6_attention_weight_min": -51.94491195678711,
      "activations/layer7_attention_weight_max": 76.00204467773438,
      "activations/layer7_attention_weight_min": -71.5845947265625,
      "activations/layer8_attention_weight_max": 43.30201721191406,
      "activations/layer8_attention_weight_min": -44.60585403442383,
      "activations/layer9_attention_weight_max": 48.21208190917969,
      "activations/layer9_attention_weight_min": -47.443077087402344,
      "epoch": 10.4,
      "learning_rate": 8.377083333333333e-05,
      "loss": 2.7962,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4736,
      "eval_samples_per_second": 506.753,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4736,
      "eval_openwebtext_samples_per_second": 506.753,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_wikitext_loss": 2.970703125,
      "eval_wikitext_ppl": 19.505629671380603,
      "eval_wikitext_runtime": 1.9636,
      "eval_wikitext_samples_per_second": 232.226,
      "step": 179000
    },
    {
      "epoch": 10.4,
      "eval_lambada_loss": 2.607421875,
      "eval_lambada_ppl": 13.564035954639001,
      "eval_lambada_runtime": 9.5417,
      "eval_lambada_samples_per_second": 510.287,
      "step": 179000
    },
    {
      "activations/layer0_attention_weight_max": 14.479623794555664,
      "activations/layer0_attention_weight_min": -14.562255859375,
      "activations/layer10_attention_weight_max": 41.91862487792969,
      "activations/layer10_attention_weight_min": -40.3642463684082,
      "activations/layer11_attention_weight_max": 34.952598571777344,
      "activations/layer11_attention_weight_min": -36.792755126953125,
      "activations/layer12_attention_weight_max": 17.978422164916992,
      "activations/layer12_attention_weight_min": -24.64469337463379,
      "activations/layer13_attention_weight_max": 37.429107666015625,
      "activations/layer13_attention_weight_min": -30.636409759521484,
      "activations/layer14_attention_weight_max": 37.45606994628906,
      "activations/layer14_attention_weight_min": -33.024940490722656,
      "activations/layer15_attention_weight_max": 34.222412109375,
      "activations/layer15_attention_weight_min": -32.46886444091797,
      "activations/layer16_attention_weight_max": 35.1796875,
      "activations/layer16_attention_weight_min": -31.477750778198242,
      "activations/layer17_attention_weight_max": 56.21018600463867,
      "activations/layer17_attention_weight_min": -48.50347137451172,
      "activations/layer18_attention_weight_max": 51.48681640625,
      "activations/layer18_attention_weight_min": -39.40087127685547,
      "activations/layer19_attention_weight_max": 22.188106536865234,
      "activations/layer19_attention_weight_min": -22.083065032958984,
      "activations/layer1_attention_weight_max": 16.431474685668945,
      "activations/layer1_attention_weight_min": -16.650836944580078,
      "activations/layer20_attention_weight_max": 24.62527084350586,
      "activations/layer20_attention_weight_min": -21.2561092376709,
      "activations/layer21_attention_weight_max": 45.24890899658203,
      "activations/layer21_attention_weight_min": -26.996749877929688,
      "activations/layer22_attention_weight_max": 31.745622634887695,
      "activations/layer22_attention_weight_min": -27.2227725982666,
      "activations/layer23_attention_weight_max": 36.30885314941406,
      "activations/layer23_attention_weight_min": -25.689937591552734,
      "activations/layer2_attention_weight_max": 32.792572021484375,
      "activations/layer2_attention_weight_min": -30.433635711669922,
      "activations/layer3_attention_weight_max": 95.28914642333984,
      "activations/layer3_attention_weight_min": -94.76996612548828,
      "activations/layer4_attention_weight_max": 101.7098617553711,
      "activations/layer4_attention_weight_min": -96.24380493164062,
      "activations/layer5_attention_weight_max": 80.71971130371094,
      "activations/layer5_attention_weight_min": -78.29483032226562,
      "activations/layer6_attention_weight_max": 55.67776870727539,
      "activations/layer6_attention_weight_min": -55.206787109375,
      "activations/layer7_attention_weight_max": 78.15377044677734,
      "activations/layer7_attention_weight_min": -75.47560119628906,
      "activations/layer8_attention_weight_max": 46.83771896362305,
      "activations/layer8_attention_weight_min": -48.80783462524414,
      "activations/layer9_attention_weight_max": 53.50905227661133,
      "activations/layer9_attention_weight_min": -55.352027893066406,
      "epoch": 10.4,
      "learning_rate": 8.375189393939392e-05,
      "loss": 2.7827,
      "step": 179050
    },
    {
      "activations/layer0_attention_weight_max": 15.055770874023438,
      "activations/layer0_attention_weight_min": -14.747020721435547,
      "activations/layer10_attention_weight_max": 35.911155700683594,
      "activations/layer10_attention_weight_min": -34.42996597290039,
      "activations/layer11_attention_weight_max": 33.24417495727539,
      "activations/layer11_attention_weight_min": -33.54286193847656,
      "activations/layer12_attention_weight_max": 17.857088088989258,
      "activations/layer12_attention_weight_min": -31.050968170166016,
      "activations/layer13_attention_weight_max": 42.970726013183594,
      "activations/layer13_attention_weight_min": -31.2196102142334,
      "activations/layer14_attention_weight_max": 37.37217330932617,
      "activations/layer14_attention_weight_min": -32.09164810180664,
      "activations/layer15_attention_weight_max": 34.156455993652344,
      "activations/layer15_attention_weight_min": -29.295799255371094,
      "activations/layer16_attention_weight_max": 29.168485641479492,
      "activations/layer16_attention_weight_min": -28.55792236328125,
      "activations/layer17_attention_weight_max": 53.04412841796875,
      "activations/layer17_attention_weight_min": -43.03350067138672,
      "activations/layer18_attention_weight_max": 43.66699981689453,
      "activations/layer18_attention_weight_min": -37.51287841796875,
      "activations/layer19_attention_weight_max": 22.412120819091797,
      "activations/layer19_attention_weight_min": -21.999807357788086,
      "activations/layer1_attention_weight_max": 17.364124298095703,
      "activations/layer1_attention_weight_min": -15.711042404174805,
      "activations/layer20_attention_weight_max": 21.691434860229492,
      "activations/layer20_attention_weight_min": -22.5313777923584,
      "activations/layer21_attention_weight_max": 38.25331497192383,
      "activations/layer21_attention_weight_min": -28.203397750854492,
      "activations/layer22_attention_weight_max": 32.165802001953125,
      "activations/layer22_attention_weight_min": -25.94420623779297,
      "activations/layer23_attention_weight_max": 40.0821533203125,
      "activations/layer23_attention_weight_min": -28.27124786376953,
      "activations/layer2_attention_weight_max": 33.005287170410156,
      "activations/layer2_attention_weight_min": -30.57025718688965,
      "activations/layer3_attention_weight_max": 99.57442474365234,
      "activations/layer3_attention_weight_min": -92.7398452758789,
      "activations/layer4_attention_weight_max": 101.49872589111328,
      "activations/layer4_attention_weight_min": -93.20297241210938,
      "activations/layer5_attention_weight_max": 75.50962829589844,
      "activations/layer5_attention_weight_min": -78.07693481445312,
      "activations/layer6_attention_weight_max": 54.34965133666992,
      "activations/layer6_attention_weight_min": -54.49648666381836,
      "activations/layer7_attention_weight_max": 75.38201904296875,
      "activations/layer7_attention_weight_min": -73.09464263916016,
      "activations/layer8_attention_weight_max": 46.36144256591797,
      "activations/layer8_attention_weight_min": -48.43757247924805,
      "activations/layer9_attention_weight_max": 54.89374542236328,
      "activations/layer9_attention_weight_min": -50.70271682739258,
      "epoch": 10.41,
      "learning_rate": 8.373295454545454e-05,
      "loss": 2.7863,
      "step": 179100
    },
    {
      "activations/layer0_attention_weight_max": 15.455406188964844,
      "activations/layer0_attention_weight_min": -14.165552139282227,
      "activations/layer10_attention_weight_max": 37.00477981567383,
      "activations/layer10_attention_weight_min": -35.47017288208008,
      "activations/layer11_attention_weight_max": 31.756244659423828,
      "activations/layer11_attention_weight_min": -32.472389221191406,
      "activations/layer12_attention_weight_max": 18.42293357849121,
      "activations/layer12_attention_weight_min": -24.71976089477539,
      "activations/layer13_attention_weight_max": 40.29304122924805,
      "activations/layer13_attention_weight_min": -31.036882400512695,
      "activations/layer14_attention_weight_max": 38.303279876708984,
      "activations/layer14_attention_weight_min": -32.1566162109375,
      "activations/layer15_attention_weight_max": 35.16633605957031,
      "activations/layer15_attention_weight_min": -29.808231353759766,
      "activations/layer16_attention_weight_max": 31.810640335083008,
      "activations/layer16_attention_weight_min": -31.03342628479004,
      "activations/layer17_attention_weight_max": 54.35078048706055,
      "activations/layer17_attention_weight_min": -46.64528274536133,
      "activations/layer18_attention_weight_max": 52.09134292602539,
      "activations/layer18_attention_weight_min": -39.28335952758789,
      "activations/layer19_attention_weight_max": 24.684165954589844,
      "activations/layer19_attention_weight_min": -21.253957748413086,
      "activations/layer1_attention_weight_max": 16.755109786987305,
      "activations/layer1_attention_weight_min": -14.968997955322266,
      "activations/layer20_attention_weight_max": 24.6258544921875,
      "activations/layer20_attention_weight_min": -23.342016220092773,
      "activations/layer21_attention_weight_max": 39.805904388427734,
      "activations/layer21_attention_weight_min": -25.23045539855957,
      "activations/layer22_attention_weight_max": 33.44923782348633,
      "activations/layer22_attention_weight_min": -27.17499542236328,
      "activations/layer23_attention_weight_max": 38.75877380371094,
      "activations/layer23_attention_weight_min": -23.306224822998047,
      "activations/layer2_attention_weight_max": 30.604352951049805,
      "activations/layer2_attention_weight_min": -28.083209991455078,
      "activations/layer3_attention_weight_max": 87.51937866210938,
      "activations/layer3_attention_weight_min": -87.47590637207031,
      "activations/layer4_attention_weight_max": 95.93000030517578,
      "activations/layer4_attention_weight_min": -92.57283020019531,
      "activations/layer5_attention_weight_max": 74.2311782836914,
      "activations/layer5_attention_weight_min": -80.51316833496094,
      "activations/layer6_attention_weight_max": 53.051963806152344,
      "activations/layer6_attention_weight_min": -55.32025909423828,
      "activations/layer7_attention_weight_max": 73.4996109008789,
      "activations/layer7_attention_weight_min": -69.884521484375,
      "activations/layer8_attention_weight_max": 43.534515380859375,
      "activations/layer8_attention_weight_min": -44.007354736328125,
      "activations/layer9_attention_weight_max": 47.541561126708984,
      "activations/layer9_attention_weight_min": -48.325843811035156,
      "epoch": 10.41,
      "learning_rate": 8.371401515151515e-05,
      "loss": 2.7994,
      "step": 179150
    },
    {
      "activations/layer0_attention_weight_max": 15.045129776000977,
      "activations/layer0_attention_weight_min": -14.261285781860352,
      "activations/layer10_attention_weight_max": 38.065284729003906,
      "activations/layer10_attention_weight_min": -37.112308502197266,
      "activations/layer11_attention_weight_max": 33.115699768066406,
      "activations/layer11_attention_weight_min": -35.02613067626953,
      "activations/layer12_attention_weight_max": 21.446231842041016,
      "activations/layer12_attention_weight_min": -25.226804733276367,
      "activations/layer13_attention_weight_max": 50.74528884887695,
      "activations/layer13_attention_weight_min": -31.107444763183594,
      "activations/layer14_attention_weight_max": 46.917205810546875,
      "activations/layer14_attention_weight_min": -32.836673736572266,
      "activations/layer15_attention_weight_max": 38.67367935180664,
      "activations/layer15_attention_weight_min": -29.69744300842285,
      "activations/layer16_attention_weight_max": 39.314022064208984,
      "activations/layer16_attention_weight_min": -32.67611312866211,
      "activations/layer17_attention_weight_max": 60.19540023803711,
      "activations/layer17_attention_weight_min": -51.6394157409668,
      "activations/layer18_attention_weight_max": 54.092594146728516,
      "activations/layer18_attention_weight_min": -43.895973205566406,
      "activations/layer19_attention_weight_max": 27.601978302001953,
      "activations/layer19_attention_weight_min": -25.147781372070312,
      "activations/layer1_attention_weight_max": 17.57765007019043,
      "activations/layer1_attention_weight_min": -13.778632164001465,
      "activations/layer20_attention_weight_max": 27.168621063232422,
      "activations/layer20_attention_weight_min": -22.929931640625,
      "activations/layer21_attention_weight_max": 45.96071243286133,
      "activations/layer21_attention_weight_min": -26.03911018371582,
      "activations/layer22_attention_weight_max": 35.49607849121094,
      "activations/layer22_attention_weight_min": -26.141965866088867,
      "activations/layer23_attention_weight_max": 45.442909240722656,
      "activations/layer23_attention_weight_min": -23.850692749023438,
      "activations/layer2_attention_weight_max": 31.026853561401367,
      "activations/layer2_attention_weight_min": -30.691688537597656,
      "activations/layer3_attention_weight_max": 93.30280303955078,
      "activations/layer3_attention_weight_min": -91.0882568359375,
      "activations/layer4_attention_weight_max": 96.68267059326172,
      "activations/layer4_attention_weight_min": -92.76959228515625,
      "activations/layer5_attention_weight_max": 74.11622619628906,
      "activations/layer5_attention_weight_min": -79.92044830322266,
      "activations/layer6_attention_weight_max": 54.80628967285156,
      "activations/layer6_attention_weight_min": -53.3288688659668,
      "activations/layer7_attention_weight_max": 77.63337707519531,
      "activations/layer7_attention_weight_min": -73.20479583740234,
      "activations/layer8_attention_weight_max": 43.990543365478516,
      "activations/layer8_attention_weight_min": -45.123878479003906,
      "activations/layer9_attention_weight_max": 49.782859802246094,
      "activations/layer9_attention_weight_min": -48.23005294799805,
      "epoch": 10.41,
      "learning_rate": 8.369507575757574e-05,
      "loss": 2.7905,
      "step": 179200
    },
    {
      "activations/layer0_attention_weight_max": 14.099898338317871,
      "activations/layer0_attention_weight_min": -14.636818885803223,
      "activations/layer10_attention_weight_max": 34.95441436767578,
      "activations/layer10_attention_weight_min": -35.62134552001953,
      "activations/layer11_attention_weight_max": 33.268917083740234,
      "activations/layer11_attention_weight_min": -31.372615814208984,
      "activations/layer12_attention_weight_max": 18.648534774780273,
      "activations/layer12_attention_weight_min": -31.175928115844727,
      "activations/layer13_attention_weight_max": 37.098899841308594,
      "activations/layer13_attention_weight_min": -31.614221572875977,
      "activations/layer14_attention_weight_max": 39.92088317871094,
      "activations/layer14_attention_weight_min": -29.694133758544922,
      "activations/layer15_attention_weight_max": 33.531402587890625,
      "activations/layer15_attention_weight_min": -29.53961753845215,
      "activations/layer16_attention_weight_max": 33.943153381347656,
      "activations/layer16_attention_weight_min": -28.373815536499023,
      "activations/layer17_attention_weight_max": 50.02156448364258,
      "activations/layer17_attention_weight_min": -42.92041778564453,
      "activations/layer18_attention_weight_max": 46.77865982055664,
      "activations/layer18_attention_weight_min": -37.33803939819336,
      "activations/layer19_attention_weight_max": 20.921306610107422,
      "activations/layer19_attention_weight_min": -21.49243927001953,
      "activations/layer1_attention_weight_max": 17.612598419189453,
      "activations/layer1_attention_weight_min": -16.038896560668945,
      "activations/layer20_attention_weight_max": 22.793519973754883,
      "activations/layer20_attention_weight_min": -22.99056625366211,
      "activations/layer21_attention_weight_max": 35.96592712402344,
      "activations/layer21_attention_weight_min": -21.1990909576416,
      "activations/layer22_attention_weight_max": 31.81386375427246,
      "activations/layer22_attention_weight_min": -25.84856414794922,
      "activations/layer23_attention_weight_max": 36.081642150878906,
      "activations/layer23_attention_weight_min": -21.885547637939453,
      "activations/layer2_attention_weight_max": 32.596839904785156,
      "activations/layer2_attention_weight_min": -29.739774703979492,
      "activations/layer3_attention_weight_max": 91.0586166381836,
      "activations/layer3_attention_weight_min": -85.42091369628906,
      "activations/layer4_attention_weight_max": 96.62969970703125,
      "activations/layer4_attention_weight_min": -90.97789764404297,
      "activations/layer5_attention_weight_max": 74.84872436523438,
      "activations/layer5_attention_weight_min": -79.20997619628906,
      "activations/layer6_attention_weight_max": 52.9665412902832,
      "activations/layer6_attention_weight_min": -53.27669143676758,
      "activations/layer7_attention_weight_max": 78.79856872558594,
      "activations/layer7_attention_weight_min": -70.37667083740234,
      "activations/layer8_attention_weight_max": 43.730186462402344,
      "activations/layer8_attention_weight_min": -44.869728088378906,
      "activations/layer9_attention_weight_max": 49.522377014160156,
      "activations/layer9_attention_weight_min": -51.32447052001953,
      "epoch": 10.42,
      "learning_rate": 8.367613636363636e-05,
      "loss": 2.7881,
      "step": 179250
    },
    {
      "activations/layer0_attention_weight_max": 16.544376373291016,
      "activations/layer0_attention_weight_min": -14.43665599822998,
      "activations/layer10_attention_weight_max": 35.70510482788086,
      "activations/layer10_attention_weight_min": -33.595924377441406,
      "activations/layer11_attention_weight_max": 29.820892333984375,
      "activations/layer11_attention_weight_min": -30.71371078491211,
      "activations/layer12_attention_weight_max": 23.35748291015625,
      "activations/layer12_attention_weight_min": -25.582101821899414,
      "activations/layer13_attention_weight_max": 38.92272186279297,
      "activations/layer13_attention_weight_min": -31.928119659423828,
      "activations/layer14_attention_weight_max": 35.44892883300781,
      "activations/layer14_attention_weight_min": -29.74452018737793,
      "activations/layer15_attention_weight_max": 34.25809860229492,
      "activations/layer15_attention_weight_min": -29.54788589477539,
      "activations/layer16_attention_weight_max": 30.079387664794922,
      "activations/layer16_attention_weight_min": -26.90330696105957,
      "activations/layer17_attention_weight_max": 50.132469177246094,
      "activations/layer17_attention_weight_min": -42.317535400390625,
      "activations/layer18_attention_weight_max": 44.85752868652344,
      "activations/layer18_attention_weight_min": -36.86919403076172,
      "activations/layer19_attention_weight_max": 21.619482040405273,
      "activations/layer19_attention_weight_min": -21.202558517456055,
      "activations/layer1_attention_weight_max": 16.474519729614258,
      "activations/layer1_attention_weight_min": -14.684624671936035,
      "activations/layer20_attention_weight_max": 20.806987762451172,
      "activations/layer20_attention_weight_min": -21.77605438232422,
      "activations/layer21_attention_weight_max": 35.5595588684082,
      "activations/layer21_attention_weight_min": -21.132736206054688,
      "activations/layer22_attention_weight_max": 31.286497116088867,
      "activations/layer22_attention_weight_min": -24.217437744140625,
      "activations/layer23_attention_weight_max": 34.746116638183594,
      "activations/layer23_attention_weight_min": -21.573978424072266,
      "activations/layer2_attention_weight_max": 30.408607482910156,
      "activations/layer2_attention_weight_min": -30.28870964050293,
      "activations/layer3_attention_weight_max": 83.42585754394531,
      "activations/layer3_attention_weight_min": -89.79714965820312,
      "activations/layer4_attention_weight_max": 88.87071990966797,
      "activations/layer4_attention_weight_min": -88.09481048583984,
      "activations/layer5_attention_weight_max": 68.82467651367188,
      "activations/layer5_attention_weight_min": -80.13434600830078,
      "activations/layer6_attention_weight_max": 50.08717346191406,
      "activations/layer6_attention_weight_min": -49.44846725463867,
      "activations/layer7_attention_weight_max": 72.65934753417969,
      "activations/layer7_attention_weight_min": -72.50838470458984,
      "activations/layer8_attention_weight_max": 45.67240905761719,
      "activations/layer8_attention_weight_min": -44.955482482910156,
      "activations/layer9_attention_weight_max": 49.07059860229492,
      "activations/layer9_attention_weight_min": -45.17612838745117,
      "epoch": 10.42,
      "learning_rate": 8.365719696969696e-05,
      "loss": 2.7897,
      "step": 179300
    },
    {
      "activations/layer0_attention_weight_max": 16.898412704467773,
      "activations/layer0_attention_weight_min": -14.217227935791016,
      "activations/layer10_attention_weight_max": 35.75954055786133,
      "activations/layer10_attention_weight_min": -34.360443115234375,
      "activations/layer11_attention_weight_max": 33.914573669433594,
      "activations/layer11_attention_weight_min": -32.32512283325195,
      "activations/layer12_attention_weight_max": 21.96731185913086,
      "activations/layer12_attention_weight_min": -30.080982208251953,
      "activations/layer13_attention_weight_max": 36.762298583984375,
      "activations/layer13_attention_weight_min": -27.02548599243164,
      "activations/layer14_attention_weight_max": 46.07849884033203,
      "activations/layer14_attention_weight_min": -30.096538543701172,
      "activations/layer15_attention_weight_max": 42.72690963745117,
      "activations/layer15_attention_weight_min": -28.940872192382812,
      "activations/layer16_attention_weight_max": 34.94943618774414,
      "activations/layer16_attention_weight_min": -27.40180206298828,
      "activations/layer17_attention_weight_max": 55.065879821777344,
      "activations/layer17_attention_weight_min": -42.18879318237305,
      "activations/layer18_attention_weight_max": 56.63850402832031,
      "activations/layer18_attention_weight_min": -38.14402389526367,
      "activations/layer19_attention_weight_max": 26.174768447875977,
      "activations/layer19_attention_weight_min": -23.58563804626465,
      "activations/layer1_attention_weight_max": 17.993928909301758,
      "activations/layer1_attention_weight_min": -14.603313446044922,
      "activations/layer20_attention_weight_max": 27.27936363220215,
      "activations/layer20_attention_weight_min": -21.482097625732422,
      "activations/layer21_attention_weight_max": 43.17963409423828,
      "activations/layer21_attention_weight_min": -24.8552303314209,
      "activations/layer22_attention_weight_max": 35.55883026123047,
      "activations/layer22_attention_weight_min": -25.91609001159668,
      "activations/layer23_attention_weight_max": 39.19124221801758,
      "activations/layer23_attention_weight_min": -23.438474655151367,
      "activations/layer2_attention_weight_max": 33.98417663574219,
      "activations/layer2_attention_weight_min": -30.50924301147461,
      "activations/layer3_attention_weight_max": 95.3418197631836,
      "activations/layer3_attention_weight_min": -95.56663513183594,
      "activations/layer4_attention_weight_max": 98.6438217163086,
      "activations/layer4_attention_weight_min": -93.14656829833984,
      "activations/layer5_attention_weight_max": 74.15287780761719,
      "activations/layer5_attention_weight_min": -80.3175277709961,
      "activations/layer6_attention_weight_max": 53.82844161987305,
      "activations/layer6_attention_weight_min": -51.67947769165039,
      "activations/layer7_attention_weight_max": 76.67093658447266,
      "activations/layer7_attention_weight_min": -69.44086456298828,
      "activations/layer8_attention_weight_max": 44.739418029785156,
      "activations/layer8_attention_weight_min": -45.72816467285156,
      "activations/layer9_attention_weight_max": 55.5088005065918,
      "activations/layer9_attention_weight_min": -51.48946762084961,
      "epoch": 10.42,
      "learning_rate": 8.363825757575757e-05,
      "loss": 2.7983,
      "step": 179350
    },
    {
      "activations/layer0_attention_weight_max": 15.051088333129883,
      "activations/layer0_attention_weight_min": -14.847554206848145,
      "activations/layer10_attention_weight_max": 37.050880432128906,
      "activations/layer10_attention_weight_min": -36.233154296875,
      "activations/layer11_attention_weight_max": 32.26935958862305,
      "activations/layer11_attention_weight_min": -31.143577575683594,
      "activations/layer12_attention_weight_max": 17.792980194091797,
      "activations/layer12_attention_weight_min": -26.566171646118164,
      "activations/layer13_attention_weight_max": 43.62522888183594,
      "activations/layer13_attention_weight_min": -31.780345916748047,
      "activations/layer14_attention_weight_max": 43.37023162841797,
      "activations/layer14_attention_weight_min": -31.03641128540039,
      "activations/layer15_attention_weight_max": 33.95098876953125,
      "activations/layer15_attention_weight_min": -28.162660598754883,
      "activations/layer16_attention_weight_max": 29.473806381225586,
      "activations/layer16_attention_weight_min": -29.227083206176758,
      "activations/layer17_attention_weight_max": 51.4623908996582,
      "activations/layer17_attention_weight_min": -45.84406280517578,
      "activations/layer18_attention_weight_max": 45.287078857421875,
      "activations/layer18_attention_weight_min": -39.29792022705078,
      "activations/layer19_attention_weight_max": 23.343965530395508,
      "activations/layer19_attention_weight_min": -22.07232666015625,
      "activations/layer1_attention_weight_max": 16.595060348510742,
      "activations/layer1_attention_weight_min": -14.528281211853027,
      "activations/layer20_attention_weight_max": 24.01886558532715,
      "activations/layer20_attention_weight_min": -21.49929428100586,
      "activations/layer21_attention_weight_max": 44.86254119873047,
      "activations/layer21_attention_weight_min": -24.911266326904297,
      "activations/layer22_attention_weight_max": 36.15433883666992,
      "activations/layer22_attention_weight_min": -24.849958419799805,
      "activations/layer23_attention_weight_max": 38.27320861816406,
      "activations/layer23_attention_weight_min": -25.013675689697266,
      "activations/layer2_attention_weight_max": 31.949071884155273,
      "activations/layer2_attention_weight_min": -30.07788848876953,
      "activations/layer3_attention_weight_max": 96.64442443847656,
      "activations/layer3_attention_weight_min": -93.87655639648438,
      "activations/layer4_attention_weight_max": 100.65507507324219,
      "activations/layer4_attention_weight_min": -98.92713928222656,
      "activations/layer5_attention_weight_max": 72.78661346435547,
      "activations/layer5_attention_weight_min": -79.86486053466797,
      "activations/layer6_attention_weight_max": 51.67897415161133,
      "activations/layer6_attention_weight_min": -52.015968322753906,
      "activations/layer7_attention_weight_max": 71.94734954833984,
      "activations/layer7_attention_weight_min": -74.2420425415039,
      "activations/layer8_attention_weight_max": 44.26961898803711,
      "activations/layer8_attention_weight_min": -45.856170654296875,
      "activations/layer9_attention_weight_max": 53.8145751953125,
      "activations/layer9_attention_weight_min": -49.45282745361328,
      "epoch": 10.42,
      "learning_rate": 8.361931818181817e-05,
      "loss": 2.7911,
      "step": 179400
    },
    {
      "activations/layer0_attention_weight_max": 17.50177574157715,
      "activations/layer0_attention_weight_min": -14.409167289733887,
      "activations/layer10_attention_weight_max": 38.50407791137695,
      "activations/layer10_attention_weight_min": -37.08414077758789,
      "activations/layer11_attention_weight_max": 34.19700241088867,
      "activations/layer11_attention_weight_min": -33.497169494628906,
      "activations/layer12_attention_weight_max": 19.307754516601562,
      "activations/layer12_attention_weight_min": -24.64807891845703,
      "activations/layer13_attention_weight_max": 47.76752471923828,
      "activations/layer13_attention_weight_min": -30.16779327392578,
      "activations/layer14_attention_weight_max": 56.588104248046875,
      "activations/layer14_attention_weight_min": -31.8631534576416,
      "activations/layer15_attention_weight_max": 43.976051330566406,
      "activations/layer15_attention_weight_min": -30.062244415283203,
      "activations/layer16_attention_weight_max": 36.1906852722168,
      "activations/layer16_attention_weight_min": -29.686811447143555,
      "activations/layer17_attention_weight_max": 58.81101608276367,
      "activations/layer17_attention_weight_min": -46.2447395324707,
      "activations/layer18_attention_weight_max": 50.64883041381836,
      "activations/layer18_attention_weight_min": -37.7229118347168,
      "activations/layer19_attention_weight_max": 24.78889274597168,
      "activations/layer19_attention_weight_min": -21.703020095825195,
      "activations/layer1_attention_weight_max": 17.492334365844727,
      "activations/layer1_attention_weight_min": -14.233542442321777,
      "activations/layer20_attention_weight_max": 25.4713134765625,
      "activations/layer20_attention_weight_min": -21.33236312866211,
      "activations/layer21_attention_weight_max": 36.37091064453125,
      "activations/layer21_attention_weight_min": -22.22336769104004,
      "activations/layer22_attention_weight_max": 33.2779655456543,
      "activations/layer22_attention_weight_min": -26.713897705078125,
      "activations/layer23_attention_weight_max": 42.89530944824219,
      "activations/layer23_attention_weight_min": -25.69275665283203,
      "activations/layer2_attention_weight_max": 30.43505859375,
      "activations/layer2_attention_weight_min": -30.006633758544922,
      "activations/layer3_attention_weight_max": 87.73123168945312,
      "activations/layer3_attention_weight_min": -90.54395294189453,
      "activations/layer4_attention_weight_max": 95.49681854248047,
      "activations/layer4_attention_weight_min": -95.60799407958984,
      "activations/layer5_attention_weight_max": 75.0015869140625,
      "activations/layer5_attention_weight_min": -80.43531799316406,
      "activations/layer6_attention_weight_max": 56.84471130371094,
      "activations/layer6_attention_weight_min": -54.734561920166016,
      "activations/layer7_attention_weight_max": 77.73503112792969,
      "activations/layer7_attention_weight_min": -76.78545379638672,
      "activations/layer8_attention_weight_max": 48.09536361694336,
      "activations/layer8_attention_weight_min": -50.26311111450195,
      "activations/layer9_attention_weight_max": 52.87980270385742,
      "activations/layer9_attention_weight_min": -52.14567947387695,
      "epoch": 10.43,
      "learning_rate": 8.360037878787877e-05,
      "loss": 2.7919,
      "step": 179450
    },
    {
      "activations/layer0_attention_weight_max": 15.107518196105957,
      "activations/layer0_attention_weight_min": -14.445125579833984,
      "activations/layer10_attention_weight_max": 33.998844146728516,
      "activations/layer10_attention_weight_min": -34.351226806640625,
      "activations/layer11_attention_weight_max": 30.362411499023438,
      "activations/layer11_attention_weight_min": -32.94823455810547,
      "activations/layer12_attention_weight_max": 16.951770782470703,
      "activations/layer12_attention_weight_min": -25.245988845825195,
      "activations/layer13_attention_weight_max": 36.655426025390625,
      "activations/layer13_attention_weight_min": -27.255802154541016,
      "activations/layer14_attention_weight_max": 36.289649963378906,
      "activations/layer14_attention_weight_min": -28.82440757751465,
      "activations/layer15_attention_weight_max": 34.38748550415039,
      "activations/layer15_attention_weight_min": -27.737947463989258,
      "activations/layer16_attention_weight_max": 31.03154182434082,
      "activations/layer16_attention_weight_min": -27.27959442138672,
      "activations/layer17_attention_weight_max": 50.157100677490234,
      "activations/layer17_attention_weight_min": -40.234825134277344,
      "activations/layer18_attention_weight_max": 42.76701354980469,
      "activations/layer18_attention_weight_min": -34.13799285888672,
      "activations/layer19_attention_weight_max": 20.561460494995117,
      "activations/layer19_attention_weight_min": -20.880062103271484,
      "activations/layer1_attention_weight_max": 15.965210914611816,
      "activations/layer1_attention_weight_min": -13.984696388244629,
      "activations/layer20_attention_weight_max": 20.809186935424805,
      "activations/layer20_attention_weight_min": -20.614343643188477,
      "activations/layer21_attention_weight_max": 35.161556243896484,
      "activations/layer21_attention_weight_min": -20.871685028076172,
      "activations/layer22_attention_weight_max": 30.90182113647461,
      "activations/layer22_attention_weight_min": -23.35027313232422,
      "activations/layer23_attention_weight_max": 37.33894348144531,
      "activations/layer23_attention_weight_min": -23.189422607421875,
      "activations/layer2_attention_weight_max": 31.26687240600586,
      "activations/layer2_attention_weight_min": -29.838882446289062,
      "activations/layer3_attention_weight_max": 89.40650177001953,
      "activations/layer3_attention_weight_min": -91.2960433959961,
      "activations/layer4_attention_weight_max": 100.16123962402344,
      "activations/layer4_attention_weight_min": -97.77737426757812,
      "activations/layer5_attention_weight_max": 78.40383911132812,
      "activations/layer5_attention_weight_min": -81.55205535888672,
      "activations/layer6_attention_weight_max": 55.39727020263672,
      "activations/layer6_attention_weight_min": -55.56066131591797,
      "activations/layer7_attention_weight_max": 72.1563491821289,
      "activations/layer7_attention_weight_min": -75.19266510009766,
      "activations/layer8_attention_weight_max": 44.27128601074219,
      "activations/layer8_attention_weight_min": -45.9962043762207,
      "activations/layer9_attention_weight_max": 47.09642028808594,
      "activations/layer9_attention_weight_min": -49.057762145996094,
      "epoch": 10.43,
      "learning_rate": 8.358143939393939e-05,
      "loss": 2.8092,
      "step": 179500
    },
    {
      "activations/layer0_attention_weight_max": 15.460609436035156,
      "activations/layer0_attention_weight_min": -14.208456993103027,
      "activations/layer10_attention_weight_max": 40.28969192504883,
      "activations/layer10_attention_weight_min": -36.862064361572266,
      "activations/layer11_attention_weight_max": 32.8655891418457,
      "activations/layer11_attention_weight_min": -32.85956573486328,
      "activations/layer12_attention_weight_max": 18.06252098083496,
      "activations/layer12_attention_weight_min": -25.438247680664062,
      "activations/layer13_attention_weight_max": 38.753108978271484,
      "activations/layer13_attention_weight_min": -27.471261978149414,
      "activations/layer14_attention_weight_max": 36.43930435180664,
      "activations/layer14_attention_weight_min": -30.45450782775879,
      "activations/layer15_attention_weight_max": 36.132911682128906,
      "activations/layer15_attention_weight_min": -29.32489585876465,
      "activations/layer16_attention_weight_max": 28.840837478637695,
      "activations/layer16_attention_weight_min": -28.167701721191406,
      "activations/layer17_attention_weight_max": 51.9210319519043,
      "activations/layer17_attention_weight_min": -40.956058502197266,
      "activations/layer18_attention_weight_max": 41.82795333862305,
      "activations/layer18_attention_weight_min": -36.77806854248047,
      "activations/layer19_attention_weight_max": 20.1911678314209,
      "activations/layer19_attention_weight_min": -22.561279296875,
      "activations/layer1_attention_weight_max": 16.995075225830078,
      "activations/layer1_attention_weight_min": -16.00365447998047,
      "activations/layer20_attention_weight_max": 21.936378479003906,
      "activations/layer20_attention_weight_min": -23.912248611450195,
      "activations/layer21_attention_weight_max": 36.31083679199219,
      "activations/layer21_attention_weight_min": -23.475784301757812,
      "activations/layer22_attention_weight_max": 34.87199020385742,
      "activations/layer22_attention_weight_min": -27.2716007232666,
      "activations/layer23_attention_weight_max": 35.934078216552734,
      "activations/layer23_attention_weight_min": -24.438426971435547,
      "activations/layer2_attention_weight_max": 31.41318130493164,
      "activations/layer2_attention_weight_min": -32.227989196777344,
      "activations/layer3_attention_weight_max": 93.49181365966797,
      "activations/layer3_attention_weight_min": -98.42687225341797,
      "activations/layer4_attention_weight_max": 97.94210815429688,
      "activations/layer4_attention_weight_min": -96.00823211669922,
      "activations/layer5_attention_weight_max": 76.10482788085938,
      "activations/layer5_attention_weight_min": -78.91510009765625,
      "activations/layer6_attention_weight_max": 50.90559387207031,
      "activations/layer6_attention_weight_min": -51.424644470214844,
      "activations/layer7_attention_weight_max": 73.49847412109375,
      "activations/layer7_attention_weight_min": -71.77498626708984,
      "activations/layer8_attention_weight_max": 45.12172317504883,
      "activations/layer8_attention_weight_min": -45.76504898071289,
      "activations/layer9_attention_weight_max": 46.813961029052734,
      "activations/layer9_attention_weight_min": -48.90140914916992,
      "epoch": 10.43,
      "learning_rate": 8.35625e-05,
      "loss": 2.776,
      "step": 179550
    },
    {
      "activations/layer0_attention_weight_max": 14.720990180969238,
      "activations/layer0_attention_weight_min": -14.698324203491211,
      "activations/layer10_attention_weight_max": 38.70748519897461,
      "activations/layer10_attention_weight_min": -40.464420318603516,
      "activations/layer11_attention_weight_max": 31.923866271972656,
      "activations/layer11_attention_weight_min": -34.31480407714844,
      "activations/layer12_attention_weight_max": 18.863548278808594,
      "activations/layer12_attention_weight_min": -24.83013153076172,
      "activations/layer13_attention_weight_max": 44.191917419433594,
      "activations/layer13_attention_weight_min": -32.96059036254883,
      "activations/layer14_attention_weight_max": 44.84508514404297,
      "activations/layer14_attention_weight_min": -31.347843170166016,
      "activations/layer15_attention_weight_max": 37.283870697021484,
      "activations/layer15_attention_weight_min": -28.818714141845703,
      "activations/layer16_attention_weight_max": 30.08371925354004,
      "activations/layer16_attention_weight_min": -28.931943893432617,
      "activations/layer17_attention_weight_max": 50.84885787963867,
      "activations/layer17_attention_weight_min": -42.113853454589844,
      "activations/layer18_attention_weight_max": 42.84575653076172,
      "activations/layer18_attention_weight_min": -36.9325065612793,
      "activations/layer19_attention_weight_max": 21.637588500976562,
      "activations/layer19_attention_weight_min": -20.451078414916992,
      "activations/layer1_attention_weight_max": 16.156326293945312,
      "activations/layer1_attention_weight_min": -15.399572372436523,
      "activations/layer20_attention_weight_max": 22.28969383239746,
      "activations/layer20_attention_weight_min": -23.48691177368164,
      "activations/layer21_attention_weight_max": 36.834110260009766,
      "activations/layer21_attention_weight_min": -24.738548278808594,
      "activations/layer22_attention_weight_max": 31.210826873779297,
      "activations/layer22_attention_weight_min": -25.93682289123535,
      "activations/layer23_attention_weight_max": 37.33184051513672,
      "activations/layer23_attention_weight_min": -26.378150939941406,
      "activations/layer2_attention_weight_max": 31.200225830078125,
      "activations/layer2_attention_weight_min": -30.748458862304688,
      "activations/layer3_attention_weight_max": 91.03105926513672,
      "activations/layer3_attention_weight_min": -88.29170989990234,
      "activations/layer4_attention_weight_max": 99.79358673095703,
      "activations/layer4_attention_weight_min": -94.17037200927734,
      "activations/layer5_attention_weight_max": 75.53033447265625,
      "activations/layer5_attention_weight_min": -79.61515808105469,
      "activations/layer6_attention_weight_max": 55.80696487426758,
      "activations/layer6_attention_weight_min": -51.33658218383789,
      "activations/layer7_attention_weight_max": 74.88326263427734,
      "activations/layer7_attention_weight_min": -71.50003051757812,
      "activations/layer8_attention_weight_max": 43.57585525512695,
      "activations/layer8_attention_weight_min": -49.532142639160156,
      "activations/layer9_attention_weight_max": 47.46590042114258,
      "activations/layer9_attention_weight_min": -50.25870895385742,
      "epoch": 10.44,
      "learning_rate": 8.354356060606059e-05,
      "loss": 2.7765,
      "step": 179600
    },
    {
      "activations/layer0_attention_weight_max": 15.99454402923584,
      "activations/layer0_attention_weight_min": -14.622756958007812,
      "activations/layer10_attention_weight_max": 37.87328338623047,
      "activations/layer10_attention_weight_min": -37.4381103515625,
      "activations/layer11_attention_weight_max": 34.864219665527344,
      "activations/layer11_attention_weight_min": -34.55897521972656,
      "activations/layer12_attention_weight_max": 19.236440658569336,
      "activations/layer12_attention_weight_min": -23.153966903686523,
      "activations/layer13_attention_weight_max": 46.05567932128906,
      "activations/layer13_attention_weight_min": -30.494665145874023,
      "activations/layer14_attention_weight_max": 41.01048278808594,
      "activations/layer14_attention_weight_min": -29.705595016479492,
      "activations/layer15_attention_weight_max": 36.91016387939453,
      "activations/layer15_attention_weight_min": -28.32927131652832,
      "activations/layer16_attention_weight_max": 35.8012809753418,
      "activations/layer16_attention_weight_min": -29.578805923461914,
      "activations/layer17_attention_weight_max": 57.61996078491211,
      "activations/layer17_attention_weight_min": -44.86002731323242,
      "activations/layer18_attention_weight_max": 53.70701599121094,
      "activations/layer18_attention_weight_min": -38.46952438354492,
      "activations/layer19_attention_weight_max": 25.291378021240234,
      "activations/layer19_attention_weight_min": -21.982397079467773,
      "activations/layer1_attention_weight_max": 16.21062469482422,
      "activations/layer1_attention_weight_min": -14.596362113952637,
      "activations/layer20_attention_weight_max": 27.167984008789062,
      "activations/layer20_attention_weight_min": -22.69483757019043,
      "activations/layer21_attention_weight_max": 47.11771011352539,
      "activations/layer21_attention_weight_min": -22.275331497192383,
      "activations/layer22_attention_weight_max": 34.36653137207031,
      "activations/layer22_attention_weight_min": -23.03676414489746,
      "activations/layer23_attention_weight_max": 46.59565734863281,
      "activations/layer23_attention_weight_min": -24.57598114013672,
      "activations/layer2_attention_weight_max": 33.46217346191406,
      "activations/layer2_attention_weight_min": -33.04137420654297,
      "activations/layer3_attention_weight_max": 95.15141296386719,
      "activations/layer3_attention_weight_min": -98.57567596435547,
      "activations/layer4_attention_weight_max": 102.35074615478516,
      "activations/layer4_attention_weight_min": -97.27529907226562,
      "activations/layer5_attention_weight_max": 78.97225189208984,
      "activations/layer5_attention_weight_min": -82.06684875488281,
      "activations/layer6_attention_weight_max": 58.35131072998047,
      "activations/layer6_attention_weight_min": -53.6523323059082,
      "activations/layer7_attention_weight_max": 76.93082427978516,
      "activations/layer7_attention_weight_min": -75.87061309814453,
      "activations/layer8_attention_weight_max": 47.68869400024414,
      "activations/layer8_attention_weight_min": -46.80101776123047,
      "activations/layer9_attention_weight_max": 53.91718292236328,
      "activations/layer9_attention_weight_min": -52.349586486816406,
      "epoch": 10.44,
      "learning_rate": 8.352462121212121e-05,
      "loss": 2.7983,
      "step": 179650
    },
    {
      "activations/layer0_attention_weight_max": 16.50823211669922,
      "activations/layer0_attention_weight_min": -14.225526809692383,
      "activations/layer10_attention_weight_max": 36.96334457397461,
      "activations/layer10_attention_weight_min": -36.4439697265625,
      "activations/layer11_attention_weight_max": 31.197500228881836,
      "activations/layer11_attention_weight_min": -32.55157470703125,
      "activations/layer12_attention_weight_max": 20.201446533203125,
      "activations/layer12_attention_weight_min": -26.360530853271484,
      "activations/layer13_attention_weight_max": 36.781978607177734,
      "activations/layer13_attention_weight_min": -30.69173240661621,
      "activations/layer14_attention_weight_max": 33.594322204589844,
      "activations/layer14_attention_weight_min": -29.197792053222656,
      "activations/layer15_attention_weight_max": 30.773080825805664,
      "activations/layer15_attention_weight_min": -28.647212982177734,
      "activations/layer16_attention_weight_max": 30.305631637573242,
      "activations/layer16_attention_weight_min": -28.396268844604492,
      "activations/layer17_attention_weight_max": 51.14739227294922,
      "activations/layer17_attention_weight_min": -42.07321548461914,
      "activations/layer18_attention_weight_max": 42.447410583496094,
      "activations/layer18_attention_weight_min": -35.773048400878906,
      "activations/layer19_attention_weight_max": 22.641754150390625,
      "activations/layer19_attention_weight_min": -21.531219482421875,
      "activations/layer1_attention_weight_max": 16.692344665527344,
      "activations/layer1_attention_weight_min": -15.572263717651367,
      "activations/layer20_attention_weight_max": 22.866369247436523,
      "activations/layer20_attention_weight_min": -19.60375213623047,
      "activations/layer21_attention_weight_max": 40.77215576171875,
      "activations/layer21_attention_weight_min": -22.17226791381836,
      "activations/layer22_attention_weight_max": 30.921371459960938,
      "activations/layer22_attention_weight_min": -24.15799903869629,
      "activations/layer23_attention_weight_max": 38.156429290771484,
      "activations/layer23_attention_weight_min": -22.00143814086914,
      "activations/layer2_attention_weight_max": 32.839324951171875,
      "activations/layer2_attention_weight_min": -31.95619010925293,
      "activations/layer3_attention_weight_max": 92.95674133300781,
      "activations/layer3_attention_weight_min": -93.83802795410156,
      "activations/layer4_attention_weight_max": 101.62000274658203,
      "activations/layer4_attention_weight_min": -93.9530258178711,
      "activations/layer5_attention_weight_max": 76.02885437011719,
      "activations/layer5_attention_weight_min": -80.36072540283203,
      "activations/layer6_attention_weight_max": 52.16910171508789,
      "activations/layer6_attention_weight_min": -52.404659271240234,
      "activations/layer7_attention_weight_max": 71.07989501953125,
      "activations/layer7_attention_weight_min": -76.28458404541016,
      "activations/layer8_attention_weight_max": 45.09319305419922,
      "activations/layer8_attention_weight_min": -47.8204460144043,
      "activations/layer9_attention_weight_max": 47.58332824707031,
      "activations/layer9_attention_weight_min": -48.18387985229492,
      "epoch": 10.44,
      "learning_rate": 8.350568181818182e-05,
      "loss": 2.7965,
      "step": 179700
    },
    {
      "activations/layer0_attention_weight_max": 15.765617370605469,
      "activations/layer0_attention_weight_min": -14.418623924255371,
      "activations/layer10_attention_weight_max": 39.01810073852539,
      "activations/layer10_attention_weight_min": -37.05539321899414,
      "activations/layer11_attention_weight_max": 33.13063430786133,
      "activations/layer11_attention_weight_min": -33.22105026245117,
      "activations/layer12_attention_weight_max": 17.366647720336914,
      "activations/layer12_attention_weight_min": -22.51278305053711,
      "activations/layer13_attention_weight_max": 37.195472717285156,
      "activations/layer13_attention_weight_min": -29.69247817993164,
      "activations/layer14_attention_weight_max": 38.31012725830078,
      "activations/layer14_attention_weight_min": -31.713319778442383,
      "activations/layer15_attention_weight_max": 33.899986267089844,
      "activations/layer15_attention_weight_min": -28.889450073242188,
      "activations/layer16_attention_weight_max": 28.672292709350586,
      "activations/layer16_attention_weight_min": -28.457426071166992,
      "activations/layer17_attention_weight_max": 52.89002990722656,
      "activations/layer17_attention_weight_min": -40.984676361083984,
      "activations/layer18_attention_weight_max": 49.144630432128906,
      "activations/layer18_attention_weight_min": -34.91242980957031,
      "activations/layer19_attention_weight_max": 19.84402847290039,
      "activations/layer19_attention_weight_min": -20.976179122924805,
      "activations/layer1_attention_weight_max": 16.603364944458008,
      "activations/layer1_attention_weight_min": -14.581062316894531,
      "activations/layer20_attention_weight_max": 20.2012882232666,
      "activations/layer20_attention_weight_min": -23.347734451293945,
      "activations/layer21_attention_weight_max": 36.85768508911133,
      "activations/layer21_attention_weight_min": -20.214962005615234,
      "activations/layer22_attention_weight_max": 30.866832733154297,
      "activations/layer22_attention_weight_min": -25.52206802368164,
      "activations/layer23_attention_weight_max": 35.8612174987793,
      "activations/layer23_attention_weight_min": -20.688627243041992,
      "activations/layer2_attention_weight_max": 31.367033004760742,
      "activations/layer2_attention_weight_min": -30.00636863708496,
      "activations/layer3_attention_weight_max": 89.39701080322266,
      "activations/layer3_attention_weight_min": -89.36740112304688,
      "activations/layer4_attention_weight_max": 101.13558197021484,
      "activations/layer4_attention_weight_min": -94.51231384277344,
      "activations/layer5_attention_weight_max": 72.41575622558594,
      "activations/layer5_attention_weight_min": -79.41007232666016,
      "activations/layer6_attention_weight_max": 52.58555603027344,
      "activations/layer6_attention_weight_min": -55.300418853759766,
      "activations/layer7_attention_weight_max": 78.05742645263672,
      "activations/layer7_attention_weight_min": -73.09426879882812,
      "activations/layer8_attention_weight_max": 46.195648193359375,
      "activations/layer8_attention_weight_min": -48.56813430786133,
      "activations/layer9_attention_weight_max": 49.14906692504883,
      "activations/layer9_attention_weight_min": -50.411155700683594,
      "epoch": 10.44,
      "learning_rate": 8.348674242424241e-05,
      "loss": 2.7672,
      "step": 179750
    },
    {
      "activations/layer0_attention_weight_max": 15.35741901397705,
      "activations/layer0_attention_weight_min": -14.070696830749512,
      "activations/layer10_attention_weight_max": 41.416900634765625,
      "activations/layer10_attention_weight_min": -36.80511474609375,
      "activations/layer11_attention_weight_max": 32.22868347167969,
      "activations/layer11_attention_weight_min": -35.6381950378418,
      "activations/layer12_attention_weight_max": 21.50074577331543,
      "activations/layer12_attention_weight_min": -26.334285736083984,
      "activations/layer13_attention_weight_max": 39.4924201965332,
      "activations/layer13_attention_weight_min": -29.50078773498535,
      "activations/layer14_attention_weight_max": 36.558929443359375,
      "activations/layer14_attention_weight_min": -29.716279983520508,
      "activations/layer15_attention_weight_max": 33.34475326538086,
      "activations/layer15_attention_weight_min": -29.148738861083984,
      "activations/layer16_attention_weight_max": 31.826932907104492,
      "activations/layer16_attention_weight_min": -28.926475524902344,
      "activations/layer17_attention_weight_max": 52.43669509887695,
      "activations/layer17_attention_weight_min": -44.47462463378906,
      "activations/layer18_attention_weight_max": 45.67375564575195,
      "activations/layer18_attention_weight_min": -38.64289474487305,
      "activations/layer19_attention_weight_max": 22.4193058013916,
      "activations/layer19_attention_weight_min": -24.04087257385254,
      "activations/layer1_attention_weight_max": 17.507080078125,
      "activations/layer1_attention_weight_min": -15.912627220153809,
      "activations/layer20_attention_weight_max": 23.706195831298828,
      "activations/layer20_attention_weight_min": -22.019134521484375,
      "activations/layer21_attention_weight_max": 37.8616828918457,
      "activations/layer21_attention_weight_min": -24.38389015197754,
      "activations/layer22_attention_weight_max": 33.70892333984375,
      "activations/layer22_attention_weight_min": -25.149627685546875,
      "activations/layer23_attention_weight_max": 39.22019958496094,
      "activations/layer23_attention_weight_min": -28.708141326904297,
      "activations/layer2_attention_weight_max": 31.151533126831055,
      "activations/layer2_attention_weight_min": -30.10262680053711,
      "activations/layer3_attention_weight_max": 91.6094741821289,
      "activations/layer3_attention_weight_min": -91.2237319946289,
      "activations/layer4_attention_weight_max": 101.41368865966797,
      "activations/layer4_attention_weight_min": -95.6103515625,
      "activations/layer5_attention_weight_max": 78.80215454101562,
      "activations/layer5_attention_weight_min": -78.98587799072266,
      "activations/layer6_attention_weight_max": 58.350616455078125,
      "activations/layer6_attention_weight_min": -57.470252990722656,
      "activations/layer7_attention_weight_max": 79.51113891601562,
      "activations/layer7_attention_weight_min": -78.91546630859375,
      "activations/layer8_attention_weight_max": 50.711708068847656,
      "activations/layer8_attention_weight_min": -49.38943862915039,
      "activations/layer9_attention_weight_max": 52.31376266479492,
      "activations/layer9_attention_weight_min": -57.24591064453125,
      "epoch": 10.45,
      "learning_rate": 8.346780303030303e-05,
      "loss": 2.7862,
      "step": 179800
    },
    {
      "activations/layer0_attention_weight_max": 14.384854316711426,
      "activations/layer0_attention_weight_min": -14.478958129882812,
      "activations/layer10_attention_weight_max": 36.21802520751953,
      "activations/layer10_attention_weight_min": -36.106258392333984,
      "activations/layer11_attention_weight_max": 32.8121337890625,
      "activations/layer11_attention_weight_min": -32.202613830566406,
      "activations/layer12_attention_weight_max": 20.830257415771484,
      "activations/layer12_attention_weight_min": -25.174503326416016,
      "activations/layer13_attention_weight_max": 46.651023864746094,
      "activations/layer13_attention_weight_min": -30.513832092285156,
      "activations/layer14_attention_weight_max": 44.26564407348633,
      "activations/layer14_attention_weight_min": -31.71625328063965,
      "activations/layer15_attention_weight_max": 40.55208969116211,
      "activations/layer15_attention_weight_min": -30.102941513061523,
      "activations/layer16_attention_weight_max": 30.973159790039062,
      "activations/layer16_attention_weight_min": -29.37870979309082,
      "activations/layer17_attention_weight_max": 57.97555923461914,
      "activations/layer17_attention_weight_min": -47.4343147277832,
      "activations/layer18_attention_weight_max": 49.40143585205078,
      "activations/layer18_attention_weight_min": -39.997982025146484,
      "activations/layer19_attention_weight_max": 26.397226333618164,
      "activations/layer19_attention_weight_min": -23.57901382446289,
      "activations/layer1_attention_weight_max": 17.1718807220459,
      "activations/layer1_attention_weight_min": -14.411590576171875,
      "activations/layer20_attention_weight_max": 23.976848602294922,
      "activations/layer20_attention_weight_min": -26.51530647277832,
      "activations/layer21_attention_weight_max": 41.08027267456055,
      "activations/layer21_attention_weight_min": -24.306381225585938,
      "activations/layer22_attention_weight_max": 32.24280548095703,
      "activations/layer22_attention_weight_min": -24.705577850341797,
      "activations/layer23_attention_weight_max": 39.142005920410156,
      "activations/layer23_attention_weight_min": -24.14642906188965,
      "activations/layer2_attention_weight_max": 31.257354736328125,
      "activations/layer2_attention_weight_min": -28.866741180419922,
      "activations/layer3_attention_weight_max": 90.22705841064453,
      "activations/layer3_attention_weight_min": -88.63294982910156,
      "activations/layer4_attention_weight_max": 97.47657012939453,
      "activations/layer4_attention_weight_min": -93.33543395996094,
      "activations/layer5_attention_weight_max": 78.71218872070312,
      "activations/layer5_attention_weight_min": -81.71665954589844,
      "activations/layer6_attention_weight_max": 54.99342346191406,
      "activations/layer6_attention_weight_min": -55.458316802978516,
      "activations/layer7_attention_weight_max": 77.46983337402344,
      "activations/layer7_attention_weight_min": -77.06629943847656,
      "activations/layer8_attention_weight_max": 45.65629959106445,
      "activations/layer8_attention_weight_min": -48.30791091918945,
      "activations/layer9_attention_weight_max": 50.01369094848633,
      "activations/layer9_attention_weight_min": -46.43442916870117,
      "epoch": 10.45,
      "learning_rate": 8.344886363636363e-05,
      "loss": 2.784,
      "step": 179850
    },
    {
      "activations/layer0_attention_weight_max": 15.754446029663086,
      "activations/layer0_attention_weight_min": -14.32015609741211,
      "activations/layer10_attention_weight_max": 40.08710479736328,
      "activations/layer10_attention_weight_min": -38.1785774230957,
      "activations/layer11_attention_weight_max": 33.835811614990234,
      "activations/layer11_attention_weight_min": -33.92484664916992,
      "activations/layer12_attention_weight_max": 21.527677536010742,
      "activations/layer12_attention_weight_min": -21.671646118164062,
      "activations/layer13_attention_weight_max": 44.17768096923828,
      "activations/layer13_attention_weight_min": -35.01968002319336,
      "activations/layer14_attention_weight_max": 42.91175842285156,
      "activations/layer14_attention_weight_min": -31.532508850097656,
      "activations/layer15_attention_weight_max": 35.81476974487305,
      "activations/layer15_attention_weight_min": -29.179548263549805,
      "activations/layer16_attention_weight_max": 33.60248947143555,
      "activations/layer16_attention_weight_min": -29.34162139892578,
      "activations/layer17_attention_weight_max": 54.65998840332031,
      "activations/layer17_attention_weight_min": -45.838069915771484,
      "activations/layer18_attention_weight_max": 48.81837463378906,
      "activations/layer18_attention_weight_min": -35.979942321777344,
      "activations/layer19_attention_weight_max": 25.35797882080078,
      "activations/layer19_attention_weight_min": -20.647241592407227,
      "activations/layer1_attention_weight_max": 17.322477340698242,
      "activations/layer1_attention_weight_min": -14.640052795410156,
      "activations/layer20_attention_weight_max": 23.071378707885742,
      "activations/layer20_attention_weight_min": -21.70035743713379,
      "activations/layer21_attention_weight_max": 45.476585388183594,
      "activations/layer21_attention_weight_min": -29.38852882385254,
      "activations/layer22_attention_weight_max": 33.87525939941406,
      "activations/layer22_attention_weight_min": -26.055988311767578,
      "activations/layer23_attention_weight_max": 40.23320770263672,
      "activations/layer23_attention_weight_min": -23.28652000427246,
      "activations/layer2_attention_weight_max": 32.039939880371094,
      "activations/layer2_attention_weight_min": -30.415420532226562,
      "activations/layer3_attention_weight_max": 89.90679168701172,
      "activations/layer3_attention_weight_min": -94.35118865966797,
      "activations/layer4_attention_weight_max": 97.68587493896484,
      "activations/layer4_attention_weight_min": -92.49217987060547,
      "activations/layer5_attention_weight_max": 78.00755310058594,
      "activations/layer5_attention_weight_min": -76.49917602539062,
      "activations/layer6_attention_weight_max": 53.16126251220703,
      "activations/layer6_attention_weight_min": -53.66104507446289,
      "activations/layer7_attention_weight_max": 76.41162872314453,
      "activations/layer7_attention_weight_min": -75.67924499511719,
      "activations/layer8_attention_weight_max": 46.29530715942383,
      "activations/layer8_attention_weight_min": -48.894508361816406,
      "activations/layer9_attention_weight_max": 51.022064208984375,
      "activations/layer9_attention_weight_min": -51.45161819458008,
      "epoch": 10.45,
      "learning_rate": 8.342992424242423e-05,
      "loss": 2.7854,
      "step": 179900
    },
    {
      "activations/layer0_attention_weight_max": 15.111084938049316,
      "activations/layer0_attention_weight_min": -14.724106788635254,
      "activations/layer10_attention_weight_max": 34.97648239135742,
      "activations/layer10_attention_weight_min": -34.365291595458984,
      "activations/layer11_attention_weight_max": 31.189624786376953,
      "activations/layer11_attention_weight_min": -32.14981460571289,
      "activations/layer12_attention_weight_max": 25.082923889160156,
      "activations/layer12_attention_weight_min": -26.283281326293945,
      "activations/layer13_attention_weight_max": 36.519004821777344,
      "activations/layer13_attention_weight_min": -29.399364471435547,
      "activations/layer14_attention_weight_max": 34.938873291015625,
      "activations/layer14_attention_weight_min": -28.847597122192383,
      "activations/layer15_attention_weight_max": 30.90867042541504,
      "activations/layer15_attention_weight_min": -27.895755767822266,
      "activations/layer16_attention_weight_max": 31.306669235229492,
      "activations/layer16_attention_weight_min": -28.013092041015625,
      "activations/layer17_attention_weight_max": 50.97898864746094,
      "activations/layer17_attention_weight_min": -41.67564392089844,
      "activations/layer18_attention_weight_max": 45.89836883544922,
      "activations/layer18_attention_weight_min": -37.31348419189453,
      "activations/layer19_attention_weight_max": 22.021146774291992,
      "activations/layer19_attention_weight_min": -23.399492263793945,
      "activations/layer1_attention_weight_max": 17.34796905517578,
      "activations/layer1_attention_weight_min": -15.87142562866211,
      "activations/layer20_attention_weight_max": 23.548656463623047,
      "activations/layer20_attention_weight_min": -23.159482955932617,
      "activations/layer21_attention_weight_max": 36.375091552734375,
      "activations/layer21_attention_weight_min": -23.30795669555664,
      "activations/layer22_attention_weight_max": 32.63713836669922,
      "activations/layer22_attention_weight_min": -25.686330795288086,
      "activations/layer23_attention_weight_max": 38.827213287353516,
      "activations/layer23_attention_weight_min": -23.32404327392578,
      "activations/layer2_attention_weight_max": 30.24951934814453,
      "activations/layer2_attention_weight_min": -31.013158798217773,
      "activations/layer3_attention_weight_max": 87.3002700805664,
      "activations/layer3_attention_weight_min": -90.50658416748047,
      "activations/layer4_attention_weight_max": 94.85194396972656,
      "activations/layer4_attention_weight_min": -90.84645080566406,
      "activations/layer5_attention_weight_max": 73.58525848388672,
      "activations/layer5_attention_weight_min": -74.4609375,
      "activations/layer6_attention_weight_max": 51.04462814331055,
      "activations/layer6_attention_weight_min": -49.967227935791016,
      "activations/layer7_attention_weight_max": 70.94515228271484,
      "activations/layer7_attention_weight_min": -69.909423828125,
      "activations/layer8_attention_weight_max": 43.112117767333984,
      "activations/layer8_attention_weight_min": -44.03235626220703,
      "activations/layer9_attention_weight_max": 45.748958587646484,
      "activations/layer9_attention_weight_min": -45.89156723022461,
      "epoch": 10.46,
      "learning_rate": 8.341098484848484e-05,
      "loss": 2.7858,
      "step": 179950
    },
    {
      "activations/layer0_attention_weight_max": 16.06133270263672,
      "activations/layer0_attention_weight_min": -14.881464004516602,
      "activations/layer10_attention_weight_max": 45.31840133666992,
      "activations/layer10_attention_weight_min": -42.1260871887207,
      "activations/layer11_attention_weight_max": 43.012420654296875,
      "activations/layer11_attention_weight_min": -36.73045349121094,
      "activations/layer12_attention_weight_max": 19.796354293823242,
      "activations/layer12_attention_weight_min": -27.615768432617188,
      "activations/layer13_attention_weight_max": 36.31480026245117,
      "activations/layer13_attention_weight_min": -29.13918113708496,
      "activations/layer14_attention_weight_max": 37.219444274902344,
      "activations/layer14_attention_weight_min": -32.194801330566406,
      "activations/layer15_attention_weight_max": 34.56859588623047,
      "activations/layer15_attention_weight_min": -30.65172004699707,
      "activations/layer16_attention_weight_max": 32.98015213012695,
      "activations/layer16_attention_weight_min": -30.882123947143555,
      "activations/layer17_attention_weight_max": 59.42451858520508,
      "activations/layer17_attention_weight_min": -48.79896926879883,
      "activations/layer18_attention_weight_max": 49.64442825317383,
      "activations/layer18_attention_weight_min": -39.70035171508789,
      "activations/layer19_attention_weight_max": 26.146896362304688,
      "activations/layer19_attention_weight_min": -21.351438522338867,
      "activations/layer1_attention_weight_max": 18.120336532592773,
      "activations/layer1_attention_weight_min": -16.06060218811035,
      "activations/layer20_attention_weight_max": 26.010969161987305,
      "activations/layer20_attention_weight_min": -19.729259490966797,
      "activations/layer21_attention_weight_max": 46.052703857421875,
      "activations/layer21_attention_weight_min": -21.691669464111328,
      "activations/layer22_attention_weight_max": 37.55785369873047,
      "activations/layer22_attention_weight_min": -25.26569938659668,
      "activations/layer23_attention_weight_max": 42.26454544067383,
      "activations/layer23_attention_weight_min": -23.949752807617188,
      "activations/layer2_attention_weight_max": 34.435516357421875,
      "activations/layer2_attention_weight_min": -33.69587707519531,
      "activations/layer3_attention_weight_max": 95.07622528076172,
      "activations/layer3_attention_weight_min": -96.29622650146484,
      "activations/layer4_attention_weight_max": 109.15496826171875,
      "activations/layer4_attention_weight_min": -95.85134887695312,
      "activations/layer5_attention_weight_max": 82.82054901123047,
      "activations/layer5_attention_weight_min": -79.65491485595703,
      "activations/layer6_attention_weight_max": 56.13291931152344,
      "activations/layer6_attention_weight_min": -54.87898635864258,
      "activations/layer7_attention_weight_max": 97.12972259521484,
      "activations/layer7_attention_weight_min": -83.2978286743164,
      "activations/layer8_attention_weight_max": 59.65250778198242,
      "activations/layer8_attention_weight_min": -50.980194091796875,
      "activations/layer9_attention_weight_max": 72.1819076538086,
      "activations/layer9_attention_weight_min": -56.5982551574707,
      "epoch": 10.46,
      "learning_rate": 8.339204545454545e-05,
      "loss": 2.7726,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4427,
      "eval_samples_per_second": 508.607,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4427,
      "eval_openwebtext_samples_per_second": 508.607,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 2.0044,
      "eval_wikitext_samples_per_second": 227.499,
      "step": 180000
    },
    {
      "epoch": 10.46,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.5844,
      "eval_lambada_samples_per_second": 508.015,
      "step": 180000
    },
    {
      "activations/layer0_attention_weight_max": 15.203797340393066,
      "activations/layer0_attention_weight_min": -14.273584365844727,
      "activations/layer10_attention_weight_max": 34.83099365234375,
      "activations/layer10_attention_weight_min": -36.780357360839844,
      "activations/layer11_attention_weight_max": 31.589719772338867,
      "activations/layer11_attention_weight_min": -33.399986267089844,
      "activations/layer12_attention_weight_max": 21.41448974609375,
      "activations/layer12_attention_weight_min": -25.353145599365234,
      "activations/layer13_attention_weight_max": 44.77033233642578,
      "activations/layer13_attention_weight_min": -30.226036071777344,
      "activations/layer14_attention_weight_max": 43.706993103027344,
      "activations/layer14_attention_weight_min": -29.032686233520508,
      "activations/layer15_attention_weight_max": 39.14091110229492,
      "activations/layer15_attention_weight_min": -29.160242080688477,
      "activations/layer16_attention_weight_max": 33.012306213378906,
      "activations/layer16_attention_weight_min": -29.01537322998047,
      "activations/layer17_attention_weight_max": 55.23460006713867,
      "activations/layer17_attention_weight_min": -44.09961700439453,
      "activations/layer18_attention_weight_max": 49.17973327636719,
      "activations/layer18_attention_weight_min": -38.26957702636719,
      "activations/layer19_attention_weight_max": 23.228755950927734,
      "activations/layer19_attention_weight_min": -23.74538230895996,
      "activations/layer1_attention_weight_max": 16.723552703857422,
      "activations/layer1_attention_weight_min": -17.764453887939453,
      "activations/layer20_attention_weight_max": 24.300132751464844,
      "activations/layer20_attention_weight_min": -22.26842498779297,
      "activations/layer21_attention_weight_max": 40.82558059692383,
      "activations/layer21_attention_weight_min": -28.95174789428711,
      "activations/layer22_attention_weight_max": 35.495845794677734,
      "activations/layer22_attention_weight_min": -26.296533584594727,
      "activations/layer23_attention_weight_max": 39.628395080566406,
      "activations/layer23_attention_weight_min": -23.9239559173584,
      "activations/layer2_attention_weight_max": 31.151302337646484,
      "activations/layer2_attention_weight_min": -30.9371395111084,
      "activations/layer3_attention_weight_max": 91.09791564941406,
      "activations/layer3_attention_weight_min": -98.488037109375,
      "activations/layer4_attention_weight_max": 96.91860961914062,
      "activations/layer4_attention_weight_min": -94.92825317382812,
      "activations/layer5_attention_weight_max": 72.98377990722656,
      "activations/layer5_attention_weight_min": -78.96343994140625,
      "activations/layer6_attention_weight_max": 53.576107025146484,
      "activations/layer6_attention_weight_min": -53.1567497253418,
      "activations/layer7_attention_weight_max": 76.94230651855469,
      "activations/layer7_attention_weight_min": -70.71344757080078,
      "activations/layer8_attention_weight_max": 45.58323669433594,
      "activations/layer8_attention_weight_min": -47.5849723815918,
      "activations/layer9_attention_weight_max": 49.43991470336914,
      "activations/layer9_attention_weight_min": -49.85868453979492,
      "epoch": 10.46,
      "learning_rate": 8.337310606060606e-05,
      "loss": 2.8012,
      "step": 180050
    },
    {
      "activations/layer0_attention_weight_max": 16.31463050842285,
      "activations/layer0_attention_weight_min": -14.486407279968262,
      "activations/layer10_attention_weight_max": 35.14163589477539,
      "activations/layer10_attention_weight_min": -35.12602996826172,
      "activations/layer11_attention_weight_max": 29.04006576538086,
      "activations/layer11_attention_weight_min": -31.295459747314453,
      "activations/layer12_attention_weight_max": 17.169647216796875,
      "activations/layer12_attention_weight_min": -26.600055694580078,
      "activations/layer13_attention_weight_max": 37.745479583740234,
      "activations/layer13_attention_weight_min": -31.27871322631836,
      "activations/layer14_attention_weight_max": 37.67652893066406,
      "activations/layer14_attention_weight_min": -30.73581314086914,
      "activations/layer15_attention_weight_max": 30.647546768188477,
      "activations/layer15_attention_weight_min": -29.344778060913086,
      "activations/layer16_attention_weight_max": 31.881059646606445,
      "activations/layer16_attention_weight_min": -31.01612091064453,
      "activations/layer17_attention_weight_max": 49.08407211303711,
      "activations/layer17_attention_weight_min": -43.669189453125,
      "activations/layer18_attention_weight_max": 51.42828369140625,
      "activations/layer18_attention_weight_min": -38.986175537109375,
      "activations/layer19_attention_weight_max": 23.31911277770996,
      "activations/layer19_attention_weight_min": -22.156103134155273,
      "activations/layer1_attention_weight_max": 16.95566177368164,
      "activations/layer1_attention_weight_min": -15.001243591308594,
      "activations/layer20_attention_weight_max": 22.069042205810547,
      "activations/layer20_attention_weight_min": -23.815704345703125,
      "activations/layer21_attention_weight_max": 39.15620803833008,
      "activations/layer21_attention_weight_min": -25.016056060791016,
      "activations/layer22_attention_weight_max": 36.937164306640625,
      "activations/layer22_attention_weight_min": -26.346025466918945,
      "activations/layer23_attention_weight_max": 34.948524475097656,
      "activations/layer23_attention_weight_min": -27.689502716064453,
      "activations/layer2_attention_weight_max": 31.78499412536621,
      "activations/layer2_attention_weight_min": -29.926345825195312,
      "activations/layer3_attention_weight_max": 89.5095443725586,
      "activations/layer3_attention_weight_min": -86.49830627441406,
      "activations/layer4_attention_weight_max": 96.84629821777344,
      "activations/layer4_attention_weight_min": -90.72200775146484,
      "activations/layer5_attention_weight_max": 69.70420837402344,
      "activations/layer5_attention_weight_min": -74.07855987548828,
      "activations/layer6_attention_weight_max": 49.056236267089844,
      "activations/layer6_attention_weight_min": -51.00725173950195,
      "activations/layer7_attention_weight_max": 74.06525421142578,
      "activations/layer7_attention_weight_min": -69.81649780273438,
      "activations/layer8_attention_weight_max": 41.140052795410156,
      "activations/layer8_attention_weight_min": -44.31217575073242,
      "activations/layer9_attention_weight_max": 45.28373336791992,
      "activations/layer9_attention_weight_min": -46.33842086791992,
      "epoch": 10.46,
      "learning_rate": 8.335416666666666e-05,
      "loss": 2.7876,
      "step": 180100
    },
    {
      "activations/layer0_attention_weight_max": 15.8895263671875,
      "activations/layer0_attention_weight_min": -14.219964027404785,
      "activations/layer10_attention_weight_max": 35.342891693115234,
      "activations/layer10_attention_weight_min": -34.279457092285156,
      "activations/layer11_attention_weight_max": 31.019168853759766,
      "activations/layer11_attention_weight_min": -32.98224639892578,
      "activations/layer12_attention_weight_max": 16.724971771240234,
      "activations/layer12_attention_weight_min": -26.49880599975586,
      "activations/layer13_attention_weight_max": 35.598934173583984,
      "activations/layer13_attention_weight_min": -28.75922203063965,
      "activations/layer14_attention_weight_max": 34.82678985595703,
      "activations/layer14_attention_weight_min": -29.571626663208008,
      "activations/layer15_attention_weight_max": 30.7563533782959,
      "activations/layer15_attention_weight_min": -27.43561553955078,
      "activations/layer16_attention_weight_max": 30.34267234802246,
      "activations/layer16_attention_weight_min": -27.316099166870117,
      "activations/layer17_attention_weight_max": 48.023345947265625,
      "activations/layer17_attention_weight_min": -40.95610427856445,
      "activations/layer18_attention_weight_max": 44.300350189208984,
      "activations/layer18_attention_weight_min": -35.15190887451172,
      "activations/layer19_attention_weight_max": 23.156930923461914,
      "activations/layer19_attention_weight_min": -20.700254440307617,
      "activations/layer1_attention_weight_max": 15.860433578491211,
      "activations/layer1_attention_weight_min": -15.58070182800293,
      "activations/layer20_attention_weight_max": 20.45676612854004,
      "activations/layer20_attention_weight_min": -24.105710983276367,
      "activations/layer21_attention_weight_max": 33.023773193359375,
      "activations/layer21_attention_weight_min": -23.346677780151367,
      "activations/layer22_attention_weight_max": 28.866798400878906,
      "activations/layer22_attention_weight_min": -25.21388053894043,
      "activations/layer23_attention_weight_max": 36.31387710571289,
      "activations/layer23_attention_weight_min": -23.541797637939453,
      "activations/layer2_attention_weight_max": 31.41460418701172,
      "activations/layer2_attention_weight_min": -30.3629150390625,
      "activations/layer3_attention_weight_max": 92.58245086669922,
      "activations/layer3_attention_weight_min": -92.67866516113281,
      "activations/layer4_attention_weight_max": 95.48606872558594,
      "activations/layer4_attention_weight_min": -91.64909362792969,
      "activations/layer5_attention_weight_max": 72.42244720458984,
      "activations/layer5_attention_weight_min": -77.46756744384766,
      "activations/layer6_attention_weight_max": 54.002891540527344,
      "activations/layer6_attention_weight_min": -52.73970031738281,
      "activations/layer7_attention_weight_max": 80.83091735839844,
      "activations/layer7_attention_weight_min": -73.19100189208984,
      "activations/layer8_attention_weight_max": 46.009422302246094,
      "activations/layer8_attention_weight_min": -47.46815872192383,
      "activations/layer9_attention_weight_max": 50.22935104370117,
      "activations/layer9_attention_weight_min": -48.459922790527344,
      "epoch": 10.47,
      "learning_rate": 8.333522727272726e-05,
      "loss": 2.7936,
      "step": 180150
    },
    {
      "activations/layer0_attention_weight_max": 18.41135597229004,
      "activations/layer0_attention_weight_min": -13.86415958404541,
      "activations/layer10_attention_weight_max": 35.6014404296875,
      "activations/layer10_attention_weight_min": -36.20705032348633,
      "activations/layer11_attention_weight_max": 30.19082260131836,
      "activations/layer11_attention_weight_min": -32.474761962890625,
      "activations/layer12_attention_weight_max": 17.457443237304688,
      "activations/layer12_attention_weight_min": -26.227893829345703,
      "activations/layer13_attention_weight_max": 39.51561737060547,
      "activations/layer13_attention_weight_min": -33.08218002319336,
      "activations/layer14_attention_weight_max": 35.233665466308594,
      "activations/layer14_attention_weight_min": -31.981658935546875,
      "activations/layer15_attention_weight_max": 30.291412353515625,
      "activations/layer15_attention_weight_min": -28.726821899414062,
      "activations/layer16_attention_weight_max": 28.02565574645996,
      "activations/layer16_attention_weight_min": -28.711894989013672,
      "activations/layer17_attention_weight_max": 50.83269119262695,
      "activations/layer17_attention_weight_min": -41.41661834716797,
      "activations/layer18_attention_weight_max": 41.5095329284668,
      "activations/layer18_attention_weight_min": -35.211395263671875,
      "activations/layer19_attention_weight_max": 20.934358596801758,
      "activations/layer19_attention_weight_min": -20.152639389038086,
      "activations/layer1_attention_weight_max": 17.316282272338867,
      "activations/layer1_attention_weight_min": -16.031877517700195,
      "activations/layer20_attention_weight_max": 19.934593200683594,
      "activations/layer20_attention_weight_min": -22.318397521972656,
      "activations/layer21_attention_weight_max": 33.110107421875,
      "activations/layer21_attention_weight_min": -24.8565616607666,
      "activations/layer22_attention_weight_max": 27.012279510498047,
      "activations/layer22_attention_weight_min": -25.65975570678711,
      "activations/layer23_attention_weight_max": 34.74845504760742,
      "activations/layer23_attention_weight_min": -24.285690307617188,
      "activations/layer2_attention_weight_max": 29.6003475189209,
      "activations/layer2_attention_weight_min": -27.08155059814453,
      "activations/layer3_attention_weight_max": 88.74771118164062,
      "activations/layer3_attention_weight_min": -83.73409271240234,
      "activations/layer4_attention_weight_max": 91.73214721679688,
      "activations/layer4_attention_weight_min": -88.25725555419922,
      "activations/layer5_attention_weight_max": 71.79077911376953,
      "activations/layer5_attention_weight_min": -78.40042877197266,
      "activations/layer6_attention_weight_max": 51.88882064819336,
      "activations/layer6_attention_weight_min": -51.1415901184082,
      "activations/layer7_attention_weight_max": 72.25113677978516,
      "activations/layer7_attention_weight_min": -71.72821044921875,
      "activations/layer8_attention_weight_max": 44.29217529296875,
      "activations/layer8_attention_weight_min": -47.23591232299805,
      "activations/layer9_attention_weight_max": 48.41121292114258,
      "activations/layer9_attention_weight_min": -49.552711486816406,
      "epoch": 10.47,
      "learning_rate": 8.331628787878788e-05,
      "loss": 2.8016,
      "step": 180200
    },
    {
      "activations/layer0_attention_weight_max": 17.600109100341797,
      "activations/layer0_attention_weight_min": -14.341052055358887,
      "activations/layer10_attention_weight_max": 41.68427658081055,
      "activations/layer10_attention_weight_min": -38.62759780883789,
      "activations/layer11_attention_weight_max": 34.318275451660156,
      "activations/layer11_attention_weight_min": -33.17761993408203,
      "activations/layer12_attention_weight_max": 18.10940933227539,
      "activations/layer12_attention_weight_min": -26.663135528564453,
      "activations/layer13_attention_weight_max": 39.58538818359375,
      "activations/layer13_attention_weight_min": -29.920181274414062,
      "activations/layer14_attention_weight_max": 40.61137390136719,
      "activations/layer14_attention_weight_min": -32.88848114013672,
      "activations/layer15_attention_weight_max": 35.61289978027344,
      "activations/layer15_attention_weight_min": -29.488170623779297,
      "activations/layer16_attention_weight_max": 31.7081298828125,
      "activations/layer16_attention_weight_min": -32.18628692626953,
      "activations/layer17_attention_weight_max": 51.85377883911133,
      "activations/layer17_attention_weight_min": -45.28777313232422,
      "activations/layer18_attention_weight_max": 44.67996597290039,
      "activations/layer18_attention_weight_min": -38.79645919799805,
      "activations/layer19_attention_weight_max": 21.10841178894043,
      "activations/layer19_attention_weight_min": -22.230377197265625,
      "activations/layer1_attention_weight_max": 18.20101547241211,
      "activations/layer1_attention_weight_min": -15.591958045959473,
      "activations/layer20_attention_weight_max": 21.775362014770508,
      "activations/layer20_attention_weight_min": -23.78862190246582,
      "activations/layer21_attention_weight_max": 32.3899040222168,
      "activations/layer21_attention_weight_min": -22.298147201538086,
      "activations/layer22_attention_weight_max": 29.941144943237305,
      "activations/layer22_attention_weight_min": -23.50653839111328,
      "activations/layer23_attention_weight_max": 34.516239166259766,
      "activations/layer23_attention_weight_min": -23.074134826660156,
      "activations/layer2_attention_weight_max": 32.669639587402344,
      "activations/layer2_attention_weight_min": -30.709430694580078,
      "activations/layer3_attention_weight_max": 94.1279067993164,
      "activations/layer3_attention_weight_min": -92.86679077148438,
      "activations/layer4_attention_weight_max": 95.91178131103516,
      "activations/layer4_attention_weight_min": -97.29967498779297,
      "activations/layer5_attention_weight_max": 72.43894958496094,
      "activations/layer5_attention_weight_min": -79.98223114013672,
      "activations/layer6_attention_weight_max": 56.52452087402344,
      "activations/layer6_attention_weight_min": -56.924591064453125,
      "activations/layer7_attention_weight_max": 76.27899932861328,
      "activations/layer7_attention_weight_min": -76.49923706054688,
      "activations/layer8_attention_weight_max": 45.14706802368164,
      "activations/layer8_attention_weight_min": -48.60631561279297,
      "activations/layer9_attention_weight_max": 54.77047348022461,
      "activations/layer9_attention_weight_min": -49.31393051147461,
      "epoch": 10.47,
      "learning_rate": 8.329734848484847e-05,
      "loss": 2.7802,
      "step": 180250
    },
    {
      "activations/layer0_attention_weight_max": 14.847505569458008,
      "activations/layer0_attention_weight_min": -14.36260986328125,
      "activations/layer10_attention_weight_max": 40.07119369506836,
      "activations/layer10_attention_weight_min": -40.0189094543457,
      "activations/layer11_attention_weight_max": 37.259002685546875,
      "activations/layer11_attention_weight_min": -34.90275573730469,
      "activations/layer12_attention_weight_max": 20.79090690612793,
      "activations/layer12_attention_weight_min": -26.340389251708984,
      "activations/layer13_attention_weight_max": 45.55079650878906,
      "activations/layer13_attention_weight_min": -32.600440979003906,
      "activations/layer14_attention_weight_max": 36.70310974121094,
      "activations/layer14_attention_weight_min": -30.959423065185547,
      "activations/layer15_attention_weight_max": 34.31495666503906,
      "activations/layer15_attention_weight_min": -29.3704776763916,
      "activations/layer16_attention_weight_max": 28.170557022094727,
      "activations/layer16_attention_weight_min": -27.96980857849121,
      "activations/layer17_attention_weight_max": 51.61205291748047,
      "activations/layer17_attention_weight_min": -43.61613845825195,
      "activations/layer18_attention_weight_max": 47.80317306518555,
      "activations/layer18_attention_weight_min": -37.80964660644531,
      "activations/layer19_attention_weight_max": 22.848403930664062,
      "activations/layer19_attention_weight_min": -21.219755172729492,
      "activations/layer1_attention_weight_max": 15.963611602783203,
      "activations/layer1_attention_weight_min": -14.41703987121582,
      "activations/layer20_attention_weight_max": 22.316490173339844,
      "activations/layer20_attention_weight_min": -21.563913345336914,
      "activations/layer21_attention_weight_max": 32.86204147338867,
      "activations/layer21_attention_weight_min": -22.139291763305664,
      "activations/layer22_attention_weight_max": 28.361814498901367,
      "activations/layer22_attention_weight_min": -25.152917861938477,
      "activations/layer23_attention_weight_max": 33.7301025390625,
      "activations/layer23_attention_weight_min": -23.882856369018555,
      "activations/layer2_attention_weight_max": 31.93861198425293,
      "activations/layer2_attention_weight_min": -29.570289611816406,
      "activations/layer3_attention_weight_max": 90.57991027832031,
      "activations/layer3_attention_weight_min": -91.24999237060547,
      "activations/layer4_attention_weight_max": 99.58204650878906,
      "activations/layer4_attention_weight_min": -93.82881164550781,
      "activations/layer5_attention_weight_max": 77.90754699707031,
      "activations/layer5_attention_weight_min": -79.39576721191406,
      "activations/layer6_attention_weight_max": 58.37786102294922,
      "activations/layer6_attention_weight_min": -58.624366760253906,
      "activations/layer7_attention_weight_max": 77.29447174072266,
      "activations/layer7_attention_weight_min": -77.8519287109375,
      "activations/layer8_attention_weight_max": 51.70547866821289,
      "activations/layer8_attention_weight_min": -54.131446838378906,
      "activations/layer9_attention_weight_max": 52.5344352722168,
      "activations/layer9_attention_weight_min": -51.6531982421875,
      "epoch": 10.48,
      "learning_rate": 8.327840909090908e-05,
      "loss": 2.8042,
      "step": 180300
    },
    {
      "activations/layer0_attention_weight_max": 15.797442436218262,
      "activations/layer0_attention_weight_min": -14.592613220214844,
      "activations/layer10_attention_weight_max": 38.26767349243164,
      "activations/layer10_attention_weight_min": -35.91642761230469,
      "activations/layer11_attention_weight_max": 33.590736389160156,
      "activations/layer11_attention_weight_min": -34.463661193847656,
      "activations/layer12_attention_weight_max": 22.36457633972168,
      "activations/layer12_attention_weight_min": -24.253433227539062,
      "activations/layer13_attention_weight_max": 47.21113586425781,
      "activations/layer13_attention_weight_min": -31.43243980407715,
      "activations/layer14_attention_weight_max": 41.87158203125,
      "activations/layer14_attention_weight_min": -30.164018630981445,
      "activations/layer15_attention_weight_max": 40.066871643066406,
      "activations/layer15_attention_weight_min": -29.016239166259766,
      "activations/layer16_attention_weight_max": 32.240257263183594,
      "activations/layer16_attention_weight_min": -28.91480255126953,
      "activations/layer17_attention_weight_max": 52.26715850830078,
      "activations/layer17_attention_weight_min": -43.37842559814453,
      "activations/layer18_attention_weight_max": 48.712337493896484,
      "activations/layer18_attention_weight_min": -37.83240509033203,
      "activations/layer19_attention_weight_max": 21.728200912475586,
      "activations/layer19_attention_weight_min": -22.536924362182617,
      "activations/layer1_attention_weight_max": 18.197078704833984,
      "activations/layer1_attention_weight_min": -15.246819496154785,
      "activations/layer20_attention_weight_max": 22.031761169433594,
      "activations/layer20_attention_weight_min": -23.310909271240234,
      "activations/layer21_attention_weight_max": 44.10734176635742,
      "activations/layer21_attention_weight_min": -29.075912475585938,
      "activations/layer22_attention_weight_max": 31.87606430053711,
      "activations/layer22_attention_weight_min": -27.107343673706055,
      "activations/layer23_attention_weight_max": 41.305816650390625,
      "activations/layer23_attention_weight_min": -24.467212677001953,
      "activations/layer2_attention_weight_max": 31.69417953491211,
      "activations/layer2_attention_weight_min": -29.617542266845703,
      "activations/layer3_attention_weight_max": 90.08744812011719,
      "activations/layer3_attention_weight_min": -92.02091979980469,
      "activations/layer4_attention_weight_max": 98.55604553222656,
      "activations/layer4_attention_weight_min": -93.2280502319336,
      "activations/layer5_attention_weight_max": 74.40478515625,
      "activations/layer5_attention_weight_min": -80.96783447265625,
      "activations/layer6_attention_weight_max": 53.92679977416992,
      "activations/layer6_attention_weight_min": -53.07367706298828,
      "activations/layer7_attention_weight_max": 74.96761322021484,
      "activations/layer7_attention_weight_min": -75.6412582397461,
      "activations/layer8_attention_weight_max": 44.944637298583984,
      "activations/layer8_attention_weight_min": -46.658782958984375,
      "activations/layer9_attention_weight_max": 49.352352142333984,
      "activations/layer9_attention_weight_min": -49.72751998901367,
      "epoch": 10.48,
      "learning_rate": 8.32594696969697e-05,
      "loss": 2.7757,
      "step": 180350
    },
    {
      "activations/layer0_attention_weight_max": 16.88782501220703,
      "activations/layer0_attention_weight_min": -14.690495491027832,
      "activations/layer10_attention_weight_max": 37.214996337890625,
      "activations/layer10_attention_weight_min": -38.46527099609375,
      "activations/layer11_attention_weight_max": 33.83647537231445,
      "activations/layer11_attention_weight_min": -33.188865661621094,
      "activations/layer12_attention_weight_max": 17.693042755126953,
      "activations/layer12_attention_weight_min": -28.69434928894043,
      "activations/layer13_attention_weight_max": 39.52373123168945,
      "activations/layer13_attention_weight_min": -30.438241958618164,
      "activations/layer14_attention_weight_max": 45.2737922668457,
      "activations/layer14_attention_weight_min": -32.32585906982422,
      "activations/layer15_attention_weight_max": 33.58171463012695,
      "activations/layer15_attention_weight_min": -29.639955520629883,
      "activations/layer16_attention_weight_max": 33.08794021606445,
      "activations/layer16_attention_weight_min": -31.507368087768555,
      "activations/layer17_attention_weight_max": 54.3739013671875,
      "activations/layer17_attention_weight_min": -47.57106018066406,
      "activations/layer18_attention_weight_max": 48.68448257446289,
      "activations/layer18_attention_weight_min": -42.45649337768555,
      "activations/layer19_attention_weight_max": 23.458858489990234,
      "activations/layer19_attention_weight_min": -22.156423568725586,
      "activations/layer1_attention_weight_max": 16.09394645690918,
      "activations/layer1_attention_weight_min": -14.76745891571045,
      "activations/layer20_attention_weight_max": 27.425569534301758,
      "activations/layer20_attention_weight_min": -21.788063049316406,
      "activations/layer21_attention_weight_max": 48.27763366699219,
      "activations/layer21_attention_weight_min": -25.98395347595215,
      "activations/layer22_attention_weight_max": 33.941566467285156,
      "activations/layer22_attention_weight_min": -25.45245933532715,
      "activations/layer23_attention_weight_max": 43.94692611694336,
      "activations/layer23_attention_weight_min": -23.958797454833984,
      "activations/layer2_attention_weight_max": 34.9915771484375,
      "activations/layer2_attention_weight_min": -31.078720092773438,
      "activations/layer3_attention_weight_max": 93.6885986328125,
      "activations/layer3_attention_weight_min": -94.01124572753906,
      "activations/layer4_attention_weight_max": 99.8098373413086,
      "activations/layer4_attention_weight_min": -98.28082275390625,
      "activations/layer5_attention_weight_max": 76.68447875976562,
      "activations/layer5_attention_weight_min": -82.87335205078125,
      "activations/layer6_attention_weight_max": 54.90637969970703,
      "activations/layer6_attention_weight_min": -54.5730094909668,
      "activations/layer7_attention_weight_max": 76.91173553466797,
      "activations/layer7_attention_weight_min": -80.26844024658203,
      "activations/layer8_attention_weight_max": 47.80731201171875,
      "activations/layer8_attention_weight_min": -48.86245346069336,
      "activations/layer9_attention_weight_max": 55.98594284057617,
      "activations/layer9_attention_weight_min": -53.01728439331055,
      "epoch": 10.48,
      "learning_rate": 8.324053030303028e-05,
      "loss": 2.7699,
      "step": 180400
    },
    {
      "activations/layer0_attention_weight_max": 15.693596839904785,
      "activations/layer0_attention_weight_min": -14.062426567077637,
      "activations/layer10_attention_weight_max": 39.9251708984375,
      "activations/layer10_attention_weight_min": -40.27203369140625,
      "activations/layer11_attention_weight_max": 34.37262725830078,
      "activations/layer11_attention_weight_min": -33.98137664794922,
      "activations/layer12_attention_weight_max": 17.348501205444336,
      "activations/layer12_attention_weight_min": -31.642805099487305,
      "activations/layer13_attention_weight_max": 51.00064468383789,
      "activations/layer13_attention_weight_min": -31.57464027404785,
      "activations/layer14_attention_weight_max": 42.370399475097656,
      "activations/layer14_attention_weight_min": -31.79250717163086,
      "activations/layer15_attention_weight_max": 34.48582458496094,
      "activations/layer15_attention_weight_min": -28.97340202331543,
      "activations/layer16_attention_weight_max": 32.400630950927734,
      "activations/layer16_attention_weight_min": -29.900169372558594,
      "activations/layer17_attention_weight_max": 60.30998611450195,
      "activations/layer17_attention_weight_min": -47.035457611083984,
      "activations/layer18_attention_weight_max": 47.9002571105957,
      "activations/layer18_attention_weight_min": -36.78550338745117,
      "activations/layer19_attention_weight_max": 20.948097229003906,
      "activations/layer19_attention_weight_min": -20.586904525756836,
      "activations/layer1_attention_weight_max": 16.163896560668945,
      "activations/layer1_attention_weight_min": -15.129605293273926,
      "activations/layer20_attention_weight_max": 23.601804733276367,
      "activations/layer20_attention_weight_min": -19.948312759399414,
      "activations/layer21_attention_weight_max": 34.639217376708984,
      "activations/layer21_attention_weight_min": -19.13812255859375,
      "activations/layer22_attention_weight_max": 31.646100997924805,
      "activations/layer22_attention_weight_min": -26.61075782775879,
      "activations/layer23_attention_weight_max": 38.7188720703125,
      "activations/layer23_attention_weight_min": -21.655559539794922,
      "activations/layer2_attention_weight_max": 32.67198944091797,
      "activations/layer2_attention_weight_min": -30.827320098876953,
      "activations/layer3_attention_weight_max": 91.66400146484375,
      "activations/layer3_attention_weight_min": -91.52700805664062,
      "activations/layer4_attention_weight_max": 101.68621063232422,
      "activations/layer4_attention_weight_min": -95.9826889038086,
      "activations/layer5_attention_weight_max": 78.33363342285156,
      "activations/layer5_attention_weight_min": -81.12445068359375,
      "activations/layer6_attention_weight_max": 59.573219299316406,
      "activations/layer6_attention_weight_min": -55.396148681640625,
      "activations/layer7_attention_weight_max": 77.26685333251953,
      "activations/layer7_attention_weight_min": -69.83905029296875,
      "activations/layer8_attention_weight_max": 45.581912994384766,
      "activations/layer8_attention_weight_min": -49.62605285644531,
      "activations/layer9_attention_weight_max": 50.449317932128906,
      "activations/layer9_attention_weight_min": -50.42647933959961,
      "epoch": 10.49,
      "learning_rate": 8.32215909090909e-05,
      "loss": 2.7753,
      "step": 180450
    },
    {
      "activations/layer0_attention_weight_max": 16.091259002685547,
      "activations/layer0_attention_weight_min": -14.338895797729492,
      "activations/layer10_attention_weight_max": 35.62798309326172,
      "activations/layer10_attention_weight_min": -36.748470306396484,
      "activations/layer11_attention_weight_max": 32.59526824951172,
      "activations/layer11_attention_weight_min": -32.942535400390625,
      "activations/layer12_attention_weight_max": 19.040483474731445,
      "activations/layer12_attention_weight_min": -28.08588409423828,
      "activations/layer13_attention_weight_max": 39.102256774902344,
      "activations/layer13_attention_weight_min": -30.10843849182129,
      "activations/layer14_attention_weight_max": 36.02590560913086,
      "activations/layer14_attention_weight_min": -31.130603790283203,
      "activations/layer15_attention_weight_max": 30.599958419799805,
      "activations/layer15_attention_weight_min": -28.344593048095703,
      "activations/layer16_attention_weight_max": 33.1478385925293,
      "activations/layer16_attention_weight_min": -28.405961990356445,
      "activations/layer17_attention_weight_max": 55.17500305175781,
      "activations/layer17_attention_weight_min": -41.711647033691406,
      "activations/layer18_attention_weight_max": 44.90522766113281,
      "activations/layer18_attention_weight_min": -36.26384735107422,
      "activations/layer19_attention_weight_max": 23.91115379333496,
      "activations/layer19_attention_weight_min": -19.124835968017578,
      "activations/layer1_attention_weight_max": 17.271926879882812,
      "activations/layer1_attention_weight_min": -16.342533111572266,
      "activations/layer20_attention_weight_max": 20.736082077026367,
      "activations/layer20_attention_weight_min": -20.342655181884766,
      "activations/layer21_attention_weight_max": 30.665023803710938,
      "activations/layer21_attention_weight_min": -22.95877456665039,
      "activations/layer22_attention_weight_max": 31.23224449157715,
      "activations/layer22_attention_weight_min": -26.32624626159668,
      "activations/layer23_attention_weight_max": 36.72385025024414,
      "activations/layer23_attention_weight_min": -21.980194091796875,
      "activations/layer2_attention_weight_max": 32.36809539794922,
      "activations/layer2_attention_weight_min": -32.42523956298828,
      "activations/layer3_attention_weight_max": 94.43325805664062,
      "activations/layer3_attention_weight_min": -94.58613586425781,
      "activations/layer4_attention_weight_max": 97.35250091552734,
      "activations/layer4_attention_weight_min": -94.21025085449219,
      "activations/layer5_attention_weight_max": 75.0572280883789,
      "activations/layer5_attention_weight_min": -78.03269958496094,
      "activations/layer6_attention_weight_max": 53.18265914916992,
      "activations/layer6_attention_weight_min": -52.284202575683594,
      "activations/layer7_attention_weight_max": 75.55453491210938,
      "activations/layer7_attention_weight_min": -74.05731201171875,
      "activations/layer8_attention_weight_max": 44.9582405090332,
      "activations/layer8_attention_weight_min": -46.5675163269043,
      "activations/layer9_attention_weight_max": 46.456764221191406,
      "activations/layer9_attention_weight_min": -49.40359115600586,
      "epoch": 10.49,
      "learning_rate": 8.320265151515151e-05,
      "loss": 2.8024,
      "step": 180500
    },
    {
      "activations/layer0_attention_weight_max": 14.897916793823242,
      "activations/layer0_attention_weight_min": -14.351483345031738,
      "activations/layer10_attention_weight_max": 38.6480712890625,
      "activations/layer10_attention_weight_min": -37.72542190551758,
      "activations/layer11_attention_weight_max": 32.598697662353516,
      "activations/layer11_attention_weight_min": -32.948486328125,
      "activations/layer12_attention_weight_max": 17.65383529663086,
      "activations/layer12_attention_weight_min": -25.52774429321289,
      "activations/layer13_attention_weight_max": 42.79143524169922,
      "activations/layer13_attention_weight_min": -32.20640563964844,
      "activations/layer14_attention_weight_max": 38.17517852783203,
      "activations/layer14_attention_weight_min": -29.953371047973633,
      "activations/layer15_attention_weight_max": 33.83635330200195,
      "activations/layer15_attention_weight_min": -29.139781951904297,
      "activations/layer16_attention_weight_max": 30.466997146606445,
      "activations/layer16_attention_weight_min": -28.195844650268555,
      "activations/layer17_attention_weight_max": 51.051300048828125,
      "activations/layer17_attention_weight_min": -42.79143524169922,
      "activations/layer18_attention_weight_max": 44.83784866333008,
      "activations/layer18_attention_weight_min": -38.20446014404297,
      "activations/layer19_attention_weight_max": 20.508501052856445,
      "activations/layer19_attention_weight_min": -21.302160263061523,
      "activations/layer1_attention_weight_max": 17.113779067993164,
      "activations/layer1_attention_weight_min": -14.672152519226074,
      "activations/layer20_attention_weight_max": 21.88384437561035,
      "activations/layer20_attention_weight_min": -23.888755798339844,
      "activations/layer21_attention_weight_max": 36.49565505981445,
      "activations/layer21_attention_weight_min": -22.914796829223633,
      "activations/layer22_attention_weight_max": 32.31781768798828,
      "activations/layer22_attention_weight_min": -26.20067596435547,
      "activations/layer23_attention_weight_max": 39.14608383178711,
      "activations/layer23_attention_weight_min": -24.488414764404297,
      "activations/layer2_attention_weight_max": 32.439666748046875,
      "activations/layer2_attention_weight_min": -32.36573028564453,
      "activations/layer3_attention_weight_max": 92.14643859863281,
      "activations/layer3_attention_weight_min": -94.14338684082031,
      "activations/layer4_attention_weight_max": 100.92469787597656,
      "activations/layer4_attention_weight_min": -96.79875183105469,
      "activations/layer5_attention_weight_max": 81.11808013916016,
      "activations/layer5_attention_weight_min": -80.45380401611328,
      "activations/layer6_attention_weight_max": 58.34727096557617,
      "activations/layer6_attention_weight_min": -55.36111831665039,
      "activations/layer7_attention_weight_max": 77.19322204589844,
      "activations/layer7_attention_weight_min": -75.21302795410156,
      "activations/layer8_attention_weight_max": 49.751583099365234,
      "activations/layer8_attention_weight_min": -46.863338470458984,
      "activations/layer9_attention_weight_max": 49.476985931396484,
      "activations/layer9_attention_weight_min": -51.99415969848633,
      "epoch": 10.49,
      "learning_rate": 8.31837121212121e-05,
      "loss": 2.808,
      "step": 180550
    },
    {
      "activations/layer0_attention_weight_max": 16.358356475830078,
      "activations/layer0_attention_weight_min": -14.16257381439209,
      "activations/layer10_attention_weight_max": 36.431610107421875,
      "activations/layer10_attention_weight_min": -36.487125396728516,
      "activations/layer11_attention_weight_max": 33.049198150634766,
      "activations/layer11_attention_weight_min": -32.086090087890625,
      "activations/layer12_attention_weight_max": 19.33405876159668,
      "activations/layer12_attention_weight_min": -27.125160217285156,
      "activations/layer13_attention_weight_max": 46.96909713745117,
      "activations/layer13_attention_weight_min": -31.440927505493164,
      "activations/layer14_attention_weight_max": 44.875328063964844,
      "activations/layer14_attention_weight_min": -34.15674591064453,
      "activations/layer15_attention_weight_max": 37.97202682495117,
      "activations/layer15_attention_weight_min": -29.396020889282227,
      "activations/layer16_attention_weight_max": 33.685386657714844,
      "activations/layer16_attention_weight_min": -28.704788208007812,
      "activations/layer17_attention_weight_max": 56.99684524536133,
      "activations/layer17_attention_weight_min": -43.7868537902832,
      "activations/layer18_attention_weight_max": 47.17273712158203,
      "activations/layer18_attention_weight_min": -36.92924499511719,
      "activations/layer19_attention_weight_max": 25.26262092590332,
      "activations/layer19_attention_weight_min": -24.333173751831055,
      "activations/layer1_attention_weight_max": 19.89788055419922,
      "activations/layer1_attention_weight_min": -15.787114143371582,
      "activations/layer20_attention_weight_max": 27.404155731201172,
      "activations/layer20_attention_weight_min": -24.463787078857422,
      "activations/layer21_attention_weight_max": 41.3328971862793,
      "activations/layer21_attention_weight_min": -22.78797149658203,
      "activations/layer22_attention_weight_max": 35.9485969543457,
      "activations/layer22_attention_weight_min": -25.377838134765625,
      "activations/layer23_attention_weight_max": 42.56737518310547,
      "activations/layer23_attention_weight_min": -24.69623374938965,
      "activations/layer2_attention_weight_max": 32.08002471923828,
      "activations/layer2_attention_weight_min": -31.893356323242188,
      "activations/layer3_attention_weight_max": 95.42879486083984,
      "activations/layer3_attention_weight_min": -98.0320053100586,
      "activations/layer4_attention_weight_max": 101.6493148803711,
      "activations/layer4_attention_weight_min": -95.8061294555664,
      "activations/layer5_attention_weight_max": 79.6286392211914,
      "activations/layer5_attention_weight_min": -77.53742980957031,
      "activations/layer6_attention_weight_max": 56.91920852661133,
      "activations/layer6_attention_weight_min": -53.59916305541992,
      "activations/layer7_attention_weight_max": 79.80583953857422,
      "activations/layer7_attention_weight_min": -73.97908782958984,
      "activations/layer8_attention_weight_max": 47.1437873840332,
      "activations/layer8_attention_weight_min": -45.7017707824707,
      "activations/layer9_attention_weight_max": 54.39085006713867,
      "activations/layer9_attention_weight_min": -46.101070404052734,
      "epoch": 10.49,
      "learning_rate": 8.316515151515151e-05,
      "loss": 2.7742,
      "step": 180600
    },
    {
      "activations/layer0_attention_weight_max": 15.984743118286133,
      "activations/layer0_attention_weight_min": -13.98894214630127,
      "activations/layer10_attention_weight_max": 35.96731948852539,
      "activations/layer10_attention_weight_min": -35.92369079589844,
      "activations/layer11_attention_weight_max": 31.436758041381836,
      "activations/layer11_attention_weight_min": -31.763652801513672,
      "activations/layer12_attention_weight_max": 18.7873592376709,
      "activations/layer12_attention_weight_min": -25.859760284423828,
      "activations/layer13_attention_weight_max": 38.78575134277344,
      "activations/layer13_attention_weight_min": -28.19610023498535,
      "activations/layer14_attention_weight_max": 41.40780258178711,
      "activations/layer14_attention_weight_min": -30.642127990722656,
      "activations/layer15_attention_weight_max": 36.21467590332031,
      "activations/layer15_attention_weight_min": -28.778793334960938,
      "activations/layer16_attention_weight_max": 33.67148208618164,
      "activations/layer16_attention_weight_min": -29.964099884033203,
      "activations/layer17_attention_weight_max": 60.0914306640625,
      "activations/layer17_attention_weight_min": -46.70216751098633,
      "activations/layer18_attention_weight_max": 53.51530838012695,
      "activations/layer18_attention_weight_min": -38.303157806396484,
      "activations/layer19_attention_weight_max": 29.27861976623535,
      "activations/layer19_attention_weight_min": -21.894010543823242,
      "activations/layer1_attention_weight_max": 16.847381591796875,
      "activations/layer1_attention_weight_min": -14.419318199157715,
      "activations/layer20_attention_weight_max": 27.155866622924805,
      "activations/layer20_attention_weight_min": -22.309598922729492,
      "activations/layer21_attention_weight_max": 44.28429412841797,
      "activations/layer21_attention_weight_min": -23.435672760009766,
      "activations/layer22_attention_weight_max": 36.81241226196289,
      "activations/layer22_attention_weight_min": -27.058795928955078,
      "activations/layer23_attention_weight_max": 46.657142639160156,
      "activations/layer23_attention_weight_min": -21.195524215698242,
      "activations/layer2_attention_weight_max": 30.89730453491211,
      "activations/layer2_attention_weight_min": -29.793882369995117,
      "activations/layer3_attention_weight_max": 93.0350341796875,
      "activations/layer3_attention_weight_min": -96.37300872802734,
      "activations/layer4_attention_weight_max": 104.97625732421875,
      "activations/layer4_attention_weight_min": -100.68258666992188,
      "activations/layer5_attention_weight_max": 82.74845886230469,
      "activations/layer5_attention_weight_min": -81.35861206054688,
      "activations/layer6_attention_weight_max": 56.430206298828125,
      "activations/layer6_attention_weight_min": -54.48334503173828,
      "activations/layer7_attention_weight_max": 79.69060516357422,
      "activations/layer7_attention_weight_min": -77.65544128417969,
      "activations/layer8_attention_weight_max": 46.12583541870117,
      "activations/layer8_attention_weight_min": -46.64006423950195,
      "activations/layer9_attention_weight_max": 51.456336975097656,
      "activations/layer9_attention_weight_min": -53.80794906616211,
      "epoch": 10.5,
      "learning_rate": 8.314621212121211e-05,
      "loss": 2.7886,
      "step": 180650
    },
    {
      "activations/layer0_attention_weight_max": 18.428239822387695,
      "activations/layer0_attention_weight_min": -13.742400169372559,
      "activations/layer10_attention_weight_max": 36.89322280883789,
      "activations/layer10_attention_weight_min": -33.6663818359375,
      "activations/layer11_attention_weight_max": 30.06275749206543,
      "activations/layer11_attention_weight_min": -31.506629943847656,
      "activations/layer12_attention_weight_max": 19.385465621948242,
      "activations/layer12_attention_weight_min": -27.576068878173828,
      "activations/layer13_attention_weight_max": 35.991294860839844,
      "activations/layer13_attention_weight_min": -30.484569549560547,
      "activations/layer14_attention_weight_max": 35.52370071411133,
      "activations/layer14_attention_weight_min": -29.09672737121582,
      "activations/layer15_attention_weight_max": 31.803983688354492,
      "activations/layer15_attention_weight_min": -29.62870216369629,
      "activations/layer16_attention_weight_max": 33.1956672668457,
      "activations/layer16_attention_weight_min": -27.981515884399414,
      "activations/layer17_attention_weight_max": 49.7375373840332,
      "activations/layer17_attention_weight_min": -43.28535842895508,
      "activations/layer18_attention_weight_max": 44.09174728393555,
      "activations/layer18_attention_weight_min": -36.617408752441406,
      "activations/layer19_attention_weight_max": 22.384923934936523,
      "activations/layer19_attention_weight_min": -20.884912490844727,
      "activations/layer1_attention_weight_max": 17.079833984375,
      "activations/layer1_attention_weight_min": -15.516825675964355,
      "activations/layer20_attention_weight_max": 22.637666702270508,
      "activations/layer20_attention_weight_min": -22.174903869628906,
      "activations/layer21_attention_weight_max": 36.862998962402344,
      "activations/layer21_attention_weight_min": -24.958885192871094,
      "activations/layer22_attention_weight_max": 34.48250961303711,
      "activations/layer22_attention_weight_min": -26.797456741333008,
      "activations/layer23_attention_weight_max": 37.001068115234375,
      "activations/layer23_attention_weight_min": -26.443517684936523,
      "activations/layer2_attention_weight_max": 33.69236755371094,
      "activations/layer2_attention_weight_min": -30.81894302368164,
      "activations/layer3_attention_weight_max": 89.57024383544922,
      "activations/layer3_attention_weight_min": -89.82696533203125,
      "activations/layer4_attention_weight_max": 95.90587615966797,
      "activations/layer4_attention_weight_min": -90.49125671386719,
      "activations/layer5_attention_weight_max": 75.50213623046875,
      "activations/layer5_attention_weight_min": -78.396728515625,
      "activations/layer6_attention_weight_max": 54.03743362426758,
      "activations/layer6_attention_weight_min": -51.866050720214844,
      "activations/layer7_attention_weight_max": 70.61582946777344,
      "activations/layer7_attention_weight_min": -70.36016082763672,
      "activations/layer8_attention_weight_max": 43.818389892578125,
      "activations/layer8_attention_weight_min": -46.40446090698242,
      "activations/layer9_attention_weight_max": 50.38271713256836,
      "activations/layer9_attention_weight_min": -49.60697555541992,
      "epoch": 10.5,
      "learning_rate": 8.312727272727272e-05,
      "loss": 2.8002,
      "step": 180700
    },
    {
      "activations/layer0_attention_weight_max": 15.621623039245605,
      "activations/layer0_attention_weight_min": -14.311639785766602,
      "activations/layer10_attention_weight_max": 34.654659271240234,
      "activations/layer10_attention_weight_min": -33.96642303466797,
      "activations/layer11_attention_weight_max": 31.105695724487305,
      "activations/layer11_attention_weight_min": -31.65961265563965,
      "activations/layer12_attention_weight_max": 18.56843376159668,
      "activations/layer12_attention_weight_min": -21.726905822753906,
      "activations/layer13_attention_weight_max": 40.37263870239258,
      "activations/layer13_attention_weight_min": -28.96752166748047,
      "activations/layer14_attention_weight_max": 37.22423553466797,
      "activations/layer14_attention_weight_min": -29.35650634765625,
      "activations/layer15_attention_weight_max": 32.255985260009766,
      "activations/layer15_attention_weight_min": -28.45993995666504,
      "activations/layer16_attention_weight_max": 29.829219818115234,
      "activations/layer16_attention_weight_min": -27.678735733032227,
      "activations/layer17_attention_weight_max": 54.02215576171875,
      "activations/layer17_attention_weight_min": -43.32855987548828,
      "activations/layer18_attention_weight_max": 44.16680908203125,
      "activations/layer18_attention_weight_min": -36.79193115234375,
      "activations/layer19_attention_weight_max": 22.296846389770508,
      "activations/layer19_attention_weight_min": -23.57999038696289,
      "activations/layer1_attention_weight_max": 17.0771427154541,
      "activations/layer1_attention_weight_min": -14.651890754699707,
      "activations/layer20_attention_weight_max": 21.73099136352539,
      "activations/layer20_attention_weight_min": -22.448060989379883,
      "activations/layer21_attention_weight_max": 38.44324493408203,
      "activations/layer21_attention_weight_min": -23.214757919311523,
      "activations/layer22_attention_weight_max": 30.76934051513672,
      "activations/layer22_attention_weight_min": -25.908912658691406,
      "activations/layer23_attention_weight_max": 37.267730712890625,
      "activations/layer23_attention_weight_min": -23.16571044921875,
      "activations/layer2_attention_weight_max": 32.387359619140625,
      "activations/layer2_attention_weight_min": -33.049652099609375,
      "activations/layer3_attention_weight_max": 91.8837890625,
      "activations/layer3_attention_weight_min": -101.62139892578125,
      "activations/layer4_attention_weight_max": 101.66361236572266,
      "activations/layer4_attention_weight_min": -94.97032928466797,
      "activations/layer5_attention_weight_max": 76.39884948730469,
      "activations/layer5_attention_weight_min": -83.43082427978516,
      "activations/layer6_attention_weight_max": 55.538578033447266,
      "activations/layer6_attention_weight_min": -54.46798324584961,
      "activations/layer7_attention_weight_max": 73.10167694091797,
      "activations/layer7_attention_weight_min": -79.41448974609375,
      "activations/layer8_attention_weight_max": 44.46652603149414,
      "activations/layer8_attention_weight_min": -47.853309631347656,
      "activations/layer9_attention_weight_max": 51.6354866027832,
      "activations/layer9_attention_weight_min": -50.0192985534668,
      "epoch": 10.5,
      "learning_rate": 8.310871212121211e-05,
      "loss": 2.7986,
      "step": 180750
    },
    {
      "activations/layer0_attention_weight_max": 15.19013500213623,
      "activations/layer0_attention_weight_min": -14.345284461975098,
      "activations/layer10_attention_weight_max": 34.945796966552734,
      "activations/layer10_attention_weight_min": -35.026798248291016,
      "activations/layer11_attention_weight_max": 31.418384552001953,
      "activations/layer11_attention_weight_min": -32.06648254394531,
      "activations/layer12_attention_weight_max": 21.237585067749023,
      "activations/layer12_attention_weight_min": -26.613636016845703,
      "activations/layer13_attention_weight_max": 39.62770080566406,
      "activations/layer13_attention_weight_min": -30.276968002319336,
      "activations/layer14_attention_weight_max": 34.54063034057617,
      "activations/layer14_attention_weight_min": -31.01323699951172,
      "activations/layer15_attention_weight_max": 33.535919189453125,
      "activations/layer15_attention_weight_min": -28.745506286621094,
      "activations/layer16_attention_weight_max": 29.273775100708008,
      "activations/layer16_attention_weight_min": -29.139963150024414,
      "activations/layer17_attention_weight_max": 54.96280288696289,
      "activations/layer17_attention_weight_min": -45.87956619262695,
      "activations/layer18_attention_weight_max": 46.3938102722168,
      "activations/layer18_attention_weight_min": -37.5617561340332,
      "activations/layer19_attention_weight_max": 22.18509292602539,
      "activations/layer19_attention_weight_min": -20.193132400512695,
      "activations/layer1_attention_weight_max": 16.075336456298828,
      "activations/layer1_attention_weight_min": -14.866455078125,
      "activations/layer20_attention_weight_max": 21.817087173461914,
      "activations/layer20_attention_weight_min": -22.75959014892578,
      "activations/layer21_attention_weight_max": 36.11797332763672,
      "activations/layer21_attention_weight_min": -22.179906845092773,
      "activations/layer22_attention_weight_max": 30.35287857055664,
      "activations/layer22_attention_weight_min": -27.52777862548828,
      "activations/layer23_attention_weight_max": 36.83699417114258,
      "activations/layer23_attention_weight_min": -22.239957809448242,
      "activations/layer2_attention_weight_max": 31.741477966308594,
      "activations/layer2_attention_weight_min": -30.078277587890625,
      "activations/layer3_attention_weight_max": 90.48434448242188,
      "activations/layer3_attention_weight_min": -90.0737533569336,
      "activations/layer4_attention_weight_max": 98.28324127197266,
      "activations/layer4_attention_weight_min": -94.29000854492188,
      "activations/layer5_attention_weight_max": 81.50289154052734,
      "activations/layer5_attention_weight_min": -86.45858764648438,
      "activations/layer6_attention_weight_max": 55.525962829589844,
      "activations/layer6_attention_weight_min": -52.3901481628418,
      "activations/layer7_attention_weight_max": 72.53336334228516,
      "activations/layer7_attention_weight_min": -72.1661605834961,
      "activations/layer8_attention_weight_max": 44.17415237426758,
      "activations/layer8_attention_weight_min": -49.38713455200195,
      "activations/layer9_attention_weight_max": 49.69143295288086,
      "activations/layer9_attention_weight_min": -48.07769775390625,
      "epoch": 10.51,
      "learning_rate": 8.308977272727272e-05,
      "loss": 2.7668,
      "step": 180800
    },
    {
      "activations/layer0_attention_weight_max": 15.276949882507324,
      "activations/layer0_attention_weight_min": -14.600276947021484,
      "activations/layer10_attention_weight_max": 39.052459716796875,
      "activations/layer10_attention_weight_min": -35.667781829833984,
      "activations/layer11_attention_weight_max": 34.551204681396484,
      "activations/layer11_attention_weight_min": -33.722999572753906,
      "activations/layer12_attention_weight_max": 18.258832931518555,
      "activations/layer12_attention_weight_min": -27.337038040161133,
      "activations/layer13_attention_weight_max": 40.87498474121094,
      "activations/layer13_attention_weight_min": -29.97957992553711,
      "activations/layer14_attention_weight_max": 41.01103210449219,
      "activations/layer14_attention_weight_min": -31.478803634643555,
      "activations/layer15_attention_weight_max": 39.617069244384766,
      "activations/layer15_attention_weight_min": -30.306472778320312,
      "activations/layer16_attention_weight_max": 31.050308227539062,
      "activations/layer16_attention_weight_min": -29.68189239501953,
      "activations/layer17_attention_weight_max": 55.75323486328125,
      "activations/layer17_attention_weight_min": -47.46419906616211,
      "activations/layer18_attention_weight_max": 49.548011779785156,
      "activations/layer18_attention_weight_min": -41.30864334106445,
      "activations/layer19_attention_weight_max": 22.974973678588867,
      "activations/layer19_attention_weight_min": -23.440340042114258,
      "activations/layer1_attention_weight_max": 17.063554763793945,
      "activations/layer1_attention_weight_min": -15.904227256774902,
      "activations/layer20_attention_weight_max": 24.304170608520508,
      "activations/layer20_attention_weight_min": -23.673362731933594,
      "activations/layer21_attention_weight_max": 43.21823501586914,
      "activations/layer21_attention_weight_min": -24.563268661499023,
      "activations/layer22_attention_weight_max": 33.1108512878418,
      "activations/layer22_attention_weight_min": -27.668476104736328,
      "activations/layer23_attention_weight_max": 36.20001983642578,
      "activations/layer23_attention_weight_min": -25.100637435913086,
      "activations/layer2_attention_weight_max": 32.32587814331055,
      "activations/layer2_attention_weight_min": -31.755657196044922,
      "activations/layer3_attention_weight_max": 90.92327117919922,
      "activations/layer3_attention_weight_min": -96.06781005859375,
      "activations/layer4_attention_weight_max": 100.62513732910156,
      "activations/layer4_attention_weight_min": -99.65255737304688,
      "activations/layer5_attention_weight_max": 76.3398208618164,
      "activations/layer5_attention_weight_min": -80.45683288574219,
      "activations/layer6_attention_weight_max": 55.18467330932617,
      "activations/layer6_attention_weight_min": -55.22990798950195,
      "activations/layer7_attention_weight_max": 78.92218017578125,
      "activations/layer7_attention_weight_min": -74.53182220458984,
      "activations/layer8_attention_weight_max": 46.53423309326172,
      "activations/layer8_attention_weight_min": -45.955787658691406,
      "activations/layer9_attention_weight_max": 51.2794303894043,
      "activations/layer9_attention_weight_min": -48.4957389831543,
      "epoch": 10.51,
      "learning_rate": 8.307083333333333e-05,
      "loss": 2.781,
      "step": 180850
    },
    {
      "activations/layer0_attention_weight_max": 15.35217571258545,
      "activations/layer0_attention_weight_min": -14.010885238647461,
      "activations/layer10_attention_weight_max": 37.904293060302734,
      "activations/layer10_attention_weight_min": -37.663002014160156,
      "activations/layer11_attention_weight_max": 36.497982025146484,
      "activations/layer11_attention_weight_min": -34.11935806274414,
      "activations/layer12_attention_weight_max": 17.707483291625977,
      "activations/layer12_attention_weight_min": -26.415714263916016,
      "activations/layer13_attention_weight_max": 40.54610824584961,
      "activations/layer13_attention_weight_min": -29.73804473876953,
      "activations/layer14_attention_weight_max": 36.482906341552734,
      "activations/layer14_attention_weight_min": -32.79848861694336,
      "activations/layer15_attention_weight_max": 35.80294418334961,
      "activations/layer15_attention_weight_min": -30.434709548950195,
      "activations/layer16_attention_weight_max": 32.069671630859375,
      "activations/layer16_attention_weight_min": -28.627437591552734,
      "activations/layer17_attention_weight_max": 54.8428840637207,
      "activations/layer17_attention_weight_min": -45.0712890625,
      "activations/layer18_attention_weight_max": 44.95847702026367,
      "activations/layer18_attention_weight_min": -36.34488296508789,
      "activations/layer19_attention_weight_max": 21.750804901123047,
      "activations/layer19_attention_weight_min": -22.121118545532227,
      "activations/layer1_attention_weight_max": 17.537734985351562,
      "activations/layer1_attention_weight_min": -13.773665428161621,
      "activations/layer20_attention_weight_max": 24.97786521911621,
      "activations/layer20_attention_weight_min": -22.18459129333496,
      "activations/layer21_attention_weight_max": 37.003841400146484,
      "activations/layer21_attention_weight_min": -22.660070419311523,
      "activations/layer22_attention_weight_max": 29.943269729614258,
      "activations/layer22_attention_weight_min": -24.31587028503418,
      "activations/layer23_attention_weight_max": 34.11695861816406,
      "activations/layer23_attention_weight_min": -21.035045623779297,
      "activations/layer2_attention_weight_max": 31.278461456298828,
      "activations/layer2_attention_weight_min": -32.097415924072266,
      "activations/layer3_attention_weight_max": 91.65693664550781,
      "activations/layer3_attention_weight_min": -96.9230728149414,
      "activations/layer4_attention_weight_max": 102.2152099609375,
      "activations/layer4_attention_weight_min": -100.1533203125,
      "activations/layer5_attention_weight_max": 77.59912109375,
      "activations/layer5_attention_weight_min": -81.06655883789062,
      "activations/layer6_attention_weight_max": 54.739315032958984,
      "activations/layer6_attention_weight_min": -56.05516052246094,
      "activations/layer7_attention_weight_max": 78.66407775878906,
      "activations/layer7_attention_weight_min": -76.30239868164062,
      "activations/layer8_attention_weight_max": 46.52285385131836,
      "activations/layer8_attention_weight_min": -48.723594665527344,
      "activations/layer9_attention_weight_max": 49.52548599243164,
      "activations/layer9_attention_weight_min": -52.1179313659668,
      "epoch": 10.51,
      "learning_rate": 8.305227272727272e-05,
      "loss": 2.7762,
      "step": 180900
    },
    {
      "activations/layer0_attention_weight_max": 16.289447784423828,
      "activations/layer0_attention_weight_min": -14.218589782714844,
      "activations/layer10_attention_weight_max": 35.698509216308594,
      "activations/layer10_attention_weight_min": -34.195945739746094,
      "activations/layer11_attention_weight_max": 30.624858856201172,
      "activations/layer11_attention_weight_min": -33.00973129272461,
      "activations/layer12_attention_weight_max": 19.815475463867188,
      "activations/layer12_attention_weight_min": -22.868202209472656,
      "activations/layer13_attention_weight_max": 46.001895904541016,
      "activations/layer13_attention_weight_min": -30.84617805480957,
      "activations/layer14_attention_weight_max": 40.249298095703125,
      "activations/layer14_attention_weight_min": -31.577219009399414,
      "activations/layer15_attention_weight_max": 35.595245361328125,
      "activations/layer15_attention_weight_min": -30.25898551940918,
      "activations/layer16_attention_weight_max": 29.69170570373535,
      "activations/layer16_attention_weight_min": -30.094255447387695,
      "activations/layer17_attention_weight_max": 55.455665588378906,
      "activations/layer17_attention_weight_min": -46.64311981201172,
      "activations/layer18_attention_weight_max": 50.487545013427734,
      "activations/layer18_attention_weight_min": -37.13791275024414,
      "activations/layer19_attention_weight_max": 26.532215118408203,
      "activations/layer19_attention_weight_min": -21.168701171875,
      "activations/layer1_attention_weight_max": 16.78905487060547,
      "activations/layer1_attention_weight_min": -14.335044860839844,
      "activations/layer20_attention_weight_max": 24.00519371032715,
      "activations/layer20_attention_weight_min": -22.392187118530273,
      "activations/layer21_attention_weight_max": 35.391841888427734,
      "activations/layer21_attention_weight_min": -20.906667709350586,
      "activations/layer22_attention_weight_max": 31.007137298583984,
      "activations/layer22_attention_weight_min": -26.97439193725586,
      "activations/layer23_attention_weight_max": 35.526466369628906,
      "activations/layer23_attention_weight_min": -25.14911460876465,
      "activations/layer2_attention_weight_max": 33.10365676879883,
      "activations/layer2_attention_weight_min": -29.6010799407959,
      "activations/layer3_attention_weight_max": 88.250732421875,
      "activations/layer3_attention_weight_min": -94.07142639160156,
      "activations/layer4_attention_weight_max": 92.7560043334961,
      "activations/layer4_attention_weight_min": -91.48506927490234,
      "activations/layer5_attention_weight_max": 71.78314971923828,
      "activations/layer5_attention_weight_min": -77.61540985107422,
      "activations/layer6_attention_weight_max": 51.94724655151367,
      "activations/layer6_attention_weight_min": -49.99867248535156,
      "activations/layer7_attention_weight_max": 78.8466567993164,
      "activations/layer7_attention_weight_min": -71.45826721191406,
      "activations/layer8_attention_weight_max": 43.22489929199219,
      "activations/layer8_attention_weight_min": -44.89152908325195,
      "activations/layer9_attention_weight_max": 48.24705123901367,
      "activations/layer9_attention_weight_min": -49.20131301879883,
      "epoch": 10.51,
      "learning_rate": 8.303333333333332e-05,
      "loss": 2.7796,
      "step": 180950
    },
    {
      "activations/layer0_attention_weight_max": 14.485237121582031,
      "activations/layer0_attention_weight_min": -13.947677612304688,
      "activations/layer10_attention_weight_max": 36.079383850097656,
      "activations/layer10_attention_weight_min": -36.211971282958984,
      "activations/layer11_attention_weight_max": 30.986095428466797,
      "activations/layer11_attention_weight_min": -31.260709762573242,
      "activations/layer12_attention_weight_max": 17.52850341796875,
      "activations/layer12_attention_weight_min": -25.928890228271484,
      "activations/layer13_attention_weight_max": 44.07362747192383,
      "activations/layer13_attention_weight_min": -30.00644874572754,
      "activations/layer14_attention_weight_max": 40.76647186279297,
      "activations/layer14_attention_weight_min": -34.30197525024414,
      "activations/layer15_attention_weight_max": 37.022159576416016,
      "activations/layer15_attention_weight_min": -31.79214859008789,
      "activations/layer16_attention_weight_max": 32.18877029418945,
      "activations/layer16_attention_weight_min": -32.76703643798828,
      "activations/layer17_attention_weight_max": 53.501060485839844,
      "activations/layer17_attention_weight_min": -49.06592559814453,
      "activations/layer18_attention_weight_max": 48.3648796081543,
      "activations/layer18_attention_weight_min": -41.59451675415039,
      "activations/layer19_attention_weight_max": 21.372591018676758,
      "activations/layer19_attention_weight_min": -24.91531753540039,
      "activations/layer1_attention_weight_max": 18.226783752441406,
      "activations/layer1_attention_weight_min": -15.081857681274414,
      "activations/layer20_attention_weight_max": 24.417076110839844,
      "activations/layer20_attention_weight_min": -22.805944442749023,
      "activations/layer21_attention_weight_max": 42.642372131347656,
      "activations/layer21_attention_weight_min": -26.84067153930664,
      "activations/layer22_attention_weight_max": 31.708526611328125,
      "activations/layer22_attention_weight_min": -29.80344009399414,
      "activations/layer23_attention_weight_max": 37.57390213012695,
      "activations/layer23_attention_weight_min": -25.805221557617188,
      "activations/layer2_attention_weight_max": 31.638242721557617,
      "activations/layer2_attention_weight_min": -30.556432723999023,
      "activations/layer3_attention_weight_max": 85.39732360839844,
      "activations/layer3_attention_weight_min": -92.90657043457031,
      "activations/layer4_attention_weight_max": 95.25255584716797,
      "activations/layer4_attention_weight_min": -91.85478973388672,
      "activations/layer5_attention_weight_max": 73.273193359375,
      "activations/layer5_attention_weight_min": -75.02195739746094,
      "activations/layer6_attention_weight_max": 52.93533706665039,
      "activations/layer6_attention_weight_min": -50.6683464050293,
      "activations/layer7_attention_weight_max": 75.25453186035156,
      "activations/layer7_attention_weight_min": -68.66675567626953,
      "activations/layer8_attention_weight_max": 43.84122085571289,
      "activations/layer8_attention_weight_min": -46.60721969604492,
      "activations/layer9_attention_weight_max": 49.593719482421875,
      "activations/layer9_attention_weight_min": -49.887664794921875,
      "epoch": 10.52,
      "learning_rate": 8.301439393939392e-05,
      "loss": 2.7644,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.4535,
      "eval_samples_per_second": 507.953,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.4535,
      "eval_openwebtext_samples_per_second": 507.953,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_wikitext_loss": 2.970703125,
      "eval_wikitext_ppl": 19.505629671380603,
      "eval_wikitext_runtime": 1.9821,
      "eval_wikitext_samples_per_second": 230.061,
      "step": 181000
    },
    {
      "epoch": 10.52,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.5746,
      "eval_lambada_samples_per_second": 508.534,
      "step": 181000
    },
    {
      "activations/layer0_attention_weight_max": 15.474699020385742,
      "activations/layer0_attention_weight_min": -13.869120597839355,
      "activations/layer10_attention_weight_max": 38.54588317871094,
      "activations/layer10_attention_weight_min": -36.83200454711914,
      "activations/layer11_attention_weight_max": 33.10327911376953,
      "activations/layer11_attention_weight_min": -32.958412170410156,
      "activations/layer12_attention_weight_max": 18.2891788482666,
      "activations/layer12_attention_weight_min": -28.558317184448242,
      "activations/layer13_attention_weight_max": 39.08797073364258,
      "activations/layer13_attention_weight_min": -32.03010559082031,
      "activations/layer14_attention_weight_max": 44.89091873168945,
      "activations/layer14_attention_weight_min": -30.41147232055664,
      "activations/layer15_attention_weight_max": 34.22358322143555,
      "activations/layer15_attention_weight_min": -29.370750427246094,
      "activations/layer16_attention_weight_max": 32.70419692993164,
      "activations/layer16_attention_weight_min": -28.27980613708496,
      "activations/layer17_attention_weight_max": 52.432228088378906,
      "activations/layer17_attention_weight_min": -44.116905212402344,
      "activations/layer18_attention_weight_max": 46.1990966796875,
      "activations/layer18_attention_weight_min": -38.60862731933594,
      "activations/layer19_attention_weight_max": 21.805471420288086,
      "activations/layer19_attention_weight_min": -23.330747604370117,
      "activations/layer1_attention_weight_max": 16.169696807861328,
      "activations/layer1_attention_weight_min": -14.855777740478516,
      "activations/layer20_attention_weight_max": 22.751192092895508,
      "activations/layer20_attention_weight_min": -20.454225540161133,
      "activations/layer21_attention_weight_max": 36.18359375,
      "activations/layer21_attention_weight_min": -22.151899337768555,
      "activations/layer22_attention_weight_max": 33.19480895996094,
      "activations/layer22_attention_weight_min": -25.43577766418457,
      "activations/layer23_attention_weight_max": 40.7506103515625,
      "activations/layer23_attention_weight_min": -24.79022216796875,
      "activations/layer2_attention_weight_max": 33.26434326171875,
      "activations/layer2_attention_weight_min": -31.5240478515625,
      "activations/layer3_attention_weight_max": 94.03914642333984,
      "activations/layer3_attention_weight_min": -93.39619445800781,
      "activations/layer4_attention_weight_max": 101.62709045410156,
      "activations/layer4_attention_weight_min": -97.13597106933594,
      "activations/layer5_attention_weight_max": 78.36100769042969,
      "activations/layer5_attention_weight_min": -80.08999633789062,
      "activations/layer6_attention_weight_max": 57.35490036010742,
      "activations/layer6_attention_weight_min": -54.71438217163086,
      "activations/layer7_attention_weight_max": 77.03113555908203,
      "activations/layer7_attention_weight_min": -75.65921020507812,
      "activations/layer8_attention_weight_max": 47.00482177734375,
      "activations/layer8_attention_weight_min": -49.096397399902344,
      "activations/layer9_attention_weight_max": 52.098052978515625,
      "activations/layer9_attention_weight_min": -52.18788528442383,
      "epoch": 10.52,
      "learning_rate": 8.299545454545454e-05,
      "loss": 2.7914,
      "step": 181050
    },
    {
      "activations/layer0_attention_weight_max": 15.148350715637207,
      "activations/layer0_attention_weight_min": -13.60012435913086,
      "activations/layer10_attention_weight_max": 37.03242492675781,
      "activations/layer10_attention_weight_min": -34.786376953125,
      "activations/layer11_attention_weight_max": 31.349645614624023,
      "activations/layer11_attention_weight_min": -32.01742935180664,
      "activations/layer12_attention_weight_max": 16.932178497314453,
      "activations/layer12_attention_weight_min": -29.86170768737793,
      "activations/layer13_attention_weight_max": 39.543617248535156,
      "activations/layer13_attention_weight_min": -28.7836856842041,
      "activations/layer14_attention_weight_max": 39.24723434448242,
      "activations/layer14_attention_weight_min": -30.673364639282227,
      "activations/layer15_attention_weight_max": 37.14662170410156,
      "activations/layer15_attention_weight_min": -29.274925231933594,
      "activations/layer16_attention_weight_max": 33.20212936401367,
      "activations/layer16_attention_weight_min": -29.72378158569336,
      "activations/layer17_attention_weight_max": 56.69087600708008,
      "activations/layer17_attention_weight_min": -46.755462646484375,
      "activations/layer18_attention_weight_max": 54.82593536376953,
      "activations/layer18_attention_weight_min": -40.628440856933594,
      "activations/layer19_attention_weight_max": 21.989946365356445,
      "activations/layer19_attention_weight_min": -21.706436157226562,
      "activations/layer1_attention_weight_max": 17.295677185058594,
      "activations/layer1_attention_weight_min": -13.05659008026123,
      "activations/layer20_attention_weight_max": 24.012584686279297,
      "activations/layer20_attention_weight_min": -22.21088409423828,
      "activations/layer21_attention_weight_max": 32.630615234375,
      "activations/layer21_attention_weight_min": -22.313007354736328,
      "activations/layer22_attention_weight_max": 28.72770881652832,
      "activations/layer22_attention_weight_min": -25.010332107543945,
      "activations/layer23_attention_weight_max": 36.79356002807617,
      "activations/layer23_attention_weight_min": -22.706817626953125,
      "activations/layer2_attention_weight_max": 30.769859313964844,
      "activations/layer2_attention_weight_min": -31.200345993041992,
      "activations/layer3_attention_weight_max": 86.16101837158203,
      "activations/layer3_attention_weight_min": -89.39469146728516,
      "activations/layer4_attention_weight_max": 94.82488250732422,
      "activations/layer4_attention_weight_min": -91.84236907958984,
      "activations/layer5_attention_weight_max": 72.18436431884766,
      "activations/layer5_attention_weight_min": -74.94657897949219,
      "activations/layer6_attention_weight_max": 52.82079315185547,
      "activations/layer6_attention_weight_min": -53.57084274291992,
      "activations/layer7_attention_weight_max": 76.57312774658203,
      "activations/layer7_attention_weight_min": -73.3503646850586,
      "activations/layer8_attention_weight_max": 47.21732711791992,
      "activations/layer8_attention_weight_min": -50.93915557861328,
      "activations/layer9_attention_weight_max": 48.14785385131836,
      "activations/layer9_attention_weight_min": -48.65079116821289,
      "epoch": 10.52,
      "learning_rate": 8.297651515151515e-05,
      "loss": 2.7903,
      "step": 181100
    },
    {
      "activations/layer0_attention_weight_max": 16.127437591552734,
      "activations/layer0_attention_weight_min": -14.212631225585938,
      "activations/layer10_attention_weight_max": 36.63714599609375,
      "activations/layer10_attention_weight_min": -36.016319274902344,
      "activations/layer11_attention_weight_max": 31.970937728881836,
      "activations/layer11_attention_weight_min": -33.43938446044922,
      "activations/layer12_attention_weight_max": 19.553266525268555,
      "activations/layer12_attention_weight_min": -26.423133850097656,
      "activations/layer13_attention_weight_max": 36.43794250488281,
      "activations/layer13_attention_weight_min": -29.09590721130371,
      "activations/layer14_attention_weight_max": 37.89706039428711,
      "activations/layer14_attention_weight_min": -29.93157958984375,
      "activations/layer15_attention_weight_max": 31.833871841430664,
      "activations/layer15_attention_weight_min": -30.814804077148438,
      "activations/layer16_attention_weight_max": 29.440793991088867,
      "activations/layer16_attention_weight_min": -28.391189575195312,
      "activations/layer17_attention_weight_max": 50.467594146728516,
      "activations/layer17_attention_weight_min": -44.13337707519531,
      "activations/layer18_attention_weight_max": 43.91647720336914,
      "activations/layer18_attention_weight_min": -37.81166076660156,
      "activations/layer19_attention_weight_max": 24.047849655151367,
      "activations/layer19_attention_weight_min": -20.392507553100586,
      "activations/layer1_attention_weight_max": 16.902706146240234,
      "activations/layer1_attention_weight_min": -14.574199676513672,
      "activations/layer20_attention_weight_max": 22.138076782226562,
      "activations/layer20_attention_weight_min": -21.53655242919922,
      "activations/layer21_attention_weight_max": 36.414310455322266,
      "activations/layer21_attention_weight_min": -23.587661743164062,
      "activations/layer22_attention_weight_max": 30.26692008972168,
      "activations/layer22_attention_weight_min": -26.847152709960938,
      "activations/layer23_attention_weight_max": 34.70759201049805,
      "activations/layer23_attention_weight_min": -24.457355499267578,
      "activations/layer2_attention_weight_max": 31.834304809570312,
      "activations/layer2_attention_weight_min": -29.388290405273438,
      "activations/layer3_attention_weight_max": 90.97627258300781,
      "activations/layer3_attention_weight_min": -96.23406982421875,
      "activations/layer4_attention_weight_max": 96.04422760009766,
      "activations/layer4_attention_weight_min": -94.14523315429688,
      "activations/layer5_attention_weight_max": 75.25396728515625,
      "activations/layer5_attention_weight_min": -81.45777893066406,
      "activations/layer6_attention_weight_max": 54.8133659362793,
      "activations/layer6_attention_weight_min": -54.428627014160156,
      "activations/layer7_attention_weight_max": 73.80207061767578,
      "activations/layer7_attention_weight_min": -73.2225112915039,
      "activations/layer8_attention_weight_max": 46.30317306518555,
      "activations/layer8_attention_weight_min": -49.59013748168945,
      "activations/layer9_attention_weight_max": 48.27627182006836,
      "activations/layer9_attention_weight_min": -51.197635650634766,
      "epoch": 10.53,
      "learning_rate": 8.295757575757574e-05,
      "loss": 2.7897,
      "step": 181150
    },
    {
      "activations/layer0_attention_weight_max": 16.93198013305664,
      "activations/layer0_attention_weight_min": -13.85364818572998,
      "activations/layer10_attention_weight_max": 35.26005935668945,
      "activations/layer10_attention_weight_min": -33.9249153137207,
      "activations/layer11_attention_weight_max": 30.321613311767578,
      "activations/layer11_attention_weight_min": -32.131988525390625,
      "activations/layer12_attention_weight_max": 18.642791748046875,
      "activations/layer12_attention_weight_min": -26.139747619628906,
      "activations/layer13_attention_weight_max": 39.288658142089844,
      "activations/layer13_attention_weight_min": -28.46314239501953,
      "activations/layer14_attention_weight_max": 36.48035430908203,
      "activations/layer14_attention_weight_min": -30.618484497070312,
      "activations/layer15_attention_weight_max": 33.26664352416992,
      "activations/layer15_attention_weight_min": -29.427574157714844,
      "activations/layer16_attention_weight_max": 30.950578689575195,
      "activations/layer16_attention_weight_min": -30.192935943603516,
      "activations/layer17_attention_weight_max": 53.96574783325195,
      "activations/layer17_attention_weight_min": -45.03591537475586,
      "activations/layer18_attention_weight_max": 48.597259521484375,
      "activations/layer18_attention_weight_min": -38.99110794067383,
      "activations/layer19_attention_weight_max": 22.83979034423828,
      "activations/layer19_attention_weight_min": -24.175289154052734,
      "activations/layer1_attention_weight_max": 17.113784790039062,
      "activations/layer1_attention_weight_min": -15.684721946716309,
      "activations/layer20_attention_weight_max": 24.265220642089844,
      "activations/layer20_attention_weight_min": -22.729480743408203,
      "activations/layer21_attention_weight_max": 41.44343566894531,
      "activations/layer21_attention_weight_min": -23.766990661621094,
      "activations/layer22_attention_weight_max": 32.00016784667969,
      "activations/layer22_attention_weight_min": -25.45145606994629,
      "activations/layer23_attention_weight_max": 38.35083770751953,
      "activations/layer23_attention_weight_min": -24.53790855407715,
      "activations/layer2_attention_weight_max": 31.5435791015625,
      "activations/layer2_attention_weight_min": -33.24146270751953,
      "activations/layer3_attention_weight_max": 86.91410827636719,
      "activations/layer3_attention_weight_min": -97.4245834350586,
      "activations/layer4_attention_weight_max": 95.36862182617188,
      "activations/layer4_attention_weight_min": -96.04783630371094,
      "activations/layer5_attention_weight_max": 71.73866271972656,
      "activations/layer5_attention_weight_min": -76.37110900878906,
      "activations/layer6_attention_weight_max": 51.10683059692383,
      "activations/layer6_attention_weight_min": -50.53423309326172,
      "activations/layer7_attention_weight_max": 72.43965911865234,
      "activations/layer7_attention_weight_min": -70.4306411743164,
      "activations/layer8_attention_weight_max": 42.27552795410156,
      "activations/layer8_attention_weight_min": -45.42512512207031,
      "activations/layer9_attention_weight_max": 47.865535736083984,
      "activations/layer9_attention_weight_min": -48.61774826049805,
      "epoch": 10.53,
      "learning_rate": 8.293863636363636e-05,
      "loss": 2.7886,
      "step": 181200
    },
    {
      "activations/layer0_attention_weight_max": 17.07671546936035,
      "activations/layer0_attention_weight_min": -13.908230781555176,
      "activations/layer10_attention_weight_max": 37.03773880004883,
      "activations/layer10_attention_weight_min": -35.064369201660156,
      "activations/layer11_attention_weight_max": 32.81852722167969,
      "activations/layer11_attention_weight_min": -32.20455551147461,
      "activations/layer12_attention_weight_max": 20.13789939880371,
      "activations/layer12_attention_weight_min": -26.1834774017334,
      "activations/layer13_attention_weight_max": 47.69269561767578,
      "activations/layer13_attention_weight_min": -31.80388069152832,
      "activations/layer14_attention_weight_max": 38.8693733215332,
      "activations/layer14_attention_weight_min": -30.955968856811523,
      "activations/layer15_attention_weight_max": 36.0727424621582,
      "activations/layer15_attention_weight_min": -31.437786102294922,
      "activations/layer16_attention_weight_max": 32.7911491394043,
      "activations/layer16_attention_weight_min": -30.35030174255371,
      "activations/layer17_attention_weight_max": 54.88423538208008,
      "activations/layer17_attention_weight_min": -47.349693298339844,
      "activations/layer18_attention_weight_max": 45.9760856628418,
      "activations/layer18_attention_weight_min": -41.33097457885742,
      "activations/layer19_attention_weight_max": 24.052106857299805,
      "activations/layer19_attention_weight_min": -23.58977699279785,
      "activations/layer1_attention_weight_max": 17.89472770690918,
      "activations/layer1_attention_weight_min": -14.933937072753906,
      "activations/layer20_attention_weight_max": 25.411882400512695,
      "activations/layer20_attention_weight_min": -25.40892219543457,
      "activations/layer21_attention_weight_max": 48.23807907104492,
      "activations/layer21_attention_weight_min": -25.4486026763916,
      "activations/layer22_attention_weight_max": 32.58907699584961,
      "activations/layer22_attention_weight_min": -30.692970275878906,
      "activations/layer23_attention_weight_max": 40.15373992919922,
      "activations/layer23_attention_weight_min": -29.295909881591797,
      "activations/layer2_attention_weight_max": 32.50465393066406,
      "activations/layer2_attention_weight_min": -31.318199157714844,
      "activations/layer3_attention_weight_max": 92.97863006591797,
      "activations/layer3_attention_weight_min": -98.3260498046875,
      "activations/layer4_attention_weight_max": 96.74396514892578,
      "activations/layer4_attention_weight_min": -92.79134368896484,
      "activations/layer5_attention_weight_max": 74.06338500976562,
      "activations/layer5_attention_weight_min": -78.31700897216797,
      "activations/layer6_attention_weight_max": 53.22073745727539,
      "activations/layer6_attention_weight_min": -50.90172576904297,
      "activations/layer7_attention_weight_max": 75.90592193603516,
      "activations/layer7_attention_weight_min": -75.63545227050781,
      "activations/layer8_attention_weight_max": 46.09547805786133,
      "activations/layer8_attention_weight_min": -48.389400482177734,
      "activations/layer9_attention_weight_max": 51.8652229309082,
      "activations/layer9_attention_weight_min": -51.12078857421875,
      "epoch": 10.53,
      "learning_rate": 8.291969696969697e-05,
      "loss": 2.7951,
      "step": 181250
    },
    {
      "activations/layer0_attention_weight_max": 14.744951248168945,
      "activations/layer0_attention_weight_min": -14.970359802246094,
      "activations/layer10_attention_weight_max": 38.45985794067383,
      "activations/layer10_attention_weight_min": -33.56324768066406,
      "activations/layer11_attention_weight_max": 34.56838607788086,
      "activations/layer11_attention_weight_min": -31.72383689880371,
      "activations/layer12_attention_weight_max": 20.286991119384766,
      "activations/layer12_attention_weight_min": -25.42803955078125,
      "activations/layer13_attention_weight_max": 38.69069290161133,
      "activations/layer13_attention_weight_min": -33.51753616333008,
      "activations/layer14_attention_weight_max": 34.406829833984375,
      "activations/layer14_attention_weight_min": -31.107051849365234,
      "activations/layer15_attention_weight_max": 34.165077209472656,
      "activations/layer15_attention_weight_min": -31.32185173034668,
      "activations/layer16_attention_weight_max": 33.25324630737305,
      "activations/layer16_attention_weight_min": -28.90414810180664,
      "activations/layer17_attention_weight_max": 52.296382904052734,
      "activations/layer17_attention_weight_min": -47.397369384765625,
      "activations/layer18_attention_weight_max": 45.51173782348633,
      "activations/layer18_attention_weight_min": -39.907562255859375,
      "activations/layer19_attention_weight_max": 24.991058349609375,
      "activations/layer19_attention_weight_min": -22.95216941833496,
      "activations/layer1_attention_weight_max": 16.04911231994629,
      "activations/layer1_attention_weight_min": -14.28158187866211,
      "activations/layer20_attention_weight_max": 23.79892349243164,
      "activations/layer20_attention_weight_min": -20.470470428466797,
      "activations/layer21_attention_weight_max": 41.05846405029297,
      "activations/layer21_attention_weight_min": -27.86467742919922,
      "activations/layer22_attention_weight_max": 31.983795166015625,
      "activations/layer22_attention_weight_min": -27.566452026367188,
      "activations/layer23_attention_weight_max": 37.105873107910156,
      "activations/layer23_attention_weight_min": -22.101314544677734,
      "activations/layer2_attention_weight_max": 30.164745330810547,
      "activations/layer2_attention_weight_min": -30.298322677612305,
      "activations/layer3_attention_weight_max": 85.93140411376953,
      "activations/layer3_attention_weight_min": -97.67466735839844,
      "activations/layer4_attention_weight_max": 94.56555938720703,
      "activations/layer4_attention_weight_min": -91.33467864990234,
      "activations/layer5_attention_weight_max": 76.64259338378906,
      "activations/layer5_attention_weight_min": -80.28750610351562,
      "activations/layer6_attention_weight_max": 54.259605407714844,
      "activations/layer6_attention_weight_min": -54.008419036865234,
      "activations/layer7_attention_weight_max": 80.4952163696289,
      "activations/layer7_attention_weight_min": -73.68560791015625,
      "activations/layer8_attention_weight_max": 47.83968734741211,
      "activations/layer8_attention_weight_min": -47.6420783996582,
      "activations/layer9_attention_weight_max": 51.08625793457031,
      "activations/layer9_attention_weight_min": -47.837093353271484,
      "epoch": 10.53,
      "learning_rate": 8.290075757575756e-05,
      "loss": 2.7865,
      "step": 181300
    },
    {
      "activations/layer0_attention_weight_max": 15.751778602600098,
      "activations/layer0_attention_weight_min": -14.677952766418457,
      "activations/layer10_attention_weight_max": 37.34604263305664,
      "activations/layer10_attention_weight_min": -36.646392822265625,
      "activations/layer11_attention_weight_max": 33.65007781982422,
      "activations/layer11_attention_weight_min": -33.49358367919922,
      "activations/layer12_attention_weight_max": 21.815086364746094,
      "activations/layer12_attention_weight_min": -22.487960815429688,
      "activations/layer13_attention_weight_max": 49.87391662597656,
      "activations/layer13_attention_weight_min": -28.10454559326172,
      "activations/layer14_attention_weight_max": 61.273681640625,
      "activations/layer14_attention_weight_min": -33.066715240478516,
      "activations/layer15_attention_weight_max": 43.46975326538086,
      "activations/layer15_attention_weight_min": -32.68531799316406,
      "activations/layer16_attention_weight_max": 35.08079528808594,
      "activations/layer16_attention_weight_min": -31.782258987426758,
      "activations/layer17_attention_weight_max": 62.64967346191406,
      "activations/layer17_attention_weight_min": -48.162174224853516,
      "activations/layer18_attention_weight_max": 54.59324264526367,
      "activations/layer18_attention_weight_min": -41.74799728393555,
      "activations/layer19_attention_weight_max": 27.945478439331055,
      "activations/layer19_attention_weight_min": -24.367382049560547,
      "activations/layer1_attention_weight_max": 18.2324161529541,
      "activations/layer1_attention_weight_min": -16.01350212097168,
      "activations/layer20_attention_weight_max": 28.877824783325195,
      "activations/layer20_attention_weight_min": -24.74041175842285,
      "activations/layer21_attention_weight_max": 42.66963577270508,
      "activations/layer21_attention_weight_min": -23.6855525970459,
      "activations/layer22_attention_weight_max": 38.63256072998047,
      "activations/layer22_attention_weight_min": -27.936189651489258,
      "activations/layer23_attention_weight_max": 43.75817108154297,
      "activations/layer23_attention_weight_min": -24.168827056884766,
      "activations/layer2_attention_weight_max": 29.811447143554688,
      "activations/layer2_attention_weight_min": -29.88107681274414,
      "activations/layer3_attention_weight_max": 84.63436889648438,
      "activations/layer3_attention_weight_min": -83.04893493652344,
      "activations/layer4_attention_weight_max": 94.8802719116211,
      "activations/layer4_attention_weight_min": -89.65098571777344,
      "activations/layer5_attention_weight_max": 73.28047180175781,
      "activations/layer5_attention_weight_min": -79.22969055175781,
      "activations/layer6_attention_weight_max": 53.73945999145508,
      "activations/layer6_attention_weight_min": -54.22685623168945,
      "activations/layer7_attention_weight_max": 74.06192016601562,
      "activations/layer7_attention_weight_min": -74.93367767333984,
      "activations/layer8_attention_weight_max": 46.59608840942383,
      "activations/layer8_attention_weight_min": -48.31521987915039,
      "activations/layer9_attention_weight_max": 55.08510971069336,
      "activations/layer9_attention_weight_min": -50.54948806762695,
      "epoch": 10.54,
      "learning_rate": 8.288181818181818e-05,
      "loss": 2.7796,
      "step": 181350
    },
    {
      "activations/layer0_attention_weight_max": 14.925288200378418,
      "activations/layer0_attention_weight_min": -14.464845657348633,
      "activations/layer10_attention_weight_max": 40.8305778503418,
      "activations/layer10_attention_weight_min": -36.52569580078125,
      "activations/layer11_attention_weight_max": 35.59918212890625,
      "activations/layer11_attention_weight_min": -34.7050666809082,
      "activations/layer12_attention_weight_max": 21.277891159057617,
      "activations/layer12_attention_weight_min": -25.685758590698242,
      "activations/layer13_attention_weight_max": 48.889278411865234,
      "activations/layer13_attention_weight_min": -33.84967803955078,
      "activations/layer14_attention_weight_max": 44.816314697265625,
      "activations/layer14_attention_weight_min": -30.382301330566406,
      "activations/layer15_attention_weight_max": 36.65532302856445,
      "activations/layer15_attention_weight_min": -28.627336502075195,
      "activations/layer16_attention_weight_max": 34.83183670043945,
      "activations/layer16_attention_weight_min": -29.541452407836914,
      "activations/layer17_attention_weight_max": 56.60118865966797,
      "activations/layer17_attention_weight_min": -44.74283218383789,
      "activations/layer18_attention_weight_max": 49.154212951660156,
      "activations/layer18_attention_weight_min": -37.51771926879883,
      "activations/layer19_attention_weight_max": 23.7407283782959,
      "activations/layer19_attention_weight_min": -21.54686164855957,
      "activations/layer1_attention_weight_max": 16.720983505249023,
      "activations/layer1_attention_weight_min": -13.663945198059082,
      "activations/layer20_attention_weight_max": 26.304636001586914,
      "activations/layer20_attention_weight_min": -22.497438430786133,
      "activations/layer21_attention_weight_max": 45.16074752807617,
      "activations/layer21_attention_weight_min": -22.204790115356445,
      "activations/layer22_attention_weight_max": 32.8648567199707,
      "activations/layer22_attention_weight_min": -25.317916870117188,
      "activations/layer23_attention_weight_max": 43.231956481933594,
      "activations/layer23_attention_weight_min": -22.174537658691406,
      "activations/layer2_attention_weight_max": 30.572040557861328,
      "activations/layer2_attention_weight_min": -29.39529037475586,
      "activations/layer3_attention_weight_max": 87.81214904785156,
      "activations/layer3_attention_weight_min": -89.59513092041016,
      "activations/layer4_attention_weight_max": 95.70657348632812,
      "activations/layer4_attention_weight_min": -92.22618865966797,
      "activations/layer5_attention_weight_max": 75.36271667480469,
      "activations/layer5_attention_weight_min": -80.95621490478516,
      "activations/layer6_attention_weight_max": 56.09866714477539,
      "activations/layer6_attention_weight_min": -55.95050048828125,
      "activations/layer7_attention_weight_max": 80.0507583618164,
      "activations/layer7_attention_weight_min": -75.26412963867188,
      "activations/layer8_attention_weight_max": 49.850730895996094,
      "activations/layer8_attention_weight_min": -49.377342224121094,
      "activations/layer9_attention_weight_max": 53.553466796875,
      "activations/layer9_attention_weight_min": -49.76376724243164,
      "epoch": 10.54,
      "learning_rate": 8.286287878787878e-05,
      "loss": 2.7997,
      "step": 181400
    },
    {
      "activations/layer0_attention_weight_max": 15.621061325073242,
      "activations/layer0_attention_weight_min": -13.885948181152344,
      "activations/layer10_attention_weight_max": 37.533695220947266,
      "activations/layer10_attention_weight_min": -36.31949234008789,
      "activations/layer11_attention_weight_max": 32.63861846923828,
      "activations/layer11_attention_weight_min": -32.22224044799805,
      "activations/layer12_attention_weight_max": 19.56863784790039,
      "activations/layer12_attention_weight_min": -26.821443557739258,
      "activations/layer13_attention_weight_max": 45.54159164428711,
      "activations/layer13_attention_weight_min": -28.575380325317383,
      "activations/layer14_attention_weight_max": 47.508140563964844,
      "activations/layer14_attention_weight_min": -34.67013168334961,
      "activations/layer15_attention_weight_max": 37.68316650390625,
      "activations/layer15_attention_weight_min": -30.74689483642578,
      "activations/layer16_attention_weight_max": 33.92483901977539,
      "activations/layer16_attention_weight_min": -30.98972511291504,
      "activations/layer17_attention_weight_max": 55.253719329833984,
      "activations/layer17_attention_weight_min": -46.954586029052734,
      "activations/layer18_attention_weight_max": 52.50229263305664,
      "activations/layer18_attention_weight_min": -40.358367919921875,
      "activations/layer19_attention_weight_max": 24.553159713745117,
      "activations/layer19_attention_weight_min": -23.081525802612305,
      "activations/layer1_attention_weight_max": 18.591032028198242,
      "activations/layer1_attention_weight_min": -15.135860443115234,
      "activations/layer20_attention_weight_max": 25.275707244873047,
      "activations/layer20_attention_weight_min": -20.24960708618164,
      "activations/layer21_attention_weight_max": 44.10933303833008,
      "activations/layer21_attention_weight_min": -21.292593002319336,
      "activations/layer22_attention_weight_max": 30.715145111083984,
      "activations/layer22_attention_weight_min": -25.269575119018555,
      "activations/layer23_attention_weight_max": 36.3619499206543,
      "activations/layer23_attention_weight_min": -22.279281616210938,
      "activations/layer2_attention_weight_max": 31.243642807006836,
      "activations/layer2_attention_weight_min": -31.30219078063965,
      "activations/layer3_attention_weight_max": 89.11034393310547,
      "activations/layer3_attention_weight_min": -91.5697250366211,
      "activations/layer4_attention_weight_max": 98.16625213623047,
      "activations/layer4_attention_weight_min": -91.9473648071289,
      "activations/layer5_attention_weight_max": 73.70999908447266,
      "activations/layer5_attention_weight_min": -76.30755615234375,
      "activations/layer6_attention_weight_max": 53.95762252807617,
      "activations/layer6_attention_weight_min": -51.61628723144531,
      "activations/layer7_attention_weight_max": 75.60741424560547,
      "activations/layer7_attention_weight_min": -69.53447723388672,
      "activations/layer8_attention_weight_max": 47.610416412353516,
      "activations/layer8_attention_weight_min": -47.9549674987793,
      "activations/layer9_attention_weight_max": 57.68781280517578,
      "activations/layer9_attention_weight_min": -51.03896713256836,
      "epoch": 10.54,
      "learning_rate": 8.284393939393938e-05,
      "loss": 2.7872,
      "step": 181450
    },
    {
      "activations/layer0_attention_weight_max": 15.911787033081055,
      "activations/layer0_attention_weight_min": -14.347094535827637,
      "activations/layer10_attention_weight_max": 43.134803771972656,
      "activations/layer10_attention_weight_min": -43.66551208496094,
      "activations/layer11_attention_weight_max": 38.76398468017578,
      "activations/layer11_attention_weight_min": -36.17913818359375,
      "activations/layer12_attention_weight_max": 20.544641494750977,
      "activations/layer12_attention_weight_min": -31.957963943481445,
      "activations/layer13_attention_weight_max": 46.07865905761719,
      "activations/layer13_attention_weight_min": -29.9991397857666,
      "activations/layer14_attention_weight_max": 39.1252326965332,
      "activations/layer14_attention_weight_min": -31.33783531188965,
      "activations/layer15_attention_weight_max": 34.608154296875,
      "activations/layer15_attention_weight_min": -32.262577056884766,
      "activations/layer16_attention_weight_max": 31.664566040039062,
      "activations/layer16_attention_weight_min": -33.143245697021484,
      "activations/layer17_attention_weight_max": 56.372413635253906,
      "activations/layer17_attention_weight_min": -46.30009841918945,
      "activations/layer18_attention_weight_max": 48.45801544189453,
      "activations/layer18_attention_weight_min": -38.060848236083984,
      "activations/layer19_attention_weight_max": 21.403043746948242,
      "activations/layer19_attention_weight_min": -20.126516342163086,
      "activations/layer1_attention_weight_max": 18.06890869140625,
      "activations/layer1_attention_weight_min": -13.683242797851562,
      "activations/layer20_attention_weight_max": 21.466604232788086,
      "activations/layer20_attention_weight_min": -21.599407196044922,
      "activations/layer21_attention_weight_max": 44.0713005065918,
      "activations/layer21_attention_weight_min": -24.010629653930664,
      "activations/layer22_attention_weight_max": 30.655427932739258,
      "activations/layer22_attention_weight_min": -26.47547149658203,
      "activations/layer23_attention_weight_max": 35.61296463012695,
      "activations/layer23_attention_weight_min": -24.83571434020996,
      "activations/layer2_attention_weight_max": 31.64998435974121,
      "activations/layer2_attention_weight_min": -29.863731384277344,
      "activations/layer3_attention_weight_max": 97.50806427001953,
      "activations/layer3_attention_weight_min": -95.70848846435547,
      "activations/layer4_attention_weight_max": 108.29302215576172,
      "activations/layer4_attention_weight_min": -99.55711364746094,
      "activations/layer5_attention_weight_max": 82.40805053710938,
      "activations/layer5_attention_weight_min": -88.65658569335938,
      "activations/layer6_attention_weight_max": 57.18279266357422,
      "activations/layer6_attention_weight_min": -55.041526794433594,
      "activations/layer7_attention_weight_max": 81.50840759277344,
      "activations/layer7_attention_weight_min": -76.40509033203125,
      "activations/layer8_attention_weight_max": 49.57733154296875,
      "activations/layer8_attention_weight_min": -51.43238830566406,
      "activations/layer9_attention_weight_max": 60.792152404785156,
      "activations/layer9_attention_weight_min": -59.2882080078125,
      "epoch": 10.55,
      "learning_rate": 8.2825e-05,
      "loss": 2.7769,
      "step": 181500
    },
    {
      "activations/layer0_attention_weight_max": 16.079723358154297,
      "activations/layer0_attention_weight_min": -13.794046401977539,
      "activations/layer10_attention_weight_max": 35.1409797668457,
      "activations/layer10_attention_weight_min": -34.28635787963867,
      "activations/layer11_attention_weight_max": 32.000858306884766,
      "activations/layer11_attention_weight_min": -30.754453659057617,
      "activations/layer12_attention_weight_max": 20.766597747802734,
      "activations/layer12_attention_weight_min": -26.01657485961914,
      "activations/layer13_attention_weight_max": 36.25490188598633,
      "activations/layer13_attention_weight_min": -32.74079132080078,
      "activations/layer14_attention_weight_max": 35.96821212768555,
      "activations/layer14_attention_weight_min": -30.000661849975586,
      "activations/layer15_attention_weight_max": 31.555204391479492,
      "activations/layer15_attention_weight_min": -28.244985580444336,
      "activations/layer16_attention_weight_max": 29.411123275756836,
      "activations/layer16_attention_weight_min": -29.25773811340332,
      "activations/layer17_attention_weight_max": 50.43828201293945,
      "activations/layer17_attention_weight_min": -42.55775451660156,
      "activations/layer18_attention_weight_max": 46.27323532104492,
      "activations/layer18_attention_weight_min": -36.81500244140625,
      "activations/layer19_attention_weight_max": 21.265775680541992,
      "activations/layer19_attention_weight_min": -20.854799270629883,
      "activations/layer1_attention_weight_max": 16.196441650390625,
      "activations/layer1_attention_weight_min": -14.288419723510742,
      "activations/layer20_attention_weight_max": 24.167339324951172,
      "activations/layer20_attention_weight_min": -23.20299530029297,
      "activations/layer21_attention_weight_max": 39.887123107910156,
      "activations/layer21_attention_weight_min": -23.49156379699707,
      "activations/layer22_attention_weight_max": 30.5968074798584,
      "activations/layer22_attention_weight_min": -24.47450065612793,
      "activations/layer23_attention_weight_max": 38.8963623046875,
      "activations/layer23_attention_weight_min": -25.39419174194336,
      "activations/layer2_attention_weight_max": 29.238773345947266,
      "activations/layer2_attention_weight_min": -27.706951141357422,
      "activations/layer3_attention_weight_max": 85.42603302001953,
      "activations/layer3_attention_weight_min": -88.54304504394531,
      "activations/layer4_attention_weight_max": 95.290771484375,
      "activations/layer4_attention_weight_min": -91.09410858154297,
      "activations/layer5_attention_weight_max": 72.05191040039062,
      "activations/layer5_attention_weight_min": -78.34684753417969,
      "activations/layer6_attention_weight_max": 52.35802459716797,
      "activations/layer6_attention_weight_min": -53.56186294555664,
      "activations/layer7_attention_weight_max": 75.18620300292969,
      "activations/layer7_attention_weight_min": -72.12937927246094,
      "activations/layer8_attention_weight_max": 44.92473220825195,
      "activations/layer8_attention_weight_min": -46.908836364746094,
      "activations/layer9_attention_weight_max": 50.347198486328125,
      "activations/layer9_attention_weight_min": -47.439632415771484,
      "epoch": 10.55,
      "learning_rate": 8.28060606060606e-05,
      "loss": 2.779,
      "step": 181550
    },
    {
      "activations/layer0_attention_weight_max": 15.212267875671387,
      "activations/layer0_attention_weight_min": -14.224702835083008,
      "activations/layer10_attention_weight_max": 34.93634796142578,
      "activations/layer10_attention_weight_min": -34.92075729370117,
      "activations/layer11_attention_weight_max": 30.17900848388672,
      "activations/layer11_attention_weight_min": -31.175268173217773,
      "activations/layer12_attention_weight_max": 21.73896598815918,
      "activations/layer12_attention_weight_min": -23.879301071166992,
      "activations/layer13_attention_weight_max": 34.64373779296875,
      "activations/layer13_attention_weight_min": -29.018951416015625,
      "activations/layer14_attention_weight_max": 35.80222702026367,
      "activations/layer14_attention_weight_min": -28.478145599365234,
      "activations/layer15_attention_weight_max": 31.056814193725586,
      "activations/layer15_attention_weight_min": -27.808963775634766,
      "activations/layer16_attention_weight_max": 29.001087188720703,
      "activations/layer16_attention_weight_min": -27.531641006469727,
      "activations/layer17_attention_weight_max": 48.6409912109375,
      "activations/layer17_attention_weight_min": -39.808319091796875,
      "activations/layer18_attention_weight_max": 44.72197723388672,
      "activations/layer18_attention_weight_min": -35.167667388916016,
      "activations/layer19_attention_weight_max": 21.485830307006836,
      "activations/layer19_attention_weight_min": -23.15578269958496,
      "activations/layer1_attention_weight_max": 16.82745361328125,
      "activations/layer1_attention_weight_min": -13.984418869018555,
      "activations/layer20_attention_weight_max": 21.539857864379883,
      "activations/layer20_attention_weight_min": -22.51979637145996,
      "activations/layer21_attention_weight_max": 35.93650436401367,
      "activations/layer21_attention_weight_min": -26.804508209228516,
      "activations/layer22_attention_weight_max": 27.88600730895996,
      "activations/layer22_attention_weight_min": -27.262481689453125,
      "activations/layer23_attention_weight_max": 34.262638092041016,
      "activations/layer23_attention_weight_min": -24.21990203857422,
      "activations/layer2_attention_weight_max": 32.31594467163086,
      "activations/layer2_attention_weight_min": -29.724781036376953,
      "activations/layer3_attention_weight_max": 83.92463684082031,
      "activations/layer3_attention_weight_min": -86.48216247558594,
      "activations/layer4_attention_weight_max": 95.08255767822266,
      "activations/layer4_attention_weight_min": -92.66487884521484,
      "activations/layer5_attention_weight_max": 74.26405334472656,
      "activations/layer5_attention_weight_min": -78.4791488647461,
      "activations/layer6_attention_weight_max": 52.6182746887207,
      "activations/layer6_attention_weight_min": -51.53959274291992,
      "activations/layer7_attention_weight_max": 75.53573608398438,
      "activations/layer7_attention_weight_min": -70.03742980957031,
      "activations/layer8_attention_weight_max": 43.63930892944336,
      "activations/layer8_attention_weight_min": -45.56315612792969,
      "activations/layer9_attention_weight_max": 45.428993225097656,
      "activations/layer9_attention_weight_min": -48.12506866455078,
      "epoch": 10.55,
      "learning_rate": 8.278712121212121e-05,
      "loss": 2.7725,
      "step": 181600
    },
    {
      "activations/layer0_attention_weight_max": 15.862378120422363,
      "activations/layer0_attention_weight_min": -13.554980278015137,
      "activations/layer10_attention_weight_max": 36.822242736816406,
      "activations/layer10_attention_weight_min": -39.675106048583984,
      "activations/layer11_attention_weight_max": 32.89252471923828,
      "activations/layer11_attention_weight_min": -34.810691833496094,
      "activations/layer12_attention_weight_max": 20.813993453979492,
      "activations/layer12_attention_weight_min": -25.160690307617188,
      "activations/layer13_attention_weight_max": 40.33634567260742,
      "activations/layer13_attention_weight_min": -28.1839542388916,
      "activations/layer14_attention_weight_max": 42.451236724853516,
      "activations/layer14_attention_weight_min": -30.531129837036133,
      "activations/layer15_attention_weight_max": 32.58327865600586,
      "activations/layer15_attention_weight_min": -28.678367614746094,
      "activations/layer16_attention_weight_max": 29.960086822509766,
      "activations/layer16_attention_weight_min": -28.040315628051758,
      "activations/layer17_attention_weight_max": 54.492591857910156,
      "activations/layer17_attention_weight_min": -44.348751068115234,
      "activations/layer18_attention_weight_max": 45.28889846801758,
      "activations/layer18_attention_weight_min": -36.731990814208984,
      "activations/layer19_attention_weight_max": 20.233858108520508,
      "activations/layer19_attention_weight_min": -20.874738693237305,
      "activations/layer1_attention_weight_max": 16.798627853393555,
      "activations/layer1_attention_weight_min": -14.314079284667969,
      "activations/layer20_attention_weight_max": 19.683467864990234,
      "activations/layer20_attention_weight_min": -25.40279197692871,
      "activations/layer21_attention_weight_max": 34.446292877197266,
      "activations/layer21_attention_weight_min": -20.515663146972656,
      "activations/layer22_attention_weight_max": 32.35805892944336,
      "activations/layer22_attention_weight_min": -25.24529457092285,
      "activations/layer23_attention_weight_max": 33.007232666015625,
      "activations/layer23_attention_weight_min": -21.703380584716797,
      "activations/layer2_attention_weight_max": 35.26203536987305,
      "activations/layer2_attention_weight_min": -31.26690673828125,
      "activations/layer3_attention_weight_max": 92.47689819335938,
      "activations/layer3_attention_weight_min": -90.10237121582031,
      "activations/layer4_attention_weight_max": 97.29631805419922,
      "activations/layer4_attention_weight_min": -93.3934555053711,
      "activations/layer5_attention_weight_max": 73.65621948242188,
      "activations/layer5_attention_weight_min": -80.23818969726562,
      "activations/layer6_attention_weight_max": 53.64427185058594,
      "activations/layer6_attention_weight_min": -53.788997650146484,
      "activations/layer7_attention_weight_max": 74.97430419921875,
      "activations/layer7_attention_weight_min": -76.5009536743164,
      "activations/layer8_attention_weight_max": 46.90408706665039,
      "activations/layer8_attention_weight_min": -49.887840270996094,
      "activations/layer9_attention_weight_max": 48.81528091430664,
      "activations/layer9_attention_weight_min": -52.6463623046875,
      "epoch": 10.55,
      "learning_rate": 8.276818181818181e-05,
      "loss": 2.7882,
      "step": 181650
    },
    {
      "activations/layer0_attention_weight_max": 15.396186828613281,
      "activations/layer0_attention_weight_min": -13.558920860290527,
      "activations/layer10_attention_weight_max": 37.97591781616211,
      "activations/layer10_attention_weight_min": -36.209999084472656,
      "activations/layer11_attention_weight_max": 32.00850296020508,
      "activations/layer11_attention_weight_min": -34.02009582519531,
      "activations/layer12_attention_weight_max": 17.579227447509766,
      "activations/layer12_attention_weight_min": -24.394580841064453,
      "activations/layer13_attention_weight_max": 37.03465270996094,
      "activations/layer13_attention_weight_min": -28.681827545166016,
      "activations/layer14_attention_weight_max": 35.30734634399414,
      "activations/layer14_attention_weight_min": -31.39116859436035,
      "activations/layer15_attention_weight_max": 33.852237701416016,
      "activations/layer15_attention_weight_min": -29.480361938476562,
      "activations/layer16_attention_weight_max": 31.77850341796875,
      "activations/layer16_attention_weight_min": -31.685197830200195,
      "activations/layer17_attention_weight_max": 51.96814727783203,
      "activations/layer17_attention_weight_min": -44.365814208984375,
      "activations/layer18_attention_weight_max": 47.39889907836914,
      "activations/layer18_attention_weight_min": -41.385868072509766,
      "activations/layer19_attention_weight_max": 21.390819549560547,
      "activations/layer19_attention_weight_min": -22.323354721069336,
      "activations/layer1_attention_weight_max": 16.920076370239258,
      "activations/layer1_attention_weight_min": -14.978829383850098,
      "activations/layer20_attention_weight_max": 22.115190505981445,
      "activations/layer20_attention_weight_min": -22.30946159362793,
      "activations/layer21_attention_weight_max": 39.26272964477539,
      "activations/layer21_attention_weight_min": -23.015655517578125,
      "activations/layer22_attention_weight_max": 30.99901008605957,
      "activations/layer22_attention_weight_min": -26.344270706176758,
      "activations/layer23_attention_weight_max": 35.76978302001953,
      "activations/layer23_attention_weight_min": -23.330984115600586,
      "activations/layer2_attention_weight_max": 32.500877380371094,
      "activations/layer2_attention_weight_min": -29.761260986328125,
      "activations/layer3_attention_weight_max": 86.8545150756836,
      "activations/layer3_attention_weight_min": -87.41524505615234,
      "activations/layer4_attention_weight_max": 94.3520736694336,
      "activations/layer4_attention_weight_min": -91.0884780883789,
      "activations/layer5_attention_weight_max": 75.39295959472656,
      "activations/layer5_attention_weight_min": -77.39810180664062,
      "activations/layer6_attention_weight_max": 52.921287536621094,
      "activations/layer6_attention_weight_min": -50.966007232666016,
      "activations/layer7_attention_weight_max": 70.53050994873047,
      "activations/layer7_attention_weight_min": -73.33441162109375,
      "activations/layer8_attention_weight_max": 45.69389724731445,
      "activations/layer8_attention_weight_min": -48.36941146850586,
      "activations/layer9_attention_weight_max": 51.42644119262695,
      "activations/layer9_attention_weight_min": -48.092491149902344,
      "epoch": 10.56,
      "learning_rate": 8.274924242424241e-05,
      "loss": 2.789,
      "step": 181700
    },
    {
      "activations/layer0_attention_weight_max": 15.208720207214355,
      "activations/layer0_attention_weight_min": -13.45945930480957,
      "activations/layer10_attention_weight_max": 40.227413177490234,
      "activations/layer10_attention_weight_min": -38.84701919555664,
      "activations/layer11_attention_weight_max": 33.750213623046875,
      "activations/layer11_attention_weight_min": -33.12818908691406,
      "activations/layer12_attention_weight_max": 18.45143699645996,
      "activations/layer12_attention_weight_min": -25.5727596282959,
      "activations/layer13_attention_weight_max": 38.10062789916992,
      "activations/layer13_attention_weight_min": -30.657936096191406,
      "activations/layer14_attention_weight_max": 37.59202575683594,
      "activations/layer14_attention_weight_min": -30.82653045654297,
      "activations/layer15_attention_weight_max": 34.812461853027344,
      "activations/layer15_attention_weight_min": -29.429040908813477,
      "activations/layer16_attention_weight_max": 31.408077239990234,
      "activations/layer16_attention_weight_min": -28.92627716064453,
      "activations/layer17_attention_weight_max": 56.39572525024414,
      "activations/layer17_attention_weight_min": -44.94890213012695,
      "activations/layer18_attention_weight_max": 52.18917465209961,
      "activations/layer18_attention_weight_min": -39.19064712524414,
      "activations/layer19_attention_weight_max": 23.715396881103516,
      "activations/layer19_attention_weight_min": -22.74222755432129,
      "activations/layer1_attention_weight_max": 16.24809455871582,
      "activations/layer1_attention_weight_min": -13.74991226196289,
      "activations/layer20_attention_weight_max": 23.4502010345459,
      "activations/layer20_attention_weight_min": -21.87549591064453,
      "activations/layer21_attention_weight_max": 39.928016662597656,
      "activations/layer21_attention_weight_min": -24.611528396606445,
      "activations/layer22_attention_weight_max": 36.112083435058594,
      "activations/layer22_attention_weight_min": -25.481613159179688,
      "activations/layer23_attention_weight_max": 40.594627380371094,
      "activations/layer23_attention_weight_min": -24.80718994140625,
      "activations/layer2_attention_weight_max": 31.562576293945312,
      "activations/layer2_attention_weight_min": -30.529645919799805,
      "activations/layer3_attention_weight_max": 99.28270721435547,
      "activations/layer3_attention_weight_min": -93.96232604980469,
      "activations/layer4_attention_weight_max": 103.98016357421875,
      "activations/layer4_attention_weight_min": -100.31945037841797,
      "activations/layer5_attention_weight_max": 82.8852310180664,
      "activations/layer5_attention_weight_min": -80.85585021972656,
      "activations/layer6_attention_weight_max": 57.29800796508789,
      "activations/layer6_attention_weight_min": -55.69988250732422,
      "activations/layer7_attention_weight_max": 82.10374450683594,
      "activations/layer7_attention_weight_min": -77.39057159423828,
      "activations/layer8_attention_weight_max": 49.602821350097656,
      "activations/layer8_attention_weight_min": -49.15916061401367,
      "activations/layer9_attention_weight_max": 50.93115234375,
      "activations/layer9_attention_weight_min": -50.58866882324219,
      "epoch": 10.56,
      "learning_rate": 8.273030303030303e-05,
      "loss": 2.7915,
      "step": 181750
    },
    {
      "activations/layer0_attention_weight_max": 15.266529083251953,
      "activations/layer0_attention_weight_min": -13.941871643066406,
      "activations/layer10_attention_weight_max": 35.1591796875,
      "activations/layer10_attention_weight_min": -34.5234375,
      "activations/layer11_attention_weight_max": 31.72937774658203,
      "activations/layer11_attention_weight_min": -32.89453887939453,
      "activations/layer12_attention_weight_max": 20.61243438720703,
      "activations/layer12_attention_weight_min": -24.546173095703125,
      "activations/layer13_attention_weight_max": 37.19902420043945,
      "activations/layer13_attention_weight_min": -33.92634201049805,
      "activations/layer14_attention_weight_max": 37.123905181884766,
      "activations/layer14_attention_weight_min": -33.7735595703125,
      "activations/layer15_attention_weight_max": 33.443115234375,
      "activations/layer15_attention_weight_min": -29.066143035888672,
      "activations/layer16_attention_weight_max": 30.185163497924805,
      "activations/layer16_attention_weight_min": -29.313247680664062,
      "activations/layer17_attention_weight_max": 50.40989685058594,
      "activations/layer17_attention_weight_min": -43.00773620605469,
      "activations/layer18_attention_weight_max": 44.45269012451172,
      "activations/layer18_attention_weight_min": -36.33229064941406,
      "activations/layer19_attention_weight_max": 21.90625,
      "activations/layer19_attention_weight_min": -22.111845016479492,
      "activations/layer1_attention_weight_max": 17.217741012573242,
      "activations/layer1_attention_weight_min": -14.425158500671387,
      "activations/layer20_attention_weight_max": 22.91429901123047,
      "activations/layer20_attention_weight_min": -21.89189338684082,
      "activations/layer21_attention_weight_max": 40.1589241027832,
      "activations/layer21_attention_weight_min": -24.491750717163086,
      "activations/layer22_attention_weight_max": 29.731443405151367,
      "activations/layer22_attention_weight_min": -25.67389488220215,
      "activations/layer23_attention_weight_max": 39.87660598754883,
      "activations/layer23_attention_weight_min": -26.102828979492188,
      "activations/layer2_attention_weight_max": 32.168548583984375,
      "activations/layer2_attention_weight_min": -32.03645706176758,
      "activations/layer3_attention_weight_max": 89.4786605834961,
      "activations/layer3_attention_weight_min": -88.9927978515625,
      "activations/layer4_attention_weight_max": 100.1667251586914,
      "activations/layer4_attention_weight_min": -94.46102142333984,
      "activations/layer5_attention_weight_max": 78.5237808227539,
      "activations/layer5_attention_weight_min": -82.3586196899414,
      "activations/layer6_attention_weight_max": 54.888309478759766,
      "activations/layer6_attention_weight_min": -54.60999298095703,
      "activations/layer7_attention_weight_max": 75.79248809814453,
      "activations/layer7_attention_weight_min": -76.63652038574219,
      "activations/layer8_attention_weight_max": 45.6214485168457,
      "activations/layer8_attention_weight_min": -46.15034484863281,
      "activations/layer9_attention_weight_max": 49.71650314331055,
      "activations/layer9_attention_weight_min": -49.943355560302734,
      "epoch": 10.56,
      "learning_rate": 8.271136363636363e-05,
      "loss": 2.7902,
      "step": 181800
    },
    {
      "activations/layer0_attention_weight_max": 15.389144897460938,
      "activations/layer0_attention_weight_min": -14.200185775756836,
      "activations/layer10_attention_weight_max": 37.22022247314453,
      "activations/layer10_attention_weight_min": -35.90032958984375,
      "activations/layer11_attention_weight_max": 32.49623107910156,
      "activations/layer11_attention_weight_min": -32.68840026855469,
      "activations/layer12_attention_weight_max": 25.86285972595215,
      "activations/layer12_attention_weight_min": -26.530139923095703,
      "activations/layer13_attention_weight_max": 37.4904670715332,
      "activations/layer13_attention_weight_min": -29.13373374938965,
      "activations/layer14_attention_weight_max": 47.04867935180664,
      "activations/layer14_attention_weight_min": -30.469600677490234,
      "activations/layer15_attention_weight_max": 37.19187927246094,
      "activations/layer15_attention_weight_min": -28.86530876159668,
      "activations/layer16_attention_weight_max": 35.706825256347656,
      "activations/layer16_attention_weight_min": -28.579998016357422,
      "activations/layer17_attention_weight_max": 51.31291580200195,
      "activations/layer17_attention_weight_min": -44.20420455932617,
      "activations/layer18_attention_weight_max": 46.536048889160156,
      "activations/layer18_attention_weight_min": -41.28352737426758,
      "activations/layer19_attention_weight_max": 21.595552444458008,
      "activations/layer19_attention_weight_min": -24.637971878051758,
      "activations/layer1_attention_weight_max": 17.370210647583008,
      "activations/layer1_attention_weight_min": -15.665176391601562,
      "activations/layer20_attention_weight_max": 23.74394989013672,
      "activations/layer20_attention_weight_min": -22.366573333740234,
      "activations/layer21_attention_weight_max": 39.16250228881836,
      "activations/layer21_attention_weight_min": -27.29606819152832,
      "activations/layer22_attention_weight_max": 32.601654052734375,
      "activations/layer22_attention_weight_min": -25.917932510375977,
      "activations/layer23_attention_weight_max": 38.51734924316406,
      "activations/layer23_attention_weight_min": -25.785078048706055,
      "activations/layer2_attention_weight_max": 31.718849182128906,
      "activations/layer2_attention_weight_min": -30.656169891357422,
      "activations/layer3_attention_weight_max": 95.16954803466797,
      "activations/layer3_attention_weight_min": -94.75273895263672,
      "activations/layer4_attention_weight_max": 99.07275390625,
      "activations/layer4_attention_weight_min": -95.31566619873047,
      "activations/layer5_attention_weight_max": 76.03205871582031,
      "activations/layer5_attention_weight_min": -78.47531127929688,
      "activations/layer6_attention_weight_max": 54.92514419555664,
      "activations/layer6_attention_weight_min": -53.82477951049805,
      "activations/layer7_attention_weight_max": 78.30563354492188,
      "activations/layer7_attention_weight_min": -71.91764068603516,
      "activations/layer8_attention_weight_max": 46.00636672973633,
      "activations/layer8_attention_weight_min": -47.32896423339844,
      "activations/layer9_attention_weight_max": 57.225257873535156,
      "activations/layer9_attention_weight_min": -51.269683837890625,
      "epoch": 10.57,
      "learning_rate": 8.269242424242423e-05,
      "loss": 2.8002,
      "step": 181850
    },
    {
      "activations/layer0_attention_weight_max": 15.859565734863281,
      "activations/layer0_attention_weight_min": -13.812385559082031,
      "activations/layer10_attention_weight_max": 36.17937469482422,
      "activations/layer10_attention_weight_min": -37.138427734375,
      "activations/layer11_attention_weight_max": 30.44763946533203,
      "activations/layer11_attention_weight_min": -31.608230590820312,
      "activations/layer12_attention_weight_max": 20.420429229736328,
      "activations/layer12_attention_weight_min": -23.26152801513672,
      "activations/layer13_attention_weight_max": 34.71736145019531,
      "activations/layer13_attention_weight_min": -28.174779891967773,
      "activations/layer14_attention_weight_max": 36.91068649291992,
      "activations/layer14_attention_weight_min": -29.28299903869629,
      "activations/layer15_attention_weight_max": 34.915321350097656,
      "activations/layer15_attention_weight_min": -30.24335289001465,
      "activations/layer16_attention_weight_max": 31.933385848999023,
      "activations/layer16_attention_weight_min": -30.596162796020508,
      "activations/layer17_attention_weight_max": 52.5853157043457,
      "activations/layer17_attention_weight_min": -46.25495529174805,
      "activations/layer18_attention_weight_max": 48.40260314941406,
      "activations/layer18_attention_weight_min": -39.30259704589844,
      "activations/layer19_attention_weight_max": 24.16086769104004,
      "activations/layer19_attention_weight_min": -22.374433517456055,
      "activations/layer1_attention_weight_max": 16.72085952758789,
      "activations/layer1_attention_weight_min": -14.090802192687988,
      "activations/layer20_attention_weight_max": 22.43703842163086,
      "activations/layer20_attention_weight_min": -22.54510498046875,
      "activations/layer21_attention_weight_max": 38.82954406738281,
      "activations/layer21_attention_weight_min": -24.430862426757812,
      "activations/layer22_attention_weight_max": 34.6146240234375,
      "activations/layer22_attention_weight_min": -26.302982330322266,
      "activations/layer23_attention_weight_max": 41.59779357910156,
      "activations/layer23_attention_weight_min": -25.594709396362305,
      "activations/layer2_attention_weight_max": 32.279727935791016,
      "activations/layer2_attention_weight_min": -29.379560470581055,
      "activations/layer3_attention_weight_max": 89.36148834228516,
      "activations/layer3_attention_weight_min": -90.98382568359375,
      "activations/layer4_attention_weight_max": 101.55387878417969,
      "activations/layer4_attention_weight_min": -95.66087341308594,
      "activations/layer5_attention_weight_max": 77.61550903320312,
      "activations/layer5_attention_weight_min": -79.03794860839844,
      "activations/layer6_attention_weight_max": 53.25590896606445,
      "activations/layer6_attention_weight_min": -54.78276824951172,
      "activations/layer7_attention_weight_max": 79.0130615234375,
      "activations/layer7_attention_weight_min": -72.54782104492188,
      "activations/layer8_attention_weight_max": 44.679595947265625,
      "activations/layer8_attention_weight_min": -48.23847961425781,
      "activations/layer9_attention_weight_max": 48.80462646484375,
      "activations/layer9_attention_weight_min": -48.44953918457031,
      "epoch": 10.57,
      "learning_rate": 8.267348484848485e-05,
      "loss": 2.7934,
      "step": 181900
    },
    {
      "activations/layer0_attention_weight_max": 15.625521659851074,
      "activations/layer0_attention_weight_min": -13.656902313232422,
      "activations/layer10_attention_weight_max": 33.5506706237793,
      "activations/layer10_attention_weight_min": -34.67546463012695,
      "activations/layer11_attention_weight_max": 31.01887321472168,
      "activations/layer11_attention_weight_min": -33.31117630004883,
      "activations/layer12_attention_weight_max": 18.287485122680664,
      "activations/layer12_attention_weight_min": -23.229089736938477,
      "activations/layer13_attention_weight_max": 31.31659698486328,
      "activations/layer13_attention_weight_min": -32.83935546875,
      "activations/layer14_attention_weight_max": 32.403228759765625,
      "activations/layer14_attention_weight_min": -30.20779800415039,
      "activations/layer15_attention_weight_max": 28.589374542236328,
      "activations/layer15_attention_weight_min": -28.634658813476562,
      "activations/layer16_attention_weight_max": 29.007158279418945,
      "activations/layer16_attention_weight_min": -29.628068923950195,
      "activations/layer17_attention_weight_max": 47.86300277709961,
      "activations/layer17_attention_weight_min": -43.59947204589844,
      "activations/layer18_attention_weight_max": 39.36503601074219,
      "activations/layer18_attention_weight_min": -35.10344696044922,
      "activations/layer19_attention_weight_max": 20.153642654418945,
      "activations/layer19_attention_weight_min": -20.51890754699707,
      "activations/layer1_attention_weight_max": 16.194385528564453,
      "activations/layer1_attention_weight_min": -15.262144088745117,
      "activations/layer20_attention_weight_max": 18.51280403137207,
      "activations/layer20_attention_weight_min": -25.070993423461914,
      "activations/layer21_attention_weight_max": 30.690555572509766,
      "activations/layer21_attention_weight_min": -22.184585571289062,
      "activations/layer22_attention_weight_max": 29.35630226135254,
      "activations/layer22_attention_weight_min": -24.76836395263672,
      "activations/layer23_attention_weight_max": 33.469879150390625,
      "activations/layer23_attention_weight_min": -25.8327693939209,
      "activations/layer2_attention_weight_max": 34.355804443359375,
      "activations/layer2_attention_weight_min": -32.333213806152344,
      "activations/layer3_attention_weight_max": 95.25730895996094,
      "activations/layer3_attention_weight_min": -95.96267700195312,
      "activations/layer4_attention_weight_max": 98.2013168334961,
      "activations/layer4_attention_weight_min": -93.77249145507812,
      "activations/layer5_attention_weight_max": 77.27337646484375,
      "activations/layer5_attention_weight_min": -80.03065490722656,
      "activations/layer6_attention_weight_max": 53.072025299072266,
      "activations/layer6_attention_weight_min": -55.090728759765625,
      "activations/layer7_attention_weight_max": 72.82525634765625,
      "activations/layer7_attention_weight_min": -70.69251251220703,
      "activations/layer8_attention_weight_max": 43.157779693603516,
      "activations/layer8_attention_weight_min": -45.91704177856445,
      "activations/layer9_attention_weight_max": 48.87519836425781,
      "activations/layer9_attention_weight_min": -45.68648910522461,
      "epoch": 10.57,
      "learning_rate": 8.265454545454543e-05,
      "loss": 2.8112,
      "step": 181950
    },
    {
      "activations/layer0_attention_weight_max": 14.508216857910156,
      "activations/layer0_attention_weight_min": -14.547898292541504,
      "activations/layer10_attention_weight_max": 36.765419006347656,
      "activations/layer10_attention_weight_min": -36.752830505371094,
      "activations/layer11_attention_weight_max": 30.61946678161621,
      "activations/layer11_attention_weight_min": -33.452110290527344,
      "activations/layer12_attention_weight_max": 18.68787384033203,
      "activations/layer12_attention_weight_min": -25.76907730102539,
      "activations/layer13_attention_weight_max": 44.562599182128906,
      "activations/layer13_attention_weight_min": -32.542999267578125,
      "activations/layer14_attention_weight_max": 39.776065826416016,
      "activations/layer14_attention_weight_min": -29.164812088012695,
      "activations/layer15_attention_weight_max": 35.859981536865234,
      "activations/layer15_attention_weight_min": -28.92395782470703,
      "activations/layer16_attention_weight_max": 33.20637512207031,
      "activations/layer16_attention_weight_min": -30.806917190551758,
      "activations/layer17_attention_weight_max": 55.858924865722656,
      "activations/layer17_attention_weight_min": -49.72626876831055,
      "activations/layer18_attention_weight_max": 50.86124038696289,
      "activations/layer18_attention_weight_min": -40.492103576660156,
      "activations/layer19_attention_weight_max": 25.79695701599121,
      "activations/layer19_attention_weight_min": -21.98769760131836,
      "activations/layer1_attention_weight_max": 16.251789093017578,
      "activations/layer1_attention_weight_min": -14.744240760803223,
      "activations/layer20_attention_weight_max": 27.363079071044922,
      "activations/layer20_attention_weight_min": -24.74697494506836,
      "activations/layer21_attention_weight_max": 43.50201416015625,
      "activations/layer21_attention_weight_min": -26.341686248779297,
      "activations/layer22_attention_weight_max": 34.254093170166016,
      "activations/layer22_attention_weight_min": -25.856342315673828,
      "activations/layer23_attention_weight_max": 41.5579833984375,
      "activations/layer23_attention_weight_min": -23.875341415405273,
      "activations/layer2_attention_weight_max": 32.91521453857422,
      "activations/layer2_attention_weight_min": -31.189985275268555,
      "activations/layer3_attention_weight_max": 95.224853515625,
      "activations/layer3_attention_weight_min": -97.82549285888672,
      "activations/layer4_attention_weight_max": 97.20873260498047,
      "activations/layer4_attention_weight_min": -100.71884155273438,
      "activations/layer5_attention_weight_max": 73.44189453125,
      "activations/layer5_attention_weight_min": -76.87734985351562,
      "activations/layer6_attention_weight_max": 54.872318267822266,
      "activations/layer6_attention_weight_min": -54.459659576416016,
      "activations/layer7_attention_weight_max": 75.16472625732422,
      "activations/layer7_attention_weight_min": -75.45161437988281,
      "activations/layer8_attention_weight_max": 46.21369552612305,
      "activations/layer8_attention_weight_min": -47.610511779785156,
      "activations/layer9_attention_weight_max": 49.63710021972656,
      "activations/layer9_attention_weight_min": -48.459041595458984,
      "epoch": 10.58,
      "learning_rate": 8.263560606060605e-05,
      "loss": 2.7835,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_loss": 2.740234375,
      "eval_runtime": 8.5144,
      "eval_samples_per_second": 504.322,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_openwebtext_loss": 2.740234375,
      "eval_openwebtext_ppl": 15.490615283867873,
      "eval_openwebtext_runtime": 8.5144,
      "eval_openwebtext_samples_per_second": 504.322,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.987,
      "eval_wikitext_samples_per_second": 229.493,
      "step": 182000
    },
    {
      "epoch": 10.58,
      "eval_lambada_loss": 2.626953125,
      "eval_lambada_ppl": 13.831562592310346,
      "eval_lambada_runtime": 9.5435,
      "eval_lambada_samples_per_second": 510.189,
      "step": 182000
    },
    {
      "activations/layer0_attention_weight_max": 15.758028030395508,
      "activations/layer0_attention_weight_min": -14.170318603515625,
      "activations/layer10_attention_weight_max": 38.186187744140625,
      "activations/layer10_attention_weight_min": -35.71162414550781,
      "activations/layer11_attention_weight_max": 33.167884826660156,
      "activations/layer11_attention_weight_min": -34.20934295654297,
      "activations/layer12_attention_weight_max": 18.057809829711914,
      "activations/layer12_attention_weight_min": -24.45458984375,
      "activations/layer13_attention_weight_max": 40.74550247192383,
      "activations/layer13_attention_weight_min": -28.346233367919922,
      "activations/layer14_attention_weight_max": 36.589298248291016,
      "activations/layer14_attention_weight_min": -31.246923446655273,
      "activations/layer15_attention_weight_max": 34.64415740966797,
      "activations/layer15_attention_weight_min": -28.355566024780273,
      "activations/layer16_attention_weight_max": 30.5439453125,
      "activations/layer16_attention_weight_min": -28.518943786621094,
      "activations/layer17_attention_weight_max": 52.8397102355957,
      "activations/layer17_attention_weight_min": -44.10175704956055,
      "activations/layer18_attention_weight_max": 48.237857818603516,
      "activations/layer18_attention_weight_min": -38.01610565185547,
      "activations/layer19_attention_weight_max": 21.852725982666016,
      "activations/layer19_attention_weight_min": -21.45639419555664,
      "activations/layer1_attention_weight_max": 16.468467712402344,
      "activations/layer1_attention_weight_min": -14.391966819763184,
      "activations/layer20_attention_weight_max": 23.318620681762695,
      "activations/layer20_attention_weight_min": -20.627769470214844,
      "activations/layer21_attention_weight_max": 42.16444778442383,
      "activations/layer21_attention_weight_min": -24.444475173950195,
      "activations/layer22_attention_weight_max": 32.212921142578125,
      "activations/layer22_attention_weight_min": -24.161197662353516,
      "activations/layer23_attention_weight_max": 39.728912353515625,
      "activations/layer23_attention_weight_min": -22.311262130737305,
      "activations/layer2_attention_weight_max": 33.20313262939453,
      "activations/layer2_attention_weight_min": -30.650846481323242,
      "activations/layer3_attention_weight_max": 90.04952239990234,
      "activations/layer3_attention_weight_min": -90.99746704101562,
      "activations/layer4_attention_weight_max": 98.0201416015625,
      "activations/layer4_attention_weight_min": -90.37661743164062,
      "activations/layer5_attention_weight_max": 74.74340057373047,
      "activations/layer5_attention_weight_min": -75.27140808105469,
      "activations/layer6_attention_weight_max": 55.503055572509766,
      "activations/layer6_attention_weight_min": -54.49562072753906,
      "activations/layer7_attention_weight_max": 75.33338165283203,
      "activations/layer7_attention_weight_min": -73.71562194824219,
      "activations/layer8_attention_weight_max": 43.7889289855957,
      "activations/layer8_attention_weight_min": -44.20535659790039,
      "activations/layer9_attention_weight_max": 49.6423454284668,
      "activations/layer9_attention_weight_min": -47.61500549316406,
      "epoch": 10.58,
      "learning_rate": 8.261666666666666e-05,
      "loss": 2.7821,
      "step": 182050
    },
    {
      "activations/layer0_attention_weight_max": 15.05075740814209,
      "activations/layer0_attention_weight_min": -14.179692268371582,
      "activations/layer10_attention_weight_max": 35.79518508911133,
      "activations/layer10_attention_weight_min": -34.73493194580078,
      "activations/layer11_attention_weight_max": 30.37885284423828,
      "activations/layer11_attention_weight_min": -31.835372924804688,
      "activations/layer12_attention_weight_max": 18.800344467163086,
      "activations/layer12_attention_weight_min": -25.03531837463379,
      "activations/layer13_attention_weight_max": 38.05724334716797,
      "activations/layer13_attention_weight_min": -31.537065505981445,
      "activations/layer14_attention_weight_max": 36.92190933227539,
      "activations/layer14_attention_weight_min": -29.690732955932617,
      "activations/layer15_attention_weight_max": 33.243289947509766,
      "activations/layer15_attention_weight_min": -28.799318313598633,
      "activations/layer16_attention_weight_max": 30.119977951049805,
      "activations/layer16_attention_weight_min": -31.115360260009766,
      "activations/layer17_attention_weight_max": 55.21186065673828,
      "activations/layer17_attention_weight_min": -46.018863677978516,
      "activations/layer18_attention_weight_max": 46.07685852050781,
      "activations/layer18_attention_weight_min": -38.350257873535156,
      "activations/layer19_attention_weight_max": 20.243154525756836,
      "activations/layer19_attention_weight_min": -21.458681106567383,
      "activations/layer1_attention_weight_max": 16.24213981628418,
      "activations/layer1_attention_weight_min": -16.566530227661133,
      "activations/layer20_attention_weight_max": 20.999164581298828,
      "activations/layer20_attention_weight_min": -23.50897216796875,
      "activations/layer21_attention_weight_max": 36.58233642578125,
      "activations/layer21_attention_weight_min": -22.895156860351562,
      "activations/layer22_attention_weight_max": 30.73029327392578,
      "activations/layer22_attention_weight_min": -25.648576736450195,
      "activations/layer23_attention_weight_max": 37.00468063354492,
      "activations/layer23_attention_weight_min": -24.271886825561523,
      "activations/layer2_attention_weight_max": 31.33808708190918,
      "activations/layer2_attention_weight_min": -30.115436553955078,
      "activations/layer3_attention_weight_max": 90.70314025878906,
      "activations/layer3_attention_weight_min": -91.48534393310547,
      "activations/layer4_attention_weight_max": 99.03585815429688,
      "activations/layer4_attention_weight_min": -93.8498306274414,
      "activations/layer5_attention_weight_max": 76.7327880859375,
      "activations/layer5_attention_weight_min": -82.00238037109375,
      "activations/layer6_attention_weight_max": 52.85588073730469,
      "activations/layer6_attention_weight_min": -52.771663665771484,
      "activations/layer7_attention_weight_max": 76.01619720458984,
      "activations/layer7_attention_weight_min": -71.36762237548828,
      "activations/layer8_attention_weight_max": 43.844818115234375,
      "activations/layer8_attention_weight_min": -44.588871002197266,
      "activations/layer9_attention_weight_max": 47.659847259521484,
      "activations/layer9_attention_weight_min": -47.0800666809082,
      "epoch": 10.58,
      "learning_rate": 8.259772727272727e-05,
      "loss": 2.7651,
      "step": 182100
    },
    {
      "activations/layer0_attention_weight_max": 15.482348442077637,
      "activations/layer0_attention_weight_min": -13.960433959960938,
      "activations/layer10_attention_weight_max": 39.42689514160156,
      "activations/layer10_attention_weight_min": -35.14470672607422,
      "activations/layer11_attention_weight_max": 34.03974914550781,
      "activations/layer11_attention_weight_min": -32.66168212890625,
      "activations/layer12_attention_weight_max": 19.051589965820312,
      "activations/layer12_attention_weight_min": -23.251583099365234,
      "activations/layer13_attention_weight_max": 42.392333984375,
      "activations/layer13_attention_weight_min": -31.279531478881836,
      "activations/layer14_attention_weight_max": 38.574249267578125,
      "activations/layer14_attention_weight_min": -29.936012268066406,
      "activations/layer15_attention_weight_max": 32.8614616394043,
      "activations/layer15_attention_weight_min": -30.214317321777344,
      "activations/layer16_attention_weight_max": 31.391780853271484,
      "activations/layer16_attention_weight_min": -29.837615966796875,
      "activations/layer17_attention_weight_max": 52.02512741088867,
      "activations/layer17_attention_weight_min": -44.33693313598633,
      "activations/layer18_attention_weight_max": 48.70355224609375,
      "activations/layer18_attention_weight_min": -41.95376205444336,
      "activations/layer19_attention_weight_max": 23.492929458618164,
      "activations/layer19_attention_weight_min": -23.107980728149414,
      "activations/layer1_attention_weight_max": 17.207284927368164,
      "activations/layer1_attention_weight_min": -15.499734878540039,
      "activations/layer20_attention_weight_max": 23.542644500732422,
      "activations/layer20_attention_weight_min": -22.482437133789062,
      "activations/layer21_attention_weight_max": 45.430049896240234,
      "activations/layer21_attention_weight_min": -24.429092407226562,
      "activations/layer22_attention_weight_max": 31.633928298950195,
      "activations/layer22_attention_weight_min": -27.076154708862305,
      "activations/layer23_attention_weight_max": 39.977027893066406,
      "activations/layer23_attention_weight_min": -23.137535095214844,
      "activations/layer2_attention_weight_max": 32.31248092651367,
      "activations/layer2_attention_weight_min": -31.099227905273438,
      "activations/layer3_attention_weight_max": 90.3542251586914,
      "activations/layer3_attention_weight_min": -93.40327453613281,
      "activations/layer4_attention_weight_max": 99.90106201171875,
      "activations/layer4_attention_weight_min": -92.54046630859375,
      "activations/layer5_attention_weight_max": 75.13030242919922,
      "activations/layer5_attention_weight_min": -77.56388092041016,
      "activations/layer6_attention_weight_max": 56.33200454711914,
      "activations/layer6_attention_weight_min": -54.8690185546875,
      "activations/layer7_attention_weight_max": 78.0064468383789,
      "activations/layer7_attention_weight_min": -74.64764404296875,
      "activations/layer8_attention_weight_max": 50.946746826171875,
      "activations/layer8_attention_weight_min": -48.91917037963867,
      "activations/layer9_attention_weight_max": 52.29568099975586,
      "activations/layer9_attention_weight_min": -47.4926872253418,
      "epoch": 10.58,
      "learning_rate": 8.257878787878787e-05,
      "loss": 2.7879,
      "step": 182150
    },
    {
      "activations/layer0_attention_weight_max": 15.347882270812988,
      "activations/layer0_attention_weight_min": -13.897645950317383,
      "activations/layer10_attention_weight_max": 36.09163284301758,
      "activations/layer10_attention_weight_min": -34.92682647705078,
      "activations/layer11_attention_weight_max": 30.14725112915039,
      "activations/layer11_attention_weight_min": -31.560277938842773,
      "activations/layer12_attention_weight_max": 16.604183197021484,
      "activations/layer12_attention_weight_min": -24.546512603759766,
      "activations/layer13_attention_weight_max": 43.9288215637207,
      "activations/layer13_attention_weight_min": -35.53485870361328,
      "activations/layer14_attention_weight_max": 35.749603271484375,
      "activations/layer14_attention_weight_min": -30.42266273498535,
      "activations/layer15_attention_weight_max": 31.869205474853516,
      "activations/layer15_attention_weight_min": -29.10869598388672,
      "activations/layer16_attention_weight_max": 29.413528442382812,
      "activations/layer16_attention_weight_min": -29.368274688720703,
      "activations/layer17_attention_weight_max": 54.202781677246094,
      "activations/layer17_attention_weight_min": -43.89056396484375,
      "activations/layer18_attention_weight_max": 44.90457534790039,
      "activations/layer18_attention_weight_min": -36.67926788330078,
      "activations/layer19_attention_weight_max": 20.95575523376465,
      "activations/layer19_attention_weight_min": -21.828550338745117,
      "activations/layer1_attention_weight_max": 16.951313018798828,
      "activations/layer1_attention_weight_min": -16.678707122802734,
      "activations/layer20_attention_weight_max": 21.8935546875,
      "activations/layer20_attention_weight_min": -24.671232223510742,
      "activations/layer21_attention_weight_max": 32.74085235595703,
      "activations/layer21_attention_weight_min": -22.66617774963379,
      "activations/layer22_attention_weight_max": 29.912519454956055,
      "activations/layer22_attention_weight_min": -26.0472354888916,
      "activations/layer23_attention_weight_max": 34.16551971435547,
      "activations/layer23_attention_weight_min": -23.65236473083496,
      "activations/layer2_attention_weight_max": 30.906288146972656,
      "activations/layer2_attention_weight_min": -30.98562240600586,
      "activations/layer3_attention_weight_max": 85.4746322631836,
      "activations/layer3_attention_weight_min": -92.8174057006836,
      "activations/layer4_attention_weight_max": 99.83731842041016,
      "activations/layer4_attention_weight_min": -98.70663452148438,
      "activations/layer5_attention_weight_max": 75.68830871582031,
      "activations/layer5_attention_weight_min": -78.76609802246094,
      "activations/layer6_attention_weight_max": 57.04731369018555,
      "activations/layer6_attention_weight_min": -55.843204498291016,
      "activations/layer7_attention_weight_max": 73.9808349609375,
      "activations/layer7_attention_weight_min": -76.09082794189453,
      "activations/layer8_attention_weight_max": 46.98744201660156,
      "activations/layer8_attention_weight_min": -48.568145751953125,
      "activations/layer9_attention_weight_max": 48.5113410949707,
      "activations/layer9_attention_weight_min": -50.955387115478516,
      "epoch": 10.59,
      "learning_rate": 8.255984848484848e-05,
      "loss": 2.7825,
      "step": 182200
    },
    {
      "activations/layer0_attention_weight_max": 15.8215913772583,
      "activations/layer0_attention_weight_min": -13.757399559020996,
      "activations/layer10_attention_weight_max": 37.71771240234375,
      "activations/layer10_attention_weight_min": -37.38611602783203,
      "activations/layer11_attention_weight_max": 33.175201416015625,
      "activations/layer11_attention_weight_min": -33.837100982666016,
      "activations/layer12_attention_weight_max": 19.091819763183594,
      "activations/layer12_attention_weight_min": -29.313312530517578,
      "activations/layer13_attention_weight_max": 52.30036544799805,
      "activations/layer13_attention_weight_min": -30.9605655670166,
      "activations/layer14_attention_weight_max": 38.805091857910156,
      "activations/layer14_attention_weight_min": -30.49455451965332,
      "activations/layer15_attention_weight_max": 35.8216667175293,
      "activations/layer15_attention_weight_min": -28.63341522216797,
      "activations/layer16_attention_weight_max": 29.577665328979492,
      "activations/layer16_attention_weight_min": -29.973974227905273,
      "activations/layer17_attention_weight_max": 50.12310791015625,
      "activations/layer17_attention_weight_min": -41.23101806640625,
      "activations/layer18_attention_weight_max": 47.100650787353516,
      "activations/layer18_attention_weight_min": -38.85784149169922,
      "activations/layer19_attention_weight_max": 23.222013473510742,
      "activations/layer19_attention_weight_min": -21.317441940307617,
      "activations/layer1_attention_weight_max": 16.649229049682617,
      "activations/layer1_attention_weight_min": -14.399070739746094,
      "activations/layer20_attention_weight_max": 20.85177993774414,
      "activations/layer20_attention_weight_min": -20.166288375854492,
      "activations/layer21_attention_weight_max": 33.73265075683594,
      "activations/layer21_attention_weight_min": -22.905315399169922,
      "activations/layer22_attention_weight_max": 30.62541389465332,
      "activations/layer22_attention_weight_min": -25.562089920043945,
      "activations/layer23_attention_weight_max": 37.963478088378906,
      "activations/layer23_attention_weight_min": -25.246414184570312,
      "activations/layer2_attention_weight_max": 31.305463790893555,
      "activations/layer2_attention_weight_min": -30.214324951171875,
      "activations/layer3_attention_weight_max": 91.7906265258789,
      "activations/layer3_attention_weight_min": -95.55577850341797,
      "activations/layer4_attention_weight_max": 96.64068603515625,
      "activations/layer4_attention_weight_min": -93.95220184326172,
      "activations/layer5_attention_weight_max": 76.96656036376953,
      "activations/layer5_attention_weight_min": -76.93872833251953,
      "activations/layer6_attention_weight_max": 55.99717330932617,
      "activations/layer6_attention_weight_min": -55.460540771484375,
      "activations/layer7_attention_weight_max": 80.65619659423828,
      "activations/layer7_attention_weight_min": -77.7542495727539,
      "activations/layer8_attention_weight_max": 46.18007278442383,
      "activations/layer8_attention_weight_min": -51.84320831298828,
      "activations/layer9_attention_weight_max": 49.590213775634766,
      "activations/layer9_attention_weight_min": -51.5373649597168,
      "epoch": 10.59,
      "learning_rate": 8.254090909090908e-05,
      "loss": 2.824,
      "step": 182250
    },
    {
      "activations/layer0_attention_weight_max": 14.951242446899414,
      "activations/layer0_attention_weight_min": -13.23181438446045,
      "activations/layer10_attention_weight_max": 40.36675262451172,
      "activations/layer10_attention_weight_min": -40.39780044555664,
      "activations/layer11_attention_weight_max": 35.990089416503906,
      "activations/layer11_attention_weight_min": -35.10882568359375,
      "activations/layer12_attention_weight_max": 19.388790130615234,
      "activations/layer12_attention_weight_min": -25.359216690063477,
      "activations/layer13_attention_weight_max": 42.8115119934082,
      "activations/layer13_attention_weight_min": -30.57082176208496,
      "activations/layer14_attention_weight_max": 42.18644714355469,
      "activations/layer14_attention_weight_min": -33.76612854003906,
      "activations/layer15_attention_weight_max": 36.469303131103516,
      "activations/layer15_attention_weight_min": -31.474821090698242,
      "activations/layer16_attention_weight_max": 39.89055633544922,
      "activations/layer16_attention_weight_min": -32.765716552734375,
      "activations/layer17_attention_weight_max": 59.42489242553711,
      "activations/layer17_attention_weight_min": -52.277320861816406,
      "activations/layer18_attention_weight_max": 55.220890045166016,
      "activations/layer18_attention_weight_min": -44.32315444946289,
      "activations/layer19_attention_weight_max": 24.118633270263672,
      "activations/layer19_attention_weight_min": -23.67076301574707,
      "activations/layer1_attention_weight_max": 16.90936851501465,
      "activations/layer1_attention_weight_min": -14.53565502166748,
      "activations/layer20_attention_weight_max": 28.052200317382812,
      "activations/layer20_attention_weight_min": -22.25258445739746,
      "activations/layer21_attention_weight_max": 47.681602478027344,
      "activations/layer21_attention_weight_min": -23.541534423828125,
      "activations/layer22_attention_weight_max": 35.23722457885742,
      "activations/layer22_attention_weight_min": -26.708148956298828,
      "activations/layer23_attention_weight_max": 41.903751373291016,
      "activations/layer23_attention_weight_min": -26.605052947998047,
      "activations/layer2_attention_weight_max": 32.52997970581055,
      "activations/layer2_attention_weight_min": -32.406044006347656,
      "activations/layer3_attention_weight_max": 94.07504272460938,
      "activations/layer3_attention_weight_min": -96.28131103515625,
      "activations/layer4_attention_weight_max": 105.38915252685547,
      "activations/layer4_attention_weight_min": -96.45439147949219,
      "activations/layer5_attention_weight_max": 77.4747314453125,
      "activations/layer5_attention_weight_min": -76.38545227050781,
      "activations/layer6_attention_weight_max": 57.674137115478516,
      "activations/layer6_attention_weight_min": -54.511531829833984,
      "activations/layer7_attention_weight_max": 82.82904815673828,
      "activations/layer7_attention_weight_min": -74.86686706542969,
      "activations/layer8_attention_weight_max": 50.90207290649414,
      "activations/layer8_attention_weight_min": -50.81086730957031,
      "activations/layer9_attention_weight_max": 56.184627532958984,
      "activations/layer9_attention_weight_min": -51.82855224609375,
      "epoch": 10.59,
      "learning_rate": 8.252196969696969e-05,
      "loss": 2.8086,
      "step": 182300
    },
    {
      "activations/layer0_attention_weight_max": 16.54380989074707,
      "activations/layer0_attention_weight_min": -13.877604484558105,
      "activations/layer10_attention_weight_max": 41.16221618652344,
      "activations/layer10_attention_weight_min": -38.76033401489258,
      "activations/layer11_attention_weight_max": 34.95184326171875,
      "activations/layer11_attention_weight_min": -35.27145767211914,
      "activations/layer12_attention_weight_max": 18.700300216674805,
      "activations/layer12_attention_weight_min": -24.819955825805664,
      "activations/layer13_attention_weight_max": 44.657413482666016,
      "activations/layer13_attention_weight_min": -34.043521881103516,
      "activations/layer14_attention_weight_max": 38.668704986572266,
      "activations/layer14_attention_weight_min": -28.765270233154297,
      "activations/layer15_attention_weight_max": 35.27415466308594,
      "activations/layer15_attention_weight_min": -28.741151809692383,
      "activations/layer16_attention_weight_max": 32.14621353149414,
      "activations/layer16_attention_weight_min": -29.890134811401367,
      "activations/layer17_attention_weight_max": 55.362850189208984,
      "activations/layer17_attention_weight_min": -42.45256805419922,
      "activations/layer18_attention_weight_max": 46.0911979675293,
      "activations/layer18_attention_weight_min": -36.841766357421875,
      "activations/layer19_attention_weight_max": 22.329404830932617,
      "activations/layer19_attention_weight_min": -25.079092025756836,
      "activations/layer1_attention_weight_max": 16.766700744628906,
      "activations/layer1_attention_weight_min": -14.278623580932617,
      "activations/layer20_attention_weight_max": 22.51681900024414,
      "activations/layer20_attention_weight_min": -22.695524215698242,
      "activations/layer21_attention_weight_max": 37.45280456542969,
      "activations/layer21_attention_weight_min": -22.927881240844727,
      "activations/layer22_attention_weight_max": 31.420501708984375,
      "activations/layer22_attention_weight_min": -24.887144088745117,
      "activations/layer23_attention_weight_max": 39.33754348754883,
      "activations/layer23_attention_weight_min": -24.570409774780273,
      "activations/layer2_attention_weight_max": 30.948734283447266,
      "activations/layer2_attention_weight_min": -30.18146514892578,
      "activations/layer3_attention_weight_max": 87.30753326416016,
      "activations/layer3_attention_weight_min": -92.31998443603516,
      "activations/layer4_attention_weight_max": 97.07109069824219,
      "activations/layer4_attention_weight_min": -93.44598388671875,
      "activations/layer5_attention_weight_max": 73.49325561523438,
      "activations/layer5_attention_weight_min": -77.71392822265625,
      "activations/layer6_attention_weight_max": 53.97354507446289,
      "activations/layer6_attention_weight_min": -52.7966194152832,
      "activations/layer7_attention_weight_max": 72.4782943725586,
      "activations/layer7_attention_weight_min": -76.0848159790039,
      "activations/layer8_attention_weight_max": 48.93621826171875,
      "activations/layer8_attention_weight_min": -48.90603256225586,
      "activations/layer9_attention_weight_max": 50.06193542480469,
      "activations/layer9_attention_weight_min": -49.21315002441406,
      "epoch": 10.6,
      "learning_rate": 8.250303030303029e-05,
      "loss": 2.7805,
      "step": 182350
    },
    {
      "activations/layer0_attention_weight_max": 15.278282165527344,
      "activations/layer0_attention_weight_min": -13.979846954345703,
      "activations/layer10_attention_weight_max": 41.683448791503906,
      "activations/layer10_attention_weight_min": -42.16201400756836,
      "activations/layer11_attention_weight_max": 35.92119598388672,
      "activations/layer11_attention_weight_min": -37.95056915283203,
      "activations/layer12_attention_weight_max": 18.93581199645996,
      "activations/layer12_attention_weight_min": -24.54181671142578,
      "activations/layer13_attention_weight_max": 44.218040466308594,
      "activations/layer13_attention_weight_min": -31.897727966308594,
      "activations/layer14_attention_weight_max": 37.15392303466797,
      "activations/layer14_attention_weight_min": -31.519983291625977,
      "activations/layer15_attention_weight_max": 34.287696838378906,
      "activations/layer15_attention_weight_min": -30.587631225585938,
      "activations/layer16_attention_weight_max": 32.69834518432617,
      "activations/layer16_attention_weight_min": -29.381309509277344,
      "activations/layer17_attention_weight_max": 57.43069839477539,
      "activations/layer17_attention_weight_min": -47.42018508911133,
      "activations/layer18_attention_weight_max": 50.37393569946289,
      "activations/layer18_attention_weight_min": -39.09926223754883,
      "activations/layer19_attention_weight_max": 22.623825073242188,
      "activations/layer19_attention_weight_min": -22.69041633605957,
      "activations/layer1_attention_weight_max": 16.65372657775879,
      "activations/layer1_attention_weight_min": -16.439788818359375,
      "activations/layer20_attention_weight_max": 22.709239959716797,
      "activations/layer20_attention_weight_min": -22.028181076049805,
      "activations/layer21_attention_weight_max": 40.46492385864258,
      "activations/layer21_attention_weight_min": -22.98887062072754,
      "activations/layer22_attention_weight_max": 32.76428985595703,
      "activations/layer22_attention_weight_min": -25.482589721679688,
      "activations/layer23_attention_weight_max": 41.50934600830078,
      "activations/layer23_attention_weight_min": -25.39235496520996,
      "activations/layer2_attention_weight_max": 32.443843841552734,
      "activations/layer2_attention_weight_min": -33.43655776977539,
      "activations/layer3_attention_weight_max": 93.01659393310547,
      "activations/layer3_attention_weight_min": -96.81861877441406,
      "activations/layer4_attention_weight_max": 97.49138641357422,
      "activations/layer4_attention_weight_min": -96.39815521240234,
      "activations/layer5_attention_weight_max": 74.79961395263672,
      "activations/layer5_attention_weight_min": -78.67082214355469,
      "activations/layer6_attention_weight_max": 53.849788665771484,
      "activations/layer6_attention_weight_min": -52.905113220214844,
      "activations/layer7_attention_weight_max": 80.3082275390625,
      "activations/layer7_attention_weight_min": -76.39044952392578,
      "activations/layer8_attention_weight_max": 50.58427429199219,
      "activations/layer8_attention_weight_min": -52.045528411865234,
      "activations/layer9_attention_weight_max": 55.310890197753906,
      "activations/layer9_attention_weight_min": -52.4937744140625,
      "epoch": 10.6,
      "learning_rate": 8.24840909090909e-05,
      "loss": 2.774,
      "step": 182400
    },
    {
      "activations/layer0_attention_weight_max": 16.31174659729004,
      "activations/layer0_attention_weight_min": -14.342415809631348,
      "activations/layer10_attention_weight_max": 40.020511627197266,
      "activations/layer10_attention_weight_min": -38.638763427734375,
      "activations/layer11_attention_weight_max": 34.064666748046875,
      "activations/layer11_attention_weight_min": -33.626426696777344,
      "activations/layer12_attention_weight_max": 21.03021240234375,
      "activations/layer12_attention_weight_min": -28.378984451293945,
      "activations/layer13_attention_weight_max": 55.03437423706055,
      "activations/layer13_attention_weight_min": -32.640872955322266,
      "activations/layer14_attention_weight_max": 50.152732849121094,
      "activations/layer14_attention_weight_min": -32.28421401977539,
      "activations/layer15_attention_weight_max": 42.138465881347656,
      "activations/layer15_attention_weight_min": -30.493919372558594,
      "activations/layer16_attention_weight_max": 33.73814010620117,
      "activations/layer16_attention_weight_min": -30.305191040039062,
      "activations/layer17_attention_weight_max": 55.25133514404297,
      "activations/layer17_attention_weight_min": -46.19178009033203,
      "activations/layer18_attention_weight_max": 51.041015625,
      "activations/layer18_attention_weight_min": -39.09844970703125,
      "activations/layer19_attention_weight_max": 24.010496139526367,
      "activations/layer19_attention_weight_min": -22.873197555541992,
      "activations/layer1_attention_weight_max": 16.35966682434082,
      "activations/layer1_attention_weight_min": -13.866923332214355,
      "activations/layer20_attention_weight_max": 27.363706588745117,
      "activations/layer20_attention_weight_min": -23.290544509887695,
      "activations/layer21_attention_weight_max": 41.59015655517578,
      "activations/layer21_attention_weight_min": -24.225078582763672,
      "activations/layer22_attention_weight_max": 34.61921310424805,
      "activations/layer22_attention_weight_min": -26.06394386291504,
      "activations/layer23_attention_weight_max": 42.01142883300781,
      "activations/layer23_attention_weight_min": -23.789609909057617,
      "activations/layer2_attention_weight_max": 32.85225296020508,
      "activations/layer2_attention_weight_min": -32.774879455566406,
      "activations/layer3_attention_weight_max": 93.72047424316406,
      "activations/layer3_attention_weight_min": -100.61172485351562,
      "activations/layer4_attention_weight_max": 100.96363067626953,
      "activations/layer4_attention_weight_min": -98.48432159423828,
      "activations/layer5_attention_weight_max": 75.33069610595703,
      "activations/layer5_attention_weight_min": -79.11878204345703,
      "activations/layer6_attention_weight_max": 55.130130767822266,
      "activations/layer6_attention_weight_min": -56.200199127197266,
      "activations/layer7_attention_weight_max": 74.12939453125,
      "activations/layer7_attention_weight_min": -74.70307159423828,
      "activations/layer8_attention_weight_max": 49.12862014770508,
      "activations/layer8_attention_weight_min": -49.181396484375,
      "activations/layer9_attention_weight_max": 49.60712814331055,
      "activations/layer9_attention_weight_min": -49.42502975463867,
      "epoch": 10.6,
      "learning_rate": 8.246515151515152e-05,
      "loss": 2.7882,
      "step": 182450
    },
    {
      "activations/layer0_attention_weight_max": 17.43402862548828,
      "activations/layer0_attention_weight_min": -14.12304973602295,
      "activations/layer10_attention_weight_max": 40.247779846191406,
      "activations/layer10_attention_weight_min": -39.59674835205078,
      "activations/layer11_attention_weight_max": 35.71348571777344,
      "activations/layer11_attention_weight_min": -34.262542724609375,
      "activations/layer12_attention_weight_max": 18.61130714416504,
      "activations/layer12_attention_weight_min": -27.104957580566406,
      "activations/layer13_attention_weight_max": 65.84922790527344,
      "activations/layer13_attention_weight_min": -32.60744857788086,
      "activations/layer14_attention_weight_max": 60.03352355957031,
      "activations/layer14_attention_weight_min": -31.48990821838379,
      "activations/layer15_attention_weight_max": 46.75303268432617,
      "activations/layer15_attention_weight_min": -28.423730850219727,
      "activations/layer16_attention_weight_max": 35.58951950073242,
      "activations/layer16_attention_weight_min": -29.425737380981445,
      "activations/layer17_attention_weight_max": 57.57915496826172,
      "activations/layer17_attention_weight_min": -47.781307220458984,
      "activations/layer18_attention_weight_max": 51.43070602416992,
      "activations/layer18_attention_weight_min": -36.535797119140625,
      "activations/layer19_attention_weight_max": 23.5145320892334,
      "activations/layer19_attention_weight_min": -20.884553909301758,
      "activations/layer1_attention_weight_max": 16.72303581237793,
      "activations/layer1_attention_weight_min": -14.421310424804688,
      "activations/layer20_attention_weight_max": 22.62785530090332,
      "activations/layer20_attention_weight_min": -21.054170608520508,
      "activations/layer21_attention_weight_max": 38.836097717285156,
      "activations/layer21_attention_weight_min": -21.826263427734375,
      "activations/layer22_attention_weight_max": 31.72981834411621,
      "activations/layer22_attention_weight_min": -23.58647918701172,
      "activations/layer23_attention_weight_max": 38.244441986083984,
      "activations/layer23_attention_weight_min": -21.0064697265625,
      "activations/layer2_attention_weight_max": 32.4260139465332,
      "activations/layer2_attention_weight_min": -31.10454559326172,
      "activations/layer3_attention_weight_max": 91.40293884277344,
      "activations/layer3_attention_weight_min": -90.62960815429688,
      "activations/layer4_attention_weight_max": 101.02979278564453,
      "activations/layer4_attention_weight_min": -96.28252410888672,
      "activations/layer5_attention_weight_max": 76.81553649902344,
      "activations/layer5_attention_weight_min": -82.75786590576172,
      "activations/layer6_attention_weight_max": 58.54264450073242,
      "activations/layer6_attention_weight_min": -53.93720245361328,
      "activations/layer7_attention_weight_max": 76.0204086303711,
      "activations/layer7_attention_weight_min": -74.45664978027344,
      "activations/layer8_attention_weight_max": 50.36773681640625,
      "activations/layer8_attention_weight_min": -49.27271270751953,
      "activations/layer9_attention_weight_max": 53.785926818847656,
      "activations/layer9_attention_weight_min": -49.93736267089844,
      "epoch": 10.6,
      "learning_rate": 8.24462121212121e-05,
      "loss": 2.776,
      "step": 182500
    },
    {
      "activations/layer0_attention_weight_max": 15.99545669555664,
      "activations/layer0_attention_weight_min": -14.459177017211914,
      "activations/layer10_attention_weight_max": 35.95719909667969,
      "activations/layer10_attention_weight_min": -36.18163299560547,
      "activations/layer11_attention_weight_max": 31.592960357666016,
      "activations/layer11_attention_weight_min": -34.65170669555664,
      "activations/layer12_attention_weight_max": 17.68088722229004,
      "activations/layer12_attention_weight_min": -30.36988067626953,
      "activations/layer13_attention_weight_max": 40.365142822265625,
      "activations/layer13_attention_weight_min": -28.167766571044922,
      "activations/layer14_attention_weight_max": 39.00532913208008,
      "activations/layer14_attention_weight_min": -31.154380798339844,
      "activations/layer15_attention_weight_max": 32.86920166015625,
      "activations/layer15_attention_weight_min": -32.0282096862793,
      "activations/layer16_attention_weight_max": 29.754066467285156,
      "activations/layer16_attention_weight_min": -28.13153648376465,
      "activations/layer17_attention_weight_max": 52.365421295166016,
      "activations/layer17_attention_weight_min": -43.45674514770508,
      "activations/layer18_attention_weight_max": 49.34453582763672,
      "activations/layer18_attention_weight_min": -37.55312728881836,
      "activations/layer19_attention_weight_max": 22.71422576904297,
      "activations/layer19_attention_weight_min": -21.7784423828125,
      "activations/layer1_attention_weight_max": 16.812562942504883,
      "activations/layer1_attention_weight_min": -15.551973342895508,
      "activations/layer20_attention_weight_max": 22.411882400512695,
      "activations/layer20_attention_weight_min": -22.817461013793945,
      "activations/layer21_attention_weight_max": 38.294132232666016,
      "activations/layer21_attention_weight_min": -22.26949119567871,
      "activations/layer22_attention_weight_max": 29.606700897216797,
      "activations/layer22_attention_weight_min": -25.573734283447266,
      "activations/layer23_attention_weight_max": 40.90021514892578,
      "activations/layer23_attention_weight_min": -22.457426071166992,
      "activations/layer2_attention_weight_max": 32.98992919921875,
      "activations/layer2_attention_weight_min": -31.894071578979492,
      "activations/layer3_attention_weight_max": 97.32804870605469,
      "activations/layer3_attention_weight_min": -95.67668914794922,
      "activations/layer4_attention_weight_max": 98.72957611083984,
      "activations/layer4_attention_weight_min": -95.8546371459961,
      "activations/layer5_attention_weight_max": 73.71622467041016,
      "activations/layer5_attention_weight_min": -77.81993103027344,
      "activations/layer6_attention_weight_max": 51.11842346191406,
      "activations/layer6_attention_weight_min": -53.89734649658203,
      "activations/layer7_attention_weight_max": 73.14965057373047,
      "activations/layer7_attention_weight_min": -70.95166778564453,
      "activations/layer8_attention_weight_max": 46.32500076293945,
      "activations/layer8_attention_weight_min": -49.96013641357422,
      "activations/layer9_attention_weight_max": 49.2242431640625,
      "activations/layer9_attention_weight_min": -48.22847366333008,
      "epoch": 10.61,
      "learning_rate": 8.242727272727272e-05,
      "loss": 2.7921,
      "step": 182550
    },
    {
      "activations/layer0_attention_weight_max": 16.36087417602539,
      "activations/layer0_attention_weight_min": -14.324350357055664,
      "activations/layer10_attention_weight_max": 37.626441955566406,
      "activations/layer10_attention_weight_min": -37.58204650878906,
      "activations/layer11_attention_weight_max": 31.77443504333496,
      "activations/layer11_attention_weight_min": -31.287921905517578,
      "activations/layer12_attention_weight_max": 19.459924697875977,
      "activations/layer12_attention_weight_min": -22.9959659576416,
      "activations/layer13_attention_weight_max": 42.687843322753906,
      "activations/layer13_attention_weight_min": -30.267662048339844,
      "activations/layer14_attention_weight_max": 47.184818267822266,
      "activations/layer14_attention_weight_min": -34.65470886230469,
      "activations/layer15_attention_weight_max": 40.644126892089844,
      "activations/layer15_attention_weight_min": -31.766799926757812,
      "activations/layer16_attention_weight_max": 31.811311721801758,
      "activations/layer16_attention_weight_min": -30.123876571655273,
      "activations/layer17_attention_weight_max": 56.18342971801758,
      "activations/layer17_attention_weight_min": -44.96282958984375,
      "activations/layer18_attention_weight_max": 51.664180755615234,
      "activations/layer18_attention_weight_min": -39.907745361328125,
      "activations/layer19_attention_weight_max": 26.751922607421875,
      "activations/layer19_attention_weight_min": -22.568389892578125,
      "activations/layer1_attention_weight_max": 17.092954635620117,
      "activations/layer1_attention_weight_min": -14.2431058883667,
      "activations/layer20_attention_weight_max": 22.90540313720703,
      "activations/layer20_attention_weight_min": -21.04634666442871,
      "activations/layer21_attention_weight_max": 40.64493942260742,
      "activations/layer21_attention_weight_min": -24.735427856445312,
      "activations/layer22_attention_weight_max": 31.114974975585938,
      "activations/layer22_attention_weight_min": -23.76871681213379,
      "activations/layer23_attention_weight_max": 38.55773162841797,
      "activations/layer23_attention_weight_min": -22.749855041503906,
      "activations/layer2_attention_weight_max": 30.829992294311523,
      "activations/layer2_attention_weight_min": -30.821735382080078,
      "activations/layer3_attention_weight_max": 88.23185729980469,
      "activations/layer3_attention_weight_min": -93.97948455810547,
      "activations/layer4_attention_weight_max": 97.63884735107422,
      "activations/layer4_attention_weight_min": -96.22946166992188,
      "activations/layer5_attention_weight_max": 77.53312683105469,
      "activations/layer5_attention_weight_min": -81.9681396484375,
      "activations/layer6_attention_weight_max": 53.86013412475586,
      "activations/layer6_attention_weight_min": -53.789512634277344,
      "activations/layer7_attention_weight_max": 71.1538314819336,
      "activations/layer7_attention_weight_min": -73.29006958007812,
      "activations/layer8_attention_weight_max": 45.8040657043457,
      "activations/layer8_attention_weight_min": -48.57642364501953,
      "activations/layer9_attention_weight_max": 48.95958709716797,
      "activations/layer9_attention_weight_min": -52.01624298095703,
      "epoch": 10.61,
      "learning_rate": 8.240833333333334e-05,
      "loss": 2.7858,
      "step": 182600
    },
    {
      "activations/layer0_attention_weight_max": 16.066490173339844,
      "activations/layer0_attention_weight_min": -14.156508445739746,
      "activations/layer10_attention_weight_max": 37.153419494628906,
      "activations/layer10_attention_weight_min": -34.59223937988281,
      "activations/layer11_attention_weight_max": 31.60822296142578,
      "activations/layer11_attention_weight_min": -32.555049896240234,
      "activations/layer12_attention_weight_max": 19.752933502197266,
      "activations/layer12_attention_weight_min": -26.551620483398438,
      "activations/layer13_attention_weight_max": 37.91439437866211,
      "activations/layer13_attention_weight_min": -35.66775894165039,
      "activations/layer14_attention_weight_max": 34.14897155761719,
      "activations/layer14_attention_weight_min": -33.84449005126953,
      "activations/layer15_attention_weight_max": 32.219886779785156,
      "activations/layer15_attention_weight_min": -34.82759475708008,
      "activations/layer16_attention_weight_max": 31.52971076965332,
      "activations/layer16_attention_weight_min": -28.892963409423828,
      "activations/layer17_attention_weight_max": 49.47431945800781,
      "activations/layer17_attention_weight_min": -48.58894729614258,
      "activations/layer18_attention_weight_max": 43.64036178588867,
      "activations/layer18_attention_weight_min": -41.73763656616211,
      "activations/layer19_attention_weight_max": 25.718643188476562,
      "activations/layer19_attention_weight_min": -23.80215072631836,
      "activations/layer1_attention_weight_max": 17.46514892578125,
      "activations/layer1_attention_weight_min": -15.929783821105957,
      "activations/layer20_attention_weight_max": 21.975549697875977,
      "activations/layer20_attention_weight_min": -22.832292556762695,
      "activations/layer21_attention_weight_max": 39.273677825927734,
      "activations/layer21_attention_weight_min": -24.273244857788086,
      "activations/layer22_attention_weight_max": 29.5865535736084,
      "activations/layer22_attention_weight_min": -25.973360061645508,
      "activations/layer23_attention_weight_max": 36.845497131347656,
      "activations/layer23_attention_weight_min": -26.71392822265625,
      "activations/layer2_attention_weight_max": 31.915212631225586,
      "activations/layer2_attention_weight_min": -30.909854888916016,
      "activations/layer3_attention_weight_max": 92.19215393066406,
      "activations/layer3_attention_weight_min": -93.86510467529297,
      "activations/layer4_attention_weight_max": 101.07918548583984,
      "activations/layer4_attention_weight_min": -96.22274017333984,
      "activations/layer5_attention_weight_max": 75.16282653808594,
      "activations/layer5_attention_weight_min": -79.49974822998047,
      "activations/layer6_attention_weight_max": 51.03276443481445,
      "activations/layer6_attention_weight_min": -53.07168960571289,
      "activations/layer7_attention_weight_max": 72.6023178100586,
      "activations/layer7_attention_weight_min": -72.17227172851562,
      "activations/layer8_attention_weight_max": 44.010555267333984,
      "activations/layer8_attention_weight_min": -47.57236862182617,
      "activations/layer9_attention_weight_max": 47.676658630371094,
      "activations/layer9_attention_weight_min": -48.66477966308594,
      "epoch": 10.61,
      "learning_rate": 8.238939393939392e-05,
      "loss": 2.7783,
      "step": 182650
    },
    {
      "activations/layer0_attention_weight_max": 16.02989387512207,
      "activations/layer0_attention_weight_min": -14.334569931030273,
      "activations/layer10_attention_weight_max": 40.21160888671875,
      "activations/layer10_attention_weight_min": -37.427711486816406,
      "activations/layer11_attention_weight_max": 33.40205001831055,
      "activations/layer11_attention_weight_min": -33.49509811401367,
      "activations/layer12_attention_weight_max": 16.915027618408203,
      "activations/layer12_attention_weight_min": -30.721744537353516,
      "activations/layer13_attention_weight_max": 39.11930465698242,
      "activations/layer13_attention_weight_min": -34.579322814941406,
      "activations/layer14_attention_weight_max": 36.46349334716797,
      "activations/layer14_attention_weight_min": -29.538442611694336,
      "activations/layer15_attention_weight_max": 33.67873764038086,
      "activations/layer15_attention_weight_min": -30.025585174560547,
      "activations/layer16_attention_weight_max": 31.040283203125,
      "activations/layer16_attention_weight_min": -30.09491539001465,
      "activations/layer17_attention_weight_max": 52.78076934814453,
      "activations/layer17_attention_weight_min": -43.45094680786133,
      "activations/layer18_attention_weight_max": 50.15306854248047,
      "activations/layer18_attention_weight_min": -37.94755554199219,
      "activations/layer19_attention_weight_max": 24.56343650817871,
      "activations/layer19_attention_weight_min": -20.722721099853516,
      "activations/layer1_attention_weight_max": 16.675933837890625,
      "activations/layer1_attention_weight_min": -16.010265350341797,
      "activations/layer20_attention_weight_max": 22.41851806640625,
      "activations/layer20_attention_weight_min": -22.199962615966797,
      "activations/layer21_attention_weight_max": 40.204124450683594,
      "activations/layer21_attention_weight_min": -22.860504150390625,
      "activations/layer22_attention_weight_max": 32.39926528930664,
      "activations/layer22_attention_weight_min": -24.54141616821289,
      "activations/layer23_attention_weight_max": 39.48767852783203,
      "activations/layer23_attention_weight_min": -22.60541534423828,
      "activations/layer2_attention_weight_max": 35.1569709777832,
      "activations/layer2_attention_weight_min": -30.75705337524414,
      "activations/layer3_attention_weight_max": 94.58336639404297,
      "activations/layer3_attention_weight_min": -90.66407012939453,
      "activations/layer4_attention_weight_max": 102.4352798461914,
      "activations/layer4_attention_weight_min": -95.5086669921875,
      "activations/layer5_attention_weight_max": 76.64217376708984,
      "activations/layer5_attention_weight_min": -76.1122817993164,
      "activations/layer6_attention_weight_max": 53.90290451049805,
      "activations/layer6_attention_weight_min": -54.127525329589844,
      "activations/layer7_attention_weight_max": 78.20415496826172,
      "activations/layer7_attention_weight_min": -73.58260345458984,
      "activations/layer8_attention_weight_max": 48.913692474365234,
      "activations/layer8_attention_weight_min": -47.62602615356445,
      "activations/layer9_attention_weight_max": 55.775856018066406,
      "activations/layer9_attention_weight_min": -50.46579360961914,
      "epoch": 10.62,
      "learning_rate": 8.237045454545454e-05,
      "loss": 2.798,
      "step": 182700
    },
    {
      "activations/layer0_attention_weight_max": 15.035819053649902,
      "activations/layer0_attention_weight_min": -14.481881141662598,
      "activations/layer10_attention_weight_max": 35.3324089050293,
      "activations/layer10_attention_weight_min": -34.1441535949707,
      "activations/layer11_attention_weight_max": 30.400253295898438,
      "activations/layer11_attention_weight_min": -30.870685577392578,
      "activations/layer12_attention_weight_max": 16.373315811157227,
      "activations/layer12_attention_weight_min": -27.325727462768555,
      "activations/layer13_attention_weight_max": 38.97084045410156,
      "activations/layer13_attention_weight_min": -32.9646110534668,
      "activations/layer14_attention_weight_max": 40.6146125793457,
      "activations/layer14_attention_weight_min": -31.4677734375,
      "activations/layer15_attention_weight_max": 33.650089263916016,
      "activations/layer15_attention_weight_min": -32.05316925048828,
      "activations/layer16_attention_weight_max": 31.02280616760254,
      "activations/layer16_attention_weight_min": -30.11989402770996,
      "activations/layer17_attention_weight_max": 56.632205963134766,
      "activations/layer17_attention_weight_min": -46.47625732421875,
      "activations/layer18_attention_weight_max": 45.37335205078125,
      "activations/layer18_attention_weight_min": -39.27424621582031,
      "activations/layer19_attention_weight_max": 22.752748489379883,
      "activations/layer19_attention_weight_min": -23.676145553588867,
      "activations/layer1_attention_weight_max": 17.54042625427246,
      "activations/layer1_attention_weight_min": -14.757259368896484,
      "activations/layer20_attention_weight_max": 22.732574462890625,
      "activations/layer20_attention_weight_min": -22.621402740478516,
      "activations/layer21_attention_weight_max": 45.87114715576172,
      "activations/layer21_attention_weight_min": -34.048954010009766,
      "activations/layer22_attention_weight_max": 29.30859375,
      "activations/layer22_attention_weight_min": -25.405475616455078,
      "activations/layer23_attention_weight_max": 40.41409683227539,
      "activations/layer23_attention_weight_min": -24.814062118530273,
      "activations/layer2_attention_weight_max": 30.93585205078125,
      "activations/layer2_attention_weight_min": -30.342634201049805,
      "activations/layer3_attention_weight_max": 87.56200408935547,
      "activations/layer3_attention_weight_min": -93.59062957763672,
      "activations/layer4_attention_weight_max": 94.96198272705078,
      "activations/layer4_attention_weight_min": -90.53539276123047,
      "activations/layer5_attention_weight_max": 70.58950805664062,
      "activations/layer5_attention_weight_min": -81.24935913085938,
      "activations/layer6_attention_weight_max": 52.427650451660156,
      "activations/layer6_attention_weight_min": -51.68198776245117,
      "activations/layer7_attention_weight_max": 72.32587432861328,
      "activations/layer7_attention_weight_min": -70.50057220458984,
      "activations/layer8_attention_weight_max": 43.59233093261719,
      "activations/layer8_attention_weight_min": -45.13261795043945,
      "activations/layer9_attention_weight_max": 45.56484603881836,
      "activations/layer9_attention_weight_min": -47.60408401489258,
      "epoch": 10.62,
      "learning_rate": 8.235151515151515e-05,
      "loss": 2.7963,
      "step": 182750
    },
    {
      "activations/layer0_attention_weight_max": 14.872637748718262,
      "activations/layer0_attention_weight_min": -14.481817245483398,
      "activations/layer10_attention_weight_max": 36.76237487792969,
      "activations/layer10_attention_weight_min": -36.63487243652344,
      "activations/layer11_attention_weight_max": 30.772571563720703,
      "activations/layer11_attention_weight_min": -32.27524948120117,
      "activations/layer12_attention_weight_max": 17.468469619750977,
      "activations/layer12_attention_weight_min": -25.019323348999023,
      "activations/layer13_attention_weight_max": 38.673439025878906,
      "activations/layer13_attention_weight_min": -28.03559112548828,
      "activations/layer14_attention_weight_max": 37.94858932495117,
      "activations/layer14_attention_weight_min": -29.796545028686523,
      "activations/layer15_attention_weight_max": 33.926544189453125,
      "activations/layer15_attention_weight_min": -28.391496658325195,
      "activations/layer16_attention_weight_max": 30.089149475097656,
      "activations/layer16_attention_weight_min": -29.167945861816406,
      "activations/layer17_attention_weight_max": 55.233882904052734,
      "activations/layer17_attention_weight_min": -43.17522048950195,
      "activations/layer18_attention_weight_max": 48.658939361572266,
      "activations/layer18_attention_weight_min": -36.781070709228516,
      "activations/layer19_attention_weight_max": 20.2497615814209,
      "activations/layer19_attention_weight_min": -20.07750129699707,
      "activations/layer1_attention_weight_max": 16.78851890563965,
      "activations/layer1_attention_weight_min": -15.380614280700684,
      "activations/layer20_attention_weight_max": 21.955543518066406,
      "activations/layer20_attention_weight_min": -20.488916397094727,
      "activations/layer21_attention_weight_max": 32.33592224121094,
      "activations/layer21_attention_weight_min": -21.951601028442383,
      "activations/layer22_attention_weight_max": 27.582178115844727,
      "activations/layer22_attention_weight_min": -25.188291549682617,
      "activations/layer23_attention_weight_max": 37.537384033203125,
      "activations/layer23_attention_weight_min": -24.348615646362305,
      "activations/layer2_attention_weight_max": 33.783447265625,
      "activations/layer2_attention_weight_min": -31.305587768554688,
      "activations/layer3_attention_weight_max": 95.54010009765625,
      "activations/layer3_attention_weight_min": -93.76671600341797,
      "activations/layer4_attention_weight_max": 99.67201232910156,
      "activations/layer4_attention_weight_min": -96.10413360595703,
      "activations/layer5_attention_weight_max": 76.6512222290039,
      "activations/layer5_attention_weight_min": -83.9088363647461,
      "activations/layer6_attention_weight_max": 56.457496643066406,
      "activations/layer6_attention_weight_min": -54.248817443847656,
      "activations/layer7_attention_weight_max": 76.20967864990234,
      "activations/layer7_attention_weight_min": -72.89820098876953,
      "activations/layer8_attention_weight_max": 49.86829376220703,
      "activations/layer8_attention_weight_min": -49.27239227294922,
      "activations/layer9_attention_weight_max": 47.21567153930664,
      "activations/layer9_attention_weight_min": -52.84037399291992,
      "epoch": 10.62,
      "learning_rate": 8.233257575757574e-05,
      "loss": 2.7955,
      "step": 182800
    },
    {
      "activations/layer0_attention_weight_max": 14.321565628051758,
      "activations/layer0_attention_weight_min": -13.779779434204102,
      "activations/layer10_attention_weight_max": 39.93140411376953,
      "activations/layer10_attention_weight_min": -36.70954132080078,
      "activations/layer11_attention_weight_max": 35.06116485595703,
      "activations/layer11_attention_weight_min": -33.054840087890625,
      "activations/layer12_attention_weight_max": 19.857053756713867,
      "activations/layer12_attention_weight_min": -28.967920303344727,
      "activations/layer13_attention_weight_max": 47.885501861572266,
      "activations/layer13_attention_weight_min": -29.937339782714844,
      "activations/layer14_attention_weight_max": 36.95504379272461,
      "activations/layer14_attention_weight_min": -31.868005752563477,
      "activations/layer15_attention_weight_max": 34.494632720947266,
      "activations/layer15_attention_weight_min": -29.334949493408203,
      "activations/layer16_attention_weight_max": 31.17307472229004,
      "activations/layer16_attention_weight_min": -27.6343994140625,
      "activations/layer17_attention_weight_max": 53.89957809448242,
      "activations/layer17_attention_weight_min": -43.64584732055664,
      "activations/layer18_attention_weight_max": 47.7631721496582,
      "activations/layer18_attention_weight_min": -37.515865325927734,
      "activations/layer19_attention_weight_max": 20.990095138549805,
      "activations/layer19_attention_weight_min": -20.728361129760742,
      "activations/layer1_attention_weight_max": 16.930997848510742,
      "activations/layer1_attention_weight_min": -14.902175903320312,
      "activations/layer20_attention_weight_max": 22.418169021606445,
      "activations/layer20_attention_weight_min": -21.341495513916016,
      "activations/layer21_attention_weight_max": 36.000850677490234,
      "activations/layer21_attention_weight_min": -20.855064392089844,
      "activations/layer22_attention_weight_max": 28.36949920654297,
      "activations/layer22_attention_weight_min": -25.43528175354004,
      "activations/layer23_attention_weight_max": 35.98111343383789,
      "activations/layer23_attention_weight_min": -23.601585388183594,
      "activations/layer2_attention_weight_max": 30.954511642456055,
      "activations/layer2_attention_weight_min": -30.85932731628418,
      "activations/layer3_attention_weight_max": 91.81339263916016,
      "activations/layer3_attention_weight_min": -95.90909576416016,
      "activations/layer4_attention_weight_max": 100.65486907958984,
      "activations/layer4_attention_weight_min": -96.29891204833984,
      "activations/layer5_attention_weight_max": 77.4773941040039,
      "activations/layer5_attention_weight_min": -80.00453186035156,
      "activations/layer6_attention_weight_max": 54.901092529296875,
      "activations/layer6_attention_weight_min": -55.22914505004883,
      "activations/layer7_attention_weight_max": 82.9738998413086,
      "activations/layer7_attention_weight_min": -75.62993621826172,
      "activations/layer8_attention_weight_max": 48.6777229309082,
      "activations/layer8_attention_weight_min": -48.15961837768555,
      "activations/layer9_attention_weight_max": 57.15582275390625,
      "activations/layer9_attention_weight_min": -50.16128158569336,
      "epoch": 10.62,
      "learning_rate": 8.231363636363636e-05,
      "loss": 2.7653,
      "step": 182850
    },
    {
      "activations/layer0_attention_weight_max": 15.005928039550781,
      "activations/layer0_attention_weight_min": -14.032600402832031,
      "activations/layer10_attention_weight_max": 32.80458450317383,
      "activations/layer10_attention_weight_min": -33.42323684692383,
      "activations/layer11_attention_weight_max": 30.965599060058594,
      "activations/layer11_attention_weight_min": -30.501495361328125,
      "activations/layer12_attention_weight_max": 16.680553436279297,
      "activations/layer12_attention_weight_min": -26.518217086791992,
      "activations/layer13_attention_weight_max": 43.31297302246094,
      "activations/layer13_attention_weight_min": -27.33847999572754,
      "activations/layer14_attention_weight_max": 34.75888442993164,
      "activations/layer14_attention_weight_min": -29.868587493896484,
      "activations/layer15_attention_weight_max": 31.025300979614258,
      "activations/layer15_attention_weight_min": -28.169700622558594,
      "activations/layer16_attention_weight_max": 30.241540908813477,
      "activations/layer16_attention_weight_min": -27.31897735595703,
      "activations/layer17_attention_weight_max": 48.2440299987793,
      "activations/layer17_attention_weight_min": -40.21831130981445,
      "activations/layer18_attention_weight_max": 44.710418701171875,
      "activations/layer18_attention_weight_min": -36.87773895263672,
      "activations/layer19_attention_weight_max": 20.54130744934082,
      "activations/layer19_attention_weight_min": -21.871479034423828,
      "activations/layer1_attention_weight_max": 17.480066299438477,
      "activations/layer1_attention_weight_min": -14.632765769958496,
      "activations/layer20_attention_weight_max": 20.31349754333496,
      "activations/layer20_attention_weight_min": -22.905454635620117,
      "activations/layer21_attention_weight_max": 34.720497131347656,
      "activations/layer21_attention_weight_min": -20.86883544921875,
      "activations/layer22_attention_weight_max": 28.752595901489258,
      "activations/layer22_attention_weight_min": -25.76485824584961,
      "activations/layer23_attention_weight_max": 35.39888000488281,
      "activations/layer23_attention_weight_min": -26.55140495300293,
      "activations/layer2_attention_weight_max": 31.095436096191406,
      "activations/layer2_attention_weight_min": -31.194313049316406,
      "activations/layer3_attention_weight_max": 88.21568298339844,
      "activations/layer3_attention_weight_min": -87.2025375366211,
      "activations/layer4_attention_weight_max": 97.29888916015625,
      "activations/layer4_attention_weight_min": -92.60201263427734,
      "activations/layer5_attention_weight_max": 73.54743194580078,
      "activations/layer5_attention_weight_min": -83.54049682617188,
      "activations/layer6_attention_weight_max": 52.09621810913086,
      "activations/layer6_attention_weight_min": -50.39988327026367,
      "activations/layer7_attention_weight_max": 72.87487030029297,
      "activations/layer7_attention_weight_min": -69.5403823852539,
      "activations/layer8_attention_weight_max": 41.336971282958984,
      "activations/layer8_attention_weight_min": -45.40028762817383,
      "activations/layer9_attention_weight_max": 57.960445404052734,
      "activations/layer9_attention_weight_min": -47.805084228515625,
      "epoch": 10.63,
      "learning_rate": 8.229469696969696e-05,
      "loss": 2.7915,
      "step": 182900
    },
    {
      "activations/layer0_attention_weight_max": 14.530001640319824,
      "activations/layer0_attention_weight_min": -13.785652160644531,
      "activations/layer10_attention_weight_max": 38.133277893066406,
      "activations/layer10_attention_weight_min": -35.07422637939453,
      "activations/layer11_attention_weight_max": 31.717849731445312,
      "activations/layer11_attention_weight_min": -32.07559585571289,
      "activations/layer12_attention_weight_max": 19.068584442138672,
      "activations/layer12_attention_weight_min": -27.63007164001465,
      "activations/layer13_attention_weight_max": 37.743282318115234,
      "activations/layer13_attention_weight_min": -31.352746963500977,
      "activations/layer14_attention_weight_max": 36.255828857421875,
      "activations/layer14_attention_weight_min": -33.59043502807617,
      "activations/layer15_attention_weight_max": 31.940677642822266,
      "activations/layer15_attention_weight_min": -29.558292388916016,
      "activations/layer16_attention_weight_max": 32.693824768066406,
      "activations/layer16_attention_weight_min": -30.02122688293457,
      "activations/layer17_attention_weight_max": 53.06325149536133,
      "activations/layer17_attention_weight_min": -47.8144645690918,
      "activations/layer18_attention_weight_max": 49.50058364868164,
      "activations/layer18_attention_weight_min": -43.266178131103516,
      "activations/layer19_attention_weight_max": 21.945737838745117,
      "activations/layer19_attention_weight_min": -23.403547286987305,
      "activations/layer1_attention_weight_max": 16.95248794555664,
      "activations/layer1_attention_weight_min": -14.234819412231445,
      "activations/layer20_attention_weight_max": 23.43653106689453,
      "activations/layer20_attention_weight_min": -23.818267822265625,
      "activations/layer21_attention_weight_max": 38.29231643676758,
      "activations/layer21_attention_weight_min": -24.4732666015625,
      "activations/layer22_attention_weight_max": 31.72623634338379,
      "activations/layer22_attention_weight_min": -29.03255271911621,
      "activations/layer23_attention_weight_max": 37.20686721801758,
      "activations/layer23_attention_weight_min": -24.72124481201172,
      "activations/layer2_attention_weight_max": 35.158836364746094,
      "activations/layer2_attention_weight_min": -31.6004638671875,
      "activations/layer3_attention_weight_max": 88.5797119140625,
      "activations/layer3_attention_weight_min": -90.12281799316406,
      "activations/layer4_attention_weight_max": 96.2270736694336,
      "activations/layer4_attention_weight_min": -91.21047973632812,
      "activations/layer5_attention_weight_max": 73.66687774658203,
      "activations/layer5_attention_weight_min": -74.0954360961914,
      "activations/layer6_attention_weight_max": 52.56718444824219,
      "activations/layer6_attention_weight_min": -55.23493576049805,
      "activations/layer7_attention_weight_max": 77.35444641113281,
      "activations/layer7_attention_weight_min": -67.50677490234375,
      "activations/layer8_attention_weight_max": 46.79476547241211,
      "activations/layer8_attention_weight_min": -44.41246032714844,
      "activations/layer9_attention_weight_max": 46.46721649169922,
      "activations/layer9_attention_weight_min": -49.1838493347168,
      "epoch": 10.63,
      "learning_rate": 8.227575757575757e-05,
      "loss": 2.7953,
      "step": 182950
    },
    {
      "activations/layer0_attention_weight_max": 15.929422378540039,
      "activations/layer0_attention_weight_min": -13.716694831848145,
      "activations/layer10_attention_weight_max": 51.16845703125,
      "activations/layer10_attention_weight_min": -50.212242126464844,
      "activations/layer11_attention_weight_max": 45.859466552734375,
      "activations/layer11_attention_weight_min": -43.96905517578125,
      "activations/layer12_attention_weight_max": 22.11246109008789,
      "activations/layer12_attention_weight_min": -27.031251907348633,
      "activations/layer13_attention_weight_max": 64.0447998046875,
      "activations/layer13_attention_weight_min": -40.11555480957031,
      "activations/layer14_attention_weight_max": 77.09419250488281,
      "activations/layer14_attention_weight_min": -43.111785888671875,
      "activations/layer15_attention_weight_max": 61.276123046875,
      "activations/layer15_attention_weight_min": -38.489990234375,
      "activations/layer16_attention_weight_max": 41.588260650634766,
      "activations/layer16_attention_weight_min": -37.12540054321289,
      "activations/layer17_attention_weight_max": 67.18215942382812,
      "activations/layer17_attention_weight_min": -57.2360725402832,
      "activations/layer18_attention_weight_max": 59.35047149658203,
      "activations/layer18_attention_weight_min": -49.558074951171875,
      "activations/layer19_attention_weight_max": 31.1729793548584,
      "activations/layer19_attention_weight_min": -28.60029411315918,
      "activations/layer1_attention_weight_max": 17.184873580932617,
      "activations/layer1_attention_weight_min": -14.419891357421875,
      "activations/layer20_attention_weight_max": 33.68573760986328,
      "activations/layer20_attention_weight_min": -25.854490280151367,
      "activations/layer21_attention_weight_max": 69.96329498291016,
      "activations/layer21_attention_weight_min": -25.964494705200195,
      "activations/layer22_attention_weight_max": 39.78278350830078,
      "activations/layer22_attention_weight_min": -28.07465171813965,
      "activations/layer23_attention_weight_max": 35.528587341308594,
      "activations/layer23_attention_weight_min": -24.16719627380371,
      "activations/layer2_attention_weight_max": 36.50267028808594,
      "activations/layer2_attention_weight_min": -33.485774993896484,
      "activations/layer3_attention_weight_max": 108.8031005859375,
      "activations/layer3_attention_weight_min": -106.44420623779297,
      "activations/layer4_attention_weight_max": 114.76332092285156,
      "activations/layer4_attention_weight_min": -107.21150970458984,
      "activations/layer5_attention_weight_max": 82.10887145996094,
      "activations/layer5_attention_weight_min": -78.87138366699219,
      "activations/layer6_attention_weight_max": 59.881683349609375,
      "activations/layer6_attention_weight_min": -58.1998291015625,
      "activations/layer7_attention_weight_max": 88.27906036376953,
      "activations/layer7_attention_weight_min": -80.3392105102539,
      "activations/layer8_attention_weight_max": 56.5769157409668,
      "activations/layer8_attention_weight_min": -55.61213684082031,
      "activations/layer9_attention_weight_max": 68.33247375488281,
      "activations/layer9_attention_weight_min": -60.554134368896484,
      "epoch": 10.63,
      "learning_rate": 8.225681818181817e-05,
      "loss": 2.7693,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_loss": 2.73828125,
      "eval_runtime": 8.5251,
      "eval_samples_per_second": 503.688,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_openwebtext_loss": 2.73828125,
      "eval_openwebtext_ppl": 15.460389702669161,
      "eval_openwebtext_runtime": 8.5251,
      "eval_openwebtext_samples_per_second": 503.688,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 1.9765,
      "eval_wikitext_samples_per_second": 230.713,
      "step": 183000
    },
    {
      "epoch": 10.63,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.5493,
      "eval_lambada_samples_per_second": 509.879,
      "step": 183000
    },
    {
      "activations/layer0_attention_weight_max": 15.243387222290039,
      "activations/layer0_attention_weight_min": -13.990245819091797,
      "activations/layer10_attention_weight_max": 35.31550979614258,
      "activations/layer10_attention_weight_min": -35.2775764465332,
      "activations/layer11_attention_weight_max": 30.675046920776367,
      "activations/layer11_attention_weight_min": -32.80228805541992,
      "activations/layer12_attention_weight_max": 21.0147647857666,
      "activations/layer12_attention_weight_min": -24.197021484375,
      "activations/layer13_attention_weight_max": 40.20402145385742,
      "activations/layer13_attention_weight_min": -26.904216766357422,
      "activations/layer14_attention_weight_max": 34.17788314819336,
      "activations/layer14_attention_weight_min": -30.479103088378906,
      "activations/layer15_attention_weight_max": 31.7648868560791,
      "activations/layer15_attention_weight_min": -28.743772506713867,
      "activations/layer16_attention_weight_max": 29.950183868408203,
      "activations/layer16_attention_weight_min": -30.45245361328125,
      "activations/layer17_attention_weight_max": 54.17620086669922,
      "activations/layer17_attention_weight_min": -44.39856719970703,
      "activations/layer18_attention_weight_max": 48.47762680053711,
      "activations/layer18_attention_weight_min": -40.49791717529297,
      "activations/layer19_attention_weight_max": 22.569143295288086,
      "activations/layer19_attention_weight_min": -22.00211524963379,
      "activations/layer1_attention_weight_max": 16.70398712158203,
      "activations/layer1_attention_weight_min": -16.264768600463867,
      "activations/layer20_attention_weight_max": 23.852046966552734,
      "activations/layer20_attention_weight_min": -22.33760643005371,
      "activations/layer21_attention_weight_max": 43.25996780395508,
      "activations/layer21_attention_weight_min": -24.282474517822266,
      "activations/layer22_attention_weight_max": 35.55590057373047,
      "activations/layer22_attention_weight_min": -26.05437469482422,
      "activations/layer23_attention_weight_max": 42.320457458496094,
      "activations/layer23_attention_weight_min": -26.08121109008789,
      "activations/layer2_attention_weight_max": 34.23601531982422,
      "activations/layer2_attention_weight_min": -33.870567321777344,
      "activations/layer3_attention_weight_max": 96.17830657958984,
      "activations/layer3_attention_weight_min": -96.34708404541016,
      "activations/layer4_attention_weight_max": 110.34942626953125,
      "activations/layer4_attention_weight_min": -104.00435638427734,
      "activations/layer5_attention_weight_max": 80.1297607421875,
      "activations/layer5_attention_weight_min": -77.50961303710938,
      "activations/layer6_attention_weight_max": 55.191837310791016,
      "activations/layer6_attention_weight_min": -55.5947380065918,
      "activations/layer7_attention_weight_max": 74.52371215820312,
      "activations/layer7_attention_weight_min": -72.93502807617188,
      "activations/layer8_attention_weight_max": 46.28952407836914,
      "activations/layer8_attention_weight_min": -45.94069290161133,
      "activations/layer9_attention_weight_max": 54.90059280395508,
      "activations/layer9_attention_weight_min": -50.227596282958984,
      "epoch": 10.64,
      "learning_rate": 8.223787878787878e-05,
      "loss": 2.7786,
      "step": 183050
    },
    {
      "activations/layer0_attention_weight_max": 15.483030319213867,
      "activations/layer0_attention_weight_min": -13.797443389892578,
      "activations/layer10_attention_weight_max": 38.14202117919922,
      "activations/layer10_attention_weight_min": -36.93282699584961,
      "activations/layer11_attention_weight_max": 34.31200408935547,
      "activations/layer11_attention_weight_min": -34.220550537109375,
      "activations/layer12_attention_weight_max": 22.254718780517578,
      "activations/layer12_attention_weight_min": -32.05952072143555,
      "activations/layer13_attention_weight_max": 41.5678825378418,
      "activations/layer13_attention_weight_min": -29.73250961303711,
      "activations/layer14_attention_weight_max": 38.706851959228516,
      "activations/layer14_attention_weight_min": -31.83321762084961,
      "activations/layer15_attention_weight_max": 32.93088150024414,
      "activations/layer15_attention_weight_min": -30.194482803344727,
      "activations/layer16_attention_weight_max": 30.067134857177734,
      "activations/layer16_attention_weight_min": -28.44457244873047,
      "activations/layer17_attention_weight_max": 52.51167678833008,
      "activations/layer17_attention_weight_min": -44.95347213745117,
      "activations/layer18_attention_weight_max": 49.58852005004883,
      "activations/layer18_attention_weight_min": -39.34465789794922,
      "activations/layer19_attention_weight_max": 20.31576919555664,
      "activations/layer19_attention_weight_min": -21.495569229125977,
      "activations/layer1_attention_weight_max": 17.29192543029785,
      "activations/layer1_attention_weight_min": -14.012955665588379,
      "activations/layer20_attention_weight_max": 20.214067459106445,
      "activations/layer20_attention_weight_min": -19.712894439697266,
      "activations/layer21_attention_weight_max": 34.10379409790039,
      "activations/layer21_attention_weight_min": -20.507822036743164,
      "activations/layer22_attention_weight_max": 30.413110733032227,
      "activations/layer22_attention_weight_min": -24.369722366333008,
      "activations/layer23_attention_weight_max": 38.154945373535156,
      "activations/layer23_attention_weight_min": -22.581403732299805,
      "activations/layer2_attention_weight_max": 34.538787841796875,
      "activations/layer2_attention_weight_min": -32.892433166503906,
      "activations/layer3_attention_weight_max": 97.13322448730469,
      "activations/layer3_attention_weight_min": -99.602294921875,
      "activations/layer4_attention_weight_max": 103.86302947998047,
      "activations/layer4_attention_weight_min": -101.79041290283203,
      "activations/layer5_attention_weight_max": 74.7779312133789,
      "activations/layer5_attention_weight_min": -77.4654541015625,
      "activations/layer6_attention_weight_max": 53.68775177001953,
      "activations/layer6_attention_weight_min": -52.15480041503906,
      "activations/layer7_attention_weight_max": 80.436279296875,
      "activations/layer7_attention_weight_min": -78.50391387939453,
      "activations/layer8_attention_weight_max": 47.25253677368164,
      "activations/layer8_attention_weight_min": -48.23651123046875,
      "activations/layer9_attention_weight_max": 53.2744140625,
      "activations/layer9_attention_weight_min": -55.3853759765625,
      "epoch": 10.64,
      "learning_rate": 8.221893939393939e-05,
      "loss": 2.7799,
      "step": 183100
    },
    {
      "activations/layer0_attention_weight_max": 14.911367416381836,
      "activations/layer0_attention_weight_min": -13.53000545501709,
      "activations/layer10_attention_weight_max": 37.80531692504883,
      "activations/layer10_attention_weight_min": -37.26247787475586,
      "activations/layer11_attention_weight_max": 33.425743103027344,
      "activations/layer11_attention_weight_min": -34.81983947753906,
      "activations/layer12_attention_weight_max": 19.23413848876953,
      "activations/layer12_attention_weight_min": -23.481876373291016,
      "activations/layer13_attention_weight_max": 45.31989288330078,
      "activations/layer13_attention_weight_min": -33.05536651611328,
      "activations/layer14_attention_weight_max": 43.78450012207031,
      "activations/layer14_attention_weight_min": -30.31757164001465,
      "activations/layer15_attention_weight_max": 37.7961540222168,
      "activations/layer15_attention_weight_min": -30.60968780517578,
      "activations/layer16_attention_weight_max": 32.92973709106445,
      "activations/layer16_attention_weight_min": -31.093128204345703,
      "activations/layer17_attention_weight_max": 57.72206497192383,
      "activations/layer17_attention_weight_min": -47.08156967163086,
      "activations/layer18_attention_weight_max": 56.64636993408203,
      "activations/layer18_attention_weight_min": -38.48073196411133,
      "activations/layer19_attention_weight_max": 30.469131469726562,
      "activations/layer19_attention_weight_min": -24.050283432006836,
      "activations/layer1_attention_weight_max": 15.649735450744629,
      "activations/layer1_attention_weight_min": -15.309331893920898,
      "activations/layer20_attention_weight_max": 28.382572174072266,
      "activations/layer20_attention_weight_min": -20.519302368164062,
      "activations/layer21_attention_weight_max": 38.6489143371582,
      "activations/layer21_attention_weight_min": -22.299768447875977,
      "activations/layer22_attention_weight_max": 39.133331298828125,
      "activations/layer22_attention_weight_min": -24.4835147857666,
      "activations/layer23_attention_weight_max": 46.940826416015625,
      "activations/layer23_attention_weight_min": -25.6152400970459,
      "activations/layer2_attention_weight_max": 31.451656341552734,
      "activations/layer2_attention_weight_min": -31.592090606689453,
      "activations/layer3_attention_weight_max": 93.4204330444336,
      "activations/layer3_attention_weight_min": -95.70069122314453,
      "activations/layer4_attention_weight_max": 99.56021881103516,
      "activations/layer4_attention_weight_min": -97.16803741455078,
      "activations/layer5_attention_weight_max": 74.82429504394531,
      "activations/layer5_attention_weight_min": -74.71755981445312,
      "activations/layer6_attention_weight_max": 54.11088180541992,
      "activations/layer6_attention_weight_min": -52.79985809326172,
      "activations/layer7_attention_weight_max": 74.24535369873047,
      "activations/layer7_attention_weight_min": -73.3348159790039,
      "activations/layer8_attention_weight_max": 46.73931884765625,
      "activations/layer8_attention_weight_min": -49.009979248046875,
      "activations/layer9_attention_weight_max": 49.8841438293457,
      "activations/layer9_attention_weight_min": -49.909034729003906,
      "epoch": 10.64,
      "learning_rate": 8.22e-05,
      "loss": 2.7663,
      "step": 183150
    },
    {
      "activations/layer0_attention_weight_max": 15.742676734924316,
      "activations/layer0_attention_weight_min": -13.946179389953613,
      "activations/layer10_attention_weight_max": 40.00067138671875,
      "activations/layer10_attention_weight_min": -38.82684326171875,
      "activations/layer11_attention_weight_max": 36.37464904785156,
      "activations/layer11_attention_weight_min": -33.777061462402344,
      "activations/layer12_attention_weight_max": 17.70808982849121,
      "activations/layer12_attention_weight_min": -25.656517028808594,
      "activations/layer13_attention_weight_max": 44.28388595581055,
      "activations/layer13_attention_weight_min": -33.6864128112793,
      "activations/layer14_attention_weight_max": 38.757545471191406,
      "activations/layer14_attention_weight_min": -30.995914459228516,
      "activations/layer15_attention_weight_max": 33.093231201171875,
      "activations/layer15_attention_weight_min": -30.108108520507812,
      "activations/layer16_attention_weight_max": 29.832088470458984,
      "activations/layer16_attention_weight_min": -28.148345947265625,
      "activations/layer17_attention_weight_max": 51.08334732055664,
      "activations/layer17_attention_weight_min": -40.852142333984375,
      "activations/layer18_attention_weight_max": 43.78024673461914,
      "activations/layer18_attention_weight_min": -34.516395568847656,
      "activations/layer19_attention_weight_max": 21.609922409057617,
      "activations/layer19_attention_weight_min": -19.704965591430664,
      "activations/layer1_attention_weight_max": 16.26236915588379,
      "activations/layer1_attention_weight_min": -14.828227996826172,
      "activations/layer20_attention_weight_max": 19.847911834716797,
      "activations/layer20_attention_weight_min": -20.387075424194336,
      "activations/layer21_attention_weight_max": 32.37665557861328,
      "activations/layer21_attention_weight_min": -21.204973220825195,
      "activations/layer22_attention_weight_max": 30.09608268737793,
      "activations/layer22_attention_weight_min": -24.5981502532959,
      "activations/layer23_attention_weight_max": 36.307861328125,
      "activations/layer23_attention_weight_min": -25.816761016845703,
      "activations/layer2_attention_weight_max": 32.14643859863281,
      "activations/layer2_attention_weight_min": -31.55073356628418,
      "activations/layer3_attention_weight_max": 94.0216293334961,
      "activations/layer3_attention_weight_min": -101.29721069335938,
      "activations/layer4_attention_weight_max": 100.70014953613281,
      "activations/layer4_attention_weight_min": -96.51315307617188,
      "activations/layer5_attention_weight_max": 76.7342529296875,
      "activations/layer5_attention_weight_min": -80.75456237792969,
      "activations/layer6_attention_weight_max": 55.93247985839844,
      "activations/layer6_attention_weight_min": -55.92143630981445,
      "activations/layer7_attention_weight_max": 75.14685821533203,
      "activations/layer7_attention_weight_min": -78.21345520019531,
      "activations/layer8_attention_weight_max": 51.4891242980957,
      "activations/layer8_attention_weight_min": -53.04188919067383,
      "activations/layer9_attention_weight_max": 51.83003616333008,
      "activations/layer9_attention_weight_min": -51.250274658203125,
      "epoch": 10.64,
      "learning_rate": 8.21810606060606e-05,
      "loss": 2.7563,
      "step": 183200
    },
    {
      "activations/layer0_attention_weight_max": 16.000743865966797,
      "activations/layer0_attention_weight_min": -13.953787803649902,
      "activations/layer10_attention_weight_max": 37.120059967041016,
      "activations/layer10_attention_weight_min": -36.44768142700195,
      "activations/layer11_attention_weight_max": 33.82942199707031,
      "activations/layer11_attention_weight_min": -32.697933197021484,
      "activations/layer12_attention_weight_max": 28.059280395507812,
      "activations/layer12_attention_weight_min": -21.175317764282227,
      "activations/layer13_attention_weight_max": 41.885677337646484,
      "activations/layer13_attention_weight_min": -30.98269271850586,
      "activations/layer14_attention_weight_max": 40.309173583984375,
      "activations/layer14_attention_weight_min": -33.2411994934082,
      "activations/layer15_attention_weight_max": 37.8486328125,
      "activations/layer15_attention_weight_min": -31.590160369873047,
      "activations/layer16_attention_weight_max": 35.24102783203125,
      "activations/layer16_attention_weight_min": -31.44743537902832,
      "activations/layer17_attention_weight_max": 60.66122055053711,
      "activations/layer17_attention_weight_min": -47.41408157348633,
      "activations/layer18_attention_weight_max": 53.811641693115234,
      "activations/layer18_attention_weight_min": -38.281768798828125,
      "activations/layer19_attention_weight_max": 23.804073333740234,
      "activations/layer19_attention_weight_min": -21.891889572143555,
      "activations/layer1_attention_weight_max": 17.979339599609375,
      "activations/layer1_attention_weight_min": -13.964031219482422,
      "activations/layer20_attention_weight_max": 25.528352737426758,
      "activations/layer20_attention_weight_min": -20.951833724975586,
      "activations/layer21_attention_weight_max": 43.88401794433594,
      "activations/layer21_attention_weight_min": -23.195457458496094,
      "activations/layer22_attention_weight_max": 39.355560302734375,
      "activations/layer22_attention_weight_min": -27.547565460205078,
      "activations/layer23_attention_weight_max": 46.644676208496094,
      "activations/layer23_attention_weight_min": -25.924678802490234,
      "activations/layer2_attention_weight_max": 31.59343719482422,
      "activations/layer2_attention_weight_min": -32.066436767578125,
      "activations/layer3_attention_weight_max": 93.86640167236328,
      "activations/layer3_attention_weight_min": -97.38957977294922,
      "activations/layer4_attention_weight_max": 99.10033416748047,
      "activations/layer4_attention_weight_min": -97.41705322265625,
      "activations/layer5_attention_weight_max": 75.09855651855469,
      "activations/layer5_attention_weight_min": -78.96170043945312,
      "activations/layer6_attention_weight_max": 58.20216369628906,
      "activations/layer6_attention_weight_min": -54.3439826965332,
      "activations/layer7_attention_weight_max": 75.83460235595703,
      "activations/layer7_attention_weight_min": -72.32037353515625,
      "activations/layer8_attention_weight_max": 44.33949661254883,
      "activations/layer8_attention_weight_min": -46.665245056152344,
      "activations/layer9_attention_weight_max": 48.24481964111328,
      "activations/layer9_attention_weight_min": -49.05354309082031,
      "epoch": 10.65,
      "learning_rate": 8.216212121212121e-05,
      "loss": 2.773,
      "step": 183250
    },
    {
      "activations/layer0_attention_weight_max": 15.142210006713867,
      "activations/layer0_attention_weight_min": -13.895625114440918,
      "activations/layer10_attention_weight_max": 34.558204650878906,
      "activations/layer10_attention_weight_min": -34.342899322509766,
      "activations/layer11_attention_weight_max": 30.36603355407715,
      "activations/layer11_attention_weight_min": -32.293277740478516,
      "activations/layer12_attention_weight_max": 17.332292556762695,
      "activations/layer12_attention_weight_min": -23.79463005065918,
      "activations/layer13_attention_weight_max": 40.63452911376953,
      "activations/layer13_attention_weight_min": -32.61300277709961,
      "activations/layer14_attention_weight_max": 36.209877014160156,
      "activations/layer14_attention_weight_min": -31.356590270996094,
      "activations/layer15_attention_weight_max": 34.28300857543945,
      "activations/layer15_attention_weight_min": -29.657243728637695,
      "activations/layer16_attention_weight_max": 31.548250198364258,
      "activations/layer16_attention_weight_min": -30.911893844604492,
      "activations/layer17_attention_weight_max": 53.45069122314453,
      "activations/layer17_attention_weight_min": -45.78850555419922,
      "activations/layer18_attention_weight_max": 49.84803771972656,
      "activations/layer18_attention_weight_min": -39.815956115722656,
      "activations/layer19_attention_weight_max": 23.968788146972656,
      "activations/layer19_attention_weight_min": -24.070154190063477,
      "activations/layer1_attention_weight_max": 16.215129852294922,
      "activations/layer1_attention_weight_min": -16.04500961303711,
      "activations/layer20_attention_weight_max": 24.865787506103516,
      "activations/layer20_attention_weight_min": -21.54411506652832,
      "activations/layer21_attention_weight_max": 38.5943489074707,
      "activations/layer21_attention_weight_min": -23.708215713500977,
      "activations/layer22_attention_weight_max": 35.01569366455078,
      "activations/layer22_attention_weight_min": -25.576644897460938,
      "activations/layer23_attention_weight_max": 42.68354415893555,
      "activations/layer23_attention_weight_min": -23.737144470214844,
      "activations/layer2_attention_weight_max": 31.237335205078125,
      "activations/layer2_attention_weight_min": -31.616498947143555,
      "activations/layer3_attention_weight_max": 87.73104095458984,
      "activations/layer3_attention_weight_min": -91.93611907958984,
      "activations/layer4_attention_weight_max": 98.384033203125,
      "activations/layer4_attention_weight_min": -95.48656463623047,
      "activations/layer5_attention_weight_max": 73.29346466064453,
      "activations/layer5_attention_weight_min": -81.4400863647461,
      "activations/layer6_attention_weight_max": 54.617008209228516,
      "activations/layer6_attention_weight_min": -55.90620040893555,
      "activations/layer7_attention_weight_max": 75.53755187988281,
      "activations/layer7_attention_weight_min": -69.07499694824219,
      "activations/layer8_attention_weight_max": 43.6657600402832,
      "activations/layer8_attention_weight_min": -45.39631271362305,
      "activations/layer9_attention_weight_max": 46.88724899291992,
      "activations/layer9_attention_weight_min": -46.990875244140625,
      "epoch": 10.65,
      "learning_rate": 8.214318181818182e-05,
      "loss": 2.7583,
      "step": 183300
    },
    {
      "activations/layer0_attention_weight_max": 15.766847610473633,
      "activations/layer0_attention_weight_min": -13.81734561920166,
      "activations/layer10_attention_weight_max": 35.941162109375,
      "activations/layer10_attention_weight_min": -34.84780502319336,
      "activations/layer11_attention_weight_max": 33.363494873046875,
      "activations/layer11_attention_weight_min": -33.87635803222656,
      "activations/layer12_attention_weight_max": 18.07415008544922,
      "activations/layer12_attention_weight_min": -22.207048416137695,
      "activations/layer13_attention_weight_max": 40.94328689575195,
      "activations/layer13_attention_weight_min": -32.82421112060547,
      "activations/layer14_attention_weight_max": 38.32130432128906,
      "activations/layer14_attention_weight_min": -30.5982666015625,
      "activations/layer15_attention_weight_max": 35.2160758972168,
      "activations/layer15_attention_weight_min": -29.694887161254883,
      "activations/layer16_attention_weight_max": 31.9373779296875,
      "activations/layer16_attention_weight_min": -29.292957305908203,
      "activations/layer17_attention_weight_max": 52.7282600402832,
      "activations/layer17_attention_weight_min": -45.86688232421875,
      "activations/layer18_attention_weight_max": 45.80341339111328,
      "activations/layer18_attention_weight_min": -38.04351043701172,
      "activations/layer19_attention_weight_max": 23.26112174987793,
      "activations/layer19_attention_weight_min": -22.210386276245117,
      "activations/layer1_attention_weight_max": 17.221111297607422,
      "activations/layer1_attention_weight_min": -15.423222541809082,
      "activations/layer20_attention_weight_max": 22.159393310546875,
      "activations/layer20_attention_weight_min": -23.22536277770996,
      "activations/layer21_attention_weight_max": 43.92791748046875,
      "activations/layer21_attention_weight_min": -24.559001922607422,
      "activations/layer22_attention_weight_max": 31.62925148010254,
      "activations/layer22_attention_weight_min": -25.75597381591797,
      "activations/layer23_attention_weight_max": 37.130775451660156,
      "activations/layer23_attention_weight_min": -28.029386520385742,
      "activations/layer2_attention_weight_max": 32.20822525024414,
      "activations/layer2_attention_weight_min": -30.189590454101562,
      "activations/layer3_attention_weight_max": 93.14949798583984,
      "activations/layer3_attention_weight_min": -94.4704360961914,
      "activations/layer4_attention_weight_max": 98.20079803466797,
      "activations/layer4_attention_weight_min": -94.11333465576172,
      "activations/layer5_attention_weight_max": 77.98464965820312,
      "activations/layer5_attention_weight_min": -78.11797332763672,
      "activations/layer6_attention_weight_max": 56.20844268798828,
      "activations/layer6_attention_weight_min": -54.059940338134766,
      "activations/layer7_attention_weight_max": 77.8841323852539,
      "activations/layer7_attention_weight_min": -73.1274642944336,
      "activations/layer8_attention_weight_max": 46.03089141845703,
      "activations/layer8_attention_weight_min": -47.46686553955078,
      "activations/layer9_attention_weight_max": 47.37548828125,
      "activations/layer9_attention_weight_min": -51.053070068359375,
      "epoch": 10.65,
      "learning_rate": 8.212424242424241e-05,
      "loss": 2.7603,
      "step": 183350
    },
    {
      "activations/layer0_attention_weight_max": 14.52772331237793,
      "activations/layer0_attention_weight_min": -13.92646598815918,
      "activations/layer10_attention_weight_max": 37.593109130859375,
      "activations/layer10_attention_weight_min": -36.018089294433594,
      "activations/layer11_attention_weight_max": 33.08050537109375,
      "activations/layer11_attention_weight_min": -32.63377380371094,
      "activations/layer12_attention_weight_max": 19.129575729370117,
      "activations/layer12_attention_weight_min": -23.20785903930664,
      "activations/layer13_attention_weight_max": 39.94770431518555,
      "activations/layer13_attention_weight_min": -30.245073318481445,
      "activations/layer14_attention_weight_max": 37.300682067871094,
      "activations/layer14_attention_weight_min": -31.033222198486328,
      "activations/layer15_attention_weight_max": 34.551910400390625,
      "activations/layer15_attention_weight_min": -29.791452407836914,
      "activations/layer16_attention_weight_max": 33.32169723510742,
      "activations/layer16_attention_weight_min": -30.197160720825195,
      "activations/layer17_attention_weight_max": 55.73805236816406,
      "activations/layer17_attention_weight_min": -43.88782501220703,
      "activations/layer18_attention_weight_max": 47.849246978759766,
      "activations/layer18_attention_weight_min": -39.13163375854492,
      "activations/layer19_attention_weight_max": 24.19687271118164,
      "activations/layer19_attention_weight_min": -23.089109420776367,
      "activations/layer1_attention_weight_max": 16.377099990844727,
      "activations/layer1_attention_weight_min": -16.625694274902344,
      "activations/layer20_attention_weight_max": 26.765296936035156,
      "activations/layer20_attention_weight_min": -22.80548858642578,
      "activations/layer21_attention_weight_max": 45.129356384277344,
      "activations/layer21_attention_weight_min": -23.075275421142578,
      "activations/layer22_attention_weight_max": 35.54181671142578,
      "activations/layer22_attention_weight_min": -25.314321517944336,
      "activations/layer23_attention_weight_max": 40.33454132080078,
      "activations/layer23_attention_weight_min": -21.60272216796875,
      "activations/layer2_attention_weight_max": 32.041351318359375,
      "activations/layer2_attention_weight_min": -32.31035232543945,
      "activations/layer3_attention_weight_max": 91.31690979003906,
      "activations/layer3_attention_weight_min": -97.35986328125,
      "activations/layer4_attention_weight_max": 101.25384521484375,
      "activations/layer4_attention_weight_min": -99.91326141357422,
      "activations/layer5_attention_weight_max": 79.9002685546875,
      "activations/layer5_attention_weight_min": -79.86473083496094,
      "activations/layer6_attention_weight_max": 54.52940368652344,
      "activations/layer6_attention_weight_min": -56.82016372680664,
      "activations/layer7_attention_weight_max": 81.4969253540039,
      "activations/layer7_attention_weight_min": -74.07040405273438,
      "activations/layer8_attention_weight_max": 47.48561096191406,
      "activations/layer8_attention_weight_min": -49.87677001953125,
      "activations/layer9_attention_weight_max": 49.51127243041992,
      "activations/layer9_attention_weight_min": -50.934871673583984,
      "epoch": 10.66,
      "learning_rate": 8.210530303030303e-05,
      "loss": 2.7896,
      "step": 183400
    },
    {
      "activations/layer0_attention_weight_max": 14.791213989257812,
      "activations/layer0_attention_weight_min": -13.841354370117188,
      "activations/layer10_attention_weight_max": 36.811058044433594,
      "activations/layer10_attention_weight_min": -36.71784973144531,
      "activations/layer11_attention_weight_max": 32.608619689941406,
      "activations/layer11_attention_weight_min": -33.39250946044922,
      "activations/layer12_attention_weight_max": 17.5982608795166,
      "activations/layer12_attention_weight_min": -26.242692947387695,
      "activations/layer13_attention_weight_max": 32.41676712036133,
      "activations/layer13_attention_weight_min": -29.662385940551758,
      "activations/layer14_attention_weight_max": 36.69412612915039,
      "activations/layer14_attention_weight_min": -32.370182037353516,
      "activations/layer15_attention_weight_max": 32.94865036010742,
      "activations/layer15_attention_weight_min": -31.062467575073242,
      "activations/layer16_attention_weight_max": 31.45648956298828,
      "activations/layer16_attention_weight_min": -30.97174835205078,
      "activations/layer17_attention_weight_max": 56.415653228759766,
      "activations/layer17_attention_weight_min": -45.00844955444336,
      "activations/layer18_attention_weight_max": 49.11835861206055,
      "activations/layer18_attention_weight_min": -38.32486343383789,
      "activations/layer19_attention_weight_max": 22.053119659423828,
      "activations/layer19_attention_weight_min": -22.82492446899414,
      "activations/layer1_attention_weight_max": 16.8035888671875,
      "activations/layer1_attention_weight_min": -17.012739181518555,
      "activations/layer20_attention_weight_max": 22.57750701904297,
      "activations/layer20_attention_weight_min": -23.656394958496094,
      "activations/layer21_attention_weight_max": 34.32947540283203,
      "activations/layer21_attention_weight_min": -22.601003646850586,
      "activations/layer22_attention_weight_max": 28.608470916748047,
      "activations/layer22_attention_weight_min": -25.5136775970459,
      "activations/layer23_attention_weight_max": 37.01305389404297,
      "activations/layer23_attention_weight_min": -25.27396011352539,
      "activations/layer2_attention_weight_max": 33.20869445800781,
      "activations/layer2_attention_weight_min": -31.01983642578125,
      "activations/layer3_attention_weight_max": 92.60719299316406,
      "activations/layer3_attention_weight_min": -96.07135009765625,
      "activations/layer4_attention_weight_max": 101.61328125,
      "activations/layer4_attention_weight_min": -95.8913345336914,
      "activations/layer5_attention_weight_max": 76.4067611694336,
      "activations/layer5_attention_weight_min": -81.55374145507812,
      "activations/layer6_attention_weight_max": 54.516639709472656,
      "activations/layer6_attention_weight_min": -54.7653923034668,
      "activations/layer7_attention_weight_max": 74.22398376464844,
      "activations/layer7_attention_weight_min": -71.06897735595703,
      "activations/layer8_attention_weight_max": 47.16347885131836,
      "activations/layer8_attention_weight_min": -48.35549545288086,
      "activations/layer9_attention_weight_max": 51.420074462890625,
      "activations/layer9_attention_weight_min": -52.257057189941406,
      "epoch": 10.66,
      "learning_rate": 8.208636363636363e-05,
      "loss": 2.772,
      "step": 183450
    },
    {
      "activations/layer0_attention_weight_max": 15.973273277282715,
      "activations/layer0_attention_weight_min": -14.273508071899414,
      "activations/layer10_attention_weight_max": 36.577598571777344,
      "activations/layer10_attention_weight_min": -35.8826789855957,
      "activations/layer11_attention_weight_max": 31.398399353027344,
      "activations/layer11_attention_weight_min": -32.782493591308594,
      "activations/layer12_attention_weight_max": 21.060890197753906,
      "activations/layer12_attention_weight_min": -23.832870483398438,
      "activations/layer13_attention_weight_max": 36.27629470825195,
      "activations/layer13_attention_weight_min": -32.645660400390625,
      "activations/layer14_attention_weight_max": 36.76226043701172,
      "activations/layer14_attention_weight_min": -30.109352111816406,
      "activations/layer15_attention_weight_max": 32.674720764160156,
      "activations/layer15_attention_weight_min": -28.698129653930664,
      "activations/layer16_attention_weight_max": 31.127832412719727,
      "activations/layer16_attention_weight_min": -29.127593994140625,
      "activations/layer17_attention_weight_max": 50.66035079956055,
      "activations/layer17_attention_weight_min": -43.4908332824707,
      "activations/layer18_attention_weight_max": 44.45203399658203,
      "activations/layer18_attention_weight_min": -36.51326370239258,
      "activations/layer19_attention_weight_max": 22.739032745361328,
      "activations/layer19_attention_weight_min": -22.510112762451172,
      "activations/layer1_attention_weight_max": 18.2073917388916,
      "activations/layer1_attention_weight_min": -13.795906066894531,
      "activations/layer20_attention_weight_max": 22.464086532592773,
      "activations/layer20_attention_weight_min": -20.98150062561035,
      "activations/layer21_attention_weight_max": 37.56201171875,
      "activations/layer21_attention_weight_min": -22.99009132385254,
      "activations/layer22_attention_weight_max": 31.529266357421875,
      "activations/layer22_attention_weight_min": -25.592323303222656,
      "activations/layer23_attention_weight_max": 42.127540588378906,
      "activations/layer23_attention_weight_min": -26.460773468017578,
      "activations/layer2_attention_weight_max": 31.60029411315918,
      "activations/layer2_attention_weight_min": -31.21514892578125,
      "activations/layer3_attention_weight_max": 91.63194274902344,
      "activations/layer3_attention_weight_min": -94.24285125732422,
      "activations/layer4_attention_weight_max": 102.02053833007812,
      "activations/layer4_attention_weight_min": -97.33483123779297,
      "activations/layer5_attention_weight_max": 75.08407592773438,
      "activations/layer5_attention_weight_min": -78.49395751953125,
      "activations/layer6_attention_weight_max": 55.37977981567383,
      "activations/layer6_attention_weight_min": -56.55756759643555,
      "activations/layer7_attention_weight_max": 84.62821960449219,
      "activations/layer7_attention_weight_min": -71.71590423583984,
      "activations/layer8_attention_weight_max": 45.787986755371094,
      "activations/layer8_attention_weight_min": -48.88167953491211,
      "activations/layer9_attention_weight_max": 47.49434280395508,
      "activations/layer9_attention_weight_min": -48.41891860961914,
      "epoch": 10.66,
      "learning_rate": 8.206742424242423e-05,
      "loss": 2.7689,
      "step": 183500
    },
    {
      "activations/layer0_attention_weight_max": 15.855157852172852,
      "activations/layer0_attention_weight_min": -14.087016105651855,
      "activations/layer10_attention_weight_max": 41.322265625,
      "activations/layer10_attention_weight_min": -39.512176513671875,
      "activations/layer11_attention_weight_max": 35.29962158203125,
      "activations/layer11_attention_weight_min": -35.794471740722656,
      "activations/layer12_attention_weight_max": 18.760366439819336,
      "activations/layer12_attention_weight_min": -26.82632827758789,
      "activations/layer13_attention_weight_max": 46.82569122314453,
      "activations/layer13_attention_weight_min": -31.81031036376953,
      "activations/layer14_attention_weight_max": 45.97605514526367,
      "activations/layer14_attention_weight_min": -30.828210830688477,
      "activations/layer15_attention_weight_max": 37.79793167114258,
      "activations/layer15_attention_weight_min": -29.33997344970703,
      "activations/layer16_attention_weight_max": 33.25498580932617,
      "activations/layer16_attention_weight_min": -27.932104110717773,
      "activations/layer17_attention_weight_max": 52.9248046875,
      "activations/layer17_attention_weight_min": -46.751853942871094,
      "activations/layer18_attention_weight_max": 50.58304214477539,
      "activations/layer18_attention_weight_min": -40.10780715942383,
      "activations/layer19_attention_weight_max": 24.82469940185547,
      "activations/layer19_attention_weight_min": -23.317718505859375,
      "activations/layer1_attention_weight_max": 17.370136260986328,
      "activations/layer1_attention_weight_min": -14.20367431640625,
      "activations/layer20_attention_weight_max": 26.035621643066406,
      "activations/layer20_attention_weight_min": -21.199857711791992,
      "activations/layer21_attention_weight_max": 40.38514709472656,
      "activations/layer21_attention_weight_min": -21.81256866455078,
      "activations/layer22_attention_weight_max": 32.43178176879883,
      "activations/layer22_attention_weight_min": -25.239532470703125,
      "activations/layer23_attention_weight_max": 38.804046630859375,
      "activations/layer23_attention_weight_min": -25.430328369140625,
      "activations/layer2_attention_weight_max": 32.397789001464844,
      "activations/layer2_attention_weight_min": -31.590389251708984,
      "activations/layer3_attention_weight_max": 98.5920639038086,
      "activations/layer3_attention_weight_min": -101.94768524169922,
      "activations/layer4_attention_weight_max": 102.87163543701172,
      "activations/layer4_attention_weight_min": -101.14501190185547,
      "activations/layer5_attention_weight_max": 76.69223022460938,
      "activations/layer5_attention_weight_min": -78.20860290527344,
      "activations/layer6_attention_weight_max": 58.88443374633789,
      "activations/layer6_attention_weight_min": -58.09000778198242,
      "activations/layer7_attention_weight_max": 76.25276184082031,
      "activations/layer7_attention_weight_min": -82.34517669677734,
      "activations/layer8_attention_weight_max": 50.23094177246094,
      "activations/layer8_attention_weight_min": -50.83503341674805,
      "activations/layer9_attention_weight_max": 59.360286712646484,
      "activations/layer9_attention_weight_min": -55.381591796875,
      "epoch": 10.67,
      "learning_rate": 8.204848484848484e-05,
      "loss": 2.7768,
      "step": 183550
    },
    {
      "activations/layer0_attention_weight_max": 15.292838096618652,
      "activations/layer0_attention_weight_min": -13.723682403564453,
      "activations/layer10_attention_weight_max": 37.349334716796875,
      "activations/layer10_attention_weight_min": -35.95547103881836,
      "activations/layer11_attention_weight_max": 33.493919372558594,
      "activations/layer11_attention_weight_min": -32.18822479248047,
      "activations/layer12_attention_weight_max": 16.945724487304688,
      "activations/layer12_attention_weight_min": -24.165502548217773,
      "activations/layer13_attention_weight_max": 40.066978454589844,
      "activations/layer13_attention_weight_min": -31.477893829345703,
      "activations/layer14_attention_weight_max": 37.330360412597656,
      "activations/layer14_attention_weight_min": -29.91595458984375,
      "activations/layer15_attention_weight_max": 33.582969665527344,
      "activations/layer15_attention_weight_min": -29.270124435424805,
      "activations/layer16_attention_weight_max": 32.781795501708984,
      "activations/layer16_attention_weight_min": -30.742542266845703,
      "activations/layer17_attention_weight_max": 50.565799713134766,
      "activations/layer17_attention_weight_min": -43.51213836669922,
      "activations/layer18_attention_weight_max": 46.22312927246094,
      "activations/layer18_attention_weight_min": -37.536293029785156,
      "activations/layer19_attention_weight_max": 21.34895133972168,
      "activations/layer19_attention_weight_min": -21.382856369018555,
      "activations/layer1_attention_weight_max": 16.477806091308594,
      "activations/layer1_attention_weight_min": -15.517064094543457,
      "activations/layer20_attention_weight_max": 20.85521697998047,
      "activations/layer20_attention_weight_min": -21.759042739868164,
      "activations/layer21_attention_weight_max": 34.917320251464844,
      "activations/layer21_attention_weight_min": -21.799436569213867,
      "activations/layer22_attention_weight_max": 30.42930793762207,
      "activations/layer22_attention_weight_min": -24.28286361694336,
      "activations/layer23_attention_weight_max": 34.244300842285156,
      "activations/layer23_attention_weight_min": -23.644445419311523,
      "activations/layer2_attention_weight_max": 31.474075317382812,
      "activations/layer2_attention_weight_min": -30.363630294799805,
      "activations/layer3_attention_weight_max": 92.09127044677734,
      "activations/layer3_attention_weight_min": -91.64652252197266,
      "activations/layer4_attention_weight_max": 101.82219696044922,
      "activations/layer4_attention_weight_min": -94.96038818359375,
      "activations/layer5_attention_weight_max": 78.19062805175781,
      "activations/layer5_attention_weight_min": -78.73640441894531,
      "activations/layer6_attention_weight_max": 56.224525451660156,
      "activations/layer6_attention_weight_min": -56.550315856933594,
      "activations/layer7_attention_weight_max": 81.4114761352539,
      "activations/layer7_attention_weight_min": -74.35540771484375,
      "activations/layer8_attention_weight_max": 48.16324996948242,
      "activations/layer8_attention_weight_min": -46.6517333984375,
      "activations/layer9_attention_weight_max": 49.156246185302734,
      "activations/layer9_attention_weight_min": -50.23389434814453,
      "epoch": 10.67,
      "learning_rate": 8.202954545454545e-05,
      "loss": 2.7764,
      "step": 183600
    },
    {
      "activations/layer0_attention_weight_max": 15.026068687438965,
      "activations/layer0_attention_weight_min": -13.688590049743652,
      "activations/layer10_attention_weight_max": 38.017669677734375,
      "activations/layer10_attention_weight_min": -37.16899871826172,
      "activations/layer11_attention_weight_max": 31.717079162597656,
      "activations/layer11_attention_weight_min": -33.2542724609375,
      "activations/layer12_attention_weight_max": 18.211095809936523,
      "activations/layer12_attention_weight_min": -24.47796630859375,
      "activations/layer13_attention_weight_max": 38.973663330078125,
      "activations/layer13_attention_weight_min": -29.304210662841797,
      "activations/layer14_attention_weight_max": 37.31755447387695,
      "activations/layer14_attention_weight_min": -32.690589904785156,
      "activations/layer15_attention_weight_max": 33.75834274291992,
      "activations/layer15_attention_weight_min": -32.29787826538086,
      "activations/layer16_attention_weight_max": 31.118562698364258,
      "activations/layer16_attention_weight_min": -30.075342178344727,
      "activations/layer17_attention_weight_max": 57.65732955932617,
      "activations/layer17_attention_weight_min": -46.590721130371094,
      "activations/layer18_attention_weight_max": 51.50480651855469,
      "activations/layer18_attention_weight_min": -40.6249885559082,
      "activations/layer19_attention_weight_max": 23.046245574951172,
      "activations/layer19_attention_weight_min": -24.301733016967773,
      "activations/layer1_attention_weight_max": 16.156208038330078,
      "activations/layer1_attention_weight_min": -14.000941276550293,
      "activations/layer20_attention_weight_max": 23.968311309814453,
      "activations/layer20_attention_weight_min": -21.888248443603516,
      "activations/layer21_attention_weight_max": 38.481483459472656,
      "activations/layer21_attention_weight_min": -22.114789962768555,
      "activations/layer22_attention_weight_max": 28.973127365112305,
      "activations/layer22_attention_weight_min": -25.012170791625977,
      "activations/layer23_attention_weight_max": 38.48708724975586,
      "activations/layer23_attention_weight_min": -22.718807220458984,
      "activations/layer2_attention_weight_max": 32.89053726196289,
      "activations/layer2_attention_weight_min": -33.98917007446289,
      "activations/layer3_attention_weight_max": 94.1002426147461,
      "activations/layer3_attention_weight_min": -99.4886703491211,
      "activations/layer4_attention_weight_max": 105.06585693359375,
      "activations/layer4_attention_weight_min": -100.35865783691406,
      "activations/layer5_attention_weight_max": 79.1456298828125,
      "activations/layer5_attention_weight_min": -78.27094268798828,
      "activations/layer6_attention_weight_max": 55.153629302978516,
      "activations/layer6_attention_weight_min": -55.390357971191406,
      "activations/layer7_attention_weight_max": 76.059814453125,
      "activations/layer7_attention_weight_min": -74.25202178955078,
      "activations/layer8_attention_weight_max": 49.119956970214844,
      "activations/layer8_attention_weight_min": -52.02627182006836,
      "activations/layer9_attention_weight_max": 48.32596969604492,
      "activations/layer9_attention_weight_min": -50.005611419677734,
      "epoch": 10.67,
      "learning_rate": 8.201060606060605e-05,
      "loss": 2.7641,
      "step": 183650
    },
    {
      "activations/layer0_attention_weight_max": 15.888731002807617,
      "activations/layer0_attention_weight_min": -14.029812812805176,
      "activations/layer10_attention_weight_max": 37.20175552368164,
      "activations/layer10_attention_weight_min": -35.08125305175781,
      "activations/layer11_attention_weight_max": 32.43002700805664,
      "activations/layer11_attention_weight_min": -33.70474624633789,
      "activations/layer12_attention_weight_max": 18.94460678100586,
      "activations/layer12_attention_weight_min": -24.828170776367188,
      "activations/layer13_attention_weight_max": 37.262081146240234,
      "activations/layer13_attention_weight_min": -32.21078109741211,
      "activations/layer14_attention_weight_max": 37.07471466064453,
      "activations/layer14_attention_weight_min": -34.49036407470703,
      "activations/layer15_attention_weight_max": 35.325557708740234,
      "activations/layer15_attention_weight_min": -30.07249641418457,
      "activations/layer16_attention_weight_max": 35.552120208740234,
      "activations/layer16_attention_weight_min": -29.5508975982666,
      "activations/layer17_attention_weight_max": 58.73117446899414,
      "activations/layer17_attention_weight_min": -46.92255401611328,
      "activations/layer18_attention_weight_max": 52.234317779541016,
      "activations/layer18_attention_weight_min": -40.96207046508789,
      "activations/layer19_attention_weight_max": 23.46082878112793,
      "activations/layer19_attention_weight_min": -20.20335578918457,
      "activations/layer1_attention_weight_max": 16.370031356811523,
      "activations/layer1_attention_weight_min": -14.808091163635254,
      "activations/layer20_attention_weight_max": 25.008583068847656,
      "activations/layer20_attention_weight_min": -20.20444107055664,
      "activations/layer21_attention_weight_max": 36.98039627075195,
      "activations/layer21_attention_weight_min": -24.49567222595215,
      "activations/layer22_attention_weight_max": 36.753639221191406,
      "activations/layer22_attention_weight_min": -25.46703338623047,
      "activations/layer23_attention_weight_max": 41.17285919189453,
      "activations/layer23_attention_weight_min": -26.54944610595703,
      "activations/layer2_attention_weight_max": 31.65361785888672,
      "activations/layer2_attention_weight_min": -30.02077865600586,
      "activations/layer3_attention_weight_max": 92.4207763671875,
      "activations/layer3_attention_weight_min": -92.88182067871094,
      "activations/layer4_attention_weight_max": 99.63228607177734,
      "activations/layer4_attention_weight_min": -99.47755432128906,
      "activations/layer5_attention_weight_max": 76.36759948730469,
      "activations/layer5_attention_weight_min": -80.50502014160156,
      "activations/layer6_attention_weight_max": 54.540348052978516,
      "activations/layer6_attention_weight_min": -52.40445327758789,
      "activations/layer7_attention_weight_max": 76.35482025146484,
      "activations/layer7_attention_weight_min": -73.04248809814453,
      "activations/layer8_attention_weight_max": 48.48305130004883,
      "activations/layer8_attention_weight_min": -48.44402313232422,
      "activations/layer9_attention_weight_max": 48.99497604370117,
      "activations/layer9_attention_weight_min": -48.646942138671875,
      "epoch": 10.67,
      "learning_rate": 8.199166666666666e-05,
      "loss": 2.7768,
      "step": 183700
    },
    {
      "activations/layer0_attention_weight_max": 15.092486381530762,
      "activations/layer0_attention_weight_min": -13.686198234558105,
      "activations/layer10_attention_weight_max": 40.50360107421875,
      "activations/layer10_attention_weight_min": -37.909515380859375,
      "activations/layer11_attention_weight_max": 34.367950439453125,
      "activations/layer11_attention_weight_min": -33.896278381347656,
      "activations/layer12_attention_weight_max": 19.18902015686035,
      "activations/layer12_attention_weight_min": -26.30428695678711,
      "activations/layer13_attention_weight_max": 41.14479446411133,
      "activations/layer13_attention_weight_min": -30.642330169677734,
      "activations/layer14_attention_weight_max": 41.81134796142578,
      "activations/layer14_attention_weight_min": -31.963699340820312,
      "activations/layer15_attention_weight_max": 35.49546432495117,
      "activations/layer15_attention_weight_min": -30.500320434570312,
      "activations/layer16_attention_weight_max": 30.182933807373047,
      "activations/layer16_attention_weight_min": -29.40340232849121,
      "activations/layer17_attention_weight_max": 49.21269989013672,
      "activations/layer17_attention_weight_min": -43.14990234375,
      "activations/layer18_attention_weight_max": 47.06647872924805,
      "activations/layer18_attention_weight_min": -39.64030075073242,
      "activations/layer19_attention_weight_max": 22.116708755493164,
      "activations/layer19_attention_weight_min": -22.679298400878906,
      "activations/layer1_attention_weight_max": 16.444772720336914,
      "activations/layer1_attention_weight_min": -14.660173416137695,
      "activations/layer20_attention_weight_max": 23.381624221801758,
      "activations/layer20_attention_weight_min": -24.532697677612305,
      "activations/layer21_attention_weight_max": 40.81711196899414,
      "activations/layer21_attention_weight_min": -24.737340927124023,
      "activations/layer22_attention_weight_max": 30.40596580505371,
      "activations/layer22_attention_weight_min": -25.718719482421875,
      "activations/layer23_attention_weight_max": 36.26891326904297,
      "activations/layer23_attention_weight_min": -24.153104782104492,
      "activations/layer2_attention_weight_max": 32.22428894042969,
      "activations/layer2_attention_weight_min": -29.756816864013672,
      "activations/layer3_attention_weight_max": 92.96984100341797,
      "activations/layer3_attention_weight_min": -98.4379653930664,
      "activations/layer4_attention_weight_max": 103.92439270019531,
      "activations/layer4_attention_weight_min": -92.64767456054688,
      "activations/layer5_attention_weight_max": 80.67462921142578,
      "activations/layer5_attention_weight_min": -77.26014709472656,
      "activations/layer6_attention_weight_max": 58.96524429321289,
      "activations/layer6_attention_weight_min": -54.420345306396484,
      "activations/layer7_attention_weight_max": 81.33794403076172,
      "activations/layer7_attention_weight_min": -72.5488052368164,
      "activations/layer8_attention_weight_max": 51.126670837402344,
      "activations/layer8_attention_weight_min": -47.8706169128418,
      "activations/layer9_attention_weight_max": 55.37382888793945,
      "activations/layer9_attention_weight_min": -49.62094497680664,
      "epoch": 10.68,
      "learning_rate": 8.197272727272726e-05,
      "loss": 2.7618,
      "step": 183750
    },
    {
      "activations/layer0_attention_weight_max": 15.520859718322754,
      "activations/layer0_attention_weight_min": -14.216546058654785,
      "activations/layer10_attention_weight_max": 37.48537063598633,
      "activations/layer10_attention_weight_min": -36.6862678527832,
      "activations/layer11_attention_weight_max": 32.56693649291992,
      "activations/layer11_attention_weight_min": -34.07086944580078,
      "activations/layer12_attention_weight_max": 19.937435150146484,
      "activations/layer12_attention_weight_min": -27.754745483398438,
      "activations/layer13_attention_weight_max": 38.99139404296875,
      "activations/layer13_attention_weight_min": -28.29720687866211,
      "activations/layer14_attention_weight_max": 36.899505615234375,
      "activations/layer14_attention_weight_min": -29.577733993530273,
      "activations/layer15_attention_weight_max": 34.76161575317383,
      "activations/layer15_attention_weight_min": -29.632843017578125,
      "activations/layer16_attention_weight_max": 32.455509185791016,
      "activations/layer16_attention_weight_min": -29.032564163208008,
      "activations/layer17_attention_weight_max": 53.355003356933594,
      "activations/layer17_attention_weight_min": -42.424346923828125,
      "activations/layer18_attention_weight_max": 47.374088287353516,
      "activations/layer18_attention_weight_min": -37.47748565673828,
      "activations/layer19_attention_weight_max": 25.44533348083496,
      "activations/layer19_attention_weight_min": -21.363021850585938,
      "activations/layer1_attention_weight_max": 17.397443771362305,
      "activations/layer1_attention_weight_min": -14.685087203979492,
      "activations/layer20_attention_weight_max": 23.631389617919922,
      "activations/layer20_attention_weight_min": -22.169219970703125,
      "activations/layer21_attention_weight_max": 40.5115966796875,
      "activations/layer21_attention_weight_min": -22.677305221557617,
      "activations/layer22_attention_weight_max": 32.625362396240234,
      "activations/layer22_attention_weight_min": -25.27579689025879,
      "activations/layer23_attention_weight_max": 40.513946533203125,
      "activations/layer23_attention_weight_min": -25.733016967773438,
      "activations/layer2_attention_weight_max": 32.004669189453125,
      "activations/layer2_attention_weight_min": -31.73999786376953,
      "activations/layer3_attention_weight_max": 97.95618438720703,
      "activations/layer3_attention_weight_min": -101.12825012207031,
      "activations/layer4_attention_weight_max": 101.50635528564453,
      "activations/layer4_attention_weight_min": -97.00938415527344,
      "activations/layer5_attention_weight_max": 74.58589172363281,
      "activations/layer5_attention_weight_min": -81.83230590820312,
      "activations/layer6_attention_weight_max": 55.81895446777344,
      "activations/layer6_attention_weight_min": -54.91001892089844,
      "activations/layer7_attention_weight_max": 72.43934631347656,
      "activations/layer7_attention_weight_min": -72.6302490234375,
      "activations/layer8_attention_weight_max": 46.17171096801758,
      "activations/layer8_attention_weight_min": -48.12040328979492,
      "activations/layer9_attention_weight_max": 46.826316833496094,
      "activations/layer9_attention_weight_min": -48.987213134765625,
      "epoch": 10.68,
      "learning_rate": 8.195378787878788e-05,
      "loss": 2.7617,
      "step": 183800
    },
    {
      "activations/layer0_attention_weight_max": 15.789813995361328,
      "activations/layer0_attention_weight_min": -14.214353561401367,
      "activations/layer10_attention_weight_max": 37.42253875732422,
      "activations/layer10_attention_weight_min": -36.38959503173828,
      "activations/layer11_attention_weight_max": 32.90272521972656,
      "activations/layer11_attention_weight_min": -33.076576232910156,
      "activations/layer12_attention_weight_max": 18.129941940307617,
      "activations/layer12_attention_weight_min": -25.1480655670166,
      "activations/layer13_attention_weight_max": 37.15102005004883,
      "activations/layer13_attention_weight_min": -30.831850051879883,
      "activations/layer14_attention_weight_max": 43.2662239074707,
      "activations/layer14_attention_weight_min": -30.017316818237305,
      "activations/layer15_attention_weight_max": 35.538761138916016,
      "activations/layer15_attention_weight_min": -28.515806198120117,
      "activations/layer16_attention_weight_max": 31.87406349182129,
      "activations/layer16_attention_weight_min": -30.238819122314453,
      "activations/layer17_attention_weight_max": 55.4893684387207,
      "activations/layer17_attention_weight_min": -43.98752975463867,
      "activations/layer18_attention_weight_max": 50.35847091674805,
      "activations/layer18_attention_weight_min": -36.6020622253418,
      "activations/layer19_attention_weight_max": 22.419553756713867,
      "activations/layer19_attention_weight_min": -21.687040328979492,
      "activations/layer1_attention_weight_max": 16.780126571655273,
      "activations/layer1_attention_weight_min": -13.391156196594238,
      "activations/layer20_attention_weight_max": 22.17681884765625,
      "activations/layer20_attention_weight_min": -20.71709632873535,
      "activations/layer21_attention_weight_max": 35.860267639160156,
      "activations/layer21_attention_weight_min": -22.53232765197754,
      "activations/layer22_attention_weight_max": 32.445640563964844,
      "activations/layer22_attention_weight_min": -23.60930824279785,
      "activations/layer23_attention_weight_max": 39.72321319580078,
      "activations/layer23_attention_weight_min": -26.96548843383789,
      "activations/layer2_attention_weight_max": 31.060279846191406,
      "activations/layer2_attention_weight_min": -29.293102264404297,
      "activations/layer3_attention_weight_max": 91.55751037597656,
      "activations/layer3_attention_weight_min": -94.14469909667969,
      "activations/layer4_attention_weight_max": 101.15998077392578,
      "activations/layer4_attention_weight_min": -103.45818328857422,
      "activations/layer5_attention_weight_max": 75.48021697998047,
      "activations/layer5_attention_weight_min": -76.14584350585938,
      "activations/layer6_attention_weight_max": 54.848384857177734,
      "activations/layer6_attention_weight_min": -52.584415435791016,
      "activations/layer7_attention_weight_max": 74.6510009765625,
      "activations/layer7_attention_weight_min": -72.76809692382812,
      "activations/layer8_attention_weight_max": 44.457820892333984,
      "activations/layer8_attention_weight_min": -45.958614349365234,
      "activations/layer9_attention_weight_max": 49.23521041870117,
      "activations/layer9_attention_weight_min": -49.05466842651367,
      "epoch": 10.68,
      "learning_rate": 8.193484848484847e-05,
      "loss": 2.7673,
      "step": 183850
    },
    {
      "activations/layer0_attention_weight_max": 15.238945007324219,
      "activations/layer0_attention_weight_min": -14.50296688079834,
      "activations/layer10_attention_weight_max": 35.564117431640625,
      "activations/layer10_attention_weight_min": -36.203369140625,
      "activations/layer11_attention_weight_max": 30.33014678955078,
      "activations/layer11_attention_weight_min": -32.312042236328125,
      "activations/layer12_attention_weight_max": 17.06043815612793,
      "activations/layer12_attention_weight_min": -24.915348052978516,
      "activations/layer13_attention_weight_max": 53.09703826904297,
      "activations/layer13_attention_weight_min": -32.2336540222168,
      "activations/layer14_attention_weight_max": 48.197608947753906,
      "activations/layer14_attention_weight_min": -31.22844886779785,
      "activations/layer15_attention_weight_max": 35.763648986816406,
      "activations/layer15_attention_weight_min": -28.931734085083008,
      "activations/layer16_attention_weight_max": 32.90018081665039,
      "activations/layer16_attention_weight_min": -29.334787368774414,
      "activations/layer17_attention_weight_max": 50.555274963378906,
      "activations/layer17_attention_weight_min": -42.50200653076172,
      "activations/layer18_attention_weight_max": 44.83210754394531,
      "activations/layer18_attention_weight_min": -38.00322723388672,
      "activations/layer19_attention_weight_max": 20.105998992919922,
      "activations/layer19_attention_weight_min": -21.474336624145508,
      "activations/layer1_attention_weight_max": 17.30141258239746,
      "activations/layer1_attention_weight_min": -14.286458969116211,
      "activations/layer20_attention_weight_max": 21.827068328857422,
      "activations/layer20_attention_weight_min": -21.864839553833008,
      "activations/layer21_attention_weight_max": 38.9394416809082,
      "activations/layer21_attention_weight_min": -22.193344116210938,
      "activations/layer22_attention_weight_max": 29.25141143798828,
      "activations/layer22_attention_weight_min": -27.669540405273438,
      "activations/layer23_attention_weight_max": 36.079864501953125,
      "activations/layer23_attention_weight_min": -26.044233322143555,
      "activations/layer2_attention_weight_max": 32.2171630859375,
      "activations/layer2_attention_weight_min": -30.949413299560547,
      "activations/layer3_attention_weight_max": 91.3320083618164,
      "activations/layer3_attention_weight_min": -95.12713623046875,
      "activations/layer4_attention_weight_max": 97.83345794677734,
      "activations/layer4_attention_weight_min": -93.51695251464844,
      "activations/layer5_attention_weight_max": 71.27470397949219,
      "activations/layer5_attention_weight_min": -77.18429565429688,
      "activations/layer6_attention_weight_max": 53.14677429199219,
      "activations/layer6_attention_weight_min": -50.07167053222656,
      "activations/layer7_attention_weight_max": 75.975341796875,
      "activations/layer7_attention_weight_min": -67.7756118774414,
      "activations/layer8_attention_weight_max": 45.29694366455078,
      "activations/layer8_attention_weight_min": -42.72979736328125,
      "activations/layer9_attention_weight_max": 45.947200775146484,
      "activations/layer9_attention_weight_min": -47.08201599121094,
      "epoch": 10.69,
      "learning_rate": 8.191590909090908e-05,
      "loss": 2.7709,
      "step": 183900
    },
    {
      "activations/layer0_attention_weight_max": 15.19690990447998,
      "activations/layer0_attention_weight_min": -13.940136909484863,
      "activations/layer10_attention_weight_max": 39.61015319824219,
      "activations/layer10_attention_weight_min": -35.53936767578125,
      "activations/layer11_attention_weight_max": 37.80384826660156,
      "activations/layer11_attention_weight_min": -33.0810546875,
      "activations/layer12_attention_weight_max": 19.941495895385742,
      "activations/layer12_attention_weight_min": -28.941604614257812,
      "activations/layer13_attention_weight_max": 43.58049011230469,
      "activations/layer13_attention_weight_min": -30.335405349731445,
      "activations/layer14_attention_weight_max": 49.94205856323242,
      "activations/layer14_attention_weight_min": -30.21405029296875,
      "activations/layer15_attention_weight_max": 39.795555114746094,
      "activations/layer15_attention_weight_min": -29.650630950927734,
      "activations/layer16_attention_weight_max": 30.65453338623047,
      "activations/layer16_attention_weight_min": -29.024120330810547,
      "activations/layer17_attention_weight_max": 52.756019592285156,
      "activations/layer17_attention_weight_min": -43.896766662597656,
      "activations/layer18_attention_weight_max": 47.20305252075195,
      "activations/layer18_attention_weight_min": -37.35637283325195,
      "activations/layer19_attention_weight_max": 21.431806564331055,
      "activations/layer19_attention_weight_min": -21.422441482543945,
      "activations/layer1_attention_weight_max": 18.1680908203125,
      "activations/layer1_attention_weight_min": -14.129766464233398,
      "activations/layer20_attention_weight_max": 21.620771408081055,
      "activations/layer20_attention_weight_min": -19.80365753173828,
      "activations/layer21_attention_weight_max": 38.78194808959961,
      "activations/layer21_attention_weight_min": -23.13816261291504,
      "activations/layer22_attention_weight_max": 28.7633056640625,
      "activations/layer22_attention_weight_min": -24.376680374145508,
      "activations/layer23_attention_weight_max": 37.1330680847168,
      "activations/layer23_attention_weight_min": -21.57445526123047,
      "activations/layer2_attention_weight_max": 34.33140563964844,
      "activations/layer2_attention_weight_min": -32.685638427734375,
      "activations/layer3_attention_weight_max": 95.12183380126953,
      "activations/layer3_attention_weight_min": -97.40509033203125,
      "activations/layer4_attention_weight_max": 103.70954895019531,
      "activations/layer4_attention_weight_min": -97.88933563232422,
      "activations/layer5_attention_weight_max": 75.00196075439453,
      "activations/layer5_attention_weight_min": -78.24075317382812,
      "activations/layer6_attention_weight_max": 54.0236701965332,
      "activations/layer6_attention_weight_min": -53.85841369628906,
      "activations/layer7_attention_weight_max": 81.30429077148438,
      "activations/layer7_attention_weight_min": -69.48176574707031,
      "activations/layer8_attention_weight_max": 47.096534729003906,
      "activations/layer8_attention_weight_min": -46.40409469604492,
      "activations/layer9_attention_weight_max": 58.643959045410156,
      "activations/layer9_attention_weight_min": -52.10036849975586,
      "epoch": 10.69,
      "learning_rate": 8.18969696969697e-05,
      "loss": 2.7748,
      "step": 183950
    },
    {
      "activations/layer0_attention_weight_max": 15.408727645874023,
      "activations/layer0_attention_weight_min": -14.332159996032715,
      "activations/layer10_attention_weight_max": 40.04136276245117,
      "activations/layer10_attention_weight_min": -39.30060577392578,
      "activations/layer11_attention_weight_max": 35.207435607910156,
      "activations/layer11_attention_weight_min": -33.32891845703125,
      "activations/layer12_attention_weight_max": 18.838977813720703,
      "activations/layer12_attention_weight_min": -25.16362953186035,
      "activations/layer13_attention_weight_max": 37.68846893310547,
      "activations/layer13_attention_weight_min": -30.790721893310547,
      "activations/layer14_attention_weight_max": 33.7769889831543,
      "activations/layer14_attention_weight_min": -29.439210891723633,
      "activations/layer15_attention_weight_max": 30.799007415771484,
      "activations/layer15_attention_weight_min": -28.537734985351562,
      "activations/layer16_attention_weight_max": 29.15092658996582,
      "activations/layer16_attention_weight_min": -28.507051467895508,
      "activations/layer17_attention_weight_max": 48.421417236328125,
      "activations/layer17_attention_weight_min": -41.39516067504883,
      "activations/layer18_attention_weight_max": 44.68773651123047,
      "activations/layer18_attention_weight_min": -35.651824951171875,
      "activations/layer19_attention_weight_max": 21.069265365600586,
      "activations/layer19_attention_weight_min": -22.37979507446289,
      "activations/layer1_attention_weight_max": 16.86859130859375,
      "activations/layer1_attention_weight_min": -15.659252166748047,
      "activations/layer20_attention_weight_max": 23.06774139404297,
      "activations/layer20_attention_weight_min": -21.19496726989746,
      "activations/layer21_attention_weight_max": 34.708553314208984,
      "activations/layer21_attention_weight_min": -21.975683212280273,
      "activations/layer22_attention_weight_max": 29.459924697875977,
      "activations/layer22_attention_weight_min": -24.84507942199707,
      "activations/layer23_attention_weight_max": 38.035057067871094,
      "activations/layer23_attention_weight_min": -24.9664363861084,
      "activations/layer2_attention_weight_max": 31.98412322998047,
      "activations/layer2_attention_weight_min": -32.43077087402344,
      "activations/layer3_attention_weight_max": 97.47533416748047,
      "activations/layer3_attention_weight_min": -102.94173431396484,
      "activations/layer4_attention_weight_max": 102.90813446044922,
      "activations/layer4_attention_weight_min": -99.3646011352539,
      "activations/layer5_attention_weight_max": 75.901123046875,
      "activations/layer5_attention_weight_min": -78.35309600830078,
      "activations/layer6_attention_weight_max": 55.70618438720703,
      "activations/layer6_attention_weight_min": -52.92586135864258,
      "activations/layer7_attention_weight_max": 72.45879364013672,
      "activations/layer7_attention_weight_min": -67.35474395751953,
      "activations/layer8_attention_weight_max": 44.96516418457031,
      "activations/layer8_attention_weight_min": -51.47245788574219,
      "activations/layer9_attention_weight_max": 47.619415283203125,
      "activations/layer9_attention_weight_min": -49.29728317260742,
      "epoch": 10.69,
      "learning_rate": 8.187840909090909e-05,
      "loss": 2.7798,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.4973,
      "eval_samples_per_second": 505.337,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.4973,
      "eval_openwebtext_samples_per_second": 505.337,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_wikitext_loss": 2.966796875,
      "eval_wikitext_ppl": 19.429584428039178,
      "eval_wikitext_runtime": 1.9858,
      "eval_wikitext_samples_per_second": 229.629,
      "step": 184000
    },
    {
      "epoch": 10.69,
      "eval_lambada_loss": 2.55859375,
      "eval_lambada_ppl": 12.917639106991658,
      "eval_lambada_runtime": 9.6058,
      "eval_lambada_samples_per_second": 506.883,
      "step": 184000
    },
    {
      "activations/layer0_attention_weight_max": 15.010479927062988,
      "activations/layer0_attention_weight_min": -14.418417930603027,
      "activations/layer10_attention_weight_max": 39.89612579345703,
      "activations/layer10_attention_weight_min": -39.72712326049805,
      "activations/layer11_attention_weight_max": 36.79866409301758,
      "activations/layer11_attention_weight_min": -35.23467254638672,
      "activations/layer12_attention_weight_max": 21.44784164428711,
      "activations/layer12_attention_weight_min": -24.741655349731445,
      "activations/layer13_attention_weight_max": 44.80171585083008,
      "activations/layer13_attention_weight_min": -33.4212760925293,
      "activations/layer14_attention_weight_max": 41.779685974121094,
      "activations/layer14_attention_weight_min": -32.45854568481445,
      "activations/layer15_attention_weight_max": 41.799076080322266,
      "activations/layer15_attention_weight_min": -30.583248138427734,
      "activations/layer16_attention_weight_max": 36.15313720703125,
      "activations/layer16_attention_weight_min": -30.68787384033203,
      "activations/layer17_attention_weight_max": 58.64784240722656,
      "activations/layer17_attention_weight_min": -48.86527633666992,
      "activations/layer18_attention_weight_max": 53.7047233581543,
      "activations/layer18_attention_weight_min": -40.852760314941406,
      "activations/layer19_attention_weight_max": 24.014860153198242,
      "activations/layer19_attention_weight_min": -22.08331871032715,
      "activations/layer1_attention_weight_max": 16.4736270904541,
      "activations/layer1_attention_weight_min": -14.052517890930176,
      "activations/layer20_attention_weight_max": 26.1768741607666,
      "activations/layer20_attention_weight_min": -19.716577529907227,
      "activations/layer21_attention_weight_max": 38.270301818847656,
      "activations/layer21_attention_weight_min": -23.785207748413086,
      "activations/layer22_attention_weight_max": 34.18973922729492,
      "activations/layer22_attention_weight_min": -27.163394927978516,
      "activations/layer23_attention_weight_max": 37.91032028198242,
      "activations/layer23_attention_weight_min": -23.86751937866211,
      "activations/layer2_attention_weight_max": 32.08864974975586,
      "activations/layer2_attention_weight_min": -30.541059494018555,
      "activations/layer3_attention_weight_max": 95.82250213623047,
      "activations/layer3_attention_weight_min": -93.51978302001953,
      "activations/layer4_attention_weight_max": 106.16936492919922,
      "activations/layer4_attention_weight_min": -101.77239227294922,
      "activations/layer5_attention_weight_max": 80.10646057128906,
      "activations/layer5_attention_weight_min": -84.28546905517578,
      "activations/layer6_attention_weight_max": 59.77792739868164,
      "activations/layer6_attention_weight_min": -57.46817398071289,
      "activations/layer7_attention_weight_max": 76.9270248413086,
      "activations/layer7_attention_weight_min": -74.44786071777344,
      "activations/layer8_attention_weight_max": 51.53865432739258,
      "activations/layer8_attention_weight_min": -53.43484878540039,
      "activations/layer9_attention_weight_max": 53.12185287475586,
      "activations/layer9_attention_weight_min": -51.21055221557617,
      "epoch": 10.69,
      "learning_rate": 8.185946969696968e-05,
      "loss": 2.7484,
      "step": 184050
    },
    {
      "activations/layer0_attention_weight_max": 15.71780776977539,
      "activations/layer0_attention_weight_min": -14.047184944152832,
      "activations/layer10_attention_weight_max": 36.421669006347656,
      "activations/layer10_attention_weight_min": -38.54878234863281,
      "activations/layer11_attention_weight_max": 32.00511169433594,
      "activations/layer11_attention_weight_min": -33.857173919677734,
      "activations/layer12_attention_weight_max": 18.738431930541992,
      "activations/layer12_attention_weight_min": -35.01011657714844,
      "activations/layer13_attention_weight_max": 39.92289352416992,
      "activations/layer13_attention_weight_min": -30.191574096679688,
      "activations/layer14_attention_weight_max": 41.76054763793945,
      "activations/layer14_attention_weight_min": -32.37335968017578,
      "activations/layer15_attention_weight_max": 34.628379821777344,
      "activations/layer15_attention_weight_min": -31.212299346923828,
      "activations/layer16_attention_weight_max": 32.85964584350586,
      "activations/layer16_attention_weight_min": -31.64544677734375,
      "activations/layer17_attention_weight_max": 55.267112731933594,
      "activations/layer17_attention_weight_min": -47.20446014404297,
      "activations/layer18_attention_weight_max": 49.267295837402344,
      "activations/layer18_attention_weight_min": -39.25067138671875,
      "activations/layer19_attention_weight_max": 27.229162216186523,
      "activations/layer19_attention_weight_min": -22.52985382080078,
      "activations/layer1_attention_weight_max": 17.409347534179688,
      "activations/layer1_attention_weight_min": -16.70168685913086,
      "activations/layer20_attention_weight_max": 25.80218505859375,
      "activations/layer20_attention_weight_min": -22.1248836517334,
      "activations/layer21_attention_weight_max": 46.92128372192383,
      "activations/layer21_attention_weight_min": -22.985170364379883,
      "activations/layer22_attention_weight_max": 33.570556640625,
      "activations/layer22_attention_weight_min": -25.182113647460938,
      "activations/layer23_attention_weight_max": 40.72978973388672,
      "activations/layer23_attention_weight_min": -22.675479888916016,
      "activations/layer2_attention_weight_max": 32.7708740234375,
      "activations/layer2_attention_weight_min": -32.34761047363281,
      "activations/layer3_attention_weight_max": 94.76244354248047,
      "activations/layer3_attention_weight_min": -93.67088317871094,
      "activations/layer4_attention_weight_max": 104.8099365234375,
      "activations/layer4_attention_weight_min": -99.27935028076172,
      "activations/layer5_attention_weight_max": 79.3272933959961,
      "activations/layer5_attention_weight_min": -81.39175415039062,
      "activations/layer6_attention_weight_max": 58.518768310546875,
      "activations/layer6_attention_weight_min": -53.982872009277344,
      "activations/layer7_attention_weight_max": 80.30085754394531,
      "activations/layer7_attention_weight_min": -74.92823028564453,
      "activations/layer8_attention_weight_max": 48.35322189331055,
      "activations/layer8_attention_weight_min": -50.1612663269043,
      "activations/layer9_attention_weight_max": 52.94281005859375,
      "activations/layer9_attention_weight_min": -55.18758773803711,
      "epoch": 10.7,
      "learning_rate": 8.184053030303029e-05,
      "loss": 2.788,
      "step": 184100
    },
    {
      "activations/layer0_attention_weight_max": 16.750978469848633,
      "activations/layer0_attention_weight_min": -14.395377159118652,
      "activations/layer10_attention_weight_max": 36.3074951171875,
      "activations/layer10_attention_weight_min": -34.57356643676758,
      "activations/layer11_attention_weight_max": 30.436939239501953,
      "activations/layer11_attention_weight_min": -32.638145446777344,
      "activations/layer12_attention_weight_max": 19.81100082397461,
      "activations/layer12_attention_weight_min": -23.429813385009766,
      "activations/layer13_attention_weight_max": 40.338714599609375,
      "activations/layer13_attention_weight_min": -30.18214988708496,
      "activations/layer14_attention_weight_max": 38.288028717041016,
      "activations/layer14_attention_weight_min": -31.03592872619629,
      "activations/layer15_attention_weight_max": 35.7365837097168,
      "activations/layer15_attention_weight_min": -30.672449111938477,
      "activations/layer16_attention_weight_max": 34.19731140136719,
      "activations/layer16_attention_weight_min": -30.844287872314453,
      "activations/layer17_attention_weight_max": 54.01726150512695,
      "activations/layer17_attention_weight_min": -47.32857894897461,
      "activations/layer18_attention_weight_max": 48.19562911987305,
      "activations/layer18_attention_weight_min": -38.79180908203125,
      "activations/layer19_attention_weight_max": 23.183902740478516,
      "activations/layer19_attention_weight_min": -21.564611434936523,
      "activations/layer1_attention_weight_max": 15.946698188781738,
      "activations/layer1_attention_weight_min": -15.133811950683594,
      "activations/layer20_attention_weight_max": 22.730398178100586,
      "activations/layer20_attention_weight_min": -24.198078155517578,
      "activations/layer21_attention_weight_max": 35.542877197265625,
      "activations/layer21_attention_weight_min": -21.278297424316406,
      "activations/layer22_attention_weight_max": 33.53485870361328,
      "activations/layer22_attention_weight_min": -25.55632781982422,
      "activations/layer23_attention_weight_max": 35.62852478027344,
      "activations/layer23_attention_weight_min": -25.012107849121094,
      "activations/layer2_attention_weight_max": 31.795568466186523,
      "activations/layer2_attention_weight_min": -29.7814998626709,
      "activations/layer3_attention_weight_max": 88.24800109863281,
      "activations/layer3_attention_weight_min": -94.16233825683594,
      "activations/layer4_attention_weight_max": 99.93145751953125,
      "activations/layer4_attention_weight_min": -94.9814224243164,
      "activations/layer5_attention_weight_max": 76.0993423461914,
      "activations/layer5_attention_weight_min": -75.51936340332031,
      "activations/layer6_attention_weight_max": 55.09297561645508,
      "activations/layer6_attention_weight_min": -54.592247009277344,
      "activations/layer7_attention_weight_max": 70.64299011230469,
      "activations/layer7_attention_weight_min": -69.4111099243164,
      "activations/layer8_attention_weight_max": 42.98408889770508,
      "activations/layer8_attention_weight_min": -44.798831939697266,
      "activations/layer9_attention_weight_max": 48.927589416503906,
      "activations/layer9_attention_weight_min": -47.3629150390625,
      "epoch": 10.7,
      "learning_rate": 8.182196969696968e-05,
      "loss": 2.7786,
      "step": 184150
    },
    {
      "activations/layer0_attention_weight_max": 16.019086837768555,
      "activations/layer0_attention_weight_min": -14.744904518127441,
      "activations/layer10_attention_weight_max": 39.18709182739258,
      "activations/layer10_attention_weight_min": -38.07383728027344,
      "activations/layer11_attention_weight_max": 35.28661346435547,
      "activations/layer11_attention_weight_min": -33.26190948486328,
      "activations/layer12_attention_weight_max": 18.09796905517578,
      "activations/layer12_attention_weight_min": -25.04261016845703,
      "activations/layer13_attention_weight_max": 38.52864456176758,
      "activations/layer13_attention_weight_min": -30.53641700744629,
      "activations/layer14_attention_weight_max": 40.4007682800293,
      "activations/layer14_attention_weight_min": -30.89745330810547,
      "activations/layer15_attention_weight_max": 33.789302825927734,
      "activations/layer15_attention_weight_min": -29.25305938720703,
      "activations/layer16_attention_weight_max": 31.043529510498047,
      "activations/layer16_attention_weight_min": -29.100906372070312,
      "activations/layer17_attention_weight_max": 50.027915954589844,
      "activations/layer17_attention_weight_min": -45.09293746948242,
      "activations/layer18_attention_weight_max": 45.21257400512695,
      "activations/layer18_attention_weight_min": -41.182044982910156,
      "activations/layer19_attention_weight_max": 22.061033248901367,
      "activations/layer19_attention_weight_min": -23.448230743408203,
      "activations/layer1_attention_weight_max": 16.643369674682617,
      "activations/layer1_attention_weight_min": -15.217598915100098,
      "activations/layer20_attention_weight_max": 22.21715545654297,
      "activations/layer20_attention_weight_min": -22.783676147460938,
      "activations/layer21_attention_weight_max": 31.830913543701172,
      "activations/layer21_attention_weight_min": -22.076095581054688,
      "activations/layer22_attention_weight_max": 30.56922149658203,
      "activations/layer22_attention_weight_min": -27.720170974731445,
      "activations/layer23_attention_weight_max": 39.347381591796875,
      "activations/layer23_attention_weight_min": -22.83607292175293,
      "activations/layer2_attention_weight_max": 34.20667266845703,
      "activations/layer2_attention_weight_min": -33.67816925048828,
      "activations/layer3_attention_weight_max": 93.7591552734375,
      "activations/layer3_attention_weight_min": -94.69344329833984,
      "activations/layer4_attention_weight_max": 106.84667205810547,
      "activations/layer4_attention_weight_min": -100.67454528808594,
      "activations/layer5_attention_weight_max": 78.60224914550781,
      "activations/layer5_attention_weight_min": -82.69134521484375,
      "activations/layer6_attention_weight_max": 56.384178161621094,
      "activations/layer6_attention_weight_min": -54.63111114501953,
      "activations/layer7_attention_weight_max": 78.56845092773438,
      "activations/layer7_attention_weight_min": -74.19049835205078,
      "activations/layer8_attention_weight_max": 49.21895217895508,
      "activations/layer8_attention_weight_min": -49.69649887084961,
      "activations/layer9_attention_weight_max": 56.11323165893555,
      "activations/layer9_attention_weight_min": -48.913169860839844,
      "epoch": 10.7,
      "learning_rate": 8.18030303030303e-05,
      "loss": 2.7741,
      "step": 184200
    },
    {
      "activations/layer0_attention_weight_max": 15.58702564239502,
      "activations/layer0_attention_weight_min": -14.052556991577148,
      "activations/layer10_attention_weight_max": 35.67290115356445,
      "activations/layer10_attention_weight_min": -35.2913818359375,
      "activations/layer11_attention_weight_max": 31.89080810546875,
      "activations/layer11_attention_weight_min": -31.309123992919922,
      "activations/layer12_attention_weight_max": 27.7359561920166,
      "activations/layer12_attention_weight_min": -28.622272491455078,
      "activations/layer13_attention_weight_max": 37.29738235473633,
      "activations/layer13_attention_weight_min": -27.35175323486328,
      "activations/layer14_attention_weight_max": 36.836307525634766,
      "activations/layer14_attention_weight_min": -28.562440872192383,
      "activations/layer15_attention_weight_max": 34.121395111083984,
      "activations/layer15_attention_weight_min": -29.856090545654297,
      "activations/layer16_attention_weight_max": 32.47080993652344,
      "activations/layer16_attention_weight_min": -28.276687622070312,
      "activations/layer17_attention_weight_max": 52.889183044433594,
      "activations/layer17_attention_weight_min": -45.94728469848633,
      "activations/layer18_attention_weight_max": 51.13414001464844,
      "activations/layer18_attention_weight_min": -36.73943328857422,
      "activations/layer19_attention_weight_max": 22.937705993652344,
      "activations/layer19_attention_weight_min": -21.974573135375977,
      "activations/layer1_attention_weight_max": 16.739543914794922,
      "activations/layer1_attention_weight_min": -15.199143409729004,
      "activations/layer20_attention_weight_max": 23.638561248779297,
      "activations/layer20_attention_weight_min": -23.541271209716797,
      "activations/layer21_attention_weight_max": 42.50223922729492,
      "activations/layer21_attention_weight_min": -22.459457397460938,
      "activations/layer22_attention_weight_max": 31.96380615234375,
      "activations/layer22_attention_weight_min": -25.09218978881836,
      "activations/layer23_attention_weight_max": 38.40529251098633,
      "activations/layer23_attention_weight_min": -22.599559783935547,
      "activations/layer2_attention_weight_max": 32.47168731689453,
      "activations/layer2_attention_weight_min": -32.01454162597656,
      "activations/layer3_attention_weight_max": 90.87718200683594,
      "activations/layer3_attention_weight_min": -92.89607238769531,
      "activations/layer4_attention_weight_max": 100.7468490600586,
      "activations/layer4_attention_weight_min": -96.8479232788086,
      "activations/layer5_attention_weight_max": 73.5904312133789,
      "activations/layer5_attention_weight_min": -76.2310562133789,
      "activations/layer6_attention_weight_max": 54.38076400756836,
      "activations/layer6_attention_weight_min": -51.41963577270508,
      "activations/layer7_attention_weight_max": 75.0667724609375,
      "activations/layer7_attention_weight_min": -71.85192108154297,
      "activations/layer8_attention_weight_max": 45.40213394165039,
      "activations/layer8_attention_weight_min": -45.502506256103516,
      "activations/layer9_attention_weight_max": 48.62644577026367,
      "activations/layer9_attention_weight_min": -44.526161193847656,
      "epoch": 10.71,
      "learning_rate": 8.178409090909091e-05,
      "loss": 2.7641,
      "step": 184250
    },
    {
      "activations/layer0_attention_weight_max": 16.935943603515625,
      "activations/layer0_attention_weight_min": -14.171123504638672,
      "activations/layer10_attention_weight_max": 34.44929885864258,
      "activations/layer10_attention_weight_min": -33.755096435546875,
      "activations/layer11_attention_weight_max": 30.452842712402344,
      "activations/layer11_attention_weight_min": -30.355213165283203,
      "activations/layer12_attention_weight_max": 20.690927505493164,
      "activations/layer12_attention_weight_min": -23.648868560791016,
      "activations/layer13_attention_weight_max": 38.707244873046875,
      "activations/layer13_attention_weight_min": -32.67230987548828,
      "activations/layer14_attention_weight_max": 36.26531982421875,
      "activations/layer14_attention_weight_min": -30.462160110473633,
      "activations/layer15_attention_weight_max": 32.239891052246094,
      "activations/layer15_attention_weight_min": -29.707014083862305,
      "activations/layer16_attention_weight_max": 34.11299133300781,
      "activations/layer16_attention_weight_min": -29.24956512451172,
      "activations/layer17_attention_weight_max": 53.72713851928711,
      "activations/layer17_attention_weight_min": -44.95449447631836,
      "activations/layer18_attention_weight_max": 48.546241760253906,
      "activations/layer18_attention_weight_min": -38.75788879394531,
      "activations/layer19_attention_weight_max": 22.372888565063477,
      "activations/layer19_attention_weight_min": -20.361587524414062,
      "activations/layer1_attention_weight_max": 16.242656707763672,
      "activations/layer1_attention_weight_min": -14.201895713806152,
      "activations/layer20_attention_weight_max": 23.748624801635742,
      "activations/layer20_attention_weight_min": -23.282575607299805,
      "activations/layer21_attention_weight_max": 44.02333450317383,
      "activations/layer21_attention_weight_min": -26.683218002319336,
      "activations/layer22_attention_weight_max": 34.15375518798828,
      "activations/layer22_attention_weight_min": -26.190401077270508,
      "activations/layer23_attention_weight_max": 42.734535217285156,
      "activations/layer23_attention_weight_min": -24.55536460876465,
      "activations/layer2_attention_weight_max": 31.30610466003418,
      "activations/layer2_attention_weight_min": -30.056373596191406,
      "activations/layer3_attention_weight_max": 86.7247543334961,
      "activations/layer3_attention_weight_min": -87.44845581054688,
      "activations/layer4_attention_weight_max": 93.57814025878906,
      "activations/layer4_attention_weight_min": -88.67527770996094,
      "activations/layer5_attention_weight_max": 69.63877868652344,
      "activations/layer5_attention_weight_min": -72.96707153320312,
      "activations/layer6_attention_weight_max": 50.217281341552734,
      "activations/layer6_attention_weight_min": -50.60663986206055,
      "activations/layer7_attention_weight_max": 68.66718292236328,
      "activations/layer7_attention_weight_min": -69.54161071777344,
      "activations/layer8_attention_weight_max": 41.997779846191406,
      "activations/layer8_attention_weight_min": -46.501312255859375,
      "activations/layer9_attention_weight_max": 44.94123458862305,
      "activations/layer9_attention_weight_min": -48.55595016479492,
      "epoch": 10.71,
      "learning_rate": 8.17651515151515e-05,
      "loss": 2.7796,
      "step": 184300
    },
    {
      "activations/layer0_attention_weight_max": 15.238574981689453,
      "activations/layer0_attention_weight_min": -13.872174263000488,
      "activations/layer10_attention_weight_max": 35.23051834106445,
      "activations/layer10_attention_weight_min": -36.85767364501953,
      "activations/layer11_attention_weight_max": 31.80034637451172,
      "activations/layer11_attention_weight_min": -31.624744415283203,
      "activations/layer12_attention_weight_max": 19.44220542907715,
      "activations/layer12_attention_weight_min": -24.04075050354004,
      "activations/layer13_attention_weight_max": 46.24974060058594,
      "activations/layer13_attention_weight_min": -31.70512580871582,
      "activations/layer14_attention_weight_max": 47.987422943115234,
      "activations/layer14_attention_weight_min": -32.049041748046875,
      "activations/layer15_attention_weight_max": 39.319278717041016,
      "activations/layer15_attention_weight_min": -30.928327560424805,
      "activations/layer16_attention_weight_max": 33.67876434326172,
      "activations/layer16_attention_weight_min": -29.903987884521484,
      "activations/layer17_attention_weight_max": 54.79100036621094,
      "activations/layer17_attention_weight_min": -42.9258918762207,
      "activations/layer18_attention_weight_max": 48.48683166503906,
      "activations/layer18_attention_weight_min": -37.59597396850586,
      "activations/layer19_attention_weight_max": 21.81695556640625,
      "activations/layer19_attention_weight_min": -22.7541561126709,
      "activations/layer1_attention_weight_max": 16.90313720703125,
      "activations/layer1_attention_weight_min": -15.506637573242188,
      "activations/layer20_attention_weight_max": 26.488866806030273,
      "activations/layer20_attention_weight_min": -20.479450225830078,
      "activations/layer21_attention_weight_max": 44.41517639160156,
      "activations/layer21_attention_weight_min": -22.680816650390625,
      "activations/layer22_attention_weight_max": 40.635345458984375,
      "activations/layer22_attention_weight_min": -25.17290687561035,
      "activations/layer23_attention_weight_max": 42.24031448364258,
      "activations/layer23_attention_weight_min": -23.711246490478516,
      "activations/layer2_attention_weight_max": 32.200340270996094,
      "activations/layer2_attention_weight_min": -31.80126953125,
      "activations/layer3_attention_weight_max": 96.61703491210938,
      "activations/layer3_attention_weight_min": -102.096923828125,
      "activations/layer4_attention_weight_max": 104.05132293701172,
      "activations/layer4_attention_weight_min": -98.97073364257812,
      "activations/layer5_attention_weight_max": 75.75706481933594,
      "activations/layer5_attention_weight_min": -81.50755310058594,
      "activations/layer6_attention_weight_max": 55.583251953125,
      "activations/layer6_attention_weight_min": -52.90752029418945,
      "activations/layer7_attention_weight_max": 73.5014419555664,
      "activations/layer7_attention_weight_min": -72.43246459960938,
      "activations/layer8_attention_weight_max": 46.02070999145508,
      "activations/layer8_attention_weight_min": -47.918243408203125,
      "activations/layer9_attention_weight_max": 51.551063537597656,
      "activations/layer9_attention_weight_min": -51.764095306396484,
      "epoch": 10.71,
      "learning_rate": 8.174621212121212e-05,
      "loss": 2.7903,
      "step": 184350
    },
    {
      "activations/layer0_attention_weight_max": 15.654976844787598,
      "activations/layer0_attention_weight_min": -14.090975761413574,
      "activations/layer10_attention_weight_max": 39.801265716552734,
      "activations/layer10_attention_weight_min": -38.475929260253906,
      "activations/layer11_attention_weight_max": 36.2276725769043,
      "activations/layer11_attention_weight_min": -32.83269119262695,
      "activations/layer12_attention_weight_max": 25.856739044189453,
      "activations/layer12_attention_weight_min": -24.251052856445312,
      "activations/layer13_attention_weight_max": 42.486534118652344,
      "activations/layer13_attention_weight_min": -34.25518798828125,
      "activations/layer14_attention_weight_max": 38.81216049194336,
      "activations/layer14_attention_weight_min": -32.380496978759766,
      "activations/layer15_attention_weight_max": 38.32651138305664,
      "activations/layer15_attention_weight_min": -29.89446258544922,
      "activations/layer16_attention_weight_max": 31.839860916137695,
      "activations/layer16_attention_weight_min": -28.991838455200195,
      "activations/layer17_attention_weight_max": 57.14803695678711,
      "activations/layer17_attention_weight_min": -48.358421325683594,
      "activations/layer18_attention_weight_max": 49.85376739501953,
      "activations/layer18_attention_weight_min": -37.63028335571289,
      "activations/layer19_attention_weight_max": 23.689273834228516,
      "activations/layer19_attention_weight_min": -19.948640823364258,
      "activations/layer1_attention_weight_max": 16.930194854736328,
      "activations/layer1_attention_weight_min": -15.419136047363281,
      "activations/layer20_attention_weight_max": 24.933359146118164,
      "activations/layer20_attention_weight_min": -20.174541473388672,
      "activations/layer21_attention_weight_max": 42.71416091918945,
      "activations/layer21_attention_weight_min": -23.754331588745117,
      "activations/layer22_attention_weight_max": 35.6258430480957,
      "activations/layer22_attention_weight_min": -26.265045166015625,
      "activations/layer23_attention_weight_max": 39.298221588134766,
      "activations/layer23_attention_weight_min": -24.812896728515625,
      "activations/layer2_attention_weight_max": 31.09579086303711,
      "activations/layer2_attention_weight_min": -30.016698837280273,
      "activations/layer3_attention_weight_max": 86.7042007446289,
      "activations/layer3_attention_weight_min": -90.56998443603516,
      "activations/layer4_attention_weight_max": 101.16850280761719,
      "activations/layer4_attention_weight_min": -103.18719482421875,
      "activations/layer5_attention_weight_max": 75.78849792480469,
      "activations/layer5_attention_weight_min": -78.13111877441406,
      "activations/layer6_attention_weight_max": 53.84282302856445,
      "activations/layer6_attention_weight_min": -57.648658752441406,
      "activations/layer7_attention_weight_max": 81.06664276123047,
      "activations/layer7_attention_weight_min": -71.28533935546875,
      "activations/layer8_attention_weight_max": 47.497215270996094,
      "activations/layer8_attention_weight_min": -50.62047576904297,
      "activations/layer9_attention_weight_max": 59.077857971191406,
      "activations/layer9_attention_weight_min": -49.92139434814453,
      "epoch": 10.71,
      "learning_rate": 8.172727272727272e-05,
      "loss": 2.7882,
      "step": 184400
    },
    {
      "activations/layer0_attention_weight_max": 16.230539321899414,
      "activations/layer0_attention_weight_min": -13.702896118164062,
      "activations/layer10_attention_weight_max": 35.42683029174805,
      "activations/layer10_attention_weight_min": -34.71721649169922,
      "activations/layer11_attention_weight_max": 31.126920700073242,
      "activations/layer11_attention_weight_min": -31.722387313842773,
      "activations/layer12_attention_weight_max": 15.6780424118042,
      "activations/layer12_attention_weight_min": -26.78422737121582,
      "activations/layer13_attention_weight_max": 42.81849670410156,
      "activations/layer13_attention_weight_min": -35.71146774291992,
      "activations/layer14_attention_weight_max": 34.48383712768555,
      "activations/layer14_attention_weight_min": -32.39431381225586,
      "activations/layer15_attention_weight_max": 31.09494400024414,
      "activations/layer15_attention_weight_min": -31.076791763305664,
      "activations/layer16_attention_weight_max": 30.224763870239258,
      "activations/layer16_attention_weight_min": -30.770153045654297,
      "activations/layer17_attention_weight_max": 48.36714553833008,
      "activations/layer17_attention_weight_min": -47.12642288208008,
      "activations/layer18_attention_weight_max": 42.097381591796875,
      "activations/layer18_attention_weight_min": -40.07133865356445,
      "activations/layer19_attention_weight_max": 20.456844329833984,
      "activations/layer19_attention_weight_min": -23.027387619018555,
      "activations/layer1_attention_weight_max": 16.93119239807129,
      "activations/layer1_attention_weight_min": -14.399250030517578,
      "activations/layer20_attention_weight_max": 19.58572006225586,
      "activations/layer20_attention_weight_min": -22.312416076660156,
      "activations/layer21_attention_weight_max": 31.481929779052734,
      "activations/layer21_attention_weight_min": -23.96709632873535,
      "activations/layer22_attention_weight_max": 30.06459617614746,
      "activations/layer22_attention_weight_min": -25.07123565673828,
      "activations/layer23_attention_weight_max": 33.09637451171875,
      "activations/layer23_attention_weight_min": -24.85395050048828,
      "activations/layer2_attention_weight_max": 31.224769592285156,
      "activations/layer2_attention_weight_min": -28.57245445251465,
      "activations/layer3_attention_weight_max": 89.02468872070312,
      "activations/layer3_attention_weight_min": -86.8475341796875,
      "activations/layer4_attention_weight_max": 95.92987060546875,
      "activations/layer4_attention_weight_min": -89.55960083007812,
      "activations/layer5_attention_weight_max": 71.76551818847656,
      "activations/layer5_attention_weight_min": -77.99600219726562,
      "activations/layer6_attention_weight_max": 54.11673355102539,
      "activations/layer6_attention_weight_min": -51.56710433959961,
      "activations/layer7_attention_weight_max": 71.19603729248047,
      "activations/layer7_attention_weight_min": -73.3212890625,
      "activations/layer8_attention_weight_max": 43.668087005615234,
      "activations/layer8_attention_weight_min": -46.04319763183594,
      "activations/layer9_attention_weight_max": 49.39142990112305,
      "activations/layer9_attention_weight_min": -51.498023986816406,
      "epoch": 10.72,
      "learning_rate": 8.170833333333333e-05,
      "loss": 2.7817,
      "step": 184450
    },
    {
      "activations/layer0_attention_weight_max": 15.974828720092773,
      "activations/layer0_attention_weight_min": -14.07352352142334,
      "activations/layer10_attention_weight_max": 34.76941680908203,
      "activations/layer10_attention_weight_min": -35.34400177001953,
      "activations/layer11_attention_weight_max": 30.513931274414062,
      "activations/layer11_attention_weight_min": -32.645423889160156,
      "activations/layer12_attention_weight_max": 20.36676025390625,
      "activations/layer12_attention_weight_min": -25.098480224609375,
      "activations/layer13_attention_weight_max": 36.5718879699707,
      "activations/layer13_attention_weight_min": -29.982330322265625,
      "activations/layer14_attention_weight_max": 34.463768005371094,
      "activations/layer14_attention_weight_min": -30.837984085083008,
      "activations/layer15_attention_weight_max": 32.389949798583984,
      "activations/layer15_attention_weight_min": -29.0621337890625,
      "activations/layer16_attention_weight_max": 30.576961517333984,
      "activations/layer16_attention_weight_min": -31.15220069885254,
      "activations/layer17_attention_weight_max": 55.90111541748047,
      "activations/layer17_attention_weight_min": -45.14534378051758,
      "activations/layer18_attention_weight_max": 46.40134048461914,
      "activations/layer18_attention_weight_min": -39.27893829345703,
      "activations/layer19_attention_weight_max": 22.246910095214844,
      "activations/layer19_attention_weight_min": -22.60463523864746,
      "activations/layer1_attention_weight_max": 16.584434509277344,
      "activations/layer1_attention_weight_min": -15.047124862670898,
      "activations/layer20_attention_weight_max": 21.72200584411621,
      "activations/layer20_attention_weight_min": -22.73712730407715,
      "activations/layer21_attention_weight_max": 36.621246337890625,
      "activations/layer21_attention_weight_min": -22.496965408325195,
      "activations/layer22_attention_weight_max": 30.775035858154297,
      "activations/layer22_attention_weight_min": -27.39031982421875,
      "activations/layer23_attention_weight_max": 36.47517776489258,
      "activations/layer23_attention_weight_min": -26.031307220458984,
      "activations/layer2_attention_weight_max": 31.580039978027344,
      "activations/layer2_attention_weight_min": -31.699310302734375,
      "activations/layer3_attention_weight_max": 85.90154266357422,
      "activations/layer3_attention_weight_min": -97.40384674072266,
      "activations/layer4_attention_weight_max": 95.7658920288086,
      "activations/layer4_attention_weight_min": -93.03446197509766,
      "activations/layer5_attention_weight_max": 73.64125061035156,
      "activations/layer5_attention_weight_min": -76.53314208984375,
      "activations/layer6_attention_weight_max": 51.70576095581055,
      "activations/layer6_attention_weight_min": -51.276432037353516,
      "activations/layer7_attention_weight_max": 71.61763763427734,
      "activations/layer7_attention_weight_min": -72.57820129394531,
      "activations/layer8_attention_weight_max": 45.643558502197266,
      "activations/layer8_attention_weight_min": -45.86305618286133,
      "activations/layer9_attention_weight_max": 48.41245651245117,
      "activations/layer9_attention_weight_min": -45.75853729248047,
      "epoch": 10.72,
      "learning_rate": 8.168939393939393e-05,
      "loss": 2.7969,
      "step": 184500
    },
    {
      "activations/layer0_attention_weight_max": 15.162429809570312,
      "activations/layer0_attention_weight_min": -13.519567489624023,
      "activations/layer10_attention_weight_max": 52.3955192565918,
      "activations/layer10_attention_weight_min": -48.78759765625,
      "activations/layer11_attention_weight_max": 47.38999557495117,
      "activations/layer11_attention_weight_min": -42.15064239501953,
      "activations/layer12_attention_weight_max": 21.597763061523438,
      "activations/layer12_attention_weight_min": -25.498125076293945,
      "activations/layer13_attention_weight_max": 72.67474365234375,
      "activations/layer13_attention_weight_min": -34.72779846191406,
      "activations/layer14_attention_weight_max": 72.16681671142578,
      "activations/layer14_attention_weight_min": -37.99538040161133,
      "activations/layer15_attention_weight_max": 66.07915496826172,
      "activations/layer15_attention_weight_min": -32.97087478637695,
      "activations/layer16_attention_weight_max": 39.58743667602539,
      "activations/layer16_attention_weight_min": -35.85066223144531,
      "activations/layer17_attention_weight_max": 66.534423828125,
      "activations/layer17_attention_weight_min": -59.59547424316406,
      "activations/layer18_attention_weight_max": 59.16563034057617,
      "activations/layer18_attention_weight_min": -50.18438720703125,
      "activations/layer19_attention_weight_max": 28.51543617248535,
      "activations/layer19_attention_weight_min": -26.097309112548828,
      "activations/layer1_attention_weight_max": 16.852903366088867,
      "activations/layer1_attention_weight_min": -14.899886131286621,
      "activations/layer20_attention_weight_max": 30.865745544433594,
      "activations/layer20_attention_weight_min": -22.36686897277832,
      "activations/layer21_attention_weight_max": 55.18944549560547,
      "activations/layer21_attention_weight_min": -22.92630958557129,
      "activations/layer22_attention_weight_max": 39.82453918457031,
      "activations/layer22_attention_weight_min": -27.5545654296875,
      "activations/layer23_attention_weight_max": 39.081932067871094,
      "activations/layer23_attention_weight_min": -26.916015625,
      "activations/layer2_attention_weight_max": 31.718914031982422,
      "activations/layer2_attention_weight_min": -31.50605583190918,
      "activations/layer3_attention_weight_max": 92.70149230957031,
      "activations/layer3_attention_weight_min": -97.79410552978516,
      "activations/layer4_attention_weight_max": 106.42903137207031,
      "activations/layer4_attention_weight_min": -100.43599700927734,
      "activations/layer5_attention_weight_max": 78.5661392211914,
      "activations/layer5_attention_weight_min": -77.64892578125,
      "activations/layer6_attention_weight_max": 55.59796905517578,
      "activations/layer6_attention_weight_min": -54.506309509277344,
      "activations/layer7_attention_weight_max": 89.11671447753906,
      "activations/layer7_attention_weight_min": -78.87371063232422,
      "activations/layer8_attention_weight_max": 59.47188949584961,
      "activations/layer8_attention_weight_min": -54.084381103515625,
      "activations/layer9_attention_weight_max": 73.20999908447266,
      "activations/layer9_attention_weight_min": -59.24741744995117,
      "epoch": 10.72,
      "learning_rate": 8.167045454545453e-05,
      "loss": 2.7905,
      "step": 184550
    },
    {
      "activations/layer0_attention_weight_max": 15.599061012268066,
      "activations/layer0_attention_weight_min": -13.559494018554688,
      "activations/layer10_attention_weight_max": 35.193939208984375,
      "activations/layer10_attention_weight_min": -33.75639343261719,
      "activations/layer11_attention_weight_max": 28.7176513671875,
      "activations/layer11_attention_weight_min": -30.49647331237793,
      "activations/layer12_attention_weight_max": 19.25673484802246,
      "activations/layer12_attention_weight_min": -23.476083755493164,
      "activations/layer13_attention_weight_max": 40.5528450012207,
      "activations/layer13_attention_weight_min": -30.539155960083008,
      "activations/layer14_attention_weight_max": 39.184669494628906,
      "activations/layer14_attention_weight_min": -31.948863983154297,
      "activations/layer15_attention_weight_max": 33.260292053222656,
      "activations/layer15_attention_weight_min": -28.84393310546875,
      "activations/layer16_attention_weight_max": 33.420860290527344,
      "activations/layer16_attention_weight_min": -30.224817276000977,
      "activations/layer17_attention_weight_max": 51.190277099609375,
      "activations/layer17_attention_weight_min": -45.592586517333984,
      "activations/layer18_attention_weight_max": 48.03437805175781,
      "activations/layer18_attention_weight_min": -40.077362060546875,
      "activations/layer19_attention_weight_max": 21.661914825439453,
      "activations/layer19_attention_weight_min": -20.97885513305664,
      "activations/layer1_attention_weight_max": 16.410085678100586,
      "activations/layer1_attention_weight_min": -14.110837936401367,
      "activations/layer20_attention_weight_max": 22.603994369506836,
      "activations/layer20_attention_weight_min": -21.320337295532227,
      "activations/layer21_attention_weight_max": 35.090824127197266,
      "activations/layer21_attention_weight_min": -22.545520782470703,
      "activations/layer22_attention_weight_max": 34.37323760986328,
      "activations/layer22_attention_weight_min": -27.474103927612305,
      "activations/layer23_attention_weight_max": 43.20933151245117,
      "activations/layer23_attention_weight_min": -24.196195602416992,
      "activations/layer2_attention_weight_max": 32.23936462402344,
      "activations/layer2_attention_weight_min": -30.343265533447266,
      "activations/layer3_attention_weight_max": 88.08830261230469,
      "activations/layer3_attention_weight_min": -90.2895736694336,
      "activations/layer4_attention_weight_max": 95.5062026977539,
      "activations/layer4_attention_weight_min": -88.8021240234375,
      "activations/layer5_attention_weight_max": 69.52153778076172,
      "activations/layer5_attention_weight_min": -75.60611724853516,
      "activations/layer6_attention_weight_max": 51.694435119628906,
      "activations/layer6_attention_weight_min": -51.00078201293945,
      "activations/layer7_attention_weight_max": 69.1860580444336,
      "activations/layer7_attention_weight_min": -73.90031433105469,
      "activations/layer8_attention_weight_max": 42.131187438964844,
      "activations/layer8_attention_weight_min": -46.675662994384766,
      "activations/layer9_attention_weight_max": 46.21105194091797,
      "activations/layer9_attention_weight_min": -46.566429138183594,
      "epoch": 10.73,
      "learning_rate": 8.165151515151515e-05,
      "loss": 2.7832,
      "step": 184600
    },
    {
      "activations/layer0_attention_weight_max": 14.653690338134766,
      "activations/layer0_attention_weight_min": -13.935813903808594,
      "activations/layer10_attention_weight_max": 36.33211135864258,
      "activations/layer10_attention_weight_min": -36.358455657958984,
      "activations/layer11_attention_weight_max": 31.27642822265625,
      "activations/layer11_attention_weight_min": -33.21521759033203,
      "activations/layer12_attention_weight_max": 19.077932357788086,
      "activations/layer12_attention_weight_min": -24.897180557250977,
      "activations/layer13_attention_weight_max": 37.082122802734375,
      "activations/layer13_attention_weight_min": -26.340280532836914,
      "activations/layer14_attention_weight_max": 34.744110107421875,
      "activations/layer14_attention_weight_min": -28.70072364807129,
      "activations/layer15_attention_weight_max": 34.02750778198242,
      "activations/layer15_attention_weight_min": -28.753816604614258,
      "activations/layer16_attention_weight_max": 30.081140518188477,
      "activations/layer16_attention_weight_min": -26.39002799987793,
      "activations/layer17_attention_weight_max": 57.7998046875,
      "activations/layer17_attention_weight_min": -42.16050720214844,
      "activations/layer18_attention_weight_max": 53.30562973022461,
      "activations/layer18_attention_weight_min": -36.87543487548828,
      "activations/layer19_attention_weight_max": 24.153423309326172,
      "activations/layer19_attention_weight_min": -21.791366577148438,
      "activations/layer1_attention_weight_max": 16.467082977294922,
      "activations/layer1_attention_weight_min": -14.802589416503906,
      "activations/layer20_attention_weight_max": 22.256288528442383,
      "activations/layer20_attention_weight_min": -20.1318302154541,
      "activations/layer21_attention_weight_max": 37.5551643371582,
      "activations/layer21_attention_weight_min": -23.30580711364746,
      "activations/layer22_attention_weight_max": 29.845937728881836,
      "activations/layer22_attention_weight_min": -25.36147117614746,
      "activations/layer23_attention_weight_max": 34.97151565551758,
      "activations/layer23_attention_weight_min": -23.445220947265625,
      "activations/layer2_attention_weight_max": 31.904438018798828,
      "activations/layer2_attention_weight_min": -31.757770538330078,
      "activations/layer3_attention_weight_max": 88.03938293457031,
      "activations/layer3_attention_weight_min": -88.63164520263672,
      "activations/layer4_attention_weight_max": 95.32099914550781,
      "activations/layer4_attention_weight_min": -94.85589599609375,
      "activations/layer5_attention_weight_max": 71.35739135742188,
      "activations/layer5_attention_weight_min": -78.31523132324219,
      "activations/layer6_attention_weight_max": 54.78788757324219,
      "activations/layer6_attention_weight_min": -53.29554748535156,
      "activations/layer7_attention_weight_max": 73.05754089355469,
      "activations/layer7_attention_weight_min": -70.39219665527344,
      "activations/layer8_attention_weight_max": 48.375301361083984,
      "activations/layer8_attention_weight_min": -47.52687454223633,
      "activations/layer9_attention_weight_max": 49.90180969238281,
      "activations/layer9_attention_weight_min": -50.14083480834961,
      "epoch": 10.73,
      "learning_rate": 8.163257575757575e-05,
      "loss": 2.7678,
      "step": 184650
    },
    {
      "activations/layer0_attention_weight_max": 16.018625259399414,
      "activations/layer0_attention_weight_min": -13.762663841247559,
      "activations/layer10_attention_weight_max": 41.64432907104492,
      "activations/layer10_attention_weight_min": -39.924861907958984,
      "activations/layer11_attention_weight_max": 35.20403289794922,
      "activations/layer11_attention_weight_min": -35.818363189697266,
      "activations/layer12_attention_weight_max": 17.507614135742188,
      "activations/layer12_attention_weight_min": -24.641698837280273,
      "activations/layer13_attention_weight_max": 39.6165885925293,
      "activations/layer13_attention_weight_min": -31.814464569091797,
      "activations/layer14_attention_weight_max": 35.67020797729492,
      "activations/layer14_attention_weight_min": -32.55132293701172,
      "activations/layer15_attention_weight_max": 34.61140441894531,
      "activations/layer15_attention_weight_min": -32.66701889038086,
      "activations/layer16_attention_weight_max": 30.632776260375977,
      "activations/layer16_attention_weight_min": -30.9720458984375,
      "activations/layer17_attention_weight_max": 54.08091735839844,
      "activations/layer17_attention_weight_min": -49.563995361328125,
      "activations/layer18_attention_weight_max": 45.92654800415039,
      "activations/layer18_attention_weight_min": -40.36724853515625,
      "activations/layer19_attention_weight_max": 20.896991729736328,
      "activations/layer19_attention_weight_min": -22.598079681396484,
      "activations/layer1_attention_weight_max": 15.481308937072754,
      "activations/layer1_attention_weight_min": -13.940742492675781,
      "activations/layer20_attention_weight_max": 22.799467086791992,
      "activations/layer20_attention_weight_min": -22.56559181213379,
      "activations/layer21_attention_weight_max": 34.17488098144531,
      "activations/layer21_attention_weight_min": -24.30246353149414,
      "activations/layer22_attention_weight_max": 32.236263275146484,
      "activations/layer22_attention_weight_min": -26.442102432250977,
      "activations/layer23_attention_weight_max": 34.48562240600586,
      "activations/layer23_attention_weight_min": -25.85700225830078,
      "activations/layer2_attention_weight_max": 31.718185424804688,
      "activations/layer2_attention_weight_min": -30.923477172851562,
      "activations/layer3_attention_weight_max": 87.74938201904297,
      "activations/layer3_attention_weight_min": -91.15264129638672,
      "activations/layer4_attention_weight_max": 95.1984634399414,
      "activations/layer4_attention_weight_min": -92.82909393310547,
      "activations/layer5_attention_weight_max": 71.47474670410156,
      "activations/layer5_attention_weight_min": -72.88848876953125,
      "activations/layer6_attention_weight_max": 54.14085388183594,
      "activations/layer6_attention_weight_min": -55.80308532714844,
      "activations/layer7_attention_weight_max": 79.64009857177734,
      "activations/layer7_attention_weight_min": -77.00062561035156,
      "activations/layer8_attention_weight_max": 49.71371078491211,
      "activations/layer8_attention_weight_min": -49.15716552734375,
      "activations/layer9_attention_weight_max": 53.32967758178711,
      "activations/layer9_attention_weight_min": -56.18994140625,
      "epoch": 10.73,
      "learning_rate": 8.161363636363635e-05,
      "loss": 2.7901,
      "step": 184700
    },
    {
      "activations/layer0_attention_weight_max": 14.90168285369873,
      "activations/layer0_attention_weight_min": -13.649068832397461,
      "activations/layer10_attention_weight_max": 38.21973419189453,
      "activations/layer10_attention_weight_min": -36.58241271972656,
      "activations/layer11_attention_weight_max": 33.81281661987305,
      "activations/layer11_attention_weight_min": -34.74203109741211,
      "activations/layer12_attention_weight_max": 18.412155151367188,
      "activations/layer12_attention_weight_min": -23.325401306152344,
      "activations/layer13_attention_weight_max": 37.070472717285156,
      "activations/layer13_attention_weight_min": -30.425113677978516,
      "activations/layer14_attention_weight_max": 40.30351257324219,
      "activations/layer14_attention_weight_min": -31.03475570678711,
      "activations/layer15_attention_weight_max": 34.613277435302734,
      "activations/layer15_attention_weight_min": -29.68461799621582,
      "activations/layer16_attention_weight_max": 30.22182273864746,
      "activations/layer16_attention_weight_min": -29.8750057220459,
      "activations/layer17_attention_weight_max": 54.014217376708984,
      "activations/layer17_attention_weight_min": -45.36460876464844,
      "activations/layer18_attention_weight_max": 50.56117248535156,
      "activations/layer18_attention_weight_min": -41.915687561035156,
      "activations/layer19_attention_weight_max": 23.209882736206055,
      "activations/layer19_attention_weight_min": -21.732227325439453,
      "activations/layer1_attention_weight_max": 18.534507751464844,
      "activations/layer1_attention_weight_min": -16.108854293823242,
      "activations/layer20_attention_weight_max": 22.555395126342773,
      "activations/layer20_attention_weight_min": -22.386625289916992,
      "activations/layer21_attention_weight_max": 46.45424270629883,
      "activations/layer21_attention_weight_min": -28.72383689880371,
      "activations/layer22_attention_weight_max": 35.078975677490234,
      "activations/layer22_attention_weight_min": -28.01245880126953,
      "activations/layer23_attention_weight_max": 39.848388671875,
      "activations/layer23_attention_weight_min": -23.56881332397461,
      "activations/layer2_attention_weight_max": 32.214942932128906,
      "activations/layer2_attention_weight_min": -30.831037521362305,
      "activations/layer3_attention_weight_max": 91.0743179321289,
      "activations/layer3_attention_weight_min": -86.64482879638672,
      "activations/layer4_attention_weight_max": 100.57176971435547,
      "activations/layer4_attention_weight_min": -92.66661071777344,
      "activations/layer5_attention_weight_max": 72.8995361328125,
      "activations/layer5_attention_weight_min": -72.99383544921875,
      "activations/layer6_attention_weight_max": 54.89080047607422,
      "activations/layer6_attention_weight_min": -55.455665588378906,
      "activations/layer7_attention_weight_max": 80.44308471679688,
      "activations/layer7_attention_weight_min": -72.14862823486328,
      "activations/layer8_attention_weight_max": 49.8367805480957,
      "activations/layer8_attention_weight_min": -50.8678092956543,
      "activations/layer9_attention_weight_max": 49.247249603271484,
      "activations/layer9_attention_weight_min": -47.991455078125,
      "epoch": 10.74,
      "learning_rate": 8.159469696969697e-05,
      "loss": 2.779,
      "step": 184750
    },
    {
      "activations/layer0_attention_weight_max": 16.475574493408203,
      "activations/layer0_attention_weight_min": -13.69510555267334,
      "activations/layer10_attention_weight_max": 37.33348846435547,
      "activations/layer10_attention_weight_min": -37.33845520019531,
      "activations/layer11_attention_weight_max": 33.83332061767578,
      "activations/layer11_attention_weight_min": -37.30043029785156,
      "activations/layer12_attention_weight_max": 19.278993606567383,
      "activations/layer12_attention_weight_min": -24.03063201904297,
      "activations/layer13_attention_weight_max": 47.937740325927734,
      "activations/layer13_attention_weight_min": -30.86620330810547,
      "activations/layer14_attention_weight_max": 41.80572509765625,
      "activations/layer14_attention_weight_min": -30.525609970092773,
      "activations/layer15_attention_weight_max": 36.994667053222656,
      "activations/layer15_attention_weight_min": -32.083648681640625,
      "activations/layer16_attention_weight_max": 33.82538604736328,
      "activations/layer16_attention_weight_min": -31.305164337158203,
      "activations/layer17_attention_weight_max": 59.89827346801758,
      "activations/layer17_attention_weight_min": -49.49055862426758,
      "activations/layer18_attention_weight_max": 55.216033935546875,
      "activations/layer18_attention_weight_min": -44.69438934326172,
      "activations/layer19_attention_weight_max": 27.963499069213867,
      "activations/layer19_attention_weight_min": -24.38081169128418,
      "activations/layer1_attention_weight_max": 18.26364517211914,
      "activations/layer1_attention_weight_min": -15.649202346801758,
      "activations/layer20_attention_weight_max": 30.543378829956055,
      "activations/layer20_attention_weight_min": -24.069971084594727,
      "activations/layer21_attention_weight_max": 50.84001541137695,
      "activations/layer21_attention_weight_min": -26.558467864990234,
      "activations/layer22_attention_weight_max": 38.17761993408203,
      "activations/layer22_attention_weight_min": -27.04125213623047,
      "activations/layer23_attention_weight_max": 47.74609375,
      "activations/layer23_attention_weight_min": -23.63753318786621,
      "activations/layer2_attention_weight_max": 32.72810745239258,
      "activations/layer2_attention_weight_min": -30.925045013427734,
      "activations/layer3_attention_weight_max": 90.20561218261719,
      "activations/layer3_attention_weight_min": -91.99850463867188,
      "activations/layer4_attention_weight_max": 96.17308044433594,
      "activations/layer4_attention_weight_min": -93.63945770263672,
      "activations/layer5_attention_weight_max": 71.66165161132812,
      "activations/layer5_attention_weight_min": -80.07229614257812,
      "activations/layer6_attention_weight_max": 54.3068962097168,
      "activations/layer6_attention_weight_min": -56.893646240234375,
      "activations/layer7_attention_weight_max": 85.5732192993164,
      "activations/layer7_attention_weight_min": -73.8865737915039,
      "activations/layer8_attention_weight_max": 48.21984100341797,
      "activations/layer8_attention_weight_min": -49.10228729248047,
      "activations/layer9_attention_weight_max": 48.33921432495117,
      "activations/layer9_attention_weight_min": -50.159358978271484,
      "epoch": 10.74,
      "learning_rate": 8.157575757575756e-05,
      "loss": 2.7826,
      "step": 184800
    },
    {
      "activations/layer0_attention_weight_max": 14.75063419342041,
      "activations/layer0_attention_weight_min": -13.80080509185791,
      "activations/layer10_attention_weight_max": 38.23346710205078,
      "activations/layer10_attention_weight_min": -37.1591796875,
      "activations/layer11_attention_weight_max": 34.087196350097656,
      "activations/layer11_attention_weight_min": -34.6934700012207,
      "activations/layer12_attention_weight_max": 18.599313735961914,
      "activations/layer12_attention_weight_min": -23.847684860229492,
      "activations/layer13_attention_weight_max": 36.933998107910156,
      "activations/layer13_attention_weight_min": -34.43122482299805,
      "activations/layer14_attention_weight_max": 35.334808349609375,
      "activations/layer14_attention_weight_min": -29.615055084228516,
      "activations/layer15_attention_weight_max": 32.839603424072266,
      "activations/layer15_attention_weight_min": -29.442594528198242,
      "activations/layer16_attention_weight_max": 32.29951095581055,
      "activations/layer16_attention_weight_min": -29.363391876220703,
      "activations/layer17_attention_weight_max": 54.122867584228516,
      "activations/layer17_attention_weight_min": -43.279945373535156,
      "activations/layer18_attention_weight_max": 51.1380729675293,
      "activations/layer18_attention_weight_min": -37.49750900268555,
      "activations/layer19_attention_weight_max": 24.331680297851562,
      "activations/layer19_attention_weight_min": -21.573938369750977,
      "activations/layer1_attention_weight_max": 15.864888191223145,
      "activations/layer1_attention_weight_min": -14.312684059143066,
      "activations/layer20_attention_weight_max": 24.106735229492188,
      "activations/layer20_attention_weight_min": -22.16820526123047,
      "activations/layer21_attention_weight_max": 46.133758544921875,
      "activations/layer21_attention_weight_min": -22.588911056518555,
      "activations/layer22_attention_weight_max": 34.5176887512207,
      "activations/layer22_attention_weight_min": -25.873994827270508,
      "activations/layer23_attention_weight_max": 37.32311248779297,
      "activations/layer23_attention_weight_min": -23.206573486328125,
      "activations/layer2_attention_weight_max": 32.18006134033203,
      "activations/layer2_attention_weight_min": -31.31439208984375,
      "activations/layer3_attention_weight_max": 91.82328033447266,
      "activations/layer3_attention_weight_min": -90.48220825195312,
      "activations/layer4_attention_weight_max": 102.1148910522461,
      "activations/layer4_attention_weight_min": -98.47154998779297,
      "activations/layer5_attention_weight_max": 75.72441864013672,
      "activations/layer5_attention_weight_min": -82.029541015625,
      "activations/layer6_attention_weight_max": 55.25694274902344,
      "activations/layer6_attention_weight_min": -54.024986267089844,
      "activations/layer7_attention_weight_max": 73.51949310302734,
      "activations/layer7_attention_weight_min": -78.82683563232422,
      "activations/layer8_attention_weight_max": 46.70648956298828,
      "activations/layer8_attention_weight_min": -49.93294906616211,
      "activations/layer9_attention_weight_max": 55.675628662109375,
      "activations/layer9_attention_weight_min": -50.85605239868164,
      "epoch": 10.74,
      "learning_rate": 8.155681818181817e-05,
      "loss": 2.777,
      "step": 184850
    },
    {
      "activations/layer0_attention_weight_max": 14.45871353149414,
      "activations/layer0_attention_weight_min": -14.080113410949707,
      "activations/layer10_attention_weight_max": 38.342430114746094,
      "activations/layer10_attention_weight_min": -39.87162399291992,
      "activations/layer11_attention_weight_max": 33.26451110839844,
      "activations/layer11_attention_weight_min": -33.44157409667969,
      "activations/layer12_attention_weight_max": 17.796730041503906,
      "activations/layer12_attention_weight_min": -23.952144622802734,
      "activations/layer13_attention_weight_max": 39.32512283325195,
      "activations/layer13_attention_weight_min": -31.476226806640625,
      "activations/layer14_attention_weight_max": 37.59036636352539,
      "activations/layer14_attention_weight_min": -33.1794548034668,
      "activations/layer15_attention_weight_max": 35.64984893798828,
      "activations/layer15_attention_weight_min": -29.64695167541504,
      "activations/layer16_attention_weight_max": 33.84634780883789,
      "activations/layer16_attention_weight_min": -32.31153869628906,
      "activations/layer17_attention_weight_max": 52.483192443847656,
      "activations/layer17_attention_weight_min": -44.989986419677734,
      "activations/layer18_attention_weight_max": 47.169273376464844,
      "activations/layer18_attention_weight_min": -38.1180534362793,
      "activations/layer19_attention_weight_max": 23.67048454284668,
      "activations/layer19_attention_weight_min": -24.860248565673828,
      "activations/layer1_attention_weight_max": 17.69509506225586,
      "activations/layer1_attention_weight_min": -16.06324577331543,
      "activations/layer20_attention_weight_max": 20.653642654418945,
      "activations/layer20_attention_weight_min": -28.789445877075195,
      "activations/layer21_attention_weight_max": 39.94355010986328,
      "activations/layer21_attention_weight_min": -28.47470474243164,
      "activations/layer22_attention_weight_max": 31.7703857421875,
      "activations/layer22_attention_weight_min": -24.77477264404297,
      "activations/layer23_attention_weight_max": 38.68949890136719,
      "activations/layer23_attention_weight_min": -24.088932037353516,
      "activations/layer2_attention_weight_max": 33.52240753173828,
      "activations/layer2_attention_weight_min": -31.880935668945312,
      "activations/layer3_attention_weight_max": 91.45925903320312,
      "activations/layer3_attention_weight_min": -97.73371124267578,
      "activations/layer4_attention_weight_max": 101.61011505126953,
      "activations/layer4_attention_weight_min": -100.38638305664062,
      "activations/layer5_attention_weight_max": 76.34741973876953,
      "activations/layer5_attention_weight_min": -80.30450439453125,
      "activations/layer6_attention_weight_max": 56.755348205566406,
      "activations/layer6_attention_weight_min": -56.966068267822266,
      "activations/layer7_attention_weight_max": 79.63574981689453,
      "activations/layer7_attention_weight_min": -78.38937377929688,
      "activations/layer8_attention_weight_max": 46.547977447509766,
      "activations/layer8_attention_weight_min": -49.46638488769531,
      "activations/layer9_attention_weight_max": 50.19358825683594,
      "activations/layer9_attention_weight_min": -52.659698486328125,
      "epoch": 10.74,
      "learning_rate": 8.153787878787879e-05,
      "loss": 2.7947,
      "step": 184900
    },
    {
      "activations/layer0_attention_weight_max": 15.132384300231934,
      "activations/layer0_attention_weight_min": -13.873275756835938,
      "activations/layer10_attention_weight_max": 38.97417068481445,
      "activations/layer10_attention_weight_min": -36.88258743286133,
      "activations/layer11_attention_weight_max": 35.93942642211914,
      "activations/layer11_attention_weight_min": -32.83857727050781,
      "activations/layer12_attention_weight_max": 19.71051788330078,
      "activations/layer12_attention_weight_min": -25.62443733215332,
      "activations/layer13_attention_weight_max": 37.233070373535156,
      "activations/layer13_attention_weight_min": -28.066293716430664,
      "activations/layer14_attention_weight_max": 41.261016845703125,
      "activations/layer14_attention_weight_min": -31.062271118164062,
      "activations/layer15_attention_weight_max": 35.40413284301758,
      "activations/layer15_attention_weight_min": -29.855134963989258,
      "activations/layer16_attention_weight_max": 33.75346374511719,
      "activations/layer16_attention_weight_min": -31.03755760192871,
      "activations/layer17_attention_weight_max": 54.910160064697266,
      "activations/layer17_attention_weight_min": -46.50558090209961,
      "activations/layer18_attention_weight_max": 48.45452117919922,
      "activations/layer18_attention_weight_min": -38.43475341796875,
      "activations/layer19_attention_weight_max": 23.136465072631836,
      "activations/layer19_attention_weight_min": -22.038789749145508,
      "activations/layer1_attention_weight_max": 17.121822357177734,
      "activations/layer1_attention_weight_min": -15.433156967163086,
      "activations/layer20_attention_weight_max": 24.278188705444336,
      "activations/layer20_attention_weight_min": -20.213134765625,
      "activations/layer21_attention_weight_max": 40.40165710449219,
      "activations/layer21_attention_weight_min": -23.197237014770508,
      "activations/layer22_attention_weight_max": 31.99947738647461,
      "activations/layer22_attention_weight_min": -27.03342056274414,
      "activations/layer23_attention_weight_max": 38.29022216796875,
      "activations/layer23_attention_weight_min": -25.27259635925293,
      "activations/layer2_attention_weight_max": 33.13787078857422,
      "activations/layer2_attention_weight_min": -31.370159149169922,
      "activations/layer3_attention_weight_max": 95.59626007080078,
      "activations/layer3_attention_weight_min": -91.34954833984375,
      "activations/layer4_attention_weight_max": 103.87627410888672,
      "activations/layer4_attention_weight_min": -96.94347381591797,
      "activations/layer5_attention_weight_max": 76.91084289550781,
      "activations/layer5_attention_weight_min": -75.6663589477539,
      "activations/layer6_attention_weight_max": 56.37022399902344,
      "activations/layer6_attention_weight_min": -53.84004211425781,
      "activations/layer7_attention_weight_max": 75.86275482177734,
      "activations/layer7_attention_weight_min": -73.47471618652344,
      "activations/layer8_attention_weight_max": 45.52685546875,
      "activations/layer8_attention_weight_min": -45.85786437988281,
      "activations/layer9_attention_weight_max": 55.831539154052734,
      "activations/layer9_attention_weight_min": -52.854644775390625,
      "epoch": 10.75,
      "learning_rate": 8.151893939393939e-05,
      "loss": 2.7828,
      "step": 184950
    },
    {
      "activations/layer0_attention_weight_max": 14.316588401794434,
      "activations/layer0_attention_weight_min": -14.473901748657227,
      "activations/layer10_attention_weight_max": 38.72850799560547,
      "activations/layer10_attention_weight_min": -35.096405029296875,
      "activations/layer11_attention_weight_max": 34.01795959472656,
      "activations/layer11_attention_weight_min": -31.792346954345703,
      "activations/layer12_attention_weight_max": 18.866880416870117,
      "activations/layer12_attention_weight_min": -26.814332962036133,
      "activations/layer13_attention_weight_max": 44.8641471862793,
      "activations/layer13_attention_weight_min": -37.15772247314453,
      "activations/layer14_attention_weight_max": 37.82231903076172,
      "activations/layer14_attention_weight_min": -31.619674682617188,
      "activations/layer15_attention_weight_max": 35.57625198364258,
      "activations/layer15_attention_weight_min": -30.692428588867188,
      "activations/layer16_attention_weight_max": 33.993412017822266,
      "activations/layer16_attention_weight_min": -31.043306350708008,
      "activations/layer17_attention_weight_max": 54.30895233154297,
      "activations/layer17_attention_weight_min": -48.03364944458008,
      "activations/layer18_attention_weight_max": 50.273990631103516,
      "activations/layer18_attention_weight_min": -40.034263610839844,
      "activations/layer19_attention_weight_max": 24.347536087036133,
      "activations/layer19_attention_weight_min": -22.875158309936523,
      "activations/layer1_attention_weight_max": 17.6063175201416,
      "activations/layer1_attention_weight_min": -15.34929084777832,
      "activations/layer20_attention_weight_max": 23.17868423461914,
      "activations/layer20_attention_weight_min": -22.6818904876709,
      "activations/layer21_attention_weight_max": 35.7042236328125,
      "activations/layer21_attention_weight_min": -22.116586685180664,
      "activations/layer22_attention_weight_max": 30.82069969177246,
      "activations/layer22_attention_weight_min": -26.641708374023438,
      "activations/layer23_attention_weight_max": 37.482635498046875,
      "activations/layer23_attention_weight_min": -22.60403823852539,
      "activations/layer2_attention_weight_max": 33.51194381713867,
      "activations/layer2_attention_weight_min": -29.694990158081055,
      "activations/layer3_attention_weight_max": 91.45853424072266,
      "activations/layer3_attention_weight_min": -89.63113403320312,
      "activations/layer4_attention_weight_max": 99.677978515625,
      "activations/layer4_attention_weight_min": -95.2626724243164,
      "activations/layer5_attention_weight_max": 71.56668853759766,
      "activations/layer5_attention_weight_min": -77.21028137207031,
      "activations/layer6_attention_weight_max": 53.214439392089844,
      "activations/layer6_attention_weight_min": -53.85263442993164,
      "activations/layer7_attention_weight_max": 74.02520751953125,
      "activations/layer7_attention_weight_min": -77.67646789550781,
      "activations/layer8_attention_weight_max": 50.646724700927734,
      "activations/layer8_attention_weight_min": -50.78948211669922,
      "activations/layer9_attention_weight_max": 48.59449768066406,
      "activations/layer9_attention_weight_min": -47.28855895996094,
      "epoch": 10.75,
      "learning_rate": 8.149999999999999e-05,
      "loss": 2.7924,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_loss": 2.734375,
      "eval_runtime": 8.4538,
      "eval_samples_per_second": 507.937,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.4538,
      "eval_openwebtext_samples_per_second": 507.937,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 1.9872,
      "eval_wikitext_samples_per_second": 229.473,
      "step": 185000
    },
    {
      "epoch": 10.75,
      "eval_lambada_loss": 2.626953125,
      "eval_lambada_ppl": 13.831562592310346,
      "eval_lambada_runtime": 9.5951,
      "eval_lambada_samples_per_second": 507.446,
      "step": 185000
    },
    {
      "activations/layer0_attention_weight_max": 15.353170394897461,
      "activations/layer0_attention_weight_min": -14.136685371398926,
      "activations/layer10_attention_weight_max": 39.31697463989258,
      "activations/layer10_attention_weight_min": -37.781829833984375,
      "activations/layer11_attention_weight_max": 33.170875549316406,
      "activations/layer11_attention_weight_min": -33.56818771362305,
      "activations/layer12_attention_weight_max": 20.494657516479492,
      "activations/layer12_attention_weight_min": -32.74845504760742,
      "activations/layer13_attention_weight_max": 40.83241271972656,
      "activations/layer13_attention_weight_min": -30.60199737548828,
      "activations/layer14_attention_weight_max": 39.87496566772461,
      "activations/layer14_attention_weight_min": -32.43717575073242,
      "activations/layer15_attention_weight_max": 35.46258544921875,
      "activations/layer15_attention_weight_min": -33.02961730957031,
      "activations/layer16_attention_weight_max": 35.6944694519043,
      "activations/layer16_attention_weight_min": -32.58642578125,
      "activations/layer17_attention_weight_max": 60.96849822998047,
      "activations/layer17_attention_weight_min": -48.11205291748047,
      "activations/layer18_attention_weight_max": 51.77799987792969,
      "activations/layer18_attention_weight_min": -41.31698989868164,
      "activations/layer19_attention_weight_max": 22.681964874267578,
      "activations/layer19_attention_weight_min": -24.062963485717773,
      "activations/layer1_attention_weight_max": 16.569856643676758,
      "activations/layer1_attention_weight_min": -16.013593673706055,
      "activations/layer20_attention_weight_max": 22.70102310180664,
      "activations/layer20_attention_weight_min": -22.94535255432129,
      "activations/layer21_attention_weight_max": 43.382198333740234,
      "activations/layer21_attention_weight_min": -21.65022850036621,
      "activations/layer22_attention_weight_max": 29.173906326293945,
      "activations/layer22_attention_weight_min": -27.836111068725586,
      "activations/layer23_attention_weight_max": 37.274696350097656,
      "activations/layer23_attention_weight_min": -22.793781280517578,
      "activations/layer2_attention_weight_max": 33.464717864990234,
      "activations/layer2_attention_weight_min": -31.118192672729492,
      "activations/layer3_attention_weight_max": 94.05142211914062,
      "activations/layer3_attention_weight_min": -97.88018798828125,
      "activations/layer4_attention_weight_max": 102.79239654541016,
      "activations/layer4_attention_weight_min": -97.07897186279297,
      "activations/layer5_attention_weight_max": 76.35391235351562,
      "activations/layer5_attention_weight_min": -76.99147033691406,
      "activations/layer6_attention_weight_max": 54.10980987548828,
      "activations/layer6_attention_weight_min": -54.0977897644043,
      "activations/layer7_attention_weight_max": 81.00660705566406,
      "activations/layer7_attention_weight_min": -72.12425231933594,
      "activations/layer8_attention_weight_max": 47.28327178955078,
      "activations/layer8_attention_weight_min": -48.979347229003906,
      "activations/layer9_attention_weight_max": 51.16427230834961,
      "activations/layer9_attention_weight_min": -52.28864669799805,
      "epoch": 10.75,
      "learning_rate": 8.14810606060606e-05,
      "loss": 2.7783,
      "step": 185050
    },
    {
      "activations/layer0_attention_weight_max": 14.858817100524902,
      "activations/layer0_attention_weight_min": -13.953333854675293,
      "activations/layer10_attention_weight_max": 38.80849075317383,
      "activations/layer10_attention_weight_min": -39.73036193847656,
      "activations/layer11_attention_weight_max": 33.831260681152344,
      "activations/layer11_attention_weight_min": -34.0064697265625,
      "activations/layer12_attention_weight_max": 20.700895309448242,
      "activations/layer12_attention_weight_min": -29.111661911010742,
      "activations/layer13_attention_weight_max": 40.45941925048828,
      "activations/layer13_attention_weight_min": -31.166603088378906,
      "activations/layer14_attention_weight_max": 35.43086242675781,
      "activations/layer14_attention_weight_min": -31.033937454223633,
      "activations/layer15_attention_weight_max": 32.474212646484375,
      "activations/layer15_attention_weight_min": -30.376577377319336,
      "activations/layer16_attention_weight_max": 29.864871978759766,
      "activations/layer16_attention_weight_min": -29.78174591064453,
      "activations/layer17_attention_weight_max": 50.93308639526367,
      "activations/layer17_attention_weight_min": -46.75425720214844,
      "activations/layer18_attention_weight_max": 48.74705123901367,
      "activations/layer18_attention_weight_min": -39.12914276123047,
      "activations/layer19_attention_weight_max": 22.501379013061523,
      "activations/layer19_attention_weight_min": -22.40108871459961,
      "activations/layer1_attention_weight_max": 16.215167999267578,
      "activations/layer1_attention_weight_min": -15.120262145996094,
      "activations/layer20_attention_weight_max": 22.518028259277344,
      "activations/layer20_attention_weight_min": -22.767271041870117,
      "activations/layer21_attention_weight_max": 42.157257080078125,
      "activations/layer21_attention_weight_min": -29.136077880859375,
      "activations/layer22_attention_weight_max": 30.612581253051758,
      "activations/layer22_attention_weight_min": -26.20709800720215,
      "activations/layer23_attention_weight_max": 35.39255905151367,
      "activations/layer23_attention_weight_min": -25.293766021728516,
      "activations/layer2_attention_weight_max": 31.979873657226562,
      "activations/layer2_attention_weight_min": -32.314208984375,
      "activations/layer3_attention_weight_max": 92.81714630126953,
      "activations/layer3_attention_weight_min": -96.86135864257812,
      "activations/layer4_attention_weight_max": 100.85418701171875,
      "activations/layer4_attention_weight_min": -100.77515411376953,
      "activations/layer5_attention_weight_max": 73.41670989990234,
      "activations/layer5_attention_weight_min": -77.08772277832031,
      "activations/layer6_attention_weight_max": 54.805198669433594,
      "activations/layer6_attention_weight_min": -53.44013977050781,
      "activations/layer7_attention_weight_max": 78.80259704589844,
      "activations/layer7_attention_weight_min": -74.09310913085938,
      "activations/layer8_attention_weight_max": 46.05779266357422,
      "activations/layer8_attention_weight_min": -49.407066345214844,
      "activations/layer9_attention_weight_max": 51.39445114135742,
      "activations/layer9_attention_weight_min": -51.031455993652344,
      "epoch": 10.76,
      "learning_rate": 8.14621212121212e-05,
      "loss": 2.7753,
      "step": 185100
    },
    {
      "activations/layer0_attention_weight_max": 18.147401809692383,
      "activations/layer0_attention_weight_min": -14.20520305633545,
      "activations/layer10_attention_weight_max": 38.56172561645508,
      "activations/layer10_attention_weight_min": -37.05570983886719,
      "activations/layer11_attention_weight_max": 31.702177047729492,
      "activations/layer11_attention_weight_min": -32.73296356201172,
      "activations/layer12_attention_weight_max": 19.11405372619629,
      "activations/layer12_attention_weight_min": -25.14317512512207,
      "activations/layer13_attention_weight_max": 47.09724044799805,
      "activations/layer13_attention_weight_min": -32.071807861328125,
      "activations/layer14_attention_weight_max": 45.59815979003906,
      "activations/layer14_attention_weight_min": -32.02291488647461,
      "activations/layer15_attention_weight_max": 45.381263732910156,
      "activations/layer15_attention_weight_min": -31.606237411499023,
      "activations/layer16_attention_weight_max": 32.13083267211914,
      "activations/layer16_attention_weight_min": -28.993412017822266,
      "activations/layer17_attention_weight_max": 54.69850158691406,
      "activations/layer17_attention_weight_min": -43.933502197265625,
      "activations/layer18_attention_weight_max": 49.692100524902344,
      "activations/layer18_attention_weight_min": -37.2996826171875,
      "activations/layer19_attention_weight_max": 21.943220138549805,
      "activations/layer19_attention_weight_min": -21.36081886291504,
      "activations/layer1_attention_weight_max": 16.31329345703125,
      "activations/layer1_attention_weight_min": -15.414318084716797,
      "activations/layer20_attention_weight_max": 25.56819725036621,
      "activations/layer20_attention_weight_min": -23.051158905029297,
      "activations/layer21_attention_weight_max": 42.009429931640625,
      "activations/layer21_attention_weight_min": -23.907033920288086,
      "activations/layer22_attention_weight_max": 33.84559631347656,
      "activations/layer22_attention_weight_min": -26.73712730407715,
      "activations/layer23_attention_weight_max": 42.670589447021484,
      "activations/layer23_attention_weight_min": -24.119770050048828,
      "activations/layer2_attention_weight_max": 33.22660827636719,
      "activations/layer2_attention_weight_min": -30.854145050048828,
      "activations/layer3_attention_weight_max": 92.75127410888672,
      "activations/layer3_attention_weight_min": -92.49105072021484,
      "activations/layer4_attention_weight_max": 100.71527099609375,
      "activations/layer4_attention_weight_min": -97.51690673828125,
      "activations/layer5_attention_weight_max": 79.4971923828125,
      "activations/layer5_attention_weight_min": -77.33187103271484,
      "activations/layer6_attention_weight_max": 53.3672981262207,
      "activations/layer6_attention_weight_min": -52.92776107788086,
      "activations/layer7_attention_weight_max": 75.54496002197266,
      "activations/layer7_attention_weight_min": -73.44232940673828,
      "activations/layer8_attention_weight_max": 44.68696594238281,
      "activations/layer8_attention_weight_min": -48.079811096191406,
      "activations/layer9_attention_weight_max": 54.500911712646484,
      "activations/layer9_attention_weight_min": -47.914634704589844,
      "epoch": 10.76,
      "learning_rate": 8.144318181818181e-05,
      "loss": 2.8026,
      "step": 185150
    },
    {
      "activations/layer0_attention_weight_max": 15.49244213104248,
      "activations/layer0_attention_weight_min": -14.078607559204102,
      "activations/layer10_attention_weight_max": 33.17730712890625,
      "activations/layer10_attention_weight_min": -33.16414260864258,
      "activations/layer11_attention_weight_max": 31.26568603515625,
      "activations/layer11_attention_weight_min": -32.11499786376953,
      "activations/layer12_attention_weight_max": 19.652997970581055,
      "activations/layer12_attention_weight_min": -22.864364624023438,
      "activations/layer13_attention_weight_max": 36.65789031982422,
      "activations/layer13_attention_weight_min": -28.384536743164062,
      "activations/layer14_attention_weight_max": 35.87861633300781,
      "activations/layer14_attention_weight_min": -29.175312042236328,
      "activations/layer15_attention_weight_max": 30.582592010498047,
      "activations/layer15_attention_weight_min": -28.128032684326172,
      "activations/layer16_attention_weight_max": 27.804689407348633,
      "activations/layer16_attention_weight_min": -26.43515968322754,
      "activations/layer17_attention_weight_max": 48.87018585205078,
      "activations/layer17_attention_weight_min": -41.616233825683594,
      "activations/layer18_attention_weight_max": 44.11296463012695,
      "activations/layer18_attention_weight_min": -35.10671615600586,
      "activations/layer19_attention_weight_max": 21.114734649658203,
      "activations/layer19_attention_weight_min": -21.039457321166992,
      "activations/layer1_attention_weight_max": 16.1135196685791,
      "activations/layer1_attention_weight_min": -14.4733304977417,
      "activations/layer20_attention_weight_max": 19.973289489746094,
      "activations/layer20_attention_weight_min": -23.343486785888672,
      "activations/layer21_attention_weight_max": 32.510398864746094,
      "activations/layer21_attention_weight_min": -23.424144744873047,
      "activations/layer22_attention_weight_max": 27.11617088317871,
      "activations/layer22_attention_weight_min": -25.32821273803711,
      "activations/layer23_attention_weight_max": 38.34619903564453,
      "activations/layer23_attention_weight_min": -23.88019371032715,
      "activations/layer2_attention_weight_max": 32.7485237121582,
      "activations/layer2_attention_weight_min": -30.69472312927246,
      "activations/layer3_attention_weight_max": 87.91960144042969,
      "activations/layer3_attention_weight_min": -92.40933227539062,
      "activations/layer4_attention_weight_max": 95.0002670288086,
      "activations/layer4_attention_weight_min": -89.31098175048828,
      "activations/layer5_attention_weight_max": 74.03319549560547,
      "activations/layer5_attention_weight_min": -72.71922302246094,
      "activations/layer6_attention_weight_max": 52.47426986694336,
      "activations/layer6_attention_weight_min": -50.56074905395508,
      "activations/layer7_attention_weight_max": 71.39053344726562,
      "activations/layer7_attention_weight_min": -69.37574768066406,
      "activations/layer8_attention_weight_max": 43.12018966674805,
      "activations/layer8_attention_weight_min": -44.16144561767578,
      "activations/layer9_attention_weight_max": 44.73897171020508,
      "activations/layer9_attention_weight_min": -44.72407913208008,
      "epoch": 10.76,
      "learning_rate": 8.142424242424242e-05,
      "loss": 2.7784,
      "step": 185200
    },
    {
      "activations/layer0_attention_weight_max": 16.21617889404297,
      "activations/layer0_attention_weight_min": -14.305154800415039,
      "activations/layer10_attention_weight_max": 40.47796630859375,
      "activations/layer10_attention_weight_min": -37.50901412963867,
      "activations/layer11_attention_weight_max": 38.85847854614258,
      "activations/layer11_attention_weight_min": -34.535194396972656,
      "activations/layer12_attention_weight_max": 20.424524307250977,
      "activations/layer12_attention_weight_min": -27.250009536743164,
      "activations/layer13_attention_weight_max": 50.28892517089844,
      "activations/layer13_attention_weight_min": -34.007362365722656,
      "activations/layer14_attention_weight_max": 56.23548126220703,
      "activations/layer14_attention_weight_min": -34.46849822998047,
      "activations/layer15_attention_weight_max": 50.010128021240234,
      "activations/layer15_attention_weight_min": -28.63098907470703,
      "activations/layer16_attention_weight_max": 30.55247688293457,
      "activations/layer16_attention_weight_min": -27.059532165527344,
      "activations/layer17_attention_weight_max": 50.893672943115234,
      "activations/layer17_attention_weight_min": -47.09364700317383,
      "activations/layer18_attention_weight_max": 47.241920471191406,
      "activations/layer18_attention_weight_min": -42.1866455078125,
      "activations/layer19_attention_weight_max": 24.269636154174805,
      "activations/layer19_attention_weight_min": -23.127376556396484,
      "activations/layer1_attention_weight_max": 17.74256134033203,
      "activations/layer1_attention_weight_min": -16.4105167388916,
      "activations/layer20_attention_weight_max": 24.792430877685547,
      "activations/layer20_attention_weight_min": -22.486448287963867,
      "activations/layer21_attention_weight_max": 36.419734954833984,
      "activations/layer21_attention_weight_min": -21.45467758178711,
      "activations/layer22_attention_weight_max": 33.50182342529297,
      "activations/layer22_attention_weight_min": -24.83837127685547,
      "activations/layer23_attention_weight_max": 37.23415756225586,
      "activations/layer23_attention_weight_min": -23.553380966186523,
      "activations/layer2_attention_weight_max": 35.161685943603516,
      "activations/layer2_attention_weight_min": -31.555997848510742,
      "activations/layer3_attention_weight_max": 93.53197479248047,
      "activations/layer3_attention_weight_min": -95.70700073242188,
      "activations/layer4_attention_weight_max": 98.87495422363281,
      "activations/layer4_attention_weight_min": -96.1488037109375,
      "activations/layer5_attention_weight_max": 78.5521469116211,
      "activations/layer5_attention_weight_min": -70.48785400390625,
      "activations/layer6_attention_weight_max": 51.11899948120117,
      "activations/layer6_attention_weight_min": -50.62783432006836,
      "activations/layer7_attention_weight_max": 91.29380798339844,
      "activations/layer7_attention_weight_min": -75.3631591796875,
      "activations/layer8_attention_weight_max": 55.054840087890625,
      "activations/layer8_attention_weight_min": -45.65803146362305,
      "activations/layer9_attention_weight_max": 66.73104858398438,
      "activations/layer9_attention_weight_min": -53.34368896484375,
      "epoch": 10.76,
      "learning_rate": 8.140530303030302e-05,
      "loss": 2.791,
      "step": 185250
    },
    {
      "activations/layer0_attention_weight_max": 15.43438720703125,
      "activations/layer0_attention_weight_min": -13.68567943572998,
      "activations/layer10_attention_weight_max": 38.71499252319336,
      "activations/layer10_attention_weight_min": -36.640933990478516,
      "activations/layer11_attention_weight_max": 32.751766204833984,
      "activations/layer11_attention_weight_min": -32.28854751586914,
      "activations/layer12_attention_weight_max": 19.650808334350586,
      "activations/layer12_attention_weight_min": -24.55341148376465,
      "activations/layer13_attention_weight_max": 45.68785095214844,
      "activations/layer13_attention_weight_min": -30.68340492248535,
      "activations/layer14_attention_weight_max": 47.00224304199219,
      "activations/layer14_attention_weight_min": -29.817716598510742,
      "activations/layer15_attention_weight_max": 40.87040328979492,
      "activations/layer15_attention_weight_min": -28.65789794921875,
      "activations/layer16_attention_weight_max": 35.81714630126953,
      "activations/layer16_attention_weight_min": -30.26897430419922,
      "activations/layer17_attention_weight_max": 58.90003967285156,
      "activations/layer17_attention_weight_min": -45.76202392578125,
      "activations/layer18_attention_weight_max": 55.527435302734375,
      "activations/layer18_attention_weight_min": -38.8308219909668,
      "activations/layer19_attention_weight_max": 24.87523651123047,
      "activations/layer19_attention_weight_min": -22.84235191345215,
      "activations/layer1_attention_weight_max": 15.793493270874023,
      "activations/layer1_attention_weight_min": -13.774922370910645,
      "activations/layer20_attention_weight_max": 31.285484313964844,
      "activations/layer20_attention_weight_min": -22.81673812866211,
      "activations/layer21_attention_weight_max": 53.63138961791992,
      "activations/layer21_attention_weight_min": -22.090364456176758,
      "activations/layer22_attention_weight_max": 41.95888137817383,
      "activations/layer22_attention_weight_min": -27.828205108642578,
      "activations/layer23_attention_weight_max": 45.191612243652344,
      "activations/layer23_attention_weight_min": -21.379653930664062,
      "activations/layer2_attention_weight_max": 30.907955169677734,
      "activations/layer2_attention_weight_min": -29.747005462646484,
      "activations/layer3_attention_weight_max": 87.08821868896484,
      "activations/layer3_attention_weight_min": -89.8812484741211,
      "activations/layer4_attention_weight_max": 95.34590148925781,
      "activations/layer4_attention_weight_min": -92.7137222290039,
      "activations/layer5_attention_weight_max": 70.58758544921875,
      "activations/layer5_attention_weight_min": -76.55722045898438,
      "activations/layer6_attention_weight_max": 53.98833084106445,
      "activations/layer6_attention_weight_min": -52.66851043701172,
      "activations/layer7_attention_weight_max": 72.41565704345703,
      "activations/layer7_attention_weight_min": -71.9200668334961,
      "activations/layer8_attention_weight_max": 48.14972686767578,
      "activations/layer8_attention_weight_min": -47.35129928588867,
      "activations/layer9_attention_weight_max": 46.84580993652344,
      "activations/layer9_attention_weight_min": -48.130741119384766,
      "epoch": 10.77,
      "learning_rate": 8.138636363636364e-05,
      "loss": 2.793,
      "step": 185300
    },
    {
      "activations/layer0_attention_weight_max": 15.256965637207031,
      "activations/layer0_attention_weight_min": -14.745477676391602,
      "activations/layer10_attention_weight_max": 39.695274353027344,
      "activations/layer10_attention_weight_min": -36.70188903808594,
      "activations/layer11_attention_weight_max": 33.578495025634766,
      "activations/layer11_attention_weight_min": -34.647945404052734,
      "activations/layer12_attention_weight_max": 19.940284729003906,
      "activations/layer12_attention_weight_min": -24.73886489868164,
      "activations/layer13_attention_weight_max": 42.16896438598633,
      "activations/layer13_attention_weight_min": -30.03843879699707,
      "activations/layer14_attention_weight_max": 41.008941650390625,
      "activations/layer14_attention_weight_min": -31.97825813293457,
      "activations/layer15_attention_weight_max": 36.83282470703125,
      "activations/layer15_attention_weight_min": -28.687232971191406,
      "activations/layer16_attention_weight_max": 31.05088233947754,
      "activations/layer16_attention_weight_min": -28.90653419494629,
      "activations/layer17_attention_weight_max": 56.15481185913086,
      "activations/layer17_attention_weight_min": -46.02582931518555,
      "activations/layer18_attention_weight_max": 49.72736358642578,
      "activations/layer18_attention_weight_min": -39.83357620239258,
      "activations/layer19_attention_weight_max": 21.49648094177246,
      "activations/layer19_attention_weight_min": -23.10993003845215,
      "activations/layer1_attention_weight_max": 17.020090103149414,
      "activations/layer1_attention_weight_min": -17.680997848510742,
      "activations/layer20_attention_weight_max": 22.323055267333984,
      "activations/layer20_attention_weight_min": -23.182498931884766,
      "activations/layer21_attention_weight_max": 44.15445327758789,
      "activations/layer21_attention_weight_min": -24.67150115966797,
      "activations/layer22_attention_weight_max": 29.917436599731445,
      "activations/layer22_attention_weight_min": -26.434988021850586,
      "activations/layer23_attention_weight_max": 41.93466567993164,
      "activations/layer23_attention_weight_min": -27.36589813232422,
      "activations/layer2_attention_weight_max": 33.44800567626953,
      "activations/layer2_attention_weight_min": -32.97984313964844,
      "activations/layer3_attention_weight_max": 96.34465789794922,
      "activations/layer3_attention_weight_min": -102.14497375488281,
      "activations/layer4_attention_weight_max": 101.7020492553711,
      "activations/layer4_attention_weight_min": -95.99134063720703,
      "activations/layer5_attention_weight_max": 75.81956481933594,
      "activations/layer5_attention_weight_min": -75.92146301269531,
      "activations/layer6_attention_weight_max": 54.01213073730469,
      "activations/layer6_attention_weight_min": -52.09955596923828,
      "activations/layer7_attention_weight_max": 74.80876159667969,
      "activations/layer7_attention_weight_min": -73.86460876464844,
      "activations/layer8_attention_weight_max": 44.94154357910156,
      "activations/layer8_attention_weight_min": -47.7935791015625,
      "activations/layer9_attention_weight_max": 47.77425003051758,
      "activations/layer9_attention_weight_min": -49.448272705078125,
      "epoch": 10.77,
      "learning_rate": 8.136742424242423e-05,
      "loss": 2.7899,
      "step": 185350
    },
    {
      "activations/layer0_attention_weight_max": 15.714129447937012,
      "activations/layer0_attention_weight_min": -14.143150329589844,
      "activations/layer10_attention_weight_max": 39.355934143066406,
      "activations/layer10_attention_weight_min": -37.738433837890625,
      "activations/layer11_attention_weight_max": 35.77689743041992,
      "activations/layer11_attention_weight_min": -34.10639190673828,
      "activations/layer12_attention_weight_max": 16.84905242919922,
      "activations/layer12_attention_weight_min": -27.442262649536133,
      "activations/layer13_attention_weight_max": 39.79122543334961,
      "activations/layer13_attention_weight_min": -29.175704956054688,
      "activations/layer14_attention_weight_max": 41.127647399902344,
      "activations/layer14_attention_weight_min": -32.31972122192383,
      "activations/layer15_attention_weight_max": 35.76067352294922,
      "activations/layer15_attention_weight_min": -29.622896194458008,
      "activations/layer16_attention_weight_max": 33.88112258911133,
      "activations/layer16_attention_weight_min": -29.713665008544922,
      "activations/layer17_attention_weight_max": 56.60615158081055,
      "activations/layer17_attention_weight_min": -47.29878616333008,
      "activations/layer18_attention_weight_max": 52.27985763549805,
      "activations/layer18_attention_weight_min": -42.09572982788086,
      "activations/layer19_attention_weight_max": 25.323015213012695,
      "activations/layer19_attention_weight_min": -25.72812843322754,
      "activations/layer1_attention_weight_max": 16.222061157226562,
      "activations/layer1_attention_weight_min": -16.10524559020996,
      "activations/layer20_attention_weight_max": 27.07965660095215,
      "activations/layer20_attention_weight_min": -24.274900436401367,
      "activations/layer21_attention_weight_max": 44.39692687988281,
      "activations/layer21_attention_weight_min": -22.73206329345703,
      "activations/layer22_attention_weight_max": 35.886810302734375,
      "activations/layer22_attention_weight_min": -27.4486141204834,
      "activations/layer23_attention_weight_max": 40.61421203613281,
      "activations/layer23_attention_weight_min": -24.250247955322266,
      "activations/layer2_attention_weight_max": 33.907718658447266,
      "activations/layer2_attention_weight_min": -31.68111801147461,
      "activations/layer3_attention_weight_max": 95.71585083007812,
      "activations/layer3_attention_weight_min": -95.97234344482422,
      "activations/layer4_attention_weight_max": 102.15237426757812,
      "activations/layer4_attention_weight_min": -95.86297607421875,
      "activations/layer5_attention_weight_max": 73.03038024902344,
      "activations/layer5_attention_weight_min": -78.61174011230469,
      "activations/layer6_attention_weight_max": 53.30314636230469,
      "activations/layer6_attention_weight_min": -51.52906036376953,
      "activations/layer7_attention_weight_max": 76.6903076171875,
      "activations/layer7_attention_weight_min": -72.34693145751953,
      "activations/layer8_attention_weight_max": 46.41478729248047,
      "activations/layer8_attention_weight_min": -48.41765213012695,
      "activations/layer9_attention_weight_max": 48.824180603027344,
      "activations/layer9_attention_weight_min": -47.54295349121094,
      "epoch": 10.77,
      "learning_rate": 8.134848484848484e-05,
      "loss": 2.8004,
      "step": 185400
    },
    {
      "activations/layer0_attention_weight_max": 15.973044395446777,
      "activations/layer0_attention_weight_min": -13.983997344970703,
      "activations/layer10_attention_weight_max": 39.04835510253906,
      "activations/layer10_attention_weight_min": -39.42351150512695,
      "activations/layer11_attention_weight_max": 35.86163330078125,
      "activations/layer11_attention_weight_min": -34.74095916748047,
      "activations/layer12_attention_weight_max": 18.01535415649414,
      "activations/layer12_attention_weight_min": -33.75299835205078,
      "activations/layer13_attention_weight_max": 51.54233169555664,
      "activations/layer13_attention_weight_min": -36.181846618652344,
      "activations/layer14_attention_weight_max": 46.47978210449219,
      "activations/layer14_attention_weight_min": -32.75579071044922,
      "activations/layer15_attention_weight_max": 40.609676361083984,
      "activations/layer15_attention_weight_min": -30.29705238342285,
      "activations/layer16_attention_weight_max": 31.698856353759766,
      "activations/layer16_attention_weight_min": -29.936420440673828,
      "activations/layer17_attention_weight_max": 51.65956115722656,
      "activations/layer17_attention_weight_min": -44.907508850097656,
      "activations/layer18_attention_weight_max": 46.91608810424805,
      "activations/layer18_attention_weight_min": -39.94878005981445,
      "activations/layer19_attention_weight_max": 21.58972930908203,
      "activations/layer19_attention_weight_min": -23.350921630859375,
      "activations/layer1_attention_weight_max": 16.46126937866211,
      "activations/layer1_attention_weight_min": -15.194058418273926,
      "activations/layer20_attention_weight_max": 23.951406478881836,
      "activations/layer20_attention_weight_min": -23.751768112182617,
      "activations/layer21_attention_weight_max": 44.36967849731445,
      "activations/layer21_attention_weight_min": -24.16497230529785,
      "activations/layer22_attention_weight_max": 29.671873092651367,
      "activations/layer22_attention_weight_min": -25.466001510620117,
      "activations/layer23_attention_weight_max": 38.63325881958008,
      "activations/layer23_attention_weight_min": -23.112201690673828,
      "activations/layer2_attention_weight_max": 32.51519012451172,
      "activations/layer2_attention_weight_min": -32.00519943237305,
      "activations/layer3_attention_weight_max": 90.02633666992188,
      "activations/layer3_attention_weight_min": -91.49054718017578,
      "activations/layer4_attention_weight_max": 100.39041900634766,
      "activations/layer4_attention_weight_min": -94.43867492675781,
      "activations/layer5_attention_weight_max": 73.3484115600586,
      "activations/layer5_attention_weight_min": -74.29969024658203,
      "activations/layer6_attention_weight_max": 54.38651657104492,
      "activations/layer6_attention_weight_min": -53.913169860839844,
      "activations/layer7_attention_weight_max": 80.08588409423828,
      "activations/layer7_attention_weight_min": -78.42250061035156,
      "activations/layer8_attention_weight_max": 49.61780548095703,
      "activations/layer8_attention_weight_min": -49.019596099853516,
      "activations/layer9_attention_weight_max": 59.56433868408203,
      "activations/layer9_attention_weight_min": -51.73651123046875,
      "epoch": 10.78,
      "learning_rate": 8.132954545454546e-05,
      "loss": 2.7869,
      "step": 185450
    },
    {
      "activations/layer0_attention_weight_max": 15.978292465209961,
      "activations/layer0_attention_weight_min": -14.289426803588867,
      "activations/layer10_attention_weight_max": 46.187469482421875,
      "activations/layer10_attention_weight_min": -44.594078063964844,
      "activations/layer11_attention_weight_max": 44.70379638671875,
      "activations/layer11_attention_weight_min": -39.254478454589844,
      "activations/layer12_attention_weight_max": 21.32798194885254,
      "activations/layer12_attention_weight_min": -24.69873046875,
      "activations/layer13_attention_weight_max": 78.29331970214844,
      "activations/layer13_attention_weight_min": -44.59299087524414,
      "activations/layer14_attention_weight_max": 71.63204956054688,
      "activations/layer14_attention_weight_min": -35.68924331665039,
      "activations/layer15_attention_weight_max": 60.757328033447266,
      "activations/layer15_attention_weight_min": -32.129276275634766,
      "activations/layer16_attention_weight_max": 38.25020217895508,
      "activations/layer16_attention_weight_min": -32.7816047668457,
      "activations/layer17_attention_weight_max": 65.18509674072266,
      "activations/layer17_attention_weight_min": -53.88438415527344,
      "activations/layer18_attention_weight_max": 57.80642318725586,
      "activations/layer18_attention_weight_min": -46.78461837768555,
      "activations/layer19_attention_weight_max": 28.187910079956055,
      "activations/layer19_attention_weight_min": -27.734729766845703,
      "activations/layer1_attention_weight_max": 18.22478675842285,
      "activations/layer1_attention_weight_min": -16.447904586791992,
      "activations/layer20_attention_weight_max": 29.779146194458008,
      "activations/layer20_attention_weight_min": -23.159896850585938,
      "activations/layer21_attention_weight_max": 48.07014083862305,
      "activations/layer21_attention_weight_min": -24.669225692749023,
      "activations/layer22_attention_weight_max": 33.84345626831055,
      "activations/layer22_attention_weight_min": -27.073091506958008,
      "activations/layer23_attention_weight_max": 34.617591857910156,
      "activations/layer23_attention_weight_min": -25.468883514404297,
      "activations/layer2_attention_weight_max": 35.53516387939453,
      "activations/layer2_attention_weight_min": -34.20613098144531,
      "activations/layer3_attention_weight_max": 101.36616516113281,
      "activations/layer3_attention_weight_min": -102.55553436279297,
      "activations/layer4_attention_weight_max": 114.40152740478516,
      "activations/layer4_attention_weight_min": -101.24675750732422,
      "activations/layer5_attention_weight_max": 80.67388916015625,
      "activations/layer5_attention_weight_min": -75.0107421875,
      "activations/layer6_attention_weight_max": 60.122413635253906,
      "activations/layer6_attention_weight_min": -55.29609680175781,
      "activations/layer7_attention_weight_max": 87.47227478027344,
      "activations/layer7_attention_weight_min": -75.3470458984375,
      "activations/layer8_attention_weight_max": 54.71824264526367,
      "activations/layer8_attention_weight_min": -50.215087890625,
      "activations/layer9_attention_weight_max": 68.15225982666016,
      "activations/layer9_attention_weight_min": -56.376373291015625,
      "epoch": 10.78,
      "learning_rate": 8.131060606060604e-05,
      "loss": 2.7677,
      "step": 185500
    },
    {
      "activations/layer0_attention_weight_max": 14.80113697052002,
      "activations/layer0_attention_weight_min": -14.320542335510254,
      "activations/layer10_attention_weight_max": 34.4505729675293,
      "activations/layer10_attention_weight_min": -34.53883743286133,
      "activations/layer11_attention_weight_max": 31.87732696533203,
      "activations/layer11_attention_weight_min": -31.97918701171875,
      "activations/layer12_attention_weight_max": 23.79969024658203,
      "activations/layer12_attention_weight_min": -23.857755661010742,
      "activations/layer13_attention_weight_max": 39.430992126464844,
      "activations/layer13_attention_weight_min": -32.0933837890625,
      "activations/layer14_attention_weight_max": 33.8662109375,
      "activations/layer14_attention_weight_min": -31.440086364746094,
      "activations/layer15_attention_weight_max": 35.30881118774414,
      "activations/layer15_attention_weight_min": -31.208415985107422,
      "activations/layer16_attention_weight_max": 31.10838508605957,
      "activations/layer16_attention_weight_min": -29.57255744934082,
      "activations/layer17_attention_weight_max": 48.59450912475586,
      "activations/layer17_attention_weight_min": -40.60636520385742,
      "activations/layer18_attention_weight_max": 43.76182174682617,
      "activations/layer18_attention_weight_min": -36.2293815612793,
      "activations/layer19_attention_weight_max": 23.58755111694336,
      "activations/layer19_attention_weight_min": -20.5814208984375,
      "activations/layer1_attention_weight_max": 16.652427673339844,
      "activations/layer1_attention_weight_min": -16.076007843017578,
      "activations/layer20_attention_weight_max": 21.750638961791992,
      "activations/layer20_attention_weight_min": -22.57691192626953,
      "activations/layer21_attention_weight_max": 36.19904708862305,
      "activations/layer21_attention_weight_min": -26.900060653686523,
      "activations/layer22_attention_weight_max": 31.94666862487793,
      "activations/layer22_attention_weight_min": -24.521224975585938,
      "activations/layer23_attention_weight_max": 35.24342346191406,
      "activations/layer23_attention_weight_min": -24.143455505371094,
      "activations/layer2_attention_weight_max": 34.70252227783203,
      "activations/layer2_attention_weight_min": -33.83441162109375,
      "activations/layer3_attention_weight_max": 99.8506088256836,
      "activations/layer3_attention_weight_min": -107.15594482421875,
      "activations/layer4_attention_weight_max": 105.27619934082031,
      "activations/layer4_attention_weight_min": -98.33293914794922,
      "activations/layer5_attention_weight_max": 76.62283325195312,
      "activations/layer5_attention_weight_min": -78.19122314453125,
      "activations/layer6_attention_weight_max": 56.736656188964844,
      "activations/layer6_attention_weight_min": -57.23422622680664,
      "activations/layer7_attention_weight_max": 78.70179748535156,
      "activations/layer7_attention_weight_min": -75.34410095214844,
      "activations/layer8_attention_weight_max": 46.05487060546875,
      "activations/layer8_attention_weight_min": -47.771942138671875,
      "activations/layer9_attention_weight_max": 49.00922775268555,
      "activations/layer9_attention_weight_min": -45.829322814941406,
      "epoch": 10.78,
      "learning_rate": 8.129166666666666e-05,
      "loss": 2.7638,
      "step": 185550
    },
    {
      "activations/layer0_attention_weight_max": 15.088095664978027,
      "activations/layer0_attention_weight_min": -14.659355163574219,
      "activations/layer10_attention_weight_max": 41.84147644042969,
      "activations/layer10_attention_weight_min": -41.800437927246094,
      "activations/layer11_attention_weight_max": 36.75511169433594,
      "activations/layer11_attention_weight_min": -35.15350341796875,
      "activations/layer12_attention_weight_max": 18.352828979492188,
      "activations/layer12_attention_weight_min": -25.358503341674805,
      "activations/layer13_attention_weight_max": 44.64929962158203,
      "activations/layer13_attention_weight_min": -33.349578857421875,
      "activations/layer14_attention_weight_max": 44.04804229736328,
      "activations/layer14_attention_weight_min": -31.238859176635742,
      "activations/layer15_attention_weight_max": 35.84794235229492,
      "activations/layer15_attention_weight_min": -30.299518585205078,
      "activations/layer16_attention_weight_max": 30.88085174560547,
      "activations/layer16_attention_weight_min": -27.684585571289062,
      "activations/layer17_attention_weight_max": 50.87910842895508,
      "activations/layer17_attention_weight_min": -42.30797576904297,
      "activations/layer18_attention_weight_max": 45.92807388305664,
      "activations/layer18_attention_weight_min": -36.59624099731445,
      "activations/layer19_attention_weight_max": 20.614423751831055,
      "activations/layer19_attention_weight_min": -23.70577049255371,
      "activations/layer1_attention_weight_max": 16.884389877319336,
      "activations/layer1_attention_weight_min": -15.441890716552734,
      "activations/layer20_attention_weight_max": 22.01421356201172,
      "activations/layer20_attention_weight_min": -22.847118377685547,
      "activations/layer21_attention_weight_max": 42.27336502075195,
      "activations/layer21_attention_weight_min": -24.954893112182617,
      "activations/layer22_attention_weight_max": 29.491256713867188,
      "activations/layer22_attention_weight_min": -24.604001998901367,
      "activations/layer23_attention_weight_max": 35.633365631103516,
      "activations/layer23_attention_weight_min": -21.817502975463867,
      "activations/layer2_attention_weight_max": 33.784584045410156,
      "activations/layer2_attention_weight_min": -31.611194610595703,
      "activations/layer3_attention_weight_max": 92.2685317993164,
      "activations/layer3_attention_weight_min": -93.23222351074219,
      "activations/layer4_attention_weight_max": 97.9538803100586,
      "activations/layer4_attention_weight_min": -98.89781188964844,
      "activations/layer5_attention_weight_max": 72.81580352783203,
      "activations/layer5_attention_weight_min": -80.20457458496094,
      "activations/layer6_attention_weight_max": 54.08222579956055,
      "activations/layer6_attention_weight_min": -54.34923553466797,
      "activations/layer7_attention_weight_max": 77.70349884033203,
      "activations/layer7_attention_weight_min": -74.0405044555664,
      "activations/layer8_attention_weight_max": 50.383758544921875,
      "activations/layer8_attention_weight_min": -51.46651077270508,
      "activations/layer9_attention_weight_max": 54.675899505615234,
      "activations/layer9_attention_weight_min": -52.97351837158203,
      "epoch": 10.78,
      "learning_rate": 8.127272727272727e-05,
      "loss": 2.7782,
      "step": 185600
    },
    {
      "activations/layer0_attention_weight_max": 15.433510780334473,
      "activations/layer0_attention_weight_min": -14.257917404174805,
      "activations/layer10_attention_weight_max": 38.68645095825195,
      "activations/layer10_attention_weight_min": -37.365753173828125,
      "activations/layer11_attention_weight_max": 33.351051330566406,
      "activations/layer11_attention_weight_min": -33.35603713989258,
      "activations/layer12_attention_weight_max": 22.7445011138916,
      "activations/layer12_attention_weight_min": -24.74321937561035,
      "activations/layer13_attention_weight_max": 46.855430603027344,
      "activations/layer13_attention_weight_min": -33.51219177246094,
      "activations/layer14_attention_weight_max": 61.97807312011719,
      "activations/layer14_attention_weight_min": -32.5657958984375,
      "activations/layer15_attention_weight_max": 45.512630462646484,
      "activations/layer15_attention_weight_min": -31.92502212524414,
      "activations/layer16_attention_weight_max": 35.94676971435547,
      "activations/layer16_attention_weight_min": -32.43104934692383,
      "activations/layer17_attention_weight_max": 59.45368194580078,
      "activations/layer17_attention_weight_min": -48.60909652709961,
      "activations/layer18_attention_weight_max": 59.28858184814453,
      "activations/layer18_attention_weight_min": -42.4926872253418,
      "activations/layer19_attention_weight_max": 30.552139282226562,
      "activations/layer19_attention_weight_min": -23.83715057373047,
      "activations/layer1_attention_weight_max": 17.446697235107422,
      "activations/layer1_attention_weight_min": -14.49508285522461,
      "activations/layer20_attention_weight_max": 33.04928970336914,
      "activations/layer20_attention_weight_min": -21.54882049560547,
      "activations/layer21_attention_weight_max": 51.79668045043945,
      "activations/layer21_attention_weight_min": -24.7801570892334,
      "activations/layer22_attention_weight_max": 39.1197395324707,
      "activations/layer22_attention_weight_min": -25.661945343017578,
      "activations/layer23_attention_weight_max": 45.37098693847656,
      "activations/layer23_attention_weight_min": -22.583707809448242,
      "activations/layer2_attention_weight_max": 30.73206329345703,
      "activations/layer2_attention_weight_min": -28.416908264160156,
      "activations/layer3_attention_weight_max": 89.10938262939453,
      "activations/layer3_attention_weight_min": -89.95277404785156,
      "activations/layer4_attention_weight_max": 101.3108139038086,
      "activations/layer4_attention_weight_min": -91.63201141357422,
      "activations/layer5_attention_weight_max": 74.51089477539062,
      "activations/layer5_attention_weight_min": -76.08642578125,
      "activations/layer6_attention_weight_max": 56.46459197998047,
      "activations/layer6_attention_weight_min": -55.04851150512695,
      "activations/layer7_attention_weight_max": 76.22571563720703,
      "activations/layer7_attention_weight_min": -77.1966552734375,
      "activations/layer8_attention_weight_max": 48.16870880126953,
      "activations/layer8_attention_weight_min": -48.17918014526367,
      "activations/layer9_attention_weight_max": 50.817317962646484,
      "activations/layer9_attention_weight_min": -52.10650634765625,
      "epoch": 10.79,
      "learning_rate": 8.125378787878786e-05,
      "loss": 2.7747,
      "step": 185650
    },
    {
      "activations/layer0_attention_weight_max": 15.300165176391602,
      "activations/layer0_attention_weight_min": -15.041836738586426,
      "activations/layer10_attention_weight_max": 38.22629928588867,
      "activations/layer10_attention_weight_min": -35.70329666137695,
      "activations/layer11_attention_weight_max": 31.150997161865234,
      "activations/layer11_attention_weight_min": -32.188987731933594,
      "activations/layer12_attention_weight_max": 25.29374885559082,
      "activations/layer12_attention_weight_min": -24.61912727355957,
      "activations/layer13_attention_weight_max": 49.55641174316406,
      "activations/layer13_attention_weight_min": -34.109256744384766,
      "activations/layer14_attention_weight_max": 45.12761688232422,
      "activations/layer14_attention_weight_min": -29.815509796142578,
      "activations/layer15_attention_weight_max": 35.48428726196289,
      "activations/layer15_attention_weight_min": -27.798364639282227,
      "activations/layer16_attention_weight_max": 30.42970848083496,
      "activations/layer16_attention_weight_min": -29.02142333984375,
      "activations/layer17_attention_weight_max": 56.518028259277344,
      "activations/layer17_attention_weight_min": -43.559852600097656,
      "activations/layer18_attention_weight_max": 46.22703170776367,
      "activations/layer18_attention_weight_min": -35.02931594848633,
      "activations/layer19_attention_weight_max": 22.816083908081055,
      "activations/layer19_attention_weight_min": -20.507888793945312,
      "activations/layer1_attention_weight_max": 17.83877182006836,
      "activations/layer1_attention_weight_min": -13.467098236083984,
      "activations/layer20_attention_weight_max": 20.101959228515625,
      "activations/layer20_attention_weight_min": -21.88262176513672,
      "activations/layer21_attention_weight_max": 39.24798583984375,
      "activations/layer21_attention_weight_min": -24.44045066833496,
      "activations/layer22_attention_weight_max": 28.65918731689453,
      "activations/layer22_attention_weight_min": -23.89354705810547,
      "activations/layer23_attention_weight_max": 34.202674865722656,
      "activations/layer23_attention_weight_min": -21.404159545898438,
      "activations/layer2_attention_weight_max": 30.75018310546875,
      "activations/layer2_attention_weight_min": -29.675457000732422,
      "activations/layer3_attention_weight_max": 88.91027069091797,
      "activations/layer3_attention_weight_min": -88.74674224853516,
      "activations/layer4_attention_weight_max": 99.99568176269531,
      "activations/layer4_attention_weight_min": -97.92646026611328,
      "activations/layer5_attention_weight_max": 73.14595794677734,
      "activations/layer5_attention_weight_min": -81.48698425292969,
      "activations/layer6_attention_weight_max": 55.05942916870117,
      "activations/layer6_attention_weight_min": -55.30118942260742,
      "activations/layer7_attention_weight_max": 74.26830291748047,
      "activations/layer7_attention_weight_min": -73.56661987304688,
      "activations/layer8_attention_weight_max": 44.86741638183594,
      "activations/layer8_attention_weight_min": -49.19928741455078,
      "activations/layer9_attention_weight_max": 49.68476486206055,
      "activations/layer9_attention_weight_min": -48.468502044677734,
      "epoch": 10.79,
      "learning_rate": 8.123484848484848e-05,
      "loss": 2.7914,
      "step": 185700
    },
    {
      "activations/layer0_attention_weight_max": 15.432353973388672,
      "activations/layer0_attention_weight_min": -14.245613098144531,
      "activations/layer10_attention_weight_max": 40.103031158447266,
      "activations/layer10_attention_weight_min": -36.55973434448242,
      "activations/layer11_attention_weight_max": 35.035465240478516,
      "activations/layer11_attention_weight_min": -34.26810073852539,
      "activations/layer12_attention_weight_max": 27.108360290527344,
      "activations/layer12_attention_weight_min": -29.823217391967773,
      "activations/layer13_attention_weight_max": 42.639522552490234,
      "activations/layer13_attention_weight_min": -30.450273513793945,
      "activations/layer14_attention_weight_max": 40.95668411254883,
      "activations/layer14_attention_weight_min": -30.54018783569336,
      "activations/layer15_attention_weight_max": 35.43104553222656,
      "activations/layer15_attention_weight_min": -29.008928298950195,
      "activations/layer16_attention_weight_max": 35.22673797607422,
      "activations/layer16_attention_weight_min": -31.625980377197266,
      "activations/layer17_attention_weight_max": 55.88103103637695,
      "activations/layer17_attention_weight_min": -46.04119110107422,
      "activations/layer18_attention_weight_max": 52.8174934387207,
      "activations/layer18_attention_weight_min": -38.91768264770508,
      "activations/layer19_attention_weight_max": 24.90754508972168,
      "activations/layer19_attention_weight_min": -22.448354721069336,
      "activations/layer1_attention_weight_max": 17.386783599853516,
      "activations/layer1_attention_weight_min": -15.721619606018066,
      "activations/layer20_attention_weight_max": 24.72087860107422,
      "activations/layer20_attention_weight_min": -20.51068115234375,
      "activations/layer21_attention_weight_max": 41.930870056152344,
      "activations/layer21_attention_weight_min": -21.915124893188477,
      "activations/layer22_attention_weight_max": 35.00154113769531,
      "activations/layer22_attention_weight_min": -24.677183151245117,
      "activations/layer23_attention_weight_max": 47.29701232910156,
      "activations/layer23_attention_weight_min": -22.590009689331055,
      "activations/layer2_attention_weight_max": 33.51959228515625,
      "activations/layer2_attention_weight_min": -31.274925231933594,
      "activations/layer3_attention_weight_max": 87.32682037353516,
      "activations/layer3_attention_weight_min": -92.52521514892578,
      "activations/layer4_attention_weight_max": 99.72758483886719,
      "activations/layer4_attention_weight_min": -95.45478057861328,
      "activations/layer5_attention_weight_max": 72.5260009765625,
      "activations/layer5_attention_weight_min": -76.639404296875,
      "activations/layer6_attention_weight_max": 54.54410171508789,
      "activations/layer6_attention_weight_min": -55.20625686645508,
      "activations/layer7_attention_weight_max": 75.07626342773438,
      "activations/layer7_attention_weight_min": -70.35029602050781,
      "activations/layer8_attention_weight_max": 48.738304138183594,
      "activations/layer8_attention_weight_min": -48.90817642211914,
      "activations/layer9_attention_weight_max": 51.765377044677734,
      "activations/layer9_attention_weight_min": -51.16515350341797,
      "epoch": 10.79,
      "learning_rate": 8.121590909090908e-05,
      "loss": 2.7784,
      "step": 185750
    },
    {
      "activations/layer0_attention_weight_max": 16.120969772338867,
      "activations/layer0_attention_weight_min": -14.933931350708008,
      "activations/layer10_attention_weight_max": 36.95886993408203,
      "activations/layer10_attention_weight_min": -36.160560607910156,
      "activations/layer11_attention_weight_max": 30.75814437866211,
      "activations/layer11_attention_weight_min": -31.702362060546875,
      "activations/layer12_attention_weight_max": 16.11390495300293,
      "activations/layer12_attention_weight_min": -23.246356964111328,
      "activations/layer13_attention_weight_max": 35.654964447021484,
      "activations/layer13_attention_weight_min": -29.17453384399414,
      "activations/layer14_attention_weight_max": 32.59907150268555,
      "activations/layer14_attention_weight_min": -29.751861572265625,
      "activations/layer15_attention_weight_max": 31.467334747314453,
      "activations/layer15_attention_weight_min": -32.62363052368164,
      "activations/layer16_attention_weight_max": 29.163177490234375,
      "activations/layer16_attention_weight_min": -27.704444885253906,
      "activations/layer17_attention_weight_max": 54.16921615600586,
      "activations/layer17_attention_weight_min": -40.78091812133789,
      "activations/layer18_attention_weight_max": 43.23116683959961,
      "activations/layer18_attention_weight_min": -35.419891357421875,
      "activations/layer19_attention_weight_max": 19.978675842285156,
      "activations/layer19_attention_weight_min": -20.49452781677246,
      "activations/layer1_attention_weight_max": 17.985891342163086,
      "activations/layer1_attention_weight_min": -15.593076705932617,
      "activations/layer20_attention_weight_max": 20.046524047851562,
      "activations/layer20_attention_weight_min": -25.114768981933594,
      "activations/layer21_attention_weight_max": 32.53349685668945,
      "activations/layer21_attention_weight_min": -22.88959312438965,
      "activations/layer22_attention_weight_max": 30.501388549804688,
      "activations/layer22_attention_weight_min": -25.173860549926758,
      "activations/layer23_attention_weight_max": 33.05519104003906,
      "activations/layer23_attention_weight_min": -23.20138168334961,
      "activations/layer2_attention_weight_max": 33.10828399658203,
      "activations/layer2_attention_weight_min": -31.572187423706055,
      "activations/layer3_attention_weight_max": 92.84441375732422,
      "activations/layer3_attention_weight_min": -99.28353881835938,
      "activations/layer4_attention_weight_max": 100.63216400146484,
      "activations/layer4_attention_weight_min": -97.5225601196289,
      "activations/layer5_attention_weight_max": 73.6041030883789,
      "activations/layer5_attention_weight_min": -74.57026672363281,
      "activations/layer6_attention_weight_max": 56.43718338012695,
      "activations/layer6_attention_weight_min": -53.77361297607422,
      "activations/layer7_attention_weight_max": 72.68731689453125,
      "activations/layer7_attention_weight_min": -73.47993469238281,
      "activations/layer8_attention_weight_max": 46.22810745239258,
      "activations/layer8_attention_weight_min": -47.697696685791016,
      "activations/layer9_attention_weight_max": 46.302337646484375,
      "activations/layer9_attention_weight_min": -49.75231170654297,
      "epoch": 10.8,
      "learning_rate": 8.11969696969697e-05,
      "loss": 2.7803,
      "step": 185800
    },
    {
      "activations/layer0_attention_weight_max": 14.62111759185791,
      "activations/layer0_attention_weight_min": -14.085370063781738,
      "activations/layer10_attention_weight_max": 38.919288635253906,
      "activations/layer10_attention_weight_min": -39.50111770629883,
      "activations/layer11_attention_weight_max": 32.73734664916992,
      "activations/layer11_attention_weight_min": -35.51587677001953,
      "activations/layer12_attention_weight_max": 18.356958389282227,
      "activations/layer12_attention_weight_min": -24.420032501220703,
      "activations/layer13_attention_weight_max": 49.04488754272461,
      "activations/layer13_attention_weight_min": -30.044363021850586,
      "activations/layer14_attention_weight_max": 59.0114860534668,
      "activations/layer14_attention_weight_min": -34.94254684448242,
      "activations/layer15_attention_weight_max": 42.448707580566406,
      "activations/layer15_attention_weight_min": -31.681344985961914,
      "activations/layer16_attention_weight_max": 33.85197067260742,
      "activations/layer16_attention_weight_min": -33.203216552734375,
      "activations/layer17_attention_weight_max": 59.97475814819336,
      "activations/layer17_attention_weight_min": -49.730751037597656,
      "activations/layer18_attention_weight_max": 53.660057067871094,
      "activations/layer18_attention_weight_min": -38.738670349121094,
      "activations/layer19_attention_weight_max": 23.545392990112305,
      "activations/layer19_attention_weight_min": -22.750234603881836,
      "activations/layer1_attention_weight_max": 16.43405532836914,
      "activations/layer1_attention_weight_min": -14.495635032653809,
      "activations/layer20_attention_weight_max": 24.41557502746582,
      "activations/layer20_attention_weight_min": -21.85439682006836,
      "activations/layer21_attention_weight_max": 42.83583068847656,
      "activations/layer21_attention_weight_min": -23.3464298248291,
      "activations/layer22_attention_weight_max": 35.584434509277344,
      "activations/layer22_attention_weight_min": -29.26738166809082,
      "activations/layer23_attention_weight_max": 43.28428649902344,
      "activations/layer23_attention_weight_min": -26.19662857055664,
      "activations/layer2_attention_weight_max": 33.26896286010742,
      "activations/layer2_attention_weight_min": -31.58698081970215,
      "activations/layer3_attention_weight_max": 90.0644760131836,
      "activations/layer3_attention_weight_min": -93.05823516845703,
      "activations/layer4_attention_weight_max": 95.23472595214844,
      "activations/layer4_attention_weight_min": -92.8259506225586,
      "activations/layer5_attention_weight_max": 71.18096160888672,
      "activations/layer5_attention_weight_min": -77.3231430053711,
      "activations/layer6_attention_weight_max": 54.539695739746094,
      "activations/layer6_attention_weight_min": -53.13056564331055,
      "activations/layer7_attention_weight_max": 74.85781860351562,
      "activations/layer7_attention_weight_min": -75.28312683105469,
      "activations/layer8_attention_weight_max": 48.02553176879883,
      "activations/layer8_attention_weight_min": -48.50777053833008,
      "activations/layer9_attention_weight_max": 50.614444732666016,
      "activations/layer9_attention_weight_min": -52.081581115722656,
      "epoch": 10.8,
      "learning_rate": 8.11780303030303e-05,
      "loss": 2.782,
      "step": 185850
    },
    {
      "activations/layer0_attention_weight_max": 15.915324211120605,
      "activations/layer0_attention_weight_min": -14.024521827697754,
      "activations/layer10_attention_weight_max": 35.71156311035156,
      "activations/layer10_attention_weight_min": -35.77082061767578,
      "activations/layer11_attention_weight_max": 31.287137985229492,
      "activations/layer11_attention_weight_min": -31.917713165283203,
      "activations/layer12_attention_weight_max": 19.398609161376953,
      "activations/layer12_attention_weight_min": -30.881086349487305,
      "activations/layer13_attention_weight_max": 38.008544921875,
      "activations/layer13_attention_weight_min": -30.166629791259766,
      "activations/layer14_attention_weight_max": 36.99583053588867,
      "activations/layer14_attention_weight_min": -30.500802993774414,
      "activations/layer15_attention_weight_max": 33.455467224121094,
      "activations/layer15_attention_weight_min": -29.033771514892578,
      "activations/layer16_attention_weight_max": 30.191686630249023,
      "activations/layer16_attention_weight_min": -30.444324493408203,
      "activations/layer17_attention_weight_max": 55.0922737121582,
      "activations/layer17_attention_weight_min": -44.943180084228516,
      "activations/layer18_attention_weight_max": 47.131309509277344,
      "activations/layer18_attention_weight_min": -38.899749755859375,
      "activations/layer19_attention_weight_max": 22.302608489990234,
      "activations/layer19_attention_weight_min": -22.445898056030273,
      "activations/layer1_attention_weight_max": 17.271034240722656,
      "activations/layer1_attention_weight_min": -14.892779350280762,
      "activations/layer20_attention_weight_max": 23.65819549560547,
      "activations/layer20_attention_weight_min": -22.416170120239258,
      "activations/layer21_attention_weight_max": 42.675559997558594,
      "activations/layer21_attention_weight_min": -24.84764862060547,
      "activations/layer22_attention_weight_max": 28.95362091064453,
      "activations/layer22_attention_weight_min": -26.4168758392334,
      "activations/layer23_attention_weight_max": 39.035247802734375,
      "activations/layer23_attention_weight_min": -21.975154876708984,
      "activations/layer2_attention_weight_max": 34.17804718017578,
      "activations/layer2_attention_weight_min": -31.453794479370117,
      "activations/layer3_attention_weight_max": 95.4190673828125,
      "activations/layer3_attention_weight_min": -91.26422882080078,
      "activations/layer4_attention_weight_max": 101.1274185180664,
      "activations/layer4_attention_weight_min": -95.58483123779297,
      "activations/layer5_attention_weight_max": 74.47969818115234,
      "activations/layer5_attention_weight_min": -80.7889633178711,
      "activations/layer6_attention_weight_max": 54.3184700012207,
      "activations/layer6_attention_weight_min": -53.4769172668457,
      "activations/layer7_attention_weight_max": 76.30448150634766,
      "activations/layer7_attention_weight_min": -74.36676788330078,
      "activations/layer8_attention_weight_max": 44.78672409057617,
      "activations/layer8_attention_weight_min": -46.77810287475586,
      "activations/layer9_attention_weight_max": 48.34918212890625,
      "activations/layer9_attention_weight_min": -48.17986297607422,
      "epoch": 10.8,
      "learning_rate": 8.11590909090909e-05,
      "loss": 2.7768,
      "step": 185900
    },
    {
      "activations/layer0_attention_weight_max": 15.69672966003418,
      "activations/layer0_attention_weight_min": -14.355111122131348,
      "activations/layer10_attention_weight_max": 37.314659118652344,
      "activations/layer10_attention_weight_min": -35.25146484375,
      "activations/layer11_attention_weight_max": 32.20379638671875,
      "activations/layer11_attention_weight_min": -32.62767028808594,
      "activations/layer12_attention_weight_max": 18.655818939208984,
      "activations/layer12_attention_weight_min": -22.90997314453125,
      "activations/layer13_attention_weight_max": 35.20848846435547,
      "activations/layer13_attention_weight_min": -30.05219268798828,
      "activations/layer14_attention_weight_max": 35.71783447265625,
      "activations/layer14_attention_weight_min": -30.544763565063477,
      "activations/layer15_attention_weight_max": 33.50634765625,
      "activations/layer15_attention_weight_min": -29.321950912475586,
      "activations/layer16_attention_weight_max": 32.95563888549805,
      "activations/layer16_attention_weight_min": -32.09258270263672,
      "activations/layer17_attention_weight_max": 58.34456253051758,
      "activations/layer17_attention_weight_min": -45.01998519897461,
      "activations/layer18_attention_weight_max": 56.16614532470703,
      "activations/layer18_attention_weight_min": -39.81153106689453,
      "activations/layer19_attention_weight_max": 27.086898803710938,
      "activations/layer19_attention_weight_min": -21.727807998657227,
      "activations/layer1_attention_weight_max": 16.32663917541504,
      "activations/layer1_attention_weight_min": -14.60774040222168,
      "activations/layer20_attention_weight_max": 26.45424461364746,
      "activations/layer20_attention_weight_min": -21.95851707458496,
      "activations/layer21_attention_weight_max": 45.265594482421875,
      "activations/layer21_attention_weight_min": -22.263717651367188,
      "activations/layer22_attention_weight_max": 41.3035774230957,
      "activations/layer22_attention_weight_min": -29.138031005859375,
      "activations/layer23_attention_weight_max": 40.743064880371094,
      "activations/layer23_attention_weight_min": -24.592357635498047,
      "activations/layer2_attention_weight_max": 33.1641845703125,
      "activations/layer2_attention_weight_min": -30.27328109741211,
      "activations/layer3_attention_weight_max": 93.5206069946289,
      "activations/layer3_attention_weight_min": -92.89411163330078,
      "activations/layer4_attention_weight_max": 99.723388671875,
      "activations/layer4_attention_weight_min": -96.9832992553711,
      "activations/layer5_attention_weight_max": 78.12682342529297,
      "activations/layer5_attention_weight_min": -81.4703598022461,
      "activations/layer6_attention_weight_max": 55.73606872558594,
      "activations/layer6_attention_weight_min": -56.313011169433594,
      "activations/layer7_attention_weight_max": 74.79814147949219,
      "activations/layer7_attention_weight_min": -78.19746398925781,
      "activations/layer8_attention_weight_max": 46.4819221496582,
      "activations/layer8_attention_weight_min": -47.155704498291016,
      "activations/layer9_attention_weight_max": 50.82725143432617,
      "activations/layer9_attention_weight_min": -47.922142028808594,
      "epoch": 10.8,
      "learning_rate": 8.114015151515151e-05,
      "loss": 2.7762,
      "step": 185950
    },
    {
      "activations/layer0_attention_weight_max": 14.942317008972168,
      "activations/layer0_attention_weight_min": -14.756882667541504,
      "activations/layer10_attention_weight_max": 36.716522216796875,
      "activations/layer10_attention_weight_min": -35.962799072265625,
      "activations/layer11_attention_weight_max": 32.149742126464844,
      "activations/layer11_attention_weight_min": -31.658424377441406,
      "activations/layer12_attention_weight_max": 19.093887329101562,
      "activations/layer12_attention_weight_min": -23.2056941986084,
      "activations/layer13_attention_weight_max": 45.26725769042969,
      "activations/layer13_attention_weight_min": -33.566314697265625,
      "activations/layer14_attention_weight_max": 40.133949279785156,
      "activations/layer14_attention_weight_min": -30.014408111572266,
      "activations/layer15_attention_weight_max": 34.826438903808594,
      "activations/layer15_attention_weight_min": -27.776315689086914,
      "activations/layer16_attention_weight_max": 29.54119110107422,
      "activations/layer16_attention_weight_min": -28.09465217590332,
      "activations/layer17_attention_weight_max": 48.88832092285156,
      "activations/layer17_attention_weight_min": -44.4282112121582,
      "activations/layer18_attention_weight_max": 44.253997802734375,
      "activations/layer18_attention_weight_min": -37.786067962646484,
      "activations/layer19_attention_weight_max": 22.024906158447266,
      "activations/layer19_attention_weight_min": -21.289962768554688,
      "activations/layer1_attention_weight_max": 16.42559242248535,
      "activations/layer1_attention_weight_min": -15.46759033203125,
      "activations/layer20_attention_weight_max": 20.83081817626953,
      "activations/layer20_attention_weight_min": -23.196714401245117,
      "activations/layer21_attention_weight_max": 37.896949768066406,
      "activations/layer21_attention_weight_min": -22.437786102294922,
      "activations/layer22_attention_weight_max": 28.582727432250977,
      "activations/layer22_attention_weight_min": -25.266727447509766,
      "activations/layer23_attention_weight_max": 33.329254150390625,
      "activations/layer23_attention_weight_min": -25.572736740112305,
      "activations/layer2_attention_weight_max": 31.924041748046875,
      "activations/layer2_attention_weight_min": -30.53866195678711,
      "activations/layer3_attention_weight_max": 91.68306732177734,
      "activations/layer3_attention_weight_min": -93.82413482666016,
      "activations/layer4_attention_weight_max": 97.820068359375,
      "activations/layer4_attention_weight_min": -94.27527618408203,
      "activations/layer5_attention_weight_max": 73.57931518554688,
      "activations/layer5_attention_weight_min": -75.12681579589844,
      "activations/layer6_attention_weight_max": 52.4455680847168,
      "activations/layer6_attention_weight_min": -50.06172180175781,
      "activations/layer7_attention_weight_max": 71.63311767578125,
      "activations/layer7_attention_weight_min": -69.515625,
      "activations/layer8_attention_weight_max": 45.916770935058594,
      "activations/layer8_attention_weight_min": -46.40304946899414,
      "activations/layer9_attention_weight_max": 46.16107177734375,
      "activations/layer9_attention_weight_min": -50.40324020385742,
      "epoch": 10.81,
      "learning_rate": 8.112121212121211e-05,
      "loss": 2.7957,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_loss": 2.734375,
      "eval_runtime": 8.4867,
      "eval_samples_per_second": 505.967,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.4867,
      "eval_openwebtext_samples_per_second": 505.967,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 1.9809,
      "eval_wikitext_samples_per_second": 230.197,
      "step": 186000
    },
    {
      "epoch": 10.81,
      "eval_lambada_loss": 2.580078125,
      "eval_lambada_ppl": 13.198169226352599,
      "eval_lambada_runtime": 9.5561,
      "eval_lambada_samples_per_second": 509.518,
      "step": 186000
    },
    {
      "activations/layer0_attention_weight_max": 14.92591381072998,
      "activations/layer0_attention_weight_min": -14.629534721374512,
      "activations/layer10_attention_weight_max": 33.55024337768555,
      "activations/layer10_attention_weight_min": -32.795570373535156,
      "activations/layer11_attention_weight_max": 28.680587768554688,
      "activations/layer11_attention_weight_min": -30.82497215270996,
      "activations/layer12_attention_weight_max": 17.453590393066406,
      "activations/layer12_attention_weight_min": -27.475605010986328,
      "activations/layer13_attention_weight_max": 33.094970703125,
      "activations/layer13_attention_weight_min": -28.972061157226562,
      "activations/layer14_attention_weight_max": 34.810386657714844,
      "activations/layer14_attention_weight_min": -29.193422317504883,
      "activations/layer15_attention_weight_max": 31.518455505371094,
      "activations/layer15_attention_weight_min": -28.330381393432617,
      "activations/layer16_attention_weight_max": 29.578330993652344,
      "activations/layer16_attention_weight_min": -29.128826141357422,
      "activations/layer17_attention_weight_max": 49.720191955566406,
      "activations/layer17_attention_weight_min": -44.60660934448242,
      "activations/layer18_attention_weight_max": 42.13280487060547,
      "activations/layer18_attention_weight_min": -39.0689811706543,
      "activations/layer19_attention_weight_max": 22.466856002807617,
      "activations/layer19_attention_weight_min": -25.369298934936523,
      "activations/layer1_attention_weight_max": 16.82347297668457,
      "activations/layer1_attention_weight_min": -15.06760025024414,
      "activations/layer20_attention_weight_max": 21.0474853515625,
      "activations/layer20_attention_weight_min": -24.815635681152344,
      "activations/layer21_attention_weight_max": 37.03940963745117,
      "activations/layer21_attention_weight_min": -30.65079689025879,
      "activations/layer22_attention_weight_max": 30.34851837158203,
      "activations/layer22_attention_weight_min": -27.93309211730957,
      "activations/layer23_attention_weight_max": 33.91797637939453,
      "activations/layer23_attention_weight_min": -25.532148361206055,
      "activations/layer2_attention_weight_max": 32.81831359863281,
      "activations/layer2_attention_weight_min": -29.58379364013672,
      "activations/layer3_attention_weight_max": 87.55379486083984,
      "activations/layer3_attention_weight_min": -90.83381652832031,
      "activations/layer4_attention_weight_max": 96.6865463256836,
      "activations/layer4_attention_weight_min": -92.4914779663086,
      "activations/layer5_attention_weight_max": 71.7196044921875,
      "activations/layer5_attention_weight_min": -75.56343078613281,
      "activations/layer6_attention_weight_max": 54.2021598815918,
      "activations/layer6_attention_weight_min": -50.4060173034668,
      "activations/layer7_attention_weight_max": 74.8245620727539,
      "activations/layer7_attention_weight_min": -68.05558013916016,
      "activations/layer8_attention_weight_max": 41.16357421875,
      "activations/layer8_attention_weight_min": -45.38640213012695,
      "activations/layer9_attention_weight_max": 42.58543395996094,
      "activations/layer9_attention_weight_min": -45.912811279296875,
      "epoch": 10.81,
      "learning_rate": 8.110227272727272e-05,
      "loss": 2.791,
      "step": 186050
    },
    {
      "activations/layer0_attention_weight_max": 15.558069229125977,
      "activations/layer0_attention_weight_min": -14.773178100585938,
      "activations/layer10_attention_weight_max": 36.1868896484375,
      "activations/layer10_attention_weight_min": -36.815433502197266,
      "activations/layer11_attention_weight_max": 31.26820182800293,
      "activations/layer11_attention_weight_min": -34.7271728515625,
      "activations/layer12_attention_weight_max": 18.442611694335938,
      "activations/layer12_attention_weight_min": -26.39459991455078,
      "activations/layer13_attention_weight_max": 32.876224517822266,
      "activations/layer13_attention_weight_min": -27.550609588623047,
      "activations/layer14_attention_weight_max": 34.74855041503906,
      "activations/layer14_attention_weight_min": -31.128173828125,
      "activations/layer15_attention_weight_max": 33.21726608276367,
      "activations/layer15_attention_weight_min": -28.352426528930664,
      "activations/layer16_attention_weight_max": 29.979692459106445,
      "activations/layer16_attention_weight_min": -28.832393646240234,
      "activations/layer17_attention_weight_max": 51.027095794677734,
      "activations/layer17_attention_weight_min": -42.995235443115234,
      "activations/layer18_attention_weight_max": 44.30791473388672,
      "activations/layer18_attention_weight_min": -37.8275146484375,
      "activations/layer19_attention_weight_max": 20.66320037841797,
      "activations/layer19_attention_weight_min": -20.735990524291992,
      "activations/layer1_attention_weight_max": 16.56487274169922,
      "activations/layer1_attention_weight_min": -14.135811805725098,
      "activations/layer20_attention_weight_max": 20.91221809387207,
      "activations/layer20_attention_weight_min": -19.245437622070312,
      "activations/layer21_attention_weight_max": 31.20440673828125,
      "activations/layer21_attention_weight_min": -21.437366485595703,
      "activations/layer22_attention_weight_max": 29.657636642456055,
      "activations/layer22_attention_weight_min": -26.98113250732422,
      "activations/layer23_attention_weight_max": 34.862796783447266,
      "activations/layer23_attention_weight_min": -22.49659538269043,
      "activations/layer2_attention_weight_max": 32.199623107910156,
      "activations/layer2_attention_weight_min": -31.515790939331055,
      "activations/layer3_attention_weight_max": 92.38923645019531,
      "activations/layer3_attention_weight_min": -92.20929718017578,
      "activations/layer4_attention_weight_max": 100.71099090576172,
      "activations/layer4_attention_weight_min": -95.9581527709961,
      "activations/layer5_attention_weight_max": 72.6251220703125,
      "activations/layer5_attention_weight_min": -75.98004150390625,
      "activations/layer6_attention_weight_max": 52.7111930847168,
      "activations/layer6_attention_weight_min": -53.37816619873047,
      "activations/layer7_attention_weight_max": 77.85736083984375,
      "activations/layer7_attention_weight_min": -71.77045440673828,
      "activations/layer8_attention_weight_max": 46.624427795410156,
      "activations/layer8_attention_weight_min": -48.405033111572266,
      "activations/layer9_attention_weight_max": 51.17259979248047,
      "activations/layer9_attention_weight_min": -48.873226165771484,
      "epoch": 10.81,
      "learning_rate": 8.108333333333333e-05,
      "loss": 2.7846,
      "step": 186100
    },
    {
      "activations/layer0_attention_weight_max": 15.243508338928223,
      "activations/layer0_attention_weight_min": -14.22465991973877,
      "activations/layer10_attention_weight_max": 37.872379302978516,
      "activations/layer10_attention_weight_min": -36.440948486328125,
      "activations/layer11_attention_weight_max": 31.596492767333984,
      "activations/layer11_attention_weight_min": -33.06511688232422,
      "activations/layer12_attention_weight_max": 20.0214786529541,
      "activations/layer12_attention_weight_min": -26.16918182373047,
      "activations/layer13_attention_weight_max": 36.33007049560547,
      "activations/layer13_attention_weight_min": -28.335460662841797,
      "activations/layer14_attention_weight_max": 37.73487854003906,
      "activations/layer14_attention_weight_min": -30.897335052490234,
      "activations/layer15_attention_weight_max": 36.33409118652344,
      "activations/layer15_attention_weight_min": -31.283090591430664,
      "activations/layer16_attention_weight_max": 31.42961311340332,
      "activations/layer16_attention_weight_min": -31.038957595825195,
      "activations/layer17_attention_weight_max": 52.35036087036133,
      "activations/layer17_attention_weight_min": -44.0085334777832,
      "activations/layer18_attention_weight_max": 47.40833282470703,
      "activations/layer18_attention_weight_min": -38.29643630981445,
      "activations/layer19_attention_weight_max": 23.848752975463867,
      "activations/layer19_attention_weight_min": -22.90802574157715,
      "activations/layer1_attention_weight_max": 17.130460739135742,
      "activations/layer1_attention_weight_min": -14.847267150878906,
      "activations/layer20_attention_weight_max": 24.10610580444336,
      "activations/layer20_attention_weight_min": -21.024839401245117,
      "activations/layer21_attention_weight_max": 38.23352813720703,
      "activations/layer21_attention_weight_min": -22.744897842407227,
      "activations/layer22_attention_weight_max": 33.2857780456543,
      "activations/layer22_attention_weight_min": -26.712726593017578,
      "activations/layer23_attention_weight_max": 37.59053421020508,
      "activations/layer23_attention_weight_min": -23.883342742919922,
      "activations/layer2_attention_weight_max": 34.23579406738281,
      "activations/layer2_attention_weight_min": -32.06890869140625,
      "activations/layer3_attention_weight_max": 96.50544738769531,
      "activations/layer3_attention_weight_min": -90.98282623291016,
      "activations/layer4_attention_weight_max": 103.6109619140625,
      "activations/layer4_attention_weight_min": -96.48314666748047,
      "activations/layer5_attention_weight_max": 73.48077392578125,
      "activations/layer5_attention_weight_min": -76.69673156738281,
      "activations/layer6_attention_weight_max": 53.793128967285156,
      "activations/layer6_attention_weight_min": -52.4515266418457,
      "activations/layer7_attention_weight_max": 76.27794647216797,
      "activations/layer7_attention_weight_min": -73.05904388427734,
      "activations/layer8_attention_weight_max": 46.36368179321289,
      "activations/layer8_attention_weight_min": -48.50370788574219,
      "activations/layer9_attention_weight_max": 49.35664749145508,
      "activations/layer9_attention_weight_min": -49.69916915893555,
      "epoch": 10.82,
      "learning_rate": 8.106439393939395e-05,
      "loss": 2.7832,
      "step": 186150
    },
    {
      "activations/layer0_attention_weight_max": 15.82402229309082,
      "activations/layer0_attention_weight_min": -14.364713668823242,
      "activations/layer10_attention_weight_max": 33.84882354736328,
      "activations/layer10_attention_weight_min": -33.99074172973633,
      "activations/layer11_attention_weight_max": 29.946990966796875,
      "activations/layer11_attention_weight_min": -31.684093475341797,
      "activations/layer12_attention_weight_max": 19.192771911621094,
      "activations/layer12_attention_weight_min": -26.95939064025879,
      "activations/layer13_attention_weight_max": 35.48875045776367,
      "activations/layer13_attention_weight_min": -27.903518676757812,
      "activations/layer14_attention_weight_max": 33.64629364013672,
      "activations/layer14_attention_weight_min": -32.94563293457031,
      "activations/layer15_attention_weight_max": 30.748878479003906,
      "activations/layer15_attention_weight_min": -30.147188186645508,
      "activations/layer16_attention_weight_max": 30.625625610351562,
      "activations/layer16_attention_weight_min": -29.401687622070312,
      "activations/layer17_attention_weight_max": 48.5582389831543,
      "activations/layer17_attention_weight_min": -43.70305633544922,
      "activations/layer18_attention_weight_max": 43.52108383178711,
      "activations/layer18_attention_weight_min": -38.6162109375,
      "activations/layer19_attention_weight_max": 20.84906768798828,
      "activations/layer19_attention_weight_min": -22.629179000854492,
      "activations/layer1_attention_weight_max": 16.240772247314453,
      "activations/layer1_attention_weight_min": -14.0574951171875,
      "activations/layer20_attention_weight_max": 22.597902297973633,
      "activations/layer20_attention_weight_min": -21.531084060668945,
      "activations/layer21_attention_weight_max": 36.597625732421875,
      "activations/layer21_attention_weight_min": -22.091232299804688,
      "activations/layer22_attention_weight_max": 27.596181869506836,
      "activations/layer22_attention_weight_min": -27.07748031616211,
      "activations/layer23_attention_weight_max": 35.48976135253906,
      "activations/layer23_attention_weight_min": -21.764690399169922,
      "activations/layer2_attention_weight_max": 31.623260498046875,
      "activations/layer2_attention_weight_min": -29.797515869140625,
      "activations/layer3_attention_weight_max": 88.07688903808594,
      "activations/layer3_attention_weight_min": -88.54961395263672,
      "activations/layer4_attention_weight_max": 95.08358764648438,
      "activations/layer4_attention_weight_min": -91.83797454833984,
      "activations/layer5_attention_weight_max": 69.4128646850586,
      "activations/layer5_attention_weight_min": -77.01350402832031,
      "activations/layer6_attention_weight_max": 51.292964935302734,
      "activations/layer6_attention_weight_min": -50.332645416259766,
      "activations/layer7_attention_weight_max": 71.13525390625,
      "activations/layer7_attention_weight_min": -70.21878814697266,
      "activations/layer8_attention_weight_max": 42.96026611328125,
      "activations/layer8_attention_weight_min": -46.996978759765625,
      "activations/layer9_attention_weight_max": 44.40126037597656,
      "activations/layer9_attention_weight_min": -46.48228073120117,
      "epoch": 10.82,
      "learning_rate": 8.104545454545453e-05,
      "loss": 2.7916,
      "step": 186200
    },
    {
      "activations/layer0_attention_weight_max": 16.243871688842773,
      "activations/layer0_attention_weight_min": -14.49596881866455,
      "activations/layer10_attention_weight_max": 40.902496337890625,
      "activations/layer10_attention_weight_min": -38.025638580322266,
      "activations/layer11_attention_weight_max": 35.34907531738281,
      "activations/layer11_attention_weight_min": -33.23896789550781,
      "activations/layer12_attention_weight_max": 21.10674476623535,
      "activations/layer12_attention_weight_min": -21.988670349121094,
      "activations/layer13_attention_weight_max": 38.45792770385742,
      "activations/layer13_attention_weight_min": -28.134761810302734,
      "activations/layer14_attention_weight_max": 38.80043411254883,
      "activations/layer14_attention_weight_min": -30.30489730834961,
      "activations/layer15_attention_weight_max": 33.74401092529297,
      "activations/layer15_attention_weight_min": -28.765167236328125,
      "activations/layer16_attention_weight_max": 32.196292877197266,
      "activations/layer16_attention_weight_min": -31.087535858154297,
      "activations/layer17_attention_weight_max": 51.342689514160156,
      "activations/layer17_attention_weight_min": -45.66082763671875,
      "activations/layer18_attention_weight_max": 47.31210708618164,
      "activations/layer18_attention_weight_min": -41.04106140136719,
      "activations/layer19_attention_weight_max": 21.96394157409668,
      "activations/layer19_attention_weight_min": -25.482065200805664,
      "activations/layer1_attention_weight_max": 17.773040771484375,
      "activations/layer1_attention_weight_min": -14.497103691101074,
      "activations/layer20_attention_weight_max": 23.4337215423584,
      "activations/layer20_attention_weight_min": -22.74155616760254,
      "activations/layer21_attention_weight_max": 44.50345993041992,
      "activations/layer21_attention_weight_min": -24.627294540405273,
      "activations/layer22_attention_weight_max": 31.72959327697754,
      "activations/layer22_attention_weight_min": -25.920801162719727,
      "activations/layer23_attention_weight_max": 36.78194046020508,
      "activations/layer23_attention_weight_min": -25.327072143554688,
      "activations/layer2_attention_weight_max": 31.75322723388672,
      "activations/layer2_attention_weight_min": -30.349384307861328,
      "activations/layer3_attention_weight_max": 96.5149154663086,
      "activations/layer3_attention_weight_min": -92.79722595214844,
      "activations/layer4_attention_weight_max": 104.39696502685547,
      "activations/layer4_attention_weight_min": -98.77970886230469,
      "activations/layer5_attention_weight_max": 75.15686798095703,
      "activations/layer5_attention_weight_min": -85.5605697631836,
      "activations/layer6_attention_weight_max": 56.63480758666992,
      "activations/layer6_attention_weight_min": -54.94171905517578,
      "activations/layer7_attention_weight_max": 79.34463500976562,
      "activations/layer7_attention_weight_min": -75.07853698730469,
      "activations/layer8_attention_weight_max": 48.27312088012695,
      "activations/layer8_attention_weight_min": -48.84831619262695,
      "activations/layer9_attention_weight_max": 49.9792594909668,
      "activations/layer9_attention_weight_min": -48.3324089050293,
      "epoch": 10.82,
      "learning_rate": 8.102651515151515e-05,
      "loss": 2.7899,
      "step": 186250
    },
    {
      "activations/layer0_attention_weight_max": 15.741398811340332,
      "activations/layer0_attention_weight_min": -14.577180862426758,
      "activations/layer10_attention_weight_max": 37.80400466918945,
      "activations/layer10_attention_weight_min": -38.66675567626953,
      "activations/layer11_attention_weight_max": 32.098228454589844,
      "activations/layer11_attention_weight_min": -32.75646209716797,
      "activations/layer12_attention_weight_max": 20.937366485595703,
      "activations/layer12_attention_weight_min": -25.71518898010254,
      "activations/layer13_attention_weight_max": 43.146766662597656,
      "activations/layer13_attention_weight_min": -31.4304256439209,
      "activations/layer14_attention_weight_max": 40.61233139038086,
      "activations/layer14_attention_weight_min": -32.35822677612305,
      "activations/layer15_attention_weight_max": 41.887550354003906,
      "activations/layer15_attention_weight_min": -31.4765625,
      "activations/layer16_attention_weight_max": 33.839195251464844,
      "activations/layer16_attention_weight_min": -33.180259704589844,
      "activations/layer17_attention_weight_max": 54.79305648803711,
      "activations/layer17_attention_weight_min": -48.35188293457031,
      "activations/layer18_attention_weight_max": 50.40180587768555,
      "activations/layer18_attention_weight_min": -40.83857345581055,
      "activations/layer19_attention_weight_max": 23.39927101135254,
      "activations/layer19_attention_weight_min": -21.925683975219727,
      "activations/layer1_attention_weight_max": 16.586181640625,
      "activations/layer1_attention_weight_min": -15.391366958618164,
      "activations/layer20_attention_weight_max": 23.874984741210938,
      "activations/layer20_attention_weight_min": -20.872905731201172,
      "activations/layer21_attention_weight_max": 45.04634475708008,
      "activations/layer21_attention_weight_min": -25.18269920349121,
      "activations/layer22_attention_weight_max": 32.8851203918457,
      "activations/layer22_attention_weight_min": -25.898069381713867,
      "activations/layer23_attention_weight_max": 37.842498779296875,
      "activations/layer23_attention_weight_min": -24.007070541381836,
      "activations/layer2_attention_weight_max": 31.965009689331055,
      "activations/layer2_attention_weight_min": -31.99355125427246,
      "activations/layer3_attention_weight_max": 90.3064193725586,
      "activations/layer3_attention_weight_min": -95.64070129394531,
      "activations/layer4_attention_weight_max": 99.43279266357422,
      "activations/layer4_attention_weight_min": -97.69097137451172,
      "activations/layer5_attention_weight_max": 71.67691040039062,
      "activations/layer5_attention_weight_min": -77.82411193847656,
      "activations/layer6_attention_weight_max": 51.71445083618164,
      "activations/layer6_attention_weight_min": -54.637413024902344,
      "activations/layer7_attention_weight_max": 71.56883239746094,
      "activations/layer7_attention_weight_min": -72.98446655273438,
      "activations/layer8_attention_weight_max": 46.926151275634766,
      "activations/layer8_attention_weight_min": -50.04098892211914,
      "activations/layer9_attention_weight_max": 50.52083969116211,
      "activations/layer9_attention_weight_min": -53.30121994018555,
      "epoch": 10.83,
      "learning_rate": 8.100757575757575e-05,
      "loss": 2.7918,
      "step": 186300
    },
    {
      "activations/layer0_attention_weight_max": 15.302266120910645,
      "activations/layer0_attention_weight_min": -14.733336448669434,
      "activations/layer10_attention_weight_max": 36.133819580078125,
      "activations/layer10_attention_weight_min": -35.81913757324219,
      "activations/layer11_attention_weight_max": 31.371444702148438,
      "activations/layer11_attention_weight_min": -32.45848083496094,
      "activations/layer12_attention_weight_max": 16.7981014251709,
      "activations/layer12_attention_weight_min": -27.892166137695312,
      "activations/layer13_attention_weight_max": 35.98163986206055,
      "activations/layer13_attention_weight_min": -30.410783767700195,
      "activations/layer14_attention_weight_max": 38.69956970214844,
      "activations/layer14_attention_weight_min": -30.39179039001465,
      "activations/layer15_attention_weight_max": 31.957292556762695,
      "activations/layer15_attention_weight_min": -30.10471534729004,
      "activations/layer16_attention_weight_max": 29.451040267944336,
      "activations/layer16_attention_weight_min": -28.189817428588867,
      "activations/layer17_attention_weight_max": 49.91202926635742,
      "activations/layer17_attention_weight_min": -41.23241424560547,
      "activations/layer18_attention_weight_max": 44.51858139038086,
      "activations/layer18_attention_weight_min": -36.00324249267578,
      "activations/layer19_attention_weight_max": 22.353736877441406,
      "activations/layer19_attention_weight_min": -22.037282943725586,
      "activations/layer1_attention_weight_max": 16.731298446655273,
      "activations/layer1_attention_weight_min": -14.513936042785645,
      "activations/layer20_attention_weight_max": 22.163330078125,
      "activations/layer20_attention_weight_min": -21.50023078918457,
      "activations/layer21_attention_weight_max": 33.37385177612305,
      "activations/layer21_attention_weight_min": -25.345537185668945,
      "activations/layer22_attention_weight_max": 28.308849334716797,
      "activations/layer22_attention_weight_min": -24.07906150817871,
      "activations/layer23_attention_weight_max": 34.548736572265625,
      "activations/layer23_attention_weight_min": -23.842199325561523,
      "activations/layer2_attention_weight_max": 33.3111686706543,
      "activations/layer2_attention_weight_min": -31.4174861907959,
      "activations/layer3_attention_weight_max": 90.75944519042969,
      "activations/layer3_attention_weight_min": -95.45701599121094,
      "activations/layer4_attention_weight_max": 97.07440948486328,
      "activations/layer4_attention_weight_min": -92.14266204833984,
      "activations/layer5_attention_weight_max": 70.87846374511719,
      "activations/layer5_attention_weight_min": -77.63956451416016,
      "activations/layer6_attention_weight_max": 53.48084259033203,
      "activations/layer6_attention_weight_min": -54.45083999633789,
      "activations/layer7_attention_weight_max": 76.2615737915039,
      "activations/layer7_attention_weight_min": -71.07847595214844,
      "activations/layer8_attention_weight_max": 43.160362243652344,
      "activations/layer8_attention_weight_min": -47.1953010559082,
      "activations/layer9_attention_weight_max": 46.6799430847168,
      "activations/layer9_attention_weight_min": -46.0037841796875,
      "epoch": 10.83,
      "learning_rate": 8.098863636363635e-05,
      "loss": 2.7786,
      "step": 186350
    },
    {
      "activations/layer0_attention_weight_max": 14.867546081542969,
      "activations/layer0_attention_weight_min": -14.984430313110352,
      "activations/layer10_attention_weight_max": 38.2029914855957,
      "activations/layer10_attention_weight_min": -38.90135192871094,
      "activations/layer11_attention_weight_max": 32.29422378540039,
      "activations/layer11_attention_weight_min": -32.61650848388672,
      "activations/layer12_attention_weight_max": 17.606382369995117,
      "activations/layer12_attention_weight_min": -24.08420753479004,
      "activations/layer13_attention_weight_max": 49.27070617675781,
      "activations/layer13_attention_weight_min": -32.76878356933594,
      "activations/layer14_attention_weight_max": 39.896400451660156,
      "activations/layer14_attention_weight_min": -31.25311851501465,
      "activations/layer15_attention_weight_max": 37.61241149902344,
      "activations/layer15_attention_weight_min": -29.945480346679688,
      "activations/layer16_attention_weight_max": 32.65351486206055,
      "activations/layer16_attention_weight_min": -29.420650482177734,
      "activations/layer17_attention_weight_max": 55.166709899902344,
      "activations/layer17_attention_weight_min": -43.2054443359375,
      "activations/layer18_attention_weight_max": 49.37822341918945,
      "activations/layer18_attention_weight_min": -37.592220306396484,
      "activations/layer19_attention_weight_max": 23.98699188232422,
      "activations/layer19_attention_weight_min": -25.28957748413086,
      "activations/layer1_attention_weight_max": 16.642480850219727,
      "activations/layer1_attention_weight_min": -15.623030662536621,
      "activations/layer20_attention_weight_max": 23.074987411499023,
      "activations/layer20_attention_weight_min": -24.93478012084961,
      "activations/layer21_attention_weight_max": 43.410377502441406,
      "activations/layer21_attention_weight_min": -24.651636123657227,
      "activations/layer22_attention_weight_max": 34.706912994384766,
      "activations/layer22_attention_weight_min": -26.99745750427246,
      "activations/layer23_attention_weight_max": 38.88602828979492,
      "activations/layer23_attention_weight_min": -23.862812042236328,
      "activations/layer2_attention_weight_max": 33.26775360107422,
      "activations/layer2_attention_weight_min": -30.441572189331055,
      "activations/layer3_attention_weight_max": 91.7225570678711,
      "activations/layer3_attention_weight_min": -92.71238708496094,
      "activations/layer4_attention_weight_max": 98.81249237060547,
      "activations/layer4_attention_weight_min": -98.71888732910156,
      "activations/layer5_attention_weight_max": 72.23934936523438,
      "activations/layer5_attention_weight_min": -74.88487243652344,
      "activations/layer6_attention_weight_max": 55.99332809448242,
      "activations/layer6_attention_weight_min": -52.406227111816406,
      "activations/layer7_attention_weight_max": 77.60075378417969,
      "activations/layer7_attention_weight_min": -70.67057800292969,
      "activations/layer8_attention_weight_max": 47.35688018798828,
      "activations/layer8_attention_weight_min": -47.371402740478516,
      "activations/layer9_attention_weight_max": 49.09858703613281,
      "activations/layer9_attention_weight_min": -47.446041107177734,
      "epoch": 10.83,
      "learning_rate": 8.097007575757574e-05,
      "loss": 2.7778,
      "step": 186400
    },
    {
      "activations/layer0_attention_weight_max": 15.205286979675293,
      "activations/layer0_attention_weight_min": -15.039863586425781,
      "activations/layer10_attention_weight_max": 37.04096221923828,
      "activations/layer10_attention_weight_min": -36.216522216796875,
      "activations/layer11_attention_weight_max": 32.94883346557617,
      "activations/layer11_attention_weight_min": -34.416194915771484,
      "activations/layer12_attention_weight_max": 16.81624412536621,
      "activations/layer12_attention_weight_min": -26.225862503051758,
      "activations/layer13_attention_weight_max": 40.39394760131836,
      "activations/layer13_attention_weight_min": -32.238319396972656,
      "activations/layer14_attention_weight_max": 37.24518585205078,
      "activations/layer14_attention_weight_min": -30.254833221435547,
      "activations/layer15_attention_weight_max": 33.26776885986328,
      "activations/layer15_attention_weight_min": -29.1417236328125,
      "activations/layer16_attention_weight_max": 30.290185928344727,
      "activations/layer16_attention_weight_min": -28.339702606201172,
      "activations/layer17_attention_weight_max": 50.482662200927734,
      "activations/layer17_attention_weight_min": -43.60462188720703,
      "activations/layer18_attention_weight_max": 44.07469940185547,
      "activations/layer18_attention_weight_min": -37.17465591430664,
      "activations/layer19_attention_weight_max": 20.93115234375,
      "activations/layer19_attention_weight_min": -20.703283309936523,
      "activations/layer1_attention_weight_max": 16.784704208374023,
      "activations/layer1_attention_weight_min": -16.34546661376953,
      "activations/layer20_attention_weight_max": 20.81586456298828,
      "activations/layer20_attention_weight_min": -18.616310119628906,
      "activations/layer21_attention_weight_max": 31.229379653930664,
      "activations/layer21_attention_weight_min": -20.67718505859375,
      "activations/layer22_attention_weight_max": 30.63735580444336,
      "activations/layer22_attention_weight_min": -23.77939796447754,
      "activations/layer23_attention_weight_max": 34.633079528808594,
      "activations/layer23_attention_weight_min": -21.379745483398438,
      "activations/layer2_attention_weight_max": 33.92052459716797,
      "activations/layer2_attention_weight_min": -32.14564514160156,
      "activations/layer3_attention_weight_max": 96.6119384765625,
      "activations/layer3_attention_weight_min": -97.22032928466797,
      "activations/layer4_attention_weight_max": 102.43529510498047,
      "activations/layer4_attention_weight_min": -95.26421356201172,
      "activations/layer5_attention_weight_max": 75.80347442626953,
      "activations/layer5_attention_weight_min": -79.41014099121094,
      "activations/layer6_attention_weight_max": 55.38022232055664,
      "activations/layer6_attention_weight_min": -52.529598236083984,
      "activations/layer7_attention_weight_max": 81.32368469238281,
      "activations/layer7_attention_weight_min": -74.24639892578125,
      "activations/layer8_attention_weight_max": 45.902366638183594,
      "activations/layer8_attention_weight_min": -47.43202590942383,
      "activations/layer9_attention_weight_max": 48.958900451660156,
      "activations/layer9_attention_weight_min": -47.479373931884766,
      "epoch": 10.83,
      "learning_rate": 8.095113636363636e-05,
      "loss": 2.7668,
      "step": 186450
    },
    {
      "activations/layer0_attention_weight_max": 15.688654899597168,
      "activations/layer0_attention_weight_min": -14.428448677062988,
      "activations/layer10_attention_weight_max": 35.61262893676758,
      "activations/layer10_attention_weight_min": -36.32040023803711,
      "activations/layer11_attention_weight_max": 31.79175567626953,
      "activations/layer11_attention_weight_min": -34.37129211425781,
      "activations/layer12_attention_weight_max": 19.628149032592773,
      "activations/layer12_attention_weight_min": -24.484466552734375,
      "activations/layer13_attention_weight_max": 36.08476257324219,
      "activations/layer13_attention_weight_min": -27.747814178466797,
      "activations/layer14_attention_weight_max": 36.92989730834961,
      "activations/layer14_attention_weight_min": -29.615388870239258,
      "activations/layer15_attention_weight_max": 35.55292510986328,
      "activations/layer15_attention_weight_min": -29.479047775268555,
      "activations/layer16_attention_weight_max": 29.866968154907227,
      "activations/layer16_attention_weight_min": -28.869434356689453,
      "activations/layer17_attention_weight_max": 52.66259002685547,
      "activations/layer17_attention_weight_min": -44.78923797607422,
      "activations/layer18_attention_weight_max": 45.066566467285156,
      "activations/layer18_attention_weight_min": -39.5810661315918,
      "activations/layer19_attention_weight_max": 22.685474395751953,
      "activations/layer19_attention_weight_min": -23.751811981201172,
      "activations/layer1_attention_weight_max": 17.247465133666992,
      "activations/layer1_attention_weight_min": -15.742984771728516,
      "activations/layer20_attention_weight_max": 22.471872329711914,
      "activations/layer20_attention_weight_min": -24.266265869140625,
      "activations/layer21_attention_weight_max": 36.486732482910156,
      "activations/layer21_attention_weight_min": -23.522930145263672,
      "activations/layer22_attention_weight_max": 29.02377700805664,
      "activations/layer22_attention_weight_min": -27.72538185119629,
      "activations/layer23_attention_weight_max": 37.579410552978516,
      "activations/layer23_attention_weight_min": -25.284191131591797,
      "activations/layer2_attention_weight_max": 32.90286636352539,
      "activations/layer2_attention_weight_min": -30.964073181152344,
      "activations/layer3_attention_weight_max": 93.45719909667969,
      "activations/layer3_attention_weight_min": -93.65370178222656,
      "activations/layer4_attention_weight_max": 99.56211853027344,
      "activations/layer4_attention_weight_min": -94.3471908569336,
      "activations/layer5_attention_weight_max": 74.9615249633789,
      "activations/layer5_attention_weight_min": -81.2470703125,
      "activations/layer6_attention_weight_max": 52.04380416870117,
      "activations/layer6_attention_weight_min": -52.01283264160156,
      "activations/layer7_attention_weight_max": 72.64977264404297,
      "activations/layer7_attention_weight_min": -73.87323760986328,
      "activations/layer8_attention_weight_max": 45.30843734741211,
      "activations/layer8_attention_weight_min": -47.325870513916016,
      "activations/layer9_attention_weight_max": 44.847782135009766,
      "activations/layer9_attention_weight_min": -48.995723724365234,
      "epoch": 10.84,
      "learning_rate": 8.093257575757575e-05,
      "loss": 2.7834,
      "step": 186500
    },
    {
      "activations/layer0_attention_weight_max": 15.089418411254883,
      "activations/layer0_attention_weight_min": -14.49901008605957,
      "activations/layer10_attention_weight_max": 32.01194381713867,
      "activations/layer10_attention_weight_min": -33.18571472167969,
      "activations/layer11_attention_weight_max": 28.693683624267578,
      "activations/layer11_attention_weight_min": -30.20612144470215,
      "activations/layer12_attention_weight_max": 17.417062759399414,
      "activations/layer12_attention_weight_min": -27.355070114135742,
      "activations/layer13_attention_weight_max": 32.924076080322266,
      "activations/layer13_attention_weight_min": -26.181718826293945,
      "activations/layer14_attention_weight_max": 34.18663024902344,
      "activations/layer14_attention_weight_min": -28.626052856445312,
      "activations/layer15_attention_weight_max": 32.959983825683594,
      "activations/layer15_attention_weight_min": -28.64600944519043,
      "activations/layer16_attention_weight_max": 29.58051300048828,
      "activations/layer16_attention_weight_min": -28.146591186523438,
      "activations/layer17_attention_weight_max": 49.370933532714844,
      "activations/layer17_attention_weight_min": -41.353816986083984,
      "activations/layer18_attention_weight_max": 42.925960540771484,
      "activations/layer18_attention_weight_min": -37.23442077636719,
      "activations/layer19_attention_weight_max": 19.291688919067383,
      "activations/layer19_attention_weight_min": -20.1503963470459,
      "activations/layer1_attention_weight_max": 17.228668212890625,
      "activations/layer1_attention_weight_min": -14.764198303222656,
      "activations/layer20_attention_weight_max": 18.50734519958496,
      "activations/layer20_attention_weight_min": -20.284809112548828,
      "activations/layer21_attention_weight_max": 35.043243408203125,
      "activations/layer21_attention_weight_min": -24.789011001586914,
      "activations/layer22_attention_weight_max": 27.497705459594727,
      "activations/layer22_attention_weight_min": -26.98869514465332,
      "activations/layer23_attention_weight_max": 35.019954681396484,
      "activations/layer23_attention_weight_min": -24.475614547729492,
      "activations/layer2_attention_weight_max": 34.35062026977539,
      "activations/layer2_attention_weight_min": -31.210485458374023,
      "activations/layer3_attention_weight_max": 94.84506225585938,
      "activations/layer3_attention_weight_min": -91.2584457397461,
      "activations/layer4_attention_weight_max": 98.57201385498047,
      "activations/layer4_attention_weight_min": -93.61719512939453,
      "activations/layer5_attention_weight_max": 72.40034484863281,
      "activations/layer5_attention_weight_min": -75.48028564453125,
      "activations/layer6_attention_weight_max": 51.837284088134766,
      "activations/layer6_attention_weight_min": -52.473628997802734,
      "activations/layer7_attention_weight_max": 69.92265319824219,
      "activations/layer7_attention_weight_min": -71.24884796142578,
      "activations/layer8_attention_weight_max": 42.2009391784668,
      "activations/layer8_attention_weight_min": -43.91944122314453,
      "activations/layer9_attention_weight_max": 46.00455093383789,
      "activations/layer9_attention_weight_min": -45.78373336791992,
      "epoch": 10.84,
      "learning_rate": 8.091363636363636e-05,
      "loss": 2.7849,
      "step": 186550
    },
    {
      "activations/layer0_attention_weight_max": 14.816043853759766,
      "activations/layer0_attention_weight_min": -15.198334693908691,
      "activations/layer10_attention_weight_max": 38.835147857666016,
      "activations/layer10_attention_weight_min": -36.6246337890625,
      "activations/layer11_attention_weight_max": 33.71185302734375,
      "activations/layer11_attention_weight_min": -33.47123718261719,
      "activations/layer12_attention_weight_max": 18.647478103637695,
      "activations/layer12_attention_weight_min": -24.05906105041504,
      "activations/layer13_attention_weight_max": 39.413291931152344,
      "activations/layer13_attention_weight_min": -27.792308807373047,
      "activations/layer14_attention_weight_max": 36.15285873413086,
      "activations/layer14_attention_weight_min": -30.819793701171875,
      "activations/layer15_attention_weight_max": 32.42083740234375,
      "activations/layer15_attention_weight_min": -29.130720138549805,
      "activations/layer16_attention_weight_max": 30.133472442626953,
      "activations/layer16_attention_weight_min": -27.49989128112793,
      "activations/layer17_attention_weight_max": 51.23992919921875,
      "activations/layer17_attention_weight_min": -41.90937805175781,
      "activations/layer18_attention_weight_max": 45.03850555419922,
      "activations/layer18_attention_weight_min": -35.745269775390625,
      "activations/layer19_attention_weight_max": 21.459964752197266,
      "activations/layer19_attention_weight_min": -22.57242774963379,
      "activations/layer1_attention_weight_max": 16.994792938232422,
      "activations/layer1_attention_weight_min": -14.111903190612793,
      "activations/layer20_attention_weight_max": 20.31616973876953,
      "activations/layer20_attention_weight_min": -21.348533630371094,
      "activations/layer21_attention_weight_max": 35.340206146240234,
      "activations/layer21_attention_weight_min": -22.445741653442383,
      "activations/layer22_attention_weight_max": 32.9194221496582,
      "activations/layer22_attention_weight_min": -25.79096031188965,
      "activations/layer23_attention_weight_max": 37.20676040649414,
      "activations/layer23_attention_weight_min": -24.665761947631836,
      "activations/layer2_attention_weight_max": 33.298561096191406,
      "activations/layer2_attention_weight_min": -31.193622589111328,
      "activations/layer3_attention_weight_max": 92.1744155883789,
      "activations/layer3_attention_weight_min": -94.01671600341797,
      "activations/layer4_attention_weight_max": 100.09105682373047,
      "activations/layer4_attention_weight_min": -94.17015838623047,
      "activations/layer5_attention_weight_max": 73.14576721191406,
      "activations/layer5_attention_weight_min": -74.9957275390625,
      "activations/layer6_attention_weight_max": 54.96366500854492,
      "activations/layer6_attention_weight_min": -52.973167419433594,
      "activations/layer7_attention_weight_max": 78.17196655273438,
      "activations/layer7_attention_weight_min": -71.76763916015625,
      "activations/layer8_attention_weight_max": 50.29275894165039,
      "activations/layer8_attention_weight_min": -48.35140609741211,
      "activations/layer9_attention_weight_max": 55.59766387939453,
      "activations/layer9_attention_weight_min": -49.20113754272461,
      "epoch": 10.84,
      "learning_rate": 8.089469696969696e-05,
      "loss": 2.7805,
      "step": 186600
    },
    {
      "activations/layer0_attention_weight_max": 14.76124382019043,
      "activations/layer0_attention_weight_min": -14.745903015136719,
      "activations/layer10_attention_weight_max": 41.839664459228516,
      "activations/layer10_attention_weight_min": -39.856353759765625,
      "activations/layer11_attention_weight_max": 37.750911712646484,
      "activations/layer11_attention_weight_min": -34.49431610107422,
      "activations/layer12_attention_weight_max": 24.335050582885742,
      "activations/layer12_attention_weight_min": -26.65169906616211,
      "activations/layer13_attention_weight_max": 44.128421783447266,
      "activations/layer13_attention_weight_min": -29.922603607177734,
      "activations/layer14_attention_weight_max": 41.24290084838867,
      "activations/layer14_attention_weight_min": -29.644094467163086,
      "activations/layer15_attention_weight_max": 36.93057632446289,
      "activations/layer15_attention_weight_min": -29.78266143798828,
      "activations/layer16_attention_weight_max": 31.19110870361328,
      "activations/layer16_attention_weight_min": -27.93282127380371,
      "activations/layer17_attention_weight_max": 53.70956802368164,
      "activations/layer17_attention_weight_min": -45.39693832397461,
      "activations/layer18_attention_weight_max": 46.783382415771484,
      "activations/layer18_attention_weight_min": -40.614830017089844,
      "activations/layer19_attention_weight_max": 20.7889404296875,
      "activations/layer19_attention_weight_min": -20.133201599121094,
      "activations/layer1_attention_weight_max": 16.615955352783203,
      "activations/layer1_attention_weight_min": -16.356491088867188,
      "activations/layer20_attention_weight_max": 22.390655517578125,
      "activations/layer20_attention_weight_min": -20.051525115966797,
      "activations/layer21_attention_weight_max": 38.17268753051758,
      "activations/layer21_attention_weight_min": -22.492441177368164,
      "activations/layer22_attention_weight_max": 30.94053077697754,
      "activations/layer22_attention_weight_min": -23.843299865722656,
      "activations/layer23_attention_weight_max": 46.0123405456543,
      "activations/layer23_attention_weight_min": -25.85849380493164,
      "activations/layer2_attention_weight_max": 33.232383728027344,
      "activations/layer2_attention_weight_min": -33.29082489013672,
      "activations/layer3_attention_weight_max": 95.58431243896484,
      "activations/layer3_attention_weight_min": -93.99394226074219,
      "activations/layer4_attention_weight_max": 102.83222961425781,
      "activations/layer4_attention_weight_min": -99.35711669921875,
      "activations/layer5_attention_weight_max": 72.14306640625,
      "activations/layer5_attention_weight_min": -82.14070129394531,
      "activations/layer6_attention_weight_max": 54.22431182861328,
      "activations/layer6_attention_weight_min": -53.31843185424805,
      "activations/layer7_attention_weight_max": 78.3850326538086,
      "activations/layer7_attention_weight_min": -78.14266967773438,
      "activations/layer8_attention_weight_max": 50.77000427246094,
      "activations/layer8_attention_weight_min": -49.12196350097656,
      "activations/layer9_attention_weight_max": 57.9351921081543,
      "activations/layer9_attention_weight_min": -48.60857391357422,
      "epoch": 10.85,
      "learning_rate": 8.087575757575757e-05,
      "loss": 2.7838,
      "step": 186650
    },
    {
      "activations/layer0_attention_weight_max": 14.588240623474121,
      "activations/layer0_attention_weight_min": -14.691038131713867,
      "activations/layer10_attention_weight_max": 33.4786376953125,
      "activations/layer10_attention_weight_min": -35.31760787963867,
      "activations/layer11_attention_weight_max": 28.488826751708984,
      "activations/layer11_attention_weight_min": -30.564411163330078,
      "activations/layer12_attention_weight_max": 20.244522094726562,
      "activations/layer12_attention_weight_min": -23.257272720336914,
      "activations/layer13_attention_weight_max": 43.57437515258789,
      "activations/layer13_attention_weight_min": -26.290842056274414,
      "activations/layer14_attention_weight_max": 34.53165817260742,
      "activations/layer14_attention_weight_min": -30.343517303466797,
      "activations/layer15_attention_weight_max": 31.432661056518555,
      "activations/layer15_attention_weight_min": -27.25172233581543,
      "activations/layer16_attention_weight_max": 29.31005096435547,
      "activations/layer16_attention_weight_min": -28.293813705444336,
      "activations/layer17_attention_weight_max": 52.05116653442383,
      "activations/layer17_attention_weight_min": -40.21572494506836,
      "activations/layer18_attention_weight_max": 42.73038101196289,
      "activations/layer18_attention_weight_min": -36.478981018066406,
      "activations/layer19_attention_weight_max": 23.553693771362305,
      "activations/layer19_attention_weight_min": -20.250377655029297,
      "activations/layer1_attention_weight_max": 17.48465347290039,
      "activations/layer1_attention_weight_min": -14.75706958770752,
      "activations/layer20_attention_weight_max": 24.412879943847656,
      "activations/layer20_attention_weight_min": -21.84485626220703,
      "activations/layer21_attention_weight_max": 36.08032989501953,
      "activations/layer21_attention_weight_min": -21.758258819580078,
      "activations/layer22_attention_weight_max": 28.98248863220215,
      "activations/layer22_attention_weight_min": -24.208141326904297,
      "activations/layer23_attention_weight_max": 38.88313293457031,
      "activations/layer23_attention_weight_min": -25.06173324584961,
      "activations/layer2_attention_weight_max": 32.24146270751953,
      "activations/layer2_attention_weight_min": -31.570348739624023,
      "activations/layer3_attention_weight_max": 88.36736297607422,
      "activations/layer3_attention_weight_min": -90.9404296875,
      "activations/layer4_attention_weight_max": 94.02094268798828,
      "activations/layer4_attention_weight_min": -91.7575912475586,
      "activations/layer5_attention_weight_max": 69.5633544921875,
      "activations/layer5_attention_weight_min": -73.96028137207031,
      "activations/layer6_attention_weight_max": 53.588436126708984,
      "activations/layer6_attention_weight_min": -50.89455795288086,
      "activations/layer7_attention_weight_max": 69.86537170410156,
      "activations/layer7_attention_weight_min": -70.89057922363281,
      "activations/layer8_attention_weight_max": 43.38161087036133,
      "activations/layer8_attention_weight_min": -44.45383834838867,
      "activations/layer9_attention_weight_max": 45.56587600708008,
      "activations/layer9_attention_weight_min": -46.01908874511719,
      "epoch": 10.85,
      "learning_rate": 8.085681818181817e-05,
      "loss": 2.7908,
      "step": 186700
    },
    {
      "activations/layer0_attention_weight_max": 15.372238159179688,
      "activations/layer0_attention_weight_min": -14.876749992370605,
      "activations/layer10_attention_weight_max": 36.725547790527344,
      "activations/layer10_attention_weight_min": -37.743072509765625,
      "activations/layer11_attention_weight_max": 33.255775451660156,
      "activations/layer11_attention_weight_min": -32.723846435546875,
      "activations/layer12_attention_weight_max": 18.16141700744629,
      "activations/layer12_attention_weight_min": -22.567703247070312,
      "activations/layer13_attention_weight_max": 39.178749084472656,
      "activations/layer13_attention_weight_min": -32.44219970703125,
      "activations/layer14_attention_weight_max": 34.830562591552734,
      "activations/layer14_attention_weight_min": -30.74643898010254,
      "activations/layer15_attention_weight_max": 33.112220764160156,
      "activations/layer15_attention_weight_min": -28.8284912109375,
      "activations/layer16_attention_weight_max": 33.335636138916016,
      "activations/layer16_attention_weight_min": -30.346446990966797,
      "activations/layer17_attention_weight_max": 50.57942581176758,
      "activations/layer17_attention_weight_min": -42.44255828857422,
      "activations/layer18_attention_weight_max": 48.88520050048828,
      "activations/layer18_attention_weight_min": -37.06786346435547,
      "activations/layer19_attention_weight_max": 22.01954460144043,
      "activations/layer19_attention_weight_min": -21.74159812927246,
      "activations/layer1_attention_weight_max": 16.550453186035156,
      "activations/layer1_attention_weight_min": -16.0408935546875,
      "activations/layer20_attention_weight_max": 22.894821166992188,
      "activations/layer20_attention_weight_min": -21.56248664855957,
      "activations/layer21_attention_weight_max": 36.368656158447266,
      "activations/layer21_attention_weight_min": -25.210372924804688,
      "activations/layer22_attention_weight_max": 30.338348388671875,
      "activations/layer22_attention_weight_min": -25.70951271057129,
      "activations/layer23_attention_weight_max": 39.3017692565918,
      "activations/layer23_attention_weight_min": -23.80976104736328,
      "activations/layer2_attention_weight_max": 33.56863784790039,
      "activations/layer2_attention_weight_min": -31.072385787963867,
      "activations/layer3_attention_weight_max": 91.54913330078125,
      "activations/layer3_attention_weight_min": -93.79496002197266,
      "activations/layer4_attention_weight_max": 99.39066314697266,
      "activations/layer4_attention_weight_min": -99.30962371826172,
      "activations/layer5_attention_weight_max": 72.91276550292969,
      "activations/layer5_attention_weight_min": -79.81204223632812,
      "activations/layer6_attention_weight_max": 52.85450744628906,
      "activations/layer6_attention_weight_min": -54.64078140258789,
      "activations/layer7_attention_weight_max": 81.35784912109375,
      "activations/layer7_attention_weight_min": -73.31787109375,
      "activations/layer8_attention_weight_max": 45.80633544921875,
      "activations/layer8_attention_weight_min": -46.79452896118164,
      "activations/layer9_attention_weight_max": 47.80250549316406,
      "activations/layer9_attention_weight_min": -51.28068923950195,
      "epoch": 10.85,
      "learning_rate": 8.083787878787878e-05,
      "loss": 2.7986,
      "step": 186750
    },
    {
      "activations/layer0_attention_weight_max": 16.098159790039062,
      "activations/layer0_attention_weight_min": -14.567559242248535,
      "activations/layer10_attention_weight_max": 35.966766357421875,
      "activations/layer10_attention_weight_min": -35.21803665161133,
      "activations/layer11_attention_weight_max": 33.053245544433594,
      "activations/layer11_attention_weight_min": -32.305938720703125,
      "activations/layer12_attention_weight_max": 19.240524291992188,
      "activations/layer12_attention_weight_min": -22.395612716674805,
      "activations/layer13_attention_weight_max": 42.57624435424805,
      "activations/layer13_attention_weight_min": -32.4317741394043,
      "activations/layer14_attention_weight_max": 39.17356872558594,
      "activations/layer14_attention_weight_min": -30.375280380249023,
      "activations/layer15_attention_weight_max": 33.694820404052734,
      "activations/layer15_attention_weight_min": -27.854446411132812,
      "activations/layer16_attention_weight_max": 29.536163330078125,
      "activations/layer16_attention_weight_min": -27.780813217163086,
      "activations/layer17_attention_weight_max": 53.98808288574219,
      "activations/layer17_attention_weight_min": -40.89021682739258,
      "activations/layer18_attention_weight_max": 46.63626480102539,
      "activations/layer18_attention_weight_min": -35.3108024597168,
      "activations/layer19_attention_weight_max": 24.649843215942383,
      "activations/layer19_attention_weight_min": -21.1661434173584,
      "activations/layer1_attention_weight_max": 17.176937103271484,
      "activations/layer1_attention_weight_min": -15.733870506286621,
      "activations/layer20_attention_weight_max": 21.490636825561523,
      "activations/layer20_attention_weight_min": -20.969083786010742,
      "activations/layer21_attention_weight_max": 40.863704681396484,
      "activations/layer21_attention_weight_min": -22.704055786132812,
      "activations/layer22_attention_weight_max": 29.600976943969727,
      "activations/layer22_attention_weight_min": -24.043977737426758,
      "activations/layer23_attention_weight_max": 34.41588592529297,
      "activations/layer23_attention_weight_min": -23.27755355834961,
      "activations/layer2_attention_weight_max": 33.21612548828125,
      "activations/layer2_attention_weight_min": -30.551929473876953,
      "activations/layer3_attention_weight_max": 88.05415344238281,
      "activations/layer3_attention_weight_min": -94.17719268798828,
      "activations/layer4_attention_weight_max": 97.39164733886719,
      "activations/layer4_attention_weight_min": -93.39083099365234,
      "activations/layer5_attention_weight_max": 68.94743347167969,
      "activations/layer5_attention_weight_min": -75.00279998779297,
      "activations/layer6_attention_weight_max": 54.731136322021484,
      "activations/layer6_attention_weight_min": -51.753238677978516,
      "activations/layer7_attention_weight_max": 77.80682373046875,
      "activations/layer7_attention_weight_min": -74.46720123291016,
      "activations/layer8_attention_weight_max": 45.50239944458008,
      "activations/layer8_attention_weight_min": -46.9786262512207,
      "activations/layer9_attention_weight_max": 48.283939361572266,
      "activations/layer9_attention_weight_min": -48.55887985229492,
      "epoch": 10.85,
      "learning_rate": 8.08189393939394e-05,
      "loss": 2.7856,
      "step": 186800
    },
    {
      "activations/layer0_attention_weight_max": 15.499967575073242,
      "activations/layer0_attention_weight_min": -14.769325256347656,
      "activations/layer10_attention_weight_max": 36.93335723876953,
      "activations/layer10_attention_weight_min": -37.38457489013672,
      "activations/layer11_attention_weight_max": 30.562088012695312,
      "activations/layer11_attention_weight_min": -33.67751693725586,
      "activations/layer12_attention_weight_max": 19.513731002807617,
      "activations/layer12_attention_weight_min": -26.278226852416992,
      "activations/layer13_attention_weight_max": 41.78953552246094,
      "activations/layer13_attention_weight_min": -30.674091339111328,
      "activations/layer14_attention_weight_max": 64.19437408447266,
      "activations/layer14_attention_weight_min": -36.24666213989258,
      "activations/layer15_attention_weight_max": 52.4545783996582,
      "activations/layer15_attention_weight_min": -31.111268997192383,
      "activations/layer16_attention_weight_max": 34.4758186340332,
      "activations/layer16_attention_weight_min": -31.230680465698242,
      "activations/layer17_attention_weight_max": 54.61704635620117,
      "activations/layer17_attention_weight_min": -47.98046875,
      "activations/layer18_attention_weight_max": 50.47380065917969,
      "activations/layer18_attention_weight_min": -40.23586654663086,
      "activations/layer19_attention_weight_max": 25.80108070373535,
      "activations/layer19_attention_weight_min": -24.545684814453125,
      "activations/layer1_attention_weight_max": 17.430776596069336,
      "activations/layer1_attention_weight_min": -15.434921264648438,
      "activations/layer20_attention_weight_max": 26.7431640625,
      "activations/layer20_attention_weight_min": -21.117534637451172,
      "activations/layer21_attention_weight_max": 51.01308059692383,
      "activations/layer21_attention_weight_min": -24.419851303100586,
      "activations/layer22_attention_weight_max": 37.643394470214844,
      "activations/layer22_attention_weight_min": -29.275001525878906,
      "activations/layer23_attention_weight_max": 45.11310958862305,
      "activations/layer23_attention_weight_min": -25.935155868530273,
      "activations/layer2_attention_weight_max": 35.04921340942383,
      "activations/layer2_attention_weight_min": -34.381622314453125,
      "activations/layer3_attention_weight_max": 95.45875549316406,
      "activations/layer3_attention_weight_min": -93.45599365234375,
      "activations/layer4_attention_weight_max": 101.59992218017578,
      "activations/layer4_attention_weight_min": -96.22920227050781,
      "activations/layer5_attention_weight_max": 75.70523071289062,
      "activations/layer5_attention_weight_min": -79.82268524169922,
      "activations/layer6_attention_weight_max": 54.47932434082031,
      "activations/layer6_attention_weight_min": -54.50593185424805,
      "activations/layer7_attention_weight_max": 77.98287963867188,
      "activations/layer7_attention_weight_min": -76.26093292236328,
      "activations/layer8_attention_weight_max": 48.68735885620117,
      "activations/layer8_attention_weight_min": -48.877891540527344,
      "activations/layer9_attention_weight_max": 52.048397064208984,
      "activations/layer9_attention_weight_min": -49.819129943847656,
      "epoch": 10.86,
      "learning_rate": 8.079999999999999e-05,
      "loss": 2.7788,
      "step": 186850
    },
    {
      "activations/layer0_attention_weight_max": 15.025308609008789,
      "activations/layer0_attention_weight_min": -14.87464714050293,
      "activations/layer10_attention_weight_max": 37.566226959228516,
      "activations/layer10_attention_weight_min": -37.06324005126953,
      "activations/layer11_attention_weight_max": 34.309532165527344,
      "activations/layer11_attention_weight_min": -34.598411560058594,
      "activations/layer12_attention_weight_max": 19.66091537475586,
      "activations/layer12_attention_weight_min": -24.048709869384766,
      "activations/layer13_attention_weight_max": 41.1336784362793,
      "activations/layer13_attention_weight_min": -29.5832462310791,
      "activations/layer14_attention_weight_max": 40.76576232910156,
      "activations/layer14_attention_weight_min": -32.28047180175781,
      "activations/layer15_attention_weight_max": 36.21299362182617,
      "activations/layer15_attention_weight_min": -32.25710678100586,
      "activations/layer16_attention_weight_max": 35.019004821777344,
      "activations/layer16_attention_weight_min": -34.35458755493164,
      "activations/layer17_attention_weight_max": 57.20808410644531,
      "activations/layer17_attention_weight_min": -51.32686996459961,
      "activations/layer18_attention_weight_max": 49.573787689208984,
      "activations/layer18_attention_weight_min": -42.109622955322266,
      "activations/layer19_attention_weight_max": 25.478168487548828,
      "activations/layer19_attention_weight_min": -22.581668853759766,
      "activations/layer1_attention_weight_max": 17.32783317565918,
      "activations/layer1_attention_weight_min": -13.82555866241455,
      "activations/layer20_attention_weight_max": 25.36101722717285,
      "activations/layer20_attention_weight_min": -28.74798583984375,
      "activations/layer21_attention_weight_max": 39.046714782714844,
      "activations/layer21_attention_weight_min": -22.988481521606445,
      "activations/layer22_attention_weight_max": 33.99571990966797,
      "activations/layer22_attention_weight_min": -27.98401641845703,
      "activations/layer23_attention_weight_max": 42.164554595947266,
      "activations/layer23_attention_weight_min": -25.596364974975586,
      "activations/layer2_attention_weight_max": 33.131080627441406,
      "activations/layer2_attention_weight_min": -31.802993774414062,
      "activations/layer3_attention_weight_max": 94.10688781738281,
      "activations/layer3_attention_weight_min": -95.58731079101562,
      "activations/layer4_attention_weight_max": 96.65436553955078,
      "activations/layer4_attention_weight_min": -92.85482025146484,
      "activations/layer5_attention_weight_max": 71.43892669677734,
      "activations/layer5_attention_weight_min": -75.86174011230469,
      "activations/layer6_attention_weight_max": 54.593299865722656,
      "activations/layer6_attention_weight_min": -51.25717544555664,
      "activations/layer7_attention_weight_max": 75.10832214355469,
      "activations/layer7_attention_weight_min": -73.83599090576172,
      "activations/layer8_attention_weight_max": 47.44695281982422,
      "activations/layer8_attention_weight_min": -49.21565246582031,
      "activations/layer9_attention_weight_max": 52.18463897705078,
      "activations/layer9_attention_weight_min": -50.026920318603516,
      "epoch": 10.86,
      "learning_rate": 8.07810606060606e-05,
      "loss": 2.7725,
      "step": 186900
    },
    {
      "activations/layer0_attention_weight_max": 15.169435501098633,
      "activations/layer0_attention_weight_min": -14.649510383605957,
      "activations/layer10_attention_weight_max": 37.15023422241211,
      "activations/layer10_attention_weight_min": -36.8269157409668,
      "activations/layer11_attention_weight_max": 31.44036865234375,
      "activations/layer11_attention_weight_min": -33.58964538574219,
      "activations/layer12_attention_weight_max": 20.100919723510742,
      "activations/layer12_attention_weight_min": -25.81781578063965,
      "activations/layer13_attention_weight_max": 42.998714447021484,
      "activations/layer13_attention_weight_min": -31.62433433532715,
      "activations/layer14_attention_weight_max": 36.964847564697266,
      "activations/layer14_attention_weight_min": -30.11160659790039,
      "activations/layer15_attention_weight_max": 34.30010986328125,
      "activations/layer15_attention_weight_min": -29.29193115234375,
      "activations/layer16_attention_weight_max": 35.09022521972656,
      "activations/layer16_attention_weight_min": -28.34568214416504,
      "activations/layer17_attention_weight_max": 49.87360763549805,
      "activations/layer17_attention_weight_min": -43.3129768371582,
      "activations/layer18_attention_weight_max": 44.19234848022461,
      "activations/layer18_attention_weight_min": -38.34765625,
      "activations/layer19_attention_weight_max": 23.99851417541504,
      "activations/layer19_attention_weight_min": -22.568693161010742,
      "activations/layer1_attention_weight_max": 18.179704666137695,
      "activations/layer1_attention_weight_min": -17.29831314086914,
      "activations/layer20_attention_weight_max": 22.565465927124023,
      "activations/layer20_attention_weight_min": -25.444053649902344,
      "activations/layer21_attention_weight_max": 36.80633544921875,
      "activations/layer21_attention_weight_min": -22.536827087402344,
      "activations/layer22_attention_weight_max": 31.186697006225586,
      "activations/layer22_attention_weight_min": -28.736604690551758,
      "activations/layer23_attention_weight_max": 40.66176986694336,
      "activations/layer23_attention_weight_min": -24.220970153808594,
      "activations/layer2_attention_weight_max": 33.438880920410156,
      "activations/layer2_attention_weight_min": -31.354049682617188,
      "activations/layer3_attention_weight_max": 95.2695541381836,
      "activations/layer3_attention_weight_min": -98.1152572631836,
      "activations/layer4_attention_weight_max": 100.24339294433594,
      "activations/layer4_attention_weight_min": -93.60265350341797,
      "activations/layer5_attention_weight_max": 73.26412963867188,
      "activations/layer5_attention_weight_min": -74.62503051757812,
      "activations/layer6_attention_weight_max": 55.77899169921875,
      "activations/layer6_attention_weight_min": -52.70524978637695,
      "activations/layer7_attention_weight_max": 83.02883911132812,
      "activations/layer7_attention_weight_min": -73.74246978759766,
      "activations/layer8_attention_weight_max": 45.113162994384766,
      "activations/layer8_attention_weight_min": -49.55290985107422,
      "activations/layer9_attention_weight_max": 48.58622360229492,
      "activations/layer9_attention_weight_min": -49.00148391723633,
      "epoch": 10.86,
      "learning_rate": 8.076212121212122e-05,
      "loss": 2.7875,
      "step": 186950
    },
    {
      "activations/layer0_attention_weight_max": 15.460381507873535,
      "activations/layer0_attention_weight_min": -14.406122207641602,
      "activations/layer10_attention_weight_max": 34.444915771484375,
      "activations/layer10_attention_weight_min": -33.42085266113281,
      "activations/layer11_attention_weight_max": 30.94391441345215,
      "activations/layer11_attention_weight_min": -32.9150505065918,
      "activations/layer12_attention_weight_max": 20.889307022094727,
      "activations/layer12_attention_weight_min": -27.66193389892578,
      "activations/layer13_attention_weight_max": 41.57662582397461,
      "activations/layer13_attention_weight_min": -30.942806243896484,
      "activations/layer14_attention_weight_max": 36.635921478271484,
      "activations/layer14_attention_weight_min": -31.385652542114258,
      "activations/layer15_attention_weight_max": 32.951229095458984,
      "activations/layer15_attention_weight_min": -29.906675338745117,
      "activations/layer16_attention_weight_max": 31.392345428466797,
      "activations/layer16_attention_weight_min": -28.050878524780273,
      "activations/layer17_attention_weight_max": 52.66151428222656,
      "activations/layer17_attention_weight_min": -43.556392669677734,
      "activations/layer18_attention_weight_max": 48.61740493774414,
      "activations/layer18_attention_weight_min": -35.652137756347656,
      "activations/layer19_attention_weight_max": 22.72199821472168,
      "activations/layer19_attention_weight_min": -19.876230239868164,
      "activations/layer1_attention_weight_max": 18.09470558166504,
      "activations/layer1_attention_weight_min": -14.541096687316895,
      "activations/layer20_attention_weight_max": 22.518478393554688,
      "activations/layer20_attention_weight_min": -21.634185791015625,
      "activations/layer21_attention_weight_max": 41.771728515625,
      "activations/layer21_attention_weight_min": -26.963077545166016,
      "activations/layer22_attention_weight_max": 31.874645233154297,
      "activations/layer22_attention_weight_min": -25.64480972290039,
      "activations/layer23_attention_weight_max": 37.65775680541992,
      "activations/layer23_attention_weight_min": -29.70037841796875,
      "activations/layer2_attention_weight_max": 31.944725036621094,
      "activations/layer2_attention_weight_min": -29.7586669921875,
      "activations/layer3_attention_weight_max": 89.19795227050781,
      "activations/layer3_attention_weight_min": -89.47816467285156,
      "activations/layer4_attention_weight_max": 97.82518005371094,
      "activations/layer4_attention_weight_min": -96.39642333984375,
      "activations/layer5_attention_weight_max": 74.19648742675781,
      "activations/layer5_attention_weight_min": -74.5712661743164,
      "activations/layer6_attention_weight_max": 52.45541763305664,
      "activations/layer6_attention_weight_min": -50.22919464111328,
      "activations/layer7_attention_weight_max": 70.6023178100586,
      "activations/layer7_attention_weight_min": -68.79801177978516,
      "activations/layer8_attention_weight_max": 42.85322570800781,
      "activations/layer8_attention_weight_min": -44.20720291137695,
      "activations/layer9_attention_weight_max": 45.8000373840332,
      "activations/layer9_attention_weight_min": -45.04069137573242,
      "epoch": 10.87,
      "learning_rate": 8.07431818181818e-05,
      "loss": 2.7864,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.4746,
      "eval_samples_per_second": 506.689,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.4746,
      "eval_openwebtext_samples_per_second": 506.689,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 1.9827,
      "eval_wikitext_samples_per_second": 229.988,
      "step": 187000
    },
    {
      "epoch": 10.87,
      "eval_lambada_loss": 2.591796875,
      "eval_lambada_ppl": 13.35374506875535,
      "eval_lambada_runtime": 9.5897,
      "eval_lambada_samples_per_second": 507.735,
      "step": 187000
    },
    {
      "activations/layer0_attention_weight_max": 16.17283058166504,
      "activations/layer0_attention_weight_min": -14.583414077758789,
      "activations/layer10_attention_weight_max": 38.94021224975586,
      "activations/layer10_attention_weight_min": -35.46458435058594,
      "activations/layer11_attention_weight_max": 34.229469299316406,
      "activations/layer11_attention_weight_min": -32.025726318359375,
      "activations/layer12_attention_weight_max": 21.6424503326416,
      "activations/layer12_attention_weight_min": -22.339859008789062,
      "activations/layer13_attention_weight_max": 45.813453674316406,
      "activations/layer13_attention_weight_min": -36.43351364135742,
      "activations/layer14_attention_weight_max": 40.76383972167969,
      "activations/layer14_attention_weight_min": -30.735876083374023,
      "activations/layer15_attention_weight_max": 33.299224853515625,
      "activations/layer15_attention_weight_min": -28.717947006225586,
      "activations/layer16_attention_weight_max": 31.085420608520508,
      "activations/layer16_attention_weight_min": -29.909929275512695,
      "activations/layer17_attention_weight_max": 49.25505065917969,
      "activations/layer17_attention_weight_min": -41.905609130859375,
      "activations/layer18_attention_weight_max": 44.225040435791016,
      "activations/layer18_attention_weight_min": -36.498958587646484,
      "activations/layer19_attention_weight_max": 20.420822143554688,
      "activations/layer19_attention_weight_min": -21.271438598632812,
      "activations/layer1_attention_weight_max": 16.925846099853516,
      "activations/layer1_attention_weight_min": -14.49433708190918,
      "activations/layer20_attention_weight_max": 21.53113555908203,
      "activations/layer20_attention_weight_min": -22.114463806152344,
      "activations/layer21_attention_weight_max": 41.804107666015625,
      "activations/layer21_attention_weight_min": -25.9647159576416,
      "activations/layer22_attention_weight_max": 28.090057373046875,
      "activations/layer22_attention_weight_min": -26.43865203857422,
      "activations/layer23_attention_weight_max": 36.00792694091797,
      "activations/layer23_attention_weight_min": -24.109725952148438,
      "activations/layer2_attention_weight_max": 33.51987075805664,
      "activations/layer2_attention_weight_min": -32.02760314941406,
      "activations/layer3_attention_weight_max": 90.84696197509766,
      "activations/layer3_attention_weight_min": -95.24166107177734,
      "activations/layer4_attention_weight_max": 102.44225311279297,
      "activations/layer4_attention_weight_min": -98.11785888671875,
      "activations/layer5_attention_weight_max": 72.49360656738281,
      "activations/layer5_attention_weight_min": -77.33572387695312,
      "activations/layer6_attention_weight_max": 55.43669891357422,
      "activations/layer6_attention_weight_min": -53.03556823730469,
      "activations/layer7_attention_weight_max": 76.19095611572266,
      "activations/layer7_attention_weight_min": -71.9718017578125,
      "activations/layer8_attention_weight_max": 48.177391052246094,
      "activations/layer8_attention_weight_min": -49.53157043457031,
      "activations/layer9_attention_weight_max": 51.80753707885742,
      "activations/layer9_attention_weight_min": -47.20478439331055,
      "epoch": 10.87,
      "learning_rate": 8.072424242424242e-05,
      "loss": 2.7763,
      "step": 187050
    },
    {
      "activations/layer0_attention_weight_max": 16.322811126708984,
      "activations/layer0_attention_weight_min": -14.792672157287598,
      "activations/layer10_attention_weight_max": 33.78394317626953,
      "activations/layer10_attention_weight_min": -35.78668975830078,
      "activations/layer11_attention_weight_max": 29.86844253540039,
      "activations/layer11_attention_weight_min": -30.500614166259766,
      "activations/layer12_attention_weight_max": 18.4642333984375,
      "activations/layer12_attention_weight_min": -24.63172721862793,
      "activations/layer13_attention_weight_max": 33.983360290527344,
      "activations/layer13_attention_weight_min": -30.22987937927246,
      "activations/layer14_attention_weight_max": 34.93280029296875,
      "activations/layer14_attention_weight_min": -30.638376235961914,
      "activations/layer15_attention_weight_max": 37.361324310302734,
      "activations/layer15_attention_weight_min": -31.41814613342285,
      "activations/layer16_attention_weight_max": 29.502296447753906,
      "activations/layer16_attention_weight_min": -28.865703582763672,
      "activations/layer17_attention_weight_max": 52.20914077758789,
      "activations/layer17_attention_weight_min": -43.78862762451172,
      "activations/layer18_attention_weight_max": 49.66114044189453,
      "activations/layer18_attention_weight_min": -39.218936920166016,
      "activations/layer19_attention_weight_max": 23.67872428894043,
      "activations/layer19_attention_weight_min": -23.71889305114746,
      "activations/layer1_attention_weight_max": 16.719839096069336,
      "activations/layer1_attention_weight_min": -15.086210250854492,
      "activations/layer20_attention_weight_max": 23.3624210357666,
      "activations/layer20_attention_weight_min": -21.606590270996094,
      "activations/layer21_attention_weight_max": 38.45201873779297,
      "activations/layer21_attention_weight_min": -22.53034019470215,
      "activations/layer22_attention_weight_max": 30.841835021972656,
      "activations/layer22_attention_weight_min": -26.487451553344727,
      "activations/layer23_attention_weight_max": 37.509361267089844,
      "activations/layer23_attention_weight_min": -22.868858337402344,
      "activations/layer2_attention_weight_max": 34.38321304321289,
      "activations/layer2_attention_weight_min": -32.217010498046875,
      "activations/layer3_attention_weight_max": 92.13028717041016,
      "activations/layer3_attention_weight_min": -94.83187103271484,
      "activations/layer4_attention_weight_max": 93.84886169433594,
      "activations/layer4_attention_weight_min": -93.99504089355469,
      "activations/layer5_attention_weight_max": 71.05860137939453,
      "activations/layer5_attention_weight_min": -75.83402252197266,
      "activations/layer6_attention_weight_max": 50.59577941894531,
      "activations/layer6_attention_weight_min": -51.93302536010742,
      "activations/layer7_attention_weight_max": 71.20962524414062,
      "activations/layer7_attention_weight_min": -69.25862121582031,
      "activations/layer8_attention_weight_max": 40.83875274658203,
      "activations/layer8_attention_weight_min": -43.91510772705078,
      "activations/layer9_attention_weight_max": 43.64614486694336,
      "activations/layer9_attention_weight_min": -44.91839599609375,
      "epoch": 10.87,
      "learning_rate": 8.070530303030303e-05,
      "loss": 2.7995,
      "step": 187100
    },
    {
      "activations/layer0_attention_weight_max": 15.856651306152344,
      "activations/layer0_attention_weight_min": -15.176897048950195,
      "activations/layer10_attention_weight_max": 35.60538101196289,
      "activations/layer10_attention_weight_min": -35.82219314575195,
      "activations/layer11_attention_weight_max": 32.33866882324219,
      "activations/layer11_attention_weight_min": -32.038475036621094,
      "activations/layer12_attention_weight_max": 17.70005989074707,
      "activations/layer12_attention_weight_min": -29.239585876464844,
      "activations/layer13_attention_weight_max": 34.6195068359375,
      "activations/layer13_attention_weight_min": -28.792613983154297,
      "activations/layer14_attention_weight_max": 35.44504928588867,
      "activations/layer14_attention_weight_min": -29.52088737487793,
      "activations/layer15_attention_weight_max": 34.0153694152832,
      "activations/layer15_attention_weight_min": -27.515451431274414,
      "activations/layer16_attention_weight_max": 28.585933685302734,
      "activations/layer16_attention_weight_min": -27.51214027404785,
      "activations/layer17_attention_weight_max": 49.549644470214844,
      "activations/layer17_attention_weight_min": -43.682090759277344,
      "activations/layer18_attention_weight_max": 43.659423828125,
      "activations/layer18_attention_weight_min": -35.270084381103516,
      "activations/layer19_attention_weight_max": 19.965009689331055,
      "activations/layer19_attention_weight_min": -22.928707122802734,
      "activations/layer1_attention_weight_max": 17.242202758789062,
      "activations/layer1_attention_weight_min": -13.248319625854492,
      "activations/layer20_attention_weight_max": 19.61817169189453,
      "activations/layer20_attention_weight_min": -21.09929084777832,
      "activations/layer21_attention_weight_max": 34.172218322753906,
      "activations/layer21_attention_weight_min": -21.16424560546875,
      "activations/layer22_attention_weight_max": 30.93655776977539,
      "activations/layer22_attention_weight_min": -25.448884963989258,
      "activations/layer23_attention_weight_max": 35.12261199951172,
      "activations/layer23_attention_weight_min": -22.60822105407715,
      "activations/layer2_attention_weight_max": 31.74623680114746,
      "activations/layer2_attention_weight_min": -30.734375,
      "activations/layer3_attention_weight_max": 91.3563232421875,
      "activations/layer3_attention_weight_min": -90.81183624267578,
      "activations/layer4_attention_weight_max": 97.85192108154297,
      "activations/layer4_attention_weight_min": -93.39116668701172,
      "activations/layer5_attention_weight_max": 71.84185791015625,
      "activations/layer5_attention_weight_min": -76.503662109375,
      "activations/layer6_attention_weight_max": 55.13434600830078,
      "activations/layer6_attention_weight_min": -55.255619049072266,
      "activations/layer7_attention_weight_max": 76.53591918945312,
      "activations/layer7_attention_weight_min": -75.31636047363281,
      "activations/layer8_attention_weight_max": 48.11029815673828,
      "activations/layer8_attention_weight_min": -48.750343322753906,
      "activations/layer9_attention_weight_max": 46.75180435180664,
      "activations/layer9_attention_weight_min": -47.30086898803711,
      "epoch": 10.87,
      "learning_rate": 8.068636363636362e-05,
      "loss": 2.7886,
      "step": 187150
    },
    {
      "activations/layer0_attention_weight_max": 15.991416931152344,
      "activations/layer0_attention_weight_min": -14.804797172546387,
      "activations/layer10_attention_weight_max": 42.80839157104492,
      "activations/layer10_attention_weight_min": -41.79884719848633,
      "activations/layer11_attention_weight_max": 37.43329620361328,
      "activations/layer11_attention_weight_min": -36.85696029663086,
      "activations/layer12_attention_weight_max": 19.438425064086914,
      "activations/layer12_attention_weight_min": -29.31949806213379,
      "activations/layer13_attention_weight_max": 44.402156829833984,
      "activations/layer13_attention_weight_min": -36.52293014526367,
      "activations/layer14_attention_weight_max": 44.79579544067383,
      "activations/layer14_attention_weight_min": -32.209449768066406,
      "activations/layer15_attention_weight_max": 39.647499084472656,
      "activations/layer15_attention_weight_min": -33.90257263183594,
      "activations/layer16_attention_weight_max": 33.84364700317383,
      "activations/layer16_attention_weight_min": -29.8153133392334,
      "activations/layer17_attention_weight_max": 55.13474655151367,
      "activations/layer17_attention_weight_min": -47.32193374633789,
      "activations/layer18_attention_weight_max": 53.08036422729492,
      "activations/layer18_attention_weight_min": -40.24374008178711,
      "activations/layer19_attention_weight_max": 26.58648109436035,
      "activations/layer19_attention_weight_min": -21.28865623474121,
      "activations/layer1_attention_weight_max": 16.50580596923828,
      "activations/layer1_attention_weight_min": -14.922880172729492,
      "activations/layer20_attention_weight_max": 26.277494430541992,
      "activations/layer20_attention_weight_min": -23.440763473510742,
      "activations/layer21_attention_weight_max": 52.042659759521484,
      "activations/layer21_attention_weight_min": -27.03348731994629,
      "activations/layer22_attention_weight_max": 35.29096221923828,
      "activations/layer22_attention_weight_min": -23.84978485107422,
      "activations/layer23_attention_weight_max": 38.138526916503906,
      "activations/layer23_attention_weight_min": -21.713613510131836,
      "activations/layer2_attention_weight_max": 34.36729049682617,
      "activations/layer2_attention_weight_min": -33.11989212036133,
      "activations/layer3_attention_weight_max": 98.66791534423828,
      "activations/layer3_attention_weight_min": -95.15239715576172,
      "activations/layer4_attention_weight_max": 105.40579986572266,
      "activations/layer4_attention_weight_min": -96.22744750976562,
      "activations/layer5_attention_weight_max": 79.91281127929688,
      "activations/layer5_attention_weight_min": -81.86325073242188,
      "activations/layer6_attention_weight_max": 61.45906448364258,
      "activations/layer6_attention_weight_min": -57.130924224853516,
      "activations/layer7_attention_weight_max": 81.96700286865234,
      "activations/layer7_attention_weight_min": -79.83214569091797,
      "activations/layer8_attention_weight_max": 54.863311767578125,
      "activations/layer8_attention_weight_min": -51.5224723815918,
      "activations/layer9_attention_weight_max": 56.360408782958984,
      "activations/layer9_attention_weight_min": -54.89652633666992,
      "epoch": 10.88,
      "learning_rate": 8.066742424242424e-05,
      "loss": 2.7948,
      "step": 187200
    },
    {
      "activations/layer0_attention_weight_max": 16.4722900390625,
      "activations/layer0_attention_weight_min": -14.047351837158203,
      "activations/layer10_attention_weight_max": 38.25386428833008,
      "activations/layer10_attention_weight_min": -36.750213623046875,
      "activations/layer11_attention_weight_max": 32.608055114746094,
      "activations/layer11_attention_weight_min": -33.552955627441406,
      "activations/layer12_attention_weight_max": 20.00338363647461,
      "activations/layer12_attention_weight_min": -26.933237075805664,
      "activations/layer13_attention_weight_max": 42.657737731933594,
      "activations/layer13_attention_weight_min": -30.12734031677246,
      "activations/layer14_attention_weight_max": 40.87647247314453,
      "activations/layer14_attention_weight_min": -32.729061126708984,
      "activations/layer15_attention_weight_max": 33.45005798339844,
      "activations/layer15_attention_weight_min": -30.93210792541504,
      "activations/layer16_attention_weight_max": 29.837736129760742,
      "activations/layer16_attention_weight_min": -28.782867431640625,
      "activations/layer17_attention_weight_max": 51.48112869262695,
      "activations/layer17_attention_weight_min": -43.0733642578125,
      "activations/layer18_attention_weight_max": 46.51426315307617,
      "activations/layer18_attention_weight_min": -36.478275299072266,
      "activations/layer19_attention_weight_max": 20.63310432434082,
      "activations/layer19_attention_weight_min": -24.498226165771484,
      "activations/layer1_attention_weight_max": 16.798831939697266,
      "activations/layer1_attention_weight_min": -14.29333782196045,
      "activations/layer20_attention_weight_max": 21.725831985473633,
      "activations/layer20_attention_weight_min": -23.544261932373047,
      "activations/layer21_attention_weight_max": 36.06229782104492,
      "activations/layer21_attention_weight_min": -21.75196075439453,
      "activations/layer22_attention_weight_max": 30.57590675354004,
      "activations/layer22_attention_weight_min": -24.846168518066406,
      "activations/layer23_attention_weight_max": 34.95720672607422,
      "activations/layer23_attention_weight_min": -22.126956939697266,
      "activations/layer2_attention_weight_max": 32.449459075927734,
      "activations/layer2_attention_weight_min": -31.07887077331543,
      "activations/layer3_attention_weight_max": 93.85057830810547,
      "activations/layer3_attention_weight_min": -90.87921142578125,
      "activations/layer4_attention_weight_max": 95.95268249511719,
      "activations/layer4_attention_weight_min": -93.79621124267578,
      "activations/layer5_attention_weight_max": 71.65408325195312,
      "activations/layer5_attention_weight_min": -80.06143188476562,
      "activations/layer6_attention_weight_max": 56.35136795043945,
      "activations/layer6_attention_weight_min": -55.17357635498047,
      "activations/layer7_attention_weight_max": 76.0888442993164,
      "activations/layer7_attention_weight_min": -73.99494934082031,
      "activations/layer8_attention_weight_max": 45.292057037353516,
      "activations/layer8_attention_weight_min": -48.67036819458008,
      "activations/layer9_attention_weight_max": 48.610198974609375,
      "activations/layer9_attention_weight_min": -48.660064697265625,
      "epoch": 10.88,
      "learning_rate": 8.064848484848484e-05,
      "loss": 2.7929,
      "step": 187250
    },
    {
      "activations/layer0_attention_weight_max": 15.76125717163086,
      "activations/layer0_attention_weight_min": -14.256132125854492,
      "activations/layer10_attention_weight_max": 36.97323226928711,
      "activations/layer10_attention_weight_min": -37.37628936767578,
      "activations/layer11_attention_weight_max": 32.71002960205078,
      "activations/layer11_attention_weight_min": -33.543155670166016,
      "activations/layer12_attention_weight_max": 17.436969757080078,
      "activations/layer12_attention_weight_min": -26.527755737304688,
      "activations/layer13_attention_weight_max": 36.334800720214844,
      "activations/layer13_attention_weight_min": -29.681121826171875,
      "activations/layer14_attention_weight_max": 35.6407356262207,
      "activations/layer14_attention_weight_min": -30.080598831176758,
      "activations/layer15_attention_weight_max": 32.25936508178711,
      "activations/layer15_attention_weight_min": -28.300607681274414,
      "activations/layer16_attention_weight_max": 29.560853958129883,
      "activations/layer16_attention_weight_min": -28.327613830566406,
      "activations/layer17_attention_weight_max": 53.085792541503906,
      "activations/layer17_attention_weight_min": -41.5828742980957,
      "activations/layer18_attention_weight_max": 47.32017517089844,
      "activations/layer18_attention_weight_min": -35.785640716552734,
      "activations/layer19_attention_weight_max": 21.7756290435791,
      "activations/layer19_attention_weight_min": -21.085660934448242,
      "activations/layer1_attention_weight_max": 16.262563705444336,
      "activations/layer1_attention_weight_min": -13.494803428649902,
      "activations/layer20_attention_weight_max": 23.00005340576172,
      "activations/layer20_attention_weight_min": -20.921201705932617,
      "activations/layer21_attention_weight_max": 31.826250076293945,
      "activations/layer21_attention_weight_min": -22.177785873413086,
      "activations/layer22_attention_weight_max": 28.60381317138672,
      "activations/layer22_attention_weight_min": -24.014911651611328,
      "activations/layer23_attention_weight_max": 38.90238952636719,
      "activations/layer23_attention_weight_min": -23.91577911376953,
      "activations/layer2_attention_weight_max": 32.106632232666016,
      "activations/layer2_attention_weight_min": -29.33734703063965,
      "activations/layer3_attention_weight_max": 93.57665252685547,
      "activations/layer3_attention_weight_min": -93.30562591552734,
      "activations/layer4_attention_weight_max": 101.47773742675781,
      "activations/layer4_attention_weight_min": -100.42729187011719,
      "activations/layer5_attention_weight_max": 73.55604553222656,
      "activations/layer5_attention_weight_min": -77.68995666503906,
      "activations/layer6_attention_weight_max": 56.89494705200195,
      "activations/layer6_attention_weight_min": -53.748023986816406,
      "activations/layer7_attention_weight_max": 75.34561920166016,
      "activations/layer7_attention_weight_min": -77.84524536132812,
      "activations/layer8_attention_weight_max": 49.25415802001953,
      "activations/layer8_attention_weight_min": -52.92573928833008,
      "activations/layer9_attention_weight_max": 52.01528549194336,
      "activations/layer9_attention_weight_min": -51.20654296875,
      "epoch": 10.88,
      "learning_rate": 8.062954545454545e-05,
      "loss": 2.7871,
      "step": 187300
    },
    {
      "activations/layer0_attention_weight_max": 17.851259231567383,
      "activations/layer0_attention_weight_min": -14.430185317993164,
      "activations/layer10_attention_weight_max": 34.29060363769531,
      "activations/layer10_attention_weight_min": -35.423492431640625,
      "activations/layer11_attention_weight_max": 29.836864471435547,
      "activations/layer11_attention_weight_min": -33.669273376464844,
      "activations/layer12_attention_weight_max": 17.90300178527832,
      "activations/layer12_attention_weight_min": -24.211870193481445,
      "activations/layer13_attention_weight_max": 39.45049285888672,
      "activations/layer13_attention_weight_min": -29.930810928344727,
      "activations/layer14_attention_weight_max": 36.80729675292969,
      "activations/layer14_attention_weight_min": -30.923831939697266,
      "activations/layer15_attention_weight_max": 33.51148223876953,
      "activations/layer15_attention_weight_min": -31.077892303466797,
      "activations/layer16_attention_weight_max": 30.469493865966797,
      "activations/layer16_attention_weight_min": -29.364700317382812,
      "activations/layer17_attention_weight_max": 51.734676361083984,
      "activations/layer17_attention_weight_min": -44.78994369506836,
      "activations/layer18_attention_weight_max": 48.287174224853516,
      "activations/layer18_attention_weight_min": -38.69982147216797,
      "activations/layer19_attention_weight_max": 22.17691421508789,
      "activations/layer19_attention_weight_min": -21.746028900146484,
      "activations/layer1_attention_weight_max": 15.97620677947998,
      "activations/layer1_attention_weight_min": -13.758528709411621,
      "activations/layer20_attention_weight_max": 20.73407745361328,
      "activations/layer20_attention_weight_min": -20.006229400634766,
      "activations/layer21_attention_weight_max": 36.58453369140625,
      "activations/layer21_attention_weight_min": -24.037822723388672,
      "activations/layer22_attention_weight_max": 28.83077621459961,
      "activations/layer22_attention_weight_min": -25.595783233642578,
      "activations/layer23_attention_weight_max": 34.36162185668945,
      "activations/layer23_attention_weight_min": -28.535091400146484,
      "activations/layer2_attention_weight_max": 30.79427146911621,
      "activations/layer2_attention_weight_min": -32.018951416015625,
      "activations/layer3_attention_weight_max": 87.79087829589844,
      "activations/layer3_attention_weight_min": -90.79423522949219,
      "activations/layer4_attention_weight_max": 96.27706146240234,
      "activations/layer4_attention_weight_min": -95.90922546386719,
      "activations/layer5_attention_weight_max": 73.4963150024414,
      "activations/layer5_attention_weight_min": -81.18841552734375,
      "activations/layer6_attention_weight_max": 52.94819641113281,
      "activations/layer6_attention_weight_min": -56.159671783447266,
      "activations/layer7_attention_weight_max": 75.17271423339844,
      "activations/layer7_attention_weight_min": -70.35364532470703,
      "activations/layer8_attention_weight_max": 44.392433166503906,
      "activations/layer8_attention_weight_min": -45.209049224853516,
      "activations/layer9_attention_weight_max": 46.28955078125,
      "activations/layer9_attention_weight_min": -46.51129913330078,
      "epoch": 10.89,
      "learning_rate": 8.061060606060605e-05,
      "loss": 2.7912,
      "step": 187350
    },
    {
      "activations/layer0_attention_weight_max": 14.95744800567627,
      "activations/layer0_attention_weight_min": -14.52923583984375,
      "activations/layer10_attention_weight_max": 37.44770812988281,
      "activations/layer10_attention_weight_min": -36.21346664428711,
      "activations/layer11_attention_weight_max": 32.30147171020508,
      "activations/layer11_attention_weight_min": -34.67449951171875,
      "activations/layer12_attention_weight_max": 17.6978816986084,
      "activations/layer12_attention_weight_min": -26.434202194213867,
      "activations/layer13_attention_weight_max": 39.07563781738281,
      "activations/layer13_attention_weight_min": -29.56435775756836,
      "activations/layer14_attention_weight_max": 40.03443145751953,
      "activations/layer14_attention_weight_min": -30.061433792114258,
      "activations/layer15_attention_weight_max": 33.94383239746094,
      "activations/layer15_attention_weight_min": -28.089651107788086,
      "activations/layer16_attention_weight_max": 29.24443244934082,
      "activations/layer16_attention_weight_min": -28.609256744384766,
      "activations/layer17_attention_weight_max": 51.90272903442383,
      "activations/layer17_attention_weight_min": -43.568294525146484,
      "activations/layer18_attention_weight_max": 47.079410552978516,
      "activations/layer18_attention_weight_min": -36.8904914855957,
      "activations/layer19_attention_weight_max": 21.96864891052246,
      "activations/layer19_attention_weight_min": -22.899396896362305,
      "activations/layer1_attention_weight_max": 17.236284255981445,
      "activations/layer1_attention_weight_min": -16.387720108032227,
      "activations/layer20_attention_weight_max": 24.35544204711914,
      "activations/layer20_attention_weight_min": -24.4258975982666,
      "activations/layer21_attention_weight_max": 41.470420837402344,
      "activations/layer21_attention_weight_min": -24.026275634765625,
      "activations/layer22_attention_weight_max": 34.83103561401367,
      "activations/layer22_attention_weight_min": -25.47885513305664,
      "activations/layer23_attention_weight_max": 38.411014556884766,
      "activations/layer23_attention_weight_min": -21.86577606201172,
      "activations/layer2_attention_weight_max": 32.730369567871094,
      "activations/layer2_attention_weight_min": -32.479766845703125,
      "activations/layer3_attention_weight_max": 91.03495788574219,
      "activations/layer3_attention_weight_min": -93.55216217041016,
      "activations/layer4_attention_weight_max": 98.27696228027344,
      "activations/layer4_attention_weight_min": -95.95515441894531,
      "activations/layer5_attention_weight_max": 71.20640563964844,
      "activations/layer5_attention_weight_min": -76.35502624511719,
      "activations/layer6_attention_weight_max": 54.952579498291016,
      "activations/layer6_attention_weight_min": -52.2201042175293,
      "activations/layer7_attention_weight_max": 73.55062103271484,
      "activations/layer7_attention_weight_min": -70.07540130615234,
      "activations/layer8_attention_weight_max": 48.03411865234375,
      "activations/layer8_attention_weight_min": -49.10365676879883,
      "activations/layer9_attention_weight_max": 49.47007369995117,
      "activations/layer9_attention_weight_min": -49.98916244506836,
      "epoch": 10.89,
      "learning_rate": 8.059166666666666e-05,
      "loss": 2.7944,
      "step": 187400
    },
    {
      "activations/layer0_attention_weight_max": 16.16679573059082,
      "activations/layer0_attention_weight_min": -14.470756530761719,
      "activations/layer10_attention_weight_max": 37.59886169433594,
      "activations/layer10_attention_weight_min": -35.30128860473633,
      "activations/layer11_attention_weight_max": 31.93800926208496,
      "activations/layer11_attention_weight_min": -31.247465133666992,
      "activations/layer12_attention_weight_max": 18.01113510131836,
      "activations/layer12_attention_weight_min": -24.38604736328125,
      "activations/layer13_attention_weight_max": 37.502410888671875,
      "activations/layer13_attention_weight_min": -30.7093505859375,
      "activations/layer14_attention_weight_max": 35.43397903442383,
      "activations/layer14_attention_weight_min": -28.938434600830078,
      "activations/layer15_attention_weight_max": 33.27964401245117,
      "activations/layer15_attention_weight_min": -27.795785903930664,
      "activations/layer16_attention_weight_max": 32.797752380371094,
      "activations/layer16_attention_weight_min": -27.796634674072266,
      "activations/layer17_attention_weight_max": 51.86442947387695,
      "activations/layer17_attention_weight_min": -43.63602828979492,
      "activations/layer18_attention_weight_max": 45.01922607421875,
      "activations/layer18_attention_weight_min": -38.4962043762207,
      "activations/layer19_attention_weight_max": 21.40768051147461,
      "activations/layer19_attention_weight_min": -19.92623519897461,
      "activations/layer1_attention_weight_max": 17.90865707397461,
      "activations/layer1_attention_weight_min": -14.016921997070312,
      "activations/layer20_attention_weight_max": 20.094837188720703,
      "activations/layer20_attention_weight_min": -22.64173126220703,
      "activations/layer21_attention_weight_max": 34.12009811401367,
      "activations/layer21_attention_weight_min": -24.028417587280273,
      "activations/layer22_attention_weight_max": 27.85694122314453,
      "activations/layer22_attention_weight_min": -25.991907119750977,
      "activations/layer23_attention_weight_max": 40.73421859741211,
      "activations/layer23_attention_weight_min": -26.821533203125,
      "activations/layer2_attention_weight_max": 32.38972091674805,
      "activations/layer2_attention_weight_min": -30.64789581298828,
      "activations/layer3_attention_weight_max": 88.3030014038086,
      "activations/layer3_attention_weight_min": -89.2934799194336,
      "activations/layer4_attention_weight_max": 96.26763153076172,
      "activations/layer4_attention_weight_min": -90.22703552246094,
      "activations/layer5_attention_weight_max": 70.62251281738281,
      "activations/layer5_attention_weight_min": -74.361572265625,
      "activations/layer6_attention_weight_max": 52.49756622314453,
      "activations/layer6_attention_weight_min": -52.020023345947266,
      "activations/layer7_attention_weight_max": 72.96492767333984,
      "activations/layer7_attention_weight_min": -75.59974670410156,
      "activations/layer8_attention_weight_max": 46.68963623046875,
      "activations/layer8_attention_weight_min": -46.226356506347656,
      "activations/layer9_attention_weight_max": 48.09453582763672,
      "activations/layer9_attention_weight_min": -46.801876068115234,
      "epoch": 10.89,
      "learning_rate": 8.057272727272727e-05,
      "loss": 2.7727,
      "step": 187450
    },
    {
      "activations/layer0_attention_weight_max": 15.920856475830078,
      "activations/layer0_attention_weight_min": -14.394655227661133,
      "activations/layer10_attention_weight_max": 36.294864654541016,
      "activations/layer10_attention_weight_min": -34.807289123535156,
      "activations/layer11_attention_weight_max": 31.73733901977539,
      "activations/layer11_attention_weight_min": -32.656341552734375,
      "activations/layer12_attention_weight_max": 18.309852600097656,
      "activations/layer12_attention_weight_min": -25.123075485229492,
      "activations/layer13_attention_weight_max": 37.41442108154297,
      "activations/layer13_attention_weight_min": -28.394760131835938,
      "activations/layer14_attention_weight_max": 36.022216796875,
      "activations/layer14_attention_weight_min": -31.750782012939453,
      "activations/layer15_attention_weight_max": 31.83946990966797,
      "activations/layer15_attention_weight_min": -28.710372924804688,
      "activations/layer16_attention_weight_max": 31.548572540283203,
      "activations/layer16_attention_weight_min": -29.647541046142578,
      "activations/layer17_attention_weight_max": 51.05592727661133,
      "activations/layer17_attention_weight_min": -42.67643737792969,
      "activations/layer18_attention_weight_max": 45.89317321777344,
      "activations/layer18_attention_weight_min": -37.535118103027344,
      "activations/layer19_attention_weight_max": 22.89644432067871,
      "activations/layer19_attention_weight_min": -23.462114334106445,
      "activations/layer1_attention_weight_max": 17.90680694580078,
      "activations/layer1_attention_weight_min": -13.451796531677246,
      "activations/layer20_attention_weight_max": 21.29197120666504,
      "activations/layer20_attention_weight_min": -21.588342666625977,
      "activations/layer21_attention_weight_max": 34.13423538208008,
      "activations/layer21_attention_weight_min": -23.33505630493164,
      "activations/layer22_attention_weight_max": 31.464603424072266,
      "activations/layer22_attention_weight_min": -26.90922737121582,
      "activations/layer23_attention_weight_max": 35.30373764038086,
      "activations/layer23_attention_weight_min": -24.540014266967773,
      "activations/layer2_attention_weight_max": 32.91300964355469,
      "activations/layer2_attention_weight_min": -31.19033432006836,
      "activations/layer3_attention_weight_max": 93.82360076904297,
      "activations/layer3_attention_weight_min": -97.12618255615234,
      "activations/layer4_attention_weight_max": 95.82929229736328,
      "activations/layer4_attention_weight_min": -96.74864959716797,
      "activations/layer5_attention_weight_max": 71.47956085205078,
      "activations/layer5_attention_weight_min": -76.89202880859375,
      "activations/layer6_attention_weight_max": 53.91742706298828,
      "activations/layer6_attention_weight_min": -50.95325469970703,
      "activations/layer7_attention_weight_max": 78.39460754394531,
      "activations/layer7_attention_weight_min": -69.92497253417969,
      "activations/layer8_attention_weight_max": 43.83578109741211,
      "activations/layer8_attention_weight_min": -46.7675666809082,
      "activations/layer9_attention_weight_max": 46.771827697753906,
      "activations/layer9_attention_weight_min": -46.52257537841797,
      "epoch": 10.89,
      "learning_rate": 8.055378787878787e-05,
      "loss": 2.7952,
      "step": 187500
    },
    {
      "activations/layer0_attention_weight_max": 14.968009948730469,
      "activations/layer0_attention_weight_min": -14.433392524719238,
      "activations/layer10_attention_weight_max": 37.796024322509766,
      "activations/layer10_attention_weight_min": -34.86420440673828,
      "activations/layer11_attention_weight_max": 34.249671936035156,
      "activations/layer11_attention_weight_min": -32.75025939941406,
      "activations/layer12_attention_weight_max": 18.34580421447754,
      "activations/layer12_attention_weight_min": -23.83869743347168,
      "activations/layer13_attention_weight_max": 42.145103454589844,
      "activations/layer13_attention_weight_min": -29.749794006347656,
      "activations/layer14_attention_weight_max": 37.98430633544922,
      "activations/layer14_attention_weight_min": -31.398880004882812,
      "activations/layer15_attention_weight_max": 33.76131057739258,
      "activations/layer15_attention_weight_min": -29.767789840698242,
      "activations/layer16_attention_weight_max": 32.343841552734375,
      "activations/layer16_attention_weight_min": -31.97013282775879,
      "activations/layer17_attention_weight_max": 52.36470413208008,
      "activations/layer17_attention_weight_min": -45.887001037597656,
      "activations/layer18_attention_weight_max": 46.17881774902344,
      "activations/layer18_attention_weight_min": -40.07006072998047,
      "activations/layer19_attention_weight_max": 21.51900291442871,
      "activations/layer19_attention_weight_min": -22.839019775390625,
      "activations/layer1_attention_weight_max": 17.67186737060547,
      "activations/layer1_attention_weight_min": -13.60261344909668,
      "activations/layer20_attention_weight_max": 21.86589241027832,
      "activations/layer20_attention_weight_min": -21.91460418701172,
      "activations/layer21_attention_weight_max": 36.52815628051758,
      "activations/layer21_attention_weight_min": -23.621170043945312,
      "activations/layer22_attention_weight_max": 31.922388076782227,
      "activations/layer22_attention_weight_min": -30.294721603393555,
      "activations/layer23_attention_weight_max": 37.512725830078125,
      "activations/layer23_attention_weight_min": -25.51825714111328,
      "activations/layer2_attention_weight_max": 31.98446273803711,
      "activations/layer2_attention_weight_min": -29.894254684448242,
      "activations/layer3_attention_weight_max": 92.22061157226562,
      "activations/layer3_attention_weight_min": -91.42064666748047,
      "activations/layer4_attention_weight_max": 96.46358489990234,
      "activations/layer4_attention_weight_min": -94.80138397216797,
      "activations/layer5_attention_weight_max": 71.331298828125,
      "activations/layer5_attention_weight_min": -78.92062377929688,
      "activations/layer6_attention_weight_max": 53.47181701660156,
      "activations/layer6_attention_weight_min": -52.51726150512695,
      "activations/layer7_attention_weight_max": 77.15088653564453,
      "activations/layer7_attention_weight_min": -75.33882141113281,
      "activations/layer8_attention_weight_max": 45.0969123840332,
      "activations/layer8_attention_weight_min": -50.234375,
      "activations/layer9_attention_weight_max": 48.91205596923828,
      "activations/layer9_attention_weight_min": -50.16670227050781,
      "epoch": 10.9,
      "learning_rate": 8.053484848484847e-05,
      "loss": 2.7684,
      "step": 187550
    },
    {
      "activations/layer0_attention_weight_max": 16.045259475708008,
      "activations/layer0_attention_weight_min": -14.67824649810791,
      "activations/layer10_attention_weight_max": 37.2415771484375,
      "activations/layer10_attention_weight_min": -35.424556732177734,
      "activations/layer11_attention_weight_max": 31.586729049682617,
      "activations/layer11_attention_weight_min": -31.78656578063965,
      "activations/layer12_attention_weight_max": 18.630361557006836,
      "activations/layer12_attention_weight_min": -26.810848236083984,
      "activations/layer13_attention_weight_max": 41.123817443847656,
      "activations/layer13_attention_weight_min": -29.501117706298828,
      "activations/layer14_attention_weight_max": 39.4428596496582,
      "activations/layer14_attention_weight_min": -30.89694595336914,
      "activations/layer15_attention_weight_max": 34.9295539855957,
      "activations/layer15_attention_weight_min": -28.049509048461914,
      "activations/layer16_attention_weight_max": 32.87814712524414,
      "activations/layer16_attention_weight_min": -27.145034790039062,
      "activations/layer17_attention_weight_max": 50.35948181152344,
      "activations/layer17_attention_weight_min": -41.249542236328125,
      "activations/layer18_attention_weight_max": 42.4802131652832,
      "activations/layer18_attention_weight_min": -38.445892333984375,
      "activations/layer19_attention_weight_max": 22.609540939331055,
      "activations/layer19_attention_weight_min": -21.108535766601562,
      "activations/layer1_attention_weight_max": 18.004255294799805,
      "activations/layer1_attention_weight_min": -15.014801979064941,
      "activations/layer20_attention_weight_max": 22.018470764160156,
      "activations/layer20_attention_weight_min": -20.981754302978516,
      "activations/layer21_attention_weight_max": 36.956573486328125,
      "activations/layer21_attention_weight_min": -24.282024383544922,
      "activations/layer22_attention_weight_max": 29.43633270263672,
      "activations/layer22_attention_weight_min": -24.44292640686035,
      "activations/layer23_attention_weight_max": 35.90499496459961,
      "activations/layer23_attention_weight_min": -25.301036834716797,
      "activations/layer2_attention_weight_max": 33.7403450012207,
      "activations/layer2_attention_weight_min": -31.03404998779297,
      "activations/layer3_attention_weight_max": 94.11964416503906,
      "activations/layer3_attention_weight_min": -99.61407470703125,
      "activations/layer4_attention_weight_max": 100.73674774169922,
      "activations/layer4_attention_weight_min": -93.50244903564453,
      "activations/layer5_attention_weight_max": 72.49308776855469,
      "activations/layer5_attention_weight_min": -73.85986328125,
      "activations/layer6_attention_weight_max": 52.7663688659668,
      "activations/layer6_attention_weight_min": -52.96943283081055,
      "activations/layer7_attention_weight_max": 74.2756118774414,
      "activations/layer7_attention_weight_min": -75.9706039428711,
      "activations/layer8_attention_weight_max": 45.33302307128906,
      "activations/layer8_attention_weight_min": -51.346038818359375,
      "activations/layer9_attention_weight_max": 48.74801254272461,
      "activations/layer9_attention_weight_min": -50.58302688598633,
      "epoch": 10.9,
      "learning_rate": 8.051590909090909e-05,
      "loss": 2.7809,
      "step": 187600
    },
    {
      "activations/layer0_attention_weight_max": 14.82793140411377,
      "activations/layer0_attention_weight_min": -15.095721244812012,
      "activations/layer10_attention_weight_max": 35.230037689208984,
      "activations/layer10_attention_weight_min": -36.626739501953125,
      "activations/layer11_attention_weight_max": 31.241863250732422,
      "activations/layer11_attention_weight_min": -33.935752868652344,
      "activations/layer12_attention_weight_max": 21.248828887939453,
      "activations/layer12_attention_weight_min": -29.46904182434082,
      "activations/layer13_attention_weight_max": 47.55646514892578,
      "activations/layer13_attention_weight_min": -34.308448791503906,
      "activations/layer14_attention_weight_max": 36.10762023925781,
      "activations/layer14_attention_weight_min": -29.82013702392578,
      "activations/layer15_attention_weight_max": 33.32078170776367,
      "activations/layer15_attention_weight_min": -28.836589813232422,
      "activations/layer16_attention_weight_max": 33.25920104980469,
      "activations/layer16_attention_weight_min": -30.17890167236328,
      "activations/layer17_attention_weight_max": 50.7398681640625,
      "activations/layer17_attention_weight_min": -43.25706100463867,
      "activations/layer18_attention_weight_max": 45.458351135253906,
      "activations/layer18_attention_weight_min": -36.85204315185547,
      "activations/layer19_attention_weight_max": 21.402244567871094,
      "activations/layer19_attention_weight_min": -20.002958297729492,
      "activations/layer1_attention_weight_max": 17.93071746826172,
      "activations/layer1_attention_weight_min": -16.056591033935547,
      "activations/layer20_attention_weight_max": 23.531476974487305,
      "activations/layer20_attention_weight_min": -20.476619720458984,
      "activations/layer21_attention_weight_max": 34.108123779296875,
      "activations/layer21_attention_weight_min": -22.5631160736084,
      "activations/layer22_attention_weight_max": 33.39432144165039,
      "activations/layer22_attention_weight_min": -26.426742553710938,
      "activations/layer23_attention_weight_max": 42.27546691894531,
      "activations/layer23_attention_weight_min": -26.13843536376953,
      "activations/layer2_attention_weight_max": 34.15226364135742,
      "activations/layer2_attention_weight_min": -30.268911361694336,
      "activations/layer3_attention_weight_max": 92.31103515625,
      "activations/layer3_attention_weight_min": -94.0691146850586,
      "activations/layer4_attention_weight_max": 99.32311248779297,
      "activations/layer4_attention_weight_min": -97.66973114013672,
      "activations/layer5_attention_weight_max": 70.90494537353516,
      "activations/layer5_attention_weight_min": -79.15470886230469,
      "activations/layer6_attention_weight_max": 55.436241149902344,
      "activations/layer6_attention_weight_min": -54.2919807434082,
      "activations/layer7_attention_weight_max": 75.37682342529297,
      "activations/layer7_attention_weight_min": -81.50672912597656,
      "activations/layer8_attention_weight_max": 46.37568283081055,
      "activations/layer8_attention_weight_min": -54.16378402709961,
      "activations/layer9_attention_weight_max": 48.686363220214844,
      "activations/layer9_attention_weight_min": -49.153865814208984,
      "epoch": 10.9,
      "learning_rate": 8.049696969696968e-05,
      "loss": 2.7864,
      "step": 187650
    },
    {
      "activations/layer0_attention_weight_max": 16.776771545410156,
      "activations/layer0_attention_weight_min": -14.903449058532715,
      "activations/layer10_attention_weight_max": 34.80604934692383,
      "activations/layer10_attention_weight_min": -33.47412109375,
      "activations/layer11_attention_weight_max": 35.223323822021484,
      "activations/layer11_attention_weight_min": -31.142578125,
      "activations/layer12_attention_weight_max": 17.687328338623047,
      "activations/layer12_attention_weight_min": -26.051298141479492,
      "activations/layer13_attention_weight_max": 40.678619384765625,
      "activations/layer13_attention_weight_min": -29.95526123046875,
      "activations/layer14_attention_weight_max": 43.67356491088867,
      "activations/layer14_attention_weight_min": -29.279541015625,
      "activations/layer15_attention_weight_max": 39.182796478271484,
      "activations/layer15_attention_weight_min": -29.395387649536133,
      "activations/layer16_attention_weight_max": 29.93732452392578,
      "activations/layer16_attention_weight_min": -30.031829833984375,
      "activations/layer17_attention_weight_max": 49.71192932128906,
      "activations/layer17_attention_weight_min": -45.32570266723633,
      "activations/layer18_attention_weight_max": 42.95560073852539,
      "activations/layer18_attention_weight_min": -36.62934494018555,
      "activations/layer19_attention_weight_max": 20.583261489868164,
      "activations/layer19_attention_weight_min": -22.61115837097168,
      "activations/layer1_attention_weight_max": 16.571453094482422,
      "activations/layer1_attention_weight_min": -14.321907997131348,
      "activations/layer20_attention_weight_max": 22.14259147644043,
      "activations/layer20_attention_weight_min": -26.460039138793945,
      "activations/layer21_attention_weight_max": 42.48744583129883,
      "activations/layer21_attention_weight_min": -24.063554763793945,
      "activations/layer22_attention_weight_max": 29.026649475097656,
      "activations/layer22_attention_weight_min": -25.905807495117188,
      "activations/layer23_attention_weight_max": 35.391807556152344,
      "activations/layer23_attention_weight_min": -23.156551361083984,
      "activations/layer2_attention_weight_max": 35.83000183105469,
      "activations/layer2_attention_weight_min": -30.335941314697266,
      "activations/layer3_attention_weight_max": 93.01425170898438,
      "activations/layer3_attention_weight_min": -93.3252944946289,
      "activations/layer4_attention_weight_max": 97.21139526367188,
      "activations/layer4_attention_weight_min": -92.66423797607422,
      "activations/layer5_attention_weight_max": 71.7873306274414,
      "activations/layer5_attention_weight_min": -73.94454956054688,
      "activations/layer6_attention_weight_max": 53.0757942199707,
      "activations/layer6_attention_weight_min": -52.10655975341797,
      "activations/layer7_attention_weight_max": 74.0597152709961,
      "activations/layer7_attention_weight_min": -69.62810516357422,
      "activations/layer8_attention_weight_max": 45.46245574951172,
      "activations/layer8_attention_weight_min": -46.42439651489258,
      "activations/layer9_attention_weight_max": 55.296443939208984,
      "activations/layer9_attention_weight_min": -48.18160629272461,
      "epoch": 10.91,
      "learning_rate": 8.047803030303029e-05,
      "loss": 2.7885,
      "step": 187700
    },
    {
      "activations/layer0_attention_weight_max": 15.04190731048584,
      "activations/layer0_attention_weight_min": -14.276714324951172,
      "activations/layer10_attention_weight_max": 36.904449462890625,
      "activations/layer10_attention_weight_min": -35.851036071777344,
      "activations/layer11_attention_weight_max": 34.643707275390625,
      "activations/layer11_attention_weight_min": -34.820823669433594,
      "activations/layer12_attention_weight_max": 18.28034782409668,
      "activations/layer12_attention_weight_min": -21.428842544555664,
      "activations/layer13_attention_weight_max": 37.08728790283203,
      "activations/layer13_attention_weight_min": -32.01852798461914,
      "activations/layer14_attention_weight_max": 35.37089157104492,
      "activations/layer14_attention_weight_min": -30.757801055908203,
      "activations/layer15_attention_weight_max": 31.723934173583984,
      "activations/layer15_attention_weight_min": -28.700796127319336,
      "activations/layer16_attention_weight_max": 29.048208236694336,
      "activations/layer16_attention_weight_min": -29.086565017700195,
      "activations/layer17_attention_weight_max": 47.60914993286133,
      "activations/layer17_attention_weight_min": -43.01082229614258,
      "activations/layer18_attention_weight_max": 43.8513298034668,
      "activations/layer18_attention_weight_min": -38.596954345703125,
      "activations/layer19_attention_weight_max": 22.439985275268555,
      "activations/layer19_attention_weight_min": -21.97677993774414,
      "activations/layer1_attention_weight_max": 18.312437057495117,
      "activations/layer1_attention_weight_min": -13.957204818725586,
      "activations/layer20_attention_weight_max": 23.1798095703125,
      "activations/layer20_attention_weight_min": -23.960205078125,
      "activations/layer21_attention_weight_max": 36.27846145629883,
      "activations/layer21_attention_weight_min": -27.393882751464844,
      "activations/layer22_attention_weight_max": 27.17038917541504,
      "activations/layer22_attention_weight_min": -26.679786682128906,
      "activations/layer23_attention_weight_max": 34.584632873535156,
      "activations/layer23_attention_weight_min": -23.019813537597656,
      "activations/layer2_attention_weight_max": 32.610069274902344,
      "activations/layer2_attention_weight_min": -30.89529037475586,
      "activations/layer3_attention_weight_max": 97.52356719970703,
      "activations/layer3_attention_weight_min": -95.97689819335938,
      "activations/layer4_attention_weight_max": 98.2699966430664,
      "activations/layer4_attention_weight_min": -94.4839096069336,
      "activations/layer5_attention_weight_max": 74.84326171875,
      "activations/layer5_attention_weight_min": -75.6378173828125,
      "activations/layer6_attention_weight_max": 54.4005241394043,
      "activations/layer6_attention_weight_min": -53.772403717041016,
      "activations/layer7_attention_weight_max": 80.48049926757812,
      "activations/layer7_attention_weight_min": -74.50170135498047,
      "activations/layer8_attention_weight_max": 45.77153396606445,
      "activations/layer8_attention_weight_min": -48.18752670288086,
      "activations/layer9_attention_weight_max": 53.33784866333008,
      "activations/layer9_attention_weight_min": -50.05989074707031,
      "epoch": 10.91,
      "learning_rate": 8.045909090909091e-05,
      "loss": 2.7823,
      "step": 187750
    },
    {
      "activations/layer0_attention_weight_max": 16.139198303222656,
      "activations/layer0_attention_weight_min": -14.4628324508667,
      "activations/layer10_attention_weight_max": 37.175819396972656,
      "activations/layer10_attention_weight_min": -37.1027946472168,
      "activations/layer11_attention_weight_max": 35.881656646728516,
      "activations/layer11_attention_weight_min": -33.33209228515625,
      "activations/layer12_attention_weight_max": 19.358169555664062,
      "activations/layer12_attention_weight_min": -27.824203491210938,
      "activations/layer13_attention_weight_max": 40.77742385864258,
      "activations/layer13_attention_weight_min": -29.8846378326416,
      "activations/layer14_attention_weight_max": 35.96770477294922,
      "activations/layer14_attention_weight_min": -29.225257873535156,
      "activations/layer15_attention_weight_max": 31.93352699279785,
      "activations/layer15_attention_weight_min": -29.019838333129883,
      "activations/layer16_attention_weight_max": 28.425382614135742,
      "activations/layer16_attention_weight_min": -29.574832916259766,
      "activations/layer17_attention_weight_max": 48.238616943359375,
      "activations/layer17_attention_weight_min": -43.703609466552734,
      "activations/layer18_attention_weight_max": 45.34601593017578,
      "activations/layer18_attention_weight_min": -39.20558547973633,
      "activations/layer19_attention_weight_max": 22.310256958007812,
      "activations/layer19_attention_weight_min": -21.600683212280273,
      "activations/layer1_attention_weight_max": 18.294347763061523,
      "activations/layer1_attention_weight_min": -16.390918731689453,
      "activations/layer20_attention_weight_max": 23.154346466064453,
      "activations/layer20_attention_weight_min": -19.718900680541992,
      "activations/layer21_attention_weight_max": 35.273231506347656,
      "activations/layer21_attention_weight_min": -24.632089614868164,
      "activations/layer22_attention_weight_max": 27.990070343017578,
      "activations/layer22_attention_weight_min": -23.79734992980957,
      "activations/layer23_attention_weight_max": 34.7769775390625,
      "activations/layer23_attention_weight_min": -27.91918182373047,
      "activations/layer2_attention_weight_max": 33.78656768798828,
      "activations/layer2_attention_weight_min": -32.15015411376953,
      "activations/layer3_attention_weight_max": 94.11336517333984,
      "activations/layer3_attention_weight_min": -96.2774887084961,
      "activations/layer4_attention_weight_max": 99.29457092285156,
      "activations/layer4_attention_weight_min": -96.08284759521484,
      "activations/layer5_attention_weight_max": 70.92786407470703,
      "activations/layer5_attention_weight_min": -78.39617156982422,
      "activations/layer6_attention_weight_max": 52.7583122253418,
      "activations/layer6_attention_weight_min": -50.34890365600586,
      "activations/layer7_attention_weight_max": 77.21894836425781,
      "activations/layer7_attention_weight_min": -71.38220977783203,
      "activations/layer8_attention_weight_max": 44.98467254638672,
      "activations/layer8_attention_weight_min": -46.27819061279297,
      "activations/layer9_attention_weight_max": 57.01367950439453,
      "activations/layer9_attention_weight_min": -46.178714752197266,
      "epoch": 10.91,
      "learning_rate": 8.044015151515151e-05,
      "loss": 2.7767,
      "step": 187800
    },
    {
      "activations/layer0_attention_weight_max": 16.232423782348633,
      "activations/layer0_attention_weight_min": -14.432698249816895,
      "activations/layer10_attention_weight_max": 39.55233383178711,
      "activations/layer10_attention_weight_min": -37.932044982910156,
      "activations/layer11_attention_weight_max": 34.429840087890625,
      "activations/layer11_attention_weight_min": -34.723628997802734,
      "activations/layer12_attention_weight_max": 19.980207443237305,
      "activations/layer12_attention_weight_min": -23.509973526000977,
      "activations/layer13_attention_weight_max": 37.165218353271484,
      "activations/layer13_attention_weight_min": -27.683427810668945,
      "activations/layer14_attention_weight_max": 39.65666961669922,
      "activations/layer14_attention_weight_min": -31.861797332763672,
      "activations/layer15_attention_weight_max": 35.10588455200195,
      "activations/layer15_attention_weight_min": -30.80006980895996,
      "activations/layer16_attention_weight_max": 32.348655700683594,
      "activations/layer16_attention_weight_min": -30.50803565979004,
      "activations/layer17_attention_weight_max": 59.82019805908203,
      "activations/layer17_attention_weight_min": -47.3345947265625,
      "activations/layer18_attention_weight_max": 52.11639404296875,
      "activations/layer18_attention_weight_min": -38.83805847167969,
      "activations/layer19_attention_weight_max": 24.276103973388672,
      "activations/layer19_attention_weight_min": -21.812402725219727,
      "activations/layer1_attention_weight_max": 16.436498641967773,
      "activations/layer1_attention_weight_min": -15.306092262268066,
      "activations/layer20_attention_weight_max": 25.59135627746582,
      "activations/layer20_attention_weight_min": -22.865617752075195,
      "activations/layer21_attention_weight_max": 42.08933639526367,
      "activations/layer21_attention_weight_min": -23.777639389038086,
      "activations/layer22_attention_weight_max": 31.826623916625977,
      "activations/layer22_attention_weight_min": -23.86811065673828,
      "activations/layer23_attention_weight_max": 39.25428771972656,
      "activations/layer23_attention_weight_min": -23.3574275970459,
      "activations/layer2_attention_weight_max": 32.58210372924805,
      "activations/layer2_attention_weight_min": -30.894025802612305,
      "activations/layer3_attention_weight_max": 88.59660339355469,
      "activations/layer3_attention_weight_min": -93.63085174560547,
      "activations/layer4_attention_weight_max": 100.93143463134766,
      "activations/layer4_attention_weight_min": -99.6898193359375,
      "activations/layer5_attention_weight_max": 74.64995574951172,
      "activations/layer5_attention_weight_min": -76.86390686035156,
      "activations/layer6_attention_weight_max": 57.3232307434082,
      "activations/layer6_attention_weight_min": -56.62590408325195,
      "activations/layer7_attention_weight_max": 78.5034408569336,
      "activations/layer7_attention_weight_min": -76.6173324584961,
      "activations/layer8_attention_weight_max": 47.77012634277344,
      "activations/layer8_attention_weight_min": -46.87630081176758,
      "activations/layer9_attention_weight_max": 49.00130844116211,
      "activations/layer9_attention_weight_min": -49.41413497924805,
      "epoch": 10.92,
      "learning_rate": 8.042121212121211e-05,
      "loss": 2.7782,
      "step": 187850
    },
    {
      "activations/layer0_attention_weight_max": 15.282112121582031,
      "activations/layer0_attention_weight_min": -14.367311477661133,
      "activations/layer10_attention_weight_max": 35.705745697021484,
      "activations/layer10_attention_weight_min": -36.21352005004883,
      "activations/layer11_attention_weight_max": 31.620079040527344,
      "activations/layer11_attention_weight_min": -33.49040222167969,
      "activations/layer12_attention_weight_max": 20.34474754333496,
      "activations/layer12_attention_weight_min": -29.551952362060547,
      "activations/layer13_attention_weight_max": 40.03062438964844,
      "activations/layer13_attention_weight_min": -32.756378173828125,
      "activations/layer14_attention_weight_max": 37.19997787475586,
      "activations/layer14_attention_weight_min": -35.15953826904297,
      "activations/layer15_attention_weight_max": 33.370887756347656,
      "activations/layer15_attention_weight_min": -33.1918830871582,
      "activations/layer16_attention_weight_max": 32.10489273071289,
      "activations/layer16_attention_weight_min": -30.44770050048828,
      "activations/layer17_attention_weight_max": 50.65496063232422,
      "activations/layer17_attention_weight_min": -45.09335708618164,
      "activations/layer18_attention_weight_max": 46.45288848876953,
      "activations/layer18_attention_weight_min": -41.87440490722656,
      "activations/layer19_attention_weight_max": 25.377788543701172,
      "activations/layer19_attention_weight_min": -20.82050132751465,
      "activations/layer1_attention_weight_max": 16.538951873779297,
      "activations/layer1_attention_weight_min": -14.842971801757812,
      "activations/layer20_attention_weight_max": 23.63504981994629,
      "activations/layer20_attention_weight_min": -21.64592170715332,
      "activations/layer21_attention_weight_max": 35.18465042114258,
      "activations/layer21_attention_weight_min": -21.870939254760742,
      "activations/layer22_attention_weight_max": 35.49580764770508,
      "activations/layer22_attention_weight_min": -25.09803009033203,
      "activations/layer23_attention_weight_max": 37.584617614746094,
      "activations/layer23_attention_weight_min": -23.526287078857422,
      "activations/layer2_attention_weight_max": 33.007179260253906,
      "activations/layer2_attention_weight_min": -31.08907699584961,
      "activations/layer3_attention_weight_max": 87.42842864990234,
      "activations/layer3_attention_weight_min": -91.7616195678711,
      "activations/layer4_attention_weight_max": 97.01895141601562,
      "activations/layer4_attention_weight_min": -91.79354858398438,
      "activations/layer5_attention_weight_max": 71.53384399414062,
      "activations/layer5_attention_weight_min": -75.20123291015625,
      "activations/layer6_attention_weight_max": 51.505897521972656,
      "activations/layer6_attention_weight_min": -52.228023529052734,
      "activations/layer7_attention_weight_max": 78.26312255859375,
      "activations/layer7_attention_weight_min": -75.04037475585938,
      "activations/layer8_attention_weight_max": 45.524169921875,
      "activations/layer8_attention_weight_min": -49.36470413208008,
      "activations/layer9_attention_weight_max": 46.53106689453125,
      "activations/layer9_attention_weight_min": -48.403053283691406,
      "epoch": 10.92,
      "learning_rate": 8.040227272727273e-05,
      "loss": 2.7954,
      "step": 187900
    },
    {
      "activations/layer0_attention_weight_max": 16.77615737915039,
      "activations/layer0_attention_weight_min": -14.781888008117676,
      "activations/layer10_attention_weight_max": 38.105308532714844,
      "activations/layer10_attention_weight_min": -35.64727020263672,
      "activations/layer11_attention_weight_max": 32.97953796386719,
      "activations/layer11_attention_weight_min": -32.15367126464844,
      "activations/layer12_attention_weight_max": 18.576316833496094,
      "activations/layer12_attention_weight_min": -23.844284057617188,
      "activations/layer13_attention_weight_max": 47.73234939575195,
      "activations/layer13_attention_weight_min": -34.90331268310547,
      "activations/layer14_attention_weight_max": 44.215396881103516,
      "activations/layer14_attention_weight_min": -32.044700622558594,
      "activations/layer15_attention_weight_max": 36.52012252807617,
      "activations/layer15_attention_weight_min": -30.446489334106445,
      "activations/layer16_attention_weight_max": 33.46733093261719,
      "activations/layer16_attention_weight_min": -30.63377571105957,
      "activations/layer17_attention_weight_max": 53.124847412109375,
      "activations/layer17_attention_weight_min": -45.486717224121094,
      "activations/layer18_attention_weight_max": 46.6318359375,
      "activations/layer18_attention_weight_min": -38.69186782836914,
      "activations/layer19_attention_weight_max": 24.26671600341797,
      "activations/layer19_attention_weight_min": -21.662023544311523,
      "activations/layer1_attention_weight_max": 16.413698196411133,
      "activations/layer1_attention_weight_min": -16.157943725585938,
      "activations/layer20_attention_weight_max": 26.348356246948242,
      "activations/layer20_attention_weight_min": -21.892532348632812,
      "activations/layer21_attention_weight_max": 44.98711013793945,
      "activations/layer21_attention_weight_min": -24.30333709716797,
      "activations/layer22_attention_weight_max": 33.08700942993164,
      "activations/layer22_attention_weight_min": -25.10616111755371,
      "activations/layer23_attention_weight_max": 42.665714263916016,
      "activations/layer23_attention_weight_min": -25.706188201904297,
      "activations/layer2_attention_weight_max": 33.39167785644531,
      "activations/layer2_attention_weight_min": -32.76789093017578,
      "activations/layer3_attention_weight_max": 95.32766723632812,
      "activations/layer3_attention_weight_min": -98.24409484863281,
      "activations/layer4_attention_weight_max": 100.4786605834961,
      "activations/layer4_attention_weight_min": -100.10438537597656,
      "activations/layer5_attention_weight_max": 74.80082702636719,
      "activations/layer5_attention_weight_min": -81.06903076171875,
      "activations/layer6_attention_weight_max": 53.486793518066406,
      "activations/layer6_attention_weight_min": -52.17691421508789,
      "activations/layer7_attention_weight_max": 74.9542007446289,
      "activations/layer7_attention_weight_min": -73.6687240600586,
      "activations/layer8_attention_weight_max": 46.345001220703125,
      "activations/layer8_attention_weight_min": -51.078582763671875,
      "activations/layer9_attention_weight_max": 47.83323287963867,
      "activations/layer9_attention_weight_min": -48.031856536865234,
      "epoch": 10.92,
      "learning_rate": 8.038333333333333e-05,
      "loss": 2.7963,
      "step": 187950
    },
    {
      "activations/layer0_attention_weight_max": 15.154691696166992,
      "activations/layer0_attention_weight_min": -14.554819107055664,
      "activations/layer10_attention_weight_max": 34.61396789550781,
      "activations/layer10_attention_weight_min": -35.08430862426758,
      "activations/layer11_attention_weight_max": 32.659828186035156,
      "activations/layer11_attention_weight_min": -33.6214599609375,
      "activations/layer12_attention_weight_max": 21.103723526000977,
      "activations/layer12_attention_weight_min": -23.695220947265625,
      "activations/layer13_attention_weight_max": 39.54680633544922,
      "activations/layer13_attention_weight_min": -27.957056045532227,
      "activations/layer14_attention_weight_max": 35.050262451171875,
      "activations/layer14_attention_weight_min": -30.44795799255371,
      "activations/layer15_attention_weight_max": 31.592010498046875,
      "activations/layer15_attention_weight_min": -28.597858428955078,
      "activations/layer16_attention_weight_max": 30.42093849182129,
      "activations/layer16_attention_weight_min": -28.821273803710938,
      "activations/layer17_attention_weight_max": 53.36299514770508,
      "activations/layer17_attention_weight_min": -44.028690338134766,
      "activations/layer18_attention_weight_max": 42.50385284423828,
      "activations/layer18_attention_weight_min": -38.1552848815918,
      "activations/layer19_attention_weight_max": 21.480892181396484,
      "activations/layer19_attention_weight_min": -22.532899856567383,
      "activations/layer1_attention_weight_max": 17.077939987182617,
      "activations/layer1_attention_weight_min": -14.565001487731934,
      "activations/layer20_attention_weight_max": 21.837053298950195,
      "activations/layer20_attention_weight_min": -21.1221866607666,
      "activations/layer21_attention_weight_max": 42.84821701049805,
      "activations/layer21_attention_weight_min": -29.06694221496582,
      "activations/layer22_attention_weight_max": 27.43153190612793,
      "activations/layer22_attention_weight_min": -24.21308135986328,
      "activations/layer23_attention_weight_max": 40.035160064697266,
      "activations/layer23_attention_weight_min": -28.543136596679688,
      "activations/layer2_attention_weight_max": 34.43507385253906,
      "activations/layer2_attention_weight_min": -30.569684982299805,
      "activations/layer3_attention_weight_max": 92.78630065917969,
      "activations/layer3_attention_weight_min": -92.18434143066406,
      "activations/layer4_attention_weight_max": 100.02080535888672,
      "activations/layer4_attention_weight_min": -97.08739471435547,
      "activations/layer5_attention_weight_max": 70.49588775634766,
      "activations/layer5_attention_weight_min": -80.53466033935547,
      "activations/layer6_attention_weight_max": 53.88154220581055,
      "activations/layer6_attention_weight_min": -52.899635314941406,
      "activations/layer7_attention_weight_max": 77.87694549560547,
      "activations/layer7_attention_weight_min": -72.2105484008789,
      "activations/layer8_attention_weight_max": 44.630367279052734,
      "activations/layer8_attention_weight_min": -46.93171691894531,
      "activations/layer9_attention_weight_max": 49.507930755615234,
      "activations/layer9_attention_weight_min": -47.579593658447266,
      "epoch": 10.92,
      "learning_rate": 8.036439393939393e-05,
      "loss": 2.7812,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.5189,
      "eval_samples_per_second": 504.057,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.5189,
      "eval_openwebtext_samples_per_second": 504.057,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.9729,
      "eval_wikitext_samples_per_second": 231.135,
      "step": 188000
    },
    {
      "epoch": 10.92,
      "eval_lambada_loss": 2.578125,
      "eval_lambada_ppl": 13.172416709211404,
      "eval_lambada_runtime": 9.5772,
      "eval_lambada_samples_per_second": 508.393,
      "step": 188000
    },
    {
      "activations/layer0_attention_weight_max": 15.230611801147461,
      "activations/layer0_attention_weight_min": -14.420896530151367,
      "activations/layer10_attention_weight_max": 35.32713317871094,
      "activations/layer10_attention_weight_min": -33.561405181884766,
      "activations/layer11_attention_weight_max": 32.08159637451172,
      "activations/layer11_attention_weight_min": -31.40359878540039,
      "activations/layer12_attention_weight_max": 19.239948272705078,
      "activations/layer12_attention_weight_min": -21.83393096923828,
      "activations/layer13_attention_weight_max": 43.940555572509766,
      "activations/layer13_attention_weight_min": -29.623096466064453,
      "activations/layer14_attention_weight_max": 37.56238555908203,
      "activations/layer14_attention_weight_min": -30.468961715698242,
      "activations/layer15_attention_weight_max": 32.73736572265625,
      "activations/layer15_attention_weight_min": -29.290063858032227,
      "activations/layer16_attention_weight_max": 32.881675720214844,
      "activations/layer16_attention_weight_min": -30.329607009887695,
      "activations/layer17_attention_weight_max": 56.736778259277344,
      "activations/layer17_attention_weight_min": -45.31792068481445,
      "activations/layer18_attention_weight_max": 48.13282775878906,
      "activations/layer18_attention_weight_min": -41.05203628540039,
      "activations/layer19_attention_weight_max": 21.652786254882812,
      "activations/layer19_attention_weight_min": -20.79416275024414,
      "activations/layer1_attention_weight_max": 16.28754234313965,
      "activations/layer1_attention_weight_min": -15.00245189666748,
      "activations/layer20_attention_weight_max": 23.51361656188965,
      "activations/layer20_attention_weight_min": -22.8653621673584,
      "activations/layer21_attention_weight_max": 39.34135818481445,
      "activations/layer21_attention_weight_min": -24.54111671447754,
      "activations/layer22_attention_weight_max": 32.86201477050781,
      "activations/layer22_attention_weight_min": -27.669933319091797,
      "activations/layer23_attention_weight_max": 38.710113525390625,
      "activations/layer23_attention_weight_min": -26.000343322753906,
      "activations/layer2_attention_weight_max": 32.91507339477539,
      "activations/layer2_attention_weight_min": -31.443553924560547,
      "activations/layer3_attention_weight_max": 89.59380340576172,
      "activations/layer3_attention_weight_min": -94.88311767578125,
      "activations/layer4_attention_weight_max": 95.94281005859375,
      "activations/layer4_attention_weight_min": -93.64177703857422,
      "activations/layer5_attention_weight_max": 70.80813598632812,
      "activations/layer5_attention_weight_min": -76.20677185058594,
      "activations/layer6_attention_weight_max": 51.08735656738281,
      "activations/layer6_attention_weight_min": -52.96110916137695,
      "activations/layer7_attention_weight_max": 76.26239013671875,
      "activations/layer7_attention_weight_min": -73.35830688476562,
      "activations/layer8_attention_weight_max": 44.227561950683594,
      "activations/layer8_attention_weight_min": -47.22328186035156,
      "activations/layer9_attention_weight_max": 46.93833541870117,
      "activations/layer9_attention_weight_min": -48.64140319824219,
      "epoch": 10.93,
      "learning_rate": 8.034545454545454e-05,
      "loss": 2.7793,
      "step": 188050
    },
    {
      "activations/layer0_attention_weight_max": 15.017105102539062,
      "activations/layer0_attention_weight_min": -14.437074661254883,
      "activations/layer10_attention_weight_max": 41.05570602416992,
      "activations/layer10_attention_weight_min": -39.79501724243164,
      "activations/layer11_attention_weight_max": 35.352638244628906,
      "activations/layer11_attention_weight_min": -35.362937927246094,
      "activations/layer12_attention_weight_max": 21.521533966064453,
      "activations/layer12_attention_weight_min": -24.635087966918945,
      "activations/layer13_attention_weight_max": 43.647457122802734,
      "activations/layer13_attention_weight_min": -28.880361557006836,
      "activations/layer14_attention_weight_max": 36.98283386230469,
      "activations/layer14_attention_weight_min": -29.732629776000977,
      "activations/layer15_attention_weight_max": 37.670291900634766,
      "activations/layer15_attention_weight_min": -29.343358993530273,
      "activations/layer16_attention_weight_max": 31.640506744384766,
      "activations/layer16_attention_weight_min": -29.20456314086914,
      "activations/layer17_attention_weight_max": 54.4352912902832,
      "activations/layer17_attention_weight_min": -44.58755111694336,
      "activations/layer18_attention_weight_max": 50.34574508666992,
      "activations/layer18_attention_weight_min": -37.3768424987793,
      "activations/layer19_attention_weight_max": 21.516040802001953,
      "activations/layer19_attention_weight_min": -21.47170066833496,
      "activations/layer1_attention_weight_max": 16.2331485748291,
      "activations/layer1_attention_weight_min": -14.575867652893066,
      "activations/layer20_attention_weight_max": 22.58150863647461,
      "activations/layer20_attention_weight_min": -21.521541595458984,
      "activations/layer21_attention_weight_max": 38.007476806640625,
      "activations/layer21_attention_weight_min": -22.1157283782959,
      "activations/layer22_attention_weight_max": 31.54119873046875,
      "activations/layer22_attention_weight_min": -26.16569709777832,
      "activations/layer23_attention_weight_max": 39.398502349853516,
      "activations/layer23_attention_weight_min": -22.242290496826172,
      "activations/layer2_attention_weight_max": 32.303253173828125,
      "activations/layer2_attention_weight_min": -30.94575309753418,
      "activations/layer3_attention_weight_max": 89.64653015136719,
      "activations/layer3_attention_weight_min": -90.41869354248047,
      "activations/layer4_attention_weight_max": 97.27099609375,
      "activations/layer4_attention_weight_min": -92.50885009765625,
      "activations/layer5_attention_weight_max": 72.94358825683594,
      "activations/layer5_attention_weight_min": -76.80084991455078,
      "activations/layer6_attention_weight_max": 55.2012939453125,
      "activations/layer6_attention_weight_min": -55.642333984375,
      "activations/layer7_attention_weight_max": 76.37991333007812,
      "activations/layer7_attention_weight_min": -77.09619903564453,
      "activations/layer8_attention_weight_max": 53.560096740722656,
      "activations/layer8_attention_weight_min": -52.06227493286133,
      "activations/layer9_attention_weight_max": 55.38420486450195,
      "activations/layer9_attention_weight_min": -53.48468780517578,
      "epoch": 10.93,
      "learning_rate": 8.032651515151514e-05,
      "loss": 2.7864,
      "step": 188100
    },
    {
      "activations/layer0_attention_weight_max": 15.301794052124023,
      "activations/layer0_attention_weight_min": -14.305392265319824,
      "activations/layer10_attention_weight_max": 37.05028533935547,
      "activations/layer10_attention_weight_min": -37.18428421020508,
      "activations/layer11_attention_weight_max": 33.026092529296875,
      "activations/layer11_attention_weight_min": -33.53163146972656,
      "activations/layer12_attention_weight_max": 17.810104370117188,
      "activations/layer12_attention_weight_min": -25.0202579498291,
      "activations/layer13_attention_weight_max": 39.31011962890625,
      "activations/layer13_attention_weight_min": -29.554153442382812,
      "activations/layer14_attention_weight_max": 35.1391716003418,
      "activations/layer14_attention_weight_min": -30.178020477294922,
      "activations/layer15_attention_weight_max": 32.05985641479492,
      "activations/layer15_attention_weight_min": -28.267391204833984,
      "activations/layer16_attention_weight_max": 30.102481842041016,
      "activations/layer16_attention_weight_min": -27.860403060913086,
      "activations/layer17_attention_weight_max": 51.57094192504883,
      "activations/layer17_attention_weight_min": -42.692283630371094,
      "activations/layer18_attention_weight_max": 44.84016799926758,
      "activations/layer18_attention_weight_min": -37.37299728393555,
      "activations/layer19_attention_weight_max": 21.119230270385742,
      "activations/layer19_attention_weight_min": -21.757490158081055,
      "activations/layer1_attention_weight_max": 17.408554077148438,
      "activations/layer1_attention_weight_min": -15.090900421142578,
      "activations/layer20_attention_weight_max": 22.209735870361328,
      "activations/layer20_attention_weight_min": -24.451114654541016,
      "activations/layer21_attention_weight_max": 39.05420684814453,
      "activations/layer21_attention_weight_min": -22.092241287231445,
      "activations/layer22_attention_weight_max": 33.34918212890625,
      "activations/layer22_attention_weight_min": -25.23171615600586,
      "activations/layer23_attention_weight_max": 38.11048889160156,
      "activations/layer23_attention_weight_min": -25.02387809753418,
      "activations/layer2_attention_weight_max": 34.6021842956543,
      "activations/layer2_attention_weight_min": -32.53179931640625,
      "activations/layer3_attention_weight_max": 96.67414855957031,
      "activations/layer3_attention_weight_min": -98.5847396850586,
      "activations/layer4_attention_weight_max": 104.799560546875,
      "activations/layer4_attention_weight_min": -97.01936340332031,
      "activations/layer5_attention_weight_max": 75.93939208984375,
      "activations/layer5_attention_weight_min": -80.46076965332031,
      "activations/layer6_attention_weight_max": 54.25190353393555,
      "activations/layer6_attention_weight_min": -52.853572845458984,
      "activations/layer7_attention_weight_max": 76.5528793334961,
      "activations/layer7_attention_weight_min": -77.36006927490234,
      "activations/layer8_attention_weight_max": 50.07966613769531,
      "activations/layer8_attention_weight_min": -49.16943359375,
      "activations/layer9_attention_weight_max": 49.445526123046875,
      "activations/layer9_attention_weight_min": -49.92464828491211,
      "epoch": 10.93,
      "learning_rate": 8.030757575757576e-05,
      "loss": 2.7877,
      "step": 188150
    },
    {
      "activations/layer0_attention_weight_max": 16.32478904724121,
      "activations/layer0_attention_weight_min": -14.428466796875,
      "activations/layer10_attention_weight_max": 35.195350646972656,
      "activations/layer10_attention_weight_min": -35.719635009765625,
      "activations/layer11_attention_weight_max": 32.57954788208008,
      "activations/layer11_attention_weight_min": -35.07953643798828,
      "activations/layer12_attention_weight_max": 18.19025230407715,
      "activations/layer12_attention_weight_min": -30.03769302368164,
      "activations/layer13_attention_weight_max": 38.108863830566406,
      "activations/layer13_attention_weight_min": -29.404733657836914,
      "activations/layer14_attention_weight_max": 37.577178955078125,
      "activations/layer14_attention_weight_min": -30.46013832092285,
      "activations/layer15_attention_weight_max": 36.36543655395508,
      "activations/layer15_attention_weight_min": -30.26741600036621,
      "activations/layer16_attention_weight_max": 35.306724548339844,
      "activations/layer16_attention_weight_min": -28.70738410949707,
      "activations/layer17_attention_weight_max": 51.717857360839844,
      "activations/layer17_attention_weight_min": -43.566558837890625,
      "activations/layer18_attention_weight_max": 46.94489288330078,
      "activations/layer18_attention_weight_min": -39.23042678833008,
      "activations/layer19_attention_weight_max": 23.394296646118164,
      "activations/layer19_attention_weight_min": -21.647212982177734,
      "activations/layer1_attention_weight_max": 17.82866096496582,
      "activations/layer1_attention_weight_min": -14.30696964263916,
      "activations/layer20_attention_weight_max": 27.40557098388672,
      "activations/layer20_attention_weight_min": -21.321788787841797,
      "activations/layer21_attention_weight_max": 37.01002502441406,
      "activations/layer21_attention_weight_min": -24.40337562561035,
      "activations/layer22_attention_weight_max": 29.757810592651367,
      "activations/layer22_attention_weight_min": -24.868560791015625,
      "activations/layer23_attention_weight_max": 38.915000915527344,
      "activations/layer23_attention_weight_min": -26.830013275146484,
      "activations/layer2_attention_weight_max": 33.90853500366211,
      "activations/layer2_attention_weight_min": -29.53204917907715,
      "activations/layer3_attention_weight_max": 101.52684020996094,
      "activations/layer3_attention_weight_min": -93.74124908447266,
      "activations/layer4_attention_weight_max": 106.35649871826172,
      "activations/layer4_attention_weight_min": -96.00343322753906,
      "activations/layer5_attention_weight_max": 72.7691650390625,
      "activations/layer5_attention_weight_min": -83.5416488647461,
      "activations/layer6_attention_weight_max": 55.96284103393555,
      "activations/layer6_attention_weight_min": -53.000728607177734,
      "activations/layer7_attention_weight_max": 84.0827865600586,
      "activations/layer7_attention_weight_min": -74.7118911743164,
      "activations/layer8_attention_weight_max": 46.43067169189453,
      "activations/layer8_attention_weight_min": -46.86671829223633,
      "activations/layer9_attention_weight_max": 51.61029815673828,
      "activations/layer9_attention_weight_min": -48.39115524291992,
      "epoch": 10.94,
      "learning_rate": 8.028863636363635e-05,
      "loss": 2.797,
      "step": 188200
    },
    {
      "activations/layer0_attention_weight_max": 14.73661994934082,
      "activations/layer0_attention_weight_min": -13.830702781677246,
      "activations/layer10_attention_weight_max": 37.45392608642578,
      "activations/layer10_attention_weight_min": -36.79262161254883,
      "activations/layer11_attention_weight_max": 34.079002380371094,
      "activations/layer11_attention_weight_min": -33.52954864501953,
      "activations/layer12_attention_weight_max": 17.633546829223633,
      "activations/layer12_attention_weight_min": -23.939823150634766,
      "activations/layer13_attention_weight_max": 42.058815002441406,
      "activations/layer13_attention_weight_min": -31.538009643554688,
      "activations/layer14_attention_weight_max": 35.95918655395508,
      "activations/layer14_attention_weight_min": -33.21953201293945,
      "activations/layer15_attention_weight_max": 34.18910598754883,
      "activations/layer15_attention_weight_min": -31.41506576538086,
      "activations/layer16_attention_weight_max": 30.057920455932617,
      "activations/layer16_attention_weight_min": -29.137344360351562,
      "activations/layer17_attention_weight_max": 49.55366897583008,
      "activations/layer17_attention_weight_min": -43.20302963256836,
      "activations/layer18_attention_weight_max": 43.92817306518555,
      "activations/layer18_attention_weight_min": -35.95405960083008,
      "activations/layer19_attention_weight_max": 20.420856475830078,
      "activations/layer19_attention_weight_min": -22.50056266784668,
      "activations/layer1_attention_weight_max": 17.039047241210938,
      "activations/layer1_attention_weight_min": -15.328640937805176,
      "activations/layer20_attention_weight_max": 22.649526596069336,
      "activations/layer20_attention_weight_min": -21.152971267700195,
      "activations/layer21_attention_weight_max": 32.503196716308594,
      "activations/layer21_attention_weight_min": -24.297714233398438,
      "activations/layer22_attention_weight_max": 26.71373176574707,
      "activations/layer22_attention_weight_min": -26.266639709472656,
      "activations/layer23_attention_weight_max": 37.93949890136719,
      "activations/layer23_attention_weight_min": -23.9752254486084,
      "activations/layer2_attention_weight_max": 32.99734115600586,
      "activations/layer2_attention_weight_min": -30.05484390258789,
      "activations/layer3_attention_weight_max": 89.31218719482422,
      "activations/layer3_attention_weight_min": -91.08345794677734,
      "activations/layer4_attention_weight_max": 97.59862518310547,
      "activations/layer4_attention_weight_min": -93.47728729248047,
      "activations/layer5_attention_weight_max": 74.05216979980469,
      "activations/layer5_attention_weight_min": -73.18566131591797,
      "activations/layer6_attention_weight_max": 53.79755783081055,
      "activations/layer6_attention_weight_min": -53.06483840942383,
      "activations/layer7_attention_weight_max": 75.4036636352539,
      "activations/layer7_attention_weight_min": -72.70511627197266,
      "activations/layer8_attention_weight_max": 45.94247055053711,
      "activations/layer8_attention_weight_min": -45.3463134765625,
      "activations/layer9_attention_weight_max": 48.954952239990234,
      "activations/layer9_attention_weight_min": -48.14683151245117,
      "epoch": 10.94,
      "learning_rate": 8.026969696969696e-05,
      "loss": 2.7785,
      "step": 188250
    },
    {
      "activations/layer0_attention_weight_max": 17.635456085205078,
      "activations/layer0_attention_weight_min": -14.28857135772705,
      "activations/layer10_attention_weight_max": 38.405364990234375,
      "activations/layer10_attention_weight_min": -39.74396514892578,
      "activations/layer11_attention_weight_max": 34.98017883300781,
      "activations/layer11_attention_weight_min": -33.0201416015625,
      "activations/layer12_attention_weight_max": 25.14885902404785,
      "activations/layer12_attention_weight_min": -22.725013732910156,
      "activations/layer13_attention_weight_max": 45.96442794799805,
      "activations/layer13_attention_weight_min": -33.056488037109375,
      "activations/layer14_attention_weight_max": 51.96370315551758,
      "activations/layer14_attention_weight_min": -32.36378479003906,
      "activations/layer15_attention_weight_max": 47.016265869140625,
      "activations/layer15_attention_weight_min": -29.714235305786133,
      "activations/layer16_attention_weight_max": 33.84351348876953,
      "activations/layer16_attention_weight_min": -32.04671859741211,
      "activations/layer17_attention_weight_max": 55.86348342895508,
      "activations/layer17_attention_weight_min": -45.25865173339844,
      "activations/layer18_attention_weight_max": 51.59832000732422,
      "activations/layer18_attention_weight_min": -39.069847106933594,
      "activations/layer19_attention_weight_max": 21.73138999938965,
      "activations/layer19_attention_weight_min": -21.301620483398438,
      "activations/layer1_attention_weight_max": 19.113630294799805,
      "activations/layer1_attention_weight_min": -14.05248737335205,
      "activations/layer20_attention_weight_max": 25.14095115661621,
      "activations/layer20_attention_weight_min": -21.635250091552734,
      "activations/layer21_attention_weight_max": 39.98460388183594,
      "activations/layer21_attention_weight_min": -24.648143768310547,
      "activations/layer22_attention_weight_max": 35.022438049316406,
      "activations/layer22_attention_weight_min": -24.10063934326172,
      "activations/layer23_attention_weight_max": 39.09749221801758,
      "activations/layer23_attention_weight_min": -23.262474060058594,
      "activations/layer2_attention_weight_max": 32.93794631958008,
      "activations/layer2_attention_weight_min": -33.22991943359375,
      "activations/layer3_attention_weight_max": 93.7435073852539,
      "activations/layer3_attention_weight_min": -99.63707733154297,
      "activations/layer4_attention_weight_max": 99.60289001464844,
      "activations/layer4_attention_weight_min": -100.32770538330078,
      "activations/layer5_attention_weight_max": 74.40553283691406,
      "activations/layer5_attention_weight_min": -81.38571166992188,
      "activations/layer6_attention_weight_max": 55.182472229003906,
      "activations/layer6_attention_weight_min": -54.24179458618164,
      "activations/layer7_attention_weight_max": 79.72437286376953,
      "activations/layer7_attention_weight_min": -72.43640899658203,
      "activations/layer8_attention_weight_max": 47.70504379272461,
      "activations/layer8_attention_weight_min": -50.562469482421875,
      "activations/layer9_attention_weight_max": 51.9840202331543,
      "activations/layer9_attention_weight_min": -49.9443473815918,
      "epoch": 10.94,
      "learning_rate": 8.025075757575758e-05,
      "loss": 2.7826,
      "step": 188300
    },
    {
      "activations/layer0_attention_weight_max": 15.472851753234863,
      "activations/layer0_attention_weight_min": -13.941865921020508,
      "activations/layer10_attention_weight_max": 34.36707305908203,
      "activations/layer10_attention_weight_min": -33.33605194091797,
      "activations/layer11_attention_weight_max": 30.541217803955078,
      "activations/layer11_attention_weight_min": -30.725881576538086,
      "activations/layer12_attention_weight_max": 19.22041893005371,
      "activations/layer12_attention_weight_min": -24.021024703979492,
      "activations/layer13_attention_weight_max": 38.02323532104492,
      "activations/layer13_attention_weight_min": -26.893789291381836,
      "activations/layer14_attention_weight_max": 36.76570129394531,
      "activations/layer14_attention_weight_min": -31.263505935668945,
      "activations/layer15_attention_weight_max": 33.63477325439453,
      "activations/layer15_attention_weight_min": -29.24152374267578,
      "activations/layer16_attention_weight_max": 31.839954376220703,
      "activations/layer16_attention_weight_min": -31.03546905517578,
      "activations/layer17_attention_weight_max": 55.36947250366211,
      "activations/layer17_attention_weight_min": -44.76079177856445,
      "activations/layer18_attention_weight_max": 47.84892654418945,
      "activations/layer18_attention_weight_min": -39.64023208618164,
      "activations/layer19_attention_weight_max": 24.08907127380371,
      "activations/layer19_attention_weight_min": -20.764272689819336,
      "activations/layer1_attention_weight_max": 18.224029541015625,
      "activations/layer1_attention_weight_min": -17.591535568237305,
      "activations/layer20_attention_weight_max": 22.57147979736328,
      "activations/layer20_attention_weight_min": -22.966032028198242,
      "activations/layer21_attention_weight_max": 38.753910064697266,
      "activations/layer21_attention_weight_min": -23.74884605407715,
      "activations/layer22_attention_weight_max": 33.09275436401367,
      "activations/layer22_attention_weight_min": -29.147390365600586,
      "activations/layer23_attention_weight_max": 37.85863494873047,
      "activations/layer23_attention_weight_min": -24.438091278076172,
      "activations/layer2_attention_weight_max": 32.914939880371094,
      "activations/layer2_attention_weight_min": -29.785606384277344,
      "activations/layer3_attention_weight_max": 92.67127227783203,
      "activations/layer3_attention_weight_min": -92.14555358886719,
      "activations/layer4_attention_weight_max": 100.62557220458984,
      "activations/layer4_attention_weight_min": -97.39729309082031,
      "activations/layer5_attention_weight_max": 72.81527709960938,
      "activations/layer5_attention_weight_min": -75.69610595703125,
      "activations/layer6_attention_weight_max": 53.196781158447266,
      "activations/layer6_attention_weight_min": -53.9339714050293,
      "activations/layer7_attention_weight_max": 72.12418365478516,
      "activations/layer7_attention_weight_min": -71.72333526611328,
      "activations/layer8_attention_weight_max": 45.190242767333984,
      "activations/layer8_attention_weight_min": -44.97700119018555,
      "activations/layer9_attention_weight_max": 45.83321762084961,
      "activations/layer9_attention_weight_min": -46.28334426879883,
      "epoch": 10.94,
      "learning_rate": 8.023181818181817e-05,
      "loss": 2.7798,
      "step": 188350
    },
    {
      "activations/layer0_attention_weight_max": 15.8405179977417,
      "activations/layer0_attention_weight_min": -14.430852890014648,
      "activations/layer10_attention_weight_max": 38.40275955200195,
      "activations/layer10_attention_weight_min": -36.60478210449219,
      "activations/layer11_attention_weight_max": 34.244503021240234,
      "activations/layer11_attention_weight_min": -33.54543685913086,
      "activations/layer12_attention_weight_max": 20.870594024658203,
      "activations/layer12_attention_weight_min": -25.093624114990234,
      "activations/layer13_attention_weight_max": 46.23323440551758,
      "activations/layer13_attention_weight_min": -29.487844467163086,
      "activations/layer14_attention_weight_max": 41.51557540893555,
      "activations/layer14_attention_weight_min": -31.286869049072266,
      "activations/layer15_attention_weight_max": 37.560699462890625,
      "activations/layer15_attention_weight_min": -29.46929359436035,
      "activations/layer16_attention_weight_max": 30.775928497314453,
      "activations/layer16_attention_weight_min": -29.473814010620117,
      "activations/layer17_attention_weight_max": 55.400394439697266,
      "activations/layer17_attention_weight_min": -45.572914123535156,
      "activations/layer18_attention_weight_max": 48.555381774902344,
      "activations/layer18_attention_weight_min": -37.03856658935547,
      "activations/layer19_attention_weight_max": 23.272045135498047,
      "activations/layer19_attention_weight_min": -21.10317039489746,
      "activations/layer1_attention_weight_max": 16.792219161987305,
      "activations/layer1_attention_weight_min": -16.15262794494629,
      "activations/layer20_attention_weight_max": 22.219379425048828,
      "activations/layer20_attention_weight_min": -22.92454719543457,
      "activations/layer21_attention_weight_max": 39.58855438232422,
      "activations/layer21_attention_weight_min": -23.984636306762695,
      "activations/layer22_attention_weight_max": 35.01041030883789,
      "activations/layer22_attention_weight_min": -27.33194351196289,
      "activations/layer23_attention_weight_max": 40.199462890625,
      "activations/layer23_attention_weight_min": -25.82769775390625,
      "activations/layer2_attention_weight_max": 33.87267303466797,
      "activations/layer2_attention_weight_min": -32.41097640991211,
      "activations/layer3_attention_weight_max": 93.42547607421875,
      "activations/layer3_attention_weight_min": -91.5538558959961,
      "activations/layer4_attention_weight_max": 99.12810516357422,
      "activations/layer4_attention_weight_min": -91.70507049560547,
      "activations/layer5_attention_weight_max": 74.76778411865234,
      "activations/layer5_attention_weight_min": -74.97059631347656,
      "activations/layer6_attention_weight_max": 54.18931579589844,
      "activations/layer6_attention_weight_min": -54.06743621826172,
      "activations/layer7_attention_weight_max": 79.38723754882812,
      "activations/layer7_attention_weight_min": -73.85954284667969,
      "activations/layer8_attention_weight_max": 47.463645935058594,
      "activations/layer8_attention_weight_min": -48.86069107055664,
      "activations/layer9_attention_weight_max": 59.7296257019043,
      "activations/layer9_attention_weight_min": -49.841854095458984,
      "epoch": 10.95,
      "learning_rate": 8.021287878787878e-05,
      "loss": 2.7801,
      "step": 188400
    },
    {
      "activations/layer0_attention_weight_max": 15.933701515197754,
      "activations/layer0_attention_weight_min": -14.509681701660156,
      "activations/layer10_attention_weight_max": 39.929054260253906,
      "activations/layer10_attention_weight_min": -39.0132942199707,
      "activations/layer11_attention_weight_max": 34.92675018310547,
      "activations/layer11_attention_weight_min": -35.401611328125,
      "activations/layer12_attention_weight_max": 19.762653350830078,
      "activations/layer12_attention_weight_min": -25.73857307434082,
      "activations/layer13_attention_weight_max": 45.67887878417969,
      "activations/layer13_attention_weight_min": -30.140926361083984,
      "activations/layer14_attention_weight_max": 43.69244384765625,
      "activations/layer14_attention_weight_min": -31.668230056762695,
      "activations/layer15_attention_weight_max": 36.2109489440918,
      "activations/layer15_attention_weight_min": -30.240758895874023,
      "activations/layer16_attention_weight_max": 34.214969635009766,
      "activations/layer16_attention_weight_min": -30.550634384155273,
      "activations/layer17_attention_weight_max": 56.33797073364258,
      "activations/layer17_attention_weight_min": -45.821693420410156,
      "activations/layer18_attention_weight_max": 54.368106842041016,
      "activations/layer18_attention_weight_min": -42.0410270690918,
      "activations/layer19_attention_weight_max": 24.763639450073242,
      "activations/layer19_attention_weight_min": -23.59490394592285,
      "activations/layer1_attention_weight_max": 16.6897029876709,
      "activations/layer1_attention_weight_min": -15.59976577758789,
      "activations/layer20_attention_weight_max": 25.29298973083496,
      "activations/layer20_attention_weight_min": -24.99233055114746,
      "activations/layer21_attention_weight_max": 45.74750518798828,
      "activations/layer21_attention_weight_min": -23.329362869262695,
      "activations/layer22_attention_weight_max": 35.21939468383789,
      "activations/layer22_attention_weight_min": -26.618738174438477,
      "activations/layer23_attention_weight_max": 42.43657684326172,
      "activations/layer23_attention_weight_min": -23.523651123046875,
      "activations/layer2_attention_weight_max": 34.40742492675781,
      "activations/layer2_attention_weight_min": -31.82522964477539,
      "activations/layer3_attention_weight_max": 95.9753189086914,
      "activations/layer3_attention_weight_min": -96.35404205322266,
      "activations/layer4_attention_weight_max": 104.47499084472656,
      "activations/layer4_attention_weight_min": -102.0261001586914,
      "activations/layer5_attention_weight_max": 74.34918212890625,
      "activations/layer5_attention_weight_min": -76.23457336425781,
      "activations/layer6_attention_weight_max": 55.01152420043945,
      "activations/layer6_attention_weight_min": -56.29425811767578,
      "activations/layer7_attention_weight_max": 78.62398529052734,
      "activations/layer7_attention_weight_min": -79.1751480102539,
      "activations/layer8_attention_weight_max": 47.25861740112305,
      "activations/layer8_attention_weight_min": -49.257503509521484,
      "activations/layer9_attention_weight_max": 50.116153717041016,
      "activations/layer9_attention_weight_min": -50.868377685546875,
      "epoch": 10.95,
      "learning_rate": 8.01939393939394e-05,
      "loss": 2.7651,
      "step": 188450
    },
    {
      "activations/layer0_attention_weight_max": 16.689367294311523,
      "activations/layer0_attention_weight_min": -14.210308074951172,
      "activations/layer10_attention_weight_max": 42.3384895324707,
      "activations/layer10_attention_weight_min": -40.36978530883789,
      "activations/layer11_attention_weight_max": 34.03314971923828,
      "activations/layer11_attention_weight_min": -35.128604888916016,
      "activations/layer12_attention_weight_max": 20.083789825439453,
      "activations/layer12_attention_weight_min": -27.96959686279297,
      "activations/layer13_attention_weight_max": 49.31000518798828,
      "activations/layer13_attention_weight_min": -34.97587966918945,
      "activations/layer14_attention_weight_max": 39.899715423583984,
      "activations/layer14_attention_weight_min": -29.8018741607666,
      "activations/layer15_attention_weight_max": 41.1606559753418,
      "activations/layer15_attention_weight_min": -29.70423126220703,
      "activations/layer16_attention_weight_max": 33.83577346801758,
      "activations/layer16_attention_weight_min": -29.768451690673828,
      "activations/layer17_attention_weight_max": 54.828861236572266,
      "activations/layer17_attention_weight_min": -44.847251892089844,
      "activations/layer18_attention_weight_max": 49.86585998535156,
      "activations/layer18_attention_weight_min": -38.57418441772461,
      "activations/layer19_attention_weight_max": 21.937667846679688,
      "activations/layer19_attention_weight_min": -22.556076049804688,
      "activations/layer1_attention_weight_max": 16.166505813598633,
      "activations/layer1_attention_weight_min": -15.410208702087402,
      "activations/layer20_attention_weight_max": 25.079015731811523,
      "activations/layer20_attention_weight_min": -23.3681697845459,
      "activations/layer21_attention_weight_max": 45.62836837768555,
      "activations/layer21_attention_weight_min": -25.829448699951172,
      "activations/layer22_attention_weight_max": 31.569286346435547,
      "activations/layer22_attention_weight_min": -26.724119186401367,
      "activations/layer23_attention_weight_max": 41.85142517089844,
      "activations/layer23_attention_weight_min": -23.72542381286621,
      "activations/layer2_attention_weight_max": 35.4510498046875,
      "activations/layer2_attention_weight_min": -33.39421081542969,
      "activations/layer3_attention_weight_max": 101.51841735839844,
      "activations/layer3_attention_weight_min": -100.7195053100586,
      "activations/layer4_attention_weight_max": 107.0057373046875,
      "activations/layer4_attention_weight_min": -99.90034484863281,
      "activations/layer5_attention_weight_max": 76.69883728027344,
      "activations/layer5_attention_weight_min": -77.48003387451172,
      "activations/layer6_attention_weight_max": 54.135765075683594,
      "activations/layer6_attention_weight_min": -54.69485855102539,
      "activations/layer7_attention_weight_max": 79.51216125488281,
      "activations/layer7_attention_weight_min": -74.92567443847656,
      "activations/layer8_attention_weight_max": 51.8425407409668,
      "activations/layer8_attention_weight_min": -51.66360855102539,
      "activations/layer9_attention_weight_max": 52.58657455444336,
      "activations/layer9_attention_weight_min": -50.69355392456055,
      "epoch": 10.95,
      "learning_rate": 8.017499999999998e-05,
      "loss": 2.769,
      "step": 188500
    },
    {
      "activations/layer0_attention_weight_max": 15.907248497009277,
      "activations/layer0_attention_weight_min": -14.229059219360352,
      "activations/layer10_attention_weight_max": 42.65181350708008,
      "activations/layer10_attention_weight_min": -38.61393356323242,
      "activations/layer11_attention_weight_max": 35.627220153808594,
      "activations/layer11_attention_weight_min": -35.460548400878906,
      "activations/layer12_attention_weight_max": 19.636219024658203,
      "activations/layer12_attention_weight_min": -26.31637954711914,
      "activations/layer13_attention_weight_max": 48.61445999145508,
      "activations/layer13_attention_weight_min": -34.86277770996094,
      "activations/layer14_attention_weight_max": 40.1611442565918,
      "activations/layer14_attention_weight_min": -30.500904083251953,
      "activations/layer15_attention_weight_max": 37.52042007446289,
      "activations/layer15_attention_weight_min": -29.302440643310547,
      "activations/layer16_attention_weight_max": 35.876956939697266,
      "activations/layer16_attention_weight_min": -29.329687118530273,
      "activations/layer17_attention_weight_max": 56.27622985839844,
      "activations/layer17_attention_weight_min": -45.6251106262207,
      "activations/layer18_attention_weight_max": 53.746055603027344,
      "activations/layer18_attention_weight_min": -41.20997619628906,
      "activations/layer19_attention_weight_max": 25.414274215698242,
      "activations/layer19_attention_weight_min": -23.58216667175293,
      "activations/layer1_attention_weight_max": 17.05351448059082,
      "activations/layer1_attention_weight_min": -15.35241413116455,
      "activations/layer20_attention_weight_max": 26.172277450561523,
      "activations/layer20_attention_weight_min": -20.95166778564453,
      "activations/layer21_attention_weight_max": 43.40192794799805,
      "activations/layer21_attention_weight_min": -23.39871597290039,
      "activations/layer22_attention_weight_max": 38.65189743041992,
      "activations/layer22_attention_weight_min": -26.367294311523438,
      "activations/layer23_attention_weight_max": 42.08006286621094,
      "activations/layer23_attention_weight_min": -24.980926513671875,
      "activations/layer2_attention_weight_max": 33.484806060791016,
      "activations/layer2_attention_weight_min": -31.038055419921875,
      "activations/layer3_attention_weight_max": 94.94109344482422,
      "activations/layer3_attention_weight_min": -94.82793426513672,
      "activations/layer4_attention_weight_max": 102.44120025634766,
      "activations/layer4_attention_weight_min": -96.78314971923828,
      "activations/layer5_attention_weight_max": 74.1595458984375,
      "activations/layer5_attention_weight_min": -76.1021728515625,
      "activations/layer6_attention_weight_max": 54.13228988647461,
      "activations/layer6_attention_weight_min": -54.313419342041016,
      "activations/layer7_attention_weight_max": 78.14564514160156,
      "activations/layer7_attention_weight_min": -72.3125,
      "activations/layer8_attention_weight_max": 48.80904006958008,
      "activations/layer8_attention_weight_min": -48.98430252075195,
      "activations/layer9_attention_weight_max": 57.33525466918945,
      "activations/layer9_attention_weight_min": -53.862972259521484,
      "epoch": 10.96,
      "learning_rate": 8.01560606060606e-05,
      "loss": 2.7777,
      "step": 188550
    },
    {
      "activations/layer0_attention_weight_max": 15.942438125610352,
      "activations/layer0_attention_weight_min": -14.173827171325684,
      "activations/layer10_attention_weight_max": 36.544166564941406,
      "activations/layer10_attention_weight_min": -34.01679611206055,
      "activations/layer11_attention_weight_max": 32.96615219116211,
      "activations/layer11_attention_weight_min": -31.7801570892334,
      "activations/layer12_attention_weight_max": 20.790292739868164,
      "activations/layer12_attention_weight_min": -27.47273063659668,
      "activations/layer13_attention_weight_max": 43.56711959838867,
      "activations/layer13_attention_weight_min": -31.461166381835938,
      "activations/layer14_attention_weight_max": 39.887794494628906,
      "activations/layer14_attention_weight_min": -29.449655532836914,
      "activations/layer15_attention_weight_max": 37.933170318603516,
      "activations/layer15_attention_weight_min": -29.59036636352539,
      "activations/layer16_attention_weight_max": 32.83073425292969,
      "activations/layer16_attention_weight_min": -28.15979766845703,
      "activations/layer17_attention_weight_max": 50.52802658081055,
      "activations/layer17_attention_weight_min": -43.989097595214844,
      "activations/layer18_attention_weight_max": 46.20328140258789,
      "activations/layer18_attention_weight_min": -36.82913589477539,
      "activations/layer19_attention_weight_max": 21.314191818237305,
      "activations/layer19_attention_weight_min": -22.389211654663086,
      "activations/layer1_attention_weight_max": 16.573135375976562,
      "activations/layer1_attention_weight_min": -13.59898567199707,
      "activations/layer20_attention_weight_max": 22.421072006225586,
      "activations/layer20_attention_weight_min": -23.271574020385742,
      "activations/layer21_attention_weight_max": 39.211181640625,
      "activations/layer21_attention_weight_min": -23.93949317932129,
      "activations/layer22_attention_weight_max": 29.599504470825195,
      "activations/layer22_attention_weight_min": -24.553955078125,
      "activations/layer23_attention_weight_max": 38.203025817871094,
      "activations/layer23_attention_weight_min": -23.239456176757812,
      "activations/layer2_attention_weight_max": 32.29276657104492,
      "activations/layer2_attention_weight_min": -31.577991485595703,
      "activations/layer3_attention_weight_max": 93.87166595458984,
      "activations/layer3_attention_weight_min": -99.5926742553711,
      "activations/layer4_attention_weight_max": 97.19225311279297,
      "activations/layer4_attention_weight_min": -98.87397003173828,
      "activations/layer5_attention_weight_max": 70.6279296875,
      "activations/layer5_attention_weight_min": -75.60752868652344,
      "activations/layer6_attention_weight_max": 51.16810607910156,
      "activations/layer6_attention_weight_min": -50.58795928955078,
      "activations/layer7_attention_weight_max": 74.48686981201172,
      "activations/layer7_attention_weight_min": -71.52318572998047,
      "activations/layer8_attention_weight_max": 46.78434753417969,
      "activations/layer8_attention_weight_min": -50.16323471069336,
      "activations/layer9_attention_weight_max": 48.9084358215332,
      "activations/layer9_attention_weight_min": -46.96989059448242,
      "epoch": 10.96,
      "learning_rate": 8.01371212121212e-05,
      "loss": 2.788,
      "step": 188600
    },
    {
      "activations/layer0_attention_weight_max": 16.56800079345703,
      "activations/layer0_attention_weight_min": -14.236291885375977,
      "activations/layer10_attention_weight_max": 38.320274353027344,
      "activations/layer10_attention_weight_min": -34.78529739379883,
      "activations/layer11_attention_weight_max": 33.557472229003906,
      "activations/layer11_attention_weight_min": -31.255657196044922,
      "activations/layer12_attention_weight_max": 18.757862091064453,
      "activations/layer12_attention_weight_min": -29.114994049072266,
      "activations/layer13_attention_weight_max": 42.868858337402344,
      "activations/layer13_attention_weight_min": -31.151493072509766,
      "activations/layer14_attention_weight_max": 42.20445251464844,
      "activations/layer14_attention_weight_min": -30.2502498626709,
      "activations/layer15_attention_weight_max": 37.31156539916992,
      "activations/layer15_attention_weight_min": -29.931005477905273,
      "activations/layer16_attention_weight_max": 32.83846664428711,
      "activations/layer16_attention_weight_min": -32.12015914916992,
      "activations/layer17_attention_weight_max": 53.79071807861328,
      "activations/layer17_attention_weight_min": -47.561222076416016,
      "activations/layer18_attention_weight_max": 43.747493743896484,
      "activations/layer18_attention_weight_min": -38.106956481933594,
      "activations/layer19_attention_weight_max": 22.650732040405273,
      "activations/layer19_attention_weight_min": -21.243144989013672,
      "activations/layer1_attention_weight_max": 16.186010360717773,
      "activations/layer1_attention_weight_min": -14.377213478088379,
      "activations/layer20_attention_weight_max": 22.617284774780273,
      "activations/layer20_attention_weight_min": -19.77480125427246,
      "activations/layer21_attention_weight_max": 48.8172607421875,
      "activations/layer21_attention_weight_min": -21.776308059692383,
      "activations/layer22_attention_weight_max": 29.8194580078125,
      "activations/layer22_attention_weight_min": -22.985376358032227,
      "activations/layer23_attention_weight_max": 38.45188903808594,
      "activations/layer23_attention_weight_min": -23.106929779052734,
      "activations/layer2_attention_weight_max": 34.87958526611328,
      "activations/layer2_attention_weight_min": -30.593849182128906,
      "activations/layer3_attention_weight_max": 95.48322296142578,
      "activations/layer3_attention_weight_min": -93.05874633789062,
      "activations/layer4_attention_weight_max": 97.0181884765625,
      "activations/layer4_attention_weight_min": -96.2533187866211,
      "activations/layer5_attention_weight_max": 71.77391052246094,
      "activations/layer5_attention_weight_min": -76.78705596923828,
      "activations/layer6_attention_weight_max": 51.354801177978516,
      "activations/layer6_attention_weight_min": -53.43760681152344,
      "activations/layer7_attention_weight_max": 77.82273864746094,
      "activations/layer7_attention_weight_min": -69.67190551757812,
      "activations/layer8_attention_weight_max": 44.87087631225586,
      "activations/layer8_attention_weight_min": -46.73295593261719,
      "activations/layer9_attention_weight_max": 50.78678512573242,
      "activations/layer9_attention_weight_min": -47.50117874145508,
      "epoch": 10.96,
      "learning_rate": 8.011818181818182e-05,
      "loss": 2.7941,
      "step": 188650
    },
    {
      "activations/layer0_attention_weight_max": 15.443984031677246,
      "activations/layer0_attention_weight_min": -14.17823314666748,
      "activations/layer10_attention_weight_max": 34.990814208984375,
      "activations/layer10_attention_weight_min": -34.339622497558594,
      "activations/layer11_attention_weight_max": 29.98940086364746,
      "activations/layer11_attention_weight_min": -31.92555046081543,
      "activations/layer12_attention_weight_max": 18.138744354248047,
      "activations/layer12_attention_weight_min": -25.289093017578125,
      "activations/layer13_attention_weight_max": 50.096988677978516,
      "activations/layer13_attention_weight_min": -31.64202308654785,
      "activations/layer14_attention_weight_max": 47.16913986206055,
      "activations/layer14_attention_weight_min": -31.373512268066406,
      "activations/layer15_attention_weight_max": 42.29720687866211,
      "activations/layer15_attention_weight_min": -31.284156799316406,
      "activations/layer16_attention_weight_max": 30.960119247436523,
      "activations/layer16_attention_weight_min": -29.897050857543945,
      "activations/layer17_attention_weight_max": 53.12537384033203,
      "activations/layer17_attention_weight_min": -43.469905853271484,
      "activations/layer18_attention_weight_max": 47.00739669799805,
      "activations/layer18_attention_weight_min": -37.261810302734375,
      "activations/layer19_attention_weight_max": 24.283967971801758,
      "activations/layer19_attention_weight_min": -24.040178298950195,
      "activations/layer1_attention_weight_max": 17.571269989013672,
      "activations/layer1_attention_weight_min": -15.125654220581055,
      "activations/layer20_attention_weight_max": 22.926410675048828,
      "activations/layer20_attention_weight_min": -22.317411422729492,
      "activations/layer21_attention_weight_max": 35.90089797973633,
      "activations/layer21_attention_weight_min": -22.259422302246094,
      "activations/layer22_attention_weight_max": 29.05564308166504,
      "activations/layer22_attention_weight_min": -25.6505184173584,
      "activations/layer23_attention_weight_max": 35.08815002441406,
      "activations/layer23_attention_weight_min": -24.69530487060547,
      "activations/layer2_attention_weight_max": 32.55915832519531,
      "activations/layer2_attention_weight_min": -30.867130279541016,
      "activations/layer3_attention_weight_max": 92.3560562133789,
      "activations/layer3_attention_weight_min": -99.94115447998047,
      "activations/layer4_attention_weight_max": 102.96236419677734,
      "activations/layer4_attention_weight_min": -98.86316680908203,
      "activations/layer5_attention_weight_max": 74.93463134765625,
      "activations/layer5_attention_weight_min": -77.52462005615234,
      "activations/layer6_attention_weight_max": 53.44344711303711,
      "activations/layer6_attention_weight_min": -56.94944381713867,
      "activations/layer7_attention_weight_max": 78.30245208740234,
      "activations/layer7_attention_weight_min": -71.9414291381836,
      "activations/layer8_attention_weight_max": 46.58485412597656,
      "activations/layer8_attention_weight_min": -48.5892448425293,
      "activations/layer9_attention_weight_max": 47.028202056884766,
      "activations/layer9_attention_weight_min": -49.64901351928711,
      "epoch": 10.96,
      "learning_rate": 8.009924242424242e-05,
      "loss": 2.7778,
      "step": 188700
    },
    {
      "activations/layer0_attention_weight_max": 15.90740966796875,
      "activations/layer0_attention_weight_min": -14.146480560302734,
      "activations/layer10_attention_weight_max": 38.0824089050293,
      "activations/layer10_attention_weight_min": -35.94837188720703,
      "activations/layer11_attention_weight_max": 32.79202651977539,
      "activations/layer11_attention_weight_min": -33.97173309326172,
      "activations/layer12_attention_weight_max": 18.10141372680664,
      "activations/layer12_attention_weight_min": -22.69834327697754,
      "activations/layer13_attention_weight_max": 40.4606819152832,
      "activations/layer13_attention_weight_min": -31.198314666748047,
      "activations/layer14_attention_weight_max": 36.3016357421875,
      "activations/layer14_attention_weight_min": -30.865190505981445,
      "activations/layer15_attention_weight_max": 33.80375289916992,
      "activations/layer15_attention_weight_min": -29.625621795654297,
      "activations/layer16_attention_weight_max": 30.77522850036621,
      "activations/layer16_attention_weight_min": -29.4356689453125,
      "activations/layer17_attention_weight_max": 51.93427658081055,
      "activations/layer17_attention_weight_min": -43.6537971496582,
      "activations/layer18_attention_weight_max": 43.64884567260742,
      "activations/layer18_attention_weight_min": -35.850894927978516,
      "activations/layer19_attention_weight_max": 22.257368087768555,
      "activations/layer19_attention_weight_min": -20.134693145751953,
      "activations/layer1_attention_weight_max": 16.96112060546875,
      "activations/layer1_attention_weight_min": -15.570525169372559,
      "activations/layer20_attention_weight_max": 21.769481658935547,
      "activations/layer20_attention_weight_min": -24.366130828857422,
      "activations/layer21_attention_weight_max": 38.940181732177734,
      "activations/layer21_attention_weight_min": -21.008590698242188,
      "activations/layer22_attention_weight_max": 33.82559585571289,
      "activations/layer22_attention_weight_min": -24.73210334777832,
      "activations/layer23_attention_weight_max": 34.975833892822266,
      "activations/layer23_attention_weight_min": -21.231491088867188,
      "activations/layer2_attention_weight_max": 36.279754638671875,
      "activations/layer2_attention_weight_min": -33.28919219970703,
      "activations/layer3_attention_weight_max": 97.90729522705078,
      "activations/layer3_attention_weight_min": -103.14838409423828,
      "activations/layer4_attention_weight_max": 101.64805603027344,
      "activations/layer4_attention_weight_min": -100.03180694580078,
      "activations/layer5_attention_weight_max": 69.91758728027344,
      "activations/layer5_attention_weight_min": -76.55457305908203,
      "activations/layer6_attention_weight_max": 53.34257507324219,
      "activations/layer6_attention_weight_min": -53.205570220947266,
      "activations/layer7_attention_weight_max": 74.33921813964844,
      "activations/layer7_attention_weight_min": -71.05644226074219,
      "activations/layer8_attention_weight_max": 46.373809814453125,
      "activations/layer8_attention_weight_min": -49.0322151184082,
      "activations/layer9_attention_weight_max": 47.80167770385742,
      "activations/layer9_attention_weight_min": -48.87173843383789,
      "epoch": 10.97,
      "learning_rate": 8.008030303030302e-05,
      "loss": 2.7702,
      "step": 188750
    },
    {
      "activations/layer0_attention_weight_max": 16.183269500732422,
      "activations/layer0_attention_weight_min": -14.41430377960205,
      "activations/layer10_attention_weight_max": 38.15711212158203,
      "activations/layer10_attention_weight_min": -36.05997085571289,
      "activations/layer11_attention_weight_max": 32.56407928466797,
      "activations/layer11_attention_weight_min": -33.00456619262695,
      "activations/layer12_attention_weight_max": 17.908946990966797,
      "activations/layer12_attention_weight_min": -24.306055068969727,
      "activations/layer13_attention_weight_max": 37.864681243896484,
      "activations/layer13_attention_weight_min": -29.20534896850586,
      "activations/layer14_attention_weight_max": 35.45771408081055,
      "activations/layer14_attention_weight_min": -31.796085357666016,
      "activations/layer15_attention_weight_max": 30.93733024597168,
      "activations/layer15_attention_weight_min": -28.94807243347168,
      "activations/layer16_attention_weight_max": 29.016618728637695,
      "activations/layer16_attention_weight_min": -29.24209213256836,
      "activations/layer17_attention_weight_max": 48.75873947143555,
      "activations/layer17_attention_weight_min": -45.98286819458008,
      "activations/layer18_attention_weight_max": 44.16938018798828,
      "activations/layer18_attention_weight_min": -37.314048767089844,
      "activations/layer19_attention_weight_max": 19.453290939331055,
      "activations/layer19_attention_weight_min": -20.657583236694336,
      "activations/layer1_attention_weight_max": 17.72496223449707,
      "activations/layer1_attention_weight_min": -14.921637535095215,
      "activations/layer20_attention_weight_max": 21.388195037841797,
      "activations/layer20_attention_weight_min": -23.45123291015625,
      "activations/layer21_attention_weight_max": 34.141319274902344,
      "activations/layer21_attention_weight_min": -23.57816505432129,
      "activations/layer22_attention_weight_max": 28.463254928588867,
      "activations/layer22_attention_weight_min": -22.92916488647461,
      "activations/layer23_attention_weight_max": 34.1588134765625,
      "activations/layer23_attention_weight_min": -22.434833526611328,
      "activations/layer2_attention_weight_max": 34.65107727050781,
      "activations/layer2_attention_weight_min": -31.83965301513672,
      "activations/layer3_attention_weight_max": 91.23302459716797,
      "activations/layer3_attention_weight_min": -96.24105834960938,
      "activations/layer4_attention_weight_max": 99.34056854248047,
      "activations/layer4_attention_weight_min": -97.35038757324219,
      "activations/layer5_attention_weight_max": 76.11748504638672,
      "activations/layer5_attention_weight_min": -77.91404724121094,
      "activations/layer6_attention_weight_max": 56.255287170410156,
      "activations/layer6_attention_weight_min": -54.50985336303711,
      "activations/layer7_attention_weight_max": 76.41645050048828,
      "activations/layer7_attention_weight_min": -73.05209350585938,
      "activations/layer8_attention_weight_max": 45.9621467590332,
      "activations/layer8_attention_weight_min": -47.60737609863281,
      "activations/layer9_attention_weight_max": 48.274085998535156,
      "activations/layer9_attention_weight_min": -47.98598861694336,
      "epoch": 10.97,
      "learning_rate": 8.006174242424242e-05,
      "loss": 2.7767,
      "step": 188800
    },
    {
      "activations/layer0_attention_weight_max": 15.8988618850708,
      "activations/layer0_attention_weight_min": -14.121070861816406,
      "activations/layer10_attention_weight_max": 37.836936950683594,
      "activations/layer10_attention_weight_min": -35.71970748901367,
      "activations/layer11_attention_weight_max": 32.50528335571289,
      "activations/layer11_attention_weight_min": -32.30543518066406,
      "activations/layer12_attention_weight_max": 18.137046813964844,
      "activations/layer12_attention_weight_min": -27.152076721191406,
      "activations/layer13_attention_weight_max": 36.69375228881836,
      "activations/layer13_attention_weight_min": -29.65880012512207,
      "activations/layer14_attention_weight_max": 36.944576263427734,
      "activations/layer14_attention_weight_min": -30.146427154541016,
      "activations/layer15_attention_weight_max": 32.39815139770508,
      "activations/layer15_attention_weight_min": -29.734460830688477,
      "activations/layer16_attention_weight_max": 30.410213470458984,
      "activations/layer16_attention_weight_min": -29.548254013061523,
      "activations/layer17_attention_weight_max": 52.300445556640625,
      "activations/layer17_attention_weight_min": -44.44261932373047,
      "activations/layer18_attention_weight_max": 42.88013458251953,
      "activations/layer18_attention_weight_min": -36.96230697631836,
      "activations/layer19_attention_weight_max": 19.780960083007812,
      "activations/layer19_attention_weight_min": -20.752939224243164,
      "activations/layer1_attention_weight_max": 17.152530670166016,
      "activations/layer1_attention_weight_min": -14.165569305419922,
      "activations/layer20_attention_weight_max": 21.82151985168457,
      "activations/layer20_attention_weight_min": -21.105436325073242,
      "activations/layer21_attention_weight_max": 34.49323272705078,
      "activations/layer21_attention_weight_min": -22.660964965820312,
      "activations/layer22_attention_weight_max": 28.025484085083008,
      "activations/layer22_attention_weight_min": -23.847009658813477,
      "activations/layer23_attention_weight_max": 31.957984924316406,
      "activations/layer23_attention_weight_min": -23.393512725830078,
      "activations/layer2_attention_weight_max": 32.97773361206055,
      "activations/layer2_attention_weight_min": -32.019203186035156,
      "activations/layer3_attention_weight_max": 92.51396942138672,
      "activations/layer3_attention_weight_min": -93.87679290771484,
      "activations/layer4_attention_weight_max": 101.07988739013672,
      "activations/layer4_attention_weight_min": -95.39605712890625,
      "activations/layer5_attention_weight_max": 71.00534057617188,
      "activations/layer5_attention_weight_min": -73.13406372070312,
      "activations/layer6_attention_weight_max": 52.07704544067383,
      "activations/layer6_attention_weight_min": -52.94259262084961,
      "activations/layer7_attention_weight_max": 77.57383728027344,
      "activations/layer7_attention_weight_min": -72.18952178955078,
      "activations/layer8_attention_weight_max": 43.81817626953125,
      "activations/layer8_attention_weight_min": -47.13459396362305,
      "activations/layer9_attention_weight_max": 47.56587219238281,
      "activations/layer9_attention_weight_min": -47.903846740722656,
      "epoch": 10.97,
      "learning_rate": 8.004280303030302e-05,
      "loss": 2.791,
      "step": 188850
    },
    {
      "activations/layer0_attention_weight_max": 15.42921257019043,
      "activations/layer0_attention_weight_min": -14.129603385925293,
      "activations/layer10_attention_weight_max": 37.70930862426758,
      "activations/layer10_attention_weight_min": -36.68027114868164,
      "activations/layer11_attention_weight_max": 34.408355712890625,
      "activations/layer11_attention_weight_min": -32.53687286376953,
      "activations/layer12_attention_weight_max": 18.32196044921875,
      "activations/layer12_attention_weight_min": -27.431486129760742,
      "activations/layer13_attention_weight_max": 49.67991256713867,
      "activations/layer13_attention_weight_min": -35.37577438354492,
      "activations/layer14_attention_weight_max": 41.655364990234375,
      "activations/layer14_attention_weight_min": -30.39948081970215,
      "activations/layer15_attention_weight_max": 39.80800247192383,
      "activations/layer15_attention_weight_min": -30.62470817565918,
      "activations/layer16_attention_weight_max": 31.88871192932129,
      "activations/layer16_attention_weight_min": -30.002559661865234,
      "activations/layer17_attention_weight_max": 50.86692810058594,
      "activations/layer17_attention_weight_min": -46.56108856201172,
      "activations/layer18_attention_weight_max": 44.73054122924805,
      "activations/layer18_attention_weight_min": -39.377471923828125,
      "activations/layer19_attention_weight_max": 23.55199432373047,
      "activations/layer19_attention_weight_min": -24.496225357055664,
      "activations/layer1_attention_weight_max": 17.363941192626953,
      "activations/layer1_attention_weight_min": -17.308992385864258,
      "activations/layer20_attention_weight_max": 22.918346405029297,
      "activations/layer20_attention_weight_min": -22.118267059326172,
      "activations/layer21_attention_weight_max": 43.06987380981445,
      "activations/layer21_attention_weight_min": -24.90323829650879,
      "activations/layer22_attention_weight_max": 27.47452163696289,
      "activations/layer22_attention_weight_min": -26.90648078918457,
      "activations/layer23_attention_weight_max": 41.127384185791016,
      "activations/layer23_attention_weight_min": -23.913188934326172,
      "activations/layer2_attention_weight_max": 34.17076873779297,
      "activations/layer2_attention_weight_min": -31.285579681396484,
      "activations/layer3_attention_weight_max": 93.65877532958984,
      "activations/layer3_attention_weight_min": -94.37549591064453,
      "activations/layer4_attention_weight_max": 102.02306365966797,
      "activations/layer4_attention_weight_min": -95.1329345703125,
      "activations/layer5_attention_weight_max": 71.69114685058594,
      "activations/layer5_attention_weight_min": -75.80536651611328,
      "activations/layer6_attention_weight_max": 58.0794792175293,
      "activations/layer6_attention_weight_min": -54.93027114868164,
      "activations/layer7_attention_weight_max": 77.8499984741211,
      "activations/layer7_attention_weight_min": -70.02690124511719,
      "activations/layer8_attention_weight_max": 46.4395866394043,
      "activations/layer8_attention_weight_min": -44.86375427246094,
      "activations/layer9_attention_weight_max": 58.954586029052734,
      "activations/layer9_attention_weight_min": -47.526432037353516,
      "epoch": 10.98,
      "learning_rate": 8.002386363636364e-05,
      "loss": 2.7855,
      "step": 188900
    },
    {
      "activations/layer0_attention_weight_max": 16.319599151611328,
      "activations/layer0_attention_weight_min": -14.072396278381348,
      "activations/layer10_attention_weight_max": 38.42243957519531,
      "activations/layer10_attention_weight_min": -35.24324035644531,
      "activations/layer11_attention_weight_max": 33.719608306884766,
      "activations/layer11_attention_weight_min": -35.54848861694336,
      "activations/layer12_attention_weight_max": 19.517240524291992,
      "activations/layer12_attention_weight_min": -25.926837921142578,
      "activations/layer13_attention_weight_max": 36.70801544189453,
      "activations/layer13_attention_weight_min": -31.203903198242188,
      "activations/layer14_attention_weight_max": 37.3775749206543,
      "activations/layer14_attention_weight_min": -31.984418869018555,
      "activations/layer15_attention_weight_max": 33.80473709106445,
      "activations/layer15_attention_weight_min": -31.033079147338867,
      "activations/layer16_attention_weight_max": 32.05317306518555,
      "activations/layer16_attention_weight_min": -29.03691864013672,
      "activations/layer17_attention_weight_max": 55.21560287475586,
      "activations/layer17_attention_weight_min": -43.9474983215332,
      "activations/layer18_attention_weight_max": 45.69357681274414,
      "activations/layer18_attention_weight_min": -36.93770980834961,
      "activations/layer19_attention_weight_max": 22.558805465698242,
      "activations/layer19_attention_weight_min": -20.663557052612305,
      "activations/layer1_attention_weight_max": 16.814632415771484,
      "activations/layer1_attention_weight_min": -16.07577896118164,
      "activations/layer20_attention_weight_max": 22.947227478027344,
      "activations/layer20_attention_weight_min": -19.93178939819336,
      "activations/layer21_attention_weight_max": 36.12272262573242,
      "activations/layer21_attention_weight_min": -21.744319915771484,
      "activations/layer22_attention_weight_max": 30.61139488220215,
      "activations/layer22_attention_weight_min": -25.694583892822266,
      "activations/layer23_attention_weight_max": 37.50883483886719,
      "activations/layer23_attention_weight_min": -24.246017456054688,
      "activations/layer2_attention_weight_max": 33.12592315673828,
      "activations/layer2_attention_weight_min": -30.704784393310547,
      "activations/layer3_attention_weight_max": 98.2378921508789,
      "activations/layer3_attention_weight_min": -93.82536315917969,
      "activations/layer4_attention_weight_max": 105.4073715209961,
      "activations/layer4_attention_weight_min": -95.83467864990234,
      "activations/layer5_attention_weight_max": 73.34451293945312,
      "activations/layer5_attention_weight_min": -81.3070068359375,
      "activations/layer6_attention_weight_max": 54.61549758911133,
      "activations/layer6_attention_weight_min": -54.924232482910156,
      "activations/layer7_attention_weight_max": 73.95980072021484,
      "activations/layer7_attention_weight_min": -72.25532531738281,
      "activations/layer8_attention_weight_max": 48.944068908691406,
      "activations/layer8_attention_weight_min": -51.054779052734375,
      "activations/layer9_attention_weight_max": 48.57451629638672,
      "activations/layer9_attention_weight_min": -49.35622787475586,
      "epoch": 10.98,
      "learning_rate": 8.000492424242423e-05,
      "loss": 2.7962,
      "step": 188950
    },
    {
      "activations/layer0_attention_weight_max": 16.80143928527832,
      "activations/layer0_attention_weight_min": -13.736014366149902,
      "activations/layer10_attention_weight_max": 35.628211975097656,
      "activations/layer10_attention_weight_min": -36.34495162963867,
      "activations/layer11_attention_weight_max": 30.8439998626709,
      "activations/layer11_attention_weight_min": -33.517208099365234,
      "activations/layer12_attention_weight_max": 18.759016036987305,
      "activations/layer12_attention_weight_min": -25.24726676940918,
      "activations/layer13_attention_weight_max": 35.34661865234375,
      "activations/layer13_attention_weight_min": -27.97455596923828,
      "activations/layer14_attention_weight_max": 34.4296760559082,
      "activations/layer14_attention_weight_min": -30.445783615112305,
      "activations/layer15_attention_weight_max": 32.345767974853516,
      "activations/layer15_attention_weight_min": -30.285295486450195,
      "activations/layer16_attention_weight_max": 28.89517593383789,
      "activations/layer16_attention_weight_min": -27.964521408081055,
      "activations/layer17_attention_weight_max": 48.98284912109375,
      "activations/layer17_attention_weight_min": -43.526371002197266,
      "activations/layer18_attention_weight_max": 43.119102478027344,
      "activations/layer18_attention_weight_min": -36.4453125,
      "activations/layer19_attention_weight_max": 21.39618682861328,
      "activations/layer19_attention_weight_min": -20.28730583190918,
      "activations/layer1_attention_weight_max": 16.361555099487305,
      "activations/layer1_attention_weight_min": -14.715361595153809,
      "activations/layer20_attention_weight_max": 21.266094207763672,
      "activations/layer20_attention_weight_min": -22.213735580444336,
      "activations/layer21_attention_weight_max": 39.81086349487305,
      "activations/layer21_attention_weight_min": -21.85852813720703,
      "activations/layer22_attention_weight_max": 32.31195831298828,
      "activations/layer22_attention_weight_min": -26.03607940673828,
      "activations/layer23_attention_weight_max": 37.03647232055664,
      "activations/layer23_attention_weight_min": -25.242958068847656,
      "activations/layer2_attention_weight_max": 31.952085494995117,
      "activations/layer2_attention_weight_min": -30.105344772338867,
      "activations/layer3_attention_weight_max": 88.8919906616211,
      "activations/layer3_attention_weight_min": -93.48914337158203,
      "activations/layer4_attention_weight_max": 94.24160766601562,
      "activations/layer4_attention_weight_min": -92.51764678955078,
      "activations/layer5_attention_weight_max": 68.94415283203125,
      "activations/layer5_attention_weight_min": -75.16148376464844,
      "activations/layer6_attention_weight_max": 52.78959274291992,
      "activations/layer6_attention_weight_min": -53.76605987548828,
      "activations/layer7_attention_weight_max": 75.87008666992188,
      "activations/layer7_attention_weight_min": -66.61293029785156,
      "activations/layer8_attention_weight_max": 44.518978118896484,
      "activations/layer8_attention_weight_min": -46.8853874206543,
      "activations/layer9_attention_weight_max": 45.06909942626953,
      "activations/layer9_attention_weight_min": -45.18526840209961,
      "epoch": 10.98,
      "learning_rate": 7.998598484848484e-05,
      "loss": 2.7909,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.4961,
      "eval_samples_per_second": 505.407,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.4961,
      "eval_openwebtext_samples_per_second": 505.407,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_wikitext_loss": 2.9765625,
      "eval_wikitext_ppl": 19.620255961497417,
      "eval_wikitext_runtime": 2.021,
      "eval_wikitext_samples_per_second": 225.626,
      "step": 189000
    },
    {
      "epoch": 10.98,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.5543,
      "eval_lambada_samples_per_second": 509.611,
      "step": 189000
    },
    {
      "activations/layer0_attention_weight_max": 16.48002815246582,
      "activations/layer0_attention_weight_min": -14.131217002868652,
      "activations/layer10_attention_weight_max": 37.94482421875,
      "activations/layer10_attention_weight_min": -38.96979522705078,
      "activations/layer11_attention_weight_max": 33.60240173339844,
      "activations/layer11_attention_weight_min": -34.63819885253906,
      "activations/layer12_attention_weight_max": 19.66431427001953,
      "activations/layer12_attention_weight_min": -24.18793296813965,
      "activations/layer13_attention_weight_max": 40.38948440551758,
      "activations/layer13_attention_weight_min": -29.529184341430664,
      "activations/layer14_attention_weight_max": 39.9792366027832,
      "activations/layer14_attention_weight_min": -30.731300354003906,
      "activations/layer15_attention_weight_max": 41.540504455566406,
      "activations/layer15_attention_weight_min": -30.39852523803711,
      "activations/layer16_attention_weight_max": 41.77351760864258,
      "activations/layer16_attention_weight_min": -31.208545684814453,
      "activations/layer17_attention_weight_max": 56.0428581237793,
      "activations/layer17_attention_weight_min": -44.5671501159668,
      "activations/layer18_attention_weight_max": 49.24958038330078,
      "activations/layer18_attention_weight_min": -37.38419723510742,
      "activations/layer19_attention_weight_max": 26.8123779296875,
      "activations/layer19_attention_weight_min": -21.47710609436035,
      "activations/layer1_attention_weight_max": 15.975288391113281,
      "activations/layer1_attention_weight_min": -14.769974708557129,
      "activations/layer20_attention_weight_max": 26.756650924682617,
      "activations/layer20_attention_weight_min": -21.14236831665039,
      "activations/layer21_attention_weight_max": 43.0819206237793,
      "activations/layer21_attention_weight_min": -22.773052215576172,
      "activations/layer22_attention_weight_max": 34.943328857421875,
      "activations/layer22_attention_weight_min": -26.087867736816406,
      "activations/layer23_attention_weight_max": 45.91337203979492,
      "activations/layer23_attention_weight_min": -22.715543746948242,
      "activations/layer2_attention_weight_max": 35.11721420288086,
      "activations/layer2_attention_weight_min": -32.53830337524414,
      "activations/layer3_attention_weight_max": 91.70172882080078,
      "activations/layer3_attention_weight_min": -95.4248046875,
      "activations/layer4_attention_weight_max": 96.6810073852539,
      "activations/layer4_attention_weight_min": -97.54157257080078,
      "activations/layer5_attention_weight_max": 73.66731262207031,
      "activations/layer5_attention_weight_min": -77.2315673828125,
      "activations/layer6_attention_weight_max": 56.61905288696289,
      "activations/layer6_attention_weight_min": -59.307952880859375,
      "activations/layer7_attention_weight_max": 75.68732452392578,
      "activations/layer7_attention_weight_min": -79.11306762695312,
      "activations/layer8_attention_weight_max": 48.246639251708984,
      "activations/layer8_attention_weight_min": -52.16371536254883,
      "activations/layer9_attention_weight_max": 52.85203170776367,
      "activations/layer9_attention_weight_min": -53.35891342163086,
      "epoch": 10.98,
      "learning_rate": 7.996742424242423e-05,
      "loss": 2.7838,
      "step": 189050
    },
    {
      "activations/layer0_attention_weight_max": 16.061290740966797,
      "activations/layer0_attention_weight_min": -14.180715560913086,
      "activations/layer10_attention_weight_max": 34.82078552246094,
      "activations/layer10_attention_weight_min": -35.7198486328125,
      "activations/layer11_attention_weight_max": 30.048858642578125,
      "activations/layer11_attention_weight_min": -31.5357723236084,
      "activations/layer12_attention_weight_max": 18.336423873901367,
      "activations/layer12_attention_weight_min": -24.47684097290039,
      "activations/layer13_attention_weight_max": 41.50931930541992,
      "activations/layer13_attention_weight_min": -32.4099006652832,
      "activations/layer14_attention_weight_max": 38.94731521606445,
      "activations/layer14_attention_weight_min": -30.746509552001953,
      "activations/layer15_attention_weight_max": 34.3917121887207,
      "activations/layer15_attention_weight_min": -29.11974334716797,
      "activations/layer16_attention_weight_max": 30.190954208374023,
      "activations/layer16_attention_weight_min": -30.460586547851562,
      "activations/layer17_attention_weight_max": 49.34113311767578,
      "activations/layer17_attention_weight_min": -44.79920959472656,
      "activations/layer18_attention_weight_max": 45.02695083618164,
      "activations/layer18_attention_weight_min": -36.75463104248047,
      "activations/layer19_attention_weight_max": 21.145017623901367,
      "activations/layer19_attention_weight_min": -22.688623428344727,
      "activations/layer1_attention_weight_max": 17.2169132232666,
      "activations/layer1_attention_weight_min": -14.585965156555176,
      "activations/layer20_attention_weight_max": 21.38616943359375,
      "activations/layer20_attention_weight_min": -23.165443420410156,
      "activations/layer21_attention_weight_max": 32.29426574707031,
      "activations/layer21_attention_weight_min": -22.66606330871582,
      "activations/layer22_attention_weight_max": 28.9466552734375,
      "activations/layer22_attention_weight_min": -27.0008544921875,
      "activations/layer23_attention_weight_max": 35.19014358520508,
      "activations/layer23_attention_weight_min": -23.09067153930664,
      "activations/layer2_attention_weight_max": 32.796112060546875,
      "activations/layer2_attention_weight_min": -31.915090560913086,
      "activations/layer3_attention_weight_max": 92.47172546386719,
      "activations/layer3_attention_weight_min": -90.71675872802734,
      "activations/layer4_attention_weight_max": 96.7166976928711,
      "activations/layer4_attention_weight_min": -94.2120590209961,
      "activations/layer5_attention_weight_max": 67.99282836914062,
      "activations/layer5_attention_weight_min": -74.1800537109375,
      "activations/layer6_attention_weight_max": 51.65929412841797,
      "activations/layer6_attention_weight_min": -53.10371780395508,
      "activations/layer7_attention_weight_max": 76.56243896484375,
      "activations/layer7_attention_weight_min": -69.58728790283203,
      "activations/layer8_attention_weight_max": 42.879112243652344,
      "activations/layer8_attention_weight_min": -48.798789978027344,
      "activations/layer9_attention_weight_max": 51.75504684448242,
      "activations/layer9_attention_weight_min": -46.020206451416016,
      "epoch": 10.99,
      "learning_rate": 7.994848484848485e-05,
      "loss": 2.7842,
      "step": 189100
    },
    {
      "activations/layer0_attention_weight_max": 15.145238876342773,
      "activations/layer0_attention_weight_min": -13.728607177734375,
      "activations/layer10_attention_weight_max": 35.00608444213867,
      "activations/layer10_attention_weight_min": -35.684200286865234,
      "activations/layer11_attention_weight_max": 31.970701217651367,
      "activations/layer11_attention_weight_min": -32.45612335205078,
      "activations/layer12_attention_weight_max": 18.976516723632812,
      "activations/layer12_attention_weight_min": -25.494678497314453,
      "activations/layer13_attention_weight_max": 37.80400085449219,
      "activations/layer13_attention_weight_min": -27.00981903076172,
      "activations/layer14_attention_weight_max": 35.91211700439453,
      "activations/layer14_attention_weight_min": -30.209491729736328,
      "activations/layer15_attention_weight_max": 31.540952682495117,
      "activations/layer15_attention_weight_min": -31.272974014282227,
      "activations/layer16_attention_weight_max": 31.628129959106445,
      "activations/layer16_attention_weight_min": -29.983238220214844,
      "activations/layer17_attention_weight_max": 48.75392532348633,
      "activations/layer17_attention_weight_min": -43.83768844604492,
      "activations/layer18_attention_weight_max": 45.27952194213867,
      "activations/layer18_attention_weight_min": -36.782684326171875,
      "activations/layer19_attention_weight_max": 20.465991973876953,
      "activations/layer19_attention_weight_min": -20.510839462280273,
      "activations/layer1_attention_weight_max": 17.522811889648438,
      "activations/layer1_attention_weight_min": -17.148527145385742,
      "activations/layer20_attention_weight_max": 20.266014099121094,
      "activations/layer20_attention_weight_min": -22.315397262573242,
      "activations/layer21_attention_weight_max": 36.73660659790039,
      "activations/layer21_attention_weight_min": -26.083024978637695,
      "activations/layer22_attention_weight_max": 29.846759796142578,
      "activations/layer22_attention_weight_min": -26.099641799926758,
      "activations/layer23_attention_weight_max": 39.95355224609375,
      "activations/layer23_attention_weight_min": -23.803503036499023,
      "activations/layer2_attention_weight_max": 33.77329635620117,
      "activations/layer2_attention_weight_min": -30.359718322753906,
      "activations/layer3_attention_weight_max": 94.97893524169922,
      "activations/layer3_attention_weight_min": -90.90193939208984,
      "activations/layer4_attention_weight_max": 101.12029266357422,
      "activations/layer4_attention_weight_min": -95.9483642578125,
      "activations/layer5_attention_weight_max": 77.10588073730469,
      "activations/layer5_attention_weight_min": -81.4640884399414,
      "activations/layer6_attention_weight_max": 55.44865798950195,
      "activations/layer6_attention_weight_min": -55.1549072265625,
      "activations/layer7_attention_weight_max": 76.61481475830078,
      "activations/layer7_attention_weight_min": -75.25132751464844,
      "activations/layer8_attention_weight_max": 44.99615478515625,
      "activations/layer8_attention_weight_min": -48.50048828125,
      "activations/layer9_attention_weight_max": 49.66986083984375,
      "activations/layer9_attention_weight_min": -49.30017852783203,
      "epoch": 10.99,
      "learning_rate": 7.992954545454544e-05,
      "loss": 2.7796,
      "step": 189150
    },
    {
      "activations/layer0_attention_weight_max": 15.425524711608887,
      "activations/layer0_attention_weight_min": -14.00055980682373,
      "activations/layer10_attention_weight_max": 35.94792556762695,
      "activations/layer10_attention_weight_min": -35.42545700073242,
      "activations/layer11_attention_weight_max": 31.82721710205078,
      "activations/layer11_attention_weight_min": -31.848724365234375,
      "activations/layer12_attention_weight_max": 18.459985733032227,
      "activations/layer12_attention_weight_min": -27.281770706176758,
      "activations/layer13_attention_weight_max": 37.0836067199707,
      "activations/layer13_attention_weight_min": -28.44378089904785,
      "activations/layer14_attention_weight_max": 36.54341125488281,
      "activations/layer14_attention_weight_min": -27.75157928466797,
      "activations/layer15_attention_weight_max": 32.207679748535156,
      "activations/layer15_attention_weight_min": -28.83761215209961,
      "activations/layer16_attention_weight_max": 30.67060089111328,
      "activations/layer16_attention_weight_min": -28.3684139251709,
      "activations/layer17_attention_weight_max": 49.72809982299805,
      "activations/layer17_attention_weight_min": -41.69150924682617,
      "activations/layer18_attention_weight_max": 46.06130599975586,
      "activations/layer18_attention_weight_min": -36.16933822631836,
      "activations/layer19_attention_weight_max": 19.8078556060791,
      "activations/layer19_attention_weight_min": -21.34450340270996,
      "activations/layer1_attention_weight_max": 16.710203170776367,
      "activations/layer1_attention_weight_min": -15.117258071899414,
      "activations/layer20_attention_weight_max": 19.921613693237305,
      "activations/layer20_attention_weight_min": -22.101049423217773,
      "activations/layer21_attention_weight_max": 32.63494873046875,
      "activations/layer21_attention_weight_min": -23.449169158935547,
      "activations/layer22_attention_weight_max": 30.98786735534668,
      "activations/layer22_attention_weight_min": -26.31515884399414,
      "activations/layer23_attention_weight_max": 37.12250518798828,
      "activations/layer23_attention_weight_min": -26.885536193847656,
      "activations/layer2_attention_weight_max": 33.92095947265625,
      "activations/layer2_attention_weight_min": -29.704198837280273,
      "activations/layer3_attention_weight_max": 88.47130584716797,
      "activations/layer3_attention_weight_min": -90.00386047363281,
      "activations/layer4_attention_weight_max": 92.41315460205078,
      "activations/layer4_attention_weight_min": -90.13774871826172,
      "activations/layer5_attention_weight_max": 68.85457611083984,
      "activations/layer5_attention_weight_min": -79.420654296875,
      "activations/layer6_attention_weight_max": 51.01810073852539,
      "activations/layer6_attention_weight_min": -50.72311019897461,
      "activations/layer7_attention_weight_max": 69.54266357421875,
      "activations/layer7_attention_weight_min": -73.02717590332031,
      "activations/layer8_attention_weight_max": 44.295623779296875,
      "activations/layer8_attention_weight_min": -47.879119873046875,
      "activations/layer9_attention_weight_max": 48.03397750854492,
      "activations/layer9_attention_weight_min": -48.26015853881836,
      "epoch": 10.99,
      "learning_rate": 7.991060606060605e-05,
      "loss": 2.7828,
      "step": 189200
    },
    {
      "activations/layer0_attention_weight_max": 17.096923828125,
      "activations/layer0_attention_weight_min": -13.595096588134766,
      "activations/layer10_attention_weight_max": 42.338905334472656,
      "activations/layer10_attention_weight_min": -40.268611907958984,
      "activations/layer11_attention_weight_max": 35.85495376586914,
      "activations/layer11_attention_weight_min": -36.705135345458984,
      "activations/layer12_attention_weight_max": 25.53693199157715,
      "activations/layer12_attention_weight_min": -26.851945877075195,
      "activations/layer13_attention_weight_max": 45.407264709472656,
      "activations/layer13_attention_weight_min": -34.142696380615234,
      "activations/layer14_attention_weight_max": 41.1555290222168,
      "activations/layer14_attention_weight_min": -31.364788055419922,
      "activations/layer15_attention_weight_max": 38.49045181274414,
      "activations/layer15_attention_weight_min": -30.524311065673828,
      "activations/layer16_attention_weight_max": 34.85946273803711,
      "activations/layer16_attention_weight_min": -31.141475677490234,
      "activations/layer17_attention_weight_max": 62.99736404418945,
      "activations/layer17_attention_weight_min": -47.75288772583008,
      "activations/layer18_attention_weight_max": 52.7095832824707,
      "activations/layer18_attention_weight_min": -39.054683685302734,
      "activations/layer19_attention_weight_max": 25.529088973999023,
      "activations/layer19_attention_weight_min": -20.970815658569336,
      "activations/layer1_attention_weight_max": 16.075878143310547,
      "activations/layer1_attention_weight_min": -15.835420608520508,
      "activations/layer20_attention_weight_max": 24.588998794555664,
      "activations/layer20_attention_weight_min": -21.95760154724121,
      "activations/layer21_attention_weight_max": 36.65264892578125,
      "activations/layer21_attention_weight_min": -24.899484634399414,
      "activations/layer22_attention_weight_max": 32.404659271240234,
      "activations/layer22_attention_weight_min": -28.951610565185547,
      "activations/layer23_attention_weight_max": 36.192909240722656,
      "activations/layer23_attention_weight_min": -22.848920822143555,
      "activations/layer2_attention_weight_max": 33.71573257446289,
      "activations/layer2_attention_weight_min": -31.96772003173828,
      "activations/layer3_attention_weight_max": 96.02751922607422,
      "activations/layer3_attention_weight_min": -94.0423812866211,
      "activations/layer4_attention_weight_max": 102.95137023925781,
      "activations/layer4_attention_weight_min": -96.88670349121094,
      "activations/layer5_attention_weight_max": 72.14360046386719,
      "activations/layer5_attention_weight_min": -76.34884643554688,
      "activations/layer6_attention_weight_max": 55.7218132019043,
      "activations/layer6_attention_weight_min": -56.44518280029297,
      "activations/layer7_attention_weight_max": 79.61470794677734,
      "activations/layer7_attention_weight_min": -74.50902557373047,
      "activations/layer8_attention_weight_max": 52.17302322387695,
      "activations/layer8_attention_weight_min": -53.49625778198242,
      "activations/layer9_attention_weight_max": 51.89951705932617,
      "activations/layer9_attention_weight_min": -52.33173370361328,
      "epoch": 11.0,
      "learning_rate": 7.989166666666667e-05,
      "loss": 2.7853,
      "step": 189250
    },
    {
      "activations/layer0_attention_weight_max": 16.1881160736084,
      "activations/layer0_attention_weight_min": -13.831369400024414,
      "activations/layer10_attention_weight_max": 38.85585403442383,
      "activations/layer10_attention_weight_min": -38.65037536621094,
      "activations/layer11_attention_weight_max": 32.4114990234375,
      "activations/layer11_attention_weight_min": -36.10309982299805,
      "activations/layer12_attention_weight_max": 18.28813934326172,
      "activations/layer12_attention_weight_min": -25.22455406188965,
      "activations/layer13_attention_weight_max": 47.29541778564453,
      "activations/layer13_attention_weight_min": -30.412805557250977,
      "activations/layer14_attention_weight_max": 36.54045104980469,
      "activations/layer14_attention_weight_min": -31.27781105041504,
      "activations/layer15_attention_weight_max": 39.30540084838867,
      "activations/layer15_attention_weight_min": -29.212427139282227,
      "activations/layer16_attention_weight_max": 32.013397216796875,
      "activations/layer16_attention_weight_min": -29.822710037231445,
      "activations/layer17_attention_weight_max": 54.3759651184082,
      "activations/layer17_attention_weight_min": -43.82002639770508,
      "activations/layer18_attention_weight_max": 46.98640823364258,
      "activations/layer18_attention_weight_min": -39.90164566040039,
      "activations/layer19_attention_weight_max": 21.611282348632812,
      "activations/layer19_attention_weight_min": -24.473262786865234,
      "activations/layer1_attention_weight_max": 17.09636116027832,
      "activations/layer1_attention_weight_min": -14.975808143615723,
      "activations/layer20_attention_weight_max": 22.5504093170166,
      "activations/layer20_attention_weight_min": -21.007455825805664,
      "activations/layer21_attention_weight_max": 38.71627426147461,
      "activations/layer21_attention_weight_min": -23.048913955688477,
      "activations/layer22_attention_weight_max": 28.78285789489746,
      "activations/layer22_attention_weight_min": -26.49591827392578,
      "activations/layer23_attention_weight_max": 38.31269454956055,
      "activations/layer23_attention_weight_min": -23.766780853271484,
      "activations/layer2_attention_weight_max": 34.5411376953125,
      "activations/layer2_attention_weight_min": -33.223045349121094,
      "activations/layer3_attention_weight_max": 91.48562622070312,
      "activations/layer3_attention_weight_min": -95.19561004638672,
      "activations/layer4_attention_weight_max": 102.6395263671875,
      "activations/layer4_attention_weight_min": -101.18714904785156,
      "activations/layer5_attention_weight_max": 72.32701110839844,
      "activations/layer5_attention_weight_min": -78.54058837890625,
      "activations/layer6_attention_weight_max": 59.12387466430664,
      "activations/layer6_attention_weight_min": -57.14728927612305,
      "activations/layer7_attention_weight_max": 76.59930419921875,
      "activations/layer7_attention_weight_min": -73.25589752197266,
      "activations/layer8_attention_weight_max": 47.350181579589844,
      "activations/layer8_attention_weight_min": -50.902713775634766,
      "activations/layer9_attention_weight_max": 50.70878219604492,
      "activations/layer9_attention_weight_min": -47.60332489013672,
      "epoch": 11.0,
      "learning_rate": 7.987272727272725e-05,
      "loss": 2.7846,
      "step": 189300
    },
    {
      "activations/layer0_attention_weight_max": 16.145919799804688,
      "activations/layer0_attention_weight_min": -13.701994895935059,
      "activations/layer10_attention_weight_max": 40.339569091796875,
      "activations/layer10_attention_weight_min": -37.03287887573242,
      "activations/layer11_attention_weight_max": 33.13031005859375,
      "activations/layer11_attention_weight_min": -32.87985610961914,
      "activations/layer12_attention_weight_max": 19.883319854736328,
      "activations/layer12_attention_weight_min": -25.658082962036133,
      "activations/layer13_attention_weight_max": 39.38819885253906,
      "activations/layer13_attention_weight_min": -29.10187339782715,
      "activations/layer14_attention_weight_max": 36.93412399291992,
      "activations/layer14_attention_weight_min": -29.472227096557617,
      "activations/layer15_attention_weight_max": 35.992218017578125,
      "activations/layer15_attention_weight_min": -29.77809715270996,
      "activations/layer16_attention_weight_max": 31.855361938476562,
      "activations/layer16_attention_weight_min": -29.514102935791016,
      "activations/layer17_attention_weight_max": 53.383304595947266,
      "activations/layer17_attention_weight_min": -43.2597770690918,
      "activations/layer18_attention_weight_max": 48.31300354003906,
      "activations/layer18_attention_weight_min": -36.645381927490234,
      "activations/layer19_attention_weight_max": 22.66974449157715,
      "activations/layer19_attention_weight_min": -19.4271297454834,
      "activations/layer1_attention_weight_max": 16.19313621520996,
      "activations/layer1_attention_weight_min": -13.597493171691895,
      "activations/layer20_attention_weight_max": 20.601253509521484,
      "activations/layer20_attention_weight_min": -19.861392974853516,
      "activations/layer21_attention_weight_max": 35.884735107421875,
      "activations/layer21_attention_weight_min": -24.181901931762695,
      "activations/layer22_attention_weight_max": 32.857093811035156,
      "activations/layer22_attention_weight_min": -25.33615493774414,
      "activations/layer23_attention_weight_max": 44.7530517578125,
      "activations/layer23_attention_weight_min": -24.720123291015625,
      "activations/layer2_attention_weight_max": 31.978572845458984,
      "activations/layer2_attention_weight_min": -30.581043243408203,
      "activations/layer3_attention_weight_max": 87.78929138183594,
      "activations/layer3_attention_weight_min": -90.39850616455078,
      "activations/layer4_attention_weight_max": 96.2967758178711,
      "activations/layer4_attention_weight_min": -94.98617553710938,
      "activations/layer5_attention_weight_max": 71.18172454833984,
      "activations/layer5_attention_weight_min": -78.4190673828125,
      "activations/layer6_attention_weight_max": 57.776756286621094,
      "activations/layer6_attention_weight_min": -54.21381378173828,
      "activations/layer7_attention_weight_max": 76.82295989990234,
      "activations/layer7_attention_weight_min": -71.8810806274414,
      "activations/layer8_attention_weight_max": 48.8754997253418,
      "activations/layer8_attention_weight_min": -50.94544219970703,
      "activations/layer9_attention_weight_max": 50.568729400634766,
      "activations/layer9_attention_weight_min": -52.285400390625,
      "epoch": 11.0,
      "learning_rate": 7.985378787878787e-05,
      "loss": 2.8379,
      "step": 189350
    },
    {
      "activations/layer0_attention_weight_max": 17.254711151123047,
      "activations/layer0_attention_weight_min": -13.440824508666992,
      "activations/layer10_attention_weight_max": 41.7836799621582,
      "activations/layer10_attention_weight_min": -42.085506439208984,
      "activations/layer11_attention_weight_max": 39.19762420654297,
      "activations/layer11_attention_weight_min": -35.79263687133789,
      "activations/layer12_attention_weight_max": 18.138925552368164,
      "activations/layer12_attention_weight_min": -23.89426612854004,
      "activations/layer13_attention_weight_max": 46.05781555175781,
      "activations/layer13_attention_weight_min": -32.34078598022461,
      "activations/layer14_attention_weight_max": 41.64562225341797,
      "activations/layer14_attention_weight_min": -29.502391815185547,
      "activations/layer15_attention_weight_max": 34.36608123779297,
      "activations/layer15_attention_weight_min": -30.94905662536621,
      "activations/layer16_attention_weight_max": 28.39152717590332,
      "activations/layer16_attention_weight_min": -28.342260360717773,
      "activations/layer17_attention_weight_max": 51.43402862548828,
      "activations/layer17_attention_weight_min": -42.84686279296875,
      "activations/layer18_attention_weight_max": 43.63344955444336,
      "activations/layer18_attention_weight_min": -35.434844970703125,
      "activations/layer19_attention_weight_max": 20.5841121673584,
      "activations/layer19_attention_weight_min": -20.313335418701172,
      "activations/layer1_attention_weight_max": 16.283388137817383,
      "activations/layer1_attention_weight_min": -14.396077156066895,
      "activations/layer20_attention_weight_max": 19.501249313354492,
      "activations/layer20_attention_weight_min": -21.44734764099121,
      "activations/layer21_attention_weight_max": 37.95120620727539,
      "activations/layer21_attention_weight_min": -21.710617065429688,
      "activations/layer22_attention_weight_max": 30.770931243896484,
      "activations/layer22_attention_weight_min": -25.0771541595459,
      "activations/layer23_attention_weight_max": 36.718544006347656,
      "activations/layer23_attention_weight_min": -23.09088706970215,
      "activations/layer2_attention_weight_max": 33.80621337890625,
      "activations/layer2_attention_weight_min": -31.684234619140625,
      "activations/layer3_attention_weight_max": 94.3998794555664,
      "activations/layer3_attention_weight_min": -95.75123596191406,
      "activations/layer4_attention_weight_max": 101.28104400634766,
      "activations/layer4_attention_weight_min": -98.09103393554688,
      "activations/layer5_attention_weight_max": 73.83235931396484,
      "activations/layer5_attention_weight_min": -79.32548522949219,
      "activations/layer6_attention_weight_max": 54.65026092529297,
      "activations/layer6_attention_weight_min": -52.91053771972656,
      "activations/layer7_attention_weight_max": 76.20604705810547,
      "activations/layer7_attention_weight_min": -78.41117095947266,
      "activations/layer8_attention_weight_max": 50.71742630004883,
      "activations/layer8_attention_weight_min": -47.93425369262695,
      "activations/layer9_attention_weight_max": 60.24915313720703,
      "activations/layer9_attention_weight_min": -49.73798751831055,
      "epoch": 11.01,
      "learning_rate": 7.983484848484848e-05,
      "loss": 2.7676,
      "step": 189400
    },
    {
      "activations/layer0_attention_weight_max": 16.32587242126465,
      "activations/layer0_attention_weight_min": -13.758723258972168,
      "activations/layer10_attention_weight_max": 41.9516487121582,
      "activations/layer10_attention_weight_min": -39.75697326660156,
      "activations/layer11_attention_weight_max": 37.32769012451172,
      "activations/layer11_attention_weight_min": -35.47154998779297,
      "activations/layer12_attention_weight_max": 18.06606674194336,
      "activations/layer12_attention_weight_min": -29.437252044677734,
      "activations/layer13_attention_weight_max": 50.49059295654297,
      "activations/layer13_attention_weight_min": -32.861427307128906,
      "activations/layer14_attention_weight_max": 38.751502990722656,
      "activations/layer14_attention_weight_min": -30.625551223754883,
      "activations/layer15_attention_weight_max": 35.0315055847168,
      "activations/layer15_attention_weight_min": -28.901630401611328,
      "activations/layer16_attention_weight_max": 30.12150001525879,
      "activations/layer16_attention_weight_min": -28.37230110168457,
      "activations/layer17_attention_weight_max": 54.30326461791992,
      "activations/layer17_attention_weight_min": -44.38948440551758,
      "activations/layer18_attention_weight_max": 48.94168472290039,
      "activations/layer18_attention_weight_min": -37.45627975463867,
      "activations/layer19_attention_weight_max": 22.655920028686523,
      "activations/layer19_attention_weight_min": -22.534149169921875,
      "activations/layer1_attention_weight_max": 16.987489700317383,
      "activations/layer1_attention_weight_min": -15.763731002807617,
      "activations/layer20_attention_weight_max": 23.184886932373047,
      "activations/layer20_attention_weight_min": -22.604642868041992,
      "activations/layer21_attention_weight_max": 35.77765655517578,
      "activations/layer21_attention_weight_min": -21.521446228027344,
      "activations/layer22_attention_weight_max": 30.75650978088379,
      "activations/layer22_attention_weight_min": -25.85128402709961,
      "activations/layer23_attention_weight_max": 37.285255432128906,
      "activations/layer23_attention_weight_min": -25.225608825683594,
      "activations/layer2_attention_weight_max": 33.16230773925781,
      "activations/layer2_attention_weight_min": -31.64423370361328,
      "activations/layer3_attention_weight_max": 90.96981048583984,
      "activations/layer3_attention_weight_min": -96.08242797851562,
      "activations/layer4_attention_weight_max": 110.6754150390625,
      "activations/layer4_attention_weight_min": -100.10567474365234,
      "activations/layer5_attention_weight_max": 75.25336456298828,
      "activations/layer5_attention_weight_min": -77.73966979980469,
      "activations/layer6_attention_weight_max": 55.88785934448242,
      "activations/layer6_attention_weight_min": -56.800743103027344,
      "activations/layer7_attention_weight_max": 78.64828491210938,
      "activations/layer7_attention_weight_min": -74.5873794555664,
      "activations/layer8_attention_weight_max": 49.28798294067383,
      "activations/layer8_attention_weight_min": -49.65485382080078,
      "activations/layer9_attention_weight_max": 56.586822509765625,
      "activations/layer9_attention_weight_min": -51.672584533691406,
      "epoch": 11.01,
      "learning_rate": 7.981628787878787e-05,
      "loss": 2.782,
      "step": 189450
    },
    {
      "activations/layer0_attention_weight_max": 16.295120239257812,
      "activations/layer0_attention_weight_min": -13.783920288085938,
      "activations/layer10_attention_weight_max": 34.56888961791992,
      "activations/layer10_attention_weight_min": -33.615966796875,
      "activations/layer11_attention_weight_max": 30.49401092529297,
      "activations/layer11_attention_weight_min": -31.506513595581055,
      "activations/layer12_attention_weight_max": 17.844562530517578,
      "activations/layer12_attention_weight_min": -26.840656280517578,
      "activations/layer13_attention_weight_max": 42.08464050292969,
      "activations/layer13_attention_weight_min": -33.165428161621094,
      "activations/layer14_attention_weight_max": 37.241050720214844,
      "activations/layer14_attention_weight_min": -31.328710556030273,
      "activations/layer15_attention_weight_max": 39.652366638183594,
      "activations/layer15_attention_weight_min": -29.040870666503906,
      "activations/layer16_attention_weight_max": 34.318538665771484,
      "activations/layer16_attention_weight_min": -29.44125747680664,
      "activations/layer17_attention_weight_max": 52.395015716552734,
      "activations/layer17_attention_weight_min": -44.847103118896484,
      "activations/layer18_attention_weight_max": 46.03160858154297,
      "activations/layer18_attention_weight_min": -37.94793701171875,
      "activations/layer19_attention_weight_max": 24.964956283569336,
      "activations/layer19_attention_weight_min": -23.84644317626953,
      "activations/layer1_attention_weight_max": 17.0727481842041,
      "activations/layer1_attention_weight_min": -15.744900703430176,
      "activations/layer20_attention_weight_max": 24.9780216217041,
      "activations/layer20_attention_weight_min": -22.730175018310547,
      "activations/layer21_attention_weight_max": 40.634437561035156,
      "activations/layer21_attention_weight_min": -25.07979393005371,
      "activations/layer22_attention_weight_max": 32.91719055175781,
      "activations/layer22_attention_weight_min": -28.479827880859375,
      "activations/layer23_attention_weight_max": 39.41761779785156,
      "activations/layer23_attention_weight_min": -23.77289390563965,
      "activations/layer2_attention_weight_max": 31.996723175048828,
      "activations/layer2_attention_weight_min": -31.135482788085938,
      "activations/layer3_attention_weight_max": 89.83313751220703,
      "activations/layer3_attention_weight_min": -91.37593078613281,
      "activations/layer4_attention_weight_max": 95.24254608154297,
      "activations/layer4_attention_weight_min": -96.53399658203125,
      "activations/layer5_attention_weight_max": 70.97932434082031,
      "activations/layer5_attention_weight_min": -77.36744689941406,
      "activations/layer6_attention_weight_max": 50.81912612915039,
      "activations/layer6_attention_weight_min": -52.046566009521484,
      "activations/layer7_attention_weight_max": 73.17887115478516,
      "activations/layer7_attention_weight_min": -75.1583480834961,
      "activations/layer8_attention_weight_max": 43.12166213989258,
      "activations/layer8_attention_weight_min": -45.036705017089844,
      "activations/layer9_attention_weight_max": 46.866886138916016,
      "activations/layer9_attention_weight_min": -45.49311828613281,
      "epoch": 11.01,
      "learning_rate": 7.979734848484848e-05,
      "loss": 2.7612,
      "step": 189500
    },
    {
      "activations/layer0_attention_weight_max": 16.108179092407227,
      "activations/layer0_attention_weight_min": -13.827841758728027,
      "activations/layer10_attention_weight_max": 33.55669403076172,
      "activations/layer10_attention_weight_min": -35.217777252197266,
      "activations/layer11_attention_weight_max": 29.959976196289062,
      "activations/layer11_attention_weight_min": -32.40259552001953,
      "activations/layer12_attention_weight_max": 20.02435302734375,
      "activations/layer12_attention_weight_min": -27.783138275146484,
      "activations/layer13_attention_weight_max": 39.10039520263672,
      "activations/layer13_attention_weight_min": -30.57159996032715,
      "activations/layer14_attention_weight_max": 51.52938461303711,
      "activations/layer14_attention_weight_min": -31.599655151367188,
      "activations/layer15_attention_weight_max": 39.50703811645508,
      "activations/layer15_attention_weight_min": -29.52828025817871,
      "activations/layer16_attention_weight_max": 33.91378402709961,
      "activations/layer16_attention_weight_min": -28.05910873413086,
      "activations/layer17_attention_weight_max": 57.45680236816406,
      "activations/layer17_attention_weight_min": -43.42266082763672,
      "activations/layer18_attention_weight_max": 59.19194793701172,
      "activations/layer18_attention_weight_min": -40.516136169433594,
      "activations/layer19_attention_weight_max": 26.459726333618164,
      "activations/layer19_attention_weight_min": -23.9785099029541,
      "activations/layer1_attention_weight_max": 16.872669219970703,
      "activations/layer1_attention_weight_min": -17.204675674438477,
      "activations/layer20_attention_weight_max": 28.955310821533203,
      "activations/layer20_attention_weight_min": -22.299774169921875,
      "activations/layer21_attention_weight_max": 49.168724060058594,
      "activations/layer21_attention_weight_min": -25.8667049407959,
      "activations/layer22_attention_weight_max": 36.400508880615234,
      "activations/layer22_attention_weight_min": -25.06329917907715,
      "activations/layer23_attention_weight_max": 45.75973129272461,
      "activations/layer23_attention_weight_min": -22.480253219604492,
      "activations/layer2_attention_weight_max": 31.41364097595215,
      "activations/layer2_attention_weight_min": -31.81053924560547,
      "activations/layer3_attention_weight_max": 95.07060241699219,
      "activations/layer3_attention_weight_min": -98.64521026611328,
      "activations/layer4_attention_weight_max": 94.67937469482422,
      "activations/layer4_attention_weight_min": -93.43614959716797,
      "activations/layer5_attention_weight_max": 66.73892211914062,
      "activations/layer5_attention_weight_min": -75.7459716796875,
      "activations/layer6_attention_weight_max": 52.94596862792969,
      "activations/layer6_attention_weight_min": -52.823421478271484,
      "activations/layer7_attention_weight_max": 74.90689849853516,
      "activations/layer7_attention_weight_min": -68.40254974365234,
      "activations/layer8_attention_weight_max": 45.76926803588867,
      "activations/layer8_attention_weight_min": -47.03257751464844,
      "activations/layer9_attention_weight_max": 46.27131271362305,
      "activations/layer9_attention_weight_min": -44.593379974365234,
      "epoch": 11.01,
      "learning_rate": 7.977840909090908e-05,
      "loss": 2.7666,
      "step": 189550
    },
    {
      "activations/layer0_attention_weight_max": 16.456157684326172,
      "activations/layer0_attention_weight_min": -14.049354553222656,
      "activations/layer10_attention_weight_max": 38.42432403564453,
      "activations/layer10_attention_weight_min": -35.923343658447266,
      "activations/layer11_attention_weight_max": 34.19581985473633,
      "activations/layer11_attention_weight_min": -34.47621154785156,
      "activations/layer12_attention_weight_max": 18.237064361572266,
      "activations/layer12_attention_weight_min": -23.894607543945312,
      "activations/layer13_attention_weight_max": 47.10060501098633,
      "activations/layer13_attention_weight_min": -32.1118278503418,
      "activations/layer14_attention_weight_max": 48.92613220214844,
      "activations/layer14_attention_weight_min": -31.8514404296875,
      "activations/layer15_attention_weight_max": 38.70549011230469,
      "activations/layer15_attention_weight_min": -28.696537017822266,
      "activations/layer16_attention_weight_max": 32.27991485595703,
      "activations/layer16_attention_weight_min": -31.208484649658203,
      "activations/layer17_attention_weight_max": 59.48308181762695,
      "activations/layer17_attention_weight_min": -46.29946517944336,
      "activations/layer18_attention_weight_max": 53.76249313354492,
      "activations/layer18_attention_weight_min": -37.641910552978516,
      "activations/layer19_attention_weight_max": 25.398418426513672,
      "activations/layer19_attention_weight_min": -22.314760208129883,
      "activations/layer1_attention_weight_max": 17.076112747192383,
      "activations/layer1_attention_weight_min": -15.857733726501465,
      "activations/layer20_attention_weight_max": 26.73979377746582,
      "activations/layer20_attention_weight_min": -21.790891647338867,
      "activations/layer21_attention_weight_max": 44.1164665222168,
      "activations/layer21_attention_weight_min": -22.80649185180664,
      "activations/layer22_attention_weight_max": 32.68375015258789,
      "activations/layer22_attention_weight_min": -25.778236389160156,
      "activations/layer23_attention_weight_max": 44.61534118652344,
      "activations/layer23_attention_weight_min": -24.73224449157715,
      "activations/layer2_attention_weight_max": 31.209278106689453,
      "activations/layer2_attention_weight_min": -30.547378540039062,
      "activations/layer3_attention_weight_max": 88.08224487304688,
      "activations/layer3_attention_weight_min": -92.43925476074219,
      "activations/layer4_attention_weight_max": 91.91690826416016,
      "activations/layer4_attention_weight_min": -89.25249481201172,
      "activations/layer5_attention_weight_max": 72.52910614013672,
      "activations/layer5_attention_weight_min": -75.71083068847656,
      "activations/layer6_attention_weight_max": 53.13003158569336,
      "activations/layer6_attention_weight_min": -52.313148498535156,
      "activations/layer7_attention_weight_max": 76.52056884765625,
      "activations/layer7_attention_weight_min": -69.6530532836914,
      "activations/layer8_attention_weight_max": 46.340450286865234,
      "activations/layer8_attention_weight_min": -45.95083999633789,
      "activations/layer9_attention_weight_max": 49.92210388183594,
      "activations/layer9_attention_weight_min": -45.4061393737793,
      "epoch": 11.02,
      "learning_rate": 7.975946969696969e-05,
      "loss": 2.7753,
      "step": 189600
    },
    {
      "activations/layer0_attention_weight_max": 15.59268856048584,
      "activations/layer0_attention_weight_min": -14.039205551147461,
      "activations/layer10_attention_weight_max": 36.89425277709961,
      "activations/layer10_attention_weight_min": -34.53367614746094,
      "activations/layer11_attention_weight_max": 33.94118118286133,
      "activations/layer11_attention_weight_min": -34.20735168457031,
      "activations/layer12_attention_weight_max": 19.265216827392578,
      "activations/layer12_attention_weight_min": -26.74317741394043,
      "activations/layer13_attention_weight_max": 46.56709289550781,
      "activations/layer13_attention_weight_min": -30.74167251586914,
      "activations/layer14_attention_weight_max": 50.27533721923828,
      "activations/layer14_attention_weight_min": -29.422273635864258,
      "activations/layer15_attention_weight_max": 35.059608459472656,
      "activations/layer15_attention_weight_min": -28.357471466064453,
      "activations/layer16_attention_weight_max": 32.754981994628906,
      "activations/layer16_attention_weight_min": -29.15123176574707,
      "activations/layer17_attention_weight_max": 53.49446487426758,
      "activations/layer17_attention_weight_min": -41.914024353027344,
      "activations/layer18_attention_weight_max": 46.99116516113281,
      "activations/layer18_attention_weight_min": -35.934471130371094,
      "activations/layer19_attention_weight_max": 21.110719680786133,
      "activations/layer19_attention_weight_min": -21.41084098815918,
      "activations/layer1_attention_weight_max": 16.135047912597656,
      "activations/layer1_attention_weight_min": -15.550915718078613,
      "activations/layer20_attention_weight_max": 21.15557289123535,
      "activations/layer20_attention_weight_min": -20.01324462890625,
      "activations/layer21_attention_weight_max": 38.234588623046875,
      "activations/layer21_attention_weight_min": -20.84208106994629,
      "activations/layer22_attention_weight_max": 30.879255294799805,
      "activations/layer22_attention_weight_min": -28.01594352722168,
      "activations/layer23_attention_weight_max": 34.314720153808594,
      "activations/layer23_attention_weight_min": -24.24782943725586,
      "activations/layer2_attention_weight_max": 32.67014694213867,
      "activations/layer2_attention_weight_min": -31.046669006347656,
      "activations/layer3_attention_weight_max": 86.77972412109375,
      "activations/layer3_attention_weight_min": -87.77871704101562,
      "activations/layer4_attention_weight_max": 92.16529846191406,
      "activations/layer4_attention_weight_min": -91.53507995605469,
      "activations/layer5_attention_weight_max": 68.75609588623047,
      "activations/layer5_attention_weight_min": -83.02297973632812,
      "activations/layer6_attention_weight_max": 54.058101654052734,
      "activations/layer6_attention_weight_min": -51.699913024902344,
      "activations/layer7_attention_weight_max": 71.8062973022461,
      "activations/layer7_attention_weight_min": -70.3592529296875,
      "activations/layer8_attention_weight_max": 43.55308151245117,
      "activations/layer8_attention_weight_min": -44.861785888671875,
      "activations/layer9_attention_weight_max": 44.39400863647461,
      "activations/layer9_attention_weight_min": -44.918373107910156,
      "epoch": 11.02,
      "learning_rate": 7.97405303030303e-05,
      "loss": 2.7547,
      "step": 189650
    },
    {
      "activations/layer0_attention_weight_max": 18.004762649536133,
      "activations/layer0_attention_weight_min": -13.786043167114258,
      "activations/layer10_attention_weight_max": 34.875553131103516,
      "activations/layer10_attention_weight_min": -36.60441207885742,
      "activations/layer11_attention_weight_max": 31.372230529785156,
      "activations/layer11_attention_weight_min": -32.2393798828125,
      "activations/layer12_attention_weight_max": 17.858943939208984,
      "activations/layer12_attention_weight_min": -24.72101593017578,
      "activations/layer13_attention_weight_max": 41.95958709716797,
      "activations/layer13_attention_weight_min": -29.667320251464844,
      "activations/layer14_attention_weight_max": 50.3311882019043,
      "activations/layer14_attention_weight_min": -31.811107635498047,
      "activations/layer15_attention_weight_max": 40.37422180175781,
      "activations/layer15_attention_weight_min": -30.84439468383789,
      "activations/layer16_attention_weight_max": 33.13326644897461,
      "activations/layer16_attention_weight_min": -28.5985164642334,
      "activations/layer17_attention_weight_max": 53.17277908325195,
      "activations/layer17_attention_weight_min": -44.59642028808594,
      "activations/layer18_attention_weight_max": 48.7389030456543,
      "activations/layer18_attention_weight_min": -36.58903121948242,
      "activations/layer19_attention_weight_max": 26.55777931213379,
      "activations/layer19_attention_weight_min": -22.320823669433594,
      "activations/layer1_attention_weight_max": 17.04779624938965,
      "activations/layer1_attention_weight_min": -14.456177711486816,
      "activations/layer20_attention_weight_max": 24.23334312438965,
      "activations/layer20_attention_weight_min": -21.104928970336914,
      "activations/layer21_attention_weight_max": 44.13743209838867,
      "activations/layer21_attention_weight_min": -23.16178321838379,
      "activations/layer22_attention_weight_max": 32.67957305908203,
      "activations/layer22_attention_weight_min": -24.339157104492188,
      "activations/layer23_attention_weight_max": 43.283321380615234,
      "activations/layer23_attention_weight_min": -23.218076705932617,
      "activations/layer2_attention_weight_max": 33.604339599609375,
      "activations/layer2_attention_weight_min": -31.123104095458984,
      "activations/layer3_attention_weight_max": 91.06055450439453,
      "activations/layer3_attention_weight_min": -91.88560485839844,
      "activations/layer4_attention_weight_max": 96.05987548828125,
      "activations/layer4_attention_weight_min": -93.22259521484375,
      "activations/layer5_attention_weight_max": 75.08189392089844,
      "activations/layer5_attention_weight_min": -74.62298583984375,
      "activations/layer6_attention_weight_max": 54.716304779052734,
      "activations/layer6_attention_weight_min": -51.22841262817383,
      "activations/layer7_attention_weight_max": 73.40021514892578,
      "activations/layer7_attention_weight_min": -72.82178497314453,
      "activations/layer8_attention_weight_max": 45.19430923461914,
      "activations/layer8_attention_weight_min": -47.45501708984375,
      "activations/layer9_attention_weight_max": 51.600276947021484,
      "activations/layer9_attention_weight_min": -51.11924743652344,
      "epoch": 11.02,
      "learning_rate": 7.972159090909091e-05,
      "loss": 2.7628,
      "step": 189700
    },
    {
      "activations/layer0_attention_weight_max": 16.533254623413086,
      "activations/layer0_attention_weight_min": -14.242109298706055,
      "activations/layer10_attention_weight_max": 42.07094192504883,
      "activations/layer10_attention_weight_min": -40.867958068847656,
      "activations/layer11_attention_weight_max": 41.366172790527344,
      "activations/layer11_attention_weight_min": -34.946754455566406,
      "activations/layer12_attention_weight_max": 19.241256713867188,
      "activations/layer12_attention_weight_min": -27.179582595825195,
      "activations/layer13_attention_weight_max": 36.67063522338867,
      "activations/layer13_attention_weight_min": -27.32362937927246,
      "activations/layer14_attention_weight_max": 39.36455535888672,
      "activations/layer14_attention_weight_min": -29.362945556640625,
      "activations/layer15_attention_weight_max": 35.40987777709961,
      "activations/layer15_attention_weight_min": -29.85076332092285,
      "activations/layer16_attention_weight_max": 32.467891693115234,
      "activations/layer16_attention_weight_min": -29.255708694458008,
      "activations/layer17_attention_weight_max": 55.5511360168457,
      "activations/layer17_attention_weight_min": -43.12295150756836,
      "activations/layer18_attention_weight_max": 51.06792449951172,
      "activations/layer18_attention_weight_min": -36.99746322631836,
      "activations/layer19_attention_weight_max": 23.23832893371582,
      "activations/layer19_attention_weight_min": -21.433998107910156,
      "activations/layer1_attention_weight_max": 18.468692779541016,
      "activations/layer1_attention_weight_min": -13.748563766479492,
      "activations/layer20_attention_weight_max": 22.093721389770508,
      "activations/layer20_attention_weight_min": -20.961362838745117,
      "activations/layer21_attention_weight_max": 35.424964904785156,
      "activations/layer21_attention_weight_min": -20.486204147338867,
      "activations/layer22_attention_weight_max": 30.69247817993164,
      "activations/layer22_attention_weight_min": -25.11363983154297,
      "activations/layer23_attention_weight_max": 41.70520782470703,
      "activations/layer23_attention_weight_min": -22.777023315429688,
      "activations/layer2_attention_weight_max": 31.474576950073242,
      "activations/layer2_attention_weight_min": -30.023183822631836,
      "activations/layer3_attention_weight_max": 89.2529296875,
      "activations/layer3_attention_weight_min": -95.91248321533203,
      "activations/layer4_attention_weight_max": 100.85403442382812,
      "activations/layer4_attention_weight_min": -99.32705688476562,
      "activations/layer5_attention_weight_max": 71.86146545410156,
      "activations/layer5_attention_weight_min": -75.34771728515625,
      "activations/layer6_attention_weight_max": 53.13583755493164,
      "activations/layer6_attention_weight_min": -53.36394119262695,
      "activations/layer7_attention_weight_max": 85.5607681274414,
      "activations/layer7_attention_weight_min": -73.89523315429688,
      "activations/layer8_attention_weight_max": 54.09076690673828,
      "activations/layer8_attention_weight_min": -45.69285202026367,
      "activations/layer9_attention_weight_max": 70.42857360839844,
      "activations/layer9_attention_weight_min": -53.455623626708984,
      "epoch": 11.03,
      "learning_rate": 7.97026515151515e-05,
      "loss": 2.769,
      "step": 189750
    },
    {
      "activations/layer0_attention_weight_max": 15.985671997070312,
      "activations/layer0_attention_weight_min": -13.97198486328125,
      "activations/layer10_attention_weight_max": 35.120338439941406,
      "activations/layer10_attention_weight_min": -35.12213134765625,
      "activations/layer11_attention_weight_max": 30.030941009521484,
      "activations/layer11_attention_weight_min": -32.56468200683594,
      "activations/layer12_attention_weight_max": 19.13649559020996,
      "activations/layer12_attention_weight_min": -30.53693962097168,
      "activations/layer13_attention_weight_max": 46.059444427490234,
      "activations/layer13_attention_weight_min": -31.467864990234375,
      "activations/layer14_attention_weight_max": 37.81755447387695,
      "activations/layer14_attention_weight_min": -28.586210250854492,
      "activations/layer15_attention_weight_max": 39.66105270385742,
      "activations/layer15_attention_weight_min": -28.928421020507812,
      "activations/layer16_attention_weight_max": 30.491003036499023,
      "activations/layer16_attention_weight_min": -28.387388229370117,
      "activations/layer17_attention_weight_max": 51.43377685546875,
      "activations/layer17_attention_weight_min": -42.28339767456055,
      "activations/layer18_attention_weight_max": 44.8996696472168,
      "activations/layer18_attention_weight_min": -36.47334671020508,
      "activations/layer19_attention_weight_max": 24.66495132446289,
      "activations/layer19_attention_weight_min": -22.84588050842285,
      "activations/layer1_attention_weight_max": 16.2059268951416,
      "activations/layer1_attention_weight_min": -18.258676528930664,
      "activations/layer20_attention_weight_max": 21.367326736450195,
      "activations/layer20_attention_weight_min": -24.1048641204834,
      "activations/layer21_attention_weight_max": 46.05633544921875,
      "activations/layer21_attention_weight_min": -28.92138671875,
      "activations/layer22_attention_weight_max": 29.96729278564453,
      "activations/layer22_attention_weight_min": -27.073871612548828,
      "activations/layer23_attention_weight_max": 38.16102600097656,
      "activations/layer23_attention_weight_min": -25.846385955810547,
      "activations/layer2_attention_weight_max": 32.630615234375,
      "activations/layer2_attention_weight_min": -31.98927879333496,
      "activations/layer3_attention_weight_max": 93.19440460205078,
      "activations/layer3_attention_weight_min": -98.06198120117188,
      "activations/layer4_attention_weight_max": 94.03190612792969,
      "activations/layer4_attention_weight_min": -90.57294464111328,
      "activations/layer5_attention_weight_max": 68.55699920654297,
      "activations/layer5_attention_weight_min": -72.72029876708984,
      "activations/layer6_attention_weight_max": 51.2576789855957,
      "activations/layer6_attention_weight_min": -52.25357437133789,
      "activations/layer7_attention_weight_max": 73.43635559082031,
      "activations/layer7_attention_weight_min": -70.131591796875,
      "activations/layer8_attention_weight_max": 42.0157470703125,
      "activations/layer8_attention_weight_min": -43.73207092285156,
      "activations/layer9_attention_weight_max": 46.96699142456055,
      "activations/layer9_attention_weight_min": -45.24580001831055,
      "epoch": 11.03,
      "learning_rate": 7.968371212121211e-05,
      "loss": 2.7675,
      "step": 189800
    },
    {
      "activations/layer0_attention_weight_max": 15.32442855834961,
      "activations/layer0_attention_weight_min": -14.075935363769531,
      "activations/layer10_attention_weight_max": 37.98853302001953,
      "activations/layer10_attention_weight_min": -38.50090408325195,
      "activations/layer11_attention_weight_max": 34.17982482910156,
      "activations/layer11_attention_weight_min": -32.47109603881836,
      "activations/layer12_attention_weight_max": 18.477033615112305,
      "activations/layer12_attention_weight_min": -25.05176544189453,
      "activations/layer13_attention_weight_max": 48.47354507446289,
      "activations/layer13_attention_weight_min": -31.836872100830078,
      "activations/layer14_attention_weight_max": 40.61288833618164,
      "activations/layer14_attention_weight_min": -30.969707489013672,
      "activations/layer15_attention_weight_max": 34.68503952026367,
      "activations/layer15_attention_weight_min": -29.51896095275879,
      "activations/layer16_attention_weight_max": 31.09099578857422,
      "activations/layer16_attention_weight_min": -29.945491790771484,
      "activations/layer17_attention_weight_max": 55.51042556762695,
      "activations/layer17_attention_weight_min": -45.2762451171875,
      "activations/layer18_attention_weight_max": 49.05667495727539,
      "activations/layer18_attention_weight_min": -40.01350021362305,
      "activations/layer19_attention_weight_max": 22.87958335876465,
      "activations/layer19_attention_weight_min": -24.513702392578125,
      "activations/layer1_attention_weight_max": 19.462812423706055,
      "activations/layer1_attention_weight_min": -18.058456420898438,
      "activations/layer20_attention_weight_max": 22.611042022705078,
      "activations/layer20_attention_weight_min": -22.097454071044922,
      "activations/layer21_attention_weight_max": 38.22903823852539,
      "activations/layer21_attention_weight_min": -22.66773796081543,
      "activations/layer22_attention_weight_max": 31.984615325927734,
      "activations/layer22_attention_weight_min": -25.399568557739258,
      "activations/layer23_attention_weight_max": 38.532012939453125,
      "activations/layer23_attention_weight_min": -22.360424041748047,
      "activations/layer2_attention_weight_max": 33.86851501464844,
      "activations/layer2_attention_weight_min": -32.032012939453125,
      "activations/layer3_attention_weight_max": 89.9725341796875,
      "activations/layer3_attention_weight_min": -92.6050033569336,
      "activations/layer4_attention_weight_max": 94.40779876708984,
      "activations/layer4_attention_weight_min": -96.12315368652344,
      "activations/layer5_attention_weight_max": 69.72444152832031,
      "activations/layer5_attention_weight_min": -73.76992797851562,
      "activations/layer6_attention_weight_max": 53.80937957763672,
      "activations/layer6_attention_weight_min": -50.970985412597656,
      "activations/layer7_attention_weight_max": 81.1053466796875,
      "activations/layer7_attention_weight_min": -76.9254150390625,
      "activations/layer8_attention_weight_max": 49.105281829833984,
      "activations/layer8_attention_weight_min": -47.17119598388672,
      "activations/layer9_attention_weight_max": 60.50132369995117,
      "activations/layer9_attention_weight_min": -50.74553680419922,
      "epoch": 11.03,
      "learning_rate": 7.966477272727273e-05,
      "loss": 2.7694,
      "step": 189850
    },
    {
      "activations/layer0_attention_weight_max": 16.178537368774414,
      "activations/layer0_attention_weight_min": -14.303376197814941,
      "activations/layer10_attention_weight_max": 43.644775390625,
      "activations/layer10_attention_weight_min": -40.9720573425293,
      "activations/layer11_attention_weight_max": 35.82152557373047,
      "activations/layer11_attention_weight_min": -35.11711120605469,
      "activations/layer12_attention_weight_max": 23.020456314086914,
      "activations/layer12_attention_weight_min": -29.623695373535156,
      "activations/layer13_attention_weight_max": 41.54817199707031,
      "activations/layer13_attention_weight_min": -30.864437103271484,
      "activations/layer14_attention_weight_max": 36.13047790527344,
      "activations/layer14_attention_weight_min": -30.178295135498047,
      "activations/layer15_attention_weight_max": 34.1658821105957,
      "activations/layer15_attention_weight_min": -29.39317512512207,
      "activations/layer16_attention_weight_max": 30.233287811279297,
      "activations/layer16_attention_weight_min": -28.604082107543945,
      "activations/layer17_attention_weight_max": 54.588768005371094,
      "activations/layer17_attention_weight_min": -43.30973434448242,
      "activations/layer18_attention_weight_max": 47.69758987426758,
      "activations/layer18_attention_weight_min": -37.45582962036133,
      "activations/layer19_attention_weight_max": 22.31722068786621,
      "activations/layer19_attention_weight_min": -21.18897247314453,
      "activations/layer1_attention_weight_max": 16.949216842651367,
      "activations/layer1_attention_weight_min": -15.841567993164062,
      "activations/layer20_attention_weight_max": 23.179922103881836,
      "activations/layer20_attention_weight_min": -20.348819732666016,
      "activations/layer21_attention_weight_max": 36.673519134521484,
      "activations/layer21_attention_weight_min": -22.770864486694336,
      "activations/layer22_attention_weight_max": 32.868812561035156,
      "activations/layer22_attention_weight_min": -24.609731674194336,
      "activations/layer23_attention_weight_max": 35.43365478515625,
      "activations/layer23_attention_weight_min": -22.95177459716797,
      "activations/layer2_attention_weight_max": 33.32731628417969,
      "activations/layer2_attention_weight_min": -30.668445587158203,
      "activations/layer3_attention_weight_max": 95.09539794921875,
      "activations/layer3_attention_weight_min": -95.59495544433594,
      "activations/layer4_attention_weight_max": 100.84197235107422,
      "activations/layer4_attention_weight_min": -97.20449829101562,
      "activations/layer5_attention_weight_max": 72.13400268554688,
      "activations/layer5_attention_weight_min": -75.39445495605469,
      "activations/layer6_attention_weight_max": 55.323570251464844,
      "activations/layer6_attention_weight_min": -53.42795181274414,
      "activations/layer7_attention_weight_max": 84.69808197021484,
      "activations/layer7_attention_weight_min": -76.62244415283203,
      "activations/layer8_attention_weight_max": 52.971275329589844,
      "activations/layer8_attention_weight_min": -52.09293746948242,
      "activations/layer9_attention_weight_max": 53.49516677856445,
      "activations/layer9_attention_weight_min": -51.18412780761719,
      "epoch": 11.03,
      "learning_rate": 7.964583333333332e-05,
      "loss": 2.7807,
      "step": 189900
    },
    {
      "activations/layer0_attention_weight_max": 15.273002624511719,
      "activations/layer0_attention_weight_min": -14.102965354919434,
      "activations/layer10_attention_weight_max": 38.030372619628906,
      "activations/layer10_attention_weight_min": -37.96773147583008,
      "activations/layer11_attention_weight_max": 31.761388778686523,
      "activations/layer11_attention_weight_min": -33.190650939941406,
      "activations/layer12_attention_weight_max": 16.72036361694336,
      "activations/layer12_attention_weight_min": -24.63500213623047,
      "activations/layer13_attention_weight_max": 36.886932373046875,
      "activations/layer13_attention_weight_min": -31.879112243652344,
      "activations/layer14_attention_weight_max": 35.19398880004883,
      "activations/layer14_attention_weight_min": -31.580989837646484,
      "activations/layer15_attention_weight_max": 31.890506744384766,
      "activations/layer15_attention_weight_min": -30.743663787841797,
      "activations/layer16_attention_weight_max": 29.419492721557617,
      "activations/layer16_attention_weight_min": -30.400753021240234,
      "activations/layer17_attention_weight_max": 50.41563034057617,
      "activations/layer17_attention_weight_min": -46.445369720458984,
      "activations/layer18_attention_weight_max": 48.04475021362305,
      "activations/layer18_attention_weight_min": -38.02882385253906,
      "activations/layer19_attention_weight_max": 21.35003089904785,
      "activations/layer19_attention_weight_min": -21.31009864807129,
      "activations/layer1_attention_weight_max": 15.947771072387695,
      "activations/layer1_attention_weight_min": -14.310791015625,
      "activations/layer20_attention_weight_max": 21.750003814697266,
      "activations/layer20_attention_weight_min": -26.174962997436523,
      "activations/layer21_attention_weight_max": 36.38087844848633,
      "activations/layer21_attention_weight_min": -24.740262985229492,
      "activations/layer22_attention_weight_max": 30.59684944152832,
      "activations/layer22_attention_weight_min": -28.62601661682129,
      "activations/layer23_attention_weight_max": 39.18744659423828,
      "activations/layer23_attention_weight_min": -24.298490524291992,
      "activations/layer2_attention_weight_max": 31.243385314941406,
      "activations/layer2_attention_weight_min": -30.72185516357422,
      "activations/layer3_attention_weight_max": 94.73883819580078,
      "activations/layer3_attention_weight_min": -94.40129089355469,
      "activations/layer4_attention_weight_max": 98.51642608642578,
      "activations/layer4_attention_weight_min": -95.47566223144531,
      "activations/layer5_attention_weight_max": 72.21729278564453,
      "activations/layer5_attention_weight_min": -75.46878051757812,
      "activations/layer6_attention_weight_max": 53.86023712158203,
      "activations/layer6_attention_weight_min": -54.88266372680664,
      "activations/layer7_attention_weight_max": 79.37325286865234,
      "activations/layer7_attention_weight_min": -73.10499572753906,
      "activations/layer8_attention_weight_max": 46.31079864501953,
      "activations/layer8_attention_weight_min": -46.26302719116211,
      "activations/layer9_attention_weight_max": 46.94778060913086,
      "activations/layer9_attention_weight_min": -45.488853454589844,
      "epoch": 11.04,
      "learning_rate": 7.962689393939393e-05,
      "loss": 2.7647,
      "step": 189950
    },
    {
      "activations/layer0_attention_weight_max": 15.380110740661621,
      "activations/layer0_attention_weight_min": -14.406645774841309,
      "activations/layer10_attention_weight_max": 37.48860549926758,
      "activations/layer10_attention_weight_min": -35.90317916870117,
      "activations/layer11_attention_weight_max": 32.34742736816406,
      "activations/layer11_attention_weight_min": -34.92202377319336,
      "activations/layer12_attention_weight_max": 19.367958068847656,
      "activations/layer12_attention_weight_min": -26.120939254760742,
      "activations/layer13_attention_weight_max": 37.50311279296875,
      "activations/layer13_attention_weight_min": -35.05620574951172,
      "activations/layer14_attention_weight_max": 37.9774055480957,
      "activations/layer14_attention_weight_min": -33.0491828918457,
      "activations/layer15_attention_weight_max": 33.038700103759766,
      "activations/layer15_attention_weight_min": -32.36643981933594,
      "activations/layer16_attention_weight_max": 32.316551208496094,
      "activations/layer16_attention_weight_min": -30.244983673095703,
      "activations/layer17_attention_weight_max": 52.800865173339844,
      "activations/layer17_attention_weight_min": -46.5308837890625,
      "activations/layer18_attention_weight_max": 46.68932342529297,
      "activations/layer18_attention_weight_min": -38.31846237182617,
      "activations/layer19_attention_weight_max": 20.38623809814453,
      "activations/layer19_attention_weight_min": -22.025861740112305,
      "activations/layer1_attention_weight_max": 16.960718154907227,
      "activations/layer1_attention_weight_min": -16.992900848388672,
      "activations/layer20_attention_weight_max": 23.153108596801758,
      "activations/layer20_attention_weight_min": -21.095014572143555,
      "activations/layer21_attention_weight_max": 35.24235916137695,
      "activations/layer21_attention_weight_min": -24.33121681213379,
      "activations/layer22_attention_weight_max": 27.300962448120117,
      "activations/layer22_attention_weight_min": -26.169477462768555,
      "activations/layer23_attention_weight_max": 36.90003204345703,
      "activations/layer23_attention_weight_min": -21.801748275756836,
      "activations/layer2_attention_weight_max": 32.29359817504883,
      "activations/layer2_attention_weight_min": -31.50328826904297,
      "activations/layer3_attention_weight_max": 91.31493377685547,
      "activations/layer3_attention_weight_min": -94.4103012084961,
      "activations/layer4_attention_weight_max": 100.77931213378906,
      "activations/layer4_attention_weight_min": -94.64855194091797,
      "activations/layer5_attention_weight_max": 70.837158203125,
      "activations/layer5_attention_weight_min": -78.88020324707031,
      "activations/layer6_attention_weight_max": 55.003662109375,
      "activations/layer6_attention_weight_min": -56.04052734375,
      "activations/layer7_attention_weight_max": 76.14904022216797,
      "activations/layer7_attention_weight_min": -73.87399291992188,
      "activations/layer8_attention_weight_max": 46.532283782958984,
      "activations/layer8_attention_weight_min": -46.34779739379883,
      "activations/layer9_attention_weight_max": 48.004146575927734,
      "activations/layer9_attention_weight_min": -47.97569274902344,
      "epoch": 11.04,
      "learning_rate": 7.960795454545455e-05,
      "loss": 2.7695,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.4664,
      "eval_samples_per_second": 507.178,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.4664,
      "eval_openwebtext_samples_per_second": 507.178,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.9826,
      "eval_wikitext_samples_per_second": 230.005,
      "step": 190000
    },
    {
      "epoch": 11.04,
      "eval_lambada_loss": 2.5546875,
      "eval_lambada_ppl": 12.867278004795136,
      "eval_lambada_runtime": 9.5372,
      "eval_lambada_samples_per_second": 510.526,
      "step": 190000
    },
    {
      "activations/layer0_attention_weight_max": 16.999475479125977,
      "activations/layer0_attention_weight_min": -13.61032772064209,
      "activations/layer10_attention_weight_max": 36.579322814941406,
      "activations/layer10_attention_weight_min": -36.60123062133789,
      "activations/layer11_attention_weight_max": 32.69990921020508,
      "activations/layer11_attention_weight_min": -33.470306396484375,
      "activations/layer12_attention_weight_max": 18.66618537902832,
      "activations/layer12_attention_weight_min": -24.723703384399414,
      "activations/layer13_attention_weight_max": 39.6944465637207,
      "activations/layer13_attention_weight_min": -29.455162048339844,
      "activations/layer14_attention_weight_max": 38.512847900390625,
      "activations/layer14_attention_weight_min": -30.86448097229004,
      "activations/layer15_attention_weight_max": 34.63838577270508,
      "activations/layer15_attention_weight_min": -28.872501373291016,
      "activations/layer16_attention_weight_max": 31.161466598510742,
      "activations/layer16_attention_weight_min": -29.357009887695312,
      "activations/layer17_attention_weight_max": 53.88935470581055,
      "activations/layer17_attention_weight_min": -43.74936294555664,
      "activations/layer18_attention_weight_max": 49.4735221862793,
      "activations/layer18_attention_weight_min": -38.30052185058594,
      "activations/layer19_attention_weight_max": 22.777929306030273,
      "activations/layer19_attention_weight_min": -22.455732345581055,
      "activations/layer1_attention_weight_max": 17.02605628967285,
      "activations/layer1_attention_weight_min": -14.814127922058105,
      "activations/layer20_attention_weight_max": 24.09342384338379,
      "activations/layer20_attention_weight_min": -23.693099975585938,
      "activations/layer21_attention_weight_max": 38.12561798095703,
      "activations/layer21_attention_weight_min": -22.95126724243164,
      "activations/layer22_attention_weight_max": 33.00121307373047,
      "activations/layer22_attention_weight_min": -24.60125160217285,
      "activations/layer23_attention_weight_max": 39.33465576171875,
      "activations/layer23_attention_weight_min": -24.704998016357422,
      "activations/layer2_attention_weight_max": 32.53498077392578,
      "activations/layer2_attention_weight_min": -30.88693618774414,
      "activations/layer3_attention_weight_max": 86.94204711914062,
      "activations/layer3_attention_weight_min": -90.10234832763672,
      "activations/layer4_attention_weight_max": 98.39704895019531,
      "activations/layer4_attention_weight_min": -94.6636734008789,
      "activations/layer5_attention_weight_max": 72.53668212890625,
      "activations/layer5_attention_weight_min": -76.6065444946289,
      "activations/layer6_attention_weight_max": 53.452415466308594,
      "activations/layer6_attention_weight_min": -53.24900436401367,
      "activations/layer7_attention_weight_max": 75.79054260253906,
      "activations/layer7_attention_weight_min": -72.0391616821289,
      "activations/layer8_attention_weight_max": 46.80488967895508,
      "activations/layer8_attention_weight_min": -48.04741668701172,
      "activations/layer9_attention_weight_max": 50.36417770385742,
      "activations/layer9_attention_weight_min": -48.30879592895508,
      "epoch": 11.04,
      "learning_rate": 7.958901515151513e-05,
      "loss": 2.7757,
      "step": 190050
    },
    {
      "activations/layer0_attention_weight_max": 16.38569450378418,
      "activations/layer0_attention_weight_min": -13.891546249389648,
      "activations/layer10_attention_weight_max": 36.98387145996094,
      "activations/layer10_attention_weight_min": -36.53480911254883,
      "activations/layer11_attention_weight_max": 30.048763275146484,
      "activations/layer11_attention_weight_min": -31.69931411743164,
      "activations/layer12_attention_weight_max": 19.74982452392578,
      "activations/layer12_attention_weight_min": -25.2502384185791,
      "activations/layer13_attention_weight_max": 37.80836868286133,
      "activations/layer13_attention_weight_min": -28.480392456054688,
      "activations/layer14_attention_weight_max": 39.21467971801758,
      "activations/layer14_attention_weight_min": -31.990257263183594,
      "activations/layer15_attention_weight_max": 33.45830535888672,
      "activations/layer15_attention_weight_min": -31.515451431274414,
      "activations/layer16_attention_weight_max": 29.684213638305664,
      "activations/layer16_attention_weight_min": -28.49112319946289,
      "activations/layer17_attention_weight_max": 49.458457946777344,
      "activations/layer17_attention_weight_min": -43.14769744873047,
      "activations/layer18_attention_weight_max": 45.347129821777344,
      "activations/layer18_attention_weight_min": -37.50490951538086,
      "activations/layer19_attention_weight_max": 21.73567771911621,
      "activations/layer19_attention_weight_min": -20.744861602783203,
      "activations/layer1_attention_weight_max": 16.16786766052246,
      "activations/layer1_attention_weight_min": -14.673294067382812,
      "activations/layer20_attention_weight_max": 23.71556282043457,
      "activations/layer20_attention_weight_min": -22.39012908935547,
      "activations/layer21_attention_weight_max": 39.109901428222656,
      "activations/layer21_attention_weight_min": -21.758224487304688,
      "activations/layer22_attention_weight_max": 29.322656631469727,
      "activations/layer22_attention_weight_min": -26.76224708557129,
      "activations/layer23_attention_weight_max": 42.06007766723633,
      "activations/layer23_attention_weight_min": -26.602664947509766,
      "activations/layer2_attention_weight_max": 33.18329620361328,
      "activations/layer2_attention_weight_min": -32.231353759765625,
      "activations/layer3_attention_weight_max": 90.61766052246094,
      "activations/layer3_attention_weight_min": -90.48930358886719,
      "activations/layer4_attention_weight_max": 94.09957122802734,
      "activations/layer4_attention_weight_min": -92.1333999633789,
      "activations/layer5_attention_weight_max": 73.64683532714844,
      "activations/layer5_attention_weight_min": -79.94859313964844,
      "activations/layer6_attention_weight_max": 51.86927032470703,
      "activations/layer6_attention_weight_min": -50.73157501220703,
      "activations/layer7_attention_weight_max": 72.90825653076172,
      "activations/layer7_attention_weight_min": -71.01033782958984,
      "activations/layer8_attention_weight_max": 44.40340042114258,
      "activations/layer8_attention_weight_min": -45.21125411987305,
      "activations/layer9_attention_weight_max": 47.75398635864258,
      "activations/layer9_attention_weight_min": -46.842079162597656,
      "epoch": 11.05,
      "learning_rate": 7.957007575757575e-05,
      "loss": 2.7698,
      "step": 190100
    },
    {
      "activations/layer0_attention_weight_max": 15.439702033996582,
      "activations/layer0_attention_weight_min": -13.553991317749023,
      "activations/layer10_attention_weight_max": 34.99655532836914,
      "activations/layer10_attention_weight_min": -35.362403869628906,
      "activations/layer11_attention_weight_max": 30.00555419921875,
      "activations/layer11_attention_weight_min": -32.274566650390625,
      "activations/layer12_attention_weight_max": 18.89943504333496,
      "activations/layer12_attention_weight_min": -24.168216705322266,
      "activations/layer13_attention_weight_max": 41.19936752319336,
      "activations/layer13_attention_weight_min": -32.312583923339844,
      "activations/layer14_attention_weight_max": 37.527339935302734,
      "activations/layer14_attention_weight_min": -30.26715660095215,
      "activations/layer15_attention_weight_max": 32.83286666870117,
      "activations/layer15_attention_weight_min": -30.521860122680664,
      "activations/layer16_attention_weight_max": 28.400489807128906,
      "activations/layer16_attention_weight_min": -28.040782928466797,
      "activations/layer17_attention_weight_max": 49.5004768371582,
      "activations/layer17_attention_weight_min": -43.382423400878906,
      "activations/layer18_attention_weight_max": 42.97059631347656,
      "activations/layer18_attention_weight_min": -38.3946418762207,
      "activations/layer19_attention_weight_max": 21.164417266845703,
      "activations/layer19_attention_weight_min": -22.26203727722168,
      "activations/layer1_attention_weight_max": 17.324514389038086,
      "activations/layer1_attention_weight_min": -14.995614051818848,
      "activations/layer20_attention_weight_max": 21.354534149169922,
      "activations/layer20_attention_weight_min": -21.678951263427734,
      "activations/layer21_attention_weight_max": 35.82551193237305,
      "activations/layer21_attention_weight_min": -27.557205200195312,
      "activations/layer22_attention_weight_max": 30.264373779296875,
      "activations/layer22_attention_weight_min": -27.39906120300293,
      "activations/layer23_attention_weight_max": 39.785980224609375,
      "activations/layer23_attention_weight_min": -29.963088989257812,
      "activations/layer2_attention_weight_max": 33.545433044433594,
      "activations/layer2_attention_weight_min": -32.02656555175781,
      "activations/layer3_attention_weight_max": 92.81612396240234,
      "activations/layer3_attention_weight_min": -94.7691879272461,
      "activations/layer4_attention_weight_max": 98.96487426757812,
      "activations/layer4_attention_weight_min": -92.1915054321289,
      "activations/layer5_attention_weight_max": 72.88996887207031,
      "activations/layer5_attention_weight_min": -76.96734619140625,
      "activations/layer6_attention_weight_max": 55.035667419433594,
      "activations/layer6_attention_weight_min": -55.436676025390625,
      "activations/layer7_attention_weight_max": 77.75469207763672,
      "activations/layer7_attention_weight_min": -70.09461975097656,
      "activations/layer8_attention_weight_max": 44.67731475830078,
      "activations/layer8_attention_weight_min": -44.594356536865234,
      "activations/layer9_attention_weight_max": 49.04847717285156,
      "activations/layer9_attention_weight_min": -47.366695404052734,
      "epoch": 11.05,
      "learning_rate": 7.955113636363636e-05,
      "loss": 2.7895,
      "step": 190150
    },
    {
      "activations/layer0_attention_weight_max": 16.164222717285156,
      "activations/layer0_attention_weight_min": -13.96208381652832,
      "activations/layer10_attention_weight_max": 38.88090133666992,
      "activations/layer10_attention_weight_min": -40.699737548828125,
      "activations/layer11_attention_weight_max": 36.83218765258789,
      "activations/layer11_attention_weight_min": -36.39068603515625,
      "activations/layer12_attention_weight_max": 18.360700607299805,
      "activations/layer12_attention_weight_min": -26.975126266479492,
      "activations/layer13_attention_weight_max": 37.98031234741211,
      "activations/layer13_attention_weight_min": -36.75584411621094,
      "activations/layer14_attention_weight_max": 37.473854064941406,
      "activations/layer14_attention_weight_min": -30.894962310791016,
      "activations/layer15_attention_weight_max": 35.330753326416016,
      "activations/layer15_attention_weight_min": -30.11562156677246,
      "activations/layer16_attention_weight_max": 32.19843292236328,
      "activations/layer16_attention_weight_min": -31.79433250427246,
      "activations/layer17_attention_weight_max": 52.794769287109375,
      "activations/layer17_attention_weight_min": -47.29966354370117,
      "activations/layer18_attention_weight_max": 49.27426528930664,
      "activations/layer18_attention_weight_min": -38.85042190551758,
      "activations/layer19_attention_weight_max": 22.925777435302734,
      "activations/layer19_attention_weight_min": -21.900510787963867,
      "activations/layer1_attention_weight_max": 16.60082244873047,
      "activations/layer1_attention_weight_min": -15.719265937805176,
      "activations/layer20_attention_weight_max": 22.795682907104492,
      "activations/layer20_attention_weight_min": -21.560991287231445,
      "activations/layer21_attention_weight_max": 35.01800537109375,
      "activations/layer21_attention_weight_min": -22.657577514648438,
      "activations/layer22_attention_weight_max": 30.713668823242188,
      "activations/layer22_attention_weight_min": -26.397294998168945,
      "activations/layer23_attention_weight_max": 41.11365509033203,
      "activations/layer23_attention_weight_min": -23.37567901611328,
      "activations/layer2_attention_weight_max": 33.34287643432617,
      "activations/layer2_attention_weight_min": -32.099571228027344,
      "activations/layer3_attention_weight_max": 93.35311126708984,
      "activations/layer3_attention_weight_min": -95.49915313720703,
      "activations/layer4_attention_weight_max": 99.76081085205078,
      "activations/layer4_attention_weight_min": -95.70785522460938,
      "activations/layer5_attention_weight_max": 76.20343780517578,
      "activations/layer5_attention_weight_min": -79.08108520507812,
      "activations/layer6_attention_weight_max": 55.88446044921875,
      "activations/layer6_attention_weight_min": -54.09199142456055,
      "activations/layer7_attention_weight_max": 80.59146881103516,
      "activations/layer7_attention_weight_min": -77.34614562988281,
      "activations/layer8_attention_weight_max": 53.26054763793945,
      "activations/layer8_attention_weight_min": -54.87318420410156,
      "activations/layer9_attention_weight_max": 64.7823715209961,
      "activations/layer9_attention_weight_min": -53.900550842285156,
      "epoch": 11.05,
      "learning_rate": 7.953219696969697e-05,
      "loss": 2.776,
      "step": 190200
    },
    {
      "activations/layer0_attention_weight_max": 15.688114166259766,
      "activations/layer0_attention_weight_min": -13.64173698425293,
      "activations/layer10_attention_weight_max": 38.493621826171875,
      "activations/layer10_attention_weight_min": -37.30296325683594,
      "activations/layer11_attention_weight_max": 33.11742401123047,
      "activations/layer11_attention_weight_min": -32.43938064575195,
      "activations/layer12_attention_weight_max": 21.040021896362305,
      "activations/layer12_attention_weight_min": -24.93999671936035,
      "activations/layer13_attention_weight_max": 35.15060043334961,
      "activations/layer13_attention_weight_min": -29.166927337646484,
      "activations/layer14_attention_weight_max": 33.615455627441406,
      "activations/layer14_attention_weight_min": -29.420907974243164,
      "activations/layer15_attention_weight_max": 31.10188865661621,
      "activations/layer15_attention_weight_min": -28.74641990661621,
      "activations/layer16_attention_weight_max": 29.14405059814453,
      "activations/layer16_attention_weight_min": -27.422163009643555,
      "activations/layer17_attention_weight_max": 50.57429504394531,
      "activations/layer17_attention_weight_min": -42.43431091308594,
      "activations/layer18_attention_weight_max": 45.59703063964844,
      "activations/layer18_attention_weight_min": -36.4445686340332,
      "activations/layer19_attention_weight_max": 20.114856719970703,
      "activations/layer19_attention_weight_min": -20.355384826660156,
      "activations/layer1_attention_weight_max": 17.337074279785156,
      "activations/layer1_attention_weight_min": -14.192705154418945,
      "activations/layer20_attention_weight_max": 21.603769302368164,
      "activations/layer20_attention_weight_min": -22.79979133605957,
      "activations/layer21_attention_weight_max": 34.17818069458008,
      "activations/layer21_attention_weight_min": -20.964981079101562,
      "activations/layer22_attention_weight_max": 32.29960632324219,
      "activations/layer22_attention_weight_min": -27.66119956970215,
      "activations/layer23_attention_weight_max": 38.58000183105469,
      "activations/layer23_attention_weight_min": -24.723121643066406,
      "activations/layer2_attention_weight_max": 34.322471618652344,
      "activations/layer2_attention_weight_min": -32.450679779052734,
      "activations/layer3_attention_weight_max": 97.23883819580078,
      "activations/layer3_attention_weight_min": -98.33946990966797,
      "activations/layer4_attention_weight_max": 100.14173126220703,
      "activations/layer4_attention_weight_min": -95.55513763427734,
      "activations/layer5_attention_weight_max": 70.25617218017578,
      "activations/layer5_attention_weight_min": -75.96257019042969,
      "activations/layer6_attention_weight_max": 50.7705192565918,
      "activations/layer6_attention_weight_min": -55.476234436035156,
      "activations/layer7_attention_weight_max": 73.2420425415039,
      "activations/layer7_attention_weight_min": -70.04261779785156,
      "activations/layer8_attention_weight_max": 44.061607360839844,
      "activations/layer8_attention_weight_min": -46.42863464355469,
      "activations/layer9_attention_weight_max": 51.33856201171875,
      "activations/layer9_attention_weight_min": -46.67049789428711,
      "epoch": 11.05,
      "learning_rate": 7.951325757575757e-05,
      "loss": 2.7638,
      "step": 190250
    },
    {
      "activations/layer0_attention_weight_max": 16.437637329101562,
      "activations/layer0_attention_weight_min": -14.289582252502441,
      "activations/layer10_attention_weight_max": 36.010353088378906,
      "activations/layer10_attention_weight_min": -34.7628059387207,
      "activations/layer11_attention_weight_max": 31.146907806396484,
      "activations/layer11_attention_weight_min": -31.47450065612793,
      "activations/layer12_attention_weight_max": 21.49199104309082,
      "activations/layer12_attention_weight_min": -24.957645416259766,
      "activations/layer13_attention_weight_max": 50.68550491333008,
      "activations/layer13_attention_weight_min": -30.74803924560547,
      "activations/layer14_attention_weight_max": 40.7424430847168,
      "activations/layer14_attention_weight_min": -30.841588973999023,
      "activations/layer15_attention_weight_max": 35.46761703491211,
      "activations/layer15_attention_weight_min": -30.181482315063477,
      "activations/layer16_attention_weight_max": 32.24319839477539,
      "activations/layer16_attention_weight_min": -28.546476364135742,
      "activations/layer17_attention_weight_max": 51.1880989074707,
      "activations/layer17_attention_weight_min": -43.673606872558594,
      "activations/layer18_attention_weight_max": 46.77915954589844,
      "activations/layer18_attention_weight_min": -39.522003173828125,
      "activations/layer19_attention_weight_max": 22.076019287109375,
      "activations/layer19_attention_weight_min": -22.24886131286621,
      "activations/layer1_attention_weight_max": 17.586095809936523,
      "activations/layer1_attention_weight_min": -15.455913543701172,
      "activations/layer20_attention_weight_max": 21.555280685424805,
      "activations/layer20_attention_weight_min": -22.10915756225586,
      "activations/layer21_attention_weight_max": 38.502254486083984,
      "activations/layer21_attention_weight_min": -23.67996597290039,
      "activations/layer22_attention_weight_max": 33.02752685546875,
      "activations/layer22_attention_weight_min": -26.47346305847168,
      "activations/layer23_attention_weight_max": 38.765342712402344,
      "activations/layer23_attention_weight_min": -23.202598571777344,
      "activations/layer2_attention_weight_max": 31.72621726989746,
      "activations/layer2_attention_weight_min": -31.99684715270996,
      "activations/layer3_attention_weight_max": 94.62165069580078,
      "activations/layer3_attention_weight_min": -97.24058532714844,
      "activations/layer4_attention_weight_max": 98.70574188232422,
      "activations/layer4_attention_weight_min": -96.25874328613281,
      "activations/layer5_attention_weight_max": 72.21226501464844,
      "activations/layer5_attention_weight_min": -77.743896484375,
      "activations/layer6_attention_weight_max": 52.405887603759766,
      "activations/layer6_attention_weight_min": -51.6961555480957,
      "activations/layer7_attention_weight_max": 76.32846069335938,
      "activations/layer7_attention_weight_min": -75.09152221679688,
      "activations/layer8_attention_weight_max": 45.21992874145508,
      "activations/layer8_attention_weight_min": -47.4320068359375,
      "activations/layer9_attention_weight_max": 47.3660774230957,
      "activations/layer9_attention_weight_min": -49.36485290527344,
      "epoch": 11.06,
      "learning_rate": 7.949431818181817e-05,
      "loss": 2.7682,
      "step": 190300
    },
    {
      "activations/layer0_attention_weight_max": 15.791391372680664,
      "activations/layer0_attention_weight_min": -13.863264083862305,
      "activations/layer10_attention_weight_max": 39.07924270629883,
      "activations/layer10_attention_weight_min": -36.16778564453125,
      "activations/layer11_attention_weight_max": 34.11244201660156,
      "activations/layer11_attention_weight_min": -34.9563102722168,
      "activations/layer12_attention_weight_max": 20.581708908081055,
      "activations/layer12_attention_weight_min": -22.4479923248291,
      "activations/layer13_attention_weight_max": 46.011051177978516,
      "activations/layer13_attention_weight_min": -28.68834114074707,
      "activations/layer14_attention_weight_max": 43.15300369262695,
      "activations/layer14_attention_weight_min": -29.676259994506836,
      "activations/layer15_attention_weight_max": 35.248863220214844,
      "activations/layer15_attention_weight_min": -29.41775894165039,
      "activations/layer16_attention_weight_max": 31.388032913208008,
      "activations/layer16_attention_weight_min": -29.329607009887695,
      "activations/layer17_attention_weight_max": 54.88240051269531,
      "activations/layer17_attention_weight_min": -42.840877532958984,
      "activations/layer18_attention_weight_max": 50.62897491455078,
      "activations/layer18_attention_weight_min": -37.31309509277344,
      "activations/layer19_attention_weight_max": 24.444591522216797,
      "activations/layer19_attention_weight_min": -22.47005844116211,
      "activations/layer1_attention_weight_max": 18.639751434326172,
      "activations/layer1_attention_weight_min": -16.015871047973633,
      "activations/layer20_attention_weight_max": 25.174604415893555,
      "activations/layer20_attention_weight_min": -22.681798934936523,
      "activations/layer21_attention_weight_max": 41.357452392578125,
      "activations/layer21_attention_weight_min": -22.881330490112305,
      "activations/layer22_attention_weight_max": 32.82927322387695,
      "activations/layer22_attention_weight_min": -26.424320220947266,
      "activations/layer23_attention_weight_max": 42.09800720214844,
      "activations/layer23_attention_weight_min": -24.7938232421875,
      "activations/layer2_attention_weight_max": 33.33063888549805,
      "activations/layer2_attention_weight_min": -32.02940368652344,
      "activations/layer3_attention_weight_max": 97.90228271484375,
      "activations/layer3_attention_weight_min": -100.3249282836914,
      "activations/layer4_attention_weight_max": 101.56400299072266,
      "activations/layer4_attention_weight_min": -97.95708465576172,
      "activations/layer5_attention_weight_max": 70.63432312011719,
      "activations/layer5_attention_weight_min": -76.35385131835938,
      "activations/layer6_attention_weight_max": 53.014381408691406,
      "activations/layer6_attention_weight_min": -52.4854850769043,
      "activations/layer7_attention_weight_max": 75.6044692993164,
      "activations/layer7_attention_weight_min": -69.70845794677734,
      "activations/layer8_attention_weight_max": 46.80952453613281,
      "activations/layer8_attention_weight_min": -48.69998550415039,
      "activations/layer9_attention_weight_max": 50.227882385253906,
      "activations/layer9_attention_weight_min": -46.81711196899414,
      "epoch": 11.06,
      "learning_rate": 7.947537878787878e-05,
      "loss": 2.7592,
      "step": 190350
    },
    {
      "activations/layer0_attention_weight_max": 15.23040771484375,
      "activations/layer0_attention_weight_min": -13.70507526397705,
      "activations/layer10_attention_weight_max": 36.52726745605469,
      "activations/layer10_attention_weight_min": -36.18666076660156,
      "activations/layer11_attention_weight_max": 31.522258758544922,
      "activations/layer11_attention_weight_min": -32.087493896484375,
      "activations/layer12_attention_weight_max": 17.116525650024414,
      "activations/layer12_attention_weight_min": -24.19414710998535,
      "activations/layer13_attention_weight_max": 41.49158477783203,
      "activations/layer13_attention_weight_min": -32.13351058959961,
      "activations/layer14_attention_weight_max": 35.27682113647461,
      "activations/layer14_attention_weight_min": -30.519567489624023,
      "activations/layer15_attention_weight_max": 33.15176773071289,
      "activations/layer15_attention_weight_min": -30.939069747924805,
      "activations/layer16_attention_weight_max": 30.998722076416016,
      "activations/layer16_attention_weight_min": -29.070018768310547,
      "activations/layer17_attention_weight_max": 51.864864349365234,
      "activations/layer17_attention_weight_min": -42.52263259887695,
      "activations/layer18_attention_weight_max": 48.0742073059082,
      "activations/layer18_attention_weight_min": -35.715415954589844,
      "activations/layer19_attention_weight_max": 21.512821197509766,
      "activations/layer19_attention_weight_min": -20.764816284179688,
      "activations/layer1_attention_weight_max": 16.441974639892578,
      "activations/layer1_attention_weight_min": -13.954517364501953,
      "activations/layer20_attention_weight_max": 21.983917236328125,
      "activations/layer20_attention_weight_min": -21.14594841003418,
      "activations/layer21_attention_weight_max": 36.22461700439453,
      "activations/layer21_attention_weight_min": -26.42104148864746,
      "activations/layer22_attention_weight_max": 28.428810119628906,
      "activations/layer22_attention_weight_min": -25.78199577331543,
      "activations/layer23_attention_weight_max": 39.194969177246094,
      "activations/layer23_attention_weight_min": -28.452306747436523,
      "activations/layer2_attention_weight_max": 30.886234283447266,
      "activations/layer2_attention_weight_min": -30.299827575683594,
      "activations/layer3_attention_weight_max": 97.47101593017578,
      "activations/layer3_attention_weight_min": -95.39875030517578,
      "activations/layer4_attention_weight_max": 99.10638427734375,
      "activations/layer4_attention_weight_min": -95.41561889648438,
      "activations/layer5_attention_weight_max": 71.93124389648438,
      "activations/layer5_attention_weight_min": -79.26203918457031,
      "activations/layer6_attention_weight_max": 54.45097351074219,
      "activations/layer6_attention_weight_min": -56.27626037597656,
      "activations/layer7_attention_weight_max": 73.55648040771484,
      "activations/layer7_attention_weight_min": -71.75751495361328,
      "activations/layer8_attention_weight_max": 45.29249954223633,
      "activations/layer8_attention_weight_min": -46.761573791503906,
      "activations/layer9_attention_weight_max": 46.141319274902344,
      "activations/layer9_attention_weight_min": -50.545711517333984,
      "epoch": 11.06,
      "learning_rate": 7.94564393939394e-05,
      "loss": 2.7806,
      "step": 190400
    },
    {
      "activations/layer0_attention_weight_max": 15.422149658203125,
      "activations/layer0_attention_weight_min": -13.952356338500977,
      "activations/layer10_attention_weight_max": 41.00807571411133,
      "activations/layer10_attention_weight_min": -41.44537353515625,
      "activations/layer11_attention_weight_max": 37.10332107543945,
      "activations/layer11_attention_weight_min": -35.813636779785156,
      "activations/layer12_attention_weight_max": 20.097158432006836,
      "activations/layer12_attention_weight_min": -24.77365493774414,
      "activations/layer13_attention_weight_max": 36.43192672729492,
      "activations/layer13_attention_weight_min": -31.051528930664062,
      "activations/layer14_attention_weight_max": 44.94538116455078,
      "activations/layer14_attention_weight_min": -29.66349983215332,
      "activations/layer15_attention_weight_max": 35.74845504760742,
      "activations/layer15_attention_weight_min": -27.90049171447754,
      "activations/layer16_attention_weight_max": 33.46938705444336,
      "activations/layer16_attention_weight_min": -29.080699920654297,
      "activations/layer17_attention_weight_max": 56.74507522583008,
      "activations/layer17_attention_weight_min": -44.76909637451172,
      "activations/layer18_attention_weight_max": 46.8421516418457,
      "activations/layer18_attention_weight_min": -37.30229949951172,
      "activations/layer19_attention_weight_max": 23.434431076049805,
      "activations/layer19_attention_weight_min": -22.102148056030273,
      "activations/layer1_attention_weight_max": 16.74308204650879,
      "activations/layer1_attention_weight_min": -14.21911334991455,
      "activations/layer20_attention_weight_max": 23.500938415527344,
      "activations/layer20_attention_weight_min": -22.452842712402344,
      "activations/layer21_attention_weight_max": 39.79320526123047,
      "activations/layer21_attention_weight_min": -24.345718383789062,
      "activations/layer22_attention_weight_max": 32.07953643798828,
      "activations/layer22_attention_weight_min": -26.84284019470215,
      "activations/layer23_attention_weight_max": 37.8221321105957,
      "activations/layer23_attention_weight_min": -25.73689842224121,
      "activations/layer2_attention_weight_max": 31.361448287963867,
      "activations/layer2_attention_weight_min": -31.482749938964844,
      "activations/layer3_attention_weight_max": 90.27718353271484,
      "activations/layer3_attention_weight_min": -95.99507904052734,
      "activations/layer4_attention_weight_max": 96.9263916015625,
      "activations/layer4_attention_weight_min": -99.79972839355469,
      "activations/layer5_attention_weight_max": 68.62030029296875,
      "activations/layer5_attention_weight_min": -75.9388198852539,
      "activations/layer6_attention_weight_max": 53.064369201660156,
      "activations/layer6_attention_weight_min": -52.13432312011719,
      "activations/layer7_attention_weight_max": 75.92533111572266,
      "activations/layer7_attention_weight_min": -71.74009704589844,
      "activations/layer8_attention_weight_max": 48.270267486572266,
      "activations/layer8_attention_weight_min": -49.73552703857422,
      "activations/layer9_attention_weight_max": 50.79301452636719,
      "activations/layer9_attention_weight_min": -50.025108337402344,
      "epoch": 11.07,
      "learning_rate": 7.943749999999999e-05,
      "loss": 2.7716,
      "step": 190450
    },
    {
      "activations/layer0_attention_weight_max": 15.446344375610352,
      "activations/layer0_attention_weight_min": -14.136624336242676,
      "activations/layer10_attention_weight_max": 38.143062591552734,
      "activations/layer10_attention_weight_min": -35.07536315917969,
      "activations/layer11_attention_weight_max": 32.781883239746094,
      "activations/layer11_attention_weight_min": -32.71072006225586,
      "activations/layer12_attention_weight_max": 18.508066177368164,
      "activations/layer12_attention_weight_min": -24.52286720275879,
      "activations/layer13_attention_weight_max": 34.00595474243164,
      "activations/layer13_attention_weight_min": -29.165918350219727,
      "activations/layer14_attention_weight_max": 35.95928192138672,
      "activations/layer14_attention_weight_min": -28.935073852539062,
      "activations/layer15_attention_weight_max": 35.61515808105469,
      "activations/layer15_attention_weight_min": -29.9497127532959,
      "activations/layer16_attention_weight_max": 30.125911712646484,
      "activations/layer16_attention_weight_min": -28.175912857055664,
      "activations/layer17_attention_weight_max": 54.50529861450195,
      "activations/layer17_attention_weight_min": -46.30891036987305,
      "activations/layer18_attention_weight_max": 45.65086364746094,
      "activations/layer18_attention_weight_min": -38.124908447265625,
      "activations/layer19_attention_weight_max": 23.069799423217773,
      "activations/layer19_attention_weight_min": -22.67137908935547,
      "activations/layer1_attention_weight_max": 16.741519927978516,
      "activations/layer1_attention_weight_min": -16.240415573120117,
      "activations/layer20_attention_weight_max": 21.247159957885742,
      "activations/layer20_attention_weight_min": -22.065073013305664,
      "activations/layer21_attention_weight_max": 35.80499267578125,
      "activations/layer21_attention_weight_min": -22.776275634765625,
      "activations/layer22_attention_weight_max": 33.511634826660156,
      "activations/layer22_attention_weight_min": -24.493732452392578,
      "activations/layer23_attention_weight_max": 37.41350555419922,
      "activations/layer23_attention_weight_min": -22.439186096191406,
      "activations/layer2_attention_weight_max": 33.48821258544922,
      "activations/layer2_attention_weight_min": -31.761943817138672,
      "activations/layer3_attention_weight_max": 90.88032531738281,
      "activations/layer3_attention_weight_min": -98.45761108398438,
      "activations/layer4_attention_weight_max": 95.42481231689453,
      "activations/layer4_attention_weight_min": -97.69731140136719,
      "activations/layer5_attention_weight_max": 68.21150207519531,
      "activations/layer5_attention_weight_min": -72.46893310546875,
      "activations/layer6_attention_weight_max": 51.98301696777344,
      "activations/layer6_attention_weight_min": -53.6746940612793,
      "activations/layer7_attention_weight_max": 79.33428955078125,
      "activations/layer7_attention_weight_min": -70.22398376464844,
      "activations/layer8_attention_weight_max": 46.504173278808594,
      "activations/layer8_attention_weight_min": -48.36933135986328,
      "activations/layer9_attention_weight_max": 48.31129455566406,
      "activations/layer9_attention_weight_min": -45.86031723022461,
      "epoch": 11.07,
      "learning_rate": 7.94185606060606e-05,
      "loss": 2.7723,
      "step": 190500
    },
    {
      "activations/layer0_attention_weight_max": 15.687163352966309,
      "activations/layer0_attention_weight_min": -14.636391639709473,
      "activations/layer10_attention_weight_max": 37.05060577392578,
      "activations/layer10_attention_weight_min": -35.80169677734375,
      "activations/layer11_attention_weight_max": 30.851139068603516,
      "activations/layer11_attention_weight_min": -32.35911178588867,
      "activations/layer12_attention_weight_max": 17.387670516967773,
      "activations/layer12_attention_weight_min": -26.206445693969727,
      "activations/layer13_attention_weight_max": 45.63011932373047,
      "activations/layer13_attention_weight_min": -31.644447326660156,
      "activations/layer14_attention_weight_max": 47.876766204833984,
      "activations/layer14_attention_weight_min": -30.7022647857666,
      "activations/layer15_attention_weight_max": 36.282981872558594,
      "activations/layer15_attention_weight_min": -28.164758682250977,
      "activations/layer16_attention_weight_max": 31.500654220581055,
      "activations/layer16_attention_weight_min": -30.28399658203125,
      "activations/layer17_attention_weight_max": 51.1651611328125,
      "activations/layer17_attention_weight_min": -44.66163635253906,
      "activations/layer18_attention_weight_max": 42.99694061279297,
      "activations/layer18_attention_weight_min": -36.831031799316406,
      "activations/layer19_attention_weight_max": 23.115127563476562,
      "activations/layer19_attention_weight_min": -20.598121643066406,
      "activations/layer1_attention_weight_max": 17.1398983001709,
      "activations/layer1_attention_weight_min": -17.01820182800293,
      "activations/layer20_attention_weight_max": 23.658966064453125,
      "activations/layer20_attention_weight_min": -20.085840225219727,
      "activations/layer21_attention_weight_max": 40.18149185180664,
      "activations/layer21_attention_weight_min": -23.69606590270996,
      "activations/layer22_attention_weight_max": 31.847938537597656,
      "activations/layer22_attention_weight_min": -24.314682006835938,
      "activations/layer23_attention_weight_max": 42.23607635498047,
      "activations/layer23_attention_weight_min": -23.421405792236328,
      "activations/layer2_attention_weight_max": 33.840667724609375,
      "activations/layer2_attention_weight_min": -33.69276428222656,
      "activations/layer3_attention_weight_max": 95.54208374023438,
      "activations/layer3_attention_weight_min": -94.13923645019531,
      "activations/layer4_attention_weight_max": 101.31954193115234,
      "activations/layer4_attention_weight_min": -97.69224548339844,
      "activations/layer5_attention_weight_max": 74.4380874633789,
      "activations/layer5_attention_weight_min": -77.13383483886719,
      "activations/layer6_attention_weight_max": 55.002201080322266,
      "activations/layer6_attention_weight_min": -53.085201263427734,
      "activations/layer7_attention_weight_max": 80.47366333007812,
      "activations/layer7_attention_weight_min": -75.09777069091797,
      "activations/layer8_attention_weight_max": 44.640377044677734,
      "activations/layer8_attention_weight_min": -46.0383186340332,
      "activations/layer9_attention_weight_max": 47.8361701965332,
      "activations/layer9_attention_weight_min": -48.6723747253418,
      "epoch": 11.07,
      "learning_rate": 7.939962121212122e-05,
      "loss": 2.7785,
      "step": 190550
    },
    {
      "activations/layer0_attention_weight_max": 15.669926643371582,
      "activations/layer0_attention_weight_min": -14.718670845031738,
      "activations/layer10_attention_weight_max": 38.427738189697266,
      "activations/layer10_attention_weight_min": -35.79407501220703,
      "activations/layer11_attention_weight_max": 32.997806549072266,
      "activations/layer11_attention_weight_min": -33.64092254638672,
      "activations/layer12_attention_weight_max": 18.991928100585938,
      "activations/layer12_attention_weight_min": -22.072595596313477,
      "activations/layer13_attention_weight_max": 37.32953643798828,
      "activations/layer13_attention_weight_min": -30.238725662231445,
      "activations/layer14_attention_weight_max": 36.45480728149414,
      "activations/layer14_attention_weight_min": -30.432191848754883,
      "activations/layer15_attention_weight_max": 33.63187789916992,
      "activations/layer15_attention_weight_min": -29.580467224121094,
      "activations/layer16_attention_weight_max": 29.70002555847168,
      "activations/layer16_attention_weight_min": -29.399831771850586,
      "activations/layer17_attention_weight_max": 53.49931716918945,
      "activations/layer17_attention_weight_min": -43.806270599365234,
      "activations/layer18_attention_weight_max": 48.27936935424805,
      "activations/layer18_attention_weight_min": -36.57320022583008,
      "activations/layer19_attention_weight_max": 23.13425064086914,
      "activations/layer19_attention_weight_min": -21.610082626342773,
      "activations/layer1_attention_weight_max": 16.559303283691406,
      "activations/layer1_attention_weight_min": -14.114246368408203,
      "activations/layer20_attention_weight_max": 21.525053024291992,
      "activations/layer20_attention_weight_min": -20.867223739624023,
      "activations/layer21_attention_weight_max": 37.19075393676758,
      "activations/layer21_attention_weight_min": -23.244216918945312,
      "activations/layer22_attention_weight_max": 31.596616744995117,
      "activations/layer22_attention_weight_min": -25.476837158203125,
      "activations/layer23_attention_weight_max": 38.23100662231445,
      "activations/layer23_attention_weight_min": -25.85371208190918,
      "activations/layer2_attention_weight_max": 30.464115142822266,
      "activations/layer2_attention_weight_min": -30.083065032958984,
      "activations/layer3_attention_weight_max": 88.81668090820312,
      "activations/layer3_attention_weight_min": -93.14788818359375,
      "activations/layer4_attention_weight_max": 98.57496643066406,
      "activations/layer4_attention_weight_min": -94.13478088378906,
      "activations/layer5_attention_weight_max": 70.99789428710938,
      "activations/layer5_attention_weight_min": -79.79659271240234,
      "activations/layer6_attention_weight_max": 55.56796646118164,
      "activations/layer6_attention_weight_min": -57.35121536254883,
      "activations/layer7_attention_weight_max": 73.44680786132812,
      "activations/layer7_attention_weight_min": -73.77860260009766,
      "activations/layer8_attention_weight_max": 46.668888092041016,
      "activations/layer8_attention_weight_min": -48.11320114135742,
      "activations/layer9_attention_weight_max": 48.28049850463867,
      "activations/layer9_attention_weight_min": -49.170528411865234,
      "epoch": 11.07,
      "learning_rate": 7.93806818181818e-05,
      "loss": 2.7799,
      "step": 190600
    },
    {
      "activations/layer0_attention_weight_max": 15.356684684753418,
      "activations/layer0_attention_weight_min": -13.861040115356445,
      "activations/layer10_attention_weight_max": 35.14687728881836,
      "activations/layer10_attention_weight_min": -35.77826690673828,
      "activations/layer11_attention_weight_max": 31.75629234313965,
      "activations/layer11_attention_weight_min": -31.410663604736328,
      "activations/layer12_attention_weight_max": 17.389446258544922,
      "activations/layer12_attention_weight_min": -24.932979583740234,
      "activations/layer13_attention_weight_max": 37.35032653808594,
      "activations/layer13_attention_weight_min": -26.418245315551758,
      "activations/layer14_attention_weight_max": 38.485652923583984,
      "activations/layer14_attention_weight_min": -28.85047721862793,
      "activations/layer15_attention_weight_max": 33.10773468017578,
      "activations/layer15_attention_weight_min": -29.68848991394043,
      "activations/layer16_attention_weight_max": 30.23063087463379,
      "activations/layer16_attention_weight_min": -28.336406707763672,
      "activations/layer17_attention_weight_max": 50.62566375732422,
      "activations/layer17_attention_weight_min": -46.355010986328125,
      "activations/layer18_attention_weight_max": 44.23973846435547,
      "activations/layer18_attention_weight_min": -38.159305572509766,
      "activations/layer19_attention_weight_max": 21.72504234313965,
      "activations/layer19_attention_weight_min": -21.677392959594727,
      "activations/layer1_attention_weight_max": 17.367525100708008,
      "activations/layer1_attention_weight_min": -16.3970947265625,
      "activations/layer20_attention_weight_max": 23.452430725097656,
      "activations/layer20_attention_weight_min": -24.061779022216797,
      "activations/layer21_attention_weight_max": 38.53387451171875,
      "activations/layer21_attention_weight_min": -22.085412979125977,
      "activations/layer22_attention_weight_max": 29.3844051361084,
      "activations/layer22_attention_weight_min": -24.975971221923828,
      "activations/layer23_attention_weight_max": 36.60206604003906,
      "activations/layer23_attention_weight_min": -25.600727081298828,
      "activations/layer2_attention_weight_max": 32.918556213378906,
      "activations/layer2_attention_weight_min": -30.749990463256836,
      "activations/layer3_attention_weight_max": 89.94593048095703,
      "activations/layer3_attention_weight_min": -99.42463684082031,
      "activations/layer4_attention_weight_max": 90.9379653930664,
      "activations/layer4_attention_weight_min": -95.17162322998047,
      "activations/layer5_attention_weight_max": 69.88705444335938,
      "activations/layer5_attention_weight_min": -73.53517150878906,
      "activations/layer6_attention_weight_max": 54.05955123901367,
      "activations/layer6_attention_weight_min": -51.68687057495117,
      "activations/layer7_attention_weight_max": 74.1615982055664,
      "activations/layer7_attention_weight_min": -66.79349517822266,
      "activations/layer8_attention_weight_max": 43.635684967041016,
      "activations/layer8_attention_weight_min": -44.777584075927734,
      "activations/layer9_attention_weight_max": 46.68412399291992,
      "activations/layer9_attention_weight_min": -46.38105392456055,
      "epoch": 11.08,
      "learning_rate": 7.936174242424242e-05,
      "loss": 2.773,
      "step": 190650
    },
    {
      "activations/layer0_attention_weight_max": 14.996912956237793,
      "activations/layer0_attention_weight_min": -14.0022611618042,
      "activations/layer10_attention_weight_max": 38.922950744628906,
      "activations/layer10_attention_weight_min": -36.151153564453125,
      "activations/layer11_attention_weight_max": 33.09003448486328,
      "activations/layer11_attention_weight_min": -33.417903900146484,
      "activations/layer12_attention_weight_max": 21.5327205657959,
      "activations/layer12_attention_weight_min": -24.129638671875,
      "activations/layer13_attention_weight_max": 39.55561447143555,
      "activations/layer13_attention_weight_min": -30.13958740234375,
      "activations/layer14_attention_weight_max": 37.00788879394531,
      "activations/layer14_attention_weight_min": -32.11457824707031,
      "activations/layer15_attention_weight_max": 34.52949142456055,
      "activations/layer15_attention_weight_min": -30.5108699798584,
      "activations/layer16_attention_weight_max": 31.437252044677734,
      "activations/layer16_attention_weight_min": -29.40645408630371,
      "activations/layer17_attention_weight_max": 55.059242248535156,
      "activations/layer17_attention_weight_min": -46.283714294433594,
      "activations/layer18_attention_weight_max": 47.49148941040039,
      "activations/layer18_attention_weight_min": -39.78828048706055,
      "activations/layer19_attention_weight_max": 22.04530143737793,
      "activations/layer19_attention_weight_min": -21.51014518737793,
      "activations/layer1_attention_weight_max": 16.81018829345703,
      "activations/layer1_attention_weight_min": -15.55073356628418,
      "activations/layer20_attention_weight_max": 21.67367172241211,
      "activations/layer20_attention_weight_min": -20.253448486328125,
      "activations/layer21_attention_weight_max": 39.98158264160156,
      "activations/layer21_attention_weight_min": -24.32305908203125,
      "activations/layer22_attention_weight_max": 32.554893493652344,
      "activations/layer22_attention_weight_min": -27.228580474853516,
      "activations/layer23_attention_weight_max": 38.56208038330078,
      "activations/layer23_attention_weight_min": -24.979270935058594,
      "activations/layer2_attention_weight_max": 33.752655029296875,
      "activations/layer2_attention_weight_min": -32.04633712768555,
      "activations/layer3_attention_weight_max": 92.52123260498047,
      "activations/layer3_attention_weight_min": -92.44049835205078,
      "activations/layer4_attention_weight_max": 96.8375473022461,
      "activations/layer4_attention_weight_min": -92.5356216430664,
      "activations/layer5_attention_weight_max": 74.63174438476562,
      "activations/layer5_attention_weight_min": -76.8899917602539,
      "activations/layer6_attention_weight_max": 55.82667541503906,
      "activations/layer6_attention_weight_min": -54.177371978759766,
      "activations/layer7_attention_weight_max": 74.01365661621094,
      "activations/layer7_attention_weight_min": -71.19951629638672,
      "activations/layer8_attention_weight_max": 47.16264343261719,
      "activations/layer8_attention_weight_min": -47.35240936279297,
      "activations/layer9_attention_weight_max": 50.17244338989258,
      "activations/layer9_attention_weight_min": -48.13600540161133,
      "epoch": 11.08,
      "learning_rate": 7.934280303030303e-05,
      "loss": 2.7669,
      "step": 190700
    },
    {
      "activations/layer0_attention_weight_max": 15.239713668823242,
      "activations/layer0_attention_weight_min": -13.939766883850098,
      "activations/layer10_attention_weight_max": 39.46072006225586,
      "activations/layer10_attention_weight_min": -37.989540100097656,
      "activations/layer11_attention_weight_max": 32.64106750488281,
      "activations/layer11_attention_weight_min": -35.34571075439453,
      "activations/layer12_attention_weight_max": 19.50443458557129,
      "activations/layer12_attention_weight_min": -28.638370513916016,
      "activations/layer13_attention_weight_max": 41.19779586791992,
      "activations/layer13_attention_weight_min": -31.940771102905273,
      "activations/layer14_attention_weight_max": 42.91404724121094,
      "activations/layer14_attention_weight_min": -29.73541259765625,
      "activations/layer15_attention_weight_max": 36.264713287353516,
      "activations/layer15_attention_weight_min": -28.155763626098633,
      "activations/layer16_attention_weight_max": 30.241113662719727,
      "activations/layer16_attention_weight_min": -29.44480323791504,
      "activations/layer17_attention_weight_max": 50.7256965637207,
      "activations/layer17_attention_weight_min": -44.3538703918457,
      "activations/layer18_attention_weight_max": 45.65735626220703,
      "activations/layer18_attention_weight_min": -40.12254333496094,
      "activations/layer19_attention_weight_max": 22.79566764831543,
      "activations/layer19_attention_weight_min": -20.49080467224121,
      "activations/layer1_attention_weight_max": 15.878044128417969,
      "activations/layer1_attention_weight_min": -14.645913124084473,
      "activations/layer20_attention_weight_max": 20.214439392089844,
      "activations/layer20_attention_weight_min": -21.148906707763672,
      "activations/layer21_attention_weight_max": 38.746673583984375,
      "activations/layer21_attention_weight_min": -27.094406127929688,
      "activations/layer22_attention_weight_max": 29.838912963867188,
      "activations/layer22_attention_weight_min": -26.59602165222168,
      "activations/layer23_attention_weight_max": 38.83623504638672,
      "activations/layer23_attention_weight_min": -24.21843719482422,
      "activations/layer2_attention_weight_max": 33.576053619384766,
      "activations/layer2_attention_weight_min": -33.90021514892578,
      "activations/layer3_attention_weight_max": 95.7139892578125,
      "activations/layer3_attention_weight_min": -96.11996459960938,
      "activations/layer4_attention_weight_max": 104.01871490478516,
      "activations/layer4_attention_weight_min": -100.11644744873047,
      "activations/layer5_attention_weight_max": 77.13174438476562,
      "activations/layer5_attention_weight_min": -81.46728515625,
      "activations/layer6_attention_weight_max": 55.90507507324219,
      "activations/layer6_attention_weight_min": -52.81743240356445,
      "activations/layer7_attention_weight_max": 76.01179504394531,
      "activations/layer7_attention_weight_min": -71.9291000366211,
      "activations/layer8_attention_weight_max": 45.57368850708008,
      "activations/layer8_attention_weight_min": -47.07126998901367,
      "activations/layer9_attention_weight_max": 51.519325256347656,
      "activations/layer9_attention_weight_min": -47.391231536865234,
      "epoch": 11.08,
      "learning_rate": 7.932386363636362e-05,
      "loss": 2.7733,
      "step": 190750
    },
    {
      "activations/layer0_attention_weight_max": 15.551029205322266,
      "activations/layer0_attention_weight_min": -13.971712112426758,
      "activations/layer10_attention_weight_max": 39.17146682739258,
      "activations/layer10_attention_weight_min": -36.99237060546875,
      "activations/layer11_attention_weight_max": 33.041908264160156,
      "activations/layer11_attention_weight_min": -35.503047943115234,
      "activations/layer12_attention_weight_max": 23.46487045288086,
      "activations/layer12_attention_weight_min": -23.380369186401367,
      "activations/layer13_attention_weight_max": 38.58240509033203,
      "activations/layer13_attention_weight_min": -28.5577449798584,
      "activations/layer14_attention_weight_max": 39.209434509277344,
      "activations/layer14_attention_weight_min": -30.178743362426758,
      "activations/layer15_attention_weight_max": 32.535377502441406,
      "activations/layer15_attention_weight_min": -30.017562866210938,
      "activations/layer16_attention_weight_max": 31.942777633666992,
      "activations/layer16_attention_weight_min": -29.09931182861328,
      "activations/layer17_attention_weight_max": 52.60462188720703,
      "activations/layer17_attention_weight_min": -43.06302261352539,
      "activations/layer18_attention_weight_max": 48.75913619995117,
      "activations/layer18_attention_weight_min": -38.757389068603516,
      "activations/layer19_attention_weight_max": 21.28676986694336,
      "activations/layer19_attention_weight_min": -22.445144653320312,
      "activations/layer1_attention_weight_max": 17.461938858032227,
      "activations/layer1_attention_weight_min": -15.305937767028809,
      "activations/layer20_attention_weight_max": 23.33262062072754,
      "activations/layer20_attention_weight_min": -19.887828826904297,
      "activations/layer21_attention_weight_max": 38.13228225708008,
      "activations/layer21_attention_weight_min": -20.379928588867188,
      "activations/layer22_attention_weight_max": 29.81965446472168,
      "activations/layer22_attention_weight_min": -24.236515045166016,
      "activations/layer23_attention_weight_max": 37.449859619140625,
      "activations/layer23_attention_weight_min": -22.010587692260742,
      "activations/layer2_attention_weight_max": 33.14888000488281,
      "activations/layer2_attention_weight_min": -31.581485748291016,
      "activations/layer3_attention_weight_max": 93.93294525146484,
      "activations/layer3_attention_weight_min": -97.03440856933594,
      "activations/layer4_attention_weight_max": 97.33568572998047,
      "activations/layer4_attention_weight_min": -98.08480072021484,
      "activations/layer5_attention_weight_max": 74.12588500976562,
      "activations/layer5_attention_weight_min": -81.09732055664062,
      "activations/layer6_attention_weight_max": 54.60342025756836,
      "activations/layer6_attention_weight_min": -58.191009521484375,
      "activations/layer7_attention_weight_max": 78.85176086425781,
      "activations/layer7_attention_weight_min": -76.27711486816406,
      "activations/layer8_attention_weight_max": 47.70094299316406,
      "activations/layer8_attention_weight_min": -50.286373138427734,
      "activations/layer9_attention_weight_max": 54.023658752441406,
      "activations/layer9_attention_weight_min": -51.252113342285156,
      "epoch": 11.09,
      "learning_rate": 7.930492424242424e-05,
      "loss": 2.7838,
      "step": 190800
    },
    {
      "activations/layer0_attention_weight_max": 17.062896728515625,
      "activations/layer0_attention_weight_min": -14.343939781188965,
      "activations/layer10_attention_weight_max": 50.9002571105957,
      "activations/layer10_attention_weight_min": -47.81632614135742,
      "activations/layer11_attention_weight_max": 50.00981140136719,
      "activations/layer11_attention_weight_min": -40.45859909057617,
      "activations/layer12_attention_weight_max": 19.339523315429688,
      "activations/layer12_attention_weight_min": -22.92144203186035,
      "activations/layer13_attention_weight_max": 41.75817108154297,
      "activations/layer13_attention_weight_min": -31.0841007232666,
      "activations/layer14_attention_weight_max": 37.13078689575195,
      "activations/layer14_attention_weight_min": -30.22562599182129,
      "activations/layer15_attention_weight_max": 33.24213409423828,
      "activations/layer15_attention_weight_min": -30.383567810058594,
      "activations/layer16_attention_weight_max": 31.158220291137695,
      "activations/layer16_attention_weight_min": -29.201637268066406,
      "activations/layer17_attention_weight_max": 51.00192642211914,
      "activations/layer17_attention_weight_min": -43.647830963134766,
      "activations/layer18_attention_weight_max": 47.98675537109375,
      "activations/layer18_attention_weight_min": -38.100711822509766,
      "activations/layer19_attention_weight_max": 22.22829246520996,
      "activations/layer19_attention_weight_min": -21.93275260925293,
      "activations/layer1_attention_weight_max": 16.759653091430664,
      "activations/layer1_attention_weight_min": -14.987184524536133,
      "activations/layer20_attention_weight_max": 21.898767471313477,
      "activations/layer20_attention_weight_min": -22.8449764251709,
      "activations/layer21_attention_weight_max": 33.775367736816406,
      "activations/layer21_attention_weight_min": -21.553884506225586,
      "activations/layer22_attention_weight_max": 33.2548828125,
      "activations/layer22_attention_weight_min": -24.569156646728516,
      "activations/layer23_attention_weight_max": 38.128238677978516,
      "activations/layer23_attention_weight_min": -22.060073852539062,
      "activations/layer2_attention_weight_max": 35.86064147949219,
      "activations/layer2_attention_weight_min": -32.000736236572266,
      "activations/layer3_attention_weight_max": 100.56594848632812,
      "activations/layer3_attention_weight_min": -104.38572692871094,
      "activations/layer4_attention_weight_max": 109.6921615600586,
      "activations/layer4_attention_weight_min": -102.49364471435547,
      "activations/layer5_attention_weight_max": 83.39012908935547,
      "activations/layer5_attention_weight_min": -72.99716186523438,
      "activations/layer6_attention_weight_max": 58.226593017578125,
      "activations/layer6_attention_weight_min": -54.366878509521484,
      "activations/layer7_attention_weight_max": 97.0986557006836,
      "activations/layer7_attention_weight_min": -73.92792510986328,
      "activations/layer8_attention_weight_max": 59.78929901123047,
      "activations/layer8_attention_weight_min": -51.94562530517578,
      "activations/layer9_attention_weight_max": 78.9554672241211,
      "activations/layer9_attention_weight_min": -58.313232421875,
      "epoch": 11.09,
      "learning_rate": 7.928598484848484e-05,
      "loss": 2.7582,
      "step": 190850
    },
    {
      "activations/layer0_attention_weight_max": 15.8585786819458,
      "activations/layer0_attention_weight_min": -15.071167945861816,
      "activations/layer10_attention_weight_max": 37.9964714050293,
      "activations/layer10_attention_weight_min": -37.15580368041992,
      "activations/layer11_attention_weight_max": 34.33740997314453,
      "activations/layer11_attention_weight_min": -32.99699401855469,
      "activations/layer12_attention_weight_max": 19.674591064453125,
      "activations/layer12_attention_weight_min": -24.759809494018555,
      "activations/layer13_attention_weight_max": 41.50128173828125,
      "activations/layer13_attention_weight_min": -33.19682693481445,
      "activations/layer14_attention_weight_max": 41.81471633911133,
      "activations/layer14_attention_weight_min": -31.276411056518555,
      "activations/layer15_attention_weight_max": 37.98223876953125,
      "activations/layer15_attention_weight_min": -31.797819137573242,
      "activations/layer16_attention_weight_max": 37.06122970581055,
      "activations/layer16_attention_weight_min": -32.21628189086914,
      "activations/layer17_attention_weight_max": 58.63410949707031,
      "activations/layer17_attention_weight_min": -49.781646728515625,
      "activations/layer18_attention_weight_max": 50.480587005615234,
      "activations/layer18_attention_weight_min": -43.58901596069336,
      "activations/layer19_attention_weight_max": 24.764005661010742,
      "activations/layer19_attention_weight_min": -23.558868408203125,
      "activations/layer1_attention_weight_max": 17.55950927734375,
      "activations/layer1_attention_weight_min": -17.26515769958496,
      "activations/layer20_attention_weight_max": 26.72454833984375,
      "activations/layer20_attention_weight_min": -23.867958068847656,
      "activations/layer21_attention_weight_max": 51.226036071777344,
      "activations/layer21_attention_weight_min": -30.298046112060547,
      "activations/layer22_attention_weight_max": 33.94832229614258,
      "activations/layer22_attention_weight_min": -28.088642120361328,
      "activations/layer23_attention_weight_max": 42.090248107910156,
      "activations/layer23_attention_weight_min": -28.616626739501953,
      "activations/layer2_attention_weight_max": 32.26646041870117,
      "activations/layer2_attention_weight_min": -31.31439208984375,
      "activations/layer3_attention_weight_max": 94.19237518310547,
      "activations/layer3_attention_weight_min": -97.03682708740234,
      "activations/layer4_attention_weight_max": 100.51417541503906,
      "activations/layer4_attention_weight_min": -97.11833190917969,
      "activations/layer5_attention_weight_max": 74.01385498046875,
      "activations/layer5_attention_weight_min": -84.25315856933594,
      "activations/layer6_attention_weight_max": 55.245887756347656,
      "activations/layer6_attention_weight_min": -55.32664108276367,
      "activations/layer7_attention_weight_max": 78.5120620727539,
      "activations/layer7_attention_weight_min": -74.4850082397461,
      "activations/layer8_attention_weight_max": 47.861480712890625,
      "activations/layer8_attention_weight_min": -50.282676696777344,
      "activations/layer9_attention_weight_max": 49.12858200073242,
      "activations/layer9_attention_weight_min": -48.369327545166016,
      "epoch": 11.09,
      "learning_rate": 7.926704545454545e-05,
      "loss": 2.7819,
      "step": 190900
    },
    {
      "activations/layer0_attention_weight_max": 15.242756843566895,
      "activations/layer0_attention_weight_min": -14.265430450439453,
      "activations/layer10_attention_weight_max": 41.095760345458984,
      "activations/layer10_attention_weight_min": -38.048011779785156,
      "activations/layer11_attention_weight_max": 35.38670349121094,
      "activations/layer11_attention_weight_min": -33.673213958740234,
      "activations/layer12_attention_weight_max": 22.057418823242188,
      "activations/layer12_attention_weight_min": -28.557621002197266,
      "activations/layer13_attention_weight_max": 66.13761901855469,
      "activations/layer13_attention_weight_min": -35.57711410522461,
      "activations/layer14_attention_weight_max": 51.00681686401367,
      "activations/layer14_attention_weight_min": -31.401565551757812,
      "activations/layer15_attention_weight_max": 37.32902908325195,
      "activations/layer15_attention_weight_min": -30.085609436035156,
      "activations/layer16_attention_weight_max": 33.49591064453125,
      "activations/layer16_attention_weight_min": -31.81360626220703,
      "activations/layer17_attention_weight_max": 58.64229202270508,
      "activations/layer17_attention_weight_min": -50.062564849853516,
      "activations/layer18_attention_weight_max": 48.4137077331543,
      "activations/layer18_attention_weight_min": -42.30385971069336,
      "activations/layer19_attention_weight_max": 23.742918014526367,
      "activations/layer19_attention_weight_min": -24.2898006439209,
      "activations/layer1_attention_weight_max": 17.788774490356445,
      "activations/layer1_attention_weight_min": -15.339983940124512,
      "activations/layer20_attention_weight_max": 26.27720832824707,
      "activations/layer20_attention_weight_min": -20.57113265991211,
      "activations/layer21_attention_weight_max": 38.551307678222656,
      "activations/layer21_attention_weight_min": -22.610483169555664,
      "activations/layer22_attention_weight_max": 31.794139862060547,
      "activations/layer22_attention_weight_min": -24.34038543701172,
      "activations/layer23_attention_weight_max": 35.129825592041016,
      "activations/layer23_attention_weight_min": -21.258085250854492,
      "activations/layer2_attention_weight_max": 33.725547790527344,
      "activations/layer2_attention_weight_min": -32.4939079284668,
      "activations/layer3_attention_weight_max": 97.572265625,
      "activations/layer3_attention_weight_min": -99.84939575195312,
      "activations/layer4_attention_weight_max": 102.22875213623047,
      "activations/layer4_attention_weight_min": -98.3911361694336,
      "activations/layer5_attention_weight_max": 71.73394775390625,
      "activations/layer5_attention_weight_min": -76.52837371826172,
      "activations/layer6_attention_weight_max": 54.91394805908203,
      "activations/layer6_attention_weight_min": -55.5039176940918,
      "activations/layer7_attention_weight_max": 77.21154022216797,
      "activations/layer7_attention_weight_min": -79.04177856445312,
      "activations/layer8_attention_weight_max": 52.78366470336914,
      "activations/layer8_attention_weight_min": -51.1192626953125,
      "activations/layer9_attention_weight_max": 57.25690841674805,
      "activations/layer9_attention_weight_min": -51.10398483276367,
      "epoch": 11.1,
      "learning_rate": 7.924810606060606e-05,
      "loss": 2.7733,
      "step": 190950
    },
    {
      "activations/layer0_attention_weight_max": 17.05938720703125,
      "activations/layer0_attention_weight_min": -14.110389709472656,
      "activations/layer10_attention_weight_max": 39.48199462890625,
      "activations/layer10_attention_weight_min": -38.77679443359375,
      "activations/layer11_attention_weight_max": 33.061317443847656,
      "activations/layer11_attention_weight_min": -33.896488189697266,
      "activations/layer12_attention_weight_max": 17.859527587890625,
      "activations/layer12_attention_weight_min": -27.756105422973633,
      "activations/layer13_attention_weight_max": 55.230613708496094,
      "activations/layer13_attention_weight_min": -38.62232971191406,
      "activations/layer14_attention_weight_max": 60.34817123413086,
      "activations/layer14_attention_weight_min": -39.41701126098633,
      "activations/layer15_attention_weight_max": 46.479583740234375,
      "activations/layer15_attention_weight_min": -33.68345260620117,
      "activations/layer16_attention_weight_max": 32.967342376708984,
      "activations/layer16_attention_weight_min": -30.961164474487305,
      "activations/layer17_attention_weight_max": 56.3875732421875,
      "activations/layer17_attention_weight_min": -44.311553955078125,
      "activations/layer18_attention_weight_max": 48.462158203125,
      "activations/layer18_attention_weight_min": -37.75049591064453,
      "activations/layer19_attention_weight_max": 23.83611488342285,
      "activations/layer19_attention_weight_min": -21.400615692138672,
      "activations/layer1_attention_weight_max": 16.277835845947266,
      "activations/layer1_attention_weight_min": -15.066426277160645,
      "activations/layer20_attention_weight_max": 26.29091453552246,
      "activations/layer20_attention_weight_min": -23.766695022583008,
      "activations/layer21_attention_weight_max": 37.87555694580078,
      "activations/layer21_attention_weight_min": -21.31029510498047,
      "activations/layer22_attention_weight_max": 30.90727424621582,
      "activations/layer22_attention_weight_min": -26.48964500427246,
      "activations/layer23_attention_weight_max": 37.54307556152344,
      "activations/layer23_attention_weight_min": -22.397483825683594,
      "activations/layer2_attention_weight_max": 33.79741668701172,
      "activations/layer2_attention_weight_min": -32.559486389160156,
      "activations/layer3_attention_weight_max": 100.85421752929688,
      "activations/layer3_attention_weight_min": -99.07439422607422,
      "activations/layer4_attention_weight_max": 104.9791488647461,
      "activations/layer4_attention_weight_min": -107.74739074707031,
      "activations/layer5_attention_weight_max": 77.79297637939453,
      "activations/layer5_attention_weight_min": -82.93647766113281,
      "activations/layer6_attention_weight_max": 58.10163497924805,
      "activations/layer6_attention_weight_min": -54.95097732543945,
      "activations/layer7_attention_weight_max": 84.02226257324219,
      "activations/layer7_attention_weight_min": -80.38634490966797,
      "activations/layer8_attention_weight_max": 52.25558090209961,
      "activations/layer8_attention_weight_min": -51.758888244628906,
      "activations/layer9_attention_weight_max": 54.037879943847656,
      "activations/layer9_attention_weight_min": -52.543365478515625,
      "epoch": 11.1,
      "learning_rate": 7.922916666666666e-05,
      "loss": 2.7959,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_loss": 2.734375,
      "eval_runtime": 8.4918,
      "eval_samples_per_second": 505.667,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.4918,
      "eval_openwebtext_samples_per_second": 505.667,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_wikitext_loss": 2.974609375,
      "eval_wikitext_ppl": 19.581972547389178,
      "eval_wikitext_runtime": 1.9789,
      "eval_wikitext_samples_per_second": 230.433,
      "step": 191000
    },
    {
      "epoch": 11.1,
      "eval_lambada_loss": 2.556640625,
      "eval_lambada_ppl": 12.892433965519283,
      "eval_lambada_runtime": 9.5807,
      "eval_lambada_samples_per_second": 508.208,
      "step": 191000
    },
    {
      "activations/layer0_attention_weight_max": 15.48666000366211,
      "activations/layer0_attention_weight_min": -14.239087104797363,
      "activations/layer10_attention_weight_max": 39.32025909423828,
      "activations/layer10_attention_weight_min": -38.26417541503906,
      "activations/layer11_attention_weight_max": 38.382102966308594,
      "activations/layer11_attention_weight_min": -34.11041259765625,
      "activations/layer12_attention_weight_max": 19.62984848022461,
      "activations/layer12_attention_weight_min": -27.73234748840332,
      "activations/layer13_attention_weight_max": 50.02056121826172,
      "activations/layer13_attention_weight_min": -34.49613571166992,
      "activations/layer14_attention_weight_max": 45.171142578125,
      "activations/layer14_attention_weight_min": -31.32298469543457,
      "activations/layer15_attention_weight_max": 41.15800094604492,
      "activations/layer15_attention_weight_min": -30.06887435913086,
      "activations/layer16_attention_weight_max": 37.9350471496582,
      "activations/layer16_attention_weight_min": -32.374732971191406,
      "activations/layer17_attention_weight_max": 59.12890625,
      "activations/layer17_attention_weight_min": -47.937740325927734,
      "activations/layer18_attention_weight_max": 57.458900451660156,
      "activations/layer18_attention_weight_min": -38.80211639404297,
      "activations/layer19_attention_weight_max": 28.610544204711914,
      "activations/layer19_attention_weight_min": -23.349031448364258,
      "activations/layer1_attention_weight_max": 17.662073135375977,
      "activations/layer1_attention_weight_min": -14.485506057739258,
      "activations/layer20_attention_weight_max": 28.11824607849121,
      "activations/layer20_attention_weight_min": -21.47804069519043,
      "activations/layer21_attention_weight_max": 46.985069274902344,
      "activations/layer21_attention_weight_min": -23.147226333618164,
      "activations/layer22_attention_weight_max": 36.034942626953125,
      "activations/layer22_attention_weight_min": -25.097551345825195,
      "activations/layer23_attention_weight_max": 41.248355865478516,
      "activations/layer23_attention_weight_min": -23.601600646972656,
      "activations/layer2_attention_weight_max": 32.11741256713867,
      "activations/layer2_attention_weight_min": -32.778472900390625,
      "activations/layer3_attention_weight_max": 93.51017761230469,
      "activations/layer3_attention_weight_min": -90.56604766845703,
      "activations/layer4_attention_weight_max": 100.83489227294922,
      "activations/layer4_attention_weight_min": -92.8300552368164,
      "activations/layer5_attention_weight_max": 73.70955657958984,
      "activations/layer5_attention_weight_min": -78.17132568359375,
      "activations/layer6_attention_weight_max": 55.26609802246094,
      "activations/layer6_attention_weight_min": -53.02286148071289,
      "activations/layer7_attention_weight_max": 74.69540405273438,
      "activations/layer7_attention_weight_min": -74.51588439941406,
      "activations/layer8_attention_weight_max": 49.812278747558594,
      "activations/layer8_attention_weight_min": -47.517215728759766,
      "activations/layer9_attention_weight_max": 54.103424072265625,
      "activations/layer9_attention_weight_min": -50.59928512573242,
      "epoch": 11.1,
      "learning_rate": 7.921022727272727e-05,
      "loss": 2.7811,
      "step": 191050
    },
    {
      "activations/layer0_attention_weight_max": 16.760814666748047,
      "activations/layer0_attention_weight_min": -13.739623069763184,
      "activations/layer10_attention_weight_max": 38.97908020019531,
      "activations/layer10_attention_weight_min": -40.28730773925781,
      "activations/layer11_attention_weight_max": 32.941978454589844,
      "activations/layer11_attention_weight_min": -33.55747604370117,
      "activations/layer12_attention_weight_max": 18.905305862426758,
      "activations/layer12_attention_weight_min": -24.53814697265625,
      "activations/layer13_attention_weight_max": 42.30946731567383,
      "activations/layer13_attention_weight_min": -33.192893981933594,
      "activations/layer14_attention_weight_max": 47.032684326171875,
      "activations/layer14_attention_weight_min": -32.656681060791016,
      "activations/layer15_attention_weight_max": 39.222049713134766,
      "activations/layer15_attention_weight_min": -28.857315063476562,
      "activations/layer16_attention_weight_max": 34.19022750854492,
      "activations/layer16_attention_weight_min": -29.054353713989258,
      "activations/layer17_attention_weight_max": 60.78063201904297,
      "activations/layer17_attention_weight_min": -46.6666145324707,
      "activations/layer18_attention_weight_max": 46.78379440307617,
      "activations/layer18_attention_weight_min": -37.825584411621094,
      "activations/layer19_attention_weight_max": 23.1595516204834,
      "activations/layer19_attention_weight_min": -21.83826446533203,
      "activations/layer1_attention_weight_max": 16.667722702026367,
      "activations/layer1_attention_weight_min": -16.3042049407959,
      "activations/layer20_attention_weight_max": 26.164094924926758,
      "activations/layer20_attention_weight_min": -23.07640838623047,
      "activations/layer21_attention_weight_max": 44.469940185546875,
      "activations/layer21_attention_weight_min": -27.19522476196289,
      "activations/layer22_attention_weight_max": 31.949542999267578,
      "activations/layer22_attention_weight_min": -25.71054458618164,
      "activations/layer23_attention_weight_max": 37.6218376159668,
      "activations/layer23_attention_weight_min": -23.321121215820312,
      "activations/layer2_attention_weight_max": 35.75909423828125,
      "activations/layer2_attention_weight_min": -35.353912353515625,
      "activations/layer3_attention_weight_max": 101.56938171386719,
      "activations/layer3_attention_weight_min": -99.2035903930664,
      "activations/layer4_attention_weight_max": 107.08504486083984,
      "activations/layer4_attention_weight_min": -99.3829574584961,
      "activations/layer5_attention_weight_max": 73.59858703613281,
      "activations/layer5_attention_weight_min": -80.02741241455078,
      "activations/layer6_attention_weight_max": 55.982154846191406,
      "activations/layer6_attention_weight_min": -58.31200408935547,
      "activations/layer7_attention_weight_max": 79.9250259399414,
      "activations/layer7_attention_weight_min": -82.90274047851562,
      "activations/layer8_attention_weight_max": 49.76790237426758,
      "activations/layer8_attention_weight_min": -51.558292388916016,
      "activations/layer9_attention_weight_max": 57.41606521606445,
      "activations/layer9_attention_weight_min": -51.679290771484375,
      "epoch": 11.1,
      "learning_rate": 7.919128787878787e-05,
      "loss": 2.7837,
      "step": 191100
    },
    {
      "activations/layer0_attention_weight_max": 16.50326156616211,
      "activations/layer0_attention_weight_min": -13.9241943359375,
      "activations/layer10_attention_weight_max": 35.97406768798828,
      "activations/layer10_attention_weight_min": -36.09272003173828,
      "activations/layer11_attention_weight_max": 31.84528350830078,
      "activations/layer11_attention_weight_min": -31.911151885986328,
      "activations/layer12_attention_weight_max": 17.30097770690918,
      "activations/layer12_attention_weight_min": -30.3647518157959,
      "activations/layer13_attention_weight_max": 45.424808502197266,
      "activations/layer13_attention_weight_min": -32.609703063964844,
      "activations/layer14_attention_weight_max": 44.88416290283203,
      "activations/layer14_attention_weight_min": -30.444795608520508,
      "activations/layer15_attention_weight_max": 37.48685073852539,
      "activations/layer15_attention_weight_min": -31.70615005493164,
      "activations/layer16_attention_weight_max": 34.252479553222656,
      "activations/layer16_attention_weight_min": -29.874393463134766,
      "activations/layer17_attention_weight_max": 52.53236389160156,
      "activations/layer17_attention_weight_min": -48.21217727661133,
      "activations/layer18_attention_weight_max": 46.857200622558594,
      "activations/layer18_attention_weight_min": -42.94124984741211,
      "activations/layer19_attention_weight_max": 21.87593650817871,
      "activations/layer19_attention_weight_min": -23.147558212280273,
      "activations/layer1_attention_weight_max": 16.560972213745117,
      "activations/layer1_attention_weight_min": -15.660568237304688,
      "activations/layer20_attention_weight_max": 22.97230339050293,
      "activations/layer20_attention_weight_min": -26.008115768432617,
      "activations/layer21_attention_weight_max": 41.91646194458008,
      "activations/layer21_attention_weight_min": -24.146764755249023,
      "activations/layer22_attention_weight_max": 31.36322593688965,
      "activations/layer22_attention_weight_min": -26.131208419799805,
      "activations/layer23_attention_weight_max": 47.30231475830078,
      "activations/layer23_attention_weight_min": -25.15786361694336,
      "activations/layer2_attention_weight_max": 35.05781936645508,
      "activations/layer2_attention_weight_min": -32.066532135009766,
      "activations/layer3_attention_weight_max": 92.13590240478516,
      "activations/layer3_attention_weight_min": -94.73876953125,
      "activations/layer4_attention_weight_max": 99.1798095703125,
      "activations/layer4_attention_weight_min": -94.27350616455078,
      "activations/layer5_attention_weight_max": 73.50672912597656,
      "activations/layer5_attention_weight_min": -77.94485473632812,
      "activations/layer6_attention_weight_max": 53.600955963134766,
      "activations/layer6_attention_weight_min": -53.08876037597656,
      "activations/layer7_attention_weight_max": 74.03534698486328,
      "activations/layer7_attention_weight_min": -72.29821014404297,
      "activations/layer8_attention_weight_max": 44.86940383911133,
      "activations/layer8_attention_weight_min": -44.92219161987305,
      "activations/layer9_attention_weight_max": 48.42951965332031,
      "activations/layer9_attention_weight_min": -49.864013671875,
      "epoch": 11.11,
      "learning_rate": 7.917234848484847e-05,
      "loss": 2.7741,
      "step": 191150
    },
    {
      "activations/layer0_attention_weight_max": 15.91027545928955,
      "activations/layer0_attention_weight_min": -14.100356101989746,
      "activations/layer10_attention_weight_max": 38.02204132080078,
      "activations/layer10_attention_weight_min": -36.24142074584961,
      "activations/layer11_attention_weight_max": 33.319393157958984,
      "activations/layer11_attention_weight_min": -33.493186950683594,
      "activations/layer12_attention_weight_max": 18.765384674072266,
      "activations/layer12_attention_weight_min": -26.656478881835938,
      "activations/layer13_attention_weight_max": 37.48681640625,
      "activations/layer13_attention_weight_min": -30.08411407470703,
      "activations/layer14_attention_weight_max": 37.660125732421875,
      "activations/layer14_attention_weight_min": -28.964282989501953,
      "activations/layer15_attention_weight_max": 32.40850067138672,
      "activations/layer15_attention_weight_min": -29.289613723754883,
      "activations/layer16_attention_weight_max": 28.628217697143555,
      "activations/layer16_attention_weight_min": -28.199399948120117,
      "activations/layer17_attention_weight_max": 50.382015228271484,
      "activations/layer17_attention_weight_min": -42.52296447753906,
      "activations/layer18_attention_weight_max": 45.03218460083008,
      "activations/layer18_attention_weight_min": -36.554046630859375,
      "activations/layer19_attention_weight_max": 20.7227840423584,
      "activations/layer19_attention_weight_min": -21.908374786376953,
      "activations/layer1_attention_weight_max": 17.347776412963867,
      "activations/layer1_attention_weight_min": -15.833016395568848,
      "activations/layer20_attention_weight_max": 21.318519592285156,
      "activations/layer20_attention_weight_min": -21.93197250366211,
      "activations/layer21_attention_weight_max": 33.200469970703125,
      "activations/layer21_attention_weight_min": -22.003665924072266,
      "activations/layer22_attention_weight_max": 27.993364334106445,
      "activations/layer22_attention_weight_min": -23.447551727294922,
      "activations/layer23_attention_weight_max": 37.18501281738281,
      "activations/layer23_attention_weight_min": -24.576034545898438,
      "activations/layer2_attention_weight_max": 34.183082580566406,
      "activations/layer2_attention_weight_min": -32.984127044677734,
      "activations/layer3_attention_weight_max": 95.12850952148438,
      "activations/layer3_attention_weight_min": -95.61768341064453,
      "activations/layer4_attention_weight_max": 102.57682037353516,
      "activations/layer4_attention_weight_min": -99.20854187011719,
      "activations/layer5_attention_weight_max": 73.23896789550781,
      "activations/layer5_attention_weight_min": -77.54548645019531,
      "activations/layer6_attention_weight_max": 56.18833541870117,
      "activations/layer6_attention_weight_min": -56.003517150878906,
      "activations/layer7_attention_weight_max": 73.72346496582031,
      "activations/layer7_attention_weight_min": -73.6839370727539,
      "activations/layer8_attention_weight_max": 48.12343215942383,
      "activations/layer8_attention_weight_min": -50.83426284790039,
      "activations/layer9_attention_weight_max": 56.70978927612305,
      "activations/layer9_attention_weight_min": -49.586708068847656,
      "epoch": 11.11,
      "learning_rate": 7.915340909090909e-05,
      "loss": 2.7829,
      "step": 191200
    },
    {
      "activations/layer0_attention_weight_max": 16.295761108398438,
      "activations/layer0_attention_weight_min": -14.04317569732666,
      "activations/layer10_attention_weight_max": 43.08839797973633,
      "activations/layer10_attention_weight_min": -40.076316833496094,
      "activations/layer11_attention_weight_max": 38.897762298583984,
      "activations/layer11_attention_weight_min": -37.068870544433594,
      "activations/layer12_attention_weight_max": 19.579336166381836,
      "activations/layer12_attention_weight_min": -22.8725528717041,
      "activations/layer13_attention_weight_max": 45.89839553833008,
      "activations/layer13_attention_weight_min": -32.80179977416992,
      "activations/layer14_attention_weight_max": 43.96750259399414,
      "activations/layer14_attention_weight_min": -32.26938247680664,
      "activations/layer15_attention_weight_max": 36.813194274902344,
      "activations/layer15_attention_weight_min": -29.70693016052246,
      "activations/layer16_attention_weight_max": 34.320594787597656,
      "activations/layer16_attention_weight_min": -29.932716369628906,
      "activations/layer17_attention_weight_max": 54.607120513916016,
      "activations/layer17_attention_weight_min": -43.878597259521484,
      "activations/layer18_attention_weight_max": 49.7160758972168,
      "activations/layer18_attention_weight_min": -39.09116744995117,
      "activations/layer19_attention_weight_max": 23.62815284729004,
      "activations/layer19_attention_weight_min": -21.800457000732422,
      "activations/layer1_attention_weight_max": 17.73179817199707,
      "activations/layer1_attention_weight_min": -15.454975128173828,
      "activations/layer20_attention_weight_max": 25.315519332885742,
      "activations/layer20_attention_weight_min": -24.23833465576172,
      "activations/layer21_attention_weight_max": 41.10121154785156,
      "activations/layer21_attention_weight_min": -23.3642520904541,
      "activations/layer22_attention_weight_max": 36.14435577392578,
      "activations/layer22_attention_weight_min": -25.77119255065918,
      "activations/layer23_attention_weight_max": 45.415992736816406,
      "activations/layer23_attention_weight_min": -23.901973724365234,
      "activations/layer2_attention_weight_max": 33.92466735839844,
      "activations/layer2_attention_weight_min": -32.30405044555664,
      "activations/layer3_attention_weight_max": 99.50067138671875,
      "activations/layer3_attention_weight_min": -101.3830795288086,
      "activations/layer4_attention_weight_max": 100.8886947631836,
      "activations/layer4_attention_weight_min": -100.42949676513672,
      "activations/layer5_attention_weight_max": 73.78089904785156,
      "activations/layer5_attention_weight_min": -76.39794921875,
      "activations/layer6_attention_weight_max": 56.90241622924805,
      "activations/layer6_attention_weight_min": -53.442264556884766,
      "activations/layer7_attention_weight_max": 79.01138305664062,
      "activations/layer7_attention_weight_min": -74.77599334716797,
      "activations/layer8_attention_weight_max": 48.991050720214844,
      "activations/layer8_attention_weight_min": -49.129493713378906,
      "activations/layer9_attention_weight_max": 55.39168930053711,
      "activations/layer9_attention_weight_min": -51.08933639526367,
      "epoch": 11.11,
      "learning_rate": 7.913446969696968e-05,
      "loss": 2.7718,
      "step": 191250
    },
    {
      "activations/layer0_attention_weight_max": 16.026111602783203,
      "activations/layer0_attention_weight_min": -14.228483200073242,
      "activations/layer10_attention_weight_max": 37.07518005371094,
      "activations/layer10_attention_weight_min": -35.8540153503418,
      "activations/layer11_attention_weight_max": 32.09180450439453,
      "activations/layer11_attention_weight_min": -32.092506408691406,
      "activations/layer12_attention_weight_max": 18.195709228515625,
      "activations/layer12_attention_weight_min": -25.79218292236328,
      "activations/layer13_attention_weight_max": 41.11881637573242,
      "activations/layer13_attention_weight_min": -31.384004592895508,
      "activations/layer14_attention_weight_max": 36.207435607910156,
      "activations/layer14_attention_weight_min": -31.288700103759766,
      "activations/layer15_attention_weight_max": 34.0408935546875,
      "activations/layer15_attention_weight_min": -30.598682403564453,
      "activations/layer16_attention_weight_max": 32.45770263671875,
      "activations/layer16_attention_weight_min": -29.09800148010254,
      "activations/layer17_attention_weight_max": 54.20613479614258,
      "activations/layer17_attention_weight_min": -44.11685562133789,
      "activations/layer18_attention_weight_max": 51.64420700073242,
      "activations/layer18_attention_weight_min": -36.44850540161133,
      "activations/layer19_attention_weight_max": 20.571165084838867,
      "activations/layer19_attention_weight_min": -20.842016220092773,
      "activations/layer1_attention_weight_max": 16.28866958618164,
      "activations/layer1_attention_weight_min": -15.389236450195312,
      "activations/layer20_attention_weight_max": 21.88689613342285,
      "activations/layer20_attention_weight_min": -20.95244026184082,
      "activations/layer21_attention_weight_max": 41.95266342163086,
      "activations/layer21_attention_weight_min": -22.28980255126953,
      "activations/layer22_attention_weight_max": 30.15854263305664,
      "activations/layer22_attention_weight_min": -25.634037017822266,
      "activations/layer23_attention_weight_max": 39.83937454223633,
      "activations/layer23_attention_weight_min": -23.437152862548828,
      "activations/layer2_attention_weight_max": 34.37652587890625,
      "activations/layer2_attention_weight_min": -32.45026779174805,
      "activations/layer3_attention_weight_max": 93.3934555053711,
      "activations/layer3_attention_weight_min": -98.17058563232422,
      "activations/layer4_attention_weight_max": 101.84716796875,
      "activations/layer4_attention_weight_min": -98.48136901855469,
      "activations/layer5_attention_weight_max": 69.2237319946289,
      "activations/layer5_attention_weight_min": -76.60906982421875,
      "activations/layer6_attention_weight_max": 53.36573028564453,
      "activations/layer6_attention_weight_min": -54.93219757080078,
      "activations/layer7_attention_weight_max": 80.1194839477539,
      "activations/layer7_attention_weight_min": -72.451904296875,
      "activations/layer8_attention_weight_max": 48.29030990600586,
      "activations/layer8_attention_weight_min": -50.91993713378906,
      "activations/layer9_attention_weight_max": 58.35415267944336,
      "activations/layer9_attention_weight_min": -47.035316467285156,
      "epoch": 11.12,
      "learning_rate": 7.911553030303029e-05,
      "loss": 2.7815,
      "step": 191300
    },
    {
      "activations/layer0_attention_weight_max": 16.176977157592773,
      "activations/layer0_attention_weight_min": -14.810812950134277,
      "activations/layer10_attention_weight_max": 36.47266387939453,
      "activations/layer10_attention_weight_min": -34.8429069519043,
      "activations/layer11_attention_weight_max": 32.74549102783203,
      "activations/layer11_attention_weight_min": -32.345916748046875,
      "activations/layer12_attention_weight_max": 18.518598556518555,
      "activations/layer12_attention_weight_min": -28.120559692382812,
      "activations/layer13_attention_weight_max": 39.77397918701172,
      "activations/layer13_attention_weight_min": -28.504552841186523,
      "activations/layer14_attention_weight_max": 40.10448455810547,
      "activations/layer14_attention_weight_min": -30.320560455322266,
      "activations/layer15_attention_weight_max": 35.86849594116211,
      "activations/layer15_attention_weight_min": -30.59225845336914,
      "activations/layer16_attention_weight_max": 32.4482536315918,
      "activations/layer16_attention_weight_min": -29.800464630126953,
      "activations/layer17_attention_weight_max": 54.86406707763672,
      "activations/layer17_attention_weight_min": -42.6717529296875,
      "activations/layer18_attention_weight_max": 52.66008758544922,
      "activations/layer18_attention_weight_min": -37.51500701904297,
      "activations/layer19_attention_weight_max": 23.893877029418945,
      "activations/layer19_attention_weight_min": -21.454301834106445,
      "activations/layer1_attention_weight_max": 18.286699295043945,
      "activations/layer1_attention_weight_min": -15.034750938415527,
      "activations/layer20_attention_weight_max": 24.03216552734375,
      "activations/layer20_attention_weight_min": -21.72646713256836,
      "activations/layer21_attention_weight_max": 36.953102111816406,
      "activations/layer21_attention_weight_min": -24.326318740844727,
      "activations/layer22_attention_weight_max": 33.72508239746094,
      "activations/layer22_attention_weight_min": -25.090547561645508,
      "activations/layer23_attention_weight_max": 39.90849304199219,
      "activations/layer23_attention_weight_min": -24.534008026123047,
      "activations/layer2_attention_weight_max": 33.683372497558594,
      "activations/layer2_attention_weight_min": -30.240102767944336,
      "activations/layer3_attention_weight_max": 89.07825469970703,
      "activations/layer3_attention_weight_min": -94.47488403320312,
      "activations/layer4_attention_weight_max": 95.73020935058594,
      "activations/layer4_attention_weight_min": -92.55205535888672,
      "activations/layer5_attention_weight_max": 69.58804321289062,
      "activations/layer5_attention_weight_min": -80.19880676269531,
      "activations/layer6_attention_weight_max": 52.76738739013672,
      "activations/layer6_attention_weight_min": -50.31697463989258,
      "activations/layer7_attention_weight_max": 73.64191436767578,
      "activations/layer7_attention_weight_min": -69.49418640136719,
      "activations/layer8_attention_weight_max": 45.26308059692383,
      "activations/layer8_attention_weight_min": -47.890541076660156,
      "activations/layer9_attention_weight_max": 44.585243225097656,
      "activations/layer9_attention_weight_min": -45.81423568725586,
      "epoch": 11.12,
      "learning_rate": 7.909659090909091e-05,
      "loss": 2.7611,
      "step": 191350
    },
    {
      "activations/layer0_attention_weight_max": 16.745105743408203,
      "activations/layer0_attention_weight_min": -14.657051086425781,
      "activations/layer10_attention_weight_max": 36.68938064575195,
      "activations/layer10_attention_weight_min": -34.77513885498047,
      "activations/layer11_attention_weight_max": 32.71064758300781,
      "activations/layer11_attention_weight_min": -32.589111328125,
      "activations/layer12_attention_weight_max": 29.405189514160156,
      "activations/layer12_attention_weight_min": -25.438154220581055,
      "activations/layer13_attention_weight_max": 41.3078498840332,
      "activations/layer13_attention_weight_min": -30.48458480834961,
      "activations/layer14_attention_weight_max": 37.98326110839844,
      "activations/layer14_attention_weight_min": -29.937255859375,
      "activations/layer15_attention_weight_max": 33.463932037353516,
      "activations/layer15_attention_weight_min": -28.15457534790039,
      "activations/layer16_attention_weight_max": 32.59734344482422,
      "activations/layer16_attention_weight_min": -29.71302604675293,
      "activations/layer17_attention_weight_max": 56.814918518066406,
      "activations/layer17_attention_weight_min": -44.92301940917969,
      "activations/layer18_attention_weight_max": 48.24825668334961,
      "activations/layer18_attention_weight_min": -36.03459930419922,
      "activations/layer19_attention_weight_max": 22.408002853393555,
      "activations/layer19_attention_weight_min": -22.751157760620117,
      "activations/layer1_attention_weight_max": 17.4606990814209,
      "activations/layer1_attention_weight_min": -14.62335205078125,
      "activations/layer20_attention_weight_max": 22.893959045410156,
      "activations/layer20_attention_weight_min": -22.2675838470459,
      "activations/layer21_attention_weight_max": 37.526206970214844,
      "activations/layer21_attention_weight_min": -24.15916633605957,
      "activations/layer22_attention_weight_max": 31.931299209594727,
      "activations/layer22_attention_weight_min": -25.381473541259766,
      "activations/layer23_attention_weight_max": 40.029510498046875,
      "activations/layer23_attention_weight_min": -22.426998138427734,
      "activations/layer2_attention_weight_max": 34.69386291503906,
      "activations/layer2_attention_weight_min": -31.592830657958984,
      "activations/layer3_attention_weight_max": 91.11499786376953,
      "activations/layer3_attention_weight_min": -95.20228576660156,
      "activations/layer4_attention_weight_max": 95.68539428710938,
      "activations/layer4_attention_weight_min": -93.16460418701172,
      "activations/layer5_attention_weight_max": 72.48331451416016,
      "activations/layer5_attention_weight_min": -76.03350830078125,
      "activations/layer6_attention_weight_max": 53.66897964477539,
      "activations/layer6_attention_weight_min": -52.75336837768555,
      "activations/layer7_attention_weight_max": 81.29015350341797,
      "activations/layer7_attention_weight_min": -77.54652404785156,
      "activations/layer8_attention_weight_max": 46.734588623046875,
      "activations/layer8_attention_weight_min": -50.73284149169922,
      "activations/layer9_attention_weight_max": 51.46167755126953,
      "activations/layer9_attention_weight_min": -46.99961471557617,
      "epoch": 11.12,
      "learning_rate": 7.90776515151515e-05,
      "loss": 2.7875,
      "step": 191400
    },
    {
      "activations/layer0_attention_weight_max": 15.768855094909668,
      "activations/layer0_attention_weight_min": -14.889203071594238,
      "activations/layer10_attention_weight_max": 35.85115432739258,
      "activations/layer10_attention_weight_min": -35.15406036376953,
      "activations/layer11_attention_weight_max": 31.083541870117188,
      "activations/layer11_attention_weight_min": -33.819374084472656,
      "activations/layer12_attention_weight_max": 18.5533447265625,
      "activations/layer12_attention_weight_min": -26.620803833007812,
      "activations/layer13_attention_weight_max": 38.4465217590332,
      "activations/layer13_attention_weight_min": -34.1370849609375,
      "activations/layer14_attention_weight_max": 36.953758239746094,
      "activations/layer14_attention_weight_min": -30.86605453491211,
      "activations/layer15_attention_weight_max": 31.018308639526367,
      "activations/layer15_attention_weight_min": -30.3211669921875,
      "activations/layer16_attention_weight_max": 31.483911514282227,
      "activations/layer16_attention_weight_min": -29.82693862915039,
      "activations/layer17_attention_weight_max": 47.054779052734375,
      "activations/layer17_attention_weight_min": -42.676395416259766,
      "activations/layer18_attention_weight_max": 42.79180145263672,
      "activations/layer18_attention_weight_min": -36.416194915771484,
      "activations/layer19_attention_weight_max": 21.37616539001465,
      "activations/layer19_attention_weight_min": -22.635454177856445,
      "activations/layer1_attention_weight_max": 16.187519073486328,
      "activations/layer1_attention_weight_min": -14.723825454711914,
      "activations/layer20_attention_weight_max": 21.03582191467285,
      "activations/layer20_attention_weight_min": -23.766965866088867,
      "activations/layer21_attention_weight_max": 40.38359832763672,
      "activations/layer21_attention_weight_min": -23.8634090423584,
      "activations/layer22_attention_weight_max": 33.096248626708984,
      "activations/layer22_attention_weight_min": -26.74675941467285,
      "activations/layer23_attention_weight_max": 39.543941497802734,
      "activations/layer23_attention_weight_min": -23.728836059570312,
      "activations/layer2_attention_weight_max": 31.80500602722168,
      "activations/layer2_attention_weight_min": -29.16460418701172,
      "activations/layer3_attention_weight_max": 89.66727447509766,
      "activations/layer3_attention_weight_min": -89.54378509521484,
      "activations/layer4_attention_weight_max": 96.64574432373047,
      "activations/layer4_attention_weight_min": -91.3022689819336,
      "activations/layer5_attention_weight_max": 69.72850036621094,
      "activations/layer5_attention_weight_min": -75.64923095703125,
      "activations/layer6_attention_weight_max": 54.55350112915039,
      "activations/layer6_attention_weight_min": -53.9209098815918,
      "activations/layer7_attention_weight_max": 76.7307357788086,
      "activations/layer7_attention_weight_min": -73.52567291259766,
      "activations/layer8_attention_weight_max": 44.47979736328125,
      "activations/layer8_attention_weight_min": -46.052581787109375,
      "activations/layer9_attention_weight_max": 48.7177848815918,
      "activations/layer9_attention_weight_min": -49.23080062866211,
      "epoch": 11.12,
      "learning_rate": 7.905871212121211e-05,
      "loss": 2.7734,
      "step": 191450
    },
    {
      "activations/layer0_attention_weight_max": 15.630176544189453,
      "activations/layer0_attention_weight_min": -14.442668914794922,
      "activations/layer10_attention_weight_max": 38.94331741333008,
      "activations/layer10_attention_weight_min": -35.313926696777344,
      "activations/layer11_attention_weight_max": 35.13177490234375,
      "activations/layer11_attention_weight_min": -33.95490264892578,
      "activations/layer12_attention_weight_max": 20.30079460144043,
      "activations/layer12_attention_weight_min": -27.58043670654297,
      "activations/layer13_attention_weight_max": 41.94386672973633,
      "activations/layer13_attention_weight_min": -31.45003890991211,
      "activations/layer14_attention_weight_max": 42.2122802734375,
      "activations/layer14_attention_weight_min": -29.28605079650879,
      "activations/layer15_attention_weight_max": 37.056060791015625,
      "activations/layer15_attention_weight_min": -30.80575180053711,
      "activations/layer16_attention_weight_max": 35.980926513671875,
      "activations/layer16_attention_weight_min": -31.19517707824707,
      "activations/layer17_attention_weight_max": 55.20591354370117,
      "activations/layer17_attention_weight_min": -47.975257873535156,
      "activations/layer18_attention_weight_max": 51.3313102722168,
      "activations/layer18_attention_weight_min": -39.3545036315918,
      "activations/layer19_attention_weight_max": 26.096710205078125,
      "activations/layer19_attention_weight_min": -24.084535598754883,
      "activations/layer1_attention_weight_max": 17.33401107788086,
      "activations/layer1_attention_weight_min": -14.589020729064941,
      "activations/layer20_attention_weight_max": 28.347566604614258,
      "activations/layer20_attention_weight_min": -24.635143280029297,
      "activations/layer21_attention_weight_max": 45.8742561340332,
      "activations/layer21_attention_weight_min": -22.98468017578125,
      "activations/layer22_attention_weight_max": 33.142601013183594,
      "activations/layer22_attention_weight_min": -27.9129638671875,
      "activations/layer23_attention_weight_max": 43.56134796142578,
      "activations/layer23_attention_weight_min": -23.578062057495117,
      "activations/layer2_attention_weight_max": 33.26019287109375,
      "activations/layer2_attention_weight_min": -29.900409698486328,
      "activations/layer3_attention_weight_max": 94.69438171386719,
      "activations/layer3_attention_weight_min": -96.24412536621094,
      "activations/layer4_attention_weight_max": 100.73890686035156,
      "activations/layer4_attention_weight_min": -91.19589233398438,
      "activations/layer5_attention_weight_max": 69.91152954101562,
      "activations/layer5_attention_weight_min": -77.04959869384766,
      "activations/layer6_attention_weight_max": 54.432682037353516,
      "activations/layer6_attention_weight_min": -50.257118225097656,
      "activations/layer7_attention_weight_max": 76.79265594482422,
      "activations/layer7_attention_weight_min": -73.664306640625,
      "activations/layer8_attention_weight_max": 45.17082977294922,
      "activations/layer8_attention_weight_min": -46.19429016113281,
      "activations/layer9_attention_weight_max": 52.0098762512207,
      "activations/layer9_attention_weight_min": -48.780155181884766,
      "epoch": 11.13,
      "learning_rate": 7.903977272727273e-05,
      "loss": 2.7734,
      "step": 191500
    },
    {
      "activations/layer0_attention_weight_max": 16.43622589111328,
      "activations/layer0_attention_weight_min": -14.633277893066406,
      "activations/layer10_attention_weight_max": 34.08855056762695,
      "activations/layer10_attention_weight_min": -33.391258239746094,
      "activations/layer11_attention_weight_max": 30.66451644897461,
      "activations/layer11_attention_weight_min": -31.55315399169922,
      "activations/layer12_attention_weight_max": 18.383569717407227,
      "activations/layer12_attention_weight_min": -25.382368087768555,
      "activations/layer13_attention_weight_max": 37.29975891113281,
      "activations/layer13_attention_weight_min": -28.714256286621094,
      "activations/layer14_attention_weight_max": 39.6513786315918,
      "activations/layer14_attention_weight_min": -30.189851760864258,
      "activations/layer15_attention_weight_max": 36.54714584350586,
      "activations/layer15_attention_weight_min": -29.104381561279297,
      "activations/layer16_attention_weight_max": 33.77851486206055,
      "activations/layer16_attention_weight_min": -32.19150161743164,
      "activations/layer17_attention_weight_max": 58.64371109008789,
      "activations/layer17_attention_weight_min": -47.73781204223633,
      "activations/layer18_attention_weight_max": 51.245094299316406,
      "activations/layer18_attention_weight_min": -36.28729248046875,
      "activations/layer19_attention_weight_max": 24.0323486328125,
      "activations/layer19_attention_weight_min": -23.947254180908203,
      "activations/layer1_attention_weight_max": 16.316539764404297,
      "activations/layer1_attention_weight_min": -16.289966583251953,
      "activations/layer20_attention_weight_max": 24.277618408203125,
      "activations/layer20_attention_weight_min": -25.043704986572266,
      "activations/layer21_attention_weight_max": 42.68082809448242,
      "activations/layer21_attention_weight_min": -25.91011619567871,
      "activations/layer22_attention_weight_max": 31.5782527923584,
      "activations/layer22_attention_weight_min": -24.929670333862305,
      "activations/layer23_attention_weight_max": 39.21263122558594,
      "activations/layer23_attention_weight_min": -25.035743713378906,
      "activations/layer2_attention_weight_max": 32.40373992919922,
      "activations/layer2_attention_weight_min": -30.509498596191406,
      "activations/layer3_attention_weight_max": 89.83869171142578,
      "activations/layer3_attention_weight_min": -94.497314453125,
      "activations/layer4_attention_weight_max": 93.93033599853516,
      "activations/layer4_attention_weight_min": -91.97429656982422,
      "activations/layer5_attention_weight_max": 67.25527954101562,
      "activations/layer5_attention_weight_min": -78.34674072265625,
      "activations/layer6_attention_weight_max": 51.85774612426758,
      "activations/layer6_attention_weight_min": -52.792442321777344,
      "activations/layer7_attention_weight_max": 73.33966827392578,
      "activations/layer7_attention_weight_min": -73.41929626464844,
      "activations/layer8_attention_weight_max": 45.35457992553711,
      "activations/layer8_attention_weight_min": -43.667869567871094,
      "activations/layer9_attention_weight_max": 46.436981201171875,
      "activations/layer9_attention_weight_min": -44.50653076171875,
      "epoch": 11.13,
      "learning_rate": 7.902083333333333e-05,
      "loss": 2.7841,
      "step": 191550
    },
    {
      "activations/layer0_attention_weight_max": 15.398144721984863,
      "activations/layer0_attention_weight_min": -14.122161865234375,
      "activations/layer10_attention_weight_max": 35.38957214355469,
      "activations/layer10_attention_weight_min": -35.33552551269531,
      "activations/layer11_attention_weight_max": 31.502655029296875,
      "activations/layer11_attention_weight_min": -30.954145431518555,
      "activations/layer12_attention_weight_max": 17.232437133789062,
      "activations/layer12_attention_weight_min": -24.878536224365234,
      "activations/layer13_attention_weight_max": 37.79038619995117,
      "activations/layer13_attention_weight_min": -30.246017456054688,
      "activations/layer14_attention_weight_max": 35.55156707763672,
      "activations/layer14_attention_weight_min": -28.94002342224121,
      "activations/layer15_attention_weight_max": 32.51775360107422,
      "activations/layer15_attention_weight_min": -28.199729919433594,
      "activations/layer16_attention_weight_max": 29.28512191772461,
      "activations/layer16_attention_weight_min": -28.77376937866211,
      "activations/layer17_attention_weight_max": 46.815216064453125,
      "activations/layer17_attention_weight_min": -41.2994270324707,
      "activations/layer18_attention_weight_max": 43.44289779663086,
      "activations/layer18_attention_weight_min": -34.73688888549805,
      "activations/layer19_attention_weight_max": 23.653413772583008,
      "activations/layer19_attention_weight_min": -20.772924423217773,
      "activations/layer1_attention_weight_max": 18.091794967651367,
      "activations/layer1_attention_weight_min": -14.324366569519043,
      "activations/layer20_attention_weight_max": 23.680997848510742,
      "activations/layer20_attention_weight_min": -20.649864196777344,
      "activations/layer21_attention_weight_max": 36.984832763671875,
      "activations/layer21_attention_weight_min": -22.838491439819336,
      "activations/layer22_attention_weight_max": 33.661346435546875,
      "activations/layer22_attention_weight_min": -25.641748428344727,
      "activations/layer23_attention_weight_max": 35.543357849121094,
      "activations/layer23_attention_weight_min": -24.13873863220215,
      "activations/layer2_attention_weight_max": 32.32394790649414,
      "activations/layer2_attention_weight_min": -28.91790771484375,
      "activations/layer3_attention_weight_max": 88.8623046875,
      "activations/layer3_attention_weight_min": -93.83370208740234,
      "activations/layer4_attention_weight_max": 96.18331146240234,
      "activations/layer4_attention_weight_min": -93.71890258789062,
      "activations/layer5_attention_weight_max": 69.42253112792969,
      "activations/layer5_attention_weight_min": -76.06010437011719,
      "activations/layer6_attention_weight_max": 51.471431732177734,
      "activations/layer6_attention_weight_min": -50.332576751708984,
      "activations/layer7_attention_weight_max": 73.788330078125,
      "activations/layer7_attention_weight_min": -69.01329040527344,
      "activations/layer8_attention_weight_max": 43.11365509033203,
      "activations/layer8_attention_weight_min": -44.49678039550781,
      "activations/layer9_attention_weight_max": 45.163818359375,
      "activations/layer9_attention_weight_min": -44.72019577026367,
      "epoch": 11.13,
      "learning_rate": 7.900189393939393e-05,
      "loss": 2.7859,
      "step": 191600
    },
    {
      "activations/layer0_attention_weight_max": 15.573416709899902,
      "activations/layer0_attention_weight_min": -14.495455741882324,
      "activations/layer10_attention_weight_max": 36.453826904296875,
      "activations/layer10_attention_weight_min": -37.05573654174805,
      "activations/layer11_attention_weight_max": 33.26268768310547,
      "activations/layer11_attention_weight_min": -33.570220947265625,
      "activations/layer12_attention_weight_max": 19.79486083984375,
      "activations/layer12_attention_weight_min": -26.128604888916016,
      "activations/layer13_attention_weight_max": 44.1533317565918,
      "activations/layer13_attention_weight_min": -37.78071594238281,
      "activations/layer14_attention_weight_max": 44.24095153808594,
      "activations/layer14_attention_weight_min": -32.57811737060547,
      "activations/layer15_attention_weight_max": 39.33608627319336,
      "activations/layer15_attention_weight_min": -30.39008903503418,
      "activations/layer16_attention_weight_max": 30.341541290283203,
      "activations/layer16_attention_weight_min": -29.30999755859375,
      "activations/layer17_attention_weight_max": 51.58058547973633,
      "activations/layer17_attention_weight_min": -44.387996673583984,
      "activations/layer18_attention_weight_max": 47.52992630004883,
      "activations/layer18_attention_weight_min": -36.72256851196289,
      "activations/layer19_attention_weight_max": 21.576461791992188,
      "activations/layer19_attention_weight_min": -22.85869598388672,
      "activations/layer1_attention_weight_max": 17.77876091003418,
      "activations/layer1_attention_weight_min": -15.10556697845459,
      "activations/layer20_attention_weight_max": 22.904647827148438,
      "activations/layer20_attention_weight_min": -19.863059997558594,
      "activations/layer21_attention_weight_max": 41.2246208190918,
      "activations/layer21_attention_weight_min": -27.015594482421875,
      "activations/layer22_attention_weight_max": 29.654308319091797,
      "activations/layer22_attention_weight_min": -23.264869689941406,
      "activations/layer23_attention_weight_max": 36.94141387939453,
      "activations/layer23_attention_weight_min": -23.855985641479492,
      "activations/layer2_attention_weight_max": 32.943817138671875,
      "activations/layer2_attention_weight_min": -30.8377742767334,
      "activations/layer3_attention_weight_max": 92.99168395996094,
      "activations/layer3_attention_weight_min": -94.57068634033203,
      "activations/layer4_attention_weight_max": 98.2110595703125,
      "activations/layer4_attention_weight_min": -94.93538665771484,
      "activations/layer5_attention_weight_max": 69.21867370605469,
      "activations/layer5_attention_weight_min": -78.63885498046875,
      "activations/layer6_attention_weight_max": 53.06035232543945,
      "activations/layer6_attention_weight_min": -53.708457946777344,
      "activations/layer7_attention_weight_max": 74.33666229248047,
      "activations/layer7_attention_weight_min": -77.95350646972656,
      "activations/layer8_attention_weight_max": 47.11342239379883,
      "activations/layer8_attention_weight_min": -48.85774612426758,
      "activations/layer9_attention_weight_max": 46.514015197753906,
      "activations/layer9_attention_weight_min": -49.03768539428711,
      "epoch": 11.14,
      "learning_rate": 7.898295454545454e-05,
      "loss": 2.775,
      "step": 191650
    },
    {
      "activations/layer0_attention_weight_max": 16.15732765197754,
      "activations/layer0_attention_weight_min": -14.534809112548828,
      "activations/layer10_attention_weight_max": 38.02647018432617,
      "activations/layer10_attention_weight_min": -36.05010986328125,
      "activations/layer11_attention_weight_max": 31.607208251953125,
      "activations/layer11_attention_weight_min": -31.378873825073242,
      "activations/layer12_attention_weight_max": 18.972341537475586,
      "activations/layer12_attention_weight_min": -24.858192443847656,
      "activations/layer13_attention_weight_max": 41.40354537963867,
      "activations/layer13_attention_weight_min": -30.24839973449707,
      "activations/layer14_attention_weight_max": 34.274925231933594,
      "activations/layer14_attention_weight_min": -28.27126121520996,
      "activations/layer15_attention_weight_max": 31.652538299560547,
      "activations/layer15_attention_weight_min": -29.31310272216797,
      "activations/layer16_attention_weight_max": 32.47430419921875,
      "activations/layer16_attention_weight_min": -28.07497215270996,
      "activations/layer17_attention_weight_max": 49.550140380859375,
      "activations/layer17_attention_weight_min": -42.8763313293457,
      "activations/layer18_attention_weight_max": 43.76791763305664,
      "activations/layer18_attention_weight_min": -36.44508361816406,
      "activations/layer19_attention_weight_max": 22.702157974243164,
      "activations/layer19_attention_weight_min": -22.23142433166504,
      "activations/layer1_attention_weight_max": 16.276180267333984,
      "activations/layer1_attention_weight_min": -15.16645622253418,
      "activations/layer20_attention_weight_max": 23.26689910888672,
      "activations/layer20_attention_weight_min": -19.896921157836914,
      "activations/layer21_attention_weight_max": 39.37652587890625,
      "activations/layer21_attention_weight_min": -23.351957321166992,
      "activations/layer22_attention_weight_max": 29.75766944885254,
      "activations/layer22_attention_weight_min": -24.470727920532227,
      "activations/layer23_attention_weight_max": 41.834983825683594,
      "activations/layer23_attention_weight_min": -25.52497673034668,
      "activations/layer2_attention_weight_max": 32.75096893310547,
      "activations/layer2_attention_weight_min": -32.78233337402344,
      "activations/layer3_attention_weight_max": 93.54459381103516,
      "activations/layer3_attention_weight_min": -101.99566650390625,
      "activations/layer4_attention_weight_max": 98.7455825805664,
      "activations/layer4_attention_weight_min": -99.80086517333984,
      "activations/layer5_attention_weight_max": 73.92892456054688,
      "activations/layer5_attention_weight_min": -75.87425231933594,
      "activations/layer6_attention_weight_max": 53.516441345214844,
      "activations/layer6_attention_weight_min": -53.12107467651367,
      "activations/layer7_attention_weight_max": 79.04251098632812,
      "activations/layer7_attention_weight_min": -71.937255859375,
      "activations/layer8_attention_weight_max": 45.28171157836914,
      "activations/layer8_attention_weight_min": -46.548927307128906,
      "activations/layer9_attention_weight_max": 53.42035675048828,
      "activations/layer9_attention_weight_min": -46.8931999206543,
      "epoch": 11.14,
      "learning_rate": 7.896439393939393e-05,
      "loss": 2.7687,
      "step": 191700
    },
    {
      "activations/layer0_attention_weight_max": 15.52151870727539,
      "activations/layer0_attention_weight_min": -14.643295288085938,
      "activations/layer10_attention_weight_max": 36.65929412841797,
      "activations/layer10_attention_weight_min": -34.65242385864258,
      "activations/layer11_attention_weight_max": 31.015926361083984,
      "activations/layer11_attention_weight_min": -32.18876647949219,
      "activations/layer12_attention_weight_max": 25.889564514160156,
      "activations/layer12_attention_weight_min": -23.96988868713379,
      "activations/layer13_attention_weight_max": 38.436424255371094,
      "activations/layer13_attention_weight_min": -28.78672218322754,
      "activations/layer14_attention_weight_max": 39.3050422668457,
      "activations/layer14_attention_weight_min": -31.1394100189209,
      "activations/layer15_attention_weight_max": 35.319984436035156,
      "activations/layer15_attention_weight_min": -29.0502986907959,
      "activations/layer16_attention_weight_max": 36.22482681274414,
      "activations/layer16_attention_weight_min": -29.618494033813477,
      "activations/layer17_attention_weight_max": 54.67279052734375,
      "activations/layer17_attention_weight_min": -43.86294174194336,
      "activations/layer18_attention_weight_max": 54.68845748901367,
      "activations/layer18_attention_weight_min": -38.84105682373047,
      "activations/layer19_attention_weight_max": 22.65775489807129,
      "activations/layer19_attention_weight_min": -21.62811279296875,
      "activations/layer1_attention_weight_max": 18.329648971557617,
      "activations/layer1_attention_weight_min": -14.63624382019043,
      "activations/layer20_attention_weight_max": 25.388427734375,
      "activations/layer20_attention_weight_min": -20.282546997070312,
      "activations/layer21_attention_weight_max": 46.1860466003418,
      "activations/layer21_attention_weight_min": -22.70250701904297,
      "activations/layer22_attention_weight_max": 33.2047119140625,
      "activations/layer22_attention_weight_min": -24.85622215270996,
      "activations/layer23_attention_weight_max": 41.886146545410156,
      "activations/layer23_attention_weight_min": -23.594951629638672,
      "activations/layer2_attention_weight_max": 32.304527282714844,
      "activations/layer2_attention_weight_min": -29.930652618408203,
      "activations/layer3_attention_weight_max": 94.84878540039062,
      "activations/layer3_attention_weight_min": -95.9844970703125,
      "activations/layer4_attention_weight_max": 97.16385650634766,
      "activations/layer4_attention_weight_min": -97.251220703125,
      "activations/layer5_attention_weight_max": 69.47499084472656,
      "activations/layer5_attention_weight_min": -75.14319610595703,
      "activations/layer6_attention_weight_max": 51.68163299560547,
      "activations/layer6_attention_weight_min": -51.42971420288086,
      "activations/layer7_attention_weight_max": 73.93224334716797,
      "activations/layer7_attention_weight_min": -70.83650970458984,
      "activations/layer8_attention_weight_max": 46.70528793334961,
      "activations/layer8_attention_weight_min": -46.821720123291016,
      "activations/layer9_attention_weight_max": 48.12217330932617,
      "activations/layer9_attention_weight_min": -48.82688522338867,
      "epoch": 11.14,
      "learning_rate": 7.894545454545454e-05,
      "loss": 2.7853,
      "step": 191750
    },
    {
      "activations/layer0_attention_weight_max": 15.967036247253418,
      "activations/layer0_attention_weight_min": -14.766942024230957,
      "activations/layer10_attention_weight_max": 40.5386848449707,
      "activations/layer10_attention_weight_min": -39.890506744384766,
      "activations/layer11_attention_weight_max": 37.18798828125,
      "activations/layer11_attention_weight_min": -36.12115478515625,
      "activations/layer12_attention_weight_max": 21.70453453063965,
      "activations/layer12_attention_weight_min": -25.011734008789062,
      "activations/layer13_attention_weight_max": 40.96867752075195,
      "activations/layer13_attention_weight_min": -28.1977596282959,
      "activations/layer14_attention_weight_max": 36.22938537597656,
      "activations/layer14_attention_weight_min": -28.110166549682617,
      "activations/layer15_attention_weight_max": 32.72484588623047,
      "activations/layer15_attention_weight_min": -28.1059513092041,
      "activations/layer16_attention_weight_max": 31.660974502563477,
      "activations/layer16_attention_weight_min": -26.904966354370117,
      "activations/layer17_attention_weight_max": 52.572757720947266,
      "activations/layer17_attention_weight_min": -39.73469161987305,
      "activations/layer18_attention_weight_max": 49.989681243896484,
      "activations/layer18_attention_weight_min": -35.99305725097656,
      "activations/layer19_attention_weight_max": 21.019916534423828,
      "activations/layer19_attention_weight_min": -19.986835479736328,
      "activations/layer1_attention_weight_max": 17.050294876098633,
      "activations/layer1_attention_weight_min": -14.325639724731445,
      "activations/layer20_attention_weight_max": 23.33422088623047,
      "activations/layer20_attention_weight_min": -23.61406135559082,
      "activations/layer21_attention_weight_max": 38.336219787597656,
      "activations/layer21_attention_weight_min": -21.864540100097656,
      "activations/layer22_attention_weight_max": 29.53939437866211,
      "activations/layer22_attention_weight_min": -24.912479400634766,
      "activations/layer23_attention_weight_max": 38.155174255371094,
      "activations/layer23_attention_weight_min": -25.816862106323242,
      "activations/layer2_attention_weight_max": 32.545902252197266,
      "activations/layer2_attention_weight_min": -31.07752799987793,
      "activations/layer3_attention_weight_max": 91.46463012695312,
      "activations/layer3_attention_weight_min": -97.82331085205078,
      "activations/layer4_attention_weight_max": 102.70403289794922,
      "activations/layer4_attention_weight_min": -97.51829528808594,
      "activations/layer5_attention_weight_max": 74.15292358398438,
      "activations/layer5_attention_weight_min": -77.88819885253906,
      "activations/layer6_attention_weight_max": 54.643638610839844,
      "activations/layer6_attention_weight_min": -53.41207504272461,
      "activations/layer7_attention_weight_max": 78.30667114257812,
      "activations/layer7_attention_weight_min": -76.35456848144531,
      "activations/layer8_attention_weight_max": 48.62571716308594,
      "activations/layer8_attention_weight_min": -49.11054992675781,
      "activations/layer9_attention_weight_max": 65.50942993164062,
      "activations/layer9_attention_weight_min": -51.07151412963867,
      "epoch": 11.14,
      "learning_rate": 7.892651515151515e-05,
      "loss": 2.7742,
      "step": 191800
    },
    {
      "activations/layer0_attention_weight_max": 16.289960861206055,
      "activations/layer0_attention_weight_min": -14.471573829650879,
      "activations/layer10_attention_weight_max": 41.7418212890625,
      "activations/layer10_attention_weight_min": -39.862396240234375,
      "activations/layer11_attention_weight_max": 34.1977653503418,
      "activations/layer11_attention_weight_min": -34.65763854980469,
      "activations/layer12_attention_weight_max": 29.281808853149414,
      "activations/layer12_attention_weight_min": -25.680944442749023,
      "activations/layer13_attention_weight_max": 46.34907531738281,
      "activations/layer13_attention_weight_min": -29.581087112426758,
      "activations/layer14_attention_weight_max": 47.23838806152344,
      "activations/layer14_attention_weight_min": -31.060405731201172,
      "activations/layer15_attention_weight_max": 40.678306579589844,
      "activations/layer15_attention_weight_min": -29.78143882751465,
      "activations/layer16_attention_weight_max": 33.4437255859375,
      "activations/layer16_attention_weight_min": -30.155961990356445,
      "activations/layer17_attention_weight_max": 58.76032257080078,
      "activations/layer17_attention_weight_min": -47.500282287597656,
      "activations/layer18_attention_weight_max": 51.266319274902344,
      "activations/layer18_attention_weight_min": -39.612159729003906,
      "activations/layer19_attention_weight_max": 24.04505729675293,
      "activations/layer19_attention_weight_min": -20.828670501708984,
      "activations/layer1_attention_weight_max": 16.89365577697754,
      "activations/layer1_attention_weight_min": -15.054337501525879,
      "activations/layer20_attention_weight_max": 22.658151626586914,
      "activations/layer20_attention_weight_min": -22.75029754638672,
      "activations/layer21_attention_weight_max": 37.53111267089844,
      "activations/layer21_attention_weight_min": -23.337615966796875,
      "activations/layer22_attention_weight_max": 32.07659149169922,
      "activations/layer22_attention_weight_min": -28.819129943847656,
      "activations/layer23_attention_weight_max": 39.91101837158203,
      "activations/layer23_attention_weight_min": -23.40685272216797,
      "activations/layer2_attention_weight_max": 34.259986877441406,
      "activations/layer2_attention_weight_min": -32.41213607788086,
      "activations/layer3_attention_weight_max": 96.79584503173828,
      "activations/layer3_attention_weight_min": -94.75577545166016,
      "activations/layer4_attention_weight_max": 104.02037811279297,
      "activations/layer4_attention_weight_min": -102.11238098144531,
      "activations/layer5_attention_weight_max": 73.0001220703125,
      "activations/layer5_attention_weight_min": -77.99488067626953,
      "activations/layer6_attention_weight_max": 55.99775695800781,
      "activations/layer6_attention_weight_min": -55.63978958129883,
      "activations/layer7_attention_weight_max": 80.85708618164062,
      "activations/layer7_attention_weight_min": -75.02403259277344,
      "activations/layer8_attention_weight_max": 49.18812561035156,
      "activations/layer8_attention_weight_min": -49.31122589111328,
      "activations/layer9_attention_weight_max": 48.42505645751953,
      "activations/layer9_attention_weight_min": -49.6938362121582,
      "epoch": 11.15,
      "learning_rate": 7.890757575757575e-05,
      "loss": 2.7856,
      "step": 191850
    },
    {
      "activations/layer0_attention_weight_max": 15.508838653564453,
      "activations/layer0_attention_weight_min": -14.517581939697266,
      "activations/layer10_attention_weight_max": 37.3134651184082,
      "activations/layer10_attention_weight_min": -36.0992431640625,
      "activations/layer11_attention_weight_max": 32.06119918823242,
      "activations/layer11_attention_weight_min": -32.03015899658203,
      "activations/layer12_attention_weight_max": 18.431396484375,
      "activations/layer12_attention_weight_min": -26.59017562866211,
      "activations/layer13_attention_weight_max": 41.37671661376953,
      "activations/layer13_attention_weight_min": -27.925392150878906,
      "activations/layer14_attention_weight_max": 38.24898147583008,
      "activations/layer14_attention_weight_min": -29.946857452392578,
      "activations/layer15_attention_weight_max": 32.83561706542969,
      "activations/layer15_attention_weight_min": -29.640798568725586,
      "activations/layer16_attention_weight_max": 30.869937896728516,
      "activations/layer16_attention_weight_min": -28.21063232421875,
      "activations/layer17_attention_weight_max": 52.83536911010742,
      "activations/layer17_attention_weight_min": -41.23940658569336,
      "activations/layer18_attention_weight_max": 44.16626739501953,
      "activations/layer18_attention_weight_min": -37.70232009887695,
      "activations/layer19_attention_weight_max": 22.86766242980957,
      "activations/layer19_attention_weight_min": -22.021760940551758,
      "activations/layer1_attention_weight_max": 16.895944595336914,
      "activations/layer1_attention_weight_min": -16.109607696533203,
      "activations/layer20_attention_weight_max": 22.619291305541992,
      "activations/layer20_attention_weight_min": -22.740123748779297,
      "activations/layer21_attention_weight_max": 38.29078674316406,
      "activations/layer21_attention_weight_min": -27.02309226989746,
      "activations/layer22_attention_weight_max": 32.60042190551758,
      "activations/layer22_attention_weight_min": -28.56157684326172,
      "activations/layer23_attention_weight_max": 37.26066589355469,
      "activations/layer23_attention_weight_min": -25.311857223510742,
      "activations/layer2_attention_weight_max": 32.42652130126953,
      "activations/layer2_attention_weight_min": -32.116920471191406,
      "activations/layer3_attention_weight_max": 95.0080337524414,
      "activations/layer3_attention_weight_min": -94.63487243652344,
      "activations/layer4_attention_weight_max": 96.64385223388672,
      "activations/layer4_attention_weight_min": -92.51790618896484,
      "activations/layer5_attention_weight_max": 68.0733413696289,
      "activations/layer5_attention_weight_min": -73.40983581542969,
      "activations/layer6_attention_weight_max": 52.61056900024414,
      "activations/layer6_attention_weight_min": -52.00444030761719,
      "activations/layer7_attention_weight_max": 76.26319885253906,
      "activations/layer7_attention_weight_min": -71.30960083007812,
      "activations/layer8_attention_weight_max": 44.461219787597656,
      "activations/layer8_attention_weight_min": -47.44664764404297,
      "activations/layer9_attention_weight_max": 48.955318450927734,
      "activations/layer9_attention_weight_min": -47.942325592041016,
      "epoch": 11.15,
      "learning_rate": 7.888863636363635e-05,
      "loss": 2.7762,
      "step": 191900
    },
    {
      "activations/layer0_attention_weight_max": 15.181139945983887,
      "activations/layer0_attention_weight_min": -14.065646171569824,
      "activations/layer10_attention_weight_max": 37.69243240356445,
      "activations/layer10_attention_weight_min": -35.393184661865234,
      "activations/layer11_attention_weight_max": 31.150623321533203,
      "activations/layer11_attention_weight_min": -34.85961151123047,
      "activations/layer12_attention_weight_max": 23.644201278686523,
      "activations/layer12_attention_weight_min": -22.79802703857422,
      "activations/layer13_attention_weight_max": 41.293846130371094,
      "activations/layer13_attention_weight_min": -30.988140106201172,
      "activations/layer14_attention_weight_max": 35.87109375,
      "activations/layer14_attention_weight_min": -30.394882202148438,
      "activations/layer15_attention_weight_max": 32.645835876464844,
      "activations/layer15_attention_weight_min": -29.021032333374023,
      "activations/layer16_attention_weight_max": 32.81049728393555,
      "activations/layer16_attention_weight_min": -30.612720489501953,
      "activations/layer17_attention_weight_max": 50.42036056518555,
      "activations/layer17_attention_weight_min": -44.72896957397461,
      "activations/layer18_attention_weight_max": 46.266212463378906,
      "activations/layer18_attention_weight_min": -36.28221893310547,
      "activations/layer19_attention_weight_max": 21.138723373413086,
      "activations/layer19_attention_weight_min": -19.90521812438965,
      "activations/layer1_attention_weight_max": 17.98939323425293,
      "activations/layer1_attention_weight_min": -18.11425018310547,
      "activations/layer20_attention_weight_max": 21.537071228027344,
      "activations/layer20_attention_weight_min": -21.535322189331055,
      "activations/layer21_attention_weight_max": 35.96923828125,
      "activations/layer21_attention_weight_min": -22.212553024291992,
      "activations/layer22_attention_weight_max": 31.431840896606445,
      "activations/layer22_attention_weight_min": -26.573762893676758,
      "activations/layer23_attention_weight_max": 37.41706085205078,
      "activations/layer23_attention_weight_min": -24.436721801757812,
      "activations/layer2_attention_weight_max": 30.449710845947266,
      "activations/layer2_attention_weight_min": -30.510353088378906,
      "activations/layer3_attention_weight_max": 92.21869659423828,
      "activations/layer3_attention_weight_min": -96.0301284790039,
      "activations/layer4_attention_weight_max": 97.1066665649414,
      "activations/layer4_attention_weight_min": -96.58891296386719,
      "activations/layer5_attention_weight_max": 76.49962615966797,
      "activations/layer5_attention_weight_min": -80.4596939086914,
      "activations/layer6_attention_weight_max": 53.20389175415039,
      "activations/layer6_attention_weight_min": -52.025611877441406,
      "activations/layer7_attention_weight_max": 76.10694122314453,
      "activations/layer7_attention_weight_min": -70.97726440429688,
      "activations/layer8_attention_weight_max": 48.46445846557617,
      "activations/layer8_attention_weight_min": -46.387718200683594,
      "activations/layer9_attention_weight_max": 48.94187927246094,
      "activations/layer9_attention_weight_min": -48.473201751708984,
      "epoch": 11.15,
      "learning_rate": 7.886969696969697e-05,
      "loss": 2.7734,
      "step": 191950
    },
    {
      "activations/layer0_attention_weight_max": 15.359006881713867,
      "activations/layer0_attention_weight_min": -14.640419006347656,
      "activations/layer10_attention_weight_max": 41.90275192260742,
      "activations/layer10_attention_weight_min": -36.19629669189453,
      "activations/layer11_attention_weight_max": 34.52804183959961,
      "activations/layer11_attention_weight_min": -32.78237533569336,
      "activations/layer12_attention_weight_max": 17.67934226989746,
      "activations/layer12_attention_weight_min": -26.93741226196289,
      "activations/layer13_attention_weight_max": 42.68156051635742,
      "activations/layer13_attention_weight_min": -31.46872329711914,
      "activations/layer14_attention_weight_max": 39.56371307373047,
      "activations/layer14_attention_weight_min": -31.42667007446289,
      "activations/layer15_attention_weight_max": 35.044898986816406,
      "activations/layer15_attention_weight_min": -31.49176597595215,
      "activations/layer16_attention_weight_max": 34.32018280029297,
      "activations/layer16_attention_weight_min": -30.12845802307129,
      "activations/layer17_attention_weight_max": 52.00468826293945,
      "activations/layer17_attention_weight_min": -44.415958404541016,
      "activations/layer18_attention_weight_max": 47.98215103149414,
      "activations/layer18_attention_weight_min": -38.5638542175293,
      "activations/layer19_attention_weight_max": 22.715293884277344,
      "activations/layer19_attention_weight_min": -22.50305938720703,
      "activations/layer1_attention_weight_max": 16.99846649169922,
      "activations/layer1_attention_weight_min": -15.286359786987305,
      "activations/layer20_attention_weight_max": 23.995800018310547,
      "activations/layer20_attention_weight_min": -20.98678970336914,
      "activations/layer21_attention_weight_max": 41.91965866088867,
      "activations/layer21_attention_weight_min": -22.52314567565918,
      "activations/layer22_attention_weight_max": 32.6276969909668,
      "activations/layer22_attention_weight_min": -25.46396827697754,
      "activations/layer23_attention_weight_max": 40.49100112915039,
      "activations/layer23_attention_weight_min": -22.315711975097656,
      "activations/layer2_attention_weight_max": 31.550830841064453,
      "activations/layer2_attention_weight_min": -30.251922607421875,
      "activations/layer3_attention_weight_max": 94.89366912841797,
      "activations/layer3_attention_weight_min": -95.86180114746094,
      "activations/layer4_attention_weight_max": 96.52425384521484,
      "activations/layer4_attention_weight_min": -95.63458251953125,
      "activations/layer5_attention_weight_max": 72.87693786621094,
      "activations/layer5_attention_weight_min": -79.06249237060547,
      "activations/layer6_attention_weight_max": 54.049598693847656,
      "activations/layer6_attention_weight_min": -51.92115020751953,
      "activations/layer7_attention_weight_max": 77.4685287475586,
      "activations/layer7_attention_weight_min": -71.5736312866211,
      "activations/layer8_attention_weight_max": 47.192718505859375,
      "activations/layer8_attention_weight_min": -48.58237838745117,
      "activations/layer9_attention_weight_max": 49.4123420715332,
      "activations/layer9_attention_weight_min": -46.86054229736328,
      "epoch": 11.16,
      "learning_rate": 7.885075757575756e-05,
      "loss": 2.7519,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.4988,
      "eval_samples_per_second": 505.247,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.4988,
      "eval_openwebtext_samples_per_second": 505.247,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_wikitext_loss": 2.97265625,
      "eval_wikitext_ppl": 19.543763832601815,
      "eval_wikitext_runtime": 1.9934,
      "eval_wikitext_samples_per_second": 228.752,
      "step": 192000
    },
    {
      "epoch": 11.16,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.5555,
      "eval_lambada_samples_per_second": 509.547,
      "step": 192000
    },
    {
      "activations/layer0_attention_weight_max": 16.157672882080078,
      "activations/layer0_attention_weight_min": -14.154563903808594,
      "activations/layer10_attention_weight_max": 38.6100959777832,
      "activations/layer10_attention_weight_min": -36.1500129699707,
      "activations/layer11_attention_weight_max": 32.617523193359375,
      "activations/layer11_attention_weight_min": -33.147727966308594,
      "activations/layer12_attention_weight_max": 17.954513549804688,
      "activations/layer12_attention_weight_min": -28.772659301757812,
      "activations/layer13_attention_weight_max": 32.87411117553711,
      "activations/layer13_attention_weight_min": -28.30350685119629,
      "activations/layer14_attention_weight_max": 36.31866455078125,
      "activations/layer14_attention_weight_min": -30.63372230529785,
      "activations/layer15_attention_weight_max": 33.64632034301758,
      "activations/layer15_attention_weight_min": -31.286354064941406,
      "activations/layer16_attention_weight_max": 32.54146194458008,
      "activations/layer16_attention_weight_min": -29.725566864013672,
      "activations/layer17_attention_weight_max": 50.754878997802734,
      "activations/layer17_attention_weight_min": -46.1961555480957,
      "activations/layer18_attention_weight_max": 48.16316223144531,
      "activations/layer18_attention_weight_min": -38.0015869140625,
      "activations/layer19_attention_weight_max": 21.62232208251953,
      "activations/layer19_attention_weight_min": -21.861501693725586,
      "activations/layer1_attention_weight_max": 18.220624923706055,
      "activations/layer1_attention_weight_min": -15.392319679260254,
      "activations/layer20_attention_weight_max": 22.410934448242188,
      "activations/layer20_attention_weight_min": -23.626859664916992,
      "activations/layer21_attention_weight_max": 40.55647659301758,
      "activations/layer21_attention_weight_min": -27.328832626342773,
      "activations/layer22_attention_weight_max": 33.04314041137695,
      "activations/layer22_attention_weight_min": -27.28903579711914,
      "activations/layer23_attention_weight_max": 38.346580505371094,
      "activations/layer23_attention_weight_min": -25.693737030029297,
      "activations/layer2_attention_weight_max": 31.562210083007812,
      "activations/layer2_attention_weight_min": -30.094356536865234,
      "activations/layer3_attention_weight_max": 91.82422637939453,
      "activations/layer3_attention_weight_min": -93.2298355102539,
      "activations/layer4_attention_weight_max": 100.38805389404297,
      "activations/layer4_attention_weight_min": -95.99822235107422,
      "activations/layer5_attention_weight_max": 72.86321258544922,
      "activations/layer5_attention_weight_min": -79.09137725830078,
      "activations/layer6_attention_weight_max": 53.94761276245117,
      "activations/layer6_attention_weight_min": -51.59523391723633,
      "activations/layer7_attention_weight_max": 74.52733612060547,
      "activations/layer7_attention_weight_min": -70.26172637939453,
      "activations/layer8_attention_weight_max": 44.637420654296875,
      "activations/layer8_attention_weight_min": -45.44819259643555,
      "activations/layer9_attention_weight_max": 46.57954025268555,
      "activations/layer9_attention_weight_min": -46.8884391784668,
      "epoch": 11.16,
      "learning_rate": 7.883181818181817e-05,
      "loss": 2.785,
      "step": 192050
    },
    {
      "activations/layer0_attention_weight_max": 17.03007698059082,
      "activations/layer0_attention_weight_min": -14.628172874450684,
      "activations/layer10_attention_weight_max": 40.381465911865234,
      "activations/layer10_attention_weight_min": -40.77070999145508,
      "activations/layer11_attention_weight_max": 37.22102355957031,
      "activations/layer11_attention_weight_min": -35.96510314941406,
      "activations/layer12_attention_weight_max": 21.240463256835938,
      "activations/layer12_attention_weight_min": -26.943605422973633,
      "activations/layer13_attention_weight_max": 44.539859771728516,
      "activations/layer13_attention_weight_min": -29.378204345703125,
      "activations/layer14_attention_weight_max": 51.547760009765625,
      "activations/layer14_attention_weight_min": -31.75383949279785,
      "activations/layer15_attention_weight_max": 42.86880111694336,
      "activations/layer15_attention_weight_min": -29.900794982910156,
      "activations/layer16_attention_weight_max": 30.73325538635254,
      "activations/layer16_attention_weight_min": -31.379472732543945,
      "activations/layer17_attention_weight_max": 54.04862594604492,
      "activations/layer17_attention_weight_min": -45.95286178588867,
      "activations/layer18_attention_weight_max": 48.73981475830078,
      "activations/layer18_attention_weight_min": -38.125911712646484,
      "activations/layer19_attention_weight_max": 21.158191680908203,
      "activations/layer19_attention_weight_min": -22.91474151611328,
      "activations/layer1_attention_weight_max": 16.622268676757812,
      "activations/layer1_attention_weight_min": -16.15852928161621,
      "activations/layer20_attention_weight_max": 22.4956111907959,
      "activations/layer20_attention_weight_min": -21.634777069091797,
      "activations/layer21_attention_weight_max": 36.2120361328125,
      "activations/layer21_attention_weight_min": -24.364511489868164,
      "activations/layer22_attention_weight_max": 30.84946632385254,
      "activations/layer22_attention_weight_min": -25.218965530395508,
      "activations/layer23_attention_weight_max": 41.28376007080078,
      "activations/layer23_attention_weight_min": -27.04444694519043,
      "activations/layer2_attention_weight_max": 33.77069091796875,
      "activations/layer2_attention_weight_min": -31.720008850097656,
      "activations/layer3_attention_weight_max": 96.60755920410156,
      "activations/layer3_attention_weight_min": -101.32538604736328,
      "activations/layer4_attention_weight_max": 102.26677703857422,
      "activations/layer4_attention_weight_min": -100.8639144897461,
      "activations/layer5_attention_weight_max": 70.08123779296875,
      "activations/layer5_attention_weight_min": -77.94796752929688,
      "activations/layer6_attention_weight_max": 51.38372039794922,
      "activations/layer6_attention_weight_min": -52.8063850402832,
      "activations/layer7_attention_weight_max": 75.95159912109375,
      "activations/layer7_attention_weight_min": -74.62431335449219,
      "activations/layer8_attention_weight_max": 47.03001403808594,
      "activations/layer8_attention_weight_min": -49.199893951416016,
      "activations/layer9_attention_weight_max": 56.81324005126953,
      "activations/layer9_attention_weight_min": -51.2546501159668,
      "epoch": 11.16,
      "learning_rate": 7.881287878787879e-05,
      "loss": 2.7709,
      "step": 192100
    },
    {
      "activations/layer0_attention_weight_max": 15.173212051391602,
      "activations/layer0_attention_weight_min": -14.917460441589355,
      "activations/layer10_attention_weight_max": 38.38482666015625,
      "activations/layer10_attention_weight_min": -36.05325698852539,
      "activations/layer11_attention_weight_max": 33.0333251953125,
      "activations/layer11_attention_weight_min": -32.99457550048828,
      "activations/layer12_attention_weight_max": 18.05683135986328,
      "activations/layer12_attention_weight_min": -28.10198211669922,
      "activations/layer13_attention_weight_max": 39.77500915527344,
      "activations/layer13_attention_weight_min": -32.87617111206055,
      "activations/layer14_attention_weight_max": 37.881351470947266,
      "activations/layer14_attention_weight_min": -31.860347747802734,
      "activations/layer15_attention_weight_max": 32.75106430053711,
      "activations/layer15_attention_weight_min": -30.272470474243164,
      "activations/layer16_attention_weight_max": 31.731351852416992,
      "activations/layer16_attention_weight_min": -30.51166534423828,
      "activations/layer17_attention_weight_max": 53.401405334472656,
      "activations/layer17_attention_weight_min": -44.8792839050293,
      "activations/layer18_attention_weight_max": 49.438663482666016,
      "activations/layer18_attention_weight_min": -37.906673431396484,
      "activations/layer19_attention_weight_max": 22.586801528930664,
      "activations/layer19_attention_weight_min": -20.798948287963867,
      "activations/layer1_attention_weight_max": 17.054670333862305,
      "activations/layer1_attention_weight_min": -15.11831283569336,
      "activations/layer20_attention_weight_max": 23.824094772338867,
      "activations/layer20_attention_weight_min": -21.735740661621094,
      "activations/layer21_attention_weight_max": 37.747318267822266,
      "activations/layer21_attention_weight_min": -23.6822509765625,
      "activations/layer22_attention_weight_max": 33.6166877746582,
      "activations/layer22_attention_weight_min": -26.701904296875,
      "activations/layer23_attention_weight_max": 37.659732818603516,
      "activations/layer23_attention_weight_min": -23.047409057617188,
      "activations/layer2_attention_weight_max": 33.540283203125,
      "activations/layer2_attention_weight_min": -31.220930099487305,
      "activations/layer3_attention_weight_max": 95.17839050292969,
      "activations/layer3_attention_weight_min": -97.80261993408203,
      "activations/layer4_attention_weight_max": 100.1561050415039,
      "activations/layer4_attention_weight_min": -100.86685943603516,
      "activations/layer5_attention_weight_max": 73.6548080444336,
      "activations/layer5_attention_weight_min": -77.5604248046875,
      "activations/layer6_attention_weight_max": 53.987056732177734,
      "activations/layer6_attention_weight_min": -53.36958694458008,
      "activations/layer7_attention_weight_max": 80.2757339477539,
      "activations/layer7_attention_weight_min": -76.66139221191406,
      "activations/layer8_attention_weight_max": 48.492000579833984,
      "activations/layer8_attention_weight_min": -51.24770736694336,
      "activations/layer9_attention_weight_max": 49.74581527709961,
      "activations/layer9_attention_weight_min": -47.87229537963867,
      "epoch": 11.17,
      "learning_rate": 7.879393939393938e-05,
      "loss": 2.7764,
      "step": 192150
    },
    {
      "activations/layer0_attention_weight_max": 16.931377410888672,
      "activations/layer0_attention_weight_min": -14.54832649230957,
      "activations/layer10_attention_weight_max": 35.071434020996094,
      "activations/layer10_attention_weight_min": -35.51054382324219,
      "activations/layer11_attention_weight_max": 31.959089279174805,
      "activations/layer11_attention_weight_min": -33.60192108154297,
      "activations/layer12_attention_weight_max": 17.729402542114258,
      "activations/layer12_attention_weight_min": -24.558834075927734,
      "activations/layer13_attention_weight_max": 36.184017181396484,
      "activations/layer13_attention_weight_min": -27.548494338989258,
      "activations/layer14_attention_weight_max": 34.19135284423828,
      "activations/layer14_attention_weight_min": -31.128681182861328,
      "activations/layer15_attention_weight_max": 32.084259033203125,
      "activations/layer15_attention_weight_min": -29.22057342529297,
      "activations/layer16_attention_weight_max": 32.45370864868164,
      "activations/layer16_attention_weight_min": -31.583984375,
      "activations/layer17_attention_weight_max": 51.224544525146484,
      "activations/layer17_attention_weight_min": -45.95329666137695,
      "activations/layer18_attention_weight_max": 46.23228073120117,
      "activations/layer18_attention_weight_min": -38.88730239868164,
      "activations/layer19_attention_weight_max": 23.41682243347168,
      "activations/layer19_attention_weight_min": -24.43105697631836,
      "activations/layer1_attention_weight_max": 16.841814041137695,
      "activations/layer1_attention_weight_min": -15.319665908813477,
      "activations/layer20_attention_weight_max": 22.074914932250977,
      "activations/layer20_attention_weight_min": -22.380752563476562,
      "activations/layer21_attention_weight_max": 36.083702087402344,
      "activations/layer21_attention_weight_min": -22.499515533447266,
      "activations/layer22_attention_weight_max": 29.57037925720215,
      "activations/layer22_attention_weight_min": -27.32073402404785,
      "activations/layer23_attention_weight_max": 36.52546691894531,
      "activations/layer23_attention_weight_min": -23.88380241394043,
      "activations/layer2_attention_weight_max": 32.1632080078125,
      "activations/layer2_attention_weight_min": -30.891454696655273,
      "activations/layer3_attention_weight_max": 91.1248779296875,
      "activations/layer3_attention_weight_min": -98.18754577636719,
      "activations/layer4_attention_weight_max": 96.78987884521484,
      "activations/layer4_attention_weight_min": -95.3205795288086,
      "activations/layer5_attention_weight_max": 72.94563293457031,
      "activations/layer5_attention_weight_min": -78.98125457763672,
      "activations/layer6_attention_weight_max": 50.24000930786133,
      "activations/layer6_attention_weight_min": -49.1789665222168,
      "activations/layer7_attention_weight_max": 74.86100006103516,
      "activations/layer7_attention_weight_min": -71.40853118896484,
      "activations/layer8_attention_weight_max": 42.37441635131836,
      "activations/layer8_attention_weight_min": -44.501747131347656,
      "activations/layer9_attention_weight_max": 47.8480339050293,
      "activations/layer9_attention_weight_min": -44.46687316894531,
      "epoch": 11.17,
      "learning_rate": 7.877499999999999e-05,
      "loss": 2.7856,
      "step": 192200
    },
    {
      "activations/layer0_attention_weight_max": 15.418075561523438,
      "activations/layer0_attention_weight_min": -14.313883781433105,
      "activations/layer10_attention_weight_max": 40.50691223144531,
      "activations/layer10_attention_weight_min": -36.968299865722656,
      "activations/layer11_attention_weight_max": 36.388755798339844,
      "activations/layer11_attention_weight_min": -33.91014099121094,
      "activations/layer12_attention_weight_max": 17.87947654724121,
      "activations/layer12_attention_weight_min": -25.561935424804688,
      "activations/layer13_attention_weight_max": 38.10365295410156,
      "activations/layer13_attention_weight_min": -31.35049819946289,
      "activations/layer14_attention_weight_max": 35.45433044433594,
      "activations/layer14_attention_weight_min": -32.69479751586914,
      "activations/layer15_attention_weight_max": 33.284759521484375,
      "activations/layer15_attention_weight_min": -31.66950225830078,
      "activations/layer16_attention_weight_max": 30.535982131958008,
      "activations/layer16_attention_weight_min": -32.08348846435547,
      "activations/layer17_attention_weight_max": 56.12614822387695,
      "activations/layer17_attention_weight_min": -49.512813568115234,
      "activations/layer18_attention_weight_max": 46.28376007080078,
      "activations/layer18_attention_weight_min": -45.717899322509766,
      "activations/layer19_attention_weight_max": 23.344411849975586,
      "activations/layer19_attention_weight_min": -27.557010650634766,
      "activations/layer1_attention_weight_max": 16.730955123901367,
      "activations/layer1_attention_weight_min": -16.013193130493164,
      "activations/layer20_attention_weight_max": 22.30729866027832,
      "activations/layer20_attention_weight_min": -25.713550567626953,
      "activations/layer21_attention_weight_max": 40.837738037109375,
      "activations/layer21_attention_weight_min": -24.06229591369629,
      "activations/layer22_attention_weight_max": 29.73870849609375,
      "activations/layer22_attention_weight_min": -28.42585563659668,
      "activations/layer23_attention_weight_max": 36.89518356323242,
      "activations/layer23_attention_weight_min": -24.108230590820312,
      "activations/layer2_attention_weight_max": 34.40043640136719,
      "activations/layer2_attention_weight_min": -31.592283248901367,
      "activations/layer3_attention_weight_max": 99.23556518554688,
      "activations/layer3_attention_weight_min": -94.02628326416016,
      "activations/layer4_attention_weight_max": 105.0496597290039,
      "activations/layer4_attention_weight_min": -96.64655303955078,
      "activations/layer5_attention_weight_max": 73.42586517333984,
      "activations/layer5_attention_weight_min": -80.19309997558594,
      "activations/layer6_attention_weight_max": 53.69180679321289,
      "activations/layer6_attention_weight_min": -52.433738708496094,
      "activations/layer7_attention_weight_max": 76.39543151855469,
      "activations/layer7_attention_weight_min": -74.01155090332031,
      "activations/layer8_attention_weight_max": 46.24345397949219,
      "activations/layer8_attention_weight_min": -48.345375061035156,
      "activations/layer9_attention_weight_max": 50.808162689208984,
      "activations/layer9_attention_weight_min": -50.644317626953125,
      "epoch": 11.17,
      "learning_rate": 7.87560606060606e-05,
      "loss": 2.7694,
      "step": 192250
    },
    {
      "activations/layer0_attention_weight_max": 17.530261993408203,
      "activations/layer0_attention_weight_min": -14.345033645629883,
      "activations/layer10_attention_weight_max": 35.68027114868164,
      "activations/layer10_attention_weight_min": -36.1291618347168,
      "activations/layer11_attention_weight_max": 32.82855224609375,
      "activations/layer11_attention_weight_min": -33.5218620300293,
      "activations/layer12_attention_weight_max": 16.846660614013672,
      "activations/layer12_attention_weight_min": -25.608375549316406,
      "activations/layer13_attention_weight_max": 37.55935287475586,
      "activations/layer13_attention_weight_min": -30.836166381835938,
      "activations/layer14_attention_weight_max": 34.922889709472656,
      "activations/layer14_attention_weight_min": -30.738487243652344,
      "activations/layer15_attention_weight_max": 32.6394157409668,
      "activations/layer15_attention_weight_min": -29.939647674560547,
      "activations/layer16_attention_weight_max": 29.654138565063477,
      "activations/layer16_attention_weight_min": -29.47184944152832,
      "activations/layer17_attention_weight_max": 50.668636322021484,
      "activations/layer17_attention_weight_min": -44.764286041259766,
      "activations/layer18_attention_weight_max": 43.30935287475586,
      "activations/layer18_attention_weight_min": -37.07642364501953,
      "activations/layer19_attention_weight_max": 20.46823501586914,
      "activations/layer19_attention_weight_min": -22.80283546447754,
      "activations/layer1_attention_weight_max": 17.364805221557617,
      "activations/layer1_attention_weight_min": -14.757149696350098,
      "activations/layer20_attention_weight_max": 20.62824249267578,
      "activations/layer20_attention_weight_min": -22.091428756713867,
      "activations/layer21_attention_weight_max": 34.142242431640625,
      "activations/layer21_attention_weight_min": -23.158571243286133,
      "activations/layer22_attention_weight_max": 28.697772979736328,
      "activations/layer22_attention_weight_min": -25.61846351623535,
      "activations/layer23_attention_weight_max": 35.85079574584961,
      "activations/layer23_attention_weight_min": -24.951215744018555,
      "activations/layer2_attention_weight_max": 33.64030456542969,
      "activations/layer2_attention_weight_min": -31.507423400878906,
      "activations/layer3_attention_weight_max": 95.3135986328125,
      "activations/layer3_attention_weight_min": -97.690185546875,
      "activations/layer4_attention_weight_max": 100.5958480834961,
      "activations/layer4_attention_weight_min": -97.1277847290039,
      "activations/layer5_attention_weight_max": 73.37382507324219,
      "activations/layer5_attention_weight_min": -76.49500274658203,
      "activations/layer6_attention_weight_max": 54.525047302246094,
      "activations/layer6_attention_weight_min": -52.8543586730957,
      "activations/layer7_attention_weight_max": 73.594482421875,
      "activations/layer7_attention_weight_min": -71.54617309570312,
      "activations/layer8_attention_weight_max": 44.70289611816406,
      "activations/layer8_attention_weight_min": -47.958309173583984,
      "activations/layer9_attention_weight_max": 45.65287399291992,
      "activations/layer9_attention_weight_min": -50.396114349365234,
      "epoch": 11.17,
      "learning_rate": 7.873712121212121e-05,
      "loss": 2.7844,
      "step": 192300
    },
    {
      "activations/layer0_attention_weight_max": 16.308368682861328,
      "activations/layer0_attention_weight_min": -14.30470085144043,
      "activations/layer10_attention_weight_max": 33.562835693359375,
      "activations/layer10_attention_weight_min": -34.956298828125,
      "activations/layer11_attention_weight_max": 31.20088768005371,
      "activations/layer11_attention_weight_min": -31.428573608398438,
      "activations/layer12_attention_weight_max": 17.733396530151367,
      "activations/layer12_attention_weight_min": -25.504478454589844,
      "activations/layer13_attention_weight_max": 36.882354736328125,
      "activations/layer13_attention_weight_min": -28.965410232543945,
      "activations/layer14_attention_weight_max": 37.273990631103516,
      "activations/layer14_attention_weight_min": -28.613332748413086,
      "activations/layer15_attention_weight_max": 31.707246780395508,
      "activations/layer15_attention_weight_min": -28.34295082092285,
      "activations/layer16_attention_weight_max": 29.92478370666504,
      "activations/layer16_attention_weight_min": -28.587295532226562,
      "activations/layer17_attention_weight_max": 50.486854553222656,
      "activations/layer17_attention_weight_min": -40.02706527709961,
      "activations/layer18_attention_weight_max": 44.822017669677734,
      "activations/layer18_attention_weight_min": -36.82661437988281,
      "activations/layer19_attention_weight_max": 20.83706283569336,
      "activations/layer19_attention_weight_min": -21.07908058166504,
      "activations/layer1_attention_weight_max": 17.267375946044922,
      "activations/layer1_attention_weight_min": -15.389777183532715,
      "activations/layer20_attention_weight_max": 20.3447322845459,
      "activations/layer20_attention_weight_min": -22.185741424560547,
      "activations/layer21_attention_weight_max": 34.874183654785156,
      "activations/layer21_attention_weight_min": -20.69986915588379,
      "activations/layer22_attention_weight_max": 29.956241607666016,
      "activations/layer22_attention_weight_min": -27.97502326965332,
      "activations/layer23_attention_weight_max": 35.55055236816406,
      "activations/layer23_attention_weight_min": -25.424116134643555,
      "activations/layer2_attention_weight_max": 34.176902770996094,
      "activations/layer2_attention_weight_min": -32.55780792236328,
      "activations/layer3_attention_weight_max": 93.53593444824219,
      "activations/layer3_attention_weight_min": -93.9468994140625,
      "activations/layer4_attention_weight_max": 96.01736450195312,
      "activations/layer4_attention_weight_min": -96.31880187988281,
      "activations/layer5_attention_weight_max": 70.66085052490234,
      "activations/layer5_attention_weight_min": -77.56143188476562,
      "activations/layer6_attention_weight_max": 53.444297790527344,
      "activations/layer6_attention_weight_min": -53.67939758300781,
      "activations/layer7_attention_weight_max": 75.75468444824219,
      "activations/layer7_attention_weight_min": -71.48319244384766,
      "activations/layer8_attention_weight_max": 44.02345657348633,
      "activations/layer8_attention_weight_min": -45.424072265625,
      "activations/layer9_attention_weight_max": 43.46292495727539,
      "activations/layer9_attention_weight_min": -48.91486740112305,
      "epoch": 11.18,
      "learning_rate": 7.871818181818181e-05,
      "loss": 2.7818,
      "step": 192350
    },
    {
      "activations/layer0_attention_weight_max": 15.672745704650879,
      "activations/layer0_attention_weight_min": -13.973993301391602,
      "activations/layer10_attention_weight_max": 36.20782470703125,
      "activations/layer10_attention_weight_min": -33.87234878540039,
      "activations/layer11_attention_weight_max": 32.13701248168945,
      "activations/layer11_attention_weight_min": -35.56170654296875,
      "activations/layer12_attention_weight_max": 17.888986587524414,
      "activations/layer12_attention_weight_min": -25.864912033081055,
      "activations/layer13_attention_weight_max": 38.692344665527344,
      "activations/layer13_attention_weight_min": -28.26605224609375,
      "activations/layer14_attention_weight_max": 39.535499572753906,
      "activations/layer14_attention_weight_min": -30.146358489990234,
      "activations/layer15_attention_weight_max": 32.97761917114258,
      "activations/layer15_attention_weight_min": -29.04440689086914,
      "activations/layer16_attention_weight_max": 31.976945877075195,
      "activations/layer16_attention_weight_min": -28.91160774230957,
      "activations/layer17_attention_weight_max": 52.45348358154297,
      "activations/layer17_attention_weight_min": -45.96037673950195,
      "activations/layer18_attention_weight_max": 47.52954864501953,
      "activations/layer18_attention_weight_min": -37.25977325439453,
      "activations/layer19_attention_weight_max": 22.69672203063965,
      "activations/layer19_attention_weight_min": -23.245269775390625,
      "activations/layer1_attention_weight_max": 17.046951293945312,
      "activations/layer1_attention_weight_min": -17.15330696105957,
      "activations/layer20_attention_weight_max": 23.56057357788086,
      "activations/layer20_attention_weight_min": -21.035911560058594,
      "activations/layer21_attention_weight_max": 36.85057067871094,
      "activations/layer21_attention_weight_min": -22.225248336791992,
      "activations/layer22_attention_weight_max": 31.260238647460938,
      "activations/layer22_attention_weight_min": -25.27073097229004,
      "activations/layer23_attention_weight_max": 37.825653076171875,
      "activations/layer23_attention_weight_min": -23.777912139892578,
      "activations/layer2_attention_weight_max": 32.190345764160156,
      "activations/layer2_attention_weight_min": -30.392601013183594,
      "activations/layer3_attention_weight_max": 88.63086700439453,
      "activations/layer3_attention_weight_min": -86.5164566040039,
      "activations/layer4_attention_weight_max": 98.2473373413086,
      "activations/layer4_attention_weight_min": -91.92015838623047,
      "activations/layer5_attention_weight_max": 71.06076049804688,
      "activations/layer5_attention_weight_min": -73.4254150390625,
      "activations/layer6_attention_weight_max": 52.835269927978516,
      "activations/layer6_attention_weight_min": -50.103633880615234,
      "activations/layer7_attention_weight_max": 72.36700439453125,
      "activations/layer7_attention_weight_min": -70.20247650146484,
      "activations/layer8_attention_weight_max": 43.2849006652832,
      "activations/layer8_attention_weight_min": -45.45907211303711,
      "activations/layer9_attention_weight_max": 44.992610931396484,
      "activations/layer9_attention_weight_min": -46.61188507080078,
      "epoch": 11.18,
      "learning_rate": 7.869924242424241e-05,
      "loss": 2.7729,
      "step": 192400
    },
    {
      "activations/layer0_attention_weight_max": 15.80157470703125,
      "activations/layer0_attention_weight_min": -14.351279258728027,
      "activations/layer10_attention_weight_max": 41.10530471801758,
      "activations/layer10_attention_weight_min": -37.88682174682617,
      "activations/layer11_attention_weight_max": 32.28849792480469,
      "activations/layer11_attention_weight_min": -32.8384895324707,
      "activations/layer12_attention_weight_max": 20.39455795288086,
      "activations/layer12_attention_weight_min": -28.122819900512695,
      "activations/layer13_attention_weight_max": 40.74609375,
      "activations/layer13_attention_weight_min": -27.529071807861328,
      "activations/layer14_attention_weight_max": 37.61171340942383,
      "activations/layer14_attention_weight_min": -29.675823211669922,
      "activations/layer15_attention_weight_max": 33.92599105834961,
      "activations/layer15_attention_weight_min": -30.13383674621582,
      "activations/layer16_attention_weight_max": 33.13175582885742,
      "activations/layer16_attention_weight_min": -28.737070083618164,
      "activations/layer17_attention_weight_max": 52.552162170410156,
      "activations/layer17_attention_weight_min": -42.72685623168945,
      "activations/layer18_attention_weight_max": 53.22024154663086,
      "activations/layer18_attention_weight_min": -38.051021575927734,
      "activations/layer19_attention_weight_max": 22.11810874938965,
      "activations/layer19_attention_weight_min": -20.899755477905273,
      "activations/layer1_attention_weight_max": 18.087263107299805,
      "activations/layer1_attention_weight_min": -16.33146858215332,
      "activations/layer20_attention_weight_max": 22.519136428833008,
      "activations/layer20_attention_weight_min": -23.57111167907715,
      "activations/layer21_attention_weight_max": 40.47977828979492,
      "activations/layer21_attention_weight_min": -22.880020141601562,
      "activations/layer22_attention_weight_max": 32.661773681640625,
      "activations/layer22_attention_weight_min": -24.68403434753418,
      "activations/layer23_attention_weight_max": 37.651493072509766,
      "activations/layer23_attention_weight_min": -23.517187118530273,
      "activations/layer2_attention_weight_max": 34.56193542480469,
      "activations/layer2_attention_weight_min": -33.4598503112793,
      "activations/layer3_attention_weight_max": 98.57029724121094,
      "activations/layer3_attention_weight_min": -100.89862060546875,
      "activations/layer4_attention_weight_max": 105.75054168701172,
      "activations/layer4_attention_weight_min": -101.83858489990234,
      "activations/layer5_attention_weight_max": 72.12811279296875,
      "activations/layer5_attention_weight_min": -81.68987274169922,
      "activations/layer6_attention_weight_max": 52.714351654052734,
      "activations/layer6_attention_weight_min": -53.524593353271484,
      "activations/layer7_attention_weight_max": 73.46623992919922,
      "activations/layer7_attention_weight_min": -74.49055480957031,
      "activations/layer8_attention_weight_max": 46.54300308227539,
      "activations/layer8_attention_weight_min": -47.49331283569336,
      "activations/layer9_attention_weight_max": 50.05701446533203,
      "activations/layer9_attention_weight_min": -48.51150894165039,
      "epoch": 11.18,
      "learning_rate": 7.868030303030302e-05,
      "loss": 2.7764,
      "step": 192450
    },
    {
      "activations/layer0_attention_weight_max": 16.80605697631836,
      "activations/layer0_attention_weight_min": -14.611523628234863,
      "activations/layer10_attention_weight_max": 42.127986907958984,
      "activations/layer10_attention_weight_min": -41.21504211425781,
      "activations/layer11_attention_weight_max": 36.84777069091797,
      "activations/layer11_attention_weight_min": -36.172607421875,
      "activations/layer12_attention_weight_max": 21.274600982666016,
      "activations/layer12_attention_weight_min": -24.7985782623291,
      "activations/layer13_attention_weight_max": 45.78962326049805,
      "activations/layer13_attention_weight_min": -31.807878494262695,
      "activations/layer14_attention_weight_max": 43.59250259399414,
      "activations/layer14_attention_weight_min": -32.07134246826172,
      "activations/layer15_attention_weight_max": 38.132972717285156,
      "activations/layer15_attention_weight_min": -31.357311248779297,
      "activations/layer16_attention_weight_max": 34.77278518676758,
      "activations/layer16_attention_weight_min": -30.30517578125,
      "activations/layer17_attention_weight_max": 57.50259780883789,
      "activations/layer17_attention_weight_min": -48.68733596801758,
      "activations/layer18_attention_weight_max": 49.34660339355469,
      "activations/layer18_attention_weight_min": -40.33111572265625,
      "activations/layer19_attention_weight_max": 23.646501541137695,
      "activations/layer19_attention_weight_min": -20.711650848388672,
      "activations/layer1_attention_weight_max": 16.160202026367188,
      "activations/layer1_attention_weight_min": -15.921822547912598,
      "activations/layer20_attention_weight_max": 23.098033905029297,
      "activations/layer20_attention_weight_min": -21.308574676513672,
      "activations/layer21_attention_weight_max": 42.45404052734375,
      "activations/layer21_attention_weight_min": -21.749662399291992,
      "activations/layer22_attention_weight_max": 34.82612609863281,
      "activations/layer22_attention_weight_min": -25.850650787353516,
      "activations/layer23_attention_weight_max": 38.18905258178711,
      "activations/layer23_attention_weight_min": -21.81226348876953,
      "activations/layer2_attention_weight_max": 32.46549987792969,
      "activations/layer2_attention_weight_min": -31.48867416381836,
      "activations/layer3_attention_weight_max": 92.8512954711914,
      "activations/layer3_attention_weight_min": -94.76581573486328,
      "activations/layer4_attention_weight_max": 102.58776092529297,
      "activations/layer4_attention_weight_min": -103.3941879272461,
      "activations/layer5_attention_weight_max": 73.45401000976562,
      "activations/layer5_attention_weight_min": -84.454345703125,
      "activations/layer6_attention_weight_max": 54.43818283081055,
      "activations/layer6_attention_weight_min": -54.589576721191406,
      "activations/layer7_attention_weight_max": 80.2781753540039,
      "activations/layer7_attention_weight_min": -77.80390167236328,
      "activations/layer8_attention_weight_max": 51.049903869628906,
      "activations/layer8_attention_weight_min": -52.420719146728516,
      "activations/layer9_attention_weight_max": 53.536651611328125,
      "activations/layer9_attention_weight_min": -51.925819396972656,
      "epoch": 11.19,
      "learning_rate": 7.866136363636364e-05,
      "loss": 2.7787,
      "step": 192500
    },
    {
      "activations/layer0_attention_weight_max": 16.905027389526367,
      "activations/layer0_attention_weight_min": -14.317936897277832,
      "activations/layer10_attention_weight_max": 48.9862060546875,
      "activations/layer10_attention_weight_min": -44.78861618041992,
      "activations/layer11_attention_weight_max": 44.47520065307617,
      "activations/layer11_attention_weight_min": -40.15678024291992,
      "activations/layer12_attention_weight_max": 19.542972564697266,
      "activations/layer12_attention_weight_min": -27.49143409729004,
      "activations/layer13_attention_weight_max": 74.1739273071289,
      "activations/layer13_attention_weight_min": -34.363189697265625,
      "activations/layer14_attention_weight_max": 65.88658142089844,
      "activations/layer14_attention_weight_min": -35.57078170776367,
      "activations/layer15_attention_weight_max": 56.116573333740234,
      "activations/layer15_attention_weight_min": -29.044734954833984,
      "activations/layer16_attention_weight_max": 36.7833137512207,
      "activations/layer16_attention_weight_min": -31.332706451416016,
      "activations/layer17_attention_weight_max": 61.58281707763672,
      "activations/layer17_attention_weight_min": -47.09052658081055,
      "activations/layer18_attention_weight_max": 50.43291473388672,
      "activations/layer18_attention_weight_min": -39.467044830322266,
      "activations/layer19_attention_weight_max": 26.715991973876953,
      "activations/layer19_attention_weight_min": -21.82466697692871,
      "activations/layer1_attention_weight_max": 16.726564407348633,
      "activations/layer1_attention_weight_min": -16.391551971435547,
      "activations/layer20_attention_weight_max": 24.133525848388672,
      "activations/layer20_attention_weight_min": -20.010601043701172,
      "activations/layer21_attention_weight_max": 42.15346145629883,
      "activations/layer21_attention_weight_min": -21.253164291381836,
      "activations/layer22_attention_weight_max": 33.511837005615234,
      "activations/layer22_attention_weight_min": -23.65439224243164,
      "activations/layer23_attention_weight_max": 42.61892318725586,
      "activations/layer23_attention_weight_min": -23.96828842163086,
      "activations/layer2_attention_weight_max": 36.40664291381836,
      "activations/layer2_attention_weight_min": -37.19242858886719,
      "activations/layer3_attention_weight_max": 102.18435668945312,
      "activations/layer3_attention_weight_min": -104.14796447753906,
      "activations/layer4_attention_weight_max": 111.24234771728516,
      "activations/layer4_attention_weight_min": -110.49739837646484,
      "activations/layer5_attention_weight_max": 73.95405578613281,
      "activations/layer5_attention_weight_min": -77.93046569824219,
      "activations/layer6_attention_weight_max": 58.56821060180664,
      "activations/layer6_attention_weight_min": -58.46292495727539,
      "activations/layer7_attention_weight_max": 83.69182586669922,
      "activations/layer7_attention_weight_min": -79.64669036865234,
      "activations/layer8_attention_weight_max": 60.85831069946289,
      "activations/layer8_attention_weight_min": -55.94475555419922,
      "activations/layer9_attention_weight_max": 63.571834564208984,
      "activations/layer9_attention_weight_min": -54.51332473754883,
      "epoch": 11.19,
      "learning_rate": 7.864242424242423e-05,
      "loss": 2.7816,
      "step": 192550
    },
    {
      "activations/layer0_attention_weight_max": 14.703374862670898,
      "activations/layer0_attention_weight_min": -14.241069793701172,
      "activations/layer10_attention_weight_max": 35.2581672668457,
      "activations/layer10_attention_weight_min": -34.91165542602539,
      "activations/layer11_attention_weight_max": 29.973308563232422,
      "activations/layer11_attention_weight_min": -31.53504753112793,
      "activations/layer12_attention_weight_max": 18.726825714111328,
      "activations/layer12_attention_weight_min": -28.571077346801758,
      "activations/layer13_attention_weight_max": 36.79624938964844,
      "activations/layer13_attention_weight_min": -29.397279739379883,
      "activations/layer14_attention_weight_max": 39.792030334472656,
      "activations/layer14_attention_weight_min": -30.60547637939453,
      "activations/layer15_attention_weight_max": 37.30919647216797,
      "activations/layer15_attention_weight_min": -28.799449920654297,
      "activations/layer16_attention_weight_max": 31.284334182739258,
      "activations/layer16_attention_weight_min": -29.636192321777344,
      "activations/layer17_attention_weight_max": 54.893123626708984,
      "activations/layer17_attention_weight_min": -48.1744384765625,
      "activations/layer18_attention_weight_max": 46.417869567871094,
      "activations/layer18_attention_weight_min": -41.471500396728516,
      "activations/layer19_attention_weight_max": 22.05177116394043,
      "activations/layer19_attention_weight_min": -22.075191497802734,
      "activations/layer1_attention_weight_max": 16.79759979248047,
      "activations/layer1_attention_weight_min": -16.983482360839844,
      "activations/layer20_attention_weight_max": 20.575536727905273,
      "activations/layer20_attention_weight_min": -24.24375343322754,
      "activations/layer21_attention_weight_max": 38.687339782714844,
      "activations/layer21_attention_weight_min": -24.695085525512695,
      "activations/layer22_attention_weight_max": 30.434520721435547,
      "activations/layer22_attention_weight_min": -29.47188377380371,
      "activations/layer23_attention_weight_max": 36.53577423095703,
      "activations/layer23_attention_weight_min": -26.43558120727539,
      "activations/layer2_attention_weight_max": 33.61094284057617,
      "activations/layer2_attention_weight_min": -31.178449630737305,
      "activations/layer3_attention_weight_max": 94.96598052978516,
      "activations/layer3_attention_weight_min": -95.95560455322266,
      "activations/layer4_attention_weight_max": 101.01483917236328,
      "activations/layer4_attention_weight_min": -99.61714172363281,
      "activations/layer5_attention_weight_max": 71.64844512939453,
      "activations/layer5_attention_weight_min": -76.19691467285156,
      "activations/layer6_attention_weight_max": 54.201820373535156,
      "activations/layer6_attention_weight_min": -54.22218322753906,
      "activations/layer7_attention_weight_max": 74.93421936035156,
      "activations/layer7_attention_weight_min": -74.05036926269531,
      "activations/layer8_attention_weight_max": 45.466957092285156,
      "activations/layer8_attention_weight_min": -47.10104751586914,
      "activations/layer9_attention_weight_max": 49.13032150268555,
      "activations/layer9_attention_weight_min": -47.98685836791992,
      "epoch": 11.19,
      "learning_rate": 7.862348484848484e-05,
      "loss": 2.7933,
      "step": 192600
    },
    {
      "activations/layer0_attention_weight_max": 16.3958683013916,
      "activations/layer0_attention_weight_min": -14.241524696350098,
      "activations/layer10_attention_weight_max": 35.608154296875,
      "activations/layer10_attention_weight_min": -33.8583869934082,
      "activations/layer11_attention_weight_max": 32.16206359863281,
      "activations/layer11_attention_weight_min": -31.00426483154297,
      "activations/layer12_attention_weight_max": 19.01722526550293,
      "activations/layer12_attention_weight_min": -23.81907844543457,
      "activations/layer13_attention_weight_max": 34.858154296875,
      "activations/layer13_attention_weight_min": -30.006366729736328,
      "activations/layer14_attention_weight_max": 42.197757720947266,
      "activations/layer14_attention_weight_min": -28.910770416259766,
      "activations/layer15_attention_weight_max": 35.09675979614258,
      "activations/layer15_attention_weight_min": -27.599102020263672,
      "activations/layer16_attention_weight_max": 28.076147079467773,
      "activations/layer16_attention_weight_min": -26.827640533447266,
      "activations/layer17_attention_weight_max": 48.02159118652344,
      "activations/layer17_attention_weight_min": -42.56770324707031,
      "activations/layer18_attention_weight_max": 42.1539421081543,
      "activations/layer18_attention_weight_min": -34.92158508300781,
      "activations/layer19_attention_weight_max": 21.261062622070312,
      "activations/layer19_attention_weight_min": -20.069366455078125,
      "activations/layer1_attention_weight_max": 17.591936111450195,
      "activations/layer1_attention_weight_min": -18.198345184326172,
      "activations/layer20_attention_weight_max": 20.45395278930664,
      "activations/layer20_attention_weight_min": -25.04213523864746,
      "activations/layer21_attention_weight_max": 36.79029083251953,
      "activations/layer21_attention_weight_min": -21.938997268676758,
      "activations/layer22_attention_weight_max": 29.78611183166504,
      "activations/layer22_attention_weight_min": -23.598352432250977,
      "activations/layer23_attention_weight_max": 34.518062591552734,
      "activations/layer23_attention_weight_min": -22.595327377319336,
      "activations/layer2_attention_weight_max": 33.79898452758789,
      "activations/layer2_attention_weight_min": -29.457538604736328,
      "activations/layer3_attention_weight_max": 92.66630554199219,
      "activations/layer3_attention_weight_min": -91.30545043945312,
      "activations/layer4_attention_weight_max": 96.46561431884766,
      "activations/layer4_attention_weight_min": -93.85000610351562,
      "activations/layer5_attention_weight_max": 71.47856140136719,
      "activations/layer5_attention_weight_min": -75.64973449707031,
      "activations/layer6_attention_weight_max": 50.6463508605957,
      "activations/layer6_attention_weight_min": -50.63844680786133,
      "activations/layer7_attention_weight_max": 72.90380096435547,
      "activations/layer7_attention_weight_min": -66.90867614746094,
      "activations/layer8_attention_weight_max": 44.68010330200195,
      "activations/layer8_attention_weight_min": -42.93899917602539,
      "activations/layer9_attention_weight_max": 44.00876235961914,
      "activations/layer9_attention_weight_min": -45.34567642211914,
      "epoch": 11.19,
      "learning_rate": 7.860454545454546e-05,
      "loss": 2.7744,
      "step": 192650
    },
    {
      "activations/layer0_attention_weight_max": 16.356088638305664,
      "activations/layer0_attention_weight_min": -14.19303035736084,
      "activations/layer10_attention_weight_max": 37.61940383911133,
      "activations/layer10_attention_weight_min": -36.75368118286133,
      "activations/layer11_attention_weight_max": 31.72938346862793,
      "activations/layer11_attention_weight_min": -35.16794204711914,
      "activations/layer12_attention_weight_max": 19.09871482849121,
      "activations/layer12_attention_weight_min": -23.80583953857422,
      "activations/layer13_attention_weight_max": 44.38022994995117,
      "activations/layer13_attention_weight_min": -28.80774688720703,
      "activations/layer14_attention_weight_max": 35.66526412963867,
      "activations/layer14_attention_weight_min": -30.926916122436523,
      "activations/layer15_attention_weight_max": 35.2237663269043,
      "activations/layer15_attention_weight_min": -30.384254455566406,
      "activations/layer16_attention_weight_max": 31.14246368408203,
      "activations/layer16_attention_weight_min": -29.72163963317871,
      "activations/layer17_attention_weight_max": 59.74088668823242,
      "activations/layer17_attention_weight_min": -46.712650299072266,
      "activations/layer18_attention_weight_max": 50.091041564941406,
      "activations/layer18_attention_weight_min": -40.9051399230957,
      "activations/layer19_attention_weight_max": 23.104299545288086,
      "activations/layer19_attention_weight_min": -22.52912712097168,
      "activations/layer1_attention_weight_max": 16.37528419494629,
      "activations/layer1_attention_weight_min": -14.598528861999512,
      "activations/layer20_attention_weight_max": 23.478092193603516,
      "activations/layer20_attention_weight_min": -22.921459197998047,
      "activations/layer21_attention_weight_max": 40.18390655517578,
      "activations/layer21_attention_weight_min": -24.531713485717773,
      "activations/layer22_attention_weight_max": 28.58686065673828,
      "activations/layer22_attention_weight_min": -27.143274307250977,
      "activations/layer23_attention_weight_max": 35.903114318847656,
      "activations/layer23_attention_weight_min": -26.318618774414062,
      "activations/layer2_attention_weight_max": 34.278404235839844,
      "activations/layer2_attention_weight_min": -31.01802635192871,
      "activations/layer3_attention_weight_max": 94.32978820800781,
      "activations/layer3_attention_weight_min": -97.10824584960938,
      "activations/layer4_attention_weight_max": 104.42132568359375,
      "activations/layer4_attention_weight_min": -98.00792694091797,
      "activations/layer5_attention_weight_max": 74.69404602050781,
      "activations/layer5_attention_weight_min": -82.33187866210938,
      "activations/layer6_attention_weight_max": 55.407196044921875,
      "activations/layer6_attention_weight_min": -55.288822174072266,
      "activations/layer7_attention_weight_max": 75.55812072753906,
      "activations/layer7_attention_weight_min": -74.18675994873047,
      "activations/layer8_attention_weight_max": 45.0451545715332,
      "activations/layer8_attention_weight_min": -47.76674270629883,
      "activations/layer9_attention_weight_max": 47.67805099487305,
      "activations/layer9_attention_weight_min": -47.726417541503906,
      "epoch": 11.2,
      "learning_rate": 7.858560606060605e-05,
      "loss": 2.7649,
      "step": 192700
    },
    {
      "activations/layer0_attention_weight_max": 17.13947105407715,
      "activations/layer0_attention_weight_min": -14.195728302001953,
      "activations/layer10_attention_weight_max": 36.19810485839844,
      "activations/layer10_attention_weight_min": -36.01030731201172,
      "activations/layer11_attention_weight_max": 32.27906036376953,
      "activations/layer11_attention_weight_min": -35.40361022949219,
      "activations/layer12_attention_weight_max": 17.407514572143555,
      "activations/layer12_attention_weight_min": -24.371976852416992,
      "activations/layer13_attention_weight_max": 36.632545471191406,
      "activations/layer13_attention_weight_min": -30.910799026489258,
      "activations/layer14_attention_weight_max": 34.103729248046875,
      "activations/layer14_attention_weight_min": -29.405004501342773,
      "activations/layer15_attention_weight_max": 31.631254196166992,
      "activations/layer15_attention_weight_min": -28.93410301208496,
      "activations/layer16_attention_weight_max": 30.540742874145508,
      "activations/layer16_attention_weight_min": -27.399295806884766,
      "activations/layer17_attention_weight_max": 47.4029426574707,
      "activations/layer17_attention_weight_min": -42.62381362915039,
      "activations/layer18_attention_weight_max": 45.06232833862305,
      "activations/layer18_attention_weight_min": -36.1161003112793,
      "activations/layer19_attention_weight_max": 20.31639862060547,
      "activations/layer19_attention_weight_min": -21.124923706054688,
      "activations/layer1_attention_weight_max": 16.525598526000977,
      "activations/layer1_attention_weight_min": -14.097807884216309,
      "activations/layer20_attention_weight_max": 21.548086166381836,
      "activations/layer20_attention_weight_min": -23.34055519104004,
      "activations/layer21_attention_weight_max": 32.73477554321289,
      "activations/layer21_attention_weight_min": -21.5985107421875,
      "activations/layer22_attention_weight_max": 30.92230224609375,
      "activations/layer22_attention_weight_min": -25.68826675415039,
      "activations/layer23_attention_weight_max": 42.15168762207031,
      "activations/layer23_attention_weight_min": -23.869972229003906,
      "activations/layer2_attention_weight_max": 32.08929443359375,
      "activations/layer2_attention_weight_min": -29.46847915649414,
      "activations/layer3_attention_weight_max": 91.41808319091797,
      "activations/layer3_attention_weight_min": -90.99549102783203,
      "activations/layer4_attention_weight_max": 98.65364074707031,
      "activations/layer4_attention_weight_min": -93.50813293457031,
      "activations/layer5_attention_weight_max": 71.16935729980469,
      "activations/layer5_attention_weight_min": -78.77987670898438,
      "activations/layer6_attention_weight_max": 52.11585235595703,
      "activations/layer6_attention_weight_min": -51.18761444091797,
      "activations/layer7_attention_weight_max": 73.52474212646484,
      "activations/layer7_attention_weight_min": -66.97684478759766,
      "activations/layer8_attention_weight_max": 46.241390228271484,
      "activations/layer8_attention_weight_min": -47.57331085205078,
      "activations/layer9_attention_weight_max": 46.20170211791992,
      "activations/layer9_attention_weight_min": -47.641197204589844,
      "epoch": 11.2,
      "learning_rate": 7.856666666666666e-05,
      "loss": 2.7813,
      "step": 192750
    },
    {
      "activations/layer0_attention_weight_max": 16.040668487548828,
      "activations/layer0_attention_weight_min": -14.346497535705566,
      "activations/layer10_attention_weight_max": 37.80846405029297,
      "activations/layer10_attention_weight_min": -38.48048782348633,
      "activations/layer11_attention_weight_max": 31.542034149169922,
      "activations/layer11_attention_weight_min": -35.0450439453125,
      "activations/layer12_attention_weight_max": 18.884597778320312,
      "activations/layer12_attention_weight_min": -24.425357818603516,
      "activations/layer13_attention_weight_max": 39.68525314331055,
      "activations/layer13_attention_weight_min": -31.26087188720703,
      "activations/layer14_attention_weight_max": 45.99456787109375,
      "activations/layer14_attention_weight_min": -32.41606140136719,
      "activations/layer15_attention_weight_max": 39.46492004394531,
      "activations/layer15_attention_weight_min": -32.68608474731445,
      "activations/layer16_attention_weight_max": 34.53500747680664,
      "activations/layer16_attention_weight_min": -32.10955810546875,
      "activations/layer17_attention_weight_max": 61.64394760131836,
      "activations/layer17_attention_weight_min": -49.23046112060547,
      "activations/layer18_attention_weight_max": 54.10133743286133,
      "activations/layer18_attention_weight_min": -40.4382438659668,
      "activations/layer19_attention_weight_max": 25.819049835205078,
      "activations/layer19_attention_weight_min": -22.582124710083008,
      "activations/layer1_attention_weight_max": 16.941017150878906,
      "activations/layer1_attention_weight_min": -15.095590591430664,
      "activations/layer20_attention_weight_max": 24.837356567382812,
      "activations/layer20_attention_weight_min": -24.23396110534668,
      "activations/layer21_attention_weight_max": 44.2869987487793,
      "activations/layer21_attention_weight_min": -23.422855377197266,
      "activations/layer22_attention_weight_max": 32.36091232299805,
      "activations/layer22_attention_weight_min": -25.959749221801758,
      "activations/layer23_attention_weight_max": 37.1810188293457,
      "activations/layer23_attention_weight_min": -21.97681427001953,
      "activations/layer2_attention_weight_max": 32.09269332885742,
      "activations/layer2_attention_weight_min": -31.204708099365234,
      "activations/layer3_attention_weight_max": 95.03225708007812,
      "activations/layer3_attention_weight_min": -90.48225402832031,
      "activations/layer4_attention_weight_max": 99.44512176513672,
      "activations/layer4_attention_weight_min": -96.7206802368164,
      "activations/layer5_attention_weight_max": 71.9310302734375,
      "activations/layer5_attention_weight_min": -80.35169982910156,
      "activations/layer6_attention_weight_max": 52.54713439941406,
      "activations/layer6_attention_weight_min": -51.99336624145508,
      "activations/layer7_attention_weight_max": 76.28022766113281,
      "activations/layer7_attention_weight_min": -77.13618469238281,
      "activations/layer8_attention_weight_max": 44.16159439086914,
      "activations/layer8_attention_weight_min": -48.5445442199707,
      "activations/layer9_attention_weight_max": 50.52040481567383,
      "activations/layer9_attention_weight_min": -48.432613372802734,
      "epoch": 11.2,
      "learning_rate": 7.854772727272728e-05,
      "loss": 2.775,
      "step": 192800
    },
    {
      "activations/layer0_attention_weight_max": 16.165699005126953,
      "activations/layer0_attention_weight_min": -14.600418090820312,
      "activations/layer10_attention_weight_max": 36.42646789550781,
      "activations/layer10_attention_weight_min": -36.31789779663086,
      "activations/layer11_attention_weight_max": 33.057159423828125,
      "activations/layer11_attention_weight_min": -32.73939895629883,
      "activations/layer12_attention_weight_max": 18.6427001953125,
      "activations/layer12_attention_weight_min": -26.673738479614258,
      "activations/layer13_attention_weight_max": 37.623382568359375,
      "activations/layer13_attention_weight_min": -30.425193786621094,
      "activations/layer14_attention_weight_max": 36.57048797607422,
      "activations/layer14_attention_weight_min": -32.59305953979492,
      "activations/layer15_attention_weight_max": 33.488277435302734,
      "activations/layer15_attention_weight_min": -30.680469512939453,
      "activations/layer16_attention_weight_max": 30.070571899414062,
      "activations/layer16_attention_weight_min": -28.78074073791504,
      "activations/layer17_attention_weight_max": 52.21365737915039,
      "activations/layer17_attention_weight_min": -45.348228454589844,
      "activations/layer18_attention_weight_max": 48.41214370727539,
      "activations/layer18_attention_weight_min": -39.01533126831055,
      "activations/layer19_attention_weight_max": 22.42047119140625,
      "activations/layer19_attention_weight_min": -24.46835708618164,
      "activations/layer1_attention_weight_max": 17.041677474975586,
      "activations/layer1_attention_weight_min": -13.784492492675781,
      "activations/layer20_attention_weight_max": 20.932966232299805,
      "activations/layer20_attention_weight_min": -21.826005935668945,
      "activations/layer21_attention_weight_max": 34.411582946777344,
      "activations/layer21_attention_weight_min": -24.460735321044922,
      "activations/layer22_attention_weight_max": 31.311988830566406,
      "activations/layer22_attention_weight_min": -26.9138240814209,
      "activations/layer23_attention_weight_max": 37.59270095825195,
      "activations/layer23_attention_weight_min": -22.579605102539062,
      "activations/layer2_attention_weight_max": 32.37267303466797,
      "activations/layer2_attention_weight_min": -30.230749130249023,
      "activations/layer3_attention_weight_max": 90.71229553222656,
      "activations/layer3_attention_weight_min": -91.28142547607422,
      "activations/layer4_attention_weight_max": 97.13457489013672,
      "activations/layer4_attention_weight_min": -93.25762176513672,
      "activations/layer5_attention_weight_max": 70.19757080078125,
      "activations/layer5_attention_weight_min": -77.66734313964844,
      "activations/layer6_attention_weight_max": 51.518898010253906,
      "activations/layer6_attention_weight_min": -50.840267181396484,
      "activations/layer7_attention_weight_max": 72.2087631225586,
      "activations/layer7_attention_weight_min": -72.4104232788086,
      "activations/layer8_attention_weight_max": 43.77139663696289,
      "activations/layer8_attention_weight_min": -45.60002136230469,
      "activations/layer9_attention_weight_max": 43.88549041748047,
      "activations/layer9_attention_weight_min": -47.91701889038086,
      "epoch": 11.21,
      "learning_rate": 7.852878787878786e-05,
      "loss": 2.7807,
      "step": 192850
    },
    {
      "activations/layer0_attention_weight_max": 15.36418628692627,
      "activations/layer0_attention_weight_min": -14.748577117919922,
      "activations/layer10_attention_weight_max": 38.49781036376953,
      "activations/layer10_attention_weight_min": -37.430484771728516,
      "activations/layer11_attention_weight_max": 36.36899948120117,
      "activations/layer11_attention_weight_min": -32.701908111572266,
      "activations/layer12_attention_weight_max": 18.787643432617188,
      "activations/layer12_attention_weight_min": -25.704410552978516,
      "activations/layer13_attention_weight_max": 39.477691650390625,
      "activations/layer13_attention_weight_min": -27.941530227661133,
      "activations/layer14_attention_weight_max": 39.31661605834961,
      "activations/layer14_attention_weight_min": -32.193511962890625,
      "activations/layer15_attention_weight_max": 36.58094024658203,
      "activations/layer15_attention_weight_min": -29.568981170654297,
      "activations/layer16_attention_weight_max": 31.02457618713379,
      "activations/layer16_attention_weight_min": -27.889493942260742,
      "activations/layer17_attention_weight_max": 49.12188720703125,
      "activations/layer17_attention_weight_min": -41.77925491333008,
      "activations/layer18_attention_weight_max": 45.53181838989258,
      "activations/layer18_attention_weight_min": -35.67144012451172,
      "activations/layer19_attention_weight_max": 23.301273345947266,
      "activations/layer19_attention_weight_min": -20.528413772583008,
      "activations/layer1_attention_weight_max": 18.285324096679688,
      "activations/layer1_attention_weight_min": -13.894524574279785,
      "activations/layer20_attention_weight_max": 22.38578987121582,
      "activations/layer20_attention_weight_min": -23.452726364135742,
      "activations/layer21_attention_weight_max": 38.583038330078125,
      "activations/layer21_attention_weight_min": -22.785842895507812,
      "activations/layer22_attention_weight_max": 30.194604873657227,
      "activations/layer22_attention_weight_min": -26.951881408691406,
      "activations/layer23_attention_weight_max": 37.24091339111328,
      "activations/layer23_attention_weight_min": -23.876178741455078,
      "activations/layer2_attention_weight_max": 32.31951141357422,
      "activations/layer2_attention_weight_min": -30.820850372314453,
      "activations/layer3_attention_weight_max": 93.00080871582031,
      "activations/layer3_attention_weight_min": -91.41606903076172,
      "activations/layer4_attention_weight_max": 100.46729278564453,
      "activations/layer4_attention_weight_min": -95.1256332397461,
      "activations/layer5_attention_weight_max": 76.47551727294922,
      "activations/layer5_attention_weight_min": -76.37481689453125,
      "activations/layer6_attention_weight_max": 49.94987106323242,
      "activations/layer6_attention_weight_min": -51.162349700927734,
      "activations/layer7_attention_weight_max": 79.99497985839844,
      "activations/layer7_attention_weight_min": -72.91826629638672,
      "activations/layer8_attention_weight_max": 48.26873016357422,
      "activations/layer8_attention_weight_min": -47.79786682128906,
      "activations/layer9_attention_weight_max": 60.99262619018555,
      "activations/layer9_attention_weight_min": -47.94514083862305,
      "epoch": 11.21,
      "learning_rate": 7.850984848484848e-05,
      "loss": 2.7707,
      "step": 192900
    },
    {
      "activations/layer0_attention_weight_max": 15.66869831085205,
      "activations/layer0_attention_weight_min": -14.697749137878418,
      "activations/layer10_attention_weight_max": 36.81437683105469,
      "activations/layer10_attention_weight_min": -36.71369552612305,
      "activations/layer11_attention_weight_max": 32.01011276245117,
      "activations/layer11_attention_weight_min": -32.072349548339844,
      "activations/layer12_attention_weight_max": 16.43792152404785,
      "activations/layer12_attention_weight_min": -22.265216827392578,
      "activations/layer13_attention_weight_max": 40.275508880615234,
      "activations/layer13_attention_weight_min": -28.824832916259766,
      "activations/layer14_attention_weight_max": 33.54275131225586,
      "activations/layer14_attention_weight_min": -28.735912322998047,
      "activations/layer15_attention_weight_max": 32.62428283691406,
      "activations/layer15_attention_weight_min": -29.449630737304688,
      "activations/layer16_attention_weight_max": 30.154186248779297,
      "activations/layer16_attention_weight_min": -27.987089157104492,
      "activations/layer17_attention_weight_max": 54.12310028076172,
      "activations/layer17_attention_weight_min": -41.44844436645508,
      "activations/layer18_attention_weight_max": 45.680606842041016,
      "activations/layer18_attention_weight_min": -39.15888595581055,
      "activations/layer19_attention_weight_max": 20.826644897460938,
      "activations/layer19_attention_weight_min": -26.5736141204834,
      "activations/layer1_attention_weight_max": 17.63604736328125,
      "activations/layer1_attention_weight_min": -16.35388946533203,
      "activations/layer20_attention_weight_max": 21.75345802307129,
      "activations/layer20_attention_weight_min": -25.82872772216797,
      "activations/layer21_attention_weight_max": 33.58772277832031,
      "activations/layer21_attention_weight_min": -32.618812561035156,
      "activations/layer22_attention_weight_max": 28.695894241333008,
      "activations/layer22_attention_weight_min": -26.260284423828125,
      "activations/layer23_attention_weight_max": 35.21666717529297,
      "activations/layer23_attention_weight_min": -25.895627975463867,
      "activations/layer2_attention_weight_max": 34.793758392333984,
      "activations/layer2_attention_weight_min": -31.019268035888672,
      "activations/layer3_attention_weight_max": 98.81927490234375,
      "activations/layer3_attention_weight_min": -98.78504943847656,
      "activations/layer4_attention_weight_max": 103.33656311035156,
      "activations/layer4_attention_weight_min": -102.31280517578125,
      "activations/layer5_attention_weight_max": 77.64004516601562,
      "activations/layer5_attention_weight_min": -79.0664291381836,
      "activations/layer6_attention_weight_max": 54.59922409057617,
      "activations/layer6_attention_weight_min": -55.15739059448242,
      "activations/layer7_attention_weight_max": 74.22430419921875,
      "activations/layer7_attention_weight_min": -74.66621398925781,
      "activations/layer8_attention_weight_max": 47.64691162109375,
      "activations/layer8_attention_weight_min": -50.914390563964844,
      "activations/layer9_attention_weight_max": 47.65449905395508,
      "activations/layer9_attention_weight_min": -48.176025390625,
      "epoch": 11.21,
      "learning_rate": 7.849090909090908e-05,
      "loss": 2.7942,
      "step": 192950
    },
    {
      "activations/layer0_attention_weight_max": 16.098474502563477,
      "activations/layer0_attention_weight_min": -14.003227233886719,
      "activations/layer10_attention_weight_max": 35.392826080322266,
      "activations/layer10_attention_weight_min": -35.80970001220703,
      "activations/layer11_attention_weight_max": 33.002540588378906,
      "activations/layer11_attention_weight_min": -35.10112380981445,
      "activations/layer12_attention_weight_max": 18.716917037963867,
      "activations/layer12_attention_weight_min": -24.884212493896484,
      "activations/layer13_attention_weight_max": 36.907527923583984,
      "activations/layer13_attention_weight_min": -29.19980812072754,
      "activations/layer14_attention_weight_max": 36.64911651611328,
      "activations/layer14_attention_weight_min": -28.28978157043457,
      "activations/layer15_attention_weight_max": 30.1212215423584,
      "activations/layer15_attention_weight_min": -28.564937591552734,
      "activations/layer16_attention_weight_max": 27.73302459716797,
      "activations/layer16_attention_weight_min": -25.682741165161133,
      "activations/layer17_attention_weight_max": 48.873291015625,
      "activations/layer17_attention_weight_min": -45.5777702331543,
      "activations/layer18_attention_weight_max": 46.89285659790039,
      "activations/layer18_attention_weight_min": -38.23527145385742,
      "activations/layer19_attention_weight_max": 22.20261573791504,
      "activations/layer19_attention_weight_min": -21.278621673583984,
      "activations/layer1_attention_weight_max": 17.941997528076172,
      "activations/layer1_attention_weight_min": -16.361494064331055,
      "activations/layer20_attention_weight_max": 20.885271072387695,
      "activations/layer20_attention_weight_min": -21.76464080810547,
      "activations/layer21_attention_weight_max": 33.37482452392578,
      "activations/layer21_attention_weight_min": -24.45470428466797,
      "activations/layer22_attention_weight_max": 29.73246192932129,
      "activations/layer22_attention_weight_min": -22.8604679107666,
      "activations/layer23_attention_weight_max": 35.062767028808594,
      "activations/layer23_attention_weight_min": -22.878461837768555,
      "activations/layer2_attention_weight_max": 39.73040771484375,
      "activations/layer2_attention_weight_min": -36.496002197265625,
      "activations/layer3_attention_weight_max": 114.01573944091797,
      "activations/layer3_attention_weight_min": -107.05783081054688,
      "activations/layer4_attention_weight_max": 108.5174789428711,
      "activations/layer4_attention_weight_min": -100.56370544433594,
      "activations/layer5_attention_weight_max": 73.56655883789062,
      "activations/layer5_attention_weight_min": -76.38766479492188,
      "activations/layer6_attention_weight_max": 51.178035736083984,
      "activations/layer6_attention_weight_min": -52.46876907348633,
      "activations/layer7_attention_weight_max": 74.61721801757812,
      "activations/layer7_attention_weight_min": -72.86225128173828,
      "activations/layer8_attention_weight_max": 46.46247100830078,
      "activations/layer8_attention_weight_min": -46.057621002197266,
      "activations/layer9_attention_weight_max": 51.365020751953125,
      "activations/layer9_attention_weight_min": -47.922691345214844,
      "epoch": 11.21,
      "learning_rate": 7.847196969696968e-05,
      "loss": 2.7738,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5198,
      "eval_samples_per_second": 504.003,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5198,
      "eval_openwebtext_samples_per_second": 504.003,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_wikitext_loss": 2.96875,
      "eval_wikitext_ppl": 19.46756991825522,
      "eval_wikitext_runtime": 1.9983,
      "eval_wikitext_samples_per_second": 228.196,
      "step": 193000
    },
    {
      "epoch": 11.21,
      "eval_lambada_loss": 2.587890625,
      "eval_lambada_ppl": 13.301683750542033,
      "eval_lambada_runtime": 9.6128,
      "eval_lambada_samples_per_second": 506.511,
      "step": 193000
    },
    {
      "activations/layer0_attention_weight_max": 17.138118743896484,
      "activations/layer0_attention_weight_min": -14.331219673156738,
      "activations/layer10_attention_weight_max": 38.808650970458984,
      "activations/layer10_attention_weight_min": -38.123146057128906,
      "activations/layer11_attention_weight_max": 34.32829666137695,
      "activations/layer11_attention_weight_min": -35.1688117980957,
      "activations/layer12_attention_weight_max": 19.693801879882812,
      "activations/layer12_attention_weight_min": -24.433019638061523,
      "activations/layer13_attention_weight_max": 40.794490814208984,
      "activations/layer13_attention_weight_min": -32.12270736694336,
      "activations/layer14_attention_weight_max": 40.4295768737793,
      "activations/layer14_attention_weight_min": -32.48234176635742,
      "activations/layer15_attention_weight_max": 33.53333282470703,
      "activations/layer15_attention_weight_min": -32.03765106201172,
      "activations/layer16_attention_weight_max": 34.342472076416016,
      "activations/layer16_attention_weight_min": -29.091144561767578,
      "activations/layer17_attention_weight_max": 55.872554779052734,
      "activations/layer17_attention_weight_min": -45.851688385009766,
      "activations/layer18_attention_weight_max": 52.9047966003418,
      "activations/layer18_attention_weight_min": -41.569698333740234,
      "activations/layer19_attention_weight_max": 24.385900497436523,
      "activations/layer19_attention_weight_min": -23.51291275024414,
      "activations/layer1_attention_weight_max": 17.292020797729492,
      "activations/layer1_attention_weight_min": -15.411148071289062,
      "activations/layer20_attention_weight_max": 25.86294174194336,
      "activations/layer20_attention_weight_min": -23.824230194091797,
      "activations/layer21_attention_weight_max": 39.85081481933594,
      "activations/layer21_attention_weight_min": -26.79030418395996,
      "activations/layer22_attention_weight_max": 36.33602523803711,
      "activations/layer22_attention_weight_min": -27.5323543548584,
      "activations/layer23_attention_weight_max": 40.02988052368164,
      "activations/layer23_attention_weight_min": -24.611167907714844,
      "activations/layer2_attention_weight_max": 34.027442932128906,
      "activations/layer2_attention_weight_min": -30.813377380371094,
      "activations/layer3_attention_weight_max": 98.85908508300781,
      "activations/layer3_attention_weight_min": -99.538330078125,
      "activations/layer4_attention_weight_max": 107.8049545288086,
      "activations/layer4_attention_weight_min": -100.2114486694336,
      "activations/layer5_attention_weight_max": 77.40859985351562,
      "activations/layer5_attention_weight_min": -79.95638275146484,
      "activations/layer6_attention_weight_max": 62.06149673461914,
      "activations/layer6_attention_weight_min": -55.40485763549805,
      "activations/layer7_attention_weight_max": 77.58563995361328,
      "activations/layer7_attention_weight_min": -77.09003448486328,
      "activations/layer8_attention_weight_max": 53.66460037231445,
      "activations/layer8_attention_weight_min": -53.09922409057617,
      "activations/layer9_attention_weight_max": 53.166500091552734,
      "activations/layer9_attention_weight_min": -49.40958023071289,
      "epoch": 11.22,
      "learning_rate": 7.84530303030303e-05,
      "loss": 2.7804,
      "step": 193050
    },
    {
      "activations/layer0_attention_weight_max": 16.320085525512695,
      "activations/layer0_attention_weight_min": -14.357897758483887,
      "activations/layer10_attention_weight_max": 39.00139617919922,
      "activations/layer10_attention_weight_min": -39.16316223144531,
      "activations/layer11_attention_weight_max": 33.26433563232422,
      "activations/layer11_attention_weight_min": -34.31661605834961,
      "activations/layer12_attention_weight_max": 24.90208625793457,
      "activations/layer12_attention_weight_min": -23.789169311523438,
      "activations/layer13_attention_weight_max": 39.733089447021484,
      "activations/layer13_attention_weight_min": -31.799386978149414,
      "activations/layer14_attention_weight_max": 35.512935638427734,
      "activations/layer14_attention_weight_min": -31.436351776123047,
      "activations/layer15_attention_weight_max": 33.490352630615234,
      "activations/layer15_attention_weight_min": -32.21208190917969,
      "activations/layer16_attention_weight_max": 34.9359245300293,
      "activations/layer16_attention_weight_min": -28.820398330688477,
      "activations/layer17_attention_weight_max": 49.76448440551758,
      "activations/layer17_attention_weight_min": -44.82251739501953,
      "activations/layer18_attention_weight_max": 47.73448944091797,
      "activations/layer18_attention_weight_min": -36.3602180480957,
      "activations/layer19_attention_weight_max": 20.016923904418945,
      "activations/layer19_attention_weight_min": -20.560733795166016,
      "activations/layer1_attention_weight_max": 15.5906982421875,
      "activations/layer1_attention_weight_min": -14.376921653747559,
      "activations/layer20_attention_weight_max": 22.164602279663086,
      "activations/layer20_attention_weight_min": -19.373931884765625,
      "activations/layer21_attention_weight_max": 36.29558181762695,
      "activations/layer21_attention_weight_min": -20.966264724731445,
      "activations/layer22_attention_weight_max": 29.06667137145996,
      "activations/layer22_attention_weight_min": -24.464305877685547,
      "activations/layer23_attention_weight_max": 33.789764404296875,
      "activations/layer23_attention_weight_min": -21.436384201049805,
      "activations/layer2_attention_weight_max": 33.088626861572266,
      "activations/layer2_attention_weight_min": -32.498390197753906,
      "activations/layer3_attention_weight_max": 94.12126159667969,
      "activations/layer3_attention_weight_min": -97.73656463623047,
      "activations/layer4_attention_weight_max": 101.3886947631836,
      "activations/layer4_attention_weight_min": -99.0399169921875,
      "activations/layer5_attention_weight_max": 72.59930419921875,
      "activations/layer5_attention_weight_min": -82.77426147460938,
      "activations/layer6_attention_weight_max": 56.838592529296875,
      "activations/layer6_attention_weight_min": -59.68465042114258,
      "activations/layer7_attention_weight_max": 77.94624328613281,
      "activations/layer7_attention_weight_min": -80.07500457763672,
      "activations/layer8_attention_weight_max": 48.90651321411133,
      "activations/layer8_attention_weight_min": -50.813777923583984,
      "activations/layer9_attention_weight_max": 48.47381591796875,
      "activations/layer9_attention_weight_min": -51.17608642578125,
      "epoch": 11.22,
      "learning_rate": 7.84340909090909e-05,
      "loss": 2.7698,
      "step": 193100
    },
    {
      "activations/layer0_attention_weight_max": 16.884180068969727,
      "activations/layer0_attention_weight_min": -14.470596313476562,
      "activations/layer10_attention_weight_max": 39.941246032714844,
      "activations/layer10_attention_weight_min": -35.73109817504883,
      "activations/layer11_attention_weight_max": 33.671600341796875,
      "activations/layer11_attention_weight_min": -33.799652099609375,
      "activations/layer12_attention_weight_max": 18.918537139892578,
      "activations/layer12_attention_weight_min": -24.433271408081055,
      "activations/layer13_attention_weight_max": 38.5799560546875,
      "activations/layer13_attention_weight_min": -31.39072036743164,
      "activations/layer14_attention_weight_max": 39.81232833862305,
      "activations/layer14_attention_weight_min": -30.41666030883789,
      "activations/layer15_attention_weight_max": 35.73016357421875,
      "activations/layer15_attention_weight_min": -30.069854736328125,
      "activations/layer16_attention_weight_max": 33.86083984375,
      "activations/layer16_attention_weight_min": -31.100618362426758,
      "activations/layer17_attention_weight_max": 55.397518157958984,
      "activations/layer17_attention_weight_min": -45.37346267700195,
      "activations/layer18_attention_weight_max": 48.392486572265625,
      "activations/layer18_attention_weight_min": -36.58906936645508,
      "activations/layer19_attention_weight_max": 24.3746395111084,
      "activations/layer19_attention_weight_min": -23.781604766845703,
      "activations/layer1_attention_weight_max": 16.9188175201416,
      "activations/layer1_attention_weight_min": -15.81903076171875,
      "activations/layer20_attention_weight_max": 24.932315826416016,
      "activations/layer20_attention_weight_min": -22.460599899291992,
      "activations/layer21_attention_weight_max": 42.16047286987305,
      "activations/layer21_attention_weight_min": -25.13470458984375,
      "activations/layer22_attention_weight_max": 35.23746109008789,
      "activations/layer22_attention_weight_min": -23.624553680419922,
      "activations/layer23_attention_weight_max": 38.08308410644531,
      "activations/layer23_attention_weight_min": -23.67110252380371,
      "activations/layer2_attention_weight_max": 34.144508361816406,
      "activations/layer2_attention_weight_min": -31.232900619506836,
      "activations/layer3_attention_weight_max": 99.8926010131836,
      "activations/layer3_attention_weight_min": -95.13320922851562,
      "activations/layer4_attention_weight_max": 104.22298431396484,
      "activations/layer4_attention_weight_min": -101.29986572265625,
      "activations/layer5_attention_weight_max": 76.18382263183594,
      "activations/layer5_attention_weight_min": -74.67059326171875,
      "activations/layer6_attention_weight_max": 54.03704071044922,
      "activations/layer6_attention_weight_min": -51.67451095581055,
      "activations/layer7_attention_weight_max": 79.82328033447266,
      "activations/layer7_attention_weight_min": -74.80558776855469,
      "activations/layer8_attention_weight_max": 50.27452087402344,
      "activations/layer8_attention_weight_min": -47.122615814208984,
      "activations/layer9_attention_weight_max": 52.72455978393555,
      "activations/layer9_attention_weight_min": -48.27912902832031,
      "epoch": 11.22,
      "learning_rate": 7.841515151515151e-05,
      "loss": 2.7811,
      "step": 193150
    },
    {
      "activations/layer0_attention_weight_max": 15.40239429473877,
      "activations/layer0_attention_weight_min": -14.129939079284668,
      "activations/layer10_attention_weight_max": 36.34103012084961,
      "activations/layer10_attention_weight_min": -35.432403564453125,
      "activations/layer11_attention_weight_max": 32.36621856689453,
      "activations/layer11_attention_weight_min": -33.24312210083008,
      "activations/layer12_attention_weight_max": 18.72584342956543,
      "activations/layer12_attention_weight_min": -25.94059181213379,
      "activations/layer13_attention_weight_max": 40.460784912109375,
      "activations/layer13_attention_weight_min": -30.045310974121094,
      "activations/layer14_attention_weight_max": 41.904510498046875,
      "activations/layer14_attention_weight_min": -31.519973754882812,
      "activations/layer15_attention_weight_max": 35.07988739013672,
      "activations/layer15_attention_weight_min": -29.024293899536133,
      "activations/layer16_attention_weight_max": 32.48358917236328,
      "activations/layer16_attention_weight_min": -30.151899337768555,
      "activations/layer17_attention_weight_max": 53.83748245239258,
      "activations/layer17_attention_weight_min": -44.607398986816406,
      "activations/layer18_attention_weight_max": 46.67041778564453,
      "activations/layer18_attention_weight_min": -35.66843032836914,
      "activations/layer19_attention_weight_max": 21.340633392333984,
      "activations/layer19_attention_weight_min": -20.04022216796875,
      "activations/layer1_attention_weight_max": 16.81175422668457,
      "activations/layer1_attention_weight_min": -14.795794486999512,
      "activations/layer20_attention_weight_max": 21.22601890563965,
      "activations/layer20_attention_weight_min": -19.629074096679688,
      "activations/layer21_attention_weight_max": 36.44926834106445,
      "activations/layer21_attention_weight_min": -22.15767478942871,
      "activations/layer22_attention_weight_max": 32.134971618652344,
      "activations/layer22_attention_weight_min": -24.934677124023438,
      "activations/layer23_attention_weight_max": 36.69844055175781,
      "activations/layer23_attention_weight_min": -23.39533042907715,
      "activations/layer2_attention_weight_max": 32.62900161743164,
      "activations/layer2_attention_weight_min": -31.004697799682617,
      "activations/layer3_attention_weight_max": 96.112060546875,
      "activations/layer3_attention_weight_min": -93.43242645263672,
      "activations/layer4_attention_weight_max": 98.29866027832031,
      "activations/layer4_attention_weight_min": -92.49283599853516,
      "activations/layer5_attention_weight_max": 70.68561553955078,
      "activations/layer5_attention_weight_min": -76.19257354736328,
      "activations/layer6_attention_weight_max": 53.20647048950195,
      "activations/layer6_attention_weight_min": -53.00040817260742,
      "activations/layer7_attention_weight_max": 77.66339874267578,
      "activations/layer7_attention_weight_min": -71.79070281982422,
      "activations/layer8_attention_weight_max": 48.29222869873047,
      "activations/layer8_attention_weight_min": -48.53427505493164,
      "activations/layer9_attention_weight_max": 49.18754196166992,
      "activations/layer9_attention_weight_min": -47.354347229003906,
      "epoch": 11.23,
      "learning_rate": 7.83965909090909e-05,
      "loss": 2.7876,
      "step": 193200
    },
    {
      "activations/layer0_attention_weight_max": 15.579313278198242,
      "activations/layer0_attention_weight_min": -14.355342864990234,
      "activations/layer10_attention_weight_max": 48.44942092895508,
      "activations/layer10_attention_weight_min": -42.45070266723633,
      "activations/layer11_attention_weight_max": 43.0490837097168,
      "activations/layer11_attention_weight_min": -36.742454528808594,
      "activations/layer12_attention_weight_max": 26.83807945251465,
      "activations/layer12_attention_weight_min": -27.083749771118164,
      "activations/layer13_attention_weight_max": 46.30996322631836,
      "activations/layer13_attention_weight_min": -32.40717697143555,
      "activations/layer14_attention_weight_max": 41.68661880493164,
      "activations/layer14_attention_weight_min": -33.50932312011719,
      "activations/layer15_attention_weight_max": 35.5413703918457,
      "activations/layer15_attention_weight_min": -31.554996490478516,
      "activations/layer16_attention_weight_max": 33.976959228515625,
      "activations/layer16_attention_weight_min": -31.58351707458496,
      "activations/layer17_attention_weight_max": 55.34972381591797,
      "activations/layer17_attention_weight_min": -48.36067581176758,
      "activations/layer18_attention_weight_max": 48.89809799194336,
      "activations/layer18_attention_weight_min": -41.59381866455078,
      "activations/layer19_attention_weight_max": 23.847213745117188,
      "activations/layer19_attention_weight_min": -23.50995445251465,
      "activations/layer1_attention_weight_max": 17.04578971862793,
      "activations/layer1_attention_weight_min": -16.679845809936523,
      "activations/layer20_attention_weight_max": 27.625732421875,
      "activations/layer20_attention_weight_min": -20.058773040771484,
      "activations/layer21_attention_weight_max": 41.26559066772461,
      "activations/layer21_attention_weight_min": -20.287065505981445,
      "activations/layer22_attention_weight_max": 33.111351013183594,
      "activations/layer22_attention_weight_min": -24.013458251953125,
      "activations/layer23_attention_weight_max": 34.979148864746094,
      "activations/layer23_attention_weight_min": -22.590023040771484,
      "activations/layer2_attention_weight_max": 36.65943145751953,
      "activations/layer2_attention_weight_min": -32.191368103027344,
      "activations/layer3_attention_weight_max": 97.20350646972656,
      "activations/layer3_attention_weight_min": -96.69656372070312,
      "activations/layer4_attention_weight_max": 102.13765716552734,
      "activations/layer4_attention_weight_min": -98.46711730957031,
      "activations/layer5_attention_weight_max": 75.6932373046875,
      "activations/layer5_attention_weight_min": -76.5585708618164,
      "activations/layer6_attention_weight_max": 61.521121978759766,
      "activations/layer6_attention_weight_min": -58.55028533935547,
      "activations/layer7_attention_weight_max": 89.63578033447266,
      "activations/layer7_attention_weight_min": -79.58332061767578,
      "activations/layer8_attention_weight_max": 55.915714263916016,
      "activations/layer8_attention_weight_min": -54.81431579589844,
      "activations/layer9_attention_weight_max": 67.5859146118164,
      "activations/layer9_attention_weight_min": -49.12303924560547,
      "epoch": 11.23,
      "learning_rate": 7.837765151515152e-05,
      "loss": 2.7641,
      "step": 193250
    },
    {
      "activations/layer0_attention_weight_max": 17.25114631652832,
      "activations/layer0_attention_weight_min": -14.16484260559082,
      "activations/layer10_attention_weight_max": 37.06746292114258,
      "activations/layer10_attention_weight_min": -34.90727615356445,
      "activations/layer11_attention_weight_max": 32.89256286621094,
      "activations/layer11_attention_weight_min": -31.48584747314453,
      "activations/layer12_attention_weight_max": 18.95508575439453,
      "activations/layer12_attention_weight_min": -24.904447555541992,
      "activations/layer13_attention_weight_max": 35.97922134399414,
      "activations/layer13_attention_weight_min": -33.72071838378906,
      "activations/layer14_attention_weight_max": 39.34474182128906,
      "activations/layer14_attention_weight_min": -29.073871612548828,
      "activations/layer15_attention_weight_max": 31.318359375,
      "activations/layer15_attention_weight_min": -27.949649810791016,
      "activations/layer16_attention_weight_max": 27.74533462524414,
      "activations/layer16_attention_weight_min": -28.300243377685547,
      "activations/layer17_attention_weight_max": 49.91337203979492,
      "activations/layer17_attention_weight_min": -42.9682502746582,
      "activations/layer18_attention_weight_max": 41.679954528808594,
      "activations/layer18_attention_weight_min": -35.59016418457031,
      "activations/layer19_attention_weight_max": 21.276906967163086,
      "activations/layer19_attention_weight_min": -20.736801147460938,
      "activations/layer1_attention_weight_max": 17.937530517578125,
      "activations/layer1_attention_weight_min": -14.758138656616211,
      "activations/layer20_attention_weight_max": 20.32862091064453,
      "activations/layer20_attention_weight_min": -20.881959915161133,
      "activations/layer21_attention_weight_max": 31.57151985168457,
      "activations/layer21_attention_weight_min": -20.622934341430664,
      "activations/layer22_attention_weight_max": 30.33041763305664,
      "activations/layer22_attention_weight_min": -24.531356811523438,
      "activations/layer23_attention_weight_max": 34.86906051635742,
      "activations/layer23_attention_weight_min": -27.325210571289062,
      "activations/layer2_attention_weight_max": 33.32304763793945,
      "activations/layer2_attention_weight_min": -30.741985321044922,
      "activations/layer3_attention_weight_max": 91.23957061767578,
      "activations/layer3_attention_weight_min": -91.12295532226562,
      "activations/layer4_attention_weight_max": 97.90497589111328,
      "activations/layer4_attention_weight_min": -91.44366455078125,
      "activations/layer5_attention_weight_max": 70.51078033447266,
      "activations/layer5_attention_weight_min": -81.75761413574219,
      "activations/layer6_attention_weight_max": 54.6379508972168,
      "activations/layer6_attention_weight_min": -51.604270935058594,
      "activations/layer7_attention_weight_max": 74.55535888671875,
      "activations/layer7_attention_weight_min": -75.43533325195312,
      "activations/layer8_attention_weight_max": 47.991390228271484,
      "activations/layer8_attention_weight_min": -47.45556640625,
      "activations/layer9_attention_weight_max": 54.74617385864258,
      "activations/layer9_attention_weight_min": -48.38534164428711,
      "epoch": 11.23,
      "learning_rate": 7.835909090909091e-05,
      "loss": 2.7608,
      "step": 193300
    },
    {
      "activations/layer0_attention_weight_max": 16.016313552856445,
      "activations/layer0_attention_weight_min": -14.099397659301758,
      "activations/layer10_attention_weight_max": 40.065391540527344,
      "activations/layer10_attention_weight_min": -37.69868850708008,
      "activations/layer11_attention_weight_max": 31.93622589111328,
      "activations/layer11_attention_weight_min": -33.00277328491211,
      "activations/layer12_attention_weight_max": 20.029911041259766,
      "activations/layer12_attention_weight_min": -25.102231979370117,
      "activations/layer13_attention_weight_max": 38.618194580078125,
      "activations/layer13_attention_weight_min": -30.432125091552734,
      "activations/layer14_attention_weight_max": 37.473567962646484,
      "activations/layer14_attention_weight_min": -29.739116668701172,
      "activations/layer15_attention_weight_max": 32.488677978515625,
      "activations/layer15_attention_weight_min": -28.81294059753418,
      "activations/layer16_attention_weight_max": 30.83201026916504,
      "activations/layer16_attention_weight_min": -30.869930267333984,
      "activations/layer17_attention_weight_max": 50.56659698486328,
      "activations/layer17_attention_weight_min": -46.77456283569336,
      "activations/layer18_attention_weight_max": 43.9642333984375,
      "activations/layer18_attention_weight_min": -39.46605682373047,
      "activations/layer19_attention_weight_max": 21.776945114135742,
      "activations/layer19_attention_weight_min": -22.39048957824707,
      "activations/layer1_attention_weight_max": 16.48453140258789,
      "activations/layer1_attention_weight_min": -14.950443267822266,
      "activations/layer20_attention_weight_max": 23.79033851623535,
      "activations/layer20_attention_weight_min": -22.36834716796875,
      "activations/layer21_attention_weight_max": 32.496795654296875,
      "activations/layer21_attention_weight_min": -24.561790466308594,
      "activations/layer22_attention_weight_max": 28.47711944580078,
      "activations/layer22_attention_weight_min": -27.32782554626465,
      "activations/layer23_attention_weight_max": 36.115081787109375,
      "activations/layer23_attention_weight_min": -24.320480346679688,
      "activations/layer2_attention_weight_max": 33.336402893066406,
      "activations/layer2_attention_weight_min": -30.082395553588867,
      "activations/layer3_attention_weight_max": 94.52348327636719,
      "activations/layer3_attention_weight_min": -96.5944595336914,
      "activations/layer4_attention_weight_max": 103.18791961669922,
      "activations/layer4_attention_weight_min": -101.78914642333984,
      "activations/layer5_attention_weight_max": 77.51937866210938,
      "activations/layer5_attention_weight_min": -82.37739562988281,
      "activations/layer6_attention_weight_max": 54.063072204589844,
      "activations/layer6_attention_weight_min": -54.389808654785156,
      "activations/layer7_attention_weight_max": 77.00468444824219,
      "activations/layer7_attention_weight_min": -78.78463745117188,
      "activations/layer8_attention_weight_max": 45.863548278808594,
      "activations/layer8_attention_weight_min": -49.56779861450195,
      "activations/layer9_attention_weight_max": 49.248374938964844,
      "activations/layer9_attention_weight_min": -48.177162170410156,
      "epoch": 11.23,
      "learning_rate": 7.83401515151515e-05,
      "loss": 2.7896,
      "step": 193350
    },
    {
      "activations/layer0_attention_weight_max": 15.949753761291504,
      "activations/layer0_attention_weight_min": -13.90450668334961,
      "activations/layer10_attention_weight_max": 33.751609802246094,
      "activations/layer10_attention_weight_min": -33.152931213378906,
      "activations/layer11_attention_weight_max": 30.889183044433594,
      "activations/layer11_attention_weight_min": -31.412443161010742,
      "activations/layer12_attention_weight_max": 19.134471893310547,
      "activations/layer12_attention_weight_min": -26.798616409301758,
      "activations/layer13_attention_weight_max": 34.37727737426758,
      "activations/layer13_attention_weight_min": -28.06529426574707,
      "activations/layer14_attention_weight_max": 35.72886657714844,
      "activations/layer14_attention_weight_min": -30.032394409179688,
      "activations/layer15_attention_weight_max": 31.813507080078125,
      "activations/layer15_attention_weight_min": -30.995756149291992,
      "activations/layer16_attention_weight_max": 32.783905029296875,
      "activations/layer16_attention_weight_min": -29.691152572631836,
      "activations/layer17_attention_weight_max": 49.674373626708984,
      "activations/layer17_attention_weight_min": -43.08476257324219,
      "activations/layer18_attention_weight_max": 46.82670211791992,
      "activations/layer18_attention_weight_min": -36.67374801635742,
      "activations/layer19_attention_weight_max": 22.108762741088867,
      "activations/layer19_attention_weight_min": -23.472557067871094,
      "activations/layer1_attention_weight_max": 16.27916145324707,
      "activations/layer1_attention_weight_min": -15.77424430847168,
      "activations/layer20_attention_weight_max": 22.089330673217773,
      "activations/layer20_attention_weight_min": -23.13543701171875,
      "activations/layer21_attention_weight_max": 37.1412353515625,
      "activations/layer21_attention_weight_min": -27.89915657043457,
      "activations/layer22_attention_weight_max": 31.152320861816406,
      "activations/layer22_attention_weight_min": -26.074155807495117,
      "activations/layer23_attention_weight_max": 38.45660400390625,
      "activations/layer23_attention_weight_min": -24.519737243652344,
      "activations/layer2_attention_weight_max": 35.208778381347656,
      "activations/layer2_attention_weight_min": -30.624235153198242,
      "activations/layer3_attention_weight_max": 96.38098907470703,
      "activations/layer3_attention_weight_min": -98.98018646240234,
      "activations/layer4_attention_weight_max": 95.66862487792969,
      "activations/layer4_attention_weight_min": -95.48109436035156,
      "activations/layer5_attention_weight_max": 69.44374084472656,
      "activations/layer5_attention_weight_min": -79.2365493774414,
      "activations/layer6_attention_weight_max": 51.696144104003906,
      "activations/layer6_attention_weight_min": -51.47637176513672,
      "activations/layer7_attention_weight_max": 71.6366195678711,
      "activations/layer7_attention_weight_min": -70.0859375,
      "activations/layer8_attention_weight_max": 42.33974075317383,
      "activations/layer8_attention_weight_min": -44.19743728637695,
      "activations/layer9_attention_weight_max": 43.922725677490234,
      "activations/layer9_attention_weight_min": -46.719337463378906,
      "epoch": 11.24,
      "learning_rate": 7.832121212121211e-05,
      "loss": 2.7918,
      "step": 193400
    },
    {
      "activations/layer0_attention_weight_max": 15.258573532104492,
      "activations/layer0_attention_weight_min": -14.149981498718262,
      "activations/layer10_attention_weight_max": 41.87236404418945,
      "activations/layer10_attention_weight_min": -37.96689224243164,
      "activations/layer11_attention_weight_max": 34.87031936645508,
      "activations/layer11_attention_weight_min": -33.28765106201172,
      "activations/layer12_attention_weight_max": 20.37335205078125,
      "activations/layer12_attention_weight_min": -24.321805953979492,
      "activations/layer13_attention_weight_max": 53.29926300048828,
      "activations/layer13_attention_weight_min": -33.417823791503906,
      "activations/layer14_attention_weight_max": 46.901676177978516,
      "activations/layer14_attention_weight_min": -30.983612060546875,
      "activations/layer15_attention_weight_max": 40.2940788269043,
      "activations/layer15_attention_weight_min": -29.4558162689209,
      "activations/layer16_attention_weight_max": 37.207271575927734,
      "activations/layer16_attention_weight_min": -29.391910552978516,
      "activations/layer17_attention_weight_max": 60.56062316894531,
      "activations/layer17_attention_weight_min": -47.413875579833984,
      "activations/layer18_attention_weight_max": 55.27775573730469,
      "activations/layer18_attention_weight_min": -39.97674560546875,
      "activations/layer19_attention_weight_max": 23.895706176757812,
      "activations/layer19_attention_weight_min": -21.39234733581543,
      "activations/layer1_attention_weight_max": 17.87859344482422,
      "activations/layer1_attention_weight_min": -18.56165885925293,
      "activations/layer20_attention_weight_max": 24.56163215637207,
      "activations/layer20_attention_weight_min": -20.008914947509766,
      "activations/layer21_attention_weight_max": 45.2063102722168,
      "activations/layer21_attention_weight_min": -26.275985717773438,
      "activations/layer22_attention_weight_max": 33.49583053588867,
      "activations/layer22_attention_weight_min": -25.36102867126465,
      "activations/layer23_attention_weight_max": 39.25848388671875,
      "activations/layer23_attention_weight_min": -25.3258113861084,
      "activations/layer2_attention_weight_max": 34.92119598388672,
      "activations/layer2_attention_weight_min": -32.542640686035156,
      "activations/layer3_attention_weight_max": 94.86378479003906,
      "activations/layer3_attention_weight_min": -94.05022430419922,
      "activations/layer4_attention_weight_max": 101.16507720947266,
      "activations/layer4_attention_weight_min": -97.02510070800781,
      "activations/layer5_attention_weight_max": 73.78309631347656,
      "activations/layer5_attention_weight_min": -74.71510314941406,
      "activations/layer6_attention_weight_max": 51.36908721923828,
      "activations/layer6_attention_weight_min": -52.44404983520508,
      "activations/layer7_attention_weight_max": 79.3304443359375,
      "activations/layer7_attention_weight_min": -71.23367309570312,
      "activations/layer8_attention_weight_max": 46.9940071105957,
      "activations/layer8_attention_weight_min": -48.07845687866211,
      "activations/layer9_attention_weight_max": 57.50883102416992,
      "activations/layer9_attention_weight_min": -46.618465423583984,
      "epoch": 11.24,
      "learning_rate": 7.830227272727273e-05,
      "loss": 2.7802,
      "step": 193450
    },
    {
      "activations/layer0_attention_weight_max": 15.573883056640625,
      "activations/layer0_attention_weight_min": -14.2272310256958,
      "activations/layer10_attention_weight_max": 35.40840530395508,
      "activations/layer10_attention_weight_min": -35.973907470703125,
      "activations/layer11_attention_weight_max": 33.41265869140625,
      "activations/layer11_attention_weight_min": -32.77354049682617,
      "activations/layer12_attention_weight_max": 17.81937026977539,
      "activations/layer12_attention_weight_min": -25.81216049194336,
      "activations/layer13_attention_weight_max": 37.575862884521484,
      "activations/layer13_attention_weight_min": -29.463293075561523,
      "activations/layer14_attention_weight_max": 36.53200912475586,
      "activations/layer14_attention_weight_min": -31.937728881835938,
      "activations/layer15_attention_weight_max": 32.949153900146484,
      "activations/layer15_attention_weight_min": -32.85495376586914,
      "activations/layer16_attention_weight_max": 31.670093536376953,
      "activations/layer16_attention_weight_min": -30.677095413208008,
      "activations/layer17_attention_weight_max": 57.25076675415039,
      "activations/layer17_attention_weight_min": -44.20447540283203,
      "activations/layer18_attention_weight_max": 51.11404800415039,
      "activations/layer18_attention_weight_min": -39.694725036621094,
      "activations/layer19_attention_weight_max": 23.533727645874023,
      "activations/layer19_attention_weight_min": -23.25484275817871,
      "activations/layer1_attention_weight_max": 17.46528434753418,
      "activations/layer1_attention_weight_min": -15.645898818969727,
      "activations/layer20_attention_weight_max": 26.663095474243164,
      "activations/layer20_attention_weight_min": -25.556583404541016,
      "activations/layer21_attention_weight_max": 42.892860412597656,
      "activations/layer21_attention_weight_min": -22.227436065673828,
      "activations/layer22_attention_weight_max": 31.394208908081055,
      "activations/layer22_attention_weight_min": -26.403127670288086,
      "activations/layer23_attention_weight_max": 39.0666618347168,
      "activations/layer23_attention_weight_min": -25.99080467224121,
      "activations/layer2_attention_weight_max": 32.382591247558594,
      "activations/layer2_attention_weight_min": -30.450490951538086,
      "activations/layer3_attention_weight_max": 94.18315887451172,
      "activations/layer3_attention_weight_min": -95.30256652832031,
      "activations/layer4_attention_weight_max": 96.61974334716797,
      "activations/layer4_attention_weight_min": -96.40010070800781,
      "activations/layer5_attention_weight_max": 72.92516326904297,
      "activations/layer5_attention_weight_min": -75.8564682006836,
      "activations/layer6_attention_weight_max": 51.78546905517578,
      "activations/layer6_attention_weight_min": -54.24320983886719,
      "activations/layer7_attention_weight_max": 74.89215087890625,
      "activations/layer7_attention_weight_min": -71.23992919921875,
      "activations/layer8_attention_weight_max": 46.53597640991211,
      "activations/layer8_attention_weight_min": -47.19697570800781,
      "activations/layer9_attention_weight_max": 57.5236701965332,
      "activations/layer9_attention_weight_min": -48.331600189208984,
      "epoch": 11.24,
      "learning_rate": 7.828333333333332e-05,
      "loss": 2.7867,
      "step": 193500
    },
    {
      "activations/layer0_attention_weight_max": 15.33627986907959,
      "activations/layer0_attention_weight_min": -13.673171043395996,
      "activations/layer10_attention_weight_max": 35.526039123535156,
      "activations/layer10_attention_weight_min": -35.07780838012695,
      "activations/layer11_attention_weight_max": 34.93736267089844,
      "activations/layer11_attention_weight_min": -35.79072570800781,
      "activations/layer12_attention_weight_max": 20.488582611083984,
      "activations/layer12_attention_weight_min": -23.661100387573242,
      "activations/layer13_attention_weight_max": 47.35151672363281,
      "activations/layer13_attention_weight_min": -28.575090408325195,
      "activations/layer14_attention_weight_max": 44.37192916870117,
      "activations/layer14_attention_weight_min": -30.344572067260742,
      "activations/layer15_attention_weight_max": 35.95035171508789,
      "activations/layer15_attention_weight_min": -28.248939514160156,
      "activations/layer16_attention_weight_max": 34.60966110229492,
      "activations/layer16_attention_weight_min": -29.552106857299805,
      "activations/layer17_attention_weight_max": 57.147216796875,
      "activations/layer17_attention_weight_min": -44.93245315551758,
      "activations/layer18_attention_weight_max": 52.77829360961914,
      "activations/layer18_attention_weight_min": -37.1337890625,
      "activations/layer19_attention_weight_max": 24.016971588134766,
      "activations/layer19_attention_weight_min": -24.187623977661133,
      "activations/layer1_attention_weight_max": 16.866436004638672,
      "activations/layer1_attention_weight_min": -15.401820182800293,
      "activations/layer20_attention_weight_max": 25.872957229614258,
      "activations/layer20_attention_weight_min": -21.152599334716797,
      "activations/layer21_attention_weight_max": 51.06901168823242,
      "activations/layer21_attention_weight_min": -25.225303649902344,
      "activations/layer22_attention_weight_max": 35.534996032714844,
      "activations/layer22_attention_weight_min": -25.617435455322266,
      "activations/layer23_attention_weight_max": 48.06377410888672,
      "activations/layer23_attention_weight_min": -25.32685661315918,
      "activations/layer2_attention_weight_max": 33.26753616333008,
      "activations/layer2_attention_weight_min": -31.15571403503418,
      "activations/layer3_attention_weight_max": 88.13090515136719,
      "activations/layer3_attention_weight_min": -90.6965103149414,
      "activations/layer4_attention_weight_max": 97.45126342773438,
      "activations/layer4_attention_weight_min": -92.81084442138672,
      "activations/layer5_attention_weight_max": 73.72501373291016,
      "activations/layer5_attention_weight_min": -79.42048645019531,
      "activations/layer6_attention_weight_max": 52.63588333129883,
      "activations/layer6_attention_weight_min": -53.99254608154297,
      "activations/layer7_attention_weight_max": 73.01749420166016,
      "activations/layer7_attention_weight_min": -75.57796478271484,
      "activations/layer8_attention_weight_max": 43.75669479370117,
      "activations/layer8_attention_weight_min": -46.08076095581055,
      "activations/layer9_attention_weight_max": 47.93372344970703,
      "activations/layer9_attention_weight_min": -45.27367401123047,
      "epoch": 11.25,
      "learning_rate": 7.826439393939393e-05,
      "loss": 2.7861,
      "step": 193550
    },
    {
      "activations/layer0_attention_weight_max": 15.653289794921875,
      "activations/layer0_attention_weight_min": -13.847317695617676,
      "activations/layer10_attention_weight_max": 35.77695846557617,
      "activations/layer10_attention_weight_min": -35.35648727416992,
      "activations/layer11_attention_weight_max": 32.11701583862305,
      "activations/layer11_attention_weight_min": -31.33349609375,
      "activations/layer12_attention_weight_max": 28.48703956604004,
      "activations/layer12_attention_weight_min": -25.16907501220703,
      "activations/layer13_attention_weight_max": 42.01676559448242,
      "activations/layer13_attention_weight_min": -31.274431228637695,
      "activations/layer14_attention_weight_max": 39.15827560424805,
      "activations/layer14_attention_weight_min": -29.06703758239746,
      "activations/layer15_attention_weight_max": 32.62113571166992,
      "activations/layer15_attention_weight_min": -29.646404266357422,
      "activations/layer16_attention_weight_max": 31.113935470581055,
      "activations/layer16_attention_weight_min": -29.706235885620117,
      "activations/layer17_attention_weight_max": 51.14348220825195,
      "activations/layer17_attention_weight_min": -45.49006652832031,
      "activations/layer18_attention_weight_max": 44.54277038574219,
      "activations/layer18_attention_weight_min": -38.545223236083984,
      "activations/layer19_attention_weight_max": 25.785207748413086,
      "activations/layer19_attention_weight_min": -21.12626075744629,
      "activations/layer1_attention_weight_max": 16.607486724853516,
      "activations/layer1_attention_weight_min": -16.43898582458496,
      "activations/layer20_attention_weight_max": 22.244230270385742,
      "activations/layer20_attention_weight_min": -21.996957778930664,
      "activations/layer21_attention_weight_max": 35.862979888916016,
      "activations/layer21_attention_weight_min": -24.90032386779785,
      "activations/layer22_attention_weight_max": 30.4007625579834,
      "activations/layer22_attention_weight_min": -26.66890525817871,
      "activations/layer23_attention_weight_max": 36.14628601074219,
      "activations/layer23_attention_weight_min": -22.597251892089844,
      "activations/layer2_attention_weight_max": 31.74283218383789,
      "activations/layer2_attention_weight_min": -30.571781158447266,
      "activations/layer3_attention_weight_max": 90.45262145996094,
      "activations/layer3_attention_weight_min": -89.80587005615234,
      "activations/layer4_attention_weight_max": 94.84467315673828,
      "activations/layer4_attention_weight_min": -93.22600555419922,
      "activations/layer5_attention_weight_max": 71.73268127441406,
      "activations/layer5_attention_weight_min": -83.97718048095703,
      "activations/layer6_attention_weight_max": 51.98908233642578,
      "activations/layer6_attention_weight_min": -51.78694152832031,
      "activations/layer7_attention_weight_max": 80.3715591430664,
      "activations/layer7_attention_weight_min": -71.05081176757812,
      "activations/layer8_attention_weight_max": 43.70188522338867,
      "activations/layer8_attention_weight_min": -46.85652542114258,
      "activations/layer9_attention_weight_max": 47.49188995361328,
      "activations/layer9_attention_weight_min": -47.8807373046875,
      "epoch": 11.25,
      "learning_rate": 7.824545454545455e-05,
      "loss": 2.7849,
      "step": 193600
    },
    {
      "activations/layer0_attention_weight_max": 15.897958755493164,
      "activations/layer0_attention_weight_min": -13.540396690368652,
      "activations/layer10_attention_weight_max": 38.472007751464844,
      "activations/layer10_attention_weight_min": -35.883296966552734,
      "activations/layer11_attention_weight_max": 35.243412017822266,
      "activations/layer11_attention_weight_min": -32.704219818115234,
      "activations/layer12_attention_weight_max": 18.983715057373047,
      "activations/layer12_attention_weight_min": -24.0849552154541,
      "activations/layer13_attention_weight_max": 45.61060333251953,
      "activations/layer13_attention_weight_min": -29.592844009399414,
      "activations/layer14_attention_weight_max": 39.870582580566406,
      "activations/layer14_attention_weight_min": -31.39201545715332,
      "activations/layer15_attention_weight_max": 37.32709884643555,
      "activations/layer15_attention_weight_min": -31.096418380737305,
      "activations/layer16_attention_weight_max": 32.816890716552734,
      "activations/layer16_attention_weight_min": -30.73173713684082,
      "activations/layer17_attention_weight_max": 54.9272346496582,
      "activations/layer17_attention_weight_min": -44.6076545715332,
      "activations/layer18_attention_weight_max": 52.33965301513672,
      "activations/layer18_attention_weight_min": -37.26023864746094,
      "activations/layer19_attention_weight_max": 22.19426918029785,
      "activations/layer19_attention_weight_min": -23.571094512939453,
      "activations/layer1_attention_weight_max": 17.207050323486328,
      "activations/layer1_attention_weight_min": -15.180034637451172,
      "activations/layer20_attention_weight_max": 23.202302932739258,
      "activations/layer20_attention_weight_min": -22.15947914123535,
      "activations/layer21_attention_weight_max": 37.861385345458984,
      "activations/layer21_attention_weight_min": -22.879257202148438,
      "activations/layer22_attention_weight_max": 31.715072631835938,
      "activations/layer22_attention_weight_min": -26.01030158996582,
      "activations/layer23_attention_weight_max": 38.703983306884766,
      "activations/layer23_attention_weight_min": -22.745952606201172,
      "activations/layer2_attention_weight_max": 33.67967987060547,
      "activations/layer2_attention_weight_min": -30.773048400878906,
      "activations/layer3_attention_weight_max": 99.38790893554688,
      "activations/layer3_attention_weight_min": -98.95784759521484,
      "activations/layer4_attention_weight_max": 102.10840606689453,
      "activations/layer4_attention_weight_min": -96.25657653808594,
      "activations/layer5_attention_weight_max": 74.95816802978516,
      "activations/layer5_attention_weight_min": -80.2451171875,
      "activations/layer6_attention_weight_max": 53.973243713378906,
      "activations/layer6_attention_weight_min": -56.643287658691406,
      "activations/layer7_attention_weight_max": 77.91779327392578,
      "activations/layer7_attention_weight_min": -78.06615447998047,
      "activations/layer8_attention_weight_max": 46.273651123046875,
      "activations/layer8_attention_weight_min": -51.07112121582031,
      "activations/layer9_attention_weight_max": 49.754398345947266,
      "activations/layer9_attention_weight_min": -48.55900192260742,
      "epoch": 11.25,
      "learning_rate": 7.822651515151513e-05,
      "loss": 2.7908,
      "step": 193650
    },
    {
      "activations/layer0_attention_weight_max": 16.070072174072266,
      "activations/layer0_attention_weight_min": -13.86281967163086,
      "activations/layer10_attention_weight_max": 39.17427062988281,
      "activations/layer10_attention_weight_min": -36.02974319458008,
      "activations/layer11_attention_weight_max": 33.628318786621094,
      "activations/layer11_attention_weight_min": -33.210243225097656,
      "activations/layer12_attention_weight_max": 18.437679290771484,
      "activations/layer12_attention_weight_min": -25.367679595947266,
      "activations/layer13_attention_weight_max": 41.26677322387695,
      "activations/layer13_attention_weight_min": -31.527080535888672,
      "activations/layer14_attention_weight_max": 34.794342041015625,
      "activations/layer14_attention_weight_min": -28.728254318237305,
      "activations/layer15_attention_weight_max": 32.57526397705078,
      "activations/layer15_attention_weight_min": -27.591079711914062,
      "activations/layer16_attention_weight_max": 29.93704605102539,
      "activations/layer16_attention_weight_min": -29.743160247802734,
      "activations/layer17_attention_weight_max": 51.381011962890625,
      "activations/layer17_attention_weight_min": -44.263572692871094,
      "activations/layer18_attention_weight_max": 44.493751525878906,
      "activations/layer18_attention_weight_min": -37.381263732910156,
      "activations/layer19_attention_weight_max": 19.534198760986328,
      "activations/layer19_attention_weight_min": -22.62557029724121,
      "activations/layer1_attention_weight_max": 17.427091598510742,
      "activations/layer1_attention_weight_min": -14.955461502075195,
      "activations/layer20_attention_weight_max": 22.48769760131836,
      "activations/layer20_attention_weight_min": -21.859861373901367,
      "activations/layer21_attention_weight_max": 32.26709747314453,
      "activations/layer21_attention_weight_min": -26.20979881286621,
      "activations/layer22_attention_weight_max": 29.603919982910156,
      "activations/layer22_attention_weight_min": -28.101093292236328,
      "activations/layer23_attention_weight_max": 35.05316925048828,
      "activations/layer23_attention_weight_min": -22.30052947998047,
      "activations/layer2_attention_weight_max": 32.409263610839844,
      "activations/layer2_attention_weight_min": -31.105201721191406,
      "activations/layer3_attention_weight_max": 95.23409271240234,
      "activations/layer3_attention_weight_min": -92.76632690429688,
      "activations/layer4_attention_weight_max": 100.2645034790039,
      "activations/layer4_attention_weight_min": -94.6335678100586,
      "activations/layer5_attention_weight_max": 71.46304321289062,
      "activations/layer5_attention_weight_min": -77.61934661865234,
      "activations/layer6_attention_weight_max": 50.47099685668945,
      "activations/layer6_attention_weight_min": -53.25902557373047,
      "activations/layer7_attention_weight_max": 74.35502624511719,
      "activations/layer7_attention_weight_min": -75.32227325439453,
      "activations/layer8_attention_weight_max": 46.01454544067383,
      "activations/layer8_attention_weight_min": -45.77833938598633,
      "activations/layer9_attention_weight_max": 49.13142395019531,
      "activations/layer9_attention_weight_min": -48.7268180847168,
      "epoch": 11.26,
      "learning_rate": 7.820757575757575e-05,
      "loss": 2.7733,
      "step": 193700
    },
    {
      "activations/layer0_attention_weight_max": 16.504138946533203,
      "activations/layer0_attention_weight_min": -13.63619327545166,
      "activations/layer10_attention_weight_max": 38.48018264770508,
      "activations/layer10_attention_weight_min": -36.73062515258789,
      "activations/layer11_attention_weight_max": 34.1065673828125,
      "activations/layer11_attention_weight_min": -33.000221252441406,
      "activations/layer12_attention_weight_max": 18.62696647644043,
      "activations/layer12_attention_weight_min": -24.44759750366211,
      "activations/layer13_attention_weight_max": 38.52094650268555,
      "activations/layer13_attention_weight_min": -28.475696563720703,
      "activations/layer14_attention_weight_max": 36.119529724121094,
      "activations/layer14_attention_weight_min": -31.014507293701172,
      "activations/layer15_attention_weight_max": 33.916725158691406,
      "activations/layer15_attention_weight_min": -30.277301788330078,
      "activations/layer16_attention_weight_max": 30.317276000976562,
      "activations/layer16_attention_weight_min": -29.22520637512207,
      "activations/layer17_attention_weight_max": 56.59577941894531,
      "activations/layer17_attention_weight_min": -42.15242385864258,
      "activations/layer18_attention_weight_max": 47.710426330566406,
      "activations/layer18_attention_weight_min": -37.111793518066406,
      "activations/layer19_attention_weight_max": 20.510650634765625,
      "activations/layer19_attention_weight_min": -20.128835678100586,
      "activations/layer1_attention_weight_max": 15.97789478302002,
      "activations/layer1_attention_weight_min": -13.8670072555542,
      "activations/layer20_attention_weight_max": 22.274761199951172,
      "activations/layer20_attention_weight_min": -20.005388259887695,
      "activations/layer21_attention_weight_max": 36.127052307128906,
      "activations/layer21_attention_weight_min": -24.34702491760254,
      "activations/layer22_attention_weight_max": 30.855609893798828,
      "activations/layer22_attention_weight_min": -25.458642959594727,
      "activations/layer23_attention_weight_max": 34.762935638427734,
      "activations/layer23_attention_weight_min": -24.828052520751953,
      "activations/layer2_attention_weight_max": 34.18498229980469,
      "activations/layer2_attention_weight_min": -30.589691162109375,
      "activations/layer3_attention_weight_max": 87.94984436035156,
      "activations/layer3_attention_weight_min": -89.5476303100586,
      "activations/layer4_attention_weight_max": 94.98979949951172,
      "activations/layer4_attention_weight_min": -89.78272247314453,
      "activations/layer5_attention_weight_max": 72.03761291503906,
      "activations/layer5_attention_weight_min": -74.84120178222656,
      "activations/layer6_attention_weight_max": 52.26700973510742,
      "activations/layer6_attention_weight_min": -50.44184112548828,
      "activations/layer7_attention_weight_max": 74.53109741210938,
      "activations/layer7_attention_weight_min": -72.57444763183594,
      "activations/layer8_attention_weight_max": 47.145267486572266,
      "activations/layer8_attention_weight_min": -47.99147415161133,
      "activations/layer9_attention_weight_max": 51.6442985534668,
      "activations/layer9_attention_weight_min": -48.74562454223633,
      "epoch": 11.26,
      "learning_rate": 7.818863636363636e-05,
      "loss": 2.7773,
      "step": 193750
    },
    {
      "activations/layer0_attention_weight_max": 16.626596450805664,
      "activations/layer0_attention_weight_min": -13.937589645385742,
      "activations/layer10_attention_weight_max": 38.14094543457031,
      "activations/layer10_attention_weight_min": -35.66944122314453,
      "activations/layer11_attention_weight_max": 34.788516998291016,
      "activations/layer11_attention_weight_min": -33.34441375732422,
      "activations/layer12_attention_weight_max": 21.501094818115234,
      "activations/layer12_attention_weight_min": -24.976572036743164,
      "activations/layer13_attention_weight_max": 43.79711151123047,
      "activations/layer13_attention_weight_min": -31.826465606689453,
      "activations/layer14_attention_weight_max": 47.520755767822266,
      "activations/layer14_attention_weight_min": -31.869144439697266,
      "activations/layer15_attention_weight_max": 38.07145309448242,
      "activations/layer15_attention_weight_min": -32.04547119140625,
      "activations/layer16_attention_weight_max": 34.64582061767578,
      "activations/layer16_attention_weight_min": -28.741104125976562,
      "activations/layer17_attention_weight_max": 54.936912536621094,
      "activations/layer17_attention_weight_min": -45.47511672973633,
      "activations/layer18_attention_weight_max": 48.001041412353516,
      "activations/layer18_attention_weight_min": -39.68473815917969,
      "activations/layer19_attention_weight_max": 22.305927276611328,
      "activations/layer19_attention_weight_min": -22.609724044799805,
      "activations/layer1_attention_weight_max": 16.543031692504883,
      "activations/layer1_attention_weight_min": -15.389416694641113,
      "activations/layer20_attention_weight_max": 23.332582473754883,
      "activations/layer20_attention_weight_min": -25.279268264770508,
      "activations/layer21_attention_weight_max": 40.26361846923828,
      "activations/layer21_attention_weight_min": -21.313579559326172,
      "activations/layer22_attention_weight_max": 33.258235931396484,
      "activations/layer22_attention_weight_min": -27.054014205932617,
      "activations/layer23_attention_weight_max": 39.43982696533203,
      "activations/layer23_attention_weight_min": -25.449848175048828,
      "activations/layer2_attention_weight_max": 32.674476623535156,
      "activations/layer2_attention_weight_min": -31.3754940032959,
      "activations/layer3_attention_weight_max": 91.8681869506836,
      "activations/layer3_attention_weight_min": -93.9905776977539,
      "activations/layer4_attention_weight_max": 97.03804779052734,
      "activations/layer4_attention_weight_min": -91.4264144897461,
      "activations/layer5_attention_weight_max": 75.12840270996094,
      "activations/layer5_attention_weight_min": -80.01066589355469,
      "activations/layer6_attention_weight_max": 56.78471755981445,
      "activations/layer6_attention_weight_min": -53.797157287597656,
      "activations/layer7_attention_weight_max": 82.77279663085938,
      "activations/layer7_attention_weight_min": -74.28117370605469,
      "activations/layer8_attention_weight_max": 52.80025100708008,
      "activations/layer8_attention_weight_min": -49.125938415527344,
      "activations/layer9_attention_weight_max": 51.64846420288086,
      "activations/layer9_attention_weight_min": -52.17280197143555,
      "epoch": 11.26,
      "learning_rate": 7.816969696969697e-05,
      "loss": 2.7741,
      "step": 193800
    },
    {
      "activations/layer0_attention_weight_max": 15.334667205810547,
      "activations/layer0_attention_weight_min": -13.841644287109375,
      "activations/layer10_attention_weight_max": 34.34528350830078,
      "activations/layer10_attention_weight_min": -33.55628967285156,
      "activations/layer11_attention_weight_max": 30.601238250732422,
      "activations/layer11_attention_weight_min": -30.199604034423828,
      "activations/layer12_attention_weight_max": 17.208969116210938,
      "activations/layer12_attention_weight_min": -23.399248123168945,
      "activations/layer13_attention_weight_max": 42.35803985595703,
      "activations/layer13_attention_weight_min": -31.698280334472656,
      "activations/layer14_attention_weight_max": 34.227108001708984,
      "activations/layer14_attention_weight_min": -28.567785263061523,
      "activations/layer15_attention_weight_max": 31.852670669555664,
      "activations/layer15_attention_weight_min": -29.90378761291504,
      "activations/layer16_attention_weight_max": 30.92707061767578,
      "activations/layer16_attention_weight_min": -29.81597328186035,
      "activations/layer17_attention_weight_max": 52.74916458129883,
      "activations/layer17_attention_weight_min": -44.43410110473633,
      "activations/layer18_attention_weight_max": 46.26350784301758,
      "activations/layer18_attention_weight_min": -36.39455032348633,
      "activations/layer19_attention_weight_max": 20.844099044799805,
      "activations/layer19_attention_weight_min": -21.72718620300293,
      "activations/layer1_attention_weight_max": 17.554197311401367,
      "activations/layer1_attention_weight_min": -13.662964820861816,
      "activations/layer20_attention_weight_max": 22.778831481933594,
      "activations/layer20_attention_weight_min": -21.25274658203125,
      "activations/layer21_attention_weight_max": 36.554630279541016,
      "activations/layer21_attention_weight_min": -24.341550827026367,
      "activations/layer22_attention_weight_max": 31.7576961517334,
      "activations/layer22_attention_weight_min": -24.66985321044922,
      "activations/layer23_attention_weight_max": 36.98649978637695,
      "activations/layer23_attention_weight_min": -23.871658325195312,
      "activations/layer2_attention_weight_max": 33.750701904296875,
      "activations/layer2_attention_weight_min": -31.715354919433594,
      "activations/layer3_attention_weight_max": 95.84036254882812,
      "activations/layer3_attention_weight_min": -87.52243041992188,
      "activations/layer4_attention_weight_max": 97.84163665771484,
      "activations/layer4_attention_weight_min": -90.76436614990234,
      "activations/layer5_attention_weight_max": 70.83562469482422,
      "activations/layer5_attention_weight_min": -76.38484191894531,
      "activations/layer6_attention_weight_max": 52.17768478393555,
      "activations/layer6_attention_weight_min": -50.772682189941406,
      "activations/layer7_attention_weight_max": 77.67567443847656,
      "activations/layer7_attention_weight_min": -70.78614044189453,
      "activations/layer8_attention_weight_max": 43.44835662841797,
      "activations/layer8_attention_weight_min": -46.394466400146484,
      "activations/layer9_attention_weight_max": 47.9466667175293,
      "activations/layer9_attention_weight_min": -48.4259033203125,
      "epoch": 11.26,
      "learning_rate": 7.815075757575757e-05,
      "loss": 2.7691,
      "step": 193850
    },
    {
      "activations/layer0_attention_weight_max": 15.342318534851074,
      "activations/layer0_attention_weight_min": -13.681756973266602,
      "activations/layer10_attention_weight_max": 49.1544303894043,
      "activations/layer10_attention_weight_min": -47.566314697265625,
      "activations/layer11_attention_weight_max": 41.97219467163086,
      "activations/layer11_attention_weight_min": -39.84278106689453,
      "activations/layer12_attention_weight_max": 17.43646812438965,
      "activations/layer12_attention_weight_min": -27.29276466369629,
      "activations/layer13_attention_weight_max": 41.718666076660156,
      "activations/layer13_attention_weight_min": -31.319435119628906,
      "activations/layer14_attention_weight_max": 38.926387786865234,
      "activations/layer14_attention_weight_min": -33.69709396362305,
      "activations/layer15_attention_weight_max": 34.75351333618164,
      "activations/layer15_attention_weight_min": -29.23774528503418,
      "activations/layer16_attention_weight_max": 29.882427215576172,
      "activations/layer16_attention_weight_min": -27.731966018676758,
      "activations/layer17_attention_weight_max": 47.31963348388672,
      "activations/layer17_attention_weight_min": -44.1717529296875,
      "activations/layer18_attention_weight_max": 43.62495422363281,
      "activations/layer18_attention_weight_min": -38.677528381347656,
      "activations/layer19_attention_weight_max": 20.385190963745117,
      "activations/layer19_attention_weight_min": -22.064002990722656,
      "activations/layer1_attention_weight_max": 16.106340408325195,
      "activations/layer1_attention_weight_min": -16.07171058654785,
      "activations/layer20_attention_weight_max": 21.916614532470703,
      "activations/layer20_attention_weight_min": -23.758007049560547,
      "activations/layer21_attention_weight_max": 34.192298889160156,
      "activations/layer21_attention_weight_min": -23.131824493408203,
      "activations/layer22_attention_weight_max": 30.53104591369629,
      "activations/layer22_attention_weight_min": -25.82806968688965,
      "activations/layer23_attention_weight_max": 37.69891357421875,
      "activations/layer23_attention_weight_min": -23.682310104370117,
      "activations/layer2_attention_weight_max": 35.10015106201172,
      "activations/layer2_attention_weight_min": -31.739017486572266,
      "activations/layer3_attention_weight_max": 95.15487670898438,
      "activations/layer3_attention_weight_min": -97.51526641845703,
      "activations/layer4_attention_weight_max": 99.48135375976562,
      "activations/layer4_attention_weight_min": -99.94881439208984,
      "activations/layer5_attention_weight_max": 73.8052749633789,
      "activations/layer5_attention_weight_min": -77.54195404052734,
      "activations/layer6_attention_weight_max": 57.17204284667969,
      "activations/layer6_attention_weight_min": -59.78388214111328,
      "activations/layer7_attention_weight_max": 91.10458374023438,
      "activations/layer7_attention_weight_min": -90.36410522460938,
      "activations/layer8_attention_weight_max": 61.55746078491211,
      "activations/layer8_attention_weight_min": -60.49977111816406,
      "activations/layer9_attention_weight_max": 68.19232177734375,
      "activations/layer9_attention_weight_min": -60.73501205444336,
      "epoch": 11.27,
      "learning_rate": 7.813181818181817e-05,
      "loss": 2.7994,
      "step": 193900
    },
    {
      "activations/layer0_attention_weight_max": 15.273608207702637,
      "activations/layer0_attention_weight_min": -13.72982406616211,
      "activations/layer10_attention_weight_max": 36.27864074707031,
      "activations/layer10_attention_weight_min": -35.00636672973633,
      "activations/layer11_attention_weight_max": 33.813594818115234,
      "activations/layer11_attention_weight_min": -34.34193420410156,
      "activations/layer12_attention_weight_max": 18.326980590820312,
      "activations/layer12_attention_weight_min": -25.120399475097656,
      "activations/layer13_attention_weight_max": 39.93648910522461,
      "activations/layer13_attention_weight_min": -30.22480010986328,
      "activations/layer14_attention_weight_max": 38.773136138916016,
      "activations/layer14_attention_weight_min": -33.08489227294922,
      "activations/layer15_attention_weight_max": 35.40530014038086,
      "activations/layer15_attention_weight_min": -29.767608642578125,
      "activations/layer16_attention_weight_max": 36.826297760009766,
      "activations/layer16_attention_weight_min": -32.15133285522461,
      "activations/layer17_attention_weight_max": 53.64226150512695,
      "activations/layer17_attention_weight_min": -45.535945892333984,
      "activations/layer18_attention_weight_max": 47.17382049560547,
      "activations/layer18_attention_weight_min": -38.96678924560547,
      "activations/layer19_attention_weight_max": 20.22246551513672,
      "activations/layer19_attention_weight_min": -21.999778747558594,
      "activations/layer1_attention_weight_max": 18.12861442565918,
      "activations/layer1_attention_weight_min": -15.165364265441895,
      "activations/layer20_attention_weight_max": 21.230430603027344,
      "activations/layer20_attention_weight_min": -22.104618072509766,
      "activations/layer21_attention_weight_max": 36.78388214111328,
      "activations/layer21_attention_weight_min": -25.117259979248047,
      "activations/layer22_attention_weight_max": 29.671615600585938,
      "activations/layer22_attention_weight_min": -24.74150276184082,
      "activations/layer23_attention_weight_max": 38.838218688964844,
      "activations/layer23_attention_weight_min": -25.97772789001465,
      "activations/layer2_attention_weight_max": 33.53319549560547,
      "activations/layer2_attention_weight_min": -32.88722229003906,
      "activations/layer3_attention_weight_max": 94.61788177490234,
      "activations/layer3_attention_weight_min": -97.67318725585938,
      "activations/layer4_attention_weight_max": 100.2371597290039,
      "activations/layer4_attention_weight_min": -101.01187896728516,
      "activations/layer5_attention_weight_max": 72.94731903076172,
      "activations/layer5_attention_weight_min": -75.2447509765625,
      "activations/layer6_attention_weight_max": 53.03360366821289,
      "activations/layer6_attention_weight_min": -50.78098678588867,
      "activations/layer7_attention_weight_max": 78.13443756103516,
      "activations/layer7_attention_weight_min": -72.68121337890625,
      "activations/layer8_attention_weight_max": 44.98017501831055,
      "activations/layer8_attention_weight_min": -44.86053466796875,
      "activations/layer9_attention_weight_max": 48.1541633605957,
      "activations/layer9_attention_weight_min": -51.22420120239258,
      "epoch": 11.27,
      "learning_rate": 7.811287878787878e-05,
      "loss": 2.7838,
      "step": 193950
    },
    {
      "activations/layer0_attention_weight_max": 15.108290672302246,
      "activations/layer0_attention_weight_min": -13.621760368347168,
      "activations/layer10_attention_weight_max": 44.25302505493164,
      "activations/layer10_attention_weight_min": -41.523075103759766,
      "activations/layer11_attention_weight_max": 38.22974395751953,
      "activations/layer11_attention_weight_min": -37.90116882324219,
      "activations/layer12_attention_weight_max": 18.547273635864258,
      "activations/layer12_attention_weight_min": -33.493553161621094,
      "activations/layer13_attention_weight_max": 42.18294906616211,
      "activations/layer13_attention_weight_min": -29.351694107055664,
      "activations/layer14_attention_weight_max": 42.02702331542969,
      "activations/layer14_attention_weight_min": -30.143844604492188,
      "activations/layer15_attention_weight_max": 36.1562385559082,
      "activations/layer15_attention_weight_min": -29.49190902709961,
      "activations/layer16_attention_weight_max": 32.15509796142578,
      "activations/layer16_attention_weight_min": -30.40320587158203,
      "activations/layer17_attention_weight_max": 59.76274871826172,
      "activations/layer17_attention_weight_min": -45.265052795410156,
      "activations/layer18_attention_weight_max": 49.05465316772461,
      "activations/layer18_attention_weight_min": -37.9954948425293,
      "activations/layer19_attention_weight_max": 22.065507888793945,
      "activations/layer19_attention_weight_min": -22.503625869750977,
      "activations/layer1_attention_weight_max": 17.154457092285156,
      "activations/layer1_attention_weight_min": -14.845767974853516,
      "activations/layer20_attention_weight_max": 24.55967903137207,
      "activations/layer20_attention_weight_min": -22.441816329956055,
      "activations/layer21_attention_weight_max": 45.891143798828125,
      "activations/layer21_attention_weight_min": -23.450302124023438,
      "activations/layer22_attention_weight_max": 31.867734909057617,
      "activations/layer22_attention_weight_min": -25.643722534179688,
      "activations/layer23_attention_weight_max": 38.36994171142578,
      "activations/layer23_attention_weight_min": -25.342971801757812,
      "activations/layer2_attention_weight_max": 33.00124740600586,
      "activations/layer2_attention_weight_min": -32.174468994140625,
      "activations/layer3_attention_weight_max": 97.8072280883789,
      "activations/layer3_attention_weight_min": -98.72869110107422,
      "activations/layer4_attention_weight_max": 107.53350067138672,
      "activations/layer4_attention_weight_min": -104.17384338378906,
      "activations/layer5_attention_weight_max": 78.16525268554688,
      "activations/layer5_attention_weight_min": -84.56971740722656,
      "activations/layer6_attention_weight_max": 57.68903732299805,
      "activations/layer6_attention_weight_min": -57.65061569213867,
      "activations/layer7_attention_weight_max": 89.4790267944336,
      "activations/layer7_attention_weight_min": -80.82263946533203,
      "activations/layer8_attention_weight_max": 56.35053634643555,
      "activations/layer8_attention_weight_min": -53.897850036621094,
      "activations/layer9_attention_weight_max": 58.9354133605957,
      "activations/layer9_attention_weight_min": -53.987701416015625,
      "epoch": 11.27,
      "learning_rate": 7.809393939393939e-05,
      "loss": 2.7789,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.4751,
      "eval_samples_per_second": 506.659,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.4751,
      "eval_openwebtext_samples_per_second": 506.659,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_wikitext_loss": 2.9609375,
      "eval_wikitext_ppl": 19.31607208722833,
      "eval_wikitext_runtime": 1.9802,
      "eval_wikitext_samples_per_second": 230.276,
      "step": 194000
    },
    {
      "epoch": 11.27,
      "eval_lambada_loss": 2.59765625,
      "eval_lambada_ppl": 13.432219349369987,
      "eval_lambada_runtime": 9.5486,
      "eval_lambada_samples_per_second": 509.919,
      "step": 194000
    },
    {
      "activations/layer0_attention_weight_max": 15.45354175567627,
      "activations/layer0_attention_weight_min": -13.857736587524414,
      "activations/layer10_attention_weight_max": 36.24541091918945,
      "activations/layer10_attention_weight_min": -35.984825134277344,
      "activations/layer11_attention_weight_max": 31.018735885620117,
      "activations/layer11_attention_weight_min": -33.28606033325195,
      "activations/layer12_attention_weight_max": 18.75494384765625,
      "activations/layer12_attention_weight_min": -26.12439727783203,
      "activations/layer13_attention_weight_max": 38.025970458984375,
      "activations/layer13_attention_weight_min": -30.192453384399414,
      "activations/layer14_attention_weight_max": 35.96315383911133,
      "activations/layer14_attention_weight_min": -30.866918563842773,
      "activations/layer15_attention_weight_max": 33.98286437988281,
      "activations/layer15_attention_weight_min": -29.665508270263672,
      "activations/layer16_attention_weight_max": 30.39299201965332,
      "activations/layer16_attention_weight_min": -27.80048179626465,
      "activations/layer17_attention_weight_max": 53.91139602661133,
      "activations/layer17_attention_weight_min": -43.488426208496094,
      "activations/layer18_attention_weight_max": 51.03081130981445,
      "activations/layer18_attention_weight_min": -36.209930419921875,
      "activations/layer19_attention_weight_max": 25.227935791015625,
      "activations/layer19_attention_weight_min": -22.317838668823242,
      "activations/layer1_attention_weight_max": 16.487653732299805,
      "activations/layer1_attention_weight_min": -14.606677055358887,
      "activations/layer20_attention_weight_max": 23.338220596313477,
      "activations/layer20_attention_weight_min": -23.167076110839844,
      "activations/layer21_attention_weight_max": 41.10908889770508,
      "activations/layer21_attention_weight_min": -25.224279403686523,
      "activations/layer22_attention_weight_max": 33.144588470458984,
      "activations/layer22_attention_weight_min": -25.98086166381836,
      "activations/layer23_attention_weight_max": 37.1829833984375,
      "activations/layer23_attention_weight_min": -21.990009307861328,
      "activations/layer2_attention_weight_max": 31.700908660888672,
      "activations/layer2_attention_weight_min": -30.368999481201172,
      "activations/layer3_attention_weight_max": 92.51910400390625,
      "activations/layer3_attention_weight_min": -93.9462661743164,
      "activations/layer4_attention_weight_max": 95.25627899169922,
      "activations/layer4_attention_weight_min": -91.98070526123047,
      "activations/layer5_attention_weight_max": 74.38426208496094,
      "activations/layer5_attention_weight_min": -83.89183044433594,
      "activations/layer6_attention_weight_max": 52.62870788574219,
      "activations/layer6_attention_weight_min": -50.457027435302734,
      "activations/layer7_attention_weight_max": 70.19597625732422,
      "activations/layer7_attention_weight_min": -74.08036041259766,
      "activations/layer8_attention_weight_max": 43.88533401489258,
      "activations/layer8_attention_weight_min": -46.88441848754883,
      "activations/layer9_attention_weight_max": 46.360260009765625,
      "activations/layer9_attention_weight_min": -48.85676956176758,
      "epoch": 11.28,
      "learning_rate": 7.807499999999999e-05,
      "loss": 2.788,
      "step": 194050
    },
    {
      "activations/layer0_attention_weight_max": 17.559810638427734,
      "activations/layer0_attention_weight_min": -13.873384475708008,
      "activations/layer10_attention_weight_max": 35.5911750793457,
      "activations/layer10_attention_weight_min": -35.165531158447266,
      "activations/layer11_attention_weight_max": 32.453941345214844,
      "activations/layer11_attention_weight_min": -32.25244903564453,
      "activations/layer12_attention_weight_max": 21.485275268554688,
      "activations/layer12_attention_weight_min": -25.315975189208984,
      "activations/layer13_attention_weight_max": 35.961944580078125,
      "activations/layer13_attention_weight_min": -28.59807586669922,
      "activations/layer14_attention_weight_max": 41.56905746459961,
      "activations/layer14_attention_weight_min": -29.676992416381836,
      "activations/layer15_attention_weight_max": 32.7702751159668,
      "activations/layer15_attention_weight_min": -29.791887283325195,
      "activations/layer16_attention_weight_max": 28.951007843017578,
      "activations/layer16_attention_weight_min": -29.188291549682617,
      "activations/layer17_attention_weight_max": 49.65709686279297,
      "activations/layer17_attention_weight_min": -46.48367691040039,
      "activations/layer18_attention_weight_max": 44.24912643432617,
      "activations/layer18_attention_weight_min": -38.472984313964844,
      "activations/layer19_attention_weight_max": 20.98174476623535,
      "activations/layer19_attention_weight_min": -22.011493682861328,
      "activations/layer1_attention_weight_max": 17.338953018188477,
      "activations/layer1_attention_weight_min": -15.377080917358398,
      "activations/layer20_attention_weight_max": 20.020566940307617,
      "activations/layer20_attention_weight_min": -20.909757614135742,
      "activations/layer21_attention_weight_max": 37.039794921875,
      "activations/layer21_attention_weight_min": -25.849491119384766,
      "activations/layer22_attention_weight_max": 29.70747947692871,
      "activations/layer22_attention_weight_min": -25.138927459716797,
      "activations/layer23_attention_weight_max": 37.96199035644531,
      "activations/layer23_attention_weight_min": -25.793323516845703,
      "activations/layer2_attention_weight_max": 33.91704559326172,
      "activations/layer2_attention_weight_min": -32.86148452758789,
      "activations/layer3_attention_weight_max": 95.85923767089844,
      "activations/layer3_attention_weight_min": -97.09307098388672,
      "activations/layer4_attention_weight_max": 103.74076080322266,
      "activations/layer4_attention_weight_min": -92.9087142944336,
      "activations/layer5_attention_weight_max": 69.4711685180664,
      "activations/layer5_attention_weight_min": -76.06304168701172,
      "activations/layer6_attention_weight_max": 48.7503547668457,
      "activations/layer6_attention_weight_min": -50.396846771240234,
      "activations/layer7_attention_weight_max": 70.90374755859375,
      "activations/layer7_attention_weight_min": -71.06272888183594,
      "activations/layer8_attention_weight_max": 41.69545364379883,
      "activations/layer8_attention_weight_min": -45.5459098815918,
      "activations/layer9_attention_weight_max": 51.96342849731445,
      "activations/layer9_attention_weight_min": -49.00566101074219,
      "epoch": 11.28,
      "learning_rate": 7.80560606060606e-05,
      "loss": 2.7808,
      "step": 194100
    },
    {
      "activations/layer0_attention_weight_max": 15.419986724853516,
      "activations/layer0_attention_weight_min": -14.488227844238281,
      "activations/layer10_attention_weight_max": 40.7842903137207,
      "activations/layer10_attention_weight_min": -40.890541076660156,
      "activations/layer11_attention_weight_max": 34.24195098876953,
      "activations/layer11_attention_weight_min": -35.46140670776367,
      "activations/layer12_attention_weight_max": 19.836450576782227,
      "activations/layer12_attention_weight_min": -26.61601448059082,
      "activations/layer13_attention_weight_max": 37.16483688354492,
      "activations/layer13_attention_weight_min": -28.965843200683594,
      "activations/layer14_attention_weight_max": 38.75873947143555,
      "activations/layer14_attention_weight_min": -31.336740493774414,
      "activations/layer15_attention_weight_max": 36.23387908935547,
      "activations/layer15_attention_weight_min": -31.135440826416016,
      "activations/layer16_attention_weight_max": 33.39451599121094,
      "activations/layer16_attention_weight_min": -32.250877380371094,
      "activations/layer17_attention_weight_max": 57.06939697265625,
      "activations/layer17_attention_weight_min": -47.654197692871094,
      "activations/layer18_attention_weight_max": 49.739219665527344,
      "activations/layer18_attention_weight_min": -40.35514450073242,
      "activations/layer19_attention_weight_max": 24.721372604370117,
      "activations/layer19_attention_weight_min": -23.904726028442383,
      "activations/layer1_attention_weight_max": 17.127613067626953,
      "activations/layer1_attention_weight_min": -14.597739219665527,
      "activations/layer20_attention_weight_max": 24.931617736816406,
      "activations/layer20_attention_weight_min": -23.109981536865234,
      "activations/layer21_attention_weight_max": 40.61518478393555,
      "activations/layer21_attention_weight_min": -23.78165054321289,
      "activations/layer22_attention_weight_max": 37.13986587524414,
      "activations/layer22_attention_weight_min": -26.70941925048828,
      "activations/layer23_attention_weight_max": 42.124671936035156,
      "activations/layer23_attention_weight_min": -25.95711898803711,
      "activations/layer2_attention_weight_max": 33.82380676269531,
      "activations/layer2_attention_weight_min": -31.161155700683594,
      "activations/layer3_attention_weight_max": 92.7826156616211,
      "activations/layer3_attention_weight_min": -91.63701629638672,
      "activations/layer4_attention_weight_max": 97.92790222167969,
      "activations/layer4_attention_weight_min": -92.5680160522461,
      "activations/layer5_attention_weight_max": 74.37815856933594,
      "activations/layer5_attention_weight_min": -76.32791137695312,
      "activations/layer6_attention_weight_max": 55.036231994628906,
      "activations/layer6_attention_weight_min": -52.928096771240234,
      "activations/layer7_attention_weight_max": 73.28682708740234,
      "activations/layer7_attention_weight_min": -72.97280883789062,
      "activations/layer8_attention_weight_max": 50.88393783569336,
      "activations/layer8_attention_weight_min": -49.69120788574219,
      "activations/layer9_attention_weight_max": 54.76626205444336,
      "activations/layer9_attention_weight_min": -48.63735580444336,
      "epoch": 11.28,
      "learning_rate": 7.803712121212122e-05,
      "loss": 2.782,
      "step": 194150
    },
    {
      "activations/layer0_attention_weight_max": 15.950922966003418,
      "activations/layer0_attention_weight_min": -14.619837760925293,
      "activations/layer10_attention_weight_max": 35.14933776855469,
      "activations/layer10_attention_weight_min": -33.48834228515625,
      "activations/layer11_attention_weight_max": 31.126094818115234,
      "activations/layer11_attention_weight_min": -31.016841888427734,
      "activations/layer12_attention_weight_max": 20.573341369628906,
      "activations/layer12_attention_weight_min": -24.886402130126953,
      "activations/layer13_attention_weight_max": 35.86305236816406,
      "activations/layer13_attention_weight_min": -28.478126525878906,
      "activations/layer14_attention_weight_max": 36.331634521484375,
      "activations/layer14_attention_weight_min": -31.129756927490234,
      "activations/layer15_attention_weight_max": 37.301021575927734,
      "activations/layer15_attention_weight_min": -29.67678451538086,
      "activations/layer16_attention_weight_max": 33.81509780883789,
      "activations/layer16_attention_weight_min": -31.28629493713379,
      "activations/layer17_attention_weight_max": 57.88947296142578,
      "activations/layer17_attention_weight_min": -49.17171096801758,
      "activations/layer18_attention_weight_max": 51.2318229675293,
      "activations/layer18_attention_weight_min": -40.87760925292969,
      "activations/layer19_attention_weight_max": 22.013090133666992,
      "activations/layer19_attention_weight_min": -22.727149963378906,
      "activations/layer1_attention_weight_max": 17.26851463317871,
      "activations/layer1_attention_weight_min": -16.530332565307617,
      "activations/layer20_attention_weight_max": 24.159704208374023,
      "activations/layer20_attention_weight_min": -23.988771438598633,
      "activations/layer21_attention_weight_max": 39.15431594848633,
      "activations/layer21_attention_weight_min": -21.13824462890625,
      "activations/layer22_attention_weight_max": 32.00202178955078,
      "activations/layer22_attention_weight_min": -24.96084213256836,
      "activations/layer23_attention_weight_max": 36.454036712646484,
      "activations/layer23_attention_weight_min": -23.26172637939453,
      "activations/layer2_attention_weight_max": 34.03802490234375,
      "activations/layer2_attention_weight_min": -31.428874969482422,
      "activations/layer3_attention_weight_max": 90.5719985961914,
      "activations/layer3_attention_weight_min": -94.05975341796875,
      "activations/layer4_attention_weight_max": 98.66668701171875,
      "activations/layer4_attention_weight_min": -96.28540802001953,
      "activations/layer5_attention_weight_max": 71.065185546875,
      "activations/layer5_attention_weight_min": -77.9561767578125,
      "activations/layer6_attention_weight_max": 52.78081130981445,
      "activations/layer6_attention_weight_min": -51.77214050292969,
      "activations/layer7_attention_weight_max": 74.47002410888672,
      "activations/layer7_attention_weight_min": -75.00735473632812,
      "activations/layer8_attention_weight_max": 41.573429107666016,
      "activations/layer8_attention_weight_min": -46.613739013671875,
      "activations/layer9_attention_weight_max": 50.181331634521484,
      "activations/layer9_attention_weight_min": -46.130001068115234,
      "epoch": 11.28,
      "learning_rate": 7.80181818181818e-05,
      "loss": 2.7642,
      "step": 194200
    },
    {
      "activations/layer0_attention_weight_max": 14.937533378601074,
      "activations/layer0_attention_weight_min": -14.39451789855957,
      "activations/layer10_attention_weight_max": 37.82471466064453,
      "activations/layer10_attention_weight_min": -36.9093017578125,
      "activations/layer11_attention_weight_max": 33.228912353515625,
      "activations/layer11_attention_weight_min": -31.807111740112305,
      "activations/layer12_attention_weight_max": 20.62640953063965,
      "activations/layer12_attention_weight_min": -25.797273635864258,
      "activations/layer13_attention_weight_max": 34.19597625732422,
      "activations/layer13_attention_weight_min": -31.88113784790039,
      "activations/layer14_attention_weight_max": 39.77613830566406,
      "activations/layer14_attention_weight_min": -32.585205078125,
      "activations/layer15_attention_weight_max": 36.40275955200195,
      "activations/layer15_attention_weight_min": -31.254972457885742,
      "activations/layer16_attention_weight_max": 31.15623664855957,
      "activations/layer16_attention_weight_min": -27.37028694152832,
      "activations/layer17_attention_weight_max": 55.539649963378906,
      "activations/layer17_attention_weight_min": -42.71449661254883,
      "activations/layer18_attention_weight_max": 47.669403076171875,
      "activations/layer18_attention_weight_min": -35.0958366394043,
      "activations/layer19_attention_weight_max": 21.889019012451172,
      "activations/layer19_attention_weight_min": -21.036590576171875,
      "activations/layer1_attention_weight_max": 17.046680450439453,
      "activations/layer1_attention_weight_min": -14.422368049621582,
      "activations/layer20_attention_weight_max": 23.256614685058594,
      "activations/layer20_attention_weight_min": -22.404142379760742,
      "activations/layer21_attention_weight_max": 32.57952880859375,
      "activations/layer21_attention_weight_min": -21.494714736938477,
      "activations/layer22_attention_weight_max": 32.55191421508789,
      "activations/layer22_attention_weight_min": -26.980663299560547,
      "activations/layer23_attention_weight_max": 36.806060791015625,
      "activations/layer23_attention_weight_min": -25.608264923095703,
      "activations/layer2_attention_weight_max": 32.368709564208984,
      "activations/layer2_attention_weight_min": -31.143964767456055,
      "activations/layer3_attention_weight_max": 91.94622039794922,
      "activations/layer3_attention_weight_min": -96.33646392822266,
      "activations/layer4_attention_weight_max": 94.19820404052734,
      "activations/layer4_attention_weight_min": -93.88585662841797,
      "activations/layer5_attention_weight_max": 69.63755798339844,
      "activations/layer5_attention_weight_min": -79.30460357666016,
      "activations/layer6_attention_weight_max": 50.049339294433594,
      "activations/layer6_attention_weight_min": -50.03157424926758,
      "activations/layer7_attention_weight_max": 74.68025207519531,
      "activations/layer7_attention_weight_min": -71.27913665771484,
      "activations/layer8_attention_weight_max": 44.83100128173828,
      "activations/layer8_attention_weight_min": -46.66849136352539,
      "activations/layer9_attention_weight_max": 50.63780975341797,
      "activations/layer9_attention_weight_min": -44.554012298583984,
      "epoch": 11.29,
      "learning_rate": 7.799924242424242e-05,
      "loss": 2.7712,
      "step": 194250
    },
    {
      "activations/layer0_attention_weight_max": 15.734047889709473,
      "activations/layer0_attention_weight_min": -14.543874740600586,
      "activations/layer10_attention_weight_max": 35.25465393066406,
      "activations/layer10_attention_weight_min": -33.38222122192383,
      "activations/layer11_attention_weight_max": 32.238487243652344,
      "activations/layer11_attention_weight_min": -30.364364624023438,
      "activations/layer12_attention_weight_max": 17.583486557006836,
      "activations/layer12_attention_weight_min": -23.41269302368164,
      "activations/layer13_attention_weight_max": 36.390899658203125,
      "activations/layer13_attention_weight_min": -27.718570709228516,
      "activations/layer14_attention_weight_max": 35.13130187988281,
      "activations/layer14_attention_weight_min": -30.3980655670166,
      "activations/layer15_attention_weight_max": 31.241249084472656,
      "activations/layer15_attention_weight_min": -28.293357849121094,
      "activations/layer16_attention_weight_max": 29.21344757080078,
      "activations/layer16_attention_weight_min": -29.690366744995117,
      "activations/layer17_attention_weight_max": 48.46821594238281,
      "activations/layer17_attention_weight_min": -44.45088577270508,
      "activations/layer18_attention_weight_max": 42.78157424926758,
      "activations/layer18_attention_weight_min": -39.64127731323242,
      "activations/layer19_attention_weight_max": 23.38169288635254,
      "activations/layer19_attention_weight_min": -26.132638931274414,
      "activations/layer1_attention_weight_max": 17.406023025512695,
      "activations/layer1_attention_weight_min": -14.236096382141113,
      "activations/layer20_attention_weight_max": 22.161949157714844,
      "activations/layer20_attention_weight_min": -22.756881713867188,
      "activations/layer21_attention_weight_max": 34.461326599121094,
      "activations/layer21_attention_weight_min": -25.84494400024414,
      "activations/layer22_attention_weight_max": 28.745086669921875,
      "activations/layer22_attention_weight_min": -27.643056869506836,
      "activations/layer23_attention_weight_max": 35.328697204589844,
      "activations/layer23_attention_weight_min": -23.238998413085938,
      "activations/layer2_attention_weight_max": 31.779521942138672,
      "activations/layer2_attention_weight_min": -30.683652877807617,
      "activations/layer3_attention_weight_max": 86.73766326904297,
      "activations/layer3_attention_weight_min": -91.00911712646484,
      "activations/layer4_attention_weight_max": 93.6927719116211,
      "activations/layer4_attention_weight_min": -91.48257446289062,
      "activations/layer5_attention_weight_max": 71.97913360595703,
      "activations/layer5_attention_weight_min": -77.83982849121094,
      "activations/layer6_attention_weight_max": 55.83710861206055,
      "activations/layer6_attention_weight_min": -51.564796447753906,
      "activations/layer7_attention_weight_max": 75.94424438476562,
      "activations/layer7_attention_weight_min": -69.4650650024414,
      "activations/layer8_attention_weight_max": 42.74968338012695,
      "activations/layer8_attention_weight_min": -46.173500061035156,
      "activations/layer9_attention_weight_max": 47.51290512084961,
      "activations/layer9_attention_weight_min": -49.370853424072266,
      "epoch": 11.29,
      "learning_rate": 7.798030303030302e-05,
      "loss": 2.7812,
      "step": 194300
    },
    {
      "activations/layer0_attention_weight_max": 15.627552032470703,
      "activations/layer0_attention_weight_min": -14.262382507324219,
      "activations/layer10_attention_weight_max": 34.52080154418945,
      "activations/layer10_attention_weight_min": -33.73377227783203,
      "activations/layer11_attention_weight_max": 30.501991271972656,
      "activations/layer11_attention_weight_min": -31.231672286987305,
      "activations/layer12_attention_weight_max": 17.025911331176758,
      "activations/layer12_attention_weight_min": -25.253990173339844,
      "activations/layer13_attention_weight_max": 33.92380905151367,
      "activations/layer13_attention_weight_min": -27.41136360168457,
      "activations/layer14_attention_weight_max": 32.752281188964844,
      "activations/layer14_attention_weight_min": -29.6441650390625,
      "activations/layer15_attention_weight_max": 29.568906784057617,
      "activations/layer15_attention_weight_min": -27.974470138549805,
      "activations/layer16_attention_weight_max": 28.711402893066406,
      "activations/layer16_attention_weight_min": -29.51732635498047,
      "activations/layer17_attention_weight_max": 45.170448303222656,
      "activations/layer17_attention_weight_min": -42.45137405395508,
      "activations/layer18_attention_weight_max": 40.0648307800293,
      "activations/layer18_attention_weight_min": -36.45073699951172,
      "activations/layer19_attention_weight_max": 18.707138061523438,
      "activations/layer19_attention_weight_min": -19.361982345581055,
      "activations/layer1_attention_weight_max": 16.88234519958496,
      "activations/layer1_attention_weight_min": -14.19605827331543,
      "activations/layer20_attention_weight_max": 20.871179580688477,
      "activations/layer20_attention_weight_min": -23.46556282043457,
      "activations/layer21_attention_weight_max": 34.577999114990234,
      "activations/layer21_attention_weight_min": -23.578289031982422,
      "activations/layer22_attention_weight_max": 25.53952980041504,
      "activations/layer22_attention_weight_min": -27.357616424560547,
      "activations/layer23_attention_weight_max": 35.98390579223633,
      "activations/layer23_attention_weight_min": -25.260372161865234,
      "activations/layer2_attention_weight_max": 32.493247985839844,
      "activations/layer2_attention_weight_min": -28.369197845458984,
      "activations/layer3_attention_weight_max": 93.26944732666016,
      "activations/layer3_attention_weight_min": -88.99784851074219,
      "activations/layer4_attention_weight_max": 95.56774139404297,
      "activations/layer4_attention_weight_min": -90.836669921875,
      "activations/layer5_attention_weight_max": 69.41804504394531,
      "activations/layer5_attention_weight_min": -73.22403717041016,
      "activations/layer6_attention_weight_max": 51.52092742919922,
      "activations/layer6_attention_weight_min": -50.90441131591797,
      "activations/layer7_attention_weight_max": 74.88758850097656,
      "activations/layer7_attention_weight_min": -71.61670684814453,
      "activations/layer8_attention_weight_max": 44.40288162231445,
      "activations/layer8_attention_weight_min": -43.79818344116211,
      "activations/layer9_attention_weight_max": 52.508663177490234,
      "activations/layer9_attention_weight_min": -45.96407699584961,
      "epoch": 11.29,
      "learning_rate": 7.796136363636362e-05,
      "loss": 2.7899,
      "step": 194350
    },
    {
      "activations/layer0_attention_weight_max": 15.13955307006836,
      "activations/layer0_attention_weight_min": -14.182378768920898,
      "activations/layer10_attention_weight_max": 37.28373336791992,
      "activations/layer10_attention_weight_min": -35.45399856567383,
      "activations/layer11_attention_weight_max": 31.523300170898438,
      "activations/layer11_attention_weight_min": -34.137542724609375,
      "activations/layer12_attention_weight_max": 17.517370223999023,
      "activations/layer12_attention_weight_min": -22.444787979125977,
      "activations/layer13_attention_weight_max": 38.27653121948242,
      "activations/layer13_attention_weight_min": -28.654958724975586,
      "activations/layer14_attention_weight_max": 37.717315673828125,
      "activations/layer14_attention_weight_min": -27.32062530517578,
      "activations/layer15_attention_weight_max": 33.6459846496582,
      "activations/layer15_attention_weight_min": -28.964574813842773,
      "activations/layer16_attention_weight_max": 33.15386199951172,
      "activations/layer16_attention_weight_min": -28.471832275390625,
      "activations/layer17_attention_weight_max": 52.56559371948242,
      "activations/layer17_attention_weight_min": -42.17129135131836,
      "activations/layer18_attention_weight_max": 47.85425567626953,
      "activations/layer18_attention_weight_min": -40.665164947509766,
      "activations/layer19_attention_weight_max": 25.274137496948242,
      "activations/layer19_attention_weight_min": -21.66514778137207,
      "activations/layer1_attention_weight_max": 16.451940536499023,
      "activations/layer1_attention_weight_min": -15.862639427185059,
      "activations/layer20_attention_weight_max": 25.448501586914062,
      "activations/layer20_attention_weight_min": -23.39818000793457,
      "activations/layer21_attention_weight_max": 42.938079833984375,
      "activations/layer21_attention_weight_min": -23.71169090270996,
      "activations/layer22_attention_weight_max": 33.803707122802734,
      "activations/layer22_attention_weight_min": -23.85088348388672,
      "activations/layer23_attention_weight_max": 40.6700325012207,
      "activations/layer23_attention_weight_min": -23.982282638549805,
      "activations/layer2_attention_weight_max": 32.39333724975586,
      "activations/layer2_attention_weight_min": -29.5205135345459,
      "activations/layer3_attention_weight_max": 88.0208969116211,
      "activations/layer3_attention_weight_min": -88.67390441894531,
      "activations/layer4_attention_weight_max": 98.05130004882812,
      "activations/layer4_attention_weight_min": -99.1234359741211,
      "activations/layer5_attention_weight_max": 75.34758758544922,
      "activations/layer5_attention_weight_min": -74.867431640625,
      "activations/layer6_attention_weight_max": 53.47311782836914,
      "activations/layer6_attention_weight_min": -52.27320098876953,
      "activations/layer7_attention_weight_max": 72.41097259521484,
      "activations/layer7_attention_weight_min": -71.3875732421875,
      "activations/layer8_attention_weight_max": 46.13657760620117,
      "activations/layer8_attention_weight_min": -46.39036178588867,
      "activations/layer9_attention_weight_max": 49.3224983215332,
      "activations/layer9_attention_weight_min": -47.44170379638672,
      "epoch": 11.3,
      "learning_rate": 7.794242424242424e-05,
      "loss": 2.7547,
      "step": 194400
    },
    {
      "activations/layer0_attention_weight_max": 14.897767066955566,
      "activations/layer0_attention_weight_min": -13.86555004119873,
      "activations/layer10_attention_weight_max": 35.11205291748047,
      "activations/layer10_attention_weight_min": -34.10920333862305,
      "activations/layer11_attention_weight_max": 32.494876861572266,
      "activations/layer11_attention_weight_min": -32.521812438964844,
      "activations/layer12_attention_weight_max": 21.097448348999023,
      "activations/layer12_attention_weight_min": -23.39472007751465,
      "activations/layer13_attention_weight_max": 39.05223846435547,
      "activations/layer13_attention_weight_min": -29.79136085510254,
      "activations/layer14_attention_weight_max": 36.00735855102539,
      "activations/layer14_attention_weight_min": -31.000436782836914,
      "activations/layer15_attention_weight_max": 33.15945053100586,
      "activations/layer15_attention_weight_min": -29.597688674926758,
      "activations/layer16_attention_weight_max": 34.56815719604492,
      "activations/layer16_attention_weight_min": -28.92707061767578,
      "activations/layer17_attention_weight_max": 54.59851837158203,
      "activations/layer17_attention_weight_min": -43.44697570800781,
      "activations/layer18_attention_weight_max": 48.54740524291992,
      "activations/layer18_attention_weight_min": -36.92423629760742,
      "activations/layer19_attention_weight_max": 26.033300399780273,
      "activations/layer19_attention_weight_min": -21.211545944213867,
      "activations/layer1_attention_weight_max": 16.472885131835938,
      "activations/layer1_attention_weight_min": -14.65682315826416,
      "activations/layer20_attention_weight_max": 24.766599655151367,
      "activations/layer20_attention_weight_min": -20.389848709106445,
      "activations/layer21_attention_weight_max": 35.54558181762695,
      "activations/layer21_attention_weight_min": -22.29262351989746,
      "activations/layer22_attention_weight_max": 32.65089416503906,
      "activations/layer22_attention_weight_min": -25.046695709228516,
      "activations/layer23_attention_weight_max": 39.843475341796875,
      "activations/layer23_attention_weight_min": -25.41914176940918,
      "activations/layer2_attention_weight_max": 32.89881134033203,
      "activations/layer2_attention_weight_min": -30.2574462890625,
      "activations/layer3_attention_weight_max": 91.82584381103516,
      "activations/layer3_attention_weight_min": -93.70527648925781,
      "activations/layer4_attention_weight_max": 96.96187591552734,
      "activations/layer4_attention_weight_min": -95.48092651367188,
      "activations/layer5_attention_weight_max": 72.10711669921875,
      "activations/layer5_attention_weight_min": -78.65666961669922,
      "activations/layer6_attention_weight_max": 57.00270462036133,
      "activations/layer6_attention_weight_min": -55.156776428222656,
      "activations/layer7_attention_weight_max": 72.22862243652344,
      "activations/layer7_attention_weight_min": -69.53084564208984,
      "activations/layer8_attention_weight_max": 49.68278884887695,
      "activations/layer8_attention_weight_min": -49.99225997924805,
      "activations/layer9_attention_weight_max": 46.03139877319336,
      "activations/layer9_attention_weight_min": -46.813720703125,
      "epoch": 11.3,
      "learning_rate": 7.792348484848484e-05,
      "loss": 2.7885,
      "step": 194450
    },
    {
      "activations/layer0_attention_weight_max": 15.192666053771973,
      "activations/layer0_attention_weight_min": -13.698217391967773,
      "activations/layer10_attention_weight_max": 35.8121223449707,
      "activations/layer10_attention_weight_min": -35.04949951171875,
      "activations/layer11_attention_weight_max": 30.85855484008789,
      "activations/layer11_attention_weight_min": -32.89114761352539,
      "activations/layer12_attention_weight_max": 18.21637535095215,
      "activations/layer12_attention_weight_min": -23.28555679321289,
      "activations/layer13_attention_weight_max": 38.608924865722656,
      "activations/layer13_attention_weight_min": -30.887203216552734,
      "activations/layer14_attention_weight_max": 50.94373321533203,
      "activations/layer14_attention_weight_min": -32.427520751953125,
      "activations/layer15_attention_weight_max": 36.416831970214844,
      "activations/layer15_attention_weight_min": -29.837120056152344,
      "activations/layer16_attention_weight_max": 31.29375457763672,
      "activations/layer16_attention_weight_min": -29.583450317382812,
      "activations/layer17_attention_weight_max": 51.371822357177734,
      "activations/layer17_attention_weight_min": -42.05461502075195,
      "activations/layer18_attention_weight_max": 44.88138198852539,
      "activations/layer18_attention_weight_min": -35.307769775390625,
      "activations/layer19_attention_weight_max": 23.285303115844727,
      "activations/layer19_attention_weight_min": -23.130754470825195,
      "activations/layer1_attention_weight_max": 16.08067512512207,
      "activations/layer1_attention_weight_min": -14.77663516998291,
      "activations/layer20_attention_weight_max": 21.687564849853516,
      "activations/layer20_attention_weight_min": -23.180315017700195,
      "activations/layer21_attention_weight_max": 34.63823318481445,
      "activations/layer21_attention_weight_min": -23.22149085998535,
      "activations/layer22_attention_weight_max": 32.31034469604492,
      "activations/layer22_attention_weight_min": -27.64719009399414,
      "activations/layer23_attention_weight_max": 40.03382873535156,
      "activations/layer23_attention_weight_min": -25.21046257019043,
      "activations/layer2_attention_weight_max": 30.753463745117188,
      "activations/layer2_attention_weight_min": -29.967069625854492,
      "activations/layer3_attention_weight_max": 91.62415313720703,
      "activations/layer3_attention_weight_min": -90.63024139404297,
      "activations/layer4_attention_weight_max": 100.6706314086914,
      "activations/layer4_attention_weight_min": -97.12914276123047,
      "activations/layer5_attention_weight_max": 75.85427856445312,
      "activations/layer5_attention_weight_min": -77.1620101928711,
      "activations/layer6_attention_weight_max": 55.639652252197266,
      "activations/layer6_attention_weight_min": -54.28630447387695,
      "activations/layer7_attention_weight_max": 77.37167358398438,
      "activations/layer7_attention_weight_min": -75.12885284423828,
      "activations/layer8_attention_weight_max": 46.21897506713867,
      "activations/layer8_attention_weight_min": -47.52988815307617,
      "activations/layer9_attention_weight_max": 48.78507614135742,
      "activations/layer9_attention_weight_min": -47.94208526611328,
      "epoch": 11.3,
      "learning_rate": 7.790454545454544e-05,
      "loss": 2.7719,
      "step": 194500
    },
    {
      "activations/layer0_attention_weight_max": 15.349687576293945,
      "activations/layer0_attention_weight_min": -14.172798156738281,
      "activations/layer10_attention_weight_max": 38.5261344909668,
      "activations/layer10_attention_weight_min": -36.90211486816406,
      "activations/layer11_attention_weight_max": 33.716861724853516,
      "activations/layer11_attention_weight_min": -35.14735412597656,
      "activations/layer12_attention_weight_max": 20.869543075561523,
      "activations/layer12_attention_weight_min": -24.781145095825195,
      "activations/layer13_attention_weight_max": 43.45478820800781,
      "activations/layer13_attention_weight_min": -34.26306915283203,
      "activations/layer14_attention_weight_max": 42.09516525268555,
      "activations/layer14_attention_weight_min": -31.917194366455078,
      "activations/layer15_attention_weight_max": 39.85988998413086,
      "activations/layer15_attention_weight_min": -31.627958297729492,
      "activations/layer16_attention_weight_max": 36.170448303222656,
      "activations/layer16_attention_weight_min": -30.60089111328125,
      "activations/layer17_attention_weight_max": 57.51060104370117,
      "activations/layer17_attention_weight_min": -43.70683670043945,
      "activations/layer18_attention_weight_max": 53.64706802368164,
      "activations/layer18_attention_weight_min": -37.31711196899414,
      "activations/layer19_attention_weight_max": 26.935474395751953,
      "activations/layer19_attention_weight_min": -21.620328903198242,
      "activations/layer1_attention_weight_max": 16.537565231323242,
      "activations/layer1_attention_weight_min": -15.13153076171875,
      "activations/layer20_attention_weight_max": 25.610740661621094,
      "activations/layer20_attention_weight_min": -21.226741790771484,
      "activations/layer21_attention_weight_max": 49.95050048828125,
      "activations/layer21_attention_weight_min": -24.876407623291016,
      "activations/layer22_attention_weight_max": 33.87643814086914,
      "activations/layer22_attention_weight_min": -26.73012351989746,
      "activations/layer23_attention_weight_max": 44.34800338745117,
      "activations/layer23_attention_weight_min": -24.51485824584961,
      "activations/layer2_attention_weight_max": 30.995380401611328,
      "activations/layer2_attention_weight_min": -29.600444793701172,
      "activations/layer3_attention_weight_max": 89.37948608398438,
      "activations/layer3_attention_weight_min": -93.07675170898438,
      "activations/layer4_attention_weight_max": 100.1003646850586,
      "activations/layer4_attention_weight_min": -94.830322265625,
      "activations/layer5_attention_weight_max": 77.48860931396484,
      "activations/layer5_attention_weight_min": -79.74332427978516,
      "activations/layer6_attention_weight_max": 56.2182731628418,
      "activations/layer6_attention_weight_min": -50.73097610473633,
      "activations/layer7_attention_weight_max": 76.98639678955078,
      "activations/layer7_attention_weight_min": -74.11739349365234,
      "activations/layer8_attention_weight_max": 47.41533279418945,
      "activations/layer8_attention_weight_min": -46.85205078125,
      "activations/layer9_attention_weight_max": 47.74668502807617,
      "activations/layer9_attention_weight_min": -50.2886848449707,
      "epoch": 11.3,
      "learning_rate": 7.788560606060606e-05,
      "loss": 2.7824,
      "step": 194550
    },
    {
      "activations/layer0_attention_weight_max": 15.655929565429688,
      "activations/layer0_attention_weight_min": -14.17634105682373,
      "activations/layer10_attention_weight_max": 39.55293655395508,
      "activations/layer10_attention_weight_min": -35.92327880859375,
      "activations/layer11_attention_weight_max": 33.49646759033203,
      "activations/layer11_attention_weight_min": -33.8798942565918,
      "activations/layer12_attention_weight_max": 18.550878524780273,
      "activations/layer12_attention_weight_min": -24.125791549682617,
      "activations/layer13_attention_weight_max": 45.58036422729492,
      "activations/layer13_attention_weight_min": -30.505693435668945,
      "activations/layer14_attention_weight_max": 39.857093811035156,
      "activations/layer14_attention_weight_min": -30.905502319335938,
      "activations/layer15_attention_weight_max": 35.074825286865234,
      "activations/layer15_attention_weight_min": -29.052188873291016,
      "activations/layer16_attention_weight_max": 31.75220489501953,
      "activations/layer16_attention_weight_min": -32.02162170410156,
      "activations/layer17_attention_weight_max": 53.7272834777832,
      "activations/layer17_attention_weight_min": -44.63430404663086,
      "activations/layer18_attention_weight_max": 46.14360427856445,
      "activations/layer18_attention_weight_min": -38.79803466796875,
      "activations/layer19_attention_weight_max": 21.949581146240234,
      "activations/layer19_attention_weight_min": -22.83007049560547,
      "activations/layer1_attention_weight_max": 16.703760147094727,
      "activations/layer1_attention_weight_min": -15.76539421081543,
      "activations/layer20_attention_weight_max": 25.3403377532959,
      "activations/layer20_attention_weight_min": -20.4897518157959,
      "activations/layer21_attention_weight_max": 41.58854293823242,
      "activations/layer21_attention_weight_min": -23.498741149902344,
      "activations/layer22_attention_weight_max": 30.745527267456055,
      "activations/layer22_attention_weight_min": -26.522565841674805,
      "activations/layer23_attention_weight_max": 38.679378509521484,
      "activations/layer23_attention_weight_min": -25.30805778503418,
      "activations/layer2_attention_weight_max": 32.049232482910156,
      "activations/layer2_attention_weight_min": -30.04265594482422,
      "activations/layer3_attention_weight_max": 94.8457260131836,
      "activations/layer3_attention_weight_min": -93.97657775878906,
      "activations/layer4_attention_weight_max": 98.66207885742188,
      "activations/layer4_attention_weight_min": -94.49544525146484,
      "activations/layer5_attention_weight_max": 72.99127960205078,
      "activations/layer5_attention_weight_min": -76.06468963623047,
      "activations/layer6_attention_weight_max": 57.86222839355469,
      "activations/layer6_attention_weight_min": -52.72062301635742,
      "activations/layer7_attention_weight_max": 75.50836181640625,
      "activations/layer7_attention_weight_min": -74.87481689453125,
      "activations/layer8_attention_weight_max": 48.5406379699707,
      "activations/layer8_attention_weight_min": -50.101619720458984,
      "activations/layer9_attention_weight_max": 51.01640319824219,
      "activations/layer9_attention_weight_min": -51.425777435302734,
      "epoch": 11.31,
      "learning_rate": 7.786666666666666e-05,
      "loss": 2.7706,
      "step": 194600
    },
    {
      "activations/layer0_attention_weight_max": 14.943229675292969,
      "activations/layer0_attention_weight_min": -14.344611167907715,
      "activations/layer10_attention_weight_max": 38.82825469970703,
      "activations/layer10_attention_weight_min": -36.616233825683594,
      "activations/layer11_attention_weight_max": 33.343414306640625,
      "activations/layer11_attention_weight_min": -32.130409240722656,
      "activations/layer12_attention_weight_max": 25.650922775268555,
      "activations/layer12_attention_weight_min": -24.103055953979492,
      "activations/layer13_attention_weight_max": 45.07224655151367,
      "activations/layer13_attention_weight_min": -31.91948890686035,
      "activations/layer14_attention_weight_max": 39.000240325927734,
      "activations/layer14_attention_weight_min": -31.41961669921875,
      "activations/layer15_attention_weight_max": 38.12946319580078,
      "activations/layer15_attention_weight_min": -31.260347366333008,
      "activations/layer16_attention_weight_max": 34.46676254272461,
      "activations/layer16_attention_weight_min": -29.703533172607422,
      "activations/layer17_attention_weight_max": 55.82502365112305,
      "activations/layer17_attention_weight_min": -43.26970672607422,
      "activations/layer18_attention_weight_max": 48.79355239868164,
      "activations/layer18_attention_weight_min": -38.526092529296875,
      "activations/layer19_attention_weight_max": 25.04557991027832,
      "activations/layer19_attention_weight_min": -21.977375030517578,
      "activations/layer1_attention_weight_max": 18.326557159423828,
      "activations/layer1_attention_weight_min": -16.135589599609375,
      "activations/layer20_attention_weight_max": 21.564489364624023,
      "activations/layer20_attention_weight_min": -21.96884536743164,
      "activations/layer21_attention_weight_max": 43.3725700378418,
      "activations/layer21_attention_weight_min": -23.113525390625,
      "activations/layer22_attention_weight_max": 31.72720718383789,
      "activations/layer22_attention_weight_min": -24.905195236206055,
      "activations/layer23_attention_weight_max": 38.58659744262695,
      "activations/layer23_attention_weight_min": -22.71181869506836,
      "activations/layer2_attention_weight_max": 32.71105194091797,
      "activations/layer2_attention_weight_min": -30.345252990722656,
      "activations/layer3_attention_weight_max": 95.26464080810547,
      "activations/layer3_attention_weight_min": -95.07925415039062,
      "activations/layer4_attention_weight_max": 98.73693084716797,
      "activations/layer4_attention_weight_min": -94.83267211914062,
      "activations/layer5_attention_weight_max": 69.61947631835938,
      "activations/layer5_attention_weight_min": -79.39098358154297,
      "activations/layer6_attention_weight_max": 52.08324432373047,
      "activations/layer6_attention_weight_min": -50.510948181152344,
      "activations/layer7_attention_weight_max": 74.36019134521484,
      "activations/layer7_attention_weight_min": -68.97309112548828,
      "activations/layer8_attention_weight_max": 44.33074951171875,
      "activations/layer8_attention_weight_min": -45.2269172668457,
      "activations/layer9_attention_weight_max": 52.8870964050293,
      "activations/layer9_attention_weight_min": -46.29445266723633,
      "epoch": 11.31,
      "learning_rate": 7.784772727272727e-05,
      "loss": 2.7653,
      "step": 194650
    },
    {
      "activations/layer0_attention_weight_max": 14.450926780700684,
      "activations/layer0_attention_weight_min": -13.950664520263672,
      "activations/layer10_attention_weight_max": 34.327789306640625,
      "activations/layer10_attention_weight_min": -35.9604606628418,
      "activations/layer11_attention_weight_max": 31.320232391357422,
      "activations/layer11_attention_weight_min": -33.85696029663086,
      "activations/layer12_attention_weight_max": 17.264179229736328,
      "activations/layer12_attention_weight_min": -25.499189376831055,
      "activations/layer13_attention_weight_max": 33.922340393066406,
      "activations/layer13_attention_weight_min": -28.30703353881836,
      "activations/layer14_attention_weight_max": 36.01493453979492,
      "activations/layer14_attention_weight_min": -30.696245193481445,
      "activations/layer15_attention_weight_max": 31.220932006835938,
      "activations/layer15_attention_weight_min": -30.26811981201172,
      "activations/layer16_attention_weight_max": 31.245166778564453,
      "activations/layer16_attention_weight_min": -28.80189323425293,
      "activations/layer17_attention_weight_max": 53.11458206176758,
      "activations/layer17_attention_weight_min": -42.73070526123047,
      "activations/layer18_attention_weight_max": 44.031410217285156,
      "activations/layer18_attention_weight_min": -37.661529541015625,
      "activations/layer19_attention_weight_max": 21.462215423583984,
      "activations/layer19_attention_weight_min": -22.0919246673584,
      "activations/layer1_attention_weight_max": 18.314067840576172,
      "activations/layer1_attention_weight_min": -14.902865409851074,
      "activations/layer20_attention_weight_max": 25.318010330200195,
      "activations/layer20_attention_weight_min": -21.709253311157227,
      "activations/layer21_attention_weight_max": 42.21873474121094,
      "activations/layer21_attention_weight_min": -22.93015480041504,
      "activations/layer22_attention_weight_max": 31.38084602355957,
      "activations/layer22_attention_weight_min": -26.30221939086914,
      "activations/layer23_attention_weight_max": 38.407012939453125,
      "activations/layer23_attention_weight_min": -26.622066497802734,
      "activations/layer2_attention_weight_max": 33.89669418334961,
      "activations/layer2_attention_weight_min": -31.491397857666016,
      "activations/layer3_attention_weight_max": 93.17667388916016,
      "activations/layer3_attention_weight_min": -98.36225891113281,
      "activations/layer4_attention_weight_max": 103.9300308227539,
      "activations/layer4_attention_weight_min": -103.74285888671875,
      "activations/layer5_attention_weight_max": 79.553466796875,
      "activations/layer5_attention_weight_min": -83.42221069335938,
      "activations/layer6_attention_weight_max": 53.73259353637695,
      "activations/layer6_attention_weight_min": -55.02348327636719,
      "activations/layer7_attention_weight_max": 80.05734252929688,
      "activations/layer7_attention_weight_min": -72.73989868164062,
      "activations/layer8_attention_weight_max": 46.722904205322266,
      "activations/layer8_attention_weight_min": -47.39375686645508,
      "activations/layer9_attention_weight_max": 45.6909294128418,
      "activations/layer9_attention_weight_min": -48.498313903808594,
      "epoch": 11.31,
      "learning_rate": 7.782878787878787e-05,
      "loss": 2.7745,
      "step": 194700
    },
    {
      "activations/layer0_attention_weight_max": 14.797740936279297,
      "activations/layer0_attention_weight_min": -14.165680885314941,
      "activations/layer10_attention_weight_max": 38.62445068359375,
      "activations/layer10_attention_weight_min": -38.911495208740234,
      "activations/layer11_attention_weight_max": 33.651100158691406,
      "activations/layer11_attention_weight_min": -36.04330825805664,
      "activations/layer12_attention_weight_max": 20.633817672729492,
      "activations/layer12_attention_weight_min": -25.096616744995117,
      "activations/layer13_attention_weight_max": 48.7330207824707,
      "activations/layer13_attention_weight_min": -29.638723373413086,
      "activations/layer14_attention_weight_max": 40.67209243774414,
      "activations/layer14_attention_weight_min": -31.105365753173828,
      "activations/layer15_attention_weight_max": 37.535518646240234,
      "activations/layer15_attention_weight_min": -30.007299423217773,
      "activations/layer16_attention_weight_max": 34.54007339477539,
      "activations/layer16_attention_weight_min": -29.846769332885742,
      "activations/layer17_attention_weight_max": 54.23857879638672,
      "activations/layer17_attention_weight_min": -45.75273513793945,
      "activations/layer18_attention_weight_max": 50.86894989013672,
      "activations/layer18_attention_weight_min": -40.6967658996582,
      "activations/layer19_attention_weight_max": 25.5860538482666,
      "activations/layer19_attention_weight_min": -22.386083602905273,
      "activations/layer1_attention_weight_max": 17.413522720336914,
      "activations/layer1_attention_weight_min": -15.794798851013184,
      "activations/layer20_attention_weight_max": 27.79009246826172,
      "activations/layer20_attention_weight_min": -22.85755729675293,
      "activations/layer21_attention_weight_max": 45.76304244995117,
      "activations/layer21_attention_weight_min": -24.1572208404541,
      "activations/layer22_attention_weight_max": 32.910911560058594,
      "activations/layer22_attention_weight_min": -25.604944229125977,
      "activations/layer23_attention_weight_max": 42.38890838623047,
      "activations/layer23_attention_weight_min": -21.730588912963867,
      "activations/layer2_attention_weight_max": 33.43553161621094,
      "activations/layer2_attention_weight_min": -31.397075653076172,
      "activations/layer3_attention_weight_max": 98.4303970336914,
      "activations/layer3_attention_weight_min": -96.77871704101562,
      "activations/layer4_attention_weight_max": 100.32566833496094,
      "activations/layer4_attention_weight_min": -101.42020416259766,
      "activations/layer5_attention_weight_max": 73.0543441772461,
      "activations/layer5_attention_weight_min": -74.87080383300781,
      "activations/layer6_attention_weight_max": 52.34070587158203,
      "activations/layer6_attention_weight_min": -53.7503776550293,
      "activations/layer7_attention_weight_max": 71.57452392578125,
      "activations/layer7_attention_weight_min": -73.35255432128906,
      "activations/layer8_attention_weight_max": 45.964019775390625,
      "activations/layer8_attention_weight_min": -49.80747985839844,
      "activations/layer9_attention_weight_max": 50.187225341796875,
      "activations/layer9_attention_weight_min": -51.644500732421875,
      "epoch": 11.32,
      "learning_rate": 7.780984848484848e-05,
      "loss": 2.7923,
      "step": 194750
    },
    {
      "activations/layer0_attention_weight_max": 15.893242835998535,
      "activations/layer0_attention_weight_min": -13.769824028015137,
      "activations/layer10_attention_weight_max": 36.17177200317383,
      "activations/layer10_attention_weight_min": -35.51337432861328,
      "activations/layer11_attention_weight_max": 32.265872955322266,
      "activations/layer11_attention_weight_min": -33.99829864501953,
      "activations/layer12_attention_weight_max": 16.40276527404785,
      "activations/layer12_attention_weight_min": -25.888477325439453,
      "activations/layer13_attention_weight_max": 38.62884521484375,
      "activations/layer13_attention_weight_min": -32.742156982421875,
      "activations/layer14_attention_weight_max": 34.011444091796875,
      "activations/layer14_attention_weight_min": -29.691041946411133,
      "activations/layer15_attention_weight_max": 29.21463394165039,
      "activations/layer15_attention_weight_min": -29.259492874145508,
      "activations/layer16_attention_weight_max": 29.30040740966797,
      "activations/layer16_attention_weight_min": -29.567655563354492,
      "activations/layer17_attention_weight_max": 46.71649169921875,
      "activations/layer17_attention_weight_min": -45.46022033691406,
      "activations/layer18_attention_weight_max": 42.906341552734375,
      "activations/layer18_attention_weight_min": -40.94953155517578,
      "activations/layer19_attention_weight_max": 21.900571823120117,
      "activations/layer19_attention_weight_min": -20.62084197998047,
      "activations/layer1_attention_weight_max": 17.029640197753906,
      "activations/layer1_attention_weight_min": -15.465340614318848,
      "activations/layer20_attention_weight_max": 21.565929412841797,
      "activations/layer20_attention_weight_min": -21.671236038208008,
      "activations/layer21_attention_weight_max": 35.49795150756836,
      "activations/layer21_attention_weight_min": -25.720046997070312,
      "activations/layer22_attention_weight_max": 29.687435150146484,
      "activations/layer22_attention_weight_min": -26.056610107421875,
      "activations/layer23_attention_weight_max": 32.35588836669922,
      "activations/layer23_attention_weight_min": -21.39578628540039,
      "activations/layer2_attention_weight_max": 32.29337692260742,
      "activations/layer2_attention_weight_min": -30.514739990234375,
      "activations/layer3_attention_weight_max": 96.30087280273438,
      "activations/layer3_attention_weight_min": -92.6986312866211,
      "activations/layer4_attention_weight_max": 104.4251480102539,
      "activations/layer4_attention_weight_min": -101.8524169921875,
      "activations/layer5_attention_weight_max": 74.0571060180664,
      "activations/layer5_attention_weight_min": -78.04434204101562,
      "activations/layer6_attention_weight_max": 54.53458023071289,
      "activations/layer6_attention_weight_min": -56.1636848449707,
      "activations/layer7_attention_weight_max": 79.83939361572266,
      "activations/layer7_attention_weight_min": -74.21094512939453,
      "activations/layer8_attention_weight_max": 46.03099822998047,
      "activations/layer8_attention_weight_min": -49.01205825805664,
      "activations/layer9_attention_weight_max": 47.51448059082031,
      "activations/layer9_attention_weight_min": -50.9835319519043,
      "epoch": 11.32,
      "learning_rate": 7.779090909090909e-05,
      "loss": 2.7722,
      "step": 194800
    },
    {
      "activations/layer0_attention_weight_max": 15.463027000427246,
      "activations/layer0_attention_weight_min": -13.57987117767334,
      "activations/layer10_attention_weight_max": 33.67509078979492,
      "activations/layer10_attention_weight_min": -35.122901916503906,
      "activations/layer11_attention_weight_max": 29.505531311035156,
      "activations/layer11_attention_weight_min": -33.2863883972168,
      "activations/layer12_attention_weight_max": 19.36162567138672,
      "activations/layer12_attention_weight_min": -24.60367774963379,
      "activations/layer13_attention_weight_max": 39.04000473022461,
      "activations/layer13_attention_weight_min": -28.917451858520508,
      "activations/layer14_attention_weight_max": 35.387168884277344,
      "activations/layer14_attention_weight_min": -29.617359161376953,
      "activations/layer15_attention_weight_max": 31.402250289916992,
      "activations/layer15_attention_weight_min": -29.063814163208008,
      "activations/layer16_attention_weight_max": 30.04523277282715,
      "activations/layer16_attention_weight_min": -28.022863388061523,
      "activations/layer17_attention_weight_max": 54.16792678833008,
      "activations/layer17_attention_weight_min": -45.630836486816406,
      "activations/layer18_attention_weight_max": 48.77784729003906,
      "activations/layer18_attention_weight_min": -39.46702194213867,
      "activations/layer19_attention_weight_max": 22.019880294799805,
      "activations/layer19_attention_weight_min": -21.683658599853516,
      "activations/layer1_attention_weight_max": 17.85854148864746,
      "activations/layer1_attention_weight_min": -16.642305374145508,
      "activations/layer20_attention_weight_max": 21.26024627685547,
      "activations/layer20_attention_weight_min": -20.965303421020508,
      "activations/layer21_attention_weight_max": 35.70520782470703,
      "activations/layer21_attention_weight_min": -24.806594848632812,
      "activations/layer22_attention_weight_max": 31.835962295532227,
      "activations/layer22_attention_weight_min": -27.17565155029297,
      "activations/layer23_attention_weight_max": 34.77754592895508,
      "activations/layer23_attention_weight_min": -24.194564819335938,
      "activations/layer2_attention_weight_max": 31.62469482421875,
      "activations/layer2_attention_weight_min": -30.266965866088867,
      "activations/layer3_attention_weight_max": 92.67303466796875,
      "activations/layer3_attention_weight_min": -91.35022735595703,
      "activations/layer4_attention_weight_max": 94.26907348632812,
      "activations/layer4_attention_weight_min": -92.45735168457031,
      "activations/layer5_attention_weight_max": 73.91329193115234,
      "activations/layer5_attention_weight_min": -78.52932739257812,
      "activations/layer6_attention_weight_max": 51.03750228881836,
      "activations/layer6_attention_weight_min": -53.501033782958984,
      "activations/layer7_attention_weight_max": 70.6331558227539,
      "activations/layer7_attention_weight_min": -73.25165557861328,
      "activations/layer8_attention_weight_max": 43.52861785888672,
      "activations/layer8_attention_weight_min": -50.82362747192383,
      "activations/layer9_attention_weight_max": 46.63179016113281,
      "activations/layer9_attention_weight_min": -47.68360137939453,
      "epoch": 11.32,
      "learning_rate": 7.777196969696968e-05,
      "loss": 2.7833,
      "step": 194850
    },
    {
      "activations/layer0_attention_weight_max": 15.615232467651367,
      "activations/layer0_attention_weight_min": -13.758332252502441,
      "activations/layer10_attention_weight_max": 38.711334228515625,
      "activations/layer10_attention_weight_min": -37.307044982910156,
      "activations/layer11_attention_weight_max": 34.145633697509766,
      "activations/layer11_attention_weight_min": -34.186492919921875,
      "activations/layer12_attention_weight_max": 24.722631454467773,
      "activations/layer12_attention_weight_min": -22.360610961914062,
      "activations/layer13_attention_weight_max": 41.7937126159668,
      "activations/layer13_attention_weight_min": -31.85724639892578,
      "activations/layer14_attention_weight_max": 37.43850326538086,
      "activations/layer14_attention_weight_min": -31.964754104614258,
      "activations/layer15_attention_weight_max": 35.643577575683594,
      "activations/layer15_attention_weight_min": -30.046981811523438,
      "activations/layer16_attention_weight_max": 34.095794677734375,
      "activations/layer16_attention_weight_min": -28.644174575805664,
      "activations/layer17_attention_weight_max": 50.830528259277344,
      "activations/layer17_attention_weight_min": -44.77738952636719,
      "activations/layer18_attention_weight_max": 47.0771369934082,
      "activations/layer18_attention_weight_min": -37.652915954589844,
      "activations/layer19_attention_weight_max": 22.90523910522461,
      "activations/layer19_attention_weight_min": -22.044418334960938,
      "activations/layer1_attention_weight_max": 17.208425521850586,
      "activations/layer1_attention_weight_min": -19.35029411315918,
      "activations/layer20_attention_weight_max": 21.563325881958008,
      "activations/layer20_attention_weight_min": -20.569974899291992,
      "activations/layer21_attention_weight_max": 34.55793380737305,
      "activations/layer21_attention_weight_min": -21.80231285095215,
      "activations/layer22_attention_weight_max": 30.128828048706055,
      "activations/layer22_attention_weight_min": -25.317535400390625,
      "activations/layer23_attention_weight_max": 38.745872497558594,
      "activations/layer23_attention_weight_min": -23.349658966064453,
      "activations/layer2_attention_weight_max": 31.240575790405273,
      "activations/layer2_attention_weight_min": -29.462133407592773,
      "activations/layer3_attention_weight_max": 91.9375991821289,
      "activations/layer3_attention_weight_min": -90.9381103515625,
      "activations/layer4_attention_weight_max": 98.66988372802734,
      "activations/layer4_attention_weight_min": -94.1570816040039,
      "activations/layer5_attention_weight_max": 75.54863739013672,
      "activations/layer5_attention_weight_min": -75.74974060058594,
      "activations/layer6_attention_weight_max": 53.50028991699219,
      "activations/layer6_attention_weight_min": -51.42586898803711,
      "activations/layer7_attention_weight_max": 75.17900848388672,
      "activations/layer7_attention_weight_min": -72.02776336669922,
      "activations/layer8_attention_weight_max": 47.2442512512207,
      "activations/layer8_attention_weight_min": -47.115478515625,
      "activations/layer9_attention_weight_max": 48.19422912597656,
      "activations/layer9_attention_weight_min": -47.36692810058594,
      "epoch": 11.32,
      "learning_rate": 7.77530303030303e-05,
      "loss": 2.7678,
      "step": 194900
    },
    {
      "activations/layer0_attention_weight_max": 15.19329833984375,
      "activations/layer0_attention_weight_min": -13.662834167480469,
      "activations/layer10_attention_weight_max": 35.081748962402344,
      "activations/layer10_attention_weight_min": -34.45066452026367,
      "activations/layer11_attention_weight_max": 32.12998580932617,
      "activations/layer11_attention_weight_min": -31.12823486328125,
      "activations/layer12_attention_weight_max": 17.42576026916504,
      "activations/layer12_attention_weight_min": -25.63552474975586,
      "activations/layer13_attention_weight_max": 33.456298828125,
      "activations/layer13_attention_weight_min": -28.151952743530273,
      "activations/layer14_attention_weight_max": 33.10063171386719,
      "activations/layer14_attention_weight_min": -28.206640243530273,
      "activations/layer15_attention_weight_max": 31.928504943847656,
      "activations/layer15_attention_weight_min": -28.035911560058594,
      "activations/layer16_attention_weight_max": 30.726716995239258,
      "activations/layer16_attention_weight_min": -28.536760330200195,
      "activations/layer17_attention_weight_max": 50.61775588989258,
      "activations/layer17_attention_weight_min": -44.344261169433594,
      "activations/layer18_attention_weight_max": 43.789730072021484,
      "activations/layer18_attention_weight_min": -38.66794967651367,
      "activations/layer19_attention_weight_max": 21.291820526123047,
      "activations/layer19_attention_weight_min": -21.519275665283203,
      "activations/layer1_attention_weight_max": 16.597759246826172,
      "activations/layer1_attention_weight_min": -14.789133071899414,
      "activations/layer20_attention_weight_max": 21.172666549682617,
      "activations/layer20_attention_weight_min": -23.06195068359375,
      "activations/layer21_attention_weight_max": 31.60373878479004,
      "activations/layer21_attention_weight_min": -22.939382553100586,
      "activations/layer22_attention_weight_max": 30.390262603759766,
      "activations/layer22_attention_weight_min": -24.872007369995117,
      "activations/layer23_attention_weight_max": 36.645660400390625,
      "activations/layer23_attention_weight_min": -22.97701072692871,
      "activations/layer2_attention_weight_max": 33.40557098388672,
      "activations/layer2_attention_weight_min": -31.208738327026367,
      "activations/layer3_attention_weight_max": 91.90213775634766,
      "activations/layer3_attention_weight_min": -90.86408233642578,
      "activations/layer4_attention_weight_max": 94.47343444824219,
      "activations/layer4_attention_weight_min": -91.64961242675781,
      "activations/layer5_attention_weight_max": 75.04289245605469,
      "activations/layer5_attention_weight_min": -77.70721435546875,
      "activations/layer6_attention_weight_max": 53.12499237060547,
      "activations/layer6_attention_weight_min": -50.64604949951172,
      "activations/layer7_attention_weight_max": 78.02008056640625,
      "activations/layer7_attention_weight_min": -67.49758911132812,
      "activations/layer8_attention_weight_max": 44.773338317871094,
      "activations/layer8_attention_weight_min": -46.108367919921875,
      "activations/layer9_attention_weight_max": 53.040828704833984,
      "activations/layer9_attention_weight_min": -46.69668960571289,
      "epoch": 11.33,
      "learning_rate": 7.773409090909091e-05,
      "loss": 2.7849,
      "step": 194950
    },
    {
      "activations/layer0_attention_weight_max": 15.41545581817627,
      "activations/layer0_attention_weight_min": -13.603087425231934,
      "activations/layer10_attention_weight_max": 36.142669677734375,
      "activations/layer10_attention_weight_min": -36.28353500366211,
      "activations/layer11_attention_weight_max": 32.899620056152344,
      "activations/layer11_attention_weight_min": -33.15531921386719,
      "activations/layer12_attention_weight_max": 19.458295822143555,
      "activations/layer12_attention_weight_min": -25.19662094116211,
      "activations/layer13_attention_weight_max": 42.81150436401367,
      "activations/layer13_attention_weight_min": -32.11680221557617,
      "activations/layer14_attention_weight_max": 36.801780700683594,
      "activations/layer14_attention_weight_min": -30.915536880493164,
      "activations/layer15_attention_weight_max": 32.989585876464844,
      "activations/layer15_attention_weight_min": -30.00434684753418,
      "activations/layer16_attention_weight_max": 33.21693420410156,
      "activations/layer16_attention_weight_min": -31.76609230041504,
      "activations/layer17_attention_weight_max": 52.22218704223633,
      "activations/layer17_attention_weight_min": -45.8627815246582,
      "activations/layer18_attention_weight_max": 47.74462890625,
      "activations/layer18_attention_weight_min": -38.986854553222656,
      "activations/layer19_attention_weight_max": 22.609638214111328,
      "activations/layer19_attention_weight_min": -21.870901107788086,
      "activations/layer1_attention_weight_max": 16.24233055114746,
      "activations/layer1_attention_weight_min": -14.58647346496582,
      "activations/layer20_attention_weight_max": 24.608320236206055,
      "activations/layer20_attention_weight_min": -23.24083709716797,
      "activations/layer21_attention_weight_max": 39.61270523071289,
      "activations/layer21_attention_weight_min": -22.093082427978516,
      "activations/layer22_attention_weight_max": 32.429527282714844,
      "activations/layer22_attention_weight_min": -24.80722427368164,
      "activations/layer23_attention_weight_max": 40.27470016479492,
      "activations/layer23_attention_weight_min": -22.18462371826172,
      "activations/layer2_attention_weight_max": 33.600120544433594,
      "activations/layer2_attention_weight_min": -30.547147750854492,
      "activations/layer3_attention_weight_max": 96.83319854736328,
      "activations/layer3_attention_weight_min": -97.00753784179688,
      "activations/layer4_attention_weight_max": 103.02497863769531,
      "activations/layer4_attention_weight_min": -104.35438537597656,
      "activations/layer5_attention_weight_max": 81.01793670654297,
      "activations/layer5_attention_weight_min": -80.69647216796875,
      "activations/layer6_attention_weight_max": 54.405941009521484,
      "activations/layer6_attention_weight_min": -52.814430236816406,
      "activations/layer7_attention_weight_max": 74.06348419189453,
      "activations/layer7_attention_weight_min": -69.61909484863281,
      "activations/layer8_attention_weight_max": 43.26944351196289,
      "activations/layer8_attention_weight_min": -46.97145462036133,
      "activations/layer9_attention_weight_max": 49.95915985107422,
      "activations/layer9_attention_weight_min": -46.93427658081055,
      "epoch": 11.33,
      "learning_rate": 7.77151515151515e-05,
      "loss": 2.7899,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_loss": 2.734375,
      "eval_runtime": 8.5118,
      "eval_samples_per_second": 504.477,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_openwebtext_loss": 2.734375,
      "eval_openwebtext_ppl": 15.400115355370453,
      "eval_openwebtext_runtime": 8.5118,
      "eval_openwebtext_samples_per_second": 504.477,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_wikitext_loss": 2.966796875,
      "eval_wikitext_ppl": 19.429584428039178,
      "eval_wikitext_runtime": 2.0487,
      "eval_wikitext_samples_per_second": 222.581,
      "step": 195000
    },
    {
      "epoch": 11.33,
      "eval_lambada_loss": 2.55859375,
      "eval_lambada_ppl": 12.917639106991658,
      "eval_lambada_runtime": 9.5888,
      "eval_lambada_samples_per_second": 507.778,
      "step": 195000
    },
    {
      "activations/layer0_attention_weight_max": 14.863215446472168,
      "activations/layer0_attention_weight_min": -13.768636703491211,
      "activations/layer10_attention_weight_max": 37.80199432373047,
      "activations/layer10_attention_weight_min": -36.584983825683594,
      "activations/layer11_attention_weight_max": 33.31393051147461,
      "activations/layer11_attention_weight_min": -33.69456481933594,
      "activations/layer12_attention_weight_max": 18.145551681518555,
      "activations/layer12_attention_weight_min": -24.26638412475586,
      "activations/layer13_attention_weight_max": 36.91958236694336,
      "activations/layer13_attention_weight_min": -29.37091827392578,
      "activations/layer14_attention_weight_max": 35.45012664794922,
      "activations/layer14_attention_weight_min": -29.286174774169922,
      "activations/layer15_attention_weight_max": 31.67385482788086,
      "activations/layer15_attention_weight_min": -28.89319610595703,
      "activations/layer16_attention_weight_max": 28.57059669494629,
      "activations/layer16_attention_weight_min": -27.95673179626465,
      "activations/layer17_attention_weight_max": 52.329097747802734,
      "activations/layer17_attention_weight_min": -43.221744537353516,
      "activations/layer18_attention_weight_max": 50.10253143310547,
      "activations/layer18_attention_weight_min": -35.787322998046875,
      "activations/layer19_attention_weight_max": 22.773984909057617,
      "activations/layer19_attention_weight_min": -22.121912002563477,
      "activations/layer1_attention_weight_max": 17.61194610595703,
      "activations/layer1_attention_weight_min": -14.362678527832031,
      "activations/layer20_attention_weight_max": 27.42516326904297,
      "activations/layer20_attention_weight_min": -18.913928985595703,
      "activations/layer21_attention_weight_max": 38.74879455566406,
      "activations/layer21_attention_weight_min": -23.60304832458496,
      "activations/layer22_attention_weight_max": 31.31082534790039,
      "activations/layer22_attention_weight_min": -25.447187423706055,
      "activations/layer23_attention_weight_max": 35.361000061035156,
      "activations/layer23_attention_weight_min": -23.425540924072266,
      "activations/layer2_attention_weight_max": 32.56035614013672,
      "activations/layer2_attention_weight_min": -33.226932525634766,
      "activations/layer3_attention_weight_max": 90.03070068359375,
      "activations/layer3_attention_weight_min": -92.20919036865234,
      "activations/layer4_attention_weight_max": 96.73030853271484,
      "activations/layer4_attention_weight_min": -93.32036590576172,
      "activations/layer5_attention_weight_max": 71.4543685913086,
      "activations/layer5_attention_weight_min": -77.18904876708984,
      "activations/layer6_attention_weight_max": 53.94581985473633,
      "activations/layer6_attention_weight_min": -52.51348114013672,
      "activations/layer7_attention_weight_max": 78.84507751464844,
      "activations/layer7_attention_weight_min": -74.51679992675781,
      "activations/layer8_attention_weight_max": 45.97004699707031,
      "activations/layer8_attention_weight_min": -47.546085357666016,
      "activations/layer9_attention_weight_max": 49.456783294677734,
      "activations/layer9_attention_weight_min": -47.18382263183594,
      "epoch": 11.33,
      "learning_rate": 7.769621212121211e-05,
      "loss": 2.7693,
      "step": 195050
    },
    {
      "activations/layer0_attention_weight_max": 15.634211540222168,
      "activations/layer0_attention_weight_min": -13.92733383178711,
      "activations/layer10_attention_weight_max": 40.730743408203125,
      "activations/layer10_attention_weight_min": -40.219505310058594,
      "activations/layer11_attention_weight_max": 35.0511360168457,
      "activations/layer11_attention_weight_min": -35.79328918457031,
      "activations/layer12_attention_weight_max": 21.18344497680664,
      "activations/layer12_attention_weight_min": -27.903095245361328,
      "activations/layer13_attention_weight_max": 38.92411804199219,
      "activations/layer13_attention_weight_min": -29.193103790283203,
      "activations/layer14_attention_weight_max": 48.959716796875,
      "activations/layer14_attention_weight_min": -32.12346267700195,
      "activations/layer15_attention_weight_max": 40.1239013671875,
      "activations/layer15_attention_weight_min": -30.82275390625,
      "activations/layer16_attention_weight_max": 32.17767333984375,
      "activations/layer16_attention_weight_min": -29.955739974975586,
      "activations/layer17_attention_weight_max": 52.97713851928711,
      "activations/layer17_attention_weight_min": -45.86787414550781,
      "activations/layer18_attention_weight_max": 47.82404708862305,
      "activations/layer18_attention_weight_min": -40.37788009643555,
      "activations/layer19_attention_weight_max": 23.34187889099121,
      "activations/layer19_attention_weight_min": -24.16594696044922,
      "activations/layer1_attention_weight_max": 16.384923934936523,
      "activations/layer1_attention_weight_min": -17.552135467529297,
      "activations/layer20_attention_weight_max": 24.578889846801758,
      "activations/layer20_attention_weight_min": -22.540708541870117,
      "activations/layer21_attention_weight_max": 40.16717529296875,
      "activations/layer21_attention_weight_min": -25.28171157836914,
      "activations/layer22_attention_weight_max": 31.824983596801758,
      "activations/layer22_attention_weight_min": -25.01236343383789,
      "activations/layer23_attention_weight_max": 43.919029235839844,
      "activations/layer23_attention_weight_min": -23.405864715576172,
      "activations/layer2_attention_weight_max": 35.372947692871094,
      "activations/layer2_attention_weight_min": -31.63887596130371,
      "activations/layer3_attention_weight_max": 102.00476837158203,
      "activations/layer3_attention_weight_min": -97.73261260986328,
      "activations/layer4_attention_weight_max": 106.23485565185547,
      "activations/layer4_attention_weight_min": -97.06519317626953,
      "activations/layer5_attention_weight_max": 77.86054992675781,
      "activations/layer5_attention_weight_min": -77.70467376708984,
      "activations/layer6_attention_weight_max": 54.841758728027344,
      "activations/layer6_attention_weight_min": -55.7940788269043,
      "activations/layer7_attention_weight_max": 83.96987915039062,
      "activations/layer7_attention_weight_min": -75.46393585205078,
      "activations/layer8_attention_weight_max": 51.598388671875,
      "activations/layer8_attention_weight_min": -51.56325149536133,
      "activations/layer9_attention_weight_max": 54.955238342285156,
      "activations/layer9_attention_weight_min": -54.960609436035156,
      "epoch": 11.34,
      "learning_rate": 7.767727272727273e-05,
      "loss": 2.7939,
      "step": 195100
    },
    {
      "activations/layer0_attention_weight_max": 14.948168754577637,
      "activations/layer0_attention_weight_min": -13.447757720947266,
      "activations/layer10_attention_weight_max": 39.67051315307617,
      "activations/layer10_attention_weight_min": -35.26953887939453,
      "activations/layer11_attention_weight_max": 33.5414924621582,
      "activations/layer11_attention_weight_min": -32.00380325317383,
      "activations/layer12_attention_weight_max": 17.85877799987793,
      "activations/layer12_attention_weight_min": -30.391555786132812,
      "activations/layer13_attention_weight_max": 36.205772399902344,
      "activations/layer13_attention_weight_min": -30.14699363708496,
      "activations/layer14_attention_weight_max": 35.388038635253906,
      "activations/layer14_attention_weight_min": -30.534025192260742,
      "activations/layer15_attention_weight_max": 32.496124267578125,
      "activations/layer15_attention_weight_min": -29.574316024780273,
      "activations/layer16_attention_weight_max": 29.72331428527832,
      "activations/layer16_attention_weight_min": -29.675737380981445,
      "activations/layer17_attention_weight_max": 52.4826545715332,
      "activations/layer17_attention_weight_min": -45.70957565307617,
      "activations/layer18_attention_weight_max": 43.446258544921875,
      "activations/layer18_attention_weight_min": -36.570404052734375,
      "activations/layer19_attention_weight_max": 26.014806747436523,
      "activations/layer19_attention_weight_min": -21.448455810546875,
      "activations/layer1_attention_weight_max": 17.011077880859375,
      "activations/layer1_attention_weight_min": -15.794211387634277,
      "activations/layer20_attention_weight_max": 21.409208297729492,
      "activations/layer20_attention_weight_min": -23.866361618041992,
      "activations/layer21_attention_weight_max": 33.90369415283203,
      "activations/layer21_attention_weight_min": -24.530136108398438,
      "activations/layer22_attention_weight_max": 30.6324520111084,
      "activations/layer22_attention_weight_min": -26.77176284790039,
      "activations/layer23_attention_weight_max": 34.421165466308594,
      "activations/layer23_attention_weight_min": -24.017105102539062,
      "activations/layer2_attention_weight_max": 33.61198425292969,
      "activations/layer2_attention_weight_min": -31.350608825683594,
      "activations/layer3_attention_weight_max": 94.45457458496094,
      "activations/layer3_attention_weight_min": -96.5223159790039,
      "activations/layer4_attention_weight_max": 98.28392028808594,
      "activations/layer4_attention_weight_min": -93.47693634033203,
      "activations/layer5_attention_weight_max": 72.81792449951172,
      "activations/layer5_attention_weight_min": -78.45829772949219,
      "activations/layer6_attention_weight_max": 53.62749099731445,
      "activations/layer6_attention_weight_min": -53.38249969482422,
      "activations/layer7_attention_weight_max": 74.06151580810547,
      "activations/layer7_attention_weight_min": -74.83926391601562,
      "activations/layer8_attention_weight_max": 45.75864028930664,
      "activations/layer8_attention_weight_min": -46.79945373535156,
      "activations/layer9_attention_weight_max": 48.0692138671875,
      "activations/layer9_attention_weight_min": -46.89680480957031,
      "epoch": 11.34,
      "learning_rate": 7.765833333333333e-05,
      "loss": 2.7769,
      "step": 195150
    },
    {
      "activations/layer0_attention_weight_max": 15.234212875366211,
      "activations/layer0_attention_weight_min": -13.651296615600586,
      "activations/layer10_attention_weight_max": 38.1423225402832,
      "activations/layer10_attention_weight_min": -36.93883514404297,
      "activations/layer11_attention_weight_max": 34.072837829589844,
      "activations/layer11_attention_weight_min": -33.073387145996094,
      "activations/layer12_attention_weight_max": 27.717525482177734,
      "activations/layer12_attention_weight_min": -30.739301681518555,
      "activations/layer13_attention_weight_max": 41.292381286621094,
      "activations/layer13_attention_weight_min": -30.32114028930664,
      "activations/layer14_attention_weight_max": 35.61626434326172,
      "activations/layer14_attention_weight_min": -29.74435043334961,
      "activations/layer15_attention_weight_max": 34.112518310546875,
      "activations/layer15_attention_weight_min": -28.784626007080078,
      "activations/layer16_attention_weight_max": 31.49962615966797,
      "activations/layer16_attention_weight_min": -27.945354461669922,
      "activations/layer17_attention_weight_max": 51.443607330322266,
      "activations/layer17_attention_weight_min": -44.51630783081055,
      "activations/layer18_attention_weight_max": 45.21800994873047,
      "activations/layer18_attention_weight_min": -36.36617660522461,
      "activations/layer19_attention_weight_max": 22.286067962646484,
      "activations/layer19_attention_weight_min": -20.912799835205078,
      "activations/layer1_attention_weight_max": 17.03412628173828,
      "activations/layer1_attention_weight_min": -16.491846084594727,
      "activations/layer20_attention_weight_max": 24.15240478515625,
      "activations/layer20_attention_weight_min": -19.273710250854492,
      "activations/layer21_attention_weight_max": 36.29140853881836,
      "activations/layer21_attention_weight_min": -22.7701358795166,
      "activations/layer22_attention_weight_max": 31.26058578491211,
      "activations/layer22_attention_weight_min": -25.439186096191406,
      "activations/layer23_attention_weight_max": 40.356544494628906,
      "activations/layer23_attention_weight_min": -23.567657470703125,
      "activations/layer2_attention_weight_max": 33.46773910522461,
      "activations/layer2_attention_weight_min": -32.75128936767578,
      "activations/layer3_attention_weight_max": 96.35317993164062,
      "activations/layer3_attention_weight_min": -100.14480590820312,
      "activations/layer4_attention_weight_max": 101.4481430053711,
      "activations/layer4_attention_weight_min": -100.75463104248047,
      "activations/layer5_attention_weight_max": 74.68904113769531,
      "activations/layer5_attention_weight_min": -77.13485717773438,
      "activations/layer6_attention_weight_max": 54.87641143798828,
      "activations/layer6_attention_weight_min": -53.84419631958008,
      "activations/layer7_attention_weight_max": 75.48239135742188,
      "activations/layer7_attention_weight_min": -75.24270629882812,
      "activations/layer8_attention_weight_max": 47.26251220703125,
      "activations/layer8_attention_weight_min": -46.952693939208984,
      "activations/layer9_attention_weight_max": 50.340911865234375,
      "activations/layer9_attention_weight_min": -49.78301239013672,
      "epoch": 11.34,
      "learning_rate": 7.763939393939393e-05,
      "loss": 2.7645,
      "step": 195200
    },
    {
      "activations/layer0_attention_weight_max": 16.101200103759766,
      "activations/layer0_attention_weight_min": -13.674595832824707,
      "activations/layer10_attention_weight_max": 33.33160400390625,
      "activations/layer10_attention_weight_min": -31.795635223388672,
      "activations/layer11_attention_weight_max": 28.6878662109375,
      "activations/layer11_attention_weight_min": -29.798206329345703,
      "activations/layer12_attention_weight_max": 18.09999656677246,
      "activations/layer12_attention_weight_min": -27.374732971191406,
      "activations/layer13_attention_weight_max": 42.279476165771484,
      "activations/layer13_attention_weight_min": -28.263206481933594,
      "activations/layer14_attention_weight_max": 35.2421875,
      "activations/layer14_attention_weight_min": -30.306257247924805,
      "activations/layer15_attention_weight_max": 33.511478424072266,
      "activations/layer15_attention_weight_min": -28.093334197998047,
      "activations/layer16_attention_weight_max": 31.35750389099121,
      "activations/layer16_attention_weight_min": -28.476551055908203,
      "activations/layer17_attention_weight_max": 51.23692321777344,
      "activations/layer17_attention_weight_min": -43.224143981933594,
      "activations/layer18_attention_weight_max": 47.00507354736328,
      "activations/layer18_attention_weight_min": -40.254554748535156,
      "activations/layer19_attention_weight_max": 22.131345748901367,
      "activations/layer19_attention_weight_min": -21.42076301574707,
      "activations/layer1_attention_weight_max": 17.549306869506836,
      "activations/layer1_attention_weight_min": -14.569189071655273,
      "activations/layer20_attention_weight_max": 28.2625732421875,
      "activations/layer20_attention_weight_min": -20.924556732177734,
      "activations/layer21_attention_weight_max": 42.00001907348633,
      "activations/layer21_attention_weight_min": -22.42783546447754,
      "activations/layer22_attention_weight_max": 34.6510009765625,
      "activations/layer22_attention_weight_min": -26.570913314819336,
      "activations/layer23_attention_weight_max": 42.8365478515625,
      "activations/layer23_attention_weight_min": -26.100149154663086,
      "activations/layer2_attention_weight_max": 32.03547668457031,
      "activations/layer2_attention_weight_min": -30.682973861694336,
      "activations/layer3_attention_weight_max": 88.36502838134766,
      "activations/layer3_attention_weight_min": -89.20958709716797,
      "activations/layer4_attention_weight_max": 96.13607025146484,
      "activations/layer4_attention_weight_min": -93.23774719238281,
      "activations/layer5_attention_weight_max": 72.77029418945312,
      "activations/layer5_attention_weight_min": -80.5123062133789,
      "activations/layer6_attention_weight_max": 50.78314971923828,
      "activations/layer6_attention_weight_min": -50.99138641357422,
      "activations/layer7_attention_weight_max": 70.36428833007812,
      "activations/layer7_attention_weight_min": -68.04802703857422,
      "activations/layer8_attention_weight_max": 41.87515640258789,
      "activations/layer8_attention_weight_min": -43.97703552246094,
      "activations/layer9_attention_weight_max": 43.09908676147461,
      "activations/layer9_attention_weight_min": -44.43781661987305,
      "epoch": 11.35,
      "learning_rate": 7.762045454545454e-05,
      "loss": 2.7869,
      "step": 195250
    },
    {
      "activations/layer0_attention_weight_max": 15.327472686767578,
      "activations/layer0_attention_weight_min": -13.77025032043457,
      "activations/layer10_attention_weight_max": 48.390953063964844,
      "activations/layer10_attention_weight_min": -44.74065017700195,
      "activations/layer11_attention_weight_max": 41.88069534301758,
      "activations/layer11_attention_weight_min": -39.56024932861328,
      "activations/layer12_attention_weight_max": 18.513813018798828,
      "activations/layer12_attention_weight_min": -27.581327438354492,
      "activations/layer13_attention_weight_max": 49.2996711730957,
      "activations/layer13_attention_weight_min": -30.01225471496582,
      "activations/layer14_attention_weight_max": 55.44086456298828,
      "activations/layer14_attention_weight_min": -31.58527374267578,
      "activations/layer15_attention_weight_max": 48.55288314819336,
      "activations/layer15_attention_weight_min": -28.87687873840332,
      "activations/layer16_attention_weight_max": 30.86604881286621,
      "activations/layer16_attention_weight_min": -28.141685485839844,
      "activations/layer17_attention_weight_max": 55.49748611450195,
      "activations/layer17_attention_weight_min": -44.26347351074219,
      "activations/layer18_attention_weight_max": 47.91018295288086,
      "activations/layer18_attention_weight_min": -42.19214630126953,
      "activations/layer19_attention_weight_max": 24.901142120361328,
      "activations/layer19_attention_weight_min": -23.020009994506836,
      "activations/layer1_attention_weight_max": 16.73870277404785,
      "activations/layer1_attention_weight_min": -18.117605209350586,
      "activations/layer20_attention_weight_max": 27.826404571533203,
      "activations/layer20_attention_weight_min": -20.66901969909668,
      "activations/layer21_attention_weight_max": 46.19273376464844,
      "activations/layer21_attention_weight_min": -21.33717155456543,
      "activations/layer22_attention_weight_max": 33.84455871582031,
      "activations/layer22_attention_weight_min": -25.40533447265625,
      "activations/layer23_attention_weight_max": 33.74827575683594,
      "activations/layer23_attention_weight_min": -24.50714874267578,
      "activations/layer2_attention_weight_max": 34.79345703125,
      "activations/layer2_attention_weight_min": -34.22523498535156,
      "activations/layer3_attention_weight_max": 95.64675903320312,
      "activations/layer3_attention_weight_min": -95.73008728027344,
      "activations/layer4_attention_weight_max": 101.76854705810547,
      "activations/layer4_attention_weight_min": -99.12708282470703,
      "activations/layer5_attention_weight_max": 73.36579132080078,
      "activations/layer5_attention_weight_min": -79.98516082763672,
      "activations/layer6_attention_weight_max": 58.73198699951172,
      "activations/layer6_attention_weight_min": -58.86785125732422,
      "activations/layer7_attention_weight_max": 95.39881896972656,
      "activations/layer7_attention_weight_min": -84.51509857177734,
      "activations/layer8_attention_weight_max": 56.31167984008789,
      "activations/layer8_attention_weight_min": -53.60700225830078,
      "activations/layer9_attention_weight_max": 65.9933090209961,
      "activations/layer9_attention_weight_min": -53.919189453125,
      "epoch": 11.35,
      "learning_rate": 7.760151515151515e-05,
      "loss": 2.7938,
      "step": 195300
    },
    {
      "activations/layer0_attention_weight_max": 14.963582992553711,
      "activations/layer0_attention_weight_min": -13.269447326660156,
      "activations/layer10_attention_weight_max": 33.71729278564453,
      "activations/layer10_attention_weight_min": -34.234256744384766,
      "activations/layer11_attention_weight_max": 30.101964950561523,
      "activations/layer11_attention_weight_min": -31.115802764892578,
      "activations/layer12_attention_weight_max": 18.440370559692383,
      "activations/layer12_attention_weight_min": -25.35747528076172,
      "activations/layer13_attention_weight_max": 32.12688446044922,
      "activations/layer13_attention_weight_min": -28.078157424926758,
      "activations/layer14_attention_weight_max": 35.402706146240234,
      "activations/layer14_attention_weight_min": -30.64209747314453,
      "activations/layer15_attention_weight_max": 31.243396759033203,
      "activations/layer15_attention_weight_min": -28.50419807434082,
      "activations/layer16_attention_weight_max": 29.032684326171875,
      "activations/layer16_attention_weight_min": -29.17705535888672,
      "activations/layer17_attention_weight_max": 49.09864807128906,
      "activations/layer17_attention_weight_min": -42.61757278442383,
      "activations/layer18_attention_weight_max": 41.23988342285156,
      "activations/layer18_attention_weight_min": -36.608848571777344,
      "activations/layer19_attention_weight_max": 21.453067779541016,
      "activations/layer19_attention_weight_min": -24.348825454711914,
      "activations/layer1_attention_weight_max": 17.108301162719727,
      "activations/layer1_attention_weight_min": -16.071866989135742,
      "activations/layer20_attention_weight_max": 21.79106903076172,
      "activations/layer20_attention_weight_min": -23.15399932861328,
      "activations/layer21_attention_weight_max": 32.56916809082031,
      "activations/layer21_attention_weight_min": -22.62057113647461,
      "activations/layer22_attention_weight_max": 27.82262420654297,
      "activations/layer22_attention_weight_min": -25.829668045043945,
      "activations/layer23_attention_weight_max": 35.15742492675781,
      "activations/layer23_attention_weight_min": -25.493507385253906,
      "activations/layer2_attention_weight_max": 33.127506256103516,
      "activations/layer2_attention_weight_min": -30.483592987060547,
      "activations/layer3_attention_weight_max": 93.08274841308594,
      "activations/layer3_attention_weight_min": -90.37496185302734,
      "activations/layer4_attention_weight_max": 97.18221282958984,
      "activations/layer4_attention_weight_min": -95.33064270019531,
      "activations/layer5_attention_weight_max": 70.4715576171875,
      "activations/layer5_attention_weight_min": -73.90418243408203,
      "activations/layer6_attention_weight_max": 52.46398162841797,
      "activations/layer6_attention_weight_min": -53.94084930419922,
      "activations/layer7_attention_weight_max": 73.92071533203125,
      "activations/layer7_attention_weight_min": -70.61544799804688,
      "activations/layer8_attention_weight_max": 42.77607727050781,
      "activations/layer8_attention_weight_min": -46.730926513671875,
      "activations/layer9_attention_weight_max": 44.24458312988281,
      "activations/layer9_attention_weight_min": -46.79159164428711,
      "epoch": 11.35,
      "learning_rate": 7.758257575757576e-05,
      "loss": 2.7736,
      "step": 195350
    },
    {
      "activations/layer0_attention_weight_max": 15.276825904846191,
      "activations/layer0_attention_weight_min": -13.998103141784668,
      "activations/layer10_attention_weight_max": 39.58769607543945,
      "activations/layer10_attention_weight_min": -38.436317443847656,
      "activations/layer11_attention_weight_max": 35.25882339477539,
      "activations/layer11_attention_weight_min": -32.39993667602539,
      "activations/layer12_attention_weight_max": 21.8043155670166,
      "activations/layer12_attention_weight_min": -27.076383590698242,
      "activations/layer13_attention_weight_max": 39.9734992980957,
      "activations/layer13_attention_weight_min": -29.93770980834961,
      "activations/layer14_attention_weight_max": 33.07825469970703,
      "activations/layer14_attention_weight_min": -29.92129135131836,
      "activations/layer15_attention_weight_max": 32.14548873901367,
      "activations/layer15_attention_weight_min": -30.535770416259766,
      "activations/layer16_attention_weight_max": 32.32965087890625,
      "activations/layer16_attention_weight_min": -30.163782119750977,
      "activations/layer17_attention_weight_max": 49.37969207763672,
      "activations/layer17_attention_weight_min": -45.551795959472656,
      "activations/layer18_attention_weight_max": 43.623443603515625,
      "activations/layer18_attention_weight_min": -37.62992477416992,
      "activations/layer19_attention_weight_max": 21.28040885925293,
      "activations/layer19_attention_weight_min": -23.804283142089844,
      "activations/layer1_attention_weight_max": 18.278919219970703,
      "activations/layer1_attention_weight_min": -14.44675064086914,
      "activations/layer20_attention_weight_max": 22.148893356323242,
      "activations/layer20_attention_weight_min": -22.559497833251953,
      "activations/layer21_attention_weight_max": 35.56480407714844,
      "activations/layer21_attention_weight_min": -21.877212524414062,
      "activations/layer22_attention_weight_max": 29.22228240966797,
      "activations/layer22_attention_weight_min": -24.8321533203125,
      "activations/layer23_attention_weight_max": 38.74610900878906,
      "activations/layer23_attention_weight_min": -24.617015838623047,
      "activations/layer2_attention_weight_max": 33.227500915527344,
      "activations/layer2_attention_weight_min": -32.65388107299805,
      "activations/layer3_attention_weight_max": 92.94117736816406,
      "activations/layer3_attention_weight_min": -91.94528198242188,
      "activations/layer4_attention_weight_max": 95.88174438476562,
      "activations/layer4_attention_weight_min": -92.51043701171875,
      "activations/layer5_attention_weight_max": 69.85736083984375,
      "activations/layer5_attention_weight_min": -85.45266723632812,
      "activations/layer6_attention_weight_max": 50.93661880493164,
      "activations/layer6_attention_weight_min": -52.45001983642578,
      "activations/layer7_attention_weight_max": 73.2506332397461,
      "activations/layer7_attention_weight_min": -74.8112564086914,
      "activations/layer8_attention_weight_max": 46.95301055908203,
      "activations/layer8_attention_weight_min": -47.98908233642578,
      "activations/layer9_attention_weight_max": 48.64115905761719,
      "activations/layer9_attention_weight_min": -48.01709747314453,
      "epoch": 11.35,
      "learning_rate": 7.756363636363635e-05,
      "loss": 2.7887,
      "step": 195400
    },
    {
      "activations/layer0_attention_weight_max": 15.154237747192383,
      "activations/layer0_attention_weight_min": -13.703235626220703,
      "activations/layer10_attention_weight_max": 34.01386642456055,
      "activations/layer10_attention_weight_min": -33.939537048339844,
      "activations/layer11_attention_weight_max": 30.110179901123047,
      "activations/layer11_attention_weight_min": -30.297943115234375,
      "activations/layer12_attention_weight_max": 19.514188766479492,
      "activations/layer12_attention_weight_min": -25.30510902404785,
      "activations/layer13_attention_weight_max": 38.07320022583008,
      "activations/layer13_attention_weight_min": -28.63985252380371,
      "activations/layer14_attention_weight_max": 36.486061096191406,
      "activations/layer14_attention_weight_min": -31.452125549316406,
      "activations/layer15_attention_weight_max": 34.14849090576172,
      "activations/layer15_attention_weight_min": -31.09710121154785,
      "activations/layer16_attention_weight_max": 34.1052360534668,
      "activations/layer16_attention_weight_min": -33.02455139160156,
      "activations/layer17_attention_weight_max": 55.17603302001953,
      "activations/layer17_attention_weight_min": -46.83763885498047,
      "activations/layer18_attention_weight_max": 45.70519256591797,
      "activations/layer18_attention_weight_min": -37.310882568359375,
      "activations/layer19_attention_weight_max": 23.04296112060547,
      "activations/layer19_attention_weight_min": -21.426931381225586,
      "activations/layer1_attention_weight_max": 16.879039764404297,
      "activations/layer1_attention_weight_min": -14.898870468139648,
      "activations/layer20_attention_weight_max": 21.68096160888672,
      "activations/layer20_attention_weight_min": -19.700986862182617,
      "activations/layer21_attention_weight_max": 34.845272064208984,
      "activations/layer21_attention_weight_min": -20.931180953979492,
      "activations/layer22_attention_weight_max": 31.16069793701172,
      "activations/layer22_attention_weight_min": -27.61517333984375,
      "activations/layer23_attention_weight_max": 36.367469787597656,
      "activations/layer23_attention_weight_min": -24.658100128173828,
      "activations/layer2_attention_weight_max": 32.248268127441406,
      "activations/layer2_attention_weight_min": -31.2525634765625,
      "activations/layer3_attention_weight_max": 94.283935546875,
      "activations/layer3_attention_weight_min": -91.5615005493164,
      "activations/layer4_attention_weight_max": 101.65123748779297,
      "activations/layer4_attention_weight_min": -96.70115661621094,
      "activations/layer5_attention_weight_max": 75.54197692871094,
      "activations/layer5_attention_weight_min": -77.08765411376953,
      "activations/layer6_attention_weight_max": 52.708438873291016,
      "activations/layer6_attention_weight_min": -51.69139099121094,
      "activations/layer7_attention_weight_max": 76.74932098388672,
      "activations/layer7_attention_weight_min": -73.2956314086914,
      "activations/layer8_attention_weight_max": 47.12774658203125,
      "activations/layer8_attention_weight_min": -46.26448059082031,
      "activations/layer9_attention_weight_max": 46.21686935424805,
      "activations/layer9_attention_weight_min": -45.21547317504883,
      "epoch": 11.36,
      "learning_rate": 7.754469696969696e-05,
      "loss": 2.7841,
      "step": 195450
    },
    {
      "activations/layer0_attention_weight_max": 14.992257118225098,
      "activations/layer0_attention_weight_min": -14.172621726989746,
      "activations/layer10_attention_weight_max": 43.99744415283203,
      "activations/layer10_attention_weight_min": -43.18804168701172,
      "activations/layer11_attention_weight_max": 40.65119934082031,
      "activations/layer11_attention_weight_min": -38.615291595458984,
      "activations/layer12_attention_weight_max": 19.456619262695312,
      "activations/layer12_attention_weight_min": -27.88379669189453,
      "activations/layer13_attention_weight_max": 52.83712387084961,
      "activations/layer13_attention_weight_min": -31.504629135131836,
      "activations/layer14_attention_weight_max": 43.54671859741211,
      "activations/layer14_attention_weight_min": -29.534990310668945,
      "activations/layer15_attention_weight_max": 40.35432815551758,
      "activations/layer15_attention_weight_min": -28.4276123046875,
      "activations/layer16_attention_weight_max": 29.717504501342773,
      "activations/layer16_attention_weight_min": -28.23125457763672,
      "activations/layer17_attention_weight_max": 51.912479400634766,
      "activations/layer17_attention_weight_min": -41.89229202270508,
      "activations/layer18_attention_weight_max": 44.93983840942383,
      "activations/layer18_attention_weight_min": -35.242950439453125,
      "activations/layer19_attention_weight_max": 21.442522048950195,
      "activations/layer19_attention_weight_min": -20.43302345275879,
      "activations/layer1_attention_weight_max": 17.591960906982422,
      "activations/layer1_attention_weight_min": -15.781745910644531,
      "activations/layer20_attention_weight_max": 21.56668472290039,
      "activations/layer20_attention_weight_min": -20.000913619995117,
      "activations/layer21_attention_weight_max": 36.369537353515625,
      "activations/layer21_attention_weight_min": -21.279766082763672,
      "activations/layer22_attention_weight_max": 29.871213912963867,
      "activations/layer22_attention_weight_min": -23.488178253173828,
      "activations/layer23_attention_weight_max": 37.89081954956055,
      "activations/layer23_attention_weight_min": -23.274288177490234,
      "activations/layer2_attention_weight_max": 33.06619644165039,
      "activations/layer2_attention_weight_min": -33.49041748046875,
      "activations/layer3_attention_weight_max": 98.15003204345703,
      "activations/layer3_attention_weight_min": -98.3773422241211,
      "activations/layer4_attention_weight_max": 103.51191711425781,
      "activations/layer4_attention_weight_min": -99.4224853515625,
      "activations/layer5_attention_weight_max": 74.30552673339844,
      "activations/layer5_attention_weight_min": -80.58859252929688,
      "activations/layer6_attention_weight_max": 54.27823257446289,
      "activations/layer6_attention_weight_min": -53.44202423095703,
      "activations/layer7_attention_weight_max": 90.58522033691406,
      "activations/layer7_attention_weight_min": -76.29168701171875,
      "activations/layer8_attention_weight_max": 51.968101501464844,
      "activations/layer8_attention_weight_min": -49.53434371948242,
      "activations/layer9_attention_weight_max": 59.225868225097656,
      "activations/layer9_attention_weight_min": -52.99946212768555,
      "epoch": 11.36,
      "learning_rate": 7.752575757575758e-05,
      "loss": 2.7728,
      "step": 195500
    },
    {
      "activations/layer0_attention_weight_max": 17.25537109375,
      "activations/layer0_attention_weight_min": -13.801145553588867,
      "activations/layer10_attention_weight_max": 38.17591094970703,
      "activations/layer10_attention_weight_min": -36.7882194519043,
      "activations/layer11_attention_weight_max": 33.544883728027344,
      "activations/layer11_attention_weight_min": -32.363285064697266,
      "activations/layer12_attention_weight_max": 21.253828048706055,
      "activations/layer12_attention_weight_min": -25.240537643432617,
      "activations/layer13_attention_weight_max": 49.47190856933594,
      "activations/layer13_attention_weight_min": -32.123470306396484,
      "activations/layer14_attention_weight_max": 44.77450942993164,
      "activations/layer14_attention_weight_min": -33.18440628051758,
      "activations/layer15_attention_weight_max": 38.60415267944336,
      "activations/layer15_attention_weight_min": -32.22061538696289,
      "activations/layer16_attention_weight_max": 34.05799865722656,
      "activations/layer16_attention_weight_min": -31.70462989807129,
      "activations/layer17_attention_weight_max": 57.86977767944336,
      "activations/layer17_attention_weight_min": -49.565093994140625,
      "activations/layer18_attention_weight_max": 50.2993278503418,
      "activations/layer18_attention_weight_min": -41.61471939086914,
      "activations/layer19_attention_weight_max": 25.320524215698242,
      "activations/layer19_attention_weight_min": -23.097074508666992,
      "activations/layer1_attention_weight_max": 17.112401962280273,
      "activations/layer1_attention_weight_min": -17.85157585144043,
      "activations/layer20_attention_weight_max": 24.099119186401367,
      "activations/layer20_attention_weight_min": -24.67803955078125,
      "activations/layer21_attention_weight_max": 42.20191955566406,
      "activations/layer21_attention_weight_min": -22.78557586669922,
      "activations/layer22_attention_weight_max": 30.61053466796875,
      "activations/layer22_attention_weight_min": -26.2534122467041,
      "activations/layer23_attention_weight_max": 40.43401336669922,
      "activations/layer23_attention_weight_min": -23.814613342285156,
      "activations/layer2_attention_weight_max": 34.30584716796875,
      "activations/layer2_attention_weight_min": -31.608848571777344,
      "activations/layer3_attention_weight_max": 99.84024810791016,
      "activations/layer3_attention_weight_min": -94.49301147460938,
      "activations/layer4_attention_weight_max": 99.85978698730469,
      "activations/layer4_attention_weight_min": -96.86964416503906,
      "activations/layer5_attention_weight_max": 72.58587646484375,
      "activations/layer5_attention_weight_min": -79.15458679199219,
      "activations/layer6_attention_weight_max": 54.48354721069336,
      "activations/layer6_attention_weight_min": -54.06373977661133,
      "activations/layer7_attention_weight_max": 78.68235778808594,
      "activations/layer7_attention_weight_min": -73.12886047363281,
      "activations/layer8_attention_weight_max": 43.921016693115234,
      "activations/layer8_attention_weight_min": -49.10572814941406,
      "activations/layer9_attention_weight_max": 50.117549896240234,
      "activations/layer9_attention_weight_min": -50.42447280883789,
      "epoch": 11.36,
      "learning_rate": 7.750681818181817e-05,
      "loss": 2.7764,
      "step": 195550
    },
    {
      "activations/layer0_attention_weight_max": 15.236255645751953,
      "activations/layer0_attention_weight_min": -13.848442077636719,
      "activations/layer10_attention_weight_max": 37.177982330322266,
      "activations/layer10_attention_weight_min": -36.329498291015625,
      "activations/layer11_attention_weight_max": 32.823883056640625,
      "activations/layer11_attention_weight_min": -33.126949310302734,
      "activations/layer12_attention_weight_max": 21.160140991210938,
      "activations/layer12_attention_weight_min": -23.546401977539062,
      "activations/layer13_attention_weight_max": 54.372337341308594,
      "activations/layer13_attention_weight_min": -34.75444412231445,
      "activations/layer14_attention_weight_max": 48.11017990112305,
      "activations/layer14_attention_weight_min": -32.0748176574707,
      "activations/layer15_attention_weight_max": 38.67195129394531,
      "activations/layer15_attention_weight_min": -32.135658264160156,
      "activations/layer16_attention_weight_max": 34.74135208129883,
      "activations/layer16_attention_weight_min": -32.68169021606445,
      "activations/layer17_attention_weight_max": 57.32460021972656,
      "activations/layer17_attention_weight_min": -50.83371353149414,
      "activations/layer18_attention_weight_max": 53.18529510498047,
      "activations/layer18_attention_weight_min": -41.268619537353516,
      "activations/layer19_attention_weight_max": 25.47652244567871,
      "activations/layer19_attention_weight_min": -24.000072479248047,
      "activations/layer1_attention_weight_max": 16.924667358398438,
      "activations/layer1_attention_weight_min": -16.096797943115234,
      "activations/layer20_attention_weight_max": 29.36662483215332,
      "activations/layer20_attention_weight_min": -21.274295806884766,
      "activations/layer21_attention_weight_max": 44.381038665771484,
      "activations/layer21_attention_weight_min": -22.54126739501953,
      "activations/layer22_attention_weight_max": 34.03757095336914,
      "activations/layer22_attention_weight_min": -25.260549545288086,
      "activations/layer23_attention_weight_max": 41.63169860839844,
      "activations/layer23_attention_weight_min": -22.557615280151367,
      "activations/layer2_attention_weight_max": 34.22856140136719,
      "activations/layer2_attention_weight_min": -31.898534774780273,
      "activations/layer3_attention_weight_max": 96.24906158447266,
      "activations/layer3_attention_weight_min": -97.03133392333984,
      "activations/layer4_attention_weight_max": 100.84712982177734,
      "activations/layer4_attention_weight_min": -94.52376556396484,
      "activations/layer5_attention_weight_max": 76.40095520019531,
      "activations/layer5_attention_weight_min": -85.81392669677734,
      "activations/layer6_attention_weight_max": 56.08366012573242,
      "activations/layer6_attention_weight_min": -54.47726058959961,
      "activations/layer7_attention_weight_max": 80.98853302001953,
      "activations/layer7_attention_weight_min": -70.69007110595703,
      "activations/layer8_attention_weight_max": 44.64030075073242,
      "activations/layer8_attention_weight_min": -48.52174758911133,
      "activations/layer9_attention_weight_max": 45.833587646484375,
      "activations/layer9_attention_weight_min": -49.630062103271484,
      "epoch": 11.37,
      "learning_rate": 7.748787878787878e-05,
      "loss": 2.7708,
      "step": 195600
    },
    {
      "activations/layer0_attention_weight_max": 15.105283737182617,
      "activations/layer0_attention_weight_min": -13.927303314208984,
      "activations/layer10_attention_weight_max": 34.028709411621094,
      "activations/layer10_attention_weight_min": -35.643157958984375,
      "activations/layer11_attention_weight_max": 29.786270141601562,
      "activations/layer11_attention_weight_min": -32.330753326416016,
      "activations/layer12_attention_weight_max": 19.1356258392334,
      "activations/layer12_attention_weight_min": -24.919269561767578,
      "activations/layer13_attention_weight_max": 37.201866149902344,
      "activations/layer13_attention_weight_min": -28.397682189941406,
      "activations/layer14_attention_weight_max": 34.566062927246094,
      "activations/layer14_attention_weight_min": -30.688426971435547,
      "activations/layer15_attention_weight_max": 32.01938247680664,
      "activations/layer15_attention_weight_min": -29.104042053222656,
      "activations/layer16_attention_weight_max": 29.526065826416016,
      "activations/layer16_attention_weight_min": -29.7842960357666,
      "activations/layer17_attention_weight_max": 49.663578033447266,
      "activations/layer17_attention_weight_min": -42.856422424316406,
      "activations/layer18_attention_weight_max": 42.722900390625,
      "activations/layer18_attention_weight_min": -35.789581298828125,
      "activations/layer19_attention_weight_max": 21.781082153320312,
      "activations/layer19_attention_weight_min": -21.934682846069336,
      "activations/layer1_attention_weight_max": 16.481809616088867,
      "activations/layer1_attention_weight_min": -15.641860961914062,
      "activations/layer20_attention_weight_max": 21.079063415527344,
      "activations/layer20_attention_weight_min": -24.532365798950195,
      "activations/layer21_attention_weight_max": 37.08012008666992,
      "activations/layer21_attention_weight_min": -22.198755264282227,
      "activations/layer22_attention_weight_max": 31.110595703125,
      "activations/layer22_attention_weight_min": -26.42899513244629,
      "activations/layer23_attention_weight_max": 37.09470748901367,
      "activations/layer23_attention_weight_min": -21.95230484008789,
      "activations/layer2_attention_weight_max": 33.833229064941406,
      "activations/layer2_attention_weight_min": -32.350860595703125,
      "activations/layer3_attention_weight_max": 90.19287872314453,
      "activations/layer3_attention_weight_min": -92.69257354736328,
      "activations/layer4_attention_weight_max": 98.89048767089844,
      "activations/layer4_attention_weight_min": -93.20870208740234,
      "activations/layer5_attention_weight_max": 70.01929473876953,
      "activations/layer5_attention_weight_min": -76.8742904663086,
      "activations/layer6_attention_weight_max": 51.02125549316406,
      "activations/layer6_attention_weight_min": -51.31262969970703,
      "activations/layer7_attention_weight_max": 72.52574157714844,
      "activations/layer7_attention_weight_min": -70.04197692871094,
      "activations/layer8_attention_weight_max": 43.4458122253418,
      "activations/layer8_attention_weight_min": -44.65974426269531,
      "activations/layer9_attention_weight_max": 46.94963836669922,
      "activations/layer9_attention_weight_min": -46.9218864440918,
      "epoch": 11.37,
      "learning_rate": 7.74689393939394e-05,
      "loss": 2.7783,
      "step": 195650
    },
    {
      "activations/layer0_attention_weight_max": 15.804144859313965,
      "activations/layer0_attention_weight_min": -13.528959274291992,
      "activations/layer10_attention_weight_max": 35.20032501220703,
      "activations/layer10_attention_weight_min": -34.437660217285156,
      "activations/layer11_attention_weight_max": 29.75082015991211,
      "activations/layer11_attention_weight_min": -31.602584838867188,
      "activations/layer12_attention_weight_max": 17.70969009399414,
      "activations/layer12_attention_weight_min": -22.59025001525879,
      "activations/layer13_attention_weight_max": 39.793601989746094,
      "activations/layer13_attention_weight_min": -31.568803787231445,
      "activations/layer14_attention_weight_max": 46.33112335205078,
      "activations/layer14_attention_weight_min": -29.582094192504883,
      "activations/layer15_attention_weight_max": 37.72371292114258,
      "activations/layer15_attention_weight_min": -28.750656127929688,
      "activations/layer16_attention_weight_max": 30.61393928527832,
      "activations/layer16_attention_weight_min": -30.205663681030273,
      "activations/layer17_attention_weight_max": 50.31050491333008,
      "activations/layer17_attention_weight_min": -45.08694076538086,
      "activations/layer18_attention_weight_max": 47.411216735839844,
      "activations/layer18_attention_weight_min": -39.29412078857422,
      "activations/layer19_attention_weight_max": 23.86771011352539,
      "activations/layer19_attention_weight_min": -24.481327056884766,
      "activations/layer1_attention_weight_max": 16.458131790161133,
      "activations/layer1_attention_weight_min": -16.945911407470703,
      "activations/layer20_attention_weight_max": 26.31952476501465,
      "activations/layer20_attention_weight_min": -21.974811553955078,
      "activations/layer21_attention_weight_max": 40.90290451049805,
      "activations/layer21_attention_weight_min": -25.3076229095459,
      "activations/layer22_attention_weight_max": 32.12236022949219,
      "activations/layer22_attention_weight_min": -28.48554801940918,
      "activations/layer23_attention_weight_max": 36.73262405395508,
      "activations/layer23_attention_weight_min": -23.86958885192871,
      "activations/layer2_attention_weight_max": 32.89439010620117,
      "activations/layer2_attention_weight_min": -30.20296287536621,
      "activations/layer3_attention_weight_max": 92.19171905517578,
      "activations/layer3_attention_weight_min": -95.61323547363281,
      "activations/layer4_attention_weight_max": 94.86174774169922,
      "activations/layer4_attention_weight_min": -90.48731994628906,
      "activations/layer5_attention_weight_max": 70.21784973144531,
      "activations/layer5_attention_weight_min": -75.6181869506836,
      "activations/layer6_attention_weight_max": 51.054439544677734,
      "activations/layer6_attention_weight_min": -50.93901824951172,
      "activations/layer7_attention_weight_max": 71.39400482177734,
      "activations/layer7_attention_weight_min": -69.5526123046875,
      "activations/layer8_attention_weight_max": 44.99864196777344,
      "activations/layer8_attention_weight_min": -44.280853271484375,
      "activations/layer9_attention_weight_max": 43.84249496459961,
      "activations/layer9_attention_weight_min": -45.40239334106445,
      "epoch": 11.37,
      "learning_rate": 7.744999999999999e-05,
      "loss": 2.7917,
      "step": 195700
    },
    {
      "activations/layer0_attention_weight_max": 15.367258071899414,
      "activations/layer0_attention_weight_min": -14.100029945373535,
      "activations/layer10_attention_weight_max": 36.221466064453125,
      "activations/layer10_attention_weight_min": -35.211151123046875,
      "activations/layer11_attention_weight_max": 30.71718978881836,
      "activations/layer11_attention_weight_min": -31.248260498046875,
      "activations/layer12_attention_weight_max": 17.720548629760742,
      "activations/layer12_attention_weight_min": -24.801815032958984,
      "activations/layer13_attention_weight_max": 36.43770217895508,
      "activations/layer13_attention_weight_min": -30.704635620117188,
      "activations/layer14_attention_weight_max": 35.52748489379883,
      "activations/layer14_attention_weight_min": -28.839250564575195,
      "activations/layer15_attention_weight_max": 32.2469482421875,
      "activations/layer15_attention_weight_min": -29.602100372314453,
      "activations/layer16_attention_weight_max": 30.435684204101562,
      "activations/layer16_attention_weight_min": -29.901479721069336,
      "activations/layer17_attention_weight_max": 52.719703674316406,
      "activations/layer17_attention_weight_min": -46.611480712890625,
      "activations/layer18_attention_weight_max": 45.112754821777344,
      "activations/layer18_attention_weight_min": -37.67438507080078,
      "activations/layer19_attention_weight_max": 21.640655517578125,
      "activations/layer19_attention_weight_min": -23.516843795776367,
      "activations/layer1_attention_weight_max": 16.29965591430664,
      "activations/layer1_attention_weight_min": -14.910958290100098,
      "activations/layer20_attention_weight_max": 21.691490173339844,
      "activations/layer20_attention_weight_min": -21.456851959228516,
      "activations/layer21_attention_weight_max": 34.15229415893555,
      "activations/layer21_attention_weight_min": -23.362279891967773,
      "activations/layer22_attention_weight_max": 33.48731231689453,
      "activations/layer22_attention_weight_min": -25.762714385986328,
      "activations/layer23_attention_weight_max": 37.200504302978516,
      "activations/layer23_attention_weight_min": -25.287504196166992,
      "activations/layer2_attention_weight_max": 32.18321228027344,
      "activations/layer2_attention_weight_min": -29.816259384155273,
      "activations/layer3_attention_weight_max": 93.25397491455078,
      "activations/layer3_attention_weight_min": -96.48622131347656,
      "activations/layer4_attention_weight_max": 99.92791748046875,
      "activations/layer4_attention_weight_min": -95.43891143798828,
      "activations/layer5_attention_weight_max": 72.48896789550781,
      "activations/layer5_attention_weight_min": -75.70133972167969,
      "activations/layer6_attention_weight_max": 55.8874626159668,
      "activations/layer6_attention_weight_min": -56.33778762817383,
      "activations/layer7_attention_weight_max": 74.28446960449219,
      "activations/layer7_attention_weight_min": -69.85208129882812,
      "activations/layer8_attention_weight_max": 42.818450927734375,
      "activations/layer8_attention_weight_min": -43.078125,
      "activations/layer9_attention_weight_max": 49.88803482055664,
      "activations/layer9_attention_weight_min": -46.18171310424805,
      "epoch": 11.37,
      "learning_rate": 7.74310606060606e-05,
      "loss": 2.7955,
      "step": 195750
    },
    {
      "activations/layer0_attention_weight_max": 15.17408561706543,
      "activations/layer0_attention_weight_min": -13.888837814331055,
      "activations/layer10_attention_weight_max": 35.1586799621582,
      "activations/layer10_attention_weight_min": -35.36466979980469,
      "activations/layer11_attention_weight_max": 31.82765769958496,
      "activations/layer11_attention_weight_min": -31.34455680847168,
      "activations/layer12_attention_weight_max": 21.07913589477539,
      "activations/layer12_attention_weight_min": -24.23239517211914,
      "activations/layer13_attention_weight_max": 44.93329620361328,
      "activations/layer13_attention_weight_min": -30.90497398376465,
      "activations/layer14_attention_weight_max": 41.3442497253418,
      "activations/layer14_attention_weight_min": -29.47221565246582,
      "activations/layer15_attention_weight_max": 33.61332702636719,
      "activations/layer15_attention_weight_min": -27.792770385742188,
      "activations/layer16_attention_weight_max": 32.165191650390625,
      "activations/layer16_attention_weight_min": -27.380786895751953,
      "activations/layer17_attention_weight_max": 51.52571105957031,
      "activations/layer17_attention_weight_min": -43.420196533203125,
      "activations/layer18_attention_weight_max": 43.3453483581543,
      "activations/layer18_attention_weight_min": -37.176971435546875,
      "activations/layer19_attention_weight_max": 22.804182052612305,
      "activations/layer19_attention_weight_min": -21.30284309387207,
      "activations/layer1_attention_weight_max": 17.16197395324707,
      "activations/layer1_attention_weight_min": -14.658463478088379,
      "activations/layer20_attention_weight_max": 21.154775619506836,
      "activations/layer20_attention_weight_min": -21.75929069519043,
      "activations/layer21_attention_weight_max": 40.025596618652344,
      "activations/layer21_attention_weight_min": -24.200986862182617,
      "activations/layer22_attention_weight_max": 30.37627410888672,
      "activations/layer22_attention_weight_min": -24.79212760925293,
      "activations/layer23_attention_weight_max": 36.602073669433594,
      "activations/layer23_attention_weight_min": -22.75137710571289,
      "activations/layer2_attention_weight_max": 32.34333038330078,
      "activations/layer2_attention_weight_min": -30.555761337280273,
      "activations/layer3_attention_weight_max": 91.40892791748047,
      "activations/layer3_attention_weight_min": -90.66349792480469,
      "activations/layer4_attention_weight_max": 96.18844604492188,
      "activations/layer4_attention_weight_min": -92.40868377685547,
      "activations/layer5_attention_weight_max": 72.5780258178711,
      "activations/layer5_attention_weight_min": -75.51466369628906,
      "activations/layer6_attention_weight_max": 53.049720764160156,
      "activations/layer6_attention_weight_min": -52.00527572631836,
      "activations/layer7_attention_weight_max": 76.29554748535156,
      "activations/layer7_attention_weight_min": -69.36273956298828,
      "activations/layer8_attention_weight_max": 44.34294509887695,
      "activations/layer8_attention_weight_min": -47.143226623535156,
      "activations/layer9_attention_weight_max": 50.418277740478516,
      "activations/layer9_attention_weight_min": -46.1854133605957,
      "epoch": 11.38,
      "learning_rate": 7.741249999999999e-05,
      "loss": 2.7572,
      "step": 195800
    },
    {
      "activations/layer0_attention_weight_max": 16.03080940246582,
      "activations/layer0_attention_weight_min": -13.745994567871094,
      "activations/layer10_attention_weight_max": 42.91989517211914,
      "activations/layer10_attention_weight_min": -38.6671257019043,
      "activations/layer11_attention_weight_max": 38.941593170166016,
      "activations/layer11_attention_weight_min": -35.03131103515625,
      "activations/layer12_attention_weight_max": 17.530895233154297,
      "activations/layer12_attention_weight_min": -30.45640754699707,
      "activations/layer13_attention_weight_max": 60.97690200805664,
      "activations/layer13_attention_weight_min": -29.511837005615234,
      "activations/layer14_attention_weight_max": 56.06981658935547,
      "activations/layer14_attention_weight_min": -28.11908531188965,
      "activations/layer15_attention_weight_max": 51.14300537109375,
      "activations/layer15_attention_weight_min": -28.436403274536133,
      "activations/layer16_attention_weight_max": 31.496170043945312,
      "activations/layer16_attention_weight_min": -29.84261703491211,
      "activations/layer17_attention_weight_max": 52.86894607543945,
      "activations/layer17_attention_weight_min": -44.5948371887207,
      "activations/layer18_attention_weight_max": 46.589561462402344,
      "activations/layer18_attention_weight_min": -38.69572067260742,
      "activations/layer19_attention_weight_max": 22.79694175720215,
      "activations/layer19_attention_weight_min": -23.529142379760742,
      "activations/layer1_attention_weight_max": 17.90098762512207,
      "activations/layer1_attention_weight_min": -14.603330612182617,
      "activations/layer20_attention_weight_max": 23.083179473876953,
      "activations/layer20_attention_weight_min": -20.952301025390625,
      "activations/layer21_attention_weight_max": 36.17485809326172,
      "activations/layer21_attention_weight_min": -22.389490127563477,
      "activations/layer22_attention_weight_max": 29.73159408569336,
      "activations/layer22_attention_weight_min": -25.37262535095215,
      "activations/layer23_attention_weight_max": 34.273258209228516,
      "activations/layer23_attention_weight_min": -23.202024459838867,
      "activations/layer2_attention_weight_max": 33.167110443115234,
      "activations/layer2_attention_weight_min": -30.45868492126465,
      "activations/layer3_attention_weight_max": 89.63477325439453,
      "activations/layer3_attention_weight_min": -92.45990753173828,
      "activations/layer4_attention_weight_max": 94.73670196533203,
      "activations/layer4_attention_weight_min": -94.6903076171875,
      "activations/layer5_attention_weight_max": 73.25350952148438,
      "activations/layer5_attention_weight_min": -76.7396469116211,
      "activations/layer6_attention_weight_max": 53.949649810791016,
      "activations/layer6_attention_weight_min": -51.09685516357422,
      "activations/layer7_attention_weight_max": 80.82611846923828,
      "activations/layer7_attention_weight_min": -70.2829818725586,
      "activations/layer8_attention_weight_max": 50.30778121948242,
      "activations/layer8_attention_weight_min": -45.453819274902344,
      "activations/layer9_attention_weight_max": 61.442195892333984,
      "activations/layer9_attention_weight_min": -55.53932571411133,
      "epoch": 11.38,
      "learning_rate": 7.73935606060606e-05,
      "loss": 2.7867,
      "step": 195850
    },
    {
      "activations/layer0_attention_weight_max": 16.078323364257812,
      "activations/layer0_attention_weight_min": -13.5149564743042,
      "activations/layer10_attention_weight_max": 43.83062744140625,
      "activations/layer10_attention_weight_min": -37.90507888793945,
      "activations/layer11_attention_weight_max": 38.83927917480469,
      "activations/layer11_attention_weight_min": -33.5251579284668,
      "activations/layer12_attention_weight_max": 27.116849899291992,
      "activations/layer12_attention_weight_min": -23.056167602539062,
      "activations/layer13_attention_weight_max": 33.727294921875,
      "activations/layer13_attention_weight_min": -26.523141860961914,
      "activations/layer14_attention_weight_max": 37.03728103637695,
      "activations/layer14_attention_weight_min": -30.784271240234375,
      "activations/layer15_attention_weight_max": 34.10540008544922,
      "activations/layer15_attention_weight_min": -29.3858642578125,
      "activations/layer16_attention_weight_max": 34.518035888671875,
      "activations/layer16_attention_weight_min": -29.56169319152832,
      "activations/layer17_attention_weight_max": 57.57087707519531,
      "activations/layer17_attention_weight_min": -45.76272964477539,
      "activations/layer18_attention_weight_max": 46.432838439941406,
      "activations/layer18_attention_weight_min": -38.70532989501953,
      "activations/layer19_attention_weight_max": 20.034364700317383,
      "activations/layer19_attention_weight_min": -20.00048828125,
      "activations/layer1_attention_weight_max": 17.25718116760254,
      "activations/layer1_attention_weight_min": -14.804099082946777,
      "activations/layer20_attention_weight_max": 21.686573028564453,
      "activations/layer20_attention_weight_min": -20.68790054321289,
      "activations/layer21_attention_weight_max": 34.88677978515625,
      "activations/layer21_attention_weight_min": -21.974903106689453,
      "activations/layer22_attention_weight_max": 28.56684684753418,
      "activations/layer22_attention_weight_min": -24.42313003540039,
      "activations/layer23_attention_weight_max": 36.23893737792969,
      "activations/layer23_attention_weight_min": -23.782440185546875,
      "activations/layer2_attention_weight_max": 33.44049072265625,
      "activations/layer2_attention_weight_min": -31.61676025390625,
      "activations/layer3_attention_weight_max": 93.668212890625,
      "activations/layer3_attention_weight_min": -94.7092514038086,
      "activations/layer4_attention_weight_max": 97.66735076904297,
      "activations/layer4_attention_weight_min": -97.45318603515625,
      "activations/layer5_attention_weight_max": 73.30516815185547,
      "activations/layer5_attention_weight_min": -78.72744750976562,
      "activations/layer6_attention_weight_max": 54.85993194580078,
      "activations/layer6_attention_weight_min": -57.40366744995117,
      "activations/layer7_attention_weight_max": 80.61052703857422,
      "activations/layer7_attention_weight_min": -76.56800842285156,
      "activations/layer8_attention_weight_max": 52.36766815185547,
      "activations/layer8_attention_weight_min": -46.77095413208008,
      "activations/layer9_attention_weight_max": 56.16887283325195,
      "activations/layer9_attention_weight_min": -48.44097900390625,
      "epoch": 11.38,
      "learning_rate": 7.737462121212121e-05,
      "loss": 2.7659,
      "step": 195900
    },
    {
      "activations/layer0_attention_weight_max": 15.280441284179688,
      "activations/layer0_attention_weight_min": -13.876852989196777,
      "activations/layer10_attention_weight_max": 38.144283294677734,
      "activations/layer10_attention_weight_min": -37.88156509399414,
      "activations/layer11_attention_weight_max": 35.262943267822266,
      "activations/layer11_attention_weight_min": -33.07258605957031,
      "activations/layer12_attention_weight_max": 18.34905433654785,
      "activations/layer12_attention_weight_min": -27.895505905151367,
      "activations/layer13_attention_weight_max": 36.364349365234375,
      "activations/layer13_attention_weight_min": -30.793825149536133,
      "activations/layer14_attention_weight_max": 39.37197494506836,
      "activations/layer14_attention_weight_min": -32.429443359375,
      "activations/layer15_attention_weight_max": 34.960323333740234,
      "activations/layer15_attention_weight_min": -33.002525329589844,
      "activations/layer16_attention_weight_max": 32.30635070800781,
      "activations/layer16_attention_weight_min": -30.90944480895996,
      "activations/layer17_attention_weight_max": 54.114288330078125,
      "activations/layer17_attention_weight_min": -45.66218566894531,
      "activations/layer18_attention_weight_max": 50.10143280029297,
      "activations/layer18_attention_weight_min": -42.28788757324219,
      "activations/layer19_attention_weight_max": 21.427927017211914,
      "activations/layer19_attention_weight_min": -22.94369125366211,
      "activations/layer1_attention_weight_max": 17.604515075683594,
      "activations/layer1_attention_weight_min": -15.3174409866333,
      "activations/layer20_attention_weight_max": 20.95238494873047,
      "activations/layer20_attention_weight_min": -21.082218170166016,
      "activations/layer21_attention_weight_max": 37.97949981689453,
      "activations/layer21_attention_weight_min": -28.21712303161621,
      "activations/layer22_attention_weight_max": 31.204866409301758,
      "activations/layer22_attention_weight_min": -27.51841926574707,
      "activations/layer23_attention_weight_max": 38.986873626708984,
      "activations/layer23_attention_weight_min": -24.501699447631836,
      "activations/layer2_attention_weight_max": 35.5594482421875,
      "activations/layer2_attention_weight_min": -33.74521255493164,
      "activations/layer3_attention_weight_max": 99.24993896484375,
      "activations/layer3_attention_weight_min": -104.20313262939453,
      "activations/layer4_attention_weight_max": 100.8161849975586,
      "activations/layer4_attention_weight_min": -95.35479736328125,
      "activations/layer5_attention_weight_max": 75.66856384277344,
      "activations/layer5_attention_weight_min": -78.40322875976562,
      "activations/layer6_attention_weight_max": 56.02455139160156,
      "activations/layer6_attention_weight_min": -52.26649856567383,
      "activations/layer7_attention_weight_max": 82.61131286621094,
      "activations/layer7_attention_weight_min": -73.97834014892578,
      "activations/layer8_attention_weight_max": 46.5650520324707,
      "activations/layer8_attention_weight_min": -50.20220947265625,
      "activations/layer9_attention_weight_max": 54.52702713012695,
      "activations/layer9_attention_weight_min": -50.71136474609375,
      "epoch": 11.39,
      "learning_rate": 7.735568181818181e-05,
      "loss": 2.7742,
      "step": 195950
    },
    {
      "activations/layer0_attention_weight_max": 15.071625709533691,
      "activations/layer0_attention_weight_min": -13.365477561950684,
      "activations/layer10_attention_weight_max": 36.85776138305664,
      "activations/layer10_attention_weight_min": -36.37140655517578,
      "activations/layer11_attention_weight_max": 33.553428649902344,
      "activations/layer11_attention_weight_min": -34.66138458251953,
      "activations/layer12_attention_weight_max": 18.330957412719727,
      "activations/layer12_attention_weight_min": -23.8763484954834,
      "activations/layer13_attention_weight_max": 40.73949432373047,
      "activations/layer13_attention_weight_min": -34.190345764160156,
      "activations/layer14_attention_weight_max": 39.31377410888672,
      "activations/layer14_attention_weight_min": -32.317989349365234,
      "activations/layer15_attention_weight_max": 34.972145080566406,
      "activations/layer15_attention_weight_min": -30.201261520385742,
      "activations/layer16_attention_weight_max": 31.64286994934082,
      "activations/layer16_attention_weight_min": -30.07396125793457,
      "activations/layer17_attention_weight_max": 55.32917404174805,
      "activations/layer17_attention_weight_min": -45.160430908203125,
      "activations/layer18_attention_weight_max": 48.545616149902344,
      "activations/layer18_attention_weight_min": -39.13767623901367,
      "activations/layer19_attention_weight_max": 25.144006729125977,
      "activations/layer19_attention_weight_min": -23.77400779724121,
      "activations/layer1_attention_weight_max": 16.381668090820312,
      "activations/layer1_attention_weight_min": -16.469545364379883,
      "activations/layer20_attention_weight_max": 23.66459846496582,
      "activations/layer20_attention_weight_min": -21.030122756958008,
      "activations/layer21_attention_weight_max": 40.20069122314453,
      "activations/layer21_attention_weight_min": -22.16027069091797,
      "activations/layer22_attention_weight_max": 33.19409942626953,
      "activations/layer22_attention_weight_min": -26.041597366333008,
      "activations/layer23_attention_weight_max": 39.68701934814453,
      "activations/layer23_attention_weight_min": -23.645429611206055,
      "activations/layer2_attention_weight_max": 32.66392135620117,
      "activations/layer2_attention_weight_min": -30.47748374938965,
      "activations/layer3_attention_weight_max": 93.3447494506836,
      "activations/layer3_attention_weight_min": -95.30290222167969,
      "activations/layer4_attention_weight_max": 94.2184066772461,
      "activations/layer4_attention_weight_min": -93.75511169433594,
      "activations/layer5_attention_weight_max": 72.67082214355469,
      "activations/layer5_attention_weight_min": -75.59288787841797,
      "activations/layer6_attention_weight_max": 53.4375114440918,
      "activations/layer6_attention_weight_min": -55.81369400024414,
      "activations/layer7_attention_weight_max": 77.3834228515625,
      "activations/layer7_attention_weight_min": -71.1800537109375,
      "activations/layer8_attention_weight_max": 45.2159538269043,
      "activations/layer8_attention_weight_min": -48.306278228759766,
      "activations/layer9_attention_weight_max": 52.78081130981445,
      "activations/layer9_attention_weight_min": -47.35539245605469,
      "epoch": 11.39,
      "learning_rate": 7.733674242424241e-05,
      "loss": 2.7819,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_loss": 2.736328125,
      "eval_runtime": 8.5354,
      "eval_samples_per_second": 503.081,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_openwebtext_loss": 2.736328125,
      "eval_openwebtext_ppl": 15.43022309819552,
      "eval_openwebtext_runtime": 8.5354,
      "eval_openwebtext_samples_per_second": 503.081,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9862,
      "eval_wikitext_samples_per_second": 229.585,
      "step": 196000
    },
    {
      "epoch": 11.39,
      "eval_lambada_loss": 2.56640625,
      "eval_lambada_ppl": 13.018953406185608,
      "eval_lambada_runtime": 9.5636,
      "eval_lambada_samples_per_second": 509.116,
      "step": 196000
    },
    {
      "activations/layer0_attention_weight_max": 15.52834701538086,
      "activations/layer0_attention_weight_min": -13.936572074890137,
      "activations/layer10_attention_weight_max": 35.35933303833008,
      "activations/layer10_attention_weight_min": -37.67506790161133,
      "activations/layer11_attention_weight_max": 31.811246871948242,
      "activations/layer11_attention_weight_min": -33.005924224853516,
      "activations/layer12_attention_weight_max": 18.210081100463867,
      "activations/layer12_attention_weight_min": -26.165470123291016,
      "activations/layer13_attention_weight_max": 39.12220764160156,
      "activations/layer13_attention_weight_min": -29.100500106811523,
      "activations/layer14_attention_weight_max": 38.610103607177734,
      "activations/layer14_attention_weight_min": -31.874271392822266,
      "activations/layer15_attention_weight_max": 35.497589111328125,
      "activations/layer15_attention_weight_min": -29.40483283996582,
      "activations/layer16_attention_weight_max": 33.2237548828125,
      "activations/layer16_attention_weight_min": -29.85689353942871,
      "activations/layer17_attention_weight_max": 53.03835678100586,
      "activations/layer17_attention_weight_min": -43.73398208618164,
      "activations/layer18_attention_weight_max": 47.058631896972656,
      "activations/layer18_attention_weight_min": -39.624122619628906,
      "activations/layer19_attention_weight_max": 21.21051597595215,
      "activations/layer19_attention_weight_min": -24.203964233398438,
      "activations/layer1_attention_weight_max": 17.11919593811035,
      "activations/layer1_attention_weight_min": -14.300894737243652,
      "activations/layer20_attention_weight_max": 20.754756927490234,
      "activations/layer20_attention_weight_min": -21.386320114135742,
      "activations/layer21_attention_weight_max": 36.57277297973633,
      "activations/layer21_attention_weight_min": -27.233882904052734,
      "activations/layer22_attention_weight_max": 35.23230743408203,
      "activations/layer22_attention_weight_min": -28.95346450805664,
      "activations/layer23_attention_weight_max": 36.079872131347656,
      "activations/layer23_attention_weight_min": -25.507125854492188,
      "activations/layer2_attention_weight_max": 32.538394927978516,
      "activations/layer2_attention_weight_min": -31.108966827392578,
      "activations/layer3_attention_weight_max": 89.02163696289062,
      "activations/layer3_attention_weight_min": -89.67670440673828,
      "activations/layer4_attention_weight_max": 97.45446014404297,
      "activations/layer4_attention_weight_min": -92.45850372314453,
      "activations/layer5_attention_weight_max": 72.74263000488281,
      "activations/layer5_attention_weight_min": -75.04163360595703,
      "activations/layer6_attention_weight_max": 51.31102752685547,
      "activations/layer6_attention_weight_min": -56.17500305175781,
      "activations/layer7_attention_weight_max": 72.64921569824219,
      "activations/layer7_attention_weight_min": -74.26177215576172,
      "activations/layer8_attention_weight_max": 44.49953842163086,
      "activations/layer8_attention_weight_min": -47.22223663330078,
      "activations/layer9_attention_weight_max": 44.09577560424805,
      "activations/layer9_attention_weight_min": -46.4140739440918,
      "epoch": 11.39,
      "learning_rate": 7.731780303030303e-05,
      "loss": 2.7888,
      "step": 196050
    },
    {
      "activations/layer0_attention_weight_max": 15.43105697631836,
      "activations/layer0_attention_weight_min": -13.75814437866211,
      "activations/layer10_attention_weight_max": 38.59290313720703,
      "activations/layer10_attention_weight_min": -37.36122512817383,
      "activations/layer11_attention_weight_max": 34.31891632080078,
      "activations/layer11_attention_weight_min": -33.8496208190918,
      "activations/layer12_attention_weight_max": 17.360294342041016,
      "activations/layer12_attention_weight_min": -25.50613784790039,
      "activations/layer13_attention_weight_max": 44.67805862426758,
      "activations/layer13_attention_weight_min": -29.381017684936523,
      "activations/layer14_attention_weight_max": 36.751834869384766,
      "activations/layer14_attention_weight_min": -33.65576171875,
      "activations/layer15_attention_weight_max": 34.204349517822266,
      "activations/layer15_attention_weight_min": -28.854963302612305,
      "activations/layer16_attention_weight_max": 30.85900115966797,
      "activations/layer16_attention_weight_min": -31.29266357421875,
      "activations/layer17_attention_weight_max": 50.540489196777344,
      "activations/layer17_attention_weight_min": -47.08565139770508,
      "activations/layer18_attention_weight_max": 45.76265335083008,
      "activations/layer18_attention_weight_min": -40.4917106628418,
      "activations/layer19_attention_weight_max": 21.612966537475586,
      "activations/layer19_attention_weight_min": -20.71681785583496,
      "activations/layer1_attention_weight_max": 15.655508041381836,
      "activations/layer1_attention_weight_min": -15.375798225402832,
      "activations/layer20_attention_weight_max": 22.16437339782715,
      "activations/layer20_attention_weight_min": -23.70905303955078,
      "activations/layer21_attention_weight_max": 31.909727096557617,
      "activations/layer21_attention_weight_min": -21.15224838256836,
      "activations/layer22_attention_weight_max": 29.850067138671875,
      "activations/layer22_attention_weight_min": -28.14383316040039,
      "activations/layer23_attention_weight_max": 34.18692398071289,
      "activations/layer23_attention_weight_min": -23.91594696044922,
      "activations/layer2_attention_weight_max": 33.680606842041016,
      "activations/layer2_attention_weight_min": -32.95439147949219,
      "activations/layer3_attention_weight_max": 92.78362274169922,
      "activations/layer3_attention_weight_min": -94.07281494140625,
      "activations/layer4_attention_weight_max": 99.7142105102539,
      "activations/layer4_attention_weight_min": -95.68336486816406,
      "activations/layer5_attention_weight_max": 73.1709213256836,
      "activations/layer5_attention_weight_min": -78.08714294433594,
      "activations/layer6_attention_weight_max": 52.450435638427734,
      "activations/layer6_attention_weight_min": -52.51149368286133,
      "activations/layer7_attention_weight_max": 76.80977630615234,
      "activations/layer7_attention_weight_min": -69.78518676757812,
      "activations/layer8_attention_weight_max": 48.111820220947266,
      "activations/layer8_attention_weight_min": -49.41914749145508,
      "activations/layer9_attention_weight_max": 48.64365005493164,
      "activations/layer9_attention_weight_min": -50.59147644042969,
      "epoch": 11.39,
      "learning_rate": 7.729886363636363e-05,
      "loss": 2.7794,
      "step": 196100
    },
    {
      "activations/layer0_attention_weight_max": 14.778292655944824,
      "activations/layer0_attention_weight_min": -14.103790283203125,
      "activations/layer10_attention_weight_max": 36.590396881103516,
      "activations/layer10_attention_weight_min": -35.744503021240234,
      "activations/layer11_attention_weight_max": 31.052000045776367,
      "activations/layer11_attention_weight_min": -31.482330322265625,
      "activations/layer12_attention_weight_max": 19.15945816040039,
      "activations/layer12_attention_weight_min": -25.828235626220703,
      "activations/layer13_attention_weight_max": 35.93522644042969,
      "activations/layer13_attention_weight_min": -27.491580963134766,
      "activations/layer14_attention_weight_max": 40.657772064208984,
      "activations/layer14_attention_weight_min": -29.78885841369629,
      "activations/layer15_attention_weight_max": 31.244821548461914,
      "activations/layer15_attention_weight_min": -28.868114471435547,
      "activations/layer16_attention_weight_max": 28.17729949951172,
      "activations/layer16_attention_weight_min": -29.61115074157715,
      "activations/layer17_attention_weight_max": 47.93532180786133,
      "activations/layer17_attention_weight_min": -42.77830123901367,
      "activations/layer18_attention_weight_max": 42.56841278076172,
      "activations/layer18_attention_weight_min": -37.27643966674805,
      "activations/layer19_attention_weight_max": 22.13168716430664,
      "activations/layer19_attention_weight_min": -20.79855728149414,
      "activations/layer1_attention_weight_max": 17.07411766052246,
      "activations/layer1_attention_weight_min": -15.3287935256958,
      "activations/layer20_attention_weight_max": 22.105545043945312,
      "activations/layer20_attention_weight_min": -22.094528198242188,
      "activations/layer21_attention_weight_max": 34.585906982421875,
      "activations/layer21_attention_weight_min": -22.499841690063477,
      "activations/layer22_attention_weight_max": 28.70677947998047,
      "activations/layer22_attention_weight_min": -28.292301177978516,
      "activations/layer23_attention_weight_max": 37.29853820800781,
      "activations/layer23_attention_weight_min": -24.662391662597656,
      "activations/layer2_attention_weight_max": 31.467552185058594,
      "activations/layer2_attention_weight_min": -29.039030075073242,
      "activations/layer3_attention_weight_max": 89.56083679199219,
      "activations/layer3_attention_weight_min": -92.4859619140625,
      "activations/layer4_attention_weight_max": 95.22786712646484,
      "activations/layer4_attention_weight_min": -95.3707046508789,
      "activations/layer5_attention_weight_max": 70.31587219238281,
      "activations/layer5_attention_weight_min": -85.22825622558594,
      "activations/layer6_attention_weight_max": 54.71648025512695,
      "activations/layer6_attention_weight_min": -52.580623626708984,
      "activations/layer7_attention_weight_max": 73.38776397705078,
      "activations/layer7_attention_weight_min": -73.21459197998047,
      "activations/layer8_attention_weight_max": 46.50884246826172,
      "activations/layer8_attention_weight_min": -49.98942947387695,
      "activations/layer9_attention_weight_max": 46.84745788574219,
      "activations/layer9_attention_weight_min": -46.80781555175781,
      "epoch": 11.4,
      "learning_rate": 7.727992424242423e-05,
      "loss": 2.7667,
      "step": 196150
    },
    {
      "activations/layer0_attention_weight_max": 16.276281356811523,
      "activations/layer0_attention_weight_min": -13.83425521850586,
      "activations/layer10_attention_weight_max": 37.47180938720703,
      "activations/layer10_attention_weight_min": -39.08985900878906,
      "activations/layer11_attention_weight_max": 33.27829360961914,
      "activations/layer11_attention_weight_min": -33.656856536865234,
      "activations/layer12_attention_weight_max": 19.396055221557617,
      "activations/layer12_attention_weight_min": -29.338808059692383,
      "activations/layer13_attention_weight_max": 49.573001861572266,
      "activations/layer13_attention_weight_min": -33.835540771484375,
      "activations/layer14_attention_weight_max": 46.07923889160156,
      "activations/layer14_attention_weight_min": -29.268693923950195,
      "activations/layer15_attention_weight_max": 41.627445220947266,
      "activations/layer15_attention_weight_min": -29.539804458618164,
      "activations/layer16_attention_weight_max": 32.463050842285156,
      "activations/layer16_attention_weight_min": -29.111431121826172,
      "activations/layer17_attention_weight_max": 54.98443603515625,
      "activations/layer17_attention_weight_min": -42.46148681640625,
      "activations/layer18_attention_weight_max": 50.123905181884766,
      "activations/layer18_attention_weight_min": -35.73117446899414,
      "activations/layer19_attention_weight_max": 21.366689682006836,
      "activations/layer19_attention_weight_min": -21.282257080078125,
      "activations/layer1_attention_weight_max": 17.03113555908203,
      "activations/layer1_attention_weight_min": -13.533071517944336,
      "activations/layer20_attention_weight_max": 23.425804138183594,
      "activations/layer20_attention_weight_min": -21.133634567260742,
      "activations/layer21_attention_weight_max": 39.493343353271484,
      "activations/layer21_attention_weight_min": -22.826168060302734,
      "activations/layer22_attention_weight_max": 32.50210952758789,
      "activations/layer22_attention_weight_min": -25.6262264251709,
      "activations/layer23_attention_weight_max": 36.69939422607422,
      "activations/layer23_attention_weight_min": -25.04224967956543,
      "activations/layer2_attention_weight_max": 32.05259323120117,
      "activations/layer2_attention_weight_min": -29.499664306640625,
      "activations/layer3_attention_weight_max": 89.10836029052734,
      "activations/layer3_attention_weight_min": -87.95592498779297,
      "activations/layer4_attention_weight_max": 95.38827514648438,
      "activations/layer4_attention_weight_min": -92.15219116210938,
      "activations/layer5_attention_weight_max": 69.69125366210938,
      "activations/layer5_attention_weight_min": -79.1384048461914,
      "activations/layer6_attention_weight_max": 49.24380874633789,
      "activations/layer6_attention_weight_min": -51.17020797729492,
      "activations/layer7_attention_weight_max": 75.60980987548828,
      "activations/layer7_attention_weight_min": -72.00756072998047,
      "activations/layer8_attention_weight_max": 44.59016799926758,
      "activations/layer8_attention_weight_min": -46.66474151611328,
      "activations/layer9_attention_weight_max": 54.57868194580078,
      "activations/layer9_attention_weight_min": -52.76963424682617,
      "epoch": 11.4,
      "learning_rate": 7.726098484848484e-05,
      "loss": 2.7873,
      "step": 196200
    },
    {
      "activations/layer0_attention_weight_max": 16.61809539794922,
      "activations/layer0_attention_weight_min": -14.00619888305664,
      "activations/layer10_attention_weight_max": 41.028568267822266,
      "activations/layer10_attention_weight_min": -37.83528518676758,
      "activations/layer11_attention_weight_max": 35.27241516113281,
      "activations/layer11_attention_weight_min": -34.21778106689453,
      "activations/layer12_attention_weight_max": 18.12015151977539,
      "activations/layer12_attention_weight_min": -27.180850982666016,
      "activations/layer13_attention_weight_max": 38.62532424926758,
      "activations/layer13_attention_weight_min": -33.589927673339844,
      "activations/layer14_attention_weight_max": 35.74787521362305,
      "activations/layer14_attention_weight_min": -33.1707649230957,
      "activations/layer15_attention_weight_max": 33.28571319580078,
      "activations/layer15_attention_weight_min": -30.799560546875,
      "activations/layer16_attention_weight_max": 32.36249542236328,
      "activations/layer16_attention_weight_min": -30.791065216064453,
      "activations/layer17_attention_weight_max": 51.194541931152344,
      "activations/layer17_attention_weight_min": -47.27703857421875,
      "activations/layer18_attention_weight_max": 44.19097137451172,
      "activations/layer18_attention_weight_min": -39.57305145263672,
      "activations/layer19_attention_weight_max": 20.497785568237305,
      "activations/layer19_attention_weight_min": -22.146921157836914,
      "activations/layer1_attention_weight_max": 16.67586326599121,
      "activations/layer1_attention_weight_min": -15.766179084777832,
      "activations/layer20_attention_weight_max": 21.81231689453125,
      "activations/layer20_attention_weight_min": -24.46450424194336,
      "activations/layer21_attention_weight_max": 43.5516357421875,
      "activations/layer21_attention_weight_min": -26.125247955322266,
      "activations/layer22_attention_weight_max": 30.477462768554688,
      "activations/layer22_attention_weight_min": -27.627546310424805,
      "activations/layer23_attention_weight_max": 39.62334442138672,
      "activations/layer23_attention_weight_min": -24.35480499267578,
      "activations/layer2_attention_weight_max": 32.988033294677734,
      "activations/layer2_attention_weight_min": -33.5672607421875,
      "activations/layer3_attention_weight_max": 94.8000717163086,
      "activations/layer3_attention_weight_min": -96.71121978759766,
      "activations/layer4_attention_weight_max": 101.81319427490234,
      "activations/layer4_attention_weight_min": -96.32855224609375,
      "activations/layer5_attention_weight_max": 73.93229675292969,
      "activations/layer5_attention_weight_min": -79.51702880859375,
      "activations/layer6_attention_weight_max": 54.95436477661133,
      "activations/layer6_attention_weight_min": -54.82206726074219,
      "activations/layer7_attention_weight_max": 76.4853744506836,
      "activations/layer7_attention_weight_min": -76.17619323730469,
      "activations/layer8_attention_weight_max": 46.77322769165039,
      "activations/layer8_attention_weight_min": -49.384090423583984,
      "activations/layer9_attention_weight_max": 49.35916519165039,
      "activations/layer9_attention_weight_min": -53.2149658203125,
      "epoch": 11.4,
      "learning_rate": 7.724204545454546e-05,
      "loss": 2.7673,
      "step": 196250
    },
    {
      "activations/layer0_attention_weight_max": 15.50048828125,
      "activations/layer0_attention_weight_min": -14.31216049194336,
      "activations/layer10_attention_weight_max": 34.71138381958008,
      "activations/layer10_attention_weight_min": -35.65685272216797,
      "activations/layer11_attention_weight_max": 30.277257919311523,
      "activations/layer11_attention_weight_min": -32.48895263671875,
      "activations/layer12_attention_weight_max": 18.395811080932617,
      "activations/layer12_attention_weight_min": -28.437410354614258,
      "activations/layer13_attention_weight_max": 33.125572204589844,
      "activations/layer13_attention_weight_min": -28.12486457824707,
      "activations/layer14_attention_weight_max": 33.306922912597656,
      "activations/layer14_attention_weight_min": -28.297517776489258,
      "activations/layer15_attention_weight_max": 30.026826858520508,
      "activations/layer15_attention_weight_min": -29.137374877929688,
      "activations/layer16_attention_weight_max": 31.819210052490234,
      "activations/layer16_attention_weight_min": -30.79854393005371,
      "activations/layer17_attention_weight_max": 50.282066345214844,
      "activations/layer17_attention_weight_min": -44.040794372558594,
      "activations/layer18_attention_weight_max": 43.204105377197266,
      "activations/layer18_attention_weight_min": -38.103179931640625,
      "activations/layer19_attention_weight_max": 21.569900512695312,
      "activations/layer19_attention_weight_min": -22.505918502807617,
      "activations/layer1_attention_weight_max": 16.38243865966797,
      "activations/layer1_attention_weight_min": -15.158585548400879,
      "activations/layer20_attention_weight_max": 22.142723083496094,
      "activations/layer20_attention_weight_min": -21.484281539916992,
      "activations/layer21_attention_weight_max": 34.1136360168457,
      "activations/layer21_attention_weight_min": -27.40736961364746,
      "activations/layer22_attention_weight_max": 32.00825119018555,
      "activations/layer22_attention_weight_min": -27.14109992980957,
      "activations/layer23_attention_weight_max": 35.184566497802734,
      "activations/layer23_attention_weight_min": -24.531696319580078,
      "activations/layer2_attention_weight_max": 34.501625061035156,
      "activations/layer2_attention_weight_min": -31.35462188720703,
      "activations/layer3_attention_weight_max": 90.43340301513672,
      "activations/layer3_attention_weight_min": -94.69066619873047,
      "activations/layer4_attention_weight_max": 96.6185073852539,
      "activations/layer4_attention_weight_min": -94.3714599609375,
      "activations/layer5_attention_weight_max": 75.27494049072266,
      "activations/layer5_attention_weight_min": -75.53743743896484,
      "activations/layer6_attention_weight_max": 53.77071762084961,
      "activations/layer6_attention_weight_min": -54.57160949707031,
      "activations/layer7_attention_weight_max": 71.96490478515625,
      "activations/layer7_attention_weight_min": -70.42947387695312,
      "activations/layer8_attention_weight_max": 45.107181549072266,
      "activations/layer8_attention_weight_min": -49.20683288574219,
      "activations/layer9_attention_weight_max": 46.743385314941406,
      "activations/layer9_attention_weight_min": -45.67476272583008,
      "epoch": 11.41,
      "learning_rate": 7.722310606060605e-05,
      "loss": 2.7845,
      "step": 196300
    },
    {
      "activations/layer0_attention_weight_max": 15.450640678405762,
      "activations/layer0_attention_weight_min": -14.056395530700684,
      "activations/layer10_attention_weight_max": 39.82188415527344,
      "activations/layer10_attention_weight_min": -36.23259353637695,
      "activations/layer11_attention_weight_max": 33.17269515991211,
      "activations/layer11_attention_weight_min": -30.63778305053711,
      "activations/layer12_attention_weight_max": 18.467697143554688,
      "activations/layer12_attention_weight_min": -27.349870681762695,
      "activations/layer13_attention_weight_max": 62.20042037963867,
      "activations/layer13_attention_weight_min": -34.47652816772461,
      "activations/layer14_attention_weight_max": 62.439857482910156,
      "activations/layer14_attention_weight_min": -33.96684646606445,
      "activations/layer15_attention_weight_max": 54.785369873046875,
      "activations/layer15_attention_weight_min": -32.51969909667969,
      "activations/layer16_attention_weight_max": 47.52688217163086,
      "activations/layer16_attention_weight_min": -31.367631912231445,
      "activations/layer17_attention_weight_max": 69.02422332763672,
      "activations/layer17_attention_weight_min": -52.98368453979492,
      "activations/layer18_attention_weight_max": 65.65892028808594,
      "activations/layer18_attention_weight_min": -46.14574432373047,
      "activations/layer19_attention_weight_max": 27.938867568969727,
      "activations/layer19_attention_weight_min": -26.07307243347168,
      "activations/layer1_attention_weight_max": 18.21721076965332,
      "activations/layer1_attention_weight_min": -15.483527183532715,
      "activations/layer20_attention_weight_max": 31.492401123046875,
      "activations/layer20_attention_weight_min": -26.14276123046875,
      "activations/layer21_attention_weight_max": 62.46601486206055,
      "activations/layer21_attention_weight_min": -24.6826229095459,
      "activations/layer22_attention_weight_max": 39.146785736083984,
      "activations/layer22_attention_weight_min": -26.861421585083008,
      "activations/layer23_attention_weight_max": 47.10761642456055,
      "activations/layer23_attention_weight_min": -25.09852409362793,
      "activations/layer2_attention_weight_max": 31.375904083251953,
      "activations/layer2_attention_weight_min": -28.870031356811523,
      "activations/layer3_attention_weight_max": 90.90847778320312,
      "activations/layer3_attention_weight_min": -101.14712524414062,
      "activations/layer4_attention_weight_max": 97.29517364501953,
      "activations/layer4_attention_weight_min": -91.17919921875,
      "activations/layer5_attention_weight_max": 75.04051208496094,
      "activations/layer5_attention_weight_min": -76.39379119873047,
      "activations/layer6_attention_weight_max": 50.74541091918945,
      "activations/layer6_attention_weight_min": -49.448848724365234,
      "activations/layer7_attention_weight_max": 72.54732513427734,
      "activations/layer7_attention_weight_min": -71.52342987060547,
      "activations/layer8_attention_weight_max": 45.4156494140625,
      "activations/layer8_attention_weight_min": -46.675537109375,
      "activations/layer9_attention_weight_max": 48.984535217285156,
      "activations/layer9_attention_weight_min": -44.29432678222656,
      "epoch": 11.41,
      "learning_rate": 7.720416666666666e-05,
      "loss": 2.7669,
      "step": 196350
    },
    {
      "activations/layer0_attention_weight_max": 15.274969100952148,
      "activations/layer0_attention_weight_min": -13.87343978881836,
      "activations/layer10_attention_weight_max": 36.203887939453125,
      "activations/layer10_attention_weight_min": -36.87846755981445,
      "activations/layer11_attention_weight_max": 31.701229095458984,
      "activations/layer11_attention_weight_min": -33.369049072265625,
      "activations/layer12_attention_weight_max": 18.421249389648438,
      "activations/layer12_attention_weight_min": -36.079063415527344,
      "activations/layer13_attention_weight_max": 38.38056182861328,
      "activations/layer13_attention_weight_min": -30.109071731567383,
      "activations/layer14_attention_weight_max": 38.259403228759766,
      "activations/layer14_attention_weight_min": -34.78097152709961,
      "activations/layer15_attention_weight_max": 32.752376556396484,
      "activations/layer15_attention_weight_min": -30.722248077392578,
      "activations/layer16_attention_weight_max": 30.533451080322266,
      "activations/layer16_attention_weight_min": -30.67976951599121,
      "activations/layer17_attention_weight_max": 52.76089096069336,
      "activations/layer17_attention_weight_min": -48.3206672668457,
      "activations/layer18_attention_weight_max": 46.97411346435547,
      "activations/layer18_attention_weight_min": -39.94835662841797,
      "activations/layer19_attention_weight_max": 21.767911911010742,
      "activations/layer19_attention_weight_min": -22.402137756347656,
      "activations/layer1_attention_weight_max": 17.60698890686035,
      "activations/layer1_attention_weight_min": -14.297872543334961,
      "activations/layer20_attention_weight_max": 19.78114128112793,
      "activations/layer20_attention_weight_min": -20.93478775024414,
      "activations/layer21_attention_weight_max": 34.49348068237305,
      "activations/layer21_attention_weight_min": -22.98918914794922,
      "activations/layer22_attention_weight_max": 27.627777099609375,
      "activations/layer22_attention_weight_min": -25.539888381958008,
      "activations/layer23_attention_weight_max": 33.097328186035156,
      "activations/layer23_attention_weight_min": -22.9732666015625,
      "activations/layer2_attention_weight_max": 32.471317291259766,
      "activations/layer2_attention_weight_min": -29.785511016845703,
      "activations/layer3_attention_weight_max": 95.62928771972656,
      "activations/layer3_attention_weight_min": -97.19097137451172,
      "activations/layer4_attention_weight_max": 98.8313980102539,
      "activations/layer4_attention_weight_min": -96.07546997070312,
      "activations/layer5_attention_weight_max": 75.328369140625,
      "activations/layer5_attention_weight_min": -79.05776977539062,
      "activations/layer6_attention_weight_max": 53.86238479614258,
      "activations/layer6_attention_weight_min": -53.03278350830078,
      "activations/layer7_attention_weight_max": 76.15192413330078,
      "activations/layer7_attention_weight_min": -77.40716552734375,
      "activations/layer8_attention_weight_max": 45.812564849853516,
      "activations/layer8_attention_weight_min": -45.36522674560547,
      "activations/layer9_attention_weight_max": 51.808712005615234,
      "activations/layer9_attention_weight_min": -49.914710998535156,
      "epoch": 11.41,
      "learning_rate": 7.718522727272728e-05,
      "loss": 2.7713,
      "step": 196400
    },
    {
      "activations/layer0_attention_weight_max": 16.126497268676758,
      "activations/layer0_attention_weight_min": -14.270785331726074,
      "activations/layer10_attention_weight_max": 38.802734375,
      "activations/layer10_attention_weight_min": -38.85869216918945,
      "activations/layer11_attention_weight_max": 35.8239860534668,
      "activations/layer11_attention_weight_min": -33.92814636230469,
      "activations/layer12_attention_weight_max": 18.260700225830078,
      "activations/layer12_attention_weight_min": -27.185531616210938,
      "activations/layer13_attention_weight_max": 40.914546966552734,
      "activations/layer13_attention_weight_min": -26.344514846801758,
      "activations/layer14_attention_weight_max": 36.24979782104492,
      "activations/layer14_attention_weight_min": -30.058046340942383,
      "activations/layer15_attention_weight_max": 31.530160903930664,
      "activations/layer15_attention_weight_min": -27.689708709716797,
      "activations/layer16_attention_weight_max": 29.991046905517578,
      "activations/layer16_attention_weight_min": -27.816808700561523,
      "activations/layer17_attention_weight_max": 51.687339782714844,
      "activations/layer17_attention_weight_min": -42.20665740966797,
      "activations/layer18_attention_weight_max": 43.028221130371094,
      "activations/layer18_attention_weight_min": -39.60139083862305,
      "activations/layer19_attention_weight_max": 21.506309509277344,
      "activations/layer19_attention_weight_min": -22.880475997924805,
      "activations/layer1_attention_weight_max": 16.231595993041992,
      "activations/layer1_attention_weight_min": -14.890096664428711,
      "activations/layer20_attention_weight_max": 22.05583953857422,
      "activations/layer20_attention_weight_min": -22.105195999145508,
      "activations/layer21_attention_weight_max": 33.2360725402832,
      "activations/layer21_attention_weight_min": -24.96067237854004,
      "activations/layer22_attention_weight_max": 28.369831085205078,
      "activations/layer22_attention_weight_min": -28.966632843017578,
      "activations/layer23_attention_weight_max": 34.06230926513672,
      "activations/layer23_attention_weight_min": -25.26742935180664,
      "activations/layer2_attention_weight_max": 36.769779205322266,
      "activations/layer2_attention_weight_min": -33.329017639160156,
      "activations/layer3_attention_weight_max": 104.36585235595703,
      "activations/layer3_attention_weight_min": -99.77491760253906,
      "activations/layer4_attention_weight_max": 112.40152740478516,
      "activations/layer4_attention_weight_min": -96.39131164550781,
      "activations/layer5_attention_weight_max": 80.3465576171875,
      "activations/layer5_attention_weight_min": -75.00115966796875,
      "activations/layer6_attention_weight_max": 56.97636413574219,
      "activations/layer6_attention_weight_min": -53.91804504394531,
      "activations/layer7_attention_weight_max": 80.34491729736328,
      "activations/layer7_attention_weight_min": -70.63868713378906,
      "activations/layer8_attention_weight_max": 45.40967559814453,
      "activations/layer8_attention_weight_min": -48.448307037353516,
      "activations/layer9_attention_weight_max": 56.934234619140625,
      "activations/layer9_attention_weight_min": -49.32175064086914,
      "epoch": 11.41,
      "learning_rate": 7.716628787878786e-05,
      "loss": 2.7863,
      "step": 196450
    },
    {
      "activations/layer0_attention_weight_max": 15.284894943237305,
      "activations/layer0_attention_weight_min": -14.05390453338623,
      "activations/layer10_attention_weight_max": 38.71527099609375,
      "activations/layer10_attention_weight_min": -37.24192428588867,
      "activations/layer11_attention_weight_max": 35.995147705078125,
      "activations/layer11_attention_weight_min": -34.229530334472656,
      "activations/layer12_attention_weight_max": 19.87544822692871,
      "activations/layer12_attention_weight_min": -23.15400505065918,
      "activations/layer13_attention_weight_max": 39.52662658691406,
      "activations/layer13_attention_weight_min": -30.378225326538086,
      "activations/layer14_attention_weight_max": 39.77040481567383,
      "activations/layer14_attention_weight_min": -30.169872283935547,
      "activations/layer15_attention_weight_max": 35.61673355102539,
      "activations/layer15_attention_weight_min": -28.354480743408203,
      "activations/layer16_attention_weight_max": 33.64406967163086,
      "activations/layer16_attention_weight_min": -30.769208908081055,
      "activations/layer17_attention_weight_max": 50.5694694519043,
      "activations/layer17_attention_weight_min": -44.46345901489258,
      "activations/layer18_attention_weight_max": 47.677284240722656,
      "activations/layer18_attention_weight_min": -35.06831741333008,
      "activations/layer19_attention_weight_max": 20.122255325317383,
      "activations/layer19_attention_weight_min": -20.87738037109375,
      "activations/layer1_attention_weight_max": 16.5445499420166,
      "activations/layer1_attention_weight_min": -14.594099044799805,
      "activations/layer20_attention_weight_max": 21.526134490966797,
      "activations/layer20_attention_weight_min": -20.425704956054688,
      "activations/layer21_attention_weight_max": 39.15895080566406,
      "activations/layer21_attention_weight_min": -25.004459381103516,
      "activations/layer22_attention_weight_max": 30.565216064453125,
      "activations/layer22_attention_weight_min": -26.227991104125977,
      "activations/layer23_attention_weight_max": 35.871009826660156,
      "activations/layer23_attention_weight_min": -22.957874298095703,
      "activations/layer2_attention_weight_max": 31.16482162475586,
      "activations/layer2_attention_weight_min": -32.70848083496094,
      "activations/layer3_attention_weight_max": 98.01309204101562,
      "activations/layer3_attention_weight_min": -103.56553649902344,
      "activations/layer4_attention_weight_max": 102.84825897216797,
      "activations/layer4_attention_weight_min": -101.96611022949219,
      "activations/layer5_attention_weight_max": 76.22846984863281,
      "activations/layer5_attention_weight_min": -85.10214233398438,
      "activations/layer6_attention_weight_max": 55.09966278076172,
      "activations/layer6_attention_weight_min": -55.26802062988281,
      "activations/layer7_attention_weight_max": 79.61553955078125,
      "activations/layer7_attention_weight_min": -76.01717376708984,
      "activations/layer8_attention_weight_max": 48.218326568603516,
      "activations/layer8_attention_weight_min": -49.357810974121094,
      "activations/layer9_attention_weight_max": 55.45766830444336,
      "activations/layer9_attention_weight_min": -48.17536926269531,
      "epoch": 11.42,
      "learning_rate": 7.714734848484848e-05,
      "loss": 2.7745,
      "step": 196500
    },
    {
      "activations/layer0_attention_weight_max": 15.566493034362793,
      "activations/layer0_attention_weight_min": -14.0509672164917,
      "activations/layer10_attention_weight_max": 36.598182678222656,
      "activations/layer10_attention_weight_min": -35.47270965576172,
      "activations/layer11_attention_weight_max": 32.18316650390625,
      "activations/layer11_attention_weight_min": -33.369842529296875,
      "activations/layer12_attention_weight_max": 18.83216667175293,
      "activations/layer12_attention_weight_min": -26.975439071655273,
      "activations/layer13_attention_weight_max": 35.41100311279297,
      "activations/layer13_attention_weight_min": -31.965057373046875,
      "activations/layer14_attention_weight_max": 34.057777404785156,
      "activations/layer14_attention_weight_min": -28.937997817993164,
      "activations/layer15_attention_weight_max": 30.533878326416016,
      "activations/layer15_attention_weight_min": -28.047922134399414,
      "activations/layer16_attention_weight_max": 27.878955841064453,
      "activations/layer16_attention_weight_min": -28.415483474731445,
      "activations/layer17_attention_weight_max": 50.174495697021484,
      "activations/layer17_attention_weight_min": -41.02397155761719,
      "activations/layer18_attention_weight_max": 43.15043258666992,
      "activations/layer18_attention_weight_min": -35.19076919555664,
      "activations/layer19_attention_weight_max": 20.945119857788086,
      "activations/layer19_attention_weight_min": -20.686880111694336,
      "activations/layer1_attention_weight_max": 16.274120330810547,
      "activations/layer1_attention_weight_min": -14.080309867858887,
      "activations/layer20_attention_weight_max": 20.324810028076172,
      "activations/layer20_attention_weight_min": -26.894079208374023,
      "activations/layer21_attention_weight_max": 36.82855224609375,
      "activations/layer21_attention_weight_min": -22.22517967224121,
      "activations/layer22_attention_weight_max": 28.682254791259766,
      "activations/layer22_attention_weight_min": -23.985027313232422,
      "activations/layer23_attention_weight_max": 34.51121520996094,
      "activations/layer23_attention_weight_min": -24.926610946655273,
      "activations/layer2_attention_weight_max": 32.41194152832031,
      "activations/layer2_attention_weight_min": -29.95197296142578,
      "activations/layer3_attention_weight_max": 92.78012084960938,
      "activations/layer3_attention_weight_min": -92.07054901123047,
      "activations/layer4_attention_weight_max": 98.87139129638672,
      "activations/layer4_attention_weight_min": -95.66942596435547,
      "activations/layer5_attention_weight_max": 72.00996398925781,
      "activations/layer5_attention_weight_min": -73.79764556884766,
      "activations/layer6_attention_weight_max": 52.867469787597656,
      "activations/layer6_attention_weight_min": -51.16789627075195,
      "activations/layer7_attention_weight_max": 72.11455535888672,
      "activations/layer7_attention_weight_min": -69.213134765625,
      "activations/layer8_attention_weight_max": 45.0052490234375,
      "activations/layer8_attention_weight_min": -46.03812789916992,
      "activations/layer9_attention_weight_max": 48.27130889892578,
      "activations/layer9_attention_weight_min": -45.73121643066406,
      "epoch": 11.42,
      "learning_rate": 7.712840909090908e-05,
      "loss": 2.8018,
      "step": 196550
    },
    {
      "activations/layer0_attention_weight_max": 15.563699722290039,
      "activations/layer0_attention_weight_min": -14.543285369873047,
      "activations/layer10_attention_weight_max": 42.1334228515625,
      "activations/layer10_attention_weight_min": -40.828250885009766,
      "activations/layer11_attention_weight_max": 35.324459075927734,
      "activations/layer11_attention_weight_min": -35.06733703613281,
      "activations/layer12_attention_weight_max": 23.3497371673584,
      "activations/layer12_attention_weight_min": -27.050918579101562,
      "activations/layer13_attention_weight_max": 38.35228729248047,
      "activations/layer13_attention_weight_min": -28.2414493560791,
      "activations/layer14_attention_weight_max": 35.59792709350586,
      "activations/layer14_attention_weight_min": -30.58673667907715,
      "activations/layer15_attention_weight_max": 33.61924362182617,
      "activations/layer15_attention_weight_min": -30.326658248901367,
      "activations/layer16_attention_weight_max": 31.189180374145508,
      "activations/layer16_attention_weight_min": -30.983047485351562,
      "activations/layer17_attention_weight_max": 55.95878601074219,
      "activations/layer17_attention_weight_min": -48.66780090332031,
      "activations/layer18_attention_weight_max": 46.85660171508789,
      "activations/layer18_attention_weight_min": -41.66624450683594,
      "activations/layer19_attention_weight_max": 26.359375,
      "activations/layer19_attention_weight_min": -23.55129051208496,
      "activations/layer1_attention_weight_max": 16.51020050048828,
      "activations/layer1_attention_weight_min": -16.527963638305664,
      "activations/layer20_attention_weight_max": 22.007431030273438,
      "activations/layer20_attention_weight_min": -21.896265029907227,
      "activations/layer21_attention_weight_max": 40.9843635559082,
      "activations/layer21_attention_weight_min": -23.481164932250977,
      "activations/layer22_attention_weight_max": 38.197818756103516,
      "activations/layer22_attention_weight_min": -27.99884605407715,
      "activations/layer23_attention_weight_max": 37.29065704345703,
      "activations/layer23_attention_weight_min": -22.92095184326172,
      "activations/layer2_attention_weight_max": 33.39450454711914,
      "activations/layer2_attention_weight_min": -32.47401428222656,
      "activations/layer3_attention_weight_max": 97.59330749511719,
      "activations/layer3_attention_weight_min": -103.66190338134766,
      "activations/layer4_attention_weight_max": 98.48841857910156,
      "activations/layer4_attention_weight_min": -98.4774398803711,
      "activations/layer5_attention_weight_max": 73.15182495117188,
      "activations/layer5_attention_weight_min": -80.2207260131836,
      "activations/layer6_attention_weight_max": 55.322330474853516,
      "activations/layer6_attention_weight_min": -55.80409622192383,
      "activations/layer7_attention_weight_max": 76.55030059814453,
      "activations/layer7_attention_weight_min": -78.37995147705078,
      "activations/layer8_attention_weight_max": 50.79716491699219,
      "activations/layer8_attention_weight_min": -52.39235305786133,
      "activations/layer9_attention_weight_max": 51.62550735473633,
      "activations/layer9_attention_weight_min": -55.34102249145508,
      "epoch": 11.42,
      "learning_rate": 7.710946969696968e-05,
      "loss": 2.7664,
      "step": 196600
    },
    {
      "activations/layer0_attention_weight_max": 15.180146217346191,
      "activations/layer0_attention_weight_min": -14.1187105178833,
      "activations/layer10_attention_weight_max": 39.4664192199707,
      "activations/layer10_attention_weight_min": -37.09904861450195,
      "activations/layer11_attention_weight_max": 33.93312072753906,
      "activations/layer11_attention_weight_min": -35.568199157714844,
      "activations/layer12_attention_weight_max": 25.17399787902832,
      "activations/layer12_attention_weight_min": -25.13946533203125,
      "activations/layer13_attention_weight_max": 44.523948669433594,
      "activations/layer13_attention_weight_min": -30.695470809936523,
      "activations/layer14_attention_weight_max": 43.09927749633789,
      "activations/layer14_attention_weight_min": -31.873308181762695,
      "activations/layer15_attention_weight_max": 38.424232482910156,
      "activations/layer15_attention_weight_min": -30.178617477416992,
      "activations/layer16_attention_weight_max": 39.20182418823242,
      "activations/layer16_attention_weight_min": -33.73236846923828,
      "activations/layer17_attention_weight_max": 65.99811553955078,
      "activations/layer17_attention_weight_min": -52.09092330932617,
      "activations/layer18_attention_weight_max": 63.47465515136719,
      "activations/layer18_attention_weight_min": -45.61383056640625,
      "activations/layer19_attention_weight_max": 27.42413902282715,
      "activations/layer19_attention_weight_min": -25.322385787963867,
      "activations/layer1_attention_weight_max": 17.849515914916992,
      "activations/layer1_attention_weight_min": -14.479647636413574,
      "activations/layer20_attention_weight_max": 29.355117797851562,
      "activations/layer20_attention_weight_min": -23.200273513793945,
      "activations/layer21_attention_weight_max": 43.22623825073242,
      "activations/layer21_attention_weight_min": -24.178945541381836,
      "activations/layer22_attention_weight_max": 34.117122650146484,
      "activations/layer22_attention_weight_min": -24.72380256652832,
      "activations/layer23_attention_weight_max": 41.817108154296875,
      "activations/layer23_attention_weight_min": -24.45831871032715,
      "activations/layer2_attention_weight_max": 30.79461669921875,
      "activations/layer2_attention_weight_min": -30.90272331237793,
      "activations/layer3_attention_weight_max": 92.14543914794922,
      "activations/layer3_attention_weight_min": -102.13691711425781,
      "activations/layer4_attention_weight_max": 103.17048645019531,
      "activations/layer4_attention_weight_min": -101.33003997802734,
      "activations/layer5_attention_weight_max": 74.25979614257812,
      "activations/layer5_attention_weight_min": -80.672607421875,
      "activations/layer6_attention_weight_max": 57.37516784667969,
      "activations/layer6_attention_weight_min": -53.06269454956055,
      "activations/layer7_attention_weight_max": 84.9411392211914,
      "activations/layer7_attention_weight_min": -74.1575927734375,
      "activations/layer8_attention_weight_max": 46.2244758605957,
      "activations/layer8_attention_weight_min": -48.675411224365234,
      "activations/layer9_attention_weight_max": 48.1320915222168,
      "activations/layer9_attention_weight_min": -49.466400146484375,
      "epoch": 11.43,
      "learning_rate": 7.70905303030303e-05,
      "loss": 2.7916,
      "step": 196650
    },
    {
      "activations/layer0_attention_weight_max": 14.805480003356934,
      "activations/layer0_attention_weight_min": -14.149319648742676,
      "activations/layer10_attention_weight_max": 38.670894622802734,
      "activations/layer10_attention_weight_min": -36.609344482421875,
      "activations/layer11_attention_weight_max": 32.876731872558594,
      "activations/layer11_attention_weight_min": -33.462181091308594,
      "activations/layer12_attention_weight_max": 19.654367446899414,
      "activations/layer12_attention_weight_min": -23.520002365112305,
      "activations/layer13_attention_weight_max": 42.6318473815918,
      "activations/layer13_attention_weight_min": -30.617345809936523,
      "activations/layer14_attention_weight_max": 40.073238372802734,
      "activations/layer14_attention_weight_min": -29.482202529907227,
      "activations/layer15_attention_weight_max": 36.00664138793945,
      "activations/layer15_attention_weight_min": -28.13197898864746,
      "activations/layer16_attention_weight_max": 36.91446304321289,
      "activations/layer16_attention_weight_min": -29.724565505981445,
      "activations/layer17_attention_weight_max": 54.25880432128906,
      "activations/layer17_attention_weight_min": -43.59730911254883,
      "activations/layer18_attention_weight_max": 49.59889221191406,
      "activations/layer18_attention_weight_min": -36.843257904052734,
      "activations/layer19_attention_weight_max": 22.667226791381836,
      "activations/layer19_attention_weight_min": -18.914308547973633,
      "activations/layer1_attention_weight_max": 16.141897201538086,
      "activations/layer1_attention_weight_min": -14.3129301071167,
      "activations/layer20_attention_weight_max": 21.295658111572266,
      "activations/layer20_attention_weight_min": -19.212356567382812,
      "activations/layer21_attention_weight_max": 35.58176040649414,
      "activations/layer21_attention_weight_min": -23.93559455871582,
      "activations/layer22_attention_weight_max": 30.28099250793457,
      "activations/layer22_attention_weight_min": -23.64324951171875,
      "activations/layer23_attention_weight_max": 40.02326965332031,
      "activations/layer23_attention_weight_min": -27.07254409790039,
      "activations/layer2_attention_weight_max": 32.10743713378906,
      "activations/layer2_attention_weight_min": -30.52983856201172,
      "activations/layer3_attention_weight_max": 96.14424133300781,
      "activations/layer3_attention_weight_min": -93.40567016601562,
      "activations/layer4_attention_weight_max": 103.77172088623047,
      "activations/layer4_attention_weight_min": -96.31230163574219,
      "activations/layer5_attention_weight_max": 74.24185180664062,
      "activations/layer5_attention_weight_min": -79.13778686523438,
      "activations/layer6_attention_weight_max": 55.19572830200195,
      "activations/layer6_attention_weight_min": -53.5100212097168,
      "activations/layer7_attention_weight_max": 76.0780029296875,
      "activations/layer7_attention_weight_min": -75.32332611083984,
      "activations/layer8_attention_weight_max": 46.5128173828125,
      "activations/layer8_attention_weight_min": -47.190162658691406,
      "activations/layer9_attention_weight_max": 49.74643325805664,
      "activations/layer9_attention_weight_min": -50.06124496459961,
      "epoch": 11.43,
      "learning_rate": 7.70715909090909e-05,
      "loss": 2.7928,
      "step": 196700
    },
    {
      "activations/layer0_attention_weight_max": 15.248485565185547,
      "activations/layer0_attention_weight_min": -14.404340744018555,
      "activations/layer10_attention_weight_max": 35.61367416381836,
      "activations/layer10_attention_weight_min": -34.0589485168457,
      "activations/layer11_attention_weight_max": 31.52442741394043,
      "activations/layer11_attention_weight_min": -32.06138610839844,
      "activations/layer12_attention_weight_max": 18.049345016479492,
      "activations/layer12_attention_weight_min": -28.107309341430664,
      "activations/layer13_attention_weight_max": 36.64215850830078,
      "activations/layer13_attention_weight_min": -27.93929672241211,
      "activations/layer14_attention_weight_max": 39.82529067993164,
      "activations/layer14_attention_weight_min": -30.300174713134766,
      "activations/layer15_attention_weight_max": 33.425537109375,
      "activations/layer15_attention_weight_min": -29.158851623535156,
      "activations/layer16_attention_weight_max": 32.76193618774414,
      "activations/layer16_attention_weight_min": -30.015960693359375,
      "activations/layer17_attention_weight_max": 52.379554748535156,
      "activations/layer17_attention_weight_min": -43.67675018310547,
      "activations/layer18_attention_weight_max": 48.557579040527344,
      "activations/layer18_attention_weight_min": -39.565494537353516,
      "activations/layer19_attention_weight_max": 23.989484786987305,
      "activations/layer19_attention_weight_min": -21.713735580444336,
      "activations/layer1_attention_weight_max": 17.04974937438965,
      "activations/layer1_attention_weight_min": -15.494470596313477,
      "activations/layer20_attention_weight_max": 22.622211456298828,
      "activations/layer20_attention_weight_min": -20.83333396911621,
      "activations/layer21_attention_weight_max": 35.73474884033203,
      "activations/layer21_attention_weight_min": -23.58274269104004,
      "activations/layer22_attention_weight_max": 32.5923957824707,
      "activations/layer22_attention_weight_min": -25.819435119628906,
      "activations/layer23_attention_weight_max": 35.8094367980957,
      "activations/layer23_attention_weight_min": -23.080312728881836,
      "activations/layer2_attention_weight_max": 30.190780639648438,
      "activations/layer2_attention_weight_min": -29.610105514526367,
      "activations/layer3_attention_weight_max": 90.451904296875,
      "activations/layer3_attention_weight_min": -92.46239471435547,
      "activations/layer4_attention_weight_max": 95.22813415527344,
      "activations/layer4_attention_weight_min": -96.08209228515625,
      "activations/layer5_attention_weight_max": 70.23762512207031,
      "activations/layer5_attention_weight_min": -79.73206329345703,
      "activations/layer6_attention_weight_max": 50.36198043823242,
      "activations/layer6_attention_weight_min": -53.40229797363281,
      "activations/layer7_attention_weight_max": 73.64673614501953,
      "activations/layer7_attention_weight_min": -70.1281509399414,
      "activations/layer8_attention_weight_max": 43.11552047729492,
      "activations/layer8_attention_weight_min": -44.97075271606445,
      "activations/layer9_attention_weight_max": 43.0993766784668,
      "activations/layer9_attention_weight_min": -45.348167419433594,
      "epoch": 11.43,
      "learning_rate": 7.705265151515151e-05,
      "loss": 2.7905,
      "step": 196750
    },
    {
      "activations/layer0_attention_weight_max": 15.679007530212402,
      "activations/layer0_attention_weight_min": -13.783536911010742,
      "activations/layer10_attention_weight_max": 37.89909744262695,
      "activations/layer10_attention_weight_min": -35.78785705566406,
      "activations/layer11_attention_weight_max": 33.574729919433594,
      "activations/layer11_attention_weight_min": -31.879606246948242,
      "activations/layer12_attention_weight_max": 24.695980072021484,
      "activations/layer12_attention_weight_min": -25.68248748779297,
      "activations/layer13_attention_weight_max": 45.366539001464844,
      "activations/layer13_attention_weight_min": -30.85108184814453,
      "activations/layer14_attention_weight_max": 41.15725326538086,
      "activations/layer14_attention_weight_min": -29.54246711730957,
      "activations/layer15_attention_weight_max": 34.27154541015625,
      "activations/layer15_attention_weight_min": -28.343992233276367,
      "activations/layer16_attention_weight_max": 34.53847885131836,
      "activations/layer16_attention_weight_min": -29.116943359375,
      "activations/layer17_attention_weight_max": 56.02684020996094,
      "activations/layer17_attention_weight_min": -45.479156494140625,
      "activations/layer18_attention_weight_max": 52.79502487182617,
      "activations/layer18_attention_weight_min": -38.66713333129883,
      "activations/layer19_attention_weight_max": 21.700979232788086,
      "activations/layer19_attention_weight_min": -19.70623779296875,
      "activations/layer1_attention_weight_max": 17.751354217529297,
      "activations/layer1_attention_weight_min": -13.553135871887207,
      "activations/layer20_attention_weight_max": 21.1887264251709,
      "activations/layer20_attention_weight_min": -20.147991180419922,
      "activations/layer21_attention_weight_max": 39.85938262939453,
      "activations/layer21_attention_weight_min": -21.942323684692383,
      "activations/layer22_attention_weight_max": 30.9621639251709,
      "activations/layer22_attention_weight_min": -23.71973991394043,
      "activations/layer23_attention_weight_max": 37.78544998168945,
      "activations/layer23_attention_weight_min": -21.913930892944336,
      "activations/layer2_attention_weight_max": 31.874446868896484,
      "activations/layer2_attention_weight_min": -30.191694259643555,
      "activations/layer3_attention_weight_max": 95.96450805664062,
      "activations/layer3_attention_weight_min": -99.1924819946289,
      "activations/layer4_attention_weight_max": 102.13841247558594,
      "activations/layer4_attention_weight_min": -99.57051849365234,
      "activations/layer5_attention_weight_max": 74.8001937866211,
      "activations/layer5_attention_weight_min": -78.76438903808594,
      "activations/layer6_attention_weight_max": 54.715579986572266,
      "activations/layer6_attention_weight_min": -52.05936050415039,
      "activations/layer7_attention_weight_max": 75.05782318115234,
      "activations/layer7_attention_weight_min": -73.84471893310547,
      "activations/layer8_attention_weight_max": 44.87372970581055,
      "activations/layer8_attention_weight_min": -47.10805892944336,
      "activations/layer9_attention_weight_max": 49.9179573059082,
      "activations/layer9_attention_weight_min": -50.1734619140625,
      "epoch": 11.44,
      "learning_rate": 7.703371212121212e-05,
      "loss": 2.7822,
      "step": 196800
    },
    {
      "activations/layer0_attention_weight_max": 15.44186782836914,
      "activations/layer0_attention_weight_min": -13.897634506225586,
      "activations/layer10_attention_weight_max": 36.42253494262695,
      "activations/layer10_attention_weight_min": -36.30391311645508,
      "activations/layer11_attention_weight_max": 32.39867401123047,
      "activations/layer11_attention_weight_min": -32.63935089111328,
      "activations/layer12_attention_weight_max": 18.0889892578125,
      "activations/layer12_attention_weight_min": -28.138996124267578,
      "activations/layer13_attention_weight_max": 36.40538787841797,
      "activations/layer13_attention_weight_min": -27.03139877319336,
      "activations/layer14_attention_weight_max": 35.43595504760742,
      "activations/layer14_attention_weight_min": -31.02225685119629,
      "activations/layer15_attention_weight_max": 32.377689361572266,
      "activations/layer15_attention_weight_min": -30.004928588867188,
      "activations/layer16_attention_weight_max": 30.16744041442871,
      "activations/layer16_attention_weight_min": -28.94725799560547,
      "activations/layer17_attention_weight_max": 54.21245193481445,
      "activations/layer17_attention_weight_min": -45.5853271484375,
      "activations/layer18_attention_weight_max": 48.09615707397461,
      "activations/layer18_attention_weight_min": -38.8602180480957,
      "activations/layer19_attention_weight_max": 22.558300018310547,
      "activations/layer19_attention_weight_min": -22.843088150024414,
      "activations/layer1_attention_weight_max": 17.199312210083008,
      "activations/layer1_attention_weight_min": -15.589941024780273,
      "activations/layer20_attention_weight_max": 23.077085494995117,
      "activations/layer20_attention_weight_min": -24.361255645751953,
      "activations/layer21_attention_weight_max": 41.40058517456055,
      "activations/layer21_attention_weight_min": -22.797395706176758,
      "activations/layer22_attention_weight_max": 29.211891174316406,
      "activations/layer22_attention_weight_min": -26.453474044799805,
      "activations/layer23_attention_weight_max": 36.458274841308594,
      "activations/layer23_attention_weight_min": -24.21017837524414,
      "activations/layer2_attention_weight_max": 33.4356689453125,
      "activations/layer2_attention_weight_min": -30.498319625854492,
      "activations/layer3_attention_weight_max": 93.56929016113281,
      "activations/layer3_attention_weight_min": -93.59706115722656,
      "activations/layer4_attention_weight_max": 101.434814453125,
      "activations/layer4_attention_weight_min": -94.16230773925781,
      "activations/layer5_attention_weight_max": 72.78885650634766,
      "activations/layer5_attention_weight_min": -75.7972640991211,
      "activations/layer6_attention_weight_max": 52.75713348388672,
      "activations/layer6_attention_weight_min": -50.334556579589844,
      "activations/layer7_attention_weight_max": 72.48953247070312,
      "activations/layer7_attention_weight_min": -69.97348022460938,
      "activations/layer8_attention_weight_max": 46.248741149902344,
      "activations/layer8_attention_weight_min": -45.558441162109375,
      "activations/layer9_attention_weight_max": 51.38483810424805,
      "activations/layer9_attention_weight_min": -46.78178787231445,
      "epoch": 11.44,
      "learning_rate": 7.701477272727272e-05,
      "loss": 2.7818,
      "step": 196850
    },
    {
      "activations/layer0_attention_weight_max": 15.067072868347168,
      "activations/layer0_attention_weight_min": -14.529810905456543,
      "activations/layer10_attention_weight_max": 36.316139221191406,
      "activations/layer10_attention_weight_min": -34.879024505615234,
      "activations/layer11_attention_weight_max": 32.88269805908203,
      "activations/layer11_attention_weight_min": -34.39633560180664,
      "activations/layer12_attention_weight_max": 20.81263542175293,
      "activations/layer12_attention_weight_min": -26.01420783996582,
      "activations/layer13_attention_weight_max": 38.696533203125,
      "activations/layer13_attention_weight_min": -31.385717391967773,
      "activations/layer14_attention_weight_max": 55.91545104980469,
      "activations/layer14_attention_weight_min": -32.66691207885742,
      "activations/layer15_attention_weight_max": 45.282310485839844,
      "activations/layer15_attention_weight_min": -30.695905685424805,
      "activations/layer16_attention_weight_max": 33.91812515258789,
      "activations/layer16_attention_weight_min": -34.59522247314453,
      "activations/layer17_attention_weight_max": 57.84325408935547,
      "activations/layer17_attention_weight_min": -50.7717399597168,
      "activations/layer18_attention_weight_max": 51.16901397705078,
      "activations/layer18_attention_weight_min": -41.39664840698242,
      "activations/layer19_attention_weight_max": 25.22763442993164,
      "activations/layer19_attention_weight_min": -25.06403923034668,
      "activations/layer1_attention_weight_max": 17.974260330200195,
      "activations/layer1_attention_weight_min": -15.192087173461914,
      "activations/layer20_attention_weight_max": 26.21173667907715,
      "activations/layer20_attention_weight_min": -22.703933715820312,
      "activations/layer21_attention_weight_max": 41.254905700683594,
      "activations/layer21_attention_weight_min": -23.351970672607422,
      "activations/layer22_attention_weight_max": 37.68273162841797,
      "activations/layer22_attention_weight_min": -28.744098663330078,
      "activations/layer23_attention_weight_max": 43.158321380615234,
      "activations/layer23_attention_weight_min": -22.525344848632812,
      "activations/layer2_attention_weight_max": 35.31232452392578,
      "activations/layer2_attention_weight_min": -32.48595428466797,
      "activations/layer3_attention_weight_max": 96.64974212646484,
      "activations/layer3_attention_weight_min": -96.7933578491211,
      "activations/layer4_attention_weight_max": 98.60889434814453,
      "activations/layer4_attention_weight_min": -96.8580551147461,
      "activations/layer5_attention_weight_max": 73.47615051269531,
      "activations/layer5_attention_weight_min": -77.19252014160156,
      "activations/layer6_attention_weight_max": 51.49439239501953,
      "activations/layer6_attention_weight_min": -52.45341873168945,
      "activations/layer7_attention_weight_max": 74.76982879638672,
      "activations/layer7_attention_weight_min": -75.900146484375,
      "activations/layer8_attention_weight_max": 46.84553527832031,
      "activations/layer8_attention_weight_min": -49.32096481323242,
      "activations/layer9_attention_weight_max": 51.862831115722656,
      "activations/layer9_attention_weight_min": -48.10441207885742,
      "epoch": 11.44,
      "learning_rate": 7.699583333333333e-05,
      "loss": 2.7793,
      "step": 196900
    },
    {
      "activations/layer0_attention_weight_max": 15.810647010803223,
      "activations/layer0_attention_weight_min": -14.036876678466797,
      "activations/layer10_attention_weight_max": 36.651363372802734,
      "activations/layer10_attention_weight_min": -34.108489990234375,
      "activations/layer11_attention_weight_max": 30.388378143310547,
      "activations/layer11_attention_weight_min": -32.35448455810547,
      "activations/layer12_attention_weight_max": 19.085655212402344,
      "activations/layer12_attention_weight_min": -27.596214294433594,
      "activations/layer13_attention_weight_max": 49.92835235595703,
      "activations/layer13_attention_weight_min": -33.016780853271484,
      "activations/layer14_attention_weight_max": 36.37234115600586,
      "activations/layer14_attention_weight_min": -31.57729721069336,
      "activations/layer15_attention_weight_max": 31.784215927124023,
      "activations/layer15_attention_weight_min": -29.164682388305664,
      "activations/layer16_attention_weight_max": 30.62059783935547,
      "activations/layer16_attention_weight_min": -30.086061477661133,
      "activations/layer17_attention_weight_max": 49.8775749206543,
      "activations/layer17_attention_weight_min": -44.46563720703125,
      "activations/layer18_attention_weight_max": 44.045345306396484,
      "activations/layer18_attention_weight_min": -36.46563720703125,
      "activations/layer19_attention_weight_max": 22.854129791259766,
      "activations/layer19_attention_weight_min": -22.56852149963379,
      "activations/layer1_attention_weight_max": 16.395793914794922,
      "activations/layer1_attention_weight_min": -13.89306926727295,
      "activations/layer20_attention_weight_max": 22.13420867919922,
      "activations/layer20_attention_weight_min": -20.25208854675293,
      "activations/layer21_attention_weight_max": 37.332855224609375,
      "activations/layer21_attention_weight_min": -24.1025333404541,
      "activations/layer22_attention_weight_max": 32.69268035888672,
      "activations/layer22_attention_weight_min": -26.496185302734375,
      "activations/layer23_attention_weight_max": 40.04383850097656,
      "activations/layer23_attention_weight_min": -24.711162567138672,
      "activations/layer2_attention_weight_max": 31.729345321655273,
      "activations/layer2_attention_weight_min": -31.895301818847656,
      "activations/layer3_attention_weight_max": 98.31097412109375,
      "activations/layer3_attention_weight_min": -94.859375,
      "activations/layer4_attention_weight_max": 100.68826293945312,
      "activations/layer4_attention_weight_min": -96.88784790039062,
      "activations/layer5_attention_weight_max": 70.37987518310547,
      "activations/layer5_attention_weight_min": -79.95954895019531,
      "activations/layer6_attention_weight_max": 54.446815490722656,
      "activations/layer6_attention_weight_min": -54.343868255615234,
      "activations/layer7_attention_weight_max": 71.4914321899414,
      "activations/layer7_attention_weight_min": -73.31029510498047,
      "activations/layer8_attention_weight_max": 44.44944381713867,
      "activations/layer8_attention_weight_min": -46.7277717590332,
      "activations/layer9_attention_weight_max": 44.92292022705078,
      "activations/layer9_attention_weight_min": -45.84724807739258,
      "epoch": 11.44,
      "learning_rate": 7.697689393939392e-05,
      "loss": 2.7636,
      "step": 196950
    },
    {
      "activations/layer0_attention_weight_max": 15.395133972167969,
      "activations/layer0_attention_weight_min": -14.14553451538086,
      "activations/layer10_attention_weight_max": 37.62199401855469,
      "activations/layer10_attention_weight_min": -37.50468826293945,
      "activations/layer11_attention_weight_max": 34.177085876464844,
      "activations/layer11_attention_weight_min": -34.22041320800781,
      "activations/layer12_attention_weight_max": 19.76913070678711,
      "activations/layer12_attention_weight_min": -22.774553298950195,
      "activations/layer13_attention_weight_max": 43.018470764160156,
      "activations/layer13_attention_weight_min": -34.36577224731445,
      "activations/layer14_attention_weight_max": 40.981754302978516,
      "activations/layer14_attention_weight_min": -32.548789978027344,
      "activations/layer15_attention_weight_max": 35.60755157470703,
      "activations/layer15_attention_weight_min": -31.82211685180664,
      "activations/layer16_attention_weight_max": 33.720558166503906,
      "activations/layer16_attention_weight_min": -30.298906326293945,
      "activations/layer17_attention_weight_max": 53.75656509399414,
      "activations/layer17_attention_weight_min": -45.26334762573242,
      "activations/layer18_attention_weight_max": 48.73276138305664,
      "activations/layer18_attention_weight_min": -40.593971252441406,
      "activations/layer19_attention_weight_max": 23.766836166381836,
      "activations/layer19_attention_weight_min": -23.239133834838867,
      "activations/layer1_attention_weight_max": 16.707420349121094,
      "activations/layer1_attention_weight_min": -14.896439552307129,
      "activations/layer20_attention_weight_max": 25.13298797607422,
      "activations/layer20_attention_weight_min": -22.198345184326172,
      "activations/layer21_attention_weight_max": 40.408241271972656,
      "activations/layer21_attention_weight_min": -22.666934967041016,
      "activations/layer22_attention_weight_max": 36.37257385253906,
      "activations/layer22_attention_weight_min": -25.074678421020508,
      "activations/layer23_attention_weight_max": 46.04612731933594,
      "activations/layer23_attention_weight_min": -23.446304321289062,
      "activations/layer2_attention_weight_max": 32.26909637451172,
      "activations/layer2_attention_weight_min": -31.15192985534668,
      "activations/layer3_attention_weight_max": 96.78484344482422,
      "activations/layer3_attention_weight_min": -100.89256286621094,
      "activations/layer4_attention_weight_max": 107.28104400634766,
      "activations/layer4_attention_weight_min": -102.67496490478516,
      "activations/layer5_attention_weight_max": 75.53435516357422,
      "activations/layer5_attention_weight_min": -82.02204132080078,
      "activations/layer6_attention_weight_max": 53.44438171386719,
      "activations/layer6_attention_weight_min": -54.858070373535156,
      "activations/layer7_attention_weight_max": 77.32987213134766,
      "activations/layer7_attention_weight_min": -74.444580078125,
      "activations/layer8_attention_weight_max": 47.20857620239258,
      "activations/layer8_attention_weight_min": -48.40787887573242,
      "activations/layer9_attention_weight_max": 47.485145568847656,
      "activations/layer9_attention_weight_min": -48.396446228027344,
      "epoch": 11.45,
      "learning_rate": 7.695795454545454e-05,
      "loss": 2.7902,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.5095,
      "eval_samples_per_second": 504.61,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.5095,
      "eval_openwebtext_samples_per_second": 504.61,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9967,
      "eval_wikitext_samples_per_second": 228.381,
      "step": 197000
    },
    {
      "epoch": 11.45,
      "eval_lambada_loss": 2.53125,
      "eval_lambada_ppl": 12.569207830853442,
      "eval_lambada_runtime": 9.5755,
      "eval_lambada_samples_per_second": 508.486,
      "step": 197000
    },
    {
      "activations/layer0_attention_weight_max": 16.35325050354004,
      "activations/layer0_attention_weight_min": -14.08548641204834,
      "activations/layer10_attention_weight_max": 37.921783447265625,
      "activations/layer10_attention_weight_min": -36.39799880981445,
      "activations/layer11_attention_weight_max": 32.287017822265625,
      "activations/layer11_attention_weight_min": -32.371551513671875,
      "activations/layer12_attention_weight_max": 19.81958770751953,
      "activations/layer12_attention_weight_min": -24.86489486694336,
      "activations/layer13_attention_weight_max": 39.39406204223633,
      "activations/layer13_attention_weight_min": -30.501375198364258,
      "activations/layer14_attention_weight_max": 37.827945709228516,
      "activations/layer14_attention_weight_min": -32.10801696777344,
      "activations/layer15_attention_weight_max": 35.96513748168945,
      "activations/layer15_attention_weight_min": -29.65593910217285,
      "activations/layer16_attention_weight_max": 34.21492004394531,
      "activations/layer16_attention_weight_min": -29.091434478759766,
      "activations/layer17_attention_weight_max": 55.55240249633789,
      "activations/layer17_attention_weight_min": -48.521026611328125,
      "activations/layer18_attention_weight_max": 46.46576690673828,
      "activations/layer18_attention_weight_min": -40.17485046386719,
      "activations/layer19_attention_weight_max": 23.231298446655273,
      "activations/layer19_attention_weight_min": -22.669279098510742,
      "activations/layer1_attention_weight_max": 18.60295867919922,
      "activations/layer1_attention_weight_min": -15.275203704833984,
      "activations/layer20_attention_weight_max": 23.658023834228516,
      "activations/layer20_attention_weight_min": -24.02460289001465,
      "activations/layer21_attention_weight_max": 36.0601806640625,
      "activations/layer21_attention_weight_min": -23.05262565612793,
      "activations/layer22_attention_weight_max": 31.116222381591797,
      "activations/layer22_attention_weight_min": -25.488252639770508,
      "activations/layer23_attention_weight_max": 41.837398529052734,
      "activations/layer23_attention_weight_min": -24.621641159057617,
      "activations/layer2_attention_weight_max": 34.418087005615234,
      "activations/layer2_attention_weight_min": -33.39671325683594,
      "activations/layer3_attention_weight_max": 98.12571716308594,
      "activations/layer3_attention_weight_min": -97.765625,
      "activations/layer4_attention_weight_max": 106.26356506347656,
      "activations/layer4_attention_weight_min": -97.60200500488281,
      "activations/layer5_attention_weight_max": 78.25746154785156,
      "activations/layer5_attention_weight_min": -79.58869934082031,
      "activations/layer6_attention_weight_max": 56.20737838745117,
      "activations/layer6_attention_weight_min": -54.39346694946289,
      "activations/layer7_attention_weight_max": 77.22468566894531,
      "activations/layer7_attention_weight_min": -76.84925079345703,
      "activations/layer8_attention_weight_max": 47.81916046142578,
      "activations/layer8_attention_weight_min": -48.654197692871094,
      "activations/layer9_attention_weight_max": 54.37724685668945,
      "activations/layer9_attention_weight_min": -48.48155212402344,
      "epoch": 11.45,
      "learning_rate": 7.693901515151515e-05,
      "loss": 2.771,
      "step": 197050
    },
    {
      "activations/layer0_attention_weight_max": 15.833723068237305,
      "activations/layer0_attention_weight_min": -14.046086311340332,
      "activations/layer10_attention_weight_max": 37.59931945800781,
      "activations/layer10_attention_weight_min": -35.67478942871094,
      "activations/layer11_attention_weight_max": 32.695865631103516,
      "activations/layer11_attention_weight_min": -34.156246185302734,
      "activations/layer12_attention_weight_max": 18.071731567382812,
      "activations/layer12_attention_weight_min": -26.885738372802734,
      "activations/layer13_attention_weight_max": 38.31007385253906,
      "activations/layer13_attention_weight_min": -29.323902130126953,
      "activations/layer14_attention_weight_max": 38.52427291870117,
      "activations/layer14_attention_weight_min": -31.813228607177734,
      "activations/layer15_attention_weight_max": 34.40687942504883,
      "activations/layer15_attention_weight_min": -31.91787338256836,
      "activations/layer16_attention_weight_max": 31.961498260498047,
      "activations/layer16_attention_weight_min": -31.338533401489258,
      "activations/layer17_attention_weight_max": 53.72827911376953,
      "activations/layer17_attention_weight_min": -46.09607696533203,
      "activations/layer18_attention_weight_max": 49.87327575683594,
      "activations/layer18_attention_weight_min": -38.594093322753906,
      "activations/layer19_attention_weight_max": 25.49940299987793,
      "activations/layer19_attention_weight_min": -23.702733993530273,
      "activations/layer1_attention_weight_max": 16.76259422302246,
      "activations/layer1_attention_weight_min": -15.459672927856445,
      "activations/layer20_attention_weight_max": 24.819345474243164,
      "activations/layer20_attention_weight_min": -21.805356979370117,
      "activations/layer21_attention_weight_max": 50.751739501953125,
      "activations/layer21_attention_weight_min": -26.294240951538086,
      "activations/layer22_attention_weight_max": 32.41543960571289,
      "activations/layer22_attention_weight_min": -25.945528030395508,
      "activations/layer23_attention_weight_max": 38.80060577392578,
      "activations/layer23_attention_weight_min": -25.114749908447266,
      "activations/layer2_attention_weight_max": 33.12893295288086,
      "activations/layer2_attention_weight_min": -30.21217918395996,
      "activations/layer3_attention_weight_max": 93.53926849365234,
      "activations/layer3_attention_weight_min": -91.82386779785156,
      "activations/layer4_attention_weight_max": 95.99361419677734,
      "activations/layer4_attention_weight_min": -92.08511352539062,
      "activations/layer5_attention_weight_max": 72.74381256103516,
      "activations/layer5_attention_weight_min": -81.15969848632812,
      "activations/layer6_attention_weight_max": 51.9562873840332,
      "activations/layer6_attention_weight_min": -55.187103271484375,
      "activations/layer7_attention_weight_max": 75.723388671875,
      "activations/layer7_attention_weight_min": -70.06279754638672,
      "activations/layer8_attention_weight_max": 43.72957229614258,
      "activations/layer8_attention_weight_min": -45.85050582885742,
      "activations/layer9_attention_weight_max": 45.67561721801758,
      "activations/layer9_attention_weight_min": -45.98448944091797,
      "epoch": 11.45,
      "learning_rate": 7.692007575757574e-05,
      "loss": 2.7748,
      "step": 197100
    },
    {
      "activations/layer0_attention_weight_max": 14.82530403137207,
      "activations/layer0_attention_weight_min": -14.157424926757812,
      "activations/layer10_attention_weight_max": 36.03266906738281,
      "activations/layer10_attention_weight_min": -34.404293060302734,
      "activations/layer11_attention_weight_max": 31.321392059326172,
      "activations/layer11_attention_weight_min": -31.497638702392578,
      "activations/layer12_attention_weight_max": 18.054716110229492,
      "activations/layer12_attention_weight_min": -26.969215393066406,
      "activations/layer13_attention_weight_max": 37.896942138671875,
      "activations/layer13_attention_weight_min": -27.194734573364258,
      "activations/layer14_attention_weight_max": 37.35444641113281,
      "activations/layer14_attention_weight_min": -32.130455017089844,
      "activations/layer15_attention_weight_max": 34.5323486328125,
      "activations/layer15_attention_weight_min": -30.74600601196289,
      "activations/layer16_attention_weight_max": 31.903446197509766,
      "activations/layer16_attention_weight_min": -32.66584777832031,
      "activations/layer17_attention_weight_max": 53.44025802612305,
      "activations/layer17_attention_weight_min": -47.04222106933594,
      "activations/layer18_attention_weight_max": 46.59868240356445,
      "activations/layer18_attention_weight_min": -37.390872955322266,
      "activations/layer19_attention_weight_max": 22.550016403198242,
      "activations/layer19_attention_weight_min": -22.107149124145508,
      "activations/layer1_attention_weight_max": 16.512590408325195,
      "activations/layer1_attention_weight_min": -15.867136001586914,
      "activations/layer20_attention_weight_max": 23.67473602294922,
      "activations/layer20_attention_weight_min": -18.696123123168945,
      "activations/layer21_attention_weight_max": 36.2177734375,
      "activations/layer21_attention_weight_min": -22.344152450561523,
      "activations/layer22_attention_weight_max": 29.382579803466797,
      "activations/layer22_attention_weight_min": -24.23557472229004,
      "activations/layer23_attention_weight_max": 33.98933029174805,
      "activations/layer23_attention_weight_min": -21.56264305114746,
      "activations/layer2_attention_weight_max": 35.84345626831055,
      "activations/layer2_attention_weight_min": -33.72395324707031,
      "activations/layer3_attention_weight_max": 95.9098892211914,
      "activations/layer3_attention_weight_min": -91.8953628540039,
      "activations/layer4_attention_weight_max": 100.51800537109375,
      "activations/layer4_attention_weight_min": -95.19719696044922,
      "activations/layer5_attention_weight_max": 73.24551391601562,
      "activations/layer5_attention_weight_min": -74.04502868652344,
      "activations/layer6_attention_weight_max": 52.60698699951172,
      "activations/layer6_attention_weight_min": -55.018131256103516,
      "activations/layer7_attention_weight_max": 75.92597198486328,
      "activations/layer7_attention_weight_min": -74.60139465332031,
      "activations/layer8_attention_weight_max": 46.601627349853516,
      "activations/layer8_attention_weight_min": -48.10064697265625,
      "activations/layer9_attention_weight_max": 48.795597076416016,
      "activations/layer9_attention_weight_min": -46.58066940307617,
      "epoch": 11.46,
      "learning_rate": 7.690113636363635e-05,
      "loss": 2.7712,
      "step": 197150
    },
    {
      "activations/layer0_attention_weight_max": 14.967529296875,
      "activations/layer0_attention_weight_min": -14.110737800598145,
      "activations/layer10_attention_weight_max": 35.53488540649414,
      "activations/layer10_attention_weight_min": -35.633121490478516,
      "activations/layer11_attention_weight_max": 31.927602767944336,
      "activations/layer11_attention_weight_min": -33.51940155029297,
      "activations/layer12_attention_weight_max": 19.868011474609375,
      "activations/layer12_attention_weight_min": -28.87405776977539,
      "activations/layer13_attention_weight_max": 43.28036117553711,
      "activations/layer13_attention_weight_min": -30.34902572631836,
      "activations/layer14_attention_weight_max": 38.95823669433594,
      "activations/layer14_attention_weight_min": -30.065519332885742,
      "activations/layer15_attention_weight_max": 36.90817642211914,
      "activations/layer15_attention_weight_min": -29.341487884521484,
      "activations/layer16_attention_weight_max": 35.359867095947266,
      "activations/layer16_attention_weight_min": -29.40829086303711,
      "activations/layer17_attention_weight_max": 54.51418685913086,
      "activations/layer17_attention_weight_min": -43.355560302734375,
      "activations/layer18_attention_weight_max": 50.541908264160156,
      "activations/layer18_attention_weight_min": -38.053749084472656,
      "activations/layer19_attention_weight_max": 26.95277214050293,
      "activations/layer19_attention_weight_min": -22.573463439941406,
      "activations/layer1_attention_weight_max": 17.176969528198242,
      "activations/layer1_attention_weight_min": -13.878421783447266,
      "activations/layer20_attention_weight_max": 27.61590003967285,
      "activations/layer20_attention_weight_min": -22.27603530883789,
      "activations/layer21_attention_weight_max": 45.884071350097656,
      "activations/layer21_attention_weight_min": -23.4862060546875,
      "activations/layer22_attention_weight_max": 34.45554733276367,
      "activations/layer22_attention_weight_min": -26.668718338012695,
      "activations/layer23_attention_weight_max": 41.59252166748047,
      "activations/layer23_attention_weight_min": -23.997467041015625,
      "activations/layer2_attention_weight_max": 33.56582260131836,
      "activations/layer2_attention_weight_min": -33.73657989501953,
      "activations/layer3_attention_weight_max": 96.71688079833984,
      "activations/layer3_attention_weight_min": -105.40050506591797,
      "activations/layer4_attention_weight_max": 98.36319732666016,
      "activations/layer4_attention_weight_min": -98.10314178466797,
      "activations/layer5_attention_weight_max": 71.47604370117188,
      "activations/layer5_attention_weight_min": -77.6783676147461,
      "activations/layer6_attention_weight_max": 52.52428436279297,
      "activations/layer6_attention_weight_min": -53.53495407104492,
      "activations/layer7_attention_weight_max": 76.71977996826172,
      "activations/layer7_attention_weight_min": -71.85877990722656,
      "activations/layer8_attention_weight_max": 47.06285095214844,
      "activations/layer8_attention_weight_min": -48.210262298583984,
      "activations/layer9_attention_weight_max": 49.684871673583984,
      "activations/layer9_attention_weight_min": -48.37205123901367,
      "epoch": 11.46,
      "learning_rate": 7.688219696969697e-05,
      "loss": 2.7914,
      "step": 197200
    },
    {
      "activations/layer0_attention_weight_max": 15.846396446228027,
      "activations/layer0_attention_weight_min": -14.449169158935547,
      "activations/layer10_attention_weight_max": 37.123390197753906,
      "activations/layer10_attention_weight_min": -33.519798278808594,
      "activations/layer11_attention_weight_max": 33.83151626586914,
      "activations/layer11_attention_weight_min": -33.908050537109375,
      "activations/layer12_attention_weight_max": 21.0341739654541,
      "activations/layer12_attention_weight_min": -28.54338264465332,
      "activations/layer13_attention_weight_max": 33.64888381958008,
      "activations/layer13_attention_weight_min": -30.36237335205078,
      "activations/layer14_attention_weight_max": 34.55465316772461,
      "activations/layer14_attention_weight_min": -27.374584197998047,
      "activations/layer15_attention_weight_max": 32.02534103393555,
      "activations/layer15_attention_weight_min": -27.93404769897461,
      "activations/layer16_attention_weight_max": 31.475826263427734,
      "activations/layer16_attention_weight_min": -27.27971076965332,
      "activations/layer17_attention_weight_max": 51.49665451049805,
      "activations/layer17_attention_weight_min": -41.13793182373047,
      "activations/layer18_attention_weight_max": 44.13014602661133,
      "activations/layer18_attention_weight_min": -34.632110595703125,
      "activations/layer19_attention_weight_max": 20.130958557128906,
      "activations/layer19_attention_weight_min": -19.55699348449707,
      "activations/layer1_attention_weight_max": 17.10015869140625,
      "activations/layer1_attention_weight_min": -13.583680152893066,
      "activations/layer20_attention_weight_max": 21.005189895629883,
      "activations/layer20_attention_weight_min": -21.571372985839844,
      "activations/layer21_attention_weight_max": 34.59227752685547,
      "activations/layer21_attention_weight_min": -22.59642791748047,
      "activations/layer22_attention_weight_max": 31.219696044921875,
      "activations/layer22_attention_weight_min": -24.06993865966797,
      "activations/layer23_attention_weight_max": 36.85076904296875,
      "activations/layer23_attention_weight_min": -25.687763214111328,
      "activations/layer2_attention_weight_max": 32.58876037597656,
      "activations/layer2_attention_weight_min": -30.619787216186523,
      "activations/layer3_attention_weight_max": 104.50566101074219,
      "activations/layer3_attention_weight_min": -104.52236938476562,
      "activations/layer4_attention_weight_max": 104.45006561279297,
      "activations/layer4_attention_weight_min": -102.1410140991211,
      "activations/layer5_attention_weight_max": 71.46615600585938,
      "activations/layer5_attention_weight_min": -76.19007873535156,
      "activations/layer6_attention_weight_max": 51.883506774902344,
      "activations/layer6_attention_weight_min": -53.9599494934082,
      "activations/layer7_attention_weight_max": 76.32599639892578,
      "activations/layer7_attention_weight_min": -69.55826568603516,
      "activations/layer8_attention_weight_max": 44.65620803833008,
      "activations/layer8_attention_weight_min": -44.71306610107422,
      "activations/layer9_attention_weight_max": 51.38076400756836,
      "activations/layer9_attention_weight_min": -46.3294792175293,
      "epoch": 11.46,
      "learning_rate": 7.686325757575757e-05,
      "loss": 2.7725,
      "step": 197250
    },
    {
      "activations/layer0_attention_weight_max": 15.947460174560547,
      "activations/layer0_attention_weight_min": -14.235468864440918,
      "activations/layer10_attention_weight_max": 38.463623046875,
      "activations/layer10_attention_weight_min": -38.55168914794922,
      "activations/layer11_attention_weight_max": 32.30350875854492,
      "activations/layer11_attention_weight_min": -30.71072006225586,
      "activations/layer12_attention_weight_max": 24.7745304107666,
      "activations/layer12_attention_weight_min": -23.56300163269043,
      "activations/layer13_attention_weight_max": 39.08351135253906,
      "activations/layer13_attention_weight_min": -28.763158798217773,
      "activations/layer14_attention_weight_max": 35.299495697021484,
      "activations/layer14_attention_weight_min": -29.224384307861328,
      "activations/layer15_attention_weight_max": 33.296085357666016,
      "activations/layer15_attention_weight_min": -30.640047073364258,
      "activations/layer16_attention_weight_max": 27.261117935180664,
      "activations/layer16_attention_weight_min": -28.489112854003906,
      "activations/layer17_attention_weight_max": 47.706390380859375,
      "activations/layer17_attention_weight_min": -41.026607513427734,
      "activations/layer18_attention_weight_max": 46.236820220947266,
      "activations/layer18_attention_weight_min": -36.0496940612793,
      "activations/layer19_attention_weight_max": 23.083290100097656,
      "activations/layer19_attention_weight_min": -19.94468879699707,
      "activations/layer1_attention_weight_max": 17.788618087768555,
      "activations/layer1_attention_weight_min": -14.64209270477295,
      "activations/layer20_attention_weight_max": 21.352758407592773,
      "activations/layer20_attention_weight_min": -22.279619216918945,
      "activations/layer21_attention_weight_max": 32.171566009521484,
      "activations/layer21_attention_weight_min": -27.05015754699707,
      "activations/layer22_attention_weight_max": 29.55146026611328,
      "activations/layer22_attention_weight_min": -26.09784507751465,
      "activations/layer23_attention_weight_max": 33.665645599365234,
      "activations/layer23_attention_weight_min": -23.19637107849121,
      "activations/layer2_attention_weight_max": 32.663795471191406,
      "activations/layer2_attention_weight_min": -33.076683044433594,
      "activations/layer3_attention_weight_max": 95.16657257080078,
      "activations/layer3_attention_weight_min": -106.06536102294922,
      "activations/layer4_attention_weight_max": 102.5628433227539,
      "activations/layer4_attention_weight_min": -106.88150787353516,
      "activations/layer5_attention_weight_max": 77.10902404785156,
      "activations/layer5_attention_weight_min": -80.23088073730469,
      "activations/layer6_attention_weight_max": 56.43811798095703,
      "activations/layer6_attention_weight_min": -54.904422760009766,
      "activations/layer7_attention_weight_max": 79.0152587890625,
      "activations/layer7_attention_weight_min": -75.42951202392578,
      "activations/layer8_attention_weight_max": 48.14894104003906,
      "activations/layer8_attention_weight_min": -49.290218353271484,
      "activations/layer9_attention_weight_max": 49.872310638427734,
      "activations/layer9_attention_weight_min": -49.81613540649414,
      "epoch": 11.46,
      "learning_rate": 7.684431818181817e-05,
      "loss": 2.7803,
      "step": 197300
    },
    {
      "activations/layer0_attention_weight_max": 17.31366539001465,
      "activations/layer0_attention_weight_min": -14.08541202545166,
      "activations/layer10_attention_weight_max": 36.134342193603516,
      "activations/layer10_attention_weight_min": -35.249267578125,
      "activations/layer11_attention_weight_max": 31.625225067138672,
      "activations/layer11_attention_weight_min": -33.66337966918945,
      "activations/layer12_attention_weight_max": 18.57892608642578,
      "activations/layer12_attention_weight_min": -24.948074340820312,
      "activations/layer13_attention_weight_max": 40.634864807128906,
      "activations/layer13_attention_weight_min": -28.208642959594727,
      "activations/layer14_attention_weight_max": 36.35609817504883,
      "activations/layer14_attention_weight_min": -29.97130012512207,
      "activations/layer15_attention_weight_max": 33.40296173095703,
      "activations/layer15_attention_weight_min": -29.69154167175293,
      "activations/layer16_attention_weight_max": 32.41020202636719,
      "activations/layer16_attention_weight_min": -29.292875289916992,
      "activations/layer17_attention_weight_max": 51.35359573364258,
      "activations/layer17_attention_weight_min": -43.151832580566406,
      "activations/layer18_attention_weight_max": 46.55706787109375,
      "activations/layer18_attention_weight_min": -39.25370407104492,
      "activations/layer19_attention_weight_max": 22.964561462402344,
      "activations/layer19_attention_weight_min": -22.620346069335938,
      "activations/layer1_attention_weight_max": 16.192440032958984,
      "activations/layer1_attention_weight_min": -14.666532516479492,
      "activations/layer20_attention_weight_max": 24.48716926574707,
      "activations/layer20_attention_weight_min": -20.787353515625,
      "activations/layer21_attention_weight_max": 37.923248291015625,
      "activations/layer21_attention_weight_min": -26.03212547302246,
      "activations/layer22_attention_weight_max": 28.55402183532715,
      "activations/layer22_attention_weight_min": -25.900022506713867,
      "activations/layer23_attention_weight_max": 38.43956756591797,
      "activations/layer23_attention_weight_min": -23.12457275390625,
      "activations/layer2_attention_weight_max": 34.03124237060547,
      "activations/layer2_attention_weight_min": -33.51213073730469,
      "activations/layer3_attention_weight_max": 98.01862335205078,
      "activations/layer3_attention_weight_min": -104.6498794555664,
      "activations/layer4_attention_weight_max": 100.90303039550781,
      "activations/layer4_attention_weight_min": -97.80107879638672,
      "activations/layer5_attention_weight_max": 73.3656234741211,
      "activations/layer5_attention_weight_min": -79.41485595703125,
      "activations/layer6_attention_weight_max": 54.74523162841797,
      "activations/layer6_attention_weight_min": -55.047203063964844,
      "activations/layer7_attention_weight_max": 74.85625457763672,
      "activations/layer7_attention_weight_min": -73.17790222167969,
      "activations/layer8_attention_weight_max": 50.01301956176758,
      "activations/layer8_attention_weight_min": -53.43738555908203,
      "activations/layer9_attention_weight_max": 51.02634048461914,
      "activations/layer9_attention_weight_min": -49.286869049072266,
      "epoch": 11.47,
      "learning_rate": 7.682537878787879e-05,
      "loss": 2.7826,
      "step": 197350
    },
    {
      "activations/layer0_attention_weight_max": 16.29938316345215,
      "activations/layer0_attention_weight_min": -14.074769973754883,
      "activations/layer10_attention_weight_max": 39.4320182800293,
      "activations/layer10_attention_weight_min": -39.08155059814453,
      "activations/layer11_attention_weight_max": 36.12358856201172,
      "activations/layer11_attention_weight_min": -33.51646423339844,
      "activations/layer12_attention_weight_max": 17.567642211914062,
      "activations/layer12_attention_weight_min": -23.05643081665039,
      "activations/layer13_attention_weight_max": 37.728755950927734,
      "activations/layer13_attention_weight_min": -34.51576232910156,
      "activations/layer14_attention_weight_max": 37.053958892822266,
      "activations/layer14_attention_weight_min": -30.929100036621094,
      "activations/layer15_attention_weight_max": 33.02820587158203,
      "activations/layer15_attention_weight_min": -30.80437469482422,
      "activations/layer16_attention_weight_max": 31.589872360229492,
      "activations/layer16_attention_weight_min": -29.453657150268555,
      "activations/layer17_attention_weight_max": 52.74080276489258,
      "activations/layer17_attention_weight_min": -44.270469665527344,
      "activations/layer18_attention_weight_max": 46.468650817871094,
      "activations/layer18_attention_weight_min": -39.805511474609375,
      "activations/layer19_attention_weight_max": 21.785615921020508,
      "activations/layer19_attention_weight_min": -23.43181610107422,
      "activations/layer1_attention_weight_max": 17.37384033203125,
      "activations/layer1_attention_weight_min": -15.361359596252441,
      "activations/layer20_attention_weight_max": 22.13780403137207,
      "activations/layer20_attention_weight_min": -20.38044548034668,
      "activations/layer21_attention_weight_max": 33.80974197387695,
      "activations/layer21_attention_weight_min": -24.26713752746582,
      "activations/layer22_attention_weight_max": 29.641220092773438,
      "activations/layer22_attention_weight_min": -27.393260955810547,
      "activations/layer23_attention_weight_max": 37.31108093261719,
      "activations/layer23_attention_weight_min": -27.608030319213867,
      "activations/layer2_attention_weight_max": 34.65493392944336,
      "activations/layer2_attention_weight_min": -34.599388122558594,
      "activations/layer3_attention_weight_max": 96.53672790527344,
      "activations/layer3_attention_weight_min": -102.20865631103516,
      "activations/layer4_attention_weight_max": 104.30859375,
      "activations/layer4_attention_weight_min": -102.40612030029297,
      "activations/layer5_attention_weight_max": 78.66165161132812,
      "activations/layer5_attention_weight_min": -79.67288208007812,
      "activations/layer6_attention_weight_max": 60.679443359375,
      "activations/layer6_attention_weight_min": -57.372596740722656,
      "activations/layer7_attention_weight_max": 77.05905151367188,
      "activations/layer7_attention_weight_min": -75.90896606445312,
      "activations/layer8_attention_weight_max": 48.49337387084961,
      "activations/layer8_attention_weight_min": -53.08883285522461,
      "activations/layer9_attention_weight_max": 56.188575744628906,
      "activations/layer9_attention_weight_min": -51.45913314819336,
      "epoch": 11.47,
      "learning_rate": 7.680643939393939e-05,
      "loss": 2.7817,
      "step": 197400
    },
    {
      "activations/layer0_attention_weight_max": 16.0738582611084,
      "activations/layer0_attention_weight_min": -14.44849967956543,
      "activations/layer10_attention_weight_max": 34.636417388916016,
      "activations/layer10_attention_weight_min": -34.22735595703125,
      "activations/layer11_attention_weight_max": 31.062763214111328,
      "activations/layer11_attention_weight_min": -31.571910858154297,
      "activations/layer12_attention_weight_max": 16.896162033081055,
      "activations/layer12_attention_weight_min": -22.719324111938477,
      "activations/layer13_attention_weight_max": 37.91891098022461,
      "activations/layer13_attention_weight_min": -28.248947143554688,
      "activations/layer14_attention_weight_max": 37.19430923461914,
      "activations/layer14_attention_weight_min": -33.18364715576172,
      "activations/layer15_attention_weight_max": 32.09907150268555,
      "activations/layer15_attention_weight_min": -29.09671401977539,
      "activations/layer16_attention_weight_max": 32.419921875,
      "activations/layer16_attention_weight_min": -31.099803924560547,
      "activations/layer17_attention_weight_max": 51.82777404785156,
      "activations/layer17_attention_weight_min": -43.78071212768555,
      "activations/layer18_attention_weight_max": 45.1995964050293,
      "activations/layer18_attention_weight_min": -37.614654541015625,
      "activations/layer19_attention_weight_max": 20.9520263671875,
      "activations/layer19_attention_weight_min": -22.139638900756836,
      "activations/layer1_attention_weight_max": 17.266050338745117,
      "activations/layer1_attention_weight_min": -15.339021682739258,
      "activations/layer20_attention_weight_max": 22.149267196655273,
      "activations/layer20_attention_weight_min": -24.367382049560547,
      "activations/layer21_attention_weight_max": 36.67000198364258,
      "activations/layer21_attention_weight_min": -25.036258697509766,
      "activations/layer22_attention_weight_max": 29.068281173706055,
      "activations/layer22_attention_weight_min": -25.67206382751465,
      "activations/layer23_attention_weight_max": 39.346458435058594,
      "activations/layer23_attention_weight_min": -23.146995544433594,
      "activations/layer2_attention_weight_max": 28.71370506286621,
      "activations/layer2_attention_weight_min": -28.545156478881836,
      "activations/layer3_attention_weight_max": 87.3341293334961,
      "activations/layer3_attention_weight_min": -98.10824584960938,
      "activations/layer4_attention_weight_max": 95.87932586669922,
      "activations/layer4_attention_weight_min": -94.11870574951172,
      "activations/layer5_attention_weight_max": 76.232666015625,
      "activations/layer5_attention_weight_min": -77.0658187866211,
      "activations/layer6_attention_weight_max": 51.74766540527344,
      "activations/layer6_attention_weight_min": -52.38020706176758,
      "activations/layer7_attention_weight_max": 72.27822875976562,
      "activations/layer7_attention_weight_min": -71.59022521972656,
      "activations/layer8_attention_weight_max": 43.88072967529297,
      "activations/layer8_attention_weight_min": -45.30085754394531,
      "activations/layer9_attention_weight_max": 47.24705505371094,
      "activations/layer9_attention_weight_min": -45.49769973754883,
      "epoch": 11.47,
      "learning_rate": 7.67875e-05,
      "loss": 2.7743,
      "step": 197450
    },
    {
      "activations/layer0_attention_weight_max": 15.837952613830566,
      "activations/layer0_attention_weight_min": -13.879255294799805,
      "activations/layer10_attention_weight_max": 39.49531936645508,
      "activations/layer10_attention_weight_min": -35.65894317626953,
      "activations/layer11_attention_weight_max": 33.603172302246094,
      "activations/layer11_attention_weight_min": -32.02527618408203,
      "activations/layer12_attention_weight_max": 18.758398056030273,
      "activations/layer12_attention_weight_min": -24.444231033325195,
      "activations/layer13_attention_weight_max": 37.17948532104492,
      "activations/layer13_attention_weight_min": -29.151145935058594,
      "activations/layer14_attention_weight_max": 34.1227912902832,
      "activations/layer14_attention_weight_min": -28.7519588470459,
      "activations/layer15_attention_weight_max": 31.970882415771484,
      "activations/layer15_attention_weight_min": -27.691858291625977,
      "activations/layer16_attention_weight_max": 28.882678985595703,
      "activations/layer16_attention_weight_min": -28.48944091796875,
      "activations/layer17_attention_weight_max": 49.14165115356445,
      "activations/layer17_attention_weight_min": -41.46128845214844,
      "activations/layer18_attention_weight_max": 45.059844970703125,
      "activations/layer18_attention_weight_min": -33.89033889770508,
      "activations/layer19_attention_weight_max": 20.745580673217773,
      "activations/layer19_attention_weight_min": -19.720216751098633,
      "activations/layer1_attention_weight_max": 16.87696647644043,
      "activations/layer1_attention_weight_min": -17.32469940185547,
      "activations/layer20_attention_weight_max": 23.116891860961914,
      "activations/layer20_attention_weight_min": -20.595394134521484,
      "activations/layer21_attention_weight_max": 33.22099685668945,
      "activations/layer21_attention_weight_min": -23.6145076751709,
      "activations/layer22_attention_weight_max": 31.041337966918945,
      "activations/layer22_attention_weight_min": -25.31612777709961,
      "activations/layer23_attention_weight_max": 34.72801208496094,
      "activations/layer23_attention_weight_min": -23.657133102416992,
      "activations/layer2_attention_weight_max": 32.151123046875,
      "activations/layer2_attention_weight_min": -31.23366355895996,
      "activations/layer3_attention_weight_max": 93.23310852050781,
      "activations/layer3_attention_weight_min": -99.85013580322266,
      "activations/layer4_attention_weight_max": 96.59163665771484,
      "activations/layer4_attention_weight_min": -96.330810546875,
      "activations/layer5_attention_weight_max": 74.15834045410156,
      "activations/layer5_attention_weight_min": -77.95784759521484,
      "activations/layer6_attention_weight_max": 55.01463317871094,
      "activations/layer6_attention_weight_min": -52.51845169067383,
      "activations/layer7_attention_weight_max": 75.2386703491211,
      "activations/layer7_attention_weight_min": -71.02289581298828,
      "activations/layer8_attention_weight_max": 45.15594482421875,
      "activations/layer8_attention_weight_min": -49.66044998168945,
      "activations/layer9_attention_weight_max": 50.39400863647461,
      "activations/layer9_attention_weight_min": -48.068443298339844,
      "epoch": 11.48,
      "learning_rate": 7.67689393939394e-05,
      "loss": 2.7815,
      "step": 197500
    },
    {
      "activations/layer0_attention_weight_max": 15.122138977050781,
      "activations/layer0_attention_weight_min": -13.845906257629395,
      "activations/layer10_attention_weight_max": 41.93648910522461,
      "activations/layer10_attention_weight_min": -41.78102493286133,
      "activations/layer11_attention_weight_max": 39.05892562866211,
      "activations/layer11_attention_weight_min": -36.881446838378906,
      "activations/layer12_attention_weight_max": 19.030059814453125,
      "activations/layer12_attention_weight_min": -31.449085235595703,
      "activations/layer13_attention_weight_max": 40.115211486816406,
      "activations/layer13_attention_weight_min": -33.27674865722656,
      "activations/layer14_attention_weight_max": 44.47357177734375,
      "activations/layer14_attention_weight_min": -32.8341064453125,
      "activations/layer15_attention_weight_max": 39.17779541015625,
      "activations/layer15_attention_weight_min": -32.788265228271484,
      "activations/layer16_attention_weight_max": 32.77577590942383,
      "activations/layer16_attention_weight_min": -30.40415382385254,
      "activations/layer17_attention_weight_max": 56.27046585083008,
      "activations/layer17_attention_weight_min": -47.31983184814453,
      "activations/layer18_attention_weight_max": 51.54581069946289,
      "activations/layer18_attention_weight_min": -41.94792556762695,
      "activations/layer19_attention_weight_max": 22.93593406677246,
      "activations/layer19_attention_weight_min": -24.57669448852539,
      "activations/layer1_attention_weight_max": 16.922077178955078,
      "activations/layer1_attention_weight_min": -17.06106185913086,
      "activations/layer20_attention_weight_max": 25.537321090698242,
      "activations/layer20_attention_weight_min": -23.091291427612305,
      "activations/layer21_attention_weight_max": 41.712913513183594,
      "activations/layer21_attention_weight_min": -24.399089813232422,
      "activations/layer22_attention_weight_max": 31.171100616455078,
      "activations/layer22_attention_weight_min": -27.598814010620117,
      "activations/layer23_attention_weight_max": 41.29768371582031,
      "activations/layer23_attention_weight_min": -24.314289093017578,
      "activations/layer2_attention_weight_max": 34.81212615966797,
      "activations/layer2_attention_weight_min": -33.29438400268555,
      "activations/layer3_attention_weight_max": 106.67851257324219,
      "activations/layer3_attention_weight_min": -103.69120788574219,
      "activations/layer4_attention_weight_max": 116.22460174560547,
      "activations/layer4_attention_weight_min": -107.39283752441406,
      "activations/layer5_attention_weight_max": 79.67693328857422,
      "activations/layer5_attention_weight_min": -78.9991226196289,
      "activations/layer6_attention_weight_max": 57.317115783691406,
      "activations/layer6_attention_weight_min": -57.026519775390625,
      "activations/layer7_attention_weight_max": 82.78870391845703,
      "activations/layer7_attention_weight_min": -73.80199432373047,
      "activations/layer8_attention_weight_max": 51.96574783325195,
      "activations/layer8_attention_weight_min": -50.528076171875,
      "activations/layer9_attention_weight_max": 61.276363372802734,
      "activations/layer9_attention_weight_min": -50.17638397216797,
      "epoch": 11.48,
      "learning_rate": 7.675e-05,
      "loss": 2.7684,
      "step": 197550
    },
    {
      "activations/layer0_attention_weight_max": 15.954795837402344,
      "activations/layer0_attention_weight_min": -14.282706260681152,
      "activations/layer10_attention_weight_max": 37.496185302734375,
      "activations/layer10_attention_weight_min": -35.48430252075195,
      "activations/layer11_attention_weight_max": 32.161964416503906,
      "activations/layer11_attention_weight_min": -31.817829132080078,
      "activations/layer12_attention_weight_max": 19.404748916625977,
      "activations/layer12_attention_weight_min": -23.17597198486328,
      "activations/layer13_attention_weight_max": 40.62425994873047,
      "activations/layer13_attention_weight_min": -32.26665496826172,
      "activations/layer14_attention_weight_max": 41.569278717041016,
      "activations/layer14_attention_weight_min": -29.464839935302734,
      "activations/layer15_attention_weight_max": 34.147029876708984,
      "activations/layer15_attention_weight_min": -27.485950469970703,
      "activations/layer16_attention_weight_max": 31.49997901916504,
      "activations/layer16_attention_weight_min": -29.731164932250977,
      "activations/layer17_attention_weight_max": 54.73859786987305,
      "activations/layer17_attention_weight_min": -42.42792510986328,
      "activations/layer18_attention_weight_max": 48.1019287109375,
      "activations/layer18_attention_weight_min": -38.27024841308594,
      "activations/layer19_attention_weight_max": 21.080854415893555,
      "activations/layer19_attention_weight_min": -20.536174774169922,
      "activations/layer1_attention_weight_max": 16.62109375,
      "activations/layer1_attention_weight_min": -15.779380798339844,
      "activations/layer20_attention_weight_max": 23.307985305786133,
      "activations/layer20_attention_weight_min": -20.8445987701416,
      "activations/layer21_attention_weight_max": 38.49231719970703,
      "activations/layer21_attention_weight_min": -26.01924705505371,
      "activations/layer22_attention_weight_max": 29.85291862487793,
      "activations/layer22_attention_weight_min": -23.852210998535156,
      "activations/layer23_attention_weight_max": 36.464019775390625,
      "activations/layer23_attention_weight_min": -26.036602020263672,
      "activations/layer2_attention_weight_max": 31.60869598388672,
      "activations/layer2_attention_weight_min": -32.58416748046875,
      "activations/layer3_attention_weight_max": 93.99862670898438,
      "activations/layer3_attention_weight_min": -98.29842376708984,
      "activations/layer4_attention_weight_max": 100.21507263183594,
      "activations/layer4_attention_weight_min": -95.45638275146484,
      "activations/layer5_attention_weight_max": 74.05731964111328,
      "activations/layer5_attention_weight_min": -76.77389526367188,
      "activations/layer6_attention_weight_max": 56.92210388183594,
      "activations/layer6_attention_weight_min": -56.385894775390625,
      "activations/layer7_attention_weight_max": 73.28345489501953,
      "activations/layer7_attention_weight_min": -77.18151092529297,
      "activations/layer8_attention_weight_max": 46.6090087890625,
      "activations/layer8_attention_weight_min": -51.95248794555664,
      "activations/layer9_attention_weight_max": 48.75836181640625,
      "activations/layer9_attention_weight_min": -48.17877197265625,
      "epoch": 11.48,
      "learning_rate": 7.67310606060606e-05,
      "loss": 2.7805,
      "step": 197600
    },
    {
      "activations/layer0_attention_weight_max": 15.230914115905762,
      "activations/layer0_attention_weight_min": -14.313026428222656,
      "activations/layer10_attention_weight_max": 38.67700958251953,
      "activations/layer10_attention_weight_min": -34.2830696105957,
      "activations/layer11_attention_weight_max": 34.16700744628906,
      "activations/layer11_attention_weight_min": -31.367877960205078,
      "activations/layer12_attention_weight_max": 18.316783905029297,
      "activations/layer12_attention_weight_min": -23.9534969329834,
      "activations/layer13_attention_weight_max": 32.68449020385742,
      "activations/layer13_attention_weight_min": -29.274538040161133,
      "activations/layer14_attention_weight_max": 32.077999114990234,
      "activations/layer14_attention_weight_min": -29.097131729125977,
      "activations/layer15_attention_weight_max": 30.109256744384766,
      "activations/layer15_attention_weight_min": -28.439069747924805,
      "activations/layer16_attention_weight_max": 28.898460388183594,
      "activations/layer16_attention_weight_min": -28.179166793823242,
      "activations/layer17_attention_weight_max": 47.161964416503906,
      "activations/layer17_attention_weight_min": -41.60208511352539,
      "activations/layer18_attention_weight_max": 41.51462173461914,
      "activations/layer18_attention_weight_min": -34.18709182739258,
      "activations/layer19_attention_weight_max": 20.66448402404785,
      "activations/layer19_attention_weight_min": -21.575599670410156,
      "activations/layer1_attention_weight_max": 16.252893447875977,
      "activations/layer1_attention_weight_min": -15.605907440185547,
      "activations/layer20_attention_weight_max": 21.31696891784668,
      "activations/layer20_attention_weight_min": -22.38210105895996,
      "activations/layer21_attention_weight_max": 33.75160598754883,
      "activations/layer21_attention_weight_min": -25.126937866210938,
      "activations/layer22_attention_weight_max": 29.83405303955078,
      "activations/layer22_attention_weight_min": -23.96555519104004,
      "activations/layer23_attention_weight_max": 38.819976806640625,
      "activations/layer23_attention_weight_min": -24.616193771362305,
      "activations/layer2_attention_weight_max": 32.2976188659668,
      "activations/layer2_attention_weight_min": -29.575132369995117,
      "activations/layer3_attention_weight_max": 95.35208892822266,
      "activations/layer3_attention_weight_min": -95.0953369140625,
      "activations/layer4_attention_weight_max": 102.81044006347656,
      "activations/layer4_attention_weight_min": -96.79174041748047,
      "activations/layer5_attention_weight_max": 77.6793441772461,
      "activations/layer5_attention_weight_min": -77.45973205566406,
      "activations/layer6_attention_weight_max": 55.93936538696289,
      "activations/layer6_attention_weight_min": -52.61501693725586,
      "activations/layer7_attention_weight_max": 73.75701141357422,
      "activations/layer7_attention_weight_min": -75.39326477050781,
      "activations/layer8_attention_weight_max": 46.65401077270508,
      "activations/layer8_attention_weight_min": -46.38071823120117,
      "activations/layer9_attention_weight_max": 51.301082611083984,
      "activations/layer9_attention_weight_min": -49.05617904663086,
      "epoch": 11.48,
      "learning_rate": 7.671212121212121e-05,
      "loss": 2.7678,
      "step": 197650
    },
    {
      "activations/layer0_attention_weight_max": 15.436322212219238,
      "activations/layer0_attention_weight_min": -14.344623565673828,
      "activations/layer10_attention_weight_max": 36.629398345947266,
      "activations/layer10_attention_weight_min": -36.78776931762695,
      "activations/layer11_attention_weight_max": 35.78140640258789,
      "activations/layer11_attention_weight_min": -34.21647644042969,
      "activations/layer12_attention_weight_max": 20.73505401611328,
      "activations/layer12_attention_weight_min": -25.939027786254883,
      "activations/layer13_attention_weight_max": 36.373497009277344,
      "activations/layer13_attention_weight_min": -29.254732131958008,
      "activations/layer14_attention_weight_max": 35.98748779296875,
      "activations/layer14_attention_weight_min": -29.8528995513916,
      "activations/layer15_attention_weight_max": 31.780717849731445,
      "activations/layer15_attention_weight_min": -29.370447158813477,
      "activations/layer16_attention_weight_max": 30.792646408081055,
      "activations/layer16_attention_weight_min": -28.58671760559082,
      "activations/layer17_attention_weight_max": 50.136940002441406,
      "activations/layer17_attention_weight_min": -44.17711639404297,
      "activations/layer18_attention_weight_max": 46.83249282836914,
      "activations/layer18_attention_weight_min": -35.95854949951172,
      "activations/layer19_attention_weight_max": 21.227033615112305,
      "activations/layer19_attention_weight_min": -20.59560203552246,
      "activations/layer1_attention_weight_max": 17.088748931884766,
      "activations/layer1_attention_weight_min": -13.809273719787598,
      "activations/layer20_attention_weight_max": 20.73847198486328,
      "activations/layer20_attention_weight_min": -20.658065795898438,
      "activations/layer21_attention_weight_max": 35.046897888183594,
      "activations/layer21_attention_weight_min": -22.026081085205078,
      "activations/layer22_attention_weight_max": 31.95599365234375,
      "activations/layer22_attention_weight_min": -24.93857192993164,
      "activations/layer23_attention_weight_max": 39.30006408691406,
      "activations/layer23_attention_weight_min": -22.59468650817871,
      "activations/layer2_attention_weight_max": 31.439956665039062,
      "activations/layer2_attention_weight_min": -29.742088317871094,
      "activations/layer3_attention_weight_max": 93.62449645996094,
      "activations/layer3_attention_weight_min": -91.81108856201172,
      "activations/layer4_attention_weight_max": 97.27171325683594,
      "activations/layer4_attention_weight_min": -94.27384185791016,
      "activations/layer5_attention_weight_max": 70.38717651367188,
      "activations/layer5_attention_weight_min": -75.96498107910156,
      "activations/layer6_attention_weight_max": 53.27519607543945,
      "activations/layer6_attention_weight_min": -54.17689895629883,
      "activations/layer7_attention_weight_max": 73.92357635498047,
      "activations/layer7_attention_weight_min": -75.28207397460938,
      "activations/layer8_attention_weight_max": 49.58037185668945,
      "activations/layer8_attention_weight_min": -48.542171478271484,
      "activations/layer9_attention_weight_max": 48.88909149169922,
      "activations/layer9_attention_weight_min": -49.5470085144043,
      "epoch": 11.49,
      "learning_rate": 7.66931818181818e-05,
      "loss": 2.7784,
      "step": 197700
    },
    {
      "activations/layer0_attention_weight_max": 14.994926452636719,
      "activations/layer0_attention_weight_min": -14.18472957611084,
      "activations/layer10_attention_weight_max": 41.99225997924805,
      "activations/layer10_attention_weight_min": -37.102294921875,
      "activations/layer11_attention_weight_max": 36.15635299682617,
      "activations/layer11_attention_weight_min": -35.04046630859375,
      "activations/layer12_attention_weight_max": 17.463512420654297,
      "activations/layer12_attention_weight_min": -27.031246185302734,
      "activations/layer13_attention_weight_max": 46.32654571533203,
      "activations/layer13_attention_weight_min": -36.64467239379883,
      "activations/layer14_attention_weight_max": 59.286781311035156,
      "activations/layer14_attention_weight_min": -38.368770599365234,
      "activations/layer15_attention_weight_max": 45.5025634765625,
      "activations/layer15_attention_weight_min": -32.46845245361328,
      "activations/layer16_attention_weight_max": 30.216577529907227,
      "activations/layer16_attention_weight_min": -29.13094139099121,
      "activations/layer17_attention_weight_max": 52.33211898803711,
      "activations/layer17_attention_weight_min": -46.56155776977539,
      "activations/layer18_attention_weight_max": 49.592227935791016,
      "activations/layer18_attention_weight_min": -41.515968322753906,
      "activations/layer19_attention_weight_max": 22.12322235107422,
      "activations/layer19_attention_weight_min": -22.660945892333984,
      "activations/layer1_attention_weight_max": 18.543561935424805,
      "activations/layer1_attention_weight_min": -15.494983673095703,
      "activations/layer20_attention_weight_max": 23.73627281188965,
      "activations/layer20_attention_weight_min": -22.80602264404297,
      "activations/layer21_attention_weight_max": 39.95838928222656,
      "activations/layer21_attention_weight_min": -28.364492416381836,
      "activations/layer22_attention_weight_max": 33.95724868774414,
      "activations/layer22_attention_weight_min": -27.995027542114258,
      "activations/layer23_attention_weight_max": 39.18763732910156,
      "activations/layer23_attention_weight_min": -25.220840454101562,
      "activations/layer2_attention_weight_max": 33.794960021972656,
      "activations/layer2_attention_weight_min": -33.10475158691406,
      "activations/layer3_attention_weight_max": 99.63209533691406,
      "activations/layer3_attention_weight_min": -102.4811782836914,
      "activations/layer4_attention_weight_max": 104.88612365722656,
      "activations/layer4_attention_weight_min": -97.75532531738281,
      "activations/layer5_attention_weight_max": 75.18144989013672,
      "activations/layer5_attention_weight_min": -78.28330993652344,
      "activations/layer6_attention_weight_max": 58.060333251953125,
      "activations/layer6_attention_weight_min": -54.97868728637695,
      "activations/layer7_attention_weight_max": 80.14974975585938,
      "activations/layer7_attention_weight_min": -72.5323715209961,
      "activations/layer8_attention_weight_max": 48.27501678466797,
      "activations/layer8_attention_weight_min": -46.81385803222656,
      "activations/layer9_attention_weight_max": 51.08795166015625,
      "activations/layer9_attention_weight_min": -48.10151672363281,
      "epoch": 11.49,
      "learning_rate": 7.667424242424241e-05,
      "loss": 2.7705,
      "step": 197750
    },
    {
      "activations/layer0_attention_weight_max": 15.657756805419922,
      "activations/layer0_attention_weight_min": -14.082069396972656,
      "activations/layer10_attention_weight_max": 34.94873046875,
      "activations/layer10_attention_weight_min": -35.25662612915039,
      "activations/layer11_attention_weight_max": 30.25259780883789,
      "activations/layer11_attention_weight_min": -31.6097469329834,
      "activations/layer12_attention_weight_max": 28.511117935180664,
      "activations/layer12_attention_weight_min": -22.527755737304688,
      "activations/layer13_attention_weight_max": 38.78704833984375,
      "activations/layer13_attention_weight_min": -30.153663635253906,
      "activations/layer14_attention_weight_max": 35.0088005065918,
      "activations/layer14_attention_weight_min": -29.35919761657715,
      "activations/layer15_attention_weight_max": 32.41640853881836,
      "activations/layer15_attention_weight_min": -29.452898025512695,
      "activations/layer16_attention_weight_max": 31.87723731994629,
      "activations/layer16_attention_weight_min": -28.553998947143555,
      "activations/layer17_attention_weight_max": 51.244205474853516,
      "activations/layer17_attention_weight_min": -40.92375564575195,
      "activations/layer18_attention_weight_max": 45.25455093383789,
      "activations/layer18_attention_weight_min": -35.523529052734375,
      "activations/layer19_attention_weight_max": 25.367206573486328,
      "activations/layer19_attention_weight_min": -20.901042938232422,
      "activations/layer1_attention_weight_max": 16.78789520263672,
      "activations/layer1_attention_weight_min": -16.1712589263916,
      "activations/layer20_attention_weight_max": 22.53888511657715,
      "activations/layer20_attention_weight_min": -22.792484283447266,
      "activations/layer21_attention_weight_max": 37.86714553833008,
      "activations/layer21_attention_weight_min": -25.302518844604492,
      "activations/layer22_attention_weight_max": 27.921228408813477,
      "activations/layer22_attention_weight_min": -27.991270065307617,
      "activations/layer23_attention_weight_max": 39.14997482299805,
      "activations/layer23_attention_weight_min": -23.695730209350586,
      "activations/layer2_attention_weight_max": 31.967723846435547,
      "activations/layer2_attention_weight_min": -29.986244201660156,
      "activations/layer3_attention_weight_max": 95.23307800292969,
      "activations/layer3_attention_weight_min": -95.14512634277344,
      "activations/layer4_attention_weight_max": 98.8358383178711,
      "activations/layer4_attention_weight_min": -99.47952270507812,
      "activations/layer5_attention_weight_max": 70.46665954589844,
      "activations/layer5_attention_weight_min": -78.07122802734375,
      "activations/layer6_attention_weight_max": 52.9096565246582,
      "activations/layer6_attention_weight_min": -53.646888732910156,
      "activations/layer7_attention_weight_max": 73.5622787475586,
      "activations/layer7_attention_weight_min": -71.82670593261719,
      "activations/layer8_attention_weight_max": 44.89358139038086,
      "activations/layer8_attention_weight_min": -46.535926818847656,
      "activations/layer9_attention_weight_max": 46.86661148071289,
      "activations/layer9_attention_weight_min": -46.219879150390625,
      "epoch": 11.49,
      "learning_rate": 7.665530303030303e-05,
      "loss": 2.7791,
      "step": 197800
    },
    {
      "activations/layer0_attention_weight_max": 15.37291431427002,
      "activations/layer0_attention_weight_min": -14.344985961914062,
      "activations/layer10_attention_weight_max": 36.43933868408203,
      "activations/layer10_attention_weight_min": -35.481781005859375,
      "activations/layer11_attention_weight_max": 34.39364242553711,
      "activations/layer11_attention_weight_min": -35.09597396850586,
      "activations/layer12_attention_weight_max": 22.481340408325195,
      "activations/layer12_attention_weight_min": -24.809354782104492,
      "activations/layer13_attention_weight_max": 37.3197135925293,
      "activations/layer13_attention_weight_min": -28.47012710571289,
      "activations/layer14_attention_weight_max": 34.96329116821289,
      "activations/layer14_attention_weight_min": -30.869434356689453,
      "activations/layer15_attention_weight_max": 31.81381607055664,
      "activations/layer15_attention_weight_min": -30.58903694152832,
      "activations/layer16_attention_weight_max": 30.65587615966797,
      "activations/layer16_attention_weight_min": -30.944637298583984,
      "activations/layer17_attention_weight_max": 51.73479461669922,
      "activations/layer17_attention_weight_min": -45.5491943359375,
      "activations/layer18_attention_weight_max": 45.555946350097656,
      "activations/layer18_attention_weight_min": -42.507537841796875,
      "activations/layer19_attention_weight_max": 23.8128719329834,
      "activations/layer19_attention_weight_min": -22.70120620727539,
      "activations/layer1_attention_weight_max": 17.838964462280273,
      "activations/layer1_attention_weight_min": -14.89181900024414,
      "activations/layer20_attention_weight_max": 25.909038543701172,
      "activations/layer20_attention_weight_min": -22.74648666381836,
      "activations/layer21_attention_weight_max": 40.13444900512695,
      "activations/layer21_attention_weight_min": -22.550878524780273,
      "activations/layer22_attention_weight_max": 30.292072296142578,
      "activations/layer22_attention_weight_min": -28.57379722595215,
      "activations/layer23_attention_weight_max": 35.967384338378906,
      "activations/layer23_attention_weight_min": -23.486156463623047,
      "activations/layer2_attention_weight_max": 33.044342041015625,
      "activations/layer2_attention_weight_min": -31.164443969726562,
      "activations/layer3_attention_weight_max": 92.83949279785156,
      "activations/layer3_attention_weight_min": -98.48171997070312,
      "activations/layer4_attention_weight_max": 103.12001037597656,
      "activations/layer4_attention_weight_min": -102.64165496826172,
      "activations/layer5_attention_weight_max": 75.26397705078125,
      "activations/layer5_attention_weight_min": -80.5613021850586,
      "activations/layer6_attention_weight_max": 54.75321960449219,
      "activations/layer6_attention_weight_min": -54.50298309326172,
      "activations/layer7_attention_weight_max": 74.4859619140625,
      "activations/layer7_attention_weight_min": -75.41674041748047,
      "activations/layer8_attention_weight_max": 46.64702224731445,
      "activations/layer8_attention_weight_min": -46.78217697143555,
      "activations/layer9_attention_weight_max": 56.23409652709961,
      "activations/layer9_attention_weight_min": -49.2774658203125,
      "epoch": 11.5,
      "learning_rate": 7.663636363636362e-05,
      "loss": 2.7732,
      "step": 197850
    },
    {
      "activations/layer0_attention_weight_max": 16.27093505859375,
      "activations/layer0_attention_weight_min": -14.321945190429688,
      "activations/layer10_attention_weight_max": 40.56580352783203,
      "activations/layer10_attention_weight_min": -38.40520095825195,
      "activations/layer11_attention_weight_max": 34.922142028808594,
      "activations/layer11_attention_weight_min": -36.048519134521484,
      "activations/layer12_attention_weight_max": 18.265417098999023,
      "activations/layer12_attention_weight_min": -28.72367286682129,
      "activations/layer13_attention_weight_max": 38.74220657348633,
      "activations/layer13_attention_weight_min": -32.134178161621094,
      "activations/layer14_attention_weight_max": 40.95735549926758,
      "activations/layer14_attention_weight_min": -32.782859802246094,
      "activations/layer15_attention_weight_max": 36.73139190673828,
      "activations/layer15_attention_weight_min": -31.120670318603516,
      "activations/layer16_attention_weight_max": 39.63418960571289,
      "activations/layer16_attention_weight_min": -31.293439865112305,
      "activations/layer17_attention_weight_max": 61.51249313354492,
      "activations/layer17_attention_weight_min": -45.92192077636719,
      "activations/layer18_attention_weight_max": 54.11614227294922,
      "activations/layer18_attention_weight_min": -40.3165168762207,
      "activations/layer19_attention_weight_max": 26.337553024291992,
      "activations/layer19_attention_weight_min": -21.79655647277832,
      "activations/layer1_attention_weight_max": 17.644559860229492,
      "activations/layer1_attention_weight_min": -16.047231674194336,
      "activations/layer20_attention_weight_max": 24.406841278076172,
      "activations/layer20_attention_weight_min": -20.536624908447266,
      "activations/layer21_attention_weight_max": 43.008514404296875,
      "activations/layer21_attention_weight_min": -23.22037124633789,
      "activations/layer22_attention_weight_max": 33.47492980957031,
      "activations/layer22_attention_weight_min": -26.3757381439209,
      "activations/layer23_attention_weight_max": 38.98286437988281,
      "activations/layer23_attention_weight_min": -24.817562103271484,
      "activations/layer2_attention_weight_max": 33.3931770324707,
      "activations/layer2_attention_weight_min": -33.46880340576172,
      "activations/layer3_attention_weight_max": 106.23284912109375,
      "activations/layer3_attention_weight_min": -103.55135345458984,
      "activations/layer4_attention_weight_max": 110.8635025024414,
      "activations/layer4_attention_weight_min": -103.684814453125,
      "activations/layer5_attention_weight_max": 78.57061767578125,
      "activations/layer5_attention_weight_min": -78.22764587402344,
      "activations/layer6_attention_weight_max": 58.3216438293457,
      "activations/layer6_attention_weight_min": -55.74677658081055,
      "activations/layer7_attention_weight_max": 81.78853607177734,
      "activations/layer7_attention_weight_min": -78.85724639892578,
      "activations/layer8_attention_weight_max": 51.86518478393555,
      "activations/layer8_attention_weight_min": -49.89665603637695,
      "activations/layer9_attention_weight_max": 58.528114318847656,
      "activations/layer9_attention_weight_min": -49.13121032714844,
      "epoch": 11.5,
      "learning_rate": 7.661742424242423e-05,
      "loss": 2.7897,
      "step": 197900
    },
    {
      "activations/layer0_attention_weight_max": 15.903738021850586,
      "activations/layer0_attention_weight_min": -14.312492370605469,
      "activations/layer10_attention_weight_max": 38.20293426513672,
      "activations/layer10_attention_weight_min": -35.99103927612305,
      "activations/layer11_attention_weight_max": 33.89387893676758,
      "activations/layer11_attention_weight_min": -33.52882385253906,
      "activations/layer12_attention_weight_max": 21.333297729492188,
      "activations/layer12_attention_weight_min": -24.281997680664062,
      "activations/layer13_attention_weight_max": 51.06766128540039,
      "activations/layer13_attention_weight_min": -31.917802810668945,
      "activations/layer14_attention_weight_max": 38.42099380493164,
      "activations/layer14_attention_weight_min": -32.16349792480469,
      "activations/layer15_attention_weight_max": 36.92842483520508,
      "activations/layer15_attention_weight_min": -30.262258529663086,
      "activations/layer16_attention_weight_max": 35.27755355834961,
      "activations/layer16_attention_weight_min": -31.879972457885742,
      "activations/layer17_attention_weight_max": 55.12980270385742,
      "activations/layer17_attention_weight_min": -45.27720642089844,
      "activations/layer18_attention_weight_max": 47.793514251708984,
      "activations/layer18_attention_weight_min": -40.25508499145508,
      "activations/layer19_attention_weight_max": 26.350082397460938,
      "activations/layer19_attention_weight_min": -22.878414154052734,
      "activations/layer1_attention_weight_max": 17.117136001586914,
      "activations/layer1_attention_weight_min": -15.093367576599121,
      "activations/layer20_attention_weight_max": 24.87860107421875,
      "activations/layer20_attention_weight_min": -23.4862060546875,
      "activations/layer21_attention_weight_max": 46.3985595703125,
      "activations/layer21_attention_weight_min": -28.385812759399414,
      "activations/layer22_attention_weight_max": 35.33705139160156,
      "activations/layer22_attention_weight_min": -27.495630264282227,
      "activations/layer23_attention_weight_max": 41.0323371887207,
      "activations/layer23_attention_weight_min": -24.435413360595703,
      "activations/layer2_attention_weight_max": 33.36029815673828,
      "activations/layer2_attention_weight_min": -32.72557830810547,
      "activations/layer3_attention_weight_max": 98.17830657958984,
      "activations/layer3_attention_weight_min": -103.39286804199219,
      "activations/layer4_attention_weight_max": 98.6521224975586,
      "activations/layer4_attention_weight_min": -99.42882537841797,
      "activations/layer5_attention_weight_max": 75.2669677734375,
      "activations/layer5_attention_weight_min": -82.3675308227539,
      "activations/layer6_attention_weight_max": 53.047935485839844,
      "activations/layer6_attention_weight_min": -52.11585235595703,
      "activations/layer7_attention_weight_max": 76.6742172241211,
      "activations/layer7_attention_weight_min": -71.4272232055664,
      "activations/layer8_attention_weight_max": 47.38544464111328,
      "activations/layer8_attention_weight_min": -48.48649215698242,
      "activations/layer9_attention_weight_max": 50.991050720214844,
      "activations/layer9_attention_weight_min": -48.3790397644043,
      "epoch": 11.5,
      "learning_rate": 7.659848484848485e-05,
      "loss": 2.7841,
      "step": 197950
    },
    {
      "activations/layer0_attention_weight_max": 15.109642028808594,
      "activations/layer0_attention_weight_min": -14.199482917785645,
      "activations/layer10_attention_weight_max": 37.295867919921875,
      "activations/layer10_attention_weight_min": -35.40632247924805,
      "activations/layer11_attention_weight_max": 33.04654312133789,
      "activations/layer11_attention_weight_min": -32.63970947265625,
      "activations/layer12_attention_weight_max": 18.588830947875977,
      "activations/layer12_attention_weight_min": -25.46792984008789,
      "activations/layer13_attention_weight_max": 38.863922119140625,
      "activations/layer13_attention_weight_min": -30.51378059387207,
      "activations/layer14_attention_weight_max": 37.036109924316406,
      "activations/layer14_attention_weight_min": -29.825159072875977,
      "activations/layer15_attention_weight_max": 34.29951095581055,
      "activations/layer15_attention_weight_min": -29.348873138427734,
      "activations/layer16_attention_weight_max": 35.208465576171875,
      "activations/layer16_attention_weight_min": -30.376712799072266,
      "activations/layer17_attention_weight_max": 53.316436767578125,
      "activations/layer17_attention_weight_min": -45.5805549621582,
      "activations/layer18_attention_weight_max": 49.89568328857422,
      "activations/layer18_attention_weight_min": -36.978511810302734,
      "activations/layer19_attention_weight_max": 23.58340835571289,
      "activations/layer19_attention_weight_min": -22.229406356811523,
      "activations/layer1_attention_weight_max": 17.336631774902344,
      "activations/layer1_attention_weight_min": -17.027433395385742,
      "activations/layer20_attention_weight_max": 26.31585121154785,
      "activations/layer20_attention_weight_min": -23.209503173828125,
      "activations/layer21_attention_weight_max": 40.9886589050293,
      "activations/layer21_attention_weight_min": -24.268877029418945,
      "activations/layer22_attention_weight_max": 31.669607162475586,
      "activations/layer22_attention_weight_min": -24.24690818786621,
      "activations/layer23_attention_weight_max": 38.85952377319336,
      "activations/layer23_attention_weight_min": -24.253774642944336,
      "activations/layer2_attention_weight_max": 32.45281219482422,
      "activations/layer2_attention_weight_min": -29.765132904052734,
      "activations/layer3_attention_weight_max": 97.77591705322266,
      "activations/layer3_attention_weight_min": -99.14952850341797,
      "activations/layer4_attention_weight_max": 98.11244201660156,
      "activations/layer4_attention_weight_min": -98.31150817871094,
      "activations/layer5_attention_weight_max": 74.49238586425781,
      "activations/layer5_attention_weight_min": -80.56143188476562,
      "activations/layer6_attention_weight_max": 55.522254943847656,
      "activations/layer6_attention_weight_min": -54.15095901489258,
      "activations/layer7_attention_weight_max": 82.1332778930664,
      "activations/layer7_attention_weight_min": -75.83206939697266,
      "activations/layer8_attention_weight_max": 49.31035232543945,
      "activations/layer8_attention_weight_min": -48.0096435546875,
      "activations/layer9_attention_weight_max": 48.38969802856445,
      "activations/layer9_attention_weight_min": -47.82160186767578,
      "epoch": 11.5,
      "learning_rate": 7.657954545454545e-05,
      "loss": 2.7848,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.4891,
      "eval_samples_per_second": 505.824,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.4891,
      "eval_openwebtext_samples_per_second": 505.824,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_wikitext_loss": 2.96484375,
      "eval_wikitext_ppl": 19.391673055829287,
      "eval_wikitext_runtime": 1.9998,
      "eval_wikitext_samples_per_second": 228.025,
      "step": 198000
    },
    {
      "epoch": 11.5,
      "eval_lambada_loss": 2.564453125,
      "eval_lambada_ppl": 12.993550578338562,
      "eval_lambada_runtime": 9.5907,
      "eval_lambada_samples_per_second": 507.677,
      "step": 198000
    },
    {
      "activations/layer0_attention_weight_max": 15.092720031738281,
      "activations/layer0_attention_weight_min": -14.584309577941895,
      "activations/layer10_attention_weight_max": 36.1260871887207,
      "activations/layer10_attention_weight_min": -36.26140213012695,
      "activations/layer11_attention_weight_max": 31.16744041442871,
      "activations/layer11_attention_weight_min": -33.86819076538086,
      "activations/layer12_attention_weight_max": 17.88685417175293,
      "activations/layer12_attention_weight_min": -24.597665786743164,
      "activations/layer13_attention_weight_max": 48.35521697998047,
      "activations/layer13_attention_weight_min": -30.959321975708008,
      "activations/layer14_attention_weight_max": 48.88929748535156,
      "activations/layer14_attention_weight_min": -30.262548446655273,
      "activations/layer15_attention_weight_max": 43.98377990722656,
      "activations/layer15_attention_weight_min": -29.861799240112305,
      "activations/layer16_attention_weight_max": 40.93355178833008,
      "activations/layer16_attention_weight_min": -31.447778701782227,
      "activations/layer17_attention_weight_max": 62.61531448364258,
      "activations/layer17_attention_weight_min": -47.16082763671875,
      "activations/layer18_attention_weight_max": 57.13955307006836,
      "activations/layer18_attention_weight_min": -42.83863830566406,
      "activations/layer19_attention_weight_max": 28.983484268188477,
      "activations/layer19_attention_weight_min": -26.57256507873535,
      "activations/layer1_attention_weight_max": 16.659839630126953,
      "activations/layer1_attention_weight_min": -15.677789688110352,
      "activations/layer20_attention_weight_max": 30.123191833496094,
      "activations/layer20_attention_weight_min": -21.799728393554688,
      "activations/layer21_attention_weight_max": 50.19096374511719,
      "activations/layer21_attention_weight_min": -27.120473861694336,
      "activations/layer22_attention_weight_max": 40.30542755126953,
      "activations/layer22_attention_weight_min": -27.100814819335938,
      "activations/layer23_attention_weight_max": 44.520416259765625,
      "activations/layer23_attention_weight_min": -25.651153564453125,
      "activations/layer2_attention_weight_max": 32.05466079711914,
      "activations/layer2_attention_weight_min": -30.876684188842773,
      "activations/layer3_attention_weight_max": 92.29715728759766,
      "activations/layer3_attention_weight_min": -96.4432601928711,
      "activations/layer4_attention_weight_max": 98.10527038574219,
      "activations/layer4_attention_weight_min": -95.2574234008789,
      "activations/layer5_attention_weight_max": 75.15052032470703,
      "activations/layer5_attention_weight_min": -83.52481079101562,
      "activations/layer6_attention_weight_max": 52.75227737426758,
      "activations/layer6_attention_weight_min": -52.728328704833984,
      "activations/layer7_attention_weight_max": 73.99395751953125,
      "activations/layer7_attention_weight_min": -70.04956817626953,
      "activations/layer8_attention_weight_max": 46.31914138793945,
      "activations/layer8_attention_weight_min": -47.61216354370117,
      "activations/layer9_attention_weight_max": 47.72933578491211,
      "activations/layer9_attention_weight_min": -46.376712799072266,
      "epoch": 11.51,
      "learning_rate": 7.656060606060605e-05,
      "loss": 2.7891,
      "step": 198050
    },
    {
      "activations/layer0_attention_weight_max": 14.46225643157959,
      "activations/layer0_attention_weight_min": -14.160475730895996,
      "activations/layer10_attention_weight_max": 40.338600158691406,
      "activations/layer10_attention_weight_min": -40.009124755859375,
      "activations/layer11_attention_weight_max": 33.600128173828125,
      "activations/layer11_attention_weight_min": -34.744667053222656,
      "activations/layer12_attention_weight_max": 21.25212287902832,
      "activations/layer12_attention_weight_min": -27.306991577148438,
      "activations/layer13_attention_weight_max": 45.1047477722168,
      "activations/layer13_attention_weight_min": -32.431007385253906,
      "activations/layer14_attention_weight_max": 49.33516311645508,
      "activations/layer14_attention_weight_min": -31.80167007446289,
      "activations/layer15_attention_weight_max": 37.41558837890625,
      "activations/layer15_attention_weight_min": -29.80419921875,
      "activations/layer16_attention_weight_max": 32.29085922241211,
      "activations/layer16_attention_weight_min": -31.61094093322754,
      "activations/layer17_attention_weight_max": 54.1945915222168,
      "activations/layer17_attention_weight_min": -45.7230224609375,
      "activations/layer18_attention_weight_max": 47.487464904785156,
      "activations/layer18_attention_weight_min": -38.388572692871094,
      "activations/layer19_attention_weight_max": 22.072317123413086,
      "activations/layer19_attention_weight_min": -22.831989288330078,
      "activations/layer1_attention_weight_max": 16.564393997192383,
      "activations/layer1_attention_weight_min": -16.057695388793945,
      "activations/layer20_attention_weight_max": 21.750167846679688,
      "activations/layer20_attention_weight_min": -21.923002243041992,
      "activations/layer21_attention_weight_max": 37.63723373413086,
      "activations/layer21_attention_weight_min": -22.016965866088867,
      "activations/layer22_attention_weight_max": 30.857086181640625,
      "activations/layer22_attention_weight_min": -23.608449935913086,
      "activations/layer23_attention_weight_max": 40.27459716796875,
      "activations/layer23_attention_weight_min": -21.532827377319336,
      "activations/layer2_attention_weight_max": 33.93709182739258,
      "activations/layer2_attention_weight_min": -32.95487976074219,
      "activations/layer3_attention_weight_max": 103.83311462402344,
      "activations/layer3_attention_weight_min": -99.53663635253906,
      "activations/layer4_attention_weight_max": 108.52298736572266,
      "activations/layer4_attention_weight_min": -99.94647216796875,
      "activations/layer5_attention_weight_max": 74.56342315673828,
      "activations/layer5_attention_weight_min": -77.18467712402344,
      "activations/layer6_attention_weight_max": 54.484317779541016,
      "activations/layer6_attention_weight_min": -55.81028366088867,
      "activations/layer7_attention_weight_max": 76.03599548339844,
      "activations/layer7_attention_weight_min": -79.54586791992188,
      "activations/layer8_attention_weight_max": 46.748939514160156,
      "activations/layer8_attention_weight_min": -50.20093536376953,
      "activations/layer9_attention_weight_max": 49.35239791870117,
      "activations/layer9_attention_weight_min": -52.14640426635742,
      "epoch": 11.51,
      "learning_rate": 7.654166666666667e-05,
      "loss": 2.7932,
      "step": 198100
    },
    {
      "activations/layer0_attention_weight_max": 16.376075744628906,
      "activations/layer0_attention_weight_min": -14.461770057678223,
      "activations/layer10_attention_weight_max": 35.634498596191406,
      "activations/layer10_attention_weight_min": -35.30805206298828,
      "activations/layer11_attention_weight_max": 32.46070098876953,
      "activations/layer11_attention_weight_min": -32.8475227355957,
      "activations/layer12_attention_weight_max": 17.661380767822266,
      "activations/layer12_attention_weight_min": -23.604148864746094,
      "activations/layer13_attention_weight_max": 41.22150421142578,
      "activations/layer13_attention_weight_min": -31.673097610473633,
      "activations/layer14_attention_weight_max": 47.055904388427734,
      "activations/layer14_attention_weight_min": -31.975189208984375,
      "activations/layer15_attention_weight_max": 34.63823699951172,
      "activations/layer15_attention_weight_min": -31.585594177246094,
      "activations/layer16_attention_weight_max": 37.85219192504883,
      "activations/layer16_attention_weight_min": -32.37104034423828,
      "activations/layer17_attention_weight_max": 58.14596939086914,
      "activations/layer17_attention_weight_min": -47.30082321166992,
      "activations/layer18_attention_weight_max": 49.85564041137695,
      "activations/layer18_attention_weight_min": -40.48458480834961,
      "activations/layer19_attention_weight_max": 23.465734481811523,
      "activations/layer19_attention_weight_min": -22.6551570892334,
      "activations/layer1_attention_weight_max": 17.145395278930664,
      "activations/layer1_attention_weight_min": -16.161481857299805,
      "activations/layer20_attention_weight_max": 25.68182945251465,
      "activations/layer20_attention_weight_min": -21.28970718383789,
      "activations/layer21_attention_weight_max": 36.528839111328125,
      "activations/layer21_attention_weight_min": -22.283891677856445,
      "activations/layer22_attention_weight_max": 33.63825988769531,
      "activations/layer22_attention_weight_min": -27.501508712768555,
      "activations/layer23_attention_weight_max": 43.17253112792969,
      "activations/layer23_attention_weight_min": -26.216129302978516,
      "activations/layer2_attention_weight_max": 33.06480026245117,
      "activations/layer2_attention_weight_min": -31.22823143005371,
      "activations/layer3_attention_weight_max": 96.3086929321289,
      "activations/layer3_attention_weight_min": -97.35310363769531,
      "activations/layer4_attention_weight_max": 103.85169982910156,
      "activations/layer4_attention_weight_min": -99.3613052368164,
      "activations/layer5_attention_weight_max": 74.3572998046875,
      "activations/layer5_attention_weight_min": -80.91978454589844,
      "activations/layer6_attention_weight_max": 52.97063446044922,
      "activations/layer6_attention_weight_min": -53.22085952758789,
      "activations/layer7_attention_weight_max": 75.8502197265625,
      "activations/layer7_attention_weight_min": -78.19429779052734,
      "activations/layer8_attention_weight_max": 43.645240783691406,
      "activations/layer8_attention_weight_min": -46.743587493896484,
      "activations/layer9_attention_weight_max": 45.67499923706055,
      "activations/layer9_attention_weight_min": -47.87971115112305,
      "epoch": 11.51,
      "learning_rate": 7.652272727272727e-05,
      "loss": 2.7867,
      "step": 198150
    },
    {
      "activations/layer0_attention_weight_max": 15.582295417785645,
      "activations/layer0_attention_weight_min": -13.98103141784668,
      "activations/layer10_attention_weight_max": 36.80799102783203,
      "activations/layer10_attention_weight_min": -36.63855743408203,
      "activations/layer11_attention_weight_max": 32.44140625,
      "activations/layer11_attention_weight_min": -32.16999816894531,
      "activations/layer12_attention_weight_max": 20.28049659729004,
      "activations/layer12_attention_weight_min": -35.927066802978516,
      "activations/layer13_attention_weight_max": 36.41541290283203,
      "activations/layer13_attention_weight_min": -27.32762336730957,
      "activations/layer14_attention_weight_max": 37.2259407043457,
      "activations/layer14_attention_weight_min": -30.837160110473633,
      "activations/layer15_attention_weight_max": 34.819461822509766,
      "activations/layer15_attention_weight_min": -30.665485382080078,
      "activations/layer16_attention_weight_max": 31.217084884643555,
      "activations/layer16_attention_weight_min": -30.391124725341797,
      "activations/layer17_attention_weight_max": 52.27391052246094,
      "activations/layer17_attention_weight_min": -44.77724075317383,
      "activations/layer18_attention_weight_max": 46.38816452026367,
      "activations/layer18_attention_weight_min": -38.709564208984375,
      "activations/layer19_attention_weight_max": 19.637191772460938,
      "activations/layer19_attention_weight_min": -21.84943962097168,
      "activations/layer1_attention_weight_max": 17.747777938842773,
      "activations/layer1_attention_weight_min": -15.062899589538574,
      "activations/layer20_attention_weight_max": 20.68326187133789,
      "activations/layer20_attention_weight_min": -22.388029098510742,
      "activations/layer21_attention_weight_max": 31.649675369262695,
      "activations/layer21_attention_weight_min": -22.23755645751953,
      "activations/layer22_attention_weight_max": 29.45802879333496,
      "activations/layer22_attention_weight_min": -24.659156799316406,
      "activations/layer23_attention_weight_max": 32.49162292480469,
      "activations/layer23_attention_weight_min": -20.504337310791016,
      "activations/layer2_attention_weight_max": 34.77473449707031,
      "activations/layer2_attention_weight_min": -30.828842163085938,
      "activations/layer3_attention_weight_max": 94.10862731933594,
      "activations/layer3_attention_weight_min": -99.70842742919922,
      "activations/layer4_attention_weight_max": 101.15231323242188,
      "activations/layer4_attention_weight_min": -97.47660064697266,
      "activations/layer5_attention_weight_max": 71.81369018554688,
      "activations/layer5_attention_weight_min": -77.38583374023438,
      "activations/layer6_attention_weight_max": 52.2800407409668,
      "activations/layer6_attention_weight_min": -53.68885803222656,
      "activations/layer7_attention_weight_max": 75.38056182861328,
      "activations/layer7_attention_weight_min": -69.62972259521484,
      "activations/layer8_attention_weight_max": 44.93330764770508,
      "activations/layer8_attention_weight_min": -46.606781005859375,
      "activations/layer9_attention_weight_max": 54.67097854614258,
      "activations/layer9_attention_weight_min": -50.57378387451172,
      "epoch": 11.52,
      "learning_rate": 7.650378787878787e-05,
      "loss": 2.7761,
      "step": 198200
    },
    {
      "activations/layer0_attention_weight_max": 15.006977081298828,
      "activations/layer0_attention_weight_min": -14.594587326049805,
      "activations/layer10_attention_weight_max": 35.34758758544922,
      "activations/layer10_attention_weight_min": -35.246185302734375,
      "activations/layer11_attention_weight_max": 30.932388305664062,
      "activations/layer11_attention_weight_min": -31.57585906982422,
      "activations/layer12_attention_weight_max": 16.988683700561523,
      "activations/layer12_attention_weight_min": -27.667722702026367,
      "activations/layer13_attention_weight_max": 36.738922119140625,
      "activations/layer13_attention_weight_min": -28.279184341430664,
      "activations/layer14_attention_weight_max": 35.617488861083984,
      "activations/layer14_attention_weight_min": -28.480566024780273,
      "activations/layer15_attention_weight_max": 32.08671188354492,
      "activations/layer15_attention_weight_min": -27.066112518310547,
      "activations/layer16_attention_weight_max": 29.054311752319336,
      "activations/layer16_attention_weight_min": -29.216737747192383,
      "activations/layer17_attention_weight_max": 52.694950103759766,
      "activations/layer17_attention_weight_min": -43.215213775634766,
      "activations/layer18_attention_weight_max": 47.48052978515625,
      "activations/layer18_attention_weight_min": -37.51197814941406,
      "activations/layer19_attention_weight_max": 23.67654037475586,
      "activations/layer19_attention_weight_min": -23.370561599731445,
      "activations/layer1_attention_weight_max": 16.289377212524414,
      "activations/layer1_attention_weight_min": -17.11518096923828,
      "activations/layer20_attention_weight_max": 23.761075973510742,
      "activations/layer20_attention_weight_min": -21.657442092895508,
      "activations/layer21_attention_weight_max": 37.52288818359375,
      "activations/layer21_attention_weight_min": -21.96193504333496,
      "activations/layer22_attention_weight_max": 29.1618709564209,
      "activations/layer22_attention_weight_min": -26.134410858154297,
      "activations/layer23_attention_weight_max": 33.48429489135742,
      "activations/layer23_attention_weight_min": -23.697240829467773,
      "activations/layer2_attention_weight_max": 33.86958312988281,
      "activations/layer2_attention_weight_min": -29.504539489746094,
      "activations/layer3_attention_weight_max": 95.41144561767578,
      "activations/layer3_attention_weight_min": -93.46058654785156,
      "activations/layer4_attention_weight_max": 105.2057113647461,
      "activations/layer4_attention_weight_min": -99.93090057373047,
      "activations/layer5_attention_weight_max": 76.07363891601562,
      "activations/layer5_attention_weight_min": -80.02332305908203,
      "activations/layer6_attention_weight_max": 53.29359436035156,
      "activations/layer6_attention_weight_min": -54.317100524902344,
      "activations/layer7_attention_weight_max": 78.45153045654297,
      "activations/layer7_attention_weight_min": -75.03695678710938,
      "activations/layer8_attention_weight_max": 46.73626708984375,
      "activations/layer8_attention_weight_min": -47.33584976196289,
      "activations/layer9_attention_weight_max": 51.01451110839844,
      "activations/layer9_attention_weight_min": -47.90260696411133,
      "epoch": 11.52,
      "learning_rate": 7.648484848484847e-05,
      "loss": 2.7664,
      "step": 198250
    },
    {
      "activations/layer0_attention_weight_max": 16.500673294067383,
      "activations/layer0_attention_weight_min": -14.070466041564941,
      "activations/layer10_attention_weight_max": 36.137245178222656,
      "activations/layer10_attention_weight_min": -35.4012336730957,
      "activations/layer11_attention_weight_max": 32.29141616821289,
      "activations/layer11_attention_weight_min": -33.39963150024414,
      "activations/layer12_attention_weight_max": 19.135440826416016,
      "activations/layer12_attention_weight_min": -21.115259170532227,
      "activations/layer13_attention_weight_max": 38.18305587768555,
      "activations/layer13_attention_weight_min": -31.372900009155273,
      "activations/layer14_attention_weight_max": 36.08198547363281,
      "activations/layer14_attention_weight_min": -32.55049514770508,
      "activations/layer15_attention_weight_max": 33.077964782714844,
      "activations/layer15_attention_weight_min": -29.738037109375,
      "activations/layer16_attention_weight_max": 30.44890022277832,
      "activations/layer16_attention_weight_min": -29.337926864624023,
      "activations/layer17_attention_weight_max": 51.31816482543945,
      "activations/layer17_attention_weight_min": -46.23886489868164,
      "activations/layer18_attention_weight_max": 47.22283935546875,
      "activations/layer18_attention_weight_min": -40.02478790283203,
      "activations/layer19_attention_weight_max": 21.74228858947754,
      "activations/layer19_attention_weight_min": -22.5893611907959,
      "activations/layer1_attention_weight_max": 17.30579948425293,
      "activations/layer1_attention_weight_min": -14.249711036682129,
      "activations/layer20_attention_weight_max": 23.807292938232422,
      "activations/layer20_attention_weight_min": -22.10839080810547,
      "activations/layer21_attention_weight_max": 39.19910430908203,
      "activations/layer21_attention_weight_min": -25.933908462524414,
      "activations/layer22_attention_weight_max": 29.49359893798828,
      "activations/layer22_attention_weight_min": -28.050174713134766,
      "activations/layer23_attention_weight_max": 38.59137725830078,
      "activations/layer23_attention_weight_min": -23.885128021240234,
      "activations/layer2_attention_weight_max": 33.87465286254883,
      "activations/layer2_attention_weight_min": -30.838722229003906,
      "activations/layer3_attention_weight_max": 93.93339538574219,
      "activations/layer3_attention_weight_min": -93.44395446777344,
      "activations/layer4_attention_weight_max": 98.3741226196289,
      "activations/layer4_attention_weight_min": -97.15460205078125,
      "activations/layer5_attention_weight_max": 72.27014923095703,
      "activations/layer5_attention_weight_min": -81.25082397460938,
      "activations/layer6_attention_weight_max": 53.25790023803711,
      "activations/layer6_attention_weight_min": -53.063880920410156,
      "activations/layer7_attention_weight_max": 75.26740264892578,
      "activations/layer7_attention_weight_min": -70.12024688720703,
      "activations/layer8_attention_weight_max": 45.7650146484375,
      "activations/layer8_attention_weight_min": -47.65742874145508,
      "activations/layer9_attention_weight_max": 51.43117141723633,
      "activations/layer9_attention_weight_min": -49.81541442871094,
      "epoch": 11.52,
      "learning_rate": 7.646628787878787e-05,
      "loss": 2.7807,
      "step": 198300
    },
    {
      "activations/layer0_attention_weight_max": 15.868321418762207,
      "activations/layer0_attention_weight_min": -14.22817325592041,
      "activations/layer10_attention_weight_max": 41.40821838378906,
      "activations/layer10_attention_weight_min": -39.502296447753906,
      "activations/layer11_attention_weight_max": 35.96123504638672,
      "activations/layer11_attention_weight_min": -35.65055847167969,
      "activations/layer12_attention_weight_max": 18.876585006713867,
      "activations/layer12_attention_weight_min": -25.668554306030273,
      "activations/layer13_attention_weight_max": 42.672916412353516,
      "activations/layer13_attention_weight_min": -31.108692169189453,
      "activations/layer14_attention_weight_max": 38.03874206542969,
      "activations/layer14_attention_weight_min": -30.074968338012695,
      "activations/layer15_attention_weight_max": 34.01382827758789,
      "activations/layer15_attention_weight_min": -29.44721221923828,
      "activations/layer16_attention_weight_max": 31.583040237426758,
      "activations/layer16_attention_weight_min": -30.401105880737305,
      "activations/layer17_attention_weight_max": 53.6165657043457,
      "activations/layer17_attention_weight_min": -45.1169319152832,
      "activations/layer18_attention_weight_max": 47.496238708496094,
      "activations/layer18_attention_weight_min": -39.742645263671875,
      "activations/layer19_attention_weight_max": 22.401351928710938,
      "activations/layer19_attention_weight_min": -21.908475875854492,
      "activations/layer1_attention_weight_max": 16.79711151123047,
      "activations/layer1_attention_weight_min": -16.02318572998047,
      "activations/layer20_attention_weight_max": 22.513254165649414,
      "activations/layer20_attention_weight_min": -22.216869354248047,
      "activations/layer21_attention_weight_max": 36.34078598022461,
      "activations/layer21_attention_weight_min": -25.7790470123291,
      "activations/layer22_attention_weight_max": 31.466264724731445,
      "activations/layer22_attention_weight_min": -25.874719619750977,
      "activations/layer23_attention_weight_max": 41.157493591308594,
      "activations/layer23_attention_weight_min": -23.31014633178711,
      "activations/layer2_attention_weight_max": 31.685888290405273,
      "activations/layer2_attention_weight_min": -30.08177947998047,
      "activations/layer3_attention_weight_max": 97.21392822265625,
      "activations/layer3_attention_weight_min": -97.40361022949219,
      "activations/layer4_attention_weight_max": 101.9675064086914,
      "activations/layer4_attention_weight_min": -99.20854187011719,
      "activations/layer5_attention_weight_max": 74.72177124023438,
      "activations/layer5_attention_weight_min": -78.6667709350586,
      "activations/layer6_attention_weight_max": 54.469947814941406,
      "activations/layer6_attention_weight_min": -54.94673156738281,
      "activations/layer7_attention_weight_max": 71.53555297851562,
      "activations/layer7_attention_weight_min": -77.52377319335938,
      "activations/layer8_attention_weight_max": 45.69383239746094,
      "activations/layer8_attention_weight_min": -48.56105422973633,
      "activations/layer9_attention_weight_max": 48.12360763549805,
      "activations/layer9_attention_weight_min": -48.23006820678711,
      "epoch": 11.53,
      "learning_rate": 7.644734848484848e-05,
      "loss": 2.7948,
      "step": 198350
    },
    {
      "activations/layer0_attention_weight_max": 16.379968643188477,
      "activations/layer0_attention_weight_min": -14.307329177856445,
      "activations/layer10_attention_weight_max": 37.83445739746094,
      "activations/layer10_attention_weight_min": -36.56388473510742,
      "activations/layer11_attention_weight_max": 34.92829895019531,
      "activations/layer11_attention_weight_min": -33.06087112426758,
      "activations/layer12_attention_weight_max": 19.111326217651367,
      "activations/layer12_attention_weight_min": -21.83566665649414,
      "activations/layer13_attention_weight_max": 41.22096252441406,
      "activations/layer13_attention_weight_min": -31.40948486328125,
      "activations/layer14_attention_weight_max": 41.86186599731445,
      "activations/layer14_attention_weight_min": -31.067657470703125,
      "activations/layer15_attention_weight_max": 36.27009201049805,
      "activations/layer15_attention_weight_min": -29.523807525634766,
      "activations/layer16_attention_weight_max": 32.46871566772461,
      "activations/layer16_attention_weight_min": -30.50337791442871,
      "activations/layer17_attention_weight_max": 54.84273910522461,
      "activations/layer17_attention_weight_min": -42.68910598754883,
      "activations/layer18_attention_weight_max": 50.91133499145508,
      "activations/layer18_attention_weight_min": -38.505035400390625,
      "activations/layer19_attention_weight_max": 24.71347999572754,
      "activations/layer19_attention_weight_min": -24.756305694580078,
      "activations/layer1_attention_weight_max": 16.128246307373047,
      "activations/layer1_attention_weight_min": -14.706670761108398,
      "activations/layer20_attention_weight_max": 25.650239944458008,
      "activations/layer20_attention_weight_min": -24.686620712280273,
      "activations/layer21_attention_weight_max": 40.38334274291992,
      "activations/layer21_attention_weight_min": -23.7950439453125,
      "activations/layer22_attention_weight_max": 32.27833557128906,
      "activations/layer22_attention_weight_min": -27.0373592376709,
      "activations/layer23_attention_weight_max": 44.217369079589844,
      "activations/layer23_attention_weight_min": -25.185956954956055,
      "activations/layer2_attention_weight_max": 33.14439392089844,
      "activations/layer2_attention_weight_min": -30.53182601928711,
      "activations/layer3_attention_weight_max": 96.2694091796875,
      "activations/layer3_attention_weight_min": -99.9819564819336,
      "activations/layer4_attention_weight_max": 102.20101165771484,
      "activations/layer4_attention_weight_min": -98.52584838867188,
      "activations/layer5_attention_weight_max": 74.09125518798828,
      "activations/layer5_attention_weight_min": -78.97314453125,
      "activations/layer6_attention_weight_max": 54.976261138916016,
      "activations/layer6_attention_weight_min": -57.50984191894531,
      "activations/layer7_attention_weight_max": 74.34288024902344,
      "activations/layer7_attention_weight_min": -71.59935760498047,
      "activations/layer8_attention_weight_max": 46.81718063354492,
      "activations/layer8_attention_weight_min": -47.44620895385742,
      "activations/layer9_attention_weight_max": 48.82206344604492,
      "activations/layer9_attention_weight_min": -49.57847213745117,
      "epoch": 11.53,
      "learning_rate": 7.642840909090909e-05,
      "loss": 2.7945,
      "step": 198400
    },
    {
      "activations/layer0_attention_weight_max": 16.861888885498047,
      "activations/layer0_attention_weight_min": -13.992325782775879,
      "activations/layer10_attention_weight_max": 36.13035583496094,
      "activations/layer10_attention_weight_min": -36.031925201416016,
      "activations/layer11_attention_weight_max": 31.900684356689453,
      "activations/layer11_attention_weight_min": -32.16741943359375,
      "activations/layer12_attention_weight_max": 19.183725357055664,
      "activations/layer12_attention_weight_min": -25.458127975463867,
      "activations/layer13_attention_weight_max": 47.83469009399414,
      "activations/layer13_attention_weight_min": -33.24314880371094,
      "activations/layer14_attention_weight_max": 41.51959991455078,
      "activations/layer14_attention_weight_min": -32.01437759399414,
      "activations/layer15_attention_weight_max": 35.53827667236328,
      "activations/layer15_attention_weight_min": -30.971595764160156,
      "activations/layer16_attention_weight_max": 33.889347076416016,
      "activations/layer16_attention_weight_min": -32.17337417602539,
      "activations/layer17_attention_weight_max": 54.78466796875,
      "activations/layer17_attention_weight_min": -50.069515228271484,
      "activations/layer18_attention_weight_max": 47.56883239746094,
      "activations/layer18_attention_weight_min": -40.84321594238281,
      "activations/layer19_attention_weight_max": 22.98036766052246,
      "activations/layer19_attention_weight_min": -22.47933006286621,
      "activations/layer1_attention_weight_max": 17.514366149902344,
      "activations/layer1_attention_weight_min": -17.130796432495117,
      "activations/layer20_attention_weight_max": 22.171859741210938,
      "activations/layer20_attention_weight_min": -22.48768424987793,
      "activations/layer21_attention_weight_max": 34.94550323486328,
      "activations/layer21_attention_weight_min": -22.3049259185791,
      "activations/layer22_attention_weight_max": 28.429189682006836,
      "activations/layer22_attention_weight_min": -25.800540924072266,
      "activations/layer23_attention_weight_max": 39.55044937133789,
      "activations/layer23_attention_weight_min": -24.608631134033203,
      "activations/layer2_attention_weight_max": 33.588111877441406,
      "activations/layer2_attention_weight_min": -32.14609909057617,
      "activations/layer3_attention_weight_max": 91.0657958984375,
      "activations/layer3_attention_weight_min": -93.87620544433594,
      "activations/layer4_attention_weight_max": 95.91484069824219,
      "activations/layer4_attention_weight_min": -94.06529998779297,
      "activations/layer5_attention_weight_max": 72.48709106445312,
      "activations/layer5_attention_weight_min": -79.22909545898438,
      "activations/layer6_attention_weight_max": 51.39133071899414,
      "activations/layer6_attention_weight_min": -51.810951232910156,
      "activations/layer7_attention_weight_max": 70.57167053222656,
      "activations/layer7_attention_weight_min": -72.61320495605469,
      "activations/layer8_attention_weight_max": 45.28837203979492,
      "activations/layer8_attention_weight_min": -48.383827209472656,
      "activations/layer9_attention_weight_max": 46.84476852416992,
      "activations/layer9_attention_weight_min": -47.788856506347656,
      "epoch": 11.53,
      "learning_rate": 7.640946969696968e-05,
      "loss": 2.7889,
      "step": 198450
    },
    {
      "activations/layer0_attention_weight_max": 15.922198295593262,
      "activations/layer0_attention_weight_min": -14.027673721313477,
      "activations/layer10_attention_weight_max": 38.30318069458008,
      "activations/layer10_attention_weight_min": -38.61879348754883,
      "activations/layer11_attention_weight_max": 35.03694152832031,
      "activations/layer11_attention_weight_min": -34.71683120727539,
      "activations/layer12_attention_weight_max": 17.550256729125977,
      "activations/layer12_attention_weight_min": -23.409629821777344,
      "activations/layer13_attention_weight_max": 42.42596435546875,
      "activations/layer13_attention_weight_min": -34.95607376098633,
      "activations/layer14_attention_weight_max": 38.48441696166992,
      "activations/layer14_attention_weight_min": -33.269065856933594,
      "activations/layer15_attention_weight_max": 40.13497543334961,
      "activations/layer15_attention_weight_min": -31.686567306518555,
      "activations/layer16_attention_weight_max": 30.91977882385254,
      "activations/layer16_attention_weight_min": -29.94908332824707,
      "activations/layer17_attention_weight_max": 52.550994873046875,
      "activations/layer17_attention_weight_min": -50.0562744140625,
      "activations/layer18_attention_weight_max": 46.531463623046875,
      "activations/layer18_attention_weight_min": -42.92116928100586,
      "activations/layer19_attention_weight_max": 21.702306747436523,
      "activations/layer19_attention_weight_min": -23.340208053588867,
      "activations/layer1_attention_weight_max": 17.34943199157715,
      "activations/layer1_attention_weight_min": -16.401033401489258,
      "activations/layer20_attention_weight_max": 21.201274871826172,
      "activations/layer20_attention_weight_min": -21.727964401245117,
      "activations/layer21_attention_weight_max": 40.26518630981445,
      "activations/layer21_attention_weight_min": -26.69948959350586,
      "activations/layer22_attention_weight_max": 31.100341796875,
      "activations/layer22_attention_weight_min": -29.02039909362793,
      "activations/layer23_attention_weight_max": 36.50716018676758,
      "activations/layer23_attention_weight_min": -26.202491760253906,
      "activations/layer2_attention_weight_max": 31.128515243530273,
      "activations/layer2_attention_weight_min": -31.832578659057617,
      "activations/layer3_attention_weight_max": 91.09681701660156,
      "activations/layer3_attention_weight_min": -93.35620880126953,
      "activations/layer4_attention_weight_max": 101.72138977050781,
      "activations/layer4_attention_weight_min": -98.64273071289062,
      "activations/layer5_attention_weight_max": 71.88336181640625,
      "activations/layer5_attention_weight_min": -80.4832992553711,
      "activations/layer6_attention_weight_max": 51.377464294433594,
      "activations/layer6_attention_weight_min": -52.15050506591797,
      "activations/layer7_attention_weight_max": 74.24578094482422,
      "activations/layer7_attention_weight_min": -71.01077270507812,
      "activations/layer8_attention_weight_max": 47.03921127319336,
      "activations/layer8_attention_weight_min": -47.52428436279297,
      "activations/layer9_attention_weight_max": 57.20781326293945,
      "activations/layer9_attention_weight_min": -51.23309326171875,
      "epoch": 11.53,
      "learning_rate": 7.63905303030303e-05,
      "loss": 2.7751,
      "step": 198500
    },
    {
      "activations/layer0_attention_weight_max": 16.46371078491211,
      "activations/layer0_attention_weight_min": -13.986083030700684,
      "activations/layer10_attention_weight_max": 38.49895095825195,
      "activations/layer10_attention_weight_min": -36.7799186706543,
      "activations/layer11_attention_weight_max": 32.527259826660156,
      "activations/layer11_attention_weight_min": -33.32501220703125,
      "activations/layer12_attention_weight_max": 19.243667602539062,
      "activations/layer12_attention_weight_min": -25.761869430541992,
      "activations/layer13_attention_weight_max": 38.943477630615234,
      "activations/layer13_attention_weight_min": -30.64081573486328,
      "activations/layer14_attention_weight_max": 34.53045654296875,
      "activations/layer14_attention_weight_min": -30.28325843811035,
      "activations/layer15_attention_weight_max": 32.50405502319336,
      "activations/layer15_attention_weight_min": -29.876577377319336,
      "activations/layer16_attention_weight_max": 30.776044845581055,
      "activations/layer16_attention_weight_min": -29.515352249145508,
      "activations/layer17_attention_weight_max": 54.9488639831543,
      "activations/layer17_attention_weight_min": -43.571102142333984,
      "activations/layer18_attention_weight_max": 47.48778533935547,
      "activations/layer18_attention_weight_min": -40.64210891723633,
      "activations/layer19_attention_weight_max": 28.569067001342773,
      "activations/layer19_attention_weight_min": -22.68190574645996,
      "activations/layer1_attention_weight_max": 16.58843994140625,
      "activations/layer1_attention_weight_min": -14.979206085205078,
      "activations/layer20_attention_weight_max": 26.0343017578125,
      "activations/layer20_attention_weight_min": -21.617765426635742,
      "activations/layer21_attention_weight_max": 40.62090301513672,
      "activations/layer21_attention_weight_min": -26.578996658325195,
      "activations/layer22_attention_weight_max": 32.11262512207031,
      "activations/layer22_attention_weight_min": -26.980602264404297,
      "activations/layer23_attention_weight_max": 41.295989990234375,
      "activations/layer23_attention_weight_min": -26.761281967163086,
      "activations/layer2_attention_weight_max": 33.0352668762207,
      "activations/layer2_attention_weight_min": -30.80671501159668,
      "activations/layer3_attention_weight_max": 100.30921936035156,
      "activations/layer3_attention_weight_min": -102.55879974365234,
      "activations/layer4_attention_weight_max": 102.79100799560547,
      "activations/layer4_attention_weight_min": -102.59513092041016,
      "activations/layer5_attention_weight_max": 74.35612487792969,
      "activations/layer5_attention_weight_min": -79.99205780029297,
      "activations/layer6_attention_weight_max": 56.26025390625,
      "activations/layer6_attention_weight_min": -56.00731658935547,
      "activations/layer7_attention_weight_max": 76.20845794677734,
      "activations/layer7_attention_weight_min": -75.07229614257812,
      "activations/layer8_attention_weight_max": 47.879886627197266,
      "activations/layer8_attention_weight_min": -48.60071563720703,
      "activations/layer9_attention_weight_max": 47.0424919128418,
      "activations/layer9_attention_weight_min": -50.634769439697266,
      "epoch": 11.54,
      "learning_rate": 7.637159090909091e-05,
      "loss": 2.7731,
      "step": 198550
    },
    {
      "activations/layer0_attention_weight_max": 16.042829513549805,
      "activations/layer0_attention_weight_min": -13.912047386169434,
      "activations/layer10_attention_weight_max": 37.403194427490234,
      "activations/layer10_attention_weight_min": -35.435882568359375,
      "activations/layer11_attention_weight_max": 32.067535400390625,
      "activations/layer11_attention_weight_min": -33.1595458984375,
      "activations/layer12_attention_weight_max": 20.188444137573242,
      "activations/layer12_attention_weight_min": -21.010000228881836,
      "activations/layer13_attention_weight_max": 42.38584899902344,
      "activations/layer13_attention_weight_min": -29.173770904541016,
      "activations/layer14_attention_weight_max": 42.0106315612793,
      "activations/layer14_attention_weight_min": -30.980789184570312,
      "activations/layer15_attention_weight_max": 37.86222839355469,
      "activations/layer15_attention_weight_min": -30.1563777923584,
      "activations/layer16_attention_weight_max": 38.18301773071289,
      "activations/layer16_attention_weight_min": -34.764190673828125,
      "activations/layer17_attention_weight_max": 61.15798568725586,
      "activations/layer17_attention_weight_min": -48.12329864501953,
      "activations/layer18_attention_weight_max": 55.4139404296875,
      "activations/layer18_attention_weight_min": -39.90536880493164,
      "activations/layer19_attention_weight_max": 27.421554565429688,
      "activations/layer19_attention_weight_min": -22.278717041015625,
      "activations/layer1_attention_weight_max": 15.957804679870605,
      "activations/layer1_attention_weight_min": -15.000880241394043,
      "activations/layer20_attention_weight_max": 26.347944259643555,
      "activations/layer20_attention_weight_min": -23.162351608276367,
      "activations/layer21_attention_weight_max": 39.431678771972656,
      "activations/layer21_attention_weight_min": -21.46684455871582,
      "activations/layer22_attention_weight_max": 35.9316291809082,
      "activations/layer22_attention_weight_min": -26.21396827697754,
      "activations/layer23_attention_weight_max": 41.78545379638672,
      "activations/layer23_attention_weight_min": -24.988422393798828,
      "activations/layer2_attention_weight_max": 28.726375579833984,
      "activations/layer2_attention_weight_min": -26.779800415039062,
      "activations/layer3_attention_weight_max": 92.52532196044922,
      "activations/layer3_attention_weight_min": -89.22989654541016,
      "activations/layer4_attention_weight_max": 102.65118408203125,
      "activations/layer4_attention_weight_min": -99.3049545288086,
      "activations/layer5_attention_weight_max": 74.3595962524414,
      "activations/layer5_attention_weight_min": -79.86150360107422,
      "activations/layer6_attention_weight_max": 52.68183135986328,
      "activations/layer6_attention_weight_min": -52.897605895996094,
      "activations/layer7_attention_weight_max": 77.35707092285156,
      "activations/layer7_attention_weight_min": -70.74475860595703,
      "activations/layer8_attention_weight_max": 46.59840774536133,
      "activations/layer8_attention_weight_min": -48.29191207885742,
      "activations/layer9_attention_weight_max": 55.83390426635742,
      "activations/layer9_attention_weight_min": -46.09602355957031,
      "epoch": 11.54,
      "learning_rate": 7.63526515151515e-05,
      "loss": 2.7665,
      "step": 198600
    },
    {
      "activations/layer0_attention_weight_max": 15.521448135375977,
      "activations/layer0_attention_weight_min": -13.340360641479492,
      "activations/layer10_attention_weight_max": 37.146514892578125,
      "activations/layer10_attention_weight_min": -36.69847106933594,
      "activations/layer11_attention_weight_max": 32.40923309326172,
      "activations/layer11_attention_weight_min": -33.48065185546875,
      "activations/layer12_attention_weight_max": 19.140226364135742,
      "activations/layer12_attention_weight_min": -26.651107788085938,
      "activations/layer13_attention_weight_max": 42.09956359863281,
      "activations/layer13_attention_weight_min": -34.865562438964844,
      "activations/layer14_attention_weight_max": 39.73579406738281,
      "activations/layer14_attention_weight_min": -31.466819763183594,
      "activations/layer15_attention_weight_max": 31.906299591064453,
      "activations/layer15_attention_weight_min": -29.90854835510254,
      "activations/layer16_attention_weight_max": 27.616159439086914,
      "activations/layer16_attention_weight_min": -27.28485107421875,
      "activations/layer17_attention_weight_max": 51.11058807373047,
      "activations/layer17_attention_weight_min": -42.8592414855957,
      "activations/layer18_attention_weight_max": 45.91730880737305,
      "activations/layer18_attention_weight_min": -37.00304412841797,
      "activations/layer19_attention_weight_max": 21.761924743652344,
      "activations/layer19_attention_weight_min": -21.8436222076416,
      "activations/layer1_attention_weight_max": 16.937368392944336,
      "activations/layer1_attention_weight_min": -14.904690742492676,
      "activations/layer20_attention_weight_max": 22.573469161987305,
      "activations/layer20_attention_weight_min": -27.017669677734375,
      "activations/layer21_attention_weight_max": 38.789161682128906,
      "activations/layer21_attention_weight_min": -21.891569137573242,
      "activations/layer22_attention_weight_max": 30.24805450439453,
      "activations/layer22_attention_weight_min": -25.646400451660156,
      "activations/layer23_attention_weight_max": 36.4463005065918,
      "activations/layer23_attention_weight_min": -23.947444915771484,
      "activations/layer2_attention_weight_max": 35.07139587402344,
      "activations/layer2_attention_weight_min": -33.22385787963867,
      "activations/layer3_attention_weight_max": 98.8765869140625,
      "activations/layer3_attention_weight_min": -101.01488494873047,
      "activations/layer4_attention_weight_max": 102.40361785888672,
      "activations/layer4_attention_weight_min": -98.29161071777344,
      "activations/layer5_attention_weight_max": 75.03721618652344,
      "activations/layer5_attention_weight_min": -81.6519775390625,
      "activations/layer6_attention_weight_max": 54.96546173095703,
      "activations/layer6_attention_weight_min": -52.23930740356445,
      "activations/layer7_attention_weight_max": 81.24304962158203,
      "activations/layer7_attention_weight_min": -74.62458038330078,
      "activations/layer8_attention_weight_max": 45.31853103637695,
      "activations/layer8_attention_weight_min": -48.042633056640625,
      "activations/layer9_attention_weight_max": 47.563480377197266,
      "activations/layer9_attention_weight_min": -48.589359283447266,
      "epoch": 11.54,
      "learning_rate": 7.633371212121211e-05,
      "loss": 2.7912,
      "step": 198650
    },
    {
      "activations/layer0_attention_weight_max": 15.40817928314209,
      "activations/layer0_attention_weight_min": -14.253443717956543,
      "activations/layer10_attention_weight_max": 35.80473709106445,
      "activations/layer10_attention_weight_min": -36.56007766723633,
      "activations/layer11_attention_weight_max": 31.9461727142334,
      "activations/layer11_attention_weight_min": -33.995811462402344,
      "activations/layer12_attention_weight_max": 16.899961471557617,
      "activations/layer12_attention_weight_min": -26.619102478027344,
      "activations/layer13_attention_weight_max": 41.438262939453125,
      "activations/layer13_attention_weight_min": -34.38682174682617,
      "activations/layer14_attention_weight_max": 41.818031311035156,
      "activations/layer14_attention_weight_min": -31.664506912231445,
      "activations/layer15_attention_weight_max": 42.054256439208984,
      "activations/layer15_attention_weight_min": -31.892471313476562,
      "activations/layer16_attention_weight_max": 31.141878128051758,
      "activations/layer16_attention_weight_min": -32.23176193237305,
      "activations/layer17_attention_weight_max": 51.43507766723633,
      "activations/layer17_attention_weight_min": -51.11729431152344,
      "activations/layer18_attention_weight_max": 46.75118637084961,
      "activations/layer18_attention_weight_min": -46.72107696533203,
      "activations/layer19_attention_weight_max": 24.518970489501953,
      "activations/layer19_attention_weight_min": -23.997678756713867,
      "activations/layer1_attention_weight_max": 17.26627540588379,
      "activations/layer1_attention_weight_min": -16.959877014160156,
      "activations/layer20_attention_weight_max": 25.956464767456055,
      "activations/layer20_attention_weight_min": -23.319425582885742,
      "activations/layer21_attention_weight_max": 36.21337890625,
      "activations/layer21_attention_weight_min": -23.957250595092773,
      "activations/layer22_attention_weight_max": 28.451690673828125,
      "activations/layer22_attention_weight_min": -26.642438888549805,
      "activations/layer23_attention_weight_max": 37.704349517822266,
      "activations/layer23_attention_weight_min": -24.932445526123047,
      "activations/layer2_attention_weight_max": 31.382713317871094,
      "activations/layer2_attention_weight_min": -30.94521713256836,
      "activations/layer3_attention_weight_max": 89.08377075195312,
      "activations/layer3_attention_weight_min": -96.5670394897461,
      "activations/layer4_attention_weight_max": 98.88800048828125,
      "activations/layer4_attention_weight_min": -100.66766357421875,
      "activations/layer5_attention_weight_max": 73.17831420898438,
      "activations/layer5_attention_weight_min": -78.33262634277344,
      "activations/layer6_attention_weight_max": 55.3370246887207,
      "activations/layer6_attention_weight_min": -55.43870162963867,
      "activations/layer7_attention_weight_max": 74.70543670654297,
      "activations/layer7_attention_weight_min": -74.98011016845703,
      "activations/layer8_attention_weight_max": 51.19603729248047,
      "activations/layer8_attention_weight_min": -50.500389099121094,
      "activations/layer9_attention_weight_max": 55.46163558959961,
      "activations/layer9_attention_weight_min": -49.32900619506836,
      "epoch": 11.55,
      "learning_rate": 7.631477272727273e-05,
      "loss": 2.7623,
      "step": 198700
    },
    {
      "activations/layer0_attention_weight_max": 14.911141395568848,
      "activations/layer0_attention_weight_min": -14.356369972229004,
      "activations/layer10_attention_weight_max": 32.74908447265625,
      "activations/layer10_attention_weight_min": -33.37778091430664,
      "activations/layer11_attention_weight_max": 29.492841720581055,
      "activations/layer11_attention_weight_min": -29.487356185913086,
      "activations/layer12_attention_weight_max": 17.798221588134766,
      "activations/layer12_attention_weight_min": -28.0681095123291,
      "activations/layer13_attention_weight_max": 42.61137008666992,
      "activations/layer13_attention_weight_min": -32.95921325683594,
      "activations/layer14_attention_weight_max": 41.79069900512695,
      "activations/layer14_attention_weight_min": -31.0925350189209,
      "activations/layer15_attention_weight_max": 38.16783142089844,
      "activations/layer15_attention_weight_min": -30.594005584716797,
      "activations/layer16_attention_weight_max": 35.19514465332031,
      "activations/layer16_attention_weight_min": -30.742740631103516,
      "activations/layer17_attention_weight_max": 57.13294219970703,
      "activations/layer17_attention_weight_min": -46.5517578125,
      "activations/layer18_attention_weight_max": 54.24473190307617,
      "activations/layer18_attention_weight_min": -41.891761779785156,
      "activations/layer19_attention_weight_max": 26.698307037353516,
      "activations/layer19_attention_weight_min": -22.455312728881836,
      "activations/layer1_attention_weight_max": 16.228193283081055,
      "activations/layer1_attention_weight_min": -14.788816452026367,
      "activations/layer20_attention_weight_max": 27.38848876953125,
      "activations/layer20_attention_weight_min": -22.22504425048828,
      "activations/layer21_attention_weight_max": 53.29825973510742,
      "activations/layer21_attention_weight_min": -26.402233123779297,
      "activations/layer22_attention_weight_max": 38.5405387878418,
      "activations/layer22_attention_weight_min": -26.31574058532715,
      "activations/layer23_attention_weight_max": 44.64628601074219,
      "activations/layer23_attention_weight_min": -24.363920211791992,
      "activations/layer2_attention_weight_max": 33.07415008544922,
      "activations/layer2_attention_weight_min": -31.398685455322266,
      "activations/layer3_attention_weight_max": 90.77784729003906,
      "activations/layer3_attention_weight_min": -96.66011810302734,
      "activations/layer4_attention_weight_max": 95.54828643798828,
      "activations/layer4_attention_weight_min": -92.62652587890625,
      "activations/layer5_attention_weight_max": 69.33390808105469,
      "activations/layer5_attention_weight_min": -77.18382263183594,
      "activations/layer6_attention_weight_max": 50.594459533691406,
      "activations/layer6_attention_weight_min": -50.585914611816406,
      "activations/layer7_attention_weight_max": 76.28279876708984,
      "activations/layer7_attention_weight_min": -68.55430603027344,
      "activations/layer8_attention_weight_max": 43.002647399902344,
      "activations/layer8_attention_weight_min": -46.3465461730957,
      "activations/layer9_attention_weight_max": 45.505104064941406,
      "activations/layer9_attention_weight_min": -46.16789627075195,
      "epoch": 11.55,
      "learning_rate": 7.629583333333333e-05,
      "loss": 2.7719,
      "step": 198750
    },
    {
      "activations/layer0_attention_weight_max": 15.590134620666504,
      "activations/layer0_attention_weight_min": -14.236522674560547,
      "activations/layer10_attention_weight_max": 38.538780212402344,
      "activations/layer10_attention_weight_min": -35.673431396484375,
      "activations/layer11_attention_weight_max": 36.130218505859375,
      "activations/layer11_attention_weight_min": -32.502113342285156,
      "activations/layer12_attention_weight_max": 20.15534782409668,
      "activations/layer12_attention_weight_min": -23.933591842651367,
      "activations/layer13_attention_weight_max": 38.42033004760742,
      "activations/layer13_attention_weight_min": -29.6826114654541,
      "activations/layer14_attention_weight_max": 35.33949279785156,
      "activations/layer14_attention_weight_min": -30.840099334716797,
      "activations/layer15_attention_weight_max": 30.44791030883789,
      "activations/layer15_attention_weight_min": -29.814191818237305,
      "activations/layer16_attention_weight_max": 30.680740356445312,
      "activations/layer16_attention_weight_min": -29.209228515625,
      "activations/layer17_attention_weight_max": 53.22861099243164,
      "activations/layer17_attention_weight_min": -43.558837890625,
      "activations/layer18_attention_weight_max": 44.91305923461914,
      "activations/layer18_attention_weight_min": -39.19481658935547,
      "activations/layer19_attention_weight_max": 22.2863826751709,
      "activations/layer19_attention_weight_min": -22.818805694580078,
      "activations/layer1_attention_weight_max": 16.903514862060547,
      "activations/layer1_attention_weight_min": -15.282478332519531,
      "activations/layer20_attention_weight_max": 22.30797576904297,
      "activations/layer20_attention_weight_min": -23.66765594482422,
      "activations/layer21_attention_weight_max": 38.74345779418945,
      "activations/layer21_attention_weight_min": -22.713817596435547,
      "activations/layer22_attention_weight_max": 30.61376190185547,
      "activations/layer22_attention_weight_min": -26.11216163635254,
      "activations/layer23_attention_weight_max": 39.123687744140625,
      "activations/layer23_attention_weight_min": -23.184478759765625,
      "activations/layer2_attention_weight_max": 32.11408233642578,
      "activations/layer2_attention_weight_min": -30.705947875976562,
      "activations/layer3_attention_weight_max": 87.8427963256836,
      "activations/layer3_attention_weight_min": -92.12318420410156,
      "activations/layer4_attention_weight_max": 99.74571990966797,
      "activations/layer4_attention_weight_min": -94.70939636230469,
      "activations/layer5_attention_weight_max": 70.76364135742188,
      "activations/layer5_attention_weight_min": -78.46180725097656,
      "activations/layer6_attention_weight_max": 54.36872100830078,
      "activations/layer6_attention_weight_min": -50.6174201965332,
      "activations/layer7_attention_weight_max": 72.42628479003906,
      "activations/layer7_attention_weight_min": -70.96271514892578,
      "activations/layer8_attention_weight_max": 46.31341552734375,
      "activations/layer8_attention_weight_min": -47.36788558959961,
      "activations/layer9_attention_weight_max": 49.34832763671875,
      "activations/layer9_attention_weight_min": -47.084083557128906,
      "epoch": 11.55,
      "learning_rate": 7.627689393939393e-05,
      "loss": 2.762,
      "step": 198800
    },
    {
      "activations/layer0_attention_weight_max": 15.967928886413574,
      "activations/layer0_attention_weight_min": -14.038355827331543,
      "activations/layer10_attention_weight_max": 37.05865478515625,
      "activations/layer10_attention_weight_min": -35.36582565307617,
      "activations/layer11_attention_weight_max": 32.364078521728516,
      "activations/layer11_attention_weight_min": -31.192319869995117,
      "activations/layer12_attention_weight_max": 27.24527931213379,
      "activations/layer12_attention_weight_min": -24.345619201660156,
      "activations/layer13_attention_weight_max": 35.68421173095703,
      "activations/layer13_attention_weight_min": -29.294710159301758,
      "activations/layer14_attention_weight_max": 34.99778366088867,
      "activations/layer14_attention_weight_min": -29.30774688720703,
      "activations/layer15_attention_weight_max": 34.21133804321289,
      "activations/layer15_attention_weight_min": -28.83045768737793,
      "activations/layer16_attention_weight_max": 30.732755661010742,
      "activations/layer16_attention_weight_min": -29.385770797729492,
      "activations/layer17_attention_weight_max": 53.88851547241211,
      "activations/layer17_attention_weight_min": -43.70659637451172,
      "activations/layer18_attention_weight_max": 45.86795425415039,
      "activations/layer18_attention_weight_min": -39.23888397216797,
      "activations/layer19_attention_weight_max": 24.66878318786621,
      "activations/layer19_attention_weight_min": -23.702932357788086,
      "activations/layer1_attention_weight_max": 17.86667251586914,
      "activations/layer1_attention_weight_min": -14.236076354980469,
      "activations/layer20_attention_weight_max": 22.230192184448242,
      "activations/layer20_attention_weight_min": -23.3731746673584,
      "activations/layer21_attention_weight_max": 39.524871826171875,
      "activations/layer21_attention_weight_min": -24.20378303527832,
      "activations/layer22_attention_weight_max": 28.2651309967041,
      "activations/layer22_attention_weight_min": -27.025375366210938,
      "activations/layer23_attention_weight_max": 35.572261810302734,
      "activations/layer23_attention_weight_min": -27.49439239501953,
      "activations/layer2_attention_weight_max": 32.68914031982422,
      "activations/layer2_attention_weight_min": -29.601255416870117,
      "activations/layer3_attention_weight_max": 89.5098876953125,
      "activations/layer3_attention_weight_min": -94.28816223144531,
      "activations/layer4_attention_weight_max": 95.8697280883789,
      "activations/layer4_attention_weight_min": -93.64989471435547,
      "activations/layer5_attention_weight_max": 72.94560241699219,
      "activations/layer5_attention_weight_min": -75.74014282226562,
      "activations/layer6_attention_weight_max": 50.082603454589844,
      "activations/layer6_attention_weight_min": -50.37983703613281,
      "activations/layer7_attention_weight_max": 73.27742767333984,
      "activations/layer7_attention_weight_min": -72.46744537353516,
      "activations/layer8_attention_weight_max": 42.610408782958984,
      "activations/layer8_attention_weight_min": -47.251548767089844,
      "activations/layer9_attention_weight_max": 44.310333251953125,
      "activations/layer9_attention_weight_min": -45.48712921142578,
      "epoch": 11.55,
      "learning_rate": 7.625795454545455e-05,
      "loss": 2.7758,
      "step": 198850
    },
    {
      "activations/layer0_attention_weight_max": 15.015741348266602,
      "activations/layer0_attention_weight_min": -14.224474906921387,
      "activations/layer10_attention_weight_max": 37.14468765258789,
      "activations/layer10_attention_weight_min": -34.90007019042969,
      "activations/layer11_attention_weight_max": 30.78087615966797,
      "activations/layer11_attention_weight_min": -33.25606155395508,
      "activations/layer12_attention_weight_max": 17.919631958007812,
      "activations/layer12_attention_weight_min": -26.17707633972168,
      "activations/layer13_attention_weight_max": 40.67890167236328,
      "activations/layer13_attention_weight_min": -28.78770637512207,
      "activations/layer14_attention_weight_max": 42.20105743408203,
      "activations/layer14_attention_weight_min": -28.380353927612305,
      "activations/layer15_attention_weight_max": 35.92982864379883,
      "activations/layer15_attention_weight_min": -29.623605728149414,
      "activations/layer16_attention_weight_max": 32.3275260925293,
      "activations/layer16_attention_weight_min": -28.134891510009766,
      "activations/layer17_attention_weight_max": 50.081695556640625,
      "activations/layer17_attention_weight_min": -40.673221588134766,
      "activations/layer18_attention_weight_max": 42.369815826416016,
      "activations/layer18_attention_weight_min": -36.18027877807617,
      "activations/layer19_attention_weight_max": 23.188507080078125,
      "activations/layer19_attention_weight_min": -20.413721084594727,
      "activations/layer1_attention_weight_max": 18.061378479003906,
      "activations/layer1_attention_weight_min": -15.543289184570312,
      "activations/layer20_attention_weight_max": 21.984373092651367,
      "activations/layer20_attention_weight_min": -21.207042694091797,
      "activations/layer21_attention_weight_max": 39.67755126953125,
      "activations/layer21_attention_weight_min": -21.8074951171875,
      "activations/layer22_attention_weight_max": 31.544757843017578,
      "activations/layer22_attention_weight_min": -25.83506965637207,
      "activations/layer23_attention_weight_max": 36.96296310424805,
      "activations/layer23_attention_weight_min": -23.310529708862305,
      "activations/layer2_attention_weight_max": 31.240293502807617,
      "activations/layer2_attention_weight_min": -30.512937545776367,
      "activations/layer3_attention_weight_max": 89.39736938476562,
      "activations/layer3_attention_weight_min": -95.69844818115234,
      "activations/layer4_attention_weight_max": 94.78191375732422,
      "activations/layer4_attention_weight_min": -95.37271118164062,
      "activations/layer5_attention_weight_max": 71.7156982421875,
      "activations/layer5_attention_weight_min": -77.98289489746094,
      "activations/layer6_attention_weight_max": 50.651607513427734,
      "activations/layer6_attention_weight_min": -51.005558013916016,
      "activations/layer7_attention_weight_max": 72.05342864990234,
      "activations/layer7_attention_weight_min": -68.47089385986328,
      "activations/layer8_attention_weight_max": 44.36847686767578,
      "activations/layer8_attention_weight_min": -48.60469055175781,
      "activations/layer9_attention_weight_max": 43.44045639038086,
      "activations/layer9_attention_weight_min": -46.932701110839844,
      "epoch": 11.56,
      "learning_rate": 7.623901515151515e-05,
      "loss": 2.7825,
      "step": 198900
    },
    {
      "activations/layer0_attention_weight_max": 16.504804611206055,
      "activations/layer0_attention_weight_min": -14.631510734558105,
      "activations/layer10_attention_weight_max": 37.288429260253906,
      "activations/layer10_attention_weight_min": -35.29368209838867,
      "activations/layer11_attention_weight_max": 32.78846740722656,
      "activations/layer11_attention_weight_min": -33.676292419433594,
      "activations/layer12_attention_weight_max": 17.91238784790039,
      "activations/layer12_attention_weight_min": -23.528743743896484,
      "activations/layer13_attention_weight_max": 35.10663604736328,
      "activations/layer13_attention_weight_min": -29.263628005981445,
      "activations/layer14_attention_weight_max": 33.19118881225586,
      "activations/layer14_attention_weight_min": -31.663557052612305,
      "activations/layer15_attention_weight_max": 31.467161178588867,
      "activations/layer15_attention_weight_min": -30.444589614868164,
      "activations/layer16_attention_weight_max": 29.184650421142578,
      "activations/layer16_attention_weight_min": -29.6331844329834,
      "activations/layer17_attention_weight_max": 48.51314163208008,
      "activations/layer17_attention_weight_min": -43.331390380859375,
      "activations/layer18_attention_weight_max": 43.44477844238281,
      "activations/layer18_attention_weight_min": -37.97598648071289,
      "activations/layer19_attention_weight_max": 23.24593162536621,
      "activations/layer19_attention_weight_min": -21.500106811523438,
      "activations/layer1_attention_weight_max": 16.691667556762695,
      "activations/layer1_attention_weight_min": -14.98221492767334,
      "activations/layer20_attention_weight_max": 23.339439392089844,
      "activations/layer20_attention_weight_min": -21.433259963989258,
      "activations/layer21_attention_weight_max": 36.712337493896484,
      "activations/layer21_attention_weight_min": -23.647817611694336,
      "activations/layer22_attention_weight_max": 28.838512420654297,
      "activations/layer22_attention_weight_min": -25.325660705566406,
      "activations/layer23_attention_weight_max": 35.80909729003906,
      "activations/layer23_attention_weight_min": -23.903470993041992,
      "activations/layer2_attention_weight_max": 32.4202995300293,
      "activations/layer2_attention_weight_min": -30.219314575195312,
      "activations/layer3_attention_weight_max": 94.89404296875,
      "activations/layer3_attention_weight_min": -93.83026123046875,
      "activations/layer4_attention_weight_max": 102.9632568359375,
      "activations/layer4_attention_weight_min": -97.8442153930664,
      "activations/layer5_attention_weight_max": 74.74903869628906,
      "activations/layer5_attention_weight_min": -81.95218658447266,
      "activations/layer6_attention_weight_max": 54.26908493041992,
      "activations/layer6_attention_weight_min": -53.42210388183594,
      "activations/layer7_attention_weight_max": 79.69500732421875,
      "activations/layer7_attention_weight_min": -71.40532684326172,
      "activations/layer8_attention_weight_max": 45.97583770751953,
      "activations/layer8_attention_weight_min": -47.61510467529297,
      "activations/layer9_attention_weight_max": 47.56870651245117,
      "activations/layer9_attention_weight_min": -47.17716979980469,
      "epoch": 11.56,
      "learning_rate": 7.622007575757575e-05,
      "loss": 2.7679,
      "step": 198950
    },
    {
      "activations/layer0_attention_weight_max": 15.516077995300293,
      "activations/layer0_attention_weight_min": -14.525790214538574,
      "activations/layer10_attention_weight_max": 36.883338928222656,
      "activations/layer10_attention_weight_min": -35.67544937133789,
      "activations/layer11_attention_weight_max": 33.93037033081055,
      "activations/layer11_attention_weight_min": -35.672386169433594,
      "activations/layer12_attention_weight_max": 21.808143615722656,
      "activations/layer12_attention_weight_min": -25.72821807861328,
      "activations/layer13_attention_weight_max": 41.62037658691406,
      "activations/layer13_attention_weight_min": -28.859771728515625,
      "activations/layer14_attention_weight_max": 40.30818557739258,
      "activations/layer14_attention_weight_min": -31.54778289794922,
      "activations/layer15_attention_weight_max": 40.11273956298828,
      "activations/layer15_attention_weight_min": -28.963150024414062,
      "activations/layer16_attention_weight_max": 32.42823028564453,
      "activations/layer16_attention_weight_min": -29.349767684936523,
      "activations/layer17_attention_weight_max": 52.79993438720703,
      "activations/layer17_attention_weight_min": -44.13423156738281,
      "activations/layer18_attention_weight_max": 50.685508728027344,
      "activations/layer18_attention_weight_min": -38.20736312866211,
      "activations/layer19_attention_weight_max": 26.029956817626953,
      "activations/layer19_attention_weight_min": -22.015419006347656,
      "activations/layer1_attention_weight_max": 16.56802749633789,
      "activations/layer1_attention_weight_min": -14.732257843017578,
      "activations/layer20_attention_weight_max": 25.94432258605957,
      "activations/layer20_attention_weight_min": -23.51190757751465,
      "activations/layer21_attention_weight_max": 46.981834411621094,
      "activations/layer21_attention_weight_min": -26.713592529296875,
      "activations/layer22_attention_weight_max": 35.8643798828125,
      "activations/layer22_attention_weight_min": -26.546066284179688,
      "activations/layer23_attention_weight_max": 49.201805114746094,
      "activations/layer23_attention_weight_min": -25.82242202758789,
      "activations/layer2_attention_weight_max": 32.82951736450195,
      "activations/layer2_attention_weight_min": -32.012611389160156,
      "activations/layer3_attention_weight_max": 95.39289855957031,
      "activations/layer3_attention_weight_min": -99.5157241821289,
      "activations/layer4_attention_weight_max": 106.7353286743164,
      "activations/layer4_attention_weight_min": -104.13579559326172,
      "activations/layer5_attention_weight_max": 79.20646667480469,
      "activations/layer5_attention_weight_min": -82.93428039550781,
      "activations/layer6_attention_weight_max": 55.13125991821289,
      "activations/layer6_attention_weight_min": -55.447914123535156,
      "activations/layer7_attention_weight_max": 76.73487854003906,
      "activations/layer7_attention_weight_min": -73.71495056152344,
      "activations/layer8_attention_weight_max": 49.359596252441406,
      "activations/layer8_attention_weight_min": -49.25767135620117,
      "activations/layer9_attention_weight_max": 49.34238815307617,
      "activations/layer9_attention_weight_min": -48.2173957824707,
      "epoch": 11.56,
      "learning_rate": 7.620113636363635e-05,
      "loss": 2.7949,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.5303,
      "eval_samples_per_second": 503.382,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.5303,
      "eval_openwebtext_samples_per_second": 503.382,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.9845,
      "eval_wikitext_samples_per_second": 229.784,
      "step": 199000
    },
    {
      "epoch": 11.56,
      "eval_lambada_loss": 2.572265625,
      "eval_lambada_ppl": 13.095460258581369,
      "eval_lambada_runtime": 9.6238,
      "eval_lambada_samples_per_second": 505.932,
      "step": 199000
    },
    {
      "activations/layer0_attention_weight_max": 15.912755012512207,
      "activations/layer0_attention_weight_min": -14.427531242370605,
      "activations/layer10_attention_weight_max": 35.646087646484375,
      "activations/layer10_attention_weight_min": -35.93974685668945,
      "activations/layer11_attention_weight_max": 33.05708312988281,
      "activations/layer11_attention_weight_min": -33.963409423828125,
      "activations/layer12_attention_weight_max": 17.882583618164062,
      "activations/layer12_attention_weight_min": -24.724029541015625,
      "activations/layer13_attention_weight_max": 40.572608947753906,
      "activations/layer13_attention_weight_min": -30.611812591552734,
      "activations/layer14_attention_weight_max": 36.33879852294922,
      "activations/layer14_attention_weight_min": -31.602710723876953,
      "activations/layer15_attention_weight_max": 35.11890411376953,
      "activations/layer15_attention_weight_min": -28.60344123840332,
      "activations/layer16_attention_weight_max": 33.90509796142578,
      "activations/layer16_attention_weight_min": -31.189952850341797,
      "activations/layer17_attention_weight_max": 56.669376373291016,
      "activations/layer17_attention_weight_min": -44.70003128051758,
      "activations/layer18_attention_weight_max": 48.8029899597168,
      "activations/layer18_attention_weight_min": -39.55629348754883,
      "activations/layer19_attention_weight_max": 23.35775375366211,
      "activations/layer19_attention_weight_min": -23.542524337768555,
      "activations/layer1_attention_weight_max": 17.1240234375,
      "activations/layer1_attention_weight_min": -14.453668594360352,
      "activations/layer20_attention_weight_max": 23.799589157104492,
      "activations/layer20_attention_weight_min": -24.276273727416992,
      "activations/layer21_attention_weight_max": 38.33993911743164,
      "activations/layer21_attention_weight_min": -25.579736709594727,
      "activations/layer22_attention_weight_max": 34.26041793823242,
      "activations/layer22_attention_weight_min": -25.23894691467285,
      "activations/layer23_attention_weight_max": 42.86733627319336,
      "activations/layer23_attention_weight_min": -23.485309600830078,
      "activations/layer2_attention_weight_max": 33.17323303222656,
      "activations/layer2_attention_weight_min": -31.87074089050293,
      "activations/layer3_attention_weight_max": 97.96101379394531,
      "activations/layer3_attention_weight_min": -100.26964569091797,
      "activations/layer4_attention_weight_max": 104.51972961425781,
      "activations/layer4_attention_weight_min": -100.627685546875,
      "activations/layer5_attention_weight_max": 75.88117980957031,
      "activations/layer5_attention_weight_min": -80.36668395996094,
      "activations/layer6_attention_weight_max": 55.769493103027344,
      "activations/layer6_attention_weight_min": -55.30244445800781,
      "activations/layer7_attention_weight_max": 76.56656646728516,
      "activations/layer7_attention_weight_min": -71.82393646240234,
      "activations/layer8_attention_weight_max": 45.634464263916016,
      "activations/layer8_attention_weight_min": -48.35365676879883,
      "activations/layer9_attention_weight_max": 48.168704986572266,
      "activations/layer9_attention_weight_min": -48.03822708129883,
      "epoch": 11.57,
      "learning_rate": 7.618219696969696e-05,
      "loss": 2.7631,
      "step": 199050
    },
    {
      "activations/layer0_attention_weight_max": 15.657902717590332,
      "activations/layer0_attention_weight_min": -14.215529441833496,
      "activations/layer10_attention_weight_max": 37.58643341064453,
      "activations/layer10_attention_weight_min": -34.21759033203125,
      "activations/layer11_attention_weight_max": 33.352943420410156,
      "activations/layer11_attention_weight_min": -34.466514587402344,
      "activations/layer12_attention_weight_max": 20.126359939575195,
      "activations/layer12_attention_weight_min": -23.062931060791016,
      "activations/layer13_attention_weight_max": 40.0484504699707,
      "activations/layer13_attention_weight_min": -32.86229705810547,
      "activations/layer14_attention_weight_max": 39.967750549316406,
      "activations/layer14_attention_weight_min": -31.305477142333984,
      "activations/layer15_attention_weight_max": 38.109458923339844,
      "activations/layer15_attention_weight_min": -30.81958770751953,
      "activations/layer16_attention_weight_max": 34.72215270996094,
      "activations/layer16_attention_weight_min": -30.09881019592285,
      "activations/layer17_attention_weight_max": 55.8129997253418,
      "activations/layer17_attention_weight_min": -45.732818603515625,
      "activations/layer18_attention_weight_max": 49.64688491821289,
      "activations/layer18_attention_weight_min": -40.25505065917969,
      "activations/layer19_attention_weight_max": 28.687335968017578,
      "activations/layer19_attention_weight_min": -23.704986572265625,
      "activations/layer1_attention_weight_max": 17.43121910095215,
      "activations/layer1_attention_weight_min": -13.543925285339355,
      "activations/layer20_attention_weight_max": 26.640209197998047,
      "activations/layer20_attention_weight_min": -21.5633487701416,
      "activations/layer21_attention_weight_max": 40.03720474243164,
      "activations/layer21_attention_weight_min": -23.6634464263916,
      "activations/layer22_attention_weight_max": 34.694114685058594,
      "activations/layer22_attention_weight_min": -28.933387756347656,
      "activations/layer23_attention_weight_max": 42.29878234863281,
      "activations/layer23_attention_weight_min": -23.206066131591797,
      "activations/layer2_attention_weight_max": 31.902530670166016,
      "activations/layer2_attention_weight_min": -30.46717643737793,
      "activations/layer3_attention_weight_max": 92.44119262695312,
      "activations/layer3_attention_weight_min": -94.40013885498047,
      "activations/layer4_attention_weight_max": 99.86404418945312,
      "activations/layer4_attention_weight_min": -97.20233154296875,
      "activations/layer5_attention_weight_max": 73.61680603027344,
      "activations/layer5_attention_weight_min": -76.49994659423828,
      "activations/layer6_attention_weight_max": 54.97862243652344,
      "activations/layer6_attention_weight_min": -53.482276916503906,
      "activations/layer7_attention_weight_max": 72.15615844726562,
      "activations/layer7_attention_weight_min": -70.32769775390625,
      "activations/layer8_attention_weight_max": 44.29277801513672,
      "activations/layer8_attention_weight_min": -45.47028350830078,
      "activations/layer9_attention_weight_max": 47.711631774902344,
      "activations/layer9_attention_weight_min": -46.43137741088867,
      "epoch": 11.57,
      "learning_rate": 7.616325757575758e-05,
      "loss": 2.7878,
      "step": 199100
    },
    {
      "activations/layer0_attention_weight_max": 15.444822311401367,
      "activations/layer0_attention_weight_min": -14.633152961730957,
      "activations/layer10_attention_weight_max": 38.99705505371094,
      "activations/layer10_attention_weight_min": -40.174903869628906,
      "activations/layer11_attention_weight_max": 34.48816680908203,
      "activations/layer11_attention_weight_min": -33.51832580566406,
      "activations/layer12_attention_weight_max": 21.38027000427246,
      "activations/layer12_attention_weight_min": -25.357316970825195,
      "activations/layer13_attention_weight_max": 45.13371658325195,
      "activations/layer13_attention_weight_min": -36.056461334228516,
      "activations/layer14_attention_weight_max": 55.64188003540039,
      "activations/layer14_attention_weight_min": -33.460479736328125,
      "activations/layer15_attention_weight_max": 46.07101821899414,
      "activations/layer15_attention_weight_min": -31.89205551147461,
      "activations/layer16_attention_weight_max": 32.54257583618164,
      "activations/layer16_attention_weight_min": -31.505054473876953,
      "activations/layer17_attention_weight_max": 51.09360885620117,
      "activations/layer17_attention_weight_min": -45.28653335571289,
      "activations/layer18_attention_weight_max": 45.07637405395508,
      "activations/layer18_attention_weight_min": -41.47039031982422,
      "activations/layer19_attention_weight_max": 21.4234676361084,
      "activations/layer19_attention_weight_min": -23.132160186767578,
      "activations/layer1_attention_weight_max": 16.12471580505371,
      "activations/layer1_attention_weight_min": -16.671772003173828,
      "activations/layer20_attention_weight_max": 23.258947372436523,
      "activations/layer20_attention_weight_min": -24.586320877075195,
      "activations/layer21_attention_weight_max": 40.37498474121094,
      "activations/layer21_attention_weight_min": -26.6314697265625,
      "activations/layer22_attention_weight_max": 30.564640045166016,
      "activations/layer22_attention_weight_min": -27.039779663085938,
      "activations/layer23_attention_weight_max": 37.291446685791016,
      "activations/layer23_attention_weight_min": -25.76596450805664,
      "activations/layer2_attention_weight_max": 34.682037353515625,
      "activations/layer2_attention_weight_min": -31.02224349975586,
      "activations/layer3_attention_weight_max": 97.44608306884766,
      "activations/layer3_attention_weight_min": -99.00993347167969,
      "activations/layer4_attention_weight_max": 102.6297836303711,
      "activations/layer4_attention_weight_min": -99.15865325927734,
      "activations/layer5_attention_weight_max": 76.61911010742188,
      "activations/layer5_attention_weight_min": -81.97183990478516,
      "activations/layer6_attention_weight_max": 54.60917663574219,
      "activations/layer6_attention_weight_min": -52.770328521728516,
      "activations/layer7_attention_weight_max": 73.40837097167969,
      "activations/layer7_attention_weight_min": -74.43546295166016,
      "activations/layer8_attention_weight_max": 48.35843276977539,
      "activations/layer8_attention_weight_min": -47.39595413208008,
      "activations/layer9_attention_weight_max": 56.258583068847656,
      "activations/layer9_attention_weight_min": -48.692901611328125,
      "epoch": 11.57,
      "learning_rate": 7.614431818181817e-05,
      "loss": 2.7843,
      "step": 199150
    },
    {
      "activations/layer0_attention_weight_max": 15.423742294311523,
      "activations/layer0_attention_weight_min": -14.83558177947998,
      "activations/layer10_attention_weight_max": 40.233585357666016,
      "activations/layer10_attention_weight_min": -35.20292663574219,
      "activations/layer11_attention_weight_max": 34.75543212890625,
      "activations/layer11_attention_weight_min": -33.8237190246582,
      "activations/layer12_attention_weight_max": 20.548507690429688,
      "activations/layer12_attention_weight_min": -25.491317749023438,
      "activations/layer13_attention_weight_max": 38.012569427490234,
      "activations/layer13_attention_weight_min": -29.844970703125,
      "activations/layer14_attention_weight_max": 37.967185974121094,
      "activations/layer14_attention_weight_min": -31.80327606201172,
      "activations/layer15_attention_weight_max": 34.444881439208984,
      "activations/layer15_attention_weight_min": -32.3648796081543,
      "activations/layer16_attention_weight_max": 30.905046463012695,
      "activations/layer16_attention_weight_min": -31.01457977294922,
      "activations/layer17_attention_weight_max": 54.661956787109375,
      "activations/layer17_attention_weight_min": -45.731597900390625,
      "activations/layer18_attention_weight_max": 45.921783447265625,
      "activations/layer18_attention_weight_min": -38.6417121887207,
      "activations/layer19_attention_weight_max": 23.607097625732422,
      "activations/layer19_attention_weight_min": -21.882156372070312,
      "activations/layer1_attention_weight_max": 16.171510696411133,
      "activations/layer1_attention_weight_min": -15.4588623046875,
      "activations/layer20_attention_weight_max": 21.31997299194336,
      "activations/layer20_attention_weight_min": -23.193574905395508,
      "activations/layer21_attention_weight_max": 36.132415771484375,
      "activations/layer21_attention_weight_min": -22.40326499938965,
      "activations/layer22_attention_weight_max": 28.969297409057617,
      "activations/layer22_attention_weight_min": -26.355379104614258,
      "activations/layer23_attention_weight_max": 40.878509521484375,
      "activations/layer23_attention_weight_min": -24.820350646972656,
      "activations/layer2_attention_weight_max": 33.08262634277344,
      "activations/layer2_attention_weight_min": -31.078672409057617,
      "activations/layer3_attention_weight_max": 94.34741973876953,
      "activations/layer3_attention_weight_min": -93.71087646484375,
      "activations/layer4_attention_weight_max": 98.63704681396484,
      "activations/layer4_attention_weight_min": -92.99962615966797,
      "activations/layer5_attention_weight_max": 70.15477752685547,
      "activations/layer5_attention_weight_min": -75.07149505615234,
      "activations/layer6_attention_weight_max": 54.25634002685547,
      "activations/layer6_attention_weight_min": -52.44648361206055,
      "activations/layer7_attention_weight_max": 73.92823791503906,
      "activations/layer7_attention_weight_min": -72.6231918334961,
      "activations/layer8_attention_weight_max": 45.83295440673828,
      "activations/layer8_attention_weight_min": -47.4771842956543,
      "activations/layer9_attention_weight_max": 47.573272705078125,
      "activations/layer9_attention_weight_min": -45.20640182495117,
      "epoch": 11.57,
      "learning_rate": 7.612537878787878e-05,
      "loss": 2.7709,
      "step": 199200
    },
    {
      "activations/layer0_attention_weight_max": 15.89488697052002,
      "activations/layer0_attention_weight_min": -14.422833442687988,
      "activations/layer10_attention_weight_max": 38.786434173583984,
      "activations/layer10_attention_weight_min": -37.858272552490234,
      "activations/layer11_attention_weight_max": 31.4832763671875,
      "activations/layer11_attention_weight_min": -33.11134719848633,
      "activations/layer12_attention_weight_max": 18.61040687561035,
      "activations/layer12_attention_weight_min": -23.56959342956543,
      "activations/layer13_attention_weight_max": 37.20009994506836,
      "activations/layer13_attention_weight_min": -29.13846206665039,
      "activations/layer14_attention_weight_max": 36.00867462158203,
      "activations/layer14_attention_weight_min": -31.13450813293457,
      "activations/layer15_attention_weight_max": 33.867027282714844,
      "activations/layer15_attention_weight_min": -29.713031768798828,
      "activations/layer16_attention_weight_max": 29.71620750427246,
      "activations/layer16_attention_weight_min": -31.922407150268555,
      "activations/layer17_attention_weight_max": 50.13188934326172,
      "activations/layer17_attention_weight_min": -45.195308685302734,
      "activations/layer18_attention_weight_max": 43.162113189697266,
      "activations/layer18_attention_weight_min": -38.1135368347168,
      "activations/layer19_attention_weight_max": 21.005596160888672,
      "activations/layer19_attention_weight_min": -21.136621475219727,
      "activations/layer1_attention_weight_max": 16.074214935302734,
      "activations/layer1_attention_weight_min": -15.298426628112793,
      "activations/layer20_attention_weight_max": 21.194963455200195,
      "activations/layer20_attention_weight_min": -22.190990447998047,
      "activations/layer21_attention_weight_max": 34.27048110961914,
      "activations/layer21_attention_weight_min": -21.9406681060791,
      "activations/layer22_attention_weight_max": 27.37994956970215,
      "activations/layer22_attention_weight_min": -25.366527557373047,
      "activations/layer23_attention_weight_max": 32.40803527832031,
      "activations/layer23_attention_weight_min": -23.835498809814453,
      "activations/layer2_attention_weight_max": 32.860755920410156,
      "activations/layer2_attention_weight_min": -30.990283966064453,
      "activations/layer3_attention_weight_max": 93.01758575439453,
      "activations/layer3_attention_weight_min": -93.80101013183594,
      "activations/layer4_attention_weight_max": 98.24983978271484,
      "activations/layer4_attention_weight_min": -96.99691772460938,
      "activations/layer5_attention_weight_max": 71.4006118774414,
      "activations/layer5_attention_weight_min": -83.45844268798828,
      "activations/layer6_attention_weight_max": 56.259735107421875,
      "activations/layer6_attention_weight_min": -53.41672897338867,
      "activations/layer7_attention_weight_max": 72.15045166015625,
      "activations/layer7_attention_weight_min": -75.14543151855469,
      "activations/layer8_attention_weight_max": 44.53679275512695,
      "activations/layer8_attention_weight_min": -48.961280822753906,
      "activations/layer9_attention_weight_max": 47.683441162109375,
      "activations/layer9_attention_weight_min": -48.62954330444336,
      "epoch": 11.58,
      "learning_rate": 7.61064393939394e-05,
      "loss": 2.7788,
      "step": 199250
    },
    {
      "activations/layer0_attention_weight_max": 16.27397918701172,
      "activations/layer0_attention_weight_min": -14.822994232177734,
      "activations/layer10_attention_weight_max": 41.21845626831055,
      "activations/layer10_attention_weight_min": -38.31104278564453,
      "activations/layer11_attention_weight_max": 36.2652473449707,
      "activations/layer11_attention_weight_min": -35.01862716674805,
      "activations/layer12_attention_weight_max": 17.906667709350586,
      "activations/layer12_attention_weight_min": -23.772397994995117,
      "activations/layer13_attention_weight_max": 49.48011779785156,
      "activations/layer13_attention_weight_min": -30.73704719543457,
      "activations/layer14_attention_weight_max": 40.803279876708984,
      "activations/layer14_attention_weight_min": -31.795406341552734,
      "activations/layer15_attention_weight_max": 34.31102752685547,
      "activations/layer15_attention_weight_min": -29.479656219482422,
      "activations/layer16_attention_weight_max": 29.154094696044922,
      "activations/layer16_attention_weight_min": -29.73701286315918,
      "activations/layer17_attention_weight_max": 48.267948150634766,
      "activations/layer17_attention_weight_min": -44.71077346801758,
      "activations/layer18_attention_weight_max": 45.41470718383789,
      "activations/layer18_attention_weight_min": -38.825679779052734,
      "activations/layer19_attention_weight_max": 20.134368896484375,
      "activations/layer19_attention_weight_min": -22.724205017089844,
      "activations/layer1_attention_weight_max": 17.748119354248047,
      "activations/layer1_attention_weight_min": -14.759642601013184,
      "activations/layer20_attention_weight_max": 20.492839813232422,
      "activations/layer20_attention_weight_min": -22.56554412841797,
      "activations/layer21_attention_weight_max": 31.30575942993164,
      "activations/layer21_attention_weight_min": -23.868621826171875,
      "activations/layer22_attention_weight_max": 30.183734893798828,
      "activations/layer22_attention_weight_min": -25.33575439453125,
      "activations/layer23_attention_weight_max": 35.426979064941406,
      "activations/layer23_attention_weight_min": -24.06509780883789,
      "activations/layer2_attention_weight_max": 32.996273040771484,
      "activations/layer2_attention_weight_min": -32.8154182434082,
      "activations/layer3_attention_weight_max": 94.05606079101562,
      "activations/layer3_attention_weight_min": -94.2043685913086,
      "activations/layer4_attention_weight_max": 102.46820068359375,
      "activations/layer4_attention_weight_min": -95.3476333618164,
      "activations/layer5_attention_weight_max": 73.7542724609375,
      "activations/layer5_attention_weight_min": -80.7823486328125,
      "activations/layer6_attention_weight_max": 53.31489181518555,
      "activations/layer6_attention_weight_min": -53.271488189697266,
      "activations/layer7_attention_weight_max": 72.69606018066406,
      "activations/layer7_attention_weight_min": -71.5778579711914,
      "activations/layer8_attention_weight_max": 47.60588073730469,
      "activations/layer8_attention_weight_min": -47.362491607666016,
      "activations/layer9_attention_weight_max": 51.81792068481445,
      "activations/layer9_attention_weight_min": -47.693634033203125,
      "epoch": 11.58,
      "learning_rate": 7.608749999999999e-05,
      "loss": 2.7891,
      "step": 199300
    },
    {
      "activations/layer0_attention_weight_max": 16.03736114501953,
      "activations/layer0_attention_weight_min": -14.451059341430664,
      "activations/layer10_attention_weight_max": 35.27997589111328,
      "activations/layer10_attention_weight_min": -37.7409782409668,
      "activations/layer11_attention_weight_max": 32.94984817504883,
      "activations/layer11_attention_weight_min": -34.01279067993164,
      "activations/layer12_attention_weight_max": 20.6298828125,
      "activations/layer12_attention_weight_min": -25.7653751373291,
      "activations/layer13_attention_weight_max": 35.0234260559082,
      "activations/layer13_attention_weight_min": -29.03601837158203,
      "activations/layer14_attention_weight_max": 41.12845230102539,
      "activations/layer14_attention_weight_min": -31.522171020507812,
      "activations/layer15_attention_weight_max": 32.74077606201172,
      "activations/layer15_attention_weight_min": -30.05394744873047,
      "activations/layer16_attention_weight_max": 33.030479431152344,
      "activations/layer16_attention_weight_min": -30.082233428955078,
      "activations/layer17_attention_weight_max": 54.07088851928711,
      "activations/layer17_attention_weight_min": -45.96529769897461,
      "activations/layer18_attention_weight_max": 49.1747932434082,
      "activations/layer18_attention_weight_min": -41.04727554321289,
      "activations/layer19_attention_weight_max": 21.265193939208984,
      "activations/layer19_attention_weight_min": -20.489145278930664,
      "activations/layer1_attention_weight_max": 16.774309158325195,
      "activations/layer1_attention_weight_min": -16.16132926940918,
      "activations/layer20_attention_weight_max": 24.202329635620117,
      "activations/layer20_attention_weight_min": -25.161497116088867,
      "activations/layer21_attention_weight_max": 38.29846954345703,
      "activations/layer21_attention_weight_min": -25.444543838500977,
      "activations/layer22_attention_weight_max": 31.15483283996582,
      "activations/layer22_attention_weight_min": -27.039854049682617,
      "activations/layer23_attention_weight_max": 37.756309509277344,
      "activations/layer23_attention_weight_min": -22.2994441986084,
      "activations/layer2_attention_weight_max": 34.70575714111328,
      "activations/layer2_attention_weight_min": -31.24226188659668,
      "activations/layer3_attention_weight_max": 98.11520385742188,
      "activations/layer3_attention_weight_min": -96.38556671142578,
      "activations/layer4_attention_weight_max": 96.3179702758789,
      "activations/layer4_attention_weight_min": -93.17752838134766,
      "activations/layer5_attention_weight_max": 70.42042541503906,
      "activations/layer5_attention_weight_min": -83.41766357421875,
      "activations/layer6_attention_weight_max": 53.809059143066406,
      "activations/layer6_attention_weight_min": -52.70881271362305,
      "activations/layer7_attention_weight_max": 74.82162475585938,
      "activations/layer7_attention_weight_min": -76.40656280517578,
      "activations/layer8_attention_weight_max": 48.19071960449219,
      "activations/layer8_attention_weight_min": -51.88606643676758,
      "activations/layer9_attention_weight_max": 46.686458587646484,
      "activations/layer9_attention_weight_min": -47.62943649291992,
      "epoch": 11.58,
      "learning_rate": 7.60685606060606e-05,
      "loss": 2.7814,
      "step": 199350
    },
    {
      "activations/layer0_attention_weight_max": 15.813451766967773,
      "activations/layer0_attention_weight_min": -14.068222999572754,
      "activations/layer10_attention_weight_max": 35.564449310302734,
      "activations/layer10_attention_weight_min": -34.16661834716797,
      "activations/layer11_attention_weight_max": 34.30769348144531,
      "activations/layer11_attention_weight_min": -33.554656982421875,
      "activations/layer12_attention_weight_max": 17.49147605895996,
      "activations/layer12_attention_weight_min": -25.370601654052734,
      "activations/layer13_attention_weight_max": 40.58816146850586,
      "activations/layer13_attention_weight_min": -32.48582077026367,
      "activations/layer14_attention_weight_max": 51.1928825378418,
      "activations/layer14_attention_weight_min": -29.690229415893555,
      "activations/layer15_attention_weight_max": 42.452789306640625,
      "activations/layer15_attention_weight_min": -30.217803955078125,
      "activations/layer16_attention_weight_max": 29.921993255615234,
      "activations/layer16_attention_weight_min": -28.990781784057617,
      "activations/layer17_attention_weight_max": 50.784149169921875,
      "activations/layer17_attention_weight_min": -42.346099853515625,
      "activations/layer18_attention_weight_max": 45.31676483154297,
      "activations/layer18_attention_weight_min": -36.87214279174805,
      "activations/layer19_attention_weight_max": 22.891496658325195,
      "activations/layer19_attention_weight_min": -21.819055557250977,
      "activations/layer1_attention_weight_max": 16.544702529907227,
      "activations/layer1_attention_weight_min": -14.89218807220459,
      "activations/layer20_attention_weight_max": 23.462928771972656,
      "activations/layer20_attention_weight_min": -21.486122131347656,
      "activations/layer21_attention_weight_max": 37.974334716796875,
      "activations/layer21_attention_weight_min": -23.35838508605957,
      "activations/layer22_attention_weight_max": 31.61151885986328,
      "activations/layer22_attention_weight_min": -26.298189163208008,
      "activations/layer23_attention_weight_max": 42.280330657958984,
      "activations/layer23_attention_weight_min": -26.126956939697266,
      "activations/layer2_attention_weight_max": 33.66498947143555,
      "activations/layer2_attention_weight_min": -31.398900985717773,
      "activations/layer3_attention_weight_max": 95.68423461914062,
      "activations/layer3_attention_weight_min": -101.9134750366211,
      "activations/layer4_attention_weight_max": 98.86773681640625,
      "activations/layer4_attention_weight_min": -96.74835205078125,
      "activations/layer5_attention_weight_max": 74.93061828613281,
      "activations/layer5_attention_weight_min": -79.24090576171875,
      "activations/layer6_attention_weight_max": 53.79595184326172,
      "activations/layer6_attention_weight_min": -54.315956115722656,
      "activations/layer7_attention_weight_max": 72.5108413696289,
      "activations/layer7_attention_weight_min": -67.89163208007812,
      "activations/layer8_attention_weight_max": 45.293365478515625,
      "activations/layer8_attention_weight_min": -47.80223083496094,
      "activations/layer9_attention_weight_max": 46.75978469848633,
      "activations/layer9_attention_weight_min": -47.99531173706055,
      "epoch": 11.59,
      "learning_rate": 7.60496212121212e-05,
      "loss": 2.7785,
      "step": 199400
    },
    {
      "activations/layer0_attention_weight_max": 15.078411102294922,
      "activations/layer0_attention_weight_min": -14.435810089111328,
      "activations/layer10_attention_weight_max": 38.3705940246582,
      "activations/layer10_attention_weight_min": -37.958702087402344,
      "activations/layer11_attention_weight_max": 34.40966796875,
      "activations/layer11_attention_weight_min": -34.66835021972656,
      "activations/layer12_attention_weight_max": 18.68848419189453,
      "activations/layer12_attention_weight_min": -24.863059997558594,
      "activations/layer13_attention_weight_max": 40.67201232910156,
      "activations/layer13_attention_weight_min": -29.844188690185547,
      "activations/layer14_attention_weight_max": 37.91942596435547,
      "activations/layer14_attention_weight_min": -31.47607421875,
      "activations/layer15_attention_weight_max": 32.900909423828125,
      "activations/layer15_attention_weight_min": -30.063875198364258,
      "activations/layer16_attention_weight_max": 32.10398864746094,
      "activations/layer16_attention_weight_min": -28.275239944458008,
      "activations/layer17_attention_weight_max": 50.53013229370117,
      "activations/layer17_attention_weight_min": -43.41426467895508,
      "activations/layer18_attention_weight_max": 44.30482864379883,
      "activations/layer18_attention_weight_min": -37.417118072509766,
      "activations/layer19_attention_weight_max": 22.76416778564453,
      "activations/layer19_attention_weight_min": -23.372478485107422,
      "activations/layer1_attention_weight_max": 16.21649742126465,
      "activations/layer1_attention_weight_min": -15.31409740447998,
      "activations/layer20_attention_weight_max": 23.584556579589844,
      "activations/layer20_attention_weight_min": -23.34597396850586,
      "activations/layer21_attention_weight_max": 38.38257598876953,
      "activations/layer21_attention_weight_min": -24.727062225341797,
      "activations/layer22_attention_weight_max": 30.628747940063477,
      "activations/layer22_attention_weight_min": -26.1122989654541,
      "activations/layer23_attention_weight_max": 40.81083679199219,
      "activations/layer23_attention_weight_min": -24.99620819091797,
      "activations/layer2_attention_weight_max": 33.470855712890625,
      "activations/layer2_attention_weight_min": -32.68273162841797,
      "activations/layer3_attention_weight_max": 98.66957092285156,
      "activations/layer3_attention_weight_min": -98.62051391601562,
      "activations/layer4_attention_weight_max": 105.63201904296875,
      "activations/layer4_attention_weight_min": -99.88716888427734,
      "activations/layer5_attention_weight_max": 74.74577331542969,
      "activations/layer5_attention_weight_min": -81.57190704345703,
      "activations/layer6_attention_weight_max": 56.16843795776367,
      "activations/layer6_attention_weight_min": -54.13960647583008,
      "activations/layer7_attention_weight_max": 75.9976806640625,
      "activations/layer7_attention_weight_min": -73.03761291503906,
      "activations/layer8_attention_weight_max": 48.56134796142578,
      "activations/layer8_attention_weight_min": -48.65020751953125,
      "activations/layer9_attention_weight_max": 52.418697357177734,
      "activations/layer9_attention_weight_min": -47.49079132080078,
      "epoch": 11.59,
      "learning_rate": 7.60306818181818e-05,
      "loss": 2.7815,
      "step": 199450
    },
    {
      "activations/layer0_attention_weight_max": 15.792587280273438,
      "activations/layer0_attention_weight_min": -14.442742347717285,
      "activations/layer10_attention_weight_max": 38.569026947021484,
      "activations/layer10_attention_weight_min": -35.463226318359375,
      "activations/layer11_attention_weight_max": 31.996795654296875,
      "activations/layer11_attention_weight_min": -33.400508880615234,
      "activations/layer12_attention_weight_max": 21.94463348388672,
      "activations/layer12_attention_weight_min": -25.387451171875,
      "activations/layer13_attention_weight_max": 42.36549377441406,
      "activations/layer13_attention_weight_min": -30.722797393798828,
      "activations/layer14_attention_weight_max": 38.128013610839844,
      "activations/layer14_attention_weight_min": -31.246538162231445,
      "activations/layer15_attention_weight_max": 35.79946517944336,
      "activations/layer15_attention_weight_min": -29.987394332885742,
      "activations/layer16_attention_weight_max": 36.27727508544922,
      "activations/layer16_attention_weight_min": -30.9371337890625,
      "activations/layer17_attention_weight_max": 57.54221725463867,
      "activations/layer17_attention_weight_min": -47.729454040527344,
      "activations/layer18_attention_weight_max": 54.818416595458984,
      "activations/layer18_attention_weight_min": -42.06548309326172,
      "activations/layer19_attention_weight_max": 25.57158088684082,
      "activations/layer19_attention_weight_min": -22.27631187438965,
      "activations/layer1_attention_weight_max": 17.13729476928711,
      "activations/layer1_attention_weight_min": -15.322108268737793,
      "activations/layer20_attention_weight_max": 24.507762908935547,
      "activations/layer20_attention_weight_min": -21.542531967163086,
      "activations/layer21_attention_weight_max": 44.07111740112305,
      "activations/layer21_attention_weight_min": -24.864151000976562,
      "activations/layer22_attention_weight_max": 36.82900619506836,
      "activations/layer22_attention_weight_min": -27.52577781677246,
      "activations/layer23_attention_weight_max": 40.57509994506836,
      "activations/layer23_attention_weight_min": -25.39275360107422,
      "activations/layer2_attention_weight_max": 34.91313934326172,
      "activations/layer2_attention_weight_min": -30.965248107910156,
      "activations/layer3_attention_weight_max": 95.36152648925781,
      "activations/layer3_attention_weight_min": -96.57318115234375,
      "activations/layer4_attention_weight_max": 101.22225189208984,
      "activations/layer4_attention_weight_min": -98.53375244140625,
      "activations/layer5_attention_weight_max": 73.8392105102539,
      "activations/layer5_attention_weight_min": -81.9006118774414,
      "activations/layer6_attention_weight_max": 58.026573181152344,
      "activations/layer6_attention_weight_min": -58.69928741455078,
      "activations/layer7_attention_weight_max": 79.29911041259766,
      "activations/layer7_attention_weight_min": -72.43565368652344,
      "activations/layer8_attention_weight_max": 44.9753532409668,
      "activations/layer8_attention_weight_min": -47.208595275878906,
      "activations/layer9_attention_weight_max": 48.396297454833984,
      "activations/layer9_attention_weight_min": -47.457881927490234,
      "epoch": 11.59,
      "learning_rate": 7.601174242424242e-05,
      "loss": 2.7768,
      "step": 199500
    },
    {
      "activations/layer0_attention_weight_max": 16.69417953491211,
      "activations/layer0_attention_weight_min": -14.559774398803711,
      "activations/layer10_attention_weight_max": 36.3952522277832,
      "activations/layer10_attention_weight_min": -36.05614471435547,
      "activations/layer11_attention_weight_max": 31.528905868530273,
      "activations/layer11_attention_weight_min": -32.039608001708984,
      "activations/layer12_attention_weight_max": 24.440521240234375,
      "activations/layer12_attention_weight_min": -25.021018981933594,
      "activations/layer13_attention_weight_max": 41.131690979003906,
      "activations/layer13_attention_weight_min": -27.21576499938965,
      "activations/layer14_attention_weight_max": 40.83235168457031,
      "activations/layer14_attention_weight_min": -31.128755569458008,
      "activations/layer15_attention_weight_max": 36.04314422607422,
      "activations/layer15_attention_weight_min": -30.14974594116211,
      "activations/layer16_attention_weight_max": 35.466556549072266,
      "activations/layer16_attention_weight_min": -29.12645149230957,
      "activations/layer17_attention_weight_max": 58.8544921875,
      "activations/layer17_attention_weight_min": -47.351402282714844,
      "activations/layer18_attention_weight_max": 49.175819396972656,
      "activations/layer18_attention_weight_min": -38.04274368286133,
      "activations/layer19_attention_weight_max": 26.17380142211914,
      "activations/layer19_attention_weight_min": -21.86016273498535,
      "activations/layer1_attention_weight_max": 17.19388198852539,
      "activations/layer1_attention_weight_min": -15.638816833496094,
      "activations/layer20_attention_weight_max": 23.7309513092041,
      "activations/layer20_attention_weight_min": -22.06228256225586,
      "activations/layer21_attention_weight_max": 39.602149963378906,
      "activations/layer21_attention_weight_min": -22.281944274902344,
      "activations/layer22_attention_weight_max": 34.73453903198242,
      "activations/layer22_attention_weight_min": -26.172212600708008,
      "activations/layer23_attention_weight_max": 40.84084701538086,
      "activations/layer23_attention_weight_min": -23.87967300415039,
      "activations/layer2_attention_weight_max": 33.93183135986328,
      "activations/layer2_attention_weight_min": -33.256134033203125,
      "activations/layer3_attention_weight_max": 95.144775390625,
      "activations/layer3_attention_weight_min": -96.34146118164062,
      "activations/layer4_attention_weight_max": 102.3701171875,
      "activations/layer4_attention_weight_min": -95.1844482421875,
      "activations/layer5_attention_weight_max": 72.8612289428711,
      "activations/layer5_attention_weight_min": -79.4163818359375,
      "activations/layer6_attention_weight_max": 56.7601318359375,
      "activations/layer6_attention_weight_min": -53.24049377441406,
      "activations/layer7_attention_weight_max": 77.0778579711914,
      "activations/layer7_attention_weight_min": -72.93174743652344,
      "activations/layer8_attention_weight_max": 47.307552337646484,
      "activations/layer8_attention_weight_min": -47.25704574584961,
      "activations/layer9_attention_weight_max": 48.704856872558594,
      "activations/layer9_attention_weight_min": -44.57929611206055,
      "epoch": 11.59,
      "learning_rate": 7.599280303030302e-05,
      "loss": 2.7704,
      "step": 199550
    },
    {
      "activations/layer0_attention_weight_max": 15.769384384155273,
      "activations/layer0_attention_weight_min": -14.349576950073242,
      "activations/layer10_attention_weight_max": 46.217613220214844,
      "activations/layer10_attention_weight_min": -41.25307083129883,
      "activations/layer11_attention_weight_max": 42.30812454223633,
      "activations/layer11_attention_weight_min": -38.15972137451172,
      "activations/layer12_attention_weight_max": 25.847970962524414,
      "activations/layer12_attention_weight_min": -28.860958099365234,
      "activations/layer13_attention_weight_max": 37.68358612060547,
      "activations/layer13_attention_weight_min": -27.882457733154297,
      "activations/layer14_attention_weight_max": 39.55671310424805,
      "activations/layer14_attention_weight_min": -29.928205490112305,
      "activations/layer15_attention_weight_max": 32.02153778076172,
      "activations/layer15_attention_weight_min": -29.63960838317871,
      "activations/layer16_attention_weight_max": 32.836669921875,
      "activations/layer16_attention_weight_min": -29.250078201293945,
      "activations/layer17_attention_weight_max": 52.0601806640625,
      "activations/layer17_attention_weight_min": -42.2880744934082,
      "activations/layer18_attention_weight_max": 44.438194274902344,
      "activations/layer18_attention_weight_min": -36.57927703857422,
      "activations/layer19_attention_weight_max": 22.42070960998535,
      "activations/layer19_attention_weight_min": -21.500486373901367,
      "activations/layer1_attention_weight_max": 16.953052520751953,
      "activations/layer1_attention_weight_min": -16.312223434448242,
      "activations/layer20_attention_weight_max": 22.930814743041992,
      "activations/layer20_attention_weight_min": -20.13698387145996,
      "activations/layer21_attention_weight_max": 33.45223617553711,
      "activations/layer21_attention_weight_min": -21.56960105895996,
      "activations/layer22_attention_weight_max": 30.11280059814453,
      "activations/layer22_attention_weight_min": -27.94473648071289,
      "activations/layer23_attention_weight_max": 40.08136749267578,
      "activations/layer23_attention_weight_min": -23.248332977294922,
      "activations/layer2_attention_weight_max": 36.69521713256836,
      "activations/layer2_attention_weight_min": -33.770809173583984,
      "activations/layer3_attention_weight_max": 101.73023223876953,
      "activations/layer3_attention_weight_min": -106.29009246826172,
      "activations/layer4_attention_weight_max": 103.8857650756836,
      "activations/layer4_attention_weight_min": -103.28466033935547,
      "activations/layer5_attention_weight_max": 77.08477783203125,
      "activations/layer5_attention_weight_min": -81.45491027832031,
      "activations/layer6_attention_weight_max": 57.346221923828125,
      "activations/layer6_attention_weight_min": -59.73865509033203,
      "activations/layer7_attention_weight_max": 83.39134216308594,
      "activations/layer7_attention_weight_min": -77.47856903076172,
      "activations/layer8_attention_weight_max": 53.98250198364258,
      "activations/layer8_attention_weight_min": -54.90060043334961,
      "activations/layer9_attention_weight_max": 68.19420623779297,
      "activations/layer9_attention_weight_min": -55.92058181762695,
      "epoch": 11.6,
      "learning_rate": 7.597386363636364e-05,
      "loss": 2.7829,
      "step": 199600
    },
    {
      "activations/layer0_attention_weight_max": 15.767363548278809,
      "activations/layer0_attention_weight_min": -14.599627494812012,
      "activations/layer10_attention_weight_max": 34.81532287597656,
      "activations/layer10_attention_weight_min": -34.06303787231445,
      "activations/layer11_attention_weight_max": 31.9013729095459,
      "activations/layer11_attention_weight_min": -32.36540603637695,
      "activations/layer12_attention_weight_max": 18.15594482421875,
      "activations/layer12_attention_weight_min": -30.570510864257812,
      "activations/layer13_attention_weight_max": 39.61017608642578,
      "activations/layer13_attention_weight_min": -31.15092658996582,
      "activations/layer14_attention_weight_max": 37.71321105957031,
      "activations/layer14_attention_weight_min": -30.588102340698242,
      "activations/layer15_attention_weight_max": 33.73036193847656,
      "activations/layer15_attention_weight_min": -29.597599029541016,
      "activations/layer16_attention_weight_max": 32.98908615112305,
      "activations/layer16_attention_weight_min": -31.853174209594727,
      "activations/layer17_attention_weight_max": 54.33644104003906,
      "activations/layer17_attention_weight_min": -42.31917190551758,
      "activations/layer18_attention_weight_max": 46.98298645019531,
      "activations/layer18_attention_weight_min": -36.12480163574219,
      "activations/layer19_attention_weight_max": 23.436782836914062,
      "activations/layer19_attention_weight_min": -21.452733993530273,
      "activations/layer1_attention_weight_max": 16.27288818359375,
      "activations/layer1_attention_weight_min": -14.684006690979004,
      "activations/layer20_attention_weight_max": 22.88890838623047,
      "activations/layer20_attention_weight_min": -23.283491134643555,
      "activations/layer21_attention_weight_max": 36.43924331665039,
      "activations/layer21_attention_weight_min": -23.997568130493164,
      "activations/layer22_attention_weight_max": 32.689884185791016,
      "activations/layer22_attention_weight_min": -24.965208053588867,
      "activations/layer23_attention_weight_max": 36.70390701293945,
      "activations/layer23_attention_weight_min": -23.28594207763672,
      "activations/layer2_attention_weight_max": 32.73232650756836,
      "activations/layer2_attention_weight_min": -30.544872283935547,
      "activations/layer3_attention_weight_max": 103.3289794921875,
      "activations/layer3_attention_weight_min": -94.0876235961914,
      "activations/layer4_attention_weight_max": 99.25279998779297,
      "activations/layer4_attention_weight_min": -92.29840850830078,
      "activations/layer5_attention_weight_max": 71.41705322265625,
      "activations/layer5_attention_weight_min": -80.39095306396484,
      "activations/layer6_attention_weight_max": 54.35512161254883,
      "activations/layer6_attention_weight_min": -51.03232192993164,
      "activations/layer7_attention_weight_max": 77.38583374023438,
      "activations/layer7_attention_weight_min": -69.53826141357422,
      "activations/layer8_attention_weight_max": 45.0487174987793,
      "activations/layer8_attention_weight_min": -47.50385284423828,
      "activations/layer9_attention_weight_max": 48.82391357421875,
      "activations/layer9_attention_weight_min": -45.669700622558594,
      "epoch": 11.6,
      "learning_rate": 7.595492424242424e-05,
      "loss": 2.7669,
      "step": 199650
    },
    {
      "activations/layer0_attention_weight_max": 15.693471908569336,
      "activations/layer0_attention_weight_min": -14.528160095214844,
      "activations/layer10_attention_weight_max": 39.34977340698242,
      "activations/layer10_attention_weight_min": -38.49592971801758,
      "activations/layer11_attention_weight_max": 35.502281188964844,
      "activations/layer11_attention_weight_min": -34.998294830322266,
      "activations/layer12_attention_weight_max": 21.022214889526367,
      "activations/layer12_attention_weight_min": -24.18223762512207,
      "activations/layer13_attention_weight_max": 44.075828552246094,
      "activations/layer13_attention_weight_min": -31.384929656982422,
      "activations/layer14_attention_weight_max": 41.420814514160156,
      "activations/layer14_attention_weight_min": -32.13944625854492,
      "activations/layer15_attention_weight_max": 37.30575180053711,
      "activations/layer15_attention_weight_min": -29.98297119140625,
      "activations/layer16_attention_weight_max": 32.02016830444336,
      "activations/layer16_attention_weight_min": -29.653892517089844,
      "activations/layer17_attention_weight_max": 54.38426208496094,
      "activations/layer17_attention_weight_min": -46.70491027832031,
      "activations/layer18_attention_weight_max": 50.021705627441406,
      "activations/layer18_attention_weight_min": -41.6397819519043,
      "activations/layer19_attention_weight_max": 22.8083553314209,
      "activations/layer19_attention_weight_min": -23.290952682495117,
      "activations/layer1_attention_weight_max": 16.95938491821289,
      "activations/layer1_attention_weight_min": -15.440898895263672,
      "activations/layer20_attention_weight_max": 24.63947105407715,
      "activations/layer20_attention_weight_min": -21.30731964111328,
      "activations/layer21_attention_weight_max": 45.7083740234375,
      "activations/layer21_attention_weight_min": -24.09647560119629,
      "activations/layer22_attention_weight_max": 31.541170120239258,
      "activations/layer22_attention_weight_min": -26.827939987182617,
      "activations/layer23_attention_weight_max": 41.60546875,
      "activations/layer23_attention_weight_min": -25.067564010620117,
      "activations/layer2_attention_weight_max": 35.06266784667969,
      "activations/layer2_attention_weight_min": -33.506526947021484,
      "activations/layer3_attention_weight_max": 100.42079162597656,
      "activations/layer3_attention_weight_min": -101.05620574951172,
      "activations/layer4_attention_weight_max": 104.7698974609375,
      "activations/layer4_attention_weight_min": -103.90294647216797,
      "activations/layer5_attention_weight_max": 73.42979431152344,
      "activations/layer5_attention_weight_min": -80.7472915649414,
      "activations/layer6_attention_weight_max": 59.44158172607422,
      "activations/layer6_attention_weight_min": -54.5938835144043,
      "activations/layer7_attention_weight_max": 77.83643341064453,
      "activations/layer7_attention_weight_min": -82.27518463134766,
      "activations/layer8_attention_weight_max": 49.79771423339844,
      "activations/layer8_attention_weight_min": -54.2181510925293,
      "activations/layer9_attention_weight_max": 53.7469367980957,
      "activations/layer9_attention_weight_min": -52.05970001220703,
      "epoch": 11.6,
      "learning_rate": 7.593598484848484e-05,
      "loss": 2.7868,
      "step": 199700
    },
    {
      "activations/layer0_attention_weight_max": 15.872763633728027,
      "activations/layer0_attention_weight_min": -14.67052173614502,
      "activations/layer10_attention_weight_max": 37.78595733642578,
      "activations/layer10_attention_weight_min": -35.907135009765625,
      "activations/layer11_attention_weight_max": 36.524574279785156,
      "activations/layer11_attention_weight_min": -32.03639602661133,
      "activations/layer12_attention_weight_max": 21.37958335876465,
      "activations/layer12_attention_weight_min": -26.001819610595703,
      "activations/layer13_attention_weight_max": 48.93728256225586,
      "activations/layer13_attention_weight_min": -33.482666015625,
      "activations/layer14_attention_weight_max": 45.883060455322266,
      "activations/layer14_attention_weight_min": -30.88858985900879,
      "activations/layer15_attention_weight_max": 40.153533935546875,
      "activations/layer15_attention_weight_min": -27.985015869140625,
      "activations/layer16_attention_weight_max": 34.018917083740234,
      "activations/layer16_attention_weight_min": -29.250261306762695,
      "activations/layer17_attention_weight_max": 54.23318099975586,
      "activations/layer17_attention_weight_min": -46.62711715698242,
      "activations/layer18_attention_weight_max": 49.96678924560547,
      "activations/layer18_attention_weight_min": -40.08357620239258,
      "activations/layer19_attention_weight_max": 21.45779800415039,
      "activations/layer19_attention_weight_min": -20.555124282836914,
      "activations/layer1_attention_weight_max": 16.05252456665039,
      "activations/layer1_attention_weight_min": -16.03372573852539,
      "activations/layer20_attention_weight_max": 22.8010311126709,
      "activations/layer20_attention_weight_min": -20.452787399291992,
      "activations/layer21_attention_weight_max": 35.42226028442383,
      "activations/layer21_attention_weight_min": -20.490951538085938,
      "activations/layer22_attention_weight_max": 33.16699981689453,
      "activations/layer22_attention_weight_min": -24.35610580444336,
      "activations/layer23_attention_weight_max": 42.8507194519043,
      "activations/layer23_attention_weight_min": -25.277725219726562,
      "activations/layer2_attention_weight_max": 36.194183349609375,
      "activations/layer2_attention_weight_min": -33.62314987182617,
      "activations/layer3_attention_weight_max": 94.72492980957031,
      "activations/layer3_attention_weight_min": -98.67562103271484,
      "activations/layer4_attention_weight_max": 100.35855865478516,
      "activations/layer4_attention_weight_min": -97.2791976928711,
      "activations/layer5_attention_weight_max": 74.88687133789062,
      "activations/layer5_attention_weight_min": -78.74239349365234,
      "activations/layer6_attention_weight_max": 51.711063385009766,
      "activations/layer6_attention_weight_min": -53.653743743896484,
      "activations/layer7_attention_weight_max": 85.95993041992188,
      "activations/layer7_attention_weight_min": -71.25312042236328,
      "activations/layer8_attention_weight_max": 47.68587112426758,
      "activations/layer8_attention_weight_min": -46.87288284301758,
      "activations/layer9_attention_weight_max": 61.46692657470703,
      "activations/layer9_attention_weight_min": -50.34200668334961,
      "epoch": 11.61,
      "learning_rate": 7.591704545454545e-05,
      "loss": 2.7877,
      "step": 199750
    },
    {
      "activations/layer0_attention_weight_max": 15.254605293273926,
      "activations/layer0_attention_weight_min": -14.531320571899414,
      "activations/layer10_attention_weight_max": 36.48781204223633,
      "activations/layer10_attention_weight_min": -33.842498779296875,
      "activations/layer11_attention_weight_max": 34.836971282958984,
      "activations/layer11_attention_weight_min": -32.556278228759766,
      "activations/layer12_attention_weight_max": 17.44591522216797,
      "activations/layer12_attention_weight_min": -25.782466888427734,
      "activations/layer13_attention_weight_max": 42.90409469604492,
      "activations/layer13_attention_weight_min": -31.020145416259766,
      "activations/layer14_attention_weight_max": 42.71499252319336,
      "activations/layer14_attention_weight_min": -29.228506088256836,
      "activations/layer15_attention_weight_max": 36.453460693359375,
      "activations/layer15_attention_weight_min": -28.990459442138672,
      "activations/layer16_attention_weight_max": 33.02249526977539,
      "activations/layer16_attention_weight_min": -28.95681381225586,
      "activations/layer17_attention_weight_max": 54.12257766723633,
      "activations/layer17_attention_weight_min": -44.944366455078125,
      "activations/layer18_attention_weight_max": 54.70215606689453,
      "activations/layer18_attention_weight_min": -38.9089469909668,
      "activations/layer19_attention_weight_max": 22.56477928161621,
      "activations/layer19_attention_weight_min": -21.99207878112793,
      "activations/layer1_attention_weight_max": 17.761926651000977,
      "activations/layer1_attention_weight_min": -13.871255874633789,
      "activations/layer20_attention_weight_max": 23.974300384521484,
      "activations/layer20_attention_weight_min": -22.148239135742188,
      "activations/layer21_attention_weight_max": 44.976104736328125,
      "activations/layer21_attention_weight_min": -22.97711753845215,
      "activations/layer22_attention_weight_max": 29.903270721435547,
      "activations/layer22_attention_weight_min": -25.890594482421875,
      "activations/layer23_attention_weight_max": 37.751243591308594,
      "activations/layer23_attention_weight_min": -23.836502075195312,
      "activations/layer2_attention_weight_max": 34.101219177246094,
      "activations/layer2_attention_weight_min": -31.504623413085938,
      "activations/layer3_attention_weight_max": 95.17764282226562,
      "activations/layer3_attention_weight_min": -95.64796447753906,
      "activations/layer4_attention_weight_max": 102.98402404785156,
      "activations/layer4_attention_weight_min": -95.90864562988281,
      "activations/layer5_attention_weight_max": 73.50713348388672,
      "activations/layer5_attention_weight_min": -79.60932922363281,
      "activations/layer6_attention_weight_max": 53.066184997558594,
      "activations/layer6_attention_weight_min": -52.90818786621094,
      "activations/layer7_attention_weight_max": 76.44920349121094,
      "activations/layer7_attention_weight_min": -73.83087158203125,
      "activations/layer8_attention_weight_max": 44.46786880493164,
      "activations/layer8_attention_weight_min": -45.503822326660156,
      "activations/layer9_attention_weight_max": 57.7918815612793,
      "activations/layer9_attention_weight_min": -48.8343505859375,
      "epoch": 11.61,
      "learning_rate": 7.589810606060607e-05,
      "loss": 2.7817,
      "step": 199800
    },
    {
      "activations/layer0_attention_weight_max": 15.993865013122559,
      "activations/layer0_attention_weight_min": -14.314159393310547,
      "activations/layer10_attention_weight_max": 36.91688919067383,
      "activations/layer10_attention_weight_min": -35.16819763183594,
      "activations/layer11_attention_weight_max": 32.589515686035156,
      "activations/layer11_attention_weight_min": -32.796226501464844,
      "activations/layer12_attention_weight_max": 20.029203414916992,
      "activations/layer12_attention_weight_min": -26.197240829467773,
      "activations/layer13_attention_weight_max": 36.08341979980469,
      "activations/layer13_attention_weight_min": -28.779159545898438,
      "activations/layer14_attention_weight_max": 38.49699783325195,
      "activations/layer14_attention_weight_min": -31.963293075561523,
      "activations/layer15_attention_weight_max": 35.461544036865234,
      "activations/layer15_attention_weight_min": -30.190935134887695,
      "activations/layer16_attention_weight_max": 31.217647552490234,
      "activations/layer16_attention_weight_min": -30.31915283203125,
      "activations/layer17_attention_weight_max": 53.339691162109375,
      "activations/layer17_attention_weight_min": -46.065391540527344,
      "activations/layer18_attention_weight_max": 49.66621398925781,
      "activations/layer18_attention_weight_min": -40.69252014160156,
      "activations/layer19_attention_weight_max": 26.3487491607666,
      "activations/layer19_attention_weight_min": -22.259727478027344,
      "activations/layer1_attention_weight_max": 16.510772705078125,
      "activations/layer1_attention_weight_min": -13.950491905212402,
      "activations/layer20_attention_weight_max": 26.386823654174805,
      "activations/layer20_attention_weight_min": -19.284629821777344,
      "activations/layer21_attention_weight_max": 34.039337158203125,
      "activations/layer21_attention_weight_min": -20.87029457092285,
      "activations/layer22_attention_weight_max": 32.80942153930664,
      "activations/layer22_attention_weight_min": -27.083786010742188,
      "activations/layer23_attention_weight_max": 47.424556732177734,
      "activations/layer23_attention_weight_min": -24.082502365112305,
      "activations/layer2_attention_weight_max": 31.778099060058594,
      "activations/layer2_attention_weight_min": -29.95265007019043,
      "activations/layer3_attention_weight_max": 95.86788940429688,
      "activations/layer3_attention_weight_min": -99.99671936035156,
      "activations/layer4_attention_weight_max": 102.78541564941406,
      "activations/layer4_attention_weight_min": -97.36259460449219,
      "activations/layer5_attention_weight_max": 73.54608917236328,
      "activations/layer5_attention_weight_min": -81.17610168457031,
      "activations/layer6_attention_weight_max": 60.78056335449219,
      "activations/layer6_attention_weight_min": -56.7249755859375,
      "activations/layer7_attention_weight_max": 82.40376281738281,
      "activations/layer7_attention_weight_min": -70.24165344238281,
      "activations/layer8_attention_weight_max": 49.002227783203125,
      "activations/layer8_attention_weight_min": -49.74274826049805,
      "activations/layer9_attention_weight_max": 48.601863861083984,
      "activations/layer9_attention_weight_min": -46.790245056152344,
      "epoch": 11.61,
      "learning_rate": 7.587916666666666e-05,
      "loss": 2.7839,
      "step": 199850
    },
    {
      "activations/layer0_attention_weight_max": 16.39600372314453,
      "activations/layer0_attention_weight_min": -14.506012916564941,
      "activations/layer10_attention_weight_max": 40.467735290527344,
      "activations/layer10_attention_weight_min": -42.022857666015625,
      "activations/layer11_attention_weight_max": 36.85150909423828,
      "activations/layer11_attention_weight_min": -35.268836975097656,
      "activations/layer12_attention_weight_max": 18.791229248046875,
      "activations/layer12_attention_weight_min": -24.726348876953125,
      "activations/layer13_attention_weight_max": 39.16343688964844,
      "activations/layer13_attention_weight_min": -30.35884666442871,
      "activations/layer14_attention_weight_max": 36.182586669921875,
      "activations/layer14_attention_weight_min": -30.235919952392578,
      "activations/layer15_attention_weight_max": 33.96715545654297,
      "activations/layer15_attention_weight_min": -29.016643524169922,
      "activations/layer16_attention_weight_max": 31.24488067626953,
      "activations/layer16_attention_weight_min": -28.54912567138672,
      "activations/layer17_attention_weight_max": 50.29117965698242,
      "activations/layer17_attention_weight_min": -41.77919006347656,
      "activations/layer18_attention_weight_max": 45.67339324951172,
      "activations/layer18_attention_weight_min": -37.2404670715332,
      "activations/layer19_attention_weight_max": 22.973051071166992,
      "activations/layer19_attention_weight_min": -19.249309539794922,
      "activations/layer1_attention_weight_max": 16.526113510131836,
      "activations/layer1_attention_weight_min": -14.714925765991211,
      "activations/layer20_attention_weight_max": 22.057218551635742,
      "activations/layer20_attention_weight_min": -23.34227180480957,
      "activations/layer21_attention_weight_max": 38.492706298828125,
      "activations/layer21_attention_weight_min": -22.39776039123535,
      "activations/layer22_attention_weight_max": 31.582048416137695,
      "activations/layer22_attention_weight_min": -26.11648941040039,
      "activations/layer23_attention_weight_max": 35.57538604736328,
      "activations/layer23_attention_weight_min": -23.663183212280273,
      "activations/layer2_attention_weight_max": 35.17404556274414,
      "activations/layer2_attention_weight_min": -31.988231658935547,
      "activations/layer3_attention_weight_max": 99.53660583496094,
      "activations/layer3_attention_weight_min": -99.24407196044922,
      "activations/layer4_attention_weight_max": 104.7109375,
      "activations/layer4_attention_weight_min": -102.49718475341797,
      "activations/layer5_attention_weight_max": 77.94215393066406,
      "activations/layer5_attention_weight_min": -81.93561553955078,
      "activations/layer6_attention_weight_max": 54.910987854003906,
      "activations/layer6_attention_weight_min": -56.55492401123047,
      "activations/layer7_attention_weight_max": 78.94100189208984,
      "activations/layer7_attention_weight_min": -81.28064727783203,
      "activations/layer8_attention_weight_max": 51.30403137207031,
      "activations/layer8_attention_weight_min": -50.84650421142578,
      "activations/layer9_attention_weight_max": 62.16974639892578,
      "activations/layer9_attention_weight_min": -50.650962829589844,
      "epoch": 11.62,
      "learning_rate": 7.586022727272727e-05,
      "loss": 2.7582,
      "step": 199900
    },
    {
      "activations/layer0_attention_weight_max": 16.611684799194336,
      "activations/layer0_attention_weight_min": -14.381669044494629,
      "activations/layer10_attention_weight_max": 36.7443733215332,
      "activations/layer10_attention_weight_min": -36.77482223510742,
      "activations/layer11_attention_weight_max": 31.66858673095703,
      "activations/layer11_attention_weight_min": -33.3450813293457,
      "activations/layer12_attention_weight_max": 18.734487533569336,
      "activations/layer12_attention_weight_min": -26.0427188873291,
      "activations/layer13_attention_weight_max": 36.43379211425781,
      "activations/layer13_attention_weight_min": -30.803455352783203,
      "activations/layer14_attention_weight_max": 37.82915496826172,
      "activations/layer14_attention_weight_min": -29.746492385864258,
      "activations/layer15_attention_weight_max": 31.569780349731445,
      "activations/layer15_attention_weight_min": -29.574918746948242,
      "activations/layer16_attention_weight_max": 31.62468719482422,
      "activations/layer16_attention_weight_min": -29.072025299072266,
      "activations/layer17_attention_weight_max": 54.71482849121094,
      "activations/layer17_attention_weight_min": -44.775028228759766,
      "activations/layer18_attention_weight_max": 44.8526496887207,
      "activations/layer18_attention_weight_min": -37.609161376953125,
      "activations/layer19_attention_weight_max": 21.422744750976562,
      "activations/layer19_attention_weight_min": -22.92690658569336,
      "activations/layer1_attention_weight_max": 16.471256256103516,
      "activations/layer1_attention_weight_min": -16.19214630126953,
      "activations/layer20_attention_weight_max": 21.830127716064453,
      "activations/layer20_attention_weight_min": -23.400020599365234,
      "activations/layer21_attention_weight_max": 29.259647369384766,
      "activations/layer21_attention_weight_min": -20.12578582763672,
      "activations/layer22_attention_weight_max": 24.72420310974121,
      "activations/layer22_attention_weight_min": -23.208032608032227,
      "activations/layer23_attention_weight_max": 32.58729553222656,
      "activations/layer23_attention_weight_min": -21.363492965698242,
      "activations/layer2_attention_weight_max": 33.322349548339844,
      "activations/layer2_attention_weight_min": -32.065425872802734,
      "activations/layer3_attention_weight_max": 93.64234161376953,
      "activations/layer3_attention_weight_min": -100.95926666259766,
      "activations/layer4_attention_weight_max": 97.72040557861328,
      "activations/layer4_attention_weight_min": -96.65332794189453,
      "activations/layer5_attention_weight_max": 71.91349792480469,
      "activations/layer5_attention_weight_min": -81.66519927978516,
      "activations/layer6_attention_weight_max": 54.407772064208984,
      "activations/layer6_attention_weight_min": -49.51201248168945,
      "activations/layer7_attention_weight_max": 72.8874282836914,
      "activations/layer7_attention_weight_min": -71.81632995605469,
      "activations/layer8_attention_weight_max": 43.68111801147461,
      "activations/layer8_attention_weight_min": -43.98830032348633,
      "activations/layer9_attention_weight_max": 44.26485824584961,
      "activations/layer9_attention_weight_min": -44.9428596496582,
      "epoch": 11.62,
      "learning_rate": 7.584128787878786e-05,
      "loss": 2.7681,
      "step": 199950
    },
    {
      "activations/layer0_attention_weight_max": 15.580601692199707,
      "activations/layer0_attention_weight_min": -14.381010055541992,
      "activations/layer10_attention_weight_max": 42.17687225341797,
      "activations/layer10_attention_weight_min": -39.915367126464844,
      "activations/layer11_attention_weight_max": 39.06218719482422,
      "activations/layer11_attention_weight_min": -33.93415069580078,
      "activations/layer12_attention_weight_max": 18.01166343688965,
      "activations/layer12_attention_weight_min": -26.74862289428711,
      "activations/layer13_attention_weight_max": 36.003761291503906,
      "activations/layer13_attention_weight_min": -28.27315902709961,
      "activations/layer14_attention_weight_max": 33.90810012817383,
      "activations/layer14_attention_weight_min": -29.59539222717285,
      "activations/layer15_attention_weight_max": 31.187414169311523,
      "activations/layer15_attention_weight_min": -29.30475616455078,
      "activations/layer16_attention_weight_max": 30.74340057373047,
      "activations/layer16_attention_weight_min": -28.442821502685547,
      "activations/layer17_attention_weight_max": 48.38371658325195,
      "activations/layer17_attention_weight_min": -41.879398345947266,
      "activations/layer18_attention_weight_max": 46.19059753417969,
      "activations/layer18_attention_weight_min": -36.44816207885742,
      "activations/layer19_attention_weight_max": 21.31780433654785,
      "activations/layer19_attention_weight_min": -21.549232482910156,
      "activations/layer1_attention_weight_max": 17.48737907409668,
      "activations/layer1_attention_weight_min": -15.502530097961426,
      "activations/layer20_attention_weight_max": 24.341880798339844,
      "activations/layer20_attention_weight_min": -21.196231842041016,
      "activations/layer21_attention_weight_max": 42.400169372558594,
      "activations/layer21_attention_weight_min": -23.910242080688477,
      "activations/layer22_attention_weight_max": 33.5170783996582,
      "activations/layer22_attention_weight_min": -25.99464225769043,
      "activations/layer23_attention_weight_max": 35.56291580200195,
      "activations/layer23_attention_weight_min": -22.52004051208496,
      "activations/layer2_attention_weight_max": 33.721527099609375,
      "activations/layer2_attention_weight_min": -32.807830810546875,
      "activations/layer3_attention_weight_max": 93.89985656738281,
      "activations/layer3_attention_weight_min": -96.23779296875,
      "activations/layer4_attention_weight_max": 99.3849105834961,
      "activations/layer4_attention_weight_min": -94.69982147216797,
      "activations/layer5_attention_weight_max": 72.76831817626953,
      "activations/layer5_attention_weight_min": -79.60404205322266,
      "activations/layer6_attention_weight_max": 52.92255783081055,
      "activations/layer6_attention_weight_min": -52.36543273925781,
      "activations/layer7_attention_weight_max": 83.1655044555664,
      "activations/layer7_attention_weight_min": -68.78734588623047,
      "activations/layer8_attention_weight_max": 49.67585754394531,
      "activations/layer8_attention_weight_min": -45.50407791137695,
      "activations/layer9_attention_weight_max": 63.36887741088867,
      "activations/layer9_attention_weight_min": -48.82891845703125,
      "epoch": 11.62,
      "learning_rate": 7.582234848484847e-05,
      "loss": 2.7773,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.5191,
      "eval_samples_per_second": 504.043,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.5191,
      "eval_openwebtext_samples_per_second": 504.043,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.9984,
      "eval_wikitext_samples_per_second": 228.188,
      "step": 200000
    },
    {
      "epoch": 11.62,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.6229,
      "eval_lambada_samples_per_second": 505.982,
      "step": 200000
    },
    {
      "activations/layer0_attention_weight_max": 16.767349243164062,
      "activations/layer0_attention_weight_min": -14.41596794128418,
      "activations/layer10_attention_weight_max": 35.48571014404297,
      "activations/layer10_attention_weight_min": -35.00788497924805,
      "activations/layer11_attention_weight_max": 32.160072326660156,
      "activations/layer11_attention_weight_min": -30.283628463745117,
      "activations/layer12_attention_weight_max": 22.047998428344727,
      "activations/layer12_attention_weight_min": -27.40732192993164,
      "activations/layer13_attention_weight_max": 38.16255187988281,
      "activations/layer13_attention_weight_min": -26.886598587036133,
      "activations/layer14_attention_weight_max": 35.61651611328125,
      "activations/layer14_attention_weight_min": -29.623138427734375,
      "activations/layer15_attention_weight_max": 31.379676818847656,
      "activations/layer15_attention_weight_min": -29.414148330688477,
      "activations/layer16_attention_weight_max": 31.79414939880371,
      "activations/layer16_attention_weight_min": -28.48111343383789,
      "activations/layer17_attention_weight_max": 55.27511978149414,
      "activations/layer17_attention_weight_min": -45.6888542175293,
      "activations/layer18_attention_weight_max": 50.77544403076172,
      "activations/layer18_attention_weight_min": -40.238685607910156,
      "activations/layer19_attention_weight_max": 23.503671646118164,
      "activations/layer19_attention_weight_min": -23.317625045776367,
      "activations/layer1_attention_weight_max": 16.798362731933594,
      "activations/layer1_attention_weight_min": -14.315500259399414,
      "activations/layer20_attention_weight_max": 23.191307067871094,
      "activations/layer20_attention_weight_min": -23.59611701965332,
      "activations/layer21_attention_weight_max": 48.41668701171875,
      "activations/layer21_attention_weight_min": -24.484825134277344,
      "activations/layer22_attention_weight_max": 30.847572326660156,
      "activations/layer22_attention_weight_min": -26.51074981689453,
      "activations/layer23_attention_weight_max": 39.73802185058594,
      "activations/layer23_attention_weight_min": -22.82757568359375,
      "activations/layer2_attention_weight_max": 33.197349548339844,
      "activations/layer2_attention_weight_min": -32.732452392578125,
      "activations/layer3_attention_weight_max": 93.94032287597656,
      "activations/layer3_attention_weight_min": -98.76045989990234,
      "activations/layer4_attention_weight_max": 101.64473724365234,
      "activations/layer4_attention_weight_min": -98.40583038330078,
      "activations/layer5_attention_weight_max": 70.06010437011719,
      "activations/layer5_attention_weight_min": -80.72855377197266,
      "activations/layer6_attention_weight_max": 54.057498931884766,
      "activations/layer6_attention_weight_min": -51.19674301147461,
      "activations/layer7_attention_weight_max": 71.68478393554688,
      "activations/layer7_attention_weight_min": -69.29149627685547,
      "activations/layer8_attention_weight_max": 44.676231384277344,
      "activations/layer8_attention_weight_min": -46.30605697631836,
      "activations/layer9_attention_weight_max": 44.88886642456055,
      "activations/layer9_attention_weight_min": -46.68478012084961,
      "epoch": 11.62,
      "learning_rate": 7.580340909090909e-05,
      "loss": 2.7766,
      "step": 200050
    },
    {
      "activations/layer0_attention_weight_max": 15.987407684326172,
      "activations/layer0_attention_weight_min": -14.392513275146484,
      "activations/layer10_attention_weight_max": 36.78401565551758,
      "activations/layer10_attention_weight_min": -34.91402816772461,
      "activations/layer11_attention_weight_max": 32.24391174316406,
      "activations/layer11_attention_weight_min": -32.53611373901367,
      "activations/layer12_attention_weight_max": 20.68195343017578,
      "activations/layer12_attention_weight_min": -26.871383666992188,
      "activations/layer13_attention_weight_max": 37.83164978027344,
      "activations/layer13_attention_weight_min": -34.881927490234375,
      "activations/layer14_attention_weight_max": 43.30539321899414,
      "activations/layer14_attention_weight_min": -31.077070236206055,
      "activations/layer15_attention_weight_max": 38.23094940185547,
      "activations/layer15_attention_weight_min": -28.138492584228516,
      "activations/layer16_attention_weight_max": 33.85131072998047,
      "activations/layer16_attention_weight_min": -29.861164093017578,
      "activations/layer17_attention_weight_max": 58.54094314575195,
      "activations/layer17_attention_weight_min": -46.570838928222656,
      "activations/layer18_attention_weight_max": 48.2892951965332,
      "activations/layer18_attention_weight_min": -39.84307098388672,
      "activations/layer19_attention_weight_max": 25.571449279785156,
      "activations/layer19_attention_weight_min": -22.10323143005371,
      "activations/layer1_attention_weight_max": 17.0911865234375,
      "activations/layer1_attention_weight_min": -16.605146408081055,
      "activations/layer20_attention_weight_max": 27.194074630737305,
      "activations/layer20_attention_weight_min": -22.794523239135742,
      "activations/layer21_attention_weight_max": 42.65837860107422,
      "activations/layer21_attention_weight_min": -20.663925170898438,
      "activations/layer22_attention_weight_max": 32.790245056152344,
      "activations/layer22_attention_weight_min": -24.27692222595215,
      "activations/layer23_attention_weight_max": 43.49567794799805,
      "activations/layer23_attention_weight_min": -24.71405029296875,
      "activations/layer2_attention_weight_max": 34.68022918701172,
      "activations/layer2_attention_weight_min": -32.478782653808594,
      "activations/layer3_attention_weight_max": 97.66355895996094,
      "activations/layer3_attention_weight_min": -101.37348937988281,
      "activations/layer4_attention_weight_max": 103.01448059082031,
      "activations/layer4_attention_weight_min": -99.86479949951172,
      "activations/layer5_attention_weight_max": 73.74246215820312,
      "activations/layer5_attention_weight_min": -82.98983764648438,
      "activations/layer6_attention_weight_max": 52.758968353271484,
      "activations/layer6_attention_weight_min": -54.87335968017578,
      "activations/layer7_attention_weight_max": 77.75704956054688,
      "activations/layer7_attention_weight_min": -73.31697082519531,
      "activations/layer8_attention_weight_max": 45.5001106262207,
      "activations/layer8_attention_weight_min": -47.803890228271484,
      "activations/layer9_attention_weight_max": 47.78418731689453,
      "activations/layer9_attention_weight_min": -46.63662338256836,
      "epoch": 11.63,
      "learning_rate": 7.578446969696969e-05,
      "loss": 2.7884,
      "step": 200100
    },
    {
      "activations/layer0_attention_weight_max": 15.575542449951172,
      "activations/layer0_attention_weight_min": -14.633877754211426,
      "activations/layer10_attention_weight_max": 39.9671516418457,
      "activations/layer10_attention_weight_min": -40.75685501098633,
      "activations/layer11_attention_weight_max": 39.16551208496094,
      "activations/layer11_attention_weight_min": -36.24987030029297,
      "activations/layer12_attention_weight_max": 17.68091583251953,
      "activations/layer12_attention_weight_min": -25.714168548583984,
      "activations/layer13_attention_weight_max": 37.993438720703125,
      "activations/layer13_attention_weight_min": -28.738576889038086,
      "activations/layer14_attention_weight_max": 35.27471923828125,
      "activations/layer14_attention_weight_min": -30.257556915283203,
      "activations/layer15_attention_weight_max": 32.01750183105469,
      "activations/layer15_attention_weight_min": -29.513160705566406,
      "activations/layer16_attention_weight_max": 31.534692764282227,
      "activations/layer16_attention_weight_min": -30.862085342407227,
      "activations/layer17_attention_weight_max": 52.98435974121094,
      "activations/layer17_attention_weight_min": -47.13200759887695,
      "activations/layer18_attention_weight_max": 45.32345962524414,
      "activations/layer18_attention_weight_min": -38.34294891357422,
      "activations/layer19_attention_weight_max": 22.399314880371094,
      "activations/layer19_attention_weight_min": -22.52985954284668,
      "activations/layer1_attention_weight_max": 17.260290145874023,
      "activations/layer1_attention_weight_min": -14.788567543029785,
      "activations/layer20_attention_weight_max": 24.133243560791016,
      "activations/layer20_attention_weight_min": -21.338218688964844,
      "activations/layer21_attention_weight_max": 31.975412368774414,
      "activations/layer21_attention_weight_min": -22.58414077758789,
      "activations/layer22_attention_weight_max": 31.816133499145508,
      "activations/layer22_attention_weight_min": -28.29261016845703,
      "activations/layer23_attention_weight_max": 37.86309051513672,
      "activations/layer23_attention_weight_min": -26.790546417236328,
      "activations/layer2_attention_weight_max": 32.941062927246094,
      "activations/layer2_attention_weight_min": -33.21217346191406,
      "activations/layer3_attention_weight_max": 94.42318725585938,
      "activations/layer3_attention_weight_min": -102.07938385009766,
      "activations/layer4_attention_weight_max": 98.869140625,
      "activations/layer4_attention_weight_min": -93.8044204711914,
      "activations/layer5_attention_weight_max": 73.29822540283203,
      "activations/layer5_attention_weight_min": -73.72695922851562,
      "activations/layer6_attention_weight_max": 51.882869720458984,
      "activations/layer6_attention_weight_min": -51.55644607543945,
      "activations/layer7_attention_weight_max": 85.4288101196289,
      "activations/layer7_attention_weight_min": -75.60401153564453,
      "activations/layer8_attention_weight_max": 54.275875091552734,
      "activations/layer8_attention_weight_min": -47.561397552490234,
      "activations/layer9_attention_weight_max": 67.26869201660156,
      "activations/layer9_attention_weight_min": -53.26032638549805,
      "epoch": 11.63,
      "learning_rate": 7.576553030303029e-05,
      "loss": 2.7867,
      "step": 200150
    },
    {
      "activations/layer0_attention_weight_max": 16.26021385192871,
      "activations/layer0_attention_weight_min": -14.137191772460938,
      "activations/layer10_attention_weight_max": 36.14315414428711,
      "activations/layer10_attention_weight_min": -35.055015563964844,
      "activations/layer11_attention_weight_max": 32.31264877319336,
      "activations/layer11_attention_weight_min": -33.33727264404297,
      "activations/layer12_attention_weight_max": 19.903676986694336,
      "activations/layer12_attention_weight_min": -23.646554946899414,
      "activations/layer13_attention_weight_max": 45.1985969543457,
      "activations/layer13_attention_weight_min": -32.417598724365234,
      "activations/layer14_attention_weight_max": 37.27429962158203,
      "activations/layer14_attention_weight_min": -29.60689353942871,
      "activations/layer15_attention_weight_max": 33.31881332397461,
      "activations/layer15_attention_weight_min": -28.34334945678711,
      "activations/layer16_attention_weight_max": 32.52876281738281,
      "activations/layer16_attention_weight_min": -30.965023040771484,
      "activations/layer17_attention_weight_max": 50.634220123291016,
      "activations/layer17_attention_weight_min": -43.87747573852539,
      "activations/layer18_attention_weight_max": 48.76020431518555,
      "activations/layer18_attention_weight_min": -37.45684051513672,
      "activations/layer19_attention_weight_max": 24.300365447998047,
      "activations/layer19_attention_weight_min": -20.76991844177246,
      "activations/layer1_attention_weight_max": 16.038188934326172,
      "activations/layer1_attention_weight_min": -18.534971237182617,
      "activations/layer20_attention_weight_max": 23.060009002685547,
      "activations/layer20_attention_weight_min": -22.005935668945312,
      "activations/layer21_attention_weight_max": 39.26826858520508,
      "activations/layer21_attention_weight_min": -21.86919593811035,
      "activations/layer22_attention_weight_max": 30.883914947509766,
      "activations/layer22_attention_weight_min": -25.49321937561035,
      "activations/layer23_attention_weight_max": 42.178314208984375,
      "activations/layer23_attention_weight_min": -24.346364974975586,
      "activations/layer2_attention_weight_max": 35.16615676879883,
      "activations/layer2_attention_weight_min": -32.844730377197266,
      "activations/layer3_attention_weight_max": 99.85592651367188,
      "activations/layer3_attention_weight_min": -97.79533386230469,
      "activations/layer4_attention_weight_max": 100.68356323242188,
      "activations/layer4_attention_weight_min": -95.2658920288086,
      "activations/layer5_attention_weight_max": 77.17250061035156,
      "activations/layer5_attention_weight_min": -80.772705078125,
      "activations/layer6_attention_weight_max": 55.720157623291016,
      "activations/layer6_attention_weight_min": -52.4419059753418,
      "activations/layer7_attention_weight_max": 81.62175750732422,
      "activations/layer7_attention_weight_min": -72.35397338867188,
      "activations/layer8_attention_weight_max": 46.202152252197266,
      "activations/layer8_attention_weight_min": -47.31897735595703,
      "activations/layer9_attention_weight_max": 50.17668914794922,
      "activations/layer9_attention_weight_min": -49.689815521240234,
      "epoch": 11.63,
      "learning_rate": 7.574659090909091e-05,
      "loss": 2.8018,
      "step": 200200
    },
    {
      "activations/layer0_attention_weight_max": 16.22462272644043,
      "activations/layer0_attention_weight_min": -14.393324851989746,
      "activations/layer10_attention_weight_max": 34.02069854736328,
      "activations/layer10_attention_weight_min": -33.33088684082031,
      "activations/layer11_attention_weight_max": 30.602949142456055,
      "activations/layer11_attention_weight_min": -31.163368225097656,
      "activations/layer12_attention_weight_max": 18.01763916015625,
      "activations/layer12_attention_weight_min": -25.51658058166504,
      "activations/layer13_attention_weight_max": 46.51064682006836,
      "activations/layer13_attention_weight_min": -31.526718139648438,
      "activations/layer14_attention_weight_max": 37.733131408691406,
      "activations/layer14_attention_weight_min": -29.81311798095703,
      "activations/layer15_attention_weight_max": 32.25068664550781,
      "activations/layer15_attention_weight_min": -28.818679809570312,
      "activations/layer16_attention_weight_max": 32.15983581542969,
      "activations/layer16_attention_weight_min": -29.17156982421875,
      "activations/layer17_attention_weight_max": 51.94706344604492,
      "activations/layer17_attention_weight_min": -44.451019287109375,
      "activations/layer18_attention_weight_max": 45.02786636352539,
      "activations/layer18_attention_weight_min": -38.407310485839844,
      "activations/layer19_attention_weight_max": 21.500059127807617,
      "activations/layer19_attention_weight_min": -23.037126541137695,
      "activations/layer1_attention_weight_max": 16.829551696777344,
      "activations/layer1_attention_weight_min": -16.816146850585938,
      "activations/layer20_attention_weight_max": 20.390405654907227,
      "activations/layer20_attention_weight_min": -24.430891036987305,
      "activations/layer21_attention_weight_max": 38.97369384765625,
      "activations/layer21_attention_weight_min": -24.43842124938965,
      "activations/layer22_attention_weight_max": 30.01425552368164,
      "activations/layer22_attention_weight_min": -24.994897842407227,
      "activations/layer23_attention_weight_max": 35.3444709777832,
      "activations/layer23_attention_weight_min": -22.927715301513672,
      "activations/layer2_attention_weight_max": 33.18019485473633,
      "activations/layer2_attention_weight_min": -31.706600189208984,
      "activations/layer3_attention_weight_max": 93.71427917480469,
      "activations/layer3_attention_weight_min": -93.9785385131836,
      "activations/layer4_attention_weight_max": 94.8076400756836,
      "activations/layer4_attention_weight_min": -94.78691864013672,
      "activations/layer5_attention_weight_max": 71.3028335571289,
      "activations/layer5_attention_weight_min": -77.22794342041016,
      "activations/layer6_attention_weight_max": 51.56985092163086,
      "activations/layer6_attention_weight_min": -52.421043395996094,
      "activations/layer7_attention_weight_max": 72.10608673095703,
      "activations/layer7_attention_weight_min": -68.21847534179688,
      "activations/layer8_attention_weight_max": 44.684974670410156,
      "activations/layer8_attention_weight_min": -46.182411193847656,
      "activations/layer9_attention_weight_max": 47.19698715209961,
      "activations/layer9_attention_weight_min": -46.61287307739258,
      "epoch": 11.64,
      "learning_rate": 7.572765151515151e-05,
      "loss": 2.781,
      "step": 200250
    },
    {
      "activations/layer0_attention_weight_max": 16.822410583496094,
      "activations/layer0_attention_weight_min": -14.251402854919434,
      "activations/layer10_attention_weight_max": 37.37482452392578,
      "activations/layer10_attention_weight_min": -36.75458526611328,
      "activations/layer11_attention_weight_max": 35.52622985839844,
      "activations/layer11_attention_weight_min": -33.061668395996094,
      "activations/layer12_attention_weight_max": 18.307601928710938,
      "activations/layer12_attention_weight_min": -26.908784866333008,
      "activations/layer13_attention_weight_max": 40.34309005737305,
      "activations/layer13_attention_weight_min": -28.51844024658203,
      "activations/layer14_attention_weight_max": 38.69203186035156,
      "activations/layer14_attention_weight_min": -31.012962341308594,
      "activations/layer15_attention_weight_max": 36.39761734008789,
      "activations/layer15_attention_weight_min": -29.42728614807129,
      "activations/layer16_attention_weight_max": 33.1650390625,
      "activations/layer16_attention_weight_min": -32.75369644165039,
      "activations/layer17_attention_weight_max": 55.73603057861328,
      "activations/layer17_attention_weight_min": -44.33452606201172,
      "activations/layer18_attention_weight_max": 51.3370475769043,
      "activations/layer18_attention_weight_min": -40.095001220703125,
      "activations/layer19_attention_weight_max": 24.058141708374023,
      "activations/layer19_attention_weight_min": -22.257158279418945,
      "activations/layer1_attention_weight_max": 16.352724075317383,
      "activations/layer1_attention_weight_min": -14.798864364624023,
      "activations/layer20_attention_weight_max": 25.12856101989746,
      "activations/layer20_attention_weight_min": -23.89842987060547,
      "activations/layer21_attention_weight_max": 40.82234191894531,
      "activations/layer21_attention_weight_min": -23.39170265197754,
      "activations/layer22_attention_weight_max": 32.67130661010742,
      "activations/layer22_attention_weight_min": -26.6782283782959,
      "activations/layer23_attention_weight_max": 42.00621795654297,
      "activations/layer23_attention_weight_min": -24.484722137451172,
      "activations/layer2_attention_weight_max": 32.95584487915039,
      "activations/layer2_attention_weight_min": -32.25164031982422,
      "activations/layer3_attention_weight_max": 94.76280212402344,
      "activations/layer3_attention_weight_min": -97.70897674560547,
      "activations/layer4_attention_weight_max": 102.60340881347656,
      "activations/layer4_attention_weight_min": -97.94728088378906,
      "activations/layer5_attention_weight_max": 74.0213851928711,
      "activations/layer5_attention_weight_min": -81.1692886352539,
      "activations/layer6_attention_weight_max": 54.46479415893555,
      "activations/layer6_attention_weight_min": -53.93535232543945,
      "activations/layer7_attention_weight_max": 73.59989929199219,
      "activations/layer7_attention_weight_min": -74.77337646484375,
      "activations/layer8_attention_weight_max": 46.47868728637695,
      "activations/layer8_attention_weight_min": -52.229408264160156,
      "activations/layer9_attention_weight_max": 47.02518844604492,
      "activations/layer9_attention_weight_min": -49.317501068115234,
      "epoch": 11.64,
      "learning_rate": 7.570871212121211e-05,
      "loss": 2.7806,
      "step": 200300
    },
    {
      "activations/layer0_attention_weight_max": 17.24266815185547,
      "activations/layer0_attention_weight_min": -14.3597993850708,
      "activations/layer10_attention_weight_max": 38.890750885009766,
      "activations/layer10_attention_weight_min": -36.167694091796875,
      "activations/layer11_attention_weight_max": 32.41968536376953,
      "activations/layer11_attention_weight_min": -32.505706787109375,
      "activations/layer12_attention_weight_max": 17.865041732788086,
      "activations/layer12_attention_weight_min": -34.669891357421875,
      "activations/layer13_attention_weight_max": 47.92543029785156,
      "activations/layer13_attention_weight_min": -30.677978515625,
      "activations/layer14_attention_weight_max": 52.97910690307617,
      "activations/layer14_attention_weight_min": -30.283491134643555,
      "activations/layer15_attention_weight_max": 47.791805267333984,
      "activations/layer15_attention_weight_min": -27.86210823059082,
      "activations/layer16_attention_weight_max": 35.086265563964844,
      "activations/layer16_attention_weight_min": -29.47951316833496,
      "activations/layer17_attention_weight_max": 56.789276123046875,
      "activations/layer17_attention_weight_min": -48.09761047363281,
      "activations/layer18_attention_weight_max": 48.79790115356445,
      "activations/layer18_attention_weight_min": -39.44330596923828,
      "activations/layer19_attention_weight_max": 22.619356155395508,
      "activations/layer19_attention_weight_min": -22.99481773376465,
      "activations/layer1_attention_weight_max": 19.063432693481445,
      "activations/layer1_attention_weight_min": -15.916860580444336,
      "activations/layer20_attention_weight_max": 24.965349197387695,
      "activations/layer20_attention_weight_min": -20.160261154174805,
      "activations/layer21_attention_weight_max": 43.323455810546875,
      "activations/layer21_attention_weight_min": -24.372339248657227,
      "activations/layer22_attention_weight_max": 33.21232223510742,
      "activations/layer22_attention_weight_min": -25.24623680114746,
      "activations/layer23_attention_weight_max": 40.22736358642578,
      "activations/layer23_attention_weight_min": -24.269752502441406,
      "activations/layer2_attention_weight_max": 33.23723602294922,
      "activations/layer2_attention_weight_min": -32.134307861328125,
      "activations/layer3_attention_weight_max": 94.08292388916016,
      "activations/layer3_attention_weight_min": -95.5099868774414,
      "activations/layer4_attention_weight_max": 97.70831298828125,
      "activations/layer4_attention_weight_min": -95.6177978515625,
      "activations/layer5_attention_weight_max": 74.73758697509766,
      "activations/layer5_attention_weight_min": -79.24066162109375,
      "activations/layer6_attention_weight_max": 53.335811614990234,
      "activations/layer6_attention_weight_min": -51.61248016357422,
      "activations/layer7_attention_weight_max": 82.49982452392578,
      "activations/layer7_attention_weight_min": -75.24523162841797,
      "activations/layer8_attention_weight_max": 46.79828643798828,
      "activations/layer8_attention_weight_min": -48.2039909362793,
      "activations/layer9_attention_weight_max": 57.07255935668945,
      "activations/layer9_attention_weight_min": -49.59840393066406,
      "epoch": 11.64,
      "learning_rate": 7.568977272727271e-05,
      "loss": 2.7789,
      "step": 200350
    },
    {
      "activations/layer0_attention_weight_max": 16.177257537841797,
      "activations/layer0_attention_weight_min": -14.6843900680542,
      "activations/layer10_attention_weight_max": 34.688331604003906,
      "activations/layer10_attention_weight_min": -36.22416687011719,
      "activations/layer11_attention_weight_max": 33.6055908203125,
      "activations/layer11_attention_weight_min": -32.002254486083984,
      "activations/layer12_attention_weight_max": 17.29880714416504,
      "activations/layer12_attention_weight_min": -28.917621612548828,
      "activations/layer13_attention_weight_max": 32.07130813598633,
      "activations/layer13_attention_weight_min": -28.083969116210938,
      "activations/layer14_attention_weight_max": 45.010643005371094,
      "activations/layer14_attention_weight_min": -31.491846084594727,
      "activations/layer15_attention_weight_max": 40.14834976196289,
      "activations/layer15_attention_weight_min": -28.515018463134766,
      "activations/layer16_attention_weight_max": 30.64720916748047,
      "activations/layer16_attention_weight_min": -27.795297622680664,
      "activations/layer17_attention_weight_max": 50.01970672607422,
      "activations/layer17_attention_weight_min": -45.0568733215332,
      "activations/layer18_attention_weight_max": 45.38070297241211,
      "activations/layer18_attention_weight_min": -38.063568115234375,
      "activations/layer19_attention_weight_max": 24.38532066345215,
      "activations/layer19_attention_weight_min": -22.387571334838867,
      "activations/layer1_attention_weight_max": 17.078062057495117,
      "activations/layer1_attention_weight_min": -16.47442054748535,
      "activations/layer20_attention_weight_max": 23.951011657714844,
      "activations/layer20_attention_weight_min": -21.262523651123047,
      "activations/layer21_attention_weight_max": 39.84099578857422,
      "activations/layer21_attention_weight_min": -23.165916442871094,
      "activations/layer22_attention_weight_max": 33.26906204223633,
      "activations/layer22_attention_weight_min": -24.941743850708008,
      "activations/layer23_attention_weight_max": 34.22175979614258,
      "activations/layer23_attention_weight_min": -23.1849422454834,
      "activations/layer2_attention_weight_max": 32.98497772216797,
      "activations/layer2_attention_weight_min": -31.104835510253906,
      "activations/layer3_attention_weight_max": 93.8404312133789,
      "activations/layer3_attention_weight_min": -92.85680389404297,
      "activations/layer4_attention_weight_max": 98.87559509277344,
      "activations/layer4_attention_weight_min": -93.7262954711914,
      "activations/layer5_attention_weight_max": 73.4374008178711,
      "activations/layer5_attention_weight_min": -77.75154876708984,
      "activations/layer6_attention_weight_max": 52.416080474853516,
      "activations/layer6_attention_weight_min": -52.6064338684082,
      "activations/layer7_attention_weight_max": 77.43475341796875,
      "activations/layer7_attention_weight_min": -69.62824249267578,
      "activations/layer8_attention_weight_max": 44.05696487426758,
      "activations/layer8_attention_weight_min": -46.02933883666992,
      "activations/layer9_attention_weight_max": 50.22660827636719,
      "activations/layer9_attention_weight_min": -47.66136932373047,
      "epoch": 11.64,
      "learning_rate": 7.567083333333333e-05,
      "loss": 2.7751,
      "step": 200400
    },
    {
      "activations/layer0_attention_weight_max": 16.130943298339844,
      "activations/layer0_attention_weight_min": -14.864114761352539,
      "activations/layer10_attention_weight_max": 36.451385498046875,
      "activations/layer10_attention_weight_min": -36.65144729614258,
      "activations/layer11_attention_weight_max": 35.842437744140625,
      "activations/layer11_attention_weight_min": -32.57722091674805,
      "activations/layer12_attention_weight_max": 19.03824806213379,
      "activations/layer12_attention_weight_min": -26.705915451049805,
      "activations/layer13_attention_weight_max": 38.90624237060547,
      "activations/layer13_attention_weight_min": -28.331459045410156,
      "activations/layer14_attention_weight_max": 37.35574722290039,
      "activations/layer14_attention_weight_min": -29.816661834716797,
      "activations/layer15_attention_weight_max": 31.91701889038086,
      "activations/layer15_attention_weight_min": -28.891315460205078,
      "activations/layer16_attention_weight_max": 31.278284072875977,
      "activations/layer16_attention_weight_min": -30.38756561279297,
      "activations/layer17_attention_weight_max": 53.00067138671875,
      "activations/layer17_attention_weight_min": -45.18145751953125,
      "activations/layer18_attention_weight_max": 54.82082748413086,
      "activations/layer18_attention_weight_min": -39.76137161254883,
      "activations/layer19_attention_weight_max": 22.991989135742188,
      "activations/layer19_attention_weight_min": -21.844144821166992,
      "activations/layer1_attention_weight_max": 16.332515716552734,
      "activations/layer1_attention_weight_min": -15.093587875366211,
      "activations/layer20_attention_weight_max": 25.90108871459961,
      "activations/layer20_attention_weight_min": -20.600658416748047,
      "activations/layer21_attention_weight_max": 40.9249267578125,
      "activations/layer21_attention_weight_min": -23.951894760131836,
      "activations/layer22_attention_weight_max": 33.40044021606445,
      "activations/layer22_attention_weight_min": -25.54302978515625,
      "activations/layer23_attention_weight_max": 40.43428039550781,
      "activations/layer23_attention_weight_min": -25.810321807861328,
      "activations/layer2_attention_weight_max": 32.133583068847656,
      "activations/layer2_attention_weight_min": -30.4528865814209,
      "activations/layer3_attention_weight_max": 90.95109558105469,
      "activations/layer3_attention_weight_min": -92.53784942626953,
      "activations/layer4_attention_weight_max": 94.91869354248047,
      "activations/layer4_attention_weight_min": -89.66605377197266,
      "activations/layer5_attention_weight_max": 73.6249008178711,
      "activations/layer5_attention_weight_min": -75.09590148925781,
      "activations/layer6_attention_weight_max": 54.00505447387695,
      "activations/layer6_attention_weight_min": -49.56901550292969,
      "activations/layer7_attention_weight_max": 77.39244842529297,
      "activations/layer7_attention_weight_min": -67.24589538574219,
      "activations/layer8_attention_weight_max": 45.772621154785156,
      "activations/layer8_attention_weight_min": -44.893009185791016,
      "activations/layer9_attention_weight_max": 46.337337493896484,
      "activations/layer9_attention_weight_min": -45.76182174682617,
      "epoch": 11.65,
      "learning_rate": 7.565189393939394e-05,
      "loss": 2.7819,
      "step": 200450
    },
    {
      "activations/layer0_attention_weight_max": 15.232487678527832,
      "activations/layer0_attention_weight_min": -14.675219535827637,
      "activations/layer10_attention_weight_max": 36.62681579589844,
      "activations/layer10_attention_weight_min": -36.075836181640625,
      "activations/layer11_attention_weight_max": 32.45334243774414,
      "activations/layer11_attention_weight_min": -32.863059997558594,
      "activations/layer12_attention_weight_max": 18.44000816345215,
      "activations/layer12_attention_weight_min": -25.95634651184082,
      "activations/layer13_attention_weight_max": 47.32073211669922,
      "activations/layer13_attention_weight_min": -33.204681396484375,
      "activations/layer14_attention_weight_max": 43.64629364013672,
      "activations/layer14_attention_weight_min": -32.60565185546875,
      "activations/layer15_attention_weight_max": 35.130924224853516,
      "activations/layer15_attention_weight_min": -31.120454788208008,
      "activations/layer16_attention_weight_max": 34.870361328125,
      "activations/layer16_attention_weight_min": -32.724369049072266,
      "activations/layer17_attention_weight_max": 53.41529083251953,
      "activations/layer17_attention_weight_min": -46.65187072753906,
      "activations/layer18_attention_weight_max": 47.49539566040039,
      "activations/layer18_attention_weight_min": -40.32504653930664,
      "activations/layer19_attention_weight_max": 23.53019905090332,
      "activations/layer19_attention_weight_min": -26.602222442626953,
      "activations/layer1_attention_weight_max": 18.00971794128418,
      "activations/layer1_attention_weight_min": -14.89085578918457,
      "activations/layer20_attention_weight_max": 25.244670867919922,
      "activations/layer20_attention_weight_min": -24.271757125854492,
      "activations/layer21_attention_weight_max": 43.22160720825195,
      "activations/layer21_attention_weight_min": -27.16591453552246,
      "activations/layer22_attention_weight_max": 31.951332092285156,
      "activations/layer22_attention_weight_min": -26.89430809020996,
      "activations/layer23_attention_weight_max": 43.45954132080078,
      "activations/layer23_attention_weight_min": -27.422191619873047,
      "activations/layer2_attention_weight_max": 34.5001220703125,
      "activations/layer2_attention_weight_min": -32.604557037353516,
      "activations/layer3_attention_weight_max": 99.09296417236328,
      "activations/layer3_attention_weight_min": -99.1412124633789,
      "activations/layer4_attention_weight_max": 102.71366119384766,
      "activations/layer4_attention_weight_min": -100.00545501708984,
      "activations/layer5_attention_weight_max": 73.447265625,
      "activations/layer5_attention_weight_min": -87.33383178710938,
      "activations/layer6_attention_weight_max": 53.264076232910156,
      "activations/layer6_attention_weight_min": -53.215450286865234,
      "activations/layer7_attention_weight_max": 74.79903411865234,
      "activations/layer7_attention_weight_min": -71.97563934326172,
      "activations/layer8_attention_weight_max": 43.541812896728516,
      "activations/layer8_attention_weight_min": -45.59050369262695,
      "activations/layer9_attention_weight_max": 45.841217041015625,
      "activations/layer9_attention_weight_min": -50.33625793457031,
      "epoch": 11.65,
      "learning_rate": 7.563295454545453e-05,
      "loss": 2.7817,
      "step": 200500
    },
    {
      "activations/layer0_attention_weight_max": 15.778677940368652,
      "activations/layer0_attention_weight_min": -14.689332008361816,
      "activations/layer10_attention_weight_max": 38.417240142822266,
      "activations/layer10_attention_weight_min": -39.2410774230957,
      "activations/layer11_attention_weight_max": 34.265235900878906,
      "activations/layer11_attention_weight_min": -35.21320343017578,
      "activations/layer12_attention_weight_max": 24.278812408447266,
      "activations/layer12_attention_weight_min": -22.473690032958984,
      "activations/layer13_attention_weight_max": 45.97533416748047,
      "activations/layer13_attention_weight_min": -36.284481048583984,
      "activations/layer14_attention_weight_max": 38.51947784423828,
      "activations/layer14_attention_weight_min": -30.190582275390625,
      "activations/layer15_attention_weight_max": 32.76171112060547,
      "activations/layer15_attention_weight_min": -28.847185134887695,
      "activations/layer16_attention_weight_max": 30.401147842407227,
      "activations/layer16_attention_weight_min": -29.27594757080078,
      "activations/layer17_attention_weight_max": 53.02866744995117,
      "activations/layer17_attention_weight_min": -45.8228645324707,
      "activations/layer18_attention_weight_max": 46.604129791259766,
      "activations/layer18_attention_weight_min": -36.292205810546875,
      "activations/layer19_attention_weight_max": 22.10706901550293,
      "activations/layer19_attention_weight_min": -19.168258666992188,
      "activations/layer1_attention_weight_max": 16.701541900634766,
      "activations/layer1_attention_weight_min": -15.164690017700195,
      "activations/layer20_attention_weight_max": 20.69900131225586,
      "activations/layer20_attention_weight_min": -21.39678382873535,
      "activations/layer21_attention_weight_max": 30.63556480407715,
      "activations/layer21_attention_weight_min": -19.424768447875977,
      "activations/layer22_attention_weight_max": 33.1364631652832,
      "activations/layer22_attention_weight_min": -25.326976776123047,
      "activations/layer23_attention_weight_max": 33.400638580322266,
      "activations/layer23_attention_weight_min": -20.557090759277344,
      "activations/layer2_attention_weight_max": 31.88528060913086,
      "activations/layer2_attention_weight_min": -30.867826461791992,
      "activations/layer3_attention_weight_max": 95.43253326416016,
      "activations/layer3_attention_weight_min": -94.44868469238281,
      "activations/layer4_attention_weight_max": 101.33554077148438,
      "activations/layer4_attention_weight_min": -99.54157257080078,
      "activations/layer5_attention_weight_max": 72.76277923583984,
      "activations/layer5_attention_weight_min": -81.57562255859375,
      "activations/layer6_attention_weight_max": 51.94746398925781,
      "activations/layer6_attention_weight_min": -53.03886032104492,
      "activations/layer7_attention_weight_max": 75.87032318115234,
      "activations/layer7_attention_weight_min": -73.19078063964844,
      "activations/layer8_attention_weight_max": 48.62560272216797,
      "activations/layer8_attention_weight_min": -49.95979690551758,
      "activations/layer9_attention_weight_max": 47.054161071777344,
      "activations/layer9_attention_weight_min": -49.18729782104492,
      "epoch": 11.65,
      "learning_rate": 7.561401515151515e-05,
      "loss": 2.784,
      "step": 200550
    },
    {
      "activations/layer0_attention_weight_max": 16.130632400512695,
      "activations/layer0_attention_weight_min": -14.77234172821045,
      "activations/layer10_attention_weight_max": 33.07319641113281,
      "activations/layer10_attention_weight_min": -36.99810791015625,
      "activations/layer11_attention_weight_max": 31.072614669799805,
      "activations/layer11_attention_weight_min": -34.58632278442383,
      "activations/layer12_attention_weight_max": 17.07057762145996,
      "activations/layer12_attention_weight_min": -32.54743194580078,
      "activations/layer13_attention_weight_max": 33.52152633666992,
      "activations/layer13_attention_weight_min": -26.29582405090332,
      "activations/layer14_attention_weight_max": 35.63763427734375,
      "activations/layer14_attention_weight_min": -27.93583106994629,
      "activations/layer15_attention_weight_max": 30.17795753479004,
      "activations/layer15_attention_weight_min": -27.16588020324707,
      "activations/layer16_attention_weight_max": 27.426706314086914,
      "activations/layer16_attention_weight_min": -26.8160343170166,
      "activations/layer17_attention_weight_max": 47.253929138183594,
      "activations/layer17_attention_weight_min": -40.30134963989258,
      "activations/layer18_attention_weight_max": 41.60988235473633,
      "activations/layer18_attention_weight_min": -34.86711120605469,
      "activations/layer19_attention_weight_max": 20.536657333374023,
      "activations/layer19_attention_weight_min": -19.629777908325195,
      "activations/layer1_attention_weight_max": 16.76590347290039,
      "activations/layer1_attention_weight_min": -14.228038787841797,
      "activations/layer20_attention_weight_max": 22.389995574951172,
      "activations/layer20_attention_weight_min": -24.392648696899414,
      "activations/layer21_attention_weight_max": 35.08919143676758,
      "activations/layer21_attention_weight_min": -21.488571166992188,
      "activations/layer22_attention_weight_max": 30.328155517578125,
      "activations/layer22_attention_weight_min": -23.038686752319336,
      "activations/layer23_attention_weight_max": 33.4944953918457,
      "activations/layer23_attention_weight_min": -23.32845115661621,
      "activations/layer2_attention_weight_max": 33.54011535644531,
      "activations/layer2_attention_weight_min": -32.49146270751953,
      "activations/layer3_attention_weight_max": 95.10234069824219,
      "activations/layer3_attention_weight_min": -103.29788208007812,
      "activations/layer4_attention_weight_max": 95.75829315185547,
      "activations/layer4_attention_weight_min": -96.1024398803711,
      "activations/layer5_attention_weight_max": 70.02288055419922,
      "activations/layer5_attention_weight_min": -76.05574035644531,
      "activations/layer6_attention_weight_max": 50.67557907104492,
      "activations/layer6_attention_weight_min": -50.0642204284668,
      "activations/layer7_attention_weight_max": 71.24332427978516,
      "activations/layer7_attention_weight_min": -69.6251220703125,
      "activations/layer8_attention_weight_max": 41.03742599487305,
      "activations/layer8_attention_weight_min": -43.5811653137207,
      "activations/layer9_attention_weight_max": 47.44761276245117,
      "activations/layer9_attention_weight_min": -46.65377426147461,
      "epoch": 11.66,
      "learning_rate": 7.559507575757576e-05,
      "loss": 2.769,
      "step": 200600
    },
    {
      "activations/layer0_attention_weight_max": 15.833598136901855,
      "activations/layer0_attention_weight_min": -14.878195762634277,
      "activations/layer10_attention_weight_max": 39.71400833129883,
      "activations/layer10_attention_weight_min": -38.92679977416992,
      "activations/layer11_attention_weight_max": 34.78898620605469,
      "activations/layer11_attention_weight_min": -34.51560974121094,
      "activations/layer12_attention_weight_max": 18.94390106201172,
      "activations/layer12_attention_weight_min": -26.074790954589844,
      "activations/layer13_attention_weight_max": 42.342403411865234,
      "activations/layer13_attention_weight_min": -29.342077255249023,
      "activations/layer14_attention_weight_max": 39.44466781616211,
      "activations/layer14_attention_weight_min": -28.740886688232422,
      "activations/layer15_attention_weight_max": 31.347322463989258,
      "activations/layer15_attention_weight_min": -28.39539337158203,
      "activations/layer16_attention_weight_max": 29.743255615234375,
      "activations/layer16_attention_weight_min": -26.966018676757812,
      "activations/layer17_attention_weight_max": 53.279598236083984,
      "activations/layer17_attention_weight_min": -40.98210906982422,
      "activations/layer18_attention_weight_max": 45.45989990234375,
      "activations/layer18_attention_weight_min": -34.87060546875,
      "activations/layer19_attention_weight_max": 20.543447494506836,
      "activations/layer19_attention_weight_min": -21.99562644958496,
      "activations/layer1_attention_weight_max": 16.80950164794922,
      "activations/layer1_attention_weight_min": -15.62589168548584,
      "activations/layer20_attention_weight_max": 22.767345428466797,
      "activations/layer20_attention_weight_min": -20.840139389038086,
      "activations/layer21_attention_weight_max": 41.2011833190918,
      "activations/layer21_attention_weight_min": -25.391263961791992,
      "activations/layer22_attention_weight_max": 31.04500961303711,
      "activations/layer22_attention_weight_min": -25.594242095947266,
      "activations/layer23_attention_weight_max": 36.566200256347656,
      "activations/layer23_attention_weight_min": -25.984535217285156,
      "activations/layer2_attention_weight_max": 34.494834899902344,
      "activations/layer2_attention_weight_min": -31.407590866088867,
      "activations/layer3_attention_weight_max": 99.75581359863281,
      "activations/layer3_attention_weight_min": -97.84649658203125,
      "activations/layer4_attention_weight_max": 104.4450454711914,
      "activations/layer4_attention_weight_min": -99.59526062011719,
      "activations/layer5_attention_weight_max": 73.29714965820312,
      "activations/layer5_attention_weight_min": -78.7436294555664,
      "activations/layer6_attention_weight_max": 55.06422805786133,
      "activations/layer6_attention_weight_min": -52.89318084716797,
      "activations/layer7_attention_weight_max": 73.91120910644531,
      "activations/layer7_attention_weight_min": -71.0257797241211,
      "activations/layer8_attention_weight_max": 45.83317565917969,
      "activations/layer8_attention_weight_min": -48.020965576171875,
      "activations/layer9_attention_weight_max": 45.8593864440918,
      "activations/layer9_attention_weight_min": -47.1615104675293,
      "epoch": 11.66,
      "learning_rate": 7.557613636363635e-05,
      "loss": 2.7679,
      "step": 200650
    },
    {
      "activations/layer0_attention_weight_max": 16.4321346282959,
      "activations/layer0_attention_weight_min": -14.831620216369629,
      "activations/layer10_attention_weight_max": 32.934078216552734,
      "activations/layer10_attention_weight_min": -33.19924545288086,
      "activations/layer11_attention_weight_max": 30.882949829101562,
      "activations/layer11_attention_weight_min": -30.331668853759766,
      "activations/layer12_attention_weight_max": 18.239030838012695,
      "activations/layer12_attention_weight_min": -27.336244583129883,
      "activations/layer13_attention_weight_max": 35.05851745605469,
      "activations/layer13_attention_weight_min": -28.120668411254883,
      "activations/layer14_attention_weight_max": 34.64490509033203,
      "activations/layer14_attention_weight_min": -31.193296432495117,
      "activations/layer15_attention_weight_max": 30.7596492767334,
      "activations/layer15_attention_weight_min": -29.262039184570312,
      "activations/layer16_attention_weight_max": 28.778905868530273,
      "activations/layer16_attention_weight_min": -28.81190299987793,
      "activations/layer17_attention_weight_max": 51.54673767089844,
      "activations/layer17_attention_weight_min": -41.23700714111328,
      "activations/layer18_attention_weight_max": 44.491493225097656,
      "activations/layer18_attention_weight_min": -35.40686798095703,
      "activations/layer19_attention_weight_max": 20.60468292236328,
      "activations/layer19_attention_weight_min": -20.035369873046875,
      "activations/layer1_attention_weight_max": 17.392866134643555,
      "activations/layer1_attention_weight_min": -15.418170928955078,
      "activations/layer20_attention_weight_max": 21.537830352783203,
      "activations/layer20_attention_weight_min": -21.161680221557617,
      "activations/layer21_attention_weight_max": 35.23542022705078,
      "activations/layer21_attention_weight_min": -24.330841064453125,
      "activations/layer22_attention_weight_max": 32.3278694152832,
      "activations/layer22_attention_weight_min": -26.921968460083008,
      "activations/layer23_attention_weight_max": 34.633056640625,
      "activations/layer23_attention_weight_min": -24.881519317626953,
      "activations/layer2_attention_weight_max": 35.75447082519531,
      "activations/layer2_attention_weight_min": -32.20588684082031,
      "activations/layer3_attention_weight_max": 102.49286651611328,
      "activations/layer3_attention_weight_min": -95.34556579589844,
      "activations/layer4_attention_weight_max": 101.54337310791016,
      "activations/layer4_attention_weight_min": -94.47212982177734,
      "activations/layer5_attention_weight_max": 72.31470489501953,
      "activations/layer5_attention_weight_min": -75.68414306640625,
      "activations/layer6_attention_weight_max": 54.6728630065918,
      "activations/layer6_attention_weight_min": -51.46792984008789,
      "activations/layer7_attention_weight_max": 69.7347640991211,
      "activations/layer7_attention_weight_min": -67.87106323242188,
      "activations/layer8_attention_weight_max": 43.6727180480957,
      "activations/layer8_attention_weight_min": -46.4091682434082,
      "activations/layer9_attention_weight_max": 45.15568542480469,
      "activations/layer9_attention_weight_min": -45.1108283996582,
      "epoch": 11.66,
      "learning_rate": 7.555719696969696e-05,
      "loss": 2.7842,
      "step": 200700
    },
    {
      "activations/layer0_attention_weight_max": 16.22661781311035,
      "activations/layer0_attention_weight_min": -15.05451774597168,
      "activations/layer10_attention_weight_max": 39.34239959716797,
      "activations/layer10_attention_weight_min": -38.29322814941406,
      "activations/layer11_attention_weight_max": 34.493953704833984,
      "activations/layer11_attention_weight_min": -33.87236022949219,
      "activations/layer12_attention_weight_max": 21.3795223236084,
      "activations/layer12_attention_weight_min": -25.304813385009766,
      "activations/layer13_attention_weight_max": 45.88434982299805,
      "activations/layer13_attention_weight_min": -28.891008377075195,
      "activations/layer14_attention_weight_max": 43.25054931640625,
      "activations/layer14_attention_weight_min": -29.94831657409668,
      "activations/layer15_attention_weight_max": 38.80520248413086,
      "activations/layer15_attention_weight_min": -30.20319938659668,
      "activations/layer16_attention_weight_max": 32.6899299621582,
      "activations/layer16_attention_weight_min": -29.602460861206055,
      "activations/layer17_attention_weight_max": 57.20209503173828,
      "activations/layer17_attention_weight_min": -44.685184478759766,
      "activations/layer18_attention_weight_max": 50.8484001159668,
      "activations/layer18_attention_weight_min": -36.8047981262207,
      "activations/layer19_attention_weight_max": 21.082883834838867,
      "activations/layer19_attention_weight_min": -19.95763397216797,
      "activations/layer1_attention_weight_max": 16.728759765625,
      "activations/layer1_attention_weight_min": -14.704611778259277,
      "activations/layer20_attention_weight_max": 22.850873947143555,
      "activations/layer20_attention_weight_min": -20.99797821044922,
      "activations/layer21_attention_weight_max": 35.756324768066406,
      "activations/layer21_attention_weight_min": -24.023778915405273,
      "activations/layer22_attention_weight_max": 30.09955596923828,
      "activations/layer22_attention_weight_min": -23.917465209960938,
      "activations/layer23_attention_weight_max": 40.02961349487305,
      "activations/layer23_attention_weight_min": -23.639869689941406,
      "activations/layer2_attention_weight_max": 31.861038208007812,
      "activations/layer2_attention_weight_min": -31.329734802246094,
      "activations/layer3_attention_weight_max": 95.98280334472656,
      "activations/layer3_attention_weight_min": -98.60377502441406,
      "activations/layer4_attention_weight_max": 101.01371765136719,
      "activations/layer4_attention_weight_min": -96.01265716552734,
      "activations/layer5_attention_weight_max": 73.13735961914062,
      "activations/layer5_attention_weight_min": -75.50678253173828,
      "activations/layer6_attention_weight_max": 54.455867767333984,
      "activations/layer6_attention_weight_min": -54.08076477050781,
      "activations/layer7_attention_weight_max": 78.1069107055664,
      "activations/layer7_attention_weight_min": -74.68446350097656,
      "activations/layer8_attention_weight_max": 46.652557373046875,
      "activations/layer8_attention_weight_min": -47.8635368347168,
      "activations/layer9_attention_weight_max": 51.38245391845703,
      "activations/layer9_attention_weight_min": -48.813575744628906,
      "epoch": 11.66,
      "learning_rate": 7.553825757575758e-05,
      "loss": 2.7621,
      "step": 200750
    },
    {
      "activations/layer0_attention_weight_max": 16.633169174194336,
      "activations/layer0_attention_weight_min": -14.471899032592773,
      "activations/layer10_attention_weight_max": 38.9302978515625,
      "activations/layer10_attention_weight_min": -39.172645568847656,
      "activations/layer11_attention_weight_max": 34.245689392089844,
      "activations/layer11_attention_weight_min": -34.38164520263672,
      "activations/layer12_attention_weight_max": 18.212148666381836,
      "activations/layer12_attention_weight_min": -22.253141403198242,
      "activations/layer13_attention_weight_max": 40.500179290771484,
      "activations/layer13_attention_weight_min": -32.386260986328125,
      "activations/layer14_attention_weight_max": 36.57364273071289,
      "activations/layer14_attention_weight_min": -29.137216567993164,
      "activations/layer15_attention_weight_max": 33.65317153930664,
      "activations/layer15_attention_weight_min": -28.538291931152344,
      "activations/layer16_attention_weight_max": 30.40416717529297,
      "activations/layer16_attention_weight_min": -29.818449020385742,
      "activations/layer17_attention_weight_max": 52.08597946166992,
      "activations/layer17_attention_weight_min": -45.21103286743164,
      "activations/layer18_attention_weight_max": 48.01961898803711,
      "activations/layer18_attention_weight_min": -36.8685417175293,
      "activations/layer19_attention_weight_max": 21.92774772644043,
      "activations/layer19_attention_weight_min": -20.44387435913086,
      "activations/layer1_attention_weight_max": 16.36267852783203,
      "activations/layer1_attention_weight_min": -17.25278091430664,
      "activations/layer20_attention_weight_max": 23.18000030517578,
      "activations/layer20_attention_weight_min": -19.54422378540039,
      "activations/layer21_attention_weight_max": 34.540992736816406,
      "activations/layer21_attention_weight_min": -22.17620277404785,
      "activations/layer22_attention_weight_max": 28.507375717163086,
      "activations/layer22_attention_weight_min": -24.133134841918945,
      "activations/layer23_attention_weight_max": 39.182125091552734,
      "activations/layer23_attention_weight_min": -26.290508270263672,
      "activations/layer2_attention_weight_max": 33.61227798461914,
      "activations/layer2_attention_weight_min": -31.95479393005371,
      "activations/layer3_attention_weight_max": 95.55589294433594,
      "activations/layer3_attention_weight_min": -99.10935974121094,
      "activations/layer4_attention_weight_max": 97.3646011352539,
      "activations/layer4_attention_weight_min": -94.8970718383789,
      "activations/layer5_attention_weight_max": 74.24888610839844,
      "activations/layer5_attention_weight_min": -79.69186401367188,
      "activations/layer6_attention_weight_max": 52.824501037597656,
      "activations/layer6_attention_weight_min": -52.28868103027344,
      "activations/layer7_attention_weight_max": 74.70425415039062,
      "activations/layer7_attention_weight_min": -76.19610595703125,
      "activations/layer8_attention_weight_max": 49.97321319580078,
      "activations/layer8_attention_weight_min": -49.69269943237305,
      "activations/layer9_attention_weight_max": 52.73940658569336,
      "activations/layer9_attention_weight_min": -51.3922233581543,
      "epoch": 11.67,
      "learning_rate": 7.551969696969697e-05,
      "loss": 2.784,
      "step": 200800
    },
    {
      "activations/layer0_attention_weight_max": 15.764307022094727,
      "activations/layer0_attention_weight_min": -14.935562133789062,
      "activations/layer10_attention_weight_max": 39.6622200012207,
      "activations/layer10_attention_weight_min": -38.79735565185547,
      "activations/layer11_attention_weight_max": 34.09272766113281,
      "activations/layer11_attention_weight_min": -35.89220428466797,
      "activations/layer12_attention_weight_max": 20.87610626220703,
      "activations/layer12_attention_weight_min": -27.563392639160156,
      "activations/layer13_attention_weight_max": 43.57048416137695,
      "activations/layer13_attention_weight_min": -29.330904006958008,
      "activations/layer14_attention_weight_max": 39.32501983642578,
      "activations/layer14_attention_weight_min": -30.534671783447266,
      "activations/layer15_attention_weight_max": 39.053287506103516,
      "activations/layer15_attention_weight_min": -30.485551834106445,
      "activations/layer16_attention_weight_max": 33.10737991333008,
      "activations/layer16_attention_weight_min": -32.88844680786133,
      "activations/layer17_attention_weight_max": 53.44472885131836,
      "activations/layer17_attention_weight_min": -46.32013702392578,
      "activations/layer18_attention_weight_max": 50.43995666503906,
      "activations/layer18_attention_weight_min": -38.15984344482422,
      "activations/layer19_attention_weight_max": 25.506574630737305,
      "activations/layer19_attention_weight_min": -20.954811096191406,
      "activations/layer1_attention_weight_max": 17.47361946105957,
      "activations/layer1_attention_weight_min": -15.267144203186035,
      "activations/layer20_attention_weight_max": 22.67928695678711,
      "activations/layer20_attention_weight_min": -20.914819717407227,
      "activations/layer21_attention_weight_max": 34.311824798583984,
      "activations/layer21_attention_weight_min": -21.943796157836914,
      "activations/layer22_attention_weight_max": 32.04718780517578,
      "activations/layer22_attention_weight_min": -24.53188133239746,
      "activations/layer23_attention_weight_max": 36.189964294433594,
      "activations/layer23_attention_weight_min": -22.102703094482422,
      "activations/layer2_attention_weight_max": 33.164371490478516,
      "activations/layer2_attention_weight_min": -33.84941101074219,
      "activations/layer3_attention_weight_max": 95.9022216796875,
      "activations/layer3_attention_weight_min": -103.09492492675781,
      "activations/layer4_attention_weight_max": 102.14263916015625,
      "activations/layer4_attention_weight_min": -100.8735580444336,
      "activations/layer5_attention_weight_max": 76.40288543701172,
      "activations/layer5_attention_weight_min": -82.94820404052734,
      "activations/layer6_attention_weight_max": 53.75181579589844,
      "activations/layer6_attention_weight_min": -55.987701416015625,
      "activations/layer7_attention_weight_max": 74.73673248291016,
      "activations/layer7_attention_weight_min": -72.52242279052734,
      "activations/layer8_attention_weight_max": 45.18807601928711,
      "activations/layer8_attention_weight_min": -48.2313346862793,
      "activations/layer9_attention_weight_max": 49.86235809326172,
      "activations/layer9_attention_weight_min": -50.9891242980957,
      "epoch": 11.67,
      "learning_rate": 7.550075757575757e-05,
      "loss": 2.7749,
      "step": 200850
    },
    {
      "activations/layer0_attention_weight_max": 15.745421409606934,
      "activations/layer0_attention_weight_min": -14.7089204788208,
      "activations/layer10_attention_weight_max": 35.533748626708984,
      "activations/layer10_attention_weight_min": -35.94164276123047,
      "activations/layer11_attention_weight_max": 31.017993927001953,
      "activations/layer11_attention_weight_min": -33.27839660644531,
      "activations/layer12_attention_weight_max": 20.549856185913086,
      "activations/layer12_attention_weight_min": -28.720436096191406,
      "activations/layer13_attention_weight_max": 42.300819396972656,
      "activations/layer13_attention_weight_min": -29.00456428527832,
      "activations/layer14_attention_weight_max": 37.58354949951172,
      "activations/layer14_attention_weight_min": -30.791282653808594,
      "activations/layer15_attention_weight_max": 33.3165397644043,
      "activations/layer15_attention_weight_min": -29.581226348876953,
      "activations/layer16_attention_weight_max": 30.38616180419922,
      "activations/layer16_attention_weight_min": -28.896520614624023,
      "activations/layer17_attention_weight_max": 51.50593185424805,
      "activations/layer17_attention_weight_min": -43.25172805786133,
      "activations/layer18_attention_weight_max": 45.752342224121094,
      "activations/layer18_attention_weight_min": -37.8114128112793,
      "activations/layer19_attention_weight_max": 22.11722183227539,
      "activations/layer19_attention_weight_min": -19.316669464111328,
      "activations/layer1_attention_weight_max": 16.07309341430664,
      "activations/layer1_attention_weight_min": -15.334177017211914,
      "activations/layer20_attention_weight_max": 20.403242111206055,
      "activations/layer20_attention_weight_min": -19.77304458618164,
      "activations/layer21_attention_weight_max": 35.17402267456055,
      "activations/layer21_attention_weight_min": -24.496837615966797,
      "activations/layer22_attention_weight_max": 27.217832565307617,
      "activations/layer22_attention_weight_min": -26.34678840637207,
      "activations/layer23_attention_weight_max": 36.76436233520508,
      "activations/layer23_attention_weight_min": -22.74752426147461,
      "activations/layer2_attention_weight_max": 33.52924728393555,
      "activations/layer2_attention_weight_min": -31.726749420166016,
      "activations/layer3_attention_weight_max": 99.93248748779297,
      "activations/layer3_attention_weight_min": -99.96546936035156,
      "activations/layer4_attention_weight_max": 105.87908172607422,
      "activations/layer4_attention_weight_min": -100.14856719970703,
      "activations/layer5_attention_weight_max": 76.64498901367188,
      "activations/layer5_attention_weight_min": -82.13766479492188,
      "activations/layer6_attention_weight_max": 56.52756881713867,
      "activations/layer6_attention_weight_min": -57.808074951171875,
      "activations/layer7_attention_weight_max": 75.64160919189453,
      "activations/layer7_attention_weight_min": -72.30230712890625,
      "activations/layer8_attention_weight_max": 46.47116470336914,
      "activations/layer8_attention_weight_min": -53.89885330200195,
      "activations/layer9_attention_weight_max": 48.301937103271484,
      "activations/layer9_attention_weight_min": -49.89849090576172,
      "epoch": 11.67,
      "learning_rate": 7.548181818181817e-05,
      "loss": 2.7746,
      "step": 200900
    },
    {
      "activations/layer0_attention_weight_max": 17.043031692504883,
      "activations/layer0_attention_weight_min": -14.578103065490723,
      "activations/layer10_attention_weight_max": 35.562713623046875,
      "activations/layer10_attention_weight_min": -35.036224365234375,
      "activations/layer11_attention_weight_max": 32.62298583984375,
      "activations/layer11_attention_weight_min": -31.83929443359375,
      "activations/layer12_attention_weight_max": 19.96018409729004,
      "activations/layer12_attention_weight_min": -24.273466110229492,
      "activations/layer13_attention_weight_max": 36.56138229370117,
      "activations/layer13_attention_weight_min": -29.568193435668945,
      "activations/layer14_attention_weight_max": 36.19548416137695,
      "activations/layer14_attention_weight_min": -31.97629165649414,
      "activations/layer15_attention_weight_max": 33.70773696899414,
      "activations/layer15_attention_weight_min": -28.487083435058594,
      "activations/layer16_attention_weight_max": 30.30660057067871,
      "activations/layer16_attention_weight_min": -29.5876522064209,
      "activations/layer17_attention_weight_max": 51.36416244506836,
      "activations/layer17_attention_weight_min": -41.92470932006836,
      "activations/layer18_attention_weight_max": 46.08599090576172,
      "activations/layer18_attention_weight_min": -35.2783317565918,
      "activations/layer19_attention_weight_max": 22.796354293823242,
      "activations/layer19_attention_weight_min": -20.694971084594727,
      "activations/layer1_attention_weight_max": 16.776016235351562,
      "activations/layer1_attention_weight_min": -14.396307945251465,
      "activations/layer20_attention_weight_max": 23.43107795715332,
      "activations/layer20_attention_weight_min": -22.326250076293945,
      "activations/layer21_attention_weight_max": 37.52092361450195,
      "activations/layer21_attention_weight_min": -25.62790870666504,
      "activations/layer22_attention_weight_max": 29.435110092163086,
      "activations/layer22_attention_weight_min": -25.31282615661621,
      "activations/layer23_attention_weight_max": 38.524356842041016,
      "activations/layer23_attention_weight_min": -24.371809005737305,
      "activations/layer2_attention_weight_max": 32.78049087524414,
      "activations/layer2_attention_weight_min": -32.997501373291016,
      "activations/layer3_attention_weight_max": 95.57649993896484,
      "activations/layer3_attention_weight_min": -107.32437896728516,
      "activations/layer4_attention_weight_max": 100.15522003173828,
      "activations/layer4_attention_weight_min": -105.15038299560547,
      "activations/layer5_attention_weight_max": 72.16358947753906,
      "activations/layer5_attention_weight_min": -79.01176452636719,
      "activations/layer6_attention_weight_max": 53.02611541748047,
      "activations/layer6_attention_weight_min": -53.69071960449219,
      "activations/layer7_attention_weight_max": 72.01776885986328,
      "activations/layer7_attention_weight_min": -69.84857177734375,
      "activations/layer8_attention_weight_max": 44.7424201965332,
      "activations/layer8_attention_weight_min": -46.724491119384766,
      "activations/layer9_attention_weight_max": 45.171512603759766,
      "activations/layer9_attention_weight_min": -47.78103256225586,
      "epoch": 11.68,
      "learning_rate": 7.546287878787879e-05,
      "loss": 2.7754,
      "step": 200950
    },
    {
      "activations/layer0_attention_weight_max": 17.1682186126709,
      "activations/layer0_attention_weight_min": -14.445876121520996,
      "activations/layer10_attention_weight_max": 38.97383499145508,
      "activations/layer10_attention_weight_min": -38.52242660522461,
      "activations/layer11_attention_weight_max": 32.03485870361328,
      "activations/layer11_attention_weight_min": -32.81557083129883,
      "activations/layer12_attention_weight_max": 24.25104522705078,
      "activations/layer12_attention_weight_min": -23.56641387939453,
      "activations/layer13_attention_weight_max": 33.79544448852539,
      "activations/layer13_attention_weight_min": -30.478355407714844,
      "activations/layer14_attention_weight_max": 35.58502960205078,
      "activations/layer14_attention_weight_min": -29.961252212524414,
      "activations/layer15_attention_weight_max": 31.121931076049805,
      "activations/layer15_attention_weight_min": -28.582393646240234,
      "activations/layer16_attention_weight_max": 31.957063674926758,
      "activations/layer16_attention_weight_min": -29.57259178161621,
      "activations/layer17_attention_weight_max": 51.35075378417969,
      "activations/layer17_attention_weight_min": -45.59193420410156,
      "activations/layer18_attention_weight_max": 49.405723571777344,
      "activations/layer18_attention_weight_min": -41.884918212890625,
      "activations/layer19_attention_weight_max": 22.530601501464844,
      "activations/layer19_attention_weight_min": -24.421276092529297,
      "activations/layer1_attention_weight_max": 16.816577911376953,
      "activations/layer1_attention_weight_min": -15.229879379272461,
      "activations/layer20_attention_weight_max": 22.857406616210938,
      "activations/layer20_attention_weight_min": -24.62726402282715,
      "activations/layer21_attention_weight_max": 34.17826461791992,
      "activations/layer21_attention_weight_min": -23.81451416015625,
      "activations/layer22_attention_weight_max": 31.41475486755371,
      "activations/layer22_attention_weight_min": -27.639129638671875,
      "activations/layer23_attention_weight_max": 37.84798049926758,
      "activations/layer23_attention_weight_min": -25.27089500427246,
      "activations/layer2_attention_weight_max": 32.96808624267578,
      "activations/layer2_attention_weight_min": -32.016761779785156,
      "activations/layer3_attention_weight_max": 96.09036254882812,
      "activations/layer3_attention_weight_min": -100.16488647460938,
      "activations/layer4_attention_weight_max": 100.00897979736328,
      "activations/layer4_attention_weight_min": -96.02880859375,
      "activations/layer5_attention_weight_max": 75.38517761230469,
      "activations/layer5_attention_weight_min": -79.24772644042969,
      "activations/layer6_attention_weight_max": 52.02363586425781,
      "activations/layer6_attention_weight_min": -52.89033889770508,
      "activations/layer7_attention_weight_max": 72.40360260009766,
      "activations/layer7_attention_weight_min": -72.84485626220703,
      "activations/layer8_attention_weight_max": 46.2017936706543,
      "activations/layer8_attention_weight_min": -50.85975646972656,
      "activations/layer9_attention_weight_max": 48.69255828857422,
      "activations/layer9_attention_weight_min": -52.29030227661133,
      "epoch": 11.68,
      "learning_rate": 7.544393939393939e-05,
      "loss": 2.7796,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_loss": 2.73046875,
      "eval_runtime": 8.5295,
      "eval_samples_per_second": 503.428,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_openwebtext_loss": 2.73046875,
      "eval_openwebtext_ppl": 15.340075995482293,
      "eval_openwebtext_runtime": 8.5295,
      "eval_openwebtext_samples_per_second": 503.428,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 2.0058,
      "eval_wikitext_samples_per_second": 227.344,
      "step": 201000
    },
    {
      "epoch": 11.68,
      "eval_lambada_loss": 2.669921875,
      "eval_lambada_ppl": 14.438841114275835,
      "eval_lambada_runtime": 9.6138,
      "eval_lambada_samples_per_second": 506.458,
      "step": 201000
    },
    {
      "activations/layer0_attention_weight_max": 15.943965911865234,
      "activations/layer0_attention_weight_min": -14.650614738464355,
      "activations/layer10_attention_weight_max": 33.43062210083008,
      "activations/layer10_attention_weight_min": -33.683868408203125,
      "activations/layer11_attention_weight_max": 30.13977813720703,
      "activations/layer11_attention_weight_min": -32.811851501464844,
      "activations/layer12_attention_weight_max": 20.291675567626953,
      "activations/layer12_attention_weight_min": -26.34954071044922,
      "activations/layer13_attention_weight_max": 40.276405334472656,
      "activations/layer13_attention_weight_min": -29.621417999267578,
      "activations/layer14_attention_weight_max": 36.907257080078125,
      "activations/layer14_attention_weight_min": -30.80352210998535,
      "activations/layer15_attention_weight_max": 35.563411712646484,
      "activations/layer15_attention_weight_min": -28.598201751708984,
      "activations/layer16_attention_weight_max": 32.64872741699219,
      "activations/layer16_attention_weight_min": -29.21462059020996,
      "activations/layer17_attention_weight_max": 56.37128829956055,
      "activations/layer17_attention_weight_min": -45.027984619140625,
      "activations/layer18_attention_weight_max": 50.030006408691406,
      "activations/layer18_attention_weight_min": -37.957130432128906,
      "activations/layer19_attention_weight_max": 24.6503963470459,
      "activations/layer19_attention_weight_min": -21.3195858001709,
      "activations/layer1_attention_weight_max": 17.538244247436523,
      "activations/layer1_attention_weight_min": -14.042750358581543,
      "activations/layer20_attention_weight_max": 23.675678253173828,
      "activations/layer20_attention_weight_min": -21.446014404296875,
      "activations/layer21_attention_weight_max": 40.55501937866211,
      "activations/layer21_attention_weight_min": -22.36016082763672,
      "activations/layer22_attention_weight_max": 35.265926361083984,
      "activations/layer22_attention_weight_min": -25.3859920501709,
      "activations/layer23_attention_weight_max": 42.24858856201172,
      "activations/layer23_attention_weight_min": -21.228168487548828,
      "activations/layer2_attention_weight_max": 32.2703971862793,
      "activations/layer2_attention_weight_min": -32.48380661010742,
      "activations/layer3_attention_weight_max": 100.9022216796875,
      "activations/layer3_attention_weight_min": -99.23709106445312,
      "activations/layer4_attention_weight_max": 104.45128631591797,
      "activations/layer4_attention_weight_min": -96.90105438232422,
      "activations/layer5_attention_weight_max": 74.50354766845703,
      "activations/layer5_attention_weight_min": -77.10467529296875,
      "activations/layer6_attention_weight_max": 54.55916213989258,
      "activations/layer6_attention_weight_min": -52.488712310791016,
      "activations/layer7_attention_weight_max": 70.94486999511719,
      "activations/layer7_attention_weight_min": -73.36898040771484,
      "activations/layer8_attention_weight_max": 45.86046600341797,
      "activations/layer8_attention_weight_min": -45.1741828918457,
      "activations/layer9_attention_weight_max": 43.82553482055664,
      "activations/layer9_attention_weight_min": -45.98673629760742,
      "epoch": 11.68,
      "learning_rate": 7.542499999999999e-05,
      "loss": 2.7928,
      "step": 201050
    },
    {
      "activations/layer0_attention_weight_max": 16.525564193725586,
      "activations/layer0_attention_weight_min": -14.404973983764648,
      "activations/layer10_attention_weight_max": 34.314125061035156,
      "activations/layer10_attention_weight_min": -33.591163635253906,
      "activations/layer11_attention_weight_max": 33.154090881347656,
      "activations/layer11_attention_weight_min": -32.45375442504883,
      "activations/layer12_attention_weight_max": 18.190052032470703,
      "activations/layer12_attention_weight_min": -25.470726013183594,
      "activations/layer13_attention_weight_max": 38.033992767333984,
      "activations/layer13_attention_weight_min": -29.782358169555664,
      "activations/layer14_attention_weight_max": 36.98401641845703,
      "activations/layer14_attention_weight_min": -34.23761749267578,
      "activations/layer15_attention_weight_max": 36.013179779052734,
      "activations/layer15_attention_weight_min": -30.368789672851562,
      "activations/layer16_attention_weight_max": 33.39034652709961,
      "activations/layer16_attention_weight_min": -30.18864631652832,
      "activations/layer17_attention_weight_max": 55.09169387817383,
      "activations/layer17_attention_weight_min": -43.63362503051758,
      "activations/layer18_attention_weight_max": 46.89548110961914,
      "activations/layer18_attention_weight_min": -37.64283752441406,
      "activations/layer19_attention_weight_max": 23.16683006286621,
      "activations/layer19_attention_weight_min": -23.5118350982666,
      "activations/layer1_attention_weight_max": 16.304906845092773,
      "activations/layer1_attention_weight_min": -15.433603286743164,
      "activations/layer20_attention_weight_max": 20.966764450073242,
      "activations/layer20_attention_weight_min": -31.33672332763672,
      "activations/layer21_attention_weight_max": 36.57768630981445,
      "activations/layer21_attention_weight_min": -23.809412002563477,
      "activations/layer22_attention_weight_max": 31.1285457611084,
      "activations/layer22_attention_weight_min": -25.52749252319336,
      "activations/layer23_attention_weight_max": 38.384830474853516,
      "activations/layer23_attention_weight_min": -24.642789840698242,
      "activations/layer2_attention_weight_max": 35.19657897949219,
      "activations/layer2_attention_weight_min": -30.924335479736328,
      "activations/layer3_attention_weight_max": 94.11366271972656,
      "activations/layer3_attention_weight_min": -92.122802734375,
      "activations/layer4_attention_weight_max": 95.17381286621094,
      "activations/layer4_attention_weight_min": -91.95955657958984,
      "activations/layer5_attention_weight_max": 73.61238098144531,
      "activations/layer5_attention_weight_min": -77.05575561523438,
      "activations/layer6_attention_weight_max": 52.101009368896484,
      "activations/layer6_attention_weight_min": -50.20859146118164,
      "activations/layer7_attention_weight_max": 70.1854248046875,
      "activations/layer7_attention_weight_min": -67.3318099975586,
      "activations/layer8_attention_weight_max": 42.950584411621094,
      "activations/layer8_attention_weight_min": -45.826995849609375,
      "activations/layer9_attention_weight_max": 44.4950065612793,
      "activations/layer9_attention_weight_min": -46.03475570678711,
      "epoch": 11.69,
      "learning_rate": 7.540606060606059e-05,
      "loss": 2.7714,
      "step": 201100
    },
    {
      "activations/layer0_attention_weight_max": 15.627337455749512,
      "activations/layer0_attention_weight_min": -15.150456428527832,
      "activations/layer10_attention_weight_max": 34.69994354248047,
      "activations/layer10_attention_weight_min": -34.65871047973633,
      "activations/layer11_attention_weight_max": 34.28781509399414,
      "activations/layer11_attention_weight_min": -34.67723083496094,
      "activations/layer12_attention_weight_max": 19.683349609375,
      "activations/layer12_attention_weight_min": -26.457008361816406,
      "activations/layer13_attention_weight_max": 43.257057189941406,
      "activations/layer13_attention_weight_min": -28.783960342407227,
      "activations/layer14_attention_weight_max": 39.827911376953125,
      "activations/layer14_attention_weight_min": -34.39729309082031,
      "activations/layer15_attention_weight_max": 35.72774124145508,
      "activations/layer15_attention_weight_min": -31.11552619934082,
      "activations/layer16_attention_weight_max": 32.47050476074219,
      "activations/layer16_attention_weight_min": -29.93667221069336,
      "activations/layer17_attention_weight_max": 55.315921783447266,
      "activations/layer17_attention_weight_min": -45.98982238769531,
      "activations/layer18_attention_weight_max": 49.0473747253418,
      "activations/layer18_attention_weight_min": -39.087432861328125,
      "activations/layer19_attention_weight_max": 23.190288543701172,
      "activations/layer19_attention_weight_min": -21.20789909362793,
      "activations/layer1_attention_weight_max": 17.605396270751953,
      "activations/layer1_attention_weight_min": -15.654460906982422,
      "activations/layer20_attention_weight_max": 24.4417724609375,
      "activations/layer20_attention_weight_min": -25.354618072509766,
      "activations/layer21_attention_weight_max": 43.12464904785156,
      "activations/layer21_attention_weight_min": -23.683202743530273,
      "activations/layer22_attention_weight_max": 31.05942153930664,
      "activations/layer22_attention_weight_min": -26.149486541748047,
      "activations/layer23_attention_weight_max": 41.47820281982422,
      "activations/layer23_attention_weight_min": -24.52874755859375,
      "activations/layer2_attention_weight_max": 31.810009002685547,
      "activations/layer2_attention_weight_min": -28.913707733154297,
      "activations/layer3_attention_weight_max": 84.60395812988281,
      "activations/layer3_attention_weight_min": -89.64116668701172,
      "activations/layer4_attention_weight_max": 97.12353515625,
      "activations/layer4_attention_weight_min": -95.3807601928711,
      "activations/layer5_attention_weight_max": 73.75241088867188,
      "activations/layer5_attention_weight_min": -77.58067321777344,
      "activations/layer6_attention_weight_max": 52.197364807128906,
      "activations/layer6_attention_weight_min": -53.43886947631836,
      "activations/layer7_attention_weight_max": 69.84333038330078,
      "activations/layer7_attention_weight_min": -68.91328430175781,
      "activations/layer8_attention_weight_max": 45.9432487487793,
      "activations/layer8_attention_weight_min": -46.54973602294922,
      "activations/layer9_attention_weight_max": 45.16766357421875,
      "activations/layer9_attention_weight_min": -49.31327438354492,
      "epoch": 11.69,
      "learning_rate": 7.538787878787879e-05,
      "loss": 2.7916,
      "step": 201150
    },
    {
      "activations/layer0_attention_weight_max": 15.869196891784668,
      "activations/layer0_attention_weight_min": -15.049670219421387,
      "activations/layer10_attention_weight_max": 42.46949768066406,
      "activations/layer10_attention_weight_min": -41.478553771972656,
      "activations/layer11_attention_weight_max": 35.724796295166016,
      "activations/layer11_attention_weight_min": -36.57459259033203,
      "activations/layer12_attention_weight_max": 19.675851821899414,
      "activations/layer12_attention_weight_min": -23.83260726928711,
      "activations/layer13_attention_weight_max": 43.20886993408203,
      "activations/layer13_attention_weight_min": -30.074254989624023,
      "activations/layer14_attention_weight_max": 36.67649459838867,
      "activations/layer14_attention_weight_min": -30.5432071685791,
      "activations/layer15_attention_weight_max": 35.439449310302734,
      "activations/layer15_attention_weight_min": -29.4831485748291,
      "activations/layer16_attention_weight_max": 31.511857986450195,
      "activations/layer16_attention_weight_min": -28.52869415283203,
      "activations/layer17_attention_weight_max": 51.65760040283203,
      "activations/layer17_attention_weight_min": -40.62979507446289,
      "activations/layer18_attention_weight_max": 45.48271179199219,
      "activations/layer18_attention_weight_min": -36.476993560791016,
      "activations/layer19_attention_weight_max": 24.190000534057617,
      "activations/layer19_attention_weight_min": -21.26703453063965,
      "activations/layer1_attention_weight_max": 17.17856788635254,
      "activations/layer1_attention_weight_min": -15.822805404663086,
      "activations/layer20_attention_weight_max": 24.394779205322266,
      "activations/layer20_attention_weight_min": -21.261058807373047,
      "activations/layer21_attention_weight_max": 46.92106628417969,
      "activations/layer21_attention_weight_min": -25.498369216918945,
      "activations/layer22_attention_weight_max": 35.65201950073242,
      "activations/layer22_attention_weight_min": -24.71442413330078,
      "activations/layer23_attention_weight_max": 42.86207580566406,
      "activations/layer23_attention_weight_min": -24.646385192871094,
      "activations/layer2_attention_weight_max": 32.89649963378906,
      "activations/layer2_attention_weight_min": -32.1202278137207,
      "activations/layer3_attention_weight_max": 95.7488784790039,
      "activations/layer3_attention_weight_min": -97.72827911376953,
      "activations/layer4_attention_weight_max": 102.63885498046875,
      "activations/layer4_attention_weight_min": -100.1850357055664,
      "activations/layer5_attention_weight_max": 73.1785888671875,
      "activations/layer5_attention_weight_min": -83.01060485839844,
      "activations/layer6_attention_weight_max": 58.682647705078125,
      "activations/layer6_attention_weight_min": -57.586341857910156,
      "activations/layer7_attention_weight_max": 76.24201965332031,
      "activations/layer7_attention_weight_min": -77.70674896240234,
      "activations/layer8_attention_weight_max": 50.32425308227539,
      "activations/layer8_attention_weight_min": -55.391136169433594,
      "activations/layer9_attention_weight_max": 53.26032257080078,
      "activations/layer9_attention_weight_min": -56.5328254699707,
      "epoch": 11.69,
      "learning_rate": 7.536893939393939e-05,
      "loss": 2.7789,
      "step": 201200
    },
    {
      "activations/layer0_attention_weight_max": 16.12939453125,
      "activations/layer0_attention_weight_min": -15.024981498718262,
      "activations/layer10_attention_weight_max": 39.84121322631836,
      "activations/layer10_attention_weight_min": -37.96028518676758,
      "activations/layer11_attention_weight_max": 34.480812072753906,
      "activations/layer11_attention_weight_min": -34.04970169067383,
      "activations/layer12_attention_weight_max": 19.859888076782227,
      "activations/layer12_attention_weight_min": -28.19577407836914,
      "activations/layer13_attention_weight_max": 43.2270622253418,
      "activations/layer13_attention_weight_min": -30.060455322265625,
      "activations/layer14_attention_weight_max": 41.81877899169922,
      "activations/layer14_attention_weight_min": -31.614036560058594,
      "activations/layer15_attention_weight_max": 35.590980529785156,
      "activations/layer15_attention_weight_min": -29.941486358642578,
      "activations/layer16_attention_weight_max": 32.75292205810547,
      "activations/layer16_attention_weight_min": -29.29836082458496,
      "activations/layer17_attention_weight_max": 54.61130142211914,
      "activations/layer17_attention_weight_min": -43.551395416259766,
      "activations/layer18_attention_weight_max": 48.48042678833008,
      "activations/layer18_attention_weight_min": -37.16471481323242,
      "activations/layer19_attention_weight_max": 25.094289779663086,
      "activations/layer19_attention_weight_min": -21.410371780395508,
      "activations/layer1_attention_weight_max": 15.757782936096191,
      "activations/layer1_attention_weight_min": -14.754533767700195,
      "activations/layer20_attention_weight_max": 24.23000144958496,
      "activations/layer20_attention_weight_min": -23.29795265197754,
      "activations/layer21_attention_weight_max": 39.61738967895508,
      "activations/layer21_attention_weight_min": -21.958404541015625,
      "activations/layer22_attention_weight_max": 33.677635192871094,
      "activations/layer22_attention_weight_min": -25.260332107543945,
      "activations/layer23_attention_weight_max": 41.73638153076172,
      "activations/layer23_attention_weight_min": -25.361562728881836,
      "activations/layer2_attention_weight_max": 31.69025230407715,
      "activations/layer2_attention_weight_min": -30.38880157470703,
      "activations/layer3_attention_weight_max": 91.69104766845703,
      "activations/layer3_attention_weight_min": -92.19467163085938,
      "activations/layer4_attention_weight_max": 100.98776245117188,
      "activations/layer4_attention_weight_min": -96.42467498779297,
      "activations/layer5_attention_weight_max": 71.8865966796875,
      "activations/layer5_attention_weight_min": -79.1636734008789,
      "activations/layer6_attention_weight_max": 56.4740104675293,
      "activations/layer6_attention_weight_min": -53.81837844848633,
      "activations/layer7_attention_weight_max": 74.57032775878906,
      "activations/layer7_attention_weight_min": -80.349365234375,
      "activations/layer8_attention_weight_max": 48.32886505126953,
      "activations/layer8_attention_weight_min": -50.47169876098633,
      "activations/layer9_attention_weight_max": 52.92626953125,
      "activations/layer9_attention_weight_min": -50.81343460083008,
      "epoch": 11.69,
      "learning_rate": 7.534999999999999e-05,
      "loss": 2.78,
      "step": 201250
    },
    {
      "activations/layer0_attention_weight_max": 16.265066146850586,
      "activations/layer0_attention_weight_min": -14.888343811035156,
      "activations/layer10_attention_weight_max": 34.37257385253906,
      "activations/layer10_attention_weight_min": -34.80085754394531,
      "activations/layer11_attention_weight_max": 30.756635665893555,
      "activations/layer11_attention_weight_min": -32.02970886230469,
      "activations/layer12_attention_weight_max": 20.216672897338867,
      "activations/layer12_attention_weight_min": -25.066864013671875,
      "activations/layer13_attention_weight_max": 34.88622283935547,
      "activations/layer13_attention_weight_min": -29.22946548461914,
      "activations/layer14_attention_weight_max": 38.20474624633789,
      "activations/layer14_attention_weight_min": -30.067829132080078,
      "activations/layer15_attention_weight_max": 34.52667236328125,
      "activations/layer15_attention_weight_min": -29.55796241760254,
      "activations/layer16_attention_weight_max": 34.749916076660156,
      "activations/layer16_attention_weight_min": -30.692386627197266,
      "activations/layer17_attention_weight_max": 57.10268020629883,
      "activations/layer17_attention_weight_min": -46.306514739990234,
      "activations/layer18_attention_weight_max": 50.69904708862305,
      "activations/layer18_attention_weight_min": -37.8280143737793,
      "activations/layer19_attention_weight_max": 25.00776481628418,
      "activations/layer19_attention_weight_min": -21.787534713745117,
      "activations/layer1_attention_weight_max": 17.601524353027344,
      "activations/layer1_attention_weight_min": -14.014430046081543,
      "activations/layer20_attention_weight_max": 22.972572326660156,
      "activations/layer20_attention_weight_min": -22.167388916015625,
      "activations/layer21_attention_weight_max": 36.63579559326172,
      "activations/layer21_attention_weight_min": -22.840011596679688,
      "activations/layer22_attention_weight_max": 41.032047271728516,
      "activations/layer22_attention_weight_min": -27.329126358032227,
      "activations/layer23_attention_weight_max": 40.82034683227539,
      "activations/layer23_attention_weight_min": -23.728736877441406,
      "activations/layer2_attention_weight_max": 31.993301391601562,
      "activations/layer2_attention_weight_min": -28.991592407226562,
      "activations/layer3_attention_weight_max": 92.00519561767578,
      "activations/layer3_attention_weight_min": -92.1067886352539,
      "activations/layer4_attention_weight_max": 95.07494354248047,
      "activations/layer4_attention_weight_min": -91.93087768554688,
      "activations/layer5_attention_weight_max": 72.97386169433594,
      "activations/layer5_attention_weight_min": -77.68851470947266,
      "activations/layer6_attention_weight_max": 53.69235610961914,
      "activations/layer6_attention_weight_min": -56.61317825317383,
      "activations/layer7_attention_weight_max": 77.72914123535156,
      "activations/layer7_attention_weight_min": -71.84564971923828,
      "activations/layer8_attention_weight_max": 43.475921630859375,
      "activations/layer8_attention_weight_min": -48.24757385253906,
      "activations/layer9_attention_weight_max": 45.0565299987793,
      "activations/layer9_attention_weight_min": -47.19416427612305,
      "epoch": 11.7,
      "learning_rate": 7.53310606060606e-05,
      "loss": 2.7625,
      "step": 201300
    },
    {
      "activations/layer0_attention_weight_max": 15.679688453674316,
      "activations/layer0_attention_weight_min": -14.941014289855957,
      "activations/layer10_attention_weight_max": 34.33628845214844,
      "activations/layer10_attention_weight_min": -35.2166748046875,
      "activations/layer11_attention_weight_max": 31.487133026123047,
      "activations/layer11_attention_weight_min": -32.245628356933594,
      "activations/layer12_attention_weight_max": 18.379467010498047,
      "activations/layer12_attention_weight_min": -23.853656768798828,
      "activations/layer13_attention_weight_max": 41.28441619873047,
      "activations/layer13_attention_weight_min": -30.5129337310791,
      "activations/layer14_attention_weight_max": 46.40367126464844,
      "activations/layer14_attention_weight_min": -30.368833541870117,
      "activations/layer15_attention_weight_max": 36.164894104003906,
      "activations/layer15_attention_weight_min": -28.38857078552246,
      "activations/layer16_attention_weight_max": 32.917198181152344,
      "activations/layer16_attention_weight_min": -31.52506446838379,
      "activations/layer17_attention_weight_max": 52.29372024536133,
      "activations/layer17_attention_weight_min": -45.423728942871094,
      "activations/layer18_attention_weight_max": 50.320404052734375,
      "activations/layer18_attention_weight_min": -37.375648498535156,
      "activations/layer19_attention_weight_max": 23.315898895263672,
      "activations/layer19_attention_weight_min": -21.642175674438477,
      "activations/layer1_attention_weight_max": 17.014495849609375,
      "activations/layer1_attention_weight_min": -15.63720417022705,
      "activations/layer20_attention_weight_max": 24.839616775512695,
      "activations/layer20_attention_weight_min": -20.147743225097656,
      "activations/layer21_attention_weight_max": 34.38215637207031,
      "activations/layer21_attention_weight_min": -23.643518447875977,
      "activations/layer22_attention_weight_max": 31.400754928588867,
      "activations/layer22_attention_weight_min": -24.968544006347656,
      "activations/layer23_attention_weight_max": 41.82445526123047,
      "activations/layer23_attention_weight_min": -25.94839096069336,
      "activations/layer2_attention_weight_max": 32.12625503540039,
      "activations/layer2_attention_weight_min": -30.36160659790039,
      "activations/layer3_attention_weight_max": 90.84342956542969,
      "activations/layer3_attention_weight_min": -90.81997680664062,
      "activations/layer4_attention_weight_max": 95.64128875732422,
      "activations/layer4_attention_weight_min": -93.55171966552734,
      "activations/layer5_attention_weight_max": 72.83343505859375,
      "activations/layer5_attention_weight_min": -80.3017349243164,
      "activations/layer6_attention_weight_max": 49.60529708862305,
      "activations/layer6_attention_weight_min": -51.95266342163086,
      "activations/layer7_attention_weight_max": 74.22080993652344,
      "activations/layer7_attention_weight_min": -68.92818450927734,
      "activations/layer8_attention_weight_max": 42.83692169189453,
      "activations/layer8_attention_weight_min": -46.533485412597656,
      "activations/layer9_attention_weight_max": 48.808990478515625,
      "activations/layer9_attention_weight_min": -47.678646087646484,
      "epoch": 11.7,
      "learning_rate": 7.531212121212122e-05,
      "loss": 2.7822,
      "step": 201350
    },
    {
      "activations/layer0_attention_weight_max": 16.39496612548828,
      "activations/layer0_attention_weight_min": -14.9815673828125,
      "activations/layer10_attention_weight_max": 34.77706527709961,
      "activations/layer10_attention_weight_min": -34.628173828125,
      "activations/layer11_attention_weight_max": 32.58533477783203,
      "activations/layer11_attention_weight_min": -32.680545806884766,
      "activations/layer12_attention_weight_max": 17.666810989379883,
      "activations/layer12_attention_weight_min": -26.155561447143555,
      "activations/layer13_attention_weight_max": 35.5966796875,
      "activations/layer13_attention_weight_min": -29.450960159301758,
      "activations/layer14_attention_weight_max": 35.4670295715332,
      "activations/layer14_attention_weight_min": -29.545656204223633,
      "activations/layer15_attention_weight_max": 32.44769287109375,
      "activations/layer15_attention_weight_min": -29.6400146484375,
      "activations/layer16_attention_weight_max": 29.72904396057129,
      "activations/layer16_attention_weight_min": -29.568260192871094,
      "activations/layer17_attention_weight_max": 50.00453186035156,
      "activations/layer17_attention_weight_min": -43.526973724365234,
      "activations/layer18_attention_weight_max": 41.71963882446289,
      "activations/layer18_attention_weight_min": -36.89375686645508,
      "activations/layer19_attention_weight_max": 19.060546875,
      "activations/layer19_attention_weight_min": -21.685626983642578,
      "activations/layer1_attention_weight_max": 17.663997650146484,
      "activations/layer1_attention_weight_min": -14.796008110046387,
      "activations/layer20_attention_weight_max": 19.806949615478516,
      "activations/layer20_attention_weight_min": -19.654518127441406,
      "activations/layer21_attention_weight_max": 36.09070587158203,
      "activations/layer21_attention_weight_min": -22.262046813964844,
      "activations/layer22_attention_weight_max": 29.434223175048828,
      "activations/layer22_attention_weight_min": -25.703983306884766,
      "activations/layer23_attention_weight_max": 35.396461486816406,
      "activations/layer23_attention_weight_min": -25.032567977905273,
      "activations/layer2_attention_weight_max": 31.83028221130371,
      "activations/layer2_attention_weight_min": -30.01699447631836,
      "activations/layer3_attention_weight_max": 95.32286834716797,
      "activations/layer3_attention_weight_min": -96.43865966796875,
      "activations/layer4_attention_weight_max": 103.45222473144531,
      "activations/layer4_attention_weight_min": -99.0091781616211,
      "activations/layer5_attention_weight_max": 73.99809265136719,
      "activations/layer5_attention_weight_min": -79.14274597167969,
      "activations/layer6_attention_weight_max": 53.323368072509766,
      "activations/layer6_attention_weight_min": -52.9267692565918,
      "activations/layer7_attention_weight_max": 74.84451293945312,
      "activations/layer7_attention_weight_min": -69.40169525146484,
      "activations/layer8_attention_weight_max": 50.12759780883789,
      "activations/layer8_attention_weight_min": -51.346439361572266,
      "activations/layer9_attention_weight_max": 53.33479690551758,
      "activations/layer9_attention_weight_min": -45.78134536743164,
      "epoch": 11.7,
      "learning_rate": 7.52931818181818e-05,
      "loss": 2.7873,
      "step": 201400
    },
    {
      "activations/layer0_attention_weight_max": 15.99126148223877,
      "activations/layer0_attention_weight_min": -14.579599380493164,
      "activations/layer10_attention_weight_max": 34.70001983642578,
      "activations/layer10_attention_weight_min": -36.762672424316406,
      "activations/layer11_attention_weight_max": 31.173673629760742,
      "activations/layer11_attention_weight_min": -33.79979705810547,
      "activations/layer12_attention_weight_max": 19.699697494506836,
      "activations/layer12_attention_weight_min": -26.243425369262695,
      "activations/layer13_attention_weight_max": 36.555816650390625,
      "activations/layer13_attention_weight_min": -31.006914138793945,
      "activations/layer14_attention_weight_max": 35.71025085449219,
      "activations/layer14_attention_weight_min": -29.80017852783203,
      "activations/layer15_attention_weight_max": 33.00797653198242,
      "activations/layer15_attention_weight_min": -28.925575256347656,
      "activations/layer16_attention_weight_max": 33.337162017822266,
      "activations/layer16_attention_weight_min": -30.785810470581055,
      "activations/layer17_attention_weight_max": 53.11317825317383,
      "activations/layer17_attention_weight_min": -44.683021545410156,
      "activations/layer18_attention_weight_max": 47.52068328857422,
      "activations/layer18_attention_weight_min": -36.854408264160156,
      "activations/layer19_attention_weight_max": 22.9453182220459,
      "activations/layer19_attention_weight_min": -20.56974983215332,
      "activations/layer1_attention_weight_max": 16.692615509033203,
      "activations/layer1_attention_weight_min": -13.727154731750488,
      "activations/layer20_attention_weight_max": 22.875896453857422,
      "activations/layer20_attention_weight_min": -20.77895736694336,
      "activations/layer21_attention_weight_max": 37.67570495605469,
      "activations/layer21_attention_weight_min": -23.63951301574707,
      "activations/layer22_attention_weight_max": 33.33845520019531,
      "activations/layer22_attention_weight_min": -24.46144676208496,
      "activations/layer23_attention_weight_max": 39.4666862487793,
      "activations/layer23_attention_weight_min": -24.81842613220215,
      "activations/layer2_attention_weight_max": 30.723644256591797,
      "activations/layer2_attention_weight_min": -29.75250244140625,
      "activations/layer3_attention_weight_max": 89.7850112915039,
      "activations/layer3_attention_weight_min": -92.10687255859375,
      "activations/layer4_attention_weight_max": 99.61940002441406,
      "activations/layer4_attention_weight_min": -95.594482421875,
      "activations/layer5_attention_weight_max": 76.00587463378906,
      "activations/layer5_attention_weight_min": -82.90525817871094,
      "activations/layer6_attention_weight_max": 55.56660079956055,
      "activations/layer6_attention_weight_min": -54.49785232543945,
      "activations/layer7_attention_weight_max": 80.61965942382812,
      "activations/layer7_attention_weight_min": -75.35552978515625,
      "activations/layer8_attention_weight_max": 44.95241165161133,
      "activations/layer8_attention_weight_min": -48.26069259643555,
      "activations/layer9_attention_weight_max": 47.047760009765625,
      "activations/layer9_attention_weight_min": -50.359859466552734,
      "epoch": 11.71,
      "learning_rate": 7.527424242424242e-05,
      "loss": 2.7901,
      "step": 201450
    },
    {
      "activations/layer0_attention_weight_max": 16.42935562133789,
      "activations/layer0_attention_weight_min": -15.250297546386719,
      "activations/layer10_attention_weight_max": 36.17084503173828,
      "activations/layer10_attention_weight_min": -34.291168212890625,
      "activations/layer11_attention_weight_max": 29.597684860229492,
      "activations/layer11_attention_weight_min": -31.658485412597656,
      "activations/layer12_attention_weight_max": 18.41769790649414,
      "activations/layer12_attention_weight_min": -28.18111801147461,
      "activations/layer13_attention_weight_max": 41.2534294128418,
      "activations/layer13_attention_weight_min": -28.246126174926758,
      "activations/layer14_attention_weight_max": 34.30291748046875,
      "activations/layer14_attention_weight_min": -32.15335464477539,
      "activations/layer15_attention_weight_max": 32.74183654785156,
      "activations/layer15_attention_weight_min": -30.57200050354004,
      "activations/layer16_attention_weight_max": 32.20954895019531,
      "activations/layer16_attention_weight_min": -29.341360092163086,
      "activations/layer17_attention_weight_max": 49.41400146484375,
      "activations/layer17_attention_weight_min": -46.76976013183594,
      "activations/layer18_attention_weight_max": 44.42094039916992,
      "activations/layer18_attention_weight_min": -38.64906311035156,
      "activations/layer19_attention_weight_max": 20.725893020629883,
      "activations/layer19_attention_weight_min": -24.705984115600586,
      "activations/layer1_attention_weight_max": 16.040231704711914,
      "activations/layer1_attention_weight_min": -13.959720611572266,
      "activations/layer20_attention_weight_max": 21.054237365722656,
      "activations/layer20_attention_weight_min": -23.171390533447266,
      "activations/layer21_attention_weight_max": 32.40373611450195,
      "activations/layer21_attention_weight_min": -24.7181453704834,
      "activations/layer22_attention_weight_max": 29.239545822143555,
      "activations/layer22_attention_weight_min": -28.52005386352539,
      "activations/layer23_attention_weight_max": 37.96824264526367,
      "activations/layer23_attention_weight_min": -26.1364803314209,
      "activations/layer2_attention_weight_max": 32.676597595214844,
      "activations/layer2_attention_weight_min": -31.703941345214844,
      "activations/layer3_attention_weight_max": 92.3085708618164,
      "activations/layer3_attention_weight_min": -94.83209991455078,
      "activations/layer4_attention_weight_max": 95.67476654052734,
      "activations/layer4_attention_weight_min": -96.90459442138672,
      "activations/layer5_attention_weight_max": 73.1966323852539,
      "activations/layer5_attention_weight_min": -78.31737518310547,
      "activations/layer6_attention_weight_max": 52.737953186035156,
      "activations/layer6_attention_weight_min": -53.84556198120117,
      "activations/layer7_attention_weight_max": 71.42743682861328,
      "activations/layer7_attention_weight_min": -69.26654815673828,
      "activations/layer8_attention_weight_max": 45.576080322265625,
      "activations/layer8_attention_weight_min": -49.018714904785156,
      "activations/layer9_attention_weight_max": 45.065242767333984,
      "activations/layer9_attention_weight_min": -44.92414093017578,
      "epoch": 11.71,
      "learning_rate": 7.525530303030301e-05,
      "loss": 2.7726,
      "step": 201500
    },
    {
      "activations/layer0_attention_weight_max": 16.11849594116211,
      "activations/layer0_attention_weight_min": -14.612451553344727,
      "activations/layer10_attention_weight_max": 36.316280364990234,
      "activations/layer10_attention_weight_min": -37.35772705078125,
      "activations/layer11_attention_weight_max": 31.10216522216797,
      "activations/layer11_attention_weight_min": -32.06080627441406,
      "activations/layer12_attention_weight_max": 17.67388916015625,
      "activations/layer12_attention_weight_min": -22.972129821777344,
      "activations/layer13_attention_weight_max": 31.980751037597656,
      "activations/layer13_attention_weight_min": -29.90445899963379,
      "activations/layer14_attention_weight_max": 32.882137298583984,
      "activations/layer14_attention_weight_min": -30.58080291748047,
      "activations/layer15_attention_weight_max": 30.548776626586914,
      "activations/layer15_attention_weight_min": -30.781030654907227,
      "activations/layer16_attention_weight_max": 29.533763885498047,
      "activations/layer16_attention_weight_min": -29.267105102539062,
      "activations/layer17_attention_weight_max": 51.3245735168457,
      "activations/layer17_attention_weight_min": -45.576080322265625,
      "activations/layer18_attention_weight_max": 45.43552017211914,
      "activations/layer18_attention_weight_min": -38.543251037597656,
      "activations/layer19_attention_weight_max": 21.008346557617188,
      "activations/layer19_attention_weight_min": -21.94710922241211,
      "activations/layer1_attention_weight_max": 16.197355270385742,
      "activations/layer1_attention_weight_min": -14.840742111206055,
      "activations/layer20_attention_weight_max": 22.621788024902344,
      "activations/layer20_attention_weight_min": -23.709800720214844,
      "activations/layer21_attention_weight_max": 37.79703140258789,
      "activations/layer21_attention_weight_min": -24.894779205322266,
      "activations/layer22_attention_weight_max": 34.77503204345703,
      "activations/layer22_attention_weight_min": -27.2725830078125,
      "activations/layer23_attention_weight_max": 36.868316650390625,
      "activations/layer23_attention_weight_min": -26.37445640563965,
      "activations/layer2_attention_weight_max": 32.018287658691406,
      "activations/layer2_attention_weight_min": -31.560964584350586,
      "activations/layer3_attention_weight_max": 91.39775848388672,
      "activations/layer3_attention_weight_min": -96.02571105957031,
      "activations/layer4_attention_weight_max": 97.1981430053711,
      "activations/layer4_attention_weight_min": -92.78779602050781,
      "activations/layer5_attention_weight_max": 74.92706298828125,
      "activations/layer5_attention_weight_min": -82.54753112792969,
      "activations/layer6_attention_weight_max": 51.447811126708984,
      "activations/layer6_attention_weight_min": -54.869544982910156,
      "activations/layer7_attention_weight_max": 76.05322265625,
      "activations/layer7_attention_weight_min": -72.1266860961914,
      "activations/layer8_attention_weight_max": 45.52106475830078,
      "activations/layer8_attention_weight_min": -50.04201889038086,
      "activations/layer9_attention_weight_max": 47.99443435668945,
      "activations/layer9_attention_weight_min": -46.699066162109375,
      "epoch": 11.71,
      "learning_rate": 7.523636363636362e-05,
      "loss": 2.7859,
      "step": 201550
    },
    {
      "activations/layer0_attention_weight_max": 16.964115142822266,
      "activations/layer0_attention_weight_min": -15.025113105773926,
      "activations/layer10_attention_weight_max": 36.0152702331543,
      "activations/layer10_attention_weight_min": -35.792030334472656,
      "activations/layer11_attention_weight_max": 31.22555160522461,
      "activations/layer11_attention_weight_min": -32.88129425048828,
      "activations/layer12_attention_weight_max": 17.646404266357422,
      "activations/layer12_attention_weight_min": -25.8465518951416,
      "activations/layer13_attention_weight_max": 37.802303314208984,
      "activations/layer13_attention_weight_min": -29.818931579589844,
      "activations/layer14_attention_weight_max": 36.146663665771484,
      "activations/layer14_attention_weight_min": -29.835073471069336,
      "activations/layer15_attention_weight_max": 33.77762985229492,
      "activations/layer15_attention_weight_min": -28.45879364013672,
      "activations/layer16_attention_weight_max": 31.373714447021484,
      "activations/layer16_attention_weight_min": -27.632341384887695,
      "activations/layer17_attention_weight_max": 53.72438430786133,
      "activations/layer17_attention_weight_min": -44.45838928222656,
      "activations/layer18_attention_weight_max": 47.20520782470703,
      "activations/layer18_attention_weight_min": -38.95258712768555,
      "activations/layer19_attention_weight_max": 22.566102981567383,
      "activations/layer19_attention_weight_min": -23.16621208190918,
      "activations/layer1_attention_weight_max": 16.608890533447266,
      "activations/layer1_attention_weight_min": -15.49238109588623,
      "activations/layer20_attention_weight_max": 24.918384552001953,
      "activations/layer20_attention_weight_min": -23.433992385864258,
      "activations/layer21_attention_weight_max": 40.590065002441406,
      "activations/layer21_attention_weight_min": -22.20859718322754,
      "activations/layer22_attention_weight_max": 32.40031814575195,
      "activations/layer22_attention_weight_min": -25.37380027770996,
      "activations/layer23_attention_weight_max": 40.605804443359375,
      "activations/layer23_attention_weight_min": -25.749107360839844,
      "activations/layer2_attention_weight_max": 31.68891143798828,
      "activations/layer2_attention_weight_min": -30.724124908447266,
      "activations/layer3_attention_weight_max": 91.97772216796875,
      "activations/layer3_attention_weight_min": -95.28020477294922,
      "activations/layer4_attention_weight_max": 101.63175201416016,
      "activations/layer4_attention_weight_min": -99.34883880615234,
      "activations/layer5_attention_weight_max": 71.87553405761719,
      "activations/layer5_attention_weight_min": -79.45370483398438,
      "activations/layer6_attention_weight_max": 53.84717559814453,
      "activations/layer6_attention_weight_min": -53.237979888916016,
      "activations/layer7_attention_weight_max": 69.99549865722656,
      "activations/layer7_attention_weight_min": -69.14578247070312,
      "activations/layer8_attention_weight_max": 44.391273498535156,
      "activations/layer8_attention_weight_min": -48.269039154052734,
      "activations/layer9_attention_weight_max": 47.66263961791992,
      "activations/layer9_attention_weight_min": -46.63436508178711,
      "epoch": 11.71,
      "learning_rate": 7.521742424242424e-05,
      "loss": 2.7706,
      "step": 201600
    },
    {
      "activations/layer0_attention_weight_max": 15.981521606445312,
      "activations/layer0_attention_weight_min": -15.218161582946777,
      "activations/layer10_attention_weight_max": 36.4285774230957,
      "activations/layer10_attention_weight_min": -35.163429260253906,
      "activations/layer11_attention_weight_max": 31.360393524169922,
      "activations/layer11_attention_weight_min": -31.51150131225586,
      "activations/layer12_attention_weight_max": 19.137531280517578,
      "activations/layer12_attention_weight_min": -23.53207015991211,
      "activations/layer13_attention_weight_max": 37.83596420288086,
      "activations/layer13_attention_weight_min": -27.520980834960938,
      "activations/layer14_attention_weight_max": 35.00630187988281,
      "activations/layer14_attention_weight_min": -29.748111724853516,
      "activations/layer15_attention_weight_max": 31.007410049438477,
      "activations/layer15_attention_weight_min": -29.673866271972656,
      "activations/layer16_attention_weight_max": 29.516624450683594,
      "activations/layer16_attention_weight_min": -29.58584976196289,
      "activations/layer17_attention_weight_max": 51.479000091552734,
      "activations/layer17_attention_weight_min": -44.11689376831055,
      "activations/layer18_attention_weight_max": 44.09361267089844,
      "activations/layer18_attention_weight_min": -35.798866271972656,
      "activations/layer19_attention_weight_max": 22.66753387451172,
      "activations/layer19_attention_weight_min": -20.859596252441406,
      "activations/layer1_attention_weight_max": 16.57258415222168,
      "activations/layer1_attention_weight_min": -16.31528091430664,
      "activations/layer20_attention_weight_max": 22.785459518432617,
      "activations/layer20_attention_weight_min": -22.690656661987305,
      "activations/layer21_attention_weight_max": 32.79753112792969,
      "activations/layer21_attention_weight_min": -20.712636947631836,
      "activations/layer22_attention_weight_max": 27.687213897705078,
      "activations/layer22_attention_weight_min": -26.316553115844727,
      "activations/layer23_attention_weight_max": 32.4586296081543,
      "activations/layer23_attention_weight_min": -22.26465606689453,
      "activations/layer2_attention_weight_max": 33.188270568847656,
      "activations/layer2_attention_weight_min": -33.014015197753906,
      "activations/layer3_attention_weight_max": 95.27490997314453,
      "activations/layer3_attention_weight_min": -98.72709655761719,
      "activations/layer4_attention_weight_max": 99.5025863647461,
      "activations/layer4_attention_weight_min": -97.88774871826172,
      "activations/layer5_attention_weight_max": 72.71531677246094,
      "activations/layer5_attention_weight_min": -82.19035339355469,
      "activations/layer6_attention_weight_max": 52.54221725463867,
      "activations/layer6_attention_weight_min": -54.42045593261719,
      "activations/layer7_attention_weight_max": 81.49626922607422,
      "activations/layer7_attention_weight_min": -72.18616485595703,
      "activations/layer8_attention_weight_max": 45.821434020996094,
      "activations/layer8_attention_weight_min": -48.66170120239258,
      "activations/layer9_attention_weight_max": 46.83713912963867,
      "activations/layer9_attention_weight_min": -45.90884017944336,
      "epoch": 11.72,
      "learning_rate": 7.519848484848484e-05,
      "loss": 2.7865,
      "step": 201650
    },
    {
      "activations/layer0_attention_weight_max": 15.562787055969238,
      "activations/layer0_attention_weight_min": -14.563278198242188,
      "activations/layer10_attention_weight_max": 35.706878662109375,
      "activations/layer10_attention_weight_min": -35.48627853393555,
      "activations/layer11_attention_weight_max": 30.601743698120117,
      "activations/layer11_attention_weight_min": -32.095794677734375,
      "activations/layer12_attention_weight_max": 18.169580459594727,
      "activations/layer12_attention_weight_min": -25.88973617553711,
      "activations/layer13_attention_weight_max": 33.59602355957031,
      "activations/layer13_attention_weight_min": -27.74452781677246,
      "activations/layer14_attention_weight_max": 35.18112564086914,
      "activations/layer14_attention_weight_min": -28.80938720703125,
      "activations/layer15_attention_weight_max": 32.238487243652344,
      "activations/layer15_attention_weight_min": -28.18084144592285,
      "activations/layer16_attention_weight_max": 29.555517196655273,
      "activations/layer16_attention_weight_min": -28.786706924438477,
      "activations/layer17_attention_weight_max": 49.9543342590332,
      "activations/layer17_attention_weight_min": -40.776039123535156,
      "activations/layer18_attention_weight_max": 42.61631393432617,
      "activations/layer18_attention_weight_min": -36.3881950378418,
      "activations/layer19_attention_weight_max": 20.30116844177246,
      "activations/layer19_attention_weight_min": -22.716222763061523,
      "activations/layer1_attention_weight_max": 16.270383834838867,
      "activations/layer1_attention_weight_min": -14.355936050415039,
      "activations/layer20_attention_weight_max": 22.687179565429688,
      "activations/layer20_attention_weight_min": -23.04414939880371,
      "activations/layer21_attention_weight_max": 35.678916931152344,
      "activations/layer21_attention_weight_min": -22.356950759887695,
      "activations/layer22_attention_weight_max": 31.709890365600586,
      "activations/layer22_attention_weight_min": -26.408994674682617,
      "activations/layer23_attention_weight_max": 34.30561447143555,
      "activations/layer23_attention_weight_min": -23.49987030029297,
      "activations/layer2_attention_weight_max": 32.6640510559082,
      "activations/layer2_attention_weight_min": -30.551795959472656,
      "activations/layer3_attention_weight_max": 96.52590942382812,
      "activations/layer3_attention_weight_min": -100.3739013671875,
      "activations/layer4_attention_weight_max": 106.52081298828125,
      "activations/layer4_attention_weight_min": -100.66191101074219,
      "activations/layer5_attention_weight_max": 74.69644165039062,
      "activations/layer5_attention_weight_min": -77.09024047851562,
      "activations/layer6_attention_weight_max": 50.63949966430664,
      "activations/layer6_attention_weight_min": -50.84149932861328,
      "activations/layer7_attention_weight_max": 72.22612762451172,
      "activations/layer7_attention_weight_min": -71.41907501220703,
      "activations/layer8_attention_weight_max": 44.553993225097656,
      "activations/layer8_attention_weight_min": -46.396263122558594,
      "activations/layer9_attention_weight_max": 47.5087776184082,
      "activations/layer9_attention_weight_min": -48.06224822998047,
      "epoch": 11.72,
      "learning_rate": 7.517954545454544e-05,
      "loss": 2.7899,
      "step": 201700
    },
    {
      "activations/layer0_attention_weight_max": 16.100177764892578,
      "activations/layer0_attention_weight_min": -15.316545486450195,
      "activations/layer10_attention_weight_max": 44.08452606201172,
      "activations/layer10_attention_weight_min": -45.082618713378906,
      "activations/layer11_attention_weight_max": 38.20201873779297,
      "activations/layer11_attention_weight_min": -38.7301025390625,
      "activations/layer12_attention_weight_max": 19.314558029174805,
      "activations/layer12_attention_weight_min": -26.10775375366211,
      "activations/layer13_attention_weight_max": 54.12138748168945,
      "activations/layer13_attention_weight_min": -32.17827224731445,
      "activations/layer14_attention_weight_max": 44.78725814819336,
      "activations/layer14_attention_weight_min": -30.573835372924805,
      "activations/layer15_attention_weight_max": 37.175655364990234,
      "activations/layer15_attention_weight_min": -31.000465393066406,
      "activations/layer16_attention_weight_max": 37.716835021972656,
      "activations/layer16_attention_weight_min": -29.97063636779785,
      "activations/layer17_attention_weight_max": 59.61064529418945,
      "activations/layer17_attention_weight_min": -46.833580017089844,
      "activations/layer18_attention_weight_max": 53.75947189331055,
      "activations/layer18_attention_weight_min": -39.37300109863281,
      "activations/layer19_attention_weight_max": 23.86851692199707,
      "activations/layer19_attention_weight_min": -22.646703720092773,
      "activations/layer1_attention_weight_max": 18.32544708251953,
      "activations/layer1_attention_weight_min": -14.216428756713867,
      "activations/layer20_attention_weight_max": 26.539806365966797,
      "activations/layer20_attention_weight_min": -21.730257034301758,
      "activations/layer21_attention_weight_max": 46.2611083984375,
      "activations/layer21_attention_weight_min": -21.565582275390625,
      "activations/layer22_attention_weight_max": 37.25086212158203,
      "activations/layer22_attention_weight_min": -26.12862205505371,
      "activations/layer23_attention_weight_max": 50.06378936767578,
      "activations/layer23_attention_weight_min": -23.09585952758789,
      "activations/layer2_attention_weight_max": 33.853546142578125,
      "activations/layer2_attention_weight_min": -29.614620208740234,
      "activations/layer3_attention_weight_max": 95.89645385742188,
      "activations/layer3_attention_weight_min": -90.44393920898438,
      "activations/layer4_attention_weight_max": 104.07501220703125,
      "activations/layer4_attention_weight_min": -96.0852279663086,
      "activations/layer5_attention_weight_max": 75.39939880371094,
      "activations/layer5_attention_weight_min": -82.47811889648438,
      "activations/layer6_attention_weight_max": 53.79695510864258,
      "activations/layer6_attention_weight_min": -52.913639068603516,
      "activations/layer7_attention_weight_max": 78.7621841430664,
      "activations/layer7_attention_weight_min": -76.1136474609375,
      "activations/layer8_attention_weight_max": 52.404048919677734,
      "activations/layer8_attention_weight_min": -51.616493225097656,
      "activations/layer9_attention_weight_max": 60.147945404052734,
      "activations/layer9_attention_weight_min": -54.4012451171875,
      "epoch": 11.72,
      "learning_rate": 7.516060606060606e-05,
      "loss": 2.7866,
      "step": 201750
    },
    {
      "activations/layer0_attention_weight_max": 16.122900009155273,
      "activations/layer0_attention_weight_min": -14.867937088012695,
      "activations/layer10_attention_weight_max": 36.08942413330078,
      "activations/layer10_attention_weight_min": -36.03190994262695,
      "activations/layer11_attention_weight_max": 31.05170440673828,
      "activations/layer11_attention_weight_min": -33.12342071533203,
      "activations/layer12_attention_weight_max": 19.280447006225586,
      "activations/layer12_attention_weight_min": -25.476314544677734,
      "activations/layer13_attention_weight_max": 35.90812301635742,
      "activations/layer13_attention_weight_min": -29.12278175354004,
      "activations/layer14_attention_weight_max": 37.04932403564453,
      "activations/layer14_attention_weight_min": -29.401290893554688,
      "activations/layer15_attention_weight_max": 33.319252014160156,
      "activations/layer15_attention_weight_min": -30.903045654296875,
      "activations/layer16_attention_weight_max": 29.754627227783203,
      "activations/layer16_attention_weight_min": -28.90616226196289,
      "activations/layer17_attention_weight_max": 51.5395393371582,
      "activations/layer17_attention_weight_min": -42.544151306152344,
      "activations/layer18_attention_weight_max": 44.88079071044922,
      "activations/layer18_attention_weight_min": -36.557884216308594,
      "activations/layer19_attention_weight_max": 22.566144943237305,
      "activations/layer19_attention_weight_min": -21.62737464904785,
      "activations/layer1_attention_weight_max": 17.042753219604492,
      "activations/layer1_attention_weight_min": -14.125743865966797,
      "activations/layer20_attention_weight_max": 23.089120864868164,
      "activations/layer20_attention_weight_min": -20.8598690032959,
      "activations/layer21_attention_weight_max": 35.049346923828125,
      "activations/layer21_attention_weight_min": -21.38287353515625,
      "activations/layer22_attention_weight_max": 30.629413604736328,
      "activations/layer22_attention_weight_min": -26.154348373413086,
      "activations/layer23_attention_weight_max": 39.4251594543457,
      "activations/layer23_attention_weight_min": -23.833847045898438,
      "activations/layer2_attention_weight_max": 31.689254760742188,
      "activations/layer2_attention_weight_min": -30.37787437438965,
      "activations/layer3_attention_weight_max": 93.13097381591797,
      "activations/layer3_attention_weight_min": -91.33293914794922,
      "activations/layer4_attention_weight_max": 99.2818832397461,
      "activations/layer4_attention_weight_min": -96.3751449584961,
      "activations/layer5_attention_weight_max": 76.52268981933594,
      "activations/layer5_attention_weight_min": -79.10594177246094,
      "activations/layer6_attention_weight_max": 53.38185119628906,
      "activations/layer6_attention_weight_min": -53.678993225097656,
      "activations/layer7_attention_weight_max": 80.32576751708984,
      "activations/layer7_attention_weight_min": -71.30839538574219,
      "activations/layer8_attention_weight_max": 46.261314392089844,
      "activations/layer8_attention_weight_min": -47.8437385559082,
      "activations/layer9_attention_weight_max": 46.34741973876953,
      "activations/layer9_attention_weight_min": -48.12104415893555,
      "epoch": 11.73,
      "learning_rate": 7.514166666666666e-05,
      "loss": 2.7858,
      "step": 201800
    },
    {
      "activations/layer0_attention_weight_max": 16.893714904785156,
      "activations/layer0_attention_weight_min": -14.57137680053711,
      "activations/layer10_attention_weight_max": 55.15214538574219,
      "activations/layer10_attention_weight_min": -49.49654006958008,
      "activations/layer11_attention_weight_max": 47.61231231689453,
      "activations/layer11_attention_weight_min": -44.04655075073242,
      "activations/layer12_attention_weight_max": 24.084190368652344,
      "activations/layer12_attention_weight_min": -24.404891967773438,
      "activations/layer13_attention_weight_max": 60.73528289794922,
      "activations/layer13_attention_weight_min": -36.8992919921875,
      "activations/layer14_attention_weight_max": 61.67803192138672,
      "activations/layer14_attention_weight_min": -37.399906158447266,
      "activations/layer15_attention_weight_max": 47.01175308227539,
      "activations/layer15_attention_weight_min": -35.32133865356445,
      "activations/layer16_attention_weight_max": 41.36765670776367,
      "activations/layer16_attention_weight_min": -37.16123962402344,
      "activations/layer17_attention_weight_max": 72.86705017089844,
      "activations/layer17_attention_weight_min": -59.5628547668457,
      "activations/layer18_attention_weight_max": 60.872886657714844,
      "activations/layer18_attention_weight_min": -46.80553436279297,
      "activations/layer19_attention_weight_max": 32.98418045043945,
      "activations/layer19_attention_weight_min": -25.687671661376953,
      "activations/layer1_attention_weight_max": 16.694168090820312,
      "activations/layer1_attention_weight_min": -14.96948528289795,
      "activations/layer20_attention_weight_max": 30.844341278076172,
      "activations/layer20_attention_weight_min": -26.8723201751709,
      "activations/layer21_attention_weight_max": 51.25391387939453,
      "activations/layer21_attention_weight_min": -24.76837730407715,
      "activations/layer22_attention_weight_max": 35.83739471435547,
      "activations/layer22_attention_weight_min": -26.569272994995117,
      "activations/layer23_attention_weight_max": 45.069149017333984,
      "activations/layer23_attention_weight_min": -22.096363067626953,
      "activations/layer2_attention_weight_max": 35.53347396850586,
      "activations/layer2_attention_weight_min": -33.212284088134766,
      "activations/layer3_attention_weight_max": 110.65912628173828,
      "activations/layer3_attention_weight_min": -108.3899154663086,
      "activations/layer4_attention_weight_max": 117.70831298828125,
      "activations/layer4_attention_weight_min": -113.8161849975586,
      "activations/layer5_attention_weight_max": 85.74282836914062,
      "activations/layer5_attention_weight_min": -82.33645629882812,
      "activations/layer6_attention_weight_max": 64.48316955566406,
      "activations/layer6_attention_weight_min": -61.97100067138672,
      "activations/layer7_attention_weight_max": 98.38380432128906,
      "activations/layer7_attention_weight_min": -82.9149169921875,
      "activations/layer8_attention_weight_max": 59.75252151489258,
      "activations/layer8_attention_weight_min": -56.60956573486328,
      "activations/layer9_attention_weight_max": 68.24286651611328,
      "activations/layer9_attention_weight_min": -57.98535919189453,
      "epoch": 11.73,
      "learning_rate": 7.512272727272727e-05,
      "loss": 2.7996,
      "step": 201850
    },
    {
      "activations/layer0_attention_weight_max": 17.460166931152344,
      "activations/layer0_attention_weight_min": -14.442695617675781,
      "activations/layer10_attention_weight_max": 39.89802932739258,
      "activations/layer10_attention_weight_min": -37.485023498535156,
      "activations/layer11_attention_weight_max": 33.493003845214844,
      "activations/layer11_attention_weight_min": -33.91072082519531,
      "activations/layer12_attention_weight_max": 18.732681274414062,
      "activations/layer12_attention_weight_min": -24.261259078979492,
      "activations/layer13_attention_weight_max": 38.3777961730957,
      "activations/layer13_attention_weight_min": -32.07862091064453,
      "activations/layer14_attention_weight_max": 36.3171272277832,
      "activations/layer14_attention_weight_min": -31.734935760498047,
      "activations/layer15_attention_weight_max": 33.01223373413086,
      "activations/layer15_attention_weight_min": -30.014963150024414,
      "activations/layer16_attention_weight_max": 31.52596092224121,
      "activations/layer16_attention_weight_min": -29.514753341674805,
      "activations/layer17_attention_weight_max": 53.523712158203125,
      "activations/layer17_attention_weight_min": -42.91939926147461,
      "activations/layer18_attention_weight_max": 50.54280471801758,
      "activations/layer18_attention_weight_min": -37.02892303466797,
      "activations/layer19_attention_weight_max": 20.696247100830078,
      "activations/layer19_attention_weight_min": -21.716346740722656,
      "activations/layer1_attention_weight_max": 16.10773277282715,
      "activations/layer1_attention_weight_min": -16.393720626831055,
      "activations/layer20_attention_weight_max": 22.869247436523438,
      "activations/layer20_attention_weight_min": -21.218772888183594,
      "activations/layer21_attention_weight_max": 39.305850982666016,
      "activations/layer21_attention_weight_min": -25.432388305664062,
      "activations/layer22_attention_weight_max": 31.32052993774414,
      "activations/layer22_attention_weight_min": -29.05841636657715,
      "activations/layer23_attention_weight_max": 40.58819580078125,
      "activations/layer23_attention_weight_min": -24.628826141357422,
      "activations/layer2_attention_weight_max": 34.29259490966797,
      "activations/layer2_attention_weight_min": -31.664281845092773,
      "activations/layer3_attention_weight_max": 101.26683044433594,
      "activations/layer3_attention_weight_min": -97.81147766113281,
      "activations/layer4_attention_weight_max": 105.90447998046875,
      "activations/layer4_attention_weight_min": -100.09699249267578,
      "activations/layer5_attention_weight_max": 76.66744995117188,
      "activations/layer5_attention_weight_min": -86.6627197265625,
      "activations/layer6_attention_weight_max": 58.803955078125,
      "activations/layer6_attention_weight_min": -54.25830841064453,
      "activations/layer7_attention_weight_max": 81.18403625488281,
      "activations/layer7_attention_weight_min": -71.80958557128906,
      "activations/layer8_attention_weight_max": 48.17255401611328,
      "activations/layer8_attention_weight_min": -48.837791442871094,
      "activations/layer9_attention_weight_max": 49.42330551147461,
      "activations/layer9_attention_weight_min": -48.45323181152344,
      "epoch": 11.73,
      "learning_rate": 7.510378787878788e-05,
      "loss": 2.7656,
      "step": 201900
    },
    {
      "activations/layer0_attention_weight_max": 16.950782775878906,
      "activations/layer0_attention_weight_min": -14.46013355255127,
      "activations/layer10_attention_weight_max": 34.42806625366211,
      "activations/layer10_attention_weight_min": -35.86589050292969,
      "activations/layer11_attention_weight_max": 29.743078231811523,
      "activations/layer11_attention_weight_min": -30.39593505859375,
      "activations/layer12_attention_weight_max": 18.661258697509766,
      "activations/layer12_attention_weight_min": -23.380008697509766,
      "activations/layer13_attention_weight_max": 38.800716400146484,
      "activations/layer13_attention_weight_min": -28.223268508911133,
      "activations/layer14_attention_weight_max": 36.39937210083008,
      "activations/layer14_attention_weight_min": -31.36277198791504,
      "activations/layer15_attention_weight_max": 31.950571060180664,
      "activations/layer15_attention_weight_min": -30.46242904663086,
      "activations/layer16_attention_weight_max": 33.00180435180664,
      "activations/layer16_attention_weight_min": -29.82952308654785,
      "activations/layer17_attention_weight_max": 53.760623931884766,
      "activations/layer17_attention_weight_min": -46.156005859375,
      "activations/layer18_attention_weight_max": 44.7454948425293,
      "activations/layer18_attention_weight_min": -40.01411056518555,
      "activations/layer19_attention_weight_max": 23.650827407836914,
      "activations/layer19_attention_weight_min": -25.55459976196289,
      "activations/layer1_attention_weight_max": 16.9766788482666,
      "activations/layer1_attention_weight_min": -15.587043762207031,
      "activations/layer20_attention_weight_max": 25.92027473449707,
      "activations/layer20_attention_weight_min": -24.733476638793945,
      "activations/layer21_attention_weight_max": 39.21672439575195,
      "activations/layer21_attention_weight_min": -24.767587661743164,
      "activations/layer22_attention_weight_max": 32.468746185302734,
      "activations/layer22_attention_weight_min": -26.342533111572266,
      "activations/layer23_attention_weight_max": 41.17086410522461,
      "activations/layer23_attention_weight_min": -24.748374938964844,
      "activations/layer2_attention_weight_max": 32.596893310546875,
      "activations/layer2_attention_weight_min": -31.476804733276367,
      "activations/layer3_attention_weight_max": 93.26138305664062,
      "activations/layer3_attention_weight_min": -88.79177856445312,
      "activations/layer4_attention_weight_max": 96.22513580322266,
      "activations/layer4_attention_weight_min": -92.72280883789062,
      "activations/layer5_attention_weight_max": 79.33003997802734,
      "activations/layer5_attention_weight_min": -81.37956237792969,
      "activations/layer6_attention_weight_max": 52.448936462402344,
      "activations/layer6_attention_weight_min": -53.276283264160156,
      "activations/layer7_attention_weight_max": 75.90259552001953,
      "activations/layer7_attention_weight_min": -73.94697570800781,
      "activations/layer8_attention_weight_max": 42.31793975830078,
      "activations/layer8_attention_weight_min": -47.20802688598633,
      "activations/layer9_attention_weight_max": 47.54911804199219,
      "activations/layer9_attention_weight_min": -50.71969985961914,
      "epoch": 11.73,
      "learning_rate": 7.508484848484848e-05,
      "loss": 2.7749,
      "step": 201950
    },
    {
      "activations/layer0_attention_weight_max": 16.808364868164062,
      "activations/layer0_attention_weight_min": -14.551451683044434,
      "activations/layer10_attention_weight_max": 35.33808517456055,
      "activations/layer10_attention_weight_min": -37.98781204223633,
      "activations/layer11_attention_weight_max": 30.58623504638672,
      "activations/layer11_attention_weight_min": -35.72007751464844,
      "activations/layer12_attention_weight_max": 17.933725357055664,
      "activations/layer12_attention_weight_min": -24.542465209960938,
      "activations/layer13_attention_weight_max": 37.500205993652344,
      "activations/layer13_attention_weight_min": -30.32246971130371,
      "activations/layer14_attention_weight_max": 35.34291458129883,
      "activations/layer14_attention_weight_min": -30.817218780517578,
      "activations/layer15_attention_weight_max": 34.358245849609375,
      "activations/layer15_attention_weight_min": -32.102516174316406,
      "activations/layer16_attention_weight_max": 31.344791412353516,
      "activations/layer16_attention_weight_min": -30.70128059387207,
      "activations/layer17_attention_weight_max": 50.44308853149414,
      "activations/layer17_attention_weight_min": -44.3251838684082,
      "activations/layer18_attention_weight_max": 46.02382278442383,
      "activations/layer18_attention_weight_min": -38.29636764526367,
      "activations/layer19_attention_weight_max": 22.145957946777344,
      "activations/layer19_attention_weight_min": -20.946813583374023,
      "activations/layer1_attention_weight_max": 16.59869956970215,
      "activations/layer1_attention_weight_min": -13.84603500366211,
      "activations/layer20_attention_weight_max": 23.085527420043945,
      "activations/layer20_attention_weight_min": -22.752727508544922,
      "activations/layer21_attention_weight_max": 34.52064895629883,
      "activations/layer21_attention_weight_min": -22.71314239501953,
      "activations/layer22_attention_weight_max": 30.961971282958984,
      "activations/layer22_attention_weight_min": -27.305770874023438,
      "activations/layer23_attention_weight_max": 34.696800231933594,
      "activations/layer23_attention_weight_min": -24.596237182617188,
      "activations/layer2_attention_weight_max": 30.523740768432617,
      "activations/layer2_attention_weight_min": -28.84174919128418,
      "activations/layer3_attention_weight_max": 89.76592254638672,
      "activations/layer3_attention_weight_min": -93.72148132324219,
      "activations/layer4_attention_weight_max": 93.06238555908203,
      "activations/layer4_attention_weight_min": -94.49523162841797,
      "activations/layer5_attention_weight_max": 73.34687042236328,
      "activations/layer5_attention_weight_min": -78.29209899902344,
      "activations/layer6_attention_weight_max": 53.14145278930664,
      "activations/layer6_attention_weight_min": -49.88065719604492,
      "activations/layer7_attention_weight_max": 71.10310363769531,
      "activations/layer7_attention_weight_min": -69.14598083496094,
      "activations/layer8_attention_weight_max": 44.349853515625,
      "activations/layer8_attention_weight_min": -45.85597610473633,
      "activations/layer9_attention_weight_max": 44.02694320678711,
      "activations/layer9_attention_weight_min": -49.150428771972656,
      "epoch": 11.74,
      "learning_rate": 7.506590909090909e-05,
      "loss": 2.7691,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_loss": 2.732421875,
      "eval_runtime": 8.5052,
      "eval_samples_per_second": 504.87,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_openwebtext_loss": 2.732421875,
      "eval_openwebtext_ppl": 15.370066359341997,
      "eval_openwebtext_runtime": 8.5052,
      "eval_openwebtext_samples_per_second": 504.87,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 2.0067,
      "eval_wikitext_samples_per_second": 227.236,
      "step": 202000
    },
    {
      "epoch": 11.74,
      "eval_lambada_loss": 2.61328125,
      "eval_lambada_ppl": 13.643746025356307,
      "eval_lambada_runtime": 9.6135,
      "eval_lambada_samples_per_second": 506.474,
      "step": 202000
    },
    {
      "activations/layer0_attention_weight_max": 16.25836181640625,
      "activations/layer0_attention_weight_min": -14.627402305603027,
      "activations/layer10_attention_weight_max": 37.43526840209961,
      "activations/layer10_attention_weight_min": -37.483985900878906,
      "activations/layer11_attention_weight_max": 32.643157958984375,
      "activations/layer11_attention_weight_min": -36.675601959228516,
      "activations/layer12_attention_weight_max": 20.78488540649414,
      "activations/layer12_attention_weight_min": -24.006946563720703,
      "activations/layer13_attention_weight_max": 36.124202728271484,
      "activations/layer13_attention_weight_min": -29.66522216796875,
      "activations/layer14_attention_weight_max": 33.429649353027344,
      "activations/layer14_attention_weight_min": -29.81278419494629,
      "activations/layer15_attention_weight_max": 30.53892707824707,
      "activations/layer15_attention_weight_min": -31.310121536254883,
      "activations/layer16_attention_weight_max": 29.319244384765625,
      "activations/layer16_attention_weight_min": -30.392650604248047,
      "activations/layer17_attention_weight_max": 48.546791076660156,
      "activations/layer17_attention_weight_min": -44.422969818115234,
      "activations/layer18_attention_weight_max": 43.91099548339844,
      "activations/layer18_attention_weight_min": -37.253902435302734,
      "activations/layer19_attention_weight_max": 21.025131225585938,
      "activations/layer19_attention_weight_min": -22.587488174438477,
      "activations/layer1_attention_weight_max": 16.64929962158203,
      "activations/layer1_attention_weight_min": -15.301689147949219,
      "activations/layer20_attention_weight_max": 22.59052848815918,
      "activations/layer20_attention_weight_min": -22.422109603881836,
      "activations/layer21_attention_weight_max": 37.013648986816406,
      "activations/layer21_attention_weight_min": -22.781352996826172,
      "activations/layer22_attention_weight_max": 30.24823570251465,
      "activations/layer22_attention_weight_min": -25.967506408691406,
      "activations/layer23_attention_weight_max": 40.120147705078125,
      "activations/layer23_attention_weight_min": -23.85106658935547,
      "activations/layer2_attention_weight_max": 32.031951904296875,
      "activations/layer2_attention_weight_min": -31.52020263671875,
      "activations/layer3_attention_weight_max": 99.81398010253906,
      "activations/layer3_attention_weight_min": -103.80856323242188,
      "activations/layer4_attention_weight_max": 113.9183578491211,
      "activations/layer4_attention_weight_min": -106.06999969482422,
      "activations/layer5_attention_weight_max": 79.96478271484375,
      "activations/layer5_attention_weight_min": -86.3868408203125,
      "activations/layer6_attention_weight_max": 61.95964050292969,
      "activations/layer6_attention_weight_min": -60.31666564941406,
      "activations/layer7_attention_weight_max": 84.14566040039062,
      "activations/layer7_attention_weight_min": -83.44051361083984,
      "activations/layer8_attention_weight_max": 50.99613571166992,
      "activations/layer8_attention_weight_min": -52.579898834228516,
      "activations/layer9_attention_weight_max": 53.691619873046875,
      "activations/layer9_attention_weight_min": -50.81379318237305,
      "epoch": 11.74,
      "learning_rate": 7.504696969696968e-05,
      "loss": 2.7828,
      "step": 202050
    },
    {
      "activations/layer0_attention_weight_max": 15.799700736999512,
      "activations/layer0_attention_weight_min": -13.58661937713623,
      "activations/layer10_attention_weight_max": 32.94698715209961,
      "activations/layer10_attention_weight_min": -34.55507278442383,
      "activations/layer11_attention_weight_max": 29.693073272705078,
      "activations/layer11_attention_weight_min": -31.27260398864746,
      "activations/layer12_attention_weight_max": 19.00563621520996,
      "activations/layer12_attention_weight_min": -27.17662811279297,
      "activations/layer13_attention_weight_max": 49.34481430053711,
      "activations/layer13_attention_weight_min": -32.168739318847656,
      "activations/layer14_attention_weight_max": 33.653953552246094,
      "activations/layer14_attention_weight_min": -29.30910301208496,
      "activations/layer15_attention_weight_max": 31.22217559814453,
      "activations/layer15_attention_weight_min": -29.064556121826172,
      "activations/layer16_attention_weight_max": 30.581193923950195,
      "activations/layer16_attention_weight_min": -26.907642364501953,
      "activations/layer17_attention_weight_max": 48.26675033569336,
      "activations/layer17_attention_weight_min": -42.0250129699707,
      "activations/layer18_attention_weight_max": 46.71561813354492,
      "activations/layer18_attention_weight_min": -37.41676330566406,
      "activations/layer19_attention_weight_max": 22.167587280273438,
      "activations/layer19_attention_weight_min": -22.768924713134766,
      "activations/layer1_attention_weight_max": 17.301620483398438,
      "activations/layer1_attention_weight_min": -13.8206787109375,
      "activations/layer20_attention_weight_max": 22.83319091796875,
      "activations/layer20_attention_weight_min": -22.017148971557617,
      "activations/layer21_attention_weight_max": 35.890201568603516,
      "activations/layer21_attention_weight_min": -24.909069061279297,
      "activations/layer22_attention_weight_max": 31.962190628051758,
      "activations/layer22_attention_weight_min": -26.956527709960938,
      "activations/layer23_attention_weight_max": 40.606529235839844,
      "activations/layer23_attention_weight_min": -24.128154754638672,
      "activations/layer2_attention_weight_max": 29.66230583190918,
      "activations/layer2_attention_weight_min": -30.33740997314453,
      "activations/layer3_attention_weight_max": 93.50774383544922,
      "activations/layer3_attention_weight_min": -95.7645034790039,
      "activations/layer4_attention_weight_max": 92.36418914794922,
      "activations/layer4_attention_weight_min": -96.97765350341797,
      "activations/layer5_attention_weight_max": 69.6102523803711,
      "activations/layer5_attention_weight_min": -80.53630065917969,
      "activations/layer6_attention_weight_max": 50.26000213623047,
      "activations/layer6_attention_weight_min": -51.60751724243164,
      "activations/layer7_attention_weight_max": 71.37504577636719,
      "activations/layer7_attention_weight_min": -69.9172592163086,
      "activations/layer8_attention_weight_max": 45.683815002441406,
      "activations/layer8_attention_weight_min": -44.954811096191406,
      "activations/layer9_attention_weight_max": 43.78402328491211,
      "activations/layer9_attention_weight_min": -45.32542037963867,
      "epoch": 11.74,
      "learning_rate": 7.50280303030303e-05,
      "loss": 2.794,
      "step": 202100
    },
    {
      "activations/layer0_attention_weight_max": 15.562978744506836,
      "activations/layer0_attention_weight_min": -13.453258514404297,
      "activations/layer10_attention_weight_max": 38.2181396484375,
      "activations/layer10_attention_weight_min": -38.476314544677734,
      "activations/layer11_attention_weight_max": 31.878568649291992,
      "activations/layer11_attention_weight_min": -34.704864501953125,
      "activations/layer12_attention_weight_max": 18.262964248657227,
      "activations/layer12_attention_weight_min": -26.584537506103516,
      "activations/layer13_attention_weight_max": 35.391632080078125,
      "activations/layer13_attention_weight_min": -27.327232360839844,
      "activations/layer14_attention_weight_max": 34.06040954589844,
      "activations/layer14_attention_weight_min": -31.23516845703125,
      "activations/layer15_attention_weight_max": 30.710391998291016,
      "activations/layer15_attention_weight_min": -29.704626083374023,
      "activations/layer16_attention_weight_max": 29.909807205200195,
      "activations/layer16_attention_weight_min": -28.787111282348633,
      "activations/layer17_attention_weight_max": 47.523216247558594,
      "activations/layer17_attention_weight_min": -43.7905158996582,
      "activations/layer18_attention_weight_max": 44.0417366027832,
      "activations/layer18_attention_weight_min": -38.180023193359375,
      "activations/layer19_attention_weight_max": 21.147905349731445,
      "activations/layer19_attention_weight_min": -20.7662296295166,
      "activations/layer1_attention_weight_max": 17.010740280151367,
      "activations/layer1_attention_weight_min": -14.979961395263672,
      "activations/layer20_attention_weight_max": 21.637374877929688,
      "activations/layer20_attention_weight_min": -21.304868698120117,
      "activations/layer21_attention_weight_max": 37.65967559814453,
      "activations/layer21_attention_weight_min": -25.707592010498047,
      "activations/layer22_attention_weight_max": 27.33489227294922,
      "activations/layer22_attention_weight_min": -25.4978084564209,
      "activations/layer23_attention_weight_max": 32.470802307128906,
      "activations/layer23_attention_weight_min": -22.898826599121094,
      "activations/layer2_attention_weight_max": 31.901525497436523,
      "activations/layer2_attention_weight_min": -32.633853912353516,
      "activations/layer3_attention_weight_max": 92.01616668701172,
      "activations/layer3_attention_weight_min": -94.05760192871094,
      "activations/layer4_attention_weight_max": 101.43708038330078,
      "activations/layer4_attention_weight_min": -95.77875518798828,
      "activations/layer5_attention_weight_max": 73.94432067871094,
      "activations/layer5_attention_weight_min": -80.13077545166016,
      "activations/layer6_attention_weight_max": 54.822227478027344,
      "activations/layer6_attention_weight_min": -54.91548156738281,
      "activations/layer7_attention_weight_max": 74.30950927734375,
      "activations/layer7_attention_weight_min": -76.21109008789062,
      "activations/layer8_attention_weight_max": 45.13561248779297,
      "activations/layer8_attention_weight_min": -48.00935745239258,
      "activations/layer9_attention_weight_max": 50.24195861816406,
      "activations/layer9_attention_weight_min": -49.346160888671875,
      "epoch": 11.75,
      "learning_rate": 7.500909090909091e-05,
      "loss": 2.7962,
      "step": 202150
    },
    {
      "activations/layer0_attention_weight_max": 15.377486228942871,
      "activations/layer0_attention_weight_min": -13.333772659301758,
      "activations/layer10_attention_weight_max": 36.91218948364258,
      "activations/layer10_attention_weight_min": -36.438236236572266,
      "activations/layer11_attention_weight_max": 33.28050231933594,
      "activations/layer11_attention_weight_min": -33.04861831665039,
      "activations/layer12_attention_weight_max": 19.35272216796875,
      "activations/layer12_attention_weight_min": -24.256301879882812,
      "activations/layer13_attention_weight_max": 41.6729736328125,
      "activations/layer13_attention_weight_min": -30.948471069335938,
      "activations/layer14_attention_weight_max": 34.62522506713867,
      "activations/layer14_attention_weight_min": -30.551149368286133,
      "activations/layer15_attention_weight_max": 31.497827529907227,
      "activations/layer15_attention_weight_min": -30.770313262939453,
      "activations/layer16_attention_weight_max": 31.730749130249023,
      "activations/layer16_attention_weight_min": -29.39506721496582,
      "activations/layer17_attention_weight_max": 53.52467727661133,
      "activations/layer17_attention_weight_min": -46.69585418701172,
      "activations/layer18_attention_weight_max": 47.07186508178711,
      "activations/layer18_attention_weight_min": -38.46861267089844,
      "activations/layer19_attention_weight_max": 24.377727508544922,
      "activations/layer19_attention_weight_min": -21.858354568481445,
      "activations/layer1_attention_weight_max": 17.400623321533203,
      "activations/layer1_attention_weight_min": -14.687664031982422,
      "activations/layer20_attention_weight_max": 24.172452926635742,
      "activations/layer20_attention_weight_min": -20.66435432434082,
      "activations/layer21_attention_weight_max": 44.83871078491211,
      "activations/layer21_attention_weight_min": -24.682035446166992,
      "activations/layer22_attention_weight_max": 31.48261260986328,
      "activations/layer22_attention_weight_min": -28.24335289001465,
      "activations/layer23_attention_weight_max": 36.140892028808594,
      "activations/layer23_attention_weight_min": -21.348880767822266,
      "activations/layer2_attention_weight_max": 30.838308334350586,
      "activations/layer2_attention_weight_min": -29.015792846679688,
      "activations/layer3_attention_weight_max": 92.80361938476562,
      "activations/layer3_attention_weight_min": -93.63822937011719,
      "activations/layer4_attention_weight_max": 98.33143615722656,
      "activations/layer4_attention_weight_min": -90.58110809326172,
      "activations/layer5_attention_weight_max": 70.1396484375,
      "activations/layer5_attention_weight_min": -78.33377075195312,
      "activations/layer6_attention_weight_max": 52.902809143066406,
      "activations/layer6_attention_weight_min": -55.48939895629883,
      "activations/layer7_attention_weight_max": 76.35807800292969,
      "activations/layer7_attention_weight_min": -75.1877670288086,
      "activations/layer8_attention_weight_max": 44.24193572998047,
      "activations/layer8_attention_weight_min": -47.22758483886719,
      "activations/layer9_attention_weight_max": 46.22594451904297,
      "activations/layer9_attention_weight_min": -48.58870315551758,
      "epoch": 11.75,
      "learning_rate": 7.499015151515151e-05,
      "loss": 2.7921,
      "step": 202200
    },
    {
      "activations/layer0_attention_weight_max": 15.549704551696777,
      "activations/layer0_attention_weight_min": -13.793402671813965,
      "activations/layer10_attention_weight_max": 39.11859130859375,
      "activations/layer10_attention_weight_min": -38.1979866027832,
      "activations/layer11_attention_weight_max": 35.234230041503906,
      "activations/layer11_attention_weight_min": -32.93397903442383,
      "activations/layer12_attention_weight_max": 19.95686149597168,
      "activations/layer12_attention_weight_min": -28.34918975830078,
      "activations/layer13_attention_weight_max": 37.6524543762207,
      "activations/layer13_attention_weight_min": -29.088520050048828,
      "activations/layer14_attention_weight_max": 35.82390594482422,
      "activations/layer14_attention_weight_min": -31.366016387939453,
      "activations/layer15_attention_weight_max": 34.14523696899414,
      "activations/layer15_attention_weight_min": -29.80885124206543,
      "activations/layer16_attention_weight_max": 31.829769134521484,
      "activations/layer16_attention_weight_min": -29.17422866821289,
      "activations/layer17_attention_weight_max": 57.85945129394531,
      "activations/layer17_attention_weight_min": -48.714088439941406,
      "activations/layer18_attention_weight_max": 47.80645751953125,
      "activations/layer18_attention_weight_min": -40.749881744384766,
      "activations/layer19_attention_weight_max": 20.63776969909668,
      "activations/layer19_attention_weight_min": -20.69441032409668,
      "activations/layer1_attention_weight_max": 18.7443790435791,
      "activations/layer1_attention_weight_min": -15.046296119689941,
      "activations/layer20_attention_weight_max": 20.623016357421875,
      "activations/layer20_attention_weight_min": -22.697988510131836,
      "activations/layer21_attention_weight_max": 38.28582763671875,
      "activations/layer21_attention_weight_min": -24.4343318939209,
      "activations/layer22_attention_weight_max": 29.754793167114258,
      "activations/layer22_attention_weight_min": -26.02416229248047,
      "activations/layer23_attention_weight_max": 34.46356964111328,
      "activations/layer23_attention_weight_min": -23.69614028930664,
      "activations/layer2_attention_weight_max": 31.749311447143555,
      "activations/layer2_attention_weight_min": -31.163183212280273,
      "activations/layer3_attention_weight_max": 89.95445251464844,
      "activations/layer3_attention_weight_min": -96.78217315673828,
      "activations/layer4_attention_weight_max": 99.49871063232422,
      "activations/layer4_attention_weight_min": -97.87911224365234,
      "activations/layer5_attention_weight_max": 73.871826171875,
      "activations/layer5_attention_weight_min": -80.85801696777344,
      "activations/layer6_attention_weight_max": 54.4404411315918,
      "activations/layer6_attention_weight_min": -53.72441864013672,
      "activations/layer7_attention_weight_max": 72.87617492675781,
      "activations/layer7_attention_weight_min": -73.39356231689453,
      "activations/layer8_attention_weight_max": 48.158973693847656,
      "activations/layer8_attention_weight_min": -51.348751068115234,
      "activations/layer9_attention_weight_max": 44.1815185546875,
      "activations/layer9_attention_weight_min": -47.20572280883789,
      "epoch": 11.75,
      "learning_rate": 7.497121212121211e-05,
      "loss": 2.7637,
      "step": 202250
    },
    {
      "activations/layer0_attention_weight_max": 16.209224700927734,
      "activations/layer0_attention_weight_min": -13.651391983032227,
      "activations/layer10_attention_weight_max": 39.91521072387695,
      "activations/layer10_attention_weight_min": -36.817569732666016,
      "activations/layer11_attention_weight_max": 32.05573654174805,
      "activations/layer11_attention_weight_min": -31.588062286376953,
      "activations/layer12_attention_weight_max": 18.121536254882812,
      "activations/layer12_attention_weight_min": -24.849576950073242,
      "activations/layer13_attention_weight_max": 40.084999084472656,
      "activations/layer13_attention_weight_min": -32.861392974853516,
      "activations/layer14_attention_weight_max": 50.215904235839844,
      "activations/layer14_attention_weight_min": -35.35207748413086,
      "activations/layer15_attention_weight_max": 40.957157135009766,
      "activations/layer15_attention_weight_min": -29.373987197875977,
      "activations/layer16_attention_weight_max": 30.333463668823242,
      "activations/layer16_attention_weight_min": -29.05666732788086,
      "activations/layer17_attention_weight_max": 50.77363586425781,
      "activations/layer17_attention_weight_min": -45.423465728759766,
      "activations/layer18_attention_weight_max": 49.27004623413086,
      "activations/layer18_attention_weight_min": -40.898773193359375,
      "activations/layer19_attention_weight_max": 21.26987075805664,
      "activations/layer19_attention_weight_min": -22.900272369384766,
      "activations/layer1_attention_weight_max": 17.812885284423828,
      "activations/layer1_attention_weight_min": -16.259244918823242,
      "activations/layer20_attention_weight_max": 21.14167594909668,
      "activations/layer20_attention_weight_min": -21.35894012451172,
      "activations/layer21_attention_weight_max": 37.620399475097656,
      "activations/layer21_attention_weight_min": -23.700119018554688,
      "activations/layer22_attention_weight_max": 30.67736053466797,
      "activations/layer22_attention_weight_min": -27.146421432495117,
      "activations/layer23_attention_weight_max": 40.93992614746094,
      "activations/layer23_attention_weight_min": -24.94083023071289,
      "activations/layer2_attention_weight_max": 31.65890121459961,
      "activations/layer2_attention_weight_min": -30.497379302978516,
      "activations/layer3_attention_weight_max": 88.6093978881836,
      "activations/layer3_attention_weight_min": -92.3315200805664,
      "activations/layer4_attention_weight_max": 97.7265853881836,
      "activations/layer4_attention_weight_min": -94.9171142578125,
      "activations/layer5_attention_weight_max": 74.1738510131836,
      "activations/layer5_attention_weight_min": -81.28076934814453,
      "activations/layer6_attention_weight_max": 52.806358337402344,
      "activations/layer6_attention_weight_min": -51.32324981689453,
      "activations/layer7_attention_weight_max": 77.20771789550781,
      "activations/layer7_attention_weight_min": -70.39620208740234,
      "activations/layer8_attention_weight_max": 49.51334762573242,
      "activations/layer8_attention_weight_min": -46.079078674316406,
      "activations/layer9_attention_weight_max": 47.40883255004883,
      "activations/layer9_attention_weight_min": -47.04203796386719,
      "epoch": 11.75,
      "learning_rate": 7.495227272727271e-05,
      "loss": 2.7764,
      "step": 202300
    },
    {
      "activations/layer0_attention_weight_max": 16.326541900634766,
      "activations/layer0_attention_weight_min": -13.607809066772461,
      "activations/layer10_attention_weight_max": 37.73972702026367,
      "activations/layer10_attention_weight_min": -36.39949417114258,
      "activations/layer11_attention_weight_max": 34.22943115234375,
      "activations/layer11_attention_weight_min": -33.3929328918457,
      "activations/layer12_attention_weight_max": 18.598228454589844,
      "activations/layer12_attention_weight_min": -25.373977661132812,
      "activations/layer13_attention_weight_max": 36.87624740600586,
      "activations/layer13_attention_weight_min": -29.164052963256836,
      "activations/layer14_attention_weight_max": 43.9379997253418,
      "activations/layer14_attention_weight_min": -30.6064395904541,
      "activations/layer15_attention_weight_max": 40.587337493896484,
      "activations/layer15_attention_weight_min": -29.567604064941406,
      "activations/layer16_attention_weight_max": 32.57648849487305,
      "activations/layer16_attention_weight_min": -28.865243911743164,
      "activations/layer17_attention_weight_max": 54.07126998901367,
      "activations/layer17_attention_weight_min": -45.695899963378906,
      "activations/layer18_attention_weight_max": 52.15663146972656,
      "activations/layer18_attention_weight_min": -38.92714309692383,
      "activations/layer19_attention_weight_max": 26.0568904876709,
      "activations/layer19_attention_weight_min": -22.203357696533203,
      "activations/layer1_attention_weight_max": 17.86469268798828,
      "activations/layer1_attention_weight_min": -13.407942771911621,
      "activations/layer20_attention_weight_max": 27.25044822692871,
      "activations/layer20_attention_weight_min": -21.93118667602539,
      "activations/layer21_attention_weight_max": 51.197593688964844,
      "activations/layer21_attention_weight_min": -26.27364158630371,
      "activations/layer22_attention_weight_max": 28.88770294189453,
      "activations/layer22_attention_weight_min": -26.437211990356445,
      "activations/layer23_attention_weight_max": 38.151123046875,
      "activations/layer23_attention_weight_min": -24.646682739257812,
      "activations/layer2_attention_weight_max": 27.761932373046875,
      "activations/layer2_attention_weight_min": -28.9670467376709,
      "activations/layer3_attention_weight_max": 85.77775573730469,
      "activations/layer3_attention_weight_min": -93.89065551757812,
      "activations/layer4_attention_weight_max": 91.83584594726562,
      "activations/layer4_attention_weight_min": -92.2697525024414,
      "activations/layer5_attention_weight_max": 69.26594543457031,
      "activations/layer5_attention_weight_min": -79.26821899414062,
      "activations/layer6_attention_weight_max": 50.4415397644043,
      "activations/layer6_attention_weight_min": -52.02846145629883,
      "activations/layer7_attention_weight_max": 69.4933090209961,
      "activations/layer7_attention_weight_min": -69.4420394897461,
      "activations/layer8_attention_weight_max": 46.224647521972656,
      "activations/layer8_attention_weight_min": -49.86580276489258,
      "activations/layer9_attention_weight_max": 50.28957748413086,
      "activations/layer9_attention_weight_min": -51.204132080078125,
      "epoch": 11.76,
      "learning_rate": 7.493333333333333e-05,
      "loss": 2.7675,
      "step": 202350
    },
    {
      "activations/layer0_attention_weight_max": 16.597747802734375,
      "activations/layer0_attention_weight_min": -14.24661922454834,
      "activations/layer10_attention_weight_max": 37.555389404296875,
      "activations/layer10_attention_weight_min": -36.34364700317383,
      "activations/layer11_attention_weight_max": 33.14449691772461,
      "activations/layer11_attention_weight_min": -34.515899658203125,
      "activations/layer12_attention_weight_max": 17.658342361450195,
      "activations/layer12_attention_weight_min": -26.668115615844727,
      "activations/layer13_attention_weight_max": 44.75214385986328,
      "activations/layer13_attention_weight_min": -35.81415557861328,
      "activations/layer14_attention_weight_max": 36.62502670288086,
      "activations/layer14_attention_weight_min": -29.705093383789062,
      "activations/layer15_attention_weight_max": 33.498069763183594,
      "activations/layer15_attention_weight_min": -28.229135513305664,
      "activations/layer16_attention_weight_max": 29.629032135009766,
      "activations/layer16_attention_weight_min": -28.590673446655273,
      "activations/layer17_attention_weight_max": 51.71474838256836,
      "activations/layer17_attention_weight_min": -43.25449752807617,
      "activations/layer18_attention_weight_max": 46.35569763183594,
      "activations/layer18_attention_weight_min": -37.31749725341797,
      "activations/layer19_attention_weight_max": 22.414356231689453,
      "activations/layer19_attention_weight_min": -21.6055965423584,
      "activations/layer1_attention_weight_max": 16.210752487182617,
      "activations/layer1_attention_weight_min": -14.279561042785645,
      "activations/layer20_attention_weight_max": 23.17011260986328,
      "activations/layer20_attention_weight_min": -23.317766189575195,
      "activations/layer21_attention_weight_max": 39.894493103027344,
      "activations/layer21_attention_weight_min": -25.113975524902344,
      "activations/layer22_attention_weight_max": 34.920291900634766,
      "activations/layer22_attention_weight_min": -25.275651931762695,
      "activations/layer23_attention_weight_max": 37.74410629272461,
      "activations/layer23_attention_weight_min": -25.580549240112305,
      "activations/layer2_attention_weight_max": 30.381771087646484,
      "activations/layer2_attention_weight_min": -30.598861694335938,
      "activations/layer3_attention_weight_max": 87.84090423583984,
      "activations/layer3_attention_weight_min": -90.51683807373047,
      "activations/layer4_attention_weight_max": 97.129638671875,
      "activations/layer4_attention_weight_min": -89.21651458740234,
      "activations/layer5_attention_weight_max": 76.16084289550781,
      "activations/layer5_attention_weight_min": -82.38709259033203,
      "activations/layer6_attention_weight_max": 56.5433235168457,
      "activations/layer6_attention_weight_min": -52.63632583618164,
      "activations/layer7_attention_weight_max": 76.27771759033203,
      "activations/layer7_attention_weight_min": -73.56803131103516,
      "activations/layer8_attention_weight_max": 48.024532318115234,
      "activations/layer8_attention_weight_min": -49.28145217895508,
      "activations/layer9_attention_weight_max": 48.7457389831543,
      "activations/layer9_attention_weight_min": -48.64083480834961,
      "epoch": 11.76,
      "learning_rate": 7.491439393939393e-05,
      "loss": 2.776,
      "step": 202400
    },
    {
      "activations/layer0_attention_weight_max": 15.825492858886719,
      "activations/layer0_attention_weight_min": -13.884233474731445,
      "activations/layer10_attention_weight_max": 38.0931510925293,
      "activations/layer10_attention_weight_min": -36.76765823364258,
      "activations/layer11_attention_weight_max": 36.645015716552734,
      "activations/layer11_attention_weight_min": -33.58935546875,
      "activations/layer12_attention_weight_max": 16.830337524414062,
      "activations/layer12_attention_weight_min": -31.946979522705078,
      "activations/layer13_attention_weight_max": 45.04631042480469,
      "activations/layer13_attention_weight_min": -30.30437469482422,
      "activations/layer14_attention_weight_max": 42.82085418701172,
      "activations/layer14_attention_weight_min": -28.6318359375,
      "activations/layer15_attention_weight_max": 41.1751594543457,
      "activations/layer15_attention_weight_min": -27.088817596435547,
      "activations/layer16_attention_weight_max": 29.921127319335938,
      "activations/layer16_attention_weight_min": -27.87350845336914,
      "activations/layer17_attention_weight_max": 51.36460494995117,
      "activations/layer17_attention_weight_min": -43.513580322265625,
      "activations/layer18_attention_weight_max": 44.43684005737305,
      "activations/layer18_attention_weight_min": -37.46921157836914,
      "activations/layer19_attention_weight_max": 22.635339736938477,
      "activations/layer19_attention_weight_min": -20.610231399536133,
      "activations/layer1_attention_weight_max": 17.632787704467773,
      "activations/layer1_attention_weight_min": -15.168724060058594,
      "activations/layer20_attention_weight_max": 23.635263442993164,
      "activations/layer20_attention_weight_min": -20.686077117919922,
      "activations/layer21_attention_weight_max": 40.02106857299805,
      "activations/layer21_attention_weight_min": -22.264507293701172,
      "activations/layer22_attention_weight_max": 25.734996795654297,
      "activations/layer22_attention_weight_min": -24.200973510742188,
      "activations/layer23_attention_weight_max": 33.43263244628906,
      "activations/layer23_attention_weight_min": -21.576946258544922,
      "activations/layer2_attention_weight_max": 33.9660758972168,
      "activations/layer2_attention_weight_min": -30.713346481323242,
      "activations/layer3_attention_weight_max": 96.6497573852539,
      "activations/layer3_attention_weight_min": -93.45963287353516,
      "activations/layer4_attention_weight_max": 100.60672760009766,
      "activations/layer4_attention_weight_min": -90.82843780517578,
      "activations/layer5_attention_weight_max": 78.0656509399414,
      "activations/layer5_attention_weight_min": -71.33251953125,
      "activations/layer6_attention_weight_max": 50.26593780517578,
      "activations/layer6_attention_weight_min": -48.828285217285156,
      "activations/layer7_attention_weight_max": 78.530517578125,
      "activations/layer7_attention_weight_min": -69.73731994628906,
      "activations/layer8_attention_weight_max": 50.58495330810547,
      "activations/layer8_attention_weight_min": -46.7379035949707,
      "activations/layer9_attention_weight_max": 59.52695846557617,
      "activations/layer9_attention_weight_min": -50.77349090576172,
      "epoch": 11.76,
      "learning_rate": 7.489545454545453e-05,
      "loss": 2.7948,
      "step": 202450
    },
    {
      "activations/layer0_attention_weight_max": 15.968282699584961,
      "activations/layer0_attention_weight_min": -13.675989151000977,
      "activations/layer10_attention_weight_max": 34.5651741027832,
      "activations/layer10_attention_weight_min": -34.120872497558594,
      "activations/layer11_attention_weight_max": 30.467100143432617,
      "activations/layer11_attention_weight_min": -31.19921875,
      "activations/layer12_attention_weight_max": 17.530832290649414,
      "activations/layer12_attention_weight_min": -26.76837730407715,
      "activations/layer13_attention_weight_max": 50.78496170043945,
      "activations/layer13_attention_weight_min": -36.264461517333984,
      "activations/layer14_attention_weight_max": 47.943702697753906,
      "activations/layer14_attention_weight_min": -29.831214904785156,
      "activations/layer15_attention_weight_max": 37.601837158203125,
      "activations/layer15_attention_weight_min": -32.38737869262695,
      "activations/layer16_attention_weight_max": 34.833702087402344,
      "activations/layer16_attention_weight_min": -32.20669937133789,
      "activations/layer17_attention_weight_max": 59.74123001098633,
      "activations/layer17_attention_weight_min": -49.520999908447266,
      "activations/layer18_attention_weight_max": 50.02540588378906,
      "activations/layer18_attention_weight_min": -41.95460510253906,
      "activations/layer19_attention_weight_max": 23.87580680847168,
      "activations/layer19_attention_weight_min": -22.2752685546875,
      "activations/layer1_attention_weight_max": 17.196481704711914,
      "activations/layer1_attention_weight_min": -14.388556480407715,
      "activations/layer20_attention_weight_max": 23.305551528930664,
      "activations/layer20_attention_weight_min": -21.938095092773438,
      "activations/layer21_attention_weight_max": 47.09442138671875,
      "activations/layer21_attention_weight_min": -26.85752296447754,
      "activations/layer22_attention_weight_max": 31.177448272705078,
      "activations/layer22_attention_weight_min": -26.147756576538086,
      "activations/layer23_attention_weight_max": 42.17463684082031,
      "activations/layer23_attention_weight_min": -23.96479034423828,
      "activations/layer2_attention_weight_max": 31.289154052734375,
      "activations/layer2_attention_weight_min": -30.248611450195312,
      "activations/layer3_attention_weight_max": 91.60912322998047,
      "activations/layer3_attention_weight_min": -94.18262481689453,
      "activations/layer4_attention_weight_max": 94.90288543701172,
      "activations/layer4_attention_weight_min": -91.68622589111328,
      "activations/layer5_attention_weight_max": 70.22699737548828,
      "activations/layer5_attention_weight_min": -76.00988006591797,
      "activations/layer6_attention_weight_max": 52.49840545654297,
      "activations/layer6_attention_weight_min": -50.68124008178711,
      "activations/layer7_attention_weight_max": 80.04798126220703,
      "activations/layer7_attention_weight_min": -75.10047149658203,
      "activations/layer8_attention_weight_max": 44.680694580078125,
      "activations/layer8_attention_weight_min": -47.96782302856445,
      "activations/layer9_attention_weight_max": 47.171199798583984,
      "activations/layer9_attention_weight_min": -45.79861068725586,
      "epoch": 11.77,
      "learning_rate": 7.487651515151515e-05,
      "loss": 2.7857,
      "step": 202500
    },
    {
      "activations/layer0_attention_weight_max": 15.630010604858398,
      "activations/layer0_attention_weight_min": -14.12157917022705,
      "activations/layer10_attention_weight_max": 38.7083740234375,
      "activations/layer10_attention_weight_min": -36.0482292175293,
      "activations/layer11_attention_weight_max": 34.57764434814453,
      "activations/layer11_attention_weight_min": -33.10885238647461,
      "activations/layer12_attention_weight_max": 18.708057403564453,
      "activations/layer12_attention_weight_min": -25.9477596282959,
      "activations/layer13_attention_weight_max": 33.93516159057617,
      "activations/layer13_attention_weight_min": -26.248502731323242,
      "activations/layer14_attention_weight_max": 35.78437423706055,
      "activations/layer14_attention_weight_min": -29.521150588989258,
      "activations/layer15_attention_weight_max": 29.740781784057617,
      "activations/layer15_attention_weight_min": -28.027734756469727,
      "activations/layer16_attention_weight_max": 29.960535049438477,
      "activations/layer16_attention_weight_min": -28.556577682495117,
      "activations/layer17_attention_weight_max": 48.255558013916016,
      "activations/layer17_attention_weight_min": -40.77886962890625,
      "activations/layer18_attention_weight_max": 41.4502067565918,
      "activations/layer18_attention_weight_min": -36.77882385253906,
      "activations/layer19_attention_weight_max": 22.57090187072754,
      "activations/layer19_attention_weight_min": -21.26746368408203,
      "activations/layer1_attention_weight_max": 16.818134307861328,
      "activations/layer1_attention_weight_min": -14.60787296295166,
      "activations/layer20_attention_weight_max": 20.087913513183594,
      "activations/layer20_attention_weight_min": -21.115835189819336,
      "activations/layer21_attention_weight_max": 30.796875,
      "activations/layer21_attention_weight_min": -20.70231056213379,
      "activations/layer22_attention_weight_max": 28.94123077392578,
      "activations/layer22_attention_weight_min": -26.37356185913086,
      "activations/layer23_attention_weight_max": 32.1758918762207,
      "activations/layer23_attention_weight_min": -23.653152465820312,
      "activations/layer2_attention_weight_max": 30.420799255371094,
      "activations/layer2_attention_weight_min": -29.647390365600586,
      "activations/layer3_attention_weight_max": 92.36861419677734,
      "activations/layer3_attention_weight_min": -93.76634979248047,
      "activations/layer4_attention_weight_max": 96.12325286865234,
      "activations/layer4_attention_weight_min": -92.49144744873047,
      "activations/layer5_attention_weight_max": 67.43612670898438,
      "activations/layer5_attention_weight_min": -78.49136352539062,
      "activations/layer6_attention_weight_max": 53.291996002197266,
      "activations/layer6_attention_weight_min": -51.5939826965332,
      "activations/layer7_attention_weight_max": 73.56303405761719,
      "activations/layer7_attention_weight_min": -73.18778991699219,
      "activations/layer8_attention_weight_max": 44.32794952392578,
      "activations/layer8_attention_weight_min": -49.15068054199219,
      "activations/layer9_attention_weight_max": 47.03708267211914,
      "activations/layer9_attention_weight_min": -48.51216125488281,
      "epoch": 11.77,
      "learning_rate": 7.485757575757575e-05,
      "loss": 2.7824,
      "step": 202550
    },
    {
      "activations/layer0_attention_weight_max": 15.348995208740234,
      "activations/layer0_attention_weight_min": -14.078780174255371,
      "activations/layer10_attention_weight_max": 37.640106201171875,
      "activations/layer10_attention_weight_min": -37.76761245727539,
      "activations/layer11_attention_weight_max": 34.2210693359375,
      "activations/layer11_attention_weight_min": -34.67698669433594,
      "activations/layer12_attention_weight_max": 18.823715209960938,
      "activations/layer12_attention_weight_min": -27.286170959472656,
      "activations/layer13_attention_weight_max": 50.17705535888672,
      "activations/layer13_attention_weight_min": -28.75433349609375,
      "activations/layer14_attention_weight_max": 38.826332092285156,
      "activations/layer14_attention_weight_min": -32.602134704589844,
      "activations/layer15_attention_weight_max": 36.553016662597656,
      "activations/layer15_attention_weight_min": -31.03459930419922,
      "activations/layer16_attention_weight_max": 32.99032974243164,
      "activations/layer16_attention_weight_min": -30.414363861083984,
      "activations/layer17_attention_weight_max": 56.70416259765625,
      "activations/layer17_attention_weight_min": -46.99262237548828,
      "activations/layer18_attention_weight_max": 51.08493423461914,
      "activations/layer18_attention_weight_min": -38.81232833862305,
      "activations/layer19_attention_weight_max": 25.145248413085938,
      "activations/layer19_attention_weight_min": -23.22474479675293,
      "activations/layer1_attention_weight_max": 16.130577087402344,
      "activations/layer1_attention_weight_min": -14.74132251739502,
      "activations/layer20_attention_weight_max": 28.577110290527344,
      "activations/layer20_attention_weight_min": -23.554710388183594,
      "activations/layer21_attention_weight_max": 43.474491119384766,
      "activations/layer21_attention_weight_min": -23.151351928710938,
      "activations/layer22_attention_weight_max": 31.471181869506836,
      "activations/layer22_attention_weight_min": -25.487459182739258,
      "activations/layer23_attention_weight_max": 40.33085632324219,
      "activations/layer23_attention_weight_min": -23.501514434814453,
      "activations/layer2_attention_weight_max": 32.87078857421875,
      "activations/layer2_attention_weight_min": -31.85287094116211,
      "activations/layer3_attention_weight_max": 100.1019515991211,
      "activations/layer3_attention_weight_min": -96.51158142089844,
      "activations/layer4_attention_weight_max": 100.2112045288086,
      "activations/layer4_attention_weight_min": -98.8115005493164,
      "activations/layer5_attention_weight_max": 70.1800537109375,
      "activations/layer5_attention_weight_min": -80.63029479980469,
      "activations/layer6_attention_weight_max": 53.407161712646484,
      "activations/layer6_attention_weight_min": -53.245609283447266,
      "activations/layer7_attention_weight_max": 72.2691421508789,
      "activations/layer7_attention_weight_min": -69.65983581542969,
      "activations/layer8_attention_weight_max": 45.384979248046875,
      "activations/layer8_attention_weight_min": -46.05756378173828,
      "activations/layer9_attention_weight_max": 45.25742721557617,
      "activations/layer9_attention_weight_min": -48.826133728027344,
      "epoch": 11.77,
      "learning_rate": 7.483863636363636e-05,
      "loss": 2.769,
      "step": 202600
    },
    {
      "activations/layer0_attention_weight_max": 16.496244430541992,
      "activations/layer0_attention_weight_min": -14.241573333740234,
      "activations/layer10_attention_weight_max": 39.88422393798828,
      "activations/layer10_attention_weight_min": -36.35177230834961,
      "activations/layer11_attention_weight_max": 33.34561538696289,
      "activations/layer11_attention_weight_min": -32.34600830078125,
      "activations/layer12_attention_weight_max": 20.127750396728516,
      "activations/layer12_attention_weight_min": -25.482786178588867,
      "activations/layer13_attention_weight_max": 38.80208969116211,
      "activations/layer13_attention_weight_min": -31.39198875427246,
      "activations/layer14_attention_weight_max": 35.474342346191406,
      "activations/layer14_attention_weight_min": -31.23462677001953,
      "activations/layer15_attention_weight_max": 33.09041213989258,
      "activations/layer15_attention_weight_min": -28.98811912536621,
      "activations/layer16_attention_weight_max": 30.217554092407227,
      "activations/layer16_attention_weight_min": -28.751968383789062,
      "activations/layer17_attention_weight_max": 51.466922760009766,
      "activations/layer17_attention_weight_min": -43.83654022216797,
      "activations/layer18_attention_weight_max": 46.713863372802734,
      "activations/layer18_attention_weight_min": -39.63195037841797,
      "activations/layer19_attention_weight_max": 21.255064010620117,
      "activations/layer19_attention_weight_min": -23.38424301147461,
      "activations/layer1_attention_weight_max": 17.426916122436523,
      "activations/layer1_attention_weight_min": -14.403450012207031,
      "activations/layer20_attention_weight_max": 22.196561813354492,
      "activations/layer20_attention_weight_min": -23.955984115600586,
      "activations/layer21_attention_weight_max": 31.61476707458496,
      "activations/layer21_attention_weight_min": -22.57691764831543,
      "activations/layer22_attention_weight_max": 28.62792205810547,
      "activations/layer22_attention_weight_min": -25.22224998474121,
      "activations/layer23_attention_weight_max": 34.274200439453125,
      "activations/layer23_attention_weight_min": -24.460857391357422,
      "activations/layer2_attention_weight_max": 30.653194427490234,
      "activations/layer2_attention_weight_min": -28.763641357421875,
      "activations/layer3_attention_weight_max": 89.67657470703125,
      "activations/layer3_attention_weight_min": -94.62069702148438,
      "activations/layer4_attention_weight_max": 100.40253448486328,
      "activations/layer4_attention_weight_min": -95.71147918701172,
      "activations/layer5_attention_weight_max": 71.448486328125,
      "activations/layer5_attention_weight_min": -78.02342224121094,
      "activations/layer6_attention_weight_max": 53.59678268432617,
      "activations/layer6_attention_weight_min": -51.92252731323242,
      "activations/layer7_attention_weight_max": 72.6395492553711,
      "activations/layer7_attention_weight_min": -71.24188232421875,
      "activations/layer8_attention_weight_max": 47.58339309692383,
      "activations/layer8_attention_weight_min": -47.75031280517578,
      "activations/layer9_attention_weight_max": 47.39896774291992,
      "activations/layer9_attention_weight_min": -48.72050476074219,
      "epoch": 11.78,
      "learning_rate": 7.481969696969697e-05,
      "loss": 2.7829,
      "step": 202650
    },
    {
      "activations/layer0_attention_weight_max": 15.238173484802246,
      "activations/layer0_attention_weight_min": -14.289019584655762,
      "activations/layer10_attention_weight_max": 42.77083206176758,
      "activations/layer10_attention_weight_min": -38.067955017089844,
      "activations/layer11_attention_weight_max": 40.0779914855957,
      "activations/layer11_attention_weight_min": -35.514549255371094,
      "activations/layer12_attention_weight_max": 20.508432388305664,
      "activations/layer12_attention_weight_min": -27.648107528686523,
      "activations/layer13_attention_weight_max": 50.58540725708008,
      "activations/layer13_attention_weight_min": -29.58624267578125,
      "activations/layer14_attention_weight_max": 49.280311584472656,
      "activations/layer14_attention_weight_min": -30.157257080078125,
      "activations/layer15_attention_weight_max": 39.248756408691406,
      "activations/layer15_attention_weight_min": -30.501585006713867,
      "activations/layer16_attention_weight_max": 30.99139404296875,
      "activations/layer16_attention_weight_min": -29.137907028198242,
      "activations/layer17_attention_weight_max": 53.26823425292969,
      "activations/layer17_attention_weight_min": -46.51258850097656,
      "activations/layer18_attention_weight_max": 47.947784423828125,
      "activations/layer18_attention_weight_min": -38.658103942871094,
      "activations/layer19_attention_weight_max": 25.438222885131836,
      "activations/layer19_attention_weight_min": -22.768728256225586,
      "activations/layer1_attention_weight_max": 16.612850189208984,
      "activations/layer1_attention_weight_min": -15.441445350646973,
      "activations/layer20_attention_weight_max": 24.7369327545166,
      "activations/layer20_attention_weight_min": -22.148544311523438,
      "activations/layer21_attention_weight_max": 46.610374450683594,
      "activations/layer21_attention_weight_min": -24.49907875061035,
      "activations/layer22_attention_weight_max": 30.623485565185547,
      "activations/layer22_attention_weight_min": -24.73589324951172,
      "activations/layer23_attention_weight_max": 36.55623245239258,
      "activations/layer23_attention_weight_min": -25.15998077392578,
      "activations/layer2_attention_weight_max": 34.00458526611328,
      "activations/layer2_attention_weight_min": -32.271034240722656,
      "activations/layer3_attention_weight_max": 99.5505142211914,
      "activations/layer3_attention_weight_min": -100.42495727539062,
      "activations/layer4_attention_weight_max": 103.21612548828125,
      "activations/layer4_attention_weight_min": -98.1949234008789,
      "activations/layer5_attention_weight_max": 74.76567077636719,
      "activations/layer5_attention_weight_min": -78.2756118774414,
      "activations/layer6_attention_weight_max": 51.65587615966797,
      "activations/layer6_attention_weight_min": -53.86829376220703,
      "activations/layer7_attention_weight_max": 81.01627349853516,
      "activations/layer7_attention_weight_min": -73.18502044677734,
      "activations/layer8_attention_weight_max": 50.834957122802734,
      "activations/layer8_attention_weight_min": -51.181846618652344,
      "activations/layer9_attention_weight_max": 58.94961166381836,
      "activations/layer9_attention_weight_min": -51.41194534301758,
      "epoch": 11.78,
      "learning_rate": 7.480075757575757e-05,
      "loss": 2.77,
      "step": 202700
    },
    {
      "activations/layer0_attention_weight_max": 15.402814865112305,
      "activations/layer0_attention_weight_min": -13.688870429992676,
      "activations/layer10_attention_weight_max": 37.807159423828125,
      "activations/layer10_attention_weight_min": -36.719566345214844,
      "activations/layer11_attention_weight_max": 34.55636978149414,
      "activations/layer11_attention_weight_min": -34.469520568847656,
      "activations/layer12_attention_weight_max": 19.592025756835938,
      "activations/layer12_attention_weight_min": -26.442691802978516,
      "activations/layer13_attention_weight_max": 38.76671600341797,
      "activations/layer13_attention_weight_min": -30.30293846130371,
      "activations/layer14_attention_weight_max": 38.529319763183594,
      "activations/layer14_attention_weight_min": -31.929325103759766,
      "activations/layer15_attention_weight_max": 34.56505584716797,
      "activations/layer15_attention_weight_min": -29.542072296142578,
      "activations/layer16_attention_weight_max": 34.38011932373047,
      "activations/layer16_attention_weight_min": -31.163923263549805,
      "activations/layer17_attention_weight_max": 53.88392639160156,
      "activations/layer17_attention_weight_min": -46.390289306640625,
      "activations/layer18_attention_weight_max": 48.25124740600586,
      "activations/layer18_attention_weight_min": -37.533416748046875,
      "activations/layer19_attention_weight_max": 26.161670684814453,
      "activations/layer19_attention_weight_min": -23.047143936157227,
      "activations/layer1_attention_weight_max": 16.396512985229492,
      "activations/layer1_attention_weight_min": -14.791500091552734,
      "activations/layer20_attention_weight_max": 25.083860397338867,
      "activations/layer20_attention_weight_min": -20.222410202026367,
      "activations/layer21_attention_weight_max": 38.37290573120117,
      "activations/layer21_attention_weight_min": -22.037673950195312,
      "activations/layer22_attention_weight_max": 35.99317169189453,
      "activations/layer22_attention_weight_min": -26.75331687927246,
      "activations/layer23_attention_weight_max": 43.302330017089844,
      "activations/layer23_attention_weight_min": -22.241947174072266,
      "activations/layer2_attention_weight_max": 31.599889755249023,
      "activations/layer2_attention_weight_min": -31.68872833251953,
      "activations/layer3_attention_weight_max": 92.02422332763672,
      "activations/layer3_attention_weight_min": -93.41734313964844,
      "activations/layer4_attention_weight_max": 100.58334350585938,
      "activations/layer4_attention_weight_min": -91.78514099121094,
      "activations/layer5_attention_weight_max": 72.62916564941406,
      "activations/layer5_attention_weight_min": -76.67474365234375,
      "activations/layer6_attention_weight_max": 55.90513610839844,
      "activations/layer6_attention_weight_min": -52.077877044677734,
      "activations/layer7_attention_weight_max": 74.001220703125,
      "activations/layer7_attention_weight_min": -70.02490234375,
      "activations/layer8_attention_weight_max": 48.779178619384766,
      "activations/layer8_attention_weight_min": -47.30385971069336,
      "activations/layer9_attention_weight_max": 49.22884750366211,
      "activations/layer9_attention_weight_min": -46.74216842651367,
      "epoch": 11.78,
      "learning_rate": 7.478181818181818e-05,
      "loss": 2.7677,
      "step": 202750
    },
    {
      "activations/layer0_attention_weight_max": 15.949874877929688,
      "activations/layer0_attention_weight_min": -13.875483512878418,
      "activations/layer10_attention_weight_max": 37.66719436645508,
      "activations/layer10_attention_weight_min": -36.944061279296875,
      "activations/layer11_attention_weight_max": 34.831886291503906,
      "activations/layer11_attention_weight_min": -33.409515380859375,
      "activations/layer12_attention_weight_max": 18.272809982299805,
      "activations/layer12_attention_weight_min": -24.54447364807129,
      "activations/layer13_attention_weight_max": 34.89686965942383,
      "activations/layer13_attention_weight_min": -28.112178802490234,
      "activations/layer14_attention_weight_max": 34.96687698364258,
      "activations/layer14_attention_weight_min": -31.91883659362793,
      "activations/layer15_attention_weight_max": 33.617034912109375,
      "activations/layer15_attention_weight_min": -32.27864074707031,
      "activations/layer16_attention_weight_max": 30.812698364257812,
      "activations/layer16_attention_weight_min": -30.584239959716797,
      "activations/layer17_attention_weight_max": 52.05875778198242,
      "activations/layer17_attention_weight_min": -43.391578674316406,
      "activations/layer18_attention_weight_max": 43.958106994628906,
      "activations/layer18_attention_weight_min": -36.523197174072266,
      "activations/layer19_attention_weight_max": 24.674320220947266,
      "activations/layer19_attention_weight_min": -21.812318801879883,
      "activations/layer1_attention_weight_max": 16.29534339904785,
      "activations/layer1_attention_weight_min": -16.5572452545166,
      "activations/layer20_attention_weight_max": 21.968278884887695,
      "activations/layer20_attention_weight_min": -23.936622619628906,
      "activations/layer21_attention_weight_max": 35.47071075439453,
      "activations/layer21_attention_weight_min": -20.235595703125,
      "activations/layer22_attention_weight_max": 32.1915283203125,
      "activations/layer22_attention_weight_min": -24.586029052734375,
      "activations/layer23_attention_weight_max": 37.300559997558594,
      "activations/layer23_attention_weight_min": -22.816387176513672,
      "activations/layer2_attention_weight_max": 31.53839683532715,
      "activations/layer2_attention_weight_min": -30.12295913696289,
      "activations/layer3_attention_weight_max": 91.16761779785156,
      "activations/layer3_attention_weight_min": -94.24552917480469,
      "activations/layer4_attention_weight_max": 97.28666687011719,
      "activations/layer4_attention_weight_min": -92.98363494873047,
      "activations/layer5_attention_weight_max": 75.66529083251953,
      "activations/layer5_attention_weight_min": -81.10523986816406,
      "activations/layer6_attention_weight_max": 53.26980209350586,
      "activations/layer6_attention_weight_min": -52.348304748535156,
      "activations/layer7_attention_weight_max": 69.08042907714844,
      "activations/layer7_attention_weight_min": -69.42920684814453,
      "activations/layer8_attention_weight_max": 48.46365737915039,
      "activations/layer8_attention_weight_min": -48.66582107543945,
      "activations/layer9_attention_weight_max": 45.659420013427734,
      "activations/layer9_attention_weight_min": -46.66299819946289,
      "epoch": 11.78,
      "learning_rate": 7.476287878787878e-05,
      "loss": 2.7949,
      "step": 202800
    },
    {
      "activations/layer0_attention_weight_max": 16.443443298339844,
      "activations/layer0_attention_weight_min": -14.211478233337402,
      "activations/layer10_attention_weight_max": 36.99515151977539,
      "activations/layer10_attention_weight_min": -36.744197845458984,
      "activations/layer11_attention_weight_max": 33.658546447753906,
      "activations/layer11_attention_weight_min": -34.57460021972656,
      "activations/layer12_attention_weight_max": 17.941640853881836,
      "activations/layer12_attention_weight_min": -25.801542282104492,
      "activations/layer13_attention_weight_max": 36.078060150146484,
      "activations/layer13_attention_weight_min": -30.17363929748535,
      "activations/layer14_attention_weight_max": 34.298702239990234,
      "activations/layer14_attention_weight_min": -30.416419982910156,
      "activations/layer15_attention_weight_max": 32.21092987060547,
      "activations/layer15_attention_weight_min": -28.639310836791992,
      "activations/layer16_attention_weight_max": 33.93896484375,
      "activations/layer16_attention_weight_min": -31.560184478759766,
      "activations/layer17_attention_weight_max": 51.22056198120117,
      "activations/layer17_attention_weight_min": -46.10792922973633,
      "activations/layer18_attention_weight_max": 46.30381774902344,
      "activations/layer18_attention_weight_min": -37.88024139404297,
      "activations/layer19_attention_weight_max": 21.0252742767334,
      "activations/layer19_attention_weight_min": -20.061359405517578,
      "activations/layer1_attention_weight_max": 17.13435935974121,
      "activations/layer1_attention_weight_min": -14.894319534301758,
      "activations/layer20_attention_weight_max": 23.64482307434082,
      "activations/layer20_attention_weight_min": -22.31157112121582,
      "activations/layer21_attention_weight_max": 43.41435623168945,
      "activations/layer21_attention_weight_min": -23.36809539794922,
      "activations/layer22_attention_weight_max": 29.703508377075195,
      "activations/layer22_attention_weight_min": -25.92527961730957,
      "activations/layer23_attention_weight_max": 38.696380615234375,
      "activations/layer23_attention_weight_min": -28.52486801147461,
      "activations/layer2_attention_weight_max": 33.954917907714844,
      "activations/layer2_attention_weight_min": -32.01984405517578,
      "activations/layer3_attention_weight_max": 95.5306167602539,
      "activations/layer3_attention_weight_min": -99.50166320800781,
      "activations/layer4_attention_weight_max": 100.23076629638672,
      "activations/layer4_attention_weight_min": -97.85133361816406,
      "activations/layer5_attention_weight_max": 70.687744140625,
      "activations/layer5_attention_weight_min": -76.74256896972656,
      "activations/layer6_attention_weight_max": 54.41604995727539,
      "activations/layer6_attention_weight_min": -52.90305709838867,
      "activations/layer7_attention_weight_max": 73.57511901855469,
      "activations/layer7_attention_weight_min": -72.3023452758789,
      "activations/layer8_attention_weight_max": 48.094337463378906,
      "activations/layer8_attention_weight_min": -50.5234489440918,
      "activations/layer9_attention_weight_max": 46.6243782043457,
      "activations/layer9_attention_weight_min": -49.1528434753418,
      "epoch": 11.79,
      "learning_rate": 7.474393939393939e-05,
      "loss": 2.7816,
      "step": 202850
    },
    {
      "activations/layer0_attention_weight_max": 18.634727478027344,
      "activations/layer0_attention_weight_min": -14.15039348602295,
      "activations/layer10_attention_weight_max": 40.65576934814453,
      "activations/layer10_attention_weight_min": -39.812740325927734,
      "activations/layer11_attention_weight_max": 35.64234924316406,
      "activations/layer11_attention_weight_min": -35.64397048950195,
      "activations/layer12_attention_weight_max": 19.378204345703125,
      "activations/layer12_attention_weight_min": -25.870018005371094,
      "activations/layer13_attention_weight_max": 55.040130615234375,
      "activations/layer13_attention_weight_min": -32.10497283935547,
      "activations/layer14_attention_weight_max": 60.312076568603516,
      "activations/layer14_attention_weight_min": -30.452428817749023,
      "activations/layer15_attention_weight_max": 52.57305145263672,
      "activations/layer15_attention_weight_min": -31.48558807373047,
      "activations/layer16_attention_weight_max": 43.71282958984375,
      "activations/layer16_attention_weight_min": -29.399076461791992,
      "activations/layer17_attention_weight_max": 57.03190994262695,
      "activations/layer17_attention_weight_min": -46.729068756103516,
      "activations/layer18_attention_weight_max": 49.64902114868164,
      "activations/layer18_attention_weight_min": -41.53144454956055,
      "activations/layer19_attention_weight_max": 24.94301414489746,
      "activations/layer19_attention_weight_min": -24.461050033569336,
      "activations/layer1_attention_weight_max": 16.302066802978516,
      "activations/layer1_attention_weight_min": -16.608844757080078,
      "activations/layer20_attention_weight_max": 27.573495864868164,
      "activations/layer20_attention_weight_min": -24.841541290283203,
      "activations/layer21_attention_weight_max": 63.35075378417969,
      "activations/layer21_attention_weight_min": -27.977703094482422,
      "activations/layer22_attention_weight_max": 36.046104431152344,
      "activations/layer22_attention_weight_min": -27.727752685546875,
      "activations/layer23_attention_weight_max": 51.33665466308594,
      "activations/layer23_attention_weight_min": -24.890213012695312,
      "activations/layer2_attention_weight_max": 33.2877311706543,
      "activations/layer2_attention_weight_min": -31.730876922607422,
      "activations/layer3_attention_weight_max": 96.19450378417969,
      "activations/layer3_attention_weight_min": -96.38737487792969,
      "activations/layer4_attention_weight_max": 99.44188690185547,
      "activations/layer4_attention_weight_min": -99.55241394042969,
      "activations/layer5_attention_weight_max": 75.60256958007812,
      "activations/layer5_attention_weight_min": -76.09651947021484,
      "activations/layer6_attention_weight_max": 54.651187896728516,
      "activations/layer6_attention_weight_min": -55.249290466308594,
      "activations/layer7_attention_weight_max": 74.39907836914062,
      "activations/layer7_attention_weight_min": -75.05230712890625,
      "activations/layer8_attention_weight_max": 49.475624084472656,
      "activations/layer8_attention_weight_min": -49.072200775146484,
      "activations/layer9_attention_weight_max": 53.16128158569336,
      "activations/layer9_attention_weight_min": -52.58075714111328,
      "epoch": 11.79,
      "learning_rate": 7.472499999999999e-05,
      "loss": 2.7793,
      "step": 202900
    },
    {
      "activations/layer0_attention_weight_max": 15.73967170715332,
      "activations/layer0_attention_weight_min": -14.280630111694336,
      "activations/layer10_attention_weight_max": 36.821449279785156,
      "activations/layer10_attention_weight_min": -35.064266204833984,
      "activations/layer11_attention_weight_max": 31.201688766479492,
      "activations/layer11_attention_weight_min": -33.12901306152344,
      "activations/layer12_attention_weight_max": 22.647607803344727,
      "activations/layer12_attention_weight_min": -23.2806339263916,
      "activations/layer13_attention_weight_max": 39.36884307861328,
      "activations/layer13_attention_weight_min": -28.867115020751953,
      "activations/layer14_attention_weight_max": 39.87434387207031,
      "activations/layer14_attention_weight_min": -30.176889419555664,
      "activations/layer15_attention_weight_max": 33.810237884521484,
      "activations/layer15_attention_weight_min": -30.141157150268555,
      "activations/layer16_attention_weight_max": 32.68357849121094,
      "activations/layer16_attention_weight_min": -31.272937774658203,
      "activations/layer17_attention_weight_max": 56.92634582519531,
      "activations/layer17_attention_weight_min": -50.52946853637695,
      "activations/layer18_attention_weight_max": 51.49541473388672,
      "activations/layer18_attention_weight_min": -42.098697662353516,
      "activations/layer19_attention_weight_max": 23.73479461669922,
      "activations/layer19_attention_weight_min": -21.92647361755371,
      "activations/layer1_attention_weight_max": 16.180334091186523,
      "activations/layer1_attention_weight_min": -15.951555252075195,
      "activations/layer20_attention_weight_max": 24.24698829650879,
      "activations/layer20_attention_weight_min": -22.14551544189453,
      "activations/layer21_attention_weight_max": 38.68269729614258,
      "activations/layer21_attention_weight_min": -23.843191146850586,
      "activations/layer22_attention_weight_max": 35.4390869140625,
      "activations/layer22_attention_weight_min": -27.677257537841797,
      "activations/layer23_attention_weight_max": 42.675079345703125,
      "activations/layer23_attention_weight_min": -23.426246643066406,
      "activations/layer2_attention_weight_max": 30.476879119873047,
      "activations/layer2_attention_weight_min": -30.81395721435547,
      "activations/layer3_attention_weight_max": 90.97162628173828,
      "activations/layer3_attention_weight_min": -94.32711029052734,
      "activations/layer4_attention_weight_max": 99.50545501708984,
      "activations/layer4_attention_weight_min": -96.1959457397461,
      "activations/layer5_attention_weight_max": 72.4632568359375,
      "activations/layer5_attention_weight_min": -76.77719116210938,
      "activations/layer6_attention_weight_max": 52.677921295166016,
      "activations/layer6_attention_weight_min": -51.97702407836914,
      "activations/layer7_attention_weight_max": 73.68148040771484,
      "activations/layer7_attention_weight_min": -72.55427551269531,
      "activations/layer8_attention_weight_max": 45.447601318359375,
      "activations/layer8_attention_weight_min": -47.34347915649414,
      "activations/layer9_attention_weight_max": 49.0941276550293,
      "activations/layer9_attention_weight_min": -48.575687408447266,
      "epoch": 11.79,
      "learning_rate": 7.47060606060606e-05,
      "loss": 2.7727,
      "step": 202950
    },
    {
      "activations/layer0_attention_weight_max": 15.172115325927734,
      "activations/layer0_attention_weight_min": -14.479660987854004,
      "activations/layer10_attention_weight_max": 37.840309143066406,
      "activations/layer10_attention_weight_min": -35.88296127319336,
      "activations/layer11_attention_weight_max": 33.12108612060547,
      "activations/layer11_attention_weight_min": -32.65414810180664,
      "activations/layer12_attention_weight_max": 17.5096492767334,
      "activations/layer12_attention_weight_min": -24.160703659057617,
      "activations/layer13_attention_weight_max": 35.17104721069336,
      "activations/layer13_attention_weight_min": -30.807544708251953,
      "activations/layer14_attention_weight_max": 36.42155075073242,
      "activations/layer14_attention_weight_min": -31.081541061401367,
      "activations/layer15_attention_weight_max": 32.640708923339844,
      "activations/layer15_attention_weight_min": -30.32719612121582,
      "activations/layer16_attention_weight_max": 31.704227447509766,
      "activations/layer16_attention_weight_min": -29.899330139160156,
      "activations/layer17_attention_weight_max": 52.323490142822266,
      "activations/layer17_attention_weight_min": -44.404029846191406,
      "activations/layer18_attention_weight_max": 45.5780029296875,
      "activations/layer18_attention_weight_min": -36.44493103027344,
      "activations/layer19_attention_weight_max": 21.46388053894043,
      "activations/layer19_attention_weight_min": -20.682546615600586,
      "activations/layer1_attention_weight_max": 16.723583221435547,
      "activations/layer1_attention_weight_min": -14.923042297363281,
      "activations/layer20_attention_weight_max": 22.90606689453125,
      "activations/layer20_attention_weight_min": -22.551511764526367,
      "activations/layer21_attention_weight_max": 36.753082275390625,
      "activations/layer21_attention_weight_min": -21.74702262878418,
      "activations/layer22_attention_weight_max": 29.00519561767578,
      "activations/layer22_attention_weight_min": -24.48995590209961,
      "activations/layer23_attention_weight_max": 32.55534362792969,
      "activations/layer23_attention_weight_min": -21.284568786621094,
      "activations/layer2_attention_weight_max": 32.506874084472656,
      "activations/layer2_attention_weight_min": -30.67653465270996,
      "activations/layer3_attention_weight_max": 98.35038757324219,
      "activations/layer3_attention_weight_min": -103.74317932128906,
      "activations/layer4_attention_weight_max": 98.55598449707031,
      "activations/layer4_attention_weight_min": -101.65557861328125,
      "activations/layer5_attention_weight_max": 71.53717041015625,
      "activations/layer5_attention_weight_min": -78.3680419921875,
      "activations/layer6_attention_weight_max": 53.25868606567383,
      "activations/layer6_attention_weight_min": -52.36054611206055,
      "activations/layer7_attention_weight_max": 72.36221313476562,
      "activations/layer7_attention_weight_min": -74.23635864257812,
      "activations/layer8_attention_weight_max": 45.30891799926758,
      "activations/layer8_attention_weight_min": -46.77653503417969,
      "activations/layer9_attention_weight_max": 47.82129669189453,
      "activations/layer9_attention_weight_min": -48.37301254272461,
      "epoch": 11.8,
      "learning_rate": 7.46871212121212e-05,
      "loss": 2.7791,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5024,
      "eval_samples_per_second": 505.033,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5024,
      "eval_openwebtext_samples_per_second": 505.033,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 2.0033,
      "eval_wikitext_samples_per_second": 227.62,
      "step": 203000
    },
    {
      "epoch": 11.8,
      "eval_lambada_loss": 2.58203125,
      "eval_lambada_ppl": 13.223972090529857,
      "eval_lambada_runtime": 9.643,
      "eval_lambada_samples_per_second": 504.926,
      "step": 203000
    },
    {
      "activations/layer0_attention_weight_max": 15.931897163391113,
      "activations/layer0_attention_weight_min": -13.95981216430664,
      "activations/layer10_attention_weight_max": 40.4990119934082,
      "activations/layer10_attention_weight_min": -40.36318588256836,
      "activations/layer11_attention_weight_max": 37.64007568359375,
      "activations/layer11_attention_weight_min": -36.7571907043457,
      "activations/layer12_attention_weight_max": 18.490711212158203,
      "activations/layer12_attention_weight_min": -23.81100845336914,
      "activations/layer13_attention_weight_max": 38.55311584472656,
      "activations/layer13_attention_weight_min": -32.738494873046875,
      "activations/layer14_attention_weight_max": 48.918663024902344,
      "activations/layer14_attention_weight_min": -31.728116989135742,
      "activations/layer15_attention_weight_max": 49.56399154663086,
      "activations/layer15_attention_weight_min": -30.20278549194336,
      "activations/layer16_attention_weight_max": 31.070573806762695,
      "activations/layer16_attention_weight_min": -28.424610137939453,
      "activations/layer17_attention_weight_max": 52.60414505004883,
      "activations/layer17_attention_weight_min": -43.312618255615234,
      "activations/layer18_attention_weight_max": 44.50090789794922,
      "activations/layer18_attention_weight_min": -39.48027038574219,
      "activations/layer19_attention_weight_max": 22.09535789489746,
      "activations/layer19_attention_weight_min": -22.767194747924805,
      "activations/layer1_attention_weight_max": 16.920446395874023,
      "activations/layer1_attention_weight_min": -14.482216835021973,
      "activations/layer20_attention_weight_max": 23.262845993041992,
      "activations/layer20_attention_weight_min": -22.059776306152344,
      "activations/layer21_attention_weight_max": 37.03438186645508,
      "activations/layer21_attention_weight_min": -23.302770614624023,
      "activations/layer22_attention_weight_max": 30.010305404663086,
      "activations/layer22_attention_weight_min": -26.235795974731445,
      "activations/layer23_attention_weight_max": 40.691349029541016,
      "activations/layer23_attention_weight_min": -25.629623413085938,
      "activations/layer2_attention_weight_max": 34.402957916259766,
      "activations/layer2_attention_weight_min": -33.126068115234375,
      "activations/layer3_attention_weight_max": 97.46196746826172,
      "activations/layer3_attention_weight_min": -99.03947448730469,
      "activations/layer4_attention_weight_max": 97.81645965576172,
      "activations/layer4_attention_weight_min": -94.05189514160156,
      "activations/layer5_attention_weight_max": 76.16596984863281,
      "activations/layer5_attention_weight_min": -76.4621353149414,
      "activations/layer6_attention_weight_max": 51.7470588684082,
      "activations/layer6_attention_weight_min": -54.29547882080078,
      "activations/layer7_attention_weight_max": 81.33143615722656,
      "activations/layer7_attention_weight_min": -70.39759826660156,
      "activations/layer8_attention_weight_max": 52.36347961425781,
      "activations/layer8_attention_weight_min": -46.95018768310547,
      "activations/layer9_attention_weight_max": 63.82813262939453,
      "activations/layer9_attention_weight_min": -51.64844512939453,
      "epoch": 11.8,
      "learning_rate": 7.46681818181818e-05,
      "loss": 2.7761,
      "step": 203050
    },
    {
      "activations/layer0_attention_weight_max": 17.027999877929688,
      "activations/layer0_attention_weight_min": -14.044022560119629,
      "activations/layer10_attention_weight_max": 33.194862365722656,
      "activations/layer10_attention_weight_min": -34.666561126708984,
      "activations/layer11_attention_weight_max": 29.98468017578125,
      "activations/layer11_attention_weight_min": -31.974166870117188,
      "activations/layer12_attention_weight_max": 17.087646484375,
      "activations/layer12_attention_weight_min": -25.84296989440918,
      "activations/layer13_attention_weight_max": 41.1391487121582,
      "activations/layer13_attention_weight_min": -29.762042999267578,
      "activations/layer14_attention_weight_max": 32.86930847167969,
      "activations/layer14_attention_weight_min": -29.93870735168457,
      "activations/layer15_attention_weight_max": 32.26110076904297,
      "activations/layer15_attention_weight_min": -28.618974685668945,
      "activations/layer16_attention_weight_max": 29.93694496154785,
      "activations/layer16_attention_weight_min": -29.924081802368164,
      "activations/layer17_attention_weight_max": 48.49605941772461,
      "activations/layer17_attention_weight_min": -44.61960983276367,
      "activations/layer18_attention_weight_max": 44.10728073120117,
      "activations/layer18_attention_weight_min": -38.21196746826172,
      "activations/layer19_attention_weight_max": 21.0391902923584,
      "activations/layer19_attention_weight_min": -23.109771728515625,
      "activations/layer1_attention_weight_max": 16.695947647094727,
      "activations/layer1_attention_weight_min": -14.609861373901367,
      "activations/layer20_attention_weight_max": 20.340763092041016,
      "activations/layer20_attention_weight_min": -22.792020797729492,
      "activations/layer21_attention_weight_max": 35.949134826660156,
      "activations/layer21_attention_weight_min": -21.07993507385254,
      "activations/layer22_attention_weight_max": 30.99195671081543,
      "activations/layer22_attention_weight_min": -23.607263565063477,
      "activations/layer23_attention_weight_max": 38.989952087402344,
      "activations/layer23_attention_weight_min": -23.635143280029297,
      "activations/layer2_attention_weight_max": 32.71684265136719,
      "activations/layer2_attention_weight_min": -31.517860412597656,
      "activations/layer3_attention_weight_max": 93.68539428710938,
      "activations/layer3_attention_weight_min": -99.92218780517578,
      "activations/layer4_attention_weight_max": 94.15699005126953,
      "activations/layer4_attention_weight_min": -95.73882293701172,
      "activations/layer5_attention_weight_max": 71.7594985961914,
      "activations/layer5_attention_weight_min": -77.6921615600586,
      "activations/layer6_attention_weight_max": 51.49559783935547,
      "activations/layer6_attention_weight_min": -50.56892395019531,
      "activations/layer7_attention_weight_max": 74.42235565185547,
      "activations/layer7_attention_weight_min": -66.46371459960938,
      "activations/layer8_attention_weight_max": 42.525115966796875,
      "activations/layer8_attention_weight_min": -43.788455963134766,
      "activations/layer9_attention_weight_max": 48.86222457885742,
      "activations/layer9_attention_weight_min": -45.25179672241211,
      "epoch": 11.8,
      "learning_rate": 7.464924242424242e-05,
      "loss": 2.7819,
      "step": 203100
    },
    {
      "activations/layer0_attention_weight_max": 14.95090389251709,
      "activations/layer0_attention_weight_min": -14.265978813171387,
      "activations/layer10_attention_weight_max": 42.761322021484375,
      "activations/layer10_attention_weight_min": -39.472755432128906,
      "activations/layer11_attention_weight_max": 34.46910858154297,
      "activations/layer11_attention_weight_min": -34.5421257019043,
      "activations/layer12_attention_weight_max": 21.98160171508789,
      "activations/layer12_attention_weight_min": -23.94589614868164,
      "activations/layer13_attention_weight_max": 37.4477653503418,
      "activations/layer13_attention_weight_min": -32.476051330566406,
      "activations/layer14_attention_weight_max": 40.51972198486328,
      "activations/layer14_attention_weight_min": -30.470537185668945,
      "activations/layer15_attention_weight_max": 35.47605895996094,
      "activations/layer15_attention_weight_min": -30.418827056884766,
      "activations/layer16_attention_weight_max": 35.31028747558594,
      "activations/layer16_attention_weight_min": -30.38554573059082,
      "activations/layer17_attention_weight_max": 57.60603713989258,
      "activations/layer17_attention_weight_min": -46.0671272277832,
      "activations/layer18_attention_weight_max": 52.235530853271484,
      "activations/layer18_attention_weight_min": -38.7622184753418,
      "activations/layer19_attention_weight_max": 27.733856201171875,
      "activations/layer19_attention_weight_min": -22.804319381713867,
      "activations/layer1_attention_weight_max": 16.49105453491211,
      "activations/layer1_attention_weight_min": -14.926701545715332,
      "activations/layer20_attention_weight_max": 26.686267852783203,
      "activations/layer20_attention_weight_min": -23.779523849487305,
      "activations/layer21_attention_weight_max": 37.83850860595703,
      "activations/layer21_attention_weight_min": -23.66925048828125,
      "activations/layer22_attention_weight_max": 35.61015701293945,
      "activations/layer22_attention_weight_min": -24.600379943847656,
      "activations/layer23_attention_weight_max": 38.4232063293457,
      "activations/layer23_attention_weight_min": -21.37301254272461,
      "activations/layer2_attention_weight_max": 33.98423385620117,
      "activations/layer2_attention_weight_min": -32.98899841308594,
      "activations/layer3_attention_weight_max": 100.04364776611328,
      "activations/layer3_attention_weight_min": -104.42352294921875,
      "activations/layer4_attention_weight_max": 105.23583221435547,
      "activations/layer4_attention_weight_min": -105.84268951416016,
      "activations/layer5_attention_weight_max": 75.52224731445312,
      "activations/layer5_attention_weight_min": -79.2403564453125,
      "activations/layer6_attention_weight_max": 53.47504425048828,
      "activations/layer6_attention_weight_min": -56.53886032104492,
      "activations/layer7_attention_weight_max": 80.37462615966797,
      "activations/layer7_attention_weight_min": -71.5658950805664,
      "activations/layer8_attention_weight_max": 46.4697265625,
      "activations/layer8_attention_weight_min": -49.74406814575195,
      "activations/layer9_attention_weight_max": 49.22091293334961,
      "activations/layer9_attention_weight_min": -47.580284118652344,
      "epoch": 11.8,
      "learning_rate": 7.463030303030302e-05,
      "loss": 2.7775,
      "step": 203150
    },
    {
      "activations/layer0_attention_weight_max": 15.345914840698242,
      "activations/layer0_attention_weight_min": -14.059480667114258,
      "activations/layer10_attention_weight_max": 36.738075256347656,
      "activations/layer10_attention_weight_min": -37.23655700683594,
      "activations/layer11_attention_weight_max": 32.84532928466797,
      "activations/layer11_attention_weight_min": -32.62662887573242,
      "activations/layer12_attention_weight_max": 19.56174087524414,
      "activations/layer12_attention_weight_min": -23.26663589477539,
      "activations/layer13_attention_weight_max": 43.40776824951172,
      "activations/layer13_attention_weight_min": -30.000560760498047,
      "activations/layer14_attention_weight_max": 39.854331970214844,
      "activations/layer14_attention_weight_min": -31.622262954711914,
      "activations/layer15_attention_weight_max": 34.70926284790039,
      "activations/layer15_attention_weight_min": -31.209819793701172,
      "activations/layer16_attention_weight_max": 33.622528076171875,
      "activations/layer16_attention_weight_min": -30.608654022216797,
      "activations/layer17_attention_weight_max": 57.73968505859375,
      "activations/layer17_attention_weight_min": -50.16320037841797,
      "activations/layer18_attention_weight_max": 48.80661392211914,
      "activations/layer18_attention_weight_min": -40.80134582519531,
      "activations/layer19_attention_weight_max": 25.24677276611328,
      "activations/layer19_attention_weight_min": -21.291614532470703,
      "activations/layer1_attention_weight_max": 16.930631637573242,
      "activations/layer1_attention_weight_min": -16.82282066345215,
      "activations/layer20_attention_weight_max": 24.8277587890625,
      "activations/layer20_attention_weight_min": -23.075237274169922,
      "activations/layer21_attention_weight_max": 39.75764846801758,
      "activations/layer21_attention_weight_min": -21.683340072631836,
      "activations/layer22_attention_weight_max": 30.685123443603516,
      "activations/layer22_attention_weight_min": -25.923730850219727,
      "activations/layer23_attention_weight_max": 39.91214370727539,
      "activations/layer23_attention_weight_min": -25.070030212402344,
      "activations/layer2_attention_weight_max": 32.30360412597656,
      "activations/layer2_attention_weight_min": -31.871952056884766,
      "activations/layer3_attention_weight_max": 99.13956451416016,
      "activations/layer3_attention_weight_min": -98.77410888671875,
      "activations/layer4_attention_weight_max": 100.49796295166016,
      "activations/layer4_attention_weight_min": -101.07476806640625,
      "activations/layer5_attention_weight_max": 77.04315185546875,
      "activations/layer5_attention_weight_min": -84.40655517578125,
      "activations/layer6_attention_weight_max": 56.44522476196289,
      "activations/layer6_attention_weight_min": -54.95071792602539,
      "activations/layer7_attention_weight_max": 78.21700286865234,
      "activations/layer7_attention_weight_min": -72.26986694335938,
      "activations/layer8_attention_weight_max": 48.13130569458008,
      "activations/layer8_attention_weight_min": -52.19270706176758,
      "activations/layer9_attention_weight_max": 47.717864990234375,
      "activations/layer9_attention_weight_min": -49.28988265991211,
      "epoch": 11.81,
      "learning_rate": 7.461136363636364e-05,
      "loss": 2.7619,
      "step": 203200
    },
    {
      "activations/layer0_attention_weight_max": 16.112897872924805,
      "activations/layer0_attention_weight_min": -13.853204727172852,
      "activations/layer10_attention_weight_max": 37.552528381347656,
      "activations/layer10_attention_weight_min": -36.69801330566406,
      "activations/layer11_attention_weight_max": 34.84398651123047,
      "activations/layer11_attention_weight_min": -33.87849426269531,
      "activations/layer12_attention_weight_max": 19.06523895263672,
      "activations/layer12_attention_weight_min": -28.144933700561523,
      "activations/layer13_attention_weight_max": 35.04252624511719,
      "activations/layer13_attention_weight_min": -29.290081024169922,
      "activations/layer14_attention_weight_max": 34.34914016723633,
      "activations/layer14_attention_weight_min": -30.333799362182617,
      "activations/layer15_attention_weight_max": 32.481727600097656,
      "activations/layer15_attention_weight_min": -30.830354690551758,
      "activations/layer16_attention_weight_max": 32.49131774902344,
      "activations/layer16_attention_weight_min": -30.78041648864746,
      "activations/layer17_attention_weight_max": 54.23996353149414,
      "activations/layer17_attention_weight_min": -50.46855545043945,
      "activations/layer18_attention_weight_max": 47.488101959228516,
      "activations/layer18_attention_weight_min": -40.699100494384766,
      "activations/layer19_attention_weight_max": 22.968584060668945,
      "activations/layer19_attention_weight_min": -21.825220108032227,
      "activations/layer1_attention_weight_max": 17.221160888671875,
      "activations/layer1_attention_weight_min": -16.70830726623535,
      "activations/layer20_attention_weight_max": 21.867088317871094,
      "activations/layer20_attention_weight_min": -21.76527214050293,
      "activations/layer21_attention_weight_max": 40.57126998901367,
      "activations/layer21_attention_weight_min": -23.924325942993164,
      "activations/layer22_attention_weight_max": 28.63599967956543,
      "activations/layer22_attention_weight_min": -26.146495819091797,
      "activations/layer23_attention_weight_max": 35.499290466308594,
      "activations/layer23_attention_weight_min": -24.867902755737305,
      "activations/layer2_attention_weight_max": 34.58313751220703,
      "activations/layer2_attention_weight_min": -31.85318374633789,
      "activations/layer3_attention_weight_max": 99.01644897460938,
      "activations/layer3_attention_weight_min": -99.85672760009766,
      "activations/layer4_attention_weight_max": 97.5928726196289,
      "activations/layer4_attention_weight_min": -94.6313705444336,
      "activations/layer5_attention_weight_max": 73.77971649169922,
      "activations/layer5_attention_weight_min": -78.72218322753906,
      "activations/layer6_attention_weight_max": 52.235191345214844,
      "activations/layer6_attention_weight_min": -52.013729095458984,
      "activations/layer7_attention_weight_max": 72.57844543457031,
      "activations/layer7_attention_weight_min": -69.08229064941406,
      "activations/layer8_attention_weight_max": 46.48604202270508,
      "activations/layer8_attention_weight_min": -46.940616607666016,
      "activations/layer9_attention_weight_max": 51.11030960083008,
      "activations/layer9_attention_weight_min": -46.01621627807617,
      "epoch": 11.81,
      "learning_rate": 7.459242424242424e-05,
      "loss": 2.7823,
      "step": 203250
    },
    {
      "activations/layer0_attention_weight_max": 16.005971908569336,
      "activations/layer0_attention_weight_min": -14.030706405639648,
      "activations/layer10_attention_weight_max": 36.907020568847656,
      "activations/layer10_attention_weight_min": -35.08479690551758,
      "activations/layer11_attention_weight_max": 31.498313903808594,
      "activations/layer11_attention_weight_min": -31.718608856201172,
      "activations/layer12_attention_weight_max": 18.31617546081543,
      "activations/layer12_attention_weight_min": -23.446945190429688,
      "activations/layer13_attention_weight_max": 36.5484733581543,
      "activations/layer13_attention_weight_min": -29.791215896606445,
      "activations/layer14_attention_weight_max": 37.28771209716797,
      "activations/layer14_attention_weight_min": -29.083293914794922,
      "activations/layer15_attention_weight_max": 28.93492317199707,
      "activations/layer15_attention_weight_min": -28.87079620361328,
      "activations/layer16_attention_weight_max": 27.477758407592773,
      "activations/layer16_attention_weight_min": -29.377290725708008,
      "activations/layer17_attention_weight_max": 45.833805084228516,
      "activations/layer17_attention_weight_min": -42.93232345581055,
      "activations/layer18_attention_weight_max": 43.323341369628906,
      "activations/layer18_attention_weight_min": -34.74407958984375,
      "activations/layer19_attention_weight_max": 20.86087989807129,
      "activations/layer19_attention_weight_min": -20.17803955078125,
      "activations/layer1_attention_weight_max": 16.319290161132812,
      "activations/layer1_attention_weight_min": -14.853203773498535,
      "activations/layer20_attention_weight_max": 21.487028121948242,
      "activations/layer20_attention_weight_min": -20.362382888793945,
      "activations/layer21_attention_weight_max": 31.771188735961914,
      "activations/layer21_attention_weight_min": -22.741756439208984,
      "activations/layer22_attention_weight_max": 28.380491256713867,
      "activations/layer22_attention_weight_min": -25.71052360534668,
      "activations/layer23_attention_weight_max": 38.45079803466797,
      "activations/layer23_attention_weight_min": -20.97244644165039,
      "activations/layer2_attention_weight_max": 34.52435302734375,
      "activations/layer2_attention_weight_min": -32.53050994873047,
      "activations/layer3_attention_weight_max": 96.5453109741211,
      "activations/layer3_attention_weight_min": -99.32711791992188,
      "activations/layer4_attention_weight_max": 99.60273742675781,
      "activations/layer4_attention_weight_min": -98.93718719482422,
      "activations/layer5_attention_weight_max": 78.52024841308594,
      "activations/layer5_attention_weight_min": -81.63499450683594,
      "activations/layer6_attention_weight_max": 52.180416107177734,
      "activations/layer6_attention_weight_min": -53.76872634887695,
      "activations/layer7_attention_weight_max": 75.24797821044922,
      "activations/layer7_attention_weight_min": -70.50120544433594,
      "activations/layer8_attention_weight_max": 49.94520950317383,
      "activations/layer8_attention_weight_min": -49.60234069824219,
      "activations/layer9_attention_weight_max": 51.15394592285156,
      "activations/layer9_attention_weight_min": -45.603790283203125,
      "epoch": 11.81,
      "learning_rate": 7.457348484848484e-05,
      "loss": 2.7805,
      "step": 203300
    },
    {
      "activations/layer0_attention_weight_max": 15.9983491897583,
      "activations/layer0_attention_weight_min": -13.984017372131348,
      "activations/layer10_attention_weight_max": 36.31133270263672,
      "activations/layer10_attention_weight_min": -35.11399841308594,
      "activations/layer11_attention_weight_max": 33.887306213378906,
      "activations/layer11_attention_weight_min": -34.34808349609375,
      "activations/layer12_attention_weight_max": 18.31650161743164,
      "activations/layer12_attention_weight_min": -23.3759822845459,
      "activations/layer13_attention_weight_max": 37.47687911987305,
      "activations/layer13_attention_weight_min": -28.377614974975586,
      "activations/layer14_attention_weight_max": 36.821720123291016,
      "activations/layer14_attention_weight_min": -29.11806297302246,
      "activations/layer15_attention_weight_max": 34.01388168334961,
      "activations/layer15_attention_weight_min": -27.685684204101562,
      "activations/layer16_attention_weight_max": 32.09531021118164,
      "activations/layer16_attention_weight_min": -29.4226131439209,
      "activations/layer17_attention_weight_max": 51.913658142089844,
      "activations/layer17_attention_weight_min": -42.88087844848633,
      "activations/layer18_attention_weight_max": 49.60593795776367,
      "activations/layer18_attention_weight_min": -36.15896987915039,
      "activations/layer19_attention_weight_max": 22.355167388916016,
      "activations/layer19_attention_weight_min": -20.413068771362305,
      "activations/layer1_attention_weight_max": 15.975519180297852,
      "activations/layer1_attention_weight_min": -14.335963249206543,
      "activations/layer20_attention_weight_max": 22.57961654663086,
      "activations/layer20_attention_weight_min": -22.69347381591797,
      "activations/layer21_attention_weight_max": 42.74171829223633,
      "activations/layer21_attention_weight_min": -21.699447631835938,
      "activations/layer22_attention_weight_max": 30.413999557495117,
      "activations/layer22_attention_weight_min": -24.67211151123047,
      "activations/layer23_attention_weight_max": 41.78556442260742,
      "activations/layer23_attention_weight_min": -25.568004608154297,
      "activations/layer2_attention_weight_max": 32.79095458984375,
      "activations/layer2_attention_weight_min": -32.5509033203125,
      "activations/layer3_attention_weight_max": 97.43097686767578,
      "activations/layer3_attention_weight_min": -96.53958129882812,
      "activations/layer4_attention_weight_max": 104.31465148925781,
      "activations/layer4_attention_weight_min": -99.8249740600586,
      "activations/layer5_attention_weight_max": 75.57904052734375,
      "activations/layer5_attention_weight_min": -84.28450012207031,
      "activations/layer6_attention_weight_max": 55.13637161254883,
      "activations/layer6_attention_weight_min": -53.96303176879883,
      "activations/layer7_attention_weight_max": 78.71754455566406,
      "activations/layer7_attention_weight_min": -75.1537094116211,
      "activations/layer8_attention_weight_max": 47.671783447265625,
      "activations/layer8_attention_weight_min": -47.73080062866211,
      "activations/layer9_attention_weight_max": 46.349342346191406,
      "activations/layer9_attention_weight_min": -47.87714767456055,
      "epoch": 11.82,
      "learning_rate": 7.455454545454545e-05,
      "loss": 2.7847,
      "step": 203350
    },
    {
      "activations/layer0_attention_weight_max": 15.78283977508545,
      "activations/layer0_attention_weight_min": -14.15206241607666,
      "activations/layer10_attention_weight_max": 37.51182556152344,
      "activations/layer10_attention_weight_min": -37.73932647705078,
      "activations/layer11_attention_weight_max": 34.89467239379883,
      "activations/layer11_attention_weight_min": -33.2053108215332,
      "activations/layer12_attention_weight_max": 19.550270080566406,
      "activations/layer12_attention_weight_min": -27.353662490844727,
      "activations/layer13_attention_weight_max": 37.51411056518555,
      "activations/layer13_attention_weight_min": -28.832565307617188,
      "activations/layer14_attention_weight_max": 40.277313232421875,
      "activations/layer14_attention_weight_min": -32.429561614990234,
      "activations/layer15_attention_weight_max": 34.06019592285156,
      "activations/layer15_attention_weight_min": -29.622224807739258,
      "activations/layer16_attention_weight_max": 33.83600616455078,
      "activations/layer16_attention_weight_min": -31.38425636291504,
      "activations/layer17_attention_weight_max": 58.1136360168457,
      "activations/layer17_attention_weight_min": -46.88175964355469,
      "activations/layer18_attention_weight_max": 50.04812240600586,
      "activations/layer18_attention_weight_min": -42.46309280395508,
      "activations/layer19_attention_weight_max": 23.753904342651367,
      "activations/layer19_attention_weight_min": -23.262859344482422,
      "activations/layer1_attention_weight_max": 16.528644561767578,
      "activations/layer1_attention_weight_min": -15.467147827148438,
      "activations/layer20_attention_weight_max": 23.97760009765625,
      "activations/layer20_attention_weight_min": -21.47113609313965,
      "activations/layer21_attention_weight_max": 42.51622772216797,
      "activations/layer21_attention_weight_min": -22.811254501342773,
      "activations/layer22_attention_weight_max": 32.56373977661133,
      "activations/layer22_attention_weight_min": -26.44468116760254,
      "activations/layer23_attention_weight_max": 42.38670349121094,
      "activations/layer23_attention_weight_min": -25.800575256347656,
      "activations/layer2_attention_weight_max": 34.19398498535156,
      "activations/layer2_attention_weight_min": -32.43246841430664,
      "activations/layer3_attention_weight_max": 99.50354766845703,
      "activations/layer3_attention_weight_min": -104.97203063964844,
      "activations/layer4_attention_weight_max": 100.18608093261719,
      "activations/layer4_attention_weight_min": -100.18743133544922,
      "activations/layer5_attention_weight_max": 74.94660186767578,
      "activations/layer5_attention_weight_min": -78.76066589355469,
      "activations/layer6_attention_weight_max": 54.01576614379883,
      "activations/layer6_attention_weight_min": -54.36670684814453,
      "activations/layer7_attention_weight_max": 75.24654388427734,
      "activations/layer7_attention_weight_min": -76.49458312988281,
      "activations/layer8_attention_weight_max": 47.74614715576172,
      "activations/layer8_attention_weight_min": -50.64142990112305,
      "activations/layer9_attention_weight_max": 54.77544021606445,
      "activations/layer9_attention_weight_min": -49.793495178222656,
      "epoch": 11.82,
      "learning_rate": 7.453560606060606e-05,
      "loss": 2.7764,
      "step": 203400
    },
    {
      "activations/layer0_attention_weight_max": 15.627403259277344,
      "activations/layer0_attention_weight_min": -13.94797420501709,
      "activations/layer10_attention_weight_max": 37.66522979736328,
      "activations/layer10_attention_weight_min": -35.5215950012207,
      "activations/layer11_attention_weight_max": 32.581111907958984,
      "activations/layer11_attention_weight_min": -31.58761978149414,
      "activations/layer12_attention_weight_max": 17.52823257446289,
      "activations/layer12_attention_weight_min": -21.805498123168945,
      "activations/layer13_attention_weight_max": 43.977481842041016,
      "activations/layer13_attention_weight_min": -32.470428466796875,
      "activations/layer14_attention_weight_max": 39.40630340576172,
      "activations/layer14_attention_weight_min": -29.92635154724121,
      "activations/layer15_attention_weight_max": 33.98058319091797,
      "activations/layer15_attention_weight_min": -30.003284454345703,
      "activations/layer16_attention_weight_max": 31.51593780517578,
      "activations/layer16_attention_weight_min": -28.272613525390625,
      "activations/layer17_attention_weight_max": 52.352359771728516,
      "activations/layer17_attention_weight_min": -41.92475509643555,
      "activations/layer18_attention_weight_max": 48.80497360229492,
      "activations/layer18_attention_weight_min": -36.63174819946289,
      "activations/layer19_attention_weight_max": 21.013307571411133,
      "activations/layer19_attention_weight_min": -21.2106876373291,
      "activations/layer1_attention_weight_max": 17.6585750579834,
      "activations/layer1_attention_weight_min": -16.12893295288086,
      "activations/layer20_attention_weight_max": 21.651460647583008,
      "activations/layer20_attention_weight_min": -20.65561866760254,
      "activations/layer21_attention_weight_max": 39.080543518066406,
      "activations/layer21_attention_weight_min": -24.23538589477539,
      "activations/layer22_attention_weight_max": 31.33353042602539,
      "activations/layer22_attention_weight_min": -27.246673583984375,
      "activations/layer23_attention_weight_max": 36.314449310302734,
      "activations/layer23_attention_weight_min": -25.378446578979492,
      "activations/layer2_attention_weight_max": 31.68440818786621,
      "activations/layer2_attention_weight_min": -31.400253295898438,
      "activations/layer3_attention_weight_max": 97.60956573486328,
      "activations/layer3_attention_weight_min": -98.6650161743164,
      "activations/layer4_attention_weight_max": 100.0770263671875,
      "activations/layer4_attention_weight_min": -97.34187316894531,
      "activations/layer5_attention_weight_max": 71.01162719726562,
      "activations/layer5_attention_weight_min": -77.24400329589844,
      "activations/layer6_attention_weight_max": 52.99630355834961,
      "activations/layer6_attention_weight_min": -53.945125579833984,
      "activations/layer7_attention_weight_max": 72.14749908447266,
      "activations/layer7_attention_weight_min": -73.67163848876953,
      "activations/layer8_attention_weight_max": 46.77952194213867,
      "activations/layer8_attention_weight_min": -46.15407180786133,
      "activations/layer9_attention_weight_max": 47.27716827392578,
      "activations/layer9_attention_weight_min": -46.93362045288086,
      "epoch": 11.82,
      "learning_rate": 7.451666666666666e-05,
      "loss": 2.7771,
      "step": 203450
    },
    {
      "activations/layer0_attention_weight_max": 15.47088623046875,
      "activations/layer0_attention_weight_min": -14.406307220458984,
      "activations/layer10_attention_weight_max": 36.40371322631836,
      "activations/layer10_attention_weight_min": -34.71623611450195,
      "activations/layer11_attention_weight_max": 29.519451141357422,
      "activations/layer11_attention_weight_min": -32.310001373291016,
      "activations/layer12_attention_weight_max": 19.12264633178711,
      "activations/layer12_attention_weight_min": -26.283666610717773,
      "activations/layer13_attention_weight_max": 36.15156173706055,
      "activations/layer13_attention_weight_min": -30.66306495666504,
      "activations/layer14_attention_weight_max": 36.24409866333008,
      "activations/layer14_attention_weight_min": -30.34553337097168,
      "activations/layer15_attention_weight_max": 34.173797607421875,
      "activations/layer15_attention_weight_min": -29.869441986083984,
      "activations/layer16_attention_weight_max": 31.027509689331055,
      "activations/layer16_attention_weight_min": -29.774803161621094,
      "activations/layer17_attention_weight_max": 53.40715408325195,
      "activations/layer17_attention_weight_min": -45.961341857910156,
      "activations/layer18_attention_weight_max": 47.335357666015625,
      "activations/layer18_attention_weight_min": -38.840972900390625,
      "activations/layer19_attention_weight_max": 23.771059036254883,
      "activations/layer19_attention_weight_min": -22.700538635253906,
      "activations/layer1_attention_weight_max": 16.979015350341797,
      "activations/layer1_attention_weight_min": -15.034651756286621,
      "activations/layer20_attention_weight_max": 22.202543258666992,
      "activations/layer20_attention_weight_min": -22.0725040435791,
      "activations/layer21_attention_weight_max": 38.817745208740234,
      "activations/layer21_attention_weight_min": -26.82526206970215,
      "activations/layer22_attention_weight_max": 28.61855125427246,
      "activations/layer22_attention_weight_min": -24.223283767700195,
      "activations/layer23_attention_weight_max": 39.769222259521484,
      "activations/layer23_attention_weight_min": -26.249279022216797,
      "activations/layer2_attention_weight_max": 34.40842056274414,
      "activations/layer2_attention_weight_min": -31.691160202026367,
      "activations/layer3_attention_weight_max": 95.25963592529297,
      "activations/layer3_attention_weight_min": -96.96327209472656,
      "activations/layer4_attention_weight_max": 95.43535614013672,
      "activations/layer4_attention_weight_min": -94.44544982910156,
      "activations/layer5_attention_weight_max": 68.75431060791016,
      "activations/layer5_attention_weight_min": -78.4537353515625,
      "activations/layer6_attention_weight_max": 55.03683853149414,
      "activations/layer6_attention_weight_min": -51.217037200927734,
      "activations/layer7_attention_weight_max": 74.49182891845703,
      "activations/layer7_attention_weight_min": -67.75421905517578,
      "activations/layer8_attention_weight_max": 44.8004035949707,
      "activations/layer8_attention_weight_min": -45.72846984863281,
      "activations/layer9_attention_weight_max": 44.377769470214844,
      "activations/layer9_attention_weight_min": -46.25004577636719,
      "epoch": 11.82,
      "learning_rate": 7.449772727272727e-05,
      "loss": 2.7812,
      "step": 203500
    },
    {
      "activations/layer0_attention_weight_max": 17.520139694213867,
      "activations/layer0_attention_weight_min": -13.929935455322266,
      "activations/layer10_attention_weight_max": 37.70048141479492,
      "activations/layer10_attention_weight_min": -37.67987060546875,
      "activations/layer11_attention_weight_max": 32.868324279785156,
      "activations/layer11_attention_weight_min": -33.79554748535156,
      "activations/layer12_attention_weight_max": 18.834026336669922,
      "activations/layer12_attention_weight_min": -25.14035987854004,
      "activations/layer13_attention_weight_max": 46.896366119384766,
      "activations/layer13_attention_weight_min": -36.47199630737305,
      "activations/layer14_attention_weight_max": 42.48611068725586,
      "activations/layer14_attention_weight_min": -34.27463150024414,
      "activations/layer15_attention_weight_max": 35.416358947753906,
      "activations/layer15_attention_weight_min": -33.13772201538086,
      "activations/layer16_attention_weight_max": 34.19878387451172,
      "activations/layer16_attention_weight_min": -32.72899627685547,
      "activations/layer17_attention_weight_max": 58.3580322265625,
      "activations/layer17_attention_weight_min": -48.87653732299805,
      "activations/layer18_attention_weight_max": 53.74698257446289,
      "activations/layer18_attention_weight_min": -43.05900573730469,
      "activations/layer19_attention_weight_max": 26.60271644592285,
      "activations/layer19_attention_weight_min": -24.76300621032715,
      "activations/layer1_attention_weight_max": 17.175867080688477,
      "activations/layer1_attention_weight_min": -16.97456932067871,
      "activations/layer20_attention_weight_max": 26.92974853515625,
      "activations/layer20_attention_weight_min": -24.039155960083008,
      "activations/layer21_attention_weight_max": 50.98715591430664,
      "activations/layer21_attention_weight_min": -24.54283905029297,
      "activations/layer22_attention_weight_max": 39.15987014770508,
      "activations/layer22_attention_weight_min": -29.18087387084961,
      "activations/layer23_attention_weight_max": 44.354095458984375,
      "activations/layer23_attention_weight_min": -23.583219528198242,
      "activations/layer2_attention_weight_max": 34.020423889160156,
      "activations/layer2_attention_weight_min": -33.18229293823242,
      "activations/layer3_attention_weight_max": 99.39165496826172,
      "activations/layer3_attention_weight_min": -101.26640319824219,
      "activations/layer4_attention_weight_max": 99.4991455078125,
      "activations/layer4_attention_weight_min": -95.99161529541016,
      "activations/layer5_attention_weight_max": 75.65773010253906,
      "activations/layer5_attention_weight_min": -83.04551696777344,
      "activations/layer6_attention_weight_max": 53.00587463378906,
      "activations/layer6_attention_weight_min": -53.54273223876953,
      "activations/layer7_attention_weight_max": 71.2837905883789,
      "activations/layer7_attention_weight_min": -73.56037902832031,
      "activations/layer8_attention_weight_max": 47.06421661376953,
      "activations/layer8_attention_weight_min": -46.79766082763672,
      "activations/layer9_attention_weight_max": 49.26366424560547,
      "activations/layer9_attention_weight_min": -48.178340911865234,
      "epoch": 11.83,
      "learning_rate": 7.447878787878787e-05,
      "loss": 2.7817,
      "step": 203550
    },
    {
      "activations/layer0_attention_weight_max": 15.379073143005371,
      "activations/layer0_attention_weight_min": -13.883048057556152,
      "activations/layer10_attention_weight_max": 39.95302200317383,
      "activations/layer10_attention_weight_min": -36.65425491333008,
      "activations/layer11_attention_weight_max": 36.19104766845703,
      "activations/layer11_attention_weight_min": -32.50055694580078,
      "activations/layer12_attention_weight_max": 20.160573959350586,
      "activations/layer12_attention_weight_min": -25.210729598999023,
      "activations/layer13_attention_weight_max": 55.42738723754883,
      "activations/layer13_attention_weight_min": -33.574493408203125,
      "activations/layer14_attention_weight_max": 59.690616607666016,
      "activations/layer14_attention_weight_min": -31.71044158935547,
      "activations/layer15_attention_weight_max": 48.13374710083008,
      "activations/layer15_attention_weight_min": -30.51612091064453,
      "activations/layer16_attention_weight_max": 39.335960388183594,
      "activations/layer16_attention_weight_min": -32.31398010253906,
      "activations/layer17_attention_weight_max": 66.30072021484375,
      "activations/layer17_attention_weight_min": -48.59148025512695,
      "activations/layer18_attention_weight_max": 61.60254669189453,
      "activations/layer18_attention_weight_min": -39.75922775268555,
      "activations/layer19_attention_weight_max": 32.254695892333984,
      "activations/layer19_attention_weight_min": -23.367063522338867,
      "activations/layer1_attention_weight_max": 17.219743728637695,
      "activations/layer1_attention_weight_min": -15.569942474365234,
      "activations/layer20_attention_weight_max": 32.19473648071289,
      "activations/layer20_attention_weight_min": -22.681846618652344,
      "activations/layer21_attention_weight_max": 54.28139114379883,
      "activations/layer21_attention_weight_min": -25.592870712280273,
      "activations/layer22_attention_weight_max": 36.11552047729492,
      "activations/layer22_attention_weight_min": -26.29737663269043,
      "activations/layer23_attention_weight_max": 49.180381774902344,
      "activations/layer23_attention_weight_min": -27.352115631103516,
      "activations/layer2_attention_weight_max": 34.330482482910156,
      "activations/layer2_attention_weight_min": -33.450035095214844,
      "activations/layer3_attention_weight_max": 101.98175811767578,
      "activations/layer3_attention_weight_min": -105.69920349121094,
      "activations/layer4_attention_weight_max": 104.1142349243164,
      "activations/layer4_attention_weight_min": -102.74244689941406,
      "activations/layer5_attention_weight_max": 73.79490661621094,
      "activations/layer5_attention_weight_min": -77.62554168701172,
      "activations/layer6_attention_weight_max": 54.58089828491211,
      "activations/layer6_attention_weight_min": -55.67656707763672,
      "activations/layer7_attention_weight_max": 76.0477066040039,
      "activations/layer7_attention_weight_min": -75.0274658203125,
      "activations/layer8_attention_weight_max": 47.17211151123047,
      "activations/layer8_attention_weight_min": -49.0939826965332,
      "activations/layer9_attention_weight_max": 51.891273498535156,
      "activations/layer9_attention_weight_min": -52.59387969970703,
      "epoch": 11.83,
      "learning_rate": 7.445984848484848e-05,
      "loss": 2.7621,
      "step": 203600
    },
    {
      "activations/layer0_attention_weight_max": 15.970513343811035,
      "activations/layer0_attention_weight_min": -13.959192276000977,
      "activations/layer10_attention_weight_max": 41.05024337768555,
      "activations/layer10_attention_weight_min": -38.05485916137695,
      "activations/layer11_attention_weight_max": 37.71540069580078,
      "activations/layer11_attention_weight_min": -34.61447525024414,
      "activations/layer12_attention_weight_max": 20.885921478271484,
      "activations/layer12_attention_weight_min": -28.6762752532959,
      "activations/layer13_attention_weight_max": 37.321022033691406,
      "activations/layer13_attention_weight_min": -28.18448257446289,
      "activations/layer14_attention_weight_max": 36.16586685180664,
      "activations/layer14_attention_weight_min": -30.878433227539062,
      "activations/layer15_attention_weight_max": 33.35527038574219,
      "activations/layer15_attention_weight_min": -29.632455825805664,
      "activations/layer16_attention_weight_max": 32.32279968261719,
      "activations/layer16_attention_weight_min": -29.356948852539062,
      "activations/layer17_attention_weight_max": 51.86626052856445,
      "activations/layer17_attention_weight_min": -45.17512512207031,
      "activations/layer18_attention_weight_max": 47.236595153808594,
      "activations/layer18_attention_weight_min": -39.34378433227539,
      "activations/layer19_attention_weight_max": 22.68686866760254,
      "activations/layer19_attention_weight_min": -22.12248992919922,
      "activations/layer1_attention_weight_max": 16.800546646118164,
      "activations/layer1_attention_weight_min": -16.582609176635742,
      "activations/layer20_attention_weight_max": 23.7655029296875,
      "activations/layer20_attention_weight_min": -20.976016998291016,
      "activations/layer21_attention_weight_max": 36.780052185058594,
      "activations/layer21_attention_weight_min": -27.173967361450195,
      "activations/layer22_attention_weight_max": 31.47889518737793,
      "activations/layer22_attention_weight_min": -27.20449447631836,
      "activations/layer23_attention_weight_max": 36.09806442260742,
      "activations/layer23_attention_weight_min": -22.875205993652344,
      "activations/layer2_attention_weight_max": 32.866493225097656,
      "activations/layer2_attention_weight_min": -30.029197692871094,
      "activations/layer3_attention_weight_max": 89.74267578125,
      "activations/layer3_attention_weight_min": -92.95521545410156,
      "activations/layer4_attention_weight_max": 99.30443572998047,
      "activations/layer4_attention_weight_min": -92.665283203125,
      "activations/layer5_attention_weight_max": 76.49291229248047,
      "activations/layer5_attention_weight_min": -74.90264892578125,
      "activations/layer6_attention_weight_max": 55.8855094909668,
      "activations/layer6_attention_weight_min": -53.606712341308594,
      "activations/layer7_attention_weight_max": 82.83277130126953,
      "activations/layer7_attention_weight_min": -72.7497329711914,
      "activations/layer8_attention_weight_max": 51.50661087036133,
      "activations/layer8_attention_weight_min": -50.97188186645508,
      "activations/layer9_attention_weight_max": 61.204925537109375,
      "activations/layer9_attention_weight_min": -49.34489440917969,
      "epoch": 11.83,
      "learning_rate": 7.444090909090908e-05,
      "loss": 2.7831,
      "step": 203650
    },
    {
      "activations/layer0_attention_weight_max": 16.777057647705078,
      "activations/layer0_attention_weight_min": -14.45808219909668,
      "activations/layer10_attention_weight_max": 38.10247039794922,
      "activations/layer10_attention_weight_min": -35.04463195800781,
      "activations/layer11_attention_weight_max": 36.80023193359375,
      "activations/layer11_attention_weight_min": -34.78583526611328,
      "activations/layer12_attention_weight_max": 18.485815048217773,
      "activations/layer12_attention_weight_min": -26.201526641845703,
      "activations/layer13_attention_weight_max": 38.8786506652832,
      "activations/layer13_attention_weight_min": -31.330154418945312,
      "activations/layer14_attention_weight_max": 45.88068771362305,
      "activations/layer14_attention_weight_min": -30.47974395751953,
      "activations/layer15_attention_weight_max": 35.188446044921875,
      "activations/layer15_attention_weight_min": -30.168472290039062,
      "activations/layer16_attention_weight_max": 31.96458625793457,
      "activations/layer16_attention_weight_min": -31.05348777770996,
      "activations/layer17_attention_weight_max": 51.92659378051758,
      "activations/layer17_attention_weight_min": -46.81661605834961,
      "activations/layer18_attention_weight_max": 46.38876724243164,
      "activations/layer18_attention_weight_min": -38.40622329711914,
      "activations/layer19_attention_weight_max": 22.363759994506836,
      "activations/layer19_attention_weight_min": -22.755142211914062,
      "activations/layer1_attention_weight_max": 16.82375717163086,
      "activations/layer1_attention_weight_min": -16.6428165435791,
      "activations/layer20_attention_weight_max": 23.482969284057617,
      "activations/layer20_attention_weight_min": -20.650497436523438,
      "activations/layer21_attention_weight_max": 38.26509094238281,
      "activations/layer21_attention_weight_min": -22.256492614746094,
      "activations/layer22_attention_weight_max": 30.38831901550293,
      "activations/layer22_attention_weight_min": -27.46285629272461,
      "activations/layer23_attention_weight_max": 36.24061584472656,
      "activations/layer23_attention_weight_min": -23.188861846923828,
      "activations/layer2_attention_weight_max": 36.283485412597656,
      "activations/layer2_attention_weight_min": -32.762977600097656,
      "activations/layer3_attention_weight_max": 99.9998550415039,
      "activations/layer3_attention_weight_min": -103.22179412841797,
      "activations/layer4_attention_weight_max": 102.03247833251953,
      "activations/layer4_attention_weight_min": -97.14110565185547,
      "activations/layer5_attention_weight_max": 73.84381103515625,
      "activations/layer5_attention_weight_min": -83.31895446777344,
      "activations/layer6_attention_weight_max": 53.61906433105469,
      "activations/layer6_attention_weight_min": -55.53804016113281,
      "activations/layer7_attention_weight_max": 73.87279510498047,
      "activations/layer7_attention_weight_min": -72.55755615234375,
      "activations/layer8_attention_weight_max": 46.316287994384766,
      "activations/layer8_attention_weight_min": -46.685768127441406,
      "activations/layer9_attention_weight_max": 57.736995697021484,
      "activations/layer9_attention_weight_min": -47.46744918823242,
      "epoch": 11.84,
      "learning_rate": 7.442196969696969e-05,
      "loss": 2.7706,
      "step": 203700
    },
    {
      "activations/layer0_attention_weight_max": 16.248001098632812,
      "activations/layer0_attention_weight_min": -14.048736572265625,
      "activations/layer10_attention_weight_max": 39.15614700317383,
      "activations/layer10_attention_weight_min": -38.45451736450195,
      "activations/layer11_attention_weight_max": 32.375640869140625,
      "activations/layer11_attention_weight_min": -32.103065490722656,
      "activations/layer12_attention_weight_max": 18.07123565673828,
      "activations/layer12_attention_weight_min": -27.162479400634766,
      "activations/layer13_attention_weight_max": 38.66237258911133,
      "activations/layer13_attention_weight_min": -29.28850555419922,
      "activations/layer14_attention_weight_max": 41.22565460205078,
      "activations/layer14_attention_weight_min": -31.741626739501953,
      "activations/layer15_attention_weight_max": 40.15938186645508,
      "activations/layer15_attention_weight_min": -30.537038803100586,
      "activations/layer16_attention_weight_max": 36.065277099609375,
      "activations/layer16_attention_weight_min": -31.388044357299805,
      "activations/layer17_attention_weight_max": 59.22899627685547,
      "activations/layer17_attention_weight_min": -46.63397979736328,
      "activations/layer18_attention_weight_max": 52.140708923339844,
      "activations/layer18_attention_weight_min": -40.71302795410156,
      "activations/layer19_attention_weight_max": 24.036827087402344,
      "activations/layer19_attention_weight_min": -23.372568130493164,
      "activations/layer1_attention_weight_max": 19.288896560668945,
      "activations/layer1_attention_weight_min": -16.498310089111328,
      "activations/layer20_attention_weight_max": 25.099424362182617,
      "activations/layer20_attention_weight_min": -23.30756378173828,
      "activations/layer21_attention_weight_max": 47.26315689086914,
      "activations/layer21_attention_weight_min": -23.885910034179688,
      "activations/layer22_attention_weight_max": 32.2813835144043,
      "activations/layer22_attention_weight_min": -27.079336166381836,
      "activations/layer23_attention_weight_max": 42.24890899658203,
      "activations/layer23_attention_weight_min": -24.712650299072266,
      "activations/layer2_attention_weight_max": 37.44081115722656,
      "activations/layer2_attention_weight_min": -36.891990661621094,
      "activations/layer3_attention_weight_max": 105.6877670288086,
      "activations/layer3_attention_weight_min": -106.65969848632812,
      "activations/layer4_attention_weight_max": 104.38411712646484,
      "activations/layer4_attention_weight_min": -104.95979309082031,
      "activations/layer5_attention_weight_max": 74.40937805175781,
      "activations/layer5_attention_weight_min": -80.14279174804688,
      "activations/layer6_attention_weight_max": 52.7519645690918,
      "activations/layer6_attention_weight_min": -53.04984664916992,
      "activations/layer7_attention_weight_max": 75.35111236572266,
      "activations/layer7_attention_weight_min": -74.56792449951172,
      "activations/layer8_attention_weight_max": 46.596229553222656,
      "activations/layer8_attention_weight_min": -46.98869705200195,
      "activations/layer9_attention_weight_max": 45.520870208740234,
      "activations/layer9_attention_weight_min": -45.29115676879883,
      "epoch": 11.84,
      "learning_rate": 7.440340909090908e-05,
      "loss": 2.7862,
      "step": 203750
    },
    {
      "activations/layer0_attention_weight_max": 15.734658241271973,
      "activations/layer0_attention_weight_min": -13.766082763671875,
      "activations/layer10_attention_weight_max": 37.651248931884766,
      "activations/layer10_attention_weight_min": -36.33345031738281,
      "activations/layer11_attention_weight_max": 32.248836517333984,
      "activations/layer11_attention_weight_min": -31.658767700195312,
      "activations/layer12_attention_weight_max": 20.721899032592773,
      "activations/layer12_attention_weight_min": -23.83706283569336,
      "activations/layer13_attention_weight_max": 44.3662223815918,
      "activations/layer13_attention_weight_min": -28.061119079589844,
      "activations/layer14_attention_weight_max": 34.96945571899414,
      "activations/layer14_attention_weight_min": -30.7899112701416,
      "activations/layer15_attention_weight_max": 35.242919921875,
      "activations/layer15_attention_weight_min": -28.2993221282959,
      "activations/layer16_attention_weight_max": 31.845136642456055,
      "activations/layer16_attention_weight_min": -28.238306045532227,
      "activations/layer17_attention_weight_max": 51.742122650146484,
      "activations/layer17_attention_weight_min": -41.44483184814453,
      "activations/layer18_attention_weight_max": 44.89680099487305,
      "activations/layer18_attention_weight_min": -35.93227005004883,
      "activations/layer19_attention_weight_max": 23.80893898010254,
      "activations/layer19_attention_weight_min": -20.75117301940918,
      "activations/layer1_attention_weight_max": 18.90608787536621,
      "activations/layer1_attention_weight_min": -14.925358772277832,
      "activations/layer20_attention_weight_max": 22.006275177001953,
      "activations/layer20_attention_weight_min": -20.350719451904297,
      "activations/layer21_attention_weight_max": 35.5192756652832,
      "activations/layer21_attention_weight_min": -23.216384887695312,
      "activations/layer22_attention_weight_max": 29.492877960205078,
      "activations/layer22_attention_weight_min": -25.21196746826172,
      "activations/layer23_attention_weight_max": 36.44150924682617,
      "activations/layer23_attention_weight_min": -24.36098289489746,
      "activations/layer2_attention_weight_max": 32.11491394042969,
      "activations/layer2_attention_weight_min": -32.05908966064453,
      "activations/layer3_attention_weight_max": 93.86200714111328,
      "activations/layer3_attention_weight_min": -96.09010314941406,
      "activations/layer4_attention_weight_max": 99.18983459472656,
      "activations/layer4_attention_weight_min": -94.59210968017578,
      "activations/layer5_attention_weight_max": 71.71292114257812,
      "activations/layer5_attention_weight_min": -78.17674255371094,
      "activations/layer6_attention_weight_max": 53.27500534057617,
      "activations/layer6_attention_weight_min": -52.12784957885742,
      "activations/layer7_attention_weight_max": 75.77215576171875,
      "activations/layer7_attention_weight_min": -69.34745025634766,
      "activations/layer8_attention_weight_max": 46.29771041870117,
      "activations/layer8_attention_weight_min": -48.86575698852539,
      "activations/layer9_attention_weight_max": 46.512001037597656,
      "activations/layer9_attention_weight_min": -46.23856735229492,
      "epoch": 11.84,
      "learning_rate": 7.438446969696968e-05,
      "loss": 2.7796,
      "step": 203800
    },
    {
      "activations/layer0_attention_weight_max": 16.33611297607422,
      "activations/layer0_attention_weight_min": -14.33251953125,
      "activations/layer10_attention_weight_max": 35.65071105957031,
      "activations/layer10_attention_weight_min": -37.761627197265625,
      "activations/layer11_attention_weight_max": 32.89485168457031,
      "activations/layer11_attention_weight_min": -35.00433349609375,
      "activations/layer12_attention_weight_max": 19.706438064575195,
      "activations/layer12_attention_weight_min": -29.198816299438477,
      "activations/layer13_attention_weight_max": 42.14390563964844,
      "activations/layer13_attention_weight_min": -28.93391990661621,
      "activations/layer14_attention_weight_max": 35.98434829711914,
      "activations/layer14_attention_weight_min": -29.048994064331055,
      "activations/layer15_attention_weight_max": 32.493263244628906,
      "activations/layer15_attention_weight_min": -28.35886573791504,
      "activations/layer16_attention_weight_max": 30.848018646240234,
      "activations/layer16_attention_weight_min": -28.359455108642578,
      "activations/layer17_attention_weight_max": 54.53968048095703,
      "activations/layer17_attention_weight_min": -46.30052947998047,
      "activations/layer18_attention_weight_max": 50.37163162231445,
      "activations/layer18_attention_weight_min": -37.49897766113281,
      "activations/layer19_attention_weight_max": 23.720014572143555,
      "activations/layer19_attention_weight_min": -21.747722625732422,
      "activations/layer1_attention_weight_max": 16.632530212402344,
      "activations/layer1_attention_weight_min": -14.677713394165039,
      "activations/layer20_attention_weight_max": 23.7387638092041,
      "activations/layer20_attention_weight_min": -25.690244674682617,
      "activations/layer21_attention_weight_max": 35.9766960144043,
      "activations/layer21_attention_weight_min": -23.048967361450195,
      "activations/layer22_attention_weight_max": 28.72974967956543,
      "activations/layer22_attention_weight_min": -25.13644027709961,
      "activations/layer23_attention_weight_max": 39.65227508544922,
      "activations/layer23_attention_weight_min": -25.460372924804688,
      "activations/layer2_attention_weight_max": 30.88983917236328,
      "activations/layer2_attention_weight_min": -31.46961784362793,
      "activations/layer3_attention_weight_max": 91.94670104980469,
      "activations/layer3_attention_weight_min": -96.89153289794922,
      "activations/layer4_attention_weight_max": 96.78675079345703,
      "activations/layer4_attention_weight_min": -93.57869720458984,
      "activations/layer5_attention_weight_max": 74.38172912597656,
      "activations/layer5_attention_weight_min": -76.8736572265625,
      "activations/layer6_attention_weight_max": 54.55775451660156,
      "activations/layer6_attention_weight_min": -53.56758117675781,
      "activations/layer7_attention_weight_max": 75.63249206542969,
      "activations/layer7_attention_weight_min": -70.76690673828125,
      "activations/layer8_attention_weight_max": 44.75337600708008,
      "activations/layer8_attention_weight_min": -48.358551025390625,
      "activations/layer9_attention_weight_max": 48.91787338256836,
      "activations/layer9_attention_weight_min": -47.94120407104492,
      "epoch": 11.84,
      "learning_rate": 7.43655303030303e-05,
      "loss": 2.7905,
      "step": 203850
    },
    {
      "activations/layer0_attention_weight_max": 16.64002799987793,
      "activations/layer0_attention_weight_min": -14.341830253601074,
      "activations/layer10_attention_weight_max": 33.98847961425781,
      "activations/layer10_attention_weight_min": -34.36243438720703,
      "activations/layer11_attention_weight_max": 30.630552291870117,
      "activations/layer11_attention_weight_min": -32.70869064331055,
      "activations/layer12_attention_weight_max": 16.952465057373047,
      "activations/layer12_attention_weight_min": -25.61716079711914,
      "activations/layer13_attention_weight_max": 40.21125030517578,
      "activations/layer13_attention_weight_min": -31.152183532714844,
      "activations/layer14_attention_weight_max": 33.5429573059082,
      "activations/layer14_attention_weight_min": -28.07439613342285,
      "activations/layer15_attention_weight_max": 30.353891372680664,
      "activations/layer15_attention_weight_min": -29.503890991210938,
      "activations/layer16_attention_weight_max": 28.7054386138916,
      "activations/layer16_attention_weight_min": -27.52631378173828,
      "activations/layer17_attention_weight_max": 47.86422348022461,
      "activations/layer17_attention_weight_min": -40.991390228271484,
      "activations/layer18_attention_weight_max": 45.21049880981445,
      "activations/layer18_attention_weight_min": -35.367828369140625,
      "activations/layer19_attention_weight_max": 22.02174186706543,
      "activations/layer19_attention_weight_min": -22.547346115112305,
      "activations/layer1_attention_weight_max": 16.75242805480957,
      "activations/layer1_attention_weight_min": -14.45134449005127,
      "activations/layer20_attention_weight_max": 20.70376968383789,
      "activations/layer20_attention_weight_min": -21.244823455810547,
      "activations/layer21_attention_weight_max": 37.54279327392578,
      "activations/layer21_attention_weight_min": -28.098983764648438,
      "activations/layer22_attention_weight_max": 30.26955795288086,
      "activations/layer22_attention_weight_min": -25.76805877685547,
      "activations/layer23_attention_weight_max": 36.65131759643555,
      "activations/layer23_attention_weight_min": -23.92780876159668,
      "activations/layer2_attention_weight_max": 32.18202209472656,
      "activations/layer2_attention_weight_min": -31.729171752929688,
      "activations/layer3_attention_weight_max": 92.85631561279297,
      "activations/layer3_attention_weight_min": -94.18115234375,
      "activations/layer4_attention_weight_max": 97.28651428222656,
      "activations/layer4_attention_weight_min": -93.0896224975586,
      "activations/layer5_attention_weight_max": 70.28167724609375,
      "activations/layer5_attention_weight_min": -76.82731628417969,
      "activations/layer6_attention_weight_max": 50.98236846923828,
      "activations/layer6_attention_weight_min": -50.279903411865234,
      "activations/layer7_attention_weight_max": 70.48188781738281,
      "activations/layer7_attention_weight_min": -68.10999298095703,
      "activations/layer8_attention_weight_max": 42.76353454589844,
      "activations/layer8_attention_weight_min": -46.48250961303711,
      "activations/layer9_attention_weight_max": 45.27997970581055,
      "activations/layer9_attention_weight_min": -46.31060791015625,
      "epoch": 11.85,
      "learning_rate": 7.43465909090909e-05,
      "loss": 2.7914,
      "step": 203900
    },
    {
      "activations/layer0_attention_weight_max": 16.367496490478516,
      "activations/layer0_attention_weight_min": -13.993917465209961,
      "activations/layer10_attention_weight_max": 34.59396743774414,
      "activations/layer10_attention_weight_min": -36.341529846191406,
      "activations/layer11_attention_weight_max": 31.59499740600586,
      "activations/layer11_attention_weight_min": -31.48428726196289,
      "activations/layer12_attention_weight_max": 16.118654251098633,
      "activations/layer12_attention_weight_min": -25.420549392700195,
      "activations/layer13_attention_weight_max": 36.3681755065918,
      "activations/layer13_attention_weight_min": -30.409015655517578,
      "activations/layer14_attention_weight_max": 32.91352462768555,
      "activations/layer14_attention_weight_min": -30.274030685424805,
      "activations/layer15_attention_weight_max": 30.334529876708984,
      "activations/layer15_attention_weight_min": -29.371353149414062,
      "activations/layer16_attention_weight_max": 28.961456298828125,
      "activations/layer16_attention_weight_min": -27.716434478759766,
      "activations/layer17_attention_weight_max": 45.6336555480957,
      "activations/layer17_attention_weight_min": -41.672950744628906,
      "activations/layer18_attention_weight_max": 43.04605484008789,
      "activations/layer18_attention_weight_min": -38.380897521972656,
      "activations/layer19_attention_weight_max": 19.91763687133789,
      "activations/layer19_attention_weight_min": -20.22541046142578,
      "activations/layer1_attention_weight_max": 17.564830780029297,
      "activations/layer1_attention_weight_min": -14.19062614440918,
      "activations/layer20_attention_weight_max": 21.49099349975586,
      "activations/layer20_attention_weight_min": -20.183372497558594,
      "activations/layer21_attention_weight_max": 31.689952850341797,
      "activations/layer21_attention_weight_min": -21.753931045532227,
      "activations/layer22_attention_weight_max": 29.458635330200195,
      "activations/layer22_attention_weight_min": -24.25688362121582,
      "activations/layer23_attention_weight_max": 32.28301239013672,
      "activations/layer23_attention_weight_min": -26.075050354003906,
      "activations/layer2_attention_weight_max": 34.57614517211914,
      "activations/layer2_attention_weight_min": -31.37434959411621,
      "activations/layer3_attention_weight_max": 95.01294708251953,
      "activations/layer3_attention_weight_min": -97.12858581542969,
      "activations/layer4_attention_weight_max": 95.10079193115234,
      "activations/layer4_attention_weight_min": -96.52947235107422,
      "activations/layer5_attention_weight_max": 72.82405853271484,
      "activations/layer5_attention_weight_min": -79.00182342529297,
      "activations/layer6_attention_weight_max": 56.63386154174805,
      "activations/layer6_attention_weight_min": -54.82556915283203,
      "activations/layer7_attention_weight_max": 70.84117889404297,
      "activations/layer7_attention_weight_min": -67.41400146484375,
      "activations/layer8_attention_weight_max": 47.093963623046875,
      "activations/layer8_attention_weight_min": -46.47172164916992,
      "activations/layer9_attention_weight_max": 47.325927734375,
      "activations/layer9_attention_weight_min": -46.99012756347656,
      "epoch": 11.85,
      "learning_rate": 7.432765151515152e-05,
      "loss": 2.775,
      "step": 203950
    },
    {
      "activations/layer0_attention_weight_max": 17.039087295532227,
      "activations/layer0_attention_weight_min": -14.733352661132812,
      "activations/layer10_attention_weight_max": 34.512115478515625,
      "activations/layer10_attention_weight_min": -36.520835876464844,
      "activations/layer11_attention_weight_max": 30.702312469482422,
      "activations/layer11_attention_weight_min": -32.752647399902344,
      "activations/layer12_attention_weight_max": 20.076887130737305,
      "activations/layer12_attention_weight_min": -24.21681022644043,
      "activations/layer13_attention_weight_max": 39.78813552856445,
      "activations/layer13_attention_weight_min": -30.392492294311523,
      "activations/layer14_attention_weight_max": 42.54849624633789,
      "activations/layer14_attention_weight_min": -29.094667434692383,
      "activations/layer15_attention_weight_max": 39.50198745727539,
      "activations/layer15_attention_weight_min": -29.736379623413086,
      "activations/layer16_attention_weight_max": 31.993602752685547,
      "activations/layer16_attention_weight_min": -29.824220657348633,
      "activations/layer17_attention_weight_max": 52.62215042114258,
      "activations/layer17_attention_weight_min": -42.063323974609375,
      "activations/layer18_attention_weight_max": 44.849632263183594,
      "activations/layer18_attention_weight_min": -37.995391845703125,
      "activations/layer19_attention_weight_max": 23.228254318237305,
      "activations/layer19_attention_weight_min": -22.73908233642578,
      "activations/layer1_attention_weight_max": 16.195812225341797,
      "activations/layer1_attention_weight_min": -14.009211540222168,
      "activations/layer20_attention_weight_max": 24.79986572265625,
      "activations/layer20_attention_weight_min": -24.059797286987305,
      "activations/layer21_attention_weight_max": 48.612762451171875,
      "activations/layer21_attention_weight_min": -29.542293548583984,
      "activations/layer22_attention_weight_max": 35.387229919433594,
      "activations/layer22_attention_weight_min": -27.700572967529297,
      "activations/layer23_attention_weight_max": 42.241302490234375,
      "activations/layer23_attention_weight_min": -25.38440704345703,
      "activations/layer2_attention_weight_max": 34.001731872558594,
      "activations/layer2_attention_weight_min": -32.335269927978516,
      "activations/layer3_attention_weight_max": 92.9805908203125,
      "activations/layer3_attention_weight_min": -99.38719177246094,
      "activations/layer4_attention_weight_max": 95.8893051147461,
      "activations/layer4_attention_weight_min": -91.5911865234375,
      "activations/layer5_attention_weight_max": 72.39252471923828,
      "activations/layer5_attention_weight_min": -74.5353012084961,
      "activations/layer6_attention_weight_max": 51.899879455566406,
      "activations/layer6_attention_weight_min": -50.01632308959961,
      "activations/layer7_attention_weight_max": 70.25801849365234,
      "activations/layer7_attention_weight_min": -67.48933410644531,
      "activations/layer8_attention_weight_max": 45.837127685546875,
      "activations/layer8_attention_weight_min": -47.6483039855957,
      "activations/layer9_attention_weight_max": 45.84366989135742,
      "activations/layer9_attention_weight_min": -47.27766036987305,
      "epoch": 11.85,
      "learning_rate": 7.430871212121212e-05,
      "loss": 2.7714,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.4908,
      "eval_samples_per_second": 505.721,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.4908,
      "eval_openwebtext_samples_per_second": 505.721,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0308,
      "eval_wikitext_samples_per_second": 224.544,
      "step": 204000
    },
    {
      "epoch": 11.85,
      "eval_lambada_loss": 2.521484375,
      "eval_lambada_ppl": 12.447059060920614,
      "eval_lambada_runtime": 9.6355,
      "eval_lambada_samples_per_second": 505.319,
      "step": 204000
    },
    {
      "activations/layer0_attention_weight_max": 15.739999771118164,
      "activations/layer0_attention_weight_min": -14.18725872039795,
      "activations/layer10_attention_weight_max": 33.398773193359375,
      "activations/layer10_attention_weight_min": -33.959381103515625,
      "activations/layer11_attention_weight_max": 30.426969528198242,
      "activations/layer11_attention_weight_min": -30.84425926208496,
      "activations/layer12_attention_weight_max": 18.650495529174805,
      "activations/layer12_attention_weight_min": -22.89517593383789,
      "activations/layer13_attention_weight_max": 34.95522689819336,
      "activations/layer13_attention_weight_min": -27.99747085571289,
      "activations/layer14_attention_weight_max": 37.15901184082031,
      "activations/layer14_attention_weight_min": -30.55780601501465,
      "activations/layer15_attention_weight_max": 33.21651077270508,
      "activations/layer15_attention_weight_min": -29.040496826171875,
      "activations/layer16_attention_weight_max": 31.293590545654297,
      "activations/layer16_attention_weight_min": -33.1530647277832,
      "activations/layer17_attention_weight_max": 50.95219039916992,
      "activations/layer17_attention_weight_min": -46.40058517456055,
      "activations/layer18_attention_weight_max": 48.760189056396484,
      "activations/layer18_attention_weight_min": -38.07842254638672,
      "activations/layer19_attention_weight_max": 24.77949333190918,
      "activations/layer19_attention_weight_min": -22.89522933959961,
      "activations/layer1_attention_weight_max": 16.57155990600586,
      "activations/layer1_attention_weight_min": -13.687932968139648,
      "activations/layer20_attention_weight_max": 24.166908264160156,
      "activations/layer20_attention_weight_min": -21.087875366210938,
      "activations/layer21_attention_weight_max": 40.6928825378418,
      "activations/layer21_attention_weight_min": -22.752540588378906,
      "activations/layer22_attention_weight_max": 30.731525421142578,
      "activations/layer22_attention_weight_min": -28.0159912109375,
      "activations/layer23_attention_weight_max": 34.24797058105469,
      "activations/layer23_attention_weight_min": -23.471647262573242,
      "activations/layer2_attention_weight_max": 30.578311920166016,
      "activations/layer2_attention_weight_min": -30.306915283203125,
      "activations/layer3_attention_weight_max": 91.41728973388672,
      "activations/layer3_attention_weight_min": -91.93666076660156,
      "activations/layer4_attention_weight_max": 93.60054779052734,
      "activations/layer4_attention_weight_min": -92.54840087890625,
      "activations/layer5_attention_weight_max": 72.99295043945312,
      "activations/layer5_attention_weight_min": -75.86731719970703,
      "activations/layer6_attention_weight_max": 51.47085952758789,
      "activations/layer6_attention_weight_min": -51.787315368652344,
      "activations/layer7_attention_weight_max": 69.86114501953125,
      "activations/layer7_attention_weight_min": -69.849609375,
      "activations/layer8_attention_weight_max": 42.46870803833008,
      "activations/layer8_attention_weight_min": -45.26543045043945,
      "activations/layer9_attention_weight_max": 45.7181396484375,
      "activations/layer9_attention_weight_min": -46.770606994628906,
      "epoch": 11.86,
      "learning_rate": 7.428977272727272e-05,
      "loss": 2.772,
      "step": 204050
    },
    {
      "activations/layer0_attention_weight_max": 15.585417747497559,
      "activations/layer0_attention_weight_min": -14.381492614746094,
      "activations/layer10_attention_weight_max": 35.970333099365234,
      "activations/layer10_attention_weight_min": -35.552276611328125,
      "activations/layer11_attention_weight_max": 33.86652374267578,
      "activations/layer11_attention_weight_min": -32.418373107910156,
      "activations/layer12_attention_weight_max": 22.45431900024414,
      "activations/layer12_attention_weight_min": -25.40377426147461,
      "activations/layer13_attention_weight_max": 49.75867462158203,
      "activations/layer13_attention_weight_min": -28.28624153137207,
      "activations/layer14_attention_weight_max": 45.18104934692383,
      "activations/layer14_attention_weight_min": -27.96639060974121,
      "activations/layer15_attention_weight_max": 33.748966217041016,
      "activations/layer15_attention_weight_min": -27.069746017456055,
      "activations/layer16_attention_weight_max": 29.95245933532715,
      "activations/layer16_attention_weight_min": -28.48729133605957,
      "activations/layer17_attention_weight_max": 52.317874908447266,
      "activations/layer17_attention_weight_min": -41.4208869934082,
      "activations/layer18_attention_weight_max": 42.853736877441406,
      "activations/layer18_attention_weight_min": -36.480621337890625,
      "activations/layer19_attention_weight_max": 22.12953758239746,
      "activations/layer19_attention_weight_min": -21.641115188598633,
      "activations/layer1_attention_weight_max": 16.373151779174805,
      "activations/layer1_attention_weight_min": -13.992356300354004,
      "activations/layer20_attention_weight_max": 23.041744232177734,
      "activations/layer20_attention_weight_min": -21.837244033813477,
      "activations/layer21_attention_weight_max": 39.83517074584961,
      "activations/layer21_attention_weight_min": -22.64630889892578,
      "activations/layer22_attention_weight_max": 29.87564468383789,
      "activations/layer22_attention_weight_min": -24.76892852783203,
      "activations/layer23_attention_weight_max": 35.158477783203125,
      "activations/layer23_attention_weight_min": -26.902679443359375,
      "activations/layer2_attention_weight_max": 35.05785369873047,
      "activations/layer2_attention_weight_min": -31.769855499267578,
      "activations/layer3_attention_weight_max": 99.48992156982422,
      "activations/layer3_attention_weight_min": -97.54808044433594,
      "activations/layer4_attention_weight_max": 98.28606414794922,
      "activations/layer4_attention_weight_min": -94.74938201904297,
      "activations/layer5_attention_weight_max": 72.00534057617188,
      "activations/layer5_attention_weight_min": -74.06719970703125,
      "activations/layer6_attention_weight_max": 54.52092361450195,
      "activations/layer6_attention_weight_min": -52.953819274902344,
      "activations/layer7_attention_weight_max": 75.89707946777344,
      "activations/layer7_attention_weight_min": -72.02558898925781,
      "activations/layer8_attention_weight_max": 44.9178352355957,
      "activations/layer8_attention_weight_min": -45.84912872314453,
      "activations/layer9_attention_weight_max": 54.6307487487793,
      "activations/layer9_attention_weight_min": -46.162139892578125,
      "epoch": 11.86,
      "learning_rate": 7.427083333333333e-05,
      "loss": 2.7702,
      "step": 204100
    },
    {
      "activations/layer0_attention_weight_max": 15.11900520324707,
      "activations/layer0_attention_weight_min": -14.372873306274414,
      "activations/layer10_attention_weight_max": 36.68476104736328,
      "activations/layer10_attention_weight_min": -37.352901458740234,
      "activations/layer11_attention_weight_max": 35.99018478393555,
      "activations/layer11_attention_weight_min": -33.37640380859375,
      "activations/layer12_attention_weight_max": 25.43567657470703,
      "activations/layer12_attention_weight_min": -22.403156280517578,
      "activations/layer13_attention_weight_max": 41.02272415161133,
      "activations/layer13_attention_weight_min": -28.703838348388672,
      "activations/layer14_attention_weight_max": 37.468997955322266,
      "activations/layer14_attention_weight_min": -29.162443161010742,
      "activations/layer15_attention_weight_max": 34.23531723022461,
      "activations/layer15_attention_weight_min": -28.334346771240234,
      "activations/layer16_attention_weight_max": 33.5030517578125,
      "activations/layer16_attention_weight_min": -28.742082595825195,
      "activations/layer17_attention_weight_max": 52.4731559753418,
      "activations/layer17_attention_weight_min": -41.85334014892578,
      "activations/layer18_attention_weight_max": 47.01823043823242,
      "activations/layer18_attention_weight_min": -38.08400344848633,
      "activations/layer19_attention_weight_max": 25.550249099731445,
      "activations/layer19_attention_weight_min": -21.924283981323242,
      "activations/layer1_attention_weight_max": 17.162094116210938,
      "activations/layer1_attention_weight_min": -15.333264350891113,
      "activations/layer20_attention_weight_max": 24.484312057495117,
      "activations/layer20_attention_weight_min": -22.463909149169922,
      "activations/layer21_attention_weight_max": 37.28483963012695,
      "activations/layer21_attention_weight_min": -23.624126434326172,
      "activations/layer22_attention_weight_max": 35.057701110839844,
      "activations/layer22_attention_weight_min": -24.44152069091797,
      "activations/layer23_attention_weight_max": 37.44976806640625,
      "activations/layer23_attention_weight_min": -27.884374618530273,
      "activations/layer2_attention_weight_max": 33.71728515625,
      "activations/layer2_attention_weight_min": -33.04896545410156,
      "activations/layer3_attention_weight_max": 95.31394958496094,
      "activations/layer3_attention_weight_min": -94.83770751953125,
      "activations/layer4_attention_weight_max": 105.31365966796875,
      "activations/layer4_attention_weight_min": -97.81038665771484,
      "activations/layer5_attention_weight_max": 74.21437072753906,
      "activations/layer5_attention_weight_min": -81.74996948242188,
      "activations/layer6_attention_weight_max": 55.710723876953125,
      "activations/layer6_attention_weight_min": -53.18003845214844,
      "activations/layer7_attention_weight_max": 77.32504272460938,
      "activations/layer7_attention_weight_min": -72.37651062011719,
      "activations/layer8_attention_weight_max": 47.16411590576172,
      "activations/layer8_attention_weight_min": -46.99872970581055,
      "activations/layer9_attention_weight_max": 49.97944641113281,
      "activations/layer9_attention_weight_min": -47.89372253417969,
      "epoch": 11.86,
      "learning_rate": 7.425189393939394e-05,
      "loss": 2.789,
      "step": 204150
    },
    {
      "activations/layer0_attention_weight_max": 15.908924102783203,
      "activations/layer0_attention_weight_min": -14.480413436889648,
      "activations/layer10_attention_weight_max": 36.534278869628906,
      "activations/layer10_attention_weight_min": -35.72898864746094,
      "activations/layer11_attention_weight_max": 33.25655746459961,
      "activations/layer11_attention_weight_min": -34.26048278808594,
      "activations/layer12_attention_weight_max": 17.024179458618164,
      "activations/layer12_attention_weight_min": -24.06298065185547,
      "activations/layer13_attention_weight_max": 36.67784118652344,
      "activations/layer13_attention_weight_min": -28.138282775878906,
      "activations/layer14_attention_weight_max": 36.447078704833984,
      "activations/layer14_attention_weight_min": -31.207324981689453,
      "activations/layer15_attention_weight_max": 32.1386604309082,
      "activations/layer15_attention_weight_min": -28.936561584472656,
      "activations/layer16_attention_weight_max": 31.455720901489258,
      "activations/layer16_attention_weight_min": -31.527576446533203,
      "activations/layer17_attention_weight_max": 51.549861907958984,
      "activations/layer17_attention_weight_min": -46.293006896972656,
      "activations/layer18_attention_weight_max": 44.13337707519531,
      "activations/layer18_attention_weight_min": -37.399131774902344,
      "activations/layer19_attention_weight_max": 24.6044921875,
      "activations/layer19_attention_weight_min": -23.049516677856445,
      "activations/layer1_attention_weight_max": 16.016164779663086,
      "activations/layer1_attention_weight_min": -15.562604904174805,
      "activations/layer20_attention_weight_max": 21.306264877319336,
      "activations/layer20_attention_weight_min": -21.91658592224121,
      "activations/layer21_attention_weight_max": 35.72964096069336,
      "activations/layer21_attention_weight_min": -22.600948333740234,
      "activations/layer22_attention_weight_max": 30.171316146850586,
      "activations/layer22_attention_weight_min": -25.218416213989258,
      "activations/layer23_attention_weight_max": 39.87966537475586,
      "activations/layer23_attention_weight_min": -25.982297897338867,
      "activations/layer2_attention_weight_max": 33.14067840576172,
      "activations/layer2_attention_weight_min": -31.446989059448242,
      "activations/layer3_attention_weight_max": 93.47047424316406,
      "activations/layer3_attention_weight_min": -94.29808807373047,
      "activations/layer4_attention_weight_max": 98.20054626464844,
      "activations/layer4_attention_weight_min": -94.19161987304688,
      "activations/layer5_attention_weight_max": 71.81632232666016,
      "activations/layer5_attention_weight_min": -79.17481994628906,
      "activations/layer6_attention_weight_max": 54.050453186035156,
      "activations/layer6_attention_weight_min": -51.001976013183594,
      "activations/layer7_attention_weight_max": 72.211669921875,
      "activations/layer7_attention_weight_min": -69.80233001708984,
      "activations/layer8_attention_weight_max": 44.5141716003418,
      "activations/layer8_attention_weight_min": -48.56538391113281,
      "activations/layer9_attention_weight_max": 45.26833724975586,
      "activations/layer9_attention_weight_min": -48.61438751220703,
      "epoch": 11.87,
      "learning_rate": 7.423295454545454e-05,
      "loss": 2.7826,
      "step": 204200
    },
    {
      "activations/layer0_attention_weight_max": 16.224853515625,
      "activations/layer0_attention_weight_min": -13.928871154785156,
      "activations/layer10_attention_weight_max": 35.40919494628906,
      "activations/layer10_attention_weight_min": -34.48909378051758,
      "activations/layer11_attention_weight_max": 29.661556243896484,
      "activations/layer11_attention_weight_min": -32.0871696472168,
      "activations/layer12_attention_weight_max": 18.979206085205078,
      "activations/layer12_attention_weight_min": -23.262351989746094,
      "activations/layer13_attention_weight_max": 35.062530517578125,
      "activations/layer13_attention_weight_min": -30.76445960998535,
      "activations/layer14_attention_weight_max": 33.352298736572266,
      "activations/layer14_attention_weight_min": -28.94022560119629,
      "activations/layer15_attention_weight_max": 30.498109817504883,
      "activations/layer15_attention_weight_min": -28.862945556640625,
      "activations/layer16_attention_weight_max": 29.363988876342773,
      "activations/layer16_attention_weight_min": -28.954410552978516,
      "activations/layer17_attention_weight_max": 48.76930618286133,
      "activations/layer17_attention_weight_min": -42.19197082519531,
      "activations/layer18_attention_weight_max": 44.81163787841797,
      "activations/layer18_attention_weight_min": -36.68451690673828,
      "activations/layer19_attention_weight_max": 24.47675895690918,
      "activations/layer19_attention_weight_min": -21.744176864624023,
      "activations/layer1_attention_weight_max": 16.56477928161621,
      "activations/layer1_attention_weight_min": -14.444535255432129,
      "activations/layer20_attention_weight_max": 22.242088317871094,
      "activations/layer20_attention_weight_min": -22.32009506225586,
      "activations/layer21_attention_weight_max": 33.72860336303711,
      "activations/layer21_attention_weight_min": -21.55111312866211,
      "activations/layer22_attention_weight_max": 32.4180793762207,
      "activations/layer22_attention_weight_min": -25.268457412719727,
      "activations/layer23_attention_weight_max": 38.70286560058594,
      "activations/layer23_attention_weight_min": -29.102195739746094,
      "activations/layer2_attention_weight_max": 32.11358642578125,
      "activations/layer2_attention_weight_min": -31.210561752319336,
      "activations/layer3_attention_weight_max": 95.04740142822266,
      "activations/layer3_attention_weight_min": -95.9979476928711,
      "activations/layer4_attention_weight_max": 98.25988006591797,
      "activations/layer4_attention_weight_min": -95.63265991210938,
      "activations/layer5_attention_weight_max": 69.02435302734375,
      "activations/layer5_attention_weight_min": -79.19556427001953,
      "activations/layer6_attention_weight_max": 54.514522552490234,
      "activations/layer6_attention_weight_min": -52.59687805175781,
      "activations/layer7_attention_weight_max": 76.96672821044922,
      "activations/layer7_attention_weight_min": -77.84072875976562,
      "activations/layer8_attention_weight_max": 43.24616622924805,
      "activations/layer8_attention_weight_min": -48.89850616455078,
      "activations/layer9_attention_weight_max": 46.194190979003906,
      "activations/layer9_attention_weight_min": -46.42866134643555,
      "epoch": 11.87,
      "learning_rate": 7.421401515151514e-05,
      "loss": 2.7571,
      "step": 204250
    },
    {
      "activations/layer0_attention_weight_max": 16.069416046142578,
      "activations/layer0_attention_weight_min": -13.950688362121582,
      "activations/layer10_attention_weight_max": 35.69374465942383,
      "activations/layer10_attention_weight_min": -36.737709045410156,
      "activations/layer11_attention_weight_max": 32.087310791015625,
      "activations/layer11_attention_weight_min": -32.77227020263672,
      "activations/layer12_attention_weight_max": 17.959875106811523,
      "activations/layer12_attention_weight_min": -25.817659378051758,
      "activations/layer13_attention_weight_max": 35.08147048950195,
      "activations/layer13_attention_weight_min": -29.755443572998047,
      "activations/layer14_attention_weight_max": 36.77787780761719,
      "activations/layer14_attention_weight_min": -32.0122184753418,
      "activations/layer15_attention_weight_max": 34.691795349121094,
      "activations/layer15_attention_weight_min": -28.66900634765625,
      "activations/layer16_attention_weight_max": 32.30221939086914,
      "activations/layer16_attention_weight_min": -30.485322952270508,
      "activations/layer17_attention_weight_max": 54.37359619140625,
      "activations/layer17_attention_weight_min": -48.04936599731445,
      "activations/layer18_attention_weight_max": 48.08088302612305,
      "activations/layer18_attention_weight_min": -38.36797332763672,
      "activations/layer19_attention_weight_max": 20.078847885131836,
      "activations/layer19_attention_weight_min": -21.74555015563965,
      "activations/layer1_attention_weight_max": 16.613622665405273,
      "activations/layer1_attention_weight_min": -13.766314506530762,
      "activations/layer20_attention_weight_max": 22.93817710876465,
      "activations/layer20_attention_weight_min": -20.744112014770508,
      "activations/layer21_attention_weight_max": 36.35191345214844,
      "activations/layer21_attention_weight_min": -22.426862716674805,
      "activations/layer22_attention_weight_max": 30.959518432617188,
      "activations/layer22_attention_weight_min": -26.252758026123047,
      "activations/layer23_attention_weight_max": 40.492919921875,
      "activations/layer23_attention_weight_min": -22.127246856689453,
      "activations/layer2_attention_weight_max": 33.35533142089844,
      "activations/layer2_attention_weight_min": -30.98361587524414,
      "activations/layer3_attention_weight_max": 99.67120361328125,
      "activations/layer3_attention_weight_min": -95.9144058227539,
      "activations/layer4_attention_weight_max": 95.60680389404297,
      "activations/layer4_attention_weight_min": -91.95610809326172,
      "activations/layer5_attention_weight_max": 73.98960876464844,
      "activations/layer5_attention_weight_min": -75.50970458984375,
      "activations/layer6_attention_weight_max": 51.89338684082031,
      "activations/layer6_attention_weight_min": -49.44930648803711,
      "activations/layer7_attention_weight_max": 71.53108978271484,
      "activations/layer7_attention_weight_min": -72.19709014892578,
      "activations/layer8_attention_weight_max": 44.42533874511719,
      "activations/layer8_attention_weight_min": -46.5063362121582,
      "activations/layer9_attention_weight_max": 46.510807037353516,
      "activations/layer9_attention_weight_min": -48.69898223876953,
      "epoch": 11.87,
      "learning_rate": 7.419507575757575e-05,
      "loss": 2.773,
      "step": 204300
    },
    {
      "activations/layer0_attention_weight_max": 16.96941566467285,
      "activations/layer0_attention_weight_min": -14.095210075378418,
      "activations/layer10_attention_weight_max": 39.93138122558594,
      "activations/layer10_attention_weight_min": -37.09933090209961,
      "activations/layer11_attention_weight_max": 34.8842658996582,
      "activations/layer11_attention_weight_min": -35.582149505615234,
      "activations/layer12_attention_weight_max": 19.49983787536621,
      "activations/layer12_attention_weight_min": -27.961023330688477,
      "activations/layer13_attention_weight_max": 49.75009536743164,
      "activations/layer13_attention_weight_min": -34.12232208251953,
      "activations/layer14_attention_weight_max": 36.51445388793945,
      "activations/layer14_attention_weight_min": -30.853607177734375,
      "activations/layer15_attention_weight_max": 35.01255798339844,
      "activations/layer15_attention_weight_min": -30.96221351623535,
      "activations/layer16_attention_weight_max": 30.74884033203125,
      "activations/layer16_attention_weight_min": -29.00241470336914,
      "activations/layer17_attention_weight_max": 54.10014724731445,
      "activations/layer17_attention_weight_min": -43.5449104309082,
      "activations/layer18_attention_weight_max": 47.03455352783203,
      "activations/layer18_attention_weight_min": -39.0359001159668,
      "activations/layer19_attention_weight_max": 25.351900100708008,
      "activations/layer19_attention_weight_min": -21.361717224121094,
      "activations/layer1_attention_weight_max": 16.199546813964844,
      "activations/layer1_attention_weight_min": -14.932806968688965,
      "activations/layer20_attention_weight_max": 24.917861938476562,
      "activations/layer20_attention_weight_min": -22.064889907836914,
      "activations/layer21_attention_weight_max": 37.51227569580078,
      "activations/layer21_attention_weight_min": -23.325542449951172,
      "activations/layer22_attention_weight_max": 29.27134132385254,
      "activations/layer22_attention_weight_min": -23.383098602294922,
      "activations/layer23_attention_weight_max": 37.827789306640625,
      "activations/layer23_attention_weight_min": -22.592025756835938,
      "activations/layer2_attention_weight_max": 34.27546310424805,
      "activations/layer2_attention_weight_min": -32.99853515625,
      "activations/layer3_attention_weight_max": 94.57553100585938,
      "activations/layer3_attention_weight_min": -96.91133117675781,
      "activations/layer4_attention_weight_max": 97.243408203125,
      "activations/layer4_attention_weight_min": -94.85350799560547,
      "activations/layer5_attention_weight_max": 71.679931640625,
      "activations/layer5_attention_weight_min": -78.16636657714844,
      "activations/layer6_attention_weight_max": 51.94222640991211,
      "activations/layer6_attention_weight_min": -52.52334976196289,
      "activations/layer7_attention_weight_max": 75.62540435791016,
      "activations/layer7_attention_weight_min": -70.27532958984375,
      "activations/layer8_attention_weight_max": 44.07147979736328,
      "activations/layer8_attention_weight_min": -45.31687545776367,
      "activations/layer9_attention_weight_max": 48.45792770385742,
      "activations/layer9_attention_weight_min": -46.74619674682617,
      "epoch": 11.87,
      "learning_rate": 7.417613636363635e-05,
      "loss": 2.7717,
      "step": 204350
    },
    {
      "activations/layer0_attention_weight_max": 15.700820922851562,
      "activations/layer0_attention_weight_min": -14.327957153320312,
      "activations/layer10_attention_weight_max": 35.156005859375,
      "activations/layer10_attention_weight_min": -36.51115417480469,
      "activations/layer11_attention_weight_max": 30.981647491455078,
      "activations/layer11_attention_weight_min": -33.28020477294922,
      "activations/layer12_attention_weight_max": 20.043046951293945,
      "activations/layer12_attention_weight_min": -24.681852340698242,
      "activations/layer13_attention_weight_max": 34.80596160888672,
      "activations/layer13_attention_weight_min": -29.021656036376953,
      "activations/layer14_attention_weight_max": 35.44084167480469,
      "activations/layer14_attention_weight_min": -30.2447509765625,
      "activations/layer15_attention_weight_max": 33.679344177246094,
      "activations/layer15_attention_weight_min": -29.124570846557617,
      "activations/layer16_attention_weight_max": 30.4674072265625,
      "activations/layer16_attention_weight_min": -31.31725311279297,
      "activations/layer17_attention_weight_max": 51.62420654296875,
      "activations/layer17_attention_weight_min": -43.89296340942383,
      "activations/layer18_attention_weight_max": 48.48100280761719,
      "activations/layer18_attention_weight_min": -38.902462005615234,
      "activations/layer19_attention_weight_max": 22.889812469482422,
      "activations/layer19_attention_weight_min": -21.601713180541992,
      "activations/layer1_attention_weight_max": 18.690784454345703,
      "activations/layer1_attention_weight_min": -14.462326049804688,
      "activations/layer20_attention_weight_max": 21.841089248657227,
      "activations/layer20_attention_weight_min": -21.571619033813477,
      "activations/layer21_attention_weight_max": 34.169986724853516,
      "activations/layer21_attention_weight_min": -20.4948673248291,
      "activations/layer22_attention_weight_max": 29.47459602355957,
      "activations/layer22_attention_weight_min": -25.915687561035156,
      "activations/layer23_attention_weight_max": 35.802513122558594,
      "activations/layer23_attention_weight_min": -24.658803939819336,
      "activations/layer2_attention_weight_max": 33.388099670410156,
      "activations/layer2_attention_weight_min": -32.14217758178711,
      "activations/layer3_attention_weight_max": 94.98572540283203,
      "activations/layer3_attention_weight_min": -96.89122009277344,
      "activations/layer4_attention_weight_max": 96.02747344970703,
      "activations/layer4_attention_weight_min": -94.15581512451172,
      "activations/layer5_attention_weight_max": 70.15082550048828,
      "activations/layer5_attention_weight_min": -81.63825988769531,
      "activations/layer6_attention_weight_max": 53.08774948120117,
      "activations/layer6_attention_weight_min": -52.22578048706055,
      "activations/layer7_attention_weight_max": 72.7745132446289,
      "activations/layer7_attention_weight_min": -72.27081298828125,
      "activations/layer8_attention_weight_max": 45.36943054199219,
      "activations/layer8_attention_weight_min": -47.02435302734375,
      "activations/layer9_attention_weight_max": 50.05377960205078,
      "activations/layer9_attention_weight_min": -47.387794494628906,
      "epoch": 11.88,
      "learning_rate": 7.415719696969696e-05,
      "loss": 2.7679,
      "step": 204400
    },
    {
      "activations/layer0_attention_weight_max": 15.809049606323242,
      "activations/layer0_attention_weight_min": -13.913250923156738,
      "activations/layer10_attention_weight_max": 37.94382858276367,
      "activations/layer10_attention_weight_min": -36.148170471191406,
      "activations/layer11_attention_weight_max": 33.761314392089844,
      "activations/layer11_attention_weight_min": -33.65801239013672,
      "activations/layer12_attention_weight_max": 18.032346725463867,
      "activations/layer12_attention_weight_min": -27.237314224243164,
      "activations/layer13_attention_weight_max": 46.63288116455078,
      "activations/layer13_attention_weight_min": -30.59540557861328,
      "activations/layer14_attention_weight_max": 35.641658782958984,
      "activations/layer14_attention_weight_min": -30.157733917236328,
      "activations/layer15_attention_weight_max": 32.444435119628906,
      "activations/layer15_attention_weight_min": -27.769248962402344,
      "activations/layer16_attention_weight_max": 30.78166389465332,
      "activations/layer16_attention_weight_min": -28.613691329956055,
      "activations/layer17_attention_weight_max": 51.55381774902344,
      "activations/layer17_attention_weight_min": -41.860904693603516,
      "activations/layer18_attention_weight_max": 46.80303955078125,
      "activations/layer18_attention_weight_min": -37.74480438232422,
      "activations/layer19_attention_weight_max": 22.46274185180664,
      "activations/layer19_attention_weight_min": -21.433927536010742,
      "activations/layer1_attention_weight_max": 16.121234893798828,
      "activations/layer1_attention_weight_min": -15.181584358215332,
      "activations/layer20_attention_weight_max": 23.831798553466797,
      "activations/layer20_attention_weight_min": -20.83854103088379,
      "activations/layer21_attention_weight_max": 40.93814468383789,
      "activations/layer21_attention_weight_min": -20.973621368408203,
      "activations/layer22_attention_weight_max": 28.621702194213867,
      "activations/layer22_attention_weight_min": -25.196802139282227,
      "activations/layer23_attention_weight_max": 38.46223068237305,
      "activations/layer23_attention_weight_min": -23.71677017211914,
      "activations/layer2_attention_weight_max": 32.54358673095703,
      "activations/layer2_attention_weight_min": -31.20839500427246,
      "activations/layer3_attention_weight_max": 101.47019958496094,
      "activations/layer3_attention_weight_min": -104.52420043945312,
      "activations/layer4_attention_weight_max": 99.3067398071289,
      "activations/layer4_attention_weight_min": -100.17294311523438,
      "activations/layer5_attention_weight_max": 77.17668151855469,
      "activations/layer5_attention_weight_min": -78.60441589355469,
      "activations/layer6_attention_weight_max": 54.969215393066406,
      "activations/layer6_attention_weight_min": -52.03077697753906,
      "activations/layer7_attention_weight_max": 73.25852966308594,
      "activations/layer7_attention_weight_min": -71.72992706298828,
      "activations/layer8_attention_weight_max": 45.09941101074219,
      "activations/layer8_attention_weight_min": -48.61283874511719,
      "activations/layer9_attention_weight_max": 46.5656623840332,
      "activations/layer9_attention_weight_min": -46.8966178894043,
      "epoch": 11.88,
      "learning_rate": 7.413825757575757e-05,
      "loss": 2.7795,
      "step": 204450
    },
    {
      "activations/layer0_attention_weight_max": 16.59427261352539,
      "activations/layer0_attention_weight_min": -14.664369583129883,
      "activations/layer10_attention_weight_max": 36.70841598510742,
      "activations/layer10_attention_weight_min": -37.32761001586914,
      "activations/layer11_attention_weight_max": 31.75105857849121,
      "activations/layer11_attention_weight_min": -33.42145538330078,
      "activations/layer12_attention_weight_max": 17.9405574798584,
      "activations/layer12_attention_weight_min": -24.53822135925293,
      "activations/layer13_attention_weight_max": 55.37065505981445,
      "activations/layer13_attention_weight_min": -36.30549621582031,
      "activations/layer14_attention_weight_max": 39.07869338989258,
      "activations/layer14_attention_weight_min": -31.027360916137695,
      "activations/layer15_attention_weight_max": 35.347625732421875,
      "activations/layer15_attention_weight_min": -30.395240783691406,
      "activations/layer16_attention_weight_max": 30.36717414855957,
      "activations/layer16_attention_weight_min": -29.32529640197754,
      "activations/layer17_attention_weight_max": 52.14491653442383,
      "activations/layer17_attention_weight_min": -46.169227600097656,
      "activations/layer18_attention_weight_max": 47.95511245727539,
      "activations/layer18_attention_weight_min": -37.16255187988281,
      "activations/layer19_attention_weight_max": 23.890539169311523,
      "activations/layer19_attention_weight_min": -23.12763023376465,
      "activations/layer1_attention_weight_max": 17.716466903686523,
      "activations/layer1_attention_weight_min": -13.858136177062988,
      "activations/layer20_attention_weight_max": 23.564170837402344,
      "activations/layer20_attention_weight_min": -21.55722427368164,
      "activations/layer21_attention_weight_max": 39.52376174926758,
      "activations/layer21_attention_weight_min": -23.407878875732422,
      "activations/layer22_attention_weight_max": 29.600753784179688,
      "activations/layer22_attention_weight_min": -24.62165641784668,
      "activations/layer23_attention_weight_max": 34.70176696777344,
      "activations/layer23_attention_weight_min": -23.527549743652344,
      "activations/layer2_attention_weight_max": 33.27457046508789,
      "activations/layer2_attention_weight_min": -31.615352630615234,
      "activations/layer3_attention_weight_max": 100.70967102050781,
      "activations/layer3_attention_weight_min": -102.63117980957031,
      "activations/layer4_attention_weight_max": 100.91832733154297,
      "activations/layer4_attention_weight_min": -99.30062103271484,
      "activations/layer5_attention_weight_max": 73.0430679321289,
      "activations/layer5_attention_weight_min": -80.83140563964844,
      "activations/layer6_attention_weight_max": 52.615116119384766,
      "activations/layer6_attention_weight_min": -52.85124969482422,
      "activations/layer7_attention_weight_max": 70.61893463134766,
      "activations/layer7_attention_weight_min": -67.5462646484375,
      "activations/layer8_attention_weight_max": 43.71917724609375,
      "activations/layer8_attention_weight_min": -47.4415168762207,
      "activations/layer9_attention_weight_max": 45.28937911987305,
      "activations/layer9_attention_weight_min": -46.53205871582031,
      "epoch": 11.88,
      "learning_rate": 7.411931818181817e-05,
      "loss": 2.7805,
      "step": 204500
    },
    {
      "activations/layer0_attention_weight_max": 15.722784042358398,
      "activations/layer0_attention_weight_min": -14.135589599609375,
      "activations/layer10_attention_weight_max": 36.29384231567383,
      "activations/layer10_attention_weight_min": -35.44737243652344,
      "activations/layer11_attention_weight_max": 32.5511474609375,
      "activations/layer11_attention_weight_min": -33.22933578491211,
      "activations/layer12_attention_weight_max": 20.365434646606445,
      "activations/layer12_attention_weight_min": -27.666034698486328,
      "activations/layer13_attention_weight_max": 49.400535583496094,
      "activations/layer13_attention_weight_min": -34.44180679321289,
      "activations/layer14_attention_weight_max": 51.103179931640625,
      "activations/layer14_attention_weight_min": -32.75825881958008,
      "activations/layer15_attention_weight_max": 40.387393951416016,
      "activations/layer15_attention_weight_min": -29.404735565185547,
      "activations/layer16_attention_weight_max": 35.63886260986328,
      "activations/layer16_attention_weight_min": -31.381181716918945,
      "activations/layer17_attention_weight_max": 57.11545944213867,
      "activations/layer17_attention_weight_min": -45.26887512207031,
      "activations/layer18_attention_weight_max": 50.74338150024414,
      "activations/layer18_attention_weight_min": -37.95235824584961,
      "activations/layer19_attention_weight_max": 24.134597778320312,
      "activations/layer19_attention_weight_min": -21.99822998046875,
      "activations/layer1_attention_weight_max": 16.79803466796875,
      "activations/layer1_attention_weight_min": -14.566200256347656,
      "activations/layer20_attention_weight_max": 25.68756675720215,
      "activations/layer20_attention_weight_min": -22.65581512451172,
      "activations/layer21_attention_weight_max": 44.77342987060547,
      "activations/layer21_attention_weight_min": -24.254112243652344,
      "activations/layer22_attention_weight_max": 35.70012283325195,
      "activations/layer22_attention_weight_min": -25.68606948852539,
      "activations/layer23_attention_weight_max": 46.18025207519531,
      "activations/layer23_attention_weight_min": -26.195228576660156,
      "activations/layer2_attention_weight_max": 33.8685302734375,
      "activations/layer2_attention_weight_min": -33.42849349975586,
      "activations/layer3_attention_weight_max": 97.47944641113281,
      "activations/layer3_attention_weight_min": -103.96382141113281,
      "activations/layer4_attention_weight_max": 105.156494140625,
      "activations/layer4_attention_weight_min": -99.57954406738281,
      "activations/layer5_attention_weight_max": 72.40637969970703,
      "activations/layer5_attention_weight_min": -76.01849365234375,
      "activations/layer6_attention_weight_max": 53.75278091430664,
      "activations/layer6_attention_weight_min": -52.87132263183594,
      "activations/layer7_attention_weight_max": 73.58209228515625,
      "activations/layer7_attention_weight_min": -68.14292907714844,
      "activations/layer8_attention_weight_max": 46.34255599975586,
      "activations/layer8_attention_weight_min": -47.335411071777344,
      "activations/layer9_attention_weight_max": 48.55156707763672,
      "activations/layer9_attention_weight_min": -46.969051361083984,
      "epoch": 11.89,
      "learning_rate": 7.410037878787877e-05,
      "loss": 2.779,
      "step": 204550
    },
    {
      "activations/layer0_attention_weight_max": 15.4218111038208,
      "activations/layer0_attention_weight_min": -14.156062126159668,
      "activations/layer10_attention_weight_max": 37.805641174316406,
      "activations/layer10_attention_weight_min": -37.06451416015625,
      "activations/layer11_attention_weight_max": 32.683650970458984,
      "activations/layer11_attention_weight_min": -32.73821258544922,
      "activations/layer12_attention_weight_max": 17.6766357421875,
      "activations/layer12_attention_weight_min": -24.281036376953125,
      "activations/layer13_attention_weight_max": 39.29754638671875,
      "activations/layer13_attention_weight_min": -30.676637649536133,
      "activations/layer14_attention_weight_max": 37.1828727722168,
      "activations/layer14_attention_weight_min": -30.395198822021484,
      "activations/layer15_attention_weight_max": 31.378103256225586,
      "activations/layer15_attention_weight_min": -30.818408966064453,
      "activations/layer16_attention_weight_max": 30.351497650146484,
      "activations/layer16_attention_weight_min": -28.89854621887207,
      "activations/layer17_attention_weight_max": 49.88656234741211,
      "activations/layer17_attention_weight_min": -42.68034362792969,
      "activations/layer18_attention_weight_max": 48.24224090576172,
      "activations/layer18_attention_weight_min": -36.98146438598633,
      "activations/layer19_attention_weight_max": 20.35526466369629,
      "activations/layer19_attention_weight_min": -22.498424530029297,
      "activations/layer1_attention_weight_max": 16.605337142944336,
      "activations/layer1_attention_weight_min": -15.452659606933594,
      "activations/layer20_attention_weight_max": 23.718042373657227,
      "activations/layer20_attention_weight_min": -22.63103485107422,
      "activations/layer21_attention_weight_max": 44.76124954223633,
      "activations/layer21_attention_weight_min": -24.19419288635254,
      "activations/layer22_attention_weight_max": 30.03810691833496,
      "activations/layer22_attention_weight_min": -25.30289649963379,
      "activations/layer23_attention_weight_max": 36.340293884277344,
      "activations/layer23_attention_weight_min": -23.257221221923828,
      "activations/layer2_attention_weight_max": 33.6552734375,
      "activations/layer2_attention_weight_min": -32.36616134643555,
      "activations/layer3_attention_weight_max": 91.73184967041016,
      "activations/layer3_attention_weight_min": -98.38491821289062,
      "activations/layer4_attention_weight_max": 96.91316223144531,
      "activations/layer4_attention_weight_min": -97.77378845214844,
      "activations/layer5_attention_weight_max": 72.49748992919922,
      "activations/layer5_attention_weight_min": -78.81859588623047,
      "activations/layer6_attention_weight_max": 51.89107131958008,
      "activations/layer6_attention_weight_min": -52.57423782348633,
      "activations/layer7_attention_weight_max": 69.02296447753906,
      "activations/layer7_attention_weight_min": -67.95011138916016,
      "activations/layer8_attention_weight_max": 43.76624298095703,
      "activations/layer8_attention_weight_min": -48.784664154052734,
      "activations/layer9_attention_weight_max": 43.757259368896484,
      "activations/layer9_attention_weight_min": -46.84724044799805,
      "epoch": 11.89,
      "learning_rate": 7.408143939393939e-05,
      "loss": 2.7552,
      "step": 204600
    },
    {
      "activations/layer0_attention_weight_max": 16.279163360595703,
      "activations/layer0_attention_weight_min": -14.387938499450684,
      "activations/layer10_attention_weight_max": 38.341800689697266,
      "activations/layer10_attention_weight_min": -39.67349624633789,
      "activations/layer11_attention_weight_max": 33.49294662475586,
      "activations/layer11_attention_weight_min": -33.75872039794922,
      "activations/layer12_attention_weight_max": 17.896278381347656,
      "activations/layer12_attention_weight_min": -31.89033317565918,
      "activations/layer13_attention_weight_max": 36.796504974365234,
      "activations/layer13_attention_weight_min": -35.23956298828125,
      "activations/layer14_attention_weight_max": 39.07356262207031,
      "activations/layer14_attention_weight_min": -32.29021072387695,
      "activations/layer15_attention_weight_max": 36.98051452636719,
      "activations/layer15_attention_weight_min": -30.89718246459961,
      "activations/layer16_attention_weight_max": 33.858184814453125,
      "activations/layer16_attention_weight_min": -30.68328094482422,
      "activations/layer17_attention_weight_max": 54.833370208740234,
      "activations/layer17_attention_weight_min": -44.56828689575195,
      "activations/layer18_attention_weight_max": 47.38579559326172,
      "activations/layer18_attention_weight_min": -36.282047271728516,
      "activations/layer19_attention_weight_max": 22.158649444580078,
      "activations/layer19_attention_weight_min": -19.503698348999023,
      "activations/layer1_attention_weight_max": 16.560531616210938,
      "activations/layer1_attention_weight_min": -14.214274406433105,
      "activations/layer20_attention_weight_max": 23.30784797668457,
      "activations/layer20_attention_weight_min": -24.078903198242188,
      "activations/layer21_attention_weight_max": 34.91603469848633,
      "activations/layer21_attention_weight_min": -22.4141902923584,
      "activations/layer22_attention_weight_max": 26.667449951171875,
      "activations/layer22_attention_weight_min": -23.55381965637207,
      "activations/layer23_attention_weight_max": 37.67080307006836,
      "activations/layer23_attention_weight_min": -22.489173889160156,
      "activations/layer2_attention_weight_max": 36.78630447387695,
      "activations/layer2_attention_weight_min": -33.29256820678711,
      "activations/layer3_attention_weight_max": 104.2225570678711,
      "activations/layer3_attention_weight_min": -102.24154663085938,
      "activations/layer4_attention_weight_max": 108.5728759765625,
      "activations/layer4_attention_weight_min": -107.40193939208984,
      "activations/layer5_attention_weight_max": 79.19937133789062,
      "activations/layer5_attention_weight_min": -88.08207702636719,
      "activations/layer6_attention_weight_max": 59.16116714477539,
      "activations/layer6_attention_weight_min": -60.75907516479492,
      "activations/layer7_attention_weight_max": 77.09710693359375,
      "activations/layer7_attention_weight_min": -78.3617935180664,
      "activations/layer8_attention_weight_max": 49.98823547363281,
      "activations/layer8_attention_weight_min": -54.202476501464844,
      "activations/layer9_attention_weight_max": 53.628292083740234,
      "activations/layer9_attention_weight_min": -52.8148307800293,
      "epoch": 11.89,
      "learning_rate": 7.406249999999999e-05,
      "loss": 2.7855,
      "step": 204650
    },
    {
      "activations/layer0_attention_weight_max": 15.970911026000977,
      "activations/layer0_attention_weight_min": -14.105796813964844,
      "activations/layer10_attention_weight_max": 39.020050048828125,
      "activations/layer10_attention_weight_min": -38.940162658691406,
      "activations/layer11_attention_weight_max": 34.300193786621094,
      "activations/layer11_attention_weight_min": -35.5009880065918,
      "activations/layer12_attention_weight_max": 18.5767879486084,
      "activations/layer12_attention_weight_min": -24.24884605407715,
      "activations/layer13_attention_weight_max": 37.14610290527344,
      "activations/layer13_attention_weight_min": -27.394277572631836,
      "activations/layer14_attention_weight_max": 40.51802444458008,
      "activations/layer14_attention_weight_min": -29.649856567382812,
      "activations/layer15_attention_weight_max": 34.240665435791016,
      "activations/layer15_attention_weight_min": -27.495759963989258,
      "activations/layer16_attention_weight_max": 32.4390983581543,
      "activations/layer16_attention_weight_min": -30.77972412109375,
      "activations/layer17_attention_weight_max": 52.73604965209961,
      "activations/layer17_attention_weight_min": -42.90607452392578,
      "activations/layer18_attention_weight_max": 48.14011001586914,
      "activations/layer18_attention_weight_min": -35.83473587036133,
      "activations/layer19_attention_weight_max": 22.347457885742188,
      "activations/layer19_attention_weight_min": -23.61199188232422,
      "activations/layer1_attention_weight_max": 16.16727066040039,
      "activations/layer1_attention_weight_min": -15.252431869506836,
      "activations/layer20_attention_weight_max": 22.184179306030273,
      "activations/layer20_attention_weight_min": -20.30166244506836,
      "activations/layer21_attention_weight_max": 39.422462463378906,
      "activations/layer21_attention_weight_min": -22.71430206298828,
      "activations/layer22_attention_weight_max": 31.8040771484375,
      "activations/layer22_attention_weight_min": -26.99781608581543,
      "activations/layer23_attention_weight_max": 41.53707504272461,
      "activations/layer23_attention_weight_min": -24.45123291015625,
      "activations/layer2_attention_weight_max": 34.45369338989258,
      "activations/layer2_attention_weight_min": -33.249027252197266,
      "activations/layer3_attention_weight_max": 95.19854736328125,
      "activations/layer3_attention_weight_min": -97.62019348144531,
      "activations/layer4_attention_weight_max": 98.7439956665039,
      "activations/layer4_attention_weight_min": -97.28803253173828,
      "activations/layer5_attention_weight_max": 74.48377227783203,
      "activations/layer5_attention_weight_min": -78.30204010009766,
      "activations/layer6_attention_weight_max": 52.47322463989258,
      "activations/layer6_attention_weight_min": -52.622581481933594,
      "activations/layer7_attention_weight_max": 70.68843841552734,
      "activations/layer7_attention_weight_min": -67.52310180664062,
      "activations/layer8_attention_weight_max": 46.39857482910156,
      "activations/layer8_attention_weight_min": -48.22639846801758,
      "activations/layer9_attention_weight_max": 51.58330154418945,
      "activations/layer9_attention_weight_min": -50.99967956542969,
      "epoch": 11.89,
      "learning_rate": 7.40435606060606e-05,
      "loss": 2.7896,
      "step": 204700
    },
    {
      "activations/layer0_attention_weight_max": 15.988663673400879,
      "activations/layer0_attention_weight_min": -14.468990325927734,
      "activations/layer10_attention_weight_max": 37.64658737182617,
      "activations/layer10_attention_weight_min": -34.24777603149414,
      "activations/layer11_attention_weight_max": 33.48531723022461,
      "activations/layer11_attention_weight_min": -32.830326080322266,
      "activations/layer12_attention_weight_max": 17.715904235839844,
      "activations/layer12_attention_weight_min": -24.821746826171875,
      "activations/layer13_attention_weight_max": 41.1804313659668,
      "activations/layer13_attention_weight_min": -30.448444366455078,
      "activations/layer14_attention_weight_max": 36.36518096923828,
      "activations/layer14_attention_weight_min": -32.32018280029297,
      "activations/layer15_attention_weight_max": 34.21997833251953,
      "activations/layer15_attention_weight_min": -32.68878936767578,
      "activations/layer16_attention_weight_max": 32.09958267211914,
      "activations/layer16_attention_weight_min": -31.221906661987305,
      "activations/layer17_attention_weight_max": 53.60535430908203,
      "activations/layer17_attention_weight_min": -46.97584915161133,
      "activations/layer18_attention_weight_max": 46.22392654418945,
      "activations/layer18_attention_weight_min": -39.64240264892578,
      "activations/layer19_attention_weight_max": 23.474393844604492,
      "activations/layer19_attention_weight_min": -22.07938575744629,
      "activations/layer1_attention_weight_max": 16.842432022094727,
      "activations/layer1_attention_weight_min": -17.049724578857422,
      "activations/layer20_attention_weight_max": 24.39842987060547,
      "activations/layer20_attention_weight_min": -23.30885887145996,
      "activations/layer21_attention_weight_max": 41.36554718017578,
      "activations/layer21_attention_weight_min": -23.746000289916992,
      "activations/layer22_attention_weight_max": 33.480499267578125,
      "activations/layer22_attention_weight_min": -26.941892623901367,
      "activations/layer23_attention_weight_max": 41.33188247680664,
      "activations/layer23_attention_weight_min": -23.9405574798584,
      "activations/layer2_attention_weight_max": 33.70045852661133,
      "activations/layer2_attention_weight_min": -31.341938018798828,
      "activations/layer3_attention_weight_max": 96.09765625,
      "activations/layer3_attention_weight_min": -100.8931655883789,
      "activations/layer4_attention_weight_max": 98.85749816894531,
      "activations/layer4_attention_weight_min": -94.82118225097656,
      "activations/layer5_attention_weight_max": 74.26873779296875,
      "activations/layer5_attention_weight_min": -76.78129577636719,
      "activations/layer6_attention_weight_max": 54.02207565307617,
      "activations/layer6_attention_weight_min": -52.8493766784668,
      "activations/layer7_attention_weight_max": 78.0289535522461,
      "activations/layer7_attention_weight_min": -70.65864562988281,
      "activations/layer8_attention_weight_max": 47.640647888183594,
      "activations/layer8_attention_weight_min": -45.929344177246094,
      "activations/layer9_attention_weight_max": 52.247169494628906,
      "activations/layer9_attention_weight_min": -46.10451126098633,
      "epoch": 11.9,
      "learning_rate": 7.402462121212121e-05,
      "loss": 2.791,
      "step": 204750
    },
    {
      "activations/layer0_attention_weight_max": 16.62889289855957,
      "activations/layer0_attention_weight_min": -14.468565940856934,
      "activations/layer10_attention_weight_max": 36.29668045043945,
      "activations/layer10_attention_weight_min": -35.487579345703125,
      "activations/layer11_attention_weight_max": 32.702728271484375,
      "activations/layer11_attention_weight_min": -32.49259948730469,
      "activations/layer12_attention_weight_max": 17.130746841430664,
      "activations/layer12_attention_weight_min": -26.77773666381836,
      "activations/layer13_attention_weight_max": 35.01572799682617,
      "activations/layer13_attention_weight_min": -26.58855628967285,
      "activations/layer14_attention_weight_max": 35.96371078491211,
      "activations/layer14_attention_weight_min": -29.193641662597656,
      "activations/layer15_attention_weight_max": 32.89665985107422,
      "activations/layer15_attention_weight_min": -27.368499755859375,
      "activations/layer16_attention_weight_max": 31.802127838134766,
      "activations/layer16_attention_weight_min": -29.329517364501953,
      "activations/layer17_attention_weight_max": 51.93545150756836,
      "activations/layer17_attention_weight_min": -42.91757583618164,
      "activations/layer18_attention_weight_max": 45.624900817871094,
      "activations/layer18_attention_weight_min": -35.90999984741211,
      "activations/layer19_attention_weight_max": 20.817869186401367,
      "activations/layer19_attention_weight_min": -19.913803100585938,
      "activations/layer1_attention_weight_max": 16.784317016601562,
      "activations/layer1_attention_weight_min": -14.908598899841309,
      "activations/layer20_attention_weight_max": 21.043228149414062,
      "activations/layer20_attention_weight_min": -20.613513946533203,
      "activations/layer21_attention_weight_max": 33.06619644165039,
      "activations/layer21_attention_weight_min": -27.15252685546875,
      "activations/layer22_attention_weight_max": 28.28410530090332,
      "activations/layer22_attention_weight_min": -25.648956298828125,
      "activations/layer23_attention_weight_max": 35.05713653564453,
      "activations/layer23_attention_weight_min": -26.29842758178711,
      "activations/layer2_attention_weight_max": 33.258914947509766,
      "activations/layer2_attention_weight_min": -33.17042541503906,
      "activations/layer3_attention_weight_max": 93.98091125488281,
      "activations/layer3_attention_weight_min": -108.68417358398438,
      "activations/layer4_attention_weight_max": 98.01888275146484,
      "activations/layer4_attention_weight_min": -97.7507553100586,
      "activations/layer5_attention_weight_max": 72.5801010131836,
      "activations/layer5_attention_weight_min": -80.81975555419922,
      "activations/layer6_attention_weight_max": 50.83962631225586,
      "activations/layer6_attention_weight_min": -53.853267669677734,
      "activations/layer7_attention_weight_max": 69.78781127929688,
      "activations/layer7_attention_weight_min": -71.12992858886719,
      "activations/layer8_attention_weight_max": 45.75806427001953,
      "activations/layer8_attention_weight_min": -48.439151763916016,
      "activations/layer9_attention_weight_max": 52.2470703125,
      "activations/layer9_attention_weight_min": -47.20902633666992,
      "epoch": 11.9,
      "learning_rate": 7.400568181818181e-05,
      "loss": 2.7861,
      "step": 204800
    },
    {
      "activations/layer0_attention_weight_max": 16.143056869506836,
      "activations/layer0_attention_weight_min": -14.200837135314941,
      "activations/layer10_attention_weight_max": 35.939727783203125,
      "activations/layer10_attention_weight_min": -33.139610290527344,
      "activations/layer11_attention_weight_max": 31.80414581298828,
      "activations/layer11_attention_weight_min": -31.687902450561523,
      "activations/layer12_attention_weight_max": 20.756391525268555,
      "activations/layer12_attention_weight_min": -25.412370681762695,
      "activations/layer13_attention_weight_max": 38.46124267578125,
      "activations/layer13_attention_weight_min": -27.296842575073242,
      "activations/layer14_attention_weight_max": 35.845123291015625,
      "activations/layer14_attention_weight_min": -30.220672607421875,
      "activations/layer15_attention_weight_max": 32.22215270996094,
      "activations/layer15_attention_weight_min": -28.442487716674805,
      "activations/layer16_attention_weight_max": 31.066463470458984,
      "activations/layer16_attention_weight_min": -29.37006950378418,
      "activations/layer17_attention_weight_max": 49.91946792602539,
      "activations/layer17_attention_weight_min": -43.65849304199219,
      "activations/layer18_attention_weight_max": 44.93906784057617,
      "activations/layer18_attention_weight_min": -36.27794647216797,
      "activations/layer19_attention_weight_max": 21.799724578857422,
      "activations/layer19_attention_weight_min": -20.46544647216797,
      "activations/layer1_attention_weight_max": 16.724796295166016,
      "activations/layer1_attention_weight_min": -15.372017860412598,
      "activations/layer20_attention_weight_max": 21.799297332763672,
      "activations/layer20_attention_weight_min": -24.636680603027344,
      "activations/layer21_attention_weight_max": 39.58620071411133,
      "activations/layer21_attention_weight_min": -23.26285743713379,
      "activations/layer22_attention_weight_max": 31.091768264770508,
      "activations/layer22_attention_weight_min": -23.75482940673828,
      "activations/layer23_attention_weight_max": 36.92665100097656,
      "activations/layer23_attention_weight_min": -22.661100387573242,
      "activations/layer2_attention_weight_max": 34.9925537109375,
      "activations/layer2_attention_weight_min": -33.127105712890625,
      "activations/layer3_attention_weight_max": 97.11164093017578,
      "activations/layer3_attention_weight_min": -104.3260498046875,
      "activations/layer4_attention_weight_max": 104.38690185546875,
      "activations/layer4_attention_weight_min": -102.77542877197266,
      "activations/layer5_attention_weight_max": 72.956298828125,
      "activations/layer5_attention_weight_min": -77.22026824951172,
      "activations/layer6_attention_weight_max": 54.98636245727539,
      "activations/layer6_attention_weight_min": -52.82179641723633,
      "activations/layer7_attention_weight_max": 76.00872039794922,
      "activations/layer7_attention_weight_min": -68.194580078125,
      "activations/layer8_attention_weight_max": 44.452152252197266,
      "activations/layer8_attention_weight_min": -45.76277542114258,
      "activations/layer9_attention_weight_max": 46.44511413574219,
      "activations/layer9_attention_weight_min": -45.55599594116211,
      "epoch": 11.9,
      "learning_rate": 7.398674242424242e-05,
      "loss": 2.7819,
      "step": 204850
    },
    {
      "activations/layer0_attention_weight_max": 15.285606384277344,
      "activations/layer0_attention_weight_min": -14.619970321655273,
      "activations/layer10_attention_weight_max": 38.080440521240234,
      "activations/layer10_attention_weight_min": -36.99083709716797,
      "activations/layer11_attention_weight_max": 33.72624206542969,
      "activations/layer11_attention_weight_min": -33.51167297363281,
      "activations/layer12_attention_weight_max": 18.86057472229004,
      "activations/layer12_attention_weight_min": -26.70877456665039,
      "activations/layer13_attention_weight_max": 42.69157791137695,
      "activations/layer13_attention_weight_min": -29.974952697753906,
      "activations/layer14_attention_weight_max": 37.94404983520508,
      "activations/layer14_attention_weight_min": -28.949203491210938,
      "activations/layer15_attention_weight_max": 35.5865478515625,
      "activations/layer15_attention_weight_min": -29.93146514892578,
      "activations/layer16_attention_weight_max": 34.3654899597168,
      "activations/layer16_attention_weight_min": -29.438425064086914,
      "activations/layer17_attention_weight_max": 54.93684768676758,
      "activations/layer17_attention_weight_min": -43.587005615234375,
      "activations/layer18_attention_weight_max": 51.06766128540039,
      "activations/layer18_attention_weight_min": -38.35996627807617,
      "activations/layer19_attention_weight_max": 24.947546005249023,
      "activations/layer19_attention_weight_min": -22.11937141418457,
      "activations/layer1_attention_weight_max": 15.97025203704834,
      "activations/layer1_attention_weight_min": -15.542125701904297,
      "activations/layer20_attention_weight_max": 26.754606246948242,
      "activations/layer20_attention_weight_min": -21.35148811340332,
      "activations/layer21_attention_weight_max": 42.79094314575195,
      "activations/layer21_attention_weight_min": -22.214229583740234,
      "activations/layer22_attention_weight_max": 35.47926712036133,
      "activations/layer22_attention_weight_min": -24.58868980407715,
      "activations/layer23_attention_weight_max": 42.864051818847656,
      "activations/layer23_attention_weight_min": -25.684797286987305,
      "activations/layer2_attention_weight_max": 33.35601043701172,
      "activations/layer2_attention_weight_min": -33.33599090576172,
      "activations/layer3_attention_weight_max": 93.27543640136719,
      "activations/layer3_attention_weight_min": -96.3965072631836,
      "activations/layer4_attention_weight_max": 99.73687744140625,
      "activations/layer4_attention_weight_min": -96.59617614746094,
      "activations/layer5_attention_weight_max": 71.56845092773438,
      "activations/layer5_attention_weight_min": -82.16754913330078,
      "activations/layer6_attention_weight_max": 55.40789794921875,
      "activations/layer6_attention_weight_min": -54.37185287475586,
      "activations/layer7_attention_weight_max": 74.71365356445312,
      "activations/layer7_attention_weight_min": -75.60563659667969,
      "activations/layer8_attention_weight_max": 47.06159210205078,
      "activations/layer8_attention_weight_min": -52.770511627197266,
      "activations/layer9_attention_weight_max": 51.770263671875,
      "activations/layer9_attention_weight_min": -48.28725814819336,
      "epoch": 11.91,
      "learning_rate": 7.396780303030303e-05,
      "loss": 2.8095,
      "step": 204900
    },
    {
      "activations/layer0_attention_weight_max": 16.48236083984375,
      "activations/layer0_attention_weight_min": -14.450784683227539,
      "activations/layer10_attention_weight_max": 35.18142318725586,
      "activations/layer10_attention_weight_min": -37.53518295288086,
      "activations/layer11_attention_weight_max": 30.834592819213867,
      "activations/layer11_attention_weight_min": -32.329776763916016,
      "activations/layer12_attention_weight_max": 21.6851806640625,
      "activations/layer12_attention_weight_min": -28.054086685180664,
      "activations/layer13_attention_weight_max": 39.07537078857422,
      "activations/layer13_attention_weight_min": -29.55834197998047,
      "activations/layer14_attention_weight_max": 40.094303131103516,
      "activations/layer14_attention_weight_min": -32.367042541503906,
      "activations/layer15_attention_weight_max": 34.47951126098633,
      "activations/layer15_attention_weight_min": -30.06934928894043,
      "activations/layer16_attention_weight_max": 29.841896057128906,
      "activations/layer16_attention_weight_min": -30.907115936279297,
      "activations/layer17_attention_weight_max": 51.26244354248047,
      "activations/layer17_attention_weight_min": -48.266151428222656,
      "activations/layer18_attention_weight_max": 45.87894058227539,
      "activations/layer18_attention_weight_min": -40.61636734008789,
      "activations/layer19_attention_weight_max": 22.22713279724121,
      "activations/layer19_attention_weight_min": -24.49717903137207,
      "activations/layer1_attention_weight_max": 16.646860122680664,
      "activations/layer1_attention_weight_min": -16.142797470092773,
      "activations/layer20_attention_weight_max": 25.05501937866211,
      "activations/layer20_attention_weight_min": -22.762163162231445,
      "activations/layer21_attention_weight_max": 35.5036506652832,
      "activations/layer21_attention_weight_min": -24.249237060546875,
      "activations/layer22_attention_weight_max": 32.87257385253906,
      "activations/layer22_attention_weight_min": -26.08671760559082,
      "activations/layer23_attention_weight_max": 37.233150482177734,
      "activations/layer23_attention_weight_min": -25.19264793395996,
      "activations/layer2_attention_weight_max": 33.81369400024414,
      "activations/layer2_attention_weight_min": -33.0460205078125,
      "activations/layer3_attention_weight_max": 94.74950408935547,
      "activations/layer3_attention_weight_min": -98.40862274169922,
      "activations/layer4_attention_weight_max": 96.24934387207031,
      "activations/layer4_attention_weight_min": -95.66761016845703,
      "activations/layer5_attention_weight_max": 72.30203247070312,
      "activations/layer5_attention_weight_min": -80.2698974609375,
      "activations/layer6_attention_weight_max": 52.59925079345703,
      "activations/layer6_attention_weight_min": -51.01941680908203,
      "activations/layer7_attention_weight_max": 72.10319519042969,
      "activations/layer7_attention_weight_min": -70.13545227050781,
      "activations/layer8_attention_weight_max": 43.347373962402344,
      "activations/layer8_attention_weight_min": -45.42424392700195,
      "activations/layer9_attention_weight_max": 44.519107818603516,
      "activations/layer9_attention_weight_min": -46.35117721557617,
      "epoch": 11.91,
      "learning_rate": 7.394886363636363e-05,
      "loss": 2.7618,
      "step": 204950
    },
    {
      "activations/layer0_attention_weight_max": 15.52646255493164,
      "activations/layer0_attention_weight_min": -14.706281661987305,
      "activations/layer10_attention_weight_max": 36.73525619506836,
      "activations/layer10_attention_weight_min": -37.427513122558594,
      "activations/layer11_attention_weight_max": 32.108028411865234,
      "activations/layer11_attention_weight_min": -35.30447006225586,
      "activations/layer12_attention_weight_max": 21.431188583374023,
      "activations/layer12_attention_weight_min": -23.7431640625,
      "activations/layer13_attention_weight_max": 38.98370361328125,
      "activations/layer13_attention_weight_min": -29.930452346801758,
      "activations/layer14_attention_weight_max": 40.23188018798828,
      "activations/layer14_attention_weight_min": -32.66929626464844,
      "activations/layer15_attention_weight_max": 36.53935623168945,
      "activations/layer15_attention_weight_min": -30.877920150756836,
      "activations/layer16_attention_weight_max": 35.26216506958008,
      "activations/layer16_attention_weight_min": -32.644203186035156,
      "activations/layer17_attention_weight_max": 54.538761138916016,
      "activations/layer17_attention_weight_min": -47.011436462402344,
      "activations/layer18_attention_weight_max": 50.88611602783203,
      "activations/layer18_attention_weight_min": -38.09103012084961,
      "activations/layer19_attention_weight_max": 21.77022361755371,
      "activations/layer19_attention_weight_min": -22.600831985473633,
      "activations/layer1_attention_weight_max": 16.912891387939453,
      "activations/layer1_attention_weight_min": -15.682943344116211,
      "activations/layer20_attention_weight_max": 23.439834594726562,
      "activations/layer20_attention_weight_min": -23.225078582763672,
      "activations/layer21_attention_weight_max": 46.11918258666992,
      "activations/layer21_attention_weight_min": -25.959732055664062,
      "activations/layer22_attention_weight_max": 36.89686584472656,
      "activations/layer22_attention_weight_min": -27.610416412353516,
      "activations/layer23_attention_weight_max": 42.74166488647461,
      "activations/layer23_attention_weight_min": -24.332366943359375,
      "activations/layer2_attention_weight_max": 31.75063133239746,
      "activations/layer2_attention_weight_min": -31.175125122070312,
      "activations/layer3_attention_weight_max": 90.65169525146484,
      "activations/layer3_attention_weight_min": -92.90396118164062,
      "activations/layer4_attention_weight_max": 97.61095428466797,
      "activations/layer4_attention_weight_min": -92.9521255493164,
      "activations/layer5_attention_weight_max": 71.80729675292969,
      "activations/layer5_attention_weight_min": -78.09030151367188,
      "activations/layer6_attention_weight_max": 52.95012283325195,
      "activations/layer6_attention_weight_min": -53.529911041259766,
      "activations/layer7_attention_weight_max": 70.66940307617188,
      "activations/layer7_attention_weight_min": -68.20697784423828,
      "activations/layer8_attention_weight_max": 48.554508209228516,
      "activations/layer8_attention_weight_min": -48.7612419128418,
      "activations/layer9_attention_weight_max": 48.396236419677734,
      "activations/layer9_attention_weight_min": -49.81945037841797,
      "epoch": 11.91,
      "learning_rate": 7.392992424242423e-05,
      "loss": 2.7867,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5131,
      "eval_samples_per_second": 504.401,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5131,
      "eval_openwebtext_samples_per_second": 504.401,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_wikitext_loss": 2.962890625,
      "eval_wikitext_ppl": 19.353835657005096,
      "eval_wikitext_runtime": 1.9796,
      "eval_wikitext_samples_per_second": 230.35,
      "step": 205000
    },
    {
      "epoch": 11.91,
      "eval_lambada_loss": 2.580078125,
      "eval_lambada_ppl": 13.198169226352599,
      "eval_lambada_runtime": 9.5558,
      "eval_lambada_samples_per_second": 509.531,
      "step": 205000
    },
    {
      "activations/layer0_attention_weight_max": 16.12360382080078,
      "activations/layer0_attention_weight_min": -14.658483505249023,
      "activations/layer10_attention_weight_max": 36.4066047668457,
      "activations/layer10_attention_weight_min": -37.715885162353516,
      "activations/layer11_attention_weight_max": 31.289548873901367,
      "activations/layer11_attention_weight_min": -35.598384857177734,
      "activations/layer12_attention_weight_max": 16.629375457763672,
      "activations/layer12_attention_weight_min": -27.229084014892578,
      "activations/layer13_attention_weight_max": 33.84833526611328,
      "activations/layer13_attention_weight_min": -29.582143783569336,
      "activations/layer14_attention_weight_max": 34.48347091674805,
      "activations/layer14_attention_weight_min": -30.48760986328125,
      "activations/layer15_attention_weight_max": 34.50718688964844,
      "activations/layer15_attention_weight_min": -31.22121810913086,
      "activations/layer16_attention_weight_max": 32.5294189453125,
      "activations/layer16_attention_weight_min": -31.092910766601562,
      "activations/layer17_attention_weight_max": 49.10810470581055,
      "activations/layer17_attention_weight_min": -44.93714904785156,
      "activations/layer18_attention_weight_max": 41.10301208496094,
      "activations/layer18_attention_weight_min": -38.256980895996094,
      "activations/layer19_attention_weight_max": 22.304784774780273,
      "activations/layer19_attention_weight_min": -20.366456985473633,
      "activations/layer1_attention_weight_max": 16.153865814208984,
      "activations/layer1_attention_weight_min": -14.486806869506836,
      "activations/layer20_attention_weight_max": 20.246980667114258,
      "activations/layer20_attention_weight_min": -21.081188201904297,
      "activations/layer21_attention_weight_max": 31.843828201293945,
      "activations/layer21_attention_weight_min": -23.11557388305664,
      "activations/layer22_attention_weight_max": 28.27985382080078,
      "activations/layer22_attention_weight_min": -25.521530151367188,
      "activations/layer23_attention_weight_max": 35.774925231933594,
      "activations/layer23_attention_weight_min": -24.502866744995117,
      "activations/layer2_attention_weight_max": 34.890472412109375,
      "activations/layer2_attention_weight_min": -33.08631134033203,
      "activations/layer3_attention_weight_max": 98.9287338256836,
      "activations/layer3_attention_weight_min": -101.10908508300781,
      "activations/layer4_attention_weight_max": 101.69149017333984,
      "activations/layer4_attention_weight_min": -95.61028289794922,
      "activations/layer5_attention_weight_max": 72.58506774902344,
      "activations/layer5_attention_weight_min": -78.49127197265625,
      "activations/layer6_attention_weight_max": 53.24313735961914,
      "activations/layer6_attention_weight_min": -53.189857482910156,
      "activations/layer7_attention_weight_max": 76.62998962402344,
      "activations/layer7_attention_weight_min": -67.22354125976562,
      "activations/layer8_attention_weight_max": 44.76414108276367,
      "activations/layer8_attention_weight_min": -46.011409759521484,
      "activations/layer9_attention_weight_max": 47.3323974609375,
      "activations/layer9_attention_weight_min": -46.802974700927734,
      "epoch": 11.91,
      "learning_rate": 7.391098484848484e-05,
      "loss": 2.7805,
      "step": 205050
    },
    {
      "activations/layer0_attention_weight_max": 16.498859405517578,
      "activations/layer0_attention_weight_min": -14.201496124267578,
      "activations/layer10_attention_weight_max": 36.64281463623047,
      "activations/layer10_attention_weight_min": -36.4028434753418,
      "activations/layer11_attention_weight_max": 32.617977142333984,
      "activations/layer11_attention_weight_min": -33.848419189453125,
      "activations/layer12_attention_weight_max": 17.472326278686523,
      "activations/layer12_attention_weight_min": -23.33793830871582,
      "activations/layer13_attention_weight_max": 37.70294189453125,
      "activations/layer13_attention_weight_min": -29.988027572631836,
      "activations/layer14_attention_weight_max": 35.34465408325195,
      "activations/layer14_attention_weight_min": -31.63534164428711,
      "activations/layer15_attention_weight_max": 33.756954193115234,
      "activations/layer15_attention_weight_min": -28.97854995727539,
      "activations/layer16_attention_weight_max": 29.723430633544922,
      "activations/layer16_attention_weight_min": -28.14744758605957,
      "activations/layer17_attention_weight_max": 50.406185150146484,
      "activations/layer17_attention_weight_min": -44.125144958496094,
      "activations/layer18_attention_weight_max": 44.53285598754883,
      "activations/layer18_attention_weight_min": -36.54864501953125,
      "activations/layer19_attention_weight_max": 22.290735244750977,
      "activations/layer19_attention_weight_min": -20.460988998413086,
      "activations/layer1_attention_weight_max": 16.515445709228516,
      "activations/layer1_attention_weight_min": -15.256341934204102,
      "activations/layer20_attention_weight_max": 21.864458084106445,
      "activations/layer20_attention_weight_min": -20.19446563720703,
      "activations/layer21_attention_weight_max": 33.55921173095703,
      "activations/layer21_attention_weight_min": -22.351041793823242,
      "activations/layer22_attention_weight_max": 32.32781982421875,
      "activations/layer22_attention_weight_min": -25.407142639160156,
      "activations/layer23_attention_weight_max": 36.83346176147461,
      "activations/layer23_attention_weight_min": -22.784971237182617,
      "activations/layer2_attention_weight_max": 34.65245819091797,
      "activations/layer2_attention_weight_min": -33.66319274902344,
      "activations/layer3_attention_weight_max": 94.12579345703125,
      "activations/layer3_attention_weight_min": -102.51557922363281,
      "activations/layer4_attention_weight_max": 99.1047592163086,
      "activations/layer4_attention_weight_min": -96.21363830566406,
      "activations/layer5_attention_weight_max": 72.36160278320312,
      "activations/layer5_attention_weight_min": -82.0551986694336,
      "activations/layer6_attention_weight_max": 53.64213180541992,
      "activations/layer6_attention_weight_min": -52.64474868774414,
      "activations/layer7_attention_weight_max": 73.2830810546875,
      "activations/layer7_attention_weight_min": -70.65357208251953,
      "activations/layer8_attention_weight_max": 45.32472229003906,
      "activations/layer8_attention_weight_min": -48.38495635986328,
      "activations/layer9_attention_weight_max": 47.381263732910156,
      "activations/layer9_attention_weight_min": -48.5922966003418,
      "epoch": 11.92,
      "learning_rate": 7.389204545454545e-05,
      "loss": 2.7937,
      "step": 205100
    },
    {
      "activations/layer0_attention_weight_max": 15.821863174438477,
      "activations/layer0_attention_weight_min": -14.240428924560547,
      "activations/layer10_attention_weight_max": 37.78560256958008,
      "activations/layer10_attention_weight_min": -40.43254089355469,
      "activations/layer11_attention_weight_max": 34.120174407958984,
      "activations/layer11_attention_weight_min": -36.63645553588867,
      "activations/layer12_attention_weight_max": 20.764968872070312,
      "activations/layer12_attention_weight_min": -25.270809173583984,
      "activations/layer13_attention_weight_max": 35.61486053466797,
      "activations/layer13_attention_weight_min": -29.102752685546875,
      "activations/layer14_attention_weight_max": 37.517173767089844,
      "activations/layer14_attention_weight_min": -33.04438018798828,
      "activations/layer15_attention_weight_max": 35.62416076660156,
      "activations/layer15_attention_weight_min": -31.078962326049805,
      "activations/layer16_attention_weight_max": 35.75920486450195,
      "activations/layer16_attention_weight_min": -32.657203674316406,
      "activations/layer17_attention_weight_max": 55.66843032836914,
      "activations/layer17_attention_weight_min": -49.50482177734375,
      "activations/layer18_attention_weight_max": 46.508140563964844,
      "activations/layer18_attention_weight_min": -39.354896545410156,
      "activations/layer19_attention_weight_max": 21.9018497467041,
      "activations/layer19_attention_weight_min": -22.680288314819336,
      "activations/layer1_attention_weight_max": 17.535259246826172,
      "activations/layer1_attention_weight_min": -16.00990867614746,
      "activations/layer20_attention_weight_max": 21.987958908081055,
      "activations/layer20_attention_weight_min": -22.084674835205078,
      "activations/layer21_attention_weight_max": 37.282318115234375,
      "activations/layer21_attention_weight_min": -23.24283790588379,
      "activations/layer22_attention_weight_max": 34.3675651550293,
      "activations/layer22_attention_weight_min": -27.689361572265625,
      "activations/layer23_attention_weight_max": 43.39216232299805,
      "activations/layer23_attention_weight_min": -23.662010192871094,
      "activations/layer2_attention_weight_max": 37.01014709472656,
      "activations/layer2_attention_weight_min": -34.75876235961914,
      "activations/layer3_attention_weight_max": 102.76606750488281,
      "activations/layer3_attention_weight_min": -102.73088073730469,
      "activations/layer4_attention_weight_max": 103.924072265625,
      "activations/layer4_attention_weight_min": -104.3786849975586,
      "activations/layer5_attention_weight_max": 74.26593780517578,
      "activations/layer5_attention_weight_min": -81.831298828125,
      "activations/layer6_attention_weight_max": 55.03428268432617,
      "activations/layer6_attention_weight_min": -55.04792785644531,
      "activations/layer7_attention_weight_max": 71.13694763183594,
      "activations/layer7_attention_weight_min": -69.52024841308594,
      "activations/layer8_attention_weight_max": 48.36911392211914,
      "activations/layer8_attention_weight_min": -48.78569030761719,
      "activations/layer9_attention_weight_max": 50.713233947753906,
      "activations/layer9_attention_weight_min": -48.639888763427734,
      "epoch": 11.92,
      "learning_rate": 7.387310606060605e-05,
      "loss": 2.7786,
      "step": 205150
    },
    {
      "activations/layer0_attention_weight_max": 16.80799674987793,
      "activations/layer0_attention_weight_min": -14.283833503723145,
      "activations/layer10_attention_weight_max": 37.72990798950195,
      "activations/layer10_attention_weight_min": -35.03280258178711,
      "activations/layer11_attention_weight_max": 33.93600082397461,
      "activations/layer11_attention_weight_min": -32.67815399169922,
      "activations/layer12_attention_weight_max": 17.9548282623291,
      "activations/layer12_attention_weight_min": -24.026090621948242,
      "activations/layer13_attention_weight_max": 40.324546813964844,
      "activations/layer13_attention_weight_min": -29.74936294555664,
      "activations/layer14_attention_weight_max": 39.258235931396484,
      "activations/layer14_attention_weight_min": -31.038183212280273,
      "activations/layer15_attention_weight_max": 35.27379608154297,
      "activations/layer15_attention_weight_min": -29.351390838623047,
      "activations/layer16_attention_weight_max": 33.70196533203125,
      "activations/layer16_attention_weight_min": -31.90349006652832,
      "activations/layer17_attention_weight_max": 56.45482635498047,
      "activations/layer17_attention_weight_min": -46.89679718017578,
      "activations/layer18_attention_weight_max": 51.53953552246094,
      "activations/layer18_attention_weight_min": -40.44868087768555,
      "activations/layer19_attention_weight_max": 24.916446685791016,
      "activations/layer19_attention_weight_min": -25.008087158203125,
      "activations/layer1_attention_weight_max": 17.297147750854492,
      "activations/layer1_attention_weight_min": -17.16184425354004,
      "activations/layer20_attention_weight_max": 24.54073715209961,
      "activations/layer20_attention_weight_min": -23.887676239013672,
      "activations/layer21_attention_weight_max": 39.43266677856445,
      "activations/layer21_attention_weight_min": -24.030712127685547,
      "activations/layer22_attention_weight_max": 31.30902671813965,
      "activations/layer22_attention_weight_min": -25.107181549072266,
      "activations/layer23_attention_weight_max": 41.76797866821289,
      "activations/layer23_attention_weight_min": -25.347043991088867,
      "activations/layer2_attention_weight_max": 31.799495697021484,
      "activations/layer2_attention_weight_min": -31.12580680847168,
      "activations/layer3_attention_weight_max": 91.61693572998047,
      "activations/layer3_attention_weight_min": -98.27210998535156,
      "activations/layer4_attention_weight_max": 98.31140899658203,
      "activations/layer4_attention_weight_min": -95.9108657836914,
      "activations/layer5_attention_weight_max": 71.68869018554688,
      "activations/layer5_attention_weight_min": -80.26838684082031,
      "activations/layer6_attention_weight_max": 52.04277801513672,
      "activations/layer6_attention_weight_min": -52.955360412597656,
      "activations/layer7_attention_weight_max": 73.6819076538086,
      "activations/layer7_attention_weight_min": -67.90531921386719,
      "activations/layer8_attention_weight_max": 46.75761795043945,
      "activations/layer8_attention_weight_min": -46.039127349853516,
      "activations/layer9_attention_weight_max": 47.9740104675293,
      "activations/layer9_attention_weight_min": -47.23664093017578,
      "epoch": 11.92,
      "learning_rate": 7.385416666666666e-05,
      "loss": 2.761,
      "step": 205200
    },
    {
      "activations/layer0_attention_weight_max": 15.912938117980957,
      "activations/layer0_attention_weight_min": -14.775127410888672,
      "activations/layer10_attention_weight_max": 37.77766799926758,
      "activations/layer10_attention_weight_min": -35.48700714111328,
      "activations/layer11_attention_weight_max": 31.72896957397461,
      "activations/layer11_attention_weight_min": -31.382675170898438,
      "activations/layer12_attention_weight_max": 18.467082977294922,
      "activations/layer12_attention_weight_min": -25.068893432617188,
      "activations/layer13_attention_weight_max": 41.37992858886719,
      "activations/layer13_attention_weight_min": -31.048477172851562,
      "activations/layer14_attention_weight_max": 35.48381423950195,
      "activations/layer14_attention_weight_min": -29.098430633544922,
      "activations/layer15_attention_weight_max": 35.037200927734375,
      "activations/layer15_attention_weight_min": -27.773433685302734,
      "activations/layer16_attention_weight_max": 30.042369842529297,
      "activations/layer16_attention_weight_min": -27.689882278442383,
      "activations/layer17_attention_weight_max": 51.06352615356445,
      "activations/layer17_attention_weight_min": -40.912235260009766,
      "activations/layer18_attention_weight_max": 43.84954071044922,
      "activations/layer18_attention_weight_min": -35.577945709228516,
      "activations/layer19_attention_weight_max": 21.528303146362305,
      "activations/layer19_attention_weight_min": -20.22511863708496,
      "activations/layer1_attention_weight_max": 16.84423065185547,
      "activations/layer1_attention_weight_min": -15.120946884155273,
      "activations/layer20_attention_weight_max": 23.189531326293945,
      "activations/layer20_attention_weight_min": -22.3515625,
      "activations/layer21_attention_weight_max": 35.92499542236328,
      "activations/layer21_attention_weight_min": -23.26761817932129,
      "activations/layer22_attention_weight_max": 28.725854873657227,
      "activations/layer22_attention_weight_min": -24.80961036682129,
      "activations/layer23_attention_weight_max": 34.947784423828125,
      "activations/layer23_attention_weight_min": -24.147628784179688,
      "activations/layer2_attention_weight_max": 34.36554718017578,
      "activations/layer2_attention_weight_min": -31.877090454101562,
      "activations/layer3_attention_weight_max": 97.56045532226562,
      "activations/layer3_attention_weight_min": -100.51786804199219,
      "activations/layer4_attention_weight_max": 101.36531829833984,
      "activations/layer4_attention_weight_min": -96.7195053100586,
      "activations/layer5_attention_weight_max": 75.37480163574219,
      "activations/layer5_attention_weight_min": -79.49723052978516,
      "activations/layer6_attention_weight_max": 54.50477600097656,
      "activations/layer6_attention_weight_min": -51.92781448364258,
      "activations/layer7_attention_weight_max": 73.65618896484375,
      "activations/layer7_attention_weight_min": -67.10905456542969,
      "activations/layer8_attention_weight_max": 44.4818000793457,
      "activations/layer8_attention_weight_min": -45.2281608581543,
      "activations/layer9_attention_weight_max": 49.18943405151367,
      "activations/layer9_attention_weight_min": -46.386070251464844,
      "epoch": 11.93,
      "learning_rate": 7.383522727272726e-05,
      "loss": 2.7852,
      "step": 205250
    },
    {
      "activations/layer0_attention_weight_max": 16.497066497802734,
      "activations/layer0_attention_weight_min": -14.180614471435547,
      "activations/layer10_attention_weight_max": 35.00210189819336,
      "activations/layer10_attention_weight_min": -32.838623046875,
      "activations/layer11_attention_weight_max": 31.336118698120117,
      "activations/layer11_attention_weight_min": -31.895912170410156,
      "activations/layer12_attention_weight_max": 15.71255111694336,
      "activations/layer12_attention_weight_min": -25.45670509338379,
      "activations/layer13_attention_weight_max": 35.798126220703125,
      "activations/layer13_attention_weight_min": -27.304216384887695,
      "activations/layer14_attention_weight_max": 35.27883529663086,
      "activations/layer14_attention_weight_min": -32.07044219970703,
      "activations/layer15_attention_weight_max": 28.903057098388672,
      "activations/layer15_attention_weight_min": -31.921506881713867,
      "activations/layer16_attention_weight_max": 28.5725154876709,
      "activations/layer16_attention_weight_min": -29.123247146606445,
      "activations/layer17_attention_weight_max": 52.29561233520508,
      "activations/layer17_attention_weight_min": -41.31547546386719,
      "activations/layer18_attention_weight_max": 41.56000900268555,
      "activations/layer18_attention_weight_min": -36.9965705871582,
      "activations/layer19_attention_weight_max": 20.9273738861084,
      "activations/layer19_attention_weight_min": -22.219655990600586,
      "activations/layer1_attention_weight_max": 16.121280670166016,
      "activations/layer1_attention_weight_min": -14.981038093566895,
      "activations/layer20_attention_weight_max": 19.599300384521484,
      "activations/layer20_attention_weight_min": -22.845727920532227,
      "activations/layer21_attention_weight_max": 29.723405838012695,
      "activations/layer21_attention_weight_min": -25.610332489013672,
      "activations/layer22_attention_weight_max": 31.191558837890625,
      "activations/layer22_attention_weight_min": -24.22728729248047,
      "activations/layer23_attention_weight_max": 35.92029571533203,
      "activations/layer23_attention_weight_min": -23.835500717163086,
      "activations/layer2_attention_weight_max": 32.167789459228516,
      "activations/layer2_attention_weight_min": -30.485984802246094,
      "activations/layer3_attention_weight_max": 90.87052154541016,
      "activations/layer3_attention_weight_min": -91.66954040527344,
      "activations/layer4_attention_weight_max": 97.18595123291016,
      "activations/layer4_attention_weight_min": -96.92115020751953,
      "activations/layer5_attention_weight_max": 72.70443725585938,
      "activations/layer5_attention_weight_min": -75.81683349609375,
      "activations/layer6_attention_weight_max": 53.73051071166992,
      "activations/layer6_attention_weight_min": -51.69110870361328,
      "activations/layer7_attention_weight_max": 71.4593276977539,
      "activations/layer7_attention_weight_min": -65.92767333984375,
      "activations/layer8_attention_weight_max": 45.76189422607422,
      "activations/layer8_attention_weight_min": -46.52651596069336,
      "activations/layer9_attention_weight_max": 44.839698791503906,
      "activations/layer9_attention_weight_min": -44.16604232788086,
      "epoch": 11.93,
      "learning_rate": 7.381628787878788e-05,
      "loss": 2.7678,
      "step": 205300
    },
    {
      "activations/layer0_attention_weight_max": 16.036376953125,
      "activations/layer0_attention_weight_min": -14.794235229492188,
      "activations/layer10_attention_weight_max": 36.364505767822266,
      "activations/layer10_attention_weight_min": -37.104522705078125,
      "activations/layer11_attention_weight_max": 31.36791229248047,
      "activations/layer11_attention_weight_min": -33.02225112915039,
      "activations/layer12_attention_weight_max": 17.235336303710938,
      "activations/layer12_attention_weight_min": -23.215579986572266,
      "activations/layer13_attention_weight_max": 35.327354431152344,
      "activations/layer13_attention_weight_min": -27.95233917236328,
      "activations/layer14_attention_weight_max": 35.912559509277344,
      "activations/layer14_attention_weight_min": -29.8126277923584,
      "activations/layer15_attention_weight_max": 34.33041763305664,
      "activations/layer15_attention_weight_min": -28.542388916015625,
      "activations/layer16_attention_weight_max": 32.433494567871094,
      "activations/layer16_attention_weight_min": -28.562509536743164,
      "activations/layer17_attention_weight_max": 48.925899505615234,
      "activations/layer17_attention_weight_min": -45.439353942871094,
      "activations/layer18_attention_weight_max": 45.82019805908203,
      "activations/layer18_attention_weight_min": -38.83867263793945,
      "activations/layer19_attention_weight_max": 19.77589988708496,
      "activations/layer19_attention_weight_min": -21.85361099243164,
      "activations/layer1_attention_weight_max": 17.00501251220703,
      "activations/layer1_attention_weight_min": -15.80709457397461,
      "activations/layer20_attention_weight_max": 22.286680221557617,
      "activations/layer20_attention_weight_min": -25.9161319732666,
      "activations/layer21_attention_weight_max": 40.68424987792969,
      "activations/layer21_attention_weight_min": -24.71917152404785,
      "activations/layer22_attention_weight_max": 33.53892135620117,
      "activations/layer22_attention_weight_min": -27.11591339111328,
      "activations/layer23_attention_weight_max": 41.16376876831055,
      "activations/layer23_attention_weight_min": -23.51370620727539,
      "activations/layer2_attention_weight_max": 33.004310607910156,
      "activations/layer2_attention_weight_min": -31.764602661132812,
      "activations/layer3_attention_weight_max": 94.1631851196289,
      "activations/layer3_attention_weight_min": -95.22437286376953,
      "activations/layer4_attention_weight_max": 98.41248321533203,
      "activations/layer4_attention_weight_min": -93.67719268798828,
      "activations/layer5_attention_weight_max": 71.26925659179688,
      "activations/layer5_attention_weight_min": -78.53437805175781,
      "activations/layer6_attention_weight_max": 52.807762145996094,
      "activations/layer6_attention_weight_min": -51.27008819580078,
      "activations/layer7_attention_weight_max": 69.002685546875,
      "activations/layer7_attention_weight_min": -67.25128173828125,
      "activations/layer8_attention_weight_max": 44.46774673461914,
      "activations/layer8_attention_weight_min": -45.45089340209961,
      "activations/layer9_attention_weight_max": 44.43601989746094,
      "activations/layer9_attention_weight_min": -44.6498908996582,
      "epoch": 11.93,
      "learning_rate": 7.379734848484848e-05,
      "loss": 2.7776,
      "step": 205350
    },
    {
      "activations/layer0_attention_weight_max": 16.064857482910156,
      "activations/layer0_attention_weight_min": -14.54262924194336,
      "activations/layer10_attention_weight_max": 30.95758056640625,
      "activations/layer10_attention_weight_min": -33.490684509277344,
      "activations/layer11_attention_weight_max": 29.285810470581055,
      "activations/layer11_attention_weight_min": -30.026676177978516,
      "activations/layer12_attention_weight_max": 17.71675682067871,
      "activations/layer12_attention_weight_min": -26.237245559692383,
      "activations/layer13_attention_weight_max": 32.93025207519531,
      "activations/layer13_attention_weight_min": -28.396350860595703,
      "activations/layer14_attention_weight_max": 33.323883056640625,
      "activations/layer14_attention_weight_min": -30.076339721679688,
      "activations/layer15_attention_weight_max": 28.751190185546875,
      "activations/layer15_attention_weight_min": -28.6372127532959,
      "activations/layer16_attention_weight_max": 29.04440689086914,
      "activations/layer16_attention_weight_min": -29.682411193847656,
      "activations/layer17_attention_weight_max": 50.577762603759766,
      "activations/layer17_attention_weight_min": -42.06135940551758,
      "activations/layer18_attention_weight_max": 41.84624481201172,
      "activations/layer18_attention_weight_min": -37.250038146972656,
      "activations/layer19_attention_weight_max": 22.299007415771484,
      "activations/layer19_attention_weight_min": -22.153188705444336,
      "activations/layer1_attention_weight_max": 16.32700538635254,
      "activations/layer1_attention_weight_min": -14.014657020568848,
      "activations/layer20_attention_weight_max": 19.851821899414062,
      "activations/layer20_attention_weight_min": -26.009428024291992,
      "activations/layer21_attention_weight_max": 32.858734130859375,
      "activations/layer21_attention_weight_min": -22.79188346862793,
      "activations/layer22_attention_weight_max": 31.734664916992188,
      "activations/layer22_attention_weight_min": -26.651020050048828,
      "activations/layer23_attention_weight_max": 37.03382110595703,
      "activations/layer23_attention_weight_min": -26.054798126220703,
      "activations/layer2_attention_weight_max": 32.78697967529297,
      "activations/layer2_attention_weight_min": -30.02247428894043,
      "activations/layer3_attention_weight_max": 86.29013061523438,
      "activations/layer3_attention_weight_min": -88.48680877685547,
      "activations/layer4_attention_weight_max": 94.33145141601562,
      "activations/layer4_attention_weight_min": -96.35125732421875,
      "activations/layer5_attention_weight_max": 69.96028900146484,
      "activations/layer5_attention_weight_min": -75.42445373535156,
      "activations/layer6_attention_weight_max": 48.557315826416016,
      "activations/layer6_attention_weight_min": -48.83131408691406,
      "activations/layer7_attention_weight_max": 65.42315673828125,
      "activations/layer7_attention_weight_min": -67.22909545898438,
      "activations/layer8_attention_weight_max": 39.96231460571289,
      "activations/layer8_attention_weight_min": -42.583412170410156,
      "activations/layer9_attention_weight_max": 42.0543327331543,
      "activations/layer9_attention_weight_min": -44.17015838623047,
      "epoch": 11.93,
      "learning_rate": 7.37784090909091e-05,
      "loss": 2.778,
      "step": 205400
    },
    {
      "activations/layer0_attention_weight_max": 15.323715209960938,
      "activations/layer0_attention_weight_min": -14.730489730834961,
      "activations/layer10_attention_weight_max": 38.13518142700195,
      "activations/layer10_attention_weight_min": -34.31527328491211,
      "activations/layer11_attention_weight_max": 32.134368896484375,
      "activations/layer11_attention_weight_min": -32.32728958129883,
      "activations/layer12_attention_weight_max": 18.031986236572266,
      "activations/layer12_attention_weight_min": -33.477516174316406,
      "activations/layer13_attention_weight_max": 39.04181671142578,
      "activations/layer13_attention_weight_min": -33.22288513183594,
      "activations/layer14_attention_weight_max": 37.48137283325195,
      "activations/layer14_attention_weight_min": -32.767269134521484,
      "activations/layer15_attention_weight_max": 32.61444854736328,
      "activations/layer15_attention_weight_min": -30.589353561401367,
      "activations/layer16_attention_weight_max": 29.343524932861328,
      "activations/layer16_attention_weight_min": -28.624801635742188,
      "activations/layer17_attention_weight_max": 49.230125427246094,
      "activations/layer17_attention_weight_min": -41.85707473754883,
      "activations/layer18_attention_weight_max": 44.72633361816406,
      "activations/layer18_attention_weight_min": -38.15264129638672,
      "activations/layer19_attention_weight_max": 21.75098991394043,
      "activations/layer19_attention_weight_min": -22.048479080200195,
      "activations/layer1_attention_weight_max": 18.37836265563965,
      "activations/layer1_attention_weight_min": -13.946414947509766,
      "activations/layer20_attention_weight_max": 21.92633056640625,
      "activations/layer20_attention_weight_min": -20.50605583190918,
      "activations/layer21_attention_weight_max": 34.92723083496094,
      "activations/layer21_attention_weight_min": -25.75604820251465,
      "activations/layer22_attention_weight_max": 30.821807861328125,
      "activations/layer22_attention_weight_min": -25.603513717651367,
      "activations/layer23_attention_weight_max": 36.09140396118164,
      "activations/layer23_attention_weight_min": -23.52541732788086,
      "activations/layer2_attention_weight_max": 32.228721618652344,
      "activations/layer2_attention_weight_min": -30.84972381591797,
      "activations/layer3_attention_weight_max": 89.01045989990234,
      "activations/layer3_attention_weight_min": -92.03755187988281,
      "activations/layer4_attention_weight_max": 95.10316467285156,
      "activations/layer4_attention_weight_min": -96.65026092529297,
      "activations/layer5_attention_weight_max": 72.56082153320312,
      "activations/layer5_attention_weight_min": -77.03750610351562,
      "activations/layer6_attention_weight_max": 52.22036361694336,
      "activations/layer6_attention_weight_min": -53.293338775634766,
      "activations/layer7_attention_weight_max": 74.5185317993164,
      "activations/layer7_attention_weight_min": -71.20220184326172,
      "activations/layer8_attention_weight_max": 44.32707214355469,
      "activations/layer8_attention_weight_min": -47.8329963684082,
      "activations/layer9_attention_weight_max": 54.4638671875,
      "activations/layer9_attention_weight_min": -51.03242874145508,
      "epoch": 11.94,
      "learning_rate": 7.37594696969697e-05,
      "loss": 2.7777,
      "step": 205450
    },
    {
      "activations/layer0_attention_weight_max": 17.031465530395508,
      "activations/layer0_attention_weight_min": -14.480827331542969,
      "activations/layer10_attention_weight_max": 46.840999603271484,
      "activations/layer10_attention_weight_min": -41.05100631713867,
      "activations/layer11_attention_weight_max": 40.655494689941406,
      "activations/layer11_attention_weight_min": -37.76945495605469,
      "activations/layer12_attention_weight_max": 20.385940551757812,
      "activations/layer12_attention_weight_min": -27.018434524536133,
      "activations/layer13_attention_weight_max": 51.161373138427734,
      "activations/layer13_attention_weight_min": -32.95207214355469,
      "activations/layer14_attention_weight_max": 55.90557861328125,
      "activations/layer14_attention_weight_min": -34.31370162963867,
      "activations/layer15_attention_weight_max": 41.091487884521484,
      "activations/layer15_attention_weight_min": -31.99774932861328,
      "activations/layer16_attention_weight_max": 36.19894790649414,
      "activations/layer16_attention_weight_min": -31.193498611450195,
      "activations/layer17_attention_weight_max": 60.81770706176758,
      "activations/layer17_attention_weight_min": -49.582096099853516,
      "activations/layer18_attention_weight_max": 57.966007232666016,
      "activations/layer18_attention_weight_min": -42.326141357421875,
      "activations/layer19_attention_weight_max": 28.630964279174805,
      "activations/layer19_attention_weight_min": -24.284650802612305,
      "activations/layer1_attention_weight_max": 16.708942413330078,
      "activations/layer1_attention_weight_min": -14.876526832580566,
      "activations/layer20_attention_weight_max": 28.902862548828125,
      "activations/layer20_attention_weight_min": -20.293930053710938,
      "activations/layer21_attention_weight_max": 52.91716003417969,
      "activations/layer21_attention_weight_min": -23.424888610839844,
      "activations/layer22_attention_weight_max": 36.53475570678711,
      "activations/layer22_attention_weight_min": -25.226451873779297,
      "activations/layer23_attention_weight_max": 43.71480941772461,
      "activations/layer23_attention_weight_min": -25.674726486206055,
      "activations/layer2_attention_weight_max": 33.35409927368164,
      "activations/layer2_attention_weight_min": -31.95277976989746,
      "activations/layer3_attention_weight_max": 94.48323822021484,
      "activations/layer3_attention_weight_min": -93.54237365722656,
      "activations/layer4_attention_weight_max": 103.53468322753906,
      "activations/layer4_attention_weight_min": -100.0309066772461,
      "activations/layer5_attention_weight_max": 78.28268432617188,
      "activations/layer5_attention_weight_min": -78.20114135742188,
      "activations/layer6_attention_weight_max": 57.98872375488281,
      "activations/layer6_attention_weight_min": -55.549705505371094,
      "activations/layer7_attention_weight_max": 85.55613708496094,
      "activations/layer7_attention_weight_min": -72.63427734375,
      "activations/layer8_attention_weight_max": 52.02587890625,
      "activations/layer8_attention_weight_min": -51.10377883911133,
      "activations/layer9_attention_weight_max": 57.1442985534668,
      "activations/layer9_attention_weight_min": -49.16718673706055,
      "epoch": 11.94,
      "learning_rate": 7.37405303030303e-05,
      "loss": 2.7923,
      "step": 205500
    },
    {
      "activations/layer0_attention_weight_max": 15.646971702575684,
      "activations/layer0_attention_weight_min": -14.313691139221191,
      "activations/layer10_attention_weight_max": 37.011627197265625,
      "activations/layer10_attention_weight_min": -37.03195571899414,
      "activations/layer11_attention_weight_max": 32.5198974609375,
      "activations/layer11_attention_weight_min": -33.0999641418457,
      "activations/layer12_attention_weight_max": 18.475278854370117,
      "activations/layer12_attention_weight_min": -25.71417808532715,
      "activations/layer13_attention_weight_max": 39.3874626159668,
      "activations/layer13_attention_weight_min": -28.396514892578125,
      "activations/layer14_attention_weight_max": 39.561500549316406,
      "activations/layer14_attention_weight_min": -32.0031852722168,
      "activations/layer15_attention_weight_max": 33.197105407714844,
      "activations/layer15_attention_weight_min": -29.00355339050293,
      "activations/layer16_attention_weight_max": 31.879838943481445,
      "activations/layer16_attention_weight_min": -29.401546478271484,
      "activations/layer17_attention_weight_max": 54.058780670166016,
      "activations/layer17_attention_weight_min": -46.5385627746582,
      "activations/layer18_attention_weight_max": 48.84387969970703,
      "activations/layer18_attention_weight_min": -38.98208236694336,
      "activations/layer19_attention_weight_max": 21.229646682739258,
      "activations/layer19_attention_weight_min": -23.35977554321289,
      "activations/layer1_attention_weight_max": 16.320096969604492,
      "activations/layer1_attention_weight_min": -16.740747451782227,
      "activations/layer20_attention_weight_max": 22.571786880493164,
      "activations/layer20_attention_weight_min": -22.857833862304688,
      "activations/layer21_attention_weight_max": 34.41259002685547,
      "activations/layer21_attention_weight_min": -21.4012508392334,
      "activations/layer22_attention_weight_max": 29.39136505126953,
      "activations/layer22_attention_weight_min": -26.14775276184082,
      "activations/layer23_attention_weight_max": 36.68242263793945,
      "activations/layer23_attention_weight_min": -23.739788055419922,
      "activations/layer2_attention_weight_max": 33.132476806640625,
      "activations/layer2_attention_weight_min": -31.780744552612305,
      "activations/layer3_attention_weight_max": 95.64202117919922,
      "activations/layer3_attention_weight_min": -95.49602508544922,
      "activations/layer4_attention_weight_max": 101.06489562988281,
      "activations/layer4_attention_weight_min": -97.55025482177734,
      "activations/layer5_attention_weight_max": 70.9111099243164,
      "activations/layer5_attention_weight_min": -85.35015106201172,
      "activations/layer6_attention_weight_max": 51.080116271972656,
      "activations/layer6_attention_weight_min": -52.773582458496094,
      "activations/layer7_attention_weight_max": 73.67826080322266,
      "activations/layer7_attention_weight_min": -70.97818756103516,
      "activations/layer8_attention_weight_max": 46.499969482421875,
      "activations/layer8_attention_weight_min": -46.0115852355957,
      "activations/layer9_attention_weight_max": 52.48262405395508,
      "activations/layer9_attention_weight_min": -50.41655731201172,
      "epoch": 11.94,
      "learning_rate": 7.37215909090909e-05,
      "loss": 2.7707,
      "step": 205550
    },
    {
      "activations/layer0_attention_weight_max": 15.438426971435547,
      "activations/layer0_attention_weight_min": -14.677801132202148,
      "activations/layer10_attention_weight_max": 38.59724807739258,
      "activations/layer10_attention_weight_min": -37.44468688964844,
      "activations/layer11_attention_weight_max": 34.868717193603516,
      "activations/layer11_attention_weight_min": -35.117610931396484,
      "activations/layer12_attention_weight_max": 20.373308181762695,
      "activations/layer12_attention_weight_min": -24.623252868652344,
      "activations/layer13_attention_weight_max": 41.364784240722656,
      "activations/layer13_attention_weight_min": -27.615673065185547,
      "activations/layer14_attention_weight_max": 36.8869514465332,
      "activations/layer14_attention_weight_min": -28.552881240844727,
      "activations/layer15_attention_weight_max": 35.04169464111328,
      "activations/layer15_attention_weight_min": -28.62099838256836,
      "activations/layer16_attention_weight_max": 32.10991287231445,
      "activations/layer16_attention_weight_min": -30.13805389404297,
      "activations/layer17_attention_weight_max": 55.739070892333984,
      "activations/layer17_attention_weight_min": -44.955265045166016,
      "activations/layer18_attention_weight_max": 50.81782531738281,
      "activations/layer18_attention_weight_min": -38.100440979003906,
      "activations/layer19_attention_weight_max": 25.237438201904297,
      "activations/layer19_attention_weight_min": -21.966144561767578,
      "activations/layer1_attention_weight_max": 16.26410484313965,
      "activations/layer1_attention_weight_min": -14.75067138671875,
      "activations/layer20_attention_weight_max": 22.099334716796875,
      "activations/layer20_attention_weight_min": -20.30063819885254,
      "activations/layer21_attention_weight_max": 38.70092010498047,
      "activations/layer21_attention_weight_min": -22.84955406188965,
      "activations/layer22_attention_weight_max": 32.36741638183594,
      "activations/layer22_attention_weight_min": -26.42128562927246,
      "activations/layer23_attention_weight_max": 40.31825256347656,
      "activations/layer23_attention_weight_min": -24.861061096191406,
      "activations/layer2_attention_weight_max": 32.67491912841797,
      "activations/layer2_attention_weight_min": -31.60036277770996,
      "activations/layer3_attention_weight_max": 94.51669311523438,
      "activations/layer3_attention_weight_min": -98.33892059326172,
      "activations/layer4_attention_weight_max": 101.89522552490234,
      "activations/layer4_attention_weight_min": -98.28223419189453,
      "activations/layer5_attention_weight_max": 75.60746765136719,
      "activations/layer5_attention_weight_min": -81.11273193359375,
      "activations/layer6_attention_weight_max": 58.0535774230957,
      "activations/layer6_attention_weight_min": -52.27351760864258,
      "activations/layer7_attention_weight_max": 78.11365509033203,
      "activations/layer7_attention_weight_min": -67.9905776977539,
      "activations/layer8_attention_weight_max": 47.446571350097656,
      "activations/layer8_attention_weight_min": -48.72432327270508,
      "activations/layer9_attention_weight_max": 48.450504302978516,
      "activations/layer9_attention_weight_min": -49.643001556396484,
      "epoch": 11.95,
      "learning_rate": 7.370265151515151e-05,
      "loss": 2.7807,
      "step": 205600
    },
    {
      "activations/layer0_attention_weight_max": 16.33793067932129,
      "activations/layer0_attention_weight_min": -14.544638633728027,
      "activations/layer10_attention_weight_max": 35.8049201965332,
      "activations/layer10_attention_weight_min": -34.403472900390625,
      "activations/layer11_attention_weight_max": 33.81511306762695,
      "activations/layer11_attention_weight_min": -32.99657440185547,
      "activations/layer12_attention_weight_max": 18.1318302154541,
      "activations/layer12_attention_weight_min": -24.782800674438477,
      "activations/layer13_attention_weight_max": 36.44508361816406,
      "activations/layer13_attention_weight_min": -26.8365421295166,
      "activations/layer14_attention_weight_max": 33.83378219604492,
      "activations/layer14_attention_weight_min": -29.013660430908203,
      "activations/layer15_attention_weight_max": 30.639175415039062,
      "activations/layer15_attention_weight_min": -27.039827346801758,
      "activations/layer16_attention_weight_max": 31.274011611938477,
      "activations/layer16_attention_weight_min": -28.2832088470459,
      "activations/layer17_attention_weight_max": 56.33839416503906,
      "activations/layer17_attention_weight_min": -47.49296951293945,
      "activations/layer18_attention_weight_max": 46.768856048583984,
      "activations/layer18_attention_weight_min": -41.51751708984375,
      "activations/layer19_attention_weight_max": 22.880022048950195,
      "activations/layer19_attention_weight_min": -21.165382385253906,
      "activations/layer1_attention_weight_max": 17.740903854370117,
      "activations/layer1_attention_weight_min": -14.905105590820312,
      "activations/layer20_attention_weight_max": 24.051956176757812,
      "activations/layer20_attention_weight_min": -21.461389541625977,
      "activations/layer21_attention_weight_max": 34.78239822387695,
      "activations/layer21_attention_weight_min": -22.38181495666504,
      "activations/layer22_attention_weight_max": 35.15393829345703,
      "activations/layer22_attention_weight_min": -23.902923583984375,
      "activations/layer23_attention_weight_max": 39.72607421875,
      "activations/layer23_attention_weight_min": -21.226402282714844,
      "activations/layer2_attention_weight_max": 33.83421325683594,
      "activations/layer2_attention_weight_min": -33.45001220703125,
      "activations/layer3_attention_weight_max": 94.05703735351562,
      "activations/layer3_attention_weight_min": -97.03266906738281,
      "activations/layer4_attention_weight_max": 101.43633270263672,
      "activations/layer4_attention_weight_min": -98.36071014404297,
      "activations/layer5_attention_weight_max": 68.608642578125,
      "activations/layer5_attention_weight_min": -76.967041015625,
      "activations/layer6_attention_weight_max": 52.41862869262695,
      "activations/layer6_attention_weight_min": -51.54862976074219,
      "activations/layer7_attention_weight_max": 72.45032501220703,
      "activations/layer7_attention_weight_min": -66.288330078125,
      "activations/layer8_attention_weight_max": 44.4289436340332,
      "activations/layer8_attention_weight_min": -46.554588317871094,
      "activations/layer9_attention_weight_max": 52.823421478271484,
      "activations/layer9_attention_weight_min": -45.78892135620117,
      "epoch": 11.95,
      "learning_rate": 7.368371212121212e-05,
      "loss": 2.7659,
      "step": 205650
    },
    {
      "activations/layer0_attention_weight_max": 15.992006301879883,
      "activations/layer0_attention_weight_min": -15.207025527954102,
      "activations/layer10_attention_weight_max": 36.56441879272461,
      "activations/layer10_attention_weight_min": -35.8377685546875,
      "activations/layer11_attention_weight_max": 31.81696319580078,
      "activations/layer11_attention_weight_min": -32.70574188232422,
      "activations/layer12_attention_weight_max": 24.992576599121094,
      "activations/layer12_attention_weight_min": -21.552122116088867,
      "activations/layer13_attention_weight_max": 34.68778610229492,
      "activations/layer13_attention_weight_min": -28.0856990814209,
      "activations/layer14_attention_weight_max": 34.15951919555664,
      "activations/layer14_attention_weight_min": -31.073719024658203,
      "activations/layer15_attention_weight_max": 30.7655086517334,
      "activations/layer15_attention_weight_min": -28.66623306274414,
      "activations/layer16_attention_weight_max": 28.766450881958008,
      "activations/layer16_attention_weight_min": -28.505117416381836,
      "activations/layer17_attention_weight_max": 48.17410659790039,
      "activations/layer17_attention_weight_min": -44.27637481689453,
      "activations/layer18_attention_weight_max": 42.24783706665039,
      "activations/layer18_attention_weight_min": -36.838932037353516,
      "activations/layer19_attention_weight_max": 24.027006149291992,
      "activations/layer19_attention_weight_min": -22.786113739013672,
      "activations/layer1_attention_weight_max": 17.954233169555664,
      "activations/layer1_attention_weight_min": -15.085171699523926,
      "activations/layer20_attention_weight_max": 22.842805862426758,
      "activations/layer20_attention_weight_min": -22.927215576171875,
      "activations/layer21_attention_weight_max": 36.029693603515625,
      "activations/layer21_attention_weight_min": -23.87699317932129,
      "activations/layer22_attention_weight_max": 27.81355857849121,
      "activations/layer22_attention_weight_min": -25.966856002807617,
      "activations/layer23_attention_weight_max": 38.2333984375,
      "activations/layer23_attention_weight_min": -22.732650756835938,
      "activations/layer2_attention_weight_max": 36.91097640991211,
      "activations/layer2_attention_weight_min": -35.764835357666016,
      "activations/layer3_attention_weight_max": 96.00010681152344,
      "activations/layer3_attention_weight_min": -97.33712005615234,
      "activations/layer4_attention_weight_max": 100.61322021484375,
      "activations/layer4_attention_weight_min": -98.90093994140625,
      "activations/layer5_attention_weight_max": 72.7065200805664,
      "activations/layer5_attention_weight_min": -81.16883850097656,
      "activations/layer6_attention_weight_max": 54.125736236572266,
      "activations/layer6_attention_weight_min": -53.47603988647461,
      "activations/layer7_attention_weight_max": 74.5639419555664,
      "activations/layer7_attention_weight_min": -67.74360656738281,
      "activations/layer8_attention_weight_max": 45.3530158996582,
      "activations/layer8_attention_weight_min": -46.040199279785156,
      "activations/layer9_attention_weight_max": 47.619224548339844,
      "activations/layer9_attention_weight_min": -47.82561111450195,
      "epoch": 11.95,
      "learning_rate": 7.366477272727272e-05,
      "loss": 2.7642,
      "step": 205700
    },
    {
      "activations/layer0_attention_weight_max": 15.778637886047363,
      "activations/layer0_attention_weight_min": -14.893097877502441,
      "activations/layer10_attention_weight_max": 34.24291229248047,
      "activations/layer10_attention_weight_min": -34.274471282958984,
      "activations/layer11_attention_weight_max": 32.323036193847656,
      "activations/layer11_attention_weight_min": -33.747901916503906,
      "activations/layer12_attention_weight_max": 17.553485870361328,
      "activations/layer12_attention_weight_min": -22.013322830200195,
      "activations/layer13_attention_weight_max": 34.681427001953125,
      "activations/layer13_attention_weight_min": -30.130390167236328,
      "activations/layer14_attention_weight_max": 35.6729621887207,
      "activations/layer14_attention_weight_min": -30.777788162231445,
      "activations/layer15_attention_weight_max": 33.422096252441406,
      "activations/layer15_attention_weight_min": -29.983835220336914,
      "activations/layer16_attention_weight_max": 31.140039443969727,
      "activations/layer16_attention_weight_min": -28.401681900024414,
      "activations/layer17_attention_weight_max": 53.3516731262207,
      "activations/layer17_attention_weight_min": -41.746360778808594,
      "activations/layer18_attention_weight_max": 47.92177963256836,
      "activations/layer18_attention_weight_min": -37.08648681640625,
      "activations/layer19_attention_weight_max": 23.059377670288086,
      "activations/layer19_attention_weight_min": -21.275495529174805,
      "activations/layer1_attention_weight_max": 17.700490951538086,
      "activations/layer1_attention_weight_min": -16.720552444458008,
      "activations/layer20_attention_weight_max": 23.802160263061523,
      "activations/layer20_attention_weight_min": -24.687692642211914,
      "activations/layer21_attention_weight_max": 35.89640426635742,
      "activations/layer21_attention_weight_min": -24.79350471496582,
      "activations/layer22_attention_weight_max": 31.663898468017578,
      "activations/layer22_attention_weight_min": -26.340744018554688,
      "activations/layer23_attention_weight_max": 38.69443893432617,
      "activations/layer23_attention_weight_min": -26.023550033569336,
      "activations/layer2_attention_weight_max": 33.6762580871582,
      "activations/layer2_attention_weight_min": -32.97467803955078,
      "activations/layer3_attention_weight_max": 96.50951385498047,
      "activations/layer3_attention_weight_min": -95.9815444946289,
      "activations/layer4_attention_weight_max": 100.07697296142578,
      "activations/layer4_attention_weight_min": -99.54496765136719,
      "activations/layer5_attention_weight_max": 75.36361694335938,
      "activations/layer5_attention_weight_min": -79.36962127685547,
      "activations/layer6_attention_weight_max": 54.02669906616211,
      "activations/layer6_attention_weight_min": -55.0725212097168,
      "activations/layer7_attention_weight_max": 77.13787841796875,
      "activations/layer7_attention_weight_min": -68.39293670654297,
      "activations/layer8_attention_weight_max": 43.82423782348633,
      "activations/layer8_attention_weight_min": -44.89426803588867,
      "activations/layer9_attention_weight_max": 48.872982025146484,
      "activations/layer9_attention_weight_min": -45.67414855957031,
      "epoch": 11.96,
      "learning_rate": 7.364583333333332e-05,
      "loss": 2.7843,
      "step": 205750
    },
    {
      "activations/layer0_attention_weight_max": 16.457983016967773,
      "activations/layer0_attention_weight_min": -14.797964096069336,
      "activations/layer10_attention_weight_max": 36.785888671875,
      "activations/layer10_attention_weight_min": -37.11591339111328,
      "activations/layer11_attention_weight_max": 31.146808624267578,
      "activations/layer11_attention_weight_min": -33.303077697753906,
      "activations/layer12_attention_weight_max": 18.733173370361328,
      "activations/layer12_attention_weight_min": -27.583602905273438,
      "activations/layer13_attention_weight_max": 39.7061653137207,
      "activations/layer13_attention_weight_min": -29.67339324951172,
      "activations/layer14_attention_weight_max": 38.505149841308594,
      "activations/layer14_attention_weight_min": -30.939620971679688,
      "activations/layer15_attention_weight_max": 37.53413772583008,
      "activations/layer15_attention_weight_min": -29.252111434936523,
      "activations/layer16_attention_weight_max": 38.66661834716797,
      "activations/layer16_attention_weight_min": -31.039453506469727,
      "activations/layer17_attention_weight_max": 59.16627883911133,
      "activations/layer17_attention_weight_min": -49.39763641357422,
      "activations/layer18_attention_weight_max": 54.47351837158203,
      "activations/layer18_attention_weight_min": -41.844993591308594,
      "activations/layer19_attention_weight_max": 22.191797256469727,
      "activations/layer19_attention_weight_min": -23.08519172668457,
      "activations/layer1_attention_weight_max": 16.41511344909668,
      "activations/layer1_attention_weight_min": -14.906991004943848,
      "activations/layer20_attention_weight_max": 23.593172073364258,
      "activations/layer20_attention_weight_min": -21.867656707763672,
      "activations/layer21_attention_weight_max": 38.149925231933594,
      "activations/layer21_attention_weight_min": -22.477102279663086,
      "activations/layer22_attention_weight_max": 33.17001724243164,
      "activations/layer22_attention_weight_min": -26.698486328125,
      "activations/layer23_attention_weight_max": 38.621551513671875,
      "activations/layer23_attention_weight_min": -23.87836265563965,
      "activations/layer2_attention_weight_max": 32.410926818847656,
      "activations/layer2_attention_weight_min": -30.576393127441406,
      "activations/layer3_attention_weight_max": 92.20761108398438,
      "activations/layer3_attention_weight_min": -92.464599609375,
      "activations/layer4_attention_weight_max": 101.40555572509766,
      "activations/layer4_attention_weight_min": -97.56454467773438,
      "activations/layer5_attention_weight_max": 72.24169921875,
      "activations/layer5_attention_weight_min": -81.923828125,
      "activations/layer6_attention_weight_max": 54.9918098449707,
      "activations/layer6_attention_weight_min": -52.05738067626953,
      "activations/layer7_attention_weight_max": 73.29646301269531,
      "activations/layer7_attention_weight_min": -68.2748031616211,
      "activations/layer8_attention_weight_max": 47.37435531616211,
      "activations/layer8_attention_weight_min": -47.68368148803711,
      "activations/layer9_attention_weight_max": 45.80112838745117,
      "activations/layer9_attention_weight_min": -46.65249252319336,
      "epoch": 11.96,
      "learning_rate": 7.362689393939393e-05,
      "loss": 2.782,
      "step": 205800
    },
    {
      "activations/layer0_attention_weight_max": 15.410989761352539,
      "activations/layer0_attention_weight_min": -14.932368278503418,
      "activations/layer10_attention_weight_max": 36.400367736816406,
      "activations/layer10_attention_weight_min": -35.858219146728516,
      "activations/layer11_attention_weight_max": 33.479488372802734,
      "activations/layer11_attention_weight_min": -32.63105392456055,
      "activations/layer12_attention_weight_max": 20.388107299804688,
      "activations/layer12_attention_weight_min": -24.496423721313477,
      "activations/layer13_attention_weight_max": 42.09424591064453,
      "activations/layer13_attention_weight_min": -29.561952590942383,
      "activations/layer14_attention_weight_max": 40.054786682128906,
      "activations/layer14_attention_weight_min": -30.054655075073242,
      "activations/layer15_attention_weight_max": 37.93962097167969,
      "activations/layer15_attention_weight_min": -28.098764419555664,
      "activations/layer16_attention_weight_max": 35.79682159423828,
      "activations/layer16_attention_weight_min": -32.42019271850586,
      "activations/layer17_attention_weight_max": 59.2110595703125,
      "activations/layer17_attention_weight_min": -47.78588104248047,
      "activations/layer18_attention_weight_max": 53.18492889404297,
      "activations/layer18_attention_weight_min": -41.25604248046875,
      "activations/layer19_attention_weight_max": 26.781879425048828,
      "activations/layer19_attention_weight_min": -21.869609832763672,
      "activations/layer1_attention_weight_max": 17.436038970947266,
      "activations/layer1_attention_weight_min": -17.024843215942383,
      "activations/layer20_attention_weight_max": 26.941333770751953,
      "activations/layer20_attention_weight_min": -21.744935989379883,
      "activations/layer21_attention_weight_max": 44.64360427856445,
      "activations/layer21_attention_weight_min": -24.207082748413086,
      "activations/layer22_attention_weight_max": 37.22043228149414,
      "activations/layer22_attention_weight_min": -28.46396827697754,
      "activations/layer23_attention_weight_max": 44.120235443115234,
      "activations/layer23_attention_weight_min": -26.28217124938965,
      "activations/layer2_attention_weight_max": 32.238555908203125,
      "activations/layer2_attention_weight_min": -31.943191528320312,
      "activations/layer3_attention_weight_max": 94.21009063720703,
      "activations/layer3_attention_weight_min": -91.56171417236328,
      "activations/layer4_attention_weight_max": 99.18145751953125,
      "activations/layer4_attention_weight_min": -96.73104858398438,
      "activations/layer5_attention_weight_max": 70.30918884277344,
      "activations/layer5_attention_weight_min": -76.56275939941406,
      "activations/layer6_attention_weight_max": 52.55817413330078,
      "activations/layer6_attention_weight_min": -50.84419631958008,
      "activations/layer7_attention_weight_max": 73.80307006835938,
      "activations/layer7_attention_weight_min": -70.2866439819336,
      "activations/layer8_attention_weight_max": 46.60902786254883,
      "activations/layer8_attention_weight_min": -48.44152069091797,
      "activations/layer9_attention_weight_max": 47.597286224365234,
      "activations/layer9_attention_weight_min": -51.835968017578125,
      "epoch": 11.96,
      "learning_rate": 7.360833333333332e-05,
      "loss": 2.7783,
      "step": 205850
    },
    {
      "activations/layer0_attention_weight_max": 16.87032127380371,
      "activations/layer0_attention_weight_min": -14.652907371520996,
      "activations/layer10_attention_weight_max": 35.72393035888672,
      "activations/layer10_attention_weight_min": -35.137176513671875,
      "activations/layer11_attention_weight_max": 31.831100463867188,
      "activations/layer11_attention_weight_min": -32.81584167480469,
      "activations/layer12_attention_weight_max": 19.981775283813477,
      "activations/layer12_attention_weight_min": -27.628664016723633,
      "activations/layer13_attention_weight_max": 40.42601013183594,
      "activations/layer13_attention_weight_min": -29.075542449951172,
      "activations/layer14_attention_weight_max": 39.497825622558594,
      "activations/layer14_attention_weight_min": -29.99431800842285,
      "activations/layer15_attention_weight_max": 36.35806655883789,
      "activations/layer15_attention_weight_min": -28.842573165893555,
      "activations/layer16_attention_weight_max": 34.69633865356445,
      "activations/layer16_attention_weight_min": -29.306743621826172,
      "activations/layer17_attention_weight_max": 56.98268508911133,
      "activations/layer17_attention_weight_min": -44.087581634521484,
      "activations/layer18_attention_weight_max": 46.01523208618164,
      "activations/layer18_attention_weight_min": -36.85319519042969,
      "activations/layer19_attention_weight_max": 26.119794845581055,
      "activations/layer19_attention_weight_min": -22.104265213012695,
      "activations/layer1_attention_weight_max": 16.25778579711914,
      "activations/layer1_attention_weight_min": -14.76502513885498,
      "activations/layer20_attention_weight_max": 22.404376983642578,
      "activations/layer20_attention_weight_min": -20.10545539855957,
      "activations/layer21_attention_weight_max": 37.275054931640625,
      "activations/layer21_attention_weight_min": -23.553478240966797,
      "activations/layer22_attention_weight_max": 33.25053024291992,
      "activations/layer22_attention_weight_min": -25.390256881713867,
      "activations/layer23_attention_weight_max": 36.020023345947266,
      "activations/layer23_attention_weight_min": -24.52227783203125,
      "activations/layer2_attention_weight_max": 32.48991012573242,
      "activations/layer2_attention_weight_min": -30.89533805847168,
      "activations/layer3_attention_weight_max": 90.927490234375,
      "activations/layer3_attention_weight_min": -93.31571197509766,
      "activations/layer4_attention_weight_max": 98.38419342041016,
      "activations/layer4_attention_weight_min": -100.33644104003906,
      "activations/layer5_attention_weight_max": 73.02005767822266,
      "activations/layer5_attention_weight_min": -78.31611633300781,
      "activations/layer6_attention_weight_max": 52.47935485839844,
      "activations/layer6_attention_weight_min": -54.765254974365234,
      "activations/layer7_attention_weight_max": 72.02253723144531,
      "activations/layer7_attention_weight_min": -66.58392333984375,
      "activations/layer8_attention_weight_max": 44.735355377197266,
      "activations/layer8_attention_weight_min": -47.31867980957031,
      "activations/layer9_attention_weight_max": 51.77348709106445,
      "activations/layer9_attention_weight_min": -45.18939971923828,
      "epoch": 11.96,
      "learning_rate": 7.358939393939393e-05,
      "loss": 2.7687,
      "step": 205900
    },
    {
      "activations/layer0_attention_weight_max": 17.573911666870117,
      "activations/layer0_attention_weight_min": -14.461362838745117,
      "activations/layer10_attention_weight_max": 38.570098876953125,
      "activations/layer10_attention_weight_min": -35.49773025512695,
      "activations/layer11_attention_weight_max": 34.717918395996094,
      "activations/layer11_attention_weight_min": -33.431766510009766,
      "activations/layer12_attention_weight_max": 29.94878578186035,
      "activations/layer12_attention_weight_min": -29.193124771118164,
      "activations/layer13_attention_weight_max": 38.30793380737305,
      "activations/layer13_attention_weight_min": -30.481468200683594,
      "activations/layer14_attention_weight_max": 36.99591827392578,
      "activations/layer14_attention_weight_min": -30.069400787353516,
      "activations/layer15_attention_weight_max": 33.193058013916016,
      "activations/layer15_attention_weight_min": -29.01218605041504,
      "activations/layer16_attention_weight_max": 30.71940803527832,
      "activations/layer16_attention_weight_min": -29.703136444091797,
      "activations/layer17_attention_weight_max": 58.47932434082031,
      "activations/layer17_attention_weight_min": -47.13622283935547,
      "activations/layer18_attention_weight_max": 49.750030517578125,
      "activations/layer18_attention_weight_min": -41.82292938232422,
      "activations/layer19_attention_weight_max": 21.96756362915039,
      "activations/layer19_attention_weight_min": -22.41386604309082,
      "activations/layer1_attention_weight_max": 17.452993392944336,
      "activations/layer1_attention_weight_min": -17.655506134033203,
      "activations/layer20_attention_weight_max": 21.39789581298828,
      "activations/layer20_attention_weight_min": -20.531494140625,
      "activations/layer21_attention_weight_max": 34.885826110839844,
      "activations/layer21_attention_weight_min": -23.479625701904297,
      "activations/layer22_attention_weight_max": 27.23946189880371,
      "activations/layer22_attention_weight_min": -25.0447940826416,
      "activations/layer23_attention_weight_max": 34.131011962890625,
      "activations/layer23_attention_weight_min": -24.77130126953125,
      "activations/layer2_attention_weight_max": 32.3465690612793,
      "activations/layer2_attention_weight_min": -30.530851364135742,
      "activations/layer3_attention_weight_max": 90.21786499023438,
      "activations/layer3_attention_weight_min": -91.32169342041016,
      "activations/layer4_attention_weight_max": 96.2199935913086,
      "activations/layer4_attention_weight_min": -92.94690704345703,
      "activations/layer5_attention_weight_max": 71.99776458740234,
      "activations/layer5_attention_weight_min": -79.26318359375,
      "activations/layer6_attention_weight_max": 53.328025817871094,
      "activations/layer6_attention_weight_min": -53.63660430908203,
      "activations/layer7_attention_weight_max": 73.74536895751953,
      "activations/layer7_attention_weight_min": -67.45945739746094,
      "activations/layer8_attention_weight_max": 45.12690353393555,
      "activations/layer8_attention_weight_min": -47.4451904296875,
      "activations/layer9_attention_weight_max": 52.350311279296875,
      "activations/layer9_attention_weight_min": -47.503108978271484,
      "epoch": 11.97,
      "learning_rate": 7.357045454545454e-05,
      "loss": 2.7756,
      "step": 205950
    },
    {
      "activations/layer0_attention_weight_max": 15.367937088012695,
      "activations/layer0_attention_weight_min": -14.417969703674316,
      "activations/layer10_attention_weight_max": 35.907203674316406,
      "activations/layer10_attention_weight_min": -39.308441162109375,
      "activations/layer11_attention_weight_max": 31.452795028686523,
      "activations/layer11_attention_weight_min": -34.56742477416992,
      "activations/layer12_attention_weight_max": 18.77630615234375,
      "activations/layer12_attention_weight_min": -26.147432327270508,
      "activations/layer13_attention_weight_max": 39.98573684692383,
      "activations/layer13_attention_weight_min": -32.64045715332031,
      "activations/layer14_attention_weight_max": 43.52324295043945,
      "activations/layer14_attention_weight_min": -32.03468322753906,
      "activations/layer15_attention_weight_max": 37.0997428894043,
      "activations/layer15_attention_weight_min": -29.958585739135742,
      "activations/layer16_attention_weight_max": 38.95156478881836,
      "activations/layer16_attention_weight_min": -33.80143356323242,
      "activations/layer17_attention_weight_max": 59.1667366027832,
      "activations/layer17_attention_weight_min": -50.9743766784668,
      "activations/layer18_attention_weight_max": 57.30790710449219,
      "activations/layer18_attention_weight_min": -42.177268981933594,
      "activations/layer19_attention_weight_max": 26.923856735229492,
      "activations/layer19_attention_weight_min": -25.42774772644043,
      "activations/layer1_attention_weight_max": 17.88967514038086,
      "activations/layer1_attention_weight_min": -16.255279541015625,
      "activations/layer20_attention_weight_max": 26.636882781982422,
      "activations/layer20_attention_weight_min": -24.177509307861328,
      "activations/layer21_attention_weight_max": 51.64084243774414,
      "activations/layer21_attention_weight_min": -22.73244285583496,
      "activations/layer22_attention_weight_max": 39.630863189697266,
      "activations/layer22_attention_weight_min": -27.945209503173828,
      "activations/layer23_attention_weight_max": 44.058349609375,
      "activations/layer23_attention_weight_min": -25.60881805419922,
      "activations/layer2_attention_weight_max": 32.38523483276367,
      "activations/layer2_attention_weight_min": -31.922039031982422,
      "activations/layer3_attention_weight_max": 93.06068420410156,
      "activations/layer3_attention_weight_min": -91.76573181152344,
      "activations/layer4_attention_weight_max": 97.94942474365234,
      "activations/layer4_attention_weight_min": -93.47990417480469,
      "activations/layer5_attention_weight_max": 70.30804443359375,
      "activations/layer5_attention_weight_min": -77.54241943359375,
      "activations/layer6_attention_weight_max": 53.3371467590332,
      "activations/layer6_attention_weight_min": -51.62638473510742,
      "activations/layer7_attention_weight_max": 74.35913848876953,
      "activations/layer7_attention_weight_min": -65.48300170898438,
      "activations/layer8_attention_weight_max": 44.373600006103516,
      "activations/layer8_attention_weight_min": -45.732391357421875,
      "activations/layer9_attention_weight_max": 44.9865837097168,
      "activations/layer9_attention_weight_min": -46.68387985229492,
      "epoch": 11.97,
      "learning_rate": 7.355151515151514e-05,
      "loss": 2.7599,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.4905,
      "eval_samples_per_second": 505.743,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.4905,
      "eval_openwebtext_samples_per_second": 505.743,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0099,
      "eval_wikitext_samples_per_second": 226.874,
      "step": 206000
    },
    {
      "epoch": 11.97,
      "eval_lambada_loss": 2.6484375,
      "eval_lambada_ppl": 14.131940228876227,
      "eval_lambada_runtime": 9.5938,
      "eval_lambada_samples_per_second": 507.516,
      "step": 206000
    },
    {
      "activations/layer0_attention_weight_max": 16.225324630737305,
      "activations/layer0_attention_weight_min": -14.574638366699219,
      "activations/layer10_attention_weight_max": 39.916259765625,
      "activations/layer10_attention_weight_min": -39.354305267333984,
      "activations/layer11_attention_weight_max": 37.60728454589844,
      "activations/layer11_attention_weight_min": -34.689048767089844,
      "activations/layer12_attention_weight_max": 18.424461364746094,
      "activations/layer12_attention_weight_min": -25.062210083007812,
      "activations/layer13_attention_weight_max": 46.282108306884766,
      "activations/layer13_attention_weight_min": -29.783348083496094,
      "activations/layer14_attention_weight_max": 37.557456970214844,
      "activations/layer14_attention_weight_min": -32.3293342590332,
      "activations/layer15_attention_weight_max": 35.150413513183594,
      "activations/layer15_attention_weight_min": -31.330026626586914,
      "activations/layer16_attention_weight_max": 31.152055740356445,
      "activations/layer16_attention_weight_min": -29.87173843383789,
      "activations/layer17_attention_weight_max": 53.60556411743164,
      "activations/layer17_attention_weight_min": -46.240116119384766,
      "activations/layer18_attention_weight_max": 47.612060546875,
      "activations/layer18_attention_weight_min": -39.0063591003418,
      "activations/layer19_attention_weight_max": 23.43494987487793,
      "activations/layer19_attention_weight_min": -23.286006927490234,
      "activations/layer1_attention_weight_max": 18.906343460083008,
      "activations/layer1_attention_weight_min": -15.160133361816406,
      "activations/layer20_attention_weight_max": 25.52779197692871,
      "activations/layer20_attention_weight_min": -22.570363998413086,
      "activations/layer21_attention_weight_max": 42.61323547363281,
      "activations/layer21_attention_weight_min": -26.85019302368164,
      "activations/layer22_attention_weight_max": 29.167633056640625,
      "activations/layer22_attention_weight_min": -25.71986198425293,
      "activations/layer23_attention_weight_max": 40.432838439941406,
      "activations/layer23_attention_weight_min": -27.09810447692871,
      "activations/layer2_attention_weight_max": 34.03852081298828,
      "activations/layer2_attention_weight_min": -33.425018310546875,
      "activations/layer3_attention_weight_max": 96.76191711425781,
      "activations/layer3_attention_weight_min": -99.09159851074219,
      "activations/layer4_attention_weight_max": 101.4455795288086,
      "activations/layer4_attention_weight_min": -101.0387191772461,
      "activations/layer5_attention_weight_max": 71.54202270507812,
      "activations/layer5_attention_weight_min": -77.36776733398438,
      "activations/layer6_attention_weight_max": 52.90073013305664,
      "activations/layer6_attention_weight_min": -55.25865173339844,
      "activations/layer7_attention_weight_max": 78.71998596191406,
      "activations/layer7_attention_weight_min": -70.48638153076172,
      "activations/layer8_attention_weight_max": 48.37645721435547,
      "activations/layer8_attention_weight_min": -51.651268005371094,
      "activations/layer9_attention_weight_max": 57.48158645629883,
      "activations/layer9_attention_weight_min": -47.753597259521484,
      "epoch": 11.97,
      "learning_rate": 7.353257575757576e-05,
      "loss": 2.7711,
      "step": 206050
    },
    {
      "activations/layer0_attention_weight_max": 16.452594757080078,
      "activations/layer0_attention_weight_min": -14.507440567016602,
      "activations/layer10_attention_weight_max": 37.87084197998047,
      "activations/layer10_attention_weight_min": -35.87211227416992,
      "activations/layer11_attention_weight_max": 34.410526275634766,
      "activations/layer11_attention_weight_min": -34.01779556274414,
      "activations/layer12_attention_weight_max": 22.311798095703125,
      "activations/layer12_attention_weight_min": -24.488788604736328,
      "activations/layer13_attention_weight_max": 39.63732147216797,
      "activations/layer13_attention_weight_min": -29.242538452148438,
      "activations/layer14_attention_weight_max": 41.90867233276367,
      "activations/layer14_attention_weight_min": -31.180809020996094,
      "activations/layer15_attention_weight_max": 36.47300720214844,
      "activations/layer15_attention_weight_min": -29.30992317199707,
      "activations/layer16_attention_weight_max": 35.36103057861328,
      "activations/layer16_attention_weight_min": -29.316482543945312,
      "activations/layer17_attention_weight_max": 59.63660430908203,
      "activations/layer17_attention_weight_min": -47.395660400390625,
      "activations/layer18_attention_weight_max": 54.8028678894043,
      "activations/layer18_attention_weight_min": -38.82246017456055,
      "activations/layer19_attention_weight_max": 26.554777145385742,
      "activations/layer19_attention_weight_min": -22.323453903198242,
      "activations/layer1_attention_weight_max": 16.738800048828125,
      "activations/layer1_attention_weight_min": -15.113929748535156,
      "activations/layer20_attention_weight_max": 26.085599899291992,
      "activations/layer20_attention_weight_min": -24.975889205932617,
      "activations/layer21_attention_weight_max": 44.78651428222656,
      "activations/layer21_attention_weight_min": -28.074295043945312,
      "activations/layer22_attention_weight_max": 36.596778869628906,
      "activations/layer22_attention_weight_min": -26.2468318939209,
      "activations/layer23_attention_weight_max": 44.07133102416992,
      "activations/layer23_attention_weight_min": -23.12228012084961,
      "activations/layer2_attention_weight_max": 32.0010986328125,
      "activations/layer2_attention_weight_min": -31.72115135192871,
      "activations/layer3_attention_weight_max": 93.50472259521484,
      "activations/layer3_attention_weight_min": -94.91436767578125,
      "activations/layer4_attention_weight_max": 100.29096221923828,
      "activations/layer4_attention_weight_min": -99.4365463256836,
      "activations/layer5_attention_weight_max": 76.3471908569336,
      "activations/layer5_attention_weight_min": -77.96186828613281,
      "activations/layer6_attention_weight_max": 55.104759216308594,
      "activations/layer6_attention_weight_min": -55.84593200683594,
      "activations/layer7_attention_weight_max": 72.35846710205078,
      "activations/layer7_attention_weight_min": -68.53512573242188,
      "activations/layer8_attention_weight_max": 48.60187530517578,
      "activations/layer8_attention_weight_min": -47.89589309692383,
      "activations/layer9_attention_weight_max": 47.40516662597656,
      "activations/layer9_attention_weight_min": -47.07781982421875,
      "epoch": 11.98,
      "learning_rate": 7.351363636363636e-05,
      "loss": 2.7692,
      "step": 206100
    },
    {
      "activations/layer0_attention_weight_max": 16.314165115356445,
      "activations/layer0_attention_weight_min": -14.46352767944336,
      "activations/layer10_attention_weight_max": 38.178218841552734,
      "activations/layer10_attention_weight_min": -37.931922912597656,
      "activations/layer11_attention_weight_max": 33.66856384277344,
      "activations/layer11_attention_weight_min": -32.7552375793457,
      "activations/layer12_attention_weight_max": 20.733022689819336,
      "activations/layer12_attention_weight_min": -25.041114807128906,
      "activations/layer13_attention_weight_max": 36.84284973144531,
      "activations/layer13_attention_weight_min": -27.030296325683594,
      "activations/layer14_attention_weight_max": 39.31848907470703,
      "activations/layer14_attention_weight_min": -29.26319694519043,
      "activations/layer15_attention_weight_max": 38.55934524536133,
      "activations/layer15_attention_weight_min": -30.182939529418945,
      "activations/layer16_attention_weight_max": 36.34766387939453,
      "activations/layer16_attention_weight_min": -29.8022403717041,
      "activations/layer17_attention_weight_max": 57.35667037963867,
      "activations/layer17_attention_weight_min": -46.694400787353516,
      "activations/layer18_attention_weight_max": 53.66790008544922,
      "activations/layer18_attention_weight_min": -39.118255615234375,
      "activations/layer19_attention_weight_max": 23.255569458007812,
      "activations/layer19_attention_weight_min": -22.732358932495117,
      "activations/layer1_attention_weight_max": 16.417274475097656,
      "activations/layer1_attention_weight_min": -13.855238914489746,
      "activations/layer20_attention_weight_max": 23.987266540527344,
      "activations/layer20_attention_weight_min": -24.107091903686523,
      "activations/layer21_attention_weight_max": 41.63960266113281,
      "activations/layer21_attention_weight_min": -22.62739372253418,
      "activations/layer22_attention_weight_max": 35.413997650146484,
      "activations/layer22_attention_weight_min": -25.16208267211914,
      "activations/layer23_attention_weight_max": 43.97311019897461,
      "activations/layer23_attention_weight_min": -24.636388778686523,
      "activations/layer2_attention_weight_max": 32.86840057373047,
      "activations/layer2_attention_weight_min": -31.616268157958984,
      "activations/layer3_attention_weight_max": 96.40913391113281,
      "activations/layer3_attention_weight_min": -96.13987731933594,
      "activations/layer4_attention_weight_max": 99.62853240966797,
      "activations/layer4_attention_weight_min": -95.0040512084961,
      "activations/layer5_attention_weight_max": 69.41197204589844,
      "activations/layer5_attention_weight_min": -75.61947631835938,
      "activations/layer6_attention_weight_max": 51.43929672241211,
      "activations/layer6_attention_weight_min": -53.42137145996094,
      "activations/layer7_attention_weight_max": 69.75590515136719,
      "activations/layer7_attention_weight_min": -66.04331970214844,
      "activations/layer8_attention_weight_max": 44.9329719543457,
      "activations/layer8_attention_weight_min": -47.320945739746094,
      "activations/layer9_attention_weight_max": 47.570045471191406,
      "activations/layer9_attention_weight_min": -48.86431121826172,
      "epoch": 11.98,
      "learning_rate": 7.349469696969696e-05,
      "loss": 2.7829,
      "step": 206150
    },
    {
      "activations/layer0_attention_weight_max": 16.409875869750977,
      "activations/layer0_attention_weight_min": -14.741559028625488,
      "activations/layer10_attention_weight_max": 40.20298767089844,
      "activations/layer10_attention_weight_min": -36.9238166809082,
      "activations/layer11_attention_weight_max": 38.78985595703125,
      "activations/layer11_attention_weight_min": -35.79501724243164,
      "activations/layer12_attention_weight_max": 17.078033447265625,
      "activations/layer12_attention_weight_min": -26.234773635864258,
      "activations/layer13_attention_weight_max": 39.2557487487793,
      "activations/layer13_attention_weight_min": -29.784996032714844,
      "activations/layer14_attention_weight_max": 45.625282287597656,
      "activations/layer14_attention_weight_min": -29.636981964111328,
      "activations/layer15_attention_weight_max": 46.04494094848633,
      "activations/layer15_attention_weight_min": -28.822269439697266,
      "activations/layer16_attention_weight_max": 30.959196090698242,
      "activations/layer16_attention_weight_min": -30.011064529418945,
      "activations/layer17_attention_weight_max": 55.35832595825195,
      "activations/layer17_attention_weight_min": -47.229278564453125,
      "activations/layer18_attention_weight_max": 46.58094024658203,
      "activations/layer18_attention_weight_min": -39.124446868896484,
      "activations/layer19_attention_weight_max": 22.384967803955078,
      "activations/layer19_attention_weight_min": -21.598039627075195,
      "activations/layer1_attention_weight_max": 18.507062911987305,
      "activations/layer1_attention_weight_min": -15.148242950439453,
      "activations/layer20_attention_weight_max": 29.304668426513672,
      "activations/layer20_attention_weight_min": -22.037508010864258,
      "activations/layer21_attention_weight_max": 43.84614944458008,
      "activations/layer21_attention_weight_min": -21.25516128540039,
      "activations/layer22_attention_weight_max": 33.82078552246094,
      "activations/layer22_attention_weight_min": -28.001556396484375,
      "activations/layer23_attention_weight_max": 36.47344970703125,
      "activations/layer23_attention_weight_min": -22.65555191040039,
      "activations/layer2_attention_weight_max": 35.31446075439453,
      "activations/layer2_attention_weight_min": -32.54197692871094,
      "activations/layer3_attention_weight_max": 95.46015167236328,
      "activations/layer3_attention_weight_min": -99.67613983154297,
      "activations/layer4_attention_weight_max": 104.219970703125,
      "activations/layer4_attention_weight_min": -100.95833587646484,
      "activations/layer5_attention_weight_max": 78.50703430175781,
      "activations/layer5_attention_weight_min": -78.58767700195312,
      "activations/layer6_attention_weight_max": 53.3682746887207,
      "activations/layer6_attention_weight_min": -54.857948303222656,
      "activations/layer7_attention_weight_max": 83.21339416503906,
      "activations/layer7_attention_weight_min": -75.01510620117188,
      "activations/layer8_attention_weight_max": 52.376461029052734,
      "activations/layer8_attention_weight_min": -49.00084686279297,
      "activations/layer9_attention_weight_max": 64.06819915771484,
      "activations/layer9_attention_weight_min": -52.464599609375,
      "epoch": 11.98,
      "learning_rate": 7.347575757575758e-05,
      "loss": 2.7788,
      "step": 206200
    },
    {
      "activations/layer0_attention_weight_max": 17.099992752075195,
      "activations/layer0_attention_weight_min": -13.867901802062988,
      "activations/layer10_attention_weight_max": 38.82234573364258,
      "activations/layer10_attention_weight_min": -36.717376708984375,
      "activations/layer11_attention_weight_max": 32.878265380859375,
      "activations/layer11_attention_weight_min": -33.130184173583984,
      "activations/layer12_attention_weight_max": 18.804866790771484,
      "activations/layer12_attention_weight_min": -24.063838958740234,
      "activations/layer13_attention_weight_max": 37.1127815246582,
      "activations/layer13_attention_weight_min": -30.13094711303711,
      "activations/layer14_attention_weight_max": 39.51081085205078,
      "activations/layer14_attention_weight_min": -29.60304832458496,
      "activations/layer15_attention_weight_max": 33.64872741699219,
      "activations/layer15_attention_weight_min": -30.6334171295166,
      "activations/layer16_attention_weight_max": 32.270538330078125,
      "activations/layer16_attention_weight_min": -29.99486541748047,
      "activations/layer17_attention_weight_max": 50.9679069519043,
      "activations/layer17_attention_weight_min": -43.71328353881836,
      "activations/layer18_attention_weight_max": 49.98310089111328,
      "activations/layer18_attention_weight_min": -37.757293701171875,
      "activations/layer19_attention_weight_max": 25.39263343811035,
      "activations/layer19_attention_weight_min": -22.42032814025879,
      "activations/layer1_attention_weight_max": 18.150150299072266,
      "activations/layer1_attention_weight_min": -14.936355590820312,
      "activations/layer20_attention_weight_max": 23.393346786499023,
      "activations/layer20_attention_weight_min": -22.72395896911621,
      "activations/layer21_attention_weight_max": 39.04707717895508,
      "activations/layer21_attention_weight_min": -24.723724365234375,
      "activations/layer22_attention_weight_max": 31.399003982543945,
      "activations/layer22_attention_weight_min": -27.517494201660156,
      "activations/layer23_attention_weight_max": 39.66692352294922,
      "activations/layer23_attention_weight_min": -31.065526962280273,
      "activations/layer2_attention_weight_max": 35.57358169555664,
      "activations/layer2_attention_weight_min": -33.04789733886719,
      "activations/layer3_attention_weight_max": 98.12383270263672,
      "activations/layer3_attention_weight_min": -98.62782287597656,
      "activations/layer4_attention_weight_max": 103.9131088256836,
      "activations/layer4_attention_weight_min": -94.95226287841797,
      "activations/layer5_attention_weight_max": 73.52806091308594,
      "activations/layer5_attention_weight_min": -79.55866241455078,
      "activations/layer6_attention_weight_max": 54.880558013916016,
      "activations/layer6_attention_weight_min": -52.61692810058594,
      "activations/layer7_attention_weight_max": 74.41806030273438,
      "activations/layer7_attention_weight_min": -67.3633804321289,
      "activations/layer8_attention_weight_max": 49.31932830810547,
      "activations/layer8_attention_weight_min": -47.96005630493164,
      "activations/layer9_attention_weight_max": 48.950462341308594,
      "activations/layer9_attention_weight_min": -46.548519134521484,
      "epoch": 11.98,
      "learning_rate": 7.345681818181818e-05,
      "loss": 2.7932,
      "step": 206250
    },
    {
      "activations/layer0_attention_weight_max": 15.225674629211426,
      "activations/layer0_attention_weight_min": -14.115642547607422,
      "activations/layer10_attention_weight_max": 33.72725296020508,
      "activations/layer10_attention_weight_min": -35.110504150390625,
      "activations/layer11_attention_weight_max": 30.802349090576172,
      "activations/layer11_attention_weight_min": -34.178741455078125,
      "activations/layer12_attention_weight_max": 16.792043685913086,
      "activations/layer12_attention_weight_min": -30.943763732910156,
      "activations/layer13_attention_weight_max": 33.72383117675781,
      "activations/layer13_attention_weight_min": -28.27630043029785,
      "activations/layer14_attention_weight_max": 35.45196533203125,
      "activations/layer14_attention_weight_min": -28.642282485961914,
      "activations/layer15_attention_weight_max": 33.57899856567383,
      "activations/layer15_attention_weight_min": -29.462684631347656,
      "activations/layer16_attention_weight_max": 31.7841796875,
      "activations/layer16_attention_weight_min": -30.032245635986328,
      "activations/layer17_attention_weight_max": 53.53523635864258,
      "activations/layer17_attention_weight_min": -42.99638748168945,
      "activations/layer18_attention_weight_max": 51.25834274291992,
      "activations/layer18_attention_weight_min": -36.30935287475586,
      "activations/layer19_attention_weight_max": 21.1086368560791,
      "activations/layer19_attention_weight_min": -19.910871505737305,
      "activations/layer1_attention_weight_max": 16.80477523803711,
      "activations/layer1_attention_weight_min": -14.809778213500977,
      "activations/layer20_attention_weight_max": 19.867650985717773,
      "activations/layer20_attention_weight_min": -22.74515724182129,
      "activations/layer21_attention_weight_max": 31.91135025024414,
      "activations/layer21_attention_weight_min": -21.695005416870117,
      "activations/layer22_attention_weight_max": 32.2972297668457,
      "activations/layer22_attention_weight_min": -23.773550033569336,
      "activations/layer23_attention_weight_max": 34.585365295410156,
      "activations/layer23_attention_weight_min": -26.533092498779297,
      "activations/layer2_attention_weight_max": 33.48430633544922,
      "activations/layer2_attention_weight_min": -30.450069427490234,
      "activations/layer3_attention_weight_max": 95.2381362915039,
      "activations/layer3_attention_weight_min": -94.72406005859375,
      "activations/layer4_attention_weight_max": 97.5174789428711,
      "activations/layer4_attention_weight_min": -93.06099700927734,
      "activations/layer5_attention_weight_max": 70.60430908203125,
      "activations/layer5_attention_weight_min": -78.79425048828125,
      "activations/layer6_attention_weight_max": 49.925514221191406,
      "activations/layer6_attention_weight_min": -53.95110321044922,
      "activations/layer7_attention_weight_max": 67.75109100341797,
      "activations/layer7_attention_weight_min": -64.98594665527344,
      "activations/layer8_attention_weight_max": 44.17421340942383,
      "activations/layer8_attention_weight_min": -44.71884536743164,
      "activations/layer9_attention_weight_max": 43.948646545410156,
      "activations/layer9_attention_weight_min": -46.30254364013672,
      "epoch": 11.99,
      "learning_rate": 7.343787878787878e-05,
      "loss": 2.7698,
      "step": 206300
    },
    {
      "activations/layer0_attention_weight_max": 16.065876007080078,
      "activations/layer0_attention_weight_min": -13.993103981018066,
      "activations/layer10_attention_weight_max": 48.10858917236328,
      "activations/layer10_attention_weight_min": -45.91093826293945,
      "activations/layer11_attention_weight_max": 46.846885681152344,
      "activations/layer11_attention_weight_min": -42.12034225463867,
      "activations/layer12_attention_weight_max": 22.60149574279785,
      "activations/layer12_attention_weight_min": -31.41250991821289,
      "activations/layer13_attention_weight_max": 43.6246452331543,
      "activations/layer13_attention_weight_min": -31.59857177734375,
      "activations/layer14_attention_weight_max": 50.973663330078125,
      "activations/layer14_attention_weight_min": -31.657726287841797,
      "activations/layer15_attention_weight_max": 41.3626594543457,
      "activations/layer15_attention_weight_min": -30.445941925048828,
      "activations/layer16_attention_weight_max": 35.33047103881836,
      "activations/layer16_attention_weight_min": -31.34346580505371,
      "activations/layer17_attention_weight_max": 55.3659553527832,
      "activations/layer17_attention_weight_min": -49.276214599609375,
      "activations/layer18_attention_weight_max": 54.031776428222656,
      "activations/layer18_attention_weight_min": -41.57742691040039,
      "activations/layer19_attention_weight_max": 24.125789642333984,
      "activations/layer19_attention_weight_min": -23.776044845581055,
      "activations/layer1_attention_weight_max": 18.721237182617188,
      "activations/layer1_attention_weight_min": -14.498697280883789,
      "activations/layer20_attention_weight_max": 27.67840576171875,
      "activations/layer20_attention_weight_min": -20.5350399017334,
      "activations/layer21_attention_weight_max": 37.78435516357422,
      "activations/layer21_attention_weight_min": -20.87274932861328,
      "activations/layer22_attention_weight_max": 35.28359603881836,
      "activations/layer22_attention_weight_min": -25.176708221435547,
      "activations/layer23_attention_weight_max": 40.999237060546875,
      "activations/layer23_attention_weight_min": -22.18880844116211,
      "activations/layer2_attention_weight_max": 34.735557556152344,
      "activations/layer2_attention_weight_min": -32.296051025390625,
      "activations/layer3_attention_weight_max": 100.20968627929688,
      "activations/layer3_attention_weight_min": -100.46858215332031,
      "activations/layer4_attention_weight_max": 112.33088684082031,
      "activations/layer4_attention_weight_min": -103.20051574707031,
      "activations/layer5_attention_weight_max": 81.32884216308594,
      "activations/layer5_attention_weight_min": -79.8645248413086,
      "activations/layer6_attention_weight_max": 59.21231460571289,
      "activations/layer6_attention_weight_min": -56.63893127441406,
      "activations/layer7_attention_weight_max": 94.94567108154297,
      "activations/layer7_attention_weight_min": -80.02216339111328,
      "activations/layer8_attention_weight_max": 61.58749771118164,
      "activations/layer8_attention_weight_min": -55.0878791809082,
      "activations/layer9_attention_weight_max": 73.9389419555664,
      "activations/layer9_attention_weight_min": -60.342891693115234,
      "epoch": 11.99,
      "learning_rate": 7.34189393939394e-05,
      "loss": 2.7793,
      "step": 206350
    },
    {
      "activations/layer0_attention_weight_max": 15.869012832641602,
      "activations/layer0_attention_weight_min": -13.717679977416992,
      "activations/layer10_attention_weight_max": 39.42957305908203,
      "activations/layer10_attention_weight_min": -37.90629577636719,
      "activations/layer11_attention_weight_max": 34.132240295410156,
      "activations/layer11_attention_weight_min": -34.19670104980469,
      "activations/layer12_attention_weight_max": 18.712247848510742,
      "activations/layer12_attention_weight_min": -23.748571395874023,
      "activations/layer13_attention_weight_max": 50.115867614746094,
      "activations/layer13_attention_weight_min": -34.905574798583984,
      "activations/layer14_attention_weight_max": 37.872562408447266,
      "activations/layer14_attention_weight_min": -30.47825813293457,
      "activations/layer15_attention_weight_max": 34.692901611328125,
      "activations/layer15_attention_weight_min": -28.39150047302246,
      "activations/layer16_attention_weight_max": 32.757911682128906,
      "activations/layer16_attention_weight_min": -31.864416122436523,
      "activations/layer17_attention_weight_max": 49.23149871826172,
      "activations/layer17_attention_weight_min": -43.18898010253906,
      "activations/layer18_attention_weight_max": 43.42801284790039,
      "activations/layer18_attention_weight_min": -37.84758758544922,
      "activations/layer19_attention_weight_max": 25.30901527404785,
      "activations/layer19_attention_weight_min": -21.08690643310547,
      "activations/layer1_attention_weight_max": 16.983291625976562,
      "activations/layer1_attention_weight_min": -15.849464416503906,
      "activations/layer20_attention_weight_max": 23.925199508666992,
      "activations/layer20_attention_weight_min": -22.09383773803711,
      "activations/layer21_attention_weight_max": 36.39665603637695,
      "activations/layer21_attention_weight_min": -27.716257095336914,
      "activations/layer22_attention_weight_max": 34.372859954833984,
      "activations/layer22_attention_weight_min": -27.09534454345703,
      "activations/layer23_attention_weight_max": 39.63222122192383,
      "activations/layer23_attention_weight_min": -24.443836212158203,
      "activations/layer2_attention_weight_max": 33.89794921875,
      "activations/layer2_attention_weight_min": -32.07008743286133,
      "activations/layer3_attention_weight_max": 95.8630599975586,
      "activations/layer3_attention_weight_min": -92.48958587646484,
      "activations/layer4_attention_weight_max": 102.13712310791016,
      "activations/layer4_attention_weight_min": -101.54843139648438,
      "activations/layer5_attention_weight_max": 74.95664978027344,
      "activations/layer5_attention_weight_min": -82.19402313232422,
      "activations/layer6_attention_weight_max": 55.25735855102539,
      "activations/layer6_attention_weight_min": -54.98975372314453,
      "activations/layer7_attention_weight_max": 71.45915222167969,
      "activations/layer7_attention_weight_min": -69.76036834716797,
      "activations/layer8_attention_weight_max": 49.78573989868164,
      "activations/layer8_attention_weight_min": -50.68709182739258,
      "activations/layer9_attention_weight_max": 49.039302825927734,
      "activations/layer9_attention_weight_min": -49.656795501708984,
      "epoch": 11.99,
      "learning_rate": 7.34e-05,
      "loss": 2.7801,
      "step": 206400
    },
    {
      "activations/layer0_attention_weight_max": 15.396435737609863,
      "activations/layer0_attention_weight_min": -14.079886436462402,
      "activations/layer10_attention_weight_max": 39.095436096191406,
      "activations/layer10_attention_weight_min": -37.59806442260742,
      "activations/layer11_attention_weight_max": 32.93408966064453,
      "activations/layer11_attention_weight_min": -33.14773941040039,
      "activations/layer12_attention_weight_max": 18.11166763305664,
      "activations/layer12_attention_weight_min": -27.089771270751953,
      "activations/layer13_attention_weight_max": 34.47840118408203,
      "activations/layer13_attention_weight_min": -27.75151252746582,
      "activations/layer14_attention_weight_max": 35.17363357543945,
      "activations/layer14_attention_weight_min": -28.398422241210938,
      "activations/layer15_attention_weight_max": 30.77693748474121,
      "activations/layer15_attention_weight_min": -28.14669418334961,
      "activations/layer16_attention_weight_max": 32.08804702758789,
      "activations/layer16_attention_weight_min": -29.159648895263672,
      "activations/layer17_attention_weight_max": 48.8393440246582,
      "activations/layer17_attention_weight_min": -43.77429962158203,
      "activations/layer18_attention_weight_max": 43.19823455810547,
      "activations/layer18_attention_weight_min": -40.729095458984375,
      "activations/layer19_attention_weight_max": 23.184720993041992,
      "activations/layer19_attention_weight_min": -21.555988311767578,
      "activations/layer1_attention_weight_max": 16.98792839050293,
      "activations/layer1_attention_weight_min": -14.835075378417969,
      "activations/layer20_attention_weight_max": 20.26434326171875,
      "activations/layer20_attention_weight_min": -21.361894607543945,
      "activations/layer21_attention_weight_max": 33.51194763183594,
      "activations/layer21_attention_weight_min": -19.967554092407227,
      "activations/layer22_attention_weight_max": 29.706344604492188,
      "activations/layer22_attention_weight_min": -26.983388900756836,
      "activations/layer23_attention_weight_max": 33.51136016845703,
      "activations/layer23_attention_weight_min": -23.091232299804688,
      "activations/layer2_attention_weight_max": 33.54735565185547,
      "activations/layer2_attention_weight_min": -32.43897247314453,
      "activations/layer3_attention_weight_max": 98.21337890625,
      "activations/layer3_attention_weight_min": -93.70940399169922,
      "activations/layer4_attention_weight_max": 101.80941009521484,
      "activations/layer4_attention_weight_min": -95.8766098022461,
      "activations/layer5_attention_weight_max": 72.34922790527344,
      "activations/layer5_attention_weight_min": -78.66747283935547,
      "activations/layer6_attention_weight_max": 53.867469787597656,
      "activations/layer6_attention_weight_min": -54.39902877807617,
      "activations/layer7_attention_weight_max": 71.11743927001953,
      "activations/layer7_attention_weight_min": -68.40133666992188,
      "activations/layer8_attention_weight_max": 45.90398025512695,
      "activations/layer8_attention_weight_min": -47.950679779052734,
      "activations/layer9_attention_weight_max": 47.20144271850586,
      "activations/layer9_attention_weight_min": -46.528167724609375,
      "epoch": 12.0,
      "learning_rate": 7.33810606060606e-05,
      "loss": 2.7729,
      "step": 206450
    },
    {
      "activations/layer0_attention_weight_max": 16.330120086669922,
      "activations/layer0_attention_weight_min": -14.180428504943848,
      "activations/layer10_attention_weight_max": 36.017765045166016,
      "activations/layer10_attention_weight_min": -35.39154052734375,
      "activations/layer11_attention_weight_max": 31.660964965820312,
      "activations/layer11_attention_weight_min": -31.957273483276367,
      "activations/layer12_attention_weight_max": 19.435409545898438,
      "activations/layer12_attention_weight_min": -25.21729278564453,
      "activations/layer13_attention_weight_max": 38.38172912597656,
      "activations/layer13_attention_weight_min": -29.76911163330078,
      "activations/layer14_attention_weight_max": 34.76127624511719,
      "activations/layer14_attention_weight_min": -30.79625129699707,
      "activations/layer15_attention_weight_max": 30.949787139892578,
      "activations/layer15_attention_weight_min": -30.852083206176758,
      "activations/layer16_attention_weight_max": 29.141986846923828,
      "activations/layer16_attention_weight_min": -28.06352996826172,
      "activations/layer17_attention_weight_max": 48.056060791015625,
      "activations/layer17_attention_weight_min": -42.835968017578125,
      "activations/layer18_attention_weight_max": 42.46620178222656,
      "activations/layer18_attention_weight_min": -36.178916931152344,
      "activations/layer19_attention_weight_max": 20.253877639770508,
      "activations/layer19_attention_weight_min": -19.7733154296875,
      "activations/layer1_attention_weight_max": 16.777616500854492,
      "activations/layer1_attention_weight_min": -14.508783340454102,
      "activations/layer20_attention_weight_max": 21.07366943359375,
      "activations/layer20_attention_weight_min": -23.79527473449707,
      "activations/layer21_attention_weight_max": 37.888214111328125,
      "activations/layer21_attention_weight_min": -23.2125301361084,
      "activations/layer22_attention_weight_max": 30.489274978637695,
      "activations/layer22_attention_weight_min": -26.68587303161621,
      "activations/layer23_attention_weight_max": 42.676795959472656,
      "activations/layer23_attention_weight_min": -25.483604431152344,
      "activations/layer2_attention_weight_max": 33.709869384765625,
      "activations/layer2_attention_weight_min": -33.581268310546875,
      "activations/layer3_attention_weight_max": 96.78113555908203,
      "activations/layer3_attention_weight_min": -104.89568328857422,
      "activations/layer4_attention_weight_max": 98.3927001953125,
      "activations/layer4_attention_weight_min": -96.92681121826172,
      "activations/layer5_attention_weight_max": 73.40760040283203,
      "activations/layer5_attention_weight_min": -75.57826232910156,
      "activations/layer6_attention_weight_max": 54.357364654541016,
      "activations/layer6_attention_weight_min": -51.89125442504883,
      "activations/layer7_attention_weight_max": 71.67790985107422,
      "activations/layer7_attention_weight_min": -66.646484375,
      "activations/layer8_attention_weight_max": 44.46337890625,
      "activations/layer8_attention_weight_min": -46.36865997314453,
      "activations/layer9_attention_weight_max": 45.17586898803711,
      "activations/layer9_attention_weight_min": -44.55693817138672,
      "epoch": 12.0,
      "learning_rate": 7.33621212121212e-05,
      "loss": 2.7686,
      "step": 206500
    },
    {
      "activations/layer0_attention_weight_max": 15.74881649017334,
      "activations/layer0_attention_weight_min": -14.066988945007324,
      "activations/layer10_attention_weight_max": 39.81174087524414,
      "activations/layer10_attention_weight_min": -34.128658294677734,
      "activations/layer11_attention_weight_max": 36.83391571044922,
      "activations/layer11_attention_weight_min": -33.45606994628906,
      "activations/layer12_attention_weight_max": 18.592544555664062,
      "activations/layer12_attention_weight_min": -37.51961898803711,
      "activations/layer13_attention_weight_max": 44.2324104309082,
      "activations/layer13_attention_weight_min": -32.0865478515625,
      "activations/layer14_attention_weight_max": 34.92802429199219,
      "activations/layer14_attention_weight_min": -29.297595977783203,
      "activations/layer15_attention_weight_max": 32.96861267089844,
      "activations/layer15_attention_weight_min": -27.526517868041992,
      "activations/layer16_attention_weight_max": 33.49191665649414,
      "activations/layer16_attention_weight_min": -29.19650650024414,
      "activations/layer17_attention_weight_max": 56.88832092285156,
      "activations/layer17_attention_weight_min": -46.154640197753906,
      "activations/layer18_attention_weight_max": 47.11713790893555,
      "activations/layer18_attention_weight_min": -41.10700607299805,
      "activations/layer19_attention_weight_max": 20.770721435546875,
      "activations/layer19_attention_weight_min": -21.66712760925293,
      "activations/layer1_attention_weight_max": 17.991222381591797,
      "activations/layer1_attention_weight_min": -14.786142349243164,
      "activations/layer20_attention_weight_max": 23.9317684173584,
      "activations/layer20_attention_weight_min": -21.829954147338867,
      "activations/layer21_attention_weight_max": 32.093868255615234,
      "activations/layer21_attention_weight_min": -20.391653060913086,
      "activations/layer22_attention_weight_max": 32.37683868408203,
      "activations/layer22_attention_weight_min": -26.54210662841797,
      "activations/layer23_attention_weight_max": 36.438236236572266,
      "activations/layer23_attention_weight_min": -23.94196128845215,
      "activations/layer2_attention_weight_max": 35.277015686035156,
      "activations/layer2_attention_weight_min": -33.25145721435547,
      "activations/layer3_attention_weight_max": 94.02330780029297,
      "activations/layer3_attention_weight_min": -100.99662780761719,
      "activations/layer4_attention_weight_max": 100.3680648803711,
      "activations/layer4_attention_weight_min": -104.72643280029297,
      "activations/layer5_attention_weight_max": 71.05085754394531,
      "activations/layer5_attention_weight_min": -75.7621078491211,
      "activations/layer6_attention_weight_max": 55.98434829711914,
      "activations/layer6_attention_weight_min": -53.07805633544922,
      "activations/layer7_attention_weight_max": 74.15763854980469,
      "activations/layer7_attention_weight_min": -66.48539733886719,
      "activations/layer8_attention_weight_max": 49.26401901245117,
      "activations/layer8_attention_weight_min": -43.9070930480957,
      "activations/layer9_attention_weight_max": 57.87109375,
      "activations/layer9_attention_weight_min": -49.36632537841797,
      "epoch": 12.0,
      "learning_rate": 7.334280303030302e-05,
      "loss": 2.8296,
      "step": 206550
    },
    {
      "activations/layer0_attention_weight_max": 16.77956199645996,
      "activations/layer0_attention_weight_min": -14.475679397583008,
      "activations/layer10_attention_weight_max": 41.58170700073242,
      "activations/layer10_attention_weight_min": -40.2110710144043,
      "activations/layer11_attention_weight_max": 37.45369338989258,
      "activations/layer11_attention_weight_min": -36.956947326660156,
      "activations/layer12_attention_weight_max": 19.43265151977539,
      "activations/layer12_attention_weight_min": -23.897865295410156,
      "activations/layer13_attention_weight_max": 50.42086410522461,
      "activations/layer13_attention_weight_min": -29.32121467590332,
      "activations/layer14_attention_weight_max": 51.531532287597656,
      "activations/layer14_attention_weight_min": -33.95232009887695,
      "activations/layer15_attention_weight_max": 45.744781494140625,
      "activations/layer15_attention_weight_min": -31.772605895996094,
      "activations/layer16_attention_weight_max": 35.23517608642578,
      "activations/layer16_attention_weight_min": -34.7995719909668,
      "activations/layer17_attention_weight_max": 57.31209182739258,
      "activations/layer17_attention_weight_min": -51.201847076416016,
      "activations/layer18_attention_weight_max": 51.742942810058594,
      "activations/layer18_attention_weight_min": -43.59270477294922,
      "activations/layer19_attention_weight_max": 25.09461784362793,
      "activations/layer19_attention_weight_min": -24.064210891723633,
      "activations/layer1_attention_weight_max": 18.046937942504883,
      "activations/layer1_attention_weight_min": -15.41070556640625,
      "activations/layer20_attention_weight_max": 22.992963790893555,
      "activations/layer20_attention_weight_min": -22.143156051635742,
      "activations/layer21_attention_weight_max": 45.76858901977539,
      "activations/layer21_attention_weight_min": -23.925487518310547,
      "activations/layer22_attention_weight_max": 34.64543151855469,
      "activations/layer22_attention_weight_min": -28.081602096557617,
      "activations/layer23_attention_weight_max": 41.35115051269531,
      "activations/layer23_attention_weight_min": -27.272594451904297,
      "activations/layer2_attention_weight_max": 34.49494171142578,
      "activations/layer2_attention_weight_min": -32.36804962158203,
      "activations/layer3_attention_weight_max": 100.9488296508789,
      "activations/layer3_attention_weight_min": -103.69319915771484,
      "activations/layer4_attention_weight_max": 108.32231903076172,
      "activations/layer4_attention_weight_min": -105.5775375366211,
      "activations/layer5_attention_weight_max": 78.01560974121094,
      "activations/layer5_attention_weight_min": -81.05226135253906,
      "activations/layer6_attention_weight_max": 54.95124435424805,
      "activations/layer6_attention_weight_min": -55.66890335083008,
      "activations/layer7_attention_weight_max": 89.67403411865234,
      "activations/layer7_attention_weight_min": -73.91443634033203,
      "activations/layer8_attention_weight_max": 54.900630950927734,
      "activations/layer8_attention_weight_min": -48.76381301879883,
      "activations/layer9_attention_weight_max": 64.24369812011719,
      "activations/layer9_attention_weight_min": -54.17036056518555,
      "epoch": 12.0,
      "learning_rate": 7.332386363636363e-05,
      "loss": 2.763,
      "step": 206600
    },
    {
      "activations/layer0_attention_weight_max": 16.058359146118164,
      "activations/layer0_attention_weight_min": -14.445100784301758,
      "activations/layer10_attention_weight_max": 35.15222930908203,
      "activations/layer10_attention_weight_min": -34.18779754638672,
      "activations/layer11_attention_weight_max": 31.626447677612305,
      "activations/layer11_attention_weight_min": -31.74441146850586,
      "activations/layer12_attention_weight_max": 20.187740325927734,
      "activations/layer12_attention_weight_min": -23.817718505859375,
      "activations/layer13_attention_weight_max": 38.59444808959961,
      "activations/layer13_attention_weight_min": -30.452157974243164,
      "activations/layer14_attention_weight_max": 38.22187423706055,
      "activations/layer14_attention_weight_min": -30.676788330078125,
      "activations/layer15_attention_weight_max": 34.73511505126953,
      "activations/layer15_attention_weight_min": -29.803590774536133,
      "activations/layer16_attention_weight_max": 32.498172760009766,
      "activations/layer16_attention_weight_min": -29.624326705932617,
      "activations/layer17_attention_weight_max": 55.19938659667969,
      "activations/layer17_attention_weight_min": -43.97624588012695,
      "activations/layer18_attention_weight_max": 49.5380744934082,
      "activations/layer18_attention_weight_min": -37.16056442260742,
      "activations/layer19_attention_weight_max": 21.531064987182617,
      "activations/layer19_attention_weight_min": -22.09219741821289,
      "activations/layer1_attention_weight_max": 16.47753143310547,
      "activations/layer1_attention_weight_min": -16.91600227355957,
      "activations/layer20_attention_weight_max": 21.387737274169922,
      "activations/layer20_attention_weight_min": -23.797142028808594,
      "activations/layer21_attention_weight_max": 41.88115692138672,
      "activations/layer21_attention_weight_min": -22.17375946044922,
      "activations/layer22_attention_weight_max": 29.804264068603516,
      "activations/layer22_attention_weight_min": -24.86988639831543,
      "activations/layer23_attention_weight_max": 40.643882751464844,
      "activations/layer23_attention_weight_min": -25.305387496948242,
      "activations/layer2_attention_weight_max": 33.31338882446289,
      "activations/layer2_attention_weight_min": -31.706342697143555,
      "activations/layer3_attention_weight_max": 92.95142364501953,
      "activations/layer3_attention_weight_min": -95.27099609375,
      "activations/layer4_attention_weight_max": 98.41316986083984,
      "activations/layer4_attention_weight_min": -94.78890228271484,
      "activations/layer5_attention_weight_max": 71.4791030883789,
      "activations/layer5_attention_weight_min": -75.63699340820312,
      "activations/layer6_attention_weight_max": 51.773014068603516,
      "activations/layer6_attention_weight_min": -53.62205123901367,
      "activations/layer7_attention_weight_max": 69.16999053955078,
      "activations/layer7_attention_weight_min": -66.28888702392578,
      "activations/layer8_attention_weight_max": 43.22921371459961,
      "activations/layer8_attention_weight_min": -45.660728454589844,
      "activations/layer9_attention_weight_max": 43.26924133300781,
      "activations/layer9_attention_weight_min": -45.44965362548828,
      "epoch": 12.01,
      "learning_rate": 7.330492424242424e-05,
      "loss": 2.7756,
      "step": 206650
    },
    {
      "activations/layer0_attention_weight_max": 15.927922248840332,
      "activations/layer0_attention_weight_min": -14.230149269104004,
      "activations/layer10_attention_weight_max": 35.682823181152344,
      "activations/layer10_attention_weight_min": -35.388301849365234,
      "activations/layer11_attention_weight_max": 33.141624450683594,
      "activations/layer11_attention_weight_min": -31.450660705566406,
      "activations/layer12_attention_weight_max": 20.23528480529785,
      "activations/layer12_attention_weight_min": -24.692684173583984,
      "activations/layer13_attention_weight_max": 43.49020004272461,
      "activations/layer13_attention_weight_min": -27.476839065551758,
      "activations/layer14_attention_weight_max": 35.85747528076172,
      "activations/layer14_attention_weight_min": -31.201581954956055,
      "activations/layer15_attention_weight_max": 32.69709777832031,
      "activations/layer15_attention_weight_min": -28.639158248901367,
      "activations/layer16_attention_weight_max": 34.987281799316406,
      "activations/layer16_attention_weight_min": -30.606136322021484,
      "activations/layer17_attention_weight_max": 53.49024200439453,
      "activations/layer17_attention_weight_min": -45.72038269042969,
      "activations/layer18_attention_weight_max": 45.51145553588867,
      "activations/layer18_attention_weight_min": -38.017059326171875,
      "activations/layer19_attention_weight_max": 25.006013870239258,
      "activations/layer19_attention_weight_min": -21.675800323486328,
      "activations/layer1_attention_weight_max": 16.713165283203125,
      "activations/layer1_attention_weight_min": -14.84238338470459,
      "activations/layer20_attention_weight_max": 22.95781135559082,
      "activations/layer20_attention_weight_min": -20.571109771728516,
      "activations/layer21_attention_weight_max": 36.299381256103516,
      "activations/layer21_attention_weight_min": -24.07177734375,
      "activations/layer22_attention_weight_max": 32.50349807739258,
      "activations/layer22_attention_weight_min": -24.751569747924805,
      "activations/layer23_attention_weight_max": 39.203243255615234,
      "activations/layer23_attention_weight_min": -22.308317184448242,
      "activations/layer2_attention_weight_max": 33.275848388671875,
      "activations/layer2_attention_weight_min": -31.03192138671875,
      "activations/layer3_attention_weight_max": 93.85021209716797,
      "activations/layer3_attention_weight_min": -95.36690521240234,
      "activations/layer4_attention_weight_max": 102.5546646118164,
      "activations/layer4_attention_weight_min": -98.01277160644531,
      "activations/layer5_attention_weight_max": 75.39159393310547,
      "activations/layer5_attention_weight_min": -80.93171691894531,
      "activations/layer6_attention_weight_max": 53.486656188964844,
      "activations/layer6_attention_weight_min": -51.738670349121094,
      "activations/layer7_attention_weight_max": 70.2637710571289,
      "activations/layer7_attention_weight_min": -68.74910736083984,
      "activations/layer8_attention_weight_max": 47.21879577636719,
      "activations/layer8_attention_weight_min": -45.7907829284668,
      "activations/layer9_attention_weight_max": 47.902156829833984,
      "activations/layer9_attention_weight_min": -47.103797912597656,
      "epoch": 12.01,
      "learning_rate": 7.328598484848484e-05,
      "loss": 2.7638,
      "step": 206700
    },
    {
      "activations/layer0_attention_weight_max": 16.16668128967285,
      "activations/layer0_attention_weight_min": -14.593036651611328,
      "activations/layer10_attention_weight_max": 33.75782012939453,
      "activations/layer10_attention_weight_min": -32.28689956665039,
      "activations/layer11_attention_weight_max": 31.036441802978516,
      "activations/layer11_attention_weight_min": -29.825048446655273,
      "activations/layer12_attention_weight_max": 17.127124786376953,
      "activations/layer12_attention_weight_min": -27.47239112854004,
      "activations/layer13_attention_weight_max": 34.55333709716797,
      "activations/layer13_attention_weight_min": -29.13074493408203,
      "activations/layer14_attention_weight_max": 34.03126525878906,
      "activations/layer14_attention_weight_min": -28.767135620117188,
      "activations/layer15_attention_weight_max": 29.61910057067871,
      "activations/layer15_attention_weight_min": -27.67599868774414,
      "activations/layer16_attention_weight_max": 29.79047966003418,
      "activations/layer16_attention_weight_min": -27.655263900756836,
      "activations/layer17_attention_weight_max": 48.10338592529297,
      "activations/layer17_attention_weight_min": -39.911319732666016,
      "activations/layer18_attention_weight_max": 40.464778900146484,
      "activations/layer18_attention_weight_min": -33.8538932800293,
      "activations/layer19_attention_weight_max": 19.761606216430664,
      "activations/layer19_attention_weight_min": -20.267675399780273,
      "activations/layer1_attention_weight_max": 16.45246124267578,
      "activations/layer1_attention_weight_min": -14.71431827545166,
      "activations/layer20_attention_weight_max": 20.290956497192383,
      "activations/layer20_attention_weight_min": -21.168760299682617,
      "activations/layer21_attention_weight_max": 32.39934158325195,
      "activations/layer21_attention_weight_min": -23.384347915649414,
      "activations/layer22_attention_weight_max": 26.36185646057129,
      "activations/layer22_attention_weight_min": -24.757911682128906,
      "activations/layer23_attention_weight_max": 32.58552169799805,
      "activations/layer23_attention_weight_min": -24.775981903076172,
      "activations/layer2_attention_weight_max": 33.66952896118164,
      "activations/layer2_attention_weight_min": -31.381465911865234,
      "activations/layer3_attention_weight_max": 91.13627624511719,
      "activations/layer3_attention_weight_min": -94.99267578125,
      "activations/layer4_attention_weight_max": 97.22295379638672,
      "activations/layer4_attention_weight_min": -91.05064392089844,
      "activations/layer5_attention_weight_max": 70.57479858398438,
      "activations/layer5_attention_weight_min": -74.7275619506836,
      "activations/layer6_attention_weight_max": 52.4470329284668,
      "activations/layer6_attention_weight_min": -52.63777160644531,
      "activations/layer7_attention_weight_max": 65.35901641845703,
      "activations/layer7_attention_weight_min": -64.43683624267578,
      "activations/layer8_attention_weight_max": 41.83576965332031,
      "activations/layer8_attention_weight_min": -44.478851318359375,
      "activations/layer9_attention_weight_max": 46.05897521972656,
      "activations/layer9_attention_weight_min": -45.44344711303711,
      "epoch": 12.01,
      "learning_rate": 7.326704545454546e-05,
      "loss": 2.7666,
      "step": 206750
    },
    {
      "activations/layer0_attention_weight_max": 15.98534107208252,
      "activations/layer0_attention_weight_min": -14.308714866638184,
      "activations/layer10_attention_weight_max": 39.24707794189453,
      "activations/layer10_attention_weight_min": -33.65212631225586,
      "activations/layer11_attention_weight_max": 32.25690460205078,
      "activations/layer11_attention_weight_min": -30.6264705657959,
      "activations/layer12_attention_weight_max": 19.825153350830078,
      "activations/layer12_attention_weight_min": -23.9775333404541,
      "activations/layer13_attention_weight_max": 41.84105682373047,
      "activations/layer13_attention_weight_min": -31.452728271484375,
      "activations/layer14_attention_weight_max": 36.99801254272461,
      "activations/layer14_attention_weight_min": -29.56353759765625,
      "activations/layer15_attention_weight_max": 34.40097427368164,
      "activations/layer15_attention_weight_min": -28.395286560058594,
      "activations/layer16_attention_weight_max": 34.038330078125,
      "activations/layer16_attention_weight_min": -31.66263771057129,
      "activations/layer17_attention_weight_max": 51.55449295043945,
      "activations/layer17_attention_weight_min": -42.98189926147461,
      "activations/layer18_attention_weight_max": 48.89499282836914,
      "activations/layer18_attention_weight_min": -38.15469741821289,
      "activations/layer19_attention_weight_max": 22.89720344543457,
      "activations/layer19_attention_weight_min": -23.029020309448242,
      "activations/layer1_attention_weight_max": 17.296146392822266,
      "activations/layer1_attention_weight_min": -13.752777099609375,
      "activations/layer20_attention_weight_max": 21.980239868164062,
      "activations/layer20_attention_weight_min": -22.149999618530273,
      "activations/layer21_attention_weight_max": 33.59836196899414,
      "activations/layer21_attention_weight_min": -20.968278884887695,
      "activations/layer22_attention_weight_max": 31.600969314575195,
      "activations/layer22_attention_weight_min": -24.42970085144043,
      "activations/layer23_attention_weight_max": 36.86756134033203,
      "activations/layer23_attention_weight_min": -23.729475021362305,
      "activations/layer2_attention_weight_max": 31.473411560058594,
      "activations/layer2_attention_weight_min": -32.02559280395508,
      "activations/layer3_attention_weight_max": 95.22080993652344,
      "activations/layer3_attention_weight_min": -92.96822357177734,
      "activations/layer4_attention_weight_max": 100.7243881225586,
      "activations/layer4_attention_weight_min": -98.7222671508789,
      "activations/layer5_attention_weight_max": 72.47737121582031,
      "activations/layer5_attention_weight_min": -79.43257141113281,
      "activations/layer6_attention_weight_max": 54.94398880004883,
      "activations/layer6_attention_weight_min": -54.034080505371094,
      "activations/layer7_attention_weight_max": 71.46125793457031,
      "activations/layer7_attention_weight_min": -70.93600463867188,
      "activations/layer8_attention_weight_max": 44.66830062866211,
      "activations/layer8_attention_weight_min": -47.02412414550781,
      "activations/layer9_attention_weight_max": 44.99709701538086,
      "activations/layer9_attention_weight_min": -44.75824737548828,
      "epoch": 12.02,
      "learning_rate": 7.324810606060606e-05,
      "loss": 2.7741,
      "step": 206800
    },
    {
      "activations/layer0_attention_weight_max": 16.313270568847656,
      "activations/layer0_attention_weight_min": -14.39318561553955,
      "activations/layer10_attention_weight_max": 36.2027587890625,
      "activations/layer10_attention_weight_min": -38.529048919677734,
      "activations/layer11_attention_weight_max": 32.112403869628906,
      "activations/layer11_attention_weight_min": -33.08938980102539,
      "activations/layer12_attention_weight_max": 21.5772647857666,
      "activations/layer12_attention_weight_min": -23.959049224853516,
      "activations/layer13_attention_weight_max": 39.33559036254883,
      "activations/layer13_attention_weight_min": -27.41714859008789,
      "activations/layer14_attention_weight_max": 34.59632873535156,
      "activations/layer14_attention_weight_min": -29.60457420349121,
      "activations/layer15_attention_weight_max": 32.62137222290039,
      "activations/layer15_attention_weight_min": -29.72217559814453,
      "activations/layer16_attention_weight_max": 32.88522720336914,
      "activations/layer16_attention_weight_min": -31.631750106811523,
      "activations/layer17_attention_weight_max": 52.35266876220703,
      "activations/layer17_attention_weight_min": -43.85768508911133,
      "activations/layer18_attention_weight_max": 45.940460205078125,
      "activations/layer18_attention_weight_min": -38.057350158691406,
      "activations/layer19_attention_weight_max": 20.61614990234375,
      "activations/layer19_attention_weight_min": -25.40175437927246,
      "activations/layer1_attention_weight_max": 16.850061416625977,
      "activations/layer1_attention_weight_min": -15.111499786376953,
      "activations/layer20_attention_weight_max": 23.22462272644043,
      "activations/layer20_attention_weight_min": -21.789108276367188,
      "activations/layer21_attention_weight_max": 32.10880661010742,
      "activations/layer21_attention_weight_min": -20.247211456298828,
      "activations/layer22_attention_weight_max": 27.602468490600586,
      "activations/layer22_attention_weight_min": -24.738014221191406,
      "activations/layer23_attention_weight_max": 36.952537536621094,
      "activations/layer23_attention_weight_min": -26.53018569946289,
      "activations/layer2_attention_weight_max": 33.391056060791016,
      "activations/layer2_attention_weight_min": -31.545011520385742,
      "activations/layer3_attention_weight_max": 98.34050750732422,
      "activations/layer3_attention_weight_min": -97.13506317138672,
      "activations/layer4_attention_weight_max": 99.20455169677734,
      "activations/layer4_attention_weight_min": -96.82381439208984,
      "activations/layer5_attention_weight_max": 72.13157653808594,
      "activations/layer5_attention_weight_min": -78.38780212402344,
      "activations/layer6_attention_weight_max": 53.04819869995117,
      "activations/layer6_attention_weight_min": -52.6873893737793,
      "activations/layer7_attention_weight_max": 68.74395751953125,
      "activations/layer7_attention_weight_min": -72.69039916992188,
      "activations/layer8_attention_weight_max": 44.90789794921875,
      "activations/layer8_attention_weight_min": -48.794715881347656,
      "activations/layer9_attention_weight_max": 46.626216888427734,
      "activations/layer9_attention_weight_min": -47.38344955444336,
      "epoch": 12.02,
      "learning_rate": 7.322916666666666e-05,
      "loss": 2.7594,
      "step": 206850
    },
    {
      "activations/layer0_attention_weight_max": 16.814376831054688,
      "activations/layer0_attention_weight_min": -14.175751686096191,
      "activations/layer10_attention_weight_max": 39.39882278442383,
      "activations/layer10_attention_weight_min": -37.66715621948242,
      "activations/layer11_attention_weight_max": 36.1748046875,
      "activations/layer11_attention_weight_min": -33.88618087768555,
      "activations/layer12_attention_weight_max": 18.443513870239258,
      "activations/layer12_attention_weight_min": -25.105661392211914,
      "activations/layer13_attention_weight_max": 37.579505920410156,
      "activations/layer13_attention_weight_min": -32.106727600097656,
      "activations/layer14_attention_weight_max": 37.78318786621094,
      "activations/layer14_attention_weight_min": -31.5059871673584,
      "activations/layer15_attention_weight_max": 34.37589645385742,
      "activations/layer15_attention_weight_min": -29.195415496826172,
      "activations/layer16_attention_weight_max": 32.87939453125,
      "activations/layer16_attention_weight_min": -31.37061309814453,
      "activations/layer17_attention_weight_max": 55.54704284667969,
      "activations/layer17_attention_weight_min": -46.271942138671875,
      "activations/layer18_attention_weight_max": 49.87210464477539,
      "activations/layer18_attention_weight_min": -37.715911865234375,
      "activations/layer19_attention_weight_max": 22.1478328704834,
      "activations/layer19_attention_weight_min": -21.203506469726562,
      "activations/layer1_attention_weight_max": 16.95183563232422,
      "activations/layer1_attention_weight_min": -15.4725341796875,
      "activations/layer20_attention_weight_max": 22.05691146850586,
      "activations/layer20_attention_weight_min": -19.86821937561035,
      "activations/layer21_attention_weight_max": 36.9448127746582,
      "activations/layer21_attention_weight_min": -21.934085845947266,
      "activations/layer22_attention_weight_max": 30.67561912536621,
      "activations/layer22_attention_weight_min": -27.31255531311035,
      "activations/layer23_attention_weight_max": 38.91292190551758,
      "activations/layer23_attention_weight_min": -24.431293487548828,
      "activations/layer2_attention_weight_max": 33.16919708251953,
      "activations/layer2_attention_weight_min": -31.824871063232422,
      "activations/layer3_attention_weight_max": 96.91754913330078,
      "activations/layer3_attention_weight_min": -100.3207778930664,
      "activations/layer4_attention_weight_max": 100.04243469238281,
      "activations/layer4_attention_weight_min": -100.6811294555664,
      "activations/layer5_attention_weight_max": 75.087890625,
      "activations/layer5_attention_weight_min": -77.42864227294922,
      "activations/layer6_attention_weight_max": 56.12580871582031,
      "activations/layer6_attention_weight_min": -55.44697952270508,
      "activations/layer7_attention_weight_max": 73.7208480834961,
      "activations/layer7_attention_weight_min": -70.02981567382812,
      "activations/layer8_attention_weight_max": 47.44895553588867,
      "activations/layer8_attention_weight_min": -49.41889953613281,
      "activations/layer9_attention_weight_max": 49.748924255371094,
      "activations/layer9_attention_weight_min": -46.30183029174805,
      "epoch": 12.02,
      "learning_rate": 7.321022727272728e-05,
      "loss": 2.7534,
      "step": 206900
    },
    {
      "activations/layer0_attention_weight_max": 15.788101196289062,
      "activations/layer0_attention_weight_min": -14.66736888885498,
      "activations/layer10_attention_weight_max": 34.35443878173828,
      "activations/layer10_attention_weight_min": -34.234825134277344,
      "activations/layer11_attention_weight_max": 28.983196258544922,
      "activations/layer11_attention_weight_min": -32.291236877441406,
      "activations/layer12_attention_weight_max": 18.04340934753418,
      "activations/layer12_attention_weight_min": -25.067279815673828,
      "activations/layer13_attention_weight_max": 38.17543029785156,
      "activations/layer13_attention_weight_min": -30.292104721069336,
      "activations/layer14_attention_weight_max": 33.745338439941406,
      "activations/layer14_attention_weight_min": -30.373458862304688,
      "activations/layer15_attention_weight_max": 30.460145950317383,
      "activations/layer15_attention_weight_min": -29.174331665039062,
      "activations/layer16_attention_weight_max": 28.605724334716797,
      "activations/layer16_attention_weight_min": -29.648143768310547,
      "activations/layer17_attention_weight_max": 50.06516647338867,
      "activations/layer17_attention_weight_min": -43.2648811340332,
      "activations/layer18_attention_weight_max": 42.066585540771484,
      "activations/layer18_attention_weight_min": -36.775657653808594,
      "activations/layer19_attention_weight_max": 22.744321823120117,
      "activations/layer19_attention_weight_min": -23.080671310424805,
      "activations/layer1_attention_weight_max": 16.254634857177734,
      "activations/layer1_attention_weight_min": -13.88683032989502,
      "activations/layer20_attention_weight_max": 22.460796356201172,
      "activations/layer20_attention_weight_min": -23.170019149780273,
      "activations/layer21_attention_weight_max": 35.28272247314453,
      "activations/layer21_attention_weight_min": -25.531871795654297,
      "activations/layer22_attention_weight_max": 27.644412994384766,
      "activations/layer22_attention_weight_min": -27.02383041381836,
      "activations/layer23_attention_weight_max": 39.50080871582031,
      "activations/layer23_attention_weight_min": -25.188276290893555,
      "activations/layer2_attention_weight_max": 33.067893981933594,
      "activations/layer2_attention_weight_min": -33.13622283935547,
      "activations/layer3_attention_weight_max": 94.77606964111328,
      "activations/layer3_attention_weight_min": -97.9145278930664,
      "activations/layer4_attention_weight_max": 98.181884765625,
      "activations/layer4_attention_weight_min": -96.6324691772461,
      "activations/layer5_attention_weight_max": 68.03043365478516,
      "activations/layer5_attention_weight_min": -75.5227279663086,
      "activations/layer6_attention_weight_max": 50.46833801269531,
      "activations/layer6_attention_weight_min": -50.818660736083984,
      "activations/layer7_attention_weight_max": 69.43171691894531,
      "activations/layer7_attention_weight_min": -65.2226333618164,
      "activations/layer8_attention_weight_max": 40.99367141723633,
      "activations/layer8_attention_weight_min": -48.616546630859375,
      "activations/layer9_attention_weight_max": 43.39547348022461,
      "activations/layer9_attention_weight_min": -47.20542526245117,
      "epoch": 12.02,
      "learning_rate": 7.319128787878788e-05,
      "loss": 2.769,
      "step": 206950
    },
    {
      "activations/layer0_attention_weight_max": 16.47536277770996,
      "activations/layer0_attention_weight_min": -14.44820499420166,
      "activations/layer10_attention_weight_max": 34.29947280883789,
      "activations/layer10_attention_weight_min": -34.71063995361328,
      "activations/layer11_attention_weight_max": 33.080745697021484,
      "activations/layer11_attention_weight_min": -34.77493667602539,
      "activations/layer12_attention_weight_max": 19.333011627197266,
      "activations/layer12_attention_weight_min": -23.531341552734375,
      "activations/layer13_attention_weight_max": 41.975860595703125,
      "activations/layer13_attention_weight_min": -29.300167083740234,
      "activations/layer14_attention_weight_max": 36.39503860473633,
      "activations/layer14_attention_weight_min": -30.73440170288086,
      "activations/layer15_attention_weight_max": 33.21213912963867,
      "activations/layer15_attention_weight_min": -30.327762603759766,
      "activations/layer16_attention_weight_max": 30.17519187927246,
      "activations/layer16_attention_weight_min": -29.307880401611328,
      "activations/layer17_attention_weight_max": 54.42319107055664,
      "activations/layer17_attention_weight_min": -44.95426940917969,
      "activations/layer18_attention_weight_max": 46.10762023925781,
      "activations/layer18_attention_weight_min": -40.11269760131836,
      "activations/layer19_attention_weight_max": 22.63861846923828,
      "activations/layer19_attention_weight_min": -22.1549129486084,
      "activations/layer1_attention_weight_max": 16.327922821044922,
      "activations/layer1_attention_weight_min": -15.354355812072754,
      "activations/layer20_attention_weight_max": 24.63365936279297,
      "activations/layer20_attention_weight_min": -21.537538528442383,
      "activations/layer21_attention_weight_max": 36.48721694946289,
      "activations/layer21_attention_weight_min": -23.146804809570312,
      "activations/layer22_attention_weight_max": 34.84990692138672,
      "activations/layer22_attention_weight_min": -27.372724533081055,
      "activations/layer23_attention_weight_max": 40.911556243896484,
      "activations/layer23_attention_weight_min": -24.081645965576172,
      "activations/layer2_attention_weight_max": 32.94157409667969,
      "activations/layer2_attention_weight_min": -30.991947174072266,
      "activations/layer3_attention_weight_max": 99.02828216552734,
      "activations/layer3_attention_weight_min": -99.48921966552734,
      "activations/layer4_attention_weight_max": 100.31339263916016,
      "activations/layer4_attention_weight_min": -99.42610168457031,
      "activations/layer5_attention_weight_max": 72.41370391845703,
      "activations/layer5_attention_weight_min": -81.12503051757812,
      "activations/layer6_attention_weight_max": 53.30564880371094,
      "activations/layer6_attention_weight_min": -53.945133209228516,
      "activations/layer7_attention_weight_max": 69.10398864746094,
      "activations/layer7_attention_weight_min": -68.78400421142578,
      "activations/layer8_attention_weight_max": 45.08028030395508,
      "activations/layer8_attention_weight_min": -48.4045524597168,
      "activations/layer9_attention_weight_max": 49.181968688964844,
      "activations/layer9_attention_weight_min": -49.882301330566406,
      "epoch": 12.03,
      "learning_rate": 7.317234848484848e-05,
      "loss": 2.7623,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5554,
      "eval_samples_per_second": 501.906,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5554,
      "eval_openwebtext_samples_per_second": 501.906,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.046,
      "eval_wikitext_samples_per_second": 222.876,
      "step": 207000
    },
    {
      "epoch": 12.03,
      "eval_lambada_loss": 2.611328125,
      "eval_lambada_ppl": 13.617124090346826,
      "eval_lambada_runtime": 9.6092,
      "eval_lambada_samples_per_second": 506.701,
      "step": 207000
    },
    {
      "activations/layer0_attention_weight_max": 15.68143081665039,
      "activations/layer0_attention_weight_min": -14.149420738220215,
      "activations/layer10_attention_weight_max": 36.79273223876953,
      "activations/layer10_attention_weight_min": -35.408843994140625,
      "activations/layer11_attention_weight_max": 36.66383361816406,
      "activations/layer11_attention_weight_min": -34.80656814575195,
      "activations/layer12_attention_weight_max": 17.240230560302734,
      "activations/layer12_attention_weight_min": -26.116531372070312,
      "activations/layer13_attention_weight_max": 40.392738342285156,
      "activations/layer13_attention_weight_min": -31.4101619720459,
      "activations/layer14_attention_weight_max": 39.886905670166016,
      "activations/layer14_attention_weight_min": -31.90284538269043,
      "activations/layer15_attention_weight_max": 37.88409423828125,
      "activations/layer15_attention_weight_min": -30.653343200683594,
      "activations/layer16_attention_weight_max": 33.42635726928711,
      "activations/layer16_attention_weight_min": -31.54424285888672,
      "activations/layer17_attention_weight_max": 51.33713150024414,
      "activations/layer17_attention_weight_min": -46.900726318359375,
      "activations/layer18_attention_weight_max": 50.51781463623047,
      "activations/layer18_attention_weight_min": -41.075904846191406,
      "activations/layer19_attention_weight_max": 25.083341598510742,
      "activations/layer19_attention_weight_min": -23.968664169311523,
      "activations/layer1_attention_weight_max": 17.280473709106445,
      "activations/layer1_attention_weight_min": -15.009177207946777,
      "activations/layer20_attention_weight_max": 24.673994064331055,
      "activations/layer20_attention_weight_min": -21.91724967956543,
      "activations/layer21_attention_weight_max": 41.58113098144531,
      "activations/layer21_attention_weight_min": -26.350278854370117,
      "activations/layer22_attention_weight_max": 28.59501838684082,
      "activations/layer22_attention_weight_min": -26.19253158569336,
      "activations/layer23_attention_weight_max": 38.5825309753418,
      "activations/layer23_attention_weight_min": -25.22310447692871,
      "activations/layer2_attention_weight_max": 33.51908493041992,
      "activations/layer2_attention_weight_min": -32.59171676635742,
      "activations/layer3_attention_weight_max": 101.6220932006836,
      "activations/layer3_attention_weight_min": -101.7497329711914,
      "activations/layer4_attention_weight_max": 106.58839416503906,
      "activations/layer4_attention_weight_min": -100.14495849609375,
      "activations/layer5_attention_weight_max": 81.97724914550781,
      "activations/layer5_attention_weight_min": -86.14175415039062,
      "activations/layer6_attention_weight_max": 54.187889099121094,
      "activations/layer6_attention_weight_min": -55.30384063720703,
      "activations/layer7_attention_weight_max": 78.2499771118164,
      "activations/layer7_attention_weight_min": -67.51192474365234,
      "activations/layer8_attention_weight_max": 47.296730041503906,
      "activations/layer8_attention_weight_min": -45.27338409423828,
      "activations/layer9_attention_weight_max": 58.345428466796875,
      "activations/layer9_attention_weight_min": -48.631046295166016,
      "epoch": 12.03,
      "learning_rate": 7.315378787878787e-05,
      "loss": 2.7647,
      "step": 207050
    },
    {
      "activations/layer0_attention_weight_max": 15.984586715698242,
      "activations/layer0_attention_weight_min": -14.627781867980957,
      "activations/layer10_attention_weight_max": 41.346343994140625,
      "activations/layer10_attention_weight_min": -37.53412628173828,
      "activations/layer11_attention_weight_max": 36.734405517578125,
      "activations/layer11_attention_weight_min": -37.749977111816406,
      "activations/layer12_attention_weight_max": 20.770841598510742,
      "activations/layer12_attention_weight_min": -21.124542236328125,
      "activations/layer13_attention_weight_max": 40.893280029296875,
      "activations/layer13_attention_weight_min": -31.054704666137695,
      "activations/layer14_attention_weight_max": 44.312400817871094,
      "activations/layer14_attention_weight_min": -30.88028335571289,
      "activations/layer15_attention_weight_max": 35.43339920043945,
      "activations/layer15_attention_weight_min": -31.25516700744629,
      "activations/layer16_attention_weight_max": 34.36017990112305,
      "activations/layer16_attention_weight_min": -33.66959762573242,
      "activations/layer17_attention_weight_max": 60.58391571044922,
      "activations/layer17_attention_weight_min": -49.217830657958984,
      "activations/layer18_attention_weight_max": 50.87470626831055,
      "activations/layer18_attention_weight_min": -42.08344650268555,
      "activations/layer19_attention_weight_max": 27.551273345947266,
      "activations/layer19_attention_weight_min": -25.68719482421875,
      "activations/layer1_attention_weight_max": 17.507366180419922,
      "activations/layer1_attention_weight_min": -17.43866539001465,
      "activations/layer20_attention_weight_max": 25.523508071899414,
      "activations/layer20_attention_weight_min": -25.22321128845215,
      "activations/layer21_attention_weight_max": 40.1887092590332,
      "activations/layer21_attention_weight_min": -25.272178649902344,
      "activations/layer22_attention_weight_max": 34.447662353515625,
      "activations/layer22_attention_weight_min": -29.979063034057617,
      "activations/layer23_attention_weight_max": 44.87315368652344,
      "activations/layer23_attention_weight_min": -30.884479522705078,
      "activations/layer2_attention_weight_max": 34.404869079589844,
      "activations/layer2_attention_weight_min": -32.82423782348633,
      "activations/layer3_attention_weight_max": 97.66089630126953,
      "activations/layer3_attention_weight_min": -97.35198974609375,
      "activations/layer4_attention_weight_max": 102.91024017333984,
      "activations/layer4_attention_weight_min": -98.02986907958984,
      "activations/layer5_attention_weight_max": 74.74896240234375,
      "activations/layer5_attention_weight_min": -78.58409881591797,
      "activations/layer6_attention_weight_max": 55.82309341430664,
      "activations/layer6_attention_weight_min": -53.93280792236328,
      "activations/layer7_attention_weight_max": 73.56396484375,
      "activations/layer7_attention_weight_min": -71.77670288085938,
      "activations/layer8_attention_weight_max": 48.13112258911133,
      "activations/layer8_attention_weight_min": -49.25190353393555,
      "activations/layer9_attention_weight_max": 50.12406539916992,
      "activations/layer9_attention_weight_min": -49.39020919799805,
      "epoch": 12.03,
      "learning_rate": 7.313484848484848e-05,
      "loss": 2.7639,
      "step": 207100
    },
    {
      "activations/layer0_attention_weight_max": 16.090211868286133,
      "activations/layer0_attention_weight_min": -14.416075706481934,
      "activations/layer10_attention_weight_max": 37.52497100830078,
      "activations/layer10_attention_weight_min": -36.39144515991211,
      "activations/layer11_attention_weight_max": 32.8367805480957,
      "activations/layer11_attention_weight_min": -31.860837936401367,
      "activations/layer12_attention_weight_max": 17.450153350830078,
      "activations/layer12_attention_weight_min": -23.650680541992188,
      "activations/layer13_attention_weight_max": 38.167236328125,
      "activations/layer13_attention_weight_min": -33.476844787597656,
      "activations/layer14_attention_weight_max": 35.709712982177734,
      "activations/layer14_attention_weight_min": -30.847009658813477,
      "activations/layer15_attention_weight_max": 32.78799057006836,
      "activations/layer15_attention_weight_min": -30.68938446044922,
      "activations/layer16_attention_weight_max": 29.608980178833008,
      "activations/layer16_attention_weight_min": -29.412988662719727,
      "activations/layer17_attention_weight_max": 52.10968780517578,
      "activations/layer17_attention_weight_min": -46.4973258972168,
      "activations/layer18_attention_weight_max": 46.9265251159668,
      "activations/layer18_attention_weight_min": -37.64581298828125,
      "activations/layer19_attention_weight_max": 20.889219284057617,
      "activations/layer19_attention_weight_min": -22.36275863647461,
      "activations/layer1_attention_weight_max": 17.506250381469727,
      "activations/layer1_attention_weight_min": -16.33355712890625,
      "activations/layer20_attention_weight_max": 20.597286224365234,
      "activations/layer20_attention_weight_min": -20.866161346435547,
      "activations/layer21_attention_weight_max": 32.727115631103516,
      "activations/layer21_attention_weight_min": -20.1353702545166,
      "activations/layer22_attention_weight_max": 28.035329818725586,
      "activations/layer22_attention_weight_min": -24.927141189575195,
      "activations/layer23_attention_weight_max": 32.02062225341797,
      "activations/layer23_attention_weight_min": -21.769733428955078,
      "activations/layer2_attention_weight_max": 33.04693603515625,
      "activations/layer2_attention_weight_min": -31.45476531982422,
      "activations/layer3_attention_weight_max": 95.365966796875,
      "activations/layer3_attention_weight_min": -100.38706970214844,
      "activations/layer4_attention_weight_max": 96.71737670898438,
      "activations/layer4_attention_weight_min": -95.82080841064453,
      "activations/layer5_attention_weight_max": 70.21769714355469,
      "activations/layer5_attention_weight_min": -77.56610870361328,
      "activations/layer6_attention_weight_max": 54.00606918334961,
      "activations/layer6_attention_weight_min": -51.03933334350586,
      "activations/layer7_attention_weight_max": 71.00711059570312,
      "activations/layer7_attention_weight_min": -66.309814453125,
      "activations/layer8_attention_weight_max": 45.10530090332031,
      "activations/layer8_attention_weight_min": -47.538658142089844,
      "activations/layer9_attention_weight_max": 43.768516540527344,
      "activations/layer9_attention_weight_min": -46.92293930053711,
      "epoch": 12.04,
      "learning_rate": 7.311590909090909e-05,
      "loss": 2.7829,
      "step": 207150
    },
    {
      "activations/layer0_attention_weight_max": 15.794153213500977,
      "activations/layer0_attention_weight_min": -14.436727523803711,
      "activations/layer10_attention_weight_max": 39.38336181640625,
      "activations/layer10_attention_weight_min": -36.561126708984375,
      "activations/layer11_attention_weight_max": 32.03762435913086,
      "activations/layer11_attention_weight_min": -34.64746856689453,
      "activations/layer12_attention_weight_max": 18.600833892822266,
      "activations/layer12_attention_weight_min": -27.731103897094727,
      "activations/layer13_attention_weight_max": 35.47848892211914,
      "activations/layer13_attention_weight_min": -28.00171661376953,
      "activations/layer14_attention_weight_max": 36.41506576538086,
      "activations/layer14_attention_weight_min": -29.814918518066406,
      "activations/layer15_attention_weight_max": 33.25652313232422,
      "activations/layer15_attention_weight_min": -28.84278678894043,
      "activations/layer16_attention_weight_max": 30.732873916625977,
      "activations/layer16_attention_weight_min": -28.964008331298828,
      "activations/layer17_attention_weight_max": 48.708648681640625,
      "activations/layer17_attention_weight_min": -42.08275604248047,
      "activations/layer18_attention_weight_max": 43.349159240722656,
      "activations/layer18_attention_weight_min": -36.39066696166992,
      "activations/layer19_attention_weight_max": 22.73821449279785,
      "activations/layer19_attention_weight_min": -21.04669952392578,
      "activations/layer1_attention_weight_max": 16.611709594726562,
      "activations/layer1_attention_weight_min": -15.01362419128418,
      "activations/layer20_attention_weight_max": 22.586729049682617,
      "activations/layer20_attention_weight_min": -22.22731590270996,
      "activations/layer21_attention_weight_max": 31.75998878479004,
      "activations/layer21_attention_weight_min": -21.081527709960938,
      "activations/layer22_attention_weight_max": 29.93446159362793,
      "activations/layer22_attention_weight_min": -24.92247200012207,
      "activations/layer23_attention_weight_max": 33.28733444213867,
      "activations/layer23_attention_weight_min": -23.75568199157715,
      "activations/layer2_attention_weight_max": 32.08312225341797,
      "activations/layer2_attention_weight_min": -32.16176223754883,
      "activations/layer3_attention_weight_max": 91.91399383544922,
      "activations/layer3_attention_weight_min": -91.46531677246094,
      "activations/layer4_attention_weight_max": 100.07051086425781,
      "activations/layer4_attention_weight_min": -92.95848083496094,
      "activations/layer5_attention_weight_max": 74.9952392578125,
      "activations/layer5_attention_weight_min": -80.68521881103516,
      "activations/layer6_attention_weight_max": 58.19251251220703,
      "activations/layer6_attention_weight_min": -53.399349212646484,
      "activations/layer7_attention_weight_max": 74.66278076171875,
      "activations/layer7_attention_weight_min": -67.12853240966797,
      "activations/layer8_attention_weight_max": 45.43907165527344,
      "activations/layer8_attention_weight_min": -49.640777587890625,
      "activations/layer9_attention_weight_max": 46.98649597167969,
      "activations/layer9_attention_weight_min": -48.15171813964844,
      "epoch": 12.04,
      "learning_rate": 7.309696969696969e-05,
      "loss": 2.7609,
      "step": 207200
    },
    {
      "activations/layer0_attention_weight_max": 16.689102172851562,
      "activations/layer0_attention_weight_min": -14.287126541137695,
      "activations/layer10_attention_weight_max": 38.19141387939453,
      "activations/layer10_attention_weight_min": -37.625030517578125,
      "activations/layer11_attention_weight_max": 33.731964111328125,
      "activations/layer11_attention_weight_min": -32.28526306152344,
      "activations/layer12_attention_weight_max": 19.455602645874023,
      "activations/layer12_attention_weight_min": -25.92381477355957,
      "activations/layer13_attention_weight_max": 37.14111328125,
      "activations/layer13_attention_weight_min": -25.87849998474121,
      "activations/layer14_attention_weight_max": 37.211143493652344,
      "activations/layer14_attention_weight_min": -31.068742752075195,
      "activations/layer15_attention_weight_max": 33.57321548461914,
      "activations/layer15_attention_weight_min": -30.700172424316406,
      "activations/layer16_attention_weight_max": 33.00971221923828,
      "activations/layer16_attention_weight_min": -29.657419204711914,
      "activations/layer17_attention_weight_max": 51.28976058959961,
      "activations/layer17_attention_weight_min": -43.851463317871094,
      "activations/layer18_attention_weight_max": 44.380592346191406,
      "activations/layer18_attention_weight_min": -37.45125961303711,
      "activations/layer19_attention_weight_max": 20.87286949157715,
      "activations/layer19_attention_weight_min": -22.132429122924805,
      "activations/layer1_attention_weight_max": 17.929155349731445,
      "activations/layer1_attention_weight_min": -18.06955909729004,
      "activations/layer20_attention_weight_max": 20.913461685180664,
      "activations/layer20_attention_weight_min": -21.391986846923828,
      "activations/layer21_attention_weight_max": 34.25598907470703,
      "activations/layer21_attention_weight_min": -22.082679748535156,
      "activations/layer22_attention_weight_max": 27.437145233154297,
      "activations/layer22_attention_weight_min": -25.75234031677246,
      "activations/layer23_attention_weight_max": 38.20880889892578,
      "activations/layer23_attention_weight_min": -26.233436584472656,
      "activations/layer2_attention_weight_max": 33.87523651123047,
      "activations/layer2_attention_weight_min": -33.54236602783203,
      "activations/layer3_attention_weight_max": 91.80430603027344,
      "activations/layer3_attention_weight_min": -95.05821228027344,
      "activations/layer4_attention_weight_max": 98.43636322021484,
      "activations/layer4_attention_weight_min": -99.2526626586914,
      "activations/layer5_attention_weight_max": 74.62345886230469,
      "activations/layer5_attention_weight_min": -74.6683349609375,
      "activations/layer6_attention_weight_max": 53.54417037963867,
      "activations/layer6_attention_weight_min": -52.00539016723633,
      "activations/layer7_attention_weight_max": 71.25790405273438,
      "activations/layer7_attention_weight_min": -68.1514663696289,
      "activations/layer8_attention_weight_max": 46.14388656616211,
      "activations/layer8_attention_weight_min": -46.97175979614258,
      "activations/layer9_attention_weight_max": 47.4919319152832,
      "activations/layer9_attention_weight_min": -48.8293571472168,
      "epoch": 12.04,
      "learning_rate": 7.307803030303029e-05,
      "loss": 2.7712,
      "step": 207250
    },
    {
      "activations/layer0_attention_weight_max": 16.38324546813965,
      "activations/layer0_attention_weight_min": -14.186388969421387,
      "activations/layer10_attention_weight_max": 33.722991943359375,
      "activations/layer10_attention_weight_min": -34.31635284423828,
      "activations/layer11_attention_weight_max": 30.947303771972656,
      "activations/layer11_attention_weight_min": -32.02625274658203,
      "activations/layer12_attention_weight_max": 17.922117233276367,
      "activations/layer12_attention_weight_min": -26.993188858032227,
      "activations/layer13_attention_weight_max": 38.21322250366211,
      "activations/layer13_attention_weight_min": -29.095226287841797,
      "activations/layer14_attention_weight_max": 34.07794952392578,
      "activations/layer14_attention_weight_min": -30.99528694152832,
      "activations/layer15_attention_weight_max": 31.545942306518555,
      "activations/layer15_attention_weight_min": -30.43247413635254,
      "activations/layer16_attention_weight_max": 29.25106430053711,
      "activations/layer16_attention_weight_min": -29.761356353759766,
      "activations/layer17_attention_weight_max": 50.809532165527344,
      "activations/layer17_attention_weight_min": -39.814308166503906,
      "activations/layer18_attention_weight_max": 46.216636657714844,
      "activations/layer18_attention_weight_min": -37.421974182128906,
      "activations/layer19_attention_weight_max": 25.185211181640625,
      "activations/layer19_attention_weight_min": -21.082134246826172,
      "activations/layer1_attention_weight_max": 16.68613624572754,
      "activations/layer1_attention_weight_min": -14.719476699829102,
      "activations/layer20_attention_weight_max": 21.451011657714844,
      "activations/layer20_attention_weight_min": -23.550703048706055,
      "activations/layer21_attention_weight_max": 36.30204772949219,
      "activations/layer21_attention_weight_min": -23.509069442749023,
      "activations/layer22_attention_weight_max": 28.315385818481445,
      "activations/layer22_attention_weight_min": -26.551469802856445,
      "activations/layer23_attention_weight_max": 36.39800262451172,
      "activations/layer23_attention_weight_min": -24.445661544799805,
      "activations/layer2_attention_weight_max": 32.738216400146484,
      "activations/layer2_attention_weight_min": -32.340511322021484,
      "activations/layer3_attention_weight_max": 91.57579803466797,
      "activations/layer3_attention_weight_min": -92.85193634033203,
      "activations/layer4_attention_weight_max": 94.71355438232422,
      "activations/layer4_attention_weight_min": -100.76970672607422,
      "activations/layer5_attention_weight_max": 68.96923828125,
      "activations/layer5_attention_weight_min": -77.8108139038086,
      "activations/layer6_attention_weight_max": 50.005184173583984,
      "activations/layer6_attention_weight_min": -53.019432067871094,
      "activations/layer7_attention_weight_max": 67.74027252197266,
      "activations/layer7_attention_weight_min": -67.80915832519531,
      "activations/layer8_attention_weight_max": 42.4035530090332,
      "activations/layer8_attention_weight_min": -46.01828384399414,
      "activations/layer9_attention_weight_max": 44.23113250732422,
      "activations/layer9_attention_weight_min": -47.8871955871582,
      "epoch": 12.05,
      "learning_rate": 7.30590909090909e-05,
      "loss": 2.7698,
      "step": 207300
    },
    {
      "activations/layer0_attention_weight_max": 15.505025863647461,
      "activations/layer0_attention_weight_min": -14.545899391174316,
      "activations/layer10_attention_weight_max": 34.63304138183594,
      "activations/layer10_attention_weight_min": -37.2082405090332,
      "activations/layer11_attention_weight_max": 31.52479362487793,
      "activations/layer11_attention_weight_min": -33.50469970703125,
      "activations/layer12_attention_weight_max": 18.647891998291016,
      "activations/layer12_attention_weight_min": -24.50485610961914,
      "activations/layer13_attention_weight_max": 36.9140510559082,
      "activations/layer13_attention_weight_min": -31.67880630493164,
      "activations/layer14_attention_weight_max": 37.7625617980957,
      "activations/layer14_attention_weight_min": -32.61216354370117,
      "activations/layer15_attention_weight_max": 36.49022674560547,
      "activations/layer15_attention_weight_min": -30.499427795410156,
      "activations/layer16_attention_weight_max": 35.063716888427734,
      "activations/layer16_attention_weight_min": -32.135040283203125,
      "activations/layer17_attention_weight_max": 60.40821838378906,
      "activations/layer17_attention_weight_min": -47.26893615722656,
      "activations/layer18_attention_weight_max": 51.11005783081055,
      "activations/layer18_attention_weight_min": -38.23795700073242,
      "activations/layer19_attention_weight_max": 22.341073989868164,
      "activations/layer19_attention_weight_min": -22.40337371826172,
      "activations/layer1_attention_weight_max": 17.028453826904297,
      "activations/layer1_attention_weight_min": -15.686935424804688,
      "activations/layer20_attention_weight_max": 22.369060516357422,
      "activations/layer20_attention_weight_min": -22.514965057373047,
      "activations/layer21_attention_weight_max": 37.548248291015625,
      "activations/layer21_attention_weight_min": -23.08209991455078,
      "activations/layer22_attention_weight_max": 32.138877868652344,
      "activations/layer22_attention_weight_min": -26.49550437927246,
      "activations/layer23_attention_weight_max": 39.9626579284668,
      "activations/layer23_attention_weight_min": -25.49098014831543,
      "activations/layer2_attention_weight_max": 34.424354553222656,
      "activations/layer2_attention_weight_min": -32.82884216308594,
      "activations/layer3_attention_weight_max": 94.4822769165039,
      "activations/layer3_attention_weight_min": -102.73843383789062,
      "activations/layer4_attention_weight_max": 100.80545043945312,
      "activations/layer4_attention_weight_min": -101.39131164550781,
      "activations/layer5_attention_weight_max": 72.8575210571289,
      "activations/layer5_attention_weight_min": -83.63536071777344,
      "activations/layer6_attention_weight_max": 54.14905548095703,
      "activations/layer6_attention_weight_min": -54.06565856933594,
      "activations/layer7_attention_weight_max": 71.23538970947266,
      "activations/layer7_attention_weight_min": -68.35340881347656,
      "activations/layer8_attention_weight_max": 45.744354248046875,
      "activations/layer8_attention_weight_min": -45.972843170166016,
      "activations/layer9_attention_weight_max": 48.16279220581055,
      "activations/layer9_attention_weight_min": -48.53134536743164,
      "epoch": 12.05,
      "learning_rate": 7.30401515151515e-05,
      "loss": 2.78,
      "step": 207350
    },
    {
      "activations/layer0_attention_weight_max": 15.863905906677246,
      "activations/layer0_attention_weight_min": -14.54282283782959,
      "activations/layer10_attention_weight_max": 36.292362213134766,
      "activations/layer10_attention_weight_min": -33.8211669921875,
      "activations/layer11_attention_weight_max": 31.25238609313965,
      "activations/layer11_attention_weight_min": -31.44143295288086,
      "activations/layer12_attention_weight_max": 18.617551803588867,
      "activations/layer12_attention_weight_min": -27.506662368774414,
      "activations/layer13_attention_weight_max": 39.57670211791992,
      "activations/layer13_attention_weight_min": -30.371103286743164,
      "activations/layer14_attention_weight_max": 39.68478012084961,
      "activations/layer14_attention_weight_min": -32.4886360168457,
      "activations/layer15_attention_weight_max": 32.2785530090332,
      "activations/layer15_attention_weight_min": -30.530344009399414,
      "activations/layer16_attention_weight_max": 31.990327835083008,
      "activations/layer16_attention_weight_min": -30.85295867919922,
      "activations/layer17_attention_weight_max": 56.3409423828125,
      "activations/layer17_attention_weight_min": -47.2570686340332,
      "activations/layer18_attention_weight_max": 48.25371170043945,
      "activations/layer18_attention_weight_min": -41.06380081176758,
      "activations/layer19_attention_weight_max": 25.156368255615234,
      "activations/layer19_attention_weight_min": -21.86903190612793,
      "activations/layer1_attention_weight_max": 16.74053382873535,
      "activations/layer1_attention_weight_min": -14.594889640808105,
      "activations/layer20_attention_weight_max": 24.0003719329834,
      "activations/layer20_attention_weight_min": -21.717998504638672,
      "activations/layer21_attention_weight_max": 37.24446487426758,
      "activations/layer21_attention_weight_min": -25.1837100982666,
      "activations/layer22_attention_weight_max": 32.248741149902344,
      "activations/layer22_attention_weight_min": -25.4981746673584,
      "activations/layer23_attention_weight_max": 39.33911895751953,
      "activations/layer23_attention_weight_min": -25.08131217956543,
      "activations/layer2_attention_weight_max": 33.41249084472656,
      "activations/layer2_attention_weight_min": -31.831314086914062,
      "activations/layer3_attention_weight_max": 90.4960708618164,
      "activations/layer3_attention_weight_min": -89.5151596069336,
      "activations/layer4_attention_weight_max": 98.52715301513672,
      "activations/layer4_attention_weight_min": -95.51292419433594,
      "activations/layer5_attention_weight_max": 72.21388244628906,
      "activations/layer5_attention_weight_min": -77.44046020507812,
      "activations/layer6_attention_weight_max": 53.22819519042969,
      "activations/layer6_attention_weight_min": -53.19149398803711,
      "activations/layer7_attention_weight_max": 73.5641098022461,
      "activations/layer7_attention_weight_min": -64.46695709228516,
      "activations/layer8_attention_weight_max": 42.289764404296875,
      "activations/layer8_attention_weight_min": -46.58455276489258,
      "activations/layer9_attention_weight_max": 43.80959701538086,
      "activations/layer9_attention_weight_min": -45.276771545410156,
      "epoch": 12.05,
      "learning_rate": 7.302121212121212e-05,
      "loss": 2.7761,
      "step": 207400
    },
    {
      "activations/layer0_attention_weight_max": 16.4166202545166,
      "activations/layer0_attention_weight_min": -14.673569679260254,
      "activations/layer10_attention_weight_max": 35.96355438232422,
      "activations/layer10_attention_weight_min": -34.75899887084961,
      "activations/layer11_attention_weight_max": 33.36487579345703,
      "activations/layer11_attention_weight_min": -32.23235321044922,
      "activations/layer12_attention_weight_max": 19.21489143371582,
      "activations/layer12_attention_weight_min": -23.77752685546875,
      "activations/layer13_attention_weight_max": 36.85197448730469,
      "activations/layer13_attention_weight_min": -29.983003616333008,
      "activations/layer14_attention_weight_max": 37.6436882019043,
      "activations/layer14_attention_weight_min": -30.628080368041992,
      "activations/layer15_attention_weight_max": 35.43593978881836,
      "activations/layer15_attention_weight_min": -30.525531768798828,
      "activations/layer16_attention_weight_max": 33.77638626098633,
      "activations/layer16_attention_weight_min": -31.16426658630371,
      "activations/layer17_attention_weight_max": 56.74504852294922,
      "activations/layer17_attention_weight_min": -47.88136672973633,
      "activations/layer18_attention_weight_max": 55.3801383972168,
      "activations/layer18_attention_weight_min": -41.80323791503906,
      "activations/layer19_attention_weight_max": 25.153854370117188,
      "activations/layer19_attention_weight_min": -24.28217315673828,
      "activations/layer1_attention_weight_max": 16.061077117919922,
      "activations/layer1_attention_weight_min": -14.375475883483887,
      "activations/layer20_attention_weight_max": 25.13666343688965,
      "activations/layer20_attention_weight_min": -21.986652374267578,
      "activations/layer21_attention_weight_max": 40.264713287353516,
      "activations/layer21_attention_weight_min": -25.28888511657715,
      "activations/layer22_attention_weight_max": 37.008541107177734,
      "activations/layer22_attention_weight_min": -28.07941436767578,
      "activations/layer23_attention_weight_max": 38.45088577270508,
      "activations/layer23_attention_weight_min": -25.0625057220459,
      "activations/layer2_attention_weight_max": 30.975019454956055,
      "activations/layer2_attention_weight_min": -28.67060089111328,
      "activations/layer3_attention_weight_max": 90.28727722167969,
      "activations/layer3_attention_weight_min": -88.65689849853516,
      "activations/layer4_attention_weight_max": 96.98299407958984,
      "activations/layer4_attention_weight_min": -94.73448944091797,
      "activations/layer5_attention_weight_max": 73.78986358642578,
      "activations/layer5_attention_weight_min": -81.1783447265625,
      "activations/layer6_attention_weight_max": 52.96442794799805,
      "activations/layer6_attention_weight_min": -52.495262145996094,
      "activations/layer7_attention_weight_max": 70.58710479736328,
      "activations/layer7_attention_weight_min": -66.95430755615234,
      "activations/layer8_attention_weight_max": 44.7187614440918,
      "activations/layer8_attention_weight_min": -46.26905059814453,
      "activations/layer9_attention_weight_max": 44.477970123291016,
      "activations/layer9_attention_weight_min": -44.881248474121094,
      "epoch": 12.05,
      "learning_rate": 7.300265151515151e-05,
      "loss": 2.7713,
      "step": 207450
    },
    {
      "activations/layer0_attention_weight_max": 15.614686965942383,
      "activations/layer0_attention_weight_min": -13.951581954956055,
      "activations/layer10_attention_weight_max": 38.6572265625,
      "activations/layer10_attention_weight_min": -37.887203216552734,
      "activations/layer11_attention_weight_max": 34.688289642333984,
      "activations/layer11_attention_weight_min": -33.60509490966797,
      "activations/layer12_attention_weight_max": 26.232858657836914,
      "activations/layer12_attention_weight_min": -23.335180282592773,
      "activations/layer13_attention_weight_max": 39.8724250793457,
      "activations/layer13_attention_weight_min": -31.458606719970703,
      "activations/layer14_attention_weight_max": 41.35567092895508,
      "activations/layer14_attention_weight_min": -32.174415588378906,
      "activations/layer15_attention_weight_max": 34.05183410644531,
      "activations/layer15_attention_weight_min": -30.897655487060547,
      "activations/layer16_attention_weight_max": 34.44135284423828,
      "activations/layer16_attention_weight_min": -31.42569923400879,
      "activations/layer17_attention_weight_max": 52.74226379394531,
      "activations/layer17_attention_weight_min": -46.34553909301758,
      "activations/layer18_attention_weight_max": 45.5094108581543,
      "activations/layer18_attention_weight_min": -39.85455322265625,
      "activations/layer19_attention_weight_max": 21.61738395690918,
      "activations/layer19_attention_weight_min": -21.918100357055664,
      "activations/layer1_attention_weight_max": 16.958406448364258,
      "activations/layer1_attention_weight_min": -11.675057411193848,
      "activations/layer20_attention_weight_max": 22.366456985473633,
      "activations/layer20_attention_weight_min": -21.582477569580078,
      "activations/layer21_attention_weight_max": 37.1817626953125,
      "activations/layer21_attention_weight_min": -22.19660758972168,
      "activations/layer22_attention_weight_max": 29.89989471435547,
      "activations/layer22_attention_weight_min": -27.256887435913086,
      "activations/layer23_attention_weight_max": 40.240638732910156,
      "activations/layer23_attention_weight_min": -24.2418212890625,
      "activations/layer2_attention_weight_max": 27.322132110595703,
      "activations/layer2_attention_weight_min": -28.445598602294922,
      "activations/layer3_attention_weight_max": 82.6268539428711,
      "activations/layer3_attention_weight_min": -81.04838562011719,
      "activations/layer4_attention_weight_max": 94.54862213134766,
      "activations/layer4_attention_weight_min": -91.73975372314453,
      "activations/layer5_attention_weight_max": 71.64371490478516,
      "activations/layer5_attention_weight_min": -81.74266052246094,
      "activations/layer6_attention_weight_max": 51.90460968017578,
      "activations/layer6_attention_weight_min": -53.81172561645508,
      "activations/layer7_attention_weight_max": 72.8532485961914,
      "activations/layer7_attention_weight_min": -70.67965698242188,
      "activations/layer8_attention_weight_max": 47.72822570800781,
      "activations/layer8_attention_weight_min": -49.80531692504883,
      "activations/layer9_attention_weight_max": 51.77520751953125,
      "activations/layer9_attention_weight_min": -49.57439422607422,
      "epoch": 12.06,
      "learning_rate": 7.298371212121211e-05,
      "loss": 2.7849,
      "step": 207500
    },
    {
      "activations/layer0_attention_weight_max": 15.024311065673828,
      "activations/layer0_attention_weight_min": -14.178125381469727,
      "activations/layer10_attention_weight_max": 36.74543762207031,
      "activations/layer10_attention_weight_min": -35.999671936035156,
      "activations/layer11_attention_weight_max": 32.684226989746094,
      "activations/layer11_attention_weight_min": -34.658729553222656,
      "activations/layer12_attention_weight_max": 17.276599884033203,
      "activations/layer12_attention_weight_min": -23.653030395507812,
      "activations/layer13_attention_weight_max": 39.17000961303711,
      "activations/layer13_attention_weight_min": -29.94027328491211,
      "activations/layer14_attention_weight_max": 33.68202209472656,
      "activations/layer14_attention_weight_min": -31.061471939086914,
      "activations/layer15_attention_weight_max": 31.289180755615234,
      "activations/layer15_attention_weight_min": -30.8878173828125,
      "activations/layer16_attention_weight_max": 30.156522750854492,
      "activations/layer16_attention_weight_min": -29.768543243408203,
      "activations/layer17_attention_weight_max": 50.81886672973633,
      "activations/layer17_attention_weight_min": -45.025901794433594,
      "activations/layer18_attention_weight_max": 47.59943771362305,
      "activations/layer18_attention_weight_min": -39.0343017578125,
      "activations/layer19_attention_weight_max": 20.57330322265625,
      "activations/layer19_attention_weight_min": -20.771772384643555,
      "activations/layer1_attention_weight_max": 16.711612701416016,
      "activations/layer1_attention_weight_min": -14.477398872375488,
      "activations/layer20_attention_weight_max": 21.13391876220703,
      "activations/layer20_attention_weight_min": -21.480588912963867,
      "activations/layer21_attention_weight_max": 34.406280517578125,
      "activations/layer21_attention_weight_min": -24.454240798950195,
      "activations/layer22_attention_weight_max": 29.046710968017578,
      "activations/layer22_attention_weight_min": -25.141368865966797,
      "activations/layer23_attention_weight_max": 35.99287796020508,
      "activations/layer23_attention_weight_min": -24.781810760498047,
      "activations/layer2_attention_weight_max": 30.39967155456543,
      "activations/layer2_attention_weight_min": -30.529516220092773,
      "activations/layer3_attention_weight_max": 87.58089447021484,
      "activations/layer3_attention_weight_min": -91.95154571533203,
      "activations/layer4_attention_weight_max": 94.1275634765625,
      "activations/layer4_attention_weight_min": -91.45275115966797,
      "activations/layer5_attention_weight_max": 76.98912048339844,
      "activations/layer5_attention_weight_min": -78.9654312133789,
      "activations/layer6_attention_weight_max": 56.16440963745117,
      "activations/layer6_attention_weight_min": -56.5392951965332,
      "activations/layer7_attention_weight_max": 72.36319732666016,
      "activations/layer7_attention_weight_min": -72.45800018310547,
      "activations/layer8_attention_weight_max": 47.062686920166016,
      "activations/layer8_attention_weight_min": -51.641056060791016,
      "activations/layer9_attention_weight_max": 51.349151611328125,
      "activations/layer9_attention_weight_min": -48.37075424194336,
      "epoch": 12.06,
      "learning_rate": 7.29651515151515e-05,
      "loss": 2.7738,
      "step": 207550
    },
    {
      "activations/layer0_attention_weight_max": 15.601197242736816,
      "activations/layer0_attention_weight_min": -14.068333625793457,
      "activations/layer10_attention_weight_max": 35.051673889160156,
      "activations/layer10_attention_weight_min": -35.1331901550293,
      "activations/layer11_attention_weight_max": 32.34320831298828,
      "activations/layer11_attention_weight_min": -32.18572235107422,
      "activations/layer12_attention_weight_max": 20.976213455200195,
      "activations/layer12_attention_weight_min": -28.04230499267578,
      "activations/layer13_attention_weight_max": 38.7107048034668,
      "activations/layer13_attention_weight_min": -28.446035385131836,
      "activations/layer14_attention_weight_max": 48.078433990478516,
      "activations/layer14_attention_weight_min": -31.035999298095703,
      "activations/layer15_attention_weight_max": 41.63951873779297,
      "activations/layer15_attention_weight_min": -30.524364471435547,
      "activations/layer16_attention_weight_max": 32.79264450073242,
      "activations/layer16_attention_weight_min": -30.472896575927734,
      "activations/layer17_attention_weight_max": 55.18907928466797,
      "activations/layer17_attention_weight_min": -46.833377838134766,
      "activations/layer18_attention_weight_max": 53.32028579711914,
      "activations/layer18_attention_weight_min": -39.18137741088867,
      "activations/layer19_attention_weight_max": 21.69685173034668,
      "activations/layer19_attention_weight_min": -23.059255599975586,
      "activations/layer1_attention_weight_max": 16.951740264892578,
      "activations/layer1_attention_weight_min": -14.416730880737305,
      "activations/layer20_attention_weight_max": 23.565031051635742,
      "activations/layer20_attention_weight_min": -24.43883514404297,
      "activations/layer21_attention_weight_max": 47.7841911315918,
      "activations/layer21_attention_weight_min": -21.51291847229004,
      "activations/layer22_attention_weight_max": 31.720199584960938,
      "activations/layer22_attention_weight_min": -25.974609375,
      "activations/layer23_attention_weight_max": 44.53459930419922,
      "activations/layer23_attention_weight_min": -25.205127716064453,
      "activations/layer2_attention_weight_max": 32.857025146484375,
      "activations/layer2_attention_weight_min": -31.961406707763672,
      "activations/layer3_attention_weight_max": 88.41317749023438,
      "activations/layer3_attention_weight_min": -90.88300323486328,
      "activations/layer4_attention_weight_max": 95.64403533935547,
      "activations/layer4_attention_weight_min": -91.01049041748047,
      "activations/layer5_attention_weight_max": 73.91143035888672,
      "activations/layer5_attention_weight_min": -75.41929626464844,
      "activations/layer6_attention_weight_max": 53.642093658447266,
      "activations/layer6_attention_weight_min": -51.3112678527832,
      "activations/layer7_attention_weight_max": 70.08268737792969,
      "activations/layer7_attention_weight_min": -64.83584594726562,
      "activations/layer8_attention_weight_max": 45.179534912109375,
      "activations/layer8_attention_weight_min": -47.47990036010742,
      "activations/layer9_attention_weight_max": 46.621395111083984,
      "activations/layer9_attention_weight_min": -44.24282455444336,
      "epoch": 12.06,
      "learning_rate": 7.294621212121212e-05,
      "loss": 2.762,
      "step": 207600
    },
    {
      "activations/layer0_attention_weight_max": 15.09013557434082,
      "activations/layer0_attention_weight_min": -14.078757286071777,
      "activations/layer10_attention_weight_max": 37.84291076660156,
      "activations/layer10_attention_weight_min": -35.39170837402344,
      "activations/layer11_attention_weight_max": 33.46875762939453,
      "activations/layer11_attention_weight_min": -33.141578674316406,
      "activations/layer12_attention_weight_max": 19.05653190612793,
      "activations/layer12_attention_weight_min": -26.95431900024414,
      "activations/layer13_attention_weight_max": 35.76309585571289,
      "activations/layer13_attention_weight_min": -29.36095428466797,
      "activations/layer14_attention_weight_max": 39.02043533325195,
      "activations/layer14_attention_weight_min": -30.038314819335938,
      "activations/layer15_attention_weight_max": 35.81655502319336,
      "activations/layer15_attention_weight_min": -30.89495086669922,
      "activations/layer16_attention_weight_max": 36.737327575683594,
      "activations/layer16_attention_weight_min": -29.007253646850586,
      "activations/layer17_attention_weight_max": 55.220924377441406,
      "activations/layer17_attention_weight_min": -44.02983093261719,
      "activations/layer18_attention_weight_max": 53.77501678466797,
      "activations/layer18_attention_weight_min": -39.597679138183594,
      "activations/layer19_attention_weight_max": 30.5135498046875,
      "activations/layer19_attention_weight_min": -24.999940872192383,
      "activations/layer1_attention_weight_max": 16.60477066040039,
      "activations/layer1_attention_weight_min": -14.626864433288574,
      "activations/layer20_attention_weight_max": 31.263687133789062,
      "activations/layer20_attention_weight_min": -21.383594512939453,
      "activations/layer21_attention_weight_max": 53.14317321777344,
      "activations/layer21_attention_weight_min": -25.91672706604004,
      "activations/layer22_attention_weight_max": 41.9655647277832,
      "activations/layer22_attention_weight_min": -25.845443725585938,
      "activations/layer23_attention_weight_max": 48.18608093261719,
      "activations/layer23_attention_weight_min": -26.306859970092773,
      "activations/layer2_attention_weight_max": 32.21943664550781,
      "activations/layer2_attention_weight_min": -31.51331329345703,
      "activations/layer3_attention_weight_max": 91.40939331054688,
      "activations/layer3_attention_weight_min": -91.0667953491211,
      "activations/layer4_attention_weight_max": 96.63280487060547,
      "activations/layer4_attention_weight_min": -92.10013580322266,
      "activations/layer5_attention_weight_max": 71.15818786621094,
      "activations/layer5_attention_weight_min": -77.76104736328125,
      "activations/layer6_attention_weight_max": 52.55797576904297,
      "activations/layer6_attention_weight_min": -52.92141342163086,
      "activations/layer7_attention_weight_max": 69.40530395507812,
      "activations/layer7_attention_weight_min": -68.3883285522461,
      "activations/layer8_attention_weight_max": 46.08700180053711,
      "activations/layer8_attention_weight_min": -49.063446044921875,
      "activations/layer9_attention_weight_max": 47.36812973022461,
      "activations/layer9_attention_weight_min": -47.47909164428711,
      "epoch": 12.07,
      "learning_rate": 7.292727272727272e-05,
      "loss": 2.7719,
      "step": 207650
    },
    {
      "activations/layer0_attention_weight_max": 15.577048301696777,
      "activations/layer0_attention_weight_min": -14.539690017700195,
      "activations/layer10_attention_weight_max": 36.57475662231445,
      "activations/layer10_attention_weight_min": -36.06621170043945,
      "activations/layer11_attention_weight_max": 31.569921493530273,
      "activations/layer11_attention_weight_min": -32.71414566040039,
      "activations/layer12_attention_weight_max": 18.137622833251953,
      "activations/layer12_attention_weight_min": -26.173158645629883,
      "activations/layer13_attention_weight_max": 35.61647033691406,
      "activations/layer13_attention_weight_min": -32.476051330566406,
      "activations/layer14_attention_weight_max": 35.34741973876953,
      "activations/layer14_attention_weight_min": -30.881196975708008,
      "activations/layer15_attention_weight_max": 34.656349182128906,
      "activations/layer15_attention_weight_min": -28.98984718322754,
      "activations/layer16_attention_weight_max": 29.707927703857422,
      "activations/layer16_attention_weight_min": -28.364578247070312,
      "activations/layer17_attention_weight_max": 50.68825912475586,
      "activations/layer17_attention_weight_min": -44.57170104980469,
      "activations/layer18_attention_weight_max": 44.488155364990234,
      "activations/layer18_attention_weight_min": -38.55860137939453,
      "activations/layer19_attention_weight_max": 20.532827377319336,
      "activations/layer19_attention_weight_min": -21.352916717529297,
      "activations/layer1_attention_weight_max": 17.197824478149414,
      "activations/layer1_attention_weight_min": -13.973003387451172,
      "activations/layer20_attention_weight_max": 22.843290328979492,
      "activations/layer20_attention_weight_min": -21.532142639160156,
      "activations/layer21_attention_weight_max": 35.65590286254883,
      "activations/layer21_attention_weight_min": -24.09395408630371,
      "activations/layer22_attention_weight_max": 32.00202178955078,
      "activations/layer22_attention_weight_min": -25.281538009643555,
      "activations/layer23_attention_weight_max": 41.1020393371582,
      "activations/layer23_attention_weight_min": -24.58148956298828,
      "activations/layer2_attention_weight_max": 31.945091247558594,
      "activations/layer2_attention_weight_min": -31.190608978271484,
      "activations/layer3_attention_weight_max": 90.53172302246094,
      "activations/layer3_attention_weight_min": -88.45011901855469,
      "activations/layer4_attention_weight_max": 92.4230728149414,
      "activations/layer4_attention_weight_min": -95.17657470703125,
      "activations/layer5_attention_weight_max": 70.41009521484375,
      "activations/layer5_attention_weight_min": -72.67822265625,
      "activations/layer6_attention_weight_max": 50.66825485229492,
      "activations/layer6_attention_weight_min": -50.498538970947266,
      "activations/layer7_attention_weight_max": 68.03208923339844,
      "activations/layer7_attention_weight_min": -62.984703063964844,
      "activations/layer8_attention_weight_max": 43.219295501708984,
      "activations/layer8_attention_weight_min": -46.826393127441406,
      "activations/layer9_attention_weight_max": 44.16792678833008,
      "activations/layer9_attention_weight_min": -45.87351608276367,
      "epoch": 12.07,
      "learning_rate": 7.290833333333333e-05,
      "loss": 2.7683,
      "step": 207700
    },
    {
      "activations/layer0_attention_weight_max": 16.566850662231445,
      "activations/layer0_attention_weight_min": -14.681716918945312,
      "activations/layer10_attention_weight_max": 35.87944030761719,
      "activations/layer10_attention_weight_min": -35.517173767089844,
      "activations/layer11_attention_weight_max": 31.693069458007812,
      "activations/layer11_attention_weight_min": -32.32250213623047,
      "activations/layer12_attention_weight_max": 16.922847747802734,
      "activations/layer12_attention_weight_min": -24.839191436767578,
      "activations/layer13_attention_weight_max": 33.92969512939453,
      "activations/layer13_attention_weight_min": -29.191150665283203,
      "activations/layer14_attention_weight_max": 34.43589401245117,
      "activations/layer14_attention_weight_min": -29.54133415222168,
      "activations/layer15_attention_weight_max": 30.013784408569336,
      "activations/layer15_attention_weight_min": -29.846677780151367,
      "activations/layer16_attention_weight_max": 29.82785987854004,
      "activations/layer16_attention_weight_min": -31.289033889770508,
      "activations/layer17_attention_weight_max": 50.25136184692383,
      "activations/layer17_attention_weight_min": -44.8834114074707,
      "activations/layer18_attention_weight_max": 43.09665298461914,
      "activations/layer18_attention_weight_min": -38.45977020263672,
      "activations/layer19_attention_weight_max": 20.765806198120117,
      "activations/layer19_attention_weight_min": -23.010467529296875,
      "activations/layer1_attention_weight_max": 17.07220458984375,
      "activations/layer1_attention_weight_min": -13.970049858093262,
      "activations/layer20_attention_weight_max": 20.386167526245117,
      "activations/layer20_attention_weight_min": -23.82658576965332,
      "activations/layer21_attention_weight_max": 33.9272575378418,
      "activations/layer21_attention_weight_min": -22.6035099029541,
      "activations/layer22_attention_weight_max": 29.86852264404297,
      "activations/layer22_attention_weight_min": -25.06541633605957,
      "activations/layer23_attention_weight_max": 34.815765380859375,
      "activations/layer23_attention_weight_min": -24.333812713623047,
      "activations/layer2_attention_weight_max": 31.42770767211914,
      "activations/layer2_attention_weight_min": -29.070690155029297,
      "activations/layer3_attention_weight_max": 89.58505249023438,
      "activations/layer3_attention_weight_min": -90.66192626953125,
      "activations/layer4_attention_weight_max": 97.11203002929688,
      "activations/layer4_attention_weight_min": -92.8389663696289,
      "activations/layer5_attention_weight_max": 70.37493896484375,
      "activations/layer5_attention_weight_min": -81.09114837646484,
      "activations/layer6_attention_weight_max": 52.702247619628906,
      "activations/layer6_attention_weight_min": -52.33747482299805,
      "activations/layer7_attention_weight_max": 68.95448303222656,
      "activations/layer7_attention_weight_min": -65.25965881347656,
      "activations/layer8_attention_weight_max": 44.494720458984375,
      "activations/layer8_attention_weight_min": -47.18305587768555,
      "activations/layer9_attention_weight_max": 46.035640716552734,
      "activations/layer9_attention_weight_min": -45.207618713378906,
      "epoch": 12.07,
      "learning_rate": 7.288939393939394e-05,
      "loss": 2.7789,
      "step": 207750
    },
    {
      "activations/layer0_attention_weight_max": 15.748722076416016,
      "activations/layer0_attention_weight_min": -14.415815353393555,
      "activations/layer10_attention_weight_max": 35.62421798706055,
      "activations/layer10_attention_weight_min": -35.747676849365234,
      "activations/layer11_attention_weight_max": 32.79129409790039,
      "activations/layer11_attention_weight_min": -33.518524169921875,
      "activations/layer12_attention_weight_max": 19.828073501586914,
      "activations/layer12_attention_weight_min": -26.58379554748535,
      "activations/layer13_attention_weight_max": 43.464900970458984,
      "activations/layer13_attention_weight_min": -29.109004974365234,
      "activations/layer14_attention_weight_max": 43.18629455566406,
      "activations/layer14_attention_weight_min": -29.756620407104492,
      "activations/layer15_attention_weight_max": 36.54914093017578,
      "activations/layer15_attention_weight_min": -28.524675369262695,
      "activations/layer16_attention_weight_max": 32.94788360595703,
      "activations/layer16_attention_weight_min": -29.05420684814453,
      "activations/layer17_attention_weight_max": 57.69441604614258,
      "activations/layer17_attention_weight_min": -43.384307861328125,
      "activations/layer18_attention_weight_max": 55.589664459228516,
      "activations/layer18_attention_weight_min": -36.767547607421875,
      "activations/layer19_attention_weight_max": 29.373397827148438,
      "activations/layer19_attention_weight_min": -21.640182495117188,
      "activations/layer1_attention_weight_max": 16.980907440185547,
      "activations/layer1_attention_weight_min": -14.950825691223145,
      "activations/layer20_attention_weight_max": 24.587112426757812,
      "activations/layer20_attention_weight_min": -21.19183349609375,
      "activations/layer21_attention_weight_max": 37.37196731567383,
      "activations/layer21_attention_weight_min": -22.90489959716797,
      "activations/layer22_attention_weight_max": 32.540443420410156,
      "activations/layer22_attention_weight_min": -25.746192932128906,
      "activations/layer23_attention_weight_max": 38.46187210083008,
      "activations/layer23_attention_weight_min": -25.095876693725586,
      "activations/layer2_attention_weight_max": 32.978248596191406,
      "activations/layer2_attention_weight_min": -30.842931747436523,
      "activations/layer3_attention_weight_max": 89.99759674072266,
      "activations/layer3_attention_weight_min": -90.08099365234375,
      "activations/layer4_attention_weight_max": 96.36231231689453,
      "activations/layer4_attention_weight_min": -96.20305633544922,
      "activations/layer5_attention_weight_max": 72.65225982666016,
      "activations/layer5_attention_weight_min": -76.34840393066406,
      "activations/layer6_attention_weight_max": 52.6825065612793,
      "activations/layer6_attention_weight_min": -51.95522689819336,
      "activations/layer7_attention_weight_max": 77.02447509765625,
      "activations/layer7_attention_weight_min": -69.19940948486328,
      "activations/layer8_attention_weight_max": 46.302391052246094,
      "activations/layer8_attention_weight_min": -46.09782028198242,
      "activations/layer9_attention_weight_max": 52.09556198120117,
      "activations/layer9_attention_weight_min": -47.55797576904297,
      "epoch": 12.07,
      "learning_rate": 7.287045454545454e-05,
      "loss": 2.7744,
      "step": 207800
    },
    {
      "activations/layer0_attention_weight_max": 15.557852745056152,
      "activations/layer0_attention_weight_min": -14.975619316101074,
      "activations/layer10_attention_weight_max": 38.35493850708008,
      "activations/layer10_attention_weight_min": -40.078346252441406,
      "activations/layer11_attention_weight_max": 33.802764892578125,
      "activations/layer11_attention_weight_min": -33.397682189941406,
      "activations/layer12_attention_weight_max": 18.735502243041992,
      "activations/layer12_attention_weight_min": -23.740447998046875,
      "activations/layer13_attention_weight_max": 40.52608871459961,
      "activations/layer13_attention_weight_min": -30.13524627685547,
      "activations/layer14_attention_weight_max": 37.14609909057617,
      "activations/layer14_attention_weight_min": -28.069379806518555,
      "activations/layer15_attention_weight_max": 35.538909912109375,
      "activations/layer15_attention_weight_min": -29.15612030029297,
      "activations/layer16_attention_weight_max": 31.652469635009766,
      "activations/layer16_attention_weight_min": -28.165212631225586,
      "activations/layer17_attention_weight_max": 54.48149490356445,
      "activations/layer17_attention_weight_min": -43.883609771728516,
      "activations/layer18_attention_weight_max": 47.020538330078125,
      "activations/layer18_attention_weight_min": -36.49745178222656,
      "activations/layer19_attention_weight_max": 22.23835563659668,
      "activations/layer19_attention_weight_min": -23.181337356567383,
      "activations/layer1_attention_weight_max": 17.928274154663086,
      "activations/layer1_attention_weight_min": -15.14215087890625,
      "activations/layer20_attention_weight_max": 23.365556716918945,
      "activations/layer20_attention_weight_min": -25.27777671813965,
      "activations/layer21_attention_weight_max": 35.02392578125,
      "activations/layer21_attention_weight_min": -20.84918785095215,
      "activations/layer22_attention_weight_max": 28.126985549926758,
      "activations/layer22_attention_weight_min": -26.78328514099121,
      "activations/layer23_attention_weight_max": 35.560577392578125,
      "activations/layer23_attention_weight_min": -23.66493797302246,
      "activations/layer2_attention_weight_max": 35.34629821777344,
      "activations/layer2_attention_weight_min": -33.08363342285156,
      "activations/layer3_attention_weight_max": 94.09664154052734,
      "activations/layer3_attention_weight_min": -93.44636535644531,
      "activations/layer4_attention_weight_max": 99.47002410888672,
      "activations/layer4_attention_weight_min": -95.3563461303711,
      "activations/layer5_attention_weight_max": 76.08755493164062,
      "activations/layer5_attention_weight_min": -80.18254089355469,
      "activations/layer6_attention_weight_max": 54.06127166748047,
      "activations/layer6_attention_weight_min": -52.797630310058594,
      "activations/layer7_attention_weight_max": 74.1747055053711,
      "activations/layer7_attention_weight_min": -67.06588745117188,
      "activations/layer8_attention_weight_max": 47.799583435058594,
      "activations/layer8_attention_weight_min": -50.599735260009766,
      "activations/layer9_attention_weight_max": 51.226802825927734,
      "activations/layer9_attention_weight_min": -46.76615524291992,
      "epoch": 12.08,
      "learning_rate": 7.285151515151514e-05,
      "loss": 2.772,
      "step": 207850
    },
    {
      "activations/layer0_attention_weight_max": 16.309228897094727,
      "activations/layer0_attention_weight_min": -14.55935001373291,
      "activations/layer10_attention_weight_max": 35.37358093261719,
      "activations/layer10_attention_weight_min": -34.63690948486328,
      "activations/layer11_attention_weight_max": 32.262596130371094,
      "activations/layer11_attention_weight_min": -32.93375778198242,
      "activations/layer12_attention_weight_max": 20.01127052307129,
      "activations/layer12_attention_weight_min": -24.931434631347656,
      "activations/layer13_attention_weight_max": 38.537410736083984,
      "activations/layer13_attention_weight_min": -32.46238327026367,
      "activations/layer14_attention_weight_max": 37.02324295043945,
      "activations/layer14_attention_weight_min": -31.31623077392578,
      "activations/layer15_attention_weight_max": 35.613372802734375,
      "activations/layer15_attention_weight_min": -30.17430877685547,
      "activations/layer16_attention_weight_max": 32.78293991088867,
      "activations/layer16_attention_weight_min": -29.803016662597656,
      "activations/layer17_attention_weight_max": 55.72193908691406,
      "activations/layer17_attention_weight_min": -45.244178771972656,
      "activations/layer18_attention_weight_max": 50.85532760620117,
      "activations/layer18_attention_weight_min": -37.93736267089844,
      "activations/layer19_attention_weight_max": 22.643239974975586,
      "activations/layer19_attention_weight_min": -23.168222427368164,
      "activations/layer1_attention_weight_max": 16.098539352416992,
      "activations/layer1_attention_weight_min": -13.647510528564453,
      "activations/layer20_attention_weight_max": 23.655071258544922,
      "activations/layer20_attention_weight_min": -21.05145835876465,
      "activations/layer21_attention_weight_max": 39.6646728515625,
      "activations/layer21_attention_weight_min": -23.257980346679688,
      "activations/layer22_attention_weight_max": 28.72161102294922,
      "activations/layer22_attention_weight_min": -27.533586502075195,
      "activations/layer23_attention_weight_max": 37.73764419555664,
      "activations/layer23_attention_weight_min": -23.986469268798828,
      "activations/layer2_attention_weight_max": 30.374324798583984,
      "activations/layer2_attention_weight_min": -27.754695892333984,
      "activations/layer3_attention_weight_max": 86.79859924316406,
      "activations/layer3_attention_weight_min": -88.22412872314453,
      "activations/layer4_attention_weight_max": 89.97217559814453,
      "activations/layer4_attention_weight_min": -88.63180541992188,
      "activations/layer5_attention_weight_max": 69.03105163574219,
      "activations/layer5_attention_weight_min": -78.39897155761719,
      "activations/layer6_attention_weight_max": 51.28632354736328,
      "activations/layer6_attention_weight_min": -51.656734466552734,
      "activations/layer7_attention_weight_max": 66.86715698242188,
      "activations/layer7_attention_weight_min": -65.37100219726562,
      "activations/layer8_attention_weight_max": 42.83375930786133,
      "activations/layer8_attention_weight_min": -45.53792190551758,
      "activations/layer9_attention_weight_max": 42.272216796875,
      "activations/layer9_attention_weight_min": -46.04448318481445,
      "epoch": 12.08,
      "learning_rate": 7.283257575757575e-05,
      "loss": 2.7596,
      "step": 207900
    },
    {
      "activations/layer0_attention_weight_max": 16.5888671875,
      "activations/layer0_attention_weight_min": -14.260677337646484,
      "activations/layer10_attention_weight_max": 35.898597717285156,
      "activations/layer10_attention_weight_min": -36.03031921386719,
      "activations/layer11_attention_weight_max": 33.035728454589844,
      "activations/layer11_attention_weight_min": -33.095054626464844,
      "activations/layer12_attention_weight_max": 20.364633560180664,
      "activations/layer12_attention_weight_min": -25.027076721191406,
      "activations/layer13_attention_weight_max": 41.73226547241211,
      "activations/layer13_attention_weight_min": -30.801965713500977,
      "activations/layer14_attention_weight_max": 37.52272033691406,
      "activations/layer14_attention_weight_min": -30.611572265625,
      "activations/layer15_attention_weight_max": 33.420005798339844,
      "activations/layer15_attention_weight_min": -29.398590087890625,
      "activations/layer16_attention_weight_max": 31.842058181762695,
      "activations/layer16_attention_weight_min": -31.114009857177734,
      "activations/layer17_attention_weight_max": 53.331809997558594,
      "activations/layer17_attention_weight_min": -48.620758056640625,
      "activations/layer18_attention_weight_max": 46.07060623168945,
      "activations/layer18_attention_weight_min": -44.64965057373047,
      "activations/layer19_attention_weight_max": 24.386024475097656,
      "activations/layer19_attention_weight_min": -22.583791732788086,
      "activations/layer1_attention_weight_max": 16.175294876098633,
      "activations/layer1_attention_weight_min": -15.536558151245117,
      "activations/layer20_attention_weight_max": 22.88455581665039,
      "activations/layer20_attention_weight_min": -20.977914810180664,
      "activations/layer21_attention_weight_max": 43.3809700012207,
      "activations/layer21_attention_weight_min": -23.955322265625,
      "activations/layer22_attention_weight_max": 30.72850227355957,
      "activations/layer22_attention_weight_min": -24.338409423828125,
      "activations/layer23_attention_weight_max": 41.5409049987793,
      "activations/layer23_attention_weight_min": -24.509740829467773,
      "activations/layer2_attention_weight_max": 33.99878692626953,
      "activations/layer2_attention_weight_min": -31.349472045898438,
      "activations/layer3_attention_weight_max": 89.38827514648438,
      "activations/layer3_attention_weight_min": -94.48406982421875,
      "activations/layer4_attention_weight_max": 97.25798034667969,
      "activations/layer4_attention_weight_min": -91.07428741455078,
      "activations/layer5_attention_weight_max": 68.03828430175781,
      "activations/layer5_attention_weight_min": -76.76583862304688,
      "activations/layer6_attention_weight_max": 52.35548400878906,
      "activations/layer6_attention_weight_min": -55.140960693359375,
      "activations/layer7_attention_weight_max": 65.83939361572266,
      "activations/layer7_attention_weight_min": -67.67594909667969,
      "activations/layer8_attention_weight_max": 44.6032829284668,
      "activations/layer8_attention_weight_min": -50.08324432373047,
      "activations/layer9_attention_weight_max": 50.422203063964844,
      "activations/layer9_attention_weight_min": -49.9534912109375,
      "epoch": 12.08,
      "learning_rate": 7.281363636363636e-05,
      "loss": 2.7626,
      "step": 207950
    },
    {
      "activations/layer0_attention_weight_max": 15.280081748962402,
      "activations/layer0_attention_weight_min": -13.861030578613281,
      "activations/layer10_attention_weight_max": 37.16663360595703,
      "activations/layer10_attention_weight_min": -34.06349182128906,
      "activations/layer11_attention_weight_max": 32.07460021972656,
      "activations/layer11_attention_weight_min": -32.90238571166992,
      "activations/layer12_attention_weight_max": 23.532440185546875,
      "activations/layer12_attention_weight_min": -21.291797637939453,
      "activations/layer13_attention_weight_max": 35.258636474609375,
      "activations/layer13_attention_weight_min": -29.704761505126953,
      "activations/layer14_attention_weight_max": 37.772132873535156,
      "activations/layer14_attention_weight_min": -30.995147705078125,
      "activations/layer15_attention_weight_max": 35.11497497558594,
      "activations/layer15_attention_weight_min": -31.23195457458496,
      "activations/layer16_attention_weight_max": 36.306339263916016,
      "activations/layer16_attention_weight_min": -30.73614501953125,
      "activations/layer17_attention_weight_max": 52.18173599243164,
      "activations/layer17_attention_weight_min": -44.767398834228516,
      "activations/layer18_attention_weight_max": 50.126861572265625,
      "activations/layer18_attention_weight_min": -38.2664909362793,
      "activations/layer19_attention_weight_max": 27.32210350036621,
      "activations/layer19_attention_weight_min": -23.648502349853516,
      "activations/layer1_attention_weight_max": 15.766256332397461,
      "activations/layer1_attention_weight_min": -14.831788063049316,
      "activations/layer20_attention_weight_max": 23.22581672668457,
      "activations/layer20_attention_weight_min": -23.091245651245117,
      "activations/layer21_attention_weight_max": 48.60324478149414,
      "activations/layer21_attention_weight_min": -24.94464111328125,
      "activations/layer22_attention_weight_max": 31.56605339050293,
      "activations/layer22_attention_weight_min": -26.662353515625,
      "activations/layer23_attention_weight_max": 39.38600540161133,
      "activations/layer23_attention_weight_min": -26.04517364501953,
      "activations/layer2_attention_weight_max": 30.908035278320312,
      "activations/layer2_attention_weight_min": -30.33308982849121,
      "activations/layer3_attention_weight_max": 87.25157165527344,
      "activations/layer3_attention_weight_min": -97.796630859375,
      "activations/layer4_attention_weight_max": 93.70870208740234,
      "activations/layer4_attention_weight_min": -91.48262023925781,
      "activations/layer5_attention_weight_max": 69.81585693359375,
      "activations/layer5_attention_weight_min": -80.37998962402344,
      "activations/layer6_attention_weight_max": 50.705223083496094,
      "activations/layer6_attention_weight_min": -52.41328811645508,
      "activations/layer7_attention_weight_max": 70.95939636230469,
      "activations/layer7_attention_weight_min": -70.31522369384766,
      "activations/layer8_attention_weight_max": 45.63735580444336,
      "activations/layer8_attention_weight_min": -47.21686935424805,
      "activations/layer9_attention_weight_max": 45.08304977416992,
      "activations/layer9_attention_weight_min": -45.16890335083008,
      "epoch": 12.09,
      "learning_rate": 7.279469696969696e-05,
      "loss": 2.7773,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5456,
      "eval_samples_per_second": 502.479,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5456,
      "eval_openwebtext_samples_per_second": 502.479,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.986,
      "eval_wikitext_samples_per_second": 229.611,
      "step": 208000
    },
    {
      "epoch": 12.09,
      "eval_lambada_loss": 2.513671875,
      "eval_lambada_ppl": 12.350195278830133,
      "eval_lambada_runtime": 9.6452,
      "eval_lambada_samples_per_second": 504.81,
      "step": 208000
    },
    {
      "activations/layer0_attention_weight_max": 15.290809631347656,
      "activations/layer0_attention_weight_min": -13.542073249816895,
      "activations/layer10_attention_weight_max": 41.053619384765625,
      "activations/layer10_attention_weight_min": -36.96998977661133,
      "activations/layer11_attention_weight_max": 32.604530334472656,
      "activations/layer11_attention_weight_min": -31.033559799194336,
      "activations/layer12_attention_weight_max": 22.897689819335938,
      "activations/layer12_attention_weight_min": -22.33176612854004,
      "activations/layer13_attention_weight_max": 47.07933807373047,
      "activations/layer13_attention_weight_min": -29.71225929260254,
      "activations/layer14_attention_weight_max": 35.142005920410156,
      "activations/layer14_attention_weight_min": -32.29306411743164,
      "activations/layer15_attention_weight_max": 38.47239685058594,
      "activations/layer15_attention_weight_min": -29.677248001098633,
      "activations/layer16_attention_weight_max": 33.055747985839844,
      "activations/layer16_attention_weight_min": -30.151111602783203,
      "activations/layer17_attention_weight_max": 53.1701774597168,
      "activations/layer17_attention_weight_min": -44.14216613769531,
      "activations/layer18_attention_weight_max": 48.54474639892578,
      "activations/layer18_attention_weight_min": -37.5432243347168,
      "activations/layer19_attention_weight_max": 25.15833282470703,
      "activations/layer19_attention_weight_min": -21.56366729736328,
      "activations/layer1_attention_weight_max": 16.19329261779785,
      "activations/layer1_attention_weight_min": -15.460000991821289,
      "activations/layer20_attention_weight_max": 25.792490005493164,
      "activations/layer20_attention_weight_min": -24.459617614746094,
      "activations/layer21_attention_weight_max": 38.51953887939453,
      "activations/layer21_attention_weight_min": -23.16695213317871,
      "activations/layer22_attention_weight_max": 33.331077575683594,
      "activations/layer22_attention_weight_min": -26.52469253540039,
      "activations/layer23_attention_weight_max": 37.676265716552734,
      "activations/layer23_attention_weight_min": -24.207416534423828,
      "activations/layer2_attention_weight_max": 31.638158798217773,
      "activations/layer2_attention_weight_min": -29.854923248291016,
      "activations/layer3_attention_weight_max": 90.45321655273438,
      "activations/layer3_attention_weight_min": -91.27295684814453,
      "activations/layer4_attention_weight_max": 99.77519989013672,
      "activations/layer4_attention_weight_min": -96.0057373046875,
      "activations/layer5_attention_weight_max": 72.94012451171875,
      "activations/layer5_attention_weight_min": -78.26083374023438,
      "activations/layer6_attention_weight_max": 52.965721130371094,
      "activations/layer6_attention_weight_min": -53.58070755004883,
      "activations/layer7_attention_weight_max": 69.53506469726562,
      "activations/layer7_attention_weight_min": -71.95979309082031,
      "activations/layer8_attention_weight_max": 46.53547286987305,
      "activations/layer8_attention_weight_min": -46.94978713989258,
      "activations/layer9_attention_weight_max": 49.230377197265625,
      "activations/layer9_attention_weight_min": -48.800601959228516,
      "epoch": 12.09,
      "learning_rate": 7.277575757575757e-05,
      "loss": 2.7725,
      "step": 208050
    },
    {
      "activations/layer0_attention_weight_max": 15.753679275512695,
      "activations/layer0_attention_weight_min": -13.847113609313965,
      "activations/layer10_attention_weight_max": 36.22369384765625,
      "activations/layer10_attention_weight_min": -35.24726486206055,
      "activations/layer11_attention_weight_max": 34.597068786621094,
      "activations/layer11_attention_weight_min": -32.096534729003906,
      "activations/layer12_attention_weight_max": 25.392526626586914,
      "activations/layer12_attention_weight_min": -25.131900787353516,
      "activations/layer13_attention_weight_max": 37.02685546875,
      "activations/layer13_attention_weight_min": -30.9288272857666,
      "activations/layer14_attention_weight_max": 36.53974914550781,
      "activations/layer14_attention_weight_min": -29.74382972717285,
      "activations/layer15_attention_weight_max": 31.72749137878418,
      "activations/layer15_attention_weight_min": -27.54795265197754,
      "activations/layer16_attention_weight_max": 29.50244140625,
      "activations/layer16_attention_weight_min": -27.949691772460938,
      "activations/layer17_attention_weight_max": 46.66115188598633,
      "activations/layer17_attention_weight_min": -41.30964279174805,
      "activations/layer18_attention_weight_max": 42.846168518066406,
      "activations/layer18_attention_weight_min": -35.015567779541016,
      "activations/layer19_attention_weight_max": 22.742002487182617,
      "activations/layer19_attention_weight_min": -21.106863021850586,
      "activations/layer1_attention_weight_max": 17.250099182128906,
      "activations/layer1_attention_weight_min": -14.410423278808594,
      "activations/layer20_attention_weight_max": 21.19246482849121,
      "activations/layer20_attention_weight_min": -24.813005447387695,
      "activations/layer21_attention_weight_max": 34.1981086730957,
      "activations/layer21_attention_weight_min": -23.12874984741211,
      "activations/layer22_attention_weight_max": 27.60846710205078,
      "activations/layer22_attention_weight_min": -24.847131729125977,
      "activations/layer23_attention_weight_max": 34.99850845336914,
      "activations/layer23_attention_weight_min": -23.637014389038086,
      "activations/layer2_attention_weight_max": 29.9957332611084,
      "activations/layer2_attention_weight_min": -29.386951446533203,
      "activations/layer3_attention_weight_max": 86.6891098022461,
      "activations/layer3_attention_weight_min": -89.34770965576172,
      "activations/layer4_attention_weight_max": 96.51028442382812,
      "activations/layer4_attention_weight_min": -92.78687286376953,
      "activations/layer5_attention_weight_max": 74.14844512939453,
      "activations/layer5_attention_weight_min": -79.84481811523438,
      "activations/layer6_attention_weight_max": 52.07927703857422,
      "activations/layer6_attention_weight_min": -52.81340026855469,
      "activations/layer7_attention_weight_max": 71.30601501464844,
      "activations/layer7_attention_weight_min": -66.18492889404297,
      "activations/layer8_attention_weight_max": 45.51576614379883,
      "activations/layer8_attention_weight_min": -45.31549835205078,
      "activations/layer9_attention_weight_max": 52.47597885131836,
      "activations/layer9_attention_weight_min": -46.16246795654297,
      "epoch": 12.09,
      "learning_rate": 7.275681818181817e-05,
      "loss": 2.766,
      "step": 208100
    },
    {
      "activations/layer0_attention_weight_max": 15.604275703430176,
      "activations/layer0_attention_weight_min": -14.084503173828125,
      "activations/layer10_attention_weight_max": 37.600257873535156,
      "activations/layer10_attention_weight_min": -36.03779220581055,
      "activations/layer11_attention_weight_max": 34.72931671142578,
      "activations/layer11_attention_weight_min": -32.966712951660156,
      "activations/layer12_attention_weight_max": 18.124588012695312,
      "activations/layer12_attention_weight_min": -26.308490753173828,
      "activations/layer13_attention_weight_max": 40.0914421081543,
      "activations/layer13_attention_weight_min": -28.881183624267578,
      "activations/layer14_attention_weight_max": 40.543819427490234,
      "activations/layer14_attention_weight_min": -33.026145935058594,
      "activations/layer15_attention_weight_max": 37.38726043701172,
      "activations/layer15_attention_weight_min": -30.062835693359375,
      "activations/layer16_attention_weight_max": 34.603084564208984,
      "activations/layer16_attention_weight_min": -30.706632614135742,
      "activations/layer17_attention_weight_max": 61.59601593017578,
      "activations/layer17_attention_weight_min": -47.148826599121094,
      "activations/layer18_attention_weight_max": 51.282081604003906,
      "activations/layer18_attention_weight_min": -37.85374450683594,
      "activations/layer19_attention_weight_max": 24.65805435180664,
      "activations/layer19_attention_weight_min": -22.451736450195312,
      "activations/layer1_attention_weight_max": 17.845733642578125,
      "activations/layer1_attention_weight_min": -15.872645378112793,
      "activations/layer20_attention_weight_max": 26.76675033569336,
      "activations/layer20_attention_weight_min": -20.974538803100586,
      "activations/layer21_attention_weight_max": 39.841800689697266,
      "activations/layer21_attention_weight_min": -22.791677474975586,
      "activations/layer22_attention_weight_max": 31.283803939819336,
      "activations/layer22_attention_weight_min": -26.40554428100586,
      "activations/layer23_attention_weight_max": 39.19181823730469,
      "activations/layer23_attention_weight_min": -23.364110946655273,
      "activations/layer2_attention_weight_max": 30.29113006591797,
      "activations/layer2_attention_weight_min": -30.04629898071289,
      "activations/layer3_attention_weight_max": 83.31439208984375,
      "activations/layer3_attention_weight_min": -93.1562728881836,
      "activations/layer4_attention_weight_max": 94.42169952392578,
      "activations/layer4_attention_weight_min": -92.2623062133789,
      "activations/layer5_attention_weight_max": 68.04576110839844,
      "activations/layer5_attention_weight_min": -73.95356750488281,
      "activations/layer6_attention_weight_max": 51.68171691894531,
      "activations/layer6_attention_weight_min": -52.71671676635742,
      "activations/layer7_attention_weight_max": 72.2284164428711,
      "activations/layer7_attention_weight_min": -75.79159545898438,
      "activations/layer8_attention_weight_max": 48.30330276489258,
      "activations/layer8_attention_weight_min": -48.76321029663086,
      "activations/layer9_attention_weight_max": 46.1394157409668,
      "activations/layer9_attention_weight_min": -47.26203155517578,
      "epoch": 12.09,
      "learning_rate": 7.273787878787878e-05,
      "loss": 2.7811,
      "step": 208150
    },
    {
      "activations/layer0_attention_weight_max": 15.221254348754883,
      "activations/layer0_attention_weight_min": -13.835104942321777,
      "activations/layer10_attention_weight_max": 35.82426071166992,
      "activations/layer10_attention_weight_min": -34.547637939453125,
      "activations/layer11_attention_weight_max": 31.875234603881836,
      "activations/layer11_attention_weight_min": -32.84403991699219,
      "activations/layer12_attention_weight_max": 20.498172760009766,
      "activations/layer12_attention_weight_min": -24.581924438476562,
      "activations/layer13_attention_weight_max": 40.0377197265625,
      "activations/layer13_attention_weight_min": -28.833141326904297,
      "activations/layer14_attention_weight_max": 40.8015251159668,
      "activations/layer14_attention_weight_min": -32.36251449584961,
      "activations/layer15_attention_weight_max": 34.074134826660156,
      "activations/layer15_attention_weight_min": -31.087251663208008,
      "activations/layer16_attention_weight_max": 32.949981689453125,
      "activations/layer16_attention_weight_min": -30.56276512145996,
      "activations/layer17_attention_weight_max": 53.81862258911133,
      "activations/layer17_attention_weight_min": -46.36605453491211,
      "activations/layer18_attention_weight_max": 48.59231948852539,
      "activations/layer18_attention_weight_min": -37.48810958862305,
      "activations/layer19_attention_weight_max": 26.247344970703125,
      "activations/layer19_attention_weight_min": -21.785572052001953,
      "activations/layer1_attention_weight_max": 16.86433982849121,
      "activations/layer1_attention_weight_min": -14.883749008178711,
      "activations/layer20_attention_weight_max": 26.587650299072266,
      "activations/layer20_attention_weight_min": -22.2625789642334,
      "activations/layer21_attention_weight_max": 39.830116271972656,
      "activations/layer21_attention_weight_min": -21.466278076171875,
      "activations/layer22_attention_weight_max": 35.29478073120117,
      "activations/layer22_attention_weight_min": -26.893749237060547,
      "activations/layer23_attention_weight_max": 40.76519775390625,
      "activations/layer23_attention_weight_min": -25.690921783447266,
      "activations/layer2_attention_weight_max": 31.201656341552734,
      "activations/layer2_attention_weight_min": -29.440547943115234,
      "activations/layer3_attention_weight_max": 85.23383331298828,
      "activations/layer3_attention_weight_min": -87.5907974243164,
      "activations/layer4_attention_weight_max": 94.4217529296875,
      "activations/layer4_attention_weight_min": -90.39588165283203,
      "activations/layer5_attention_weight_max": 68.5980224609375,
      "activations/layer5_attention_weight_min": -76.1446304321289,
      "activations/layer6_attention_weight_max": 50.99632263183594,
      "activations/layer6_attention_weight_min": -52.636817932128906,
      "activations/layer7_attention_weight_max": 68.12680053710938,
      "activations/layer7_attention_weight_min": -70.2029037475586,
      "activations/layer8_attention_weight_max": 45.65363693237305,
      "activations/layer8_attention_weight_min": -45.11265182495117,
      "activations/layer9_attention_weight_max": 45.67991256713867,
      "activations/layer9_attention_weight_min": -46.732730865478516,
      "epoch": 12.1,
      "learning_rate": 7.271893939393939e-05,
      "loss": 2.7857,
      "step": 208200
    },
    {
      "activations/layer0_attention_weight_max": 15.276836395263672,
      "activations/layer0_attention_weight_min": -13.845041275024414,
      "activations/layer10_attention_weight_max": 37.21344757080078,
      "activations/layer10_attention_weight_min": -35.89485549926758,
      "activations/layer11_attention_weight_max": 31.3565731048584,
      "activations/layer11_attention_weight_min": -31.035263061523438,
      "activations/layer12_attention_weight_max": 17.539928436279297,
      "activations/layer12_attention_weight_min": -27.363407135009766,
      "activations/layer13_attention_weight_max": 31.86929702758789,
      "activations/layer13_attention_weight_min": -28.26984214782715,
      "activations/layer14_attention_weight_max": 33.01747512817383,
      "activations/layer14_attention_weight_min": -29.120956420898438,
      "activations/layer15_attention_weight_max": 31.378252029418945,
      "activations/layer15_attention_weight_min": -27.52444839477539,
      "activations/layer16_attention_weight_max": 30.019826889038086,
      "activations/layer16_attention_weight_min": -29.009143829345703,
      "activations/layer17_attention_weight_max": 51.058692932128906,
      "activations/layer17_attention_weight_min": -42.84953308105469,
      "activations/layer18_attention_weight_max": 46.33788299560547,
      "activations/layer18_attention_weight_min": -36.13857650756836,
      "activations/layer19_attention_weight_max": 23.45871353149414,
      "activations/layer19_attention_weight_min": -21.47498321533203,
      "activations/layer1_attention_weight_max": 17.21953010559082,
      "activations/layer1_attention_weight_min": -14.66176986694336,
      "activations/layer20_attention_weight_max": 21.601472854614258,
      "activations/layer20_attention_weight_min": -19.889049530029297,
      "activations/layer21_attention_weight_max": 37.59223937988281,
      "activations/layer21_attention_weight_min": -20.913591384887695,
      "activations/layer22_attention_weight_max": 37.80951690673828,
      "activations/layer22_attention_weight_min": -24.293977737426758,
      "activations/layer23_attention_weight_max": 36.137977600097656,
      "activations/layer23_attention_weight_min": -22.978164672851562,
      "activations/layer2_attention_weight_max": 31.480581283569336,
      "activations/layer2_attention_weight_min": -28.440000534057617,
      "activations/layer3_attention_weight_max": 85.82616424560547,
      "activations/layer3_attention_weight_min": -87.83729553222656,
      "activations/layer4_attention_weight_max": 95.08999633789062,
      "activations/layer4_attention_weight_min": -90.02606201171875,
      "activations/layer5_attention_weight_max": 71.22247314453125,
      "activations/layer5_attention_weight_min": -79.26251220703125,
      "activations/layer6_attention_weight_max": 51.748741149902344,
      "activations/layer6_attention_weight_min": -53.632930755615234,
      "activations/layer7_attention_weight_max": 70.02490997314453,
      "activations/layer7_attention_weight_min": -66.0808334350586,
      "activations/layer8_attention_weight_max": 46.6525764465332,
      "activations/layer8_attention_weight_min": -48.19594192504883,
      "activations/layer9_attention_weight_max": 50.58066940307617,
      "activations/layer9_attention_weight_min": -47.31319808959961,
      "epoch": 12.1,
      "learning_rate": 7.269999999999999e-05,
      "loss": 2.7602,
      "step": 208250
    },
    {
      "activations/layer0_attention_weight_max": 16.44611358642578,
      "activations/layer0_attention_weight_min": -13.985786437988281,
      "activations/layer10_attention_weight_max": 35.31248474121094,
      "activations/layer10_attention_weight_min": -35.761810302734375,
      "activations/layer11_attention_weight_max": 31.104373931884766,
      "activations/layer11_attention_weight_min": -32.06023406982422,
      "activations/layer12_attention_weight_max": 17.694805145263672,
      "activations/layer12_attention_weight_min": -25.03515625,
      "activations/layer13_attention_weight_max": 33.9656867980957,
      "activations/layer13_attention_weight_min": -29.012605667114258,
      "activations/layer14_attention_weight_max": 33.045894622802734,
      "activations/layer14_attention_weight_min": -30.75181007385254,
      "activations/layer15_attention_weight_max": 30.64541244506836,
      "activations/layer15_attention_weight_min": -29.70958137512207,
      "activations/layer16_attention_weight_max": 30.632556915283203,
      "activations/layer16_attention_weight_min": -29.006122589111328,
      "activations/layer17_attention_weight_max": 47.2991943359375,
      "activations/layer17_attention_weight_min": -41.72028350830078,
      "activations/layer18_attention_weight_max": 38.62888717651367,
      "activations/layer18_attention_weight_min": -37.0157585144043,
      "activations/layer19_attention_weight_max": 21.13239097595215,
      "activations/layer19_attention_weight_min": -21.898672103881836,
      "activations/layer1_attention_weight_max": 16.139081954956055,
      "activations/layer1_attention_weight_min": -14.903929710388184,
      "activations/layer20_attention_weight_max": 20.394718170166016,
      "activations/layer20_attention_weight_min": -24.437358856201172,
      "activations/layer21_attention_weight_max": 31.508953094482422,
      "activations/layer21_attention_weight_min": -22.846599578857422,
      "activations/layer22_attention_weight_max": 28.67110824584961,
      "activations/layer22_attention_weight_min": -25.699527740478516,
      "activations/layer23_attention_weight_max": 34.78360366821289,
      "activations/layer23_attention_weight_min": -23.960899353027344,
      "activations/layer2_attention_weight_max": 29.454944610595703,
      "activations/layer2_attention_weight_min": -28.27535629272461,
      "activations/layer3_attention_weight_max": 90.16880798339844,
      "activations/layer3_attention_weight_min": -94.80297088623047,
      "activations/layer4_attention_weight_max": 93.62041473388672,
      "activations/layer4_attention_weight_min": -90.49600219726562,
      "activations/layer5_attention_weight_max": 73.25173950195312,
      "activations/layer5_attention_weight_min": -79.78431701660156,
      "activations/layer6_attention_weight_max": 50.74325180053711,
      "activations/layer6_attention_weight_min": -48.26229476928711,
      "activations/layer7_attention_weight_max": 66.22295379638672,
      "activations/layer7_attention_weight_min": -68.09945678710938,
      "activations/layer8_attention_weight_max": 43.11390686035156,
      "activations/layer8_attention_weight_min": -45.9491081237793,
      "activations/layer9_attention_weight_max": 44.922611236572266,
      "activations/layer9_attention_weight_min": -45.50248718261719,
      "epoch": 12.1,
      "learning_rate": 7.268106060606061e-05,
      "loss": 2.763,
      "step": 208300
    },
    {
      "activations/layer0_attention_weight_max": 15.8128662109375,
      "activations/layer0_attention_weight_min": -13.657234191894531,
      "activations/layer10_attention_weight_max": 36.755035400390625,
      "activations/layer10_attention_weight_min": -36.28334426879883,
      "activations/layer11_attention_weight_max": 33.94392395019531,
      "activations/layer11_attention_weight_min": -33.656105041503906,
      "activations/layer12_attention_weight_max": 17.32709503173828,
      "activations/layer12_attention_weight_min": -24.259000778198242,
      "activations/layer13_attention_weight_max": 35.25250244140625,
      "activations/layer13_attention_weight_min": -34.214195251464844,
      "activations/layer14_attention_weight_max": 35.21354293823242,
      "activations/layer14_attention_weight_min": -29.37455177307129,
      "activations/layer15_attention_weight_max": 32.327598571777344,
      "activations/layer15_attention_weight_min": -30.3156795501709,
      "activations/layer16_attention_weight_max": 30.30064582824707,
      "activations/layer16_attention_weight_min": -28.017681121826172,
      "activations/layer17_attention_weight_max": 48.56107711791992,
      "activations/layer17_attention_weight_min": -42.5717658996582,
      "activations/layer18_attention_weight_max": 40.6415901184082,
      "activations/layer18_attention_weight_min": -37.613643646240234,
      "activations/layer19_attention_weight_max": 18.82520866394043,
      "activations/layer19_attention_weight_min": -24.95404624938965,
      "activations/layer1_attention_weight_max": 17.02161979675293,
      "activations/layer1_attention_weight_min": -15.60393238067627,
      "activations/layer20_attention_weight_max": 19.868541717529297,
      "activations/layer20_attention_weight_min": -21.86445426940918,
      "activations/layer21_attention_weight_max": 31.98710060119629,
      "activations/layer21_attention_weight_min": -22.707977294921875,
      "activations/layer22_attention_weight_max": 27.75828742980957,
      "activations/layer22_attention_weight_min": -26.557878494262695,
      "activations/layer23_attention_weight_max": 35.473968505859375,
      "activations/layer23_attention_weight_min": -25.779212951660156,
      "activations/layer2_attention_weight_max": 31.981870651245117,
      "activations/layer2_attention_weight_min": -29.699819564819336,
      "activations/layer3_attention_weight_max": 90.98629760742188,
      "activations/layer3_attention_weight_min": -90.74082946777344,
      "activations/layer4_attention_weight_max": 104.4751205444336,
      "activations/layer4_attention_weight_min": -96.84628295898438,
      "activations/layer5_attention_weight_max": 75.93461608886719,
      "activations/layer5_attention_weight_min": -77.30935668945312,
      "activations/layer6_attention_weight_max": 56.980167388916016,
      "activations/layer6_attention_weight_min": -53.59697341918945,
      "activations/layer7_attention_weight_max": 73.7579116821289,
      "activations/layer7_attention_weight_min": -68.85154724121094,
      "activations/layer8_attention_weight_max": 45.855308532714844,
      "activations/layer8_attention_weight_min": -48.91822052001953,
      "activations/layer9_attention_weight_max": 48.446250915527344,
      "activations/layer9_attention_weight_min": -50.284542083740234,
      "epoch": 12.11,
      "learning_rate": 7.266212121212121e-05,
      "loss": 2.7935,
      "step": 208350
    },
    {
      "activations/layer0_attention_weight_max": 15.39837646484375,
      "activations/layer0_attention_weight_min": -13.163681030273438,
      "activations/layer10_attention_weight_max": 37.43256759643555,
      "activations/layer10_attention_weight_min": -35.22343063354492,
      "activations/layer11_attention_weight_max": 32.488197326660156,
      "activations/layer11_attention_weight_min": -32.155662536621094,
      "activations/layer12_attention_weight_max": 21.1866455078125,
      "activations/layer12_attention_weight_min": -24.960193634033203,
      "activations/layer13_attention_weight_max": 33.65910339355469,
      "activations/layer13_attention_weight_min": -27.428932189941406,
      "activations/layer14_attention_weight_max": 37.79078674316406,
      "activations/layer14_attention_weight_min": -32.413387298583984,
      "activations/layer15_attention_weight_max": 34.89046859741211,
      "activations/layer15_attention_weight_min": -31.75712013244629,
      "activations/layer16_attention_weight_max": 32.612178802490234,
      "activations/layer16_attention_weight_min": -33.24677276611328,
      "activations/layer17_attention_weight_max": 55.30221939086914,
      "activations/layer17_attention_weight_min": -50.262420654296875,
      "activations/layer18_attention_weight_max": 49.7781982421875,
      "activations/layer18_attention_weight_min": -40.391685485839844,
      "activations/layer19_attention_weight_max": 23.506818771362305,
      "activations/layer19_attention_weight_min": -24.80677032470703,
      "activations/layer1_attention_weight_max": 16.999486923217773,
      "activations/layer1_attention_weight_min": -14.607416152954102,
      "activations/layer20_attention_weight_max": 25.024106979370117,
      "activations/layer20_attention_weight_min": -22.839523315429688,
      "activations/layer21_attention_weight_max": 40.37306594848633,
      "activations/layer21_attention_weight_min": -24.64000129699707,
      "activations/layer22_attention_weight_max": 34.15421676635742,
      "activations/layer22_attention_weight_min": -27.190080642700195,
      "activations/layer23_attention_weight_max": 43.40472412109375,
      "activations/layer23_attention_weight_min": -24.638980865478516,
      "activations/layer2_attention_weight_max": 30.478759765625,
      "activations/layer2_attention_weight_min": -29.57697868347168,
      "activations/layer3_attention_weight_max": 83.83065795898438,
      "activations/layer3_attention_weight_min": -86.45586395263672,
      "activations/layer4_attention_weight_max": 94.66625213623047,
      "activations/layer4_attention_weight_min": -90.52872467041016,
      "activations/layer5_attention_weight_max": 71.9378890991211,
      "activations/layer5_attention_weight_min": -74.15879821777344,
      "activations/layer6_attention_weight_max": 53.19123458862305,
      "activations/layer6_attention_weight_min": -50.39706802368164,
      "activations/layer7_attention_weight_max": 67.97753143310547,
      "activations/layer7_attention_weight_min": -66.22789001464844,
      "activations/layer8_attention_weight_max": 45.17680740356445,
      "activations/layer8_attention_weight_min": -48.39412307739258,
      "activations/layer9_attention_weight_max": 46.303871154785156,
      "activations/layer9_attention_weight_min": -44.125667572021484,
      "epoch": 12.11,
      "learning_rate": 7.264318181818181e-05,
      "loss": 2.7767,
      "step": 208400
    },
    {
      "activations/layer0_attention_weight_max": 17.43129539489746,
      "activations/layer0_attention_weight_min": -13.365459442138672,
      "activations/layer10_attention_weight_max": 33.01939010620117,
      "activations/layer10_attention_weight_min": -33.293270111083984,
      "activations/layer11_attention_weight_max": 30.39089012145996,
      "activations/layer11_attention_weight_min": -31.228445053100586,
      "activations/layer12_attention_weight_max": 18.436504364013672,
      "activations/layer12_attention_weight_min": -25.70151138305664,
      "activations/layer13_attention_weight_max": 32.74882507324219,
      "activations/layer13_attention_weight_min": -28.277589797973633,
      "activations/layer14_attention_weight_max": 35.67331314086914,
      "activations/layer14_attention_weight_min": -29.347259521484375,
      "activations/layer15_attention_weight_max": 32.39894485473633,
      "activations/layer15_attention_weight_min": -28.4548397064209,
      "activations/layer16_attention_weight_max": 28.614458084106445,
      "activations/layer16_attention_weight_min": -29.068906784057617,
      "activations/layer17_attention_weight_max": 46.142433166503906,
      "activations/layer17_attention_weight_min": -41.32499694824219,
      "activations/layer18_attention_weight_max": 45.0945930480957,
      "activations/layer18_attention_weight_min": -37.6253547668457,
      "activations/layer19_attention_weight_max": 20.016429901123047,
      "activations/layer19_attention_weight_min": -20.199295043945312,
      "activations/layer1_attention_weight_max": 16.673227310180664,
      "activations/layer1_attention_weight_min": -13.692567825317383,
      "activations/layer20_attention_weight_max": 19.096435546875,
      "activations/layer20_attention_weight_min": -19.838579177856445,
      "activations/layer21_attention_weight_max": 30.959239959716797,
      "activations/layer21_attention_weight_min": -21.435319900512695,
      "activations/layer22_attention_weight_max": 26.120250701904297,
      "activations/layer22_attention_weight_min": -27.140296936035156,
      "activations/layer23_attention_weight_max": 35.15376281738281,
      "activations/layer23_attention_weight_min": -23.854183197021484,
      "activations/layer2_attention_weight_max": 31.577877044677734,
      "activations/layer2_attention_weight_min": -28.76374626159668,
      "activations/layer3_attention_weight_max": 87.08147430419922,
      "activations/layer3_attention_weight_min": -93.9499740600586,
      "activations/layer4_attention_weight_max": 96.406494140625,
      "activations/layer4_attention_weight_min": -90.2064437866211,
      "activations/layer5_attention_weight_max": 69.81368255615234,
      "activations/layer5_attention_weight_min": -72.79367065429688,
      "activations/layer6_attention_weight_max": 51.31474685668945,
      "activations/layer6_attention_weight_min": -50.78032684326172,
      "activations/layer7_attention_weight_max": 69.42241668701172,
      "activations/layer7_attention_weight_min": -65.13760375976562,
      "activations/layer8_attention_weight_max": 44.04878616333008,
      "activations/layer8_attention_weight_min": -45.78025817871094,
      "activations/layer9_attention_weight_max": 47.7591667175293,
      "activations/layer9_attention_weight_min": -48.64187240600586,
      "epoch": 12.11,
      "learning_rate": 7.262424242424242e-05,
      "loss": 2.7693,
      "step": 208450
    },
    {
      "activations/layer0_attention_weight_max": 15.807893753051758,
      "activations/layer0_attention_weight_min": -13.931965827941895,
      "activations/layer10_attention_weight_max": 41.649017333984375,
      "activations/layer10_attention_weight_min": -42.47754669189453,
      "activations/layer11_attention_weight_max": 35.58110809326172,
      "activations/layer11_attention_weight_min": -36.15496826171875,
      "activations/layer12_attention_weight_max": 18.657737731933594,
      "activations/layer12_attention_weight_min": -28.2004451751709,
      "activations/layer13_attention_weight_max": 43.23044967651367,
      "activations/layer13_attention_weight_min": -33.42453384399414,
      "activations/layer14_attention_weight_max": 34.38184356689453,
      "activations/layer14_attention_weight_min": -29.374414443969727,
      "activations/layer15_attention_weight_max": 34.79521942138672,
      "activations/layer15_attention_weight_min": -28.41961097717285,
      "activations/layer16_attention_weight_max": 32.560367584228516,
      "activations/layer16_attention_weight_min": -30.19634246826172,
      "activations/layer17_attention_weight_max": 52.074951171875,
      "activations/layer17_attention_weight_min": -41.5622444152832,
      "activations/layer18_attention_weight_max": 49.0263557434082,
      "activations/layer18_attention_weight_min": -38.117591857910156,
      "activations/layer19_attention_weight_max": 24.150522232055664,
      "activations/layer19_attention_weight_min": -21.222488403320312,
      "activations/layer1_attention_weight_max": 16.452720642089844,
      "activations/layer1_attention_weight_min": -14.935401916503906,
      "activations/layer20_attention_weight_max": 22.583351135253906,
      "activations/layer20_attention_weight_min": -19.99480438232422,
      "activations/layer21_attention_weight_max": 34.9593620300293,
      "activations/layer21_attention_weight_min": -21.275054931640625,
      "activations/layer22_attention_weight_max": 28.87785530090332,
      "activations/layer22_attention_weight_min": -26.56276512145996,
      "activations/layer23_attention_weight_max": 39.81635284423828,
      "activations/layer23_attention_weight_min": -24.320554733276367,
      "activations/layer2_attention_weight_max": 30.396970748901367,
      "activations/layer2_attention_weight_min": -30.224912643432617,
      "activations/layer3_attention_weight_max": 89.18684387207031,
      "activations/layer3_attention_weight_min": -91.9134521484375,
      "activations/layer4_attention_weight_max": 95.2514877319336,
      "activations/layer4_attention_weight_min": -96.75355529785156,
      "activations/layer5_attention_weight_max": 72.15757751464844,
      "activations/layer5_attention_weight_min": -76.18505096435547,
      "activations/layer6_attention_weight_max": 52.81167221069336,
      "activations/layer6_attention_weight_min": -52.86183166503906,
      "activations/layer7_attention_weight_max": 69.44305419921875,
      "activations/layer7_attention_weight_min": -67.68372344970703,
      "activations/layer8_attention_weight_max": 49.55559539794922,
      "activations/layer8_attention_weight_min": -51.05630111694336,
      "activations/layer9_attention_weight_max": 52.839900970458984,
      "activations/layer9_attention_weight_min": -50.10029602050781,
      "epoch": 12.12,
      "learning_rate": 7.260530303030303e-05,
      "loss": 2.7677,
      "step": 208500
    },
    {
      "activations/layer0_attention_weight_max": 15.144647598266602,
      "activations/layer0_attention_weight_min": -14.090629577636719,
      "activations/layer10_attention_weight_max": 37.769229888916016,
      "activations/layer10_attention_weight_min": -39.82390594482422,
      "activations/layer11_attention_weight_max": 33.052154541015625,
      "activations/layer11_attention_weight_min": -37.859352111816406,
      "activations/layer12_attention_weight_max": 16.98630142211914,
      "activations/layer12_attention_weight_min": -26.948888778686523,
      "activations/layer13_attention_weight_max": 34.30522155761719,
      "activations/layer13_attention_weight_min": -27.426279067993164,
      "activations/layer14_attention_weight_max": 33.09245681762695,
      "activations/layer14_attention_weight_min": -30.62586784362793,
      "activations/layer15_attention_weight_max": 30.105607986450195,
      "activations/layer15_attention_weight_min": -30.366361618041992,
      "activations/layer16_attention_weight_max": 28.95505142211914,
      "activations/layer16_attention_weight_min": -28.683595657348633,
      "activations/layer17_attention_weight_max": 47.091835021972656,
      "activations/layer17_attention_weight_min": -43.49217224121094,
      "activations/layer18_attention_weight_max": 41.05698013305664,
      "activations/layer18_attention_weight_min": -36.500335693359375,
      "activations/layer19_attention_weight_max": 21.909414291381836,
      "activations/layer19_attention_weight_min": -23.822141647338867,
      "activations/layer1_attention_weight_max": 17.139375686645508,
      "activations/layer1_attention_weight_min": -15.954230308532715,
      "activations/layer20_attention_weight_max": 22.31169891357422,
      "activations/layer20_attention_weight_min": -22.68463134765625,
      "activations/layer21_attention_weight_max": 37.64220428466797,
      "activations/layer21_attention_weight_min": -23.656280517578125,
      "activations/layer22_attention_weight_max": 27.485883712768555,
      "activations/layer22_attention_weight_min": -24.05896759033203,
      "activations/layer23_attention_weight_max": 39.89906692504883,
      "activations/layer23_attention_weight_min": -23.5128173828125,
      "activations/layer2_attention_weight_max": 30.419353485107422,
      "activations/layer2_attention_weight_min": -29.738277435302734,
      "activations/layer3_attention_weight_max": 92.92353057861328,
      "activations/layer3_attention_weight_min": -91.09087371826172,
      "activations/layer4_attention_weight_max": 94.33173370361328,
      "activations/layer4_attention_weight_min": -94.95304870605469,
      "activations/layer5_attention_weight_max": 70.08154296875,
      "activations/layer5_attention_weight_min": -74.9972152709961,
      "activations/layer6_attention_weight_max": 53.532920837402344,
      "activations/layer6_attention_weight_min": -53.90483474731445,
      "activations/layer7_attention_weight_max": 69.01109313964844,
      "activations/layer7_attention_weight_min": -67.04789733886719,
      "activations/layer8_attention_weight_max": 46.52561569213867,
      "activations/layer8_attention_weight_min": -50.57562255859375,
      "activations/layer9_attention_weight_max": 47.6635856628418,
      "activations/layer9_attention_weight_min": -48.205745697021484,
      "epoch": 12.12,
      "learning_rate": 7.258636363636363e-05,
      "loss": 2.786,
      "step": 208550
    },
    {
      "activations/layer0_attention_weight_max": 16.40345573425293,
      "activations/layer0_attention_weight_min": -13.801939964294434,
      "activations/layer10_attention_weight_max": 34.85874557495117,
      "activations/layer10_attention_weight_min": -35.14702224731445,
      "activations/layer11_attention_weight_max": 34.32246017456055,
      "activations/layer11_attention_weight_min": -33.37105178833008,
      "activations/layer12_attention_weight_max": 19.08076286315918,
      "activations/layer12_attention_weight_min": -26.292964935302734,
      "activations/layer13_attention_weight_max": 50.48517990112305,
      "activations/layer13_attention_weight_min": -30.40131378173828,
      "activations/layer14_attention_weight_max": 69.05142211914062,
      "activations/layer14_attention_weight_min": -35.61292266845703,
      "activations/layer15_attention_weight_max": 53.78388977050781,
      "activations/layer15_attention_weight_min": -29.650348663330078,
      "activations/layer16_attention_weight_max": 33.316524505615234,
      "activations/layer16_attention_weight_min": -27.980731964111328,
      "activations/layer17_attention_weight_max": 52.848731994628906,
      "activations/layer17_attention_weight_min": -44.77124786376953,
      "activations/layer18_attention_weight_max": 48.01601028442383,
      "activations/layer18_attention_weight_min": -37.494544982910156,
      "activations/layer19_attention_weight_max": 27.199758529663086,
      "activations/layer19_attention_weight_min": -20.751670837402344,
      "activations/layer1_attention_weight_max": 16.936050415039062,
      "activations/layer1_attention_weight_min": -14.03385066986084,
      "activations/layer20_attention_weight_max": 29.4449520111084,
      "activations/layer20_attention_weight_min": -25.562686920166016,
      "activations/layer21_attention_weight_max": 41.6014404296875,
      "activations/layer21_attention_weight_min": -21.60401725769043,
      "activations/layer22_attention_weight_max": 32.2836799621582,
      "activations/layer22_attention_weight_min": -24.072534561157227,
      "activations/layer23_attention_weight_max": 39.784053802490234,
      "activations/layer23_attention_weight_min": -24.616111755371094,
      "activations/layer2_attention_weight_max": 29.257740020751953,
      "activations/layer2_attention_weight_min": -29.321208953857422,
      "activations/layer3_attention_weight_max": 87.2036361694336,
      "activations/layer3_attention_weight_min": -90.30533599853516,
      "activations/layer4_attention_weight_max": 94.20183563232422,
      "activations/layer4_attention_weight_min": -93.86310577392578,
      "activations/layer5_attention_weight_max": 73.12449645996094,
      "activations/layer5_attention_weight_min": -78.37696075439453,
      "activations/layer6_attention_weight_max": 51.644779205322266,
      "activations/layer6_attention_weight_min": -51.73188400268555,
      "activations/layer7_attention_weight_max": 71.38538360595703,
      "activations/layer7_attention_weight_min": -68.69356536865234,
      "activations/layer8_attention_weight_max": 43.467987060546875,
      "activations/layer8_attention_weight_min": -45.6404914855957,
      "activations/layer9_attention_weight_max": 50.71842575073242,
      "activations/layer9_attention_weight_min": -47.73088455200195,
      "epoch": 12.12,
      "learning_rate": 7.256742424242423e-05,
      "loss": 2.7756,
      "step": 208600
    },
    {
      "activations/layer0_attention_weight_max": 15.730876922607422,
      "activations/layer0_attention_weight_min": -13.682890892028809,
      "activations/layer10_attention_weight_max": 37.4333381652832,
      "activations/layer10_attention_weight_min": -39.34674835205078,
      "activations/layer11_attention_weight_max": 31.52338218688965,
      "activations/layer11_attention_weight_min": -33.655555725097656,
      "activations/layer12_attention_weight_max": 18.08948516845703,
      "activations/layer12_attention_weight_min": -23.713340759277344,
      "activations/layer13_attention_weight_max": 34.99506759643555,
      "activations/layer13_attention_weight_min": -28.824703216552734,
      "activations/layer14_attention_weight_max": 40.48338317871094,
      "activations/layer14_attention_weight_min": -32.71236038208008,
      "activations/layer15_attention_weight_max": 36.27942657470703,
      "activations/layer15_attention_weight_min": -30.661222457885742,
      "activations/layer16_attention_weight_max": 35.17437744140625,
      "activations/layer16_attention_weight_min": -30.651199340820312,
      "activations/layer17_attention_weight_max": 57.55733108520508,
      "activations/layer17_attention_weight_min": -48.58350372314453,
      "activations/layer18_attention_weight_max": 52.63555908203125,
      "activations/layer18_attention_weight_min": -41.95834732055664,
      "activations/layer19_attention_weight_max": 22.136770248413086,
      "activations/layer19_attention_weight_min": -25.335491180419922,
      "activations/layer1_attention_weight_max": 16.449220657348633,
      "activations/layer1_attention_weight_min": -16.1732120513916,
      "activations/layer20_attention_weight_max": 24.056310653686523,
      "activations/layer20_attention_weight_min": -23.786243438720703,
      "activations/layer21_attention_weight_max": 39.59383773803711,
      "activations/layer21_attention_weight_min": -22.841567993164062,
      "activations/layer22_attention_weight_max": 31.669349670410156,
      "activations/layer22_attention_weight_min": -28.02822494506836,
      "activations/layer23_attention_weight_max": 38.22153091430664,
      "activations/layer23_attention_weight_min": -25.65283203125,
      "activations/layer2_attention_weight_max": 33.347904205322266,
      "activations/layer2_attention_weight_min": -31.393787384033203,
      "activations/layer3_attention_weight_max": 89.05681610107422,
      "activations/layer3_attention_weight_min": -94.19874572753906,
      "activations/layer4_attention_weight_max": 92.9140853881836,
      "activations/layer4_attention_weight_min": -91.06645202636719,
      "activations/layer5_attention_weight_max": 70.00357818603516,
      "activations/layer5_attention_weight_min": -74.81853485107422,
      "activations/layer6_attention_weight_max": 53.934024810791016,
      "activations/layer6_attention_weight_min": -52.84748840332031,
      "activations/layer7_attention_weight_max": 73.0665054321289,
      "activations/layer7_attention_weight_min": -68.21556854248047,
      "activations/layer8_attention_weight_max": 46.80678176879883,
      "activations/layer8_attention_weight_min": -49.5557746887207,
      "activations/layer9_attention_weight_max": 49.17832565307617,
      "activations/layer9_attention_weight_min": -47.14944839477539,
      "epoch": 12.12,
      "learning_rate": 7.254848484848484e-05,
      "loss": 2.7674,
      "step": 208650
    },
    {
      "activations/layer0_attention_weight_max": 17.45248031616211,
      "activations/layer0_attention_weight_min": -14.146223068237305,
      "activations/layer10_attention_weight_max": 33.8087158203125,
      "activations/layer10_attention_weight_min": -34.96161651611328,
      "activations/layer11_attention_weight_max": 33.382877349853516,
      "activations/layer11_attention_weight_min": -32.42984390258789,
      "activations/layer12_attention_weight_max": 18.42378807067871,
      "activations/layer12_attention_weight_min": -25.54863929748535,
      "activations/layer13_attention_weight_max": 37.190162658691406,
      "activations/layer13_attention_weight_min": -26.500469207763672,
      "activations/layer14_attention_weight_max": 43.95692825317383,
      "activations/layer14_attention_weight_min": -31.30680274963379,
      "activations/layer15_attention_weight_max": 35.3079948425293,
      "activations/layer15_attention_weight_min": -28.952579498291016,
      "activations/layer16_attention_weight_max": 31.46251106262207,
      "activations/layer16_attention_weight_min": -30.488306045532227,
      "activations/layer17_attention_weight_max": 52.909095764160156,
      "activations/layer17_attention_weight_min": -44.02983474731445,
      "activations/layer18_attention_weight_max": 49.934661865234375,
      "activations/layer18_attention_weight_min": -38.587982177734375,
      "activations/layer19_attention_weight_max": 24.42232894897461,
      "activations/layer19_attention_weight_min": -22.78498077392578,
      "activations/layer1_attention_weight_max": 17.150470733642578,
      "activations/layer1_attention_weight_min": -14.48662281036377,
      "activations/layer20_attention_weight_max": 25.184480667114258,
      "activations/layer20_attention_weight_min": -20.989259719848633,
      "activations/layer21_attention_weight_max": 40.937015533447266,
      "activations/layer21_attention_weight_min": -26.32560157775879,
      "activations/layer22_attention_weight_max": 32.95435333251953,
      "activations/layer22_attention_weight_min": -25.197948455810547,
      "activations/layer23_attention_weight_max": 41.44892501831055,
      "activations/layer23_attention_weight_min": -26.101581573486328,
      "activations/layer2_attention_weight_max": 29.480323791503906,
      "activations/layer2_attention_weight_min": -29.268463134765625,
      "activations/layer3_attention_weight_max": 91.61805725097656,
      "activations/layer3_attention_weight_min": -87.08379364013672,
      "activations/layer4_attention_weight_max": 98.6777572631836,
      "activations/layer4_attention_weight_min": -88.41328430175781,
      "activations/layer5_attention_weight_max": 69.84066009521484,
      "activations/layer5_attention_weight_min": -75.14031982421875,
      "activations/layer6_attention_weight_max": 54.51261520385742,
      "activations/layer6_attention_weight_min": -52.46051788330078,
      "activations/layer7_attention_weight_max": 68.63226318359375,
      "activations/layer7_attention_weight_min": -63.7317008972168,
      "activations/layer8_attention_weight_max": 43.45991897583008,
      "activations/layer8_attention_weight_min": -45.89527130126953,
      "activations/layer9_attention_weight_max": 49.91246032714844,
      "activations/layer9_attention_weight_min": -48.35258102416992,
      "epoch": 12.13,
      "learning_rate": 7.252954545454545e-05,
      "loss": 2.7709,
      "step": 208700
    },
    {
      "activations/layer0_attention_weight_max": 15.775650024414062,
      "activations/layer0_attention_weight_min": -13.686366081237793,
      "activations/layer10_attention_weight_max": 36.36075210571289,
      "activations/layer10_attention_weight_min": -34.851402282714844,
      "activations/layer11_attention_weight_max": 32.61941909790039,
      "activations/layer11_attention_weight_min": -31.512611389160156,
      "activations/layer12_attention_weight_max": 18.871051788330078,
      "activations/layer12_attention_weight_min": -23.878684997558594,
      "activations/layer13_attention_weight_max": 49.86655807495117,
      "activations/layer13_attention_weight_min": -37.28548812866211,
      "activations/layer14_attention_weight_max": 47.50374221801758,
      "activations/layer14_attention_weight_min": -31.899845123291016,
      "activations/layer15_attention_weight_max": 43.48810958862305,
      "activations/layer15_attention_weight_min": -30.659578323364258,
      "activations/layer16_attention_weight_max": 33.350460052490234,
      "activations/layer16_attention_weight_min": -30.940017700195312,
      "activations/layer17_attention_weight_max": 54.70566940307617,
      "activations/layer17_attention_weight_min": -48.025936126708984,
      "activations/layer18_attention_weight_max": 50.26591491699219,
      "activations/layer18_attention_weight_min": -39.912506103515625,
      "activations/layer19_attention_weight_max": 22.670650482177734,
      "activations/layer19_attention_weight_min": -24.139537811279297,
      "activations/layer1_attention_weight_max": 16.670869827270508,
      "activations/layer1_attention_weight_min": -14.55576229095459,
      "activations/layer20_attention_weight_max": 23.736814498901367,
      "activations/layer20_attention_weight_min": -23.600351333618164,
      "activations/layer21_attention_weight_max": 49.78314971923828,
      "activations/layer21_attention_weight_min": -28.701766967773438,
      "activations/layer22_attention_weight_max": 31.850326538085938,
      "activations/layer22_attention_weight_min": -26.1787109375,
      "activations/layer23_attention_weight_max": 38.396095275878906,
      "activations/layer23_attention_weight_min": -24.96446990966797,
      "activations/layer2_attention_weight_max": 30.51129913330078,
      "activations/layer2_attention_weight_min": -31.12480926513672,
      "activations/layer3_attention_weight_max": 90.09327697753906,
      "activations/layer3_attention_weight_min": -96.0551528930664,
      "activations/layer4_attention_weight_max": 95.77446746826172,
      "activations/layer4_attention_weight_min": -88.95296478271484,
      "activations/layer5_attention_weight_max": 70.06645202636719,
      "activations/layer5_attention_weight_min": -79.64651489257812,
      "activations/layer6_attention_weight_max": 52.188663482666016,
      "activations/layer6_attention_weight_min": -50.55710220336914,
      "activations/layer7_attention_weight_max": 71.16950225830078,
      "activations/layer7_attention_weight_min": -62.87639236450195,
      "activations/layer8_attention_weight_max": 45.00063705444336,
      "activations/layer8_attention_weight_min": -44.54031753540039,
      "activations/layer9_attention_weight_max": 45.796016693115234,
      "activations/layer9_attention_weight_min": -45.483577728271484,
      "epoch": 12.13,
      "learning_rate": 7.251060606060605e-05,
      "loss": 2.7699,
      "step": 208750
    },
    {
      "activations/layer0_attention_weight_max": 16.17074966430664,
      "activations/layer0_attention_weight_min": -14.144721031188965,
      "activations/layer10_attention_weight_max": 37.84779739379883,
      "activations/layer10_attention_weight_min": -34.33762741088867,
      "activations/layer11_attention_weight_max": 32.037147521972656,
      "activations/layer11_attention_weight_min": -32.63127517700195,
      "activations/layer12_attention_weight_max": 19.57537841796875,
      "activations/layer12_attention_weight_min": -24.818811416625977,
      "activations/layer13_attention_weight_max": 39.15459060668945,
      "activations/layer13_attention_weight_min": -30.045059204101562,
      "activations/layer14_attention_weight_max": 42.21720886230469,
      "activations/layer14_attention_weight_min": -29.387775421142578,
      "activations/layer15_attention_weight_max": 35.91925048828125,
      "activations/layer15_attention_weight_min": -30.20684814453125,
      "activations/layer16_attention_weight_max": 35.308189392089844,
      "activations/layer16_attention_weight_min": -31.29693603515625,
      "activations/layer17_attention_weight_max": 59.43537139892578,
      "activations/layer17_attention_weight_min": -48.49486541748047,
      "activations/layer18_attention_weight_max": 51.29684066772461,
      "activations/layer18_attention_weight_min": -42.15303039550781,
      "activations/layer19_attention_weight_max": 21.7753963470459,
      "activations/layer19_attention_weight_min": -21.587261199951172,
      "activations/layer1_attention_weight_max": 17.277324676513672,
      "activations/layer1_attention_weight_min": -14.761248588562012,
      "activations/layer20_attention_weight_max": 23.270952224731445,
      "activations/layer20_attention_weight_min": -20.370006561279297,
      "activations/layer21_attention_weight_max": 43.930816650390625,
      "activations/layer21_attention_weight_min": -22.731231689453125,
      "activations/layer22_attention_weight_max": 33.57842254638672,
      "activations/layer22_attention_weight_min": -24.425262451171875,
      "activations/layer23_attention_weight_max": 39.46375274658203,
      "activations/layer23_attention_weight_min": -22.097999572753906,
      "activations/layer2_attention_weight_max": 32.207584381103516,
      "activations/layer2_attention_weight_min": -29.714004516601562,
      "activations/layer3_attention_weight_max": 89.93980407714844,
      "activations/layer3_attention_weight_min": -89.83672332763672,
      "activations/layer4_attention_weight_max": 97.22786712646484,
      "activations/layer4_attention_weight_min": -94.3170166015625,
      "activations/layer5_attention_weight_max": 70.41861724853516,
      "activations/layer5_attention_weight_min": -77.5733642578125,
      "activations/layer6_attention_weight_max": 51.894954681396484,
      "activations/layer6_attention_weight_min": -53.23215103149414,
      "activations/layer7_attention_weight_max": 74.46651458740234,
      "activations/layer7_attention_weight_min": -66.97511291503906,
      "activations/layer8_attention_weight_max": 46.24147033691406,
      "activations/layer8_attention_weight_min": -45.23728561401367,
      "activations/layer9_attention_weight_max": 48.81385803222656,
      "activations/layer9_attention_weight_min": -46.1319580078125,
      "epoch": 12.13,
      "learning_rate": 7.249166666666666e-05,
      "loss": 2.7698,
      "step": 208800
    },
    {
      "activations/layer0_attention_weight_max": 15.394075393676758,
      "activations/layer0_attention_weight_min": -14.126298904418945,
      "activations/layer10_attention_weight_max": 40.34445571899414,
      "activations/layer10_attention_weight_min": -37.18943786621094,
      "activations/layer11_attention_weight_max": 35.70036315917969,
      "activations/layer11_attention_weight_min": -36.37489318847656,
      "activations/layer12_attention_weight_max": 25.82602310180664,
      "activations/layer12_attention_weight_min": -25.23126983642578,
      "activations/layer13_attention_weight_max": 46.194374084472656,
      "activations/layer13_attention_weight_min": -32.149330139160156,
      "activations/layer14_attention_weight_max": 43.9943733215332,
      "activations/layer14_attention_weight_min": -31.555627822875977,
      "activations/layer15_attention_weight_max": 39.634483337402344,
      "activations/layer15_attention_weight_min": -30.529315948486328,
      "activations/layer16_attention_weight_max": 35.75160217285156,
      "activations/layer16_attention_weight_min": -34.127197265625,
      "activations/layer17_attention_weight_max": 58.03580093383789,
      "activations/layer17_attention_weight_min": -51.103302001953125,
      "activations/layer18_attention_weight_max": 53.095001220703125,
      "activations/layer18_attention_weight_min": -45.27300262451172,
      "activations/layer19_attention_weight_max": 30.261539459228516,
      "activations/layer19_attention_weight_min": -26.54847526550293,
      "activations/layer1_attention_weight_max": 17.451025009155273,
      "activations/layer1_attention_weight_min": -13.975278854370117,
      "activations/layer20_attention_weight_max": 28.187532424926758,
      "activations/layer20_attention_weight_min": -22.88888168334961,
      "activations/layer21_attention_weight_max": 45.06464385986328,
      "activations/layer21_attention_weight_min": -25.865739822387695,
      "activations/layer22_attention_weight_max": 33.992244720458984,
      "activations/layer22_attention_weight_min": -29.3463191986084,
      "activations/layer23_attention_weight_max": 48.526824951171875,
      "activations/layer23_attention_weight_min": -25.977237701416016,
      "activations/layer2_attention_weight_max": 32.872161865234375,
      "activations/layer2_attention_weight_min": -30.960569381713867,
      "activations/layer3_attention_weight_max": 94.13761138916016,
      "activations/layer3_attention_weight_min": -92.26805877685547,
      "activations/layer4_attention_weight_max": 96.37435150146484,
      "activations/layer4_attention_weight_min": -93.99567413330078,
      "activations/layer5_attention_weight_max": 69.06968688964844,
      "activations/layer5_attention_weight_min": -75.93171691894531,
      "activations/layer6_attention_weight_max": 52.81475067138672,
      "activations/layer6_attention_weight_min": -52.47303009033203,
      "activations/layer7_attention_weight_max": 73.1119384765625,
      "activations/layer7_attention_weight_min": -68.88461303710938,
      "activations/layer8_attention_weight_max": 46.17964553833008,
      "activations/layer8_attention_weight_min": -46.157047271728516,
      "activations/layer9_attention_weight_max": 50.04301452636719,
      "activations/layer9_attention_weight_min": -53.6810302734375,
      "epoch": 12.14,
      "learning_rate": 7.247272727272726e-05,
      "loss": 2.7796,
      "step": 208850
    },
    {
      "activations/layer0_attention_weight_max": 15.882383346557617,
      "activations/layer0_attention_weight_min": -13.995199203491211,
      "activations/layer10_attention_weight_max": 38.89984130859375,
      "activations/layer10_attention_weight_min": -38.333492279052734,
      "activations/layer11_attention_weight_max": 35.354827880859375,
      "activations/layer11_attention_weight_min": -35.31047058105469,
      "activations/layer12_attention_weight_max": 18.63958168029785,
      "activations/layer12_attention_weight_min": -24.037078857421875,
      "activations/layer13_attention_weight_max": 40.18156814575195,
      "activations/layer13_attention_weight_min": -30.72650909423828,
      "activations/layer14_attention_weight_max": 35.17715072631836,
      "activations/layer14_attention_weight_min": -30.688304901123047,
      "activations/layer15_attention_weight_max": 33.71015930175781,
      "activations/layer15_attention_weight_min": -28.879682540893555,
      "activations/layer16_attention_weight_max": 34.493412017822266,
      "activations/layer16_attention_weight_min": -30.09885597229004,
      "activations/layer17_attention_weight_max": 52.702117919921875,
      "activations/layer17_attention_weight_min": -44.5113525390625,
      "activations/layer18_attention_weight_max": 50.88975524902344,
      "activations/layer18_attention_weight_min": -40.92461013793945,
      "activations/layer19_attention_weight_max": 23.51899528503418,
      "activations/layer19_attention_weight_min": -22.549636840820312,
      "activations/layer1_attention_weight_max": 17.109773635864258,
      "activations/layer1_attention_weight_min": -14.04516315460205,
      "activations/layer20_attention_weight_max": 24.388633728027344,
      "activations/layer20_attention_weight_min": -22.4378719329834,
      "activations/layer21_attention_weight_max": 38.875244140625,
      "activations/layer21_attention_weight_min": -23.25763511657715,
      "activations/layer22_attention_weight_max": 32.35964584350586,
      "activations/layer22_attention_weight_min": -27.340757369995117,
      "activations/layer23_attention_weight_max": 39.405426025390625,
      "activations/layer23_attention_weight_min": -25.236003875732422,
      "activations/layer2_attention_weight_max": 32.210227966308594,
      "activations/layer2_attention_weight_min": -30.264850616455078,
      "activations/layer3_attention_weight_max": 92.56526184082031,
      "activations/layer3_attention_weight_min": -92.16613006591797,
      "activations/layer4_attention_weight_max": 97.90992736816406,
      "activations/layer4_attention_weight_min": -92.82695770263672,
      "activations/layer5_attention_weight_max": 69.11833953857422,
      "activations/layer5_attention_weight_min": -76.55581665039062,
      "activations/layer6_attention_weight_max": 54.05508041381836,
      "activations/layer6_attention_weight_min": -53.860496520996094,
      "activations/layer7_attention_weight_max": 67.58694458007812,
      "activations/layer7_attention_weight_min": -66.85012817382812,
      "activations/layer8_attention_weight_max": 47.33959197998047,
      "activations/layer8_attention_weight_min": -48.94701385498047,
      "activations/layer9_attention_weight_max": 47.892303466796875,
      "activations/layer9_attention_weight_min": -48.61308288574219,
      "epoch": 12.14,
      "learning_rate": 7.245378787878788e-05,
      "loss": 2.7786,
      "step": 208900
    },
    {
      "activations/layer0_attention_weight_max": 15.67619800567627,
      "activations/layer0_attention_weight_min": -14.22746753692627,
      "activations/layer10_attention_weight_max": 37.76536178588867,
      "activations/layer10_attention_weight_min": -36.63566589355469,
      "activations/layer11_attention_weight_max": 31.850439071655273,
      "activations/layer11_attention_weight_min": -33.67315673828125,
      "activations/layer12_attention_weight_max": 21.766788482666016,
      "activations/layer12_attention_weight_min": -26.00766944885254,
      "activations/layer13_attention_weight_max": 37.9038200378418,
      "activations/layer13_attention_weight_min": -27.235410690307617,
      "activations/layer14_attention_weight_max": 37.91722106933594,
      "activations/layer14_attention_weight_min": -29.733503341674805,
      "activations/layer15_attention_weight_max": 35.3590087890625,
      "activations/layer15_attention_weight_min": -31.117380142211914,
      "activations/layer16_attention_weight_max": 30.948734283447266,
      "activations/layer16_attention_weight_min": -29.85525131225586,
      "activations/layer17_attention_weight_max": 56.06385040283203,
      "activations/layer17_attention_weight_min": -49.06159973144531,
      "activations/layer18_attention_weight_max": 48.280113220214844,
      "activations/layer18_attention_weight_min": -39.989227294921875,
      "activations/layer19_attention_weight_max": 23.58985710144043,
      "activations/layer19_attention_weight_min": -23.0830135345459,
      "activations/layer1_attention_weight_max": 17.27724838256836,
      "activations/layer1_attention_weight_min": -15.726197242736816,
      "activations/layer20_attention_weight_max": 22.331693649291992,
      "activations/layer20_attention_weight_min": -22.69681167602539,
      "activations/layer21_attention_weight_max": 40.17567825317383,
      "activations/layer21_attention_weight_min": -22.241849899291992,
      "activations/layer22_attention_weight_max": 29.718017578125,
      "activations/layer22_attention_weight_min": -26.57776641845703,
      "activations/layer23_attention_weight_max": 35.09626770019531,
      "activations/layer23_attention_weight_min": -24.24392318725586,
      "activations/layer2_attention_weight_max": 33.50689697265625,
      "activations/layer2_attention_weight_min": -33.23509216308594,
      "activations/layer3_attention_weight_max": 92.26609802246094,
      "activations/layer3_attention_weight_min": -92.94407653808594,
      "activations/layer4_attention_weight_max": 94.99983215332031,
      "activations/layer4_attention_weight_min": -91.40512084960938,
      "activations/layer5_attention_weight_max": 72.68365478515625,
      "activations/layer5_attention_weight_min": -75.7666244506836,
      "activations/layer6_attention_weight_max": 51.46162033081055,
      "activations/layer6_attention_weight_min": -50.33751678466797,
      "activations/layer7_attention_weight_max": 66.71370697021484,
      "activations/layer7_attention_weight_min": -65.9815902709961,
      "activations/layer8_attention_weight_max": 44.40465545654297,
      "activations/layer8_attention_weight_min": -46.88167190551758,
      "activations/layer9_attention_weight_max": 44.2575798034668,
      "activations/layer9_attention_weight_min": -47.69342041015625,
      "epoch": 12.14,
      "learning_rate": 7.243484848484848e-05,
      "loss": 2.7729,
      "step": 208950
    },
    {
      "activations/layer0_attention_weight_max": 16.85196876525879,
      "activations/layer0_attention_weight_min": -14.061827659606934,
      "activations/layer10_attention_weight_max": 36.03931427001953,
      "activations/layer10_attention_weight_min": -34.97047805786133,
      "activations/layer11_attention_weight_max": 31.448078155517578,
      "activations/layer11_attention_weight_min": -32.428733825683594,
      "activations/layer12_attention_weight_max": 20.461912155151367,
      "activations/layer12_attention_weight_min": -24.850982666015625,
      "activations/layer13_attention_weight_max": 41.34184646606445,
      "activations/layer13_attention_weight_min": -31.3956298828125,
      "activations/layer14_attention_weight_max": 40.49457931518555,
      "activations/layer14_attention_weight_min": -33.009517669677734,
      "activations/layer15_attention_weight_max": 35.359519958496094,
      "activations/layer15_attention_weight_min": -30.392953872680664,
      "activations/layer16_attention_weight_max": 33.537879943847656,
      "activations/layer16_attention_weight_min": -30.98731231689453,
      "activations/layer17_attention_weight_max": 59.05502700805664,
      "activations/layer17_attention_weight_min": -47.04413604736328,
      "activations/layer18_attention_weight_max": 51.65986251831055,
      "activations/layer18_attention_weight_min": -39.63215255737305,
      "activations/layer19_attention_weight_max": 26.87896728515625,
      "activations/layer19_attention_weight_min": -23.66230583190918,
      "activations/layer1_attention_weight_max": 16.639572143554688,
      "activations/layer1_attention_weight_min": -14.953485488891602,
      "activations/layer20_attention_weight_max": 26.738903045654297,
      "activations/layer20_attention_weight_min": -22.9959659576416,
      "activations/layer21_attention_weight_max": 40.95418930053711,
      "activations/layer21_attention_weight_min": -24.645963668823242,
      "activations/layer22_attention_weight_max": 32.47090148925781,
      "activations/layer22_attention_weight_min": -26.635095596313477,
      "activations/layer23_attention_weight_max": 41.03245162963867,
      "activations/layer23_attention_weight_min": -25.693281173706055,
      "activations/layer2_attention_weight_max": 29.6674861907959,
      "activations/layer2_attention_weight_min": -29.17683219909668,
      "activations/layer3_attention_weight_max": 89.86258697509766,
      "activations/layer3_attention_weight_min": -92.67855834960938,
      "activations/layer4_attention_weight_max": 97.45418548583984,
      "activations/layer4_attention_weight_min": -93.97808074951172,
      "activations/layer5_attention_weight_max": 69.05851745605469,
      "activations/layer5_attention_weight_min": -77.02367401123047,
      "activations/layer6_attention_weight_max": 53.149837493896484,
      "activations/layer6_attention_weight_min": -53.47496032714844,
      "activations/layer7_attention_weight_max": 68.78486633300781,
      "activations/layer7_attention_weight_min": -66.19499206542969,
      "activations/layer8_attention_weight_max": 45.0477409362793,
      "activations/layer8_attention_weight_min": -46.98170852661133,
      "activations/layer9_attention_weight_max": 45.630523681640625,
      "activations/layer9_attention_weight_min": -44.51805877685547,
      "epoch": 12.14,
      "learning_rate": 7.241590909090908e-05,
      "loss": 2.7659,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.5091,
      "eval_samples_per_second": 504.637,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.5091,
      "eval_openwebtext_samples_per_second": 504.637,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.975,
      "eval_wikitext_samples_per_second": 230.89,
      "step": 209000
    },
    {
      "epoch": 12.14,
      "eval_lambada_loss": 2.548828125,
      "eval_lambada_ppl": 12.792104248423877,
      "eval_lambada_runtime": 9.5857,
      "eval_lambada_samples_per_second": 507.942,
      "step": 209000
    },
    {
      "activations/layer0_attention_weight_max": 15.86600399017334,
      "activations/layer0_attention_weight_min": -14.471041679382324,
      "activations/layer10_attention_weight_max": 40.91904830932617,
      "activations/layer10_attention_weight_min": -37.52082443237305,
      "activations/layer11_attention_weight_max": 35.10042190551758,
      "activations/layer11_attention_weight_min": -34.39295196533203,
      "activations/layer12_attention_weight_max": 19.417903900146484,
      "activations/layer12_attention_weight_min": -26.91706657409668,
      "activations/layer13_attention_weight_max": 45.7360725402832,
      "activations/layer13_attention_weight_min": -36.68975067138672,
      "activations/layer14_attention_weight_max": 40.418270111083984,
      "activations/layer14_attention_weight_min": -35.30054473876953,
      "activations/layer15_attention_weight_max": 41.28056335449219,
      "activations/layer15_attention_weight_min": -30.865392684936523,
      "activations/layer16_attention_weight_max": 31.814266204833984,
      "activations/layer16_attention_weight_min": -31.033267974853516,
      "activations/layer17_attention_weight_max": 51.592567443847656,
      "activations/layer17_attention_weight_min": -46.242706298828125,
      "activations/layer18_attention_weight_max": 50.13741683959961,
      "activations/layer18_attention_weight_min": -39.76698684692383,
      "activations/layer19_attention_weight_max": 21.687864303588867,
      "activations/layer19_attention_weight_min": -21.48842430114746,
      "activations/layer1_attention_weight_max": 17.882959365844727,
      "activations/layer1_attention_weight_min": -14.5292387008667,
      "activations/layer20_attention_weight_max": 22.362077713012695,
      "activations/layer20_attention_weight_min": -21.265396118164062,
      "activations/layer21_attention_weight_max": 38.588417053222656,
      "activations/layer21_attention_weight_min": -23.604345321655273,
      "activations/layer22_attention_weight_max": 29.993099212646484,
      "activations/layer22_attention_weight_min": -26.137191772460938,
      "activations/layer23_attention_weight_max": 37.39204406738281,
      "activations/layer23_attention_weight_min": -24.851232528686523,
      "activations/layer2_attention_weight_max": 30.558462142944336,
      "activations/layer2_attention_weight_min": -31.505573272705078,
      "activations/layer3_attention_weight_max": 91.43737030029297,
      "activations/layer3_attention_weight_min": -93.32624816894531,
      "activations/layer4_attention_weight_max": 95.55377197265625,
      "activations/layer4_attention_weight_min": -92.24526977539062,
      "activations/layer5_attention_weight_max": 70.99574279785156,
      "activations/layer5_attention_weight_min": -72.19418334960938,
      "activations/layer6_attention_weight_max": 54.79329299926758,
      "activations/layer6_attention_weight_min": -54.48296356201172,
      "activations/layer7_attention_weight_max": 74.09889221191406,
      "activations/layer7_attention_weight_min": -73.49956512451172,
      "activations/layer8_attention_weight_max": 49.87755584716797,
      "activations/layer8_attention_weight_min": -49.57644271850586,
      "activations/layer9_attention_weight_max": 54.35584259033203,
      "activations/layer9_attention_weight_min": -49.358455657958984,
      "epoch": 12.15,
      "learning_rate": 7.23969696969697e-05,
      "loss": 2.7751,
      "step": 209050
    },
    {
      "activations/layer0_attention_weight_max": 17.440603256225586,
      "activations/layer0_attention_weight_min": -14.846217155456543,
      "activations/layer10_attention_weight_max": 37.9310188293457,
      "activations/layer10_attention_weight_min": -36.65571975708008,
      "activations/layer11_attention_weight_max": 36.16248321533203,
      "activations/layer11_attention_weight_min": -34.32279586791992,
      "activations/layer12_attention_weight_max": 20.19915771484375,
      "activations/layer12_attention_weight_min": -25.55885124206543,
      "activations/layer13_attention_weight_max": 40.89107131958008,
      "activations/layer13_attention_weight_min": -31.58177375793457,
      "activations/layer14_attention_weight_max": 47.24217224121094,
      "activations/layer14_attention_weight_min": -32.13377380371094,
      "activations/layer15_attention_weight_max": 37.829097747802734,
      "activations/layer15_attention_weight_min": -31.125200271606445,
      "activations/layer16_attention_weight_max": 35.26791763305664,
      "activations/layer16_attention_weight_min": -33.2076530456543,
      "activations/layer17_attention_weight_max": 55.815853118896484,
      "activations/layer17_attention_weight_min": -51.04130935668945,
      "activations/layer18_attention_weight_max": 50.6361198425293,
      "activations/layer18_attention_weight_min": -41.27341079711914,
      "activations/layer19_attention_weight_max": 22.274991989135742,
      "activations/layer19_attention_weight_min": -23.189693450927734,
      "activations/layer1_attention_weight_max": 16.7961483001709,
      "activations/layer1_attention_weight_min": -17.107948303222656,
      "activations/layer20_attention_weight_max": 24.353483200073242,
      "activations/layer20_attention_weight_min": -21.835542678833008,
      "activations/layer21_attention_weight_max": 51.75642776489258,
      "activations/layer21_attention_weight_min": -28.592327117919922,
      "activations/layer22_attention_weight_max": 29.7454891204834,
      "activations/layer22_attention_weight_min": -26.78354835510254,
      "activations/layer23_attention_weight_max": 37.28693389892578,
      "activations/layer23_attention_weight_min": -22.19307518005371,
      "activations/layer2_attention_weight_max": 33.05026626586914,
      "activations/layer2_attention_weight_min": -31.24810791015625,
      "activations/layer3_attention_weight_max": 93.38955688476562,
      "activations/layer3_attention_weight_min": -93.37749481201172,
      "activations/layer4_attention_weight_max": 104.0646743774414,
      "activations/layer4_attention_weight_min": -101.4875717163086,
      "activations/layer5_attention_weight_max": 78.56010437011719,
      "activations/layer5_attention_weight_min": -76.87760162353516,
      "activations/layer6_attention_weight_max": 52.88685989379883,
      "activations/layer6_attention_weight_min": -55.15427780151367,
      "activations/layer7_attention_weight_max": 72.04935455322266,
      "activations/layer7_attention_weight_min": -68.27764892578125,
      "activations/layer8_attention_weight_max": 51.22473907470703,
      "activations/layer8_attention_weight_min": -49.51417922973633,
      "activations/layer9_attention_weight_max": 53.438777923583984,
      "activations/layer9_attention_weight_min": -48.36994552612305,
      "epoch": 12.15,
      "learning_rate": 7.23780303030303e-05,
      "loss": 2.7645,
      "step": 209100
    },
    {
      "activations/layer0_attention_weight_max": 16.419008255004883,
      "activations/layer0_attention_weight_min": -14.443273544311523,
      "activations/layer10_attention_weight_max": 41.62702560424805,
      "activations/layer10_attention_weight_min": -38.660987854003906,
      "activations/layer11_attention_weight_max": 36.43461608886719,
      "activations/layer11_attention_weight_min": -34.59599685668945,
      "activations/layer12_attention_weight_max": 19.02775001525879,
      "activations/layer12_attention_weight_min": -22.43484878540039,
      "activations/layer13_attention_weight_max": 33.33958435058594,
      "activations/layer13_attention_weight_min": -29.50970458984375,
      "activations/layer14_attention_weight_max": 33.81339645385742,
      "activations/layer14_attention_weight_min": -29.65338706970215,
      "activations/layer15_attention_weight_max": 31.805557250976562,
      "activations/layer15_attention_weight_min": -29.255611419677734,
      "activations/layer16_attention_weight_max": 30.033283233642578,
      "activations/layer16_attention_weight_min": -28.38677406311035,
      "activations/layer17_attention_weight_max": 47.807247161865234,
      "activations/layer17_attention_weight_min": -42.36772537231445,
      "activations/layer18_attention_weight_max": 46.66364669799805,
      "activations/layer18_attention_weight_min": -37.24633026123047,
      "activations/layer19_attention_weight_max": 21.535398483276367,
      "activations/layer19_attention_weight_min": -20.907865524291992,
      "activations/layer1_attention_weight_max": 16.177526473999023,
      "activations/layer1_attention_weight_min": -14.652729988098145,
      "activations/layer20_attention_weight_max": 20.667922973632812,
      "activations/layer20_attention_weight_min": -23.390588760375977,
      "activations/layer21_attention_weight_max": 37.47589111328125,
      "activations/layer21_attention_weight_min": -24.304380416870117,
      "activations/layer22_attention_weight_max": 28.976728439331055,
      "activations/layer22_attention_weight_min": -24.985076904296875,
      "activations/layer23_attention_weight_max": 35.46321105957031,
      "activations/layer23_attention_weight_min": -24.21600914001465,
      "activations/layer2_attention_weight_max": 31.018512725830078,
      "activations/layer2_attention_weight_min": -29.013072967529297,
      "activations/layer3_attention_weight_max": 85.41171264648438,
      "activations/layer3_attention_weight_min": -90.10404968261719,
      "activations/layer4_attention_weight_max": 98.07151794433594,
      "activations/layer4_attention_weight_min": -92.84014129638672,
      "activations/layer5_attention_weight_max": 69.81404113769531,
      "activations/layer5_attention_weight_min": -75.93869018554688,
      "activations/layer6_attention_weight_max": 54.131099700927734,
      "activations/layer6_attention_weight_min": -53.886817932128906,
      "activations/layer7_attention_weight_max": 72.78125762939453,
      "activations/layer7_attention_weight_min": -70.77386474609375,
      "activations/layer8_attention_weight_max": 49.43440246582031,
      "activations/layer8_attention_weight_min": -51.35152816772461,
      "activations/layer9_attention_weight_max": 53.46622848510742,
      "activations/layer9_attention_weight_min": -51.83088302612305,
      "epoch": 12.15,
      "learning_rate": 7.23590909090909e-05,
      "loss": 2.7706,
      "step": 209150
    },
    {
      "activations/layer0_attention_weight_max": 16.703201293945312,
      "activations/layer0_attention_weight_min": -14.095915794372559,
      "activations/layer10_attention_weight_max": 34.0911750793457,
      "activations/layer10_attention_weight_min": -36.472084045410156,
      "activations/layer11_attention_weight_max": 31.24664878845215,
      "activations/layer11_attention_weight_min": -31.764915466308594,
      "activations/layer12_attention_weight_max": 16.776126861572266,
      "activations/layer12_attention_weight_min": -27.155122756958008,
      "activations/layer13_attention_weight_max": 34.63291549682617,
      "activations/layer13_attention_weight_min": -25.654958724975586,
      "activations/layer14_attention_weight_max": 34.18315505981445,
      "activations/layer14_attention_weight_min": -27.58289337158203,
      "activations/layer15_attention_weight_max": 28.524497985839844,
      "activations/layer15_attention_weight_min": -27.272279739379883,
      "activations/layer16_attention_weight_max": 29.995309829711914,
      "activations/layer16_attention_weight_min": -28.956987380981445,
      "activations/layer17_attention_weight_max": 49.34857940673828,
      "activations/layer17_attention_weight_min": -45.563201904296875,
      "activations/layer18_attention_weight_max": 44.88949966430664,
      "activations/layer18_attention_weight_min": -38.4708366394043,
      "activations/layer19_attention_weight_max": 19.551136016845703,
      "activations/layer19_attention_weight_min": -23.838109970092773,
      "activations/layer1_attention_weight_max": 17.332460403442383,
      "activations/layer1_attention_weight_min": -14.903700828552246,
      "activations/layer20_attention_weight_max": 19.869354248046875,
      "activations/layer20_attention_weight_min": -23.150606155395508,
      "activations/layer21_attention_weight_max": 30.35893440246582,
      "activations/layer21_attention_weight_min": -23.87141990661621,
      "activations/layer22_attention_weight_max": 29.131547927856445,
      "activations/layer22_attention_weight_min": -27.413795471191406,
      "activations/layer23_attention_weight_max": 37.33515167236328,
      "activations/layer23_attention_weight_min": -25.940641403198242,
      "activations/layer2_attention_weight_max": 31.680593490600586,
      "activations/layer2_attention_weight_min": -32.14084243774414,
      "activations/layer3_attention_weight_max": 86.7641372680664,
      "activations/layer3_attention_weight_min": -94.48445892333984,
      "activations/layer4_attention_weight_max": 98.26119995117188,
      "activations/layer4_attention_weight_min": -94.60448455810547,
      "activations/layer5_attention_weight_max": 75.89322662353516,
      "activations/layer5_attention_weight_min": -75.60738372802734,
      "activations/layer6_attention_weight_max": 51.66313552856445,
      "activations/layer6_attention_weight_min": -52.97245788574219,
      "activations/layer7_attention_weight_max": 68.4600830078125,
      "activations/layer7_attention_weight_min": -65.71603393554688,
      "activations/layer8_attention_weight_max": 45.392791748046875,
      "activations/layer8_attention_weight_min": -44.326942443847656,
      "activations/layer9_attention_weight_max": 52.79668045043945,
      "activations/layer9_attention_weight_min": -44.683494567871094,
      "epoch": 12.16,
      "learning_rate": 7.234015151515151e-05,
      "loss": 2.7578,
      "step": 209200
    },
    {
      "activations/layer0_attention_weight_max": 16.31173324584961,
      "activations/layer0_attention_weight_min": -14.512520790100098,
      "activations/layer10_attention_weight_max": 35.48444366455078,
      "activations/layer10_attention_weight_min": -35.6158332824707,
      "activations/layer11_attention_weight_max": 32.04979705810547,
      "activations/layer11_attention_weight_min": -32.6746940612793,
      "activations/layer12_attention_weight_max": 17.848934173583984,
      "activations/layer12_attention_weight_min": -24.396522521972656,
      "activations/layer13_attention_weight_max": 36.51683807373047,
      "activations/layer13_attention_weight_min": -31.67864227294922,
      "activations/layer14_attention_weight_max": 37.171688079833984,
      "activations/layer14_attention_weight_min": -31.464134216308594,
      "activations/layer15_attention_weight_max": 31.678329467773438,
      "activations/layer15_attention_weight_min": -29.8765811920166,
      "activations/layer16_attention_weight_max": 30.82646942138672,
      "activations/layer16_attention_weight_min": -29.618656158447266,
      "activations/layer17_attention_weight_max": 55.42632293701172,
      "activations/layer17_attention_weight_min": -46.071285247802734,
      "activations/layer18_attention_weight_max": 48.548404693603516,
      "activations/layer18_attention_weight_min": -41.28968811035156,
      "activations/layer19_attention_weight_max": 22.824113845825195,
      "activations/layer19_attention_weight_min": -21.663375854492188,
      "activations/layer1_attention_weight_max": 16.366748809814453,
      "activations/layer1_attention_weight_min": -15.012709617614746,
      "activations/layer20_attention_weight_max": 22.199142456054688,
      "activations/layer20_attention_weight_min": -20.352977752685547,
      "activations/layer21_attention_weight_max": 39.04375076293945,
      "activations/layer21_attention_weight_min": -22.6297550201416,
      "activations/layer22_attention_weight_max": 31.425086975097656,
      "activations/layer22_attention_weight_min": -25.029260635375977,
      "activations/layer23_attention_weight_max": 36.419620513916016,
      "activations/layer23_attention_weight_min": -24.862995147705078,
      "activations/layer2_attention_weight_max": 33.964942932128906,
      "activations/layer2_attention_weight_min": -31.427001953125,
      "activations/layer3_attention_weight_max": 91.43623352050781,
      "activations/layer3_attention_weight_min": -95.4413833618164,
      "activations/layer4_attention_weight_max": 97.6778564453125,
      "activations/layer4_attention_weight_min": -93.65132904052734,
      "activations/layer5_attention_weight_max": 70.73249816894531,
      "activations/layer5_attention_weight_min": -82.78541564941406,
      "activations/layer6_attention_weight_max": 55.357322692871094,
      "activations/layer6_attention_weight_min": -52.35000991821289,
      "activations/layer7_attention_weight_max": 73.26966094970703,
      "activations/layer7_attention_weight_min": -69.54314422607422,
      "activations/layer8_attention_weight_max": 44.2501106262207,
      "activations/layer8_attention_weight_min": -47.96625518798828,
      "activations/layer9_attention_weight_max": 47.59895706176758,
      "activations/layer9_attention_weight_min": -48.1309814453125,
      "epoch": 12.16,
      "learning_rate": 7.232121212121212e-05,
      "loss": 2.7829,
      "step": 209250
    },
    {
      "activations/layer0_attention_weight_max": 16.25098991394043,
      "activations/layer0_attention_weight_min": -14.180541038513184,
      "activations/layer10_attention_weight_max": 38.87944412231445,
      "activations/layer10_attention_weight_min": -39.50339126586914,
      "activations/layer11_attention_weight_max": 33.656150817871094,
      "activations/layer11_attention_weight_min": -34.54273223876953,
      "activations/layer12_attention_weight_max": 18.893674850463867,
      "activations/layer12_attention_weight_min": -26.072229385375977,
      "activations/layer13_attention_weight_max": 37.50737762451172,
      "activations/layer13_attention_weight_min": -30.307485580444336,
      "activations/layer14_attention_weight_max": 36.17408752441406,
      "activations/layer14_attention_weight_min": -32.214359283447266,
      "activations/layer15_attention_weight_max": 31.77264404296875,
      "activations/layer15_attention_weight_min": -28.73322868347168,
      "activations/layer16_attention_weight_max": 30.72186279296875,
      "activations/layer16_attention_weight_min": -29.97721290588379,
      "activations/layer17_attention_weight_max": 52.8774299621582,
      "activations/layer17_attention_weight_min": -44.53377151489258,
      "activations/layer18_attention_weight_max": 42.599308013916016,
      "activations/layer18_attention_weight_min": -38.9173698425293,
      "activations/layer19_attention_weight_max": 21.48213768005371,
      "activations/layer19_attention_weight_min": -24.52920913696289,
      "activations/layer1_attention_weight_max": 17.77521514892578,
      "activations/layer1_attention_weight_min": -14.226181983947754,
      "activations/layer20_attention_weight_max": 20.253549575805664,
      "activations/layer20_attention_weight_min": -23.37715721130371,
      "activations/layer21_attention_weight_max": 31.044647216796875,
      "activations/layer21_attention_weight_min": -29.831533432006836,
      "activations/layer22_attention_weight_max": 27.58955192565918,
      "activations/layer22_attention_weight_min": -25.362695693969727,
      "activations/layer23_attention_weight_max": 35.12413787841797,
      "activations/layer23_attention_weight_min": -26.040464401245117,
      "activations/layer2_attention_weight_max": 31.141685485839844,
      "activations/layer2_attention_weight_min": -29.375507354736328,
      "activations/layer3_attention_weight_max": 86.83126831054688,
      "activations/layer3_attention_weight_min": -92.099853515625,
      "activations/layer4_attention_weight_max": 93.9864730834961,
      "activations/layer4_attention_weight_min": -92.97212982177734,
      "activations/layer5_attention_weight_max": 69.89234924316406,
      "activations/layer5_attention_weight_min": -82.02943420410156,
      "activations/layer6_attention_weight_max": 54.06010437011719,
      "activations/layer6_attention_weight_min": -50.51203536987305,
      "activations/layer7_attention_weight_max": 69.71790313720703,
      "activations/layer7_attention_weight_min": -65.77420043945312,
      "activations/layer8_attention_weight_max": 45.021202087402344,
      "activations/layer8_attention_weight_min": -45.55654525756836,
      "activations/layer9_attention_weight_max": 48.10240936279297,
      "activations/layer9_attention_weight_min": -46.44843292236328,
      "epoch": 12.16,
      "learning_rate": 7.230227272727272e-05,
      "loss": 2.7789,
      "step": 209300
    },
    {
      "activations/layer0_attention_weight_max": 16.711450576782227,
      "activations/layer0_attention_weight_min": -14.379478454589844,
      "activations/layer10_attention_weight_max": 37.15705108642578,
      "activations/layer10_attention_weight_min": -37.36553955078125,
      "activations/layer11_attention_weight_max": 33.491397857666016,
      "activations/layer11_attention_weight_min": -34.67146301269531,
      "activations/layer12_attention_weight_max": 19.524917602539062,
      "activations/layer12_attention_weight_min": -23.872459411621094,
      "activations/layer13_attention_weight_max": 33.82160568237305,
      "activations/layer13_attention_weight_min": -29.009410858154297,
      "activations/layer14_attention_weight_max": 37.90800476074219,
      "activations/layer14_attention_weight_min": -31.615821838378906,
      "activations/layer15_attention_weight_max": 34.35165786743164,
      "activations/layer15_attention_weight_min": -32.852657318115234,
      "activations/layer16_attention_weight_max": 32.80189514160156,
      "activations/layer16_attention_weight_min": -31.65935516357422,
      "activations/layer17_attention_weight_max": 57.02119827270508,
      "activations/layer17_attention_weight_min": -46.71638870239258,
      "activations/layer18_attention_weight_max": 51.59906005859375,
      "activations/layer18_attention_weight_min": -38.7105827331543,
      "activations/layer19_attention_weight_max": 23.390310287475586,
      "activations/layer19_attention_weight_min": -23.211627960205078,
      "activations/layer1_attention_weight_max": 16.302526473999023,
      "activations/layer1_attention_weight_min": -14.684632301330566,
      "activations/layer20_attention_weight_max": 22.81246566772461,
      "activations/layer20_attention_weight_min": -22.362627029418945,
      "activations/layer21_attention_weight_max": 34.98064422607422,
      "activations/layer21_attention_weight_min": -23.150968551635742,
      "activations/layer22_attention_weight_max": 33.09010314941406,
      "activations/layer22_attention_weight_min": -26.50460433959961,
      "activations/layer23_attention_weight_max": 37.51335906982422,
      "activations/layer23_attention_weight_min": -22.494474411010742,
      "activations/layer2_attention_weight_max": 30.606983184814453,
      "activations/layer2_attention_weight_min": -29.716426849365234,
      "activations/layer3_attention_weight_max": 90.47216796875,
      "activations/layer3_attention_weight_min": -97.82755279541016,
      "activations/layer4_attention_weight_max": 96.28270721435547,
      "activations/layer4_attention_weight_min": -98.49175262451172,
      "activations/layer5_attention_weight_max": 73.38321685791016,
      "activations/layer5_attention_weight_min": -79.66234588623047,
      "activations/layer6_attention_weight_max": 54.22205352783203,
      "activations/layer6_attention_weight_min": -55.60546112060547,
      "activations/layer7_attention_weight_max": 70.65229797363281,
      "activations/layer7_attention_weight_min": -71.2322769165039,
      "activations/layer8_attention_weight_max": 46.2620849609375,
      "activations/layer8_attention_weight_min": -54.83052444458008,
      "activations/layer9_attention_weight_max": 48.42208480834961,
      "activations/layer9_attention_weight_min": -49.3607177734375,
      "epoch": 12.16,
      "learning_rate": 7.228333333333332e-05,
      "loss": 2.778,
      "step": 209350
    },
    {
      "activations/layer0_attention_weight_max": 15.206679344177246,
      "activations/layer0_attention_weight_min": -14.712264060974121,
      "activations/layer10_attention_weight_max": 36.38677978515625,
      "activations/layer10_attention_weight_min": -35.97663116455078,
      "activations/layer11_attention_weight_max": 32.416263580322266,
      "activations/layer11_attention_weight_min": -32.97142028808594,
      "activations/layer12_attention_weight_max": 18.042098999023438,
      "activations/layer12_attention_weight_min": -27.497127532958984,
      "activations/layer13_attention_weight_max": 35.291988372802734,
      "activations/layer13_attention_weight_min": -29.96902847290039,
      "activations/layer14_attention_weight_max": 34.11663818359375,
      "activations/layer14_attention_weight_min": -30.466773986816406,
      "activations/layer15_attention_weight_max": 32.45250701904297,
      "activations/layer15_attention_weight_min": -29.512218475341797,
      "activations/layer16_attention_weight_max": 31.993473052978516,
      "activations/layer16_attention_weight_min": -29.134502410888672,
      "activations/layer17_attention_weight_max": 50.93601989746094,
      "activations/layer17_attention_weight_min": -42.04595184326172,
      "activations/layer18_attention_weight_max": 45.87944030761719,
      "activations/layer18_attention_weight_min": -35.79418182373047,
      "activations/layer19_attention_weight_max": 25.521587371826172,
      "activations/layer19_attention_weight_min": -22.02545928955078,
      "activations/layer1_attention_weight_max": 16.545589447021484,
      "activations/layer1_attention_weight_min": -13.297082901000977,
      "activations/layer20_attention_weight_max": 23.02815055847168,
      "activations/layer20_attention_weight_min": -21.803035736083984,
      "activations/layer21_attention_weight_max": 36.9387321472168,
      "activations/layer21_attention_weight_min": -22.901153564453125,
      "activations/layer22_attention_weight_max": 33.523216247558594,
      "activations/layer22_attention_weight_min": -26.103113174438477,
      "activations/layer23_attention_weight_max": 38.177734375,
      "activations/layer23_attention_weight_min": -23.103458404541016,
      "activations/layer2_attention_weight_max": 32.127925872802734,
      "activations/layer2_attention_weight_min": -31.30697250366211,
      "activations/layer3_attention_weight_max": 90.85081481933594,
      "activations/layer3_attention_weight_min": -94.09811401367188,
      "activations/layer4_attention_weight_max": 95.57134246826172,
      "activations/layer4_attention_weight_min": -92.49633026123047,
      "activations/layer5_attention_weight_max": 73.30101776123047,
      "activations/layer5_attention_weight_min": -79.79360961914062,
      "activations/layer6_attention_weight_max": 53.09152603149414,
      "activations/layer6_attention_weight_min": -52.91232681274414,
      "activations/layer7_attention_weight_max": 71.89938354492188,
      "activations/layer7_attention_weight_min": -67.42135620117188,
      "activations/layer8_attention_weight_max": 44.41024398803711,
      "activations/layer8_attention_weight_min": -47.7729606628418,
      "activations/layer9_attention_weight_max": 54.84088897705078,
      "activations/layer9_attention_weight_min": -49.27286911010742,
      "epoch": 12.17,
      "learning_rate": 7.226439393939393e-05,
      "loss": 2.7694,
      "step": 209400
    },
    {
      "activations/layer0_attention_weight_max": 16.40290069580078,
      "activations/layer0_attention_weight_min": -14.461844444274902,
      "activations/layer10_attention_weight_max": 35.47856140136719,
      "activations/layer10_attention_weight_min": -34.21741485595703,
      "activations/layer11_attention_weight_max": 32.60432052612305,
      "activations/layer11_attention_weight_min": -32.55031967163086,
      "activations/layer12_attention_weight_max": 17.98179054260254,
      "activations/layer12_attention_weight_min": -24.013187408447266,
      "activations/layer13_attention_weight_max": 36.322509765625,
      "activations/layer13_attention_weight_min": -28.74983787536621,
      "activations/layer14_attention_weight_max": 39.45429992675781,
      "activations/layer14_attention_weight_min": -32.56285095214844,
      "activations/layer15_attention_weight_max": 35.82149887084961,
      "activations/layer15_attention_weight_min": -32.324302673339844,
      "activations/layer16_attention_weight_max": 32.98399353027344,
      "activations/layer16_attention_weight_min": -30.446590423583984,
      "activations/layer17_attention_weight_max": 60.12267303466797,
      "activations/layer17_attention_weight_min": -46.167667388916016,
      "activations/layer18_attention_weight_max": 51.62244415283203,
      "activations/layer18_attention_weight_min": -39.807464599609375,
      "activations/layer19_attention_weight_max": 22.736289978027344,
      "activations/layer19_attention_weight_min": -23.07266616821289,
      "activations/layer1_attention_weight_max": 16.28074073791504,
      "activations/layer1_attention_weight_min": -15.113309860229492,
      "activations/layer20_attention_weight_max": 24.620525360107422,
      "activations/layer20_attention_weight_min": -24.240447998046875,
      "activations/layer21_attention_weight_max": 36.4317512512207,
      "activations/layer21_attention_weight_min": -24.957489013671875,
      "activations/layer22_attention_weight_max": 30.27065658569336,
      "activations/layer22_attention_weight_min": -26.16733741760254,
      "activations/layer23_attention_weight_max": 33.912078857421875,
      "activations/layer23_attention_weight_min": -24.4115047454834,
      "activations/layer2_attention_weight_max": 31.41458511352539,
      "activations/layer2_attention_weight_min": -31.059009552001953,
      "activations/layer3_attention_weight_max": 91.82062530517578,
      "activations/layer3_attention_weight_min": -93.54596710205078,
      "activations/layer4_attention_weight_max": 99.76354217529297,
      "activations/layer4_attention_weight_min": -98.04155731201172,
      "activations/layer5_attention_weight_max": 71.8161392211914,
      "activations/layer5_attention_weight_min": -77.82911682128906,
      "activations/layer6_attention_weight_max": 55.13226318359375,
      "activations/layer6_attention_weight_min": -53.8233757019043,
      "activations/layer7_attention_weight_max": 72.15115356445312,
      "activations/layer7_attention_weight_min": -67.00162506103516,
      "activations/layer8_attention_weight_max": 46.888275146484375,
      "activations/layer8_attention_weight_min": -47.1346549987793,
      "activations/layer9_attention_weight_max": 43.838035583496094,
      "activations/layer9_attention_weight_min": -46.95603561401367,
      "epoch": 12.17,
      "learning_rate": 7.224545454545454e-05,
      "loss": 2.77,
      "step": 209450
    },
    {
      "activations/layer0_attention_weight_max": 15.651625633239746,
      "activations/layer0_attention_weight_min": -14.428619384765625,
      "activations/layer10_attention_weight_max": 39.34789276123047,
      "activations/layer10_attention_weight_min": -35.80451583862305,
      "activations/layer11_attention_weight_max": 33.79331588745117,
      "activations/layer11_attention_weight_min": -34.75660705566406,
      "activations/layer12_attention_weight_max": 17.686792373657227,
      "activations/layer12_attention_weight_min": -23.51673698425293,
      "activations/layer13_attention_weight_max": 36.83289337158203,
      "activations/layer13_attention_weight_min": -31.578420639038086,
      "activations/layer14_attention_weight_max": 38.000328063964844,
      "activations/layer14_attention_weight_min": -31.48831558227539,
      "activations/layer15_attention_weight_max": 33.207786560058594,
      "activations/layer15_attention_weight_min": -30.691394805908203,
      "activations/layer16_attention_weight_max": 30.548946380615234,
      "activations/layer16_attention_weight_min": -28.35588264465332,
      "activations/layer17_attention_weight_max": 50.49533462524414,
      "activations/layer17_attention_weight_min": -43.99995803833008,
      "activations/layer18_attention_weight_max": 47.37548065185547,
      "activations/layer18_attention_weight_min": -36.346099853515625,
      "activations/layer19_attention_weight_max": 20.599891662597656,
      "activations/layer19_attention_weight_min": -20.65776252746582,
      "activations/layer1_attention_weight_max": 16.747507095336914,
      "activations/layer1_attention_weight_min": -13.07265853881836,
      "activations/layer20_attention_weight_max": 22.462324142456055,
      "activations/layer20_attention_weight_min": -20.430320739746094,
      "activations/layer21_attention_weight_max": 34.70528793334961,
      "activations/layer21_attention_weight_min": -20.606426239013672,
      "activations/layer22_attention_weight_max": 31.299707412719727,
      "activations/layer22_attention_weight_min": -25.522357940673828,
      "activations/layer23_attention_weight_max": 36.814815521240234,
      "activations/layer23_attention_weight_min": -25.552562713623047,
      "activations/layer2_attention_weight_max": 31.21875,
      "activations/layer2_attention_weight_min": -30.37259292602539,
      "activations/layer3_attention_weight_max": 88.85186004638672,
      "activations/layer3_attention_weight_min": -93.3172836303711,
      "activations/layer4_attention_weight_max": 99.42143249511719,
      "activations/layer4_attention_weight_min": -91.3773422241211,
      "activations/layer5_attention_weight_max": 70.14799499511719,
      "activations/layer5_attention_weight_min": -79.10884094238281,
      "activations/layer6_attention_weight_max": 51.575767517089844,
      "activations/layer6_attention_weight_min": -53.93111801147461,
      "activations/layer7_attention_weight_max": 70.47911834716797,
      "activations/layer7_attention_weight_min": -65.7839584350586,
      "activations/layer8_attention_weight_max": 48.57085418701172,
      "activations/layer8_attention_weight_min": -48.005802154541016,
      "activations/layer9_attention_weight_max": 51.059200286865234,
      "activations/layer9_attention_weight_min": -48.72669982910156,
      "epoch": 12.17,
      "learning_rate": 7.222651515151514e-05,
      "loss": 2.7715,
      "step": 209500
    },
    {
      "activations/layer0_attention_weight_max": 16.95208168029785,
      "activations/layer0_attention_weight_min": -14.583415985107422,
      "activations/layer10_attention_weight_max": 38.55484390258789,
      "activations/layer10_attention_weight_min": -39.82387161254883,
      "activations/layer11_attention_weight_max": 35.938690185546875,
      "activations/layer11_attention_weight_min": -33.43821716308594,
      "activations/layer12_attention_weight_max": 16.990306854248047,
      "activations/layer12_attention_weight_min": -26.06757164001465,
      "activations/layer13_attention_weight_max": 44.15964889526367,
      "activations/layer13_attention_weight_min": -30.60027503967285,
      "activations/layer14_attention_weight_max": 53.49281311035156,
      "activations/layer14_attention_weight_min": -33.357845306396484,
      "activations/layer15_attention_weight_max": 50.97499465942383,
      "activations/layer15_attention_weight_min": -32.03656005859375,
      "activations/layer16_attention_weight_max": 32.60507583618164,
      "activations/layer16_attention_weight_min": -29.721391677856445,
      "activations/layer17_attention_weight_max": 53.872840881347656,
      "activations/layer17_attention_weight_min": -46.636688232421875,
      "activations/layer18_attention_weight_max": 50.7762565612793,
      "activations/layer18_attention_weight_min": -40.611209869384766,
      "activations/layer19_attention_weight_max": 22.49700355529785,
      "activations/layer19_attention_weight_min": -20.32904815673828,
      "activations/layer1_attention_weight_max": 17.218996047973633,
      "activations/layer1_attention_weight_min": -13.963593482971191,
      "activations/layer20_attention_weight_max": 24.725297927856445,
      "activations/layer20_attention_weight_min": -21.784440994262695,
      "activations/layer21_attention_weight_max": 42.691680908203125,
      "activations/layer21_attention_weight_min": -23.39861297607422,
      "activations/layer22_attention_weight_max": 31.199216842651367,
      "activations/layer22_attention_weight_min": -24.405969619750977,
      "activations/layer23_attention_weight_max": 36.80896759033203,
      "activations/layer23_attention_weight_min": -22.1340389251709,
      "activations/layer2_attention_weight_max": 31.033588409423828,
      "activations/layer2_attention_weight_min": -31.105878829956055,
      "activations/layer3_attention_weight_max": 91.0868148803711,
      "activations/layer3_attention_weight_min": -98.05824279785156,
      "activations/layer4_attention_weight_max": 96.42044830322266,
      "activations/layer4_attention_weight_min": -99.65079498291016,
      "activations/layer5_attention_weight_max": 68.99834442138672,
      "activations/layer5_attention_weight_min": -80.32290649414062,
      "activations/layer6_attention_weight_max": 55.38679122924805,
      "activations/layer6_attention_weight_min": -53.480751037597656,
      "activations/layer7_attention_weight_max": 71.05513763427734,
      "activations/layer7_attention_weight_min": -69.46177673339844,
      "activations/layer8_attention_weight_max": 48.09111404418945,
      "activations/layer8_attention_weight_min": -51.16282653808594,
      "activations/layer9_attention_weight_max": 56.59757614135742,
      "activations/layer9_attention_weight_min": -49.4903678894043,
      "epoch": 12.18,
      "learning_rate": 7.220757575757575e-05,
      "loss": 2.7767,
      "step": 209550
    },
    {
      "activations/layer0_attention_weight_max": 16.393545150756836,
      "activations/layer0_attention_weight_min": -14.245170593261719,
      "activations/layer10_attention_weight_max": 41.68185043334961,
      "activations/layer10_attention_weight_min": -41.365745544433594,
      "activations/layer11_attention_weight_max": 34.92643737792969,
      "activations/layer11_attention_weight_min": -35.631412506103516,
      "activations/layer12_attention_weight_max": 19.6975040435791,
      "activations/layer12_attention_weight_min": -24.344287872314453,
      "activations/layer13_attention_weight_max": 38.897491455078125,
      "activations/layer13_attention_weight_min": -28.65204429626465,
      "activations/layer14_attention_weight_max": 37.93336868286133,
      "activations/layer14_attention_weight_min": -30.232074737548828,
      "activations/layer15_attention_weight_max": 33.96137619018555,
      "activations/layer15_attention_weight_min": -30.229785919189453,
      "activations/layer16_attention_weight_max": 31.755773544311523,
      "activations/layer16_attention_weight_min": -29.563243865966797,
      "activations/layer17_attention_weight_max": 53.191463470458984,
      "activations/layer17_attention_weight_min": -42.54753494262695,
      "activations/layer18_attention_weight_max": 46.12729263305664,
      "activations/layer18_attention_weight_min": -38.98519515991211,
      "activations/layer19_attention_weight_max": 24.71139144897461,
      "activations/layer19_attention_weight_min": -21.48828125,
      "activations/layer1_attention_weight_max": 17.02767562866211,
      "activations/layer1_attention_weight_min": -16.498476028442383,
      "activations/layer20_attention_weight_max": 22.040746688842773,
      "activations/layer20_attention_weight_min": -21.674278259277344,
      "activations/layer21_attention_weight_max": 36.16984176635742,
      "activations/layer21_attention_weight_min": -25.66744041442871,
      "activations/layer22_attention_weight_max": 28.44744300842285,
      "activations/layer22_attention_weight_min": -24.90605926513672,
      "activations/layer23_attention_weight_max": 38.961341857910156,
      "activations/layer23_attention_weight_min": -22.958919525146484,
      "activations/layer2_attention_weight_max": 33.76244354248047,
      "activations/layer2_attention_weight_min": -32.131614685058594,
      "activations/layer3_attention_weight_max": 97.11836242675781,
      "activations/layer3_attention_weight_min": -97.94475555419922,
      "activations/layer4_attention_weight_max": 100.7711410522461,
      "activations/layer4_attention_weight_min": -99.10649108886719,
      "activations/layer5_attention_weight_max": 72.96150970458984,
      "activations/layer5_attention_weight_min": -81.74967193603516,
      "activations/layer6_attention_weight_max": 59.458106994628906,
      "activations/layer6_attention_weight_min": -58.23344039916992,
      "activations/layer7_attention_weight_max": 74.6884994506836,
      "activations/layer7_attention_weight_min": -75.49118041992188,
      "activations/layer8_attention_weight_max": 52.391056060791016,
      "activations/layer8_attention_weight_min": -56.83632278442383,
      "activations/layer9_attention_weight_max": 54.956024169921875,
      "activations/layer9_attention_weight_min": -52.423011779785156,
      "epoch": 12.18,
      "learning_rate": 7.218863636363635e-05,
      "loss": 2.7755,
      "step": 209600
    },
    {
      "activations/layer0_attention_weight_max": 17.449756622314453,
      "activations/layer0_attention_weight_min": -14.31944465637207,
      "activations/layer10_attention_weight_max": 35.885101318359375,
      "activations/layer10_attention_weight_min": -36.19035339355469,
      "activations/layer11_attention_weight_max": 32.55443572998047,
      "activations/layer11_attention_weight_min": -32.76438903808594,
      "activations/layer12_attention_weight_max": 16.540613174438477,
      "activations/layer12_attention_weight_min": -25.922677993774414,
      "activations/layer13_attention_weight_max": 38.31887435913086,
      "activations/layer13_attention_weight_min": -30.876182556152344,
      "activations/layer14_attention_weight_max": 34.942359924316406,
      "activations/layer14_attention_weight_min": -31.589746475219727,
      "activations/layer15_attention_weight_max": 32.36771011352539,
      "activations/layer15_attention_weight_min": -29.303253173828125,
      "activations/layer16_attention_weight_max": 32.309757232666016,
      "activations/layer16_attention_weight_min": -31.73522186279297,
      "activations/layer17_attention_weight_max": 51.630489349365234,
      "activations/layer17_attention_weight_min": -45.11201477050781,
      "activations/layer18_attention_weight_max": 45.262203216552734,
      "activations/layer18_attention_weight_min": -39.30049133300781,
      "activations/layer19_attention_weight_max": 24.12672996520996,
      "activations/layer19_attention_weight_min": -20.19184684753418,
      "activations/layer1_attention_weight_max": 16.823135375976562,
      "activations/layer1_attention_weight_min": -14.66984748840332,
      "activations/layer20_attention_weight_max": 22.238264083862305,
      "activations/layer20_attention_weight_min": -21.988309860229492,
      "activations/layer21_attention_weight_max": 35.26451110839844,
      "activations/layer21_attention_weight_min": -22.28809928894043,
      "activations/layer22_attention_weight_max": 30.1469783782959,
      "activations/layer22_attention_weight_min": -25.720767974853516,
      "activations/layer23_attention_weight_max": 36.75568771362305,
      "activations/layer23_attention_weight_min": -25.232608795166016,
      "activations/layer2_attention_weight_max": 31.508434295654297,
      "activations/layer2_attention_weight_min": -30.236427307128906,
      "activations/layer3_attention_weight_max": 92.06986236572266,
      "activations/layer3_attention_weight_min": -93.70067596435547,
      "activations/layer4_attention_weight_max": 94.80039978027344,
      "activations/layer4_attention_weight_min": -93.89659881591797,
      "activations/layer5_attention_weight_max": 71.98992919921875,
      "activations/layer5_attention_weight_min": -81.59259033203125,
      "activations/layer6_attention_weight_max": 52.48155975341797,
      "activations/layer6_attention_weight_min": -51.477821350097656,
      "activations/layer7_attention_weight_max": 69.90159606933594,
      "activations/layer7_attention_weight_min": -66.25249481201172,
      "activations/layer8_attention_weight_max": 44.897708892822266,
      "activations/layer8_attention_weight_min": -47.15962600708008,
      "activations/layer9_attention_weight_max": 46.784610748291016,
      "activations/layer9_attention_weight_min": -46.254417419433594,
      "epoch": 12.18,
      "learning_rate": 7.216969696969697e-05,
      "loss": 2.77,
      "step": 209650
    },
    {
      "activations/layer0_attention_weight_max": 16.396467208862305,
      "activations/layer0_attention_weight_min": -14.575271606445312,
      "activations/layer10_attention_weight_max": 41.359825134277344,
      "activations/layer10_attention_weight_min": -39.02047348022461,
      "activations/layer11_attention_weight_max": 38.15060043334961,
      "activations/layer11_attention_weight_min": -35.10198211669922,
      "activations/layer12_attention_weight_max": 19.199556350708008,
      "activations/layer12_attention_weight_min": -27.81877899169922,
      "activations/layer13_attention_weight_max": 39.16515350341797,
      "activations/layer13_attention_weight_min": -29.521778106689453,
      "activations/layer14_attention_weight_max": 38.17812728881836,
      "activations/layer14_attention_weight_min": -31.199777603149414,
      "activations/layer15_attention_weight_max": 34.06386184692383,
      "activations/layer15_attention_weight_min": -30.799274444580078,
      "activations/layer16_attention_weight_max": 34.21213912963867,
      "activations/layer16_attention_weight_min": -31.399593353271484,
      "activations/layer17_attention_weight_max": 54.7508430480957,
      "activations/layer17_attention_weight_min": -44.40766525268555,
      "activations/layer18_attention_weight_max": 48.80461120605469,
      "activations/layer18_attention_weight_min": -39.44414520263672,
      "activations/layer19_attention_weight_max": 24.68393898010254,
      "activations/layer19_attention_weight_min": -24.421096801757812,
      "activations/layer1_attention_weight_max": 16.292346954345703,
      "activations/layer1_attention_weight_min": -15.305487632751465,
      "activations/layer20_attention_weight_max": 24.79918670654297,
      "activations/layer20_attention_weight_min": -21.44658660888672,
      "activations/layer21_attention_weight_max": 37.041114807128906,
      "activations/layer21_attention_weight_min": -27.30276870727539,
      "activations/layer22_attention_weight_max": 34.8535041809082,
      "activations/layer22_attention_weight_min": -28.101051330566406,
      "activations/layer23_attention_weight_max": 40.898990631103516,
      "activations/layer23_attention_weight_min": -24.28371238708496,
      "activations/layer2_attention_weight_max": 33.212921142578125,
      "activations/layer2_attention_weight_min": -31.874502182006836,
      "activations/layer3_attention_weight_max": 95.19791412353516,
      "activations/layer3_attention_weight_min": -95.33013916015625,
      "activations/layer4_attention_weight_max": 102.46923065185547,
      "activations/layer4_attention_weight_min": -100.20638275146484,
      "activations/layer5_attention_weight_max": 74.35174560546875,
      "activations/layer5_attention_weight_min": -80.76773071289062,
      "activations/layer6_attention_weight_max": 54.36348342895508,
      "activations/layer6_attention_weight_min": -55.72543716430664,
      "activations/layer7_attention_weight_max": 77.17350006103516,
      "activations/layer7_attention_weight_min": -70.96824645996094,
      "activations/layer8_attention_weight_max": 50.80217742919922,
      "activations/layer8_attention_weight_min": -52.79267501831055,
      "activations/layer9_attention_weight_max": 57.909908294677734,
      "activations/layer9_attention_weight_min": -49.80160140991211,
      "epoch": 12.18,
      "learning_rate": 7.215075757575757e-05,
      "loss": 2.7762,
      "step": 209700
    },
    {
      "activations/layer0_attention_weight_max": 14.657135009765625,
      "activations/layer0_attention_weight_min": -14.225342750549316,
      "activations/layer10_attention_weight_max": 36.800079345703125,
      "activations/layer10_attention_weight_min": -35.892364501953125,
      "activations/layer11_attention_weight_max": 31.74295425415039,
      "activations/layer11_attention_weight_min": -34.415714263916016,
      "activations/layer12_attention_weight_max": 17.241422653198242,
      "activations/layer12_attention_weight_min": -25.1236515045166,
      "activations/layer13_attention_weight_max": 38.0382194519043,
      "activations/layer13_attention_weight_min": -29.88651466369629,
      "activations/layer14_attention_weight_max": 35.540283203125,
      "activations/layer14_attention_weight_min": -29.97395133972168,
      "activations/layer15_attention_weight_max": 30.963682174682617,
      "activations/layer15_attention_weight_min": -29.370119094848633,
      "activations/layer16_attention_weight_max": 28.275922775268555,
      "activations/layer16_attention_weight_min": -31.686479568481445,
      "activations/layer17_attention_weight_max": 48.219093322753906,
      "activations/layer17_attention_weight_min": -43.625511169433594,
      "activations/layer18_attention_weight_max": 42.74600601196289,
      "activations/layer18_attention_weight_min": -39.014068603515625,
      "activations/layer19_attention_weight_max": 25.944995880126953,
      "activations/layer19_attention_weight_min": -22.05284881591797,
      "activations/layer1_attention_weight_max": 17.743309020996094,
      "activations/layer1_attention_weight_min": -17.405803680419922,
      "activations/layer20_attention_weight_max": 21.098373413085938,
      "activations/layer20_attention_weight_min": -20.822265625,
      "activations/layer21_attention_weight_max": 34.23353958129883,
      "activations/layer21_attention_weight_min": -22.758960723876953,
      "activations/layer22_attention_weight_max": 29.796220779418945,
      "activations/layer22_attention_weight_min": -26.05497932434082,
      "activations/layer23_attention_weight_max": 35.603004455566406,
      "activations/layer23_attention_weight_min": -22.823318481445312,
      "activations/layer2_attention_weight_max": 34.59576416015625,
      "activations/layer2_attention_weight_min": -34.13117980957031,
      "activations/layer3_attention_weight_max": 100.32044982910156,
      "activations/layer3_attention_weight_min": -107.20269012451172,
      "activations/layer4_attention_weight_max": 104.79390716552734,
      "activations/layer4_attention_weight_min": -103.30803680419922,
      "activations/layer5_attention_weight_max": 73.1588134765625,
      "activations/layer5_attention_weight_min": -78.32994079589844,
      "activations/layer6_attention_weight_max": 54.711524963378906,
      "activations/layer6_attention_weight_min": -54.26783752441406,
      "activations/layer7_attention_weight_max": 70.34707641601562,
      "activations/layer7_attention_weight_min": -71.39864349365234,
      "activations/layer8_attention_weight_max": 47.71318817138672,
      "activations/layer8_attention_weight_min": -49.90311050415039,
      "activations/layer9_attention_weight_max": 46.93415069580078,
      "activations/layer9_attention_weight_min": -50.37358474731445,
      "epoch": 12.19,
      "learning_rate": 7.213181818181819e-05,
      "loss": 2.7655,
      "step": 209750
    },
    {
      "activations/layer0_attention_weight_max": 15.550362586975098,
      "activations/layer0_attention_weight_min": -13.992103576660156,
      "activations/layer10_attention_weight_max": 41.7122917175293,
      "activations/layer10_attention_weight_min": -38.7684440612793,
      "activations/layer11_attention_weight_max": 35.77113723754883,
      "activations/layer11_attention_weight_min": -33.62677764892578,
      "activations/layer12_attention_weight_max": 18.829957962036133,
      "activations/layer12_attention_weight_min": -25.3944034576416,
      "activations/layer13_attention_weight_max": 38.24366760253906,
      "activations/layer13_attention_weight_min": -29.821046829223633,
      "activations/layer14_attention_weight_max": 37.73871994018555,
      "activations/layer14_attention_weight_min": -30.02881622314453,
      "activations/layer15_attention_weight_max": 35.41463851928711,
      "activations/layer15_attention_weight_min": -29.506086349487305,
      "activations/layer16_attention_weight_max": 37.149471282958984,
      "activations/layer16_attention_weight_min": -30.517724990844727,
      "activations/layer17_attention_weight_max": 56.311153411865234,
      "activations/layer17_attention_weight_min": -47.28483581542969,
      "activations/layer18_attention_weight_max": 48.649967193603516,
      "activations/layer18_attention_weight_min": -41.01007843017578,
      "activations/layer19_attention_weight_max": 23.20289421081543,
      "activations/layer19_attention_weight_min": -21.846363067626953,
      "activations/layer1_attention_weight_max": 16.123088836669922,
      "activations/layer1_attention_weight_min": -14.864594459533691,
      "activations/layer20_attention_weight_max": 24.297874450683594,
      "activations/layer20_attention_weight_min": -21.77918243408203,
      "activations/layer21_attention_weight_max": 44.75318908691406,
      "activations/layer21_attention_weight_min": -22.002182006835938,
      "activations/layer22_attention_weight_max": 34.14046859741211,
      "activations/layer22_attention_weight_min": -27.06565284729004,
      "activations/layer23_attention_weight_max": 42.955421447753906,
      "activations/layer23_attention_weight_min": -24.31979751586914,
      "activations/layer2_attention_weight_max": 31.666088104248047,
      "activations/layer2_attention_weight_min": -30.189420700073242,
      "activations/layer3_attention_weight_max": 94.34687042236328,
      "activations/layer3_attention_weight_min": -93.0340347290039,
      "activations/layer4_attention_weight_max": 98.31048583984375,
      "activations/layer4_attention_weight_min": -98.24102020263672,
      "activations/layer5_attention_weight_max": 75.9230728149414,
      "activations/layer5_attention_weight_min": -78.74317932128906,
      "activations/layer6_attention_weight_max": 54.12271499633789,
      "activations/layer6_attention_weight_min": -54.7097282409668,
      "activations/layer7_attention_weight_max": 69.1243896484375,
      "activations/layer7_attention_weight_min": -68.705810546875,
      "activations/layer8_attention_weight_max": 50.1489143371582,
      "activations/layer8_attention_weight_min": -51.07836151123047,
      "activations/layer9_attention_weight_max": 51.26395797729492,
      "activations/layer9_attention_weight_min": -50.39545440673828,
      "epoch": 12.19,
      "learning_rate": 7.211287878787879e-05,
      "loss": 2.7817,
      "step": 209800
    },
    {
      "activations/layer0_attention_weight_max": 16.36610984802246,
      "activations/layer0_attention_weight_min": -14.17854118347168,
      "activations/layer10_attention_weight_max": 42.37214279174805,
      "activations/layer10_attention_weight_min": -43.829803466796875,
      "activations/layer11_attention_weight_max": 36.0341796875,
      "activations/layer11_attention_weight_min": -38.065250396728516,
      "activations/layer12_attention_weight_max": 19.331146240234375,
      "activations/layer12_attention_weight_min": -27.842647552490234,
      "activations/layer13_attention_weight_max": 38.96780014038086,
      "activations/layer13_attention_weight_min": -32.6367301940918,
      "activations/layer14_attention_weight_max": 39.84494400024414,
      "activations/layer14_attention_weight_min": -30.011201858520508,
      "activations/layer15_attention_weight_max": 37.375022888183594,
      "activations/layer15_attention_weight_min": -31.31481170654297,
      "activations/layer16_attention_weight_max": 32.979278564453125,
      "activations/layer16_attention_weight_min": -29.785667419433594,
      "activations/layer17_attention_weight_max": 56.87053680419922,
      "activations/layer17_attention_weight_min": -45.79301452636719,
      "activations/layer18_attention_weight_max": 48.2210807800293,
      "activations/layer18_attention_weight_min": -39.05752944946289,
      "activations/layer19_attention_weight_max": 23.388952255249023,
      "activations/layer19_attention_weight_min": -22.28459930419922,
      "activations/layer1_attention_weight_max": 16.210187911987305,
      "activations/layer1_attention_weight_min": -15.426512718200684,
      "activations/layer20_attention_weight_max": 23.25413703918457,
      "activations/layer20_attention_weight_min": -24.159685134887695,
      "activations/layer21_attention_weight_max": 43.45515823364258,
      "activations/layer21_attention_weight_min": -26.099956512451172,
      "activations/layer22_attention_weight_max": 32.76020812988281,
      "activations/layer22_attention_weight_min": -24.678634643554688,
      "activations/layer23_attention_weight_max": 40.07758331298828,
      "activations/layer23_attention_weight_min": -23.63452911376953,
      "activations/layer2_attention_weight_max": 35.535552978515625,
      "activations/layer2_attention_weight_min": -36.06755828857422,
      "activations/layer3_attention_weight_max": 100.44258117675781,
      "activations/layer3_attention_weight_min": -104.28533172607422,
      "activations/layer4_attention_weight_max": 107.93729400634766,
      "activations/layer4_attention_weight_min": -103.6440658569336,
      "activations/layer5_attention_weight_max": 81.97227478027344,
      "activations/layer5_attention_weight_min": -76.93812561035156,
      "activations/layer6_attention_weight_max": 61.75614547729492,
      "activations/layer6_attention_weight_min": -60.99797821044922,
      "activations/layer7_attention_weight_max": 83.60859680175781,
      "activations/layer7_attention_weight_min": -76.40884399414062,
      "activations/layer8_attention_weight_max": 57.04267120361328,
      "activations/layer8_attention_weight_min": -53.02863311767578,
      "activations/layer9_attention_weight_max": 58.926055908203125,
      "activations/layer9_attention_weight_min": -51.6057014465332,
      "epoch": 12.19,
      "learning_rate": 7.209393939393939e-05,
      "loss": 2.7623,
      "step": 209850
    },
    {
      "activations/layer0_attention_weight_max": 15.40691089630127,
      "activations/layer0_attention_weight_min": -14.37517261505127,
      "activations/layer10_attention_weight_max": 38.133750915527344,
      "activations/layer10_attention_weight_min": -38.89240264892578,
      "activations/layer11_attention_weight_max": 32.56775665283203,
      "activations/layer11_attention_weight_min": -34.77362060546875,
      "activations/layer12_attention_weight_max": 17.726253509521484,
      "activations/layer12_attention_weight_min": -27.104188919067383,
      "activations/layer13_attention_weight_max": 46.01306915283203,
      "activations/layer13_attention_weight_min": -34.171810150146484,
      "activations/layer14_attention_weight_max": 42.7265510559082,
      "activations/layer14_attention_weight_min": -34.100650787353516,
      "activations/layer15_attention_weight_max": 34.33841323852539,
      "activations/layer15_attention_weight_min": -30.53706932067871,
      "activations/layer16_attention_weight_max": 33.24200439453125,
      "activations/layer16_attention_weight_min": -29.61133575439453,
      "activations/layer17_attention_weight_max": 53.84966278076172,
      "activations/layer17_attention_weight_min": -44.087249755859375,
      "activations/layer18_attention_weight_max": 47.353206634521484,
      "activations/layer18_attention_weight_min": -39.385528564453125,
      "activations/layer19_attention_weight_max": 22.366987228393555,
      "activations/layer19_attention_weight_min": -25.079490661621094,
      "activations/layer1_attention_weight_max": 17.874773025512695,
      "activations/layer1_attention_weight_min": -14.507120132446289,
      "activations/layer20_attention_weight_max": 21.414493560791016,
      "activations/layer20_attention_weight_min": -24.268346786499023,
      "activations/layer21_attention_weight_max": 36.4631233215332,
      "activations/layer21_attention_weight_min": -27.495914459228516,
      "activations/layer22_attention_weight_max": 29.72975730895996,
      "activations/layer22_attention_weight_min": -26.094743728637695,
      "activations/layer23_attention_weight_max": 37.08686828613281,
      "activations/layer23_attention_weight_min": -28.501754760742188,
      "activations/layer2_attention_weight_max": 33.265708923339844,
      "activations/layer2_attention_weight_min": -33.58582305908203,
      "activations/layer3_attention_weight_max": 100.6179428100586,
      "activations/layer3_attention_weight_min": -102.09979248046875,
      "activations/layer4_attention_weight_max": 98.98856353759766,
      "activations/layer4_attention_weight_min": -96.23687744140625,
      "activations/layer5_attention_weight_max": 71.45733642578125,
      "activations/layer5_attention_weight_min": -77.94180297851562,
      "activations/layer6_attention_weight_max": 54.15158462524414,
      "activations/layer6_attention_weight_min": -53.06990051269531,
      "activations/layer7_attention_weight_max": 71.57379150390625,
      "activations/layer7_attention_weight_min": -71.02764892578125,
      "activations/layer8_attention_weight_max": 45.955657958984375,
      "activations/layer8_attention_weight_min": -47.40059280395508,
      "activations/layer9_attention_weight_max": 45.91311264038086,
      "activations/layer9_attention_weight_min": -47.66069030761719,
      "epoch": 12.2,
      "learning_rate": 7.207499999999999e-05,
      "loss": 2.7746,
      "step": 209900
    },
    {
      "activations/layer0_attention_weight_max": 15.649028778076172,
      "activations/layer0_attention_weight_min": -13.844144821166992,
      "activations/layer10_attention_weight_max": 32.63330841064453,
      "activations/layer10_attention_weight_min": -33.910545349121094,
      "activations/layer11_attention_weight_max": 29.697742462158203,
      "activations/layer11_attention_weight_min": -31.95705795288086,
      "activations/layer12_attention_weight_max": 19.080320358276367,
      "activations/layer12_attention_weight_min": -27.55460548400879,
      "activations/layer13_attention_weight_max": 36.39446258544922,
      "activations/layer13_attention_weight_min": -28.050079345703125,
      "activations/layer14_attention_weight_max": 34.38081741333008,
      "activations/layer14_attention_weight_min": -29.561904907226562,
      "activations/layer15_attention_weight_max": 33.0705680847168,
      "activations/layer15_attention_weight_min": -28.271879196166992,
      "activations/layer16_attention_weight_max": 30.0717830657959,
      "activations/layer16_attention_weight_min": -28.7906494140625,
      "activations/layer17_attention_weight_max": 47.82466506958008,
      "activations/layer17_attention_weight_min": -44.192501068115234,
      "activations/layer18_attention_weight_max": 43.78604507446289,
      "activations/layer18_attention_weight_min": -35.99382781982422,
      "activations/layer19_attention_weight_max": 21.396324157714844,
      "activations/layer19_attention_weight_min": -22.796981811523438,
      "activations/layer1_attention_weight_max": 16.3716983795166,
      "activations/layer1_attention_weight_min": -14.41519832611084,
      "activations/layer20_attention_weight_max": 23.046180725097656,
      "activations/layer20_attention_weight_min": -20.19700050354004,
      "activations/layer21_attention_weight_max": 39.45636749267578,
      "activations/layer21_attention_weight_min": -22.119121551513672,
      "activations/layer22_attention_weight_max": 33.56355285644531,
      "activations/layer22_attention_weight_min": -24.45830726623535,
      "activations/layer23_attention_weight_max": 33.43362808227539,
      "activations/layer23_attention_weight_min": -23.802593231201172,
      "activations/layer2_attention_weight_max": 33.302825927734375,
      "activations/layer2_attention_weight_min": -34.24124526977539,
      "activations/layer3_attention_weight_max": 93.26531219482422,
      "activations/layer3_attention_weight_min": -95.8003921508789,
      "activations/layer4_attention_weight_max": 96.26551818847656,
      "activations/layer4_attention_weight_min": -92.5728530883789,
      "activations/layer5_attention_weight_max": 69.99725341796875,
      "activations/layer5_attention_weight_min": -75.07437133789062,
      "activations/layer6_attention_weight_max": 50.15058517456055,
      "activations/layer6_attention_weight_min": -51.692752838134766,
      "activations/layer7_attention_weight_max": 68.20684051513672,
      "activations/layer7_attention_weight_min": -64.2637710571289,
      "activations/layer8_attention_weight_max": 46.006526947021484,
      "activations/layer8_attention_weight_min": -45.41646194458008,
      "activations/layer9_attention_weight_max": 47.244869232177734,
      "activations/layer9_attention_weight_min": -45.71751403808594,
      "epoch": 12.2,
      "learning_rate": 7.20560606060606e-05,
      "loss": 2.7574,
      "step": 209950
    },
    {
      "activations/layer0_attention_weight_max": 15.894479751586914,
      "activations/layer0_attention_weight_min": -14.351287841796875,
      "activations/layer10_attention_weight_max": 37.230079650878906,
      "activations/layer10_attention_weight_min": -34.55518341064453,
      "activations/layer11_attention_weight_max": 33.20560836791992,
      "activations/layer11_attention_weight_min": -33.165672302246094,
      "activations/layer12_attention_weight_max": 22.777292251586914,
      "activations/layer12_attention_weight_min": -24.06486701965332,
      "activations/layer13_attention_weight_max": 39.40949630737305,
      "activations/layer13_attention_weight_min": -29.724477767944336,
      "activations/layer14_attention_weight_max": 37.224727630615234,
      "activations/layer14_attention_weight_min": -31.30137825012207,
      "activations/layer15_attention_weight_max": 33.29926681518555,
      "activations/layer15_attention_weight_min": -30.538854598999023,
      "activations/layer16_attention_weight_max": 34.33787155151367,
      "activations/layer16_attention_weight_min": -31.70685577392578,
      "activations/layer17_attention_weight_max": 52.97514724731445,
      "activations/layer17_attention_weight_min": -45.19584274291992,
      "activations/layer18_attention_weight_max": 45.63396453857422,
      "activations/layer18_attention_weight_min": -37.14598846435547,
      "activations/layer19_attention_weight_max": 22.391815185546875,
      "activations/layer19_attention_weight_min": -21.553754806518555,
      "activations/layer1_attention_weight_max": 16.56142234802246,
      "activations/layer1_attention_weight_min": -14.106399536132812,
      "activations/layer20_attention_weight_max": 23.894136428833008,
      "activations/layer20_attention_weight_min": -23.176122665405273,
      "activations/layer21_attention_weight_max": 39.43108367919922,
      "activations/layer21_attention_weight_min": -20.388978958129883,
      "activations/layer22_attention_weight_max": 27.741079330444336,
      "activations/layer22_attention_weight_min": -25.587753295898438,
      "activations/layer23_attention_weight_max": 36.34867858886719,
      "activations/layer23_attention_weight_min": -23.364368438720703,
      "activations/layer2_attention_weight_max": 32.94768524169922,
      "activations/layer2_attention_weight_min": -33.57288360595703,
      "activations/layer3_attention_weight_max": 99.87693786621094,
      "activations/layer3_attention_weight_min": -100.40953826904297,
      "activations/layer4_attention_weight_max": 103.27501678466797,
      "activations/layer4_attention_weight_min": -98.60616302490234,
      "activations/layer5_attention_weight_max": 74.03032684326172,
      "activations/layer5_attention_weight_min": -80.64559173583984,
      "activations/layer6_attention_weight_max": 54.00507354736328,
      "activations/layer6_attention_weight_min": -55.51629638671875,
      "activations/layer7_attention_weight_max": 73.94058990478516,
      "activations/layer7_attention_weight_min": -69.1663818359375,
      "activations/layer8_attention_weight_max": 46.7583122253418,
      "activations/layer8_attention_weight_min": -47.7839469909668,
      "activations/layer9_attention_weight_max": 46.83540725708008,
      "activations/layer9_attention_weight_min": -46.76753616333008,
      "epoch": 12.2,
      "learning_rate": 7.20371212121212e-05,
      "loss": 2.7833,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_loss": 2.728515625,
      "eval_runtime": 8.566,
      "eval_samples_per_second": 501.284,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_openwebtext_loss": 2.728515625,
      "eval_openwebtext_ppl": 15.310144149387147,
      "eval_openwebtext_runtime": 8.566,
      "eval_openwebtext_samples_per_second": 501.284,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 2.0056,
      "eval_wikitext_samples_per_second": 227.36,
      "step": 210000
    },
    {
      "epoch": 12.2,
      "eval_lambada_loss": 2.53125,
      "eval_lambada_ppl": 12.569207830853442,
      "eval_lambada_runtime": 9.5922,
      "eval_lambada_samples_per_second": 507.602,
      "step": 210000
    },
    {
      "activations/layer0_attention_weight_max": 16.433326721191406,
      "activations/layer0_attention_weight_min": -14.041013717651367,
      "activations/layer10_attention_weight_max": 35.66180419921875,
      "activations/layer10_attention_weight_min": -34.69537353515625,
      "activations/layer11_attention_weight_max": 34.117733001708984,
      "activations/layer11_attention_weight_min": -33.181053161621094,
      "activations/layer12_attention_weight_max": 18.778884887695312,
      "activations/layer12_attention_weight_min": -25.886198043823242,
      "activations/layer13_attention_weight_max": 36.541473388671875,
      "activations/layer13_attention_weight_min": -24.53142547607422,
      "activations/layer14_attention_weight_max": 35.345115661621094,
      "activations/layer14_attention_weight_min": -29.233182907104492,
      "activations/layer15_attention_weight_max": 33.13064193725586,
      "activations/layer15_attention_weight_min": -29.297924041748047,
      "activations/layer16_attention_weight_max": 32.54243850708008,
      "activations/layer16_attention_weight_min": -30.462793350219727,
      "activations/layer17_attention_weight_max": 50.48892593383789,
      "activations/layer17_attention_weight_min": -42.79937744140625,
      "activations/layer18_attention_weight_max": 46.53837966918945,
      "activations/layer18_attention_weight_min": -35.42254638671875,
      "activations/layer19_attention_weight_max": 22.458967208862305,
      "activations/layer19_attention_weight_min": -20.739700317382812,
      "activations/layer1_attention_weight_max": 17.67381477355957,
      "activations/layer1_attention_weight_min": -14.148083686828613,
      "activations/layer20_attention_weight_max": 24.814420700073242,
      "activations/layer20_attention_weight_min": -22.314010620117188,
      "activations/layer21_attention_weight_max": 37.669803619384766,
      "activations/layer21_attention_weight_min": -20.31731605529785,
      "activations/layer22_attention_weight_max": 29.435396194458008,
      "activations/layer22_attention_weight_min": -24.945215225219727,
      "activations/layer23_attention_weight_max": 35.40772247314453,
      "activations/layer23_attention_weight_min": -22.60776138305664,
      "activations/layer2_attention_weight_max": 31.69786834716797,
      "activations/layer2_attention_weight_min": -31.16109275817871,
      "activations/layer3_attention_weight_max": 92.59541320800781,
      "activations/layer3_attention_weight_min": -95.19512176513672,
      "activations/layer4_attention_weight_max": 99.09526062011719,
      "activations/layer4_attention_weight_min": -92.09810638427734,
      "activations/layer5_attention_weight_max": 74.16154479980469,
      "activations/layer5_attention_weight_min": -78.75819396972656,
      "activations/layer6_attention_weight_max": 54.6748161315918,
      "activations/layer6_attention_weight_min": -52.238285064697266,
      "activations/layer7_attention_weight_max": 71.53138732910156,
      "activations/layer7_attention_weight_min": -62.403385162353516,
      "activations/layer8_attention_weight_max": 45.550941467285156,
      "activations/layer8_attention_weight_min": -45.926273345947266,
      "activations/layer9_attention_weight_max": 43.962120056152344,
      "activations/layer9_attention_weight_min": -43.77621078491211,
      "epoch": 12.21,
      "learning_rate": 7.201818181818181e-05,
      "loss": 2.7693,
      "step": 210050
    },
    {
      "activations/layer0_attention_weight_max": 16.078834533691406,
      "activations/layer0_attention_weight_min": -13.842473030090332,
      "activations/layer10_attention_weight_max": 35.49189758300781,
      "activations/layer10_attention_weight_min": -35.16341018676758,
      "activations/layer11_attention_weight_max": 31.12706756591797,
      "activations/layer11_attention_weight_min": -33.206809997558594,
      "activations/layer12_attention_weight_max": 17.673686981201172,
      "activations/layer12_attention_weight_min": -32.313148498535156,
      "activations/layer13_attention_weight_max": 38.164154052734375,
      "activations/layer13_attention_weight_min": -32.51998519897461,
      "activations/layer14_attention_weight_max": 35.79829025268555,
      "activations/layer14_attention_weight_min": -33.37010955810547,
      "activations/layer15_attention_weight_max": 33.586753845214844,
      "activations/layer15_attention_weight_min": -29.880878448486328,
      "activations/layer16_attention_weight_max": 32.268985748291016,
      "activations/layer16_attention_weight_min": -32.22597122192383,
      "activations/layer17_attention_weight_max": 54.95671844482422,
      "activations/layer17_attention_weight_min": -47.047760009765625,
      "activations/layer18_attention_weight_max": 49.71965026855469,
      "activations/layer18_attention_weight_min": -40.300323486328125,
      "activations/layer19_attention_weight_max": 24.08344078063965,
      "activations/layer19_attention_weight_min": -24.99515151977539,
      "activations/layer1_attention_weight_max": 17.123342514038086,
      "activations/layer1_attention_weight_min": -14.267766952514648,
      "activations/layer20_attention_weight_max": 24.432050704956055,
      "activations/layer20_attention_weight_min": -23.486787796020508,
      "activations/layer21_attention_weight_max": 35.353172302246094,
      "activations/layer21_attention_weight_min": -23.920751571655273,
      "activations/layer22_attention_weight_max": 31.701141357421875,
      "activations/layer22_attention_weight_min": -27.52345085144043,
      "activations/layer23_attention_weight_max": 35.803489685058594,
      "activations/layer23_attention_weight_min": -24.377304077148438,
      "activations/layer2_attention_weight_max": 33.92524719238281,
      "activations/layer2_attention_weight_min": -30.212020874023438,
      "activations/layer3_attention_weight_max": 99.89879608154297,
      "activations/layer3_attention_weight_min": -90.67375946044922,
      "activations/layer4_attention_weight_max": 97.75575256347656,
      "activations/layer4_attention_weight_min": -95.30365753173828,
      "activations/layer5_attention_weight_max": 71.25901794433594,
      "activations/layer5_attention_weight_min": -78.2020263671875,
      "activations/layer6_attention_weight_max": 53.89425277709961,
      "activations/layer6_attention_weight_min": -52.92899703979492,
      "activations/layer7_attention_weight_max": 69.83871459960938,
      "activations/layer7_attention_weight_min": -72.0623550415039,
      "activations/layer8_attention_weight_max": 44.36770248413086,
      "activations/layer8_attention_weight_min": -49.98698806762695,
      "activations/layer9_attention_weight_max": 46.723663330078125,
      "activations/layer9_attention_weight_min": -47.09194564819336,
      "epoch": 12.21,
      "learning_rate": 7.199924242424241e-05,
      "loss": 2.7651,
      "step": 210100
    },
    {
      "activations/layer0_attention_weight_max": 16.094484329223633,
      "activations/layer0_attention_weight_min": -14.072368621826172,
      "activations/layer10_attention_weight_max": 40.164344787597656,
      "activations/layer10_attention_weight_min": -39.12532424926758,
      "activations/layer11_attention_weight_max": 33.655574798583984,
      "activations/layer11_attention_weight_min": -34.64336395263672,
      "activations/layer12_attention_weight_max": 21.93960189819336,
      "activations/layer12_attention_weight_min": -24.770605087280273,
      "activations/layer13_attention_weight_max": 47.08079147338867,
      "activations/layer13_attention_weight_min": -31.622779846191406,
      "activations/layer14_attention_weight_max": 45.47471618652344,
      "activations/layer14_attention_weight_min": -31.730955123901367,
      "activations/layer15_attention_weight_max": 37.108619689941406,
      "activations/layer15_attention_weight_min": -29.977893829345703,
      "activations/layer16_attention_weight_max": 35.9176025390625,
      "activations/layer16_attention_weight_min": -31.259231567382812,
      "activations/layer17_attention_weight_max": 55.52587890625,
      "activations/layer17_attention_weight_min": -46.484222412109375,
      "activations/layer18_attention_weight_max": 52.2833137512207,
      "activations/layer18_attention_weight_min": -39.750282287597656,
      "activations/layer19_attention_weight_max": 24.1495304107666,
      "activations/layer19_attention_weight_min": -23.952192306518555,
      "activations/layer1_attention_weight_max": 16.356184005737305,
      "activations/layer1_attention_weight_min": -16.340579986572266,
      "activations/layer20_attention_weight_max": 25.538854598999023,
      "activations/layer20_attention_weight_min": -24.23904800415039,
      "activations/layer21_attention_weight_max": 41.57969665527344,
      "activations/layer21_attention_weight_min": -25.21421241760254,
      "activations/layer22_attention_weight_max": 32.771522521972656,
      "activations/layer22_attention_weight_min": -24.84943389892578,
      "activations/layer23_attention_weight_max": 42.441410064697266,
      "activations/layer23_attention_weight_min": -24.221729278564453,
      "activations/layer2_attention_weight_max": 33.91163635253906,
      "activations/layer2_attention_weight_min": -33.195125579833984,
      "activations/layer3_attention_weight_max": 96.23654174804688,
      "activations/layer3_attention_weight_min": -95.2792739868164,
      "activations/layer4_attention_weight_max": 101.41114807128906,
      "activations/layer4_attention_weight_min": -97.83061218261719,
      "activations/layer5_attention_weight_max": 76.2269287109375,
      "activations/layer5_attention_weight_min": -79.35462951660156,
      "activations/layer6_attention_weight_max": 54.644344329833984,
      "activations/layer6_attention_weight_min": -56.720001220703125,
      "activations/layer7_attention_weight_max": 70.59703826904297,
      "activations/layer7_attention_weight_min": -69.37186431884766,
      "activations/layer8_attention_weight_max": 46.9647102355957,
      "activations/layer8_attention_weight_min": -49.73674392700195,
      "activations/layer9_attention_weight_max": 46.472530364990234,
      "activations/layer9_attention_weight_min": -49.26414108276367,
      "epoch": 12.21,
      "learning_rate": 7.198030303030302e-05,
      "loss": 2.772,
      "step": 210150
    },
    {
      "activations/layer0_attention_weight_max": 15.587675094604492,
      "activations/layer0_attention_weight_min": -14.033117294311523,
      "activations/layer10_attention_weight_max": 36.06319808959961,
      "activations/layer10_attention_weight_min": -36.05333709716797,
      "activations/layer11_attention_weight_max": 33.778839111328125,
      "activations/layer11_attention_weight_min": -33.907493591308594,
      "activations/layer12_attention_weight_max": 16.37407684326172,
      "activations/layer12_attention_weight_min": -23.53221893310547,
      "activations/layer13_attention_weight_max": 32.89740753173828,
      "activations/layer13_attention_weight_min": -26.624229431152344,
      "activations/layer14_attention_weight_max": 31.770896911621094,
      "activations/layer14_attention_weight_min": -28.412994384765625,
      "activations/layer15_attention_weight_max": 31.137990951538086,
      "activations/layer15_attention_weight_min": -28.84552574157715,
      "activations/layer16_attention_weight_max": 32.1189079284668,
      "activations/layer16_attention_weight_min": -31.181310653686523,
      "activations/layer17_attention_weight_max": 50.43819046020508,
      "activations/layer17_attention_weight_min": -46.350765228271484,
      "activations/layer18_attention_weight_max": 40.4673957824707,
      "activations/layer18_attention_weight_min": -34.810218811035156,
      "activations/layer19_attention_weight_max": 20.362585067749023,
      "activations/layer19_attention_weight_min": -22.326810836791992,
      "activations/layer1_attention_weight_max": 15.767748832702637,
      "activations/layer1_attention_weight_min": -15.579928398132324,
      "activations/layer20_attention_weight_max": 19.80959701538086,
      "activations/layer20_attention_weight_min": -20.90666961669922,
      "activations/layer21_attention_weight_max": 33.316314697265625,
      "activations/layer21_attention_weight_min": -22.499364852905273,
      "activations/layer22_attention_weight_max": 29.2340087890625,
      "activations/layer22_attention_weight_min": -24.98604965209961,
      "activations/layer23_attention_weight_max": 37.101322174072266,
      "activations/layer23_attention_weight_min": -24.739105224609375,
      "activations/layer2_attention_weight_max": 31.692306518554688,
      "activations/layer2_attention_weight_min": -29.02205467224121,
      "activations/layer3_attention_weight_max": 96.13572692871094,
      "activations/layer3_attention_weight_min": -93.08705139160156,
      "activations/layer4_attention_weight_max": 99.1035385131836,
      "activations/layer4_attention_weight_min": -91.95343017578125,
      "activations/layer5_attention_weight_max": 69.51168823242188,
      "activations/layer5_attention_weight_min": -74.99740600585938,
      "activations/layer6_attention_weight_max": 55.53394317626953,
      "activations/layer6_attention_weight_min": -51.055747985839844,
      "activations/layer7_attention_weight_max": 70.90416717529297,
      "activations/layer7_attention_weight_min": -70.87239074707031,
      "activations/layer8_attention_weight_max": 45.74462127685547,
      "activations/layer8_attention_weight_min": -47.87897491455078,
      "activations/layer9_attention_weight_max": 46.09029769897461,
      "activations/layer9_attention_weight_min": -44.95024871826172,
      "epoch": 12.21,
      "learning_rate": 7.196136363636363e-05,
      "loss": 2.7839,
      "step": 210200
    },
    {
      "activations/layer0_attention_weight_max": 15.030441284179688,
      "activations/layer0_attention_weight_min": -14.182043075561523,
      "activations/layer10_attention_weight_max": 35.81227111816406,
      "activations/layer10_attention_weight_min": -35.769535064697266,
      "activations/layer11_attention_weight_max": 32.68441390991211,
      "activations/layer11_attention_weight_min": -34.115234375,
      "activations/layer12_attention_weight_max": 19.435455322265625,
      "activations/layer12_attention_weight_min": -24.374719619750977,
      "activations/layer13_attention_weight_max": 35.698272705078125,
      "activations/layer13_attention_weight_min": -30.363868713378906,
      "activations/layer14_attention_weight_max": 40.1634635925293,
      "activations/layer14_attention_weight_min": -30.261842727661133,
      "activations/layer15_attention_weight_max": 34.68730926513672,
      "activations/layer15_attention_weight_min": -30.863906860351562,
      "activations/layer16_attention_weight_max": 33.60618209838867,
      "activations/layer16_attention_weight_min": -30.462068557739258,
      "activations/layer17_attention_weight_max": 55.57249069213867,
      "activations/layer17_attention_weight_min": -44.87025833129883,
      "activations/layer18_attention_weight_max": 48.09325408935547,
      "activations/layer18_attention_weight_min": -41.95707321166992,
      "activations/layer19_attention_weight_max": 24.28952407836914,
      "activations/layer19_attention_weight_min": -23.875944137573242,
      "activations/layer1_attention_weight_max": 18.482250213623047,
      "activations/layer1_attention_weight_min": -14.895978927612305,
      "activations/layer20_attention_weight_max": 26.248876571655273,
      "activations/layer20_attention_weight_min": -21.882402420043945,
      "activations/layer21_attention_weight_max": 38.87364196777344,
      "activations/layer21_attention_weight_min": -26.03379249572754,
      "activations/layer22_attention_weight_max": 29.71781349182129,
      "activations/layer22_attention_weight_min": -27.57130241394043,
      "activations/layer23_attention_weight_max": 40.731014251708984,
      "activations/layer23_attention_weight_min": -23.833433151245117,
      "activations/layer2_attention_weight_max": 31.35417938232422,
      "activations/layer2_attention_weight_min": -28.782901763916016,
      "activations/layer3_attention_weight_max": 92.92922973632812,
      "activations/layer3_attention_weight_min": -90.41316223144531,
      "activations/layer4_attention_weight_max": 102.94233703613281,
      "activations/layer4_attention_weight_min": -92.29150390625,
      "activations/layer5_attention_weight_max": 74.30303955078125,
      "activations/layer5_attention_weight_min": -76.76564025878906,
      "activations/layer6_attention_weight_max": 54.3867073059082,
      "activations/layer6_attention_weight_min": -53.898414611816406,
      "activations/layer7_attention_weight_max": 74.03323364257812,
      "activations/layer7_attention_weight_min": -70.09661102294922,
      "activations/layer8_attention_weight_max": 45.28449249267578,
      "activations/layer8_attention_weight_min": -48.12461853027344,
      "activations/layer9_attention_weight_max": 46.35990905761719,
      "activations/layer9_attention_weight_min": -46.72966003417969,
      "epoch": 12.22,
      "learning_rate": 7.194242424242424e-05,
      "loss": 2.7711,
      "step": 210250
    },
    {
      "activations/layer0_attention_weight_max": 15.655562400817871,
      "activations/layer0_attention_weight_min": -14.35412311553955,
      "activations/layer10_attention_weight_max": 35.2554817199707,
      "activations/layer10_attention_weight_min": -34.8401985168457,
      "activations/layer11_attention_weight_max": 32.17736053466797,
      "activations/layer11_attention_weight_min": -31.036062240600586,
      "activations/layer12_attention_weight_max": 17.597200393676758,
      "activations/layer12_attention_weight_min": -23.565425872802734,
      "activations/layer13_attention_weight_max": 37.237579345703125,
      "activations/layer13_attention_weight_min": -28.531970977783203,
      "activations/layer14_attention_weight_max": 36.28414535522461,
      "activations/layer14_attention_weight_min": -29.86883544921875,
      "activations/layer15_attention_weight_max": 32.11325454711914,
      "activations/layer15_attention_weight_min": -28.76246452331543,
      "activations/layer16_attention_weight_max": 31.074548721313477,
      "activations/layer16_attention_weight_min": -28.249441146850586,
      "activations/layer17_attention_weight_max": 49.36082077026367,
      "activations/layer17_attention_weight_min": -42.75049591064453,
      "activations/layer18_attention_weight_max": 48.8279914855957,
      "activations/layer18_attention_weight_min": -34.47534942626953,
      "activations/layer19_attention_weight_max": 23.396833419799805,
      "activations/layer19_attention_weight_min": -20.48419761657715,
      "activations/layer1_attention_weight_max": 17.57200813293457,
      "activations/layer1_attention_weight_min": -15.597314834594727,
      "activations/layer20_attention_weight_max": 20.821672439575195,
      "activations/layer20_attention_weight_min": -21.064029693603516,
      "activations/layer21_attention_weight_max": 37.88394546508789,
      "activations/layer21_attention_weight_min": -22.82696533203125,
      "activations/layer22_attention_weight_max": 29.673751831054688,
      "activations/layer22_attention_weight_min": -25.03708839416504,
      "activations/layer23_attention_weight_max": 38.604827880859375,
      "activations/layer23_attention_weight_min": -22.11302947998047,
      "activations/layer2_attention_weight_max": 33.02644729614258,
      "activations/layer2_attention_weight_min": -31.159086227416992,
      "activations/layer3_attention_weight_max": 92.62755584716797,
      "activations/layer3_attention_weight_min": -92.84908294677734,
      "activations/layer4_attention_weight_max": 95.76493072509766,
      "activations/layer4_attention_weight_min": -92.41709899902344,
      "activations/layer5_attention_weight_max": 71.97041320800781,
      "activations/layer5_attention_weight_min": -77.71150970458984,
      "activations/layer6_attention_weight_max": 52.6002311706543,
      "activations/layer6_attention_weight_min": -54.00232696533203,
      "activations/layer7_attention_weight_max": 67.42789459228516,
      "activations/layer7_attention_weight_min": -67.54562377929688,
      "activations/layer8_attention_weight_max": 42.602989196777344,
      "activations/layer8_attention_weight_min": -49.179779052734375,
      "activations/layer9_attention_weight_max": 45.16899871826172,
      "activations/layer9_attention_weight_min": -43.730159759521484,
      "epoch": 12.22,
      "learning_rate": 7.192348484848484e-05,
      "loss": 2.7763,
      "step": 210300
    },
    {
      "activations/layer0_attention_weight_max": 16.41725730895996,
      "activations/layer0_attention_weight_min": -14.198866844177246,
      "activations/layer10_attention_weight_max": 33.771888732910156,
      "activations/layer10_attention_weight_min": -34.498199462890625,
      "activations/layer11_attention_weight_max": 32.353370666503906,
      "activations/layer11_attention_weight_min": -35.37755584716797,
      "activations/layer12_attention_weight_max": 18.852277755737305,
      "activations/layer12_attention_weight_min": -26.124595642089844,
      "activations/layer13_attention_weight_max": 33.320838928222656,
      "activations/layer13_attention_weight_min": -29.482036590576172,
      "activations/layer14_attention_weight_max": 37.0582160949707,
      "activations/layer14_attention_weight_min": -30.850109100341797,
      "activations/layer15_attention_weight_max": 32.76234817504883,
      "activations/layer15_attention_weight_min": -29.5404052734375,
      "activations/layer16_attention_weight_max": 30.723237991333008,
      "activations/layer16_attention_weight_min": -30.43701934814453,
      "activations/layer17_attention_weight_max": 49.07744216918945,
      "activations/layer17_attention_weight_min": -42.890647888183594,
      "activations/layer18_attention_weight_max": 44.91889572143555,
      "activations/layer18_attention_weight_min": -38.894840240478516,
      "activations/layer19_attention_weight_max": 21.136451721191406,
      "activations/layer19_attention_weight_min": -21.735584259033203,
      "activations/layer1_attention_weight_max": 17.23335075378418,
      "activations/layer1_attention_weight_min": -14.391586303710938,
      "activations/layer20_attention_weight_max": 21.089387893676758,
      "activations/layer20_attention_weight_min": -21.537736892700195,
      "activations/layer21_attention_weight_max": 35.20929718017578,
      "activations/layer21_attention_weight_min": -21.907949447631836,
      "activations/layer22_attention_weight_max": 29.301097869873047,
      "activations/layer22_attention_weight_min": -28.35405921936035,
      "activations/layer23_attention_weight_max": 37.275901794433594,
      "activations/layer23_attention_weight_min": -23.09689712524414,
      "activations/layer2_attention_weight_max": 33.189300537109375,
      "activations/layer2_attention_weight_min": -32.074913024902344,
      "activations/layer3_attention_weight_max": 93.02947998046875,
      "activations/layer3_attention_weight_min": -90.13074493408203,
      "activations/layer4_attention_weight_max": 94.33196258544922,
      "activations/layer4_attention_weight_min": -90.02650451660156,
      "activations/layer5_attention_weight_max": 69.283203125,
      "activations/layer5_attention_weight_min": -77.94132995605469,
      "activations/layer6_attention_weight_max": 51.83602523803711,
      "activations/layer6_attention_weight_min": -51.51719665527344,
      "activations/layer7_attention_weight_max": 69.2171859741211,
      "activations/layer7_attention_weight_min": -64.87461853027344,
      "activations/layer8_attention_weight_max": 43.211570739746094,
      "activations/layer8_attention_weight_min": -46.777713775634766,
      "activations/layer9_attention_weight_max": 44.85743713378906,
      "activations/layer9_attention_weight_min": -45.561336517333984,
      "epoch": 12.22,
      "learning_rate": 7.190454545454544e-05,
      "loss": 2.7923,
      "step": 210350
    },
    {
      "activations/layer0_attention_weight_max": 15.561412811279297,
      "activations/layer0_attention_weight_min": -14.124299049377441,
      "activations/layer10_attention_weight_max": 38.688106536865234,
      "activations/layer10_attention_weight_min": -36.729915618896484,
      "activations/layer11_attention_weight_max": 34.574127197265625,
      "activations/layer11_attention_weight_min": -34.054931640625,
      "activations/layer12_attention_weight_max": 18.857135772705078,
      "activations/layer12_attention_weight_min": -25.681949615478516,
      "activations/layer13_attention_weight_max": 62.36087417602539,
      "activations/layer13_attention_weight_min": -31.052379608154297,
      "activations/layer14_attention_weight_max": 51.38803482055664,
      "activations/layer14_attention_weight_min": -29.387813568115234,
      "activations/layer15_attention_weight_max": 42.86390686035156,
      "activations/layer15_attention_weight_min": -28.28234100341797,
      "activations/layer16_attention_weight_max": 29.716588973999023,
      "activations/layer16_attention_weight_min": -28.747100830078125,
      "activations/layer17_attention_weight_max": 51.81992721557617,
      "activations/layer17_attention_weight_min": -42.75088882446289,
      "activations/layer18_attention_weight_max": 45.04240417480469,
      "activations/layer18_attention_weight_min": -36.761287689208984,
      "activations/layer19_attention_weight_max": 20.918180465698242,
      "activations/layer19_attention_weight_min": -20.528532028198242,
      "activations/layer1_attention_weight_max": 17.896312713623047,
      "activations/layer1_attention_weight_min": -13.742663383483887,
      "activations/layer20_attention_weight_max": 20.725366592407227,
      "activations/layer20_attention_weight_min": -21.726594924926758,
      "activations/layer21_attention_weight_max": 34.60865020751953,
      "activations/layer21_attention_weight_min": -22.319910049438477,
      "activations/layer22_attention_weight_max": 27.85694122314453,
      "activations/layer22_attention_weight_min": -25.452484130859375,
      "activations/layer23_attention_weight_max": 34.50907897949219,
      "activations/layer23_attention_weight_min": -22.906095504760742,
      "activations/layer2_attention_weight_max": 33.510047912597656,
      "activations/layer2_attention_weight_min": -31.562049865722656,
      "activations/layer3_attention_weight_max": 99.32464599609375,
      "activations/layer3_attention_weight_min": -94.47814178466797,
      "activations/layer4_attention_weight_max": 102.16471099853516,
      "activations/layer4_attention_weight_min": -97.2601547241211,
      "activations/layer5_attention_weight_max": 75.59701538085938,
      "activations/layer5_attention_weight_min": -79.03059387207031,
      "activations/layer6_attention_weight_max": 58.40857696533203,
      "activations/layer6_attention_weight_min": -57.17426681518555,
      "activations/layer7_attention_weight_max": 71.05615234375,
      "activations/layer7_attention_weight_min": -65.57931518554688,
      "activations/layer8_attention_weight_max": 47.453712463378906,
      "activations/layer8_attention_weight_min": -53.2312126159668,
      "activations/layer9_attention_weight_max": 55.797760009765625,
      "activations/layer9_attention_weight_min": -49.18745803833008,
      "epoch": 12.23,
      "learning_rate": 7.188560606060606e-05,
      "loss": 2.7693,
      "step": 210400
    },
    {
      "activations/layer0_attention_weight_max": 15.744380950927734,
      "activations/layer0_attention_weight_min": -14.369295120239258,
      "activations/layer10_attention_weight_max": 37.9246711730957,
      "activations/layer10_attention_weight_min": -37.427921295166016,
      "activations/layer11_attention_weight_max": 34.79194641113281,
      "activations/layer11_attention_weight_min": -34.30378723144531,
      "activations/layer12_attention_weight_max": 19.099573135375977,
      "activations/layer12_attention_weight_min": -34.047142028808594,
      "activations/layer13_attention_weight_max": 42.14873504638672,
      "activations/layer13_attention_weight_min": -34.20328140258789,
      "activations/layer14_attention_weight_max": 38.00362014770508,
      "activations/layer14_attention_weight_min": -31.869726181030273,
      "activations/layer15_attention_weight_max": 35.48096466064453,
      "activations/layer15_attention_weight_min": -30.002107620239258,
      "activations/layer16_attention_weight_max": 32.66473388671875,
      "activations/layer16_attention_weight_min": -30.41190528869629,
      "activations/layer17_attention_weight_max": 51.67994689941406,
      "activations/layer17_attention_weight_min": -44.26205825805664,
      "activations/layer18_attention_weight_max": 46.96419143676758,
      "activations/layer18_attention_weight_min": -37.8919563293457,
      "activations/layer19_attention_weight_max": 24.38130760192871,
      "activations/layer19_attention_weight_min": -22.969486236572266,
      "activations/layer1_attention_weight_max": 16.75177764892578,
      "activations/layer1_attention_weight_min": -14.43608570098877,
      "activations/layer20_attention_weight_max": 22.792644500732422,
      "activations/layer20_attention_weight_min": -22.140148162841797,
      "activations/layer21_attention_weight_max": 42.58354568481445,
      "activations/layer21_attention_weight_min": -26.019187927246094,
      "activations/layer22_attention_weight_max": 30.42527961730957,
      "activations/layer22_attention_weight_min": -26.09156036376953,
      "activations/layer23_attention_weight_max": 41.50532531738281,
      "activations/layer23_attention_weight_min": -25.372007369995117,
      "activations/layer2_attention_weight_max": 32.14007568359375,
      "activations/layer2_attention_weight_min": -30.948108673095703,
      "activations/layer3_attention_weight_max": 94.9793930053711,
      "activations/layer3_attention_weight_min": -93.63046264648438,
      "activations/layer4_attention_weight_max": 98.59249877929688,
      "activations/layer4_attention_weight_min": -93.05648040771484,
      "activations/layer5_attention_weight_max": 71.33003997802734,
      "activations/layer5_attention_weight_min": -81.170166015625,
      "activations/layer6_attention_weight_max": 56.10179138183594,
      "activations/layer6_attention_weight_min": -53.393775939941406,
      "activations/layer7_attention_weight_max": 69.24613952636719,
      "activations/layer7_attention_weight_min": -71.48719024658203,
      "activations/layer8_attention_weight_max": 47.719505310058594,
      "activations/layer8_attention_weight_min": -48.96625518798828,
      "activations/layer9_attention_weight_max": 48.49406433105469,
      "activations/layer9_attention_weight_min": -50.841793060302734,
      "epoch": 12.23,
      "learning_rate": 7.186666666666666e-05,
      "loss": 2.7631,
      "step": 210450
    },
    {
      "activations/layer0_attention_weight_max": 15.794380187988281,
      "activations/layer0_attention_weight_min": -13.999614715576172,
      "activations/layer10_attention_weight_max": 36.62781524658203,
      "activations/layer10_attention_weight_min": -35.708984375,
      "activations/layer11_attention_weight_max": 35.96159362792969,
      "activations/layer11_attention_weight_min": -33.17591857910156,
      "activations/layer12_attention_weight_max": 17.3964786529541,
      "activations/layer12_attention_weight_min": -28.97361946105957,
      "activations/layer13_attention_weight_max": 40.32649612426758,
      "activations/layer13_attention_weight_min": -29.91307258605957,
      "activations/layer14_attention_weight_max": 40.34653854370117,
      "activations/layer14_attention_weight_min": -28.878231048583984,
      "activations/layer15_attention_weight_max": 34.99520492553711,
      "activations/layer15_attention_weight_min": -29.627779006958008,
      "activations/layer16_attention_weight_max": 32.21756362915039,
      "activations/layer16_attention_weight_min": -29.99437141418457,
      "activations/layer17_attention_weight_max": 51.69973373413086,
      "activations/layer17_attention_weight_min": -44.28302764892578,
      "activations/layer18_attention_weight_max": 49.54376220703125,
      "activations/layer18_attention_weight_min": -38.21721267700195,
      "activations/layer19_attention_weight_max": 22.006071090698242,
      "activations/layer19_attention_weight_min": -22.407655715942383,
      "activations/layer1_attention_weight_max": 17.664691925048828,
      "activations/layer1_attention_weight_min": -13.93191146850586,
      "activations/layer20_attention_weight_max": 21.65666961669922,
      "activations/layer20_attention_weight_min": -24.0612850189209,
      "activations/layer21_attention_weight_max": 38.353336334228516,
      "activations/layer21_attention_weight_min": -22.881954193115234,
      "activations/layer22_attention_weight_max": 31.536972045898438,
      "activations/layer22_attention_weight_min": -24.82367706298828,
      "activations/layer23_attention_weight_max": 33.866905212402344,
      "activations/layer23_attention_weight_min": -23.667926788330078,
      "activations/layer2_attention_weight_max": 31.76381492614746,
      "activations/layer2_attention_weight_min": -33.55529022216797,
      "activations/layer3_attention_weight_max": 97.63726043701172,
      "activations/layer3_attention_weight_min": -98.06236267089844,
      "activations/layer4_attention_weight_max": 99.22762298583984,
      "activations/layer4_attention_weight_min": -94.57640838623047,
      "activations/layer5_attention_weight_max": 72.35125732421875,
      "activations/layer5_attention_weight_min": -76.14765930175781,
      "activations/layer6_attention_weight_max": 53.84951400756836,
      "activations/layer6_attention_weight_min": -52.631874084472656,
      "activations/layer7_attention_weight_max": 76.1908187866211,
      "activations/layer7_attention_weight_min": -64.62847900390625,
      "activations/layer8_attention_weight_max": 47.82658004760742,
      "activations/layer8_attention_weight_min": -48.796791076660156,
      "activations/layer9_attention_weight_max": 49.163536071777344,
      "activations/layer9_attention_weight_min": -47.18403244018555,
      "epoch": 12.23,
      "learning_rate": 7.184772727272726e-05,
      "loss": 2.7643,
      "step": 210500
    },
    {
      "activations/layer0_attention_weight_max": 15.284671783447266,
      "activations/layer0_attention_weight_min": -14.196457862854004,
      "activations/layer10_attention_weight_max": 36.802955627441406,
      "activations/layer10_attention_weight_min": -36.00747299194336,
      "activations/layer11_attention_weight_max": 32.244102478027344,
      "activations/layer11_attention_weight_min": -32.6281852722168,
      "activations/layer12_attention_weight_max": 18.024505615234375,
      "activations/layer12_attention_weight_min": -24.781503677368164,
      "activations/layer13_attention_weight_max": 44.08422088623047,
      "activations/layer13_attention_weight_min": -31.135826110839844,
      "activations/layer14_attention_weight_max": 39.78130340576172,
      "activations/layer14_attention_weight_min": -30.598352432250977,
      "activations/layer15_attention_weight_max": 33.515647888183594,
      "activations/layer15_attention_weight_min": -29.917377471923828,
      "activations/layer16_attention_weight_max": 32.33083724975586,
      "activations/layer16_attention_weight_min": -30.51304054260254,
      "activations/layer17_attention_weight_max": 56.09247970581055,
      "activations/layer17_attention_weight_min": -45.63557815551758,
      "activations/layer18_attention_weight_max": 50.334659576416016,
      "activations/layer18_attention_weight_min": -38.38379669189453,
      "activations/layer19_attention_weight_max": 24.132802963256836,
      "activations/layer19_attention_weight_min": -21.405065536499023,
      "activations/layer1_attention_weight_max": 18.55059051513672,
      "activations/layer1_attention_weight_min": -14.857987403869629,
      "activations/layer20_attention_weight_max": 22.952619552612305,
      "activations/layer20_attention_weight_min": -21.39801788330078,
      "activations/layer21_attention_weight_max": 39.23435974121094,
      "activations/layer21_attention_weight_min": -21.8299560546875,
      "activations/layer22_attention_weight_max": 32.25962448120117,
      "activations/layer22_attention_weight_min": -25.42926025390625,
      "activations/layer23_attention_weight_max": 39.32503890991211,
      "activations/layer23_attention_weight_min": -23.42565155029297,
      "activations/layer2_attention_weight_max": 30.22689437866211,
      "activations/layer2_attention_weight_min": -30.588321685791016,
      "activations/layer3_attention_weight_max": 89.37818145751953,
      "activations/layer3_attention_weight_min": -92.77005004882812,
      "activations/layer4_attention_weight_max": 100.88534545898438,
      "activations/layer4_attention_weight_min": -97.4571762084961,
      "activations/layer5_attention_weight_max": 72.18425750732422,
      "activations/layer5_attention_weight_min": -81.29685974121094,
      "activations/layer6_attention_weight_max": 56.615943908691406,
      "activations/layer6_attention_weight_min": -55.93379211425781,
      "activations/layer7_attention_weight_max": 70.71275329589844,
      "activations/layer7_attention_weight_min": -67.1357421875,
      "activations/layer8_attention_weight_max": 46.743675231933594,
      "activations/layer8_attention_weight_min": -49.169189453125,
      "activations/layer9_attention_weight_max": 45.99165344238281,
      "activations/layer9_attention_weight_min": -46.81618118286133,
      "epoch": 12.23,
      "learning_rate": 7.182878787878788e-05,
      "loss": 2.7739,
      "step": 210550
    },
    {
      "activations/layer0_attention_weight_max": 16.201730728149414,
      "activations/layer0_attention_weight_min": -14.00290298461914,
      "activations/layer10_attention_weight_max": 34.3502197265625,
      "activations/layer10_attention_weight_min": -37.66876983642578,
      "activations/layer11_attention_weight_max": 32.88821792602539,
      "activations/layer11_attention_weight_min": -36.5844612121582,
      "activations/layer12_attention_weight_max": 19.069290161132812,
      "activations/layer12_attention_weight_min": -27.624467849731445,
      "activations/layer13_attention_weight_max": 39.08972930908203,
      "activations/layer13_attention_weight_min": -31.942028045654297,
      "activations/layer14_attention_weight_max": 37.01744842529297,
      "activations/layer14_attention_weight_min": -31.268901824951172,
      "activations/layer15_attention_weight_max": 34.18839645385742,
      "activations/layer15_attention_weight_min": -29.213693618774414,
      "activations/layer16_attention_weight_max": 35.54032897949219,
      "activations/layer16_attention_weight_min": -30.18300437927246,
      "activations/layer17_attention_weight_max": 53.76784133911133,
      "activations/layer17_attention_weight_min": -44.88003921508789,
      "activations/layer18_attention_weight_max": 46.80371856689453,
      "activations/layer18_attention_weight_min": -38.96067810058594,
      "activations/layer19_attention_weight_max": 20.961851119995117,
      "activations/layer19_attention_weight_min": -22.27574348449707,
      "activations/layer1_attention_weight_max": 18.177366256713867,
      "activations/layer1_attention_weight_min": -15.772856712341309,
      "activations/layer20_attention_weight_max": 22.21782112121582,
      "activations/layer20_attention_weight_min": -22.785837173461914,
      "activations/layer21_attention_weight_max": 41.790042877197266,
      "activations/layer21_attention_weight_min": -22.4346981048584,
      "activations/layer22_attention_weight_max": 28.177339553833008,
      "activations/layer22_attention_weight_min": -28.249061584472656,
      "activations/layer23_attention_weight_max": 35.955806732177734,
      "activations/layer23_attention_weight_min": -24.612783432006836,
      "activations/layer2_attention_weight_max": 33.03116226196289,
      "activations/layer2_attention_weight_min": -31.99927520751953,
      "activations/layer3_attention_weight_max": 97.34843444824219,
      "activations/layer3_attention_weight_min": -98.87741088867188,
      "activations/layer4_attention_weight_max": 99.33023834228516,
      "activations/layer4_attention_weight_min": -96.6095199584961,
      "activations/layer5_attention_weight_max": 73.62801361083984,
      "activations/layer5_attention_weight_min": -77.58878326416016,
      "activations/layer6_attention_weight_max": 52.10720443725586,
      "activations/layer6_attention_weight_min": -52.57624053955078,
      "activations/layer7_attention_weight_max": 68.2010269165039,
      "activations/layer7_attention_weight_min": -66.46158599853516,
      "activations/layer8_attention_weight_max": 44.613380432128906,
      "activations/layer8_attention_weight_min": -47.12099075317383,
      "activations/layer9_attention_weight_max": 50.85139083862305,
      "activations/layer9_attention_weight_min": -50.25208282470703,
      "epoch": 12.24,
      "learning_rate": 7.180984848484848e-05,
      "loss": 2.7721,
      "step": 210600
    },
    {
      "activations/layer0_attention_weight_max": 16.372514724731445,
      "activations/layer0_attention_weight_min": -14.408641815185547,
      "activations/layer10_attention_weight_max": 33.17138671875,
      "activations/layer10_attention_weight_min": -33.63481903076172,
      "activations/layer11_attention_weight_max": 29.155750274658203,
      "activations/layer11_attention_weight_min": -31.7983455657959,
      "activations/layer12_attention_weight_max": 17.571874618530273,
      "activations/layer12_attention_weight_min": -25.659194946289062,
      "activations/layer13_attention_weight_max": 36.03807830810547,
      "activations/layer13_attention_weight_min": -27.552852630615234,
      "activations/layer14_attention_weight_max": 35.28029251098633,
      "activations/layer14_attention_weight_min": -30.061866760253906,
      "activations/layer15_attention_weight_max": 32.235267639160156,
      "activations/layer15_attention_weight_min": -30.438798904418945,
      "activations/layer16_attention_weight_max": 31.882644653320312,
      "activations/layer16_attention_weight_min": -31.999608993530273,
      "activations/layer17_attention_weight_max": 53.42268371582031,
      "activations/layer17_attention_weight_min": -47.87771987915039,
      "activations/layer18_attention_weight_max": 44.57526397705078,
      "activations/layer18_attention_weight_min": -39.39315414428711,
      "activations/layer19_attention_weight_max": 26.503747940063477,
      "activations/layer19_attention_weight_min": -26.569782257080078,
      "activations/layer1_attention_weight_max": 16.617835998535156,
      "activations/layer1_attention_weight_min": -15.411582946777344,
      "activations/layer20_attention_weight_max": 22.52379608154297,
      "activations/layer20_attention_weight_min": -23.71146583557129,
      "activations/layer21_attention_weight_max": 35.3139533996582,
      "activations/layer21_attention_weight_min": -27.16856575012207,
      "activations/layer22_attention_weight_max": 30.095937728881836,
      "activations/layer22_attention_weight_min": -26.529178619384766,
      "activations/layer23_attention_weight_max": 34.08210754394531,
      "activations/layer23_attention_weight_min": -25.913639068603516,
      "activations/layer2_attention_weight_max": 32.07586669921875,
      "activations/layer2_attention_weight_min": -29.72907257080078,
      "activations/layer3_attention_weight_max": 90.80403137207031,
      "activations/layer3_attention_weight_min": -88.31161499023438,
      "activations/layer4_attention_weight_max": 91.66197967529297,
      "activations/layer4_attention_weight_min": -92.62481689453125,
      "activations/layer5_attention_weight_max": 68.1415023803711,
      "activations/layer5_attention_weight_min": -78.15071868896484,
      "activations/layer6_attention_weight_max": 53.17206573486328,
      "activations/layer6_attention_weight_min": -52.40218734741211,
      "activations/layer7_attention_weight_max": 65.30619049072266,
      "activations/layer7_attention_weight_min": -64.2055892944336,
      "activations/layer8_attention_weight_max": 43.21460723876953,
      "activations/layer8_attention_weight_min": -44.39059829711914,
      "activations/layer9_attention_weight_max": 46.41413497924805,
      "activations/layer9_attention_weight_min": -44.382755279541016,
      "epoch": 12.24,
      "learning_rate": 7.179090909090908e-05,
      "loss": 2.7793,
      "step": 210650
    },
    {
      "activations/layer0_attention_weight_max": 15.328317642211914,
      "activations/layer0_attention_weight_min": -14.130346298217773,
      "activations/layer10_attention_weight_max": 40.45896911621094,
      "activations/layer10_attention_weight_min": -36.580604553222656,
      "activations/layer11_attention_weight_max": 35.267189025878906,
      "activations/layer11_attention_weight_min": -33.33702850341797,
      "activations/layer12_attention_weight_max": 18.476673126220703,
      "activations/layer12_attention_weight_min": -23.61802101135254,
      "activations/layer13_attention_weight_max": 48.151710510253906,
      "activations/layer13_attention_weight_min": -33.02605438232422,
      "activations/layer14_attention_weight_max": 42.43299865722656,
      "activations/layer14_attention_weight_min": -31.481260299682617,
      "activations/layer15_attention_weight_max": 39.94525909423828,
      "activations/layer15_attention_weight_min": -30.508718490600586,
      "activations/layer16_attention_weight_max": 38.1866455078125,
      "activations/layer16_attention_weight_min": -30.832616806030273,
      "activations/layer17_attention_weight_max": 57.87776565551758,
      "activations/layer17_attention_weight_min": -46.50103759765625,
      "activations/layer18_attention_weight_max": 55.11904525756836,
      "activations/layer18_attention_weight_min": -40.57750701904297,
      "activations/layer19_attention_weight_max": 28.11917495727539,
      "activations/layer19_attention_weight_min": -24.74080467224121,
      "activations/layer1_attention_weight_max": 16.460033416748047,
      "activations/layer1_attention_weight_min": -15.874913215637207,
      "activations/layer20_attention_weight_max": 24.875164031982422,
      "activations/layer20_attention_weight_min": -20.956233978271484,
      "activations/layer21_attention_weight_max": 43.28084182739258,
      "activations/layer21_attention_weight_min": -23.263694763183594,
      "activations/layer22_attention_weight_max": 35.89173889160156,
      "activations/layer22_attention_weight_min": -27.401426315307617,
      "activations/layer23_attention_weight_max": 44.03651428222656,
      "activations/layer23_attention_weight_min": -23.741477966308594,
      "activations/layer2_attention_weight_max": 30.96224021911621,
      "activations/layer2_attention_weight_min": -31.187740325927734,
      "activations/layer3_attention_weight_max": 92.35063934326172,
      "activations/layer3_attention_weight_min": -90.43280029296875,
      "activations/layer4_attention_weight_max": 99.88442993164062,
      "activations/layer4_attention_weight_min": -96.30681610107422,
      "activations/layer5_attention_weight_max": 74.10175323486328,
      "activations/layer5_attention_weight_min": -79.51432800292969,
      "activations/layer6_attention_weight_max": 55.919219970703125,
      "activations/layer6_attention_weight_min": -53.287349700927734,
      "activations/layer7_attention_weight_max": 75.72351837158203,
      "activations/layer7_attention_weight_min": -68.33272552490234,
      "activations/layer8_attention_weight_max": 48.889183044433594,
      "activations/layer8_attention_weight_min": -49.958518981933594,
      "activations/layer9_attention_weight_max": 50.572601318359375,
      "activations/layer9_attention_weight_min": -48.133331298828125,
      "epoch": 12.24,
      "learning_rate": 7.17719696969697e-05,
      "loss": 2.7494,
      "step": 210700
    },
    {
      "activations/layer0_attention_weight_max": 16.127222061157227,
      "activations/layer0_attention_weight_min": -14.008732795715332,
      "activations/layer10_attention_weight_max": 37.280094146728516,
      "activations/layer10_attention_weight_min": -37.2663688659668,
      "activations/layer11_attention_weight_max": 32.08742141723633,
      "activations/layer11_attention_weight_min": -32.80234146118164,
      "activations/layer12_attention_weight_max": 17.656246185302734,
      "activations/layer12_attention_weight_min": -25.793434143066406,
      "activations/layer13_attention_weight_max": 36.467796325683594,
      "activations/layer13_attention_weight_min": -27.4464111328125,
      "activations/layer14_attention_weight_max": 34.84469223022461,
      "activations/layer14_attention_weight_min": -29.77714729309082,
      "activations/layer15_attention_weight_max": 31.716814041137695,
      "activations/layer15_attention_weight_min": -28.965625762939453,
      "activations/layer16_attention_weight_max": 29.794086456298828,
      "activations/layer16_attention_weight_min": -28.14654541015625,
      "activations/layer17_attention_weight_max": 49.698951721191406,
      "activations/layer17_attention_weight_min": -44.245277404785156,
      "activations/layer18_attention_weight_max": 44.29172134399414,
      "activations/layer18_attention_weight_min": -38.660560607910156,
      "activations/layer19_attention_weight_max": 23.18641471862793,
      "activations/layer19_attention_weight_min": -22.412860870361328,
      "activations/layer1_attention_weight_max": 16.688568115234375,
      "activations/layer1_attention_weight_min": -13.29945182800293,
      "activations/layer20_attention_weight_max": 21.917022705078125,
      "activations/layer20_attention_weight_min": -21.303491592407227,
      "activations/layer21_attention_weight_max": 36.879791259765625,
      "activations/layer21_attention_weight_min": -23.521371841430664,
      "activations/layer22_attention_weight_max": 29.5327205657959,
      "activations/layer22_attention_weight_min": -25.35091209411621,
      "activations/layer23_attention_weight_max": 39.47327423095703,
      "activations/layer23_attention_weight_min": -25.261493682861328,
      "activations/layer2_attention_weight_max": 30.88448715209961,
      "activations/layer2_attention_weight_min": -29.16012191772461,
      "activations/layer3_attention_weight_max": 91.69410705566406,
      "activations/layer3_attention_weight_min": -99.16127014160156,
      "activations/layer4_attention_weight_max": 98.79962921142578,
      "activations/layer4_attention_weight_min": -98.00669860839844,
      "activations/layer5_attention_weight_max": 70.60183715820312,
      "activations/layer5_attention_weight_min": -79.98818969726562,
      "activations/layer6_attention_weight_max": 53.69807052612305,
      "activations/layer6_attention_weight_min": -52.53987121582031,
      "activations/layer7_attention_weight_max": 69.27275848388672,
      "activations/layer7_attention_weight_min": -68.40011596679688,
      "activations/layer8_attention_weight_max": 44.621482849121094,
      "activations/layer8_attention_weight_min": -47.27678298950195,
      "activations/layer9_attention_weight_max": 46.13816833496094,
      "activations/layer9_attention_weight_min": -47.84880828857422,
      "epoch": 12.25,
      "learning_rate": 7.17530303030303e-05,
      "loss": 2.7627,
      "step": 210750
    },
    {
      "activations/layer0_attention_weight_max": 15.94079875946045,
      "activations/layer0_attention_weight_min": -14.197884559631348,
      "activations/layer10_attention_weight_max": 36.8033332824707,
      "activations/layer10_attention_weight_min": -34.92354202270508,
      "activations/layer11_attention_weight_max": 32.6011962890625,
      "activations/layer11_attention_weight_min": -32.12706756591797,
      "activations/layer12_attention_weight_max": 18.749094009399414,
      "activations/layer12_attention_weight_min": -24.147497177124023,
      "activations/layer13_attention_weight_max": 41.97634506225586,
      "activations/layer13_attention_weight_min": -32.99704360961914,
      "activations/layer14_attention_weight_max": 35.68701934814453,
      "activations/layer14_attention_weight_min": -30.583871841430664,
      "activations/layer15_attention_weight_max": 34.85927963256836,
      "activations/layer15_attention_weight_min": -29.40333366394043,
      "activations/layer16_attention_weight_max": 32.51182174682617,
      "activations/layer16_attention_weight_min": -31.255769729614258,
      "activations/layer17_attention_weight_max": 49.719722747802734,
      "activations/layer17_attention_weight_min": -45.332008361816406,
      "activations/layer18_attention_weight_max": 45.228797912597656,
      "activations/layer18_attention_weight_min": -38.60723114013672,
      "activations/layer19_attention_weight_max": 23.86626625061035,
      "activations/layer19_attention_weight_min": -24.174230575561523,
      "activations/layer1_attention_weight_max": 16.696651458740234,
      "activations/layer1_attention_weight_min": -15.712409019470215,
      "activations/layer20_attention_weight_max": 24.36155128479004,
      "activations/layer20_attention_weight_min": -25.506181716918945,
      "activations/layer21_attention_weight_max": 35.2046012878418,
      "activations/layer21_attention_weight_min": -25.479196548461914,
      "activations/layer22_attention_weight_max": 30.793338775634766,
      "activations/layer22_attention_weight_min": -27.532503128051758,
      "activations/layer23_attention_weight_max": 40.432044982910156,
      "activations/layer23_attention_weight_min": -28.88599395751953,
      "activations/layer2_attention_weight_max": 33.44585418701172,
      "activations/layer2_attention_weight_min": -31.404821395874023,
      "activations/layer3_attention_weight_max": 96.20673370361328,
      "activations/layer3_attention_weight_min": -96.08627319335938,
      "activations/layer4_attention_weight_max": 96.34458923339844,
      "activations/layer4_attention_weight_min": -94.13404083251953,
      "activations/layer5_attention_weight_max": 69.80430603027344,
      "activations/layer5_attention_weight_min": -81.02224731445312,
      "activations/layer6_attention_weight_max": 52.938541412353516,
      "activations/layer6_attention_weight_min": -54.98178482055664,
      "activations/layer7_attention_weight_max": 65.99726867675781,
      "activations/layer7_attention_weight_min": -66.51802825927734,
      "activations/layer8_attention_weight_max": 45.86355972290039,
      "activations/layer8_attention_weight_min": -47.59063720703125,
      "activations/layer9_attention_weight_max": 44.23617172241211,
      "activations/layer9_attention_weight_min": -47.59040451049805,
      "epoch": 12.25,
      "learning_rate": 7.17340909090909e-05,
      "loss": 2.7836,
      "step": 210800
    },
    {
      "activations/layer0_attention_weight_max": 16.53214454650879,
      "activations/layer0_attention_weight_min": -13.774333953857422,
      "activations/layer10_attention_weight_max": 40.12545394897461,
      "activations/layer10_attention_weight_min": -36.54397201538086,
      "activations/layer11_attention_weight_max": 33.885650634765625,
      "activations/layer11_attention_weight_min": -34.505470275878906,
      "activations/layer12_attention_weight_max": 21.123226165771484,
      "activations/layer12_attention_weight_min": -24.67498016357422,
      "activations/layer13_attention_weight_max": 45.469276428222656,
      "activations/layer13_attention_weight_min": -29.9700927734375,
      "activations/layer14_attention_weight_max": 39.7171745300293,
      "activations/layer14_attention_weight_min": -30.915016174316406,
      "activations/layer15_attention_weight_max": 35.84177780151367,
      "activations/layer15_attention_weight_min": -30.38684844970703,
      "activations/layer16_attention_weight_max": 35.15542984008789,
      "activations/layer16_attention_weight_min": -30.47324562072754,
      "activations/layer17_attention_weight_max": 53.59077835083008,
      "activations/layer17_attention_weight_min": -44.496944427490234,
      "activations/layer18_attention_weight_max": 49.46171951293945,
      "activations/layer18_attention_weight_min": -38.46757888793945,
      "activations/layer19_attention_weight_max": 23.67873764038086,
      "activations/layer19_attention_weight_min": -21.45218276977539,
      "activations/layer1_attention_weight_max": 15.911094665527344,
      "activations/layer1_attention_weight_min": -16.563308715820312,
      "activations/layer20_attention_weight_max": 23.09080696105957,
      "activations/layer20_attention_weight_min": -21.589143753051758,
      "activations/layer21_attention_weight_max": 36.993560791015625,
      "activations/layer21_attention_weight_min": -20.761240005493164,
      "activations/layer22_attention_weight_max": 32.039695739746094,
      "activations/layer22_attention_weight_min": -25.04118537902832,
      "activations/layer23_attention_weight_max": 41.86467361450195,
      "activations/layer23_attention_weight_min": -26.130428314208984,
      "activations/layer2_attention_weight_max": 32.955936431884766,
      "activations/layer2_attention_weight_min": -30.567113876342773,
      "activations/layer3_attention_weight_max": 88.89127349853516,
      "activations/layer3_attention_weight_min": -91.57262420654297,
      "activations/layer4_attention_weight_max": 97.50376892089844,
      "activations/layer4_attention_weight_min": -96.16022491455078,
      "activations/layer5_attention_weight_max": 70.4623794555664,
      "activations/layer5_attention_weight_min": -78.56767272949219,
      "activations/layer6_attention_weight_max": 55.327152252197266,
      "activations/layer6_attention_weight_min": -54.6547737121582,
      "activations/layer7_attention_weight_max": 77.41473388671875,
      "activations/layer7_attention_weight_min": -67.85633087158203,
      "activations/layer8_attention_weight_max": 46.82769012451172,
      "activations/layer8_attention_weight_min": -48.3218879699707,
      "activations/layer9_attention_weight_max": 48.10958480834961,
      "activations/layer9_attention_weight_min": -48.592342376708984,
      "epoch": 12.25,
      "learning_rate": 7.17151515151515e-05,
      "loss": 2.7713,
      "step": 210850
    },
    {
      "activations/layer0_attention_weight_max": 15.846672058105469,
      "activations/layer0_attention_weight_min": -13.887433052062988,
      "activations/layer10_attention_weight_max": 36.83417510986328,
      "activations/layer10_attention_weight_min": -34.96076583862305,
      "activations/layer11_attention_weight_max": 33.12592315673828,
      "activations/layer11_attention_weight_min": -32.38111114501953,
      "activations/layer12_attention_weight_max": 18.490570068359375,
      "activations/layer12_attention_weight_min": -27.248884201049805,
      "activations/layer13_attention_weight_max": 40.443965911865234,
      "activations/layer13_attention_weight_min": -30.230844497680664,
      "activations/layer14_attention_weight_max": 39.96583557128906,
      "activations/layer14_attention_weight_min": -30.068805694580078,
      "activations/layer15_attention_weight_max": 34.09794616699219,
      "activations/layer15_attention_weight_min": -29.448612213134766,
      "activations/layer16_attention_weight_max": 35.96519088745117,
      "activations/layer16_attention_weight_min": -33.13524627685547,
      "activations/layer17_attention_weight_max": 56.092796325683594,
      "activations/layer17_attention_weight_min": -47.67426681518555,
      "activations/layer18_attention_weight_max": 49.95491027832031,
      "activations/layer18_attention_weight_min": -39.73418426513672,
      "activations/layer19_attention_weight_max": 23.518884658813477,
      "activations/layer19_attention_weight_min": -24.688793182373047,
      "activations/layer1_attention_weight_max": 17.519649505615234,
      "activations/layer1_attention_weight_min": -15.22037410736084,
      "activations/layer20_attention_weight_max": 26.547679901123047,
      "activations/layer20_attention_weight_min": -21.27608299255371,
      "activations/layer21_attention_weight_max": 40.78642272949219,
      "activations/layer21_attention_weight_min": -23.755332946777344,
      "activations/layer22_attention_weight_max": 35.85580825805664,
      "activations/layer22_attention_weight_min": -27.306779861450195,
      "activations/layer23_attention_weight_max": 39.62504196166992,
      "activations/layer23_attention_weight_min": -24.837173461914062,
      "activations/layer2_attention_weight_max": 30.902311325073242,
      "activations/layer2_attention_weight_min": -30.499923706054688,
      "activations/layer3_attention_weight_max": 89.78607177734375,
      "activations/layer3_attention_weight_min": -93.08497619628906,
      "activations/layer4_attention_weight_max": 94.7542724609375,
      "activations/layer4_attention_weight_min": -92.68907165527344,
      "activations/layer5_attention_weight_max": 70.60600280761719,
      "activations/layer5_attention_weight_min": -74.81375885009766,
      "activations/layer6_attention_weight_max": 56.90980529785156,
      "activations/layer6_attention_weight_min": -51.09392166137695,
      "activations/layer7_attention_weight_max": 68.18107604980469,
      "activations/layer7_attention_weight_min": -63.36433792114258,
      "activations/layer8_attention_weight_max": 44.87436294555664,
      "activations/layer8_attention_weight_min": -43.94749069213867,
      "activations/layer9_attention_weight_max": 43.03666687011719,
      "activations/layer9_attention_weight_min": -45.848018646240234,
      "epoch": 12.25,
      "learning_rate": 7.169621212121211e-05,
      "loss": 2.7735,
      "step": 210900
    },
    {
      "activations/layer0_attention_weight_max": 15.405712127685547,
      "activations/layer0_attention_weight_min": -13.810276985168457,
      "activations/layer10_attention_weight_max": 36.21634292602539,
      "activations/layer10_attention_weight_min": -35.4702262878418,
      "activations/layer11_attention_weight_max": 31.735618591308594,
      "activations/layer11_attention_weight_min": -33.84226989746094,
      "activations/layer12_attention_weight_max": 17.19632339477539,
      "activations/layer12_attention_weight_min": -27.821266174316406,
      "activations/layer13_attention_weight_max": 35.754661560058594,
      "activations/layer13_attention_weight_min": -26.846208572387695,
      "activations/layer14_attention_weight_max": 36.96064758300781,
      "activations/layer14_attention_weight_min": -29.444211959838867,
      "activations/layer15_attention_weight_max": 31.260189056396484,
      "activations/layer15_attention_weight_min": -27.712522506713867,
      "activations/layer16_attention_weight_max": 32.77273941040039,
      "activations/layer16_attention_weight_min": -28.256858825683594,
      "activations/layer17_attention_weight_max": 48.45785140991211,
      "activations/layer17_attention_weight_min": -42.013404846191406,
      "activations/layer18_attention_weight_max": 45.07603073120117,
      "activations/layer18_attention_weight_min": -35.69693374633789,
      "activations/layer19_attention_weight_max": 22.000246047973633,
      "activations/layer19_attention_weight_min": -21.56438446044922,
      "activations/layer1_attention_weight_max": 16.774084091186523,
      "activations/layer1_attention_weight_min": -15.34102725982666,
      "activations/layer20_attention_weight_max": 24.755292892456055,
      "activations/layer20_attention_weight_min": -22.403337478637695,
      "activations/layer21_attention_weight_max": 39.210784912109375,
      "activations/layer21_attention_weight_min": -22.786420822143555,
      "activations/layer22_attention_weight_max": 37.72914505004883,
      "activations/layer22_attention_weight_min": -26.09654998779297,
      "activations/layer23_attention_weight_max": 44.70250701904297,
      "activations/layer23_attention_weight_min": -24.23688507080078,
      "activations/layer2_attention_weight_max": 32.6038932800293,
      "activations/layer2_attention_weight_min": -31.83984375,
      "activations/layer3_attention_weight_max": 96.17413330078125,
      "activations/layer3_attention_weight_min": -92.79766082763672,
      "activations/layer4_attention_weight_max": 96.94857788085938,
      "activations/layer4_attention_weight_min": -91.3091049194336,
      "activations/layer5_attention_weight_max": 67.50054931640625,
      "activations/layer5_attention_weight_min": -77.14305114746094,
      "activations/layer6_attention_weight_max": 50.32289123535156,
      "activations/layer6_attention_weight_min": -51.757694244384766,
      "activations/layer7_attention_weight_max": 66.0986328125,
      "activations/layer7_attention_weight_min": -65.7015609741211,
      "activations/layer8_attention_weight_max": 45.24506378173828,
      "activations/layer8_attention_weight_min": -44.24715805053711,
      "activations/layer9_attention_weight_max": 48.352108001708984,
      "activations/layer9_attention_weight_min": -45.163185119628906,
      "epoch": 12.26,
      "learning_rate": 7.167727272727272e-05,
      "loss": 2.7694,
      "step": 210950
    },
    {
      "activations/layer0_attention_weight_max": 15.642430305480957,
      "activations/layer0_attention_weight_min": -13.759279251098633,
      "activations/layer10_attention_weight_max": 36.609474182128906,
      "activations/layer10_attention_weight_min": -36.5792121887207,
      "activations/layer11_attention_weight_max": 35.0155029296875,
      "activations/layer11_attention_weight_min": -34.5907096862793,
      "activations/layer12_attention_weight_max": 19.213348388671875,
      "activations/layer12_attention_weight_min": -25.910375595092773,
      "activations/layer13_attention_weight_max": 40.25570297241211,
      "activations/layer13_attention_weight_min": -29.616249084472656,
      "activations/layer14_attention_weight_max": 37.55821990966797,
      "activations/layer14_attention_weight_min": -32.25714111328125,
      "activations/layer15_attention_weight_max": 36.80770492553711,
      "activations/layer15_attention_weight_min": -33.042259216308594,
      "activations/layer16_attention_weight_max": 32.26840591430664,
      "activations/layer16_attention_weight_min": -29.47142219543457,
      "activations/layer17_attention_weight_max": 53.04450607299805,
      "activations/layer17_attention_weight_min": -43.902122497558594,
      "activations/layer18_attention_weight_max": 47.564884185791016,
      "activations/layer18_attention_weight_min": -39.57427978515625,
      "activations/layer19_attention_weight_max": 24.24881362915039,
      "activations/layer19_attention_weight_min": -23.464374542236328,
      "activations/layer1_attention_weight_max": 16.68709373474121,
      "activations/layer1_attention_weight_min": -15.526644706726074,
      "activations/layer20_attention_weight_max": 28.322187423706055,
      "activations/layer20_attention_weight_min": -20.92026138305664,
      "activations/layer21_attention_weight_max": 46.970603942871094,
      "activations/layer21_attention_weight_min": -26.737504959106445,
      "activations/layer22_attention_weight_max": 32.04424285888672,
      "activations/layer22_attention_weight_min": -26.156843185424805,
      "activations/layer23_attention_weight_max": 42.324974060058594,
      "activations/layer23_attention_weight_min": -25.623271942138672,
      "activations/layer2_attention_weight_max": 34.977447509765625,
      "activations/layer2_attention_weight_min": -32.4957275390625,
      "activations/layer3_attention_weight_max": 97.24305725097656,
      "activations/layer3_attention_weight_min": -96.40848541259766,
      "activations/layer4_attention_weight_max": 104.63175201416016,
      "activations/layer4_attention_weight_min": -98.28353118896484,
      "activations/layer5_attention_weight_max": 74.68302154541016,
      "activations/layer5_attention_weight_min": -80.79547882080078,
      "activations/layer6_attention_weight_max": 56.4473876953125,
      "activations/layer6_attention_weight_min": -57.47008514404297,
      "activations/layer7_attention_weight_max": 71.04306030273438,
      "activations/layer7_attention_weight_min": -72.3756332397461,
      "activations/layer8_attention_weight_max": 46.08524703979492,
      "activations/layer8_attention_weight_min": -50.27907180786133,
      "activations/layer9_attention_weight_max": 49.06730651855469,
      "activations/layer9_attention_weight_min": -49.03446578979492,
      "epoch": 12.26,
      "learning_rate": 7.165833333333333e-05,
      "loss": 2.7718,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.4657,
      "eval_samples_per_second": 507.221,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.4657,
      "eval_openwebtext_samples_per_second": 507.221,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 1.9869,
      "eval_wikitext_samples_per_second": 229.507,
      "step": 211000
    },
    {
      "epoch": 12.26,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.5687,
      "eval_lambada_samples_per_second": 508.845,
      "step": 211000
    },
    {
      "activations/layer0_attention_weight_max": 15.75216007232666,
      "activations/layer0_attention_weight_min": -13.873744010925293,
      "activations/layer10_attention_weight_max": 41.22039794921875,
      "activations/layer10_attention_weight_min": -38.82171630859375,
      "activations/layer11_attention_weight_max": 36.37681579589844,
      "activations/layer11_attention_weight_min": -35.33210372924805,
      "activations/layer12_attention_weight_max": 17.621807098388672,
      "activations/layer12_attention_weight_min": -23.625064849853516,
      "activations/layer13_attention_weight_max": 52.84070587158203,
      "activations/layer13_attention_weight_min": -32.2514762878418,
      "activations/layer14_attention_weight_max": 50.604827880859375,
      "activations/layer14_attention_weight_min": -33.892845153808594,
      "activations/layer15_attention_weight_max": 41.63231658935547,
      "activations/layer15_attention_weight_min": -30.2081241607666,
      "activations/layer16_attention_weight_max": 32.89854431152344,
      "activations/layer16_attention_weight_min": -31.73316192626953,
      "activations/layer17_attention_weight_max": 56.61900329589844,
      "activations/layer17_attention_weight_min": -46.03506851196289,
      "activations/layer18_attention_weight_max": 47.676231384277344,
      "activations/layer18_attention_weight_min": -40.170654296875,
      "activations/layer19_attention_weight_max": 21.902862548828125,
      "activations/layer19_attention_weight_min": -22.64162254333496,
      "activations/layer1_attention_weight_max": 16.576683044433594,
      "activations/layer1_attention_weight_min": -18.160507202148438,
      "activations/layer20_attention_weight_max": 23.813194274902344,
      "activations/layer20_attention_weight_min": -22.564245223999023,
      "activations/layer21_attention_weight_max": 39.387237548828125,
      "activations/layer21_attention_weight_min": -25.05051040649414,
      "activations/layer22_attention_weight_max": 31.327335357666016,
      "activations/layer22_attention_weight_min": -26.727170944213867,
      "activations/layer23_attention_weight_max": 42.98545837402344,
      "activations/layer23_attention_weight_min": -24.714797973632812,
      "activations/layer2_attention_weight_max": 34.05820846557617,
      "activations/layer2_attention_weight_min": -31.689064025878906,
      "activations/layer3_attention_weight_max": 100.04975891113281,
      "activations/layer3_attention_weight_min": -96.43929290771484,
      "activations/layer4_attention_weight_max": 102.05709075927734,
      "activations/layer4_attention_weight_min": -98.1002426147461,
      "activations/layer5_attention_weight_max": 79.66032409667969,
      "activations/layer5_attention_weight_min": -80.42689514160156,
      "activations/layer6_attention_weight_max": 55.514137268066406,
      "activations/layer6_attention_weight_min": -56.03948974609375,
      "activations/layer7_attention_weight_max": 78.63563537597656,
      "activations/layer7_attention_weight_min": -70.8851089477539,
      "activations/layer8_attention_weight_max": 51.3309211730957,
      "activations/layer8_attention_weight_min": -51.78318405151367,
      "activations/layer9_attention_weight_max": 51.63393783569336,
      "activations/layer9_attention_weight_min": -48.42932891845703,
      "epoch": 12.26,
      "learning_rate": 7.163939393939393e-05,
      "loss": 2.7787,
      "step": 211050
    },
    {
      "activations/layer0_attention_weight_max": 15.312093734741211,
      "activations/layer0_attention_weight_min": -14.148393630981445,
      "activations/layer10_attention_weight_max": 36.62696838378906,
      "activations/layer10_attention_weight_min": -35.61349868774414,
      "activations/layer11_attention_weight_max": 31.259550094604492,
      "activations/layer11_attention_weight_min": -34.18876647949219,
      "activations/layer12_attention_weight_max": 18.084598541259766,
      "activations/layer12_attention_weight_min": -25.366897583007812,
      "activations/layer13_attention_weight_max": 33.14623260498047,
      "activations/layer13_attention_weight_min": -31.912067413330078,
      "activations/layer14_attention_weight_max": 36.01945877075195,
      "activations/layer14_attention_weight_min": -32.0348014831543,
      "activations/layer15_attention_weight_max": 33.14916229248047,
      "activations/layer15_attention_weight_min": -32.14594650268555,
      "activations/layer16_attention_weight_max": 34.0971565246582,
      "activations/layer16_attention_weight_min": -31.169559478759766,
      "activations/layer17_attention_weight_max": 53.40768814086914,
      "activations/layer17_attention_weight_min": -45.64485168457031,
      "activations/layer18_attention_weight_max": 44.66716766357422,
      "activations/layer18_attention_weight_min": -37.05143356323242,
      "activations/layer19_attention_weight_max": 20.3855037689209,
      "activations/layer19_attention_weight_min": -20.08360481262207,
      "activations/layer1_attention_weight_max": 16.148141860961914,
      "activations/layer1_attention_weight_min": -15.386932373046875,
      "activations/layer20_attention_weight_max": 21.284008026123047,
      "activations/layer20_attention_weight_min": -22.051267623901367,
      "activations/layer21_attention_weight_max": 30.756975173950195,
      "activations/layer21_attention_weight_min": -20.84018325805664,
      "activations/layer22_attention_weight_max": 27.9671630859375,
      "activations/layer22_attention_weight_min": -26.320873260498047,
      "activations/layer23_attention_weight_max": 34.257816314697266,
      "activations/layer23_attention_weight_min": -23.489166259765625,
      "activations/layer2_attention_weight_max": 32.747249603271484,
      "activations/layer2_attention_weight_min": -31.438312530517578,
      "activations/layer3_attention_weight_max": 96.9090805053711,
      "activations/layer3_attention_weight_min": -100.04840087890625,
      "activations/layer4_attention_weight_max": 98.55742645263672,
      "activations/layer4_attention_weight_min": -95.72189331054688,
      "activations/layer5_attention_weight_max": 70.31230926513672,
      "activations/layer5_attention_weight_min": -79.939453125,
      "activations/layer6_attention_weight_max": 54.9100227355957,
      "activations/layer6_attention_weight_min": -54.0273323059082,
      "activations/layer7_attention_weight_max": 73.68351745605469,
      "activations/layer7_attention_weight_min": -65.84086608886719,
      "activations/layer8_attention_weight_max": 46.37926483154297,
      "activations/layer8_attention_weight_min": -49.063297271728516,
      "activations/layer9_attention_weight_max": 49.61616516113281,
      "activations/layer9_attention_weight_min": -50.456233978271484,
      "epoch": 12.27,
      "learning_rate": 7.162045454545455e-05,
      "loss": 2.7619,
      "step": 211100
    },
    {
      "activations/layer0_attention_weight_max": 15.835015296936035,
      "activations/layer0_attention_weight_min": -13.990253448486328,
      "activations/layer10_attention_weight_max": 36.758544921875,
      "activations/layer10_attention_weight_min": -34.23784255981445,
      "activations/layer11_attention_weight_max": 33.09801483154297,
      "activations/layer11_attention_weight_min": -33.583763122558594,
      "activations/layer12_attention_weight_max": 19.37633514404297,
      "activations/layer12_attention_weight_min": -27.396207809448242,
      "activations/layer13_attention_weight_max": 41.73088836669922,
      "activations/layer13_attention_weight_min": -35.36332702636719,
      "activations/layer14_attention_weight_max": 39.913291931152344,
      "activations/layer14_attention_weight_min": -30.11183738708496,
      "activations/layer15_attention_weight_max": 34.338043212890625,
      "activations/layer15_attention_weight_min": -30.278606414794922,
      "activations/layer16_attention_weight_max": 32.67869186401367,
      "activations/layer16_attention_weight_min": -29.4829044342041,
      "activations/layer17_attention_weight_max": 53.48410415649414,
      "activations/layer17_attention_weight_min": -44.94700622558594,
      "activations/layer18_attention_weight_max": 51.364437103271484,
      "activations/layer18_attention_weight_min": -38.3774299621582,
      "activations/layer19_attention_weight_max": 22.999929428100586,
      "activations/layer19_attention_weight_min": -22.239946365356445,
      "activations/layer1_attention_weight_max": 16.423311233520508,
      "activations/layer1_attention_weight_min": -15.171626091003418,
      "activations/layer20_attention_weight_max": 24.181297302246094,
      "activations/layer20_attention_weight_min": -21.69441032409668,
      "activations/layer21_attention_weight_max": 40.8680305480957,
      "activations/layer21_attention_weight_min": -29.89408302307129,
      "activations/layer22_attention_weight_max": 31.17725372314453,
      "activations/layer22_attention_weight_min": -27.96727752685547,
      "activations/layer23_attention_weight_max": 38.600650787353516,
      "activations/layer23_attention_weight_min": -26.920124053955078,
      "activations/layer2_attention_weight_max": 32.88990783691406,
      "activations/layer2_attention_weight_min": -32.7695198059082,
      "activations/layer3_attention_weight_max": 91.9732666015625,
      "activations/layer3_attention_weight_min": -101.49424743652344,
      "activations/layer4_attention_weight_max": 97.9809799194336,
      "activations/layer4_attention_weight_min": -94.42137145996094,
      "activations/layer5_attention_weight_max": 75.07975769042969,
      "activations/layer5_attention_weight_min": -79.06809997558594,
      "activations/layer6_attention_weight_max": 53.01325607299805,
      "activations/layer6_attention_weight_min": -51.81167984008789,
      "activations/layer7_attention_weight_max": 68.02225494384766,
      "activations/layer7_attention_weight_min": -64.54769134521484,
      "activations/layer8_attention_weight_max": 45.6463508605957,
      "activations/layer8_attention_weight_min": -46.82941436767578,
      "activations/layer9_attention_weight_max": 46.74528884887695,
      "activations/layer9_attention_weight_min": -44.92805099487305,
      "epoch": 12.27,
      "learning_rate": 7.160151515151515e-05,
      "loss": 2.7724,
      "step": 211150
    },
    {
      "activations/layer0_attention_weight_max": 16.296955108642578,
      "activations/layer0_attention_weight_min": -14.304926872253418,
      "activations/layer10_attention_weight_max": 44.29117202758789,
      "activations/layer10_attention_weight_min": -41.203617095947266,
      "activations/layer11_attention_weight_max": 39.12708282470703,
      "activations/layer11_attention_weight_min": -35.41778564453125,
      "activations/layer12_attention_weight_max": 18.002613067626953,
      "activations/layer12_attention_weight_min": -23.845014572143555,
      "activations/layer13_attention_weight_max": 53.510257720947266,
      "activations/layer13_attention_weight_min": -31.57248878479004,
      "activations/layer14_attention_weight_max": 49.186553955078125,
      "activations/layer14_attention_weight_min": -31.349475860595703,
      "activations/layer15_attention_weight_max": 45.72565841674805,
      "activations/layer15_attention_weight_min": -28.370838165283203,
      "activations/layer16_attention_weight_max": 30.328079223632812,
      "activations/layer16_attention_weight_min": -28.682464599609375,
      "activations/layer17_attention_weight_max": 51.046932220458984,
      "activations/layer17_attention_weight_min": -42.0549201965332,
      "activations/layer18_attention_weight_max": 45.24186325073242,
      "activations/layer18_attention_weight_min": -36.63544845581055,
      "activations/layer19_attention_weight_max": 20.671472549438477,
      "activations/layer19_attention_weight_min": -20.525285720825195,
      "activations/layer1_attention_weight_max": 17.17633819580078,
      "activations/layer1_attention_weight_min": -18.451879501342773,
      "activations/layer20_attention_weight_max": 22.47086524963379,
      "activations/layer20_attention_weight_min": -24.045255661010742,
      "activations/layer21_attention_weight_max": 36.44868469238281,
      "activations/layer21_attention_weight_min": -23.7375545501709,
      "activations/layer22_attention_weight_max": 30.20835304260254,
      "activations/layer22_attention_weight_min": -25.67087173461914,
      "activations/layer23_attention_weight_max": 37.47703170776367,
      "activations/layer23_attention_weight_min": -24.712909698486328,
      "activations/layer2_attention_weight_max": 32.39118957519531,
      "activations/layer2_attention_weight_min": -31.700284957885742,
      "activations/layer3_attention_weight_max": 95.86199188232422,
      "activations/layer3_attention_weight_min": -97.1299819946289,
      "activations/layer4_attention_weight_max": 102.33580780029297,
      "activations/layer4_attention_weight_min": -97.74462127685547,
      "activations/layer5_attention_weight_max": 72.3652114868164,
      "activations/layer5_attention_weight_min": -81.59341430664062,
      "activations/layer6_attention_weight_max": 54.77521514892578,
      "activations/layer6_attention_weight_min": -52.985843658447266,
      "activations/layer7_attention_weight_max": 80.61017608642578,
      "activations/layer7_attention_weight_min": -67.90083312988281,
      "activations/layer8_attention_weight_max": 54.246681213378906,
      "activations/layer8_attention_weight_min": -47.66021728515625,
      "activations/layer9_attention_weight_max": 62.36269760131836,
      "activations/layer9_attention_weight_min": -51.868892669677734,
      "epoch": 12.27,
      "learning_rate": 7.158257575757575e-05,
      "loss": 2.7512,
      "step": 211200
    },
    {
      "activations/layer0_attention_weight_max": 16.023345947265625,
      "activations/layer0_attention_weight_min": -13.93974494934082,
      "activations/layer10_attention_weight_max": 39.661346435546875,
      "activations/layer10_attention_weight_min": -39.147186279296875,
      "activations/layer11_attention_weight_max": 36.036224365234375,
      "activations/layer11_attention_weight_min": -33.97071838378906,
      "activations/layer12_attention_weight_max": 20.379560470581055,
      "activations/layer12_attention_weight_min": -24.14052963256836,
      "activations/layer13_attention_weight_max": 37.74507522583008,
      "activations/layer13_attention_weight_min": -29.925167083740234,
      "activations/layer14_attention_weight_max": 36.637123107910156,
      "activations/layer14_attention_weight_min": -33.40753173828125,
      "activations/layer15_attention_weight_max": 34.777381896972656,
      "activations/layer15_attention_weight_min": -32.70339584350586,
      "activations/layer16_attention_weight_max": 36.182125091552734,
      "activations/layer16_attention_weight_min": -32.63507843017578,
      "activations/layer17_attention_weight_max": 57.65240478515625,
      "activations/layer17_attention_weight_min": -51.40164566040039,
      "activations/layer18_attention_weight_max": 52.9848747253418,
      "activations/layer18_attention_weight_min": -42.83372497558594,
      "activations/layer19_attention_weight_max": 23.361257553100586,
      "activations/layer19_attention_weight_min": -23.185495376586914,
      "activations/layer1_attention_weight_max": 16.275423049926758,
      "activations/layer1_attention_weight_min": -15.822643280029297,
      "activations/layer20_attention_weight_max": 23.609529495239258,
      "activations/layer20_attention_weight_min": -22.626510620117188,
      "activations/layer21_attention_weight_max": 40.9012336730957,
      "activations/layer21_attention_weight_min": -22.329090118408203,
      "activations/layer22_attention_weight_max": 35.87757873535156,
      "activations/layer22_attention_weight_min": -27.310564041137695,
      "activations/layer23_attention_weight_max": 43.10258483886719,
      "activations/layer23_attention_weight_min": -23.212093353271484,
      "activations/layer2_attention_weight_max": 33.279579162597656,
      "activations/layer2_attention_weight_min": -32.60710906982422,
      "activations/layer3_attention_weight_max": 93.76415252685547,
      "activations/layer3_attention_weight_min": -97.54557037353516,
      "activations/layer4_attention_weight_max": 101.59243774414062,
      "activations/layer4_attention_weight_min": -97.21776580810547,
      "activations/layer5_attention_weight_max": 69.00618743896484,
      "activations/layer5_attention_weight_min": -81.54257202148438,
      "activations/layer6_attention_weight_max": 52.17726516723633,
      "activations/layer6_attention_weight_min": -52.72611999511719,
      "activations/layer7_attention_weight_max": 68.70532989501953,
      "activations/layer7_attention_weight_min": -73.00279998779297,
      "activations/layer8_attention_weight_max": 48.95347595214844,
      "activations/layer8_attention_weight_min": -47.0526123046875,
      "activations/layer9_attention_weight_max": 55.80292892456055,
      "activations/layer9_attention_weight_min": -49.51224899291992,
      "epoch": 12.27,
      "learning_rate": 7.156363636363635e-05,
      "loss": 2.781,
      "step": 211250
    },
    {
      "activations/layer0_attention_weight_max": 15.2304105758667,
      "activations/layer0_attention_weight_min": -14.269646644592285,
      "activations/layer10_attention_weight_max": 37.292999267578125,
      "activations/layer10_attention_weight_min": -37.02216720581055,
      "activations/layer11_attention_weight_max": 31.75495147705078,
      "activations/layer11_attention_weight_min": -33.10914993286133,
      "activations/layer12_attention_weight_max": 17.08334732055664,
      "activations/layer12_attention_weight_min": -24.38800621032715,
      "activations/layer13_attention_weight_max": 36.679298400878906,
      "activations/layer13_attention_weight_min": -27.65769386291504,
      "activations/layer14_attention_weight_max": 35.48664093017578,
      "activations/layer14_attention_weight_min": -29.56879234313965,
      "activations/layer15_attention_weight_max": 30.57189178466797,
      "activations/layer15_attention_weight_min": -28.992416381835938,
      "activations/layer16_attention_weight_max": 29.21611785888672,
      "activations/layer16_attention_weight_min": -29.12858772277832,
      "activations/layer17_attention_weight_max": 48.02296829223633,
      "activations/layer17_attention_weight_min": -41.856590270996094,
      "activations/layer18_attention_weight_max": 42.528724670410156,
      "activations/layer18_attention_weight_min": -35.26704406738281,
      "activations/layer19_attention_weight_max": 22.046655654907227,
      "activations/layer19_attention_weight_min": -21.90489959716797,
      "activations/layer1_attention_weight_max": 16.848508834838867,
      "activations/layer1_attention_weight_min": -15.280553817749023,
      "activations/layer20_attention_weight_max": 21.47756004333496,
      "activations/layer20_attention_weight_min": -21.0961856842041,
      "activations/layer21_attention_weight_max": 33.95061111450195,
      "activations/layer21_attention_weight_min": -22.504711151123047,
      "activations/layer22_attention_weight_max": 28.465303421020508,
      "activations/layer22_attention_weight_min": -24.131338119506836,
      "activations/layer23_attention_weight_max": 36.489776611328125,
      "activations/layer23_attention_weight_min": -29.065120697021484,
      "activations/layer2_attention_weight_max": 33.93006134033203,
      "activations/layer2_attention_weight_min": -34.033878326416016,
      "activations/layer3_attention_weight_max": 96.34642791748047,
      "activations/layer3_attention_weight_min": -99.77201843261719,
      "activations/layer4_attention_weight_max": 103.5964126586914,
      "activations/layer4_attention_weight_min": -96.64946746826172,
      "activations/layer5_attention_weight_max": 70.90400695800781,
      "activations/layer5_attention_weight_min": -80.37767028808594,
      "activations/layer6_attention_weight_max": 56.16530227661133,
      "activations/layer6_attention_weight_min": -54.47602844238281,
      "activations/layer7_attention_weight_max": 71.36258697509766,
      "activations/layer7_attention_weight_min": -66.12944793701172,
      "activations/layer8_attention_weight_max": 47.42893981933594,
      "activations/layer8_attention_weight_min": -47.42654037475586,
      "activations/layer9_attention_weight_max": 47.09866714477539,
      "activations/layer9_attention_weight_min": -47.304378509521484,
      "epoch": 12.28,
      "learning_rate": 7.154469696969697e-05,
      "loss": 2.7922,
      "step": 211300
    },
    {
      "activations/layer0_attention_weight_max": 15.77546215057373,
      "activations/layer0_attention_weight_min": -14.107705116271973,
      "activations/layer10_attention_weight_max": 40.363346099853516,
      "activations/layer10_attention_weight_min": -36.98283767700195,
      "activations/layer11_attention_weight_max": 36.08409881591797,
      "activations/layer11_attention_weight_min": -35.56053161621094,
      "activations/layer12_attention_weight_max": 20.30570411682129,
      "activations/layer12_attention_weight_min": -27.71935272216797,
      "activations/layer13_attention_weight_max": 61.52214050292969,
      "activations/layer13_attention_weight_min": -37.726539611816406,
      "activations/layer14_attention_weight_max": 49.13111114501953,
      "activations/layer14_attention_weight_min": -31.01109504699707,
      "activations/layer15_attention_weight_max": 39.21831130981445,
      "activations/layer15_attention_weight_min": -29.649988174438477,
      "activations/layer16_attention_weight_max": 36.00212478637695,
      "activations/layer16_attention_weight_min": -30.201135635375977,
      "activations/layer17_attention_weight_max": 55.78007507324219,
      "activations/layer17_attention_weight_min": -44.93706130981445,
      "activations/layer18_attention_weight_max": 51.36282730102539,
      "activations/layer18_attention_weight_min": -36.54130935668945,
      "activations/layer19_attention_weight_max": 25.6922550201416,
      "activations/layer19_attention_weight_min": -20.64542579650879,
      "activations/layer1_attention_weight_max": 15.932456016540527,
      "activations/layer1_attention_weight_min": -15.879258155822754,
      "activations/layer20_attention_weight_max": 27.0915470123291,
      "activations/layer20_attention_weight_min": -21.680057525634766,
      "activations/layer21_attention_weight_max": 39.25496292114258,
      "activations/layer21_attention_weight_min": -23.309890747070312,
      "activations/layer22_attention_weight_max": 32.724544525146484,
      "activations/layer22_attention_weight_min": -23.773361206054688,
      "activations/layer23_attention_weight_max": 41.951072692871094,
      "activations/layer23_attention_weight_min": -22.42561912536621,
      "activations/layer2_attention_weight_max": 32.470401763916016,
      "activations/layer2_attention_weight_min": -32.8399543762207,
      "activations/layer3_attention_weight_max": 96.57295227050781,
      "activations/layer3_attention_weight_min": -101.5898666381836,
      "activations/layer4_attention_weight_max": 103.6762466430664,
      "activations/layer4_attention_weight_min": -100.94770812988281,
      "activations/layer5_attention_weight_max": 76.59539794921875,
      "activations/layer5_attention_weight_min": -80.66495513916016,
      "activations/layer6_attention_weight_max": 54.86972427368164,
      "activations/layer6_attention_weight_min": -58.43016815185547,
      "activations/layer7_attention_weight_max": 80.80863952636719,
      "activations/layer7_attention_weight_min": -72.32425689697266,
      "activations/layer8_attention_weight_max": 48.30377197265625,
      "activations/layer8_attention_weight_min": -51.00900650024414,
      "activations/layer9_attention_weight_max": 50.3416748046875,
      "activations/layer9_attention_weight_min": -47.010704040527344,
      "epoch": 12.28,
      "learning_rate": 7.152575757575757e-05,
      "loss": 2.755,
      "step": 211350
    },
    {
      "activations/layer0_attention_weight_max": 15.329706192016602,
      "activations/layer0_attention_weight_min": -13.931159019470215,
      "activations/layer10_attention_weight_max": 38.06938171386719,
      "activations/layer10_attention_weight_min": -35.44966506958008,
      "activations/layer11_attention_weight_max": 35.239376068115234,
      "activations/layer11_attention_weight_min": -32.39283752441406,
      "activations/layer12_attention_weight_max": 16.75946807861328,
      "activations/layer12_attention_weight_min": -24.646854400634766,
      "activations/layer13_attention_weight_max": 44.55967330932617,
      "activations/layer13_attention_weight_min": -32.08210372924805,
      "activations/layer14_attention_weight_max": 43.098148345947266,
      "activations/layer14_attention_weight_min": -31.627504348754883,
      "activations/layer15_attention_weight_max": 34.261104583740234,
      "activations/layer15_attention_weight_min": -30.762699127197266,
      "activations/layer16_attention_weight_max": 32.37148666381836,
      "activations/layer16_attention_weight_min": -30.57375144958496,
      "activations/layer17_attention_weight_max": 52.68818283081055,
      "activations/layer17_attention_weight_min": -44.202110290527344,
      "activations/layer18_attention_weight_max": 47.78460693359375,
      "activations/layer18_attention_weight_min": -38.31444549560547,
      "activations/layer19_attention_weight_max": 22.91061019897461,
      "activations/layer19_attention_weight_min": -22.9550838470459,
      "activations/layer1_attention_weight_max": 17.14219856262207,
      "activations/layer1_attention_weight_min": -15.307110786437988,
      "activations/layer20_attention_weight_max": 22.460739135742188,
      "activations/layer20_attention_weight_min": -21.709230422973633,
      "activations/layer21_attention_weight_max": 40.831180572509766,
      "activations/layer21_attention_weight_min": -24.86502456665039,
      "activations/layer22_attention_weight_max": 30.855560302734375,
      "activations/layer22_attention_weight_min": -25.489521026611328,
      "activations/layer23_attention_weight_max": 40.92164993286133,
      "activations/layer23_attention_weight_min": -26.817256927490234,
      "activations/layer2_attention_weight_max": 34.173973083496094,
      "activations/layer2_attention_weight_min": -31.096675872802734,
      "activations/layer3_attention_weight_max": 96.99368286132812,
      "activations/layer3_attention_weight_min": -96.8788070678711,
      "activations/layer4_attention_weight_max": 97.80430603027344,
      "activations/layer4_attention_weight_min": -93.16838073730469,
      "activations/layer5_attention_weight_max": 73.19207763671875,
      "activations/layer5_attention_weight_min": -80.15271759033203,
      "activations/layer6_attention_weight_max": 56.2241096496582,
      "activations/layer6_attention_weight_min": -51.341556549072266,
      "activations/layer7_attention_weight_max": 69.69203186035156,
      "activations/layer7_attention_weight_min": -65.09320831298828,
      "activations/layer8_attention_weight_max": 46.03560256958008,
      "activations/layer8_attention_weight_min": -46.831504821777344,
      "activations/layer9_attention_weight_max": 45.28333282470703,
      "activations/layer9_attention_weight_min": -45.98301315307617,
      "epoch": 12.28,
      "learning_rate": 7.150681818181817e-05,
      "loss": 2.7655,
      "step": 211400
    },
    {
      "activations/layer0_attention_weight_max": 16.605640411376953,
      "activations/layer0_attention_weight_min": -13.907103538513184,
      "activations/layer10_attention_weight_max": 39.6445198059082,
      "activations/layer10_attention_weight_min": -38.27444839477539,
      "activations/layer11_attention_weight_max": 34.103904724121094,
      "activations/layer11_attention_weight_min": -32.82047653198242,
      "activations/layer12_attention_weight_max": 19.948820114135742,
      "activations/layer12_attention_weight_min": -23.326353073120117,
      "activations/layer13_attention_weight_max": 38.92254638671875,
      "activations/layer13_attention_weight_min": -29.84280014038086,
      "activations/layer14_attention_weight_max": 38.272945404052734,
      "activations/layer14_attention_weight_min": -30.638751983642578,
      "activations/layer15_attention_weight_max": 34.673133850097656,
      "activations/layer15_attention_weight_min": -30.45476531982422,
      "activations/layer16_attention_weight_max": 35.895050048828125,
      "activations/layer16_attention_weight_min": -30.972469329833984,
      "activations/layer17_attention_weight_max": 55.26847457885742,
      "activations/layer17_attention_weight_min": -44.31782913208008,
      "activations/layer18_attention_weight_max": 48.69289779663086,
      "activations/layer18_attention_weight_min": -38.371490478515625,
      "activations/layer19_attention_weight_max": 23.257387161254883,
      "activations/layer19_attention_weight_min": -23.069761276245117,
      "activations/layer1_attention_weight_max": 19.182022094726562,
      "activations/layer1_attention_weight_min": -16.25926971435547,
      "activations/layer20_attention_weight_max": 23.909255981445312,
      "activations/layer20_attention_weight_min": -21.756860733032227,
      "activations/layer21_attention_weight_max": 42.44599151611328,
      "activations/layer21_attention_weight_min": -23.665653228759766,
      "activations/layer22_attention_weight_max": 32.15704345703125,
      "activations/layer22_attention_weight_min": -27.340362548828125,
      "activations/layer23_attention_weight_max": 41.14292907714844,
      "activations/layer23_attention_weight_min": -24.644603729248047,
      "activations/layer2_attention_weight_max": 33.433929443359375,
      "activations/layer2_attention_weight_min": -34.5208625793457,
      "activations/layer3_attention_weight_max": 99.95198059082031,
      "activations/layer3_attention_weight_min": -100.09921264648438,
      "activations/layer4_attention_weight_max": 100.04312896728516,
      "activations/layer4_attention_weight_min": -99.6634750366211,
      "activations/layer5_attention_weight_max": 72.79764556884766,
      "activations/layer5_attention_weight_min": -81.41167449951172,
      "activations/layer6_attention_weight_max": 53.86758041381836,
      "activations/layer6_attention_weight_min": -54.18574523925781,
      "activations/layer7_attention_weight_max": 69.70724487304688,
      "activations/layer7_attention_weight_min": -65.44608306884766,
      "activations/layer8_attention_weight_max": 48.56724548339844,
      "activations/layer8_attention_weight_min": -48.40657424926758,
      "activations/layer9_attention_weight_max": 48.787166595458984,
      "activations/layer9_attention_weight_min": -47.819976806640625,
      "epoch": 12.29,
      "learning_rate": 7.148787878787879e-05,
      "loss": 2.7722,
      "step": 211450
    },
    {
      "activations/layer0_attention_weight_max": 14.938810348510742,
      "activations/layer0_attention_weight_min": -13.844389915466309,
      "activations/layer10_attention_weight_max": 35.13750076293945,
      "activations/layer10_attention_weight_min": -37.02985382080078,
      "activations/layer11_attention_weight_max": 32.28375244140625,
      "activations/layer11_attention_weight_min": -32.432350158691406,
      "activations/layer12_attention_weight_max": 18.544328689575195,
      "activations/layer12_attention_weight_min": -26.126108169555664,
      "activations/layer13_attention_weight_max": 33.52959442138672,
      "activations/layer13_attention_weight_min": -26.187816619873047,
      "activations/layer14_attention_weight_max": 33.52149200439453,
      "activations/layer14_attention_weight_min": -30.13654899597168,
      "activations/layer15_attention_weight_max": 32.4970817565918,
      "activations/layer15_attention_weight_min": -28.432998657226562,
      "activations/layer16_attention_weight_max": 29.73850440979004,
      "activations/layer16_attention_weight_min": -29.08805274963379,
      "activations/layer17_attention_weight_max": 51.17742919921875,
      "activations/layer17_attention_weight_min": -42.89778137207031,
      "activations/layer18_attention_weight_max": 46.189056396484375,
      "activations/layer18_attention_weight_min": -38.32753372192383,
      "activations/layer19_attention_weight_max": 24.382915496826172,
      "activations/layer19_attention_weight_min": -23.145999908447266,
      "activations/layer1_attention_weight_max": 16.969881057739258,
      "activations/layer1_attention_weight_min": -15.755332946777344,
      "activations/layer20_attention_weight_max": 24.99473762512207,
      "activations/layer20_attention_weight_min": -22.47988510131836,
      "activations/layer21_attention_weight_max": 37.126068115234375,
      "activations/layer21_attention_weight_min": -23.9611759185791,
      "activations/layer22_attention_weight_max": 32.08197021484375,
      "activations/layer22_attention_weight_min": -25.84108543395996,
      "activations/layer23_attention_weight_max": 42.659454345703125,
      "activations/layer23_attention_weight_min": -26.08637046813965,
      "activations/layer2_attention_weight_max": 32.50783920288086,
      "activations/layer2_attention_weight_min": -31.80133056640625,
      "activations/layer3_attention_weight_max": 93.36567687988281,
      "activations/layer3_attention_weight_min": -98.3193588256836,
      "activations/layer4_attention_weight_max": 96.8808364868164,
      "activations/layer4_attention_weight_min": -92.16936492919922,
      "activations/layer5_attention_weight_max": 69.91493225097656,
      "activations/layer5_attention_weight_min": -76.72077178955078,
      "activations/layer6_attention_weight_max": 54.54072570800781,
      "activations/layer6_attention_weight_min": -52.83070373535156,
      "activations/layer7_attention_weight_max": 65.5743179321289,
      "activations/layer7_attention_weight_min": -70.72712707519531,
      "activations/layer8_attention_weight_max": 45.672359466552734,
      "activations/layer8_attention_weight_min": -46.424625396728516,
      "activations/layer9_attention_weight_max": 45.617034912109375,
      "activations/layer9_attention_weight_min": -45.96625900268555,
      "epoch": 12.29,
      "learning_rate": 7.146893939393939e-05,
      "loss": 2.7532,
      "step": 211500
    },
    {
      "activations/layer0_attention_weight_max": 15.936375617980957,
      "activations/layer0_attention_weight_min": -14.00815486907959,
      "activations/layer10_attention_weight_max": 38.73760986328125,
      "activations/layer10_attention_weight_min": -35.757816314697266,
      "activations/layer11_attention_weight_max": 32.98923873901367,
      "activations/layer11_attention_weight_min": -31.621795654296875,
      "activations/layer12_attention_weight_max": 17.330001831054688,
      "activations/layer12_attention_weight_min": -26.727750778198242,
      "activations/layer13_attention_weight_max": 34.974700927734375,
      "activations/layer13_attention_weight_min": -28.530200958251953,
      "activations/layer14_attention_weight_max": 33.15932846069336,
      "activations/layer14_attention_weight_min": -29.188766479492188,
      "activations/layer15_attention_weight_max": 31.53558921813965,
      "activations/layer15_attention_weight_min": -28.305253982543945,
      "activations/layer16_attention_weight_max": 33.59531021118164,
      "activations/layer16_attention_weight_min": -29.17985725402832,
      "activations/layer17_attention_weight_max": 51.6744384765625,
      "activations/layer17_attention_weight_min": -41.697059631347656,
      "activations/layer18_attention_weight_max": 48.56064987182617,
      "activations/layer18_attention_weight_min": -35.97361755371094,
      "activations/layer19_attention_weight_max": 26.746381759643555,
      "activations/layer19_attention_weight_min": -21.84092903137207,
      "activations/layer1_attention_weight_max": 16.6766300201416,
      "activations/layer1_attention_weight_min": -14.774534225463867,
      "activations/layer20_attention_weight_max": 22.749744415283203,
      "activations/layer20_attention_weight_min": -22.42041778564453,
      "activations/layer21_attention_weight_max": 38.42818832397461,
      "activations/layer21_attention_weight_min": -21.667566299438477,
      "activations/layer22_attention_weight_max": 32.40415573120117,
      "activations/layer22_attention_weight_min": -24.445459365844727,
      "activations/layer23_attention_weight_max": 40.93158721923828,
      "activations/layer23_attention_weight_min": -23.49091148376465,
      "activations/layer2_attention_weight_max": 32.8450813293457,
      "activations/layer2_attention_weight_min": -32.53176498413086,
      "activations/layer3_attention_weight_max": 96.51371765136719,
      "activations/layer3_attention_weight_min": -100.89189147949219,
      "activations/layer4_attention_weight_max": 96.76192474365234,
      "activations/layer4_attention_weight_min": -97.50994873046875,
      "activations/layer5_attention_weight_max": 70.71604919433594,
      "activations/layer5_attention_weight_min": -78.84568786621094,
      "activations/layer6_attention_weight_max": 52.34829330444336,
      "activations/layer6_attention_weight_min": -52.51780319213867,
      "activations/layer7_attention_weight_max": 67.43890380859375,
      "activations/layer7_attention_weight_min": -70.89696502685547,
      "activations/layer8_attention_weight_max": 48.3587532043457,
      "activations/layer8_attention_weight_min": -48.881500244140625,
      "activations/layer9_attention_weight_max": 48.06601333618164,
      "activations/layer9_attention_weight_min": -46.054962158203125,
      "epoch": 12.29,
      "learning_rate": 7.144999999999999e-05,
      "loss": 2.7694,
      "step": 211550
    },
    {
      "activations/layer0_attention_weight_max": 16.00179672241211,
      "activations/layer0_attention_weight_min": -14.11921215057373,
      "activations/layer10_attention_weight_max": 36.82327651977539,
      "activations/layer10_attention_weight_min": -35.302162170410156,
      "activations/layer11_attention_weight_max": 32.634605407714844,
      "activations/layer11_attention_weight_min": -31.53957748413086,
      "activations/layer12_attention_weight_max": 22.69274139404297,
      "activations/layer12_attention_weight_min": -24.774696350097656,
      "activations/layer13_attention_weight_max": 45.82581329345703,
      "activations/layer13_attention_weight_min": -30.58836555480957,
      "activations/layer14_attention_weight_max": 44.075645446777344,
      "activations/layer14_attention_weight_min": -29.81550407409668,
      "activations/layer15_attention_weight_max": 35.08252716064453,
      "activations/layer15_attention_weight_min": -28.777118682861328,
      "activations/layer16_attention_weight_max": 31.35329246520996,
      "activations/layer16_attention_weight_min": -31.029361724853516,
      "activations/layer17_attention_weight_max": 52.51165771484375,
      "activations/layer17_attention_weight_min": -46.04168701171875,
      "activations/layer18_attention_weight_max": 48.83757400512695,
      "activations/layer18_attention_weight_min": -40.4700813293457,
      "activations/layer19_attention_weight_max": 22.63033676147461,
      "activations/layer19_attention_weight_min": -23.159635543823242,
      "activations/layer1_attention_weight_max": 17.86008071899414,
      "activations/layer1_attention_weight_min": -14.864459991455078,
      "activations/layer20_attention_weight_max": 22.225419998168945,
      "activations/layer20_attention_weight_min": -19.90164566040039,
      "activations/layer21_attention_weight_max": 33.170475006103516,
      "activations/layer21_attention_weight_min": -24.31089210510254,
      "activations/layer22_attention_weight_max": 29.32469940185547,
      "activations/layer22_attention_weight_min": -25.55188751220703,
      "activations/layer23_attention_weight_max": 35.59822463989258,
      "activations/layer23_attention_weight_min": -25.910945892333984,
      "activations/layer2_attention_weight_max": 31.658241271972656,
      "activations/layer2_attention_weight_min": -31.192556381225586,
      "activations/layer3_attention_weight_max": 97.02836608886719,
      "activations/layer3_attention_weight_min": -95.91175842285156,
      "activations/layer4_attention_weight_max": 100.05876922607422,
      "activations/layer4_attention_weight_min": -94.90786743164062,
      "activations/layer5_attention_weight_max": 69.69204711914062,
      "activations/layer5_attention_weight_min": -78.02613830566406,
      "activations/layer6_attention_weight_max": 54.75017166137695,
      "activations/layer6_attention_weight_min": -53.922794342041016,
      "activations/layer7_attention_weight_max": 66.755859375,
      "activations/layer7_attention_weight_min": -65.9120864868164,
      "activations/layer8_attention_weight_max": 46.35641098022461,
      "activations/layer8_attention_weight_min": -46.777095794677734,
      "activations/layer9_attention_weight_max": 44.69993591308594,
      "activations/layer9_attention_weight_min": -48.31733322143555,
      "epoch": 12.3,
      "learning_rate": 7.14310606060606e-05,
      "loss": 2.7853,
      "step": 211600
    },
    {
      "activations/layer0_attention_weight_max": 15.534884452819824,
      "activations/layer0_attention_weight_min": -14.106010437011719,
      "activations/layer10_attention_weight_max": 35.72645568847656,
      "activations/layer10_attention_weight_min": -35.66545104980469,
      "activations/layer11_attention_weight_max": 33.31775665283203,
      "activations/layer11_attention_weight_min": -32.139869689941406,
      "activations/layer12_attention_weight_max": 19.008630752563477,
      "activations/layer12_attention_weight_min": -24.270776748657227,
      "activations/layer13_attention_weight_max": 40.74524688720703,
      "activations/layer13_attention_weight_min": -29.562273025512695,
      "activations/layer14_attention_weight_max": 36.447853088378906,
      "activations/layer14_attention_weight_min": -30.60183334350586,
      "activations/layer15_attention_weight_max": 33.23335266113281,
      "activations/layer15_attention_weight_min": -29.36111831665039,
      "activations/layer16_attention_weight_max": 32.77384948730469,
      "activations/layer16_attention_weight_min": -28.81795883178711,
      "activations/layer17_attention_weight_max": 51.417476654052734,
      "activations/layer17_attention_weight_min": -43.98799514770508,
      "activations/layer18_attention_weight_max": 50.0056266784668,
      "activations/layer18_attention_weight_min": -36.00716018676758,
      "activations/layer19_attention_weight_max": 22.80199432373047,
      "activations/layer19_attention_weight_min": -22.022226333618164,
      "activations/layer1_attention_weight_max": 17.371715545654297,
      "activations/layer1_attention_weight_min": -17.680566787719727,
      "activations/layer20_attention_weight_max": 22.726957321166992,
      "activations/layer20_attention_weight_min": -21.498291015625,
      "activations/layer21_attention_weight_max": 35.145843505859375,
      "activations/layer21_attention_weight_min": -23.05436134338379,
      "activations/layer22_attention_weight_max": 28.79785919189453,
      "activations/layer22_attention_weight_min": -25.8104305267334,
      "activations/layer23_attention_weight_max": 35.72727966308594,
      "activations/layer23_attention_weight_min": -22.929767608642578,
      "activations/layer2_attention_weight_max": 33.749813079833984,
      "activations/layer2_attention_weight_min": -31.515522003173828,
      "activations/layer3_attention_weight_max": 97.19856262207031,
      "activations/layer3_attention_weight_min": -96.71831512451172,
      "activations/layer4_attention_weight_max": 97.08402252197266,
      "activations/layer4_attention_weight_min": -95.97262573242188,
      "activations/layer5_attention_weight_max": 67.04013061523438,
      "activations/layer5_attention_weight_min": -78.93238830566406,
      "activations/layer6_attention_weight_max": 51.02602767944336,
      "activations/layer6_attention_weight_min": -51.471927642822266,
      "activations/layer7_attention_weight_max": 68.09766387939453,
      "activations/layer7_attention_weight_min": -66.0779037475586,
      "activations/layer8_attention_weight_max": 47.29140090942383,
      "activations/layer8_attention_weight_min": -48.83711242675781,
      "activations/layer9_attention_weight_max": 46.1402702331543,
      "activations/layer9_attention_weight_min": -46.44856643676758,
      "epoch": 12.3,
      "learning_rate": 7.14121212121212e-05,
      "loss": 2.7792,
      "step": 211650
    },
    {
      "activations/layer0_attention_weight_max": 15.70092487335205,
      "activations/layer0_attention_weight_min": -14.305328369140625,
      "activations/layer10_attention_weight_max": 38.17888259887695,
      "activations/layer10_attention_weight_min": -34.67879104614258,
      "activations/layer11_attention_weight_max": 31.183319091796875,
      "activations/layer11_attention_weight_min": -30.817264556884766,
      "activations/layer12_attention_weight_max": 17.619131088256836,
      "activations/layer12_attention_weight_min": -24.596221923828125,
      "activations/layer13_attention_weight_max": 32.284427642822266,
      "activations/layer13_attention_weight_min": -29.477331161499023,
      "activations/layer14_attention_weight_max": 34.372398376464844,
      "activations/layer14_attention_weight_min": -28.979530334472656,
      "activations/layer15_attention_weight_max": 28.984567642211914,
      "activations/layer15_attention_weight_min": -29.018728256225586,
      "activations/layer16_attention_weight_max": 29.437089920043945,
      "activations/layer16_attention_weight_min": -29.813302993774414,
      "activations/layer17_attention_weight_max": 46.567298889160156,
      "activations/layer17_attention_weight_min": -44.12872314453125,
      "activations/layer18_attention_weight_max": 40.97103500366211,
      "activations/layer18_attention_weight_min": -38.57144546508789,
      "activations/layer19_attention_weight_max": 20.526647567749023,
      "activations/layer19_attention_weight_min": -22.43305015563965,
      "activations/layer1_attention_weight_max": 17.29384422302246,
      "activations/layer1_attention_weight_min": -16.188329696655273,
      "activations/layer20_attention_weight_max": 21.042734146118164,
      "activations/layer20_attention_weight_min": -21.455652236938477,
      "activations/layer21_attention_weight_max": 33.357784271240234,
      "activations/layer21_attention_weight_min": -21.87852668762207,
      "activations/layer22_attention_weight_max": 30.001605987548828,
      "activations/layer22_attention_weight_min": -27.144460678100586,
      "activations/layer23_attention_weight_max": 35.515960693359375,
      "activations/layer23_attention_weight_min": -25.138458251953125,
      "activations/layer2_attention_weight_max": 32.63145065307617,
      "activations/layer2_attention_weight_min": -30.881711959838867,
      "activations/layer3_attention_weight_max": 94.48811340332031,
      "activations/layer3_attention_weight_min": -94.2498779296875,
      "activations/layer4_attention_weight_max": 96.49010467529297,
      "activations/layer4_attention_weight_min": -92.77471923828125,
      "activations/layer5_attention_weight_max": 71.30149841308594,
      "activations/layer5_attention_weight_min": -76.6156997680664,
      "activations/layer6_attention_weight_max": 54.65946578979492,
      "activations/layer6_attention_weight_min": -52.153648376464844,
      "activations/layer7_attention_weight_max": 68.14066314697266,
      "activations/layer7_attention_weight_min": -65.47818756103516,
      "activations/layer8_attention_weight_max": 45.95765686035156,
      "activations/layer8_attention_weight_min": -46.36783981323242,
      "activations/layer9_attention_weight_max": 44.27998733520508,
      "activations/layer9_attention_weight_min": -44.91023635864258,
      "epoch": 12.3,
      "learning_rate": 7.139318181818182e-05,
      "loss": 2.7684,
      "step": 211700
    },
    {
      "activations/layer0_attention_weight_max": 16.468671798706055,
      "activations/layer0_attention_weight_min": -14.375301361083984,
      "activations/layer10_attention_weight_max": 34.743778228759766,
      "activations/layer10_attention_weight_min": -34.87057113647461,
      "activations/layer11_attention_weight_max": 30.8646297454834,
      "activations/layer11_attention_weight_min": -31.764801025390625,
      "activations/layer12_attention_weight_max": 17.151473999023438,
      "activations/layer12_attention_weight_min": -24.214786529541016,
      "activations/layer13_attention_weight_max": 33.75426483154297,
      "activations/layer13_attention_weight_min": -32.02001190185547,
      "activations/layer14_attention_weight_max": 34.21390151977539,
      "activations/layer14_attention_weight_min": -30.884668350219727,
      "activations/layer15_attention_weight_max": 29.710119247436523,
      "activations/layer15_attention_weight_min": -31.694900512695312,
      "activations/layer16_attention_weight_max": 28.832897186279297,
      "activations/layer16_attention_weight_min": -31.451189041137695,
      "activations/layer17_attention_weight_max": 48.487693786621094,
      "activations/layer17_attention_weight_min": -43.3421516418457,
      "activations/layer18_attention_weight_max": 46.26560974121094,
      "activations/layer18_attention_weight_min": -40.00474166870117,
      "activations/layer19_attention_weight_max": 23.033681869506836,
      "activations/layer19_attention_weight_min": -23.702194213867188,
      "activations/layer1_attention_weight_max": 17.07314682006836,
      "activations/layer1_attention_weight_min": -15.76751708984375,
      "activations/layer20_attention_weight_max": 23.09302520751953,
      "activations/layer20_attention_weight_min": -22.862531661987305,
      "activations/layer21_attention_weight_max": 34.18626022338867,
      "activations/layer21_attention_weight_min": -21.781566619873047,
      "activations/layer22_attention_weight_max": 31.721797943115234,
      "activations/layer22_attention_weight_min": -30.407665252685547,
      "activations/layer23_attention_weight_max": 34.91428756713867,
      "activations/layer23_attention_weight_min": -24.24547576904297,
      "activations/layer2_attention_weight_max": 33.25163269042969,
      "activations/layer2_attention_weight_min": -31.017988204956055,
      "activations/layer3_attention_weight_max": 99.59552764892578,
      "activations/layer3_attention_weight_min": -98.1077880859375,
      "activations/layer4_attention_weight_max": 101.0796127319336,
      "activations/layer4_attention_weight_min": -100.10247039794922,
      "activations/layer5_attention_weight_max": 71.41777801513672,
      "activations/layer5_attention_weight_min": -79.59920501708984,
      "activations/layer6_attention_weight_max": 53.298709869384766,
      "activations/layer6_attention_weight_min": -56.22862243652344,
      "activations/layer7_attention_weight_max": 73.24439239501953,
      "activations/layer7_attention_weight_min": -68.0063705444336,
      "activations/layer8_attention_weight_max": 44.2453498840332,
      "activations/layer8_attention_weight_min": -48.17935562133789,
      "activations/layer9_attention_weight_max": 46.395389556884766,
      "activations/layer9_attention_weight_min": -45.50259780883789,
      "epoch": 12.3,
      "learning_rate": 7.137424242424242e-05,
      "loss": 2.7762,
      "step": 211750
    },
    {
      "activations/layer0_attention_weight_max": 15.717748641967773,
      "activations/layer0_attention_weight_min": -14.473808288574219,
      "activations/layer10_attention_weight_max": 40.69404220581055,
      "activations/layer10_attention_weight_min": -36.8400764465332,
      "activations/layer11_attention_weight_max": 33.95183563232422,
      "activations/layer11_attention_weight_min": -32.775848388671875,
      "activations/layer12_attention_weight_max": 19.51335334777832,
      "activations/layer12_attention_weight_min": -26.02678108215332,
      "activations/layer13_attention_weight_max": 43.88272476196289,
      "activations/layer13_attention_weight_min": -30.798507690429688,
      "activations/layer14_attention_weight_max": 50.18153762817383,
      "activations/layer14_attention_weight_min": -31.347278594970703,
      "activations/layer15_attention_weight_max": 40.93257141113281,
      "activations/layer15_attention_weight_min": -29.34100341796875,
      "activations/layer16_attention_weight_max": 34.487701416015625,
      "activations/layer16_attention_weight_min": -32.05216979980469,
      "activations/layer17_attention_weight_max": 56.75510787963867,
      "activations/layer17_attention_weight_min": -48.930397033691406,
      "activations/layer18_attention_weight_max": 53.54437255859375,
      "activations/layer18_attention_weight_min": -43.93817901611328,
      "activations/layer19_attention_weight_max": 26.191486358642578,
      "activations/layer19_attention_weight_min": -24.34029197692871,
      "activations/layer1_attention_weight_max": 17.38445281982422,
      "activations/layer1_attention_weight_min": -16.703121185302734,
      "activations/layer20_attention_weight_max": 26.774608612060547,
      "activations/layer20_attention_weight_min": -23.187742233276367,
      "activations/layer21_attention_weight_max": 49.5041618347168,
      "activations/layer21_attention_weight_min": -24.015289306640625,
      "activations/layer22_attention_weight_max": 35.57132339477539,
      "activations/layer22_attention_weight_min": -25.9073429107666,
      "activations/layer23_attention_weight_max": 42.865394592285156,
      "activations/layer23_attention_weight_min": -24.284175872802734,
      "activations/layer2_attention_weight_max": 34.21733093261719,
      "activations/layer2_attention_weight_min": -31.587230682373047,
      "activations/layer3_attention_weight_max": 94.62449645996094,
      "activations/layer3_attention_weight_min": -96.11700439453125,
      "activations/layer4_attention_weight_max": 98.19083404541016,
      "activations/layer4_attention_weight_min": -93.86854553222656,
      "activations/layer5_attention_weight_max": 71.00092315673828,
      "activations/layer5_attention_weight_min": -76.74609375,
      "activations/layer6_attention_weight_max": 53.081546783447266,
      "activations/layer6_attention_weight_min": -53.77163314819336,
      "activations/layer7_attention_weight_max": 69.0252685546875,
      "activations/layer7_attention_weight_min": -65.79627990722656,
      "activations/layer8_attention_weight_max": 47.977657318115234,
      "activations/layer8_attention_weight_min": -49.06343460083008,
      "activations/layer9_attention_weight_max": 48.520301818847656,
      "activations/layer9_attention_weight_min": -49.524192810058594,
      "epoch": 12.31,
      "learning_rate": 7.135530303030302e-05,
      "loss": 2.7896,
      "step": 211800
    },
    {
      "activations/layer0_attention_weight_max": 16.13758087158203,
      "activations/layer0_attention_weight_min": -14.357349395751953,
      "activations/layer10_attention_weight_max": 37.4344367980957,
      "activations/layer10_attention_weight_min": -35.228084564208984,
      "activations/layer11_attention_weight_max": 32.332027435302734,
      "activations/layer11_attention_weight_min": -31.954946517944336,
      "activations/layer12_attention_weight_max": 18.036462783813477,
      "activations/layer12_attention_weight_min": -24.735374450683594,
      "activations/layer13_attention_weight_max": 36.29002380371094,
      "activations/layer13_attention_weight_min": -28.91111946105957,
      "activations/layer14_attention_weight_max": 34.902313232421875,
      "activations/layer14_attention_weight_min": -30.459980010986328,
      "activations/layer15_attention_weight_max": 32.343536376953125,
      "activations/layer15_attention_weight_min": -29.912160873413086,
      "activations/layer16_attention_weight_max": 31.38743019104004,
      "activations/layer16_attention_weight_min": -29.902311325073242,
      "activations/layer17_attention_weight_max": 54.14802169799805,
      "activations/layer17_attention_weight_min": -46.01335525512695,
      "activations/layer18_attention_weight_max": 45.79764175415039,
      "activations/layer18_attention_weight_min": -39.00828552246094,
      "activations/layer19_attention_weight_max": 22.4500675201416,
      "activations/layer19_attention_weight_min": -20.769975662231445,
      "activations/layer1_attention_weight_max": 16.145824432373047,
      "activations/layer1_attention_weight_min": -15.153587341308594,
      "activations/layer20_attention_weight_max": 21.88591766357422,
      "activations/layer20_attention_weight_min": -23.68714714050293,
      "activations/layer21_attention_weight_max": 33.176856994628906,
      "activations/layer21_attention_weight_min": -21.379791259765625,
      "activations/layer22_attention_weight_max": 27.407012939453125,
      "activations/layer22_attention_weight_min": -25.320556640625,
      "activations/layer23_attention_weight_max": 34.803306579589844,
      "activations/layer23_attention_weight_min": -23.844343185424805,
      "activations/layer2_attention_weight_max": 33.470664978027344,
      "activations/layer2_attention_weight_min": -31.662979125976562,
      "activations/layer3_attention_weight_max": 98.51832580566406,
      "activations/layer3_attention_weight_min": -98.99874877929688,
      "activations/layer4_attention_weight_max": 101.74419403076172,
      "activations/layer4_attention_weight_min": -98.54497528076172,
      "activations/layer5_attention_weight_max": 76.40472412109375,
      "activations/layer5_attention_weight_min": -76.85308074951172,
      "activations/layer6_attention_weight_max": 54.27772903442383,
      "activations/layer6_attention_weight_min": -54.02405548095703,
      "activations/layer7_attention_weight_max": 70.64282989501953,
      "activations/layer7_attention_weight_min": -66.89846801757812,
      "activations/layer8_attention_weight_max": 44.5060920715332,
      "activations/layer8_attention_weight_min": -47.270835876464844,
      "activations/layer9_attention_weight_max": 44.51072311401367,
      "activations/layer9_attention_weight_min": -46.71939468383789,
      "epoch": 12.31,
      "learning_rate": 7.133674242424243e-05,
      "loss": 2.7476,
      "step": 211850
    },
    {
      "activations/layer0_attention_weight_max": 16.20486831665039,
      "activations/layer0_attention_weight_min": -14.117731094360352,
      "activations/layer10_attention_weight_max": 39.33507537841797,
      "activations/layer10_attention_weight_min": -36.711185455322266,
      "activations/layer11_attention_weight_max": 32.50751876831055,
      "activations/layer11_attention_weight_min": -31.35802459716797,
      "activations/layer12_attention_weight_max": 20.113956451416016,
      "activations/layer12_attention_weight_min": -29.204452514648438,
      "activations/layer13_attention_weight_max": 40.87538146972656,
      "activations/layer13_attention_weight_min": -27.20296287536621,
      "activations/layer14_attention_weight_max": 43.1279411315918,
      "activations/layer14_attention_weight_min": -27.693553924560547,
      "activations/layer15_attention_weight_max": 34.69685745239258,
      "activations/layer15_attention_weight_min": -28.60320281982422,
      "activations/layer16_attention_weight_max": 34.234153747558594,
      "activations/layer16_attention_weight_min": -28.65635871887207,
      "activations/layer17_attention_weight_max": 50.578067779541016,
      "activations/layer17_attention_weight_min": -43.311187744140625,
      "activations/layer18_attention_weight_max": 49.121009826660156,
      "activations/layer18_attention_weight_min": -36.626251220703125,
      "activations/layer19_attention_weight_max": 25.179908752441406,
      "activations/layer19_attention_weight_min": -21.499868392944336,
      "activations/layer1_attention_weight_max": 16.714242935180664,
      "activations/layer1_attention_weight_min": -16.401851654052734,
      "activations/layer20_attention_weight_max": 25.743284225463867,
      "activations/layer20_attention_weight_min": -20.00937843322754,
      "activations/layer21_attention_weight_max": 37.75623321533203,
      "activations/layer21_attention_weight_min": -22.385818481445312,
      "activations/layer22_attention_weight_max": 30.49370002746582,
      "activations/layer22_attention_weight_min": -26.522071838378906,
      "activations/layer23_attention_weight_max": 35.096702575683594,
      "activations/layer23_attention_weight_min": -25.310163497924805,
      "activations/layer2_attention_weight_max": 33.659297943115234,
      "activations/layer2_attention_weight_min": -31.384389877319336,
      "activations/layer3_attention_weight_max": 104.93540954589844,
      "activations/layer3_attention_weight_min": -101.44710540771484,
      "activations/layer4_attention_weight_max": 107.38785552978516,
      "activations/layer4_attention_weight_min": -102.1458969116211,
      "activations/layer5_attention_weight_max": 76.09185791015625,
      "activations/layer5_attention_weight_min": -78.47882080078125,
      "activations/layer6_attention_weight_max": 56.880584716796875,
      "activations/layer6_attention_weight_min": -55.31306838989258,
      "activations/layer7_attention_weight_max": 75.14735412597656,
      "activations/layer7_attention_weight_min": -65.84435272216797,
      "activations/layer8_attention_weight_max": 48.338401794433594,
      "activations/layer8_attention_weight_min": -48.0558967590332,
      "activations/layer9_attention_weight_max": 49.783687591552734,
      "activations/layer9_attention_weight_min": -47.04681396484375,
      "epoch": 12.31,
      "learning_rate": 7.131780303030303e-05,
      "loss": 2.7731,
      "step": 211900
    },
    {
      "activations/layer0_attention_weight_max": 16.49497413635254,
      "activations/layer0_attention_weight_min": -14.270730018615723,
      "activations/layer10_attention_weight_max": 37.064414978027344,
      "activations/layer10_attention_weight_min": -35.780574798583984,
      "activations/layer11_attention_weight_max": 33.107364654541016,
      "activations/layer11_attention_weight_min": -31.42206573486328,
      "activations/layer12_attention_weight_max": 21.22218894958496,
      "activations/layer12_attention_weight_min": -22.56233024597168,
      "activations/layer13_attention_weight_max": 46.83103561401367,
      "activations/layer13_attention_weight_min": -30.973175048828125,
      "activations/layer14_attention_weight_max": 34.89683532714844,
      "activations/layer14_attention_weight_min": -31.179332733154297,
      "activations/layer15_attention_weight_max": 31.920495986938477,
      "activations/layer15_attention_weight_min": -28.314258575439453,
      "activations/layer16_attention_weight_max": 33.356048583984375,
      "activations/layer16_attention_weight_min": -29.02580451965332,
      "activations/layer17_attention_weight_max": 50.13173294067383,
      "activations/layer17_attention_weight_min": -41.77323532104492,
      "activations/layer18_attention_weight_max": 41.79601287841797,
      "activations/layer18_attention_weight_min": -35.28845977783203,
      "activations/layer19_attention_weight_max": 20.825464248657227,
      "activations/layer19_attention_weight_min": -19.790922164916992,
      "activations/layer1_attention_weight_max": 16.497852325439453,
      "activations/layer1_attention_weight_min": -16.89668846130371,
      "activations/layer20_attention_weight_max": 21.22712516784668,
      "activations/layer20_attention_weight_min": -21.737958908081055,
      "activations/layer21_attention_weight_max": 32.670928955078125,
      "activations/layer21_attention_weight_min": -19.869718551635742,
      "activations/layer22_attention_weight_max": 32.45821762084961,
      "activations/layer22_attention_weight_min": -26.01521110534668,
      "activations/layer23_attention_weight_max": 36.286964416503906,
      "activations/layer23_attention_weight_min": -23.246740341186523,
      "activations/layer2_attention_weight_max": 31.637998580932617,
      "activations/layer2_attention_weight_min": -31.918088912963867,
      "activations/layer3_attention_weight_max": 95.98355102539062,
      "activations/layer3_attention_weight_min": -98.69047546386719,
      "activations/layer4_attention_weight_max": 98.08255004882812,
      "activations/layer4_attention_weight_min": -97.16835021972656,
      "activations/layer5_attention_weight_max": 70.48819732666016,
      "activations/layer5_attention_weight_min": -80.0687255859375,
      "activations/layer6_attention_weight_max": 53.86978530883789,
      "activations/layer6_attention_weight_min": -55.35831832885742,
      "activations/layer7_attention_weight_max": 69.67610931396484,
      "activations/layer7_attention_weight_min": -68.77186584472656,
      "activations/layer8_attention_weight_max": 45.847129821777344,
      "activations/layer8_attention_weight_min": -50.03913116455078,
      "activations/layer9_attention_weight_max": 46.783016204833984,
      "activations/layer9_attention_weight_min": -49.483821868896484,
      "epoch": 12.32,
      "learning_rate": 7.129886363636363e-05,
      "loss": 2.7799,
      "step": 211950
    },
    {
      "activations/layer0_attention_weight_max": 15.786286354064941,
      "activations/layer0_attention_weight_min": -14.162969589233398,
      "activations/layer10_attention_weight_max": 38.885772705078125,
      "activations/layer10_attention_weight_min": -36.36130142211914,
      "activations/layer11_attention_weight_max": 33.4976921081543,
      "activations/layer11_attention_weight_min": -35.63212966918945,
      "activations/layer12_attention_weight_max": 16.76851463317871,
      "activations/layer12_attention_weight_min": -27.035560607910156,
      "activations/layer13_attention_weight_max": 35.837646484375,
      "activations/layer13_attention_weight_min": -31.150836944580078,
      "activations/layer14_attention_weight_max": 33.29325485229492,
      "activations/layer14_attention_weight_min": -30.644182205200195,
      "activations/layer15_attention_weight_max": 32.490169525146484,
      "activations/layer15_attention_weight_min": -31.30844497680664,
      "activations/layer16_attention_weight_max": 30.21234130859375,
      "activations/layer16_attention_weight_min": -29.426677703857422,
      "activations/layer17_attention_weight_max": 48.84514236450195,
      "activations/layer17_attention_weight_min": -47.079566955566406,
      "activations/layer18_attention_weight_max": 45.26569747924805,
      "activations/layer18_attention_weight_min": -41.52509689331055,
      "activations/layer19_attention_weight_max": 21.07611656188965,
      "activations/layer19_attention_weight_min": -24.95046043395996,
      "activations/layer1_attention_weight_max": 16.982980728149414,
      "activations/layer1_attention_weight_min": -16.9132022857666,
      "activations/layer20_attention_weight_max": 22.250349044799805,
      "activations/layer20_attention_weight_min": -23.132089614868164,
      "activations/layer21_attention_weight_max": 37.534793853759766,
      "activations/layer21_attention_weight_min": -28.226455688476562,
      "activations/layer22_attention_weight_max": 31.98145294189453,
      "activations/layer22_attention_weight_min": -29.397146224975586,
      "activations/layer23_attention_weight_max": 39.846763610839844,
      "activations/layer23_attention_weight_min": -26.09744644165039,
      "activations/layer2_attention_weight_max": 34.12165069580078,
      "activations/layer2_attention_weight_min": -30.96963882446289,
      "activations/layer3_attention_weight_max": 95.62316131591797,
      "activations/layer3_attention_weight_min": -96.38725280761719,
      "activations/layer4_attention_weight_max": 99.33353424072266,
      "activations/layer4_attention_weight_min": -96.83517456054688,
      "activations/layer5_attention_weight_max": 73.7453842163086,
      "activations/layer5_attention_weight_min": -81.41886901855469,
      "activations/layer6_attention_weight_max": 56.2674446105957,
      "activations/layer6_attention_weight_min": -54.33919143676758,
      "activations/layer7_attention_weight_max": 71.74581146240234,
      "activations/layer7_attention_weight_min": -70.11841583251953,
      "activations/layer8_attention_weight_max": 46.09939956665039,
      "activations/layer8_attention_weight_min": -50.16523361206055,
      "activations/layer9_attention_weight_max": 45.92327880859375,
      "activations/layer9_attention_weight_min": -47.821258544921875,
      "epoch": 12.32,
      "learning_rate": 7.127992424242423e-05,
      "loss": 2.79,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.5302,
      "eval_samples_per_second": 503.386,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.5302,
      "eval_openwebtext_samples_per_second": 503.386,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9962,
      "eval_wikitext_samples_per_second": 228.439,
      "step": 212000
    },
    {
      "epoch": 12.32,
      "eval_lambada_loss": 2.537109375,
      "eval_lambada_ppl": 12.643071719773257,
      "eval_lambada_runtime": 9.5872,
      "eval_lambada_samples_per_second": 507.867,
      "step": 212000
    },
    {
      "activations/layer0_attention_weight_max": 17.142440795898438,
      "activations/layer0_attention_weight_min": -14.377842903137207,
      "activations/layer10_attention_weight_max": 35.67979049682617,
      "activations/layer10_attention_weight_min": -36.77821731567383,
      "activations/layer11_attention_weight_max": 31.68318748474121,
      "activations/layer11_attention_weight_min": -34.92156982421875,
      "activations/layer12_attention_weight_max": 18.331716537475586,
      "activations/layer12_attention_weight_min": -25.331233978271484,
      "activations/layer13_attention_weight_max": 35.34520721435547,
      "activations/layer13_attention_weight_min": -28.246461868286133,
      "activations/layer14_attention_weight_max": 38.73462677001953,
      "activations/layer14_attention_weight_min": -30.644596099853516,
      "activations/layer15_attention_weight_max": 33.761192321777344,
      "activations/layer15_attention_weight_min": -30.584991455078125,
      "activations/layer16_attention_weight_max": 34.30496597290039,
      "activations/layer16_attention_weight_min": -30.792598724365234,
      "activations/layer17_attention_weight_max": 55.615177154541016,
      "activations/layer17_attention_weight_min": -47.98573684692383,
      "activations/layer18_attention_weight_max": 48.168270111083984,
      "activations/layer18_attention_weight_min": -40.00170135498047,
      "activations/layer19_attention_weight_max": 21.467565536499023,
      "activations/layer19_attention_weight_min": -23.340417861938477,
      "activations/layer1_attention_weight_max": 16.240680694580078,
      "activations/layer1_attention_weight_min": -15.127653121948242,
      "activations/layer20_attention_weight_max": 21.477581024169922,
      "activations/layer20_attention_weight_min": -24.193845748901367,
      "activations/layer21_attention_weight_max": 35.410552978515625,
      "activations/layer21_attention_weight_min": -23.72139549255371,
      "activations/layer22_attention_weight_max": 29.148712158203125,
      "activations/layer22_attention_weight_min": -25.554290771484375,
      "activations/layer23_attention_weight_max": 40.354591369628906,
      "activations/layer23_attention_weight_min": -24.922948837280273,
      "activations/layer2_attention_weight_max": 33.731956481933594,
      "activations/layer2_attention_weight_min": -33.173038482666016,
      "activations/layer3_attention_weight_max": 99.95526885986328,
      "activations/layer3_attention_weight_min": -98.44471740722656,
      "activations/layer4_attention_weight_max": 98.13402557373047,
      "activations/layer4_attention_weight_min": -95.62185668945312,
      "activations/layer5_attention_weight_max": 73.6526870727539,
      "activations/layer5_attention_weight_min": -75.3975601196289,
      "activations/layer6_attention_weight_max": 53.424354553222656,
      "activations/layer6_attention_weight_min": -55.18123245239258,
      "activations/layer7_attention_weight_max": 69.6491928100586,
      "activations/layer7_attention_weight_min": -66.86103820800781,
      "activations/layer8_attention_weight_max": 46.74337387084961,
      "activations/layer8_attention_weight_min": -46.18959426879883,
      "activations/layer9_attention_weight_max": 45.09305953979492,
      "activations/layer9_attention_weight_min": -48.1510124206543,
      "epoch": 12.32,
      "learning_rate": 7.126098484848485e-05,
      "loss": 2.753,
      "step": 212050
    },
    {
      "activations/layer0_attention_weight_max": 15.365118026733398,
      "activations/layer0_attention_weight_min": -14.35720443725586,
      "activations/layer10_attention_weight_max": 35.810882568359375,
      "activations/layer10_attention_weight_min": -33.94915771484375,
      "activations/layer11_attention_weight_max": 30.892208099365234,
      "activations/layer11_attention_weight_min": -32.358558654785156,
      "activations/layer12_attention_weight_max": 18.64805793762207,
      "activations/layer12_attention_weight_min": -24.12110137939453,
      "activations/layer13_attention_weight_max": 34.504886627197266,
      "activations/layer13_attention_weight_min": -27.991853713989258,
      "activations/layer14_attention_weight_max": 34.86033248901367,
      "activations/layer14_attention_weight_min": -31.9361629486084,
      "activations/layer15_attention_weight_max": 32.38340759277344,
      "activations/layer15_attention_weight_min": -30.69263458251953,
      "activations/layer16_attention_weight_max": 32.28390884399414,
      "activations/layer16_attention_weight_min": -32.645477294921875,
      "activations/layer17_attention_weight_max": 50.079593658447266,
      "activations/layer17_attention_weight_min": -45.97827911376953,
      "activations/layer18_attention_weight_max": 46.83938980102539,
      "activations/layer18_attention_weight_min": -36.19178009033203,
      "activations/layer19_attention_weight_max": 21.294103622436523,
      "activations/layer19_attention_weight_min": -20.744815826416016,
      "activations/layer1_attention_weight_max": 16.600025177001953,
      "activations/layer1_attention_weight_min": -17.061203002929688,
      "activations/layer20_attention_weight_max": 22.478416442871094,
      "activations/layer20_attention_weight_min": -22.66582679748535,
      "activations/layer21_attention_weight_max": 36.465702056884766,
      "activations/layer21_attention_weight_min": -23.45985984802246,
      "activations/layer22_attention_weight_max": 31.13109016418457,
      "activations/layer22_attention_weight_min": -25.021320343017578,
      "activations/layer23_attention_weight_max": 38.05548858642578,
      "activations/layer23_attention_weight_min": -22.222675323486328,
      "activations/layer2_attention_weight_max": 33.58005905151367,
      "activations/layer2_attention_weight_min": -32.43778610229492,
      "activations/layer3_attention_weight_max": 99.90225982666016,
      "activations/layer3_attention_weight_min": -95.71412658691406,
      "activations/layer4_attention_weight_max": 100.5343246459961,
      "activations/layer4_attention_weight_min": -95.08683013916016,
      "activations/layer5_attention_weight_max": 77.67289733886719,
      "activations/layer5_attention_weight_min": -79.87629699707031,
      "activations/layer6_attention_weight_max": 53.560428619384766,
      "activations/layer6_attention_weight_min": -53.333839416503906,
      "activations/layer7_attention_weight_max": 70.19503784179688,
      "activations/layer7_attention_weight_min": -68.59024810791016,
      "activations/layer8_attention_weight_max": 47.00031280517578,
      "activations/layer8_attention_weight_min": -51.18438720703125,
      "activations/layer9_attention_weight_max": 50.272151947021484,
      "activations/layer9_attention_weight_min": -47.0215950012207,
      "epoch": 12.32,
      "learning_rate": 7.124242424242424e-05,
      "loss": 2.7632,
      "step": 212100
    },
    {
      "activations/layer0_attention_weight_max": 16.30620002746582,
      "activations/layer0_attention_weight_min": -14.171856880187988,
      "activations/layer10_attention_weight_max": 35.575218200683594,
      "activations/layer10_attention_weight_min": -35.5011100769043,
      "activations/layer11_attention_weight_max": 32.01424789428711,
      "activations/layer11_attention_weight_min": -33.379310607910156,
      "activations/layer12_attention_weight_max": 18.23456573486328,
      "activations/layer12_attention_weight_min": -25.016834259033203,
      "activations/layer13_attention_weight_max": 37.26954650878906,
      "activations/layer13_attention_weight_min": -28.679779052734375,
      "activations/layer14_attention_weight_max": 35.32801055908203,
      "activations/layer14_attention_weight_min": -30.255359649658203,
      "activations/layer15_attention_weight_max": 32.582767486572266,
      "activations/layer15_attention_weight_min": -29.143211364746094,
      "activations/layer16_attention_weight_max": 29.601858139038086,
      "activations/layer16_attention_weight_min": -28.800180435180664,
      "activations/layer17_attention_weight_max": 49.68423080444336,
      "activations/layer17_attention_weight_min": -41.828338623046875,
      "activations/layer18_attention_weight_max": 45.989036560058594,
      "activations/layer18_attention_weight_min": -35.40431594848633,
      "activations/layer19_attention_weight_max": 21.038665771484375,
      "activations/layer19_attention_weight_min": -21.28042221069336,
      "activations/layer1_attention_weight_max": 17.40948486328125,
      "activations/layer1_attention_weight_min": -16.682022094726562,
      "activations/layer20_attention_weight_max": 21.084840774536133,
      "activations/layer20_attention_weight_min": -22.620834350585938,
      "activations/layer21_attention_weight_max": 32.38771438598633,
      "activations/layer21_attention_weight_min": -23.068361282348633,
      "activations/layer22_attention_weight_max": 28.47366714477539,
      "activations/layer22_attention_weight_min": -25.614715576171875,
      "activations/layer23_attention_weight_max": 37.89061737060547,
      "activations/layer23_attention_weight_min": -23.955665588378906,
      "activations/layer2_attention_weight_max": 33.52550506591797,
      "activations/layer2_attention_weight_min": -32.76215362548828,
      "activations/layer3_attention_weight_max": 95.38015747070312,
      "activations/layer3_attention_weight_min": -98.78871154785156,
      "activations/layer4_attention_weight_max": 101.72248840332031,
      "activations/layer4_attention_weight_min": -101.67806243896484,
      "activations/layer5_attention_weight_max": 73.06147766113281,
      "activations/layer5_attention_weight_min": -80.32598114013672,
      "activations/layer6_attention_weight_max": 53.174110412597656,
      "activations/layer6_attention_weight_min": -52.97669219970703,
      "activations/layer7_attention_weight_max": 70.16319274902344,
      "activations/layer7_attention_weight_min": -70.44818115234375,
      "activations/layer8_attention_weight_max": 46.63786315917969,
      "activations/layer8_attention_weight_min": -47.308048248291016,
      "activations/layer9_attention_weight_max": 45.49782180786133,
      "activations/layer9_attention_weight_min": -46.975738525390625,
      "epoch": 12.33,
      "learning_rate": 7.122348484848484e-05,
      "loss": 2.761,
      "step": 212150
    },
    {
      "activations/layer0_attention_weight_max": 15.962774276733398,
      "activations/layer0_attention_weight_min": -14.509047508239746,
      "activations/layer10_attention_weight_max": 37.89826202392578,
      "activations/layer10_attention_weight_min": -37.73187255859375,
      "activations/layer11_attention_weight_max": 34.253055572509766,
      "activations/layer11_attention_weight_min": -32.39162063598633,
      "activations/layer12_attention_weight_max": 17.997224807739258,
      "activations/layer12_attention_weight_min": -25.906953811645508,
      "activations/layer13_attention_weight_max": 37.40029525756836,
      "activations/layer13_attention_weight_min": -30.364425659179688,
      "activations/layer14_attention_weight_max": 37.906211853027344,
      "activations/layer14_attention_weight_min": -31.4805850982666,
      "activations/layer15_attention_weight_max": 34.0092887878418,
      "activations/layer15_attention_weight_min": -28.099822998046875,
      "activations/layer16_attention_weight_max": 30.694307327270508,
      "activations/layer16_attention_weight_min": -29.13225555419922,
      "activations/layer17_attention_weight_max": 50.317359924316406,
      "activations/layer17_attention_weight_min": -41.04658126831055,
      "activations/layer18_attention_weight_max": 44.093448638916016,
      "activations/layer18_attention_weight_min": -36.780303955078125,
      "activations/layer19_attention_weight_max": 21.057226181030273,
      "activations/layer19_attention_weight_min": -21.865955352783203,
      "activations/layer1_attention_weight_max": 17.32499122619629,
      "activations/layer1_attention_weight_min": -14.368916511535645,
      "activations/layer20_attention_weight_max": 20.103235244750977,
      "activations/layer20_attention_weight_min": -20.454687118530273,
      "activations/layer21_attention_weight_max": 35.6762580871582,
      "activations/layer21_attention_weight_min": -22.21247673034668,
      "activations/layer22_attention_weight_max": 29.406749725341797,
      "activations/layer22_attention_weight_min": -25.995361328125,
      "activations/layer23_attention_weight_max": 33.971458435058594,
      "activations/layer23_attention_weight_min": -23.050453186035156,
      "activations/layer2_attention_weight_max": 32.269920349121094,
      "activations/layer2_attention_weight_min": -34.141868591308594,
      "activations/layer3_attention_weight_max": 99.7149658203125,
      "activations/layer3_attention_weight_min": -101.06121063232422,
      "activations/layer4_attention_weight_max": 103.95934295654297,
      "activations/layer4_attention_weight_min": -101.80635833740234,
      "activations/layer5_attention_weight_max": 76.52163696289062,
      "activations/layer5_attention_weight_min": -79.60736846923828,
      "activations/layer6_attention_weight_max": 56.23086929321289,
      "activations/layer6_attention_weight_min": -52.76529312133789,
      "activations/layer7_attention_weight_max": 73.45722961425781,
      "activations/layer7_attention_weight_min": -68.9180679321289,
      "activations/layer8_attention_weight_max": 50.595279693603516,
      "activations/layer8_attention_weight_min": -48.67115020751953,
      "activations/layer9_attention_weight_max": 49.52653503417969,
      "activations/layer9_attention_weight_min": -47.13653564453125,
      "epoch": 12.33,
      "learning_rate": 7.120454545454544e-05,
      "loss": 2.7612,
      "step": 212200
    },
    {
      "activations/layer0_attention_weight_max": 15.649983406066895,
      "activations/layer0_attention_weight_min": -13.828897476196289,
      "activations/layer10_attention_weight_max": 39.4837760925293,
      "activations/layer10_attention_weight_min": -38.626136779785156,
      "activations/layer11_attention_weight_max": 35.408348083496094,
      "activations/layer11_attention_weight_min": -34.201568603515625,
      "activations/layer12_attention_weight_max": 19.141889572143555,
      "activations/layer12_attention_weight_min": -26.563722610473633,
      "activations/layer13_attention_weight_max": 43.54813766479492,
      "activations/layer13_attention_weight_min": -30.73207664489746,
      "activations/layer14_attention_weight_max": 41.431236267089844,
      "activations/layer14_attention_weight_min": -32.408390045166016,
      "activations/layer15_attention_weight_max": 36.6920166015625,
      "activations/layer15_attention_weight_min": -32.43976593017578,
      "activations/layer16_attention_weight_max": 34.26102066040039,
      "activations/layer16_attention_weight_min": -32.579933166503906,
      "activations/layer17_attention_weight_max": 54.475154876708984,
      "activations/layer17_attention_weight_min": -44.694786071777344,
      "activations/layer18_attention_weight_max": 49.162445068359375,
      "activations/layer18_attention_weight_min": -37.298519134521484,
      "activations/layer19_attention_weight_max": 25.328628540039062,
      "activations/layer19_attention_weight_min": -22.333906173706055,
      "activations/layer1_attention_weight_max": 16.62539291381836,
      "activations/layer1_attention_weight_min": -15.236595153808594,
      "activations/layer20_attention_weight_max": 24.272218704223633,
      "activations/layer20_attention_weight_min": -20.63103485107422,
      "activations/layer21_attention_weight_max": 43.721412658691406,
      "activations/layer21_attention_weight_min": -24.285598754882812,
      "activations/layer22_attention_weight_max": 32.313228607177734,
      "activations/layer22_attention_weight_min": -25.283336639404297,
      "activations/layer23_attention_weight_max": 38.056068420410156,
      "activations/layer23_attention_weight_min": -23.094711303710938,
      "activations/layer2_attention_weight_max": 33.49601364135742,
      "activations/layer2_attention_weight_min": -33.321189880371094,
      "activations/layer3_attention_weight_max": 102.62088775634766,
      "activations/layer3_attention_weight_min": -105.49110412597656,
      "activations/layer4_attention_weight_max": 102.57518768310547,
      "activations/layer4_attention_weight_min": -99.37960052490234,
      "activations/layer5_attention_weight_max": 72.98419189453125,
      "activations/layer5_attention_weight_min": -80.88201904296875,
      "activations/layer6_attention_weight_max": 54.07522964477539,
      "activations/layer6_attention_weight_min": -54.35221481323242,
      "activations/layer7_attention_weight_max": 74.69853973388672,
      "activations/layer7_attention_weight_min": -69.08572387695312,
      "activations/layer8_attention_weight_max": 49.56772232055664,
      "activations/layer8_attention_weight_min": -48.72504425048828,
      "activations/layer9_attention_weight_max": 57.54106521606445,
      "activations/layer9_attention_weight_min": -50.36930847167969,
      "epoch": 12.33,
      "learning_rate": 7.118560606060606e-05,
      "loss": 2.7456,
      "step": 212250
    },
    {
      "activations/layer0_attention_weight_max": 14.646096229553223,
      "activations/layer0_attention_weight_min": -13.950153350830078,
      "activations/layer10_attention_weight_max": 40.44376754760742,
      "activations/layer10_attention_weight_min": -37.6099853515625,
      "activations/layer11_attention_weight_max": 34.688987731933594,
      "activations/layer11_attention_weight_min": -34.15498352050781,
      "activations/layer12_attention_weight_max": 18.596113204956055,
      "activations/layer12_attention_weight_min": -23.01310920715332,
      "activations/layer13_attention_weight_max": 41.80312728881836,
      "activations/layer13_attention_weight_min": -36.04560852050781,
      "activations/layer14_attention_weight_max": 36.20705032348633,
      "activations/layer14_attention_weight_min": -34.318355560302734,
      "activations/layer15_attention_weight_max": 34.37862777709961,
      "activations/layer15_attention_weight_min": -30.98661231994629,
      "activations/layer16_attention_weight_max": 33.481781005859375,
      "activations/layer16_attention_weight_min": -32.78595733642578,
      "activations/layer17_attention_weight_max": 53.38308334350586,
      "activations/layer17_attention_weight_min": -47.824371337890625,
      "activations/layer18_attention_weight_max": 45.197208404541016,
      "activations/layer18_attention_weight_min": -40.79579544067383,
      "activations/layer19_attention_weight_max": 23.134044647216797,
      "activations/layer19_attention_weight_min": -23.236835479736328,
      "activations/layer1_attention_weight_max": 16.799888610839844,
      "activations/layer1_attention_weight_min": -14.794946670532227,
      "activations/layer20_attention_weight_max": 22.21148109436035,
      "activations/layer20_attention_weight_min": -21.78152847290039,
      "activations/layer21_attention_weight_max": 36.52280807495117,
      "activations/layer21_attention_weight_min": -24.24650001525879,
      "activations/layer22_attention_weight_max": 28.61829948425293,
      "activations/layer22_attention_weight_min": -27.16204261779785,
      "activations/layer23_attention_weight_max": 36.817447662353516,
      "activations/layer23_attention_weight_min": -24.658344268798828,
      "activations/layer2_attention_weight_max": 34.05754089355469,
      "activations/layer2_attention_weight_min": -31.540586471557617,
      "activations/layer3_attention_weight_max": 100.89143371582031,
      "activations/layer3_attention_weight_min": -94.8303451538086,
      "activations/layer4_attention_weight_max": 103.26836395263672,
      "activations/layer4_attention_weight_min": -95.84663391113281,
      "activations/layer5_attention_weight_max": 74.47344970703125,
      "activations/layer5_attention_weight_min": -78.31515502929688,
      "activations/layer6_attention_weight_max": 57.84373092651367,
      "activations/layer6_attention_weight_min": -55.44032669067383,
      "activations/layer7_attention_weight_max": 73.02205657958984,
      "activations/layer7_attention_weight_min": -70.25850677490234,
      "activations/layer8_attention_weight_max": 51.719173431396484,
      "activations/layer8_attention_weight_min": -51.94614028930664,
      "activations/layer9_attention_weight_max": 52.63248825073242,
      "activations/layer9_attention_weight_min": -47.218502044677734,
      "epoch": 12.34,
      "learning_rate": 7.116666666666666e-05,
      "loss": 2.7677,
      "step": 212300
    },
    {
      "activations/layer0_attention_weight_max": 15.498747825622559,
      "activations/layer0_attention_weight_min": -14.016498565673828,
      "activations/layer10_attention_weight_max": 38.44118881225586,
      "activations/layer10_attention_weight_min": -37.73835372924805,
      "activations/layer11_attention_weight_max": 32.47327423095703,
      "activations/layer11_attention_weight_min": -33.363502502441406,
      "activations/layer12_attention_weight_max": 18.9973087310791,
      "activations/layer12_attention_weight_min": -26.415983200073242,
      "activations/layer13_attention_weight_max": 46.41311264038086,
      "activations/layer13_attention_weight_min": -30.01766586303711,
      "activations/layer14_attention_weight_max": 38.856082916259766,
      "activations/layer14_attention_weight_min": -31.062114715576172,
      "activations/layer15_attention_weight_max": 36.26401901245117,
      "activations/layer15_attention_weight_min": -31.354642868041992,
      "activations/layer16_attention_weight_max": 34.806060791015625,
      "activations/layer16_attention_weight_min": -30.699413299560547,
      "activations/layer17_attention_weight_max": 55.682342529296875,
      "activations/layer17_attention_weight_min": -44.82799530029297,
      "activations/layer18_attention_weight_max": 47.47311782836914,
      "activations/layer18_attention_weight_min": -38.92272186279297,
      "activations/layer19_attention_weight_max": 23.699642181396484,
      "activations/layer19_attention_weight_min": -22.61366081237793,
      "activations/layer1_attention_weight_max": 16.507831573486328,
      "activations/layer1_attention_weight_min": -16.209274291992188,
      "activations/layer20_attention_weight_max": 27.231334686279297,
      "activations/layer20_attention_weight_min": -21.63511848449707,
      "activations/layer21_attention_weight_max": 50.14219284057617,
      "activations/layer21_attention_weight_min": -23.978818893432617,
      "activations/layer22_attention_weight_max": 33.35430145263672,
      "activations/layer22_attention_weight_min": -28.395614624023438,
      "activations/layer23_attention_weight_max": 40.405738830566406,
      "activations/layer23_attention_weight_min": -24.982925415039062,
      "activations/layer2_attention_weight_max": 32.91149139404297,
      "activations/layer2_attention_weight_min": -31.24359703063965,
      "activations/layer3_attention_weight_max": 101.45986938476562,
      "activations/layer3_attention_weight_min": -97.81404113769531,
      "activations/layer4_attention_weight_max": 103.29361724853516,
      "activations/layer4_attention_weight_min": -98.5794906616211,
      "activations/layer5_attention_weight_max": 73.08140563964844,
      "activations/layer5_attention_weight_min": -79.08313751220703,
      "activations/layer6_attention_weight_max": 53.07524490356445,
      "activations/layer6_attention_weight_min": -52.97835159301758,
      "activations/layer7_attention_weight_max": 69.68755340576172,
      "activations/layer7_attention_weight_min": -65.95149993896484,
      "activations/layer8_attention_weight_max": 46.46479797363281,
      "activations/layer8_attention_weight_min": -52.38786697387695,
      "activations/layer9_attention_weight_max": 47.62095642089844,
      "activations/layer9_attention_weight_min": -47.227516174316406,
      "epoch": 12.34,
      "learning_rate": 7.114772727272726e-05,
      "loss": 2.7706,
      "step": 212350
    },
    {
      "activations/layer0_attention_weight_max": 16.074417114257812,
      "activations/layer0_attention_weight_min": -13.935576438903809,
      "activations/layer10_attention_weight_max": 36.83200454711914,
      "activations/layer10_attention_weight_min": -36.533287048339844,
      "activations/layer11_attention_weight_max": 32.82262420654297,
      "activations/layer11_attention_weight_min": -33.25788116455078,
      "activations/layer12_attention_weight_max": 18.71588134765625,
      "activations/layer12_attention_weight_min": -24.105375289916992,
      "activations/layer13_attention_weight_max": 35.59981155395508,
      "activations/layer13_attention_weight_min": -29.404556274414062,
      "activations/layer14_attention_weight_max": 38.98113250732422,
      "activations/layer14_attention_weight_min": -31.13277816772461,
      "activations/layer15_attention_weight_max": 33.43307876586914,
      "activations/layer15_attention_weight_min": -31.079010009765625,
      "activations/layer16_attention_weight_max": 32.53813171386719,
      "activations/layer16_attention_weight_min": -30.022478103637695,
      "activations/layer17_attention_weight_max": 54.72890853881836,
      "activations/layer17_attention_weight_min": -44.12709426879883,
      "activations/layer18_attention_weight_max": 51.19826126098633,
      "activations/layer18_attention_weight_min": -37.45591735839844,
      "activations/layer19_attention_weight_max": 23.007631301879883,
      "activations/layer19_attention_weight_min": -22.403106689453125,
      "activations/layer1_attention_weight_max": 16.707117080688477,
      "activations/layer1_attention_weight_min": -15.756921768188477,
      "activations/layer20_attention_weight_max": 20.437463760375977,
      "activations/layer20_attention_weight_min": -20.812767028808594,
      "activations/layer21_attention_weight_max": 34.115291595458984,
      "activations/layer21_attention_weight_min": -22.27142333984375,
      "activations/layer22_attention_weight_max": 30.68984031677246,
      "activations/layer22_attention_weight_min": -27.549673080444336,
      "activations/layer23_attention_weight_max": 36.99209976196289,
      "activations/layer23_attention_weight_min": -23.999197006225586,
      "activations/layer2_attention_weight_max": 33.523780822753906,
      "activations/layer2_attention_weight_min": -31.80726432800293,
      "activations/layer3_attention_weight_max": 100.18069458007812,
      "activations/layer3_attention_weight_min": -94.95286560058594,
      "activations/layer4_attention_weight_max": 99.12969970703125,
      "activations/layer4_attention_weight_min": -95.3545150756836,
      "activations/layer5_attention_weight_max": 72.27381896972656,
      "activations/layer5_attention_weight_min": -80.35185241699219,
      "activations/layer6_attention_weight_max": 54.31306457519531,
      "activations/layer6_attention_weight_min": -54.97700500488281,
      "activations/layer7_attention_weight_max": 73.11161041259766,
      "activations/layer7_attention_weight_min": -67.06070709228516,
      "activations/layer8_attention_weight_max": 45.636539459228516,
      "activations/layer8_attention_weight_min": -47.54882049560547,
      "activations/layer9_attention_weight_max": 48.05425262451172,
      "activations/layer9_attention_weight_min": -47.3216438293457,
      "epoch": 12.34,
      "learning_rate": 7.112878787878787e-05,
      "loss": 2.7501,
      "step": 212400
    },
    {
      "activations/layer0_attention_weight_max": 16.144969940185547,
      "activations/layer0_attention_weight_min": -14.262906074523926,
      "activations/layer10_attention_weight_max": 37.880977630615234,
      "activations/layer10_attention_weight_min": -37.820255279541016,
      "activations/layer11_attention_weight_max": 32.47932815551758,
      "activations/layer11_attention_weight_min": -32.91437530517578,
      "activations/layer12_attention_weight_max": 17.51053237915039,
      "activations/layer12_attention_weight_min": -26.513294219970703,
      "activations/layer13_attention_weight_max": 37.63704299926758,
      "activations/layer13_attention_weight_min": -28.291948318481445,
      "activations/layer14_attention_weight_max": 40.33340072631836,
      "activations/layer14_attention_weight_min": -29.353349685668945,
      "activations/layer15_attention_weight_max": 33.0383186340332,
      "activations/layer15_attention_weight_min": -29.40082359313965,
      "activations/layer16_attention_weight_max": 30.36608123779297,
      "activations/layer16_attention_weight_min": -29.451221466064453,
      "activations/layer17_attention_weight_max": 48.930416107177734,
      "activations/layer17_attention_weight_min": -43.17715835571289,
      "activations/layer18_attention_weight_max": 45.85502243041992,
      "activations/layer18_attention_weight_min": -37.79289627075195,
      "activations/layer19_attention_weight_max": 21.37782096862793,
      "activations/layer19_attention_weight_min": -22.093902587890625,
      "activations/layer1_attention_weight_max": 16.773038864135742,
      "activations/layer1_attention_weight_min": -16.104598999023438,
      "activations/layer20_attention_weight_max": 21.361186981201172,
      "activations/layer20_attention_weight_min": -22.462940216064453,
      "activations/layer21_attention_weight_max": 35.44316482543945,
      "activations/layer21_attention_weight_min": -26.98116683959961,
      "activations/layer22_attention_weight_max": 28.14611053466797,
      "activations/layer22_attention_weight_min": -24.01552391052246,
      "activations/layer23_attention_weight_max": 37.33124542236328,
      "activations/layer23_attention_weight_min": -24.56635284423828,
      "activations/layer2_attention_weight_max": 33.182254791259766,
      "activations/layer2_attention_weight_min": -31.693052291870117,
      "activations/layer3_attention_weight_max": 98.0019302368164,
      "activations/layer3_attention_weight_min": -99.48335266113281,
      "activations/layer4_attention_weight_max": 100.53022766113281,
      "activations/layer4_attention_weight_min": -96.74590301513672,
      "activations/layer5_attention_weight_max": 74.5979995727539,
      "activations/layer5_attention_weight_min": -81.44786834716797,
      "activations/layer6_attention_weight_max": 53.275630950927734,
      "activations/layer6_attention_weight_min": -53.591026306152344,
      "activations/layer7_attention_weight_max": 69.94600677490234,
      "activations/layer7_attention_weight_min": -66.27873229980469,
      "activations/layer8_attention_weight_max": 47.153133392333984,
      "activations/layer8_attention_weight_min": -48.70840072631836,
      "activations/layer9_attention_weight_max": 47.41851043701172,
      "activations/layer9_attention_weight_min": -47.015228271484375,
      "epoch": 12.34,
      "learning_rate": 7.111022727272726e-05,
      "loss": 2.7395,
      "step": 212450
    },
    {
      "activations/layer0_attention_weight_max": 16.520748138427734,
      "activations/layer0_attention_weight_min": -14.04784107208252,
      "activations/layer10_attention_weight_max": 35.622840881347656,
      "activations/layer10_attention_weight_min": -34.79196548461914,
      "activations/layer11_attention_weight_max": 31.757434844970703,
      "activations/layer11_attention_weight_min": -33.94489669799805,
      "activations/layer12_attention_weight_max": 19.845699310302734,
      "activations/layer12_attention_weight_min": -23.83338165283203,
      "activations/layer13_attention_weight_max": 35.55441665649414,
      "activations/layer13_attention_weight_min": -29.18142318725586,
      "activations/layer14_attention_weight_max": 37.98229217529297,
      "activations/layer14_attention_weight_min": -30.025142669677734,
      "activations/layer15_attention_weight_max": 34.293060302734375,
      "activations/layer15_attention_weight_min": -29.386489868164062,
      "activations/layer16_attention_weight_max": 33.39566421508789,
      "activations/layer16_attention_weight_min": -31.998994827270508,
      "activations/layer17_attention_weight_max": 54.662750244140625,
      "activations/layer17_attention_weight_min": -44.70492172241211,
      "activations/layer18_attention_weight_max": 49.00965118408203,
      "activations/layer18_attention_weight_min": -38.26628875732422,
      "activations/layer19_attention_weight_max": 22.970752716064453,
      "activations/layer19_attention_weight_min": -21.730314254760742,
      "activations/layer1_attention_weight_max": 15.765750885009766,
      "activations/layer1_attention_weight_min": -16.20936393737793,
      "activations/layer20_attention_weight_max": 23.91020965576172,
      "activations/layer20_attention_weight_min": -22.2890625,
      "activations/layer21_attention_weight_max": 35.08436584472656,
      "activations/layer21_attention_weight_min": -25.694610595703125,
      "activations/layer22_attention_weight_max": 31.656492233276367,
      "activations/layer22_attention_weight_min": -26.75113868713379,
      "activations/layer23_attention_weight_max": 39.19594192504883,
      "activations/layer23_attention_weight_min": -24.641559600830078,
      "activations/layer2_attention_weight_max": 32.033843994140625,
      "activations/layer2_attention_weight_min": -30.77237319946289,
      "activations/layer3_attention_weight_max": 92.99357604980469,
      "activations/layer3_attention_weight_min": -93.64771270751953,
      "activations/layer4_attention_weight_max": 97.92147064208984,
      "activations/layer4_attention_weight_min": -93.8306655883789,
      "activations/layer5_attention_weight_max": 69.60430908203125,
      "activations/layer5_attention_weight_min": -76.52613067626953,
      "activations/layer6_attention_weight_max": 50.54465103149414,
      "activations/layer6_attention_weight_min": -52.78260803222656,
      "activations/layer7_attention_weight_max": 66.70294952392578,
      "activations/layer7_attention_weight_min": -65.59466552734375,
      "activations/layer8_attention_weight_max": 46.00163650512695,
      "activations/layer8_attention_weight_min": -45.91384506225586,
      "activations/layer9_attention_weight_max": 44.0361328125,
      "activations/layer9_attention_weight_min": -46.7922477722168,
      "epoch": 12.35,
      "learning_rate": 7.109128787878787e-05,
      "loss": 2.7711,
      "step": 212500
    },
    {
      "activations/layer0_attention_weight_max": 17.1702823638916,
      "activations/layer0_attention_weight_min": -14.17715835571289,
      "activations/layer10_attention_weight_max": 43.83220291137695,
      "activations/layer10_attention_weight_min": -40.720741271972656,
      "activations/layer11_attention_weight_max": 37.5025520324707,
      "activations/layer11_attention_weight_min": -38.105506896972656,
      "activations/layer12_attention_weight_max": 24.37505340576172,
      "activations/layer12_attention_weight_min": -24.19620132446289,
      "activations/layer13_attention_weight_max": 36.435997009277344,
      "activations/layer13_attention_weight_min": -27.82610321044922,
      "activations/layer14_attention_weight_max": 40.385154724121094,
      "activations/layer14_attention_weight_min": -32.653656005859375,
      "activations/layer15_attention_weight_max": 39.53416442871094,
      "activations/layer15_attention_weight_min": -32.197330474853516,
      "activations/layer16_attention_weight_max": 36.14313507080078,
      "activations/layer16_attention_weight_min": -32.1480598449707,
      "activations/layer17_attention_weight_max": 63.06105422973633,
      "activations/layer17_attention_weight_min": -48.92948913574219,
      "activations/layer18_attention_weight_max": 58.91733169555664,
      "activations/layer18_attention_weight_min": -40.259254455566406,
      "activations/layer19_attention_weight_max": 23.45399284362793,
      "activations/layer19_attention_weight_min": -20.940420150756836,
      "activations/layer1_attention_weight_max": 17.5216064453125,
      "activations/layer1_attention_weight_min": -14.866230010986328,
      "activations/layer20_attention_weight_max": 24.281879425048828,
      "activations/layer20_attention_weight_min": -19.85484504699707,
      "activations/layer21_attention_weight_max": 38.787742614746094,
      "activations/layer21_attention_weight_min": -23.934988021850586,
      "activations/layer22_attention_weight_max": 32.722442626953125,
      "activations/layer22_attention_weight_min": -24.701488494873047,
      "activations/layer23_attention_weight_max": 41.0117073059082,
      "activations/layer23_attention_weight_min": -24.214126586914062,
      "activations/layer2_attention_weight_max": 32.104469299316406,
      "activations/layer2_attention_weight_min": -31.860464096069336,
      "activations/layer3_attention_weight_max": 98.78079223632812,
      "activations/layer3_attention_weight_min": -101.87327575683594,
      "activations/layer4_attention_weight_max": 104.30519104003906,
      "activations/layer4_attention_weight_min": -100.56675720214844,
      "activations/layer5_attention_weight_max": 74.48051452636719,
      "activations/layer5_attention_weight_min": -83.25250244140625,
      "activations/layer6_attention_weight_max": 56.31501388549805,
      "activations/layer6_attention_weight_min": -56.58865737915039,
      "activations/layer7_attention_weight_max": 74.37234497070312,
      "activations/layer7_attention_weight_min": -71.73084259033203,
      "activations/layer8_attention_weight_max": 51.22917556762695,
      "activations/layer8_attention_weight_min": -50.518341064453125,
      "activations/layer9_attention_weight_max": 51.47541427612305,
      "activations/layer9_attention_weight_min": -50.17991638183594,
      "epoch": 12.35,
      "learning_rate": 7.107234848484848e-05,
      "loss": 2.7568,
      "step": 212550
    },
    {
      "activations/layer0_attention_weight_max": 16.589139938354492,
      "activations/layer0_attention_weight_min": -13.764021873474121,
      "activations/layer10_attention_weight_max": 38.28465270996094,
      "activations/layer10_attention_weight_min": -36.83695983886719,
      "activations/layer11_attention_weight_max": 34.066650390625,
      "activations/layer11_attention_weight_min": -33.74414825439453,
      "activations/layer12_attention_weight_max": 20.123733520507812,
      "activations/layer12_attention_weight_min": -22.84164047241211,
      "activations/layer13_attention_weight_max": 38.012359619140625,
      "activations/layer13_attention_weight_min": -31.87465476989746,
      "activations/layer14_attention_weight_max": 35.698219299316406,
      "activations/layer14_attention_weight_min": -31.71135139465332,
      "activations/layer15_attention_weight_max": 34.5984001159668,
      "activations/layer15_attention_weight_min": -30.20292854309082,
      "activations/layer16_attention_weight_max": 33.48417663574219,
      "activations/layer16_attention_weight_min": -31.637479782104492,
      "activations/layer17_attention_weight_max": 57.33279800415039,
      "activations/layer17_attention_weight_min": -46.603843688964844,
      "activations/layer18_attention_weight_max": 52.3713493347168,
      "activations/layer18_attention_weight_min": -41.947593688964844,
      "activations/layer19_attention_weight_max": 23.714387893676758,
      "activations/layer19_attention_weight_min": -25.77365493774414,
      "activations/layer1_attention_weight_max": 15.925994873046875,
      "activations/layer1_attention_weight_min": -17.019533157348633,
      "activations/layer20_attention_weight_max": 26.60686492919922,
      "activations/layer20_attention_weight_min": -24.367565155029297,
      "activations/layer21_attention_weight_max": 44.229888916015625,
      "activations/layer21_attention_weight_min": -22.0023193359375,
      "activations/layer22_attention_weight_max": 33.14702224731445,
      "activations/layer22_attention_weight_min": -25.170543670654297,
      "activations/layer23_attention_weight_max": 45.65080261230469,
      "activations/layer23_attention_weight_min": -23.058677673339844,
      "activations/layer2_attention_weight_max": 33.84688949584961,
      "activations/layer2_attention_weight_min": -32.67359924316406,
      "activations/layer3_attention_weight_max": 97.4823989868164,
      "activations/layer3_attention_weight_min": -100.35508728027344,
      "activations/layer4_attention_weight_max": 105.88519287109375,
      "activations/layer4_attention_weight_min": -104.45946502685547,
      "activations/layer5_attention_weight_max": 70.0849609375,
      "activations/layer5_attention_weight_min": -77.27122497558594,
      "activations/layer6_attention_weight_max": 55.727848052978516,
      "activations/layer6_attention_weight_min": -53.80235290527344,
      "activations/layer7_attention_weight_max": 73.93391418457031,
      "activations/layer7_attention_weight_min": -69.8233871459961,
      "activations/layer8_attention_weight_max": 47.39043426513672,
      "activations/layer8_attention_weight_min": -49.69721603393555,
      "activations/layer9_attention_weight_max": 52.00294876098633,
      "activations/layer9_attention_weight_min": -49.94367599487305,
      "epoch": 12.35,
      "learning_rate": 7.105340909090908e-05,
      "loss": 2.7711,
      "step": 212600
    },
    {
      "activations/layer0_attention_weight_max": 15.438037872314453,
      "activations/layer0_attention_weight_min": -13.945551872253418,
      "activations/layer10_attention_weight_max": 35.79108428955078,
      "activations/layer10_attention_weight_min": -35.283363342285156,
      "activations/layer11_attention_weight_max": 30.715576171875,
      "activations/layer11_attention_weight_min": -31.218791961669922,
      "activations/layer12_attention_weight_max": 18.27408790588379,
      "activations/layer12_attention_weight_min": -29.23426628112793,
      "activations/layer13_attention_weight_max": 34.52932357788086,
      "activations/layer13_attention_weight_min": -27.27338981628418,
      "activations/layer14_attention_weight_max": 36.60297393798828,
      "activations/layer14_attention_weight_min": -35.32384490966797,
      "activations/layer15_attention_weight_max": 34.015682220458984,
      "activations/layer15_attention_weight_min": -32.18686294555664,
      "activations/layer16_attention_weight_max": 32.02708053588867,
      "activations/layer16_attention_weight_min": -30.548274993896484,
      "activations/layer17_attention_weight_max": 56.50675964355469,
      "activations/layer17_attention_weight_min": -46.08623123168945,
      "activations/layer18_attention_weight_max": 47.38206100463867,
      "activations/layer18_attention_weight_min": -41.0615234375,
      "activations/layer19_attention_weight_max": 21.745210647583008,
      "activations/layer19_attention_weight_min": -23.406164169311523,
      "activations/layer1_attention_weight_max": 15.684895515441895,
      "activations/layer1_attention_weight_min": -15.21959400177002,
      "activations/layer20_attention_weight_max": 22.57286834716797,
      "activations/layer20_attention_weight_min": -22.437557220458984,
      "activations/layer21_attention_weight_max": 36.01968765258789,
      "activations/layer21_attention_weight_min": -24.46160888671875,
      "activations/layer22_attention_weight_max": 32.34759521484375,
      "activations/layer22_attention_weight_min": -25.83972930908203,
      "activations/layer23_attention_weight_max": 39.91407012939453,
      "activations/layer23_attention_weight_min": -24.11505126953125,
      "activations/layer2_attention_weight_max": 32.89115905761719,
      "activations/layer2_attention_weight_min": -31.225006103515625,
      "activations/layer3_attention_weight_max": 98.43257904052734,
      "activations/layer3_attention_weight_min": -98.94821166992188,
      "activations/layer4_attention_weight_max": 100.08522033691406,
      "activations/layer4_attention_weight_min": -96.32398986816406,
      "activations/layer5_attention_weight_max": 72.10693359375,
      "activations/layer5_attention_weight_min": -78.06642150878906,
      "activations/layer6_attention_weight_max": 53.57151794433594,
      "activations/layer6_attention_weight_min": -53.01994705200195,
      "activations/layer7_attention_weight_max": 68.04369354248047,
      "activations/layer7_attention_weight_min": -65.86174774169922,
      "activations/layer8_attention_weight_max": 45.55097579956055,
      "activations/layer8_attention_weight_min": -46.474281311035156,
      "activations/layer9_attention_weight_max": 45.83013916015625,
      "activations/layer9_attention_weight_min": -46.05904006958008,
      "epoch": 12.36,
      "learning_rate": 7.10344696969697e-05,
      "loss": 2.7465,
      "step": 212650
    },
    {
      "activations/layer0_attention_weight_max": 16.293073654174805,
      "activations/layer0_attention_weight_min": -13.948373794555664,
      "activations/layer10_attention_weight_max": 39.3140754699707,
      "activations/layer10_attention_weight_min": -36.7236328125,
      "activations/layer11_attention_weight_max": 33.764007568359375,
      "activations/layer11_attention_weight_min": -33.70359802246094,
      "activations/layer12_attention_weight_max": 18.842348098754883,
      "activations/layer12_attention_weight_min": -28.564437866210938,
      "activations/layer13_attention_weight_max": 41.50783157348633,
      "activations/layer13_attention_weight_min": -30.02989387512207,
      "activations/layer14_attention_weight_max": 37.22545623779297,
      "activations/layer14_attention_weight_min": -31.86087989807129,
      "activations/layer15_attention_weight_max": 34.87746047973633,
      "activations/layer15_attention_weight_min": -29.761032104492188,
      "activations/layer16_attention_weight_max": 33.930667877197266,
      "activations/layer16_attention_weight_min": -30.032230377197266,
      "activations/layer17_attention_weight_max": 56.63083267211914,
      "activations/layer17_attention_weight_min": -47.123226165771484,
      "activations/layer18_attention_weight_max": 49.82889938354492,
      "activations/layer18_attention_weight_min": -39.83450698852539,
      "activations/layer19_attention_weight_max": 24.37864875793457,
      "activations/layer19_attention_weight_min": -23.310611724853516,
      "activations/layer1_attention_weight_max": 16.814292907714844,
      "activations/layer1_attention_weight_min": -15.411794662475586,
      "activations/layer20_attention_weight_max": 26.42446517944336,
      "activations/layer20_attention_weight_min": -24.800514221191406,
      "activations/layer21_attention_weight_max": 48.85942077636719,
      "activations/layer21_attention_weight_min": -24.119915008544922,
      "activations/layer22_attention_weight_max": 33.079010009765625,
      "activations/layer22_attention_weight_min": -27.829790115356445,
      "activations/layer23_attention_weight_max": 36.65296936035156,
      "activations/layer23_attention_weight_min": -26.92306900024414,
      "activations/layer2_attention_weight_max": 34.538944244384766,
      "activations/layer2_attention_weight_min": -31.480392456054688,
      "activations/layer3_attention_weight_max": 97.7802734375,
      "activations/layer3_attention_weight_min": -99.46495056152344,
      "activations/layer4_attention_weight_max": 99.6854476928711,
      "activations/layer4_attention_weight_min": -95.9651870727539,
      "activations/layer5_attention_weight_max": 70.91786193847656,
      "activations/layer5_attention_weight_min": -79.19866943359375,
      "activations/layer6_attention_weight_max": 54.71590042114258,
      "activations/layer6_attention_weight_min": -51.45787811279297,
      "activations/layer7_attention_weight_max": 69.17891693115234,
      "activations/layer7_attention_weight_min": -67.65454864501953,
      "activations/layer8_attention_weight_max": 46.58940124511719,
      "activations/layer8_attention_weight_min": -46.39189910888672,
      "activations/layer9_attention_weight_max": 48.626251220703125,
      "activations/layer9_attention_weight_min": -47.629798889160156,
      "epoch": 12.36,
      "learning_rate": 7.10155303030303e-05,
      "loss": 2.7818,
      "step": 212700
    },
    {
      "activations/layer0_attention_weight_max": 15.973014831542969,
      "activations/layer0_attention_weight_min": -14.284005165100098,
      "activations/layer10_attention_weight_max": 42.970611572265625,
      "activations/layer10_attention_weight_min": -38.131221771240234,
      "activations/layer11_attention_weight_max": 40.86459732055664,
      "activations/layer11_attention_weight_min": -36.32597732543945,
      "activations/layer12_attention_weight_max": 18.22356414794922,
      "activations/layer12_attention_weight_min": -28.208961486816406,
      "activations/layer13_attention_weight_max": 37.813167572021484,
      "activations/layer13_attention_weight_min": -28.592370986938477,
      "activations/layer14_attention_weight_max": 39.246063232421875,
      "activations/layer14_attention_weight_min": -30.069520950317383,
      "activations/layer15_attention_weight_max": 35.2548713684082,
      "activations/layer15_attention_weight_min": -28.75775909423828,
      "activations/layer16_attention_weight_max": 33.02678680419922,
      "activations/layer16_attention_weight_min": -30.309797286987305,
      "activations/layer17_attention_weight_max": 57.23942184448242,
      "activations/layer17_attention_weight_min": -44.40463638305664,
      "activations/layer18_attention_weight_max": 47.01241683959961,
      "activations/layer18_attention_weight_min": -39.4964714050293,
      "activations/layer19_attention_weight_max": 22.635177612304688,
      "activations/layer19_attention_weight_min": -22.00654411315918,
      "activations/layer1_attention_weight_max": 17.520469665527344,
      "activations/layer1_attention_weight_min": -16.164958953857422,
      "activations/layer20_attention_weight_max": 21.5285701751709,
      "activations/layer20_attention_weight_min": -21.29588508605957,
      "activations/layer21_attention_weight_max": 38.716922760009766,
      "activations/layer21_attention_weight_min": -21.742212295532227,
      "activations/layer22_attention_weight_max": 27.553728103637695,
      "activations/layer22_attention_weight_min": -25.818803787231445,
      "activations/layer23_attention_weight_max": 31.639083862304688,
      "activations/layer23_attention_weight_min": -21.993566513061523,
      "activations/layer2_attention_weight_max": 37.6634635925293,
      "activations/layer2_attention_weight_min": -34.154640197753906,
      "activations/layer3_attention_weight_max": 105.79074096679688,
      "activations/layer3_attention_weight_min": -102.02196502685547,
      "activations/layer4_attention_weight_max": 106.26134490966797,
      "activations/layer4_attention_weight_min": -101.19772338867188,
      "activations/layer5_attention_weight_max": 76.57135772705078,
      "activations/layer5_attention_weight_min": -79.27020263671875,
      "activations/layer6_attention_weight_max": 60.2169189453125,
      "activations/layer6_attention_weight_min": -54.79643630981445,
      "activations/layer7_attention_weight_max": 81.34487915039062,
      "activations/layer7_attention_weight_min": -68.69889831542969,
      "activations/layer8_attention_weight_max": 56.624942779541016,
      "activations/layer8_attention_weight_min": -52.636817932128906,
      "activations/layer9_attention_weight_max": 61.29368209838867,
      "activations/layer9_attention_weight_min": -53.130794525146484,
      "epoch": 12.36,
      "learning_rate": 7.09965909090909e-05,
      "loss": 2.766,
      "step": 212750
    },
    {
      "activations/layer0_attention_weight_max": 16.583595275878906,
      "activations/layer0_attention_weight_min": -13.970317840576172,
      "activations/layer10_attention_weight_max": 37.046142578125,
      "activations/layer10_attention_weight_min": -38.96916580200195,
      "activations/layer11_attention_weight_max": 32.582557678222656,
      "activations/layer11_attention_weight_min": -33.98429870605469,
      "activations/layer12_attention_weight_max": 16.317501068115234,
      "activations/layer12_attention_weight_min": -23.6469669342041,
      "activations/layer13_attention_weight_max": 41.056636810302734,
      "activations/layer13_attention_weight_min": -34.675384521484375,
      "activations/layer14_attention_weight_max": 42.09220504760742,
      "activations/layer14_attention_weight_min": -33.0882568359375,
      "activations/layer15_attention_weight_max": 38.23351287841797,
      "activations/layer15_attention_weight_min": -31.781757354736328,
      "activations/layer16_attention_weight_max": 31.173595428466797,
      "activations/layer16_attention_weight_min": -30.335309982299805,
      "activations/layer17_attention_weight_max": 51.449684143066406,
      "activations/layer17_attention_weight_min": -45.46757888793945,
      "activations/layer18_attention_weight_max": 46.90996551513672,
      "activations/layer18_attention_weight_min": -36.83979797363281,
      "activations/layer19_attention_weight_max": 21.884164810180664,
      "activations/layer19_attention_weight_min": -21.37726402282715,
      "activations/layer1_attention_weight_max": 17.82899284362793,
      "activations/layer1_attention_weight_min": -19.275630950927734,
      "activations/layer20_attention_weight_max": 21.350229263305664,
      "activations/layer20_attention_weight_min": -21.999555587768555,
      "activations/layer21_attention_weight_max": 38.9379997253418,
      "activations/layer21_attention_weight_min": -21.642126083374023,
      "activations/layer22_attention_weight_max": 29.39400291442871,
      "activations/layer22_attention_weight_min": -25.8367977142334,
      "activations/layer23_attention_weight_max": 38.63246536254883,
      "activations/layer23_attention_weight_min": -24.813331604003906,
      "activations/layer2_attention_weight_max": 34.197509765625,
      "activations/layer2_attention_weight_min": -34.30354309082031,
      "activations/layer3_attention_weight_max": 101.0584487915039,
      "activations/layer3_attention_weight_min": -102.0250015258789,
      "activations/layer4_attention_weight_max": 105.684326171875,
      "activations/layer4_attention_weight_min": -105.26844787597656,
      "activations/layer5_attention_weight_max": 77.65130615234375,
      "activations/layer5_attention_weight_min": -80.64423370361328,
      "activations/layer6_attention_weight_max": 54.52385330200195,
      "activations/layer6_attention_weight_min": -56.70596694946289,
      "activations/layer7_attention_weight_max": 74.7438735961914,
      "activations/layer7_attention_weight_min": -72.08890533447266,
      "activations/layer8_attention_weight_max": 49.12102508544922,
      "activations/layer8_attention_weight_min": -53.76197814941406,
      "activations/layer9_attention_weight_max": 51.415557861328125,
      "activations/layer9_attention_weight_min": -53.765724182128906,
      "epoch": 12.36,
      "learning_rate": 7.097765151515152e-05,
      "loss": 2.7583,
      "step": 212800
    },
    {
      "activations/layer0_attention_weight_max": 15.644536972045898,
      "activations/layer0_attention_weight_min": -14.268839836120605,
      "activations/layer10_attention_weight_max": 41.01114273071289,
      "activations/layer10_attention_weight_min": -37.38737869262695,
      "activations/layer11_attention_weight_max": 33.86363983154297,
      "activations/layer11_attention_weight_min": -34.925228118896484,
      "activations/layer12_attention_weight_max": 18.779874801635742,
      "activations/layer12_attention_weight_min": -27.120492935180664,
      "activations/layer13_attention_weight_max": 40.04090881347656,
      "activations/layer13_attention_weight_min": -29.18263816833496,
      "activations/layer14_attention_weight_max": 40.803104400634766,
      "activations/layer14_attention_weight_min": -30.86573028564453,
      "activations/layer15_attention_weight_max": 32.3004264831543,
      "activations/layer15_attention_weight_min": -29.0961856842041,
      "activations/layer16_attention_weight_max": 32.37217712402344,
      "activations/layer16_attention_weight_min": -29.498310089111328,
      "activations/layer17_attention_weight_max": 50.55134582519531,
      "activations/layer17_attention_weight_min": -43.39780044555664,
      "activations/layer18_attention_weight_max": 46.1224365234375,
      "activations/layer18_attention_weight_min": -38.82904815673828,
      "activations/layer19_attention_weight_max": 23.847610473632812,
      "activations/layer19_attention_weight_min": -20.385177612304688,
      "activations/layer1_attention_weight_max": 16.07340431213379,
      "activations/layer1_attention_weight_min": -14.867118835449219,
      "activations/layer20_attention_weight_max": 25.15624237060547,
      "activations/layer20_attention_weight_min": -25.652257919311523,
      "activations/layer21_attention_weight_max": 39.92210006713867,
      "activations/layer21_attention_weight_min": -21.729684829711914,
      "activations/layer22_attention_weight_max": 32.6901969909668,
      "activations/layer22_attention_weight_min": -26.35610008239746,
      "activations/layer23_attention_weight_max": 40.26380157470703,
      "activations/layer23_attention_weight_min": -25.64583969116211,
      "activations/layer2_attention_weight_max": 32.43956756591797,
      "activations/layer2_attention_weight_min": -32.160186767578125,
      "activations/layer3_attention_weight_max": 95.5983657836914,
      "activations/layer3_attention_weight_min": -99.7253646850586,
      "activations/layer4_attention_weight_max": 100.52503967285156,
      "activations/layer4_attention_weight_min": -98.79881286621094,
      "activations/layer5_attention_weight_max": 74.25546264648438,
      "activations/layer5_attention_weight_min": -78.07716369628906,
      "activations/layer6_attention_weight_max": 56.92880630493164,
      "activations/layer6_attention_weight_min": -53.390750885009766,
      "activations/layer7_attention_weight_max": 69.32757568359375,
      "activations/layer7_attention_weight_min": -68.56898498535156,
      "activations/layer8_attention_weight_max": 46.89783477783203,
      "activations/layer8_attention_weight_min": -50.15003967285156,
      "activations/layer9_attention_weight_max": 48.14863967895508,
      "activations/layer9_attention_weight_min": -46.45585250854492,
      "epoch": 12.37,
      "learning_rate": 7.095871212121212e-05,
      "loss": 2.7704,
      "step": 212850
    },
    {
      "activations/layer0_attention_weight_max": 16.54554557800293,
      "activations/layer0_attention_weight_min": -13.791389465332031,
      "activations/layer10_attention_weight_max": 36.3140754699707,
      "activations/layer10_attention_weight_min": -39.19289016723633,
      "activations/layer11_attention_weight_max": 32.13125991821289,
      "activations/layer11_attention_weight_min": -33.0041618347168,
      "activations/layer12_attention_weight_max": 28.734865188598633,
      "activations/layer12_attention_weight_min": -26.06546401977539,
      "activations/layer13_attention_weight_max": 36.883758544921875,
      "activations/layer13_attention_weight_min": -26.848615646362305,
      "activations/layer14_attention_weight_max": 35.95949935913086,
      "activations/layer14_attention_weight_min": -29.178016662597656,
      "activations/layer15_attention_weight_max": 32.237091064453125,
      "activations/layer15_attention_weight_min": -28.826196670532227,
      "activations/layer16_attention_weight_max": 32.76156234741211,
      "activations/layer16_attention_weight_min": -31.40694236755371,
      "activations/layer17_attention_weight_max": 53.241188049316406,
      "activations/layer17_attention_weight_min": -44.552520751953125,
      "activations/layer18_attention_weight_max": 47.91576385498047,
      "activations/layer18_attention_weight_min": -37.852230072021484,
      "activations/layer19_attention_weight_max": 22.50373077392578,
      "activations/layer19_attention_weight_min": -24.875120162963867,
      "activations/layer1_attention_weight_max": 16.28366470336914,
      "activations/layer1_attention_weight_min": -14.305001258850098,
      "activations/layer20_attention_weight_max": 25.479795455932617,
      "activations/layer20_attention_weight_min": -25.007572174072266,
      "activations/layer21_attention_weight_max": 42.37671661376953,
      "activations/layer21_attention_weight_min": -24.475244522094727,
      "activations/layer22_attention_weight_max": 31.65320587158203,
      "activations/layer22_attention_weight_min": -25.394691467285156,
      "activations/layer23_attention_weight_max": 38.54081344604492,
      "activations/layer23_attention_weight_min": -25.529464721679688,
      "activations/layer2_attention_weight_max": 34.479942321777344,
      "activations/layer2_attention_weight_min": -34.43848419189453,
      "activations/layer3_attention_weight_max": 100.32254791259766,
      "activations/layer3_attention_weight_min": -105.64922332763672,
      "activations/layer4_attention_weight_max": 105.8392333984375,
      "activations/layer4_attention_weight_min": -102.91448974609375,
      "activations/layer5_attention_weight_max": 74.72241973876953,
      "activations/layer5_attention_weight_min": -82.20480346679688,
      "activations/layer6_attention_weight_max": 57.670684814453125,
      "activations/layer6_attention_weight_min": -57.273101806640625,
      "activations/layer7_attention_weight_max": 69.2204360961914,
      "activations/layer7_attention_weight_min": -70.11857604980469,
      "activations/layer8_attention_weight_max": 44.2037239074707,
      "activations/layer8_attention_weight_min": -46.494873046875,
      "activations/layer9_attention_weight_max": 46.6199836730957,
      "activations/layer9_attention_weight_min": -49.94288635253906,
      "epoch": 12.37,
      "learning_rate": 7.093977272727272e-05,
      "loss": 2.7615,
      "step": 212900
    },
    {
      "activations/layer0_attention_weight_max": 16.523149490356445,
      "activations/layer0_attention_weight_min": -13.630165100097656,
      "activations/layer10_attention_weight_max": 37.03632354736328,
      "activations/layer10_attention_weight_min": -37.17975616455078,
      "activations/layer11_attention_weight_max": 32.967201232910156,
      "activations/layer11_attention_weight_min": -34.38859558105469,
      "activations/layer12_attention_weight_max": 21.693323135375977,
      "activations/layer12_attention_weight_min": -24.561363220214844,
      "activations/layer13_attention_weight_max": 39.4478759765625,
      "activations/layer13_attention_weight_min": -29.214553833007812,
      "activations/layer14_attention_weight_max": 50.794551849365234,
      "activations/layer14_attention_weight_min": -33.457557678222656,
      "activations/layer15_attention_weight_max": 40.09302520751953,
      "activations/layer15_attention_weight_min": -31.283063888549805,
      "activations/layer16_attention_weight_max": 35.74604797363281,
      "activations/layer16_attention_weight_min": -32.73705291748047,
      "activations/layer17_attention_weight_max": 59.40625,
      "activations/layer17_attention_weight_min": -47.870826721191406,
      "activations/layer18_attention_weight_max": 54.8918342590332,
      "activations/layer18_attention_weight_min": -42.64257049560547,
      "activations/layer19_attention_weight_max": 28.868484497070312,
      "activations/layer19_attention_weight_min": -24.902318954467773,
      "activations/layer1_attention_weight_max": 16.086380004882812,
      "activations/layer1_attention_weight_min": -14.42632007598877,
      "activations/layer20_attention_weight_max": 28.89461326599121,
      "activations/layer20_attention_weight_min": -23.308269500732422,
      "activations/layer21_attention_weight_max": 46.59018325805664,
      "activations/layer21_attention_weight_min": -25.471323013305664,
      "activations/layer22_attention_weight_max": 38.62059783935547,
      "activations/layer22_attention_weight_min": -27.453922271728516,
      "activations/layer23_attention_weight_max": 45.11719512939453,
      "activations/layer23_attention_weight_min": -27.864301681518555,
      "activations/layer2_attention_weight_max": 32.373355865478516,
      "activations/layer2_attention_weight_min": -32.84852981567383,
      "activations/layer3_attention_weight_max": 92.55230712890625,
      "activations/layer3_attention_weight_min": -97.6765365600586,
      "activations/layer4_attention_weight_max": 93.50582122802734,
      "activations/layer4_attention_weight_min": -97.47010803222656,
      "activations/layer5_attention_weight_max": 67.10832214355469,
      "activations/layer5_attention_weight_min": -83.25856018066406,
      "activations/layer6_attention_weight_max": 51.75008773803711,
      "activations/layer6_attention_weight_min": -53.33903884887695,
      "activations/layer7_attention_weight_max": 69.18985748291016,
      "activations/layer7_attention_weight_min": -67.20508575439453,
      "activations/layer8_attention_weight_max": 43.826839447021484,
      "activations/layer8_attention_weight_min": -47.371707916259766,
      "activations/layer9_attention_weight_max": 47.175716400146484,
      "activations/layer9_attention_weight_min": -48.49796676635742,
      "epoch": 12.37,
      "learning_rate": 7.092083333333332e-05,
      "loss": 2.7614,
      "step": 212950
    },
    {
      "activations/layer0_attention_weight_max": 17.7843017578125,
      "activations/layer0_attention_weight_min": -14.110295295715332,
      "activations/layer10_attention_weight_max": 33.4892692565918,
      "activations/layer10_attention_weight_min": -34.03231430053711,
      "activations/layer11_attention_weight_max": 30.080577850341797,
      "activations/layer11_attention_weight_min": -32.37996292114258,
      "activations/layer12_attention_weight_max": 17.215669631958008,
      "activations/layer12_attention_weight_min": -24.564651489257812,
      "activations/layer13_attention_weight_max": 34.643062591552734,
      "activations/layer13_attention_weight_min": -28.868881225585938,
      "activations/layer14_attention_weight_max": 34.04187774658203,
      "activations/layer14_attention_weight_min": -29.193462371826172,
      "activations/layer15_attention_weight_max": 30.32611656188965,
      "activations/layer15_attention_weight_min": -29.36073112487793,
      "activations/layer16_attention_weight_max": 29.274560928344727,
      "activations/layer16_attention_weight_min": -28.675582885742188,
      "activations/layer17_attention_weight_max": 49.32954788208008,
      "activations/layer17_attention_weight_min": -42.65800857543945,
      "activations/layer18_attention_weight_max": 44.157203674316406,
      "activations/layer18_attention_weight_min": -37.512298583984375,
      "activations/layer19_attention_weight_max": 22.116405487060547,
      "activations/layer19_attention_weight_min": -21.86273956298828,
      "activations/layer1_attention_weight_max": 16.668075561523438,
      "activations/layer1_attention_weight_min": -14.829957008361816,
      "activations/layer20_attention_weight_max": 22.987346649169922,
      "activations/layer20_attention_weight_min": -27.609336853027344,
      "activations/layer21_attention_weight_max": 37.773681640625,
      "activations/layer21_attention_weight_min": -24.15706443786621,
      "activations/layer22_attention_weight_max": 28.949644088745117,
      "activations/layer22_attention_weight_min": -25.73427963256836,
      "activations/layer23_attention_weight_max": 33.63455581665039,
      "activations/layer23_attention_weight_min": -23.150394439697266,
      "activations/layer2_attention_weight_max": 33.898597717285156,
      "activations/layer2_attention_weight_min": -31.830360412597656,
      "activations/layer3_attention_weight_max": 94.53788757324219,
      "activations/layer3_attention_weight_min": -94.46461486816406,
      "activations/layer4_attention_weight_max": 96.39665222167969,
      "activations/layer4_attention_weight_min": -90.4001693725586,
      "activations/layer5_attention_weight_max": 70.13838195800781,
      "activations/layer5_attention_weight_min": -77.84938049316406,
      "activations/layer6_attention_weight_max": 54.46030807495117,
      "activations/layer6_attention_weight_min": -53.003543853759766,
      "activations/layer7_attention_weight_max": 69.21014404296875,
      "activations/layer7_attention_weight_min": -64.93144226074219,
      "activations/layer8_attention_weight_max": 44.90620803833008,
      "activations/layer8_attention_weight_min": -49.66913986206055,
      "activations/layer9_attention_weight_max": 45.314178466796875,
      "activations/layer9_attention_weight_min": -45.511314392089844,
      "epoch": 12.38,
      "learning_rate": 7.090189393939394e-05,
      "loss": 2.7632,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_loss": 2.71875,
      "eval_runtime": 8.5239,
      "eval_samples_per_second": 503.759,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.5239,
      "eval_openwebtext_samples_per_second": 503.759,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_wikitext_loss": 2.953125,
      "eval_wikitext_ppl": 19.165753221675935,
      "eval_wikitext_runtime": 2.0116,
      "eval_wikitext_samples_per_second": 226.689,
      "step": 213000
    },
    {
      "epoch": 12.38,
      "eval_lambada_loss": 2.525390625,
      "eval_lambada_ppl": 12.495775472672934,
      "eval_lambada_runtime": 9.6047,
      "eval_lambada_samples_per_second": 506.938,
      "step": 213000
    },
    {
      "activations/layer0_attention_weight_max": 15.621589660644531,
      "activations/layer0_attention_weight_min": -13.581889152526855,
      "activations/layer10_attention_weight_max": 37.881744384765625,
      "activations/layer10_attention_weight_min": -36.56967544555664,
      "activations/layer11_attention_weight_max": 38.22367858886719,
      "activations/layer11_attention_weight_min": -36.3484992980957,
      "activations/layer12_attention_weight_max": 18.945093154907227,
      "activations/layer12_attention_weight_min": -26.56009864807129,
      "activations/layer13_attention_weight_max": 33.85911178588867,
      "activations/layer13_attention_weight_min": -28.306079864501953,
      "activations/layer14_attention_weight_max": 37.8326530456543,
      "activations/layer14_attention_weight_min": -31.251434326171875,
      "activations/layer15_attention_weight_max": 32.633636474609375,
      "activations/layer15_attention_weight_min": -30.046998977661133,
      "activations/layer16_attention_weight_max": 31.748754501342773,
      "activations/layer16_attention_weight_min": -30.14182472229004,
      "activations/layer17_attention_weight_max": 53.093528747558594,
      "activations/layer17_attention_weight_min": -45.45635986328125,
      "activations/layer18_attention_weight_max": 44.20876693725586,
      "activations/layer18_attention_weight_min": -36.91404342651367,
      "activations/layer19_attention_weight_max": 22.584163665771484,
      "activations/layer19_attention_weight_min": -21.78241539001465,
      "activations/layer1_attention_weight_max": 16.097925186157227,
      "activations/layer1_attention_weight_min": -17.02191734313965,
      "activations/layer20_attention_weight_max": 23.7653751373291,
      "activations/layer20_attention_weight_min": -20.615537643432617,
      "activations/layer21_attention_weight_max": 39.43031692504883,
      "activations/layer21_attention_weight_min": -26.447250366210938,
      "activations/layer22_attention_weight_max": 31.142345428466797,
      "activations/layer22_attention_weight_min": -25.14384651184082,
      "activations/layer23_attention_weight_max": 38.39543151855469,
      "activations/layer23_attention_weight_min": -24.561443328857422,
      "activations/layer2_attention_weight_max": 33.8123779296875,
      "activations/layer2_attention_weight_min": -33.276756286621094,
      "activations/layer3_attention_weight_max": 100.17173767089844,
      "activations/layer3_attention_weight_min": -100.34529876708984,
      "activations/layer4_attention_weight_max": 103.6157455444336,
      "activations/layer4_attention_weight_min": -97.27099609375,
      "activations/layer5_attention_weight_max": 78.77838134765625,
      "activations/layer5_attention_weight_min": -80.01052856445312,
      "activations/layer6_attention_weight_max": 57.99476623535156,
      "activations/layer6_attention_weight_min": -53.06829071044922,
      "activations/layer7_attention_weight_max": 76.06982421875,
      "activations/layer7_attention_weight_min": -66.34663391113281,
      "activations/layer8_attention_weight_max": 46.30561828613281,
      "activations/layer8_attention_weight_min": -48.18544387817383,
      "activations/layer9_attention_weight_max": 51.0321159362793,
      "activations/layer9_attention_weight_min": -49.76450729370117,
      "epoch": 12.38,
      "learning_rate": 7.088295454545454e-05,
      "loss": 2.7704,
      "step": 213050
    },
    {
      "activations/layer0_attention_weight_max": 15.50955867767334,
      "activations/layer0_attention_weight_min": -13.67363166809082,
      "activations/layer10_attention_weight_max": 35.62250518798828,
      "activations/layer10_attention_weight_min": -38.33953857421875,
      "activations/layer11_attention_weight_max": 34.55228042602539,
      "activations/layer11_attention_weight_min": -33.554840087890625,
      "activations/layer12_attention_weight_max": 19.49382972717285,
      "activations/layer12_attention_weight_min": -24.74409294128418,
      "activations/layer13_attention_weight_max": 40.87946701049805,
      "activations/layer13_attention_weight_min": -31.210756301879883,
      "activations/layer14_attention_weight_max": 35.62218475341797,
      "activations/layer14_attention_weight_min": -30.532276153564453,
      "activations/layer15_attention_weight_max": 32.02080535888672,
      "activations/layer15_attention_weight_min": -29.641572952270508,
      "activations/layer16_attention_weight_max": 30.920255661010742,
      "activations/layer16_attention_weight_min": -29.40020179748535,
      "activations/layer17_attention_weight_max": 52.89054489135742,
      "activations/layer17_attention_weight_min": -45.56983947753906,
      "activations/layer18_attention_weight_max": 47.10483169555664,
      "activations/layer18_attention_weight_min": -40.76310348510742,
      "activations/layer19_attention_weight_max": 24.315282821655273,
      "activations/layer19_attention_weight_min": -23.71165657043457,
      "activations/layer1_attention_weight_max": 15.943892478942871,
      "activations/layer1_attention_weight_min": -14.785604476928711,
      "activations/layer20_attention_weight_max": 22.447614669799805,
      "activations/layer20_attention_weight_min": -21.456127166748047,
      "activations/layer21_attention_weight_max": 38.21107482910156,
      "activations/layer21_attention_weight_min": -23.08483123779297,
      "activations/layer22_attention_weight_max": 32.24934005737305,
      "activations/layer22_attention_weight_min": -26.78363037109375,
      "activations/layer23_attention_weight_max": 38.772518157958984,
      "activations/layer23_attention_weight_min": -25.86007308959961,
      "activations/layer2_attention_weight_max": 33.14532470703125,
      "activations/layer2_attention_weight_min": -31.914371490478516,
      "activations/layer3_attention_weight_max": 96.5588150024414,
      "activations/layer3_attention_weight_min": -101.36677551269531,
      "activations/layer4_attention_weight_max": 102.56929779052734,
      "activations/layer4_attention_weight_min": -101.48127746582031,
      "activations/layer5_attention_weight_max": 72.0638198852539,
      "activations/layer5_attention_weight_min": -84.94842529296875,
      "activations/layer6_attention_weight_max": 55.111968994140625,
      "activations/layer6_attention_weight_min": -56.19172668457031,
      "activations/layer7_attention_weight_max": 73.28787231445312,
      "activations/layer7_attention_weight_min": -69.63725280761719,
      "activations/layer8_attention_weight_max": 45.20450973510742,
      "activations/layer8_attention_weight_min": -48.999759674072266,
      "activations/layer9_attention_weight_max": 47.224021911621094,
      "activations/layer9_attention_weight_min": -50.45966720581055,
      "epoch": 12.38,
      "learning_rate": 7.086401515151514e-05,
      "loss": 2.7785,
      "step": 213100
    },
    {
      "activations/layer0_attention_weight_max": 15.98679256439209,
      "activations/layer0_attention_weight_min": -14.641162872314453,
      "activations/layer10_attention_weight_max": 34.782508850097656,
      "activations/layer10_attention_weight_min": -34.88436508178711,
      "activations/layer11_attention_weight_max": 31.18103790283203,
      "activations/layer11_attention_weight_min": -33.06269454956055,
      "activations/layer12_attention_weight_max": 20.029088973999023,
      "activations/layer12_attention_weight_min": -25.240249633789062,
      "activations/layer13_attention_weight_max": 33.8671760559082,
      "activations/layer13_attention_weight_min": -28.995025634765625,
      "activations/layer14_attention_weight_max": 36.86455535888672,
      "activations/layer14_attention_weight_min": -30.45339012145996,
      "activations/layer15_attention_weight_max": 33.61521911621094,
      "activations/layer15_attention_weight_min": -27.397132873535156,
      "activations/layer16_attention_weight_max": 30.226579666137695,
      "activations/layer16_attention_weight_min": -30.584741592407227,
      "activations/layer17_attention_weight_max": 50.536495208740234,
      "activations/layer17_attention_weight_min": -42.08412170410156,
      "activations/layer18_attention_weight_max": 45.68400955200195,
      "activations/layer18_attention_weight_min": -36.928321838378906,
      "activations/layer19_attention_weight_max": 20.73285484313965,
      "activations/layer19_attention_weight_min": -22.2309627532959,
      "activations/layer1_attention_weight_max": 16.37358856201172,
      "activations/layer1_attention_weight_min": -15.328899383544922,
      "activations/layer20_attention_weight_max": 21.2384033203125,
      "activations/layer20_attention_weight_min": -21.157236099243164,
      "activations/layer21_attention_weight_max": 34.92993927001953,
      "activations/layer21_attention_weight_min": -23.20549964904785,
      "activations/layer22_attention_weight_max": 28.496952056884766,
      "activations/layer22_attention_weight_min": -26.23331069946289,
      "activations/layer23_attention_weight_max": 35.83100891113281,
      "activations/layer23_attention_weight_min": -21.993778228759766,
      "activations/layer2_attention_weight_max": 33.01136779785156,
      "activations/layer2_attention_weight_min": -31.864614486694336,
      "activations/layer3_attention_weight_max": 96.04757690429688,
      "activations/layer3_attention_weight_min": -102.25996398925781,
      "activations/layer4_attention_weight_max": 99.72960662841797,
      "activations/layer4_attention_weight_min": -94.72042083740234,
      "activations/layer5_attention_weight_max": 72.18025970458984,
      "activations/layer5_attention_weight_min": -77.53636169433594,
      "activations/layer6_attention_weight_max": 52.03017807006836,
      "activations/layer6_attention_weight_min": -52.584571838378906,
      "activations/layer7_attention_weight_max": 71.4361343383789,
      "activations/layer7_attention_weight_min": -66.63382720947266,
      "activations/layer8_attention_weight_max": 45.71902084350586,
      "activations/layer8_attention_weight_min": -47.56591033935547,
      "activations/layer9_attention_weight_max": 44.38003158569336,
      "activations/layer9_attention_weight_min": -45.60126876831055,
      "epoch": 12.39,
      "learning_rate": 7.084507575757575e-05,
      "loss": 2.7803,
      "step": 213150
    },
    {
      "activations/layer0_attention_weight_max": 15.896366119384766,
      "activations/layer0_attention_weight_min": -14.703885078430176,
      "activations/layer10_attention_weight_max": 37.141693115234375,
      "activations/layer10_attention_weight_min": -35.6457405090332,
      "activations/layer11_attention_weight_max": 34.427757263183594,
      "activations/layer11_attention_weight_min": -33.49196243286133,
      "activations/layer12_attention_weight_max": 18.752500534057617,
      "activations/layer12_attention_weight_min": -25.73862075805664,
      "activations/layer13_attention_weight_max": 42.32662582397461,
      "activations/layer13_attention_weight_min": -30.84406089782715,
      "activations/layer14_attention_weight_max": 37.558876037597656,
      "activations/layer14_attention_weight_min": -32.55417251586914,
      "activations/layer15_attention_weight_max": 33.90468215942383,
      "activations/layer15_attention_weight_min": -29.92524528503418,
      "activations/layer16_attention_weight_max": 35.486572265625,
      "activations/layer16_attention_weight_min": -33.587955474853516,
      "activations/layer17_attention_weight_max": 51.72356033325195,
      "activations/layer17_attention_weight_min": -48.62294387817383,
      "activations/layer18_attention_weight_max": 46.98739242553711,
      "activations/layer18_attention_weight_min": -42.39794158935547,
      "activations/layer19_attention_weight_max": 23.144241333007812,
      "activations/layer19_attention_weight_min": -24.426921844482422,
      "activations/layer1_attention_weight_max": 17.984619140625,
      "activations/layer1_attention_weight_min": -16.54773712158203,
      "activations/layer20_attention_weight_max": 22.45288848876953,
      "activations/layer20_attention_weight_min": -22.018436431884766,
      "activations/layer21_attention_weight_max": 36.7796516418457,
      "activations/layer21_attention_weight_min": -22.483877182006836,
      "activations/layer22_attention_weight_max": 30.02585792541504,
      "activations/layer22_attention_weight_min": -25.560606002807617,
      "activations/layer23_attention_weight_max": 35.746612548828125,
      "activations/layer23_attention_weight_min": -27.047882080078125,
      "activations/layer2_attention_weight_max": 34.358306884765625,
      "activations/layer2_attention_weight_min": -35.226749420166016,
      "activations/layer3_attention_weight_max": 102.0135498046875,
      "activations/layer3_attention_weight_min": -110.07630920410156,
      "activations/layer4_attention_weight_max": 103.80859375,
      "activations/layer4_attention_weight_min": -105.81131744384766,
      "activations/layer5_attention_weight_max": 76.77513885498047,
      "activations/layer5_attention_weight_min": -86.71126556396484,
      "activations/layer6_attention_weight_max": 54.849159240722656,
      "activations/layer6_attention_weight_min": -55.67713928222656,
      "activations/layer7_attention_weight_max": 71.67691040039062,
      "activations/layer7_attention_weight_min": -68.25177001953125,
      "activations/layer8_attention_weight_max": 47.298072814941406,
      "activations/layer8_attention_weight_min": -48.525390625,
      "activations/layer9_attention_weight_max": 46.92708969116211,
      "activations/layer9_attention_weight_min": -49.99180221557617,
      "epoch": 12.39,
      "learning_rate": 7.082613636363635e-05,
      "loss": 2.7698,
      "step": 213200
    },
    {
      "activations/layer0_attention_weight_max": 15.71034049987793,
      "activations/layer0_attention_weight_min": -13.988608360290527,
      "activations/layer10_attention_weight_max": 40.21464920043945,
      "activations/layer10_attention_weight_min": -37.820457458496094,
      "activations/layer11_attention_weight_max": 34.60274887084961,
      "activations/layer11_attention_weight_min": -34.38152313232422,
      "activations/layer12_attention_weight_max": 20.934818267822266,
      "activations/layer12_attention_weight_min": -31.40247917175293,
      "activations/layer13_attention_weight_max": 38.846553802490234,
      "activations/layer13_attention_weight_min": -30.351430892944336,
      "activations/layer14_attention_weight_max": 41.20335006713867,
      "activations/layer14_attention_weight_min": -29.248567581176758,
      "activations/layer15_attention_weight_max": 35.78710174560547,
      "activations/layer15_attention_weight_min": -28.530311584472656,
      "activations/layer16_attention_weight_max": 32.386627197265625,
      "activations/layer16_attention_weight_min": -30.189664840698242,
      "activations/layer17_attention_weight_max": 55.002681732177734,
      "activations/layer17_attention_weight_min": -43.39746856689453,
      "activations/layer18_attention_weight_max": 46.5008430480957,
      "activations/layer18_attention_weight_min": -36.96288299560547,
      "activations/layer19_attention_weight_max": 23.81207847595215,
      "activations/layer19_attention_weight_min": -23.39736557006836,
      "activations/layer1_attention_weight_max": 16.72473907470703,
      "activations/layer1_attention_weight_min": -13.602090835571289,
      "activations/layer20_attention_weight_max": 24.28902816772461,
      "activations/layer20_attention_weight_min": -21.553213119506836,
      "activations/layer21_attention_weight_max": 43.660308837890625,
      "activations/layer21_attention_weight_min": -29.813718795776367,
      "activations/layer22_attention_weight_max": 28.71760368347168,
      "activations/layer22_attention_weight_min": -26.11902618408203,
      "activations/layer23_attention_weight_max": 35.78108215332031,
      "activations/layer23_attention_weight_min": -26.442195892333984,
      "activations/layer2_attention_weight_max": 34.15510559082031,
      "activations/layer2_attention_weight_min": -31.264537811279297,
      "activations/layer3_attention_weight_max": 100.19842529296875,
      "activations/layer3_attention_weight_min": -97.52083587646484,
      "activations/layer4_attention_weight_max": 104.64900970458984,
      "activations/layer4_attention_weight_min": -99.21037292480469,
      "activations/layer5_attention_weight_max": 80.0114974975586,
      "activations/layer5_attention_weight_min": -83.29312133789062,
      "activations/layer6_attention_weight_max": 61.42341995239258,
      "activations/layer6_attention_weight_min": -59.6297721862793,
      "activations/layer7_attention_weight_max": 83.33075714111328,
      "activations/layer7_attention_weight_min": -75.5030746459961,
      "activations/layer8_attention_weight_max": 53.42926788330078,
      "activations/layer8_attention_weight_min": -54.19718933105469,
      "activations/layer9_attention_weight_max": 55.78260040283203,
      "activations/layer9_attention_weight_min": -54.58803939819336,
      "epoch": 12.39,
      "learning_rate": 7.080719696969697e-05,
      "loss": 2.7673,
      "step": 213250
    },
    {
      "activations/layer0_attention_weight_max": 15.694975852966309,
      "activations/layer0_attention_weight_min": -14.39525318145752,
      "activations/layer10_attention_weight_max": 38.00324249267578,
      "activations/layer10_attention_weight_min": -35.95231628417969,
      "activations/layer11_attention_weight_max": 31.969139099121094,
      "activations/layer11_attention_weight_min": -32.797264099121094,
      "activations/layer12_attention_weight_max": 17.68825340270996,
      "activations/layer12_attention_weight_min": -23.35255241394043,
      "activations/layer13_attention_weight_max": 35.66529083251953,
      "activations/layer13_attention_weight_min": -28.14495277404785,
      "activations/layer14_attention_weight_max": 35.19118118286133,
      "activations/layer14_attention_weight_min": -30.319791793823242,
      "activations/layer15_attention_weight_max": 32.71647262573242,
      "activations/layer15_attention_weight_min": -31.18061065673828,
      "activations/layer16_attention_weight_max": 30.89697265625,
      "activations/layer16_attention_weight_min": -28.709596633911133,
      "activations/layer17_attention_weight_max": 51.09694290161133,
      "activations/layer17_attention_weight_min": -44.660465240478516,
      "activations/layer18_attention_weight_max": 43.0778923034668,
      "activations/layer18_attention_weight_min": -35.48255157470703,
      "activations/layer19_attention_weight_max": 22.44280242919922,
      "activations/layer19_attention_weight_min": -22.253171920776367,
      "activations/layer1_attention_weight_max": 17.09206771850586,
      "activations/layer1_attention_weight_min": -14.504559516906738,
      "activations/layer20_attention_weight_max": 21.562448501586914,
      "activations/layer20_attention_weight_min": -21.11618995666504,
      "activations/layer21_attention_weight_max": 33.1197509765625,
      "activations/layer21_attention_weight_min": -21.179006576538086,
      "activations/layer22_attention_weight_max": 31.266857147216797,
      "activations/layer22_attention_weight_min": -26.246967315673828,
      "activations/layer23_attention_weight_max": 38.40707015991211,
      "activations/layer23_attention_weight_min": -25.60356903076172,
      "activations/layer2_attention_weight_max": 32.151771545410156,
      "activations/layer2_attention_weight_min": -31.526756286621094,
      "activations/layer3_attention_weight_max": 90.10250091552734,
      "activations/layer3_attention_weight_min": -93.71647644042969,
      "activations/layer4_attention_weight_max": 96.83432006835938,
      "activations/layer4_attention_weight_min": -93.60614776611328,
      "activations/layer5_attention_weight_max": 70.00152587890625,
      "activations/layer5_attention_weight_min": -79.56925201416016,
      "activations/layer6_attention_weight_max": 55.15180206298828,
      "activations/layer6_attention_weight_min": -53.38821029663086,
      "activations/layer7_attention_weight_max": 68.77790069580078,
      "activations/layer7_attention_weight_min": -66.51842498779297,
      "activations/layer8_attention_weight_max": 47.49849319458008,
      "activations/layer8_attention_weight_min": -50.77216339111328,
      "activations/layer9_attention_weight_max": 47.57845687866211,
      "activations/layer9_attention_weight_min": -49.552242279052734,
      "epoch": 12.39,
      "learning_rate": 7.078825757575757e-05,
      "loss": 2.7674,
      "step": 213300
    },
    {
      "activations/layer0_attention_weight_max": 15.16687297821045,
      "activations/layer0_attention_weight_min": -14.736873626708984,
      "activations/layer10_attention_weight_max": 38.329994201660156,
      "activations/layer10_attention_weight_min": -35.391387939453125,
      "activations/layer11_attention_weight_max": 31.705371856689453,
      "activations/layer11_attention_weight_min": -33.14536666870117,
      "activations/layer12_attention_weight_max": 17.601482391357422,
      "activations/layer12_attention_weight_min": -22.708267211914062,
      "activations/layer13_attention_weight_max": 36.82835006713867,
      "activations/layer13_attention_weight_min": -28.187517166137695,
      "activations/layer14_attention_weight_max": 37.004051208496094,
      "activations/layer14_attention_weight_min": -31.070392608642578,
      "activations/layer15_attention_weight_max": 34.9111213684082,
      "activations/layer15_attention_weight_min": -28.79904556274414,
      "activations/layer16_attention_weight_max": 30.403629302978516,
      "activations/layer16_attention_weight_min": -30.27911376953125,
      "activations/layer17_attention_weight_max": 54.17036056518555,
      "activations/layer17_attention_weight_min": -44.804351806640625,
      "activations/layer18_attention_weight_max": 45.46787643432617,
      "activations/layer18_attention_weight_min": -38.39950180053711,
      "activations/layer19_attention_weight_max": 22.00787353515625,
      "activations/layer19_attention_weight_min": -22.015769958496094,
      "activations/layer1_attention_weight_max": 16.5094051361084,
      "activations/layer1_attention_weight_min": -13.83336353302002,
      "activations/layer20_attention_weight_max": 22.1000919342041,
      "activations/layer20_attention_weight_min": -21.14592170715332,
      "activations/layer21_attention_weight_max": 35.97089385986328,
      "activations/layer21_attention_weight_min": -21.94539451599121,
      "activations/layer22_attention_weight_max": 27.383487701416016,
      "activations/layer22_attention_weight_min": -25.284034729003906,
      "activations/layer23_attention_weight_max": 35.64787292480469,
      "activations/layer23_attention_weight_min": -26.906919479370117,
      "activations/layer2_attention_weight_max": 34.02275085449219,
      "activations/layer2_attention_weight_min": -31.46945571899414,
      "activations/layer3_attention_weight_max": 94.01908111572266,
      "activations/layer3_attention_weight_min": -91.24169158935547,
      "activations/layer4_attention_weight_max": 101.82653045654297,
      "activations/layer4_attention_weight_min": -95.05496978759766,
      "activations/layer5_attention_weight_max": 73.07688903808594,
      "activations/layer5_attention_weight_min": -76.57135009765625,
      "activations/layer6_attention_weight_max": 53.11552810668945,
      "activations/layer6_attention_weight_min": -53.83660125732422,
      "activations/layer7_attention_weight_max": 68.31285858154297,
      "activations/layer7_attention_weight_min": -67.1633071899414,
      "activations/layer8_attention_weight_max": 47.414852142333984,
      "activations/layer8_attention_weight_min": -47.49769973754883,
      "activations/layer9_attention_weight_max": 46.139068603515625,
      "activations/layer9_attention_weight_min": -47.16035079956055,
      "epoch": 12.4,
      "learning_rate": 7.076931818181817e-05,
      "loss": 2.7653,
      "step": 213350
    },
    {
      "activations/layer0_attention_weight_max": 16.397268295288086,
      "activations/layer0_attention_weight_min": -14.535331726074219,
      "activations/layer10_attention_weight_max": 35.17069625854492,
      "activations/layer10_attention_weight_min": -33.97576141357422,
      "activations/layer11_attention_weight_max": 32.466732025146484,
      "activations/layer11_attention_weight_min": -31.013090133666992,
      "activations/layer12_attention_weight_max": 17.34868621826172,
      "activations/layer12_attention_weight_min": -25.667118072509766,
      "activations/layer13_attention_weight_max": 35.167083740234375,
      "activations/layer13_attention_weight_min": -29.871606826782227,
      "activations/layer14_attention_weight_max": 34.17030334472656,
      "activations/layer14_attention_weight_min": -31.65458869934082,
      "activations/layer15_attention_weight_max": 29.577587127685547,
      "activations/layer15_attention_weight_min": -32.580322265625,
      "activations/layer16_attention_weight_max": 29.197105407714844,
      "activations/layer16_attention_weight_min": -30.09383201599121,
      "activations/layer17_attention_weight_max": 48.4025993347168,
      "activations/layer17_attention_weight_min": -45.377174377441406,
      "activations/layer18_attention_weight_max": 44.75933074951172,
      "activations/layer18_attention_weight_min": -39.86577224731445,
      "activations/layer19_attention_weight_max": 21.597349166870117,
      "activations/layer19_attention_weight_min": -21.997154235839844,
      "activations/layer1_attention_weight_max": 17.46331214904785,
      "activations/layer1_attention_weight_min": -14.90883731842041,
      "activations/layer20_attention_weight_max": 19.69098663330078,
      "activations/layer20_attention_weight_min": -24.360336303710938,
      "activations/layer21_attention_weight_max": 34.66379165649414,
      "activations/layer21_attention_weight_min": -23.497838973999023,
      "activations/layer22_attention_weight_max": 30.37661361694336,
      "activations/layer22_attention_weight_min": -29.246973037719727,
      "activations/layer23_attention_weight_max": 35.002891540527344,
      "activations/layer23_attention_weight_min": -24.899085998535156,
      "activations/layer2_attention_weight_max": 32.77368927001953,
      "activations/layer2_attention_weight_min": -31.905269622802734,
      "activations/layer3_attention_weight_max": 96.69054412841797,
      "activations/layer3_attention_weight_min": -97.18281555175781,
      "activations/layer4_attention_weight_max": 99.87261962890625,
      "activations/layer4_attention_weight_min": -95.6988754272461,
      "activations/layer5_attention_weight_max": 70.3431396484375,
      "activations/layer5_attention_weight_min": -79.31578063964844,
      "activations/layer6_attention_weight_max": 52.8568229675293,
      "activations/layer6_attention_weight_min": -54.192745208740234,
      "activations/layer7_attention_weight_max": 65.88640594482422,
      "activations/layer7_attention_weight_min": -68.42699432373047,
      "activations/layer8_attention_weight_max": 44.58452224731445,
      "activations/layer8_attention_weight_min": -46.08694076538086,
      "activations/layer9_attention_weight_max": 45.38250732421875,
      "activations/layer9_attention_weight_min": -46.61152267456055,
      "epoch": 12.4,
      "learning_rate": 7.075037878787879e-05,
      "loss": 2.7672,
      "step": 213400
    },
    {
      "activations/layer0_attention_weight_max": 16.609203338623047,
      "activations/layer0_attention_weight_min": -14.43489933013916,
      "activations/layer10_attention_weight_max": 38.77964401245117,
      "activations/layer10_attention_weight_min": -38.90494155883789,
      "activations/layer11_attention_weight_max": 35.82727813720703,
      "activations/layer11_attention_weight_min": -36.715538024902344,
      "activations/layer12_attention_weight_max": 17.441884994506836,
      "activations/layer12_attention_weight_min": -26.22813606262207,
      "activations/layer13_attention_weight_max": 35.908939361572266,
      "activations/layer13_attention_weight_min": -30.318775177001953,
      "activations/layer14_attention_weight_max": 36.696041107177734,
      "activations/layer14_attention_weight_min": -32.363956451416016,
      "activations/layer15_attention_weight_max": 32.535301208496094,
      "activations/layer15_attention_weight_min": -28.702688217163086,
      "activations/layer16_attention_weight_max": 31.331823348999023,
      "activations/layer16_attention_weight_min": -30.43937110900879,
      "activations/layer17_attention_weight_max": 50.78520965576172,
      "activations/layer17_attention_weight_min": -43.04854202270508,
      "activations/layer18_attention_weight_max": 44.9747428894043,
      "activations/layer18_attention_weight_min": -37.21064376831055,
      "activations/layer19_attention_weight_max": 22.07469940185547,
      "activations/layer19_attention_weight_min": -22.576200485229492,
      "activations/layer1_attention_weight_max": 16.59191131591797,
      "activations/layer1_attention_weight_min": -14.535213470458984,
      "activations/layer20_attention_weight_max": 21.21835708618164,
      "activations/layer20_attention_weight_min": -22.208715438842773,
      "activations/layer21_attention_weight_max": 34.669063568115234,
      "activations/layer21_attention_weight_min": -20.990053176879883,
      "activations/layer22_attention_weight_max": 26.966691970825195,
      "activations/layer22_attention_weight_min": -25.70803451538086,
      "activations/layer23_attention_weight_max": 35.07939910888672,
      "activations/layer23_attention_weight_min": -24.28905487060547,
      "activations/layer2_attention_weight_max": 33.89069366455078,
      "activations/layer2_attention_weight_min": -32.62531280517578,
      "activations/layer3_attention_weight_max": 97.7177963256836,
      "activations/layer3_attention_weight_min": -98.52643585205078,
      "activations/layer4_attention_weight_max": 99.9586181640625,
      "activations/layer4_attention_weight_min": -98.86287689208984,
      "activations/layer5_attention_weight_max": 75.21732330322266,
      "activations/layer5_attention_weight_min": -78.3123779296875,
      "activations/layer6_attention_weight_max": 54.90410614013672,
      "activations/layer6_attention_weight_min": -54.07487869262695,
      "activations/layer7_attention_weight_max": 72.60521697998047,
      "activations/layer7_attention_weight_min": -71.44326782226562,
      "activations/layer8_attention_weight_max": 46.46998596191406,
      "activations/layer8_attention_weight_min": -49.17763137817383,
      "activations/layer9_attention_weight_max": 51.02537155151367,
      "activations/layer9_attention_weight_min": -53.26337814331055,
      "epoch": 12.4,
      "learning_rate": 7.073143939393939e-05,
      "loss": 2.7755,
      "step": 213450
    },
    {
      "activations/layer0_attention_weight_max": 16.616039276123047,
      "activations/layer0_attention_weight_min": -14.719121932983398,
      "activations/layer10_attention_weight_max": 35.70586013793945,
      "activations/layer10_attention_weight_min": -37.7755012512207,
      "activations/layer11_attention_weight_max": 33.22381591796875,
      "activations/layer11_attention_weight_min": -34.133689880371094,
      "activations/layer12_attention_weight_max": 18.9345645904541,
      "activations/layer12_attention_weight_min": -28.41901397705078,
      "activations/layer13_attention_weight_max": 38.15497589111328,
      "activations/layer13_attention_weight_min": -29.993715286254883,
      "activations/layer14_attention_weight_max": 34.84841537475586,
      "activations/layer14_attention_weight_min": -29.66094970703125,
      "activations/layer15_attention_weight_max": 33.67655944824219,
      "activations/layer15_attention_weight_min": -29.81379508972168,
      "activations/layer16_attention_weight_max": 31.45508575439453,
      "activations/layer16_attention_weight_min": -29.04981803894043,
      "activations/layer17_attention_weight_max": 49.973876953125,
      "activations/layer17_attention_weight_min": -43.595333099365234,
      "activations/layer18_attention_weight_max": 43.44868850708008,
      "activations/layer18_attention_weight_min": -36.36614227294922,
      "activations/layer19_attention_weight_max": 21.56514549255371,
      "activations/layer19_attention_weight_min": -22.33846092224121,
      "activations/layer1_attention_weight_max": 16.334030151367188,
      "activations/layer1_attention_weight_min": -14.753196716308594,
      "activations/layer20_attention_weight_max": 24.192325592041016,
      "activations/layer20_attention_weight_min": -22.099266052246094,
      "activations/layer21_attention_weight_max": 40.61775588989258,
      "activations/layer21_attention_weight_min": -21.567113876342773,
      "activations/layer22_attention_weight_max": 29.804534912109375,
      "activations/layer22_attention_weight_min": -27.15472984313965,
      "activations/layer23_attention_weight_max": 34.867431640625,
      "activations/layer23_attention_weight_min": -25.008581161499023,
      "activations/layer2_attention_weight_max": 35.45973205566406,
      "activations/layer2_attention_weight_min": -32.92900848388672,
      "activations/layer3_attention_weight_max": 98.03788757324219,
      "activations/layer3_attention_weight_min": -99.93397521972656,
      "activations/layer4_attention_weight_max": 103.13160705566406,
      "activations/layer4_attention_weight_min": -100.82723999023438,
      "activations/layer5_attention_weight_max": 77.43846130371094,
      "activations/layer5_attention_weight_min": -83.00155639648438,
      "activations/layer6_attention_weight_max": 56.4355583190918,
      "activations/layer6_attention_weight_min": -54.59150695800781,
      "activations/layer7_attention_weight_max": 71.86888122558594,
      "activations/layer7_attention_weight_min": -71.92259979248047,
      "activations/layer8_attention_weight_max": 47.3277473449707,
      "activations/layer8_attention_weight_min": -51.24540710449219,
      "activations/layer9_attention_weight_max": 47.55301284790039,
      "activations/layer9_attention_weight_min": -51.7998046875,
      "epoch": 12.41,
      "learning_rate": 7.071249999999999e-05,
      "loss": 2.7788,
      "step": 213500
    },
    {
      "activations/layer0_attention_weight_max": 18.447343826293945,
      "activations/layer0_attention_weight_min": -14.558940887451172,
      "activations/layer10_attention_weight_max": 40.135555267333984,
      "activations/layer10_attention_weight_min": -39.735355377197266,
      "activations/layer11_attention_weight_max": 35.653194427490234,
      "activations/layer11_attention_weight_min": -36.390377044677734,
      "activations/layer12_attention_weight_max": 20.368738174438477,
      "activations/layer12_attention_weight_min": -27.424060821533203,
      "activations/layer13_attention_weight_max": 42.73142623901367,
      "activations/layer13_attention_weight_min": -31.586645126342773,
      "activations/layer14_attention_weight_max": 40.29186248779297,
      "activations/layer14_attention_weight_min": -32.90717697143555,
      "activations/layer15_attention_weight_max": 37.83888626098633,
      "activations/layer15_attention_weight_min": -32.61391067504883,
      "activations/layer16_attention_weight_max": 35.86691665649414,
      "activations/layer16_attention_weight_min": -33.72901153564453,
      "activations/layer17_attention_weight_max": 61.41989517211914,
      "activations/layer17_attention_weight_min": -48.35469436645508,
      "activations/layer18_attention_weight_max": 53.48943328857422,
      "activations/layer18_attention_weight_min": -38.99279022216797,
      "activations/layer19_attention_weight_max": 23.025283813476562,
      "activations/layer19_attention_weight_min": -23.149520874023438,
      "activations/layer1_attention_weight_max": 17.657695770263672,
      "activations/layer1_attention_weight_min": -15.532014846801758,
      "activations/layer20_attention_weight_max": 26.506511688232422,
      "activations/layer20_attention_weight_min": -21.49555778503418,
      "activations/layer21_attention_weight_max": 48.9244499206543,
      "activations/layer21_attention_weight_min": -22.6656494140625,
      "activations/layer22_attention_weight_max": 34.13803482055664,
      "activations/layer22_attention_weight_min": -28.360572814941406,
      "activations/layer23_attention_weight_max": 38.23619079589844,
      "activations/layer23_attention_weight_min": -30.49010467529297,
      "activations/layer2_attention_weight_max": 35.332725524902344,
      "activations/layer2_attention_weight_min": -33.06013488769531,
      "activations/layer3_attention_weight_max": 98.18324279785156,
      "activations/layer3_attention_weight_min": -98.78182220458984,
      "activations/layer4_attention_weight_max": 102.09912109375,
      "activations/layer4_attention_weight_min": -98.87621307373047,
      "activations/layer5_attention_weight_max": 72.70440673828125,
      "activations/layer5_attention_weight_min": -79.87222290039062,
      "activations/layer6_attention_weight_max": 57.23540496826172,
      "activations/layer6_attention_weight_min": -57.56216812133789,
      "activations/layer7_attention_weight_max": 73.40460205078125,
      "activations/layer7_attention_weight_min": -70.99260711669922,
      "activations/layer8_attention_weight_max": 50.565826416015625,
      "activations/layer8_attention_weight_min": -52.84498977661133,
      "activations/layer9_attention_weight_max": 52.66743087768555,
      "activations/layer9_attention_weight_min": -54.39636993408203,
      "epoch": 12.41,
      "learning_rate": 7.06935606060606e-05,
      "loss": 2.7655,
      "step": 213550
    },
    {
      "activations/layer0_attention_weight_max": 16.215131759643555,
      "activations/layer0_attention_weight_min": -13.95050048828125,
      "activations/layer10_attention_weight_max": 36.25689697265625,
      "activations/layer10_attention_weight_min": -37.85760498046875,
      "activations/layer11_attention_weight_max": 32.23294448852539,
      "activations/layer11_attention_weight_min": -32.45109558105469,
      "activations/layer12_attention_weight_max": 18.273401260375977,
      "activations/layer12_attention_weight_min": -25.858718872070312,
      "activations/layer13_attention_weight_max": 47.784061431884766,
      "activations/layer13_attention_weight_min": -34.08629608154297,
      "activations/layer14_attention_weight_max": 44.97404861450195,
      "activations/layer14_attention_weight_min": -35.1307487487793,
      "activations/layer15_attention_weight_max": 35.658470153808594,
      "activations/layer15_attention_weight_min": -30.67919921875,
      "activations/layer16_attention_weight_max": 34.14626693725586,
      "activations/layer16_attention_weight_min": -31.766735076904297,
      "activations/layer17_attention_weight_max": 54.23783493041992,
      "activations/layer17_attention_weight_min": -45.16082763671875,
      "activations/layer18_attention_weight_max": 49.294456481933594,
      "activations/layer18_attention_weight_min": -38.48619842529297,
      "activations/layer19_attention_weight_max": 22.617033004760742,
      "activations/layer19_attention_weight_min": -21.683134078979492,
      "activations/layer1_attention_weight_max": 16.13164710998535,
      "activations/layer1_attention_weight_min": -15.874289512634277,
      "activations/layer20_attention_weight_max": 23.391719818115234,
      "activations/layer20_attention_weight_min": -22.355960845947266,
      "activations/layer21_attention_weight_max": 36.88119888305664,
      "activations/layer21_attention_weight_min": -23.26839828491211,
      "activations/layer22_attention_weight_max": 32.709205627441406,
      "activations/layer22_attention_weight_min": -24.930248260498047,
      "activations/layer23_attention_weight_max": 34.200096130371094,
      "activations/layer23_attention_weight_min": -25.369468688964844,
      "activations/layer2_attention_weight_max": 32.988441467285156,
      "activations/layer2_attention_weight_min": -31.62325668334961,
      "activations/layer3_attention_weight_max": 100.3641128540039,
      "activations/layer3_attention_weight_min": -96.1991958618164,
      "activations/layer4_attention_weight_max": 100.06475067138672,
      "activations/layer4_attention_weight_min": -99.52041625976562,
      "activations/layer5_attention_weight_max": 70.16093444824219,
      "activations/layer5_attention_weight_min": -80.280029296875,
      "activations/layer6_attention_weight_max": 54.872581481933594,
      "activations/layer6_attention_weight_min": -53.58695602416992,
      "activations/layer7_attention_weight_max": 68.95787048339844,
      "activations/layer7_attention_weight_min": -71.07239532470703,
      "activations/layer8_attention_weight_max": 45.564170837402344,
      "activations/layer8_attention_weight_min": -47.2137336730957,
      "activations/layer9_attention_weight_max": 45.887107849121094,
      "activations/layer9_attention_weight_min": -49.290985107421875,
      "epoch": 12.41,
      "learning_rate": 7.067462121212121e-05,
      "loss": 2.7628,
      "step": 213600
    },
    {
      "activations/layer0_attention_weight_max": 17.245534896850586,
      "activations/layer0_attention_weight_min": -14.034520149230957,
      "activations/layer10_attention_weight_max": 34.93431091308594,
      "activations/layer10_attention_weight_min": -33.73990249633789,
      "activations/layer11_attention_weight_max": 31.50788116455078,
      "activations/layer11_attention_weight_min": -31.74379539489746,
      "activations/layer12_attention_weight_max": 19.908283233642578,
      "activations/layer12_attention_weight_min": -25.196552276611328,
      "activations/layer13_attention_weight_max": 36.3268928527832,
      "activations/layer13_attention_weight_min": -26.981374740600586,
      "activations/layer14_attention_weight_max": 36.62106704711914,
      "activations/layer14_attention_weight_min": -28.700435638427734,
      "activations/layer15_attention_weight_max": 32.716026306152344,
      "activations/layer15_attention_weight_min": -28.26620864868164,
      "activations/layer16_attention_weight_max": 35.51172637939453,
      "activations/layer16_attention_weight_min": -28.514402389526367,
      "activations/layer17_attention_weight_max": 51.43019104003906,
      "activations/layer17_attention_weight_min": -43.25291442871094,
      "activations/layer18_attention_weight_max": 47.1555290222168,
      "activations/layer18_attention_weight_min": -33.6390266418457,
      "activations/layer19_attention_weight_max": 24.486669540405273,
      "activations/layer19_attention_weight_min": -20.836410522460938,
      "activations/layer1_attention_weight_max": 16.372900009155273,
      "activations/layer1_attention_weight_min": -15.60897159576416,
      "activations/layer20_attention_weight_max": 25.144493103027344,
      "activations/layer20_attention_weight_min": -21.779911041259766,
      "activations/layer21_attention_weight_max": 41.248069763183594,
      "activations/layer21_attention_weight_min": -25.22515106201172,
      "activations/layer22_attention_weight_max": 32.62439727783203,
      "activations/layer22_attention_weight_min": -24.84065818786621,
      "activations/layer23_attention_weight_max": 41.83821105957031,
      "activations/layer23_attention_weight_min": -26.26288604736328,
      "activations/layer2_attention_weight_max": 35.12016677856445,
      "activations/layer2_attention_weight_min": -32.46784210205078,
      "activations/layer3_attention_weight_max": 96.17109680175781,
      "activations/layer3_attention_weight_min": -102.45577239990234,
      "activations/layer4_attention_weight_max": 99.59066772460938,
      "activations/layer4_attention_weight_min": -100.7637710571289,
      "activations/layer5_attention_weight_max": 74.19524383544922,
      "activations/layer5_attention_weight_min": -81.86772155761719,
      "activations/layer6_attention_weight_max": 54.78209686279297,
      "activations/layer6_attention_weight_min": -53.923423767089844,
      "activations/layer7_attention_weight_max": 74.20645141601562,
      "activations/layer7_attention_weight_min": -71.43423461914062,
      "activations/layer8_attention_weight_max": 46.42506790161133,
      "activations/layer8_attention_weight_min": -49.297935485839844,
      "activations/layer9_attention_weight_max": 48.77272415161133,
      "activations/layer9_attention_weight_min": -48.101165771484375,
      "epoch": 12.41,
      "learning_rate": 7.065568181818181e-05,
      "loss": 2.7679,
      "step": 213650
    },
    {
      "activations/layer0_attention_weight_max": 15.934805870056152,
      "activations/layer0_attention_weight_min": -14.082338333129883,
      "activations/layer10_attention_weight_max": 37.239227294921875,
      "activations/layer10_attention_weight_min": -36.228511810302734,
      "activations/layer11_attention_weight_max": 33.743133544921875,
      "activations/layer11_attention_weight_min": -34.097015380859375,
      "activations/layer12_attention_weight_max": 17.71833610534668,
      "activations/layer12_attention_weight_min": -24.302539825439453,
      "activations/layer13_attention_weight_max": 38.73345947265625,
      "activations/layer13_attention_weight_min": -30.816749572753906,
      "activations/layer14_attention_weight_max": 36.30710983276367,
      "activations/layer14_attention_weight_min": -31.007387161254883,
      "activations/layer15_attention_weight_max": 32.145355224609375,
      "activations/layer15_attention_weight_min": -30.197813034057617,
      "activations/layer16_attention_weight_max": 32.40930938720703,
      "activations/layer16_attention_weight_min": -30.390779495239258,
      "activations/layer17_attention_weight_max": 54.745765686035156,
      "activations/layer17_attention_weight_min": -50.189231872558594,
      "activations/layer18_attention_weight_max": 50.265594482421875,
      "activations/layer18_attention_weight_min": -42.50880432128906,
      "activations/layer19_attention_weight_max": 25.47854232788086,
      "activations/layer19_attention_weight_min": -24.129911422729492,
      "activations/layer1_attention_weight_max": 16.61827278137207,
      "activations/layer1_attention_weight_min": -14.256674766540527,
      "activations/layer20_attention_weight_max": 23.496049880981445,
      "activations/layer20_attention_weight_min": -21.847612380981445,
      "activations/layer21_attention_weight_max": 36.17633819580078,
      "activations/layer21_attention_weight_min": -23.044565200805664,
      "activations/layer22_attention_weight_max": 35.175289154052734,
      "activations/layer22_attention_weight_min": -27.19837760925293,
      "activations/layer23_attention_weight_max": 39.69063186645508,
      "activations/layer23_attention_weight_min": -24.867406845092773,
      "activations/layer2_attention_weight_max": 34.71183395385742,
      "activations/layer2_attention_weight_min": -32.58958435058594,
      "activations/layer3_attention_weight_max": 98.65882873535156,
      "activations/layer3_attention_weight_min": -97.1587142944336,
      "activations/layer4_attention_weight_max": 105.28138732910156,
      "activations/layer4_attention_weight_min": -99.54863739013672,
      "activations/layer5_attention_weight_max": 76.64562225341797,
      "activations/layer5_attention_weight_min": -79.85743713378906,
      "activations/layer6_attention_weight_max": 55.65784454345703,
      "activations/layer6_attention_weight_min": -54.75859451293945,
      "activations/layer7_attention_weight_max": 69.26189422607422,
      "activations/layer7_attention_weight_min": -68.65415954589844,
      "activations/layer8_attention_weight_max": 48.1142463684082,
      "activations/layer8_attention_weight_min": -51.05366897583008,
      "activations/layer9_attention_weight_max": 47.95013427734375,
      "activations/layer9_attention_weight_min": -49.60385513305664,
      "epoch": 12.42,
      "learning_rate": 7.063674242424241e-05,
      "loss": 2.7647,
      "step": 213700
    },
    {
      "activations/layer0_attention_weight_max": 15.992423057556152,
      "activations/layer0_attention_weight_min": -14.382905960083008,
      "activations/layer10_attention_weight_max": 38.649749755859375,
      "activations/layer10_attention_weight_min": -36.20941925048828,
      "activations/layer11_attention_weight_max": 36.173133850097656,
      "activations/layer11_attention_weight_min": -34.56252670288086,
      "activations/layer12_attention_weight_max": 19.542102813720703,
      "activations/layer12_attention_weight_min": -23.8643856048584,
      "activations/layer13_attention_weight_max": 47.015445709228516,
      "activations/layer13_attention_weight_min": -32.71379470825195,
      "activations/layer14_attention_weight_max": 41.65286636352539,
      "activations/layer14_attention_weight_min": -32.09718322753906,
      "activations/layer15_attention_weight_max": 37.01063919067383,
      "activations/layer15_attention_weight_min": -32.75981521606445,
      "activations/layer16_attention_weight_max": 35.21938705444336,
      "activations/layer16_attention_weight_min": -31.302160263061523,
      "activations/layer17_attention_weight_max": 55.0994987487793,
      "activations/layer17_attention_weight_min": -45.21570587158203,
      "activations/layer18_attention_weight_max": 54.938411712646484,
      "activations/layer18_attention_weight_min": -40.88899612426758,
      "activations/layer19_attention_weight_max": 24.92293930053711,
      "activations/layer19_attention_weight_min": -23.31166648864746,
      "activations/layer1_attention_weight_max": 16.075969696044922,
      "activations/layer1_attention_weight_min": -15.771896362304688,
      "activations/layer20_attention_weight_max": 24.185922622680664,
      "activations/layer20_attention_weight_min": -22.4737491607666,
      "activations/layer21_attention_weight_max": 40.339210510253906,
      "activations/layer21_attention_weight_min": -23.977880477905273,
      "activations/layer22_attention_weight_max": 37.64414596557617,
      "activations/layer22_attention_weight_min": -26.77988052368164,
      "activations/layer23_attention_weight_max": 41.003822326660156,
      "activations/layer23_attention_weight_min": -25.012245178222656,
      "activations/layer2_attention_weight_max": 33.3701057434082,
      "activations/layer2_attention_weight_min": -31.715810775756836,
      "activations/layer3_attention_weight_max": 92.53594970703125,
      "activations/layer3_attention_weight_min": -96.60051727294922,
      "activations/layer4_attention_weight_max": 99.82653045654297,
      "activations/layer4_attention_weight_min": -94.38031005859375,
      "activations/layer5_attention_weight_max": 72.30158996582031,
      "activations/layer5_attention_weight_min": -78.30738830566406,
      "activations/layer6_attention_weight_max": 55.791019439697266,
      "activations/layer6_attention_weight_min": -54.402286529541016,
      "activations/layer7_attention_weight_max": 71.87511444091797,
      "activations/layer7_attention_weight_min": -69.74830627441406,
      "activations/layer8_attention_weight_max": 48.134117126464844,
      "activations/layer8_attention_weight_min": -49.15346145629883,
      "activations/layer9_attention_weight_max": 50.05339813232422,
      "activations/layer9_attention_weight_min": -49.43210983276367,
      "epoch": 12.42,
      "learning_rate": 7.061780303030303e-05,
      "loss": 2.7721,
      "step": 213750
    },
    {
      "activations/layer0_attention_weight_max": 16.736486434936523,
      "activations/layer0_attention_weight_min": -14.256468772888184,
      "activations/layer10_attention_weight_max": 36.67570114135742,
      "activations/layer10_attention_weight_min": -34.858699798583984,
      "activations/layer11_attention_weight_max": 32.38405990600586,
      "activations/layer11_attention_weight_min": -33.56480026245117,
      "activations/layer12_attention_weight_max": 18.51995277404785,
      "activations/layer12_attention_weight_min": -25.464902877807617,
      "activations/layer13_attention_weight_max": 40.85976028442383,
      "activations/layer13_attention_weight_min": -30.44675636291504,
      "activations/layer14_attention_weight_max": 36.39546585083008,
      "activations/layer14_attention_weight_min": -30.65656280517578,
      "activations/layer15_attention_weight_max": 31.29676055908203,
      "activations/layer15_attention_weight_min": -31.737064361572266,
      "activations/layer16_attention_weight_max": 31.840530395507812,
      "activations/layer16_attention_weight_min": -29.830446243286133,
      "activations/layer17_attention_weight_max": 51.53294372558594,
      "activations/layer17_attention_weight_min": -45.50753402709961,
      "activations/layer18_attention_weight_max": 46.538917541503906,
      "activations/layer18_attention_weight_min": -37.26569747924805,
      "activations/layer19_attention_weight_max": 23.491830825805664,
      "activations/layer19_attention_weight_min": -23.10004234313965,
      "activations/layer1_attention_weight_max": 17.882356643676758,
      "activations/layer1_attention_weight_min": -14.938933372497559,
      "activations/layer20_attention_weight_max": 23.441375732421875,
      "activations/layer20_attention_weight_min": -21.44352149963379,
      "activations/layer21_attention_weight_max": 42.04178237915039,
      "activations/layer21_attention_weight_min": -23.967575073242188,
      "activations/layer22_attention_weight_max": 30.847457885742188,
      "activations/layer22_attention_weight_min": -26.1755428314209,
      "activations/layer23_attention_weight_max": 43.49596405029297,
      "activations/layer23_attention_weight_min": -23.2841854095459,
      "activations/layer2_attention_weight_max": 33.10859298706055,
      "activations/layer2_attention_weight_min": -31.91417694091797,
      "activations/layer3_attention_weight_max": 91.17472839355469,
      "activations/layer3_attention_weight_min": -91.88297271728516,
      "activations/layer4_attention_weight_max": 95.68882751464844,
      "activations/layer4_attention_weight_min": -93.88626098632812,
      "activations/layer5_attention_weight_max": 72.1131591796875,
      "activations/layer5_attention_weight_min": -74.97868347167969,
      "activations/layer6_attention_weight_max": 53.15156555175781,
      "activations/layer6_attention_weight_min": -52.900699615478516,
      "activations/layer7_attention_weight_max": 70.07521057128906,
      "activations/layer7_attention_weight_min": -73.0602798461914,
      "activations/layer8_attention_weight_max": 44.85186767578125,
      "activations/layer8_attention_weight_min": -46.37319564819336,
      "activations/layer9_attention_weight_max": 47.19216537475586,
      "activations/layer9_attention_weight_min": -47.472618103027344,
      "epoch": 12.42,
      "learning_rate": 7.059886363636363e-05,
      "loss": 2.7743,
      "step": 213800
    },
    {
      "activations/layer0_attention_weight_max": 17.440834045410156,
      "activations/layer0_attention_weight_min": -13.807278633117676,
      "activations/layer10_attention_weight_max": 36.653465270996094,
      "activations/layer10_attention_weight_min": -38.17232131958008,
      "activations/layer11_attention_weight_max": 33.97633743286133,
      "activations/layer11_attention_weight_min": -35.980133056640625,
      "activations/layer12_attention_weight_max": 17.849077224731445,
      "activations/layer12_attention_weight_min": -24.595678329467773,
      "activations/layer13_attention_weight_max": 43.89900207519531,
      "activations/layer13_attention_weight_min": -34.25043869018555,
      "activations/layer14_attention_weight_max": 47.617027282714844,
      "activations/layer14_attention_weight_min": -33.65203857421875,
      "activations/layer15_attention_weight_max": 37.07994842529297,
      "activations/layer15_attention_weight_min": -34.381614685058594,
      "activations/layer16_attention_weight_max": 32.139442443847656,
      "activations/layer16_attention_weight_min": -31.107141494750977,
      "activations/layer17_attention_weight_max": 51.132041931152344,
      "activations/layer17_attention_weight_min": -43.9989128112793,
      "activations/layer18_attention_weight_max": 43.6674919128418,
      "activations/layer18_attention_weight_min": -38.55883026123047,
      "activations/layer19_attention_weight_max": 21.754098892211914,
      "activations/layer19_attention_weight_min": -23.401695251464844,
      "activations/layer1_attention_weight_max": 16.559085845947266,
      "activations/layer1_attention_weight_min": -17.11424446105957,
      "activations/layer20_attention_weight_max": 22.879167556762695,
      "activations/layer20_attention_weight_min": -22.05536460876465,
      "activations/layer21_attention_weight_max": 31.56929588317871,
      "activations/layer21_attention_weight_min": -22.952913284301758,
      "activations/layer22_attention_weight_max": 29.919879913330078,
      "activations/layer22_attention_weight_min": -25.3027286529541,
      "activations/layer23_attention_weight_max": 39.609375,
      "activations/layer23_attention_weight_min": -24.833066940307617,
      "activations/layer2_attention_weight_max": 33.191802978515625,
      "activations/layer2_attention_weight_min": -34.102752685546875,
      "activations/layer3_attention_weight_max": 94.29291534423828,
      "activations/layer3_attention_weight_min": -103.43919372558594,
      "activations/layer4_attention_weight_max": 105.65533447265625,
      "activations/layer4_attention_weight_min": -95.74407196044922,
      "activations/layer5_attention_weight_max": 74.54332733154297,
      "activations/layer5_attention_weight_min": -83.11527252197266,
      "activations/layer6_attention_weight_max": 57.39323806762695,
      "activations/layer6_attention_weight_min": -55.8106575012207,
      "activations/layer7_attention_weight_max": 72.235595703125,
      "activations/layer7_attention_weight_min": -70.62369537353516,
      "activations/layer8_attention_weight_max": 46.36115646362305,
      "activations/layer8_attention_weight_min": -51.35329055786133,
      "activations/layer9_attention_weight_max": 47.585060119628906,
      "activations/layer9_attention_weight_min": -50.81480026245117,
      "epoch": 12.43,
      "learning_rate": 7.057992424242423e-05,
      "loss": 2.779,
      "step": 213850
    },
    {
      "activations/layer0_attention_weight_max": 16.034109115600586,
      "activations/layer0_attention_weight_min": -13.791025161743164,
      "activations/layer10_attention_weight_max": 34.805809020996094,
      "activations/layer10_attention_weight_min": -33.8220100402832,
      "activations/layer11_attention_weight_max": 30.805892944335938,
      "activations/layer11_attention_weight_min": -32.07002258300781,
      "activations/layer12_attention_weight_max": 19.355070114135742,
      "activations/layer12_attention_weight_min": -23.496286392211914,
      "activations/layer13_attention_weight_max": 39.35663986206055,
      "activations/layer13_attention_weight_min": -28.83624839782715,
      "activations/layer14_attention_weight_max": 35.7701416015625,
      "activations/layer14_attention_weight_min": -30.29204750061035,
      "activations/layer15_attention_weight_max": 34.7867431640625,
      "activations/layer15_attention_weight_min": -30.44045066833496,
      "activations/layer16_attention_weight_max": 32.475433349609375,
      "activations/layer16_attention_weight_min": -30.562942504882812,
      "activations/layer17_attention_weight_max": 52.09574508666992,
      "activations/layer17_attention_weight_min": -46.741912841796875,
      "activations/layer18_attention_weight_max": 43.3465461730957,
      "activations/layer18_attention_weight_min": -42.16802978515625,
      "activations/layer19_attention_weight_max": 22.601367950439453,
      "activations/layer19_attention_weight_min": -23.152982711791992,
      "activations/layer1_attention_weight_max": 17.116933822631836,
      "activations/layer1_attention_weight_min": -15.293978691101074,
      "activations/layer20_attention_weight_max": 21.143871307373047,
      "activations/layer20_attention_weight_min": -20.787235260009766,
      "activations/layer21_attention_weight_max": 34.64179229736328,
      "activations/layer21_attention_weight_min": -23.47685432434082,
      "activations/layer22_attention_weight_max": 31.87854766845703,
      "activations/layer22_attention_weight_min": -26.932403564453125,
      "activations/layer23_attention_weight_max": 34.27006530761719,
      "activations/layer23_attention_weight_min": -24.065013885498047,
      "activations/layer2_attention_weight_max": 34.6793212890625,
      "activations/layer2_attention_weight_min": -33.27484893798828,
      "activations/layer3_attention_weight_max": 100.67012786865234,
      "activations/layer3_attention_weight_min": -111.69197845458984,
      "activations/layer4_attention_weight_max": 106.75421142578125,
      "activations/layer4_attention_weight_min": -105.14867401123047,
      "activations/layer5_attention_weight_max": 72.10028076171875,
      "activations/layer5_attention_weight_min": -78.62937927246094,
      "activations/layer6_attention_weight_max": 55.994224548339844,
      "activations/layer6_attention_weight_min": -54.4913215637207,
      "activations/layer7_attention_weight_max": 67.02658081054688,
      "activations/layer7_attention_weight_min": -65.67838287353516,
      "activations/layer8_attention_weight_max": 44.18087387084961,
      "activations/layer8_attention_weight_min": -47.21896743774414,
      "activations/layer9_attention_weight_max": 46.02210235595703,
      "activations/layer9_attention_weight_min": -44.73992919921875,
      "epoch": 12.43,
      "learning_rate": 7.056098484848484e-05,
      "loss": 2.7722,
      "step": 213900
    },
    {
      "activations/layer0_attention_weight_max": 16.447885513305664,
      "activations/layer0_attention_weight_min": -13.736741065979004,
      "activations/layer10_attention_weight_max": 35.32881164550781,
      "activations/layer10_attention_weight_min": -34.28046417236328,
      "activations/layer11_attention_weight_max": 32.45880126953125,
      "activations/layer11_attention_weight_min": -33.706085205078125,
      "activations/layer12_attention_weight_max": 18.69289779663086,
      "activations/layer12_attention_weight_min": -23.307971954345703,
      "activations/layer13_attention_weight_max": 40.488380432128906,
      "activations/layer13_attention_weight_min": -27.973262786865234,
      "activations/layer14_attention_weight_max": 35.84714126586914,
      "activations/layer14_attention_weight_min": -32.69995880126953,
      "activations/layer15_attention_weight_max": 32.69063949584961,
      "activations/layer15_attention_weight_min": -30.695589065551758,
      "activations/layer16_attention_weight_max": 33.41933822631836,
      "activations/layer16_attention_weight_min": -30.09935760498047,
      "activations/layer17_attention_weight_max": 51.41548538208008,
      "activations/layer17_attention_weight_min": -42.68492889404297,
      "activations/layer18_attention_weight_max": 47.89540481567383,
      "activations/layer18_attention_weight_min": -37.22968673706055,
      "activations/layer19_attention_weight_max": 23.384300231933594,
      "activations/layer19_attention_weight_min": -19.92102813720703,
      "activations/layer1_attention_weight_max": 16.398225784301758,
      "activations/layer1_attention_weight_min": -14.162230491638184,
      "activations/layer20_attention_weight_max": 22.866966247558594,
      "activations/layer20_attention_weight_min": -20.546348571777344,
      "activations/layer21_attention_weight_max": 34.52294921875,
      "activations/layer21_attention_weight_min": -19.51066780090332,
      "activations/layer22_attention_weight_max": 30.408573150634766,
      "activations/layer22_attention_weight_min": -24.975929260253906,
      "activations/layer23_attention_weight_max": 39.322967529296875,
      "activations/layer23_attention_weight_min": -24.729095458984375,
      "activations/layer2_attention_weight_max": 31.70376205444336,
      "activations/layer2_attention_weight_min": -29.494884490966797,
      "activations/layer3_attention_weight_max": 88.52340698242188,
      "activations/layer3_attention_weight_min": -89.801025390625,
      "activations/layer4_attention_weight_max": 92.88079071044922,
      "activations/layer4_attention_weight_min": -90.66967010498047,
      "activations/layer5_attention_weight_max": 70.47579956054688,
      "activations/layer5_attention_weight_min": -78.08084106445312,
      "activations/layer6_attention_weight_max": 53.2038459777832,
      "activations/layer6_attention_weight_min": -54.08567810058594,
      "activations/layer7_attention_weight_max": 71.03641510009766,
      "activations/layer7_attention_weight_min": -69.72894287109375,
      "activations/layer8_attention_weight_max": 45.809696197509766,
      "activations/layer8_attention_weight_min": -48.11494827270508,
      "activations/layer9_attention_weight_max": 51.22926712036133,
      "activations/layer9_attention_weight_min": -47.8635368347168,
      "epoch": 12.43,
      "learning_rate": 7.054242424242423e-05,
      "loss": 2.7788,
      "step": 213950
    },
    {
      "activations/layer0_attention_weight_max": 15.798795700073242,
      "activations/layer0_attention_weight_min": -13.560827255249023,
      "activations/layer10_attention_weight_max": 36.86045455932617,
      "activations/layer10_attention_weight_min": -35.18740463256836,
      "activations/layer11_attention_weight_max": 33.980621337890625,
      "activations/layer11_attention_weight_min": -35.31740188598633,
      "activations/layer12_attention_weight_max": 19.18671226501465,
      "activations/layer12_attention_weight_min": -23.07870864868164,
      "activations/layer13_attention_weight_max": 35.589481353759766,
      "activations/layer13_attention_weight_min": -26.859086990356445,
      "activations/layer14_attention_weight_max": 40.233577728271484,
      "activations/layer14_attention_weight_min": -30.26349639892578,
      "activations/layer15_attention_weight_max": 36.343875885009766,
      "activations/layer15_attention_weight_min": -29.5664005279541,
      "activations/layer16_attention_weight_max": 32.14841079711914,
      "activations/layer16_attention_weight_min": -30.784961700439453,
      "activations/layer17_attention_weight_max": 55.241947174072266,
      "activations/layer17_attention_weight_min": -45.83440017700195,
      "activations/layer18_attention_weight_max": 42.77225875854492,
      "activations/layer18_attention_weight_min": -36.86962890625,
      "activations/layer19_attention_weight_max": 23.36033821105957,
      "activations/layer19_attention_weight_min": -21.496273040771484,
      "activations/layer1_attention_weight_max": 16.557552337646484,
      "activations/layer1_attention_weight_min": -12.913990020751953,
      "activations/layer20_attention_weight_max": 23.54377555847168,
      "activations/layer20_attention_weight_min": -22.79006576538086,
      "activations/layer21_attention_weight_max": 33.776771545410156,
      "activations/layer21_attention_weight_min": -23.816692352294922,
      "activations/layer22_attention_weight_max": 34.076377868652344,
      "activations/layer22_attention_weight_min": -26.042125701904297,
      "activations/layer23_attention_weight_max": 35.5318603515625,
      "activations/layer23_attention_weight_min": -27.025150299072266,
      "activations/layer2_attention_weight_max": 32.89643096923828,
      "activations/layer2_attention_weight_min": -28.88509178161621,
      "activations/layer3_attention_weight_max": 84.13629150390625,
      "activations/layer3_attention_weight_min": -90.17127990722656,
      "activations/layer4_attention_weight_max": 92.14798736572266,
      "activations/layer4_attention_weight_min": -88.53307342529297,
      "activations/layer5_attention_weight_max": 74.29473876953125,
      "activations/layer5_attention_weight_min": -78.75375366210938,
      "activations/layer6_attention_weight_max": 55.269859313964844,
      "activations/layer6_attention_weight_min": -57.600135803222656,
      "activations/layer7_attention_weight_max": 79.27655029296875,
      "activations/layer7_attention_weight_min": -76.21702575683594,
      "activations/layer8_attention_weight_max": 52.0316276550293,
      "activations/layer8_attention_weight_min": -49.189727783203125,
      "activations/layer9_attention_weight_max": 51.93999099731445,
      "activations/layer9_attention_weight_min": -47.98091125488281,
      "epoch": 12.43,
      "learning_rate": 7.052348484848485e-05,
      "loss": 2.7748,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_loss": 2.7265625,
      "eval_runtime": 8.5504,
      "eval_samples_per_second": 502.196,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_openwebtext_loss": 2.7265625,
      "eval_openwebtext_ppl": 15.280270706875589,
      "eval_openwebtext_runtime": 8.5504,
      "eval_openwebtext_samples_per_second": 502.196,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 2.002,
      "eval_wikitext_samples_per_second": 227.776,
      "step": 214000
    },
    {
      "epoch": 12.43,
      "eval_lambada_loss": 2.5078125,
      "eval_lambada_ppl": 12.278042445054236,
      "eval_lambada_runtime": 9.5604,
      "eval_lambada_samples_per_second": 509.29,
      "step": 214000
    },
    {
      "activations/layer0_attention_weight_max": 16.274221420288086,
      "activations/layer0_attention_weight_min": -13.863092422485352,
      "activations/layer10_attention_weight_max": 35.3785400390625,
      "activations/layer10_attention_weight_min": -35.81254959106445,
      "activations/layer11_attention_weight_max": 31.049152374267578,
      "activations/layer11_attention_weight_min": -33.70040512084961,
      "activations/layer12_attention_weight_max": 20.198022842407227,
      "activations/layer12_attention_weight_min": -27.713520050048828,
      "activations/layer13_attention_weight_max": 36.46349334716797,
      "activations/layer13_attention_weight_min": -31.6116943359375,
      "activations/layer14_attention_weight_max": 38.85469436645508,
      "activations/layer14_attention_weight_min": -34.76689529418945,
      "activations/layer15_attention_weight_max": 35.58861541748047,
      "activations/layer15_attention_weight_min": -33.44711685180664,
      "activations/layer16_attention_weight_max": 31.54113006591797,
      "activations/layer16_attention_weight_min": -31.42528533935547,
      "activations/layer17_attention_weight_max": 54.154483795166016,
      "activations/layer17_attention_weight_min": -49.506534576416016,
      "activations/layer18_attention_weight_max": 51.06009292602539,
      "activations/layer18_attention_weight_min": -42.109493255615234,
      "activations/layer19_attention_weight_max": 22.613069534301758,
      "activations/layer19_attention_weight_min": -23.984651565551758,
      "activations/layer1_attention_weight_max": 17.198711395263672,
      "activations/layer1_attention_weight_min": -14.588004112243652,
      "activations/layer20_attention_weight_max": 23.458097457885742,
      "activations/layer20_attention_weight_min": -24.005146026611328,
      "activations/layer21_attention_weight_max": 39.32403564453125,
      "activations/layer21_attention_weight_min": -25.74102020263672,
      "activations/layer22_attention_weight_max": 28.042898178100586,
      "activations/layer22_attention_weight_min": -25.97832679748535,
      "activations/layer23_attention_weight_max": 32.87166213989258,
      "activations/layer23_attention_weight_min": -25.333616256713867,
      "activations/layer2_attention_weight_max": 30.19559097290039,
      "activations/layer2_attention_weight_min": -30.152477264404297,
      "activations/layer3_attention_weight_max": 83.73802185058594,
      "activations/layer3_attention_weight_min": -91.91900634765625,
      "activations/layer4_attention_weight_max": 93.9576416015625,
      "activations/layer4_attention_weight_min": -93.34892272949219,
      "activations/layer5_attention_weight_max": 71.27201843261719,
      "activations/layer5_attention_weight_min": -80.08965301513672,
      "activations/layer6_attention_weight_max": 55.33279800415039,
      "activations/layer6_attention_weight_min": -55.052616119384766,
      "activations/layer7_attention_weight_max": 68.78657531738281,
      "activations/layer7_attention_weight_min": -74.75491333007812,
      "activations/layer8_attention_weight_max": 47.18363952636719,
      "activations/layer8_attention_weight_min": -48.847328186035156,
      "activations/layer9_attention_weight_max": 47.621337890625,
      "activations/layer9_attention_weight_min": -47.2193717956543,
      "epoch": 12.44,
      "learning_rate": 7.050454545454545e-05,
      "loss": 2.766,
      "step": 214050
    },
    {
      "activations/layer0_attention_weight_max": 15.763315200805664,
      "activations/layer0_attention_weight_min": -13.646201133728027,
      "activations/layer10_attention_weight_max": 37.3214225769043,
      "activations/layer10_attention_weight_min": -36.26250076293945,
      "activations/layer11_attention_weight_max": 31.063396453857422,
      "activations/layer11_attention_weight_min": -34.80055236816406,
      "activations/layer12_attention_weight_max": 20.81879234313965,
      "activations/layer12_attention_weight_min": -22.786970138549805,
      "activations/layer13_attention_weight_max": 36.483978271484375,
      "activations/layer13_attention_weight_min": -29.038747787475586,
      "activations/layer14_attention_weight_max": 36.227378845214844,
      "activations/layer14_attention_weight_min": -29.30145835876465,
      "activations/layer15_attention_weight_max": 35.397674560546875,
      "activations/layer15_attention_weight_min": -29.739160537719727,
      "activations/layer16_attention_weight_max": 32.73716354370117,
      "activations/layer16_attention_weight_min": -30.06163215637207,
      "activations/layer17_attention_weight_max": 50.16935729980469,
      "activations/layer17_attention_weight_min": -45.49698257446289,
      "activations/layer18_attention_weight_max": 50.103050231933594,
      "activations/layer18_attention_weight_min": -37.51662826538086,
      "activations/layer19_attention_weight_max": 24.097917556762695,
      "activations/layer19_attention_weight_min": -23.555343627929688,
      "activations/layer1_attention_weight_max": 16.669363021850586,
      "activations/layer1_attention_weight_min": -14.609851837158203,
      "activations/layer20_attention_weight_max": 23.152883529663086,
      "activations/layer20_attention_weight_min": -24.488069534301758,
      "activations/layer21_attention_weight_max": 38.83843994140625,
      "activations/layer21_attention_weight_min": -24.493759155273438,
      "activations/layer22_attention_weight_max": 34.77219009399414,
      "activations/layer22_attention_weight_min": -27.87238883972168,
      "activations/layer23_attention_weight_max": 39.130619049072266,
      "activations/layer23_attention_weight_min": -24.369346618652344,
      "activations/layer2_attention_weight_max": 32.57626724243164,
      "activations/layer2_attention_weight_min": -31.12172508239746,
      "activations/layer3_attention_weight_max": 86.98676300048828,
      "activations/layer3_attention_weight_min": -95.11846923828125,
      "activations/layer4_attention_weight_max": 93.2187271118164,
      "activations/layer4_attention_weight_min": -89.35677337646484,
      "activations/layer5_attention_weight_max": 69.32072448730469,
      "activations/layer5_attention_weight_min": -80.11495208740234,
      "activations/layer6_attention_weight_max": 52.61408615112305,
      "activations/layer6_attention_weight_min": -53.5061149597168,
      "activations/layer7_attention_weight_max": 65.7158432006836,
      "activations/layer7_attention_weight_min": -64.73271179199219,
      "activations/layer8_attention_weight_max": 45.4017448425293,
      "activations/layer8_attention_weight_min": -49.47832107543945,
      "activations/layer9_attention_weight_max": 47.57316207885742,
      "activations/layer9_attention_weight_min": -47.91689682006836,
      "epoch": 12.44,
      "learning_rate": 7.048560606060605e-05,
      "loss": 2.7662,
      "step": 214100
    },
    {
      "activations/layer0_attention_weight_max": 16.66357421875,
      "activations/layer0_attention_weight_min": -13.936769485473633,
      "activations/layer10_attention_weight_max": 40.17566680908203,
      "activations/layer10_attention_weight_min": -38.942901611328125,
      "activations/layer11_attention_weight_max": 34.533973693847656,
      "activations/layer11_attention_weight_min": -36.0140380859375,
      "activations/layer12_attention_weight_max": 18.158018112182617,
      "activations/layer12_attention_weight_min": -24.679574966430664,
      "activations/layer13_attention_weight_max": 37.2650032043457,
      "activations/layer13_attention_weight_min": -28.611806869506836,
      "activations/layer14_attention_weight_max": 37.844818115234375,
      "activations/layer14_attention_weight_min": -31.63556671142578,
      "activations/layer15_attention_weight_max": 33.50674057006836,
      "activations/layer15_attention_weight_min": -29.642820358276367,
      "activations/layer16_attention_weight_max": 31.630884170532227,
      "activations/layer16_attention_weight_min": -31.356693267822266,
      "activations/layer17_attention_weight_max": 54.74700927734375,
      "activations/layer17_attention_weight_min": -45.47625732421875,
      "activations/layer18_attention_weight_max": 51.763553619384766,
      "activations/layer18_attention_weight_min": -39.066017150878906,
      "activations/layer19_attention_weight_max": 24.153430938720703,
      "activations/layer19_attention_weight_min": -24.37041664123535,
      "activations/layer1_attention_weight_max": 16.88417625427246,
      "activations/layer1_attention_weight_min": -15.675945281982422,
      "activations/layer20_attention_weight_max": 27.531875610351562,
      "activations/layer20_attention_weight_min": -21.25164794921875,
      "activations/layer21_attention_weight_max": 40.36003494262695,
      "activations/layer21_attention_weight_min": -24.833433151245117,
      "activations/layer22_attention_weight_max": 31.582979202270508,
      "activations/layer22_attention_weight_min": -26.596927642822266,
      "activations/layer23_attention_weight_max": 39.54364013671875,
      "activations/layer23_attention_weight_min": -25.288494110107422,
      "activations/layer2_attention_weight_max": 33.84156036376953,
      "activations/layer2_attention_weight_min": -32.77081298828125,
      "activations/layer3_attention_weight_max": 94.2467041015625,
      "activations/layer3_attention_weight_min": -100.20509338378906,
      "activations/layer4_attention_weight_max": 104.5433578491211,
      "activations/layer4_attention_weight_min": -99.99760437011719,
      "activations/layer5_attention_weight_max": 77.718994140625,
      "activations/layer5_attention_weight_min": -84.89800262451172,
      "activations/layer6_attention_weight_max": 61.57817459106445,
      "activations/layer6_attention_weight_min": -61.076271057128906,
      "activations/layer7_attention_weight_max": 72.88166046142578,
      "activations/layer7_attention_weight_min": -70.66111755371094,
      "activations/layer8_attention_weight_max": 54.20088577270508,
      "activations/layer8_attention_weight_min": -55.20878219604492,
      "activations/layer9_attention_weight_max": 50.83058166503906,
      "activations/layer9_attention_weight_min": -52.343505859375,
      "epoch": 12.44,
      "learning_rate": 7.046666666666667e-05,
      "loss": 2.7442,
      "step": 214150
    },
    {
      "activations/layer0_attention_weight_max": 15.924178123474121,
      "activations/layer0_attention_weight_min": -13.942326545715332,
      "activations/layer10_attention_weight_max": 38.492462158203125,
      "activations/layer10_attention_weight_min": -38.17497634887695,
      "activations/layer11_attention_weight_max": 35.27423095703125,
      "activations/layer11_attention_weight_min": -35.368568420410156,
      "activations/layer12_attention_weight_max": 20.458011627197266,
      "activations/layer12_attention_weight_min": -25.642311096191406,
      "activations/layer13_attention_weight_max": 43.42192459106445,
      "activations/layer13_attention_weight_min": -28.42918586730957,
      "activations/layer14_attention_weight_max": 40.88197326660156,
      "activations/layer14_attention_weight_min": -32.20750427246094,
      "activations/layer15_attention_weight_max": 38.85808181762695,
      "activations/layer15_attention_weight_min": -33.16086196899414,
      "activations/layer16_attention_weight_max": 34.89854049682617,
      "activations/layer16_attention_weight_min": -32.33964920043945,
      "activations/layer17_attention_weight_max": 55.26348114013672,
      "activations/layer17_attention_weight_min": -46.93739700317383,
      "activations/layer18_attention_weight_max": 50.430301666259766,
      "activations/layer18_attention_weight_min": -40.182716369628906,
      "activations/layer19_attention_weight_max": 24.39021110534668,
      "activations/layer19_attention_weight_min": -24.368452072143555,
      "activations/layer1_attention_weight_max": 16.902202606201172,
      "activations/layer1_attention_weight_min": -14.649724960327148,
      "activations/layer20_attention_weight_max": 25.4787540435791,
      "activations/layer20_attention_weight_min": -23.89208221435547,
      "activations/layer21_attention_weight_max": 43.39686965942383,
      "activations/layer21_attention_weight_min": -24.744915008544922,
      "activations/layer22_attention_weight_max": 29.036808013916016,
      "activations/layer22_attention_weight_min": -26.73410415649414,
      "activations/layer23_attention_weight_max": 37.745140075683594,
      "activations/layer23_attention_weight_min": -23.524999618530273,
      "activations/layer2_attention_weight_max": 36.14216995239258,
      "activations/layer2_attention_weight_min": -33.538185119628906,
      "activations/layer3_attention_weight_max": 89.98668670654297,
      "activations/layer3_attention_weight_min": -94.81295013427734,
      "activations/layer4_attention_weight_max": 95.70933532714844,
      "activations/layer4_attention_weight_min": -89.60221099853516,
      "activations/layer5_attention_weight_max": 67.06604766845703,
      "activations/layer5_attention_weight_min": -75.58558654785156,
      "activations/layer6_attention_weight_max": 51.91645050048828,
      "activations/layer6_attention_weight_min": -52.14389419555664,
      "activations/layer7_attention_weight_max": 66.77501678466797,
      "activations/layer7_attention_weight_min": -66.70368194580078,
      "activations/layer8_attention_weight_max": 45.8883056640625,
      "activations/layer8_attention_weight_min": -48.10746383666992,
      "activations/layer9_attention_weight_max": 48.30440902709961,
      "activations/layer9_attention_weight_min": -48.356666564941406,
      "epoch": 12.45,
      "learning_rate": 7.044772727272727e-05,
      "loss": 2.7616,
      "step": 214200
    },
    {
      "activations/layer0_attention_weight_max": 16.670108795166016,
      "activations/layer0_attention_weight_min": -13.90625286102295,
      "activations/layer10_attention_weight_max": 39.3970947265625,
      "activations/layer10_attention_weight_min": -38.81517028808594,
      "activations/layer11_attention_weight_max": 34.605506896972656,
      "activations/layer11_attention_weight_min": -36.066810607910156,
      "activations/layer12_attention_weight_max": 18.250995635986328,
      "activations/layer12_attention_weight_min": -24.874589920043945,
      "activations/layer13_attention_weight_max": 38.27918243408203,
      "activations/layer13_attention_weight_min": -28.87162208557129,
      "activations/layer14_attention_weight_max": 36.47285842895508,
      "activations/layer14_attention_weight_min": -31.122314453125,
      "activations/layer15_attention_weight_max": 32.44374084472656,
      "activations/layer15_attention_weight_min": -30.454021453857422,
      "activations/layer16_attention_weight_max": 30.479387283325195,
      "activations/layer16_attention_weight_min": -30.93376350402832,
      "activations/layer17_attention_weight_max": 52.58711242675781,
      "activations/layer17_attention_weight_min": -44.71125793457031,
      "activations/layer18_attention_weight_max": 43.174198150634766,
      "activations/layer18_attention_weight_min": -39.04641342163086,
      "activations/layer19_attention_weight_max": 21.74233055114746,
      "activations/layer19_attention_weight_min": -21.109888076782227,
      "activations/layer1_attention_weight_max": 16.86897087097168,
      "activations/layer1_attention_weight_min": -15.040698051452637,
      "activations/layer20_attention_weight_max": 22.514657974243164,
      "activations/layer20_attention_weight_min": -22.14421272277832,
      "activations/layer21_attention_weight_max": 36.67543411254883,
      "activations/layer21_attention_weight_min": -24.13907814025879,
      "activations/layer22_attention_weight_max": 28.60284996032715,
      "activations/layer22_attention_weight_min": -26.70908546447754,
      "activations/layer23_attention_weight_max": 41.291526794433594,
      "activations/layer23_attention_weight_min": -24.54644775390625,
      "activations/layer2_attention_weight_max": 31.129928588867188,
      "activations/layer2_attention_weight_min": -31.080368041992188,
      "activations/layer3_attention_weight_max": 88.99560546875,
      "activations/layer3_attention_weight_min": -99.95040130615234,
      "activations/layer4_attention_weight_max": 98.28936767578125,
      "activations/layer4_attention_weight_min": -102.73140716552734,
      "activations/layer5_attention_weight_max": 72.19113159179688,
      "activations/layer5_attention_weight_min": -81.27729797363281,
      "activations/layer6_attention_weight_max": 53.10742950439453,
      "activations/layer6_attention_weight_min": -54.001956939697266,
      "activations/layer7_attention_weight_max": 75.04009246826172,
      "activations/layer7_attention_weight_min": -68.83090209960938,
      "activations/layer8_attention_weight_max": 46.90842056274414,
      "activations/layer8_attention_weight_min": -49.61603546142578,
      "activations/layer9_attention_weight_max": 50.97878646850586,
      "activations/layer9_attention_weight_min": -48.69232177734375,
      "epoch": 12.45,
      "learning_rate": 7.042878787878787e-05,
      "loss": 2.7433,
      "step": 214250
    },
    {
      "activations/layer0_attention_weight_max": 16.143342971801758,
      "activations/layer0_attention_weight_min": -13.754851341247559,
      "activations/layer10_attention_weight_max": 34.305545806884766,
      "activations/layer10_attention_weight_min": -36.051300048828125,
      "activations/layer11_attention_weight_max": 30.992094039916992,
      "activations/layer11_attention_weight_min": -33.27970886230469,
      "activations/layer12_attention_weight_max": 17.53546714782715,
      "activations/layer12_attention_weight_min": -22.75229263305664,
      "activations/layer13_attention_weight_max": 39.180145263671875,
      "activations/layer13_attention_weight_min": -29.655532836914062,
      "activations/layer14_attention_weight_max": 33.8194694519043,
      "activations/layer14_attention_weight_min": -29.15901756286621,
      "activations/layer15_attention_weight_max": 31.3310546875,
      "activations/layer15_attention_weight_min": -29.71915054321289,
      "activations/layer16_attention_weight_max": 32.05548858642578,
      "activations/layer16_attention_weight_min": -30.830577850341797,
      "activations/layer17_attention_weight_max": 52.50469207763672,
      "activations/layer17_attention_weight_min": -45.68837356567383,
      "activations/layer18_attention_weight_max": 44.86320877075195,
      "activations/layer18_attention_weight_min": -40.919803619384766,
      "activations/layer19_attention_weight_max": 22.63533592224121,
      "activations/layer19_attention_weight_min": -23.625965118408203,
      "activations/layer1_attention_weight_max": 16.288564682006836,
      "activations/layer1_attention_weight_min": -15.253923416137695,
      "activations/layer20_attention_weight_max": 23.62346076965332,
      "activations/layer20_attention_weight_min": -22.939382553100586,
      "activations/layer21_attention_weight_max": 35.84278106689453,
      "activations/layer21_attention_weight_min": -24.286527633666992,
      "activations/layer22_attention_weight_max": 29.36572265625,
      "activations/layer22_attention_weight_min": -25.885150909423828,
      "activations/layer23_attention_weight_max": 35.753082275390625,
      "activations/layer23_attention_weight_min": -25.29379653930664,
      "activations/layer2_attention_weight_max": 32.86448669433594,
      "activations/layer2_attention_weight_min": -32.75833511352539,
      "activations/layer3_attention_weight_max": 94.64447021484375,
      "activations/layer3_attention_weight_min": -102.29338073730469,
      "activations/layer4_attention_weight_max": 99.384765625,
      "activations/layer4_attention_weight_min": -100.51114654541016,
      "activations/layer5_attention_weight_max": 71.41110229492188,
      "activations/layer5_attention_weight_min": -76.9569320678711,
      "activations/layer6_attention_weight_max": 54.06180191040039,
      "activations/layer6_attention_weight_min": -53.565982818603516,
      "activations/layer7_attention_weight_max": 71.61589050292969,
      "activations/layer7_attention_weight_min": -68.57955169677734,
      "activations/layer8_attention_weight_max": 43.54047775268555,
      "activations/layer8_attention_weight_min": -51.00282669067383,
      "activations/layer9_attention_weight_max": 45.11101150512695,
      "activations/layer9_attention_weight_min": -48.07042694091797,
      "epoch": 12.45,
      "learning_rate": 7.040984848484847e-05,
      "loss": 2.7479,
      "step": 214300
    },
    {
      "activations/layer0_attention_weight_max": 16.903234481811523,
      "activations/layer0_attention_weight_min": -14.173954010009766,
      "activations/layer10_attention_weight_max": 39.41294860839844,
      "activations/layer10_attention_weight_min": -36.45844268798828,
      "activations/layer11_attention_weight_max": 33.5842399597168,
      "activations/layer11_attention_weight_min": -34.45390319824219,
      "activations/layer12_attention_weight_max": 18.650554656982422,
      "activations/layer12_attention_weight_min": -25.896263122558594,
      "activations/layer13_attention_weight_max": 37.5406608581543,
      "activations/layer13_attention_weight_min": -28.4012393951416,
      "activations/layer14_attention_weight_max": 39.54948425292969,
      "activations/layer14_attention_weight_min": -29.22339630126953,
      "activations/layer15_attention_weight_max": 33.389156341552734,
      "activations/layer15_attention_weight_min": -30.141952514648438,
      "activations/layer16_attention_weight_max": 33.14157485961914,
      "activations/layer16_attention_weight_min": -30.752286911010742,
      "activations/layer17_attention_weight_max": 54.319252014160156,
      "activations/layer17_attention_weight_min": -49.45708465576172,
      "activations/layer18_attention_weight_max": 48.65229415893555,
      "activations/layer18_attention_weight_min": -39.790775299072266,
      "activations/layer19_attention_weight_max": 24.674917221069336,
      "activations/layer19_attention_weight_min": -25.649877548217773,
      "activations/layer1_attention_weight_max": 16.821239471435547,
      "activations/layer1_attention_weight_min": -16.507871627807617,
      "activations/layer20_attention_weight_max": 23.142566680908203,
      "activations/layer20_attention_weight_min": -21.20753288269043,
      "activations/layer21_attention_weight_max": 39.54366683959961,
      "activations/layer21_attention_weight_min": -23.58841323852539,
      "activations/layer22_attention_weight_max": 29.863162994384766,
      "activations/layer22_attention_weight_min": -27.66956329345703,
      "activations/layer23_attention_weight_max": 39.3463249206543,
      "activations/layer23_attention_weight_min": -22.11812973022461,
      "activations/layer2_attention_weight_max": 32.54148483276367,
      "activations/layer2_attention_weight_min": -32.19377136230469,
      "activations/layer3_attention_weight_max": 92.00131225585938,
      "activations/layer3_attention_weight_min": -101.46112823486328,
      "activations/layer4_attention_weight_max": 102.30984497070312,
      "activations/layer4_attention_weight_min": -98.50873565673828,
      "activations/layer5_attention_weight_max": 74.08911895751953,
      "activations/layer5_attention_weight_min": -78.5813980102539,
      "activations/layer6_attention_weight_max": 54.99516677856445,
      "activations/layer6_attention_weight_min": -55.094913482666016,
      "activations/layer7_attention_weight_max": 73.10038757324219,
      "activations/layer7_attention_weight_min": -70.81407928466797,
      "activations/layer8_attention_weight_max": 47.71590042114258,
      "activations/layer8_attention_weight_min": -49.60130310058594,
      "activations/layer9_attention_weight_max": 45.55088424682617,
      "activations/layer9_attention_weight_min": -46.87477493286133,
      "epoch": 12.45,
      "learning_rate": 7.039090909090909e-05,
      "loss": 2.7604,
      "step": 214350
    },
    {
      "activations/layer0_attention_weight_max": 15.805076599121094,
      "activations/layer0_attention_weight_min": -13.636269569396973,
      "activations/layer10_attention_weight_max": 37.352046966552734,
      "activations/layer10_attention_weight_min": -36.28828811645508,
      "activations/layer11_attention_weight_max": 34.39647674560547,
      "activations/layer11_attention_weight_min": -33.436309814453125,
      "activations/layer12_attention_weight_max": 20.598569869995117,
      "activations/layer12_attention_weight_min": -25.02482032775879,
      "activations/layer13_attention_weight_max": 36.56283187866211,
      "activations/layer13_attention_weight_min": -28.472064971923828,
      "activations/layer14_attention_weight_max": 36.664852142333984,
      "activations/layer14_attention_weight_min": -32.6186408996582,
      "activations/layer15_attention_weight_max": 34.213478088378906,
      "activations/layer15_attention_weight_min": -29.869810104370117,
      "activations/layer16_attention_weight_max": 34.060604095458984,
      "activations/layer16_attention_weight_min": -32.181007385253906,
      "activations/layer17_attention_weight_max": 54.96879577636719,
      "activations/layer17_attention_weight_min": -46.59977340698242,
      "activations/layer18_attention_weight_max": 50.01727294921875,
      "activations/layer18_attention_weight_min": -42.5151252746582,
      "activations/layer19_attention_weight_max": 23.628154754638672,
      "activations/layer19_attention_weight_min": -23.437793731689453,
      "activations/layer1_attention_weight_max": 16.869155883789062,
      "activations/layer1_attention_weight_min": -14.979970932006836,
      "activations/layer20_attention_weight_max": 28.021331787109375,
      "activations/layer20_attention_weight_min": -21.232311248779297,
      "activations/layer21_attention_weight_max": 40.6685905456543,
      "activations/layer21_attention_weight_min": -23.315475463867188,
      "activations/layer22_attention_weight_max": 35.34196472167969,
      "activations/layer22_attention_weight_min": -26.67405128479004,
      "activations/layer23_attention_weight_max": 44.34890365600586,
      "activations/layer23_attention_weight_min": -23.742244720458984,
      "activations/layer2_attention_weight_max": 32.0588493347168,
      "activations/layer2_attention_weight_min": -32.70329284667969,
      "activations/layer3_attention_weight_max": 93.3720932006836,
      "activations/layer3_attention_weight_min": -100.86257934570312,
      "activations/layer4_attention_weight_max": 97.44634246826172,
      "activations/layer4_attention_weight_min": -93.88967895507812,
      "activations/layer5_attention_weight_max": 70.35752868652344,
      "activations/layer5_attention_weight_min": -82.04507446289062,
      "activations/layer6_attention_weight_max": 56.551944732666016,
      "activations/layer6_attention_weight_min": -54.560462951660156,
      "activations/layer7_attention_weight_max": 71.58525085449219,
      "activations/layer7_attention_weight_min": -69.12906646728516,
      "activations/layer8_attention_weight_max": 47.18381881713867,
      "activations/layer8_attention_weight_min": -49.85586166381836,
      "activations/layer9_attention_weight_max": 46.32563781738281,
      "activations/layer9_attention_weight_min": -49.98627853393555,
      "epoch": 12.46,
      "learning_rate": 7.037196969696969e-05,
      "loss": 2.7523,
      "step": 214400
    },
    {
      "activations/layer0_attention_weight_max": 15.981369018554688,
      "activations/layer0_attention_weight_min": -13.737716674804688,
      "activations/layer10_attention_weight_max": 37.832603454589844,
      "activations/layer10_attention_weight_min": -37.54130935668945,
      "activations/layer11_attention_weight_max": 34.436973571777344,
      "activations/layer11_attention_weight_min": -34.539527893066406,
      "activations/layer12_attention_weight_max": 19.956825256347656,
      "activations/layer12_attention_weight_min": -24.08099365234375,
      "activations/layer13_attention_weight_max": 37.60442352294922,
      "activations/layer13_attention_weight_min": -28.597002029418945,
      "activations/layer14_attention_weight_max": 37.15083694458008,
      "activations/layer14_attention_weight_min": -29.971738815307617,
      "activations/layer15_attention_weight_max": 32.79583740234375,
      "activations/layer15_attention_weight_min": -29.804229736328125,
      "activations/layer16_attention_weight_max": 32.993499755859375,
      "activations/layer16_attention_weight_min": -32.63513946533203,
      "activations/layer17_attention_weight_max": 55.18037414550781,
      "activations/layer17_attention_weight_min": -48.00469207763672,
      "activations/layer18_attention_weight_max": 50.04296112060547,
      "activations/layer18_attention_weight_min": -40.0908088684082,
      "activations/layer19_attention_weight_max": 26.53793716430664,
      "activations/layer19_attention_weight_min": -20.823755264282227,
      "activations/layer1_attention_weight_max": 17.303098678588867,
      "activations/layer1_attention_weight_min": -16.206506729125977,
      "activations/layer20_attention_weight_max": 23.620452880859375,
      "activations/layer20_attention_weight_min": -21.02672004699707,
      "activations/layer21_attention_weight_max": 37.104557037353516,
      "activations/layer21_attention_weight_min": -24.11026954650879,
      "activations/layer22_attention_weight_max": 33.15741729736328,
      "activations/layer22_attention_weight_min": -25.073030471801758,
      "activations/layer23_attention_weight_max": 39.63690948486328,
      "activations/layer23_attention_weight_min": -23.73986053466797,
      "activations/layer2_attention_weight_max": 32.60151672363281,
      "activations/layer2_attention_weight_min": -30.45998191833496,
      "activations/layer3_attention_weight_max": 94.94398498535156,
      "activations/layer3_attention_weight_min": -94.33333587646484,
      "activations/layer4_attention_weight_max": 97.95425415039062,
      "activations/layer4_attention_weight_min": -97.13347625732422,
      "activations/layer5_attention_weight_max": 74.70498657226562,
      "activations/layer5_attention_weight_min": -77.04898071289062,
      "activations/layer6_attention_weight_max": 55.27741241455078,
      "activations/layer6_attention_weight_min": -56.46944046020508,
      "activations/layer7_attention_weight_max": 73.46910858154297,
      "activations/layer7_attention_weight_min": -69.67855072021484,
      "activations/layer8_attention_weight_max": 49.12836456298828,
      "activations/layer8_attention_weight_min": -52.65510940551758,
      "activations/layer9_attention_weight_max": 51.05910110473633,
      "activations/layer9_attention_weight_min": -50.855430603027344,
      "epoch": 12.46,
      "learning_rate": 7.035303030303029e-05,
      "loss": 2.7631,
      "step": 214450
    },
    {
      "activations/layer0_attention_weight_max": 15.900391578674316,
      "activations/layer0_attention_weight_min": -13.684123039245605,
      "activations/layer10_attention_weight_max": 35.323204040527344,
      "activations/layer10_attention_weight_min": -36.00312805175781,
      "activations/layer11_attention_weight_max": 31.686721801757812,
      "activations/layer11_attention_weight_min": -34.18815612792969,
      "activations/layer12_attention_weight_max": 19.942928314208984,
      "activations/layer12_attention_weight_min": -28.51498794555664,
      "activations/layer13_attention_weight_max": 38.10535430908203,
      "activations/layer13_attention_weight_min": -31.45613670349121,
      "activations/layer14_attention_weight_max": 35.35306930541992,
      "activations/layer14_attention_weight_min": -30.535661697387695,
      "activations/layer15_attention_weight_max": 31.424999237060547,
      "activations/layer15_attention_weight_min": -30.33977699279785,
      "activations/layer16_attention_weight_max": 29.619657516479492,
      "activations/layer16_attention_weight_min": -30.41669464111328,
      "activations/layer17_attention_weight_max": 50.55519485473633,
      "activations/layer17_attention_weight_min": -46.266361236572266,
      "activations/layer18_attention_weight_max": 44.240447998046875,
      "activations/layer18_attention_weight_min": -38.67314529418945,
      "activations/layer19_attention_weight_max": 21.172855377197266,
      "activations/layer19_attention_weight_min": -22.39948844909668,
      "activations/layer1_attention_weight_max": 16.207521438598633,
      "activations/layer1_attention_weight_min": -13.829557418823242,
      "activations/layer20_attention_weight_max": 19.658428192138672,
      "activations/layer20_attention_weight_min": -25.92638397216797,
      "activations/layer21_attention_weight_max": 35.219573974609375,
      "activations/layer21_attention_weight_min": -21.632944107055664,
      "activations/layer22_attention_weight_max": 30.174306869506836,
      "activations/layer22_attention_weight_min": -26.464006423950195,
      "activations/layer23_attention_weight_max": 36.71355056762695,
      "activations/layer23_attention_weight_min": -23.7862548828125,
      "activations/layer2_attention_weight_max": 32.570457458496094,
      "activations/layer2_attention_weight_min": -33.17420959472656,
      "activations/layer3_attention_weight_max": 94.60543060302734,
      "activations/layer3_attention_weight_min": -93.87748718261719,
      "activations/layer4_attention_weight_max": 101.85464477539062,
      "activations/layer4_attention_weight_min": -95.39214324951172,
      "activations/layer5_attention_weight_max": 72.472412109375,
      "activations/layer5_attention_weight_min": -77.1099853515625,
      "activations/layer6_attention_weight_max": 55.89017105102539,
      "activations/layer6_attention_weight_min": -53.44740676879883,
      "activations/layer7_attention_weight_max": 67.53988647460938,
      "activations/layer7_attention_weight_min": -68.86204528808594,
      "activations/layer8_attention_weight_max": 46.41965866088867,
      "activations/layer8_attention_weight_min": -48.26205062866211,
      "activations/layer9_attention_weight_max": 46.52748489379883,
      "activations/layer9_attention_weight_min": -48.81686019897461,
      "epoch": 12.46,
      "learning_rate": 7.03340909090909e-05,
      "loss": 2.756,
      "step": 214500
    },
    {
      "activations/layer0_attention_weight_max": 16.74353790283203,
      "activations/layer0_attention_weight_min": -14.139484405517578,
      "activations/layer10_attention_weight_max": 37.377662658691406,
      "activations/layer10_attention_weight_min": -37.373226165771484,
      "activations/layer11_attention_weight_max": 33.85797119140625,
      "activations/layer11_attention_weight_min": -33.51646423339844,
      "activations/layer12_attention_weight_max": 20.29572296142578,
      "activations/layer12_attention_weight_min": -23.587663650512695,
      "activations/layer13_attention_weight_max": 41.53638458251953,
      "activations/layer13_attention_weight_min": -32.17870330810547,
      "activations/layer14_attention_weight_max": 43.78190231323242,
      "activations/layer14_attention_weight_min": -32.31547546386719,
      "activations/layer15_attention_weight_max": 37.98271560668945,
      "activations/layer15_attention_weight_min": -29.84268569946289,
      "activations/layer16_attention_weight_max": 33.8953742980957,
      "activations/layer16_attention_weight_min": -29.776002883911133,
      "activations/layer17_attention_weight_max": 57.75086975097656,
      "activations/layer17_attention_weight_min": -46.86186981201172,
      "activations/layer18_attention_weight_max": 50.428619384765625,
      "activations/layer18_attention_weight_min": -40.365997314453125,
      "activations/layer19_attention_weight_max": 23.971271514892578,
      "activations/layer19_attention_weight_min": -23.173215866088867,
      "activations/layer1_attention_weight_max": 16.071470260620117,
      "activations/layer1_attention_weight_min": -16.08550453186035,
      "activations/layer20_attention_weight_max": 24.40154457092285,
      "activations/layer20_attention_weight_min": -23.99363899230957,
      "activations/layer21_attention_weight_max": 39.946563720703125,
      "activations/layer21_attention_weight_min": -24.662878036499023,
      "activations/layer22_attention_weight_max": 34.4552001953125,
      "activations/layer22_attention_weight_min": -27.30316162109375,
      "activations/layer23_attention_weight_max": 42.616615295410156,
      "activations/layer23_attention_weight_min": -24.045066833496094,
      "activations/layer2_attention_weight_max": 33.40947723388672,
      "activations/layer2_attention_weight_min": -31.632545471191406,
      "activations/layer3_attention_weight_max": 93.16178894042969,
      "activations/layer3_attention_weight_min": -99.08625030517578,
      "activations/layer4_attention_weight_max": 98.64806365966797,
      "activations/layer4_attention_weight_min": -92.51233673095703,
      "activations/layer5_attention_weight_max": 75.73884582519531,
      "activations/layer5_attention_weight_min": -78.36039733886719,
      "activations/layer6_attention_weight_max": 54.84195327758789,
      "activations/layer6_attention_weight_min": -54.96577453613281,
      "activations/layer7_attention_weight_max": 72.11498260498047,
      "activations/layer7_attention_weight_min": -69.70536804199219,
      "activations/layer8_attention_weight_max": 46.316410064697266,
      "activations/layer8_attention_weight_min": -48.12541961669922,
      "activations/layer9_attention_weight_max": 48.10298156738281,
      "activations/layer9_attention_weight_min": -48.840492248535156,
      "epoch": 12.47,
      "learning_rate": 7.03151515151515e-05,
      "loss": 2.7625,
      "step": 214550
    },
    {
      "activations/layer0_attention_weight_max": 17.066118240356445,
      "activations/layer0_attention_weight_min": -13.765021324157715,
      "activations/layer10_attention_weight_max": 39.6555290222168,
      "activations/layer10_attention_weight_min": -36.1755485534668,
      "activations/layer11_attention_weight_max": 35.78679275512695,
      "activations/layer11_attention_weight_min": -34.20001220703125,
      "activations/layer12_attention_weight_max": 19.126495361328125,
      "activations/layer12_attention_weight_min": -23.06336212158203,
      "activations/layer13_attention_weight_max": 50.7291374206543,
      "activations/layer13_attention_weight_min": -30.212547302246094,
      "activations/layer14_attention_weight_max": 39.64566421508789,
      "activations/layer14_attention_weight_min": -32.89046096801758,
      "activations/layer15_attention_weight_max": 42.783416748046875,
      "activations/layer15_attention_weight_min": -31.58504295349121,
      "activations/layer16_attention_weight_max": 36.10348892211914,
      "activations/layer16_attention_weight_min": -34.572593688964844,
      "activations/layer17_attention_weight_max": 59.92733383178711,
      "activations/layer17_attention_weight_min": -51.4111442565918,
      "activations/layer18_attention_weight_max": 52.23189926147461,
      "activations/layer18_attention_weight_min": -43.15906524658203,
      "activations/layer19_attention_weight_max": 25.441923141479492,
      "activations/layer19_attention_weight_min": -25.075305938720703,
      "activations/layer1_attention_weight_max": 17.21750259399414,
      "activations/layer1_attention_weight_min": -15.824723243713379,
      "activations/layer20_attention_weight_max": 25.840747833251953,
      "activations/layer20_attention_weight_min": -22.917465209960938,
      "activations/layer21_attention_weight_max": 46.22465896606445,
      "activations/layer21_attention_weight_min": -25.77170181274414,
      "activations/layer22_attention_weight_max": 35.329856872558594,
      "activations/layer22_attention_weight_min": -26.503950119018555,
      "activations/layer23_attention_weight_max": 40.77776336669922,
      "activations/layer23_attention_weight_min": -27.49787139892578,
      "activations/layer2_attention_weight_max": 33.951507568359375,
      "activations/layer2_attention_weight_min": -32.80430221557617,
      "activations/layer3_attention_weight_max": 92.294677734375,
      "activations/layer3_attention_weight_min": -97.1779556274414,
      "activations/layer4_attention_weight_max": 98.6778335571289,
      "activations/layer4_attention_weight_min": -93.07915496826172,
      "activations/layer5_attention_weight_max": 71.24603271484375,
      "activations/layer5_attention_weight_min": -76.19332885742188,
      "activations/layer6_attention_weight_max": 53.99388122558594,
      "activations/layer6_attention_weight_min": -55.2445068359375,
      "activations/layer7_attention_weight_max": 69.7018814086914,
      "activations/layer7_attention_weight_min": -68.6371078491211,
      "activations/layer8_attention_weight_max": 46.42914581298828,
      "activations/layer8_attention_weight_min": -50.419071197509766,
      "activations/layer9_attention_weight_max": 47.51620101928711,
      "activations/layer9_attention_weight_min": -47.44723892211914,
      "epoch": 12.47,
      "learning_rate": 7.029621212121211e-05,
      "loss": 2.7419,
      "step": 214600
    },
    {
      "activations/layer0_attention_weight_max": 15.462244033813477,
      "activations/layer0_attention_weight_min": -14.21319580078125,
      "activations/layer10_attention_weight_max": 38.022525787353516,
      "activations/layer10_attention_weight_min": -37.49347686767578,
      "activations/layer11_attention_weight_max": 35.813720703125,
      "activations/layer11_attention_weight_min": -32.9632568359375,
      "activations/layer12_attention_weight_max": 18.59375762939453,
      "activations/layer12_attention_weight_min": -28.320514678955078,
      "activations/layer13_attention_weight_max": 34.03265380859375,
      "activations/layer13_attention_weight_min": -28.39211082458496,
      "activations/layer14_attention_weight_max": 36.77973556518555,
      "activations/layer14_attention_weight_min": -31.89081573486328,
      "activations/layer15_attention_weight_max": 35.223114013671875,
      "activations/layer15_attention_weight_min": -29.999277114868164,
      "activations/layer16_attention_weight_max": 31.583349227905273,
      "activations/layer16_attention_weight_min": -29.12546157836914,
      "activations/layer17_attention_weight_max": 52.496726989746094,
      "activations/layer17_attention_weight_min": -47.40353012084961,
      "activations/layer18_attention_weight_max": 46.25936508178711,
      "activations/layer18_attention_weight_min": -40.2760009765625,
      "activations/layer19_attention_weight_max": 22.689353942871094,
      "activations/layer19_attention_weight_min": -22.713571548461914,
      "activations/layer1_attention_weight_max": 16.819677352905273,
      "activations/layer1_attention_weight_min": -14.616435050964355,
      "activations/layer20_attention_weight_max": 22.182798385620117,
      "activations/layer20_attention_weight_min": -22.68356704711914,
      "activations/layer21_attention_weight_max": 36.41667938232422,
      "activations/layer21_attention_weight_min": -22.95703125,
      "activations/layer22_attention_weight_max": 32.92656707763672,
      "activations/layer22_attention_weight_min": -25.398712158203125,
      "activations/layer23_attention_weight_max": 37.8432731628418,
      "activations/layer23_attention_weight_min": -23.907398223876953,
      "activations/layer2_attention_weight_max": 34.875587463378906,
      "activations/layer2_attention_weight_min": -32.763427734375,
      "activations/layer3_attention_weight_max": 95.37863159179688,
      "activations/layer3_attention_weight_min": -99.96128845214844,
      "activations/layer4_attention_weight_max": 102.03175354003906,
      "activations/layer4_attention_weight_min": -103.42097473144531,
      "activations/layer5_attention_weight_max": 75.64541625976562,
      "activations/layer5_attention_weight_min": -80.23492431640625,
      "activations/layer6_attention_weight_max": 55.653350830078125,
      "activations/layer6_attention_weight_min": -56.08304214477539,
      "activations/layer7_attention_weight_max": 74.193359375,
      "activations/layer7_attention_weight_min": -73.80752563476562,
      "activations/layer8_attention_weight_max": 55.1191291809082,
      "activations/layer8_attention_weight_min": -51.21390151977539,
      "activations/layer9_attention_weight_max": 60.07719802856445,
      "activations/layer9_attention_weight_min": -49.4037971496582,
      "epoch": 12.47,
      "learning_rate": 7.027727272727272e-05,
      "loss": 2.7546,
      "step": 214650
    },
    {
      "activations/layer0_attention_weight_max": 15.982619285583496,
      "activations/layer0_attention_weight_min": -13.992934226989746,
      "activations/layer10_attention_weight_max": 36.145442962646484,
      "activations/layer10_attention_weight_min": -33.52096939086914,
      "activations/layer11_attention_weight_max": 29.97823143005371,
      "activations/layer11_attention_weight_min": -31.39826774597168,
      "activations/layer12_attention_weight_max": 18.187271118164062,
      "activations/layer12_attention_weight_min": -24.594276428222656,
      "activations/layer13_attention_weight_max": 36.942073822021484,
      "activations/layer13_attention_weight_min": -27.495576858520508,
      "activations/layer14_attention_weight_max": 34.62675857543945,
      "activations/layer14_attention_weight_min": -31.11789894104004,
      "activations/layer15_attention_weight_max": 31.677675247192383,
      "activations/layer15_attention_weight_min": -31.474576950073242,
      "activations/layer16_attention_weight_max": 29.736751556396484,
      "activations/layer16_attention_weight_min": -30.2872314453125,
      "activations/layer17_attention_weight_max": 51.577186584472656,
      "activations/layer17_attention_weight_min": -46.175357818603516,
      "activations/layer18_attention_weight_max": 47.75316619873047,
      "activations/layer18_attention_weight_min": -37.539581298828125,
      "activations/layer19_attention_weight_max": 21.78675651550293,
      "activations/layer19_attention_weight_min": -21.658340454101562,
      "activations/layer1_attention_weight_max": 16.323396682739258,
      "activations/layer1_attention_weight_min": -15.181138038635254,
      "activations/layer20_attention_weight_max": 23.424781799316406,
      "activations/layer20_attention_weight_min": -21.855863571166992,
      "activations/layer21_attention_weight_max": 33.77870178222656,
      "activations/layer21_attention_weight_min": -23.756134033203125,
      "activations/layer22_attention_weight_max": 30.581384658813477,
      "activations/layer22_attention_weight_min": -26.926158905029297,
      "activations/layer23_attention_weight_max": 35.88318634033203,
      "activations/layer23_attention_weight_min": -24.548480987548828,
      "activations/layer2_attention_weight_max": 31.623416900634766,
      "activations/layer2_attention_weight_min": -30.478063583374023,
      "activations/layer3_attention_weight_max": 89.58264923095703,
      "activations/layer3_attention_weight_min": -90.20054626464844,
      "activations/layer4_attention_weight_max": 90.00664520263672,
      "activations/layer4_attention_weight_min": -89.20706939697266,
      "activations/layer5_attention_weight_max": 66.85697937011719,
      "activations/layer5_attention_weight_min": -76.86808776855469,
      "activations/layer6_attention_weight_max": 52.47254180908203,
      "activations/layer6_attention_weight_min": -56.02937316894531,
      "activations/layer7_attention_weight_max": 66.89456176757812,
      "activations/layer7_attention_weight_min": -64.76661682128906,
      "activations/layer8_attention_weight_max": 42.566524505615234,
      "activations/layer8_attention_weight_min": -45.50027847290039,
      "activations/layer9_attention_weight_max": 41.439884185791016,
      "activations/layer9_attention_weight_min": -43.43033981323242,
      "epoch": 12.48,
      "learning_rate": 7.025833333333332e-05,
      "loss": 2.7575,
      "step": 214700
    },
    {
      "activations/layer0_attention_weight_max": 15.715927124023438,
      "activations/layer0_attention_weight_min": -13.90226936340332,
      "activations/layer10_attention_weight_max": 34.039127349853516,
      "activations/layer10_attention_weight_min": -32.906898498535156,
      "activations/layer11_attention_weight_max": 33.32731246948242,
      "activations/layer11_attention_weight_min": -31.45074462890625,
      "activations/layer12_attention_weight_max": 16.68216323852539,
      "activations/layer12_attention_weight_min": -24.38079071044922,
      "activations/layer13_attention_weight_max": 48.14057159423828,
      "activations/layer13_attention_weight_min": -32.032161712646484,
      "activations/layer14_attention_weight_max": 34.313392639160156,
      "activations/layer14_attention_weight_min": -30.109424591064453,
      "activations/layer15_attention_weight_max": 31.039363861083984,
      "activations/layer15_attention_weight_min": -33.45783615112305,
      "activations/layer16_attention_weight_max": 29.9970645904541,
      "activations/layer16_attention_weight_min": -30.52447509765625,
      "activations/layer17_attention_weight_max": 50.6651496887207,
      "activations/layer17_attention_weight_min": -45.35610580444336,
      "activations/layer18_attention_weight_max": 42.9621467590332,
      "activations/layer18_attention_weight_min": -40.067848205566406,
      "activations/layer19_attention_weight_max": 21.319982528686523,
      "activations/layer19_attention_weight_min": -22.0427303314209,
      "activations/layer1_attention_weight_max": 16.764753341674805,
      "activations/layer1_attention_weight_min": -15.513440132141113,
      "activations/layer20_attention_weight_max": 20.915706634521484,
      "activations/layer20_attention_weight_min": -20.579002380371094,
      "activations/layer21_attention_weight_max": 35.7886848449707,
      "activations/layer21_attention_weight_min": -22.699581146240234,
      "activations/layer22_attention_weight_max": 29.009891510009766,
      "activations/layer22_attention_weight_min": -27.684038162231445,
      "activations/layer23_attention_weight_max": 37.16569519042969,
      "activations/layer23_attention_weight_min": -24.630382537841797,
      "activations/layer2_attention_weight_max": 33.46820068359375,
      "activations/layer2_attention_weight_min": -30.512107849121094,
      "activations/layer3_attention_weight_max": 93.77769470214844,
      "activations/layer3_attention_weight_min": -95.04914093017578,
      "activations/layer4_attention_weight_max": 95.00263977050781,
      "activations/layer4_attention_weight_min": -94.0857162475586,
      "activations/layer5_attention_weight_max": 68.65971374511719,
      "activations/layer5_attention_weight_min": -76.92835998535156,
      "activations/layer6_attention_weight_max": 52.64894104003906,
      "activations/layer6_attention_weight_min": -51.86886978149414,
      "activations/layer7_attention_weight_max": 69.78050231933594,
      "activations/layer7_attention_weight_min": -69.29376983642578,
      "activations/layer8_attention_weight_max": 46.23649597167969,
      "activations/layer8_attention_weight_min": -47.75149917602539,
      "activations/layer9_attention_weight_max": 45.92910385131836,
      "activations/layer9_attention_weight_min": -45.58464813232422,
      "epoch": 12.48,
      "learning_rate": 7.023939393939394e-05,
      "loss": 2.7703,
      "step": 214750
    },
    {
      "activations/layer0_attention_weight_max": 15.38271427154541,
      "activations/layer0_attention_weight_min": -14.406027793884277,
      "activations/layer10_attention_weight_max": 37.16632080078125,
      "activations/layer10_attention_weight_min": -35.02070999145508,
      "activations/layer11_attention_weight_max": 32.97441482543945,
      "activations/layer11_attention_weight_min": -32.63820266723633,
      "activations/layer12_attention_weight_max": 21.84791374206543,
      "activations/layer12_attention_weight_min": -26.30455207824707,
      "activations/layer13_attention_weight_max": 40.21208572387695,
      "activations/layer13_attention_weight_min": -30.006174087524414,
      "activations/layer14_attention_weight_max": 40.099761962890625,
      "activations/layer14_attention_weight_min": -30.963361740112305,
      "activations/layer15_attention_weight_max": 38.23354721069336,
      "activations/layer15_attention_weight_min": -30.607074737548828,
      "activations/layer16_attention_weight_max": 32.51716613769531,
      "activations/layer16_attention_weight_min": -31.583044052124023,
      "activations/layer17_attention_weight_max": 53.63808059692383,
      "activations/layer17_attention_weight_min": -45.132930755615234,
      "activations/layer18_attention_weight_max": 46.714359283447266,
      "activations/layer18_attention_weight_min": -38.01869583129883,
      "activations/layer19_attention_weight_max": 22.689857482910156,
      "activations/layer19_attention_weight_min": -22.870582580566406,
      "activations/layer1_attention_weight_max": 15.365143775939941,
      "activations/layer1_attention_weight_min": -15.464780807495117,
      "activations/layer20_attention_weight_max": 26.900590896606445,
      "activations/layer20_attention_weight_min": -24.476634979248047,
      "activations/layer21_attention_weight_max": 41.6080322265625,
      "activations/layer21_attention_weight_min": -26.34603500366211,
      "activations/layer22_attention_weight_max": 33.54808807373047,
      "activations/layer22_attention_weight_min": -26.944265365600586,
      "activations/layer23_attention_weight_max": 40.443267822265625,
      "activations/layer23_attention_weight_min": -24.40789031982422,
      "activations/layer2_attention_weight_max": 32.74088668823242,
      "activations/layer2_attention_weight_min": -30.647003173828125,
      "activations/layer3_attention_weight_max": 91.07898712158203,
      "activations/layer3_attention_weight_min": -93.95610046386719,
      "activations/layer4_attention_weight_max": 95.01421356201172,
      "activations/layer4_attention_weight_min": -93.8089599609375,
      "activations/layer5_attention_weight_max": 73.18338012695312,
      "activations/layer5_attention_weight_min": -78.04991912841797,
      "activations/layer6_attention_weight_max": 55.3232536315918,
      "activations/layer6_attention_weight_min": -54.393497467041016,
      "activations/layer7_attention_weight_max": 71.59383392333984,
      "activations/layer7_attention_weight_min": -70.41646575927734,
      "activations/layer8_attention_weight_max": 47.596214294433594,
      "activations/layer8_attention_weight_min": -51.2906494140625,
      "activations/layer9_attention_weight_max": 47.22417068481445,
      "activations/layer9_attention_weight_min": -47.83498764038086,
      "epoch": 12.48,
      "learning_rate": 7.022045454545454e-05,
      "loss": 2.7686,
      "step": 214800
    },
    {
      "activations/layer0_attention_weight_max": 17.198383331298828,
      "activations/layer0_attention_weight_min": -14.46318531036377,
      "activations/layer10_attention_weight_max": 37.29993438720703,
      "activations/layer10_attention_weight_min": -36.36915969848633,
      "activations/layer11_attention_weight_max": 36.107139587402344,
      "activations/layer11_attention_weight_min": -34.465179443359375,
      "activations/layer12_attention_weight_max": 19.3966121673584,
      "activations/layer12_attention_weight_min": -27.524593353271484,
      "activations/layer13_attention_weight_max": 38.53656768798828,
      "activations/layer13_attention_weight_min": -27.230457305908203,
      "activations/layer14_attention_weight_max": 43.26403045654297,
      "activations/layer14_attention_weight_min": -28.117305755615234,
      "activations/layer15_attention_weight_max": 39.857112884521484,
      "activations/layer15_attention_weight_min": -28.89223861694336,
      "activations/layer16_attention_weight_max": 31.43375587463379,
      "activations/layer16_attention_weight_min": -28.85737419128418,
      "activations/layer17_attention_weight_max": 51.2988395690918,
      "activations/layer17_attention_weight_min": -43.79341506958008,
      "activations/layer18_attention_weight_max": 44.233394622802734,
      "activations/layer18_attention_weight_min": -36.27640151977539,
      "activations/layer19_attention_weight_max": 21.82465934753418,
      "activations/layer19_attention_weight_min": -22.754125595092773,
      "activations/layer1_attention_weight_max": 17.78101348876953,
      "activations/layer1_attention_weight_min": -15.120550155639648,
      "activations/layer20_attention_weight_max": 20.95415496826172,
      "activations/layer20_attention_weight_min": -22.10173225402832,
      "activations/layer21_attention_weight_max": 36.67890548706055,
      "activations/layer21_attention_weight_min": -23.266437530517578,
      "activations/layer22_attention_weight_max": 29.76715850830078,
      "activations/layer22_attention_weight_min": -25.626314163208008,
      "activations/layer23_attention_weight_max": 35.501983642578125,
      "activations/layer23_attention_weight_min": -24.532453536987305,
      "activations/layer2_attention_weight_max": 36.46201705932617,
      "activations/layer2_attention_weight_min": -32.27362823486328,
      "activations/layer3_attention_weight_max": 103.82532501220703,
      "activations/layer3_attention_weight_min": -104.32826232910156,
      "activations/layer4_attention_weight_max": 107.2950668334961,
      "activations/layer4_attention_weight_min": -97.69173431396484,
      "activations/layer5_attention_weight_max": 71.44133758544922,
      "activations/layer5_attention_weight_min": -82.58489990234375,
      "activations/layer6_attention_weight_max": 53.754695892333984,
      "activations/layer6_attention_weight_min": -55.33244705200195,
      "activations/layer7_attention_weight_max": 74.46131134033203,
      "activations/layer7_attention_weight_min": -71.9609146118164,
      "activations/layer8_attention_weight_max": 46.594520568847656,
      "activations/layer8_attention_weight_min": -49.284603118896484,
      "activations/layer9_attention_weight_max": 54.15234375,
      "activations/layer9_attention_weight_min": -50.56949234008789,
      "epoch": 12.48,
      "learning_rate": 7.020151515151514e-05,
      "loss": 2.7621,
      "step": 214850
    },
    {
      "activations/layer0_attention_weight_max": 18.016117095947266,
      "activations/layer0_attention_weight_min": -14.129447937011719,
      "activations/layer10_attention_weight_max": 36.916507720947266,
      "activations/layer10_attention_weight_min": -35.12158203125,
      "activations/layer11_attention_weight_max": 32.82688903808594,
      "activations/layer11_attention_weight_min": -32.856239318847656,
      "activations/layer12_attention_weight_max": 19.075092315673828,
      "activations/layer12_attention_weight_min": -24.588211059570312,
      "activations/layer13_attention_weight_max": 36.94951629638672,
      "activations/layer13_attention_weight_min": -30.604965209960938,
      "activations/layer14_attention_weight_max": 36.28718948364258,
      "activations/layer14_attention_weight_min": -29.093780517578125,
      "activations/layer15_attention_weight_max": 34.70671844482422,
      "activations/layer15_attention_weight_min": -29.373077392578125,
      "activations/layer16_attention_weight_max": 31.567461013793945,
      "activations/layer16_attention_weight_min": -29.34576988220215,
      "activations/layer17_attention_weight_max": 52.48714065551758,
      "activations/layer17_attention_weight_min": -43.701904296875,
      "activations/layer18_attention_weight_max": 47.677207946777344,
      "activations/layer18_attention_weight_min": -39.337013244628906,
      "activations/layer19_attention_weight_max": 22.265371322631836,
      "activations/layer19_attention_weight_min": -23.224897384643555,
      "activations/layer1_attention_weight_max": 16.426427841186523,
      "activations/layer1_attention_weight_min": -15.166969299316406,
      "activations/layer20_attention_weight_max": 22.88304328918457,
      "activations/layer20_attention_weight_min": -24.36104965209961,
      "activations/layer21_attention_weight_max": 39.33867645263672,
      "activations/layer21_attention_weight_min": -25.132108688354492,
      "activations/layer22_attention_weight_max": 30.596040725708008,
      "activations/layer22_attention_weight_min": -25.877717971801758,
      "activations/layer23_attention_weight_max": 38.18160629272461,
      "activations/layer23_attention_weight_min": -24.88442611694336,
      "activations/layer2_attention_weight_max": 30.950620651245117,
      "activations/layer2_attention_weight_min": -29.560827255249023,
      "activations/layer3_attention_weight_max": 91.17440795898438,
      "activations/layer3_attention_weight_min": -101.3863525390625,
      "activations/layer4_attention_weight_max": 95.27039337158203,
      "activations/layer4_attention_weight_min": -100.7097396850586,
      "activations/layer5_attention_weight_max": 67.72894287109375,
      "activations/layer5_attention_weight_min": -78.97380065917969,
      "activations/layer6_attention_weight_max": 53.393428802490234,
      "activations/layer6_attention_weight_min": -52.443702697753906,
      "activations/layer7_attention_weight_max": 67.0396957397461,
      "activations/layer7_attention_weight_min": -67.91197967529297,
      "activations/layer8_attention_weight_max": 44.770442962646484,
      "activations/layer8_attention_weight_min": -47.815460205078125,
      "activations/layer9_attention_weight_max": 43.8325080871582,
      "activations/layer9_attention_weight_min": -47.618194580078125,
      "epoch": 12.49,
      "learning_rate": 7.018257575757576e-05,
      "loss": 2.7634,
      "step": 214900
    },
    {
      "activations/layer0_attention_weight_max": 15.950407028198242,
      "activations/layer0_attention_weight_min": -14.451350212097168,
      "activations/layer10_attention_weight_max": 34.96697235107422,
      "activations/layer10_attention_weight_min": -33.3699951171875,
      "activations/layer11_attention_weight_max": 30.815265655517578,
      "activations/layer11_attention_weight_min": -30.26844596862793,
      "activations/layer12_attention_weight_max": 20.601648330688477,
      "activations/layer12_attention_weight_min": -25.687620162963867,
      "activations/layer13_attention_weight_max": 36.93657302856445,
      "activations/layer13_attention_weight_min": -29.86321449279785,
      "activations/layer14_attention_weight_max": 35.33418655395508,
      "activations/layer14_attention_weight_min": -32.79454803466797,
      "activations/layer15_attention_weight_max": 32.98668670654297,
      "activations/layer15_attention_weight_min": -31.839998245239258,
      "activations/layer16_attention_weight_max": 31.244352340698242,
      "activations/layer16_attention_weight_min": -29.984905242919922,
      "activations/layer17_attention_weight_max": 50.10173034667969,
      "activations/layer17_attention_weight_min": -42.38172149658203,
      "activations/layer18_attention_weight_max": 44.101139068603516,
      "activations/layer18_attention_weight_min": -37.779823303222656,
      "activations/layer19_attention_weight_max": 23.405506134033203,
      "activations/layer19_attention_weight_min": -22.175918579101562,
      "activations/layer1_attention_weight_max": 16.615468978881836,
      "activations/layer1_attention_weight_min": -14.311210632324219,
      "activations/layer20_attention_weight_max": 21.71226692199707,
      "activations/layer20_attention_weight_min": -21.64971923828125,
      "activations/layer21_attention_weight_max": 38.31990051269531,
      "activations/layer21_attention_weight_min": -24.597055435180664,
      "activations/layer22_attention_weight_max": 33.95891189575195,
      "activations/layer22_attention_weight_min": -25.857378005981445,
      "activations/layer23_attention_weight_max": 39.1047248840332,
      "activations/layer23_attention_weight_min": -25.75174331665039,
      "activations/layer2_attention_weight_max": 32.105751037597656,
      "activations/layer2_attention_weight_min": -29.930526733398438,
      "activations/layer3_attention_weight_max": 89.35771942138672,
      "activations/layer3_attention_weight_min": -98.1040267944336,
      "activations/layer4_attention_weight_max": 91.12566375732422,
      "activations/layer4_attention_weight_min": -93.425537109375,
      "activations/layer5_attention_weight_max": 72.07586669921875,
      "activations/layer5_attention_weight_min": -74.94242858886719,
      "activations/layer6_attention_weight_max": 51.125038146972656,
      "activations/layer6_attention_weight_min": -50.74045181274414,
      "activations/layer7_attention_weight_max": 67.71871185302734,
      "activations/layer7_attention_weight_min": -66.60004425048828,
      "activations/layer8_attention_weight_max": 42.3787956237793,
      "activations/layer8_attention_weight_min": -46.83480453491211,
      "activations/layer9_attention_weight_max": 43.98244857788086,
      "activations/layer9_attention_weight_min": -43.93783187866211,
      "epoch": 12.49,
      "learning_rate": 7.016363636363636e-05,
      "loss": 2.7561,
      "step": 214950
    },
    {
      "activations/layer0_attention_weight_max": 15.629097938537598,
      "activations/layer0_attention_weight_min": -14.07390308380127,
      "activations/layer10_attention_weight_max": 37.42815017700195,
      "activations/layer10_attention_weight_min": -35.767269134521484,
      "activations/layer11_attention_weight_max": 32.179351806640625,
      "activations/layer11_attention_weight_min": -32.9533576965332,
      "activations/layer12_attention_weight_max": 16.682538986206055,
      "activations/layer12_attention_weight_min": -27.37590217590332,
      "activations/layer13_attention_weight_max": 38.547279357910156,
      "activations/layer13_attention_weight_min": -29.431488037109375,
      "activations/layer14_attention_weight_max": 32.73033142089844,
      "activations/layer14_attention_weight_min": -31.181594848632812,
      "activations/layer15_attention_weight_max": 31.71560287475586,
      "activations/layer15_attention_weight_min": -31.694194793701172,
      "activations/layer16_attention_weight_max": 30.66213035583496,
      "activations/layer16_attention_weight_min": -30.973962783813477,
      "activations/layer17_attention_weight_max": 48.91086196899414,
      "activations/layer17_attention_weight_min": -44.2409553527832,
      "activations/layer18_attention_weight_max": 45.42937469482422,
      "activations/layer18_attention_weight_min": -40.495323181152344,
      "activations/layer19_attention_weight_max": 21.512929916381836,
      "activations/layer19_attention_weight_min": -20.615751266479492,
      "activations/layer1_attention_weight_max": 17.18642807006836,
      "activations/layer1_attention_weight_min": -15.179801940917969,
      "activations/layer20_attention_weight_max": 21.991352081298828,
      "activations/layer20_attention_weight_min": -20.339872360229492,
      "activations/layer21_attention_weight_max": 35.566558837890625,
      "activations/layer21_attention_weight_min": -22.94757080078125,
      "activations/layer22_attention_weight_max": 30.41806983947754,
      "activations/layer22_attention_weight_min": -24.113712310791016,
      "activations/layer23_attention_weight_max": 37.383445739746094,
      "activations/layer23_attention_weight_min": -23.37749481201172,
      "activations/layer2_attention_weight_max": 32.96721649169922,
      "activations/layer2_attention_weight_min": -30.860502243041992,
      "activations/layer3_attention_weight_max": 93.22765350341797,
      "activations/layer3_attention_weight_min": -93.6005630493164,
      "activations/layer4_attention_weight_max": 96.31623077392578,
      "activations/layer4_attention_weight_min": -92.95613861083984,
      "activations/layer5_attention_weight_max": 69.50337982177734,
      "activations/layer5_attention_weight_min": -81.33197784423828,
      "activations/layer6_attention_weight_max": 53.233848571777344,
      "activations/layer6_attention_weight_min": -53.57021713256836,
      "activations/layer7_attention_weight_max": 68.21188354492188,
      "activations/layer7_attention_weight_min": -65.28463745117188,
      "activations/layer8_attention_weight_max": 46.63093948364258,
      "activations/layer8_attention_weight_min": -49.03017807006836,
      "activations/layer9_attention_weight_max": 46.266693115234375,
      "activations/layer9_attention_weight_min": -47.37519454956055,
      "epoch": 12.49,
      "learning_rate": 7.014469696969696e-05,
      "loss": 2.7588,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_loss": 2.71875,
      "eval_runtime": 8.544,
      "eval_samples_per_second": 502.575,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.544,
      "eval_openwebtext_samples_per_second": 502.575,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 1.9739,
      "eval_wikitext_samples_per_second": 231.013,
      "step": 215000
    },
    {
      "epoch": 12.49,
      "eval_lambada_loss": 2.525390625,
      "eval_lambada_ppl": 12.495775472672934,
      "eval_lambada_runtime": 9.603,
      "eval_lambada_samples_per_second": 507.03,
      "step": 215000
    },
    {
      "activations/layer0_attention_weight_max": 16.490291595458984,
      "activations/layer0_attention_weight_min": -14.111716270446777,
      "activations/layer10_attention_weight_max": 37.27134704589844,
      "activations/layer10_attention_weight_min": -35.961029052734375,
      "activations/layer11_attention_weight_max": 34.680335998535156,
      "activations/layer11_attention_weight_min": -35.10580062866211,
      "activations/layer12_attention_weight_max": 18.31631851196289,
      "activations/layer12_attention_weight_min": -22.684261322021484,
      "activations/layer13_attention_weight_max": 32.29216003417969,
      "activations/layer13_attention_weight_min": -31.372827529907227,
      "activations/layer14_attention_weight_max": 33.71944808959961,
      "activations/layer14_attention_weight_min": -32.60809326171875,
      "activations/layer15_attention_weight_max": 29.793540954589844,
      "activations/layer15_attention_weight_min": -31.456724166870117,
      "activations/layer16_attention_weight_max": 31.123559951782227,
      "activations/layer16_attention_weight_min": -30.8660888671875,
      "activations/layer17_attention_weight_max": 50.75725173950195,
      "activations/layer17_attention_weight_min": -44.27127456665039,
      "activations/layer18_attention_weight_max": 44.13878631591797,
      "activations/layer18_attention_weight_min": -39.964378356933594,
      "activations/layer19_attention_weight_max": 20.8370304107666,
      "activations/layer19_attention_weight_min": -22.325536727905273,
      "activations/layer1_attention_weight_max": 15.40088176727295,
      "activations/layer1_attention_weight_min": -14.769604682922363,
      "activations/layer20_attention_weight_max": 22.495397567749023,
      "activations/layer20_attention_weight_min": -22.45131492614746,
      "activations/layer21_attention_weight_max": 38.95903015136719,
      "activations/layer21_attention_weight_min": -21.59783935546875,
      "activations/layer22_attention_weight_max": 28.985124588012695,
      "activations/layer22_attention_weight_min": -25.402204513549805,
      "activations/layer23_attention_weight_max": 36.508331298828125,
      "activations/layer23_attention_weight_min": -22.71718406677246,
      "activations/layer2_attention_weight_max": 31.690109252929688,
      "activations/layer2_attention_weight_min": -30.877479553222656,
      "activations/layer3_attention_weight_max": 90.17163848876953,
      "activations/layer3_attention_weight_min": -94.2931900024414,
      "activations/layer4_attention_weight_max": 95.34735870361328,
      "activations/layer4_attention_weight_min": -94.4997787475586,
      "activations/layer5_attention_weight_max": 73.28863525390625,
      "activations/layer5_attention_weight_min": -74.03023529052734,
      "activations/layer6_attention_weight_max": 54.23023986816406,
      "activations/layer6_attention_weight_min": -53.74669647216797,
      "activations/layer7_attention_weight_max": 74.29436492919922,
      "activations/layer7_attention_weight_min": -65.67237091064453,
      "activations/layer8_attention_weight_max": 44.65250015258789,
      "activations/layer8_attention_weight_min": -47.2982177734375,
      "activations/layer9_attention_weight_max": 44.59662628173828,
      "activations/layer9_attention_weight_min": -45.3302001953125,
      "epoch": 12.5,
      "learning_rate": 7.012575757575756e-05,
      "loss": 2.7751,
      "step": 215050
    },
    {
      "activations/layer0_attention_weight_max": 15.050938606262207,
      "activations/layer0_attention_weight_min": -14.218688011169434,
      "activations/layer10_attention_weight_max": 34.52336502075195,
      "activations/layer10_attention_weight_min": -36.375938415527344,
      "activations/layer11_attention_weight_max": 31.574270248413086,
      "activations/layer11_attention_weight_min": -35.891761779785156,
      "activations/layer12_attention_weight_max": 17.413524627685547,
      "activations/layer12_attention_weight_min": -25.515106201171875,
      "activations/layer13_attention_weight_max": 35.8271598815918,
      "activations/layer13_attention_weight_min": -26.84938621520996,
      "activations/layer14_attention_weight_max": 33.33189392089844,
      "activations/layer14_attention_weight_min": -30.68108367919922,
      "activations/layer15_attention_weight_max": 31.510793685913086,
      "activations/layer15_attention_weight_min": -30.15297508239746,
      "activations/layer16_attention_weight_max": 30.79907989501953,
      "activations/layer16_attention_weight_min": -31.00104522705078,
      "activations/layer17_attention_weight_max": 51.32301330566406,
      "activations/layer17_attention_weight_min": -47.140567779541016,
      "activations/layer18_attention_weight_max": 45.20075607299805,
      "activations/layer18_attention_weight_min": -38.51144027709961,
      "activations/layer19_attention_weight_max": 22.085220336914062,
      "activations/layer19_attention_weight_min": -24.003908157348633,
      "activations/layer1_attention_weight_max": 16.756519317626953,
      "activations/layer1_attention_weight_min": -15.688299179077148,
      "activations/layer20_attention_weight_max": 22.553720474243164,
      "activations/layer20_attention_weight_min": -20.87495994567871,
      "activations/layer21_attention_weight_max": 35.112831115722656,
      "activations/layer21_attention_weight_min": -23.04933738708496,
      "activations/layer22_attention_weight_max": 30.6102294921875,
      "activations/layer22_attention_weight_min": -26.482316970825195,
      "activations/layer23_attention_weight_max": 36.319881439208984,
      "activations/layer23_attention_weight_min": -24.883560180664062,
      "activations/layer2_attention_weight_max": 32.90501022338867,
      "activations/layer2_attention_weight_min": -29.486753463745117,
      "activations/layer3_attention_weight_max": 91.13670349121094,
      "activations/layer3_attention_weight_min": -91.3507308959961,
      "activations/layer4_attention_weight_max": 93.58633422851562,
      "activations/layer4_attention_weight_min": -90.4441146850586,
      "activations/layer5_attention_weight_max": 72.9713363647461,
      "activations/layer5_attention_weight_min": -74.62985229492188,
      "activations/layer6_attention_weight_max": 52.23783493041992,
      "activations/layer6_attention_weight_min": -51.571346282958984,
      "activations/layer7_attention_weight_max": 70.92571258544922,
      "activations/layer7_attention_weight_min": -68.06617736816406,
      "activations/layer8_attention_weight_max": 44.56727600097656,
      "activations/layer8_attention_weight_min": -49.08415222167969,
      "activations/layer9_attention_weight_max": 42.68022918701172,
      "activations/layer9_attention_weight_min": -45.622955322265625,
      "epoch": 12.5,
      "learning_rate": 7.010681818181818e-05,
      "loss": 2.7772,
      "step": 215100
    },
    {
      "activations/layer0_attention_weight_max": 15.745892524719238,
      "activations/layer0_attention_weight_min": -13.974542617797852,
      "activations/layer10_attention_weight_max": 42.624053955078125,
      "activations/layer10_attention_weight_min": -41.74275207519531,
      "activations/layer11_attention_weight_max": 39.40242004394531,
      "activations/layer11_attention_weight_min": -37.656341552734375,
      "activations/layer12_attention_weight_max": 21.392515182495117,
      "activations/layer12_attention_weight_min": -24.430387496948242,
      "activations/layer13_attention_weight_max": 49.4094352722168,
      "activations/layer13_attention_weight_min": -33.170692443847656,
      "activations/layer14_attention_weight_max": 37.78369903564453,
      "activations/layer14_attention_weight_min": -32.435115814208984,
      "activations/layer15_attention_weight_max": 35.60757064819336,
      "activations/layer15_attention_weight_min": -31.619869232177734,
      "activations/layer16_attention_weight_max": 33.4256477355957,
      "activations/layer16_attention_weight_min": -31.948991775512695,
      "activations/layer17_attention_weight_max": 55.6790657043457,
      "activations/layer17_attention_weight_min": -49.88580322265625,
      "activations/layer18_attention_weight_max": 49.4393196105957,
      "activations/layer18_attention_weight_min": -39.35248947143555,
      "activations/layer19_attention_weight_max": 22.815460205078125,
      "activations/layer19_attention_weight_min": -22.39125633239746,
      "activations/layer1_attention_weight_max": 17.52140998840332,
      "activations/layer1_attention_weight_min": -16.022174835205078,
      "activations/layer20_attention_weight_max": 23.264179229736328,
      "activations/layer20_attention_weight_min": -23.156108856201172,
      "activations/layer21_attention_weight_max": 36.78590774536133,
      "activations/layer21_attention_weight_min": -23.116369247436523,
      "activations/layer22_attention_weight_max": 27.872949600219727,
      "activations/layer22_attention_weight_min": -26.10542869567871,
      "activations/layer23_attention_weight_max": 34.79371643066406,
      "activations/layer23_attention_weight_min": -23.93759536743164,
      "activations/layer2_attention_weight_max": 34.37141418457031,
      "activations/layer2_attention_weight_min": -32.28791809082031,
      "activations/layer3_attention_weight_max": 104.4700698852539,
      "activations/layer3_attention_weight_min": -107.24803924560547,
      "activations/layer4_attention_weight_max": 112.32186126708984,
      "activations/layer4_attention_weight_min": -107.08902740478516,
      "activations/layer5_attention_weight_max": 77.35420989990234,
      "activations/layer5_attention_weight_min": -86.20852661132812,
      "activations/layer6_attention_weight_max": 64.01322937011719,
      "activations/layer6_attention_weight_min": -62.527992248535156,
      "activations/layer7_attention_weight_max": 74.81421661376953,
      "activations/layer7_attention_weight_min": -74.14073181152344,
      "activations/layer8_attention_weight_max": 52.20569610595703,
      "activations/layer8_attention_weight_min": -50.339500427246094,
      "activations/layer9_attention_weight_max": 55.16794967651367,
      "activations/layer9_attention_weight_min": -52.06640625,
      "epoch": 12.5,
      "learning_rate": 7.008787878787878e-05,
      "loss": 2.7625,
      "step": 215150
    },
    {
      "activations/layer0_attention_weight_max": 15.595281600952148,
      "activations/layer0_attention_weight_min": -14.366290092468262,
      "activations/layer10_attention_weight_max": 35.80072784423828,
      "activations/layer10_attention_weight_min": -37.5185432434082,
      "activations/layer11_attention_weight_max": 33.35699462890625,
      "activations/layer11_attention_weight_min": -36.67788314819336,
      "activations/layer12_attention_weight_max": 17.690765380859375,
      "activations/layer12_attention_weight_min": -25.739009857177734,
      "activations/layer13_attention_weight_max": 34.72111129760742,
      "activations/layer13_attention_weight_min": -30.290550231933594,
      "activations/layer14_attention_weight_max": 35.55728530883789,
      "activations/layer14_attention_weight_min": -30.77613639831543,
      "activations/layer15_attention_weight_max": 32.09975051879883,
      "activations/layer15_attention_weight_min": -30.82091522216797,
      "activations/layer16_attention_weight_max": 31.32686424255371,
      "activations/layer16_attention_weight_min": -31.83898162841797,
      "activations/layer17_attention_weight_max": 49.9765739440918,
      "activations/layer17_attention_weight_min": -44.73373794555664,
      "activations/layer18_attention_weight_max": 44.49189758300781,
      "activations/layer18_attention_weight_min": -37.72408676147461,
      "activations/layer19_attention_weight_max": 21.591218948364258,
      "activations/layer19_attention_weight_min": -22.485389709472656,
      "activations/layer1_attention_weight_max": 16.285043716430664,
      "activations/layer1_attention_weight_min": -14.580790519714355,
      "activations/layer20_attention_weight_max": 24.099365234375,
      "activations/layer20_attention_weight_min": -20.485183715820312,
      "activations/layer21_attention_weight_max": 38.8688850402832,
      "activations/layer21_attention_weight_min": -22.308427810668945,
      "activations/layer22_attention_weight_max": 27.15342140197754,
      "activations/layer22_attention_weight_min": -26.909427642822266,
      "activations/layer23_attention_weight_max": 36.759666442871094,
      "activations/layer23_attention_weight_min": -22.49288558959961,
      "activations/layer2_attention_weight_max": 32.52192687988281,
      "activations/layer2_attention_weight_min": -30.79519271850586,
      "activations/layer3_attention_weight_max": 92.25846099853516,
      "activations/layer3_attention_weight_min": -94.55903625488281,
      "activations/layer4_attention_weight_max": 100.13288879394531,
      "activations/layer4_attention_weight_min": -97.2403564453125,
      "activations/layer5_attention_weight_max": 72.219970703125,
      "activations/layer5_attention_weight_min": -76.15399169921875,
      "activations/layer6_attention_weight_max": 52.88395309448242,
      "activations/layer6_attention_weight_min": -51.401954650878906,
      "activations/layer7_attention_weight_max": 72.79171752929688,
      "activations/layer7_attention_weight_min": -66.64183807373047,
      "activations/layer8_attention_weight_max": 44.479190826416016,
      "activations/layer8_attention_weight_min": -50.18300247192383,
      "activations/layer9_attention_weight_max": 47.500789642333984,
      "activations/layer9_attention_weight_min": -48.62617492675781,
      "epoch": 12.5,
      "learning_rate": 7.006893939393938e-05,
      "loss": 2.766,
      "step": 215200
    },
    {
      "activations/layer0_attention_weight_max": 15.368452072143555,
      "activations/layer0_attention_weight_min": -13.8349609375,
      "activations/layer10_attention_weight_max": 37.92873001098633,
      "activations/layer10_attention_weight_min": -37.885982513427734,
      "activations/layer11_attention_weight_max": 34.214752197265625,
      "activations/layer11_attention_weight_min": -32.50670623779297,
      "activations/layer12_attention_weight_max": 19.326324462890625,
      "activations/layer12_attention_weight_min": -22.28880500793457,
      "activations/layer13_attention_weight_max": 51.81800842285156,
      "activations/layer13_attention_weight_min": -33.75911331176758,
      "activations/layer14_attention_weight_max": 46.11054611206055,
      "activations/layer14_attention_weight_min": -34.46393585205078,
      "activations/layer15_attention_weight_max": 36.18902587890625,
      "activations/layer15_attention_weight_min": -33.07748031616211,
      "activations/layer16_attention_weight_max": 36.04658126831055,
      "activations/layer16_attention_weight_min": -31.97362518310547,
      "activations/layer17_attention_weight_max": 56.95639419555664,
      "activations/layer17_attention_weight_min": -48.35413360595703,
      "activations/layer18_attention_weight_max": 55.50182342529297,
      "activations/layer18_attention_weight_min": -42.7898063659668,
      "activations/layer19_attention_weight_max": 25.695758819580078,
      "activations/layer19_attention_weight_min": -22.802213668823242,
      "activations/layer1_attention_weight_max": 16.97659683227539,
      "activations/layer1_attention_weight_min": -15.639643669128418,
      "activations/layer20_attention_weight_max": 29.999725341796875,
      "activations/layer20_attention_weight_min": -23.172258377075195,
      "activations/layer21_attention_weight_max": 44.02995300292969,
      "activations/layer21_attention_weight_min": -24.167638778686523,
      "activations/layer22_attention_weight_max": 35.66431427001953,
      "activations/layer22_attention_weight_min": -26.5357723236084,
      "activations/layer23_attention_weight_max": 46.30228805541992,
      "activations/layer23_attention_weight_min": -23.976776123046875,
      "activations/layer2_attention_weight_max": 33.0842170715332,
      "activations/layer2_attention_weight_min": -34.70795822143555,
      "activations/layer3_attention_weight_max": 92.99412536621094,
      "activations/layer3_attention_weight_min": -91.21924591064453,
      "activations/layer4_attention_weight_max": 97.1184310913086,
      "activations/layer4_attention_weight_min": -93.4081802368164,
      "activations/layer5_attention_weight_max": 71.7549057006836,
      "activations/layer5_attention_weight_min": -77.12594604492188,
      "activations/layer6_attention_weight_max": 54.91074752807617,
      "activations/layer6_attention_weight_min": -53.22779083251953,
      "activations/layer7_attention_weight_max": 72.82703399658203,
      "activations/layer7_attention_weight_min": -68.08352661132812,
      "activations/layer8_attention_weight_max": 49.74713134765625,
      "activations/layer8_attention_weight_min": -48.67811965942383,
      "activations/layer9_attention_weight_max": 51.52704620361328,
      "activations/layer9_attention_weight_min": -48.57516098022461,
      "epoch": 12.51,
      "learning_rate": 7.005e-05,
      "loss": 2.7653,
      "step": 215250
    },
    {
      "activations/layer0_attention_weight_max": 15.975800514221191,
      "activations/layer0_attention_weight_min": -13.768416404724121,
      "activations/layer10_attention_weight_max": 37.168617248535156,
      "activations/layer10_attention_weight_min": -37.00197982788086,
      "activations/layer11_attention_weight_max": 35.80061721801758,
      "activations/layer11_attention_weight_min": -35.81840515136719,
      "activations/layer12_attention_weight_max": 18.760480880737305,
      "activations/layer12_attention_weight_min": -25.135150909423828,
      "activations/layer13_attention_weight_max": 42.00157165527344,
      "activations/layer13_attention_weight_min": -29.79924201965332,
      "activations/layer14_attention_weight_max": 40.33403015136719,
      "activations/layer14_attention_weight_min": -32.68401336669922,
      "activations/layer15_attention_weight_max": 35.46778106689453,
      "activations/layer15_attention_weight_min": -32.068580627441406,
      "activations/layer16_attention_weight_max": 33.807979583740234,
      "activations/layer16_attention_weight_min": -31.03133201599121,
      "activations/layer17_attention_weight_max": 56.41383743286133,
      "activations/layer17_attention_weight_min": -45.232933044433594,
      "activations/layer18_attention_weight_max": 50.74104690551758,
      "activations/layer18_attention_weight_min": -39.734519958496094,
      "activations/layer19_attention_weight_max": 25.480350494384766,
      "activations/layer19_attention_weight_min": -20.196853637695312,
      "activations/layer1_attention_weight_max": 16.700210571289062,
      "activations/layer1_attention_weight_min": -13.04597282409668,
      "activations/layer20_attention_weight_max": 25.804719924926758,
      "activations/layer20_attention_weight_min": -21.280746459960938,
      "activations/layer21_attention_weight_max": 37.47354507446289,
      "activations/layer21_attention_weight_min": -19.98981475830078,
      "activations/layer22_attention_weight_max": 32.69096755981445,
      "activations/layer22_attention_weight_min": -24.52726173400879,
      "activations/layer23_attention_weight_max": 43.02021789550781,
      "activations/layer23_attention_weight_min": -21.589155197143555,
      "activations/layer2_attention_weight_max": 33.80146789550781,
      "activations/layer2_attention_weight_min": -31.135547637939453,
      "activations/layer3_attention_weight_max": 92.35543823242188,
      "activations/layer3_attention_weight_min": -99.0550537109375,
      "activations/layer4_attention_weight_max": 92.25741577148438,
      "activations/layer4_attention_weight_min": -89.6213607788086,
      "activations/layer5_attention_weight_max": 70.17268371582031,
      "activations/layer5_attention_weight_min": -78.2420425415039,
      "activations/layer6_attention_weight_max": 52.38108444213867,
      "activations/layer6_attention_weight_min": -52.611759185791016,
      "activations/layer7_attention_weight_max": 71.28782653808594,
      "activations/layer7_attention_weight_min": -63.921722412109375,
      "activations/layer8_attention_weight_max": 46.652767181396484,
      "activations/layer8_attention_weight_min": -49.4026985168457,
      "activations/layer9_attention_weight_max": 46.78044128417969,
      "activations/layer9_attention_weight_min": -48.448787689208984,
      "epoch": 12.51,
      "learning_rate": 7.003143939393939e-05,
      "loss": 2.7679,
      "step": 215300
    },
    {
      "activations/layer0_attention_weight_max": 16.05385971069336,
      "activations/layer0_attention_weight_min": -13.478970527648926,
      "activations/layer10_attention_weight_max": 38.38047409057617,
      "activations/layer10_attention_weight_min": -36.5719108581543,
      "activations/layer11_attention_weight_max": 34.040992736816406,
      "activations/layer11_attention_weight_min": -35.121116638183594,
      "activations/layer12_attention_weight_max": 18.212583541870117,
      "activations/layer12_attention_weight_min": -23.53583526611328,
      "activations/layer13_attention_weight_max": 35.57189178466797,
      "activations/layer13_attention_weight_min": -28.499788284301758,
      "activations/layer14_attention_weight_max": 37.49557113647461,
      "activations/layer14_attention_weight_min": -31.52116584777832,
      "activations/layer15_attention_weight_max": 33.66688537597656,
      "activations/layer15_attention_weight_min": -29.878150939941406,
      "activations/layer16_attention_weight_max": 31.925073623657227,
      "activations/layer16_attention_weight_min": -30.945877075195312,
      "activations/layer17_attention_weight_max": 51.53370666503906,
      "activations/layer17_attention_weight_min": -45.851348876953125,
      "activations/layer18_attention_weight_max": 46.23046875,
      "activations/layer18_attention_weight_min": -38.47684860229492,
      "activations/layer19_attention_weight_max": 20.41390037536621,
      "activations/layer19_attention_weight_min": -22.928916931152344,
      "activations/layer1_attention_weight_max": 16.68552589416504,
      "activations/layer1_attention_weight_min": -13.952126502990723,
      "activations/layer20_attention_weight_max": 20.48771095275879,
      "activations/layer20_attention_weight_min": -22.826515197753906,
      "activations/layer21_attention_weight_max": 36.28437805175781,
      "activations/layer21_attention_weight_min": -23.718767166137695,
      "activations/layer22_attention_weight_max": 28.8950138092041,
      "activations/layer22_attention_weight_min": -29.787731170654297,
      "activations/layer23_attention_weight_max": 34.461883544921875,
      "activations/layer23_attention_weight_min": -23.288795471191406,
      "activations/layer2_attention_weight_max": 33.18108367919922,
      "activations/layer2_attention_weight_min": -30.278907775878906,
      "activations/layer3_attention_weight_max": 97.86650848388672,
      "activations/layer3_attention_weight_min": -100.40587615966797,
      "activations/layer4_attention_weight_max": 100.20476531982422,
      "activations/layer4_attention_weight_min": -101.14498138427734,
      "activations/layer5_attention_weight_max": 71.65399169921875,
      "activations/layer5_attention_weight_min": -78.80363464355469,
      "activations/layer6_attention_weight_max": 57.53886413574219,
      "activations/layer6_attention_weight_min": -57.99018478393555,
      "activations/layer7_attention_weight_max": 71.00086975097656,
      "activations/layer7_attention_weight_min": -72.1289291381836,
      "activations/layer8_attention_weight_max": 48.85894012451172,
      "activations/layer8_attention_weight_min": -49.07866668701172,
      "activations/layer9_attention_weight_max": 49.65293884277344,
      "activations/layer9_attention_weight_min": -50.52042770385742,
      "epoch": 12.51,
      "learning_rate": 7.001249999999999e-05,
      "loss": 2.7659,
      "step": 215350
    },
    {
      "activations/layer0_attention_weight_max": 15.492326736450195,
      "activations/layer0_attention_weight_min": -14.138642311096191,
      "activations/layer10_attention_weight_max": 35.94656753540039,
      "activations/layer10_attention_weight_min": -34.9742431640625,
      "activations/layer11_attention_weight_max": 33.60712814331055,
      "activations/layer11_attention_weight_min": -32.86341094970703,
      "activations/layer12_attention_weight_max": 18.0847225189209,
      "activations/layer12_attention_weight_min": -26.043012619018555,
      "activations/layer13_attention_weight_max": 40.665321350097656,
      "activations/layer13_attention_weight_min": -26.76326560974121,
      "activations/layer14_attention_weight_max": 32.8723258972168,
      "activations/layer14_attention_weight_min": -28.944412231445312,
      "activations/layer15_attention_weight_max": 30.076536178588867,
      "activations/layer15_attention_weight_min": -28.660648345947266,
      "activations/layer16_attention_weight_max": 29.45993423461914,
      "activations/layer16_attention_weight_min": -28.34173011779785,
      "activations/layer17_attention_weight_max": 48.76416778564453,
      "activations/layer17_attention_weight_min": -43.0888557434082,
      "activations/layer18_attention_weight_max": 44.112464904785156,
      "activations/layer18_attention_weight_min": -37.503562927246094,
      "activations/layer19_attention_weight_max": 20.235546112060547,
      "activations/layer19_attention_weight_min": -21.275009155273438,
      "activations/layer1_attention_weight_max": 16.032602310180664,
      "activations/layer1_attention_weight_min": -13.689088821411133,
      "activations/layer20_attention_weight_max": 19.984134674072266,
      "activations/layer20_attention_weight_min": -20.476821899414062,
      "activations/layer21_attention_weight_max": 31.1980037689209,
      "activations/layer21_attention_weight_min": -21.932598114013672,
      "activations/layer22_attention_weight_max": 26.75611114501953,
      "activations/layer22_attention_weight_min": -23.865764617919922,
      "activations/layer23_attention_weight_max": 33.862545013427734,
      "activations/layer23_attention_weight_min": -23.75408935546875,
      "activations/layer2_attention_weight_max": 29.625150680541992,
      "activations/layer2_attention_weight_min": -28.909564971923828,
      "activations/layer3_attention_weight_max": 86.50395202636719,
      "activations/layer3_attention_weight_min": -87.8090591430664,
      "activations/layer4_attention_weight_max": 91.18409729003906,
      "activations/layer4_attention_weight_min": -88.84412384033203,
      "activations/layer5_attention_weight_max": 70.81956481933594,
      "activations/layer5_attention_weight_min": -77.48999786376953,
      "activations/layer6_attention_weight_max": 52.93478775024414,
      "activations/layer6_attention_weight_min": -53.05338668823242,
      "activations/layer7_attention_weight_max": 70.30396270751953,
      "activations/layer7_attention_weight_min": -67.38770294189453,
      "activations/layer8_attention_weight_max": 44.275299072265625,
      "activations/layer8_attention_weight_min": -44.99980163574219,
      "activations/layer9_attention_weight_max": 45.703914642333984,
      "activations/layer9_attention_weight_min": -43.790771484375,
      "epoch": 12.52,
      "learning_rate": 6.99935606060606e-05,
      "loss": 2.7685,
      "step": 215400
    },
    {
      "activations/layer0_attention_weight_max": 19.025299072265625,
      "activations/layer0_attention_weight_min": -14.015142440795898,
      "activations/layer10_attention_weight_max": 37.952842712402344,
      "activations/layer10_attention_weight_min": -38.27631759643555,
      "activations/layer11_attention_weight_max": 33.34648895263672,
      "activations/layer11_attention_weight_min": -34.26407241821289,
      "activations/layer12_attention_weight_max": 18.682361602783203,
      "activations/layer12_attention_weight_min": -32.094703674316406,
      "activations/layer13_attention_weight_max": 37.76399612426758,
      "activations/layer13_attention_weight_min": -29.814178466796875,
      "activations/layer14_attention_weight_max": 35.9178352355957,
      "activations/layer14_attention_weight_min": -29.467802047729492,
      "activations/layer15_attention_weight_max": 33.50149917602539,
      "activations/layer15_attention_weight_min": -31.3950138092041,
      "activations/layer16_attention_weight_max": 29.47124481201172,
      "activations/layer16_attention_weight_min": -29.766414642333984,
      "activations/layer17_attention_weight_max": 54.68975067138672,
      "activations/layer17_attention_weight_min": -42.47422409057617,
      "activations/layer18_attention_weight_max": 45.960609436035156,
      "activations/layer18_attention_weight_min": -37.22068405151367,
      "activations/layer19_attention_weight_max": 21.200855255126953,
      "activations/layer19_attention_weight_min": -23.693145751953125,
      "activations/layer1_attention_weight_max": 16.57087516784668,
      "activations/layer1_attention_weight_min": -13.345890045166016,
      "activations/layer20_attention_weight_max": 22.369266510009766,
      "activations/layer20_attention_weight_min": -20.560514450073242,
      "activations/layer21_attention_weight_max": 32.385128021240234,
      "activations/layer21_attention_weight_min": -21.86084747314453,
      "activations/layer22_attention_weight_max": 31.625810623168945,
      "activations/layer22_attention_weight_min": -24.8558349609375,
      "activations/layer23_attention_weight_max": 39.25868225097656,
      "activations/layer23_attention_weight_min": -26.30678939819336,
      "activations/layer2_attention_weight_max": 32.15296173095703,
      "activations/layer2_attention_weight_min": -30.464866638183594,
      "activations/layer3_attention_weight_max": 89.1453628540039,
      "activations/layer3_attention_weight_min": -97.05955505371094,
      "activations/layer4_attention_weight_max": 98.16889953613281,
      "activations/layer4_attention_weight_min": -92.90959930419922,
      "activations/layer5_attention_weight_max": 77.74526977539062,
      "activations/layer5_attention_weight_min": -74.94651794433594,
      "activations/layer6_attention_weight_max": 52.192195892333984,
      "activations/layer6_attention_weight_min": -53.12013626098633,
      "activations/layer7_attention_weight_max": 74.48509979248047,
      "activations/layer7_attention_weight_min": -70.78352355957031,
      "activations/layer8_attention_weight_max": 47.32487106323242,
      "activations/layer8_attention_weight_min": -51.6431999206543,
      "activations/layer9_attention_weight_max": 57.512062072753906,
      "activations/layer9_attention_weight_min": -49.6998176574707,
      "epoch": 12.52,
      "learning_rate": 6.99746212121212e-05,
      "loss": 2.7763,
      "step": 215450
    },
    {
      "activations/layer0_attention_weight_max": 16.369478225708008,
      "activations/layer0_attention_weight_min": -13.910432815551758,
      "activations/layer10_attention_weight_max": 35.530757904052734,
      "activations/layer10_attention_weight_min": -33.887428283691406,
      "activations/layer11_attention_weight_max": 32.99176788330078,
      "activations/layer11_attention_weight_min": -31.929227828979492,
      "activations/layer12_attention_weight_max": 23.45298194885254,
      "activations/layer12_attention_weight_min": -25.751102447509766,
      "activations/layer13_attention_weight_max": 34.84296417236328,
      "activations/layer13_attention_weight_min": -26.62201690673828,
      "activations/layer14_attention_weight_max": 35.54195022583008,
      "activations/layer14_attention_weight_min": -29.41257095336914,
      "activations/layer15_attention_weight_max": 31.96623420715332,
      "activations/layer15_attention_weight_min": -28.9574031829834,
      "activations/layer16_attention_weight_max": 30.735918045043945,
      "activations/layer16_attention_weight_min": -28.677213668823242,
      "activations/layer17_attention_weight_max": 50.014190673828125,
      "activations/layer17_attention_weight_min": -44.02968978881836,
      "activations/layer18_attention_weight_max": 45.712345123291016,
      "activations/layer18_attention_weight_min": -35.76900863647461,
      "activations/layer19_attention_weight_max": 20.698213577270508,
      "activations/layer19_attention_weight_min": -19.617795944213867,
      "activations/layer1_attention_weight_max": 16.78678321838379,
      "activations/layer1_attention_weight_min": -14.18594741821289,
      "activations/layer20_attention_weight_max": 23.12968635559082,
      "activations/layer20_attention_weight_min": -19.844249725341797,
      "activations/layer21_attention_weight_max": 37.89747619628906,
      "activations/layer21_attention_weight_min": -22.079870223999023,
      "activations/layer22_attention_weight_max": 34.40813064575195,
      "activations/layer22_attention_weight_min": -24.635414123535156,
      "activations/layer23_attention_weight_max": 43.06121826171875,
      "activations/layer23_attention_weight_min": -21.855518341064453,
      "activations/layer2_attention_weight_max": 31.78287124633789,
      "activations/layer2_attention_weight_min": -30.343368530273438,
      "activations/layer3_attention_weight_max": 91.96086883544922,
      "activations/layer3_attention_weight_min": -96.59098052978516,
      "activations/layer4_attention_weight_max": 95.02528381347656,
      "activations/layer4_attention_weight_min": -93.09793853759766,
      "activations/layer5_attention_weight_max": 71.83132934570312,
      "activations/layer5_attention_weight_min": -74.90111541748047,
      "activations/layer6_attention_weight_max": 51.0809440612793,
      "activations/layer6_attention_weight_min": -52.18250274658203,
      "activations/layer7_attention_weight_max": 70.14189910888672,
      "activations/layer7_attention_weight_min": -68.18360900878906,
      "activations/layer8_attention_weight_max": 43.58845901489258,
      "activations/layer8_attention_weight_min": -45.01676559448242,
      "activations/layer9_attention_weight_max": 53.420631408691406,
      "activations/layer9_attention_weight_min": -46.24042892456055,
      "epoch": 12.52,
      "learning_rate": 6.995568181818182e-05,
      "loss": 2.7927,
      "step": 215500
    },
    {
      "activations/layer0_attention_weight_max": 15.718094825744629,
      "activations/layer0_attention_weight_min": -14.161415100097656,
      "activations/layer10_attention_weight_max": 38.95579528808594,
      "activations/layer10_attention_weight_min": -41.05794143676758,
      "activations/layer11_attention_weight_max": 33.94103240966797,
      "activations/layer11_attention_weight_min": -35.189605712890625,
      "activations/layer12_attention_weight_max": 25.0447998046875,
      "activations/layer12_attention_weight_min": -26.834060668945312,
      "activations/layer13_attention_weight_max": 60.70647430419922,
      "activations/layer13_attention_weight_min": -30.518564224243164,
      "activations/layer14_attention_weight_max": 56.49711227416992,
      "activations/layer14_attention_weight_min": -31.328752517700195,
      "activations/layer15_attention_weight_max": 54.02720642089844,
      "activations/layer15_attention_weight_min": -29.9778995513916,
      "activations/layer16_attention_weight_max": 33.56938171386719,
      "activations/layer16_attention_weight_min": -33.14572525024414,
      "activations/layer17_attention_weight_max": 56.89436721801758,
      "activations/layer17_attention_weight_min": -51.47706604003906,
      "activations/layer18_attention_weight_max": 51.63050079345703,
      "activations/layer18_attention_weight_min": -46.60572052001953,
      "activations/layer19_attention_weight_max": 26.60638427734375,
      "activations/layer19_attention_weight_min": -26.304630279541016,
      "activations/layer1_attention_weight_max": 16.36215591430664,
      "activations/layer1_attention_weight_min": -14.293630599975586,
      "activations/layer20_attention_weight_max": 27.653099060058594,
      "activations/layer20_attention_weight_min": -21.512414932250977,
      "activations/layer21_attention_weight_max": 41.854888916015625,
      "activations/layer21_attention_weight_min": -21.84961700439453,
      "activations/layer22_attention_weight_max": 35.04214859008789,
      "activations/layer22_attention_weight_min": -25.527740478515625,
      "activations/layer23_attention_weight_max": 32.29496765136719,
      "activations/layer23_attention_weight_min": -22.568687438964844,
      "activations/layer2_attention_weight_max": 33.71946716308594,
      "activations/layer2_attention_weight_min": -32.31100082397461,
      "activations/layer3_attention_weight_max": 94.78128051757812,
      "activations/layer3_attention_weight_min": -95.83644104003906,
      "activations/layer4_attention_weight_max": 103.53273010253906,
      "activations/layer4_attention_weight_min": -95.6420669555664,
      "activations/layer5_attention_weight_max": 70.31974029541016,
      "activations/layer5_attention_weight_min": -76.18157958984375,
      "activations/layer6_attention_weight_max": 55.89630126953125,
      "activations/layer6_attention_weight_min": -52.86533737182617,
      "activations/layer7_attention_weight_max": 71.04092407226562,
      "activations/layer7_attention_weight_min": -69.22379302978516,
      "activations/layer8_attention_weight_max": 51.3543701171875,
      "activations/layer8_attention_weight_min": -48.34791564941406,
      "activations/layer9_attention_weight_max": 51.2541389465332,
      "activations/layer9_attention_weight_min": -47.12667465209961,
      "epoch": 12.52,
      "learning_rate": 6.993674242424242e-05,
      "loss": 2.7764,
      "step": 215550
    },
    {
      "activations/layer0_attention_weight_max": 16.05274772644043,
      "activations/layer0_attention_weight_min": -14.112247467041016,
      "activations/layer10_attention_weight_max": 35.29927444458008,
      "activations/layer10_attention_weight_min": -34.63438034057617,
      "activations/layer11_attention_weight_max": 30.839202880859375,
      "activations/layer11_attention_weight_min": -32.63993835449219,
      "activations/layer12_attention_weight_max": 19.055946350097656,
      "activations/layer12_attention_weight_min": -25.695009231567383,
      "activations/layer13_attention_weight_max": 42.43149948120117,
      "activations/layer13_attention_weight_min": -29.452529907226562,
      "activations/layer14_attention_weight_max": 44.19381332397461,
      "activations/layer14_attention_weight_min": -30.547382354736328,
      "activations/layer15_attention_weight_max": 43.39825439453125,
      "activations/layer15_attention_weight_min": -30.798166275024414,
      "activations/layer16_attention_weight_max": 32.92445373535156,
      "activations/layer16_attention_weight_min": -29.753984451293945,
      "activations/layer17_attention_weight_max": 58.98200988769531,
      "activations/layer17_attention_weight_min": -46.8261833190918,
      "activations/layer18_attention_weight_max": 47.85289764404297,
      "activations/layer18_attention_weight_min": -40.23570251464844,
      "activations/layer19_attention_weight_max": 25.35445785522461,
      "activations/layer19_attention_weight_min": -21.982776641845703,
      "activations/layer1_attention_weight_max": 16.757164001464844,
      "activations/layer1_attention_weight_min": -15.342702865600586,
      "activations/layer20_attention_weight_max": 22.785228729248047,
      "activations/layer20_attention_weight_min": -21.149295806884766,
      "activations/layer21_attention_weight_max": 41.67380142211914,
      "activations/layer21_attention_weight_min": -22.289091110229492,
      "activations/layer22_attention_weight_max": 29.15536880493164,
      "activations/layer22_attention_weight_min": -25.07600975036621,
      "activations/layer23_attention_weight_max": 36.20410919189453,
      "activations/layer23_attention_weight_min": -24.257896423339844,
      "activations/layer2_attention_weight_max": 31.121854782104492,
      "activations/layer2_attention_weight_min": -29.850383758544922,
      "activations/layer3_attention_weight_max": 90.94457244873047,
      "activations/layer3_attention_weight_min": -97.60538482666016,
      "activations/layer4_attention_weight_max": 96.90922546386719,
      "activations/layer4_attention_weight_min": -94.73884582519531,
      "activations/layer5_attention_weight_max": 70.28260803222656,
      "activations/layer5_attention_weight_min": -82.4566879272461,
      "activations/layer6_attention_weight_max": 51.282752990722656,
      "activations/layer6_attention_weight_min": -51.41787338256836,
      "activations/layer7_attention_weight_max": 71.39212799072266,
      "activations/layer7_attention_weight_min": -68.39793395996094,
      "activations/layer8_attention_weight_max": 43.60921096801758,
      "activations/layer8_attention_weight_min": -47.27320098876953,
      "activations/layer9_attention_weight_max": 46.07609939575195,
      "activations/layer9_attention_weight_min": -48.20792770385742,
      "epoch": 12.53,
      "learning_rate": 6.991780303030302e-05,
      "loss": 2.7582,
      "step": 215600
    },
    {
      "activations/layer0_attention_weight_max": 15.872239112854004,
      "activations/layer0_attention_weight_min": -14.13649845123291,
      "activations/layer10_attention_weight_max": 36.22602081298828,
      "activations/layer10_attention_weight_min": -36.84737014770508,
      "activations/layer11_attention_weight_max": 31.887340545654297,
      "activations/layer11_attention_weight_min": -31.96938705444336,
      "activations/layer12_attention_weight_max": 20.08155632019043,
      "activations/layer12_attention_weight_min": -27.914148330688477,
      "activations/layer13_attention_weight_max": 39.353729248046875,
      "activations/layer13_attention_weight_min": -31.750972747802734,
      "activations/layer14_attention_weight_max": 40.268638610839844,
      "activations/layer14_attention_weight_min": -30.455097198486328,
      "activations/layer15_attention_weight_max": 32.754844665527344,
      "activations/layer15_attention_weight_min": -30.75054359436035,
      "activations/layer16_attention_weight_max": 31.934635162353516,
      "activations/layer16_attention_weight_min": -31.06737518310547,
      "activations/layer17_attention_weight_max": 52.89543914794922,
      "activations/layer17_attention_weight_min": -45.12670135498047,
      "activations/layer18_attention_weight_max": 45.15237045288086,
      "activations/layer18_attention_weight_min": -39.47697067260742,
      "activations/layer19_attention_weight_max": 22.733919143676758,
      "activations/layer19_attention_weight_min": -22.373647689819336,
      "activations/layer1_attention_weight_max": 17.23090934753418,
      "activations/layer1_attention_weight_min": -13.649295806884766,
      "activations/layer20_attention_weight_max": 20.862781524658203,
      "activations/layer20_attention_weight_min": -21.341869354248047,
      "activations/layer21_attention_weight_max": 38.18689727783203,
      "activations/layer21_attention_weight_min": -22.88051414489746,
      "activations/layer22_attention_weight_max": 29.951231002807617,
      "activations/layer22_attention_weight_min": -25.28026580810547,
      "activations/layer23_attention_weight_max": 38.96492004394531,
      "activations/layer23_attention_weight_min": -24.056135177612305,
      "activations/layer2_attention_weight_max": 30.11626434326172,
      "activations/layer2_attention_weight_min": -29.14914321899414,
      "activations/layer3_attention_weight_max": 82.63321685791016,
      "activations/layer3_attention_weight_min": -87.63739776611328,
      "activations/layer4_attention_weight_max": 89.5422134399414,
      "activations/layer4_attention_weight_min": -90.63768768310547,
      "activations/layer5_attention_weight_max": 69.29325866699219,
      "activations/layer5_attention_weight_min": -76.98986053466797,
      "activations/layer6_attention_weight_max": 49.77806091308594,
      "activations/layer6_attention_weight_min": -49.86854553222656,
      "activations/layer7_attention_weight_max": 68.31197357177734,
      "activations/layer7_attention_weight_min": -66.15935516357422,
      "activations/layer8_attention_weight_max": 43.72478103637695,
      "activations/layer8_attention_weight_min": -46.66374206542969,
      "activations/layer9_attention_weight_max": 45.72086715698242,
      "activations/layer9_attention_weight_min": -45.19192886352539,
      "epoch": 12.53,
      "learning_rate": 6.989886363636364e-05,
      "loss": 2.7633,
      "step": 215650
    },
    {
      "activations/layer0_attention_weight_max": 16.060617446899414,
      "activations/layer0_attention_weight_min": -14.070513725280762,
      "activations/layer10_attention_weight_max": 37.95710754394531,
      "activations/layer10_attention_weight_min": -37.47962951660156,
      "activations/layer11_attention_weight_max": 36.9989013671875,
      "activations/layer11_attention_weight_min": -35.76547622680664,
      "activations/layer12_attention_weight_max": 17.212696075439453,
      "activations/layer12_attention_weight_min": -30.039220809936523,
      "activations/layer13_attention_weight_max": 41.20808410644531,
      "activations/layer13_attention_weight_min": -27.25850486755371,
      "activations/layer14_attention_weight_max": 39.06198501586914,
      "activations/layer14_attention_weight_min": -30.28036880493164,
      "activations/layer15_attention_weight_max": 32.86513137817383,
      "activations/layer15_attention_weight_min": -29.114564895629883,
      "activations/layer16_attention_weight_max": 32.6722412109375,
      "activations/layer16_attention_weight_min": -29.719247817993164,
      "activations/layer17_attention_weight_max": 57.31438064575195,
      "activations/layer17_attention_weight_min": -44.857330322265625,
      "activations/layer18_attention_weight_max": 49.53178787231445,
      "activations/layer18_attention_weight_min": -39.371986389160156,
      "activations/layer19_attention_weight_max": 21.27610969543457,
      "activations/layer19_attention_weight_min": -21.537992477416992,
      "activations/layer1_attention_weight_max": 16.950040817260742,
      "activations/layer1_attention_weight_min": -16.364612579345703,
      "activations/layer20_attention_weight_max": 21.57921028137207,
      "activations/layer20_attention_weight_min": -20.688175201416016,
      "activations/layer21_attention_weight_max": 34.29871368408203,
      "activations/layer21_attention_weight_min": -25.5250186920166,
      "activations/layer22_attention_weight_max": 26.76401710510254,
      "activations/layer22_attention_weight_min": -26.44292640686035,
      "activations/layer23_attention_weight_max": 37.02867889404297,
      "activations/layer23_attention_weight_min": -25.77014923095703,
      "activations/layer2_attention_weight_max": 32.610015869140625,
      "activations/layer2_attention_weight_min": -33.25855255126953,
      "activations/layer3_attention_weight_max": 96.11616516113281,
      "activations/layer3_attention_weight_min": -97.48637390136719,
      "activations/layer4_attention_weight_max": 95.49635314941406,
      "activations/layer4_attention_weight_min": -96.21981048583984,
      "activations/layer5_attention_weight_max": 74.4327392578125,
      "activations/layer5_attention_weight_min": -77.38897705078125,
      "activations/layer6_attention_weight_max": 53.91918182373047,
      "activations/layer6_attention_weight_min": -56.50404357910156,
      "activations/layer7_attention_weight_max": 78.86732482910156,
      "activations/layer7_attention_weight_min": -76.72303009033203,
      "activations/layer8_attention_weight_max": 51.02044677734375,
      "activations/layer8_attention_weight_min": -51.63975143432617,
      "activations/layer9_attention_weight_max": 60.902862548828125,
      "activations/layer9_attention_weight_min": -52.3669319152832,
      "epoch": 12.53,
      "learning_rate": 6.987992424242424e-05,
      "loss": 2.7819,
      "step": 215700
    },
    {
      "activations/layer0_attention_weight_max": 16.447315216064453,
      "activations/layer0_attention_weight_min": -14.231498718261719,
      "activations/layer10_attention_weight_max": 35.612632751464844,
      "activations/layer10_attention_weight_min": -34.32987594604492,
      "activations/layer11_attention_weight_max": 30.77449607849121,
      "activations/layer11_attention_weight_min": -32.28009796142578,
      "activations/layer12_attention_weight_max": 16.70586585998535,
      "activations/layer12_attention_weight_min": -25.45149040222168,
      "activations/layer13_attention_weight_max": 32.155967712402344,
      "activations/layer13_attention_weight_min": -29.35431671142578,
      "activations/layer14_attention_weight_max": 34.994544982910156,
      "activations/layer14_attention_weight_min": -32.560245513916016,
      "activations/layer15_attention_weight_max": 32.01411819458008,
      "activations/layer15_attention_weight_min": -29.767528533935547,
      "activations/layer16_attention_weight_max": 33.95037841796875,
      "activations/layer16_attention_weight_min": -29.865863800048828,
      "activations/layer17_attention_weight_max": 48.60498046875,
      "activations/layer17_attention_weight_min": -43.81612014770508,
      "activations/layer18_attention_weight_max": 49.554595947265625,
      "activations/layer18_attention_weight_min": -38.24423599243164,
      "activations/layer19_attention_weight_max": 23.982297897338867,
      "activations/layer19_attention_weight_min": -22.80267333984375,
      "activations/layer1_attention_weight_max": 17.28685188293457,
      "activations/layer1_attention_weight_min": -14.570440292358398,
      "activations/layer20_attention_weight_max": 21.03915786743164,
      "activations/layer20_attention_weight_min": -22.044918060302734,
      "activations/layer21_attention_weight_max": 36.10795974731445,
      "activations/layer21_attention_weight_min": -21.31695556640625,
      "activations/layer22_attention_weight_max": 27.897275924682617,
      "activations/layer22_attention_weight_min": -25.032047271728516,
      "activations/layer23_attention_weight_max": 35.37239456176758,
      "activations/layer23_attention_weight_min": -24.134641647338867,
      "activations/layer2_attention_weight_max": 32.731258392333984,
      "activations/layer2_attention_weight_min": -30.735145568847656,
      "activations/layer3_attention_weight_max": 90.71646118164062,
      "activations/layer3_attention_weight_min": -97.56255340576172,
      "activations/layer4_attention_weight_max": 94.7253646850586,
      "activations/layer4_attention_weight_min": -97.02281188964844,
      "activations/layer5_attention_weight_max": 69.85092163085938,
      "activations/layer5_attention_weight_min": -79.05831909179688,
      "activations/layer6_attention_weight_max": 53.36978530883789,
      "activations/layer6_attention_weight_min": -54.52243423461914,
      "activations/layer7_attention_weight_max": 68.76322174072266,
      "activations/layer7_attention_weight_min": -65.53392791748047,
      "activations/layer8_attention_weight_max": 44.82542037963867,
      "activations/layer8_attention_weight_min": -47.60813522338867,
      "activations/layer9_attention_weight_max": 49.50310516357422,
      "activations/layer9_attention_weight_min": -46.57156753540039,
      "epoch": 12.54,
      "learning_rate": 6.986098484848484e-05,
      "loss": 2.7766,
      "step": 215750
    },
    {
      "activations/layer0_attention_weight_max": 17.758281707763672,
      "activations/layer0_attention_weight_min": -13.850811004638672,
      "activations/layer10_attention_weight_max": 40.299560546875,
      "activations/layer10_attention_weight_min": -37.15093994140625,
      "activations/layer11_attention_weight_max": 33.709800720214844,
      "activations/layer11_attention_weight_min": -35.39832305908203,
      "activations/layer12_attention_weight_max": 18.24713897705078,
      "activations/layer12_attention_weight_min": -24.82987403869629,
      "activations/layer13_attention_weight_max": 44.71334457397461,
      "activations/layer13_attention_weight_min": -29.690874099731445,
      "activations/layer14_attention_weight_max": 39.84516525268555,
      "activations/layer14_attention_weight_min": -34.12770462036133,
      "activations/layer15_attention_weight_max": 35.75224685668945,
      "activations/layer15_attention_weight_min": -30.894140243530273,
      "activations/layer16_attention_weight_max": 35.73830032348633,
      "activations/layer16_attention_weight_min": -32.20791244506836,
      "activations/layer17_attention_weight_max": 59.95781326293945,
      "activations/layer17_attention_weight_min": -49.624568939208984,
      "activations/layer18_attention_weight_max": 56.33473587036133,
      "activations/layer18_attention_weight_min": -40.37955856323242,
      "activations/layer19_attention_weight_max": 25.84285545349121,
      "activations/layer19_attention_weight_min": -25.1392765045166,
      "activations/layer1_attention_weight_max": 17.47638702392578,
      "activations/layer1_attention_weight_min": -14.725239753723145,
      "activations/layer20_attention_weight_max": 29.600669860839844,
      "activations/layer20_attention_weight_min": -19.603309631347656,
      "activations/layer21_attention_weight_max": 46.329524993896484,
      "activations/layer21_attention_weight_min": -24.406705856323242,
      "activations/layer22_attention_weight_max": 34.27411651611328,
      "activations/layer22_attention_weight_min": -26.159257888793945,
      "activations/layer23_attention_weight_max": 45.28932189941406,
      "activations/layer23_attention_weight_min": -24.79570770263672,
      "activations/layer2_attention_weight_max": 32.57899856567383,
      "activations/layer2_attention_weight_min": -30.771015167236328,
      "activations/layer3_attention_weight_max": 93.8996353149414,
      "activations/layer3_attention_weight_min": -93.98912048339844,
      "activations/layer4_attention_weight_max": 96.17756652832031,
      "activations/layer4_attention_weight_min": -92.4170150756836,
      "activations/layer5_attention_weight_max": 71.31088256835938,
      "activations/layer5_attention_weight_min": -83.37875366210938,
      "activations/layer6_attention_weight_max": 53.03754806518555,
      "activations/layer6_attention_weight_min": -54.80876159667969,
      "activations/layer7_attention_weight_max": 71.6539535522461,
      "activations/layer7_attention_weight_min": -68.96163177490234,
      "activations/layer8_attention_weight_max": 47.8482780456543,
      "activations/layer8_attention_weight_min": -47.261390686035156,
      "activations/layer9_attention_weight_max": 48.26212692260742,
      "activations/layer9_attention_weight_min": -48.68218994140625,
      "epoch": 12.54,
      "learning_rate": 6.984204545454544e-05,
      "loss": 2.7651,
      "step": 215800
    },
    {
      "activations/layer0_attention_weight_max": 16.283037185668945,
      "activations/layer0_attention_weight_min": -14.45627212524414,
      "activations/layer10_attention_weight_max": 35.27579116821289,
      "activations/layer10_attention_weight_min": -35.522796630859375,
      "activations/layer11_attention_weight_max": 31.033794403076172,
      "activations/layer11_attention_weight_min": -31.446918487548828,
      "activations/layer12_attention_weight_max": 18.21409034729004,
      "activations/layer12_attention_weight_min": -26.13071060180664,
      "activations/layer13_attention_weight_max": 38.90251922607422,
      "activations/layer13_attention_weight_min": -28.7220458984375,
      "activations/layer14_attention_weight_max": 34.99043273925781,
      "activations/layer14_attention_weight_min": -29.377731323242188,
      "activations/layer15_attention_weight_max": 31.391056060791016,
      "activations/layer15_attention_weight_min": -27.178136825561523,
      "activations/layer16_attention_weight_max": 30.661909103393555,
      "activations/layer16_attention_weight_min": -28.569120407104492,
      "activations/layer17_attention_weight_max": 51.85268783569336,
      "activations/layer17_attention_weight_min": -43.17578887939453,
      "activations/layer18_attention_weight_max": 46.228981018066406,
      "activations/layer18_attention_weight_min": -36.66426086425781,
      "activations/layer19_attention_weight_max": 23.26849937438965,
      "activations/layer19_attention_weight_min": -21.68122100830078,
      "activations/layer1_attention_weight_max": 17.01776695251465,
      "activations/layer1_attention_weight_min": -14.02178955078125,
      "activations/layer20_attention_weight_max": 21.903892517089844,
      "activations/layer20_attention_weight_min": -21.004032135009766,
      "activations/layer21_attention_weight_max": 33.94511032104492,
      "activations/layer21_attention_weight_min": -22.49854278564453,
      "activations/layer22_attention_weight_max": 32.33716583251953,
      "activations/layer22_attention_weight_min": -28.40032386779785,
      "activations/layer23_attention_weight_max": 35.4796142578125,
      "activations/layer23_attention_weight_min": -23.955894470214844,
      "activations/layer2_attention_weight_max": 32.61288833618164,
      "activations/layer2_attention_weight_min": -31.508773803710938,
      "activations/layer3_attention_weight_max": 91.2397232055664,
      "activations/layer3_attention_weight_min": -103.55416107177734,
      "activations/layer4_attention_weight_max": 98.35234832763672,
      "activations/layer4_attention_weight_min": -100.53060150146484,
      "activations/layer5_attention_weight_max": 71.22061157226562,
      "activations/layer5_attention_weight_min": -77.3725357055664,
      "activations/layer6_attention_weight_max": 50.19074249267578,
      "activations/layer6_attention_weight_min": -52.45962142944336,
      "activations/layer7_attention_weight_max": 69.24644470214844,
      "activations/layer7_attention_weight_min": -66.03546905517578,
      "activations/layer8_attention_weight_max": 45.85551834106445,
      "activations/layer8_attention_weight_min": -45.53179168701172,
      "activations/layer9_attention_weight_max": 45.616859436035156,
      "activations/layer9_attention_weight_min": -45.358951568603516,
      "epoch": 12.54,
      "learning_rate": 6.982310606060606e-05,
      "loss": 2.7634,
      "step": 215850
    },
    {
      "activations/layer0_attention_weight_max": 16.078481674194336,
      "activations/layer0_attention_weight_min": -14.165623664855957,
      "activations/layer10_attention_weight_max": 35.54977798461914,
      "activations/layer10_attention_weight_min": -36.121681213378906,
      "activations/layer11_attention_weight_max": 32.48656463623047,
      "activations/layer11_attention_weight_min": -33.39495086669922,
      "activations/layer12_attention_weight_max": 19.003978729248047,
      "activations/layer12_attention_weight_min": -23.794021606445312,
      "activations/layer13_attention_weight_max": 36.09157180786133,
      "activations/layer13_attention_weight_min": -30.527057647705078,
      "activations/layer14_attention_weight_max": 35.6292610168457,
      "activations/layer14_attention_weight_min": -29.844057083129883,
      "activations/layer15_attention_weight_max": 32.88478088378906,
      "activations/layer15_attention_weight_min": -29.5815486907959,
      "activations/layer16_attention_weight_max": 32.06246566772461,
      "activations/layer16_attention_weight_min": -31.377397537231445,
      "activations/layer17_attention_weight_max": 56.4167594909668,
      "activations/layer17_attention_weight_min": -46.54090881347656,
      "activations/layer18_attention_weight_max": 50.11668014526367,
      "activations/layer18_attention_weight_min": -40.63069152832031,
      "activations/layer19_attention_weight_max": 24.004697799682617,
      "activations/layer19_attention_weight_min": -24.77451515197754,
      "activations/layer1_attention_weight_max": 16.25856590270996,
      "activations/layer1_attention_weight_min": -14.829874992370605,
      "activations/layer20_attention_weight_max": 24.023576736450195,
      "activations/layer20_attention_weight_min": -22.786191940307617,
      "activations/layer21_attention_weight_max": 39.135154724121094,
      "activations/layer21_attention_weight_min": -30.19656753540039,
      "activations/layer22_attention_weight_max": 29.26362419128418,
      "activations/layer22_attention_weight_min": -27.282217025756836,
      "activations/layer23_attention_weight_max": 36.027137756347656,
      "activations/layer23_attention_weight_min": -25.053421020507812,
      "activations/layer2_attention_weight_max": 33.49815368652344,
      "activations/layer2_attention_weight_min": -31.306503295898438,
      "activations/layer3_attention_weight_max": 91.03020477294922,
      "activations/layer3_attention_weight_min": -96.95587158203125,
      "activations/layer4_attention_weight_max": 96.95925903320312,
      "activations/layer4_attention_weight_min": -92.93887329101562,
      "activations/layer5_attention_weight_max": 71.81732177734375,
      "activations/layer5_attention_weight_min": -80.30221557617188,
      "activations/layer6_attention_weight_max": 58.12131881713867,
      "activations/layer6_attention_weight_min": -53.34536361694336,
      "activations/layer7_attention_weight_max": 73.60740661621094,
      "activations/layer7_attention_weight_min": -75.07026672363281,
      "activations/layer8_attention_weight_max": 50.177330017089844,
      "activations/layer8_attention_weight_min": -50.45499801635742,
      "activations/layer9_attention_weight_max": 49.84656524658203,
      "activations/layer9_attention_weight_min": -52.69892501831055,
      "epoch": 12.55,
      "learning_rate": 6.980416666666666e-05,
      "loss": 2.7689,
      "step": 215900
    },
    {
      "activations/layer0_attention_weight_max": 15.909481048583984,
      "activations/layer0_attention_weight_min": -14.293822288513184,
      "activations/layer10_attention_weight_max": 35.5900764465332,
      "activations/layer10_attention_weight_min": -36.39689636230469,
      "activations/layer11_attention_weight_max": 30.29852867126465,
      "activations/layer11_attention_weight_min": -31.962512969970703,
      "activations/layer12_attention_weight_max": 18.728788375854492,
      "activations/layer12_attention_weight_min": -24.902099609375,
      "activations/layer13_attention_weight_max": 44.00578689575195,
      "activations/layer13_attention_weight_min": -32.98870849609375,
      "activations/layer14_attention_weight_max": 37.92842483520508,
      "activations/layer14_attention_weight_min": -33.580257415771484,
      "activations/layer15_attention_weight_max": 33.99990463256836,
      "activations/layer15_attention_weight_min": -30.23946189880371,
      "activations/layer16_attention_weight_max": 32.28850555419922,
      "activations/layer16_attention_weight_min": -30.238697052001953,
      "activations/layer17_attention_weight_max": 51.42902755737305,
      "activations/layer17_attention_weight_min": -46.46220397949219,
      "activations/layer18_attention_weight_max": 46.978843688964844,
      "activations/layer18_attention_weight_min": -39.705787658691406,
      "activations/layer19_attention_weight_max": 23.1502628326416,
      "activations/layer19_attention_weight_min": -22.528255462646484,
      "activations/layer1_attention_weight_max": 16.947494506835938,
      "activations/layer1_attention_weight_min": -14.299714088439941,
      "activations/layer20_attention_weight_max": 22.3111515045166,
      "activations/layer20_attention_weight_min": -20.893041610717773,
      "activations/layer21_attention_weight_max": 38.522254943847656,
      "activations/layer21_attention_weight_min": -24.728540420532227,
      "activations/layer22_attention_weight_max": 30.477096557617188,
      "activations/layer22_attention_weight_min": -27.086957931518555,
      "activations/layer23_attention_weight_max": 36.241119384765625,
      "activations/layer23_attention_weight_min": -25.667356491088867,
      "activations/layer2_attention_weight_max": 31.144506454467773,
      "activations/layer2_attention_weight_min": -31.112953186035156,
      "activations/layer3_attention_weight_max": 95.47547912597656,
      "activations/layer3_attention_weight_min": -97.98371124267578,
      "activations/layer4_attention_weight_max": 95.6454086303711,
      "activations/layer4_attention_weight_min": -96.96831512451172,
      "activations/layer5_attention_weight_max": 69.79935455322266,
      "activations/layer5_attention_weight_min": -77.688720703125,
      "activations/layer6_attention_weight_max": 52.537052154541016,
      "activations/layer6_attention_weight_min": -55.121551513671875,
      "activations/layer7_attention_weight_max": 70.21393585205078,
      "activations/layer7_attention_weight_min": -67.60212707519531,
      "activations/layer8_attention_weight_max": 45.97502136230469,
      "activations/layer8_attention_weight_min": -47.49617004394531,
      "activations/layer9_attention_weight_max": 47.36787414550781,
      "activations/layer9_attention_weight_min": -49.77150344848633,
      "epoch": 12.55,
      "learning_rate": 6.978522727272726e-05,
      "loss": 2.7542,
      "step": 215950
    },
    {
      "activations/layer0_attention_weight_max": 17.212404251098633,
      "activations/layer0_attention_weight_min": -13.927969932556152,
      "activations/layer10_attention_weight_max": 36.877559661865234,
      "activations/layer10_attention_weight_min": -35.628719329833984,
      "activations/layer11_attention_weight_max": 31.685148239135742,
      "activations/layer11_attention_weight_min": -32.070030212402344,
      "activations/layer12_attention_weight_max": 16.839706420898438,
      "activations/layer12_attention_weight_min": -27.766077041625977,
      "activations/layer13_attention_weight_max": 34.34699630737305,
      "activations/layer13_attention_weight_min": -30.93730354309082,
      "activations/layer14_attention_weight_max": 32.8597412109375,
      "activations/layer14_attention_weight_min": -29.48964500427246,
      "activations/layer15_attention_weight_max": 32.5303955078125,
      "activations/layer15_attention_weight_min": -28.63239288330078,
      "activations/layer16_attention_weight_max": 30.80773162841797,
      "activations/layer16_attention_weight_min": -29.857133865356445,
      "activations/layer17_attention_weight_max": 50.720767974853516,
      "activations/layer17_attention_weight_min": -44.227508544921875,
      "activations/layer18_attention_weight_max": 45.92314910888672,
      "activations/layer18_attention_weight_min": -38.341060638427734,
      "activations/layer19_attention_weight_max": 25.023813247680664,
      "activations/layer19_attention_weight_min": -21.311424255371094,
      "activations/layer1_attention_weight_max": 17.34531593322754,
      "activations/layer1_attention_weight_min": -14.955814361572266,
      "activations/layer20_attention_weight_max": 20.953567504882812,
      "activations/layer20_attention_weight_min": -20.99675178527832,
      "activations/layer21_attention_weight_max": 34.1794548034668,
      "activations/layer21_attention_weight_min": -22.639150619506836,
      "activations/layer22_attention_weight_max": 31.59543800354004,
      "activations/layer22_attention_weight_min": -25.157224655151367,
      "activations/layer23_attention_weight_max": 36.136871337890625,
      "activations/layer23_attention_weight_min": -21.952056884765625,
      "activations/layer2_attention_weight_max": 31.901103973388672,
      "activations/layer2_attention_weight_min": -31.573902130126953,
      "activations/layer3_attention_weight_max": 94.63865661621094,
      "activations/layer3_attention_weight_min": -97.20846557617188,
      "activations/layer4_attention_weight_max": 94.37971496582031,
      "activations/layer4_attention_weight_min": -93.5350112915039,
      "activations/layer5_attention_weight_max": 75.82194519042969,
      "activations/layer5_attention_weight_min": -80.9744873046875,
      "activations/layer6_attention_weight_max": 52.46455001831055,
      "activations/layer6_attention_weight_min": -52.51705551147461,
      "activations/layer7_attention_weight_max": 72.33240509033203,
      "activations/layer7_attention_weight_min": -70.70825958251953,
      "activations/layer8_attention_weight_max": 48.18089294433594,
      "activations/layer8_attention_weight_min": -48.40616989135742,
      "activations/layer9_attention_weight_max": 48.12637710571289,
      "activations/layer9_attention_weight_min": -46.31159591674805,
      "epoch": 12.55,
      "learning_rate": 6.976628787878787e-05,
      "loss": 2.7776,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.5167,
      "eval_samples_per_second": 504.183,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.5167,
      "eval_openwebtext_samples_per_second": 504.183,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 1.9551,
      "eval_wikitext_samples_per_second": 233.232,
      "step": 216000
    },
    {
      "epoch": 12.55,
      "eval_lambada_loss": 2.50390625,
      "eval_lambada_ppl": 12.230174893930755,
      "eval_lambada_runtime": 9.524,
      "eval_lambada_samples_per_second": 511.235,
      "step": 216000
    },
    {
      "activations/layer0_attention_weight_max": 16.82301139831543,
      "activations/layer0_attention_weight_min": -13.983530044555664,
      "activations/layer10_attention_weight_max": 37.286590576171875,
      "activations/layer10_attention_weight_min": -33.83185577392578,
      "activations/layer11_attention_weight_max": 34.877105712890625,
      "activations/layer11_attention_weight_min": -34.42595291137695,
      "activations/layer12_attention_weight_max": 17.935266494750977,
      "activations/layer12_attention_weight_min": -23.954187393188477,
      "activations/layer13_attention_weight_max": 36.24166488647461,
      "activations/layer13_attention_weight_min": -29.369340896606445,
      "activations/layer14_attention_weight_max": 35.8832893371582,
      "activations/layer14_attention_weight_min": -29.30906867980957,
      "activations/layer15_attention_weight_max": 32.16387939453125,
      "activations/layer15_attention_weight_min": -29.05019760131836,
      "activations/layer16_attention_weight_max": 30.67522430419922,
      "activations/layer16_attention_weight_min": -29.884912490844727,
      "activations/layer17_attention_weight_max": 51.462467193603516,
      "activations/layer17_attention_weight_min": -43.11660385131836,
      "activations/layer18_attention_weight_max": 49.385616302490234,
      "activations/layer18_attention_weight_min": -38.60869598388672,
      "activations/layer19_attention_weight_max": 23.349088668823242,
      "activations/layer19_attention_weight_min": -21.4979248046875,
      "activations/layer1_attention_weight_max": 17.265724182128906,
      "activations/layer1_attention_weight_min": -14.4443359375,
      "activations/layer20_attention_weight_max": 23.875452041625977,
      "activations/layer20_attention_weight_min": -27.864477157592773,
      "activations/layer21_attention_weight_max": 35.50754165649414,
      "activations/layer21_attention_weight_min": -21.335752487182617,
      "activations/layer22_attention_weight_max": 31.76062774658203,
      "activations/layer22_attention_weight_min": -24.90437126159668,
      "activations/layer23_attention_weight_max": 40.39715576171875,
      "activations/layer23_attention_weight_min": -23.0736083984375,
      "activations/layer2_attention_weight_max": 31.51301383972168,
      "activations/layer2_attention_weight_min": -30.049991607666016,
      "activations/layer3_attention_weight_max": 95.94444274902344,
      "activations/layer3_attention_weight_min": -98.0086669921875,
      "activations/layer4_attention_weight_max": 95.03793334960938,
      "activations/layer4_attention_weight_min": -93.64452362060547,
      "activations/layer5_attention_weight_max": 70.07463073730469,
      "activations/layer5_attention_weight_min": -80.73395538330078,
      "activations/layer6_attention_weight_max": 51.62675857543945,
      "activations/layer6_attention_weight_min": -51.35247039794922,
      "activations/layer7_attention_weight_max": 68.07197570800781,
      "activations/layer7_attention_weight_min": -66.48590850830078,
      "activations/layer8_attention_weight_max": 46.16447830200195,
      "activations/layer8_attention_weight_min": -46.4112434387207,
      "activations/layer9_attention_weight_max": 45.456993103027344,
      "activations/layer9_attention_weight_min": -46.01377487182617,
      "epoch": 12.55,
      "learning_rate": 6.974734848484848e-05,
      "loss": 2.7705,
      "step": 216050
    },
    {
      "activations/layer0_attention_weight_max": 16.0599308013916,
      "activations/layer0_attention_weight_min": -13.98261547088623,
      "activations/layer10_attention_weight_max": 34.47587585449219,
      "activations/layer10_attention_weight_min": -35.081180572509766,
      "activations/layer11_attention_weight_max": 31.643230438232422,
      "activations/layer11_attention_weight_min": -32.451412200927734,
      "activations/layer12_attention_weight_max": 19.573373794555664,
      "activations/layer12_attention_weight_min": -22.916112899780273,
      "activations/layer13_attention_weight_max": 44.91837692260742,
      "activations/layer13_attention_weight_min": -32.317466735839844,
      "activations/layer14_attention_weight_max": 39.40479278564453,
      "activations/layer14_attention_weight_min": -30.7806396484375,
      "activations/layer15_attention_weight_max": 33.171939849853516,
      "activations/layer15_attention_weight_min": -31.021068572998047,
      "activations/layer16_attention_weight_max": 34.268775939941406,
      "activations/layer16_attention_weight_min": -31.842769622802734,
      "activations/layer17_attention_weight_max": 54.7188720703125,
      "activations/layer17_attention_weight_min": -44.70045471191406,
      "activations/layer18_attention_weight_max": 46.829322814941406,
      "activations/layer18_attention_weight_min": -36.669219970703125,
      "activations/layer19_attention_weight_max": 27.323453903198242,
      "activations/layer19_attention_weight_min": -21.885007858276367,
      "activations/layer1_attention_weight_max": 15.852437973022461,
      "activations/layer1_attention_weight_min": -14.707491874694824,
      "activations/layer20_attention_weight_max": 22.9029598236084,
      "activations/layer20_attention_weight_min": -20.72266387939453,
      "activations/layer21_attention_weight_max": 40.16901397705078,
      "activations/layer21_attention_weight_min": -21.80084228515625,
      "activations/layer22_attention_weight_max": 33.72452926635742,
      "activations/layer22_attention_weight_min": -25.21804428100586,
      "activations/layer23_attention_weight_max": 38.50399398803711,
      "activations/layer23_attention_weight_min": -23.74844741821289,
      "activations/layer2_attention_weight_max": 31.130218505859375,
      "activations/layer2_attention_weight_min": -29.304922103881836,
      "activations/layer3_attention_weight_max": 93.49951171875,
      "activations/layer3_attention_weight_min": -97.7547378540039,
      "activations/layer4_attention_weight_max": 94.0644302368164,
      "activations/layer4_attention_weight_min": -89.88607025146484,
      "activations/layer5_attention_weight_max": 74.86309814453125,
      "activations/layer5_attention_weight_min": -77.8943862915039,
      "activations/layer6_attention_weight_max": 51.3575325012207,
      "activations/layer6_attention_weight_min": -51.53388214111328,
      "activations/layer7_attention_weight_max": 73.22290802001953,
      "activations/layer7_attention_weight_min": -65.87933349609375,
      "activations/layer8_attention_weight_max": 46.995174407958984,
      "activations/layer8_attention_weight_min": -46.599571228027344,
      "activations/layer9_attention_weight_max": 46.40458297729492,
      "activations/layer9_attention_weight_min": -49.17354965209961,
      "epoch": 12.56,
      "learning_rate": 6.972840909090909e-05,
      "loss": 2.7658,
      "step": 216100
    },
    {
      "activations/layer0_attention_weight_max": 15.777352333068848,
      "activations/layer0_attention_weight_min": -14.07130241394043,
      "activations/layer10_attention_weight_max": 34.622657775878906,
      "activations/layer10_attention_weight_min": -34.88081359863281,
      "activations/layer11_attention_weight_max": 32.14498519897461,
      "activations/layer11_attention_weight_min": -32.05427551269531,
      "activations/layer12_attention_weight_max": 22.91263771057129,
      "activations/layer12_attention_weight_min": -25.320648193359375,
      "activations/layer13_attention_weight_max": 44.65148162841797,
      "activations/layer13_attention_weight_min": -27.840042114257812,
      "activations/layer14_attention_weight_max": 37.358238220214844,
      "activations/layer14_attention_weight_min": -29.829845428466797,
      "activations/layer15_attention_weight_max": 32.96037673950195,
      "activations/layer15_attention_weight_min": -30.344558715820312,
      "activations/layer16_attention_weight_max": 32.29075241088867,
      "activations/layer16_attention_weight_min": -30.44554901123047,
      "activations/layer17_attention_weight_max": 53.35746383666992,
      "activations/layer17_attention_weight_min": -44.227081298828125,
      "activations/layer18_attention_weight_max": 49.3867301940918,
      "activations/layer18_attention_weight_min": -38.99101638793945,
      "activations/layer19_attention_weight_max": 22.18705940246582,
      "activations/layer19_attention_weight_min": -23.670623779296875,
      "activations/layer1_attention_weight_max": 16.228939056396484,
      "activations/layer1_attention_weight_min": -14.386622428894043,
      "activations/layer20_attention_weight_max": 22.907840728759766,
      "activations/layer20_attention_weight_min": -24.15204429626465,
      "activations/layer21_attention_weight_max": 41.6112060546875,
      "activations/layer21_attention_weight_min": -24.50148582458496,
      "activations/layer22_attention_weight_max": 28.359386444091797,
      "activations/layer22_attention_weight_min": -25.01202964782715,
      "activations/layer23_attention_weight_max": 39.21657180786133,
      "activations/layer23_attention_weight_min": -23.296611785888672,
      "activations/layer2_attention_weight_max": 33.621246337890625,
      "activations/layer2_attention_weight_min": -32.83572006225586,
      "activations/layer3_attention_weight_max": 97.6503677368164,
      "activations/layer3_attention_weight_min": -100.18492889404297,
      "activations/layer4_attention_weight_max": 99.73351287841797,
      "activations/layer4_attention_weight_min": -95.18328857421875,
      "activations/layer5_attention_weight_max": 69.77104949951172,
      "activations/layer5_attention_weight_min": -76.6082534790039,
      "activations/layer6_attention_weight_max": 53.374332427978516,
      "activations/layer6_attention_weight_min": -54.46239471435547,
      "activations/layer7_attention_weight_max": 70.29698944091797,
      "activations/layer7_attention_weight_min": -72.43557739257812,
      "activations/layer8_attention_weight_max": 44.98579788208008,
      "activations/layer8_attention_weight_min": -47.862152099609375,
      "activations/layer9_attention_weight_max": 50.53606033325195,
      "activations/layer9_attention_weight_min": -47.63040542602539,
      "epoch": 12.56,
      "learning_rate": 6.970946969696969e-05,
      "loss": 2.7724,
      "step": 216150
    },
    {
      "activations/layer0_attention_weight_max": 15.978371620178223,
      "activations/layer0_attention_weight_min": -14.839120864868164,
      "activations/layer10_attention_weight_max": 36.020484924316406,
      "activations/layer10_attention_weight_min": -36.94948196411133,
      "activations/layer11_attention_weight_max": 38.93974685668945,
      "activations/layer11_attention_weight_min": -33.23303985595703,
      "activations/layer12_attention_weight_max": 18.687341690063477,
      "activations/layer12_attention_weight_min": -37.37964630126953,
      "activations/layer13_attention_weight_max": 38.3477783203125,
      "activations/layer13_attention_weight_min": -26.192501068115234,
      "activations/layer14_attention_weight_max": 31.310688018798828,
      "activations/layer14_attention_weight_min": -27.686077117919922,
      "activations/layer15_attention_weight_max": 28.111291885375977,
      "activations/layer15_attention_weight_min": -27.326242446899414,
      "activations/layer16_attention_weight_max": 29.011266708374023,
      "activations/layer16_attention_weight_min": -28.365596771240234,
      "activations/layer17_attention_weight_max": 46.93185043334961,
      "activations/layer17_attention_weight_min": -45.64066696166992,
      "activations/layer18_attention_weight_max": 43.54050064086914,
      "activations/layer18_attention_weight_min": -36.90485382080078,
      "activations/layer19_attention_weight_max": 21.71198081970215,
      "activations/layer19_attention_weight_min": -22.33241844177246,
      "activations/layer1_attention_weight_max": 17.964950561523438,
      "activations/layer1_attention_weight_min": -16.038667678833008,
      "activations/layer20_attention_weight_max": 19.85848617553711,
      "activations/layer20_attention_weight_min": -19.93391227722168,
      "activations/layer21_attention_weight_max": 35.794281005859375,
      "activations/layer21_attention_weight_min": -21.55634117126465,
      "activations/layer22_attention_weight_max": 26.37959861755371,
      "activations/layer22_attention_weight_min": -28.25124168395996,
      "activations/layer23_attention_weight_max": 35.545387268066406,
      "activations/layer23_attention_weight_min": -24.28661346435547,
      "activations/layer2_attention_weight_max": 32.784210205078125,
      "activations/layer2_attention_weight_min": -32.0665397644043,
      "activations/layer3_attention_weight_max": 94.8967056274414,
      "activations/layer3_attention_weight_min": -96.79735565185547,
      "activations/layer4_attention_weight_max": 97.7744369506836,
      "activations/layer4_attention_weight_min": -91.3241958618164,
      "activations/layer5_attention_weight_max": 71.585205078125,
      "activations/layer5_attention_weight_min": -73.83795166015625,
      "activations/layer6_attention_weight_max": 49.90615463256836,
      "activations/layer6_attention_weight_min": -50.72819137573242,
      "activations/layer7_attention_weight_max": 80.00283813476562,
      "activations/layer7_attention_weight_min": -68.65054321289062,
      "activations/layer8_attention_weight_max": 51.929080963134766,
      "activations/layer8_attention_weight_min": -47.20131301879883,
      "activations/layer9_attention_weight_max": 60.79677963256836,
      "activations/layer9_attention_weight_min": -48.224403381347656,
      "epoch": 12.56,
      "learning_rate": 6.96905303030303e-05,
      "loss": 2.7836,
      "step": 216200
    },
    {
      "activations/layer0_attention_weight_max": 16.437992095947266,
      "activations/layer0_attention_weight_min": -14.43539047241211,
      "activations/layer10_attention_weight_max": 33.38459014892578,
      "activations/layer10_attention_weight_min": -34.31515121459961,
      "activations/layer11_attention_weight_max": 31.867042541503906,
      "activations/layer11_attention_weight_min": -32.81993865966797,
      "activations/layer12_attention_weight_max": 17.515628814697266,
      "activations/layer12_attention_weight_min": -25.698993682861328,
      "activations/layer13_attention_weight_max": 35.40187454223633,
      "activations/layer13_attention_weight_min": -35.66668701171875,
      "activations/layer14_attention_weight_max": 36.72566223144531,
      "activations/layer14_attention_weight_min": -33.2293815612793,
      "activations/layer15_attention_weight_max": 30.996341705322266,
      "activations/layer15_attention_weight_min": -31.065500259399414,
      "activations/layer16_attention_weight_max": 29.279340744018555,
      "activations/layer16_attention_weight_min": -28.981340408325195,
      "activations/layer17_attention_weight_max": 46.57854461669922,
      "activations/layer17_attention_weight_min": -41.457435607910156,
      "activations/layer18_attention_weight_max": 41.6281623840332,
      "activations/layer18_attention_weight_min": -35.94868087768555,
      "activations/layer19_attention_weight_max": 22.17466926574707,
      "activations/layer19_attention_weight_min": -24.156728744506836,
      "activations/layer1_attention_weight_max": 16.700258255004883,
      "activations/layer1_attention_weight_min": -15.970061302185059,
      "activations/layer20_attention_weight_max": 20.668928146362305,
      "activations/layer20_attention_weight_min": -24.281700134277344,
      "activations/layer21_attention_weight_max": 35.50674819946289,
      "activations/layer21_attention_weight_min": -26.641159057617188,
      "activations/layer22_attention_weight_max": 26.156145095825195,
      "activations/layer22_attention_weight_min": -24.912384033203125,
      "activations/layer23_attention_weight_max": 35.34089660644531,
      "activations/layer23_attention_weight_min": -25.054990768432617,
      "activations/layer2_attention_weight_max": 35.42237854003906,
      "activations/layer2_attention_weight_min": -31.029216766357422,
      "activations/layer3_attention_weight_max": 96.65198516845703,
      "activations/layer3_attention_weight_min": -90.1031723022461,
      "activations/layer4_attention_weight_max": 100.09770965576172,
      "activations/layer4_attention_weight_min": -87.54032897949219,
      "activations/layer5_attention_weight_max": 79.19259643554688,
      "activations/layer5_attention_weight_min": -76.74897766113281,
      "activations/layer6_attention_weight_max": 55.91245651245117,
      "activations/layer6_attention_weight_min": -52.947959899902344,
      "activations/layer7_attention_weight_max": 70.27493286132812,
      "activations/layer7_attention_weight_min": -66.08843231201172,
      "activations/layer8_attention_weight_max": 46.85696029663086,
      "activations/layer8_attention_weight_min": -45.87638854980469,
      "activations/layer9_attention_weight_max": 47.41743850708008,
      "activations/layer9_attention_weight_min": -43.5210075378418,
      "epoch": 12.57,
      "learning_rate": 6.967159090909091e-05,
      "loss": 2.7765,
      "step": 216250
    },
    {
      "activations/layer0_attention_weight_max": 15.917710304260254,
      "activations/layer0_attention_weight_min": -14.18040943145752,
      "activations/layer10_attention_weight_max": 33.13706970214844,
      "activations/layer10_attention_weight_min": -32.17163848876953,
      "activations/layer11_attention_weight_max": 30.48666763305664,
      "activations/layer11_attention_weight_min": -31.71295166015625,
      "activations/layer12_attention_weight_max": 22.322006225585938,
      "activations/layer12_attention_weight_min": -27.697908401489258,
      "activations/layer13_attention_weight_max": 41.42046356201172,
      "activations/layer13_attention_weight_min": -32.7474365234375,
      "activations/layer14_attention_weight_max": 41.60993957519531,
      "activations/layer14_attention_weight_min": -30.84539794921875,
      "activations/layer15_attention_weight_max": 36.854164123535156,
      "activations/layer15_attention_weight_min": -28.72296714782715,
      "activations/layer16_attention_weight_max": 30.969987869262695,
      "activations/layer16_attention_weight_min": -30.248071670532227,
      "activations/layer17_attention_weight_max": 50.054710388183594,
      "activations/layer17_attention_weight_min": -43.07895278930664,
      "activations/layer18_attention_weight_max": 45.68161392211914,
      "activations/layer18_attention_weight_min": -38.569984436035156,
      "activations/layer19_attention_weight_max": 21.564809799194336,
      "activations/layer19_attention_weight_min": -21.830785751342773,
      "activations/layer1_attention_weight_max": 17.43730926513672,
      "activations/layer1_attention_weight_min": -15.10168170928955,
      "activations/layer20_attention_weight_max": 21.942476272583008,
      "activations/layer20_attention_weight_min": -20.465660095214844,
      "activations/layer21_attention_weight_max": 34.84074783325195,
      "activations/layer21_attention_weight_min": -23.62236785888672,
      "activations/layer22_attention_weight_max": 29.115476608276367,
      "activations/layer22_attention_weight_min": -24.7796573638916,
      "activations/layer23_attention_weight_max": 35.353031158447266,
      "activations/layer23_attention_weight_min": -23.094463348388672,
      "activations/layer2_attention_weight_max": 31.236562728881836,
      "activations/layer2_attention_weight_min": -29.095640182495117,
      "activations/layer3_attention_weight_max": 86.27086639404297,
      "activations/layer3_attention_weight_min": -93.08502960205078,
      "activations/layer4_attention_weight_max": 90.01747131347656,
      "activations/layer4_attention_weight_min": -87.86225891113281,
      "activations/layer5_attention_weight_max": 66.8893814086914,
      "activations/layer5_attention_weight_min": -84.16889190673828,
      "activations/layer6_attention_weight_max": 50.566219329833984,
      "activations/layer6_attention_weight_min": -51.7352409362793,
      "activations/layer7_attention_weight_max": 65.77237701416016,
      "activations/layer7_attention_weight_min": -65.61634826660156,
      "activations/layer8_attention_weight_max": 41.68214797973633,
      "activations/layer8_attention_weight_min": -44.318626403808594,
      "activations/layer9_attention_weight_max": 41.76882553100586,
      "activations/layer9_attention_weight_min": -45.08799362182617,
      "epoch": 12.57,
      "learning_rate": 6.965265151515151e-05,
      "loss": 2.7613,
      "step": 216300
    },
    {
      "activations/layer0_attention_weight_max": 15.65699577331543,
      "activations/layer0_attention_weight_min": -14.209068298339844,
      "activations/layer10_attention_weight_max": 37.57981872558594,
      "activations/layer10_attention_weight_min": -36.8132209777832,
      "activations/layer11_attention_weight_max": 34.21828079223633,
      "activations/layer11_attention_weight_min": -33.87631607055664,
      "activations/layer12_attention_weight_max": 19.049495697021484,
      "activations/layer12_attention_weight_min": -21.9726619720459,
      "activations/layer13_attention_weight_max": 37.59151077270508,
      "activations/layer13_attention_weight_min": -32.94594955444336,
      "activations/layer14_attention_weight_max": 37.7836799621582,
      "activations/layer14_attention_weight_min": -30.567140579223633,
      "activations/layer15_attention_weight_max": 37.13447952270508,
      "activations/layer15_attention_weight_min": -28.99409294128418,
      "activations/layer16_attention_weight_max": 34.4703254699707,
      "activations/layer16_attention_weight_min": -30.936588287353516,
      "activations/layer17_attention_weight_max": 56.34246063232422,
      "activations/layer17_attention_weight_min": -44.70391845703125,
      "activations/layer18_attention_weight_max": 49.443572998046875,
      "activations/layer18_attention_weight_min": -38.912925720214844,
      "activations/layer19_attention_weight_max": 24.88827896118164,
      "activations/layer19_attention_weight_min": -24.09438133239746,
      "activations/layer1_attention_weight_max": 17.0653018951416,
      "activations/layer1_attention_weight_min": -16.175086975097656,
      "activations/layer20_attention_weight_max": 23.88819694519043,
      "activations/layer20_attention_weight_min": -21.730417251586914,
      "activations/layer21_attention_weight_max": 37.94237518310547,
      "activations/layer21_attention_weight_min": -24.41852378845215,
      "activations/layer22_attention_weight_max": 31.474300384521484,
      "activations/layer22_attention_weight_min": -27.40509796142578,
      "activations/layer23_attention_weight_max": 37.54184341430664,
      "activations/layer23_attention_weight_min": -23.81610107421875,
      "activations/layer2_attention_weight_max": 33.32960510253906,
      "activations/layer2_attention_weight_min": -31.92522430419922,
      "activations/layer3_attention_weight_max": 93.92257690429688,
      "activations/layer3_attention_weight_min": -97.01766967773438,
      "activations/layer4_attention_weight_max": 97.20165252685547,
      "activations/layer4_attention_weight_min": -95.13387298583984,
      "activations/layer5_attention_weight_max": 71.56167602539062,
      "activations/layer5_attention_weight_min": -83.00129699707031,
      "activations/layer6_attention_weight_max": 53.87947082519531,
      "activations/layer6_attention_weight_min": -54.69011306762695,
      "activations/layer7_attention_weight_max": 69.54568481445312,
      "activations/layer7_attention_weight_min": -73.20357513427734,
      "activations/layer8_attention_weight_max": 47.87492752075195,
      "activations/layer8_attention_weight_min": -49.863372802734375,
      "activations/layer9_attention_weight_max": 45.799598693847656,
      "activations/layer9_attention_weight_min": -47.39727020263672,
      "epoch": 12.57,
      "learning_rate": 6.963371212121211e-05,
      "loss": 2.7619,
      "step": 216350
    },
    {
      "activations/layer0_attention_weight_max": 16.85470962524414,
      "activations/layer0_attention_weight_min": -14.718226432800293,
      "activations/layer10_attention_weight_max": 36.11875534057617,
      "activations/layer10_attention_weight_min": -36.23189163208008,
      "activations/layer11_attention_weight_max": 32.75477600097656,
      "activations/layer11_attention_weight_min": -34.130741119384766,
      "activations/layer12_attention_weight_max": 18.30072021484375,
      "activations/layer12_attention_weight_min": -25.36767578125,
      "activations/layer13_attention_weight_max": 39.44685363769531,
      "activations/layer13_attention_weight_min": -28.4251708984375,
      "activations/layer14_attention_weight_max": 38.23346710205078,
      "activations/layer14_attention_weight_min": -30.909351348876953,
      "activations/layer15_attention_weight_max": 34.174522399902344,
      "activations/layer15_attention_weight_min": -29.545791625976562,
      "activations/layer16_attention_weight_max": 33.189697265625,
      "activations/layer16_attention_weight_min": -32.11972427368164,
      "activations/layer17_attention_weight_max": 55.6757698059082,
      "activations/layer17_attention_weight_min": -45.60104751586914,
      "activations/layer18_attention_weight_max": 49.11170196533203,
      "activations/layer18_attention_weight_min": -39.226715087890625,
      "activations/layer19_attention_weight_max": 22.79833221435547,
      "activations/layer19_attention_weight_min": -21.769855499267578,
      "activations/layer1_attention_weight_max": 16.74849510192871,
      "activations/layer1_attention_weight_min": -16.481088638305664,
      "activations/layer20_attention_weight_max": 22.674089431762695,
      "activations/layer20_attention_weight_min": -23.42778968811035,
      "activations/layer21_attention_weight_max": 36.19475555419922,
      "activations/layer21_attention_weight_min": -21.609222412109375,
      "activations/layer22_attention_weight_max": 31.605045318603516,
      "activations/layer22_attention_weight_min": -25.330810546875,
      "activations/layer23_attention_weight_max": 42.8534049987793,
      "activations/layer23_attention_weight_min": -25.278470993041992,
      "activations/layer2_attention_weight_max": 32.17986297607422,
      "activations/layer2_attention_weight_min": -31.189620971679688,
      "activations/layer3_attention_weight_max": 90.24341583251953,
      "activations/layer3_attention_weight_min": -92.69622802734375,
      "activations/layer4_attention_weight_max": 96.42540740966797,
      "activations/layer4_attention_weight_min": -94.01265716552734,
      "activations/layer5_attention_weight_max": 74.32595825195312,
      "activations/layer5_attention_weight_min": -78.53950500488281,
      "activations/layer6_attention_weight_max": 55.78076934814453,
      "activations/layer6_attention_weight_min": -55.89862060546875,
      "activations/layer7_attention_weight_max": 73.95978546142578,
      "activations/layer7_attention_weight_min": -71.08659362792969,
      "activations/layer8_attention_weight_max": 47.85062789916992,
      "activations/layer8_attention_weight_min": -48.05171585083008,
      "activations/layer9_attention_weight_max": 48.80755615234375,
      "activations/layer9_attention_weight_min": -50.99697494506836,
      "epoch": 12.57,
      "learning_rate": 6.961477272727273e-05,
      "loss": 2.7693,
      "step": 216400
    },
    {
      "activations/layer0_attention_weight_max": 16.97211456298828,
      "activations/layer0_attention_weight_min": -14.317766189575195,
      "activations/layer10_attention_weight_max": 35.44633865356445,
      "activations/layer10_attention_weight_min": -32.752471923828125,
      "activations/layer11_attention_weight_max": 30.086210250854492,
      "activations/layer11_attention_weight_min": -30.73968505859375,
      "activations/layer12_attention_weight_max": 17.431550979614258,
      "activations/layer12_attention_weight_min": -28.014339447021484,
      "activations/layer13_attention_weight_max": 36.90773010253906,
      "activations/layer13_attention_weight_min": -29.58907699584961,
      "activations/layer14_attention_weight_max": 34.18891525268555,
      "activations/layer14_attention_weight_min": -28.99162483215332,
      "activations/layer15_attention_weight_max": 30.572172164916992,
      "activations/layer15_attention_weight_min": -28.241601943969727,
      "activations/layer16_attention_weight_max": 29.464693069458008,
      "activations/layer16_attention_weight_min": -28.422208786010742,
      "activations/layer17_attention_weight_max": 50.54523468017578,
      "activations/layer17_attention_weight_min": -42.62932586669922,
      "activations/layer18_attention_weight_max": 42.84302520751953,
      "activations/layer18_attention_weight_min": -35.53376007080078,
      "activations/layer19_attention_weight_max": 24.407310485839844,
      "activations/layer19_attention_weight_min": -22.790063858032227,
      "activations/layer1_attention_weight_max": 15.629136085510254,
      "activations/layer1_attention_weight_min": -15.963833808898926,
      "activations/layer20_attention_weight_max": 23.939809799194336,
      "activations/layer20_attention_weight_min": -22.504335403442383,
      "activations/layer21_attention_weight_max": 40.36556625366211,
      "activations/layer21_attention_weight_min": -21.235687255859375,
      "activations/layer22_attention_weight_max": 29.007837295532227,
      "activations/layer22_attention_weight_min": -25.675275802612305,
      "activations/layer23_attention_weight_max": 35.383270263671875,
      "activations/layer23_attention_weight_min": -23.758129119873047,
      "activations/layer2_attention_weight_max": 33.46743392944336,
      "activations/layer2_attention_weight_min": -32.91349792480469,
      "activations/layer3_attention_weight_max": 91.10493469238281,
      "activations/layer3_attention_weight_min": -91.95946502685547,
      "activations/layer4_attention_weight_max": 91.89905548095703,
      "activations/layer4_attention_weight_min": -90.93206787109375,
      "activations/layer5_attention_weight_max": 69.69863891601562,
      "activations/layer5_attention_weight_min": -78.94463348388672,
      "activations/layer6_attention_weight_max": 54.36354064941406,
      "activations/layer6_attention_weight_min": -54.77152633666992,
      "activations/layer7_attention_weight_max": 66.23360443115234,
      "activations/layer7_attention_weight_min": -67.34812927246094,
      "activations/layer8_attention_weight_max": 46.67524337768555,
      "activations/layer8_attention_weight_min": -47.75062942504883,
      "activations/layer9_attention_weight_max": 43.84418487548828,
      "activations/layer9_attention_weight_min": -46.31315612792969,
      "epoch": 12.58,
      "learning_rate": 6.959583333333333e-05,
      "loss": 2.7684,
      "step": 216450
    },
    {
      "activations/layer0_attention_weight_max": 15.461057662963867,
      "activations/layer0_attention_weight_min": -15.037915229797363,
      "activations/layer10_attention_weight_max": 37.984619140625,
      "activations/layer10_attention_weight_min": -38.38752365112305,
      "activations/layer11_attention_weight_max": 33.15287780761719,
      "activations/layer11_attention_weight_min": -34.53626251220703,
      "activations/layer12_attention_weight_max": 19.861425399780273,
      "activations/layer12_attention_weight_min": -25.377567291259766,
      "activations/layer13_attention_weight_max": 36.24311828613281,
      "activations/layer13_attention_weight_min": -27.51071548461914,
      "activations/layer14_attention_weight_max": 38.226356506347656,
      "activations/layer14_attention_weight_min": -31.1878662109375,
      "activations/layer15_attention_weight_max": 35.22356414794922,
      "activations/layer15_attention_weight_min": -31.463895797729492,
      "activations/layer16_attention_weight_max": 34.718257904052734,
      "activations/layer16_attention_weight_min": -32.542972564697266,
      "activations/layer17_attention_weight_max": 56.93466567993164,
      "activations/layer17_attention_weight_min": -46.49602508544922,
      "activations/layer18_attention_weight_max": 51.328250885009766,
      "activations/layer18_attention_weight_min": -41.081298828125,
      "activations/layer19_attention_weight_max": 23.496742248535156,
      "activations/layer19_attention_weight_min": -22.900699615478516,
      "activations/layer1_attention_weight_max": 17.080045700073242,
      "activations/layer1_attention_weight_min": -14.589324951171875,
      "activations/layer20_attention_weight_max": 23.531702041625977,
      "activations/layer20_attention_weight_min": -23.107885360717773,
      "activations/layer21_attention_weight_max": 42.215476989746094,
      "activations/layer21_attention_weight_min": -24.296483993530273,
      "activations/layer22_attention_weight_max": 32.9575080871582,
      "activations/layer22_attention_weight_min": -27.776105880737305,
      "activations/layer23_attention_weight_max": 40.255828857421875,
      "activations/layer23_attention_weight_min": -24.405677795410156,
      "activations/layer2_attention_weight_max": 33.18843460083008,
      "activations/layer2_attention_weight_min": -32.788265228271484,
      "activations/layer3_attention_weight_max": 95.9501724243164,
      "activations/layer3_attention_weight_min": -99.36564636230469,
      "activations/layer4_attention_weight_max": 95.92349243164062,
      "activations/layer4_attention_weight_min": -95.59756469726562,
      "activations/layer5_attention_weight_max": 75.48675537109375,
      "activations/layer5_attention_weight_min": -83.35456848144531,
      "activations/layer6_attention_weight_max": 54.28019332885742,
      "activations/layer6_attention_weight_min": -54.956092834472656,
      "activations/layer7_attention_weight_max": 72.52230834960938,
      "activations/layer7_attention_weight_min": -71.16728210449219,
      "activations/layer8_attention_weight_max": 47.68674087524414,
      "activations/layer8_attention_weight_min": -48.137977600097656,
      "activations/layer9_attention_weight_max": 47.03386688232422,
      "activations/layer9_attention_weight_min": -47.53053283691406,
      "epoch": 12.58,
      "learning_rate": 6.957689393939393e-05,
      "loss": 2.7635,
      "step": 216500
    },
    {
      "activations/layer0_attention_weight_max": 15.737212181091309,
      "activations/layer0_attention_weight_min": -14.21249771118164,
      "activations/layer10_attention_weight_max": 35.95717239379883,
      "activations/layer10_attention_weight_min": -36.7236213684082,
      "activations/layer11_attention_weight_max": 32.5751953125,
      "activations/layer11_attention_weight_min": -32.398834228515625,
      "activations/layer12_attention_weight_max": 19.06479835510254,
      "activations/layer12_attention_weight_min": -27.986425399780273,
      "activations/layer13_attention_weight_max": 38.381248474121094,
      "activations/layer13_attention_weight_min": -31.571035385131836,
      "activations/layer14_attention_weight_max": 35.40203857421875,
      "activations/layer14_attention_weight_min": -31.14954376220703,
      "activations/layer15_attention_weight_max": 30.958942413330078,
      "activations/layer15_attention_weight_min": -32.851295471191406,
      "activations/layer16_attention_weight_max": 30.007829666137695,
      "activations/layer16_attention_weight_min": -28.633647918701172,
      "activations/layer17_attention_weight_max": 49.94221878051758,
      "activations/layer17_attention_weight_min": -40.85625076293945,
      "activations/layer18_attention_weight_max": 43.96337890625,
      "activations/layer18_attention_weight_min": -37.18775177001953,
      "activations/layer19_attention_weight_max": 22.01220703125,
      "activations/layer19_attention_weight_min": -21.017669677734375,
      "activations/layer1_attention_weight_max": 17.10408592224121,
      "activations/layer1_attention_weight_min": -15.53450870513916,
      "activations/layer20_attention_weight_max": 20.232952117919922,
      "activations/layer20_attention_weight_min": -22.557268142700195,
      "activations/layer21_attention_weight_max": 32.000816345214844,
      "activations/layer21_attention_weight_min": -23.55376625061035,
      "activations/layer22_attention_weight_max": 26.68793487548828,
      "activations/layer22_attention_weight_min": -26.059465408325195,
      "activations/layer23_attention_weight_max": 39.932403564453125,
      "activations/layer23_attention_weight_min": -24.65359878540039,
      "activations/layer2_attention_weight_max": 34.56673049926758,
      "activations/layer2_attention_weight_min": -29.91158103942871,
      "activations/layer3_attention_weight_max": 92.47520446777344,
      "activations/layer3_attention_weight_min": -91.73262786865234,
      "activations/layer4_attention_weight_max": 96.16827392578125,
      "activations/layer4_attention_weight_min": -91.09957885742188,
      "activations/layer5_attention_weight_max": 71.52596282958984,
      "activations/layer5_attention_weight_min": -77.37684631347656,
      "activations/layer6_attention_weight_max": 54.22089385986328,
      "activations/layer6_attention_weight_min": -56.521446228027344,
      "activations/layer7_attention_weight_max": 70.7366943359375,
      "activations/layer7_attention_weight_min": -68.70245361328125,
      "activations/layer8_attention_weight_max": 46.2101936340332,
      "activations/layer8_attention_weight_min": -46.583740234375,
      "activations/layer9_attention_weight_max": 45.98465347290039,
      "activations/layer9_attention_weight_min": -47.437782287597656,
      "epoch": 12.58,
      "learning_rate": 6.955795454545453e-05,
      "loss": 2.7841,
      "step": 216550
    },
    {
      "activations/layer0_attention_weight_max": 16.080768585205078,
      "activations/layer0_attention_weight_min": -14.249895095825195,
      "activations/layer10_attention_weight_max": 34.89311981201172,
      "activations/layer10_attention_weight_min": -34.903324127197266,
      "activations/layer11_attention_weight_max": 30.85426902770996,
      "activations/layer11_attention_weight_min": -31.542648315429688,
      "activations/layer12_attention_weight_max": 17.579788208007812,
      "activations/layer12_attention_weight_min": -24.885961532592773,
      "activations/layer13_attention_weight_max": 37.15877151489258,
      "activations/layer13_attention_weight_min": -27.27669334411621,
      "activations/layer14_attention_weight_max": 35.57742691040039,
      "activations/layer14_attention_weight_min": -29.182126998901367,
      "activations/layer15_attention_weight_max": 32.27672576904297,
      "activations/layer15_attention_weight_min": -29.802202224731445,
      "activations/layer16_attention_weight_max": 30.371936798095703,
      "activations/layer16_attention_weight_min": -30.577449798583984,
      "activations/layer17_attention_weight_max": 49.64805603027344,
      "activations/layer17_attention_weight_min": -41.83436584472656,
      "activations/layer18_attention_weight_max": 42.49750518798828,
      "activations/layer18_attention_weight_min": -37.8496208190918,
      "activations/layer19_attention_weight_max": 22.927371978759766,
      "activations/layer19_attention_weight_min": -21.430557250976562,
      "activations/layer1_attention_weight_max": 16.088937759399414,
      "activations/layer1_attention_weight_min": -17.543684005737305,
      "activations/layer20_attention_weight_max": 23.6626033782959,
      "activations/layer20_attention_weight_min": -21.675342559814453,
      "activations/layer21_attention_weight_max": 35.085323333740234,
      "activations/layer21_attention_weight_min": -22.295019149780273,
      "activations/layer22_attention_weight_max": 32.14912033081055,
      "activations/layer22_attention_weight_min": -28.067739486694336,
      "activations/layer23_attention_weight_max": 35.42349624633789,
      "activations/layer23_attention_weight_min": -24.287992477416992,
      "activations/layer2_attention_weight_max": 31.24526023864746,
      "activations/layer2_attention_weight_min": -30.949460983276367,
      "activations/layer3_attention_weight_max": 95.859619140625,
      "activations/layer3_attention_weight_min": -99.33760070800781,
      "activations/layer4_attention_weight_max": 93.80396270751953,
      "activations/layer4_attention_weight_min": -92.46135711669922,
      "activations/layer5_attention_weight_max": 69.11038208007812,
      "activations/layer5_attention_weight_min": -78.61157989501953,
      "activations/layer6_attention_weight_max": 53.40782928466797,
      "activations/layer6_attention_weight_min": -52.19191360473633,
      "activations/layer7_attention_weight_max": 67.59362030029297,
      "activations/layer7_attention_weight_min": -68.7166519165039,
      "activations/layer8_attention_weight_max": 44.226043701171875,
      "activations/layer8_attention_weight_min": -46.4744758605957,
      "activations/layer9_attention_weight_max": 45.35711669921875,
      "activations/layer9_attention_weight_min": -47.162471771240234,
      "epoch": 12.59,
      "learning_rate": 6.953901515151515e-05,
      "loss": 2.7676,
      "step": 216600
    },
    {
      "activations/layer0_attention_weight_max": 16.676921844482422,
      "activations/layer0_attention_weight_min": -14.124737739562988,
      "activations/layer10_attention_weight_max": 34.97532272338867,
      "activations/layer10_attention_weight_min": -33.908355712890625,
      "activations/layer11_attention_weight_max": 29.944684982299805,
      "activations/layer11_attention_weight_min": -30.653629302978516,
      "activations/layer12_attention_weight_max": 19.80333137512207,
      "activations/layer12_attention_weight_min": -21.534992218017578,
      "activations/layer13_attention_weight_max": 35.223052978515625,
      "activations/layer13_attention_weight_min": -27.52953338623047,
      "activations/layer14_attention_weight_max": 35.9697151184082,
      "activations/layer14_attention_weight_min": -29.69495391845703,
      "activations/layer15_attention_weight_max": 32.90530014038086,
      "activations/layer15_attention_weight_min": -27.786603927612305,
      "activations/layer16_attention_weight_max": 31.63789939880371,
      "activations/layer16_attention_weight_min": -29.454782485961914,
      "activations/layer17_attention_weight_max": 50.82468795776367,
      "activations/layer17_attention_weight_min": -43.46737289428711,
      "activations/layer18_attention_weight_max": 46.98875045776367,
      "activations/layer18_attention_weight_min": -38.00080108642578,
      "activations/layer19_attention_weight_max": 24.539936065673828,
      "activations/layer19_attention_weight_min": -22.556591033935547,
      "activations/layer1_attention_weight_max": 16.154239654541016,
      "activations/layer1_attention_weight_min": -14.998634338378906,
      "activations/layer20_attention_weight_max": 23.396793365478516,
      "activations/layer20_attention_weight_min": -24.602439880371094,
      "activations/layer21_attention_weight_max": 36.2001838684082,
      "activations/layer21_attention_weight_min": -22.93677520751953,
      "activations/layer22_attention_weight_max": 33.05387496948242,
      "activations/layer22_attention_weight_min": -25.81772232055664,
      "activations/layer23_attention_weight_max": 42.0137825012207,
      "activations/layer23_attention_weight_min": -28.090866088867188,
      "activations/layer2_attention_weight_max": 31.254323959350586,
      "activations/layer2_attention_weight_min": -29.074554443359375,
      "activations/layer3_attention_weight_max": 85.81008911132812,
      "activations/layer3_attention_weight_min": -87.10562133789062,
      "activations/layer4_attention_weight_max": 87.75575256347656,
      "activations/layer4_attention_weight_min": -87.96776580810547,
      "activations/layer5_attention_weight_max": 67.99932861328125,
      "activations/layer5_attention_weight_min": -84.03105926513672,
      "activations/layer6_attention_weight_max": 49.873172760009766,
      "activations/layer6_attention_weight_min": -50.3811149597168,
      "activations/layer7_attention_weight_max": 67.70401000976562,
      "activations/layer7_attention_weight_min": -64.01969909667969,
      "activations/layer8_attention_weight_max": 41.52024459838867,
      "activations/layer8_attention_weight_min": -44.8090705871582,
      "activations/layer9_attention_weight_max": 44.76050567626953,
      "activations/layer9_attention_weight_min": -48.72007751464844,
      "epoch": 12.59,
      "learning_rate": 6.952007575757575e-05,
      "loss": 2.7693,
      "step": 216650
    },
    {
      "activations/layer0_attention_weight_max": 15.562996864318848,
      "activations/layer0_attention_weight_min": -14.373403549194336,
      "activations/layer10_attention_weight_max": 37.41910934448242,
      "activations/layer10_attention_weight_min": -37.008609771728516,
      "activations/layer11_attention_weight_max": 32.510719299316406,
      "activations/layer11_attention_weight_min": -34.61552429199219,
      "activations/layer12_attention_weight_max": 18.367704391479492,
      "activations/layer12_attention_weight_min": -23.254167556762695,
      "activations/layer13_attention_weight_max": 39.340492248535156,
      "activations/layer13_attention_weight_min": -32.42539596557617,
      "activations/layer14_attention_weight_max": 35.24174499511719,
      "activations/layer14_attention_weight_min": -28.63633918762207,
      "activations/layer15_attention_weight_max": 30.625478744506836,
      "activations/layer15_attention_weight_min": -28.98368263244629,
      "activations/layer16_attention_weight_max": 30.991077423095703,
      "activations/layer16_attention_weight_min": -30.210359573364258,
      "activations/layer17_attention_weight_max": 53.01277542114258,
      "activations/layer17_attention_weight_min": -43.25901412963867,
      "activations/layer18_attention_weight_max": 48.513858795166016,
      "activations/layer18_attention_weight_min": -37.2187385559082,
      "activations/layer19_attention_weight_max": 21.11623764038086,
      "activations/layer19_attention_weight_min": -21.575382232666016,
      "activations/layer1_attention_weight_max": 17.388935089111328,
      "activations/layer1_attention_weight_min": -15.311830520629883,
      "activations/layer20_attention_weight_max": 20.678152084350586,
      "activations/layer20_attention_weight_min": -20.594202041625977,
      "activations/layer21_attention_weight_max": 31.398983001708984,
      "activations/layer21_attention_weight_min": -21.689651489257812,
      "activations/layer22_attention_weight_max": 33.58592224121094,
      "activations/layer22_attention_weight_min": -26.419282913208008,
      "activations/layer23_attention_weight_max": 37.402793884277344,
      "activations/layer23_attention_weight_min": -24.580997467041016,
      "activations/layer2_attention_weight_max": 32.37993621826172,
      "activations/layer2_attention_weight_min": -29.56375503540039,
      "activations/layer3_attention_weight_max": 86.62041473388672,
      "activations/layer3_attention_weight_min": -90.11908721923828,
      "activations/layer4_attention_weight_max": 89.78202819824219,
      "activations/layer4_attention_weight_min": -88.40597534179688,
      "activations/layer5_attention_weight_max": 70.22736358642578,
      "activations/layer5_attention_weight_min": -76.52046203613281,
      "activations/layer6_attention_weight_max": 50.85420608520508,
      "activations/layer6_attention_weight_min": -50.63279724121094,
      "activations/layer7_attention_weight_max": 68.63712310791016,
      "activations/layer7_attention_weight_min": -67.19080352783203,
      "activations/layer8_attention_weight_max": 43.797054290771484,
      "activations/layer8_attention_weight_min": -46.43203353881836,
      "activations/layer9_attention_weight_max": 45.44508361816406,
      "activations/layer9_attention_weight_min": -45.812679290771484,
      "epoch": 12.59,
      "learning_rate": 6.950113636363635e-05,
      "loss": 2.7717,
      "step": 216700
    },
    {
      "activations/layer0_attention_weight_max": 15.705842971801758,
      "activations/layer0_attention_weight_min": -14.101337432861328,
      "activations/layer10_attention_weight_max": 34.95143508911133,
      "activations/layer10_attention_weight_min": -35.47832489013672,
      "activations/layer11_attention_weight_max": 31.4119873046875,
      "activations/layer11_attention_weight_min": -34.16242599487305,
      "activations/layer12_attention_weight_max": 18.620267868041992,
      "activations/layer12_attention_weight_min": -25.1092529296875,
      "activations/layer13_attention_weight_max": 35.82182312011719,
      "activations/layer13_attention_weight_min": -29.484533309936523,
      "activations/layer14_attention_weight_max": 38.64834213256836,
      "activations/layer14_attention_weight_min": -30.124406814575195,
      "activations/layer15_attention_weight_max": 36.339229583740234,
      "activations/layer15_attention_weight_min": -30.566556930541992,
      "activations/layer16_attention_weight_max": 33.592044830322266,
      "activations/layer16_attention_weight_min": -30.88958740234375,
      "activations/layer17_attention_weight_max": 53.9810676574707,
      "activations/layer17_attention_weight_min": -47.356651306152344,
      "activations/layer18_attention_weight_max": 50.29877853393555,
      "activations/layer18_attention_weight_min": -40.026554107666016,
      "activations/layer19_attention_weight_max": 24.469324111938477,
      "activations/layer19_attention_weight_min": -22.128576278686523,
      "activations/layer1_attention_weight_max": 16.206832885742188,
      "activations/layer1_attention_weight_min": -14.182598114013672,
      "activations/layer20_attention_weight_max": 23.36741065979004,
      "activations/layer20_attention_weight_min": -20.789146423339844,
      "activations/layer21_attention_weight_max": 39.23491287231445,
      "activations/layer21_attention_weight_min": -22.2075138092041,
      "activations/layer22_attention_weight_max": 31.434995651245117,
      "activations/layer22_attention_weight_min": -25.74790382385254,
      "activations/layer23_attention_weight_max": 37.14434051513672,
      "activations/layer23_attention_weight_min": -25.361305236816406,
      "activations/layer2_attention_weight_max": 33.66667175292969,
      "activations/layer2_attention_weight_min": -31.499242782592773,
      "activations/layer3_attention_weight_max": 91.1620101928711,
      "activations/layer3_attention_weight_min": -98.62090301513672,
      "activations/layer4_attention_weight_max": 95.05024719238281,
      "activations/layer4_attention_weight_min": -91.02687072753906,
      "activations/layer5_attention_weight_max": 71.55966186523438,
      "activations/layer5_attention_weight_min": -79.49756622314453,
      "activations/layer6_attention_weight_max": 53.8988151550293,
      "activations/layer6_attention_weight_min": -55.178810119628906,
      "activations/layer7_attention_weight_max": 68.83554077148438,
      "activations/layer7_attention_weight_min": -68.07542419433594,
      "activations/layer8_attention_weight_max": 47.07692337036133,
      "activations/layer8_attention_weight_min": -52.52449417114258,
      "activations/layer9_attention_weight_max": 45.94048309326172,
      "activations/layer9_attention_weight_min": -47.71013641357422,
      "epoch": 12.59,
      "learning_rate": 6.948219696969696e-05,
      "loss": 2.7737,
      "step": 216750
    },
    {
      "activations/layer0_attention_weight_max": 15.597041130065918,
      "activations/layer0_attention_weight_min": -13.850438117980957,
      "activations/layer10_attention_weight_max": 35.89188003540039,
      "activations/layer10_attention_weight_min": -35.40244674682617,
      "activations/layer11_attention_weight_max": 32.7359619140625,
      "activations/layer11_attention_weight_min": -33.28450393676758,
      "activations/layer12_attention_weight_max": 18.814937591552734,
      "activations/layer12_attention_weight_min": -25.247909545898438,
      "activations/layer13_attention_weight_max": 40.60186767578125,
      "activations/layer13_attention_weight_min": -30.131380081176758,
      "activations/layer14_attention_weight_max": 41.49909210205078,
      "activations/layer14_attention_weight_min": -33.86177444458008,
      "activations/layer15_attention_weight_max": 40.619903564453125,
      "activations/layer15_attention_weight_min": -31.397661209106445,
      "activations/layer16_attention_weight_max": 32.588932037353516,
      "activations/layer16_attention_weight_min": -29.94428825378418,
      "activations/layer17_attention_weight_max": 51.99851989746094,
      "activations/layer17_attention_weight_min": -46.430992126464844,
      "activations/layer18_attention_weight_max": 46.49296569824219,
      "activations/layer18_attention_weight_min": -40.19116973876953,
      "activations/layer19_attention_weight_max": 23.08790397644043,
      "activations/layer19_attention_weight_min": -21.832359313964844,
      "activations/layer1_attention_weight_max": 17.919679641723633,
      "activations/layer1_attention_weight_min": -14.202576637268066,
      "activations/layer20_attention_weight_max": 24.60595703125,
      "activations/layer20_attention_weight_min": -20.425168991088867,
      "activations/layer21_attention_weight_max": 44.43919372558594,
      "activations/layer21_attention_weight_min": -25.866500854492188,
      "activations/layer22_attention_weight_max": 31.136796951293945,
      "activations/layer22_attention_weight_min": -25.729726791381836,
      "activations/layer23_attention_weight_max": 38.123531341552734,
      "activations/layer23_attention_weight_min": -23.020315170288086,
      "activations/layer2_attention_weight_max": 34.004234313964844,
      "activations/layer2_attention_weight_min": -32.07057571411133,
      "activations/layer3_attention_weight_max": 92.43539428710938,
      "activations/layer3_attention_weight_min": -92.01813507080078,
      "activations/layer4_attention_weight_max": 100.03907775878906,
      "activations/layer4_attention_weight_min": -92.9286117553711,
      "activations/layer5_attention_weight_max": 74.30731964111328,
      "activations/layer5_attention_weight_min": -76.70172119140625,
      "activations/layer6_attention_weight_max": 55.37702560424805,
      "activations/layer6_attention_weight_min": -53.50936508178711,
      "activations/layer7_attention_weight_max": 71.74468994140625,
      "activations/layer7_attention_weight_min": -70.66444396972656,
      "activations/layer8_attention_weight_max": 47.00810241699219,
      "activations/layer8_attention_weight_min": -47.8611946105957,
      "activations/layer9_attention_weight_max": 48.99710464477539,
      "activations/layer9_attention_weight_min": -48.828155517578125,
      "epoch": 12.6,
      "learning_rate": 6.946363636363636e-05,
      "loss": 2.7832,
      "step": 216800
    },
    {
      "activations/layer0_attention_weight_max": 16.30170440673828,
      "activations/layer0_attention_weight_min": -14.043067932128906,
      "activations/layer10_attention_weight_max": 39.573272705078125,
      "activations/layer10_attention_weight_min": -36.91533279418945,
      "activations/layer11_attention_weight_max": 34.298858642578125,
      "activations/layer11_attention_weight_min": -33.847511291503906,
      "activations/layer12_attention_weight_max": 19.066131591796875,
      "activations/layer12_attention_weight_min": -26.1328182220459,
      "activations/layer13_attention_weight_max": 40.745784759521484,
      "activations/layer13_attention_weight_min": -30.04034423828125,
      "activations/layer14_attention_weight_max": 42.40101623535156,
      "activations/layer14_attention_weight_min": -29.5527286529541,
      "activations/layer15_attention_weight_max": 34.315555572509766,
      "activations/layer15_attention_weight_min": -29.251161575317383,
      "activations/layer16_attention_weight_max": 33.83315658569336,
      "activations/layer16_attention_weight_min": -31.359676361083984,
      "activations/layer17_attention_weight_max": 57.72742462158203,
      "activations/layer17_attention_weight_min": -50.65484619140625,
      "activations/layer18_attention_weight_max": 51.33321762084961,
      "activations/layer18_attention_weight_min": -41.29301452636719,
      "activations/layer19_attention_weight_max": 22.206031799316406,
      "activations/layer19_attention_weight_min": -22.03522491455078,
      "activations/layer1_attention_weight_max": 17.58821678161621,
      "activations/layer1_attention_weight_min": -15.688294410705566,
      "activations/layer20_attention_weight_max": 20.829553604125977,
      "activations/layer20_attention_weight_min": -22.15398597717285,
      "activations/layer21_attention_weight_max": 33.880950927734375,
      "activations/layer21_attention_weight_min": -24.819578170776367,
      "activations/layer22_attention_weight_max": 29.373228073120117,
      "activations/layer22_attention_weight_min": -24.83041000366211,
      "activations/layer23_attention_weight_max": 38.6060791015625,
      "activations/layer23_attention_weight_min": -24.1893310546875,
      "activations/layer2_attention_weight_max": 32.080169677734375,
      "activations/layer2_attention_weight_min": -30.591054916381836,
      "activations/layer3_attention_weight_max": 93.43875885009766,
      "activations/layer3_attention_weight_min": -91.75081634521484,
      "activations/layer4_attention_weight_max": 93.70476531982422,
      "activations/layer4_attention_weight_min": -90.30842590332031,
      "activations/layer5_attention_weight_max": 70.09368896484375,
      "activations/layer5_attention_weight_min": -77.66616821289062,
      "activations/layer6_attention_weight_max": 55.61576461791992,
      "activations/layer6_attention_weight_min": -55.11956024169922,
      "activations/layer7_attention_weight_max": 71.10745239257812,
      "activations/layer7_attention_weight_min": -69.46029663085938,
      "activations/layer8_attention_weight_max": 45.98838424682617,
      "activations/layer8_attention_weight_min": -47.3427619934082,
      "activations/layer9_attention_weight_max": 56.10408401489258,
      "activations/layer9_attention_weight_min": -47.38975524902344,
      "epoch": 12.6,
      "learning_rate": 6.944469696969697e-05,
      "loss": 2.7592,
      "step": 216850
    },
    {
      "activations/layer0_attention_weight_max": 15.794175148010254,
      "activations/layer0_attention_weight_min": -14.088724136352539,
      "activations/layer10_attention_weight_max": 33.60304260253906,
      "activations/layer10_attention_weight_min": -34.08868408203125,
      "activations/layer11_attention_weight_max": 30.903484344482422,
      "activations/layer11_attention_weight_min": -31.926801681518555,
      "activations/layer12_attention_weight_max": 17.580537796020508,
      "activations/layer12_attention_weight_min": -27.10512924194336,
      "activations/layer13_attention_weight_max": 41.0681037902832,
      "activations/layer13_attention_weight_min": -30.84444236755371,
      "activations/layer14_attention_weight_max": 37.143959045410156,
      "activations/layer14_attention_weight_min": -31.274377822875977,
      "activations/layer15_attention_weight_max": 31.713253021240234,
      "activations/layer15_attention_weight_min": -29.55059814453125,
      "activations/layer16_attention_weight_max": 30.63368797302246,
      "activations/layer16_attention_weight_min": -30.976099014282227,
      "activations/layer17_attention_weight_max": 48.69145584106445,
      "activations/layer17_attention_weight_min": -46.65987777709961,
      "activations/layer18_attention_weight_max": 46.764530181884766,
      "activations/layer18_attention_weight_min": -38.46405792236328,
      "activations/layer19_attention_weight_max": 20.486894607543945,
      "activations/layer19_attention_weight_min": -22.215436935424805,
      "activations/layer1_attention_weight_max": 16.571279525756836,
      "activations/layer1_attention_weight_min": -15.252598762512207,
      "activations/layer20_attention_weight_max": 20.94481086730957,
      "activations/layer20_attention_weight_min": -21.66106605529785,
      "activations/layer21_attention_weight_max": 36.379539489746094,
      "activations/layer21_attention_weight_min": -22.84883689880371,
      "activations/layer22_attention_weight_max": 27.225543975830078,
      "activations/layer22_attention_weight_min": -26.213376998901367,
      "activations/layer23_attention_weight_max": 34.12321853637695,
      "activations/layer23_attention_weight_min": -22.11867332458496,
      "activations/layer2_attention_weight_max": 31.077041625976562,
      "activations/layer2_attention_weight_min": -29.58082389831543,
      "activations/layer3_attention_weight_max": 89.7553482055664,
      "activations/layer3_attention_weight_min": -89.67546844482422,
      "activations/layer4_attention_weight_max": 95.62776947021484,
      "activations/layer4_attention_weight_min": -89.71499633789062,
      "activations/layer5_attention_weight_max": 74.97235870361328,
      "activations/layer5_attention_weight_min": -74.82200622558594,
      "activations/layer6_attention_weight_max": 53.256004333496094,
      "activations/layer6_attention_weight_min": -52.743614196777344,
      "activations/layer7_attention_weight_max": 69.62528991699219,
      "activations/layer7_attention_weight_min": -68.17623901367188,
      "activations/layer8_attention_weight_max": 44.16852569580078,
      "activations/layer8_attention_weight_min": -44.98971176147461,
      "activations/layer9_attention_weight_max": 46.702693939208984,
      "activations/layer9_attention_weight_min": -46.95122146606445,
      "epoch": 12.6,
      "learning_rate": 6.942575757575757e-05,
      "loss": 2.7607,
      "step": 216900
    },
    {
      "activations/layer0_attention_weight_max": 15.804461479187012,
      "activations/layer0_attention_weight_min": -13.55154800415039,
      "activations/layer10_attention_weight_max": 40.990623474121094,
      "activations/layer10_attention_weight_min": -41.552467346191406,
      "activations/layer11_attention_weight_max": 37.5133056640625,
      "activations/layer11_attention_weight_min": -36.057838439941406,
      "activations/layer12_attention_weight_max": 18.666095733642578,
      "activations/layer12_attention_weight_min": -25.950883865356445,
      "activations/layer13_attention_weight_max": 38.16164779663086,
      "activations/layer13_attention_weight_min": -30.15048599243164,
      "activations/layer14_attention_weight_max": 43.4869499206543,
      "activations/layer14_attention_weight_min": -31.50320053100586,
      "activations/layer15_attention_weight_max": 35.6226806640625,
      "activations/layer15_attention_weight_min": -30.233312606811523,
      "activations/layer16_attention_weight_max": 31.116426467895508,
      "activations/layer16_attention_weight_min": -29.281009674072266,
      "activations/layer17_attention_weight_max": 49.047813415527344,
      "activations/layer17_attention_weight_min": -43.44272994995117,
      "activations/layer18_attention_weight_max": 42.54924774169922,
      "activations/layer18_attention_weight_min": -37.21255874633789,
      "activations/layer19_attention_weight_max": 22.022253036499023,
      "activations/layer19_attention_weight_min": -20.704395294189453,
      "activations/layer1_attention_weight_max": 16.66555404663086,
      "activations/layer1_attention_weight_min": -13.910506248474121,
      "activations/layer20_attention_weight_max": 23.581012725830078,
      "activations/layer20_attention_weight_min": -20.166629791259766,
      "activations/layer21_attention_weight_max": 32.71723937988281,
      "activations/layer21_attention_weight_min": -20.11094093322754,
      "activations/layer22_attention_weight_max": 29.37976837158203,
      "activations/layer22_attention_weight_min": -23.07140350341797,
      "activations/layer23_attention_weight_max": 31.55718994140625,
      "activations/layer23_attention_weight_min": -21.70606231689453,
      "activations/layer2_attention_weight_max": 33.56590270996094,
      "activations/layer2_attention_weight_min": -31.34895133972168,
      "activations/layer3_attention_weight_max": 95.31661224365234,
      "activations/layer3_attention_weight_min": -96.56993103027344,
      "activations/layer4_attention_weight_max": 101.03731536865234,
      "activations/layer4_attention_weight_min": -93.96734619140625,
      "activations/layer5_attention_weight_max": 70.96723937988281,
      "activations/layer5_attention_weight_min": -79.12928771972656,
      "activations/layer6_attention_weight_max": 53.19822692871094,
      "activations/layer6_attention_weight_min": -55.22711181640625,
      "activations/layer7_attention_weight_max": 75.44353485107422,
      "activations/layer7_attention_weight_min": -66.41780090332031,
      "activations/layer8_attention_weight_max": 46.77714920043945,
      "activations/layer8_attention_weight_min": -47.526241302490234,
      "activations/layer9_attention_weight_max": 60.777137756347656,
      "activations/layer9_attention_weight_min": -47.26437759399414,
      "epoch": 12.61,
      "learning_rate": 6.940681818181817e-05,
      "loss": 2.754,
      "step": 216950
    },
    {
      "activations/layer0_attention_weight_max": 16.093429565429688,
      "activations/layer0_attention_weight_min": -13.796360969543457,
      "activations/layer10_attention_weight_max": 43.1350212097168,
      "activations/layer10_attention_weight_min": -39.579833984375,
      "activations/layer11_attention_weight_max": 36.64537811279297,
      "activations/layer11_attention_weight_min": -34.8613395690918,
      "activations/layer12_attention_weight_max": 20.51011085510254,
      "activations/layer12_attention_weight_min": -22.293094635009766,
      "activations/layer13_attention_weight_max": 44.83439636230469,
      "activations/layer13_attention_weight_min": -31.244190216064453,
      "activations/layer14_attention_weight_max": 42.79443359375,
      "activations/layer14_attention_weight_min": -32.42029571533203,
      "activations/layer15_attention_weight_max": 39.60618591308594,
      "activations/layer15_attention_weight_min": -30.265426635742188,
      "activations/layer16_attention_weight_max": 37.93716812133789,
      "activations/layer16_attention_weight_min": -33.922523498535156,
      "activations/layer17_attention_weight_max": 65.75690460205078,
      "activations/layer17_attention_weight_min": -52.253299713134766,
      "activations/layer18_attention_weight_max": 60.851806640625,
      "activations/layer18_attention_weight_min": -46.30533218383789,
      "activations/layer19_attention_weight_max": 28.579870223999023,
      "activations/layer19_attention_weight_min": -24.83597183227539,
      "activations/layer1_attention_weight_max": 16.538930892944336,
      "activations/layer1_attention_weight_min": -14.52938461303711,
      "activations/layer20_attention_weight_max": 31.0050106048584,
      "activations/layer20_attention_weight_min": -21.281240463256836,
      "activations/layer21_attention_weight_max": 46.91337585449219,
      "activations/layer21_attention_weight_min": -24.59905242919922,
      "activations/layer22_attention_weight_max": 36.495445251464844,
      "activations/layer22_attention_weight_min": -25.758163452148438,
      "activations/layer23_attention_weight_max": 39.69767761230469,
      "activations/layer23_attention_weight_min": -22.43604278564453,
      "activations/layer2_attention_weight_max": 32.35271453857422,
      "activations/layer2_attention_weight_min": -31.367084503173828,
      "activations/layer3_attention_weight_max": 100.36912536621094,
      "activations/layer3_attention_weight_min": -98.30718994140625,
      "activations/layer4_attention_weight_max": 104.029052734375,
      "activations/layer4_attention_weight_min": -97.03490447998047,
      "activations/layer5_attention_weight_max": 72.70205688476562,
      "activations/layer5_attention_weight_min": -76.2008056640625,
      "activations/layer6_attention_weight_max": 54.83509063720703,
      "activations/layer6_attention_weight_min": -57.589088439941406,
      "activations/layer7_attention_weight_max": 73.82101440429688,
      "activations/layer7_attention_weight_min": -69.73738861083984,
      "activations/layer8_attention_weight_max": 50.41545104980469,
      "activations/layer8_attention_weight_min": -49.60624694824219,
      "activations/layer9_attention_weight_max": 51.208595275878906,
      "activations/layer9_attention_weight_min": -49.444332122802734,
      "epoch": 12.61,
      "learning_rate": 6.938787878787879e-05,
      "loss": 2.7713,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4719,
      "eval_samples_per_second": 506.85,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4719,
      "eval_openwebtext_samples_per_second": 506.85,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_wikitext_loss": 2.958984375,
      "eval_wikitext_ppl": 19.278382202442366,
      "eval_wikitext_runtime": 1.9684,
      "eval_wikitext_samples_per_second": 231.661,
      "step": 217000
    },
    {
      "epoch": 12.61,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.5182,
      "eval_lambada_samples_per_second": 511.544,
      "step": 217000
    },
    {
      "activations/layer0_attention_weight_max": 15.499996185302734,
      "activations/layer0_attention_weight_min": -13.646947860717773,
      "activations/layer10_attention_weight_max": 35.68380355834961,
      "activations/layer10_attention_weight_min": -35.113548278808594,
      "activations/layer11_attention_weight_max": 32.63320541381836,
      "activations/layer11_attention_weight_min": -32.77816390991211,
      "activations/layer12_attention_weight_max": 16.6696720123291,
      "activations/layer12_attention_weight_min": -26.836990356445312,
      "activations/layer13_attention_weight_max": 36.420833587646484,
      "activations/layer13_attention_weight_min": -28.81810760498047,
      "activations/layer14_attention_weight_max": 34.98030090332031,
      "activations/layer14_attention_weight_min": -29.94032859802246,
      "activations/layer15_attention_weight_max": 32.066612243652344,
      "activations/layer15_attention_weight_min": -27.97464370727539,
      "activations/layer16_attention_weight_max": 30.462072372436523,
      "activations/layer16_attention_weight_min": -30.333467483520508,
      "activations/layer17_attention_weight_max": 50.772823333740234,
      "activations/layer17_attention_weight_min": -45.402320861816406,
      "activations/layer18_attention_weight_max": 43.2645263671875,
      "activations/layer18_attention_weight_min": -36.6123161315918,
      "activations/layer19_attention_weight_max": 21.57402801513672,
      "activations/layer19_attention_weight_min": -21.967052459716797,
      "activations/layer1_attention_weight_max": 16.8804931640625,
      "activations/layer1_attention_weight_min": -14.649064064025879,
      "activations/layer20_attention_weight_max": 20.949260711669922,
      "activations/layer20_attention_weight_min": -23.816322326660156,
      "activations/layer21_attention_weight_max": 33.763038635253906,
      "activations/layer21_attention_weight_min": -21.574146270751953,
      "activations/layer22_attention_weight_max": 26.91692543029785,
      "activations/layer22_attention_weight_min": -26.61065101623535,
      "activations/layer23_attention_weight_max": 35.276634216308594,
      "activations/layer23_attention_weight_min": -26.253067016601562,
      "activations/layer2_attention_weight_max": 30.67242431640625,
      "activations/layer2_attention_weight_min": -30.32073211669922,
      "activations/layer3_attention_weight_max": 89.45687103271484,
      "activations/layer3_attention_weight_min": -92.65565490722656,
      "activations/layer4_attention_weight_max": 90.71124267578125,
      "activations/layer4_attention_weight_min": -91.05139923095703,
      "activations/layer5_attention_weight_max": 68.71307373046875,
      "activations/layer5_attention_weight_min": -74.31065368652344,
      "activations/layer6_attention_weight_max": 53.01081085205078,
      "activations/layer6_attention_weight_min": -50.734127044677734,
      "activations/layer7_attention_weight_max": 68.3541488647461,
      "activations/layer7_attention_weight_min": -62.5280876159668,
      "activations/layer8_attention_weight_max": 47.61418914794922,
      "activations/layer8_attention_weight_min": -46.090126037597656,
      "activations/layer9_attention_weight_max": 46.40304183959961,
      "activations/layer9_attention_weight_min": -44.04738998413086,
      "epoch": 12.61,
      "learning_rate": 6.936893939393939e-05,
      "loss": 2.7693,
      "step": 217050
    },
    {
      "activations/layer0_attention_weight_max": 15.174217224121094,
      "activations/layer0_attention_weight_min": -14.367185592651367,
      "activations/layer10_attention_weight_max": 36.320274353027344,
      "activations/layer10_attention_weight_min": -37.07428741455078,
      "activations/layer11_attention_weight_max": 31.747610092163086,
      "activations/layer11_attention_weight_min": -32.43687438964844,
      "activations/layer12_attention_weight_max": 17.873279571533203,
      "activations/layer12_attention_weight_min": -22.3845272064209,
      "activations/layer13_attention_weight_max": 33.84041213989258,
      "activations/layer13_attention_weight_min": -29.608360290527344,
      "activations/layer14_attention_weight_max": 40.320796966552734,
      "activations/layer14_attention_weight_min": -32.215126037597656,
      "activations/layer15_attention_weight_max": 33.605281829833984,
      "activations/layer15_attention_weight_min": -30.703641891479492,
      "activations/layer16_attention_weight_max": 31.02775001525879,
      "activations/layer16_attention_weight_min": -30.243913650512695,
      "activations/layer17_attention_weight_max": 54.21169662475586,
      "activations/layer17_attention_weight_min": -42.414241790771484,
      "activations/layer18_attention_weight_max": 50.538761138916016,
      "activations/layer18_attention_weight_min": -37.47919845581055,
      "activations/layer19_attention_weight_max": 23.647722244262695,
      "activations/layer19_attention_weight_min": -21.754491806030273,
      "activations/layer1_attention_weight_max": 15.631295204162598,
      "activations/layer1_attention_weight_min": -14.439778327941895,
      "activations/layer20_attention_weight_max": 21.936288833618164,
      "activations/layer20_attention_weight_min": -24.5753116607666,
      "activations/layer21_attention_weight_max": 38.67747116088867,
      "activations/layer21_attention_weight_min": -22.801319122314453,
      "activations/layer22_attention_weight_max": 29.850690841674805,
      "activations/layer22_attention_weight_min": -27.16942596435547,
      "activations/layer23_attention_weight_max": 37.803977966308594,
      "activations/layer23_attention_weight_min": -24.744449615478516,
      "activations/layer2_attention_weight_max": 31.34273910522461,
      "activations/layer2_attention_weight_min": -31.00983428955078,
      "activations/layer3_attention_weight_max": 92.07816314697266,
      "activations/layer3_attention_weight_min": -95.60018157958984,
      "activations/layer4_attention_weight_max": 100.662109375,
      "activations/layer4_attention_weight_min": -95.82715606689453,
      "activations/layer5_attention_weight_max": 71.10704803466797,
      "activations/layer5_attention_weight_min": -78.69454193115234,
      "activations/layer6_attention_weight_max": 54.794986724853516,
      "activations/layer6_attention_weight_min": -52.94612121582031,
      "activations/layer7_attention_weight_max": 74.07184600830078,
      "activations/layer7_attention_weight_min": -69.09263610839844,
      "activations/layer8_attention_weight_max": 47.288002014160156,
      "activations/layer8_attention_weight_min": -50.12076187133789,
      "activations/layer9_attention_weight_max": 50.06108093261719,
      "activations/layer9_attention_weight_min": -47.57438278198242,
      "epoch": 12.61,
      "learning_rate": 6.934999999999999e-05,
      "loss": 2.7749,
      "step": 217100
    },
    {
      "activations/layer0_attention_weight_max": 16.254600524902344,
      "activations/layer0_attention_weight_min": -13.921675682067871,
      "activations/layer10_attention_weight_max": 36.72925567626953,
      "activations/layer10_attention_weight_min": -33.78752899169922,
      "activations/layer11_attention_weight_max": 31.88525390625,
      "activations/layer11_attention_weight_min": -31.75349235534668,
      "activations/layer12_attention_weight_max": 19.16874885559082,
      "activations/layer12_attention_weight_min": -27.764352798461914,
      "activations/layer13_attention_weight_max": 36.70966720581055,
      "activations/layer13_attention_weight_min": -25.63532829284668,
      "activations/layer14_attention_weight_max": 33.36515426635742,
      "activations/layer14_attention_weight_min": -28.793752670288086,
      "activations/layer15_attention_weight_max": 31.577463150024414,
      "activations/layer15_attention_weight_min": -27.723228454589844,
      "activations/layer16_attention_weight_max": 29.78746795654297,
      "activations/layer16_attention_weight_min": -28.78034210205078,
      "activations/layer17_attention_weight_max": 47.26293182373047,
      "activations/layer17_attention_weight_min": -43.3759651184082,
      "activations/layer18_attention_weight_max": 44.19301986694336,
      "activations/layer18_attention_weight_min": -38.58866500854492,
      "activations/layer19_attention_weight_max": 22.876737594604492,
      "activations/layer19_attention_weight_min": -22.790206909179688,
      "activations/layer1_attention_weight_max": 17.47968292236328,
      "activations/layer1_attention_weight_min": -14.279659271240234,
      "activations/layer20_attention_weight_max": 19.879472732543945,
      "activations/layer20_attention_weight_min": -22.722383499145508,
      "activations/layer21_attention_weight_max": 33.59843826293945,
      "activations/layer21_attention_weight_min": -20.824329376220703,
      "activations/layer22_attention_weight_max": 29.35744285583496,
      "activations/layer22_attention_weight_min": -27.54876708984375,
      "activations/layer23_attention_weight_max": 36.136417388916016,
      "activations/layer23_attention_weight_min": -24.90786361694336,
      "activations/layer2_attention_weight_max": 31.37030029296875,
      "activations/layer2_attention_weight_min": -29.271568298339844,
      "activations/layer3_attention_weight_max": 90.8507080078125,
      "activations/layer3_attention_weight_min": -88.90550994873047,
      "activations/layer4_attention_weight_max": 93.69747161865234,
      "activations/layer4_attention_weight_min": -85.6808853149414,
      "activations/layer5_attention_weight_max": 67.76380157470703,
      "activations/layer5_attention_weight_min": -73.4183349609375,
      "activations/layer6_attention_weight_max": 49.65754699707031,
      "activations/layer6_attention_weight_min": -49.608314514160156,
      "activations/layer7_attention_weight_max": 66.00894165039062,
      "activations/layer7_attention_weight_min": -64.31218719482422,
      "activations/layer8_attention_weight_max": 42.151817321777344,
      "activations/layer8_attention_weight_min": -43.916351318359375,
      "activations/layer9_attention_weight_max": 45.67011642456055,
      "activations/layer9_attention_weight_min": -45.77180480957031,
      "epoch": 12.62,
      "learning_rate": 6.93310606060606e-05,
      "loss": 2.7828,
      "step": 217150
    },
    {
      "activations/layer0_attention_weight_max": 16.13863182067871,
      "activations/layer0_attention_weight_min": -15.390890121459961,
      "activations/layer10_attention_weight_max": 36.06224822998047,
      "activations/layer10_attention_weight_min": -36.61347198486328,
      "activations/layer11_attention_weight_max": 32.66773223876953,
      "activations/layer11_attention_weight_min": -33.23100280761719,
      "activations/layer12_attention_weight_max": 22.741716384887695,
      "activations/layer12_attention_weight_min": -24.99608612060547,
      "activations/layer13_attention_weight_max": 33.02284622192383,
      "activations/layer13_attention_weight_min": -29.20133399963379,
      "activations/layer14_attention_weight_max": 34.28407669067383,
      "activations/layer14_attention_weight_min": -30.923898696899414,
      "activations/layer15_attention_weight_max": 31.690147399902344,
      "activations/layer15_attention_weight_min": -27.445980072021484,
      "activations/layer16_attention_weight_max": 30.422073364257812,
      "activations/layer16_attention_weight_min": -29.155275344848633,
      "activations/layer17_attention_weight_max": 50.671390533447266,
      "activations/layer17_attention_weight_min": -43.425899505615234,
      "activations/layer18_attention_weight_max": 42.32114791870117,
      "activations/layer18_attention_weight_min": -38.35557556152344,
      "activations/layer19_attention_weight_max": 20.23745346069336,
      "activations/layer19_attention_weight_min": -21.021549224853516,
      "activations/layer1_attention_weight_max": 18.0650634765625,
      "activations/layer1_attention_weight_min": -17.855512619018555,
      "activations/layer20_attention_weight_max": 19.95773696899414,
      "activations/layer20_attention_weight_min": -21.22694969177246,
      "activations/layer21_attention_weight_max": 33.77602767944336,
      "activations/layer21_attention_weight_min": -21.955581665039062,
      "activations/layer22_attention_weight_max": 29.00153350830078,
      "activations/layer22_attention_weight_min": -25.20780372619629,
      "activations/layer23_attention_weight_max": 36.949440002441406,
      "activations/layer23_attention_weight_min": -23.342132568359375,
      "activations/layer2_attention_weight_max": 33.073524475097656,
      "activations/layer2_attention_weight_min": -31.145185470581055,
      "activations/layer3_attention_weight_max": 93.90451049804688,
      "activations/layer3_attention_weight_min": -95.38665771484375,
      "activations/layer4_attention_weight_max": 100.3124008178711,
      "activations/layer4_attention_weight_min": -94.42347717285156,
      "activations/layer5_attention_weight_max": 70.40965270996094,
      "activations/layer5_attention_weight_min": -82.05829620361328,
      "activations/layer6_attention_weight_max": 54.39077377319336,
      "activations/layer6_attention_weight_min": -53.13454818725586,
      "activations/layer7_attention_weight_max": 67.79683685302734,
      "activations/layer7_attention_weight_min": -64.9619140625,
      "activations/layer8_attention_weight_max": 45.03252410888672,
      "activations/layer8_attention_weight_min": -49.460147857666016,
      "activations/layer9_attention_weight_max": 45.74196243286133,
      "activations/layer9_attention_weight_min": -45.230079650878906,
      "epoch": 12.62,
      "learning_rate": 6.931212121212121e-05,
      "loss": 2.7768,
      "step": 217200
    },
    {
      "activations/layer0_attention_weight_max": 18.013036727905273,
      "activations/layer0_attention_weight_min": -13.995144844055176,
      "activations/layer10_attention_weight_max": 36.962764739990234,
      "activations/layer10_attention_weight_min": -36.9694938659668,
      "activations/layer11_attention_weight_max": 33.170162200927734,
      "activations/layer11_attention_weight_min": -35.14213180541992,
      "activations/layer12_attention_weight_max": 17.192270278930664,
      "activations/layer12_attention_weight_min": -29.311744689941406,
      "activations/layer13_attention_weight_max": 41.55025863647461,
      "activations/layer13_attention_weight_min": -33.982452392578125,
      "activations/layer14_attention_weight_max": 40.0110969543457,
      "activations/layer14_attention_weight_min": -31.4419002532959,
      "activations/layer15_attention_weight_max": 34.36846923828125,
      "activations/layer15_attention_weight_min": -31.309289932250977,
      "activations/layer16_attention_weight_max": 32.116363525390625,
      "activations/layer16_attention_weight_min": -29.567373275756836,
      "activations/layer17_attention_weight_max": 49.7577018737793,
      "activations/layer17_attention_weight_min": -42.47267532348633,
      "activations/layer18_attention_weight_max": 43.54424285888672,
      "activations/layer18_attention_weight_min": -38.67621612548828,
      "activations/layer19_attention_weight_max": 21.95796775817871,
      "activations/layer19_attention_weight_min": -21.443134307861328,
      "activations/layer1_attention_weight_max": 15.821674346923828,
      "activations/layer1_attention_weight_min": -14.418916702270508,
      "activations/layer20_attention_weight_max": 22.41937828063965,
      "activations/layer20_attention_weight_min": -21.329252243041992,
      "activations/layer21_attention_weight_max": 37.797340393066406,
      "activations/layer21_attention_weight_min": -23.089609146118164,
      "activations/layer22_attention_weight_max": 32.23283004760742,
      "activations/layer22_attention_weight_min": -25.35507583618164,
      "activations/layer23_attention_weight_max": 38.907386779785156,
      "activations/layer23_attention_weight_min": -23.935211181640625,
      "activations/layer2_attention_weight_max": 33.526885986328125,
      "activations/layer2_attention_weight_min": -30.105243682861328,
      "activations/layer3_attention_weight_max": 94.88693237304688,
      "activations/layer3_attention_weight_min": -96.90912628173828,
      "activations/layer4_attention_weight_max": 94.86290740966797,
      "activations/layer4_attention_weight_min": -93.73983001708984,
      "activations/layer5_attention_weight_max": 69.58891296386719,
      "activations/layer5_attention_weight_min": -75.6673583984375,
      "activations/layer6_attention_weight_max": 51.269718170166016,
      "activations/layer6_attention_weight_min": -50.81028747558594,
      "activations/layer7_attention_weight_max": 72.26264190673828,
      "activations/layer7_attention_weight_min": -71.06575012207031,
      "activations/layer8_attention_weight_max": 44.56088638305664,
      "activations/layer8_attention_weight_min": -45.56876754760742,
      "activations/layer9_attention_weight_max": 50.905738830566406,
      "activations/layer9_attention_weight_min": -51.96952438354492,
      "epoch": 12.62,
      "learning_rate": 6.929318181818181e-05,
      "loss": 2.7711,
      "step": 217250
    },
    {
      "activations/layer0_attention_weight_max": 16.342796325683594,
      "activations/layer0_attention_weight_min": -14.422856330871582,
      "activations/layer10_attention_weight_max": 38.54886245727539,
      "activations/layer10_attention_weight_min": -34.45797348022461,
      "activations/layer11_attention_weight_max": 32.64297103881836,
      "activations/layer11_attention_weight_min": -33.90757751464844,
      "activations/layer12_attention_weight_max": 16.699867248535156,
      "activations/layer12_attention_weight_min": -25.27083969116211,
      "activations/layer13_attention_weight_max": 32.10040283203125,
      "activations/layer13_attention_weight_min": -29.043733596801758,
      "activations/layer14_attention_weight_max": 32.61741638183594,
      "activations/layer14_attention_weight_min": -29.363191604614258,
      "activations/layer15_attention_weight_max": 29.158639907836914,
      "activations/layer15_attention_weight_min": -28.188547134399414,
      "activations/layer16_attention_weight_max": 27.296327590942383,
      "activations/layer16_attention_weight_min": -27.909242630004883,
      "activations/layer17_attention_weight_max": 45.90420150756836,
      "activations/layer17_attention_weight_min": -42.18043518066406,
      "activations/layer18_attention_weight_max": 42.21393966674805,
      "activations/layer18_attention_weight_min": -37.58005905151367,
      "activations/layer19_attention_weight_max": 20.375173568725586,
      "activations/layer19_attention_weight_min": -20.925199508666992,
      "activations/layer1_attention_weight_max": 16.177932739257812,
      "activations/layer1_attention_weight_min": -16.058876037597656,
      "activations/layer20_attention_weight_max": 20.595321655273438,
      "activations/layer20_attention_weight_min": -22.47109603881836,
      "activations/layer21_attention_weight_max": 36.202396392822266,
      "activations/layer21_attention_weight_min": -22.8792667388916,
      "activations/layer22_attention_weight_max": 29.597187042236328,
      "activations/layer22_attention_weight_min": -27.57973861694336,
      "activations/layer23_attention_weight_max": 35.3566780090332,
      "activations/layer23_attention_weight_min": -22.74891471862793,
      "activations/layer2_attention_weight_max": 33.44175338745117,
      "activations/layer2_attention_weight_min": -31.14169692993164,
      "activations/layer3_attention_weight_max": 93.87167358398438,
      "activations/layer3_attention_weight_min": -96.89385223388672,
      "activations/layer4_attention_weight_max": 101.15018463134766,
      "activations/layer4_attention_weight_min": -97.18869018554688,
      "activations/layer5_attention_weight_max": 72.17805480957031,
      "activations/layer5_attention_weight_min": -79.64878845214844,
      "activations/layer6_attention_weight_max": 56.31129837036133,
      "activations/layer6_attention_weight_min": -56.58933639526367,
      "activations/layer7_attention_weight_max": 73.00130462646484,
      "activations/layer7_attention_weight_min": -71.25347900390625,
      "activations/layer8_attention_weight_max": 50.460723876953125,
      "activations/layer8_attention_weight_min": -49.1497688293457,
      "activations/layer9_attention_weight_max": 51.7056999206543,
      "activations/layer9_attention_weight_min": -46.9600830078125,
      "epoch": 12.63,
      "learning_rate": 6.927424242424241e-05,
      "loss": 2.7725,
      "step": 217300
    },
    {
      "activations/layer0_attention_weight_max": 15.841407775878906,
      "activations/layer0_attention_weight_min": -14.346272468566895,
      "activations/layer10_attention_weight_max": 42.91404342651367,
      "activations/layer10_attention_weight_min": -41.31867218017578,
      "activations/layer11_attention_weight_max": 38.72810363769531,
      "activations/layer11_attention_weight_min": -39.03262710571289,
      "activations/layer12_attention_weight_max": 20.472171783447266,
      "activations/layer12_attention_weight_min": -23.62054443359375,
      "activations/layer13_attention_weight_max": 43.03776550292969,
      "activations/layer13_attention_weight_min": -30.623165130615234,
      "activations/layer14_attention_weight_max": 41.27655792236328,
      "activations/layer14_attention_weight_min": -31.204875946044922,
      "activations/layer15_attention_weight_max": 34.81425476074219,
      "activations/layer15_attention_weight_min": -30.81316566467285,
      "activations/layer16_attention_weight_max": 32.91604995727539,
      "activations/layer16_attention_weight_min": -30.879444122314453,
      "activations/layer17_attention_weight_max": 54.520999908447266,
      "activations/layer17_attention_weight_min": -46.87323760986328,
      "activations/layer18_attention_weight_max": 53.92117691040039,
      "activations/layer18_attention_weight_min": -39.809654235839844,
      "activations/layer19_attention_weight_max": 29.134693145751953,
      "activations/layer19_attention_weight_min": -21.897214889526367,
      "activations/layer1_attention_weight_max": 16.596633911132812,
      "activations/layer1_attention_weight_min": -13.31491470336914,
      "activations/layer20_attention_weight_max": 25.11864471435547,
      "activations/layer20_attention_weight_min": -22.5856876373291,
      "activations/layer21_attention_weight_max": 37.630706787109375,
      "activations/layer21_attention_weight_min": -21.70810890197754,
      "activations/layer22_attention_weight_max": 31.60605812072754,
      "activations/layer22_attention_weight_min": -25.273603439331055,
      "activations/layer23_attention_weight_max": 40.41730499267578,
      "activations/layer23_attention_weight_min": -24.851654052734375,
      "activations/layer2_attention_weight_max": 32.521263122558594,
      "activations/layer2_attention_weight_min": -30.585079193115234,
      "activations/layer3_attention_weight_max": 93.26573944091797,
      "activations/layer3_attention_weight_min": -96.6619644165039,
      "activations/layer4_attention_weight_max": 101.08451843261719,
      "activations/layer4_attention_weight_min": -98.76993560791016,
      "activations/layer5_attention_weight_max": 72.4583511352539,
      "activations/layer5_attention_weight_min": -81.64633178710938,
      "activations/layer6_attention_weight_max": 51.96195983886719,
      "activations/layer6_attention_weight_min": -55.89522171020508,
      "activations/layer7_attention_weight_max": 76.70513153076172,
      "activations/layer7_attention_weight_min": -69.72645568847656,
      "activations/layer8_attention_weight_max": 48.63718032836914,
      "activations/layer8_attention_weight_min": -47.24354553222656,
      "activations/layer9_attention_weight_max": 54.19709396362305,
      "activations/layer9_attention_weight_min": -49.08642578125,
      "epoch": 12.63,
      "learning_rate": 6.925530303030303e-05,
      "loss": 2.7673,
      "step": 217350
    },
    {
      "activations/layer0_attention_weight_max": 16.343992233276367,
      "activations/layer0_attention_weight_min": -13.953753471374512,
      "activations/layer10_attention_weight_max": 35.11684799194336,
      "activations/layer10_attention_weight_min": -34.0727653503418,
      "activations/layer11_attention_weight_max": 31.38271141052246,
      "activations/layer11_attention_weight_min": -32.38437271118164,
      "activations/layer12_attention_weight_max": 20.67125129699707,
      "activations/layer12_attention_weight_min": -23.31293487548828,
      "activations/layer13_attention_weight_max": 35.61004638671875,
      "activations/layer13_attention_weight_min": -30.207565307617188,
      "activations/layer14_attention_weight_max": 36.67131805419922,
      "activations/layer14_attention_weight_min": -30.54537582397461,
      "activations/layer15_attention_weight_max": 32.998374938964844,
      "activations/layer15_attention_weight_min": -29.20984649658203,
      "activations/layer16_attention_weight_max": 33.342689514160156,
      "activations/layer16_attention_weight_min": -30.458696365356445,
      "activations/layer17_attention_weight_max": 51.95759582519531,
      "activations/layer17_attention_weight_min": -45.046539306640625,
      "activations/layer18_attention_weight_max": 46.78992462158203,
      "activations/layer18_attention_weight_min": -39.24905776977539,
      "activations/layer19_attention_weight_max": 25.993274688720703,
      "activations/layer19_attention_weight_min": -21.60890007019043,
      "activations/layer1_attention_weight_max": 17.669435501098633,
      "activations/layer1_attention_weight_min": -13.676787376403809,
      "activations/layer20_attention_weight_max": 24.038097381591797,
      "activations/layer20_attention_weight_min": -22.820436477661133,
      "activations/layer21_attention_weight_max": 37.64068603515625,
      "activations/layer21_attention_weight_min": -22.19550895690918,
      "activations/layer22_attention_weight_max": 34.12947082519531,
      "activations/layer22_attention_weight_min": -27.01934242248535,
      "activations/layer23_attention_weight_max": 38.749229431152344,
      "activations/layer23_attention_weight_min": -24.498973846435547,
      "activations/layer2_attention_weight_max": 31.95961570739746,
      "activations/layer2_attention_weight_min": -29.8707332611084,
      "activations/layer3_attention_weight_max": 95.47704315185547,
      "activations/layer3_attention_weight_min": -94.183837890625,
      "activations/layer4_attention_weight_max": 96.13534545898438,
      "activations/layer4_attention_weight_min": -90.67476654052734,
      "activations/layer5_attention_weight_max": 71.4833984375,
      "activations/layer5_attention_weight_min": -77.19657897949219,
      "activations/layer6_attention_weight_max": 53.08436965942383,
      "activations/layer6_attention_weight_min": -55.84602737426758,
      "activations/layer7_attention_weight_max": 70.7123794555664,
      "activations/layer7_attention_weight_min": -69.41797637939453,
      "activations/layer8_attention_weight_max": 44.23867416381836,
      "activations/layer8_attention_weight_min": -46.31098556518555,
      "activations/layer9_attention_weight_max": 46.90972137451172,
      "activations/layer9_attention_weight_min": -45.94411849975586,
      "epoch": 12.63,
      "learning_rate": 6.923636363636363e-05,
      "loss": 2.7593,
      "step": 217400
    },
    {
      "activations/layer0_attention_weight_max": 15.273908615112305,
      "activations/layer0_attention_weight_min": -14.003738403320312,
      "activations/layer10_attention_weight_max": 33.1018180847168,
      "activations/layer10_attention_weight_min": -33.650718688964844,
      "activations/layer11_attention_weight_max": 31.139747619628906,
      "activations/layer11_attention_weight_min": -31.010272979736328,
      "activations/layer12_attention_weight_max": 16.814132690429688,
      "activations/layer12_attention_weight_min": -27.523826599121094,
      "activations/layer13_attention_weight_max": 35.90152359008789,
      "activations/layer13_attention_weight_min": -26.996671676635742,
      "activations/layer14_attention_weight_max": 34.89934539794922,
      "activations/layer14_attention_weight_min": -29.655946731567383,
      "activations/layer15_attention_weight_max": 31.182458877563477,
      "activations/layer15_attention_weight_min": -28.753643035888672,
      "activations/layer16_attention_weight_max": 29.45820426940918,
      "activations/layer16_attention_weight_min": -27.24020004272461,
      "activations/layer17_attention_weight_max": 47.05821228027344,
      "activations/layer17_attention_weight_min": -41.964908599853516,
      "activations/layer18_attention_weight_max": 39.09450912475586,
      "activations/layer18_attention_weight_min": -36.517337799072266,
      "activations/layer19_attention_weight_max": 20.45331573486328,
      "activations/layer19_attention_weight_min": -22.040494918823242,
      "activations/layer1_attention_weight_max": 16.005489349365234,
      "activations/layer1_attention_weight_min": -14.998624801635742,
      "activations/layer20_attention_weight_max": 19.41290283203125,
      "activations/layer20_attention_weight_min": -20.802001953125,
      "activations/layer21_attention_weight_max": 30.481821060180664,
      "activations/layer21_attention_weight_min": -21.906614303588867,
      "activations/layer22_attention_weight_max": 27.468326568603516,
      "activations/layer22_attention_weight_min": -26.949405670166016,
      "activations/layer23_attention_weight_max": 32.10954284667969,
      "activations/layer23_attention_weight_min": -22.341094970703125,
      "activations/layer2_attention_weight_max": 33.84169387817383,
      "activations/layer2_attention_weight_min": -31.952945709228516,
      "activations/layer3_attention_weight_max": 95.80873107910156,
      "activations/layer3_attention_weight_min": -99.54629516601562,
      "activations/layer4_attention_weight_max": 96.70863342285156,
      "activations/layer4_attention_weight_min": -91.18547058105469,
      "activations/layer5_attention_weight_max": 70.9580078125,
      "activations/layer5_attention_weight_min": -74.51579284667969,
      "activations/layer6_attention_weight_max": 51.5370979309082,
      "activations/layer6_attention_weight_min": -50.50904083251953,
      "activations/layer7_attention_weight_max": 68.151611328125,
      "activations/layer7_attention_weight_min": -66.60686492919922,
      "activations/layer8_attention_weight_max": 45.256656646728516,
      "activations/layer8_attention_weight_min": -46.88031005859375,
      "activations/layer9_attention_weight_max": 45.344520568847656,
      "activations/layer9_attention_weight_min": -46.97617721557617,
      "epoch": 12.64,
      "learning_rate": 6.921742424242423e-05,
      "loss": 2.7939,
      "step": 217450
    },
    {
      "activations/layer0_attention_weight_max": 15.807204246520996,
      "activations/layer0_attention_weight_min": -14.558582305908203,
      "activations/layer10_attention_weight_max": 35.071197509765625,
      "activations/layer10_attention_weight_min": -36.09101486206055,
      "activations/layer11_attention_weight_max": 32.680320739746094,
      "activations/layer11_attention_weight_min": -31.978958129882812,
      "activations/layer12_attention_weight_max": 18.43878746032715,
      "activations/layer12_attention_weight_min": -24.202529907226562,
      "activations/layer13_attention_weight_max": 39.3251953125,
      "activations/layer13_attention_weight_min": -28.25494384765625,
      "activations/layer14_attention_weight_max": 37.759979248046875,
      "activations/layer14_attention_weight_min": -31.851516723632812,
      "activations/layer15_attention_weight_max": 35.49895477294922,
      "activations/layer15_attention_weight_min": -29.650793075561523,
      "activations/layer16_attention_weight_max": 33.6181640625,
      "activations/layer16_attention_weight_min": -31.006391525268555,
      "activations/layer17_attention_weight_max": 52.0631217956543,
      "activations/layer17_attention_weight_min": -46.98481750488281,
      "activations/layer18_attention_weight_max": 48.3834342956543,
      "activations/layer18_attention_weight_min": -37.994110107421875,
      "activations/layer19_attention_weight_max": 22.044851303100586,
      "activations/layer19_attention_weight_min": -22.38702392578125,
      "activations/layer1_attention_weight_max": 16.48272705078125,
      "activations/layer1_attention_weight_min": -15.1608304977417,
      "activations/layer20_attention_weight_max": 22.54291534423828,
      "activations/layer20_attention_weight_min": -21.70206069946289,
      "activations/layer21_attention_weight_max": 39.5052490234375,
      "activations/layer21_attention_weight_min": -22.647476196289062,
      "activations/layer22_attention_weight_max": 29.956317901611328,
      "activations/layer22_attention_weight_min": -27.65395164489746,
      "activations/layer23_attention_weight_max": 35.57662582397461,
      "activations/layer23_attention_weight_min": -25.411590576171875,
      "activations/layer2_attention_weight_max": 32.11619567871094,
      "activations/layer2_attention_weight_min": -31.01852798461914,
      "activations/layer3_attention_weight_max": 93.45064544677734,
      "activations/layer3_attention_weight_min": -95.75159454345703,
      "activations/layer4_attention_weight_max": 98.0277099609375,
      "activations/layer4_attention_weight_min": -98.52037811279297,
      "activations/layer5_attention_weight_max": 69.8934097290039,
      "activations/layer5_attention_weight_min": -84.89259338378906,
      "activations/layer6_attention_weight_max": 53.665016174316406,
      "activations/layer6_attention_weight_min": -54.09339904785156,
      "activations/layer7_attention_weight_max": 68.56356811523438,
      "activations/layer7_attention_weight_min": -73.33135223388672,
      "activations/layer8_attention_weight_max": 45.20077896118164,
      "activations/layer8_attention_weight_min": -48.02931213378906,
      "activations/layer9_attention_weight_max": 42.1652946472168,
      "activations/layer9_attention_weight_min": -48.83708190917969,
      "epoch": 12.64,
      "learning_rate": 6.919848484848484e-05,
      "loss": 2.7701,
      "step": 217500
    },
    {
      "activations/layer0_attention_weight_max": 16.010059356689453,
      "activations/layer0_attention_weight_min": -14.302688598632812,
      "activations/layer10_attention_weight_max": 34.19148635864258,
      "activations/layer10_attention_weight_min": -33.457000732421875,
      "activations/layer11_attention_weight_max": 29.104278564453125,
      "activations/layer11_attention_weight_min": -31.666929244995117,
      "activations/layer12_attention_weight_max": 19.27206802368164,
      "activations/layer12_attention_weight_min": -23.075153350830078,
      "activations/layer13_attention_weight_max": 32.64549255371094,
      "activations/layer13_attention_weight_min": -26.7766170501709,
      "activations/layer14_attention_weight_max": 34.202049255371094,
      "activations/layer14_attention_weight_min": -29.523672103881836,
      "activations/layer15_attention_weight_max": 32.78196716308594,
      "activations/layer15_attention_weight_min": -29.297178268432617,
      "activations/layer16_attention_weight_max": 31.750492095947266,
      "activations/layer16_attention_weight_min": -29.726240158081055,
      "activations/layer17_attention_weight_max": 53.06095504760742,
      "activations/layer17_attention_weight_min": -45.648284912109375,
      "activations/layer18_attention_weight_max": 47.7208366394043,
      "activations/layer18_attention_weight_min": -38.074501037597656,
      "activations/layer19_attention_weight_max": 23.4520320892334,
      "activations/layer19_attention_weight_min": -22.377233505249023,
      "activations/layer1_attention_weight_max": 17.431310653686523,
      "activations/layer1_attention_weight_min": -15.5169095993042,
      "activations/layer20_attention_weight_max": 21.46268081665039,
      "activations/layer20_attention_weight_min": -20.26222801208496,
      "activations/layer21_attention_weight_max": 34.87276077270508,
      "activations/layer21_attention_weight_min": -25.025182723999023,
      "activations/layer22_attention_weight_max": 28.602624893188477,
      "activations/layer22_attention_weight_min": -25.118507385253906,
      "activations/layer23_attention_weight_max": 36.2093505859375,
      "activations/layer23_attention_weight_min": -25.55305290222168,
      "activations/layer2_attention_weight_max": 32.58076477050781,
      "activations/layer2_attention_weight_min": -29.578872680664062,
      "activations/layer3_attention_weight_max": 94.28575897216797,
      "activations/layer3_attention_weight_min": -92.49591064453125,
      "activations/layer4_attention_weight_max": 95.8006591796875,
      "activations/layer4_attention_weight_min": -90.7889633178711,
      "activations/layer5_attention_weight_max": 69.45661926269531,
      "activations/layer5_attention_weight_min": -82.87946319580078,
      "activations/layer6_attention_weight_max": 51.173866271972656,
      "activations/layer6_attention_weight_min": -52.93525314331055,
      "activations/layer7_attention_weight_max": 70.05340576171875,
      "activations/layer7_attention_weight_min": -69.7590103149414,
      "activations/layer8_attention_weight_max": 43.50627136230469,
      "activations/layer8_attention_weight_min": -46.331790924072266,
      "activations/layer9_attention_weight_max": 44.52770233154297,
      "activations/layer9_attention_weight_min": -48.32099151611328,
      "epoch": 12.64,
      "learning_rate": 6.917954545454545e-05,
      "loss": 2.7825,
      "step": 217550
    },
    {
      "activations/layer0_attention_weight_max": 16.058691024780273,
      "activations/layer0_attention_weight_min": -14.314229011535645,
      "activations/layer10_attention_weight_max": 40.30474090576172,
      "activations/layer10_attention_weight_min": -38.734039306640625,
      "activations/layer11_attention_weight_max": 35.0728759765625,
      "activations/layer11_attention_weight_min": -35.762325286865234,
      "activations/layer12_attention_weight_max": 20.257781982421875,
      "activations/layer12_attention_weight_min": -23.81943130493164,
      "activations/layer13_attention_weight_max": 39.40184783935547,
      "activations/layer13_attention_weight_min": -32.26752471923828,
      "activations/layer14_attention_weight_max": 36.960018157958984,
      "activations/layer14_attention_weight_min": -32.887916564941406,
      "activations/layer15_attention_weight_max": 32.90510940551758,
      "activations/layer15_attention_weight_min": -31.692455291748047,
      "activations/layer16_attention_weight_max": 33.311805725097656,
      "activations/layer16_attention_weight_min": -31.46449851989746,
      "activations/layer17_attention_weight_max": 52.02833938598633,
      "activations/layer17_attention_weight_min": -44.90597915649414,
      "activations/layer18_attention_weight_max": 47.37321090698242,
      "activations/layer18_attention_weight_min": -39.181884765625,
      "activations/layer19_attention_weight_max": 22.465038299560547,
      "activations/layer19_attention_weight_min": -23.452756881713867,
      "activations/layer1_attention_weight_max": 17.702472686767578,
      "activations/layer1_attention_weight_min": -15.944137573242188,
      "activations/layer20_attention_weight_max": 23.240550994873047,
      "activations/layer20_attention_weight_min": -23.056537628173828,
      "activations/layer21_attention_weight_max": 35.881534576416016,
      "activations/layer21_attention_weight_min": -22.506912231445312,
      "activations/layer22_attention_weight_max": 30.95252227783203,
      "activations/layer22_attention_weight_min": -27.429725646972656,
      "activations/layer23_attention_weight_max": 38.537662506103516,
      "activations/layer23_attention_weight_min": -24.598247528076172,
      "activations/layer2_attention_weight_max": 32.03271484375,
      "activations/layer2_attention_weight_min": -33.36826705932617,
      "activations/layer3_attention_weight_max": 93.74393463134766,
      "activations/layer3_attention_weight_min": -100.62493133544922,
      "activations/layer4_attention_weight_max": 96.84696960449219,
      "activations/layer4_attention_weight_min": -97.36382293701172,
      "activations/layer5_attention_weight_max": 73.27519989013672,
      "activations/layer5_attention_weight_min": -79.66731262207031,
      "activations/layer6_attention_weight_max": 54.703372955322266,
      "activations/layer6_attention_weight_min": -54.619407653808594,
      "activations/layer7_attention_weight_max": 73.56832885742188,
      "activations/layer7_attention_weight_min": -77.84355163574219,
      "activations/layer8_attention_weight_max": 46.20718002319336,
      "activations/layer8_attention_weight_min": -51.35042953491211,
      "activations/layer9_attention_weight_max": 50.908470153808594,
      "activations/layer9_attention_weight_min": -51.3466911315918,
      "epoch": 12.64,
      "learning_rate": 6.916060606060606e-05,
      "loss": 2.7853,
      "step": 217600
    },
    {
      "activations/layer0_attention_weight_max": 15.732626914978027,
      "activations/layer0_attention_weight_min": -14.441300392150879,
      "activations/layer10_attention_weight_max": 35.658878326416016,
      "activations/layer10_attention_weight_min": -35.622032165527344,
      "activations/layer11_attention_weight_max": 31.09347152709961,
      "activations/layer11_attention_weight_min": -31.506084442138672,
      "activations/layer12_attention_weight_max": 27.914857864379883,
      "activations/layer12_attention_weight_min": -24.741355895996094,
      "activations/layer13_attention_weight_max": 40.2072639465332,
      "activations/layer13_attention_weight_min": -27.31508445739746,
      "activations/layer14_attention_weight_max": 34.55223083496094,
      "activations/layer14_attention_weight_min": -28.31528663635254,
      "activations/layer15_attention_weight_max": 32.00321578979492,
      "activations/layer15_attention_weight_min": -28.12853240966797,
      "activations/layer16_attention_weight_max": 31.012361526489258,
      "activations/layer16_attention_weight_min": -30.520828247070312,
      "activations/layer17_attention_weight_max": 52.04213333129883,
      "activations/layer17_attention_weight_min": -43.20039367675781,
      "activations/layer18_attention_weight_max": 49.40011215209961,
      "activations/layer18_attention_weight_min": -37.03960418701172,
      "activations/layer19_attention_weight_max": 22.936840057373047,
      "activations/layer19_attention_weight_min": -21.414945602416992,
      "activations/layer1_attention_weight_max": 16.920093536376953,
      "activations/layer1_attention_weight_min": -15.565311431884766,
      "activations/layer20_attention_weight_max": 22.610368728637695,
      "activations/layer20_attention_weight_min": -20.926837921142578,
      "activations/layer21_attention_weight_max": 37.5084114074707,
      "activations/layer21_attention_weight_min": -24.369205474853516,
      "activations/layer22_attention_weight_max": 30.68606185913086,
      "activations/layer22_attention_weight_min": -24.384098052978516,
      "activations/layer23_attention_weight_max": 36.82056427001953,
      "activations/layer23_attention_weight_min": -24.50704574584961,
      "activations/layer2_attention_weight_max": 31.8594970703125,
      "activations/layer2_attention_weight_min": -30.046161651611328,
      "activations/layer3_attention_weight_max": 93.81092834472656,
      "activations/layer3_attention_weight_min": -90.4391860961914,
      "activations/layer4_attention_weight_max": 94.54067993164062,
      "activations/layer4_attention_weight_min": -90.9952392578125,
      "activations/layer5_attention_weight_max": 71.8497314453125,
      "activations/layer5_attention_weight_min": -76.63668823242188,
      "activations/layer6_attention_weight_max": 50.48387145996094,
      "activations/layer6_attention_weight_min": -50.22775650024414,
      "activations/layer7_attention_weight_max": 66.93567657470703,
      "activations/layer7_attention_weight_min": -68.15181732177734,
      "activations/layer8_attention_weight_max": 43.4813117980957,
      "activations/layer8_attention_weight_min": -44.38143539428711,
      "activations/layer9_attention_weight_max": 43.13201141357422,
      "activations/layer9_attention_weight_min": -47.36354064941406,
      "epoch": 12.65,
      "learning_rate": 6.914166666666666e-05,
      "loss": 2.7607,
      "step": 217650
    },
    {
      "activations/layer0_attention_weight_max": 15.430254936218262,
      "activations/layer0_attention_weight_min": -14.139933586120605,
      "activations/layer10_attention_weight_max": 36.26591873168945,
      "activations/layer10_attention_weight_min": -39.30358123779297,
      "activations/layer11_attention_weight_max": 32.774784088134766,
      "activations/layer11_attention_weight_min": -33.09151077270508,
      "activations/layer12_attention_weight_max": 20.782901763916016,
      "activations/layer12_attention_weight_min": -23.727249145507812,
      "activations/layer13_attention_weight_max": 43.38986587524414,
      "activations/layer13_attention_weight_min": -29.161052703857422,
      "activations/layer14_attention_weight_max": 42.23968505859375,
      "activations/layer14_attention_weight_min": -30.284955978393555,
      "activations/layer15_attention_weight_max": 36.42885971069336,
      "activations/layer15_attention_weight_min": -29.70068359375,
      "activations/layer16_attention_weight_max": 37.110538482666016,
      "activations/layer16_attention_weight_min": -30.43860626220703,
      "activations/layer17_attention_weight_max": 61.9265022277832,
      "activations/layer17_attention_weight_min": -46.538875579833984,
      "activations/layer18_attention_weight_max": 57.06376647949219,
      "activations/layer18_attention_weight_min": -40.713680267333984,
      "activations/layer19_attention_weight_max": 25.162067413330078,
      "activations/layer19_attention_weight_min": -21.854934692382812,
      "activations/layer1_attention_weight_max": 17.24618148803711,
      "activations/layer1_attention_weight_min": -14.66864013671875,
      "activations/layer20_attention_weight_max": 28.546396255493164,
      "activations/layer20_attention_weight_min": -21.923551559448242,
      "activations/layer21_attention_weight_max": 43.9980354309082,
      "activations/layer21_attention_weight_min": -21.033538818359375,
      "activations/layer22_attention_weight_max": 34.76764678955078,
      "activations/layer22_attention_weight_min": -23.559101104736328,
      "activations/layer23_attention_weight_max": 44.661888122558594,
      "activations/layer23_attention_weight_min": -24.467992782592773,
      "activations/layer2_attention_weight_max": 35.11200714111328,
      "activations/layer2_attention_weight_min": -33.41572570800781,
      "activations/layer3_attention_weight_max": 102.20601654052734,
      "activations/layer3_attention_weight_min": -95.9740219116211,
      "activations/layer4_attention_weight_max": 99.31627655029297,
      "activations/layer4_attention_weight_min": -93.16606903076172,
      "activations/layer5_attention_weight_max": 69.00770568847656,
      "activations/layer5_attention_weight_min": -79.74658203125,
      "activations/layer6_attention_weight_max": 52.42943572998047,
      "activations/layer6_attention_weight_min": -51.01371765136719,
      "activations/layer7_attention_weight_max": 68.42305755615234,
      "activations/layer7_attention_weight_min": -66.70914459228516,
      "activations/layer8_attention_weight_max": 45.55385208129883,
      "activations/layer8_attention_weight_min": -47.09236145019531,
      "activations/layer9_attention_weight_max": 48.574642181396484,
      "activations/layer9_attention_weight_min": -48.199466705322266,
      "epoch": 12.65,
      "learning_rate": 6.912272727272726e-05,
      "loss": 2.7597,
      "step": 217700
    },
    {
      "activations/layer0_attention_weight_max": 16.02379608154297,
      "activations/layer0_attention_weight_min": -15.166150093078613,
      "activations/layer10_attention_weight_max": 35.41496276855469,
      "activations/layer10_attention_weight_min": -34.943267822265625,
      "activations/layer11_attention_weight_max": 30.898807525634766,
      "activations/layer11_attention_weight_min": -32.82722091674805,
      "activations/layer12_attention_weight_max": 19.034440994262695,
      "activations/layer12_attention_weight_min": -22.314958572387695,
      "activations/layer13_attention_weight_max": 34.642696380615234,
      "activations/layer13_attention_weight_min": -27.775901794433594,
      "activations/layer14_attention_weight_max": 34.45295715332031,
      "activations/layer14_attention_weight_min": -29.949373245239258,
      "activations/layer15_attention_weight_max": 31.982315063476562,
      "activations/layer15_attention_weight_min": -29.90053367614746,
      "activations/layer16_attention_weight_max": 33.7335205078125,
      "activations/layer16_attention_weight_min": -31.60974884033203,
      "activations/layer17_attention_weight_max": 52.818572998046875,
      "activations/layer17_attention_weight_min": -46.41482925415039,
      "activations/layer18_attention_weight_max": 44.67208480834961,
      "activations/layer18_attention_weight_min": -39.85710906982422,
      "activations/layer19_attention_weight_max": 21.186403274536133,
      "activations/layer19_attention_weight_min": -21.141780853271484,
      "activations/layer1_attention_weight_max": 15.910333633422852,
      "activations/layer1_attention_weight_min": -15.006932258605957,
      "activations/layer20_attention_weight_max": 22.07036590576172,
      "activations/layer20_attention_weight_min": -23.47332763671875,
      "activations/layer21_attention_weight_max": 33.398834228515625,
      "activations/layer21_attention_weight_min": -21.012943267822266,
      "activations/layer22_attention_weight_max": 32.50654220581055,
      "activations/layer22_attention_weight_min": -25.943431854248047,
      "activations/layer23_attention_weight_max": 34.70014572143555,
      "activations/layer23_attention_weight_min": -24.681026458740234,
      "activations/layer2_attention_weight_max": 32.57733154296875,
      "activations/layer2_attention_weight_min": -30.489253997802734,
      "activations/layer3_attention_weight_max": 91.0810546875,
      "activations/layer3_attention_weight_min": -90.6729736328125,
      "activations/layer4_attention_weight_max": 95.56182098388672,
      "activations/layer4_attention_weight_min": -90.71221160888672,
      "activations/layer5_attention_weight_max": 72.40973663330078,
      "activations/layer5_attention_weight_min": -82.49298095703125,
      "activations/layer6_attention_weight_max": 52.401893615722656,
      "activations/layer6_attention_weight_min": -51.609230041503906,
      "activations/layer7_attention_weight_max": 68.37674713134766,
      "activations/layer7_attention_weight_min": -69.13902282714844,
      "activations/layer8_attention_weight_max": 45.0738410949707,
      "activations/layer8_attention_weight_min": -45.8355598449707,
      "activations/layer9_attention_weight_max": 43.55339050292969,
      "activations/layer9_attention_weight_min": -44.83771514892578,
      "epoch": 12.65,
      "learning_rate": 6.910378787878788e-05,
      "loss": 2.7568,
      "step": 217750
    },
    {
      "activations/layer0_attention_weight_max": 16.181236267089844,
      "activations/layer0_attention_weight_min": -15.050056457519531,
      "activations/layer10_attention_weight_max": 33.80006790161133,
      "activations/layer10_attention_weight_min": -33.10698699951172,
      "activations/layer11_attention_weight_max": 29.939069747924805,
      "activations/layer11_attention_weight_min": -30.948711395263672,
      "activations/layer12_attention_weight_max": 23.210966110229492,
      "activations/layer12_attention_weight_min": -24.367469787597656,
      "activations/layer13_attention_weight_max": 42.00165939331055,
      "activations/layer13_attention_weight_min": -27.165882110595703,
      "activations/layer14_attention_weight_max": 37.942970275878906,
      "activations/layer14_attention_weight_min": -28.440807342529297,
      "activations/layer15_attention_weight_max": 32.47737503051758,
      "activations/layer15_attention_weight_min": -28.72195816040039,
      "activations/layer16_attention_weight_max": 33.077911376953125,
      "activations/layer16_attention_weight_min": -29.038856506347656,
      "activations/layer17_attention_weight_max": 51.69203567504883,
      "activations/layer17_attention_weight_min": -43.4340705871582,
      "activations/layer18_attention_weight_max": 48.13911056518555,
      "activations/layer18_attention_weight_min": -37.6278190612793,
      "activations/layer19_attention_weight_max": 24.420875549316406,
      "activations/layer19_attention_weight_min": -22.794443130493164,
      "activations/layer1_attention_weight_max": 16.441532135009766,
      "activations/layer1_attention_weight_min": -14.955931663513184,
      "activations/layer20_attention_weight_max": 23.18206024169922,
      "activations/layer20_attention_weight_min": -20.63962745666504,
      "activations/layer21_attention_weight_max": 41.52035903930664,
      "activations/layer21_attention_weight_min": -21.720857620239258,
      "activations/layer22_attention_weight_max": 30.767671585083008,
      "activations/layer22_attention_weight_min": -24.24883460998535,
      "activations/layer23_attention_weight_max": 42.441959381103516,
      "activations/layer23_attention_weight_min": -25.603727340698242,
      "activations/layer2_attention_weight_max": 31.25916290283203,
      "activations/layer2_attention_weight_min": -30.13753318786621,
      "activations/layer3_attention_weight_max": 90.98274230957031,
      "activations/layer3_attention_weight_min": -94.60501861572266,
      "activations/layer4_attention_weight_max": 91.28445434570312,
      "activations/layer4_attention_weight_min": -89.7324447631836,
      "activations/layer5_attention_weight_max": 68.77790832519531,
      "activations/layer5_attention_weight_min": -77.4000015258789,
      "activations/layer6_attention_weight_max": 50.287742614746094,
      "activations/layer6_attention_weight_min": -49.88859939575195,
      "activations/layer7_attention_weight_max": 68.03421783447266,
      "activations/layer7_attention_weight_min": -65.4310073852539,
      "activations/layer8_attention_weight_max": 40.870391845703125,
      "activations/layer8_attention_weight_min": -44.201751708984375,
      "activations/layer9_attention_weight_max": 40.84046936035156,
      "activations/layer9_attention_weight_min": -43.290504455566406,
      "epoch": 12.66,
      "learning_rate": 6.908484848484848e-05,
      "loss": 2.7762,
      "step": 217800
    },
    {
      "activations/layer0_attention_weight_max": 15.736781120300293,
      "activations/layer0_attention_weight_min": -14.355118751525879,
      "activations/layer10_attention_weight_max": 33.56194305419922,
      "activations/layer10_attention_weight_min": -35.187599182128906,
      "activations/layer11_attention_weight_max": 31.799514770507812,
      "activations/layer11_attention_weight_min": -31.858030319213867,
      "activations/layer12_attention_weight_max": 18.187923431396484,
      "activations/layer12_attention_weight_min": -24.38528823852539,
      "activations/layer13_attention_weight_max": 34.2020378112793,
      "activations/layer13_attention_weight_min": -29.9251708984375,
      "activations/layer14_attention_weight_max": 33.99970245361328,
      "activations/layer14_attention_weight_min": -29.531400680541992,
      "activations/layer15_attention_weight_max": 32.44721984863281,
      "activations/layer15_attention_weight_min": -29.04075813293457,
      "activations/layer16_attention_weight_max": 31.622583389282227,
      "activations/layer16_attention_weight_min": -29.302799224853516,
      "activations/layer17_attention_weight_max": 50.137882232666016,
      "activations/layer17_attention_weight_min": -45.87755584716797,
      "activations/layer18_attention_weight_max": 43.421600341796875,
      "activations/layer18_attention_weight_min": -39.936798095703125,
      "activations/layer19_attention_weight_max": 19.468276977539062,
      "activations/layer19_attention_weight_min": -22.4295597076416,
      "activations/layer1_attention_weight_max": 16.844594955444336,
      "activations/layer1_attention_weight_min": -14.913217544555664,
      "activations/layer20_attention_weight_max": 20.52806282043457,
      "activations/layer20_attention_weight_min": -21.37067985534668,
      "activations/layer21_attention_weight_max": 35.41804885864258,
      "activations/layer21_attention_weight_min": -23.82805061340332,
      "activations/layer22_attention_weight_max": 25.77188491821289,
      "activations/layer22_attention_weight_min": -25.433502197265625,
      "activations/layer23_attention_weight_max": 34.19526672363281,
      "activations/layer23_attention_weight_min": -26.332439422607422,
      "activations/layer2_attention_weight_max": 32.456031799316406,
      "activations/layer2_attention_weight_min": -31.745861053466797,
      "activations/layer3_attention_weight_max": 95.15049743652344,
      "activations/layer3_attention_weight_min": -97.39265441894531,
      "activations/layer4_attention_weight_max": 94.03255462646484,
      "activations/layer4_attention_weight_min": -93.2566146850586,
      "activations/layer5_attention_weight_max": 68.26322937011719,
      "activations/layer5_attention_weight_min": -77.10282897949219,
      "activations/layer6_attention_weight_max": 52.26996994018555,
      "activations/layer6_attention_weight_min": -52.16612243652344,
      "activations/layer7_attention_weight_max": 67.37332153320312,
      "activations/layer7_attention_weight_min": -68.22815704345703,
      "activations/layer8_attention_weight_max": 46.30610656738281,
      "activations/layer8_attention_weight_min": -47.288814544677734,
      "activations/layer9_attention_weight_max": 44.41990280151367,
      "activations/layer9_attention_weight_min": -44.55528259277344,
      "epoch": 12.66,
      "learning_rate": 6.906590909090908e-05,
      "loss": 2.7525,
      "step": 217850
    },
    {
      "activations/layer0_attention_weight_max": 15.268387794494629,
      "activations/layer0_attention_weight_min": -14.683754920959473,
      "activations/layer10_attention_weight_max": 36.512977600097656,
      "activations/layer10_attention_weight_min": -34.69512939453125,
      "activations/layer11_attention_weight_max": 33.69504165649414,
      "activations/layer11_attention_weight_min": -32.77394104003906,
      "activations/layer12_attention_weight_max": 19.468902587890625,
      "activations/layer12_attention_weight_min": -24.50416374206543,
      "activations/layer13_attention_weight_max": 41.01654052734375,
      "activations/layer13_attention_weight_min": -31.363069534301758,
      "activations/layer14_attention_weight_max": 36.037776947021484,
      "activations/layer14_attention_weight_min": -31.62657928466797,
      "activations/layer15_attention_weight_max": 31.059762954711914,
      "activations/layer15_attention_weight_min": -29.98444366455078,
      "activations/layer16_attention_weight_max": 29.073974609375,
      "activations/layer16_attention_weight_min": -31.184574127197266,
      "activations/layer17_attention_weight_max": 48.70283889770508,
      "activations/layer17_attention_weight_min": -44.29420852661133,
      "activations/layer18_attention_weight_max": 42.24541473388672,
      "activations/layer18_attention_weight_min": -37.16219711303711,
      "activations/layer19_attention_weight_max": 22.54224395751953,
      "activations/layer19_attention_weight_min": -22.66878318786621,
      "activations/layer1_attention_weight_max": 15.43773078918457,
      "activations/layer1_attention_weight_min": -15.71021556854248,
      "activations/layer20_attention_weight_max": 20.324583053588867,
      "activations/layer20_attention_weight_min": -22.69500160217285,
      "activations/layer21_attention_weight_max": 33.459686279296875,
      "activations/layer21_attention_weight_min": -22.887968063354492,
      "activations/layer22_attention_weight_max": 28.19795799255371,
      "activations/layer22_attention_weight_min": -25.54543685913086,
      "activations/layer23_attention_weight_max": 36.34225082397461,
      "activations/layer23_attention_weight_min": -22.970741271972656,
      "activations/layer2_attention_weight_max": 32.58850860595703,
      "activations/layer2_attention_weight_min": -30.864501953125,
      "activations/layer3_attention_weight_max": 92.6351089477539,
      "activations/layer3_attention_weight_min": -91.0391845703125,
      "activations/layer4_attention_weight_max": 90.9919662475586,
      "activations/layer4_attention_weight_min": -87.90728759765625,
      "activations/layer5_attention_weight_max": 70.32112121582031,
      "activations/layer5_attention_weight_min": -76.05247497558594,
      "activations/layer6_attention_weight_max": 50.467506408691406,
      "activations/layer6_attention_weight_min": -51.84914779663086,
      "activations/layer7_attention_weight_max": 72.8149185180664,
      "activations/layer7_attention_weight_min": -64.99390411376953,
      "activations/layer8_attention_weight_max": 45.868263244628906,
      "activations/layer8_attention_weight_min": -45.36239242553711,
      "activations/layer9_attention_weight_max": 50.52378845214844,
      "activations/layer9_attention_weight_min": -45.01729965209961,
      "epoch": 12.66,
      "learning_rate": 6.90469696969697e-05,
      "loss": 2.7647,
      "step": 217900
    },
    {
      "activations/layer0_attention_weight_max": 15.884821891784668,
      "activations/layer0_attention_weight_min": -13.774077415466309,
      "activations/layer10_attention_weight_max": 34.766963958740234,
      "activations/layer10_attention_weight_min": -34.755096435546875,
      "activations/layer11_attention_weight_max": 32.65996551513672,
      "activations/layer11_attention_weight_min": -35.07783508300781,
      "activations/layer12_attention_weight_max": 20.436338424682617,
      "activations/layer12_attention_weight_min": -23.56782341003418,
      "activations/layer13_attention_weight_max": 41.49653244018555,
      "activations/layer13_attention_weight_min": -31.012073516845703,
      "activations/layer14_attention_weight_max": 42.62467956542969,
      "activations/layer14_attention_weight_min": -33.081661224365234,
      "activations/layer15_attention_weight_max": 32.8282470703125,
      "activations/layer15_attention_weight_min": -29.9354305267334,
      "activations/layer16_attention_weight_max": 32.345096588134766,
      "activations/layer16_attention_weight_min": -31.142656326293945,
      "activations/layer17_attention_weight_max": 55.557647705078125,
      "activations/layer17_attention_weight_min": -45.95357894897461,
      "activations/layer18_attention_weight_max": 48.58861541748047,
      "activations/layer18_attention_weight_min": -39.92083740234375,
      "activations/layer19_attention_weight_max": 27.886707305908203,
      "activations/layer19_attention_weight_min": -21.467304229736328,
      "activations/layer1_attention_weight_max": 17.268362045288086,
      "activations/layer1_attention_weight_min": -15.345151901245117,
      "activations/layer20_attention_weight_max": 24.913114547729492,
      "activations/layer20_attention_weight_min": -22.146821975708008,
      "activations/layer21_attention_weight_max": 46.16159439086914,
      "activations/layer21_attention_weight_min": -26.487333297729492,
      "activations/layer22_attention_weight_max": 33.340572357177734,
      "activations/layer22_attention_weight_min": -25.476919174194336,
      "activations/layer23_attention_weight_max": 43.879127502441406,
      "activations/layer23_attention_weight_min": -24.980215072631836,
      "activations/layer2_attention_weight_max": 34.919673919677734,
      "activations/layer2_attention_weight_min": -34.55464172363281,
      "activations/layer3_attention_weight_max": 94.91510772705078,
      "activations/layer3_attention_weight_min": -96.3531723022461,
      "activations/layer4_attention_weight_max": 96.52068328857422,
      "activations/layer4_attention_weight_min": -93.33946228027344,
      "activations/layer5_attention_weight_max": 69.25521850585938,
      "activations/layer5_attention_weight_min": -76.97151184082031,
      "activations/layer6_attention_weight_max": 51.77699279785156,
      "activations/layer6_attention_weight_min": -51.02188491821289,
      "activations/layer7_attention_weight_max": 66.39851379394531,
      "activations/layer7_attention_weight_min": -67.92887878417969,
      "activations/layer8_attention_weight_max": 46.41950988769531,
      "activations/layer8_attention_weight_min": -47.273868560791016,
      "activations/layer9_attention_weight_max": 44.88005828857422,
      "activations/layer9_attention_weight_min": -48.169559478759766,
      "epoch": 12.66,
      "learning_rate": 6.90280303030303e-05,
      "loss": 2.7763,
      "step": 217950
    },
    {
      "activations/layer0_attention_weight_max": 16.304946899414062,
      "activations/layer0_attention_weight_min": -13.611401557922363,
      "activations/layer10_attention_weight_max": 38.17917251586914,
      "activations/layer10_attention_weight_min": -36.57603073120117,
      "activations/layer11_attention_weight_max": 33.700233459472656,
      "activations/layer11_attention_weight_min": -32.72036361694336,
      "activations/layer12_attention_weight_max": 19.001131057739258,
      "activations/layer12_attention_weight_min": -23.347455978393555,
      "activations/layer13_attention_weight_max": 37.3369140625,
      "activations/layer13_attention_weight_min": -27.829599380493164,
      "activations/layer14_attention_weight_max": 38.4985466003418,
      "activations/layer14_attention_weight_min": -31.04737663269043,
      "activations/layer15_attention_weight_max": 35.2115592956543,
      "activations/layer15_attention_weight_min": -29.09357261657715,
      "activations/layer16_attention_weight_max": 32.55924606323242,
      "activations/layer16_attention_weight_min": -32.22781753540039,
      "activations/layer17_attention_weight_max": 56.641456604003906,
      "activations/layer17_attention_weight_min": -44.4361457824707,
      "activations/layer18_attention_weight_max": 49.99480438232422,
      "activations/layer18_attention_weight_min": -40.73336410522461,
      "activations/layer19_attention_weight_max": 24.908788681030273,
      "activations/layer19_attention_weight_min": -22.90431022644043,
      "activations/layer1_attention_weight_max": 16.984634399414062,
      "activations/layer1_attention_weight_min": -15.769635200500488,
      "activations/layer20_attention_weight_max": 24.744062423706055,
      "activations/layer20_attention_weight_min": -21.385272979736328,
      "activations/layer21_attention_weight_max": 39.955535888671875,
      "activations/layer21_attention_weight_min": -22.266006469726562,
      "activations/layer22_attention_weight_max": 31.822750091552734,
      "activations/layer22_attention_weight_min": -26.315603256225586,
      "activations/layer23_attention_weight_max": 36.88755798339844,
      "activations/layer23_attention_weight_min": -24.092666625976562,
      "activations/layer2_attention_weight_max": 32.15302276611328,
      "activations/layer2_attention_weight_min": -33.213035583496094,
      "activations/layer3_attention_weight_max": 91.27957916259766,
      "activations/layer3_attention_weight_min": -90.24950408935547,
      "activations/layer4_attention_weight_max": 93.47660064697266,
      "activations/layer4_attention_weight_min": -89.75518798828125,
      "activations/layer5_attention_weight_max": 68.99848937988281,
      "activations/layer5_attention_weight_min": -77.03387451171875,
      "activations/layer6_attention_weight_max": 52.39165496826172,
      "activations/layer6_attention_weight_min": -53.98844909667969,
      "activations/layer7_attention_weight_max": 73.75885009765625,
      "activations/layer7_attention_weight_min": -69.07318115234375,
      "activations/layer8_attention_weight_max": 46.7775764465332,
      "activations/layer8_attention_weight_min": -52.072967529296875,
      "activations/layer9_attention_weight_max": 46.332664489746094,
      "activations/layer9_attention_weight_min": -46.49190902709961,
      "epoch": 12.67,
      "learning_rate": 6.90090909090909e-05,
      "loss": 2.7731,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4395,
      "eval_samples_per_second": 508.795,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4395,
      "eval_openwebtext_samples_per_second": 508.795,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 1.9361,
      "eval_wikitext_samples_per_second": 235.519,
      "step": 218000
    },
    {
      "epoch": 12.67,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.5251,
      "eval_lambada_samples_per_second": 511.177,
      "step": 218000
    },
    {
      "activations/layer0_attention_weight_max": 16.345434188842773,
      "activations/layer0_attention_weight_min": -14.247040748596191,
      "activations/layer10_attention_weight_max": 42.660926818847656,
      "activations/layer10_attention_weight_min": -43.164955139160156,
      "activations/layer11_attention_weight_max": 35.38229751586914,
      "activations/layer11_attention_weight_min": -38.2504768371582,
      "activations/layer12_attention_weight_max": 21.129207611083984,
      "activations/layer12_attention_weight_min": -25.62211799621582,
      "activations/layer13_attention_weight_max": 45.844234466552734,
      "activations/layer13_attention_weight_min": -33.69877624511719,
      "activations/layer14_attention_weight_max": 47.06742477416992,
      "activations/layer14_attention_weight_min": -30.439943313598633,
      "activations/layer15_attention_weight_max": 38.55781936645508,
      "activations/layer15_attention_weight_min": -30.939348220825195,
      "activations/layer16_attention_weight_max": 32.13037109375,
      "activations/layer16_attention_weight_min": -30.76352882385254,
      "activations/layer17_attention_weight_max": 52.59294891357422,
      "activations/layer17_attention_weight_min": -46.611995697021484,
      "activations/layer18_attention_weight_max": 46.316402435302734,
      "activations/layer18_attention_weight_min": -41.15816116333008,
      "activations/layer19_attention_weight_max": 22.81214714050293,
      "activations/layer19_attention_weight_min": -24.852224349975586,
      "activations/layer1_attention_weight_max": 16.46910285949707,
      "activations/layer1_attention_weight_min": -14.310379981994629,
      "activations/layer20_attention_weight_max": 26.480350494384766,
      "activations/layer20_attention_weight_min": -22.077112197875977,
      "activations/layer21_attention_weight_max": 37.42262649536133,
      "activations/layer21_attention_weight_min": -20.88773536682129,
      "activations/layer22_attention_weight_max": 31.212921142578125,
      "activations/layer22_attention_weight_min": -25.12543296813965,
      "activations/layer23_attention_weight_max": 41.4293212890625,
      "activations/layer23_attention_weight_min": -24.151302337646484,
      "activations/layer2_attention_weight_max": 33.1374397277832,
      "activations/layer2_attention_weight_min": -32.12870788574219,
      "activations/layer3_attention_weight_max": 97.1029281616211,
      "activations/layer3_attention_weight_min": -94.7562255859375,
      "activations/layer4_attention_weight_max": 99.51201629638672,
      "activations/layer4_attention_weight_min": -99.98670959472656,
      "activations/layer5_attention_weight_max": 73.22457122802734,
      "activations/layer5_attention_weight_min": -84.24878692626953,
      "activations/layer6_attention_weight_max": 56.85715103149414,
      "activations/layer6_attention_weight_min": -57.267303466796875,
      "activations/layer7_attention_weight_max": 80.35920715332031,
      "activations/layer7_attention_weight_min": -78.99398803710938,
      "activations/layer8_attention_weight_max": 47.3504753112793,
      "activations/layer8_attention_weight_min": -50.90721130371094,
      "activations/layer9_attention_weight_max": 55.92094802856445,
      "activations/layer9_attention_weight_min": -53.37060546875,
      "epoch": 12.67,
      "learning_rate": 6.89901515151515e-05,
      "loss": 2.7563,
      "step": 218050
    },
    {
      "activations/layer0_attention_weight_max": 16.92598533630371,
      "activations/layer0_attention_weight_min": -14.372162818908691,
      "activations/layer10_attention_weight_max": 34.28154754638672,
      "activations/layer10_attention_weight_min": -33.54083251953125,
      "activations/layer11_attention_weight_max": 31.993804931640625,
      "activations/layer11_attention_weight_min": -32.37250900268555,
      "activations/layer12_attention_weight_max": 17.176471710205078,
      "activations/layer12_attention_weight_min": -25.379892349243164,
      "activations/layer13_attention_weight_max": 36.654945373535156,
      "activations/layer13_attention_weight_min": -28.562463760375977,
      "activations/layer14_attention_weight_max": 39.00996017456055,
      "activations/layer14_attention_weight_min": -29.091543197631836,
      "activations/layer15_attention_weight_max": 33.557106018066406,
      "activations/layer15_attention_weight_min": -28.68146324157715,
      "activations/layer16_attention_weight_max": 31.451154708862305,
      "activations/layer16_attention_weight_min": -28.027297973632812,
      "activations/layer17_attention_weight_max": 50.96916961669922,
      "activations/layer17_attention_weight_min": -44.24464797973633,
      "activations/layer18_attention_weight_max": 51.10215759277344,
      "activations/layer18_attention_weight_min": -37.92490768432617,
      "activations/layer19_attention_weight_max": 25.70725440979004,
      "activations/layer19_attention_weight_min": -22.221586227416992,
      "activations/layer1_attention_weight_max": 17.145618438720703,
      "activations/layer1_attention_weight_min": -14.275790214538574,
      "activations/layer20_attention_weight_max": 23.53921127319336,
      "activations/layer20_attention_weight_min": -22.421560287475586,
      "activations/layer21_attention_weight_max": 45.210227966308594,
      "activations/layer21_attention_weight_min": -29.340862274169922,
      "activations/layer22_attention_weight_max": 31.99782371520996,
      "activations/layer22_attention_weight_min": -25.49371910095215,
      "activations/layer23_attention_weight_max": 41.40454864501953,
      "activations/layer23_attention_weight_min": -25.382091522216797,
      "activations/layer2_attention_weight_max": 30.616811752319336,
      "activations/layer2_attention_weight_min": -28.49944305419922,
      "activations/layer3_attention_weight_max": 90.02723693847656,
      "activations/layer3_attention_weight_min": -87.26634979248047,
      "activations/layer4_attention_weight_max": 92.78638458251953,
      "activations/layer4_attention_weight_min": -92.20646667480469,
      "activations/layer5_attention_weight_max": 74.29684448242188,
      "activations/layer5_attention_weight_min": -76.58893585205078,
      "activations/layer6_attention_weight_max": 53.945594787597656,
      "activations/layer6_attention_weight_min": -50.736690521240234,
      "activations/layer7_attention_weight_max": 69.09925079345703,
      "activations/layer7_attention_weight_min": -68.04163360595703,
      "activations/layer8_attention_weight_max": 43.6530876159668,
      "activations/layer8_attention_weight_min": -48.33286666870117,
      "activations/layer9_attention_weight_max": 42.95164108276367,
      "activations/layer9_attention_weight_min": -46.65342330932617,
      "epoch": 12.67,
      "learning_rate": 6.897121212121212e-05,
      "loss": 2.7687,
      "step": 218100
    },
    {
      "activations/layer0_attention_weight_max": 15.691514015197754,
      "activations/layer0_attention_weight_min": -14.607682228088379,
      "activations/layer10_attention_weight_max": 33.62831497192383,
      "activations/layer10_attention_weight_min": -34.267822265625,
      "activations/layer11_attention_weight_max": 29.884092330932617,
      "activations/layer11_attention_weight_min": -32.13737869262695,
      "activations/layer12_attention_weight_max": 19.61026954650879,
      "activations/layer12_attention_weight_min": -26.7430362701416,
      "activations/layer13_attention_weight_max": 36.697486877441406,
      "activations/layer13_attention_weight_min": -31.175941467285156,
      "activations/layer14_attention_weight_max": 34.7737922668457,
      "activations/layer14_attention_weight_min": -29.314098358154297,
      "activations/layer15_attention_weight_max": 32.56736755371094,
      "activations/layer15_attention_weight_min": -29.298702239990234,
      "activations/layer16_attention_weight_max": 31.96500587463379,
      "activations/layer16_attention_weight_min": -28.899085998535156,
      "activations/layer17_attention_weight_max": 49.946773529052734,
      "activations/layer17_attention_weight_min": -41.7006721496582,
      "activations/layer18_attention_weight_max": 44.68757247924805,
      "activations/layer18_attention_weight_min": -35.9447135925293,
      "activations/layer19_attention_weight_max": 23.579294204711914,
      "activations/layer19_attention_weight_min": -21.220861434936523,
      "activations/layer1_attention_weight_max": 16.81863021850586,
      "activations/layer1_attention_weight_min": -14.125907897949219,
      "activations/layer20_attention_weight_max": 19.48772430419922,
      "activations/layer20_attention_weight_min": -21.25237464904785,
      "activations/layer21_attention_weight_max": 36.17763137817383,
      "activations/layer21_attention_weight_min": -22.891109466552734,
      "activations/layer22_attention_weight_max": 30.07241439819336,
      "activations/layer22_attention_weight_min": -26.150218963623047,
      "activations/layer23_attention_weight_max": 38.025535583496094,
      "activations/layer23_attention_weight_min": -25.621158599853516,
      "activations/layer2_attention_weight_max": 34.253482818603516,
      "activations/layer2_attention_weight_min": -32.438941955566406,
      "activations/layer3_attention_weight_max": 98.08586883544922,
      "activations/layer3_attention_weight_min": -96.21973419189453,
      "activations/layer4_attention_weight_max": 98.2278823852539,
      "activations/layer4_attention_weight_min": -91.97169494628906,
      "activations/layer5_attention_weight_max": 73.25212097167969,
      "activations/layer5_attention_weight_min": -75.20402526855469,
      "activations/layer6_attention_weight_max": 51.71788787841797,
      "activations/layer6_attention_weight_min": -51.80638885498047,
      "activations/layer7_attention_weight_max": 72.5021743774414,
      "activations/layer7_attention_weight_min": -64.70130920410156,
      "activations/layer8_attention_weight_max": 45.09882354736328,
      "activations/layer8_attention_weight_min": -46.71522521972656,
      "activations/layer9_attention_weight_max": 44.860286712646484,
      "activations/layer9_attention_weight_min": -47.784481048583984,
      "epoch": 12.68,
      "learning_rate": 6.895227272727272e-05,
      "loss": 2.7636,
      "step": 218150
    },
    {
      "activations/layer0_attention_weight_max": 16.855560302734375,
      "activations/layer0_attention_weight_min": -14.11308765411377,
      "activations/layer10_attention_weight_max": 35.09076690673828,
      "activations/layer10_attention_weight_min": -35.415626525878906,
      "activations/layer11_attention_weight_max": 32.37171173095703,
      "activations/layer11_attention_weight_min": -32.824951171875,
      "activations/layer12_attention_weight_max": 19.278623580932617,
      "activations/layer12_attention_weight_min": -29.643733978271484,
      "activations/layer13_attention_weight_max": 36.462642669677734,
      "activations/layer13_attention_weight_min": -32.393089294433594,
      "activations/layer14_attention_weight_max": 38.25965118408203,
      "activations/layer14_attention_weight_min": -30.513093948364258,
      "activations/layer15_attention_weight_max": 35.29305648803711,
      "activations/layer15_attention_weight_min": -29.57660675048828,
      "activations/layer16_attention_weight_max": 32.84173583984375,
      "activations/layer16_attention_weight_min": -29.86585235595703,
      "activations/layer17_attention_weight_max": 54.86643600463867,
      "activations/layer17_attention_weight_min": -47.065311431884766,
      "activations/layer18_attention_weight_max": 48.954593658447266,
      "activations/layer18_attention_weight_min": -40.45573806762695,
      "activations/layer19_attention_weight_max": 20.780080795288086,
      "activations/layer19_attention_weight_min": -22.194393157958984,
      "activations/layer1_attention_weight_max": 16.604143142700195,
      "activations/layer1_attention_weight_min": -15.947883605957031,
      "activations/layer20_attention_weight_max": 21.23093605041504,
      "activations/layer20_attention_weight_min": -22.575178146362305,
      "activations/layer21_attention_weight_max": 36.34310531616211,
      "activations/layer21_attention_weight_min": -25.0052433013916,
      "activations/layer22_attention_weight_max": 30.756502151489258,
      "activations/layer22_attention_weight_min": -24.92828369140625,
      "activations/layer23_attention_weight_max": 39.446266174316406,
      "activations/layer23_attention_weight_min": -22.42990493774414,
      "activations/layer2_attention_weight_max": 31.960216522216797,
      "activations/layer2_attention_weight_min": -31.108131408691406,
      "activations/layer3_attention_weight_max": 93.92655181884766,
      "activations/layer3_attention_weight_min": -99.41316986083984,
      "activations/layer4_attention_weight_max": 95.67485809326172,
      "activations/layer4_attention_weight_min": -96.43671417236328,
      "activations/layer5_attention_weight_max": 71.32508850097656,
      "activations/layer5_attention_weight_min": -79.04747772216797,
      "activations/layer6_attention_weight_max": 53.61454772949219,
      "activations/layer6_attention_weight_min": -55.13252258300781,
      "activations/layer7_attention_weight_max": 67.15503692626953,
      "activations/layer7_attention_weight_min": -74.39449310302734,
      "activations/layer8_attention_weight_max": 45.02593994140625,
      "activations/layer8_attention_weight_min": -47.722110748291016,
      "activations/layer9_attention_weight_max": 46.8660888671875,
      "activations/layer9_attention_weight_min": -49.28505325317383,
      "epoch": 12.68,
      "learning_rate": 6.893333333333333e-05,
      "loss": 2.7777,
      "step": 218200
    },
    {
      "activations/layer0_attention_weight_max": 15.505844116210938,
      "activations/layer0_attention_weight_min": -14.346230506896973,
      "activations/layer10_attention_weight_max": 39.03656768798828,
      "activations/layer10_attention_weight_min": -36.9650764465332,
      "activations/layer11_attention_weight_max": 32.72167205810547,
      "activations/layer11_attention_weight_min": -33.4696044921875,
      "activations/layer12_attention_weight_max": 17.56928825378418,
      "activations/layer12_attention_weight_min": -21.88125228881836,
      "activations/layer13_attention_weight_max": 32.845516204833984,
      "activations/layer13_attention_weight_min": -28.651622772216797,
      "activations/layer14_attention_weight_max": 33.36562728881836,
      "activations/layer14_attention_weight_min": -30.41254425048828,
      "activations/layer15_attention_weight_max": 31.92879867553711,
      "activations/layer15_attention_weight_min": -30.69964599609375,
      "activations/layer16_attention_weight_max": 31.602998733520508,
      "activations/layer16_attention_weight_min": -30.371875762939453,
      "activations/layer17_attention_weight_max": 50.83377456665039,
      "activations/layer17_attention_weight_min": -43.961490631103516,
      "activations/layer18_attention_weight_max": 44.87702178955078,
      "activations/layer18_attention_weight_min": -37.12763214111328,
      "activations/layer19_attention_weight_max": 20.08147621154785,
      "activations/layer19_attention_weight_min": -21.491722106933594,
      "activations/layer1_attention_weight_max": 16.977338790893555,
      "activations/layer1_attention_weight_min": -15.498255729675293,
      "activations/layer20_attention_weight_max": 23.456628799438477,
      "activations/layer20_attention_weight_min": -21.65195083618164,
      "activations/layer21_attention_weight_max": 32.047508239746094,
      "activations/layer21_attention_weight_min": -21.200979232788086,
      "activations/layer22_attention_weight_max": 27.95818328857422,
      "activations/layer22_attention_weight_min": -25.897390365600586,
      "activations/layer23_attention_weight_max": 36.06896209716797,
      "activations/layer23_attention_weight_min": -26.841384887695312,
      "activations/layer2_attention_weight_max": 32.10456085205078,
      "activations/layer2_attention_weight_min": -30.405105590820312,
      "activations/layer3_attention_weight_max": 93.901611328125,
      "activations/layer3_attention_weight_min": -93.10199737548828,
      "activations/layer4_attention_weight_max": 96.35149383544922,
      "activations/layer4_attention_weight_min": -93.05247497558594,
      "activations/layer5_attention_weight_max": 70.01115417480469,
      "activations/layer5_attention_weight_min": -78.44099426269531,
      "activations/layer6_attention_weight_max": 52.414100646972656,
      "activations/layer6_attention_weight_min": -55.082340240478516,
      "activations/layer7_attention_weight_max": 66.76976776123047,
      "activations/layer7_attention_weight_min": -70.32923126220703,
      "activations/layer8_attention_weight_max": 45.26311111450195,
      "activations/layer8_attention_weight_min": -50.98783874511719,
      "activations/layer9_attention_weight_max": 47.076045989990234,
      "activations/layer9_attention_weight_min": -48.54497146606445,
      "epoch": 12.68,
      "learning_rate": 6.891439393939393e-05,
      "loss": 2.7675,
      "step": 218250
    },
    {
      "activations/layer0_attention_weight_max": 16.16120147705078,
      "activations/layer0_attention_weight_min": -13.877968788146973,
      "activations/layer10_attention_weight_max": 35.63745880126953,
      "activations/layer10_attention_weight_min": -37.13739013671875,
      "activations/layer11_attention_weight_max": 31.159046173095703,
      "activations/layer11_attention_weight_min": -33.523460388183594,
      "activations/layer12_attention_weight_max": 18.492948532104492,
      "activations/layer12_attention_weight_min": -28.708492279052734,
      "activations/layer13_attention_weight_max": 34.50927734375,
      "activations/layer13_attention_weight_min": -29.216529846191406,
      "activations/layer14_attention_weight_max": 36.176055908203125,
      "activations/layer14_attention_weight_min": -30.071195602416992,
      "activations/layer15_attention_weight_max": 32.309715270996094,
      "activations/layer15_attention_weight_min": -30.209115982055664,
      "activations/layer16_attention_weight_max": 30.26137351989746,
      "activations/layer16_attention_weight_min": -31.024364471435547,
      "activations/layer17_attention_weight_max": 50.334228515625,
      "activations/layer17_attention_weight_min": -45.232601165771484,
      "activations/layer18_attention_weight_max": 45.48660659790039,
      "activations/layer18_attention_weight_min": -39.27054977416992,
      "activations/layer19_attention_weight_max": 21.439579010009766,
      "activations/layer19_attention_weight_min": -20.873571395874023,
      "activations/layer1_attention_weight_max": 16.047395706176758,
      "activations/layer1_attention_weight_min": -14.429417610168457,
      "activations/layer20_attention_weight_max": 20.80967140197754,
      "activations/layer20_attention_weight_min": -21.110143661499023,
      "activations/layer21_attention_weight_max": 34.942298889160156,
      "activations/layer21_attention_weight_min": -25.31700897216797,
      "activations/layer22_attention_weight_max": 29.238582611083984,
      "activations/layer22_attention_weight_min": -26.849409103393555,
      "activations/layer23_attention_weight_max": 36.641971588134766,
      "activations/layer23_attention_weight_min": -24.897197723388672,
      "activations/layer2_attention_weight_max": 32.54079055786133,
      "activations/layer2_attention_weight_min": -32.73442840576172,
      "activations/layer3_attention_weight_max": 96.100341796875,
      "activations/layer3_attention_weight_min": -91.87940216064453,
      "activations/layer4_attention_weight_max": 98.38151550292969,
      "activations/layer4_attention_weight_min": -96.31837463378906,
      "activations/layer5_attention_weight_max": 70.61447143554688,
      "activations/layer5_attention_weight_min": -88.12004089355469,
      "activations/layer6_attention_weight_max": 54.556461334228516,
      "activations/layer6_attention_weight_min": -52.169960021972656,
      "activations/layer7_attention_weight_max": 70.1869125366211,
      "activations/layer7_attention_weight_min": -66.5005111694336,
      "activations/layer8_attention_weight_max": 45.6301155090332,
      "activations/layer8_attention_weight_min": -49.030113220214844,
      "activations/layer9_attention_weight_max": 44.19526672363281,
      "activations/layer9_attention_weight_min": -46.14976119995117,
      "epoch": 12.68,
      "learning_rate": 6.889545454545454e-05,
      "loss": 2.774,
      "step": 218300
    },
    {
      "activations/layer0_attention_weight_max": 16.25543212890625,
      "activations/layer0_attention_weight_min": -13.909624099731445,
      "activations/layer10_attention_weight_max": 36.06159591674805,
      "activations/layer10_attention_weight_min": -34.663429260253906,
      "activations/layer11_attention_weight_max": 32.488243103027344,
      "activations/layer11_attention_weight_min": -33.578372955322266,
      "activations/layer12_attention_weight_max": 19.119873046875,
      "activations/layer12_attention_weight_min": -24.04816436767578,
      "activations/layer13_attention_weight_max": 39.18388748168945,
      "activations/layer13_attention_weight_min": -28.223215103149414,
      "activations/layer14_attention_weight_max": 40.98173522949219,
      "activations/layer14_attention_weight_min": -32.15849685668945,
      "activations/layer15_attention_weight_max": 34.829986572265625,
      "activations/layer15_attention_weight_min": -31.113203048706055,
      "activations/layer16_attention_weight_max": 30.43300437927246,
      "activations/layer16_attention_weight_min": -29.275726318359375,
      "activations/layer17_attention_weight_max": 53.02455520629883,
      "activations/layer17_attention_weight_min": -45.92787551879883,
      "activations/layer18_attention_weight_max": 47.15253448486328,
      "activations/layer18_attention_weight_min": -37.670005798339844,
      "activations/layer19_attention_weight_max": 24.505064010620117,
      "activations/layer19_attention_weight_min": -21.675214767456055,
      "activations/layer1_attention_weight_max": 17.075756072998047,
      "activations/layer1_attention_weight_min": -15.118452072143555,
      "activations/layer20_attention_weight_max": 23.02824592590332,
      "activations/layer20_attention_weight_min": -26.028602600097656,
      "activations/layer21_attention_weight_max": 39.93242263793945,
      "activations/layer21_attention_weight_min": -22.69732666015625,
      "activations/layer22_attention_weight_max": 31.758792877197266,
      "activations/layer22_attention_weight_min": -25.20862579345703,
      "activations/layer23_attention_weight_max": 40.74799346923828,
      "activations/layer23_attention_weight_min": -21.735321044921875,
      "activations/layer2_attention_weight_max": 26.269878387451172,
      "activations/layer2_attention_weight_min": -27.228750228881836,
      "activations/layer3_attention_weight_max": 77.0827865600586,
      "activations/layer3_attention_weight_min": -84.31455993652344,
      "activations/layer4_attention_weight_max": 85.0530776977539,
      "activations/layer4_attention_weight_min": -88.7525634765625,
      "activations/layer5_attention_weight_max": 66.79753875732422,
      "activations/layer5_attention_weight_min": -78.64732360839844,
      "activations/layer6_attention_weight_max": 46.74858093261719,
      "activations/layer6_attention_weight_min": -48.87648391723633,
      "activations/layer7_attention_weight_max": 64.8697509765625,
      "activations/layer7_attention_weight_min": -61.377986907958984,
      "activations/layer8_attention_weight_max": 43.575035095214844,
      "activations/layer8_attention_weight_min": -44.851749420166016,
      "activations/layer9_attention_weight_max": 42.48396301269531,
      "activations/layer9_attention_weight_min": -42.83687973022461,
      "epoch": 12.69,
      "learning_rate": 6.887651515151515e-05,
      "loss": 2.7869,
      "step": 218350
    },
    {
      "activations/layer0_attention_weight_max": 16.94325828552246,
      "activations/layer0_attention_weight_min": -14.012811660766602,
      "activations/layer10_attention_weight_max": 35.998817443847656,
      "activations/layer10_attention_weight_min": -36.06713104248047,
      "activations/layer11_attention_weight_max": 31.572181701660156,
      "activations/layer11_attention_weight_min": -30.981239318847656,
      "activations/layer12_attention_weight_max": 18.89275550842285,
      "activations/layer12_attention_weight_min": -25.804109573364258,
      "activations/layer13_attention_weight_max": 38.74953842163086,
      "activations/layer13_attention_weight_min": -28.162662506103516,
      "activations/layer14_attention_weight_max": 37.659271240234375,
      "activations/layer14_attention_weight_min": -29.425704956054688,
      "activations/layer15_attention_weight_max": 32.225704193115234,
      "activations/layer15_attention_weight_min": -28.516807556152344,
      "activations/layer16_attention_weight_max": 30.91893196105957,
      "activations/layer16_attention_weight_min": -29.141904830932617,
      "activations/layer17_attention_weight_max": 53.14563751220703,
      "activations/layer17_attention_weight_min": -44.13987731933594,
      "activations/layer18_attention_weight_max": 45.017547607421875,
      "activations/layer18_attention_weight_min": -39.07360076904297,
      "activations/layer19_attention_weight_max": 21.601747512817383,
      "activations/layer19_attention_weight_min": -21.312620162963867,
      "activations/layer1_attention_weight_max": 16.4371395111084,
      "activations/layer1_attention_weight_min": -13.57353401184082,
      "activations/layer20_attention_weight_max": 21.891557693481445,
      "activations/layer20_attention_weight_min": -20.433799743652344,
      "activations/layer21_attention_weight_max": 40.410423278808594,
      "activations/layer21_attention_weight_min": -23.91731834411621,
      "activations/layer22_attention_weight_max": 29.65592384338379,
      "activations/layer22_attention_weight_min": -25.548940658569336,
      "activations/layer23_attention_weight_max": 36.83694076538086,
      "activations/layer23_attention_weight_min": -23.870403289794922,
      "activations/layer2_attention_weight_max": 30.694204330444336,
      "activations/layer2_attention_weight_min": -28.630455017089844,
      "activations/layer3_attention_weight_max": 91.37240600585938,
      "activations/layer3_attention_weight_min": -88.6500473022461,
      "activations/layer4_attention_weight_max": 95.10285949707031,
      "activations/layer4_attention_weight_min": -89.94530487060547,
      "activations/layer5_attention_weight_max": 72.54161834716797,
      "activations/layer5_attention_weight_min": -81.94053649902344,
      "activations/layer6_attention_weight_max": 53.420162200927734,
      "activations/layer6_attention_weight_min": -53.75948715209961,
      "activations/layer7_attention_weight_max": 67.50166320800781,
      "activations/layer7_attention_weight_min": -71.21918487548828,
      "activations/layer8_attention_weight_max": 45.8452033996582,
      "activations/layer8_attention_weight_min": -48.41337966918945,
      "activations/layer9_attention_weight_max": 45.53575134277344,
      "activations/layer9_attention_weight_min": -47.82828903198242,
      "epoch": 12.69,
      "learning_rate": 6.885757575757575e-05,
      "loss": 2.7544,
      "step": 218400
    },
    {
      "activations/layer0_attention_weight_max": 17.12153434753418,
      "activations/layer0_attention_weight_min": -13.862778663635254,
      "activations/layer10_attention_weight_max": 36.000667572021484,
      "activations/layer10_attention_weight_min": -37.0103759765625,
      "activations/layer11_attention_weight_max": 33.70320129394531,
      "activations/layer11_attention_weight_min": -34.33180236816406,
      "activations/layer12_attention_weight_max": 18.64778709411621,
      "activations/layer12_attention_weight_min": -26.2479305267334,
      "activations/layer13_attention_weight_max": 34.67499542236328,
      "activations/layer13_attention_weight_min": -29.234716415405273,
      "activations/layer14_attention_weight_max": 38.834327697753906,
      "activations/layer14_attention_weight_min": -30.317537307739258,
      "activations/layer15_attention_weight_max": 35.78739929199219,
      "activations/layer15_attention_weight_min": -28.776397705078125,
      "activations/layer16_attention_weight_max": 30.333866119384766,
      "activations/layer16_attention_weight_min": -29.748580932617188,
      "activations/layer17_attention_weight_max": 49.48285675048828,
      "activations/layer17_attention_weight_min": -44.307498931884766,
      "activations/layer18_attention_weight_max": 46.98444366455078,
      "activations/layer18_attention_weight_min": -39.7420654296875,
      "activations/layer19_attention_weight_max": 24.488903045654297,
      "activations/layer19_attention_weight_min": -23.415494918823242,
      "activations/layer1_attention_weight_max": 17.026124954223633,
      "activations/layer1_attention_weight_min": -15.768579483032227,
      "activations/layer20_attention_weight_max": 22.817127227783203,
      "activations/layer20_attention_weight_min": -20.57683563232422,
      "activations/layer21_attention_weight_max": 40.6947021484375,
      "activations/layer21_attention_weight_min": -25.070276260375977,
      "activations/layer22_attention_weight_max": 31.187641143798828,
      "activations/layer22_attention_weight_min": -24.92466163635254,
      "activations/layer23_attention_weight_max": 42.407569885253906,
      "activations/layer23_attention_weight_min": -22.399717330932617,
      "activations/layer2_attention_weight_max": 33.70271301269531,
      "activations/layer2_attention_weight_min": -31.076141357421875,
      "activations/layer3_attention_weight_max": 93.54252624511719,
      "activations/layer3_attention_weight_min": -90.0999755859375,
      "activations/layer4_attention_weight_max": 89.09579467773438,
      "activations/layer4_attention_weight_min": -86.90654754638672,
      "activations/layer5_attention_weight_max": 68.0606918334961,
      "activations/layer5_attention_weight_min": -70.7589111328125,
      "activations/layer6_attention_weight_max": 49.27305603027344,
      "activations/layer6_attention_weight_min": -49.11168670654297,
      "activations/layer7_attention_weight_max": 74.95306396484375,
      "activations/layer7_attention_weight_min": -64.06979370117188,
      "activations/layer8_attention_weight_max": 45.7027473449707,
      "activations/layer8_attention_weight_min": -49.94392776489258,
      "activations/layer9_attention_weight_max": 52.464473724365234,
      "activations/layer9_attention_weight_min": -46.19990158081055,
      "epoch": 12.69,
      "learning_rate": 6.883863636363635e-05,
      "loss": 2.7777,
      "step": 218450
    },
    {
      "activations/layer0_attention_weight_max": 16.88539695739746,
      "activations/layer0_attention_weight_min": -14.132806777954102,
      "activations/layer10_attention_weight_max": 39.50666046142578,
      "activations/layer10_attention_weight_min": -36.650184631347656,
      "activations/layer11_attention_weight_max": 34.07085418701172,
      "activations/layer11_attention_weight_min": -33.98846435546875,
      "activations/layer12_attention_weight_max": 18.7592830657959,
      "activations/layer12_attention_weight_min": -25.359575271606445,
      "activations/layer13_attention_weight_max": 40.502357482910156,
      "activations/layer13_attention_weight_min": -27.089765548706055,
      "activations/layer14_attention_weight_max": 45.277313232421875,
      "activations/layer14_attention_weight_min": -32.789512634277344,
      "activations/layer15_attention_weight_max": 42.69857406616211,
      "activations/layer15_attention_weight_min": -32.097862243652344,
      "activations/layer16_attention_weight_max": 36.71438980102539,
      "activations/layer16_attention_weight_min": -35.580657958984375,
      "activations/layer17_attention_weight_max": 58.47722244262695,
      "activations/layer17_attention_weight_min": -52.43807601928711,
      "activations/layer18_attention_weight_max": 51.49057388305664,
      "activations/layer18_attention_weight_min": -42.88032531738281,
      "activations/layer19_attention_weight_max": 23.438751220703125,
      "activations/layer19_attention_weight_min": -22.989028930664062,
      "activations/layer1_attention_weight_max": 18.297582626342773,
      "activations/layer1_attention_weight_min": -15.38702392578125,
      "activations/layer20_attention_weight_max": 26.375381469726562,
      "activations/layer20_attention_weight_min": -23.331188201904297,
      "activations/layer21_attention_weight_max": 55.446006774902344,
      "activations/layer21_attention_weight_min": -30.707603454589844,
      "activations/layer22_attention_weight_max": 28.89724349975586,
      "activations/layer22_attention_weight_min": -25.48277473449707,
      "activations/layer23_attention_weight_max": 34.600135803222656,
      "activations/layer23_attention_weight_min": -25.246719360351562,
      "activations/layer2_attention_weight_max": 32.0633544921875,
      "activations/layer2_attention_weight_min": -31.684463500976562,
      "activations/layer3_attention_weight_max": 96.0404052734375,
      "activations/layer3_attention_weight_min": -98.31584167480469,
      "activations/layer4_attention_weight_max": 98.42803955078125,
      "activations/layer4_attention_weight_min": -96.42571258544922,
      "activations/layer5_attention_weight_max": 72.6899185180664,
      "activations/layer5_attention_weight_min": -81.65025329589844,
      "activations/layer6_attention_weight_max": 53.77330017089844,
      "activations/layer6_attention_weight_min": -54.84988784790039,
      "activations/layer7_attention_weight_max": 67.5404052734375,
      "activations/layer7_attention_weight_min": -71.091796875,
      "activations/layer8_attention_weight_max": 47.09939956665039,
      "activations/layer8_attention_weight_min": -49.204402923583984,
      "activations/layer9_attention_weight_max": 47.7404899597168,
      "activations/layer9_attention_weight_min": -48.92923355102539,
      "epoch": 12.7,
      "learning_rate": 6.881969696969697e-05,
      "loss": 2.7761,
      "step": 218500
    },
    {
      "activations/layer0_attention_weight_max": 17.49247169494629,
      "activations/layer0_attention_weight_min": -14.264704704284668,
      "activations/layer10_attention_weight_max": 38.40174102783203,
      "activations/layer10_attention_weight_min": -35.1112060546875,
      "activations/layer11_attention_weight_max": 33.791748046875,
      "activations/layer11_attention_weight_min": -35.90761184692383,
      "activations/layer12_attention_weight_max": 18.486770629882812,
      "activations/layer12_attention_weight_min": -26.97052574157715,
      "activations/layer13_attention_weight_max": 33.87849044799805,
      "activations/layer13_attention_weight_min": -29.12730598449707,
      "activations/layer14_attention_weight_max": 37.76152801513672,
      "activations/layer14_attention_weight_min": -31.15277671813965,
      "activations/layer15_attention_weight_max": 33.10004425048828,
      "activations/layer15_attention_weight_min": -29.282026290893555,
      "activations/layer16_attention_weight_max": 31.555143356323242,
      "activations/layer16_attention_weight_min": -32.804344177246094,
      "activations/layer17_attention_weight_max": 52.91326904296875,
      "activations/layer17_attention_weight_min": -46.6586799621582,
      "activations/layer18_attention_weight_max": 47.20364761352539,
      "activations/layer18_attention_weight_min": -38.365821838378906,
      "activations/layer19_attention_weight_max": 22.619831085205078,
      "activations/layer19_attention_weight_min": -23.79169464111328,
      "activations/layer1_attention_weight_max": 16.814117431640625,
      "activations/layer1_attention_weight_min": -14.847102165222168,
      "activations/layer20_attention_weight_max": 23.537492752075195,
      "activations/layer20_attention_weight_min": -21.184917449951172,
      "activations/layer21_attention_weight_max": 36.662681579589844,
      "activations/layer21_attention_weight_min": -21.42939567565918,
      "activations/layer22_attention_weight_max": 30.856985092163086,
      "activations/layer22_attention_weight_min": -26.884214401245117,
      "activations/layer23_attention_weight_max": 36.0581169128418,
      "activations/layer23_attention_weight_min": -25.402191162109375,
      "activations/layer2_attention_weight_max": 30.996564865112305,
      "activations/layer2_attention_weight_min": -29.353832244873047,
      "activations/layer3_attention_weight_max": 91.10643005371094,
      "activations/layer3_attention_weight_min": -88.66777801513672,
      "activations/layer4_attention_weight_max": 90.94182586669922,
      "activations/layer4_attention_weight_min": -90.91305541992188,
      "activations/layer5_attention_weight_max": 73.27943420410156,
      "activations/layer5_attention_weight_min": -80.27505493164062,
      "activations/layer6_attention_weight_max": 52.94729995727539,
      "activations/layer6_attention_weight_min": -51.65721130371094,
      "activations/layer7_attention_weight_max": 71.28697204589844,
      "activations/layer7_attention_weight_min": -68.82093811035156,
      "activations/layer8_attention_weight_max": 49.7659797668457,
      "activations/layer8_attention_weight_min": -46.99958038330078,
      "activations/layer9_attention_weight_max": 46.50978469848633,
      "activations/layer9_attention_weight_min": -48.86530685424805,
      "epoch": 12.7,
      "learning_rate": 6.880075757575757e-05,
      "loss": 2.7697,
      "step": 218550
    },
    {
      "activations/layer0_attention_weight_max": 16.323352813720703,
      "activations/layer0_attention_weight_min": -14.301797866821289,
      "activations/layer10_attention_weight_max": 35.270545959472656,
      "activations/layer10_attention_weight_min": -36.43721389770508,
      "activations/layer11_attention_weight_max": 33.07673645019531,
      "activations/layer11_attention_weight_min": -34.38584899902344,
      "activations/layer12_attention_weight_max": 18.39820098876953,
      "activations/layer12_attention_weight_min": -28.457712173461914,
      "activations/layer13_attention_weight_max": 42.86489486694336,
      "activations/layer13_attention_weight_min": -30.366769790649414,
      "activations/layer14_attention_weight_max": 34.102500915527344,
      "activations/layer14_attention_weight_min": -29.93717384338379,
      "activations/layer15_attention_weight_max": 31.21634292602539,
      "activations/layer15_attention_weight_min": -30.940053939819336,
      "activations/layer16_attention_weight_max": 31.407974243164062,
      "activations/layer16_attention_weight_min": -30.657201766967773,
      "activations/layer17_attention_weight_max": 49.91945266723633,
      "activations/layer17_attention_weight_min": -42.70741271972656,
      "activations/layer18_attention_weight_max": 46.05223846435547,
      "activations/layer18_attention_weight_min": -37.808685302734375,
      "activations/layer19_attention_weight_max": 21.35680389404297,
      "activations/layer19_attention_weight_min": -21.900606155395508,
      "activations/layer1_attention_weight_max": 16.188486099243164,
      "activations/layer1_attention_weight_min": -13.87180233001709,
      "activations/layer20_attention_weight_max": 21.456932067871094,
      "activations/layer20_attention_weight_min": -23.978504180908203,
      "activations/layer21_attention_weight_max": 37.87942886352539,
      "activations/layer21_attention_weight_min": -27.36159324645996,
      "activations/layer22_attention_weight_max": 34.23445510864258,
      "activations/layer22_attention_weight_min": -26.545135498046875,
      "activations/layer23_attention_weight_max": 39.33599853515625,
      "activations/layer23_attention_weight_min": -24.054489135742188,
      "activations/layer2_attention_weight_max": 32.335723876953125,
      "activations/layer2_attention_weight_min": -30.244426727294922,
      "activations/layer3_attention_weight_max": 94.42282104492188,
      "activations/layer3_attention_weight_min": -94.16874694824219,
      "activations/layer4_attention_weight_max": 98.26264953613281,
      "activations/layer4_attention_weight_min": -93.48491668701172,
      "activations/layer5_attention_weight_max": 71.70219421386719,
      "activations/layer5_attention_weight_min": -81.38245391845703,
      "activations/layer6_attention_weight_max": 54.423641204833984,
      "activations/layer6_attention_weight_min": -52.818965911865234,
      "activations/layer7_attention_weight_max": 72.30240631103516,
      "activations/layer7_attention_weight_min": -72.30612182617188,
      "activations/layer8_attention_weight_max": 44.26543045043945,
      "activations/layer8_attention_weight_min": -48.27757263183594,
      "activations/layer9_attention_weight_max": 54.40657424926758,
      "activations/layer9_attention_weight_min": -50.74200439453125,
      "epoch": 12.7,
      "learning_rate": 6.878181818181817e-05,
      "loss": 2.7773,
      "step": 218600
    },
    {
      "activations/layer0_attention_weight_max": 15.397732734680176,
      "activations/layer0_attention_weight_min": -13.85383129119873,
      "activations/layer10_attention_weight_max": 34.917240142822266,
      "activations/layer10_attention_weight_min": -33.88526153564453,
      "activations/layer11_attention_weight_max": 31.020450592041016,
      "activations/layer11_attention_weight_min": -30.695341110229492,
      "activations/layer12_attention_weight_max": 18.294490814208984,
      "activations/layer12_attention_weight_min": -24.453195571899414,
      "activations/layer13_attention_weight_max": 36.655479431152344,
      "activations/layer13_attention_weight_min": -25.686067581176758,
      "activations/layer14_attention_weight_max": 37.10548400878906,
      "activations/layer14_attention_weight_min": -29.23410987854004,
      "activations/layer15_attention_weight_max": 35.70413589477539,
      "activations/layer15_attention_weight_min": -28.720943450927734,
      "activations/layer16_attention_weight_max": 31.676908493041992,
      "activations/layer16_attention_weight_min": -29.217378616333008,
      "activations/layer17_attention_weight_max": 50.91481018066406,
      "activations/layer17_attention_weight_min": -44.02799606323242,
      "activations/layer18_attention_weight_max": 45.86217498779297,
      "activations/layer18_attention_weight_min": -36.63457107543945,
      "activations/layer19_attention_weight_max": 27.51468276977539,
      "activations/layer19_attention_weight_min": -22.26991081237793,
      "activations/layer1_attention_weight_max": 17.014408111572266,
      "activations/layer1_attention_weight_min": -14.71833324432373,
      "activations/layer20_attention_weight_max": 22.6784725189209,
      "activations/layer20_attention_weight_min": -20.076112747192383,
      "activations/layer21_attention_weight_max": 38.07042694091797,
      "activations/layer21_attention_weight_min": -22.944774627685547,
      "activations/layer22_attention_weight_max": 35.08243179321289,
      "activations/layer22_attention_weight_min": -27.6871337890625,
      "activations/layer23_attention_weight_max": 37.348594665527344,
      "activations/layer23_attention_weight_min": -23.835620880126953,
      "activations/layer2_attention_weight_max": 33.2247200012207,
      "activations/layer2_attention_weight_min": -31.5521240234375,
      "activations/layer3_attention_weight_max": 93.7836685180664,
      "activations/layer3_attention_weight_min": -92.33338165283203,
      "activations/layer4_attention_weight_max": 97.15633392333984,
      "activations/layer4_attention_weight_min": -92.09715270996094,
      "activations/layer5_attention_weight_max": 70.62774658203125,
      "activations/layer5_attention_weight_min": -78.7413101196289,
      "activations/layer6_attention_weight_max": 53.90116500854492,
      "activations/layer6_attention_weight_min": -51.63444519042969,
      "activations/layer7_attention_weight_max": 67.19744873046875,
      "activations/layer7_attention_weight_min": -67.87444305419922,
      "activations/layer8_attention_weight_max": 45.406070709228516,
      "activations/layer8_attention_weight_min": -46.205020904541016,
      "activations/layer9_attention_weight_max": 44.959320068359375,
      "activations/layer9_attention_weight_min": -45.255428314208984,
      "epoch": 12.7,
      "learning_rate": 6.876287878787877e-05,
      "loss": 2.7656,
      "step": 218650
    },
    {
      "activations/layer0_attention_weight_max": 15.812703132629395,
      "activations/layer0_attention_weight_min": -14.222489356994629,
      "activations/layer10_attention_weight_max": 37.5282096862793,
      "activations/layer10_attention_weight_min": -35.203529357910156,
      "activations/layer11_attention_weight_max": 35.26858901977539,
      "activations/layer11_attention_weight_min": -31.70648765563965,
      "activations/layer12_attention_weight_max": 18.614818572998047,
      "activations/layer12_attention_weight_min": -26.105894088745117,
      "activations/layer13_attention_weight_max": 38.0779914855957,
      "activations/layer13_attention_weight_min": -35.03330612182617,
      "activations/layer14_attention_weight_max": 44.319549560546875,
      "activations/layer14_attention_weight_min": -32.32001876831055,
      "activations/layer15_attention_weight_max": 39.15129852294922,
      "activations/layer15_attention_weight_min": -31.69916534423828,
      "activations/layer16_attention_weight_max": 32.592952728271484,
      "activations/layer16_attention_weight_min": -33.391136169433594,
      "activations/layer17_attention_weight_max": 50.005340576171875,
      "activations/layer17_attention_weight_min": -49.88835525512695,
      "activations/layer18_attention_weight_max": 45.80850601196289,
      "activations/layer18_attention_weight_min": -42.66664123535156,
      "activations/layer19_attention_weight_max": 23.54994010925293,
      "activations/layer19_attention_weight_min": -21.407428741455078,
      "activations/layer1_attention_weight_max": 16.314863204956055,
      "activations/layer1_attention_weight_min": -15.568115234375,
      "activations/layer20_attention_weight_max": 23.338499069213867,
      "activations/layer20_attention_weight_min": -22.14862632751465,
      "activations/layer21_attention_weight_max": 42.63654327392578,
      "activations/layer21_attention_weight_min": -29.521615982055664,
      "activations/layer22_attention_weight_max": 30.04124641418457,
      "activations/layer22_attention_weight_min": -27.2186336517334,
      "activations/layer23_attention_weight_max": 41.62268829345703,
      "activations/layer23_attention_weight_min": -25.508487701416016,
      "activations/layer2_attention_weight_max": 32.44438934326172,
      "activations/layer2_attention_weight_min": -31.5107421875,
      "activations/layer3_attention_weight_max": 94.0832748413086,
      "activations/layer3_attention_weight_min": -95.65673065185547,
      "activations/layer4_attention_weight_max": 100.20735931396484,
      "activations/layer4_attention_weight_min": -92.75470733642578,
      "activations/layer5_attention_weight_max": 71.13599395751953,
      "activations/layer5_attention_weight_min": -79.0020980834961,
      "activations/layer6_attention_weight_max": 51.307228088378906,
      "activations/layer6_attention_weight_min": -52.13779067993164,
      "activations/layer7_attention_weight_max": 70.3328628540039,
      "activations/layer7_attention_weight_min": -67.30297088623047,
      "activations/layer8_attention_weight_max": 46.25532150268555,
      "activations/layer8_attention_weight_min": -48.05024719238281,
      "activations/layer9_attention_weight_max": 50.68674850463867,
      "activations/layer9_attention_weight_min": -48.51415252685547,
      "epoch": 12.71,
      "learning_rate": 6.874393939393939e-05,
      "loss": 2.7695,
      "step": 218700
    },
    {
      "activations/layer0_attention_weight_max": 16.08143424987793,
      "activations/layer0_attention_weight_min": -13.94041633605957,
      "activations/layer10_attention_weight_max": 34.18487548828125,
      "activations/layer10_attention_weight_min": -34.005279541015625,
      "activations/layer11_attention_weight_max": 32.78091049194336,
      "activations/layer11_attention_weight_min": -32.8182258605957,
      "activations/layer12_attention_weight_max": 17.760507583618164,
      "activations/layer12_attention_weight_min": -25.4279842376709,
      "activations/layer13_attention_weight_max": 32.83613204956055,
      "activations/layer13_attention_weight_min": -28.09457778930664,
      "activations/layer14_attention_weight_max": 34.291473388671875,
      "activations/layer14_attention_weight_min": -29.194278717041016,
      "activations/layer15_attention_weight_max": 32.74976348876953,
      "activations/layer15_attention_weight_min": -28.621885299682617,
      "activations/layer16_attention_weight_max": 30.35979461669922,
      "activations/layer16_attention_weight_min": -30.170808792114258,
      "activations/layer17_attention_weight_max": 51.461429595947266,
      "activations/layer17_attention_weight_min": -42.15156936645508,
      "activations/layer18_attention_weight_max": 47.432674407958984,
      "activations/layer18_attention_weight_min": -37.760005950927734,
      "activations/layer19_attention_weight_max": 21.512662887573242,
      "activations/layer19_attention_weight_min": -23.206317901611328,
      "activations/layer1_attention_weight_max": 16.09329605102539,
      "activations/layer1_attention_weight_min": -16.011938095092773,
      "activations/layer20_attention_weight_max": 20.961925506591797,
      "activations/layer20_attention_weight_min": -25.77779197692871,
      "activations/layer21_attention_weight_max": 34.66339111328125,
      "activations/layer21_attention_weight_min": -21.1690731048584,
      "activations/layer22_attention_weight_max": 28.510087966918945,
      "activations/layer22_attention_weight_min": -25.91963768005371,
      "activations/layer23_attention_weight_max": 36.3275146484375,
      "activations/layer23_attention_weight_min": -26.766071319580078,
      "activations/layer2_attention_weight_max": 32.701316833496094,
      "activations/layer2_attention_weight_min": -30.661239624023438,
      "activations/layer3_attention_weight_max": 95.49388122558594,
      "activations/layer3_attention_weight_min": -91.75653839111328,
      "activations/layer4_attention_weight_max": 97.05644989013672,
      "activations/layer4_attention_weight_min": -91.26129150390625,
      "activations/layer5_attention_weight_max": 70.68321990966797,
      "activations/layer5_attention_weight_min": -78.83505249023438,
      "activations/layer6_attention_weight_max": 50.20814514160156,
      "activations/layer6_attention_weight_min": -52.137901306152344,
      "activations/layer7_attention_weight_max": 65.61023712158203,
      "activations/layer7_attention_weight_min": -68.35133361816406,
      "activations/layer8_attention_weight_max": 42.713138580322266,
      "activations/layer8_attention_weight_min": -47.85457992553711,
      "activations/layer9_attention_weight_max": 43.94873809814453,
      "activations/layer9_attention_weight_min": -44.30781936645508,
      "epoch": 12.71,
      "learning_rate": 6.872499999999999e-05,
      "loss": 2.7642,
      "step": 218750
    },
    {
      "activations/layer0_attention_weight_max": 17.15970230102539,
      "activations/layer0_attention_weight_min": -13.702192306518555,
      "activations/layer10_attention_weight_max": 35.903621673583984,
      "activations/layer10_attention_weight_min": -36.20557403564453,
      "activations/layer11_attention_weight_max": 31.15463638305664,
      "activations/layer11_attention_weight_min": -35.038639068603516,
      "activations/layer12_attention_weight_max": 18.232624053955078,
      "activations/layer12_attention_weight_min": -26.25238800048828,
      "activations/layer13_attention_weight_max": 40.76014709472656,
      "activations/layer13_attention_weight_min": -30.2525691986084,
      "activations/layer14_attention_weight_max": 34.38027572631836,
      "activations/layer14_attention_weight_min": -31.33309555053711,
      "activations/layer15_attention_weight_max": 31.778322219848633,
      "activations/layer15_attention_weight_min": -30.187519073486328,
      "activations/layer16_attention_weight_max": 30.910123825073242,
      "activations/layer16_attention_weight_min": -29.69135284423828,
      "activations/layer17_attention_weight_max": 49.499183654785156,
      "activations/layer17_attention_weight_min": -44.74862289428711,
      "activations/layer18_attention_weight_max": 45.66405487060547,
      "activations/layer18_attention_weight_min": -38.28181838989258,
      "activations/layer19_attention_weight_max": 20.665847778320312,
      "activations/layer19_attention_weight_min": -20.86922836303711,
      "activations/layer1_attention_weight_max": 16.531753540039062,
      "activations/layer1_attention_weight_min": -14.08995246887207,
      "activations/layer20_attention_weight_max": 20.843645095825195,
      "activations/layer20_attention_weight_min": -20.38880157470703,
      "activations/layer21_attention_weight_max": 40.55450439453125,
      "activations/layer21_attention_weight_min": -23.523183822631836,
      "activations/layer22_attention_weight_max": 28.755857467651367,
      "activations/layer22_attention_weight_min": -25.639408111572266,
      "activations/layer23_attention_weight_max": 34.513023376464844,
      "activations/layer23_attention_weight_min": -28.861427307128906,
      "activations/layer2_attention_weight_max": 33.582942962646484,
      "activations/layer2_attention_weight_min": -31.40785026550293,
      "activations/layer3_attention_weight_max": 101.01127624511719,
      "activations/layer3_attention_weight_min": -92.06040954589844,
      "activations/layer4_attention_weight_max": 98.59163665771484,
      "activations/layer4_attention_weight_min": -93.70178985595703,
      "activations/layer5_attention_weight_max": 72.62576293945312,
      "activations/layer5_attention_weight_min": -84.03538513183594,
      "activations/layer6_attention_weight_max": 54.98276138305664,
      "activations/layer6_attention_weight_min": -52.473384857177734,
      "activations/layer7_attention_weight_max": 69.71824645996094,
      "activations/layer7_attention_weight_min": -65.25260162353516,
      "activations/layer8_attention_weight_max": 46.925716400146484,
      "activations/layer8_attention_weight_min": -47.17134475708008,
      "activations/layer9_attention_weight_max": 47.14793395996094,
      "activations/layer9_attention_weight_min": -45.62190246582031,
      "epoch": 12.71,
      "learning_rate": 6.870606060606059e-05,
      "loss": 2.766,
      "step": 218800
    },
    {
      "activations/layer0_attention_weight_max": 16.66033935546875,
      "activations/layer0_attention_weight_min": -13.958870887756348,
      "activations/layer10_attention_weight_max": 37.00523376464844,
      "activations/layer10_attention_weight_min": -36.001617431640625,
      "activations/layer11_attention_weight_max": 31.78554344177246,
      "activations/layer11_attention_weight_min": -32.1568717956543,
      "activations/layer12_attention_weight_max": 20.36386489868164,
      "activations/layer12_attention_weight_min": -26.868934631347656,
      "activations/layer13_attention_weight_max": 35.65400695800781,
      "activations/layer13_attention_weight_min": -27.291574478149414,
      "activations/layer14_attention_weight_max": 39.26404571533203,
      "activations/layer14_attention_weight_min": -27.204700469970703,
      "activations/layer15_attention_weight_max": 34.2827033996582,
      "activations/layer15_attention_weight_min": -28.472990036010742,
      "activations/layer16_attention_weight_max": 33.23957824707031,
      "activations/layer16_attention_weight_min": -29.30051040649414,
      "activations/layer17_attention_weight_max": 51.947879791259766,
      "activations/layer17_attention_weight_min": -41.67771530151367,
      "activations/layer18_attention_weight_max": 47.64141082763672,
      "activations/layer18_attention_weight_min": -36.83723068237305,
      "activations/layer19_attention_weight_max": 22.222091674804688,
      "activations/layer19_attention_weight_min": -23.25580406188965,
      "activations/layer1_attention_weight_max": 17.791818618774414,
      "activations/layer1_attention_weight_min": -16.445096969604492,
      "activations/layer20_attention_weight_max": 23.043907165527344,
      "activations/layer20_attention_weight_min": -22.80038070678711,
      "activations/layer21_attention_weight_max": 37.8613166809082,
      "activations/layer21_attention_weight_min": -20.523122787475586,
      "activations/layer22_attention_weight_max": 29.283098220825195,
      "activations/layer22_attention_weight_min": -27.474531173706055,
      "activations/layer23_attention_weight_max": 39.24128723144531,
      "activations/layer23_attention_weight_min": -23.799488067626953,
      "activations/layer2_attention_weight_max": 33.34762191772461,
      "activations/layer2_attention_weight_min": -33.01213455200195,
      "activations/layer3_attention_weight_max": 94.96072387695312,
      "activations/layer3_attention_weight_min": -92.38178253173828,
      "activations/layer4_attention_weight_max": 94.61495208740234,
      "activations/layer4_attention_weight_min": -94.07972717285156,
      "activations/layer5_attention_weight_max": 70.26155090332031,
      "activations/layer5_attention_weight_min": -76.71282196044922,
      "activations/layer6_attention_weight_max": 51.483787536621094,
      "activations/layer6_attention_weight_min": -51.760337829589844,
      "activations/layer7_attention_weight_max": 72.70879364013672,
      "activations/layer7_attention_weight_min": -68.79067993164062,
      "activations/layer8_attention_weight_max": 44.69050979614258,
      "activations/layer8_attention_weight_min": -50.23063659667969,
      "activations/layer9_attention_weight_max": 48.07664489746094,
      "activations/layer9_attention_weight_min": -46.68876266479492,
      "epoch": 12.72,
      "learning_rate": 6.86871212121212e-05,
      "loss": 2.7612,
      "step": 218850
    },
    {
      "activations/layer0_attention_weight_max": 15.985891342163086,
      "activations/layer0_attention_weight_min": -13.882152557373047,
      "activations/layer10_attention_weight_max": 34.27559280395508,
      "activations/layer10_attention_weight_min": -35.74922180175781,
      "activations/layer11_attention_weight_max": 29.289138793945312,
      "activations/layer11_attention_weight_min": -33.547950744628906,
      "activations/layer12_attention_weight_max": 18.048614501953125,
      "activations/layer12_attention_weight_min": -27.84374237060547,
      "activations/layer13_attention_weight_max": 42.101585388183594,
      "activations/layer13_attention_weight_min": -32.50238800048828,
      "activations/layer14_attention_weight_max": 40.69807815551758,
      "activations/layer14_attention_weight_min": -30.989242553710938,
      "activations/layer15_attention_weight_max": 36.183441162109375,
      "activations/layer15_attention_weight_min": -30.0805721282959,
      "activations/layer16_attention_weight_max": 30.560773849487305,
      "activations/layer16_attention_weight_min": -28.875560760498047,
      "activations/layer17_attention_weight_max": 48.75634765625,
      "activations/layer17_attention_weight_min": -45.91918182373047,
      "activations/layer18_attention_weight_max": 44.927734375,
      "activations/layer18_attention_weight_min": -39.296600341796875,
      "activations/layer19_attention_weight_max": 20.107303619384766,
      "activations/layer19_attention_weight_min": -22.03866958618164,
      "activations/layer1_attention_weight_max": 16.806171417236328,
      "activations/layer1_attention_weight_min": -15.499232292175293,
      "activations/layer20_attention_weight_max": 20.73558235168457,
      "activations/layer20_attention_weight_min": -22.562402725219727,
      "activations/layer21_attention_weight_max": 34.749290466308594,
      "activations/layer21_attention_weight_min": -22.91011619567871,
      "activations/layer22_attention_weight_max": 30.98073387145996,
      "activations/layer22_attention_weight_min": -26.597566604614258,
      "activations/layer23_attention_weight_max": 38.55485153198242,
      "activations/layer23_attention_weight_min": -22.66901206970215,
      "activations/layer2_attention_weight_max": 33.64423751831055,
      "activations/layer2_attention_weight_min": -31.486417770385742,
      "activations/layer3_attention_weight_max": 94.33871459960938,
      "activations/layer3_attention_weight_min": -94.22260284423828,
      "activations/layer4_attention_weight_max": 95.39435577392578,
      "activations/layer4_attention_weight_min": -91.43620300292969,
      "activations/layer5_attention_weight_max": 71.0492935180664,
      "activations/layer5_attention_weight_min": -74.84801483154297,
      "activations/layer6_attention_weight_max": 54.37166976928711,
      "activations/layer6_attention_weight_min": -51.3421516418457,
      "activations/layer7_attention_weight_max": 69.8497085571289,
      "activations/layer7_attention_weight_min": -66.6653060913086,
      "activations/layer8_attention_weight_max": 44.03291320800781,
      "activations/layer8_attention_weight_min": -45.83949661254883,
      "activations/layer9_attention_weight_max": 44.385372161865234,
      "activations/layer9_attention_weight_min": -43.68528366088867,
      "epoch": 12.72,
      "learning_rate": 6.866818181818181e-05,
      "loss": 2.7776,
      "step": 218900
    },
    {
      "activations/layer0_attention_weight_max": 15.802814483642578,
      "activations/layer0_attention_weight_min": -14.452201843261719,
      "activations/layer10_attention_weight_max": 34.712493896484375,
      "activations/layer10_attention_weight_min": -36.20846176147461,
      "activations/layer11_attention_weight_max": 32.96478271484375,
      "activations/layer11_attention_weight_min": -34.37183380126953,
      "activations/layer12_attention_weight_max": 17.98207664489746,
      "activations/layer12_attention_weight_min": -27.25745391845703,
      "activations/layer13_attention_weight_max": 35.060157775878906,
      "activations/layer13_attention_weight_min": -28.40719985961914,
      "activations/layer14_attention_weight_max": 37.64246368408203,
      "activations/layer14_attention_weight_min": -31.008956909179688,
      "activations/layer15_attention_weight_max": 34.23500442504883,
      "activations/layer15_attention_weight_min": -30.860837936401367,
      "activations/layer16_attention_weight_max": 32.77236557006836,
      "activations/layer16_attention_weight_min": -30.155227661132812,
      "activations/layer17_attention_weight_max": 57.52066421508789,
      "activations/layer17_attention_weight_min": -47.06882858276367,
      "activations/layer18_attention_weight_max": 51.2726936340332,
      "activations/layer18_attention_weight_min": -38.133148193359375,
      "activations/layer19_attention_weight_max": 25.42716407775879,
      "activations/layer19_attention_weight_min": -25.98917579650879,
      "activations/layer1_attention_weight_max": 17.150074005126953,
      "activations/layer1_attention_weight_min": -15.547093391418457,
      "activations/layer20_attention_weight_max": 27.933237075805664,
      "activations/layer20_attention_weight_min": -23.958616256713867,
      "activations/layer21_attention_weight_max": 47.7702522277832,
      "activations/layer21_attention_weight_min": -23.8027286529541,
      "activations/layer22_attention_weight_max": 33.74269485473633,
      "activations/layer22_attention_weight_min": -27.79581069946289,
      "activations/layer23_attention_weight_max": 42.42592239379883,
      "activations/layer23_attention_weight_min": -22.962310791015625,
      "activations/layer2_attention_weight_max": 32.79656219482422,
      "activations/layer2_attention_weight_min": -32.830604553222656,
      "activations/layer3_attention_weight_max": 97.7109375,
      "activations/layer3_attention_weight_min": -104.3492202758789,
      "activations/layer4_attention_weight_max": 96.62511444091797,
      "activations/layer4_attention_weight_min": -95.96278381347656,
      "activations/layer5_attention_weight_max": 71.99137115478516,
      "activations/layer5_attention_weight_min": -81.91551971435547,
      "activations/layer6_attention_weight_max": 51.19778823852539,
      "activations/layer6_attention_weight_min": -52.00884246826172,
      "activations/layer7_attention_weight_max": 67.01237487792969,
      "activations/layer7_attention_weight_min": -66.53905487060547,
      "activations/layer8_attention_weight_max": 46.1252326965332,
      "activations/layer8_attention_weight_min": -45.33768844604492,
      "activations/layer9_attention_weight_max": 43.60169219970703,
      "activations/layer9_attention_weight_min": -45.415470123291016,
      "epoch": 12.72,
      "learning_rate": 6.864924242424242e-05,
      "loss": 2.7767,
      "step": 218950
    },
    {
      "activations/layer0_attention_weight_max": 16.434452056884766,
      "activations/layer0_attention_weight_min": -14.135978698730469,
      "activations/layer10_attention_weight_max": 35.15509033203125,
      "activations/layer10_attention_weight_min": -35.84998321533203,
      "activations/layer11_attention_weight_max": 31.95502281188965,
      "activations/layer11_attention_weight_min": -32.83404541015625,
      "activations/layer12_attention_weight_max": 19.806575775146484,
      "activations/layer12_attention_weight_min": -29.895158767700195,
      "activations/layer13_attention_weight_max": 37.30015563964844,
      "activations/layer13_attention_weight_min": -29.48877716064453,
      "activations/layer14_attention_weight_max": 39.24985122680664,
      "activations/layer14_attention_weight_min": -29.65280532836914,
      "activations/layer15_attention_weight_max": 37.34202194213867,
      "activations/layer15_attention_weight_min": -30.23813247680664,
      "activations/layer16_attention_weight_max": 32.47412872314453,
      "activations/layer16_attention_weight_min": -28.728412628173828,
      "activations/layer17_attention_weight_max": 53.48705291748047,
      "activations/layer17_attention_weight_min": -43.71674346923828,
      "activations/layer18_attention_weight_max": 49.11048889160156,
      "activations/layer18_attention_weight_min": -38.594818115234375,
      "activations/layer19_attention_weight_max": 24.52167510986328,
      "activations/layer19_attention_weight_min": -23.72911834716797,
      "activations/layer1_attention_weight_max": 16.471750259399414,
      "activations/layer1_attention_weight_min": -15.377320289611816,
      "activations/layer20_attention_weight_max": 22.602378845214844,
      "activations/layer20_attention_weight_min": -22.359020233154297,
      "activations/layer21_attention_weight_max": 54.25181198120117,
      "activations/layer21_attention_weight_min": -29.97203826904297,
      "activations/layer22_attention_weight_max": 29.8248348236084,
      "activations/layer22_attention_weight_min": -24.73484992980957,
      "activations/layer23_attention_weight_max": 37.00218200683594,
      "activations/layer23_attention_weight_min": -28.131271362304688,
      "activations/layer2_attention_weight_max": 32.97880935668945,
      "activations/layer2_attention_weight_min": -31.192977905273438,
      "activations/layer3_attention_weight_max": 96.3296127319336,
      "activations/layer3_attention_weight_min": -97.15699768066406,
      "activations/layer4_attention_weight_max": 97.07121276855469,
      "activations/layer4_attention_weight_min": -90.49807739257812,
      "activations/layer5_attention_weight_max": 69.43077087402344,
      "activations/layer5_attention_weight_min": -77.01622772216797,
      "activations/layer6_attention_weight_max": 49.84529495239258,
      "activations/layer6_attention_weight_min": -48.368675231933594,
      "activations/layer7_attention_weight_max": 67.86237335205078,
      "activations/layer7_attention_weight_min": -64.50274658203125,
      "activations/layer8_attention_weight_max": 43.2919807434082,
      "activations/layer8_attention_weight_min": -42.81028366088867,
      "activations/layer9_attention_weight_max": 46.65700912475586,
      "activations/layer9_attention_weight_min": -47.89149475097656,
      "epoch": 12.73,
      "learning_rate": 6.863030303030302e-05,
      "loss": 2.7469,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4754,
      "eval_samples_per_second": 506.645,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4754,
      "eval_openwebtext_samples_per_second": 506.645,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9767,
      "eval_wikitext_samples_per_second": 230.693,
      "step": 219000
    },
    {
      "epoch": 12.73,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.5584,
      "eval_lambada_samples_per_second": 509.393,
      "step": 219000
    },
    {
      "activations/layer0_attention_weight_max": 16.9683895111084,
      "activations/layer0_attention_weight_min": -14.148721694946289,
      "activations/layer10_attention_weight_max": 44.27848434448242,
      "activations/layer10_attention_weight_min": -41.5000114440918,
      "activations/layer11_attention_weight_max": 40.90373992919922,
      "activations/layer11_attention_weight_min": -36.484840393066406,
      "activations/layer12_attention_weight_max": 18.48484230041504,
      "activations/layer12_attention_weight_min": -26.41233253479004,
      "activations/layer13_attention_weight_max": 56.02473831176758,
      "activations/layer13_attention_weight_min": -31.438060760498047,
      "activations/layer14_attention_weight_max": 45.095619201660156,
      "activations/layer14_attention_weight_min": -32.9128532409668,
      "activations/layer15_attention_weight_max": 42.108360290527344,
      "activations/layer15_attention_weight_min": -30.33531379699707,
      "activations/layer16_attention_weight_max": 35.58414077758789,
      "activations/layer16_attention_weight_min": -29.580488204956055,
      "activations/layer17_attention_weight_max": 59.98948287963867,
      "activations/layer17_attention_weight_min": -48.19499588012695,
      "activations/layer18_attention_weight_max": 55.80620574951172,
      "activations/layer18_attention_weight_min": -43.10844421386719,
      "activations/layer19_attention_weight_max": 23.744386672973633,
      "activations/layer19_attention_weight_min": -19.601224899291992,
      "activations/layer1_attention_weight_max": 16.734655380249023,
      "activations/layer1_attention_weight_min": -15.85600757598877,
      "activations/layer20_attention_weight_max": 26.79149627685547,
      "activations/layer20_attention_weight_min": -22.258039474487305,
      "activations/layer21_attention_weight_max": 48.17679977416992,
      "activations/layer21_attention_weight_min": -24.122356414794922,
      "activations/layer22_attention_weight_max": 34.07431411743164,
      "activations/layer22_attention_weight_min": -24.50492286682129,
      "activations/layer23_attention_weight_max": 47.04235076904297,
      "activations/layer23_attention_weight_min": -23.099576950073242,
      "activations/layer2_attention_weight_max": 34.90700912475586,
      "activations/layer2_attention_weight_min": -34.304176330566406,
      "activations/layer3_attention_weight_max": 103.02482604980469,
      "activations/layer3_attention_weight_min": -99.89997863769531,
      "activations/layer4_attention_weight_max": 105.38258361816406,
      "activations/layer4_attention_weight_min": -101.05501556396484,
      "activations/layer5_attention_weight_max": 71.98650360107422,
      "activations/layer5_attention_weight_min": -82.27747344970703,
      "activations/layer6_attention_weight_max": 56.10569763183594,
      "activations/layer6_attention_weight_min": -55.8745002746582,
      "activations/layer7_attention_weight_max": 74.52788543701172,
      "activations/layer7_attention_weight_min": -84.43399810791016,
      "activations/layer8_attention_weight_max": 54.15027618408203,
      "activations/layer8_attention_weight_min": -54.74886703491211,
      "activations/layer9_attention_weight_max": 53.7203254699707,
      "activations/layer9_attention_weight_min": -56.65625,
      "epoch": 12.73,
      "learning_rate": 6.861136363636364e-05,
      "loss": 2.7552,
      "step": 219050
    },
    {
      "activations/layer0_attention_weight_max": 15.472994804382324,
      "activations/layer0_attention_weight_min": -13.844527244567871,
      "activations/layer10_attention_weight_max": 39.91443634033203,
      "activations/layer10_attention_weight_min": -38.34538269042969,
      "activations/layer11_attention_weight_max": 33.20005416870117,
      "activations/layer11_attention_weight_min": -34.67918014526367,
      "activations/layer12_attention_weight_max": 18.055923461914062,
      "activations/layer12_attention_weight_min": -23.345151901245117,
      "activations/layer13_attention_weight_max": 38.67396545410156,
      "activations/layer13_attention_weight_min": -29.085758209228516,
      "activations/layer14_attention_weight_max": 39.897369384765625,
      "activations/layer14_attention_weight_min": -30.1129150390625,
      "activations/layer15_attention_weight_max": 34.33292007446289,
      "activations/layer15_attention_weight_min": -30.233938217163086,
      "activations/layer16_attention_weight_max": 32.258914947509766,
      "activations/layer16_attention_weight_min": -29.59023094177246,
      "activations/layer17_attention_weight_max": 55.51051712036133,
      "activations/layer17_attention_weight_min": -45.32533645629883,
      "activations/layer18_attention_weight_max": 49.47201919555664,
      "activations/layer18_attention_weight_min": -39.04697036743164,
      "activations/layer19_attention_weight_max": 24.898517608642578,
      "activations/layer19_attention_weight_min": -22.900772094726562,
      "activations/layer1_attention_weight_max": 17.00619888305664,
      "activations/layer1_attention_weight_min": -15.605433464050293,
      "activations/layer20_attention_weight_max": 27.91665267944336,
      "activations/layer20_attention_weight_min": -21.721073150634766,
      "activations/layer21_attention_weight_max": 41.08978271484375,
      "activations/layer21_attention_weight_min": -23.01743507385254,
      "activations/layer22_attention_weight_max": 37.046260833740234,
      "activations/layer22_attention_weight_min": -25.2316951751709,
      "activations/layer23_attention_weight_max": 46.83826446533203,
      "activations/layer23_attention_weight_min": -24.24972915649414,
      "activations/layer2_attention_weight_max": 32.49040985107422,
      "activations/layer2_attention_weight_min": -31.69827651977539,
      "activations/layer3_attention_weight_max": 99.67404174804688,
      "activations/layer3_attention_weight_min": -100.1352767944336,
      "activations/layer4_attention_weight_max": 99.37723541259766,
      "activations/layer4_attention_weight_min": -95.98113250732422,
      "activations/layer5_attention_weight_max": 71.05853271484375,
      "activations/layer5_attention_weight_min": -77.17703247070312,
      "activations/layer6_attention_weight_max": 53.34420394897461,
      "activations/layer6_attention_weight_min": -52.03357696533203,
      "activations/layer7_attention_weight_max": 70.86693572998047,
      "activations/layer7_attention_weight_min": -69.19570922851562,
      "activations/layer8_attention_weight_max": 46.85017776489258,
      "activations/layer8_attention_weight_min": -49.163150787353516,
      "activations/layer9_attention_weight_max": 50.12684631347656,
      "activations/layer9_attention_weight_min": -50.441898345947266,
      "epoch": 12.73,
      "learning_rate": 6.859242424242424e-05,
      "loss": 2.7563,
      "step": 219100
    },
    {
      "activations/layer0_attention_weight_max": 16.07891082763672,
      "activations/layer0_attention_weight_min": -14.067304611206055,
      "activations/layer10_attention_weight_max": 35.986427307128906,
      "activations/layer10_attention_weight_min": -36.250328063964844,
      "activations/layer11_attention_weight_max": 30.71605682373047,
      "activations/layer11_attention_weight_min": -33.444862365722656,
      "activations/layer12_attention_weight_max": 17.32938003540039,
      "activations/layer12_attention_weight_min": -24.41675567626953,
      "activations/layer13_attention_weight_max": 32.509674072265625,
      "activations/layer13_attention_weight_min": -27.324800491333008,
      "activations/layer14_attention_weight_max": 32.85453796386719,
      "activations/layer14_attention_weight_min": -29.75438117980957,
      "activations/layer15_attention_weight_max": 31.625280380249023,
      "activations/layer15_attention_weight_min": -28.885047912597656,
      "activations/layer16_attention_weight_max": 30.190746307373047,
      "activations/layer16_attention_weight_min": -28.553592681884766,
      "activations/layer17_attention_weight_max": 48.64583206176758,
      "activations/layer17_attention_weight_min": -42.00470733642578,
      "activations/layer18_attention_weight_max": 41.8363151550293,
      "activations/layer18_attention_weight_min": -35.97808074951172,
      "activations/layer19_attention_weight_max": 21.197778701782227,
      "activations/layer19_attention_weight_min": -21.368886947631836,
      "activations/layer1_attention_weight_max": 16.900039672851562,
      "activations/layer1_attention_weight_min": -14.846566200256348,
      "activations/layer20_attention_weight_max": 21.131250381469727,
      "activations/layer20_attention_weight_min": -21.51399803161621,
      "activations/layer21_attention_weight_max": 39.51298904418945,
      "activations/layer21_attention_weight_min": -24.69645881652832,
      "activations/layer22_attention_weight_max": 29.10498046875,
      "activations/layer22_attention_weight_min": -26.040029525756836,
      "activations/layer23_attention_weight_max": 37.07185363769531,
      "activations/layer23_attention_weight_min": -27.75965690612793,
      "activations/layer2_attention_weight_max": 32.64357376098633,
      "activations/layer2_attention_weight_min": -30.968753814697266,
      "activations/layer3_attention_weight_max": 93.9161605834961,
      "activations/layer3_attention_weight_min": -94.81149291992188,
      "activations/layer4_attention_weight_max": 100.45602416992188,
      "activations/layer4_attention_weight_min": -97.53694915771484,
      "activations/layer5_attention_weight_max": 75.06243133544922,
      "activations/layer5_attention_weight_min": -80.27139282226562,
      "activations/layer6_attention_weight_max": 53.49042892456055,
      "activations/layer6_attention_weight_min": -54.85219955444336,
      "activations/layer7_attention_weight_max": 73.32726287841797,
      "activations/layer7_attention_weight_min": -66.67369079589844,
      "activations/layer8_attention_weight_max": 47.60087203979492,
      "activations/layer8_attention_weight_min": -48.005802154541016,
      "activations/layer9_attention_weight_max": 48.03800582885742,
      "activations/layer9_attention_weight_min": -46.486576080322266,
      "epoch": 12.73,
      "learning_rate": 6.857348484848484e-05,
      "loss": 2.7576,
      "step": 219150
    },
    {
      "activations/layer0_attention_weight_max": 16.064125061035156,
      "activations/layer0_attention_weight_min": -13.951557159423828,
      "activations/layer10_attention_weight_max": 32.15367126464844,
      "activations/layer10_attention_weight_min": -34.04039764404297,
      "activations/layer11_attention_weight_max": 30.08711814880371,
      "activations/layer11_attention_weight_min": -32.080047607421875,
      "activations/layer12_attention_weight_max": 17.515514373779297,
      "activations/layer12_attention_weight_min": -26.166383743286133,
      "activations/layer13_attention_weight_max": 35.41633605957031,
      "activations/layer13_attention_weight_min": -30.57834815979004,
      "activations/layer14_attention_weight_max": 33.78163528442383,
      "activations/layer14_attention_weight_min": -28.494096755981445,
      "activations/layer15_attention_weight_max": 30.514911651611328,
      "activations/layer15_attention_weight_min": -28.308385848999023,
      "activations/layer16_attention_weight_max": 29.23993492126465,
      "activations/layer16_attention_weight_min": -28.803512573242188,
      "activations/layer17_attention_weight_max": 49.023616790771484,
      "activations/layer17_attention_weight_min": -42.69688034057617,
      "activations/layer18_attention_weight_max": 43.99067306518555,
      "activations/layer18_attention_weight_min": -37.080169677734375,
      "activations/layer19_attention_weight_max": 21.10896110534668,
      "activations/layer19_attention_weight_min": -22.929927825927734,
      "activations/layer1_attention_weight_max": 16.46309471130371,
      "activations/layer1_attention_weight_min": -16.712114334106445,
      "activations/layer20_attention_weight_max": 21.583755493164062,
      "activations/layer20_attention_weight_min": -22.2677059173584,
      "activations/layer21_attention_weight_max": 41.176395416259766,
      "activations/layer21_attention_weight_min": -22.64120864868164,
      "activations/layer22_attention_weight_max": 28.225481033325195,
      "activations/layer22_attention_weight_min": -25.18063735961914,
      "activations/layer23_attention_weight_max": 35.7081184387207,
      "activations/layer23_attention_weight_min": -25.768918991088867,
      "activations/layer2_attention_weight_max": 35.71525192260742,
      "activations/layer2_attention_weight_min": -31.595853805541992,
      "activations/layer3_attention_weight_max": 100.54423522949219,
      "activations/layer3_attention_weight_min": -94.61991882324219,
      "activations/layer4_attention_weight_max": 98.00959777832031,
      "activations/layer4_attention_weight_min": -94.72088623046875,
      "activations/layer5_attention_weight_max": 69.63228607177734,
      "activations/layer5_attention_weight_min": -80.04988098144531,
      "activations/layer6_attention_weight_max": 51.19328308105469,
      "activations/layer6_attention_weight_min": -51.51999282836914,
      "activations/layer7_attention_weight_max": 76.66970825195312,
      "activations/layer7_attention_weight_min": -71.39494323730469,
      "activations/layer8_attention_weight_max": 43.86610412597656,
      "activations/layer8_attention_weight_min": -45.525550842285156,
      "activations/layer9_attention_weight_max": 46.1442756652832,
      "activations/layer9_attention_weight_min": -44.2519645690918,
      "epoch": 12.74,
      "learning_rate": 6.855454545454544e-05,
      "loss": 2.7749,
      "step": 219200
    },
    {
      "activations/layer0_attention_weight_max": 16.385915756225586,
      "activations/layer0_attention_weight_min": -14.179007530212402,
      "activations/layer10_attention_weight_max": 35.92946243286133,
      "activations/layer10_attention_weight_min": -34.97035598754883,
      "activations/layer11_attention_weight_max": 31.944807052612305,
      "activations/layer11_attention_weight_min": -33.74855041503906,
      "activations/layer12_attention_weight_max": 19.706186294555664,
      "activations/layer12_attention_weight_min": -22.954309463500977,
      "activations/layer13_attention_weight_max": 41.89141082763672,
      "activations/layer13_attention_weight_min": -33.01494216918945,
      "activations/layer14_attention_weight_max": 38.7879753112793,
      "activations/layer14_attention_weight_min": -30.98386001586914,
      "activations/layer15_attention_weight_max": 35.585968017578125,
      "activations/layer15_attention_weight_min": -29.540565490722656,
      "activations/layer16_attention_weight_max": 33.86237335205078,
      "activations/layer16_attention_weight_min": -31.708736419677734,
      "activations/layer17_attention_weight_max": 57.84265899658203,
      "activations/layer17_attention_weight_min": -45.858360290527344,
      "activations/layer18_attention_weight_max": 52.008460998535156,
      "activations/layer18_attention_weight_min": -38.13225555419922,
      "activations/layer19_attention_weight_max": 26.759008407592773,
      "activations/layer19_attention_weight_min": -21.724958419799805,
      "activations/layer1_attention_weight_max": 16.670625686645508,
      "activations/layer1_attention_weight_min": -14.440784454345703,
      "activations/layer20_attention_weight_max": 24.541608810424805,
      "activations/layer20_attention_weight_min": -22.074026107788086,
      "activations/layer21_attention_weight_max": 35.093902587890625,
      "activations/layer21_attention_weight_min": -24.9625244140625,
      "activations/layer22_attention_weight_max": 35.6108283996582,
      "activations/layer22_attention_weight_min": -25.838777542114258,
      "activations/layer23_attention_weight_max": 37.1308479309082,
      "activations/layer23_attention_weight_min": -24.851459503173828,
      "activations/layer2_attention_weight_max": 33.5473747253418,
      "activations/layer2_attention_weight_min": -31.612768173217773,
      "activations/layer3_attention_weight_max": 96.25215911865234,
      "activations/layer3_attention_weight_min": -95.24783325195312,
      "activations/layer4_attention_weight_max": 94.8399887084961,
      "activations/layer4_attention_weight_min": -94.84324645996094,
      "activations/layer5_attention_weight_max": 68.99615478515625,
      "activations/layer5_attention_weight_min": -76.40674591064453,
      "activations/layer6_attention_weight_max": 51.039730072021484,
      "activations/layer6_attention_weight_min": -51.72071838378906,
      "activations/layer7_attention_weight_max": 67.07069396972656,
      "activations/layer7_attention_weight_min": -65.52271270751953,
      "activations/layer8_attention_weight_max": 46.33794403076172,
      "activations/layer8_attention_weight_min": -47.8660888671875,
      "activations/layer9_attention_weight_max": 45.149749755859375,
      "activations/layer9_attention_weight_min": -45.00379180908203,
      "epoch": 12.74,
      "learning_rate": 6.853560606060606e-05,
      "loss": 2.7927,
      "step": 219250
    },
    {
      "activations/layer0_attention_weight_max": 16.134777069091797,
      "activations/layer0_attention_weight_min": -14.1419095993042,
      "activations/layer10_attention_weight_max": 36.46295166015625,
      "activations/layer10_attention_weight_min": -36.914974212646484,
      "activations/layer11_attention_weight_max": 32.203495025634766,
      "activations/layer11_attention_weight_min": -36.31587219238281,
      "activations/layer12_attention_weight_max": 18.86174964904785,
      "activations/layer12_attention_weight_min": -27.349760055541992,
      "activations/layer13_attention_weight_max": 33.94917297363281,
      "activations/layer13_attention_weight_min": -30.46822738647461,
      "activations/layer14_attention_weight_max": 37.089962005615234,
      "activations/layer14_attention_weight_min": -31.87053871154785,
      "activations/layer15_attention_weight_max": 35.13666534423828,
      "activations/layer15_attention_weight_min": -32.827476501464844,
      "activations/layer16_attention_weight_max": 35.264747619628906,
      "activations/layer16_attention_weight_min": -31.247379302978516,
      "activations/layer17_attention_weight_max": 61.27008056640625,
      "activations/layer17_attention_weight_min": -49.23567581176758,
      "activations/layer18_attention_weight_max": 54.86447525024414,
      "activations/layer18_attention_weight_min": -39.9642333984375,
      "activations/layer19_attention_weight_max": 23.675947189331055,
      "activations/layer19_attention_weight_min": -21.861080169677734,
      "activations/layer1_attention_weight_max": 17.84496307373047,
      "activations/layer1_attention_weight_min": -14.087601661682129,
      "activations/layer20_attention_weight_max": 22.92662811279297,
      "activations/layer20_attention_weight_min": -20.278841018676758,
      "activations/layer21_attention_weight_max": 35.812068939208984,
      "activations/layer21_attention_weight_min": -24.654979705810547,
      "activations/layer22_attention_weight_max": 30.09833526611328,
      "activations/layer22_attention_weight_min": -25.67298126220703,
      "activations/layer23_attention_weight_max": 36.29743576049805,
      "activations/layer23_attention_weight_min": -24.210926055908203,
      "activations/layer2_attention_weight_max": 32.27471923828125,
      "activations/layer2_attention_weight_min": -32.5472526550293,
      "activations/layer3_attention_weight_max": 93.4576416015625,
      "activations/layer3_attention_weight_min": -101.5455093383789,
      "activations/layer4_attention_weight_max": 96.92432403564453,
      "activations/layer4_attention_weight_min": -95.49149322509766,
      "activations/layer5_attention_weight_max": 71.3196792602539,
      "activations/layer5_attention_weight_min": -78.44239807128906,
      "activations/layer6_attention_weight_max": 51.84855651855469,
      "activations/layer6_attention_weight_min": -54.699684143066406,
      "activations/layer7_attention_weight_max": 71.93517303466797,
      "activations/layer7_attention_weight_min": -71.83268737792969,
      "activations/layer8_attention_weight_max": 44.65066909790039,
      "activations/layer8_attention_weight_min": -47.47092056274414,
      "activations/layer9_attention_weight_max": 46.95270919799805,
      "activations/layer9_attention_weight_min": -46.77657699584961,
      "epoch": 12.74,
      "learning_rate": 6.851666666666666e-05,
      "loss": 2.7605,
      "step": 219300
    },
    {
      "activations/layer0_attention_weight_max": 17.244871139526367,
      "activations/layer0_attention_weight_min": -13.929865837097168,
      "activations/layer10_attention_weight_max": 36.49037170410156,
      "activations/layer10_attention_weight_min": -35.11072540283203,
      "activations/layer11_attention_weight_max": 33.96525192260742,
      "activations/layer11_attention_weight_min": -33.84193420410156,
      "activations/layer12_attention_weight_max": 19.134654998779297,
      "activations/layer12_attention_weight_min": -24.04302406311035,
      "activations/layer13_attention_weight_max": 41.28347396850586,
      "activations/layer13_attention_weight_min": -28.17958641052246,
      "activations/layer14_attention_weight_max": 36.55394744873047,
      "activations/layer14_attention_weight_min": -32.19123077392578,
      "activations/layer15_attention_weight_max": 37.081871032714844,
      "activations/layer15_attention_weight_min": -32.08631896972656,
      "activations/layer16_attention_weight_max": 31.854991912841797,
      "activations/layer16_attention_weight_min": -30.7510986328125,
      "activations/layer17_attention_weight_max": 50.60225296020508,
      "activations/layer17_attention_weight_min": -44.55412673950195,
      "activations/layer18_attention_weight_max": 44.57223129272461,
      "activations/layer18_attention_weight_min": -39.31715393066406,
      "activations/layer19_attention_weight_max": 19.952295303344727,
      "activations/layer19_attention_weight_min": -22.322725296020508,
      "activations/layer1_attention_weight_max": 16.5429630279541,
      "activations/layer1_attention_weight_min": -15.559395790100098,
      "activations/layer20_attention_weight_max": 21.318660736083984,
      "activations/layer20_attention_weight_min": -23.63960838317871,
      "activations/layer21_attention_weight_max": 35.36609649658203,
      "activations/layer21_attention_weight_min": -20.999988555908203,
      "activations/layer22_attention_weight_max": 33.63785934448242,
      "activations/layer22_attention_weight_min": -24.280080795288086,
      "activations/layer23_attention_weight_max": 38.482704162597656,
      "activations/layer23_attention_weight_min": -21.271821975708008,
      "activations/layer2_attention_weight_max": 32.617671966552734,
      "activations/layer2_attention_weight_min": -31.217315673828125,
      "activations/layer3_attention_weight_max": 93.72374725341797,
      "activations/layer3_attention_weight_min": -99.58735656738281,
      "activations/layer4_attention_weight_max": 98.76371765136719,
      "activations/layer4_attention_weight_min": -92.92386627197266,
      "activations/layer5_attention_weight_max": 72.36251831054688,
      "activations/layer5_attention_weight_min": -77.71630859375,
      "activations/layer6_attention_weight_max": 52.3138542175293,
      "activations/layer6_attention_weight_min": -50.64891815185547,
      "activations/layer7_attention_weight_max": 67.00444793701172,
      "activations/layer7_attention_weight_min": -66.67242431640625,
      "activations/layer8_attention_weight_max": 45.25494384765625,
      "activations/layer8_attention_weight_min": -47.39201736450195,
      "activations/layer9_attention_weight_max": 43.95748519897461,
      "activations/layer9_attention_weight_min": -45.749202728271484,
      "epoch": 12.75,
      "learning_rate": 6.849810606060605e-05,
      "loss": 2.7623,
      "step": 219350
    },
    {
      "activations/layer0_attention_weight_max": 16.107013702392578,
      "activations/layer0_attention_weight_min": -14.38011360168457,
      "activations/layer10_attention_weight_max": 41.49708557128906,
      "activations/layer10_attention_weight_min": -40.70819854736328,
      "activations/layer11_attention_weight_max": 40.58333206176758,
      "activations/layer11_attention_weight_min": -38.444129943847656,
      "activations/layer12_attention_weight_max": 17.96876335144043,
      "activations/layer12_attention_weight_min": -24.5531005859375,
      "activations/layer13_attention_weight_max": 36.12834167480469,
      "activations/layer13_attention_weight_min": -28.790014266967773,
      "activations/layer14_attention_weight_max": 39.78466033935547,
      "activations/layer14_attention_weight_min": -31.894960403442383,
      "activations/layer15_attention_weight_max": 42.97651672363281,
      "activations/layer15_attention_weight_min": -32.13701248168945,
      "activations/layer16_attention_weight_max": 32.512752532958984,
      "activations/layer16_attention_weight_min": -31.970659255981445,
      "activations/layer17_attention_weight_max": 56.82216262817383,
      "activations/layer17_attention_weight_min": -44.33314895629883,
      "activations/layer18_attention_weight_max": 45.39011001586914,
      "activations/layer18_attention_weight_min": -39.67942428588867,
      "activations/layer19_attention_weight_max": 23.21226692199707,
      "activations/layer19_attention_weight_min": -23.66259765625,
      "activations/layer1_attention_weight_max": 16.48583221435547,
      "activations/layer1_attention_weight_min": -15.949370384216309,
      "activations/layer20_attention_weight_max": 21.15214729309082,
      "activations/layer20_attention_weight_min": -23.794218063354492,
      "activations/layer21_attention_weight_max": 41.811893463134766,
      "activations/layer21_attention_weight_min": -23.310470581054688,
      "activations/layer22_attention_weight_max": 28.964075088500977,
      "activations/layer22_attention_weight_min": -27.072988510131836,
      "activations/layer23_attention_weight_max": 35.1546745300293,
      "activations/layer23_attention_weight_min": -22.791725158691406,
      "activations/layer2_attention_weight_max": 33.88037872314453,
      "activations/layer2_attention_weight_min": -32.12194061279297,
      "activations/layer3_attention_weight_max": 99.46662902832031,
      "activations/layer3_attention_weight_min": -100.58352661132812,
      "activations/layer4_attention_weight_max": 98.53575134277344,
      "activations/layer4_attention_weight_min": -96.4189682006836,
      "activations/layer5_attention_weight_max": 72.8416748046875,
      "activations/layer5_attention_weight_min": -74.0855484008789,
      "activations/layer6_attention_weight_max": 54.25257110595703,
      "activations/layer6_attention_weight_min": -53.07503890991211,
      "activations/layer7_attention_weight_max": 73.54669952392578,
      "activations/layer7_attention_weight_min": -68.19119262695312,
      "activations/layer8_attention_weight_max": 50.00424575805664,
      "activations/layer8_attention_weight_min": -48.306549072265625,
      "activations/layer9_attention_weight_max": 61.09534454345703,
      "activations/layer9_attention_weight_min": -50.1203498840332,
      "epoch": 12.75,
      "learning_rate": 6.847916666666665e-05,
      "loss": 2.7644,
      "step": 219400
    },
    {
      "activations/layer0_attention_weight_max": 15.528099060058594,
      "activations/layer0_attention_weight_min": -14.110241889953613,
      "activations/layer10_attention_weight_max": 36.371498107910156,
      "activations/layer10_attention_weight_min": -35.61198043823242,
      "activations/layer11_attention_weight_max": 31.925884246826172,
      "activations/layer11_attention_weight_min": -32.543983459472656,
      "activations/layer12_attention_weight_max": 17.437395095825195,
      "activations/layer12_attention_weight_min": -23.193586349487305,
      "activations/layer13_attention_weight_max": 35.767601013183594,
      "activations/layer13_attention_weight_min": -27.50821304321289,
      "activations/layer14_attention_weight_max": 35.54176712036133,
      "activations/layer14_attention_weight_min": -30.520797729492188,
      "activations/layer15_attention_weight_max": 31.730079650878906,
      "activations/layer15_attention_weight_min": -28.35379981994629,
      "activations/layer16_attention_weight_max": 28.979053497314453,
      "activations/layer16_attention_weight_min": -28.808990478515625,
      "activations/layer17_attention_weight_max": 51.702457427978516,
      "activations/layer17_attention_weight_min": -42.3601188659668,
      "activations/layer18_attention_weight_max": 45.48435592651367,
      "activations/layer18_attention_weight_min": -36.24709701538086,
      "activations/layer19_attention_weight_max": 23.408308029174805,
      "activations/layer19_attention_weight_min": -21.418392181396484,
      "activations/layer1_attention_weight_max": 16.499433517456055,
      "activations/layer1_attention_weight_min": -15.71909236907959,
      "activations/layer20_attention_weight_max": 21.042512893676758,
      "activations/layer20_attention_weight_min": -22.093399047851562,
      "activations/layer21_attention_weight_max": 34.53927230834961,
      "activations/layer21_attention_weight_min": -26.257902145385742,
      "activations/layer22_attention_weight_max": 26.101430892944336,
      "activations/layer22_attention_weight_min": -25.2885799407959,
      "activations/layer23_attention_weight_max": 33.57756042480469,
      "activations/layer23_attention_weight_min": -25.534467697143555,
      "activations/layer2_attention_weight_max": 31.86144256591797,
      "activations/layer2_attention_weight_min": -31.80954360961914,
      "activations/layer3_attention_weight_max": 96.1998519897461,
      "activations/layer3_attention_weight_min": -99.25817108154297,
      "activations/layer4_attention_weight_max": 100.99195861816406,
      "activations/layer4_attention_weight_min": -95.759033203125,
      "activations/layer5_attention_weight_max": 70.88793182373047,
      "activations/layer5_attention_weight_min": -78.43034362792969,
      "activations/layer6_attention_weight_max": 53.37773513793945,
      "activations/layer6_attention_weight_min": -53.219886779785156,
      "activations/layer7_attention_weight_max": 64.75611877441406,
      "activations/layer7_attention_weight_min": -65.59799194335938,
      "activations/layer8_attention_weight_max": 45.63100051879883,
      "activations/layer8_attention_weight_min": -44.8115234375,
      "activations/layer9_attention_weight_max": 45.832275390625,
      "activations/layer9_attention_weight_min": -43.96451187133789,
      "epoch": 12.75,
      "learning_rate": 6.846022727272727e-05,
      "loss": 2.7682,
      "step": 219450
    },
    {
      "activations/layer0_attention_weight_max": 15.859105110168457,
      "activations/layer0_attention_weight_min": -14.077706336975098,
      "activations/layer10_attention_weight_max": 39.57537078857422,
      "activations/layer10_attention_weight_min": -40.336082458496094,
      "activations/layer11_attention_weight_max": 33.88774490356445,
      "activations/layer11_attention_weight_min": -34.36494064331055,
      "activations/layer12_attention_weight_max": 23.84310531616211,
      "activations/layer12_attention_weight_min": -27.340492248535156,
      "activations/layer13_attention_weight_max": 39.26833724975586,
      "activations/layer13_attention_weight_min": -27.901809692382812,
      "activations/layer14_attention_weight_max": 37.48738098144531,
      "activations/layer14_attention_weight_min": -30.12822151184082,
      "activations/layer15_attention_weight_max": 31.88802146911621,
      "activations/layer15_attention_weight_min": -28.32520294189453,
      "activations/layer16_attention_weight_max": 31.695995330810547,
      "activations/layer16_attention_weight_min": -29.544193267822266,
      "activations/layer17_attention_weight_max": 51.49635696411133,
      "activations/layer17_attention_weight_min": -45.25475311279297,
      "activations/layer18_attention_weight_max": 51.71235275268555,
      "activations/layer18_attention_weight_min": -36.961181640625,
      "activations/layer19_attention_weight_max": 24.37822723388672,
      "activations/layer19_attention_weight_min": -22.01983642578125,
      "activations/layer1_attention_weight_max": 16.528154373168945,
      "activations/layer1_attention_weight_min": -14.860285758972168,
      "activations/layer20_attention_weight_max": 24.502708435058594,
      "activations/layer20_attention_weight_min": -19.517974853515625,
      "activations/layer21_attention_weight_max": 35.31608200073242,
      "activations/layer21_attention_weight_min": -20.18051528930664,
      "activations/layer22_attention_weight_max": 34.729881286621094,
      "activations/layer22_attention_weight_min": -23.602752685546875,
      "activations/layer23_attention_weight_max": 38.100128173828125,
      "activations/layer23_attention_weight_min": -22.579303741455078,
      "activations/layer2_attention_weight_max": 33.42787551879883,
      "activations/layer2_attention_weight_min": -30.486181259155273,
      "activations/layer3_attention_weight_max": 96.75274658203125,
      "activations/layer3_attention_weight_min": -99.55074310302734,
      "activations/layer4_attention_weight_max": 99.10626983642578,
      "activations/layer4_attention_weight_min": -100.672607421875,
      "activations/layer5_attention_weight_max": 71.84669494628906,
      "activations/layer5_attention_weight_min": -77.75917053222656,
      "activations/layer6_attention_weight_max": 52.653865814208984,
      "activations/layer6_attention_weight_min": -54.30793762207031,
      "activations/layer7_attention_weight_max": 68.96639251708984,
      "activations/layer7_attention_weight_min": -65.37420654296875,
      "activations/layer8_attention_weight_max": 47.74152374267578,
      "activations/layer8_attention_weight_min": -49.72220230102539,
      "activations/layer9_attention_weight_max": 50.3591194152832,
      "activations/layer9_attention_weight_min": -51.44989013671875,
      "epoch": 12.75,
      "learning_rate": 6.844128787878787e-05,
      "loss": 2.7719,
      "step": 219500
    },
    {
      "activations/layer0_attention_weight_max": 16.19036102294922,
      "activations/layer0_attention_weight_min": -14.068982124328613,
      "activations/layer10_attention_weight_max": 41.58857345581055,
      "activations/layer10_attention_weight_min": -38.18696975708008,
      "activations/layer11_attention_weight_max": 35.07810592651367,
      "activations/layer11_attention_weight_min": -34.81645202636719,
      "activations/layer12_attention_weight_max": 23.17909049987793,
      "activations/layer12_attention_weight_min": -25.279895782470703,
      "activations/layer13_attention_weight_max": 38.568702697753906,
      "activations/layer13_attention_weight_min": -30.042863845825195,
      "activations/layer14_attention_weight_max": 34.38028335571289,
      "activations/layer14_attention_weight_min": -30.93939208984375,
      "activations/layer15_attention_weight_max": 32.98556900024414,
      "activations/layer15_attention_weight_min": -27.394184112548828,
      "activations/layer16_attention_weight_max": 32.57814025878906,
      "activations/layer16_attention_weight_min": -29.600526809692383,
      "activations/layer17_attention_weight_max": 55.813751220703125,
      "activations/layer17_attention_weight_min": -43.52901077270508,
      "activations/layer18_attention_weight_max": 45.67034149169922,
      "activations/layer18_attention_weight_min": -36.34939956665039,
      "activations/layer19_attention_weight_max": 23.88608169555664,
      "activations/layer19_attention_weight_min": -21.448144912719727,
      "activations/layer1_attention_weight_max": 17.331026077270508,
      "activations/layer1_attention_weight_min": -15.682619094848633,
      "activations/layer20_attention_weight_max": 21.052492141723633,
      "activations/layer20_attention_weight_min": -21.774852752685547,
      "activations/layer21_attention_weight_max": 34.989479064941406,
      "activations/layer21_attention_weight_min": -21.5479679107666,
      "activations/layer22_attention_weight_max": 29.177175521850586,
      "activations/layer22_attention_weight_min": -22.323078155517578,
      "activations/layer23_attention_weight_max": 35.79175567626953,
      "activations/layer23_attention_weight_min": -21.46785545349121,
      "activations/layer2_attention_weight_max": 32.99809265136719,
      "activations/layer2_attention_weight_min": -31.74966049194336,
      "activations/layer3_attention_weight_max": 96.13409423828125,
      "activations/layer3_attention_weight_min": -98.35033416748047,
      "activations/layer4_attention_weight_max": 95.43690490722656,
      "activations/layer4_attention_weight_min": -93.15438842773438,
      "activations/layer5_attention_weight_max": 72.72061157226562,
      "activations/layer5_attention_weight_min": -75.85291290283203,
      "activations/layer6_attention_weight_max": 52.53950881958008,
      "activations/layer6_attention_weight_min": -53.26978302001953,
      "activations/layer7_attention_weight_max": 68.17279052734375,
      "activations/layer7_attention_weight_min": -67.48649597167969,
      "activations/layer8_attention_weight_max": 46.224178314208984,
      "activations/layer8_attention_weight_min": -49.278770446777344,
      "activations/layer9_attention_weight_max": 48.963985443115234,
      "activations/layer9_attention_weight_min": -46.90153884887695,
      "epoch": 12.76,
      "learning_rate": 6.842234848484847e-05,
      "loss": 2.7723,
      "step": 219550
    },
    {
      "activations/layer0_attention_weight_max": 17.400362014770508,
      "activations/layer0_attention_weight_min": -13.947957038879395,
      "activations/layer10_attention_weight_max": 36.91693115234375,
      "activations/layer10_attention_weight_min": -36.115413665771484,
      "activations/layer11_attention_weight_max": 34.544071197509766,
      "activations/layer11_attention_weight_min": -35.23518371582031,
      "activations/layer12_attention_weight_max": 21.03219223022461,
      "activations/layer12_attention_weight_min": -28.762189865112305,
      "activations/layer13_attention_weight_max": 49.76978302001953,
      "activations/layer13_attention_weight_min": -33.51282501220703,
      "activations/layer14_attention_weight_max": 38.10313034057617,
      "activations/layer14_attention_weight_min": -30.280908584594727,
      "activations/layer15_attention_weight_max": 34.93074035644531,
      "activations/layer15_attention_weight_min": -29.817853927612305,
      "activations/layer16_attention_weight_max": 32.64017105102539,
      "activations/layer16_attention_weight_min": -30.42052459716797,
      "activations/layer17_attention_weight_max": 55.02225112915039,
      "activations/layer17_attention_weight_min": -45.06496047973633,
      "activations/layer18_attention_weight_max": 48.7785758972168,
      "activations/layer18_attention_weight_min": -39.04536056518555,
      "activations/layer19_attention_weight_max": 21.940044403076172,
      "activations/layer19_attention_weight_min": -21.696706771850586,
      "activations/layer1_attention_weight_max": 16.78946876525879,
      "activations/layer1_attention_weight_min": -14.357757568359375,
      "activations/layer20_attention_weight_max": 22.491405487060547,
      "activations/layer20_attention_weight_min": -23.286462783813477,
      "activations/layer21_attention_weight_max": 38.044891357421875,
      "activations/layer21_attention_weight_min": -23.315319061279297,
      "activations/layer22_attention_weight_max": 33.60745620727539,
      "activations/layer22_attention_weight_min": -26.048534393310547,
      "activations/layer23_attention_weight_max": 39.17847442626953,
      "activations/layer23_attention_weight_min": -26.577247619628906,
      "activations/layer2_attention_weight_max": 33.780296325683594,
      "activations/layer2_attention_weight_min": -31.31198501586914,
      "activations/layer3_attention_weight_max": 98.07685852050781,
      "activations/layer3_attention_weight_min": -97.86646270751953,
      "activations/layer4_attention_weight_max": 102.17459106445312,
      "activations/layer4_attention_weight_min": -98.03968048095703,
      "activations/layer5_attention_weight_max": 72.80374145507812,
      "activations/layer5_attention_weight_min": -81.63565826416016,
      "activations/layer6_attention_weight_max": 55.99088668823242,
      "activations/layer6_attention_weight_min": -56.07099151611328,
      "activations/layer7_attention_weight_max": 66.21053314208984,
      "activations/layer7_attention_weight_min": -69.6804428100586,
      "activations/layer8_attention_weight_max": 46.38408279418945,
      "activations/layer8_attention_weight_min": -50.26047897338867,
      "activations/layer9_attention_weight_max": 49.45995330810547,
      "activations/layer9_attention_weight_min": -51.06819152832031,
      "epoch": 12.76,
      "learning_rate": 6.840340909090909e-05,
      "loss": 2.7611,
      "step": 219600
    },
    {
      "activations/layer0_attention_weight_max": 16.213247299194336,
      "activations/layer0_attention_weight_min": -14.390270233154297,
      "activations/layer10_attention_weight_max": 38.472320556640625,
      "activations/layer10_attention_weight_min": -36.396034240722656,
      "activations/layer11_attention_weight_max": 34.88212585449219,
      "activations/layer11_attention_weight_min": -34.28889465332031,
      "activations/layer12_attention_weight_max": 19.802005767822266,
      "activations/layer12_attention_weight_min": -24.921443939208984,
      "activations/layer13_attention_weight_max": 41.82635498046875,
      "activations/layer13_attention_weight_min": -30.614465713500977,
      "activations/layer14_attention_weight_max": 40.80747985839844,
      "activations/layer14_attention_weight_min": -33.630470275878906,
      "activations/layer15_attention_weight_max": 38.53482437133789,
      "activations/layer15_attention_weight_min": -30.613941192626953,
      "activations/layer16_attention_weight_max": 36.02882766723633,
      "activations/layer16_attention_weight_min": -30.034931182861328,
      "activations/layer17_attention_weight_max": 54.8724250793457,
      "activations/layer17_attention_weight_min": -43.82905578613281,
      "activations/layer18_attention_weight_max": 52.00056076049805,
      "activations/layer18_attention_weight_min": -37.02300262451172,
      "activations/layer19_attention_weight_max": 27.007001876831055,
      "activations/layer19_attention_weight_min": -21.74022674560547,
      "activations/layer1_attention_weight_max": 15.974076271057129,
      "activations/layer1_attention_weight_min": -15.306955337524414,
      "activations/layer20_attention_weight_max": 27.588539123535156,
      "activations/layer20_attention_weight_min": -22.346080780029297,
      "activations/layer21_attention_weight_max": 47.50982666015625,
      "activations/layer21_attention_weight_min": -26.45632553100586,
      "activations/layer22_attention_weight_max": 29.338762283325195,
      "activations/layer22_attention_weight_min": -27.263357162475586,
      "activations/layer23_attention_weight_max": 38.235565185546875,
      "activations/layer23_attention_weight_min": -27.06166648864746,
      "activations/layer2_attention_weight_max": 33.48178482055664,
      "activations/layer2_attention_weight_min": -31.849096298217773,
      "activations/layer3_attention_weight_max": 96.38572692871094,
      "activations/layer3_attention_weight_min": -102.6587142944336,
      "activations/layer4_attention_weight_max": 99.11650085449219,
      "activations/layer4_attention_weight_min": -96.38736724853516,
      "activations/layer5_attention_weight_max": 74.85906219482422,
      "activations/layer5_attention_weight_min": -82.9516830444336,
      "activations/layer6_attention_weight_max": 54.90704345703125,
      "activations/layer6_attention_weight_min": -54.77885818481445,
      "activations/layer7_attention_weight_max": 71.91278076171875,
      "activations/layer7_attention_weight_min": -71.75626373291016,
      "activations/layer8_attention_weight_max": 47.9796028137207,
      "activations/layer8_attention_weight_min": -50.78544998168945,
      "activations/layer9_attention_weight_max": 51.78347396850586,
      "activations/layer9_attention_weight_min": -48.82930374145508,
      "epoch": 12.76,
      "learning_rate": 6.838446969696969e-05,
      "loss": 2.7611,
      "step": 219650
    },
    {
      "activations/layer0_attention_weight_max": 16.388681411743164,
      "activations/layer0_attention_weight_min": -13.99887466430664,
      "activations/layer10_attention_weight_max": 34.97329330444336,
      "activations/layer10_attention_weight_min": -33.667724609375,
      "activations/layer11_attention_weight_max": 30.476802825927734,
      "activations/layer11_attention_weight_min": -32.711280822753906,
      "activations/layer12_attention_weight_max": 17.367206573486328,
      "activations/layer12_attention_weight_min": -24.388975143432617,
      "activations/layer13_attention_weight_max": 34.33979034423828,
      "activations/layer13_attention_weight_min": -30.376678466796875,
      "activations/layer14_attention_weight_max": 34.395301818847656,
      "activations/layer14_attention_weight_min": -32.43607711791992,
      "activations/layer15_attention_weight_max": 33.59336471557617,
      "activations/layer15_attention_weight_min": -30.69624900817871,
      "activations/layer16_attention_weight_max": 31.179821014404297,
      "activations/layer16_attention_weight_min": -30.704317092895508,
      "activations/layer17_attention_weight_max": 53.18474197387695,
      "activations/layer17_attention_weight_min": -46.581634521484375,
      "activations/layer18_attention_weight_max": 46.669246673583984,
      "activations/layer18_attention_weight_min": -37.74369430541992,
      "activations/layer19_attention_weight_max": 21.43988800048828,
      "activations/layer19_attention_weight_min": -21.675466537475586,
      "activations/layer1_attention_weight_max": 16.02840805053711,
      "activations/layer1_attention_weight_min": -15.23128604888916,
      "activations/layer20_attention_weight_max": 22.438255310058594,
      "activations/layer20_attention_weight_min": -24.228960037231445,
      "activations/layer21_attention_weight_max": 33.122703552246094,
      "activations/layer21_attention_weight_min": -22.04998207092285,
      "activations/layer22_attention_weight_max": 29.569549560546875,
      "activations/layer22_attention_weight_min": -24.647811889648438,
      "activations/layer23_attention_weight_max": 35.379051208496094,
      "activations/layer23_attention_weight_min": -23.123594284057617,
      "activations/layer2_attention_weight_max": 32.80232620239258,
      "activations/layer2_attention_weight_min": -32.371795654296875,
      "activations/layer3_attention_weight_max": 94.54161834716797,
      "activations/layer3_attention_weight_min": -96.99169921875,
      "activations/layer4_attention_weight_max": 97.49295806884766,
      "activations/layer4_attention_weight_min": -94.4755859375,
      "activations/layer5_attention_weight_max": 72.80260467529297,
      "activations/layer5_attention_weight_min": -77.57735443115234,
      "activations/layer6_attention_weight_max": 49.89118576049805,
      "activations/layer6_attention_weight_min": -51.10740280151367,
      "activations/layer7_attention_weight_max": 69.73617553710938,
      "activations/layer7_attention_weight_min": -68.04783630371094,
      "activations/layer8_attention_weight_max": 42.30125427246094,
      "activations/layer8_attention_weight_min": -46.495670318603516,
      "activations/layer9_attention_weight_max": 42.764591217041016,
      "activations/layer9_attention_weight_min": -45.82294464111328,
      "epoch": 12.77,
      "learning_rate": 6.83655303030303e-05,
      "loss": 2.7703,
      "step": 219700
    },
    {
      "activations/layer0_attention_weight_max": 16.06546401977539,
      "activations/layer0_attention_weight_min": -14.15269947052002,
      "activations/layer10_attention_weight_max": 35.499568939208984,
      "activations/layer10_attention_weight_min": -34.27260208129883,
      "activations/layer11_attention_weight_max": 32.560157775878906,
      "activations/layer11_attention_weight_min": -32.20431900024414,
      "activations/layer12_attention_weight_max": 17.457857131958008,
      "activations/layer12_attention_weight_min": -26.082185745239258,
      "activations/layer13_attention_weight_max": 37.555049896240234,
      "activations/layer13_attention_weight_min": -28.040294647216797,
      "activations/layer14_attention_weight_max": 36.868202209472656,
      "activations/layer14_attention_weight_min": -28.620161056518555,
      "activations/layer15_attention_weight_max": 33.06393051147461,
      "activations/layer15_attention_weight_min": -27.314289093017578,
      "activations/layer16_attention_weight_max": 30.429155349731445,
      "activations/layer16_attention_weight_min": -31.236705780029297,
      "activations/layer17_attention_weight_max": 50.98735809326172,
      "activations/layer17_attention_weight_min": -42.01070785522461,
      "activations/layer18_attention_weight_max": 46.46807098388672,
      "activations/layer18_attention_weight_min": -35.54772186279297,
      "activations/layer19_attention_weight_max": 20.884033203125,
      "activations/layer19_attention_weight_min": -21.379011154174805,
      "activations/layer1_attention_weight_max": 16.004878997802734,
      "activations/layer1_attention_weight_min": -14.207026481628418,
      "activations/layer20_attention_weight_max": 21.36244010925293,
      "activations/layer20_attention_weight_min": -19.959196090698242,
      "activations/layer21_attention_weight_max": 31.729183197021484,
      "activations/layer21_attention_weight_min": -21.243209838867188,
      "activations/layer22_attention_weight_max": 29.816301345825195,
      "activations/layer22_attention_weight_min": -24.11109161376953,
      "activations/layer23_attention_weight_max": 38.51023864746094,
      "activations/layer23_attention_weight_min": -23.107885360717773,
      "activations/layer2_attention_weight_max": 33.52894592285156,
      "activations/layer2_attention_weight_min": -30.909215927124023,
      "activations/layer3_attention_weight_max": 92.8927993774414,
      "activations/layer3_attention_weight_min": -90.59080505371094,
      "activations/layer4_attention_weight_max": 96.7214584350586,
      "activations/layer4_attention_weight_min": -92.04625701904297,
      "activations/layer5_attention_weight_max": 71.51683044433594,
      "activations/layer5_attention_weight_min": -81.32624816894531,
      "activations/layer6_attention_weight_max": 52.9951057434082,
      "activations/layer6_attention_weight_min": -51.56772232055664,
      "activations/layer7_attention_weight_max": 71.77880096435547,
      "activations/layer7_attention_weight_min": -65.72440338134766,
      "activations/layer8_attention_weight_max": 44.45372009277344,
      "activations/layer8_attention_weight_min": -47.06324005126953,
      "activations/layer9_attention_weight_max": 45.549190521240234,
      "activations/layer9_attention_weight_min": -46.057579040527344,
      "epoch": 12.77,
      "learning_rate": 6.83465909090909e-05,
      "loss": 2.7611,
      "step": 219750
    },
    {
      "activations/layer0_attention_weight_max": 15.827553749084473,
      "activations/layer0_attention_weight_min": -13.627327919006348,
      "activations/layer10_attention_weight_max": 35.42636489868164,
      "activations/layer10_attention_weight_min": -34.12004470825195,
      "activations/layer11_attention_weight_max": 30.206680297851562,
      "activations/layer11_attention_weight_min": -32.13473892211914,
      "activations/layer12_attention_weight_max": 18.065048217773438,
      "activations/layer12_attention_weight_min": -22.58679962158203,
      "activations/layer13_attention_weight_max": 36.40998077392578,
      "activations/layer13_attention_weight_min": -26.506242752075195,
      "activations/layer14_attention_weight_max": 33.77674102783203,
      "activations/layer14_attention_weight_min": -31.528966903686523,
      "activations/layer15_attention_weight_max": 30.200599670410156,
      "activations/layer15_attention_weight_min": -30.496164321899414,
      "activations/layer16_attention_weight_max": 28.609573364257812,
      "activations/layer16_attention_weight_min": -29.953062057495117,
      "activations/layer17_attention_weight_max": 49.217750549316406,
      "activations/layer17_attention_weight_min": -41.41609191894531,
      "activations/layer18_attention_weight_max": 45.77615737915039,
      "activations/layer18_attention_weight_min": -37.03972625732422,
      "activations/layer19_attention_weight_max": 22.30939292907715,
      "activations/layer19_attention_weight_min": -21.75447654724121,
      "activations/layer1_attention_weight_max": 16.242557525634766,
      "activations/layer1_attention_weight_min": -15.165108680725098,
      "activations/layer20_attention_weight_max": 22.818674087524414,
      "activations/layer20_attention_weight_min": -19.29843521118164,
      "activations/layer21_attention_weight_max": 39.01355743408203,
      "activations/layer21_attention_weight_min": -19.688236236572266,
      "activations/layer22_attention_weight_max": 29.39204978942871,
      "activations/layer22_attention_weight_min": -25.46953010559082,
      "activations/layer23_attention_weight_max": 41.299461364746094,
      "activations/layer23_attention_weight_min": -23.637250900268555,
      "activations/layer2_attention_weight_max": 34.60462188720703,
      "activations/layer2_attention_weight_min": -30.996990203857422,
      "activations/layer3_attention_weight_max": 98.0125961303711,
      "activations/layer3_attention_weight_min": -96.62385559082031,
      "activations/layer4_attention_weight_max": 97.50374603271484,
      "activations/layer4_attention_weight_min": -95.19779205322266,
      "activations/layer5_attention_weight_max": 74.19721984863281,
      "activations/layer5_attention_weight_min": -79.3155746459961,
      "activations/layer6_attention_weight_max": 50.8167839050293,
      "activations/layer6_attention_weight_min": -51.301570892333984,
      "activations/layer7_attention_weight_max": 68.1077880859375,
      "activations/layer7_attention_weight_min": -62.94166946411133,
      "activations/layer8_attention_weight_max": 46.228981018066406,
      "activations/layer8_attention_weight_min": -49.89467239379883,
      "activations/layer9_attention_weight_max": 48.38760757446289,
      "activations/layer9_attention_weight_min": -44.79869842529297,
      "epoch": 12.77,
      "learning_rate": 6.832765151515152e-05,
      "loss": 2.7553,
      "step": 219800
    },
    {
      "activations/layer0_attention_weight_max": 15.927578926086426,
      "activations/layer0_attention_weight_min": -13.900468826293945,
      "activations/layer10_attention_weight_max": 39.139442443847656,
      "activations/layer10_attention_weight_min": -37.669464111328125,
      "activations/layer11_attention_weight_max": 32.72535705566406,
      "activations/layer11_attention_weight_min": -33.26319122314453,
      "activations/layer12_attention_weight_max": 20.570493698120117,
      "activations/layer12_attention_weight_min": -29.265520095825195,
      "activations/layer13_attention_weight_max": 43.42814254760742,
      "activations/layer13_attention_weight_min": -29.894283294677734,
      "activations/layer14_attention_weight_max": 39.32758331298828,
      "activations/layer14_attention_weight_min": -30.912124633789062,
      "activations/layer15_attention_weight_max": 37.1810417175293,
      "activations/layer15_attention_weight_min": -29.638399124145508,
      "activations/layer16_attention_weight_max": 32.09189224243164,
      "activations/layer16_attention_weight_min": -31.177364349365234,
      "activations/layer17_attention_weight_max": 51.76980209350586,
      "activations/layer17_attention_weight_min": -47.0097770690918,
      "activations/layer18_attention_weight_max": 47.28987503051758,
      "activations/layer18_attention_weight_min": -39.823116302490234,
      "activations/layer19_attention_weight_max": 25.652286529541016,
      "activations/layer19_attention_weight_min": -25.806711196899414,
      "activations/layer1_attention_weight_max": 17.653493881225586,
      "activations/layer1_attention_weight_min": -15.042016983032227,
      "activations/layer20_attention_weight_max": 28.110424041748047,
      "activations/layer20_attention_weight_min": -22.627286911010742,
      "activations/layer21_attention_weight_max": 40.55214309692383,
      "activations/layer21_attention_weight_min": -22.95835304260254,
      "activations/layer22_attention_weight_max": 31.42687225341797,
      "activations/layer22_attention_weight_min": -28.977054595947266,
      "activations/layer23_attention_weight_max": 41.12526321411133,
      "activations/layer23_attention_weight_min": -23.283836364746094,
      "activations/layer2_attention_weight_max": 34.290706634521484,
      "activations/layer2_attention_weight_min": -31.47926139831543,
      "activations/layer3_attention_weight_max": 98.25,
      "activations/layer3_attention_weight_min": -92.48686981201172,
      "activations/layer4_attention_weight_max": 94.3248519897461,
      "activations/layer4_attention_weight_min": -91.98116302490234,
      "activations/layer5_attention_weight_max": 71.58556365966797,
      "activations/layer5_attention_weight_min": -76.19430541992188,
      "activations/layer6_attention_weight_max": 51.371192932128906,
      "activations/layer6_attention_weight_min": -51.53083801269531,
      "activations/layer7_attention_weight_max": 70.27302551269531,
      "activations/layer7_attention_weight_min": -71.42633056640625,
      "activations/layer8_attention_weight_max": 47.1544303894043,
      "activations/layer8_attention_weight_min": -50.5016975402832,
      "activations/layer9_attention_weight_max": 53.18549346923828,
      "activations/layer9_attention_weight_min": -49.43647384643555,
      "epoch": 12.77,
      "learning_rate": 6.830871212121212e-05,
      "loss": 2.7664,
      "step": 219850
    },
    {
      "activations/layer0_attention_weight_max": 16.251644134521484,
      "activations/layer0_attention_weight_min": -13.916379928588867,
      "activations/layer10_attention_weight_max": 34.41878890991211,
      "activations/layer10_attention_weight_min": -34.68098831176758,
      "activations/layer11_attention_weight_max": 30.2987060546875,
      "activations/layer11_attention_weight_min": -32.63032531738281,
      "activations/layer12_attention_weight_max": 17.834165573120117,
      "activations/layer12_attention_weight_min": -30.951553344726562,
      "activations/layer13_attention_weight_max": 38.104736328125,
      "activations/layer13_attention_weight_min": -27.184797286987305,
      "activations/layer14_attention_weight_max": 35.37428283691406,
      "activations/layer14_attention_weight_min": -31.132957458496094,
      "activations/layer15_attention_weight_max": 31.884281158447266,
      "activations/layer15_attention_weight_min": -29.858449935913086,
      "activations/layer16_attention_weight_max": 30.846057891845703,
      "activations/layer16_attention_weight_min": -30.336023330688477,
      "activations/layer17_attention_weight_max": 53.00920486450195,
      "activations/layer17_attention_weight_min": -42.786678314208984,
      "activations/layer18_attention_weight_max": 43.375823974609375,
      "activations/layer18_attention_weight_min": -36.865455627441406,
      "activations/layer19_attention_weight_max": 21.535795211791992,
      "activations/layer19_attention_weight_min": -23.03160858154297,
      "activations/layer1_attention_weight_max": 16.282794952392578,
      "activations/layer1_attention_weight_min": -14.687016487121582,
      "activations/layer20_attention_weight_max": 20.602703094482422,
      "activations/layer20_attention_weight_min": -21.894895553588867,
      "activations/layer21_attention_weight_max": 33.45540237426758,
      "activations/layer21_attention_weight_min": -20.358945846557617,
      "activations/layer22_attention_weight_max": 28.495168685913086,
      "activations/layer22_attention_weight_min": -27.601884841918945,
      "activations/layer23_attention_weight_max": 33.69408416748047,
      "activations/layer23_attention_weight_min": -23.295181274414062,
      "activations/layer2_attention_weight_max": 33.58427810668945,
      "activations/layer2_attention_weight_min": -29.770488739013672,
      "activations/layer3_attention_weight_max": 96.71031951904297,
      "activations/layer3_attention_weight_min": -96.3984146118164,
      "activations/layer4_attention_weight_max": 98.65428161621094,
      "activations/layer4_attention_weight_min": -98.64080810546875,
      "activations/layer5_attention_weight_max": 70.7982177734375,
      "activations/layer5_attention_weight_min": -77.47714233398438,
      "activations/layer6_attention_weight_max": 52.14603042602539,
      "activations/layer6_attention_weight_min": -51.95872116088867,
      "activations/layer7_attention_weight_max": 67.32040405273438,
      "activations/layer7_attention_weight_min": -67.27666473388672,
      "activations/layer8_attention_weight_max": 45.11908721923828,
      "activations/layer8_attention_weight_min": -46.20978546142578,
      "activations/layer9_attention_weight_max": 45.77542495727539,
      "activations/layer9_attention_weight_min": -47.35527801513672,
      "epoch": 12.78,
      "learning_rate": 6.828977272727272e-05,
      "loss": 2.7771,
      "step": 219900
    },
    {
      "activations/layer0_attention_weight_max": 16.27193260192871,
      "activations/layer0_attention_weight_min": -13.864245414733887,
      "activations/layer10_attention_weight_max": 37.09027099609375,
      "activations/layer10_attention_weight_min": -35.319881439208984,
      "activations/layer11_attention_weight_max": 32.541717529296875,
      "activations/layer11_attention_weight_min": -31.641103744506836,
      "activations/layer12_attention_weight_max": 20.979328155517578,
      "activations/layer12_attention_weight_min": -23.9710750579834,
      "activations/layer13_attention_weight_max": 36.8966178894043,
      "activations/layer13_attention_weight_min": -27.288911819458008,
      "activations/layer14_attention_weight_max": 34.24330139160156,
      "activations/layer14_attention_weight_min": -28.95758819580078,
      "activations/layer15_attention_weight_max": 35.508140563964844,
      "activations/layer15_attention_weight_min": -28.874011993408203,
      "activations/layer16_attention_weight_max": 31.329442977905273,
      "activations/layer16_attention_weight_min": -29.800025939941406,
      "activations/layer17_attention_weight_max": 51.35205078125,
      "activations/layer17_attention_weight_min": -41.490203857421875,
      "activations/layer18_attention_weight_max": 45.41637420654297,
      "activations/layer18_attention_weight_min": -35.59966278076172,
      "activations/layer19_attention_weight_max": 20.404661178588867,
      "activations/layer19_attention_weight_min": -19.924570083618164,
      "activations/layer1_attention_weight_max": 16.00290298461914,
      "activations/layer1_attention_weight_min": -17.496915817260742,
      "activations/layer20_attention_weight_max": 22.053905487060547,
      "activations/layer20_attention_weight_min": -21.526044845581055,
      "activations/layer21_attention_weight_max": 33.2455940246582,
      "activations/layer21_attention_weight_min": -22.44271469116211,
      "activations/layer22_attention_weight_max": 30.009336471557617,
      "activations/layer22_attention_weight_min": -25.21723175048828,
      "activations/layer23_attention_weight_max": 37.207584381103516,
      "activations/layer23_attention_weight_min": -25.252065658569336,
      "activations/layer2_attention_weight_max": 34.926475524902344,
      "activations/layer2_attention_weight_min": -30.769363403320312,
      "activations/layer3_attention_weight_max": 96.33892822265625,
      "activations/layer3_attention_weight_min": -93.24835968017578,
      "activations/layer4_attention_weight_max": 98.08035278320312,
      "activations/layer4_attention_weight_min": -94.43041229248047,
      "activations/layer5_attention_weight_max": 69.82858276367188,
      "activations/layer5_attention_weight_min": -80.71806335449219,
      "activations/layer6_attention_weight_max": 51.69198226928711,
      "activations/layer6_attention_weight_min": -51.511287689208984,
      "activations/layer7_attention_weight_max": 67.65518951416016,
      "activations/layer7_attention_weight_min": -69.44580841064453,
      "activations/layer8_attention_weight_max": 47.79568862915039,
      "activations/layer8_attention_weight_min": -52.59434509277344,
      "activations/layer9_attention_weight_max": 46.356040954589844,
      "activations/layer9_attention_weight_min": -46.558074951171875,
      "epoch": 12.78,
      "learning_rate": 6.827083333333332e-05,
      "loss": 2.7779,
      "step": 219950
    },
    {
      "activations/layer0_attention_weight_max": 16.055561065673828,
      "activations/layer0_attention_weight_min": -14.05561637878418,
      "activations/layer10_attention_weight_max": 33.345252990722656,
      "activations/layer10_attention_weight_min": -36.04811096191406,
      "activations/layer11_attention_weight_max": 28.507232666015625,
      "activations/layer11_attention_weight_min": -32.70624542236328,
      "activations/layer12_attention_weight_max": 17.047992706298828,
      "activations/layer12_attention_weight_min": -26.30246353149414,
      "activations/layer13_attention_weight_max": 31.187450408935547,
      "activations/layer13_attention_weight_min": -27.334392547607422,
      "activations/layer14_attention_weight_max": 37.665687561035156,
      "activations/layer14_attention_weight_min": -29.063465118408203,
      "activations/layer15_attention_weight_max": 30.568172454833984,
      "activations/layer15_attention_weight_min": -28.40349578857422,
      "activations/layer16_attention_weight_max": 30.3829345703125,
      "activations/layer16_attention_weight_min": -29.678192138671875,
      "activations/layer17_attention_weight_max": 50.68586349487305,
      "activations/layer17_attention_weight_min": -43.77727127075195,
      "activations/layer18_attention_weight_max": 43.254432678222656,
      "activations/layer18_attention_weight_min": -38.69371795654297,
      "activations/layer19_attention_weight_max": 20.79891014099121,
      "activations/layer19_attention_weight_min": -22.744626998901367,
      "activations/layer1_attention_weight_max": 17.150766372680664,
      "activations/layer1_attention_weight_min": -15.70230484008789,
      "activations/layer20_attention_weight_max": 19.808252334594727,
      "activations/layer20_attention_weight_min": -21.641687393188477,
      "activations/layer21_attention_weight_max": 33.741336822509766,
      "activations/layer21_attention_weight_min": -20.651762008666992,
      "activations/layer22_attention_weight_max": 28.068214416503906,
      "activations/layer22_attention_weight_min": -27.42909049987793,
      "activations/layer23_attention_weight_max": 36.607765197753906,
      "activations/layer23_attention_weight_min": -24.849552154541016,
      "activations/layer2_attention_weight_max": 32.622047424316406,
      "activations/layer2_attention_weight_min": -32.366851806640625,
      "activations/layer3_attention_weight_max": 94.8841552734375,
      "activations/layer3_attention_weight_min": -100.53087615966797,
      "activations/layer4_attention_weight_max": 90.80504608154297,
      "activations/layer4_attention_weight_min": -92.5045394897461,
      "activations/layer5_attention_weight_max": 70.14242553710938,
      "activations/layer5_attention_weight_min": -77.79106140136719,
      "activations/layer6_attention_weight_max": 50.59183883666992,
      "activations/layer6_attention_weight_min": -51.76700973510742,
      "activations/layer7_attention_weight_max": 65.89183044433594,
      "activations/layer7_attention_weight_min": -63.8387336730957,
      "activations/layer8_attention_weight_max": 44.10160827636719,
      "activations/layer8_attention_weight_min": -45.58381652832031,
      "activations/layer9_attention_weight_max": 41.960933685302734,
      "activations/layer9_attention_weight_min": -45.8043098449707,
      "epoch": 12.78,
      "learning_rate": 6.825189393939394e-05,
      "loss": 2.7737,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_loss": 2.724609375,
      "eval_runtime": 8.4883,
      "eval_samples_per_second": 505.873,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_openwebtext_loss": 2.724609375,
      "eval_openwebtext_ppl": 15.250455553989443,
      "eval_openwebtext_runtime": 8.4883,
      "eval_openwebtext_samples_per_second": 505.873,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9533,
      "eval_wikitext_samples_per_second": 233.449,
      "step": 220000
    },
    {
      "epoch": 12.78,
      "eval_lambada_loss": 2.498046875,
      "eval_lambada_ppl": 12.158723248329283,
      "eval_lambada_runtime": 9.6223,
      "eval_lambada_samples_per_second": 506.012,
      "step": 220000
    },
    {
      "activations/layer0_attention_weight_max": 15.317978858947754,
      "activations/layer0_attention_weight_min": -13.99996280670166,
      "activations/layer10_attention_weight_max": 41.343162536621094,
      "activations/layer10_attention_weight_min": -38.10081481933594,
      "activations/layer11_attention_weight_max": 34.673526763916016,
      "activations/layer11_attention_weight_min": -34.16593933105469,
      "activations/layer12_attention_weight_max": 19.411508560180664,
      "activations/layer12_attention_weight_min": -24.714717864990234,
      "activations/layer13_attention_weight_max": 38.61524963378906,
      "activations/layer13_attention_weight_min": -31.8652400970459,
      "activations/layer14_attention_weight_max": 36.51576614379883,
      "activations/layer14_attention_weight_min": -30.19709587097168,
      "activations/layer15_attention_weight_max": 34.954933166503906,
      "activations/layer15_attention_weight_min": -29.648710250854492,
      "activations/layer16_attention_weight_max": 31.417818069458008,
      "activations/layer16_attention_weight_min": -29.150728225708008,
      "activations/layer17_attention_weight_max": 54.17172622680664,
      "activations/layer17_attention_weight_min": -42.31157302856445,
      "activations/layer18_attention_weight_max": 44.27911376953125,
      "activations/layer18_attention_weight_min": -37.150638580322266,
      "activations/layer19_attention_weight_max": 26.729816436767578,
      "activations/layer19_attention_weight_min": -23.251968383789062,
      "activations/layer1_attention_weight_max": 17.31070899963379,
      "activations/layer1_attention_weight_min": -16.08119010925293,
      "activations/layer20_attention_weight_max": 22.280004501342773,
      "activations/layer20_attention_weight_min": -22.44147491455078,
      "activations/layer21_attention_weight_max": 34.6805534362793,
      "activations/layer21_attention_weight_min": -22.927560806274414,
      "activations/layer22_attention_weight_max": 30.910465240478516,
      "activations/layer22_attention_weight_min": -28.490285873413086,
      "activations/layer23_attention_weight_max": 40.22077560424805,
      "activations/layer23_attention_weight_min": -23.976787567138672,
      "activations/layer2_attention_weight_max": 32.69239807128906,
      "activations/layer2_attention_weight_min": -31.531362533569336,
      "activations/layer3_attention_weight_max": 94.01882934570312,
      "activations/layer3_attention_weight_min": -93.67654418945312,
      "activations/layer4_attention_weight_max": 95.71978759765625,
      "activations/layer4_attention_weight_min": -94.45490264892578,
      "activations/layer5_attention_weight_max": 71.07232666015625,
      "activations/layer5_attention_weight_min": -75.97431945800781,
      "activations/layer6_attention_weight_max": 52.0958137512207,
      "activations/layer6_attention_weight_min": -52.997108459472656,
      "activations/layer7_attention_weight_max": 67.7253646850586,
      "activations/layer7_attention_weight_min": -66.76148223876953,
      "activations/layer8_attention_weight_max": 47.28844451904297,
      "activations/layer8_attention_weight_min": -50.97635269165039,
      "activations/layer9_attention_weight_max": 47.90766906738281,
      "activations/layer9_attention_weight_min": -48.932437896728516,
      "epoch": 12.79,
      "learning_rate": 6.823295454545454e-05,
      "loss": 2.7662,
      "step": 220050
    },
    {
      "activations/layer0_attention_weight_max": 18.164522171020508,
      "activations/layer0_attention_weight_min": -13.873003959655762,
      "activations/layer10_attention_weight_max": 35.93450927734375,
      "activations/layer10_attention_weight_min": -36.20262908935547,
      "activations/layer11_attention_weight_max": 33.002437591552734,
      "activations/layer11_attention_weight_min": -33.38425827026367,
      "activations/layer12_attention_weight_max": 19.418973922729492,
      "activations/layer12_attention_weight_min": -23.456722259521484,
      "activations/layer13_attention_weight_max": 42.51178741455078,
      "activations/layer13_attention_weight_min": -31.3010196685791,
      "activations/layer14_attention_weight_max": 37.986568450927734,
      "activations/layer14_attention_weight_min": -30.482194900512695,
      "activations/layer15_attention_weight_max": 34.240055084228516,
      "activations/layer15_attention_weight_min": -30.19042205810547,
      "activations/layer16_attention_weight_max": 34.91099166870117,
      "activations/layer16_attention_weight_min": -33.59221267700195,
      "activations/layer17_attention_weight_max": 55.82236099243164,
      "activations/layer17_attention_weight_min": -48.36580276489258,
      "activations/layer18_attention_weight_max": 48.43709945678711,
      "activations/layer18_attention_weight_min": -39.105804443359375,
      "activations/layer19_attention_weight_max": 23.61199188232422,
      "activations/layer19_attention_weight_min": -23.261852264404297,
      "activations/layer1_attention_weight_max": 17.674327850341797,
      "activations/layer1_attention_weight_min": -15.973258972167969,
      "activations/layer20_attention_weight_max": 23.217172622680664,
      "activations/layer20_attention_weight_min": -22.549911499023438,
      "activations/layer21_attention_weight_max": 38.149810791015625,
      "activations/layer21_attention_weight_min": -22.761877059936523,
      "activations/layer22_attention_weight_max": 31.61111831665039,
      "activations/layer22_attention_weight_min": -27.20569610595703,
      "activations/layer23_attention_weight_max": 41.33019256591797,
      "activations/layer23_attention_weight_min": -23.32405662536621,
      "activations/layer2_attention_weight_max": 32.41020202636719,
      "activations/layer2_attention_weight_min": -30.65024185180664,
      "activations/layer3_attention_weight_max": 98.61637115478516,
      "activations/layer3_attention_weight_min": -101.64225006103516,
      "activations/layer4_attention_weight_max": 99.64400482177734,
      "activations/layer4_attention_weight_min": -100.26366424560547,
      "activations/layer5_attention_weight_max": 70.82583618164062,
      "activations/layer5_attention_weight_min": -81.0356674194336,
      "activations/layer6_attention_weight_max": 52.313968658447266,
      "activations/layer6_attention_weight_min": -54.357601165771484,
      "activations/layer7_attention_weight_max": 67.68472290039062,
      "activations/layer7_attention_weight_min": -69.0617446899414,
      "activations/layer8_attention_weight_max": 44.2413444519043,
      "activations/layer8_attention_weight_min": -49.64687728881836,
      "activations/layer9_attention_weight_max": 46.511226654052734,
      "activations/layer9_attention_weight_min": -48.92987060546875,
      "epoch": 12.79,
      "learning_rate": 6.821401515151514e-05,
      "loss": 2.7669,
      "step": 220100
    },
    {
      "activations/layer0_attention_weight_max": 17.55584144592285,
      "activations/layer0_attention_weight_min": -13.956216812133789,
      "activations/layer10_attention_weight_max": 35.66777420043945,
      "activations/layer10_attention_weight_min": -34.78471374511719,
      "activations/layer11_attention_weight_max": 31.215560913085938,
      "activations/layer11_attention_weight_min": -33.17627716064453,
      "activations/layer12_attention_weight_max": 16.48801040649414,
      "activations/layer12_attention_weight_min": -25.943723678588867,
      "activations/layer13_attention_weight_max": 31.211437225341797,
      "activations/layer13_attention_weight_min": -28.6346435546875,
      "activations/layer14_attention_weight_max": 31.739028930664062,
      "activations/layer14_attention_weight_min": -31.240886688232422,
      "activations/layer15_attention_weight_max": 30.20880889892578,
      "activations/layer15_attention_weight_min": -30.008960723876953,
      "activations/layer16_attention_weight_max": 29.951257705688477,
      "activations/layer16_attention_weight_min": -29.660417556762695,
      "activations/layer17_attention_weight_max": 48.612186431884766,
      "activations/layer17_attention_weight_min": -42.355106353759766,
      "activations/layer18_attention_weight_max": 43.13633728027344,
      "activations/layer18_attention_weight_min": -35.77123260498047,
      "activations/layer19_attention_weight_max": 21.777379989624023,
      "activations/layer19_attention_weight_min": -20.323715209960938,
      "activations/layer1_attention_weight_max": 15.918439865112305,
      "activations/layer1_attention_weight_min": -15.624526023864746,
      "activations/layer20_attention_weight_max": 20.865360260009766,
      "activations/layer20_attention_weight_min": -22.317190170288086,
      "activations/layer21_attention_weight_max": 31.3397274017334,
      "activations/layer21_attention_weight_min": -25.710708618164062,
      "activations/layer22_attention_weight_max": 29.65934944152832,
      "activations/layer22_attention_weight_min": -25.43900489807129,
      "activations/layer23_attention_weight_max": 36.94318389892578,
      "activations/layer23_attention_weight_min": -24.60524559020996,
      "activations/layer2_attention_weight_max": 31.271652221679688,
      "activations/layer2_attention_weight_min": -29.634916305541992,
      "activations/layer3_attention_weight_max": 88.96546173095703,
      "activations/layer3_attention_weight_min": -93.68975067138672,
      "activations/layer4_attention_weight_max": 94.71318054199219,
      "activations/layer4_attention_weight_min": -89.97357940673828,
      "activations/layer5_attention_weight_max": 68.8037109375,
      "activations/layer5_attention_weight_min": -77.15773010253906,
      "activations/layer6_attention_weight_max": 52.36521530151367,
      "activations/layer6_attention_weight_min": -52.1278190612793,
      "activations/layer7_attention_weight_max": 64.61644744873047,
      "activations/layer7_attention_weight_min": -64.72126770019531,
      "activations/layer8_attention_weight_max": 43.99089050292969,
      "activations/layer8_attention_weight_min": -46.43162155151367,
      "activations/layer9_attention_weight_max": 42.8188362121582,
      "activations/layer9_attention_weight_min": -44.92184066772461,
      "epoch": 12.79,
      "learning_rate": 6.819507575757574e-05,
      "loss": 2.7663,
      "step": 220150
    },
    {
      "activations/layer0_attention_weight_max": 16.4453067779541,
      "activations/layer0_attention_weight_min": -14.473068237304688,
      "activations/layer10_attention_weight_max": 38.313602447509766,
      "activations/layer10_attention_weight_min": -40.87078857421875,
      "activations/layer11_attention_weight_max": 36.820068359375,
      "activations/layer11_attention_weight_min": -37.353355407714844,
      "activations/layer12_attention_weight_max": 18.911027908325195,
      "activations/layer12_attention_weight_min": -33.036216735839844,
      "activations/layer13_attention_weight_max": 39.83359909057617,
      "activations/layer13_attention_weight_min": -30.006853103637695,
      "activations/layer14_attention_weight_max": 43.49965286254883,
      "activations/layer14_attention_weight_min": -31.111095428466797,
      "activations/layer15_attention_weight_max": 39.26142120361328,
      "activations/layer15_attention_weight_min": -31.49279022216797,
      "activations/layer16_attention_weight_max": 32.09974670410156,
      "activations/layer16_attention_weight_min": -30.645084381103516,
      "activations/layer17_attention_weight_max": 52.24998474121094,
      "activations/layer17_attention_weight_min": -44.24705123901367,
      "activations/layer18_attention_weight_max": 43.83534622192383,
      "activations/layer18_attention_weight_min": -38.403541564941406,
      "activations/layer19_attention_weight_max": 21.130765914916992,
      "activations/layer19_attention_weight_min": -21.077428817749023,
      "activations/layer1_attention_weight_max": 17.248523712158203,
      "activations/layer1_attention_weight_min": -15.200662612915039,
      "activations/layer20_attention_weight_max": 22.33839225769043,
      "activations/layer20_attention_weight_min": -20.51999282836914,
      "activations/layer21_attention_weight_max": 37.57764434814453,
      "activations/layer21_attention_weight_min": -24.980350494384766,
      "activations/layer22_attention_weight_max": 29.920454025268555,
      "activations/layer22_attention_weight_min": -25.61612892150879,
      "activations/layer23_attention_weight_max": 32.31547927856445,
      "activations/layer23_attention_weight_min": -26.106369018554688,
      "activations/layer2_attention_weight_max": 33.413612365722656,
      "activations/layer2_attention_weight_min": -34.22781753540039,
      "activations/layer3_attention_weight_max": 95.28477478027344,
      "activations/layer3_attention_weight_min": -96.94922637939453,
      "activations/layer4_attention_weight_max": 95.6650619506836,
      "activations/layer4_attention_weight_min": -95.66861724853516,
      "activations/layer5_attention_weight_max": 71.3693618774414,
      "activations/layer5_attention_weight_min": -79.78679656982422,
      "activations/layer6_attention_weight_max": 51.482364654541016,
      "activations/layer6_attention_weight_min": -53.05107879638672,
      "activations/layer7_attention_weight_max": 72.45372772216797,
      "activations/layer7_attention_weight_min": -68.55097198486328,
      "activations/layer8_attention_weight_max": 50.8140869140625,
      "activations/layer8_attention_weight_min": -51.17741394042969,
      "activations/layer9_attention_weight_max": 61.60647201538086,
      "activations/layer9_attention_weight_min": -48.86859130859375,
      "epoch": 12.79,
      "learning_rate": 6.817613636363636e-05,
      "loss": 2.7625,
      "step": 220200
    },
    {
      "activations/layer0_attention_weight_max": 16.863130569458008,
      "activations/layer0_attention_weight_min": -14.331660270690918,
      "activations/layer10_attention_weight_max": 34.3936882019043,
      "activations/layer10_attention_weight_min": -36.968605041503906,
      "activations/layer11_attention_weight_max": 32.126373291015625,
      "activations/layer11_attention_weight_min": -33.26137161254883,
      "activations/layer12_attention_weight_max": 19.390605926513672,
      "activations/layer12_attention_weight_min": -25.19308853149414,
      "activations/layer13_attention_weight_max": 34.341636657714844,
      "activations/layer13_attention_weight_min": -30.84072494506836,
      "activations/layer14_attention_weight_max": 34.56374740600586,
      "activations/layer14_attention_weight_min": -29.582380294799805,
      "activations/layer15_attention_weight_max": 34.1437873840332,
      "activations/layer15_attention_weight_min": -29.786102294921875,
      "activations/layer16_attention_weight_max": 31.200700759887695,
      "activations/layer16_attention_weight_min": -29.32227325439453,
      "activations/layer17_attention_weight_max": 53.48670959472656,
      "activations/layer17_attention_weight_min": -44.467777252197266,
      "activations/layer18_attention_weight_max": 49.21384048461914,
      "activations/layer18_attention_weight_min": -39.069190979003906,
      "activations/layer19_attention_weight_max": 21.534711837768555,
      "activations/layer19_attention_weight_min": -21.182058334350586,
      "activations/layer1_attention_weight_max": 16.306337356567383,
      "activations/layer1_attention_weight_min": -16.912273406982422,
      "activations/layer20_attention_weight_max": 21.857349395751953,
      "activations/layer20_attention_weight_min": -20.15876579284668,
      "activations/layer21_attention_weight_max": 36.12513732910156,
      "activations/layer21_attention_weight_min": -25.334259033203125,
      "activations/layer22_attention_weight_max": 32.397727966308594,
      "activations/layer22_attention_weight_min": -24.30706024169922,
      "activations/layer23_attention_weight_max": 39.43985366821289,
      "activations/layer23_attention_weight_min": -24.934768676757812,
      "activations/layer2_attention_weight_max": 32.357337951660156,
      "activations/layer2_attention_weight_min": -30.169729232788086,
      "activations/layer3_attention_weight_max": 92.4123764038086,
      "activations/layer3_attention_weight_min": -95.6072006225586,
      "activations/layer4_attention_weight_max": 97.41670989990234,
      "activations/layer4_attention_weight_min": -95.33293914794922,
      "activations/layer5_attention_weight_max": 70.43875122070312,
      "activations/layer5_attention_weight_min": -78.03498077392578,
      "activations/layer6_attention_weight_max": 51.80864334106445,
      "activations/layer6_attention_weight_min": -50.30766296386719,
      "activations/layer7_attention_weight_max": 69.96052551269531,
      "activations/layer7_attention_weight_min": -74.95176696777344,
      "activations/layer8_attention_weight_max": 48.114891052246094,
      "activations/layer8_attention_weight_min": -47.614009857177734,
      "activations/layer9_attention_weight_max": 46.449886322021484,
      "activations/layer9_attention_weight_min": -45.729278564453125,
      "epoch": 12.8,
      "learning_rate": 6.815719696969696e-05,
      "loss": 2.7659,
      "step": 220250
    },
    {
      "activations/layer0_attention_weight_max": 16.432401657104492,
      "activations/layer0_attention_weight_min": -14.417905807495117,
      "activations/layer10_attention_weight_max": 34.21405029296875,
      "activations/layer10_attention_weight_min": -35.4470100402832,
      "activations/layer11_attention_weight_max": 33.373619079589844,
      "activations/layer11_attention_weight_min": -32.65360641479492,
      "activations/layer12_attention_weight_max": 19.6579532623291,
      "activations/layer12_attention_weight_min": -23.507652282714844,
      "activations/layer13_attention_weight_max": 38.51508712768555,
      "activations/layer13_attention_weight_min": -29.14813995361328,
      "activations/layer14_attention_weight_max": 36.83147048950195,
      "activations/layer14_attention_weight_min": -29.99439239501953,
      "activations/layer15_attention_weight_max": 34.40694808959961,
      "activations/layer15_attention_weight_min": -30.34467315673828,
      "activations/layer16_attention_weight_max": 32.11943817138672,
      "activations/layer16_attention_weight_min": -28.816423416137695,
      "activations/layer17_attention_weight_max": 51.604087829589844,
      "activations/layer17_attention_weight_min": -46.17616653442383,
      "activations/layer18_attention_weight_max": 49.752838134765625,
      "activations/layer18_attention_weight_min": -40.69339370727539,
      "activations/layer19_attention_weight_max": 24.038063049316406,
      "activations/layer19_attention_weight_min": -22.74407958984375,
      "activations/layer1_attention_weight_max": 16.037187576293945,
      "activations/layer1_attention_weight_min": -16.01702308654785,
      "activations/layer20_attention_weight_max": 23.20011329650879,
      "activations/layer20_attention_weight_min": -22.263702392578125,
      "activations/layer21_attention_weight_max": 39.77739334106445,
      "activations/layer21_attention_weight_min": -24.312515258789062,
      "activations/layer22_attention_weight_max": 31.61994171142578,
      "activations/layer22_attention_weight_min": -29.838594436645508,
      "activations/layer23_attention_weight_max": 42.279563903808594,
      "activations/layer23_attention_weight_min": -24.020618438720703,
      "activations/layer2_attention_weight_max": 32.56021499633789,
      "activations/layer2_attention_weight_min": -31.512115478515625,
      "activations/layer3_attention_weight_max": 92.98114776611328,
      "activations/layer3_attention_weight_min": -99.10963439941406,
      "activations/layer4_attention_weight_max": 94.1329116821289,
      "activations/layer4_attention_weight_min": -95.24047088623047,
      "activations/layer5_attention_weight_max": 73.00602722167969,
      "activations/layer5_attention_weight_min": -74.68064880371094,
      "activations/layer6_attention_weight_max": 51.180091857910156,
      "activations/layer6_attention_weight_min": -51.19353485107422,
      "activations/layer7_attention_weight_max": 69.82005310058594,
      "activations/layer7_attention_weight_min": -66.3477554321289,
      "activations/layer8_attention_weight_max": 48.59947967529297,
      "activations/layer8_attention_weight_min": -52.4675178527832,
      "activations/layer9_attention_weight_max": 44.460426330566406,
      "activations/layer9_attention_weight_min": -47.70765686035156,
      "epoch": 12.8,
      "learning_rate": 6.813825757575757e-05,
      "loss": 2.7869,
      "step": 220300
    },
    {
      "activations/layer0_attention_weight_max": 15.92999267578125,
      "activations/layer0_attention_weight_min": -14.54512882232666,
      "activations/layer10_attention_weight_max": 34.77943801879883,
      "activations/layer10_attention_weight_min": -36.92909240722656,
      "activations/layer11_attention_weight_max": 31.39946746826172,
      "activations/layer11_attention_weight_min": -33.24112319946289,
      "activations/layer12_attention_weight_max": 21.295150756835938,
      "activations/layer12_attention_weight_min": -30.486923217773438,
      "activations/layer13_attention_weight_max": 34.44173812866211,
      "activations/layer13_attention_weight_min": -26.920656204223633,
      "activations/layer14_attention_weight_max": 36.38026428222656,
      "activations/layer14_attention_weight_min": -30.738384246826172,
      "activations/layer15_attention_weight_max": 33.88974380493164,
      "activations/layer15_attention_weight_min": -30.90903663635254,
      "activations/layer16_attention_weight_max": 32.44286346435547,
      "activations/layer16_attention_weight_min": -31.49947166442871,
      "activations/layer17_attention_weight_max": 52.43230056762695,
      "activations/layer17_attention_weight_min": -45.17558670043945,
      "activations/layer18_attention_weight_max": 48.43321990966797,
      "activations/layer18_attention_weight_min": -37.32206344604492,
      "activations/layer19_attention_weight_max": 21.646503448486328,
      "activations/layer19_attention_weight_min": -20.841346740722656,
      "activations/layer1_attention_weight_max": 17.03202247619629,
      "activations/layer1_attention_weight_min": -14.696603775024414,
      "activations/layer20_attention_weight_max": 23.398462295532227,
      "activations/layer20_attention_weight_min": -21.164440155029297,
      "activations/layer21_attention_weight_max": 39.2956428527832,
      "activations/layer21_attention_weight_min": -22.769506454467773,
      "activations/layer22_attention_weight_max": 33.23805236816406,
      "activations/layer22_attention_weight_min": -24.998851776123047,
      "activations/layer23_attention_weight_max": 39.496116638183594,
      "activations/layer23_attention_weight_min": -24.064138412475586,
      "activations/layer2_attention_weight_max": 32.52923583984375,
      "activations/layer2_attention_weight_min": -31.016630172729492,
      "activations/layer3_attention_weight_max": 93.62640380859375,
      "activations/layer3_attention_weight_min": -98.15991973876953,
      "activations/layer4_attention_weight_max": 95.95950317382812,
      "activations/layer4_attention_weight_min": -95.37210083007812,
      "activations/layer5_attention_weight_max": 71.06103515625,
      "activations/layer5_attention_weight_min": -81.08847045898438,
      "activations/layer6_attention_weight_max": 54.85966110229492,
      "activations/layer6_attention_weight_min": -53.26481628417969,
      "activations/layer7_attention_weight_max": 73.26757049560547,
      "activations/layer7_attention_weight_min": -66.15721893310547,
      "activations/layer8_attention_weight_max": 47.07011413574219,
      "activations/layer8_attention_weight_min": -48.44203567504883,
      "activations/layer9_attention_weight_max": 51.41328048706055,
      "activations/layer9_attention_weight_min": -46.930206298828125,
      "epoch": 12.8,
      "learning_rate": 6.811931818181818e-05,
      "loss": 2.772,
      "step": 220350
    },
    {
      "activations/layer0_attention_weight_max": 16.96564483642578,
      "activations/layer0_attention_weight_min": -14.089394569396973,
      "activations/layer10_attention_weight_max": 34.227054595947266,
      "activations/layer10_attention_weight_min": -33.7400016784668,
      "activations/layer11_attention_weight_max": 30.46622085571289,
      "activations/layer11_attention_weight_min": -32.54228973388672,
      "activations/layer12_attention_weight_max": 18.608123779296875,
      "activations/layer12_attention_weight_min": -26.40641975402832,
      "activations/layer13_attention_weight_max": 35.65131378173828,
      "activations/layer13_attention_weight_min": -32.033287048339844,
      "activations/layer14_attention_weight_max": 37.35469436645508,
      "activations/layer14_attention_weight_min": -29.445951461791992,
      "activations/layer15_attention_weight_max": 34.1013069152832,
      "activations/layer15_attention_weight_min": -28.768688201904297,
      "activations/layer16_attention_weight_max": 34.36166000366211,
      "activations/layer16_attention_weight_min": -32.25872039794922,
      "activations/layer17_attention_weight_max": 56.62103271484375,
      "activations/layer17_attention_weight_min": -46.502967834472656,
      "activations/layer18_attention_weight_max": 52.190025329589844,
      "activations/layer18_attention_weight_min": -40.86277770996094,
      "activations/layer19_attention_weight_max": 22.07954978942871,
      "activations/layer19_attention_weight_min": -22.424192428588867,
      "activations/layer1_attention_weight_max": 16.63576316833496,
      "activations/layer1_attention_weight_min": -15.957067489624023,
      "activations/layer20_attention_weight_max": 27.01266098022461,
      "activations/layer20_attention_weight_min": -22.44274139404297,
      "activations/layer21_attention_weight_max": 42.45304489135742,
      "activations/layer21_attention_weight_min": -25.10845375061035,
      "activations/layer22_attention_weight_max": 35.76218032836914,
      "activations/layer22_attention_weight_min": -26.04764747619629,
      "activations/layer23_attention_weight_max": 36.67766571044922,
      "activations/layer23_attention_weight_min": -23.687192916870117,
      "activations/layer2_attention_weight_max": 31.299901962280273,
      "activations/layer2_attention_weight_min": -29.089038848876953,
      "activations/layer3_attention_weight_max": 91.60478973388672,
      "activations/layer3_attention_weight_min": -98.9677734375,
      "activations/layer4_attention_weight_max": 93.57931518554688,
      "activations/layer4_attention_weight_min": -90.79804992675781,
      "activations/layer5_attention_weight_max": 69.9385986328125,
      "activations/layer5_attention_weight_min": -80.30672454833984,
      "activations/layer6_attention_weight_max": 50.14936065673828,
      "activations/layer6_attention_weight_min": -49.7478141784668,
      "activations/layer7_attention_weight_max": 68.0800552368164,
      "activations/layer7_attention_weight_min": -63.203819274902344,
      "activations/layer8_attention_weight_max": 44.1513557434082,
      "activations/layer8_attention_weight_min": -42.827491760253906,
      "activations/layer9_attention_weight_max": 45.94980239868164,
      "activations/layer9_attention_weight_min": -46.48896408081055,
      "epoch": 12.81,
      "learning_rate": 6.810075757575757e-05,
      "loss": 2.7681,
      "step": 220400
    },
    {
      "activations/layer0_attention_weight_max": 15.71760368347168,
      "activations/layer0_attention_weight_min": -13.927295684814453,
      "activations/layer10_attention_weight_max": 34.93231964111328,
      "activations/layer10_attention_weight_min": -34.14326477050781,
      "activations/layer11_attention_weight_max": 34.48699951171875,
      "activations/layer11_attention_weight_min": -32.74032974243164,
      "activations/layer12_attention_weight_max": 19.535070419311523,
      "activations/layer12_attention_weight_min": -25.717790603637695,
      "activations/layer13_attention_weight_max": 35.10160827636719,
      "activations/layer13_attention_weight_min": -27.083995819091797,
      "activations/layer14_attention_weight_max": 37.18611526489258,
      "activations/layer14_attention_weight_min": -32.49544143676758,
      "activations/layer15_attention_weight_max": 33.858985900878906,
      "activations/layer15_attention_weight_min": -29.8294620513916,
      "activations/layer16_attention_weight_max": 33.68133544921875,
      "activations/layer16_attention_weight_min": -34.984474182128906,
      "activations/layer17_attention_weight_max": 52.33503341674805,
      "activations/layer17_attention_weight_min": -44.54911422729492,
      "activations/layer18_attention_weight_max": 47.56343078613281,
      "activations/layer18_attention_weight_min": -38.273414611816406,
      "activations/layer19_attention_weight_max": 23.4094181060791,
      "activations/layer19_attention_weight_min": -23.3084659576416,
      "activations/layer1_attention_weight_max": 17.334056854248047,
      "activations/layer1_attention_weight_min": -16.682092666625977,
      "activations/layer20_attention_weight_max": 23.61257553100586,
      "activations/layer20_attention_weight_min": -20.720684051513672,
      "activations/layer21_attention_weight_max": 33.49562454223633,
      "activations/layer21_attention_weight_min": -23.07085609436035,
      "activations/layer22_attention_weight_max": 30.870515823364258,
      "activations/layer22_attention_weight_min": -26.528358459472656,
      "activations/layer23_attention_weight_max": 36.914588928222656,
      "activations/layer23_attention_weight_min": -24.771785736083984,
      "activations/layer2_attention_weight_max": 33.719051361083984,
      "activations/layer2_attention_weight_min": -33.333126068115234,
      "activations/layer3_attention_weight_max": 90.97256469726562,
      "activations/layer3_attention_weight_min": -94.11351013183594,
      "activations/layer4_attention_weight_max": 91.60592651367188,
      "activations/layer4_attention_weight_min": -90.80802154541016,
      "activations/layer5_attention_weight_max": 73.24833679199219,
      "activations/layer5_attention_weight_min": -72.93446350097656,
      "activations/layer6_attention_weight_max": 49.56199645996094,
      "activations/layer6_attention_weight_min": -51.53667068481445,
      "activations/layer7_attention_weight_max": 77.48385620117188,
      "activations/layer7_attention_weight_min": -70.65159606933594,
      "activations/layer8_attention_weight_max": 47.35403060913086,
      "activations/layer8_attention_weight_min": -44.320281982421875,
      "activations/layer9_attention_weight_max": 58.135284423828125,
      "activations/layer9_attention_weight_min": -49.79819869995117,
      "epoch": 12.81,
      "learning_rate": 6.808181818181818e-05,
      "loss": 2.7748,
      "step": 220450
    },
    {
      "activations/layer0_attention_weight_max": 16.54520034790039,
      "activations/layer0_attention_weight_min": -13.975698471069336,
      "activations/layer10_attention_weight_max": 34.82875442504883,
      "activations/layer10_attention_weight_min": -35.300559997558594,
      "activations/layer11_attention_weight_max": 36.902374267578125,
      "activations/layer11_attention_weight_min": -33.64307403564453,
      "activations/layer12_attention_weight_max": 16.90207862854004,
      "activations/layer12_attention_weight_min": -28.00185775756836,
      "activations/layer13_attention_weight_max": 36.19633483886719,
      "activations/layer13_attention_weight_min": -27.0141544342041,
      "activations/layer14_attention_weight_max": 33.140235900878906,
      "activations/layer14_attention_weight_min": -26.437156677246094,
      "activations/layer15_attention_weight_max": 31.08559226989746,
      "activations/layer15_attention_weight_min": -27.280113220214844,
      "activations/layer16_attention_weight_max": 29.590679168701172,
      "activations/layer16_attention_weight_min": -27.86179542541504,
      "activations/layer17_attention_weight_max": 50.52364730834961,
      "activations/layer17_attention_weight_min": -40.13905715942383,
      "activations/layer18_attention_weight_max": 41.42613983154297,
      "activations/layer18_attention_weight_min": -36.376258850097656,
      "activations/layer19_attention_weight_max": 20.368505477905273,
      "activations/layer19_attention_weight_min": -21.36069679260254,
      "activations/layer1_attention_weight_max": 16.143892288208008,
      "activations/layer1_attention_weight_min": -14.125425338745117,
      "activations/layer20_attention_weight_max": 20.39195442199707,
      "activations/layer20_attention_weight_min": -21.257761001586914,
      "activations/layer21_attention_weight_max": 39.34123611450195,
      "activations/layer21_attention_weight_min": -21.713966369628906,
      "activations/layer22_attention_weight_max": 26.89828872680664,
      "activations/layer22_attention_weight_min": -23.894901275634766,
      "activations/layer23_attention_weight_max": 35.626869201660156,
      "activations/layer23_attention_weight_min": -25.524497985839844,
      "activations/layer2_attention_weight_max": 33.44129943847656,
      "activations/layer2_attention_weight_min": -31.072187423706055,
      "activations/layer3_attention_weight_max": 101.70565032958984,
      "activations/layer3_attention_weight_min": -100.35774993896484,
      "activations/layer4_attention_weight_max": 101.21160888671875,
      "activations/layer4_attention_weight_min": -94.78662872314453,
      "activations/layer5_attention_weight_max": 71.07725524902344,
      "activations/layer5_attention_weight_min": -75.5889663696289,
      "activations/layer6_attention_weight_max": 49.748252868652344,
      "activations/layer6_attention_weight_min": -50.17507553100586,
      "activations/layer7_attention_weight_max": 72.83467102050781,
      "activations/layer7_attention_weight_min": -65.141845703125,
      "activations/layer8_attention_weight_max": 44.165016174316406,
      "activations/layer8_attention_weight_min": -46.03971862792969,
      "activations/layer9_attention_weight_max": 55.05463409423828,
      "activations/layer9_attention_weight_min": -45.96085739135742,
      "epoch": 12.81,
      "learning_rate": 6.806287878787878e-05,
      "loss": 2.77,
      "step": 220500
    },
    {
      "activations/layer0_attention_weight_max": 15.978056907653809,
      "activations/layer0_attention_weight_min": -14.226591110229492,
      "activations/layer10_attention_weight_max": 36.07448196411133,
      "activations/layer10_attention_weight_min": -34.333431243896484,
      "activations/layer11_attention_weight_max": 30.2662410736084,
      "activations/layer11_attention_weight_min": -32.61830520629883,
      "activations/layer12_attention_weight_max": 17.377849578857422,
      "activations/layer12_attention_weight_min": -24.949066162109375,
      "activations/layer13_attention_weight_max": 36.48442840576172,
      "activations/layer13_attention_weight_min": -29.24490737915039,
      "activations/layer14_attention_weight_max": 34.88579177856445,
      "activations/layer14_attention_weight_min": -30.651248931884766,
      "activations/layer15_attention_weight_max": 33.81060791015625,
      "activations/layer15_attention_weight_min": -30.070457458496094,
      "activations/layer16_attention_weight_max": 31.35333251953125,
      "activations/layer16_attention_weight_min": -30.652799606323242,
      "activations/layer17_attention_weight_max": 53.85614776611328,
      "activations/layer17_attention_weight_min": -44.159881591796875,
      "activations/layer18_attention_weight_max": 47.27286148071289,
      "activations/layer18_attention_weight_min": -40.12239456176758,
      "activations/layer19_attention_weight_max": 24.439119338989258,
      "activations/layer19_attention_weight_min": -22.552732467651367,
      "activations/layer1_attention_weight_max": 17.115741729736328,
      "activations/layer1_attention_weight_min": -17.98935890197754,
      "activations/layer20_attention_weight_max": 24.750648498535156,
      "activations/layer20_attention_weight_min": -23.20124053955078,
      "activations/layer21_attention_weight_max": 38.42507553100586,
      "activations/layer21_attention_weight_min": -22.350088119506836,
      "activations/layer22_attention_weight_max": 29.77937889099121,
      "activations/layer22_attention_weight_min": -25.696048736572266,
      "activations/layer23_attention_weight_max": 40.254493713378906,
      "activations/layer23_attention_weight_min": -25.45203971862793,
      "activations/layer2_attention_weight_max": 31.94551658630371,
      "activations/layer2_attention_weight_min": -31.435649871826172,
      "activations/layer3_attention_weight_max": 90.25431060791016,
      "activations/layer3_attention_weight_min": -93.26485443115234,
      "activations/layer4_attention_weight_max": 94.93529510498047,
      "activations/layer4_attention_weight_min": -91.5086898803711,
      "activations/layer5_attention_weight_max": 74.56466674804688,
      "activations/layer5_attention_weight_min": -75.42660522460938,
      "activations/layer6_attention_weight_max": 52.35416030883789,
      "activations/layer6_attention_weight_min": -49.29108810424805,
      "activations/layer7_attention_weight_max": 69.56827545166016,
      "activations/layer7_attention_weight_min": -64.66132354736328,
      "activations/layer8_attention_weight_max": 43.10466766357422,
      "activations/layer8_attention_weight_min": -44.35427474975586,
      "activations/layer9_attention_weight_max": 44.015193939208984,
      "activations/layer9_attention_weight_min": -45.45250701904297,
      "epoch": 12.82,
      "learning_rate": 6.804393939393939e-05,
      "loss": 2.7696,
      "step": 220550
    },
    {
      "activations/layer0_attention_weight_max": 16.376075744628906,
      "activations/layer0_attention_weight_min": -14.031792640686035,
      "activations/layer10_attention_weight_max": 34.72868728637695,
      "activations/layer10_attention_weight_min": -33.799896240234375,
      "activations/layer11_attention_weight_max": 32.479942321777344,
      "activations/layer11_attention_weight_min": -33.649009704589844,
      "activations/layer12_attention_weight_max": 18.467939376831055,
      "activations/layer12_attention_weight_min": -24.61203956604004,
      "activations/layer13_attention_weight_max": 34.53437423706055,
      "activations/layer13_attention_weight_min": -27.199993133544922,
      "activations/layer14_attention_weight_max": 34.13420867919922,
      "activations/layer14_attention_weight_min": -29.79737663269043,
      "activations/layer15_attention_weight_max": 31.234975814819336,
      "activations/layer15_attention_weight_min": -29.449968338012695,
      "activations/layer16_attention_weight_max": 28.92055320739746,
      "activations/layer16_attention_weight_min": -31.204586029052734,
      "activations/layer17_attention_weight_max": 46.82490921020508,
      "activations/layer17_attention_weight_min": -44.9904899597168,
      "activations/layer18_attention_weight_max": 43.2022590637207,
      "activations/layer18_attention_weight_min": -38.9927978515625,
      "activations/layer19_attention_weight_max": 20.788311004638672,
      "activations/layer19_attention_weight_min": -20.760143280029297,
      "activations/layer1_attention_weight_max": 15.716694831848145,
      "activations/layer1_attention_weight_min": -14.610318183898926,
      "activations/layer20_attention_weight_max": 22.063413619995117,
      "activations/layer20_attention_weight_min": -19.981361389160156,
      "activations/layer21_attention_weight_max": 32.104942321777344,
      "activations/layer21_attention_weight_min": -19.87799072265625,
      "activations/layer22_attention_weight_max": 27.02376365661621,
      "activations/layer22_attention_weight_min": -23.858810424804688,
      "activations/layer23_attention_weight_max": 37.125396728515625,
      "activations/layer23_attention_weight_min": -22.488811492919922,
      "activations/layer2_attention_weight_max": 32.66474151611328,
      "activations/layer2_attention_weight_min": -31.726409912109375,
      "activations/layer3_attention_weight_max": 92.2262954711914,
      "activations/layer3_attention_weight_min": -95.38938903808594,
      "activations/layer4_attention_weight_max": 92.68306732177734,
      "activations/layer4_attention_weight_min": -93.9383544921875,
      "activations/layer5_attention_weight_max": 69.34513854980469,
      "activations/layer5_attention_weight_min": -75.28693389892578,
      "activations/layer6_attention_weight_max": 48.71434020996094,
      "activations/layer6_attention_weight_min": -48.40475845336914,
      "activations/layer7_attention_weight_max": 65.19390869140625,
      "activations/layer7_attention_weight_min": -62.61225891113281,
      "activations/layer8_attention_weight_max": 44.39213180541992,
      "activations/layer8_attention_weight_min": -46.59282684326172,
      "activations/layer9_attention_weight_max": 42.40558624267578,
      "activations/layer9_attention_weight_min": -45.36747360229492,
      "epoch": 12.82,
      "learning_rate": 6.8025e-05,
      "loss": 2.7782,
      "step": 220600
    },
    {
      "activations/layer0_attention_weight_max": 16.405555725097656,
      "activations/layer0_attention_weight_min": -13.98568058013916,
      "activations/layer10_attention_weight_max": 34.73706817626953,
      "activations/layer10_attention_weight_min": -35.053741455078125,
      "activations/layer11_attention_weight_max": 31.1685848236084,
      "activations/layer11_attention_weight_min": -30.612728118896484,
      "activations/layer12_attention_weight_max": 17.253177642822266,
      "activations/layer12_attention_weight_min": -24.957096099853516,
      "activations/layer13_attention_weight_max": 36.160396575927734,
      "activations/layer13_attention_weight_min": -28.407869338989258,
      "activations/layer14_attention_weight_max": 34.62661361694336,
      "activations/layer14_attention_weight_min": -30.318653106689453,
      "activations/layer15_attention_weight_max": 31.88422966003418,
      "activations/layer15_attention_weight_min": -29.361841201782227,
      "activations/layer16_attention_weight_max": 31.61318016052246,
      "activations/layer16_attention_weight_min": -29.731834411621094,
      "activations/layer17_attention_weight_max": 50.12478256225586,
      "activations/layer17_attention_weight_min": -43.84986877441406,
      "activations/layer18_attention_weight_max": 47.87238693237305,
      "activations/layer18_attention_weight_min": -38.29231643676758,
      "activations/layer19_attention_weight_max": 23.29704475402832,
      "activations/layer19_attention_weight_min": -21.684412002563477,
      "activations/layer1_attention_weight_max": 16.519329071044922,
      "activations/layer1_attention_weight_min": -15.631999015808105,
      "activations/layer20_attention_weight_max": 25.116539001464844,
      "activations/layer20_attention_weight_min": -20.089229583740234,
      "activations/layer21_attention_weight_max": 33.318397521972656,
      "activations/layer21_attention_weight_min": -24.128398895263672,
      "activations/layer22_attention_weight_max": 30.554367065429688,
      "activations/layer22_attention_weight_min": -27.189865112304688,
      "activations/layer23_attention_weight_max": 36.285362243652344,
      "activations/layer23_attention_weight_min": -25.535259246826172,
      "activations/layer2_attention_weight_max": 33.16624450683594,
      "activations/layer2_attention_weight_min": -30.265567779541016,
      "activations/layer3_attention_weight_max": 95.96463775634766,
      "activations/layer3_attention_weight_min": -94.45494842529297,
      "activations/layer4_attention_weight_max": 93.6449203491211,
      "activations/layer4_attention_weight_min": -95.96340942382812,
      "activations/layer5_attention_weight_max": 68.51448059082031,
      "activations/layer5_attention_weight_min": -77.19519805908203,
      "activations/layer6_attention_weight_max": 50.53840637207031,
      "activations/layer6_attention_weight_min": -50.75948715209961,
      "activations/layer7_attention_weight_max": 70.3837890625,
      "activations/layer7_attention_weight_min": -66.2171401977539,
      "activations/layer8_attention_weight_max": 45.41939926147461,
      "activations/layer8_attention_weight_min": -48.323089599609375,
      "activations/layer9_attention_weight_max": 50.16151809692383,
      "activations/layer9_attention_weight_min": -45.41276168823242,
      "epoch": 12.82,
      "learning_rate": 6.80060606060606e-05,
      "loss": 2.7573,
      "step": 220650
    },
    {
      "activations/layer0_attention_weight_max": 15.84645938873291,
      "activations/layer0_attention_weight_min": -14.192336082458496,
      "activations/layer10_attention_weight_max": 36.75767135620117,
      "activations/layer10_attention_weight_min": -37.95908737182617,
      "activations/layer11_attention_weight_max": 33.68199920654297,
      "activations/layer11_attention_weight_min": -38.215904235839844,
      "activations/layer12_attention_weight_max": 18.54775619506836,
      "activations/layer12_attention_weight_min": -25.032732009887695,
      "activations/layer13_attention_weight_max": 38.88884735107422,
      "activations/layer13_attention_weight_min": -31.44171905517578,
      "activations/layer14_attention_weight_max": 36.43980407714844,
      "activations/layer14_attention_weight_min": -29.35324478149414,
      "activations/layer15_attention_weight_max": 33.03764343261719,
      "activations/layer15_attention_weight_min": -28.47394561767578,
      "activations/layer16_attention_weight_max": 33.36109924316406,
      "activations/layer16_attention_weight_min": -29.7388858795166,
      "activations/layer17_attention_weight_max": 50.86967849731445,
      "activations/layer17_attention_weight_min": -44.615150451660156,
      "activations/layer18_attention_weight_max": 45.490745544433594,
      "activations/layer18_attention_weight_min": -38.174720764160156,
      "activations/layer19_attention_weight_max": 21.16017723083496,
      "activations/layer19_attention_weight_min": -23.617223739624023,
      "activations/layer1_attention_weight_max": 17.402084350585938,
      "activations/layer1_attention_weight_min": -16.181495666503906,
      "activations/layer20_attention_weight_max": 22.418704986572266,
      "activations/layer20_attention_weight_min": -26.373340606689453,
      "activations/layer21_attention_weight_max": 31.172317504882812,
      "activations/layer21_attention_weight_min": -21.44106101989746,
      "activations/layer22_attention_weight_max": 28.770105361938477,
      "activations/layer22_attention_weight_min": -26.094825744628906,
      "activations/layer23_attention_weight_max": 36.72584533691406,
      "activations/layer23_attention_weight_min": -24.69498062133789,
      "activations/layer2_attention_weight_max": 33.29253005981445,
      "activations/layer2_attention_weight_min": -34.19710159301758,
      "activations/layer3_attention_weight_max": 98.17638397216797,
      "activations/layer3_attention_weight_min": -97.92090606689453,
      "activations/layer4_attention_weight_max": 99.70965576171875,
      "activations/layer4_attention_weight_min": -98.73402404785156,
      "activations/layer5_attention_weight_max": 70.98304748535156,
      "activations/layer5_attention_weight_min": -83.08148193359375,
      "activations/layer6_attention_weight_max": 51.39612579345703,
      "activations/layer6_attention_weight_min": -54.27212905883789,
      "activations/layer7_attention_weight_max": 69.25872039794922,
      "activations/layer7_attention_weight_min": -69.61979675292969,
      "activations/layer8_attention_weight_max": 46.96123123168945,
      "activations/layer8_attention_weight_min": -50.04594802856445,
      "activations/layer9_attention_weight_max": 53.311614990234375,
      "activations/layer9_attention_weight_min": -49.275936126708984,
      "epoch": 12.82,
      "learning_rate": 6.79871212121212e-05,
      "loss": 2.7717,
      "step": 220700
    },
    {
      "activations/layer0_attention_weight_max": 15.918197631835938,
      "activations/layer0_attention_weight_min": -14.104034423828125,
      "activations/layer10_attention_weight_max": 41.85377883911133,
      "activations/layer10_attention_weight_min": -36.7866096496582,
      "activations/layer11_attention_weight_max": 36.210426330566406,
      "activations/layer11_attention_weight_min": -34.77838134765625,
      "activations/layer12_attention_weight_max": 16.99751091003418,
      "activations/layer12_attention_weight_min": -25.409265518188477,
      "activations/layer13_attention_weight_max": 39.11585998535156,
      "activations/layer13_attention_weight_min": -32.7191047668457,
      "activations/layer14_attention_weight_max": 34.25638198852539,
      "activations/layer14_attention_weight_min": -31.26361083984375,
      "activations/layer15_attention_weight_max": 30.367414474487305,
      "activations/layer15_attention_weight_min": -31.981386184692383,
      "activations/layer16_attention_weight_max": 31.581790924072266,
      "activations/layer16_attention_weight_min": -27.839202880859375,
      "activations/layer17_attention_weight_max": 53.18241882324219,
      "activations/layer17_attention_weight_min": -46.295265197753906,
      "activations/layer18_attention_weight_max": 45.97128677368164,
      "activations/layer18_attention_weight_min": -37.53675079345703,
      "activations/layer19_attention_weight_max": 22.07076644897461,
      "activations/layer19_attention_weight_min": -23.287105560302734,
      "activations/layer1_attention_weight_max": 17.428226470947266,
      "activations/layer1_attention_weight_min": -15.163495063781738,
      "activations/layer20_attention_weight_max": 23.36361312866211,
      "activations/layer20_attention_weight_min": -22.663164138793945,
      "activations/layer21_attention_weight_max": 34.91284942626953,
      "activations/layer21_attention_weight_min": -21.205280303955078,
      "activations/layer22_attention_weight_max": 28.36833953857422,
      "activations/layer22_attention_weight_min": -24.823347091674805,
      "activations/layer23_attention_weight_max": 37.208885192871094,
      "activations/layer23_attention_weight_min": -25.01773452758789,
      "activations/layer2_attention_weight_max": 35.56248474121094,
      "activations/layer2_attention_weight_min": -34.63288879394531,
      "activations/layer3_attention_weight_max": 103.5924072265625,
      "activations/layer3_attention_weight_min": -111.16786193847656,
      "activations/layer4_attention_weight_max": 97.96158599853516,
      "activations/layer4_attention_weight_min": -103.51058197021484,
      "activations/layer5_attention_weight_max": 69.3946533203125,
      "activations/layer5_attention_weight_min": -76.62173461914062,
      "activations/layer6_attention_weight_max": 51.93733215332031,
      "activations/layer6_attention_weight_min": -54.639102935791016,
      "activations/layer7_attention_weight_max": 67.9324951171875,
      "activations/layer7_attention_weight_min": -71.27761840820312,
      "activations/layer8_attention_weight_max": 49.9008903503418,
      "activations/layer8_attention_weight_min": -48.594886779785156,
      "activations/layer9_attention_weight_max": 52.74660873413086,
      "activations/layer9_attention_weight_min": -47.02168655395508,
      "epoch": 12.83,
      "learning_rate": 6.796818181818182e-05,
      "loss": 2.7692,
      "step": 220750
    },
    {
      "activations/layer0_attention_weight_max": 15.523666381835938,
      "activations/layer0_attention_weight_min": -14.37978744506836,
      "activations/layer10_attention_weight_max": 37.04336166381836,
      "activations/layer10_attention_weight_min": -35.98224639892578,
      "activations/layer11_attention_weight_max": 34.737091064453125,
      "activations/layer11_attention_weight_min": -33.30703353881836,
      "activations/layer12_attention_weight_max": 18.44766616821289,
      "activations/layer12_attention_weight_min": -23.386621475219727,
      "activations/layer13_attention_weight_max": 34.08028793334961,
      "activations/layer13_attention_weight_min": -29.6655330657959,
      "activations/layer14_attention_weight_max": 36.27257537841797,
      "activations/layer14_attention_weight_min": -29.532472610473633,
      "activations/layer15_attention_weight_max": 32.02231216430664,
      "activations/layer15_attention_weight_min": -30.42609214782715,
      "activations/layer16_attention_weight_max": 30.534549713134766,
      "activations/layer16_attention_weight_min": -29.61590576171875,
      "activations/layer17_attention_weight_max": 52.42683410644531,
      "activations/layer17_attention_weight_min": -44.268653869628906,
      "activations/layer18_attention_weight_max": 46.341426849365234,
      "activations/layer18_attention_weight_min": -38.55956268310547,
      "activations/layer19_attention_weight_max": 21.841779708862305,
      "activations/layer19_attention_weight_min": -21.365741729736328,
      "activations/layer1_attention_weight_max": 15.918410301208496,
      "activations/layer1_attention_weight_min": -14.069801330566406,
      "activations/layer20_attention_weight_max": 21.942180633544922,
      "activations/layer20_attention_weight_min": -23.256357192993164,
      "activations/layer21_attention_weight_max": 34.78557205200195,
      "activations/layer21_attention_weight_min": -21.303895950317383,
      "activations/layer22_attention_weight_max": 28.787200927734375,
      "activations/layer22_attention_weight_min": -25.411542892456055,
      "activations/layer23_attention_weight_max": 36.38390350341797,
      "activations/layer23_attention_weight_min": -24.440771102905273,
      "activations/layer2_attention_weight_max": 32.967369079589844,
      "activations/layer2_attention_weight_min": -32.76131820678711,
      "activations/layer3_attention_weight_max": 98.66934204101562,
      "activations/layer3_attention_weight_min": -102.86658477783203,
      "activations/layer4_attention_weight_max": 95.4791259765625,
      "activations/layer4_attention_weight_min": -94.13462829589844,
      "activations/layer5_attention_weight_max": 71.44755554199219,
      "activations/layer5_attention_weight_min": -79.17022705078125,
      "activations/layer6_attention_weight_max": 51.61396026611328,
      "activations/layer6_attention_weight_min": -53.375362396240234,
      "activations/layer7_attention_weight_max": 67.482177734375,
      "activations/layer7_attention_weight_min": -65.7995376586914,
      "activations/layer8_attention_weight_max": 44.1063346862793,
      "activations/layer8_attention_weight_min": -45.56114196777344,
      "activations/layer9_attention_weight_max": 44.012413024902344,
      "activations/layer9_attention_weight_min": -48.720603942871094,
      "epoch": 12.83,
      "learning_rate": 6.794924242424242e-05,
      "loss": 2.7465,
      "step": 220800
    },
    {
      "activations/layer0_attention_weight_max": 16.3170108795166,
      "activations/layer0_attention_weight_min": -14.326822280883789,
      "activations/layer10_attention_weight_max": 37.62255859375,
      "activations/layer10_attention_weight_min": -36.29024124145508,
      "activations/layer11_attention_weight_max": 33.234596252441406,
      "activations/layer11_attention_weight_min": -33.837432861328125,
      "activations/layer12_attention_weight_max": 19.752771377563477,
      "activations/layer12_attention_weight_min": -22.787738800048828,
      "activations/layer13_attention_weight_max": 44.75278091430664,
      "activations/layer13_attention_weight_min": -29.339988708496094,
      "activations/layer14_attention_weight_max": 44.694007873535156,
      "activations/layer14_attention_weight_min": -32.60980987548828,
      "activations/layer15_attention_weight_max": 36.73698806762695,
      "activations/layer15_attention_weight_min": -30.633886337280273,
      "activations/layer16_attention_weight_max": 34.175052642822266,
      "activations/layer16_attention_weight_min": -30.958234786987305,
      "activations/layer17_attention_weight_max": 54.08039474487305,
      "activations/layer17_attention_weight_min": -48.30519485473633,
      "activations/layer18_attention_weight_max": 52.53261947631836,
      "activations/layer18_attention_weight_min": -43.679840087890625,
      "activations/layer19_attention_weight_max": 26.2867431640625,
      "activations/layer19_attention_weight_min": -22.77912712097168,
      "activations/layer1_attention_weight_max": 16.87932586669922,
      "activations/layer1_attention_weight_min": -15.201631546020508,
      "activations/layer20_attention_weight_max": 26.168073654174805,
      "activations/layer20_attention_weight_min": -22.00657844543457,
      "activations/layer21_attention_weight_max": 37.88567352294922,
      "activations/layer21_attention_weight_min": -25.381675720214844,
      "activations/layer22_attention_weight_max": 33.34695053100586,
      "activations/layer22_attention_weight_min": -27.902545928955078,
      "activations/layer23_attention_weight_max": 43.58980941772461,
      "activations/layer23_attention_weight_min": -24.59210777282715,
      "activations/layer2_attention_weight_max": 32.24767303466797,
      "activations/layer2_attention_weight_min": -32.39588165283203,
      "activations/layer3_attention_weight_max": 95.25250244140625,
      "activations/layer3_attention_weight_min": -100.71821594238281,
      "activations/layer4_attention_weight_max": 98.05630493164062,
      "activations/layer4_attention_weight_min": -95.70753479003906,
      "activations/layer5_attention_weight_max": 73.84854888916016,
      "activations/layer5_attention_weight_min": -77.83185577392578,
      "activations/layer6_attention_weight_max": 55.37141799926758,
      "activations/layer6_attention_weight_min": -55.711307525634766,
      "activations/layer7_attention_weight_max": 71.01103210449219,
      "activations/layer7_attention_weight_min": -70.02496337890625,
      "activations/layer8_attention_weight_max": 47.429466247558594,
      "activations/layer8_attention_weight_min": -53.69903564453125,
      "activations/layer9_attention_weight_max": 45.47684860229492,
      "activations/layer9_attention_weight_min": -48.446876525878906,
      "epoch": 12.83,
      "learning_rate": 6.793030303030302e-05,
      "loss": 2.7875,
      "step": 220850
    },
    {
      "activations/layer0_attention_weight_max": 17.06935691833496,
      "activations/layer0_attention_weight_min": -14.468320846557617,
      "activations/layer10_attention_weight_max": 38.406494140625,
      "activations/layer10_attention_weight_min": -37.19761657714844,
      "activations/layer11_attention_weight_max": 32.8332633972168,
      "activations/layer11_attention_weight_min": -32.26758575439453,
      "activations/layer12_attention_weight_max": 20.674667358398438,
      "activations/layer12_attention_weight_min": -25.9654598236084,
      "activations/layer13_attention_weight_max": 39.365386962890625,
      "activations/layer13_attention_weight_min": -29.674068450927734,
      "activations/layer14_attention_weight_max": 40.46567916870117,
      "activations/layer14_attention_weight_min": -29.748939514160156,
      "activations/layer15_attention_weight_max": 34.1086311340332,
      "activations/layer15_attention_weight_min": -30.427993774414062,
      "activations/layer16_attention_weight_max": 34.525291442871094,
      "activations/layer16_attention_weight_min": -33.06237030029297,
      "activations/layer17_attention_weight_max": 57.3543586730957,
      "activations/layer17_attention_weight_min": -45.935306549072266,
      "activations/layer18_attention_weight_max": 52.86545944213867,
      "activations/layer18_attention_weight_min": -39.28819274902344,
      "activations/layer19_attention_weight_max": 26.25528907775879,
      "activations/layer19_attention_weight_min": -22.94672203063965,
      "activations/layer1_attention_weight_max": 16.09311866760254,
      "activations/layer1_attention_weight_min": -14.711823463439941,
      "activations/layer20_attention_weight_max": 26.203533172607422,
      "activations/layer20_attention_weight_min": -23.336509704589844,
      "activations/layer21_attention_weight_max": 40.973358154296875,
      "activations/layer21_attention_weight_min": -23.812408447265625,
      "activations/layer22_attention_weight_max": 34.79684829711914,
      "activations/layer22_attention_weight_min": -29.92330551147461,
      "activations/layer23_attention_weight_max": 41.67945098876953,
      "activations/layer23_attention_weight_min": -24.016803741455078,
      "activations/layer2_attention_weight_max": 32.27491760253906,
      "activations/layer2_attention_weight_min": -30.479421615600586,
      "activations/layer3_attention_weight_max": 93.5506591796875,
      "activations/layer3_attention_weight_min": -90.91191864013672,
      "activations/layer4_attention_weight_max": 94.42424774169922,
      "activations/layer4_attention_weight_min": -92.5845947265625,
      "activations/layer5_attention_weight_max": 74.67636108398438,
      "activations/layer5_attention_weight_min": -77.45496368408203,
      "activations/layer6_attention_weight_max": 52.36788558959961,
      "activations/layer6_attention_weight_min": -50.43767547607422,
      "activations/layer7_attention_weight_max": 67.75482177734375,
      "activations/layer7_attention_weight_min": -68.16260528564453,
      "activations/layer8_attention_weight_max": 47.33868408203125,
      "activations/layer8_attention_weight_min": -47.57185363769531,
      "activations/layer9_attention_weight_max": 45.911495208740234,
      "activations/layer9_attention_weight_min": -46.984771728515625,
      "epoch": 12.84,
      "learning_rate": 6.791136363636362e-05,
      "loss": 2.7642,
      "step": 220900
    },
    {
      "activations/layer0_attention_weight_max": 16.43517303466797,
      "activations/layer0_attention_weight_min": -14.262775421142578,
      "activations/layer10_attention_weight_max": 35.86420440673828,
      "activations/layer10_attention_weight_min": -34.31481170654297,
      "activations/layer11_attention_weight_max": 34.184425354003906,
      "activations/layer11_attention_weight_min": -32.95664978027344,
      "activations/layer12_attention_weight_max": 18.08953285217285,
      "activations/layer12_attention_weight_min": -27.128524780273438,
      "activations/layer13_attention_weight_max": 36.23105239868164,
      "activations/layer13_attention_weight_min": -27.448104858398438,
      "activations/layer14_attention_weight_max": 36.04885482788086,
      "activations/layer14_attention_weight_min": -30.45694351196289,
      "activations/layer15_attention_weight_max": 33.565242767333984,
      "activations/layer15_attention_weight_min": -29.746702194213867,
      "activations/layer16_attention_weight_max": 32.493011474609375,
      "activations/layer16_attention_weight_min": -31.074588775634766,
      "activations/layer17_attention_weight_max": 52.019554138183594,
      "activations/layer17_attention_weight_min": -44.61250686645508,
      "activations/layer18_attention_weight_max": 47.10562515258789,
      "activations/layer18_attention_weight_min": -39.52810287475586,
      "activations/layer19_attention_weight_max": 22.80158805847168,
      "activations/layer19_attention_weight_min": -21.756568908691406,
      "activations/layer1_attention_weight_max": 18.21622657775879,
      "activations/layer1_attention_weight_min": -14.595085144042969,
      "activations/layer20_attention_weight_max": 27.356616973876953,
      "activations/layer20_attention_weight_min": -20.80937957763672,
      "activations/layer21_attention_weight_max": 37.87615966796875,
      "activations/layer21_attention_weight_min": -21.29518699645996,
      "activations/layer22_attention_weight_max": 29.972009658813477,
      "activations/layer22_attention_weight_min": -25.63445472717285,
      "activations/layer23_attention_weight_max": 38.87897491455078,
      "activations/layer23_attention_weight_min": -26.94968032836914,
      "activations/layer2_attention_weight_max": 33.77273941040039,
      "activations/layer2_attention_weight_min": -32.24393081665039,
      "activations/layer3_attention_weight_max": 94.83438110351562,
      "activations/layer3_attention_weight_min": -99.01207733154297,
      "activations/layer4_attention_weight_max": 100.08263397216797,
      "activations/layer4_attention_weight_min": -96.69673919677734,
      "activations/layer5_attention_weight_max": 70.9527816772461,
      "activations/layer5_attention_weight_min": -83.58977508544922,
      "activations/layer6_attention_weight_max": 51.63965606689453,
      "activations/layer6_attention_weight_min": -53.49199295043945,
      "activations/layer7_attention_weight_max": 68.90949249267578,
      "activations/layer7_attention_weight_min": -67.6654281616211,
      "activations/layer8_attention_weight_max": 45.100494384765625,
      "activations/layer8_attention_weight_min": -47.22102355957031,
      "activations/layer9_attention_weight_max": 49.76043701171875,
      "activations/layer9_attention_weight_min": -47.94816207885742,
      "epoch": 12.84,
      "learning_rate": 6.789242424242424e-05,
      "loss": 2.7811,
      "step": 220950
    },
    {
      "activations/layer0_attention_weight_max": 17.03789520263672,
      "activations/layer0_attention_weight_min": -14.3628511428833,
      "activations/layer10_attention_weight_max": 39.76934814453125,
      "activations/layer10_attention_weight_min": -38.11286544799805,
      "activations/layer11_attention_weight_max": 33.177825927734375,
      "activations/layer11_attention_weight_min": -34.297828674316406,
      "activations/layer12_attention_weight_max": 19.000152587890625,
      "activations/layer12_attention_weight_min": -24.099353790283203,
      "activations/layer13_attention_weight_max": 43.32167434692383,
      "activations/layer13_attention_weight_min": -29.285594940185547,
      "activations/layer14_attention_weight_max": 42.14008712768555,
      "activations/layer14_attention_weight_min": -29.684463500976562,
      "activations/layer15_attention_weight_max": 36.61477279663086,
      "activations/layer15_attention_weight_min": -31.056514739990234,
      "activations/layer16_attention_weight_max": 33.69855499267578,
      "activations/layer16_attention_weight_min": -29.61395835876465,
      "activations/layer17_attention_weight_max": 53.34437942504883,
      "activations/layer17_attention_weight_min": -47.67136001586914,
      "activations/layer18_attention_weight_max": 51.80939865112305,
      "activations/layer18_attention_weight_min": -41.5509147644043,
      "activations/layer19_attention_weight_max": 25.697561264038086,
      "activations/layer19_attention_weight_min": -25.11800765991211,
      "activations/layer1_attention_weight_max": 18.383392333984375,
      "activations/layer1_attention_weight_min": -15.557878494262695,
      "activations/layer20_attention_weight_max": 25.928049087524414,
      "activations/layer20_attention_weight_min": -22.307758331298828,
      "activations/layer21_attention_weight_max": 43.44377517700195,
      "activations/layer21_attention_weight_min": -23.655590057373047,
      "activations/layer22_attention_weight_max": 35.38841247558594,
      "activations/layer22_attention_weight_min": -28.560375213623047,
      "activations/layer23_attention_weight_max": 43.95317077636719,
      "activations/layer23_attention_weight_min": -23.966596603393555,
      "activations/layer2_attention_weight_max": 34.4034309387207,
      "activations/layer2_attention_weight_min": -32.269309997558594,
      "activations/layer3_attention_weight_max": 103.52899169921875,
      "activations/layer3_attention_weight_min": -100.62873077392578,
      "activations/layer4_attention_weight_max": 103.54544830322266,
      "activations/layer4_attention_weight_min": -97.72821044921875,
      "activations/layer5_attention_weight_max": 77.31661987304688,
      "activations/layer5_attention_weight_min": -80.84556579589844,
      "activations/layer6_attention_weight_max": 55.9124641418457,
      "activations/layer6_attention_weight_min": -52.37282943725586,
      "activations/layer7_attention_weight_max": 71.70927429199219,
      "activations/layer7_attention_weight_min": -71.15899658203125,
      "activations/layer8_attention_weight_max": 49.984466552734375,
      "activations/layer8_attention_weight_min": -50.44719314575195,
      "activations/layer9_attention_weight_max": 49.265159606933594,
      "activations/layer9_attention_weight_min": -46.91544723510742,
      "epoch": 12.84,
      "learning_rate": 6.787348484848484e-05,
      "loss": 2.7683,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4649,
      "eval_samples_per_second": 507.272,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4649,
      "eval_openwebtext_samples_per_second": 507.272,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9507,
      "eval_wikitext_samples_per_second": 233.767,
      "step": 221000
    },
    {
      "epoch": 12.84,
      "eval_lambada_loss": 2.587890625,
      "eval_lambada_ppl": 13.301683750542033,
      "eval_lambada_runtime": 9.5138,
      "eval_lambada_samples_per_second": 511.785,
      "step": 221000
    },
    {
      "activations/layer0_attention_weight_max": 16.686660766601562,
      "activations/layer0_attention_weight_min": -14.147175788879395,
      "activations/layer10_attention_weight_max": 34.65300750732422,
      "activations/layer10_attention_weight_min": -34.09006118774414,
      "activations/layer11_attention_weight_max": 32.377052307128906,
      "activations/layer11_attention_weight_min": -33.63513946533203,
      "activations/layer12_attention_weight_max": 21.59256362915039,
      "activations/layer12_attention_weight_min": -27.328384399414062,
      "activations/layer13_attention_weight_max": 40.55837631225586,
      "activations/layer13_attention_weight_min": -29.663660049438477,
      "activations/layer14_attention_weight_max": 39.31956100463867,
      "activations/layer14_attention_weight_min": -32.4427375793457,
      "activations/layer15_attention_weight_max": 34.817989349365234,
      "activations/layer15_attention_weight_min": -30.6528377532959,
      "activations/layer16_attention_weight_max": 35.09672164916992,
      "activations/layer16_attention_weight_min": -33.45536422729492,
      "activations/layer17_attention_weight_max": 60.64251708984375,
      "activations/layer17_attention_weight_min": -53.02129364013672,
      "activations/layer18_attention_weight_max": 48.4510383605957,
      "activations/layer18_attention_weight_min": -42.81577682495117,
      "activations/layer19_attention_weight_max": 21.986486434936523,
      "activations/layer19_attention_weight_min": -23.072050094604492,
      "activations/layer1_attention_weight_max": 17.32206916809082,
      "activations/layer1_attention_weight_min": -16.207788467407227,
      "activations/layer20_attention_weight_max": 26.41228485107422,
      "activations/layer20_attention_weight_min": -22.972034454345703,
      "activations/layer21_attention_weight_max": 41.79915237426758,
      "activations/layer21_attention_weight_min": -27.94845962524414,
      "activations/layer22_attention_weight_max": 33.40572738647461,
      "activations/layer22_attention_weight_min": -26.12665557861328,
      "activations/layer23_attention_weight_max": 38.32647705078125,
      "activations/layer23_attention_weight_min": -25.04273796081543,
      "activations/layer2_attention_weight_max": 33.30717849731445,
      "activations/layer2_attention_weight_min": -32.13037109375,
      "activations/layer3_attention_weight_max": 94.1317367553711,
      "activations/layer3_attention_weight_min": -96.37527465820312,
      "activations/layer4_attention_weight_max": 97.04944610595703,
      "activations/layer4_attention_weight_min": -92.03484344482422,
      "activations/layer5_attention_weight_max": 72.49809265136719,
      "activations/layer5_attention_weight_min": -76.33511352539062,
      "activations/layer6_attention_weight_max": 54.74173355102539,
      "activations/layer6_attention_weight_min": -55.94095230102539,
      "activations/layer7_attention_weight_max": 74.24405670166016,
      "activations/layer7_attention_weight_min": -67.6110610961914,
      "activations/layer8_attention_weight_max": 43.65595245361328,
      "activations/layer8_attention_weight_min": -47.31308364868164,
      "activations/layer9_attention_weight_max": 46.07648849487305,
      "activations/layer9_attention_weight_min": -46.180416107177734,
      "epoch": 12.84,
      "learning_rate": 6.785454545454545e-05,
      "loss": 2.7688,
      "step": 221050
    },
    {
      "activations/layer0_attention_weight_max": 15.765463829040527,
      "activations/layer0_attention_weight_min": -14.170198440551758,
      "activations/layer10_attention_weight_max": 37.01178741455078,
      "activations/layer10_attention_weight_min": -35.63188934326172,
      "activations/layer11_attention_weight_max": 33.106468200683594,
      "activations/layer11_attention_weight_min": -33.763980865478516,
      "activations/layer12_attention_weight_max": 19.74728775024414,
      "activations/layer12_attention_weight_min": -23.640296936035156,
      "activations/layer13_attention_weight_max": 34.58460235595703,
      "activations/layer13_attention_weight_min": -30.867040634155273,
      "activations/layer14_attention_weight_max": 35.57207107543945,
      "activations/layer14_attention_weight_min": -29.373605728149414,
      "activations/layer15_attention_weight_max": 34.709903717041016,
      "activations/layer15_attention_weight_min": -29.857282638549805,
      "activations/layer16_attention_weight_max": 30.685197830200195,
      "activations/layer16_attention_weight_min": -29.089189529418945,
      "activations/layer17_attention_weight_max": 49.441123962402344,
      "activations/layer17_attention_weight_min": -43.216590881347656,
      "activations/layer18_attention_weight_max": 47.24835968017578,
      "activations/layer18_attention_weight_min": -36.94858169555664,
      "activations/layer19_attention_weight_max": 22.237401962280273,
      "activations/layer19_attention_weight_min": -20.556943893432617,
      "activations/layer1_attention_weight_max": 17.502649307250977,
      "activations/layer1_attention_weight_min": -17.183345794677734,
      "activations/layer20_attention_weight_max": 23.339611053466797,
      "activations/layer20_attention_weight_min": -23.261369705200195,
      "activations/layer21_attention_weight_max": 41.40732955932617,
      "activations/layer21_attention_weight_min": -28.619407653808594,
      "activations/layer22_attention_weight_max": 28.60956573486328,
      "activations/layer22_attention_weight_min": -27.456872940063477,
      "activations/layer23_attention_weight_max": 34.30824279785156,
      "activations/layer23_attention_weight_min": -23.0064754486084,
      "activations/layer2_attention_weight_max": 31.35619354248047,
      "activations/layer2_attention_weight_min": -30.60377311706543,
      "activations/layer3_attention_weight_max": 90.02828979492188,
      "activations/layer3_attention_weight_min": -95.1904296875,
      "activations/layer4_attention_weight_max": 94.60450744628906,
      "activations/layer4_attention_weight_min": -91.13311004638672,
      "activations/layer5_attention_weight_max": 71.98091125488281,
      "activations/layer5_attention_weight_min": -74.84178161621094,
      "activations/layer6_attention_weight_max": 50.24382400512695,
      "activations/layer6_attention_weight_min": -51.26798629760742,
      "activations/layer7_attention_weight_max": 67.41253662109375,
      "activations/layer7_attention_weight_min": -67.72492980957031,
      "activations/layer8_attention_weight_max": 44.65297317504883,
      "activations/layer8_attention_weight_min": -48.277748107910156,
      "activations/layer9_attention_weight_max": 45.67898941040039,
      "activations/layer9_attention_weight_min": -45.951942443847656,
      "epoch": 12.85,
      "learning_rate": 6.783560606060606e-05,
      "loss": 2.7658,
      "step": 221100
    },
    {
      "activations/layer0_attention_weight_max": 16.303770065307617,
      "activations/layer0_attention_weight_min": -14.557165145874023,
      "activations/layer10_attention_weight_max": 34.40001678466797,
      "activations/layer10_attention_weight_min": -32.44427490234375,
      "activations/layer11_attention_weight_max": 32.89820861816406,
      "activations/layer11_attention_weight_min": -30.766132354736328,
      "activations/layer12_attention_weight_max": 17.382658004760742,
      "activations/layer12_attention_weight_min": -24.052942276000977,
      "activations/layer13_attention_weight_max": 37.85140609741211,
      "activations/layer13_attention_weight_min": -34.36115264892578,
      "activations/layer14_attention_weight_max": 34.341094970703125,
      "activations/layer14_attention_weight_min": -29.027326583862305,
      "activations/layer15_attention_weight_max": 31.997886657714844,
      "activations/layer15_attention_weight_min": -29.331918716430664,
      "activations/layer16_attention_weight_max": 29.234350204467773,
      "activations/layer16_attention_weight_min": -29.823925018310547,
      "activations/layer17_attention_weight_max": 52.09727478027344,
      "activations/layer17_attention_weight_min": -43.2586555480957,
      "activations/layer18_attention_weight_max": 43.81726837158203,
      "activations/layer18_attention_weight_min": -37.349491119384766,
      "activations/layer19_attention_weight_max": 25.7048397064209,
      "activations/layer19_attention_weight_min": -22.19071388244629,
      "activations/layer1_attention_weight_max": 17.419767379760742,
      "activations/layer1_attention_weight_min": -14.858601570129395,
      "activations/layer20_attention_weight_max": 23.114225387573242,
      "activations/layer20_attention_weight_min": -21.026248931884766,
      "activations/layer21_attention_weight_max": 38.331146240234375,
      "activations/layer21_attention_weight_min": -23.616058349609375,
      "activations/layer22_attention_weight_max": 30.774784088134766,
      "activations/layer22_attention_weight_min": -25.553722381591797,
      "activations/layer23_attention_weight_max": 38.485382080078125,
      "activations/layer23_attention_weight_min": -25.298627853393555,
      "activations/layer2_attention_weight_max": 31.23656463623047,
      "activations/layer2_attention_weight_min": -32.0944938659668,
      "activations/layer3_attention_weight_max": 93.55390167236328,
      "activations/layer3_attention_weight_min": -98.81073760986328,
      "activations/layer4_attention_weight_max": 95.34967803955078,
      "activations/layer4_attention_weight_min": -93.66566467285156,
      "activations/layer5_attention_weight_max": 72.47755432128906,
      "activations/layer5_attention_weight_min": -81.43733215332031,
      "activations/layer6_attention_weight_max": 52.286415100097656,
      "activations/layer6_attention_weight_min": -52.835079193115234,
      "activations/layer7_attention_weight_max": 70.18903350830078,
      "activations/layer7_attention_weight_min": -66.50593566894531,
      "activations/layer8_attention_weight_max": 46.385066986083984,
      "activations/layer8_attention_weight_min": -44.75758743286133,
      "activations/layer9_attention_weight_max": 43.479618072509766,
      "activations/layer9_attention_weight_min": -44.595314025878906,
      "epoch": 12.85,
      "learning_rate": 6.781666666666666e-05,
      "loss": 2.7816,
      "step": 221150
    },
    {
      "activations/layer0_attention_weight_max": 17.763601303100586,
      "activations/layer0_attention_weight_min": -14.817152976989746,
      "activations/layer10_attention_weight_max": 33.27935028076172,
      "activations/layer10_attention_weight_min": -34.78303527832031,
      "activations/layer11_attention_weight_max": 29.700368881225586,
      "activations/layer11_attention_weight_min": -32.087493896484375,
      "activations/layer12_attention_weight_max": 16.843582153320312,
      "activations/layer12_attention_weight_min": -23.801816940307617,
      "activations/layer13_attention_weight_max": 31.5593204498291,
      "activations/layer13_attention_weight_min": -27.416704177856445,
      "activations/layer14_attention_weight_max": 34.16427230834961,
      "activations/layer14_attention_weight_min": -28.650110244750977,
      "activations/layer15_attention_weight_max": 30.893604278564453,
      "activations/layer15_attention_weight_min": -28.533721923828125,
      "activations/layer16_attention_weight_max": 29.073585510253906,
      "activations/layer16_attention_weight_min": -28.506868362426758,
      "activations/layer17_attention_weight_max": 46.87697219848633,
      "activations/layer17_attention_weight_min": -41.40810012817383,
      "activations/layer18_attention_weight_max": 43.575660705566406,
      "activations/layer18_attention_weight_min": -34.92563247680664,
      "activations/layer19_attention_weight_max": 21.422536849975586,
      "activations/layer19_attention_weight_min": -21.846221923828125,
      "activations/layer1_attention_weight_max": 17.012264251708984,
      "activations/layer1_attention_weight_min": -17.119380950927734,
      "activations/layer20_attention_weight_max": 21.532794952392578,
      "activations/layer20_attention_weight_min": -25.468034744262695,
      "activations/layer21_attention_weight_max": 31.983165740966797,
      "activations/layer21_attention_weight_min": -22.70123291015625,
      "activations/layer22_attention_weight_max": 28.6595401763916,
      "activations/layer22_attention_weight_min": -25.323843002319336,
      "activations/layer23_attention_weight_max": 37.022003173828125,
      "activations/layer23_attention_weight_min": -24.155109405517578,
      "activations/layer2_attention_weight_max": 31.73343276977539,
      "activations/layer2_attention_weight_min": -30.761131286621094,
      "activations/layer3_attention_weight_max": 89.66268157958984,
      "activations/layer3_attention_weight_min": -95.5772933959961,
      "activations/layer4_attention_weight_max": 92.72903442382812,
      "activations/layer4_attention_weight_min": -92.35759735107422,
      "activations/layer5_attention_weight_max": 69.72225952148438,
      "activations/layer5_attention_weight_min": -73.9500732421875,
      "activations/layer6_attention_weight_max": 47.080814361572266,
      "activations/layer6_attention_weight_min": -48.56977844238281,
      "activations/layer7_attention_weight_max": 71.58134460449219,
      "activations/layer7_attention_weight_min": -62.8072395324707,
      "activations/layer8_attention_weight_max": 41.95500564575195,
      "activations/layer8_attention_weight_min": -44.619537353515625,
      "activations/layer9_attention_weight_max": 42.39431381225586,
      "activations/layer9_attention_weight_min": -46.27096939086914,
      "epoch": 12.85,
      "learning_rate": 6.779772727272727e-05,
      "loss": 2.7805,
      "step": 221200
    },
    {
      "activations/layer0_attention_weight_max": 17.206424713134766,
      "activations/layer0_attention_weight_min": -14.412353515625,
      "activations/layer10_attention_weight_max": 33.6552848815918,
      "activations/layer10_attention_weight_min": -34.75008773803711,
      "activations/layer11_attention_weight_max": 29.984769821166992,
      "activations/layer11_attention_weight_min": -33.52825164794922,
      "activations/layer12_attention_weight_max": 19.443037033081055,
      "activations/layer12_attention_weight_min": -23.38005256652832,
      "activations/layer13_attention_weight_max": 41.47090148925781,
      "activations/layer13_attention_weight_min": -27.766742706298828,
      "activations/layer14_attention_weight_max": 36.71090316772461,
      "activations/layer14_attention_weight_min": -30.533761978149414,
      "activations/layer15_attention_weight_max": 32.47844696044922,
      "activations/layer15_attention_weight_min": -29.843881607055664,
      "activations/layer16_attention_weight_max": 32.06300735473633,
      "activations/layer16_attention_weight_min": -29.66529083251953,
      "activations/layer17_attention_weight_max": 52.7145881652832,
      "activations/layer17_attention_weight_min": -42.89350891113281,
      "activations/layer18_attention_weight_max": 50.02403259277344,
      "activations/layer18_attention_weight_min": -39.26002883911133,
      "activations/layer19_attention_weight_max": 22.199918746948242,
      "activations/layer19_attention_weight_min": -22.801733016967773,
      "activations/layer1_attention_weight_max": 16.63115119934082,
      "activations/layer1_attention_weight_min": -15.065566062927246,
      "activations/layer20_attention_weight_max": 22.78412437438965,
      "activations/layer20_attention_weight_min": -22.701086044311523,
      "activations/layer21_attention_weight_max": 35.41020202636719,
      "activations/layer21_attention_weight_min": -22.59103012084961,
      "activations/layer22_attention_weight_max": 28.94278907775879,
      "activations/layer22_attention_weight_min": -26.78063201904297,
      "activations/layer23_attention_weight_max": 38.61435317993164,
      "activations/layer23_attention_weight_min": -24.744678497314453,
      "activations/layer2_attention_weight_max": 31.78822135925293,
      "activations/layer2_attention_weight_min": -29.852508544921875,
      "activations/layer3_attention_weight_max": 91.4563980102539,
      "activations/layer3_attention_weight_min": -93.49302673339844,
      "activations/layer4_attention_weight_max": 92.60486602783203,
      "activations/layer4_attention_weight_min": -92.3533935546875,
      "activations/layer5_attention_weight_max": 68.9534912109375,
      "activations/layer5_attention_weight_min": -77.16609954833984,
      "activations/layer6_attention_weight_max": 49.802772521972656,
      "activations/layer6_attention_weight_min": -50.632877349853516,
      "activations/layer7_attention_weight_max": 63.36665725708008,
      "activations/layer7_attention_weight_min": -64.56913757324219,
      "activations/layer8_attention_weight_max": 43.915184020996094,
      "activations/layer8_attention_weight_min": -48.718318939208984,
      "activations/layer9_attention_weight_max": 44.74245834350586,
      "activations/layer9_attention_weight_min": -47.6185417175293,
      "epoch": 12.86,
      "learning_rate": 6.777878787878787e-05,
      "loss": 2.7683,
      "step": 221250
    },
    {
      "activations/layer0_attention_weight_max": 16.751312255859375,
      "activations/layer0_attention_weight_min": -14.530281066894531,
      "activations/layer10_attention_weight_max": 36.81303787231445,
      "activations/layer10_attention_weight_min": -36.63827133178711,
      "activations/layer11_attention_weight_max": 32.491539001464844,
      "activations/layer11_attention_weight_min": -36.3367919921875,
      "activations/layer12_attention_weight_max": 17.455516815185547,
      "activations/layer12_attention_weight_min": -27.2470645904541,
      "activations/layer13_attention_weight_max": 35.0355339050293,
      "activations/layer13_attention_weight_min": -28.21430015563965,
      "activations/layer14_attention_weight_max": 36.72493362426758,
      "activations/layer14_attention_weight_min": -30.679597854614258,
      "activations/layer15_attention_weight_max": 34.850826263427734,
      "activations/layer15_attention_weight_min": -29.49234390258789,
      "activations/layer16_attention_weight_max": 35.23807144165039,
      "activations/layer16_attention_weight_min": -31.19322967529297,
      "activations/layer17_attention_weight_max": 54.215980529785156,
      "activations/layer17_attention_weight_min": -45.577457427978516,
      "activations/layer18_attention_weight_max": 47.746891021728516,
      "activations/layer18_attention_weight_min": -36.99493408203125,
      "activations/layer19_attention_weight_max": 23.026151657104492,
      "activations/layer19_attention_weight_min": -21.135379791259766,
      "activations/layer1_attention_weight_max": 17.680953979492188,
      "activations/layer1_attention_weight_min": -14.649991989135742,
      "activations/layer20_attention_weight_max": 20.55608367919922,
      "activations/layer20_attention_weight_min": -21.465919494628906,
      "activations/layer21_attention_weight_max": 36.5734977722168,
      "activations/layer21_attention_weight_min": -21.927797317504883,
      "activations/layer22_attention_weight_max": 28.04944610595703,
      "activations/layer22_attention_weight_min": -25.551450729370117,
      "activations/layer23_attention_weight_max": 40.67882537841797,
      "activations/layer23_attention_weight_min": -24.698287963867188,
      "activations/layer2_attention_weight_max": 32.73088836669922,
      "activations/layer2_attention_weight_min": -31.31401824951172,
      "activations/layer3_attention_weight_max": 95.38106536865234,
      "activations/layer3_attention_weight_min": -93.22276306152344,
      "activations/layer4_attention_weight_max": 98.7491455078125,
      "activations/layer4_attention_weight_min": -94.89554595947266,
      "activations/layer5_attention_weight_max": 73.71105194091797,
      "activations/layer5_attention_weight_min": -80.19855499267578,
      "activations/layer6_attention_weight_max": 53.823848724365234,
      "activations/layer6_attention_weight_min": -54.866065979003906,
      "activations/layer7_attention_weight_max": 71.89381408691406,
      "activations/layer7_attention_weight_min": -70.91985321044922,
      "activations/layer8_attention_weight_max": 48.8607292175293,
      "activations/layer8_attention_weight_min": -50.85338592529297,
      "activations/layer9_attention_weight_max": 46.58401870727539,
      "activations/layer9_attention_weight_min": -47.6854248046875,
      "epoch": 12.86,
      "learning_rate": 6.775984848484848e-05,
      "loss": 2.7693,
      "step": 221300
    },
    {
      "activations/layer0_attention_weight_max": 16.108768463134766,
      "activations/layer0_attention_weight_min": -14.485806465148926,
      "activations/layer10_attention_weight_max": 38.122154235839844,
      "activations/layer10_attention_weight_min": -38.590091705322266,
      "activations/layer11_attention_weight_max": 35.479759216308594,
      "activations/layer11_attention_weight_min": -35.68075180053711,
      "activations/layer12_attention_weight_max": 18.489749908447266,
      "activations/layer12_attention_weight_min": -28.539478302001953,
      "activations/layer13_attention_weight_max": 38.26713943481445,
      "activations/layer13_attention_weight_min": -28.9155216217041,
      "activations/layer14_attention_weight_max": 34.81416702270508,
      "activations/layer14_attention_weight_min": -29.09542465209961,
      "activations/layer15_attention_weight_max": 30.671445846557617,
      "activations/layer15_attention_weight_min": -29.173917770385742,
      "activations/layer16_attention_weight_max": 33.2428092956543,
      "activations/layer16_attention_weight_min": -30.121728897094727,
      "activations/layer17_attention_weight_max": 56.00701141357422,
      "activations/layer17_attention_weight_min": -48.44718933105469,
      "activations/layer18_attention_weight_max": 50.81993865966797,
      "activations/layer18_attention_weight_min": -47.191287994384766,
      "activations/layer19_attention_weight_max": 22.18277931213379,
      "activations/layer19_attention_weight_min": -24.523006439208984,
      "activations/layer1_attention_weight_max": 18.090553283691406,
      "activations/layer1_attention_weight_min": -19.2689266204834,
      "activations/layer20_attention_weight_max": 22.17947769165039,
      "activations/layer20_attention_weight_min": -22.493303298950195,
      "activations/layer21_attention_weight_max": 35.59242248535156,
      "activations/layer21_attention_weight_min": -22.584007263183594,
      "activations/layer22_attention_weight_max": 29.648319244384766,
      "activations/layer22_attention_weight_min": -27.09527587890625,
      "activations/layer23_attention_weight_max": 34.57540512084961,
      "activations/layer23_attention_weight_min": -23.004451751708984,
      "activations/layer2_attention_weight_max": 33.073143005371094,
      "activations/layer2_attention_weight_min": -32.89898681640625,
      "activations/layer3_attention_weight_max": 99.0249252319336,
      "activations/layer3_attention_weight_min": -101.9833984375,
      "activations/layer4_attention_weight_max": 99.38421630859375,
      "activations/layer4_attention_weight_min": -100.12293243408203,
      "activations/layer5_attention_weight_max": 75.80439758300781,
      "activations/layer5_attention_weight_min": -79.99722290039062,
      "activations/layer6_attention_weight_max": 52.943580627441406,
      "activations/layer6_attention_weight_min": -54.787967681884766,
      "activations/layer7_attention_weight_max": 74.15087890625,
      "activations/layer7_attention_weight_min": -72.43900299072266,
      "activations/layer8_attention_weight_max": 48.23625564575195,
      "activations/layer8_attention_weight_min": -52.22914505004883,
      "activations/layer9_attention_weight_max": 51.51072311401367,
      "activations/layer9_attention_weight_min": -48.01754379272461,
      "epoch": 12.86,
      "learning_rate": 6.774090909090909e-05,
      "loss": 2.7745,
      "step": 221350
    },
    {
      "activations/layer0_attention_weight_max": 17.71926498413086,
      "activations/layer0_attention_weight_min": -14.23960018157959,
      "activations/layer10_attention_weight_max": 37.50447463989258,
      "activations/layer10_attention_weight_min": -36.78365707397461,
      "activations/layer11_attention_weight_max": 33.64161682128906,
      "activations/layer11_attention_weight_min": -33.12824249267578,
      "activations/layer12_attention_weight_max": 18.164443969726562,
      "activations/layer12_attention_weight_min": -24.30457305908203,
      "activations/layer13_attention_weight_max": 42.428810119628906,
      "activations/layer13_attention_weight_min": -28.113454818725586,
      "activations/layer14_attention_weight_max": 36.73231887817383,
      "activations/layer14_attention_weight_min": -29.034067153930664,
      "activations/layer15_attention_weight_max": 33.69341278076172,
      "activations/layer15_attention_weight_min": -28.018339157104492,
      "activations/layer16_attention_weight_max": 32.72844696044922,
      "activations/layer16_attention_weight_min": -29.913145065307617,
      "activations/layer17_attention_weight_max": 52.84026336669922,
      "activations/layer17_attention_weight_min": -43.59489440917969,
      "activations/layer18_attention_weight_max": 48.32485580444336,
      "activations/layer18_attention_weight_min": -39.0150032043457,
      "activations/layer19_attention_weight_max": 23.73695945739746,
      "activations/layer19_attention_weight_min": -22.34822654724121,
      "activations/layer1_attention_weight_max": 16.222877502441406,
      "activations/layer1_attention_weight_min": -14.939189910888672,
      "activations/layer20_attention_weight_max": 25.291563034057617,
      "activations/layer20_attention_weight_min": -20.443784713745117,
      "activations/layer21_attention_weight_max": 41.78702926635742,
      "activations/layer21_attention_weight_min": -25.82115936279297,
      "activations/layer22_attention_weight_max": 34.50309753417969,
      "activations/layer22_attention_weight_min": -24.361339569091797,
      "activations/layer23_attention_weight_max": 43.155147552490234,
      "activations/layer23_attention_weight_min": -24.11446762084961,
      "activations/layer2_attention_weight_max": 32.10907745361328,
      "activations/layer2_attention_weight_min": -31.39484214782715,
      "activations/layer3_attention_weight_max": 96.53562927246094,
      "activations/layer3_attention_weight_min": -99.44649505615234,
      "activations/layer4_attention_weight_max": 98.76193237304688,
      "activations/layer4_attention_weight_min": -95.13697814941406,
      "activations/layer5_attention_weight_max": 71.58419036865234,
      "activations/layer5_attention_weight_min": -79.40554809570312,
      "activations/layer6_attention_weight_max": 52.222328186035156,
      "activations/layer6_attention_weight_min": -51.77351379394531,
      "activations/layer7_attention_weight_max": 69.27883911132812,
      "activations/layer7_attention_weight_min": -69.13070678710938,
      "activations/layer8_attention_weight_max": 48.60456466674805,
      "activations/layer8_attention_weight_min": -48.328033447265625,
      "activations/layer9_attention_weight_max": 46.77146911621094,
      "activations/layer9_attention_weight_min": -48.70295333862305,
      "epoch": 12.86,
      "learning_rate": 6.772196969696969e-05,
      "loss": 2.7804,
      "step": 221400
    },
    {
      "activations/layer0_attention_weight_max": 17.39486312866211,
      "activations/layer0_attention_weight_min": -14.682533264160156,
      "activations/layer10_attention_weight_max": 34.4100227355957,
      "activations/layer10_attention_weight_min": -35.844573974609375,
      "activations/layer11_attention_weight_max": 32.61825942993164,
      "activations/layer11_attention_weight_min": -35.0862922668457,
      "activations/layer12_attention_weight_max": 17.131258010864258,
      "activations/layer12_attention_weight_min": -23.012510299682617,
      "activations/layer13_attention_weight_max": 35.934730529785156,
      "activations/layer13_attention_weight_min": -29.997222900390625,
      "activations/layer14_attention_weight_max": 35.155330657958984,
      "activations/layer14_attention_weight_min": -28.760108947753906,
      "activations/layer15_attention_weight_max": 31.017311096191406,
      "activations/layer15_attention_weight_min": -29.914426803588867,
      "activations/layer16_attention_weight_max": 29.219430923461914,
      "activations/layer16_attention_weight_min": -28.38750457763672,
      "activations/layer17_attention_weight_max": 47.78038787841797,
      "activations/layer17_attention_weight_min": -41.227500915527344,
      "activations/layer18_attention_weight_max": 42.69375991821289,
      "activations/layer18_attention_weight_min": -36.65459442138672,
      "activations/layer19_attention_weight_max": 22.03855323791504,
      "activations/layer19_attention_weight_min": -23.046506881713867,
      "activations/layer1_attention_weight_max": 15.837569236755371,
      "activations/layer1_attention_weight_min": -14.736777305603027,
      "activations/layer20_attention_weight_max": 21.79623794555664,
      "activations/layer20_attention_weight_min": -21.958749771118164,
      "activations/layer21_attention_weight_max": 35.79849624633789,
      "activations/layer21_attention_weight_min": -23.319860458374023,
      "activations/layer22_attention_weight_max": 28.88156509399414,
      "activations/layer22_attention_weight_min": -27.19411277770996,
      "activations/layer23_attention_weight_max": 36.800209045410156,
      "activations/layer23_attention_weight_min": -23.101560592651367,
      "activations/layer2_attention_weight_max": 30.431194305419922,
      "activations/layer2_attention_weight_min": -30.820188522338867,
      "activations/layer3_attention_weight_max": 92.8691635131836,
      "activations/layer3_attention_weight_min": -97.77273559570312,
      "activations/layer4_attention_weight_max": 97.00858306884766,
      "activations/layer4_attention_weight_min": -94.03048706054688,
      "activations/layer5_attention_weight_max": 70.61048889160156,
      "activations/layer5_attention_weight_min": -85.99974060058594,
      "activations/layer6_attention_weight_max": 49.71843338012695,
      "activations/layer6_attention_weight_min": -51.293052673339844,
      "activations/layer7_attention_weight_max": 71.20903015136719,
      "activations/layer7_attention_weight_min": -64.68325805664062,
      "activations/layer8_attention_weight_max": 44.50736618041992,
      "activations/layer8_attention_weight_min": -50.21220779418945,
      "activations/layer9_attention_weight_max": 45.78084945678711,
      "activations/layer9_attention_weight_min": -45.48612594604492,
      "epoch": 12.87,
      "learning_rate": 6.770303030303029e-05,
      "loss": 2.7724,
      "step": 221450
    },
    {
      "activations/layer0_attention_weight_max": 16.702817916870117,
      "activations/layer0_attention_weight_min": -14.75672721862793,
      "activations/layer10_attention_weight_max": 36.640174865722656,
      "activations/layer10_attention_weight_min": -36.7532844543457,
      "activations/layer11_attention_weight_max": 33.5338134765625,
      "activations/layer11_attention_weight_min": -33.96673583984375,
      "activations/layer12_attention_weight_max": 19.3741512298584,
      "activations/layer12_attention_weight_min": -22.044837951660156,
      "activations/layer13_attention_weight_max": 42.50349426269531,
      "activations/layer13_attention_weight_min": -31.41265869140625,
      "activations/layer14_attention_weight_max": 37.92780303955078,
      "activations/layer14_attention_weight_min": -31.263227462768555,
      "activations/layer15_attention_weight_max": 33.262725830078125,
      "activations/layer15_attention_weight_min": -29.081085205078125,
      "activations/layer16_attention_weight_max": 32.608375549316406,
      "activations/layer16_attention_weight_min": -31.683568954467773,
      "activations/layer17_attention_weight_max": 54.567237854003906,
      "activations/layer17_attention_weight_min": -46.9547119140625,
      "activations/layer18_attention_weight_max": 48.36314010620117,
      "activations/layer18_attention_weight_min": -39.40196990966797,
      "activations/layer19_attention_weight_max": 24.12232780456543,
      "activations/layer19_attention_weight_min": -23.276504516601562,
      "activations/layer1_attention_weight_max": 16.6210880279541,
      "activations/layer1_attention_weight_min": -15.32917308807373,
      "activations/layer20_attention_weight_max": 23.108633041381836,
      "activations/layer20_attention_weight_min": -24.028886795043945,
      "activations/layer21_attention_weight_max": 40.64871597290039,
      "activations/layer21_attention_weight_min": -24.26169776916504,
      "activations/layer22_attention_weight_max": 33.34352493286133,
      "activations/layer22_attention_weight_min": -27.234928131103516,
      "activations/layer23_attention_weight_max": 39.720401763916016,
      "activations/layer23_attention_weight_min": -23.858999252319336,
      "activations/layer2_attention_weight_max": 33.29242706298828,
      "activations/layer2_attention_weight_min": -30.44538688659668,
      "activations/layer3_attention_weight_max": 102.62652587890625,
      "activations/layer3_attention_weight_min": -97.09419250488281,
      "activations/layer4_attention_weight_max": 98.44916534423828,
      "activations/layer4_attention_weight_min": -95.02367401123047,
      "activations/layer5_attention_weight_max": 73.0850830078125,
      "activations/layer5_attention_weight_min": -77.25035095214844,
      "activations/layer6_attention_weight_max": 53.65266418457031,
      "activations/layer6_attention_weight_min": -55.42863464355469,
      "activations/layer7_attention_weight_max": 68.90211486816406,
      "activations/layer7_attention_weight_min": -69.79537963867188,
      "activations/layer8_attention_weight_max": 47.86307907104492,
      "activations/layer8_attention_weight_min": -46.92961502075195,
      "activations/layer9_attention_weight_max": 51.83645248413086,
      "activations/layer9_attention_weight_min": -47.30203628540039,
      "epoch": 12.87,
      "learning_rate": 6.768409090909091e-05,
      "loss": 2.7758,
      "step": 221500
    },
    {
      "activations/layer0_attention_weight_max": 16.899808883666992,
      "activations/layer0_attention_weight_min": -14.634590148925781,
      "activations/layer10_attention_weight_max": 37.119083404541016,
      "activations/layer10_attention_weight_min": -37.93281936645508,
      "activations/layer11_attention_weight_max": 35.346473693847656,
      "activations/layer11_attention_weight_min": -32.59748458862305,
      "activations/layer12_attention_weight_max": 19.057723999023438,
      "activations/layer12_attention_weight_min": -28.634546279907227,
      "activations/layer13_attention_weight_max": 43.31558609008789,
      "activations/layer13_attention_weight_min": -29.198928833007812,
      "activations/layer14_attention_weight_max": 48.1834602355957,
      "activations/layer14_attention_weight_min": -31.143451690673828,
      "activations/layer15_attention_weight_max": 44.973541259765625,
      "activations/layer15_attention_weight_min": -30.445816040039062,
      "activations/layer16_attention_weight_max": 36.55718231201172,
      "activations/layer16_attention_weight_min": -31.21872901916504,
      "activations/layer17_attention_weight_max": 56.258750915527344,
      "activations/layer17_attention_weight_min": -47.64553451538086,
      "activations/layer18_attention_weight_max": 52.04689407348633,
      "activations/layer18_attention_weight_min": -40.22907638549805,
      "activations/layer19_attention_weight_max": 27.30844497680664,
      "activations/layer19_attention_weight_min": -23.52020835876465,
      "activations/layer1_attention_weight_max": 16.081174850463867,
      "activations/layer1_attention_weight_min": -14.803262710571289,
      "activations/layer20_attention_weight_max": 29.716135025024414,
      "activations/layer20_attention_weight_min": -23.381296157836914,
      "activations/layer21_attention_weight_max": 53.09263610839844,
      "activations/layer21_attention_weight_min": -25.529016494750977,
      "activations/layer22_attention_weight_max": 35.98554229736328,
      "activations/layer22_attention_weight_min": -27.06692886352539,
      "activations/layer23_attention_weight_max": 39.3150749206543,
      "activations/layer23_attention_weight_min": -26.077632904052734,
      "activations/layer2_attention_weight_max": 35.04212188720703,
      "activations/layer2_attention_weight_min": -32.82324981689453,
      "activations/layer3_attention_weight_max": 104.49034881591797,
      "activations/layer3_attention_weight_min": -106.99398040771484,
      "activations/layer4_attention_weight_max": 102.86177062988281,
      "activations/layer4_attention_weight_min": -99.48379516601562,
      "activations/layer5_attention_weight_max": 72.66824340820312,
      "activations/layer5_attention_weight_min": -79.3114242553711,
      "activations/layer6_attention_weight_max": 55.76567077636719,
      "activations/layer6_attention_weight_min": -54.84803771972656,
      "activations/layer7_attention_weight_max": 74.01761627197266,
      "activations/layer7_attention_weight_min": -71.23759460449219,
      "activations/layer8_attention_weight_max": 46.97749710083008,
      "activations/layer8_attention_weight_min": -48.42127227783203,
      "activations/layer9_attention_weight_max": 49.698081970214844,
      "activations/layer9_attention_weight_min": -49.88547134399414,
      "epoch": 12.87,
      "learning_rate": 6.766515151515151e-05,
      "loss": 2.7998,
      "step": 221550
    },
    {
      "activations/layer0_attention_weight_max": 15.729708671569824,
      "activations/layer0_attention_weight_min": -14.464333534240723,
      "activations/layer10_attention_weight_max": 37.654541015625,
      "activations/layer10_attention_weight_min": -38.36729049682617,
      "activations/layer11_attention_weight_max": 34.08369445800781,
      "activations/layer11_attention_weight_min": -35.464622497558594,
      "activations/layer12_attention_weight_max": 17.106182098388672,
      "activations/layer12_attention_weight_min": -24.628997802734375,
      "activations/layer13_attention_weight_max": 50.638919830322266,
      "activations/layer13_attention_weight_min": -30.19235610961914,
      "activations/layer14_attention_weight_max": 40.266456604003906,
      "activations/layer14_attention_weight_min": -31.268775939941406,
      "activations/layer15_attention_weight_max": 32.252532958984375,
      "activations/layer15_attention_weight_min": -29.276668548583984,
      "activations/layer16_attention_weight_max": 31.56113624572754,
      "activations/layer16_attention_weight_min": -30.11322784423828,
      "activations/layer17_attention_weight_max": 51.730831146240234,
      "activations/layer17_attention_weight_min": -42.94048309326172,
      "activations/layer18_attention_weight_max": 44.19013214111328,
      "activations/layer18_attention_weight_min": -37.18998718261719,
      "activations/layer19_attention_weight_max": 20.031118392944336,
      "activations/layer19_attention_weight_min": -21.068540573120117,
      "activations/layer1_attention_weight_max": 18.230112075805664,
      "activations/layer1_attention_weight_min": -14.824999809265137,
      "activations/layer20_attention_weight_max": 22.043235778808594,
      "activations/layer20_attention_weight_min": -21.736783981323242,
      "activations/layer21_attention_weight_max": 35.164310455322266,
      "activations/layer21_attention_weight_min": -23.23163414001465,
      "activations/layer22_attention_weight_max": 29.14072608947754,
      "activations/layer22_attention_weight_min": -25.273252487182617,
      "activations/layer23_attention_weight_max": 41.45136260986328,
      "activations/layer23_attention_weight_min": -24.691951751708984,
      "activations/layer2_attention_weight_max": 32.53885269165039,
      "activations/layer2_attention_weight_min": -32.45989227294922,
      "activations/layer3_attention_weight_max": 91.5914077758789,
      "activations/layer3_attention_weight_min": -94.42853546142578,
      "activations/layer4_attention_weight_max": 94.99352264404297,
      "activations/layer4_attention_weight_min": -94.50910186767578,
      "activations/layer5_attention_weight_max": 68.17625427246094,
      "activations/layer5_attention_weight_min": -77.82133483886719,
      "activations/layer6_attention_weight_max": 50.99665069580078,
      "activations/layer6_attention_weight_min": -51.85564422607422,
      "activations/layer7_attention_weight_max": 67.84942626953125,
      "activations/layer7_attention_weight_min": -63.082698822021484,
      "activations/layer8_attention_weight_max": 48.81831359863281,
      "activations/layer8_attention_weight_min": -45.602420806884766,
      "activations/layer9_attention_weight_max": 51.54500961303711,
      "activations/layer9_attention_weight_min": -45.00664138793945,
      "epoch": 12.88,
      "learning_rate": 6.764621212121211e-05,
      "loss": 2.7722,
      "step": 221600
    },
    {
      "activations/layer0_attention_weight_max": 16.54998207092285,
      "activations/layer0_attention_weight_min": -14.487579345703125,
      "activations/layer10_attention_weight_max": 35.63875198364258,
      "activations/layer10_attention_weight_min": -33.570953369140625,
      "activations/layer11_attention_weight_max": 30.689573287963867,
      "activations/layer11_attention_weight_min": -30.253173828125,
      "activations/layer12_attention_weight_max": 17.101831436157227,
      "activations/layer12_attention_weight_min": -30.19001007080078,
      "activations/layer13_attention_weight_max": 31.780994415283203,
      "activations/layer13_attention_weight_min": -29.36661720275879,
      "activations/layer14_attention_weight_max": 31.781291961669922,
      "activations/layer14_attention_weight_min": -30.588001251220703,
      "activations/layer15_attention_weight_max": 30.13516616821289,
      "activations/layer15_attention_weight_min": -28.182126998901367,
      "activations/layer16_attention_weight_max": 29.34768295288086,
      "activations/layer16_attention_weight_min": -30.446727752685547,
      "activations/layer17_attention_weight_max": 48.55941390991211,
      "activations/layer17_attention_weight_min": -43.5531120300293,
      "activations/layer18_attention_weight_max": 45.07807159423828,
      "activations/layer18_attention_weight_min": -36.604530334472656,
      "activations/layer19_attention_weight_max": 19.701684951782227,
      "activations/layer19_attention_weight_min": -21.978445053100586,
      "activations/layer1_attention_weight_max": 17.233240127563477,
      "activations/layer1_attention_weight_min": -15.913625717163086,
      "activations/layer20_attention_weight_max": 21.209686279296875,
      "activations/layer20_attention_weight_min": -26.138559341430664,
      "activations/layer21_attention_weight_max": 32.13164520263672,
      "activations/layer21_attention_weight_min": -24.578649520874023,
      "activations/layer22_attention_weight_max": 27.267332077026367,
      "activations/layer22_attention_weight_min": -26.00783348083496,
      "activations/layer23_attention_weight_max": 32.83356475830078,
      "activations/layer23_attention_weight_min": -25.8846435546875,
      "activations/layer2_attention_weight_max": 31.933162689208984,
      "activations/layer2_attention_weight_min": -31.59383201599121,
      "activations/layer3_attention_weight_max": 94.23307037353516,
      "activations/layer3_attention_weight_min": -106.65489959716797,
      "activations/layer4_attention_weight_max": 95.90385437011719,
      "activations/layer4_attention_weight_min": -91.13860321044922,
      "activations/layer5_attention_weight_max": 69.32295989990234,
      "activations/layer5_attention_weight_min": -79.27685546875,
      "activations/layer6_attention_weight_max": 50.97003936767578,
      "activations/layer6_attention_weight_min": -48.75736999511719,
      "activations/layer7_attention_weight_max": 66.41706085205078,
      "activations/layer7_attention_weight_min": -63.37742614746094,
      "activations/layer8_attention_weight_max": 43.67528533935547,
      "activations/layer8_attention_weight_min": -46.44599533081055,
      "activations/layer9_attention_weight_max": 44.719749450683594,
      "activations/layer9_attention_weight_min": -46.12049102783203,
      "epoch": 12.88,
      "learning_rate": 6.762727272727271e-05,
      "loss": 2.7859,
      "step": 221650
    },
    {
      "activations/layer0_attention_weight_max": 16.420127868652344,
      "activations/layer0_attention_weight_min": -14.53261947631836,
      "activations/layer10_attention_weight_max": 36.100154876708984,
      "activations/layer10_attention_weight_min": -35.7513313293457,
      "activations/layer11_attention_weight_max": 32.74787139892578,
      "activations/layer11_attention_weight_min": -31.562091827392578,
      "activations/layer12_attention_weight_max": 17.869918823242188,
      "activations/layer12_attention_weight_min": -28.370859146118164,
      "activations/layer13_attention_weight_max": 32.29682159423828,
      "activations/layer13_attention_weight_min": -28.956960678100586,
      "activations/layer14_attention_weight_max": 34.81805419921875,
      "activations/layer14_attention_weight_min": -29.84201431274414,
      "activations/layer15_attention_weight_max": 30.433700561523438,
      "activations/layer15_attention_weight_min": -29.827152252197266,
      "activations/layer16_attention_weight_max": 29.532329559326172,
      "activations/layer16_attention_weight_min": -29.171754837036133,
      "activations/layer17_attention_weight_max": 51.1038703918457,
      "activations/layer17_attention_weight_min": -43.514854431152344,
      "activations/layer18_attention_weight_max": 44.71261978149414,
      "activations/layer18_attention_weight_min": -37.26508712768555,
      "activations/layer19_attention_weight_max": 20.454957962036133,
      "activations/layer19_attention_weight_min": -21.433584213256836,
      "activations/layer1_attention_weight_max": 16.445953369140625,
      "activations/layer1_attention_weight_min": -15.606334686279297,
      "activations/layer20_attention_weight_max": 20.620750427246094,
      "activations/layer20_attention_weight_min": -23.118106842041016,
      "activations/layer21_attention_weight_max": 38.30801010131836,
      "activations/layer21_attention_weight_min": -24.302724838256836,
      "activations/layer22_attention_weight_max": 28.789047241210938,
      "activations/layer22_attention_weight_min": -26.072717666625977,
      "activations/layer23_attention_weight_max": 34.17604446411133,
      "activations/layer23_attention_weight_min": -23.874252319335938,
      "activations/layer2_attention_weight_max": 35.40119171142578,
      "activations/layer2_attention_weight_min": -34.37153244018555,
      "activations/layer3_attention_weight_max": 98.85327911376953,
      "activations/layer3_attention_weight_min": -96.444091796875,
      "activations/layer4_attention_weight_max": 97.89310455322266,
      "activations/layer4_attention_weight_min": -94.24932861328125,
      "activations/layer5_attention_weight_max": 70.32907104492188,
      "activations/layer5_attention_weight_min": -76.66250610351562,
      "activations/layer6_attention_weight_max": 54.47118377685547,
      "activations/layer6_attention_weight_min": -51.38833236694336,
      "activations/layer7_attention_weight_max": 67.70706176757812,
      "activations/layer7_attention_weight_min": -63.48754119873047,
      "activations/layer8_attention_weight_max": 44.423606872558594,
      "activations/layer8_attention_weight_min": -46.2742805480957,
      "activations/layer9_attention_weight_max": 46.00217056274414,
      "activations/layer9_attention_weight_min": -46.361141204833984,
      "epoch": 12.88,
      "learning_rate": 6.760833333333333e-05,
      "loss": 2.767,
      "step": 221700
    },
    {
      "activations/layer0_attention_weight_max": 16.07465934753418,
      "activations/layer0_attention_weight_min": -14.471410751342773,
      "activations/layer10_attention_weight_max": 46.85615158081055,
      "activations/layer10_attention_weight_min": -46.090370178222656,
      "activations/layer11_attention_weight_max": 40.8858642578125,
      "activations/layer11_attention_weight_min": -40.91593551635742,
      "activations/layer12_attention_weight_max": 18.31064224243164,
      "activations/layer12_attention_weight_min": -23.74281120300293,
      "activations/layer13_attention_weight_max": 56.10291290283203,
      "activations/layer13_attention_weight_min": -35.27861785888672,
      "activations/layer14_attention_weight_max": 64.857177734375,
      "activations/layer14_attention_weight_min": -36.4376335144043,
      "activations/layer15_attention_weight_max": 53.766639709472656,
      "activations/layer15_attention_weight_min": -31.81426429748535,
      "activations/layer16_attention_weight_max": 33.70817184448242,
      "activations/layer16_attention_weight_min": -30.94377899169922,
      "activations/layer17_attention_weight_max": 53.4676399230957,
      "activations/layer17_attention_weight_min": -45.127689361572266,
      "activations/layer18_attention_weight_max": 48.0886344909668,
      "activations/layer18_attention_weight_min": -39.065711975097656,
      "activations/layer19_attention_weight_max": 22.224027633666992,
      "activations/layer19_attention_weight_min": -21.687673568725586,
      "activations/layer1_attention_weight_max": 18.933334350585938,
      "activations/layer1_attention_weight_min": -16.654273986816406,
      "activations/layer20_attention_weight_max": 25.18791389465332,
      "activations/layer20_attention_weight_min": -21.504911422729492,
      "activations/layer21_attention_weight_max": 37.12093734741211,
      "activations/layer21_attention_weight_min": -23.62041473388672,
      "activations/layer22_attention_weight_max": 32.25092315673828,
      "activations/layer22_attention_weight_min": -25.812578201293945,
      "activations/layer23_attention_weight_max": 39.49966812133789,
      "activations/layer23_attention_weight_min": -23.391019821166992,
      "activations/layer2_attention_weight_max": 34.114959716796875,
      "activations/layer2_attention_weight_min": -33.557220458984375,
      "activations/layer3_attention_weight_max": 102.6446762084961,
      "activations/layer3_attention_weight_min": -105.21529388427734,
      "activations/layer4_attention_weight_max": 103.57564544677734,
      "activations/layer4_attention_weight_min": -101.17484283447266,
      "activations/layer5_attention_weight_max": 71.6322021484375,
      "activations/layer5_attention_weight_min": -78.6979751586914,
      "activations/layer6_attention_weight_max": 57.6827507019043,
      "activations/layer6_attention_weight_min": -58.44376754760742,
      "activations/layer7_attention_weight_max": 86.12648010253906,
      "activations/layer7_attention_weight_min": -80.01683044433594,
      "activations/layer8_attention_weight_max": 56.871421813964844,
      "activations/layer8_attention_weight_min": -58.24848937988281,
      "activations/layer9_attention_weight_max": 64.97051239013672,
      "activations/layer9_attention_weight_min": -58.134979248046875,
      "epoch": 12.88,
      "learning_rate": 6.758939393939393e-05,
      "loss": 2.7654,
      "step": 221750
    },
    {
      "activations/layer0_attention_weight_max": 15.627975463867188,
      "activations/layer0_attention_weight_min": -14.534085273742676,
      "activations/layer10_attention_weight_max": 38.76017379760742,
      "activations/layer10_attention_weight_min": -37.701629638671875,
      "activations/layer11_attention_weight_max": 34.79640579223633,
      "activations/layer11_attention_weight_min": -32.99932861328125,
      "activations/layer12_attention_weight_max": 19.254064559936523,
      "activations/layer12_attention_weight_min": -25.087539672851562,
      "activations/layer13_attention_weight_max": 41.01113510131836,
      "activations/layer13_attention_weight_min": -26.349342346191406,
      "activations/layer14_attention_weight_max": 34.29112243652344,
      "activations/layer14_attention_weight_min": -29.957714080810547,
      "activations/layer15_attention_weight_max": 33.52663803100586,
      "activations/layer15_attention_weight_min": -32.58369827270508,
      "activations/layer16_attention_weight_max": 31.944950103759766,
      "activations/layer16_attention_weight_min": -28.293048858642578,
      "activations/layer17_attention_weight_max": 50.23027801513672,
      "activations/layer17_attention_weight_min": -43.15207290649414,
      "activations/layer18_attention_weight_max": 48.34792709350586,
      "activations/layer18_attention_weight_min": -40.308143615722656,
      "activations/layer19_attention_weight_max": 24.25096321105957,
      "activations/layer19_attention_weight_min": -21.355527877807617,
      "activations/layer1_attention_weight_max": 16.927461624145508,
      "activations/layer1_attention_weight_min": -14.534098625183105,
      "activations/layer20_attention_weight_max": 23.90424919128418,
      "activations/layer20_attention_weight_min": -21.842449188232422,
      "activations/layer21_attention_weight_max": 33.320274353027344,
      "activations/layer21_attention_weight_min": -22.690074920654297,
      "activations/layer22_attention_weight_max": 30.581499099731445,
      "activations/layer22_attention_weight_min": -24.846643447875977,
      "activations/layer23_attention_weight_max": 37.77253723144531,
      "activations/layer23_attention_weight_min": -25.39729118347168,
      "activations/layer2_attention_weight_max": 33.74916458129883,
      "activations/layer2_attention_weight_min": -31.016416549682617,
      "activations/layer3_attention_weight_max": 98.27156829833984,
      "activations/layer3_attention_weight_min": -96.24482727050781,
      "activations/layer4_attention_weight_max": 97.48761749267578,
      "activations/layer4_attention_weight_min": -96.89757537841797,
      "activations/layer5_attention_weight_max": 70.11360931396484,
      "activations/layer5_attention_weight_min": -79.68759155273438,
      "activations/layer6_attention_weight_max": 51.409568786621094,
      "activations/layer6_attention_weight_min": -52.137088775634766,
      "activations/layer7_attention_weight_max": 70.56208801269531,
      "activations/layer7_attention_weight_min": -68.51919555664062,
      "activations/layer8_attention_weight_max": 47.12575149536133,
      "activations/layer8_attention_weight_min": -48.91725158691406,
      "activations/layer9_attention_weight_max": 50.76769256591797,
      "activations/layer9_attention_weight_min": -49.003623962402344,
      "epoch": 12.89,
      "learning_rate": 6.757045454545454e-05,
      "loss": 2.7618,
      "step": 221800
    },
    {
      "activations/layer0_attention_weight_max": 16.54063606262207,
      "activations/layer0_attention_weight_min": -14.459637641906738,
      "activations/layer10_attention_weight_max": 38.576087951660156,
      "activations/layer10_attention_weight_min": -36.015987396240234,
      "activations/layer11_attention_weight_max": 33.548377990722656,
      "activations/layer11_attention_weight_min": -32.05902099609375,
      "activations/layer12_attention_weight_max": 19.42076873779297,
      "activations/layer12_attention_weight_min": -26.518266677856445,
      "activations/layer13_attention_weight_max": 39.072998046875,
      "activations/layer13_attention_weight_min": -30.609161376953125,
      "activations/layer14_attention_weight_max": 38.02783966064453,
      "activations/layer14_attention_weight_min": -30.935649871826172,
      "activations/layer15_attention_weight_max": 34.220924377441406,
      "activations/layer15_attention_weight_min": -30.664226531982422,
      "activations/layer16_attention_weight_max": 32.93466567993164,
      "activations/layer16_attention_weight_min": -29.230113983154297,
      "activations/layer17_attention_weight_max": 51.70089340209961,
      "activations/layer17_attention_weight_min": -45.46967315673828,
      "activations/layer18_attention_weight_max": 46.3471794128418,
      "activations/layer18_attention_weight_min": -37.98134231567383,
      "activations/layer19_attention_weight_max": 22.759445190429688,
      "activations/layer19_attention_weight_min": -25.010562896728516,
      "activations/layer1_attention_weight_max": 16.277660369873047,
      "activations/layer1_attention_weight_min": -14.891448020935059,
      "activations/layer20_attention_weight_max": 24.657779693603516,
      "activations/layer20_attention_weight_min": -23.584430694580078,
      "activations/layer21_attention_weight_max": 42.2896614074707,
      "activations/layer21_attention_weight_min": -24.620849609375,
      "activations/layer22_attention_weight_max": 30.297409057617188,
      "activations/layer22_attention_weight_min": -25.691287994384766,
      "activations/layer23_attention_weight_max": 42.05081558227539,
      "activations/layer23_attention_weight_min": -24.94832420349121,
      "activations/layer2_attention_weight_max": 34.78584289550781,
      "activations/layer2_attention_weight_min": -32.34138870239258,
      "activations/layer3_attention_weight_max": 101.770751953125,
      "activations/layer3_attention_weight_min": -102.05799102783203,
      "activations/layer4_attention_weight_max": 102.95880126953125,
      "activations/layer4_attention_weight_min": -97.7742691040039,
      "activations/layer5_attention_weight_max": 74.77699279785156,
      "activations/layer5_attention_weight_min": -82.70783996582031,
      "activations/layer6_attention_weight_max": 54.58082580566406,
      "activations/layer6_attention_weight_min": -54.60625076293945,
      "activations/layer7_attention_weight_max": 74.81449890136719,
      "activations/layer7_attention_weight_min": -65.93948364257812,
      "activations/layer8_attention_weight_max": 50.62550735473633,
      "activations/layer8_attention_weight_min": -49.53181838989258,
      "activations/layer9_attention_weight_max": 50.172454833984375,
      "activations/layer9_attention_weight_min": -47.84548568725586,
      "epoch": 12.89,
      "learning_rate": 6.755151515151515e-05,
      "loss": 2.7708,
      "step": 221850
    },
    {
      "activations/layer0_attention_weight_max": 16.463632583618164,
      "activations/layer0_attention_weight_min": -14.560007095336914,
      "activations/layer10_attention_weight_max": 34.95751190185547,
      "activations/layer10_attention_weight_min": -34.3433952331543,
      "activations/layer11_attention_weight_max": 31.387678146362305,
      "activations/layer11_attention_weight_min": -32.24763107299805,
      "activations/layer12_attention_weight_max": 19.239103317260742,
      "activations/layer12_attention_weight_min": -24.74264144897461,
      "activations/layer13_attention_weight_max": 36.231292724609375,
      "activations/layer13_attention_weight_min": -27.728538513183594,
      "activations/layer14_attention_weight_max": 34.49359130859375,
      "activations/layer14_attention_weight_min": -33.01041793823242,
      "activations/layer15_attention_weight_max": 33.66520690917969,
      "activations/layer15_attention_weight_min": -31.910280227661133,
      "activations/layer16_attention_weight_max": 33.67562484741211,
      "activations/layer16_attention_weight_min": -32.844600677490234,
      "activations/layer17_attention_weight_max": 50.372459411621094,
      "activations/layer17_attention_weight_min": -46.93013000488281,
      "activations/layer18_attention_weight_max": 46.16170883178711,
      "activations/layer18_attention_weight_min": -40.70653533935547,
      "activations/layer19_attention_weight_max": 23.500377655029297,
      "activations/layer19_attention_weight_min": -24.041519165039062,
      "activations/layer1_attention_weight_max": 17.873065948486328,
      "activations/layer1_attention_weight_min": -15.150199890136719,
      "activations/layer20_attention_weight_max": 23.50526237487793,
      "activations/layer20_attention_weight_min": -23.474958419799805,
      "activations/layer21_attention_weight_max": 38.23754119873047,
      "activations/layer21_attention_weight_min": -23.55594253540039,
      "activations/layer22_attention_weight_max": 30.665729522705078,
      "activations/layer22_attention_weight_min": -26.83715057373047,
      "activations/layer23_attention_weight_max": 37.620018005371094,
      "activations/layer23_attention_weight_min": -24.194442749023438,
      "activations/layer2_attention_weight_max": 32.64231872558594,
      "activations/layer2_attention_weight_min": -31.84687614440918,
      "activations/layer3_attention_weight_max": 97.4790267944336,
      "activations/layer3_attention_weight_min": -101.3501205444336,
      "activations/layer4_attention_weight_max": 95.08690643310547,
      "activations/layer4_attention_weight_min": -92.61164855957031,
      "activations/layer5_attention_weight_max": 71.236083984375,
      "activations/layer5_attention_weight_min": -75.52983856201172,
      "activations/layer6_attention_weight_max": 52.17739486694336,
      "activations/layer6_attention_weight_min": -53.353904724121094,
      "activations/layer7_attention_weight_max": 68.81269073486328,
      "activations/layer7_attention_weight_min": -63.70393753051758,
      "activations/layer8_attention_weight_max": 42.24501419067383,
      "activations/layer8_attention_weight_min": -44.51668930053711,
      "activations/layer9_attention_weight_max": 44.755977630615234,
      "activations/layer9_attention_weight_min": -46.396324157714844,
      "epoch": 12.89,
      "learning_rate": 6.753257575757576e-05,
      "loss": 2.7834,
      "step": 221900
    },
    {
      "activations/layer0_attention_weight_max": 16.243200302124023,
      "activations/layer0_attention_weight_min": -14.270981788635254,
      "activations/layer10_attention_weight_max": 34.665321350097656,
      "activations/layer10_attention_weight_min": -34.474586486816406,
      "activations/layer11_attention_weight_max": 29.298627853393555,
      "activations/layer11_attention_weight_min": -32.07804870605469,
      "activations/layer12_attention_weight_max": 18.73577117919922,
      "activations/layer12_attention_weight_min": -26.355905532836914,
      "activations/layer13_attention_weight_max": 36.62330627441406,
      "activations/layer13_attention_weight_min": -27.565887451171875,
      "activations/layer14_attention_weight_max": 36.58921813964844,
      "activations/layer14_attention_weight_min": -29.149852752685547,
      "activations/layer15_attention_weight_max": 32.366512298583984,
      "activations/layer15_attention_weight_min": -28.71572494506836,
      "activations/layer16_attention_weight_max": 30.87404441833496,
      "activations/layer16_attention_weight_min": -30.38736915588379,
      "activations/layer17_attention_weight_max": 54.19586181640625,
      "activations/layer17_attention_weight_min": -44.10133743286133,
      "activations/layer18_attention_weight_max": 48.02488708496094,
      "activations/layer18_attention_weight_min": -39.721412658691406,
      "activations/layer19_attention_weight_max": 22.570697784423828,
      "activations/layer19_attention_weight_min": -21.84305191040039,
      "activations/layer1_attention_weight_max": 17.281909942626953,
      "activations/layer1_attention_weight_min": -18.055959701538086,
      "activations/layer20_attention_weight_max": 26.0757999420166,
      "activations/layer20_attention_weight_min": -20.986515045166016,
      "activations/layer21_attention_weight_max": 38.72958755493164,
      "activations/layer21_attention_weight_min": -22.57459259033203,
      "activations/layer22_attention_weight_max": 28.304908752441406,
      "activations/layer22_attention_weight_min": -25.266382217407227,
      "activations/layer23_attention_weight_max": 37.73590850830078,
      "activations/layer23_attention_weight_min": -23.830646514892578,
      "activations/layer2_attention_weight_max": 31.869115829467773,
      "activations/layer2_attention_weight_min": -31.58333969116211,
      "activations/layer3_attention_weight_max": 94.99967956542969,
      "activations/layer3_attention_weight_min": -95.37013244628906,
      "activations/layer4_attention_weight_max": 94.90331268310547,
      "activations/layer4_attention_weight_min": -92.95764923095703,
      "activations/layer5_attention_weight_max": 71.50003051757812,
      "activations/layer5_attention_weight_min": -76.9872055053711,
      "activations/layer6_attention_weight_max": 52.804325103759766,
      "activations/layer6_attention_weight_min": -51.684688568115234,
      "activations/layer7_attention_weight_max": 66.10787963867188,
      "activations/layer7_attention_weight_min": -67.82498168945312,
      "activations/layer8_attention_weight_max": 43.89393615722656,
      "activations/layer8_attention_weight_min": -46.21025466918945,
      "activations/layer9_attention_weight_max": 45.8015251159668,
      "activations/layer9_attention_weight_min": -45.25187301635742,
      "epoch": 12.9,
      "learning_rate": 6.751363636363636e-05,
      "loss": 2.7629,
      "step": 221950
    },
    {
      "activations/layer0_attention_weight_max": 16.4451847076416,
      "activations/layer0_attention_weight_min": -14.324402809143066,
      "activations/layer10_attention_weight_max": 37.54905319213867,
      "activations/layer10_attention_weight_min": -38.73701858520508,
      "activations/layer11_attention_weight_max": 33.32782745361328,
      "activations/layer11_attention_weight_min": -34.78407287597656,
      "activations/layer12_attention_weight_max": 22.222312927246094,
      "activations/layer12_attention_weight_min": -26.509464263916016,
      "activations/layer13_attention_weight_max": 44.74518585205078,
      "activations/layer13_attention_weight_min": -34.70771026611328,
      "activations/layer14_attention_weight_max": 36.55501174926758,
      "activations/layer14_attention_weight_min": -30.06167984008789,
      "activations/layer15_attention_weight_max": 32.89178466796875,
      "activations/layer15_attention_weight_min": -30.647287368774414,
      "activations/layer16_attention_weight_max": 32.59555435180664,
      "activations/layer16_attention_weight_min": -30.467260360717773,
      "activations/layer17_attention_weight_max": 54.451045989990234,
      "activations/layer17_attention_weight_min": -44.93107986450195,
      "activations/layer18_attention_weight_max": 46.377342224121094,
      "activations/layer18_attention_weight_min": -37.97589874267578,
      "activations/layer19_attention_weight_max": 23.553253173828125,
      "activations/layer19_attention_weight_min": -21.28720474243164,
      "activations/layer1_attention_weight_max": 17.124643325805664,
      "activations/layer1_attention_weight_min": -14.133430480957031,
      "activations/layer20_attention_weight_max": 22.3373966217041,
      "activations/layer20_attention_weight_min": -20.3049259185791,
      "activations/layer21_attention_weight_max": 37.14449691772461,
      "activations/layer21_attention_weight_min": -19.90873146057129,
      "activations/layer22_attention_weight_max": 31.05643653869629,
      "activations/layer22_attention_weight_min": -25.69049072265625,
      "activations/layer23_attention_weight_max": 37.127479553222656,
      "activations/layer23_attention_weight_min": -23.92694854736328,
      "activations/layer2_attention_weight_max": 33.90517044067383,
      "activations/layer2_attention_weight_min": -32.77105712890625,
      "activations/layer3_attention_weight_max": 98.08399200439453,
      "activations/layer3_attention_weight_min": -102.38801574707031,
      "activations/layer4_attention_weight_max": 99.38666534423828,
      "activations/layer4_attention_weight_min": -96.54448699951172,
      "activations/layer5_attention_weight_max": 72.78755950927734,
      "activations/layer5_attention_weight_min": -85.18773651123047,
      "activations/layer6_attention_weight_max": 51.04678726196289,
      "activations/layer6_attention_weight_min": -52.346778869628906,
      "activations/layer7_attention_weight_max": 72.14909362792969,
      "activations/layer7_attention_weight_min": -70.88639068603516,
      "activations/layer8_attention_weight_max": 43.81796646118164,
      "activations/layer8_attention_weight_min": -50.36671829223633,
      "activations/layer9_attention_weight_max": 49.081565856933594,
      "activations/layer9_attention_weight_min": -46.80821228027344,
      "epoch": 12.9,
      "learning_rate": 6.749469696969696e-05,
      "loss": 2.7626,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.5197,
      "eval_samples_per_second": 504.007,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.5197,
      "eval_openwebtext_samples_per_second": 504.007,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9519,
      "eval_wikitext_samples_per_second": 233.621,
      "step": 222000
    },
    {
      "epoch": 12.9,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.5185,
      "eval_lambada_samples_per_second": 511.532,
      "step": 222000
    },
    {
      "activations/layer0_attention_weight_max": 16.47241973876953,
      "activations/layer0_attention_weight_min": -14.593683242797852,
      "activations/layer10_attention_weight_max": 36.72768020629883,
      "activations/layer10_attention_weight_min": -36.83108139038086,
      "activations/layer11_attention_weight_max": 32.159732818603516,
      "activations/layer11_attention_weight_min": -35.31068801879883,
      "activations/layer12_attention_weight_max": 20.5450439453125,
      "activations/layer12_attention_weight_min": -25.85990333557129,
      "activations/layer13_attention_weight_max": 39.66463851928711,
      "activations/layer13_attention_weight_min": -32.21553039550781,
      "activations/layer14_attention_weight_max": 39.216835021972656,
      "activations/layer14_attention_weight_min": -30.988727569580078,
      "activations/layer15_attention_weight_max": 35.53225326538086,
      "activations/layer15_attention_weight_min": -30.145557403564453,
      "activations/layer16_attention_weight_max": 34.1884651184082,
      "activations/layer16_attention_weight_min": -32.753578186035156,
      "activations/layer17_attention_weight_max": 57.42871856689453,
      "activations/layer17_attention_weight_min": -46.561893463134766,
      "activations/layer18_attention_weight_max": 49.32209396362305,
      "activations/layer18_attention_weight_min": -41.13496017456055,
      "activations/layer19_attention_weight_max": 26.629499435424805,
      "activations/layer19_attention_weight_min": -21.09278678894043,
      "activations/layer1_attention_weight_max": 16.8773250579834,
      "activations/layer1_attention_weight_min": -14.867015838623047,
      "activations/layer20_attention_weight_max": 23.245792388916016,
      "activations/layer20_attention_weight_min": -22.260892868041992,
      "activations/layer21_attention_weight_max": 42.08180236816406,
      "activations/layer21_attention_weight_min": -22.147869110107422,
      "activations/layer22_attention_weight_max": 34.447383880615234,
      "activations/layer22_attention_weight_min": -25.061920166015625,
      "activations/layer23_attention_weight_max": 44.33674240112305,
      "activations/layer23_attention_weight_min": -23.867778778076172,
      "activations/layer2_attention_weight_max": 32.36195373535156,
      "activations/layer2_attention_weight_min": -32.377098083496094,
      "activations/layer3_attention_weight_max": 93.51046752929688,
      "activations/layer3_attention_weight_min": -94.0797348022461,
      "activations/layer4_attention_weight_max": 97.05694580078125,
      "activations/layer4_attention_weight_min": -92.08546447753906,
      "activations/layer5_attention_weight_max": 71.84584045410156,
      "activations/layer5_attention_weight_min": -77.95610046386719,
      "activations/layer6_attention_weight_max": 52.74647903442383,
      "activations/layer6_attention_weight_min": -53.598819732666016,
      "activations/layer7_attention_weight_max": 71.93156433105469,
      "activations/layer7_attention_weight_min": -67.73555755615234,
      "activations/layer8_attention_weight_max": 46.914878845214844,
      "activations/layer8_attention_weight_min": -46.99912643432617,
      "activations/layer9_attention_weight_max": 47.97447204589844,
      "activations/layer9_attention_weight_min": -47.46812057495117,
      "epoch": 12.9,
      "learning_rate": 6.747575757575757e-05,
      "loss": 2.7735,
      "step": 222050
    },
    {
      "activations/layer0_attention_weight_max": 16.633140563964844,
      "activations/layer0_attention_weight_min": -15.352669715881348,
      "activations/layer10_attention_weight_max": 38.10316467285156,
      "activations/layer10_attention_weight_min": -35.421016693115234,
      "activations/layer11_attention_weight_max": 33.87214660644531,
      "activations/layer11_attention_weight_min": -33.37384796142578,
      "activations/layer12_attention_weight_max": 18.452119827270508,
      "activations/layer12_attention_weight_min": -24.87199592590332,
      "activations/layer13_attention_weight_max": 39.46339416503906,
      "activations/layer13_attention_weight_min": -34.20048141479492,
      "activations/layer14_attention_weight_max": 33.746498107910156,
      "activations/layer14_attention_weight_min": -30.838191986083984,
      "activations/layer15_attention_weight_max": 31.751638412475586,
      "activations/layer15_attention_weight_min": -30.203323364257812,
      "activations/layer16_attention_weight_max": 30.904382705688477,
      "activations/layer16_attention_weight_min": -30.251237869262695,
      "activations/layer17_attention_weight_max": 50.36805725097656,
      "activations/layer17_attention_weight_min": -46.49939727783203,
      "activations/layer18_attention_weight_max": 45.40842819213867,
      "activations/layer18_attention_weight_min": -38.40833282470703,
      "activations/layer19_attention_weight_max": 21.85075569152832,
      "activations/layer19_attention_weight_min": -23.209951400756836,
      "activations/layer1_attention_weight_max": 19.280668258666992,
      "activations/layer1_attention_weight_min": -17.403032302856445,
      "activations/layer20_attention_weight_max": 22.232177734375,
      "activations/layer20_attention_weight_min": -22.7750244140625,
      "activations/layer21_attention_weight_max": 40.4127197265625,
      "activations/layer21_attention_weight_min": -22.20427894592285,
      "activations/layer22_attention_weight_max": 32.41727066040039,
      "activations/layer22_attention_weight_min": -25.815948486328125,
      "activations/layer23_attention_weight_max": 35.26994323730469,
      "activations/layer23_attention_weight_min": -24.602025985717773,
      "activations/layer2_attention_weight_max": 32.332881927490234,
      "activations/layer2_attention_weight_min": -31.095691680908203,
      "activations/layer3_attention_weight_max": 96.68751525878906,
      "activations/layer3_attention_weight_min": -95.08476257324219,
      "activations/layer4_attention_weight_max": 100.0560531616211,
      "activations/layer4_attention_weight_min": -91.32496643066406,
      "activations/layer5_attention_weight_max": 70.92630004882812,
      "activations/layer5_attention_weight_min": -77.84989929199219,
      "activations/layer6_attention_weight_max": 54.178611755371094,
      "activations/layer6_attention_weight_min": -49.93276596069336,
      "activations/layer7_attention_weight_max": 69.68341064453125,
      "activations/layer7_attention_weight_min": -65.6917724609375,
      "activations/layer8_attention_weight_max": 45.30649185180664,
      "activations/layer8_attention_weight_min": -45.05624771118164,
      "activations/layer9_attention_weight_max": 46.533660888671875,
      "activations/layer9_attention_weight_min": -46.323001861572266,
      "epoch": 12.91,
      "learning_rate": 6.745681818181818e-05,
      "loss": 2.7678,
      "step": 222100
    },
    {
      "activations/layer0_attention_weight_max": 16.12663459777832,
      "activations/layer0_attention_weight_min": -14.656846046447754,
      "activations/layer10_attention_weight_max": 39.89141845703125,
      "activations/layer10_attention_weight_min": -36.16352081298828,
      "activations/layer11_attention_weight_max": 34.28228759765625,
      "activations/layer11_attention_weight_min": -34.04100799560547,
      "activations/layer12_attention_weight_max": 18.608808517456055,
      "activations/layer12_attention_weight_min": -29.176673889160156,
      "activations/layer13_attention_weight_max": 32.20121765136719,
      "activations/layer13_attention_weight_min": -28.266216278076172,
      "activations/layer14_attention_weight_max": 35.88854217529297,
      "activations/layer14_attention_weight_min": -31.273862838745117,
      "activations/layer15_attention_weight_max": 31.941255569458008,
      "activations/layer15_attention_weight_min": -29.17011260986328,
      "activations/layer16_attention_weight_max": 31.15423583984375,
      "activations/layer16_attention_weight_min": -31.286523818969727,
      "activations/layer17_attention_weight_max": 55.1038703918457,
      "activations/layer17_attention_weight_min": -45.93606948852539,
      "activations/layer18_attention_weight_max": 45.0601692199707,
      "activations/layer18_attention_weight_min": -38.024776458740234,
      "activations/layer19_attention_weight_max": 23.966054916381836,
      "activations/layer19_attention_weight_min": -23.51288414001465,
      "activations/layer1_attention_weight_max": 16.953798294067383,
      "activations/layer1_attention_weight_min": -15.592370986938477,
      "activations/layer20_attention_weight_max": 22.70119285583496,
      "activations/layer20_attention_weight_min": -21.76734161376953,
      "activations/layer21_attention_weight_max": 33.38212966918945,
      "activations/layer21_attention_weight_min": -22.429601669311523,
      "activations/layer22_attention_weight_max": 28.485641479492188,
      "activations/layer22_attention_weight_min": -25.383243560791016,
      "activations/layer23_attention_weight_max": 40.10041809082031,
      "activations/layer23_attention_weight_min": -23.820541381835938,
      "activations/layer2_attention_weight_max": 34.94507598876953,
      "activations/layer2_attention_weight_min": -29.426952362060547,
      "activations/layer3_attention_weight_max": 101.9289321899414,
      "activations/layer3_attention_weight_min": -98.17723083496094,
      "activations/layer4_attention_weight_max": 100.39689636230469,
      "activations/layer4_attention_weight_min": -91.43428039550781,
      "activations/layer5_attention_weight_max": 69.58514404296875,
      "activations/layer5_attention_weight_min": -77.92190551757812,
      "activations/layer6_attention_weight_max": 53.63197326660156,
      "activations/layer6_attention_weight_min": -55.8031005859375,
      "activations/layer7_attention_weight_max": 69.49058532714844,
      "activations/layer7_attention_weight_min": -64.71292877197266,
      "activations/layer8_attention_weight_max": 48.69853591918945,
      "activations/layer8_attention_weight_min": -51.23869323730469,
      "activations/layer9_attention_weight_max": 48.26045608520508,
      "activations/layer9_attention_weight_min": -48.66891098022461,
      "epoch": 12.91,
      "learning_rate": 6.743787878787878e-05,
      "loss": 2.7662,
      "step": 222150
    },
    {
      "activations/layer0_attention_weight_max": 17.105045318603516,
      "activations/layer0_attention_weight_min": -14.783971786499023,
      "activations/layer10_attention_weight_max": 36.288639068603516,
      "activations/layer10_attention_weight_min": -35.8980827331543,
      "activations/layer11_attention_weight_max": 33.38070297241211,
      "activations/layer11_attention_weight_min": -34.04344177246094,
      "activations/layer12_attention_weight_max": 19.8978328704834,
      "activations/layer12_attention_weight_min": -27.19504737854004,
      "activations/layer13_attention_weight_max": 39.34010696411133,
      "activations/layer13_attention_weight_min": -28.673118591308594,
      "activations/layer14_attention_weight_max": 36.58552932739258,
      "activations/layer14_attention_weight_min": -30.739974975585938,
      "activations/layer15_attention_weight_max": 31.70647430419922,
      "activations/layer15_attention_weight_min": -30.127573013305664,
      "activations/layer16_attention_weight_max": 29.638954162597656,
      "activations/layer16_attention_weight_min": -28.31587028503418,
      "activations/layer17_attention_weight_max": 49.23982620239258,
      "activations/layer17_attention_weight_min": -43.09796142578125,
      "activations/layer18_attention_weight_max": 45.76335906982422,
      "activations/layer18_attention_weight_min": -37.5821533203125,
      "activations/layer19_attention_weight_max": 23.41876792907715,
      "activations/layer19_attention_weight_min": -21.04050064086914,
      "activations/layer1_attention_weight_max": 17.311431884765625,
      "activations/layer1_attention_weight_min": -15.713337898254395,
      "activations/layer20_attention_weight_max": 21.20355224609375,
      "activations/layer20_attention_weight_min": -20.894983291625977,
      "activations/layer21_attention_weight_max": 34.6928825378418,
      "activations/layer21_attention_weight_min": -22.950218200683594,
      "activations/layer22_attention_weight_max": 30.441425323486328,
      "activations/layer22_attention_weight_min": -25.38888168334961,
      "activations/layer23_attention_weight_max": 36.887393951416016,
      "activations/layer23_attention_weight_min": -25.944997787475586,
      "activations/layer2_attention_weight_max": 31.63167381286621,
      "activations/layer2_attention_weight_min": -29.992176055908203,
      "activations/layer3_attention_weight_max": 91.45899963378906,
      "activations/layer3_attention_weight_min": -95.35157012939453,
      "activations/layer4_attention_weight_max": 94.68379974365234,
      "activations/layer4_attention_weight_min": -94.83870697021484,
      "activations/layer5_attention_weight_max": 71.01884460449219,
      "activations/layer5_attention_weight_min": -79.8670654296875,
      "activations/layer6_attention_weight_max": 51.47719955444336,
      "activations/layer6_attention_weight_min": -51.29694366455078,
      "activations/layer7_attention_weight_max": 68.85669708251953,
      "activations/layer7_attention_weight_min": -70.69564056396484,
      "activations/layer8_attention_weight_max": 46.101776123046875,
      "activations/layer8_attention_weight_min": -48.7711067199707,
      "activations/layer9_attention_weight_max": 45.21997833251953,
      "activations/layer9_attention_weight_min": -45.947113037109375,
      "epoch": 12.91,
      "learning_rate": 6.741893939393938e-05,
      "loss": 2.7834,
      "step": 222200
    },
    {
      "activations/layer0_attention_weight_max": 16.382802963256836,
      "activations/layer0_attention_weight_min": -14.889962196350098,
      "activations/layer10_attention_weight_max": 37.34901809692383,
      "activations/layer10_attention_weight_min": -39.31634521484375,
      "activations/layer11_attention_weight_max": 32.48804473876953,
      "activations/layer11_attention_weight_min": -34.573997497558594,
      "activations/layer12_attention_weight_max": 20.631637573242188,
      "activations/layer12_attention_weight_min": -26.569116592407227,
      "activations/layer13_attention_weight_max": 31.777381896972656,
      "activations/layer13_attention_weight_min": -28.937423706054688,
      "activations/layer14_attention_weight_max": 35.1049690246582,
      "activations/layer14_attention_weight_min": -30.039182662963867,
      "activations/layer15_attention_weight_max": 31.520315170288086,
      "activations/layer15_attention_weight_min": -29.477041244506836,
      "activations/layer16_attention_weight_max": 30.500476837158203,
      "activations/layer16_attention_weight_min": -29.44780158996582,
      "activations/layer17_attention_weight_max": 50.90553283691406,
      "activations/layer17_attention_weight_min": -46.45609664916992,
      "activations/layer18_attention_weight_max": 41.30425262451172,
      "activations/layer18_attention_weight_min": -39.93296432495117,
      "activations/layer19_attention_weight_max": 20.8647518157959,
      "activations/layer19_attention_weight_min": -22.624366760253906,
      "activations/layer1_attention_weight_max": 17.13267707824707,
      "activations/layer1_attention_weight_min": -20.219533920288086,
      "activations/layer20_attention_weight_max": 20.78144073486328,
      "activations/layer20_attention_weight_min": -21.76885223388672,
      "activations/layer21_attention_weight_max": 32.436309814453125,
      "activations/layer21_attention_weight_min": -22.197256088256836,
      "activations/layer22_attention_weight_max": 27.64095687866211,
      "activations/layer22_attention_weight_min": -25.09280014038086,
      "activations/layer23_attention_weight_max": 36.86031723022461,
      "activations/layer23_attention_weight_min": -24.441085815429688,
      "activations/layer2_attention_weight_max": 33.309417724609375,
      "activations/layer2_attention_weight_min": -30.75826644897461,
      "activations/layer3_attention_weight_max": 99.06221771240234,
      "activations/layer3_attention_weight_min": -99.89503479003906,
      "activations/layer4_attention_weight_max": 94.82938385009766,
      "activations/layer4_attention_weight_min": -94.30838012695312,
      "activations/layer5_attention_weight_max": 67.86898803710938,
      "activations/layer5_attention_weight_min": -76.92378234863281,
      "activations/layer6_attention_weight_max": 51.81452560424805,
      "activations/layer6_attention_weight_min": -51.674800872802734,
      "activations/layer7_attention_weight_max": 68.76538848876953,
      "activations/layer7_attention_weight_min": -66.74063873291016,
      "activations/layer8_attention_weight_max": 45.55470275878906,
      "activations/layer8_attention_weight_min": -48.390411376953125,
      "activations/layer9_attention_weight_max": 50.345497131347656,
      "activations/layer9_attention_weight_min": -52.27863311767578,
      "epoch": 12.91,
      "learning_rate": 6.739999999999998e-05,
      "loss": 2.7719,
      "step": 222250
    },
    {
      "activations/layer0_attention_weight_max": 15.949957847595215,
      "activations/layer0_attention_weight_min": -14.918492317199707,
      "activations/layer10_attention_weight_max": 37.103878021240234,
      "activations/layer10_attention_weight_min": -36.27055358886719,
      "activations/layer11_attention_weight_max": 32.91881561279297,
      "activations/layer11_attention_weight_min": -33.316062927246094,
      "activations/layer12_attention_weight_max": 24.035993576049805,
      "activations/layer12_attention_weight_min": -22.776395797729492,
      "activations/layer13_attention_weight_max": 37.33848571777344,
      "activations/layer13_attention_weight_min": -27.89434814453125,
      "activations/layer14_attention_weight_max": 34.617279052734375,
      "activations/layer14_attention_weight_min": -30.372312545776367,
      "activations/layer15_attention_weight_max": 32.91151428222656,
      "activations/layer15_attention_weight_min": -28.79939842224121,
      "activations/layer16_attention_weight_max": 31.267581939697266,
      "activations/layer16_attention_weight_min": -29.198984146118164,
      "activations/layer17_attention_weight_max": 54.14430236816406,
      "activations/layer17_attention_weight_min": -42.02251434326172,
      "activations/layer18_attention_weight_max": 47.55992126464844,
      "activations/layer18_attention_weight_min": -36.818206787109375,
      "activations/layer19_attention_weight_max": 21.7606258392334,
      "activations/layer19_attention_weight_min": -21.763654708862305,
      "activations/layer1_attention_weight_max": 16.38747787475586,
      "activations/layer1_attention_weight_min": -16.12395668029785,
      "activations/layer20_attention_weight_max": 23.335050582885742,
      "activations/layer20_attention_weight_min": -20.806907653808594,
      "activations/layer21_attention_weight_max": 34.570796966552734,
      "activations/layer21_attention_weight_min": -22.250688552856445,
      "activations/layer22_attention_weight_max": 28.705116271972656,
      "activations/layer22_attention_weight_min": -25.22658920288086,
      "activations/layer23_attention_weight_max": 38.803199768066406,
      "activations/layer23_attention_weight_min": -23.540740966796875,
      "activations/layer2_attention_weight_max": 32.788719177246094,
      "activations/layer2_attention_weight_min": -30.945310592651367,
      "activations/layer3_attention_weight_max": 92.6533203125,
      "activations/layer3_attention_weight_min": -96.97293853759766,
      "activations/layer4_attention_weight_max": 96.0905532836914,
      "activations/layer4_attention_weight_min": -95.40795135498047,
      "activations/layer5_attention_weight_max": 75.54559326171875,
      "activations/layer5_attention_weight_min": -79.21171569824219,
      "activations/layer6_attention_weight_max": 52.827980041503906,
      "activations/layer6_attention_weight_min": -52.42692947387695,
      "activations/layer7_attention_weight_max": 66.40721130371094,
      "activations/layer7_attention_weight_min": -67.08708190917969,
      "activations/layer8_attention_weight_max": 45.025753021240234,
      "activations/layer8_attention_weight_min": -46.89725112915039,
      "activations/layer9_attention_weight_max": 46.41203689575195,
      "activations/layer9_attention_weight_min": -48.03317642211914,
      "epoch": 12.92,
      "learning_rate": 6.73810606060606e-05,
      "loss": 2.7715,
      "step": 222300
    },
    {
      "activations/layer0_attention_weight_max": 16.182029724121094,
      "activations/layer0_attention_weight_min": -14.974775314331055,
      "activations/layer10_attention_weight_max": 37.31543731689453,
      "activations/layer10_attention_weight_min": -36.31781768798828,
      "activations/layer11_attention_weight_max": 31.771299362182617,
      "activations/layer11_attention_weight_min": -34.42002487182617,
      "activations/layer12_attention_weight_max": 18.686614990234375,
      "activations/layer12_attention_weight_min": -24.17308235168457,
      "activations/layer13_attention_weight_max": 50.03258514404297,
      "activations/layer13_attention_weight_min": -31.90787696838379,
      "activations/layer14_attention_weight_max": 34.06767654418945,
      "activations/layer14_attention_weight_min": -31.271591186523438,
      "activations/layer15_attention_weight_max": 30.99178123474121,
      "activations/layer15_attention_weight_min": -30.31671905517578,
      "activations/layer16_attention_weight_max": 30.26976203918457,
      "activations/layer16_attention_weight_min": -30.347671508789062,
      "activations/layer17_attention_weight_max": 51.677696228027344,
      "activations/layer17_attention_weight_min": -44.81692123413086,
      "activations/layer18_attention_weight_max": 42.42461395263672,
      "activations/layer18_attention_weight_min": -38.25857925415039,
      "activations/layer19_attention_weight_max": 21.23953628540039,
      "activations/layer19_attention_weight_min": -21.679378509521484,
      "activations/layer1_attention_weight_max": 19.34349822998047,
      "activations/layer1_attention_weight_min": -16.333589553833008,
      "activations/layer20_attention_weight_max": 21.115310668945312,
      "activations/layer20_attention_weight_min": -23.77711296081543,
      "activations/layer21_attention_weight_max": 35.03572463989258,
      "activations/layer21_attention_weight_min": -23.679033279418945,
      "activations/layer22_attention_weight_max": 26.73626136779785,
      "activations/layer22_attention_weight_min": -26.485761642456055,
      "activations/layer23_attention_weight_max": 36.203495025634766,
      "activations/layer23_attention_weight_min": -23.96575927734375,
      "activations/layer2_attention_weight_max": 33.45492935180664,
      "activations/layer2_attention_weight_min": -32.53733444213867,
      "activations/layer3_attention_weight_max": 98.75366973876953,
      "activations/layer3_attention_weight_min": -110.40055084228516,
      "activations/layer4_attention_weight_max": 97.090087890625,
      "activations/layer4_attention_weight_min": -100.93904876708984,
      "activations/layer5_attention_weight_max": 70.30809020996094,
      "activations/layer5_attention_weight_min": -79.86578369140625,
      "activations/layer6_attention_weight_max": 54.02949905395508,
      "activations/layer6_attention_weight_min": -55.54557418823242,
      "activations/layer7_attention_weight_max": 68.8323974609375,
      "activations/layer7_attention_weight_min": -68.38633728027344,
      "activations/layer8_attention_weight_max": 46.86342239379883,
      "activations/layer8_attention_weight_min": -48.92802047729492,
      "activations/layer9_attention_weight_max": 46.56446075439453,
      "activations/layer9_attention_weight_min": -47.38530349731445,
      "epoch": 12.92,
      "learning_rate": 6.73621212121212e-05,
      "loss": 2.7634,
      "step": 222350
    },
    {
      "activations/layer0_attention_weight_max": 17.04609489440918,
      "activations/layer0_attention_weight_min": -14.796241760253906,
      "activations/layer10_attention_weight_max": 36.143672943115234,
      "activations/layer10_attention_weight_min": -36.09278106689453,
      "activations/layer11_attention_weight_max": 32.050941467285156,
      "activations/layer11_attention_weight_min": -32.40222930908203,
      "activations/layer12_attention_weight_max": 17.42523193359375,
      "activations/layer12_attention_weight_min": -24.981948852539062,
      "activations/layer13_attention_weight_max": 35.778289794921875,
      "activations/layer13_attention_weight_min": -28.641643524169922,
      "activations/layer14_attention_weight_max": 40.463626861572266,
      "activations/layer14_attention_weight_min": -31.39649772644043,
      "activations/layer15_attention_weight_max": 33.87831115722656,
      "activations/layer15_attention_weight_min": -29.634387969970703,
      "activations/layer16_attention_weight_max": 32.636146545410156,
      "activations/layer16_attention_weight_min": -30.80397605895996,
      "activations/layer17_attention_weight_max": 50.285465240478516,
      "activations/layer17_attention_weight_min": -44.23115921020508,
      "activations/layer18_attention_weight_max": 45.42867660522461,
      "activations/layer18_attention_weight_min": -36.743560791015625,
      "activations/layer19_attention_weight_max": 22.46811294555664,
      "activations/layer19_attention_weight_min": -23.490278244018555,
      "activations/layer1_attention_weight_max": 16.753990173339844,
      "activations/layer1_attention_weight_min": -14.113943099975586,
      "activations/layer20_attention_weight_max": 23.246965408325195,
      "activations/layer20_attention_weight_min": -20.451276779174805,
      "activations/layer21_attention_weight_max": 40.91606140136719,
      "activations/layer21_attention_weight_min": -23.01230239868164,
      "activations/layer22_attention_weight_max": 29.340173721313477,
      "activations/layer22_attention_weight_min": -23.778493881225586,
      "activations/layer23_attention_weight_max": 35.96705627441406,
      "activations/layer23_attention_weight_min": -22.49496078491211,
      "activations/layer2_attention_weight_max": 32.78017807006836,
      "activations/layer2_attention_weight_min": -30.077512741088867,
      "activations/layer3_attention_weight_max": 93.52478790283203,
      "activations/layer3_attention_weight_min": -99.27790832519531,
      "activations/layer4_attention_weight_max": 96.3365707397461,
      "activations/layer4_attention_weight_min": -95.9206771850586,
      "activations/layer5_attention_weight_max": 71.14701843261719,
      "activations/layer5_attention_weight_min": -75.3826904296875,
      "activations/layer6_attention_weight_max": 51.00933074951172,
      "activations/layer6_attention_weight_min": -53.280765533447266,
      "activations/layer7_attention_weight_max": 69.994384765625,
      "activations/layer7_attention_weight_min": -69.29936981201172,
      "activations/layer8_attention_weight_max": 44.972572326660156,
      "activations/layer8_attention_weight_min": -48.06547927856445,
      "activations/layer9_attention_weight_max": 47.115108489990234,
      "activations/layer9_attention_weight_min": -45.64126205444336,
      "epoch": 12.92,
      "learning_rate": 6.734318181818182e-05,
      "loss": 2.7776,
      "step": 222400
    },
    {
      "activations/layer0_attention_weight_max": 16.421688079833984,
      "activations/layer0_attention_weight_min": -14.654397010803223,
      "activations/layer10_attention_weight_max": 40.76023483276367,
      "activations/layer10_attention_weight_min": -37.208274841308594,
      "activations/layer11_attention_weight_max": 33.835205078125,
      "activations/layer11_attention_weight_min": -32.992069244384766,
      "activations/layer12_attention_weight_max": 19.55326271057129,
      "activations/layer12_attention_weight_min": -22.736141204833984,
      "activations/layer13_attention_weight_max": 38.01375198364258,
      "activations/layer13_attention_weight_min": -29.639705657958984,
      "activations/layer14_attention_weight_max": 35.789710998535156,
      "activations/layer14_attention_weight_min": -29.99144172668457,
      "activations/layer15_attention_weight_max": 32.60399627685547,
      "activations/layer15_attention_weight_min": -28.850906372070312,
      "activations/layer16_attention_weight_max": 29.60641098022461,
      "activations/layer16_attention_weight_min": -30.51670265197754,
      "activations/layer17_attention_weight_max": 49.90175247192383,
      "activations/layer17_attention_weight_min": -43.43511962890625,
      "activations/layer18_attention_weight_max": 44.8157958984375,
      "activations/layer18_attention_weight_min": -39.45058822631836,
      "activations/layer19_attention_weight_max": 23.07844352722168,
      "activations/layer19_attention_weight_min": -24.703269958496094,
      "activations/layer1_attention_weight_max": 16.886449813842773,
      "activations/layer1_attention_weight_min": -16.67548942565918,
      "activations/layer20_attention_weight_max": 23.640892028808594,
      "activations/layer20_attention_weight_min": -22.46503448486328,
      "activations/layer21_attention_weight_max": 36.72096633911133,
      "activations/layer21_attention_weight_min": -22.454246520996094,
      "activations/layer22_attention_weight_max": 31.02635383605957,
      "activations/layer22_attention_weight_min": -25.04210662841797,
      "activations/layer23_attention_weight_max": 38.694618225097656,
      "activations/layer23_attention_weight_min": -24.432775497436523,
      "activations/layer2_attention_weight_max": 33.85515594482422,
      "activations/layer2_attention_weight_min": -32.260398864746094,
      "activations/layer3_attention_weight_max": 95.54180908203125,
      "activations/layer3_attention_weight_min": -99.53978729248047,
      "activations/layer4_attention_weight_max": 96.2713851928711,
      "activations/layer4_attention_weight_min": -90.6615219116211,
      "activations/layer5_attention_weight_max": 72.77288818359375,
      "activations/layer5_attention_weight_min": -81.83592987060547,
      "activations/layer6_attention_weight_max": 53.160160064697266,
      "activations/layer6_attention_weight_min": -50.94832229614258,
      "activations/layer7_attention_weight_max": 65.21591186523438,
      "activations/layer7_attention_weight_min": -66.3286361694336,
      "activations/layer8_attention_weight_max": 47.09298324584961,
      "activations/layer8_attention_weight_min": -51.02604293823242,
      "activations/layer9_attention_weight_max": 48.39133071899414,
      "activations/layer9_attention_weight_min": -47.57905197143555,
      "epoch": 12.93,
      "learning_rate": 6.732424242424242e-05,
      "loss": 2.7568,
      "step": 222450
    },
    {
      "activations/layer0_attention_weight_max": 17.151485443115234,
      "activations/layer0_attention_weight_min": -14.344658851623535,
      "activations/layer10_attention_weight_max": 36.26953887939453,
      "activations/layer10_attention_weight_min": -35.41059875488281,
      "activations/layer11_attention_weight_max": 32.69599533081055,
      "activations/layer11_attention_weight_min": -31.51034164428711,
      "activations/layer12_attention_weight_max": 19.17744255065918,
      "activations/layer12_attention_weight_min": -25.332990646362305,
      "activations/layer13_attention_weight_max": 37.72462463378906,
      "activations/layer13_attention_weight_min": -28.244237899780273,
      "activations/layer14_attention_weight_max": 36.4135627746582,
      "activations/layer14_attention_weight_min": -31.511510848999023,
      "activations/layer15_attention_weight_max": 34.9382209777832,
      "activations/layer15_attention_weight_min": -29.2872257232666,
      "activations/layer16_attention_weight_max": 32.12340545654297,
      "activations/layer16_attention_weight_min": -31.675003051757812,
      "activations/layer17_attention_weight_max": 53.94590759277344,
      "activations/layer17_attention_weight_min": -45.40347671508789,
      "activations/layer18_attention_weight_max": 46.1524543762207,
      "activations/layer18_attention_weight_min": -38.83150863647461,
      "activations/layer19_attention_weight_max": 22.328144073486328,
      "activations/layer19_attention_weight_min": -21.530824661254883,
      "activations/layer1_attention_weight_max": 17.248600006103516,
      "activations/layer1_attention_weight_min": -17.557344436645508,
      "activations/layer20_attention_weight_max": 20.644058227539062,
      "activations/layer20_attention_weight_min": -21.52682876586914,
      "activations/layer21_attention_weight_max": 37.893253326416016,
      "activations/layer21_attention_weight_min": -23.716697692871094,
      "activations/layer22_attention_weight_max": 29.748517990112305,
      "activations/layer22_attention_weight_min": -26.278724670410156,
      "activations/layer23_attention_weight_max": 34.11457061767578,
      "activations/layer23_attention_weight_min": -23.962909698486328,
      "activations/layer2_attention_weight_max": 34.08172607421875,
      "activations/layer2_attention_weight_min": -31.695079803466797,
      "activations/layer3_attention_weight_max": 96.1888656616211,
      "activations/layer3_attention_weight_min": -93.37249755859375,
      "activations/layer4_attention_weight_max": 96.29045867919922,
      "activations/layer4_attention_weight_min": -90.1126937866211,
      "activations/layer5_attention_weight_max": 72.27644348144531,
      "activations/layer5_attention_weight_min": -82.81257629394531,
      "activations/layer6_attention_weight_max": 51.98728561401367,
      "activations/layer6_attention_weight_min": -53.40258026123047,
      "activations/layer7_attention_weight_max": 67.4596176147461,
      "activations/layer7_attention_weight_min": -67.07324981689453,
      "activations/layer8_attention_weight_max": 44.8621940612793,
      "activations/layer8_attention_weight_min": -44.79513168334961,
      "activations/layer9_attention_weight_max": 43.96739959716797,
      "activations/layer9_attention_weight_min": -44.89463424682617,
      "epoch": 12.93,
      "learning_rate": 6.730530303030302e-05,
      "loss": 2.7845,
      "step": 222500
    },
    {
      "activations/layer0_attention_weight_max": 17.31973648071289,
      "activations/layer0_attention_weight_min": -14.857664108276367,
      "activations/layer10_attention_weight_max": 33.94279861450195,
      "activations/layer10_attention_weight_min": -33.850250244140625,
      "activations/layer11_attention_weight_max": 31.082942962646484,
      "activations/layer11_attention_weight_min": -34.231964111328125,
      "activations/layer12_attention_weight_max": 17.213756561279297,
      "activations/layer12_attention_weight_min": -22.209184646606445,
      "activations/layer13_attention_weight_max": 34.0971794128418,
      "activations/layer13_attention_weight_min": -28.98207664489746,
      "activations/layer14_attention_weight_max": 34.60203170776367,
      "activations/layer14_attention_weight_min": -30.72624969482422,
      "activations/layer15_attention_weight_max": 33.398773193359375,
      "activations/layer15_attention_weight_min": -28.739215850830078,
      "activations/layer16_attention_weight_max": 28.96596336364746,
      "activations/layer16_attention_weight_min": -29.33506965637207,
      "activations/layer17_attention_weight_max": 49.650672912597656,
      "activations/layer17_attention_weight_min": -43.513851165771484,
      "activations/layer18_attention_weight_max": 43.352542877197266,
      "activations/layer18_attention_weight_min": -39.26742935180664,
      "activations/layer19_attention_weight_max": 22.620466232299805,
      "activations/layer19_attention_weight_min": -22.347190856933594,
      "activations/layer1_attention_weight_max": 17.32003402709961,
      "activations/layer1_attention_weight_min": -17.522855758666992,
      "activations/layer20_attention_weight_max": 21.067798614501953,
      "activations/layer20_attention_weight_min": -20.72776222229004,
      "activations/layer21_attention_weight_max": 34.15839767456055,
      "activations/layer21_attention_weight_min": -20.481632232666016,
      "activations/layer22_attention_weight_max": 27.199060440063477,
      "activations/layer22_attention_weight_min": -24.86383056640625,
      "activations/layer23_attention_weight_max": 34.13470458984375,
      "activations/layer23_attention_weight_min": -23.665939331054688,
      "activations/layer2_attention_weight_max": 33.040367126464844,
      "activations/layer2_attention_weight_min": -31.559467315673828,
      "activations/layer3_attention_weight_max": 91.85845184326172,
      "activations/layer3_attention_weight_min": -95.08543395996094,
      "activations/layer4_attention_weight_max": 98.27217864990234,
      "activations/layer4_attention_weight_min": -94.3709487915039,
      "activations/layer5_attention_weight_max": 69.68084716796875,
      "activations/layer5_attention_weight_min": -82.07550811767578,
      "activations/layer6_attention_weight_max": 50.434635162353516,
      "activations/layer6_attention_weight_min": -51.3321647644043,
      "activations/layer7_attention_weight_max": 64.51934814453125,
      "activations/layer7_attention_weight_min": -66.40351104736328,
      "activations/layer8_attention_weight_max": 43.18455505371094,
      "activations/layer8_attention_weight_min": -43.76130294799805,
      "activations/layer9_attention_weight_max": 47.42446517944336,
      "activations/layer9_attention_weight_min": -44.676170349121094,
      "epoch": 12.93,
      "learning_rate": 6.728636363636363e-05,
      "loss": 2.785,
      "step": 222550
    },
    {
      "activations/layer0_attention_weight_max": 17.45683479309082,
      "activations/layer0_attention_weight_min": -14.445942878723145,
      "activations/layer10_attention_weight_max": 37.58161163330078,
      "activations/layer10_attention_weight_min": -38.03655242919922,
      "activations/layer11_attention_weight_max": 33.7149543762207,
      "activations/layer11_attention_weight_min": -33.865394592285156,
      "activations/layer12_attention_weight_max": 19.08223533630371,
      "activations/layer12_attention_weight_min": -25.18182373046875,
      "activations/layer13_attention_weight_max": 35.52177047729492,
      "activations/layer13_attention_weight_min": -27.846643447875977,
      "activations/layer14_attention_weight_max": 38.362430572509766,
      "activations/layer14_attention_weight_min": -29.59701156616211,
      "activations/layer15_attention_weight_max": 34.420143127441406,
      "activations/layer15_attention_weight_min": -30.198322296142578,
      "activations/layer16_attention_weight_max": 31.446863174438477,
      "activations/layer16_attention_weight_min": -30.334867477416992,
      "activations/layer17_attention_weight_max": 56.20289611816406,
      "activations/layer17_attention_weight_min": -46.24656677246094,
      "activations/layer18_attention_weight_max": 48.007774353027344,
      "activations/layer18_attention_weight_min": -37.768104553222656,
      "activations/layer19_attention_weight_max": 21.15485191345215,
      "activations/layer19_attention_weight_min": -20.438467025756836,
      "activations/layer1_attention_weight_max": 17.457042694091797,
      "activations/layer1_attention_weight_min": -17.334856033325195,
      "activations/layer20_attention_weight_max": 22.416784286499023,
      "activations/layer20_attention_weight_min": -21.506101608276367,
      "activations/layer21_attention_weight_max": 33.29850769042969,
      "activations/layer21_attention_weight_min": -21.016860961914062,
      "activations/layer22_attention_weight_max": 28.37713050842285,
      "activations/layer22_attention_weight_min": -24.758134841918945,
      "activations/layer23_attention_weight_max": 35.58063507080078,
      "activations/layer23_attention_weight_min": -24.067428588867188,
      "activations/layer2_attention_weight_max": 30.52334976196289,
      "activations/layer2_attention_weight_min": -31.210552215576172,
      "activations/layer3_attention_weight_max": 91.73591613769531,
      "activations/layer3_attention_weight_min": -93.04024505615234,
      "activations/layer4_attention_weight_max": 94.93193054199219,
      "activations/layer4_attention_weight_min": -94.92913055419922,
      "activations/layer5_attention_weight_max": 68.54470825195312,
      "activations/layer5_attention_weight_min": -77.93974304199219,
      "activations/layer6_attention_weight_max": 48.57862091064453,
      "activations/layer6_attention_weight_min": -51.481163024902344,
      "activations/layer7_attention_weight_max": 65.40023803710938,
      "activations/layer7_attention_weight_min": -67.63188171386719,
      "activations/layer8_attention_weight_max": 43.72086715698242,
      "activations/layer8_attention_weight_min": -47.3227424621582,
      "activations/layer9_attention_weight_max": 46.12371063232422,
      "activations/layer9_attention_weight_min": -50.366065979003906,
      "epoch": 12.93,
      "learning_rate": 6.726742424242424e-05,
      "loss": 2.7688,
      "step": 222600
    },
    {
      "activations/layer0_attention_weight_max": 16.083635330200195,
      "activations/layer0_attention_weight_min": -14.4530029296875,
      "activations/layer10_attention_weight_max": 38.99154281616211,
      "activations/layer10_attention_weight_min": -35.78836441040039,
      "activations/layer11_attention_weight_max": 35.78825378417969,
      "activations/layer11_attention_weight_min": -32.99470901489258,
      "activations/layer12_attention_weight_max": 18.945470809936523,
      "activations/layer12_attention_weight_min": -26.01463508605957,
      "activations/layer13_attention_weight_max": 34.67414093017578,
      "activations/layer13_attention_weight_min": -28.335641860961914,
      "activations/layer14_attention_weight_max": 34.48621368408203,
      "activations/layer14_attention_weight_min": -29.741403579711914,
      "activations/layer15_attention_weight_max": 29.703777313232422,
      "activations/layer15_attention_weight_min": -28.41826629638672,
      "activations/layer16_attention_weight_max": 32.03378677368164,
      "activations/layer16_attention_weight_min": -28.038915634155273,
      "activations/layer17_attention_weight_max": 49.445770263671875,
      "activations/layer17_attention_weight_min": -42.81208801269531,
      "activations/layer18_attention_weight_max": 45.00040054321289,
      "activations/layer18_attention_weight_min": -38.64834976196289,
      "activations/layer19_attention_weight_max": 22.51027488708496,
      "activations/layer19_attention_weight_min": -22.458120346069336,
      "activations/layer1_attention_weight_max": 16.59062957763672,
      "activations/layer1_attention_weight_min": -15.068764686584473,
      "activations/layer20_attention_weight_max": 24.281810760498047,
      "activations/layer20_attention_weight_min": -23.294994354248047,
      "activations/layer21_attention_weight_max": 37.723602294921875,
      "activations/layer21_attention_weight_min": -23.59276008605957,
      "activations/layer22_attention_weight_max": 33.011959075927734,
      "activations/layer22_attention_weight_min": -25.18904685974121,
      "activations/layer23_attention_weight_max": 37.491050720214844,
      "activations/layer23_attention_weight_min": -26.184797286987305,
      "activations/layer2_attention_weight_max": 31.05203628540039,
      "activations/layer2_attention_weight_min": -29.7352294921875,
      "activations/layer3_attention_weight_max": 90.42546081542969,
      "activations/layer3_attention_weight_min": -94.89457702636719,
      "activations/layer4_attention_weight_max": 95.70533752441406,
      "activations/layer4_attention_weight_min": -99.3559799194336,
      "activations/layer5_attention_weight_max": 68.89581298828125,
      "activations/layer5_attention_weight_min": -81.65206146240234,
      "activations/layer6_attention_weight_max": 54.011837005615234,
      "activations/layer6_attention_weight_min": -51.777915954589844,
      "activations/layer7_attention_weight_max": 75.67938232421875,
      "activations/layer7_attention_weight_min": -70.02432250976562,
      "activations/layer8_attention_weight_max": 48.0890007019043,
      "activations/layer8_attention_weight_min": -48.491050720214844,
      "activations/layer9_attention_weight_max": 47.37921905517578,
      "activations/layer9_attention_weight_min": -46.82863235473633,
      "epoch": 12.94,
      "learning_rate": 6.724848484848485e-05,
      "loss": 2.787,
      "step": 222650
    },
    {
      "activations/layer0_attention_weight_max": 15.89173412322998,
      "activations/layer0_attention_weight_min": -14.338687896728516,
      "activations/layer10_attention_weight_max": 39.804935455322266,
      "activations/layer10_attention_weight_min": -38.06266403198242,
      "activations/layer11_attention_weight_max": 34.010337829589844,
      "activations/layer11_attention_weight_min": -35.14513397216797,
      "activations/layer12_attention_weight_max": 25.938997268676758,
      "activations/layer12_attention_weight_min": -25.804723739624023,
      "activations/layer13_attention_weight_max": 43.4083366394043,
      "activations/layer13_attention_weight_min": -30.105459213256836,
      "activations/layer14_attention_weight_max": 38.284507751464844,
      "activations/layer14_attention_weight_min": -29.304372787475586,
      "activations/layer15_attention_weight_max": 36.8521614074707,
      "activations/layer15_attention_weight_min": -29.25674819946289,
      "activations/layer16_attention_weight_max": 32.373939514160156,
      "activations/layer16_attention_weight_min": -30.01247787475586,
      "activations/layer17_attention_weight_max": 54.526893615722656,
      "activations/layer17_attention_weight_min": -46.14097595214844,
      "activations/layer18_attention_weight_max": 47.69259262084961,
      "activations/layer18_attention_weight_min": -42.55385971069336,
      "activations/layer19_attention_weight_max": 25.88295555114746,
      "activations/layer19_attention_weight_min": -23.428668975830078,
      "activations/layer1_attention_weight_max": 17.217437744140625,
      "activations/layer1_attention_weight_min": -15.009668350219727,
      "activations/layer20_attention_weight_max": 27.05486297607422,
      "activations/layer20_attention_weight_min": -20.06426239013672,
      "activations/layer21_attention_weight_max": 36.261051177978516,
      "activations/layer21_attention_weight_min": -23.32480812072754,
      "activations/layer22_attention_weight_max": 35.64945602416992,
      "activations/layer22_attention_weight_min": -25.412446975708008,
      "activations/layer23_attention_weight_max": 39.61231994628906,
      "activations/layer23_attention_weight_min": -25.77972984313965,
      "activations/layer2_attention_weight_max": 30.986684799194336,
      "activations/layer2_attention_weight_min": -30.28661346435547,
      "activations/layer3_attention_weight_max": 98.48003387451172,
      "activations/layer3_attention_weight_min": -94.79035949707031,
      "activations/layer4_attention_weight_max": 99.1056137084961,
      "activations/layer4_attention_weight_min": -92.53557586669922,
      "activations/layer5_attention_weight_max": 71.92662048339844,
      "activations/layer5_attention_weight_min": -78.86285400390625,
      "activations/layer6_attention_weight_max": 53.05282974243164,
      "activations/layer6_attention_weight_min": -53.793006896972656,
      "activations/layer7_attention_weight_max": 69.24898529052734,
      "activations/layer7_attention_weight_min": -67.30314636230469,
      "activations/layer8_attention_weight_max": 45.932456970214844,
      "activations/layer8_attention_weight_min": -48.79397201538086,
      "activations/layer9_attention_weight_max": 50.484535217285156,
      "activations/layer9_attention_weight_min": -47.74275588989258,
      "epoch": 12.94,
      "learning_rate": 6.722954545454545e-05,
      "loss": 2.7641,
      "step": 222700
    },
    {
      "activations/layer0_attention_weight_max": 16.76605224609375,
      "activations/layer0_attention_weight_min": -14.643416404724121,
      "activations/layer10_attention_weight_max": 35.98843765258789,
      "activations/layer10_attention_weight_min": -34.98692321777344,
      "activations/layer11_attention_weight_max": 32.53349304199219,
      "activations/layer11_attention_weight_min": -33.657352447509766,
      "activations/layer12_attention_weight_max": 18.96062660217285,
      "activations/layer12_attention_weight_min": -25.421783447265625,
      "activations/layer13_attention_weight_max": 48.59819793701172,
      "activations/layer13_attention_weight_min": -36.16828155517578,
      "activations/layer14_attention_weight_max": 41.62578582763672,
      "activations/layer14_attention_weight_min": -32.91532516479492,
      "activations/layer15_attention_weight_max": 38.12895202636719,
      "activations/layer15_attention_weight_min": -30.07558250427246,
      "activations/layer16_attention_weight_max": 34.65449905395508,
      "activations/layer16_attention_weight_min": -31.189697265625,
      "activations/layer17_attention_weight_max": 51.280574798583984,
      "activations/layer17_attention_weight_min": -47.245277404785156,
      "activations/layer18_attention_weight_max": 47.59921646118164,
      "activations/layer18_attention_weight_min": -42.19784164428711,
      "activations/layer19_attention_weight_max": 22.345111846923828,
      "activations/layer19_attention_weight_min": -22.566099166870117,
      "activations/layer1_attention_weight_max": 17.226245880126953,
      "activations/layer1_attention_weight_min": -14.934377670288086,
      "activations/layer20_attention_weight_max": 23.07862663269043,
      "activations/layer20_attention_weight_min": -22.529674530029297,
      "activations/layer21_attention_weight_max": 47.18534469604492,
      "activations/layer21_attention_weight_min": -26.215526580810547,
      "activations/layer22_attention_weight_max": 32.61086654663086,
      "activations/layer22_attention_weight_min": -27.25498390197754,
      "activations/layer23_attention_weight_max": 34.56713104248047,
      "activations/layer23_attention_weight_min": -24.144306182861328,
      "activations/layer2_attention_weight_max": 33.258148193359375,
      "activations/layer2_attention_weight_min": -30.753585815429688,
      "activations/layer3_attention_weight_max": 98.01007843017578,
      "activations/layer3_attention_weight_min": -99.87667083740234,
      "activations/layer4_attention_weight_max": 100.15563201904297,
      "activations/layer4_attention_weight_min": -95.1356430053711,
      "activations/layer5_attention_weight_max": 76.21896362304688,
      "activations/layer5_attention_weight_min": -79.79722595214844,
      "activations/layer6_attention_weight_max": 52.31747055053711,
      "activations/layer6_attention_weight_min": -54.63460922241211,
      "activations/layer7_attention_weight_max": 77.5731201171875,
      "activations/layer7_attention_weight_min": -68.83544921875,
      "activations/layer8_attention_weight_max": 46.978824615478516,
      "activations/layer8_attention_weight_min": -47.42472839355469,
      "activations/layer9_attention_weight_max": 49.95187759399414,
      "activations/layer9_attention_weight_min": -47.155391693115234,
      "epoch": 12.94,
      "learning_rate": 6.721098484848484e-05,
      "loss": 2.7567,
      "step": 222750
    },
    {
      "activations/layer0_attention_weight_max": 16.435163497924805,
      "activations/layer0_attention_weight_min": -14.212671279907227,
      "activations/layer10_attention_weight_max": 37.49997329711914,
      "activations/layer10_attention_weight_min": -37.821006774902344,
      "activations/layer11_attention_weight_max": 35.982051849365234,
      "activations/layer11_attention_weight_min": -34.09700012207031,
      "activations/layer12_attention_weight_max": 20.837665557861328,
      "activations/layer12_attention_weight_min": -26.03430938720703,
      "activations/layer13_attention_weight_max": 44.5090217590332,
      "activations/layer13_attention_weight_min": -31.271095275878906,
      "activations/layer14_attention_weight_max": 41.829322814941406,
      "activations/layer14_attention_weight_min": -31.15229034423828,
      "activations/layer15_attention_weight_max": 39.72571563720703,
      "activations/layer15_attention_weight_min": -28.628372192382812,
      "activations/layer16_attention_weight_max": 30.662263870239258,
      "activations/layer16_attention_weight_min": -30.091829299926758,
      "activations/layer17_attention_weight_max": 50.33052062988281,
      "activations/layer17_attention_weight_min": -45.33774948120117,
      "activations/layer18_attention_weight_max": 47.48324203491211,
      "activations/layer18_attention_weight_min": -39.29661560058594,
      "activations/layer19_attention_weight_max": 23.83991813659668,
      "activations/layer19_attention_weight_min": -24.539003372192383,
      "activations/layer1_attention_weight_max": 17.532001495361328,
      "activations/layer1_attention_weight_min": -18.48615074157715,
      "activations/layer20_attention_weight_max": 25.10002326965332,
      "activations/layer20_attention_weight_min": -24.420469284057617,
      "activations/layer21_attention_weight_max": 35.12800598144531,
      "activations/layer21_attention_weight_min": -22.2072811126709,
      "activations/layer22_attention_weight_max": 32.03559112548828,
      "activations/layer22_attention_weight_min": -26.61884307861328,
      "activations/layer23_attention_weight_max": 41.66673278808594,
      "activations/layer23_attention_weight_min": -26.738576889038086,
      "activations/layer2_attention_weight_max": 31.945899963378906,
      "activations/layer2_attention_weight_min": -31.163318634033203,
      "activations/layer3_attention_weight_max": 96.39669036865234,
      "activations/layer3_attention_weight_min": -102.80050659179688,
      "activations/layer4_attention_weight_max": 97.61040496826172,
      "activations/layer4_attention_weight_min": -95.51152038574219,
      "activations/layer5_attention_weight_max": 72.21820831298828,
      "activations/layer5_attention_weight_min": -79.15150451660156,
      "activations/layer6_attention_weight_max": 50.08778762817383,
      "activations/layer6_attention_weight_min": -55.93232727050781,
      "activations/layer7_attention_weight_max": 73.6895980834961,
      "activations/layer7_attention_weight_min": -67.62274169921875,
      "activations/layer8_attention_weight_max": 46.02593231201172,
      "activations/layer8_attention_weight_min": -50.38995361328125,
      "activations/layer9_attention_weight_max": 56.60114669799805,
      "activations/layer9_attention_weight_min": -46.7598991394043,
      "epoch": 12.95,
      "learning_rate": 6.719204545454544e-05,
      "loss": 2.769,
      "step": 222800
    },
    {
      "activations/layer0_attention_weight_max": 16.10109519958496,
      "activations/layer0_attention_weight_min": -14.433659553527832,
      "activations/layer10_attention_weight_max": 34.67643737792969,
      "activations/layer10_attention_weight_min": -36.15408706665039,
      "activations/layer11_attention_weight_max": 34.28923797607422,
      "activations/layer11_attention_weight_min": -33.28360366821289,
      "activations/layer12_attention_weight_max": 20.60309600830078,
      "activations/layer12_attention_weight_min": -26.205053329467773,
      "activations/layer13_attention_weight_max": 48.55387496948242,
      "activations/layer13_attention_weight_min": -31.99551010131836,
      "activations/layer14_attention_weight_max": 56.698204040527344,
      "activations/layer14_attention_weight_min": -31.01248550415039,
      "activations/layer15_attention_weight_max": 48.511043548583984,
      "activations/layer15_attention_weight_min": -30.28877067565918,
      "activations/layer16_attention_weight_max": 35.1977653503418,
      "activations/layer16_attention_weight_min": -31.599271774291992,
      "activations/layer17_attention_weight_max": 59.62861633300781,
      "activations/layer17_attention_weight_min": -47.56638717651367,
      "activations/layer18_attention_weight_max": 55.38691711425781,
      "activations/layer18_attention_weight_min": -43.04591751098633,
      "activations/layer19_attention_weight_max": 29.289457321166992,
      "activations/layer19_attention_weight_min": -25.77699851989746,
      "activations/layer1_attention_weight_max": 17.963401794433594,
      "activations/layer1_attention_weight_min": -16.810728073120117,
      "activations/layer20_attention_weight_max": 29.31954002380371,
      "activations/layer20_attention_weight_min": -23.568994522094727,
      "activations/layer21_attention_weight_max": 49.49253845214844,
      "activations/layer21_attention_weight_min": -24.515344619750977,
      "activations/layer22_attention_weight_max": 37.22443389892578,
      "activations/layer22_attention_weight_min": -27.63558006286621,
      "activations/layer23_attention_weight_max": 49.012779235839844,
      "activations/layer23_attention_weight_min": -22.613515853881836,
      "activations/layer2_attention_weight_max": 34.81843566894531,
      "activations/layer2_attention_weight_min": -32.214622497558594,
      "activations/layer3_attention_weight_max": 101.78469848632812,
      "activations/layer3_attention_weight_min": -97.26277923583984,
      "activations/layer4_attention_weight_max": 99.26399993896484,
      "activations/layer4_attention_weight_min": -96.52306365966797,
      "activations/layer5_attention_weight_max": 70.54076385498047,
      "activations/layer5_attention_weight_min": -77.77154541015625,
      "activations/layer6_attention_weight_max": 50.65745544433594,
      "activations/layer6_attention_weight_min": -50.74239730834961,
      "activations/layer7_attention_weight_max": 66.97260284423828,
      "activations/layer7_attention_weight_min": -66.60098266601562,
      "activations/layer8_attention_weight_max": 46.2607307434082,
      "activations/layer8_attention_weight_min": -49.24464416503906,
      "activations/layer9_attention_weight_max": 48.419490814208984,
      "activations/layer9_attention_weight_min": -49.28364944458008,
      "epoch": 12.95,
      "learning_rate": 6.717310606060606e-05,
      "loss": 2.7704,
      "step": 222850
    },
    {
      "activations/layer0_attention_weight_max": 16.354999542236328,
      "activations/layer0_attention_weight_min": -14.415891647338867,
      "activations/layer10_attention_weight_max": 34.40957260131836,
      "activations/layer10_attention_weight_min": -36.41106033325195,
      "activations/layer11_attention_weight_max": 31.812528610229492,
      "activations/layer11_attention_weight_min": -32.549537658691406,
      "activations/layer12_attention_weight_max": 17.4407901763916,
      "activations/layer12_attention_weight_min": -25.509164810180664,
      "activations/layer13_attention_weight_max": 32.55727005004883,
      "activations/layer13_attention_weight_min": -28.33282470703125,
      "activations/layer14_attention_weight_max": 36.19962692260742,
      "activations/layer14_attention_weight_min": -29.246931076049805,
      "activations/layer15_attention_weight_max": 31.1177921295166,
      "activations/layer15_attention_weight_min": -29.665603637695312,
      "activations/layer16_attention_weight_max": 29.795995712280273,
      "activations/layer16_attention_weight_min": -28.645647048950195,
      "activations/layer17_attention_weight_max": 47.515106201171875,
      "activations/layer17_attention_weight_min": -43.00496292114258,
      "activations/layer18_attention_weight_max": 42.92074966430664,
      "activations/layer18_attention_weight_min": -35.73176956176758,
      "activations/layer19_attention_weight_max": 22.515745162963867,
      "activations/layer19_attention_weight_min": -23.214956283569336,
      "activations/layer1_attention_weight_max": 17.484058380126953,
      "activations/layer1_attention_weight_min": -14.7467679977417,
      "activations/layer20_attention_weight_max": 21.155542373657227,
      "activations/layer20_attention_weight_min": -21.59177017211914,
      "activations/layer21_attention_weight_max": 36.50332260131836,
      "activations/layer21_attention_weight_min": -22.499099731445312,
      "activations/layer22_attention_weight_max": 29.00436782836914,
      "activations/layer22_attention_weight_min": -25.77007293701172,
      "activations/layer23_attention_weight_max": 36.30034255981445,
      "activations/layer23_attention_weight_min": -23.25714874267578,
      "activations/layer2_attention_weight_max": 31.95197868347168,
      "activations/layer2_attention_weight_min": -30.65337371826172,
      "activations/layer3_attention_weight_max": 99.22174072265625,
      "activations/layer3_attention_weight_min": -98.69712829589844,
      "activations/layer4_attention_weight_max": 94.42500305175781,
      "activations/layer4_attention_weight_min": -98.53655242919922,
      "activations/layer5_attention_weight_max": 68.23157501220703,
      "activations/layer5_attention_weight_min": -79.81124114990234,
      "activations/layer6_attention_weight_max": 50.774600982666016,
      "activations/layer6_attention_weight_min": -52.21584701538086,
      "activations/layer7_attention_weight_max": 66.35018157958984,
      "activations/layer7_attention_weight_min": -68.0394058227539,
      "activations/layer8_attention_weight_max": 44.41630935668945,
      "activations/layer8_attention_weight_min": -47.71799850463867,
      "activations/layer9_attention_weight_max": 47.70589065551758,
      "activations/layer9_attention_weight_min": -46.75938034057617,
      "epoch": 12.95,
      "learning_rate": 6.715416666666666e-05,
      "loss": 2.7901,
      "step": 222900
    },
    {
      "activations/layer0_attention_weight_max": 16.38907814025879,
      "activations/layer0_attention_weight_min": -14.722050666809082,
      "activations/layer10_attention_weight_max": 43.96171951293945,
      "activations/layer10_attention_weight_min": -42.942569732666016,
      "activations/layer11_attention_weight_max": 40.31379699707031,
      "activations/layer11_attention_weight_min": -37.48628234863281,
      "activations/layer12_attention_weight_max": 20.364953994750977,
      "activations/layer12_attention_weight_min": -26.623029708862305,
      "activations/layer13_attention_weight_max": 68.64724731445312,
      "activations/layer13_attention_weight_min": -36.04556655883789,
      "activations/layer14_attention_weight_max": 71.46453094482422,
      "activations/layer14_attention_weight_min": -34.75813674926758,
      "activations/layer15_attention_weight_max": 59.83725357055664,
      "activations/layer15_attention_weight_min": -29.76911735534668,
      "activations/layer16_attention_weight_max": 38.60396957397461,
      "activations/layer16_attention_weight_min": -32.48165512084961,
      "activations/layer17_attention_weight_max": 61.85139465332031,
      "activations/layer17_attention_weight_min": -50.8719367980957,
      "activations/layer18_attention_weight_max": 54.57964324951172,
      "activations/layer18_attention_weight_min": -42.94650650024414,
      "activations/layer19_attention_weight_max": 25.323740005493164,
      "activations/layer19_attention_weight_min": -24.090864181518555,
      "activations/layer1_attention_weight_max": 17.156442642211914,
      "activations/layer1_attention_weight_min": -16.346656799316406,
      "activations/layer20_attention_weight_max": 26.76416015625,
      "activations/layer20_attention_weight_min": -21.598176956176758,
      "activations/layer21_attention_weight_max": 52.149559020996094,
      "activations/layer21_attention_weight_min": -24.690555572509766,
      "activations/layer22_attention_weight_max": 36.606510162353516,
      "activations/layer22_attention_weight_min": -26.201387405395508,
      "activations/layer23_attention_weight_max": 40.489891052246094,
      "activations/layer23_attention_weight_min": -23.65114402770996,
      "activations/layer2_attention_weight_max": 34.77415466308594,
      "activations/layer2_attention_weight_min": -35.008338928222656,
      "activations/layer3_attention_weight_max": 102.9632568359375,
      "activations/layer3_attention_weight_min": -102.76831817626953,
      "activations/layer4_attention_weight_max": 102.68773651123047,
      "activations/layer4_attention_weight_min": -100.08441162109375,
      "activations/layer5_attention_weight_max": 78.10025024414062,
      "activations/layer5_attention_weight_min": -74.27755737304688,
      "activations/layer6_attention_weight_max": 63.6290397644043,
      "activations/layer6_attention_weight_min": -59.11204147338867,
      "activations/layer7_attention_weight_max": 75.93759155273438,
      "activations/layer7_attention_weight_min": -76.71755981445312,
      "activations/layer8_attention_weight_max": 58.0550537109375,
      "activations/layer8_attention_weight_min": -55.46568298339844,
      "activations/layer9_attention_weight_max": 57.24452590942383,
      "activations/layer9_attention_weight_min": -54.594703674316406,
      "epoch": 12.95,
      "learning_rate": 6.713522727272726e-05,
      "loss": 2.7664,
      "step": 222950
    },
    {
      "activations/layer0_attention_weight_max": 15.931107521057129,
      "activations/layer0_attention_weight_min": -14.591607093811035,
      "activations/layer10_attention_weight_max": 42.45741653442383,
      "activations/layer10_attention_weight_min": -41.71051788330078,
      "activations/layer11_attention_weight_max": 38.452274322509766,
      "activations/layer11_attention_weight_min": -36.88257598876953,
      "activations/layer12_attention_weight_max": 18.77168846130371,
      "activations/layer12_attention_weight_min": -23.063968658447266,
      "activations/layer13_attention_weight_max": 48.56038284301758,
      "activations/layer13_attention_weight_min": -31.49734115600586,
      "activations/layer14_attention_weight_max": 56.14229965209961,
      "activations/layer14_attention_weight_min": -35.39517593383789,
      "activations/layer15_attention_weight_max": 52.88236999511719,
      "activations/layer15_attention_weight_min": -32.125465393066406,
      "activations/layer16_attention_weight_max": 40.5059928894043,
      "activations/layer16_attention_weight_min": -32.69332504272461,
      "activations/layer17_attention_weight_max": 57.6807861328125,
      "activations/layer17_attention_weight_min": -50.243247985839844,
      "activations/layer18_attention_weight_max": 53.982120513916016,
      "activations/layer18_attention_weight_min": -43.587249755859375,
      "activations/layer19_attention_weight_max": 29.91054344177246,
      "activations/layer19_attention_weight_min": -22.626161575317383,
      "activations/layer1_attention_weight_max": 16.72588539123535,
      "activations/layer1_attention_weight_min": -15.088239669799805,
      "activations/layer20_attention_weight_max": 26.84739875793457,
      "activations/layer20_attention_weight_min": -22.601354598999023,
      "activations/layer21_attention_weight_max": 40.34436798095703,
      "activations/layer21_attention_weight_min": -24.9141788482666,
      "activations/layer22_attention_weight_max": 32.94721221923828,
      "activations/layer22_attention_weight_min": -25.065982818603516,
      "activations/layer23_attention_weight_max": 38.798500061035156,
      "activations/layer23_attention_weight_min": -22.233604431152344,
      "activations/layer2_attention_weight_max": 35.08100128173828,
      "activations/layer2_attention_weight_min": -33.529380798339844,
      "activations/layer3_attention_weight_max": 99.43759155273438,
      "activations/layer3_attention_weight_min": -100.9867935180664,
      "activations/layer4_attention_weight_max": 97.96794128417969,
      "activations/layer4_attention_weight_min": -98.0190658569336,
      "activations/layer5_attention_weight_max": 79.14601135253906,
      "activations/layer5_attention_weight_min": -80.90218353271484,
      "activations/layer6_attention_weight_max": 57.22906494140625,
      "activations/layer6_attention_weight_min": -57.060943603515625,
      "activations/layer7_attention_weight_max": 80.65880584716797,
      "activations/layer7_attention_weight_min": -70.5403060913086,
      "activations/layer8_attention_weight_max": 55.56540298461914,
      "activations/layer8_attention_weight_min": -52.90130615234375,
      "activations/layer9_attention_weight_max": 60.3476676940918,
      "activations/layer9_attention_weight_min": -49.497440338134766,
      "epoch": 12.96,
      "learning_rate": 6.711628787878786e-05,
      "loss": 2.7576,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.495,
      "eval_samples_per_second": 505.474,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.495,
      "eval_openwebtext_samples_per_second": 505.474,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_wikitext_loss": 2.95703125,
      "eval_wikitext_ppl": 19.24076585887165,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.287,
      "step": 223000
    },
    {
      "epoch": 12.96,
      "eval_lambada_loss": 2.63671875,
      "eval_lambada_ppl": 13.96729813824377,
      "eval_lambada_runtime": 9.5319,
      "eval_lambada_samples_per_second": 510.814,
      "step": 223000
    },
    {
      "activations/layer0_attention_weight_max": 15.874995231628418,
      "activations/layer0_attention_weight_min": -14.661575317382812,
      "activations/layer10_attention_weight_max": 35.300148010253906,
      "activations/layer10_attention_weight_min": -33.78459167480469,
      "activations/layer11_attention_weight_max": 29.527263641357422,
      "activations/layer11_attention_weight_min": -31.95755386352539,
      "activations/layer12_attention_weight_max": 17.431055068969727,
      "activations/layer12_attention_weight_min": -28.475065231323242,
      "activations/layer13_attention_weight_max": 32.99386215209961,
      "activations/layer13_attention_weight_min": -26.92156219482422,
      "activations/layer14_attention_weight_max": 34.895042419433594,
      "activations/layer14_attention_weight_min": -28.60478401184082,
      "activations/layer15_attention_weight_max": 31.93708038330078,
      "activations/layer15_attention_weight_min": -29.435199737548828,
      "activations/layer16_attention_weight_max": 29.673288345336914,
      "activations/layer16_attention_weight_min": -28.39703941345215,
      "activations/layer17_attention_weight_max": 52.4257926940918,
      "activations/layer17_attention_weight_min": -42.28391647338867,
      "activations/layer18_attention_weight_max": 41.690399169921875,
      "activations/layer18_attention_weight_min": -37.20663070678711,
      "activations/layer19_attention_weight_max": 22.149372100830078,
      "activations/layer19_attention_weight_min": -21.976024627685547,
      "activations/layer1_attention_weight_max": 17.092205047607422,
      "activations/layer1_attention_weight_min": -14.51037311553955,
      "activations/layer20_attention_weight_max": 21.709056854248047,
      "activations/layer20_attention_weight_min": -21.37479019165039,
      "activations/layer21_attention_weight_max": 34.27295684814453,
      "activations/layer21_attention_weight_min": -23.997434616088867,
      "activations/layer22_attention_weight_max": 29.15557861328125,
      "activations/layer22_attention_weight_min": -25.39864158630371,
      "activations/layer23_attention_weight_max": 33.17912292480469,
      "activations/layer23_attention_weight_min": -26.75275993347168,
      "activations/layer2_attention_weight_max": 32.458309173583984,
      "activations/layer2_attention_weight_min": -31.009410858154297,
      "activations/layer3_attention_weight_max": 95.4293212890625,
      "activations/layer3_attention_weight_min": -95.47162628173828,
      "activations/layer4_attention_weight_max": 95.30561065673828,
      "activations/layer4_attention_weight_min": -92.52865600585938,
      "activations/layer5_attention_weight_max": 74.49124908447266,
      "activations/layer5_attention_weight_min": -75.8011474609375,
      "activations/layer6_attention_weight_max": 52.0292854309082,
      "activations/layer6_attention_weight_min": -54.088584899902344,
      "activations/layer7_attention_weight_max": 65.68305969238281,
      "activations/layer7_attention_weight_min": -66.3918685913086,
      "activations/layer8_attention_weight_max": 43.10520553588867,
      "activations/layer8_attention_weight_min": -49.252296447753906,
      "activations/layer9_attention_weight_max": 41.66212844848633,
      "activations/layer9_attention_weight_min": -48.26182556152344,
      "epoch": 12.96,
      "learning_rate": 6.709734848484848e-05,
      "loss": 2.7779,
      "step": 223050
    },
    {
      "activations/layer0_attention_weight_max": 15.986248970031738,
      "activations/layer0_attention_weight_min": -14.371844291687012,
      "activations/layer10_attention_weight_max": 34.583656311035156,
      "activations/layer10_attention_weight_min": -35.73268508911133,
      "activations/layer11_attention_weight_max": 31.75790786743164,
      "activations/layer11_attention_weight_min": -34.138633728027344,
      "activations/layer12_attention_weight_max": 19.490402221679688,
      "activations/layer12_attention_weight_min": -29.060266494750977,
      "activations/layer13_attention_weight_max": 35.31313705444336,
      "activations/layer13_attention_weight_min": -30.242389678955078,
      "activations/layer14_attention_weight_max": 34.69824981689453,
      "activations/layer14_attention_weight_min": -30.57568359375,
      "activations/layer15_attention_weight_max": 30.9825439453125,
      "activations/layer15_attention_weight_min": -28.84380531311035,
      "activations/layer16_attention_weight_max": 33.3235969543457,
      "activations/layer16_attention_weight_min": -31.224267959594727,
      "activations/layer17_attention_weight_max": 51.002471923828125,
      "activations/layer17_attention_weight_min": -47.42074203491211,
      "activations/layer18_attention_weight_max": 44.299346923828125,
      "activations/layer18_attention_weight_min": -44.05987548828125,
      "activations/layer19_attention_weight_max": 23.097421646118164,
      "activations/layer19_attention_weight_min": -26.962839126586914,
      "activations/layer1_attention_weight_max": 16.48873519897461,
      "activations/layer1_attention_weight_min": -15.59830093383789,
      "activations/layer20_attention_weight_max": 22.333507537841797,
      "activations/layer20_attention_weight_min": -25.507827758789062,
      "activations/layer21_attention_weight_max": 34.178443908691406,
      "activations/layer21_attention_weight_min": -29.293533325195312,
      "activations/layer22_attention_weight_max": 32.44369125366211,
      "activations/layer22_attention_weight_min": -24.47696304321289,
      "activations/layer23_attention_weight_max": 35.895790100097656,
      "activations/layer23_attention_weight_min": -25.247554779052734,
      "activations/layer2_attention_weight_max": 33.985111236572266,
      "activations/layer2_attention_weight_min": -30.782583236694336,
      "activations/layer3_attention_weight_max": 99.1017837524414,
      "activations/layer3_attention_weight_min": -97.68901062011719,
      "activations/layer4_attention_weight_max": 103.03633880615234,
      "activations/layer4_attention_weight_min": -96.28810119628906,
      "activations/layer5_attention_weight_max": 76.46260070800781,
      "activations/layer5_attention_weight_min": -81.96470642089844,
      "activations/layer6_attention_weight_max": 53.81665802001953,
      "activations/layer6_attention_weight_min": -52.77530288696289,
      "activations/layer7_attention_weight_max": 75.11724090576172,
      "activations/layer7_attention_weight_min": -66.65480041503906,
      "activations/layer8_attention_weight_max": 49.63569259643555,
      "activations/layer8_attention_weight_min": -49.52386474609375,
      "activations/layer9_attention_weight_max": 46.50833511352539,
      "activations/layer9_attention_weight_min": -49.54459762573242,
      "epoch": 12.96,
      "learning_rate": 6.707878787878787e-05,
      "loss": 2.7809,
      "step": 223100
    },
    {
      "activations/layer0_attention_weight_max": 17.373395919799805,
      "activations/layer0_attention_weight_min": -14.519580841064453,
      "activations/layer10_attention_weight_max": 36.42124938964844,
      "activations/layer10_attention_weight_min": -34.922813415527344,
      "activations/layer11_attention_weight_max": 31.858169555664062,
      "activations/layer11_attention_weight_min": -35.6708984375,
      "activations/layer12_attention_weight_max": 19.441781997680664,
      "activations/layer12_attention_weight_min": -24.51927947998047,
      "activations/layer13_attention_weight_max": 37.42524337768555,
      "activations/layer13_attention_weight_min": -28.597896575927734,
      "activations/layer14_attention_weight_max": 36.18135452270508,
      "activations/layer14_attention_weight_min": -30.379194259643555,
      "activations/layer15_attention_weight_max": 32.94766616821289,
      "activations/layer15_attention_weight_min": -29.858198165893555,
      "activations/layer16_attention_weight_max": 30.685256958007812,
      "activations/layer16_attention_weight_min": -31.274255752563477,
      "activations/layer17_attention_weight_max": 52.94733810424805,
      "activations/layer17_attention_weight_min": -45.5093994140625,
      "activations/layer18_attention_weight_max": 46.87439727783203,
      "activations/layer18_attention_weight_min": -39.22412109375,
      "activations/layer19_attention_weight_max": 23.959959030151367,
      "activations/layer19_attention_weight_min": -22.770038604736328,
      "activations/layer1_attention_weight_max": 16.172988891601562,
      "activations/layer1_attention_weight_min": -15.463252067565918,
      "activations/layer20_attention_weight_max": 21.221967697143555,
      "activations/layer20_attention_weight_min": -21.27812957763672,
      "activations/layer21_attention_weight_max": 34.629119873046875,
      "activations/layer21_attention_weight_min": -23.294815063476562,
      "activations/layer22_attention_weight_max": 31.914522171020508,
      "activations/layer22_attention_weight_min": -25.489757537841797,
      "activations/layer23_attention_weight_max": 37.454734802246094,
      "activations/layer23_attention_weight_min": -27.47121810913086,
      "activations/layer2_attention_weight_max": 30.938087463378906,
      "activations/layer2_attention_weight_min": -29.553325653076172,
      "activations/layer3_attention_weight_max": 92.00274658203125,
      "activations/layer3_attention_weight_min": -90.05099487304688,
      "activations/layer4_attention_weight_max": 93.47649383544922,
      "activations/layer4_attention_weight_min": -91.01202392578125,
      "activations/layer5_attention_weight_max": 72.80119323730469,
      "activations/layer5_attention_weight_min": -75.40697479248047,
      "activations/layer6_attention_weight_max": 50.06096267700195,
      "activations/layer6_attention_weight_min": -50.800941467285156,
      "activations/layer7_attention_weight_max": 68.38372802734375,
      "activations/layer7_attention_weight_min": -66.57183074951172,
      "activations/layer8_attention_weight_max": 44.762821197509766,
      "activations/layer8_attention_weight_min": -47.945194244384766,
      "activations/layer9_attention_weight_max": 46.555091857910156,
      "activations/layer9_attention_weight_min": -48.750144958496094,
      "epoch": 12.97,
      "learning_rate": 6.705984848484847e-05,
      "loss": 2.7756,
      "step": 223150
    },
    {
      "activations/layer0_attention_weight_max": 15.828275680541992,
      "activations/layer0_attention_weight_min": -14.304903030395508,
      "activations/layer10_attention_weight_max": 47.9888801574707,
      "activations/layer10_attention_weight_min": -47.21458435058594,
      "activations/layer11_attention_weight_max": 39.614227294921875,
      "activations/layer11_attention_weight_min": -38.85649108886719,
      "activations/layer12_attention_weight_max": 20.67843246459961,
      "activations/layer12_attention_weight_min": -28.546388626098633,
      "activations/layer13_attention_weight_max": 40.80516052246094,
      "activations/layer13_attention_weight_min": -30.687143325805664,
      "activations/layer14_attention_weight_max": 39.04273223876953,
      "activations/layer14_attention_weight_min": -34.578243255615234,
      "activations/layer15_attention_weight_max": 36.656734466552734,
      "activations/layer15_attention_weight_min": -33.92918395996094,
      "activations/layer16_attention_weight_max": 34.67033386230469,
      "activations/layer16_attention_weight_min": -32.923683166503906,
      "activations/layer17_attention_weight_max": 54.71875,
      "activations/layer17_attention_weight_min": -48.50657272338867,
      "activations/layer18_attention_weight_max": 48.84096908569336,
      "activations/layer18_attention_weight_min": -40.78007507324219,
      "activations/layer19_attention_weight_max": 23.470809936523438,
      "activations/layer19_attention_weight_min": -22.74857521057129,
      "activations/layer1_attention_weight_max": 19.10381507873535,
      "activations/layer1_attention_weight_min": -15.11414909362793,
      "activations/layer20_attention_weight_max": 23.622209548950195,
      "activations/layer20_attention_weight_min": -22.792131423950195,
      "activations/layer21_attention_weight_max": 38.72937774658203,
      "activations/layer21_attention_weight_min": -30.384525299072266,
      "activations/layer22_attention_weight_max": 30.75258445739746,
      "activations/layer22_attention_weight_min": -26.12324333190918,
      "activations/layer23_attention_weight_max": 40.76601028442383,
      "activations/layer23_attention_weight_min": -24.622634887695312,
      "activations/layer2_attention_weight_max": 35.963409423828125,
      "activations/layer2_attention_weight_min": -33.4090576171875,
      "activations/layer3_attention_weight_max": 94.97309112548828,
      "activations/layer3_attention_weight_min": -99.19584655761719,
      "activations/layer4_attention_weight_max": 107.99104309082031,
      "activations/layer4_attention_weight_min": -103.00676727294922,
      "activations/layer5_attention_weight_max": 75.62811279296875,
      "activations/layer5_attention_weight_min": -81.41395568847656,
      "activations/layer6_attention_weight_max": 55.383544921875,
      "activations/layer6_attention_weight_min": -56.64350509643555,
      "activations/layer7_attention_weight_max": 82.19015502929688,
      "activations/layer7_attention_weight_min": -71.76539611816406,
      "activations/layer8_attention_weight_max": 56.77935028076172,
      "activations/layer8_attention_weight_min": -51.13636016845703,
      "activations/layer9_attention_weight_max": 63.46851348876953,
      "activations/layer9_attention_weight_min": -52.682098388671875,
      "epoch": 12.97,
      "learning_rate": 6.704090909090909e-05,
      "loss": 2.7685,
      "step": 223200
    },
    {
      "activations/layer0_attention_weight_max": 15.741921424865723,
      "activations/layer0_attention_weight_min": -13.574832916259766,
      "activations/layer10_attention_weight_max": 35.82598114013672,
      "activations/layer10_attention_weight_min": -35.372718811035156,
      "activations/layer11_attention_weight_max": 31.049266815185547,
      "activations/layer11_attention_weight_min": -33.224525451660156,
      "activations/layer12_attention_weight_max": 18.169189453125,
      "activations/layer12_attention_weight_min": -23.052631378173828,
      "activations/layer13_attention_weight_max": 35.8240966796875,
      "activations/layer13_attention_weight_min": -28.549259185791016,
      "activations/layer14_attention_weight_max": 35.584896087646484,
      "activations/layer14_attention_weight_min": -30.34791374206543,
      "activations/layer15_attention_weight_max": 33.160823822021484,
      "activations/layer15_attention_weight_min": -30.47442054748535,
      "activations/layer16_attention_weight_max": 31.144481658935547,
      "activations/layer16_attention_weight_min": -29.89348602294922,
      "activations/layer17_attention_weight_max": 55.874969482421875,
      "activations/layer17_attention_weight_min": -47.43196105957031,
      "activations/layer18_attention_weight_max": 47.722755432128906,
      "activations/layer18_attention_weight_min": -39.83141326904297,
      "activations/layer19_attention_weight_max": 26.90802574157715,
      "activations/layer19_attention_weight_min": -21.521699905395508,
      "activations/layer1_attention_weight_max": 18.105262756347656,
      "activations/layer1_attention_weight_min": -14.282750129699707,
      "activations/layer20_attention_weight_max": 23.543867111206055,
      "activations/layer20_attention_weight_min": -21.22081756591797,
      "activations/layer21_attention_weight_max": 41.38523483276367,
      "activations/layer21_attention_weight_min": -25.036474227905273,
      "activations/layer22_attention_weight_max": 30.99909210205078,
      "activations/layer22_attention_weight_min": -24.287691116333008,
      "activations/layer23_attention_weight_max": 37.62623977661133,
      "activations/layer23_attention_weight_min": -24.512378692626953,
      "activations/layer2_attention_weight_max": 30.87154769897461,
      "activations/layer2_attention_weight_min": -29.73308563232422,
      "activations/layer3_attention_weight_max": 97.12871551513672,
      "activations/layer3_attention_weight_min": -101.17444610595703,
      "activations/layer4_attention_weight_max": 97.0057373046875,
      "activations/layer4_attention_weight_min": -94.41007232666016,
      "activations/layer5_attention_weight_max": 72.1253890991211,
      "activations/layer5_attention_weight_min": -75.675048828125,
      "activations/layer6_attention_weight_max": 51.83927536010742,
      "activations/layer6_attention_weight_min": -52.75489044189453,
      "activations/layer7_attention_weight_max": 70.30247497558594,
      "activations/layer7_attention_weight_min": -65.06726837158203,
      "activations/layer8_attention_weight_max": 43.9554328918457,
      "activations/layer8_attention_weight_min": -47.72361373901367,
      "activations/layer9_attention_weight_max": 45.709774017333984,
      "activations/layer9_attention_weight_min": -47.34892654418945,
      "epoch": 12.97,
      "learning_rate": 6.702196969696969e-05,
      "loss": 2.7796,
      "step": 223250
    },
    {
      "activations/layer0_attention_weight_max": 15.760263442993164,
      "activations/layer0_attention_weight_min": -14.085702896118164,
      "activations/layer10_attention_weight_max": 37.21240234375,
      "activations/layer10_attention_weight_min": -35.4034309387207,
      "activations/layer11_attention_weight_max": 32.298370361328125,
      "activations/layer11_attention_weight_min": -32.906768798828125,
      "activations/layer12_attention_weight_max": 19.02417755126953,
      "activations/layer12_attention_weight_min": -26.80903434753418,
      "activations/layer13_attention_weight_max": 41.738319396972656,
      "activations/layer13_attention_weight_min": -29.459590911865234,
      "activations/layer14_attention_weight_max": 39.88771438598633,
      "activations/layer14_attention_weight_min": -30.776033401489258,
      "activations/layer15_attention_weight_max": 38.14863204956055,
      "activations/layer15_attention_weight_min": -30.715635299682617,
      "activations/layer16_attention_weight_max": 33.18095016479492,
      "activations/layer16_attention_weight_min": -30.989078521728516,
      "activations/layer17_attention_weight_max": 55.25294876098633,
      "activations/layer17_attention_weight_min": -43.30959701538086,
      "activations/layer18_attention_weight_max": 48.90210723876953,
      "activations/layer18_attention_weight_min": -37.06795883178711,
      "activations/layer19_attention_weight_max": 23.68964385986328,
      "activations/layer19_attention_weight_min": -24.019615173339844,
      "activations/layer1_attention_weight_max": 16.539236068725586,
      "activations/layer1_attention_weight_min": -15.51483154296875,
      "activations/layer20_attention_weight_max": 23.922832489013672,
      "activations/layer20_attention_weight_min": -21.75017547607422,
      "activations/layer21_attention_weight_max": 39.69068145751953,
      "activations/layer21_attention_weight_min": -23.90610122680664,
      "activations/layer22_attention_weight_max": 31.019142150878906,
      "activations/layer22_attention_weight_min": -26.263404846191406,
      "activations/layer23_attention_weight_max": 42.2598876953125,
      "activations/layer23_attention_weight_min": -24.330825805664062,
      "activations/layer2_attention_weight_max": 31.391021728515625,
      "activations/layer2_attention_weight_min": -30.156604766845703,
      "activations/layer3_attention_weight_max": 95.89532470703125,
      "activations/layer3_attention_weight_min": -93.74478912353516,
      "activations/layer4_attention_weight_max": 97.87718963623047,
      "activations/layer4_attention_weight_min": -95.19416809082031,
      "activations/layer5_attention_weight_max": 73.73828125,
      "activations/layer5_attention_weight_min": -78.92219543457031,
      "activations/layer6_attention_weight_max": 51.34851837158203,
      "activations/layer6_attention_weight_min": -53.12397384643555,
      "activations/layer7_attention_weight_max": 70.66231536865234,
      "activations/layer7_attention_weight_min": -70.68441772460938,
      "activations/layer8_attention_weight_max": 45.74567413330078,
      "activations/layer8_attention_weight_min": -48.7856330871582,
      "activations/layer9_attention_weight_max": 49.47026824951172,
      "activations/layer9_attention_weight_min": -50.007240295410156,
      "epoch": 12.97,
      "learning_rate": 6.70030303030303e-05,
      "loss": 2.7793,
      "step": 223300
    },
    {
      "activations/layer0_attention_weight_max": 16.550764083862305,
      "activations/layer0_attention_weight_min": -14.78565502166748,
      "activations/layer10_attention_weight_max": 37.74506378173828,
      "activations/layer10_attention_weight_min": -36.7610969543457,
      "activations/layer11_attention_weight_max": 31.856067657470703,
      "activations/layer11_attention_weight_min": -32.430599212646484,
      "activations/layer12_attention_weight_max": 17.646596908569336,
      "activations/layer12_attention_weight_min": -24.988073348999023,
      "activations/layer13_attention_weight_max": 35.221553802490234,
      "activations/layer13_attention_weight_min": -29.298503875732422,
      "activations/layer14_attention_weight_max": 37.3391227722168,
      "activations/layer14_attention_weight_min": -30.92567253112793,
      "activations/layer15_attention_weight_max": 33.710323333740234,
      "activations/layer15_attention_weight_min": -29.502178192138672,
      "activations/layer16_attention_weight_max": 31.561250686645508,
      "activations/layer16_attention_weight_min": -30.812015533447266,
      "activations/layer17_attention_weight_max": 50.788455963134766,
      "activations/layer17_attention_weight_min": -42.264591217041016,
      "activations/layer18_attention_weight_max": 46.08348846435547,
      "activations/layer18_attention_weight_min": -36.795597076416016,
      "activations/layer19_attention_weight_max": 20.557281494140625,
      "activations/layer19_attention_weight_min": -20.097814559936523,
      "activations/layer1_attention_weight_max": 17.26241683959961,
      "activations/layer1_attention_weight_min": -14.057943344116211,
      "activations/layer20_attention_weight_max": 20.362430572509766,
      "activations/layer20_attention_weight_min": -19.932771682739258,
      "activations/layer21_attention_weight_max": 33.241943359375,
      "activations/layer21_attention_weight_min": -22.664026260375977,
      "activations/layer22_attention_weight_max": 29.16729736328125,
      "activations/layer22_attention_weight_min": -24.31494140625,
      "activations/layer23_attention_weight_max": 35.197025299072266,
      "activations/layer23_attention_weight_min": -24.429779052734375,
      "activations/layer2_attention_weight_max": 31.866209030151367,
      "activations/layer2_attention_weight_min": -30.12078094482422,
      "activations/layer3_attention_weight_max": 93.75360107421875,
      "activations/layer3_attention_weight_min": -96.03219604492188,
      "activations/layer4_attention_weight_max": 98.51842498779297,
      "activations/layer4_attention_weight_min": -93.12884521484375,
      "activations/layer5_attention_weight_max": 74.23210144042969,
      "activations/layer5_attention_weight_min": -81.4209213256836,
      "activations/layer6_attention_weight_max": 53.20589828491211,
      "activations/layer6_attention_weight_min": -52.27836608886719,
      "activations/layer7_attention_weight_max": 72.74890899658203,
      "activations/layer7_attention_weight_min": -68.1628646850586,
      "activations/layer8_attention_weight_max": 44.81498336791992,
      "activations/layer8_attention_weight_min": -48.03137969970703,
      "activations/layer9_attention_weight_max": 45.627410888671875,
      "activations/layer9_attention_weight_min": -46.735897064208984,
      "epoch": 12.98,
      "learning_rate": 6.69844696969697e-05,
      "loss": 2.7763,
      "step": 223350
    },
    {
      "activations/layer0_attention_weight_max": 16.40268898010254,
      "activations/layer0_attention_weight_min": -14.630154609680176,
      "activations/layer10_attention_weight_max": 31.87540626525879,
      "activations/layer10_attention_weight_min": -34.655906677246094,
      "activations/layer11_attention_weight_max": 29.97040557861328,
      "activations/layer11_attention_weight_min": -32.49256896972656,
      "activations/layer12_attention_weight_max": 17.3107852935791,
      "activations/layer12_attention_weight_min": -28.054292678833008,
      "activations/layer13_attention_weight_max": 34.53099822998047,
      "activations/layer13_attention_weight_min": -30.83819580078125,
      "activations/layer14_attention_weight_max": 30.978973388671875,
      "activations/layer14_attention_weight_min": -29.831951141357422,
      "activations/layer15_attention_weight_max": 29.36026954650879,
      "activations/layer15_attention_weight_min": -29.101261138916016,
      "activations/layer16_attention_weight_max": 26.719423294067383,
      "activations/layer16_attention_weight_min": -27.049034118652344,
      "activations/layer17_attention_weight_max": 45.70710754394531,
      "activations/layer17_attention_weight_min": -39.77597427368164,
      "activations/layer18_attention_weight_max": 42.41497039794922,
      "activations/layer18_attention_weight_min": -34.1084098815918,
      "activations/layer19_attention_weight_max": 19.65869903564453,
      "activations/layer19_attention_weight_min": -19.884183883666992,
      "activations/layer1_attention_weight_max": 19.412334442138672,
      "activations/layer1_attention_weight_min": -15.074671745300293,
      "activations/layer20_attention_weight_max": 18.50699234008789,
      "activations/layer20_attention_weight_min": -20.185094833374023,
      "activations/layer21_attention_weight_max": 30.37396240234375,
      "activations/layer21_attention_weight_min": -25.441787719726562,
      "activations/layer22_attention_weight_max": 28.6373348236084,
      "activations/layer22_attention_weight_min": -25.370195388793945,
      "activations/layer23_attention_weight_max": 34.449798583984375,
      "activations/layer23_attention_weight_min": -26.333959579467773,
      "activations/layer2_attention_weight_max": 30.682035446166992,
      "activations/layer2_attention_weight_min": -28.109153747558594,
      "activations/layer3_attention_weight_max": 86.86532592773438,
      "activations/layer3_attention_weight_min": -86.91038513183594,
      "activations/layer4_attention_weight_max": 87.03998565673828,
      "activations/layer4_attention_weight_min": -82.28321075439453,
      "activations/layer5_attention_weight_max": 68.09502410888672,
      "activations/layer5_attention_weight_min": -71.7408218383789,
      "activations/layer6_attention_weight_max": 48.97892761230469,
      "activations/layer6_attention_weight_min": -50.1125373840332,
      "activations/layer7_attention_weight_max": 68.54517364501953,
      "activations/layer7_attention_weight_min": -66.25509643554688,
      "activations/layer8_attention_weight_max": 42.9300422668457,
      "activations/layer8_attention_weight_min": -46.35581970214844,
      "activations/layer9_attention_weight_max": 49.44487762451172,
      "activations/layer9_attention_weight_min": -46.34637451171875,
      "epoch": 12.98,
      "learning_rate": 6.69655303030303e-05,
      "loss": 2.7951,
      "step": 223400
    },
    {
      "activations/layer0_attention_weight_max": 15.971456527709961,
      "activations/layer0_attention_weight_min": -14.366050720214844,
      "activations/layer10_attention_weight_max": 36.24444580078125,
      "activations/layer10_attention_weight_min": -36.6851921081543,
      "activations/layer11_attention_weight_max": 30.14729881286621,
      "activations/layer11_attention_weight_min": -31.353912353515625,
      "activations/layer12_attention_weight_max": 19.271360397338867,
      "activations/layer12_attention_weight_min": -22.66947364807129,
      "activations/layer13_attention_weight_max": 36.192813873291016,
      "activations/layer13_attention_weight_min": -31.447763442993164,
      "activations/layer14_attention_weight_max": 37.776126861572266,
      "activations/layer14_attention_weight_min": -30.773666381835938,
      "activations/layer15_attention_weight_max": 36.0208625793457,
      "activations/layer15_attention_weight_min": -29.270830154418945,
      "activations/layer16_attention_weight_max": 31.74578094482422,
      "activations/layer16_attention_weight_min": -28.95541000366211,
      "activations/layer17_attention_weight_max": 50.073211669921875,
      "activations/layer17_attention_weight_min": -44.02825164794922,
      "activations/layer18_attention_weight_max": 46.14017105102539,
      "activations/layer18_attention_weight_min": -37.3692512512207,
      "activations/layer19_attention_weight_max": 22.689373016357422,
      "activations/layer19_attention_weight_min": -21.768491744995117,
      "activations/layer1_attention_weight_max": 17.22228240966797,
      "activations/layer1_attention_weight_min": -14.654474258422852,
      "activations/layer20_attention_weight_max": 23.088232040405273,
      "activations/layer20_attention_weight_min": -22.099027633666992,
      "activations/layer21_attention_weight_max": 36.76491165161133,
      "activations/layer21_attention_weight_min": -23.660076141357422,
      "activations/layer22_attention_weight_max": 28.1119441986084,
      "activations/layer22_attention_weight_min": -24.58780860900879,
      "activations/layer23_attention_weight_max": 37.17759704589844,
      "activations/layer23_attention_weight_min": -25.033916473388672,
      "activations/layer2_attention_weight_max": 33.067771911621094,
      "activations/layer2_attention_weight_min": -29.78750228881836,
      "activations/layer3_attention_weight_max": 93.54553985595703,
      "activations/layer3_attention_weight_min": -91.04418182373047,
      "activations/layer4_attention_weight_max": 97.64883422851562,
      "activations/layer4_attention_weight_min": -88.57929992675781,
      "activations/layer5_attention_weight_max": 73.5046615600586,
      "activations/layer5_attention_weight_min": -83.04478454589844,
      "activations/layer6_attention_weight_max": 51.46349334716797,
      "activations/layer6_attention_weight_min": -49.34598922729492,
      "activations/layer7_attention_weight_max": 69.72647094726562,
      "activations/layer7_attention_weight_min": -66.00957489013672,
      "activations/layer8_attention_weight_max": 46.08639144897461,
      "activations/layer8_attention_weight_min": -48.24638748168945,
      "activations/layer9_attention_weight_max": 43.31939697265625,
      "activations/layer9_attention_weight_min": -46.114280700683594,
      "epoch": 12.98,
      "learning_rate": 6.694659090909091e-05,
      "loss": 2.7909,
      "step": 223450
    },
    {
      "activations/layer0_attention_weight_max": 15.003035545349121,
      "activations/layer0_attention_weight_min": -14.74459171295166,
      "activations/layer10_attention_weight_max": 36.547996520996094,
      "activations/layer10_attention_weight_min": -35.9007568359375,
      "activations/layer11_attention_weight_max": 33.65654754638672,
      "activations/layer11_attention_weight_min": -34.632965087890625,
      "activations/layer12_attention_weight_max": 17.750268936157227,
      "activations/layer12_attention_weight_min": -25.799060821533203,
      "activations/layer13_attention_weight_max": 38.669769287109375,
      "activations/layer13_attention_weight_min": -28.321014404296875,
      "activations/layer14_attention_weight_max": 36.56239318847656,
      "activations/layer14_attention_weight_min": -29.06717300415039,
      "activations/layer15_attention_weight_max": 32.26618194580078,
      "activations/layer15_attention_weight_min": -31.16557502746582,
      "activations/layer16_attention_weight_max": 33.32955551147461,
      "activations/layer16_attention_weight_min": -31.68912696838379,
      "activations/layer17_attention_weight_max": 57.58182907104492,
      "activations/layer17_attention_weight_min": -47.2483024597168,
      "activations/layer18_attention_weight_max": 47.668121337890625,
      "activations/layer18_attention_weight_min": -40.69108200073242,
      "activations/layer19_attention_weight_max": 23.50935935974121,
      "activations/layer19_attention_weight_min": -22.89035987854004,
      "activations/layer1_attention_weight_max": 18.051755905151367,
      "activations/layer1_attention_weight_min": -16.967344284057617,
      "activations/layer20_attention_weight_max": 21.860807418823242,
      "activations/layer20_attention_weight_min": -20.875133514404297,
      "activations/layer21_attention_weight_max": 35.89260482788086,
      "activations/layer21_attention_weight_min": -23.590164184570312,
      "activations/layer22_attention_weight_max": 30.163673400878906,
      "activations/layer22_attention_weight_min": -26.4953556060791,
      "activations/layer23_attention_weight_max": 38.4815673828125,
      "activations/layer23_attention_weight_min": -22.47822380065918,
      "activations/layer2_attention_weight_max": 31.527427673339844,
      "activations/layer2_attention_weight_min": -30.682151794433594,
      "activations/layer3_attention_weight_max": 96.40894317626953,
      "activations/layer3_attention_weight_min": -100.14579772949219,
      "activations/layer4_attention_weight_max": 90.89173889160156,
      "activations/layer4_attention_weight_min": -88.82158660888672,
      "activations/layer5_attention_weight_max": 73.95036315917969,
      "activations/layer5_attention_weight_min": -77.88803100585938,
      "activations/layer6_attention_weight_max": 50.30929946899414,
      "activations/layer6_attention_weight_min": -50.18942642211914,
      "activations/layer7_attention_weight_max": 68.90755462646484,
      "activations/layer7_attention_weight_min": -67.64700317382812,
      "activations/layer8_attention_weight_max": 44.9305305480957,
      "activations/layer8_attention_weight_min": -47.10710525512695,
      "activations/layer9_attention_weight_max": 46.9213981628418,
      "activations/layer9_attention_weight_min": -46.16815185546875,
      "epoch": 12.99,
      "learning_rate": 6.692765151515151e-05,
      "loss": 2.7679,
      "step": 223500
    },
    {
      "activations/layer0_attention_weight_max": 15.320796966552734,
      "activations/layer0_attention_weight_min": -14.019651412963867,
      "activations/layer10_attention_weight_max": 37.156944274902344,
      "activations/layer10_attention_weight_min": -35.66428756713867,
      "activations/layer11_attention_weight_max": 31.979366302490234,
      "activations/layer11_attention_weight_min": -31.82990074157715,
      "activations/layer12_attention_weight_max": 17.841655731201172,
      "activations/layer12_attention_weight_min": -24.72012710571289,
      "activations/layer13_attention_weight_max": 36.3830680847168,
      "activations/layer13_attention_weight_min": -28.6867733001709,
      "activations/layer14_attention_weight_max": 35.555625915527344,
      "activations/layer14_attention_weight_min": -29.798418045043945,
      "activations/layer15_attention_weight_max": 34.91889190673828,
      "activations/layer15_attention_weight_min": -29.282249450683594,
      "activations/layer16_attention_weight_max": 30.923534393310547,
      "activations/layer16_attention_weight_min": -27.216333389282227,
      "activations/layer17_attention_weight_max": 48.84757614135742,
      "activations/layer17_attention_weight_min": -39.5696907043457,
      "activations/layer18_attention_weight_max": 45.06491470336914,
      "activations/layer18_attention_weight_min": -35.26335144042969,
      "activations/layer19_attention_weight_max": 23.681591033935547,
      "activations/layer19_attention_weight_min": -20.19806671142578,
      "activations/layer1_attention_weight_max": 17.34423828125,
      "activations/layer1_attention_weight_min": -13.949782371520996,
      "activations/layer20_attention_weight_max": 21.772232055664062,
      "activations/layer20_attention_weight_min": -20.3851318359375,
      "activations/layer21_attention_weight_max": 33.923744201660156,
      "activations/layer21_attention_weight_min": -23.577621459960938,
      "activations/layer22_attention_weight_max": 30.607093811035156,
      "activations/layer22_attention_weight_min": -22.916276931762695,
      "activations/layer23_attention_weight_max": 35.780860900878906,
      "activations/layer23_attention_weight_min": -23.904827117919922,
      "activations/layer2_attention_weight_max": 32.44378662109375,
      "activations/layer2_attention_weight_min": -30.436744689941406,
      "activations/layer3_attention_weight_max": 91.65438842773438,
      "activations/layer3_attention_weight_min": -98.23806762695312,
      "activations/layer4_attention_weight_max": 91.93755340576172,
      "activations/layer4_attention_weight_min": -91.52656555175781,
      "activations/layer5_attention_weight_max": 73.32537078857422,
      "activations/layer5_attention_weight_min": -80.0920639038086,
      "activations/layer6_attention_weight_max": 52.783164978027344,
      "activations/layer6_attention_weight_min": -52.60995101928711,
      "activations/layer7_attention_weight_max": 69.62073516845703,
      "activations/layer7_attention_weight_min": -66.83319854736328,
      "activations/layer8_attention_weight_max": 44.34004211425781,
      "activations/layer8_attention_weight_min": -49.555152893066406,
      "activations/layer9_attention_weight_max": 43.159915924072266,
      "activations/layer9_attention_weight_min": -47.91067123413086,
      "epoch": 12.99,
      "learning_rate": 6.690871212121211e-05,
      "loss": 2.7708,
      "step": 223550
    },
    {
      "activations/layer0_attention_weight_max": 16.014575958251953,
      "activations/layer0_attention_weight_min": -14.30533218383789,
      "activations/layer10_attention_weight_max": 38.20093536376953,
      "activations/layer10_attention_weight_min": -35.6214714050293,
      "activations/layer11_attention_weight_max": 35.940032958984375,
      "activations/layer11_attention_weight_min": -34.43479919433594,
      "activations/layer12_attention_weight_max": 18.769441604614258,
      "activations/layer12_attention_weight_min": -26.726646423339844,
      "activations/layer13_attention_weight_max": 39.112548828125,
      "activations/layer13_attention_weight_min": -29.669036865234375,
      "activations/layer14_attention_weight_max": 36.15967559814453,
      "activations/layer14_attention_weight_min": -30.568269729614258,
      "activations/layer15_attention_weight_max": 32.79871368408203,
      "activations/layer15_attention_weight_min": -30.717710494995117,
      "activations/layer16_attention_weight_max": 32.79641342163086,
      "activations/layer16_attention_weight_min": -29.718177795410156,
      "activations/layer17_attention_weight_max": 51.231319427490234,
      "activations/layer17_attention_weight_min": -43.41225814819336,
      "activations/layer18_attention_weight_max": 45.697940826416016,
      "activations/layer18_attention_weight_min": -37.28429412841797,
      "activations/layer19_attention_weight_max": 23.487346649169922,
      "activations/layer19_attention_weight_min": -22.17212677001953,
      "activations/layer1_attention_weight_max": 17.206459045410156,
      "activations/layer1_attention_weight_min": -16.52695083618164,
      "activations/layer20_attention_weight_max": 23.85613441467285,
      "activations/layer20_attention_weight_min": -21.10411834716797,
      "activations/layer21_attention_weight_max": 40.56932067871094,
      "activations/layer21_attention_weight_min": -25.496606826782227,
      "activations/layer22_attention_weight_max": 28.62457275390625,
      "activations/layer22_attention_weight_min": -24.341747283935547,
      "activations/layer23_attention_weight_max": 37.10966873168945,
      "activations/layer23_attention_weight_min": -24.671255111694336,
      "activations/layer2_attention_weight_max": 32.69401931762695,
      "activations/layer2_attention_weight_min": -30.315053939819336,
      "activations/layer3_attention_weight_max": 92.34931945800781,
      "activations/layer3_attention_weight_min": -98.88165283203125,
      "activations/layer4_attention_weight_max": 99.21359252929688,
      "activations/layer4_attention_weight_min": -93.42263793945312,
      "activations/layer5_attention_weight_max": 74.90509796142578,
      "activations/layer5_attention_weight_min": -81.25463104248047,
      "activations/layer6_attention_weight_max": 56.30205535888672,
      "activations/layer6_attention_weight_min": -57.58416748046875,
      "activations/layer7_attention_weight_max": 77.78059387207031,
      "activations/layer7_attention_weight_min": -68.79080963134766,
      "activations/layer8_attention_weight_max": 50.22456359863281,
      "activations/layer8_attention_weight_min": -50.22748947143555,
      "activations/layer9_attention_weight_max": 49.52220916748047,
      "activations/layer9_attention_weight_min": -48.83845901489258,
      "epoch": 12.99,
      "learning_rate": 6.688977272727273e-05,
      "loss": 2.7708,
      "step": 223600
    },
    {
      "activations/layer0_attention_weight_max": 15.939823150634766,
      "activations/layer0_attention_weight_min": -14.516158103942871,
      "activations/layer10_attention_weight_max": 38.422943115234375,
      "activations/layer10_attention_weight_min": -38.575260162353516,
      "activations/layer11_attention_weight_max": 32.91867446899414,
      "activations/layer11_attention_weight_min": -34.074005126953125,
      "activations/layer12_attention_weight_max": 21.373891830444336,
      "activations/layer12_attention_weight_min": -22.983091354370117,
      "activations/layer13_attention_weight_max": 36.415279388427734,
      "activations/layer13_attention_weight_min": -29.76892852783203,
      "activations/layer14_attention_weight_max": 35.70978927612305,
      "activations/layer14_attention_weight_min": -30.176036834716797,
      "activations/layer15_attention_weight_max": 32.41516876220703,
      "activations/layer15_attention_weight_min": -28.910240173339844,
      "activations/layer16_attention_weight_max": 28.20941925048828,
      "activations/layer16_attention_weight_min": -29.277700424194336,
      "activations/layer17_attention_weight_max": 51.53471374511719,
      "activations/layer17_attention_weight_min": -41.44115447998047,
      "activations/layer18_attention_weight_max": 42.1707763671875,
      "activations/layer18_attention_weight_min": -38.29047775268555,
      "activations/layer19_attention_weight_max": 20.571260452270508,
      "activations/layer19_attention_weight_min": -21.600481033325195,
      "activations/layer1_attention_weight_max": 16.12929916381836,
      "activations/layer1_attention_weight_min": -16.080530166625977,
      "activations/layer20_attention_weight_max": 19.900070190429688,
      "activations/layer20_attention_weight_min": -22.96746826171875,
      "activations/layer21_attention_weight_max": 31.09104347229004,
      "activations/layer21_attention_weight_min": -25.25798225402832,
      "activations/layer22_attention_weight_max": 26.21616554260254,
      "activations/layer22_attention_weight_min": -23.48520851135254,
      "activations/layer23_attention_weight_max": 34.83717346191406,
      "activations/layer23_attention_weight_min": -23.320518493652344,
      "activations/layer2_attention_weight_max": 31.923757553100586,
      "activations/layer2_attention_weight_min": -30.14345359802246,
      "activations/layer3_attention_weight_max": 91.16173553466797,
      "activations/layer3_attention_weight_min": -94.75927734375,
      "activations/layer4_attention_weight_max": 92.63243865966797,
      "activations/layer4_attention_weight_min": -90.06819915771484,
      "activations/layer5_attention_weight_max": 68.87062072753906,
      "activations/layer5_attention_weight_min": -75.25572967529297,
      "activations/layer6_attention_weight_max": 51.01981735229492,
      "activations/layer6_attention_weight_min": -51.14387893676758,
      "activations/layer7_attention_weight_max": 72.09721374511719,
      "activations/layer7_attention_weight_min": -65.72150421142578,
      "activations/layer8_attention_weight_max": 44.957374572753906,
      "activations/layer8_attention_weight_min": -46.65983963012695,
      "activations/layer9_attention_weight_max": 46.545860290527344,
      "activations/layer9_attention_weight_min": -48.0626220703125,
      "epoch": 13.0,
      "learning_rate": 6.687083333333333e-05,
      "loss": 2.7631,
      "step": 223650
    },
    {
      "activations/layer0_attention_weight_max": 17.034168243408203,
      "activations/layer0_attention_weight_min": -14.34538459777832,
      "activations/layer10_attention_weight_max": 37.03084182739258,
      "activations/layer10_attention_weight_min": -35.84584426879883,
      "activations/layer11_attention_weight_max": 31.972606658935547,
      "activations/layer11_attention_weight_min": -33.123348236083984,
      "activations/layer12_attention_weight_max": 23.799880981445312,
      "activations/layer12_attention_weight_min": -25.583736419677734,
      "activations/layer13_attention_weight_max": 33.975494384765625,
      "activations/layer13_attention_weight_min": -30.12766456604004,
      "activations/layer14_attention_weight_max": 33.01406478881836,
      "activations/layer14_attention_weight_min": -31.928325653076172,
      "activations/layer15_attention_weight_max": 31.051177978515625,
      "activations/layer15_attention_weight_min": -29.489416122436523,
      "activations/layer16_attention_weight_max": 29.945024490356445,
      "activations/layer16_attention_weight_min": -30.752477645874023,
      "activations/layer17_attention_weight_max": 52.64577865600586,
      "activations/layer17_attention_weight_min": -48.85930252075195,
      "activations/layer18_attention_weight_max": 46.57677459716797,
      "activations/layer18_attention_weight_min": -41.04206466674805,
      "activations/layer19_attention_weight_max": 22.622541427612305,
      "activations/layer19_attention_weight_min": -22.663820266723633,
      "activations/layer1_attention_weight_max": 16.313304901123047,
      "activations/layer1_attention_weight_min": -16.32888412475586,
      "activations/layer20_attention_weight_max": 23.070327758789062,
      "activations/layer20_attention_weight_min": -22.147232055664062,
      "activations/layer21_attention_weight_max": 37.33771514892578,
      "activations/layer21_attention_weight_min": -24.979507446289062,
      "activations/layer22_attention_weight_max": 32.83964920043945,
      "activations/layer22_attention_weight_min": -27.94880485534668,
      "activations/layer23_attention_weight_max": 37.30567932128906,
      "activations/layer23_attention_weight_min": -25.57990074157715,
      "activations/layer2_attention_weight_max": 31.494075775146484,
      "activations/layer2_attention_weight_min": -30.16969108581543,
      "activations/layer3_attention_weight_max": 91.85786437988281,
      "activations/layer3_attention_weight_min": -92.38153839111328,
      "activations/layer4_attention_weight_max": 92.07039642333984,
      "activations/layer4_attention_weight_min": -87.5218734741211,
      "activations/layer5_attention_weight_max": 71.64879608154297,
      "activations/layer5_attention_weight_min": -77.01940155029297,
      "activations/layer6_attention_weight_max": 49.82605743408203,
      "activations/layer6_attention_weight_min": -50.907554626464844,
      "activations/layer7_attention_weight_max": 67.97187042236328,
      "activations/layer7_attention_weight_min": -63.137821197509766,
      "activations/layer8_attention_weight_max": 44.48135757446289,
      "activations/layer8_attention_weight_min": -46.10203552246094,
      "activations/layer9_attention_weight_max": 46.93342590332031,
      "activations/layer9_attention_weight_min": -45.76460266113281,
      "epoch": 13.0,
      "learning_rate": 6.685189393939393e-05,
      "loss": 2.756,
      "step": 223700
    },
    {
      "activations/layer0_attention_weight_max": 16.8475399017334,
      "activations/layer0_attention_weight_min": -13.908957481384277,
      "activations/layer10_attention_weight_max": 36.65652084350586,
      "activations/layer10_attention_weight_min": -37.9726448059082,
      "activations/layer11_attention_weight_max": 33.3643684387207,
      "activations/layer11_attention_weight_min": -33.20414733886719,
      "activations/layer12_attention_weight_max": 17.636648178100586,
      "activations/layer12_attention_weight_min": -28.967802047729492,
      "activations/layer13_attention_weight_max": 44.46212387084961,
      "activations/layer13_attention_weight_min": -31.323497772216797,
      "activations/layer14_attention_weight_max": 45.054893493652344,
      "activations/layer14_attention_weight_min": -32.85747146606445,
      "activations/layer15_attention_weight_max": 35.40447235107422,
      "activations/layer15_attention_weight_min": -29.948802947998047,
      "activations/layer16_attention_weight_max": 32.90031814575195,
      "activations/layer16_attention_weight_min": -30.59337043762207,
      "activations/layer17_attention_weight_max": 51.04582977294922,
      "activations/layer17_attention_weight_min": -44.795352935791016,
      "activations/layer18_attention_weight_max": 46.279937744140625,
      "activations/layer18_attention_weight_min": -39.25761413574219,
      "activations/layer19_attention_weight_max": 23.141599655151367,
      "activations/layer19_attention_weight_min": -22.333864212036133,
      "activations/layer1_attention_weight_max": 17.143325805664062,
      "activations/layer1_attention_weight_min": -14.702855110168457,
      "activations/layer20_attention_weight_max": 21.844032287597656,
      "activations/layer20_attention_weight_min": -23.894607543945312,
      "activations/layer21_attention_weight_max": 44.287471771240234,
      "activations/layer21_attention_weight_min": -23.868717193603516,
      "activations/layer22_attention_weight_max": 30.394250869750977,
      "activations/layer22_attention_weight_min": -26.160541534423828,
      "activations/layer23_attention_weight_max": 38.29277801513672,
      "activations/layer23_attention_weight_min": -25.448017120361328,
      "activations/layer2_attention_weight_max": 32.42298126220703,
      "activations/layer2_attention_weight_min": -31.050640106201172,
      "activations/layer3_attention_weight_max": 88.43209838867188,
      "activations/layer3_attention_weight_min": -91.73689270019531,
      "activations/layer4_attention_weight_max": 92.30847930908203,
      "activations/layer4_attention_weight_min": -90.05611419677734,
      "activations/layer5_attention_weight_max": 70.43354797363281,
      "activations/layer5_attention_weight_min": -75.52153015136719,
      "activations/layer6_attention_weight_max": 53.011592864990234,
      "activations/layer6_attention_weight_min": -50.982582092285156,
      "activations/layer7_attention_weight_max": 67.78633880615234,
      "activations/layer7_attention_weight_min": -64.34292602539062,
      "activations/layer8_attention_weight_max": 43.94029998779297,
      "activations/layer8_attention_weight_min": -45.94243240356445,
      "activations/layer9_attention_weight_max": 48.99821090698242,
      "activations/layer9_attention_weight_min": -46.03895950317383,
      "epoch": 13.0,
      "learning_rate": 6.683295454545453e-05,
      "loss": 2.8075,
      "step": 223750
    },
    {
      "activations/layer0_attention_weight_max": 15.965307235717773,
      "activations/layer0_attention_weight_min": -14.16353702545166,
      "activations/layer10_attention_weight_max": 34.38655090332031,
      "activations/layer10_attention_weight_min": -35.35686111450195,
      "activations/layer11_attention_weight_max": 33.39679718017578,
      "activations/layer11_attention_weight_min": -33.079593658447266,
      "activations/layer12_attention_weight_max": 18.031692504882812,
      "activations/layer12_attention_weight_min": -27.684890747070312,
      "activations/layer13_attention_weight_max": 40.7984619140625,
      "activations/layer13_attention_weight_min": -35.940025329589844,
      "activations/layer14_attention_weight_max": 36.24503707885742,
      "activations/layer14_attention_weight_min": -32.019081115722656,
      "activations/layer15_attention_weight_max": 32.11422348022461,
      "activations/layer15_attention_weight_min": -30.490638732910156,
      "activations/layer16_attention_weight_max": 34.469085693359375,
      "activations/layer16_attention_weight_min": -31.034683227539062,
      "activations/layer17_attention_weight_max": 55.983455657958984,
      "activations/layer17_attention_weight_min": -44.41107177734375,
      "activations/layer18_attention_weight_max": 46.50688171386719,
      "activations/layer18_attention_weight_min": -38.72404098510742,
      "activations/layer19_attention_weight_max": 23.015100479125977,
      "activations/layer19_attention_weight_min": -22.464624404907227,
      "activations/layer1_attention_weight_max": 16.258424758911133,
      "activations/layer1_attention_weight_min": -17.616662979125977,
      "activations/layer20_attention_weight_max": 22.560903549194336,
      "activations/layer20_attention_weight_min": -24.066362380981445,
      "activations/layer21_attention_weight_max": 40.69496536254883,
      "activations/layer21_attention_weight_min": -23.765960693359375,
      "activations/layer22_attention_weight_max": 31.07571792602539,
      "activations/layer22_attention_weight_min": -24.864973068237305,
      "activations/layer23_attention_weight_max": 40.57352828979492,
      "activations/layer23_attention_weight_min": -22.69482421875,
      "activations/layer2_attention_weight_max": 32.53934097290039,
      "activations/layer2_attention_weight_min": -30.271778106689453,
      "activations/layer3_attention_weight_max": 95.86299133300781,
      "activations/layer3_attention_weight_min": -97.14353942871094,
      "activations/layer4_attention_weight_max": 97.79911804199219,
      "activations/layer4_attention_weight_min": -89.88883209228516,
      "activations/layer5_attention_weight_max": 72.14151000976562,
      "activations/layer5_attention_weight_min": -74.10892486572266,
      "activations/layer6_attention_weight_max": 52.84305953979492,
      "activations/layer6_attention_weight_min": -49.88545227050781,
      "activations/layer7_attention_weight_max": 72.81134796142578,
      "activations/layer7_attention_weight_min": -64.164794921875,
      "activations/layer8_attention_weight_max": 42.331336975097656,
      "activations/layer8_attention_weight_min": -45.9826774597168,
      "activations/layer9_attention_weight_max": 43.59038543701172,
      "activations/layer9_attention_weight_min": -42.58462142944336,
      "epoch": 13.0,
      "learning_rate": 6.681401515151515e-05,
      "loss": 2.7683,
      "step": 223800
    },
    {
      "activations/layer0_attention_weight_max": 16.11819839477539,
      "activations/layer0_attention_weight_min": -14.059683799743652,
      "activations/layer10_attention_weight_max": 34.14397048950195,
      "activations/layer10_attention_weight_min": -33.9775390625,
      "activations/layer11_attention_weight_max": 30.673477172851562,
      "activations/layer11_attention_weight_min": -32.56422424316406,
      "activations/layer12_attention_weight_max": 19.33277702331543,
      "activations/layer12_attention_weight_min": -26.519973754882812,
      "activations/layer13_attention_weight_max": 44.486507415771484,
      "activations/layer13_attention_weight_min": -27.4189510345459,
      "activations/layer14_attention_weight_max": 39.70760726928711,
      "activations/layer14_attention_weight_min": -33.17795944213867,
      "activations/layer15_attention_weight_max": 37.46460723876953,
      "activations/layer15_attention_weight_min": -29.01195526123047,
      "activations/layer16_attention_weight_max": 32.7799186706543,
      "activations/layer16_attention_weight_min": -29.30160903930664,
      "activations/layer17_attention_weight_max": 52.96955871582031,
      "activations/layer17_attention_weight_min": -44.01241683959961,
      "activations/layer18_attention_weight_max": 48.06608963012695,
      "activations/layer18_attention_weight_min": -37.98430633544922,
      "activations/layer19_attention_weight_max": 26.51470947265625,
      "activations/layer19_attention_weight_min": -21.75398826599121,
      "activations/layer1_attention_weight_max": 17.152502059936523,
      "activations/layer1_attention_weight_min": -15.841361045837402,
      "activations/layer20_attention_weight_max": 27.519840240478516,
      "activations/layer20_attention_weight_min": -21.83612060546875,
      "activations/layer21_attention_weight_max": 41.863006591796875,
      "activations/layer21_attention_weight_min": -22.096895217895508,
      "activations/layer22_attention_weight_max": 32.95091247558594,
      "activations/layer22_attention_weight_min": -26.42702293395996,
      "activations/layer23_attention_weight_max": 41.834354400634766,
      "activations/layer23_attention_weight_min": -24.78432846069336,
      "activations/layer2_attention_weight_max": 33.966529846191406,
      "activations/layer2_attention_weight_min": -32.21087646484375,
      "activations/layer3_attention_weight_max": 96.4189224243164,
      "activations/layer3_attention_weight_min": -101.01751708984375,
      "activations/layer4_attention_weight_max": 95.00665283203125,
      "activations/layer4_attention_weight_min": -91.72631072998047,
      "activations/layer5_attention_weight_max": 70.85481262207031,
      "activations/layer5_attention_weight_min": -76.0634765625,
      "activations/layer6_attention_weight_max": 53.867618560791016,
      "activations/layer6_attention_weight_min": -52.74541473388672,
      "activations/layer7_attention_weight_max": 67.84766387939453,
      "activations/layer7_attention_weight_min": -64.8443603515625,
      "activations/layer8_attention_weight_max": 43.11222839355469,
      "activations/layer8_attention_weight_min": -45.26942443847656,
      "activations/layer9_attention_weight_max": 43.277244567871094,
      "activations/layer9_attention_weight_min": -44.69160079956055,
      "epoch": 13.01,
      "learning_rate": 6.679507575757575e-05,
      "loss": 2.7639,
      "step": 223850
    },
    {
      "activations/layer0_attention_weight_max": 16.080886840820312,
      "activations/layer0_attention_weight_min": -14.144434928894043,
      "activations/layer10_attention_weight_max": 35.10292434692383,
      "activations/layer10_attention_weight_min": -35.17802810668945,
      "activations/layer11_attention_weight_max": 30.780231475830078,
      "activations/layer11_attention_weight_min": -31.659955978393555,
      "activations/layer12_attention_weight_max": 20.467239379882812,
      "activations/layer12_attention_weight_min": -24.80002784729004,
      "activations/layer13_attention_weight_max": 38.11372756958008,
      "activations/layer13_attention_weight_min": -28.950531005859375,
      "activations/layer14_attention_weight_max": 36.858619689941406,
      "activations/layer14_attention_weight_min": -31.113483428955078,
      "activations/layer15_attention_weight_max": 33.062034606933594,
      "activations/layer15_attention_weight_min": -28.906030654907227,
      "activations/layer16_attention_weight_max": 33.563297271728516,
      "activations/layer16_attention_weight_min": -31.042272567749023,
      "activations/layer17_attention_weight_max": 51.117897033691406,
      "activations/layer17_attention_weight_min": -44.94719696044922,
      "activations/layer18_attention_weight_max": 47.97949981689453,
      "activations/layer18_attention_weight_min": -39.73871994018555,
      "activations/layer19_attention_weight_max": 25.365856170654297,
      "activations/layer19_attention_weight_min": -22.566694259643555,
      "activations/layer1_attention_weight_max": 17.31505012512207,
      "activations/layer1_attention_weight_min": -15.33843994140625,
      "activations/layer20_attention_weight_max": 24.454479217529297,
      "activations/layer20_attention_weight_min": -21.916257858276367,
      "activations/layer21_attention_weight_max": 36.533538818359375,
      "activations/layer21_attention_weight_min": -22.869958877563477,
      "activations/layer22_attention_weight_max": 32.203338623046875,
      "activations/layer22_attention_weight_min": -25.2733097076416,
      "activations/layer23_attention_weight_max": 41.196659088134766,
      "activations/layer23_attention_weight_min": -24.75518035888672,
      "activations/layer2_attention_weight_max": 31.76809310913086,
      "activations/layer2_attention_weight_min": -30.730663299560547,
      "activations/layer3_attention_weight_max": 93.1112289428711,
      "activations/layer3_attention_weight_min": -93.53001403808594,
      "activations/layer4_attention_weight_max": 93.1040267944336,
      "activations/layer4_attention_weight_min": -91.31645965576172,
      "activations/layer5_attention_weight_max": 69.7537841796875,
      "activations/layer5_attention_weight_min": -76.14595794677734,
      "activations/layer6_attention_weight_max": 50.258853912353516,
      "activations/layer6_attention_weight_min": -50.566287994384766,
      "activations/layer7_attention_weight_max": 67.18861389160156,
      "activations/layer7_attention_weight_min": -68.3416748046875,
      "activations/layer8_attention_weight_max": 45.030609130859375,
      "activations/layer8_attention_weight_min": -48.12275695800781,
      "activations/layer9_attention_weight_max": 42.91862487792969,
      "activations/layer9_attention_weight_min": -45.006683349609375,
      "epoch": 13.01,
      "learning_rate": 6.677613636363635e-05,
      "loss": 2.7377,
      "step": 223900
    },
    {
      "activations/layer0_attention_weight_max": 16.409549713134766,
      "activations/layer0_attention_weight_min": -14.23138427734375,
      "activations/layer10_attention_weight_max": 37.06853485107422,
      "activations/layer10_attention_weight_min": -35.592506408691406,
      "activations/layer11_attention_weight_max": 32.14335632324219,
      "activations/layer11_attention_weight_min": -32.73439025878906,
      "activations/layer12_attention_weight_max": 17.29111099243164,
      "activations/layer12_attention_weight_min": -23.73392105102539,
      "activations/layer13_attention_weight_max": 41.72526168823242,
      "activations/layer13_attention_weight_min": -30.293081283569336,
      "activations/layer14_attention_weight_max": 33.50779342651367,
      "activations/layer14_attention_weight_min": -28.287199020385742,
      "activations/layer15_attention_weight_max": 29.8140811920166,
      "activations/layer15_attention_weight_min": -28.167552947998047,
      "activations/layer16_attention_weight_max": 30.949995040893555,
      "activations/layer16_attention_weight_min": -29.365219116210938,
      "activations/layer17_attention_weight_max": 48.155338287353516,
      "activations/layer17_attention_weight_min": -42.757137298583984,
      "activations/layer18_attention_weight_max": 44.69465255737305,
      "activations/layer18_attention_weight_min": -36.87370681762695,
      "activations/layer19_attention_weight_max": 21.021724700927734,
      "activations/layer19_attention_weight_min": -22.505144119262695,
      "activations/layer1_attention_weight_max": 16.596303939819336,
      "activations/layer1_attention_weight_min": -16.089723587036133,
      "activations/layer20_attention_weight_max": 21.974870681762695,
      "activations/layer20_attention_weight_min": -20.5673770904541,
      "activations/layer21_attention_weight_max": 32.0981559753418,
      "activations/layer21_attention_weight_min": -20.124319076538086,
      "activations/layer22_attention_weight_max": 28.7126407623291,
      "activations/layer22_attention_weight_min": -25.86798095703125,
      "activations/layer23_attention_weight_max": 35.495338439941406,
      "activations/layer23_attention_weight_min": -22.152008056640625,
      "activations/layer2_attention_weight_max": 31.883832931518555,
      "activations/layer2_attention_weight_min": -32.264381408691406,
      "activations/layer3_attention_weight_max": 95.0025634765625,
      "activations/layer3_attention_weight_min": -95.87542724609375,
      "activations/layer4_attention_weight_max": 102.1522445678711,
      "activations/layer4_attention_weight_min": -98.95095825195312,
      "activations/layer5_attention_weight_max": 73.1236572265625,
      "activations/layer5_attention_weight_min": -82.08258056640625,
      "activations/layer6_attention_weight_max": 52.14332962036133,
      "activations/layer6_attention_weight_min": -53.49052047729492,
      "activations/layer7_attention_weight_max": 69.86060333251953,
      "activations/layer7_attention_weight_min": -68.9599838256836,
      "activations/layer8_attention_weight_max": 46.684200286865234,
      "activations/layer8_attention_weight_min": -46.65131759643555,
      "activations/layer9_attention_weight_max": 46.4592399597168,
      "activations/layer9_attention_weight_min": -46.49732208251953,
      "epoch": 13.01,
      "learning_rate": 6.675719696969697e-05,
      "loss": 2.7529,
      "step": 223950
    },
    {
      "activations/layer0_attention_weight_max": 15.833667755126953,
      "activations/layer0_attention_weight_min": -14.278646469116211,
      "activations/layer10_attention_weight_max": 36.34440994262695,
      "activations/layer10_attention_weight_min": -35.36455154418945,
      "activations/layer11_attention_weight_max": 33.45710372924805,
      "activations/layer11_attention_weight_min": -32.36684799194336,
      "activations/layer12_attention_weight_max": 18.05872344970703,
      "activations/layer12_attention_weight_min": -28.838855743408203,
      "activations/layer13_attention_weight_max": 33.75313186645508,
      "activations/layer13_attention_weight_min": -28.27271270751953,
      "activations/layer14_attention_weight_max": 33.715476989746094,
      "activations/layer14_attention_weight_min": -29.796037673950195,
      "activations/layer15_attention_weight_max": 30.969816207885742,
      "activations/layer15_attention_weight_min": -28.787389755249023,
      "activations/layer16_attention_weight_max": 29.035091400146484,
      "activations/layer16_attention_weight_min": -27.299253463745117,
      "activations/layer17_attention_weight_max": 47.0552864074707,
      "activations/layer17_attention_weight_min": -42.51415252685547,
      "activations/layer18_attention_weight_max": 44.140228271484375,
      "activations/layer18_attention_weight_min": -37.80207443237305,
      "activations/layer19_attention_weight_max": 19.900972366333008,
      "activations/layer19_attention_weight_min": -21.03999137878418,
      "activations/layer1_attention_weight_max": 16.25383758544922,
      "activations/layer1_attention_weight_min": -14.714694023132324,
      "activations/layer20_attention_weight_max": 19.50288963317871,
      "activations/layer20_attention_weight_min": -20.070919036865234,
      "activations/layer21_attention_weight_max": 33.21369934082031,
      "activations/layer21_attention_weight_min": -22.06490135192871,
      "activations/layer22_attention_weight_max": 28.125837326049805,
      "activations/layer22_attention_weight_min": -28.761295318603516,
      "activations/layer23_attention_weight_max": 35.446678161621094,
      "activations/layer23_attention_weight_min": -23.81990623474121,
      "activations/layer2_attention_weight_max": 31.748943328857422,
      "activations/layer2_attention_weight_min": -30.588979721069336,
      "activations/layer3_attention_weight_max": 93.84674072265625,
      "activations/layer3_attention_weight_min": -95.26254272460938,
      "activations/layer4_attention_weight_max": 95.4140853881836,
      "activations/layer4_attention_weight_min": -92.73737335205078,
      "activations/layer5_attention_weight_max": 73.4289779663086,
      "activations/layer5_attention_weight_min": -75.97869110107422,
      "activations/layer6_attention_weight_max": 51.799625396728516,
      "activations/layer6_attention_weight_min": -52.322959899902344,
      "activations/layer7_attention_weight_max": 73.91830444335938,
      "activations/layer7_attention_weight_min": -66.90994262695312,
      "activations/layer8_attention_weight_max": 46.58119583129883,
      "activations/layer8_attention_weight_min": -47.35931396484375,
      "activations/layer9_attention_weight_max": 54.16730880737305,
      "activations/layer9_attention_weight_min": -48.63832473754883,
      "epoch": 13.02,
      "learning_rate": 6.673825757575757e-05,
      "loss": 2.7751,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.466,
      "eval_samples_per_second": 507.205,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.466,
      "eval_openwebtext_samples_per_second": 507.205,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 1.9356,
      "eval_wikitext_samples_per_second": 235.58,
      "step": 224000
    },
    {
      "epoch": 13.02,
      "eval_lambada_loss": 2.626953125,
      "eval_lambada_ppl": 13.831562592310346,
      "eval_lambada_runtime": 9.5397,
      "eval_lambada_samples_per_second": 510.391,
      "step": 224000
    },
    {
      "activations/layer0_attention_weight_max": 16.06924819946289,
      "activations/layer0_attention_weight_min": -14.0787935256958,
      "activations/layer10_attention_weight_max": 36.69077682495117,
      "activations/layer10_attention_weight_min": -37.67580032348633,
      "activations/layer11_attention_weight_max": 32.31554412841797,
      "activations/layer11_attention_weight_min": -32.92261505126953,
      "activations/layer12_attention_weight_max": 17.561817169189453,
      "activations/layer12_attention_weight_min": -26.946046829223633,
      "activations/layer13_attention_weight_max": 36.70278549194336,
      "activations/layer13_attention_weight_min": -28.717382431030273,
      "activations/layer14_attention_weight_max": 39.1215934753418,
      "activations/layer14_attention_weight_min": -31.613855361938477,
      "activations/layer15_attention_weight_max": 33.927886962890625,
      "activations/layer15_attention_weight_min": -29.988082885742188,
      "activations/layer16_attention_weight_max": 33.466590881347656,
      "activations/layer16_attention_weight_min": -31.213123321533203,
      "activations/layer17_attention_weight_max": 58.47479248046875,
      "activations/layer17_attention_weight_min": -50.24846267700195,
      "activations/layer18_attention_weight_max": 52.9024772644043,
      "activations/layer18_attention_weight_min": -42.61954879760742,
      "activations/layer19_attention_weight_max": 22.971189498901367,
      "activations/layer19_attention_weight_min": -21.934621810913086,
      "activations/layer1_attention_weight_max": 15.84509563446045,
      "activations/layer1_attention_weight_min": -14.735858917236328,
      "activations/layer20_attention_weight_max": 23.659788131713867,
      "activations/layer20_attention_weight_min": -20.888675689697266,
      "activations/layer21_attention_weight_max": 34.03968048095703,
      "activations/layer21_attention_weight_min": -23.55686378479004,
      "activations/layer22_attention_weight_max": 29.120094299316406,
      "activations/layer22_attention_weight_min": -25.724634170532227,
      "activations/layer23_attention_weight_max": 36.75086975097656,
      "activations/layer23_attention_weight_min": -25.353153228759766,
      "activations/layer2_attention_weight_max": 30.52509307861328,
      "activations/layer2_attention_weight_min": -30.379932403564453,
      "activations/layer3_attention_weight_max": 90.46936798095703,
      "activations/layer3_attention_weight_min": -94.31254577636719,
      "activations/layer4_attention_weight_max": 92.61997985839844,
      "activations/layer4_attention_weight_min": -90.90570068359375,
      "activations/layer5_attention_weight_max": 69.38032531738281,
      "activations/layer5_attention_weight_min": -79.43679809570312,
      "activations/layer6_attention_weight_max": 52.63784408569336,
      "activations/layer6_attention_weight_min": -52.050880432128906,
      "activations/layer7_attention_weight_max": 66.54971313476562,
      "activations/layer7_attention_weight_min": -66.7975845336914,
      "activations/layer8_attention_weight_max": 45.93764877319336,
      "activations/layer8_attention_weight_min": -49.02687072753906,
      "activations/layer9_attention_weight_max": 46.408782958984375,
      "activations/layer9_attention_weight_min": -46.56156921386719,
      "epoch": 13.02,
      "learning_rate": 6.671931818181818e-05,
      "loss": 2.7713,
      "step": 224050
    },
    {
      "activations/layer0_attention_weight_max": 15.712920188903809,
      "activations/layer0_attention_weight_min": -13.860869407653809,
      "activations/layer10_attention_weight_max": 39.25990295410156,
      "activations/layer10_attention_weight_min": -39.471595764160156,
      "activations/layer11_attention_weight_max": 33.62333679199219,
      "activations/layer11_attention_weight_min": -35.61627197265625,
      "activations/layer12_attention_weight_max": 18.617977142333984,
      "activations/layer12_attention_weight_min": -22.93107795715332,
      "activations/layer13_attention_weight_max": 38.93054962158203,
      "activations/layer13_attention_weight_min": -29.869192123413086,
      "activations/layer14_attention_weight_max": 38.266387939453125,
      "activations/layer14_attention_weight_min": -31.071605682373047,
      "activations/layer15_attention_weight_max": 34.539669036865234,
      "activations/layer15_attention_weight_min": -29.727388381958008,
      "activations/layer16_attention_weight_max": 33.092647552490234,
      "activations/layer16_attention_weight_min": -31.53352928161621,
      "activations/layer17_attention_weight_max": 53.72736740112305,
      "activations/layer17_attention_weight_min": -46.736602783203125,
      "activations/layer18_attention_weight_max": 48.89243698120117,
      "activations/layer18_attention_weight_min": -40.565086364746094,
      "activations/layer19_attention_weight_max": 22.809772491455078,
      "activations/layer19_attention_weight_min": -25.47584342956543,
      "activations/layer1_attention_weight_max": 17.09646224975586,
      "activations/layer1_attention_weight_min": -14.405752182006836,
      "activations/layer20_attention_weight_max": 21.94229507446289,
      "activations/layer20_attention_weight_min": -20.986722946166992,
      "activations/layer21_attention_weight_max": 34.1505012512207,
      "activations/layer21_attention_weight_min": -19.334543228149414,
      "activations/layer22_attention_weight_max": 30.623567581176758,
      "activations/layer22_attention_weight_min": -25.89830780029297,
      "activations/layer23_attention_weight_max": 36.93578338623047,
      "activations/layer23_attention_weight_min": -24.210508346557617,
      "activations/layer2_attention_weight_max": 30.933700561523438,
      "activations/layer2_attention_weight_min": -29.2239990234375,
      "activations/layer3_attention_weight_max": 97.42068481445312,
      "activations/layer3_attention_weight_min": -97.52912139892578,
      "activations/layer4_attention_weight_max": 99.56002044677734,
      "activations/layer4_attention_weight_min": -97.74752044677734,
      "activations/layer5_attention_weight_max": 73.5467300415039,
      "activations/layer5_attention_weight_min": -72.69190979003906,
      "activations/layer6_attention_weight_max": 53.01639175415039,
      "activations/layer6_attention_weight_min": -52.09421920776367,
      "activations/layer7_attention_weight_max": 71.97123718261719,
      "activations/layer7_attention_weight_min": -65.92993927001953,
      "activations/layer8_attention_weight_max": 45.20045852661133,
      "activations/layer8_attention_weight_min": -49.18074417114258,
      "activations/layer9_attention_weight_max": 49.99570846557617,
      "activations/layer9_attention_weight_min": -47.22469711303711,
      "epoch": 13.02,
      "learning_rate": 6.670037878787878e-05,
      "loss": 2.7609,
      "step": 224100
    },
    {
      "activations/layer0_attention_weight_max": 15.425339698791504,
      "activations/layer0_attention_weight_min": -14.267363548278809,
      "activations/layer10_attention_weight_max": 34.28910446166992,
      "activations/layer10_attention_weight_min": -34.908451080322266,
      "activations/layer11_attention_weight_max": 31.794139862060547,
      "activations/layer11_attention_weight_min": -32.50339126586914,
      "activations/layer12_attention_weight_max": 16.754993438720703,
      "activations/layer12_attention_weight_min": -22.507482528686523,
      "activations/layer13_attention_weight_max": 37.030418395996094,
      "activations/layer13_attention_weight_min": -27.65852928161621,
      "activations/layer14_attention_weight_max": 34.59830856323242,
      "activations/layer14_attention_weight_min": -29.56434440612793,
      "activations/layer15_attention_weight_max": 34.09831237792969,
      "activations/layer15_attention_weight_min": -29.04478645324707,
      "activations/layer16_attention_weight_max": 31.79706573486328,
      "activations/layer16_attention_weight_min": -30.57469367980957,
      "activations/layer17_attention_weight_max": 53.129432678222656,
      "activations/layer17_attention_weight_min": -45.5444221496582,
      "activations/layer18_attention_weight_max": 47.53663635253906,
      "activations/layer18_attention_weight_min": -37.41600036621094,
      "activations/layer19_attention_weight_max": 21.28352165222168,
      "activations/layer19_attention_weight_min": -21.299612045288086,
      "activations/layer1_attention_weight_max": 17.416452407836914,
      "activations/layer1_attention_weight_min": -14.98205280303955,
      "activations/layer20_attention_weight_max": 23.275144577026367,
      "activations/layer20_attention_weight_min": -25.911100387573242,
      "activations/layer21_attention_weight_max": 33.925289154052734,
      "activations/layer21_attention_weight_min": -21.31279754638672,
      "activations/layer22_attention_weight_max": 30.424068450927734,
      "activations/layer22_attention_weight_min": -26.485641479492188,
      "activations/layer23_attention_weight_max": 38.2956428527832,
      "activations/layer23_attention_weight_min": -23.29528045654297,
      "activations/layer2_attention_weight_max": 31.176326751708984,
      "activations/layer2_attention_weight_min": -30.184986114501953,
      "activations/layer3_attention_weight_max": 95.15184020996094,
      "activations/layer3_attention_weight_min": -96.84666442871094,
      "activations/layer4_attention_weight_max": 98.37203216552734,
      "activations/layer4_attention_weight_min": -95.3156967163086,
      "activations/layer5_attention_weight_max": 68.37443542480469,
      "activations/layer5_attention_weight_min": -75.51756286621094,
      "activations/layer6_attention_weight_max": 51.88591003417969,
      "activations/layer6_attention_weight_min": -52.81830978393555,
      "activations/layer7_attention_weight_max": 66.29113006591797,
      "activations/layer7_attention_weight_min": -65.23766326904297,
      "activations/layer8_attention_weight_max": 46.82060623168945,
      "activations/layer8_attention_weight_min": -48.65515899658203,
      "activations/layer9_attention_weight_max": 44.17544174194336,
      "activations/layer9_attention_weight_min": -45.731422424316406,
      "epoch": 13.02,
      "learning_rate": 6.668143939393939e-05,
      "loss": 2.7563,
      "step": 224150
    },
    {
      "activations/layer0_attention_weight_max": 15.936415672302246,
      "activations/layer0_attention_weight_min": -13.96440601348877,
      "activations/layer10_attention_weight_max": 42.2038688659668,
      "activations/layer10_attention_weight_min": -38.75074005126953,
      "activations/layer11_attention_weight_max": 40.02337646484375,
      "activations/layer11_attention_weight_min": -37.67627716064453,
      "activations/layer12_attention_weight_max": 16.239152908325195,
      "activations/layer12_attention_weight_min": -27.75190544128418,
      "activations/layer13_attention_weight_max": 39.48189926147461,
      "activations/layer13_attention_weight_min": -31.932653427124023,
      "activations/layer14_attention_weight_max": 45.220096588134766,
      "activations/layer14_attention_weight_min": -32.78223419189453,
      "activations/layer15_attention_weight_max": 38.281654357910156,
      "activations/layer15_attention_weight_min": -32.504493713378906,
      "activations/layer16_attention_weight_max": 31.495031356811523,
      "activations/layer16_attention_weight_min": -30.090784072875977,
      "activations/layer17_attention_weight_max": 50.41819763183594,
      "activations/layer17_attention_weight_min": -46.005069732666016,
      "activations/layer18_attention_weight_max": 42.77825164794922,
      "activations/layer18_attention_weight_min": -39.701148986816406,
      "activations/layer19_attention_weight_max": 20.93255043029785,
      "activations/layer19_attention_weight_min": -23.50082015991211,
      "activations/layer1_attention_weight_max": 16.946075439453125,
      "activations/layer1_attention_weight_min": -14.767890930175781,
      "activations/layer20_attention_weight_max": 22.117000579833984,
      "activations/layer20_attention_weight_min": -21.129051208496094,
      "activations/layer21_attention_weight_max": 34.687957763671875,
      "activations/layer21_attention_weight_min": -24.255605697631836,
      "activations/layer22_attention_weight_max": 29.070758819580078,
      "activations/layer22_attention_weight_min": -25.37582015991211,
      "activations/layer23_attention_weight_max": 36.54500961303711,
      "activations/layer23_attention_weight_min": -26.251724243164062,
      "activations/layer2_attention_weight_max": 32.57696533203125,
      "activations/layer2_attention_weight_min": -30.681095123291016,
      "activations/layer3_attention_weight_max": 94.5645523071289,
      "activations/layer3_attention_weight_min": -99.64210510253906,
      "activations/layer4_attention_weight_max": 94.54940032958984,
      "activations/layer4_attention_weight_min": -92.5486831665039,
      "activations/layer5_attention_weight_max": 72.48370361328125,
      "activations/layer5_attention_weight_min": -78.47052001953125,
      "activations/layer6_attention_weight_max": 51.04629898071289,
      "activations/layer6_attention_weight_min": -51.65674591064453,
      "activations/layer7_attention_weight_max": 80.2994613647461,
      "activations/layer7_attention_weight_min": -69.29279327392578,
      "activations/layer8_attention_weight_max": 51.861839294433594,
      "activations/layer8_attention_weight_min": -48.929874420166016,
      "activations/layer9_attention_weight_max": 57.5037841796875,
      "activations/layer9_attention_weight_min": -48.953468322753906,
      "epoch": 13.03,
      "learning_rate": 6.66625e-05,
      "loss": 2.7684,
      "step": 224200
    },
    {
      "activations/layer0_attention_weight_max": 16.451133728027344,
      "activations/layer0_attention_weight_min": -13.67159652709961,
      "activations/layer10_attention_weight_max": 37.11488723754883,
      "activations/layer10_attention_weight_min": -37.71598434448242,
      "activations/layer11_attention_weight_max": 32.448177337646484,
      "activations/layer11_attention_weight_min": -33.84703826904297,
      "activations/layer12_attention_weight_max": 16.559982299804688,
      "activations/layer12_attention_weight_min": -24.23025131225586,
      "activations/layer13_attention_weight_max": 36.55434799194336,
      "activations/layer13_attention_weight_min": -28.019811630249023,
      "activations/layer14_attention_weight_max": 34.43309020996094,
      "activations/layer14_attention_weight_min": -30.344280242919922,
      "activations/layer15_attention_weight_max": 33.615657806396484,
      "activations/layer15_attention_weight_min": -30.66963768005371,
      "activations/layer16_attention_weight_max": 31.513553619384766,
      "activations/layer16_attention_weight_min": -30.475568771362305,
      "activations/layer17_attention_weight_max": 47.80937576293945,
      "activations/layer17_attention_weight_min": -43.474422454833984,
      "activations/layer18_attention_weight_max": 43.605045318603516,
      "activations/layer18_attention_weight_min": -37.533687591552734,
      "activations/layer19_attention_weight_max": 22.761240005493164,
      "activations/layer19_attention_weight_min": -22.858192443847656,
      "activations/layer1_attention_weight_max": 16.337194442749023,
      "activations/layer1_attention_weight_min": -15.327427864074707,
      "activations/layer20_attention_weight_max": 23.190656661987305,
      "activations/layer20_attention_weight_min": -21.400663375854492,
      "activations/layer21_attention_weight_max": 32.34974670410156,
      "activations/layer21_attention_weight_min": -20.174762725830078,
      "activations/layer22_attention_weight_max": 31.47669219970703,
      "activations/layer22_attention_weight_min": -25.104326248168945,
      "activations/layer23_attention_weight_max": 35.97564697265625,
      "activations/layer23_attention_weight_min": -24.407880783081055,
      "activations/layer2_attention_weight_max": 33.2265739440918,
      "activations/layer2_attention_weight_min": -31.401229858398438,
      "activations/layer3_attention_weight_max": 98.33209228515625,
      "activations/layer3_attention_weight_min": -97.15693664550781,
      "activations/layer4_attention_weight_max": 97.74514770507812,
      "activations/layer4_attention_weight_min": -96.08358764648438,
      "activations/layer5_attention_weight_max": 70.49566650390625,
      "activations/layer5_attention_weight_min": -78.52667999267578,
      "activations/layer6_attention_weight_max": 54.63850402832031,
      "activations/layer6_attention_weight_min": -53.02941131591797,
      "activations/layer7_attention_weight_max": 71.60399627685547,
      "activations/layer7_attention_weight_min": -68.2986068725586,
      "activations/layer8_attention_weight_max": 46.15956115722656,
      "activations/layer8_attention_weight_min": -48.770816802978516,
      "activations/layer9_attention_weight_max": 46.3232307434082,
      "activations/layer9_attention_weight_min": -46.474334716796875,
      "epoch": 13.03,
      "learning_rate": 6.66435606060606e-05,
      "loss": 2.7844,
      "step": 224250
    },
    {
      "activations/layer0_attention_weight_max": 16.072084426879883,
      "activations/layer0_attention_weight_min": -13.602128982543945,
      "activations/layer10_attention_weight_max": 37.736053466796875,
      "activations/layer10_attention_weight_min": -36.85163879394531,
      "activations/layer11_attention_weight_max": 31.387807846069336,
      "activations/layer11_attention_weight_min": -34.36096954345703,
      "activations/layer12_attention_weight_max": 19.49997329711914,
      "activations/layer12_attention_weight_min": -23.332975387573242,
      "activations/layer13_attention_weight_max": 38.79250717163086,
      "activations/layer13_attention_weight_min": -31.595603942871094,
      "activations/layer14_attention_weight_max": 38.167640686035156,
      "activations/layer14_attention_weight_min": -29.737092971801758,
      "activations/layer15_attention_weight_max": 32.945526123046875,
      "activations/layer15_attention_weight_min": -30.931842803955078,
      "activations/layer16_attention_weight_max": 32.74017333984375,
      "activations/layer16_attention_weight_min": -29.43556785583496,
      "activations/layer17_attention_weight_max": 54.753990173339844,
      "activations/layer17_attention_weight_min": -44.00771713256836,
      "activations/layer18_attention_weight_max": 49.39776611328125,
      "activations/layer18_attention_weight_min": -37.25279998779297,
      "activations/layer19_attention_weight_max": 24.98468017578125,
      "activations/layer19_attention_weight_min": -24.147525787353516,
      "activations/layer1_attention_weight_max": 17.788558959960938,
      "activations/layer1_attention_weight_min": -14.6581392288208,
      "activations/layer20_attention_weight_max": 24.261926651000977,
      "activations/layer20_attention_weight_min": -24.13449478149414,
      "activations/layer21_attention_weight_max": 39.337677001953125,
      "activations/layer21_attention_weight_min": -23.045270919799805,
      "activations/layer22_attention_weight_max": 28.022626876831055,
      "activations/layer22_attention_weight_min": -24.805816650390625,
      "activations/layer23_attention_weight_max": 38.77125930786133,
      "activations/layer23_attention_weight_min": -23.907459259033203,
      "activations/layer2_attention_weight_max": 31.069046020507812,
      "activations/layer2_attention_weight_min": -31.255329132080078,
      "activations/layer3_attention_weight_max": 90.11181640625,
      "activations/layer3_attention_weight_min": -94.51448822021484,
      "activations/layer4_attention_weight_max": 92.67809295654297,
      "activations/layer4_attention_weight_min": -91.4256820678711,
      "activations/layer5_attention_weight_max": 69.05754089355469,
      "activations/layer5_attention_weight_min": -76.94290161132812,
      "activations/layer6_attention_weight_max": 51.55228042602539,
      "activations/layer6_attention_weight_min": -53.967124938964844,
      "activations/layer7_attention_weight_max": 66.29004669189453,
      "activations/layer7_attention_weight_min": -67.51214599609375,
      "activations/layer8_attention_weight_max": 45.29157257080078,
      "activations/layer8_attention_weight_min": -47.58891677856445,
      "activations/layer9_attention_weight_max": 46.38600540161133,
      "activations/layer9_attention_weight_min": -48.03268814086914,
      "epoch": 13.03,
      "learning_rate": 6.66246212121212e-05,
      "loss": 2.7725,
      "step": 224300
    },
    {
      "activations/layer0_attention_weight_max": 15.858037948608398,
      "activations/layer0_attention_weight_min": -13.627514839172363,
      "activations/layer10_attention_weight_max": 38.59357452392578,
      "activations/layer10_attention_weight_min": -36.81132888793945,
      "activations/layer11_attention_weight_max": 33.02934265136719,
      "activations/layer11_attention_weight_min": -34.750450134277344,
      "activations/layer12_attention_weight_max": 20.354019165039062,
      "activations/layer12_attention_weight_min": -24.310272216796875,
      "activations/layer13_attention_weight_max": 38.34624481201172,
      "activations/layer13_attention_weight_min": -31.825929641723633,
      "activations/layer14_attention_weight_max": 39.39918899536133,
      "activations/layer14_attention_weight_min": -32.84500503540039,
      "activations/layer15_attention_weight_max": 38.462913513183594,
      "activations/layer15_attention_weight_min": -30.297710418701172,
      "activations/layer16_attention_weight_max": 36.848350524902344,
      "activations/layer16_attention_weight_min": -35.43852996826172,
      "activations/layer17_attention_weight_max": 60.7523193359375,
      "activations/layer17_attention_weight_min": -53.25728225708008,
      "activations/layer18_attention_weight_max": 58.353939056396484,
      "activations/layer18_attention_weight_min": -43.41335678100586,
      "activations/layer19_attention_weight_max": 25.427326202392578,
      "activations/layer19_attention_weight_min": -21.44626235961914,
      "activations/layer1_attention_weight_max": 15.465907096862793,
      "activations/layer1_attention_weight_min": -14.091073036193848,
      "activations/layer20_attention_weight_max": 23.426055908203125,
      "activations/layer20_attention_weight_min": -22.274248123168945,
      "activations/layer21_attention_weight_max": 38.05976104736328,
      "activations/layer21_attention_weight_min": -20.77039909362793,
      "activations/layer22_attention_weight_max": 34.31825256347656,
      "activations/layer22_attention_weight_min": -23.915069580078125,
      "activations/layer23_attention_weight_max": 36.851009368896484,
      "activations/layer23_attention_weight_min": -22.887920379638672,
      "activations/layer2_attention_weight_max": 31.87392234802246,
      "activations/layer2_attention_weight_min": -29.95563507080078,
      "activations/layer3_attention_weight_max": 94.82809448242188,
      "activations/layer3_attention_weight_min": -101.46898651123047,
      "activations/layer4_attention_weight_max": 97.8513412475586,
      "activations/layer4_attention_weight_min": -96.90705108642578,
      "activations/layer5_attention_weight_max": 70.43400573730469,
      "activations/layer5_attention_weight_min": -77.5034408569336,
      "activations/layer6_attention_weight_max": 54.14521789550781,
      "activations/layer6_attention_weight_min": -56.28074264526367,
      "activations/layer7_attention_weight_max": 67.89005279541016,
      "activations/layer7_attention_weight_min": -67.7977523803711,
      "activations/layer8_attention_weight_max": 48.888038635253906,
      "activations/layer8_attention_weight_min": -50.37111282348633,
      "activations/layer9_attention_weight_max": 47.49677658081055,
      "activations/layer9_attention_weight_min": -48.30457305908203,
      "epoch": 13.04,
      "learning_rate": 6.660568181818182e-05,
      "loss": 2.7686,
      "step": 224350
    },
    {
      "activations/layer0_attention_weight_max": 16.680341720581055,
      "activations/layer0_attention_weight_min": -14.314217567443848,
      "activations/layer10_attention_weight_max": 34.141334533691406,
      "activations/layer10_attention_weight_min": -36.22816848754883,
      "activations/layer11_attention_weight_max": 31.94952392578125,
      "activations/layer11_attention_weight_min": -35.4178466796875,
      "activations/layer12_attention_weight_max": 18.39266586303711,
      "activations/layer12_attention_weight_min": -30.35230255126953,
      "activations/layer13_attention_weight_max": 42.141883850097656,
      "activations/layer13_attention_weight_min": -28.980188369750977,
      "activations/layer14_attention_weight_max": 42.315040588378906,
      "activations/layer14_attention_weight_min": -31.59215545654297,
      "activations/layer15_attention_weight_max": 34.323883056640625,
      "activations/layer15_attention_weight_min": -29.162126541137695,
      "activations/layer16_attention_weight_max": 34.76740264892578,
      "activations/layer16_attention_weight_min": -33.4477424621582,
      "activations/layer17_attention_weight_max": 55.3941764831543,
      "activations/layer17_attention_weight_min": -48.967708587646484,
      "activations/layer18_attention_weight_max": 50.759586334228516,
      "activations/layer18_attention_weight_min": -42.63445281982422,
      "activations/layer19_attention_weight_max": 24.559743881225586,
      "activations/layer19_attention_weight_min": -23.76145362854004,
      "activations/layer1_attention_weight_max": 16.926525115966797,
      "activations/layer1_attention_weight_min": -14.63390827178955,
      "activations/layer20_attention_weight_max": 28.1377010345459,
      "activations/layer20_attention_weight_min": -21.031375885009766,
      "activations/layer21_attention_weight_max": 44.21916580200195,
      "activations/layer21_attention_weight_min": -24.42000389099121,
      "activations/layer22_attention_weight_max": 36.05262756347656,
      "activations/layer22_attention_weight_min": -26.67571449279785,
      "activations/layer23_attention_weight_max": 42.187591552734375,
      "activations/layer23_attention_weight_min": -28.068950653076172,
      "activations/layer2_attention_weight_max": 30.911121368408203,
      "activations/layer2_attention_weight_min": -29.453474044799805,
      "activations/layer3_attention_weight_max": 98.57588958740234,
      "activations/layer3_attention_weight_min": -101.20659637451172,
      "activations/layer4_attention_weight_max": 93.59910583496094,
      "activations/layer4_attention_weight_min": -88.52249145507812,
      "activations/layer5_attention_weight_max": 67.48616027832031,
      "activations/layer5_attention_weight_min": -78.28540802001953,
      "activations/layer6_attention_weight_max": 50.17250442504883,
      "activations/layer6_attention_weight_min": -48.98875427246094,
      "activations/layer7_attention_weight_max": 69.83045959472656,
      "activations/layer7_attention_weight_min": -64.49337768554688,
      "activations/layer8_attention_weight_max": 42.34846878051758,
      "activations/layer8_attention_weight_min": -46.83383560180664,
      "activations/layer9_attention_weight_max": 46.20309829711914,
      "activations/layer9_attention_weight_min": -45.61038589477539,
      "epoch": 13.04,
      "learning_rate": 6.658674242424242e-05,
      "loss": 2.762,
      "step": 224400
    },
    {
      "activations/layer0_attention_weight_max": 15.74428939819336,
      "activations/layer0_attention_weight_min": -13.90339469909668,
      "activations/layer10_attention_weight_max": 34.64594268798828,
      "activations/layer10_attention_weight_min": -34.84788513183594,
      "activations/layer11_attention_weight_max": 31.76434326171875,
      "activations/layer11_attention_weight_min": -32.68638229370117,
      "activations/layer12_attention_weight_max": 18.01771354675293,
      "activations/layer12_attention_weight_min": -26.34722328186035,
      "activations/layer13_attention_weight_max": 39.423152923583984,
      "activations/layer13_attention_weight_min": -27.473724365234375,
      "activations/layer14_attention_weight_max": 37.544822692871094,
      "activations/layer14_attention_weight_min": -29.16497230529785,
      "activations/layer15_attention_weight_max": 35.137840270996094,
      "activations/layer15_attention_weight_min": -29.071455001831055,
      "activations/layer16_attention_weight_max": 32.898521423339844,
      "activations/layer16_attention_weight_min": -28.821706771850586,
      "activations/layer17_attention_weight_max": 52.580265045166016,
      "activations/layer17_attention_weight_min": -43.61127853393555,
      "activations/layer18_attention_weight_max": 49.95603561401367,
      "activations/layer18_attention_weight_min": -36.429378509521484,
      "activations/layer19_attention_weight_max": 23.561447143554688,
      "activations/layer19_attention_weight_min": -21.976408004760742,
      "activations/layer1_attention_weight_max": 16.580228805541992,
      "activations/layer1_attention_weight_min": -16.902992248535156,
      "activations/layer20_attention_weight_max": 22.998889923095703,
      "activations/layer20_attention_weight_min": -19.984270095825195,
      "activations/layer21_attention_weight_max": 41.30878448486328,
      "activations/layer21_attention_weight_min": -22.90622901916504,
      "activations/layer22_attention_weight_max": 33.81648254394531,
      "activations/layer22_attention_weight_min": -24.65021324157715,
      "activations/layer23_attention_weight_max": 46.506752014160156,
      "activations/layer23_attention_weight_min": -25.36475944519043,
      "activations/layer2_attention_weight_max": 32.07660675048828,
      "activations/layer2_attention_weight_min": -29.790863037109375,
      "activations/layer3_attention_weight_max": 93.78092956542969,
      "activations/layer3_attention_weight_min": -95.1043701171875,
      "activations/layer4_attention_weight_max": 97.81333923339844,
      "activations/layer4_attention_weight_min": -91.78356170654297,
      "activations/layer5_attention_weight_max": 71.95347595214844,
      "activations/layer5_attention_weight_min": -81.82144927978516,
      "activations/layer6_attention_weight_max": 51.86939239501953,
      "activations/layer6_attention_weight_min": -50.90827178955078,
      "activations/layer7_attention_weight_max": 67.59700012207031,
      "activations/layer7_attention_weight_min": -65.05840301513672,
      "activations/layer8_attention_weight_max": 46.28212356567383,
      "activations/layer8_attention_weight_min": -49.90419006347656,
      "activations/layer9_attention_weight_max": 44.34598922729492,
      "activations/layer9_attention_weight_min": -44.73832702636719,
      "epoch": 13.04,
      "learning_rate": 6.656780303030302e-05,
      "loss": 2.7618,
      "step": 224450
    },
    {
      "activations/layer0_attention_weight_max": 15.163822174072266,
      "activations/layer0_attention_weight_min": -13.901955604553223,
      "activations/layer10_attention_weight_max": 38.92374801635742,
      "activations/layer10_attention_weight_min": -36.48116683959961,
      "activations/layer11_attention_weight_max": 34.73167037963867,
      "activations/layer11_attention_weight_min": -32.47614669799805,
      "activations/layer12_attention_weight_max": 18.99293327331543,
      "activations/layer12_attention_weight_min": -23.61991310119629,
      "activations/layer13_attention_weight_max": 40.88703536987305,
      "activations/layer13_attention_weight_min": -33.471858978271484,
      "activations/layer14_attention_weight_max": 37.518638610839844,
      "activations/layer14_attention_weight_min": -29.64897346496582,
      "activations/layer15_attention_weight_max": 34.274574279785156,
      "activations/layer15_attention_weight_min": -29.80592918395996,
      "activations/layer16_attention_weight_max": 32.74532699584961,
      "activations/layer16_attention_weight_min": -30.065404891967773,
      "activations/layer17_attention_weight_max": 54.160030364990234,
      "activations/layer17_attention_weight_min": -45.35907745361328,
      "activations/layer18_attention_weight_max": 46.03083419799805,
      "activations/layer18_attention_weight_min": -38.78097915649414,
      "activations/layer19_attention_weight_max": 23.057960510253906,
      "activations/layer19_attention_weight_min": -22.966495513916016,
      "activations/layer1_attention_weight_max": 16.1959228515625,
      "activations/layer1_attention_weight_min": -14.758550643920898,
      "activations/layer20_attention_weight_max": 20.70030975341797,
      "activations/layer20_attention_weight_min": -23.699621200561523,
      "activations/layer21_attention_weight_max": 33.84999465942383,
      "activations/layer21_attention_weight_min": -21.148000717163086,
      "activations/layer22_attention_weight_max": 28.621858596801758,
      "activations/layer22_attention_weight_min": -26.394163131713867,
      "activations/layer23_attention_weight_max": 36.514488220214844,
      "activations/layer23_attention_weight_min": -23.829479217529297,
      "activations/layer2_attention_weight_max": 33.572967529296875,
      "activations/layer2_attention_weight_min": -32.594940185546875,
      "activations/layer3_attention_weight_max": 99.33321380615234,
      "activations/layer3_attention_weight_min": -103.67826080322266,
      "activations/layer4_attention_weight_max": 99.98345184326172,
      "activations/layer4_attention_weight_min": -98.23409271240234,
      "activations/layer5_attention_weight_max": 75.08155822753906,
      "activations/layer5_attention_weight_min": -81.93927001953125,
      "activations/layer6_attention_weight_max": 52.734458923339844,
      "activations/layer6_attention_weight_min": -54.540916442871094,
      "activations/layer7_attention_weight_max": 70.65400695800781,
      "activations/layer7_attention_weight_min": -67.21971893310547,
      "activations/layer8_attention_weight_max": 48.72305679321289,
      "activations/layer8_attention_weight_min": -45.43465042114258,
      "activations/layer9_attention_weight_max": 47.81743621826172,
      "activations/layer9_attention_weight_min": -45.66718292236328,
      "epoch": 13.04,
      "learning_rate": 6.654886363636362e-05,
      "loss": 2.7643,
      "step": 224500
    },
    {
      "activations/layer0_attention_weight_max": 16.518449783325195,
      "activations/layer0_attention_weight_min": -14.052643775939941,
      "activations/layer10_attention_weight_max": 34.33755874633789,
      "activations/layer10_attention_weight_min": -36.32754898071289,
      "activations/layer11_attention_weight_max": 31.898801803588867,
      "activations/layer11_attention_weight_min": -32.120208740234375,
      "activations/layer12_attention_weight_max": 15.963069915771484,
      "activations/layer12_attention_weight_min": -24.533767700195312,
      "activations/layer13_attention_weight_max": 33.24968338012695,
      "activations/layer13_attention_weight_min": -25.576885223388672,
      "activations/layer14_attention_weight_max": 35.52743911743164,
      "activations/layer14_attention_weight_min": -30.418148040771484,
      "activations/layer15_attention_weight_max": 33.564510345458984,
      "activations/layer15_attention_weight_min": -30.210847854614258,
      "activations/layer16_attention_weight_max": 29.879377365112305,
      "activations/layer16_attention_weight_min": -29.544940948486328,
      "activations/layer17_attention_weight_max": 50.737728118896484,
      "activations/layer17_attention_weight_min": -43.618370056152344,
      "activations/layer18_attention_weight_max": 41.641693115234375,
      "activations/layer18_attention_weight_min": -37.782997131347656,
      "activations/layer19_attention_weight_max": 22.599563598632812,
      "activations/layer19_attention_weight_min": -21.476985931396484,
      "activations/layer1_attention_weight_max": 16.73053550720215,
      "activations/layer1_attention_weight_min": -14.307554244995117,
      "activations/layer20_attention_weight_max": 19.771015167236328,
      "activations/layer20_attention_weight_min": -24.472576141357422,
      "activations/layer21_attention_weight_max": 33.54727554321289,
      "activations/layer21_attention_weight_min": -21.202478408813477,
      "activations/layer22_attention_weight_max": 31.35191535949707,
      "activations/layer22_attention_weight_min": -27.35130500793457,
      "activations/layer23_attention_weight_max": 35.572349548339844,
      "activations/layer23_attention_weight_min": -22.755516052246094,
      "activations/layer2_attention_weight_max": 32.31725311279297,
      "activations/layer2_attention_weight_min": -30.209980010986328,
      "activations/layer3_attention_weight_max": 95.01556396484375,
      "activations/layer3_attention_weight_min": -91.749267578125,
      "activations/layer4_attention_weight_max": 94.58173370361328,
      "activations/layer4_attention_weight_min": -88.6245346069336,
      "activations/layer5_attention_weight_max": 70.44336700439453,
      "activations/layer5_attention_weight_min": -76.78504943847656,
      "activations/layer6_attention_weight_max": 51.55289840698242,
      "activations/layer6_attention_weight_min": -51.308753967285156,
      "activations/layer7_attention_weight_max": 65.78600311279297,
      "activations/layer7_attention_weight_min": -64.54058837890625,
      "activations/layer8_attention_weight_max": 44.396568298339844,
      "activations/layer8_attention_weight_min": -45.193382263183594,
      "activations/layer9_attention_weight_max": 43.678462982177734,
      "activations/layer9_attention_weight_min": -46.80739212036133,
      "epoch": 13.05,
      "learning_rate": 6.652992424242424e-05,
      "loss": 2.7725,
      "step": 224550
    },
    {
      "activations/layer0_attention_weight_max": 16.170730590820312,
      "activations/layer0_attention_weight_min": -14.294573783874512,
      "activations/layer10_attention_weight_max": 32.81801223754883,
      "activations/layer10_attention_weight_min": -33.37822723388672,
      "activations/layer11_attention_weight_max": 29.94475555419922,
      "activations/layer11_attention_weight_min": -31.924434661865234,
      "activations/layer12_attention_weight_max": 18.1801700592041,
      "activations/layer12_attention_weight_min": -22.504838943481445,
      "activations/layer13_attention_weight_max": 34.030330657958984,
      "activations/layer13_attention_weight_min": -27.190204620361328,
      "activations/layer14_attention_weight_max": 32.66374206542969,
      "activations/layer14_attention_weight_min": -29.315431594848633,
      "activations/layer15_attention_weight_max": 30.406726837158203,
      "activations/layer15_attention_weight_min": -28.209354400634766,
      "activations/layer16_attention_weight_max": 28.99154281616211,
      "activations/layer16_attention_weight_min": -29.002063751220703,
      "activations/layer17_attention_weight_max": 47.158348083496094,
      "activations/layer17_attention_weight_min": -40.84721755981445,
      "activations/layer18_attention_weight_max": 41.05678939819336,
      "activations/layer18_attention_weight_min": -35.437843322753906,
      "activations/layer19_attention_weight_max": 20.698261260986328,
      "activations/layer19_attention_weight_min": -20.247940063476562,
      "activations/layer1_attention_weight_max": 17.150577545166016,
      "activations/layer1_attention_weight_min": -14.53941535949707,
      "activations/layer20_attention_weight_max": 22.964860916137695,
      "activations/layer20_attention_weight_min": -21.024259567260742,
      "activations/layer21_attention_weight_max": 33.94315719604492,
      "activations/layer21_attention_weight_min": -25.397069931030273,
      "activations/layer22_attention_weight_max": 29.700788497924805,
      "activations/layer22_attention_weight_min": -25.108125686645508,
      "activations/layer23_attention_weight_max": 39.2237548828125,
      "activations/layer23_attention_weight_min": -26.283100128173828,
      "activations/layer2_attention_weight_max": 33.039955139160156,
      "activations/layer2_attention_weight_min": -32.36784362792969,
      "activations/layer3_attention_weight_max": 96.40657806396484,
      "activations/layer3_attention_weight_min": -99.79676818847656,
      "activations/layer4_attention_weight_max": 95.69585418701172,
      "activations/layer4_attention_weight_min": -93.11067199707031,
      "activations/layer5_attention_weight_max": 67.41104888916016,
      "activations/layer5_attention_weight_min": -83.0594482421875,
      "activations/layer6_attention_weight_max": 52.384117126464844,
      "activations/layer6_attention_weight_min": -53.54677200317383,
      "activations/layer7_attention_weight_max": 63.51045227050781,
      "activations/layer7_attention_weight_min": -64.8525161743164,
      "activations/layer8_attention_weight_max": 43.348915100097656,
      "activations/layer8_attention_weight_min": -44.59507369995117,
      "activations/layer9_attention_weight_max": 43.91065216064453,
      "activations/layer9_attention_weight_min": -45.66560363769531,
      "epoch": 13.05,
      "learning_rate": 6.651098484848484e-05,
      "loss": 2.7598,
      "step": 224600
    },
    {
      "activations/layer0_attention_weight_max": 14.838241577148438,
      "activations/layer0_attention_weight_min": -14.131389617919922,
      "activations/layer10_attention_weight_max": 37.50190734863281,
      "activations/layer10_attention_weight_min": -35.01814651489258,
      "activations/layer11_attention_weight_max": 31.85286521911621,
      "activations/layer11_attention_weight_min": -32.68125915527344,
      "activations/layer12_attention_weight_max": 18.374027252197266,
      "activations/layer12_attention_weight_min": -26.83472442626953,
      "activations/layer13_attention_weight_max": 33.131832122802734,
      "activations/layer13_attention_weight_min": -27.455081939697266,
      "activations/layer14_attention_weight_max": 38.9191780090332,
      "activations/layer14_attention_weight_min": -30.131746292114258,
      "activations/layer15_attention_weight_max": 33.73307800292969,
      "activations/layer15_attention_weight_min": -29.164480209350586,
      "activations/layer16_attention_weight_max": 34.012901306152344,
      "activations/layer16_attention_weight_min": -30.322282791137695,
      "activations/layer17_attention_weight_max": 52.90861511230469,
      "activations/layer17_attention_weight_min": -46.21953201293945,
      "activations/layer18_attention_weight_max": 47.34109115600586,
      "activations/layer18_attention_weight_min": -37.377925872802734,
      "activations/layer19_attention_weight_max": 21.235485076904297,
      "activations/layer19_attention_weight_min": -20.234525680541992,
      "activations/layer1_attention_weight_max": 17.83991241455078,
      "activations/layer1_attention_weight_min": -15.726920127868652,
      "activations/layer20_attention_weight_max": 23.126605987548828,
      "activations/layer20_attention_weight_min": -20.134336471557617,
      "activations/layer21_attention_weight_max": 39.361751556396484,
      "activations/layer21_attention_weight_min": -20.64076805114746,
      "activations/layer22_attention_weight_max": 29.66473388671875,
      "activations/layer22_attention_weight_min": -24.98276710510254,
      "activations/layer23_attention_weight_max": 39.757816314697266,
      "activations/layer23_attention_weight_min": -23.31399917602539,
      "activations/layer2_attention_weight_max": 31.63467788696289,
      "activations/layer2_attention_weight_min": -29.63568115234375,
      "activations/layer3_attention_weight_max": 92.3264389038086,
      "activations/layer3_attention_weight_min": -93.90265655517578,
      "activations/layer4_attention_weight_max": 94.45397186279297,
      "activations/layer4_attention_weight_min": -91.87662506103516,
      "activations/layer5_attention_weight_max": 74.12389373779297,
      "activations/layer5_attention_weight_min": -75.45755767822266,
      "activations/layer6_attention_weight_max": 52.64747619628906,
      "activations/layer6_attention_weight_min": -54.33913040161133,
      "activations/layer7_attention_weight_max": 69.93773651123047,
      "activations/layer7_attention_weight_min": -67.76116943359375,
      "activations/layer8_attention_weight_max": 46.84412384033203,
      "activations/layer8_attention_weight_min": -50.52456283569336,
      "activations/layer9_attention_weight_max": 48.88174057006836,
      "activations/layer9_attention_weight_min": -49.654964447021484,
      "epoch": 13.05,
      "learning_rate": 6.649204545454544e-05,
      "loss": 2.751,
      "step": 224650
    },
    {
      "activations/layer0_attention_weight_max": 16.558786392211914,
      "activations/layer0_attention_weight_min": -13.470731735229492,
      "activations/layer10_attention_weight_max": 35.699668884277344,
      "activations/layer10_attention_weight_min": -34.57057189941406,
      "activations/layer11_attention_weight_max": 31.43608856201172,
      "activations/layer11_attention_weight_min": -31.431987762451172,
      "activations/layer12_attention_weight_max": 17.515708923339844,
      "activations/layer12_attention_weight_min": -26.222562789916992,
      "activations/layer13_attention_weight_max": 35.6981086730957,
      "activations/layer13_attention_weight_min": -29.080322265625,
      "activations/layer14_attention_weight_max": 33.77873611450195,
      "activations/layer14_attention_weight_min": -28.34063720703125,
      "activations/layer15_attention_weight_max": 30.274961471557617,
      "activations/layer15_attention_weight_min": -29.161916732788086,
      "activations/layer16_attention_weight_max": 28.52716636657715,
      "activations/layer16_attention_weight_min": -27.684490203857422,
      "activations/layer17_attention_weight_max": 49.94400405883789,
      "activations/layer17_attention_weight_min": -42.614566802978516,
      "activations/layer18_attention_weight_max": 44.591949462890625,
      "activations/layer18_attention_weight_min": -37.56412124633789,
      "activations/layer19_attention_weight_max": 20.696529388427734,
      "activations/layer19_attention_weight_min": -21.855224609375,
      "activations/layer1_attention_weight_max": 16.31661605834961,
      "activations/layer1_attention_weight_min": -14.04395866394043,
      "activations/layer20_attention_weight_max": 21.17143440246582,
      "activations/layer20_attention_weight_min": -24.0137996673584,
      "activations/layer21_attention_weight_max": 32.88081741333008,
      "activations/layer21_attention_weight_min": -24.566293716430664,
      "activations/layer22_attention_weight_max": 27.741209030151367,
      "activations/layer22_attention_weight_min": -25.61629867553711,
      "activations/layer23_attention_weight_max": 34.313079833984375,
      "activations/layer23_attention_weight_min": -25.49523162841797,
      "activations/layer2_attention_weight_max": 32.45978927612305,
      "activations/layer2_attention_weight_min": -31.05266571044922,
      "activations/layer3_attention_weight_max": 96.13848876953125,
      "activations/layer3_attention_weight_min": -98.8516616821289,
      "activations/layer4_attention_weight_max": 93.74581909179688,
      "activations/layer4_attention_weight_min": -93.01459503173828,
      "activations/layer5_attention_weight_max": 71.59384155273438,
      "activations/layer5_attention_weight_min": -75.90245056152344,
      "activations/layer6_attention_weight_max": 52.38656234741211,
      "activations/layer6_attention_weight_min": -51.747169494628906,
      "activations/layer7_attention_weight_max": 68.68141174316406,
      "activations/layer7_attention_weight_min": -66.20540618896484,
      "activations/layer8_attention_weight_max": 45.667720794677734,
      "activations/layer8_attention_weight_min": -45.66722869873047,
      "activations/layer9_attention_weight_max": 44.409915924072266,
      "activations/layer9_attention_weight_min": -46.15290451049805,
      "epoch": 13.06,
      "learning_rate": 6.647310606060606e-05,
      "loss": 2.7617,
      "step": 224700
    },
    {
      "activations/layer0_attention_weight_max": 15.068032264709473,
      "activations/layer0_attention_weight_min": -13.621664047241211,
      "activations/layer10_attention_weight_max": 37.307861328125,
      "activations/layer10_attention_weight_min": -35.92634582519531,
      "activations/layer11_attention_weight_max": 32.248329162597656,
      "activations/layer11_attention_weight_min": -34.661258697509766,
      "activations/layer12_attention_weight_max": 19.455516815185547,
      "activations/layer12_attention_weight_min": -24.654464721679688,
      "activations/layer13_attention_weight_max": 33.76612091064453,
      "activations/layer13_attention_weight_min": -30.393712997436523,
      "activations/layer14_attention_weight_max": 34.70024871826172,
      "activations/layer14_attention_weight_min": -31.773591995239258,
      "activations/layer15_attention_weight_max": 35.31375503540039,
      "activations/layer15_attention_weight_min": -30.181621551513672,
      "activations/layer16_attention_weight_max": 33.89201736450195,
      "activations/layer16_attention_weight_min": -29.777868270874023,
      "activations/layer17_attention_weight_max": 51.72816467285156,
      "activations/layer17_attention_weight_min": -45.3518180847168,
      "activations/layer18_attention_weight_max": 49.0594596862793,
      "activations/layer18_attention_weight_min": -39.44243621826172,
      "activations/layer19_attention_weight_max": 23.14007568359375,
      "activations/layer19_attention_weight_min": -21.783100128173828,
      "activations/layer1_attention_weight_max": 18.125436782836914,
      "activations/layer1_attention_weight_min": -14.472299575805664,
      "activations/layer20_attention_weight_max": 21.515037536621094,
      "activations/layer20_attention_weight_min": -20.893938064575195,
      "activations/layer21_attention_weight_max": 38.00123977661133,
      "activations/layer21_attention_weight_min": -25.35245132446289,
      "activations/layer22_attention_weight_max": 30.075063705444336,
      "activations/layer22_attention_weight_min": -25.48455047607422,
      "activations/layer23_attention_weight_max": 39.06580352783203,
      "activations/layer23_attention_weight_min": -28.19666290283203,
      "activations/layer2_attention_weight_max": 31.259063720703125,
      "activations/layer2_attention_weight_min": -30.906185150146484,
      "activations/layer3_attention_weight_max": 96.80937194824219,
      "activations/layer3_attention_weight_min": -95.56221008300781,
      "activations/layer4_attention_weight_max": 94.81002044677734,
      "activations/layer4_attention_weight_min": -92.23934173583984,
      "activations/layer5_attention_weight_max": 73.03311157226562,
      "activations/layer5_attention_weight_min": -73.07157897949219,
      "activations/layer6_attention_weight_max": 51.08287048339844,
      "activations/layer6_attention_weight_min": -49.517189025878906,
      "activations/layer7_attention_weight_max": 66.18612670898438,
      "activations/layer7_attention_weight_min": -63.7739143371582,
      "activations/layer8_attention_weight_max": 46.586395263671875,
      "activations/layer8_attention_weight_min": -49.30110549926758,
      "activations/layer9_attention_weight_max": 47.01083755493164,
      "activations/layer9_attention_weight_min": -47.958377838134766,
      "epoch": 13.06,
      "learning_rate": 6.645416666666666e-05,
      "loss": 2.7718,
      "step": 224750
    },
    {
      "activations/layer0_attention_weight_max": 15.720520973205566,
      "activations/layer0_attention_weight_min": -13.87459945678711,
      "activations/layer10_attention_weight_max": 35.000553131103516,
      "activations/layer10_attention_weight_min": -34.66543960571289,
      "activations/layer11_attention_weight_max": 31.689970016479492,
      "activations/layer11_attention_weight_min": -32.68304443359375,
      "activations/layer12_attention_weight_max": 19.155399322509766,
      "activations/layer12_attention_weight_min": -26.283973693847656,
      "activations/layer13_attention_weight_max": 38.16264343261719,
      "activations/layer13_attention_weight_min": -30.674413681030273,
      "activations/layer14_attention_weight_max": 37.40699768066406,
      "activations/layer14_attention_weight_min": -33.60936737060547,
      "activations/layer15_attention_weight_max": 34.13640213012695,
      "activations/layer15_attention_weight_min": -32.28993225097656,
      "activations/layer16_attention_weight_max": 33.88315200805664,
      "activations/layer16_attention_weight_min": -32.39491653442383,
      "activations/layer17_attention_weight_max": 53.3587532043457,
      "activations/layer17_attention_weight_min": -45.444061279296875,
      "activations/layer18_attention_weight_max": 46.01802444458008,
      "activations/layer18_attention_weight_min": -38.650604248046875,
      "activations/layer19_attention_weight_max": 22.70601463317871,
      "activations/layer19_attention_weight_min": -21.28147315979004,
      "activations/layer1_attention_weight_max": 16.000410079956055,
      "activations/layer1_attention_weight_min": -14.619248390197754,
      "activations/layer20_attention_weight_max": 21.687191009521484,
      "activations/layer20_attention_weight_min": -21.393558502197266,
      "activations/layer21_attention_weight_max": 34.82427978515625,
      "activations/layer21_attention_weight_min": -24.793859481811523,
      "activations/layer22_attention_weight_max": 31.67681121826172,
      "activations/layer22_attention_weight_min": -27.423383712768555,
      "activations/layer23_attention_weight_max": 37.32979202270508,
      "activations/layer23_attention_weight_min": -24.187198638916016,
      "activations/layer2_attention_weight_max": 32.32649612426758,
      "activations/layer2_attention_weight_min": -30.217662811279297,
      "activations/layer3_attention_weight_max": 93.18499755859375,
      "activations/layer3_attention_weight_min": -104.05994415283203,
      "activations/layer4_attention_weight_max": 93.47884368896484,
      "activations/layer4_attention_weight_min": -93.86886596679688,
      "activations/layer5_attention_weight_max": 73.82146453857422,
      "activations/layer5_attention_weight_min": -77.6927490234375,
      "activations/layer6_attention_weight_max": 51.61414337158203,
      "activations/layer6_attention_weight_min": -53.006507873535156,
      "activations/layer7_attention_weight_max": 66.02870178222656,
      "activations/layer7_attention_weight_min": -64.42413330078125,
      "activations/layer8_attention_weight_max": 46.15757751464844,
      "activations/layer8_attention_weight_min": -45.807289123535156,
      "activations/layer9_attention_weight_max": 47.39562225341797,
      "activations/layer9_attention_weight_min": -44.707523345947266,
      "epoch": 13.06,
      "learning_rate": 6.643522727272727e-05,
      "loss": 2.7397,
      "step": 224800
    },
    {
      "activations/layer0_attention_weight_max": 16.09487533569336,
      "activations/layer0_attention_weight_min": -13.870609283447266,
      "activations/layer10_attention_weight_max": 35.06251525878906,
      "activations/layer10_attention_weight_min": -35.91553497314453,
      "activations/layer11_attention_weight_max": 32.56175231933594,
      "activations/layer11_attention_weight_min": -34.87378692626953,
      "activations/layer12_attention_weight_max": 17.668621063232422,
      "activations/layer12_attention_weight_min": -23.3524227142334,
      "activations/layer13_attention_weight_max": 34.9146614074707,
      "activations/layer13_attention_weight_min": -29.250171661376953,
      "activations/layer14_attention_weight_max": 36.80451965332031,
      "activations/layer14_attention_weight_min": -30.031267166137695,
      "activations/layer15_attention_weight_max": 31.902441024780273,
      "activations/layer15_attention_weight_min": -28.13016700744629,
      "activations/layer16_attention_weight_max": 30.35778045654297,
      "activations/layer16_attention_weight_min": -28.545915603637695,
      "activations/layer17_attention_weight_max": 49.49809265136719,
      "activations/layer17_attention_weight_min": -43.4763069152832,
      "activations/layer18_attention_weight_max": 47.08186721801758,
      "activations/layer18_attention_weight_min": -36.25678634643555,
      "activations/layer19_attention_weight_max": 21.72743797302246,
      "activations/layer19_attention_weight_min": -21.05083656311035,
      "activations/layer1_attention_weight_max": 16.685609817504883,
      "activations/layer1_attention_weight_min": -15.105330467224121,
      "activations/layer20_attention_weight_max": 23.62464141845703,
      "activations/layer20_attention_weight_min": -22.587064743041992,
      "activations/layer21_attention_weight_max": 41.61371612548828,
      "activations/layer21_attention_weight_min": -23.701824188232422,
      "activations/layer22_attention_weight_max": 30.80429458618164,
      "activations/layer22_attention_weight_min": -24.78060531616211,
      "activations/layer23_attention_weight_max": 37.00082778930664,
      "activations/layer23_attention_weight_min": -23.721981048583984,
      "activations/layer2_attention_weight_max": 32.261260986328125,
      "activations/layer2_attention_weight_min": -30.329788208007812,
      "activations/layer3_attention_weight_max": 96.9896469116211,
      "activations/layer3_attention_weight_min": -99.55471801757812,
      "activations/layer4_attention_weight_max": 99.77501678466797,
      "activations/layer4_attention_weight_min": -96.08721923828125,
      "activations/layer5_attention_weight_max": 75.89704132080078,
      "activations/layer5_attention_weight_min": -83.21244812011719,
      "activations/layer6_attention_weight_max": 54.70600509643555,
      "activations/layer6_attention_weight_min": -52.17428207397461,
      "activations/layer7_attention_weight_max": 77.76272583007812,
      "activations/layer7_attention_weight_min": -65.27425384521484,
      "activations/layer8_attention_weight_max": 46.877716064453125,
      "activations/layer8_attention_weight_min": -48.11720275878906,
      "activations/layer9_attention_weight_max": 48.47156524658203,
      "activations/layer9_attention_weight_min": -45.31947326660156,
      "epoch": 13.07,
      "learning_rate": 6.641628787878787e-05,
      "loss": 2.7761,
      "step": 224850
    },
    {
      "activations/layer0_attention_weight_max": 16.635908126831055,
      "activations/layer0_attention_weight_min": -13.57961368560791,
      "activations/layer10_attention_weight_max": 44.42097091674805,
      "activations/layer10_attention_weight_min": -42.31584548950195,
      "activations/layer11_attention_weight_max": 39.45539855957031,
      "activations/layer11_attention_weight_min": -36.705196380615234,
      "activations/layer12_attention_weight_max": 19.975601196289062,
      "activations/layer12_attention_weight_min": -28.953195571899414,
      "activations/layer13_attention_weight_max": 40.47399139404297,
      "activations/layer13_attention_weight_min": -29.020448684692383,
      "activations/layer14_attention_weight_max": 39.03127670288086,
      "activations/layer14_attention_weight_min": -32.14443588256836,
      "activations/layer15_attention_weight_max": 35.16227722167969,
      "activations/layer15_attention_weight_min": -29.844377517700195,
      "activations/layer16_attention_weight_max": 37.324249267578125,
      "activations/layer16_attention_weight_min": -34.44733810424805,
      "activations/layer17_attention_weight_max": 59.20286178588867,
      "activations/layer17_attention_weight_min": -48.89047622680664,
      "activations/layer18_attention_weight_max": 50.6828727722168,
      "activations/layer18_attention_weight_min": -43.301666259765625,
      "activations/layer19_attention_weight_max": 23.766477584838867,
      "activations/layer19_attention_weight_min": -23.078704833984375,
      "activations/layer1_attention_weight_max": 17.28617286682129,
      "activations/layer1_attention_weight_min": -14.211441993713379,
      "activations/layer20_attention_weight_max": 24.4527645111084,
      "activations/layer20_attention_weight_min": -23.783157348632812,
      "activations/layer21_attention_weight_max": 39.693321228027344,
      "activations/layer21_attention_weight_min": -23.601470947265625,
      "activations/layer22_attention_weight_max": 31.183774948120117,
      "activations/layer22_attention_weight_min": -26.32275390625,
      "activations/layer23_attention_weight_max": 42.685665130615234,
      "activations/layer23_attention_weight_min": -24.92896842956543,
      "activations/layer2_attention_weight_max": 31.372190475463867,
      "activations/layer2_attention_weight_min": -31.00641441345215,
      "activations/layer3_attention_weight_max": 95.24246978759766,
      "activations/layer3_attention_weight_min": -99.64579772949219,
      "activations/layer4_attention_weight_max": 96.76913452148438,
      "activations/layer4_attention_weight_min": -94.04705047607422,
      "activations/layer5_attention_weight_max": 73.8707046508789,
      "activations/layer5_attention_weight_min": -78.71234130859375,
      "activations/layer6_attention_weight_max": 53.90792465209961,
      "activations/layer6_attention_weight_min": -51.46681594848633,
      "activations/layer7_attention_weight_max": 77.98965454101562,
      "activations/layer7_attention_weight_min": -71.20236206054688,
      "activations/layer8_attention_weight_max": 55.37361526489258,
      "activations/layer8_attention_weight_min": -49.98613739013672,
      "activations/layer9_attention_weight_max": 63.82639694213867,
      "activations/layer9_attention_weight_min": -52.45893478393555,
      "epoch": 13.07,
      "learning_rate": 6.639734848484848e-05,
      "loss": 2.7588,
      "step": 224900
    },
    {
      "activations/layer0_attention_weight_max": 16.100534439086914,
      "activations/layer0_attention_weight_min": -13.961101531982422,
      "activations/layer10_attention_weight_max": 32.79475784301758,
      "activations/layer10_attention_weight_min": -33.61480712890625,
      "activations/layer11_attention_weight_max": 28.900936126708984,
      "activations/layer11_attention_weight_min": -32.668052673339844,
      "activations/layer12_attention_weight_max": 17.993179321289062,
      "activations/layer12_attention_weight_min": -26.589799880981445,
      "activations/layer13_attention_weight_max": 33.75048065185547,
      "activations/layer13_attention_weight_min": -26.869218826293945,
      "activations/layer14_attention_weight_max": 33.7215461730957,
      "activations/layer14_attention_weight_min": -30.032752990722656,
      "activations/layer15_attention_weight_max": 30.40726661682129,
      "activations/layer15_attention_weight_min": -28.92153549194336,
      "activations/layer16_attention_weight_max": 32.222347259521484,
      "activations/layer16_attention_weight_min": -31.049360275268555,
      "activations/layer17_attention_weight_max": 53.41044998168945,
      "activations/layer17_attention_weight_min": -46.28543472290039,
      "activations/layer18_attention_weight_max": 48.00700759887695,
      "activations/layer18_attention_weight_min": -37.709075927734375,
      "activations/layer19_attention_weight_max": 23.770179748535156,
      "activations/layer19_attention_weight_min": -21.666296005249023,
      "activations/layer1_attention_weight_max": 16.094900131225586,
      "activations/layer1_attention_weight_min": -16.339385986328125,
      "activations/layer20_attention_weight_max": 22.176368713378906,
      "activations/layer20_attention_weight_min": -20.805503845214844,
      "activations/layer21_attention_weight_max": 33.46223449707031,
      "activations/layer21_attention_weight_min": -21.97336769104004,
      "activations/layer22_attention_weight_max": 28.9417781829834,
      "activations/layer22_attention_weight_min": -26.34952163696289,
      "activations/layer23_attention_weight_max": 36.027076721191406,
      "activations/layer23_attention_weight_min": -24.671634674072266,
      "activations/layer2_attention_weight_max": 31.598371505737305,
      "activations/layer2_attention_weight_min": -31.08243179321289,
      "activations/layer3_attention_weight_max": 95.47422790527344,
      "activations/layer3_attention_weight_min": -97.1720962524414,
      "activations/layer4_attention_weight_max": 97.08393859863281,
      "activations/layer4_attention_weight_min": -96.463623046875,
      "activations/layer5_attention_weight_max": 73.279541015625,
      "activations/layer5_attention_weight_min": -82.42532348632812,
      "activations/layer6_attention_weight_max": 49.79049301147461,
      "activations/layer6_attention_weight_min": -53.715476989746094,
      "activations/layer7_attention_weight_max": 66.58126831054688,
      "activations/layer7_attention_weight_min": -66.97270965576172,
      "activations/layer8_attention_weight_max": 44.94276809692383,
      "activations/layer8_attention_weight_min": -46.0606575012207,
      "activations/layer9_attention_weight_max": 44.40578842163086,
      "activations/layer9_attention_weight_min": -44.966365814208984,
      "epoch": 13.07,
      "learning_rate": 6.637840909090909e-05,
      "loss": 2.7634,
      "step": 224950
    },
    {
      "activations/layer0_attention_weight_max": 16.206562042236328,
      "activations/layer0_attention_weight_min": -13.782073020935059,
      "activations/layer10_attention_weight_max": 35.77553939819336,
      "activations/layer10_attention_weight_min": -36.270423889160156,
      "activations/layer11_attention_weight_max": 32.955787658691406,
      "activations/layer11_attention_weight_min": -33.42283630371094,
      "activations/layer12_attention_weight_max": 19.896127700805664,
      "activations/layer12_attention_weight_min": -25.46518325805664,
      "activations/layer13_attention_weight_max": 31.67792320251465,
      "activations/layer13_attention_weight_min": -28.45639991760254,
      "activations/layer14_attention_weight_max": 33.377464294433594,
      "activations/layer14_attention_weight_min": -29.893402099609375,
      "activations/layer15_attention_weight_max": 31.7372989654541,
      "activations/layer15_attention_weight_min": -28.900447845458984,
      "activations/layer16_attention_weight_max": 30.52356719970703,
      "activations/layer16_attention_weight_min": -29.86216926574707,
      "activations/layer17_attention_weight_max": 49.89164352416992,
      "activations/layer17_attention_weight_min": -43.7011604309082,
      "activations/layer18_attention_weight_max": 48.862754821777344,
      "activations/layer18_attention_weight_min": -36.95060729980469,
      "activations/layer19_attention_weight_max": 20.937387466430664,
      "activations/layer19_attention_weight_min": -21.131399154663086,
      "activations/layer1_attention_weight_max": 18.025798797607422,
      "activations/layer1_attention_weight_min": -16.573972702026367,
      "activations/layer20_attention_weight_max": 21.582035064697266,
      "activations/layer20_attention_weight_min": -22.515422821044922,
      "activations/layer21_attention_weight_max": 37.02325439453125,
      "activations/layer21_attention_weight_min": -22.897443771362305,
      "activations/layer22_attention_weight_max": 30.29456329345703,
      "activations/layer22_attention_weight_min": -25.843612670898438,
      "activations/layer23_attention_weight_max": 31.869731903076172,
      "activations/layer23_attention_weight_min": -22.754043579101562,
      "activations/layer2_attention_weight_max": 30.42209243774414,
      "activations/layer2_attention_weight_min": -28.78266716003418,
      "activations/layer3_attention_weight_max": 97.12574768066406,
      "activations/layer3_attention_weight_min": -95.07051086425781,
      "activations/layer4_attention_weight_max": 94.27313995361328,
      "activations/layer4_attention_weight_min": -94.77886962890625,
      "activations/layer5_attention_weight_max": 72.00033569335938,
      "activations/layer5_attention_weight_min": -77.93684387207031,
      "activations/layer6_attention_weight_max": 54.779197692871094,
      "activations/layer6_attention_weight_min": -53.623451232910156,
      "activations/layer7_attention_weight_max": 67.20111083984375,
      "activations/layer7_attention_weight_min": -66.74268341064453,
      "activations/layer8_attention_weight_max": 45.880645751953125,
      "activations/layer8_attention_weight_min": -47.4152717590332,
      "activations/layer9_attention_weight_max": 46.55526351928711,
      "activations/layer9_attention_weight_min": -45.156585693359375,
      "epoch": 13.07,
      "learning_rate": 6.635946969696969e-05,
      "loss": 2.7601,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.4555,
      "eval_samples_per_second": 507.836,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.4555,
      "eval_openwebtext_samples_per_second": 507.836,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 1.9587,
      "eval_wikitext_samples_per_second": 232.813,
      "step": 225000
    },
    {
      "epoch": 13.07,
      "eval_lambada_loss": 2.607421875,
      "eval_lambada_ppl": 13.564035954639001,
      "eval_lambada_runtime": 9.4986,
      "eval_lambada_samples_per_second": 512.6,
      "step": 225000
    },
    {
      "activations/layer0_attention_weight_max": 15.663165092468262,
      "activations/layer0_attention_weight_min": -14.054593086242676,
      "activations/layer10_attention_weight_max": 38.9847412109375,
      "activations/layer10_attention_weight_min": -37.21419906616211,
      "activations/layer11_attention_weight_max": 32.73289489746094,
      "activations/layer11_attention_weight_min": -32.20092010498047,
      "activations/layer12_attention_weight_max": 17.288761138916016,
      "activations/layer12_attention_weight_min": -29.242074966430664,
      "activations/layer13_attention_weight_max": 35.09153747558594,
      "activations/layer13_attention_weight_min": -28.9960880279541,
      "activations/layer14_attention_weight_max": 32.67538070678711,
      "activations/layer14_attention_weight_min": -29.047786712646484,
      "activations/layer15_attention_weight_max": 30.959430694580078,
      "activations/layer15_attention_weight_min": -28.840742111206055,
      "activations/layer16_attention_weight_max": 28.597763061523438,
      "activations/layer16_attention_weight_min": -27.90772819519043,
      "activations/layer17_attention_weight_max": 48.01347351074219,
      "activations/layer17_attention_weight_min": -44.59830856323242,
      "activations/layer18_attention_weight_max": 42.51289749145508,
      "activations/layer18_attention_weight_min": -37.573577880859375,
      "activations/layer19_attention_weight_max": 18.959487915039062,
      "activations/layer19_attention_weight_min": -20.596403121948242,
      "activations/layer1_attention_weight_max": 17.13545036315918,
      "activations/layer1_attention_weight_min": -14.642317771911621,
      "activations/layer20_attention_weight_max": 19.098562240600586,
      "activations/layer20_attention_weight_min": -23.015222549438477,
      "activations/layer21_attention_weight_max": 30.91888999938965,
      "activations/layer21_attention_weight_min": -23.808874130249023,
      "activations/layer22_attention_weight_max": 27.409557342529297,
      "activations/layer22_attention_weight_min": -24.441118240356445,
      "activations/layer23_attention_weight_max": 36.6429557800293,
      "activations/layer23_attention_weight_min": -23.741323471069336,
      "activations/layer2_attention_weight_max": 31.71902084350586,
      "activations/layer2_attention_weight_min": -32.18431091308594,
      "activations/layer3_attention_weight_max": 99.05943298339844,
      "activations/layer3_attention_weight_min": -98.4085464477539,
      "activations/layer4_attention_weight_max": 100.6019058227539,
      "activations/layer4_attention_weight_min": -92.39989471435547,
      "activations/layer5_attention_weight_max": 73.54269409179688,
      "activations/layer5_attention_weight_min": -76.9370346069336,
      "activations/layer6_attention_weight_max": 53.79230499267578,
      "activations/layer6_attention_weight_min": -54.56680679321289,
      "activations/layer7_attention_weight_max": 66.95643615722656,
      "activations/layer7_attention_weight_min": -67.22482299804688,
      "activations/layer8_attention_weight_max": 46.817771911621094,
      "activations/layer8_attention_weight_min": -48.88780975341797,
      "activations/layer9_attention_weight_max": 53.13997268676758,
      "activations/layer9_attention_weight_min": -52.54256820678711,
      "epoch": 13.08,
      "learning_rate": 6.63405303030303e-05,
      "loss": 2.7587,
      "step": 225050
    },
    {
      "activations/layer0_attention_weight_max": 15.875478744506836,
      "activations/layer0_attention_weight_min": -13.779029846191406,
      "activations/layer10_attention_weight_max": 40.105926513671875,
      "activations/layer10_attention_weight_min": -37.573036193847656,
      "activations/layer11_attention_weight_max": 32.475624084472656,
      "activations/layer11_attention_weight_min": -33.528167724609375,
      "activations/layer12_attention_weight_max": 18.007596969604492,
      "activations/layer12_attention_weight_min": -23.918901443481445,
      "activations/layer13_attention_weight_max": 38.10485076904297,
      "activations/layer13_attention_weight_min": -27.645572662353516,
      "activations/layer14_attention_weight_max": 36.104305267333984,
      "activations/layer14_attention_weight_min": -32.06792449951172,
      "activations/layer15_attention_weight_max": 32.622230529785156,
      "activations/layer15_attention_weight_min": -31.219200134277344,
      "activations/layer16_attention_weight_max": 31.913183212280273,
      "activations/layer16_attention_weight_min": -29.9461669921875,
      "activations/layer17_attention_weight_max": 50.71419143676758,
      "activations/layer17_attention_weight_min": -46.60036849975586,
      "activations/layer18_attention_weight_max": 48.37141036987305,
      "activations/layer18_attention_weight_min": -39.53640365600586,
      "activations/layer19_attention_weight_max": 22.817060470581055,
      "activations/layer19_attention_weight_min": -22.672683715820312,
      "activations/layer1_attention_weight_max": 16.42506980895996,
      "activations/layer1_attention_weight_min": -15.634963035583496,
      "activations/layer20_attention_weight_max": 23.921833038330078,
      "activations/layer20_attention_weight_min": -21.76865005493164,
      "activations/layer21_attention_weight_max": 35.75763702392578,
      "activations/layer21_attention_weight_min": -24.399587631225586,
      "activations/layer22_attention_weight_max": 29.13831901550293,
      "activations/layer22_attention_weight_min": -25.4134521484375,
      "activations/layer23_attention_weight_max": 38.16254806518555,
      "activations/layer23_attention_weight_min": -27.34511947631836,
      "activations/layer2_attention_weight_max": 32.16349411010742,
      "activations/layer2_attention_weight_min": -31.836185455322266,
      "activations/layer3_attention_weight_max": 96.5892562866211,
      "activations/layer3_attention_weight_min": -99.47974395751953,
      "activations/layer4_attention_weight_max": 99.32178497314453,
      "activations/layer4_attention_weight_min": -101.10343170166016,
      "activations/layer5_attention_weight_max": 71.79203796386719,
      "activations/layer5_attention_weight_min": -77.71391296386719,
      "activations/layer6_attention_weight_max": 54.51583480834961,
      "activations/layer6_attention_weight_min": -52.59674835205078,
      "activations/layer7_attention_weight_max": 67.93561553955078,
      "activations/layer7_attention_weight_min": -65.02630615234375,
      "activations/layer8_attention_weight_max": 46.52580642700195,
      "activations/layer8_attention_weight_min": -46.7820930480957,
      "activations/layer9_attention_weight_max": 45.735748291015625,
      "activations/layer9_attention_weight_min": -44.78578567504883,
      "epoch": 13.08,
      "learning_rate": 6.63215909090909e-05,
      "loss": 2.7612,
      "step": 225100
    },
    {
      "activations/layer0_attention_weight_max": 14.95114803314209,
      "activations/layer0_attention_weight_min": -13.891090393066406,
      "activations/layer10_attention_weight_max": 35.863887786865234,
      "activations/layer10_attention_weight_min": -34.076683044433594,
      "activations/layer11_attention_weight_max": 31.57339096069336,
      "activations/layer11_attention_weight_min": -32.66536331176758,
      "activations/layer12_attention_weight_max": 17.614656448364258,
      "activations/layer12_attention_weight_min": -27.869338989257812,
      "activations/layer13_attention_weight_max": 35.077239990234375,
      "activations/layer13_attention_weight_min": -28.010520935058594,
      "activations/layer14_attention_weight_max": 33.953826904296875,
      "activations/layer14_attention_weight_min": -29.186054229736328,
      "activations/layer15_attention_weight_max": 32.33608627319336,
      "activations/layer15_attention_weight_min": -27.145631790161133,
      "activations/layer16_attention_weight_max": 29.611526489257812,
      "activations/layer16_attention_weight_min": -27.391101837158203,
      "activations/layer17_attention_weight_max": 46.92329788208008,
      "activations/layer17_attention_weight_min": -39.14314651489258,
      "activations/layer18_attention_weight_max": 41.7626953125,
      "activations/layer18_attention_weight_min": -34.30180358886719,
      "activations/layer19_attention_weight_max": 20.180021286010742,
      "activations/layer19_attention_weight_min": -21.186437606811523,
      "activations/layer1_attention_weight_max": 16.405248641967773,
      "activations/layer1_attention_weight_min": -16.476438522338867,
      "activations/layer20_attention_weight_max": 21.573802947998047,
      "activations/layer20_attention_weight_min": -20.46440887451172,
      "activations/layer21_attention_weight_max": 27.94857406616211,
      "activations/layer21_attention_weight_min": -21.70225715637207,
      "activations/layer22_attention_weight_max": 28.932891845703125,
      "activations/layer22_attention_weight_min": -22.220195770263672,
      "activations/layer23_attention_weight_max": 36.98595428466797,
      "activations/layer23_attention_weight_min": -22.96880340576172,
      "activations/layer2_attention_weight_max": 30.932777404785156,
      "activations/layer2_attention_weight_min": -29.958106994628906,
      "activations/layer3_attention_weight_max": 94.78176879882812,
      "activations/layer3_attention_weight_min": -102.14673614501953,
      "activations/layer4_attention_weight_max": 94.56161499023438,
      "activations/layer4_attention_weight_min": -91.46428680419922,
      "activations/layer5_attention_weight_max": 70.21116638183594,
      "activations/layer5_attention_weight_min": -77.11549377441406,
      "activations/layer6_attention_weight_max": 50.916221618652344,
      "activations/layer6_attention_weight_min": -50.19027328491211,
      "activations/layer7_attention_weight_max": 64.56407928466797,
      "activations/layer7_attention_weight_min": -69.06804656982422,
      "activations/layer8_attention_weight_max": 44.17551040649414,
      "activations/layer8_attention_weight_min": -46.17161560058594,
      "activations/layer9_attention_weight_max": 43.62441635131836,
      "activations/layer9_attention_weight_min": -43.625606536865234,
      "epoch": 13.08,
      "learning_rate": 6.630265151515151e-05,
      "loss": 2.7624,
      "step": 225150
    },
    {
      "activations/layer0_attention_weight_max": 15.477771759033203,
      "activations/layer0_attention_weight_min": -16.095928192138672,
      "activations/layer10_attention_weight_max": 33.61137771606445,
      "activations/layer10_attention_weight_min": -33.97260665893555,
      "activations/layer11_attention_weight_max": 31.610286712646484,
      "activations/layer11_attention_weight_min": -32.02317428588867,
      "activations/layer12_attention_weight_max": 20.549909591674805,
      "activations/layer12_attention_weight_min": -28.915895462036133,
      "activations/layer13_attention_weight_max": 41.399288177490234,
      "activations/layer13_attention_weight_min": -26.7187442779541,
      "activations/layer14_attention_weight_max": 35.263545989990234,
      "activations/layer14_attention_weight_min": -29.41783332824707,
      "activations/layer15_attention_weight_max": 33.28337860107422,
      "activations/layer15_attention_weight_min": -29.19669532775879,
      "activations/layer16_attention_weight_max": 30.476490020751953,
      "activations/layer16_attention_weight_min": -30.29958152770996,
      "activations/layer17_attention_weight_max": 52.183937072753906,
      "activations/layer17_attention_weight_min": -42.870399475097656,
      "activations/layer18_attention_weight_max": 45.555477142333984,
      "activations/layer18_attention_weight_min": -36.2257080078125,
      "activations/layer19_attention_weight_max": 23.028728485107422,
      "activations/layer19_attention_weight_min": -21.736906051635742,
      "activations/layer1_attention_weight_max": 16.346193313598633,
      "activations/layer1_attention_weight_min": -17.030590057373047,
      "activations/layer20_attention_weight_max": 24.125566482543945,
      "activations/layer20_attention_weight_min": -24.577537536621094,
      "activations/layer21_attention_weight_max": 35.70866394042969,
      "activations/layer21_attention_weight_min": -23.37703514099121,
      "activations/layer22_attention_weight_max": 29.892839431762695,
      "activations/layer22_attention_weight_min": -25.207836151123047,
      "activations/layer23_attention_weight_max": 38.28313064575195,
      "activations/layer23_attention_weight_min": -23.693878173828125,
      "activations/layer2_attention_weight_max": 31.448780059814453,
      "activations/layer2_attention_weight_min": -30.5518856048584,
      "activations/layer3_attention_weight_max": 95.21495056152344,
      "activations/layer3_attention_weight_min": -98.59776306152344,
      "activations/layer4_attention_weight_max": 96.2134017944336,
      "activations/layer4_attention_weight_min": -95.91307830810547,
      "activations/layer5_attention_weight_max": 73.83120727539062,
      "activations/layer5_attention_weight_min": -75.14759063720703,
      "activations/layer6_attention_weight_max": 50.16748809814453,
      "activations/layer6_attention_weight_min": -50.964256286621094,
      "activations/layer7_attention_weight_max": 68.01917266845703,
      "activations/layer7_attention_weight_min": -64.55105590820312,
      "activations/layer8_attention_weight_max": 44.52852249145508,
      "activations/layer8_attention_weight_min": -45.50613784790039,
      "activations/layer9_attention_weight_max": 45.01278305053711,
      "activations/layer9_attention_weight_min": -45.364776611328125,
      "epoch": 13.09,
      "learning_rate": 6.628371212121211e-05,
      "loss": 2.7498,
      "step": 225200
    },
    {
      "activations/layer0_attention_weight_max": 15.125389099121094,
      "activations/layer0_attention_weight_min": -13.709033966064453,
      "activations/layer10_attention_weight_max": 32.44540786743164,
      "activations/layer10_attention_weight_min": -32.84706497192383,
      "activations/layer11_attention_weight_max": 30.083105087280273,
      "activations/layer11_attention_weight_min": -30.606422424316406,
      "activations/layer12_attention_weight_max": 21.55555534362793,
      "activations/layer12_attention_weight_min": -25.60784149169922,
      "activations/layer13_attention_weight_max": 39.16939163208008,
      "activations/layer13_attention_weight_min": -30.88116455078125,
      "activations/layer14_attention_weight_max": 44.272220611572266,
      "activations/layer14_attention_weight_min": -30.304563522338867,
      "activations/layer15_attention_weight_max": 41.210262298583984,
      "activations/layer15_attention_weight_min": -30.017080307006836,
      "activations/layer16_attention_weight_max": 29.769805908203125,
      "activations/layer16_attention_weight_min": -31.37718963623047,
      "activations/layer17_attention_weight_max": 50.163082122802734,
      "activations/layer17_attention_weight_min": -49.115169525146484,
      "activations/layer18_attention_weight_max": 45.34130096435547,
      "activations/layer18_attention_weight_min": -42.10817337036133,
      "activations/layer19_attention_weight_max": 22.810558319091797,
      "activations/layer19_attention_weight_min": -24.042823791503906,
      "activations/layer1_attention_weight_max": 16.38282585144043,
      "activations/layer1_attention_weight_min": -18.42764663696289,
      "activations/layer20_attention_weight_max": 24.477500915527344,
      "activations/layer20_attention_weight_min": -25.211984634399414,
      "activations/layer21_attention_weight_max": 37.5875129699707,
      "activations/layer21_attention_weight_min": -21.591705322265625,
      "activations/layer22_attention_weight_max": 33.88278579711914,
      "activations/layer22_attention_weight_min": -29.49830436706543,
      "activations/layer23_attention_weight_max": 42.176090240478516,
      "activations/layer23_attention_weight_min": -27.951446533203125,
      "activations/layer2_attention_weight_max": 32.73409652709961,
      "activations/layer2_attention_weight_min": -29.49648666381836,
      "activations/layer3_attention_weight_max": 91.19217681884766,
      "activations/layer3_attention_weight_min": -92.53207397460938,
      "activations/layer4_attention_weight_max": 93.6932601928711,
      "activations/layer4_attention_weight_min": -94.67288208007812,
      "activations/layer5_attention_weight_max": 67.1539306640625,
      "activations/layer5_attention_weight_min": -76.81188201904297,
      "activations/layer6_attention_weight_max": 48.38487243652344,
      "activations/layer6_attention_weight_min": -49.364261627197266,
      "activations/layer7_attention_weight_max": 65.8165512084961,
      "activations/layer7_attention_weight_min": -65.03564453125,
      "activations/layer8_attention_weight_max": 43.53898620605469,
      "activations/layer8_attention_weight_min": -46.37856674194336,
      "activations/layer9_attention_weight_max": 47.75885009765625,
      "activations/layer9_attention_weight_min": -43.572731018066406,
      "epoch": 13.09,
      "learning_rate": 6.626477272727271e-05,
      "loss": 2.7767,
      "step": 225250
    },
    {
      "activations/layer0_attention_weight_max": 16.475984573364258,
      "activations/layer0_attention_weight_min": -13.40913200378418,
      "activations/layer10_attention_weight_max": 33.1611213684082,
      "activations/layer10_attention_weight_min": -32.1336669921875,
      "activations/layer11_attention_weight_max": 31.569509506225586,
      "activations/layer11_attention_weight_min": -30.816326141357422,
      "activations/layer12_attention_weight_max": 18.813554763793945,
      "activations/layer12_attention_weight_min": -26.84990882873535,
      "activations/layer13_attention_weight_max": 35.82105255126953,
      "activations/layer13_attention_weight_min": -26.700977325439453,
      "activations/layer14_attention_weight_max": 37.96965408325195,
      "activations/layer14_attention_weight_min": -30.09227180480957,
      "activations/layer15_attention_weight_max": 33.92491149902344,
      "activations/layer15_attention_weight_min": -29.14761734008789,
      "activations/layer16_attention_weight_max": 31.361223220825195,
      "activations/layer16_attention_weight_min": -29.995145797729492,
      "activations/layer17_attention_weight_max": 55.590328216552734,
      "activations/layer17_attention_weight_min": -44.022335052490234,
      "activations/layer18_attention_weight_max": 50.01668930053711,
      "activations/layer18_attention_weight_min": -37.88103485107422,
      "activations/layer19_attention_weight_max": 25.76655387878418,
      "activations/layer19_attention_weight_min": -22.188072204589844,
      "activations/layer1_attention_weight_max": 16.97283172607422,
      "activations/layer1_attention_weight_min": -14.442320823669434,
      "activations/layer20_attention_weight_max": 27.134387969970703,
      "activations/layer20_attention_weight_min": -22.25971031188965,
      "activations/layer21_attention_weight_max": 40.21164321899414,
      "activations/layer21_attention_weight_min": -21.07091522216797,
      "activations/layer22_attention_weight_max": 34.158241271972656,
      "activations/layer22_attention_weight_min": -27.400556564331055,
      "activations/layer23_attention_weight_max": 40.68817138671875,
      "activations/layer23_attention_weight_min": -24.338584899902344,
      "activations/layer2_attention_weight_max": 29.901649475097656,
      "activations/layer2_attention_weight_min": -31.25008201599121,
      "activations/layer3_attention_weight_max": 89.81643676757812,
      "activations/layer3_attention_weight_min": -101.07391357421875,
      "activations/layer4_attention_weight_max": 92.63837432861328,
      "activations/layer4_attention_weight_min": -90.4562759399414,
      "activations/layer5_attention_weight_max": 66.56190490722656,
      "activations/layer5_attention_weight_min": -75.81815338134766,
      "activations/layer6_attention_weight_max": 50.50459289550781,
      "activations/layer6_attention_weight_min": -51.84455490112305,
      "activations/layer7_attention_weight_max": 63.62647247314453,
      "activations/layer7_attention_weight_min": -63.58667755126953,
      "activations/layer8_attention_weight_max": 41.36518478393555,
      "activations/layer8_attention_weight_min": -44.39522171020508,
      "activations/layer9_attention_weight_max": 50.21653366088867,
      "activations/layer9_attention_weight_min": -43.86492156982422,
      "epoch": 13.09,
      "learning_rate": 6.624583333333333e-05,
      "loss": 2.7675,
      "step": 225300
    },
    {
      "activations/layer0_attention_weight_max": 16.70742416381836,
      "activations/layer0_attention_weight_min": -13.67894172668457,
      "activations/layer10_attention_weight_max": 38.16832733154297,
      "activations/layer10_attention_weight_min": -36.25079345703125,
      "activations/layer11_attention_weight_max": 34.85426330566406,
      "activations/layer11_attention_weight_min": -34.36360168457031,
      "activations/layer12_attention_weight_max": 19.302295684814453,
      "activations/layer12_attention_weight_min": -23.666379928588867,
      "activations/layer13_attention_weight_max": 40.716426849365234,
      "activations/layer13_attention_weight_min": -30.870073318481445,
      "activations/layer14_attention_weight_max": 41.614646911621094,
      "activations/layer14_attention_weight_min": -32.793861389160156,
      "activations/layer15_attention_weight_max": 34.97425842285156,
      "activations/layer15_attention_weight_min": -31.447040557861328,
      "activations/layer16_attention_weight_max": 39.45232391357422,
      "activations/layer16_attention_weight_min": -33.737144470214844,
      "activations/layer17_attention_weight_max": 57.8126220703125,
      "activations/layer17_attention_weight_min": -48.31492614746094,
      "activations/layer18_attention_weight_max": 50.79821014404297,
      "activations/layer18_attention_weight_min": -40.18754577636719,
      "activations/layer19_attention_weight_max": 22.087324142456055,
      "activations/layer19_attention_weight_min": -21.620046615600586,
      "activations/layer1_attention_weight_max": 18.350339889526367,
      "activations/layer1_attention_weight_min": -14.348788261413574,
      "activations/layer20_attention_weight_max": 23.037599563598633,
      "activations/layer20_attention_weight_min": -20.11888313293457,
      "activations/layer21_attention_weight_max": 36.55986785888672,
      "activations/layer21_attention_weight_min": -20.52419090270996,
      "activations/layer22_attention_weight_max": 30.863401412963867,
      "activations/layer22_attention_weight_min": -26.495325088500977,
      "activations/layer23_attention_weight_max": 38.098697662353516,
      "activations/layer23_attention_weight_min": -24.698665618896484,
      "activations/layer2_attention_weight_max": 33.59911346435547,
      "activations/layer2_attention_weight_min": -30.83248519897461,
      "activations/layer3_attention_weight_max": 95.12703704833984,
      "activations/layer3_attention_weight_min": -95.36531066894531,
      "activations/layer4_attention_weight_max": 95.33915710449219,
      "activations/layer4_attention_weight_min": -91.17498016357422,
      "activations/layer5_attention_weight_max": 73.23563385009766,
      "activations/layer5_attention_weight_min": -79.93273162841797,
      "activations/layer6_attention_weight_max": 52.89572525024414,
      "activations/layer6_attention_weight_min": -56.29918670654297,
      "activations/layer7_attention_weight_max": 73.85055541992188,
      "activations/layer7_attention_weight_min": -74.41004180908203,
      "activations/layer8_attention_weight_max": 47.35316467285156,
      "activations/layer8_attention_weight_min": -53.519874572753906,
      "activations/layer9_attention_weight_max": 46.943363189697266,
      "activations/layer9_attention_weight_min": -48.797828674316406,
      "epoch": 13.09,
      "learning_rate": 6.622689393939393e-05,
      "loss": 2.7885,
      "step": 225350
    },
    {
      "activations/layer0_attention_weight_max": 16.606966018676758,
      "activations/layer0_attention_weight_min": -13.54391860961914,
      "activations/layer10_attention_weight_max": 39.462158203125,
      "activations/layer10_attention_weight_min": -38.38365173339844,
      "activations/layer11_attention_weight_max": 38.41388702392578,
      "activations/layer11_attention_weight_min": -34.66200637817383,
      "activations/layer12_attention_weight_max": 21.224830627441406,
      "activations/layer12_attention_weight_min": -24.561420440673828,
      "activations/layer13_attention_weight_max": 43.963096618652344,
      "activations/layer13_attention_weight_min": -29.090545654296875,
      "activations/layer14_attention_weight_max": 36.9359016418457,
      "activations/layer14_attention_weight_min": -31.693256378173828,
      "activations/layer15_attention_weight_max": 32.443565368652344,
      "activations/layer15_attention_weight_min": -29.079498291015625,
      "activations/layer16_attention_weight_max": 31.264978408813477,
      "activations/layer16_attention_weight_min": -28.489749908447266,
      "activations/layer17_attention_weight_max": 47.59556198120117,
      "activations/layer17_attention_weight_min": -46.337371826171875,
      "activations/layer18_attention_weight_max": 43.53948211669922,
      "activations/layer18_attention_weight_min": -37.14037322998047,
      "activations/layer19_attention_weight_max": 19.77812385559082,
      "activations/layer19_attention_weight_min": -20.538171768188477,
      "activations/layer1_attention_weight_max": 17.262691497802734,
      "activations/layer1_attention_weight_min": -15.265829086303711,
      "activations/layer20_attention_weight_max": 21.403371810913086,
      "activations/layer20_attention_weight_min": -20.360851287841797,
      "activations/layer21_attention_weight_max": 38.089962005615234,
      "activations/layer21_attention_weight_min": -21.11076545715332,
      "activations/layer22_attention_weight_max": 28.12680435180664,
      "activations/layer22_attention_weight_min": -25.92888832092285,
      "activations/layer23_attention_weight_max": 34.7478141784668,
      "activations/layer23_attention_weight_min": -24.323911666870117,
      "activations/layer2_attention_weight_max": 32.737613677978516,
      "activations/layer2_attention_weight_min": -31.570119857788086,
      "activations/layer3_attention_weight_max": 95.86672973632812,
      "activations/layer3_attention_weight_min": -96.68870544433594,
      "activations/layer4_attention_weight_max": 97.51860046386719,
      "activations/layer4_attention_weight_min": -97.90390014648438,
      "activations/layer5_attention_weight_max": 70.4699478149414,
      "activations/layer5_attention_weight_min": -76.00281524658203,
      "activations/layer6_attention_weight_max": 53.548805236816406,
      "activations/layer6_attention_weight_min": -51.858909606933594,
      "activations/layer7_attention_weight_max": 73.3310546875,
      "activations/layer7_attention_weight_min": -65.8171615600586,
      "activations/layer8_attention_weight_max": 48.693843841552734,
      "activations/layer8_attention_weight_min": -47.76786422729492,
      "activations/layer9_attention_weight_max": 55.1817512512207,
      "activations/layer9_attention_weight_min": -46.0687370300293,
      "epoch": 13.1,
      "learning_rate": 6.620795454545455e-05,
      "loss": 2.7415,
      "step": 225400
    },
    {
      "activations/layer0_attention_weight_max": 16.390966415405273,
      "activations/layer0_attention_weight_min": -13.730295181274414,
      "activations/layer10_attention_weight_max": 44.3091926574707,
      "activations/layer10_attention_weight_min": -43.0238151550293,
      "activations/layer11_attention_weight_max": 42.92985534667969,
      "activations/layer11_attention_weight_min": -37.343631744384766,
      "activations/layer12_attention_weight_max": 19.737089157104492,
      "activations/layer12_attention_weight_min": -28.94525909423828,
      "activations/layer13_attention_weight_max": 38.076576232910156,
      "activations/layer13_attention_weight_min": -28.102724075317383,
      "activations/layer14_attention_weight_max": 39.623958587646484,
      "activations/layer14_attention_weight_min": -29.865154266357422,
      "activations/layer15_attention_weight_max": 37.95572280883789,
      "activations/layer15_attention_weight_min": -28.255823135375977,
      "activations/layer16_attention_weight_max": 33.73064041137695,
      "activations/layer16_attention_weight_min": -29.76381492614746,
      "activations/layer17_attention_weight_max": 55.49208450317383,
      "activations/layer17_attention_weight_min": -49.59175109863281,
      "activations/layer18_attention_weight_max": 51.75677490234375,
      "activations/layer18_attention_weight_min": -45.68578338623047,
      "activations/layer19_attention_weight_max": 23.82697868347168,
      "activations/layer19_attention_weight_min": -23.436697006225586,
      "activations/layer1_attention_weight_max": 18.34017562866211,
      "activations/layer1_attention_weight_min": -15.188075065612793,
      "activations/layer20_attention_weight_max": 25.544347763061523,
      "activations/layer20_attention_weight_min": -21.21976089477539,
      "activations/layer21_attention_weight_max": 35.94166946411133,
      "activations/layer21_attention_weight_min": -22.0648193359375,
      "activations/layer22_attention_weight_max": 33.653411865234375,
      "activations/layer22_attention_weight_min": -23.53053092956543,
      "activations/layer23_attention_weight_max": 36.901241302490234,
      "activations/layer23_attention_weight_min": -23.923330307006836,
      "activations/layer2_attention_weight_max": 35.70140838623047,
      "activations/layer2_attention_weight_min": -31.820392608642578,
      "activations/layer3_attention_weight_max": 116.48179626464844,
      "activations/layer3_attention_weight_min": -114.43417358398438,
      "activations/layer4_attention_weight_max": 123.32476806640625,
      "activations/layer4_attention_weight_min": -106.64327239990234,
      "activations/layer5_attention_weight_max": 79.05714416503906,
      "activations/layer5_attention_weight_min": -79.07164001464844,
      "activations/layer6_attention_weight_max": 55.76497268676758,
      "activations/layer6_attention_weight_min": -56.41312026977539,
      "activations/layer7_attention_weight_max": 83.42828369140625,
      "activations/layer7_attention_weight_min": -73.10964965820312,
      "activations/layer8_attention_weight_max": 55.565895080566406,
      "activations/layer8_attention_weight_min": -52.02894973754883,
      "activations/layer9_attention_weight_max": 62.52421188354492,
      "activations/layer9_attention_weight_min": -56.6550178527832,
      "epoch": 13.1,
      "learning_rate": 6.618901515151515e-05,
      "loss": 2.7706,
      "step": 225450
    },
    {
      "activations/layer0_attention_weight_max": 17.178115844726562,
      "activations/layer0_attention_weight_min": -14.203609466552734,
      "activations/layer10_attention_weight_max": 36.67619705200195,
      "activations/layer10_attention_weight_min": -36.248382568359375,
      "activations/layer11_attention_weight_max": 33.1002311706543,
      "activations/layer11_attention_weight_min": -33.58879470825195,
      "activations/layer12_attention_weight_max": 19.46979331970215,
      "activations/layer12_attention_weight_min": -24.479644775390625,
      "activations/layer13_attention_weight_max": 41.189903259277344,
      "activations/layer13_attention_weight_min": -29.137775421142578,
      "activations/layer14_attention_weight_max": 40.43404006958008,
      "activations/layer14_attention_weight_min": -29.880184173583984,
      "activations/layer15_attention_weight_max": 33.35516357421875,
      "activations/layer15_attention_weight_min": -30.855661392211914,
      "activations/layer16_attention_weight_max": 31.610509872436523,
      "activations/layer16_attention_weight_min": -29.621238708496094,
      "activations/layer17_attention_weight_max": 52.64727783203125,
      "activations/layer17_attention_weight_min": -44.10946273803711,
      "activations/layer18_attention_weight_max": 46.5704460144043,
      "activations/layer18_attention_weight_min": -37.329673767089844,
      "activations/layer19_attention_weight_max": 24.164113998413086,
      "activations/layer19_attention_weight_min": -22.234375,
      "activations/layer1_attention_weight_max": 16.56093978881836,
      "activations/layer1_attention_weight_min": -15.202856063842773,
      "activations/layer20_attention_weight_max": 24.450321197509766,
      "activations/layer20_attention_weight_min": -23.513044357299805,
      "activations/layer21_attention_weight_max": 43.93953323364258,
      "activations/layer21_attention_weight_min": -25.178905487060547,
      "activations/layer22_attention_weight_max": 33.54093933105469,
      "activations/layer22_attention_weight_min": -25.97130584716797,
      "activations/layer23_attention_weight_max": 41.37044906616211,
      "activations/layer23_attention_weight_min": -24.54400634765625,
      "activations/layer2_attention_weight_max": 33.244876861572266,
      "activations/layer2_attention_weight_min": -32.924774169921875,
      "activations/layer3_attention_weight_max": 96.55607604980469,
      "activations/layer3_attention_weight_min": -96.87763214111328,
      "activations/layer4_attention_weight_max": 97.13546752929688,
      "activations/layer4_attention_weight_min": -94.62616729736328,
      "activations/layer5_attention_weight_max": 72.87109375,
      "activations/layer5_attention_weight_min": -77.76578521728516,
      "activations/layer6_attention_weight_max": 55.1351432800293,
      "activations/layer6_attention_weight_min": -52.32950973510742,
      "activations/layer7_attention_weight_max": 72.34542846679688,
      "activations/layer7_attention_weight_min": -65.1312026977539,
      "activations/layer8_attention_weight_max": 47.2186279296875,
      "activations/layer8_attention_weight_min": -47.94832229614258,
      "activations/layer9_attention_weight_max": 45.24705505371094,
      "activations/layer9_attention_weight_min": -46.672950744628906,
      "epoch": 13.1,
      "learning_rate": 6.617007575757576e-05,
      "loss": 2.7695,
      "step": 225500
    },
    {
      "activations/layer0_attention_weight_max": 15.64281940460205,
      "activations/layer0_attention_weight_min": -14.425222396850586,
      "activations/layer10_attention_weight_max": 35.726844787597656,
      "activations/layer10_attention_weight_min": -36.34724426269531,
      "activations/layer11_attention_weight_max": 31.193553924560547,
      "activations/layer11_attention_weight_min": -32.35103225708008,
      "activations/layer12_attention_weight_max": 17.282377243041992,
      "activations/layer12_attention_weight_min": -24.983545303344727,
      "activations/layer13_attention_weight_max": 34.037841796875,
      "activations/layer13_attention_weight_min": -28.905736923217773,
      "activations/layer14_attention_weight_max": 35.14073181152344,
      "activations/layer14_attention_weight_min": -28.60810661315918,
      "activations/layer15_attention_weight_max": 32.05400085449219,
      "activations/layer15_attention_weight_min": -27.62851333618164,
      "activations/layer16_attention_weight_max": 29.137048721313477,
      "activations/layer16_attention_weight_min": -27.806854248046875,
      "activations/layer17_attention_weight_max": 48.36879348754883,
      "activations/layer17_attention_weight_min": -40.5460205078125,
      "activations/layer18_attention_weight_max": 41.45005798339844,
      "activations/layer18_attention_weight_min": -35.851417541503906,
      "activations/layer19_attention_weight_max": 21.343046188354492,
      "activations/layer19_attention_weight_min": -19.91596221923828,
      "activations/layer1_attention_weight_max": 16.136371612548828,
      "activations/layer1_attention_weight_min": -14.146149635314941,
      "activations/layer20_attention_weight_max": 19.276033401489258,
      "activations/layer20_attention_weight_min": -20.46531105041504,
      "activations/layer21_attention_weight_max": 31.618009567260742,
      "activations/layer21_attention_weight_min": -21.184083938598633,
      "activations/layer22_attention_weight_max": 27.842493057250977,
      "activations/layer22_attention_weight_min": -25.5255184173584,
      "activations/layer23_attention_weight_max": 37.672149658203125,
      "activations/layer23_attention_weight_min": -24.39362144470215,
      "activations/layer2_attention_weight_max": 32.99122619628906,
      "activations/layer2_attention_weight_min": -31.944000244140625,
      "activations/layer3_attention_weight_max": 90.53553771972656,
      "activations/layer3_attention_weight_min": -93.0903091430664,
      "activations/layer4_attention_weight_max": 93.07672882080078,
      "activations/layer4_attention_weight_min": -88.88679504394531,
      "activations/layer5_attention_weight_max": 70.33554077148438,
      "activations/layer5_attention_weight_min": -78.29480743408203,
      "activations/layer6_attention_weight_max": 49.120182037353516,
      "activations/layer6_attention_weight_min": -50.2828254699707,
      "activations/layer7_attention_weight_max": 64.4203872680664,
      "activations/layer7_attention_weight_min": -63.81389236450195,
      "activations/layer8_attention_weight_max": 44.48158645629883,
      "activations/layer8_attention_weight_min": -47.58716583251953,
      "activations/layer9_attention_weight_max": 50.43012619018555,
      "activations/layer9_attention_weight_min": -48.30339050292969,
      "epoch": 13.11,
      "learning_rate": 6.615113636363636e-05,
      "loss": 2.7712,
      "step": 225550
    },
    {
      "activations/layer0_attention_weight_max": 15.928705215454102,
      "activations/layer0_attention_weight_min": -14.160722732543945,
      "activations/layer10_attention_weight_max": 34.724273681640625,
      "activations/layer10_attention_weight_min": -33.39986038208008,
      "activations/layer11_attention_weight_max": 31.9139461517334,
      "activations/layer11_attention_weight_min": -30.551528930664062,
      "activations/layer12_attention_weight_max": 17.640283584594727,
      "activations/layer12_attention_weight_min": -23.052268981933594,
      "activations/layer13_attention_weight_max": 36.39466857910156,
      "activations/layer13_attention_weight_min": -26.87299156188965,
      "activations/layer14_attention_weight_max": 37.41597366333008,
      "activations/layer14_attention_weight_min": -30.344751358032227,
      "activations/layer15_attention_weight_max": 34.18132400512695,
      "activations/layer15_attention_weight_min": -30.28961181640625,
      "activations/layer16_attention_weight_max": 30.61625099182129,
      "activations/layer16_attention_weight_min": -31.172330856323242,
      "activations/layer17_attention_weight_max": 52.188385009765625,
      "activations/layer17_attention_weight_min": -43.930076599121094,
      "activations/layer18_attention_weight_max": 46.945831298828125,
      "activations/layer18_attention_weight_min": -38.71659851074219,
      "activations/layer19_attention_weight_max": 22.165891647338867,
      "activations/layer19_attention_weight_min": -22.52731704711914,
      "activations/layer1_attention_weight_max": 18.75006675720215,
      "activations/layer1_attention_weight_min": -16.296314239501953,
      "activations/layer20_attention_weight_max": 23.202524185180664,
      "activations/layer20_attention_weight_min": -24.128427505493164,
      "activations/layer21_attention_weight_max": 37.59743881225586,
      "activations/layer21_attention_weight_min": -25.071374893188477,
      "activations/layer22_attention_weight_max": 28.09585952758789,
      "activations/layer22_attention_weight_min": -25.805160522460938,
      "activations/layer23_attention_weight_max": 35.42510223388672,
      "activations/layer23_attention_weight_min": -26.05434226989746,
      "activations/layer2_attention_weight_max": 31.42250633239746,
      "activations/layer2_attention_weight_min": -30.325130462646484,
      "activations/layer3_attention_weight_max": 94.82652282714844,
      "activations/layer3_attention_weight_min": -96.16844177246094,
      "activations/layer4_attention_weight_max": 98.6962661743164,
      "activations/layer4_attention_weight_min": -95.37484741210938,
      "activations/layer5_attention_weight_max": 72.3189468383789,
      "activations/layer5_attention_weight_min": -76.09440612792969,
      "activations/layer6_attention_weight_max": 52.79850769042969,
      "activations/layer6_attention_weight_min": -50.916419982910156,
      "activations/layer7_attention_weight_max": 67.68753814697266,
      "activations/layer7_attention_weight_min": -68.87596893310547,
      "activations/layer8_attention_weight_max": 44.53616714477539,
      "activations/layer8_attention_weight_min": -44.33013153076172,
      "activations/layer9_attention_weight_max": 44.30105972290039,
      "activations/layer9_attention_weight_min": -45.59657287597656,
      "epoch": 13.11,
      "learning_rate": 6.613219696969696e-05,
      "loss": 2.7641,
      "step": 225600
    },
    {
      "activations/layer0_attention_weight_max": 16.456693649291992,
      "activations/layer0_attention_weight_min": -14.70401382446289,
      "activations/layer10_attention_weight_max": 39.41343307495117,
      "activations/layer10_attention_weight_min": -36.43313980102539,
      "activations/layer11_attention_weight_max": 35.740840911865234,
      "activations/layer11_attention_weight_min": -34.7439079284668,
      "activations/layer12_attention_weight_max": 18.39170265197754,
      "activations/layer12_attention_weight_min": -25.432693481445312,
      "activations/layer13_attention_weight_max": 39.20930480957031,
      "activations/layer13_attention_weight_min": -30.493993759155273,
      "activations/layer14_attention_weight_max": 34.970542907714844,
      "activations/layer14_attention_weight_min": -30.76433753967285,
      "activations/layer15_attention_weight_max": 33.37559127807617,
      "activations/layer15_attention_weight_min": -30.337745666503906,
      "activations/layer16_attention_weight_max": 31.30827522277832,
      "activations/layer16_attention_weight_min": -31.504369735717773,
      "activations/layer17_attention_weight_max": 55.639060974121094,
      "activations/layer17_attention_weight_min": -44.14012908935547,
      "activations/layer18_attention_weight_max": 48.65176773071289,
      "activations/layer18_attention_weight_min": -39.037532806396484,
      "activations/layer19_attention_weight_max": 23.665428161621094,
      "activations/layer19_attention_weight_min": -22.710134506225586,
      "activations/layer1_attention_weight_max": 16.578996658325195,
      "activations/layer1_attention_weight_min": -15.282302856445312,
      "activations/layer20_attention_weight_max": 22.03299331665039,
      "activations/layer20_attention_weight_min": -21.208538055419922,
      "activations/layer21_attention_weight_max": 39.10955810546875,
      "activations/layer21_attention_weight_min": -22.633554458618164,
      "activations/layer22_attention_weight_max": 29.89995002746582,
      "activations/layer22_attention_weight_min": -26.213003158569336,
      "activations/layer23_attention_weight_max": 41.470947265625,
      "activations/layer23_attention_weight_min": -23.050094604492188,
      "activations/layer2_attention_weight_max": 33.093223571777344,
      "activations/layer2_attention_weight_min": -32.590736389160156,
      "activations/layer3_attention_weight_max": 95.92282104492188,
      "activations/layer3_attention_weight_min": -101.52238464355469,
      "activations/layer4_attention_weight_max": 100.17465209960938,
      "activations/layer4_attention_weight_min": -95.91919708251953,
      "activations/layer5_attention_weight_max": 71.62476348876953,
      "activations/layer5_attention_weight_min": -78.72315216064453,
      "activations/layer6_attention_weight_max": 54.363990783691406,
      "activations/layer6_attention_weight_min": -53.13796615600586,
      "activations/layer7_attention_weight_max": 70.136962890625,
      "activations/layer7_attention_weight_min": -66.75260162353516,
      "activations/layer8_attention_weight_max": 48.5251350402832,
      "activations/layer8_attention_weight_min": -48.37449645996094,
      "activations/layer9_attention_weight_max": 46.825965881347656,
      "activations/layer9_attention_weight_min": -46.89637756347656,
      "epoch": 13.11,
      "learning_rate": 6.611325757575757e-05,
      "loss": 2.7721,
      "step": 225650
    },
    {
      "activations/layer0_attention_weight_max": 16.089637756347656,
      "activations/layer0_attention_weight_min": -14.526097297668457,
      "activations/layer10_attention_weight_max": 38.52044677734375,
      "activations/layer10_attention_weight_min": -37.81659698486328,
      "activations/layer11_attention_weight_max": 33.5839729309082,
      "activations/layer11_attention_weight_min": -33.470279693603516,
      "activations/layer12_attention_weight_max": 18.83343505859375,
      "activations/layer12_attention_weight_min": -30.143352508544922,
      "activations/layer13_attention_weight_max": 37.24374008178711,
      "activations/layer13_attention_weight_min": -28.805675506591797,
      "activations/layer14_attention_weight_max": 39.790550231933594,
      "activations/layer14_attention_weight_min": -31.68793487548828,
      "activations/layer15_attention_weight_max": 34.500572204589844,
      "activations/layer15_attention_weight_min": -30.269746780395508,
      "activations/layer16_attention_weight_max": 31.827665328979492,
      "activations/layer16_attention_weight_min": -30.7635498046875,
      "activations/layer17_attention_weight_max": 54.424713134765625,
      "activations/layer17_attention_weight_min": -47.07470703125,
      "activations/layer18_attention_weight_max": 45.72484588623047,
      "activations/layer18_attention_weight_min": -37.79113006591797,
      "activations/layer19_attention_weight_max": 22.15001678466797,
      "activations/layer19_attention_weight_min": -20.382890701293945,
      "activations/layer1_attention_weight_max": 16.92386245727539,
      "activations/layer1_attention_weight_min": -14.729464530944824,
      "activations/layer20_attention_weight_max": 21.213626861572266,
      "activations/layer20_attention_weight_min": -20.264171600341797,
      "activations/layer21_attention_weight_max": 32.751869201660156,
      "activations/layer21_attention_weight_min": -21.506961822509766,
      "activations/layer22_attention_weight_max": 30.988523483276367,
      "activations/layer22_attention_weight_min": -27.62593650817871,
      "activations/layer23_attention_weight_max": 38.097145080566406,
      "activations/layer23_attention_weight_min": -21.948637008666992,
      "activations/layer2_attention_weight_max": 32.920204162597656,
      "activations/layer2_attention_weight_min": -31.697742462158203,
      "activations/layer3_attention_weight_max": 97.64201354980469,
      "activations/layer3_attention_weight_min": -97.84246063232422,
      "activations/layer4_attention_weight_max": 104.8510513305664,
      "activations/layer4_attention_weight_min": -100.40801239013672,
      "activations/layer5_attention_weight_max": 74.03275299072266,
      "activations/layer5_attention_weight_min": -77.05242156982422,
      "activations/layer6_attention_weight_max": 54.01847457885742,
      "activations/layer6_attention_weight_min": -55.498779296875,
      "activations/layer7_attention_weight_max": 70.72225189208984,
      "activations/layer7_attention_weight_min": -70.60246276855469,
      "activations/layer8_attention_weight_max": 47.65997314453125,
      "activations/layer8_attention_weight_min": -50.810462951660156,
      "activations/layer9_attention_weight_max": 50.54379653930664,
      "activations/layer9_attention_weight_min": -51.167938232421875,
      "epoch": 13.11,
      "learning_rate": 6.609431818181818e-05,
      "loss": 2.7805,
      "step": 225700
    },
    {
      "activations/layer0_attention_weight_max": 16.98828125,
      "activations/layer0_attention_weight_min": -14.383834838867188,
      "activations/layer10_attention_weight_max": 37.86975860595703,
      "activations/layer10_attention_weight_min": -35.0179328918457,
      "activations/layer11_attention_weight_max": 33.78828811645508,
      "activations/layer11_attention_weight_min": -33.903480529785156,
      "activations/layer12_attention_weight_max": 19.126667022705078,
      "activations/layer12_attention_weight_min": -23.059677124023438,
      "activations/layer13_attention_weight_max": 43.82339096069336,
      "activations/layer13_attention_weight_min": -32.35377502441406,
      "activations/layer14_attention_weight_max": 37.769256591796875,
      "activations/layer14_attention_weight_min": -31.811052322387695,
      "activations/layer15_attention_weight_max": 35.385555267333984,
      "activations/layer15_attention_weight_min": -30.770456314086914,
      "activations/layer16_attention_weight_max": 32.92230987548828,
      "activations/layer16_attention_weight_min": -32.01531982421875,
      "activations/layer17_attention_weight_max": 53.34402084350586,
      "activations/layer17_attention_weight_min": -45.640323638916016,
      "activations/layer18_attention_weight_max": 49.52665328979492,
      "activations/layer18_attention_weight_min": -41.178890228271484,
      "activations/layer19_attention_weight_max": 22.30350685119629,
      "activations/layer19_attention_weight_min": -23.561222076416016,
      "activations/layer1_attention_weight_max": 16.477436065673828,
      "activations/layer1_attention_weight_min": -15.339251518249512,
      "activations/layer20_attention_weight_max": 22.94577407836914,
      "activations/layer20_attention_weight_min": -21.97699737548828,
      "activations/layer21_attention_weight_max": 43.40999984741211,
      "activations/layer21_attention_weight_min": -24.29800796508789,
      "activations/layer22_attention_weight_max": 31.155040740966797,
      "activations/layer22_attention_weight_min": -26.462570190429688,
      "activations/layer23_attention_weight_max": 38.38972473144531,
      "activations/layer23_attention_weight_min": -25.991737365722656,
      "activations/layer2_attention_weight_max": 32.649444580078125,
      "activations/layer2_attention_weight_min": -32.09016036987305,
      "activations/layer3_attention_weight_max": 92.7319107055664,
      "activations/layer3_attention_weight_min": -92.81597137451172,
      "activations/layer4_attention_weight_max": 96.43272399902344,
      "activations/layer4_attention_weight_min": -89.551025390625,
      "activations/layer5_attention_weight_max": 71.78112030029297,
      "activations/layer5_attention_weight_min": -78.02593994140625,
      "activations/layer6_attention_weight_max": 54.12755584716797,
      "activations/layer6_attention_weight_min": -51.95600128173828,
      "activations/layer7_attention_weight_max": 68.71597290039062,
      "activations/layer7_attention_weight_min": -67.68775939941406,
      "activations/layer8_attention_weight_max": 45.94173049926758,
      "activations/layer8_attention_weight_min": -47.3570442199707,
      "activations/layer9_attention_weight_max": 46.591461181640625,
      "activations/layer9_attention_weight_min": -45.15488052368164,
      "epoch": 13.12,
      "learning_rate": 6.607537878787878e-05,
      "loss": 2.7696,
      "step": 225750
    },
    {
      "activations/layer0_attention_weight_max": 15.686492919921875,
      "activations/layer0_attention_weight_min": -14.023130416870117,
      "activations/layer10_attention_weight_max": 40.63251495361328,
      "activations/layer10_attention_weight_min": -38.004398345947266,
      "activations/layer11_attention_weight_max": 34.1807746887207,
      "activations/layer11_attention_weight_min": -34.877471923828125,
      "activations/layer12_attention_weight_max": 19.498367309570312,
      "activations/layer12_attention_weight_min": -24.46478843688965,
      "activations/layer13_attention_weight_max": 40.4415283203125,
      "activations/layer13_attention_weight_min": -29.449495315551758,
      "activations/layer14_attention_weight_max": 47.502655029296875,
      "activations/layer14_attention_weight_min": -29.427263259887695,
      "activations/layer15_attention_weight_max": 40.4986457824707,
      "activations/layer15_attention_weight_min": -28.70167350769043,
      "activations/layer16_attention_weight_max": 31.11695098876953,
      "activations/layer16_attention_weight_min": -31.264497756958008,
      "activations/layer17_attention_weight_max": 51.4887580871582,
      "activations/layer17_attention_weight_min": -45.36473846435547,
      "activations/layer18_attention_weight_max": 45.17021942138672,
      "activations/layer18_attention_weight_min": -38.51167678833008,
      "activations/layer19_attention_weight_max": 21.440431594848633,
      "activations/layer19_attention_weight_min": -20.74966049194336,
      "activations/layer1_attention_weight_max": 16.551721572875977,
      "activations/layer1_attention_weight_min": -16.092557907104492,
      "activations/layer20_attention_weight_max": 21.775897979736328,
      "activations/layer20_attention_weight_min": -20.595609664916992,
      "activations/layer21_attention_weight_max": 35.5400390625,
      "activations/layer21_attention_weight_min": -20.068588256835938,
      "activations/layer22_attention_weight_max": 31.23439598083496,
      "activations/layer22_attention_weight_min": -24.496234893798828,
      "activations/layer23_attention_weight_max": 43.01374053955078,
      "activations/layer23_attention_weight_min": -28.425731658935547,
      "activations/layer2_attention_weight_max": 34.595863342285156,
      "activations/layer2_attention_weight_min": -33.55476379394531,
      "activations/layer3_attention_weight_max": 101.76385498046875,
      "activations/layer3_attention_weight_min": -95.30083465576172,
      "activations/layer4_attention_weight_max": 102.47216796875,
      "activations/layer4_attention_weight_min": -95.64603424072266,
      "activations/layer5_attention_weight_max": 76.10832214355469,
      "activations/layer5_attention_weight_min": -81.97518920898438,
      "activations/layer6_attention_weight_max": 53.99128341674805,
      "activations/layer6_attention_weight_min": -53.39942169189453,
      "activations/layer7_attention_weight_max": 72.24920654296875,
      "activations/layer7_attention_weight_min": -73.83297729492188,
      "activations/layer8_attention_weight_max": 47.81563186645508,
      "activations/layer8_attention_weight_min": -49.92876434326172,
      "activations/layer9_attention_weight_max": 50.59176254272461,
      "activations/layer9_attention_weight_min": -51.13528060913086,
      "epoch": 13.12,
      "learning_rate": 6.605643939393938e-05,
      "loss": 2.7594,
      "step": 225800
    },
    {
      "activations/layer0_attention_weight_max": 16.504436492919922,
      "activations/layer0_attention_weight_min": -13.91460132598877,
      "activations/layer10_attention_weight_max": 36.796268463134766,
      "activations/layer10_attention_weight_min": -36.74886703491211,
      "activations/layer11_attention_weight_max": 32.527645111083984,
      "activations/layer11_attention_weight_min": -32.36385726928711,
      "activations/layer12_attention_weight_max": 20.544097900390625,
      "activations/layer12_attention_weight_min": -24.516727447509766,
      "activations/layer13_attention_weight_max": 34.48149871826172,
      "activations/layer13_attention_weight_min": -28.546100616455078,
      "activations/layer14_attention_weight_max": 33.84258270263672,
      "activations/layer14_attention_weight_min": -29.878971099853516,
      "activations/layer15_attention_weight_max": 31.617597579956055,
      "activations/layer15_attention_weight_min": -30.64478874206543,
      "activations/layer16_attention_weight_max": 30.559280395507812,
      "activations/layer16_attention_weight_min": -28.489303588867188,
      "activations/layer17_attention_weight_max": 52.699493408203125,
      "activations/layer17_attention_weight_min": -44.4581298828125,
      "activations/layer18_attention_weight_max": 49.33827590942383,
      "activations/layer18_attention_weight_min": -42.27438735961914,
      "activations/layer19_attention_weight_max": 22.921804428100586,
      "activations/layer19_attention_weight_min": -23.83970069885254,
      "activations/layer1_attention_weight_max": 16.728036880493164,
      "activations/layer1_attention_weight_min": -15.959365844726562,
      "activations/layer20_attention_weight_max": 21.758960723876953,
      "activations/layer20_attention_weight_min": -22.973318099975586,
      "activations/layer21_attention_weight_max": 31.17068862915039,
      "activations/layer21_attention_weight_min": -21.380233764648438,
      "activations/layer22_attention_weight_max": 29.882890701293945,
      "activations/layer22_attention_weight_min": -24.365802764892578,
      "activations/layer23_attention_weight_max": 36.54290771484375,
      "activations/layer23_attention_weight_min": -23.422330856323242,
      "activations/layer2_attention_weight_max": 35.336280822753906,
      "activations/layer2_attention_weight_min": -34.42027282714844,
      "activations/layer3_attention_weight_max": 98.48359680175781,
      "activations/layer3_attention_weight_min": -100.86760711669922,
      "activations/layer4_attention_weight_max": 95.6342544555664,
      "activations/layer4_attention_weight_min": -96.0613021850586,
      "activations/layer5_attention_weight_max": 72.58470153808594,
      "activations/layer5_attention_weight_min": -79.13829040527344,
      "activations/layer6_attention_weight_max": 51.610755920410156,
      "activations/layer6_attention_weight_min": -51.40630340576172,
      "activations/layer7_attention_weight_max": 68.94334411621094,
      "activations/layer7_attention_weight_min": -66.31529998779297,
      "activations/layer8_attention_weight_max": 46.771728515625,
      "activations/layer8_attention_weight_min": -48.07637405395508,
      "activations/layer9_attention_weight_max": 48.38747024536133,
      "activations/layer9_attention_weight_min": -47.122806549072266,
      "epoch": 13.12,
      "learning_rate": 6.603749999999999e-05,
      "loss": 2.7558,
      "step": 225850
    },
    {
      "activations/layer0_attention_weight_max": 15.256400108337402,
      "activations/layer0_attention_weight_min": -14.037457466125488,
      "activations/layer10_attention_weight_max": 38.98596954345703,
      "activations/layer10_attention_weight_min": -37.48173904418945,
      "activations/layer11_attention_weight_max": 36.575042724609375,
      "activations/layer11_attention_weight_min": -35.72658920288086,
      "activations/layer12_attention_weight_max": 19.401107788085938,
      "activations/layer12_attention_weight_min": -25.238021850585938,
      "activations/layer13_attention_weight_max": 38.49702072143555,
      "activations/layer13_attention_weight_min": -29.964340209960938,
      "activations/layer14_attention_weight_max": 44.080108642578125,
      "activations/layer14_attention_weight_min": -32.46137237548828,
      "activations/layer15_attention_weight_max": 38.403202056884766,
      "activations/layer15_attention_weight_min": -30.889951705932617,
      "activations/layer16_attention_weight_max": 31.36588478088379,
      "activations/layer16_attention_weight_min": -28.966880798339844,
      "activations/layer17_attention_weight_max": 51.69794845581055,
      "activations/layer17_attention_weight_min": -45.29777145385742,
      "activations/layer18_attention_weight_max": 47.2797737121582,
      "activations/layer18_attention_weight_min": -36.93082809448242,
      "activations/layer19_attention_weight_max": 24.09795379638672,
      "activations/layer19_attention_weight_min": -22.4543399810791,
      "activations/layer1_attention_weight_max": 16.773046493530273,
      "activations/layer1_attention_weight_min": -17.57299041748047,
      "activations/layer20_attention_weight_max": 24.74213218688965,
      "activations/layer20_attention_weight_min": -21.150604248046875,
      "activations/layer21_attention_weight_max": 39.584781646728516,
      "activations/layer21_attention_weight_min": -23.21521759033203,
      "activations/layer22_attention_weight_max": 32.07711410522461,
      "activations/layer22_attention_weight_min": -25.62391471862793,
      "activations/layer23_attention_weight_max": 37.79572677612305,
      "activations/layer23_attention_weight_min": -24.213050842285156,
      "activations/layer2_attention_weight_max": 32.827110290527344,
      "activations/layer2_attention_weight_min": -32.530120849609375,
      "activations/layer3_attention_weight_max": 101.10566711425781,
      "activations/layer3_attention_weight_min": -100.75176239013672,
      "activations/layer4_attention_weight_max": 95.00868225097656,
      "activations/layer4_attention_weight_min": -93.05525207519531,
      "activations/layer5_attention_weight_max": 69.36656188964844,
      "activations/layer5_attention_weight_min": -76.36263275146484,
      "activations/layer6_attention_weight_max": 51.615074157714844,
      "activations/layer6_attention_weight_min": -52.595787048339844,
      "activations/layer7_attention_weight_max": 64.92096710205078,
      "activations/layer7_attention_weight_min": -68.668701171875,
      "activations/layer8_attention_weight_max": 48.6731071472168,
      "activations/layer8_attention_weight_min": -50.91887283325195,
      "activations/layer9_attention_weight_max": 44.69840621948242,
      "activations/layer9_attention_weight_min": -46.57925796508789,
      "epoch": 13.13,
      "learning_rate": 6.60185606060606e-05,
      "loss": 2.7611,
      "step": 225900
    },
    {
      "activations/layer0_attention_weight_max": 16.15414047241211,
      "activations/layer0_attention_weight_min": -14.32576847076416,
      "activations/layer10_attention_weight_max": 35.505924224853516,
      "activations/layer10_attention_weight_min": -35.728641510009766,
      "activations/layer11_attention_weight_max": 32.902748107910156,
      "activations/layer11_attention_weight_min": -36.15943908691406,
      "activations/layer12_attention_weight_max": 17.804101943969727,
      "activations/layer12_attention_weight_min": -23.604883193969727,
      "activations/layer13_attention_weight_max": 35.83172607421875,
      "activations/layer13_attention_weight_min": -31.512805938720703,
      "activations/layer14_attention_weight_max": 37.07913589477539,
      "activations/layer14_attention_weight_min": -30.895212173461914,
      "activations/layer15_attention_weight_max": 34.02266311645508,
      "activations/layer15_attention_weight_min": -29.121679306030273,
      "activations/layer16_attention_weight_max": 31.26093292236328,
      "activations/layer16_attention_weight_min": -28.659488677978516,
      "activations/layer17_attention_weight_max": 49.14433670043945,
      "activations/layer17_attention_weight_min": -43.74368667602539,
      "activations/layer18_attention_weight_max": 46.072296142578125,
      "activations/layer18_attention_weight_min": -37.61174392700195,
      "activations/layer19_attention_weight_max": 22.63459587097168,
      "activations/layer19_attention_weight_min": -22.782169342041016,
      "activations/layer1_attention_weight_max": 16.647558212280273,
      "activations/layer1_attention_weight_min": -15.330448150634766,
      "activations/layer20_attention_weight_max": 22.196134567260742,
      "activations/layer20_attention_weight_min": -21.57620620727539,
      "activations/layer21_attention_weight_max": 37.26502227783203,
      "activations/layer21_attention_weight_min": -25.111780166625977,
      "activations/layer22_attention_weight_max": 29.42232894897461,
      "activations/layer22_attention_weight_min": -26.98405647277832,
      "activations/layer23_attention_weight_max": 34.94671630859375,
      "activations/layer23_attention_weight_min": -26.186506271362305,
      "activations/layer2_attention_weight_max": 31.789777755737305,
      "activations/layer2_attention_weight_min": -32.593299865722656,
      "activations/layer3_attention_weight_max": 98.07439422607422,
      "activations/layer3_attention_weight_min": -103.56558227539062,
      "activations/layer4_attention_weight_max": 99.63336944580078,
      "activations/layer4_attention_weight_min": -98.04812622070312,
      "activations/layer5_attention_weight_max": 72.83259582519531,
      "activations/layer5_attention_weight_min": -76.78485107421875,
      "activations/layer6_attention_weight_max": 51.47943878173828,
      "activations/layer6_attention_weight_min": -51.833988189697266,
      "activations/layer7_attention_weight_max": 71.1348648071289,
      "activations/layer7_attention_weight_min": -65.45287322998047,
      "activations/layer8_attention_weight_max": 45.08300018310547,
      "activations/layer8_attention_weight_min": -46.73638153076172,
      "activations/layer9_attention_weight_max": 48.06662368774414,
      "activations/layer9_attention_weight_min": -47.905006408691406,
      "epoch": 13.13,
      "learning_rate": 6.59996212121212e-05,
      "loss": 2.7621,
      "step": 225950
    },
    {
      "activations/layer0_attention_weight_max": 15.961235046386719,
      "activations/layer0_attention_weight_min": -14.01652717590332,
      "activations/layer10_attention_weight_max": 42.96156692504883,
      "activations/layer10_attention_weight_min": -39.52869415283203,
      "activations/layer11_attention_weight_max": 39.955604553222656,
      "activations/layer11_attention_weight_min": -36.60773468017578,
      "activations/layer12_attention_weight_max": 22.99355697631836,
      "activations/layer12_attention_weight_min": -23.27265739440918,
      "activations/layer13_attention_weight_max": 58.51221466064453,
      "activations/layer13_attention_weight_min": -34.89828109741211,
      "activations/layer14_attention_weight_max": 47.4901237487793,
      "activations/layer14_attention_weight_min": -34.95003128051758,
      "activations/layer15_attention_weight_max": 45.680057525634766,
      "activations/layer15_attention_weight_min": -35.27460479736328,
      "activations/layer16_attention_weight_max": 42.955718994140625,
      "activations/layer16_attention_weight_min": -35.63753890991211,
      "activations/layer17_attention_weight_max": 66.171142578125,
      "activations/layer17_attention_weight_min": -55.038063049316406,
      "activations/layer18_attention_weight_max": 60.460174560546875,
      "activations/layer18_attention_weight_min": -47.90140151977539,
      "activations/layer19_attention_weight_max": 28.819950103759766,
      "activations/layer19_attention_weight_min": -26.0311222076416,
      "activations/layer1_attention_weight_max": 16.591386795043945,
      "activations/layer1_attention_weight_min": -16.416736602783203,
      "activations/layer20_attention_weight_max": 30.79168128967285,
      "activations/layer20_attention_weight_min": -22.926834106445312,
      "activations/layer21_attention_weight_max": 54.196014404296875,
      "activations/layer21_attention_weight_min": -23.361919403076172,
      "activations/layer22_attention_weight_max": 40.83615493774414,
      "activations/layer22_attention_weight_min": -26.28479766845703,
      "activations/layer23_attention_weight_max": 51.214447021484375,
      "activations/layer23_attention_weight_min": -23.629032135009766,
      "activations/layer2_attention_weight_max": 35.04309844970703,
      "activations/layer2_attention_weight_min": -34.569358825683594,
      "activations/layer3_attention_weight_max": 104.13883972167969,
      "activations/layer3_attention_weight_min": -104.69767761230469,
      "activations/layer4_attention_weight_max": 104.08089447021484,
      "activations/layer4_attention_weight_min": -99.37603759765625,
      "activations/layer5_attention_weight_max": 71.04454040527344,
      "activations/layer5_attention_weight_min": -83.3411865234375,
      "activations/layer6_attention_weight_max": 56.915809631347656,
      "activations/layer6_attention_weight_min": -57.208648681640625,
      "activations/layer7_attention_weight_max": 72.14888000488281,
      "activations/layer7_attention_weight_min": -73.69123077392578,
      "activations/layer8_attention_weight_max": 52.79024124145508,
      "activations/layer8_attention_weight_min": -49.936981201171875,
      "activations/layer9_attention_weight_max": 55.02959060668945,
      "activations/layer9_attention_weight_min": -51.35369873046875,
      "epoch": 13.13,
      "learning_rate": 6.59806818181818e-05,
      "loss": 2.7497,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.5018,
      "eval_samples_per_second": 505.067,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.5018,
      "eval_openwebtext_samples_per_second": 505.067,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9686,
      "eval_wikitext_samples_per_second": 231.633,
      "step": 226000
    },
    {
      "epoch": 13.13,
      "eval_lambada_loss": 2.517578125,
      "eval_lambada_ppl": 12.398532576458457,
      "eval_lambada_runtime": 9.555,
      "eval_lambada_samples_per_second": 509.577,
      "step": 226000
    },
    {
      "activations/layer0_attention_weight_max": 16.8968448638916,
      "activations/layer0_attention_weight_min": -14.067830085754395,
      "activations/layer10_attention_weight_max": 33.02961349487305,
      "activations/layer10_attention_weight_min": -33.78911209106445,
      "activations/layer11_attention_weight_max": 30.83241081237793,
      "activations/layer11_attention_weight_min": -30.09333610534668,
      "activations/layer12_attention_weight_max": 17.588268280029297,
      "activations/layer12_attention_weight_min": -26.03902816772461,
      "activations/layer13_attention_weight_max": 33.38102722167969,
      "activations/layer13_attention_weight_min": -31.016433715820312,
      "activations/layer14_attention_weight_max": 36.14733123779297,
      "activations/layer14_attention_weight_min": -29.674455642700195,
      "activations/layer15_attention_weight_max": 31.70783805847168,
      "activations/layer15_attention_weight_min": -28.993799209594727,
      "activations/layer16_attention_weight_max": 29.55768585205078,
      "activations/layer16_attention_weight_min": -30.034687042236328,
      "activations/layer17_attention_weight_max": 48.98545455932617,
      "activations/layer17_attention_weight_min": -45.46595001220703,
      "activations/layer18_attention_weight_max": 41.554752349853516,
      "activations/layer18_attention_weight_min": -37.47889709472656,
      "activations/layer19_attention_weight_max": 20.98102569580078,
      "activations/layer19_attention_weight_min": -20.593801498413086,
      "activations/layer1_attention_weight_max": 17.017541885375977,
      "activations/layer1_attention_weight_min": -17.11871337890625,
      "activations/layer20_attention_weight_max": 20.556102752685547,
      "activations/layer20_attention_weight_min": -22.61747169494629,
      "activations/layer21_attention_weight_max": 32.22734069824219,
      "activations/layer21_attention_weight_min": -21.930593490600586,
      "activations/layer22_attention_weight_max": 28.5880069732666,
      "activations/layer22_attention_weight_min": -25.38539695739746,
      "activations/layer23_attention_weight_max": 34.25236511230469,
      "activations/layer23_attention_weight_min": -26.73416519165039,
      "activations/layer2_attention_weight_max": 34.01396942138672,
      "activations/layer2_attention_weight_min": -31.986417770385742,
      "activations/layer3_attention_weight_max": 94.6843032836914,
      "activations/layer3_attention_weight_min": -97.49738311767578,
      "activations/layer4_attention_weight_max": 93.98042297363281,
      "activations/layer4_attention_weight_min": -91.8588638305664,
      "activations/layer5_attention_weight_max": 71.04165649414062,
      "activations/layer5_attention_weight_min": -79.9402847290039,
      "activations/layer6_attention_weight_max": 50.40935516357422,
      "activations/layer6_attention_weight_min": -50.44704818725586,
      "activations/layer7_attention_weight_max": 66.07304382324219,
      "activations/layer7_attention_weight_min": -65.5771713256836,
      "activations/layer8_attention_weight_max": 41.218692779541016,
      "activations/layer8_attention_weight_min": -42.77614974975586,
      "activations/layer9_attention_weight_max": 44.032161712646484,
      "activations/layer9_attention_weight_min": -44.20266342163086,
      "epoch": 13.13,
      "learning_rate": 6.596174242424242e-05,
      "loss": 2.7784,
      "step": 226050
    },
    {
      "activations/layer0_attention_weight_max": 15.12672233581543,
      "activations/layer0_attention_weight_min": -15.19701099395752,
      "activations/layer10_attention_weight_max": 34.750465393066406,
      "activations/layer10_attention_weight_min": -34.3669319152832,
      "activations/layer11_attention_weight_max": 30.525344848632812,
      "activations/layer11_attention_weight_min": -33.97825622558594,
      "activations/layer12_attention_weight_max": 21.828092575073242,
      "activations/layer12_attention_weight_min": -24.30019760131836,
      "activations/layer13_attention_weight_max": 37.689048767089844,
      "activations/layer13_attention_weight_min": -31.582538604736328,
      "activations/layer14_attention_weight_max": 32.54823303222656,
      "activations/layer14_attention_weight_min": -31.421428680419922,
      "activations/layer15_attention_weight_max": 29.932931900024414,
      "activations/layer15_attention_weight_min": -29.246164321899414,
      "activations/layer16_attention_weight_max": 29.757081985473633,
      "activations/layer16_attention_weight_min": -28.538908004760742,
      "activations/layer17_attention_weight_max": 48.69363021850586,
      "activations/layer17_attention_weight_min": -43.34367752075195,
      "activations/layer18_attention_weight_max": 44.83506774902344,
      "activations/layer18_attention_weight_min": -36.47726821899414,
      "activations/layer19_attention_weight_max": 21.6966495513916,
      "activations/layer19_attention_weight_min": -21.430828094482422,
      "activations/layer1_attention_weight_max": 16.113449096679688,
      "activations/layer1_attention_weight_min": -15.120789527893066,
      "activations/layer20_attention_weight_max": 21.788331985473633,
      "activations/layer20_attention_weight_min": -23.41324806213379,
      "activations/layer21_attention_weight_max": 33.9837646484375,
      "activations/layer21_attention_weight_min": -23.7554988861084,
      "activations/layer22_attention_weight_max": 28.375120162963867,
      "activations/layer22_attention_weight_min": -25.21880340576172,
      "activations/layer23_attention_weight_max": 34.249263763427734,
      "activations/layer23_attention_weight_min": -27.310497283935547,
      "activations/layer2_attention_weight_max": 32.21368408203125,
      "activations/layer2_attention_weight_min": -29.987951278686523,
      "activations/layer3_attention_weight_max": 91.485595703125,
      "activations/layer3_attention_weight_min": -92.64189910888672,
      "activations/layer4_attention_weight_max": 95.01824188232422,
      "activations/layer4_attention_weight_min": -91.66826629638672,
      "activations/layer5_attention_weight_max": 73.22271728515625,
      "activations/layer5_attention_weight_min": -78.49501037597656,
      "activations/layer6_attention_weight_max": 50.025814056396484,
      "activations/layer6_attention_weight_min": -51.535438537597656,
      "activations/layer7_attention_weight_max": 66.50527954101562,
      "activations/layer7_attention_weight_min": -62.9636344909668,
      "activations/layer8_attention_weight_max": 42.97332000732422,
      "activations/layer8_attention_weight_min": -46.43402099609375,
      "activations/layer9_attention_weight_max": 43.70335006713867,
      "activations/layer9_attention_weight_min": -46.20287322998047,
      "epoch": 13.14,
      "learning_rate": 6.594280303030302e-05,
      "loss": 2.7768,
      "step": 226100
    },
    {
      "activations/layer0_attention_weight_max": 15.59138298034668,
      "activations/layer0_attention_weight_min": -14.124855995178223,
      "activations/layer10_attention_weight_max": 40.7315559387207,
      "activations/layer10_attention_weight_min": -38.42795944213867,
      "activations/layer11_attention_weight_max": 34.627376556396484,
      "activations/layer11_attention_weight_min": -35.18035125732422,
      "activations/layer12_attention_weight_max": 17.928699493408203,
      "activations/layer12_attention_weight_min": -24.339717864990234,
      "activations/layer13_attention_weight_max": 35.14022445678711,
      "activations/layer13_attention_weight_min": -28.674556732177734,
      "activations/layer14_attention_weight_max": 35.05338668823242,
      "activations/layer14_attention_weight_min": -29.911972045898438,
      "activations/layer15_attention_weight_max": 32.650123596191406,
      "activations/layer15_attention_weight_min": -28.423707962036133,
      "activations/layer16_attention_weight_max": 29.18366813659668,
      "activations/layer16_attention_weight_min": -28.635271072387695,
      "activations/layer17_attention_weight_max": 47.93171691894531,
      "activations/layer17_attention_weight_min": -42.04563903808594,
      "activations/layer18_attention_weight_max": 45.00796127319336,
      "activations/layer18_attention_weight_min": -36.191200256347656,
      "activations/layer19_attention_weight_max": 21.787416458129883,
      "activations/layer19_attention_weight_min": -20.126461029052734,
      "activations/layer1_attention_weight_max": 17.493330001831055,
      "activations/layer1_attention_weight_min": -16.35991668701172,
      "activations/layer20_attention_weight_max": 22.225543975830078,
      "activations/layer20_attention_weight_min": -21.25657844543457,
      "activations/layer21_attention_weight_max": 37.010433197021484,
      "activations/layer21_attention_weight_min": -26.80573081970215,
      "activations/layer22_attention_weight_max": 29.4220027923584,
      "activations/layer22_attention_weight_min": -26.21773338317871,
      "activations/layer23_attention_weight_max": 33.72407150268555,
      "activations/layer23_attention_weight_min": -23.741552352905273,
      "activations/layer2_attention_weight_max": 32.421470642089844,
      "activations/layer2_attention_weight_min": -30.55847930908203,
      "activations/layer3_attention_weight_max": 92.28913879394531,
      "activations/layer3_attention_weight_min": -95.58067321777344,
      "activations/layer4_attention_weight_max": 98.27592468261719,
      "activations/layer4_attention_weight_min": -96.10611724853516,
      "activations/layer5_attention_weight_max": 72.08880615234375,
      "activations/layer5_attention_weight_min": -80.9617919921875,
      "activations/layer6_attention_weight_max": 53.650272369384766,
      "activations/layer6_attention_weight_min": -56.04792022705078,
      "activations/layer7_attention_weight_max": 71.53705596923828,
      "activations/layer7_attention_weight_min": -69.9474868774414,
      "activations/layer8_attention_weight_max": 49.43259048461914,
      "activations/layer8_attention_weight_min": -56.56840896606445,
      "activations/layer9_attention_weight_max": 49.20457077026367,
      "activations/layer9_attention_weight_min": -52.380035400390625,
      "epoch": 13.14,
      "learning_rate": 6.592386363636364e-05,
      "loss": 2.7504,
      "step": 226150
    },
    {
      "activations/layer0_attention_weight_max": 15.643842697143555,
      "activations/layer0_attention_weight_min": -13.824105262756348,
      "activations/layer10_attention_weight_max": 36.766326904296875,
      "activations/layer10_attention_weight_min": -36.062191009521484,
      "activations/layer11_attention_weight_max": 32.113033294677734,
      "activations/layer11_attention_weight_min": -32.76482009887695,
      "activations/layer12_attention_weight_max": 19.720998764038086,
      "activations/layer12_attention_weight_min": -26.380691528320312,
      "activations/layer13_attention_weight_max": 44.17787551879883,
      "activations/layer13_attention_weight_min": -29.138626098632812,
      "activations/layer14_attention_weight_max": 37.056461334228516,
      "activations/layer14_attention_weight_min": -31.238628387451172,
      "activations/layer15_attention_weight_max": 33.91238021850586,
      "activations/layer15_attention_weight_min": -29.462783813476562,
      "activations/layer16_attention_weight_max": 32.683616638183594,
      "activations/layer16_attention_weight_min": -32.268836975097656,
      "activations/layer17_attention_weight_max": 51.3809814453125,
      "activations/layer17_attention_weight_min": -48.016204833984375,
      "activations/layer18_attention_weight_max": 50.09965133666992,
      "activations/layer18_attention_weight_min": -39.96155548095703,
      "activations/layer19_attention_weight_max": 22.711904525756836,
      "activations/layer19_attention_weight_min": -22.821094512939453,
      "activations/layer1_attention_weight_max": 16.140230178833008,
      "activations/layer1_attention_weight_min": -14.565491676330566,
      "activations/layer20_attention_weight_max": 22.449199676513672,
      "activations/layer20_attention_weight_min": -27.285015106201172,
      "activations/layer21_attention_weight_max": 37.76482009887695,
      "activations/layer21_attention_weight_min": -21.831186294555664,
      "activations/layer22_attention_weight_max": 32.54206848144531,
      "activations/layer22_attention_weight_min": -26.39330291748047,
      "activations/layer23_attention_weight_max": 37.20629119873047,
      "activations/layer23_attention_weight_min": -25.315855026245117,
      "activations/layer2_attention_weight_max": 32.50789260864258,
      "activations/layer2_attention_weight_min": -29.933738708496094,
      "activations/layer3_attention_weight_max": 95.30633544921875,
      "activations/layer3_attention_weight_min": -96.0969009399414,
      "activations/layer4_attention_weight_max": 97.14360046386719,
      "activations/layer4_attention_weight_min": -92.85431671142578,
      "activations/layer5_attention_weight_max": 70.400634765625,
      "activations/layer5_attention_weight_min": -79.43333435058594,
      "activations/layer6_attention_weight_max": 51.03688430786133,
      "activations/layer6_attention_weight_min": -51.19965362548828,
      "activations/layer7_attention_weight_max": 71.02769470214844,
      "activations/layer7_attention_weight_min": -65.39570617675781,
      "activations/layer8_attention_weight_max": 47.59300994873047,
      "activations/layer8_attention_weight_min": -51.38817596435547,
      "activations/layer9_attention_weight_max": 48.0068473815918,
      "activations/layer9_attention_weight_min": -46.56031799316406,
      "epoch": 13.14,
      "learning_rate": 6.590492424242424e-05,
      "loss": 2.7628,
      "step": 226200
    },
    {
      "activations/layer0_attention_weight_max": 16.45445442199707,
      "activations/layer0_attention_weight_min": -13.768720626831055,
      "activations/layer10_attention_weight_max": 37.10350036621094,
      "activations/layer10_attention_weight_min": -35.07928466796875,
      "activations/layer11_attention_weight_max": 31.99201774597168,
      "activations/layer11_attention_weight_min": -33.76813888549805,
      "activations/layer12_attention_weight_max": 18.175434112548828,
      "activations/layer12_attention_weight_min": -27.607282638549805,
      "activations/layer13_attention_weight_max": 36.73991775512695,
      "activations/layer13_attention_weight_min": -30.793960571289062,
      "activations/layer14_attention_weight_max": 37.3122444152832,
      "activations/layer14_attention_weight_min": -30.004653930664062,
      "activations/layer15_attention_weight_max": 32.548484802246094,
      "activations/layer15_attention_weight_min": -28.875505447387695,
      "activations/layer16_attention_weight_max": 33.23239517211914,
      "activations/layer16_attention_weight_min": -30.52604103088379,
      "activations/layer17_attention_weight_max": 51.996822357177734,
      "activations/layer17_attention_weight_min": -43.41260528564453,
      "activations/layer18_attention_weight_max": 48.758567810058594,
      "activations/layer18_attention_weight_min": -37.286590576171875,
      "activations/layer19_attention_weight_max": 23.86414909362793,
      "activations/layer19_attention_weight_min": -21.642261505126953,
      "activations/layer1_attention_weight_max": 15.254355430603027,
      "activations/layer1_attention_weight_min": -14.578157424926758,
      "activations/layer20_attention_weight_max": 22.486064910888672,
      "activations/layer20_attention_weight_min": -22.56463050842285,
      "activations/layer21_attention_weight_max": 39.6893424987793,
      "activations/layer21_attention_weight_min": -26.993255615234375,
      "activations/layer22_attention_weight_max": 27.952239990234375,
      "activations/layer22_attention_weight_min": -25.43367576599121,
      "activations/layer23_attention_weight_max": 35.262149810791016,
      "activations/layer23_attention_weight_min": -23.838050842285156,
      "activations/layer2_attention_weight_max": 32.04617691040039,
      "activations/layer2_attention_weight_min": -31.316871643066406,
      "activations/layer3_attention_weight_max": 93.08727264404297,
      "activations/layer3_attention_weight_min": -98.09717559814453,
      "activations/layer4_attention_weight_max": 93.65182495117188,
      "activations/layer4_attention_weight_min": -91.91442108154297,
      "activations/layer5_attention_weight_max": 72.95797729492188,
      "activations/layer5_attention_weight_min": -74.52052307128906,
      "activations/layer6_attention_weight_max": 49.89098358154297,
      "activations/layer6_attention_weight_min": -49.12499237060547,
      "activations/layer7_attention_weight_max": 68.19063568115234,
      "activations/layer7_attention_weight_min": -69.08599090576172,
      "activations/layer8_attention_weight_max": 45.15644454956055,
      "activations/layer8_attention_weight_min": -48.07280731201172,
      "activations/layer9_attention_weight_max": 48.35685348510742,
      "activations/layer9_attention_weight_min": -47.035648345947266,
      "epoch": 13.15,
      "learning_rate": 6.588598484848485e-05,
      "loss": 2.7804,
      "step": 226250
    },
    {
      "activations/layer0_attention_weight_max": 15.564566612243652,
      "activations/layer0_attention_weight_min": -14.209866523742676,
      "activations/layer10_attention_weight_max": 37.65302276611328,
      "activations/layer10_attention_weight_min": -36.04371643066406,
      "activations/layer11_attention_weight_max": 31.938457489013672,
      "activations/layer11_attention_weight_min": -32.82860565185547,
      "activations/layer12_attention_weight_max": 20.6850643157959,
      "activations/layer12_attention_weight_min": -28.66682243347168,
      "activations/layer13_attention_weight_max": 37.327293395996094,
      "activations/layer13_attention_weight_min": -28.01513671875,
      "activations/layer14_attention_weight_max": 35.10108947753906,
      "activations/layer14_attention_weight_min": -29.54722785949707,
      "activations/layer15_attention_weight_max": 33.10673904418945,
      "activations/layer15_attention_weight_min": -27.890159606933594,
      "activations/layer16_attention_weight_max": 32.66868591308594,
      "activations/layer16_attention_weight_min": -27.66468048095703,
      "activations/layer17_attention_weight_max": 48.78935241699219,
      "activations/layer17_attention_weight_min": -41.67989730834961,
      "activations/layer18_attention_weight_max": 43.404056549072266,
      "activations/layer18_attention_weight_min": -37.49470520019531,
      "activations/layer19_attention_weight_max": 21.10141372680664,
      "activations/layer19_attention_weight_min": -19.884178161621094,
      "activations/layer1_attention_weight_max": 16.040742874145508,
      "activations/layer1_attention_weight_min": -15.47143268585205,
      "activations/layer20_attention_weight_max": 21.916915893554688,
      "activations/layer20_attention_weight_min": -20.323923110961914,
      "activations/layer21_attention_weight_max": 29.748083114624023,
      "activations/layer21_attention_weight_min": -21.87969398498535,
      "activations/layer22_attention_weight_max": 32.102909088134766,
      "activations/layer22_attention_weight_min": -23.145071029663086,
      "activations/layer23_attention_weight_max": 40.39094161987305,
      "activations/layer23_attention_weight_min": -20.789918899536133,
      "activations/layer2_attention_weight_max": 31.294599533081055,
      "activations/layer2_attention_weight_min": -30.786869049072266,
      "activations/layer3_attention_weight_max": 92.2145767211914,
      "activations/layer3_attention_weight_min": -95.79830169677734,
      "activations/layer4_attention_weight_max": 95.4505844116211,
      "activations/layer4_attention_weight_min": -93.06787109375,
      "activations/layer5_attention_weight_max": 71.1773681640625,
      "activations/layer5_attention_weight_min": -76.43315887451172,
      "activations/layer6_attention_weight_max": 52.08009719848633,
      "activations/layer6_attention_weight_min": -51.64558792114258,
      "activations/layer7_attention_weight_max": 70.4120864868164,
      "activations/layer7_attention_weight_min": -66.3202133178711,
      "activations/layer8_attention_weight_max": 45.49820327758789,
      "activations/layer8_attention_weight_min": -47.55438232421875,
      "activations/layer9_attention_weight_max": 49.42411804199219,
      "activations/layer9_attention_weight_min": -46.9854850769043,
      "epoch": 13.15,
      "learning_rate": 6.586704545454545e-05,
      "loss": 2.7726,
      "step": 226300
    },
    {
      "activations/layer0_attention_weight_max": 15.045838356018066,
      "activations/layer0_attention_weight_min": -13.714225769042969,
      "activations/layer10_attention_weight_max": 41.54710006713867,
      "activations/layer10_attention_weight_min": -39.78913116455078,
      "activations/layer11_attention_weight_max": 35.95225524902344,
      "activations/layer11_attention_weight_min": -35.12876892089844,
      "activations/layer12_attention_weight_max": 19.21439552307129,
      "activations/layer12_attention_weight_min": -29.98540687561035,
      "activations/layer13_attention_weight_max": 40.5955810546875,
      "activations/layer13_attention_weight_min": -29.26517105102539,
      "activations/layer14_attention_weight_max": 39.50602722167969,
      "activations/layer14_attention_weight_min": -31.74892807006836,
      "activations/layer15_attention_weight_max": 38.90766525268555,
      "activations/layer15_attention_weight_min": -30.238359451293945,
      "activations/layer16_attention_weight_max": 34.9316520690918,
      "activations/layer16_attention_weight_min": -31.633663177490234,
      "activations/layer17_attention_weight_max": 58.01692581176758,
      "activations/layer17_attention_weight_min": -45.474998474121094,
      "activations/layer18_attention_weight_max": 50.33345031738281,
      "activations/layer18_attention_weight_min": -38.605865478515625,
      "activations/layer19_attention_weight_max": 23.157135009765625,
      "activations/layer19_attention_weight_min": -21.5794734954834,
      "activations/layer1_attention_weight_max": 16.953693389892578,
      "activations/layer1_attention_weight_min": -16.42143440246582,
      "activations/layer20_attention_weight_max": 25.436063766479492,
      "activations/layer20_attention_weight_min": -21.403501510620117,
      "activations/layer21_attention_weight_max": 36.519126892089844,
      "activations/layer21_attention_weight_min": -25.136411666870117,
      "activations/layer22_attention_weight_max": 31.392547607421875,
      "activations/layer22_attention_weight_min": -25.2420654296875,
      "activations/layer23_attention_weight_max": 38.214080810546875,
      "activations/layer23_attention_weight_min": -27.594669342041016,
      "activations/layer2_attention_weight_max": 35.168617248535156,
      "activations/layer2_attention_weight_min": -32.972103118896484,
      "activations/layer3_attention_weight_max": 99.15562438964844,
      "activations/layer3_attention_weight_min": -95.75257110595703,
      "activations/layer4_attention_weight_max": 101.08318328857422,
      "activations/layer4_attention_weight_min": -99.98561096191406,
      "activations/layer5_attention_weight_max": 71.47830963134766,
      "activations/layer5_attention_weight_min": -75.6790771484375,
      "activations/layer6_attention_weight_max": 57.6204719543457,
      "activations/layer6_attention_weight_min": -54.342018127441406,
      "activations/layer7_attention_weight_max": 73.72991943359375,
      "activations/layer7_attention_weight_min": -69.70941162109375,
      "activations/layer8_attention_weight_max": 50.10195541381836,
      "activations/layer8_attention_weight_min": -51.77511978149414,
      "activations/layer9_attention_weight_max": 53.202003479003906,
      "activations/layer9_attention_weight_min": -51.97528076171875,
      "epoch": 13.15,
      "learning_rate": 6.584848484848484e-05,
      "loss": 2.7795,
      "step": 226350
    },
    {
      "activations/layer0_attention_weight_max": 16.18041229248047,
      "activations/layer0_attention_weight_min": -13.80923843383789,
      "activations/layer10_attention_weight_max": 37.840850830078125,
      "activations/layer10_attention_weight_min": -40.15544509887695,
      "activations/layer11_attention_weight_max": 32.00481414794922,
      "activations/layer11_attention_weight_min": -35.465782165527344,
      "activations/layer12_attention_weight_max": 17.088659286499023,
      "activations/layer12_attention_weight_min": -29.286399841308594,
      "activations/layer13_attention_weight_max": 36.45338439941406,
      "activations/layer13_attention_weight_min": -28.616262435913086,
      "activations/layer14_attention_weight_max": 34.9332275390625,
      "activations/layer14_attention_weight_min": -30.048873901367188,
      "activations/layer15_attention_weight_max": 35.90214538574219,
      "activations/layer15_attention_weight_min": -28.376140594482422,
      "activations/layer16_attention_weight_max": 30.248746871948242,
      "activations/layer16_attention_weight_min": -30.08122444152832,
      "activations/layer17_attention_weight_max": 51.31537628173828,
      "activations/layer17_attention_weight_min": -45.99026870727539,
      "activations/layer18_attention_weight_max": 45.562339782714844,
      "activations/layer18_attention_weight_min": -37.76795196533203,
      "activations/layer19_attention_weight_max": 24.005510330200195,
      "activations/layer19_attention_weight_min": -21.30658531188965,
      "activations/layer1_attention_weight_max": 15.883238792419434,
      "activations/layer1_attention_weight_min": -12.757083892822266,
      "activations/layer20_attention_weight_max": 23.34149169921875,
      "activations/layer20_attention_weight_min": -19.642833709716797,
      "activations/layer21_attention_weight_max": 39.69756317138672,
      "activations/layer21_attention_weight_min": -23.707422256469727,
      "activations/layer22_attention_weight_max": 31.19306755065918,
      "activations/layer22_attention_weight_min": -25.31772804260254,
      "activations/layer23_attention_weight_max": 38.38419723510742,
      "activations/layer23_attention_weight_min": -22.107664108276367,
      "activations/layer2_attention_weight_max": 28.893218994140625,
      "activations/layer2_attention_weight_min": -27.56230926513672,
      "activations/layer3_attention_weight_max": 91.05986022949219,
      "activations/layer3_attention_weight_min": -93.77424621582031,
      "activations/layer4_attention_weight_max": 92.6645736694336,
      "activations/layer4_attention_weight_min": -92.11226654052734,
      "activations/layer5_attention_weight_max": 71.46498107910156,
      "activations/layer5_attention_weight_min": -80.32806396484375,
      "activations/layer6_attention_weight_max": 50.44879913330078,
      "activations/layer6_attention_weight_min": -51.18272018432617,
      "activations/layer7_attention_weight_max": 68.88671112060547,
      "activations/layer7_attention_weight_min": -68.93009948730469,
      "activations/layer8_attention_weight_max": 45.52265930175781,
      "activations/layer8_attention_weight_min": -49.19929122924805,
      "activations/layer9_attention_weight_max": 49.1567497253418,
      "activations/layer9_attention_weight_min": -47.86039352416992,
      "epoch": 13.16,
      "learning_rate": 6.582954545454545e-05,
      "loss": 2.7409,
      "step": 226400
    },
    {
      "activations/layer0_attention_weight_max": 16.070127487182617,
      "activations/layer0_attention_weight_min": -13.874757766723633,
      "activations/layer10_attention_weight_max": 38.772064208984375,
      "activations/layer10_attention_weight_min": -39.43211364746094,
      "activations/layer11_attention_weight_max": 33.16557312011719,
      "activations/layer11_attention_weight_min": -33.707584381103516,
      "activations/layer12_attention_weight_max": 21.611722946166992,
      "activations/layer12_attention_weight_min": -23.787179946899414,
      "activations/layer13_attention_weight_max": 40.70751190185547,
      "activations/layer13_attention_weight_min": -31.722026824951172,
      "activations/layer14_attention_weight_max": 37.439971923828125,
      "activations/layer14_attention_weight_min": -30.56205177307129,
      "activations/layer15_attention_weight_max": 34.30741500854492,
      "activations/layer15_attention_weight_min": -29.719362258911133,
      "activations/layer16_attention_weight_max": 33.85047149658203,
      "activations/layer16_attention_weight_min": -29.629493713378906,
      "activations/layer17_attention_weight_max": 53.19134521484375,
      "activations/layer17_attention_weight_min": -44.27681350708008,
      "activations/layer18_attention_weight_max": 50.41751480102539,
      "activations/layer18_attention_weight_min": -38.15861511230469,
      "activations/layer19_attention_weight_max": 23.56452178955078,
      "activations/layer19_attention_weight_min": -23.80262565612793,
      "activations/layer1_attention_weight_max": 16.609947204589844,
      "activations/layer1_attention_weight_min": -15.1936674118042,
      "activations/layer20_attention_weight_max": 24.0278377532959,
      "activations/layer20_attention_weight_min": -22.858617782592773,
      "activations/layer21_attention_weight_max": 43.37708282470703,
      "activations/layer21_attention_weight_min": -25.44353675842285,
      "activations/layer22_attention_weight_max": 30.641557693481445,
      "activations/layer22_attention_weight_min": -29.355012893676758,
      "activations/layer23_attention_weight_max": 37.763214111328125,
      "activations/layer23_attention_weight_min": -26.711864471435547,
      "activations/layer2_attention_weight_max": 34.22527313232422,
      "activations/layer2_attention_weight_min": -32.6973876953125,
      "activations/layer3_attention_weight_max": 92.20286560058594,
      "activations/layer3_attention_weight_min": -96.688720703125,
      "activations/layer4_attention_weight_max": 95.05757904052734,
      "activations/layer4_attention_weight_min": -92.85465240478516,
      "activations/layer5_attention_weight_max": 72.1192855834961,
      "activations/layer5_attention_weight_min": -77.16094970703125,
      "activations/layer6_attention_weight_max": 51.104305267333984,
      "activations/layer6_attention_weight_min": -52.19572067260742,
      "activations/layer7_attention_weight_max": 74.34195709228516,
      "activations/layer7_attention_weight_min": -66.25456237792969,
      "activations/layer8_attention_weight_max": 48.36180114746094,
      "activations/layer8_attention_weight_min": -48.727542877197266,
      "activations/layer9_attention_weight_max": 49.384395599365234,
      "activations/layer9_attention_weight_min": -47.4080924987793,
      "epoch": 13.16,
      "learning_rate": 6.581060606060606e-05,
      "loss": 2.7478,
      "step": 226450
    },
    {
      "activations/layer0_attention_weight_max": 16.003114700317383,
      "activations/layer0_attention_weight_min": -13.902048110961914,
      "activations/layer10_attention_weight_max": 34.11312484741211,
      "activations/layer10_attention_weight_min": -36.66666793823242,
      "activations/layer11_attention_weight_max": 31.9910945892334,
      "activations/layer11_attention_weight_min": -32.463539123535156,
      "activations/layer12_attention_weight_max": 16.269023895263672,
      "activations/layer12_attention_weight_min": -25.689895629882812,
      "activations/layer13_attention_weight_max": 36.45456314086914,
      "activations/layer13_attention_weight_min": -27.340368270874023,
      "activations/layer14_attention_weight_max": 37.70413589477539,
      "activations/layer14_attention_weight_min": -28.715557098388672,
      "activations/layer15_attention_weight_max": 36.1098747253418,
      "activations/layer15_attention_weight_min": -27.860698699951172,
      "activations/layer16_attention_weight_max": 30.58681297302246,
      "activations/layer16_attention_weight_min": -28.036550521850586,
      "activations/layer17_attention_weight_max": 44.27674865722656,
      "activations/layer17_attention_weight_min": -43.039581298828125,
      "activations/layer18_attention_weight_max": 42.067203521728516,
      "activations/layer18_attention_weight_min": -38.702423095703125,
      "activations/layer19_attention_weight_max": 23.011159896850586,
      "activations/layer19_attention_weight_min": -20.465702056884766,
      "activations/layer1_attention_weight_max": 15.644216537475586,
      "activations/layer1_attention_weight_min": -14.410289764404297,
      "activations/layer20_attention_weight_max": 21.955516815185547,
      "activations/layer20_attention_weight_min": -21.86650276184082,
      "activations/layer21_attention_weight_max": 39.03031921386719,
      "activations/layer21_attention_weight_min": -24.813371658325195,
      "activations/layer22_attention_weight_max": 29.547151565551758,
      "activations/layer22_attention_weight_min": -25.633441925048828,
      "activations/layer23_attention_weight_max": 35.302406311035156,
      "activations/layer23_attention_weight_min": -26.8199462890625,
      "activations/layer2_attention_weight_max": 33.22499465942383,
      "activations/layer2_attention_weight_min": -31.192237854003906,
      "activations/layer3_attention_weight_max": 98.60729217529297,
      "activations/layer3_attention_weight_min": -99.19435119628906,
      "activations/layer4_attention_weight_max": 97.90084838867188,
      "activations/layer4_attention_weight_min": -95.25806427001953,
      "activations/layer5_attention_weight_max": 75.0997314453125,
      "activations/layer5_attention_weight_min": -78.2098388671875,
      "activations/layer6_attention_weight_max": 51.70275115966797,
      "activations/layer6_attention_weight_min": -53.103816986083984,
      "activations/layer7_attention_weight_max": 72.41357421875,
      "activations/layer7_attention_weight_min": -65.59587860107422,
      "activations/layer8_attention_weight_max": 43.60270309448242,
      "activations/layer8_attention_weight_min": -44.336265563964844,
      "activations/layer9_attention_weight_max": 46.29250717163086,
      "activations/layer9_attention_weight_min": -46.268497467041016,
      "epoch": 13.16,
      "learning_rate": 6.579166666666666e-05,
      "loss": 2.7795,
      "step": 226500
    },
    {
      "activations/layer0_attention_weight_max": 15.696639060974121,
      "activations/layer0_attention_weight_min": -13.747608184814453,
      "activations/layer10_attention_weight_max": 34.04487991333008,
      "activations/layer10_attention_weight_min": -34.018436431884766,
      "activations/layer11_attention_weight_max": 30.02532958984375,
      "activations/layer11_attention_weight_min": -32.644798278808594,
      "activations/layer12_attention_weight_max": 18.487470626831055,
      "activations/layer12_attention_weight_min": -25.46542739868164,
      "activations/layer13_attention_weight_max": 38.22058868408203,
      "activations/layer13_attention_weight_min": -28.903823852539062,
      "activations/layer14_attention_weight_max": 35.852928161621094,
      "activations/layer14_attention_weight_min": -32.61419677734375,
      "activations/layer15_attention_weight_max": 34.52088928222656,
      "activations/layer15_attention_weight_min": -32.17550277709961,
      "activations/layer16_attention_weight_max": 34.14984893798828,
      "activations/layer16_attention_weight_min": -31.73695945739746,
      "activations/layer17_attention_weight_max": 51.869911193847656,
      "activations/layer17_attention_weight_min": -48.47907257080078,
      "activations/layer18_attention_weight_max": 49.966793060302734,
      "activations/layer18_attention_weight_min": -43.393951416015625,
      "activations/layer19_attention_weight_max": 22.839906692504883,
      "activations/layer19_attention_weight_min": -23.8675594329834,
      "activations/layer1_attention_weight_max": 18.15744400024414,
      "activations/layer1_attention_weight_min": -16.704627990722656,
      "activations/layer20_attention_weight_max": 22.788127899169922,
      "activations/layer20_attention_weight_min": -23.62863540649414,
      "activations/layer21_attention_weight_max": 39.16809844970703,
      "activations/layer21_attention_weight_min": -24.335702896118164,
      "activations/layer22_attention_weight_max": 32.828250885009766,
      "activations/layer22_attention_weight_min": -28.100549697875977,
      "activations/layer23_attention_weight_max": 38.3251953125,
      "activations/layer23_attention_weight_min": -27.284988403320312,
      "activations/layer2_attention_weight_max": 32.53681182861328,
      "activations/layer2_attention_weight_min": -31.00558853149414,
      "activations/layer3_attention_weight_max": 87.24717712402344,
      "activations/layer3_attention_weight_min": -89.02084350585938,
      "activations/layer4_attention_weight_max": 89.99605560302734,
      "activations/layer4_attention_weight_min": -87.34081268310547,
      "activations/layer5_attention_weight_max": 74.67231750488281,
      "activations/layer5_attention_weight_min": -80.78848266601562,
      "activations/layer6_attention_weight_max": 50.97997283935547,
      "activations/layer6_attention_weight_min": -50.410396575927734,
      "activations/layer7_attention_weight_max": 68.10751342773438,
      "activations/layer7_attention_weight_min": -66.5272216796875,
      "activations/layer8_attention_weight_max": 45.00653076171875,
      "activations/layer8_attention_weight_min": -47.87513732910156,
      "activations/layer9_attention_weight_max": 45.427364349365234,
      "activations/layer9_attention_weight_min": -44.646820068359375,
      "epoch": 13.16,
      "learning_rate": 6.577348484848484e-05,
      "loss": 2.7555,
      "step": 226550
    },
    {
      "activations/layer0_attention_weight_max": 15.50195598602295,
      "activations/layer0_attention_weight_min": -13.714614868164062,
      "activations/layer10_attention_weight_max": 34.871585845947266,
      "activations/layer10_attention_weight_min": -37.185665130615234,
      "activations/layer11_attention_weight_max": 32.40984344482422,
      "activations/layer11_attention_weight_min": -36.93993377685547,
      "activations/layer12_attention_weight_max": 18.45224380493164,
      "activations/layer12_attention_weight_min": -25.180557250976562,
      "activations/layer13_attention_weight_max": 39.652034759521484,
      "activations/layer13_attention_weight_min": -31.34372901916504,
      "activations/layer14_attention_weight_max": 43.626773834228516,
      "activations/layer14_attention_weight_min": -32.0777702331543,
      "activations/layer15_attention_weight_max": 38.00697708129883,
      "activations/layer15_attention_weight_min": -30.27404022216797,
      "activations/layer16_attention_weight_max": 33.45035934448242,
      "activations/layer16_attention_weight_min": -31.48638916015625,
      "activations/layer17_attention_weight_max": 51.94427490234375,
      "activations/layer17_attention_weight_min": -44.44023895263672,
      "activations/layer18_attention_weight_max": 47.72718048095703,
      "activations/layer18_attention_weight_min": -37.82456588745117,
      "activations/layer19_attention_weight_max": 24.220550537109375,
      "activations/layer19_attention_weight_min": -22.417604446411133,
      "activations/layer1_attention_weight_max": 16.254724502563477,
      "activations/layer1_attention_weight_min": -16.491979598999023,
      "activations/layer20_attention_weight_max": 22.737546920776367,
      "activations/layer20_attention_weight_min": -22.002683639526367,
      "activations/layer21_attention_weight_max": 40.9930534362793,
      "activations/layer21_attention_weight_min": -24.35391616821289,
      "activations/layer22_attention_weight_max": 32.168067932128906,
      "activations/layer22_attention_weight_min": -26.41154670715332,
      "activations/layer23_attention_weight_max": 38.9600830078125,
      "activations/layer23_attention_weight_min": -30.1112060546875,
      "activations/layer2_attention_weight_max": 30.419124603271484,
      "activations/layer2_attention_weight_min": -30.52178955078125,
      "activations/layer3_attention_weight_max": 91.8222885131836,
      "activations/layer3_attention_weight_min": -91.16019439697266,
      "activations/layer4_attention_weight_max": 92.0259017944336,
      "activations/layer4_attention_weight_min": -89.8900146484375,
      "activations/layer5_attention_weight_max": 69.3829345703125,
      "activations/layer5_attention_weight_min": -76.76454162597656,
      "activations/layer6_attention_weight_max": 51.96957015991211,
      "activations/layer6_attention_weight_min": -53.043582916259766,
      "activations/layer7_attention_weight_max": 66.97193145751953,
      "activations/layer7_attention_weight_min": -69.50392150878906,
      "activations/layer8_attention_weight_max": 44.29788589477539,
      "activations/layer8_attention_weight_min": -45.16812515258789,
      "activations/layer9_attention_weight_max": 45.95922088623047,
      "activations/layer9_attention_weight_min": -46.47050094604492,
      "epoch": 13.17,
      "learning_rate": 6.575454545454544e-05,
      "loss": 2.7672,
      "step": 226600
    },
    {
      "activations/layer0_attention_weight_max": 16.83537483215332,
      "activations/layer0_attention_weight_min": -13.855545997619629,
      "activations/layer10_attention_weight_max": 35.162349700927734,
      "activations/layer10_attention_weight_min": -36.006629943847656,
      "activations/layer11_attention_weight_max": 31.27088737487793,
      "activations/layer11_attention_weight_min": -32.21332931518555,
      "activations/layer12_attention_weight_max": 18.33827781677246,
      "activations/layer12_attention_weight_min": -25.52037239074707,
      "activations/layer13_attention_weight_max": 36.81072235107422,
      "activations/layer13_attention_weight_min": -30.717342376708984,
      "activations/layer14_attention_weight_max": 36.95088195800781,
      "activations/layer14_attention_weight_min": -30.053287506103516,
      "activations/layer15_attention_weight_max": 36.212459564208984,
      "activations/layer15_attention_weight_min": -30.140777587890625,
      "activations/layer16_attention_weight_max": 34.172176361083984,
      "activations/layer16_attention_weight_min": -30.69521141052246,
      "activations/layer17_attention_weight_max": 52.21388626098633,
      "activations/layer17_attention_weight_min": -44.72872543334961,
      "activations/layer18_attention_weight_max": 46.362430572509766,
      "activations/layer18_attention_weight_min": -41.03303146362305,
      "activations/layer19_attention_weight_max": 25.203598022460938,
      "activations/layer19_attention_weight_min": -22.414276123046875,
      "activations/layer1_attention_weight_max": 19.016014099121094,
      "activations/layer1_attention_weight_min": -18.318103790283203,
      "activations/layer20_attention_weight_max": 22.70978355407715,
      "activations/layer20_attention_weight_min": -24.554292678833008,
      "activations/layer21_attention_weight_max": 39.50213623046875,
      "activations/layer21_attention_weight_min": -25.996843338012695,
      "activations/layer22_attention_weight_max": 30.80332374572754,
      "activations/layer22_attention_weight_min": -26.577295303344727,
      "activations/layer23_attention_weight_max": 38.92485809326172,
      "activations/layer23_attention_weight_min": -24.108564376831055,
      "activations/layer2_attention_weight_max": 30.608871459960938,
      "activations/layer2_attention_weight_min": -27.84804916381836,
      "activations/layer3_attention_weight_max": 88.99331665039062,
      "activations/layer3_attention_weight_min": -86.36507415771484,
      "activations/layer4_attention_weight_max": 97.86587524414062,
      "activations/layer4_attention_weight_min": -93.79476165771484,
      "activations/layer5_attention_weight_max": 71.95852661132812,
      "activations/layer5_attention_weight_min": -78.68901824951172,
      "activations/layer6_attention_weight_max": 50.10796356201172,
      "activations/layer6_attention_weight_min": -54.172054290771484,
      "activations/layer7_attention_weight_max": 69.04247283935547,
      "activations/layer7_attention_weight_min": -65.58390045166016,
      "activations/layer8_attention_weight_max": 45.5672492980957,
      "activations/layer8_attention_weight_min": -46.166847229003906,
      "activations/layer9_attention_weight_max": 46.17572784423828,
      "activations/layer9_attention_weight_min": -45.40163803100586,
      "epoch": 13.17,
      "learning_rate": 6.573560606060606e-05,
      "loss": 2.7706,
      "step": 226650
    },
    {
      "activations/layer0_attention_weight_max": 15.037232398986816,
      "activations/layer0_attention_weight_min": -14.212540626525879,
      "activations/layer10_attention_weight_max": 35.30859375,
      "activations/layer10_attention_weight_min": -37.89357376098633,
      "activations/layer11_attention_weight_max": 32.06586837768555,
      "activations/layer11_attention_weight_min": -33.20902633666992,
      "activations/layer12_attention_weight_max": 20.726303100585938,
      "activations/layer12_attention_weight_min": -23.575576782226562,
      "activations/layer13_attention_weight_max": 40.79658508300781,
      "activations/layer13_attention_weight_min": -29.300182342529297,
      "activations/layer14_attention_weight_max": 37.12202072143555,
      "activations/layer14_attention_weight_min": -29.729461669921875,
      "activations/layer15_attention_weight_max": 34.19065856933594,
      "activations/layer15_attention_weight_min": -27.484634399414062,
      "activations/layer16_attention_weight_max": 32.076866149902344,
      "activations/layer16_attention_weight_min": -29.231639862060547,
      "activations/layer17_attention_weight_max": 52.43913650512695,
      "activations/layer17_attention_weight_min": -44.231807708740234,
      "activations/layer18_attention_weight_max": 49.75666427612305,
      "activations/layer18_attention_weight_min": -35.643272399902344,
      "activations/layer19_attention_weight_max": 23.560455322265625,
      "activations/layer19_attention_weight_min": -21.06171417236328,
      "activations/layer1_attention_weight_max": 16.21503448486328,
      "activations/layer1_attention_weight_min": -15.814290046691895,
      "activations/layer20_attention_weight_max": 25.912353515625,
      "activations/layer20_attention_weight_min": -20.61465835571289,
      "activations/layer21_attention_weight_max": 41.45794677734375,
      "activations/layer21_attention_weight_min": -22.17892074584961,
      "activations/layer22_attention_weight_max": 34.37095642089844,
      "activations/layer22_attention_weight_min": -25.273014068603516,
      "activations/layer23_attention_weight_max": 42.62717819213867,
      "activations/layer23_attention_weight_min": -25.159751892089844,
      "activations/layer2_attention_weight_max": 33.45399856567383,
      "activations/layer2_attention_weight_min": -29.533790588378906,
      "activations/layer3_attention_weight_max": 93.27033996582031,
      "activations/layer3_attention_weight_min": -89.22311401367188,
      "activations/layer4_attention_weight_max": 95.09001922607422,
      "activations/layer4_attention_weight_min": -90.9217758178711,
      "activations/layer5_attention_weight_max": 74.9732666015625,
      "activations/layer5_attention_weight_min": -77.53282165527344,
      "activations/layer6_attention_weight_max": 51.28919982910156,
      "activations/layer6_attention_weight_min": -53.0644416809082,
      "activations/layer7_attention_weight_max": 71.25354766845703,
      "activations/layer7_attention_weight_min": -69.7544174194336,
      "activations/layer8_attention_weight_max": 44.020240783691406,
      "activations/layer8_attention_weight_min": -47.85114288330078,
      "activations/layer9_attention_weight_max": 45.0883903503418,
      "activations/layer9_attention_weight_min": -45.42117691040039,
      "epoch": 13.17,
      "learning_rate": 6.571666666666666e-05,
      "loss": 2.7618,
      "step": 226700
    },
    {
      "activations/layer0_attention_weight_max": 16.32281494140625,
      "activations/layer0_attention_weight_min": -14.411767959594727,
      "activations/layer10_attention_weight_max": 32.72895812988281,
      "activations/layer10_attention_weight_min": -33.208797454833984,
      "activations/layer11_attention_weight_max": 29.914127349853516,
      "activations/layer11_attention_weight_min": -31.016605377197266,
      "activations/layer12_attention_weight_max": 17.69234848022461,
      "activations/layer12_attention_weight_min": -32.54021453857422,
      "activations/layer13_attention_weight_max": 35.481990814208984,
      "activations/layer13_attention_weight_min": -25.83903694152832,
      "activations/layer14_attention_weight_max": 33.9845085144043,
      "activations/layer14_attention_weight_min": -30.198394775390625,
      "activations/layer15_attention_weight_max": 33.31995391845703,
      "activations/layer15_attention_weight_min": -28.059724807739258,
      "activations/layer16_attention_weight_max": 30.605257034301758,
      "activations/layer16_attention_weight_min": -29.55412483215332,
      "activations/layer17_attention_weight_max": 47.64079666137695,
      "activations/layer17_attention_weight_min": -43.1751594543457,
      "activations/layer18_attention_weight_max": 43.487022399902344,
      "activations/layer18_attention_weight_min": -35.816097259521484,
      "activations/layer19_attention_weight_max": 20.61223793029785,
      "activations/layer19_attention_weight_min": -21.366655349731445,
      "activations/layer1_attention_weight_max": 17.25618553161621,
      "activations/layer1_attention_weight_min": -15.263567924499512,
      "activations/layer20_attention_weight_max": 19.400968551635742,
      "activations/layer20_attention_weight_min": -19.853151321411133,
      "activations/layer21_attention_weight_max": 32.306644439697266,
      "activations/layer21_attention_weight_min": -22.320758819580078,
      "activations/layer22_attention_weight_max": 29.637252807617188,
      "activations/layer22_attention_weight_min": -26.37407112121582,
      "activations/layer23_attention_weight_max": 32.33864212036133,
      "activations/layer23_attention_weight_min": -22.724409103393555,
      "activations/layer2_attention_weight_max": 28.824127197265625,
      "activations/layer2_attention_weight_min": -28.002120971679688,
      "activations/layer3_attention_weight_max": 86.53916931152344,
      "activations/layer3_attention_weight_min": -91.94908142089844,
      "activations/layer4_attention_weight_max": 90.62700653076172,
      "activations/layer4_attention_weight_min": -88.58849334716797,
      "activations/layer5_attention_weight_max": 67.98686218261719,
      "activations/layer5_attention_weight_min": -81.54940795898438,
      "activations/layer6_attention_weight_max": 48.46326446533203,
      "activations/layer6_attention_weight_min": -51.92734909057617,
      "activations/layer7_attention_weight_max": 65.32445526123047,
      "activations/layer7_attention_weight_min": -63.10512924194336,
      "activations/layer8_attention_weight_max": 42.53826904296875,
      "activations/layer8_attention_weight_min": -47.65674591064453,
      "activations/layer9_attention_weight_max": 45.1517219543457,
      "activations/layer9_attention_weight_min": -44.615299224853516,
      "epoch": 13.18,
      "learning_rate": 6.569772727272726e-05,
      "loss": 2.751,
      "step": 226750
    },
    {
      "activations/layer0_attention_weight_max": 15.361932754516602,
      "activations/layer0_attention_weight_min": -14.599194526672363,
      "activations/layer10_attention_weight_max": 34.46644973754883,
      "activations/layer10_attention_weight_min": -34.31853103637695,
      "activations/layer11_attention_weight_max": 32.33576965332031,
      "activations/layer11_attention_weight_min": -32.26886749267578,
      "activations/layer12_attention_weight_max": 20.42011070251465,
      "activations/layer12_attention_weight_min": -26.302217483520508,
      "activations/layer13_attention_weight_max": 40.53864288330078,
      "activations/layer13_attention_weight_min": -30.898727416992188,
      "activations/layer14_attention_weight_max": 36.41242599487305,
      "activations/layer14_attention_weight_min": -30.02595329284668,
      "activations/layer15_attention_weight_max": 33.55382537841797,
      "activations/layer15_attention_weight_min": -32.04840850830078,
      "activations/layer16_attention_weight_max": 32.231163024902344,
      "activations/layer16_attention_weight_min": -32.77444076538086,
      "activations/layer17_attention_weight_max": 52.327484130859375,
      "activations/layer17_attention_weight_min": -45.425296783447266,
      "activations/layer18_attention_weight_max": 47.71151351928711,
      "activations/layer18_attention_weight_min": -39.88835144042969,
      "activations/layer19_attention_weight_max": 29.532316207885742,
      "activations/layer19_attention_weight_min": -23.44550132751465,
      "activations/layer1_attention_weight_max": 16.512174606323242,
      "activations/layer1_attention_weight_min": -14.186339378356934,
      "activations/layer20_attention_weight_max": 26.90874671936035,
      "activations/layer20_attention_weight_min": -24.52056121826172,
      "activations/layer21_attention_weight_max": 36.40839385986328,
      "activations/layer21_attention_weight_min": -23.941247940063477,
      "activations/layer22_attention_weight_max": 33.14391326904297,
      "activations/layer22_attention_weight_min": -26.223100662231445,
      "activations/layer23_attention_weight_max": 42.56281661987305,
      "activations/layer23_attention_weight_min": -25.91823387145996,
      "activations/layer2_attention_weight_max": 31.377506256103516,
      "activations/layer2_attention_weight_min": -28.774211883544922,
      "activations/layer3_attention_weight_max": 90.6778335571289,
      "activations/layer3_attention_weight_min": -87.52919006347656,
      "activations/layer4_attention_weight_max": 88.94380950927734,
      "activations/layer4_attention_weight_min": -85.71342468261719,
      "activations/layer5_attention_weight_max": 70.21470642089844,
      "activations/layer5_attention_weight_min": -71.62737274169922,
      "activations/layer6_attention_weight_max": 49.392337799072266,
      "activations/layer6_attention_weight_min": -49.829795837402344,
      "activations/layer7_attention_weight_max": 66.13111877441406,
      "activations/layer7_attention_weight_min": -64.9236831665039,
      "activations/layer8_attention_weight_max": 49.23726272583008,
      "activations/layer8_attention_weight_min": -45.056434631347656,
      "activations/layer9_attention_weight_max": 45.66604995727539,
      "activations/layer9_attention_weight_min": -46.38745880126953,
      "epoch": 13.18,
      "learning_rate": 6.567878787878786e-05,
      "loss": 2.7733,
      "step": 226800
    },
    {
      "activations/layer0_attention_weight_max": 16.218276977539062,
      "activations/layer0_attention_weight_min": -14.196464538574219,
      "activations/layer10_attention_weight_max": 38.934288024902344,
      "activations/layer10_attention_weight_min": -35.76942825317383,
      "activations/layer11_attention_weight_max": 33.35643005371094,
      "activations/layer11_attention_weight_min": -32.47624588012695,
      "activations/layer12_attention_weight_max": 16.769420623779297,
      "activations/layer12_attention_weight_min": -24.862628936767578,
      "activations/layer13_attention_weight_max": 33.607460021972656,
      "activations/layer13_attention_weight_min": -27.60443878173828,
      "activations/layer14_attention_weight_max": 34.807395935058594,
      "activations/layer14_attention_weight_min": -32.42769241333008,
      "activations/layer15_attention_weight_max": 31.872026443481445,
      "activations/layer15_attention_weight_min": -33.20254135131836,
      "activations/layer16_attention_weight_max": 30.34991455078125,
      "activations/layer16_attention_weight_min": -29.8291072845459,
      "activations/layer17_attention_weight_max": 47.5150260925293,
      "activations/layer17_attention_weight_min": -44.7468147277832,
      "activations/layer18_attention_weight_max": 40.85942077636719,
      "activations/layer18_attention_weight_min": -37.08308792114258,
      "activations/layer19_attention_weight_max": 20.871017456054688,
      "activations/layer19_attention_weight_min": -21.761505126953125,
      "activations/layer1_attention_weight_max": 16.67448616027832,
      "activations/layer1_attention_weight_min": -14.580826759338379,
      "activations/layer20_attention_weight_max": 21.686418533325195,
      "activations/layer20_attention_weight_min": -23.000911712646484,
      "activations/layer21_attention_weight_max": 38.12301254272461,
      "activations/layer21_attention_weight_min": -21.7718448638916,
      "activations/layer22_attention_weight_max": 30.963254928588867,
      "activations/layer22_attention_weight_min": -26.7812442779541,
      "activations/layer23_attention_weight_max": 35.487892150878906,
      "activations/layer23_attention_weight_min": -23.263526916503906,
      "activations/layer2_attention_weight_max": 31.496564865112305,
      "activations/layer2_attention_weight_min": -30.811485290527344,
      "activations/layer3_attention_weight_max": 92.63533782958984,
      "activations/layer3_attention_weight_min": -94.15058135986328,
      "activations/layer4_attention_weight_max": 97.66425323486328,
      "activations/layer4_attention_weight_min": -93.4400405883789,
      "activations/layer5_attention_weight_max": 71.44952392578125,
      "activations/layer5_attention_weight_min": -83.52800750732422,
      "activations/layer6_attention_weight_max": 56.60721206665039,
      "activations/layer6_attention_weight_min": -52.59391784667969,
      "activations/layer7_attention_weight_max": 71.65816497802734,
      "activations/layer7_attention_weight_min": -70.27250671386719,
      "activations/layer8_attention_weight_max": 45.64747619628906,
      "activations/layer8_attention_weight_min": -47.83163070678711,
      "activations/layer9_attention_weight_max": 45.35990524291992,
      "activations/layer9_attention_weight_min": -46.39621353149414,
      "epoch": 13.18,
      "learning_rate": 6.565984848484848e-05,
      "loss": 2.7557,
      "step": 226850
    },
    {
      "activations/layer0_attention_weight_max": 15.273784637451172,
      "activations/layer0_attention_weight_min": -14.701187133789062,
      "activations/layer10_attention_weight_max": 35.739749908447266,
      "activations/layer10_attention_weight_min": -37.952720642089844,
      "activations/layer11_attention_weight_max": 31.320240020751953,
      "activations/layer11_attention_weight_min": -34.60940170288086,
      "activations/layer12_attention_weight_max": 19.3958740234375,
      "activations/layer12_attention_weight_min": -27.99891471862793,
      "activations/layer13_attention_weight_max": 38.150352478027344,
      "activations/layer13_attention_weight_min": -27.803667068481445,
      "activations/layer14_attention_weight_max": 44.49446105957031,
      "activations/layer14_attention_weight_min": -28.797700881958008,
      "activations/layer15_attention_weight_max": 38.16344451904297,
      "activations/layer15_attention_weight_min": -27.085081100463867,
      "activations/layer16_attention_weight_max": 33.33271789550781,
      "activations/layer16_attention_weight_min": -32.062984466552734,
      "activations/layer17_attention_weight_max": 52.844505310058594,
      "activations/layer17_attention_weight_min": -43.67458724975586,
      "activations/layer18_attention_weight_max": 49.40751266479492,
      "activations/layer18_attention_weight_min": -39.37200927734375,
      "activations/layer19_attention_weight_max": 22.125680923461914,
      "activations/layer19_attention_weight_min": -21.98068618774414,
      "activations/layer1_attention_weight_max": 17.428619384765625,
      "activations/layer1_attention_weight_min": -17.05345916748047,
      "activations/layer20_attention_weight_max": 21.245603561401367,
      "activations/layer20_attention_weight_min": -21.173734664916992,
      "activations/layer21_attention_weight_max": 34.57381057739258,
      "activations/layer21_attention_weight_min": -26.772464752197266,
      "activations/layer22_attention_weight_max": 30.97418212890625,
      "activations/layer22_attention_weight_min": -29.05234718322754,
      "activations/layer23_attention_weight_max": 38.27757263183594,
      "activations/layer23_attention_weight_min": -26.238567352294922,
      "activations/layer2_attention_weight_max": 31.385448455810547,
      "activations/layer2_attention_weight_min": -30.02535629272461,
      "activations/layer3_attention_weight_max": 93.8304214477539,
      "activations/layer3_attention_weight_min": -93.91236114501953,
      "activations/layer4_attention_weight_max": 95.52642059326172,
      "activations/layer4_attention_weight_min": -89.04276275634766,
      "activations/layer5_attention_weight_max": 73.61407470703125,
      "activations/layer5_attention_weight_min": -78.61421203613281,
      "activations/layer6_attention_weight_max": 48.91975402832031,
      "activations/layer6_attention_weight_min": -51.51124572753906,
      "activations/layer7_attention_weight_max": 67.79368591308594,
      "activations/layer7_attention_weight_min": -65.65145111083984,
      "activations/layer8_attention_weight_max": 45.89695358276367,
      "activations/layer8_attention_weight_min": -49.35353469848633,
      "activations/layer9_attention_weight_max": 46.304683685302734,
      "activations/layer9_attention_weight_min": -46.685176849365234,
      "epoch": 13.18,
      "learning_rate": 6.564090909090908e-05,
      "loss": 2.7566,
      "step": 226900
    },
    {
      "activations/layer0_attention_weight_max": 16.02467155456543,
      "activations/layer0_attention_weight_min": -13.699694633483887,
      "activations/layer10_attention_weight_max": 39.16560745239258,
      "activations/layer10_attention_weight_min": -35.843135833740234,
      "activations/layer11_attention_weight_max": 33.394264221191406,
      "activations/layer11_attention_weight_min": -33.435691833496094,
      "activations/layer12_attention_weight_max": 18.59650230407715,
      "activations/layer12_attention_weight_min": -25.706499099731445,
      "activations/layer13_attention_weight_max": 35.0179557800293,
      "activations/layer13_attention_weight_min": -29.143630981445312,
      "activations/layer14_attention_weight_max": 37.003883361816406,
      "activations/layer14_attention_weight_min": -28.931270599365234,
      "activations/layer15_attention_weight_max": 36.075439453125,
      "activations/layer15_attention_weight_min": -29.18292236328125,
      "activations/layer16_attention_weight_max": 30.02178192138672,
      "activations/layer16_attention_weight_min": -27.846601486206055,
      "activations/layer17_attention_weight_max": 47.81700134277344,
      "activations/layer17_attention_weight_min": -42.61958694458008,
      "activations/layer18_attention_weight_max": 47.69047927856445,
      "activations/layer18_attention_weight_min": -36.399078369140625,
      "activations/layer19_attention_weight_max": 25.837207794189453,
      "activations/layer19_attention_weight_min": -20.941505432128906,
      "activations/layer1_attention_weight_max": 16.52617835998535,
      "activations/layer1_attention_weight_min": -16.503494262695312,
      "activations/layer20_attention_weight_max": 24.131834030151367,
      "activations/layer20_attention_weight_min": -20.90721893310547,
      "activations/layer21_attention_weight_max": 37.0713005065918,
      "activations/layer21_attention_weight_min": -21.831872940063477,
      "activations/layer22_attention_weight_max": 30.35349464416504,
      "activations/layer22_attention_weight_min": -24.944562911987305,
      "activations/layer23_attention_weight_max": 39.243003845214844,
      "activations/layer23_attention_weight_min": -24.314769744873047,
      "activations/layer2_attention_weight_max": 31.254230499267578,
      "activations/layer2_attention_weight_min": -29.695919036865234,
      "activations/layer3_attention_weight_max": 94.28656005859375,
      "activations/layer3_attention_weight_min": -89.3386459350586,
      "activations/layer4_attention_weight_max": 98.23543548583984,
      "activations/layer4_attention_weight_min": -98.495361328125,
      "activations/layer5_attention_weight_max": 70.76304626464844,
      "activations/layer5_attention_weight_min": -84.9483871459961,
      "activations/layer6_attention_weight_max": 51.60485076904297,
      "activations/layer6_attention_weight_min": -53.220787048339844,
      "activations/layer7_attention_weight_max": 67.06778717041016,
      "activations/layer7_attention_weight_min": -69.70680236816406,
      "activations/layer8_attention_weight_max": 47.790855407714844,
      "activations/layer8_attention_weight_min": -51.14598846435547,
      "activations/layer9_attention_weight_max": 47.35105514526367,
      "activations/layer9_attention_weight_min": -48.66134262084961,
      "epoch": 13.19,
      "learning_rate": 6.56219696969697e-05,
      "loss": 2.7584,
      "step": 226950
    },
    {
      "activations/layer0_attention_weight_max": 16.59750747680664,
      "activations/layer0_attention_weight_min": -13.35462760925293,
      "activations/layer10_attention_weight_max": 40.78098678588867,
      "activations/layer10_attention_weight_min": -40.82780838012695,
      "activations/layer11_attention_weight_max": 34.1767578125,
      "activations/layer11_attention_weight_min": -34.68762969970703,
      "activations/layer12_attention_weight_max": 20.286569595336914,
      "activations/layer12_attention_weight_min": -25.505550384521484,
      "activations/layer13_attention_weight_max": 37.58412551879883,
      "activations/layer13_attention_weight_min": -29.241167068481445,
      "activations/layer14_attention_weight_max": 36.494285583496094,
      "activations/layer14_attention_weight_min": -31.060270309448242,
      "activations/layer15_attention_weight_max": 34.747074127197266,
      "activations/layer15_attention_weight_min": -32.407840728759766,
      "activations/layer16_attention_weight_max": 33.09181594848633,
      "activations/layer16_attention_weight_min": -30.91536521911621,
      "activations/layer17_attention_weight_max": 52.16679382324219,
      "activations/layer17_attention_weight_min": -46.93138885498047,
      "activations/layer18_attention_weight_max": 50.5432243347168,
      "activations/layer18_attention_weight_min": -39.03788757324219,
      "activations/layer19_attention_weight_max": 22.806655883789062,
      "activations/layer19_attention_weight_min": -23.548540115356445,
      "activations/layer1_attention_weight_max": 18.608631134033203,
      "activations/layer1_attention_weight_min": -16.949785232543945,
      "activations/layer20_attention_weight_max": 21.83062171936035,
      "activations/layer20_attention_weight_min": -22.154447555541992,
      "activations/layer21_attention_weight_max": 38.488895416259766,
      "activations/layer21_attention_weight_min": -24.673065185546875,
      "activations/layer22_attention_weight_max": 30.164806365966797,
      "activations/layer22_attention_weight_min": -25.58917236328125,
      "activations/layer23_attention_weight_max": 36.037803649902344,
      "activations/layer23_attention_weight_min": -25.062942504882812,
      "activations/layer2_attention_weight_max": 31.20740509033203,
      "activations/layer2_attention_weight_min": -32.20263671875,
      "activations/layer3_attention_weight_max": 90.90166473388672,
      "activations/layer3_attention_weight_min": -91.5634765625,
      "activations/layer4_attention_weight_max": 94.41808319091797,
      "activations/layer4_attention_weight_min": -93.88571166992188,
      "activations/layer5_attention_weight_max": 68.76112365722656,
      "activations/layer5_attention_weight_min": -76.97799682617188,
      "activations/layer6_attention_weight_max": 51.488380432128906,
      "activations/layer6_attention_weight_min": -53.51862716674805,
      "activations/layer7_attention_weight_max": 68.98320770263672,
      "activations/layer7_attention_weight_min": -69.22482299804688,
      "activations/layer8_attention_weight_max": 49.33693313598633,
      "activations/layer8_attention_weight_min": -49.81003952026367,
      "activations/layer9_attention_weight_max": 49.803279876708984,
      "activations/layer9_attention_weight_min": -52.19134521484375,
      "epoch": 13.19,
      "learning_rate": 6.56030303030303e-05,
      "loss": 2.7686,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4649,
      "eval_samples_per_second": 507.269,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4649,
      "eval_openwebtext_samples_per_second": 507.269,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.9676,
      "eval_wikitext_samples_per_second": 231.749,
      "step": 227000
    },
    {
      "epoch": 13.19,
      "eval_lambada_loss": 2.603515625,
      "eval_lambada_ppl": 13.511154789957821,
      "eval_lambada_runtime": 9.5741,
      "eval_lambada_samples_per_second": 508.559,
      "step": 227000
    },
    {
      "activations/layer0_attention_weight_max": 15.974335670471191,
      "activations/layer0_attention_weight_min": -13.518320083618164,
      "activations/layer10_attention_weight_max": 35.50108337402344,
      "activations/layer10_attention_weight_min": -35.66461181640625,
      "activations/layer11_attention_weight_max": 32.510162353515625,
      "activations/layer11_attention_weight_min": -33.606910705566406,
      "activations/layer12_attention_weight_max": 17.118202209472656,
      "activations/layer12_attention_weight_min": -23.62946128845215,
      "activations/layer13_attention_weight_max": 36.74116134643555,
      "activations/layer13_attention_weight_min": -28.25478172302246,
      "activations/layer14_attention_weight_max": 34.66884231567383,
      "activations/layer14_attention_weight_min": -29.481496810913086,
      "activations/layer15_attention_weight_max": 32.09545135498047,
      "activations/layer15_attention_weight_min": -29.912107467651367,
      "activations/layer16_attention_weight_max": 30.41827392578125,
      "activations/layer16_attention_weight_min": -29.738473892211914,
      "activations/layer17_attention_weight_max": 51.97652053833008,
      "activations/layer17_attention_weight_min": -45.973331451416016,
      "activations/layer18_attention_weight_max": 44.79603958129883,
      "activations/layer18_attention_weight_min": -40.41208267211914,
      "activations/layer19_attention_weight_max": 21.36703109741211,
      "activations/layer19_attention_weight_min": -22.54194450378418,
      "activations/layer1_attention_weight_max": 18.80381202697754,
      "activations/layer1_attention_weight_min": -16.820186614990234,
      "activations/layer20_attention_weight_max": 20.391735076904297,
      "activations/layer20_attention_weight_min": -23.4945068359375,
      "activations/layer21_attention_weight_max": 31.800247192382812,
      "activations/layer21_attention_weight_min": -21.603046417236328,
      "activations/layer22_attention_weight_max": 24.938434600830078,
      "activations/layer22_attention_weight_min": -24.558231353759766,
      "activations/layer23_attention_weight_max": 33.62533950805664,
      "activations/layer23_attention_weight_min": -24.475383758544922,
      "activations/layer2_attention_weight_max": 30.88088035583496,
      "activations/layer2_attention_weight_min": -29.464370727539062,
      "activations/layer3_attention_weight_max": 92.00164794921875,
      "activations/layer3_attention_weight_min": -90.64920043945312,
      "activations/layer4_attention_weight_max": 95.2359619140625,
      "activations/layer4_attention_weight_min": -96.43364715576172,
      "activations/layer5_attention_weight_max": 68.26326751708984,
      "activations/layer5_attention_weight_min": -82.61111450195312,
      "activations/layer6_attention_weight_max": 52.68043899536133,
      "activations/layer6_attention_weight_min": -54.073612213134766,
      "activations/layer7_attention_weight_max": 67.4415283203125,
      "activations/layer7_attention_weight_min": -71.19136810302734,
      "activations/layer8_attention_weight_max": 45.744163513183594,
      "activations/layer8_attention_weight_min": -48.63332748413086,
      "activations/layer9_attention_weight_max": 46.096309661865234,
      "activations/layer9_attention_weight_min": -50.49346160888672,
      "epoch": 13.19,
      "learning_rate": 6.558409090909091e-05,
      "loss": 2.7641,
      "step": 227050
    },
    {
      "activations/layer0_attention_weight_max": 15.90005111694336,
      "activations/layer0_attention_weight_min": -13.457466125488281,
      "activations/layer10_attention_weight_max": 35.399234771728516,
      "activations/layer10_attention_weight_min": -37.28871154785156,
      "activations/layer11_attention_weight_max": 35.60224914550781,
      "activations/layer11_attention_weight_min": -33.8797607421875,
      "activations/layer12_attention_weight_max": 18.559995651245117,
      "activations/layer12_attention_weight_min": -30.860313415527344,
      "activations/layer13_attention_weight_max": 32.321044921875,
      "activations/layer13_attention_weight_min": -28.498491287231445,
      "activations/layer14_attention_weight_max": 40.06502914428711,
      "activations/layer14_attention_weight_min": -31.02497673034668,
      "activations/layer15_attention_weight_max": 38.691158294677734,
      "activations/layer15_attention_weight_min": -30.74227523803711,
      "activations/layer16_attention_weight_max": 33.04301452636719,
      "activations/layer16_attention_weight_min": -30.391300201416016,
      "activations/layer17_attention_weight_max": 52.190956115722656,
      "activations/layer17_attention_weight_min": -44.79993438720703,
      "activations/layer18_attention_weight_max": 46.030433654785156,
      "activations/layer18_attention_weight_min": -40.10926818847656,
      "activations/layer19_attention_weight_max": 21.727354049682617,
      "activations/layer19_attention_weight_min": -22.313146591186523,
      "activations/layer1_attention_weight_max": 17.36290740966797,
      "activations/layer1_attention_weight_min": -14.940671920776367,
      "activations/layer20_attention_weight_max": 21.71611213684082,
      "activations/layer20_attention_weight_min": -21.11781883239746,
      "activations/layer21_attention_weight_max": 46.60127639770508,
      "activations/layer21_attention_weight_min": -21.719724655151367,
      "activations/layer22_attention_weight_max": 29.863862991333008,
      "activations/layer22_attention_weight_min": -27.1246395111084,
      "activations/layer23_attention_weight_max": 33.97730255126953,
      "activations/layer23_attention_weight_min": -23.626251220703125,
      "activations/layer2_attention_weight_max": 34.264564514160156,
      "activations/layer2_attention_weight_min": -29.871902465820312,
      "activations/layer3_attention_weight_max": 101.23578643798828,
      "activations/layer3_attention_weight_min": -94.98182678222656,
      "activations/layer4_attention_weight_max": 97.98756408691406,
      "activations/layer4_attention_weight_min": -90.2421646118164,
      "activations/layer5_attention_weight_max": 70.01126098632812,
      "activations/layer5_attention_weight_min": -75.27617645263672,
      "activations/layer6_attention_weight_max": 48.62358856201172,
      "activations/layer6_attention_weight_min": -50.070655822753906,
      "activations/layer7_attention_weight_max": 67.8948745727539,
      "activations/layer7_attention_weight_min": -64.63848114013672,
      "activations/layer8_attention_weight_max": 45.982295989990234,
      "activations/layer8_attention_weight_min": -47.89807891845703,
      "activations/layer9_attention_weight_max": 56.09375,
      "activations/layer9_attention_weight_min": -49.39652633666992,
      "epoch": 13.2,
      "learning_rate": 6.556515151515151e-05,
      "loss": 2.7698,
      "step": 227100
    },
    {
      "activations/layer0_attention_weight_max": 15.753739356994629,
      "activations/layer0_attention_weight_min": -13.454564094543457,
      "activations/layer10_attention_weight_max": 35.41715621948242,
      "activations/layer10_attention_weight_min": -34.856082916259766,
      "activations/layer11_attention_weight_max": 32.41124725341797,
      "activations/layer11_attention_weight_min": -32.70911407470703,
      "activations/layer12_attention_weight_max": 20.597999572753906,
      "activations/layer12_attention_weight_min": -25.900474548339844,
      "activations/layer13_attention_weight_max": 38.1385612487793,
      "activations/layer13_attention_weight_min": -28.16326904296875,
      "activations/layer14_attention_weight_max": 38.4294319152832,
      "activations/layer14_attention_weight_min": -30.77141571044922,
      "activations/layer15_attention_weight_max": 35.325843811035156,
      "activations/layer15_attention_weight_min": -29.66741180419922,
      "activations/layer16_attention_weight_max": 31.002944946289062,
      "activations/layer16_attention_weight_min": -30.191455841064453,
      "activations/layer17_attention_weight_max": 52.96175003051758,
      "activations/layer17_attention_weight_min": -44.7314338684082,
      "activations/layer18_attention_weight_max": 48.42650604248047,
      "activations/layer18_attention_weight_min": -38.16130828857422,
      "activations/layer19_attention_weight_max": 25.688552856445312,
      "activations/layer19_attention_weight_min": -22.66441535949707,
      "activations/layer1_attention_weight_max": 16.664886474609375,
      "activations/layer1_attention_weight_min": -15.494418144226074,
      "activations/layer20_attention_weight_max": 26.766361236572266,
      "activations/layer20_attention_weight_min": -21.239816665649414,
      "activations/layer21_attention_weight_max": 45.819942474365234,
      "activations/layer21_attention_weight_min": -23.139537811279297,
      "activations/layer22_attention_weight_max": 30.08934211730957,
      "activations/layer22_attention_weight_min": -25.95348358154297,
      "activations/layer23_attention_weight_max": 36.516212463378906,
      "activations/layer23_attention_weight_min": -22.785537719726562,
      "activations/layer2_attention_weight_max": 30.902339935302734,
      "activations/layer2_attention_weight_min": -29.066694259643555,
      "activations/layer3_attention_weight_max": 94.36906433105469,
      "activations/layer3_attention_weight_min": -98.04484558105469,
      "activations/layer4_attention_weight_max": 105.2171630859375,
      "activations/layer4_attention_weight_min": -99.9589614868164,
      "activations/layer5_attention_weight_max": 73.32002258300781,
      "activations/layer5_attention_weight_min": -80.5903091430664,
      "activations/layer6_attention_weight_max": 54.577415466308594,
      "activations/layer6_attention_weight_min": -52.06199645996094,
      "activations/layer7_attention_weight_max": 69.93495178222656,
      "activations/layer7_attention_weight_min": -67.00603485107422,
      "activations/layer8_attention_weight_max": 46.23036575317383,
      "activations/layer8_attention_weight_min": -46.60283279418945,
      "activations/layer9_attention_weight_max": 44.63615036010742,
      "activations/layer9_attention_weight_min": -45.2168083190918,
      "epoch": 13.2,
      "learning_rate": 6.554621212121211e-05,
      "loss": 2.7597,
      "step": 227150
    },
    {
      "activations/layer0_attention_weight_max": 16.07564926147461,
      "activations/layer0_attention_weight_min": -13.570895195007324,
      "activations/layer10_attention_weight_max": 42.0662956237793,
      "activations/layer10_attention_weight_min": -39.502479553222656,
      "activations/layer11_attention_weight_max": 36.92890930175781,
      "activations/layer11_attention_weight_min": -34.584205627441406,
      "activations/layer12_attention_weight_max": 19.122705459594727,
      "activations/layer12_attention_weight_min": -28.004501342773438,
      "activations/layer13_attention_weight_max": 42.913211822509766,
      "activations/layer13_attention_weight_min": -32.55990219116211,
      "activations/layer14_attention_weight_max": 56.746212005615234,
      "activations/layer14_attention_weight_min": -31.334300994873047,
      "activations/layer15_attention_weight_max": 43.9625358581543,
      "activations/layer15_attention_weight_min": -34.97881317138672,
      "activations/layer16_attention_weight_max": 36.31717300415039,
      "activations/layer16_attention_weight_min": -32.07809829711914,
      "activations/layer17_attention_weight_max": 60.41080093383789,
      "activations/layer17_attention_weight_min": -50.31302261352539,
      "activations/layer18_attention_weight_max": 55.02341842651367,
      "activations/layer18_attention_weight_min": -44.16572570800781,
      "activations/layer19_attention_weight_max": 26.1979923248291,
      "activations/layer19_attention_weight_min": -24.41133689880371,
      "activations/layer1_attention_weight_max": 16.607675552368164,
      "activations/layer1_attention_weight_min": -16.19293785095215,
      "activations/layer20_attention_weight_max": 30.515872955322266,
      "activations/layer20_attention_weight_min": -25.210472106933594,
      "activations/layer21_attention_weight_max": 48.36704635620117,
      "activations/layer21_attention_weight_min": -25.504873275756836,
      "activations/layer22_attention_weight_max": 33.14239501953125,
      "activations/layer22_attention_weight_min": -26.425758361816406,
      "activations/layer23_attention_weight_max": 36.97187805175781,
      "activations/layer23_attention_weight_min": -24.568098068237305,
      "activations/layer2_attention_weight_max": 31.74500846862793,
      "activations/layer2_attention_weight_min": -31.223419189453125,
      "activations/layer3_attention_weight_max": 98.90032958984375,
      "activations/layer3_attention_weight_min": -101.47193908691406,
      "activations/layer4_attention_weight_max": 100.03345489501953,
      "activations/layer4_attention_weight_min": -98.01134490966797,
      "activations/layer5_attention_weight_max": 71.92411804199219,
      "activations/layer5_attention_weight_min": -77.40827178955078,
      "activations/layer6_attention_weight_max": 52.807334899902344,
      "activations/layer6_attention_weight_min": -51.663570404052734,
      "activations/layer7_attention_weight_max": 75.1202392578125,
      "activations/layer7_attention_weight_min": -75.3433609008789,
      "activations/layer8_attention_weight_max": 49.10942459106445,
      "activations/layer8_attention_weight_min": -48.76298522949219,
      "activations/layer9_attention_weight_max": 58.54880905151367,
      "activations/layer9_attention_weight_min": -48.28481674194336,
      "epoch": 13.2,
      "learning_rate": 6.552727272727273e-05,
      "loss": 2.7647,
      "step": 227200
    },
    {
      "activations/layer0_attention_weight_max": 16.1505126953125,
      "activations/layer0_attention_weight_min": -13.93768310546875,
      "activations/layer10_attention_weight_max": 35.604583740234375,
      "activations/layer10_attention_weight_min": -37.4778938293457,
      "activations/layer11_attention_weight_max": 32.86603546142578,
      "activations/layer11_attention_weight_min": -34.36530303955078,
      "activations/layer12_attention_weight_max": 19.782699584960938,
      "activations/layer12_attention_weight_min": -24.266616821289062,
      "activations/layer13_attention_weight_max": 35.16701889038086,
      "activations/layer13_attention_weight_min": -28.503345489501953,
      "activations/layer14_attention_weight_max": 35.2041015625,
      "activations/layer14_attention_weight_min": -28.713970184326172,
      "activations/layer15_attention_weight_max": 31.194475173950195,
      "activations/layer15_attention_weight_min": -28.97618293762207,
      "activations/layer16_attention_weight_max": 31.13448143005371,
      "activations/layer16_attention_weight_min": -28.804044723510742,
      "activations/layer17_attention_weight_max": 50.20744705200195,
      "activations/layer17_attention_weight_min": -45.046714782714844,
      "activations/layer18_attention_weight_max": 44.33141326904297,
      "activations/layer18_attention_weight_min": -38.700321197509766,
      "activations/layer19_attention_weight_max": 22.105749130249023,
      "activations/layer19_attention_weight_min": -22.921276092529297,
      "activations/layer1_attention_weight_max": 16.50458335876465,
      "activations/layer1_attention_weight_min": -12.783204078674316,
      "activations/layer20_attention_weight_max": 23.649110794067383,
      "activations/layer20_attention_weight_min": -23.00370979309082,
      "activations/layer21_attention_weight_max": 35.50153732299805,
      "activations/layer21_attention_weight_min": -22.69550132751465,
      "activations/layer22_attention_weight_max": 30.217988967895508,
      "activations/layer22_attention_weight_min": -27.499494552612305,
      "activations/layer23_attention_weight_max": 35.74694061279297,
      "activations/layer23_attention_weight_min": -24.67079734802246,
      "activations/layer2_attention_weight_max": 28.314687728881836,
      "activations/layer2_attention_weight_min": -28.69686508178711,
      "activations/layer3_attention_weight_max": 87.2098388671875,
      "activations/layer3_attention_weight_min": -93.48072052001953,
      "activations/layer4_attention_weight_max": 90.98331451416016,
      "activations/layer4_attention_weight_min": -90.3105697631836,
      "activations/layer5_attention_weight_max": 67.59809112548828,
      "activations/layer5_attention_weight_min": -79.80117797851562,
      "activations/layer6_attention_weight_max": 51.05051040649414,
      "activations/layer6_attention_weight_min": -51.607601165771484,
      "activations/layer7_attention_weight_max": 65.87687683105469,
      "activations/layer7_attention_weight_min": -67.48699951171875,
      "activations/layer8_attention_weight_max": 45.024532318115234,
      "activations/layer8_attention_weight_min": -48.365516662597656,
      "activations/layer9_attention_weight_max": 45.894527435302734,
      "activations/layer9_attention_weight_min": -47.24542236328125,
      "epoch": 13.2,
      "learning_rate": 6.550833333333333e-05,
      "loss": 2.7606,
      "step": 227250
    },
    {
      "activations/layer0_attention_weight_max": 15.083813667297363,
      "activations/layer0_attention_weight_min": -13.412031173706055,
      "activations/layer10_attention_weight_max": 37.526832580566406,
      "activations/layer10_attention_weight_min": -37.90769577026367,
      "activations/layer11_attention_weight_max": 33.78715515136719,
      "activations/layer11_attention_weight_min": -35.939537048339844,
      "activations/layer12_attention_weight_max": 18.03854751586914,
      "activations/layer12_attention_weight_min": -23.11414337158203,
      "activations/layer13_attention_weight_max": 34.794254302978516,
      "activations/layer13_attention_weight_min": -26.81516456604004,
      "activations/layer14_attention_weight_max": 34.56221008300781,
      "activations/layer14_attention_weight_min": -31.552160263061523,
      "activations/layer15_attention_weight_max": 32.89988708496094,
      "activations/layer15_attention_weight_min": -30.055076599121094,
      "activations/layer16_attention_weight_max": 30.708940505981445,
      "activations/layer16_attention_weight_min": -30.41387176513672,
      "activations/layer17_attention_weight_max": 50.09469223022461,
      "activations/layer17_attention_weight_min": -45.92425537109375,
      "activations/layer18_attention_weight_max": 41.98967742919922,
      "activations/layer18_attention_weight_min": -38.54570007324219,
      "activations/layer19_attention_weight_max": 21.204151153564453,
      "activations/layer19_attention_weight_min": -23.806991577148438,
      "activations/layer1_attention_weight_max": 17.401464462280273,
      "activations/layer1_attention_weight_min": -15.194130897521973,
      "activations/layer20_attention_weight_max": 21.902009963989258,
      "activations/layer20_attention_weight_min": -22.155424118041992,
      "activations/layer21_attention_weight_max": 34.52773666381836,
      "activations/layer21_attention_weight_min": -26.589500427246094,
      "activations/layer22_attention_weight_max": 30.125423431396484,
      "activations/layer22_attention_weight_min": -28.84273910522461,
      "activations/layer23_attention_weight_max": 37.77040100097656,
      "activations/layer23_attention_weight_min": -26.209157943725586,
      "activations/layer2_attention_weight_max": 34.258819580078125,
      "activations/layer2_attention_weight_min": -33.28900146484375,
      "activations/layer3_attention_weight_max": 96.9818344116211,
      "activations/layer3_attention_weight_min": -98.59689331054688,
      "activations/layer4_attention_weight_max": 100.13436126708984,
      "activations/layer4_attention_weight_min": -94.01470184326172,
      "activations/layer5_attention_weight_max": 76.447265625,
      "activations/layer5_attention_weight_min": -79.03387451171875,
      "activations/layer6_attention_weight_max": 52.86689758300781,
      "activations/layer6_attention_weight_min": -52.83979797363281,
      "activations/layer7_attention_weight_max": 68.97293853759766,
      "activations/layer7_attention_weight_min": -72.9000244140625,
      "activations/layer8_attention_weight_max": 47.73062515258789,
      "activations/layer8_attention_weight_min": -48.898372650146484,
      "activations/layer9_attention_weight_max": 46.370845794677734,
      "activations/layer9_attention_weight_min": -48.41154098510742,
      "epoch": 13.21,
      "learning_rate": 6.548939393939393e-05,
      "loss": 2.7717,
      "step": 227300
    },
    {
      "activations/layer0_attention_weight_max": 15.914121627807617,
      "activations/layer0_attention_weight_min": -13.668146133422852,
      "activations/layer10_attention_weight_max": 35.31208038330078,
      "activations/layer10_attention_weight_min": -36.43473815917969,
      "activations/layer11_attention_weight_max": 32.026329040527344,
      "activations/layer11_attention_weight_min": -35.33198547363281,
      "activations/layer12_attention_weight_max": 18.260452270507812,
      "activations/layer12_attention_weight_min": -25.31201171875,
      "activations/layer13_attention_weight_max": 38.1865119934082,
      "activations/layer13_attention_weight_min": -28.016300201416016,
      "activations/layer14_attention_weight_max": 45.39633560180664,
      "activations/layer14_attention_weight_min": -29.90085792541504,
      "activations/layer15_attention_weight_max": 36.842140197753906,
      "activations/layer15_attention_weight_min": -30.641611099243164,
      "activations/layer16_attention_weight_max": 33.577293395996094,
      "activations/layer16_attention_weight_min": -31.629165649414062,
      "activations/layer17_attention_weight_max": 52.03596878051758,
      "activations/layer17_attention_weight_min": -48.26449966430664,
      "activations/layer18_attention_weight_max": 47.06221389770508,
      "activations/layer18_attention_weight_min": -39.39927673339844,
      "activations/layer19_attention_weight_max": 22.781869888305664,
      "activations/layer19_attention_weight_min": -22.196205139160156,
      "activations/layer1_attention_weight_max": 16.74884033203125,
      "activations/layer1_attention_weight_min": -15.023237228393555,
      "activations/layer20_attention_weight_max": 24.00299072265625,
      "activations/layer20_attention_weight_min": -22.82291603088379,
      "activations/layer21_attention_weight_max": 37.807411193847656,
      "activations/layer21_attention_weight_min": -20.324161529541016,
      "activations/layer22_attention_weight_max": 31.890216827392578,
      "activations/layer22_attention_weight_min": -28.011058807373047,
      "activations/layer23_attention_weight_max": 32.76517105102539,
      "activations/layer23_attention_weight_min": -24.354591369628906,
      "activations/layer2_attention_weight_max": 30.970905303955078,
      "activations/layer2_attention_weight_min": -28.709705352783203,
      "activations/layer3_attention_weight_max": 90.0182113647461,
      "activations/layer3_attention_weight_min": -92.56488037109375,
      "activations/layer4_attention_weight_max": 97.00796508789062,
      "activations/layer4_attention_weight_min": -93.95360565185547,
      "activations/layer5_attention_weight_max": 70.0728988647461,
      "activations/layer5_attention_weight_min": -78.32330322265625,
      "activations/layer6_attention_weight_max": 51.46742630004883,
      "activations/layer6_attention_weight_min": -51.883766174316406,
      "activations/layer7_attention_weight_max": 65.46208953857422,
      "activations/layer7_attention_weight_min": -66.97694396972656,
      "activations/layer8_attention_weight_max": 45.18165588378906,
      "activations/layer8_attention_weight_min": -46.365108489990234,
      "activations/layer9_attention_weight_max": 44.995811462402344,
      "activations/layer9_attention_weight_min": -47.45231246948242,
      "epoch": 13.21,
      "learning_rate": 6.547045454545453e-05,
      "loss": 2.7634,
      "step": 227350
    },
    {
      "activations/layer0_attention_weight_max": 15.134626388549805,
      "activations/layer0_attention_weight_min": -13.924527168273926,
      "activations/layer10_attention_weight_max": 35.582862854003906,
      "activations/layer10_attention_weight_min": -36.16201400756836,
      "activations/layer11_attention_weight_max": 32.74391555786133,
      "activations/layer11_attention_weight_min": -33.99422836303711,
      "activations/layer12_attention_weight_max": 18.58464241027832,
      "activations/layer12_attention_weight_min": -22.9182071685791,
      "activations/layer13_attention_weight_max": 33.62769317626953,
      "activations/layer13_attention_weight_min": -28.43250274658203,
      "activations/layer14_attention_weight_max": 35.56817626953125,
      "activations/layer14_attention_weight_min": -29.427724838256836,
      "activations/layer15_attention_weight_max": 31.54659080505371,
      "activations/layer15_attention_weight_min": -29.212745666503906,
      "activations/layer16_attention_weight_max": 32.70781326293945,
      "activations/layer16_attention_weight_min": -28.844982147216797,
      "activations/layer17_attention_weight_max": 52.315677642822266,
      "activations/layer17_attention_weight_min": -44.24602127075195,
      "activations/layer18_attention_weight_max": 46.46452713012695,
      "activations/layer18_attention_weight_min": -37.9630012512207,
      "activations/layer19_attention_weight_max": 22.270130157470703,
      "activations/layer19_attention_weight_min": -21.917675018310547,
      "activations/layer1_attention_weight_max": 18.081472396850586,
      "activations/layer1_attention_weight_min": -13.931458473205566,
      "activations/layer20_attention_weight_max": 21.311031341552734,
      "activations/layer20_attention_weight_min": -21.71630096435547,
      "activations/layer21_attention_weight_max": 41.131011962890625,
      "activations/layer21_attention_weight_min": -25.474639892578125,
      "activations/layer22_attention_weight_max": 33.34609603881836,
      "activations/layer22_attention_weight_min": -27.048662185668945,
      "activations/layer23_attention_weight_max": 38.0944709777832,
      "activations/layer23_attention_weight_min": -24.52056884765625,
      "activations/layer2_attention_weight_max": 30.355533599853516,
      "activations/layer2_attention_weight_min": -28.176204681396484,
      "activations/layer3_attention_weight_max": 93.96392059326172,
      "activations/layer3_attention_weight_min": -90.4742660522461,
      "activations/layer4_attention_weight_max": 95.67179870605469,
      "activations/layer4_attention_weight_min": -92.33668518066406,
      "activations/layer5_attention_weight_max": 70.9130630493164,
      "activations/layer5_attention_weight_min": -76.65048217773438,
      "activations/layer6_attention_weight_max": 50.66347122192383,
      "activations/layer6_attention_weight_min": -50.96988296508789,
      "activations/layer7_attention_weight_max": 66.20389556884766,
      "activations/layer7_attention_weight_min": -63.62765884399414,
      "activations/layer8_attention_weight_max": 44.29354476928711,
      "activations/layer8_attention_weight_min": -47.924102783203125,
      "activations/layer9_attention_weight_max": 44.26761245727539,
      "activations/layer9_attention_weight_min": -45.20270538330078,
      "epoch": 13.21,
      "learning_rate": 6.545151515151515e-05,
      "loss": 2.7621,
      "step": 227400
    },
    {
      "activations/layer0_attention_weight_max": 15.44339370727539,
      "activations/layer0_attention_weight_min": -13.70127010345459,
      "activations/layer10_attention_weight_max": 37.5877685546875,
      "activations/layer10_attention_weight_min": -37.92302322387695,
      "activations/layer11_attention_weight_max": 34.615745544433594,
      "activations/layer11_attention_weight_min": -33.757816314697266,
      "activations/layer12_attention_weight_max": 18.920562744140625,
      "activations/layer12_attention_weight_min": -27.83091926574707,
      "activations/layer13_attention_weight_max": 38.4305419921875,
      "activations/layer13_attention_weight_min": -26.987621307373047,
      "activations/layer14_attention_weight_max": 34.71070098876953,
      "activations/layer14_attention_weight_min": -30.337377548217773,
      "activations/layer15_attention_weight_max": 31.145606994628906,
      "activations/layer15_attention_weight_min": -29.555559158325195,
      "activations/layer16_attention_weight_max": 32.851341247558594,
      "activations/layer16_attention_weight_min": -30.5382080078125,
      "activations/layer17_attention_weight_max": 54.92605209350586,
      "activations/layer17_attention_weight_min": -46.099178314208984,
      "activations/layer18_attention_weight_max": 49.26634979248047,
      "activations/layer18_attention_weight_min": -39.770538330078125,
      "activations/layer19_attention_weight_max": 22.510358810424805,
      "activations/layer19_attention_weight_min": -24.069082260131836,
      "activations/layer1_attention_weight_max": 16.484556198120117,
      "activations/layer1_attention_weight_min": -15.895767211914062,
      "activations/layer20_attention_weight_max": 23.95558738708496,
      "activations/layer20_attention_weight_min": -24.71757698059082,
      "activations/layer21_attention_weight_max": 40.38727569580078,
      "activations/layer21_attention_weight_min": -21.848613739013672,
      "activations/layer22_attention_weight_max": 32.80095672607422,
      "activations/layer22_attention_weight_min": -27.08856773376465,
      "activations/layer23_attention_weight_max": 40.90169906616211,
      "activations/layer23_attention_weight_min": -23.194082260131836,
      "activations/layer2_attention_weight_max": 32.91549301147461,
      "activations/layer2_attention_weight_min": -31.72307586669922,
      "activations/layer3_attention_weight_max": 95.75035858154297,
      "activations/layer3_attention_weight_min": -92.39567565917969,
      "activations/layer4_attention_weight_max": 97.15142059326172,
      "activations/layer4_attention_weight_min": -94.47417449951172,
      "activations/layer5_attention_weight_max": 69.42449951171875,
      "activations/layer5_attention_weight_min": -78.0728759765625,
      "activations/layer6_attention_weight_max": 52.43689727783203,
      "activations/layer6_attention_weight_min": -53.02838897705078,
      "activations/layer7_attention_weight_max": 70.41814422607422,
      "activations/layer7_attention_weight_min": -68.44437408447266,
      "activations/layer8_attention_weight_max": 47.456844329833984,
      "activations/layer8_attention_weight_min": -49.9492073059082,
      "activations/layer9_attention_weight_max": 47.18612289428711,
      "activations/layer9_attention_weight_min": -48.42496109008789,
      "epoch": 13.22,
      "learning_rate": 6.543257575757575e-05,
      "loss": 2.7686,
      "step": 227450
    },
    {
      "activations/layer0_attention_weight_max": 16.5692195892334,
      "activations/layer0_attention_weight_min": -14.07036018371582,
      "activations/layer10_attention_weight_max": 36.52995300292969,
      "activations/layer10_attention_weight_min": -36.64884948730469,
      "activations/layer11_attention_weight_max": 33.44884490966797,
      "activations/layer11_attention_weight_min": -32.57470703125,
      "activations/layer12_attention_weight_max": 20.08115005493164,
      "activations/layer12_attention_weight_min": -23.27069854736328,
      "activations/layer13_attention_weight_max": 42.188865661621094,
      "activations/layer13_attention_weight_min": -28.251312255859375,
      "activations/layer14_attention_weight_max": 38.733917236328125,
      "activations/layer14_attention_weight_min": -31.38812255859375,
      "activations/layer15_attention_weight_max": 36.50684356689453,
      "activations/layer15_attention_weight_min": -31.129974365234375,
      "activations/layer16_attention_weight_max": 35.32479476928711,
      "activations/layer16_attention_weight_min": -32.88851547241211,
      "activations/layer17_attention_weight_max": 55.343910217285156,
      "activations/layer17_attention_weight_min": -47.672332763671875,
      "activations/layer18_attention_weight_max": 52.807098388671875,
      "activations/layer18_attention_weight_min": -40.29752731323242,
      "activations/layer19_attention_weight_max": 24.28510093688965,
      "activations/layer19_attention_weight_min": -22.648881912231445,
      "activations/layer1_attention_weight_max": 16.277942657470703,
      "activations/layer1_attention_weight_min": -14.949796676635742,
      "activations/layer20_attention_weight_max": 25.81179428100586,
      "activations/layer20_attention_weight_min": -21.692626953125,
      "activations/layer21_attention_weight_max": 40.25441360473633,
      "activations/layer21_attention_weight_min": -22.53173828125,
      "activations/layer22_attention_weight_max": 33.8546028137207,
      "activations/layer22_attention_weight_min": -24.335895538330078,
      "activations/layer23_attention_weight_max": 40.880760192871094,
      "activations/layer23_attention_weight_min": -22.833263397216797,
      "activations/layer2_attention_weight_max": 31.915464401245117,
      "activations/layer2_attention_weight_min": -29.791296005249023,
      "activations/layer3_attention_weight_max": 90.75312805175781,
      "activations/layer3_attention_weight_min": -92.40836334228516,
      "activations/layer4_attention_weight_max": 93.72249603271484,
      "activations/layer4_attention_weight_min": -94.05830383300781,
      "activations/layer5_attention_weight_max": 70.4663314819336,
      "activations/layer5_attention_weight_min": -78.06730651855469,
      "activations/layer6_attention_weight_max": 51.535621643066406,
      "activations/layer6_attention_weight_min": -56.148014068603516,
      "activations/layer7_attention_weight_max": 67.59907531738281,
      "activations/layer7_attention_weight_min": -64.32342529296875,
      "activations/layer8_attention_weight_max": 48.28855895996094,
      "activations/layer8_attention_weight_min": -50.592166900634766,
      "activations/layer9_attention_weight_max": 46.200870513916016,
      "activations/layer9_attention_weight_min": -48.18581008911133,
      "epoch": 13.22,
      "learning_rate": 6.541363636363635e-05,
      "loss": 2.7614,
      "step": 227500
    },
    {
      "activations/layer0_attention_weight_max": 15.732723236083984,
      "activations/layer0_attention_weight_min": -13.883011817932129,
      "activations/layer10_attention_weight_max": 38.31482696533203,
      "activations/layer10_attention_weight_min": -37.664894104003906,
      "activations/layer11_attention_weight_max": 31.728710174560547,
      "activations/layer11_attention_weight_min": -34.63853454589844,
      "activations/layer12_attention_weight_max": 19.718412399291992,
      "activations/layer12_attention_weight_min": -23.767377853393555,
      "activations/layer13_attention_weight_max": 44.06045150756836,
      "activations/layer13_attention_weight_min": -34.97697448730469,
      "activations/layer14_attention_weight_max": 42.96179962158203,
      "activations/layer14_attention_weight_min": -32.57379150390625,
      "activations/layer15_attention_weight_max": 35.57584762573242,
      "activations/layer15_attention_weight_min": -30.985315322875977,
      "activations/layer16_attention_weight_max": 34.75947570800781,
      "activations/layer16_attention_weight_min": -32.68211364746094,
      "activations/layer17_attention_weight_max": 57.59523010253906,
      "activations/layer17_attention_weight_min": -47.19499588012695,
      "activations/layer18_attention_weight_max": 52.61096954345703,
      "activations/layer18_attention_weight_min": -39.34782028198242,
      "activations/layer19_attention_weight_max": 24.210721969604492,
      "activations/layer19_attention_weight_min": -22.07487678527832,
      "activations/layer1_attention_weight_max": 16.568159103393555,
      "activations/layer1_attention_weight_min": -15.507582664489746,
      "activations/layer20_attention_weight_max": 24.37767219543457,
      "activations/layer20_attention_weight_min": -27.29207992553711,
      "activations/layer21_attention_weight_max": 42.0932731628418,
      "activations/layer21_attention_weight_min": -21.595199584960938,
      "activations/layer22_attention_weight_max": 33.68849182128906,
      "activations/layer22_attention_weight_min": -26.09954833984375,
      "activations/layer23_attention_weight_max": 40.058441162109375,
      "activations/layer23_attention_weight_min": -25.735034942626953,
      "activations/layer2_attention_weight_max": 30.39935302734375,
      "activations/layer2_attention_weight_min": -29.170406341552734,
      "activations/layer3_attention_weight_max": 91.43739318847656,
      "activations/layer3_attention_weight_min": -92.06859588623047,
      "activations/layer4_attention_weight_max": 94.9566879272461,
      "activations/layer4_attention_weight_min": -92.14672088623047,
      "activations/layer5_attention_weight_max": 74.6392822265625,
      "activations/layer5_attention_weight_min": -76.41883850097656,
      "activations/layer6_attention_weight_max": 55.1209716796875,
      "activations/layer6_attention_weight_min": -50.84984588623047,
      "activations/layer7_attention_weight_max": 74.94505310058594,
      "activations/layer7_attention_weight_min": -70.27310180664062,
      "activations/layer8_attention_weight_max": 47.135528564453125,
      "activations/layer8_attention_weight_min": -51.31235885620117,
      "activations/layer9_attention_weight_max": 47.811527252197266,
      "activations/layer9_attention_weight_min": -47.68345642089844,
      "epoch": 13.22,
      "learning_rate": 6.539469696969697e-05,
      "loss": 2.7721,
      "step": 227550
    },
    {
      "activations/layer0_attention_weight_max": 15.210358619689941,
      "activations/layer0_attention_weight_min": -14.012633323669434,
      "activations/layer10_attention_weight_max": 35.507469177246094,
      "activations/layer10_attention_weight_min": -36.439964294433594,
      "activations/layer11_attention_weight_max": 32.502952575683594,
      "activations/layer11_attention_weight_min": -32.297149658203125,
      "activations/layer12_attention_weight_max": 17.05434799194336,
      "activations/layer12_attention_weight_min": -29.139902114868164,
      "activations/layer13_attention_weight_max": 39.607086181640625,
      "activations/layer13_attention_weight_min": -28.23020362854004,
      "activations/layer14_attention_weight_max": 32.78186798095703,
      "activations/layer14_attention_weight_min": -28.888349533081055,
      "activations/layer15_attention_weight_max": 31.996828079223633,
      "activations/layer15_attention_weight_min": -27.63209342956543,
      "activations/layer16_attention_weight_max": 30.48271369934082,
      "activations/layer16_attention_weight_min": -29.557048797607422,
      "activations/layer17_attention_weight_max": 53.9694709777832,
      "activations/layer17_attention_weight_min": -43.87156295776367,
      "activations/layer18_attention_weight_max": 44.8266716003418,
      "activations/layer18_attention_weight_min": -40.117889404296875,
      "activations/layer19_attention_weight_max": 23.601171493530273,
      "activations/layer19_attention_weight_min": -23.566566467285156,
      "activations/layer1_attention_weight_max": 16.90711212158203,
      "activations/layer1_attention_weight_min": -15.886852264404297,
      "activations/layer20_attention_weight_max": 25.14412498474121,
      "activations/layer20_attention_weight_min": -22.02665901184082,
      "activations/layer21_attention_weight_max": 33.558048248291016,
      "activations/layer21_attention_weight_min": -23.890310287475586,
      "activations/layer22_attention_weight_max": 34.23329162597656,
      "activations/layer22_attention_weight_min": -29.30630874633789,
      "activations/layer23_attention_weight_max": 37.1088981628418,
      "activations/layer23_attention_weight_min": -24.97579574584961,
      "activations/layer2_attention_weight_max": 32.31761932373047,
      "activations/layer2_attention_weight_min": -31.59766387939453,
      "activations/layer3_attention_weight_max": 95.79166412353516,
      "activations/layer3_attention_weight_min": -98.53736114501953,
      "activations/layer4_attention_weight_max": 99.85625457763672,
      "activations/layer4_attention_weight_min": -100.98626708984375,
      "activations/layer5_attention_weight_max": 74.57099914550781,
      "activations/layer5_attention_weight_min": -78.45490264892578,
      "activations/layer6_attention_weight_max": 54.112186431884766,
      "activations/layer6_attention_weight_min": -53.227333068847656,
      "activations/layer7_attention_weight_max": 66.28816223144531,
      "activations/layer7_attention_weight_min": -66.40557861328125,
      "activations/layer8_attention_weight_max": 44.230533599853516,
      "activations/layer8_attention_weight_min": -45.316158294677734,
      "activations/layer9_attention_weight_max": 49.45840835571289,
      "activations/layer9_attention_weight_min": -46.841617584228516,
      "epoch": 13.22,
      "learning_rate": 6.537575757575757e-05,
      "loss": 2.7791,
      "step": 227600
    },
    {
      "activations/layer0_attention_weight_max": 15.450691223144531,
      "activations/layer0_attention_weight_min": -13.763669967651367,
      "activations/layer10_attention_weight_max": 34.81165313720703,
      "activations/layer10_attention_weight_min": -34.605079650878906,
      "activations/layer11_attention_weight_max": 32.48548889160156,
      "activations/layer11_attention_weight_min": -31.503095626831055,
      "activations/layer12_attention_weight_max": 18.30036735534668,
      "activations/layer12_attention_weight_min": -24.087005615234375,
      "activations/layer13_attention_weight_max": 34.18318557739258,
      "activations/layer13_attention_weight_min": -27.856637954711914,
      "activations/layer14_attention_weight_max": 38.09378433227539,
      "activations/layer14_attention_weight_min": -31.04302978515625,
      "activations/layer15_attention_weight_max": 32.72025680541992,
      "activations/layer15_attention_weight_min": -28.50760269165039,
      "activations/layer16_attention_weight_max": 31.316007614135742,
      "activations/layer16_attention_weight_min": -27.976457595825195,
      "activations/layer17_attention_weight_max": 52.29326248168945,
      "activations/layer17_attention_weight_min": -44.01015090942383,
      "activations/layer18_attention_weight_max": 48.61266326904297,
      "activations/layer18_attention_weight_min": -37.23291778564453,
      "activations/layer19_attention_weight_max": 23.26678466796875,
      "activations/layer19_attention_weight_min": -22.930177688598633,
      "activations/layer1_attention_weight_max": 16.013042449951172,
      "activations/layer1_attention_weight_min": -16.555919647216797,
      "activations/layer20_attention_weight_max": 22.993061065673828,
      "activations/layer20_attention_weight_min": -21.668926239013672,
      "activations/layer21_attention_weight_max": 35.01864242553711,
      "activations/layer21_attention_weight_min": -21.903976440429688,
      "activations/layer22_attention_weight_max": 29.4580135345459,
      "activations/layer22_attention_weight_min": -29.157747268676758,
      "activations/layer23_attention_weight_max": 37.24592971801758,
      "activations/layer23_attention_weight_min": -27.04959487915039,
      "activations/layer2_attention_weight_max": 31.815475463867188,
      "activations/layer2_attention_weight_min": -30.956642150878906,
      "activations/layer3_attention_weight_max": 91.91973114013672,
      "activations/layer3_attention_weight_min": -97.080078125,
      "activations/layer4_attention_weight_max": 93.27941131591797,
      "activations/layer4_attention_weight_min": -87.66808319091797,
      "activations/layer5_attention_weight_max": 69.45451354980469,
      "activations/layer5_attention_weight_min": -76.05532836914062,
      "activations/layer6_attention_weight_max": 48.21210861206055,
      "activations/layer6_attention_weight_min": -49.30228805541992,
      "activations/layer7_attention_weight_max": 64.24288177490234,
      "activations/layer7_attention_weight_min": -64.52192687988281,
      "activations/layer8_attention_weight_max": 43.377681732177734,
      "activations/layer8_attention_weight_min": -46.06489181518555,
      "activations/layer9_attention_weight_max": 42.36906814575195,
      "activations/layer9_attention_weight_min": -45.513484954833984,
      "epoch": 13.23,
      "learning_rate": 6.535681818181818e-05,
      "loss": 2.7664,
      "step": 227650
    },
    {
      "activations/layer0_attention_weight_max": 16.08011245727539,
      "activations/layer0_attention_weight_min": -14.072813987731934,
      "activations/layer10_attention_weight_max": 33.35138702392578,
      "activations/layer10_attention_weight_min": -34.43720245361328,
      "activations/layer11_attention_weight_max": 35.99459457397461,
      "activations/layer11_attention_weight_min": -31.350238800048828,
      "activations/layer12_attention_weight_max": 18.22978973388672,
      "activations/layer12_attention_weight_min": -25.300689697265625,
      "activations/layer13_attention_weight_max": 41.005699157714844,
      "activations/layer13_attention_weight_min": -28.61901092529297,
      "activations/layer14_attention_weight_max": 33.5317268371582,
      "activations/layer14_attention_weight_min": -31.521650314331055,
      "activations/layer15_attention_weight_max": 30.05585479736328,
      "activations/layer15_attention_weight_min": -31.27455711364746,
      "activations/layer16_attention_weight_max": 30.185651779174805,
      "activations/layer16_attention_weight_min": -28.804189682006836,
      "activations/layer17_attention_weight_max": 50.29423141479492,
      "activations/layer17_attention_weight_min": -45.037654876708984,
      "activations/layer18_attention_weight_max": 43.31798553466797,
      "activations/layer18_attention_weight_min": -39.249881744384766,
      "activations/layer19_attention_weight_max": 23.93851661682129,
      "activations/layer19_attention_weight_min": -23.0881404876709,
      "activations/layer1_attention_weight_max": 16.633243560791016,
      "activations/layer1_attention_weight_min": -15.764189720153809,
      "activations/layer20_attention_weight_max": 22.08134651184082,
      "activations/layer20_attention_weight_min": -25.660022735595703,
      "activations/layer21_attention_weight_max": 37.259891510009766,
      "activations/layer21_attention_weight_min": -22.475269317626953,
      "activations/layer22_attention_weight_max": 25.96915626525879,
      "activations/layer22_attention_weight_min": -26.03502655029297,
      "activations/layer23_attention_weight_max": 32.81187057495117,
      "activations/layer23_attention_weight_min": -24.88018035888672,
      "activations/layer2_attention_weight_max": 32.092018127441406,
      "activations/layer2_attention_weight_min": -30.162832260131836,
      "activations/layer3_attention_weight_max": 98.22195434570312,
      "activations/layer3_attention_weight_min": -103.8659439086914,
      "activations/layer4_attention_weight_max": 101.63601684570312,
      "activations/layer4_attention_weight_min": -98.86534118652344,
      "activations/layer5_attention_weight_max": 75.65484619140625,
      "activations/layer5_attention_weight_min": -77.32151794433594,
      "activations/layer6_attention_weight_max": 54.363197326660156,
      "activations/layer6_attention_weight_min": -52.123085021972656,
      "activations/layer7_attention_weight_max": 77.16883850097656,
      "activations/layer7_attention_weight_min": -66.58951568603516,
      "activations/layer8_attention_weight_max": 48.3917121887207,
      "activations/layer8_attention_weight_min": -43.54154586791992,
      "activations/layer9_attention_weight_max": 60.58988571166992,
      "activations/layer9_attention_weight_min": -46.6297721862793,
      "epoch": 13.23,
      "learning_rate": 6.533787878787879e-05,
      "loss": 2.7632,
      "step": 227700
    },
    {
      "activations/layer0_attention_weight_max": 18.623577117919922,
      "activations/layer0_attention_weight_min": -13.568550109863281,
      "activations/layer10_attention_weight_max": 33.31315612792969,
      "activations/layer10_attention_weight_min": -34.133880615234375,
      "activations/layer11_attention_weight_max": 29.538944244384766,
      "activations/layer11_attention_weight_min": -30.84223747253418,
      "activations/layer12_attention_weight_max": 16.937580108642578,
      "activations/layer12_attention_weight_min": -24.863656997680664,
      "activations/layer13_attention_weight_max": 34.93317413330078,
      "activations/layer13_attention_weight_min": -30.03853416442871,
      "activations/layer14_attention_weight_max": 33.81425857543945,
      "activations/layer14_attention_weight_min": -33.593467712402344,
      "activations/layer15_attention_weight_max": 31.08854866027832,
      "activations/layer15_attention_weight_min": -33.311683654785156,
      "activations/layer16_attention_weight_max": 29.49656867980957,
      "activations/layer16_attention_weight_min": -30.738065719604492,
      "activations/layer17_attention_weight_max": 48.4433479309082,
      "activations/layer17_attention_weight_min": -45.97233963012695,
      "activations/layer18_attention_weight_max": 44.41401290893555,
      "activations/layer18_attention_weight_min": -40.55070114135742,
      "activations/layer19_attention_weight_max": 22.500370025634766,
      "activations/layer19_attention_weight_min": -24.527103424072266,
      "activations/layer1_attention_weight_max": 16.328968048095703,
      "activations/layer1_attention_weight_min": -15.743828773498535,
      "activations/layer20_attention_weight_max": 23.4163875579834,
      "activations/layer20_attention_weight_min": -25.021305084228516,
      "activations/layer21_attention_weight_max": 35.03950500488281,
      "activations/layer21_attention_weight_min": -23.524927139282227,
      "activations/layer22_attention_weight_max": 28.178653717041016,
      "activations/layer22_attention_weight_min": -24.698083877563477,
      "activations/layer23_attention_weight_max": 35.33351135253906,
      "activations/layer23_attention_weight_min": -24.980648040771484,
      "activations/layer2_attention_weight_max": 30.57369041442871,
      "activations/layer2_attention_weight_min": -30.954071044921875,
      "activations/layer3_attention_weight_max": 91.07564544677734,
      "activations/layer3_attention_weight_min": -96.719482421875,
      "activations/layer4_attention_weight_max": 93.35533142089844,
      "activations/layer4_attention_weight_min": -91.26558685302734,
      "activations/layer5_attention_weight_max": 68.7779541015625,
      "activations/layer5_attention_weight_min": -74.21976470947266,
      "activations/layer6_attention_weight_max": 51.00844955444336,
      "activations/layer6_attention_weight_min": -50.257572174072266,
      "activations/layer7_attention_weight_max": 68.82234191894531,
      "activations/layer7_attention_weight_min": -64.8736343383789,
      "activations/layer8_attention_weight_max": 44.79020309448242,
      "activations/layer8_attention_weight_min": -45.72794723510742,
      "activations/layer9_attention_weight_max": 40.54609680175781,
      "activations/layer9_attention_weight_min": -45.2890625,
      "epoch": 13.23,
      "learning_rate": 6.531893939393939e-05,
      "loss": 2.7536,
      "step": 227750
    },
    {
      "activations/layer0_attention_weight_max": 16.105361938476562,
      "activations/layer0_attention_weight_min": -13.954224586486816,
      "activations/layer10_attention_weight_max": 39.40499496459961,
      "activations/layer10_attention_weight_min": -37.51826477050781,
      "activations/layer11_attention_weight_max": 36.399845123291016,
      "activations/layer11_attention_weight_min": -34.73839569091797,
      "activations/layer12_attention_weight_max": 19.524789810180664,
      "activations/layer12_attention_weight_min": -25.762672424316406,
      "activations/layer13_attention_weight_max": 40.44038391113281,
      "activations/layer13_attention_weight_min": -29.174938201904297,
      "activations/layer14_attention_weight_max": 43.00859832763672,
      "activations/layer14_attention_weight_min": -31.562768936157227,
      "activations/layer15_attention_weight_max": 38.46512985229492,
      "activations/layer15_attention_weight_min": -29.1561279296875,
      "activations/layer16_attention_weight_max": 36.044498443603516,
      "activations/layer16_attention_weight_min": -29.712352752685547,
      "activations/layer17_attention_weight_max": 57.99268341064453,
      "activations/layer17_attention_weight_min": -43.70669937133789,
      "activations/layer18_attention_weight_max": 54.670326232910156,
      "activations/layer18_attention_weight_min": -41.45029830932617,
      "activations/layer19_attention_weight_max": 24.524463653564453,
      "activations/layer19_attention_weight_min": -22.270347595214844,
      "activations/layer1_attention_weight_max": 16.623994827270508,
      "activations/layer1_attention_weight_min": -17.673336029052734,
      "activations/layer20_attention_weight_max": 27.011838912963867,
      "activations/layer20_attention_weight_min": -21.09813690185547,
      "activations/layer21_attention_weight_max": 56.03434371948242,
      "activations/layer21_attention_weight_min": -25.076623916625977,
      "activations/layer22_attention_weight_max": 33.93375778198242,
      "activations/layer22_attention_weight_min": -24.058021545410156,
      "activations/layer23_attention_weight_max": 39.82513427734375,
      "activations/layer23_attention_weight_min": -24.983304977416992,
      "activations/layer2_attention_weight_max": 33.55828857421875,
      "activations/layer2_attention_weight_min": -30.83912467956543,
      "activations/layer3_attention_weight_max": 96.12679290771484,
      "activations/layer3_attention_weight_min": -99.20674133300781,
      "activations/layer4_attention_weight_max": 97.4424057006836,
      "activations/layer4_attention_weight_min": -94.60187530517578,
      "activations/layer5_attention_weight_max": 69.54827880859375,
      "activations/layer5_attention_weight_min": -76.49629211425781,
      "activations/layer6_attention_weight_max": 50.025596618652344,
      "activations/layer6_attention_weight_min": -53.092952728271484,
      "activations/layer7_attention_weight_max": 67.83141326904297,
      "activations/layer7_attention_weight_min": -66.03504180908203,
      "activations/layer8_attention_weight_max": 45.75404739379883,
      "activations/layer8_attention_weight_min": -48.619529724121094,
      "activations/layer9_attention_weight_max": 53.44167709350586,
      "activations/layer9_attention_weight_min": -50.30549240112305,
      "epoch": 13.24,
      "learning_rate": 6.53e-05,
      "loss": 2.777,
      "step": 227800
    },
    {
      "activations/layer0_attention_weight_max": 15.758194923400879,
      "activations/layer0_attention_weight_min": -13.923596382141113,
      "activations/layer10_attention_weight_max": 36.751094818115234,
      "activations/layer10_attention_weight_min": -36.04708480834961,
      "activations/layer11_attention_weight_max": 30.530881881713867,
      "activations/layer11_attention_weight_min": -33.19606018066406,
      "activations/layer12_attention_weight_max": 19.555269241333008,
      "activations/layer12_attention_weight_min": -26.19156265258789,
      "activations/layer13_attention_weight_max": 37.18376159667969,
      "activations/layer13_attention_weight_min": -34.59190368652344,
      "activations/layer14_attention_weight_max": 35.850189208984375,
      "activations/layer14_attention_weight_min": -30.597604751586914,
      "activations/layer15_attention_weight_max": 31.114784240722656,
      "activations/layer15_attention_weight_min": -29.02471160888672,
      "activations/layer16_attention_weight_max": 30.320430755615234,
      "activations/layer16_attention_weight_min": -30.690759658813477,
      "activations/layer17_attention_weight_max": 50.49668502807617,
      "activations/layer17_attention_weight_min": -43.98432540893555,
      "activations/layer18_attention_weight_max": 45.440547943115234,
      "activations/layer18_attention_weight_min": -37.164371490478516,
      "activations/layer19_attention_weight_max": 22.506181716918945,
      "activations/layer19_attention_weight_min": -22.097637176513672,
      "activations/layer1_attention_weight_max": 16.45684814453125,
      "activations/layer1_attention_weight_min": -14.564170837402344,
      "activations/layer20_attention_weight_max": 24.58705711364746,
      "activations/layer20_attention_weight_min": -21.855459213256836,
      "activations/layer21_attention_weight_max": 35.39450454711914,
      "activations/layer21_attention_weight_min": -23.53297996520996,
      "activations/layer22_attention_weight_max": 30.696247100830078,
      "activations/layer22_attention_weight_min": -26.276472091674805,
      "activations/layer23_attention_weight_max": 35.825950622558594,
      "activations/layer23_attention_weight_min": -23.664579391479492,
      "activations/layer2_attention_weight_max": 31.998809814453125,
      "activations/layer2_attention_weight_min": -30.52449607849121,
      "activations/layer3_attention_weight_max": 90.1058578491211,
      "activations/layer3_attention_weight_min": -97.20889282226562,
      "activations/layer4_attention_weight_max": 97.04116821289062,
      "activations/layer4_attention_weight_min": -91.70813751220703,
      "activations/layer5_attention_weight_max": 73.26008605957031,
      "activations/layer5_attention_weight_min": -76.88142395019531,
      "activations/layer6_attention_weight_max": 52.254554748535156,
      "activations/layer6_attention_weight_min": -51.29376983642578,
      "activations/layer7_attention_weight_max": 68.78858947753906,
      "activations/layer7_attention_weight_min": -65.70062255859375,
      "activations/layer8_attention_weight_max": 44.38230514526367,
      "activations/layer8_attention_weight_min": -45.32168197631836,
      "activations/layer9_attention_weight_max": 43.23371505737305,
      "activations/layer9_attention_weight_min": -46.38395309448242,
      "epoch": 13.24,
      "learning_rate": 6.52810606060606e-05,
      "loss": 2.7726,
      "step": 227850
    },
    {
      "activations/layer0_attention_weight_max": 15.790860176086426,
      "activations/layer0_attention_weight_min": -13.469670295715332,
      "activations/layer10_attention_weight_max": 37.1225700378418,
      "activations/layer10_attention_weight_min": -36.34611892700195,
      "activations/layer11_attention_weight_max": 32.49335861206055,
      "activations/layer11_attention_weight_min": -33.95970916748047,
      "activations/layer12_attention_weight_max": 18.472370147705078,
      "activations/layer12_attention_weight_min": -25.3940486907959,
      "activations/layer13_attention_weight_max": 38.4401969909668,
      "activations/layer13_attention_weight_min": -27.8682918548584,
      "activations/layer14_attention_weight_max": 36.00434875488281,
      "activations/layer14_attention_weight_min": -31.163949966430664,
      "activations/layer15_attention_weight_max": 32.84867858886719,
      "activations/layer15_attention_weight_min": -30.636747360229492,
      "activations/layer16_attention_weight_max": 30.661758422851562,
      "activations/layer16_attention_weight_min": -30.59001922607422,
      "activations/layer17_attention_weight_max": 51.020111083984375,
      "activations/layer17_attention_weight_min": -44.79987335205078,
      "activations/layer18_attention_weight_max": 45.80094909667969,
      "activations/layer18_attention_weight_min": -40.21552658081055,
      "activations/layer19_attention_weight_max": 24.54678726196289,
      "activations/layer19_attention_weight_min": -24.43427085876465,
      "activations/layer1_attention_weight_max": 16.384437561035156,
      "activations/layer1_attention_weight_min": -16.809371948242188,
      "activations/layer20_attention_weight_max": 23.035655975341797,
      "activations/layer20_attention_weight_min": -25.855918884277344,
      "activations/layer21_attention_weight_max": 36.796302795410156,
      "activations/layer21_attention_weight_min": -23.234272003173828,
      "activations/layer22_attention_weight_max": 29.619701385498047,
      "activations/layer22_attention_weight_min": -27.916366577148438,
      "activations/layer23_attention_weight_max": 35.77125549316406,
      "activations/layer23_attention_weight_min": -24.379051208496094,
      "activations/layer2_attention_weight_max": 30.457454681396484,
      "activations/layer2_attention_weight_min": -30.971881866455078,
      "activations/layer3_attention_weight_max": 99.87028503417969,
      "activations/layer3_attention_weight_min": -103.03716278076172,
      "activations/layer4_attention_weight_max": 99.15668487548828,
      "activations/layer4_attention_weight_min": -98.3745346069336,
      "activations/layer5_attention_weight_max": 70.43720245361328,
      "activations/layer5_attention_weight_min": -80.17066955566406,
      "activations/layer6_attention_weight_max": 51.702796936035156,
      "activations/layer6_attention_weight_min": -53.584232330322266,
      "activations/layer7_attention_weight_max": 66.66435241699219,
      "activations/layer7_attention_weight_min": -63.96767807006836,
      "activations/layer8_attention_weight_max": 45.72478103637695,
      "activations/layer8_attention_weight_min": -48.33966827392578,
      "activations/layer9_attention_weight_max": 44.26018524169922,
      "activations/layer9_attention_weight_min": -46.266544342041016,
      "epoch": 13.24,
      "learning_rate": 6.52625e-05,
      "loss": 2.7577,
      "step": 227900
    },
    {
      "activations/layer0_attention_weight_max": 17.26609230041504,
      "activations/layer0_attention_weight_min": -13.974507331848145,
      "activations/layer10_attention_weight_max": 36.927650451660156,
      "activations/layer10_attention_weight_min": -37.2319221496582,
      "activations/layer11_attention_weight_max": 33.80070495605469,
      "activations/layer11_attention_weight_min": -33.02629852294922,
      "activations/layer12_attention_weight_max": 19.79570198059082,
      "activations/layer12_attention_weight_min": -26.551298141479492,
      "activations/layer13_attention_weight_max": 35.31124496459961,
      "activations/layer13_attention_weight_min": -28.654693603515625,
      "activations/layer14_attention_weight_max": 38.20005798339844,
      "activations/layer14_attention_weight_min": -32.38987731933594,
      "activations/layer15_attention_weight_max": 35.39534378051758,
      "activations/layer15_attention_weight_min": -29.795154571533203,
      "activations/layer16_attention_weight_max": 33.01614761352539,
      "activations/layer16_attention_weight_min": -30.94795799255371,
      "activations/layer17_attention_weight_max": 53.6706428527832,
      "activations/layer17_attention_weight_min": -46.51119613647461,
      "activations/layer18_attention_weight_max": 50.4771842956543,
      "activations/layer18_attention_weight_min": -40.747802734375,
      "activations/layer19_attention_weight_max": 22.339818954467773,
      "activations/layer19_attention_weight_min": -22.932186126708984,
      "activations/layer1_attention_weight_max": 16.88875389099121,
      "activations/layer1_attention_weight_min": -16.112516403198242,
      "activations/layer20_attention_weight_max": 23.03936767578125,
      "activations/layer20_attention_weight_min": -23.49652671813965,
      "activations/layer21_attention_weight_max": 38.10197830200195,
      "activations/layer21_attention_weight_min": -22.821929931640625,
      "activations/layer22_attention_weight_max": 31.670551300048828,
      "activations/layer22_attention_weight_min": -25.969942092895508,
      "activations/layer23_attention_weight_max": 40.4210090637207,
      "activations/layer23_attention_weight_min": -24.23418426513672,
      "activations/layer2_attention_weight_max": 30.03168487548828,
      "activations/layer2_attention_weight_min": -28.541183471679688,
      "activations/layer3_attention_weight_max": 92.17282104492188,
      "activations/layer3_attention_weight_min": -93.70846557617188,
      "activations/layer4_attention_weight_max": 98.83495330810547,
      "activations/layer4_attention_weight_min": -95.40876770019531,
      "activations/layer5_attention_weight_max": 70.5511474609375,
      "activations/layer5_attention_weight_min": -76.30732727050781,
      "activations/layer6_attention_weight_max": 51.98424530029297,
      "activations/layer6_attention_weight_min": -52.26871109008789,
      "activations/layer7_attention_weight_max": 67.10533142089844,
      "activations/layer7_attention_weight_min": -64.67322540283203,
      "activations/layer8_attention_weight_max": 47.17995834350586,
      "activations/layer8_attention_weight_min": -50.8109016418457,
      "activations/layer9_attention_weight_max": 46.66796875,
      "activations/layer9_attention_weight_min": -47.500274658203125,
      "epoch": 13.25,
      "learning_rate": 6.52435606060606e-05,
      "loss": 2.7655,
      "step": 227950
    },
    {
      "activations/layer0_attention_weight_max": 16.373443603515625,
      "activations/layer0_attention_weight_min": -13.713302612304688,
      "activations/layer10_attention_weight_max": 34.481998443603516,
      "activations/layer10_attention_weight_min": -35.74453353881836,
      "activations/layer11_attention_weight_max": 31.393095016479492,
      "activations/layer11_attention_weight_min": -32.864479064941406,
      "activations/layer12_attention_weight_max": 17.506071090698242,
      "activations/layer12_attention_weight_min": -29.206274032592773,
      "activations/layer13_attention_weight_max": 28.79405403137207,
      "activations/layer13_attention_weight_min": -26.767620086669922,
      "activations/layer14_attention_weight_max": 33.911598205566406,
      "activations/layer14_attention_weight_min": -31.472017288208008,
      "activations/layer15_attention_weight_max": 28.79563331604004,
      "activations/layer15_attention_weight_min": -29.431583404541016,
      "activations/layer16_attention_weight_max": 29.871492385864258,
      "activations/layer16_attention_weight_min": -29.61722755432129,
      "activations/layer17_attention_weight_max": 51.0595817565918,
      "activations/layer17_attention_weight_min": -43.909873962402344,
      "activations/layer18_attention_weight_max": 40.70808029174805,
      "activations/layer18_attention_weight_min": -38.087852478027344,
      "activations/layer19_attention_weight_max": 21.48495101928711,
      "activations/layer19_attention_weight_min": -22.131479263305664,
      "activations/layer1_attention_weight_max": 16.527135848999023,
      "activations/layer1_attention_weight_min": -14.26207160949707,
      "activations/layer20_attention_weight_max": 19.1521053314209,
      "activations/layer20_attention_weight_min": -23.10649299621582,
      "activations/layer21_attention_weight_max": 29.740983963012695,
      "activations/layer21_attention_weight_min": -20.292560577392578,
      "activations/layer22_attention_weight_max": 26.013906478881836,
      "activations/layer22_attention_weight_min": -25.08281707763672,
      "activations/layer23_attention_weight_max": 33.18342590332031,
      "activations/layer23_attention_weight_min": -23.660438537597656,
      "activations/layer2_attention_weight_max": 30.995647430419922,
      "activations/layer2_attention_weight_min": -28.965293884277344,
      "activations/layer3_attention_weight_max": 95.66864013671875,
      "activations/layer3_attention_weight_min": -94.43407440185547,
      "activations/layer4_attention_weight_max": 100.88021087646484,
      "activations/layer4_attention_weight_min": -99.10826110839844,
      "activations/layer5_attention_weight_max": 75.639892578125,
      "activations/layer5_attention_weight_min": -78.27821350097656,
      "activations/layer6_attention_weight_max": 49.70179748535156,
      "activations/layer6_attention_weight_min": -50.701480865478516,
      "activations/layer7_attention_weight_max": 65.15876770019531,
      "activations/layer7_attention_weight_min": -66.11467742919922,
      "activations/layer8_attention_weight_max": 42.03284454345703,
      "activations/layer8_attention_weight_min": -46.6302490234375,
      "activations/layer9_attention_weight_max": 47.442291259765625,
      "activations/layer9_attention_weight_min": -44.411537170410156,
      "epoch": 13.25,
      "learning_rate": 6.522462121212121e-05,
      "loss": 2.759,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_loss": 2.72265625,
      "eval_runtime": 8.4693,
      "eval_samples_per_second": 507.009,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_openwebtext_loss": 2.72265625,
      "eval_openwebtext_ppl": 15.220698576992893,
      "eval_openwebtext_runtime": 8.4693,
      "eval_openwebtext_samples_per_second": 507.009,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9663,
      "eval_wikitext_samples_per_second": 231.905,
      "step": 228000
    },
    {
      "epoch": 13.25,
      "eval_lambada_loss": 2.578125,
      "eval_lambada_ppl": 13.172416709211404,
      "eval_lambada_runtime": 9.5633,
      "eval_lambada_samples_per_second": 509.134,
      "step": 228000
    },
    {
      "activations/layer0_attention_weight_max": 15.426790237426758,
      "activations/layer0_attention_weight_min": -13.946815490722656,
      "activations/layer10_attention_weight_max": 36.96173095703125,
      "activations/layer10_attention_weight_min": -37.83642578125,
      "activations/layer11_attention_weight_max": 32.96174240112305,
      "activations/layer11_attention_weight_min": -34.52259826660156,
      "activations/layer12_attention_weight_max": 20.729843139648438,
      "activations/layer12_attention_weight_min": -24.116256713867188,
      "activations/layer13_attention_weight_max": 39.92737579345703,
      "activations/layer13_attention_weight_min": -29.107208251953125,
      "activations/layer14_attention_weight_max": 37.22825241088867,
      "activations/layer14_attention_weight_min": -30.049631118774414,
      "activations/layer15_attention_weight_max": 35.03429412841797,
      "activations/layer15_attention_weight_min": -31.42845916748047,
      "activations/layer16_attention_weight_max": 34.65207290649414,
      "activations/layer16_attention_weight_min": -30.624467849731445,
      "activations/layer17_attention_weight_max": 54.133934020996094,
      "activations/layer17_attention_weight_min": -44.47843551635742,
      "activations/layer18_attention_weight_max": 50.299922943115234,
      "activations/layer18_attention_weight_min": -41.00224304199219,
      "activations/layer19_attention_weight_max": 25.370439529418945,
      "activations/layer19_attention_weight_min": -24.261953353881836,
      "activations/layer1_attention_weight_max": 16.312496185302734,
      "activations/layer1_attention_weight_min": -14.916349411010742,
      "activations/layer20_attention_weight_max": 28.668046951293945,
      "activations/layer20_attention_weight_min": -22.16978645324707,
      "activations/layer21_attention_weight_max": 43.739559173583984,
      "activations/layer21_attention_weight_min": -26.442123413085938,
      "activations/layer22_attention_weight_max": 33.058067321777344,
      "activations/layer22_attention_weight_min": -25.92340850830078,
      "activations/layer23_attention_weight_max": 38.199127197265625,
      "activations/layer23_attention_weight_min": -23.17847442626953,
      "activations/layer2_attention_weight_max": 29.541332244873047,
      "activations/layer2_attention_weight_min": -27.93828582763672,
      "activations/layer3_attention_weight_max": 88.39338684082031,
      "activations/layer3_attention_weight_min": -86.09524536132812,
      "activations/layer4_attention_weight_max": 92.1108627319336,
      "activations/layer4_attention_weight_min": -92.31524658203125,
      "activations/layer5_attention_weight_max": 71.2320556640625,
      "activations/layer5_attention_weight_min": -82.40542602539062,
      "activations/layer6_attention_weight_max": 50.66486358642578,
      "activations/layer6_attention_weight_min": -49.506141662597656,
      "activations/layer7_attention_weight_max": 71.12736511230469,
      "activations/layer7_attention_weight_min": -67.44255065917969,
      "activations/layer8_attention_weight_max": 44.78539276123047,
      "activations/layer8_attention_weight_min": -48.15521240234375,
      "activations/layer9_attention_weight_max": 50.042415618896484,
      "activations/layer9_attention_weight_min": -47.274349212646484,
      "epoch": 13.25,
      "learning_rate": 6.520568181818181e-05,
      "loss": 2.7605,
      "step": 228050
    },
    {
      "activations/layer0_attention_weight_max": 17.0449275970459,
      "activations/layer0_attention_weight_min": -13.864282608032227,
      "activations/layer10_attention_weight_max": 39.203163146972656,
      "activations/layer10_attention_weight_min": -36.191566467285156,
      "activations/layer11_attention_weight_max": 36.435977935791016,
      "activations/layer11_attention_weight_min": -33.78204345703125,
      "activations/layer12_attention_weight_max": 22.71058464050293,
      "activations/layer12_attention_weight_min": -26.234773635864258,
      "activations/layer13_attention_weight_max": 43.85606384277344,
      "activations/layer13_attention_weight_min": -31.236412048339844,
      "activations/layer14_attention_weight_max": 40.75911331176758,
      "activations/layer14_attention_weight_min": -30.403963088989258,
      "activations/layer15_attention_weight_max": 34.96734619140625,
      "activations/layer15_attention_weight_min": -29.78138542175293,
      "activations/layer16_attention_weight_max": 32.39859390258789,
      "activations/layer16_attention_weight_min": -30.577701568603516,
      "activations/layer17_attention_weight_max": 52.543540954589844,
      "activations/layer17_attention_weight_min": -46.0585823059082,
      "activations/layer18_attention_weight_max": 49.48764419555664,
      "activations/layer18_attention_weight_min": -40.287139892578125,
      "activations/layer19_attention_weight_max": 24.077016830444336,
      "activations/layer19_attention_weight_min": -21.66021728515625,
      "activations/layer1_attention_weight_max": 17.305892944335938,
      "activations/layer1_attention_weight_min": -14.494577407836914,
      "activations/layer20_attention_weight_max": 26.05077362060547,
      "activations/layer20_attention_weight_min": -21.6861515045166,
      "activations/layer21_attention_weight_max": 42.031494140625,
      "activations/layer21_attention_weight_min": -25.712804794311523,
      "activations/layer22_attention_weight_max": 32.494773864746094,
      "activations/layer22_attention_weight_min": -29.2685604095459,
      "activations/layer23_attention_weight_max": 41.05804443359375,
      "activations/layer23_attention_weight_min": -23.54842185974121,
      "activations/layer2_attention_weight_max": 31.54950523376465,
      "activations/layer2_attention_weight_min": -32.54921340942383,
      "activations/layer3_attention_weight_max": 96.13919830322266,
      "activations/layer3_attention_weight_min": -93.895751953125,
      "activations/layer4_attention_weight_max": 97.39907836914062,
      "activations/layer4_attention_weight_min": -93.94804382324219,
      "activations/layer5_attention_weight_max": 73.50547790527344,
      "activations/layer5_attention_weight_min": -75.48934173583984,
      "activations/layer6_attention_weight_max": 53.29347229003906,
      "activations/layer6_attention_weight_min": -51.195987701416016,
      "activations/layer7_attention_weight_max": 69.10769653320312,
      "activations/layer7_attention_weight_min": -70.78063201904297,
      "activations/layer8_attention_weight_max": 46.23884201049805,
      "activations/layer8_attention_weight_min": -47.448184967041016,
      "activations/layer9_attention_weight_max": 51.50636672973633,
      "activations/layer9_attention_weight_min": -47.38737106323242,
      "epoch": 13.25,
      "learning_rate": 6.518674242424241e-05,
      "loss": 2.7766,
      "step": 228100
    },
    {
      "activations/layer0_attention_weight_max": 15.998963356018066,
      "activations/layer0_attention_weight_min": -14.270018577575684,
      "activations/layer10_attention_weight_max": 38.54099655151367,
      "activations/layer10_attention_weight_min": -39.26961898803711,
      "activations/layer11_attention_weight_max": 34.870445251464844,
      "activations/layer11_attention_weight_min": -36.36194610595703,
      "activations/layer12_attention_weight_max": 19.59757423400879,
      "activations/layer12_attention_weight_min": -27.695714950561523,
      "activations/layer13_attention_weight_max": 41.366512298583984,
      "activations/layer13_attention_weight_min": -32.20663070678711,
      "activations/layer14_attention_weight_max": 46.01579666137695,
      "activations/layer14_attention_weight_min": -33.5931510925293,
      "activations/layer15_attention_weight_max": 40.06077575683594,
      "activations/layer15_attention_weight_min": -32.77143859863281,
      "activations/layer16_attention_weight_max": 36.61807632446289,
      "activations/layer16_attention_weight_min": -33.24631881713867,
      "activations/layer17_attention_weight_max": 57.44853973388672,
      "activations/layer17_attention_weight_min": -48.36246109008789,
      "activations/layer18_attention_weight_max": 53.34400177001953,
      "activations/layer18_attention_weight_min": -40.489173889160156,
      "activations/layer19_attention_weight_max": 25.197179794311523,
      "activations/layer19_attention_weight_min": -21.891576766967773,
      "activations/layer1_attention_weight_max": 16.154918670654297,
      "activations/layer1_attention_weight_min": -16.594369888305664,
      "activations/layer20_attention_weight_max": 25.67605209350586,
      "activations/layer20_attention_weight_min": -22.072105407714844,
      "activations/layer21_attention_weight_max": 42.81515884399414,
      "activations/layer21_attention_weight_min": -25.513675689697266,
      "activations/layer22_attention_weight_max": 35.50099182128906,
      "activations/layer22_attention_weight_min": -27.764680862426758,
      "activations/layer23_attention_weight_max": 38.43836975097656,
      "activations/layer23_attention_weight_min": -28.290769577026367,
      "activations/layer2_attention_weight_max": 31.98409652709961,
      "activations/layer2_attention_weight_min": -31.14348602294922,
      "activations/layer3_attention_weight_max": 95.72734069824219,
      "activations/layer3_attention_weight_min": -94.44764709472656,
      "activations/layer4_attention_weight_max": 98.9157943725586,
      "activations/layer4_attention_weight_min": -95.87196350097656,
      "activations/layer5_attention_weight_max": 70.40542602539062,
      "activations/layer5_attention_weight_min": -76.69099426269531,
      "activations/layer6_attention_weight_max": 53.43722152709961,
      "activations/layer6_attention_weight_min": -51.945499420166016,
      "activations/layer7_attention_weight_max": 71.39128875732422,
      "activations/layer7_attention_weight_min": -70.29872131347656,
      "activations/layer8_attention_weight_max": 49.54181671142578,
      "activations/layer8_attention_weight_min": -51.91734313964844,
      "activations/layer9_attention_weight_max": 48.61046600341797,
      "activations/layer9_attention_weight_min": -49.215538024902344,
      "epoch": 13.26,
      "learning_rate": 6.516780303030303e-05,
      "loss": 2.7653,
      "step": 228150
    },
    {
      "activations/layer0_attention_weight_max": 17.400503158569336,
      "activations/layer0_attention_weight_min": -13.771430969238281,
      "activations/layer10_attention_weight_max": 37.15033721923828,
      "activations/layer10_attention_weight_min": -35.26106643676758,
      "activations/layer11_attention_weight_max": 35.72087097167969,
      "activations/layer11_attention_weight_min": -33.568878173828125,
      "activations/layer12_attention_weight_max": 18.25886344909668,
      "activations/layer12_attention_weight_min": -26.86256217956543,
      "activations/layer13_attention_weight_max": 40.338218688964844,
      "activations/layer13_attention_weight_min": -25.487598419189453,
      "activations/layer14_attention_weight_max": 34.49202346801758,
      "activations/layer14_attention_weight_min": -30.112079620361328,
      "activations/layer15_attention_weight_max": 31.0458984375,
      "activations/layer15_attention_weight_min": -29.158864974975586,
      "activations/layer16_attention_weight_max": 29.99262809753418,
      "activations/layer16_attention_weight_min": -29.67400360107422,
      "activations/layer17_attention_weight_max": 50.42768096923828,
      "activations/layer17_attention_weight_min": -45.12407302856445,
      "activations/layer18_attention_weight_max": 50.28799057006836,
      "activations/layer18_attention_weight_min": -40.279659271240234,
      "activations/layer19_attention_weight_max": 23.54279136657715,
      "activations/layer19_attention_weight_min": -23.737157821655273,
      "activations/layer1_attention_weight_max": 16.903825759887695,
      "activations/layer1_attention_weight_min": -15.026628494262695,
      "activations/layer20_attention_weight_max": 24.4716796875,
      "activations/layer20_attention_weight_min": -22.329444885253906,
      "activations/layer21_attention_weight_max": 34.55235290527344,
      "activations/layer21_attention_weight_min": -24.573423385620117,
      "activations/layer22_attention_weight_max": 31.227460861206055,
      "activations/layer22_attention_weight_min": -28.1838436126709,
      "activations/layer23_attention_weight_max": 37.247650146484375,
      "activations/layer23_attention_weight_min": -26.112503051757812,
      "activations/layer2_attention_weight_max": 33.02142333984375,
      "activations/layer2_attention_weight_min": -31.096824645996094,
      "activations/layer3_attention_weight_max": 94.58822631835938,
      "activations/layer3_attention_weight_min": -100.08311462402344,
      "activations/layer4_attention_weight_max": 104.77980041503906,
      "activations/layer4_attention_weight_min": -98.56849670410156,
      "activations/layer5_attention_weight_max": 73.62308502197266,
      "activations/layer5_attention_weight_min": -79.951171875,
      "activations/layer6_attention_weight_max": 53.573543548583984,
      "activations/layer6_attention_weight_min": -53.40494918823242,
      "activations/layer7_attention_weight_max": 73.80360412597656,
      "activations/layer7_attention_weight_min": -69.48078918457031,
      "activations/layer8_attention_weight_max": 49.83075714111328,
      "activations/layer8_attention_weight_min": -50.94935607910156,
      "activations/layer9_attention_weight_max": 49.11109161376953,
      "activations/layer9_attention_weight_min": -47.29186248779297,
      "epoch": 13.26,
      "learning_rate": 6.514886363636363e-05,
      "loss": 2.7697,
      "step": 228200
    },
    {
      "activations/layer0_attention_weight_max": 15.612693786621094,
      "activations/layer0_attention_weight_min": -13.89666748046875,
      "activations/layer10_attention_weight_max": 37.379539489746094,
      "activations/layer10_attention_weight_min": -37.24510192871094,
      "activations/layer11_attention_weight_max": 31.644939422607422,
      "activations/layer11_attention_weight_min": -32.45718002319336,
      "activations/layer12_attention_weight_max": 16.872148513793945,
      "activations/layer12_attention_weight_min": -24.51093292236328,
      "activations/layer13_attention_weight_max": 33.166316986083984,
      "activations/layer13_attention_weight_min": -28.01189613342285,
      "activations/layer14_attention_weight_max": 35.06214141845703,
      "activations/layer14_attention_weight_min": -29.023757934570312,
      "activations/layer15_attention_weight_max": 29.72749137878418,
      "activations/layer15_attention_weight_min": -28.139873504638672,
      "activations/layer16_attention_weight_max": 29.448747634887695,
      "activations/layer16_attention_weight_min": -28.550254821777344,
      "activations/layer17_attention_weight_max": 47.18764114379883,
      "activations/layer17_attention_weight_min": -41.852813720703125,
      "activations/layer18_attention_weight_max": 42.11677169799805,
      "activations/layer18_attention_weight_min": -34.943546295166016,
      "activations/layer19_attention_weight_max": 18.96900749206543,
      "activations/layer19_attention_weight_min": -19.923341751098633,
      "activations/layer1_attention_weight_max": 16.935197830200195,
      "activations/layer1_attention_weight_min": -14.259441375732422,
      "activations/layer20_attention_weight_max": 18.29196548461914,
      "activations/layer20_attention_weight_min": -21.94223403930664,
      "activations/layer21_attention_weight_max": 32.64877700805664,
      "activations/layer21_attention_weight_min": -21.728212356567383,
      "activations/layer22_attention_weight_max": 27.59111785888672,
      "activations/layer22_attention_weight_min": -26.531641006469727,
      "activations/layer23_attention_weight_max": 32.826290130615234,
      "activations/layer23_attention_weight_min": -24.860637664794922,
      "activations/layer2_attention_weight_max": 30.151906967163086,
      "activations/layer2_attention_weight_min": -29.528064727783203,
      "activations/layer3_attention_weight_max": 88.65542602539062,
      "activations/layer3_attention_weight_min": -91.46036529541016,
      "activations/layer4_attention_weight_max": 91.24073791503906,
      "activations/layer4_attention_weight_min": -90.0440902709961,
      "activations/layer5_attention_weight_max": 72.419677734375,
      "activations/layer5_attention_weight_min": -78.27902221679688,
      "activations/layer6_attention_weight_max": 49.28639221191406,
      "activations/layer6_attention_weight_min": -52.24150085449219,
      "activations/layer7_attention_weight_max": 67.46395874023438,
      "activations/layer7_attention_weight_min": -65.0112075805664,
      "activations/layer8_attention_weight_max": 47.57757568359375,
      "activations/layer8_attention_weight_min": -48.45552062988281,
      "activations/layer9_attention_weight_max": 46.3072624206543,
      "activations/layer9_attention_weight_min": -48.84432601928711,
      "epoch": 13.26,
      "learning_rate": 6.512992424242423e-05,
      "loss": 2.7554,
      "step": 228250
    },
    {
      "activations/layer0_attention_weight_max": 15.698071479797363,
      "activations/layer0_attention_weight_min": -14.288491249084473,
      "activations/layer10_attention_weight_max": 35.977115631103516,
      "activations/layer10_attention_weight_min": -36.931148529052734,
      "activations/layer11_attention_weight_max": 33.51460266113281,
      "activations/layer11_attention_weight_min": -34.47926712036133,
      "activations/layer12_attention_weight_max": 20.024179458618164,
      "activations/layer12_attention_weight_min": -23.687170028686523,
      "activations/layer13_attention_weight_max": 42.04146957397461,
      "activations/layer13_attention_weight_min": -36.06388854980469,
      "activations/layer14_attention_weight_max": 40.24802017211914,
      "activations/layer14_attention_weight_min": -32.83440399169922,
      "activations/layer15_attention_weight_max": 41.3660888671875,
      "activations/layer15_attention_weight_min": -33.74751281738281,
      "activations/layer16_attention_weight_max": 37.95881271362305,
      "activations/layer16_attention_weight_min": -36.04765319824219,
      "activations/layer17_attention_weight_max": 57.41284942626953,
      "activations/layer17_attention_weight_min": -55.19257736206055,
      "activations/layer18_attention_weight_max": 51.5720100402832,
      "activations/layer18_attention_weight_min": -46.28474426269531,
      "activations/layer19_attention_weight_max": 24.535829544067383,
      "activations/layer19_attention_weight_min": -26.10170555114746,
      "activations/layer1_attention_weight_max": 16.442934036254883,
      "activations/layer1_attention_weight_min": -15.840606689453125,
      "activations/layer20_attention_weight_max": 24.528446197509766,
      "activations/layer20_attention_weight_min": -23.84050750732422,
      "activations/layer21_attention_weight_max": 44.427799224853516,
      "activations/layer21_attention_weight_min": -27.772960662841797,
      "activations/layer22_attention_weight_max": 32.08732223510742,
      "activations/layer22_attention_weight_min": -29.179426193237305,
      "activations/layer23_attention_weight_max": 37.852909088134766,
      "activations/layer23_attention_weight_min": -28.231155395507812,
      "activations/layer2_attention_weight_max": 31.482254028320312,
      "activations/layer2_attention_weight_min": -29.10281753540039,
      "activations/layer3_attention_weight_max": 93.41895294189453,
      "activations/layer3_attention_weight_min": -94.69377899169922,
      "activations/layer4_attention_weight_max": 101.21435546875,
      "activations/layer4_attention_weight_min": -95.49815368652344,
      "activations/layer5_attention_weight_max": 71.87457275390625,
      "activations/layer5_attention_weight_min": -77.49015808105469,
      "activations/layer6_attention_weight_max": 53.32252502441406,
      "activations/layer6_attention_weight_min": -54.909889221191406,
      "activations/layer7_attention_weight_max": 68.79280090332031,
      "activations/layer7_attention_weight_min": -73.84322357177734,
      "activations/layer8_attention_weight_max": 50.74467849731445,
      "activations/layer8_attention_weight_min": -54.25952911376953,
      "activations/layer9_attention_weight_max": 48.37635803222656,
      "activations/layer9_attention_weight_min": -51.88813400268555,
      "epoch": 13.27,
      "learning_rate": 6.511098484848485e-05,
      "loss": 2.7692,
      "step": 228300
    },
    {
      "activations/layer0_attention_weight_max": 16.657737731933594,
      "activations/layer0_attention_weight_min": -14.045411109924316,
      "activations/layer10_attention_weight_max": 37.89219665527344,
      "activations/layer10_attention_weight_min": -37.52965545654297,
      "activations/layer11_attention_weight_max": 32.84235763549805,
      "activations/layer11_attention_weight_min": -32.572269439697266,
      "activations/layer12_attention_weight_max": 17.916339874267578,
      "activations/layer12_attention_weight_min": -25.91978645324707,
      "activations/layer13_attention_weight_max": 40.9234504699707,
      "activations/layer13_attention_weight_min": -30.08991050720215,
      "activations/layer14_attention_weight_max": 35.99953842163086,
      "activations/layer14_attention_weight_min": -31.604759216308594,
      "activations/layer15_attention_weight_max": 32.39159393310547,
      "activations/layer15_attention_weight_min": -29.85327911376953,
      "activations/layer16_attention_weight_max": 32.3718376159668,
      "activations/layer16_attention_weight_min": -30.7362060546875,
      "activations/layer17_attention_weight_max": 53.1125373840332,
      "activations/layer17_attention_weight_min": -48.43901062011719,
      "activations/layer18_attention_weight_max": 49.371063232421875,
      "activations/layer18_attention_weight_min": -41.54304122924805,
      "activations/layer19_attention_weight_max": 25.4642333984375,
      "activations/layer19_attention_weight_min": -21.497907638549805,
      "activations/layer1_attention_weight_max": 18.076290130615234,
      "activations/layer1_attention_weight_min": -14.88538646697998,
      "activations/layer20_attention_weight_max": 26.419910430908203,
      "activations/layer20_attention_weight_min": -20.748489379882812,
      "activations/layer21_attention_weight_max": 39.96227264404297,
      "activations/layer21_attention_weight_min": -21.568666458129883,
      "activations/layer22_attention_weight_max": 31.565027236938477,
      "activations/layer22_attention_weight_min": -25.7204647064209,
      "activations/layer23_attention_weight_max": 37.175106048583984,
      "activations/layer23_attention_weight_min": -24.663742065429688,
      "activations/layer2_attention_weight_max": 33.191383361816406,
      "activations/layer2_attention_weight_min": -31.014690399169922,
      "activations/layer3_attention_weight_max": 94.14527893066406,
      "activations/layer3_attention_weight_min": -90.8531723022461,
      "activations/layer4_attention_weight_max": 96.41107177734375,
      "activations/layer4_attention_weight_min": -92.70964813232422,
      "activations/layer5_attention_weight_max": 72.96729278564453,
      "activations/layer5_attention_weight_min": -76.49700927734375,
      "activations/layer6_attention_weight_max": 54.773155212402344,
      "activations/layer6_attention_weight_min": -51.96841049194336,
      "activations/layer7_attention_weight_max": 66.87891387939453,
      "activations/layer7_attention_weight_min": -66.58142852783203,
      "activations/layer8_attention_weight_max": 46.9396858215332,
      "activations/layer8_attention_weight_min": -52.32439422607422,
      "activations/layer9_attention_weight_max": 50.780941009521484,
      "activations/layer9_attention_weight_min": -47.60074234008789,
      "epoch": 13.27,
      "learning_rate": 6.509204545454545e-05,
      "loss": 2.7737,
      "step": 228350
    },
    {
      "activations/layer0_attention_weight_max": 15.785128593444824,
      "activations/layer0_attention_weight_min": -14.892264366149902,
      "activations/layer10_attention_weight_max": 34.25509262084961,
      "activations/layer10_attention_weight_min": -35.49351501464844,
      "activations/layer11_attention_weight_max": 31.05956268310547,
      "activations/layer11_attention_weight_min": -31.802785873413086,
      "activations/layer12_attention_weight_max": 16.92481231689453,
      "activations/layer12_attention_weight_min": -25.445138931274414,
      "activations/layer13_attention_weight_max": 33.578086853027344,
      "activations/layer13_attention_weight_min": -25.489431381225586,
      "activations/layer14_attention_weight_max": 32.91068649291992,
      "activations/layer14_attention_weight_min": -29.711124420166016,
      "activations/layer15_attention_weight_max": 30.323566436767578,
      "activations/layer15_attention_weight_min": -29.02691650390625,
      "activations/layer16_attention_weight_max": 30.278718948364258,
      "activations/layer16_attention_weight_min": -28.717912673950195,
      "activations/layer17_attention_weight_max": 47.465728759765625,
      "activations/layer17_attention_weight_min": -40.65716552734375,
      "activations/layer18_attention_weight_max": 42.45450210571289,
      "activations/layer18_attention_weight_min": -35.062950134277344,
      "activations/layer19_attention_weight_max": 23.45384979248047,
      "activations/layer19_attention_weight_min": -22.98660659790039,
      "activations/layer1_attention_weight_max": 15.744531631469727,
      "activations/layer1_attention_weight_min": -14.78830623626709,
      "activations/layer20_attention_weight_max": 21.126787185668945,
      "activations/layer20_attention_weight_min": -22.2369384765625,
      "activations/layer21_attention_weight_max": 33.316200256347656,
      "activations/layer21_attention_weight_min": -23.33937644958496,
      "activations/layer22_attention_weight_max": 28.0534725189209,
      "activations/layer22_attention_weight_min": -25.351221084594727,
      "activations/layer23_attention_weight_max": 36.415977478027344,
      "activations/layer23_attention_weight_min": -25.232421875,
      "activations/layer2_attention_weight_max": 30.988676071166992,
      "activations/layer2_attention_weight_min": -31.0150146484375,
      "activations/layer3_attention_weight_max": 91.34918212890625,
      "activations/layer3_attention_weight_min": -91.99970245361328,
      "activations/layer4_attention_weight_max": 94.37322235107422,
      "activations/layer4_attention_weight_min": -93.2977294921875,
      "activations/layer5_attention_weight_max": 73.71832275390625,
      "activations/layer5_attention_weight_min": -78.05144500732422,
      "activations/layer6_attention_weight_max": 55.033935546875,
      "activations/layer6_attention_weight_min": -54.15253829956055,
      "activations/layer7_attention_weight_max": 68.95388793945312,
      "activations/layer7_attention_weight_min": -66.87035369873047,
      "activations/layer8_attention_weight_max": 44.58430862426758,
      "activations/layer8_attention_weight_min": -45.76234817504883,
      "activations/layer9_attention_weight_max": 45.82430648803711,
      "activations/layer9_attention_weight_min": -44.701900482177734,
      "epoch": 13.27,
      "learning_rate": 6.507310606060605e-05,
      "loss": 2.7794,
      "step": 228400
    },
    {
      "activations/layer0_attention_weight_max": 17.068954467773438,
      "activations/layer0_attention_weight_min": -14.219696044921875,
      "activations/layer10_attention_weight_max": 34.989166259765625,
      "activations/layer10_attention_weight_min": -34.89765930175781,
      "activations/layer11_attention_weight_max": 31.495609283447266,
      "activations/layer11_attention_weight_min": -35.07283020019531,
      "activations/layer12_attention_weight_max": 18.68197250366211,
      "activations/layer12_attention_weight_min": -25.2855281829834,
      "activations/layer13_attention_weight_max": 41.19773864746094,
      "activations/layer13_attention_weight_min": -30.900598526000977,
      "activations/layer14_attention_weight_max": 46.9519157409668,
      "activations/layer14_attention_weight_min": -31.552919387817383,
      "activations/layer15_attention_weight_max": 40.33864212036133,
      "activations/layer15_attention_weight_min": -32.20951843261719,
      "activations/layer16_attention_weight_max": 31.545536041259766,
      "activations/layer16_attention_weight_min": -30.229541778564453,
      "activations/layer17_attention_weight_max": 53.036006927490234,
      "activations/layer17_attention_weight_min": -43.90003204345703,
      "activations/layer18_attention_weight_max": 49.02686309814453,
      "activations/layer18_attention_weight_min": -36.06346893310547,
      "activations/layer19_attention_weight_max": 24.60982322692871,
      "activations/layer19_attention_weight_min": -22.079784393310547,
      "activations/layer1_attention_weight_max": 16.46006965637207,
      "activations/layer1_attention_weight_min": -15.307001113891602,
      "activations/layer20_attention_weight_max": 23.949995040893555,
      "activations/layer20_attention_weight_min": -21.225217819213867,
      "activations/layer21_attention_weight_max": 47.66526794433594,
      "activations/layer21_attention_weight_min": -26.5399227142334,
      "activations/layer22_attention_weight_max": 29.95291519165039,
      "activations/layer22_attention_weight_min": -26.42188262939453,
      "activations/layer23_attention_weight_max": 37.43971252441406,
      "activations/layer23_attention_weight_min": -24.254247665405273,
      "activations/layer2_attention_weight_max": 31.811737060546875,
      "activations/layer2_attention_weight_min": -31.037885665893555,
      "activations/layer3_attention_weight_max": 91.2025146484375,
      "activations/layer3_attention_weight_min": -94.5053482055664,
      "activations/layer4_attention_weight_max": 92.90081787109375,
      "activations/layer4_attention_weight_min": -94.55653381347656,
      "activations/layer5_attention_weight_max": 72.21297454833984,
      "activations/layer5_attention_weight_min": -72.83384704589844,
      "activations/layer6_attention_weight_max": 50.633975982666016,
      "activations/layer6_attention_weight_min": -51.962276458740234,
      "activations/layer7_attention_weight_max": 67.45301818847656,
      "activations/layer7_attention_weight_min": -63.77309799194336,
      "activations/layer8_attention_weight_max": 45.25127410888672,
      "activations/layer8_attention_weight_min": -49.45145034790039,
      "activations/layer9_attention_weight_max": 45.47540283203125,
      "activations/layer9_attention_weight_min": -44.30228805541992,
      "epoch": 13.27,
      "learning_rate": 6.505416666666666e-05,
      "loss": 2.7809,
      "step": 228450
    },
    {
      "activations/layer0_attention_weight_max": 16.60922622680664,
      "activations/layer0_attention_weight_min": -14.4353609085083,
      "activations/layer10_attention_weight_max": 39.18072509765625,
      "activations/layer10_attention_weight_min": -37.88849639892578,
      "activations/layer11_attention_weight_max": 35.760459899902344,
      "activations/layer11_attention_weight_min": -34.221622467041016,
      "activations/layer12_attention_weight_max": 22.148096084594727,
      "activations/layer12_attention_weight_min": -26.478580474853516,
      "activations/layer13_attention_weight_max": 38.695186614990234,
      "activations/layer13_attention_weight_min": -28.63869857788086,
      "activations/layer14_attention_weight_max": 36.06178665161133,
      "activations/layer14_attention_weight_min": -30.0762939453125,
      "activations/layer15_attention_weight_max": 32.69728088378906,
      "activations/layer15_attention_weight_min": -28.94136619567871,
      "activations/layer16_attention_weight_max": 32.28233337402344,
      "activations/layer16_attention_weight_min": -28.81882095336914,
      "activations/layer17_attention_weight_max": 52.791873931884766,
      "activations/layer17_attention_weight_min": -45.46636199951172,
      "activations/layer18_attention_weight_max": 47.3186149597168,
      "activations/layer18_attention_weight_min": -38.092247009277344,
      "activations/layer19_attention_weight_max": 22.87285041809082,
      "activations/layer19_attention_weight_min": -23.867753982543945,
      "activations/layer1_attention_weight_max": 16.011131286621094,
      "activations/layer1_attention_weight_min": -15.011242866516113,
      "activations/layer20_attention_weight_max": 22.705656051635742,
      "activations/layer20_attention_weight_min": -22.05251121520996,
      "activations/layer21_attention_weight_max": 36.60694122314453,
      "activations/layer21_attention_weight_min": -23.068401336669922,
      "activations/layer22_attention_weight_max": 31.719343185424805,
      "activations/layer22_attention_weight_min": -27.6744441986084,
      "activations/layer23_attention_weight_max": 40.35893630981445,
      "activations/layer23_attention_weight_min": -21.751827239990234,
      "activations/layer2_attention_weight_max": 34.995361328125,
      "activations/layer2_attention_weight_min": -35.53033447265625,
      "activations/layer3_attention_weight_max": 96.2835922241211,
      "activations/layer3_attention_weight_min": -97.99344635009766,
      "activations/layer4_attention_weight_max": 98.82269287109375,
      "activations/layer4_attention_weight_min": -91.7048110961914,
      "activations/layer5_attention_weight_max": 72.17005920410156,
      "activations/layer5_attention_weight_min": -75.05815124511719,
      "activations/layer6_attention_weight_max": 53.17241668701172,
      "activations/layer6_attention_weight_min": -51.26694107055664,
      "activations/layer7_attention_weight_max": 69.84709167480469,
      "activations/layer7_attention_weight_min": -67.88294219970703,
      "activations/layer8_attention_weight_max": 48.06682586669922,
      "activations/layer8_attention_weight_min": -52.65631866455078,
      "activations/layer9_attention_weight_max": 51.52600860595703,
      "activations/layer9_attention_weight_min": -49.105037689208984,
      "epoch": 13.28,
      "learning_rate": 6.503522727272727e-05,
      "loss": 2.748,
      "step": 228500
    },
    {
      "activations/layer0_attention_weight_max": 17.07097625732422,
      "activations/layer0_attention_weight_min": -14.438429832458496,
      "activations/layer10_attention_weight_max": 34.38025665283203,
      "activations/layer10_attention_weight_min": -34.436561584472656,
      "activations/layer11_attention_weight_max": 29.74117660522461,
      "activations/layer11_attention_weight_min": -32.19475173950195,
      "activations/layer12_attention_weight_max": 18.220273971557617,
      "activations/layer12_attention_weight_min": -23.10350227355957,
      "activations/layer13_attention_weight_max": 33.41340255737305,
      "activations/layer13_attention_weight_min": -29.46389389038086,
      "activations/layer14_attention_weight_max": 36.25093078613281,
      "activations/layer14_attention_weight_min": -29.583925247192383,
      "activations/layer15_attention_weight_max": 32.157562255859375,
      "activations/layer15_attention_weight_min": -29.6602840423584,
      "activations/layer16_attention_weight_max": 29.45301628112793,
      "activations/layer16_attention_weight_min": -29.329729080200195,
      "activations/layer17_attention_weight_max": 45.7745246887207,
      "activations/layer17_attention_weight_min": -44.04182815551758,
      "activations/layer18_attention_weight_max": 40.69876480102539,
      "activations/layer18_attention_weight_min": -40.868568420410156,
      "activations/layer19_attention_weight_max": 19.96438980102539,
      "activations/layer19_attention_weight_min": -22.11931037902832,
      "activations/layer1_attention_weight_max": 15.836526870727539,
      "activations/layer1_attention_weight_min": -15.536759376525879,
      "activations/layer20_attention_weight_max": 20.549232482910156,
      "activations/layer20_attention_weight_min": -23.39752769470215,
      "activations/layer21_attention_weight_max": 31.58818817138672,
      "activations/layer21_attention_weight_min": -23.454313278198242,
      "activations/layer22_attention_weight_max": 29.2877254486084,
      "activations/layer22_attention_weight_min": -29.402936935424805,
      "activations/layer23_attention_weight_max": 38.51874542236328,
      "activations/layer23_attention_weight_min": -23.91619110107422,
      "activations/layer2_attention_weight_max": 32.1641731262207,
      "activations/layer2_attention_weight_min": -29.376218795776367,
      "activations/layer3_attention_weight_max": 87.19683074951172,
      "activations/layer3_attention_weight_min": -91.93708038330078,
      "activations/layer4_attention_weight_max": 95.82872772216797,
      "activations/layer4_attention_weight_min": -90.87493896484375,
      "activations/layer5_attention_weight_max": 71.26881408691406,
      "activations/layer5_attention_weight_min": -80.8400650024414,
      "activations/layer6_attention_weight_max": 50.1534538269043,
      "activations/layer6_attention_weight_min": -51.08571243286133,
      "activations/layer7_attention_weight_max": 67.70084381103516,
      "activations/layer7_attention_weight_min": -65.33587646484375,
      "activations/layer8_attention_weight_max": 44.095157623291016,
      "activations/layer8_attention_weight_min": -45.88157653808594,
      "activations/layer9_attention_weight_max": 42.93023681640625,
      "activations/layer9_attention_weight_min": -46.015541076660156,
      "epoch": 13.28,
      "learning_rate": 6.501628787878788e-05,
      "loss": 2.7743,
      "step": 228550
    },
    {
      "activations/layer0_attention_weight_max": 15.661514282226562,
      "activations/layer0_attention_weight_min": -14.146651268005371,
      "activations/layer10_attention_weight_max": 35.53285598754883,
      "activations/layer10_attention_weight_min": -34.888877868652344,
      "activations/layer11_attention_weight_max": 31.846763610839844,
      "activations/layer11_attention_weight_min": -34.22813034057617,
      "activations/layer12_attention_weight_max": 18.38161277770996,
      "activations/layer12_attention_weight_min": -26.839981079101562,
      "activations/layer13_attention_weight_max": 37.757667541503906,
      "activations/layer13_attention_weight_min": -28.339893341064453,
      "activations/layer14_attention_weight_max": 42.00583267211914,
      "activations/layer14_attention_weight_min": -32.100730895996094,
      "activations/layer15_attention_weight_max": 35.81287384033203,
      "activations/layer15_attention_weight_min": -29.330062866210938,
      "activations/layer16_attention_weight_max": 33.537906646728516,
      "activations/layer16_attention_weight_min": -30.799007415771484,
      "activations/layer17_attention_weight_max": 52.46467590332031,
      "activations/layer17_attention_weight_min": -45.590484619140625,
      "activations/layer18_attention_weight_max": 48.93294906616211,
      "activations/layer18_attention_weight_min": -41.68058395385742,
      "activations/layer19_attention_weight_max": 23.003812789916992,
      "activations/layer19_attention_weight_min": -25.05819320678711,
      "activations/layer1_attention_weight_max": 16.599760055541992,
      "activations/layer1_attention_weight_min": -14.402591705322266,
      "activations/layer20_attention_weight_max": 22.77581024169922,
      "activations/layer20_attention_weight_min": -28.89415168762207,
      "activations/layer21_attention_weight_max": 39.30699157714844,
      "activations/layer21_attention_weight_min": -25.1146240234375,
      "activations/layer22_attention_weight_max": 33.19969940185547,
      "activations/layer22_attention_weight_min": -27.21197509765625,
      "activations/layer23_attention_weight_max": 37.51567459106445,
      "activations/layer23_attention_weight_min": -26.806106567382812,
      "activations/layer2_attention_weight_max": 30.361080169677734,
      "activations/layer2_attention_weight_min": -29.562131881713867,
      "activations/layer3_attention_weight_max": 94.14857482910156,
      "activations/layer3_attention_weight_min": -98.2804946899414,
      "activations/layer4_attention_weight_max": 95.03255462646484,
      "activations/layer4_attention_weight_min": -94.42610931396484,
      "activations/layer5_attention_weight_max": 71.53657531738281,
      "activations/layer5_attention_weight_min": -82.15868377685547,
      "activations/layer6_attention_weight_max": 55.35829544067383,
      "activations/layer6_attention_weight_min": -54.76205825805664,
      "activations/layer7_attention_weight_max": 73.51551055908203,
      "activations/layer7_attention_weight_min": -67.67890167236328,
      "activations/layer8_attention_weight_max": 44.268310546875,
      "activations/layer8_attention_weight_min": -46.0656623840332,
      "activations/layer9_attention_weight_max": 42.79081344604492,
      "activations/layer9_attention_weight_min": -47.37495422363281,
      "epoch": 13.28,
      "learning_rate": 6.499734848484848e-05,
      "loss": 2.75,
      "step": 228600
    },
    {
      "activations/layer0_attention_weight_max": 16.028038024902344,
      "activations/layer0_attention_weight_min": -13.95234203338623,
      "activations/layer10_attention_weight_max": 34.920536041259766,
      "activations/layer10_attention_weight_min": -35.05204772949219,
      "activations/layer11_attention_weight_max": 31.733884811401367,
      "activations/layer11_attention_weight_min": -31.87428855895996,
      "activations/layer12_attention_weight_max": 17.699995040893555,
      "activations/layer12_attention_weight_min": -25.687137603759766,
      "activations/layer13_attention_weight_max": 36.192440032958984,
      "activations/layer13_attention_weight_min": -28.27945899963379,
      "activations/layer14_attention_weight_max": 42.58150100708008,
      "activations/layer14_attention_weight_min": -30.482988357543945,
      "activations/layer15_attention_weight_max": 42.98554992675781,
      "activations/layer15_attention_weight_min": -30.482553482055664,
      "activations/layer16_attention_weight_max": 32.63778305053711,
      "activations/layer16_attention_weight_min": -31.624507904052734,
      "activations/layer17_attention_weight_max": 51.0322151184082,
      "activations/layer17_attention_weight_min": -44.24713134765625,
      "activations/layer18_attention_weight_max": 46.10763168334961,
      "activations/layer18_attention_weight_min": -38.72004318237305,
      "activations/layer19_attention_weight_max": 21.167612075805664,
      "activations/layer19_attention_weight_min": -20.37337303161621,
      "activations/layer1_attention_weight_max": 15.844931602478027,
      "activations/layer1_attention_weight_min": -14.977006912231445,
      "activations/layer20_attention_weight_max": 20.590702056884766,
      "activations/layer20_attention_weight_min": -23.19689178466797,
      "activations/layer21_attention_weight_max": 36.13072204589844,
      "activations/layer21_attention_weight_min": -21.729537963867188,
      "activations/layer22_attention_weight_max": 32.7738037109375,
      "activations/layer22_attention_weight_min": -25.645877838134766,
      "activations/layer23_attention_weight_max": 31.993240356445312,
      "activations/layer23_attention_weight_min": -24.67230224609375,
      "activations/layer2_attention_weight_max": 31.582195281982422,
      "activations/layer2_attention_weight_min": -29.963764190673828,
      "activations/layer3_attention_weight_max": 93.11217498779297,
      "activations/layer3_attention_weight_min": -96.21408081054688,
      "activations/layer4_attention_weight_max": 92.4154281616211,
      "activations/layer4_attention_weight_min": -92.37230682373047,
      "activations/layer5_attention_weight_max": 71.70492553710938,
      "activations/layer5_attention_weight_min": -79.558837890625,
      "activations/layer6_attention_weight_max": 52.449703216552734,
      "activations/layer6_attention_weight_min": -51.38750457763672,
      "activations/layer7_attention_weight_max": 67.78670501708984,
      "activations/layer7_attention_weight_min": -67.57876586914062,
      "activations/layer8_attention_weight_max": 43.0350456237793,
      "activations/layer8_attention_weight_min": -46.76396560668945,
      "activations/layer9_attention_weight_max": 44.961669921875,
      "activations/layer9_attention_weight_min": -46.58146286010742,
      "epoch": 13.29,
      "learning_rate": 6.497840909090908e-05,
      "loss": 2.7684,
      "step": 228650
    },
    {
      "activations/layer0_attention_weight_max": 15.233390808105469,
      "activations/layer0_attention_weight_min": -16.397205352783203,
      "activations/layer10_attention_weight_max": 38.982975006103516,
      "activations/layer10_attention_weight_min": -38.75546646118164,
      "activations/layer11_attention_weight_max": 33.79664611816406,
      "activations/layer11_attention_weight_min": -35.94658279418945,
      "activations/layer12_attention_weight_max": 17.73249053955078,
      "activations/layer12_attention_weight_min": -25.8519344329834,
      "activations/layer13_attention_weight_max": 44.233726501464844,
      "activations/layer13_attention_weight_min": -29.92776870727539,
      "activations/layer14_attention_weight_max": 34.19211959838867,
      "activations/layer14_attention_weight_min": -30.824731826782227,
      "activations/layer15_attention_weight_max": 33.73918914794922,
      "activations/layer15_attention_weight_min": -30.144145965576172,
      "activations/layer16_attention_weight_max": 30.747034072875977,
      "activations/layer16_attention_weight_min": -30.588497161865234,
      "activations/layer17_attention_weight_max": 50.77174377441406,
      "activations/layer17_attention_weight_min": -44.581424713134766,
      "activations/layer18_attention_weight_max": 44.75313949584961,
      "activations/layer18_attention_weight_min": -40.18108367919922,
      "activations/layer19_attention_weight_max": 22.78095245361328,
      "activations/layer19_attention_weight_min": -22.67391014099121,
      "activations/layer1_attention_weight_max": 16.226964950561523,
      "activations/layer1_attention_weight_min": -14.604625701904297,
      "activations/layer20_attention_weight_max": 22.74483299255371,
      "activations/layer20_attention_weight_min": -20.745677947998047,
      "activations/layer21_attention_weight_max": 39.43537521362305,
      "activations/layer21_attention_weight_min": -24.283695220947266,
      "activations/layer22_attention_weight_max": 29.72561264038086,
      "activations/layer22_attention_weight_min": -28.214305877685547,
      "activations/layer23_attention_weight_max": 38.430721282958984,
      "activations/layer23_attention_weight_min": -25.56136703491211,
      "activations/layer2_attention_weight_max": 31.685529708862305,
      "activations/layer2_attention_weight_min": -31.49395751953125,
      "activations/layer3_attention_weight_max": 93.83848571777344,
      "activations/layer3_attention_weight_min": -95.40709686279297,
      "activations/layer4_attention_weight_max": 101.53666687011719,
      "activations/layer4_attention_weight_min": -104.34999084472656,
      "activations/layer5_attention_weight_max": 72.9190673828125,
      "activations/layer5_attention_weight_min": -79.02029418945312,
      "activations/layer6_attention_weight_max": 56.48974609375,
      "activations/layer6_attention_weight_min": -57.95170211791992,
      "activations/layer7_attention_weight_max": 69.53661346435547,
      "activations/layer7_attention_weight_min": -71.46591186523438,
      "activations/layer8_attention_weight_max": 48.22281265258789,
      "activations/layer8_attention_weight_min": -51.38062286376953,
      "activations/layer9_attention_weight_max": 49.1355094909668,
      "activations/layer9_attention_weight_min": -49.26285934448242,
      "epoch": 13.29,
      "learning_rate": 6.495946969696969e-05,
      "loss": 2.7557,
      "step": 228700
    },
    {
      "activations/layer0_attention_weight_max": 15.390177726745605,
      "activations/layer0_attention_weight_min": -14.109323501586914,
      "activations/layer10_attention_weight_max": 35.95195770263672,
      "activations/layer10_attention_weight_min": -35.518775939941406,
      "activations/layer11_attention_weight_max": 32.858333587646484,
      "activations/layer11_attention_weight_min": -32.1163444519043,
      "activations/layer12_attention_weight_max": 18.153244018554688,
      "activations/layer12_attention_weight_min": -27.740394592285156,
      "activations/layer13_attention_weight_max": 33.39851760864258,
      "activations/layer13_attention_weight_min": -27.83298110961914,
      "activations/layer14_attention_weight_max": 34.74504852294922,
      "activations/layer14_attention_weight_min": -32.57691955566406,
      "activations/layer15_attention_weight_max": 31.404163360595703,
      "activations/layer15_attention_weight_min": -32.324039459228516,
      "activations/layer16_attention_weight_max": 30.01024627685547,
      "activations/layer16_attention_weight_min": -29.907833099365234,
      "activations/layer17_attention_weight_max": 51.39790725708008,
      "activations/layer17_attention_weight_min": -43.886444091796875,
      "activations/layer18_attention_weight_max": 45.18812561035156,
      "activations/layer18_attention_weight_min": -39.03881072998047,
      "activations/layer19_attention_weight_max": 22.13752555847168,
      "activations/layer19_attention_weight_min": -23.29419708251953,
      "activations/layer1_attention_weight_max": 15.390199661254883,
      "activations/layer1_attention_weight_min": -14.760819435119629,
      "activations/layer20_attention_weight_max": 21.072559356689453,
      "activations/layer20_attention_weight_min": -21.409141540527344,
      "activations/layer21_attention_weight_max": 36.5363883972168,
      "activations/layer21_attention_weight_min": -21.440567016601562,
      "activations/layer22_attention_weight_max": 28.198505401611328,
      "activations/layer22_attention_weight_min": -23.92681312561035,
      "activations/layer23_attention_weight_max": 37.246726989746094,
      "activations/layer23_attention_weight_min": -24.62118148803711,
      "activations/layer2_attention_weight_max": 33.9665641784668,
      "activations/layer2_attention_weight_min": -31.954147338867188,
      "activations/layer3_attention_weight_max": 94.0299301147461,
      "activations/layer3_attention_weight_min": -100.30491638183594,
      "activations/layer4_attention_weight_max": 93.53162384033203,
      "activations/layer4_attention_weight_min": -91.9774398803711,
      "activations/layer5_attention_weight_max": 70.20276641845703,
      "activations/layer5_attention_weight_min": -77.21250915527344,
      "activations/layer6_attention_weight_max": 51.829219818115234,
      "activations/layer6_attention_weight_min": -50.54499435424805,
      "activations/layer7_attention_weight_max": 69.1020278930664,
      "activations/layer7_attention_weight_min": -64.49974822998047,
      "activations/layer8_attention_weight_max": 44.3729248046875,
      "activations/layer8_attention_weight_min": -47.71297836303711,
      "activations/layer9_attention_weight_max": 43.50090026855469,
      "activations/layer9_attention_weight_min": -46.82044982910156,
      "epoch": 13.29,
      "learning_rate": 6.49405303030303e-05,
      "loss": 2.7601,
      "step": 228750
    },
    {
      "activations/layer0_attention_weight_max": 15.591438293457031,
      "activations/layer0_attention_weight_min": -13.47965145111084,
      "activations/layer10_attention_weight_max": 37.81495666503906,
      "activations/layer10_attention_weight_min": -41.09806442260742,
      "activations/layer11_attention_weight_max": 35.248050689697266,
      "activations/layer11_attention_weight_min": -35.92261505126953,
      "activations/layer12_attention_weight_max": 18.536283493041992,
      "activations/layer12_attention_weight_min": -25.19367790222168,
      "activations/layer13_attention_weight_max": 38.645198822021484,
      "activations/layer13_attention_weight_min": -29.503671646118164,
      "activations/layer14_attention_weight_max": 42.646568298339844,
      "activations/layer14_attention_weight_min": -29.026784896850586,
      "activations/layer15_attention_weight_max": 36.747615814208984,
      "activations/layer15_attention_weight_min": -28.936813354492188,
      "activations/layer16_attention_weight_max": 29.580015182495117,
      "activations/layer16_attention_weight_min": -30.24829864501953,
      "activations/layer17_attention_weight_max": 50.05259704589844,
      "activations/layer17_attention_weight_min": -45.69813537597656,
      "activations/layer18_attention_weight_max": 45.43528747558594,
      "activations/layer18_attention_weight_min": -37.99354553222656,
      "activations/layer19_attention_weight_max": 25.71480369567871,
      "activations/layer19_attention_weight_min": -21.946914672851562,
      "activations/layer1_attention_weight_max": 16.15691566467285,
      "activations/layer1_attention_weight_min": -14.874788284301758,
      "activations/layer20_attention_weight_max": 22.995065689086914,
      "activations/layer20_attention_weight_min": -24.23739242553711,
      "activations/layer21_attention_weight_max": 34.00886917114258,
      "activations/layer21_attention_weight_min": -22.609182357788086,
      "activations/layer22_attention_weight_max": 30.400239944458008,
      "activations/layer22_attention_weight_min": -26.147329330444336,
      "activations/layer23_attention_weight_max": 39.44091033935547,
      "activations/layer23_attention_weight_min": -24.224790573120117,
      "activations/layer2_attention_weight_max": 32.414459228515625,
      "activations/layer2_attention_weight_min": -31.726882934570312,
      "activations/layer3_attention_weight_max": 93.32648468017578,
      "activations/layer3_attention_weight_min": -94.99996948242188,
      "activations/layer4_attention_weight_max": 103.6414566040039,
      "activations/layer4_attention_weight_min": -102.50439453125,
      "activations/layer5_attention_weight_max": 75.64111328125,
      "activations/layer5_attention_weight_min": -84.37916564941406,
      "activations/layer6_attention_weight_max": 56.507450103759766,
      "activations/layer6_attention_weight_min": -57.22549819946289,
      "activations/layer7_attention_weight_max": 69.26568603515625,
      "activations/layer7_attention_weight_min": -71.40253448486328,
      "activations/layer8_attention_weight_max": 48.16038513183594,
      "activations/layer8_attention_weight_min": -50.81007385253906,
      "activations/layer9_attention_weight_max": 52.90012741088867,
      "activations/layer9_attention_weight_min": -51.285064697265625,
      "epoch": 13.29,
      "learning_rate": 6.49215909090909e-05,
      "loss": 2.7671,
      "step": 228800
    },
    {
      "activations/layer0_attention_weight_max": 15.627025604248047,
      "activations/layer0_attention_weight_min": -13.928911209106445,
      "activations/layer10_attention_weight_max": 40.191524505615234,
      "activations/layer10_attention_weight_min": -38.099857330322266,
      "activations/layer11_attention_weight_max": 35.639041900634766,
      "activations/layer11_attention_weight_min": -35.58811950683594,
      "activations/layer12_attention_weight_max": 19.577037811279297,
      "activations/layer12_attention_weight_min": -24.3369083404541,
      "activations/layer13_attention_weight_max": 37.88814926147461,
      "activations/layer13_attention_weight_min": -33.589935302734375,
      "activations/layer14_attention_weight_max": 34.19047546386719,
      "activations/layer14_attention_weight_min": -31.016857147216797,
      "activations/layer15_attention_weight_max": 32.32576370239258,
      "activations/layer15_attention_weight_min": -30.891130447387695,
      "activations/layer16_attention_weight_max": 31.431629180908203,
      "activations/layer16_attention_weight_min": -29.264455795288086,
      "activations/layer17_attention_weight_max": 53.37233352661133,
      "activations/layer17_attention_weight_min": -44.744014739990234,
      "activations/layer18_attention_weight_max": 45.48699188232422,
      "activations/layer18_attention_weight_min": -39.671173095703125,
      "activations/layer19_attention_weight_max": 21.612314224243164,
      "activations/layer19_attention_weight_min": -21.226634979248047,
      "activations/layer1_attention_weight_max": 18.534101486206055,
      "activations/layer1_attention_weight_min": -15.218039512634277,
      "activations/layer20_attention_weight_max": 22.16747283935547,
      "activations/layer20_attention_weight_min": -21.659894943237305,
      "activations/layer21_attention_weight_max": 39.216243743896484,
      "activations/layer21_attention_weight_min": -23.74777603149414,
      "activations/layer22_attention_weight_max": 30.550594329833984,
      "activations/layer22_attention_weight_min": -26.310102462768555,
      "activations/layer23_attention_weight_max": 37.5986213684082,
      "activations/layer23_attention_weight_min": -22.070188522338867,
      "activations/layer2_attention_weight_max": 30.742673873901367,
      "activations/layer2_attention_weight_min": -30.414783477783203,
      "activations/layer3_attention_weight_max": 91.98652648925781,
      "activations/layer3_attention_weight_min": -91.97473907470703,
      "activations/layer4_attention_weight_max": 94.63511657714844,
      "activations/layer4_attention_weight_min": -95.39097595214844,
      "activations/layer5_attention_weight_max": 76.21715545654297,
      "activations/layer5_attention_weight_min": -81.82829284667969,
      "activations/layer6_attention_weight_max": 54.06829071044922,
      "activations/layer6_attention_weight_min": -52.74653625488281,
      "activations/layer7_attention_weight_max": 71.49486541748047,
      "activations/layer7_attention_weight_min": -71.03095245361328,
      "activations/layer8_attention_weight_max": 46.363861083984375,
      "activations/layer8_attention_weight_min": -50.08452224731445,
      "activations/layer9_attention_weight_max": 44.91206741333008,
      "activations/layer9_attention_weight_min": -48.915313720703125,
      "epoch": 13.3,
      "learning_rate": 6.49026515151515e-05,
      "loss": 2.7686,
      "step": 228850
    },
    {
      "activations/layer0_attention_weight_max": 15.994245529174805,
      "activations/layer0_attention_weight_min": -13.438728332519531,
      "activations/layer10_attention_weight_max": 36.850276947021484,
      "activations/layer10_attention_weight_min": -36.59892654418945,
      "activations/layer11_attention_weight_max": 32.89520263671875,
      "activations/layer11_attention_weight_min": -33.81728744506836,
      "activations/layer12_attention_weight_max": 19.111677169799805,
      "activations/layer12_attention_weight_min": -27.55485725402832,
      "activations/layer13_attention_weight_max": 35.624000549316406,
      "activations/layer13_attention_weight_min": -28.500534057617188,
      "activations/layer14_attention_weight_max": 41.07710647583008,
      "activations/layer14_attention_weight_min": -31.704240798950195,
      "activations/layer15_attention_weight_max": 36.432125091552734,
      "activations/layer15_attention_weight_min": -30.11566162109375,
      "activations/layer16_attention_weight_max": 33.88490676879883,
      "activations/layer16_attention_weight_min": -32.610782623291016,
      "activations/layer17_attention_weight_max": 55.287837982177734,
      "activations/layer17_attention_weight_min": -48.01105499267578,
      "activations/layer18_attention_weight_max": 52.28194808959961,
      "activations/layer18_attention_weight_min": -40.82432556152344,
      "activations/layer19_attention_weight_max": 23.119306564331055,
      "activations/layer19_attention_weight_min": -23.653804779052734,
      "activations/layer1_attention_weight_max": 16.666250228881836,
      "activations/layer1_attention_weight_min": -14.780069351196289,
      "activations/layer20_attention_weight_max": 21.800146102905273,
      "activations/layer20_attention_weight_min": -21.681806564331055,
      "activations/layer21_attention_weight_max": 38.844635009765625,
      "activations/layer21_attention_weight_min": -24.08527183532715,
      "activations/layer22_attention_weight_max": 34.59740447998047,
      "activations/layer22_attention_weight_min": -24.78412628173828,
      "activations/layer23_attention_weight_max": 40.570186614990234,
      "activations/layer23_attention_weight_min": -24.962371826171875,
      "activations/layer2_attention_weight_max": 32.23169708251953,
      "activations/layer2_attention_weight_min": -30.961406707763672,
      "activations/layer3_attention_weight_max": 96.4852523803711,
      "activations/layer3_attention_weight_min": -96.23208618164062,
      "activations/layer4_attention_weight_max": 95.59565734863281,
      "activations/layer4_attention_weight_min": -96.39468383789062,
      "activations/layer5_attention_weight_max": 71.2388687133789,
      "activations/layer5_attention_weight_min": -78.68952941894531,
      "activations/layer6_attention_weight_max": 52.36743927001953,
      "activations/layer6_attention_weight_min": -51.856361389160156,
      "activations/layer7_attention_weight_max": 63.91583251953125,
      "activations/layer7_attention_weight_min": -63.81967544555664,
      "activations/layer8_attention_weight_max": 46.148860931396484,
      "activations/layer8_attention_weight_min": -46.57120132446289,
      "activations/layer9_attention_weight_max": 42.74518585205078,
      "activations/layer9_attention_weight_min": -46.674617767333984,
      "epoch": 13.3,
      "learning_rate": 6.48837121212121e-05,
      "loss": 2.7737,
      "step": 228900
    },
    {
      "activations/layer0_attention_weight_max": 16.596094131469727,
      "activations/layer0_attention_weight_min": -13.491495132446289,
      "activations/layer10_attention_weight_max": 37.48158645629883,
      "activations/layer10_attention_weight_min": -35.54998016357422,
      "activations/layer11_attention_weight_max": 33.28260040283203,
      "activations/layer11_attention_weight_min": -32.55158233642578,
      "activations/layer12_attention_weight_max": 19.161462783813477,
      "activations/layer12_attention_weight_min": -24.096817016601562,
      "activations/layer13_attention_weight_max": 38.883323669433594,
      "activations/layer13_attention_weight_min": -29.952064514160156,
      "activations/layer14_attention_weight_max": 36.528865814208984,
      "activations/layer14_attention_weight_min": -30.724700927734375,
      "activations/layer15_attention_weight_max": 35.45962905883789,
      "activations/layer15_attention_weight_min": -30.278905868530273,
      "activations/layer16_attention_weight_max": 32.26387023925781,
      "activations/layer16_attention_weight_min": -29.514493942260742,
      "activations/layer17_attention_weight_max": 50.74387741088867,
      "activations/layer17_attention_weight_min": -47.042701721191406,
      "activations/layer18_attention_weight_max": 48.6026725769043,
      "activations/layer18_attention_weight_min": -40.113590240478516,
      "activations/layer19_attention_weight_max": 21.701749801635742,
      "activations/layer19_attention_weight_min": -23.590723037719727,
      "activations/layer1_attention_weight_max": 16.89259910583496,
      "activations/layer1_attention_weight_min": -13.990283012390137,
      "activations/layer20_attention_weight_max": 23.773412704467773,
      "activations/layer20_attention_weight_min": -23.73590660095215,
      "activations/layer21_attention_weight_max": 40.6976203918457,
      "activations/layer21_attention_weight_min": -25.223602294921875,
      "activations/layer22_attention_weight_max": 31.53317642211914,
      "activations/layer22_attention_weight_min": -28.056562423706055,
      "activations/layer23_attention_weight_max": 38.02793884277344,
      "activations/layer23_attention_weight_min": -21.62615203857422,
      "activations/layer2_attention_weight_max": 31.352767944335938,
      "activations/layer2_attention_weight_min": -29.792102813720703,
      "activations/layer3_attention_weight_max": 90.49874877929688,
      "activations/layer3_attention_weight_min": -90.3255844116211,
      "activations/layer4_attention_weight_max": 92.95497131347656,
      "activations/layer4_attention_weight_min": -90.37226104736328,
      "activations/layer5_attention_weight_max": 73.76766204833984,
      "activations/layer5_attention_weight_min": -74.60673522949219,
      "activations/layer6_attention_weight_max": 50.88397979736328,
      "activations/layer6_attention_weight_min": -52.14803695678711,
      "activations/layer7_attention_weight_max": 69.01463317871094,
      "activations/layer7_attention_weight_min": -64.22294616699219,
      "activations/layer8_attention_weight_max": 46.88597869873047,
      "activations/layer8_attention_weight_min": -48.13973617553711,
      "activations/layer9_attention_weight_max": 43.76162338256836,
      "activations/layer9_attention_weight_min": -45.64353561401367,
      "epoch": 13.3,
      "learning_rate": 6.486477272727272e-05,
      "loss": 2.7637,
      "step": 228950
    },
    {
      "activations/layer0_attention_weight_max": 16.993488311767578,
      "activations/layer0_attention_weight_min": -13.515117645263672,
      "activations/layer10_attention_weight_max": 37.45317077636719,
      "activations/layer10_attention_weight_min": -36.84347152709961,
      "activations/layer11_attention_weight_max": 32.48217010498047,
      "activations/layer11_attention_weight_min": -33.0988883972168,
      "activations/layer12_attention_weight_max": 18.70560646057129,
      "activations/layer12_attention_weight_min": -25.980688095092773,
      "activations/layer13_attention_weight_max": 41.47428512573242,
      "activations/layer13_attention_weight_min": -27.682151794433594,
      "activations/layer14_attention_weight_max": 38.13046646118164,
      "activations/layer14_attention_weight_min": -31.80352020263672,
      "activations/layer15_attention_weight_max": 33.897682189941406,
      "activations/layer15_attention_weight_min": -30.758451461791992,
      "activations/layer16_attention_weight_max": 33.16497039794922,
      "activations/layer16_attention_weight_min": -31.33412742614746,
      "activations/layer17_attention_weight_max": 55.84952163696289,
      "activations/layer17_attention_weight_min": -45.662384033203125,
      "activations/layer18_attention_weight_max": 47.792259216308594,
      "activations/layer18_attention_weight_min": -39.78761291503906,
      "activations/layer19_attention_weight_max": 24.32276725769043,
      "activations/layer19_attention_weight_min": -21.54087257385254,
      "activations/layer1_attention_weight_max": 16.27259063720703,
      "activations/layer1_attention_weight_min": -16.357297897338867,
      "activations/layer20_attention_weight_max": 23.106948852539062,
      "activations/layer20_attention_weight_min": -20.738996505737305,
      "activations/layer21_attention_weight_max": 40.545631408691406,
      "activations/layer21_attention_weight_min": -23.769582748413086,
      "activations/layer22_attention_weight_max": 32.894126892089844,
      "activations/layer22_attention_weight_min": -25.64525032043457,
      "activations/layer23_attention_weight_max": 37.81087112426758,
      "activations/layer23_attention_weight_min": -24.164180755615234,
      "activations/layer2_attention_weight_max": 31.599136352539062,
      "activations/layer2_attention_weight_min": -29.756046295166016,
      "activations/layer3_attention_weight_max": 94.1873779296875,
      "activations/layer3_attention_weight_min": -92.15491485595703,
      "activations/layer4_attention_weight_max": 94.30728149414062,
      "activations/layer4_attention_weight_min": -93.55794525146484,
      "activations/layer5_attention_weight_max": 70.09884643554688,
      "activations/layer5_attention_weight_min": -72.71862030029297,
      "activations/layer6_attention_weight_max": 54.04227828979492,
      "activations/layer6_attention_weight_min": -50.777408599853516,
      "activations/layer7_attention_weight_max": 72.47830963134766,
      "activations/layer7_attention_weight_min": -62.961429595947266,
      "activations/layer8_attention_weight_max": 46.47704315185547,
      "activations/layer8_attention_weight_min": -47.75978469848633,
      "activations/layer9_attention_weight_max": 54.43194580078125,
      "activations/layer9_attention_weight_min": -45.402862548828125,
      "epoch": 13.31,
      "learning_rate": 6.484583333333332e-05,
      "loss": 2.7587,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.251,
      "eval_samples_per_second": 520.42,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.251,
      "eval_openwebtext_samples_per_second": 520.42,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.6365,
      "eval_wikitext_samples_per_second": 278.651,
      "step": 229000
    },
    {
      "epoch": 13.31,
      "eval_lambada_loss": 2.5234375,
      "eval_lambada_ppl": 12.471393479493909,
      "eval_lambada_runtime": 9.3254,
      "eval_lambada_samples_per_second": 522.123,
      "step": 229000
    },
    {
      "activations/layer0_attention_weight_max": 15.637832641601562,
      "activations/layer0_attention_weight_min": -14.080255508422852,
      "activations/layer10_attention_weight_max": 38.4766845703125,
      "activations/layer10_attention_weight_min": -36.70376968383789,
      "activations/layer11_attention_weight_max": 32.391357421875,
      "activations/layer11_attention_weight_min": -33.02307891845703,
      "activations/layer12_attention_weight_max": 19.216283798217773,
      "activations/layer12_attention_weight_min": -28.695322036743164,
      "activations/layer13_attention_weight_max": 39.899837493896484,
      "activations/layer13_attention_weight_min": -31.373462677001953,
      "activations/layer14_attention_weight_max": 38.92899703979492,
      "activations/layer14_attention_weight_min": -32.0150032043457,
      "activations/layer15_attention_weight_max": 35.29647445678711,
      "activations/layer15_attention_weight_min": -32.64375686645508,
      "activations/layer16_attention_weight_max": 34.25748062133789,
      "activations/layer16_attention_weight_min": -31.408580780029297,
      "activations/layer17_attention_weight_max": 56.70564270019531,
      "activations/layer17_attention_weight_min": -45.711143493652344,
      "activations/layer18_attention_weight_max": 51.3869743347168,
      "activations/layer18_attention_weight_min": -40.862579345703125,
      "activations/layer19_attention_weight_max": 23.6106014251709,
      "activations/layer19_attention_weight_min": -24.283292770385742,
      "activations/layer1_attention_weight_max": 17.118310928344727,
      "activations/layer1_attention_weight_min": -14.040290832519531,
      "activations/layer20_attention_weight_max": 22.94295883178711,
      "activations/layer20_attention_weight_min": -23.489437103271484,
      "activations/layer21_attention_weight_max": 37.909095764160156,
      "activations/layer21_attention_weight_min": -23.93156623840332,
      "activations/layer22_attention_weight_max": 31.28182601928711,
      "activations/layer22_attention_weight_min": -26.19916343688965,
      "activations/layer23_attention_weight_max": 36.528236389160156,
      "activations/layer23_attention_weight_min": -27.450881958007812,
      "activations/layer2_attention_weight_max": 30.935684204101562,
      "activations/layer2_attention_weight_min": -31.098392486572266,
      "activations/layer3_attention_weight_max": 94.77117919921875,
      "activations/layer3_attention_weight_min": -96.23262023925781,
      "activations/layer4_attention_weight_max": 94.53315734863281,
      "activations/layer4_attention_weight_min": -95.89817810058594,
      "activations/layer5_attention_weight_max": 71.9704360961914,
      "activations/layer5_attention_weight_min": -79.88947296142578,
      "activations/layer6_attention_weight_max": 50.4080696105957,
      "activations/layer6_attention_weight_min": -54.52674865722656,
      "activations/layer7_attention_weight_max": 66.46314239501953,
      "activations/layer7_attention_weight_min": -69.74115753173828,
      "activations/layer8_attention_weight_max": 44.737979888916016,
      "activations/layer8_attention_weight_min": -48.866905212402344,
      "activations/layer9_attention_weight_max": 45.883907318115234,
      "activations/layer9_attention_weight_min": -49.88960266113281,
      "epoch": 13.31,
      "learning_rate": 6.482689393939394e-05,
      "loss": 2.7719,
      "step": 229050
    },
    {
      "activations/layer0_attention_weight_max": 15.433154106140137,
      "activations/layer0_attention_weight_min": -13.543455123901367,
      "activations/layer10_attention_weight_max": 37.11225128173828,
      "activations/layer10_attention_weight_min": -37.628089904785156,
      "activations/layer11_attention_weight_max": 33.357177734375,
      "activations/layer11_attention_weight_min": -34.23754119873047,
      "activations/layer12_attention_weight_max": 18.005525588989258,
      "activations/layer12_attention_weight_min": -27.51400375366211,
      "activations/layer13_attention_weight_max": 38.15865707397461,
      "activations/layer13_attention_weight_min": -28.48067855834961,
      "activations/layer14_attention_weight_max": 36.28571701049805,
      "activations/layer14_attention_weight_min": -30.901386260986328,
      "activations/layer15_attention_weight_max": 33.10211181640625,
      "activations/layer15_attention_weight_min": -30.799354553222656,
      "activations/layer16_attention_weight_max": 30.870548248291016,
      "activations/layer16_attention_weight_min": -30.976579666137695,
      "activations/layer17_attention_weight_max": 53.20490646362305,
      "activations/layer17_attention_weight_min": -45.60588836669922,
      "activations/layer18_attention_weight_max": 46.88076400756836,
      "activations/layer18_attention_weight_min": -39.207275390625,
      "activations/layer19_attention_weight_max": 19.797462463378906,
      "activations/layer19_attention_weight_min": -21.33234977722168,
      "activations/layer1_attention_weight_max": 15.932881355285645,
      "activations/layer1_attention_weight_min": -15.149762153625488,
      "activations/layer20_attention_weight_max": 21.370569229125977,
      "activations/layer20_attention_weight_min": -21.344161987304688,
      "activations/layer21_attention_weight_max": 34.0341796875,
      "activations/layer21_attention_weight_min": -24.297964096069336,
      "activations/layer22_attention_weight_max": 31.20335578918457,
      "activations/layer22_attention_weight_min": -26.2506160736084,
      "activations/layer23_attention_weight_max": 34.67430114746094,
      "activations/layer23_attention_weight_min": -26.041332244873047,
      "activations/layer2_attention_weight_max": 31.90392303466797,
      "activations/layer2_attention_weight_min": -31.215627670288086,
      "activations/layer3_attention_weight_max": 96.82726287841797,
      "activations/layer3_attention_weight_min": -102.76048278808594,
      "activations/layer4_attention_weight_max": 98.9450912475586,
      "activations/layer4_attention_weight_min": -99.21100616455078,
      "activations/layer5_attention_weight_max": 75.41450500488281,
      "activations/layer5_attention_weight_min": -82.72576904296875,
      "activations/layer6_attention_weight_max": 57.01295471191406,
      "activations/layer6_attention_weight_min": -53.57947540283203,
      "activations/layer7_attention_weight_max": 72.40870666503906,
      "activations/layer7_attention_weight_min": -65.10941314697266,
      "activations/layer8_attention_weight_max": 48.920597076416016,
      "activations/layer8_attention_weight_min": -50.92110824584961,
      "activations/layer9_attention_weight_max": 45.67103958129883,
      "activations/layer9_attention_weight_min": -46.68606948852539,
      "epoch": 13.31,
      "learning_rate": 6.480795454545454e-05,
      "loss": 2.7693,
      "step": 229100
    },
    {
      "activations/layer0_attention_weight_max": 15.252619743347168,
      "activations/layer0_attention_weight_min": -13.764259338378906,
      "activations/layer10_attention_weight_max": 38.84563064575195,
      "activations/layer10_attention_weight_min": -35.8172492980957,
      "activations/layer11_attention_weight_max": 32.71439743041992,
      "activations/layer11_attention_weight_min": -33.250946044921875,
      "activations/layer12_attention_weight_max": 21.754058837890625,
      "activations/layer12_attention_weight_min": -23.58397102355957,
      "activations/layer13_attention_weight_max": 35.66502380371094,
      "activations/layer13_attention_weight_min": -28.56404685974121,
      "activations/layer14_attention_weight_max": 38.775875091552734,
      "activations/layer14_attention_weight_min": -29.888978958129883,
      "activations/layer15_attention_weight_max": 36.06203079223633,
      "activations/layer15_attention_weight_min": -28.89311981201172,
      "activations/layer16_attention_weight_max": 33.25386428833008,
      "activations/layer16_attention_weight_min": -29.961498260498047,
      "activations/layer17_attention_weight_max": 56.73429489135742,
      "activations/layer17_attention_weight_min": -45.26418685913086,
      "activations/layer18_attention_weight_max": 53.27168273925781,
      "activations/layer18_attention_weight_min": -37.82973098754883,
      "activations/layer19_attention_weight_max": 22.392484664916992,
      "activations/layer19_attention_weight_min": -22.584402084350586,
      "activations/layer1_attention_weight_max": 16.622207641601562,
      "activations/layer1_attention_weight_min": -14.199421882629395,
      "activations/layer20_attention_weight_max": 22.85697364807129,
      "activations/layer20_attention_weight_min": -22.355993270874023,
      "activations/layer21_attention_weight_max": 33.6700325012207,
      "activations/layer21_attention_weight_min": -21.597484588623047,
      "activations/layer22_attention_weight_max": 32.12424087524414,
      "activations/layer22_attention_weight_min": -30.60538673400879,
      "activations/layer23_attention_weight_max": 40.011070251464844,
      "activations/layer23_attention_weight_min": -26.373106002807617,
      "activations/layer2_attention_weight_max": 32.07884216308594,
      "activations/layer2_attention_weight_min": -32.188140869140625,
      "activations/layer3_attention_weight_max": 94.15921783447266,
      "activations/layer3_attention_weight_min": -98.38885498046875,
      "activations/layer4_attention_weight_max": 100.3609390258789,
      "activations/layer4_attention_weight_min": -101.23013305664062,
      "activations/layer5_attention_weight_max": 73.5395736694336,
      "activations/layer5_attention_weight_min": -83.31819152832031,
      "activations/layer6_attention_weight_max": 53.201683044433594,
      "activations/layer6_attention_weight_min": -54.53386688232422,
      "activations/layer7_attention_weight_max": 69.30781555175781,
      "activations/layer7_attention_weight_min": -67.85281372070312,
      "activations/layer8_attention_weight_max": 50.07298278808594,
      "activations/layer8_attention_weight_min": -50.59415054321289,
      "activations/layer9_attention_weight_max": 50.0898323059082,
      "activations/layer9_attention_weight_min": -49.06830978393555,
      "epoch": 13.31,
      "learning_rate": 6.478901515151515e-05,
      "loss": 2.7607,
      "step": 229150
    },
    {
      "activations/layer0_attention_weight_max": 15.867151260375977,
      "activations/layer0_attention_weight_min": -13.502559661865234,
      "activations/layer10_attention_weight_max": 37.691795349121094,
      "activations/layer10_attention_weight_min": -35.35506057739258,
      "activations/layer11_attention_weight_max": 33.35596466064453,
      "activations/layer11_attention_weight_min": -32.70029830932617,
      "activations/layer12_attention_weight_max": 19.919431686401367,
      "activations/layer12_attention_weight_min": -24.503873825073242,
      "activations/layer13_attention_weight_max": 37.5849723815918,
      "activations/layer13_attention_weight_min": -29.90428352355957,
      "activations/layer14_attention_weight_max": 35.50739288330078,
      "activations/layer14_attention_weight_min": -31.36483383178711,
      "activations/layer15_attention_weight_max": 32.802433013916016,
      "activations/layer15_attention_weight_min": -29.157489776611328,
      "activations/layer16_attention_weight_max": 32.24138259887695,
      "activations/layer16_attention_weight_min": -30.21520233154297,
      "activations/layer17_attention_weight_max": 50.118804931640625,
      "activations/layer17_attention_weight_min": -43.99245071411133,
      "activations/layer18_attention_weight_max": 45.31719207763672,
      "activations/layer18_attention_weight_min": -37.35068893432617,
      "activations/layer19_attention_weight_max": 22.824316024780273,
      "activations/layer19_attention_weight_min": -21.592304229736328,
      "activations/layer1_attention_weight_max": 16.55050277709961,
      "activations/layer1_attention_weight_min": -16.31338882446289,
      "activations/layer20_attention_weight_max": 21.39560890197754,
      "activations/layer20_attention_weight_min": -27.937986373901367,
      "activations/layer21_attention_weight_max": 39.21235656738281,
      "activations/layer21_attention_weight_min": -24.284543991088867,
      "activations/layer22_attention_weight_max": 28.066186904907227,
      "activations/layer22_attention_weight_min": -25.220491409301758,
      "activations/layer23_attention_weight_max": 38.352378845214844,
      "activations/layer23_attention_weight_min": -26.14198112487793,
      "activations/layer2_attention_weight_max": 32.55138397216797,
      "activations/layer2_attention_weight_min": -30.997726440429688,
      "activations/layer3_attention_weight_max": 92.62967681884766,
      "activations/layer3_attention_weight_min": -98.19942474365234,
      "activations/layer4_attention_weight_max": 96.0068588256836,
      "activations/layer4_attention_weight_min": -94.28894805908203,
      "activations/layer5_attention_weight_max": 71.9024429321289,
      "activations/layer5_attention_weight_min": -77.09590911865234,
      "activations/layer6_attention_weight_max": 52.27983856201172,
      "activations/layer6_attention_weight_min": -51.646934509277344,
      "activations/layer7_attention_weight_max": 67.4090576171875,
      "activations/layer7_attention_weight_min": -66.09770965576172,
      "activations/layer8_attention_weight_max": 46.137996673583984,
      "activations/layer8_attention_weight_min": -48.34033203125,
      "activations/layer9_attention_weight_max": 43.47921371459961,
      "activations/layer9_attention_weight_min": -45.7497673034668,
      "epoch": 13.32,
      "learning_rate": 6.477007575757575e-05,
      "loss": 2.7597,
      "step": 229200
    },
    {
      "activations/layer0_attention_weight_max": 15.561216354370117,
      "activations/layer0_attention_weight_min": -13.7764253616333,
      "activations/layer10_attention_weight_max": 34.80610275268555,
      "activations/layer10_attention_weight_min": -35.58033752441406,
      "activations/layer11_attention_weight_max": 31.549732208251953,
      "activations/layer11_attention_weight_min": -34.20599365234375,
      "activations/layer12_attention_weight_max": 20.001514434814453,
      "activations/layer12_attention_weight_min": -23.367338180541992,
      "activations/layer13_attention_weight_max": 33.686248779296875,
      "activations/layer13_attention_weight_min": -27.2262020111084,
      "activations/layer14_attention_weight_max": 34.2744026184082,
      "activations/layer14_attention_weight_min": -29.31185531616211,
      "activations/layer15_attention_weight_max": 30.589597702026367,
      "activations/layer15_attention_weight_min": -28.686279296875,
      "activations/layer16_attention_weight_max": 30.880273818969727,
      "activations/layer16_attention_weight_min": -29.214696884155273,
      "activations/layer17_attention_weight_max": 48.581085205078125,
      "activations/layer17_attention_weight_min": -45.36647415161133,
      "activations/layer18_attention_weight_max": 44.62891387939453,
      "activations/layer18_attention_weight_min": -36.69292449951172,
      "activations/layer19_attention_weight_max": 21.923192977905273,
      "activations/layer19_attention_weight_min": -21.761770248413086,
      "activations/layer1_attention_weight_max": 17.08269500732422,
      "activations/layer1_attention_weight_min": -16.753328323364258,
      "activations/layer20_attention_weight_max": 22.262710571289062,
      "activations/layer20_attention_weight_min": -22.20993995666504,
      "activations/layer21_attention_weight_max": 34.42455291748047,
      "activations/layer21_attention_weight_min": -22.451152801513672,
      "activations/layer22_attention_weight_max": 28.934120178222656,
      "activations/layer22_attention_weight_min": -26.420698165893555,
      "activations/layer23_attention_weight_max": 36.22868347167969,
      "activations/layer23_attention_weight_min": -24.996322631835938,
      "activations/layer2_attention_weight_max": 33.66583251953125,
      "activations/layer2_attention_weight_min": -31.167537689208984,
      "activations/layer3_attention_weight_max": 96.24462127685547,
      "activations/layer3_attention_weight_min": -98.0061264038086,
      "activations/layer4_attention_weight_max": 97.49687957763672,
      "activations/layer4_attention_weight_min": -95.4697265625,
      "activations/layer5_attention_weight_max": 70.52491760253906,
      "activations/layer5_attention_weight_min": -76.6964111328125,
      "activations/layer6_attention_weight_max": 50.670650482177734,
      "activations/layer6_attention_weight_min": -52.212894439697266,
      "activations/layer7_attention_weight_max": 66.48582458496094,
      "activations/layer7_attention_weight_min": -64.07746887207031,
      "activations/layer8_attention_weight_max": 44.7278938293457,
      "activations/layer8_attention_weight_min": -46.06467819213867,
      "activations/layer9_attention_weight_max": 46.01008224487305,
      "activations/layer9_attention_weight_min": -46.26825714111328,
      "epoch": 13.32,
      "learning_rate": 6.475113636363636e-05,
      "loss": 2.7617,
      "step": 229250
    },
    {
      "activations/layer0_attention_weight_max": 15.571944236755371,
      "activations/layer0_attention_weight_min": -13.921754837036133,
      "activations/layer10_attention_weight_max": 36.015140533447266,
      "activations/layer10_attention_weight_min": -36.42584991455078,
      "activations/layer11_attention_weight_max": 32.087371826171875,
      "activations/layer11_attention_weight_min": -31.972972869873047,
      "activations/layer12_attention_weight_max": 18.368209838867188,
      "activations/layer12_attention_weight_min": -24.388994216918945,
      "activations/layer13_attention_weight_max": 39.64079284667969,
      "activations/layer13_attention_weight_min": -30.45041847229004,
      "activations/layer14_attention_weight_max": 33.737606048583984,
      "activations/layer14_attention_weight_min": -28.785429000854492,
      "activations/layer15_attention_weight_max": 30.388011932373047,
      "activations/layer15_attention_weight_min": -28.105892181396484,
      "activations/layer16_attention_weight_max": 27.98212432861328,
      "activations/layer16_attention_weight_min": -27.657405853271484,
      "activations/layer17_attention_weight_max": 47.04462432861328,
      "activations/layer17_attention_weight_min": -39.5041389465332,
      "activations/layer18_attention_weight_max": 41.01298141479492,
      "activations/layer18_attention_weight_min": -38.35689163208008,
      "activations/layer19_attention_weight_max": 21.619041442871094,
      "activations/layer19_attention_weight_min": -21.848018646240234,
      "activations/layer1_attention_weight_max": 18.261587142944336,
      "activations/layer1_attention_weight_min": -15.087367057800293,
      "activations/layer20_attention_weight_max": 22.803678512573242,
      "activations/layer20_attention_weight_min": -21.539875030517578,
      "activations/layer21_attention_weight_max": 29.80974006652832,
      "activations/layer21_attention_weight_min": -22.993913650512695,
      "activations/layer22_attention_weight_max": 30.89296531677246,
      "activations/layer22_attention_weight_min": -25.361474990844727,
      "activations/layer23_attention_weight_max": 34.56235122680664,
      "activations/layer23_attention_weight_min": -24.187490463256836,
      "activations/layer2_attention_weight_max": 33.07100296020508,
      "activations/layer2_attention_weight_min": -31.217039108276367,
      "activations/layer3_attention_weight_max": 93.7529296875,
      "activations/layer3_attention_weight_min": -97.42533111572266,
      "activations/layer4_attention_weight_max": 97.96652221679688,
      "activations/layer4_attention_weight_min": -92.8799819946289,
      "activations/layer5_attention_weight_max": 73.32550048828125,
      "activations/layer5_attention_weight_min": -79.27986907958984,
      "activations/layer6_attention_weight_max": 50.04976272583008,
      "activations/layer6_attention_weight_min": -49.73019027709961,
      "activations/layer7_attention_weight_max": 66.72938537597656,
      "activations/layer7_attention_weight_min": -72.76852416992188,
      "activations/layer8_attention_weight_max": 43.743900299072266,
      "activations/layer8_attention_weight_min": -45.05384063720703,
      "activations/layer9_attention_weight_max": 44.647220611572266,
      "activations/layer9_attention_weight_min": -45.48250198364258,
      "epoch": 13.32,
      "learning_rate": 6.473219696969697e-05,
      "loss": 2.7597,
      "step": 229300
    },
    {
      "activations/layer0_attention_weight_max": 16.24056625366211,
      "activations/layer0_attention_weight_min": -13.898423194885254,
      "activations/layer10_attention_weight_max": 47.68405532836914,
      "activations/layer10_attention_weight_min": -44.260215759277344,
      "activations/layer11_attention_weight_max": 40.80571365356445,
      "activations/layer11_attention_weight_min": -39.26530838012695,
      "activations/layer12_attention_weight_max": 18.603403091430664,
      "activations/layer12_attention_weight_min": -25.088529586791992,
      "activations/layer13_attention_weight_max": 43.24687957763672,
      "activations/layer13_attention_weight_min": -34.6779899597168,
      "activations/layer14_attention_weight_max": 47.81867980957031,
      "activations/layer14_attention_weight_min": -30.50421714782715,
      "activations/layer15_attention_weight_max": 40.89895248413086,
      "activations/layer15_attention_weight_min": -28.385202407836914,
      "activations/layer16_attention_weight_max": 34.25749206542969,
      "activations/layer16_attention_weight_min": -29.840070724487305,
      "activations/layer17_attention_weight_max": 56.2266731262207,
      "activations/layer17_attention_weight_min": -46.942970275878906,
      "activations/layer18_attention_weight_max": 49.225643157958984,
      "activations/layer18_attention_weight_min": -40.9905891418457,
      "activations/layer19_attention_weight_max": 24.75538444519043,
      "activations/layer19_attention_weight_min": -23.654584884643555,
      "activations/layer1_attention_weight_max": 17.33150291442871,
      "activations/layer1_attention_weight_min": -14.962942123413086,
      "activations/layer20_attention_weight_max": 27.751874923706055,
      "activations/layer20_attention_weight_min": -23.54327964782715,
      "activations/layer21_attention_weight_max": 43.64397048950195,
      "activations/layer21_attention_weight_min": -26.436906814575195,
      "activations/layer22_attention_weight_max": 37.11982727050781,
      "activations/layer22_attention_weight_min": -25.745319366455078,
      "activations/layer23_attention_weight_max": 41.470001220703125,
      "activations/layer23_attention_weight_min": -24.73768424987793,
      "activations/layer2_attention_weight_max": 34.155853271484375,
      "activations/layer2_attention_weight_min": -32.903778076171875,
      "activations/layer3_attention_weight_max": 101.11722564697266,
      "activations/layer3_attention_weight_min": -103.9683609008789,
      "activations/layer4_attention_weight_max": 101.42234802246094,
      "activations/layer4_attention_weight_min": -96.53532409667969,
      "activations/layer5_attention_weight_max": 72.11029815673828,
      "activations/layer5_attention_weight_min": -80.20893859863281,
      "activations/layer6_attention_weight_max": 55.12807083129883,
      "activations/layer6_attention_weight_min": -55.273597717285156,
      "activations/layer7_attention_weight_max": 82.20487213134766,
      "activations/layer7_attention_weight_min": -78.48380279541016,
      "activations/layer8_attention_weight_max": 58.6014404296875,
      "activations/layer8_attention_weight_min": -56.14455032348633,
      "activations/layer9_attention_weight_max": 62.379676818847656,
      "activations/layer9_attention_weight_min": -54.67336654663086,
      "epoch": 13.33,
      "learning_rate": 6.471325757575757e-05,
      "loss": 2.7563,
      "step": 229350
    },
    {
      "activations/layer0_attention_weight_max": 16.733928680419922,
      "activations/layer0_attention_weight_min": -13.828507423400879,
      "activations/layer10_attention_weight_max": 38.32381057739258,
      "activations/layer10_attention_weight_min": -36.62653732299805,
      "activations/layer11_attention_weight_max": 34.08221435546875,
      "activations/layer11_attention_weight_min": -33.642696380615234,
      "activations/layer12_attention_weight_max": 18.755943298339844,
      "activations/layer12_attention_weight_min": -23.338945388793945,
      "activations/layer13_attention_weight_max": 41.04602813720703,
      "activations/layer13_attention_weight_min": -28.645248413085938,
      "activations/layer14_attention_weight_max": 37.99654769897461,
      "activations/layer14_attention_weight_min": -29.06121826171875,
      "activations/layer15_attention_weight_max": 33.143314361572266,
      "activations/layer15_attention_weight_min": -28.240089416503906,
      "activations/layer16_attention_weight_max": 34.77857971191406,
      "activations/layer16_attention_weight_min": -31.596086502075195,
      "activations/layer17_attention_weight_max": 55.34212875366211,
      "activations/layer17_attention_weight_min": -44.881980895996094,
      "activations/layer18_attention_weight_max": 51.30112075805664,
      "activations/layer18_attention_weight_min": -38.60089111328125,
      "activations/layer19_attention_weight_max": 26.38492774963379,
      "activations/layer19_attention_weight_min": -22.779436111450195,
      "activations/layer1_attention_weight_max": 16.32329750061035,
      "activations/layer1_attention_weight_min": -16.390573501586914,
      "activations/layer20_attention_weight_max": 24.052030563354492,
      "activations/layer20_attention_weight_min": -23.428218841552734,
      "activations/layer21_attention_weight_max": 38.039363861083984,
      "activations/layer21_attention_weight_min": -19.683671951293945,
      "activations/layer22_attention_weight_max": 30.937171936035156,
      "activations/layer22_attention_weight_min": -25.09912872314453,
      "activations/layer23_attention_weight_max": 40.46413803100586,
      "activations/layer23_attention_weight_min": -23.971168518066406,
      "activations/layer2_attention_weight_max": 31.867855072021484,
      "activations/layer2_attention_weight_min": -31.073287963867188,
      "activations/layer3_attention_weight_max": 93.5498275756836,
      "activations/layer3_attention_weight_min": -95.7421875,
      "activations/layer4_attention_weight_max": 96.43331909179688,
      "activations/layer4_attention_weight_min": -95.37477111816406,
      "activations/layer5_attention_weight_max": 71.60136413574219,
      "activations/layer5_attention_weight_min": -77.54444885253906,
      "activations/layer6_attention_weight_max": 50.856075286865234,
      "activations/layer6_attention_weight_min": -53.45945358276367,
      "activations/layer7_attention_weight_max": 68.75032806396484,
      "activations/layer7_attention_weight_min": -66.59842681884766,
      "activations/layer8_attention_weight_max": 47.05295944213867,
      "activations/layer8_attention_weight_min": -49.74892044067383,
      "activations/layer9_attention_weight_max": 47.37815856933594,
      "activations/layer9_attention_weight_min": -47.42093276977539,
      "epoch": 13.33,
      "learning_rate": 6.469431818181817e-05,
      "loss": 2.7727,
      "step": 229400
    },
    {
      "activations/layer0_attention_weight_max": 16.355409622192383,
      "activations/layer0_attention_weight_min": -14.142582893371582,
      "activations/layer10_attention_weight_max": 34.2875862121582,
      "activations/layer10_attention_weight_min": -33.53857421875,
      "activations/layer11_attention_weight_max": 30.410011291503906,
      "activations/layer11_attention_weight_min": -30.323307037353516,
      "activations/layer12_attention_weight_max": 17.024662017822266,
      "activations/layer12_attention_weight_min": -26.010831832885742,
      "activations/layer13_attention_weight_max": 36.869258880615234,
      "activations/layer13_attention_weight_min": -30.02927589416504,
      "activations/layer14_attention_weight_max": 33.93049240112305,
      "activations/layer14_attention_weight_min": -29.58331298828125,
      "activations/layer15_attention_weight_max": 30.806228637695312,
      "activations/layer15_attention_weight_min": -29.129770278930664,
      "activations/layer16_attention_weight_max": 31.212352752685547,
      "activations/layer16_attention_weight_min": -30.668750762939453,
      "activations/layer17_attention_weight_max": 51.0659065246582,
      "activations/layer17_attention_weight_min": -44.47789764404297,
      "activations/layer18_attention_weight_max": 42.37892150878906,
      "activations/layer18_attention_weight_min": -35.44208908081055,
      "activations/layer19_attention_weight_max": 22.274999618530273,
      "activations/layer19_attention_weight_min": -20.376012802124023,
      "activations/layer1_attention_weight_max": 16.82967185974121,
      "activations/layer1_attention_weight_min": -13.858869552612305,
      "activations/layer20_attention_weight_max": 21.33013153076172,
      "activations/layer20_attention_weight_min": -25.550491333007812,
      "activations/layer21_attention_weight_max": 31.394134521484375,
      "activations/layer21_attention_weight_min": -23.84264373779297,
      "activations/layer22_attention_weight_max": 27.862945556640625,
      "activations/layer22_attention_weight_min": -26.51046371459961,
      "activations/layer23_attention_weight_max": 35.29147720336914,
      "activations/layer23_attention_weight_min": -24.494483947753906,
      "activations/layer2_attention_weight_max": 30.308828353881836,
      "activations/layer2_attention_weight_min": -30.900989532470703,
      "activations/layer3_attention_weight_max": 94.04315185546875,
      "activations/layer3_attention_weight_min": -98.51663970947266,
      "activations/layer4_attention_weight_max": 92.38301849365234,
      "activations/layer4_attention_weight_min": -91.7946548461914,
      "activations/layer5_attention_weight_max": 68.54319763183594,
      "activations/layer5_attention_weight_min": -76.29434204101562,
      "activations/layer6_attention_weight_max": 46.438846588134766,
      "activations/layer6_attention_weight_min": -50.21110153198242,
      "activations/layer7_attention_weight_max": 64.89168548583984,
      "activations/layer7_attention_weight_min": -63.21482849121094,
      "activations/layer8_attention_weight_max": 41.192508697509766,
      "activations/layer8_attention_weight_min": -41.55607604980469,
      "activations/layer9_attention_weight_max": 42.316490173339844,
      "activations/layer9_attention_weight_min": -42.83118438720703,
      "epoch": 13.33,
      "learning_rate": 6.467537878787878e-05,
      "loss": 2.761,
      "step": 229450
    },
    {
      "activations/layer0_attention_weight_max": 16.89996910095215,
      "activations/layer0_attention_weight_min": -13.898119926452637,
      "activations/layer10_attention_weight_max": 39.433170318603516,
      "activations/layer10_attention_weight_min": -40.07807922363281,
      "activations/layer11_attention_weight_max": 38.83177185058594,
      "activations/layer11_attention_weight_min": -34.54734420776367,
      "activations/layer12_attention_weight_max": 18.36505699157715,
      "activations/layer12_attention_weight_min": -24.01598358154297,
      "activations/layer13_attention_weight_max": 42.74186706542969,
      "activations/layer13_attention_weight_min": -34.16193771362305,
      "activations/layer14_attention_weight_max": 37.17679977416992,
      "activations/layer14_attention_weight_min": -31.180736541748047,
      "activations/layer15_attention_weight_max": 34.04987716674805,
      "activations/layer15_attention_weight_min": -28.629732131958008,
      "activations/layer16_attention_weight_max": 34.42770004272461,
      "activations/layer16_attention_weight_min": -30.800365447998047,
      "activations/layer17_attention_weight_max": 53.874813079833984,
      "activations/layer17_attention_weight_min": -41.5476188659668,
      "activations/layer18_attention_weight_max": 45.47718811035156,
      "activations/layer18_attention_weight_min": -38.151432037353516,
      "activations/layer19_attention_weight_max": 26.937490463256836,
      "activations/layer19_attention_weight_min": -22.31325340270996,
      "activations/layer1_attention_weight_max": 16.410146713256836,
      "activations/layer1_attention_weight_min": -15.756890296936035,
      "activations/layer20_attention_weight_max": 25.19342613220215,
      "activations/layer20_attention_weight_min": -23.26546287536621,
      "activations/layer21_attention_weight_max": 44.34538650512695,
      "activations/layer21_attention_weight_min": -27.328527450561523,
      "activations/layer22_attention_weight_max": 30.188922882080078,
      "activations/layer22_attention_weight_min": -27.58013916015625,
      "activations/layer23_attention_weight_max": 36.192264556884766,
      "activations/layer23_attention_weight_min": -25.372665405273438,
      "activations/layer2_attention_weight_max": 33.0991325378418,
      "activations/layer2_attention_weight_min": -31.258995056152344,
      "activations/layer3_attention_weight_max": 101.84671783447266,
      "activations/layer3_attention_weight_min": -97.65331268310547,
      "activations/layer4_attention_weight_max": 100.48560333251953,
      "activations/layer4_attention_weight_min": -94.24011993408203,
      "activations/layer5_attention_weight_max": 71.32920837402344,
      "activations/layer5_attention_weight_min": -76.34825134277344,
      "activations/layer6_attention_weight_max": 50.48804473876953,
      "activations/layer6_attention_weight_min": -52.27313995361328,
      "activations/layer7_attention_weight_max": 75.07684326171875,
      "activations/layer7_attention_weight_min": -68.98884582519531,
      "activations/layer8_attention_weight_max": 50.00779342651367,
      "activations/layer8_attention_weight_min": -48.30226516723633,
      "activations/layer9_attention_weight_max": 60.860595703125,
      "activations/layer9_attention_weight_min": -48.154869079589844,
      "epoch": 13.34,
      "learning_rate": 6.465643939393939e-05,
      "loss": 2.7657,
      "step": 229500
    },
    {
      "activations/layer0_attention_weight_max": 15.72286605834961,
      "activations/layer0_attention_weight_min": -14.120805740356445,
      "activations/layer10_attention_weight_max": 36.522884368896484,
      "activations/layer10_attention_weight_min": -35.28329086303711,
      "activations/layer11_attention_weight_max": 31.791128158569336,
      "activations/layer11_attention_weight_min": -33.60820770263672,
      "activations/layer12_attention_weight_max": 18.788480758666992,
      "activations/layer12_attention_weight_min": -26.265792846679688,
      "activations/layer13_attention_weight_max": 38.67962646484375,
      "activations/layer13_attention_weight_min": -28.438203811645508,
      "activations/layer14_attention_weight_max": 35.406341552734375,
      "activations/layer14_attention_weight_min": -30.0766544342041,
      "activations/layer15_attention_weight_max": 33.36545181274414,
      "activations/layer15_attention_weight_min": -30.591251373291016,
      "activations/layer16_attention_weight_max": 33.925716400146484,
      "activations/layer16_attention_weight_min": -33.371604919433594,
      "activations/layer17_attention_weight_max": 54.642330169677734,
      "activations/layer17_attention_weight_min": -44.547996520996094,
      "activations/layer18_attention_weight_max": 49.339195251464844,
      "activations/layer18_attention_weight_min": -41.190834045410156,
      "activations/layer19_attention_weight_max": 22.538227081298828,
      "activations/layer19_attention_weight_min": -21.54334259033203,
      "activations/layer1_attention_weight_max": 16.343461990356445,
      "activations/layer1_attention_weight_min": -15.631099700927734,
      "activations/layer20_attention_weight_max": 23.742046356201172,
      "activations/layer20_attention_weight_min": -22.004716873168945,
      "activations/layer21_attention_weight_max": 41.74571990966797,
      "activations/layer21_attention_weight_min": -23.026535034179688,
      "activations/layer22_attention_weight_max": 33.69975662231445,
      "activations/layer22_attention_weight_min": -26.612220764160156,
      "activations/layer23_attention_weight_max": 34.71352005004883,
      "activations/layer23_attention_weight_min": -25.53716278076172,
      "activations/layer2_attention_weight_max": 33.989280700683594,
      "activations/layer2_attention_weight_min": -32.01177215576172,
      "activations/layer3_attention_weight_max": 97.99252319335938,
      "activations/layer3_attention_weight_min": -102.41666412353516,
      "activations/layer4_attention_weight_max": 95.37203216552734,
      "activations/layer4_attention_weight_min": -96.99017333984375,
      "activations/layer5_attention_weight_max": 71.88298034667969,
      "activations/layer5_attention_weight_min": -77.89530181884766,
      "activations/layer6_attention_weight_max": 52.59675216674805,
      "activations/layer6_attention_weight_min": -54.897544860839844,
      "activations/layer7_attention_weight_max": 68.07315826416016,
      "activations/layer7_attention_weight_min": -69.08474731445312,
      "activations/layer8_attention_weight_max": 46.9478759765625,
      "activations/layer8_attention_weight_min": -50.44429397583008,
      "activations/layer9_attention_weight_max": 46.15499496459961,
      "activations/layer9_attention_weight_min": -51.6727180480957,
      "epoch": 13.34,
      "learning_rate": 6.463749999999999e-05,
      "loss": 2.7725,
      "step": 229550
    },
    {
      "activations/layer0_attention_weight_max": 16.751850128173828,
      "activations/layer0_attention_weight_min": -13.886404991149902,
      "activations/layer10_attention_weight_max": 36.11441421508789,
      "activations/layer10_attention_weight_min": -34.78367614746094,
      "activations/layer11_attention_weight_max": 34.33211135864258,
      "activations/layer11_attention_weight_min": -33.267093658447266,
      "activations/layer12_attention_weight_max": 21.056241989135742,
      "activations/layer12_attention_weight_min": -25.16823959350586,
      "activations/layer13_attention_weight_max": 34.53263854980469,
      "activations/layer13_attention_weight_min": -27.680604934692383,
      "activations/layer14_attention_weight_max": 36.54601287841797,
      "activations/layer14_attention_weight_min": -29.39483070373535,
      "activations/layer15_attention_weight_max": 31.687725067138672,
      "activations/layer15_attention_weight_min": -28.662155151367188,
      "activations/layer16_attention_weight_max": 31.99158477783203,
      "activations/layer16_attention_weight_min": -28.61370849609375,
      "activations/layer17_attention_weight_max": 49.21852493286133,
      "activations/layer17_attention_weight_min": -42.24629592895508,
      "activations/layer18_attention_weight_max": 43.5257682800293,
      "activations/layer18_attention_weight_min": -37.42741012573242,
      "activations/layer19_attention_weight_max": 22.392200469970703,
      "activations/layer19_attention_weight_min": -21.794403076171875,
      "activations/layer1_attention_weight_max": 16.53849220275879,
      "activations/layer1_attention_weight_min": -15.273809432983398,
      "activations/layer20_attention_weight_max": 22.509599685668945,
      "activations/layer20_attention_weight_min": -21.999666213989258,
      "activations/layer21_attention_weight_max": 35.04273986816406,
      "activations/layer21_attention_weight_min": -21.57919692993164,
      "activations/layer22_attention_weight_max": 29.14036750793457,
      "activations/layer22_attention_weight_min": -24.12664031982422,
      "activations/layer23_attention_weight_max": 37.65143966674805,
      "activations/layer23_attention_weight_min": -23.580322265625,
      "activations/layer2_attention_weight_max": 32.18277359008789,
      "activations/layer2_attention_weight_min": -30.14180564880371,
      "activations/layer3_attention_weight_max": 92.49449157714844,
      "activations/layer3_attention_weight_min": -90.76444244384766,
      "activations/layer4_attention_weight_max": 93.9963150024414,
      "activations/layer4_attention_weight_min": -89.7130355834961,
      "activations/layer5_attention_weight_max": 72.26660919189453,
      "activations/layer5_attention_weight_min": -72.86323547363281,
      "activations/layer6_attention_weight_max": 51.05121612548828,
      "activations/layer6_attention_weight_min": -52.19859313964844,
      "activations/layer7_attention_weight_max": 66.76675415039062,
      "activations/layer7_attention_weight_min": -65.07284545898438,
      "activations/layer8_attention_weight_max": 44.59669876098633,
      "activations/layer8_attention_weight_min": -45.8915901184082,
      "activations/layer9_attention_weight_max": 42.825462341308594,
      "activations/layer9_attention_weight_min": -45.32740020751953,
      "epoch": 13.34,
      "learning_rate": 6.46185606060606e-05,
      "loss": 2.7573,
      "step": 229600
    },
    {
      "activations/layer0_attention_weight_max": 16.332048416137695,
      "activations/layer0_attention_weight_min": -14.046754837036133,
      "activations/layer10_attention_weight_max": 43.281551361083984,
      "activations/layer10_attention_weight_min": -38.22456741333008,
      "activations/layer11_attention_weight_max": 36.20183563232422,
      "activations/layer11_attention_weight_min": -35.547630310058594,
      "activations/layer12_attention_weight_max": 17.816282272338867,
      "activations/layer12_attention_weight_min": -25.901351928710938,
      "activations/layer13_attention_weight_max": 47.22806930541992,
      "activations/layer13_attention_weight_min": -31.882036209106445,
      "activations/layer14_attention_weight_max": 36.49917984008789,
      "activations/layer14_attention_weight_min": -29.90341567993164,
      "activations/layer15_attention_weight_max": 32.003597259521484,
      "activations/layer15_attention_weight_min": -29.36547088623047,
      "activations/layer16_attention_weight_max": 32.895599365234375,
      "activations/layer16_attention_weight_min": -30.986270904541016,
      "activations/layer17_attention_weight_max": 52.33086013793945,
      "activations/layer17_attention_weight_min": -44.319515228271484,
      "activations/layer18_attention_weight_max": 48.134979248046875,
      "activations/layer18_attention_weight_min": -38.68026351928711,
      "activations/layer19_attention_weight_max": 22.630155563354492,
      "activations/layer19_attention_weight_min": -21.4096736907959,
      "activations/layer1_attention_weight_max": 16.82513427734375,
      "activations/layer1_attention_weight_min": -16.763887405395508,
      "activations/layer20_attention_weight_max": 22.408628463745117,
      "activations/layer20_attention_weight_min": -21.459989547729492,
      "activations/layer21_attention_weight_max": 38.63525390625,
      "activations/layer21_attention_weight_min": -23.657360076904297,
      "activations/layer22_attention_weight_max": 31.64759635925293,
      "activations/layer22_attention_weight_min": -25.630172729492188,
      "activations/layer23_attention_weight_max": 38.483917236328125,
      "activations/layer23_attention_weight_min": -24.587736129760742,
      "activations/layer2_attention_weight_max": 33.054115295410156,
      "activations/layer2_attention_weight_min": -32.949256896972656,
      "activations/layer3_attention_weight_max": 99.40567016601562,
      "activations/layer3_attention_weight_min": -102.5567855834961,
      "activations/layer4_attention_weight_max": 106.79985809326172,
      "activations/layer4_attention_weight_min": -101.1551284790039,
      "activations/layer5_attention_weight_max": 77.6871566772461,
      "activations/layer5_attention_weight_min": -78.40265655517578,
      "activations/layer6_attention_weight_max": 55.2965202331543,
      "activations/layer6_attention_weight_min": -53.92972946166992,
      "activations/layer7_attention_weight_max": 70.35564422607422,
      "activations/layer7_attention_weight_min": -66.94615173339844,
      "activations/layer8_attention_weight_max": 51.067962646484375,
      "activations/layer8_attention_weight_min": -50.1797981262207,
      "activations/layer9_attention_weight_max": 48.22291564941406,
      "activations/layer9_attention_weight_min": -46.59543991088867,
      "epoch": 13.34,
      "learning_rate": 6.459962121212121e-05,
      "loss": 2.7661,
      "step": 229650
    },
    {
      "activations/layer0_attention_weight_max": 15.419182777404785,
      "activations/layer0_attention_weight_min": -13.991022109985352,
      "activations/layer10_attention_weight_max": 36.380287170410156,
      "activations/layer10_attention_weight_min": -36.874061584472656,
      "activations/layer11_attention_weight_max": 32.39226150512695,
      "activations/layer11_attention_weight_min": -35.33867645263672,
      "activations/layer12_attention_weight_max": 19.1484317779541,
      "activations/layer12_attention_weight_min": -29.404930114746094,
      "activations/layer13_attention_weight_max": 41.516632080078125,
      "activations/layer13_attention_weight_min": -33.25107955932617,
      "activations/layer14_attention_weight_max": 35.3923454284668,
      "activations/layer14_attention_weight_min": -28.944887161254883,
      "activations/layer15_attention_weight_max": 32.64006042480469,
      "activations/layer15_attention_weight_min": -28.767982482910156,
      "activations/layer16_attention_weight_max": 31.504894256591797,
      "activations/layer16_attention_weight_min": -28.337974548339844,
      "activations/layer17_attention_weight_max": 50.83498001098633,
      "activations/layer17_attention_weight_min": -41.475746154785156,
      "activations/layer18_attention_weight_max": 45.66328048706055,
      "activations/layer18_attention_weight_min": -35.572845458984375,
      "activations/layer19_attention_weight_max": 21.668420791625977,
      "activations/layer19_attention_weight_min": -21.54436683654785,
      "activations/layer1_attention_weight_max": 16.652751922607422,
      "activations/layer1_attention_weight_min": -15.073811531066895,
      "activations/layer20_attention_weight_max": 21.900272369384766,
      "activations/layer20_attention_weight_min": -21.54909324645996,
      "activations/layer21_attention_weight_max": 39.750762939453125,
      "activations/layer21_attention_weight_min": -21.914173126220703,
      "activations/layer22_attention_weight_max": 28.43360137939453,
      "activations/layer22_attention_weight_min": -24.03866958618164,
      "activations/layer23_attention_weight_max": 37.2849006652832,
      "activations/layer23_attention_weight_min": -23.1590576171875,
      "activations/layer2_attention_weight_max": 33.50709533691406,
      "activations/layer2_attention_weight_min": -32.29340362548828,
      "activations/layer3_attention_weight_max": 95.14021301269531,
      "activations/layer3_attention_weight_min": -101.17354583740234,
      "activations/layer4_attention_weight_max": 98.78043365478516,
      "activations/layer4_attention_weight_min": -94.06671905517578,
      "activations/layer5_attention_weight_max": 70.17900085449219,
      "activations/layer5_attention_weight_min": -75.81094360351562,
      "activations/layer6_attention_weight_max": 53.77216720581055,
      "activations/layer6_attention_weight_min": -51.72295379638672,
      "activations/layer7_attention_weight_max": 68.00335693359375,
      "activations/layer7_attention_weight_min": -67.14002227783203,
      "activations/layer8_attention_weight_max": 48.529476165771484,
      "activations/layer8_attention_weight_min": -49.63984680175781,
      "activations/layer9_attention_weight_max": 44.48834991455078,
      "activations/layer9_attention_weight_min": -46.69633102416992,
      "epoch": 13.35,
      "learning_rate": 6.458068181818181e-05,
      "loss": 2.7728,
      "step": 229700
    },
    {
      "activations/layer0_attention_weight_max": 15.290979385375977,
      "activations/layer0_attention_weight_min": -14.035202026367188,
      "activations/layer10_attention_weight_max": 38.13664245605469,
      "activations/layer10_attention_weight_min": -35.169246673583984,
      "activations/layer11_attention_weight_max": 33.866477966308594,
      "activations/layer11_attention_weight_min": -33.761878967285156,
      "activations/layer12_attention_weight_max": 17.955322265625,
      "activations/layer12_attention_weight_min": -25.68899154663086,
      "activations/layer13_attention_weight_max": 43.11478042602539,
      "activations/layer13_attention_weight_min": -28.385276794433594,
      "activations/layer14_attention_weight_max": 32.936683654785156,
      "activations/layer14_attention_weight_min": -29.18256187438965,
      "activations/layer15_attention_weight_max": 31.828365325927734,
      "activations/layer15_attention_weight_min": -29.20546531677246,
      "activations/layer16_attention_weight_max": 29.328243255615234,
      "activations/layer16_attention_weight_min": -29.695737838745117,
      "activations/layer17_attention_weight_max": 53.636390686035156,
      "activations/layer17_attention_weight_min": -45.95891189575195,
      "activations/layer18_attention_weight_max": 45.59280014038086,
      "activations/layer18_attention_weight_min": -39.89683532714844,
      "activations/layer19_attention_weight_max": 23.467533111572266,
      "activations/layer19_attention_weight_min": -25.003469467163086,
      "activations/layer1_attention_weight_max": 17.783422470092773,
      "activations/layer1_attention_weight_min": -15.819231986999512,
      "activations/layer20_attention_weight_max": 24.240419387817383,
      "activations/layer20_attention_weight_min": -23.6381893157959,
      "activations/layer21_attention_weight_max": 36.589088439941406,
      "activations/layer21_attention_weight_min": -23.91570281982422,
      "activations/layer22_attention_weight_max": 29.40260124206543,
      "activations/layer22_attention_weight_min": -26.759506225585938,
      "activations/layer23_attention_weight_max": 41.04545593261719,
      "activations/layer23_attention_weight_min": -28.296314239501953,
      "activations/layer2_attention_weight_max": 33.272727966308594,
      "activations/layer2_attention_weight_min": -30.674070358276367,
      "activations/layer3_attention_weight_max": 94.20501708984375,
      "activations/layer3_attention_weight_min": -98.47827911376953,
      "activations/layer4_attention_weight_max": 98.01403045654297,
      "activations/layer4_attention_weight_min": -96.43197631835938,
      "activations/layer5_attention_weight_max": 70.91167449951172,
      "activations/layer5_attention_weight_min": -79.4888916015625,
      "activations/layer6_attention_weight_max": 52.498435974121094,
      "activations/layer6_attention_weight_min": -51.02649688720703,
      "activations/layer7_attention_weight_max": 65.05068969726562,
      "activations/layer7_attention_weight_min": -68.04637908935547,
      "activations/layer8_attention_weight_max": 46.54323196411133,
      "activations/layer8_attention_weight_min": -47.15066146850586,
      "activations/layer9_attention_weight_max": 44.285850524902344,
      "activations/layer9_attention_weight_min": -44.036251068115234,
      "epoch": 13.35,
      "learning_rate": 6.456174242424243e-05,
      "loss": 2.7661,
      "step": 229750
    },
    {
      "activations/layer0_attention_weight_max": 16.011526107788086,
      "activations/layer0_attention_weight_min": -14.289271354675293,
      "activations/layer10_attention_weight_max": 37.44382095336914,
      "activations/layer10_attention_weight_min": -35.085975646972656,
      "activations/layer11_attention_weight_max": 32.100311279296875,
      "activations/layer11_attention_weight_min": -33.07487106323242,
      "activations/layer12_attention_weight_max": 20.64668083190918,
      "activations/layer12_attention_weight_min": -25.445892333984375,
      "activations/layer13_attention_weight_max": 52.415565490722656,
      "activations/layer13_attention_weight_min": -36.67234802246094,
      "activations/layer14_attention_weight_max": 56.720542907714844,
      "activations/layer14_attention_weight_min": -33.44359588623047,
      "activations/layer15_attention_weight_max": 43.826541900634766,
      "activations/layer15_attention_weight_min": -30.82413101196289,
      "activations/layer16_attention_weight_max": 34.467159271240234,
      "activations/layer16_attention_weight_min": -30.88121223449707,
      "activations/layer17_attention_weight_max": 57.93491744995117,
      "activations/layer17_attention_weight_min": -48.00665283203125,
      "activations/layer18_attention_weight_max": 58.17028045654297,
      "activations/layer18_attention_weight_min": -42.4560661315918,
      "activations/layer19_attention_weight_max": 25.988910675048828,
      "activations/layer19_attention_weight_min": -22.929597854614258,
      "activations/layer1_attention_weight_max": 16.92078971862793,
      "activations/layer1_attention_weight_min": -15.362391471862793,
      "activations/layer20_attention_weight_max": 26.441160202026367,
      "activations/layer20_attention_weight_min": -28.852476119995117,
      "activations/layer21_attention_weight_max": 44.09476852416992,
      "activations/layer21_attention_weight_min": -23.807321548461914,
      "activations/layer22_attention_weight_max": 30.68914031982422,
      "activations/layer22_attention_weight_min": -26.014652252197266,
      "activations/layer23_attention_weight_max": 40.661930084228516,
      "activations/layer23_attention_weight_min": -23.07381248474121,
      "activations/layer2_attention_weight_max": 31.94745635986328,
      "activations/layer2_attention_weight_min": -32.070396423339844,
      "activations/layer3_attention_weight_max": 95.07894134521484,
      "activations/layer3_attention_weight_min": -98.51377868652344,
      "activations/layer4_attention_weight_max": 99.20796203613281,
      "activations/layer4_attention_weight_min": -89.82585906982422,
      "activations/layer5_attention_weight_max": 67.62393951416016,
      "activations/layer5_attention_weight_min": -80.95720672607422,
      "activations/layer6_attention_weight_max": 55.0341796875,
      "activations/layer6_attention_weight_min": -53.26543426513672,
      "activations/layer7_attention_weight_max": 69.3625717163086,
      "activations/layer7_attention_weight_min": -63.30314254760742,
      "activations/layer8_attention_weight_max": 45.81222152709961,
      "activations/layer8_attention_weight_min": -46.41012954711914,
      "activations/layer9_attention_weight_max": 47.45158004760742,
      "activations/layer9_attention_weight_min": -47.014102935791016,
      "epoch": 13.35,
      "learning_rate": 6.454280303030303e-05,
      "loss": 2.7734,
      "step": 229800
    },
    {
      "activations/layer0_attention_weight_max": 15.891339302062988,
      "activations/layer0_attention_weight_min": -14.412152290344238,
      "activations/layer10_attention_weight_max": 36.31549835205078,
      "activations/layer10_attention_weight_min": -35.477882385253906,
      "activations/layer11_attention_weight_max": 32.766056060791016,
      "activations/layer11_attention_weight_min": -33.82450485229492,
      "activations/layer12_attention_weight_max": 18.268091201782227,
      "activations/layer12_attention_weight_min": -31.395484924316406,
      "activations/layer13_attention_weight_max": 37.820281982421875,
      "activations/layer13_attention_weight_min": -28.256824493408203,
      "activations/layer14_attention_weight_max": 36.88847732543945,
      "activations/layer14_attention_weight_min": -29.852813720703125,
      "activations/layer15_attention_weight_max": 34.938106536865234,
      "activations/layer15_attention_weight_min": -28.471275329589844,
      "activations/layer16_attention_weight_max": 31.53990364074707,
      "activations/layer16_attention_weight_min": -28.36137580871582,
      "activations/layer17_attention_weight_max": 48.5319709777832,
      "activations/layer17_attention_weight_min": -40.95262145996094,
      "activations/layer18_attention_weight_max": 43.6143913269043,
      "activations/layer18_attention_weight_min": -35.18000411987305,
      "activations/layer19_attention_weight_max": 22.726154327392578,
      "activations/layer19_attention_weight_min": -21.22493553161621,
      "activations/layer1_attention_weight_max": 16.518795013427734,
      "activations/layer1_attention_weight_min": -16.435291290283203,
      "activations/layer20_attention_weight_max": 21.137662887573242,
      "activations/layer20_attention_weight_min": -21.617618560791016,
      "activations/layer21_attention_weight_max": 30.917314529418945,
      "activations/layer21_attention_weight_min": -20.858362197875977,
      "activations/layer22_attention_weight_max": 32.39481735229492,
      "activations/layer22_attention_weight_min": -25.459428787231445,
      "activations/layer23_attention_weight_max": 37.57797622680664,
      "activations/layer23_attention_weight_min": -23.486074447631836,
      "activations/layer2_attention_weight_max": 31.90970230102539,
      "activations/layer2_attention_weight_min": -30.943084716796875,
      "activations/layer3_attention_weight_max": 96.94878387451172,
      "activations/layer3_attention_weight_min": -93.3035659790039,
      "activations/layer4_attention_weight_max": 98.15229034423828,
      "activations/layer4_attention_weight_min": -95.04894256591797,
      "activations/layer5_attention_weight_max": 70.42953491210938,
      "activations/layer5_attention_weight_min": -79.15923309326172,
      "activations/layer6_attention_weight_max": 52.906700134277344,
      "activations/layer6_attention_weight_min": -54.51877975463867,
      "activations/layer7_attention_weight_max": 65.37108612060547,
      "activations/layer7_attention_weight_min": -66.36164855957031,
      "activations/layer8_attention_weight_max": 46.15047073364258,
      "activations/layer8_attention_weight_min": -48.78642654418945,
      "activations/layer9_attention_weight_max": 45.73996353149414,
      "activations/layer9_attention_weight_min": -47.2454719543457,
      "epoch": 13.36,
      "learning_rate": 6.452386363636363e-05,
      "loss": 2.7544,
      "step": 229850
    },
    {
      "activations/layer0_attention_weight_max": 16.3156795501709,
      "activations/layer0_attention_weight_min": -14.388258934020996,
      "activations/layer10_attention_weight_max": 35.59832763671875,
      "activations/layer10_attention_weight_min": -35.575294494628906,
      "activations/layer11_attention_weight_max": 30.99901580810547,
      "activations/layer11_attention_weight_min": -33.571617126464844,
      "activations/layer12_attention_weight_max": 17.80562973022461,
      "activations/layer12_attention_weight_min": -23.751691818237305,
      "activations/layer13_attention_weight_max": 37.95143127441406,
      "activations/layer13_attention_weight_min": -31.757427215576172,
      "activations/layer14_attention_weight_max": 32.95037078857422,
      "activations/layer14_attention_weight_min": -27.811288833618164,
      "activations/layer15_attention_weight_max": 30.6009521484375,
      "activations/layer15_attention_weight_min": -27.787996292114258,
      "activations/layer16_attention_weight_max": 31.38572883605957,
      "activations/layer16_attention_weight_min": -30.193445205688477,
      "activations/layer17_attention_weight_max": 49.42476272583008,
      "activations/layer17_attention_weight_min": -45.582088470458984,
      "activations/layer18_attention_weight_max": 42.97245788574219,
      "activations/layer18_attention_weight_min": -36.3951416015625,
      "activations/layer19_attention_weight_max": 21.581756591796875,
      "activations/layer19_attention_weight_min": -23.847312927246094,
      "activations/layer1_attention_weight_max": 16.561447143554688,
      "activations/layer1_attention_weight_min": -15.402359008789062,
      "activations/layer20_attention_weight_max": 22.066238403320312,
      "activations/layer20_attention_weight_min": -23.13966178894043,
      "activations/layer21_attention_weight_max": 31.240978240966797,
      "activations/layer21_attention_weight_min": -23.79817771911621,
      "activations/layer22_attention_weight_max": 28.42941665649414,
      "activations/layer22_attention_weight_min": -25.10921859741211,
      "activations/layer23_attention_weight_max": 34.92140197753906,
      "activations/layer23_attention_weight_min": -24.76320457458496,
      "activations/layer2_attention_weight_max": 32.4998779296875,
      "activations/layer2_attention_weight_min": -31.626079559326172,
      "activations/layer3_attention_weight_max": 98.44586181640625,
      "activations/layer3_attention_weight_min": -104.44078826904297,
      "activations/layer4_attention_weight_max": 100.33550262451172,
      "activations/layer4_attention_weight_min": -97.77963256835938,
      "activations/layer5_attention_weight_max": 69.9228286743164,
      "activations/layer5_attention_weight_min": -82.00777435302734,
      "activations/layer6_attention_weight_max": 50.41541290283203,
      "activations/layer6_attention_weight_min": -50.02866744995117,
      "activations/layer7_attention_weight_max": 66.67212677001953,
      "activations/layer7_attention_weight_min": -65.01654815673828,
      "activations/layer8_attention_weight_max": 44.83147430419922,
      "activations/layer8_attention_weight_min": -46.49748611450195,
      "activations/layer9_attention_weight_max": 44.626338958740234,
      "activations/layer9_attention_weight_min": -46.26258850097656,
      "epoch": 13.36,
      "learning_rate": 6.450492424242424e-05,
      "loss": 2.7707,
      "step": 229900
    },
    {
      "activations/layer0_attention_weight_max": 15.95911693572998,
      "activations/layer0_attention_weight_min": -13.930285453796387,
      "activations/layer10_attention_weight_max": 36.727291107177734,
      "activations/layer10_attention_weight_min": -33.907997131347656,
      "activations/layer11_attention_weight_max": 31.219762802124023,
      "activations/layer11_attention_weight_min": -32.23094940185547,
      "activations/layer12_attention_weight_max": 22.33626937866211,
      "activations/layer12_attention_weight_min": -25.322717666625977,
      "activations/layer13_attention_weight_max": 56.46727752685547,
      "activations/layer13_attention_weight_min": -35.39704132080078,
      "activations/layer14_attention_weight_max": 48.953556060791016,
      "activations/layer14_attention_weight_min": -29.98814582824707,
      "activations/layer15_attention_weight_max": 38.75251007080078,
      "activations/layer15_attention_weight_min": -28.74991798400879,
      "activations/layer16_attention_weight_max": 34.437232971191406,
      "activations/layer16_attention_weight_min": -31.021516799926758,
      "activations/layer17_attention_weight_max": 57.45340347290039,
      "activations/layer17_attention_weight_min": -45.426292419433594,
      "activations/layer18_attention_weight_max": 51.91208267211914,
      "activations/layer18_attention_weight_min": -40.90372085571289,
      "activations/layer19_attention_weight_max": 25.878299713134766,
      "activations/layer19_attention_weight_min": -23.90139389038086,
      "activations/layer1_attention_weight_max": 16.003747940063477,
      "activations/layer1_attention_weight_min": -16.507978439331055,
      "activations/layer20_attention_weight_max": 28.389747619628906,
      "activations/layer20_attention_weight_min": -21.80096435546875,
      "activations/layer21_attention_weight_max": 48.12408447265625,
      "activations/layer21_attention_weight_min": -23.66522216796875,
      "activations/layer22_attention_weight_max": 30.439952850341797,
      "activations/layer22_attention_weight_min": -22.910934448242188,
      "activations/layer23_attention_weight_max": 36.93042755126953,
      "activations/layer23_attention_weight_min": -28.240558624267578,
      "activations/layer2_attention_weight_max": 32.07936477661133,
      "activations/layer2_attention_weight_min": -30.953243255615234,
      "activations/layer3_attention_weight_max": 96.44758605957031,
      "activations/layer3_attention_weight_min": -100.32963562011719,
      "activations/layer4_attention_weight_max": 97.69132232666016,
      "activations/layer4_attention_weight_min": -100.06437683105469,
      "activations/layer5_attention_weight_max": 69.09733581542969,
      "activations/layer5_attention_weight_min": -80.95748138427734,
      "activations/layer6_attention_weight_max": 52.57978439331055,
      "activations/layer6_attention_weight_min": -50.10028839111328,
      "activations/layer7_attention_weight_max": 63.932228088378906,
      "activations/layer7_attention_weight_min": -64.31668090820312,
      "activations/layer8_attention_weight_max": 44.672821044921875,
      "activations/layer8_attention_weight_min": -47.11659622192383,
      "activations/layer9_attention_weight_max": 47.79787826538086,
      "activations/layer9_attention_weight_min": -44.29929733276367,
      "epoch": 13.36,
      "learning_rate": 6.448598484848484e-05,
      "loss": 2.7447,
      "step": 229950
    },
    {
      "activations/layer0_attention_weight_max": 15.07970905303955,
      "activations/layer0_attention_weight_min": -14.227450370788574,
      "activations/layer10_attention_weight_max": 36.92623519897461,
      "activations/layer10_attention_weight_min": -36.004730224609375,
      "activations/layer11_attention_weight_max": 32.76420211791992,
      "activations/layer11_attention_weight_min": -34.927207946777344,
      "activations/layer12_attention_weight_max": 18.486181259155273,
      "activations/layer12_attention_weight_min": -24.329452514648438,
      "activations/layer13_attention_weight_max": 40.57533264160156,
      "activations/layer13_attention_weight_min": -30.41571044921875,
      "activations/layer14_attention_weight_max": 38.434932708740234,
      "activations/layer14_attention_weight_min": -30.906095504760742,
      "activations/layer15_attention_weight_max": 34.95894241333008,
      "activations/layer15_attention_weight_min": -31.37076759338379,
      "activations/layer16_attention_weight_max": 33.41266632080078,
      "activations/layer16_attention_weight_min": -31.455305099487305,
      "activations/layer17_attention_weight_max": 52.84425735473633,
      "activations/layer17_attention_weight_min": -43.79896926879883,
      "activations/layer18_attention_weight_max": 48.32073211669922,
      "activations/layer18_attention_weight_min": -36.66535568237305,
      "activations/layer19_attention_weight_max": 25.378101348876953,
      "activations/layer19_attention_weight_min": -21.92445182800293,
      "activations/layer1_attention_weight_max": 16.823299407958984,
      "activations/layer1_attention_weight_min": -14.582365989685059,
      "activations/layer20_attention_weight_max": 23.53173065185547,
      "activations/layer20_attention_weight_min": -25.074748992919922,
      "activations/layer21_attention_weight_max": 40.37917709350586,
      "activations/layer21_attention_weight_min": -25.035491943359375,
      "activations/layer22_attention_weight_max": 32.04059982299805,
      "activations/layer22_attention_weight_min": -27.144447326660156,
      "activations/layer23_attention_weight_max": 38.42021560668945,
      "activations/layer23_attention_weight_min": -25.50252914428711,
      "activations/layer2_attention_weight_max": 34.03214645385742,
      "activations/layer2_attention_weight_min": -33.18367004394531,
      "activations/layer3_attention_weight_max": 97.52064514160156,
      "activations/layer3_attention_weight_min": -94.61487579345703,
      "activations/layer4_attention_weight_max": 103.35212707519531,
      "activations/layer4_attention_weight_min": -95.65442657470703,
      "activations/layer5_attention_weight_max": 72.10514068603516,
      "activations/layer5_attention_weight_min": -81.31367492675781,
      "activations/layer6_attention_weight_max": 56.23732376098633,
      "activations/layer6_attention_weight_min": -55.38994216918945,
      "activations/layer7_attention_weight_max": 70.49544525146484,
      "activations/layer7_attention_weight_min": -68.49214935302734,
      "activations/layer8_attention_weight_max": 45.55217742919922,
      "activations/layer8_attention_weight_min": -48.73765182495117,
      "activations/layer9_attention_weight_max": 43.773895263671875,
      "activations/layer9_attention_weight_min": -46.51567459106445,
      "epoch": 13.36,
      "learning_rate": 6.446704545454545e-05,
      "loss": 2.7798,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.5123,
      "eval_samples_per_second": 504.449,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.5123,
      "eval_openwebtext_samples_per_second": 504.449,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 1.9784,
      "eval_wikitext_samples_per_second": 230.493,
      "step": 230000
    },
    {
      "epoch": 13.36,
      "eval_lambada_loss": 2.552734375,
      "eval_lambada_ppl": 12.842171128856814,
      "eval_lambada_runtime": 9.5412,
      "eval_lambada_samples_per_second": 510.311,
      "step": 230000
    },
    {
      "activations/layer0_attention_weight_max": 15.91153621673584,
      "activations/layer0_attention_weight_min": -14.04665756225586,
      "activations/layer10_attention_weight_max": 52.42289733886719,
      "activations/layer10_attention_weight_min": -48.892818450927734,
      "activations/layer11_attention_weight_max": 49.24488830566406,
      "activations/layer11_attention_weight_min": -46.26829528808594,
      "activations/layer12_attention_weight_max": 18.206680297851562,
      "activations/layer12_attention_weight_min": -26.311500549316406,
      "activations/layer13_attention_weight_max": 42.94900894165039,
      "activations/layer13_attention_weight_min": -35.04723358154297,
      "activations/layer14_attention_weight_max": 55.215492248535156,
      "activations/layer14_attention_weight_min": -35.72859191894531,
      "activations/layer15_attention_weight_max": 56.01952362060547,
      "activations/layer15_attention_weight_min": -34.34525680541992,
      "activations/layer16_attention_weight_max": 35.11379623413086,
      "activations/layer16_attention_weight_min": -34.062705993652344,
      "activations/layer17_attention_weight_max": 49.3090705871582,
      "activations/layer17_attention_weight_min": -49.697269439697266,
      "activations/layer18_attention_weight_max": 44.332340240478516,
      "activations/layer18_attention_weight_min": -43.34048843383789,
      "activations/layer19_attention_weight_max": 20.80778694152832,
      "activations/layer19_attention_weight_min": -22.365276336669922,
      "activations/layer1_attention_weight_max": 17.9491024017334,
      "activations/layer1_attention_weight_min": -14.836868286132812,
      "activations/layer20_attention_weight_max": 21.469030380249023,
      "activations/layer20_attention_weight_min": -20.903690338134766,
      "activations/layer21_attention_weight_max": 31.652944564819336,
      "activations/layer21_attention_weight_min": -22.323184967041016,
      "activations/layer22_attention_weight_max": 28.63742446899414,
      "activations/layer22_attention_weight_min": -24.263065338134766,
      "activations/layer23_attention_weight_max": 34.34589767456055,
      "activations/layer23_attention_weight_min": -26.217388153076172,
      "activations/layer2_attention_weight_max": 38.886940002441406,
      "activations/layer2_attention_weight_min": -35.57073974609375,
      "activations/layer3_attention_weight_max": 118.711181640625,
      "activations/layer3_attention_weight_min": -117.36556243896484,
      "activations/layer4_attention_weight_max": 123.00341796875,
      "activations/layer4_attention_weight_min": -112.6535415649414,
      "activations/layer5_attention_weight_max": 83.95358276367188,
      "activations/layer5_attention_weight_min": -77.37347412109375,
      "activations/layer6_attention_weight_max": 61.30445861816406,
      "activations/layer6_attention_weight_min": -59.89722442626953,
      "activations/layer7_attention_weight_max": 85.5157241821289,
      "activations/layer7_attention_weight_min": -75.32394409179688,
      "activations/layer8_attention_weight_max": 60.84090042114258,
      "activations/layer8_attention_weight_min": -56.761322021484375,
      "activations/layer9_attention_weight_max": 69.20394134521484,
      "activations/layer9_attention_weight_min": -61.11579132080078,
      "epoch": 13.37,
      "learning_rate": 6.444810606060606e-05,
      "loss": 2.7683,
      "step": 230050
    },
    {
      "activations/layer0_attention_weight_max": 14.951234817504883,
      "activations/layer0_attention_weight_min": -14.163735389709473,
      "activations/layer10_attention_weight_max": 35.0723876953125,
      "activations/layer10_attention_weight_min": -34.607948303222656,
      "activations/layer11_attention_weight_max": 32.42274475097656,
      "activations/layer11_attention_weight_min": -31.84267234802246,
      "activations/layer12_attention_weight_max": 18.621803283691406,
      "activations/layer12_attention_weight_min": -24.93870735168457,
      "activations/layer13_attention_weight_max": 35.78306579589844,
      "activations/layer13_attention_weight_min": -25.65593147277832,
      "activations/layer14_attention_weight_max": 37.78227996826172,
      "activations/layer14_attention_weight_min": -31.492340087890625,
      "activations/layer15_attention_weight_max": 33.025386810302734,
      "activations/layer15_attention_weight_min": -29.681406021118164,
      "activations/layer16_attention_weight_max": 33.425537109375,
      "activations/layer16_attention_weight_min": -31.80528450012207,
      "activations/layer17_attention_weight_max": 54.97802734375,
      "activations/layer17_attention_weight_min": -46.9578742980957,
      "activations/layer18_attention_weight_max": 49.566246032714844,
      "activations/layer18_attention_weight_min": -40.855140686035156,
      "activations/layer19_attention_weight_max": 23.66032600402832,
      "activations/layer19_attention_weight_min": -22.2631893157959,
      "activations/layer1_attention_weight_max": 16.377368927001953,
      "activations/layer1_attention_weight_min": -14.778982162475586,
      "activations/layer20_attention_weight_max": 22.552459716796875,
      "activations/layer20_attention_weight_min": -21.457820892333984,
      "activations/layer21_attention_weight_max": 42.491886138916016,
      "activations/layer21_attention_weight_min": -20.682538986206055,
      "activations/layer22_attention_weight_max": 33.158870697021484,
      "activations/layer22_attention_weight_min": -24.543413162231445,
      "activations/layer23_attention_weight_max": 36.03299331665039,
      "activations/layer23_attention_weight_min": -21.260805130004883,
      "activations/layer2_attention_weight_max": 33.21229553222656,
      "activations/layer2_attention_weight_min": -34.485816955566406,
      "activations/layer3_attention_weight_max": 98.12850189208984,
      "activations/layer3_attention_weight_min": -96.05471801757812,
      "activations/layer4_attention_weight_max": 99.73927307128906,
      "activations/layer4_attention_weight_min": -97.73180389404297,
      "activations/layer5_attention_weight_max": 71.21507263183594,
      "activations/layer5_attention_weight_min": -73.26487731933594,
      "activations/layer6_attention_weight_max": 49.76238250732422,
      "activations/layer6_attention_weight_min": -52.83393859863281,
      "activations/layer7_attention_weight_max": 65.35183715820312,
      "activations/layer7_attention_weight_min": -66.35337829589844,
      "activations/layer8_attention_weight_max": 43.07371139526367,
      "activations/layer8_attention_weight_min": -45.48810958862305,
      "activations/layer9_attention_weight_max": 44.218875885009766,
      "activations/layer9_attention_weight_min": -45.14994430541992,
      "epoch": 13.37,
      "learning_rate": 6.442916666666666e-05,
      "loss": 2.7663,
      "step": 230100
    },
    {
      "activations/layer0_attention_weight_max": 16.048715591430664,
      "activations/layer0_attention_weight_min": -13.78017520904541,
      "activations/layer10_attention_weight_max": 35.67737579345703,
      "activations/layer10_attention_weight_min": -36.472633361816406,
      "activations/layer11_attention_weight_max": 32.619049072265625,
      "activations/layer11_attention_weight_min": -34.39656066894531,
      "activations/layer12_attention_weight_max": 19.374469757080078,
      "activations/layer12_attention_weight_min": -24.26738739013672,
      "activations/layer13_attention_weight_max": 37.077667236328125,
      "activations/layer13_attention_weight_min": -29.096986770629883,
      "activations/layer14_attention_weight_max": 38.934593200683594,
      "activations/layer14_attention_weight_min": -30.290529251098633,
      "activations/layer15_attention_weight_max": 32.24845504760742,
      "activations/layer15_attention_weight_min": -29.666900634765625,
      "activations/layer16_attention_weight_max": 31.43846321105957,
      "activations/layer16_attention_weight_min": -29.000228881835938,
      "activations/layer17_attention_weight_max": 52.89983367919922,
      "activations/layer17_attention_weight_min": -45.50747299194336,
      "activations/layer18_attention_weight_max": 46.84297180175781,
      "activations/layer18_attention_weight_min": -37.04288864135742,
      "activations/layer19_attention_weight_max": 20.466205596923828,
      "activations/layer19_attention_weight_min": -22.80478286743164,
      "activations/layer1_attention_weight_max": 16.212984085083008,
      "activations/layer1_attention_weight_min": -16.508991241455078,
      "activations/layer20_attention_weight_max": 21.841564178466797,
      "activations/layer20_attention_weight_min": -20.96927261352539,
      "activations/layer21_attention_weight_max": 32.72252655029297,
      "activations/layer21_attention_weight_min": -21.342180252075195,
      "activations/layer22_attention_weight_max": 28.753437042236328,
      "activations/layer22_attention_weight_min": -26.3731689453125,
      "activations/layer23_attention_weight_max": 37.25483703613281,
      "activations/layer23_attention_weight_min": -27.939258575439453,
      "activations/layer2_attention_weight_max": 31.978668212890625,
      "activations/layer2_attention_weight_min": -31.546512603759766,
      "activations/layer3_attention_weight_max": 96.29668426513672,
      "activations/layer3_attention_weight_min": -94.25390625,
      "activations/layer4_attention_weight_max": 98.97755432128906,
      "activations/layer4_attention_weight_min": -92.06390380859375,
      "activations/layer5_attention_weight_max": 73.14100646972656,
      "activations/layer5_attention_weight_min": -79.62632751464844,
      "activations/layer6_attention_weight_max": 50.429744720458984,
      "activations/layer6_attention_weight_min": -51.40207290649414,
      "activations/layer7_attention_weight_max": 64.19950866699219,
      "activations/layer7_attention_weight_min": -64.32050323486328,
      "activations/layer8_attention_weight_max": 42.20642852783203,
      "activations/layer8_attention_weight_min": -45.50062561035156,
      "activations/layer9_attention_weight_max": 43.31184768676758,
      "activations/layer9_attention_weight_min": -46.98501205444336,
      "epoch": 13.37,
      "learning_rate": 6.441022727272726e-05,
      "loss": 2.7554,
      "step": 230150
    },
    {
      "activations/layer0_attention_weight_max": 17.271808624267578,
      "activations/layer0_attention_weight_min": -13.478166580200195,
      "activations/layer10_attention_weight_max": 34.9920768737793,
      "activations/layer10_attention_weight_min": -35.124717712402344,
      "activations/layer11_attention_weight_max": 31.61640739440918,
      "activations/layer11_attention_weight_min": -33.59367752075195,
      "activations/layer12_attention_weight_max": 17.127357482910156,
      "activations/layer12_attention_weight_min": -23.63309669494629,
      "activations/layer13_attention_weight_max": 35.475955963134766,
      "activations/layer13_attention_weight_min": -27.47486114501953,
      "activations/layer14_attention_weight_max": 35.00645446777344,
      "activations/layer14_attention_weight_min": -31.155994415283203,
      "activations/layer15_attention_weight_max": 33.039947509765625,
      "activations/layer15_attention_weight_min": -30.169191360473633,
      "activations/layer16_attention_weight_max": 31.43387794494629,
      "activations/layer16_attention_weight_min": -31.298107147216797,
      "activations/layer17_attention_weight_max": 53.7125129699707,
      "activations/layer17_attention_weight_min": -50.309322357177734,
      "activations/layer18_attention_weight_max": 48.58352279663086,
      "activations/layer18_attention_weight_min": -40.84371566772461,
      "activations/layer19_attention_weight_max": 25.88761329650879,
      "activations/layer19_attention_weight_min": -24.49757957458496,
      "activations/layer1_attention_weight_max": 16.20014762878418,
      "activations/layer1_attention_weight_min": -15.840341567993164,
      "activations/layer20_attention_weight_max": 24.27259063720703,
      "activations/layer20_attention_weight_min": -22.50914192199707,
      "activations/layer21_attention_weight_max": 37.759239196777344,
      "activations/layer21_attention_weight_min": -25.57523536682129,
      "activations/layer22_attention_weight_max": 32.45566940307617,
      "activations/layer22_attention_weight_min": -26.532060623168945,
      "activations/layer23_attention_weight_max": 38.470367431640625,
      "activations/layer23_attention_weight_min": -26.027984619140625,
      "activations/layer2_attention_weight_max": 36.603111267089844,
      "activations/layer2_attention_weight_min": -34.46531677246094,
      "activations/layer3_attention_weight_max": 103.08954620361328,
      "activations/layer3_attention_weight_min": -100.47752380371094,
      "activations/layer4_attention_weight_max": 99.73278045654297,
      "activations/layer4_attention_weight_min": -96.95759582519531,
      "activations/layer5_attention_weight_max": 73.56227111816406,
      "activations/layer5_attention_weight_min": -78.75064086914062,
      "activations/layer6_attention_weight_max": 55.89657974243164,
      "activations/layer6_attention_weight_min": -53.49251174926758,
      "activations/layer7_attention_weight_max": 65.89388275146484,
      "activations/layer7_attention_weight_min": -66.50267028808594,
      "activations/layer8_attention_weight_max": 48.10004806518555,
      "activations/layer8_attention_weight_min": -46.26466369628906,
      "activations/layer9_attention_weight_max": 46.29429244995117,
      "activations/layer9_attention_weight_min": -45.378448486328125,
      "epoch": 13.38,
      "learning_rate": 6.439128787878787e-05,
      "loss": 2.7958,
      "step": 230200
    },
    {
      "activations/layer0_attention_weight_max": 15.795031547546387,
      "activations/layer0_attention_weight_min": -13.930259704589844,
      "activations/layer10_attention_weight_max": 38.326568603515625,
      "activations/layer10_attention_weight_min": -37.13677978515625,
      "activations/layer11_attention_weight_max": 34.85267639160156,
      "activations/layer11_attention_weight_min": -35.99256896972656,
      "activations/layer12_attention_weight_max": 23.55630111694336,
      "activations/layer12_attention_weight_min": -24.717727661132812,
      "activations/layer13_attention_weight_max": 36.48509216308594,
      "activations/layer13_attention_weight_min": -30.774765014648438,
      "activations/layer14_attention_weight_max": 36.54445266723633,
      "activations/layer14_attention_weight_min": -29.927631378173828,
      "activations/layer15_attention_weight_max": 32.90086364746094,
      "activations/layer15_attention_weight_min": -30.73011589050293,
      "activations/layer16_attention_weight_max": 31.910709381103516,
      "activations/layer16_attention_weight_min": -30.68266487121582,
      "activations/layer17_attention_weight_max": 51.907470703125,
      "activations/layer17_attention_weight_min": -45.652557373046875,
      "activations/layer18_attention_weight_max": 45.917598724365234,
      "activations/layer18_attention_weight_min": -38.24230194091797,
      "activations/layer19_attention_weight_max": 23.790023803710938,
      "activations/layer19_attention_weight_min": -22.725318908691406,
      "activations/layer1_attention_weight_max": 16.028478622436523,
      "activations/layer1_attention_weight_min": -14.461702346801758,
      "activations/layer20_attention_weight_max": 22.123062133789062,
      "activations/layer20_attention_weight_min": -22.48956871032715,
      "activations/layer21_attention_weight_max": 37.03947067260742,
      "activations/layer21_attention_weight_min": -21.263076782226562,
      "activations/layer22_attention_weight_max": 31.115158081054688,
      "activations/layer22_attention_weight_min": -24.7042293548584,
      "activations/layer23_attention_weight_max": 37.865745544433594,
      "activations/layer23_attention_weight_min": -22.198875427246094,
      "activations/layer2_attention_weight_max": 34.02056121826172,
      "activations/layer2_attention_weight_min": -33.424659729003906,
      "activations/layer3_attention_weight_max": 97.81890869140625,
      "activations/layer3_attention_weight_min": -102.52787780761719,
      "activations/layer4_attention_weight_max": 96.5595703125,
      "activations/layer4_attention_weight_min": -97.45932006835938,
      "activations/layer5_attention_weight_max": 73.69961547851562,
      "activations/layer5_attention_weight_min": -77.97538757324219,
      "activations/layer6_attention_weight_max": 52.6458854675293,
      "activations/layer6_attention_weight_min": -51.310813903808594,
      "activations/layer7_attention_weight_max": 66.49876403808594,
      "activations/layer7_attention_weight_min": -66.79979705810547,
      "activations/layer8_attention_weight_max": 47.899818420410156,
      "activations/layer8_attention_weight_min": -52.264137268066406,
      "activations/layer9_attention_weight_max": 45.57520294189453,
      "activations/layer9_attention_weight_min": -48.07686996459961,
      "epoch": 13.38,
      "learning_rate": 6.437234848484848e-05,
      "loss": 2.7762,
      "step": 230250
    },
    {
      "activations/layer0_attention_weight_max": 16.48503875732422,
      "activations/layer0_attention_weight_min": -13.908300399780273,
      "activations/layer10_attention_weight_max": 34.821537017822266,
      "activations/layer10_attention_weight_min": -34.92387771606445,
      "activations/layer11_attention_weight_max": 31.34511375427246,
      "activations/layer11_attention_weight_min": -32.30364227294922,
      "activations/layer12_attention_weight_max": 16.92151641845703,
      "activations/layer12_attention_weight_min": -24.701486587524414,
      "activations/layer13_attention_weight_max": 35.92036819458008,
      "activations/layer13_attention_weight_min": -26.89838981628418,
      "activations/layer14_attention_weight_max": 35.033424377441406,
      "activations/layer14_attention_weight_min": -29.050931930541992,
      "activations/layer15_attention_weight_max": 33.1574592590332,
      "activations/layer15_attention_weight_min": -28.831865310668945,
      "activations/layer16_attention_weight_max": 30.27484130859375,
      "activations/layer16_attention_weight_min": -28.430110931396484,
      "activations/layer17_attention_weight_max": 52.90076446533203,
      "activations/layer17_attention_weight_min": -41.730316162109375,
      "activations/layer18_attention_weight_max": 47.24467086791992,
      "activations/layer18_attention_weight_min": -37.2178955078125,
      "activations/layer19_attention_weight_max": 20.510967254638672,
      "activations/layer19_attention_weight_min": -19.72694969177246,
      "activations/layer1_attention_weight_max": 16.51753044128418,
      "activations/layer1_attention_weight_min": -15.014212608337402,
      "activations/layer20_attention_weight_max": 21.544864654541016,
      "activations/layer20_attention_weight_min": -21.685930252075195,
      "activations/layer21_attention_weight_max": 36.42542266845703,
      "activations/layer21_attention_weight_min": -20.442670822143555,
      "activations/layer22_attention_weight_max": 29.58000373840332,
      "activations/layer22_attention_weight_min": -25.23727035522461,
      "activations/layer23_attention_weight_max": 39.01254653930664,
      "activations/layer23_attention_weight_min": -22.177465438842773,
      "activations/layer2_attention_weight_max": 31.207138061523438,
      "activations/layer2_attention_weight_min": -31.879398345947266,
      "activations/layer3_attention_weight_max": 94.25509643554688,
      "activations/layer3_attention_weight_min": -98.34769439697266,
      "activations/layer4_attention_weight_max": 94.15715026855469,
      "activations/layer4_attention_weight_min": -90.46139526367188,
      "activations/layer5_attention_weight_max": 73.17060089111328,
      "activations/layer5_attention_weight_min": -76.8986587524414,
      "activations/layer6_attention_weight_max": 49.59870529174805,
      "activations/layer6_attention_weight_min": -49.792640686035156,
      "activations/layer7_attention_weight_max": 68.51692199707031,
      "activations/layer7_attention_weight_min": -63.09267044067383,
      "activations/layer8_attention_weight_max": 44.23930358886719,
      "activations/layer8_attention_weight_min": -45.79070281982422,
      "activations/layer9_attention_weight_max": 43.7354621887207,
      "activations/layer9_attention_weight_min": -45.47002029418945,
      "epoch": 13.38,
      "learning_rate": 6.435340909090908e-05,
      "loss": 2.7688,
      "step": 230300
    },
    {
      "activations/layer0_attention_weight_max": 16.950803756713867,
      "activations/layer0_attention_weight_min": -13.803146362304688,
      "activations/layer10_attention_weight_max": 37.72428894042969,
      "activations/layer10_attention_weight_min": -40.63898468017578,
      "activations/layer11_attention_weight_max": 33.613685607910156,
      "activations/layer11_attention_weight_min": -36.927555084228516,
      "activations/layer12_attention_weight_max": 19.23566436767578,
      "activations/layer12_attention_weight_min": -23.470544815063477,
      "activations/layer13_attention_weight_max": 41.318172454833984,
      "activations/layer13_attention_weight_min": -30.418039321899414,
      "activations/layer14_attention_weight_max": 39.154510498046875,
      "activations/layer14_attention_weight_min": -31.03931427001953,
      "activations/layer15_attention_weight_max": 36.57115173339844,
      "activations/layer15_attention_weight_min": -32.30577087402344,
      "activations/layer16_attention_weight_max": 37.71337127685547,
      "activations/layer16_attention_weight_min": -32.752281188964844,
      "activations/layer17_attention_weight_max": 58.56132888793945,
      "activations/layer17_attention_weight_min": -47.87688064575195,
      "activations/layer18_attention_weight_max": 52.724517822265625,
      "activations/layer18_attention_weight_min": -40.211090087890625,
      "activations/layer19_attention_weight_max": 25.203351974487305,
      "activations/layer19_attention_weight_min": -23.660247802734375,
      "activations/layer1_attention_weight_max": 17.02942657470703,
      "activations/layer1_attention_weight_min": -17.37010955810547,
      "activations/layer20_attention_weight_max": 27.35211753845215,
      "activations/layer20_attention_weight_min": -20.575456619262695,
      "activations/layer21_attention_weight_max": 44.85755157470703,
      "activations/layer21_attention_weight_min": -22.139352798461914,
      "activations/layer22_attention_weight_max": 34.88123321533203,
      "activations/layer22_attention_weight_min": -26.9953556060791,
      "activations/layer23_attention_weight_max": 45.72303771972656,
      "activations/layer23_attention_weight_min": -25.422758102416992,
      "activations/layer2_attention_weight_max": 31.963581085205078,
      "activations/layer2_attention_weight_min": -30.537303924560547,
      "activations/layer3_attention_weight_max": 95.89383697509766,
      "activations/layer3_attention_weight_min": -101.47862243652344,
      "activations/layer4_attention_weight_max": 99.82301330566406,
      "activations/layer4_attention_weight_min": -99.80928802490234,
      "activations/layer5_attention_weight_max": 78.51896667480469,
      "activations/layer5_attention_weight_min": -88.11117553710938,
      "activations/layer6_attention_weight_max": 53.37197494506836,
      "activations/layer6_attention_weight_min": -53.47834014892578,
      "activations/layer7_attention_weight_max": 69.76306915283203,
      "activations/layer7_attention_weight_min": -65.51485443115234,
      "activations/layer8_attention_weight_max": 46.4321174621582,
      "activations/layer8_attention_weight_min": -52.16890335083008,
      "activations/layer9_attention_weight_max": 45.55172348022461,
      "activations/layer9_attention_weight_min": -49.33787155151367,
      "epoch": 13.38,
      "learning_rate": 6.433446969696968e-05,
      "loss": 2.7715,
      "step": 230350
    },
    {
      "activations/layer0_attention_weight_max": 15.911993026733398,
      "activations/layer0_attention_weight_min": -13.949414253234863,
      "activations/layer10_attention_weight_max": 35.71870040893555,
      "activations/layer10_attention_weight_min": -36.3478889465332,
      "activations/layer11_attention_weight_max": 33.28162384033203,
      "activations/layer11_attention_weight_min": -33.597251892089844,
      "activations/layer12_attention_weight_max": 19.84091567993164,
      "activations/layer12_attention_weight_min": -26.853792190551758,
      "activations/layer13_attention_weight_max": 35.851863861083984,
      "activations/layer13_attention_weight_min": -26.866010665893555,
      "activations/layer14_attention_weight_max": 39.62510299682617,
      "activations/layer14_attention_weight_min": -30.641321182250977,
      "activations/layer15_attention_weight_max": 34.64772415161133,
      "activations/layer15_attention_weight_min": -30.15814208984375,
      "activations/layer16_attention_weight_max": 31.945524215698242,
      "activations/layer16_attention_weight_min": -30.168676376342773,
      "activations/layer17_attention_weight_max": 52.5197639465332,
      "activations/layer17_attention_weight_min": -40.94707489013672,
      "activations/layer18_attention_weight_max": 49.491722106933594,
      "activations/layer18_attention_weight_min": -36.39518356323242,
      "activations/layer19_attention_weight_max": 22.877395629882812,
      "activations/layer19_attention_weight_min": -20.745933532714844,
      "activations/layer1_attention_weight_max": 15.897025108337402,
      "activations/layer1_attention_weight_min": -14.5764799118042,
      "activations/layer20_attention_weight_max": 24.61806297302246,
      "activations/layer20_attention_weight_min": -20.950443267822266,
      "activations/layer21_attention_weight_max": 36.26740264892578,
      "activations/layer21_attention_weight_min": -23.652990341186523,
      "activations/layer22_attention_weight_max": 32.90863037109375,
      "activations/layer22_attention_weight_min": -25.264558792114258,
      "activations/layer23_attention_weight_max": 39.59032440185547,
      "activations/layer23_attention_weight_min": -23.56787109375,
      "activations/layer2_attention_weight_max": 31.570173263549805,
      "activations/layer2_attention_weight_min": -31.471965789794922,
      "activations/layer3_attention_weight_max": 96.80482482910156,
      "activations/layer3_attention_weight_min": -99.58061981201172,
      "activations/layer4_attention_weight_max": 99.9094467163086,
      "activations/layer4_attention_weight_min": -98.58171844482422,
      "activations/layer5_attention_weight_max": 72.08389282226562,
      "activations/layer5_attention_weight_min": -82.5562515258789,
      "activations/layer6_attention_weight_max": 52.06708908081055,
      "activations/layer6_attention_weight_min": -54.04021072387695,
      "activations/layer7_attention_weight_max": 69.51286315917969,
      "activations/layer7_attention_weight_min": -65.96234893798828,
      "activations/layer8_attention_weight_max": 46.2869873046875,
      "activations/layer8_attention_weight_min": -48.92051696777344,
      "activations/layer9_attention_weight_max": 46.539024353027344,
      "activations/layer9_attention_weight_min": -48.468101501464844,
      "epoch": 13.39,
      "learning_rate": 6.43155303030303e-05,
      "loss": 2.7587,
      "step": 230400
    },
    {
      "activations/layer0_attention_weight_max": 15.000886917114258,
      "activations/layer0_attention_weight_min": -14.132789611816406,
      "activations/layer10_attention_weight_max": 35.55451583862305,
      "activations/layer10_attention_weight_min": -37.269622802734375,
      "activations/layer11_attention_weight_max": 34.2642822265625,
      "activations/layer11_attention_weight_min": -33.43489456176758,
      "activations/layer12_attention_weight_max": 20.24041748046875,
      "activations/layer12_attention_weight_min": -25.314096450805664,
      "activations/layer13_attention_weight_max": 32.81979751586914,
      "activations/layer13_attention_weight_min": -27.35458755493164,
      "activations/layer14_attention_weight_max": 34.93827438354492,
      "activations/layer14_attention_weight_min": -28.699752807617188,
      "activations/layer15_attention_weight_max": 33.392417907714844,
      "activations/layer15_attention_weight_min": -28.49791145324707,
      "activations/layer16_attention_weight_max": 33.11839294433594,
      "activations/layer16_attention_weight_min": -30.609813690185547,
      "activations/layer17_attention_weight_max": 52.54071044921875,
      "activations/layer17_attention_weight_min": -44.80128479003906,
      "activations/layer18_attention_weight_max": 44.92548370361328,
      "activations/layer18_attention_weight_min": -39.17781448364258,
      "activations/layer19_attention_weight_max": 22.854816436767578,
      "activations/layer19_attention_weight_min": -21.127073287963867,
      "activations/layer1_attention_weight_max": 16.229671478271484,
      "activations/layer1_attention_weight_min": -13.824113845825195,
      "activations/layer20_attention_weight_max": 24.615697860717773,
      "activations/layer20_attention_weight_min": -21.152021408081055,
      "activations/layer21_attention_weight_max": 35.92665481567383,
      "activations/layer21_attention_weight_min": -21.461029052734375,
      "activations/layer22_attention_weight_max": 29.283451080322266,
      "activations/layer22_attention_weight_min": -24.591815948486328,
      "activations/layer23_attention_weight_max": 35.357765197753906,
      "activations/layer23_attention_weight_min": -24.986539840698242,
      "activations/layer2_attention_weight_max": 33.197689056396484,
      "activations/layer2_attention_weight_min": -32.41712188720703,
      "activations/layer3_attention_weight_max": 100.84307098388672,
      "activations/layer3_attention_weight_min": -98.03652954101562,
      "activations/layer4_attention_weight_max": 94.15098571777344,
      "activations/layer4_attention_weight_min": -93.89366912841797,
      "activations/layer5_attention_weight_max": 69.10202026367188,
      "activations/layer5_attention_weight_min": -78.4471435546875,
      "activations/layer6_attention_weight_max": 49.66730880737305,
      "activations/layer6_attention_weight_min": -51.10178756713867,
      "activations/layer7_attention_weight_max": 67.72644805908203,
      "activations/layer7_attention_weight_min": -61.82684326171875,
      "activations/layer8_attention_weight_max": 44.357688903808594,
      "activations/layer8_attention_weight_min": -47.66817092895508,
      "activations/layer9_attention_weight_max": 51.5934944152832,
      "activations/layer9_attention_weight_min": -46.96776580810547,
      "epoch": 13.39,
      "learning_rate": 6.42965909090909e-05,
      "loss": 2.7652,
      "step": 230450
    },
    {
      "activations/layer0_attention_weight_max": 15.665151596069336,
      "activations/layer0_attention_weight_min": -14.724172592163086,
      "activations/layer10_attention_weight_max": 36.69622039794922,
      "activations/layer10_attention_weight_min": -33.403343200683594,
      "activations/layer11_attention_weight_max": 32.50144958496094,
      "activations/layer11_attention_weight_min": -31.646499633789062,
      "activations/layer12_attention_weight_max": 18.43996810913086,
      "activations/layer12_attention_weight_min": -22.389358520507812,
      "activations/layer13_attention_weight_max": 35.59284210205078,
      "activations/layer13_attention_weight_min": -28.048431396484375,
      "activations/layer14_attention_weight_max": 36.50868606567383,
      "activations/layer14_attention_weight_min": -30.485857009887695,
      "activations/layer15_attention_weight_max": 33.041969299316406,
      "activations/layer15_attention_weight_min": -29.815576553344727,
      "activations/layer16_attention_weight_max": 31.14296531677246,
      "activations/layer16_attention_weight_min": -29.8243408203125,
      "activations/layer17_attention_weight_max": 50.08257293701172,
      "activations/layer17_attention_weight_min": -41.02271270751953,
      "activations/layer18_attention_weight_max": 50.1370964050293,
      "activations/layer18_attention_weight_min": -38.2004508972168,
      "activations/layer19_attention_weight_max": 24.535062789916992,
      "activations/layer19_attention_weight_min": -20.713884353637695,
      "activations/layer1_attention_weight_max": 16.2470760345459,
      "activations/layer1_attention_weight_min": -14.8183012008667,
      "activations/layer20_attention_weight_max": 24.563859939575195,
      "activations/layer20_attention_weight_min": -22.090110778808594,
      "activations/layer21_attention_weight_max": 46.522735595703125,
      "activations/layer21_attention_weight_min": -26.669816970825195,
      "activations/layer22_attention_weight_max": 30.86128807067871,
      "activations/layer22_attention_weight_min": -26.63082504272461,
      "activations/layer23_attention_weight_max": 41.07918167114258,
      "activations/layer23_attention_weight_min": -24.772544860839844,
      "activations/layer2_attention_weight_max": 28.013992309570312,
      "activations/layer2_attention_weight_min": -26.652694702148438,
      "activations/layer3_attention_weight_max": 91.0485610961914,
      "activations/layer3_attention_weight_min": -87.56278228759766,
      "activations/layer4_attention_weight_max": 95.38216400146484,
      "activations/layer4_attention_weight_min": -90.15484619140625,
      "activations/layer5_attention_weight_max": 71.34073638916016,
      "activations/layer5_attention_weight_min": -78.14102935791016,
      "activations/layer6_attention_weight_max": 53.25381088256836,
      "activations/layer6_attention_weight_min": -49.67305374145508,
      "activations/layer7_attention_weight_max": 65.30360412597656,
      "activations/layer7_attention_weight_min": -64.73472595214844,
      "activations/layer8_attention_weight_max": 45.44966506958008,
      "activations/layer8_attention_weight_min": -47.86980056762695,
      "activations/layer9_attention_weight_max": 46.330787658691406,
      "activations/layer9_attention_weight_min": -49.83811569213867,
      "epoch": 13.39,
      "learning_rate": 6.427803030303029e-05,
      "loss": 2.7584,
      "step": 230500
    },
    {
      "activations/layer0_attention_weight_max": 16.660568237304688,
      "activations/layer0_attention_weight_min": -14.20838737487793,
      "activations/layer10_attention_weight_max": 35.77643966674805,
      "activations/layer10_attention_weight_min": -38.14581298828125,
      "activations/layer11_attention_weight_max": 33.21534729003906,
      "activations/layer11_attention_weight_min": -36.021026611328125,
      "activations/layer12_attention_weight_max": 17.869102478027344,
      "activations/layer12_attention_weight_min": -24.01371192932129,
      "activations/layer13_attention_weight_max": 37.87065505981445,
      "activations/layer13_attention_weight_min": -32.06795883178711,
      "activations/layer14_attention_weight_max": 34.66661071777344,
      "activations/layer14_attention_weight_min": -31.021696090698242,
      "activations/layer15_attention_weight_max": 31.1262149810791,
      "activations/layer15_attention_weight_min": -32.861328125,
      "activations/layer16_attention_weight_max": 29.530412673950195,
      "activations/layer16_attention_weight_min": -30.763288497924805,
      "activations/layer17_attention_weight_max": 51.797733306884766,
      "activations/layer17_attention_weight_min": -44.469390869140625,
      "activations/layer18_attention_weight_max": 45.62885284423828,
      "activations/layer18_attention_weight_min": -38.39204025268555,
      "activations/layer19_attention_weight_max": 21.947851181030273,
      "activations/layer19_attention_weight_min": -22.07962417602539,
      "activations/layer1_attention_weight_max": 16.325578689575195,
      "activations/layer1_attention_weight_min": -15.479181289672852,
      "activations/layer20_attention_weight_max": 20.274993896484375,
      "activations/layer20_attention_weight_min": -21.975812911987305,
      "activations/layer21_attention_weight_max": 34.31224060058594,
      "activations/layer21_attention_weight_min": -21.240427017211914,
      "activations/layer22_attention_weight_max": 27.05501365661621,
      "activations/layer22_attention_weight_min": -25.414087295532227,
      "activations/layer23_attention_weight_max": 34.587711334228516,
      "activations/layer23_attention_weight_min": -26.187713623046875,
      "activations/layer2_attention_weight_max": 30.961036682128906,
      "activations/layer2_attention_weight_min": -30.00365447998047,
      "activations/layer3_attention_weight_max": 93.56318664550781,
      "activations/layer3_attention_weight_min": -93.58537292480469,
      "activations/layer4_attention_weight_max": 98.85494995117188,
      "activations/layer4_attention_weight_min": -93.06023406982422,
      "activations/layer5_attention_weight_max": 72.31998443603516,
      "activations/layer5_attention_weight_min": -80.18019104003906,
      "activations/layer6_attention_weight_max": 52.48728942871094,
      "activations/layer6_attention_weight_min": -51.71355438232422,
      "activations/layer7_attention_weight_max": 66.44583129882812,
      "activations/layer7_attention_weight_min": -66.87848663330078,
      "activations/layer8_attention_weight_max": 45.73310089111328,
      "activations/layer8_attention_weight_min": -49.94792556762695,
      "activations/layer9_attention_weight_max": 46.50702667236328,
      "activations/layer9_attention_weight_min": -48.114410400390625,
      "epoch": 13.4,
      "learning_rate": 6.42590909090909e-05,
      "loss": 2.7505,
      "step": 230550
    },
    {
      "activations/layer0_attention_weight_max": 17.007387161254883,
      "activations/layer0_attention_weight_min": -14.331157684326172,
      "activations/layer10_attention_weight_max": 36.75291442871094,
      "activations/layer10_attention_weight_min": -36.76670837402344,
      "activations/layer11_attention_weight_max": 32.27347183227539,
      "activations/layer11_attention_weight_min": -36.01919174194336,
      "activations/layer12_attention_weight_max": 19.27176856994629,
      "activations/layer12_attention_weight_min": -27.70899200439453,
      "activations/layer13_attention_weight_max": 37.76286315917969,
      "activations/layer13_attention_weight_min": -30.24846839904785,
      "activations/layer14_attention_weight_max": 38.15656661987305,
      "activations/layer14_attention_weight_min": -29.656055450439453,
      "activations/layer15_attention_weight_max": 32.67823028564453,
      "activations/layer15_attention_weight_min": -29.995298385620117,
      "activations/layer16_attention_weight_max": 33.5914192199707,
      "activations/layer16_attention_weight_min": -31.07084083557129,
      "activations/layer17_attention_weight_max": 55.04174041748047,
      "activations/layer17_attention_weight_min": -43.43224334716797,
      "activations/layer18_attention_weight_max": 49.802364349365234,
      "activations/layer18_attention_weight_min": -37.97736740112305,
      "activations/layer19_attention_weight_max": 22.48021125793457,
      "activations/layer19_attention_weight_min": -23.640901565551758,
      "activations/layer1_attention_weight_max": 16.087675094604492,
      "activations/layer1_attention_weight_min": -15.18409252166748,
      "activations/layer20_attention_weight_max": 23.319608688354492,
      "activations/layer20_attention_weight_min": -22.599382400512695,
      "activations/layer21_attention_weight_max": 36.80794143676758,
      "activations/layer21_attention_weight_min": -25.98360252380371,
      "activations/layer22_attention_weight_max": 27.56673240661621,
      "activations/layer22_attention_weight_min": -25.933002471923828,
      "activations/layer23_attention_weight_max": 37.995548248291016,
      "activations/layer23_attention_weight_min": -26.419052124023438,
      "activations/layer2_attention_weight_max": 30.92458152770996,
      "activations/layer2_attention_weight_min": -29.429357528686523,
      "activations/layer3_attention_weight_max": 91.4134521484375,
      "activations/layer3_attention_weight_min": -95.14766693115234,
      "activations/layer4_attention_weight_max": 96.71439361572266,
      "activations/layer4_attention_weight_min": -97.64872741699219,
      "activations/layer5_attention_weight_max": 71.25849914550781,
      "activations/layer5_attention_weight_min": -75.83432006835938,
      "activations/layer6_attention_weight_max": 50.92228698730469,
      "activations/layer6_attention_weight_min": -53.027835845947266,
      "activations/layer7_attention_weight_max": 64.85733795166016,
      "activations/layer7_attention_weight_min": -66.05115509033203,
      "activations/layer8_attention_weight_max": 48.1566047668457,
      "activations/layer8_attention_weight_min": -48.8997917175293,
      "activations/layer9_attention_weight_max": 46.99422073364258,
      "activations/layer9_attention_weight_min": -48.1966667175293,
      "epoch": 13.4,
      "learning_rate": 6.424015151515151e-05,
      "loss": 2.7756,
      "step": 230600
    },
    {
      "activations/layer0_attention_weight_max": 16.79717254638672,
      "activations/layer0_attention_weight_min": -14.525975227355957,
      "activations/layer10_attention_weight_max": 34.555904388427734,
      "activations/layer10_attention_weight_min": -34.17303466796875,
      "activations/layer11_attention_weight_max": 31.189132690429688,
      "activations/layer11_attention_weight_min": -31.90736961364746,
      "activations/layer12_attention_weight_max": 17.259281158447266,
      "activations/layer12_attention_weight_min": -27.16764259338379,
      "activations/layer13_attention_weight_max": 28.756017684936523,
      "activations/layer13_attention_weight_min": -28.551530838012695,
      "activations/layer14_attention_weight_max": 31.533855438232422,
      "activations/layer14_attention_weight_min": -28.113554000854492,
      "activations/layer15_attention_weight_max": 31.22200584411621,
      "activations/layer15_attention_weight_min": -28.37666130065918,
      "activations/layer16_attention_weight_max": 27.58565902709961,
      "activations/layer16_attention_weight_min": -27.140731811523438,
      "activations/layer17_attention_weight_max": 46.250213623046875,
      "activations/layer17_attention_weight_min": -40.80594253540039,
      "activations/layer18_attention_weight_max": 39.15630340576172,
      "activations/layer18_attention_weight_min": -35.717403411865234,
      "activations/layer19_attention_weight_max": 20.70990753173828,
      "activations/layer19_attention_weight_min": -23.006057739257812,
      "activations/layer1_attention_weight_max": 16.15958595275879,
      "activations/layer1_attention_weight_min": -15.536482810974121,
      "activations/layer20_attention_weight_max": 19.784482955932617,
      "activations/layer20_attention_weight_min": -20.743328094482422,
      "activations/layer21_attention_weight_max": 31.82562828063965,
      "activations/layer21_attention_weight_min": -21.727121353149414,
      "activations/layer22_attention_weight_max": 25.822446823120117,
      "activations/layer22_attention_weight_min": -26.48626136779785,
      "activations/layer23_attention_weight_max": 33.62447738647461,
      "activations/layer23_attention_weight_min": -25.01679039001465,
      "activations/layer2_attention_weight_max": 32.165931701660156,
      "activations/layer2_attention_weight_min": -31.042783737182617,
      "activations/layer3_attention_weight_max": 95.3096923828125,
      "activations/layer3_attention_weight_min": -92.19306182861328,
      "activations/layer4_attention_weight_max": 94.87092590332031,
      "activations/layer4_attention_weight_min": -92.02149963378906,
      "activations/layer5_attention_weight_max": 67.87886810302734,
      "activations/layer5_attention_weight_min": -80.12515258789062,
      "activations/layer6_attention_weight_max": 50.58101272583008,
      "activations/layer6_attention_weight_min": -51.31544494628906,
      "activations/layer7_attention_weight_max": 64.47987365722656,
      "activations/layer7_attention_weight_min": -64.20310974121094,
      "activations/layer8_attention_weight_max": 42.23536682128906,
      "activations/layer8_attention_weight_min": -46.4587287902832,
      "activations/layer9_attention_weight_max": 43.46055221557617,
      "activations/layer9_attention_weight_min": -45.41831588745117,
      "epoch": 13.4,
      "learning_rate": 6.422121212121212e-05,
      "loss": 2.7568,
      "step": 230650
    },
    {
      "activations/layer0_attention_weight_max": 15.380311965942383,
      "activations/layer0_attention_weight_min": -14.220247268676758,
      "activations/layer10_attention_weight_max": 35.39631652832031,
      "activations/layer10_attention_weight_min": -36.22444534301758,
      "activations/layer11_attention_weight_max": 31.324399948120117,
      "activations/layer11_attention_weight_min": -33.29255294799805,
      "activations/layer12_attention_weight_max": 19.13968849182129,
      "activations/layer12_attention_weight_min": -25.990571975708008,
      "activations/layer13_attention_weight_max": 34.8624267578125,
      "activations/layer13_attention_weight_min": -28.542423248291016,
      "activations/layer14_attention_weight_max": 36.45408248901367,
      "activations/layer14_attention_weight_min": -29.875444412231445,
      "activations/layer15_attention_weight_max": 35.215084075927734,
      "activations/layer15_attention_weight_min": -28.3085994720459,
      "activations/layer16_attention_weight_max": 34.964420318603516,
      "activations/layer16_attention_weight_min": -31.24433135986328,
      "activations/layer17_attention_weight_max": 53.30152893066406,
      "activations/layer17_attention_weight_min": -45.97612762451172,
      "activations/layer18_attention_weight_max": 49.27794647216797,
      "activations/layer18_attention_weight_min": -38.92811584472656,
      "activations/layer19_attention_weight_max": 23.12895393371582,
      "activations/layer19_attention_weight_min": -23.91315269470215,
      "activations/layer1_attention_weight_max": 17.55190086364746,
      "activations/layer1_attention_weight_min": -15.152220726013184,
      "activations/layer20_attention_weight_max": 23.513540267944336,
      "activations/layer20_attention_weight_min": -22.813451766967773,
      "activations/layer21_attention_weight_max": 40.39592742919922,
      "activations/layer21_attention_weight_min": -26.316259384155273,
      "activations/layer22_attention_weight_max": 36.55763244628906,
      "activations/layer22_attention_weight_min": -25.947189331054688,
      "activations/layer23_attention_weight_max": 40.71392059326172,
      "activations/layer23_attention_weight_min": -24.13412857055664,
      "activations/layer2_attention_weight_max": 33.11822509765625,
      "activations/layer2_attention_weight_min": -30.91714096069336,
      "activations/layer3_attention_weight_max": 96.39424133300781,
      "activations/layer3_attention_weight_min": -96.37641906738281,
      "activations/layer4_attention_weight_max": 98.91939544677734,
      "activations/layer4_attention_weight_min": -94.5951919555664,
      "activations/layer5_attention_weight_max": 70.71421813964844,
      "activations/layer5_attention_weight_min": -79.14006042480469,
      "activations/layer6_attention_weight_max": 54.67466354370117,
      "activations/layer6_attention_weight_min": -53.26039123535156,
      "activations/layer7_attention_weight_max": 66.93799591064453,
      "activations/layer7_attention_weight_min": -67.5616455078125,
      "activations/layer8_attention_weight_max": 46.59072494506836,
      "activations/layer8_attention_weight_min": -47.98603439331055,
      "activations/layer9_attention_weight_max": 44.38273239135742,
      "activations/layer9_attention_weight_min": -47.03658676147461,
      "epoch": 13.4,
      "learning_rate": 6.420227272727272e-05,
      "loss": 2.7677,
      "step": 230700
    },
    {
      "activations/layer0_attention_weight_max": 15.479995727539062,
      "activations/layer0_attention_weight_min": -14.238490104675293,
      "activations/layer10_attention_weight_max": 34.3656120300293,
      "activations/layer10_attention_weight_min": -35.37836837768555,
      "activations/layer11_attention_weight_max": 30.180313110351562,
      "activations/layer11_attention_weight_min": -33.284812927246094,
      "activations/layer12_attention_weight_max": 17.472827911376953,
      "activations/layer12_attention_weight_min": -22.721067428588867,
      "activations/layer13_attention_weight_max": 35.58320617675781,
      "activations/layer13_attention_weight_min": -31.494205474853516,
      "activations/layer14_attention_weight_max": 39.45912551879883,
      "activations/layer14_attention_weight_min": -29.299108505249023,
      "activations/layer15_attention_weight_max": 34.66805648803711,
      "activations/layer15_attention_weight_min": -33.08582305908203,
      "activations/layer16_attention_weight_max": 35.536190032958984,
      "activations/layer16_attention_weight_min": -31.479236602783203,
      "activations/layer17_attention_weight_max": 54.84384536743164,
      "activations/layer17_attention_weight_min": -46.340145111083984,
      "activations/layer18_attention_weight_max": 44.61779022216797,
      "activations/layer18_attention_weight_min": -41.268699645996094,
      "activations/layer19_attention_weight_max": 23.060874938964844,
      "activations/layer19_attention_weight_min": -21.402067184448242,
      "activations/layer1_attention_weight_max": 16.537477493286133,
      "activations/layer1_attention_weight_min": -14.78779411315918,
      "activations/layer20_attention_weight_max": 26.91197967529297,
      "activations/layer20_attention_weight_min": -22.246435165405273,
      "activations/layer21_attention_weight_max": 34.44716262817383,
      "activations/layer21_attention_weight_min": -24.695110321044922,
      "activations/layer22_attention_weight_max": 32.04123306274414,
      "activations/layer22_attention_weight_min": -25.59092140197754,
      "activations/layer23_attention_weight_max": 43.984004974365234,
      "activations/layer23_attention_weight_min": -26.730247497558594,
      "activations/layer2_attention_weight_max": 33.45897674560547,
      "activations/layer2_attention_weight_min": -31.003751754760742,
      "activations/layer3_attention_weight_max": 96.41143035888672,
      "activations/layer3_attention_weight_min": -101.96721649169922,
      "activations/layer4_attention_weight_max": 93.79024505615234,
      "activations/layer4_attention_weight_min": -97.1227798461914,
      "activations/layer5_attention_weight_max": 71.33524322509766,
      "activations/layer5_attention_weight_min": -80.88970184326172,
      "activations/layer6_attention_weight_max": 51.875526428222656,
      "activations/layer6_attention_weight_min": -50.121944427490234,
      "activations/layer7_attention_weight_max": 63.15665054321289,
      "activations/layer7_attention_weight_min": -63.6431770324707,
      "activations/layer8_attention_weight_max": 43.79818344116211,
      "activations/layer8_attention_weight_min": -46.10131072998047,
      "activations/layer9_attention_weight_max": 42.999267578125,
      "activations/layer9_attention_weight_min": -44.581241607666016,
      "epoch": 13.41,
      "learning_rate": 6.418333333333333e-05,
      "loss": 2.7608,
      "step": 230750
    },
    {
      "activations/layer0_attention_weight_max": 17.18384552001953,
      "activations/layer0_attention_weight_min": -14.06292724609375,
      "activations/layer10_attention_weight_max": 37.45734405517578,
      "activations/layer10_attention_weight_min": -36.41319274902344,
      "activations/layer11_attention_weight_max": 33.96043014526367,
      "activations/layer11_attention_weight_min": -34.55786895751953,
      "activations/layer12_attention_weight_max": 18.092487335205078,
      "activations/layer12_attention_weight_min": -27.970664978027344,
      "activations/layer13_attention_weight_max": 33.86027145385742,
      "activations/layer13_attention_weight_min": -35.64384841918945,
      "activations/layer14_attention_weight_max": 38.23056411743164,
      "activations/layer14_attention_weight_min": -31.6721134185791,
      "activations/layer15_attention_weight_max": 31.649343490600586,
      "activations/layer15_attention_weight_min": -29.744504928588867,
      "activations/layer16_attention_weight_max": 32.1622428894043,
      "activations/layer16_attention_weight_min": -32.06961441040039,
      "activations/layer17_attention_weight_max": 55.241153717041016,
      "activations/layer17_attention_weight_min": -45.43495559692383,
      "activations/layer18_attention_weight_max": 46.34555435180664,
      "activations/layer18_attention_weight_min": -39.3801383972168,
      "activations/layer19_attention_weight_max": 22.982925415039062,
      "activations/layer19_attention_weight_min": -22.675621032714844,
      "activations/layer1_attention_weight_max": 16.71566390991211,
      "activations/layer1_attention_weight_min": -15.475066184997559,
      "activations/layer20_attention_weight_max": 23.500791549682617,
      "activations/layer20_attention_weight_min": -24.686166763305664,
      "activations/layer21_attention_weight_max": 36.50804138183594,
      "activations/layer21_attention_weight_min": -27.181638717651367,
      "activations/layer22_attention_weight_max": 32.095611572265625,
      "activations/layer22_attention_weight_min": -25.90312957763672,
      "activations/layer23_attention_weight_max": 38.524261474609375,
      "activations/layer23_attention_weight_min": -25.301475524902344,
      "activations/layer2_attention_weight_max": 33.20218276977539,
      "activations/layer2_attention_weight_min": -30.919355392456055,
      "activations/layer3_attention_weight_max": 97.05534362792969,
      "activations/layer3_attention_weight_min": -95.02411651611328,
      "activations/layer4_attention_weight_max": 95.5431137084961,
      "activations/layer4_attention_weight_min": -95.69292449951172,
      "activations/layer5_attention_weight_max": 73.75930786132812,
      "activations/layer5_attention_weight_min": -85.58755493164062,
      "activations/layer6_attention_weight_max": 51.187747955322266,
      "activations/layer6_attention_weight_min": -52.007057189941406,
      "activations/layer7_attention_weight_max": 68.66136169433594,
      "activations/layer7_attention_weight_min": -66.8667984008789,
      "activations/layer8_attention_weight_max": 47.3469123840332,
      "activations/layer8_attention_weight_min": -47.82271957397461,
      "activations/layer9_attention_weight_max": 50.81403732299805,
      "activations/layer9_attention_weight_min": -47.96759796142578,
      "epoch": 13.41,
      "learning_rate": 6.416439393939394e-05,
      "loss": 2.7628,
      "step": 230800
    },
    {
      "activations/layer0_attention_weight_max": 16.205846786499023,
      "activations/layer0_attention_weight_min": -14.619837760925293,
      "activations/layer10_attention_weight_max": 35.1712646484375,
      "activations/layer10_attention_weight_min": -36.03095626831055,
      "activations/layer11_attention_weight_max": 32.01393127441406,
      "activations/layer11_attention_weight_min": -32.55593490600586,
      "activations/layer12_attention_weight_max": 19.09097671508789,
      "activations/layer12_attention_weight_min": -32.03813934326172,
      "activations/layer13_attention_weight_max": 43.44548034667969,
      "activations/layer13_attention_weight_min": -29.28704261779785,
      "activations/layer14_attention_weight_max": 39.478065490722656,
      "activations/layer14_attention_weight_min": -28.631742477416992,
      "activations/layer15_attention_weight_max": 33.80675506591797,
      "activations/layer15_attention_weight_min": -28.36920166015625,
      "activations/layer16_attention_weight_max": 32.77499771118164,
      "activations/layer16_attention_weight_min": -30.33184814453125,
      "activations/layer17_attention_weight_max": 55.41265106201172,
      "activations/layer17_attention_weight_min": -44.07301712036133,
      "activations/layer18_attention_weight_max": 48.3325309753418,
      "activations/layer18_attention_weight_min": -39.82481002807617,
      "activations/layer19_attention_weight_max": 23.760541915893555,
      "activations/layer19_attention_weight_min": -22.40183448791504,
      "activations/layer1_attention_weight_max": 16.617141723632812,
      "activations/layer1_attention_weight_min": -15.838884353637695,
      "activations/layer20_attention_weight_max": 24.216367721557617,
      "activations/layer20_attention_weight_min": -20.99009132385254,
      "activations/layer21_attention_weight_max": 40.293067932128906,
      "activations/layer21_attention_weight_min": -22.176355361938477,
      "activations/layer22_attention_weight_max": 33.71378707885742,
      "activations/layer22_attention_weight_min": -26.446430206298828,
      "activations/layer23_attention_weight_max": 42.60020065307617,
      "activations/layer23_attention_weight_min": -25.529743194580078,
      "activations/layer2_attention_weight_max": 32.85090637207031,
      "activations/layer2_attention_weight_min": -32.27560043334961,
      "activations/layer3_attention_weight_max": 101.39728546142578,
      "activations/layer3_attention_weight_min": -104.02579498291016,
      "activations/layer4_attention_weight_max": 100.01544952392578,
      "activations/layer4_attention_weight_min": -96.1836166381836,
      "activations/layer5_attention_weight_max": 69.585693359375,
      "activations/layer5_attention_weight_min": -77.8511962890625,
      "activations/layer6_attention_weight_max": 52.50002670288086,
      "activations/layer6_attention_weight_min": -53.69619369506836,
      "activations/layer7_attention_weight_max": 68.38555145263672,
      "activations/layer7_attention_weight_min": -65.4287109375,
      "activations/layer8_attention_weight_max": 45.086639404296875,
      "activations/layer8_attention_weight_min": -53.64480209350586,
      "activations/layer9_attention_weight_max": 50.72859573364258,
      "activations/layer9_attention_weight_min": -47.64141845703125,
      "epoch": 13.41,
      "learning_rate": 6.414545454545454e-05,
      "loss": 2.767,
      "step": 230850
    },
    {
      "activations/layer0_attention_weight_max": 15.457645416259766,
      "activations/layer0_attention_weight_min": -14.529808044433594,
      "activations/layer10_attention_weight_max": 35.75546646118164,
      "activations/layer10_attention_weight_min": -34.5623779296875,
      "activations/layer11_attention_weight_max": 32.45172882080078,
      "activations/layer11_attention_weight_min": -32.06598663330078,
      "activations/layer12_attention_weight_max": 20.193836212158203,
      "activations/layer12_attention_weight_min": -23.24592399597168,
      "activations/layer13_attention_weight_max": 48.695796966552734,
      "activations/layer13_attention_weight_min": -30.53635597229004,
      "activations/layer14_attention_weight_max": 40.31230545043945,
      "activations/layer14_attention_weight_min": -29.137792587280273,
      "activations/layer15_attention_weight_max": 36.576026916503906,
      "activations/layer15_attention_weight_min": -29.418968200683594,
      "activations/layer16_attention_weight_max": 33.6376953125,
      "activations/layer16_attention_weight_min": -30.10710334777832,
      "activations/layer17_attention_weight_max": 56.852455139160156,
      "activations/layer17_attention_weight_min": -44.08690643310547,
      "activations/layer18_attention_weight_max": 53.48981857299805,
      "activations/layer18_attention_weight_min": -36.86260223388672,
      "activations/layer19_attention_weight_max": 24.341047286987305,
      "activations/layer19_attention_weight_min": -22.270559310913086,
      "activations/layer1_attention_weight_max": 18.01520347595215,
      "activations/layer1_attention_weight_min": -15.360235214233398,
      "activations/layer20_attention_weight_max": 27.421539306640625,
      "activations/layer20_attention_weight_min": -20.75821304321289,
      "activations/layer21_attention_weight_max": 41.3117790222168,
      "activations/layer21_attention_weight_min": -25.78388023376465,
      "activations/layer22_attention_weight_max": 35.48103332519531,
      "activations/layer22_attention_weight_min": -27.36139488220215,
      "activations/layer23_attention_weight_max": 40.98173522949219,
      "activations/layer23_attention_weight_min": -24.527423858642578,
      "activations/layer2_attention_weight_max": 32.236289978027344,
      "activations/layer2_attention_weight_min": -30.890623092651367,
      "activations/layer3_attention_weight_max": 92.7764663696289,
      "activations/layer3_attention_weight_min": -95.57103729248047,
      "activations/layer4_attention_weight_max": 93.5382080078125,
      "activations/layer4_attention_weight_min": -92.13468170166016,
      "activations/layer5_attention_weight_max": 68.94404602050781,
      "activations/layer5_attention_weight_min": -76.48334503173828,
      "activations/layer6_attention_weight_max": 52.23728942871094,
      "activations/layer6_attention_weight_min": -49.87013626098633,
      "activations/layer7_attention_weight_max": 64.74855041503906,
      "activations/layer7_attention_weight_min": -64.10758209228516,
      "activations/layer8_attention_weight_max": 45.16423797607422,
      "activations/layer8_attention_weight_min": -46.475582122802734,
      "activations/layer9_attention_weight_max": 42.82950973510742,
      "activations/layer9_attention_weight_min": -45.80495834350586,
      "epoch": 13.42,
      "learning_rate": 6.412651515151514e-05,
      "loss": 2.7462,
      "step": 230900
    },
    {
      "activations/layer0_attention_weight_max": 16.185569763183594,
      "activations/layer0_attention_weight_min": -14.100017547607422,
      "activations/layer10_attention_weight_max": 34.7410774230957,
      "activations/layer10_attention_weight_min": -34.45149612426758,
      "activations/layer11_attention_weight_max": 31.71218490600586,
      "activations/layer11_attention_weight_min": -34.228515625,
      "activations/layer12_attention_weight_max": 19.321441650390625,
      "activations/layer12_attention_weight_min": -25.15568733215332,
      "activations/layer13_attention_weight_max": 38.8509635925293,
      "activations/layer13_attention_weight_min": -30.39879608154297,
      "activations/layer14_attention_weight_max": 34.31576156616211,
      "activations/layer14_attention_weight_min": -30.36298179626465,
      "activations/layer15_attention_weight_max": 31.739450454711914,
      "activations/layer15_attention_weight_min": -28.517982482910156,
      "activations/layer16_attention_weight_max": 28.489469528198242,
      "activations/layer16_attention_weight_min": -28.129291534423828,
      "activations/layer17_attention_weight_max": 46.972984313964844,
      "activations/layer17_attention_weight_min": -43.67991256713867,
      "activations/layer18_attention_weight_max": 44.092864990234375,
      "activations/layer18_attention_weight_min": -38.774986267089844,
      "activations/layer19_attention_weight_max": 22.172834396362305,
      "activations/layer19_attention_weight_min": -21.19114112854004,
      "activations/layer1_attention_weight_max": 16.831289291381836,
      "activations/layer1_attention_weight_min": -14.087214469909668,
      "activations/layer20_attention_weight_max": 20.52390480041504,
      "activations/layer20_attention_weight_min": -22.9067440032959,
      "activations/layer21_attention_weight_max": 36.28036880493164,
      "activations/layer21_attention_weight_min": -23.824548721313477,
      "activations/layer22_attention_weight_max": 33.52713394165039,
      "activations/layer22_attention_weight_min": -26.01897621154785,
      "activations/layer23_attention_weight_max": 37.591575622558594,
      "activations/layer23_attention_weight_min": -22.922229766845703,
      "activations/layer2_attention_weight_max": 31.16175651550293,
      "activations/layer2_attention_weight_min": -31.841062545776367,
      "activations/layer3_attention_weight_max": 93.32222747802734,
      "activations/layer3_attention_weight_min": -99.09468841552734,
      "activations/layer4_attention_weight_max": 94.3539047241211,
      "activations/layer4_attention_weight_min": -91.15946197509766,
      "activations/layer5_attention_weight_max": 72.86880493164062,
      "activations/layer5_attention_weight_min": -77.77115631103516,
      "activations/layer6_attention_weight_max": 50.464351654052734,
      "activations/layer6_attention_weight_min": -52.08271789550781,
      "activations/layer7_attention_weight_max": 67.83497619628906,
      "activations/layer7_attention_weight_min": -61.68787384033203,
      "activations/layer8_attention_weight_max": 43.624568939208984,
      "activations/layer8_attention_weight_min": -44.78092575073242,
      "activations/layer9_attention_weight_max": 46.13664627075195,
      "activations/layer9_attention_weight_min": -45.90233612060547,
      "epoch": 13.42,
      "learning_rate": 6.410757575757575e-05,
      "loss": 2.7697,
      "step": 230950
    },
    {
      "activations/layer0_attention_weight_max": 15.992841720581055,
      "activations/layer0_attention_weight_min": -14.171308517456055,
      "activations/layer10_attention_weight_max": 34.6605110168457,
      "activations/layer10_attention_weight_min": -33.07209014892578,
      "activations/layer11_attention_weight_max": 29.65792465209961,
      "activations/layer11_attention_weight_min": -31.089405059814453,
      "activations/layer12_attention_weight_max": 17.556854248046875,
      "activations/layer12_attention_weight_min": -26.6861629486084,
      "activations/layer13_attention_weight_max": 34.863624572753906,
      "activations/layer13_attention_weight_min": -26.59971809387207,
      "activations/layer14_attention_weight_max": 36.71364974975586,
      "activations/layer14_attention_weight_min": -29.672420501708984,
      "activations/layer15_attention_weight_max": 31.631114959716797,
      "activations/layer15_attention_weight_min": -28.069671630859375,
      "activations/layer16_attention_weight_max": 28.3431453704834,
      "activations/layer16_attention_weight_min": -29.16804313659668,
      "activations/layer17_attention_weight_max": 48.14792251586914,
      "activations/layer17_attention_weight_min": -42.9747200012207,
      "activations/layer18_attention_weight_max": 44.158870697021484,
      "activations/layer18_attention_weight_min": -37.30076217651367,
      "activations/layer19_attention_weight_max": 20.683746337890625,
      "activations/layer19_attention_weight_min": -21.658456802368164,
      "activations/layer1_attention_weight_max": 17.19832992553711,
      "activations/layer1_attention_weight_min": -15.360601425170898,
      "activations/layer20_attention_weight_max": 21.88190269470215,
      "activations/layer20_attention_weight_min": -21.78230094909668,
      "activations/layer21_attention_weight_max": 33.9582405090332,
      "activations/layer21_attention_weight_min": -28.526639938354492,
      "activations/layer22_attention_weight_max": 32.331138610839844,
      "activations/layer22_attention_weight_min": -25.926504135131836,
      "activations/layer23_attention_weight_max": 37.39854431152344,
      "activations/layer23_attention_weight_min": -25.988677978515625,
      "activations/layer2_attention_weight_max": 31.245197296142578,
      "activations/layer2_attention_weight_min": -30.457677841186523,
      "activations/layer3_attention_weight_max": 93.83516693115234,
      "activations/layer3_attention_weight_min": -99.30560302734375,
      "activations/layer4_attention_weight_max": 97.02053833007812,
      "activations/layer4_attention_weight_min": -95.29206085205078,
      "activations/layer5_attention_weight_max": 72.09529113769531,
      "activations/layer5_attention_weight_min": -78.33918762207031,
      "activations/layer6_attention_weight_max": 52.52729415893555,
      "activations/layer6_attention_weight_min": -53.2150993347168,
      "activations/layer7_attention_weight_max": 70.0992202758789,
      "activations/layer7_attention_weight_min": -66.14668273925781,
      "activations/layer8_attention_weight_max": 44.41774368286133,
      "activations/layer8_attention_weight_min": -50.02735137939453,
      "activations/layer9_attention_weight_max": 44.12266540527344,
      "activations/layer9_attention_weight_min": -45.20735549926758,
      "epoch": 13.42,
      "learning_rate": 6.408863636363636e-05,
      "loss": 2.7594,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_loss": 2.71875,
      "eval_runtime": 8.2728,
      "eval_samples_per_second": 519.049,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.2728,
      "eval_openwebtext_samples_per_second": 519.049,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_wikitext_loss": 2.955078125,
      "eval_wikitext_ppl": 19.20322291302118,
      "eval_wikitext_runtime": 1.7643,
      "eval_wikitext_samples_per_second": 258.462,
      "step": 231000
    },
    {
      "epoch": 13.42,
      "eval_lambada_loss": 2.59375,
      "eval_lambada_ppl": 13.379852088930456,
      "eval_lambada_runtime": 9.5358,
      "eval_lambada_samples_per_second": 510.601,
      "step": 231000
    },
    {
      "activations/layer0_attention_weight_max": 17.08155632019043,
      "activations/layer0_attention_weight_min": -14.358722686767578,
      "activations/layer10_attention_weight_max": 35.70343780517578,
      "activations/layer10_attention_weight_min": -37.39984893798828,
      "activations/layer11_attention_weight_max": 32.88277816772461,
      "activations/layer11_attention_weight_min": -33.988136291503906,
      "activations/layer12_attention_weight_max": 18.171323776245117,
      "activations/layer12_attention_weight_min": -26.985031127929688,
      "activations/layer13_attention_weight_max": 34.69538497924805,
      "activations/layer13_attention_weight_min": -27.596710205078125,
      "activations/layer14_attention_weight_max": 37.9228401184082,
      "activations/layer14_attention_weight_min": -32.29388427734375,
      "activations/layer15_attention_weight_max": 35.117942810058594,
      "activations/layer15_attention_weight_min": -31.02277183532715,
      "activations/layer16_attention_weight_max": 35.19867706298828,
      "activations/layer16_attention_weight_min": -33.36482238769531,
      "activations/layer17_attention_weight_max": 52.03166961669922,
      "activations/layer17_attention_weight_min": -45.5872802734375,
      "activations/layer18_attention_weight_max": 53.8256721496582,
      "activations/layer18_attention_weight_min": -39.755645751953125,
      "activations/layer19_attention_weight_max": 23.38053321838379,
      "activations/layer19_attention_weight_min": -22.373144149780273,
      "activations/layer1_attention_weight_max": 16.517038345336914,
      "activations/layer1_attention_weight_min": -14.99074935913086,
      "activations/layer20_attention_weight_max": 23.5081787109375,
      "activations/layer20_attention_weight_min": -20.022001266479492,
      "activations/layer21_attention_weight_max": 34.83064270019531,
      "activations/layer21_attention_weight_min": -22.248775482177734,
      "activations/layer22_attention_weight_max": 28.13740348815918,
      "activations/layer22_attention_weight_min": -25.26837921142578,
      "activations/layer23_attention_weight_max": 34.455650329589844,
      "activations/layer23_attention_weight_min": -24.37555694580078,
      "activations/layer2_attention_weight_max": 32.11488723754883,
      "activations/layer2_attention_weight_min": -31.09186363220215,
      "activations/layer3_attention_weight_max": 92.9999771118164,
      "activations/layer3_attention_weight_min": -95.21034240722656,
      "activations/layer4_attention_weight_max": 98.34320068359375,
      "activations/layer4_attention_weight_min": -91.92949676513672,
      "activations/layer5_attention_weight_max": 75.62124633789062,
      "activations/layer5_attention_weight_min": -79.31137084960938,
      "activations/layer6_attention_weight_max": 51.66267395019531,
      "activations/layer6_attention_weight_min": -50.80400466918945,
      "activations/layer7_attention_weight_max": 69.0264663696289,
      "activations/layer7_attention_weight_min": -69.6144027709961,
      "activations/layer8_attention_weight_max": 42.97151184082031,
      "activations/layer8_attention_weight_min": -45.179115295410156,
      "activations/layer9_attention_weight_max": 45.57371139526367,
      "activations/layer9_attention_weight_min": -49.8574333190918,
      "epoch": 13.43,
      "learning_rate": 6.406969696969696e-05,
      "loss": 2.7609,
      "step": 231050
    },
    {
      "activations/layer0_attention_weight_max": 17.259971618652344,
      "activations/layer0_attention_weight_min": -14.481266021728516,
      "activations/layer10_attention_weight_max": 38.4736213684082,
      "activations/layer10_attention_weight_min": -35.082313537597656,
      "activations/layer11_attention_weight_max": 31.657062530517578,
      "activations/layer11_attention_weight_min": -32.41114044189453,
      "activations/layer12_attention_weight_max": 18.856019973754883,
      "activations/layer12_attention_weight_min": -24.222070693969727,
      "activations/layer13_attention_weight_max": 37.6005859375,
      "activations/layer13_attention_weight_min": -31.069068908691406,
      "activations/layer14_attention_weight_max": 36.90878677368164,
      "activations/layer14_attention_weight_min": -33.161293029785156,
      "activations/layer15_attention_weight_max": 32.60832214355469,
      "activations/layer15_attention_weight_min": -31.013248443603516,
      "activations/layer16_attention_weight_max": 32.25093078613281,
      "activations/layer16_attention_weight_min": -31.185867309570312,
      "activations/layer17_attention_weight_max": 51.058998107910156,
      "activations/layer17_attention_weight_min": -47.00071334838867,
      "activations/layer18_attention_weight_max": 44.5197639465332,
      "activations/layer18_attention_weight_min": -39.075523376464844,
      "activations/layer19_attention_weight_max": 24.32099723815918,
      "activations/layer19_attention_weight_min": -23.842153549194336,
      "activations/layer1_attention_weight_max": 17.26486587524414,
      "activations/layer1_attention_weight_min": -15.910603523254395,
      "activations/layer20_attention_weight_max": 22.91328239440918,
      "activations/layer20_attention_weight_min": -21.698307037353516,
      "activations/layer21_attention_weight_max": 33.443058013916016,
      "activations/layer21_attention_weight_min": -24.02391815185547,
      "activations/layer22_attention_weight_max": 32.70475769042969,
      "activations/layer22_attention_weight_min": -27.95517921447754,
      "activations/layer23_attention_weight_max": 38.717437744140625,
      "activations/layer23_attention_weight_min": -27.188278198242188,
      "activations/layer2_attention_weight_max": 31.60689926147461,
      "activations/layer2_attention_weight_min": -30.92690658569336,
      "activations/layer3_attention_weight_max": 97.73788452148438,
      "activations/layer3_attention_weight_min": -98.51756286621094,
      "activations/layer4_attention_weight_max": 96.3647689819336,
      "activations/layer4_attention_weight_min": -94.13917541503906,
      "activations/layer5_attention_weight_max": 74.34436798095703,
      "activations/layer5_attention_weight_min": -76.52071380615234,
      "activations/layer6_attention_weight_max": 52.29762649536133,
      "activations/layer6_attention_weight_min": -50.96504211425781,
      "activations/layer7_attention_weight_max": 66.15352630615234,
      "activations/layer7_attention_weight_min": -63.75425720214844,
      "activations/layer8_attention_weight_max": 46.03541564941406,
      "activations/layer8_attention_weight_min": -46.00456237792969,
      "activations/layer9_attention_weight_max": 48.841575622558594,
      "activations/layer9_attention_weight_min": -47.607425689697266,
      "epoch": 13.43,
      "learning_rate": 6.405075757575756e-05,
      "loss": 2.7693,
      "step": 231100
    },
    {
      "activations/layer0_attention_weight_max": 15.922406196594238,
      "activations/layer0_attention_weight_min": -14.66970443725586,
      "activations/layer10_attention_weight_max": 37.457252502441406,
      "activations/layer10_attention_weight_min": -35.67647171020508,
      "activations/layer11_attention_weight_max": 31.969860076904297,
      "activations/layer11_attention_weight_min": -33.51758575439453,
      "activations/layer12_attention_weight_max": 19.64529037475586,
      "activations/layer12_attention_weight_min": -23.3293399810791,
      "activations/layer13_attention_weight_max": 43.00149154663086,
      "activations/layer13_attention_weight_min": -32.12805938720703,
      "activations/layer14_attention_weight_max": 35.75702667236328,
      "activations/layer14_attention_weight_min": -29.851177215576172,
      "activations/layer15_attention_weight_max": 33.59075164794922,
      "activations/layer15_attention_weight_min": -30.375913619995117,
      "activations/layer16_attention_weight_max": 32.09397506713867,
      "activations/layer16_attention_weight_min": -30.281213760375977,
      "activations/layer17_attention_weight_max": 52.833351135253906,
      "activations/layer17_attention_weight_min": -42.70355987548828,
      "activations/layer18_attention_weight_max": 47.3880729675293,
      "activations/layer18_attention_weight_min": -38.055728912353516,
      "activations/layer19_attention_weight_max": 23.22294807434082,
      "activations/layer19_attention_weight_min": -21.450075149536133,
      "activations/layer1_attention_weight_max": 16.317068099975586,
      "activations/layer1_attention_weight_min": -14.735429763793945,
      "activations/layer20_attention_weight_max": 20.628517150878906,
      "activations/layer20_attention_weight_min": -20.44707489013672,
      "activations/layer21_attention_weight_max": 35.736915588378906,
      "activations/layer21_attention_weight_min": -22.21076011657715,
      "activations/layer22_attention_weight_max": 30.12759780883789,
      "activations/layer22_attention_weight_min": -25.285812377929688,
      "activations/layer23_attention_weight_max": 33.73763656616211,
      "activations/layer23_attention_weight_min": -22.68844223022461,
      "activations/layer2_attention_weight_max": 32.857276916503906,
      "activations/layer2_attention_weight_min": -31.306278228759766,
      "activations/layer3_attention_weight_max": 97.146728515625,
      "activations/layer3_attention_weight_min": -94.61924743652344,
      "activations/layer4_attention_weight_max": 94.25480651855469,
      "activations/layer4_attention_weight_min": -94.06856536865234,
      "activations/layer5_attention_weight_max": 68.96044158935547,
      "activations/layer5_attention_weight_min": -81.687744140625,
      "activations/layer6_attention_weight_max": 53.95085525512695,
      "activations/layer6_attention_weight_min": -53.231685638427734,
      "activations/layer7_attention_weight_max": 64.90957641601562,
      "activations/layer7_attention_weight_min": -64.69660186767578,
      "activations/layer8_attention_weight_max": 47.85108947753906,
      "activations/layer8_attention_weight_min": -48.64326858520508,
      "activations/layer9_attention_weight_max": 45.303260803222656,
      "activations/layer9_attention_weight_min": -46.50980758666992,
      "epoch": 13.43,
      "learning_rate": 6.403181818181818e-05,
      "loss": 2.7444,
      "step": 231150
    },
    {
      "activations/layer0_attention_weight_max": 16.143430709838867,
      "activations/layer0_attention_weight_min": -14.708637237548828,
      "activations/layer10_attention_weight_max": 38.110286712646484,
      "activations/layer10_attention_weight_min": -36.35337829589844,
      "activations/layer11_attention_weight_max": 34.00727844238281,
      "activations/layer11_attention_weight_min": -34.095481872558594,
      "activations/layer12_attention_weight_max": 16.875713348388672,
      "activations/layer12_attention_weight_min": -22.540189743041992,
      "activations/layer13_attention_weight_max": 38.254486083984375,
      "activations/layer13_attention_weight_min": -26.185091018676758,
      "activations/layer14_attention_weight_max": 36.21525955200195,
      "activations/layer14_attention_weight_min": -29.29836082458496,
      "activations/layer15_attention_weight_max": 30.915510177612305,
      "activations/layer15_attention_weight_min": -28.779983520507812,
      "activations/layer16_attention_weight_max": 30.11614418029785,
      "activations/layer16_attention_weight_min": -28.86264991760254,
      "activations/layer17_attention_weight_max": 52.52298355102539,
      "activations/layer17_attention_weight_min": -42.24540710449219,
      "activations/layer18_attention_weight_max": 43.551700592041016,
      "activations/layer18_attention_weight_min": -38.68513870239258,
      "activations/layer19_attention_weight_max": 21.783029556274414,
      "activations/layer19_attention_weight_min": -25.82522964477539,
      "activations/layer1_attention_weight_max": 17.51152229309082,
      "activations/layer1_attention_weight_min": -14.038848876953125,
      "activations/layer20_attention_weight_max": 20.170726776123047,
      "activations/layer20_attention_weight_min": -22.56199836730957,
      "activations/layer21_attention_weight_max": 36.2470588684082,
      "activations/layer21_attention_weight_min": -24.81418800354004,
      "activations/layer22_attention_weight_max": 30.173696517944336,
      "activations/layer22_attention_weight_min": -25.560537338256836,
      "activations/layer23_attention_weight_max": 35.0691032409668,
      "activations/layer23_attention_weight_min": -25.398944854736328,
      "activations/layer2_attention_weight_max": 31.407154083251953,
      "activations/layer2_attention_weight_min": -29.898101806640625,
      "activations/layer3_attention_weight_max": 90.25324249267578,
      "activations/layer3_attention_weight_min": -91.31590270996094,
      "activations/layer4_attention_weight_max": 95.86083221435547,
      "activations/layer4_attention_weight_min": -93.48533630371094,
      "activations/layer5_attention_weight_max": 70.54522705078125,
      "activations/layer5_attention_weight_min": -79.91413879394531,
      "activations/layer6_attention_weight_max": 51.57382583618164,
      "activations/layer6_attention_weight_min": -52.11659622192383,
      "activations/layer7_attention_weight_max": 71.78221130371094,
      "activations/layer7_attention_weight_min": -65.38468933105469,
      "activations/layer8_attention_weight_max": 46.995365142822266,
      "activations/layer8_attention_weight_min": -50.90488052368164,
      "activations/layer9_attention_weight_max": 45.77595138549805,
      "activations/layer9_attention_weight_min": -49.3272705078125,
      "epoch": 13.43,
      "learning_rate": 6.401287878787878e-05,
      "loss": 2.7662,
      "step": 231200
    },
    {
      "activations/layer0_attention_weight_max": 15.922419548034668,
      "activations/layer0_attention_weight_min": -14.320748329162598,
      "activations/layer10_attention_weight_max": 36.117671966552734,
      "activations/layer10_attention_weight_min": -36.60341262817383,
      "activations/layer11_attention_weight_max": 31.359745025634766,
      "activations/layer11_attention_weight_min": -32.68506622314453,
      "activations/layer12_attention_weight_max": 18.424510955810547,
      "activations/layer12_attention_weight_min": -26.141542434692383,
      "activations/layer13_attention_weight_max": 44.46910858154297,
      "activations/layer13_attention_weight_min": -31.626436233520508,
      "activations/layer14_attention_weight_max": 37.99379348754883,
      "activations/layer14_attention_weight_min": -31.49989128112793,
      "activations/layer15_attention_weight_max": 34.04938888549805,
      "activations/layer15_attention_weight_min": -29.56587791442871,
      "activations/layer16_attention_weight_max": 33.175254821777344,
      "activations/layer16_attention_weight_min": -32.204105377197266,
      "activations/layer17_attention_weight_max": 51.3501091003418,
      "activations/layer17_attention_weight_min": -46.53926467895508,
      "activations/layer18_attention_weight_max": 47.45493698120117,
      "activations/layer18_attention_weight_min": -39.295284271240234,
      "activations/layer19_attention_weight_max": 22.684755325317383,
      "activations/layer19_attention_weight_min": -22.104185104370117,
      "activations/layer1_attention_weight_max": 16.194372177124023,
      "activations/layer1_attention_weight_min": -13.836040496826172,
      "activations/layer20_attention_weight_max": 23.57094955444336,
      "activations/layer20_attention_weight_min": -21.541582107543945,
      "activations/layer21_attention_weight_max": 36.417388916015625,
      "activations/layer21_attention_weight_min": -23.755216598510742,
      "activations/layer22_attention_weight_max": 32.1201057434082,
      "activations/layer22_attention_weight_min": -27.040056228637695,
      "activations/layer23_attention_weight_max": 38.957645416259766,
      "activations/layer23_attention_weight_min": -22.668209075927734,
      "activations/layer2_attention_weight_max": 32.3831787109375,
      "activations/layer2_attention_weight_min": -32.23181915283203,
      "activations/layer3_attention_weight_max": 94.1788330078125,
      "activations/layer3_attention_weight_min": -100.53809356689453,
      "activations/layer4_attention_weight_max": 97.83366394042969,
      "activations/layer4_attention_weight_min": -94.7669677734375,
      "activations/layer5_attention_weight_max": 72.46865844726562,
      "activations/layer5_attention_weight_min": -80.63912963867188,
      "activations/layer6_attention_weight_max": 50.54123306274414,
      "activations/layer6_attention_weight_min": -51.181068420410156,
      "activations/layer7_attention_weight_max": 69.34812927246094,
      "activations/layer7_attention_weight_min": -68.20210266113281,
      "activations/layer8_attention_weight_max": 44.80757522583008,
      "activations/layer8_attention_weight_min": -47.750186920166016,
      "activations/layer9_attention_weight_max": 46.633113861083984,
      "activations/layer9_attention_weight_min": -47.53486633300781,
      "epoch": 13.44,
      "learning_rate": 6.39939393939394e-05,
      "loss": 2.77,
      "step": 231250
    },
    {
      "activations/layer0_attention_weight_max": 15.828857421875,
      "activations/layer0_attention_weight_min": -14.513381004333496,
      "activations/layer10_attention_weight_max": 37.21175003051758,
      "activations/layer10_attention_weight_min": -34.36529541015625,
      "activations/layer11_attention_weight_max": 36.18998718261719,
      "activations/layer11_attention_weight_min": -33.27136993408203,
      "activations/layer12_attention_weight_max": 18.1055908203125,
      "activations/layer12_attention_weight_min": -25.702497482299805,
      "activations/layer13_attention_weight_max": 39.00876998901367,
      "activations/layer13_attention_weight_min": -28.805965423583984,
      "activations/layer14_attention_weight_max": 35.3714714050293,
      "activations/layer14_attention_weight_min": -30.67271614074707,
      "activations/layer15_attention_weight_max": 33.27700424194336,
      "activations/layer15_attention_weight_min": -29.89035415649414,
      "activations/layer16_attention_weight_max": 30.97969627380371,
      "activations/layer16_attention_weight_min": -29.89251136779785,
      "activations/layer17_attention_weight_max": 51.25319290161133,
      "activations/layer17_attention_weight_min": -45.79072189331055,
      "activations/layer18_attention_weight_max": 45.393592834472656,
      "activations/layer18_attention_weight_min": -41.232242584228516,
      "activations/layer19_attention_weight_max": 23.37796401977539,
      "activations/layer19_attention_weight_min": -23.173246383666992,
      "activations/layer1_attention_weight_max": 16.33066749572754,
      "activations/layer1_attention_weight_min": -14.04979133605957,
      "activations/layer20_attention_weight_max": 22.623268127441406,
      "activations/layer20_attention_weight_min": -22.584304809570312,
      "activations/layer21_attention_weight_max": 36.887413024902344,
      "activations/layer21_attention_weight_min": -21.810260772705078,
      "activations/layer22_attention_weight_max": 33.50197982788086,
      "activations/layer22_attention_weight_min": -28.68749237060547,
      "activations/layer23_attention_weight_max": 39.6970329284668,
      "activations/layer23_attention_weight_min": -22.88394546508789,
      "activations/layer2_attention_weight_max": 32.657100677490234,
      "activations/layer2_attention_weight_min": -31.585582733154297,
      "activations/layer3_attention_weight_max": 91.58740997314453,
      "activations/layer3_attention_weight_min": -94.87830352783203,
      "activations/layer4_attention_weight_max": 93.64983367919922,
      "activations/layer4_attention_weight_min": -89.19207763671875,
      "activations/layer5_attention_weight_max": 69.7166519165039,
      "activations/layer5_attention_weight_min": -78.3216323852539,
      "activations/layer6_attention_weight_max": 49.34663391113281,
      "activations/layer6_attention_weight_min": -51.103355407714844,
      "activations/layer7_attention_weight_max": 63.817291259765625,
      "activations/layer7_attention_weight_min": -62.11558532714844,
      "activations/layer8_attention_weight_max": 43.52286148071289,
      "activations/layer8_attention_weight_min": -46.5977897644043,
      "activations/layer9_attention_weight_max": 43.57648849487305,
      "activations/layer9_attention_weight_min": -45.853240966796875,
      "epoch": 13.44,
      "learning_rate": 6.3975e-05,
      "loss": 2.7695,
      "step": 231300
    },
    {
      "activations/layer0_attention_weight_max": 15.180815696716309,
      "activations/layer0_attention_weight_min": -14.22916030883789,
      "activations/layer10_attention_weight_max": 38.83543395996094,
      "activations/layer10_attention_weight_min": -35.95771408081055,
      "activations/layer11_attention_weight_max": 32.542659759521484,
      "activations/layer11_attention_weight_min": -31.576725006103516,
      "activations/layer12_attention_weight_max": 22.12363624572754,
      "activations/layer12_attention_weight_min": -23.03215789794922,
      "activations/layer13_attention_weight_max": 41.84025192260742,
      "activations/layer13_attention_weight_min": -30.751073837280273,
      "activations/layer14_attention_weight_max": 42.343292236328125,
      "activations/layer14_attention_weight_min": -31.683813095092773,
      "activations/layer15_attention_weight_max": 36.70943832397461,
      "activations/layer15_attention_weight_min": -30.161575317382812,
      "activations/layer16_attention_weight_max": 33.46226501464844,
      "activations/layer16_attention_weight_min": -32.08957290649414,
      "activations/layer17_attention_weight_max": 54.584434509277344,
      "activations/layer17_attention_weight_min": -47.25932312011719,
      "activations/layer18_attention_weight_max": 51.26288986206055,
      "activations/layer18_attention_weight_min": -38.88997268676758,
      "activations/layer19_attention_weight_max": 26.263288497924805,
      "activations/layer19_attention_weight_min": -22.851119995117188,
      "activations/layer1_attention_weight_max": 16.65169334411621,
      "activations/layer1_attention_weight_min": -14.94762134552002,
      "activations/layer20_attention_weight_max": 27.879404067993164,
      "activations/layer20_attention_weight_min": -21.454326629638672,
      "activations/layer21_attention_weight_max": 45.06228256225586,
      "activations/layer21_attention_weight_min": -25.51220703125,
      "activations/layer22_attention_weight_max": 33.22120666503906,
      "activations/layer22_attention_weight_min": -25.537384033203125,
      "activations/layer23_attention_weight_max": 42.64155960083008,
      "activations/layer23_attention_weight_min": -25.991247177124023,
      "activations/layer2_attention_weight_max": 31.467130661010742,
      "activations/layer2_attention_weight_min": -30.274892807006836,
      "activations/layer3_attention_weight_max": 95.29950714111328,
      "activations/layer3_attention_weight_min": -95.81221771240234,
      "activations/layer4_attention_weight_max": 98.40032196044922,
      "activations/layer4_attention_weight_min": -90.04419708251953,
      "activations/layer5_attention_weight_max": 73.33358764648438,
      "activations/layer5_attention_weight_min": -83.64293670654297,
      "activations/layer6_attention_weight_max": 51.08794021606445,
      "activations/layer6_attention_weight_min": -51.46946716308594,
      "activations/layer7_attention_weight_max": 69.89207458496094,
      "activations/layer7_attention_weight_min": -67.32322692871094,
      "activations/layer8_attention_weight_max": 48.4682731628418,
      "activations/layer8_attention_weight_min": -48.67910385131836,
      "activations/layer9_attention_weight_max": 46.9255485534668,
      "activations/layer9_attention_weight_min": -44.25631332397461,
      "epoch": 13.44,
      "learning_rate": 6.39560606060606e-05,
      "loss": 2.7613,
      "step": 231350
    },
    {
      "activations/layer0_attention_weight_max": 15.838372230529785,
      "activations/layer0_attention_weight_min": -14.960654258728027,
      "activations/layer10_attention_weight_max": 38.34636306762695,
      "activations/layer10_attention_weight_min": -35.35088348388672,
      "activations/layer11_attention_weight_max": 34.09209442138672,
      "activations/layer11_attention_weight_min": -34.809391021728516,
      "activations/layer12_attention_weight_max": 21.00839614868164,
      "activations/layer12_attention_weight_min": -22.132728576660156,
      "activations/layer13_attention_weight_max": 39.32911682128906,
      "activations/layer13_attention_weight_min": -32.49053955078125,
      "activations/layer14_attention_weight_max": 36.61483383178711,
      "activations/layer14_attention_weight_min": -30.29927635192871,
      "activations/layer15_attention_weight_max": 33.09684371948242,
      "activations/layer15_attention_weight_min": -31.294063568115234,
      "activations/layer16_attention_weight_max": 30.36183738708496,
      "activations/layer16_attention_weight_min": -30.004962921142578,
      "activations/layer17_attention_weight_max": 53.00502014160156,
      "activations/layer17_attention_weight_min": -45.13324737548828,
      "activations/layer18_attention_weight_max": 49.99628829956055,
      "activations/layer18_attention_weight_min": -41.58340835571289,
      "activations/layer19_attention_weight_max": 25.53017234802246,
      "activations/layer19_attention_weight_min": -21.885086059570312,
      "activations/layer1_attention_weight_max": 15.760759353637695,
      "activations/layer1_attention_weight_min": -15.009626388549805,
      "activations/layer20_attention_weight_max": 23.31136703491211,
      "activations/layer20_attention_weight_min": -21.727344512939453,
      "activations/layer21_attention_weight_max": 40.03054428100586,
      "activations/layer21_attention_weight_min": -27.825780868530273,
      "activations/layer22_attention_weight_max": 33.82428741455078,
      "activations/layer22_attention_weight_min": -27.732778549194336,
      "activations/layer23_attention_weight_max": 40.22474670410156,
      "activations/layer23_attention_weight_min": -26.457263946533203,
      "activations/layer2_attention_weight_max": 33.523616790771484,
      "activations/layer2_attention_weight_min": -31.060779571533203,
      "activations/layer3_attention_weight_max": 99.65283203125,
      "activations/layer3_attention_weight_min": -99.30433654785156,
      "activations/layer4_attention_weight_max": 103.57905578613281,
      "activations/layer4_attention_weight_min": -97.98881530761719,
      "activations/layer5_attention_weight_max": 74.481201171875,
      "activations/layer5_attention_weight_min": -83.4683837890625,
      "activations/layer6_attention_weight_max": 52.851863861083984,
      "activations/layer6_attention_weight_min": -54.13664245605469,
      "activations/layer7_attention_weight_max": 72.21995544433594,
      "activations/layer7_attention_weight_min": -67.4227066040039,
      "activations/layer8_attention_weight_max": 47.277809143066406,
      "activations/layer8_attention_weight_min": -53.2249755859375,
      "activations/layer9_attention_weight_max": 48.87089157104492,
      "activations/layer9_attention_weight_min": -48.16969299316406,
      "epoch": 13.45,
      "learning_rate": 6.393712121212121e-05,
      "loss": 2.7566,
      "step": 231400
    },
    {
      "activations/layer0_attention_weight_max": 16.389055252075195,
      "activations/layer0_attention_weight_min": -14.755910873413086,
      "activations/layer10_attention_weight_max": 34.432029724121094,
      "activations/layer10_attention_weight_min": -35.11227798461914,
      "activations/layer11_attention_weight_max": 31.06167984008789,
      "activations/layer11_attention_weight_min": -31.911190032958984,
      "activations/layer12_attention_weight_max": 19.611160278320312,
      "activations/layer12_attention_weight_min": -27.640104293823242,
      "activations/layer13_attention_weight_max": 37.710262298583984,
      "activations/layer13_attention_weight_min": -30.411327362060547,
      "activations/layer14_attention_weight_max": 36.92057800292969,
      "activations/layer14_attention_weight_min": -30.24092674255371,
      "activations/layer15_attention_weight_max": 31.846397399902344,
      "activations/layer15_attention_weight_min": -30.728496551513672,
      "activations/layer16_attention_weight_max": 33.212989807128906,
      "activations/layer16_attention_weight_min": -33.763240814208984,
      "activations/layer17_attention_weight_max": 51.77702713012695,
      "activations/layer17_attention_weight_min": -44.04888916015625,
      "activations/layer18_attention_weight_max": 48.11497116088867,
      "activations/layer18_attention_weight_min": -38.48629379272461,
      "activations/layer19_attention_weight_max": 23.357511520385742,
      "activations/layer19_attention_weight_min": -22.422101974487305,
      "activations/layer1_attention_weight_max": 16.52468490600586,
      "activations/layer1_attention_weight_min": -14.652970314025879,
      "activations/layer20_attention_weight_max": 21.899417877197266,
      "activations/layer20_attention_weight_min": -23.998353958129883,
      "activations/layer21_attention_weight_max": 41.781036376953125,
      "activations/layer21_attention_weight_min": -26.738866806030273,
      "activations/layer22_attention_weight_max": 30.52181625366211,
      "activations/layer22_attention_weight_min": -27.25661277770996,
      "activations/layer23_attention_weight_max": 41.410743713378906,
      "activations/layer23_attention_weight_min": -25.769649505615234,
      "activations/layer2_attention_weight_max": 34.05188751220703,
      "activations/layer2_attention_weight_min": -30.452619552612305,
      "activations/layer3_attention_weight_max": 96.63053131103516,
      "activations/layer3_attention_weight_min": -94.49739837646484,
      "activations/layer4_attention_weight_max": 93.72164154052734,
      "activations/layer4_attention_weight_min": -91.99264526367188,
      "activations/layer5_attention_weight_max": 73.75080871582031,
      "activations/layer5_attention_weight_min": -75.88641357421875,
      "activations/layer6_attention_weight_max": 50.85300064086914,
      "activations/layer6_attention_weight_min": -49.64616394042969,
      "activations/layer7_attention_weight_max": 67.87847137451172,
      "activations/layer7_attention_weight_min": -64.39720153808594,
      "activations/layer8_attention_weight_max": 44.383113861083984,
      "activations/layer8_attention_weight_min": -47.26784133911133,
      "activations/layer9_attention_weight_max": 46.33744812011719,
      "activations/layer9_attention_weight_min": -46.29542922973633,
      "epoch": 13.45,
      "learning_rate": 6.391818181818181e-05,
      "loss": 2.7647,
      "step": 231450
    },
    {
      "activations/layer0_attention_weight_max": 15.83234977722168,
      "activations/layer0_attention_weight_min": -14.993109703063965,
      "activations/layer10_attention_weight_max": 37.23316955566406,
      "activations/layer10_attention_weight_min": -34.97136688232422,
      "activations/layer11_attention_weight_max": 32.36671829223633,
      "activations/layer11_attention_weight_min": -33.08325958251953,
      "activations/layer12_attention_weight_max": 17.813526153564453,
      "activations/layer12_attention_weight_min": -22.278770446777344,
      "activations/layer13_attention_weight_max": 34.802555084228516,
      "activations/layer13_attention_weight_min": -27.95438575744629,
      "activations/layer14_attention_weight_max": 34.78557205200195,
      "activations/layer14_attention_weight_min": -31.961755752563477,
      "activations/layer15_attention_weight_max": 31.69365119934082,
      "activations/layer15_attention_weight_min": -28.698322296142578,
      "activations/layer16_attention_weight_max": 29.564743041992188,
      "activations/layer16_attention_weight_min": -30.663883209228516,
      "activations/layer17_attention_weight_max": 50.97938919067383,
      "activations/layer17_attention_weight_min": -42.2226676940918,
      "activations/layer18_attention_weight_max": 45.746742248535156,
      "activations/layer18_attention_weight_min": -36.81965255737305,
      "activations/layer19_attention_weight_max": 22.42171287536621,
      "activations/layer19_attention_weight_min": -21.166210174560547,
      "activations/layer1_attention_weight_max": 16.81755256652832,
      "activations/layer1_attention_weight_min": -16.391357421875,
      "activations/layer20_attention_weight_max": 22.016761779785156,
      "activations/layer20_attention_weight_min": -19.106660842895508,
      "activations/layer21_attention_weight_max": 34.03302764892578,
      "activations/layer21_attention_weight_min": -27.52174949645996,
      "activations/layer22_attention_weight_max": 29.699298858642578,
      "activations/layer22_attention_weight_min": -25.551008224487305,
      "activations/layer23_attention_weight_max": 37.244895935058594,
      "activations/layer23_attention_weight_min": -23.7891845703125,
      "activations/layer2_attention_weight_max": 33.07857894897461,
      "activations/layer2_attention_weight_min": -31.99654769897461,
      "activations/layer3_attention_weight_max": 94.51678466796875,
      "activations/layer3_attention_weight_min": -96.46820068359375,
      "activations/layer4_attention_weight_max": 97.5692367553711,
      "activations/layer4_attention_weight_min": -93.3332748413086,
      "activations/layer5_attention_weight_max": 69.85330200195312,
      "activations/layer5_attention_weight_min": -79.31881713867188,
      "activations/layer6_attention_weight_max": 52.26639938354492,
      "activations/layer6_attention_weight_min": -53.54115676879883,
      "activations/layer7_attention_weight_max": 66.09818267822266,
      "activations/layer7_attention_weight_min": -64.50751495361328,
      "activations/layer8_attention_weight_max": 45.97184371948242,
      "activations/layer8_attention_weight_min": -48.951805114746094,
      "activations/layer9_attention_weight_max": 45.634368896484375,
      "activations/layer9_attention_weight_min": -45.81342697143555,
      "epoch": 13.45,
      "learning_rate": 6.389924242424242e-05,
      "loss": 2.7628,
      "step": 231500
    },
    {
      "activations/layer0_attention_weight_max": 15.995535850524902,
      "activations/layer0_attention_weight_min": -15.333759307861328,
      "activations/layer10_attention_weight_max": 34.67148208618164,
      "activations/layer10_attention_weight_min": -34.83613967895508,
      "activations/layer11_attention_weight_max": 32.19165802001953,
      "activations/layer11_attention_weight_min": -32.52846145629883,
      "activations/layer12_attention_weight_max": 20.988088607788086,
      "activations/layer12_attention_weight_min": -22.713768005371094,
      "activations/layer13_attention_weight_max": 34.527130126953125,
      "activations/layer13_attention_weight_min": -27.087919235229492,
      "activations/layer14_attention_weight_max": 32.85728073120117,
      "activations/layer14_attention_weight_min": -30.2914981842041,
      "activations/layer15_attention_weight_max": 33.8131217956543,
      "activations/layer15_attention_weight_min": -29.618425369262695,
      "activations/layer16_attention_weight_max": 29.658218383789062,
      "activations/layer16_attention_weight_min": -29.17407989501953,
      "activations/layer17_attention_weight_max": 49.2216682434082,
      "activations/layer17_attention_weight_min": -45.20512390136719,
      "activations/layer18_attention_weight_max": 44.89816665649414,
      "activations/layer18_attention_weight_min": -39.01540756225586,
      "activations/layer19_attention_weight_max": 21.455825805664062,
      "activations/layer19_attention_weight_min": -20.9738826751709,
      "activations/layer1_attention_weight_max": 16.965227127075195,
      "activations/layer1_attention_weight_min": -14.343937873840332,
      "activations/layer20_attention_weight_max": 19.763486862182617,
      "activations/layer20_attention_weight_min": -21.208114624023438,
      "activations/layer21_attention_weight_max": 32.9343147277832,
      "activations/layer21_attention_weight_min": -22.390581130981445,
      "activations/layer22_attention_weight_max": 28.514041900634766,
      "activations/layer22_attention_weight_min": -25.01946449279785,
      "activations/layer23_attention_weight_max": 35.013893127441406,
      "activations/layer23_attention_weight_min": -23.69036102294922,
      "activations/layer2_attention_weight_max": 31.475536346435547,
      "activations/layer2_attention_weight_min": -28.9447078704834,
      "activations/layer3_attention_weight_max": 94.13420867919922,
      "activations/layer3_attention_weight_min": -95.99066162109375,
      "activations/layer4_attention_weight_max": 97.27433776855469,
      "activations/layer4_attention_weight_min": -93.21958923339844,
      "activations/layer5_attention_weight_max": 71.82380676269531,
      "activations/layer5_attention_weight_min": -79.09251403808594,
      "activations/layer6_attention_weight_max": 51.53987121582031,
      "activations/layer6_attention_weight_min": -51.93844985961914,
      "activations/layer7_attention_weight_max": 66.2630844116211,
      "activations/layer7_attention_weight_min": -66.37580871582031,
      "activations/layer8_attention_weight_max": 46.85206985473633,
      "activations/layer8_attention_weight_min": -49.13982009887695,
      "activations/layer9_attention_weight_max": 46.67517852783203,
      "activations/layer9_attention_weight_min": -49.325645446777344,
      "epoch": 13.45,
      "learning_rate": 6.388030303030302e-05,
      "loss": 2.7621,
      "step": 231550
    },
    {
      "activations/layer0_attention_weight_max": 15.850227355957031,
      "activations/layer0_attention_weight_min": -14.799222946166992,
      "activations/layer10_attention_weight_max": 37.76849365234375,
      "activations/layer10_attention_weight_min": -37.18912124633789,
      "activations/layer11_attention_weight_max": 33.72575759887695,
      "activations/layer11_attention_weight_min": -34.32268524169922,
      "activations/layer12_attention_weight_max": 17.718713760375977,
      "activations/layer12_attention_weight_min": -24.569477081298828,
      "activations/layer13_attention_weight_max": 39.46296691894531,
      "activations/layer13_attention_weight_min": -29.306976318359375,
      "activations/layer14_attention_weight_max": 37.1970329284668,
      "activations/layer14_attention_weight_min": -28.728485107421875,
      "activations/layer15_attention_weight_max": 33.06028747558594,
      "activations/layer15_attention_weight_min": -29.90157127380371,
      "activations/layer16_attention_weight_max": 33.928321838378906,
      "activations/layer16_attention_weight_min": -30.135644912719727,
      "activations/layer17_attention_weight_max": 50.89876174926758,
      "activations/layer17_attention_weight_min": -44.354190826416016,
      "activations/layer18_attention_weight_max": 45.67272186279297,
      "activations/layer18_attention_weight_min": -36.98823547363281,
      "activations/layer19_attention_weight_max": 23.434083938598633,
      "activations/layer19_attention_weight_min": -22.170761108398438,
      "activations/layer1_attention_weight_max": 16.522645950317383,
      "activations/layer1_attention_weight_min": -15.236722946166992,
      "activations/layer20_attention_weight_max": 22.26276397705078,
      "activations/layer20_attention_weight_min": -21.779279708862305,
      "activations/layer21_attention_weight_max": 38.873023986816406,
      "activations/layer21_attention_weight_min": -25.02875328063965,
      "activations/layer22_attention_weight_max": 30.99728775024414,
      "activations/layer22_attention_weight_min": -24.721147537231445,
      "activations/layer23_attention_weight_max": 38.41103744506836,
      "activations/layer23_attention_weight_min": -23.937963485717773,
      "activations/layer2_attention_weight_max": 34.09888458251953,
      "activations/layer2_attention_weight_min": -31.202159881591797,
      "activations/layer3_attention_weight_max": 100.01457977294922,
      "activations/layer3_attention_weight_min": -98.11003875732422,
      "activations/layer4_attention_weight_max": 95.1955795288086,
      "activations/layer4_attention_weight_min": -96.90199279785156,
      "activations/layer5_attention_weight_max": 73.87924194335938,
      "activations/layer5_attention_weight_min": -74.69223022460938,
      "activations/layer6_attention_weight_max": 51.01323318481445,
      "activations/layer6_attention_weight_min": -50.967350006103516,
      "activations/layer7_attention_weight_max": 67.66866302490234,
      "activations/layer7_attention_weight_min": -66.26605224609375,
      "activations/layer8_attention_weight_max": 45.48461151123047,
      "activations/layer8_attention_weight_min": -47.20479202270508,
      "activations/layer9_attention_weight_max": 51.7853889465332,
      "activations/layer9_attention_weight_min": -49.56791687011719,
      "epoch": 13.46,
      "learning_rate": 6.386136363636363e-05,
      "loss": 2.7696,
      "step": 231600
    },
    {
      "activations/layer0_attention_weight_max": 15.71725082397461,
      "activations/layer0_attention_weight_min": -15.072813987731934,
      "activations/layer10_attention_weight_max": 34.49741744995117,
      "activations/layer10_attention_weight_min": -33.79280090332031,
      "activations/layer11_attention_weight_max": 30.726341247558594,
      "activations/layer11_attention_weight_min": -33.033958435058594,
      "activations/layer12_attention_weight_max": 17.556739807128906,
      "activations/layer12_attention_weight_min": -23.34373664855957,
      "activations/layer13_attention_weight_max": 33.84897994995117,
      "activations/layer13_attention_weight_min": -28.945955276489258,
      "activations/layer14_attention_weight_max": 34.75931930541992,
      "activations/layer14_attention_weight_min": -28.76803207397461,
      "activations/layer15_attention_weight_max": 30.666072845458984,
      "activations/layer15_attention_weight_min": -27.777849197387695,
      "activations/layer16_attention_weight_max": 29.474123001098633,
      "activations/layer16_attention_weight_min": -31.324602127075195,
      "activations/layer17_attention_weight_max": 49.18744659423828,
      "activations/layer17_attention_weight_min": -45.74354934692383,
      "activations/layer18_attention_weight_max": 44.167877197265625,
      "activations/layer18_attention_weight_min": -38.42485809326172,
      "activations/layer19_attention_weight_max": 24.03668212890625,
      "activations/layer19_attention_weight_min": -21.819061279296875,
      "activations/layer1_attention_weight_max": 16.29543685913086,
      "activations/layer1_attention_weight_min": -14.82673168182373,
      "activations/layer20_attention_weight_max": 20.853168487548828,
      "activations/layer20_attention_weight_min": -20.5791072845459,
      "activations/layer21_attention_weight_max": 32.35933303833008,
      "activations/layer21_attention_weight_min": -21.909738540649414,
      "activations/layer22_attention_weight_max": 29.47382164001465,
      "activations/layer22_attention_weight_min": -26.87152671813965,
      "activations/layer23_attention_weight_max": 36.86808776855469,
      "activations/layer23_attention_weight_min": -22.424800872802734,
      "activations/layer2_attention_weight_max": 31.386144638061523,
      "activations/layer2_attention_weight_min": -30.427345275878906,
      "activations/layer3_attention_weight_max": 93.42959594726562,
      "activations/layer3_attention_weight_min": -94.47443389892578,
      "activations/layer4_attention_weight_max": 96.81388092041016,
      "activations/layer4_attention_weight_min": -95.04796600341797,
      "activations/layer5_attention_weight_max": 70.42366027832031,
      "activations/layer5_attention_weight_min": -79.68659210205078,
      "activations/layer6_attention_weight_max": 53.57270431518555,
      "activations/layer6_attention_weight_min": -52.96360397338867,
      "activations/layer7_attention_weight_max": 68.36949920654297,
      "activations/layer7_attention_weight_min": -66.1964340209961,
      "activations/layer8_attention_weight_max": 45.47441101074219,
      "activations/layer8_attention_weight_min": -46.905521392822266,
      "activations/layer9_attention_weight_max": 46.686241149902344,
      "activations/layer9_attention_weight_min": -47.225196838378906,
      "epoch": 13.46,
      "learning_rate": 6.384280303030302e-05,
      "loss": 2.7819,
      "step": 231650
    },
    {
      "activations/layer0_attention_weight_max": 17.48796844482422,
      "activations/layer0_attention_weight_min": -15.049881935119629,
      "activations/layer10_attention_weight_max": 35.247718811035156,
      "activations/layer10_attention_weight_min": -34.19804382324219,
      "activations/layer11_attention_weight_max": 31.701377868652344,
      "activations/layer11_attention_weight_min": -32.9311637878418,
      "activations/layer12_attention_weight_max": 18.158248901367188,
      "activations/layer12_attention_weight_min": -24.302677154541016,
      "activations/layer13_attention_weight_max": 34.96474075317383,
      "activations/layer13_attention_weight_min": -26.484453201293945,
      "activations/layer14_attention_weight_max": 33.22971725463867,
      "activations/layer14_attention_weight_min": -29.450727462768555,
      "activations/layer15_attention_weight_max": 31.626379013061523,
      "activations/layer15_attention_weight_min": -28.311559677124023,
      "activations/layer16_attention_weight_max": 29.894821166992188,
      "activations/layer16_attention_weight_min": -28.049835205078125,
      "activations/layer17_attention_weight_max": 48.67879104614258,
      "activations/layer17_attention_weight_min": -42.254581451416016,
      "activations/layer18_attention_weight_max": 42.519954681396484,
      "activations/layer18_attention_weight_min": -36.59690475463867,
      "activations/layer19_attention_weight_max": 21.65583610534668,
      "activations/layer19_attention_weight_min": -20.612245559692383,
      "activations/layer1_attention_weight_max": 16.973854064941406,
      "activations/layer1_attention_weight_min": -14.115121841430664,
      "activations/layer20_attention_weight_max": 19.066509246826172,
      "activations/layer20_attention_weight_min": -21.438461303710938,
      "activations/layer21_attention_weight_max": 31.02130699157715,
      "activations/layer21_attention_weight_min": -20.902191162109375,
      "activations/layer22_attention_weight_max": 28.130708694458008,
      "activations/layer22_attention_weight_min": -24.129974365234375,
      "activations/layer23_attention_weight_max": 35.04975891113281,
      "activations/layer23_attention_weight_min": -22.300796508789062,
      "activations/layer2_attention_weight_max": 31.735563278198242,
      "activations/layer2_attention_weight_min": -30.48345184326172,
      "activations/layer3_attention_weight_max": 92.46923065185547,
      "activations/layer3_attention_weight_min": -91.71137237548828,
      "activations/layer4_attention_weight_max": 97.6439208984375,
      "activations/layer4_attention_weight_min": -93.59907531738281,
      "activations/layer5_attention_weight_max": 72.76715087890625,
      "activations/layer5_attention_weight_min": -75.87171936035156,
      "activations/layer6_attention_weight_max": 49.842979431152344,
      "activations/layer6_attention_weight_min": -51.27092742919922,
      "activations/layer7_attention_weight_max": 65.80022430419922,
      "activations/layer7_attention_weight_min": -62.75099182128906,
      "activations/layer8_attention_weight_max": 44.07014465332031,
      "activations/layer8_attention_weight_min": -46.95647430419922,
      "activations/layer9_attention_weight_max": 45.03348922729492,
      "activations/layer9_attention_weight_min": -44.857669830322266,
      "epoch": 13.46,
      "learning_rate": 6.382386363636362e-05,
      "loss": 2.7821,
      "step": 231700
    },
    {
      "activations/layer0_attention_weight_max": 15.519118309020996,
      "activations/layer0_attention_weight_min": -14.562753677368164,
      "activations/layer10_attention_weight_max": 43.58433532714844,
      "activations/layer10_attention_weight_min": -41.05545425415039,
      "activations/layer11_attention_weight_max": 41.64052200317383,
      "activations/layer11_attention_weight_min": -35.63886260986328,
      "activations/layer12_attention_weight_max": 19.116016387939453,
      "activations/layer12_attention_weight_min": -23.310056686401367,
      "activations/layer13_attention_weight_max": 40.097511291503906,
      "activations/layer13_attention_weight_min": -28.440298080444336,
      "activations/layer14_attention_weight_max": 39.07803726196289,
      "activations/layer14_attention_weight_min": -32.596412658691406,
      "activations/layer15_attention_weight_max": 37.765960693359375,
      "activations/layer15_attention_weight_min": -31.084352493286133,
      "activations/layer16_attention_weight_max": 37.84638214111328,
      "activations/layer16_attention_weight_min": -35.49113082885742,
      "activations/layer17_attention_weight_max": 62.40383529663086,
      "activations/layer17_attention_weight_min": -49.72412109375,
      "activations/layer18_attention_weight_max": 55.26930618286133,
      "activations/layer18_attention_weight_min": -42.34962844848633,
      "activations/layer19_attention_weight_max": 25.8375301361084,
      "activations/layer19_attention_weight_min": -22.43549346923828,
      "activations/layer1_attention_weight_max": 17.525270462036133,
      "activations/layer1_attention_weight_min": -16.46160316467285,
      "activations/layer20_attention_weight_max": 28.193225860595703,
      "activations/layer20_attention_weight_min": -25.046030044555664,
      "activations/layer21_attention_weight_max": 50.58953857421875,
      "activations/layer21_attention_weight_min": -25.66192626953125,
      "activations/layer22_attention_weight_max": 38.69486999511719,
      "activations/layer22_attention_weight_min": -25.80333137512207,
      "activations/layer23_attention_weight_max": 47.10663604736328,
      "activations/layer23_attention_weight_min": -27.949356079101562,
      "activations/layer2_attention_weight_max": 33.05158233642578,
      "activations/layer2_attention_weight_min": -31.992713928222656,
      "activations/layer3_attention_weight_max": 97.43909454345703,
      "activations/layer3_attention_weight_min": -97.23180389404297,
      "activations/layer4_attention_weight_max": 105.13172912597656,
      "activations/layer4_attention_weight_min": -97.8563461303711,
      "activations/layer5_attention_weight_max": 71.88494873046875,
      "activations/layer5_attention_weight_min": -77.61581420898438,
      "activations/layer6_attention_weight_max": 54.768516540527344,
      "activations/layer6_attention_weight_min": -57.1884651184082,
      "activations/layer7_attention_weight_max": 74.53132629394531,
      "activations/layer7_attention_weight_min": -74.69961547851562,
      "activations/layer8_attention_weight_max": 51.71876907348633,
      "activations/layer8_attention_weight_min": -55.98863220214844,
      "activations/layer9_attention_weight_max": 59.61072540283203,
      "activations/layer9_attention_weight_min": -55.6880989074707,
      "epoch": 13.47,
      "learning_rate": 6.380492424242424e-05,
      "loss": 2.7464,
      "step": 231750
    },
    {
      "activations/layer0_attention_weight_max": 15.88073444366455,
      "activations/layer0_attention_weight_min": -14.580942153930664,
      "activations/layer10_attention_weight_max": 35.227455139160156,
      "activations/layer10_attention_weight_min": -35.621185302734375,
      "activations/layer11_attention_weight_max": 31.290016174316406,
      "activations/layer11_attention_weight_min": -32.074195861816406,
      "activations/layer12_attention_weight_max": 16.78782081604004,
      "activations/layer12_attention_weight_min": -27.652809143066406,
      "activations/layer13_attention_weight_max": 34.0589599609375,
      "activations/layer13_attention_weight_min": -29.78546905517578,
      "activations/layer14_attention_weight_max": 33.79966735839844,
      "activations/layer14_attention_weight_min": -32.368770599365234,
      "activations/layer15_attention_weight_max": 31.186880111694336,
      "activations/layer15_attention_weight_min": -29.860801696777344,
      "activations/layer16_attention_weight_max": 34.345115661621094,
      "activations/layer16_attention_weight_min": -29.355865478515625,
      "activations/layer17_attention_weight_max": 51.738677978515625,
      "activations/layer17_attention_weight_min": -45.60927200317383,
      "activations/layer18_attention_weight_max": 44.72612762451172,
      "activations/layer18_attention_weight_min": -40.10260772705078,
      "activations/layer19_attention_weight_max": 20.587934494018555,
      "activations/layer19_attention_weight_min": -22.897560119628906,
      "activations/layer1_attention_weight_max": 16.202220916748047,
      "activations/layer1_attention_weight_min": -15.368764877319336,
      "activations/layer20_attention_weight_max": 21.87439727783203,
      "activations/layer20_attention_weight_min": -21.99211883544922,
      "activations/layer21_attention_weight_max": 38.37715148925781,
      "activations/layer21_attention_weight_min": -22.34930419921875,
      "activations/layer22_attention_weight_max": 28.679840087890625,
      "activations/layer22_attention_weight_min": -26.532543182373047,
      "activations/layer23_attention_weight_max": 33.34211730957031,
      "activations/layer23_attention_weight_min": -24.589580535888672,
      "activations/layer2_attention_weight_max": 32.35859298706055,
      "activations/layer2_attention_weight_min": -31.411130905151367,
      "activations/layer3_attention_weight_max": 102.54889678955078,
      "activations/layer3_attention_weight_min": -107.85939025878906,
      "activations/layer4_attention_weight_max": 96.53965759277344,
      "activations/layer4_attention_weight_min": -95.0948715209961,
      "activations/layer5_attention_weight_max": 73.68417358398438,
      "activations/layer5_attention_weight_min": -81.83351135253906,
      "activations/layer6_attention_weight_max": 51.73223114013672,
      "activations/layer6_attention_weight_min": -53.00692367553711,
      "activations/layer7_attention_weight_max": 68.79366302490234,
      "activations/layer7_attention_weight_min": -63.06486129760742,
      "activations/layer8_attention_weight_max": 45.14320373535156,
      "activations/layer8_attention_weight_min": -49.01788330078125,
      "activations/layer9_attention_weight_max": 45.93235397338867,
      "activations/layer9_attention_weight_min": -45.81098175048828,
      "epoch": 13.47,
      "learning_rate": 6.378598484848484e-05,
      "loss": 2.7526,
      "step": 231800
    },
    {
      "activations/layer0_attention_weight_max": 16.535654067993164,
      "activations/layer0_attention_weight_min": -13.806806564331055,
      "activations/layer10_attention_weight_max": 34.97566604614258,
      "activations/layer10_attention_weight_min": -35.41218948364258,
      "activations/layer11_attention_weight_max": 31.684097290039062,
      "activations/layer11_attention_weight_min": -31.308462142944336,
      "activations/layer12_attention_weight_max": 19.437650680541992,
      "activations/layer12_attention_weight_min": -25.781076431274414,
      "activations/layer13_attention_weight_max": 37.85976791381836,
      "activations/layer13_attention_weight_min": -29.978784561157227,
      "activations/layer14_attention_weight_max": 33.0512809753418,
      "activations/layer14_attention_weight_min": -31.419374465942383,
      "activations/layer15_attention_weight_max": 33.14234161376953,
      "activations/layer15_attention_weight_min": -30.206995010375977,
      "activations/layer16_attention_weight_max": 30.678035736083984,
      "activations/layer16_attention_weight_min": -30.04537010192871,
      "activations/layer17_attention_weight_max": 51.027488708496094,
      "activations/layer17_attention_weight_min": -46.27992630004883,
      "activations/layer18_attention_weight_max": 47.737300872802734,
      "activations/layer18_attention_weight_min": -39.56645202636719,
      "activations/layer19_attention_weight_max": 26.05732536315918,
      "activations/layer19_attention_weight_min": -25.497827529907227,
      "activations/layer1_attention_weight_max": 17.35332679748535,
      "activations/layer1_attention_weight_min": -14.026103019714355,
      "activations/layer20_attention_weight_max": 21.785913467407227,
      "activations/layer20_attention_weight_min": -24.184341430664062,
      "activations/layer21_attention_weight_max": 45.1877326965332,
      "activations/layer21_attention_weight_min": -24.580034255981445,
      "activations/layer22_attention_weight_max": 29.61037254333496,
      "activations/layer22_attention_weight_min": -29.492918014526367,
      "activations/layer23_attention_weight_max": 37.45392990112305,
      "activations/layer23_attention_weight_min": -24.33148956298828,
      "activations/layer2_attention_weight_max": 31.08942222595215,
      "activations/layer2_attention_weight_min": -31.714139938354492,
      "activations/layer3_attention_weight_max": 91.961181640625,
      "activations/layer3_attention_weight_min": -97.28219604492188,
      "activations/layer4_attention_weight_max": 94.42626190185547,
      "activations/layer4_attention_weight_min": -92.66173553466797,
      "activations/layer5_attention_weight_max": 74.42485046386719,
      "activations/layer5_attention_weight_min": -72.61880493164062,
      "activations/layer6_attention_weight_max": 53.0460090637207,
      "activations/layer6_attention_weight_min": -51.209774017333984,
      "activations/layer7_attention_weight_max": 68.2378921508789,
      "activations/layer7_attention_weight_min": -63.09550857543945,
      "activations/layer8_attention_weight_max": 47.748802185058594,
      "activations/layer8_attention_weight_min": -46.22832489013672,
      "activations/layer9_attention_weight_max": 46.84410095214844,
      "activations/layer9_attention_weight_min": -44.70331954956055,
      "epoch": 13.47,
      "learning_rate": 6.376704545454544e-05,
      "loss": 2.7597,
      "step": 231850
    },
    {
      "activations/layer0_attention_weight_max": 15.823054313659668,
      "activations/layer0_attention_weight_min": -13.938480377197266,
      "activations/layer10_attention_weight_max": 35.9553337097168,
      "activations/layer10_attention_weight_min": -35.58041763305664,
      "activations/layer11_attention_weight_max": 30.96906852722168,
      "activations/layer11_attention_weight_min": -33.297508239746094,
      "activations/layer12_attention_weight_max": 18.140161514282227,
      "activations/layer12_attention_weight_min": -30.3959903717041,
      "activations/layer13_attention_weight_max": 36.123435974121094,
      "activations/layer13_attention_weight_min": -27.990245819091797,
      "activations/layer14_attention_weight_max": 34.99801254272461,
      "activations/layer14_attention_weight_min": -29.114009857177734,
      "activations/layer15_attention_weight_max": 32.697689056396484,
      "activations/layer15_attention_weight_min": -27.648906707763672,
      "activations/layer16_attention_weight_max": 28.981475830078125,
      "activations/layer16_attention_weight_min": -29.481739044189453,
      "activations/layer17_attention_weight_max": 48.72822952270508,
      "activations/layer17_attention_weight_min": -46.793453216552734,
      "activations/layer18_attention_weight_max": 43.66085433959961,
      "activations/layer18_attention_weight_min": -37.27187728881836,
      "activations/layer19_attention_weight_max": 19.564725875854492,
      "activations/layer19_attention_weight_min": -22.342876434326172,
      "activations/layer1_attention_weight_max": 17.11268424987793,
      "activations/layer1_attention_weight_min": -15.447696685791016,
      "activations/layer20_attention_weight_max": 20.19572639465332,
      "activations/layer20_attention_weight_min": -19.760377883911133,
      "activations/layer21_attention_weight_max": 39.35638427734375,
      "activations/layer21_attention_weight_min": -22.227718353271484,
      "activations/layer22_attention_weight_max": 26.649839401245117,
      "activations/layer22_attention_weight_min": -22.696062088012695,
      "activations/layer23_attention_weight_max": 36.24622344970703,
      "activations/layer23_attention_weight_min": -22.721487045288086,
      "activations/layer2_attention_weight_max": 31.036895751953125,
      "activations/layer2_attention_weight_min": -29.81772232055664,
      "activations/layer3_attention_weight_max": 96.28677368164062,
      "activations/layer3_attention_weight_min": -98.37298583984375,
      "activations/layer4_attention_weight_max": 97.8134765625,
      "activations/layer4_attention_weight_min": -98.5385513305664,
      "activations/layer5_attention_weight_max": 71.06591033935547,
      "activations/layer5_attention_weight_min": -77.69851684570312,
      "activations/layer6_attention_weight_max": 49.91318893432617,
      "activations/layer6_attention_weight_min": -52.853858947753906,
      "activations/layer7_attention_weight_max": 65.08625793457031,
      "activations/layer7_attention_weight_min": -64.48438262939453,
      "activations/layer8_attention_weight_max": 46.30073928833008,
      "activations/layer8_attention_weight_min": -48.31504821777344,
      "activations/layer9_attention_weight_max": 45.34599304199219,
      "activations/layer9_attention_weight_min": -48.516815185546875,
      "epoch": 13.47,
      "learning_rate": 6.374810606060606e-05,
      "loss": 2.7622,
      "step": 231900
    },
    {
      "activations/layer0_attention_weight_max": 15.558667182922363,
      "activations/layer0_attention_weight_min": -14.204534530639648,
      "activations/layer10_attention_weight_max": 37.773475646972656,
      "activations/layer10_attention_weight_min": -35.575286865234375,
      "activations/layer11_attention_weight_max": 33.895973205566406,
      "activations/layer11_attention_weight_min": -34.35612487792969,
      "activations/layer12_attention_weight_max": 20.60587501525879,
      "activations/layer12_attention_weight_min": -26.4232234954834,
      "activations/layer13_attention_weight_max": 36.00532531738281,
      "activations/layer13_attention_weight_min": -29.470043182373047,
      "activations/layer14_attention_weight_max": 37.78888702392578,
      "activations/layer14_attention_weight_min": -32.047149658203125,
      "activations/layer15_attention_weight_max": 36.27296447753906,
      "activations/layer15_attention_weight_min": -29.473241806030273,
      "activations/layer16_attention_weight_max": 33.365333557128906,
      "activations/layer16_attention_weight_min": -29.892141342163086,
      "activations/layer17_attention_weight_max": 52.55636215209961,
      "activations/layer17_attention_weight_min": -44.10725784301758,
      "activations/layer18_attention_weight_max": 48.05643844604492,
      "activations/layer18_attention_weight_min": -38.308528900146484,
      "activations/layer19_attention_weight_max": 23.772615432739258,
      "activations/layer19_attention_weight_min": -24.330190658569336,
      "activations/layer1_attention_weight_max": 16.80925750732422,
      "activations/layer1_attention_weight_min": -15.23130989074707,
      "activations/layer20_attention_weight_max": 22.993389129638672,
      "activations/layer20_attention_weight_min": -23.5802001953125,
      "activations/layer21_attention_weight_max": 39.934505462646484,
      "activations/layer21_attention_weight_min": -22.211580276489258,
      "activations/layer22_attention_weight_max": 31.476696014404297,
      "activations/layer22_attention_weight_min": -27.727890014648438,
      "activations/layer23_attention_weight_max": 43.079673767089844,
      "activations/layer23_attention_weight_min": -26.570068359375,
      "activations/layer2_attention_weight_max": 33.46120834350586,
      "activations/layer2_attention_weight_min": -30.40833854675293,
      "activations/layer3_attention_weight_max": 96.5820541381836,
      "activations/layer3_attention_weight_min": -97.46479034423828,
      "activations/layer4_attention_weight_max": 98.67681884765625,
      "activations/layer4_attention_weight_min": -91.6568603515625,
      "activations/layer5_attention_weight_max": 68.62129974365234,
      "activations/layer5_attention_weight_min": -77.33219146728516,
      "activations/layer6_attention_weight_max": 50.477928161621094,
      "activations/layer6_attention_weight_min": -51.90073776245117,
      "activations/layer7_attention_weight_max": 64.73701477050781,
      "activations/layer7_attention_weight_min": -65.34183502197266,
      "activations/layer8_attention_weight_max": 45.182891845703125,
      "activations/layer8_attention_weight_min": -46.37198257446289,
      "activations/layer9_attention_weight_max": 45.86122131347656,
      "activations/layer9_attention_weight_min": -47.11791229248047,
      "epoch": 13.48,
      "learning_rate": 6.372916666666666e-05,
      "loss": 2.7535,
      "step": 231950
    },
    {
      "activations/layer0_attention_weight_max": 16.010507583618164,
      "activations/layer0_attention_weight_min": -14.452432632446289,
      "activations/layer10_attention_weight_max": 37.78548049926758,
      "activations/layer10_attention_weight_min": -37.213436126708984,
      "activations/layer11_attention_weight_max": 33.763694763183594,
      "activations/layer11_attention_weight_min": -34.27436065673828,
      "activations/layer12_attention_weight_max": 17.292377471923828,
      "activations/layer12_attention_weight_min": -26.990737915039062,
      "activations/layer13_attention_weight_max": 37.55935287475586,
      "activations/layer13_attention_weight_min": -27.74562644958496,
      "activations/layer14_attention_weight_max": 35.50423049926758,
      "activations/layer14_attention_weight_min": -32.92508316040039,
      "activations/layer15_attention_weight_max": 33.562522888183594,
      "activations/layer15_attention_weight_min": -31.09430694580078,
      "activations/layer16_attention_weight_max": 30.783679962158203,
      "activations/layer16_attention_weight_min": -30.721895217895508,
      "activations/layer17_attention_weight_max": 52.01036071777344,
      "activations/layer17_attention_weight_min": -47.05998992919922,
      "activations/layer18_attention_weight_max": 46.811973571777344,
      "activations/layer18_attention_weight_min": -39.99348449707031,
      "activations/layer19_attention_weight_max": 22.523193359375,
      "activations/layer19_attention_weight_min": -22.803245544433594,
      "activations/layer1_attention_weight_max": 16.588733673095703,
      "activations/layer1_attention_weight_min": -14.855416297912598,
      "activations/layer20_attention_weight_max": 22.51752471923828,
      "activations/layer20_attention_weight_min": -21.22150421142578,
      "activations/layer21_attention_weight_max": 33.47743225097656,
      "activations/layer21_attention_weight_min": -22.142078399658203,
      "activations/layer22_attention_weight_max": 34.44624710083008,
      "activations/layer22_attention_weight_min": -26.35526466369629,
      "activations/layer23_attention_weight_max": 40.94827651977539,
      "activations/layer23_attention_weight_min": -23.73828887939453,
      "activations/layer2_attention_weight_max": 28.970495223999023,
      "activations/layer2_attention_weight_min": -28.428749084472656,
      "activations/layer3_attention_weight_max": 94.84610748291016,
      "activations/layer3_attention_weight_min": -92.27400207519531,
      "activations/layer4_attention_weight_max": 95.4986572265625,
      "activations/layer4_attention_weight_min": -95.04573822021484,
      "activations/layer5_attention_weight_max": 71.64784240722656,
      "activations/layer5_attention_weight_min": -78.67443084716797,
      "activations/layer6_attention_weight_max": 52.520870208740234,
      "activations/layer6_attention_weight_min": -53.23946762084961,
      "activations/layer7_attention_weight_max": 67.7926025390625,
      "activations/layer7_attention_weight_min": -70.34451293945312,
      "activations/layer8_attention_weight_max": 47.072898864746094,
      "activations/layer8_attention_weight_min": -47.544918060302734,
      "activations/layer9_attention_weight_max": 48.600521087646484,
      "activations/layer9_attention_weight_min": -49.35136032104492,
      "epoch": 13.48,
      "learning_rate": 6.371022727272727e-05,
      "loss": 2.7488,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_loss": 2.720703125,
      "eval_runtime": 8.4933,
      "eval_samples_per_second": 505.573,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_openwebtext_loss": 2.720703125,
      "eval_openwebtext_ppl": 15.190999662372045,
      "eval_openwebtext_runtime": 8.4933,
      "eval_openwebtext_samples_per_second": 505.573,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_wikitext_loss": 2.951171875,
      "eval_wikitext_ppl": 19.128356641900336,
      "eval_wikitext_runtime": 1.9977,
      "eval_wikitext_samples_per_second": 228.261,
      "step": 232000
    },
    {
      "epoch": 13.48,
      "eval_lambada_loss": 2.583984375,
      "eval_lambada_ppl": 13.249825400173327,
      "eval_lambada_runtime": 9.5867,
      "eval_lambada_samples_per_second": 507.892,
      "step": 232000
    },
    {
      "activations/layer0_attention_weight_max": 15.711585998535156,
      "activations/layer0_attention_weight_min": -15.040403366088867,
      "activations/layer10_attention_weight_max": 38.73879623413086,
      "activations/layer10_attention_weight_min": -38.23651885986328,
      "activations/layer11_attention_weight_max": 33.80400085449219,
      "activations/layer11_attention_weight_min": -35.227272033691406,
      "activations/layer12_attention_weight_max": 19.6835880279541,
      "activations/layer12_attention_weight_min": -24.24154281616211,
      "activations/layer13_attention_weight_max": 34.3900032043457,
      "activations/layer13_attention_weight_min": -30.38330841064453,
      "activations/layer14_attention_weight_max": 35.22679901123047,
      "activations/layer14_attention_weight_min": -33.064998626708984,
      "activations/layer15_attention_weight_max": 32.45836639404297,
      "activations/layer15_attention_weight_min": -30.974163055419922,
      "activations/layer16_attention_weight_max": 35.89371109008789,
      "activations/layer16_attention_weight_min": -32.522003173828125,
      "activations/layer17_attention_weight_max": 53.0595588684082,
      "activations/layer17_attention_weight_min": -47.54784393310547,
      "activations/layer18_attention_weight_max": 47.6953239440918,
      "activations/layer18_attention_weight_min": -38.91963577270508,
      "activations/layer19_attention_weight_max": 22.974254608154297,
      "activations/layer19_attention_weight_min": -22.174285888671875,
      "activations/layer1_attention_weight_max": 17.361677169799805,
      "activations/layer1_attention_weight_min": -15.078529357910156,
      "activations/layer20_attention_weight_max": 24.43035316467285,
      "activations/layer20_attention_weight_min": -22.223587036132812,
      "activations/layer21_attention_weight_max": 38.389686584472656,
      "activations/layer21_attention_weight_min": -23.364177703857422,
      "activations/layer22_attention_weight_max": 33.49921417236328,
      "activations/layer22_attention_weight_min": -27.89529037475586,
      "activations/layer23_attention_weight_max": 44.953609466552734,
      "activations/layer23_attention_weight_min": -24.77138328552246,
      "activations/layer2_attention_weight_max": 33.690486907958984,
      "activations/layer2_attention_weight_min": -31.290943145751953,
      "activations/layer3_attention_weight_max": 97.82667541503906,
      "activations/layer3_attention_weight_min": -100.97942352294922,
      "activations/layer4_attention_weight_max": 102.67642974853516,
      "activations/layer4_attention_weight_min": -95.39453125,
      "activations/layer5_attention_weight_max": 75.96251678466797,
      "activations/layer5_attention_weight_min": -81.22705078125,
      "activations/layer6_attention_weight_max": 53.152793884277344,
      "activations/layer6_attention_weight_min": -52.64474868774414,
      "activations/layer7_attention_weight_max": 67.54502868652344,
      "activations/layer7_attention_weight_min": -66.5992431640625,
      "activations/layer8_attention_weight_max": 48.514041900634766,
      "activations/layer8_attention_weight_min": -49.30247497558594,
      "activations/layer9_attention_weight_max": 48.331298828125,
      "activations/layer9_attention_weight_min": -49.26367950439453,
      "epoch": 13.48,
      "learning_rate": 6.369128787878788e-05,
      "loss": 2.7627,
      "step": 232050
    },
    {
      "activations/layer0_attention_weight_max": 16.195981979370117,
      "activations/layer0_attention_weight_min": -14.515947341918945,
      "activations/layer10_attention_weight_max": 40.94013595581055,
      "activations/layer10_attention_weight_min": -40.89896774291992,
      "activations/layer11_attention_weight_max": 34.58890151977539,
      "activations/layer11_attention_weight_min": -34.98412322998047,
      "activations/layer12_attention_weight_max": 17.827741622924805,
      "activations/layer12_attention_weight_min": -25.216644287109375,
      "activations/layer13_attention_weight_max": 41.85775375366211,
      "activations/layer13_attention_weight_min": -31.72641372680664,
      "activations/layer14_attention_weight_max": 39.91560745239258,
      "activations/layer14_attention_weight_min": -35.01099395751953,
      "activations/layer15_attention_weight_max": 36.54808044433594,
      "activations/layer15_attention_weight_min": -29.887304306030273,
      "activations/layer16_attention_weight_max": 33.843910217285156,
      "activations/layer16_attention_weight_min": -32.498905181884766,
      "activations/layer17_attention_weight_max": 54.956302642822266,
      "activations/layer17_attention_weight_min": -48.44710159301758,
      "activations/layer18_attention_weight_max": 47.080604553222656,
      "activations/layer18_attention_weight_min": -40.93152618408203,
      "activations/layer19_attention_weight_max": 23.8179874420166,
      "activations/layer19_attention_weight_min": -23.010324478149414,
      "activations/layer1_attention_weight_max": 16.38443946838379,
      "activations/layer1_attention_weight_min": -15.866438865661621,
      "activations/layer20_attention_weight_max": 25.37367820739746,
      "activations/layer20_attention_weight_min": -26.135692596435547,
      "activations/layer21_attention_weight_max": 43.11088180541992,
      "activations/layer21_attention_weight_min": -26.61322021484375,
      "activations/layer22_attention_weight_max": 31.790815353393555,
      "activations/layer22_attention_weight_min": -28.726154327392578,
      "activations/layer23_attention_weight_max": 35.86711120605469,
      "activations/layer23_attention_weight_min": -24.634098052978516,
      "activations/layer2_attention_weight_max": 34.73291778564453,
      "activations/layer2_attention_weight_min": -32.19422149658203,
      "activations/layer3_attention_weight_max": 98.38693237304688,
      "activations/layer3_attention_weight_min": -98.38323211669922,
      "activations/layer4_attention_weight_max": 98.30167388916016,
      "activations/layer4_attention_weight_min": -96.19481658935547,
      "activations/layer5_attention_weight_max": 70.4979019165039,
      "activations/layer5_attention_weight_min": -79.31912231445312,
      "activations/layer6_attention_weight_max": 51.98903274536133,
      "activations/layer6_attention_weight_min": -53.53520584106445,
      "activations/layer7_attention_weight_max": 69.27416229248047,
      "activations/layer7_attention_weight_min": -69.48517608642578,
      "activations/layer8_attention_weight_max": 51.992427825927734,
      "activations/layer8_attention_weight_min": -51.87788009643555,
      "activations/layer9_attention_weight_max": 57.045223236083984,
      "activations/layer9_attention_weight_min": -50.30172348022461,
      "epoch": 13.49,
      "learning_rate": 6.367234848484848e-05,
      "loss": 2.7741,
      "step": 232100
    },
    {
      "activations/layer0_attention_weight_max": 16.252140045166016,
      "activations/layer0_attention_weight_min": -14.438920974731445,
      "activations/layer10_attention_weight_max": 38.60232925415039,
      "activations/layer10_attention_weight_min": -36.73359298706055,
      "activations/layer11_attention_weight_max": 35.185951232910156,
      "activations/layer11_attention_weight_min": -33.77812194824219,
      "activations/layer12_attention_weight_max": 21.6717586517334,
      "activations/layer12_attention_weight_min": -25.669431686401367,
      "activations/layer13_attention_weight_max": 44.55241012573242,
      "activations/layer13_attention_weight_min": -28.646804809570312,
      "activations/layer14_attention_weight_max": 37.622291564941406,
      "activations/layer14_attention_weight_min": -30.725770950317383,
      "activations/layer15_attention_weight_max": 35.082706451416016,
      "activations/layer15_attention_weight_min": -30.6968994140625,
      "activations/layer16_attention_weight_max": 32.51968765258789,
      "activations/layer16_attention_weight_min": -30.902509689331055,
      "activations/layer17_attention_weight_max": 57.38330841064453,
      "activations/layer17_attention_weight_min": -45.174034118652344,
      "activations/layer18_attention_weight_max": 47.019161224365234,
      "activations/layer18_attention_weight_min": -37.00467300415039,
      "activations/layer19_attention_weight_max": 25.383169174194336,
      "activations/layer19_attention_weight_min": -23.11439323425293,
      "activations/layer1_attention_weight_max": 16.53714370727539,
      "activations/layer1_attention_weight_min": -15.338078498840332,
      "activations/layer20_attention_weight_max": 24.831953048706055,
      "activations/layer20_attention_weight_min": -23.78802490234375,
      "activations/layer21_attention_weight_max": 39.994140625,
      "activations/layer21_attention_weight_min": -22.98065757751465,
      "activations/layer22_attention_weight_max": 32.38044738769531,
      "activations/layer22_attention_weight_min": -24.828643798828125,
      "activations/layer23_attention_weight_max": 39.809661865234375,
      "activations/layer23_attention_weight_min": -24.459148406982422,
      "activations/layer2_attention_weight_max": 33.48532485961914,
      "activations/layer2_attention_weight_min": -31.55738067626953,
      "activations/layer3_attention_weight_max": 97.72065734863281,
      "activations/layer3_attention_weight_min": -98.21431732177734,
      "activations/layer4_attention_weight_max": 98.04646301269531,
      "activations/layer4_attention_weight_min": -97.48074340820312,
      "activations/layer5_attention_weight_max": 73.22113800048828,
      "activations/layer5_attention_weight_min": -80.31742858886719,
      "activations/layer6_attention_weight_max": 51.41299057006836,
      "activations/layer6_attention_weight_min": -54.99604034423828,
      "activations/layer7_attention_weight_max": 67.63053131103516,
      "activations/layer7_attention_weight_min": -68.6346664428711,
      "activations/layer8_attention_weight_max": 48.989253997802734,
      "activations/layer8_attention_weight_min": -51.146244049072266,
      "activations/layer9_attention_weight_max": 48.73686218261719,
      "activations/layer9_attention_weight_min": -48.700077056884766,
      "epoch": 13.49,
      "learning_rate": 6.365340909090909e-05,
      "loss": 2.7651,
      "step": 232150
    },
    {
      "activations/layer0_attention_weight_max": 15.512433052062988,
      "activations/layer0_attention_weight_min": -14.192468643188477,
      "activations/layer10_attention_weight_max": 46.5181999206543,
      "activations/layer10_attention_weight_min": -47.6226921081543,
      "activations/layer11_attention_weight_max": 44.14870834350586,
      "activations/layer11_attention_weight_min": -40.694793701171875,
      "activations/layer12_attention_weight_max": 20.437557220458984,
      "activations/layer12_attention_weight_min": -33.40668487548828,
      "activations/layer13_attention_weight_max": 36.59877014160156,
      "activations/layer13_attention_weight_min": -29.033641815185547,
      "activations/layer14_attention_weight_max": 36.281837463378906,
      "activations/layer14_attention_weight_min": -31.846637725830078,
      "activations/layer15_attention_weight_max": 34.4589958190918,
      "activations/layer15_attention_weight_min": -28.927820205688477,
      "activations/layer16_attention_weight_max": 32.70951461791992,
      "activations/layer16_attention_weight_min": -31.02722930908203,
      "activations/layer17_attention_weight_max": 51.255279541015625,
      "activations/layer17_attention_weight_min": -46.469722747802734,
      "activations/layer18_attention_weight_max": 47.4217414855957,
      "activations/layer18_attention_weight_min": -41.19913101196289,
      "activations/layer19_attention_weight_max": 21.859060287475586,
      "activations/layer19_attention_weight_min": -21.273399353027344,
      "activations/layer1_attention_weight_max": 17.955501556396484,
      "activations/layer1_attention_weight_min": -14.58279037475586,
      "activations/layer20_attention_weight_max": 22.409412384033203,
      "activations/layer20_attention_weight_min": -23.424358367919922,
      "activations/layer21_attention_weight_max": 37.2496223449707,
      "activations/layer21_attention_weight_min": -26.542850494384766,
      "activations/layer22_attention_weight_max": 33.35979080200195,
      "activations/layer22_attention_weight_min": -28.1151123046875,
      "activations/layer23_attention_weight_max": 38.14122009277344,
      "activations/layer23_attention_weight_min": -24.86362075805664,
      "activations/layer2_attention_weight_max": 34.13389587402344,
      "activations/layer2_attention_weight_min": -34.46859359741211,
      "activations/layer3_attention_weight_max": 102.43136596679688,
      "activations/layer3_attention_weight_min": -104.873291015625,
      "activations/layer4_attention_weight_max": 105.26496887207031,
      "activations/layer4_attention_weight_min": -100.2361068725586,
      "activations/layer5_attention_weight_max": 79.49270629882812,
      "activations/layer5_attention_weight_min": -81.01882934570312,
      "activations/layer6_attention_weight_max": 56.21356964111328,
      "activations/layer6_attention_weight_min": -56.85710906982422,
      "activations/layer7_attention_weight_max": 82.63179779052734,
      "activations/layer7_attention_weight_min": -73.3654556274414,
      "activations/layer8_attention_weight_max": 59.61736297607422,
      "activations/layer8_attention_weight_min": -57.49348449707031,
      "activations/layer9_attention_weight_max": 65.3376235961914,
      "activations/layer9_attention_weight_min": -57.50765609741211,
      "epoch": 13.49,
      "learning_rate": 6.36344696969697e-05,
      "loss": 2.7614,
      "step": 232200
    },
    {
      "activations/layer0_attention_weight_max": 15.258550643920898,
      "activations/layer0_attention_weight_min": -14.208492279052734,
      "activations/layer10_attention_weight_max": 36.28049850463867,
      "activations/layer10_attention_weight_min": -36.62126541137695,
      "activations/layer11_attention_weight_max": 32.43610382080078,
      "activations/layer11_attention_weight_min": -34.30764389038086,
      "activations/layer12_attention_weight_max": 21.004114151000977,
      "activations/layer12_attention_weight_min": -25.74016761779785,
      "activations/layer13_attention_weight_max": 53.71220397949219,
      "activations/layer13_attention_weight_min": -33.219276428222656,
      "activations/layer14_attention_weight_max": 38.38837432861328,
      "activations/layer14_attention_weight_min": -31.559202194213867,
      "activations/layer15_attention_weight_max": 35.22763442993164,
      "activations/layer15_attention_weight_min": -29.163692474365234,
      "activations/layer16_attention_weight_max": 36.198089599609375,
      "activations/layer16_attention_weight_min": -31.336572647094727,
      "activations/layer17_attention_weight_max": 55.775062561035156,
      "activations/layer17_attention_weight_min": -46.2158203125,
      "activations/layer18_attention_weight_max": 53.04479217529297,
      "activations/layer18_attention_weight_min": -39.68901824951172,
      "activations/layer19_attention_weight_max": 28.661502838134766,
      "activations/layer19_attention_weight_min": -22.97186279296875,
      "activations/layer1_attention_weight_max": 17.077991485595703,
      "activations/layer1_attention_weight_min": -15.816018104553223,
      "activations/layer20_attention_weight_max": 27.18816566467285,
      "activations/layer20_attention_weight_min": -21.72578239440918,
      "activations/layer21_attention_weight_max": 52.23102951049805,
      "activations/layer21_attention_weight_min": -23.9899845123291,
      "activations/layer22_attention_weight_max": 43.464698791503906,
      "activations/layer22_attention_weight_min": -27.504331588745117,
      "activations/layer23_attention_weight_max": 43.7480583190918,
      "activations/layer23_attention_weight_min": -25.832813262939453,
      "activations/layer2_attention_weight_max": 33.20547866821289,
      "activations/layer2_attention_weight_min": -32.298439025878906,
      "activations/layer3_attention_weight_max": 93.435546875,
      "activations/layer3_attention_weight_min": -94.99016571044922,
      "activations/layer4_attention_weight_max": 97.26101684570312,
      "activations/layer4_attention_weight_min": -94.37889862060547,
      "activations/layer5_attention_weight_max": 68.844970703125,
      "activations/layer5_attention_weight_min": -78.72394561767578,
      "activations/layer6_attention_weight_max": 53.17063522338867,
      "activations/layer6_attention_weight_min": -52.32523727416992,
      "activations/layer7_attention_weight_max": 67.47511291503906,
      "activations/layer7_attention_weight_min": -66.9754638671875,
      "activations/layer8_attention_weight_max": 44.585472106933594,
      "activations/layer8_attention_weight_min": -48.73704528808594,
      "activations/layer9_attention_weight_max": 54.1926383972168,
      "activations/layer9_attention_weight_min": -50.365718841552734,
      "epoch": 13.5,
      "learning_rate": 6.36155303030303e-05,
      "loss": 2.7763,
      "step": 232250
    },
    {
      "activations/layer0_attention_weight_max": 15.469367027282715,
      "activations/layer0_attention_weight_min": -13.691132545471191,
      "activations/layer10_attention_weight_max": 34.225868225097656,
      "activations/layer10_attention_weight_min": -34.39960861206055,
      "activations/layer11_attention_weight_max": 30.643218994140625,
      "activations/layer11_attention_weight_min": -33.3217658996582,
      "activations/layer12_attention_weight_max": 17.706857681274414,
      "activations/layer12_attention_weight_min": -23.48087501525879,
      "activations/layer13_attention_weight_max": 36.41009521484375,
      "activations/layer13_attention_weight_min": -28.535701751708984,
      "activations/layer14_attention_weight_max": 33.3388671875,
      "activations/layer14_attention_weight_min": -30.211183547973633,
      "activations/layer15_attention_weight_max": 33.39115524291992,
      "activations/layer15_attention_weight_min": -28.283870697021484,
      "activations/layer16_attention_weight_max": 32.052310943603516,
      "activations/layer16_attention_weight_min": -29.863853454589844,
      "activations/layer17_attention_weight_max": 50.104496002197266,
      "activations/layer17_attention_weight_min": -42.33829116821289,
      "activations/layer18_attention_weight_max": 43.92046356201172,
      "activations/layer18_attention_weight_min": -36.18030548095703,
      "activations/layer19_attention_weight_max": 24.2478084564209,
      "activations/layer19_attention_weight_min": -21.188779830932617,
      "activations/layer1_attention_weight_max": 16.711389541625977,
      "activations/layer1_attention_weight_min": -14.023472785949707,
      "activations/layer20_attention_weight_max": 19.638042449951172,
      "activations/layer20_attention_weight_min": -21.391267776489258,
      "activations/layer21_attention_weight_max": 36.26914978027344,
      "activations/layer21_attention_weight_min": -20.34389305114746,
      "activations/layer22_attention_weight_max": 31.80359649658203,
      "activations/layer22_attention_weight_min": -26.399580001831055,
      "activations/layer23_attention_weight_max": 37.3917236328125,
      "activations/layer23_attention_weight_min": -24.16341781616211,
      "activations/layer2_attention_weight_max": 33.69315719604492,
      "activations/layer2_attention_weight_min": -30.52853775024414,
      "activations/layer3_attention_weight_max": 93.27420806884766,
      "activations/layer3_attention_weight_min": -95.7203140258789,
      "activations/layer4_attention_weight_max": 95.11427307128906,
      "activations/layer4_attention_weight_min": -92.04255676269531,
      "activations/layer5_attention_weight_max": 70.13895416259766,
      "activations/layer5_attention_weight_min": -77.75102233886719,
      "activations/layer6_attention_weight_max": 50.741275787353516,
      "activations/layer6_attention_weight_min": -50.2552375793457,
      "activations/layer7_attention_weight_max": 67.30540466308594,
      "activations/layer7_attention_weight_min": -64.4937744140625,
      "activations/layer8_attention_weight_max": 43.6283073425293,
      "activations/layer8_attention_weight_min": -45.42131042480469,
      "activations/layer9_attention_weight_max": 42.22390365600586,
      "activations/layer9_attention_weight_min": -46.80561065673828,
      "epoch": 13.5,
      "learning_rate": 6.35965909090909e-05,
      "loss": 2.7557,
      "step": 232300
    },
    {
      "activations/layer0_attention_weight_max": 14.882761001586914,
      "activations/layer0_attention_weight_min": -13.761007308959961,
      "activations/layer10_attention_weight_max": 34.9725456237793,
      "activations/layer10_attention_weight_min": -35.34478759765625,
      "activations/layer11_attention_weight_max": 32.18678665161133,
      "activations/layer11_attention_weight_min": -32.18413543701172,
      "activations/layer12_attention_weight_max": 18.37590789794922,
      "activations/layer12_attention_weight_min": -27.296890258789062,
      "activations/layer13_attention_weight_max": 37.022151947021484,
      "activations/layer13_attention_weight_min": -28.708656311035156,
      "activations/layer14_attention_weight_max": 38.1270637512207,
      "activations/layer14_attention_weight_min": -31.82696533203125,
      "activations/layer15_attention_weight_max": 31.92194366455078,
      "activations/layer15_attention_weight_min": -29.040599822998047,
      "activations/layer16_attention_weight_max": 33.06798553466797,
      "activations/layer16_attention_weight_min": -31.558223724365234,
      "activations/layer17_attention_weight_max": 54.663002014160156,
      "activations/layer17_attention_weight_min": -46.443721771240234,
      "activations/layer18_attention_weight_max": 46.243080139160156,
      "activations/layer18_attention_weight_min": -40.15603256225586,
      "activations/layer19_attention_weight_max": 23.528820037841797,
      "activations/layer19_attention_weight_min": -24.16979217529297,
      "activations/layer1_attention_weight_max": 16.980987548828125,
      "activations/layer1_attention_weight_min": -16.486356735229492,
      "activations/layer20_attention_weight_max": 25.901166915893555,
      "activations/layer20_attention_weight_min": -21.789579391479492,
      "activations/layer21_attention_weight_max": 47.43488693237305,
      "activations/layer21_attention_weight_min": -22.243785858154297,
      "activations/layer22_attention_weight_max": 34.252708435058594,
      "activations/layer22_attention_weight_min": -25.478193283081055,
      "activations/layer23_attention_weight_max": 38.380828857421875,
      "activations/layer23_attention_weight_min": -24.910995483398438,
      "activations/layer2_attention_weight_max": 32.475555419921875,
      "activations/layer2_attention_weight_min": -32.31203079223633,
      "activations/layer3_attention_weight_max": 97.6975326538086,
      "activations/layer3_attention_weight_min": -97.9238510131836,
      "activations/layer4_attention_weight_max": 94.56035614013672,
      "activations/layer4_attention_weight_min": -92.15074157714844,
      "activations/layer5_attention_weight_max": 72.39974212646484,
      "activations/layer5_attention_weight_min": -79.21270751953125,
      "activations/layer6_attention_weight_max": 52.09897232055664,
      "activations/layer6_attention_weight_min": -49.847999572753906,
      "activations/layer7_attention_weight_max": 69.14460754394531,
      "activations/layer7_attention_weight_min": -63.108585357666016,
      "activations/layer8_attention_weight_max": 46.54540252685547,
      "activations/layer8_attention_weight_min": -47.6821174621582,
      "activations/layer9_attention_weight_max": 45.35762405395508,
      "activations/layer9_attention_weight_min": -47.43751525878906,
      "epoch": 13.5,
      "learning_rate": 6.357765151515151e-05,
      "loss": 2.7618,
      "step": 232350
    },
    {
      "activations/layer0_attention_weight_max": 15.92634391784668,
      "activations/layer0_attention_weight_min": -14.445019721984863,
      "activations/layer10_attention_weight_max": 36.053558349609375,
      "activations/layer10_attention_weight_min": -34.524105072021484,
      "activations/layer11_attention_weight_max": 31.61761474609375,
      "activations/layer11_attention_weight_min": -33.037933349609375,
      "activations/layer12_attention_weight_max": 18.720932006835938,
      "activations/layer12_attention_weight_min": -25.721017837524414,
      "activations/layer13_attention_weight_max": 36.70067596435547,
      "activations/layer13_attention_weight_min": -31.43326759338379,
      "activations/layer14_attention_weight_max": 36.67777633666992,
      "activations/layer14_attention_weight_min": -30.21419334411621,
      "activations/layer15_attention_weight_max": 32.333412170410156,
      "activations/layer15_attention_weight_min": -30.41576385498047,
      "activations/layer16_attention_weight_max": 33.606807708740234,
      "activations/layer16_attention_weight_min": -30.76932716369629,
      "activations/layer17_attention_weight_max": 54.439971923828125,
      "activations/layer17_attention_weight_min": -44.252628326416016,
      "activations/layer18_attention_weight_max": 45.14725875854492,
      "activations/layer18_attention_weight_min": -37.83085632324219,
      "activations/layer19_attention_weight_max": 22.460115432739258,
      "activations/layer19_attention_weight_min": -22.018890380859375,
      "activations/layer1_attention_weight_max": 17.285245895385742,
      "activations/layer1_attention_weight_min": -15.852842330932617,
      "activations/layer20_attention_weight_max": 23.94200897216797,
      "activations/layer20_attention_weight_min": -21.073833465576172,
      "activations/layer21_attention_weight_max": 34.11566162109375,
      "activations/layer21_attention_weight_min": -22.872650146484375,
      "activations/layer22_attention_weight_max": 30.31708335876465,
      "activations/layer22_attention_weight_min": -25.09742546081543,
      "activations/layer23_attention_weight_max": 37.933349609375,
      "activations/layer23_attention_weight_min": -24.201087951660156,
      "activations/layer2_attention_weight_max": 31.752410888671875,
      "activations/layer2_attention_weight_min": -30.03424644470215,
      "activations/layer3_attention_weight_max": 96.0809097290039,
      "activations/layer3_attention_weight_min": -94.52379608154297,
      "activations/layer4_attention_weight_max": 95.29278564453125,
      "activations/layer4_attention_weight_min": -93.32575225830078,
      "activations/layer5_attention_weight_max": 71.08729553222656,
      "activations/layer5_attention_weight_min": -79.43211364746094,
      "activations/layer6_attention_weight_max": 52.914676666259766,
      "activations/layer6_attention_weight_min": -51.28261947631836,
      "activations/layer7_attention_weight_max": 69.27910614013672,
      "activations/layer7_attention_weight_min": -64.28028106689453,
      "activations/layer8_attention_weight_max": 43.65869140625,
      "activations/layer8_attention_weight_min": -46.05485916137695,
      "activations/layer9_attention_weight_max": 43.80026626586914,
      "activations/layer9_attention_weight_min": -47.6180534362793,
      "epoch": 13.5,
      "learning_rate": 6.355871212121211e-05,
      "loss": 2.7637,
      "step": 232400
    },
    {
      "activations/layer0_attention_weight_max": 15.434564590454102,
      "activations/layer0_attention_weight_min": -14.329793930053711,
      "activations/layer10_attention_weight_max": 35.5225715637207,
      "activations/layer10_attention_weight_min": -36.43376159667969,
      "activations/layer11_attention_weight_max": 29.90846824645996,
      "activations/layer11_attention_weight_min": -32.003517150878906,
      "activations/layer12_attention_weight_max": 18.392122268676758,
      "activations/layer12_attention_weight_min": -24.804052352905273,
      "activations/layer13_attention_weight_max": 40.83351516723633,
      "activations/layer13_attention_weight_min": -30.478355407714844,
      "activations/layer14_attention_weight_max": 35.36561584472656,
      "activations/layer14_attention_weight_min": -28.972612380981445,
      "activations/layer15_attention_weight_max": 33.18121337890625,
      "activations/layer15_attention_weight_min": -29.086606979370117,
      "activations/layer16_attention_weight_max": 30.553010940551758,
      "activations/layer16_attention_weight_min": -29.0909366607666,
      "activations/layer17_attention_weight_max": 49.56386947631836,
      "activations/layer17_attention_weight_min": -42.32261657714844,
      "activations/layer18_attention_weight_max": 42.55494689941406,
      "activations/layer18_attention_weight_min": -37.80899429321289,
      "activations/layer19_attention_weight_max": 22.666332244873047,
      "activations/layer19_attention_weight_min": -22.658613204956055,
      "activations/layer1_attention_weight_max": 17.04617691040039,
      "activations/layer1_attention_weight_min": -13.854619979858398,
      "activations/layer20_attention_weight_max": 22.87211036682129,
      "activations/layer20_attention_weight_min": -23.401729583740234,
      "activations/layer21_attention_weight_max": 33.940460205078125,
      "activations/layer21_attention_weight_min": -22.769773483276367,
      "activations/layer22_attention_weight_max": 29.48282814025879,
      "activations/layer22_attention_weight_min": -25.79146957397461,
      "activations/layer23_attention_weight_max": 39.72972869873047,
      "activations/layer23_attention_weight_min": -25.31610679626465,
      "activations/layer2_attention_weight_max": 33.141448974609375,
      "activations/layer2_attention_weight_min": -31.238800048828125,
      "activations/layer3_attention_weight_max": 99.1593017578125,
      "activations/layer3_attention_weight_min": -98.72464752197266,
      "activations/layer4_attention_weight_max": 97.76558685302734,
      "activations/layer4_attention_weight_min": -94.6364974975586,
      "activations/layer5_attention_weight_max": 73.39115905761719,
      "activations/layer5_attention_weight_min": -80.06951904296875,
      "activations/layer6_attention_weight_max": 53.97947692871094,
      "activations/layer6_attention_weight_min": -52.07029724121094,
      "activations/layer7_attention_weight_max": 69.72843933105469,
      "activations/layer7_attention_weight_min": -67.71468353271484,
      "activations/layer8_attention_weight_max": 45.55729293823242,
      "activations/layer8_attention_weight_min": -48.29141616821289,
      "activations/layer9_attention_weight_max": 43.85056686401367,
      "activations/layer9_attention_weight_min": -49.188575744628906,
      "epoch": 13.51,
      "learning_rate": 6.353977272727272e-05,
      "loss": 2.7836,
      "step": 232450
    },
    {
      "activations/layer0_attention_weight_max": 15.341958999633789,
      "activations/layer0_attention_weight_min": -14.246053695678711,
      "activations/layer10_attention_weight_max": 39.977169036865234,
      "activations/layer10_attention_weight_min": -36.380428314208984,
      "activations/layer11_attention_weight_max": 34.74159622192383,
      "activations/layer11_attention_weight_min": -34.306121826171875,
      "activations/layer12_attention_weight_max": 20.170621871948242,
      "activations/layer12_attention_weight_min": -24.777231216430664,
      "activations/layer13_attention_weight_max": 40.94956588745117,
      "activations/layer13_attention_weight_min": -33.59954833984375,
      "activations/layer14_attention_weight_max": 39.02688980102539,
      "activations/layer14_attention_weight_min": -29.209136962890625,
      "activations/layer15_attention_weight_max": 32.50434494018555,
      "activations/layer15_attention_weight_min": -29.9346866607666,
      "activations/layer16_attention_weight_max": 32.52841567993164,
      "activations/layer16_attention_weight_min": -29.462997436523438,
      "activations/layer17_attention_weight_max": 53.214332580566406,
      "activations/layer17_attention_weight_min": -43.48773193359375,
      "activations/layer18_attention_weight_max": 50.7399787902832,
      "activations/layer18_attention_weight_min": -39.22683334350586,
      "activations/layer19_attention_weight_max": 28.846059799194336,
      "activations/layer19_attention_weight_min": -24.308837890625,
      "activations/layer1_attention_weight_max": 16.656457901000977,
      "activations/layer1_attention_weight_min": -15.882033348083496,
      "activations/layer20_attention_weight_max": 27.07244300842285,
      "activations/layer20_attention_weight_min": -21.188528060913086,
      "activations/layer21_attention_weight_max": 43.898536682128906,
      "activations/layer21_attention_weight_min": -27.683841705322266,
      "activations/layer22_attention_weight_max": 35.91239547729492,
      "activations/layer22_attention_weight_min": -27.035846710205078,
      "activations/layer23_attention_weight_max": 39.63209533691406,
      "activations/layer23_attention_weight_min": -25.227123260498047,
      "activations/layer2_attention_weight_max": 31.743804931640625,
      "activations/layer2_attention_weight_min": -31.771167755126953,
      "activations/layer3_attention_weight_max": 94.44839477539062,
      "activations/layer3_attention_weight_min": -100.95390319824219,
      "activations/layer4_attention_weight_max": 95.68891143798828,
      "activations/layer4_attention_weight_min": -98.83888244628906,
      "activations/layer5_attention_weight_max": 72.45926666259766,
      "activations/layer5_attention_weight_min": -77.09075927734375,
      "activations/layer6_attention_weight_max": 49.49348449707031,
      "activations/layer6_attention_weight_min": -49.66440200805664,
      "activations/layer7_attention_weight_max": 68.40945434570312,
      "activations/layer7_attention_weight_min": -66.03508758544922,
      "activations/layer8_attention_weight_max": 45.4186897277832,
      "activations/layer8_attention_weight_min": -48.42843246459961,
      "activations/layer9_attention_weight_max": 46.27644348144531,
      "activations/layer9_attention_weight_min": -51.445098876953125,
      "epoch": 13.51,
      "learning_rate": 6.352083333333333e-05,
      "loss": 2.7594,
      "step": 232500
    },
    {
      "activations/layer0_attention_weight_max": 15.547718048095703,
      "activations/layer0_attention_weight_min": -14.360488891601562,
      "activations/layer10_attention_weight_max": 35.594329833984375,
      "activations/layer10_attention_weight_min": -35.86246109008789,
      "activations/layer11_attention_weight_max": 32.46868133544922,
      "activations/layer11_attention_weight_min": -33.263893127441406,
      "activations/layer12_attention_weight_max": 18.677349090576172,
      "activations/layer12_attention_weight_min": -30.559829711914062,
      "activations/layer13_attention_weight_max": 35.670047760009766,
      "activations/layer13_attention_weight_min": -30.03916358947754,
      "activations/layer14_attention_weight_max": 37.30327224731445,
      "activations/layer14_attention_weight_min": -29.2490234375,
      "activations/layer15_attention_weight_max": 32.10382080078125,
      "activations/layer15_attention_weight_min": -28.682157516479492,
      "activations/layer16_attention_weight_max": 30.00701904296875,
      "activations/layer16_attention_weight_min": -29.46224021911621,
      "activations/layer17_attention_weight_max": 48.85746765136719,
      "activations/layer17_attention_weight_min": -43.25876998901367,
      "activations/layer18_attention_weight_max": 44.098575592041016,
      "activations/layer18_attention_weight_min": -40.639793395996094,
      "activations/layer19_attention_weight_max": 22.28212547302246,
      "activations/layer19_attention_weight_min": -22.988277435302734,
      "activations/layer1_attention_weight_max": 16.112735748291016,
      "activations/layer1_attention_weight_min": -14.560726165771484,
      "activations/layer20_attention_weight_max": 21.16675567626953,
      "activations/layer20_attention_weight_min": -20.669010162353516,
      "activations/layer21_attention_weight_max": 34.69089889526367,
      "activations/layer21_attention_weight_min": -22.207138061523438,
      "activations/layer22_attention_weight_max": 28.183074951171875,
      "activations/layer22_attention_weight_min": -24.815488815307617,
      "activations/layer23_attention_weight_max": 35.12873840332031,
      "activations/layer23_attention_weight_min": -23.66412925720215,
      "activations/layer2_attention_weight_max": 32.54571533203125,
      "activations/layer2_attention_weight_min": -32.10803985595703,
      "activations/layer3_attention_weight_max": 96.2022705078125,
      "activations/layer3_attention_weight_min": -104.08457946777344,
      "activations/layer4_attention_weight_max": 95.09339904785156,
      "activations/layer4_attention_weight_min": -92.04450225830078,
      "activations/layer5_attention_weight_max": 69.48084259033203,
      "activations/layer5_attention_weight_min": -80.30299377441406,
      "activations/layer6_attention_weight_max": 50.93689727783203,
      "activations/layer6_attention_weight_min": -52.07059860229492,
      "activations/layer7_attention_weight_max": 65.99858856201172,
      "activations/layer7_attention_weight_min": -67.73578643798828,
      "activations/layer8_attention_weight_max": 42.373634338378906,
      "activations/layer8_attention_weight_min": -47.41679763793945,
      "activations/layer9_attention_weight_max": 45.57188034057617,
      "activations/layer9_attention_weight_min": -51.934852600097656,
      "epoch": 13.51,
      "learning_rate": 6.350189393939393e-05,
      "loss": 2.7736,
      "step": 232550
    },
    {
      "activations/layer0_attention_weight_max": 16.37469482421875,
      "activations/layer0_attention_weight_min": -14.10622787475586,
      "activations/layer10_attention_weight_max": 34.37060546875,
      "activations/layer10_attention_weight_min": -35.355079650878906,
      "activations/layer11_attention_weight_max": 31.72602081298828,
      "activations/layer11_attention_weight_min": -32.66162872314453,
      "activations/layer12_attention_weight_max": 18.288293838500977,
      "activations/layer12_attention_weight_min": -25.736902236938477,
      "activations/layer13_attention_weight_max": 36.053829193115234,
      "activations/layer13_attention_weight_min": -27.387102127075195,
      "activations/layer14_attention_weight_max": 35.376686096191406,
      "activations/layer14_attention_weight_min": -32.70119094848633,
      "activations/layer15_attention_weight_max": 33.34614181518555,
      "activations/layer15_attention_weight_min": -30.84986686706543,
      "activations/layer16_attention_weight_max": 30.999149322509766,
      "activations/layer16_attention_weight_min": -30.79283332824707,
      "activations/layer17_attention_weight_max": 52.995086669921875,
      "activations/layer17_attention_weight_min": -45.44040298461914,
      "activations/layer18_attention_weight_max": 45.12787628173828,
      "activations/layer18_attention_weight_min": -36.828285217285156,
      "activations/layer19_attention_weight_max": 21.079252243041992,
      "activations/layer19_attention_weight_min": -23.676315307617188,
      "activations/layer1_attention_weight_max": 17.65826988220215,
      "activations/layer1_attention_weight_min": -14.874282836914062,
      "activations/layer20_attention_weight_max": 21.69992446899414,
      "activations/layer20_attention_weight_min": -21.079925537109375,
      "activations/layer21_attention_weight_max": 34.91435623168945,
      "activations/layer21_attention_weight_min": -21.332468032836914,
      "activations/layer22_attention_weight_max": 28.396516799926758,
      "activations/layer22_attention_weight_min": -27.802745819091797,
      "activations/layer23_attention_weight_max": 35.75779342651367,
      "activations/layer23_attention_weight_min": -24.042434692382812,
      "activations/layer2_attention_weight_max": 31.248531341552734,
      "activations/layer2_attention_weight_min": -30.912893295288086,
      "activations/layer3_attention_weight_max": 94.83772277832031,
      "activations/layer3_attention_weight_min": -102.552734375,
      "activations/layer4_attention_weight_max": 94.66597747802734,
      "activations/layer4_attention_weight_min": -93.59330749511719,
      "activations/layer5_attention_weight_max": 76.0596694946289,
      "activations/layer5_attention_weight_min": -76.52428436279297,
      "activations/layer6_attention_weight_max": 50.84998321533203,
      "activations/layer6_attention_weight_min": -52.40053176879883,
      "activations/layer7_attention_weight_max": 68.8563232421875,
      "activations/layer7_attention_weight_min": -64.45018005371094,
      "activations/layer8_attention_weight_max": 45.0051383972168,
      "activations/layer8_attention_weight_min": -49.844905853271484,
      "activations/layer9_attention_weight_max": 43.66813659667969,
      "activations/layer9_attention_weight_min": -49.017616271972656,
      "epoch": 13.52,
      "learning_rate": 6.348295454545453e-05,
      "loss": 2.7721,
      "step": 232600
    },
    {
      "activations/layer0_attention_weight_max": 17.670215606689453,
      "activations/layer0_attention_weight_min": -14.719123840332031,
      "activations/layer10_attention_weight_max": 37.25813674926758,
      "activations/layer10_attention_weight_min": -34.673072814941406,
      "activations/layer11_attention_weight_max": 33.73511505126953,
      "activations/layer11_attention_weight_min": -31.995574951171875,
      "activations/layer12_attention_weight_max": 18.29729652404785,
      "activations/layer12_attention_weight_min": -27.91245460510254,
      "activations/layer13_attention_weight_max": 34.51275634765625,
      "activations/layer13_attention_weight_min": -27.45294189453125,
      "activations/layer14_attention_weight_max": 33.48653793334961,
      "activations/layer14_attention_weight_min": -30.039505004882812,
      "activations/layer15_attention_weight_max": 31.690935134887695,
      "activations/layer15_attention_weight_min": -29.459745407104492,
      "activations/layer16_attention_weight_max": 31.00032615661621,
      "activations/layer16_attention_weight_min": -29.352678298950195,
      "activations/layer17_attention_weight_max": 49.72364807128906,
      "activations/layer17_attention_weight_min": -42.268943786621094,
      "activations/layer18_attention_weight_max": 45.72962188720703,
      "activations/layer18_attention_weight_min": -36.12127685546875,
      "activations/layer19_attention_weight_max": 21.194379806518555,
      "activations/layer19_attention_weight_min": -20.642501831054688,
      "activations/layer1_attention_weight_max": 17.672903060913086,
      "activations/layer1_attention_weight_min": -15.509079933166504,
      "activations/layer20_attention_weight_max": 21.006258010864258,
      "activations/layer20_attention_weight_min": -18.17487335205078,
      "activations/layer21_attention_weight_max": 29.805822372436523,
      "activations/layer21_attention_weight_min": -19.49051284790039,
      "activations/layer22_attention_weight_max": 28.59148406982422,
      "activations/layer22_attention_weight_min": -23.98749351501465,
      "activations/layer23_attention_weight_max": 36.62523651123047,
      "activations/layer23_attention_weight_min": -24.85190200805664,
      "activations/layer2_attention_weight_max": 31.258853912353516,
      "activations/layer2_attention_weight_min": -31.215566635131836,
      "activations/layer3_attention_weight_max": 92.58567810058594,
      "activations/layer3_attention_weight_min": -92.07262420654297,
      "activations/layer4_attention_weight_max": 93.81644439697266,
      "activations/layer4_attention_weight_min": -90.0988540649414,
      "activations/layer5_attention_weight_max": 73.17070007324219,
      "activations/layer5_attention_weight_min": -78.71507263183594,
      "activations/layer6_attention_weight_max": 51.10603713989258,
      "activations/layer6_attention_weight_min": -51.26061248779297,
      "activations/layer7_attention_weight_max": 73.53186798095703,
      "activations/layer7_attention_weight_min": -63.44842529296875,
      "activations/layer8_attention_weight_max": 47.919498443603516,
      "activations/layer8_attention_weight_min": -46.944908142089844,
      "activations/layer9_attention_weight_max": 59.927101135253906,
      "activations/layer9_attention_weight_min": -46.59568405151367,
      "epoch": 13.52,
      "learning_rate": 6.346401515151515e-05,
      "loss": 2.753,
      "step": 232650
    },
    {
      "activations/layer0_attention_weight_max": 15.625384330749512,
      "activations/layer0_attention_weight_min": -14.442367553710938,
      "activations/layer10_attention_weight_max": 36.653568267822266,
      "activations/layer10_attention_weight_min": -33.674598693847656,
      "activations/layer11_attention_weight_max": 30.82349967956543,
      "activations/layer11_attention_weight_min": -31.843524932861328,
      "activations/layer12_attention_weight_max": 17.425762176513672,
      "activations/layer12_attention_weight_min": -25.90115737915039,
      "activations/layer13_attention_weight_max": 40.75175094604492,
      "activations/layer13_attention_weight_min": -28.105350494384766,
      "activations/layer14_attention_weight_max": 34.94872283935547,
      "activations/layer14_attention_weight_min": -30.86164093017578,
      "activations/layer15_attention_weight_max": 32.40056228637695,
      "activations/layer15_attention_weight_min": -30.31599235534668,
      "activations/layer16_attention_weight_max": 32.91753005981445,
      "activations/layer16_attention_weight_min": -29.12962532043457,
      "activations/layer17_attention_weight_max": 52.61726379394531,
      "activations/layer17_attention_weight_min": -47.35899353027344,
      "activations/layer18_attention_weight_max": 50.18492126464844,
      "activations/layer18_attention_weight_min": -40.76713943481445,
      "activations/layer19_attention_weight_max": 22.71419906616211,
      "activations/layer19_attention_weight_min": -23.929521560668945,
      "activations/layer1_attention_weight_max": 16.770601272583008,
      "activations/layer1_attention_weight_min": -15.45026683807373,
      "activations/layer20_attention_weight_max": 23.590322494506836,
      "activations/layer20_attention_weight_min": -24.554338455200195,
      "activations/layer21_attention_weight_max": 39.15711212158203,
      "activations/layer21_attention_weight_min": -26.22869110107422,
      "activations/layer22_attention_weight_max": 31.076292037963867,
      "activations/layer22_attention_weight_min": -25.190250396728516,
      "activations/layer23_attention_weight_max": 36.49211883544922,
      "activations/layer23_attention_weight_min": -27.580299377441406,
      "activations/layer2_attention_weight_max": 32.24363708496094,
      "activations/layer2_attention_weight_min": -30.140724182128906,
      "activations/layer3_attention_weight_max": 94.83439636230469,
      "activations/layer3_attention_weight_min": -97.302001953125,
      "activations/layer4_attention_weight_max": 92.77064514160156,
      "activations/layer4_attention_weight_min": -92.7894058227539,
      "activations/layer5_attention_weight_max": 68.43988037109375,
      "activations/layer5_attention_weight_min": -78.76398468017578,
      "activations/layer6_attention_weight_max": 54.294593811035156,
      "activations/layer6_attention_weight_min": -52.48857498168945,
      "activations/layer7_attention_weight_max": 67.17643737792969,
      "activations/layer7_attention_weight_min": -65.55901336669922,
      "activations/layer8_attention_weight_max": 48.72325134277344,
      "activations/layer8_attention_weight_min": -45.21808624267578,
      "activations/layer9_attention_weight_max": 43.45635986328125,
      "activations/layer9_attention_weight_min": -46.15148162841797,
      "epoch": 13.52,
      "learning_rate": 6.344507575757575e-05,
      "loss": 2.7752,
      "step": 232700
    },
    {
      "activations/layer0_attention_weight_max": 15.308675765991211,
      "activations/layer0_attention_weight_min": -14.443350791931152,
      "activations/layer10_attention_weight_max": 37.886192321777344,
      "activations/layer10_attention_weight_min": -37.14643478393555,
      "activations/layer11_attention_weight_max": 33.65350341796875,
      "activations/layer11_attention_weight_min": -33.84745788574219,
      "activations/layer12_attention_weight_max": 21.674392700195312,
      "activations/layer12_attention_weight_min": -25.456743240356445,
      "activations/layer13_attention_weight_max": 53.77236557006836,
      "activations/layer13_attention_weight_min": -31.33452033996582,
      "activations/layer14_attention_weight_max": 40.564205169677734,
      "activations/layer14_attention_weight_min": -32.008872985839844,
      "activations/layer15_attention_weight_max": 37.78948974609375,
      "activations/layer15_attention_weight_min": -33.85464859008789,
      "activations/layer16_attention_weight_max": 36.14114761352539,
      "activations/layer16_attention_weight_min": -32.083160400390625,
      "activations/layer17_attention_weight_max": 59.97441864013672,
      "activations/layer17_attention_weight_min": -47.89393997192383,
      "activations/layer18_attention_weight_max": 52.803955078125,
      "activations/layer18_attention_weight_min": -40.61613082885742,
      "activations/layer19_attention_weight_max": 24.56283187866211,
      "activations/layer19_attention_weight_min": -23.585142135620117,
      "activations/layer1_attention_weight_max": 17.609689712524414,
      "activations/layer1_attention_weight_min": -16.668872833251953,
      "activations/layer20_attention_weight_max": 23.71260643005371,
      "activations/layer20_attention_weight_min": -22.863279342651367,
      "activations/layer21_attention_weight_max": 39.090545654296875,
      "activations/layer21_attention_weight_min": -27.28445816040039,
      "activations/layer22_attention_weight_max": 28.802278518676758,
      "activations/layer22_attention_weight_min": -25.50564956665039,
      "activations/layer23_attention_weight_max": 36.7009162902832,
      "activations/layer23_attention_weight_min": -25.722976684570312,
      "activations/layer2_attention_weight_max": 31.525306701660156,
      "activations/layer2_attention_weight_min": -31.441307067871094,
      "activations/layer3_attention_weight_max": 95.87065124511719,
      "activations/layer3_attention_weight_min": -100.8131332397461,
      "activations/layer4_attention_weight_max": 97.55613708496094,
      "activations/layer4_attention_weight_min": -95.55361938476562,
      "activations/layer5_attention_weight_max": 70.6456527709961,
      "activations/layer5_attention_weight_min": -77.4000473022461,
      "activations/layer6_attention_weight_max": 51.419795989990234,
      "activations/layer6_attention_weight_min": -52.693721771240234,
      "activations/layer7_attention_weight_max": 66.96700286865234,
      "activations/layer7_attention_weight_min": -64.20049285888672,
      "activations/layer8_attention_weight_max": 46.00471496582031,
      "activations/layer8_attention_weight_min": -49.568199157714844,
      "activations/layer9_attention_weight_max": 46.43995666503906,
      "activations/layer9_attention_weight_min": -46.64835739135742,
      "epoch": 13.52,
      "learning_rate": 6.342613636363636e-05,
      "loss": 2.7675,
      "step": 232750
    },
    {
      "activations/layer0_attention_weight_max": 15.97184944152832,
      "activations/layer0_attention_weight_min": -14.126105308532715,
      "activations/layer10_attention_weight_max": 37.91972732543945,
      "activations/layer10_attention_weight_min": -36.35194778442383,
      "activations/layer11_attention_weight_max": 38.731712341308594,
      "activations/layer11_attention_weight_min": -33.387725830078125,
      "activations/layer12_attention_weight_max": 17.76708984375,
      "activations/layer12_attention_weight_min": -25.667051315307617,
      "activations/layer13_attention_weight_max": 36.519317626953125,
      "activations/layer13_attention_weight_min": -28.949533462524414,
      "activations/layer14_attention_weight_max": 35.18739318847656,
      "activations/layer14_attention_weight_min": -29.928329467773438,
      "activations/layer15_attention_weight_max": 33.211463928222656,
      "activations/layer15_attention_weight_min": -29.284038543701172,
      "activations/layer16_attention_weight_max": 33.13144302368164,
      "activations/layer16_attention_weight_min": -30.295108795166016,
      "activations/layer17_attention_weight_max": 52.379791259765625,
      "activations/layer17_attention_weight_min": -43.94873809814453,
      "activations/layer18_attention_weight_max": 46.68560028076172,
      "activations/layer18_attention_weight_min": -37.8574333190918,
      "activations/layer19_attention_weight_max": 23.244361877441406,
      "activations/layer19_attention_weight_min": -22.698867797851562,
      "activations/layer1_attention_weight_max": 16.6114559173584,
      "activations/layer1_attention_weight_min": -14.939316749572754,
      "activations/layer20_attention_weight_max": 21.70900535583496,
      "activations/layer20_attention_weight_min": -20.348695755004883,
      "activations/layer21_attention_weight_max": 35.87824249267578,
      "activations/layer21_attention_weight_min": -23.9110050201416,
      "activations/layer22_attention_weight_max": 30.95113182067871,
      "activations/layer22_attention_weight_min": -27.1149959564209,
      "activations/layer23_attention_weight_max": 37.661109924316406,
      "activations/layer23_attention_weight_min": -24.279598236083984,
      "activations/layer2_attention_weight_max": 31.713584899902344,
      "activations/layer2_attention_weight_min": -30.45714569091797,
      "activations/layer3_attention_weight_max": 98.17942810058594,
      "activations/layer3_attention_weight_min": -96.27526092529297,
      "activations/layer4_attention_weight_max": 97.84736633300781,
      "activations/layer4_attention_weight_min": -95.62455749511719,
      "activations/layer5_attention_weight_max": 69.99124145507812,
      "activations/layer5_attention_weight_min": -78.10505676269531,
      "activations/layer6_attention_weight_max": 49.913368225097656,
      "activations/layer6_attention_weight_min": -50.00635528564453,
      "activations/layer7_attention_weight_max": 69.26705932617188,
      "activations/layer7_attention_weight_min": -62.39435958862305,
      "activations/layer8_attention_weight_max": 47.0257453918457,
      "activations/layer8_attention_weight_min": -44.86831283569336,
      "activations/layer9_attention_weight_max": 54.504695892333984,
      "activations/layer9_attention_weight_min": -44.1689567565918,
      "epoch": 13.53,
      "learning_rate": 6.340719696969697e-05,
      "loss": 2.7436,
      "step": 232800
    },
    {
      "activations/layer0_attention_weight_max": 16.807266235351562,
      "activations/layer0_attention_weight_min": -14.039517402648926,
      "activations/layer10_attention_weight_max": 32.9015998840332,
      "activations/layer10_attention_weight_min": -35.35319519042969,
      "activations/layer11_attention_weight_max": 28.974905014038086,
      "activations/layer11_attention_weight_min": -33.86890411376953,
      "activations/layer12_attention_weight_max": 17.446584701538086,
      "activations/layer12_attention_weight_min": -25.501523971557617,
      "activations/layer13_attention_weight_max": 29.81000518798828,
      "activations/layer13_attention_weight_min": -27.15293312072754,
      "activations/layer14_attention_weight_max": 32.02249526977539,
      "activations/layer14_attention_weight_min": -29.509798049926758,
      "activations/layer15_attention_weight_max": 32.96441650390625,
      "activations/layer15_attention_weight_min": -29.526681900024414,
      "activations/layer16_attention_weight_max": 30.208051681518555,
      "activations/layer16_attention_weight_min": -30.278629302978516,
      "activations/layer17_attention_weight_max": 51.27778625488281,
      "activations/layer17_attention_weight_min": -45.502620697021484,
      "activations/layer18_attention_weight_max": 43.74325942993164,
      "activations/layer18_attention_weight_min": -36.57080078125,
      "activations/layer19_attention_weight_max": 20.025413513183594,
      "activations/layer19_attention_weight_min": -22.178512573242188,
      "activations/layer1_attention_weight_max": 16.61457633972168,
      "activations/layer1_attention_weight_min": -14.072696685791016,
      "activations/layer20_attention_weight_max": 20.787155151367188,
      "activations/layer20_attention_weight_min": -24.14890480041504,
      "activations/layer21_attention_weight_max": 37.02973175048828,
      "activations/layer21_attention_weight_min": -26.358530044555664,
      "activations/layer22_attention_weight_max": 29.768552780151367,
      "activations/layer22_attention_weight_min": -27.184005737304688,
      "activations/layer23_attention_weight_max": 34.512962341308594,
      "activations/layer23_attention_weight_min": -23.15816307067871,
      "activations/layer2_attention_weight_max": 33.80046463012695,
      "activations/layer2_attention_weight_min": -31.497760772705078,
      "activations/layer3_attention_weight_max": 96.4430160522461,
      "activations/layer3_attention_weight_min": -94.31494903564453,
      "activations/layer4_attention_weight_max": 94.73448944091797,
      "activations/layer4_attention_weight_min": -92.89723205566406,
      "activations/layer5_attention_weight_max": 68.72189331054688,
      "activations/layer5_attention_weight_min": -78.13978576660156,
      "activations/layer6_attention_weight_max": 51.082088470458984,
      "activations/layer6_attention_weight_min": -50.67116928100586,
      "activations/layer7_attention_weight_max": 66.5022964477539,
      "activations/layer7_attention_weight_min": -63.63376235961914,
      "activations/layer8_attention_weight_max": 40.892635345458984,
      "activations/layer8_attention_weight_min": -46.68354415893555,
      "activations/layer9_attention_weight_max": 44.52472686767578,
      "activations/layer9_attention_weight_min": -46.72773742675781,
      "epoch": 13.53,
      "learning_rate": 6.338825757575757e-05,
      "loss": 2.7702,
      "step": 232850
    },
    {
      "activations/layer0_attention_weight_max": 17.395458221435547,
      "activations/layer0_attention_weight_min": -14.180425643920898,
      "activations/layer10_attention_weight_max": 37.57688903808594,
      "activations/layer10_attention_weight_min": -37.20436477661133,
      "activations/layer11_attention_weight_max": 33.086448669433594,
      "activations/layer11_attention_weight_min": -33.56346893310547,
      "activations/layer12_attention_weight_max": 16.36292839050293,
      "activations/layer12_attention_weight_min": -23.515050888061523,
      "activations/layer13_attention_weight_max": 37.08366394042969,
      "activations/layer13_attention_weight_min": -34.5948486328125,
      "activations/layer14_attention_weight_max": 31.749740600585938,
      "activations/layer14_attention_weight_min": -28.63612174987793,
      "activations/layer15_attention_weight_max": 30.02782440185547,
      "activations/layer15_attention_weight_min": -27.608388900756836,
      "activations/layer16_attention_weight_max": 28.239683151245117,
      "activations/layer16_attention_weight_min": -27.578502655029297,
      "activations/layer17_attention_weight_max": 48.382625579833984,
      "activations/layer17_attention_weight_min": -40.082828521728516,
      "activations/layer18_attention_weight_max": 40.99872970581055,
      "activations/layer18_attention_weight_min": -34.70378494262695,
      "activations/layer19_attention_weight_max": 19.69233512878418,
      "activations/layer19_attention_weight_min": -20.671417236328125,
      "activations/layer1_attention_weight_max": 16.906293869018555,
      "activations/layer1_attention_weight_min": -15.579675674438477,
      "activations/layer20_attention_weight_max": 20.086938858032227,
      "activations/layer20_attention_weight_min": -22.32394027709961,
      "activations/layer21_attention_weight_max": 31.7464599609375,
      "activations/layer21_attention_weight_min": -20.861194610595703,
      "activations/layer22_attention_weight_max": 26.652517318725586,
      "activations/layer22_attention_weight_min": -25.471357345581055,
      "activations/layer23_attention_weight_max": 34.5402717590332,
      "activations/layer23_attention_weight_min": -24.369842529296875,
      "activations/layer2_attention_weight_max": 32.387718200683594,
      "activations/layer2_attention_weight_min": -32.324947357177734,
      "activations/layer3_attention_weight_max": 96.2200698852539,
      "activations/layer3_attention_weight_min": -99.00885009765625,
      "activations/layer4_attention_weight_max": 94.92765045166016,
      "activations/layer4_attention_weight_min": -91.96337127685547,
      "activations/layer5_attention_weight_max": 71.04949188232422,
      "activations/layer5_attention_weight_min": -76.71849060058594,
      "activations/layer6_attention_weight_max": 51.157657623291016,
      "activations/layer6_attention_weight_min": -52.41948318481445,
      "activations/layer7_attention_weight_max": 68.54336547851562,
      "activations/layer7_attention_weight_min": -66.41849517822266,
      "activations/layer8_attention_weight_max": 45.4630126953125,
      "activations/layer8_attention_weight_min": -47.1046142578125,
      "activations/layer9_attention_weight_max": 44.989715576171875,
      "activations/layer9_attention_weight_min": -45.756431579589844,
      "epoch": 13.53,
      "learning_rate": 6.336931818181818e-05,
      "loss": 2.7648,
      "step": 232900
    },
    {
      "activations/layer0_attention_weight_max": 16.601428985595703,
      "activations/layer0_attention_weight_min": -14.276586532592773,
      "activations/layer10_attention_weight_max": 37.462162017822266,
      "activations/layer10_attention_weight_min": -35.13287353515625,
      "activations/layer11_attention_weight_max": 31.62621307373047,
      "activations/layer11_attention_weight_min": -34.960853576660156,
      "activations/layer12_attention_weight_max": 19.178028106689453,
      "activations/layer12_attention_weight_min": -25.426502227783203,
      "activations/layer13_attention_weight_max": 37.989105224609375,
      "activations/layer13_attention_weight_min": -27.594839096069336,
      "activations/layer14_attention_weight_max": 40.125892639160156,
      "activations/layer14_attention_weight_min": -30.847698211669922,
      "activations/layer15_attention_weight_max": 35.06199645996094,
      "activations/layer15_attention_weight_min": -30.49662971496582,
      "activations/layer16_attention_weight_max": 34.315677642822266,
      "activations/layer16_attention_weight_min": -30.94184112548828,
      "activations/layer17_attention_weight_max": 58.73800277709961,
      "activations/layer17_attention_weight_min": -46.581398010253906,
      "activations/layer18_attention_weight_max": 51.5057487487793,
      "activations/layer18_attention_weight_min": -41.51712417602539,
      "activations/layer19_attention_weight_max": 27.088607788085938,
      "activations/layer19_attention_weight_min": -24.507041931152344,
      "activations/layer1_attention_weight_max": 16.31140899658203,
      "activations/layer1_attention_weight_min": -14.938563346862793,
      "activations/layer20_attention_weight_max": 26.627573013305664,
      "activations/layer20_attention_weight_min": -25.097450256347656,
      "activations/layer21_attention_weight_max": 45.60036087036133,
      "activations/layer21_attention_weight_min": -24.452194213867188,
      "activations/layer22_attention_weight_max": 33.74617385864258,
      "activations/layer22_attention_weight_min": -26.15720558166504,
      "activations/layer23_attention_weight_max": 45.8093376159668,
      "activations/layer23_attention_weight_min": -23.44003677368164,
      "activations/layer2_attention_weight_max": 34.41651153564453,
      "activations/layer2_attention_weight_min": -32.79878234863281,
      "activations/layer3_attention_weight_max": 99.2993392944336,
      "activations/layer3_attention_weight_min": -107.23372650146484,
      "activations/layer4_attention_weight_max": 98.1666259765625,
      "activations/layer4_attention_weight_min": -92.93302154541016,
      "activations/layer5_attention_weight_max": 73.62356567382812,
      "activations/layer5_attention_weight_min": -83.88101196289062,
      "activations/layer6_attention_weight_max": 52.56952667236328,
      "activations/layer6_attention_weight_min": -53.456966400146484,
      "activations/layer7_attention_weight_max": 68.16571807861328,
      "activations/layer7_attention_weight_min": -65.42323303222656,
      "activations/layer8_attention_weight_max": 47.222686767578125,
      "activations/layer8_attention_weight_min": -47.74850082397461,
      "activations/layer9_attention_weight_max": 47.95698547363281,
      "activations/layer9_attention_weight_min": -47.23228073120117,
      "epoch": 13.54,
      "learning_rate": 6.335037878787878e-05,
      "loss": 2.7576,
      "step": 232950
    },
    {
      "activations/layer0_attention_weight_max": 16.322978973388672,
      "activations/layer0_attention_weight_min": -13.91765308380127,
      "activations/layer10_attention_weight_max": 38.93572235107422,
      "activations/layer10_attention_weight_min": -34.450931549072266,
      "activations/layer11_attention_weight_max": 31.35574722290039,
      "activations/layer11_attention_weight_min": -34.179962158203125,
      "activations/layer12_attention_weight_max": 19.075809478759766,
      "activations/layer12_attention_weight_min": -29.809978485107422,
      "activations/layer13_attention_weight_max": 37.15491485595703,
      "activations/layer13_attention_weight_min": -28.697708129882812,
      "activations/layer14_attention_weight_max": 36.403987884521484,
      "activations/layer14_attention_weight_min": -28.244625091552734,
      "activations/layer15_attention_weight_max": 33.982295989990234,
      "activations/layer15_attention_weight_min": -29.53042221069336,
      "activations/layer16_attention_weight_max": 35.881404876708984,
      "activations/layer16_attention_weight_min": -31.276521682739258,
      "activations/layer17_attention_weight_max": 52.69908142089844,
      "activations/layer17_attention_weight_min": -43.604591369628906,
      "activations/layer18_attention_weight_max": 52.17108154296875,
      "activations/layer18_attention_weight_min": -38.11259841918945,
      "activations/layer19_attention_weight_max": 23.588699340820312,
      "activations/layer19_attention_weight_min": -22.51829719543457,
      "activations/layer1_attention_weight_max": 16.6602783203125,
      "activations/layer1_attention_weight_min": -15.642870903015137,
      "activations/layer20_attention_weight_max": 23.992353439331055,
      "activations/layer20_attention_weight_min": -22.083818435668945,
      "activations/layer21_attention_weight_max": 39.11916732788086,
      "activations/layer21_attention_weight_min": -23.04886245727539,
      "activations/layer22_attention_weight_max": 30.92401695251465,
      "activations/layer22_attention_weight_min": -24.309295654296875,
      "activations/layer23_attention_weight_max": 43.805389404296875,
      "activations/layer23_attention_weight_min": -23.175350189208984,
      "activations/layer2_attention_weight_max": 31.55788230895996,
      "activations/layer2_attention_weight_min": -31.48894500732422,
      "activations/layer3_attention_weight_max": 90.88352966308594,
      "activations/layer3_attention_weight_min": -92.22945404052734,
      "activations/layer4_attention_weight_max": 89.033935546875,
      "activations/layer4_attention_weight_min": -89.71195220947266,
      "activations/layer5_attention_weight_max": 72.89532470703125,
      "activations/layer5_attention_weight_min": -74.30644989013672,
      "activations/layer6_attention_weight_max": 47.948524475097656,
      "activations/layer6_attention_weight_min": -50.13780212402344,
      "activations/layer7_attention_weight_max": 66.17562103271484,
      "activations/layer7_attention_weight_min": -63.40922927856445,
      "activations/layer8_attention_weight_max": 43.06233596801758,
      "activations/layer8_attention_weight_min": -44.75312423706055,
      "activations/layer9_attention_weight_max": 44.82950973510742,
      "activations/layer9_attention_weight_min": -44.328399658203125,
      "epoch": 13.54,
      "learning_rate": 6.333143939393939e-05,
      "loss": 2.7538,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_loss": 2.71875,
      "eval_runtime": 8.498,
      "eval_samples_per_second": 505.295,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.498,
      "eval_openwebtext_samples_per_second": 505.295,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_wikitext_loss": 2.9609375,
      "eval_wikitext_ppl": 19.31607208722833,
      "eval_wikitext_runtime": 1.9623,
      "eval_wikitext_samples_per_second": 232.381,
      "step": 233000
    },
    {
      "epoch": 13.54,
      "eval_lambada_loss": 2.607421875,
      "eval_lambada_ppl": 13.564035954639001,
      "eval_lambada_runtime": 9.5511,
      "eval_lambada_samples_per_second": 509.784,
      "step": 233000
    },
    {
      "activations/layer0_attention_weight_max": 16.717134475708008,
      "activations/layer0_attention_weight_min": -13.708709716796875,
      "activations/layer10_attention_weight_max": 32.59694290161133,
      "activations/layer10_attention_weight_min": -35.528358459472656,
      "activations/layer11_attention_weight_max": 29.04793930053711,
      "activations/layer11_attention_weight_min": -31.9477596282959,
      "activations/layer12_attention_weight_max": 16.999357223510742,
      "activations/layer12_attention_weight_min": -23.08796501159668,
      "activations/layer13_attention_weight_max": 34.18775177001953,
      "activations/layer13_attention_weight_min": -27.91336441040039,
      "activations/layer14_attention_weight_max": 35.12588882446289,
      "activations/layer14_attention_weight_min": -27.885940551757812,
      "activations/layer15_attention_weight_max": 32.00432586669922,
      "activations/layer15_attention_weight_min": -27.743724822998047,
      "activations/layer16_attention_weight_max": 29.935440063476562,
      "activations/layer16_attention_weight_min": -30.233728408813477,
      "activations/layer17_attention_weight_max": 52.609920501708984,
      "activations/layer17_attention_weight_min": -43.62171173095703,
      "activations/layer18_attention_weight_max": 44.47938919067383,
      "activations/layer18_attention_weight_min": -36.713199615478516,
      "activations/layer19_attention_weight_max": 23.28792381286621,
      "activations/layer19_attention_weight_min": -21.851598739624023,
      "activations/layer1_attention_weight_max": 15.84692096710205,
      "activations/layer1_attention_weight_min": -16.542150497436523,
      "activations/layer20_attention_weight_max": 21.393651962280273,
      "activations/layer20_attention_weight_min": -21.361331939697266,
      "activations/layer21_attention_weight_max": 36.83898162841797,
      "activations/layer21_attention_weight_min": -21.298322677612305,
      "activations/layer22_attention_weight_max": 29.24513053894043,
      "activations/layer22_attention_weight_min": -26.383625030517578,
      "activations/layer23_attention_weight_max": 37.65509796142578,
      "activations/layer23_attention_weight_min": -23.067325592041016,
      "activations/layer2_attention_weight_max": 32.304508209228516,
      "activations/layer2_attention_weight_min": -30.972827911376953,
      "activations/layer3_attention_weight_max": 92.88711547851562,
      "activations/layer3_attention_weight_min": -95.97026824951172,
      "activations/layer4_attention_weight_max": 92.26151275634766,
      "activations/layer4_attention_weight_min": -89.64257049560547,
      "activations/layer5_attention_weight_max": 72.41593933105469,
      "activations/layer5_attention_weight_min": -78.05123901367188,
      "activations/layer6_attention_weight_max": 51.72279739379883,
      "activations/layer6_attention_weight_min": -48.924373626708984,
      "activations/layer7_attention_weight_max": 66.0890121459961,
      "activations/layer7_attention_weight_min": -61.46421432495117,
      "activations/layer8_attention_weight_max": 44.808773040771484,
      "activations/layer8_attention_weight_min": -44.79060363769531,
      "activations/layer9_attention_weight_max": 43.509456634521484,
      "activations/layer9_attention_weight_min": -44.92060470581055,
      "epoch": 13.54,
      "learning_rate": 6.331287878787878e-05,
      "loss": 2.7604,
      "step": 233050
    },
    {
      "activations/layer0_attention_weight_max": 15.831448554992676,
      "activations/layer0_attention_weight_min": -14.05394458770752,
      "activations/layer10_attention_weight_max": 38.551780700683594,
      "activations/layer10_attention_weight_min": -36.558292388916016,
      "activations/layer11_attention_weight_max": 35.843231201171875,
      "activations/layer11_attention_weight_min": -33.716453552246094,
      "activations/layer12_attention_weight_max": 17.430910110473633,
      "activations/layer12_attention_weight_min": -25.809289932250977,
      "activations/layer13_attention_weight_max": 35.023094177246094,
      "activations/layer13_attention_weight_min": -28.014596939086914,
      "activations/layer14_attention_weight_max": 33.226470947265625,
      "activations/layer14_attention_weight_min": -30.38208770751953,
      "activations/layer15_attention_weight_max": 31.058027267456055,
      "activations/layer15_attention_weight_min": -28.480499267578125,
      "activations/layer16_attention_weight_max": 29.936880111694336,
      "activations/layer16_attention_weight_min": -29.337507247924805,
      "activations/layer17_attention_weight_max": 48.655845642089844,
      "activations/layer17_attention_weight_min": -43.07475280761719,
      "activations/layer18_attention_weight_max": 41.90188980102539,
      "activations/layer18_attention_weight_min": -35.34566879272461,
      "activations/layer19_attention_weight_max": 20.692913055419922,
      "activations/layer19_attention_weight_min": -19.597515106201172,
      "activations/layer1_attention_weight_max": 15.95202922821045,
      "activations/layer1_attention_weight_min": -16.39815902709961,
      "activations/layer20_attention_weight_max": 19.615201950073242,
      "activations/layer20_attention_weight_min": -20.091060638427734,
      "activations/layer21_attention_weight_max": 31.686513900756836,
      "activations/layer21_attention_weight_min": -18.663171768188477,
      "activations/layer22_attention_weight_max": 25.522724151611328,
      "activations/layer22_attention_weight_min": -25.232770919799805,
      "activations/layer23_attention_weight_max": 34.5261116027832,
      "activations/layer23_attention_weight_min": -22.02736473083496,
      "activations/layer2_attention_weight_max": 33.963218688964844,
      "activations/layer2_attention_weight_min": -32.82957458496094,
      "activations/layer3_attention_weight_max": 103.09769439697266,
      "activations/layer3_attention_weight_min": -101.9692153930664,
      "activations/layer4_attention_weight_max": 101.85368347167969,
      "activations/layer4_attention_weight_min": -99.9145278930664,
      "activations/layer5_attention_weight_max": 73.81172180175781,
      "activations/layer5_attention_weight_min": -76.40310668945312,
      "activations/layer6_attention_weight_max": 54.668025970458984,
      "activations/layer6_attention_weight_min": -53.91453170776367,
      "activations/layer7_attention_weight_max": 69.7027359008789,
      "activations/layer7_attention_weight_min": -65.89279174804688,
      "activations/layer8_attention_weight_max": 45.9355354309082,
      "activations/layer8_attention_weight_min": -50.285526275634766,
      "activations/layer9_attention_weight_max": 46.348365783691406,
      "activations/layer9_attention_weight_min": -46.897769927978516,
      "epoch": 13.54,
      "learning_rate": 6.329393939393939e-05,
      "loss": 2.7508,
      "step": 233100
    },
    {
      "activations/layer0_attention_weight_max": 15.674518585205078,
      "activations/layer0_attention_weight_min": -14.152109146118164,
      "activations/layer10_attention_weight_max": 35.700382232666016,
      "activations/layer10_attention_weight_min": -35.52097702026367,
      "activations/layer11_attention_weight_max": 32.326271057128906,
      "activations/layer11_attention_weight_min": -32.6706657409668,
      "activations/layer12_attention_weight_max": 18.883962631225586,
      "activations/layer12_attention_weight_min": -24.50547218322754,
      "activations/layer13_attention_weight_max": 35.38418960571289,
      "activations/layer13_attention_weight_min": -30.884119033813477,
      "activations/layer14_attention_weight_max": 35.54407501220703,
      "activations/layer14_attention_weight_min": -30.293460845947266,
      "activations/layer15_attention_weight_max": 32.6328010559082,
      "activations/layer15_attention_weight_min": -29.438207626342773,
      "activations/layer16_attention_weight_max": 34.98482894897461,
      "activations/layer16_attention_weight_min": -34.1368408203125,
      "activations/layer17_attention_weight_max": 52.65656280517578,
      "activations/layer17_attention_weight_min": -49.84957504272461,
      "activations/layer18_attention_weight_max": 46.78532409667969,
      "activations/layer18_attention_weight_min": -41.7119255065918,
      "activations/layer19_attention_weight_max": 22.208829879760742,
      "activations/layer19_attention_weight_min": -22.6580753326416,
      "activations/layer1_attention_weight_max": 16.018600463867188,
      "activations/layer1_attention_weight_min": -15.628580093383789,
      "activations/layer20_attention_weight_max": 21.31325340270996,
      "activations/layer20_attention_weight_min": -22.033334732055664,
      "activations/layer21_attention_weight_max": 34.59156036376953,
      "activations/layer21_attention_weight_min": -25.095354080200195,
      "activations/layer22_attention_weight_max": 29.562368392944336,
      "activations/layer22_attention_weight_min": -25.151073455810547,
      "activations/layer23_attention_weight_max": 39.98115539550781,
      "activations/layer23_attention_weight_min": -24.641082763671875,
      "activations/layer2_attention_weight_max": 32.28737258911133,
      "activations/layer2_attention_weight_min": -30.018978118896484,
      "activations/layer3_attention_weight_max": 91.180908203125,
      "activations/layer3_attention_weight_min": -95.33584594726562,
      "activations/layer4_attention_weight_max": 93.84431457519531,
      "activations/layer4_attention_weight_min": -94.52725219726562,
      "activations/layer5_attention_weight_max": 67.19535827636719,
      "activations/layer5_attention_weight_min": -80.35675811767578,
      "activations/layer6_attention_weight_max": 50.60668182373047,
      "activations/layer6_attention_weight_min": -54.41210174560547,
      "activations/layer7_attention_weight_max": 68.48348236083984,
      "activations/layer7_attention_weight_min": -67.57776641845703,
      "activations/layer8_attention_weight_max": 44.6613883972168,
      "activations/layer8_attention_weight_min": -47.1536750793457,
      "activations/layer9_attention_weight_max": 45.83720779418945,
      "activations/layer9_attention_weight_min": -46.22187423706055,
      "epoch": 13.55,
      "learning_rate": 6.327499999999999e-05,
      "loss": 2.7628,
      "step": 233150
    },
    {
      "activations/layer0_attention_weight_max": 16.275846481323242,
      "activations/layer0_attention_weight_min": -14.365674018859863,
      "activations/layer10_attention_weight_max": 37.50909423828125,
      "activations/layer10_attention_weight_min": -35.97648620605469,
      "activations/layer11_attention_weight_max": 33.816001892089844,
      "activations/layer11_attention_weight_min": -33.67262649536133,
      "activations/layer12_attention_weight_max": 18.526123046875,
      "activations/layer12_attention_weight_min": -26.242414474487305,
      "activations/layer13_attention_weight_max": 37.831722259521484,
      "activations/layer13_attention_weight_min": -32.82806396484375,
      "activations/layer14_attention_weight_max": 38.62964630126953,
      "activations/layer14_attention_weight_min": -30.72026824951172,
      "activations/layer15_attention_weight_max": 33.44038391113281,
      "activations/layer15_attention_weight_min": -29.187524795532227,
      "activations/layer16_attention_weight_max": 32.38776779174805,
      "activations/layer16_attention_weight_min": -30.489620208740234,
      "activations/layer17_attention_weight_max": 50.86351776123047,
      "activations/layer17_attention_weight_min": -43.46596145629883,
      "activations/layer18_attention_weight_max": 46.727699279785156,
      "activations/layer18_attention_weight_min": -38.49297332763672,
      "activations/layer19_attention_weight_max": 23.78444480895996,
      "activations/layer19_attention_weight_min": -23.245086669921875,
      "activations/layer1_attention_weight_max": 17.04294204711914,
      "activations/layer1_attention_weight_min": -14.536333084106445,
      "activations/layer20_attention_weight_max": 23.030885696411133,
      "activations/layer20_attention_weight_min": -21.382915496826172,
      "activations/layer21_attention_weight_max": 35.152549743652344,
      "activations/layer21_attention_weight_min": -24.843822479248047,
      "activations/layer22_attention_weight_max": 32.5189094543457,
      "activations/layer22_attention_weight_min": -26.725088119506836,
      "activations/layer23_attention_weight_max": 44.196075439453125,
      "activations/layer23_attention_weight_min": -26.114912033081055,
      "activations/layer2_attention_weight_max": 33.668861389160156,
      "activations/layer2_attention_weight_min": -32.181434631347656,
      "activations/layer3_attention_weight_max": 100.1366958618164,
      "activations/layer3_attention_weight_min": -103.33084869384766,
      "activations/layer4_attention_weight_max": 98.25492858886719,
      "activations/layer4_attention_weight_min": -96.17522430419922,
      "activations/layer5_attention_weight_max": 73.44979095458984,
      "activations/layer5_attention_weight_min": -77.79119873046875,
      "activations/layer6_attention_weight_max": 53.297183990478516,
      "activations/layer6_attention_weight_min": -52.313148498535156,
      "activations/layer7_attention_weight_max": 70.14993286132812,
      "activations/layer7_attention_weight_min": -63.89429473876953,
      "activations/layer8_attention_weight_max": 47.546592712402344,
      "activations/layer8_attention_weight_min": -49.68522262573242,
      "activations/layer9_attention_weight_max": 46.191864013671875,
      "activations/layer9_attention_weight_min": -46.5225715637207,
      "epoch": 13.55,
      "learning_rate": 6.32560606060606e-05,
      "loss": 2.7637,
      "step": 233200
    },
    {
      "activations/layer0_attention_weight_max": 15.95388412475586,
      "activations/layer0_attention_weight_min": -14.048233032226562,
      "activations/layer10_attention_weight_max": 38.794883728027344,
      "activations/layer10_attention_weight_min": -37.76841354370117,
      "activations/layer11_attention_weight_max": 33.53118133544922,
      "activations/layer11_attention_weight_min": -34.35050964355469,
      "activations/layer12_attention_weight_max": 18.035659790039062,
      "activations/layer12_attention_weight_min": -25.52444076538086,
      "activations/layer13_attention_weight_max": 34.04974365234375,
      "activations/layer13_attention_weight_min": -27.16095542907715,
      "activations/layer14_attention_weight_max": 35.38792419433594,
      "activations/layer14_attention_weight_min": -28.878061294555664,
      "activations/layer15_attention_weight_max": 32.00160598754883,
      "activations/layer15_attention_weight_min": -28.409814834594727,
      "activations/layer16_attention_weight_max": 31.47752571105957,
      "activations/layer16_attention_weight_min": -28.246959686279297,
      "activations/layer17_attention_weight_max": 52.14622497558594,
      "activations/layer17_attention_weight_min": -44.18180847167969,
      "activations/layer18_attention_weight_max": 44.40195083618164,
      "activations/layer18_attention_weight_min": -37.86244583129883,
      "activations/layer19_attention_weight_max": 22.425914764404297,
      "activations/layer19_attention_weight_min": -20.53321075439453,
      "activations/layer1_attention_weight_max": 17.205202102661133,
      "activations/layer1_attention_weight_min": -15.096988677978516,
      "activations/layer20_attention_weight_max": 22.08252716064453,
      "activations/layer20_attention_weight_min": -20.96756362915039,
      "activations/layer21_attention_weight_max": 34.77446365356445,
      "activations/layer21_attention_weight_min": -22.206871032714844,
      "activations/layer22_attention_weight_max": 27.4487247467041,
      "activations/layer22_attention_weight_min": -25.455097198486328,
      "activations/layer23_attention_weight_max": 36.025177001953125,
      "activations/layer23_attention_weight_min": -24.361839294433594,
      "activations/layer2_attention_weight_max": 34.550323486328125,
      "activations/layer2_attention_weight_min": -31.73915672302246,
      "activations/layer3_attention_weight_max": 104.96125030517578,
      "activations/layer3_attention_weight_min": -105.94738006591797,
      "activations/layer4_attention_weight_max": 101.54816436767578,
      "activations/layer4_attention_weight_min": -96.80655670166016,
      "activations/layer5_attention_weight_max": 74.47101593017578,
      "activations/layer5_attention_weight_min": -80.16320037841797,
      "activations/layer6_attention_weight_max": 54.3410530090332,
      "activations/layer6_attention_weight_min": -53.70744323730469,
      "activations/layer7_attention_weight_max": 74.15257263183594,
      "activations/layer7_attention_weight_min": -70.56057739257812,
      "activations/layer8_attention_weight_max": 47.31720733642578,
      "activations/layer8_attention_weight_min": -48.389625549316406,
      "activations/layer9_attention_weight_max": 48.39818572998047,
      "activations/layer9_attention_weight_min": -49.98380661010742,
      "epoch": 13.55,
      "learning_rate": 6.323712121212121e-05,
      "loss": 2.761,
      "step": 233250
    },
    {
      "activations/layer0_attention_weight_max": 16.096721649169922,
      "activations/layer0_attention_weight_min": -14.13358211517334,
      "activations/layer10_attention_weight_max": 34.46168899536133,
      "activations/layer10_attention_weight_min": -34.22063064575195,
      "activations/layer11_attention_weight_max": 31.392929077148438,
      "activations/layer11_attention_weight_min": -31.696455001831055,
      "activations/layer12_attention_weight_max": 21.66148567199707,
      "activations/layer12_attention_weight_min": -21.30192756652832,
      "activations/layer13_attention_weight_max": 38.13054275512695,
      "activations/layer13_attention_weight_min": -26.726581573486328,
      "activations/layer14_attention_weight_max": 36.711700439453125,
      "activations/layer14_attention_weight_min": -29.74308967590332,
      "activations/layer15_attention_weight_max": 33.8252067565918,
      "activations/layer15_attention_weight_min": -28.87397003173828,
      "activations/layer16_attention_weight_max": 33.98685836791992,
      "activations/layer16_attention_weight_min": -31.03725242614746,
      "activations/layer17_attention_weight_max": 51.84449768066406,
      "activations/layer17_attention_weight_min": -43.95965576171875,
      "activations/layer18_attention_weight_max": 47.68795394897461,
      "activations/layer18_attention_weight_min": -36.902496337890625,
      "activations/layer19_attention_weight_max": 24.739927291870117,
      "activations/layer19_attention_weight_min": -23.361173629760742,
      "activations/layer1_attention_weight_max": 15.772100448608398,
      "activations/layer1_attention_weight_min": -15.837770462036133,
      "activations/layer20_attention_weight_max": 25.355375289916992,
      "activations/layer20_attention_weight_min": -27.328096389770508,
      "activations/layer21_attention_weight_max": 37.046485900878906,
      "activations/layer21_attention_weight_min": -22.606718063354492,
      "activations/layer22_attention_weight_max": 30.78072166442871,
      "activations/layer22_attention_weight_min": -25.078292846679688,
      "activations/layer23_attention_weight_max": 36.88995361328125,
      "activations/layer23_attention_weight_min": -25.2781982421875,
      "activations/layer2_attention_weight_max": 32.93882751464844,
      "activations/layer2_attention_weight_min": -31.16851806640625,
      "activations/layer3_attention_weight_max": 97.3106918334961,
      "activations/layer3_attention_weight_min": -99.01046752929688,
      "activations/layer4_attention_weight_max": 96.79158020019531,
      "activations/layer4_attention_weight_min": -97.09858703613281,
      "activations/layer5_attention_weight_max": 71.72527313232422,
      "activations/layer5_attention_weight_min": -74.9732666015625,
      "activations/layer6_attention_weight_max": 53.48330307006836,
      "activations/layer6_attention_weight_min": -49.169456481933594,
      "activations/layer7_attention_weight_max": 65.3550796508789,
      "activations/layer7_attention_weight_min": -59.4728889465332,
      "activations/layer8_attention_weight_max": 42.63363265991211,
      "activations/layer8_attention_weight_min": -43.61991500854492,
      "activations/layer9_attention_weight_max": 42.992515563964844,
      "activations/layer9_attention_weight_min": -43.12356185913086,
      "epoch": 13.56,
      "learning_rate": 6.321818181818181e-05,
      "loss": 2.7536,
      "step": 233300
    },
    {
      "activations/layer0_attention_weight_max": 15.749789237976074,
      "activations/layer0_attention_weight_min": -14.120871543884277,
      "activations/layer10_attention_weight_max": 37.59480667114258,
      "activations/layer10_attention_weight_min": -36.17021942138672,
      "activations/layer11_attention_weight_max": 34.15764617919922,
      "activations/layer11_attention_weight_min": -32.738929748535156,
      "activations/layer12_attention_weight_max": 20.17310905456543,
      "activations/layer12_attention_weight_min": -31.48021697998047,
      "activations/layer13_attention_weight_max": 38.6262321472168,
      "activations/layer13_attention_weight_min": -29.414125442504883,
      "activations/layer14_attention_weight_max": 43.5078125,
      "activations/layer14_attention_weight_min": -30.46446418762207,
      "activations/layer15_attention_weight_max": 43.0542106628418,
      "activations/layer15_attention_weight_min": -29.685707092285156,
      "activations/layer16_attention_weight_max": 31.605663299560547,
      "activations/layer16_attention_weight_min": -28.88087272644043,
      "activations/layer17_attention_weight_max": 51.3023567199707,
      "activations/layer17_attention_weight_min": -41.80986404418945,
      "activations/layer18_attention_weight_max": 44.3165397644043,
      "activations/layer18_attention_weight_min": -36.9709587097168,
      "activations/layer19_attention_weight_max": 22.935684204101562,
      "activations/layer19_attention_weight_min": -21.67829132080078,
      "activations/layer1_attention_weight_max": 17.558813095092773,
      "activations/layer1_attention_weight_min": -18.83270835876465,
      "activations/layer20_attention_weight_max": 20.84248924255371,
      "activations/layer20_attention_weight_min": -21.927024841308594,
      "activations/layer21_attention_weight_max": 37.198726654052734,
      "activations/layer21_attention_weight_min": -24.08096694946289,
      "activations/layer22_attention_weight_max": 33.080039978027344,
      "activations/layer22_attention_weight_min": -27.513261795043945,
      "activations/layer23_attention_weight_max": 37.90301513671875,
      "activations/layer23_attention_weight_min": -25.420551300048828,
      "activations/layer2_attention_weight_max": 35.32710266113281,
      "activations/layer2_attention_weight_min": -33.30058288574219,
      "activations/layer3_attention_weight_max": 102.38341522216797,
      "activations/layer3_attention_weight_min": -101.31227111816406,
      "activations/layer4_attention_weight_max": 100.1109848022461,
      "activations/layer4_attention_weight_min": -95.13947296142578,
      "activations/layer5_attention_weight_max": 77.9913558959961,
      "activations/layer5_attention_weight_min": -75.32047271728516,
      "activations/layer6_attention_weight_max": 54.03878402709961,
      "activations/layer6_attention_weight_min": -53.186458587646484,
      "activations/layer7_attention_weight_max": 76.48294067382812,
      "activations/layer7_attention_weight_min": -71.05525970458984,
      "activations/layer8_attention_weight_max": 51.21528244018555,
      "activations/layer8_attention_weight_min": -49.90449142456055,
      "activations/layer9_attention_weight_max": 57.979522705078125,
      "activations/layer9_attention_weight_min": -51.078853607177734,
      "epoch": 13.56,
      "learning_rate": 6.319924242424241e-05,
      "loss": 2.763,
      "step": 233350
    },
    {
      "activations/layer0_attention_weight_max": 16.5719051361084,
      "activations/layer0_attention_weight_min": -13.89542007446289,
      "activations/layer10_attention_weight_max": 38.01083755493164,
      "activations/layer10_attention_weight_min": -37.541011810302734,
      "activations/layer11_attention_weight_max": 33.6734619140625,
      "activations/layer11_attention_weight_min": -33.817222595214844,
      "activations/layer12_attention_weight_max": 21.980615615844727,
      "activations/layer12_attention_weight_min": -24.44744300842285,
      "activations/layer13_attention_weight_max": 38.997039794921875,
      "activations/layer13_attention_weight_min": -30.383092880249023,
      "activations/layer14_attention_weight_max": 36.852752685546875,
      "activations/layer14_attention_weight_min": -31.024011611938477,
      "activations/layer15_attention_weight_max": 33.40773010253906,
      "activations/layer15_attention_weight_min": -28.551773071289062,
      "activations/layer16_attention_weight_max": 31.606067657470703,
      "activations/layer16_attention_weight_min": -28.85787010192871,
      "activations/layer17_attention_weight_max": 51.87092208862305,
      "activations/layer17_attention_weight_min": -45.58060836791992,
      "activations/layer18_attention_weight_max": 43.28668212890625,
      "activations/layer18_attention_weight_min": -38.107887268066406,
      "activations/layer19_attention_weight_max": 24.32794761657715,
      "activations/layer19_attention_weight_min": -20.361143112182617,
      "activations/layer1_attention_weight_max": 16.905261993408203,
      "activations/layer1_attention_weight_min": -17.04236602783203,
      "activations/layer20_attention_weight_max": 22.97600555419922,
      "activations/layer20_attention_weight_min": -22.150848388671875,
      "activations/layer21_attention_weight_max": 38.86234664916992,
      "activations/layer21_attention_weight_min": -22.60477638244629,
      "activations/layer22_attention_weight_max": 29.96174430847168,
      "activations/layer22_attention_weight_min": -25.10333251953125,
      "activations/layer23_attention_weight_max": 38.80284118652344,
      "activations/layer23_attention_weight_min": -23.939781188964844,
      "activations/layer2_attention_weight_max": 34.23048400878906,
      "activations/layer2_attention_weight_min": -33.22962188720703,
      "activations/layer3_attention_weight_max": 100.43093872070312,
      "activations/layer3_attention_weight_min": -97.4937973022461,
      "activations/layer4_attention_weight_max": 99.4096908569336,
      "activations/layer4_attention_weight_min": -94.9048080444336,
      "activations/layer5_attention_weight_max": 72.85676574707031,
      "activations/layer5_attention_weight_min": -76.30096435546875,
      "activations/layer6_attention_weight_max": 52.73320770263672,
      "activations/layer6_attention_weight_min": -53.21843719482422,
      "activations/layer7_attention_weight_max": 65.98922729492188,
      "activations/layer7_attention_weight_min": -66.60530853271484,
      "activations/layer8_attention_weight_max": 46.83646011352539,
      "activations/layer8_attention_weight_min": -50.49915313720703,
      "activations/layer9_attention_weight_max": 49.49846649169922,
      "activations/layer9_attention_weight_min": -48.87470245361328,
      "epoch": 13.56,
      "learning_rate": 6.318068181818182e-05,
      "loss": 2.778,
      "step": 233400
    },
    {
      "activations/layer0_attention_weight_max": 14.873553276062012,
      "activations/layer0_attention_weight_min": -14.23328971862793,
      "activations/layer10_attention_weight_max": 36.01099395751953,
      "activations/layer10_attention_weight_min": -36.57093811035156,
      "activations/layer11_attention_weight_max": 31.715259552001953,
      "activations/layer11_attention_weight_min": -33.214481353759766,
      "activations/layer12_attention_weight_max": 17.478042602539062,
      "activations/layer12_attention_weight_min": -27.236324310302734,
      "activations/layer13_attention_weight_max": 35.87895965576172,
      "activations/layer13_attention_weight_min": -28.411884307861328,
      "activations/layer14_attention_weight_max": 36.48509979248047,
      "activations/layer14_attention_weight_min": -32.71757507324219,
      "activations/layer15_attention_weight_max": 32.723182678222656,
      "activations/layer15_attention_weight_min": -32.91685104370117,
      "activations/layer16_attention_weight_max": 32.03463363647461,
      "activations/layer16_attention_weight_min": -31.996761322021484,
      "activations/layer17_attention_weight_max": 50.23918533325195,
      "activations/layer17_attention_weight_min": -47.853275299072266,
      "activations/layer18_attention_weight_max": 47.892799377441406,
      "activations/layer18_attention_weight_min": -41.86968231201172,
      "activations/layer19_attention_weight_max": 23.523296356201172,
      "activations/layer19_attention_weight_min": -24.516014099121094,
      "activations/layer1_attention_weight_max": 16.604469299316406,
      "activations/layer1_attention_weight_min": -16.30964469909668,
      "activations/layer20_attention_weight_max": 23.97817611694336,
      "activations/layer20_attention_weight_min": -25.981605529785156,
      "activations/layer21_attention_weight_max": 41.16572952270508,
      "activations/layer21_attention_weight_min": -23.638761520385742,
      "activations/layer22_attention_weight_max": 31.559839248657227,
      "activations/layer22_attention_weight_min": -27.228073120117188,
      "activations/layer23_attention_weight_max": 35.99339294433594,
      "activations/layer23_attention_weight_min": -28.224689483642578,
      "activations/layer2_attention_weight_max": 34.274574279785156,
      "activations/layer2_attention_weight_min": -31.794111251831055,
      "activations/layer3_attention_weight_max": 97.91893768310547,
      "activations/layer3_attention_weight_min": -100.00192260742188,
      "activations/layer4_attention_weight_max": 101.01971435546875,
      "activations/layer4_attention_weight_min": -98.74818420410156,
      "activations/layer5_attention_weight_max": 74.26439666748047,
      "activations/layer5_attention_weight_min": -83.27717590332031,
      "activations/layer6_attention_weight_max": 53.39295959472656,
      "activations/layer6_attention_weight_min": -54.848567962646484,
      "activations/layer7_attention_weight_max": 70.81773376464844,
      "activations/layer7_attention_weight_min": -65.09002685546875,
      "activations/layer8_attention_weight_max": 43.40166091918945,
      "activations/layer8_attention_weight_min": -47.51704788208008,
      "activations/layer9_attention_weight_max": 44.375450134277344,
      "activations/layer9_attention_weight_min": -47.805816650390625,
      "epoch": 13.56,
      "learning_rate": 6.316174242424242e-05,
      "loss": 2.7583,
      "step": 233450
    },
    {
      "activations/layer0_attention_weight_max": 16.73440933227539,
      "activations/layer0_attention_weight_min": -14.425581932067871,
      "activations/layer10_attention_weight_max": 40.982051849365234,
      "activations/layer10_attention_weight_min": -40.9542236328125,
      "activations/layer11_attention_weight_max": 36.22541809082031,
      "activations/layer11_attention_weight_min": -38.06168746948242,
      "activations/layer12_attention_weight_max": 17.68834686279297,
      "activations/layer12_attention_weight_min": -26.63465118408203,
      "activations/layer13_attention_weight_max": 39.764015197753906,
      "activations/layer13_attention_weight_min": -31.28173828125,
      "activations/layer14_attention_weight_max": 38.991294860839844,
      "activations/layer14_attention_weight_min": -30.13902473449707,
      "activations/layer15_attention_weight_max": 33.186588287353516,
      "activations/layer15_attention_weight_min": -29.492868423461914,
      "activations/layer16_attention_weight_max": 31.996355056762695,
      "activations/layer16_attention_weight_min": -30.946500778198242,
      "activations/layer17_attention_weight_max": 52.71440505981445,
      "activations/layer17_attention_weight_min": -44.409908294677734,
      "activations/layer18_attention_weight_max": 45.93745040893555,
      "activations/layer18_attention_weight_min": -36.7793083190918,
      "activations/layer19_attention_weight_max": 20.28666114807129,
      "activations/layer19_attention_weight_min": -20.85398292541504,
      "activations/layer1_attention_weight_max": 16.777236938476562,
      "activations/layer1_attention_weight_min": -14.6304931640625,
      "activations/layer20_attention_weight_max": 21.14151382446289,
      "activations/layer20_attention_weight_min": -23.411386489868164,
      "activations/layer21_attention_weight_max": 34.935848236083984,
      "activations/layer21_attention_weight_min": -21.448013305664062,
      "activations/layer22_attention_weight_max": 28.826549530029297,
      "activations/layer22_attention_weight_min": -25.320270538330078,
      "activations/layer23_attention_weight_max": 37.645599365234375,
      "activations/layer23_attention_weight_min": -25.074222564697266,
      "activations/layer2_attention_weight_max": 32.167911529541016,
      "activations/layer2_attention_weight_min": -30.728992462158203,
      "activations/layer3_attention_weight_max": 94.7227554321289,
      "activations/layer3_attention_weight_min": -99.90950775146484,
      "activations/layer4_attention_weight_max": 96.5782699584961,
      "activations/layer4_attention_weight_min": -94.47472381591797,
      "activations/layer5_attention_weight_max": 70.95514678955078,
      "activations/layer5_attention_weight_min": -77.84817504882812,
      "activations/layer6_attention_weight_max": 52.76033020019531,
      "activations/layer6_attention_weight_min": -54.57908630371094,
      "activations/layer7_attention_weight_max": 68.8604965209961,
      "activations/layer7_attention_weight_min": -67.87760162353516,
      "activations/layer8_attention_weight_max": 49.87348937988281,
      "activations/layer8_attention_weight_min": -54.565860748291016,
      "activations/layer9_attention_weight_max": 51.0608024597168,
      "activations/layer9_attention_weight_min": -50.94930648803711,
      "epoch": 13.57,
      "learning_rate": 6.314280303030303e-05,
      "loss": 2.7588,
      "step": 233500
    },
    {
      "activations/layer0_attention_weight_max": 15.409557342529297,
      "activations/layer0_attention_weight_min": -14.300971031188965,
      "activations/layer10_attention_weight_max": 37.823890686035156,
      "activations/layer10_attention_weight_min": -38.259498596191406,
      "activations/layer11_attention_weight_max": 33.33802032470703,
      "activations/layer11_attention_weight_min": -33.47035598754883,
      "activations/layer12_attention_weight_max": 17.693483352661133,
      "activations/layer12_attention_weight_min": -24.868194580078125,
      "activations/layer13_attention_weight_max": 34.22654342651367,
      "activations/layer13_attention_weight_min": -31.957977294921875,
      "activations/layer14_attention_weight_max": 36.758853912353516,
      "activations/layer14_attention_weight_min": -30.595605850219727,
      "activations/layer15_attention_weight_max": 34.2476921081543,
      "activations/layer15_attention_weight_min": -29.249509811401367,
      "activations/layer16_attention_weight_max": 31.135986328125,
      "activations/layer16_attention_weight_min": -30.59441375732422,
      "activations/layer17_attention_weight_max": 52.78119659423828,
      "activations/layer17_attention_weight_min": -46.922119140625,
      "activations/layer18_attention_weight_max": 50.64470672607422,
      "activations/layer18_attention_weight_min": -43.36539077758789,
      "activations/layer19_attention_weight_max": 22.451343536376953,
      "activations/layer19_attention_weight_min": -24.475082397460938,
      "activations/layer1_attention_weight_max": 18.280118942260742,
      "activations/layer1_attention_weight_min": -15.396681785583496,
      "activations/layer20_attention_weight_max": 22.30284309387207,
      "activations/layer20_attention_weight_min": -21.31122589111328,
      "activations/layer21_attention_weight_max": 33.485267639160156,
      "activations/layer21_attention_weight_min": -23.461833953857422,
      "activations/layer22_attention_weight_max": 29.50188446044922,
      "activations/layer22_attention_weight_min": -25.885009765625,
      "activations/layer23_attention_weight_max": 37.39284133911133,
      "activations/layer23_attention_weight_min": -22.414167404174805,
      "activations/layer2_attention_weight_max": 32.658935546875,
      "activations/layer2_attention_weight_min": -31.409521102905273,
      "activations/layer3_attention_weight_max": 93.86212158203125,
      "activations/layer3_attention_weight_min": -94.10465240478516,
      "activations/layer4_attention_weight_max": 97.02256774902344,
      "activations/layer4_attention_weight_min": -93.57406616210938,
      "activations/layer5_attention_weight_max": 70.59056091308594,
      "activations/layer5_attention_weight_min": -80.12554168701172,
      "activations/layer6_attention_weight_max": 53.23091506958008,
      "activations/layer6_attention_weight_min": -52.87480545043945,
      "activations/layer7_attention_weight_max": 70.87396240234375,
      "activations/layer7_attention_weight_min": -67.47586059570312,
      "activations/layer8_attention_weight_max": 47.70860290527344,
      "activations/layer8_attention_weight_min": -51.07556915283203,
      "activations/layer9_attention_weight_max": 48.929595947265625,
      "activations/layer9_attention_weight_min": -48.50751876831055,
      "epoch": 13.57,
      "learning_rate": 6.312386363636363e-05,
      "loss": 2.7766,
      "step": 233550
    },
    {
      "activations/layer0_attention_weight_max": 16.134490966796875,
      "activations/layer0_attention_weight_min": -14.03212833404541,
      "activations/layer10_attention_weight_max": 36.210086822509766,
      "activations/layer10_attention_weight_min": -35.002197265625,
      "activations/layer11_attention_weight_max": 32.21586608886719,
      "activations/layer11_attention_weight_min": -33.35944366455078,
      "activations/layer12_attention_weight_max": 17.148956298828125,
      "activations/layer12_attention_weight_min": -26.138444900512695,
      "activations/layer13_attention_weight_max": 37.75498962402344,
      "activations/layer13_attention_weight_min": -29.81309700012207,
      "activations/layer14_attention_weight_max": 34.528350830078125,
      "activations/layer14_attention_weight_min": -31.8492488861084,
      "activations/layer15_attention_weight_max": 35.64181137084961,
      "activations/layer15_attention_weight_min": -31.544347763061523,
      "activations/layer16_attention_weight_max": 31.324182510375977,
      "activations/layer16_attention_weight_min": -31.38981056213379,
      "activations/layer17_attention_weight_max": 50.671634674072266,
      "activations/layer17_attention_weight_min": -44.67803192138672,
      "activations/layer18_attention_weight_max": 43.73929214477539,
      "activations/layer18_attention_weight_min": -37.561065673828125,
      "activations/layer19_attention_weight_max": 22.959171295166016,
      "activations/layer19_attention_weight_min": -20.489103317260742,
      "activations/layer1_attention_weight_max": 16.131563186645508,
      "activations/layer1_attention_weight_min": -15.390501022338867,
      "activations/layer20_attention_weight_max": 20.349376678466797,
      "activations/layer20_attention_weight_min": -22.684921264648438,
      "activations/layer21_attention_weight_max": 35.999752044677734,
      "activations/layer21_attention_weight_min": -21.963281631469727,
      "activations/layer22_attention_weight_max": 29.677412033081055,
      "activations/layer22_attention_weight_min": -27.6699161529541,
      "activations/layer23_attention_weight_max": 35.991676330566406,
      "activations/layer23_attention_weight_min": -26.610137939453125,
      "activations/layer2_attention_weight_max": 31.826618194580078,
      "activations/layer2_attention_weight_min": -29.555204391479492,
      "activations/layer3_attention_weight_max": 92.37117767333984,
      "activations/layer3_attention_weight_min": -92.33675384521484,
      "activations/layer4_attention_weight_max": 93.59808349609375,
      "activations/layer4_attention_weight_min": -89.49327087402344,
      "activations/layer5_attention_weight_max": 70.31837463378906,
      "activations/layer5_attention_weight_min": -81.83015441894531,
      "activations/layer6_attention_weight_max": 49.54550552368164,
      "activations/layer6_attention_weight_min": -51.74546432495117,
      "activations/layer7_attention_weight_max": 67.44882202148438,
      "activations/layer7_attention_weight_min": -64.34234619140625,
      "activations/layer8_attention_weight_max": 43.53471374511719,
      "activations/layer8_attention_weight_min": -48.67252731323242,
      "activations/layer9_attention_weight_max": 45.94702911376953,
      "activations/layer9_attention_weight_min": -47.0385856628418,
      "epoch": 13.57,
      "learning_rate": 6.310492424242424e-05,
      "loss": 2.7749,
      "step": 233600
    },
    {
      "activations/layer0_attention_weight_max": 16.302091598510742,
      "activations/layer0_attention_weight_min": -15.468819618225098,
      "activations/layer10_attention_weight_max": 38.10649490356445,
      "activations/layer10_attention_weight_min": -38.707366943359375,
      "activations/layer11_attention_weight_max": 32.47658157348633,
      "activations/layer11_attention_weight_min": -33.2059211730957,
      "activations/layer12_attention_weight_max": 19.48975372314453,
      "activations/layer12_attention_weight_min": -26.97038459777832,
      "activations/layer13_attention_weight_max": 44.1060676574707,
      "activations/layer13_attention_weight_min": -28.676076889038086,
      "activations/layer14_attention_weight_max": 37.25456619262695,
      "activations/layer14_attention_weight_min": -30.360456466674805,
      "activations/layer15_attention_weight_max": 34.78593063354492,
      "activations/layer15_attention_weight_min": -30.996389389038086,
      "activations/layer16_attention_weight_max": 35.16440963745117,
      "activations/layer16_attention_weight_min": -32.088836669921875,
      "activations/layer17_attention_weight_max": 57.88692855834961,
      "activations/layer17_attention_weight_min": -46.0535774230957,
      "activations/layer18_attention_weight_max": 55.232845306396484,
      "activations/layer18_attention_weight_min": -40.99546813964844,
      "activations/layer19_attention_weight_max": 21.698545455932617,
      "activations/layer19_attention_weight_min": -22.11983871459961,
      "activations/layer1_attention_weight_max": 16.249231338500977,
      "activations/layer1_attention_weight_min": -15.940422058105469,
      "activations/layer20_attention_weight_max": 24.658557891845703,
      "activations/layer20_attention_weight_min": -22.372825622558594,
      "activations/layer21_attention_weight_max": 36.498138427734375,
      "activations/layer21_attention_weight_min": -24.264780044555664,
      "activations/layer22_attention_weight_max": 30.16275405883789,
      "activations/layer22_attention_weight_min": -26.423494338989258,
      "activations/layer23_attention_weight_max": 35.62754821777344,
      "activations/layer23_attention_weight_min": -24.312122344970703,
      "activations/layer2_attention_weight_max": 32.27623748779297,
      "activations/layer2_attention_weight_min": -30.901004791259766,
      "activations/layer3_attention_weight_max": 95.92633819580078,
      "activations/layer3_attention_weight_min": -95.76014709472656,
      "activations/layer4_attention_weight_max": 97.30455780029297,
      "activations/layer4_attention_weight_min": -90.76329803466797,
      "activations/layer5_attention_weight_max": 73.94485473632812,
      "activations/layer5_attention_weight_min": -79.93730163574219,
      "activations/layer6_attention_weight_max": 51.46451187133789,
      "activations/layer6_attention_weight_min": -52.0294303894043,
      "activations/layer7_attention_weight_max": 67.05142974853516,
      "activations/layer7_attention_weight_min": -62.854610443115234,
      "activations/layer8_attention_weight_max": 45.08715057373047,
      "activations/layer8_attention_weight_min": -50.961177825927734,
      "activations/layer9_attention_weight_max": 47.0239372253418,
      "activations/layer9_attention_weight_min": -48.30968475341797,
      "epoch": 13.58,
      "learning_rate": 6.308598484848485e-05,
      "loss": 2.7558,
      "step": 233650
    },
    {
      "activations/layer0_attention_weight_max": 15.844245910644531,
      "activations/layer0_attention_weight_min": -14.590642929077148,
      "activations/layer10_attention_weight_max": 35.69782638549805,
      "activations/layer10_attention_weight_min": -34.65039825439453,
      "activations/layer11_attention_weight_max": 33.2302131652832,
      "activations/layer11_attention_weight_min": -34.99385070800781,
      "activations/layer12_attention_weight_max": 17.955862045288086,
      "activations/layer12_attention_weight_min": -26.86334800720215,
      "activations/layer13_attention_weight_max": 31.380435943603516,
      "activations/layer13_attention_weight_min": -32.4516487121582,
      "activations/layer14_attention_weight_max": 34.83515167236328,
      "activations/layer14_attention_weight_min": -29.163249969482422,
      "activations/layer15_attention_weight_max": 33.47223663330078,
      "activations/layer15_attention_weight_min": -29.20248031616211,
      "activations/layer16_attention_weight_max": 31.813709259033203,
      "activations/layer16_attention_weight_min": -30.82042121887207,
      "activations/layer17_attention_weight_max": 52.714168548583984,
      "activations/layer17_attention_weight_min": -46.402069091796875,
      "activations/layer18_attention_weight_max": 44.96330642700195,
      "activations/layer18_attention_weight_min": -38.339027404785156,
      "activations/layer19_attention_weight_max": 22.637813568115234,
      "activations/layer19_attention_weight_min": -20.771724700927734,
      "activations/layer1_attention_weight_max": 17.640134811401367,
      "activations/layer1_attention_weight_min": -18.485565185546875,
      "activations/layer20_attention_weight_max": 19.76169204711914,
      "activations/layer20_attention_weight_min": -20.235736846923828,
      "activations/layer21_attention_weight_max": 29.265987396240234,
      "activations/layer21_attention_weight_min": -19.5965576171875,
      "activations/layer22_attention_weight_max": 28.63949203491211,
      "activations/layer22_attention_weight_min": -24.739004135131836,
      "activations/layer23_attention_weight_max": 37.10188674926758,
      "activations/layer23_attention_weight_min": -23.326078414916992,
      "activations/layer2_attention_weight_max": 34.112091064453125,
      "activations/layer2_attention_weight_min": -32.859962463378906,
      "activations/layer3_attention_weight_max": 99.29000091552734,
      "activations/layer3_attention_weight_min": -97.46690368652344,
      "activations/layer4_attention_weight_max": 96.90501403808594,
      "activations/layer4_attention_weight_min": -94.64067840576172,
      "activations/layer5_attention_weight_max": 72.24403381347656,
      "activations/layer5_attention_weight_min": -78.60126495361328,
      "activations/layer6_attention_weight_max": 54.67747497558594,
      "activations/layer6_attention_weight_min": -52.041908264160156,
      "activations/layer7_attention_weight_max": 66.32998657226562,
      "activations/layer7_attention_weight_min": -68.11906433105469,
      "activations/layer8_attention_weight_max": 45.60612106323242,
      "activations/layer8_attention_weight_min": -48.384002685546875,
      "activations/layer9_attention_weight_max": 43.86037063598633,
      "activations/layer9_attention_weight_min": -45.975765228271484,
      "epoch": 13.58,
      "learning_rate": 6.306704545454545e-05,
      "loss": 2.7603,
      "step": 233700
    },
    {
      "activations/layer0_attention_weight_max": 15.690584182739258,
      "activations/layer0_attention_weight_min": -14.314102172851562,
      "activations/layer10_attention_weight_max": 34.503639221191406,
      "activations/layer10_attention_weight_min": -34.49424743652344,
      "activations/layer11_attention_weight_max": 32.21416473388672,
      "activations/layer11_attention_weight_min": -32.510154724121094,
      "activations/layer12_attention_weight_max": 19.017309188842773,
      "activations/layer12_attention_weight_min": -25.682775497436523,
      "activations/layer13_attention_weight_max": 39.45805358886719,
      "activations/layer13_attention_weight_min": -30.19064712524414,
      "activations/layer14_attention_weight_max": 37.496402740478516,
      "activations/layer14_attention_weight_min": -29.581621170043945,
      "activations/layer15_attention_weight_max": 35.59121322631836,
      "activations/layer15_attention_weight_min": -29.537540435791016,
      "activations/layer16_attention_weight_max": 33.694541931152344,
      "activations/layer16_attention_weight_min": -31.581148147583008,
      "activations/layer17_attention_weight_max": 53.06843948364258,
      "activations/layer17_attention_weight_min": -44.62199020385742,
      "activations/layer18_attention_weight_max": 49.89146423339844,
      "activations/layer18_attention_weight_min": -38.069679260253906,
      "activations/layer19_attention_weight_max": 22.01022720336914,
      "activations/layer19_attention_weight_min": -23.113901138305664,
      "activations/layer1_attention_weight_max": 16.261981964111328,
      "activations/layer1_attention_weight_min": -15.315827369689941,
      "activations/layer20_attention_weight_max": 23.091136932373047,
      "activations/layer20_attention_weight_min": -21.624704360961914,
      "activations/layer21_attention_weight_max": 33.62495040893555,
      "activations/layer21_attention_weight_min": -21.556640625,
      "activations/layer22_attention_weight_max": 29.10234260559082,
      "activations/layer22_attention_weight_min": -25.34465980529785,
      "activations/layer23_attention_weight_max": 37.06797409057617,
      "activations/layer23_attention_weight_min": -22.448806762695312,
      "activations/layer2_attention_weight_max": 33.8980598449707,
      "activations/layer2_attention_weight_min": -34.43882751464844,
      "activations/layer3_attention_weight_max": 94.63540649414062,
      "activations/layer3_attention_weight_min": -95.29202270507812,
      "activations/layer4_attention_weight_max": 97.87345123291016,
      "activations/layer4_attention_weight_min": -95.1628189086914,
      "activations/layer5_attention_weight_max": 70.79346466064453,
      "activations/layer5_attention_weight_min": -77.88460540771484,
      "activations/layer6_attention_weight_max": 51.3468132019043,
      "activations/layer6_attention_weight_min": -52.55449295043945,
      "activations/layer7_attention_weight_max": 69.74838256835938,
      "activations/layer7_attention_weight_min": -64.70478820800781,
      "activations/layer8_attention_weight_max": 42.924564361572266,
      "activations/layer8_attention_weight_min": -45.80610656738281,
      "activations/layer9_attention_weight_max": 45.187557220458984,
      "activations/layer9_attention_weight_min": -44.924766540527344,
      "epoch": 13.58,
      "learning_rate": 6.304810606060605e-05,
      "loss": 2.7732,
      "step": 233750
    },
    {
      "activations/layer0_attention_weight_max": 15.286311149597168,
      "activations/layer0_attention_weight_min": -14.248735427856445,
      "activations/layer10_attention_weight_max": 40.14500427246094,
      "activations/layer10_attention_weight_min": -41.997005462646484,
      "activations/layer11_attention_weight_max": 34.840484619140625,
      "activations/layer11_attention_weight_min": -38.011207580566406,
      "activations/layer12_attention_weight_max": 19.100299835205078,
      "activations/layer12_attention_weight_min": -36.52328109741211,
      "activations/layer13_attention_weight_max": 42.91706848144531,
      "activations/layer13_attention_weight_min": -27.98159408569336,
      "activations/layer14_attention_weight_max": 36.79155349731445,
      "activations/layer14_attention_weight_min": -33.006248474121094,
      "activations/layer15_attention_weight_max": 35.90977096557617,
      "activations/layer15_attention_weight_min": -32.74605941772461,
      "activations/layer16_attention_weight_max": 35.17954635620117,
      "activations/layer16_attention_weight_min": -33.84557342529297,
      "activations/layer17_attention_weight_max": 56.721527099609375,
      "activations/layer17_attention_weight_min": -45.76155090332031,
      "activations/layer18_attention_weight_max": 52.92443084716797,
      "activations/layer18_attention_weight_min": -39.62442398071289,
      "activations/layer19_attention_weight_max": 24.888362884521484,
      "activations/layer19_attention_weight_min": -23.13338279724121,
      "activations/layer1_attention_weight_max": 17.75275230407715,
      "activations/layer1_attention_weight_min": -15.113668441772461,
      "activations/layer20_attention_weight_max": 23.1328182220459,
      "activations/layer20_attention_weight_min": -21.677387237548828,
      "activations/layer21_attention_weight_max": 41.1751823425293,
      "activations/layer21_attention_weight_min": -26.704849243164062,
      "activations/layer22_attention_weight_max": 32.95621871948242,
      "activations/layer22_attention_weight_min": -27.192232131958008,
      "activations/layer23_attention_weight_max": 41.090904235839844,
      "activations/layer23_attention_weight_min": -24.700183868408203,
      "activations/layer2_attention_weight_max": 34.228790283203125,
      "activations/layer2_attention_weight_min": -32.28789520263672,
      "activations/layer3_attention_weight_max": 94.8402099609375,
      "activations/layer3_attention_weight_min": -100.76598358154297,
      "activations/layer4_attention_weight_max": 101.45882415771484,
      "activations/layer4_attention_weight_min": -101.34276580810547,
      "activations/layer5_attention_weight_max": 80.68313598632812,
      "activations/layer5_attention_weight_min": -89.62447357177734,
      "activations/layer6_attention_weight_max": 58.42140579223633,
      "activations/layer6_attention_weight_min": -59.220584869384766,
      "activations/layer7_attention_weight_max": 77.74508666992188,
      "activations/layer7_attention_weight_min": -70.33550262451172,
      "activations/layer8_attention_weight_max": 51.57609176635742,
      "activations/layer8_attention_weight_min": -53.3757209777832,
      "activations/layer9_attention_weight_max": 53.4662971496582,
      "activations/layer9_attention_weight_min": -54.45695114135742,
      "epoch": 13.59,
      "learning_rate": 6.302916666666666e-05,
      "loss": 2.7728,
      "step": 233800
    },
    {
      "activations/layer0_attention_weight_max": 15.603822708129883,
      "activations/layer0_attention_weight_min": -14.324151992797852,
      "activations/layer10_attention_weight_max": 36.184444427490234,
      "activations/layer10_attention_weight_min": -36.50800323486328,
      "activations/layer11_attention_weight_max": 34.00558853149414,
      "activations/layer11_attention_weight_min": -34.281333923339844,
      "activations/layer12_attention_weight_max": 17.26053237915039,
      "activations/layer12_attention_weight_min": -24.733699798583984,
      "activations/layer13_attention_weight_max": 37.54819869995117,
      "activations/layer13_attention_weight_min": -29.174285888671875,
      "activations/layer14_attention_weight_max": 37.30018615722656,
      "activations/layer14_attention_weight_min": -29.557313919067383,
      "activations/layer15_attention_weight_max": 32.34357452392578,
      "activations/layer15_attention_weight_min": -30.072235107421875,
      "activations/layer16_attention_weight_max": 31.668888092041016,
      "activations/layer16_attention_weight_min": -29.22166633605957,
      "activations/layer17_attention_weight_max": 54.87251663208008,
      "activations/layer17_attention_weight_min": -43.1424560546875,
      "activations/layer18_attention_weight_max": 47.1058464050293,
      "activations/layer18_attention_weight_min": -40.641666412353516,
      "activations/layer19_attention_weight_max": 23.467153549194336,
      "activations/layer19_attention_weight_min": -23.25018882751465,
      "activations/layer1_attention_weight_max": 16.945932388305664,
      "activations/layer1_attention_weight_min": -15.400856018066406,
      "activations/layer20_attention_weight_max": 23.557199478149414,
      "activations/layer20_attention_weight_min": -21.211814880371094,
      "activations/layer21_attention_weight_max": 40.80067825317383,
      "activations/layer21_attention_weight_min": -26.85702133178711,
      "activations/layer22_attention_weight_max": 31.061120986938477,
      "activations/layer22_attention_weight_min": -26.27924156188965,
      "activations/layer23_attention_weight_max": 43.79175567626953,
      "activations/layer23_attention_weight_min": -22.82904052734375,
      "activations/layer2_attention_weight_max": 32.20378112792969,
      "activations/layer2_attention_weight_min": -30.766189575195312,
      "activations/layer3_attention_weight_max": 92.9915542602539,
      "activations/layer3_attention_weight_min": -94.49958038330078,
      "activations/layer4_attention_weight_max": 97.54495239257812,
      "activations/layer4_attention_weight_min": -97.15128326416016,
      "activations/layer5_attention_weight_max": 74.51239776611328,
      "activations/layer5_attention_weight_min": -77.50147247314453,
      "activations/layer6_attention_weight_max": 51.99360656738281,
      "activations/layer6_attention_weight_min": -52.280338287353516,
      "activations/layer7_attention_weight_max": 70.66706085205078,
      "activations/layer7_attention_weight_min": -63.3431510925293,
      "activations/layer8_attention_weight_max": 45.080440521240234,
      "activations/layer8_attention_weight_min": -48.73377227783203,
      "activations/layer9_attention_weight_max": 48.618629455566406,
      "activations/layer9_attention_weight_min": -47.8079719543457,
      "epoch": 13.59,
      "learning_rate": 6.301022727272727e-05,
      "loss": 2.7514,
      "step": 233850
    },
    {
      "activations/layer0_attention_weight_max": 15.66821002960205,
      "activations/layer0_attention_weight_min": -14.029459953308105,
      "activations/layer10_attention_weight_max": 35.57374954223633,
      "activations/layer10_attention_weight_min": -37.11091995239258,
      "activations/layer11_attention_weight_max": 33.71249008178711,
      "activations/layer11_attention_weight_min": -33.781646728515625,
      "activations/layer12_attention_weight_max": 18.67417335510254,
      "activations/layer12_attention_weight_min": -25.587316513061523,
      "activations/layer13_attention_weight_max": 39.48297882080078,
      "activations/layer13_attention_weight_min": -29.57731056213379,
      "activations/layer14_attention_weight_max": 36.460792541503906,
      "activations/layer14_attention_weight_min": -30.98543930053711,
      "activations/layer15_attention_weight_max": 32.425777435302734,
      "activations/layer15_attention_weight_min": -30.566438674926758,
      "activations/layer16_attention_weight_max": 33.51660919189453,
      "activations/layer16_attention_weight_min": -30.04117774963379,
      "activations/layer17_attention_weight_max": 51.975730895996094,
      "activations/layer17_attention_weight_min": -45.82746505737305,
      "activations/layer18_attention_weight_max": 47.11788558959961,
      "activations/layer18_attention_weight_min": -39.34379196166992,
      "activations/layer19_attention_weight_max": 25.370243072509766,
      "activations/layer19_attention_weight_min": -22.92719078063965,
      "activations/layer1_attention_weight_max": 17.69190216064453,
      "activations/layer1_attention_weight_min": -15.990400314331055,
      "activations/layer20_attention_weight_max": 21.132652282714844,
      "activations/layer20_attention_weight_min": -21.264638900756836,
      "activations/layer21_attention_weight_max": 39.73388671875,
      "activations/layer21_attention_weight_min": -23.215139389038086,
      "activations/layer22_attention_weight_max": 34.523216247558594,
      "activations/layer22_attention_weight_min": -28.018173217773438,
      "activations/layer23_attention_weight_max": 36.94301223754883,
      "activations/layer23_attention_weight_min": -24.43819808959961,
      "activations/layer2_attention_weight_max": 37.82706832885742,
      "activations/layer2_attention_weight_min": -33.706642150878906,
      "activations/layer3_attention_weight_max": 98.06111145019531,
      "activations/layer3_attention_weight_min": -98.18448638916016,
      "activations/layer4_attention_weight_max": 98.86827850341797,
      "activations/layer4_attention_weight_min": -96.45586395263672,
      "activations/layer5_attention_weight_max": 72.69540405273438,
      "activations/layer5_attention_weight_min": -80.07623291015625,
      "activations/layer6_attention_weight_max": 52.55152893066406,
      "activations/layer6_attention_weight_min": -55.17689895629883,
      "activations/layer7_attention_weight_max": 67.11819458007812,
      "activations/layer7_attention_weight_min": -69.41758728027344,
      "activations/layer8_attention_weight_max": 46.904685974121094,
      "activations/layer8_attention_weight_min": -51.264556884765625,
      "activations/layer9_attention_weight_max": 49.71436309814453,
      "activations/layer9_attention_weight_min": -47.78174591064453,
      "epoch": 13.59,
      "learning_rate": 6.299128787878787e-05,
      "loss": 2.7767,
      "step": 233900
    },
    {
      "activations/layer0_attention_weight_max": 15.809359550476074,
      "activations/layer0_attention_weight_min": -14.391674995422363,
      "activations/layer10_attention_weight_max": 35.49524688720703,
      "activations/layer10_attention_weight_min": -36.4394416809082,
      "activations/layer11_attention_weight_max": 32.312103271484375,
      "activations/layer11_attention_weight_min": -33.20930480957031,
      "activations/layer12_attention_weight_max": 23.530168533325195,
      "activations/layer12_attention_weight_min": -24.145809173583984,
      "activations/layer13_attention_weight_max": 37.586280822753906,
      "activations/layer13_attention_weight_min": -28.333023071289062,
      "activations/layer14_attention_weight_max": 36.48581314086914,
      "activations/layer14_attention_weight_min": -29.518808364868164,
      "activations/layer15_attention_weight_max": 33.228458404541016,
      "activations/layer15_attention_weight_min": -28.492263793945312,
      "activations/layer16_attention_weight_max": 33.06330490112305,
      "activations/layer16_attention_weight_min": -29.787208557128906,
      "activations/layer17_attention_weight_max": 54.039981842041016,
      "activations/layer17_attention_weight_min": -46.25077438354492,
      "activations/layer18_attention_weight_max": 48.367271423339844,
      "activations/layer18_attention_weight_min": -37.702884674072266,
      "activations/layer19_attention_weight_max": 22.8101863861084,
      "activations/layer19_attention_weight_min": -22.481876373291016,
      "activations/layer1_attention_weight_max": 16.07451820373535,
      "activations/layer1_attention_weight_min": -15.37972640991211,
      "activations/layer20_attention_weight_max": 25.738508224487305,
      "activations/layer20_attention_weight_min": -21.60334014892578,
      "activations/layer21_attention_weight_max": 41.25953674316406,
      "activations/layer21_attention_weight_min": -23.18590545654297,
      "activations/layer22_attention_weight_max": 32.624473571777344,
      "activations/layer22_attention_weight_min": -25.328828811645508,
      "activations/layer23_attention_weight_max": 43.549110412597656,
      "activations/layer23_attention_weight_min": -26.155040740966797,
      "activations/layer2_attention_weight_max": 31.768735885620117,
      "activations/layer2_attention_weight_min": -31.496191024780273,
      "activations/layer3_attention_weight_max": 95.93782043457031,
      "activations/layer3_attention_weight_min": -100.79856872558594,
      "activations/layer4_attention_weight_max": 101.45331573486328,
      "activations/layer4_attention_weight_min": -97.83489990234375,
      "activations/layer5_attention_weight_max": 74.06488037109375,
      "activations/layer5_attention_weight_min": -84.30921173095703,
      "activations/layer6_attention_weight_max": 53.418663024902344,
      "activations/layer6_attention_weight_min": -55.641624450683594,
      "activations/layer7_attention_weight_max": 71.36443328857422,
      "activations/layer7_attention_weight_min": -67.5069351196289,
      "activations/layer8_attention_weight_max": 47.0660514831543,
      "activations/layer8_attention_weight_min": -47.89725875854492,
      "activations/layer9_attention_weight_max": 47.049652099609375,
      "activations/layer9_attention_weight_min": -48.551795959472656,
      "epoch": 13.59,
      "learning_rate": 6.297234848484847e-05,
      "loss": 2.765,
      "step": 233950
    },
    {
      "activations/layer0_attention_weight_max": 17.034353256225586,
      "activations/layer0_attention_weight_min": -13.956473350524902,
      "activations/layer10_attention_weight_max": 35.848907470703125,
      "activations/layer10_attention_weight_min": -35.01726531982422,
      "activations/layer11_attention_weight_max": 34.132930755615234,
      "activations/layer11_attention_weight_min": -33.7291259765625,
      "activations/layer12_attention_weight_max": 18.30060386657715,
      "activations/layer12_attention_weight_min": -26.552478790283203,
      "activations/layer13_attention_weight_max": 39.40304183959961,
      "activations/layer13_attention_weight_min": -32.321510314941406,
      "activations/layer14_attention_weight_max": 38.31992721557617,
      "activations/layer14_attention_weight_min": -30.65324592590332,
      "activations/layer15_attention_weight_max": 33.71273422241211,
      "activations/layer15_attention_weight_min": -30.1054744720459,
      "activations/layer16_attention_weight_max": 32.79218673706055,
      "activations/layer16_attention_weight_min": -31.715991973876953,
      "activations/layer17_attention_weight_max": 53.0188102722168,
      "activations/layer17_attention_weight_min": -46.76753616333008,
      "activations/layer18_attention_weight_max": 48.09096145629883,
      "activations/layer18_attention_weight_min": -38.9791259765625,
      "activations/layer19_attention_weight_max": 22.04074478149414,
      "activations/layer19_attention_weight_min": -21.36234474182129,
      "activations/layer1_attention_weight_max": 15.862322807312012,
      "activations/layer1_attention_weight_min": -14.464433670043945,
      "activations/layer20_attention_weight_max": 21.6712589263916,
      "activations/layer20_attention_weight_min": -20.74425506591797,
      "activations/layer21_attention_weight_max": 34.157196044921875,
      "activations/layer21_attention_weight_min": -22.127580642700195,
      "activations/layer22_attention_weight_max": 29.17997932434082,
      "activations/layer22_attention_weight_min": -25.3181209564209,
      "activations/layer23_attention_weight_max": 37.57803726196289,
      "activations/layer23_attention_weight_min": -23.858997344970703,
      "activations/layer2_attention_weight_max": 31.419645309448242,
      "activations/layer2_attention_weight_min": -30.362823486328125,
      "activations/layer3_attention_weight_max": 99.3357925415039,
      "activations/layer3_attention_weight_min": -98.59605407714844,
      "activations/layer4_attention_weight_max": 102.71663665771484,
      "activations/layer4_attention_weight_min": -93.58548736572266,
      "activations/layer5_attention_weight_max": 72.70116424560547,
      "activations/layer5_attention_weight_min": -77.95470428466797,
      "activations/layer6_attention_weight_max": 53.11615753173828,
      "activations/layer6_attention_weight_min": -51.22783279418945,
      "activations/layer7_attention_weight_max": 67.19670104980469,
      "activations/layer7_attention_weight_min": -62.78079605102539,
      "activations/layer8_attention_weight_max": 44.37275695800781,
      "activations/layer8_attention_weight_min": -44.97765350341797,
      "activations/layer9_attention_weight_max": 49.54751205444336,
      "activations/layer9_attention_weight_min": -43.324668884277344,
      "epoch": 13.6,
      "learning_rate": 6.295340909090908e-05,
      "loss": 2.7694,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_loss": 2.71875,
      "eval_runtime": 8.5167,
      "eval_samples_per_second": 504.188,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.5167,
      "eval_openwebtext_samples_per_second": 504.188,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9966,
      "eval_wikitext_samples_per_second": 228.389,
      "step": 234000
    },
    {
      "epoch": 13.6,
      "eval_lambada_loss": 2.572265625,
      "eval_lambada_ppl": 13.095460258581369,
      "eval_lambada_runtime": 9.5808,
      "eval_lambada_samples_per_second": 508.204,
      "step": 234000
    },
    {
      "activations/layer0_attention_weight_max": 15.965497970581055,
      "activations/layer0_attention_weight_min": -14.472771644592285,
      "activations/layer10_attention_weight_max": 36.630306243896484,
      "activations/layer10_attention_weight_min": -35.80549240112305,
      "activations/layer11_attention_weight_max": 33.710227966308594,
      "activations/layer11_attention_weight_min": -35.13602828979492,
      "activations/layer12_attention_weight_max": 20.021240234375,
      "activations/layer12_attention_weight_min": -26.68708038330078,
      "activations/layer13_attention_weight_max": 35.247520446777344,
      "activations/layer13_attention_weight_min": -26.9672794342041,
      "activations/layer14_attention_weight_max": 35.42449188232422,
      "activations/layer14_attention_weight_min": -30.578292846679688,
      "activations/layer15_attention_weight_max": 32.27847671508789,
      "activations/layer15_attention_weight_min": -31.186176300048828,
      "activations/layer16_attention_weight_max": 32.66460037231445,
      "activations/layer16_attention_weight_min": -31.427322387695312,
      "activations/layer17_attention_weight_max": 52.51304626464844,
      "activations/layer17_attention_weight_min": -45.166351318359375,
      "activations/layer18_attention_weight_max": 50.851924896240234,
      "activations/layer18_attention_weight_min": -40.754295349121094,
      "activations/layer19_attention_weight_max": 21.43931770324707,
      "activations/layer19_attention_weight_min": -21.926883697509766,
      "activations/layer1_attention_weight_max": 17.14497184753418,
      "activations/layer1_attention_weight_min": -15.102209091186523,
      "activations/layer20_attention_weight_max": 20.66039276123047,
      "activations/layer20_attention_weight_min": -22.657344818115234,
      "activations/layer21_attention_weight_max": 35.36566925048828,
      "activations/layer21_attention_weight_min": -24.467710494995117,
      "activations/layer22_attention_weight_max": 30.648483276367188,
      "activations/layer22_attention_weight_min": -27.274723052978516,
      "activations/layer23_attention_weight_max": 36.45021057128906,
      "activations/layer23_attention_weight_min": -24.853635787963867,
      "activations/layer2_attention_weight_max": 32.66907501220703,
      "activations/layer2_attention_weight_min": -31.594409942626953,
      "activations/layer3_attention_weight_max": 96.50304412841797,
      "activations/layer3_attention_weight_min": -97.65017700195312,
      "activations/layer4_attention_weight_max": 98.98059844970703,
      "activations/layer4_attention_weight_min": -95.0446548461914,
      "activations/layer5_attention_weight_max": 73.08068084716797,
      "activations/layer5_attention_weight_min": -85.51336669921875,
      "activations/layer6_attention_weight_max": 57.53594207763672,
      "activations/layer6_attention_weight_min": -52.14582824707031,
      "activations/layer7_attention_weight_max": 71.63815307617188,
      "activations/layer7_attention_weight_min": -65.6838150024414,
      "activations/layer8_attention_weight_max": 49.00363540649414,
      "activations/layer8_attention_weight_min": -52.30508041381836,
      "activations/layer9_attention_weight_max": 46.93791198730469,
      "activations/layer9_attention_weight_min": -47.16345977783203,
      "epoch": 13.6,
      "learning_rate": 6.293446969696969e-05,
      "loss": 2.7696,
      "step": 234050
    },
    {
      "activations/layer0_attention_weight_max": 16.043188095092773,
      "activations/layer0_attention_weight_min": -14.671363830566406,
      "activations/layer10_attention_weight_max": 34.682716369628906,
      "activations/layer10_attention_weight_min": -35.25209045410156,
      "activations/layer11_attention_weight_max": 29.272443771362305,
      "activations/layer11_attention_weight_min": -31.690427780151367,
      "activations/layer12_attention_weight_max": 18.729543685913086,
      "activations/layer12_attention_weight_min": -24.3622989654541,
      "activations/layer13_attention_weight_max": 33.59674835205078,
      "activations/layer13_attention_weight_min": -25.77204132080078,
      "activations/layer14_attention_weight_max": 34.860137939453125,
      "activations/layer14_attention_weight_min": -28.083412170410156,
      "activations/layer15_attention_weight_max": 32.4583625793457,
      "activations/layer15_attention_weight_min": -27.408830642700195,
      "activations/layer16_attention_weight_max": 32.213741302490234,
      "activations/layer16_attention_weight_min": -29.14531135559082,
      "activations/layer17_attention_weight_max": 52.351531982421875,
      "activations/layer17_attention_weight_min": -45.600059509277344,
      "activations/layer18_attention_weight_max": 44.00997543334961,
      "activations/layer18_attention_weight_min": -40.45018768310547,
      "activations/layer19_attention_weight_max": 23.23109245300293,
      "activations/layer19_attention_weight_min": -24.54434585571289,
      "activations/layer1_attention_weight_max": 16.717418670654297,
      "activations/layer1_attention_weight_min": -14.822846412658691,
      "activations/layer20_attention_weight_max": 24.59971809387207,
      "activations/layer20_attention_weight_min": -22.525251388549805,
      "activations/layer21_attention_weight_max": 37.9803352355957,
      "activations/layer21_attention_weight_min": -20.066680908203125,
      "activations/layer22_attention_weight_max": 29.434751510620117,
      "activations/layer22_attention_weight_min": -25.63367462158203,
      "activations/layer23_attention_weight_max": 38.20320129394531,
      "activations/layer23_attention_weight_min": -23.813980102539062,
      "activations/layer2_attention_weight_max": 31.752864837646484,
      "activations/layer2_attention_weight_min": -30.432966232299805,
      "activations/layer3_attention_weight_max": 92.86071014404297,
      "activations/layer3_attention_weight_min": -91.59346008300781,
      "activations/layer4_attention_weight_max": 94.03858947753906,
      "activations/layer4_attention_weight_min": -93.1129150390625,
      "activations/layer5_attention_weight_max": 73.6531982421875,
      "activations/layer5_attention_weight_min": -77.808349609375,
      "activations/layer6_attention_weight_max": 52.33473205566406,
      "activations/layer6_attention_weight_min": -52.24843215942383,
      "activations/layer7_attention_weight_max": 66.7189712524414,
      "activations/layer7_attention_weight_min": -64.02440643310547,
      "activations/layer8_attention_weight_max": 45.48301315307617,
      "activations/layer8_attention_weight_min": -47.70517349243164,
      "activations/layer9_attention_weight_max": 47.15761947631836,
      "activations/layer9_attention_weight_min": -45.49327087402344,
      "epoch": 13.6,
      "learning_rate": 6.291553030303029e-05,
      "loss": 2.7333,
      "step": 234100
    },
    {
      "activations/layer0_attention_weight_max": 15.814204216003418,
      "activations/layer0_attention_weight_min": -14.412943840026855,
      "activations/layer10_attention_weight_max": 40.666778564453125,
      "activations/layer10_attention_weight_min": -40.48529052734375,
      "activations/layer11_attention_weight_max": 38.60674285888672,
      "activations/layer11_attention_weight_min": -36.817535400390625,
      "activations/layer12_attention_weight_max": 18.881290435791016,
      "activations/layer12_attention_weight_min": -34.34080123901367,
      "activations/layer13_attention_weight_max": 41.16139221191406,
      "activations/layer13_attention_weight_min": -31.57547950744629,
      "activations/layer14_attention_weight_max": 39.08053970336914,
      "activations/layer14_attention_weight_min": -30.04352569580078,
      "activations/layer15_attention_weight_max": 41.641929626464844,
      "activations/layer15_attention_weight_min": -29.48377227783203,
      "activations/layer16_attention_weight_max": 31.338512420654297,
      "activations/layer16_attention_weight_min": -28.146894454956055,
      "activations/layer17_attention_weight_max": 51.853736877441406,
      "activations/layer17_attention_weight_min": -41.698246002197266,
      "activations/layer18_attention_weight_max": 43.34396743774414,
      "activations/layer18_attention_weight_min": -36.86867141723633,
      "activations/layer19_attention_weight_max": 20.2559871673584,
      "activations/layer19_attention_weight_min": -22.450368881225586,
      "activations/layer1_attention_weight_max": 18.46352195739746,
      "activations/layer1_attention_weight_min": -15.226024627685547,
      "activations/layer20_attention_weight_max": 20.465959548950195,
      "activations/layer20_attention_weight_min": -21.589616775512695,
      "activations/layer21_attention_weight_max": 33.064361572265625,
      "activations/layer21_attention_weight_min": -26.91244888305664,
      "activations/layer22_attention_weight_max": 27.11004638671875,
      "activations/layer22_attention_weight_min": -23.72231674194336,
      "activations/layer23_attention_weight_max": 34.554901123046875,
      "activations/layer23_attention_weight_min": -25.46490478515625,
      "activations/layer2_attention_weight_max": 37.3270378112793,
      "activations/layer2_attention_weight_min": -34.037261962890625,
      "activations/layer3_attention_weight_max": 95.6234130859375,
      "activations/layer3_attention_weight_min": -100.9554672241211,
      "activations/layer4_attention_weight_max": 94.2414779663086,
      "activations/layer4_attention_weight_min": -96.79432678222656,
      "activations/layer5_attention_weight_max": 71.99577331542969,
      "activations/layer5_attention_weight_min": -76.53435516357422,
      "activations/layer6_attention_weight_max": 51.52188491821289,
      "activations/layer6_attention_weight_min": -49.46487045288086,
      "activations/layer7_attention_weight_max": 80.03450775146484,
      "activations/layer7_attention_weight_min": -65.3740463256836,
      "activations/layer8_attention_weight_max": 54.912574768066406,
      "activations/layer8_attention_weight_min": -48.85014724731445,
      "activations/layer9_attention_weight_max": 64.54801940917969,
      "activations/layer9_attention_weight_min": -52.488739013671875,
      "epoch": 13.61,
      "learning_rate": 6.289659090909091e-05,
      "loss": 2.7578,
      "step": 234150
    },
    {
      "activations/layer0_attention_weight_max": 16.340648651123047,
      "activations/layer0_attention_weight_min": -14.730889320373535,
      "activations/layer10_attention_weight_max": 37.731834411621094,
      "activations/layer10_attention_weight_min": -35.875526428222656,
      "activations/layer11_attention_weight_max": 32.46388626098633,
      "activations/layer11_attention_weight_min": -33.23112106323242,
      "activations/layer12_attention_weight_max": 18.880216598510742,
      "activations/layer12_attention_weight_min": -29.14596176147461,
      "activations/layer13_attention_weight_max": 30.953920364379883,
      "activations/layer13_attention_weight_min": -26.179283142089844,
      "activations/layer14_attention_weight_max": 32.417198181152344,
      "activations/layer14_attention_weight_min": -28.878713607788086,
      "activations/layer15_attention_weight_max": 31.573284149169922,
      "activations/layer15_attention_weight_min": -29.562639236450195,
      "activations/layer16_attention_weight_max": 30.92987060546875,
      "activations/layer16_attention_weight_min": -30.54979133605957,
      "activations/layer17_attention_weight_max": 53.2852668762207,
      "activations/layer17_attention_weight_min": -46.96122360229492,
      "activations/layer18_attention_weight_max": 42.54294967651367,
      "activations/layer18_attention_weight_min": -38.27238082885742,
      "activations/layer19_attention_weight_max": 20.302095413208008,
      "activations/layer19_attention_weight_min": -20.52758026123047,
      "activations/layer1_attention_weight_max": 17.925630569458008,
      "activations/layer1_attention_weight_min": -16.039501190185547,
      "activations/layer20_attention_weight_max": 22.037288665771484,
      "activations/layer20_attention_weight_min": -20.97331428527832,
      "activations/layer21_attention_weight_max": 31.34806251525879,
      "activations/layer21_attention_weight_min": -22.794572830200195,
      "activations/layer22_attention_weight_max": 31.00441551208496,
      "activations/layer22_attention_weight_min": -24.32250213623047,
      "activations/layer23_attention_weight_max": 37.763648986816406,
      "activations/layer23_attention_weight_min": -23.955631256103516,
      "activations/layer2_attention_weight_max": 32.83885192871094,
      "activations/layer2_attention_weight_min": -31.039825439453125,
      "activations/layer3_attention_weight_max": 99.08213806152344,
      "activations/layer3_attention_weight_min": -92.46812438964844,
      "activations/layer4_attention_weight_max": 103.35711669921875,
      "activations/layer4_attention_weight_min": -96.71189880371094,
      "activations/layer5_attention_weight_max": 73.12368774414062,
      "activations/layer5_attention_weight_min": -75.78191375732422,
      "activations/layer6_attention_weight_max": 52.57056427001953,
      "activations/layer6_attention_weight_min": -53.31452941894531,
      "activations/layer7_attention_weight_max": 65.99290466308594,
      "activations/layer7_attention_weight_min": -68.38654327392578,
      "activations/layer8_attention_weight_max": 47.272640228271484,
      "activations/layer8_attention_weight_min": -47.86899185180664,
      "activations/layer9_attention_weight_max": 46.710540771484375,
      "activations/layer9_attention_weight_min": -44.490020751953125,
      "epoch": 13.61,
      "learning_rate": 6.287765151515151e-05,
      "loss": 2.775,
      "step": 234200
    },
    {
      "activations/layer0_attention_weight_max": 16.689783096313477,
      "activations/layer0_attention_weight_min": -14.262242317199707,
      "activations/layer10_attention_weight_max": 38.47395706176758,
      "activations/layer10_attention_weight_min": -38.39501953125,
      "activations/layer11_attention_weight_max": 35.77366638183594,
      "activations/layer11_attention_weight_min": -36.510597229003906,
      "activations/layer12_attention_weight_max": 22.135873794555664,
      "activations/layer12_attention_weight_min": -27.008140563964844,
      "activations/layer13_attention_weight_max": 43.6535530090332,
      "activations/layer13_attention_weight_min": -32.320640563964844,
      "activations/layer14_attention_weight_max": 38.764102935791016,
      "activations/layer14_attention_weight_min": -32.54951095581055,
      "activations/layer15_attention_weight_max": 35.25176239013672,
      "activations/layer15_attention_weight_min": -30.218917846679688,
      "activations/layer16_attention_weight_max": 34.920108795166016,
      "activations/layer16_attention_weight_min": -32.96721267700195,
      "activations/layer17_attention_weight_max": 53.819034576416016,
      "activations/layer17_attention_weight_min": -48.28276443481445,
      "activations/layer18_attention_weight_max": 55.449989318847656,
      "activations/layer18_attention_weight_min": -41.74433517456055,
      "activations/layer19_attention_weight_max": 23.43681526184082,
      "activations/layer19_attention_weight_min": -23.53723907470703,
      "activations/layer1_attention_weight_max": 15.95721435546875,
      "activations/layer1_attention_weight_min": -15.903672218322754,
      "activations/layer20_attention_weight_max": 23.419296264648438,
      "activations/layer20_attention_weight_min": -23.13146209716797,
      "activations/layer21_attention_weight_max": 41.89113235473633,
      "activations/layer21_attention_weight_min": -23.34800910949707,
      "activations/layer22_attention_weight_max": 34.51545333862305,
      "activations/layer22_attention_weight_min": -27.30854034423828,
      "activations/layer23_attention_weight_max": 42.057464599609375,
      "activations/layer23_attention_weight_min": -27.66280174255371,
      "activations/layer2_attention_weight_max": 34.49465560913086,
      "activations/layer2_attention_weight_min": -31.54461669921875,
      "activations/layer3_attention_weight_max": 96.44631958007812,
      "activations/layer3_attention_weight_min": -94.78592681884766,
      "activations/layer4_attention_weight_max": 96.16925048828125,
      "activations/layer4_attention_weight_min": -95.12458801269531,
      "activations/layer5_attention_weight_max": 76.61436462402344,
      "activations/layer5_attention_weight_min": -77.56207275390625,
      "activations/layer6_attention_weight_max": 51.32280731201172,
      "activations/layer6_attention_weight_min": -49.99137878417969,
      "activations/layer7_attention_weight_max": 69.27336883544922,
      "activations/layer7_attention_weight_min": -65.47354125976562,
      "activations/layer8_attention_weight_max": 44.47599792480469,
      "activations/layer8_attention_weight_min": -48.113712310791016,
      "activations/layer9_attention_weight_max": 51.55660629272461,
      "activations/layer9_attention_weight_min": -50.64805603027344,
      "epoch": 13.61,
      "learning_rate": 6.285871212121212e-05,
      "loss": 2.7702,
      "step": 234250
    },
    {
      "activations/layer0_attention_weight_max": 15.359392166137695,
      "activations/layer0_attention_weight_min": -14.452845573425293,
      "activations/layer10_attention_weight_max": 35.509185791015625,
      "activations/layer10_attention_weight_min": -37.2137336730957,
      "activations/layer11_attention_weight_max": 31.93240737915039,
      "activations/layer11_attention_weight_min": -32.20912551879883,
      "activations/layer12_attention_weight_max": 20.1046142578125,
      "activations/layer12_attention_weight_min": -29.18992042541504,
      "activations/layer13_attention_weight_max": 36.744686126708984,
      "activations/layer13_attention_weight_min": -26.76534080505371,
      "activations/layer14_attention_weight_max": 34.105018615722656,
      "activations/layer14_attention_weight_min": -28.66391372680664,
      "activations/layer15_attention_weight_max": 31.90093421936035,
      "activations/layer15_attention_weight_min": -29.12386703491211,
      "activations/layer16_attention_weight_max": 30.804019927978516,
      "activations/layer16_attention_weight_min": -30.04716682434082,
      "activations/layer17_attention_weight_max": 49.49205017089844,
      "activations/layer17_attention_weight_min": -43.472225189208984,
      "activations/layer18_attention_weight_max": 48.4361686706543,
      "activations/layer18_attention_weight_min": -36.80356979370117,
      "activations/layer19_attention_weight_max": 23.02950096130371,
      "activations/layer19_attention_weight_min": -21.669490814208984,
      "activations/layer1_attention_weight_max": 16.87778663635254,
      "activations/layer1_attention_weight_min": -15.003487586975098,
      "activations/layer20_attention_weight_max": 23.525543212890625,
      "activations/layer20_attention_weight_min": -22.758586883544922,
      "activations/layer21_attention_weight_max": 40.679847717285156,
      "activations/layer21_attention_weight_min": -20.9906005859375,
      "activations/layer22_attention_weight_max": 31.16982078552246,
      "activations/layer22_attention_weight_min": -24.75101089477539,
      "activations/layer23_attention_weight_max": 40.954166412353516,
      "activations/layer23_attention_weight_min": -22.847415924072266,
      "activations/layer2_attention_weight_max": 32.475669860839844,
      "activations/layer2_attention_weight_min": -32.51603698730469,
      "activations/layer3_attention_weight_max": 92.29595184326172,
      "activations/layer3_attention_weight_min": -91.75093841552734,
      "activations/layer4_attention_weight_max": 92.30870056152344,
      "activations/layer4_attention_weight_min": -91.65370178222656,
      "activations/layer5_attention_weight_max": 71.11094665527344,
      "activations/layer5_attention_weight_min": -78.99676513671875,
      "activations/layer6_attention_weight_max": 48.360572814941406,
      "activations/layer6_attention_weight_min": -50.098819732666016,
      "activations/layer7_attention_weight_max": 64.20975494384766,
      "activations/layer7_attention_weight_min": -64.9438705444336,
      "activations/layer8_attention_weight_max": 43.714481353759766,
      "activations/layer8_attention_weight_min": -43.82657241821289,
      "activations/layer9_attention_weight_max": 50.4542350769043,
      "activations/layer9_attention_weight_min": -42.849918365478516,
      "epoch": 13.61,
      "learning_rate": 6.283977272727273e-05,
      "loss": 2.7666,
      "step": 234300
    },
    {
      "activations/layer0_attention_weight_max": 16.46140480041504,
      "activations/layer0_attention_weight_min": -14.254216194152832,
      "activations/layer10_attention_weight_max": 35.71856689453125,
      "activations/layer10_attention_weight_min": -36.376197814941406,
      "activations/layer11_attention_weight_max": 31.692699432373047,
      "activations/layer11_attention_weight_min": -32.734832763671875,
      "activations/layer12_attention_weight_max": 21.147111892700195,
      "activations/layer12_attention_weight_min": -23.06869888305664,
      "activations/layer13_attention_weight_max": 39.43514633178711,
      "activations/layer13_attention_weight_min": -30.314516067504883,
      "activations/layer14_attention_weight_max": 36.547828674316406,
      "activations/layer14_attention_weight_min": -29.44011116027832,
      "activations/layer15_attention_weight_max": 34.45400619506836,
      "activations/layer15_attention_weight_min": -29.871889114379883,
      "activations/layer16_attention_weight_max": 31.729061126708984,
      "activations/layer16_attention_weight_min": -30.588932037353516,
      "activations/layer17_attention_weight_max": 53.468326568603516,
      "activations/layer17_attention_weight_min": -46.07862854003906,
      "activations/layer18_attention_weight_max": 47.37568283081055,
      "activations/layer18_attention_weight_min": -38.87392807006836,
      "activations/layer19_attention_weight_max": 23.467130661010742,
      "activations/layer19_attention_weight_min": -22.18394660949707,
      "activations/layer1_attention_weight_max": 17.255619049072266,
      "activations/layer1_attention_weight_min": -16.453453063964844,
      "activations/layer20_attention_weight_max": 23.932092666625977,
      "activations/layer20_attention_weight_min": -20.966472625732422,
      "activations/layer21_attention_weight_max": 39.44267272949219,
      "activations/layer21_attention_weight_min": -21.3797607421875,
      "activations/layer22_attention_weight_max": 33.75537872314453,
      "activations/layer22_attention_weight_min": -25.291576385498047,
      "activations/layer23_attention_weight_max": 37.319766998291016,
      "activations/layer23_attention_weight_min": -24.998878479003906,
      "activations/layer2_attention_weight_max": 32.67153549194336,
      "activations/layer2_attention_weight_min": -31.261398315429688,
      "activations/layer3_attention_weight_max": 93.80702209472656,
      "activations/layer3_attention_weight_min": -94.99600982666016,
      "activations/layer4_attention_weight_max": 98.6860122680664,
      "activations/layer4_attention_weight_min": -93.27151489257812,
      "activations/layer5_attention_weight_max": 72.4434585571289,
      "activations/layer5_attention_weight_min": -76.90754699707031,
      "activations/layer6_attention_weight_max": 52.315147399902344,
      "activations/layer6_attention_weight_min": -50.90870666503906,
      "activations/layer7_attention_weight_max": 66.53388977050781,
      "activations/layer7_attention_weight_min": -65.36708068847656,
      "activations/layer8_attention_weight_max": 46.23824691772461,
      "activations/layer8_attention_weight_min": -50.55143737792969,
      "activations/layer9_attention_weight_max": 45.06791305541992,
      "activations/layer9_attention_weight_min": -46.793975830078125,
      "epoch": 13.62,
      "learning_rate": 6.282083333333333e-05,
      "loss": 2.763,
      "step": 234350
    },
    {
      "activations/layer0_attention_weight_max": 15.841374397277832,
      "activations/layer0_attention_weight_min": -13.853043556213379,
      "activations/layer10_attention_weight_max": 35.35322570800781,
      "activations/layer10_attention_weight_min": -35.64361572265625,
      "activations/layer11_attention_weight_max": 33.64964294433594,
      "activations/layer11_attention_weight_min": -33.913291931152344,
      "activations/layer12_attention_weight_max": 20.248220443725586,
      "activations/layer12_attention_weight_min": -24.43232536315918,
      "activations/layer13_attention_weight_max": 36.11595916748047,
      "activations/layer13_attention_weight_min": -26.680517196655273,
      "activations/layer14_attention_weight_max": 34.399166107177734,
      "activations/layer14_attention_weight_min": -29.464065551757812,
      "activations/layer15_attention_weight_max": 35.0495719909668,
      "activations/layer15_attention_weight_min": -29.348968505859375,
      "activations/layer16_attention_weight_max": 32.993980407714844,
      "activations/layer16_attention_weight_min": -31.922664642333984,
      "activations/layer17_attention_weight_max": 50.60150146484375,
      "activations/layer17_attention_weight_min": -44.96493148803711,
      "activations/layer18_attention_weight_max": 47.460086822509766,
      "activations/layer18_attention_weight_min": -41.19401931762695,
      "activations/layer19_attention_weight_max": 22.003646850585938,
      "activations/layer19_attention_weight_min": -21.89366340637207,
      "activations/layer1_attention_weight_max": 16.94440460205078,
      "activations/layer1_attention_weight_min": -15.695035934448242,
      "activations/layer20_attention_weight_max": 23.16387367248535,
      "activations/layer20_attention_weight_min": -22.751998901367188,
      "activations/layer21_attention_weight_max": 48.131072998046875,
      "activations/layer21_attention_weight_min": -27.006132125854492,
      "activations/layer22_attention_weight_max": 33.87496566772461,
      "activations/layer22_attention_weight_min": -26.17633628845215,
      "activations/layer23_attention_weight_max": 38.43757629394531,
      "activations/layer23_attention_weight_min": -21.651641845703125,
      "activations/layer2_attention_weight_max": 33.38671875,
      "activations/layer2_attention_weight_min": -31.181964874267578,
      "activations/layer3_attention_weight_max": 95.74483489990234,
      "activations/layer3_attention_weight_min": -93.64867401123047,
      "activations/layer4_attention_weight_max": 100.19287872314453,
      "activations/layer4_attention_weight_min": -97.6330795288086,
      "activations/layer5_attention_weight_max": 72.31100463867188,
      "activations/layer5_attention_weight_min": -79.3577880859375,
      "activations/layer6_attention_weight_max": 52.6325798034668,
      "activations/layer6_attention_weight_min": -55.829349517822266,
      "activations/layer7_attention_weight_max": 68.72623443603516,
      "activations/layer7_attention_weight_min": -68.46456146240234,
      "activations/layer8_attention_weight_max": 44.55354309082031,
      "activations/layer8_attention_weight_min": -48.00215530395508,
      "activations/layer9_attention_weight_max": 47.8167724609375,
      "activations/layer9_attention_weight_min": -45.65749740600586,
      "epoch": 13.62,
      "learning_rate": 6.280189393939394e-05,
      "loss": 2.7588,
      "step": 234400
    },
    {
      "activations/layer0_attention_weight_max": 16.393238067626953,
      "activations/layer0_attention_weight_min": -13.964343070983887,
      "activations/layer10_attention_weight_max": 34.7186279296875,
      "activations/layer10_attention_weight_min": -34.130619049072266,
      "activations/layer11_attention_weight_max": 32.583099365234375,
      "activations/layer11_attention_weight_min": -32.35072326660156,
      "activations/layer12_attention_weight_max": 18.04305076599121,
      "activations/layer12_attention_weight_min": -22.450096130371094,
      "activations/layer13_attention_weight_max": 33.01627731323242,
      "activations/layer13_attention_weight_min": -27.8819522857666,
      "activations/layer14_attention_weight_max": 34.77356719970703,
      "activations/layer14_attention_weight_min": -30.359270095825195,
      "activations/layer15_attention_weight_max": 31.833065032958984,
      "activations/layer15_attention_weight_min": -30.205114364624023,
      "activations/layer16_attention_weight_max": 30.307796478271484,
      "activations/layer16_attention_weight_min": -28.795835494995117,
      "activations/layer17_attention_weight_max": 52.93840026855469,
      "activations/layer17_attention_weight_min": -43.628902435302734,
      "activations/layer18_attention_weight_max": 44.17460250854492,
      "activations/layer18_attention_weight_min": -40.67853546142578,
      "activations/layer19_attention_weight_max": 22.07361602783203,
      "activations/layer19_attention_weight_min": -22.9793643951416,
      "activations/layer1_attention_weight_max": 17.390565872192383,
      "activations/layer1_attention_weight_min": -15.0086088180542,
      "activations/layer20_attention_weight_max": 23.164579391479492,
      "activations/layer20_attention_weight_min": -23.840442657470703,
      "activations/layer21_attention_weight_max": 35.52766799926758,
      "activations/layer21_attention_weight_min": -23.944961547851562,
      "activations/layer22_attention_weight_max": 29.769466400146484,
      "activations/layer22_attention_weight_min": -27.287866592407227,
      "activations/layer23_attention_weight_max": 36.058406829833984,
      "activations/layer23_attention_weight_min": -25.1892147064209,
      "activations/layer2_attention_weight_max": 33.903785705566406,
      "activations/layer2_attention_weight_min": -33.943416595458984,
      "activations/layer3_attention_weight_max": 90.61366271972656,
      "activations/layer3_attention_weight_min": -98.28291320800781,
      "activations/layer4_attention_weight_max": 93.98397064208984,
      "activations/layer4_attention_weight_min": -93.7521743774414,
      "activations/layer5_attention_weight_max": 68.1767578125,
      "activations/layer5_attention_weight_min": -75.0450439453125,
      "activations/layer6_attention_weight_max": 51.34174346923828,
      "activations/layer6_attention_weight_min": -49.98039627075195,
      "activations/layer7_attention_weight_max": 66.72737121582031,
      "activations/layer7_attention_weight_min": -65.77592468261719,
      "activations/layer8_attention_weight_max": 43.46855926513672,
      "activations/layer8_attention_weight_min": -44.019779205322266,
      "activations/layer9_attention_weight_max": 53.33955764770508,
      "activations/layer9_attention_weight_min": -45.70903396606445,
      "epoch": 13.62,
      "learning_rate": 6.278295454545454e-05,
      "loss": 2.7782,
      "step": 234450
    },
    {
      "activations/layer0_attention_weight_max": 16.799869537353516,
      "activations/layer0_attention_weight_min": -14.533462524414062,
      "activations/layer10_attention_weight_max": 37.95137023925781,
      "activations/layer10_attention_weight_min": -36.73291778564453,
      "activations/layer11_attention_weight_max": 33.779266357421875,
      "activations/layer11_attention_weight_min": -34.57558059692383,
      "activations/layer12_attention_weight_max": 17.915422439575195,
      "activations/layer12_attention_weight_min": -23.0786075592041,
      "activations/layer13_attention_weight_max": 39.305946350097656,
      "activations/layer13_attention_weight_min": -30.136112213134766,
      "activations/layer14_attention_weight_max": 41.19380187988281,
      "activations/layer14_attention_weight_min": -33.32703399658203,
      "activations/layer15_attention_weight_max": 38.2940673828125,
      "activations/layer15_attention_weight_min": -31.53388214111328,
      "activations/layer16_attention_weight_max": 37.61378479003906,
      "activations/layer16_attention_weight_min": -34.22102737426758,
      "activations/layer17_attention_weight_max": 59.12434387207031,
      "activations/layer17_attention_weight_min": -49.06818771362305,
      "activations/layer18_attention_weight_max": 52.573612213134766,
      "activations/layer18_attention_weight_min": -41.47148895263672,
      "activations/layer19_attention_weight_max": 27.019323348999023,
      "activations/layer19_attention_weight_min": -22.2110538482666,
      "activations/layer1_attention_weight_max": 16.460126876831055,
      "activations/layer1_attention_weight_min": -16.545743942260742,
      "activations/layer20_attention_weight_max": 25.987327575683594,
      "activations/layer20_attention_weight_min": -20.454715728759766,
      "activations/layer21_attention_weight_max": 42.01496887207031,
      "activations/layer21_attention_weight_min": -23.659452438354492,
      "activations/layer22_attention_weight_max": 37.76620864868164,
      "activations/layer22_attention_weight_min": -24.303939819335938,
      "activations/layer23_attention_weight_max": 42.418670654296875,
      "activations/layer23_attention_weight_min": -21.726425170898438,
      "activations/layer2_attention_weight_max": 33.002132415771484,
      "activations/layer2_attention_weight_min": -31.200544357299805,
      "activations/layer3_attention_weight_max": 96.43244934082031,
      "activations/layer3_attention_weight_min": -96.49847412109375,
      "activations/layer4_attention_weight_max": 97.49918365478516,
      "activations/layer4_attention_weight_min": -94.20613098144531,
      "activations/layer5_attention_weight_max": 72.94916534423828,
      "activations/layer5_attention_weight_min": -77.17347717285156,
      "activations/layer6_attention_weight_max": 53.9026985168457,
      "activations/layer6_attention_weight_min": -55.8491325378418,
      "activations/layer7_attention_weight_max": 74.07001495361328,
      "activations/layer7_attention_weight_min": -66.27426147460938,
      "activations/layer8_attention_weight_max": 46.19270706176758,
      "activations/layer8_attention_weight_min": -49.29248046875,
      "activations/layer9_attention_weight_max": 49.13123321533203,
      "activations/layer9_attention_weight_min": -46.51922607421875,
      "epoch": 13.63,
      "learning_rate": 6.276401515151514e-05,
      "loss": 2.7604,
      "step": 234500
    },
    {
      "activations/layer0_attention_weight_max": 15.65175724029541,
      "activations/layer0_attention_weight_min": -14.552404403686523,
      "activations/layer10_attention_weight_max": 38.83928298950195,
      "activations/layer10_attention_weight_min": -39.568077087402344,
      "activations/layer11_attention_weight_max": 32.785484313964844,
      "activations/layer11_attention_weight_min": -34.064720153808594,
      "activations/layer12_attention_weight_max": 17.9594669342041,
      "activations/layer12_attention_weight_min": -22.84154510498047,
      "activations/layer13_attention_weight_max": 36.77558135986328,
      "activations/layer13_attention_weight_min": -27.435039520263672,
      "activations/layer14_attention_weight_max": 36.587093353271484,
      "activations/layer14_attention_weight_min": -29.606159210205078,
      "activations/layer15_attention_weight_max": 33.1953239440918,
      "activations/layer15_attention_weight_min": -30.330432891845703,
      "activations/layer16_attention_weight_max": 31.00882911682129,
      "activations/layer16_attention_weight_min": -30.843929290771484,
      "activations/layer17_attention_weight_max": 53.809478759765625,
      "activations/layer17_attention_weight_min": -44.22718811035156,
      "activations/layer18_attention_weight_max": 46.128780364990234,
      "activations/layer18_attention_weight_min": -41.99789810180664,
      "activations/layer19_attention_weight_max": 22.13698387145996,
      "activations/layer19_attention_weight_min": -23.294118881225586,
      "activations/layer1_attention_weight_max": 15.624397277832031,
      "activations/layer1_attention_weight_min": -15.695405960083008,
      "activations/layer20_attention_weight_max": 21.977100372314453,
      "activations/layer20_attention_weight_min": -22.2587890625,
      "activations/layer21_attention_weight_max": 39.251705169677734,
      "activations/layer21_attention_weight_min": -22.9202880859375,
      "activations/layer22_attention_weight_max": 30.54828643798828,
      "activations/layer22_attention_weight_min": -27.42424774169922,
      "activations/layer23_attention_weight_max": 37.45309066772461,
      "activations/layer23_attention_weight_min": -24.375057220458984,
      "activations/layer2_attention_weight_max": 32.62361145019531,
      "activations/layer2_attention_weight_min": -32.52952194213867,
      "activations/layer3_attention_weight_max": 100.77169799804688,
      "activations/layer3_attention_weight_min": -106.02839660644531,
      "activations/layer4_attention_weight_max": 99.41053009033203,
      "activations/layer4_attention_weight_min": -100.78560638427734,
      "activations/layer5_attention_weight_max": 70.00106048583984,
      "activations/layer5_attention_weight_min": -81.61121368408203,
      "activations/layer6_attention_weight_max": 52.8728141784668,
      "activations/layer6_attention_weight_min": -52.72013854980469,
      "activations/layer7_attention_weight_max": 66.8246078491211,
      "activations/layer7_attention_weight_min": -69.24974822998047,
      "activations/layer8_attention_weight_max": 47.47043991088867,
      "activations/layer8_attention_weight_min": -50.5097541809082,
      "activations/layer9_attention_weight_max": 50.735286712646484,
      "activations/layer9_attention_weight_min": -47.12609100341797,
      "epoch": 13.63,
      "learning_rate": 6.274507575757575e-05,
      "loss": 2.7569,
      "step": 234550
    },
    {
      "activations/layer0_attention_weight_max": 16.594541549682617,
      "activations/layer0_attention_weight_min": -14.627199172973633,
      "activations/layer10_attention_weight_max": 36.4470100402832,
      "activations/layer10_attention_weight_min": -36.546295166015625,
      "activations/layer11_attention_weight_max": 32.17717361450195,
      "activations/layer11_attention_weight_min": -32.05619812011719,
      "activations/layer12_attention_weight_max": 20.198450088500977,
      "activations/layer12_attention_weight_min": -29.127887725830078,
      "activations/layer13_attention_weight_max": 35.13364028930664,
      "activations/layer13_attention_weight_min": -27.747360229492188,
      "activations/layer14_attention_weight_max": 36.14025115966797,
      "activations/layer14_attention_weight_min": -29.084745407104492,
      "activations/layer15_attention_weight_max": 32.259857177734375,
      "activations/layer15_attention_weight_min": -29.591976165771484,
      "activations/layer16_attention_weight_max": 31.81835174560547,
      "activations/layer16_attention_weight_min": -30.78771209716797,
      "activations/layer17_attention_weight_max": 51.966148376464844,
      "activations/layer17_attention_weight_min": -44.050758361816406,
      "activations/layer18_attention_weight_max": 50.62814712524414,
      "activations/layer18_attention_weight_min": -38.250370025634766,
      "activations/layer19_attention_weight_max": 24.520275115966797,
      "activations/layer19_attention_weight_min": -22.597015380859375,
      "activations/layer1_attention_weight_max": 16.71050262451172,
      "activations/layer1_attention_weight_min": -16.919784545898438,
      "activations/layer20_attention_weight_max": 22.82208824157715,
      "activations/layer20_attention_weight_min": -21.794048309326172,
      "activations/layer21_attention_weight_max": 31.18784523010254,
      "activations/layer21_attention_weight_min": -23.4047794342041,
      "activations/layer22_attention_weight_max": 29.072723388671875,
      "activations/layer22_attention_weight_min": -26.7175235748291,
      "activations/layer23_attention_weight_max": 35.50700378417969,
      "activations/layer23_attention_weight_min": -26.888809204101562,
      "activations/layer2_attention_weight_max": 34.00147247314453,
      "activations/layer2_attention_weight_min": -31.94427490234375,
      "activations/layer3_attention_weight_max": 96.52613830566406,
      "activations/layer3_attention_weight_min": -98.93907165527344,
      "activations/layer4_attention_weight_max": 99.0317611694336,
      "activations/layer4_attention_weight_min": -96.41106414794922,
      "activations/layer5_attention_weight_max": 74.99166870117188,
      "activations/layer5_attention_weight_min": -82.82279968261719,
      "activations/layer6_attention_weight_max": 51.1453742980957,
      "activations/layer6_attention_weight_min": -54.575321197509766,
      "activations/layer7_attention_weight_max": 68.21772003173828,
      "activations/layer7_attention_weight_min": -68.5794448852539,
      "activations/layer8_attention_weight_max": 45.01633834838867,
      "activations/layer8_attention_weight_min": -48.69736862182617,
      "activations/layer9_attention_weight_max": 45.953189849853516,
      "activations/layer9_attention_weight_min": -47.264156341552734,
      "epoch": 13.63,
      "learning_rate": 6.272613636363636e-05,
      "loss": 2.7615,
      "step": 234600
    },
    {
      "activations/layer0_attention_weight_max": 16.936498641967773,
      "activations/layer0_attention_weight_min": -14.123398780822754,
      "activations/layer10_attention_weight_max": 37.75454330444336,
      "activations/layer10_attention_weight_min": -36.17091751098633,
      "activations/layer11_attention_weight_max": 32.9692268371582,
      "activations/layer11_attention_weight_min": -34.28345489501953,
      "activations/layer12_attention_weight_max": 19.396928787231445,
      "activations/layer12_attention_weight_min": -28.997760772705078,
      "activations/layer13_attention_weight_max": 54.63795471191406,
      "activations/layer13_attention_weight_min": -33.125858306884766,
      "activations/layer14_attention_weight_max": 51.844581604003906,
      "activations/layer14_attention_weight_min": -32.64149475097656,
      "activations/layer15_attention_weight_max": 48.058231353759766,
      "activations/layer15_attention_weight_min": -31.29741668701172,
      "activations/layer16_attention_weight_max": 37.39853286743164,
      "activations/layer16_attention_weight_min": -31.486934661865234,
      "activations/layer17_attention_weight_max": 60.32297897338867,
      "activations/layer17_attention_weight_min": -45.71340560913086,
      "activations/layer18_attention_weight_max": 53.152008056640625,
      "activations/layer18_attention_weight_min": -40.02507781982422,
      "activations/layer19_attention_weight_max": 27.59220314025879,
      "activations/layer19_attention_weight_min": -23.54254150390625,
      "activations/layer1_attention_weight_max": 17.040388107299805,
      "activations/layer1_attention_weight_min": -15.75275993347168,
      "activations/layer20_attention_weight_max": 26.16510009765625,
      "activations/layer20_attention_weight_min": -22.46405601501465,
      "activations/layer21_attention_weight_max": 46.746795654296875,
      "activations/layer21_attention_weight_min": -24.745433807373047,
      "activations/layer22_attention_weight_max": 31.79290008544922,
      "activations/layer22_attention_weight_min": -26.614980697631836,
      "activations/layer23_attention_weight_max": 41.498077392578125,
      "activations/layer23_attention_weight_min": -23.77368927001953,
      "activations/layer2_attention_weight_max": 33.98914337158203,
      "activations/layer2_attention_weight_min": -32.0903434753418,
      "activations/layer3_attention_weight_max": 98.53275299072266,
      "activations/layer3_attention_weight_min": -99.24506378173828,
      "activations/layer4_attention_weight_max": 100.19011688232422,
      "activations/layer4_attention_weight_min": -94.18118286132812,
      "activations/layer5_attention_weight_max": 75.45143127441406,
      "activations/layer5_attention_weight_min": -80.45487976074219,
      "activations/layer6_attention_weight_max": 52.94676208496094,
      "activations/layer6_attention_weight_min": -54.14095687866211,
      "activations/layer7_attention_weight_max": 73.66097259521484,
      "activations/layer7_attention_weight_min": -67.27591705322266,
      "activations/layer8_attention_weight_max": 47.89765548706055,
      "activations/layer8_attention_weight_min": -50.451507568359375,
      "activations/layer9_attention_weight_max": 49.555965423583984,
      "activations/layer9_attention_weight_min": -50.50202178955078,
      "epoch": 13.63,
      "learning_rate": 6.270719696969696e-05,
      "loss": 2.7673,
      "step": 234650
    },
    {
      "activations/layer0_attention_weight_max": 15.825547218322754,
      "activations/layer0_attention_weight_min": -14.289305686950684,
      "activations/layer10_attention_weight_max": 41.01523208618164,
      "activations/layer10_attention_weight_min": -37.13507843017578,
      "activations/layer11_attention_weight_max": 33.82118606567383,
      "activations/layer11_attention_weight_min": -33.59736633300781,
      "activations/layer12_attention_weight_max": 18.02120018005371,
      "activations/layer12_attention_weight_min": -27.538042068481445,
      "activations/layer13_attention_weight_max": 33.3449821472168,
      "activations/layer13_attention_weight_min": -27.369558334350586,
      "activations/layer14_attention_weight_max": 34.96057891845703,
      "activations/layer14_attention_weight_min": -30.084033966064453,
      "activations/layer15_attention_weight_max": 32.160926818847656,
      "activations/layer15_attention_weight_min": -31.100112915039062,
      "activations/layer16_attention_weight_max": 32.21904754638672,
      "activations/layer16_attention_weight_min": -31.546918869018555,
      "activations/layer17_attention_weight_max": 48.92643737792969,
      "activations/layer17_attention_weight_min": -43.031837463378906,
      "activations/layer18_attention_weight_max": 46.735321044921875,
      "activations/layer18_attention_weight_min": -36.719635009765625,
      "activations/layer19_attention_weight_max": 22.831567764282227,
      "activations/layer19_attention_weight_min": -20.346059799194336,
      "activations/layer1_attention_weight_max": 16.676767349243164,
      "activations/layer1_attention_weight_min": -14.537650108337402,
      "activations/layer20_attention_weight_max": 20.856786727905273,
      "activations/layer20_attention_weight_min": -23.42588996887207,
      "activations/layer21_attention_weight_max": 32.497711181640625,
      "activations/layer21_attention_weight_min": -22.798364639282227,
      "activations/layer22_attention_weight_max": 29.488969802856445,
      "activations/layer22_attention_weight_min": -24.086761474609375,
      "activations/layer23_attention_weight_max": 35.00638198852539,
      "activations/layer23_attention_weight_min": -23.696609497070312,
      "activations/layer2_attention_weight_max": 32.811187744140625,
      "activations/layer2_attention_weight_min": -30.789318084716797,
      "activations/layer3_attention_weight_max": 94.68879699707031,
      "activations/layer3_attention_weight_min": -96.42828369140625,
      "activations/layer4_attention_weight_max": 92.9472885131836,
      "activations/layer4_attention_weight_min": -90.59297943115234,
      "activations/layer5_attention_weight_max": 71.95552062988281,
      "activations/layer5_attention_weight_min": -77.68965148925781,
      "activations/layer6_attention_weight_max": 51.34210968017578,
      "activations/layer6_attention_weight_min": -51.46369552612305,
      "activations/layer7_attention_weight_max": 62.566715240478516,
      "activations/layer7_attention_weight_min": -65.12923431396484,
      "activations/layer8_attention_weight_max": 44.722625732421875,
      "activations/layer8_attention_weight_min": -48.00758743286133,
      "activations/layer9_attention_weight_max": 43.375762939453125,
      "activations/layer9_attention_weight_min": -46.05521011352539,
      "epoch": 13.64,
      "learning_rate": 6.268825757575756e-05,
      "loss": 2.7633,
      "step": 234700
    },
    {
      "activations/layer0_attention_weight_max": 16.12803077697754,
      "activations/layer0_attention_weight_min": -14.548225402832031,
      "activations/layer10_attention_weight_max": 34.74800491333008,
      "activations/layer10_attention_weight_min": -33.84335708618164,
      "activations/layer11_attention_weight_max": 30.577484130859375,
      "activations/layer11_attention_weight_min": -32.34614944458008,
      "activations/layer12_attention_weight_max": 17.533588409423828,
      "activations/layer12_attention_weight_min": -24.187986373901367,
      "activations/layer13_attention_weight_max": 38.27724838256836,
      "activations/layer13_attention_weight_min": -29.1999568939209,
      "activations/layer14_attention_weight_max": 32.56523895263672,
      "activations/layer14_attention_weight_min": -28.97515296936035,
      "activations/layer15_attention_weight_max": 30.287517547607422,
      "activations/layer15_attention_weight_min": -27.884077072143555,
      "activations/layer16_attention_weight_max": 30.05327606201172,
      "activations/layer16_attention_weight_min": -29.73600959777832,
      "activations/layer17_attention_weight_max": 49.820655822753906,
      "activations/layer17_attention_weight_min": -44.500450134277344,
      "activations/layer18_attention_weight_max": 43.3917121887207,
      "activations/layer18_attention_weight_min": -38.568946838378906,
      "activations/layer19_attention_weight_max": 26.074481964111328,
      "activations/layer19_attention_weight_min": -22.90489959716797,
      "activations/layer1_attention_weight_max": 16.4063777923584,
      "activations/layer1_attention_weight_min": -14.992321014404297,
      "activations/layer20_attention_weight_max": 22.429521560668945,
      "activations/layer20_attention_weight_min": -20.809717178344727,
      "activations/layer21_attention_weight_max": 36.02008056640625,
      "activations/layer21_attention_weight_min": -20.69261360168457,
      "activations/layer22_attention_weight_max": 30.947683334350586,
      "activations/layer22_attention_weight_min": -25.32062339782715,
      "activations/layer23_attention_weight_max": 35.414512634277344,
      "activations/layer23_attention_weight_min": -25.07605743408203,
      "activations/layer2_attention_weight_max": 31.512123107910156,
      "activations/layer2_attention_weight_min": -31.408292770385742,
      "activations/layer3_attention_weight_max": 93.4889907836914,
      "activations/layer3_attention_weight_min": -98.22803497314453,
      "activations/layer4_attention_weight_max": 92.59497833251953,
      "activations/layer4_attention_weight_min": -91.78968048095703,
      "activations/layer5_attention_weight_max": 74.23019409179688,
      "activations/layer5_attention_weight_min": -79.13594055175781,
      "activations/layer6_attention_weight_max": 50.53773880004883,
      "activations/layer6_attention_weight_min": -51.13786697387695,
      "activations/layer7_attention_weight_max": 64.47953796386719,
      "activations/layer7_attention_weight_min": -62.921348571777344,
      "activations/layer8_attention_weight_max": 41.25896072387695,
      "activations/layer8_attention_weight_min": -46.66261291503906,
      "activations/layer9_attention_weight_max": 49.44130325317383,
      "activations/layer9_attention_weight_min": -45.757118225097656,
      "epoch": 13.64,
      "learning_rate": 6.266931818181818e-05,
      "loss": 2.7759,
      "step": 234750
    },
    {
      "activations/layer0_attention_weight_max": 16.49560546875,
      "activations/layer0_attention_weight_min": -14.617992401123047,
      "activations/layer10_attention_weight_max": 38.025753021240234,
      "activations/layer10_attention_weight_min": -35.401206970214844,
      "activations/layer11_attention_weight_max": 33.90603256225586,
      "activations/layer11_attention_weight_min": -34.16466522216797,
      "activations/layer12_attention_weight_max": 19.926889419555664,
      "activations/layer12_attention_weight_min": -24.41529083251953,
      "activations/layer13_attention_weight_max": 38.659889221191406,
      "activations/layer13_attention_weight_min": -31.071107864379883,
      "activations/layer14_attention_weight_max": 36.29886245727539,
      "activations/layer14_attention_weight_min": -31.859106063842773,
      "activations/layer15_attention_weight_max": 34.174415588378906,
      "activations/layer15_attention_weight_min": -31.15302085876465,
      "activations/layer16_attention_weight_max": 33.95123291015625,
      "activations/layer16_attention_weight_min": -31.117815017700195,
      "activations/layer17_attention_weight_max": 54.951969146728516,
      "activations/layer17_attention_weight_min": -46.93163299560547,
      "activations/layer18_attention_weight_max": 47.80614471435547,
      "activations/layer18_attention_weight_min": -40.87908172607422,
      "activations/layer19_attention_weight_max": 25.433366775512695,
      "activations/layer19_attention_weight_min": -21.63641357421875,
      "activations/layer1_attention_weight_max": 17.109264373779297,
      "activations/layer1_attention_weight_min": -15.960214614868164,
      "activations/layer20_attention_weight_max": 21.383134841918945,
      "activations/layer20_attention_weight_min": -19.231138229370117,
      "activations/layer21_attention_weight_max": 34.55044174194336,
      "activations/layer21_attention_weight_min": -21.77471351623535,
      "activations/layer22_attention_weight_max": 30.57338523864746,
      "activations/layer22_attention_weight_min": -26.016448974609375,
      "activations/layer23_attention_weight_max": 36.62036895751953,
      "activations/layer23_attention_weight_min": -25.916282653808594,
      "activations/layer2_attention_weight_max": 32.80849838256836,
      "activations/layer2_attention_weight_min": -30.846921920776367,
      "activations/layer3_attention_weight_max": 98.86331176757812,
      "activations/layer3_attention_weight_min": -96.01349639892578,
      "activations/layer4_attention_weight_max": 97.8823471069336,
      "activations/layer4_attention_weight_min": -94.3030014038086,
      "activations/layer5_attention_weight_max": 72.19363403320312,
      "activations/layer5_attention_weight_min": -77.0844955444336,
      "activations/layer6_attention_weight_max": 54.68624496459961,
      "activations/layer6_attention_weight_min": -51.5004997253418,
      "activations/layer7_attention_weight_max": 69.03793334960938,
      "activations/layer7_attention_weight_min": -67.825927734375,
      "activations/layer8_attention_weight_max": 46.78128433227539,
      "activations/layer8_attention_weight_min": -48.85962677001953,
      "activations/layer9_attention_weight_max": 46.34326934814453,
      "activations/layer9_attention_weight_min": -45.01144790649414,
      "epoch": 13.64,
      "learning_rate": 6.265037878787878e-05,
      "loss": 2.7659,
      "step": 234800
    },
    {
      "activations/layer0_attention_weight_max": 16.339269638061523,
      "activations/layer0_attention_weight_min": -14.561358451843262,
      "activations/layer10_attention_weight_max": 36.03076934814453,
      "activations/layer10_attention_weight_min": -35.59893798828125,
      "activations/layer11_attention_weight_max": 32.26468276977539,
      "activations/layer11_attention_weight_min": -32.876182556152344,
      "activations/layer12_attention_weight_max": 17.26140785217285,
      "activations/layer12_attention_weight_min": -25.0230712890625,
      "activations/layer13_attention_weight_max": 36.90614318847656,
      "activations/layer13_attention_weight_min": -29.597061157226562,
      "activations/layer14_attention_weight_max": 33.30592346191406,
      "activations/layer14_attention_weight_min": -29.999866485595703,
      "activations/layer15_attention_weight_max": 33.33061981201172,
      "activations/layer15_attention_weight_min": -29.88005828857422,
      "activations/layer16_attention_weight_max": 30.20391082763672,
      "activations/layer16_attention_weight_min": -30.529386520385742,
      "activations/layer17_attention_weight_max": 51.249786376953125,
      "activations/layer17_attention_weight_min": -45.70722198486328,
      "activations/layer18_attention_weight_max": 47.95299530029297,
      "activations/layer18_attention_weight_min": -37.2454948425293,
      "activations/layer19_attention_weight_max": 22.74346923828125,
      "activations/layer19_attention_weight_min": -21.6524715423584,
      "activations/layer1_attention_weight_max": 16.646465301513672,
      "activations/layer1_attention_weight_min": -15.31550407409668,
      "activations/layer20_attention_weight_max": 20.4857120513916,
      "activations/layer20_attention_weight_min": -19.90042495727539,
      "activations/layer21_attention_weight_max": 35.32155227661133,
      "activations/layer21_attention_weight_min": -22.97957420349121,
      "activations/layer22_attention_weight_max": 28.33516502380371,
      "activations/layer22_attention_weight_min": -23.776458740234375,
      "activations/layer23_attention_weight_max": 32.85877990722656,
      "activations/layer23_attention_weight_min": -26.106918334960938,
      "activations/layer2_attention_weight_max": 33.33766555786133,
      "activations/layer2_attention_weight_min": -32.88446044921875,
      "activations/layer3_attention_weight_max": 96.84717559814453,
      "activations/layer3_attention_weight_min": -101.94622039794922,
      "activations/layer4_attention_weight_max": 97.29693603515625,
      "activations/layer4_attention_weight_min": -96.61271667480469,
      "activations/layer5_attention_weight_max": 75.2816162109375,
      "activations/layer5_attention_weight_min": -78.78639221191406,
      "activations/layer6_attention_weight_max": 53.629703521728516,
      "activations/layer6_attention_weight_min": -54.00613784790039,
      "activations/layer7_attention_weight_max": 69.26618957519531,
      "activations/layer7_attention_weight_min": -69.22235107421875,
      "activations/layer8_attention_weight_max": 48.32551574707031,
      "activations/layer8_attention_weight_min": -50.70125198364258,
      "activations/layer9_attention_weight_max": 47.69313430786133,
      "activations/layer9_attention_weight_min": -47.45819854736328,
      "epoch": 13.65,
      "learning_rate": 6.26314393939394e-05,
      "loss": 2.7689,
      "step": 234850
    },
    {
      "activations/layer0_attention_weight_max": 16.81287384033203,
      "activations/layer0_attention_weight_min": -14.330941200256348,
      "activations/layer10_attention_weight_max": 39.19107437133789,
      "activations/layer10_attention_weight_min": -37.069091796875,
      "activations/layer11_attention_weight_max": 36.89397048950195,
      "activations/layer11_attention_weight_min": -33.97570037841797,
      "activations/layer12_attention_weight_max": 19.586284637451172,
      "activations/layer12_attention_weight_min": -28.40915298461914,
      "activations/layer13_attention_weight_max": 41.51027297973633,
      "activations/layer13_attention_weight_min": -32.628665924072266,
      "activations/layer14_attention_weight_max": 38.508384704589844,
      "activations/layer14_attention_weight_min": -31.425537109375,
      "activations/layer15_attention_weight_max": 34.24172592163086,
      "activations/layer15_attention_weight_min": -30.256467819213867,
      "activations/layer16_attention_weight_max": 32.99325942993164,
      "activations/layer16_attention_weight_min": -31.027713775634766,
      "activations/layer17_attention_weight_max": 53.98161697387695,
      "activations/layer17_attention_weight_min": -44.75483703613281,
      "activations/layer18_attention_weight_max": 50.40749740600586,
      "activations/layer18_attention_weight_min": -40.06940841674805,
      "activations/layer19_attention_weight_max": 23.21015739440918,
      "activations/layer19_attention_weight_min": -22.745882034301758,
      "activations/layer1_attention_weight_max": 16.90059471130371,
      "activations/layer1_attention_weight_min": -15.35842514038086,
      "activations/layer20_attention_weight_max": 25.157163619995117,
      "activations/layer20_attention_weight_min": -22.096599578857422,
      "activations/layer21_attention_weight_max": 43.00834274291992,
      "activations/layer21_attention_weight_min": -23.653379440307617,
      "activations/layer22_attention_weight_max": 29.88492774963379,
      "activations/layer22_attention_weight_min": -25.34315299987793,
      "activations/layer23_attention_weight_max": 39.70112228393555,
      "activations/layer23_attention_weight_min": -22.015514373779297,
      "activations/layer2_attention_weight_max": 33.429725646972656,
      "activations/layer2_attention_weight_min": -32.115478515625,
      "activations/layer3_attention_weight_max": 96.37405395507812,
      "activations/layer3_attention_weight_min": -98.36978912353516,
      "activations/layer4_attention_weight_max": 102.21688079833984,
      "activations/layer4_attention_weight_min": -101.13863372802734,
      "activations/layer5_attention_weight_max": 73.58938598632812,
      "activations/layer5_attention_weight_min": -77.41654968261719,
      "activations/layer6_attention_weight_max": 55.477256774902344,
      "activations/layer6_attention_weight_min": -54.45724868774414,
      "activations/layer7_attention_weight_max": 66.00574493408203,
      "activations/layer7_attention_weight_min": -68.08489227294922,
      "activations/layer8_attention_weight_max": 46.48479461669922,
      "activations/layer8_attention_weight_min": -50.06292724609375,
      "activations/layer9_attention_weight_max": 48.36198043823242,
      "activations/layer9_attention_weight_min": -46.4692497253418,
      "epoch": 13.65,
      "learning_rate": 6.26125e-05,
      "loss": 2.7711,
      "step": 234900
    },
    {
      "activations/layer0_attention_weight_max": 16.828569412231445,
      "activations/layer0_attention_weight_min": -14.354374885559082,
      "activations/layer10_attention_weight_max": 34.63514709472656,
      "activations/layer10_attention_weight_min": -32.343265533447266,
      "activations/layer11_attention_weight_max": 30.296958923339844,
      "activations/layer11_attention_weight_min": -31.598560333251953,
      "activations/layer12_attention_weight_max": 19.037769317626953,
      "activations/layer12_attention_weight_min": -23.794206619262695,
      "activations/layer13_attention_weight_max": 33.1120719909668,
      "activations/layer13_attention_weight_min": -28.379257202148438,
      "activations/layer14_attention_weight_max": 32.62965393066406,
      "activations/layer14_attention_weight_min": -28.783126831054688,
      "activations/layer15_attention_weight_max": 31.058589935302734,
      "activations/layer15_attention_weight_min": -29.179885864257812,
      "activations/layer16_attention_weight_max": 31.187549591064453,
      "activations/layer16_attention_weight_min": -31.674694061279297,
      "activations/layer17_attention_weight_max": 48.681949615478516,
      "activations/layer17_attention_weight_min": -45.02370071411133,
      "activations/layer18_attention_weight_max": 43.37810516357422,
      "activations/layer18_attention_weight_min": -38.29057312011719,
      "activations/layer19_attention_weight_max": 21.37977409362793,
      "activations/layer19_attention_weight_min": -21.472837448120117,
      "activations/layer1_attention_weight_max": 17.088422775268555,
      "activations/layer1_attention_weight_min": -15.936132431030273,
      "activations/layer20_attention_weight_max": 20.927936553955078,
      "activations/layer20_attention_weight_min": -22.398887634277344,
      "activations/layer21_attention_weight_max": 31.868179321289062,
      "activations/layer21_attention_weight_min": -24.573545455932617,
      "activations/layer22_attention_weight_max": 30.64332389831543,
      "activations/layer22_attention_weight_min": -25.450830459594727,
      "activations/layer23_attention_weight_max": 34.658851623535156,
      "activations/layer23_attention_weight_min": -24.849515914916992,
      "activations/layer2_attention_weight_max": 32.449398040771484,
      "activations/layer2_attention_weight_min": -31.031768798828125,
      "activations/layer3_attention_weight_max": 94.75019836425781,
      "activations/layer3_attention_weight_min": -95.62770080566406,
      "activations/layer4_attention_weight_max": 98.70809173583984,
      "activations/layer4_attention_weight_min": -98.25463104248047,
      "activations/layer5_attention_weight_max": 75.97221374511719,
      "activations/layer5_attention_weight_min": -80.16320037841797,
      "activations/layer6_attention_weight_max": 52.41530227661133,
      "activations/layer6_attention_weight_min": -54.27725601196289,
      "activations/layer7_attention_weight_max": 69.34577178955078,
      "activations/layer7_attention_weight_min": -66.63777160644531,
      "activations/layer8_attention_weight_max": 43.34476852416992,
      "activations/layer8_attention_weight_min": -45.58803176879883,
      "activations/layer9_attention_weight_max": 44.180030822753906,
      "activations/layer9_attention_weight_min": -43.46979522705078,
      "epoch": 13.65,
      "learning_rate": 6.25935606060606e-05,
      "loss": 2.7703,
      "step": 234950
    },
    {
      "activations/layer0_attention_weight_max": 15.886296272277832,
      "activations/layer0_attention_weight_min": -14.441048622131348,
      "activations/layer10_attention_weight_max": 35.71226119995117,
      "activations/layer10_attention_weight_min": -34.97418212890625,
      "activations/layer11_attention_weight_max": 30.818965911865234,
      "activations/layer11_attention_weight_min": -32.412315368652344,
      "activations/layer12_attention_weight_max": 18.747901916503906,
      "activations/layer12_attention_weight_min": -25.140111923217773,
      "activations/layer13_attention_weight_max": 34.6859130859375,
      "activations/layer13_attention_weight_min": -29.887332916259766,
      "activations/layer14_attention_weight_max": 34.1318473815918,
      "activations/layer14_attention_weight_min": -28.405391693115234,
      "activations/layer15_attention_weight_max": 29.74312400817871,
      "activations/layer15_attention_weight_min": -29.398113250732422,
      "activations/layer16_attention_weight_max": 30.81970977783203,
      "activations/layer16_attention_weight_min": -29.02168083190918,
      "activations/layer17_attention_weight_max": 46.053260803222656,
      "activations/layer17_attention_weight_min": -40.64604568481445,
      "activations/layer18_attention_weight_max": 41.69970703125,
      "activations/layer18_attention_weight_min": -34.774234771728516,
      "activations/layer19_attention_weight_max": 19.670289993286133,
      "activations/layer19_attention_weight_min": -20.78840446472168,
      "activations/layer1_attention_weight_max": 16.06199836730957,
      "activations/layer1_attention_weight_min": -15.536561965942383,
      "activations/layer20_attention_weight_max": 19.60228729248047,
      "activations/layer20_attention_weight_min": -20.69523811340332,
      "activations/layer21_attention_weight_max": 32.484588623046875,
      "activations/layer21_attention_weight_min": -21.759187698364258,
      "activations/layer22_attention_weight_max": 29.674585342407227,
      "activations/layer22_attention_weight_min": -24.760332107543945,
      "activations/layer23_attention_weight_max": 35.24494552612305,
      "activations/layer23_attention_weight_min": -23.123273849487305,
      "activations/layer2_attention_weight_max": 31.888389587402344,
      "activations/layer2_attention_weight_min": -31.78089714050293,
      "activations/layer3_attention_weight_max": 99.20573425292969,
      "activations/layer3_attention_weight_min": -102.97444915771484,
      "activations/layer4_attention_weight_max": 99.24394989013672,
      "activations/layer4_attention_weight_min": -94.82914733886719,
      "activations/layer5_attention_weight_max": 69.76895141601562,
      "activations/layer5_attention_weight_min": -77.16319274902344,
      "activations/layer6_attention_weight_max": 51.67576217651367,
      "activations/layer6_attention_weight_min": -52.139747619628906,
      "activations/layer7_attention_weight_max": 64.3499984741211,
      "activations/layer7_attention_weight_min": -64.28469848632812,
      "activations/layer8_attention_weight_max": 43.06354904174805,
      "activations/layer8_attention_weight_min": -44.695838928222656,
      "activations/layer9_attention_weight_max": 43.60609436035156,
      "activations/layer9_attention_weight_min": -46.5851936340332,
      "epoch": 13.65,
      "learning_rate": 6.257462121212121e-05,
      "loss": 2.7737,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4493,
      "eval_samples_per_second": 508.207,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4493,
      "eval_openwebtext_samples_per_second": 508.207,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.973,
      "eval_wikitext_samples_per_second": 231.117,
      "step": 235000
    },
    {
      "epoch": 13.65,
      "eval_lambada_loss": 2.486328125,
      "eval_lambada_ppl": 12.01706983184115,
      "eval_lambada_runtime": 9.5699,
      "eval_lambada_samples_per_second": 508.783,
      "step": 235000
    },
    {
      "activations/layer0_attention_weight_max": 16.425342559814453,
      "activations/layer0_attention_weight_min": -14.283799171447754,
      "activations/layer10_attention_weight_max": 37.964683532714844,
      "activations/layer10_attention_weight_min": -37.10870361328125,
      "activations/layer11_attention_weight_max": 33.7051887512207,
      "activations/layer11_attention_weight_min": -33.58287048339844,
      "activations/layer12_attention_weight_max": 19.181638717651367,
      "activations/layer12_attention_weight_min": -28.62757682800293,
      "activations/layer13_attention_weight_max": 44.806480407714844,
      "activations/layer13_attention_weight_min": -31.42948341369629,
      "activations/layer14_attention_weight_max": 45.52030563354492,
      "activations/layer14_attention_weight_min": -30.613935470581055,
      "activations/layer15_attention_weight_max": 37.21528244018555,
      "activations/layer15_attention_weight_min": -29.401615142822266,
      "activations/layer16_attention_weight_max": 34.122215270996094,
      "activations/layer16_attention_weight_min": -31.28297996520996,
      "activations/layer17_attention_weight_max": 56.047035217285156,
      "activations/layer17_attention_weight_min": -45.54264450073242,
      "activations/layer18_attention_weight_max": 55.77812194824219,
      "activations/layer18_attention_weight_min": -39.5146598815918,
      "activations/layer19_attention_weight_max": 25.126359939575195,
      "activations/layer19_attention_weight_min": -21.5283203125,
      "activations/layer1_attention_weight_max": 17.079496383666992,
      "activations/layer1_attention_weight_min": -16.2955379486084,
      "activations/layer20_attention_weight_max": 30.2132568359375,
      "activations/layer20_attention_weight_min": -21.464067459106445,
      "activations/layer21_attention_weight_max": 44.81571578979492,
      "activations/layer21_attention_weight_min": -23.716032028198242,
      "activations/layer22_attention_weight_max": 34.28404235839844,
      "activations/layer22_attention_weight_min": -29.299434661865234,
      "activations/layer23_attention_weight_max": 44.109718322753906,
      "activations/layer23_attention_weight_min": -23.06182861328125,
      "activations/layer2_attention_weight_max": 34.8609619140625,
      "activations/layer2_attention_weight_min": -34.4722900390625,
      "activations/layer3_attention_weight_max": 94.719482421875,
      "activations/layer3_attention_weight_min": -100.87808990478516,
      "activations/layer4_attention_weight_max": 96.95870208740234,
      "activations/layer4_attention_weight_min": -94.7989501953125,
      "activations/layer5_attention_weight_max": 71.78357696533203,
      "activations/layer5_attention_weight_min": -79.7408447265625,
      "activations/layer6_attention_weight_max": 53.56242752075195,
      "activations/layer6_attention_weight_min": -52.609657287597656,
      "activations/layer7_attention_weight_max": 68.36286926269531,
      "activations/layer7_attention_weight_min": -67.04356384277344,
      "activations/layer8_attention_weight_max": 46.87944412231445,
      "activations/layer8_attention_weight_min": -49.8286018371582,
      "activations/layer9_attention_weight_max": 50.69033432006836,
      "activations/layer9_attention_weight_min": -46.07573318481445,
      "epoch": 13.66,
      "learning_rate": 6.255568181818182e-05,
      "loss": 2.7616,
      "step": 235050
    },
    {
      "activations/layer0_attention_weight_max": 16.551435470581055,
      "activations/layer0_attention_weight_min": -14.14484691619873,
      "activations/layer10_attention_weight_max": 37.59978103637695,
      "activations/layer10_attention_weight_min": -37.69595718383789,
      "activations/layer11_attention_weight_max": 33.43537902832031,
      "activations/layer11_attention_weight_min": -34.11986541748047,
      "activations/layer12_attention_weight_max": 19.94056510925293,
      "activations/layer12_attention_weight_min": -22.900711059570312,
      "activations/layer13_attention_weight_max": 35.94947052001953,
      "activations/layer13_attention_weight_min": -27.034582138061523,
      "activations/layer14_attention_weight_max": 35.89400100708008,
      "activations/layer14_attention_weight_min": -30.128780364990234,
      "activations/layer15_attention_weight_max": 31.105024337768555,
      "activations/layer15_attention_weight_min": -29.22834587097168,
      "activations/layer16_attention_weight_max": 32.88695526123047,
      "activations/layer16_attention_weight_min": -30.32398223876953,
      "activations/layer17_attention_weight_max": 53.19878387451172,
      "activations/layer17_attention_weight_min": -42.74959182739258,
      "activations/layer18_attention_weight_max": 47.40260696411133,
      "activations/layer18_attention_weight_min": -37.99897384643555,
      "activations/layer19_attention_weight_max": 21.076831817626953,
      "activations/layer19_attention_weight_min": -21.068166732788086,
      "activations/layer1_attention_weight_max": 16.39440155029297,
      "activations/layer1_attention_weight_min": -14.790892601013184,
      "activations/layer20_attention_weight_max": 21.827796936035156,
      "activations/layer20_attention_weight_min": -20.70206069946289,
      "activations/layer21_attention_weight_max": 31.066112518310547,
      "activations/layer21_attention_weight_min": -18.784204483032227,
      "activations/layer22_attention_weight_max": 31.899330139160156,
      "activations/layer22_attention_weight_min": -26.16990089416504,
      "activations/layer23_attention_weight_max": 36.738922119140625,
      "activations/layer23_attention_weight_min": -23.391843795776367,
      "activations/layer2_attention_weight_max": 33.057823181152344,
      "activations/layer2_attention_weight_min": -31.53799819946289,
      "activations/layer3_attention_weight_max": 96.97431182861328,
      "activations/layer3_attention_weight_min": -93.2443618774414,
      "activations/layer4_attention_weight_max": 97.94661712646484,
      "activations/layer4_attention_weight_min": -98.92645263671875,
      "activations/layer5_attention_weight_max": 74.45111846923828,
      "activations/layer5_attention_weight_min": -83.49496459960938,
      "activations/layer6_attention_weight_max": 52.99988555908203,
      "activations/layer6_attention_weight_min": -58.35944366455078,
      "activations/layer7_attention_weight_max": 67.85662841796875,
      "activations/layer7_attention_weight_min": -66.86943054199219,
      "activations/layer8_attention_weight_max": 49.05699920654297,
      "activations/layer8_attention_weight_min": -53.755958557128906,
      "activations/layer9_attention_weight_max": 48.44044494628906,
      "activations/layer9_attention_weight_min": -47.47268295288086,
      "epoch": 13.66,
      "learning_rate": 6.253674242424242e-05,
      "loss": 2.769,
      "step": 235100
    },
    {
      "activations/layer0_attention_weight_max": 16.368541717529297,
      "activations/layer0_attention_weight_min": -14.634909629821777,
      "activations/layer10_attention_weight_max": 35.1096076965332,
      "activations/layer10_attention_weight_min": -33.07942199707031,
      "activations/layer11_attention_weight_max": 31.397335052490234,
      "activations/layer11_attention_weight_min": -31.815868377685547,
      "activations/layer12_attention_weight_max": 17.997665405273438,
      "activations/layer12_attention_weight_min": -32.26008987426758,
      "activations/layer13_attention_weight_max": 37.43819808959961,
      "activations/layer13_attention_weight_min": -30.693544387817383,
      "activations/layer14_attention_weight_max": 40.452701568603516,
      "activations/layer14_attention_weight_min": -32.22627639770508,
      "activations/layer15_attention_weight_max": 43.72406005859375,
      "activations/layer15_attention_weight_min": -30.4755916595459,
      "activations/layer16_attention_weight_max": 38.981651306152344,
      "activations/layer16_attention_weight_min": -35.0803337097168,
      "activations/layer17_attention_weight_max": 66.90471649169922,
      "activations/layer17_attention_weight_min": -49.810813903808594,
      "activations/layer18_attention_weight_max": 53.88639831542969,
      "activations/layer18_attention_weight_min": -42.12639236450195,
      "activations/layer19_attention_weight_max": 26.84212303161621,
      "activations/layer19_attention_weight_min": -27.54703140258789,
      "activations/layer1_attention_weight_max": 17.248865127563477,
      "activations/layer1_attention_weight_min": -16.634904861450195,
      "activations/layer20_attention_weight_max": 29.860706329345703,
      "activations/layer20_attention_weight_min": -22.3975830078125,
      "activations/layer21_attention_weight_max": 50.592620849609375,
      "activations/layer21_attention_weight_min": -27.956979751586914,
      "activations/layer22_attention_weight_max": 41.831748962402344,
      "activations/layer22_attention_weight_min": -28.9473876953125,
      "activations/layer23_attention_weight_max": 38.64141082763672,
      "activations/layer23_attention_weight_min": -26.797531127929688,
      "activations/layer2_attention_weight_max": 32.696327209472656,
      "activations/layer2_attention_weight_min": -33.30380630493164,
      "activations/layer3_attention_weight_max": 95.87316131591797,
      "activations/layer3_attention_weight_min": -96.75861358642578,
      "activations/layer4_attention_weight_max": 101.58289337158203,
      "activations/layer4_attention_weight_min": -98.62065887451172,
      "activations/layer5_attention_weight_max": 72.21920776367188,
      "activations/layer5_attention_weight_min": -84.11036682128906,
      "activations/layer6_attention_weight_max": 52.23502731323242,
      "activations/layer6_attention_weight_min": -52.69596862792969,
      "activations/layer7_attention_weight_max": 68.70125579833984,
      "activations/layer7_attention_weight_min": -66.70773315429688,
      "activations/layer8_attention_weight_max": 45.5990104675293,
      "activations/layer8_attention_weight_min": -47.19414138793945,
      "activations/layer9_attention_weight_max": 46.59370040893555,
      "activations/layer9_attention_weight_min": -45.0103759765625,
      "epoch": 13.66,
      "learning_rate": 6.251780303030302e-05,
      "loss": 2.7598,
      "step": 235150
    },
    {
      "activations/layer0_attention_weight_max": 15.781301498413086,
      "activations/layer0_attention_weight_min": -14.304698944091797,
      "activations/layer10_attention_weight_max": 38.7983283996582,
      "activations/layer10_attention_weight_min": -39.841854095458984,
      "activations/layer11_attention_weight_max": 36.402530670166016,
      "activations/layer11_attention_weight_min": -37.710426330566406,
      "activations/layer12_attention_weight_max": 18.84878921508789,
      "activations/layer12_attention_weight_min": -25.957685470581055,
      "activations/layer13_attention_weight_max": 43.52849197387695,
      "activations/layer13_attention_weight_min": -31.30125617980957,
      "activations/layer14_attention_weight_max": 37.065616607666016,
      "activations/layer14_attention_weight_min": -31.683473587036133,
      "activations/layer15_attention_weight_max": 33.847267150878906,
      "activations/layer15_attention_weight_min": -29.93939781188965,
      "activations/layer16_attention_weight_max": 32.56053161621094,
      "activations/layer16_attention_weight_min": -31.766048431396484,
      "activations/layer17_attention_weight_max": 53.601287841796875,
      "activations/layer17_attention_weight_min": -46.16242980957031,
      "activations/layer18_attention_weight_max": 50.430213928222656,
      "activations/layer18_attention_weight_min": -39.037628173828125,
      "activations/layer19_attention_weight_max": 21.730276107788086,
      "activations/layer19_attention_weight_min": -20.886140823364258,
      "activations/layer1_attention_weight_max": 16.5718994140625,
      "activations/layer1_attention_weight_min": -15.53458309173584,
      "activations/layer20_attention_weight_max": 23.090654373168945,
      "activations/layer20_attention_weight_min": -22.073442459106445,
      "activations/layer21_attention_weight_max": 38.95135498046875,
      "activations/layer21_attention_weight_min": -22.636903762817383,
      "activations/layer22_attention_weight_max": 30.18768882751465,
      "activations/layer22_attention_weight_min": -25.358823776245117,
      "activations/layer23_attention_weight_max": 38.06175231933594,
      "activations/layer23_attention_weight_min": -23.12847900390625,
      "activations/layer2_attention_weight_max": 35.968711853027344,
      "activations/layer2_attention_weight_min": -33.351707458496094,
      "activations/layer3_attention_weight_max": 96.88258361816406,
      "activations/layer3_attention_weight_min": -97.38591003417969,
      "activations/layer4_attention_weight_max": 94.9227294921875,
      "activations/layer4_attention_weight_min": -96.7933578491211,
      "activations/layer5_attention_weight_max": 70.14979553222656,
      "activations/layer5_attention_weight_min": -78.39603424072266,
      "activations/layer6_attention_weight_max": 50.15106964111328,
      "activations/layer6_attention_weight_min": -52.122047424316406,
      "activations/layer7_attention_weight_max": 67.05149841308594,
      "activations/layer7_attention_weight_min": -65.38492584228516,
      "activations/layer8_attention_weight_max": 47.061004638671875,
      "activations/layer8_attention_weight_min": -49.554500579833984,
      "activations/layer9_attention_weight_max": 49.15373229980469,
      "activations/layer9_attention_weight_min": -47.22808837890625,
      "epoch": 13.67,
      "learning_rate": 6.249886363636363e-05,
      "loss": 2.7663,
      "step": 235200
    },
    {
      "activations/layer0_attention_weight_max": 16.661022186279297,
      "activations/layer0_attention_weight_min": -14.444840431213379,
      "activations/layer10_attention_weight_max": 35.4032096862793,
      "activations/layer10_attention_weight_min": -35.962135314941406,
      "activations/layer11_attention_weight_max": 33.766090393066406,
      "activations/layer11_attention_weight_min": -32.5894660949707,
      "activations/layer12_attention_weight_max": 17.005966186523438,
      "activations/layer12_attention_weight_min": -29.114530563354492,
      "activations/layer13_attention_weight_max": 36.57796096801758,
      "activations/layer13_attention_weight_min": -29.45854949951172,
      "activations/layer14_attention_weight_max": 37.01797866821289,
      "activations/layer14_attention_weight_min": -29.928924560546875,
      "activations/layer15_attention_weight_max": 30.518123626708984,
      "activations/layer15_attention_weight_min": -28.596696853637695,
      "activations/layer16_attention_weight_max": 32.14419937133789,
      "activations/layer16_attention_weight_min": -31.935375213623047,
      "activations/layer17_attention_weight_max": 55.314170837402344,
      "activations/layer17_attention_weight_min": -48.33515167236328,
      "activations/layer18_attention_weight_max": 51.81155014038086,
      "activations/layer18_attention_weight_min": -41.45079803466797,
      "activations/layer19_attention_weight_max": 25.216869354248047,
      "activations/layer19_attention_weight_min": -23.479633331298828,
      "activations/layer1_attention_weight_max": 15.949268341064453,
      "activations/layer1_attention_weight_min": -14.147229194641113,
      "activations/layer20_attention_weight_max": 23.50402069091797,
      "activations/layer20_attention_weight_min": -22.215177536010742,
      "activations/layer21_attention_weight_max": 40.24331283569336,
      "activations/layer21_attention_weight_min": -20.816131591796875,
      "activations/layer22_attention_weight_max": 29.9215145111084,
      "activations/layer22_attention_weight_min": -27.501508712768555,
      "activations/layer23_attention_weight_max": 36.56416702270508,
      "activations/layer23_attention_weight_min": -23.948322296142578,
      "activations/layer2_attention_weight_max": 33.25818634033203,
      "activations/layer2_attention_weight_min": -31.458295822143555,
      "activations/layer3_attention_weight_max": 93.04434204101562,
      "activations/layer3_attention_weight_min": -92.84564971923828,
      "activations/layer4_attention_weight_max": 95.31861114501953,
      "activations/layer4_attention_weight_min": -89.38697052001953,
      "activations/layer5_attention_weight_max": 74.45198059082031,
      "activations/layer5_attention_weight_min": -76.12985229492188,
      "activations/layer6_attention_weight_max": 50.72746658325195,
      "activations/layer6_attention_weight_min": -52.152217864990234,
      "activations/layer7_attention_weight_max": 64.96068572998047,
      "activations/layer7_attention_weight_min": -63.2797737121582,
      "activations/layer8_attention_weight_max": 50.23435974121094,
      "activations/layer8_attention_weight_min": -48.33818435668945,
      "activations/layer9_attention_weight_max": 49.49595260620117,
      "activations/layer9_attention_weight_min": -44.2336540222168,
      "epoch": 13.67,
      "learning_rate": 6.247992424242423e-05,
      "loss": 2.7624,
      "step": 235250
    },
    {
      "activations/layer0_attention_weight_max": 16.464611053466797,
      "activations/layer0_attention_weight_min": -14.410534858703613,
      "activations/layer10_attention_weight_max": 35.36784744262695,
      "activations/layer10_attention_weight_min": -34.29166793823242,
      "activations/layer11_attention_weight_max": 28.877685546875,
      "activations/layer11_attention_weight_min": -31.121109008789062,
      "activations/layer12_attention_weight_max": 19.430404663085938,
      "activations/layer12_attention_weight_min": -24.071491241455078,
      "activations/layer13_attention_weight_max": 33.4820671081543,
      "activations/layer13_attention_weight_min": -27.735925674438477,
      "activations/layer14_attention_weight_max": 33.96516036987305,
      "activations/layer14_attention_weight_min": -30.57962417602539,
      "activations/layer15_attention_weight_max": 30.69379425048828,
      "activations/layer15_attention_weight_min": -29.333871841430664,
      "activations/layer16_attention_weight_max": 30.500288009643555,
      "activations/layer16_attention_weight_min": -30.611494064331055,
      "activations/layer17_attention_weight_max": 46.613399505615234,
      "activations/layer17_attention_weight_min": -45.14151382446289,
      "activations/layer18_attention_weight_max": 44.05089569091797,
      "activations/layer18_attention_weight_min": -40.57083511352539,
      "activations/layer19_attention_weight_max": 20.83437156677246,
      "activations/layer19_attention_weight_min": -22.62800407409668,
      "activations/layer1_attention_weight_max": 16.956302642822266,
      "activations/layer1_attention_weight_min": -14.914510726928711,
      "activations/layer20_attention_weight_max": 23.10809898376465,
      "activations/layer20_attention_weight_min": -21.72124671936035,
      "activations/layer21_attention_weight_max": 37.35957717895508,
      "activations/layer21_attention_weight_min": -25.45794105529785,
      "activations/layer22_attention_weight_max": 29.817562103271484,
      "activations/layer22_attention_weight_min": -25.58320426940918,
      "activations/layer23_attention_weight_max": 40.14147186279297,
      "activations/layer23_attention_weight_min": -25.960752487182617,
      "activations/layer2_attention_weight_max": 32.74638366699219,
      "activations/layer2_attention_weight_min": -31.308324813842773,
      "activations/layer3_attention_weight_max": 91.78128051757812,
      "activations/layer3_attention_weight_min": -93.88150787353516,
      "activations/layer4_attention_weight_max": 93.9023666381836,
      "activations/layer4_attention_weight_min": -90.8603515625,
      "activations/layer5_attention_weight_max": 72.14027404785156,
      "activations/layer5_attention_weight_min": -83.11735534667969,
      "activations/layer6_attention_weight_max": 54.36732864379883,
      "activations/layer6_attention_weight_min": -51.68014144897461,
      "activations/layer7_attention_weight_max": 65.8750991821289,
      "activations/layer7_attention_weight_min": -65.53839111328125,
      "activations/layer8_attention_weight_max": 42.38947677612305,
      "activations/layer8_attention_weight_min": -46.27140808105469,
      "activations/layer9_attention_weight_max": 42.55754089355469,
      "activations/layer9_attention_weight_min": -43.988399505615234,
      "epoch": 13.67,
      "learning_rate": 6.246098484848484e-05,
      "loss": 2.7634,
      "step": 235300
    },
    {
      "activations/layer0_attention_weight_max": 16.305652618408203,
      "activations/layer0_attention_weight_min": -14.189315795898438,
      "activations/layer10_attention_weight_max": 33.97615432739258,
      "activations/layer10_attention_weight_min": -34.807861328125,
      "activations/layer11_attention_weight_max": 31.211111068725586,
      "activations/layer11_attention_weight_min": -32.37788772583008,
      "activations/layer12_attention_weight_max": 18.761079788208008,
      "activations/layer12_attention_weight_min": -24.824878692626953,
      "activations/layer13_attention_weight_max": 35.25509262084961,
      "activations/layer13_attention_weight_min": -29.700870513916016,
      "activations/layer14_attention_weight_max": 34.643611907958984,
      "activations/layer14_attention_weight_min": -30.031658172607422,
      "activations/layer15_attention_weight_max": 31.214513778686523,
      "activations/layer15_attention_weight_min": -30.97775650024414,
      "activations/layer16_attention_weight_max": 29.895864486694336,
      "activations/layer16_attention_weight_min": -29.724388122558594,
      "activations/layer17_attention_weight_max": 49.99690628051758,
      "activations/layer17_attention_weight_min": -44.33055877685547,
      "activations/layer18_attention_weight_max": 41.31940460205078,
      "activations/layer18_attention_weight_min": -36.95115661621094,
      "activations/layer19_attention_weight_max": 22.498714447021484,
      "activations/layer19_attention_weight_min": -22.29375457763672,
      "activations/layer1_attention_weight_max": 16.32111358642578,
      "activations/layer1_attention_weight_min": -15.698735237121582,
      "activations/layer20_attention_weight_max": 19.9520320892334,
      "activations/layer20_attention_weight_min": -23.316631317138672,
      "activations/layer21_attention_weight_max": 32.36941146850586,
      "activations/layer21_attention_weight_min": -22.764019012451172,
      "activations/layer22_attention_weight_max": 27.8818416595459,
      "activations/layer22_attention_weight_min": -25.24730682373047,
      "activations/layer23_attention_weight_max": 36.50347900390625,
      "activations/layer23_attention_weight_min": -26.133403778076172,
      "activations/layer2_attention_weight_max": 32.954681396484375,
      "activations/layer2_attention_weight_min": -30.353740692138672,
      "activations/layer3_attention_weight_max": 91.29973602294922,
      "activations/layer3_attention_weight_min": -92.49456787109375,
      "activations/layer4_attention_weight_max": 94.27794647216797,
      "activations/layer4_attention_weight_min": -91.4482650756836,
      "activations/layer5_attention_weight_max": 70.41638946533203,
      "activations/layer5_attention_weight_min": -72.33024597167969,
      "activations/layer6_attention_weight_max": 51.463050842285156,
      "activations/layer6_attention_weight_min": -50.76689529418945,
      "activations/layer7_attention_weight_max": 65.41559600830078,
      "activations/layer7_attention_weight_min": -62.29054260253906,
      "activations/layer8_attention_weight_max": 42.89644241333008,
      "activations/layer8_attention_weight_min": -43.41225051879883,
      "activations/layer9_attention_weight_max": 45.344879150390625,
      "activations/layer9_attention_weight_min": -43.507198333740234,
      "epoch": 13.68,
      "learning_rate": 6.244204545454545e-05,
      "loss": 2.7544,
      "step": 235350
    },
    {
      "activations/layer0_attention_weight_max": 17.064037322998047,
      "activations/layer0_attention_weight_min": -14.0416841506958,
      "activations/layer10_attention_weight_max": 39.5960693359375,
      "activations/layer10_attention_weight_min": -37.969459533691406,
      "activations/layer11_attention_weight_max": 31.970012664794922,
      "activations/layer11_attention_weight_min": -34.28881072998047,
      "activations/layer12_attention_weight_max": 18.37763214111328,
      "activations/layer12_attention_weight_min": -26.89206886291504,
      "activations/layer13_attention_weight_max": 35.865234375,
      "activations/layer13_attention_weight_min": -30.119165420532227,
      "activations/layer14_attention_weight_max": 35.00199890136719,
      "activations/layer14_attention_weight_min": -30.143056869506836,
      "activations/layer15_attention_weight_max": 33.76188659667969,
      "activations/layer15_attention_weight_min": -28.925106048583984,
      "activations/layer16_attention_weight_max": 32.25432586669922,
      "activations/layer16_attention_weight_min": -29.441055297851562,
      "activations/layer17_attention_weight_max": 52.78316116333008,
      "activations/layer17_attention_weight_min": -42.67934799194336,
      "activations/layer18_attention_weight_max": 46.71872329711914,
      "activations/layer18_attention_weight_min": -37.302696228027344,
      "activations/layer19_attention_weight_max": 22.22340965270996,
      "activations/layer19_attention_weight_min": -23.119739532470703,
      "activations/layer1_attention_weight_max": 16.417736053466797,
      "activations/layer1_attention_weight_min": -15.375388145446777,
      "activations/layer20_attention_weight_max": 23.502782821655273,
      "activations/layer20_attention_weight_min": -21.771902084350586,
      "activations/layer21_attention_weight_max": 42.878570556640625,
      "activations/layer21_attention_weight_min": -27.246238708496094,
      "activations/layer22_attention_weight_max": 30.118038177490234,
      "activations/layer22_attention_weight_min": -25.635028839111328,
      "activations/layer23_attention_weight_max": 41.713233947753906,
      "activations/layer23_attention_weight_min": -23.611347198486328,
      "activations/layer2_attention_weight_max": 33.055023193359375,
      "activations/layer2_attention_weight_min": -31.317296981811523,
      "activations/layer3_attention_weight_max": 93.1510238647461,
      "activations/layer3_attention_weight_min": -96.77423858642578,
      "activations/layer4_attention_weight_max": 95.23722839355469,
      "activations/layer4_attention_weight_min": -93.98690032958984,
      "activations/layer5_attention_weight_max": 70.81040954589844,
      "activations/layer5_attention_weight_min": -74.22920227050781,
      "activations/layer6_attention_weight_max": 51.008846282958984,
      "activations/layer6_attention_weight_min": -50.9854621887207,
      "activations/layer7_attention_weight_max": 64.86791229248047,
      "activations/layer7_attention_weight_min": -64.68509674072266,
      "activations/layer8_attention_weight_max": 45.945106506347656,
      "activations/layer8_attention_weight_min": -48.81844711303711,
      "activations/layer9_attention_weight_max": 45.95553207397461,
      "activations/layer9_attention_weight_min": -47.062992095947266,
      "epoch": 13.68,
      "learning_rate": 6.242310606060605e-05,
      "loss": 2.7662,
      "step": 235400
    },
    {
      "activations/layer0_attention_weight_max": 16.95175552368164,
      "activations/layer0_attention_weight_min": -14.358278274536133,
      "activations/layer10_attention_weight_max": 36.71394729614258,
      "activations/layer10_attention_weight_min": -37.10584259033203,
      "activations/layer11_attention_weight_max": 32.379173278808594,
      "activations/layer11_attention_weight_min": -33.5141487121582,
      "activations/layer12_attention_weight_max": 19.666994094848633,
      "activations/layer12_attention_weight_min": -27.749448776245117,
      "activations/layer13_attention_weight_max": 40.1688117980957,
      "activations/layer13_attention_weight_min": -29.23604393005371,
      "activations/layer14_attention_weight_max": 37.927757263183594,
      "activations/layer14_attention_weight_min": -31.487163543701172,
      "activations/layer15_attention_weight_max": 35.12822341918945,
      "activations/layer15_attention_weight_min": -31.917165756225586,
      "activations/layer16_attention_weight_max": 32.41598892211914,
      "activations/layer16_attention_weight_min": -30.24854278564453,
      "activations/layer17_attention_weight_max": 52.42320251464844,
      "activations/layer17_attention_weight_min": -44.190208435058594,
      "activations/layer18_attention_weight_max": 51.386871337890625,
      "activations/layer18_attention_weight_min": -39.07773971557617,
      "activations/layer19_attention_weight_max": 26.465238571166992,
      "activations/layer19_attention_weight_min": -25.173381805419922,
      "activations/layer1_attention_weight_max": 16.50628089904785,
      "activations/layer1_attention_weight_min": -14.819194793701172,
      "activations/layer20_attention_weight_max": 27.50231170654297,
      "activations/layer20_attention_weight_min": -24.45174789428711,
      "activations/layer21_attention_weight_max": 40.63802719116211,
      "activations/layer21_attention_weight_min": -24.88896369934082,
      "activations/layer22_attention_weight_max": 30.72341537475586,
      "activations/layer22_attention_weight_min": -25.4116268157959,
      "activations/layer23_attention_weight_max": 39.31114196777344,
      "activations/layer23_attention_weight_min": -23.178434371948242,
      "activations/layer2_attention_weight_max": 34.68424987792969,
      "activations/layer2_attention_weight_min": -31.936063766479492,
      "activations/layer3_attention_weight_max": 101.02021026611328,
      "activations/layer3_attention_weight_min": -104.58346557617188,
      "activations/layer4_attention_weight_max": 103.93717193603516,
      "activations/layer4_attention_weight_min": -98.52323913574219,
      "activations/layer5_attention_weight_max": 74.33638000488281,
      "activations/layer5_attention_weight_min": -84.31198120117188,
      "activations/layer6_attention_weight_max": 54.415870666503906,
      "activations/layer6_attention_weight_min": -57.60786056518555,
      "activations/layer7_attention_weight_max": 66.31429290771484,
      "activations/layer7_attention_weight_min": -70.39604187011719,
      "activations/layer8_attention_weight_max": 47.1786994934082,
      "activations/layer8_attention_weight_min": -51.25307083129883,
      "activations/layer9_attention_weight_max": 49.033199310302734,
      "activations/layer9_attention_weight_min": -48.65426254272461,
      "epoch": 13.68,
      "learning_rate": 6.240416666666665e-05,
      "loss": 2.748,
      "step": 235450
    },
    {
      "activations/layer0_attention_weight_max": 16.92565155029297,
      "activations/layer0_attention_weight_min": -14.349776268005371,
      "activations/layer10_attention_weight_max": 37.91797637939453,
      "activations/layer10_attention_weight_min": -37.09986114501953,
      "activations/layer11_attention_weight_max": 33.30141067504883,
      "activations/layer11_attention_weight_min": -32.819305419921875,
      "activations/layer12_attention_weight_max": 18.52347183227539,
      "activations/layer12_attention_weight_min": -24.2699031829834,
      "activations/layer13_attention_weight_max": 40.41314697265625,
      "activations/layer13_attention_weight_min": -26.065387725830078,
      "activations/layer14_attention_weight_max": 34.992698669433594,
      "activations/layer14_attention_weight_min": -30.22951889038086,
      "activations/layer15_attention_weight_max": 32.56171798706055,
      "activations/layer15_attention_weight_min": -28.731891632080078,
      "activations/layer16_attention_weight_max": 33.395286560058594,
      "activations/layer16_attention_weight_min": -29.84286880493164,
      "activations/layer17_attention_weight_max": 53.96492385864258,
      "activations/layer17_attention_weight_min": -44.129539489746094,
      "activations/layer18_attention_weight_max": 50.35248947143555,
      "activations/layer18_attention_weight_min": -36.41100311279297,
      "activations/layer19_attention_weight_max": 22.10564613342285,
      "activations/layer19_attention_weight_min": -19.726369857788086,
      "activations/layer1_attention_weight_max": 16.397687911987305,
      "activations/layer1_attention_weight_min": -14.099732398986816,
      "activations/layer20_attention_weight_max": 23.842985153198242,
      "activations/layer20_attention_weight_min": -20.212675094604492,
      "activations/layer21_attention_weight_max": 42.21792221069336,
      "activations/layer21_attention_weight_min": -22.96270179748535,
      "activations/layer22_attention_weight_max": 31.31216812133789,
      "activations/layer22_attention_weight_min": -27.237125396728516,
      "activations/layer23_attention_weight_max": 36.442466735839844,
      "activations/layer23_attention_weight_min": -21.544004440307617,
      "activations/layer2_attention_weight_max": 34.40637969970703,
      "activations/layer2_attention_weight_min": -32.120018005371094,
      "activations/layer3_attention_weight_max": 94.12940979003906,
      "activations/layer3_attention_weight_min": -101.7362289428711,
      "activations/layer4_attention_weight_max": 100.4176254272461,
      "activations/layer4_attention_weight_min": -97.03659057617188,
      "activations/layer5_attention_weight_max": 73.86687469482422,
      "activations/layer5_attention_weight_min": -79.44659423828125,
      "activations/layer6_attention_weight_max": 55.99656295776367,
      "activations/layer6_attention_weight_min": -54.11634826660156,
      "activations/layer7_attention_weight_max": 70.0992202758789,
      "activations/layer7_attention_weight_min": -65.97633361816406,
      "activations/layer8_attention_weight_max": 45.65370559692383,
      "activations/layer8_attention_weight_min": -48.4245491027832,
      "activations/layer9_attention_weight_max": 46.93954849243164,
      "activations/layer9_attention_weight_min": -48.66826248168945,
      "epoch": 13.68,
      "learning_rate": 6.238522727272727e-05,
      "loss": 2.7635,
      "step": 235500
    },
    {
      "activations/layer0_attention_weight_max": 16.55756187438965,
      "activations/layer0_attention_weight_min": -14.657917976379395,
      "activations/layer10_attention_weight_max": 38.127197265625,
      "activations/layer10_attention_weight_min": -34.946739196777344,
      "activations/layer11_attention_weight_max": 32.83869171142578,
      "activations/layer11_attention_weight_min": -32.211036682128906,
      "activations/layer12_attention_weight_max": 18.594867706298828,
      "activations/layer12_attention_weight_min": -25.069700241088867,
      "activations/layer13_attention_weight_max": 39.58378219604492,
      "activations/layer13_attention_weight_min": -31.642230987548828,
      "activations/layer14_attention_weight_max": 39.823768615722656,
      "activations/layer14_attention_weight_min": -30.317150115966797,
      "activations/layer15_attention_weight_max": 37.37268829345703,
      "activations/layer15_attention_weight_min": -30.545398712158203,
      "activations/layer16_attention_weight_max": 34.31427001953125,
      "activations/layer16_attention_weight_min": -30.508556365966797,
      "activations/layer17_attention_weight_max": 56.70393753051758,
      "activations/layer17_attention_weight_min": -44.26832962036133,
      "activations/layer18_attention_weight_max": 49.78863525390625,
      "activations/layer18_attention_weight_min": -39.046104431152344,
      "activations/layer19_attention_weight_max": 22.80657958984375,
      "activations/layer19_attention_weight_min": -20.613941192626953,
      "activations/layer1_attention_weight_max": 17.405223846435547,
      "activations/layer1_attention_weight_min": -15.064335823059082,
      "activations/layer20_attention_weight_max": 23.738054275512695,
      "activations/layer20_attention_weight_min": -19.214656829833984,
      "activations/layer21_attention_weight_max": 34.005943298339844,
      "activations/layer21_attention_weight_min": -21.188282012939453,
      "activations/layer22_attention_weight_max": 30.110851287841797,
      "activations/layer22_attention_weight_min": -24.872678756713867,
      "activations/layer23_attention_weight_max": 36.50895309448242,
      "activations/layer23_attention_weight_min": -25.63386344909668,
      "activations/layer2_attention_weight_max": 33.911903381347656,
      "activations/layer2_attention_weight_min": -30.407794952392578,
      "activations/layer3_attention_weight_max": 94.47586059570312,
      "activations/layer3_attention_weight_min": -92.27063751220703,
      "activations/layer4_attention_weight_max": 98.58560943603516,
      "activations/layer4_attention_weight_min": -97.16072845458984,
      "activations/layer5_attention_weight_max": 75.15037536621094,
      "activations/layer5_attention_weight_min": -78.27880859375,
      "activations/layer6_attention_weight_max": 52.55440902709961,
      "activations/layer6_attention_weight_min": -53.65770721435547,
      "activations/layer7_attention_weight_max": 72.1620864868164,
      "activations/layer7_attention_weight_min": -69.85960388183594,
      "activations/layer8_attention_weight_max": 46.988609313964844,
      "activations/layer8_attention_weight_min": -49.53331756591797,
      "activations/layer9_attention_weight_max": 54.49824905395508,
      "activations/layer9_attention_weight_min": -47.74456024169922,
      "epoch": 13.69,
      "learning_rate": 6.236666666666666e-05,
      "loss": 2.7704,
      "step": 235550
    },
    {
      "activations/layer0_attention_weight_max": 16.45985221862793,
      "activations/layer0_attention_weight_min": -14.517136573791504,
      "activations/layer10_attention_weight_max": 37.85041809082031,
      "activations/layer10_attention_weight_min": -36.763450622558594,
      "activations/layer11_attention_weight_max": 35.81403732299805,
      "activations/layer11_attention_weight_min": -36.42656326293945,
      "activations/layer12_attention_weight_max": 18.70829963684082,
      "activations/layer12_attention_weight_min": -22.85022735595703,
      "activations/layer13_attention_weight_max": 36.913055419921875,
      "activations/layer13_attention_weight_min": -26.644723892211914,
      "activations/layer14_attention_weight_max": 36.84848403930664,
      "activations/layer14_attention_weight_min": -30.152408599853516,
      "activations/layer15_attention_weight_max": 33.122711181640625,
      "activations/layer15_attention_weight_min": -30.292251586914062,
      "activations/layer16_attention_weight_max": 31.748205184936523,
      "activations/layer16_attention_weight_min": -32.25279235839844,
      "activations/layer17_attention_weight_max": 54.12028884887695,
      "activations/layer17_attention_weight_min": -45.567142486572266,
      "activations/layer18_attention_weight_max": 46.97939682006836,
      "activations/layer18_attention_weight_min": -39.630001068115234,
      "activations/layer19_attention_weight_max": 23.174480438232422,
      "activations/layer19_attention_weight_min": -22.09340476989746,
      "activations/layer1_attention_weight_max": 17.191991806030273,
      "activations/layer1_attention_weight_min": -15.775866508483887,
      "activations/layer20_attention_weight_max": 23.389604568481445,
      "activations/layer20_attention_weight_min": -19.180660247802734,
      "activations/layer21_attention_weight_max": 36.83689498901367,
      "activations/layer21_attention_weight_min": -20.822492599487305,
      "activations/layer22_attention_weight_max": 30.449996948242188,
      "activations/layer22_attention_weight_min": -24.27994155883789,
      "activations/layer23_attention_weight_max": 35.174034118652344,
      "activations/layer23_attention_weight_min": -23.613285064697266,
      "activations/layer2_attention_weight_max": 33.20525360107422,
      "activations/layer2_attention_weight_min": -32.42723846435547,
      "activations/layer3_attention_weight_max": 96.59230041503906,
      "activations/layer3_attention_weight_min": -97.54202270507812,
      "activations/layer4_attention_weight_max": 97.67647552490234,
      "activations/layer4_attention_weight_min": -95.44349670410156,
      "activations/layer5_attention_weight_max": 74.78890991210938,
      "activations/layer5_attention_weight_min": -82.94662475585938,
      "activations/layer6_attention_weight_max": 55.94548416137695,
      "activations/layer6_attention_weight_min": -54.822227478027344,
      "activations/layer7_attention_weight_max": 71.27709197998047,
      "activations/layer7_attention_weight_min": -68.69206237792969,
      "activations/layer8_attention_weight_max": 46.73737716674805,
      "activations/layer8_attention_weight_min": -48.77119827270508,
      "activations/layer9_attention_weight_max": 45.13433837890625,
      "activations/layer9_attention_weight_min": -45.525447845458984,
      "epoch": 13.69,
      "learning_rate": 6.234772727272728e-05,
      "loss": 2.7628,
      "step": 235600
    },
    {
      "activations/layer0_attention_weight_max": 16.63277816772461,
      "activations/layer0_attention_weight_min": -14.431852340698242,
      "activations/layer10_attention_weight_max": 39.48893356323242,
      "activations/layer10_attention_weight_min": -38.553993225097656,
      "activations/layer11_attention_weight_max": 35.27838897705078,
      "activations/layer11_attention_weight_min": -33.9163703918457,
      "activations/layer12_attention_weight_max": 18.45587730407715,
      "activations/layer12_attention_weight_min": -25.171600341796875,
      "activations/layer13_attention_weight_max": 36.68947219848633,
      "activations/layer13_attention_weight_min": -27.362703323364258,
      "activations/layer14_attention_weight_max": 35.34707260131836,
      "activations/layer14_attention_weight_min": -28.88993263244629,
      "activations/layer15_attention_weight_max": 31.142370223999023,
      "activations/layer15_attention_weight_min": -28.026268005371094,
      "activations/layer16_attention_weight_max": 31.641307830810547,
      "activations/layer16_attention_weight_min": -27.755924224853516,
      "activations/layer17_attention_weight_max": 50.1400032043457,
      "activations/layer17_attention_weight_min": -39.7968635559082,
      "activations/layer18_attention_weight_max": 44.902774810791016,
      "activations/layer18_attention_weight_min": -36.14912414550781,
      "activations/layer19_attention_weight_max": 21.0917911529541,
      "activations/layer19_attention_weight_min": -20.986679077148438,
      "activations/layer1_attention_weight_max": 16.289928436279297,
      "activations/layer1_attention_weight_min": -14.721694946289062,
      "activations/layer20_attention_weight_max": 23.792926788330078,
      "activations/layer20_attention_weight_min": -20.83673858642578,
      "activations/layer21_attention_weight_max": 40.405029296875,
      "activations/layer21_attention_weight_min": -22.40665626525879,
      "activations/layer22_attention_weight_max": 29.001495361328125,
      "activations/layer22_attention_weight_min": -25.842269897460938,
      "activations/layer23_attention_weight_max": 41.07966995239258,
      "activations/layer23_attention_weight_min": -23.259159088134766,
      "activations/layer2_attention_weight_max": 34.34750747680664,
      "activations/layer2_attention_weight_min": -32.44186019897461,
      "activations/layer3_attention_weight_max": 96.72419738769531,
      "activations/layer3_attention_weight_min": -99.4249496459961,
      "activations/layer4_attention_weight_max": 96.15538024902344,
      "activations/layer4_attention_weight_min": -92.73910522460938,
      "activations/layer5_attention_weight_max": 71.15653228759766,
      "activations/layer5_attention_weight_min": -77.90038299560547,
      "activations/layer6_attention_weight_max": 52.117713928222656,
      "activations/layer6_attention_weight_min": -53.30995559692383,
      "activations/layer7_attention_weight_max": 65.40255737304688,
      "activations/layer7_attention_weight_min": -70.50545501708984,
      "activations/layer8_attention_weight_max": 46.60905075073242,
      "activations/layer8_attention_weight_min": -51.055545806884766,
      "activations/layer9_attention_weight_max": 48.159297943115234,
      "activations/layer9_attention_weight_min": -49.63616180419922,
      "epoch": 13.69,
      "learning_rate": 6.232878787878788e-05,
      "loss": 2.7753,
      "step": 235650
    },
    {
      "activations/layer0_attention_weight_max": 16.56365203857422,
      "activations/layer0_attention_weight_min": -14.31164836883545,
      "activations/layer10_attention_weight_max": 37.31770324707031,
      "activations/layer10_attention_weight_min": -35.76682662963867,
      "activations/layer11_attention_weight_max": 33.0107307434082,
      "activations/layer11_attention_weight_min": -34.209320068359375,
      "activations/layer12_attention_weight_max": 21.37169075012207,
      "activations/layer12_attention_weight_min": -24.042062759399414,
      "activations/layer13_attention_weight_max": 40.71012878417969,
      "activations/layer13_attention_weight_min": -32.128440856933594,
      "activations/layer14_attention_weight_max": 35.99840545654297,
      "activations/layer14_attention_weight_min": -31.82957649230957,
      "activations/layer15_attention_weight_max": 34.012088775634766,
      "activations/layer15_attention_weight_min": -30.745590209960938,
      "activations/layer16_attention_weight_max": 33.81802749633789,
      "activations/layer16_attention_weight_min": -32.16061782836914,
      "activations/layer17_attention_weight_max": 56.01837158203125,
      "activations/layer17_attention_weight_min": -46.99407958984375,
      "activations/layer18_attention_weight_max": 50.103641510009766,
      "activations/layer18_attention_weight_min": -45.05498504638672,
      "activations/layer19_attention_weight_max": 27.567312240600586,
      "activations/layer19_attention_weight_min": -26.799320220947266,
      "activations/layer1_attention_weight_max": 17.068918228149414,
      "activations/layer1_attention_weight_min": -14.261963844299316,
      "activations/layer20_attention_weight_max": 29.3956356048584,
      "activations/layer20_attention_weight_min": -22.378576278686523,
      "activations/layer21_attention_weight_max": 37.89930725097656,
      "activations/layer21_attention_weight_min": -23.69371223449707,
      "activations/layer22_attention_weight_max": 33.464111328125,
      "activations/layer22_attention_weight_min": -26.28518295288086,
      "activations/layer23_attention_weight_max": 44.95799255371094,
      "activations/layer23_attention_weight_min": -23.321876525878906,
      "activations/layer2_attention_weight_max": 32.66191482543945,
      "activations/layer2_attention_weight_min": -31.906482696533203,
      "activations/layer3_attention_weight_max": 96.68212890625,
      "activations/layer3_attention_weight_min": -96.1357421875,
      "activations/layer4_attention_weight_max": 94.78565979003906,
      "activations/layer4_attention_weight_min": -92.61836242675781,
      "activations/layer5_attention_weight_max": 75.3101577758789,
      "activations/layer5_attention_weight_min": -77.69505310058594,
      "activations/layer6_attention_weight_max": 54.93540954589844,
      "activations/layer6_attention_weight_min": -52.29767608642578,
      "activations/layer7_attention_weight_max": 66.3677749633789,
      "activations/layer7_attention_weight_min": -67.63497924804688,
      "activations/layer8_attention_weight_max": 47.73386001586914,
      "activations/layer8_attention_weight_min": -50.22206497192383,
      "activations/layer9_attention_weight_max": 46.748722076416016,
      "activations/layer9_attention_weight_min": -47.34797668457031,
      "epoch": 13.7,
      "learning_rate": 6.230984848484848e-05,
      "loss": 2.7714,
      "step": 235700
    },
    {
      "activations/layer0_attention_weight_max": 17.580074310302734,
      "activations/layer0_attention_weight_min": -14.37522029876709,
      "activations/layer10_attention_weight_max": 36.1033821105957,
      "activations/layer10_attention_weight_min": -34.301944732666016,
      "activations/layer11_attention_weight_max": 32.746829986572266,
      "activations/layer11_attention_weight_min": -32.59990310668945,
      "activations/layer12_attention_weight_max": 19.829885482788086,
      "activations/layer12_attention_weight_min": -27.765888214111328,
      "activations/layer13_attention_weight_max": 58.42911911010742,
      "activations/layer13_attention_weight_min": -33.247398376464844,
      "activations/layer14_attention_weight_max": 47.11273956298828,
      "activations/layer14_attention_weight_min": -29.93939781188965,
      "activations/layer15_attention_weight_max": 48.744956970214844,
      "activations/layer15_attention_weight_min": -29.764616012573242,
      "activations/layer16_attention_weight_max": 35.52260208129883,
      "activations/layer16_attention_weight_min": -31.134912490844727,
      "activations/layer17_attention_weight_max": 58.08685302734375,
      "activations/layer17_attention_weight_min": -49.29740524291992,
      "activations/layer18_attention_weight_max": 49.79948043823242,
      "activations/layer18_attention_weight_min": -39.85541534423828,
      "activations/layer19_attention_weight_max": 25.681121826171875,
      "activations/layer19_attention_weight_min": -24.202943801879883,
      "activations/layer1_attention_weight_max": 16.880632400512695,
      "activations/layer1_attention_weight_min": -16.955650329589844,
      "activations/layer20_attention_weight_max": 27.838415145874023,
      "activations/layer20_attention_weight_min": -22.082530975341797,
      "activations/layer21_attention_weight_max": 42.40143585205078,
      "activations/layer21_attention_weight_min": -21.53064727783203,
      "activations/layer22_attention_weight_max": 35.3036003112793,
      "activations/layer22_attention_weight_min": -24.53241539001465,
      "activations/layer23_attention_weight_max": 36.88103103637695,
      "activations/layer23_attention_weight_min": -23.495697021484375,
      "activations/layer2_attention_weight_max": 32.2260627746582,
      "activations/layer2_attention_weight_min": -31.9593563079834,
      "activations/layer3_attention_weight_max": 97.14125061035156,
      "activations/layer3_attention_weight_min": -95.27567291259766,
      "activations/layer4_attention_weight_max": 94.3257064819336,
      "activations/layer4_attention_weight_min": -93.5238265991211,
      "activations/layer5_attention_weight_max": 74.09147644042969,
      "activations/layer5_attention_weight_min": -78.83259582519531,
      "activations/layer6_attention_weight_max": 52.11885070800781,
      "activations/layer6_attention_weight_min": -51.794395446777344,
      "activations/layer7_attention_weight_max": 66.86821746826172,
      "activations/layer7_attention_weight_min": -66.89077758789062,
      "activations/layer8_attention_weight_max": 44.8056755065918,
      "activations/layer8_attention_weight_min": -47.431190490722656,
      "activations/layer9_attention_weight_max": 48.33852005004883,
      "activations/layer9_attention_weight_min": -44.49565505981445,
      "epoch": 13.7,
      "learning_rate": 6.22909090909091e-05,
      "loss": 2.7722,
      "step": 235750
    },
    {
      "activations/layer0_attention_weight_max": 17.2370548248291,
      "activations/layer0_attention_weight_min": -14.124024391174316,
      "activations/layer10_attention_weight_max": 34.687583923339844,
      "activations/layer10_attention_weight_min": -36.41282272338867,
      "activations/layer11_attention_weight_max": 31.059322357177734,
      "activations/layer11_attention_weight_min": -34.04656219482422,
      "activations/layer12_attention_weight_max": 19.685171127319336,
      "activations/layer12_attention_weight_min": -25.473119735717773,
      "activations/layer13_attention_weight_max": 40.05446243286133,
      "activations/layer13_attention_weight_min": -30.076271057128906,
      "activations/layer14_attention_weight_max": 36.344688415527344,
      "activations/layer14_attention_weight_min": -30.603971481323242,
      "activations/layer15_attention_weight_max": 33.745697021484375,
      "activations/layer15_attention_weight_min": -30.69696044921875,
      "activations/layer16_attention_weight_max": 31.740934371948242,
      "activations/layer16_attention_weight_min": -29.385168075561523,
      "activations/layer17_attention_weight_max": 49.58638381958008,
      "activations/layer17_attention_weight_min": -42.89194107055664,
      "activations/layer18_attention_weight_max": 47.79945373535156,
      "activations/layer18_attention_weight_min": -36.70791244506836,
      "activations/layer19_attention_weight_max": 24.162940979003906,
      "activations/layer19_attention_weight_min": -21.164127349853516,
      "activations/layer1_attention_weight_max": 17.765138626098633,
      "activations/layer1_attention_weight_min": -15.27577018737793,
      "activations/layer20_attention_weight_max": 26.807178497314453,
      "activations/layer20_attention_weight_min": -22.05336570739746,
      "activations/layer21_attention_weight_max": 36.885292053222656,
      "activations/layer21_attention_weight_min": -23.856992721557617,
      "activations/layer22_attention_weight_max": 30.112060546875,
      "activations/layer22_attention_weight_min": -26.60120391845703,
      "activations/layer23_attention_weight_max": 36.58816909790039,
      "activations/layer23_attention_weight_min": -22.62660026550293,
      "activations/layer2_attention_weight_max": 32.860633850097656,
      "activations/layer2_attention_weight_min": -32.65495681762695,
      "activations/layer3_attention_weight_max": 92.71253967285156,
      "activations/layer3_attention_weight_min": -96.60710906982422,
      "activations/layer4_attention_weight_max": 98.73751068115234,
      "activations/layer4_attention_weight_min": -94.32659912109375,
      "activations/layer5_attention_weight_max": 67.99775695800781,
      "activations/layer5_attention_weight_min": -77.58587646484375,
      "activations/layer6_attention_weight_max": 53.37935256958008,
      "activations/layer6_attention_weight_min": -54.27711486816406,
      "activations/layer7_attention_weight_max": 65.11505126953125,
      "activations/layer7_attention_weight_min": -65.34300231933594,
      "activations/layer8_attention_weight_max": 44.13981628417969,
      "activations/layer8_attention_weight_min": -47.592933654785156,
      "activations/layer9_attention_weight_max": 43.75068664550781,
      "activations/layer9_attention_weight_min": -44.48533248901367,
      "epoch": 13.7,
      "learning_rate": 6.22719696969697e-05,
      "loss": 2.7585,
      "step": 235800
    },
    {
      "activations/layer0_attention_weight_max": 17.643922805786133,
      "activations/layer0_attention_weight_min": -14.411145210266113,
      "activations/layer10_attention_weight_max": 34.49835968017578,
      "activations/layer10_attention_weight_min": -34.90298080444336,
      "activations/layer11_attention_weight_max": 33.995155334472656,
      "activations/layer11_attention_weight_min": -32.16952896118164,
      "activations/layer12_attention_weight_max": 18.174949645996094,
      "activations/layer12_attention_weight_min": -27.808744430541992,
      "activations/layer13_attention_weight_max": 45.067989349365234,
      "activations/layer13_attention_weight_min": -29.195114135742188,
      "activations/layer14_attention_weight_max": 39.17637634277344,
      "activations/layer14_attention_weight_min": -31.006465911865234,
      "activations/layer15_attention_weight_max": 29.6306209564209,
      "activations/layer15_attention_weight_min": -29.509662628173828,
      "activations/layer16_attention_weight_max": 29.431529998779297,
      "activations/layer16_attention_weight_min": -29.270553588867188,
      "activations/layer17_attention_weight_max": 49.37538146972656,
      "activations/layer17_attention_weight_min": -41.89765930175781,
      "activations/layer18_attention_weight_max": 42.706966400146484,
      "activations/layer18_attention_weight_min": -36.55862808227539,
      "activations/layer19_attention_weight_max": 22.477798461914062,
      "activations/layer19_attention_weight_min": -21.155851364135742,
      "activations/layer1_attention_weight_max": 16.312660217285156,
      "activations/layer1_attention_weight_min": -16.331178665161133,
      "activations/layer20_attention_weight_max": 24.97280502319336,
      "activations/layer20_attention_weight_min": -22.152408599853516,
      "activations/layer21_attention_weight_max": 34.938453674316406,
      "activations/layer21_attention_weight_min": -20.378616333007812,
      "activations/layer22_attention_weight_max": 27.65793800354004,
      "activations/layer22_attention_weight_min": -23.87017059326172,
      "activations/layer23_attention_weight_max": 37.03996276855469,
      "activations/layer23_attention_weight_min": -21.907272338867188,
      "activations/layer2_attention_weight_max": 33.509613037109375,
      "activations/layer2_attention_weight_min": -33.26258087158203,
      "activations/layer3_attention_weight_max": 91.27008819580078,
      "activations/layer3_attention_weight_min": -97.01641082763672,
      "activations/layer4_attention_weight_max": 96.30052185058594,
      "activations/layer4_attention_weight_min": -94.4309310913086,
      "activations/layer5_attention_weight_max": 68.97550201416016,
      "activations/layer5_attention_weight_min": -78.2895278930664,
      "activations/layer6_attention_weight_max": 49.54782485961914,
      "activations/layer6_attention_weight_min": -55.389434814453125,
      "activations/layer7_attention_weight_max": 67.85585021972656,
      "activations/layer7_attention_weight_min": -63.866939544677734,
      "activations/layer8_attention_weight_max": 44.55463790893555,
      "activations/layer8_attention_weight_min": -49.09246826171875,
      "activations/layer9_attention_weight_max": 56.04298782348633,
      "activations/layer9_attention_weight_min": -45.29304885864258,
      "epoch": 13.7,
      "learning_rate": 6.22530303030303e-05,
      "loss": 2.7679,
      "step": 235850
    },
    {
      "activations/layer0_attention_weight_max": 16.822025299072266,
      "activations/layer0_attention_weight_min": -14.151354789733887,
      "activations/layer10_attention_weight_max": 32.575130462646484,
      "activations/layer10_attention_weight_min": -33.24271774291992,
      "activations/layer11_attention_weight_max": 31.200592041015625,
      "activations/layer11_attention_weight_min": -31.413930892944336,
      "activations/layer12_attention_weight_max": 17.638431549072266,
      "activations/layer12_attention_weight_min": -24.524139404296875,
      "activations/layer13_attention_weight_max": 41.152809143066406,
      "activations/layer13_attention_weight_min": -28.6184139251709,
      "activations/layer14_attention_weight_max": 34.85808181762695,
      "activations/layer14_attention_weight_min": -31.071138381958008,
      "activations/layer15_attention_weight_max": 34.0197639465332,
      "activations/layer15_attention_weight_min": -29.32273292541504,
      "activations/layer16_attention_weight_max": 32.73590087890625,
      "activations/layer16_attention_weight_min": -31.97716522216797,
      "activations/layer17_attention_weight_max": 50.89942169189453,
      "activations/layer17_attention_weight_min": -44.65180206298828,
      "activations/layer18_attention_weight_max": 45.25345993041992,
      "activations/layer18_attention_weight_min": -41.16162109375,
      "activations/layer19_attention_weight_max": 23.26988983154297,
      "activations/layer19_attention_weight_min": -22.468158721923828,
      "activations/layer1_attention_weight_max": 17.188100814819336,
      "activations/layer1_attention_weight_min": -14.714240074157715,
      "activations/layer20_attention_weight_max": 23.55158233642578,
      "activations/layer20_attention_weight_min": -20.874786376953125,
      "activations/layer21_attention_weight_max": 38.254615783691406,
      "activations/layer21_attention_weight_min": -20.670185089111328,
      "activations/layer22_attention_weight_max": 30.52643394470215,
      "activations/layer22_attention_weight_min": -24.419654846191406,
      "activations/layer23_attention_weight_max": 33.93637466430664,
      "activations/layer23_attention_weight_min": -24.456981658935547,
      "activations/layer2_attention_weight_max": 29.870468139648438,
      "activations/layer2_attention_weight_min": -29.368057250976562,
      "activations/layer3_attention_weight_max": 90.62803649902344,
      "activations/layer3_attention_weight_min": -90.77662658691406,
      "activations/layer4_attention_weight_max": 92.8634262084961,
      "activations/layer4_attention_weight_min": -94.04141998291016,
      "activations/layer5_attention_weight_max": 70.65192413330078,
      "activations/layer5_attention_weight_min": -75.2362060546875,
      "activations/layer6_attention_weight_max": 48.69234085083008,
      "activations/layer6_attention_weight_min": -52.96830368041992,
      "activations/layer7_attention_weight_max": 71.67009735107422,
      "activations/layer7_attention_weight_min": -64.1705551147461,
      "activations/layer8_attention_weight_max": 43.63590621948242,
      "activations/layer8_attention_weight_min": -46.65629959106445,
      "activations/layer9_attention_weight_max": 50.42828369140625,
      "activations/layer9_attention_weight_min": -44.642696380615234,
      "epoch": 13.71,
      "learning_rate": 6.223446969696969e-05,
      "loss": 2.764,
      "step": 235900
    },
    {
      "activations/layer0_attention_weight_max": 16.84839630126953,
      "activations/layer0_attention_weight_min": -14.678265571594238,
      "activations/layer10_attention_weight_max": 37.763492584228516,
      "activations/layer10_attention_weight_min": -36.67763900756836,
      "activations/layer11_attention_weight_max": 34.62871170043945,
      "activations/layer11_attention_weight_min": -32.02641296386719,
      "activations/layer12_attention_weight_max": 22.61241912841797,
      "activations/layer12_attention_weight_min": -26.884605407714844,
      "activations/layer13_attention_weight_max": 36.58970260620117,
      "activations/layer13_attention_weight_min": -28.599857330322266,
      "activations/layer14_attention_weight_max": 36.2581787109375,
      "activations/layer14_attention_weight_min": -29.46114158630371,
      "activations/layer15_attention_weight_max": 32.22364044189453,
      "activations/layer15_attention_weight_min": -28.53508758544922,
      "activations/layer16_attention_weight_max": 32.83203125,
      "activations/layer16_attention_weight_min": -31.323869705200195,
      "activations/layer17_attention_weight_max": 53.9197883605957,
      "activations/layer17_attention_weight_min": -49.34284210205078,
      "activations/layer18_attention_weight_max": 48.74059295654297,
      "activations/layer18_attention_weight_min": -41.10454559326172,
      "activations/layer19_attention_weight_max": 21.975431442260742,
      "activations/layer19_attention_weight_min": -21.196683883666992,
      "activations/layer1_attention_weight_max": 18.23908233642578,
      "activations/layer1_attention_weight_min": -16.208024978637695,
      "activations/layer20_attention_weight_max": 24.238252639770508,
      "activations/layer20_attention_weight_min": -26.992748260498047,
      "activations/layer21_attention_weight_max": 35.6415901184082,
      "activations/layer21_attention_weight_min": -22.150726318359375,
      "activations/layer22_attention_weight_max": 27.720441818237305,
      "activations/layer22_attention_weight_min": -26.35111427307129,
      "activations/layer23_attention_weight_max": 33.174415588378906,
      "activations/layer23_attention_weight_min": -25.28923797607422,
      "activations/layer2_attention_weight_max": 34.25510787963867,
      "activations/layer2_attention_weight_min": -32.948402404785156,
      "activations/layer3_attention_weight_max": 97.29081726074219,
      "activations/layer3_attention_weight_min": -102.83274841308594,
      "activations/layer4_attention_weight_max": 96.13713836669922,
      "activations/layer4_attention_weight_min": -95.57209777832031,
      "activations/layer5_attention_weight_max": 73.25379943847656,
      "activations/layer5_attention_weight_min": -80.21774291992188,
      "activations/layer6_attention_weight_max": 54.30813980102539,
      "activations/layer6_attention_weight_min": -53.37968063354492,
      "activations/layer7_attention_weight_max": 64.78936767578125,
      "activations/layer7_attention_weight_min": -68.69404602050781,
      "activations/layer8_attention_weight_max": 47.277923583984375,
      "activations/layer8_attention_weight_min": -49.357059478759766,
      "activations/layer9_attention_weight_max": 47.89550018310547,
      "activations/layer9_attention_weight_min": -45.82637405395508,
      "epoch": 13.71,
      "learning_rate": 6.22155303030303e-05,
      "loss": 2.7752,
      "step": 235950
    },
    {
      "activations/layer0_attention_weight_max": 15.99891471862793,
      "activations/layer0_attention_weight_min": -14.371899604797363,
      "activations/layer10_attention_weight_max": 39.33812713623047,
      "activations/layer10_attention_weight_min": -38.500518798828125,
      "activations/layer11_attention_weight_max": 34.49310302734375,
      "activations/layer11_attention_weight_min": -34.57691955566406,
      "activations/layer12_attention_weight_max": 20.58186912536621,
      "activations/layer12_attention_weight_min": -25.69094467163086,
      "activations/layer13_attention_weight_max": 39.531795501708984,
      "activations/layer13_attention_weight_min": -32.333580017089844,
      "activations/layer14_attention_weight_max": 34.650203704833984,
      "activations/layer14_attention_weight_min": -28.837636947631836,
      "activations/layer15_attention_weight_max": 33.783355712890625,
      "activations/layer15_attention_weight_min": -27.748706817626953,
      "activations/layer16_attention_weight_max": 29.743656158447266,
      "activations/layer16_attention_weight_min": -29.787364959716797,
      "activations/layer17_attention_weight_max": 50.787078857421875,
      "activations/layer17_attention_weight_min": -44.718143463134766,
      "activations/layer18_attention_weight_max": 44.71119689941406,
      "activations/layer18_attention_weight_min": -38.52155685424805,
      "activations/layer19_attention_weight_max": 21.697738647460938,
      "activations/layer19_attention_weight_min": -22.408077239990234,
      "activations/layer1_attention_weight_max": 16.569725036621094,
      "activations/layer1_attention_weight_min": -16.20486068725586,
      "activations/layer20_attention_weight_max": 23.381925582885742,
      "activations/layer20_attention_weight_min": -22.75535011291504,
      "activations/layer21_attention_weight_max": 36.92219924926758,
      "activations/layer21_attention_weight_min": -22.19392204284668,
      "activations/layer22_attention_weight_max": 36.24396896362305,
      "activations/layer22_attention_weight_min": -27.404775619506836,
      "activations/layer23_attention_weight_max": 37.42477798461914,
      "activations/layer23_attention_weight_min": -24.159034729003906,
      "activations/layer2_attention_weight_max": 33.25132751464844,
      "activations/layer2_attention_weight_min": -32.21862030029297,
      "activations/layer3_attention_weight_max": 93.60078430175781,
      "activations/layer3_attention_weight_min": -97.66285705566406,
      "activations/layer4_attention_weight_max": 97.95699310302734,
      "activations/layer4_attention_weight_min": -99.89807891845703,
      "activations/layer5_attention_weight_max": 74.02581787109375,
      "activations/layer5_attention_weight_min": -79.74273681640625,
      "activations/layer6_attention_weight_max": 54.173397064208984,
      "activations/layer6_attention_weight_min": -55.68645095825195,
      "activations/layer7_attention_weight_max": 70.56893157958984,
      "activations/layer7_attention_weight_min": -70.17532348632812,
      "activations/layer8_attention_weight_max": 50.14937210083008,
      "activations/layer8_attention_weight_min": -50.642555236816406,
      "activations/layer9_attention_weight_max": 52.1078987121582,
      "activations/layer9_attention_weight_min": -47.761192321777344,
      "epoch": 13.71,
      "learning_rate": 6.21965909090909e-05,
      "loss": 2.7789,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4978,
      "eval_samples_per_second": 505.309,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4978,
      "eval_openwebtext_samples_per_second": 505.309,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 2.0272,
      "eval_wikitext_samples_per_second": 224.944,
      "step": 236000
    },
    {
      "epoch": 13.71,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.5312,
      "eval_lambada_samples_per_second": 510.849,
      "step": 236000
    },
    {
      "activations/layer0_attention_weight_max": 16.342756271362305,
      "activations/layer0_attention_weight_min": -14.452571868896484,
      "activations/layer10_attention_weight_max": 38.42594909667969,
      "activations/layer10_attention_weight_min": -35.77967071533203,
      "activations/layer11_attention_weight_max": 33.269100189208984,
      "activations/layer11_attention_weight_min": -34.01450729370117,
      "activations/layer12_attention_weight_max": 19.158376693725586,
      "activations/layer12_attention_weight_min": -25.282413482666016,
      "activations/layer13_attention_weight_max": 33.346961975097656,
      "activations/layer13_attention_weight_min": -26.786388397216797,
      "activations/layer14_attention_weight_max": 36.77088928222656,
      "activations/layer14_attention_weight_min": -30.37954330444336,
      "activations/layer15_attention_weight_max": 31.633499145507812,
      "activations/layer15_attention_weight_min": -29.7595157623291,
      "activations/layer16_attention_weight_max": 31.556119918823242,
      "activations/layer16_attention_weight_min": -30.527502059936523,
      "activations/layer17_attention_weight_max": 48.30266571044922,
      "activations/layer17_attention_weight_min": -44.67198944091797,
      "activations/layer18_attention_weight_max": 44.27103042602539,
      "activations/layer18_attention_weight_min": -38.587310791015625,
      "activations/layer19_attention_weight_max": 21.740589141845703,
      "activations/layer19_attention_weight_min": -25.159406661987305,
      "activations/layer1_attention_weight_max": 16.531494140625,
      "activations/layer1_attention_weight_min": -16.565664291381836,
      "activations/layer20_attention_weight_max": 23.719655990600586,
      "activations/layer20_attention_weight_min": -25.615013122558594,
      "activations/layer21_attention_weight_max": 38.08734130859375,
      "activations/layer21_attention_weight_min": -28.18276023864746,
      "activations/layer22_attention_weight_max": 29.23908233642578,
      "activations/layer22_attention_weight_min": -26.592206954956055,
      "activations/layer23_attention_weight_max": 33.333839416503906,
      "activations/layer23_attention_weight_min": -28.390705108642578,
      "activations/layer2_attention_weight_max": 32.32722473144531,
      "activations/layer2_attention_weight_min": -30.379497528076172,
      "activations/layer3_attention_weight_max": 88.12079620361328,
      "activations/layer3_attention_weight_min": -89.28298950195312,
      "activations/layer4_attention_weight_max": 94.11785125732422,
      "activations/layer4_attention_weight_min": -89.87989807128906,
      "activations/layer5_attention_weight_max": 70.33370971679688,
      "activations/layer5_attention_weight_min": -76.07022094726562,
      "activations/layer6_attention_weight_max": 52.20915222167969,
      "activations/layer6_attention_weight_min": -51.315391540527344,
      "activations/layer7_attention_weight_max": 69.59027862548828,
      "activations/layer7_attention_weight_min": -63.845462799072266,
      "activations/layer8_attention_weight_max": 44.68351364135742,
      "activations/layer8_attention_weight_min": -48.20130157470703,
      "activations/layer9_attention_weight_max": 43.14516830444336,
      "activations/layer9_attention_weight_min": -45.321617126464844,
      "epoch": 13.72,
      "learning_rate": 6.21776515151515e-05,
      "loss": 2.764,
      "step": 236050
    },
    {
      "activations/layer0_attention_weight_max": 17.44698715209961,
      "activations/layer0_attention_weight_min": -14.87153434753418,
      "activations/layer10_attention_weight_max": 36.92180252075195,
      "activations/layer10_attention_weight_min": -36.73598098754883,
      "activations/layer11_attention_weight_max": 32.31789016723633,
      "activations/layer11_attention_weight_min": -34.82683563232422,
      "activations/layer12_attention_weight_max": 19.280710220336914,
      "activations/layer12_attention_weight_min": -24.12879753112793,
      "activations/layer13_attention_weight_max": 41.42632293701172,
      "activations/layer13_attention_weight_min": -29.183748245239258,
      "activations/layer14_attention_weight_max": 39.81778335571289,
      "activations/layer14_attention_weight_min": -31.19303321838379,
      "activations/layer15_attention_weight_max": 34.800315856933594,
      "activations/layer15_attention_weight_min": -30.9210147857666,
      "activations/layer16_attention_weight_max": 32.93925857543945,
      "activations/layer16_attention_weight_min": -30.87670135498047,
      "activations/layer17_attention_weight_max": 54.03845977783203,
      "activations/layer17_attention_weight_min": -45.10708999633789,
      "activations/layer18_attention_weight_max": 50.72622299194336,
      "activations/layer18_attention_weight_min": -39.898685455322266,
      "activations/layer19_attention_weight_max": 21.691957473754883,
      "activations/layer19_attention_weight_min": -23.465911865234375,
      "activations/layer1_attention_weight_max": 17.446786880493164,
      "activations/layer1_attention_weight_min": -16.30633544921875,
      "activations/layer20_attention_weight_max": 22.779216766357422,
      "activations/layer20_attention_weight_min": -20.615406036376953,
      "activations/layer21_attention_weight_max": 36.47314453125,
      "activations/layer21_attention_weight_min": -23.531339645385742,
      "activations/layer22_attention_weight_max": 30.921859741210938,
      "activations/layer22_attention_weight_min": -25.05954360961914,
      "activations/layer23_attention_weight_max": 39.65251159667969,
      "activations/layer23_attention_weight_min": -22.848846435546875,
      "activations/layer2_attention_weight_max": 32.99009704589844,
      "activations/layer2_attention_weight_min": -30.419034957885742,
      "activations/layer3_attention_weight_max": 91.48530578613281,
      "activations/layer3_attention_weight_min": -98.0844955444336,
      "activations/layer4_attention_weight_max": 96.2293930053711,
      "activations/layer4_attention_weight_min": -98.8487319946289,
      "activations/layer5_attention_weight_max": 75.5322265625,
      "activations/layer5_attention_weight_min": -84.14411926269531,
      "activations/layer6_attention_weight_max": 53.34334945678711,
      "activations/layer6_attention_weight_min": -54.62630081176758,
      "activations/layer7_attention_weight_max": 67.52703094482422,
      "activations/layer7_attention_weight_min": -68.98593139648438,
      "activations/layer8_attention_weight_max": 49.80216598510742,
      "activations/layer8_attention_weight_min": -51.44529724121094,
      "activations/layer9_attention_weight_max": 50.969661712646484,
      "activations/layer9_attention_weight_min": -49.48318099975586,
      "epoch": 13.72,
      "learning_rate": 6.21587121212121e-05,
      "loss": 2.7835,
      "step": 236100
    },
    {
      "activations/layer0_attention_weight_max": 17.914445877075195,
      "activations/layer0_attention_weight_min": -14.7709379196167,
      "activations/layer10_attention_weight_max": 36.59660720825195,
      "activations/layer10_attention_weight_min": -34.903602600097656,
      "activations/layer11_attention_weight_max": 31.590789794921875,
      "activations/layer11_attention_weight_min": -32.01740264892578,
      "activations/layer12_attention_weight_max": 17.845666885375977,
      "activations/layer12_attention_weight_min": -26.650293350219727,
      "activations/layer13_attention_weight_max": 30.54684829711914,
      "activations/layer13_attention_weight_min": -27.094005584716797,
      "activations/layer14_attention_weight_max": 37.156803131103516,
      "activations/layer14_attention_weight_min": -33.2324333190918,
      "activations/layer15_attention_weight_max": 34.51545333862305,
      "activations/layer15_attention_weight_min": -32.95600509643555,
      "activations/layer16_attention_weight_max": 33.9488410949707,
      "activations/layer16_attention_weight_min": -32.04684066772461,
      "activations/layer17_attention_weight_max": 53.19822311401367,
      "activations/layer17_attention_weight_min": -45.83848571777344,
      "activations/layer18_attention_weight_max": 50.09284973144531,
      "activations/layer18_attention_weight_min": -42.57665252685547,
      "activations/layer19_attention_weight_max": 24.796184539794922,
      "activations/layer19_attention_weight_min": -24.558048248291016,
      "activations/layer1_attention_weight_max": 16.52094078063965,
      "activations/layer1_attention_weight_min": -14.439628601074219,
      "activations/layer20_attention_weight_max": 26.21212387084961,
      "activations/layer20_attention_weight_min": -24.275827407836914,
      "activations/layer21_attention_weight_max": 45.87069320678711,
      "activations/layer21_attention_weight_min": -27.514066696166992,
      "activations/layer22_attention_weight_max": 32.23349380493164,
      "activations/layer22_attention_weight_min": -30.5662841796875,
      "activations/layer23_attention_weight_max": 39.2077751159668,
      "activations/layer23_attention_weight_min": -25.14394760131836,
      "activations/layer2_attention_weight_max": 33.53162384033203,
      "activations/layer2_attention_weight_min": -30.91261863708496,
      "activations/layer3_attention_weight_max": 92.79376983642578,
      "activations/layer3_attention_weight_min": -95.38902282714844,
      "activations/layer4_attention_weight_max": 94.50851440429688,
      "activations/layer4_attention_weight_min": -92.99187469482422,
      "activations/layer5_attention_weight_max": 69.17138671875,
      "activations/layer5_attention_weight_min": -81.81135559082031,
      "activations/layer6_attention_weight_max": 52.155635833740234,
      "activations/layer6_attention_weight_min": -49.4183235168457,
      "activations/layer7_attention_weight_max": 62.81467819213867,
      "activations/layer7_attention_weight_min": -62.6197509765625,
      "activations/layer8_attention_weight_max": 46.753028869628906,
      "activations/layer8_attention_weight_min": -47.643863677978516,
      "activations/layer9_attention_weight_max": 45.0053825378418,
      "activations/layer9_attention_weight_min": -45.285316467285156,
      "epoch": 13.72,
      "learning_rate": 6.213977272727272e-05,
      "loss": 2.7636,
      "step": 236150
    },
    {
      "activations/layer0_attention_weight_max": 16.511268615722656,
      "activations/layer0_attention_weight_min": -15.053071022033691,
      "activations/layer10_attention_weight_max": 36.689308166503906,
      "activations/layer10_attention_weight_min": -36.66056442260742,
      "activations/layer11_attention_weight_max": 34.166839599609375,
      "activations/layer11_attention_weight_min": -36.09019088745117,
      "activations/layer12_attention_weight_max": 19.953927993774414,
      "activations/layer12_attention_weight_min": -28.190351486206055,
      "activations/layer13_attention_weight_max": 45.39577102661133,
      "activations/layer13_attention_weight_min": -27.724855422973633,
      "activations/layer14_attention_weight_max": 49.22412109375,
      "activations/layer14_attention_weight_min": -32.46278762817383,
      "activations/layer15_attention_weight_max": 37.16038513183594,
      "activations/layer15_attention_weight_min": -33.704959869384766,
      "activations/layer16_attention_weight_max": 36.711708068847656,
      "activations/layer16_attention_weight_min": -33.04571533203125,
      "activations/layer17_attention_weight_max": 60.0255012512207,
      "activations/layer17_attention_weight_min": -50.77699279785156,
      "activations/layer18_attention_weight_max": 56.01670455932617,
      "activations/layer18_attention_weight_min": -39.478485107421875,
      "activations/layer19_attention_weight_max": 25.600873947143555,
      "activations/layer19_attention_weight_min": -23.97801971435547,
      "activations/layer1_attention_weight_max": 16.88516616821289,
      "activations/layer1_attention_weight_min": -14.960227966308594,
      "activations/layer20_attention_weight_max": 30.403364181518555,
      "activations/layer20_attention_weight_min": -22.716773986816406,
      "activations/layer21_attention_weight_max": 45.402034759521484,
      "activations/layer21_attention_weight_min": -23.61128807067871,
      "activations/layer22_attention_weight_max": 36.60074234008789,
      "activations/layer22_attention_weight_min": -30.30645751953125,
      "activations/layer23_attention_weight_max": 46.96086883544922,
      "activations/layer23_attention_weight_min": -23.67041015625,
      "activations/layer2_attention_weight_max": 31.224058151245117,
      "activations/layer2_attention_weight_min": -31.330623626708984,
      "activations/layer3_attention_weight_max": 94.6710433959961,
      "activations/layer3_attention_weight_min": -95.70108795166016,
      "activations/layer4_attention_weight_max": 95.2357406616211,
      "activations/layer4_attention_weight_min": -93.94182586669922,
      "activations/layer5_attention_weight_max": 73.91078186035156,
      "activations/layer5_attention_weight_min": -77.58831024169922,
      "activations/layer6_attention_weight_max": 53.418338775634766,
      "activations/layer6_attention_weight_min": -51.97940444946289,
      "activations/layer7_attention_weight_max": 66.73651885986328,
      "activations/layer7_attention_weight_min": -67.82508850097656,
      "activations/layer8_attention_weight_max": 46.99220275878906,
      "activations/layer8_attention_weight_min": -47.096923828125,
      "activations/layer9_attention_weight_max": 44.92107391357422,
      "activations/layer9_attention_weight_min": -48.76095962524414,
      "epoch": 13.72,
      "learning_rate": 6.212083333333332e-05,
      "loss": 2.7633,
      "step": 236200
    },
    {
      "activations/layer0_attention_weight_max": 16.452524185180664,
      "activations/layer0_attention_weight_min": -14.486520767211914,
      "activations/layer10_attention_weight_max": 38.82866668701172,
      "activations/layer10_attention_weight_min": -37.2902717590332,
      "activations/layer11_attention_weight_max": 33.90976333618164,
      "activations/layer11_attention_weight_min": -34.1390380859375,
      "activations/layer12_attention_weight_max": 21.292821884155273,
      "activations/layer12_attention_weight_min": -23.91703224182129,
      "activations/layer13_attention_weight_max": 41.18883514404297,
      "activations/layer13_attention_weight_min": -28.664669036865234,
      "activations/layer14_attention_weight_max": 37.056819915771484,
      "activations/layer14_attention_weight_min": -29.27980613708496,
      "activations/layer15_attention_weight_max": 32.167213439941406,
      "activations/layer15_attention_weight_min": -28.484901428222656,
      "activations/layer16_attention_weight_max": 32.28038787841797,
      "activations/layer16_attention_weight_min": -29.924137115478516,
      "activations/layer17_attention_weight_max": 50.406532287597656,
      "activations/layer17_attention_weight_min": -46.30363082885742,
      "activations/layer18_attention_weight_max": 46.61800003051758,
      "activations/layer18_attention_weight_min": -38.20380401611328,
      "activations/layer19_attention_weight_max": 23.4892635345459,
      "activations/layer19_attention_weight_min": -21.243825912475586,
      "activations/layer1_attention_weight_max": 17.06575584411621,
      "activations/layer1_attention_weight_min": -15.248661041259766,
      "activations/layer20_attention_weight_max": 21.791929244995117,
      "activations/layer20_attention_weight_min": -20.685977935791016,
      "activations/layer21_attention_weight_max": 39.947296142578125,
      "activations/layer21_attention_weight_min": -23.742191314697266,
      "activations/layer22_attention_weight_max": 31.93421173095703,
      "activations/layer22_attention_weight_min": -24.2307186126709,
      "activations/layer23_attention_weight_max": 38.89812088012695,
      "activations/layer23_attention_weight_min": -24.417509078979492,
      "activations/layer2_attention_weight_max": 32.088050842285156,
      "activations/layer2_attention_weight_min": -30.01776885986328,
      "activations/layer3_attention_weight_max": 95.11815643310547,
      "activations/layer3_attention_weight_min": -92.3365478515625,
      "activations/layer4_attention_weight_max": 93.66446685791016,
      "activations/layer4_attention_weight_min": -94.34935760498047,
      "activations/layer5_attention_weight_max": 70.0280990600586,
      "activations/layer5_attention_weight_min": -79.42385864257812,
      "activations/layer6_attention_weight_max": 49.30863952636719,
      "activations/layer6_attention_weight_min": -53.334495544433594,
      "activations/layer7_attention_weight_max": 64.67668151855469,
      "activations/layer7_attention_weight_min": -67.88115692138672,
      "activations/layer8_attention_weight_max": 47.21809768676758,
      "activations/layer8_attention_weight_min": -54.98967361450195,
      "activations/layer9_attention_weight_max": 47.24370574951172,
      "activations/layer9_attention_weight_min": -49.89412307739258,
      "epoch": 13.73,
      "learning_rate": 6.210227272727271e-05,
      "loss": 2.7618,
      "step": 236250
    },
    {
      "activations/layer0_attention_weight_max": 15.316155433654785,
      "activations/layer0_attention_weight_min": -14.520305633544922,
      "activations/layer10_attention_weight_max": 36.32167053222656,
      "activations/layer10_attention_weight_min": -38.04900360107422,
      "activations/layer11_attention_weight_max": 32.729713439941406,
      "activations/layer11_attention_weight_min": -34.559898376464844,
      "activations/layer12_attention_weight_max": 20.519882202148438,
      "activations/layer12_attention_weight_min": -33.0888786315918,
      "activations/layer13_attention_weight_max": 41.090293884277344,
      "activations/layer13_attention_weight_min": -29.868730545043945,
      "activations/layer14_attention_weight_max": 35.09885025024414,
      "activations/layer14_attention_weight_min": -30.66845703125,
      "activations/layer15_attention_weight_max": 31.36752700805664,
      "activations/layer15_attention_weight_min": -29.08089256286621,
      "activations/layer16_attention_weight_max": 31.23163604736328,
      "activations/layer16_attention_weight_min": -30.383955001831055,
      "activations/layer17_attention_weight_max": 52.136070251464844,
      "activations/layer17_attention_weight_min": -44.55474090576172,
      "activations/layer18_attention_weight_max": 46.201995849609375,
      "activations/layer18_attention_weight_min": -39.02742385864258,
      "activations/layer19_attention_weight_max": 22.692106246948242,
      "activations/layer19_attention_weight_min": -21.61699867248535,
      "activations/layer1_attention_weight_max": 17.081974029541016,
      "activations/layer1_attention_weight_min": -13.937715530395508,
      "activations/layer20_attention_weight_max": 23.972867965698242,
      "activations/layer20_attention_weight_min": -22.209566116333008,
      "activations/layer21_attention_weight_max": 38.22512435913086,
      "activations/layer21_attention_weight_min": -23.14464569091797,
      "activations/layer22_attention_weight_max": 30.501094818115234,
      "activations/layer22_attention_weight_min": -26.096412658691406,
      "activations/layer23_attention_weight_max": 38.91471481323242,
      "activations/layer23_attention_weight_min": -23.028484344482422,
      "activations/layer2_attention_weight_max": 32.66101837158203,
      "activations/layer2_attention_weight_min": -29.97246551513672,
      "activations/layer3_attention_weight_max": 95.06584930419922,
      "activations/layer3_attention_weight_min": -97.11412048339844,
      "activations/layer4_attention_weight_max": 95.7385025024414,
      "activations/layer4_attention_weight_min": -95.6968765258789,
      "activations/layer5_attention_weight_max": 71.71009826660156,
      "activations/layer5_attention_weight_min": -81.89763641357422,
      "activations/layer6_attention_weight_max": 50.82503128051758,
      "activations/layer6_attention_weight_min": -53.302940368652344,
      "activations/layer7_attention_weight_max": 68.93939208984375,
      "activations/layer7_attention_weight_min": -63.884761810302734,
      "activations/layer8_attention_weight_max": 44.76930618286133,
      "activations/layer8_attention_weight_min": -51.521217346191406,
      "activations/layer9_attention_weight_max": 49.41328048706055,
      "activations/layer9_attention_weight_min": -48.257789611816406,
      "epoch": 13.73,
      "learning_rate": 6.208333333333333e-05,
      "loss": 2.7705,
      "step": 236300
    },
    {
      "activations/layer0_attention_weight_max": 16.282934188842773,
      "activations/layer0_attention_weight_min": -14.302968978881836,
      "activations/layer10_attention_weight_max": 37.919410705566406,
      "activations/layer10_attention_weight_min": -39.38766098022461,
      "activations/layer11_attention_weight_max": 35.747886657714844,
      "activations/layer11_attention_weight_min": -36.865440368652344,
      "activations/layer12_attention_weight_max": 19.241254806518555,
      "activations/layer12_attention_weight_min": -24.345476150512695,
      "activations/layer13_attention_weight_max": 34.635562896728516,
      "activations/layer13_attention_weight_min": -31.98895835876465,
      "activations/layer14_attention_weight_max": 34.95162582397461,
      "activations/layer14_attention_weight_min": -30.54249382019043,
      "activations/layer15_attention_weight_max": 32.2422981262207,
      "activations/layer15_attention_weight_min": -30.1552677154541,
      "activations/layer16_attention_weight_max": 31.82524299621582,
      "activations/layer16_attention_weight_min": -30.899450302124023,
      "activations/layer17_attention_weight_max": 49.11442184448242,
      "activations/layer17_attention_weight_min": -43.26043701171875,
      "activations/layer18_attention_weight_max": 46.60515594482422,
      "activations/layer18_attention_weight_min": -38.54661560058594,
      "activations/layer19_attention_weight_max": 20.33668327331543,
      "activations/layer19_attention_weight_min": -22.043704986572266,
      "activations/layer1_attention_weight_max": 15.746451377868652,
      "activations/layer1_attention_weight_min": -14.286890029907227,
      "activations/layer20_attention_weight_max": 21.217330932617188,
      "activations/layer20_attention_weight_min": -22.452627182006836,
      "activations/layer21_attention_weight_max": 32.03160858154297,
      "activations/layer21_attention_weight_min": -21.60995864868164,
      "activations/layer22_attention_weight_max": 28.204025268554688,
      "activations/layer22_attention_weight_min": -25.112268447875977,
      "activations/layer23_attention_weight_max": 37.95899963378906,
      "activations/layer23_attention_weight_min": -23.597196578979492,
      "activations/layer2_attention_weight_max": 32.882537841796875,
      "activations/layer2_attention_weight_min": -31.061981201171875,
      "activations/layer3_attention_weight_max": 96.71138763427734,
      "activations/layer3_attention_weight_min": -95.68879699707031,
      "activations/layer4_attention_weight_max": 94.31725311279297,
      "activations/layer4_attention_weight_min": -95.68851470947266,
      "activations/layer5_attention_weight_max": 71.91221618652344,
      "activations/layer5_attention_weight_min": -76.78887939453125,
      "activations/layer6_attention_weight_max": 53.66627502441406,
      "activations/layer6_attention_weight_min": -53.83869552612305,
      "activations/layer7_attention_weight_max": 68.8392333984375,
      "activations/layer7_attention_weight_min": -68.27802276611328,
      "activations/layer8_attention_weight_max": 50.85026931762695,
      "activations/layer8_attention_weight_min": -55.02001190185547,
      "activations/layer9_attention_weight_max": 50.3262825012207,
      "activations/layer9_attention_weight_min": -52.66265869140625,
      "epoch": 13.73,
      "learning_rate": 6.206439393939393e-05,
      "loss": 2.7664,
      "step": 236350
    },
    {
      "activations/layer0_attention_weight_max": 15.930493354797363,
      "activations/layer0_attention_weight_min": -15.12711238861084,
      "activations/layer10_attention_weight_max": 39.61628341674805,
      "activations/layer10_attention_weight_min": -38.3187141418457,
      "activations/layer11_attention_weight_max": 34.70463562011719,
      "activations/layer11_attention_weight_min": -34.72538757324219,
      "activations/layer12_attention_weight_max": 22.912473678588867,
      "activations/layer12_attention_weight_min": -23.61899757385254,
      "activations/layer13_attention_weight_max": 39.28410339355469,
      "activations/layer13_attention_weight_min": -31.665481567382812,
      "activations/layer14_attention_weight_max": 35.56824493408203,
      "activations/layer14_attention_weight_min": -29.06487464904785,
      "activations/layer15_attention_weight_max": 33.02547836303711,
      "activations/layer15_attention_weight_min": -29.199277877807617,
      "activations/layer16_attention_weight_max": 30.266809463500977,
      "activations/layer16_attention_weight_min": -29.351776123046875,
      "activations/layer17_attention_weight_max": 53.00493240356445,
      "activations/layer17_attention_weight_min": -44.0071907043457,
      "activations/layer18_attention_weight_max": 45.484657287597656,
      "activations/layer18_attention_weight_min": -35.999473571777344,
      "activations/layer19_attention_weight_max": 20.106935501098633,
      "activations/layer19_attention_weight_min": -20.434925079345703,
      "activations/layer1_attention_weight_max": 18.347841262817383,
      "activations/layer1_attention_weight_min": -13.971590995788574,
      "activations/layer20_attention_weight_max": 20.15924072265625,
      "activations/layer20_attention_weight_min": -21.67099380493164,
      "activations/layer21_attention_weight_max": 30.949663162231445,
      "activations/layer21_attention_weight_min": -20.8929386138916,
      "activations/layer22_attention_weight_max": 26.475101470947266,
      "activations/layer22_attention_weight_min": -26.86861228942871,
      "activations/layer23_attention_weight_max": 34.82482147216797,
      "activations/layer23_attention_weight_min": -23.72240447998047,
      "activations/layer2_attention_weight_max": 31.809940338134766,
      "activations/layer2_attention_weight_min": -30.081836700439453,
      "activations/layer3_attention_weight_max": 93.80424499511719,
      "activations/layer3_attention_weight_min": -89.56124877929688,
      "activations/layer4_attention_weight_max": 95.56981658935547,
      "activations/layer4_attention_weight_min": -89.87895202636719,
      "activations/layer5_attention_weight_max": 73.06364440917969,
      "activations/layer5_attention_weight_min": -78.74189758300781,
      "activations/layer6_attention_weight_max": 54.08622360229492,
      "activations/layer6_attention_weight_min": -52.41710662841797,
      "activations/layer7_attention_weight_max": 69.11475372314453,
      "activations/layer7_attention_weight_min": -64.37486267089844,
      "activations/layer8_attention_weight_max": 48.627933502197266,
      "activations/layer8_attention_weight_min": -49.52882385253906,
      "activations/layer9_attention_weight_max": 47.39594268798828,
      "activations/layer9_attention_weight_min": -47.399539947509766,
      "epoch": 13.74,
      "learning_rate": 6.204545454545455e-05,
      "loss": 2.7856,
      "step": 236400
    },
    {
      "activations/layer0_attention_weight_max": 16.35079002380371,
      "activations/layer0_attention_weight_min": -14.484705924987793,
      "activations/layer10_attention_weight_max": 36.43722915649414,
      "activations/layer10_attention_weight_min": -34.10202407836914,
      "activations/layer11_attention_weight_max": 31.74385643005371,
      "activations/layer11_attention_weight_min": -32.579078674316406,
      "activations/layer12_attention_weight_max": 18.83804702758789,
      "activations/layer12_attention_weight_min": -23.548505783081055,
      "activations/layer13_attention_weight_max": 33.90293502807617,
      "activations/layer13_attention_weight_min": -29.008209228515625,
      "activations/layer14_attention_weight_max": 35.65171813964844,
      "activations/layer14_attention_weight_min": -29.78510284423828,
      "activations/layer15_attention_weight_max": 31.509366989135742,
      "activations/layer15_attention_weight_min": -29.787734985351562,
      "activations/layer16_attention_weight_max": 32.01343536376953,
      "activations/layer16_attention_weight_min": -30.692520141601562,
      "activations/layer17_attention_weight_max": 53.63624572753906,
      "activations/layer17_attention_weight_min": -48.205284118652344,
      "activations/layer18_attention_weight_max": 48.98542404174805,
      "activations/layer18_attention_weight_min": -42.95773696899414,
      "activations/layer19_attention_weight_max": 21.61817741394043,
      "activations/layer19_attention_weight_min": -21.284101486206055,
      "activations/layer1_attention_weight_max": 17.27594566345215,
      "activations/layer1_attention_weight_min": -13.42977523803711,
      "activations/layer20_attention_weight_max": 20.041276931762695,
      "activations/layer20_attention_weight_min": -23.11908721923828,
      "activations/layer21_attention_weight_max": 32.88966751098633,
      "activations/layer21_attention_weight_min": -22.666624069213867,
      "activations/layer22_attention_weight_max": 29.515249252319336,
      "activations/layer22_attention_weight_min": -28.32664680480957,
      "activations/layer23_attention_weight_max": 38.765846252441406,
      "activations/layer23_attention_weight_min": -25.32935333251953,
      "activations/layer2_attention_weight_max": 28.552684783935547,
      "activations/layer2_attention_weight_min": -28.791988372802734,
      "activations/layer3_attention_weight_max": 90.33930969238281,
      "activations/layer3_attention_weight_min": -91.32477569580078,
      "activations/layer4_attention_weight_max": 94.52168273925781,
      "activations/layer4_attention_weight_min": -88.60469818115234,
      "activations/layer5_attention_weight_max": 71.03677368164062,
      "activations/layer5_attention_weight_min": -78.30583190917969,
      "activations/layer6_attention_weight_max": 53.67112350463867,
      "activations/layer6_attention_weight_min": -51.24392318725586,
      "activations/layer7_attention_weight_max": 64.85364532470703,
      "activations/layer7_attention_weight_min": -64.80817413330078,
      "activations/layer8_attention_weight_max": 43.32866287231445,
      "activations/layer8_attention_weight_min": -48.593631744384766,
      "activations/layer9_attention_weight_max": 44.22245407104492,
      "activations/layer9_attention_weight_min": -43.8295783996582,
      "epoch": 13.74,
      "learning_rate": 6.202651515151515e-05,
      "loss": 2.7702,
      "step": 236450
    },
    {
      "activations/layer0_attention_weight_max": 15.789119720458984,
      "activations/layer0_attention_weight_min": -14.710592269897461,
      "activations/layer10_attention_weight_max": 40.24871826171875,
      "activations/layer10_attention_weight_min": -40.008419036865234,
      "activations/layer11_attention_weight_max": 36.23054122924805,
      "activations/layer11_attention_weight_min": -36.64649200439453,
      "activations/layer12_attention_weight_max": 19.240686416625977,
      "activations/layer12_attention_weight_min": -25.638513565063477,
      "activations/layer13_attention_weight_max": 38.861732482910156,
      "activations/layer13_attention_weight_min": -31.868616104125977,
      "activations/layer14_attention_weight_max": 36.17270278930664,
      "activations/layer14_attention_weight_min": -33.03174591064453,
      "activations/layer15_attention_weight_max": 35.25853729248047,
      "activations/layer15_attention_weight_min": -32.244258880615234,
      "activations/layer16_attention_weight_max": 35.780982971191406,
      "activations/layer16_attention_weight_min": -32.23048782348633,
      "activations/layer17_attention_weight_max": 53.122737884521484,
      "activations/layer17_attention_weight_min": -48.40913772583008,
      "activations/layer18_attention_weight_max": 50.23745346069336,
      "activations/layer18_attention_weight_min": -43.041404724121094,
      "activations/layer19_attention_weight_max": 22.273839950561523,
      "activations/layer19_attention_weight_min": -23.720584869384766,
      "activations/layer1_attention_weight_max": 16.58027458190918,
      "activations/layer1_attention_weight_min": -14.325959205627441,
      "activations/layer20_attention_weight_max": 22.486228942871094,
      "activations/layer20_attention_weight_min": -21.732921600341797,
      "activations/layer21_attention_weight_max": 33.286869049072266,
      "activations/layer21_attention_weight_min": -23.698999404907227,
      "activations/layer22_attention_weight_max": 29.42493438720703,
      "activations/layer22_attention_weight_min": -27.63743019104004,
      "activations/layer23_attention_weight_max": 34.99043273925781,
      "activations/layer23_attention_weight_min": -25.893699645996094,
      "activations/layer2_attention_weight_max": 31.228557586669922,
      "activations/layer2_attention_weight_min": -28.417613983154297,
      "activations/layer3_attention_weight_max": 91.98208618164062,
      "activations/layer3_attention_weight_min": -90.28987884521484,
      "activations/layer4_attention_weight_max": 97.94700622558594,
      "activations/layer4_attention_weight_min": -97.0765609741211,
      "activations/layer5_attention_weight_max": 71.65036010742188,
      "activations/layer5_attention_weight_min": -81.80644989013672,
      "activations/layer6_attention_weight_max": 55.444480895996094,
      "activations/layer6_attention_weight_min": -56.03559112548828,
      "activations/layer7_attention_weight_max": 68.8472671508789,
      "activations/layer7_attention_weight_min": -71.31692504882812,
      "activations/layer8_attention_weight_max": 48.99104309082031,
      "activations/layer8_attention_weight_min": -51.89958953857422,
      "activations/layer9_attention_weight_max": 47.597721099853516,
      "activations/layer9_attention_weight_min": -51.62886428833008,
      "epoch": 13.74,
      "learning_rate": 6.200757575757576e-05,
      "loss": 2.7657,
      "step": 236500
    },
    {
      "activations/layer0_attention_weight_max": 16.934452056884766,
      "activations/layer0_attention_weight_min": -14.395672798156738,
      "activations/layer10_attention_weight_max": 35.81467819213867,
      "activations/layer10_attention_weight_min": -35.60887908935547,
      "activations/layer11_attention_weight_max": 32.52091598510742,
      "activations/layer11_attention_weight_min": -33.31172180175781,
      "activations/layer12_attention_weight_max": 17.718050003051758,
      "activations/layer12_attention_weight_min": -25.89828872680664,
      "activations/layer13_attention_weight_max": 31.353090286254883,
      "activations/layer13_attention_weight_min": -27.55176544189453,
      "activations/layer14_attention_weight_max": 34.23759078979492,
      "activations/layer14_attention_weight_min": -30.10611343383789,
      "activations/layer15_attention_weight_max": 30.48710823059082,
      "activations/layer15_attention_weight_min": -28.83106231689453,
      "activations/layer16_attention_weight_max": 28.397056579589844,
      "activations/layer16_attention_weight_min": -28.95794105529785,
      "activations/layer17_attention_weight_max": 47.82576370239258,
      "activations/layer17_attention_weight_min": -44.05860900878906,
      "activations/layer18_attention_weight_max": 42.505313873291016,
      "activations/layer18_attention_weight_min": -38.37114334106445,
      "activations/layer19_attention_weight_max": 20.59270668029785,
      "activations/layer19_attention_weight_min": -22.23164176940918,
      "activations/layer1_attention_weight_max": 17.0153751373291,
      "activations/layer1_attention_weight_min": -18.092784881591797,
      "activations/layer20_attention_weight_max": 20.182876586914062,
      "activations/layer20_attention_weight_min": -21.524547576904297,
      "activations/layer21_attention_weight_max": 39.48004913330078,
      "activations/layer21_attention_weight_min": -22.53249168395996,
      "activations/layer22_attention_weight_max": 27.19026756286621,
      "activations/layer22_attention_weight_min": -24.880788803100586,
      "activations/layer23_attention_weight_max": 38.11138153076172,
      "activations/layer23_attention_weight_min": -25.774486541748047,
      "activations/layer2_attention_weight_max": 31.149776458740234,
      "activations/layer2_attention_weight_min": -29.0859375,
      "activations/layer3_attention_weight_max": 90.7916488647461,
      "activations/layer3_attention_weight_min": -98.68974304199219,
      "activations/layer4_attention_weight_max": 94.35025787353516,
      "activations/layer4_attention_weight_min": -90.66568756103516,
      "activations/layer5_attention_weight_max": 70.35682678222656,
      "activations/layer5_attention_weight_min": -79.91096496582031,
      "activations/layer6_attention_weight_max": 49.60024642944336,
      "activations/layer6_attention_weight_min": -50.92876434326172,
      "activations/layer7_attention_weight_max": 68.68160247802734,
      "activations/layer7_attention_weight_min": -66.62065124511719,
      "activations/layer8_attention_weight_max": 45.09712600708008,
      "activations/layer8_attention_weight_min": -45.78859329223633,
      "activations/layer9_attention_weight_max": 44.2181510925293,
      "activations/layer9_attention_weight_min": -46.21286392211914,
      "epoch": 13.74,
      "learning_rate": 6.198863636363636e-05,
      "loss": 2.7797,
      "step": 236550
    },
    {
      "activations/layer0_attention_weight_max": 16.365257263183594,
      "activations/layer0_attention_weight_min": -14.320219039916992,
      "activations/layer10_attention_weight_max": 38.193817138671875,
      "activations/layer10_attention_weight_min": -36.974708557128906,
      "activations/layer11_attention_weight_max": 34.546844482421875,
      "activations/layer11_attention_weight_min": -34.17828369140625,
      "activations/layer12_attention_weight_max": 20.29355812072754,
      "activations/layer12_attention_weight_min": -28.37122917175293,
      "activations/layer13_attention_weight_max": 46.509178161621094,
      "activations/layer13_attention_weight_min": -30.853151321411133,
      "activations/layer14_attention_weight_max": 45.80409240722656,
      "activations/layer14_attention_weight_min": -32.43524169921875,
      "activations/layer15_attention_weight_max": 43.248207092285156,
      "activations/layer15_attention_weight_min": -31.15196990966797,
      "activations/layer16_attention_weight_max": 32.46225357055664,
      "activations/layer16_attention_weight_min": -30.185176849365234,
      "activations/layer17_attention_weight_max": 50.529598236083984,
      "activations/layer17_attention_weight_min": -44.414676666259766,
      "activations/layer18_attention_weight_max": 44.79465103149414,
      "activations/layer18_attention_weight_min": -37.88167953491211,
      "activations/layer19_attention_weight_max": 21.576581954956055,
      "activations/layer19_attention_weight_min": -22.58384132385254,
      "activations/layer1_attention_weight_max": 15.546607971191406,
      "activations/layer1_attention_weight_min": -12.981046676635742,
      "activations/layer20_attention_weight_max": 23.829486846923828,
      "activations/layer20_attention_weight_min": -22.593229293823242,
      "activations/layer21_attention_weight_max": 33.60916519165039,
      "activations/layer21_attention_weight_min": -22.31437110900879,
      "activations/layer22_attention_weight_max": 30.45001983642578,
      "activations/layer22_attention_weight_min": -26.63578987121582,
      "activations/layer23_attention_weight_max": 37.149681091308594,
      "activations/layer23_attention_weight_min": -23.4397029876709,
      "activations/layer2_attention_weight_max": 28.95173454284668,
      "activations/layer2_attention_weight_min": -28.91535758972168,
      "activations/layer3_attention_weight_max": 92.39193725585938,
      "activations/layer3_attention_weight_min": -93.59475708007812,
      "activations/layer4_attention_weight_max": 96.95650482177734,
      "activations/layer4_attention_weight_min": -98.27425384521484,
      "activations/layer5_attention_weight_max": 75.18843078613281,
      "activations/layer5_attention_weight_min": -82.01841735839844,
      "activations/layer6_attention_weight_max": 51.669551849365234,
      "activations/layer6_attention_weight_min": -55.19279479980469,
      "activations/layer7_attention_weight_max": 74.52445983886719,
      "activations/layer7_attention_weight_min": -65.569091796875,
      "activations/layer8_attention_weight_max": 47.3437385559082,
      "activations/layer8_attention_weight_min": -49.073646545410156,
      "activations/layer9_attention_weight_max": 58.88408279418945,
      "activations/layer9_attention_weight_min": -47.32972717285156,
      "epoch": 13.75,
      "learning_rate": 6.196969696969697e-05,
      "loss": 2.7546,
      "step": 236600
    },
    {
      "activations/layer0_attention_weight_max": 15.939640998840332,
      "activations/layer0_attention_weight_min": -14.365556716918945,
      "activations/layer10_attention_weight_max": 41.065704345703125,
      "activations/layer10_attention_weight_min": -39.57111358642578,
      "activations/layer11_attention_weight_max": 38.94828796386719,
      "activations/layer11_attention_weight_min": -36.69580841064453,
      "activations/layer12_attention_weight_max": 20.202707290649414,
      "activations/layer12_attention_weight_min": -25.136930465698242,
      "activations/layer13_attention_weight_max": 40.60707092285156,
      "activations/layer13_attention_weight_min": -32.4647331237793,
      "activations/layer14_attention_weight_max": 36.472381591796875,
      "activations/layer14_attention_weight_min": -32.0597038269043,
      "activations/layer15_attention_weight_max": 33.89188003540039,
      "activations/layer15_attention_weight_min": -32.78990173339844,
      "activations/layer16_attention_weight_max": 33.11898422241211,
      "activations/layer16_attention_weight_min": -32.321044921875,
      "activations/layer17_attention_weight_max": 54.11243438720703,
      "activations/layer17_attention_weight_min": -46.55720520019531,
      "activations/layer18_attention_weight_max": 48.09122085571289,
      "activations/layer18_attention_weight_min": -40.4743766784668,
      "activations/layer19_attention_weight_max": 23.490591049194336,
      "activations/layer19_attention_weight_min": -23.722620010375977,
      "activations/layer1_attention_weight_max": 17.08538055419922,
      "activations/layer1_attention_weight_min": -16.10567855834961,
      "activations/layer20_attention_weight_max": 24.075010299682617,
      "activations/layer20_attention_weight_min": -22.981735229492188,
      "activations/layer21_attention_weight_max": 33.774044036865234,
      "activations/layer21_attention_weight_min": -24.60588264465332,
      "activations/layer22_attention_weight_max": 36.318336486816406,
      "activations/layer22_attention_weight_min": -29.423954010009766,
      "activations/layer23_attention_weight_max": 39.18916320800781,
      "activations/layer23_attention_weight_min": -25.195613861083984,
      "activations/layer2_attention_weight_max": 32.89579772949219,
      "activations/layer2_attention_weight_min": -32.10914993286133,
      "activations/layer3_attention_weight_max": 86.44950866699219,
      "activations/layer3_attention_weight_min": -83.8067398071289,
      "activations/layer4_attention_weight_max": 88.59638214111328,
      "activations/layer4_attention_weight_min": -87.11402893066406,
      "activations/layer5_attention_weight_max": 69.76017761230469,
      "activations/layer5_attention_weight_min": -81.54020690917969,
      "activations/layer6_attention_weight_max": 56.44871139526367,
      "activations/layer6_attention_weight_min": -51.70815658569336,
      "activations/layer7_attention_weight_max": 79.44572448730469,
      "activations/layer7_attention_weight_min": -68.49514770507812,
      "activations/layer8_attention_weight_max": 56.1402702331543,
      "activations/layer8_attention_weight_min": -54.77980422973633,
      "activations/layer9_attention_weight_max": 57.95050048828125,
      "activations/layer9_attention_weight_min": -48.08695602416992,
      "epoch": 13.75,
      "learning_rate": 6.195075757575757e-05,
      "loss": 2.7851,
      "step": 236650
    },
    {
      "activations/layer0_attention_weight_max": 17.509567260742188,
      "activations/layer0_attention_weight_min": -14.594517707824707,
      "activations/layer10_attention_weight_max": 37.20263671875,
      "activations/layer10_attention_weight_min": -37.10135269165039,
      "activations/layer11_attention_weight_max": 33.40663146972656,
      "activations/layer11_attention_weight_min": -35.10313415527344,
      "activations/layer12_attention_weight_max": 17.236345291137695,
      "activations/layer12_attention_weight_min": -24.830501556396484,
      "activations/layer13_attention_weight_max": 30.692399978637695,
      "activations/layer13_attention_weight_min": -27.366798400878906,
      "activations/layer14_attention_weight_max": 32.51595687866211,
      "activations/layer14_attention_weight_min": -29.980039596557617,
      "activations/layer15_attention_weight_max": 30.867944717407227,
      "activations/layer15_attention_weight_min": -29.309022903442383,
      "activations/layer16_attention_weight_max": 30.079010009765625,
      "activations/layer16_attention_weight_min": -29.991832733154297,
      "activations/layer17_attention_weight_max": 49.209503173828125,
      "activations/layer17_attention_weight_min": -45.8908576965332,
      "activations/layer18_attention_weight_max": 42.370704650878906,
      "activations/layer18_attention_weight_min": -36.94624710083008,
      "activations/layer19_attention_weight_max": 20.233118057250977,
      "activations/layer19_attention_weight_min": -22.040014266967773,
      "activations/layer1_attention_weight_max": 16.011140823364258,
      "activations/layer1_attention_weight_min": -16.69725227355957,
      "activations/layer20_attention_weight_max": 21.305736541748047,
      "activations/layer20_attention_weight_min": -22.20225715637207,
      "activations/layer21_attention_weight_max": 32.72829818725586,
      "activations/layer21_attention_weight_min": -20.69117546081543,
      "activations/layer22_attention_weight_max": 27.45699119567871,
      "activations/layer22_attention_weight_min": -25.37586784362793,
      "activations/layer23_attention_weight_max": 34.380332946777344,
      "activations/layer23_attention_weight_min": -22.94788360595703,
      "activations/layer2_attention_weight_max": 29.167638778686523,
      "activations/layer2_attention_weight_min": -29.509267807006836,
      "activations/layer3_attention_weight_max": 88.78247833251953,
      "activations/layer3_attention_weight_min": -90.76241302490234,
      "activations/layer4_attention_weight_max": 90.96009063720703,
      "activations/layer4_attention_weight_min": -95.65303802490234,
      "activations/layer5_attention_weight_max": 74.28433227539062,
      "activations/layer5_attention_weight_min": -77.89724731445312,
      "activations/layer6_attention_weight_max": 51.70667266845703,
      "activations/layer6_attention_weight_min": -53.19733810424805,
      "activations/layer7_attention_weight_max": 67.50259399414062,
      "activations/layer7_attention_weight_min": -68.3387451171875,
      "activations/layer8_attention_weight_max": 44.6547737121582,
      "activations/layer8_attention_weight_min": -47.26946258544922,
      "activations/layer9_attention_weight_max": 44.25734329223633,
      "activations/layer9_attention_weight_min": -46.80851745605469,
      "epoch": 13.75,
      "learning_rate": 6.193181818181818e-05,
      "loss": 2.7667,
      "step": 236700
    },
    {
      "activations/layer0_attention_weight_max": 16.465484619140625,
      "activations/layer0_attention_weight_min": -14.265020370483398,
      "activations/layer10_attention_weight_max": 46.33503723144531,
      "activations/layer10_attention_weight_min": -45.14909744262695,
      "activations/layer11_attention_weight_max": 42.05068588256836,
      "activations/layer11_attention_weight_min": -42.101318359375,
      "activations/layer12_attention_weight_max": 20.334081649780273,
      "activations/layer12_attention_weight_min": -25.22353744506836,
      "activations/layer13_attention_weight_max": 40.70886993408203,
      "activations/layer13_attention_weight_min": -32.37013244628906,
      "activations/layer14_attention_weight_max": 43.22180938720703,
      "activations/layer14_attention_weight_min": -35.4145622253418,
      "activations/layer15_attention_weight_max": 42.77708053588867,
      "activations/layer15_attention_weight_min": -34.95444107055664,
      "activations/layer16_attention_weight_max": 37.11081314086914,
      "activations/layer16_attention_weight_min": -35.46046447753906,
      "activations/layer17_attention_weight_max": 60.82665252685547,
      "activations/layer17_attention_weight_min": -49.51474380493164,
      "activations/layer18_attention_weight_max": 52.92896270751953,
      "activations/layer18_attention_weight_min": -40.949581146240234,
      "activations/layer19_attention_weight_max": 25.785888671875,
      "activations/layer19_attention_weight_min": -24.201641082763672,
      "activations/layer1_attention_weight_max": 16.37967300415039,
      "activations/layer1_attention_weight_min": -16.040067672729492,
      "activations/layer20_attention_weight_max": 24.786354064941406,
      "activations/layer20_attention_weight_min": -25.762989044189453,
      "activations/layer21_attention_weight_max": 44.92124557495117,
      "activations/layer21_attention_weight_min": -26.249740600585938,
      "activations/layer22_attention_weight_max": 33.09196090698242,
      "activations/layer22_attention_weight_min": -27.194169998168945,
      "activations/layer23_attention_weight_max": 38.53386306762695,
      "activations/layer23_attention_weight_min": -26.963788986206055,
      "activations/layer2_attention_weight_max": 34.53627014160156,
      "activations/layer2_attention_weight_min": -30.836841583251953,
      "activations/layer3_attention_weight_max": 95.61257934570312,
      "activations/layer3_attention_weight_min": -95.66754150390625,
      "activations/layer4_attention_weight_max": 99.98213195800781,
      "activations/layer4_attention_weight_min": -90.97710418701172,
      "activations/layer5_attention_weight_max": 75.09060668945312,
      "activations/layer5_attention_weight_min": -76.15291595458984,
      "activations/layer6_attention_weight_max": 57.67406463623047,
      "activations/layer6_attention_weight_min": -54.21040344238281,
      "activations/layer7_attention_weight_max": 71.76862335205078,
      "activations/layer7_attention_weight_min": -71.64244842529297,
      "activations/layer8_attention_weight_max": 55.8920783996582,
      "activations/layer8_attention_weight_min": -53.551300048828125,
      "activations/layer9_attention_weight_max": 55.320220947265625,
      "activations/layer9_attention_weight_min": -51.57068634033203,
      "epoch": 13.76,
      "learning_rate": 6.191287878787878e-05,
      "loss": 2.7823,
      "step": 236750
    },
    {
      "activations/layer0_attention_weight_max": 16.36410140991211,
      "activations/layer0_attention_weight_min": -14.14089584350586,
      "activations/layer10_attention_weight_max": 37.311126708984375,
      "activations/layer10_attention_weight_min": -36.91408920288086,
      "activations/layer11_attention_weight_max": 33.98921585083008,
      "activations/layer11_attention_weight_min": -34.962547302246094,
      "activations/layer12_attention_weight_max": 17.972917556762695,
      "activations/layer12_attention_weight_min": -26.095474243164062,
      "activations/layer13_attention_weight_max": 47.47932434082031,
      "activations/layer13_attention_weight_min": -29.98990821838379,
      "activations/layer14_attention_weight_max": 35.52651596069336,
      "activations/layer14_attention_weight_min": -32.34925079345703,
      "activations/layer15_attention_weight_max": 32.20970153808594,
      "activations/layer15_attention_weight_min": -32.50604248046875,
      "activations/layer16_attention_weight_max": 31.120447158813477,
      "activations/layer16_attention_weight_min": -32.20219421386719,
      "activations/layer17_attention_weight_max": 48.75775909423828,
      "activations/layer17_attention_weight_min": -46.84025573730469,
      "activations/layer18_attention_weight_max": 45.62218475341797,
      "activations/layer18_attention_weight_min": -40.65917205810547,
      "activations/layer19_attention_weight_max": 21.35406494140625,
      "activations/layer19_attention_weight_min": -22.543190002441406,
      "activations/layer1_attention_weight_max": 16.349010467529297,
      "activations/layer1_attention_weight_min": -16.448867797851562,
      "activations/layer20_attention_weight_max": 22.805509567260742,
      "activations/layer20_attention_weight_min": -23.165956497192383,
      "activations/layer21_attention_weight_max": 34.330467224121094,
      "activations/layer21_attention_weight_min": -23.34482192993164,
      "activations/layer22_attention_weight_max": 29.375337600708008,
      "activations/layer22_attention_weight_min": -27.160444259643555,
      "activations/layer23_attention_weight_max": 34.672523498535156,
      "activations/layer23_attention_weight_min": -24.48699188232422,
      "activations/layer2_attention_weight_max": 32.307159423828125,
      "activations/layer2_attention_weight_min": -30.18456268310547,
      "activations/layer3_attention_weight_max": 93.30743408203125,
      "activations/layer3_attention_weight_min": -90.92664337158203,
      "activations/layer4_attention_weight_max": 91.50709533691406,
      "activations/layer4_attention_weight_min": -90.0293197631836,
      "activations/layer5_attention_weight_max": 74.80673217773438,
      "activations/layer5_attention_weight_min": -80.61920166015625,
      "activations/layer6_attention_weight_max": 52.50650405883789,
      "activations/layer6_attention_weight_min": -52.560123443603516,
      "activations/layer7_attention_weight_max": 66.70626831054688,
      "activations/layer7_attention_weight_min": -71.03395080566406,
      "activations/layer8_attention_weight_max": 48.807472229003906,
      "activations/layer8_attention_weight_min": -51.510799407958984,
      "activations/layer9_attention_weight_max": 51.336090087890625,
      "activations/layer9_attention_weight_min": -47.76553726196289,
      "epoch": 13.76,
      "learning_rate": 6.189393939393938e-05,
      "loss": 2.7799,
      "step": 236800
    },
    {
      "activations/layer0_attention_weight_max": 15.924120903015137,
      "activations/layer0_attention_weight_min": -14.299153327941895,
      "activations/layer10_attention_weight_max": 35.755130767822266,
      "activations/layer10_attention_weight_min": -36.736976623535156,
      "activations/layer11_attention_weight_max": 31.322080612182617,
      "activations/layer11_attention_weight_min": -33.856834411621094,
      "activations/layer12_attention_weight_max": 18.35987663269043,
      "activations/layer12_attention_weight_min": -24.86188316345215,
      "activations/layer13_attention_weight_max": 33.121009826660156,
      "activations/layer13_attention_weight_min": -26.457990646362305,
      "activations/layer14_attention_weight_max": 38.010440826416016,
      "activations/layer14_attention_weight_min": -29.405323028564453,
      "activations/layer15_attention_weight_max": 31.6683349609375,
      "activations/layer15_attention_weight_min": -30.06148338317871,
      "activations/layer16_attention_weight_max": 31.53786277770996,
      "activations/layer16_attention_weight_min": -30.810827255249023,
      "activations/layer17_attention_weight_max": 49.47597122192383,
      "activations/layer17_attention_weight_min": -42.702693939208984,
      "activations/layer18_attention_weight_max": 43.8537483215332,
      "activations/layer18_attention_weight_min": -37.56723403930664,
      "activations/layer19_attention_weight_max": 21.790424346923828,
      "activations/layer19_attention_weight_min": -24.00994873046875,
      "activations/layer1_attention_weight_max": 17.25986671447754,
      "activations/layer1_attention_weight_min": -18.073762893676758,
      "activations/layer20_attention_weight_max": 21.668800354003906,
      "activations/layer20_attention_weight_min": -21.20490074157715,
      "activations/layer21_attention_weight_max": 33.305511474609375,
      "activations/layer21_attention_weight_min": -23.400583267211914,
      "activations/layer22_attention_weight_max": 28.13985824584961,
      "activations/layer22_attention_weight_min": -25.32537269592285,
      "activations/layer23_attention_weight_max": 34.955299377441406,
      "activations/layer23_attention_weight_min": -22.43025779724121,
      "activations/layer2_attention_weight_max": 31.403005599975586,
      "activations/layer2_attention_weight_min": -30.393310546875,
      "activations/layer3_attention_weight_max": 93.4288330078125,
      "activations/layer3_attention_weight_min": -97.55120849609375,
      "activations/layer4_attention_weight_max": 97.07862091064453,
      "activations/layer4_attention_weight_min": -93.4811782836914,
      "activations/layer5_attention_weight_max": 74.48350524902344,
      "activations/layer5_attention_weight_min": -74.3578109741211,
      "activations/layer6_attention_weight_max": 51.89950180053711,
      "activations/layer6_attention_weight_min": -51.550228118896484,
      "activations/layer7_attention_weight_max": 67.17514038085938,
      "activations/layer7_attention_weight_min": -65.79420471191406,
      "activations/layer8_attention_weight_max": 44.250244140625,
      "activations/layer8_attention_weight_min": -47.9940185546875,
      "activations/layer9_attention_weight_max": 45.119712829589844,
      "activations/layer9_attention_weight_min": -44.616065979003906,
      "epoch": 13.76,
      "learning_rate": 6.187499999999999e-05,
      "loss": 2.7734,
      "step": 236850
    },
    {
      "activations/layer0_attention_weight_max": 15.351082801818848,
      "activations/layer0_attention_weight_min": -14.247383117675781,
      "activations/layer10_attention_weight_max": 51.44708251953125,
      "activations/layer10_attention_weight_min": -47.98883056640625,
      "activations/layer11_attention_weight_max": 49.452266693115234,
      "activations/layer11_attention_weight_min": -42.892478942871094,
      "activations/layer12_attention_weight_max": 20.3524227142334,
      "activations/layer12_attention_weight_min": -27.82084083557129,
      "activations/layer13_attention_weight_max": 36.00886535644531,
      "activations/layer13_attention_weight_min": -27.91504669189453,
      "activations/layer14_attention_weight_max": 43.23200988769531,
      "activations/layer14_attention_weight_min": -30.457605361938477,
      "activations/layer15_attention_weight_max": 36.69008255004883,
      "activations/layer15_attention_weight_min": -29.13819694519043,
      "activations/layer16_attention_weight_max": 31.786645889282227,
      "activations/layer16_attention_weight_min": -29.718490600585938,
      "activations/layer17_attention_weight_max": 54.41529846191406,
      "activations/layer17_attention_weight_min": -46.456260681152344,
      "activations/layer18_attention_weight_max": 48.63601303100586,
      "activations/layer18_attention_weight_min": -40.91183853149414,
      "activations/layer19_attention_weight_max": 24.360132217407227,
      "activations/layer19_attention_weight_min": -23.123640060424805,
      "activations/layer1_attention_weight_max": 17.450992584228516,
      "activations/layer1_attention_weight_min": -17.167530059814453,
      "activations/layer20_attention_weight_max": 25.13210105895996,
      "activations/layer20_attention_weight_min": -22.475309371948242,
      "activations/layer21_attention_weight_max": 38.046875,
      "activations/layer21_attention_weight_min": -20.90706443786621,
      "activations/layer22_attention_weight_max": 32.774234771728516,
      "activations/layer22_attention_weight_min": -25.884206771850586,
      "activations/layer23_attention_weight_max": 37.40670394897461,
      "activations/layer23_attention_weight_min": -22.052547454833984,
      "activations/layer2_attention_weight_max": 36.441978454589844,
      "activations/layer2_attention_weight_min": -32.908424377441406,
      "activations/layer3_attention_weight_max": 106.47293853759766,
      "activations/layer3_attention_weight_min": -104.72753143310547,
      "activations/layer4_attention_weight_max": 109.87420654296875,
      "activations/layer4_attention_weight_min": -98.62163543701172,
      "activations/layer5_attention_weight_max": 79.19793701171875,
      "activations/layer5_attention_weight_min": -77.47572326660156,
      "activations/layer6_attention_weight_max": 57.95487976074219,
      "activations/layer6_attention_weight_min": -55.781410217285156,
      "activations/layer7_attention_weight_max": 81.90787506103516,
      "activations/layer7_attention_weight_min": -72.64828491210938,
      "activations/layer8_attention_weight_max": 61.98029327392578,
      "activations/layer8_attention_weight_min": -54.01176834106445,
      "activations/layer9_attention_weight_max": 69.37528228759766,
      "activations/layer9_attention_weight_min": -57.584388732910156,
      "epoch": 13.77,
      "learning_rate": 6.18560606060606e-05,
      "loss": 2.7698,
      "step": 236900
    },
    {
      "activations/layer0_attention_weight_max": 15.54684066772461,
      "activations/layer0_attention_weight_min": -14.176398277282715,
      "activations/layer10_attention_weight_max": 36.28037643432617,
      "activations/layer10_attention_weight_min": -34.80510330200195,
      "activations/layer11_attention_weight_max": 32.298919677734375,
      "activations/layer11_attention_weight_min": -33.1595458984375,
      "activations/layer12_attention_weight_max": 18.266477584838867,
      "activations/layer12_attention_weight_min": -26.199846267700195,
      "activations/layer13_attention_weight_max": 36.41460037231445,
      "activations/layer13_attention_weight_min": -28.653276443481445,
      "activations/layer14_attention_weight_max": 37.76010513305664,
      "activations/layer14_attention_weight_min": -31.415679931640625,
      "activations/layer15_attention_weight_max": 35.636390686035156,
      "activations/layer15_attention_weight_min": -29.639652252197266,
      "activations/layer16_attention_weight_max": 36.4560546875,
      "activations/layer16_attention_weight_min": -32.04804229736328,
      "activations/layer17_attention_weight_max": 55.932403564453125,
      "activations/layer17_attention_weight_min": -46.01832580566406,
      "activations/layer18_attention_weight_max": 49.50579071044922,
      "activations/layer18_attention_weight_min": -39.60934066772461,
      "activations/layer19_attention_weight_max": 22.315919876098633,
      "activations/layer19_attention_weight_min": -22.025142669677734,
      "activations/layer1_attention_weight_max": 17.635929107666016,
      "activations/layer1_attention_weight_min": -15.89309024810791,
      "activations/layer20_attention_weight_max": 22.90592384338379,
      "activations/layer20_attention_weight_min": -21.653850555419922,
      "activations/layer21_attention_weight_max": 37.35112380981445,
      "activations/layer21_attention_weight_min": -23.2526912689209,
      "activations/layer22_attention_weight_max": 31.458412170410156,
      "activations/layer22_attention_weight_min": -25.15980339050293,
      "activations/layer23_attention_weight_max": 42.19983673095703,
      "activations/layer23_attention_weight_min": -22.798505783081055,
      "activations/layer2_attention_weight_max": 31.612394332885742,
      "activations/layer2_attention_weight_min": -30.328704833984375,
      "activations/layer3_attention_weight_max": 96.23350524902344,
      "activations/layer3_attention_weight_min": -99.00733184814453,
      "activations/layer4_attention_weight_max": 96.24410247802734,
      "activations/layer4_attention_weight_min": -91.43171691894531,
      "activations/layer5_attention_weight_max": 69.83355712890625,
      "activations/layer5_attention_weight_min": -80.33180236816406,
      "activations/layer6_attention_weight_max": 52.31186294555664,
      "activations/layer6_attention_weight_min": -49.2935905456543,
      "activations/layer7_attention_weight_max": 66.996337890625,
      "activations/layer7_attention_weight_min": -67.38613891601562,
      "activations/layer8_attention_weight_max": 45.43118667602539,
      "activations/layer8_attention_weight_min": -47.89467239379883,
      "activations/layer9_attention_weight_max": 43.74261474609375,
      "activations/layer9_attention_weight_min": -44.3410758972168,
      "epoch": 13.77,
      "learning_rate": 6.18371212121212e-05,
      "loss": 2.7533,
      "step": 236950
    },
    {
      "activations/layer0_attention_weight_max": 15.819448471069336,
      "activations/layer0_attention_weight_min": -14.38655948638916,
      "activations/layer10_attention_weight_max": 38.5062255859375,
      "activations/layer10_attention_weight_min": -38.43509292602539,
      "activations/layer11_attention_weight_max": 36.36212921142578,
      "activations/layer11_attention_weight_min": -39.24223327636719,
      "activations/layer12_attention_weight_max": 20.08321762084961,
      "activations/layer12_attention_weight_min": -24.74042320251465,
      "activations/layer13_attention_weight_max": 41.416194915771484,
      "activations/layer13_attention_weight_min": -33.24418258666992,
      "activations/layer14_attention_weight_max": 38.418113708496094,
      "activations/layer14_attention_weight_min": -31.56186866760254,
      "activations/layer15_attention_weight_max": 32.95008087158203,
      "activations/layer15_attention_weight_min": -30.210952758789062,
      "activations/layer16_attention_weight_max": 31.015592575073242,
      "activations/layer16_attention_weight_min": -30.046764373779297,
      "activations/layer17_attention_weight_max": 52.1528205871582,
      "activations/layer17_attention_weight_min": -43.17549133300781,
      "activations/layer18_attention_weight_max": 45.35153579711914,
      "activations/layer18_attention_weight_min": -38.57421875,
      "activations/layer19_attention_weight_max": 22.924814224243164,
      "activations/layer19_attention_weight_min": -21.05060386657715,
      "activations/layer1_attention_weight_max": 16.107481002807617,
      "activations/layer1_attention_weight_min": -15.693338394165039,
      "activations/layer20_attention_weight_max": 22.52526092529297,
      "activations/layer20_attention_weight_min": -23.67918586730957,
      "activations/layer21_attention_weight_max": 40.974945068359375,
      "activations/layer21_attention_weight_min": -24.978923797607422,
      "activations/layer22_attention_weight_max": 30.687273025512695,
      "activations/layer22_attention_weight_min": -26.972070693969727,
      "activations/layer23_attention_weight_max": 40.66897201538086,
      "activations/layer23_attention_weight_min": -25.473854064941406,
      "activations/layer2_attention_weight_max": 32.131378173828125,
      "activations/layer2_attention_weight_min": -30.127159118652344,
      "activations/layer3_attention_weight_max": 96.17198181152344,
      "activations/layer3_attention_weight_min": -93.68515014648438,
      "activations/layer4_attention_weight_max": 100.88253784179688,
      "activations/layer4_attention_weight_min": -92.1021499633789,
      "activations/layer5_attention_weight_max": 73.75444030761719,
      "activations/layer5_attention_weight_min": -83.3899154663086,
      "activations/layer6_attention_weight_max": 53.2430305480957,
      "activations/layer6_attention_weight_min": -51.745826721191406,
      "activations/layer7_attention_weight_max": 70.59635162353516,
      "activations/layer7_attention_weight_min": -70.60868835449219,
      "activations/layer8_attention_weight_max": 47.614253997802734,
      "activations/layer8_attention_weight_min": -49.95599365234375,
      "activations/layer9_attention_weight_max": 49.5409049987793,
      "activations/layer9_attention_weight_min": -48.36463928222656,
      "epoch": 13.77,
      "learning_rate": 6.181818181818182e-05,
      "loss": 2.7432,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_loss": 2.71875,
      "eval_runtime": 8.4862,
      "eval_samples_per_second": 505.997,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_openwebtext_loss": 2.71875,
      "eval_openwebtext_ppl": 15.16135869683449,
      "eval_openwebtext_runtime": 8.4862,
      "eval_openwebtext_samples_per_second": 505.997,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9782,
      "eval_wikitext_samples_per_second": 230.509,
      "step": 237000
    },
    {
      "epoch": 13.77,
      "eval_lambada_loss": 2.556640625,
      "eval_lambada_ppl": 12.892433965519283,
      "eval_lambada_runtime": 9.5606,
      "eval_lambada_samples_per_second": 509.28,
      "step": 237000
    },
    {
      "activations/layer0_attention_weight_max": 16.378929138183594,
      "activations/layer0_attention_weight_min": -14.07730770111084,
      "activations/layer10_attention_weight_max": 37.59043884277344,
      "activations/layer10_attention_weight_min": -36.74958038330078,
      "activations/layer11_attention_weight_max": 32.567352294921875,
      "activations/layer11_attention_weight_min": -34.92931365966797,
      "activations/layer12_attention_weight_max": 19.864187240600586,
      "activations/layer12_attention_weight_min": -24.613187789916992,
      "activations/layer13_attention_weight_max": 33.84371566772461,
      "activations/layer13_attention_weight_min": -27.980440139770508,
      "activations/layer14_attention_weight_max": 35.16215515136719,
      "activations/layer14_attention_weight_min": -29.186023712158203,
      "activations/layer15_attention_weight_max": 32.18009567260742,
      "activations/layer15_attention_weight_min": -28.872087478637695,
      "activations/layer16_attention_weight_max": 32.786563873291016,
      "activations/layer16_attention_weight_min": -30.289655685424805,
      "activations/layer17_attention_weight_max": 53.57259750366211,
      "activations/layer17_attention_weight_min": -45.89280319213867,
      "activations/layer18_attention_weight_max": 46.34172439575195,
      "activations/layer18_attention_weight_min": -39.2750244140625,
      "activations/layer19_attention_weight_max": 22.238967895507812,
      "activations/layer19_attention_weight_min": -21.776697158813477,
      "activations/layer1_attention_weight_max": 16.956342697143555,
      "activations/layer1_attention_weight_min": -16.692670822143555,
      "activations/layer20_attention_weight_max": 24.072988510131836,
      "activations/layer20_attention_weight_min": -22.43939208984375,
      "activations/layer21_attention_weight_max": 36.222023010253906,
      "activations/layer21_attention_weight_min": -22.03334617614746,
      "activations/layer22_attention_weight_max": 35.812740325927734,
      "activations/layer22_attention_weight_min": -27.59806251525879,
      "activations/layer23_attention_weight_max": 37.70571517944336,
      "activations/layer23_attention_weight_min": -23.536344528198242,
      "activations/layer2_attention_weight_max": 30.614608764648438,
      "activations/layer2_attention_weight_min": -29.12270736694336,
      "activations/layer3_attention_weight_max": 91.88458251953125,
      "activations/layer3_attention_weight_min": -91.80679321289062,
      "activations/layer4_attention_weight_max": 93.63570404052734,
      "activations/layer4_attention_weight_min": -91.6131362915039,
      "activations/layer5_attention_weight_max": 73.31463623046875,
      "activations/layer5_attention_weight_min": -81.47628784179688,
      "activations/layer6_attention_weight_max": 53.46957015991211,
      "activations/layer6_attention_weight_min": -55.05657958984375,
      "activations/layer7_attention_weight_max": 67.57508850097656,
      "activations/layer7_attention_weight_min": -66.05894470214844,
      "activations/layer8_attention_weight_max": 48.0999755859375,
      "activations/layer8_attention_weight_min": -49.633689880371094,
      "activations/layer9_attention_weight_max": 48.25068283081055,
      "activations/layer9_attention_weight_min": -45.77788543701172,
      "epoch": 13.77,
      "learning_rate": 6.179924242424242e-05,
      "loss": 2.7702,
      "step": 237050
    },
    {
      "activations/layer0_attention_weight_max": 17.309120178222656,
      "activations/layer0_attention_weight_min": -14.282697677612305,
      "activations/layer10_attention_weight_max": 37.58134460449219,
      "activations/layer10_attention_weight_min": -36.160064697265625,
      "activations/layer11_attention_weight_max": 34.586753845214844,
      "activations/layer11_attention_weight_min": -34.47434616088867,
      "activations/layer12_attention_weight_max": 18.844280242919922,
      "activations/layer12_attention_weight_min": -28.116559982299805,
      "activations/layer13_attention_weight_max": 39.180091857910156,
      "activations/layer13_attention_weight_min": -30.77898597717285,
      "activations/layer14_attention_weight_max": 40.52363967895508,
      "activations/layer14_attention_weight_min": -30.799888610839844,
      "activations/layer15_attention_weight_max": 36.43693542480469,
      "activations/layer15_attention_weight_min": -29.94518280029297,
      "activations/layer16_attention_weight_max": 37.10964584350586,
      "activations/layer16_attention_weight_min": -32.198970794677734,
      "activations/layer17_attention_weight_max": 61.7900390625,
      "activations/layer17_attention_weight_min": -48.361812591552734,
      "activations/layer18_attention_weight_max": 52.04525375366211,
      "activations/layer18_attention_weight_min": -39.702911376953125,
      "activations/layer19_attention_weight_max": 26.0892391204834,
      "activations/layer19_attention_weight_min": -22.29370880126953,
      "activations/layer1_attention_weight_max": 16.229280471801758,
      "activations/layer1_attention_weight_min": -15.662698745727539,
      "activations/layer20_attention_weight_max": 25.161453247070312,
      "activations/layer20_attention_weight_min": -24.7654972076416,
      "activations/layer21_attention_weight_max": 39.961395263671875,
      "activations/layer21_attention_weight_min": -20.728979110717773,
      "activations/layer22_attention_weight_max": 31.406898498535156,
      "activations/layer22_attention_weight_min": -25.780563354492188,
      "activations/layer23_attention_weight_max": 36.99151611328125,
      "activations/layer23_attention_weight_min": -24.04678726196289,
      "activations/layer2_attention_weight_max": 32.20687484741211,
      "activations/layer2_attention_weight_min": -30.110321044921875,
      "activations/layer3_attention_weight_max": 94.68444061279297,
      "activations/layer3_attention_weight_min": -97.41265106201172,
      "activations/layer4_attention_weight_max": 97.49797821044922,
      "activations/layer4_attention_weight_min": -95.04251861572266,
      "activations/layer5_attention_weight_max": 69.81809997558594,
      "activations/layer5_attention_weight_min": -80.05595397949219,
      "activations/layer6_attention_weight_max": 53.18458938598633,
      "activations/layer6_attention_weight_min": -52.56522750854492,
      "activations/layer7_attention_weight_max": 72.30096435546875,
      "activations/layer7_attention_weight_min": -66.57674407958984,
      "activations/layer8_attention_weight_max": 47.52731704711914,
      "activations/layer8_attention_weight_min": -49.23381042480469,
      "activations/layer9_attention_weight_max": 47.22658920288086,
      "activations/layer9_attention_weight_min": -45.581459045410156,
      "epoch": 13.78,
      "learning_rate": 6.178030303030302e-05,
      "loss": 2.7644,
      "step": 237100
    },
    {
      "activations/layer0_attention_weight_max": 16.45075225830078,
      "activations/layer0_attention_weight_min": -14.51677417755127,
      "activations/layer10_attention_weight_max": 37.98029708862305,
      "activations/layer10_attention_weight_min": -37.25847625732422,
      "activations/layer11_attention_weight_max": 33.01703643798828,
      "activations/layer11_attention_weight_min": -34.073883056640625,
      "activations/layer12_attention_weight_max": 18.404327392578125,
      "activations/layer12_attention_weight_min": -25.36050796508789,
      "activations/layer13_attention_weight_max": 36.4351921081543,
      "activations/layer13_attention_weight_min": -28.422670364379883,
      "activations/layer14_attention_weight_max": 35.65220260620117,
      "activations/layer14_attention_weight_min": -29.335739135742188,
      "activations/layer15_attention_weight_max": 34.399662017822266,
      "activations/layer15_attention_weight_min": -30.666175842285156,
      "activations/layer16_attention_weight_max": 32.04532241821289,
      "activations/layer16_attention_weight_min": -30.860774993896484,
      "activations/layer17_attention_weight_max": 54.834007263183594,
      "activations/layer17_attention_weight_min": -44.562984466552734,
      "activations/layer18_attention_weight_max": 46.91403579711914,
      "activations/layer18_attention_weight_min": -40.719486236572266,
      "activations/layer19_attention_weight_max": 23.730627059936523,
      "activations/layer19_attention_weight_min": -22.503564834594727,
      "activations/layer1_attention_weight_max": 16.874366760253906,
      "activations/layer1_attention_weight_min": -14.615447044372559,
      "activations/layer20_attention_weight_max": 22.50554084777832,
      "activations/layer20_attention_weight_min": -22.222089767456055,
      "activations/layer21_attention_weight_max": 30.677709579467773,
      "activations/layer21_attention_weight_min": -20.1575870513916,
      "activations/layer22_attention_weight_max": 30.594127655029297,
      "activations/layer22_attention_weight_min": -24.305253982543945,
      "activations/layer23_attention_weight_max": 37.563560485839844,
      "activations/layer23_attention_weight_min": -23.749252319335938,
      "activations/layer2_attention_weight_max": 31.88605499267578,
      "activations/layer2_attention_weight_min": -30.509098052978516,
      "activations/layer3_attention_weight_max": 92.44251251220703,
      "activations/layer3_attention_weight_min": -90.3913803100586,
      "activations/layer4_attention_weight_max": 93.4250259399414,
      "activations/layer4_attention_weight_min": -91.5512466430664,
      "activations/layer5_attention_weight_max": 71.29396057128906,
      "activations/layer5_attention_weight_min": -77.44052124023438,
      "activations/layer6_attention_weight_max": 50.5933723449707,
      "activations/layer6_attention_weight_min": -52.067344665527344,
      "activations/layer7_attention_weight_max": 67.57290649414062,
      "activations/layer7_attention_weight_min": -65.96592712402344,
      "activations/layer8_attention_weight_max": 45.848358154296875,
      "activations/layer8_attention_weight_min": -51.03944396972656,
      "activations/layer9_attention_weight_max": 46.66120529174805,
      "activations/layer9_attention_weight_min": -47.02259826660156,
      "epoch": 13.78,
      "learning_rate": 6.176136363636364e-05,
      "loss": 2.7735,
      "step": 237150
    },
    {
      "activations/layer0_attention_weight_max": 16.761262893676758,
      "activations/layer0_attention_weight_min": -14.425642013549805,
      "activations/layer10_attention_weight_max": 41.46955871582031,
      "activations/layer10_attention_weight_min": -38.74127197265625,
      "activations/layer11_attention_weight_max": 37.306434631347656,
      "activations/layer11_attention_weight_min": -37.234107971191406,
      "activations/layer12_attention_weight_max": 20.108243942260742,
      "activations/layer12_attention_weight_min": -25.957685470581055,
      "activations/layer13_attention_weight_max": 35.33643341064453,
      "activations/layer13_attention_weight_min": -28.40866470336914,
      "activations/layer14_attention_weight_max": 31.80365562438965,
      "activations/layer14_attention_weight_min": -31.279727935791016,
      "activations/layer15_attention_weight_max": 31.46257209777832,
      "activations/layer15_attention_weight_min": -29.357471466064453,
      "activations/layer16_attention_weight_max": 29.40656089782715,
      "activations/layer16_attention_weight_min": -28.748023986816406,
      "activations/layer17_attention_weight_max": 50.39262390136719,
      "activations/layer17_attention_weight_min": -43.53124237060547,
      "activations/layer18_attention_weight_max": 40.79799270629883,
      "activations/layer18_attention_weight_min": -36.28487777709961,
      "activations/layer19_attention_weight_max": 20.500043869018555,
      "activations/layer19_attention_weight_min": -20.815427780151367,
      "activations/layer1_attention_weight_max": 17.56916046142578,
      "activations/layer1_attention_weight_min": -14.817065238952637,
      "activations/layer20_attention_weight_max": 23.1317195892334,
      "activations/layer20_attention_weight_min": -24.10025978088379,
      "activations/layer21_attention_weight_max": 27.735366821289062,
      "activations/layer21_attention_weight_min": -19.552154541015625,
      "activations/layer22_attention_weight_max": 29.52172088623047,
      "activations/layer22_attention_weight_min": -24.276744842529297,
      "activations/layer23_attention_weight_max": 30.923070907592773,
      "activations/layer23_attention_weight_min": -22.971914291381836,
      "activations/layer2_attention_weight_max": 34.272125244140625,
      "activations/layer2_attention_weight_min": -31.051986694335938,
      "activations/layer3_attention_weight_max": 100.48788452148438,
      "activations/layer3_attention_weight_min": -98.73719024658203,
      "activations/layer4_attention_weight_max": 103.13408660888672,
      "activations/layer4_attention_weight_min": -100.0234146118164,
      "activations/layer5_attention_weight_max": 76.23631286621094,
      "activations/layer5_attention_weight_min": -83.68385314941406,
      "activations/layer6_attention_weight_max": 54.05928039550781,
      "activations/layer6_attention_weight_min": -53.88615417480469,
      "activations/layer7_attention_weight_max": 70.31575012207031,
      "activations/layer7_attention_weight_min": -68.32299041748047,
      "activations/layer8_attention_weight_max": 49.030517578125,
      "activations/layer8_attention_weight_min": -51.54255676269531,
      "activations/layer9_attention_weight_max": 48.35734176635742,
      "activations/layer9_attention_weight_min": -48.40433883666992,
      "epoch": 13.78,
      "learning_rate": 6.174242424242424e-05,
      "loss": 2.7685,
      "step": 237200
    },
    {
      "activations/layer0_attention_weight_max": 15.691278457641602,
      "activations/layer0_attention_weight_min": -14.640697479248047,
      "activations/layer10_attention_weight_max": 37.297088623046875,
      "activations/layer10_attention_weight_min": -35.31787872314453,
      "activations/layer11_attention_weight_max": 31.887325286865234,
      "activations/layer11_attention_weight_min": -33.520687103271484,
      "activations/layer12_attention_weight_max": 19.337839126586914,
      "activations/layer12_attention_weight_min": -24.220722198486328,
      "activations/layer13_attention_weight_max": 36.260833740234375,
      "activations/layer13_attention_weight_min": -26.36675453186035,
      "activations/layer14_attention_weight_max": 36.12807846069336,
      "activations/layer14_attention_weight_min": -30.632312774658203,
      "activations/layer15_attention_weight_max": 35.772499084472656,
      "activations/layer15_attention_weight_min": -31.06423568725586,
      "activations/layer16_attention_weight_max": 34.819549560546875,
      "activations/layer16_attention_weight_min": -29.945348739624023,
      "activations/layer17_attention_weight_max": 53.21413040161133,
      "activations/layer17_attention_weight_min": -47.20051574707031,
      "activations/layer18_attention_weight_max": 47.95989227294922,
      "activations/layer18_attention_weight_min": -40.7283935546875,
      "activations/layer19_attention_weight_max": 21.289499282836914,
      "activations/layer19_attention_weight_min": -22.37118911743164,
      "activations/layer1_attention_weight_max": 17.011022567749023,
      "activations/layer1_attention_weight_min": -14.4012451171875,
      "activations/layer20_attention_weight_max": 21.112140655517578,
      "activations/layer20_attention_weight_min": -24.897537231445312,
      "activations/layer21_attention_weight_max": 36.519622802734375,
      "activations/layer21_attention_weight_min": -22.97398567199707,
      "activations/layer22_attention_weight_max": 32.90068435668945,
      "activations/layer22_attention_weight_min": -25.65141487121582,
      "activations/layer23_attention_weight_max": 39.5065803527832,
      "activations/layer23_attention_weight_min": -24.41022491455078,
      "activations/layer2_attention_weight_max": 33.0410041809082,
      "activations/layer2_attention_weight_min": -30.69396209716797,
      "activations/layer3_attention_weight_max": 94.60637664794922,
      "activations/layer3_attention_weight_min": -94.46591186523438,
      "activations/layer4_attention_weight_max": 94.92428588867188,
      "activations/layer4_attention_weight_min": -91.3218002319336,
      "activations/layer5_attention_weight_max": 73.17839050292969,
      "activations/layer5_attention_weight_min": -78.36149597167969,
      "activations/layer6_attention_weight_max": 52.83149337768555,
      "activations/layer6_attention_weight_min": -53.13801574707031,
      "activations/layer7_attention_weight_max": 67.74066925048828,
      "activations/layer7_attention_weight_min": -65.91461181640625,
      "activations/layer8_attention_weight_max": 46.198760986328125,
      "activations/layer8_attention_weight_min": -48.270118713378906,
      "activations/layer9_attention_weight_max": 45.084598541259766,
      "activations/layer9_attention_weight_min": -45.94286346435547,
      "epoch": 13.79,
      "learning_rate": 6.172348484848485e-05,
      "loss": 2.7758,
      "step": 237250
    },
    {
      "activations/layer0_attention_weight_max": 17.107391357421875,
      "activations/layer0_attention_weight_min": -14.115446090698242,
      "activations/layer10_attention_weight_max": 40.33372497558594,
      "activations/layer10_attention_weight_min": -37.960418701171875,
      "activations/layer11_attention_weight_max": 34.266357421875,
      "activations/layer11_attention_weight_min": -33.889530181884766,
      "activations/layer12_attention_weight_max": 18.331790924072266,
      "activations/layer12_attention_weight_min": -25.207061767578125,
      "activations/layer13_attention_weight_max": 43.90456771850586,
      "activations/layer13_attention_weight_min": -29.411808013916016,
      "activations/layer14_attention_weight_max": 37.54300308227539,
      "activations/layer14_attention_weight_min": -30.778987884521484,
      "activations/layer15_attention_weight_max": 32.936248779296875,
      "activations/layer15_attention_weight_min": -28.92759132385254,
      "activations/layer16_attention_weight_max": 33.61971664428711,
      "activations/layer16_attention_weight_min": -31.435657501220703,
      "activations/layer17_attention_weight_max": 50.190528869628906,
      "activations/layer17_attention_weight_min": -42.996089935302734,
      "activations/layer18_attention_weight_max": 48.00673294067383,
      "activations/layer18_attention_weight_min": -38.853912353515625,
      "activations/layer19_attention_weight_max": 22.820980072021484,
      "activations/layer19_attention_weight_min": -22.853696823120117,
      "activations/layer1_attention_weight_max": 16.189908981323242,
      "activations/layer1_attention_weight_min": -13.288837432861328,
      "activations/layer20_attention_weight_max": 24.345029830932617,
      "activations/layer20_attention_weight_min": -20.440610885620117,
      "activations/layer21_attention_weight_max": 37.866241455078125,
      "activations/layer21_attention_weight_min": -21.949462890625,
      "activations/layer22_attention_weight_max": 30.696346282958984,
      "activations/layer22_attention_weight_min": -27.16994857788086,
      "activations/layer23_attention_weight_max": 41.840187072753906,
      "activations/layer23_attention_weight_min": -26.684202194213867,
      "activations/layer2_attention_weight_max": 32.37664794921875,
      "activations/layer2_attention_weight_min": -30.264577865600586,
      "activations/layer3_attention_weight_max": 93.3364028930664,
      "activations/layer3_attention_weight_min": -94.07996368408203,
      "activations/layer4_attention_weight_max": 93.96111297607422,
      "activations/layer4_attention_weight_min": -96.20979309082031,
      "activations/layer5_attention_weight_max": 72.8729019165039,
      "activations/layer5_attention_weight_min": -84.13927459716797,
      "activations/layer6_attention_weight_max": 54.48215103149414,
      "activations/layer6_attention_weight_min": -54.23573684692383,
      "activations/layer7_attention_weight_max": 71.25728607177734,
      "activations/layer7_attention_weight_min": -70.33299255371094,
      "activations/layer8_attention_weight_max": 46.27192306518555,
      "activations/layer8_attention_weight_min": -51.505760192871094,
      "activations/layer9_attention_weight_max": 49.523414611816406,
      "activations/layer9_attention_weight_min": -52.962154388427734,
      "epoch": 13.79,
      "learning_rate": 6.170454545454545e-05,
      "loss": 2.7716,
      "step": 237300
    },
    {
      "activations/layer0_attention_weight_max": 16.266799926757812,
      "activations/layer0_attention_weight_min": -14.265939712524414,
      "activations/layer10_attention_weight_max": 39.00349807739258,
      "activations/layer10_attention_weight_min": -38.27252197265625,
      "activations/layer11_attention_weight_max": 33.80812072753906,
      "activations/layer11_attention_weight_min": -34.06480407714844,
      "activations/layer12_attention_weight_max": 18.334768295288086,
      "activations/layer12_attention_weight_min": -23.1688289642334,
      "activations/layer13_attention_weight_max": 41.95957946777344,
      "activations/layer13_attention_weight_min": -29.31744384765625,
      "activations/layer14_attention_weight_max": 38.862815856933594,
      "activations/layer14_attention_weight_min": -29.4805965423584,
      "activations/layer15_attention_weight_max": 35.047115325927734,
      "activations/layer15_attention_weight_min": -30.103286743164062,
      "activations/layer16_attention_weight_max": 32.999969482421875,
      "activations/layer16_attention_weight_min": -30.445308685302734,
      "activations/layer17_attention_weight_max": 54.77029800415039,
      "activations/layer17_attention_weight_min": -43.267845153808594,
      "activations/layer18_attention_weight_max": 48.665679931640625,
      "activations/layer18_attention_weight_min": -37.960594177246094,
      "activations/layer19_attention_weight_max": 23.83679962158203,
      "activations/layer19_attention_weight_min": -21.994361877441406,
      "activations/layer1_attention_weight_max": 16.398590087890625,
      "activations/layer1_attention_weight_min": -15.517048835754395,
      "activations/layer20_attention_weight_max": 23.771028518676758,
      "activations/layer20_attention_weight_min": -22.0218448638916,
      "activations/layer21_attention_weight_max": 37.59120559692383,
      "activations/layer21_attention_weight_min": -21.525943756103516,
      "activations/layer22_attention_weight_max": 29.26247787475586,
      "activations/layer22_attention_weight_min": -25.465513229370117,
      "activations/layer23_attention_weight_max": 37.698638916015625,
      "activations/layer23_attention_weight_min": -22.78659439086914,
      "activations/layer2_attention_weight_max": 31.918304443359375,
      "activations/layer2_attention_weight_min": -32.3197021484375,
      "activations/layer3_attention_weight_max": 95.20816802978516,
      "activations/layer3_attention_weight_min": -97.38522338867188,
      "activations/layer4_attention_weight_max": 99.63086700439453,
      "activations/layer4_attention_weight_min": -99.00743103027344,
      "activations/layer5_attention_weight_max": 74.59628295898438,
      "activations/layer5_attention_weight_min": -79.31101989746094,
      "activations/layer6_attention_weight_max": 53.22145462036133,
      "activations/layer6_attention_weight_min": -57.73762130737305,
      "activations/layer7_attention_weight_max": 70.32391357421875,
      "activations/layer7_attention_weight_min": -69.60662078857422,
      "activations/layer8_attention_weight_max": 52.06142044067383,
      "activations/layer8_attention_weight_min": -50.30998229980469,
      "activations/layer9_attention_weight_max": 48.67189025878906,
      "activations/layer9_attention_weight_min": -51.97263717651367,
      "epoch": 13.79,
      "learning_rate": 6.168560606060606e-05,
      "loss": 2.7729,
      "step": 237350
    },
    {
      "activations/layer0_attention_weight_max": 16.195388793945312,
      "activations/layer0_attention_weight_min": -14.282916069030762,
      "activations/layer10_attention_weight_max": 37.868038177490234,
      "activations/layer10_attention_weight_min": -36.76457595825195,
      "activations/layer11_attention_weight_max": 33.83049011230469,
      "activations/layer11_attention_weight_min": -34.89366149902344,
      "activations/layer12_attention_weight_max": 19.193735122680664,
      "activations/layer12_attention_weight_min": -25.213281631469727,
      "activations/layer13_attention_weight_max": 36.72529983520508,
      "activations/layer13_attention_weight_min": -29.194133758544922,
      "activations/layer14_attention_weight_max": 42.456031799316406,
      "activations/layer14_attention_weight_min": -29.47780418395996,
      "activations/layer15_attention_weight_max": 39.14253234863281,
      "activations/layer15_attention_weight_min": -30.25743865966797,
      "activations/layer16_attention_weight_max": 33.26628112792969,
      "activations/layer16_attention_weight_min": -29.938007354736328,
      "activations/layer17_attention_weight_max": 57.73460006713867,
      "activations/layer17_attention_weight_min": -44.49034881591797,
      "activations/layer18_attention_weight_max": 52.06163024902344,
      "activations/layer18_attention_weight_min": -39.40503692626953,
      "activations/layer19_attention_weight_max": 25.110492706298828,
      "activations/layer19_attention_weight_min": -25.569555282592773,
      "activations/layer1_attention_weight_max": 16.814613342285156,
      "activations/layer1_attention_weight_min": -14.20226001739502,
      "activations/layer20_attention_weight_max": 26.29994773864746,
      "activations/layer20_attention_weight_min": -24.082239151000977,
      "activations/layer21_attention_weight_max": 49.15790557861328,
      "activations/layer21_attention_weight_min": -26.326904296875,
      "activations/layer22_attention_weight_max": 31.010007858276367,
      "activations/layer22_attention_weight_min": -26.0307674407959,
      "activations/layer23_attention_weight_max": 37.38939666748047,
      "activations/layer23_attention_weight_min": -23.59441375732422,
      "activations/layer2_attention_weight_max": 32.73994064331055,
      "activations/layer2_attention_weight_min": -30.069164276123047,
      "activations/layer3_attention_weight_max": 100.80487060546875,
      "activations/layer3_attention_weight_min": -97.93444061279297,
      "activations/layer4_attention_weight_max": 99.92089080810547,
      "activations/layer4_attention_weight_min": -95.99990844726562,
      "activations/layer5_attention_weight_max": 70.60206604003906,
      "activations/layer5_attention_weight_min": -78.60289001464844,
      "activations/layer6_attention_weight_max": 53.684818267822266,
      "activations/layer6_attention_weight_min": -52.564815521240234,
      "activations/layer7_attention_weight_max": 64.99189758300781,
      "activations/layer7_attention_weight_min": -64.306640625,
      "activations/layer8_attention_weight_max": 45.72694778442383,
      "activations/layer8_attention_weight_min": -47.87516784667969,
      "activations/layer9_attention_weight_max": 43.9826545715332,
      "activations/layer9_attention_weight_min": -47.25616455078125,
      "epoch": 13.79,
      "learning_rate": 6.166666666666666e-05,
      "loss": 2.7583,
      "step": 237400
    },
    {
      "activations/layer0_attention_weight_max": 16.13477897644043,
      "activations/layer0_attention_weight_min": -14.337574005126953,
      "activations/layer10_attention_weight_max": 35.01068115234375,
      "activations/layer10_attention_weight_min": -33.861820220947266,
      "activations/layer11_attention_weight_max": 30.516868591308594,
      "activations/layer11_attention_weight_min": -32.869407653808594,
      "activations/layer12_attention_weight_max": 18.594573974609375,
      "activations/layer12_attention_weight_min": -24.841089248657227,
      "activations/layer13_attention_weight_max": 33.33557891845703,
      "activations/layer13_attention_weight_min": -28.021596908569336,
      "activations/layer14_attention_weight_max": 34.47667694091797,
      "activations/layer14_attention_weight_min": -32.1482048034668,
      "activations/layer15_attention_weight_max": 32.43889617919922,
      "activations/layer15_attention_weight_min": -29.70782470703125,
      "activations/layer16_attention_weight_max": 32.004173278808594,
      "activations/layer16_attention_weight_min": -29.855182647705078,
      "activations/layer17_attention_weight_max": 52.01503372192383,
      "activations/layer17_attention_weight_min": -43.674007415771484,
      "activations/layer18_attention_weight_max": 47.18313217163086,
      "activations/layer18_attention_weight_min": -40.473575592041016,
      "activations/layer19_attention_weight_max": 21.655052185058594,
      "activations/layer19_attention_weight_min": -22.498172760009766,
      "activations/layer1_attention_weight_max": 16.861234664916992,
      "activations/layer1_attention_weight_min": -15.940970420837402,
      "activations/layer20_attention_weight_max": 23.37132453918457,
      "activations/layer20_attention_weight_min": -22.768306732177734,
      "activations/layer21_attention_weight_max": 37.878822326660156,
      "activations/layer21_attention_weight_min": -23.2601318359375,
      "activations/layer22_attention_weight_max": 30.050018310546875,
      "activations/layer22_attention_weight_min": -28.65879249572754,
      "activations/layer23_attention_weight_max": 36.78308868408203,
      "activations/layer23_attention_weight_min": -25.532154083251953,
      "activations/layer2_attention_weight_max": 32.665306091308594,
      "activations/layer2_attention_weight_min": -31.59578514099121,
      "activations/layer3_attention_weight_max": 93.85652160644531,
      "activations/layer3_attention_weight_min": -96.89397430419922,
      "activations/layer4_attention_weight_max": 94.06673431396484,
      "activations/layer4_attention_weight_min": -90.51138305664062,
      "activations/layer5_attention_weight_max": 70.0596694946289,
      "activations/layer5_attention_weight_min": -77.25788879394531,
      "activations/layer6_attention_weight_max": 51.07767105102539,
      "activations/layer6_attention_weight_min": -49.77028274536133,
      "activations/layer7_attention_weight_max": 69.4852066040039,
      "activations/layer7_attention_weight_min": -66.02764129638672,
      "activations/layer8_attention_weight_max": 45.88394546508789,
      "activations/layer8_attention_weight_min": -47.82392120361328,
      "activations/layer9_attention_weight_max": 43.96745300292969,
      "activations/layer9_attention_weight_min": -47.727867126464844,
      "epoch": 13.8,
      "learning_rate": 6.164772727272727e-05,
      "loss": 2.7542,
      "step": 237450
    },
    {
      "activations/layer0_attention_weight_max": 16.117719650268555,
      "activations/layer0_attention_weight_min": -14.580375671386719,
      "activations/layer10_attention_weight_max": 43.36667251586914,
      "activations/layer10_attention_weight_min": -40.86491775512695,
      "activations/layer11_attention_weight_max": 37.63508605957031,
      "activations/layer11_attention_weight_min": -35.560546875,
      "activations/layer12_attention_weight_max": 21.660306930541992,
      "activations/layer12_attention_weight_min": -25.449939727783203,
      "activations/layer13_attention_weight_max": 37.431392669677734,
      "activations/layer13_attention_weight_min": -32.634742736816406,
      "activations/layer14_attention_weight_max": 40.02355194091797,
      "activations/layer14_attention_weight_min": -31.018478393554688,
      "activations/layer15_attention_weight_max": 38.01925277709961,
      "activations/layer15_attention_weight_min": -30.55535888671875,
      "activations/layer16_attention_weight_max": 35.347660064697266,
      "activations/layer16_attention_weight_min": -33.099037170410156,
      "activations/layer17_attention_weight_max": 60.31029510498047,
      "activations/layer17_attention_weight_min": -54.033226013183594,
      "activations/layer18_attention_weight_max": 51.47046661376953,
      "activations/layer18_attention_weight_min": -40.91627883911133,
      "activations/layer19_attention_weight_max": 21.784317016601562,
      "activations/layer19_attention_weight_min": -22.600828170776367,
      "activations/layer1_attention_weight_max": 16.9288272857666,
      "activations/layer1_attention_weight_min": -14.573086738586426,
      "activations/layer20_attention_weight_max": 21.654808044433594,
      "activations/layer20_attention_weight_min": -23.419958114624023,
      "activations/layer21_attention_weight_max": 33.871360778808594,
      "activations/layer21_attention_weight_min": -21.725252151489258,
      "activations/layer22_attention_weight_max": 30.620616912841797,
      "activations/layer22_attention_weight_min": -25.82492446899414,
      "activations/layer23_attention_weight_max": 40.023860931396484,
      "activations/layer23_attention_weight_min": -24.17390251159668,
      "activations/layer2_attention_weight_max": 33.123069763183594,
      "activations/layer2_attention_weight_min": -30.122867584228516,
      "activations/layer3_attention_weight_max": 94.89534759521484,
      "activations/layer3_attention_weight_min": -95.36737823486328,
      "activations/layer4_attention_weight_max": 99.43612670898438,
      "activations/layer4_attention_weight_min": -99.10089111328125,
      "activations/layer5_attention_weight_max": 72.85558319091797,
      "activations/layer5_attention_weight_min": -83.89976501464844,
      "activations/layer6_attention_weight_max": 53.40916061401367,
      "activations/layer6_attention_weight_min": -54.261844635009766,
      "activations/layer7_attention_weight_max": 71.55928039550781,
      "activations/layer7_attention_weight_min": -65.30596160888672,
      "activations/layer8_attention_weight_max": 51.0331916809082,
      "activations/layer8_attention_weight_min": -50.49831008911133,
      "activations/layer9_attention_weight_max": 57.9732666015625,
      "activations/layer9_attention_weight_min": -47.9109992980957,
      "epoch": 13.8,
      "learning_rate": 6.162878787878787e-05,
      "loss": 2.7774,
      "step": 237500
    },
    {
      "activations/layer0_attention_weight_max": 15.802096366882324,
      "activations/layer0_attention_weight_min": -14.280872344970703,
      "activations/layer10_attention_weight_max": 38.426109313964844,
      "activations/layer10_attention_weight_min": -36.73659133911133,
      "activations/layer11_attention_weight_max": 33.719970703125,
      "activations/layer11_attention_weight_min": -34.70380401611328,
      "activations/layer12_attention_weight_max": 19.229267120361328,
      "activations/layer12_attention_weight_min": -21.68938446044922,
      "activations/layer13_attention_weight_max": 39.25102996826172,
      "activations/layer13_attention_weight_min": -28.967336654663086,
      "activations/layer14_attention_weight_max": 36.99313735961914,
      "activations/layer14_attention_weight_min": -30.755626678466797,
      "activations/layer15_attention_weight_max": 33.09877014160156,
      "activations/layer15_attention_weight_min": -29.672290802001953,
      "activations/layer16_attention_weight_max": 32.55699157714844,
      "activations/layer16_attention_weight_min": -32.00152587890625,
      "activations/layer17_attention_weight_max": 52.33094787597656,
      "activations/layer17_attention_weight_min": -44.11514663696289,
      "activations/layer18_attention_weight_max": 50.0027961730957,
      "activations/layer18_attention_weight_min": -39.40068817138672,
      "activations/layer19_attention_weight_max": 25.71430778503418,
      "activations/layer19_attention_weight_min": -23.62839126586914,
      "activations/layer1_attention_weight_max": 17.568267822265625,
      "activations/layer1_attention_weight_min": -15.953533172607422,
      "activations/layer20_attention_weight_max": 25.07204818725586,
      "activations/layer20_attention_weight_min": -26.022666931152344,
      "activations/layer21_attention_weight_max": 36.37335205078125,
      "activations/layer21_attention_weight_min": -22.543914794921875,
      "activations/layer22_attention_weight_max": 30.463436126708984,
      "activations/layer22_attention_weight_min": -26.308361053466797,
      "activations/layer23_attention_weight_max": 36.73225021362305,
      "activations/layer23_attention_weight_min": -26.618955612182617,
      "activations/layer2_attention_weight_max": 33.1596565246582,
      "activations/layer2_attention_weight_min": -31.843406677246094,
      "activations/layer3_attention_weight_max": 96.48590850830078,
      "activations/layer3_attention_weight_min": -97.59439849853516,
      "activations/layer4_attention_weight_max": 96.35216522216797,
      "activations/layer4_attention_weight_min": -99.09019470214844,
      "activations/layer5_attention_weight_max": 76.22807312011719,
      "activations/layer5_attention_weight_min": -81.68690490722656,
      "activations/layer6_attention_weight_max": 55.74127197265625,
      "activations/layer6_attention_weight_min": -53.2839469909668,
      "activations/layer7_attention_weight_max": 69.03620910644531,
      "activations/layer7_attention_weight_min": -67.3546371459961,
      "activations/layer8_attention_weight_max": 50.12664794921875,
      "activations/layer8_attention_weight_min": -52.625701904296875,
      "activations/layer9_attention_weight_max": 50.173648834228516,
      "activations/layer9_attention_weight_min": -49.54301071166992,
      "epoch": 13.8,
      "learning_rate": 6.160984848484847e-05,
      "loss": 2.7675,
      "step": 237550
    },
    {
      "activations/layer0_attention_weight_max": 15.595008850097656,
      "activations/layer0_attention_weight_min": -14.206053733825684,
      "activations/layer10_attention_weight_max": 34.82257080078125,
      "activations/layer10_attention_weight_min": -34.51533889770508,
      "activations/layer11_attention_weight_max": 29.829153060913086,
      "activations/layer11_attention_weight_min": -32.68064880371094,
      "activations/layer12_attention_weight_max": 17.35713768005371,
      "activations/layer12_attention_weight_min": -26.22001838684082,
      "activations/layer13_attention_weight_max": 37.516414642333984,
      "activations/layer13_attention_weight_min": -27.05536460876465,
      "activations/layer14_attention_weight_max": 35.03977966308594,
      "activations/layer14_attention_weight_min": -29.251249313354492,
      "activations/layer15_attention_weight_max": 32.58885192871094,
      "activations/layer15_attention_weight_min": -28.460195541381836,
      "activations/layer16_attention_weight_max": 30.461023330688477,
      "activations/layer16_attention_weight_min": -29.80596351623535,
      "activations/layer17_attention_weight_max": 50.48917770385742,
      "activations/layer17_attention_weight_min": -43.83053207397461,
      "activations/layer18_attention_weight_max": 45.99324035644531,
      "activations/layer18_attention_weight_min": -37.91692352294922,
      "activations/layer19_attention_weight_max": 21.247386932373047,
      "activations/layer19_attention_weight_min": -22.311376571655273,
      "activations/layer1_attention_weight_max": 15.825798988342285,
      "activations/layer1_attention_weight_min": -14.78308391571045,
      "activations/layer20_attention_weight_max": 20.69725227355957,
      "activations/layer20_attention_weight_min": -20.49199676513672,
      "activations/layer21_attention_weight_max": 33.93352127075195,
      "activations/layer21_attention_weight_min": -21.336130142211914,
      "activations/layer22_attention_weight_max": 28.629108428955078,
      "activations/layer22_attention_weight_min": -25.461891174316406,
      "activations/layer23_attention_weight_max": 36.98134994506836,
      "activations/layer23_attention_weight_min": -24.499290466308594,
      "activations/layer2_attention_weight_max": 31.575313568115234,
      "activations/layer2_attention_weight_min": -29.374319076538086,
      "activations/layer3_attention_weight_max": 92.87155151367188,
      "activations/layer3_attention_weight_min": -95.13936614990234,
      "activations/layer4_attention_weight_max": 93.0583724975586,
      "activations/layer4_attention_weight_min": -92.58161926269531,
      "activations/layer5_attention_weight_max": 71.0817642211914,
      "activations/layer5_attention_weight_min": -75.43205261230469,
      "activations/layer6_attention_weight_max": 52.28049850463867,
      "activations/layer6_attention_weight_min": -51.48950958251953,
      "activations/layer7_attention_weight_max": 66.91706085205078,
      "activations/layer7_attention_weight_min": -63.23527908325195,
      "activations/layer8_attention_weight_max": 46.39296340942383,
      "activations/layer8_attention_weight_min": -49.857505798339844,
      "activations/layer9_attention_weight_max": 42.9963264465332,
      "activations/layer9_attention_weight_min": -44.56020736694336,
      "epoch": 13.81,
      "learning_rate": 6.159090909090908e-05,
      "loss": 2.7647,
      "step": 237600
    },
    {
      "activations/layer0_attention_weight_max": 16.255062103271484,
      "activations/layer0_attention_weight_min": -14.476297378540039,
      "activations/layer10_attention_weight_max": 40.79900360107422,
      "activations/layer10_attention_weight_min": -37.264068603515625,
      "activations/layer11_attention_weight_max": 34.929046630859375,
      "activations/layer11_attention_weight_min": -34.73073959350586,
      "activations/layer12_attention_weight_max": 19.473087310791016,
      "activations/layer12_attention_weight_min": -25.30235481262207,
      "activations/layer13_attention_weight_max": 46.75186538696289,
      "activations/layer13_attention_weight_min": -29.114686965942383,
      "activations/layer14_attention_weight_max": 44.18025207519531,
      "activations/layer14_attention_weight_min": -30.835142135620117,
      "activations/layer15_attention_weight_max": 38.10603713989258,
      "activations/layer15_attention_weight_min": -30.840030670166016,
      "activations/layer16_attention_weight_max": 34.649471282958984,
      "activations/layer16_attention_weight_min": -31.86737823486328,
      "activations/layer17_attention_weight_max": 55.026100158691406,
      "activations/layer17_attention_weight_min": -44.609073638916016,
      "activations/layer18_attention_weight_max": 47.90645217895508,
      "activations/layer18_attention_weight_min": -38.034088134765625,
      "activations/layer19_attention_weight_max": 22.57447052001953,
      "activations/layer19_attention_weight_min": -20.491737365722656,
      "activations/layer1_attention_weight_max": 16.378345489501953,
      "activations/layer1_attention_weight_min": -15.23125171661377,
      "activations/layer20_attention_weight_max": 22.870065689086914,
      "activations/layer20_attention_weight_min": -20.62952423095703,
      "activations/layer21_attention_weight_max": 43.46236038208008,
      "activations/layer21_attention_weight_min": -23.054819107055664,
      "activations/layer22_attention_weight_max": 32.273719787597656,
      "activations/layer22_attention_weight_min": -24.7875919342041,
      "activations/layer23_attention_weight_max": 41.43832015991211,
      "activations/layer23_attention_weight_min": -23.990673065185547,
      "activations/layer2_attention_weight_max": 34.06066131591797,
      "activations/layer2_attention_weight_min": -30.032516479492188,
      "activations/layer3_attention_weight_max": 95.24102783203125,
      "activations/layer3_attention_weight_min": -96.03693389892578,
      "activations/layer4_attention_weight_max": 96.46480560302734,
      "activations/layer4_attention_weight_min": -91.279052734375,
      "activations/layer5_attention_weight_max": 74.26409912109375,
      "activations/layer5_attention_weight_min": -74.24485778808594,
      "activations/layer6_attention_weight_max": 53.42479705810547,
      "activations/layer6_attention_weight_min": -52.670841217041016,
      "activations/layer7_attention_weight_max": 65.82795715332031,
      "activations/layer7_attention_weight_min": -65.1849136352539,
      "activations/layer8_attention_weight_max": 49.55746078491211,
      "activations/layer8_attention_weight_min": -50.03017044067383,
      "activations/layer9_attention_weight_max": 48.3557243347168,
      "activations/layer9_attention_weight_min": -46.833255767822266,
      "epoch": 13.81,
      "learning_rate": 6.157196969696969e-05,
      "loss": 2.7442,
      "step": 237650
    },
    {
      "activations/layer0_attention_weight_max": 15.723015785217285,
      "activations/layer0_attention_weight_min": -14.267114639282227,
      "activations/layer10_attention_weight_max": 38.071861267089844,
      "activations/layer10_attention_weight_min": -36.72728729248047,
      "activations/layer11_attention_weight_max": 33.13447570800781,
      "activations/layer11_attention_weight_min": -34.637672424316406,
      "activations/layer12_attention_weight_max": 19.12110710144043,
      "activations/layer12_attention_weight_min": -29.62027359008789,
      "activations/layer13_attention_weight_max": 39.231544494628906,
      "activations/layer13_attention_weight_min": -27.803234100341797,
      "activations/layer14_attention_weight_max": 35.75041198730469,
      "activations/layer14_attention_weight_min": -29.89368438720703,
      "activations/layer15_attention_weight_max": 31.016170501708984,
      "activations/layer15_attention_weight_min": -28.90911865234375,
      "activations/layer16_attention_weight_max": 31.57073974609375,
      "activations/layer16_attention_weight_min": -29.491456985473633,
      "activations/layer17_attention_weight_max": 49.18364334106445,
      "activations/layer17_attention_weight_min": -43.48012161254883,
      "activations/layer18_attention_weight_max": 48.537662506103516,
      "activations/layer18_attention_weight_min": -39.92530822753906,
      "activations/layer19_attention_weight_max": 23.938987731933594,
      "activations/layer19_attention_weight_min": -20.91183853149414,
      "activations/layer1_attention_weight_max": 16.988523483276367,
      "activations/layer1_attention_weight_min": -14.84911823272705,
      "activations/layer20_attention_weight_max": 23.317153930664062,
      "activations/layer20_attention_weight_min": -19.66745948791504,
      "activations/layer21_attention_weight_max": 36.146217346191406,
      "activations/layer21_attention_weight_min": -21.276653289794922,
      "activations/layer22_attention_weight_max": 36.23811340332031,
      "activations/layer22_attention_weight_min": -26.29498863220215,
      "activations/layer23_attention_weight_max": 37.484375,
      "activations/layer23_attention_weight_min": -23.452939987182617,
      "activations/layer2_attention_weight_max": 32.435577392578125,
      "activations/layer2_attention_weight_min": -29.041542053222656,
      "activations/layer3_attention_weight_max": 92.94596862792969,
      "activations/layer3_attention_weight_min": -94.90013122558594,
      "activations/layer4_attention_weight_max": 96.70098876953125,
      "activations/layer4_attention_weight_min": -91.51537322998047,
      "activations/layer5_attention_weight_max": 74.9982681274414,
      "activations/layer5_attention_weight_min": -76.54707336425781,
      "activations/layer6_attention_weight_max": 53.7291374206543,
      "activations/layer6_attention_weight_min": -51.33336639404297,
      "activations/layer7_attention_weight_max": 65.05433654785156,
      "activations/layer7_attention_weight_min": -68.91557312011719,
      "activations/layer8_attention_weight_max": 45.934139251708984,
      "activations/layer8_attention_weight_min": -49.9680290222168,
      "activations/layer9_attention_weight_max": 50.77008056640625,
      "activations/layer9_attention_weight_min": -46.827110290527344,
      "epoch": 13.81,
      "learning_rate": 6.155303030303029e-05,
      "loss": 2.7516,
      "step": 237700
    },
    {
      "activations/layer0_attention_weight_max": 16.60837745666504,
      "activations/layer0_attention_weight_min": -14.358909606933594,
      "activations/layer10_attention_weight_max": 37.20565414428711,
      "activations/layer10_attention_weight_min": -34.433815002441406,
      "activations/layer11_attention_weight_max": 34.51565933227539,
      "activations/layer11_attention_weight_min": -33.88629150390625,
      "activations/layer12_attention_weight_max": 17.99173355102539,
      "activations/layer12_attention_weight_min": -22.66480827331543,
      "activations/layer13_attention_weight_max": 38.65279769897461,
      "activations/layer13_attention_weight_min": -30.69898223876953,
      "activations/layer14_attention_weight_max": 38.872867584228516,
      "activations/layer14_attention_weight_min": -28.534313201904297,
      "activations/layer15_attention_weight_max": 34.71026611328125,
      "activations/layer15_attention_weight_min": -27.905288696289062,
      "activations/layer16_attention_weight_max": 31.105709075927734,
      "activations/layer16_attention_weight_min": -29.439552307128906,
      "activations/layer17_attention_weight_max": 50.707576751708984,
      "activations/layer17_attention_weight_min": -43.76705551147461,
      "activations/layer18_attention_weight_max": 44.39735412597656,
      "activations/layer18_attention_weight_min": -36.27873992919922,
      "activations/layer19_attention_weight_max": 21.737525939941406,
      "activations/layer19_attention_weight_min": -19.51274871826172,
      "activations/layer1_attention_weight_max": 15.967727661132812,
      "activations/layer1_attention_weight_min": -14.427153587341309,
      "activations/layer20_attention_weight_max": 21.958114624023438,
      "activations/layer20_attention_weight_min": -21.311521530151367,
      "activations/layer21_attention_weight_max": 31.489599227905273,
      "activations/layer21_attention_weight_min": -20.206403732299805,
      "activations/layer22_attention_weight_max": 30.325469970703125,
      "activations/layer22_attention_weight_min": -23.235431671142578,
      "activations/layer23_attention_weight_max": 38.329627990722656,
      "activations/layer23_attention_weight_min": -23.1444091796875,
      "activations/layer2_attention_weight_max": 31.257484436035156,
      "activations/layer2_attention_weight_min": -30.012401580810547,
      "activations/layer3_attention_weight_max": 97.60523223876953,
      "activations/layer3_attention_weight_min": -102.4079360961914,
      "activations/layer4_attention_weight_max": 94.3429183959961,
      "activations/layer4_attention_weight_min": -96.51590728759766,
      "activations/layer5_attention_weight_max": 70.84993743896484,
      "activations/layer5_attention_weight_min": -81.75035095214844,
      "activations/layer6_attention_weight_max": 51.10816192626953,
      "activations/layer6_attention_weight_min": -51.257381439208984,
      "activations/layer7_attention_weight_max": 64.7768783569336,
      "activations/layer7_attention_weight_min": -63.61444854736328,
      "activations/layer8_attention_weight_max": 45.23377227783203,
      "activations/layer8_attention_weight_min": -48.69403076171875,
      "activations/layer9_attention_weight_max": 42.8227424621582,
      "activations/layer9_attention_weight_min": -45.728782653808594,
      "epoch": 13.81,
      "learning_rate": 6.153409090909091e-05,
      "loss": 2.7592,
      "step": 237750
    },
    {
      "activations/layer0_attention_weight_max": 16.931819915771484,
      "activations/layer0_attention_weight_min": -14.20313549041748,
      "activations/layer10_attention_weight_max": 35.11517333984375,
      "activations/layer10_attention_weight_min": -35.60740280151367,
      "activations/layer11_attention_weight_max": 30.267066955566406,
      "activations/layer11_attention_weight_min": -35.467063903808594,
      "activations/layer12_attention_weight_max": 17.507455825805664,
      "activations/layer12_attention_weight_min": -24.61960792541504,
      "activations/layer13_attention_weight_max": 33.76518630981445,
      "activations/layer13_attention_weight_min": -29.426982879638672,
      "activations/layer14_attention_weight_max": 34.9799690246582,
      "activations/layer14_attention_weight_min": -30.11994171142578,
      "activations/layer15_attention_weight_max": 31.92620849609375,
      "activations/layer15_attention_weight_min": -30.381330490112305,
      "activations/layer16_attention_weight_max": 30.151941299438477,
      "activations/layer16_attention_weight_min": -30.360973358154297,
      "activations/layer17_attention_weight_max": 52.36353302001953,
      "activations/layer17_attention_weight_min": -44.06131362915039,
      "activations/layer18_attention_weight_max": 43.64674758911133,
      "activations/layer18_attention_weight_min": -37.48464584350586,
      "activations/layer19_attention_weight_max": 22.84048843383789,
      "activations/layer19_attention_weight_min": -22.66353416442871,
      "activations/layer1_attention_weight_max": 15.872147560119629,
      "activations/layer1_attention_weight_min": -15.594143867492676,
      "activations/layer20_attention_weight_max": 20.823410034179688,
      "activations/layer20_attention_weight_min": -21.072872161865234,
      "activations/layer21_attention_weight_max": 34.1694221496582,
      "activations/layer21_attention_weight_min": -24.0965633392334,
      "activations/layer22_attention_weight_max": 28.857812881469727,
      "activations/layer22_attention_weight_min": -24.961181640625,
      "activations/layer23_attention_weight_max": 35.735633850097656,
      "activations/layer23_attention_weight_min": -23.55611228942871,
      "activations/layer2_attention_weight_max": 32.34657287597656,
      "activations/layer2_attention_weight_min": -30.25815200805664,
      "activations/layer3_attention_weight_max": 95.33641052246094,
      "activations/layer3_attention_weight_min": -95.31790161132812,
      "activations/layer4_attention_weight_max": 94.62288665771484,
      "activations/layer4_attention_weight_min": -95.34967803955078,
      "activations/layer5_attention_weight_max": 72.93534851074219,
      "activations/layer5_attention_weight_min": -86.598876953125,
      "activations/layer6_attention_weight_max": 51.45583724975586,
      "activations/layer6_attention_weight_min": -51.29502868652344,
      "activations/layer7_attention_weight_max": 70.46219635009766,
      "activations/layer7_attention_weight_min": -62.97536087036133,
      "activations/layer8_attention_weight_max": 45.369346618652344,
      "activations/layer8_attention_weight_min": -47.066829681396484,
      "activations/layer9_attention_weight_max": 43.748050689697266,
      "activations/layer9_attention_weight_min": -45.26119613647461,
      "epoch": 13.82,
      "learning_rate": 6.151515151515151e-05,
      "loss": 2.7409,
      "step": 237800
    },
    {
      "activations/layer0_attention_weight_max": 16.592924118041992,
      "activations/layer0_attention_weight_min": -14.296503067016602,
      "activations/layer10_attention_weight_max": 40.49810791015625,
      "activations/layer10_attention_weight_min": -37.78847122192383,
      "activations/layer11_attention_weight_max": 34.79838180541992,
      "activations/layer11_attention_weight_min": -34.00712585449219,
      "activations/layer12_attention_weight_max": 17.24176025390625,
      "activations/layer12_attention_weight_min": -25.890853881835938,
      "activations/layer13_attention_weight_max": 36.51862716674805,
      "activations/layer13_attention_weight_min": -30.42464828491211,
      "activations/layer14_attention_weight_max": 48.81264114379883,
      "activations/layer14_attention_weight_min": -31.762248992919922,
      "activations/layer15_attention_weight_max": 42.64693832397461,
      "activations/layer15_attention_weight_min": -31.23843002319336,
      "activations/layer16_attention_weight_max": 31.446191787719727,
      "activations/layer16_attention_weight_min": -30.428890228271484,
      "activations/layer17_attention_weight_max": 47.91028594970703,
      "activations/layer17_attention_weight_min": -44.34827423095703,
      "activations/layer18_attention_weight_max": 48.851890563964844,
      "activations/layer18_attention_weight_min": -41.003326416015625,
      "activations/layer19_attention_weight_max": 22.326868057250977,
      "activations/layer19_attention_weight_min": -22.368227005004883,
      "activations/layer1_attention_weight_max": 17.537193298339844,
      "activations/layer1_attention_weight_min": -15.381589889526367,
      "activations/layer20_attention_weight_max": 23.332109451293945,
      "activations/layer20_attention_weight_min": -22.93529510498047,
      "activations/layer21_attention_weight_max": 42.474342346191406,
      "activations/layer21_attention_weight_min": -24.257200241088867,
      "activations/layer22_attention_weight_max": 31.171947479248047,
      "activations/layer22_attention_weight_min": -27.182270050048828,
      "activations/layer23_attention_weight_max": 34.091468811035156,
      "activations/layer23_attention_weight_min": -25.027915954589844,
      "activations/layer2_attention_weight_max": 33.415008544921875,
      "activations/layer2_attention_weight_min": -33.57697296142578,
      "activations/layer3_attention_weight_max": 98.76160430908203,
      "activations/layer3_attention_weight_min": -97.86952209472656,
      "activations/layer4_attention_weight_max": 100.51427459716797,
      "activations/layer4_attention_weight_min": -99.6362533569336,
      "activations/layer5_attention_weight_max": 72.47720336914062,
      "activations/layer5_attention_weight_min": -81.80699157714844,
      "activations/layer6_attention_weight_max": 51.13319396972656,
      "activations/layer6_attention_weight_min": -54.48073959350586,
      "activations/layer7_attention_weight_max": 66.9048843383789,
      "activations/layer7_attention_weight_min": -67.22786712646484,
      "activations/layer8_attention_weight_max": 47.04179382324219,
      "activations/layer8_attention_weight_min": -50.61071014404297,
      "activations/layer9_attention_weight_max": 50.19406509399414,
      "activations/layer9_attention_weight_min": -49.113338470458984,
      "epoch": 13.82,
      "learning_rate": 6.149621212121212e-05,
      "loss": 2.7495,
      "step": 237850
    },
    {
      "activations/layer0_attention_weight_max": 16.39424705505371,
      "activations/layer0_attention_weight_min": -14.006843566894531,
      "activations/layer10_attention_weight_max": 36.5751953125,
      "activations/layer10_attention_weight_min": -37.157684326171875,
      "activations/layer11_attention_weight_max": 33.252967834472656,
      "activations/layer11_attention_weight_min": -34.2498664855957,
      "activations/layer12_attention_weight_max": 19.810951232910156,
      "activations/layer12_attention_weight_min": -24.158761978149414,
      "activations/layer13_attention_weight_max": 37.72603225708008,
      "activations/layer13_attention_weight_min": -29.687946319580078,
      "activations/layer14_attention_weight_max": 39.607391357421875,
      "activations/layer14_attention_weight_min": -29.761089324951172,
      "activations/layer15_attention_weight_max": 34.886898040771484,
      "activations/layer15_attention_weight_min": -28.43571662902832,
      "activations/layer16_attention_weight_max": 33.635093688964844,
      "activations/layer16_attention_weight_min": -28.25694465637207,
      "activations/layer17_attention_weight_max": 52.17736053466797,
      "activations/layer17_attention_weight_min": -42.52841567993164,
      "activations/layer18_attention_weight_max": 45.398014068603516,
      "activations/layer18_attention_weight_min": -36.152713775634766,
      "activations/layer19_attention_weight_max": 24.131010055541992,
      "activations/layer19_attention_weight_min": -20.982099533081055,
      "activations/layer1_attention_weight_max": 17.02050018310547,
      "activations/layer1_attention_weight_min": -14.18338680267334,
      "activations/layer20_attention_weight_max": 24.243993759155273,
      "activations/layer20_attention_weight_min": -20.66708755493164,
      "activations/layer21_attention_weight_max": 37.4915771484375,
      "activations/layer21_attention_weight_min": -24.126440048217773,
      "activations/layer22_attention_weight_max": 32.83491516113281,
      "activations/layer22_attention_weight_min": -24.896610260009766,
      "activations/layer23_attention_weight_max": 40.03645324707031,
      "activations/layer23_attention_weight_min": -23.682842254638672,
      "activations/layer2_attention_weight_max": 31.763317108154297,
      "activations/layer2_attention_weight_min": -30.07649040222168,
      "activations/layer3_attention_weight_max": 93.22647094726562,
      "activations/layer3_attention_weight_min": -95.3847427368164,
      "activations/layer4_attention_weight_max": 97.56343841552734,
      "activations/layer4_attention_weight_min": -96.16289520263672,
      "activations/layer5_attention_weight_max": 74.17337799072266,
      "activations/layer5_attention_weight_min": -83.34124755859375,
      "activations/layer6_attention_weight_max": 54.04885482788086,
      "activations/layer6_attention_weight_min": -53.543880462646484,
      "activations/layer7_attention_weight_max": 72.6601333618164,
      "activations/layer7_attention_weight_min": -65.7082748413086,
      "activations/layer8_attention_weight_max": 48.449954986572266,
      "activations/layer8_attention_weight_min": -51.61870193481445,
      "activations/layer9_attention_weight_max": 47.27509689331055,
      "activations/layer9_attention_weight_min": -47.888214111328125,
      "epoch": 13.82,
      "learning_rate": 6.147727272727273e-05,
      "loss": 2.7716,
      "step": 237900
    },
    {
      "activations/layer0_attention_weight_max": 15.80843448638916,
      "activations/layer0_attention_weight_min": -14.330907821655273,
      "activations/layer10_attention_weight_max": 35.40697479248047,
      "activations/layer10_attention_weight_min": -34.52816390991211,
      "activations/layer11_attention_weight_max": 32.34674072265625,
      "activations/layer11_attention_weight_min": -31.77037811279297,
      "activations/layer12_attention_weight_max": 18.3497314453125,
      "activations/layer12_attention_weight_min": -23.56851577758789,
      "activations/layer13_attention_weight_max": 39.279090881347656,
      "activations/layer13_attention_weight_min": -28.520315170288086,
      "activations/layer14_attention_weight_max": 45.28745651245117,
      "activations/layer14_attention_weight_min": -33.7575569152832,
      "activations/layer15_attention_weight_max": 40.57778549194336,
      "activations/layer15_attention_weight_min": -28.111787796020508,
      "activations/layer16_attention_weight_max": 30.77259063720703,
      "activations/layer16_attention_weight_min": -29.693145751953125,
      "activations/layer17_attention_weight_max": 47.29499053955078,
      "activations/layer17_attention_weight_min": -42.83519744873047,
      "activations/layer18_attention_weight_max": 42.70645523071289,
      "activations/layer18_attention_weight_min": -36.50507354736328,
      "activations/layer19_attention_weight_max": 21.00214195251465,
      "activations/layer19_attention_weight_min": -20.594764709472656,
      "activations/layer1_attention_weight_max": 17.238754272460938,
      "activations/layer1_attention_weight_min": -16.825820922851562,
      "activations/layer20_attention_weight_max": 20.194143295288086,
      "activations/layer20_attention_weight_min": -19.914554595947266,
      "activations/layer21_attention_weight_max": 34.93745803833008,
      "activations/layer21_attention_weight_min": -22.317123413085938,
      "activations/layer22_attention_weight_max": 27.530916213989258,
      "activations/layer22_attention_weight_min": -27.27549171447754,
      "activations/layer23_attention_weight_max": 38.26835632324219,
      "activations/layer23_attention_weight_min": -26.10687828063965,
      "activations/layer2_attention_weight_max": 33.96912384033203,
      "activations/layer2_attention_weight_min": -31.934280395507812,
      "activations/layer3_attention_weight_max": 98.22444152832031,
      "activations/layer3_attention_weight_min": -99.1187744140625,
      "activations/layer4_attention_weight_max": 95.70386505126953,
      "activations/layer4_attention_weight_min": -91.68799591064453,
      "activations/layer5_attention_weight_max": 72.56818389892578,
      "activations/layer5_attention_weight_min": -80.95808410644531,
      "activations/layer6_attention_weight_max": 53.653255462646484,
      "activations/layer6_attention_weight_min": -51.094966888427734,
      "activations/layer7_attention_weight_max": 71.39061737060547,
      "activations/layer7_attention_weight_min": -62.46876525878906,
      "activations/layer8_attention_weight_max": 44.139427185058594,
      "activations/layer8_attention_weight_min": -47.1380729675293,
      "activations/layer9_attention_weight_max": 45.05118179321289,
      "activations/layer9_attention_weight_min": -44.77285385131836,
      "epoch": 13.83,
      "learning_rate": 6.145833333333333e-05,
      "loss": 2.7698,
      "step": 237950
    },
    {
      "activations/layer0_attention_weight_max": 16.834794998168945,
      "activations/layer0_attention_weight_min": -14.490363121032715,
      "activations/layer10_attention_weight_max": 34.726341247558594,
      "activations/layer10_attention_weight_min": -35.873077392578125,
      "activations/layer11_attention_weight_max": 32.02819061279297,
      "activations/layer11_attention_weight_min": -32.18280029296875,
      "activations/layer12_attention_weight_max": 18.021549224853516,
      "activations/layer12_attention_weight_min": -28.47945785522461,
      "activations/layer13_attention_weight_max": 34.13206100463867,
      "activations/layer13_attention_weight_min": -28.86259651184082,
      "activations/layer14_attention_weight_max": 37.086029052734375,
      "activations/layer14_attention_weight_min": -32.0298957824707,
      "activations/layer15_attention_weight_max": 32.351558685302734,
      "activations/layer15_attention_weight_min": -30.882200241088867,
      "activations/layer16_attention_weight_max": 32.32914733886719,
      "activations/layer16_attention_weight_min": -32.941009521484375,
      "activations/layer17_attention_weight_max": 51.43149948120117,
      "activations/layer17_attention_weight_min": -46.820796966552734,
      "activations/layer18_attention_weight_max": 46.48656463623047,
      "activations/layer18_attention_weight_min": -41.07342529296875,
      "activations/layer19_attention_weight_max": 24.03589630126953,
      "activations/layer19_attention_weight_min": -21.84540367126465,
      "activations/layer1_attention_weight_max": 16.461061477661133,
      "activations/layer1_attention_weight_min": -14.737591743469238,
      "activations/layer20_attention_weight_max": 23.867006301879883,
      "activations/layer20_attention_weight_min": -23.961990356445312,
      "activations/layer21_attention_weight_max": 38.527374267578125,
      "activations/layer21_attention_weight_min": -21.97626304626465,
      "activations/layer22_attention_weight_max": 30.061704635620117,
      "activations/layer22_attention_weight_min": -25.160228729248047,
      "activations/layer23_attention_weight_max": 39.587738037109375,
      "activations/layer23_attention_weight_min": -24.873558044433594,
      "activations/layer2_attention_weight_max": 35.275489807128906,
      "activations/layer2_attention_weight_min": -32.88690948486328,
      "activations/layer3_attention_weight_max": 102.41586303710938,
      "activations/layer3_attention_weight_min": -109.98277282714844,
      "activations/layer4_attention_weight_max": 104.80167388916016,
      "activations/layer4_attention_weight_min": -106.77190399169922,
      "activations/layer5_attention_weight_max": 77.12649536132812,
      "activations/layer5_attention_weight_min": -83.00852966308594,
      "activations/layer6_attention_weight_max": 55.249481201171875,
      "activations/layer6_attention_weight_min": -55.148197174072266,
      "activations/layer7_attention_weight_max": 70.41776275634766,
      "activations/layer7_attention_weight_min": -68.78569030761719,
      "activations/layer8_attention_weight_max": 50.78310775756836,
      "activations/layer8_attention_weight_min": -49.752925872802734,
      "activations/layer9_attention_weight_max": 47.41136169433594,
      "activations/layer9_attention_weight_min": -46.89124298095703,
      "epoch": 13.83,
      "learning_rate": 6.143939393939393e-05,
      "loss": 2.7743,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4306,
      "eval_samples_per_second": 509.336,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4306,
      "eval_openwebtext_samples_per_second": 509.336,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9666,
      "eval_wikitext_samples_per_second": 231.872,
      "step": 238000
    },
    {
      "epoch": 13.83,
      "eval_lambada_loss": 2.53515625,
      "eval_lambada_ppl": 12.618402319374093,
      "eval_lambada_runtime": 9.5787,
      "eval_lambada_samples_per_second": 508.314,
      "step": 238000
    },
    {
      "activations/layer0_attention_weight_max": 16.711593627929688,
      "activations/layer0_attention_weight_min": -14.370949745178223,
      "activations/layer10_attention_weight_max": 36.98686599731445,
      "activations/layer10_attention_weight_min": -35.903141021728516,
      "activations/layer11_attention_weight_max": 32.67420959472656,
      "activations/layer11_attention_weight_min": -33.844871520996094,
      "activations/layer12_attention_weight_max": 18.396024703979492,
      "activations/layer12_attention_weight_min": -26.779067993164062,
      "activations/layer13_attention_weight_max": 38.5194091796875,
      "activations/layer13_attention_weight_min": -32.89563751220703,
      "activations/layer14_attention_weight_max": 39.015621185302734,
      "activations/layer14_attention_weight_min": -31.82624053955078,
      "activations/layer15_attention_weight_max": 33.46141052246094,
      "activations/layer15_attention_weight_min": -31.00902557373047,
      "activations/layer16_attention_weight_max": 32.75007247924805,
      "activations/layer16_attention_weight_min": -30.167150497436523,
      "activations/layer17_attention_weight_max": 55.3166389465332,
      "activations/layer17_attention_weight_min": -45.54609680175781,
      "activations/layer18_attention_weight_max": 49.00715255737305,
      "activations/layer18_attention_weight_min": -42.78963851928711,
      "activations/layer19_attention_weight_max": 26.330322265625,
      "activations/layer19_attention_weight_min": -22.464950561523438,
      "activations/layer1_attention_weight_max": 16.787508010864258,
      "activations/layer1_attention_weight_min": -14.283434867858887,
      "activations/layer20_attention_weight_max": 23.999746322631836,
      "activations/layer20_attention_weight_min": -22.09949493408203,
      "activations/layer21_attention_weight_max": 39.48019790649414,
      "activations/layer21_attention_weight_min": -26.74625015258789,
      "activations/layer22_attention_weight_max": 31.94046974182129,
      "activations/layer22_attention_weight_min": -26.47203254699707,
      "activations/layer23_attention_weight_max": 39.57561492919922,
      "activations/layer23_attention_weight_min": -27.588125228881836,
      "activations/layer2_attention_weight_max": 34.605384826660156,
      "activations/layer2_attention_weight_min": -34.26176452636719,
      "activations/layer3_attention_weight_max": 96.39506530761719,
      "activations/layer3_attention_weight_min": -105.23136138916016,
      "activations/layer4_attention_weight_max": 94.5449447631836,
      "activations/layer4_attention_weight_min": -92.44514465332031,
      "activations/layer5_attention_weight_max": 71.31568908691406,
      "activations/layer5_attention_weight_min": -81.685302734375,
      "activations/layer6_attention_weight_max": 52.760982513427734,
      "activations/layer6_attention_weight_min": -51.42887496948242,
      "activations/layer7_attention_weight_max": 66.71639251708984,
      "activations/layer7_attention_weight_min": -61.926815032958984,
      "activations/layer8_attention_weight_max": 48.82467269897461,
      "activations/layer8_attention_weight_min": -49.722904205322266,
      "activations/layer9_attention_weight_max": 46.364566802978516,
      "activations/layer9_attention_weight_min": -45.66979217529297,
      "epoch": 13.83,
      "learning_rate": 6.142045454545454e-05,
      "loss": 2.78,
      "step": 238050
    },
    {
      "activations/layer0_attention_weight_max": 16.121660232543945,
      "activations/layer0_attention_weight_min": -14.396228790283203,
      "activations/layer10_attention_weight_max": 37.06507110595703,
      "activations/layer10_attention_weight_min": -37.35872268676758,
      "activations/layer11_attention_weight_max": 32.808876037597656,
      "activations/layer11_attention_weight_min": -35.34397506713867,
      "activations/layer12_attention_weight_max": 18.691394805908203,
      "activations/layer12_attention_weight_min": -25.923824310302734,
      "activations/layer13_attention_weight_max": 35.92318344116211,
      "activations/layer13_attention_weight_min": -27.81187629699707,
      "activations/layer14_attention_weight_max": 37.435333251953125,
      "activations/layer14_attention_weight_min": -29.39459228515625,
      "activations/layer15_attention_weight_max": 34.140106201171875,
      "activations/layer15_attention_weight_min": -27.597511291503906,
      "activations/layer16_attention_weight_max": 34.9991569519043,
      "activations/layer16_attention_weight_min": -31.124414443969727,
      "activations/layer17_attention_weight_max": 54.06112289428711,
      "activations/layer17_attention_weight_min": -44.74202346801758,
      "activations/layer18_attention_weight_max": 50.23899459838867,
      "activations/layer18_attention_weight_min": -39.064910888671875,
      "activations/layer19_attention_weight_max": 23.55501365661621,
      "activations/layer19_attention_weight_min": -24.379962921142578,
      "activations/layer1_attention_weight_max": 15.62053394317627,
      "activations/layer1_attention_weight_min": -15.155763626098633,
      "activations/layer20_attention_weight_max": 22.61427879333496,
      "activations/layer20_attention_weight_min": -21.453126907348633,
      "activations/layer21_attention_weight_max": 39.15898513793945,
      "activations/layer21_attention_weight_min": -21.227876663208008,
      "activations/layer22_attention_weight_max": 33.1834716796875,
      "activations/layer22_attention_weight_min": -26.424314498901367,
      "activations/layer23_attention_weight_max": 40.76011276245117,
      "activations/layer23_attention_weight_min": -24.50668716430664,
      "activations/layer2_attention_weight_max": 35.12110900878906,
      "activations/layer2_attention_weight_min": -33.783851623535156,
      "activations/layer3_attention_weight_max": 96.38064575195312,
      "activations/layer3_attention_weight_min": -100.33802795410156,
      "activations/layer4_attention_weight_max": 97.90007781982422,
      "activations/layer4_attention_weight_min": -94.71969604492188,
      "activations/layer5_attention_weight_max": 73.16798400878906,
      "activations/layer5_attention_weight_min": -78.80989837646484,
      "activations/layer6_attention_weight_max": 55.27143859863281,
      "activations/layer6_attention_weight_min": -54.27232360839844,
      "activations/layer7_attention_weight_max": 65.95805358886719,
      "activations/layer7_attention_weight_min": -65.7988510131836,
      "activations/layer8_attention_weight_max": 46.3334846496582,
      "activations/layer8_attention_weight_min": -47.83731460571289,
      "activations/layer9_attention_weight_max": 43.52299499511719,
      "activations/layer9_attention_weight_min": -45.90913009643555,
      "epoch": 13.83,
      "learning_rate": 6.140151515151515e-05,
      "loss": 2.7722,
      "step": 238100
    },
    {
      "activations/layer0_attention_weight_max": 16.888545989990234,
      "activations/layer0_attention_weight_min": -14.133105278015137,
      "activations/layer10_attention_weight_max": 43.55886459350586,
      "activations/layer10_attention_weight_min": -43.293785095214844,
      "activations/layer11_attention_weight_max": 42.14775848388672,
      "activations/layer11_attention_weight_min": -38.23455047607422,
      "activations/layer12_attention_weight_max": 17.71053123474121,
      "activations/layer12_attention_weight_min": -30.114749908447266,
      "activations/layer13_attention_weight_max": 34.947181701660156,
      "activations/layer13_attention_weight_min": -30.813579559326172,
      "activations/layer14_attention_weight_max": 38.693458557128906,
      "activations/layer14_attention_weight_min": -31.121061325073242,
      "activations/layer15_attention_weight_max": 34.196617126464844,
      "activations/layer15_attention_weight_min": -31.530385971069336,
      "activations/layer16_attention_weight_max": 29.423625946044922,
      "activations/layer16_attention_weight_min": -30.031551361083984,
      "activations/layer17_attention_weight_max": 47.690521240234375,
      "activations/layer17_attention_weight_min": -43.13703918457031,
      "activations/layer18_attention_weight_max": 42.54900360107422,
      "activations/layer18_attention_weight_min": -36.728519439697266,
      "activations/layer19_attention_weight_max": 21.349084854125977,
      "activations/layer19_attention_weight_min": -22.066364288330078,
      "activations/layer1_attention_weight_max": 16.352279663085938,
      "activations/layer1_attention_weight_min": -15.407472610473633,
      "activations/layer20_attention_weight_max": 21.773000717163086,
      "activations/layer20_attention_weight_min": -22.440322875976562,
      "activations/layer21_attention_weight_max": 32.80713653564453,
      "activations/layer21_attention_weight_min": -23.547460556030273,
      "activations/layer22_attention_weight_max": 29.90715217590332,
      "activations/layer22_attention_weight_min": -26.871065139770508,
      "activations/layer23_attention_weight_max": 34.3275032043457,
      "activations/layer23_attention_weight_min": -22.317808151245117,
      "activations/layer2_attention_weight_max": 32.718353271484375,
      "activations/layer2_attention_weight_min": -33.41472244262695,
      "activations/layer3_attention_weight_max": 89.66688537597656,
      "activations/layer3_attention_weight_min": -94.45830535888672,
      "activations/layer4_attention_weight_max": 96.58238983154297,
      "activations/layer4_attention_weight_min": -92.73441314697266,
      "activations/layer5_attention_weight_max": 71.6897964477539,
      "activations/layer5_attention_weight_min": -76.28842163085938,
      "activations/layer6_attention_weight_max": 54.85354232788086,
      "activations/layer6_attention_weight_min": -53.83616256713867,
      "activations/layer7_attention_weight_max": 77.19588470458984,
      "activations/layer7_attention_weight_min": -63.24407196044922,
      "activations/layer8_attention_weight_max": 54.99467086791992,
      "activations/layer8_attention_weight_min": -47.75211715698242,
      "activations/layer9_attention_weight_max": 69.93206024169922,
      "activations/layer9_attention_weight_min": -49.79286575317383,
      "epoch": 13.84,
      "learning_rate": 6.138257575757575e-05,
      "loss": 2.7715,
      "step": 238150
    },
    {
      "activations/layer0_attention_weight_max": 16.66303825378418,
      "activations/layer0_attention_weight_min": -14.1520414352417,
      "activations/layer10_attention_weight_max": 37.47844314575195,
      "activations/layer10_attention_weight_min": -37.94731903076172,
      "activations/layer11_attention_weight_max": 35.594696044921875,
      "activations/layer11_attention_weight_min": -35.36421203613281,
      "activations/layer12_attention_weight_max": 18.890480041503906,
      "activations/layer12_attention_weight_min": -23.0736141204834,
      "activations/layer13_attention_weight_max": 51.77598571777344,
      "activations/layer13_attention_weight_min": -34.47309112548828,
      "activations/layer14_attention_weight_max": 40.81801986694336,
      "activations/layer14_attention_weight_min": -30.73977279663086,
      "activations/layer15_attention_weight_max": 34.567108154296875,
      "activations/layer15_attention_weight_min": -30.303516387939453,
      "activations/layer16_attention_weight_max": 32.03200912475586,
      "activations/layer16_attention_weight_min": -31.51224708557129,
      "activations/layer17_attention_weight_max": 51.38539123535156,
      "activations/layer17_attention_weight_min": -46.98895263671875,
      "activations/layer18_attention_weight_max": 48.97552490234375,
      "activations/layer18_attention_weight_min": -42.95655822753906,
      "activations/layer19_attention_weight_max": 25.106708526611328,
      "activations/layer19_attention_weight_min": -26.162038803100586,
      "activations/layer1_attention_weight_max": 16.843320846557617,
      "activations/layer1_attention_weight_min": -14.134439468383789,
      "activations/layer20_attention_weight_max": 25.287147521972656,
      "activations/layer20_attention_weight_min": -22.54503631591797,
      "activations/layer21_attention_weight_max": 44.08441925048828,
      "activations/layer21_attention_weight_min": -25.046249389648438,
      "activations/layer22_attention_weight_max": 31.49580192565918,
      "activations/layer22_attention_weight_min": -28.427371978759766,
      "activations/layer23_attention_weight_max": 42.85304260253906,
      "activations/layer23_attention_weight_min": -28.46057891845703,
      "activations/layer2_attention_weight_max": 33.197303771972656,
      "activations/layer2_attention_weight_min": -32.73304748535156,
      "activations/layer3_attention_weight_max": 96.24796295166016,
      "activations/layer3_attention_weight_min": -98.22930145263672,
      "activations/layer4_attention_weight_max": 98.890380859375,
      "activations/layer4_attention_weight_min": -98.6650619506836,
      "activations/layer5_attention_weight_max": 74.13270568847656,
      "activations/layer5_attention_weight_min": -82.77951049804688,
      "activations/layer6_attention_weight_max": 54.40882110595703,
      "activations/layer6_attention_weight_min": -53.805233001708984,
      "activations/layer7_attention_weight_max": 71.8976058959961,
      "activations/layer7_attention_weight_min": -65.17217254638672,
      "activations/layer8_attention_weight_max": 50.126075744628906,
      "activations/layer8_attention_weight_min": -50.214698791503906,
      "activations/layer9_attention_weight_max": 51.29934310913086,
      "activations/layer9_attention_weight_min": -46.380584716796875,
      "epoch": 13.84,
      "learning_rate": 6.136363636363636e-05,
      "loss": 2.7632,
      "step": 238200
    },
    {
      "activations/layer0_attention_weight_max": 17.497556686401367,
      "activations/layer0_attention_weight_min": -14.68298053741455,
      "activations/layer10_attention_weight_max": 37.35135269165039,
      "activations/layer10_attention_weight_min": -37.7310791015625,
      "activations/layer11_attention_weight_max": 33.48992156982422,
      "activations/layer11_attention_weight_min": -34.95195007324219,
      "activations/layer12_attention_weight_max": 19.00857162475586,
      "activations/layer12_attention_weight_min": -25.300512313842773,
      "activations/layer13_attention_weight_max": 38.95743942260742,
      "activations/layer13_attention_weight_min": -30.78615951538086,
      "activations/layer14_attention_weight_max": 38.606632232666016,
      "activations/layer14_attention_weight_min": -34.258033752441406,
      "activations/layer15_attention_weight_max": 34.74531555175781,
      "activations/layer15_attention_weight_min": -31.6382999420166,
      "activations/layer16_attention_weight_max": 36.686214447021484,
      "activations/layer16_attention_weight_min": -32.0133056640625,
      "activations/layer17_attention_weight_max": 53.379005432128906,
      "activations/layer17_attention_weight_min": -47.54389572143555,
      "activations/layer18_attention_weight_max": 48.458106994628906,
      "activations/layer18_attention_weight_min": -39.8765983581543,
      "activations/layer19_attention_weight_max": 21.458284378051758,
      "activations/layer19_attention_weight_min": -22.537561416625977,
      "activations/layer1_attention_weight_max": 15.909358024597168,
      "activations/layer1_attention_weight_min": -14.063641548156738,
      "activations/layer20_attention_weight_max": 21.248701095581055,
      "activations/layer20_attention_weight_min": -22.842557907104492,
      "activations/layer21_attention_weight_max": 38.21025848388672,
      "activations/layer21_attention_weight_min": -22.834012985229492,
      "activations/layer22_attention_weight_max": 28.459110260009766,
      "activations/layer22_attention_weight_min": -26.015851974487305,
      "activations/layer23_attention_weight_max": 38.50746154785156,
      "activations/layer23_attention_weight_min": -24.591976165771484,
      "activations/layer2_attention_weight_max": 34.266822814941406,
      "activations/layer2_attention_weight_min": -31.061965942382812,
      "activations/layer3_attention_weight_max": 91.60913848876953,
      "activations/layer3_attention_weight_min": -91.49075317382812,
      "activations/layer4_attention_weight_max": 94.44332885742188,
      "activations/layer4_attention_weight_min": -88.45358276367188,
      "activations/layer5_attention_weight_max": 71.23869323730469,
      "activations/layer5_attention_weight_min": -80.46697998046875,
      "activations/layer6_attention_weight_max": 52.840755462646484,
      "activations/layer6_attention_weight_min": -51.70859146118164,
      "activations/layer7_attention_weight_max": 69.58718872070312,
      "activations/layer7_attention_weight_min": -63.972572326660156,
      "activations/layer8_attention_weight_max": 46.71376419067383,
      "activations/layer8_attention_weight_min": -50.30955505371094,
      "activations/layer9_attention_weight_max": 46.95026779174805,
      "activations/layer9_attention_weight_min": -46.80160903930664,
      "epoch": 13.84,
      "learning_rate": 6.134469696969696e-05,
      "loss": 2.7641,
      "step": 238250
    },
    {
      "activations/layer0_attention_weight_max": 17.549541473388672,
      "activations/layer0_attention_weight_min": -14.887093544006348,
      "activations/layer10_attention_weight_max": 39.314697265625,
      "activations/layer10_attention_weight_min": -37.033355712890625,
      "activations/layer11_attention_weight_max": 33.896568298339844,
      "activations/layer11_attention_weight_min": -34.40340042114258,
      "activations/layer12_attention_weight_max": 20.154544830322266,
      "activations/layer12_attention_weight_min": -27.900854110717773,
      "activations/layer13_attention_weight_max": 38.5130500793457,
      "activations/layer13_attention_weight_min": -31.6568546295166,
      "activations/layer14_attention_weight_max": 37.71879577636719,
      "activations/layer14_attention_weight_min": -29.81832504272461,
      "activations/layer15_attention_weight_max": 33.16761779785156,
      "activations/layer15_attention_weight_min": -30.356739044189453,
      "activations/layer16_attention_weight_max": 31.923587799072266,
      "activations/layer16_attention_weight_min": -31.7236328125,
      "activations/layer17_attention_weight_max": 54.496490478515625,
      "activations/layer17_attention_weight_min": -44.30897903442383,
      "activations/layer18_attention_weight_max": 48.626312255859375,
      "activations/layer18_attention_weight_min": -38.477779388427734,
      "activations/layer19_attention_weight_max": 21.815109252929688,
      "activations/layer19_attention_weight_min": -24.200088500976562,
      "activations/layer1_attention_weight_max": 16.252155303955078,
      "activations/layer1_attention_weight_min": -14.86912727355957,
      "activations/layer20_attention_weight_max": 22.904518127441406,
      "activations/layer20_attention_weight_min": -21.98345184326172,
      "activations/layer21_attention_weight_max": 39.191097259521484,
      "activations/layer21_attention_weight_min": -23.246387481689453,
      "activations/layer22_attention_weight_max": 30.851715087890625,
      "activations/layer22_attention_weight_min": -26.192768096923828,
      "activations/layer23_attention_weight_max": 39.77741622924805,
      "activations/layer23_attention_weight_min": -25.328367233276367,
      "activations/layer2_attention_weight_max": 33.435157775878906,
      "activations/layer2_attention_weight_min": -32.21656799316406,
      "activations/layer3_attention_weight_max": 97.69752502441406,
      "activations/layer3_attention_weight_min": -100.32671356201172,
      "activations/layer4_attention_weight_max": 101.0705795288086,
      "activations/layer4_attention_weight_min": -99.4660415649414,
      "activations/layer5_attention_weight_max": 72.07726287841797,
      "activations/layer5_attention_weight_min": -78.75914001464844,
      "activations/layer6_attention_weight_max": 56.882266998291016,
      "activations/layer6_attention_weight_min": -56.57482147216797,
      "activations/layer7_attention_weight_max": 69.9228286743164,
      "activations/layer7_attention_weight_min": -70.24337768554688,
      "activations/layer8_attention_weight_max": 50.48535919189453,
      "activations/layer8_attention_weight_min": -53.25322341918945,
      "activations/layer9_attention_weight_max": 50.02487564086914,
      "activations/layer9_attention_weight_min": -50.57308578491211,
      "epoch": 13.85,
      "learning_rate": 6.132575757575756e-05,
      "loss": 2.778,
      "step": 238300
    },
    {
      "activations/layer0_attention_weight_max": 15.876753807067871,
      "activations/layer0_attention_weight_min": -14.774380683898926,
      "activations/layer10_attention_weight_max": 36.41972732543945,
      "activations/layer10_attention_weight_min": -36.46208190917969,
      "activations/layer11_attention_weight_max": 30.600997924804688,
      "activations/layer11_attention_weight_min": -31.184608459472656,
      "activations/layer12_attention_weight_max": 17.59644317626953,
      "activations/layer12_attention_weight_min": -26.989152908325195,
      "activations/layer13_attention_weight_max": 34.809139251708984,
      "activations/layer13_attention_weight_min": -27.702959060668945,
      "activations/layer14_attention_weight_max": 35.77142333984375,
      "activations/layer14_attention_weight_min": -29.689363479614258,
      "activations/layer15_attention_weight_max": 33.58419418334961,
      "activations/layer15_attention_weight_min": -29.87470054626465,
      "activations/layer16_attention_weight_max": 31.239030838012695,
      "activations/layer16_attention_weight_min": -30.73771095275879,
      "activations/layer17_attention_weight_max": 50.4676628112793,
      "activations/layer17_attention_weight_min": -43.08495330810547,
      "activations/layer18_attention_weight_max": 45.68407440185547,
      "activations/layer18_attention_weight_min": -38.43671798706055,
      "activations/layer19_attention_weight_max": 23.761756896972656,
      "activations/layer19_attention_weight_min": -21.546628952026367,
      "activations/layer1_attention_weight_max": 16.90224266052246,
      "activations/layer1_attention_weight_min": -14.439237594604492,
      "activations/layer20_attention_weight_max": 23.95989227294922,
      "activations/layer20_attention_weight_min": -21.13408660888672,
      "activations/layer21_attention_weight_max": 38.24969482421875,
      "activations/layer21_attention_weight_min": -23.749547958374023,
      "activations/layer22_attention_weight_max": 30.497127532958984,
      "activations/layer22_attention_weight_min": -25.521942138671875,
      "activations/layer23_attention_weight_max": 37.06222915649414,
      "activations/layer23_attention_weight_min": -23.956579208374023,
      "activations/layer2_attention_weight_max": 32.38397979736328,
      "activations/layer2_attention_weight_min": -31.946636199951172,
      "activations/layer3_attention_weight_max": 94.41887664794922,
      "activations/layer3_attention_weight_min": -99.92527770996094,
      "activations/layer4_attention_weight_max": 94.06639862060547,
      "activations/layer4_attention_weight_min": -93.0328140258789,
      "activations/layer5_attention_weight_max": 71.85179138183594,
      "activations/layer5_attention_weight_min": -79.74653625488281,
      "activations/layer6_attention_weight_max": 53.578407287597656,
      "activations/layer6_attention_weight_min": -53.523563385009766,
      "activations/layer7_attention_weight_max": 65.16668701171875,
      "activations/layer7_attention_weight_min": -65.64559173583984,
      "activations/layer8_attention_weight_max": 44.09515380859375,
      "activations/layer8_attention_weight_min": -46.850502014160156,
      "activations/layer9_attention_weight_max": 43.47600555419922,
      "activations/layer9_attention_weight_min": -48.753807067871094,
      "epoch": 13.85,
      "learning_rate": 6.130681818181818e-05,
      "loss": 2.7471,
      "step": 238350
    },
    {
      "activations/layer0_attention_weight_max": 15.407756805419922,
      "activations/layer0_attention_weight_min": -14.558088302612305,
      "activations/layer10_attention_weight_max": 40.13526916503906,
      "activations/layer10_attention_weight_min": -38.92752456665039,
      "activations/layer11_attention_weight_max": 36.324501037597656,
      "activations/layer11_attention_weight_min": -35.77142333984375,
      "activations/layer12_attention_weight_max": 20.539981842041016,
      "activations/layer12_attention_weight_min": -24.668167114257812,
      "activations/layer13_attention_weight_max": 40.014366149902344,
      "activations/layer13_attention_weight_min": -31.787683486938477,
      "activations/layer14_attention_weight_max": 38.09639358520508,
      "activations/layer14_attention_weight_min": -32.71200180053711,
      "activations/layer15_attention_weight_max": 33.11241149902344,
      "activations/layer15_attention_weight_min": -31.1909122467041,
      "activations/layer16_attention_weight_max": 33.35800552368164,
      "activations/layer16_attention_weight_min": -30.773393630981445,
      "activations/layer17_attention_weight_max": 53.78197479248047,
      "activations/layer17_attention_weight_min": -45.85982131958008,
      "activations/layer18_attention_weight_max": 48.84990310668945,
      "activations/layer18_attention_weight_min": -40.86948776245117,
      "activations/layer19_attention_weight_max": 24.692455291748047,
      "activations/layer19_attention_weight_min": -24.05621910095215,
      "activations/layer1_attention_weight_max": 16.54517364501953,
      "activations/layer1_attention_weight_min": -14.513338088989258,
      "activations/layer20_attention_weight_max": 26.24855613708496,
      "activations/layer20_attention_weight_min": -20.943878173828125,
      "activations/layer21_attention_weight_max": 34.538150787353516,
      "activations/layer21_attention_weight_min": -21.608129501342773,
      "activations/layer22_attention_weight_max": 32.963584899902344,
      "activations/layer22_attention_weight_min": -28.1929874420166,
      "activations/layer23_attention_weight_max": 38.02893829345703,
      "activations/layer23_attention_weight_min": -23.33466148376465,
      "activations/layer2_attention_weight_max": 35.102699279785156,
      "activations/layer2_attention_weight_min": -32.82832336425781,
      "activations/layer3_attention_weight_max": 98.67488098144531,
      "activations/layer3_attention_weight_min": -103.43579864501953,
      "activations/layer4_attention_weight_max": 102.6854019165039,
      "activations/layer4_attention_weight_min": -100.52802276611328,
      "activations/layer5_attention_weight_max": 79.05902099609375,
      "activations/layer5_attention_weight_min": -80.19075775146484,
      "activations/layer6_attention_weight_max": 55.40524673461914,
      "activations/layer6_attention_weight_min": -54.7224006652832,
      "activations/layer7_attention_weight_max": 73.68515014648438,
      "activations/layer7_attention_weight_min": -66.561767578125,
      "activations/layer8_attention_weight_max": 48.59086227416992,
      "activations/layer8_attention_weight_min": -51.55772399902344,
      "activations/layer9_attention_weight_max": 50.156211853027344,
      "activations/layer9_attention_weight_min": -49.06237030029297,
      "epoch": 13.85,
      "learning_rate": 6.128787878787878e-05,
      "loss": 2.7512,
      "step": 238400
    },
    {
      "activations/layer0_attention_weight_max": 15.702190399169922,
      "activations/layer0_attention_weight_min": -14.661382675170898,
      "activations/layer10_attention_weight_max": 36.98527145385742,
      "activations/layer10_attention_weight_min": -35.95153045654297,
      "activations/layer11_attention_weight_max": 36.28431701660156,
      "activations/layer11_attention_weight_min": -33.68749237060547,
      "activations/layer12_attention_weight_max": 19.554967880249023,
      "activations/layer12_attention_weight_min": -27.29485321044922,
      "activations/layer13_attention_weight_max": 45.1182975769043,
      "activations/layer13_attention_weight_min": -31.27072525024414,
      "activations/layer14_attention_weight_max": 63.14893341064453,
      "activations/layer14_attention_weight_min": -32.74660110473633,
      "activations/layer15_attention_weight_max": 51.4401741027832,
      "activations/layer15_attention_weight_min": -30.642656326293945,
      "activations/layer16_attention_weight_max": 37.28274917602539,
      "activations/layer16_attention_weight_min": -30.76812744140625,
      "activations/layer17_attention_weight_max": 56.82067108154297,
      "activations/layer17_attention_weight_min": -48.48014831542969,
      "activations/layer18_attention_weight_max": 54.955867767333984,
      "activations/layer18_attention_weight_min": -39.4857063293457,
      "activations/layer19_attention_weight_max": 29.575565338134766,
      "activations/layer19_attention_weight_min": -23.614477157592773,
      "activations/layer1_attention_weight_max": 16.58499526977539,
      "activations/layer1_attention_weight_min": -13.658700942993164,
      "activations/layer20_attention_weight_max": 30.140396118164062,
      "activations/layer20_attention_weight_min": -22.201480865478516,
      "activations/layer21_attention_weight_max": 50.84260940551758,
      "activations/layer21_attention_weight_min": -24.46999168395996,
      "activations/layer22_attention_weight_max": 37.71370315551758,
      "activations/layer22_attention_weight_min": -25.645526885986328,
      "activations/layer23_attention_weight_max": 42.50937271118164,
      "activations/layer23_attention_weight_min": -24.564006805419922,
      "activations/layer2_attention_weight_max": 34.210655212402344,
      "activations/layer2_attention_weight_min": -30.898204803466797,
      "activations/layer3_attention_weight_max": 104.55888366699219,
      "activations/layer3_attention_weight_min": -98.38607788085938,
      "activations/layer4_attention_weight_max": 98.85295867919922,
      "activations/layer4_attention_weight_min": -94.83794403076172,
      "activations/layer5_attention_weight_max": 72.89036560058594,
      "activations/layer5_attention_weight_min": -76.21805572509766,
      "activations/layer6_attention_weight_max": 50.03400802612305,
      "activations/layer6_attention_weight_min": -51.57994079589844,
      "activations/layer7_attention_weight_max": 66.81141662597656,
      "activations/layer7_attention_weight_min": -65.46300506591797,
      "activations/layer8_attention_weight_max": 46.536808013916016,
      "activations/layer8_attention_weight_min": -50.060951232910156,
      "activations/layer9_attention_weight_max": 54.9171142578125,
      "activations/layer9_attention_weight_min": -47.06960678100586,
      "epoch": 13.86,
      "learning_rate": 6.126893939393938e-05,
      "loss": 2.7699,
      "step": 238450
    },
    {
      "activations/layer0_attention_weight_max": 16.373334884643555,
      "activations/layer0_attention_weight_min": -14.713424682617188,
      "activations/layer10_attention_weight_max": 37.696205139160156,
      "activations/layer10_attention_weight_min": -33.86161804199219,
      "activations/layer11_attention_weight_max": 32.52674102783203,
      "activations/layer11_attention_weight_min": -33.170623779296875,
      "activations/layer12_attention_weight_max": 21.612979888916016,
      "activations/layer12_attention_weight_min": -23.669857025146484,
      "activations/layer13_attention_weight_max": 47.48893737792969,
      "activations/layer13_attention_weight_min": -28.051326751708984,
      "activations/layer14_attention_weight_max": 49.826385498046875,
      "activations/layer14_attention_weight_min": -29.08978271484375,
      "activations/layer15_attention_weight_max": 46.1982421875,
      "activations/layer15_attention_weight_min": -29.734914779663086,
      "activations/layer16_attention_weight_max": 35.76313781738281,
      "activations/layer16_attention_weight_min": -30.76347541809082,
      "activations/layer17_attention_weight_max": 57.1658821105957,
      "activations/layer17_attention_weight_min": -43.83018112182617,
      "activations/layer18_attention_weight_max": 50.20111083984375,
      "activations/layer18_attention_weight_min": -38.68867874145508,
      "activations/layer19_attention_weight_max": 27.348377227783203,
      "activations/layer19_attention_weight_min": -23.290760040283203,
      "activations/layer1_attention_weight_max": 16.0978946685791,
      "activations/layer1_attention_weight_min": -14.915022850036621,
      "activations/layer20_attention_weight_max": 24.76058006286621,
      "activations/layer20_attention_weight_min": -23.297971725463867,
      "activations/layer21_attention_weight_max": 37.77980422973633,
      "activations/layer21_attention_weight_min": -24.20621109008789,
      "activations/layer22_attention_weight_max": 32.70417404174805,
      "activations/layer22_attention_weight_min": -27.344406127929688,
      "activations/layer23_attention_weight_max": 39.90803909301758,
      "activations/layer23_attention_weight_min": -23.491544723510742,
      "activations/layer2_attention_weight_max": 32.546024322509766,
      "activations/layer2_attention_weight_min": -32.539737701416016,
      "activations/layer3_attention_weight_max": 95.74333190917969,
      "activations/layer3_attention_weight_min": -97.66935729980469,
      "activations/layer4_attention_weight_max": 94.09384155273438,
      "activations/layer4_attention_weight_min": -91.61461639404297,
      "activations/layer5_attention_weight_max": 74.6945571899414,
      "activations/layer5_attention_weight_min": -78.86827850341797,
      "activations/layer6_attention_weight_max": 52.961997985839844,
      "activations/layer6_attention_weight_min": -52.66090774536133,
      "activations/layer7_attention_weight_max": 67.17753601074219,
      "activations/layer7_attention_weight_min": -60.25201416015625,
      "activations/layer8_attention_weight_max": 45.829898834228516,
      "activations/layer8_attention_weight_min": -47.89627456665039,
      "activations/layer9_attention_weight_max": 45.20256423950195,
      "activations/layer9_attention_weight_min": -46.406890869140625,
      "epoch": 13.86,
      "learning_rate": 6.125e-05,
      "loss": 2.763,
      "step": 238500
    },
    {
      "activations/layer0_attention_weight_max": 18.51628875732422,
      "activations/layer0_attention_weight_min": -14.425054550170898,
      "activations/layer10_attention_weight_max": 37.881744384765625,
      "activations/layer10_attention_weight_min": -38.424583435058594,
      "activations/layer11_attention_weight_max": 31.0775203704834,
      "activations/layer11_attention_weight_min": -34.27760314941406,
      "activations/layer12_attention_weight_max": 18.32185935974121,
      "activations/layer12_attention_weight_min": -27.084077835083008,
      "activations/layer13_attention_weight_max": 39.638587951660156,
      "activations/layer13_attention_weight_min": -31.229036331176758,
      "activations/layer14_attention_weight_max": 35.29418182373047,
      "activations/layer14_attention_weight_min": -30.27638053894043,
      "activations/layer15_attention_weight_max": 37.071434020996094,
      "activations/layer15_attention_weight_min": -29.67877197265625,
      "activations/layer16_attention_weight_max": 32.21438980102539,
      "activations/layer16_attention_weight_min": -31.466503143310547,
      "activations/layer17_attention_weight_max": 55.112911224365234,
      "activations/layer17_attention_weight_min": -42.35293197631836,
      "activations/layer18_attention_weight_max": 46.395694732666016,
      "activations/layer18_attention_weight_min": -38.67306137084961,
      "activations/layer19_attention_weight_max": 23.141576766967773,
      "activations/layer19_attention_weight_min": -21.065893173217773,
      "activations/layer1_attention_weight_max": 16.29076385498047,
      "activations/layer1_attention_weight_min": -16.725257873535156,
      "activations/layer20_attention_weight_max": 21.951087951660156,
      "activations/layer20_attention_weight_min": -21.363487243652344,
      "activations/layer21_attention_weight_max": 45.00592041015625,
      "activations/layer21_attention_weight_min": -26.17685890197754,
      "activations/layer22_attention_weight_max": 31.518064498901367,
      "activations/layer22_attention_weight_min": -24.65218162536621,
      "activations/layer23_attention_weight_max": 38.279151916503906,
      "activations/layer23_attention_weight_min": -26.82185935974121,
      "activations/layer2_attention_weight_max": 32.19479751586914,
      "activations/layer2_attention_weight_min": -31.883800506591797,
      "activations/layer3_attention_weight_max": 93.72795104980469,
      "activations/layer3_attention_weight_min": -96.78103637695312,
      "activations/layer4_attention_weight_max": 94.23873901367188,
      "activations/layer4_attention_weight_min": -89.17066955566406,
      "activations/layer5_attention_weight_max": 72.58402252197266,
      "activations/layer5_attention_weight_min": -79.78937530517578,
      "activations/layer6_attention_weight_max": 53.099693298339844,
      "activations/layer6_attention_weight_min": -57.45512771606445,
      "activations/layer7_attention_weight_max": 67.91142272949219,
      "activations/layer7_attention_weight_min": -67.51634216308594,
      "activations/layer8_attention_weight_max": 44.60344696044922,
      "activations/layer8_attention_weight_min": -48.37368392944336,
      "activations/layer9_attention_weight_max": 46.829978942871094,
      "activations/layer9_attention_weight_min": -44.90144348144531,
      "epoch": 13.86,
      "learning_rate": 6.12310606060606e-05,
      "loss": 2.7576,
      "step": 238550
    },
    {
      "activations/layer0_attention_weight_max": 15.96325397491455,
      "activations/layer0_attention_weight_min": -14.261323928833008,
      "activations/layer10_attention_weight_max": 40.71757888793945,
      "activations/layer10_attention_weight_min": -37.84486770629883,
      "activations/layer11_attention_weight_max": 37.16059875488281,
      "activations/layer11_attention_weight_min": -34.61740493774414,
      "activations/layer12_attention_weight_max": 18.253280639648438,
      "activations/layer12_attention_weight_min": -23.99301528930664,
      "activations/layer13_attention_weight_max": 36.52015686035156,
      "activations/layer13_attention_weight_min": -29.50320816040039,
      "activations/layer14_attention_weight_max": 37.705718994140625,
      "activations/layer14_attention_weight_min": -28.961261749267578,
      "activations/layer15_attention_weight_max": 33.485450744628906,
      "activations/layer15_attention_weight_min": -28.13108253479004,
      "activations/layer16_attention_weight_max": 31.873071670532227,
      "activations/layer16_attention_weight_min": -30.248470306396484,
      "activations/layer17_attention_weight_max": 51.96650695800781,
      "activations/layer17_attention_weight_min": -46.272300720214844,
      "activations/layer18_attention_weight_max": 48.002044677734375,
      "activations/layer18_attention_weight_min": -39.03058624267578,
      "activations/layer19_attention_weight_max": 26.662250518798828,
      "activations/layer19_attention_weight_min": -20.87598419189453,
      "activations/layer1_attention_weight_max": 15.917204856872559,
      "activations/layer1_attention_weight_min": -14.444872856140137,
      "activations/layer20_attention_weight_max": 25.05874252319336,
      "activations/layer20_attention_weight_min": -20.186912536621094,
      "activations/layer21_attention_weight_max": 41.50374221801758,
      "activations/layer21_attention_weight_min": -21.0740966796875,
      "activations/layer22_attention_weight_max": 32.82844543457031,
      "activations/layer22_attention_weight_min": -25.706052780151367,
      "activations/layer23_attention_weight_max": 39.42803955078125,
      "activations/layer23_attention_weight_min": -24.02748680114746,
      "activations/layer2_attention_weight_max": 33.414615631103516,
      "activations/layer2_attention_weight_min": -31.627052307128906,
      "activations/layer3_attention_weight_max": 95.7009506225586,
      "activations/layer3_attention_weight_min": -96.65818786621094,
      "activations/layer4_attention_weight_max": 98.75594329833984,
      "activations/layer4_attention_weight_min": -93.63784790039062,
      "activations/layer5_attention_weight_max": 72.6962661743164,
      "activations/layer5_attention_weight_min": -76.93035125732422,
      "activations/layer6_attention_weight_max": 55.8410758972168,
      "activations/layer6_attention_weight_min": -54.404293060302734,
      "activations/layer7_attention_weight_max": 75.63614654541016,
      "activations/layer7_attention_weight_min": -72.31688690185547,
      "activations/layer8_attention_weight_max": 54.23505401611328,
      "activations/layer8_attention_weight_min": -52.9580078125,
      "activations/layer9_attention_weight_max": 51.799224853515625,
      "activations/layer9_attention_weight_min": -46.220699310302734,
      "epoch": 13.86,
      "learning_rate": 6.121212121212121e-05,
      "loss": 2.7727,
      "step": 238600
    },
    {
      "activations/layer0_attention_weight_max": 15.87183952331543,
      "activations/layer0_attention_weight_min": -14.249852180480957,
      "activations/layer10_attention_weight_max": 37.44171905517578,
      "activations/layer10_attention_weight_min": -37.114776611328125,
      "activations/layer11_attention_weight_max": 33.746578216552734,
      "activations/layer11_attention_weight_min": -34.581764221191406,
      "activations/layer12_attention_weight_max": 18.05704116821289,
      "activations/layer12_attention_weight_min": -24.218467712402344,
      "activations/layer13_attention_weight_max": 34.76954650878906,
      "activations/layer13_attention_weight_min": -30.171123504638672,
      "activations/layer14_attention_weight_max": 50.1098747253418,
      "activations/layer14_attention_weight_min": -32.07894515991211,
      "activations/layer15_attention_weight_max": 43.104774475097656,
      "activations/layer15_attention_weight_min": -29.41373634338379,
      "activations/layer16_attention_weight_max": 31.568708419799805,
      "activations/layer16_attention_weight_min": -29.078264236450195,
      "activations/layer17_attention_weight_max": 50.4676399230957,
      "activations/layer17_attention_weight_min": -43.140262603759766,
      "activations/layer18_attention_weight_max": 51.87263107299805,
      "activations/layer18_attention_weight_min": -39.71958541870117,
      "activations/layer19_attention_weight_max": 22.53801918029785,
      "activations/layer19_attention_weight_min": -21.513792037963867,
      "activations/layer1_attention_weight_max": 16.48440170288086,
      "activations/layer1_attention_weight_min": -15.11678695678711,
      "activations/layer20_attention_weight_max": 23.369853973388672,
      "activations/layer20_attention_weight_min": -25.607738494873047,
      "activations/layer21_attention_weight_max": 40.840553283691406,
      "activations/layer21_attention_weight_min": -24.820714950561523,
      "activations/layer22_attention_weight_max": 30.49933624267578,
      "activations/layer22_attention_weight_min": -25.807567596435547,
      "activations/layer23_attention_weight_max": 37.77227020263672,
      "activations/layer23_attention_weight_min": -23.71401023864746,
      "activations/layer2_attention_weight_max": 33.00386047363281,
      "activations/layer2_attention_weight_min": -31.260883331298828,
      "activations/layer3_attention_weight_max": 98.30607604980469,
      "activations/layer3_attention_weight_min": -96.38951110839844,
      "activations/layer4_attention_weight_max": 95.3980484008789,
      "activations/layer4_attention_weight_min": -90.63374328613281,
      "activations/layer5_attention_weight_max": 75.50444793701172,
      "activations/layer5_attention_weight_min": -83.41590881347656,
      "activations/layer6_attention_weight_max": 55.11650848388672,
      "activations/layer6_attention_weight_min": -52.186588287353516,
      "activations/layer7_attention_weight_max": 70.95075225830078,
      "activations/layer7_attention_weight_min": -67.89090728759766,
      "activations/layer8_attention_weight_max": 48.06744384765625,
      "activations/layer8_attention_weight_min": -49.505699157714844,
      "activations/layer9_attention_weight_max": 51.717830657958984,
      "activations/layer9_attention_weight_min": -48.80060958862305,
      "epoch": 13.87,
      "learning_rate": 6.119318181818182e-05,
      "loss": 2.7657,
      "step": 238650
    },
    {
      "activations/layer0_attention_weight_max": 15.747563362121582,
      "activations/layer0_attention_weight_min": -13.986479759216309,
      "activations/layer10_attention_weight_max": 34.388710021972656,
      "activations/layer10_attention_weight_min": -34.59644317626953,
      "activations/layer11_attention_weight_max": 31.55487632751465,
      "activations/layer11_attention_weight_min": -31.678382873535156,
      "activations/layer12_attention_weight_max": 16.723255157470703,
      "activations/layer12_attention_weight_min": -24.075849533081055,
      "activations/layer13_attention_weight_max": 33.185035705566406,
      "activations/layer13_attention_weight_min": -27.49283790588379,
      "activations/layer14_attention_weight_max": 35.03864288330078,
      "activations/layer14_attention_weight_min": -29.37188148498535,
      "activations/layer15_attention_weight_max": 30.140338897705078,
      "activations/layer15_attention_weight_min": -29.350494384765625,
      "activations/layer16_attention_weight_max": 30.299148559570312,
      "activations/layer16_attention_weight_min": -29.509830474853516,
      "activations/layer17_attention_weight_max": 48.94948196411133,
      "activations/layer17_attention_weight_min": -41.55485916137695,
      "activations/layer18_attention_weight_max": 46.279624938964844,
      "activations/layer18_attention_weight_min": -37.180179595947266,
      "activations/layer19_attention_weight_max": 23.89882469177246,
      "activations/layer19_attention_weight_min": -21.896406173706055,
      "activations/layer1_attention_weight_max": 16.166122436523438,
      "activations/layer1_attention_weight_min": -13.547849655151367,
      "activations/layer20_attention_weight_max": 21.528671264648438,
      "activations/layer20_attention_weight_min": -25.30217742919922,
      "activations/layer21_attention_weight_max": 31.351760864257812,
      "activations/layer21_attention_weight_min": -23.057235717773438,
      "activations/layer22_attention_weight_max": 30.4697265625,
      "activations/layer22_attention_weight_min": -26.51299285888672,
      "activations/layer23_attention_weight_max": 36.882938385009766,
      "activations/layer23_attention_weight_min": -23.03218650817871,
      "activations/layer2_attention_weight_max": 32.34258270263672,
      "activations/layer2_attention_weight_min": -30.916616439819336,
      "activations/layer3_attention_weight_max": 91.31558227539062,
      "activations/layer3_attention_weight_min": -92.15428161621094,
      "activations/layer4_attention_weight_max": 92.13115692138672,
      "activations/layer4_attention_weight_min": -90.32078552246094,
      "activations/layer5_attention_weight_max": 71.10260772705078,
      "activations/layer5_attention_weight_min": -79.96847534179688,
      "activations/layer6_attention_weight_max": 53.18558120727539,
      "activations/layer6_attention_weight_min": -50.54492950439453,
      "activations/layer7_attention_weight_max": 66.11460876464844,
      "activations/layer7_attention_weight_min": -64.92352294921875,
      "activations/layer8_attention_weight_max": 49.0191535949707,
      "activations/layer8_attention_weight_min": -47.79222106933594,
      "activations/layer9_attention_weight_max": 47.788177490234375,
      "activations/layer9_attention_weight_min": -43.965057373046875,
      "epoch": 13.87,
      "learning_rate": 6.117424242424242e-05,
      "loss": 2.7683,
      "step": 238700
    },
    {
      "activations/layer0_attention_weight_max": 17.505477905273438,
      "activations/layer0_attention_weight_min": -13.951866149902344,
      "activations/layer10_attention_weight_max": 40.318092346191406,
      "activations/layer10_attention_weight_min": -40.58962631225586,
      "activations/layer11_attention_weight_max": 38.120628356933594,
      "activations/layer11_attention_weight_min": -36.55510330200195,
      "activations/layer12_attention_weight_max": 19.50615119934082,
      "activations/layer12_attention_weight_min": -28.15184783935547,
      "activations/layer13_attention_weight_max": 34.976776123046875,
      "activations/layer13_attention_weight_min": -27.749191284179688,
      "activations/layer14_attention_weight_max": 38.00300979614258,
      "activations/layer14_attention_weight_min": -29.45819664001465,
      "activations/layer15_attention_weight_max": 33.989925384521484,
      "activations/layer15_attention_weight_min": -29.504884719848633,
      "activations/layer16_attention_weight_max": 32.9293098449707,
      "activations/layer16_attention_weight_min": -30.286880493164062,
      "activations/layer17_attention_weight_max": 53.40537643432617,
      "activations/layer17_attention_weight_min": -42.19761276245117,
      "activations/layer18_attention_weight_max": 46.54706954956055,
      "activations/layer18_attention_weight_min": -37.12667465209961,
      "activations/layer19_attention_weight_max": 21.734264373779297,
      "activations/layer19_attention_weight_min": -21.750410079956055,
      "activations/layer1_attention_weight_max": 17.074668884277344,
      "activations/layer1_attention_weight_min": -16.65452003479004,
      "activations/layer20_attention_weight_max": 21.104223251342773,
      "activations/layer20_attention_weight_min": -21.219125747680664,
      "activations/layer21_attention_weight_max": 39.24524688720703,
      "activations/layer21_attention_weight_min": -23.635257720947266,
      "activations/layer22_attention_weight_max": 29.574188232421875,
      "activations/layer22_attention_weight_min": -26.158329010009766,
      "activations/layer23_attention_weight_max": 38.2584342956543,
      "activations/layer23_attention_weight_min": -22.406715393066406,
      "activations/layer2_attention_weight_max": 34.92974853515625,
      "activations/layer2_attention_weight_min": -32.789955139160156,
      "activations/layer3_attention_weight_max": 104.70891571044922,
      "activations/layer3_attention_weight_min": -103.22457885742188,
      "activations/layer4_attention_weight_max": 100.49239349365234,
      "activations/layer4_attention_weight_min": -100.26349639892578,
      "activations/layer5_attention_weight_max": 73.47831726074219,
      "activations/layer5_attention_weight_min": -79.9074478149414,
      "activations/layer6_attention_weight_max": 54.81853485107422,
      "activations/layer6_attention_weight_min": -54.97949981689453,
      "activations/layer7_attention_weight_max": 74.3704605102539,
      "activations/layer7_attention_weight_min": -70.70454406738281,
      "activations/layer8_attention_weight_max": 51.12137985229492,
      "activations/layer8_attention_weight_min": -50.903770446777344,
      "activations/layer9_attention_weight_max": 60.192901611328125,
      "activations/layer9_attention_weight_min": -48.365447998046875,
      "epoch": 13.87,
      "learning_rate": 6.115530303030302e-05,
      "loss": 2.7504,
      "step": 238750
    },
    {
      "activations/layer0_attention_weight_max": 15.831342697143555,
      "activations/layer0_attention_weight_min": -14.005197525024414,
      "activations/layer10_attention_weight_max": 35.43897247314453,
      "activations/layer10_attention_weight_min": -34.62420654296875,
      "activations/layer11_attention_weight_max": 31.537334442138672,
      "activations/layer11_attention_weight_min": -31.46272087097168,
      "activations/layer12_attention_weight_max": 18.882558822631836,
      "activations/layer12_attention_weight_min": -22.16049575805664,
      "activations/layer13_attention_weight_max": 40.28028106689453,
      "activations/layer13_attention_weight_min": -29.401752471923828,
      "activations/layer14_attention_weight_max": 37.1927375793457,
      "activations/layer14_attention_weight_min": -29.322084426879883,
      "activations/layer15_attention_weight_max": 33.70736312866211,
      "activations/layer15_attention_weight_min": -28.536476135253906,
      "activations/layer16_attention_weight_max": 31.59747314453125,
      "activations/layer16_attention_weight_min": -29.862112045288086,
      "activations/layer17_attention_weight_max": 51.81142044067383,
      "activations/layer17_attention_weight_min": -45.14995574951172,
      "activations/layer18_attention_weight_max": 48.17131805419922,
      "activations/layer18_attention_weight_min": -41.96520233154297,
      "activations/layer19_attention_weight_max": 23.059659957885742,
      "activations/layer19_attention_weight_min": -23.736835479736328,
      "activations/layer1_attention_weight_max": 16.62311553955078,
      "activations/layer1_attention_weight_min": -14.18526840209961,
      "activations/layer20_attention_weight_max": 24.028160095214844,
      "activations/layer20_attention_weight_min": -23.440845489501953,
      "activations/layer21_attention_weight_max": 44.02354049682617,
      "activations/layer21_attention_weight_min": -25.952341079711914,
      "activations/layer22_attention_weight_max": 29.911054611206055,
      "activations/layer22_attention_weight_min": -28.1121826171875,
      "activations/layer23_attention_weight_max": 41.086997985839844,
      "activations/layer23_attention_weight_min": -26.0496826171875,
      "activations/layer2_attention_weight_max": 30.805221557617188,
      "activations/layer2_attention_weight_min": -30.77802085876465,
      "activations/layer3_attention_weight_max": 89.84463500976562,
      "activations/layer3_attention_weight_min": -93.63037109375,
      "activations/layer4_attention_weight_max": 90.17540740966797,
      "activations/layer4_attention_weight_min": -88.83403778076172,
      "activations/layer5_attention_weight_max": 69.71115112304688,
      "activations/layer5_attention_weight_min": -78.14369201660156,
      "activations/layer6_attention_weight_max": 50.98483657836914,
      "activations/layer6_attention_weight_min": -52.59840774536133,
      "activations/layer7_attention_weight_max": 64.89596557617188,
      "activations/layer7_attention_weight_min": -67.44437408447266,
      "activations/layer8_attention_weight_max": 42.375675201416016,
      "activations/layer8_attention_weight_min": -47.69008255004883,
      "activations/layer9_attention_weight_max": 43.79092788696289,
      "activations/layer9_attention_weight_min": -48.40978240966797,
      "epoch": 13.88,
      "learning_rate": 6.113636363636363e-05,
      "loss": 2.7765,
      "step": 238800
    },
    {
      "activations/layer0_attention_weight_max": 15.58611011505127,
      "activations/layer0_attention_weight_min": -13.861852645874023,
      "activations/layer10_attention_weight_max": 38.17951202392578,
      "activations/layer10_attention_weight_min": -36.22260284423828,
      "activations/layer11_attention_weight_max": 33.40130615234375,
      "activations/layer11_attention_weight_min": -32.203426361083984,
      "activations/layer12_attention_weight_max": 19.341400146484375,
      "activations/layer12_attention_weight_min": -27.37966537475586,
      "activations/layer13_attention_weight_max": 44.185874938964844,
      "activations/layer13_attention_weight_min": -34.22705841064453,
      "activations/layer14_attention_weight_max": 40.49843215942383,
      "activations/layer14_attention_weight_min": -30.020313262939453,
      "activations/layer15_attention_weight_max": 34.07051086425781,
      "activations/layer15_attention_weight_min": -29.67748260498047,
      "activations/layer16_attention_weight_max": 32.08066940307617,
      "activations/layer16_attention_weight_min": -29.104490280151367,
      "activations/layer17_attention_weight_max": 52.10935974121094,
      "activations/layer17_attention_weight_min": -43.666542053222656,
      "activations/layer18_attention_weight_max": 47.61022186279297,
      "activations/layer18_attention_weight_min": -38.47133255004883,
      "activations/layer19_attention_weight_max": 21.982120513916016,
      "activations/layer19_attention_weight_min": -23.958646774291992,
      "activations/layer1_attention_weight_max": 16.135557174682617,
      "activations/layer1_attention_weight_min": -17.592975616455078,
      "activations/layer20_attention_weight_max": 22.950727462768555,
      "activations/layer20_attention_weight_min": -22.67548179626465,
      "activations/layer21_attention_weight_max": 43.79860305786133,
      "activations/layer21_attention_weight_min": -29.60354995727539,
      "activations/layer22_attention_weight_max": 30.10844612121582,
      "activations/layer22_attention_weight_min": -25.343286514282227,
      "activations/layer23_attention_weight_max": 39.4002571105957,
      "activations/layer23_attention_weight_min": -23.616870880126953,
      "activations/layer2_attention_weight_max": 35.72840118408203,
      "activations/layer2_attention_weight_min": -35.141387939453125,
      "activations/layer3_attention_weight_max": 99.89124298095703,
      "activations/layer3_attention_weight_min": -102.0474624633789,
      "activations/layer4_attention_weight_max": 99.2363510131836,
      "activations/layer4_attention_weight_min": -97.34860229492188,
      "activations/layer5_attention_weight_max": 72.99789428710938,
      "activations/layer5_attention_weight_min": -80.5323715209961,
      "activations/layer6_attention_weight_max": 55.29438400268555,
      "activations/layer6_attention_weight_min": -56.45561599731445,
      "activations/layer7_attention_weight_max": 67.96340942382812,
      "activations/layer7_attention_weight_min": -66.6188735961914,
      "activations/layer8_attention_weight_max": 48.54115295410156,
      "activations/layer8_attention_weight_min": -51.7219123840332,
      "activations/layer9_attention_weight_max": 46.93911361694336,
      "activations/layer9_attention_weight_min": -46.94902801513672,
      "epoch": 13.88,
      "learning_rate": 6.111742424242424e-05,
      "loss": 2.7459,
      "step": 238850
    },
    {
      "activations/layer0_attention_weight_max": 15.621681213378906,
      "activations/layer0_attention_weight_min": -14.192994117736816,
      "activations/layer10_attention_weight_max": 37.821434020996094,
      "activations/layer10_attention_weight_min": -37.0141487121582,
      "activations/layer11_attention_weight_max": 35.25862503051758,
      "activations/layer11_attention_weight_min": -35.085941314697266,
      "activations/layer12_attention_weight_max": 19.7699031829834,
      "activations/layer12_attention_weight_min": -27.612770080566406,
      "activations/layer13_attention_weight_max": 51.51388931274414,
      "activations/layer13_attention_weight_min": -31.02341079711914,
      "activations/layer14_attention_weight_max": 59.713863372802734,
      "activations/layer14_attention_weight_min": -32.667564392089844,
      "activations/layer15_attention_weight_max": 47.901973724365234,
      "activations/layer15_attention_weight_min": -31.993444442749023,
      "activations/layer16_attention_weight_max": 37.660945892333984,
      "activations/layer16_attention_weight_min": -34.07004928588867,
      "activations/layer17_attention_weight_max": 63.13884735107422,
      "activations/layer17_attention_weight_min": -51.71131134033203,
      "activations/layer18_attention_weight_max": 57.854068756103516,
      "activations/layer18_attention_weight_min": -46.5009765625,
      "activations/layer19_attention_weight_max": 25.064428329467773,
      "activations/layer19_attention_weight_min": -26.46683692932129,
      "activations/layer1_attention_weight_max": 16.275432586669922,
      "activations/layer1_attention_weight_min": -14.377277374267578,
      "activations/layer20_attention_weight_max": 26.88469886779785,
      "activations/layer20_attention_weight_min": -23.28331756591797,
      "activations/layer21_attention_weight_max": 43.40068054199219,
      "activations/layer21_attention_weight_min": -22.81922721862793,
      "activations/layer22_attention_weight_max": 36.42179489135742,
      "activations/layer22_attention_weight_min": -27.08754539489746,
      "activations/layer23_attention_weight_max": 38.98824691772461,
      "activations/layer23_attention_weight_min": -25.943031311035156,
      "activations/layer2_attention_weight_max": 35.48969268798828,
      "activations/layer2_attention_weight_min": -31.98019027709961,
      "activations/layer3_attention_weight_max": 97.6764144897461,
      "activations/layer3_attention_weight_min": -94.8985824584961,
      "activations/layer4_attention_weight_max": 99.53422546386719,
      "activations/layer4_attention_weight_min": -94.23289489746094,
      "activations/layer5_attention_weight_max": 74.63619232177734,
      "activations/layer5_attention_weight_min": -82.10847473144531,
      "activations/layer6_attention_weight_max": 55.005409240722656,
      "activations/layer6_attention_weight_min": -53.52490997314453,
      "activations/layer7_attention_weight_max": 69.02864074707031,
      "activations/layer7_attention_weight_min": -66.78551483154297,
      "activations/layer8_attention_weight_max": 48.67825698852539,
      "activations/layer8_attention_weight_min": -48.8674430847168,
      "activations/layer9_attention_weight_max": 51.49929428100586,
      "activations/layer9_attention_weight_min": -51.91443634033203,
      "epoch": 13.88,
      "learning_rate": 6.109848484848484e-05,
      "loss": 2.7655,
      "step": 238900
    },
    {
      "activations/layer0_attention_weight_max": 16.165817260742188,
      "activations/layer0_attention_weight_min": -14.14608383178711,
      "activations/layer10_attention_weight_max": 35.38105010986328,
      "activations/layer10_attention_weight_min": -34.8756103515625,
      "activations/layer11_attention_weight_max": 30.710866928100586,
      "activations/layer11_attention_weight_min": -33.3687744140625,
      "activations/layer12_attention_weight_max": 16.85687255859375,
      "activations/layer12_attention_weight_min": -27.24342155456543,
      "activations/layer13_attention_weight_max": 31.2499942779541,
      "activations/layer13_attention_weight_min": -27.498340606689453,
      "activations/layer14_attention_weight_max": 31.80135726928711,
      "activations/layer14_attention_weight_min": -30.930545806884766,
      "activations/layer15_attention_weight_max": 29.487581253051758,
      "activations/layer15_attention_weight_min": -30.5885009765625,
      "activations/layer16_attention_weight_max": 29.519014358520508,
      "activations/layer16_attention_weight_min": -30.45184326171875,
      "activations/layer17_attention_weight_max": 47.79869842529297,
      "activations/layer17_attention_weight_min": -44.31586837768555,
      "activations/layer18_attention_weight_max": 45.84623336791992,
      "activations/layer18_attention_weight_min": -38.672000885009766,
      "activations/layer19_attention_weight_max": 25.657575607299805,
      "activations/layer19_attention_weight_min": -23.23204803466797,
      "activations/layer1_attention_weight_max": 16.772207260131836,
      "activations/layer1_attention_weight_min": -14.998550415039062,
      "activations/layer20_attention_weight_max": 19.993310928344727,
      "activations/layer20_attention_weight_min": -23.454254150390625,
      "activations/layer21_attention_weight_max": 37.84879684448242,
      "activations/layer21_attention_weight_min": -24.09227752685547,
      "activations/layer22_attention_weight_max": 29.145437240600586,
      "activations/layer22_attention_weight_min": -25.42417335510254,
      "activations/layer23_attention_weight_max": 34.77178955078125,
      "activations/layer23_attention_weight_min": -25.94208526611328,
      "activations/layer2_attention_weight_max": 31.93216323852539,
      "activations/layer2_attention_weight_min": -31.84735679626465,
      "activations/layer3_attention_weight_max": 93.90281677246094,
      "activations/layer3_attention_weight_min": -98.8270034790039,
      "activations/layer4_attention_weight_max": 96.89170837402344,
      "activations/layer4_attention_weight_min": -92.32823181152344,
      "activations/layer5_attention_weight_max": 76.08975219726562,
      "activations/layer5_attention_weight_min": -82.86532592773438,
      "activations/layer6_attention_weight_max": 53.067501068115234,
      "activations/layer6_attention_weight_min": -54.19293975830078,
      "activations/layer7_attention_weight_max": 69.59553527832031,
      "activations/layer7_attention_weight_min": -67.51686096191406,
      "activations/layer8_attention_weight_max": 45.77273941040039,
      "activations/layer8_attention_weight_min": -49.01835250854492,
      "activations/layer9_attention_weight_max": 43.68650436401367,
      "activations/layer9_attention_weight_min": -47.83456039428711,
      "epoch": 13.88,
      "learning_rate": 6.107954545454545e-05,
      "loss": 2.7611,
      "step": 238950
    },
    {
      "activations/layer0_attention_weight_max": 15.840126991271973,
      "activations/layer0_attention_weight_min": -14.141325950622559,
      "activations/layer10_attention_weight_max": 39.20832061767578,
      "activations/layer10_attention_weight_min": -36.84469985961914,
      "activations/layer11_attention_weight_max": 33.256446838378906,
      "activations/layer11_attention_weight_min": -33.69417953491211,
      "activations/layer12_attention_weight_max": 19.1787166595459,
      "activations/layer12_attention_weight_min": -30.023527145385742,
      "activations/layer13_attention_weight_max": 38.176246643066406,
      "activations/layer13_attention_weight_min": -28.53821563720703,
      "activations/layer14_attention_weight_max": 42.2901496887207,
      "activations/layer14_attention_weight_min": -29.828725814819336,
      "activations/layer15_attention_weight_max": 37.58141326904297,
      "activations/layer15_attention_weight_min": -30.57645034790039,
      "activations/layer16_attention_weight_max": 36.407264709472656,
      "activations/layer16_attention_weight_min": -32.05453872680664,
      "activations/layer17_attention_weight_max": 59.97027587890625,
      "activations/layer17_attention_weight_min": -46.881935119628906,
      "activations/layer18_attention_weight_max": 55.13638687133789,
      "activations/layer18_attention_weight_min": -42.50959014892578,
      "activations/layer19_attention_weight_max": 28.0312557220459,
      "activations/layer19_attention_weight_min": -23.978193283081055,
      "activations/layer1_attention_weight_max": 15.79832649230957,
      "activations/layer1_attention_weight_min": -16.180191040039062,
      "activations/layer20_attention_weight_max": 27.111480712890625,
      "activations/layer20_attention_weight_min": -22.512887954711914,
      "activations/layer21_attention_weight_max": 49.78239440917969,
      "activations/layer21_attention_weight_min": -22.64772605895996,
      "activations/layer22_attention_weight_max": 33.94721603393555,
      "activations/layer22_attention_weight_min": -24.46311378479004,
      "activations/layer23_attention_weight_max": 44.61476135253906,
      "activations/layer23_attention_weight_min": -24.252500534057617,
      "activations/layer2_attention_weight_max": 33.88788986206055,
      "activations/layer2_attention_weight_min": -30.82435417175293,
      "activations/layer3_attention_weight_max": 99.4111099243164,
      "activations/layer3_attention_weight_min": -100.92123413085938,
      "activations/layer4_attention_weight_max": 96.61009979248047,
      "activations/layer4_attention_weight_min": -94.04367065429688,
      "activations/layer5_attention_weight_max": 71.71556091308594,
      "activations/layer5_attention_weight_min": -75.29818725585938,
      "activations/layer6_attention_weight_max": 51.218482971191406,
      "activations/layer6_attention_weight_min": -52.11882781982422,
      "activations/layer7_attention_weight_max": 66.65523529052734,
      "activations/layer7_attention_weight_min": -64.9277114868164,
      "activations/layer8_attention_weight_max": 46.28108596801758,
      "activations/layer8_attention_weight_min": -49.499534606933594,
      "activations/layer9_attention_weight_max": 46.16721725463867,
      "activations/layer9_attention_weight_min": -47.24479293823242,
      "epoch": 13.89,
      "learning_rate": 6.106060606060605e-05,
      "loss": 2.7594,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.5258,
      "eval_samples_per_second": 503.647,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.5258,
      "eval_openwebtext_samples_per_second": 503.647,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9917,
      "eval_wikitext_samples_per_second": 228.946,
      "step": 239000
    },
    {
      "epoch": 13.89,
      "eval_lambada_loss": 2.431640625,
      "eval_lambada_ppl": 11.3775330413472,
      "eval_lambada_runtime": 9.5612,
      "eval_lambada_samples_per_second": 509.244,
      "step": 239000
    },
    {
      "activations/layer0_attention_weight_max": 15.772626876831055,
      "activations/layer0_attention_weight_min": -14.164874076843262,
      "activations/layer10_attention_weight_max": 34.6900520324707,
      "activations/layer10_attention_weight_min": -34.488189697265625,
      "activations/layer11_attention_weight_max": 32.36994934082031,
      "activations/layer11_attention_weight_min": -32.09793472290039,
      "activations/layer12_attention_weight_max": 18.420047760009766,
      "activations/layer12_attention_weight_min": -25.80992889404297,
      "activations/layer13_attention_weight_max": 43.141353607177734,
      "activations/layer13_attention_weight_min": -30.656280517578125,
      "activations/layer14_attention_weight_max": 37.071407318115234,
      "activations/layer14_attention_weight_min": -31.224082946777344,
      "activations/layer15_attention_weight_max": 36.941349029541016,
      "activations/layer15_attention_weight_min": -30.592893600463867,
      "activations/layer16_attention_weight_max": 31.901554107666016,
      "activations/layer16_attention_weight_min": -32.07889175415039,
      "activations/layer17_attention_weight_max": 51.09337615966797,
      "activations/layer17_attention_weight_min": -46.262393951416016,
      "activations/layer18_attention_weight_max": 45.946205139160156,
      "activations/layer18_attention_weight_min": -37.89718246459961,
      "activations/layer19_attention_weight_max": 22.645658493041992,
      "activations/layer19_attention_weight_min": -20.11951446533203,
      "activations/layer1_attention_weight_max": 17.41691017150879,
      "activations/layer1_attention_weight_min": -14.05960750579834,
      "activations/layer20_attention_weight_max": 21.9135799407959,
      "activations/layer20_attention_weight_min": -20.46098518371582,
      "activations/layer21_attention_weight_max": 36.069183349609375,
      "activations/layer21_attention_weight_min": -21.791711807250977,
      "activations/layer22_attention_weight_max": 29.493669509887695,
      "activations/layer22_attention_weight_min": -26.12254524230957,
      "activations/layer23_attention_weight_max": 39.45717239379883,
      "activations/layer23_attention_weight_min": -24.030866622924805,
      "activations/layer2_attention_weight_max": 31.368276596069336,
      "activations/layer2_attention_weight_min": -30.75585174560547,
      "activations/layer3_attention_weight_max": 90.20591735839844,
      "activations/layer3_attention_weight_min": -91.4650650024414,
      "activations/layer4_attention_weight_max": 90.12152099609375,
      "activations/layer4_attention_weight_min": -91.04467010498047,
      "activations/layer5_attention_weight_max": 72.64669036865234,
      "activations/layer5_attention_weight_min": -80.93391418457031,
      "activations/layer6_attention_weight_max": 51.527305603027344,
      "activations/layer6_attention_weight_min": -51.55963897705078,
      "activations/layer7_attention_weight_max": 65.75525665283203,
      "activations/layer7_attention_weight_min": -67.16864013671875,
      "activations/layer8_attention_weight_max": 45.32215118408203,
      "activations/layer8_attention_weight_min": -49.01207733154297,
      "activations/layer9_attention_weight_max": 45.412471771240234,
      "activations/layer9_attention_weight_min": -48.327693939208984,
      "epoch": 13.89,
      "learning_rate": 6.104166666666666e-05,
      "loss": 2.756,
      "step": 239050
    },
    {
      "activations/layer0_attention_weight_max": 15.915081977844238,
      "activations/layer0_attention_weight_min": -14.066475868225098,
      "activations/layer10_attention_weight_max": 39.25202178955078,
      "activations/layer10_attention_weight_min": -36.48065185546875,
      "activations/layer11_attention_weight_max": 33.94666290283203,
      "activations/layer11_attention_weight_min": -34.719017028808594,
      "activations/layer12_attention_weight_max": 20.972034454345703,
      "activations/layer12_attention_weight_min": -23.16448402404785,
      "activations/layer13_attention_weight_max": 35.01405715942383,
      "activations/layer13_attention_weight_min": -29.01039695739746,
      "activations/layer14_attention_weight_max": 35.30411148071289,
      "activations/layer14_attention_weight_min": -30.08405113220215,
      "activations/layer15_attention_weight_max": 32.082698822021484,
      "activations/layer15_attention_weight_min": -31.80954933166504,
      "activations/layer16_attention_weight_max": 32.13042068481445,
      "activations/layer16_attention_weight_min": -32.4582405090332,
      "activations/layer17_attention_weight_max": 52.9233512878418,
      "activations/layer17_attention_weight_min": -45.38397979736328,
      "activations/layer18_attention_weight_max": 48.656105041503906,
      "activations/layer18_attention_weight_min": -41.055580139160156,
      "activations/layer19_attention_weight_max": 27.894271850585938,
      "activations/layer19_attention_weight_min": -24.386266708374023,
      "activations/layer1_attention_weight_max": 16.211774826049805,
      "activations/layer1_attention_weight_min": -16.28416633605957,
      "activations/layer20_attention_weight_max": 23.30561065673828,
      "activations/layer20_attention_weight_min": -23.218130111694336,
      "activations/layer21_attention_weight_max": 34.384239196777344,
      "activations/layer21_attention_weight_min": -21.242904663085938,
      "activations/layer22_attention_weight_max": 31.425989151000977,
      "activations/layer22_attention_weight_min": -25.65696907043457,
      "activations/layer23_attention_weight_max": 35.20319366455078,
      "activations/layer23_attention_weight_min": -25.054779052734375,
      "activations/layer2_attention_weight_max": 35.335479736328125,
      "activations/layer2_attention_weight_min": -30.896709442138672,
      "activations/layer3_attention_weight_max": 93.82190704345703,
      "activations/layer3_attention_weight_min": -93.94992065429688,
      "activations/layer4_attention_weight_max": 91.76060485839844,
      "activations/layer4_attention_weight_min": -90.26348114013672,
      "activations/layer5_attention_weight_max": 70.06836700439453,
      "activations/layer5_attention_weight_min": -79.028076171875,
      "activations/layer6_attention_weight_max": 50.41061019897461,
      "activations/layer6_attention_weight_min": -53.86247634887695,
      "activations/layer7_attention_weight_max": 64.85057067871094,
      "activations/layer7_attention_weight_min": -62.555320739746094,
      "activations/layer8_attention_weight_max": 43.561180114746094,
      "activations/layer8_attention_weight_min": -49.181007385253906,
      "activations/layer9_attention_weight_max": 43.101497650146484,
      "activations/layer9_attention_weight_min": -45.14177322387695,
      "epoch": 13.89,
      "learning_rate": 6.102272727272726e-05,
      "loss": 2.7798,
      "step": 239100
    },
    {
      "activations/layer0_attention_weight_max": 16.755313873291016,
      "activations/layer0_attention_weight_min": -14.123369216918945,
      "activations/layer10_attention_weight_max": 40.816986083984375,
      "activations/layer10_attention_weight_min": -40.43769836425781,
      "activations/layer11_attention_weight_max": 38.97856140136719,
      "activations/layer11_attention_weight_min": -35.03944778442383,
      "activations/layer12_attention_weight_max": 22.75189971923828,
      "activations/layer12_attention_weight_min": -23.556026458740234,
      "activations/layer13_attention_weight_max": 40.60912322998047,
      "activations/layer13_attention_weight_min": -30.644426345825195,
      "activations/layer14_attention_weight_max": 51.99472427368164,
      "activations/layer14_attention_weight_min": -30.404993057250977,
      "activations/layer15_attention_weight_max": 44.318702697753906,
      "activations/layer15_attention_weight_min": -30.060197830200195,
      "activations/layer16_attention_weight_max": 33.39335250854492,
      "activations/layer16_attention_weight_min": -30.03323745727539,
      "activations/layer17_attention_weight_max": 53.583229064941406,
      "activations/layer17_attention_weight_min": -48.212989807128906,
      "activations/layer18_attention_weight_max": 48.67524337768555,
      "activations/layer18_attention_weight_min": -38.559906005859375,
      "activations/layer19_attention_weight_max": 25.8797664642334,
      "activations/layer19_attention_weight_min": -20.400909423828125,
      "activations/layer1_attention_weight_max": 17.495145797729492,
      "activations/layer1_attention_weight_min": -14.944336891174316,
      "activations/layer20_attention_weight_max": 23.610748291015625,
      "activations/layer20_attention_weight_min": -19.97246551513672,
      "activations/layer21_attention_weight_max": 37.755516052246094,
      "activations/layer21_attention_weight_min": -21.278533935546875,
      "activations/layer22_attention_weight_max": 32.61406707763672,
      "activations/layer22_attention_weight_min": -25.09812355041504,
      "activations/layer23_attention_weight_max": 40.04230499267578,
      "activations/layer23_attention_weight_min": -25.02902603149414,
      "activations/layer2_attention_weight_max": 36.792633056640625,
      "activations/layer2_attention_weight_min": -33.40357971191406,
      "activations/layer3_attention_weight_max": 100.38270568847656,
      "activations/layer3_attention_weight_min": -99.90153503417969,
      "activations/layer4_attention_weight_max": 90.97450256347656,
      "activations/layer4_attention_weight_min": -89.2414779663086,
      "activations/layer5_attention_weight_max": 72.1329574584961,
      "activations/layer5_attention_weight_min": -82.70115661621094,
      "activations/layer6_attention_weight_max": 51.21420669555664,
      "activations/layer6_attention_weight_min": -54.628475189208984,
      "activations/layer7_attention_weight_max": 77.3069076538086,
      "activations/layer7_attention_weight_min": -65.64876556396484,
      "activations/layer8_attention_weight_max": 50.92815017700195,
      "activations/layer8_attention_weight_min": -50.70174026489258,
      "activations/layer9_attention_weight_max": 64.74766540527344,
      "activations/layer9_attention_weight_min": -46.08879852294922,
      "epoch": 13.9,
      "learning_rate": 6.100416666666667e-05,
      "loss": 2.7731,
      "step": 239150
    },
    {
      "activations/layer0_attention_weight_max": 17.017911911010742,
      "activations/layer0_attention_weight_min": -14.097784042358398,
      "activations/layer10_attention_weight_max": 37.67817687988281,
      "activations/layer10_attention_weight_min": -36.65594482421875,
      "activations/layer11_attention_weight_max": 34.063140869140625,
      "activations/layer11_attention_weight_min": -33.05425262451172,
      "activations/layer12_attention_weight_max": 21.859107971191406,
      "activations/layer12_attention_weight_min": -31.173641204833984,
      "activations/layer13_attention_weight_max": 43.97728729248047,
      "activations/layer13_attention_weight_min": -29.15498924255371,
      "activations/layer14_attention_weight_max": 37.537662506103516,
      "activations/layer14_attention_weight_min": -30.34259605407715,
      "activations/layer15_attention_weight_max": 35.53185272216797,
      "activations/layer15_attention_weight_min": -29.69727897644043,
      "activations/layer16_attention_weight_max": 32.953826904296875,
      "activations/layer16_attention_weight_min": -31.00775146484375,
      "activations/layer17_attention_weight_max": 53.916866302490234,
      "activations/layer17_attention_weight_min": -44.5785026550293,
      "activations/layer18_attention_weight_max": 50.3260383605957,
      "activations/layer18_attention_weight_min": -38.937320709228516,
      "activations/layer19_attention_weight_max": 26.545669555664062,
      "activations/layer19_attention_weight_min": -21.414894104003906,
      "activations/layer1_attention_weight_max": 17.128171920776367,
      "activations/layer1_attention_weight_min": -14.556276321411133,
      "activations/layer20_attention_weight_max": 23.49230194091797,
      "activations/layer20_attention_weight_min": -21.01953887939453,
      "activations/layer21_attention_weight_max": 39.986019134521484,
      "activations/layer21_attention_weight_min": -23.396697998046875,
      "activations/layer22_attention_weight_max": 32.74978256225586,
      "activations/layer22_attention_weight_min": -25.265670776367188,
      "activations/layer23_attention_weight_max": 40.87465286254883,
      "activations/layer23_attention_weight_min": -24.415935516357422,
      "activations/layer2_attention_weight_max": 33.41490936279297,
      "activations/layer2_attention_weight_min": -30.843650817871094,
      "activations/layer3_attention_weight_max": 96.05541229248047,
      "activations/layer3_attention_weight_min": -97.16272735595703,
      "activations/layer4_attention_weight_max": 97.09335327148438,
      "activations/layer4_attention_weight_min": -95.29190826416016,
      "activations/layer5_attention_weight_max": 70.70814514160156,
      "activations/layer5_attention_weight_min": -77.71763610839844,
      "activations/layer6_attention_weight_max": 51.60891342163086,
      "activations/layer6_attention_weight_min": -52.25630187988281,
      "activations/layer7_attention_weight_max": 63.88756561279297,
      "activations/layer7_attention_weight_min": -66.13311767578125,
      "activations/layer8_attention_weight_max": 48.85017776489258,
      "activations/layer8_attention_weight_min": -50.26282501220703,
      "activations/layer9_attention_weight_max": 46.57899856567383,
      "activations/layer9_attention_weight_min": -47.097286224365234,
      "epoch": 13.9,
      "learning_rate": 6.098522727272727e-05,
      "loss": 2.7623,
      "step": 239200
    },
    {
      "activations/layer0_attention_weight_max": 16.137235641479492,
      "activations/layer0_attention_weight_min": -14.178237915039062,
      "activations/layer10_attention_weight_max": 45.41679763793945,
      "activations/layer10_attention_weight_min": -42.358421325683594,
      "activations/layer11_attention_weight_max": 39.65475845336914,
      "activations/layer11_attention_weight_min": -34.417747497558594,
      "activations/layer12_attention_weight_max": 19.95513916015625,
      "activations/layer12_attention_weight_min": -25.241989135742188,
      "activations/layer13_attention_weight_max": 44.07911682128906,
      "activations/layer13_attention_weight_min": -28.850744247436523,
      "activations/layer14_attention_weight_max": 40.946205139160156,
      "activations/layer14_attention_weight_min": -30.202939987182617,
      "activations/layer15_attention_weight_max": 37.19728088378906,
      "activations/layer15_attention_weight_min": -29.137548446655273,
      "activations/layer16_attention_weight_max": 30.387954711914062,
      "activations/layer16_attention_weight_min": -29.737567901611328,
      "activations/layer17_attention_weight_max": 50.739662170410156,
      "activations/layer17_attention_weight_min": -44.24786376953125,
      "activations/layer18_attention_weight_max": 44.0738525390625,
      "activations/layer18_attention_weight_min": -36.31230163574219,
      "activations/layer19_attention_weight_max": 21.685256958007812,
      "activations/layer19_attention_weight_min": -19.9921932220459,
      "activations/layer1_attention_weight_max": 16.736103057861328,
      "activations/layer1_attention_weight_min": -15.202910423278809,
      "activations/layer20_attention_weight_max": 22.04939079284668,
      "activations/layer20_attention_weight_min": -20.90726089477539,
      "activations/layer21_attention_weight_max": 33.657474517822266,
      "activations/layer21_attention_weight_min": -22.165678024291992,
      "activations/layer22_attention_weight_max": 30.626667022705078,
      "activations/layer22_attention_weight_min": -23.055179595947266,
      "activations/layer23_attention_weight_max": 35.69781494140625,
      "activations/layer23_attention_weight_min": -24.362064361572266,
      "activations/layer2_attention_weight_max": 33.45359802246094,
      "activations/layer2_attention_weight_min": -32.69209671020508,
      "activations/layer3_attention_weight_max": 99.41773223876953,
      "activations/layer3_attention_weight_min": -103.406005859375,
      "activations/layer4_attention_weight_max": 103.42986297607422,
      "activations/layer4_attention_weight_min": -98.79444122314453,
      "activations/layer5_attention_weight_max": 71.46284484863281,
      "activations/layer5_attention_weight_min": -80.49362182617188,
      "activations/layer6_attention_weight_max": 53.06544494628906,
      "activations/layer6_attention_weight_min": -57.879669189453125,
      "activations/layer7_attention_weight_max": 74.77409362792969,
      "activations/layer7_attention_weight_min": -72.4241943359375,
      "activations/layer8_attention_weight_max": 55.610111236572266,
      "activations/layer8_attention_weight_min": -52.45621871948242,
      "activations/layer9_attention_weight_max": 58.00584030151367,
      "activations/layer9_attention_weight_min": -51.09882354736328,
      "epoch": 13.9,
      "learning_rate": 6.096628787878787e-05,
      "loss": 2.7545,
      "step": 239250
    },
    {
      "activations/layer0_attention_weight_max": 16.382610321044922,
      "activations/layer0_attention_weight_min": -14.293044090270996,
      "activations/layer10_attention_weight_max": 37.918983459472656,
      "activations/layer10_attention_weight_min": -38.93695831298828,
      "activations/layer11_attention_weight_max": 35.27901840209961,
      "activations/layer11_attention_weight_min": -37.96742248535156,
      "activations/layer12_attention_weight_max": 19.28013801574707,
      "activations/layer12_attention_weight_min": -25.877017974853516,
      "activations/layer13_attention_weight_max": 41.107093811035156,
      "activations/layer13_attention_weight_min": -32.01274871826172,
      "activations/layer14_attention_weight_max": 41.76255798339844,
      "activations/layer14_attention_weight_min": -33.522586822509766,
      "activations/layer15_attention_weight_max": 40.49556350708008,
      "activations/layer15_attention_weight_min": -31.753040313720703,
      "activations/layer16_attention_weight_max": 38.81897735595703,
      "activations/layer16_attention_weight_min": -33.62215042114258,
      "activations/layer17_attention_weight_max": 60.13392639160156,
      "activations/layer17_attention_weight_min": -47.42009735107422,
      "activations/layer18_attention_weight_max": 58.8731803894043,
      "activations/layer18_attention_weight_min": -41.9848747253418,
      "activations/layer19_attention_weight_max": 25.677762985229492,
      "activations/layer19_attention_weight_min": -23.495487213134766,
      "activations/layer1_attention_weight_max": 16.217588424682617,
      "activations/layer1_attention_weight_min": -14.587890625,
      "activations/layer20_attention_weight_max": 27.66695213317871,
      "activations/layer20_attention_weight_min": -22.41453742980957,
      "activations/layer21_attention_weight_max": 53.799591064453125,
      "activations/layer21_attention_weight_min": -26.940399169921875,
      "activations/layer22_attention_weight_max": 38.803226470947266,
      "activations/layer22_attention_weight_min": -27.281444549560547,
      "activations/layer23_attention_weight_max": 42.68999481201172,
      "activations/layer23_attention_weight_min": -25.8609619140625,
      "activations/layer2_attention_weight_max": 32.94106674194336,
      "activations/layer2_attention_weight_min": -32.94110107421875,
      "activations/layer3_attention_weight_max": 97.84185791015625,
      "activations/layer3_attention_weight_min": -104.56181335449219,
      "activations/layer4_attention_weight_max": 95.57344818115234,
      "activations/layer4_attention_weight_min": -94.37655639648438,
      "activations/layer5_attention_weight_max": 71.22212219238281,
      "activations/layer5_attention_weight_min": -79.29580688476562,
      "activations/layer6_attention_weight_max": 51.04185104370117,
      "activations/layer6_attention_weight_min": -50.80677032470703,
      "activations/layer7_attention_weight_max": 67.78958892822266,
      "activations/layer7_attention_weight_min": -65.9233627319336,
      "activations/layer8_attention_weight_max": 46.4765625,
      "activations/layer8_attention_weight_min": -52.252197265625,
      "activations/layer9_attention_weight_max": 48.35879135131836,
      "activations/layer9_attention_weight_min": -48.063262939453125,
      "epoch": 13.9,
      "learning_rate": 6.094734848484848e-05,
      "loss": 2.7432,
      "step": 239300
    },
    {
      "activations/layer0_attention_weight_max": 16.49805450439453,
      "activations/layer0_attention_weight_min": -14.313557624816895,
      "activations/layer10_attention_weight_max": 37.53297424316406,
      "activations/layer10_attention_weight_min": -36.17796325683594,
      "activations/layer11_attention_weight_max": 33.9744758605957,
      "activations/layer11_attention_weight_min": -33.34150695800781,
      "activations/layer12_attention_weight_max": 19.415618896484375,
      "activations/layer12_attention_weight_min": -26.610183715820312,
      "activations/layer13_attention_weight_max": 38.95948791503906,
      "activations/layer13_attention_weight_min": -30.495624542236328,
      "activations/layer14_attention_weight_max": 36.0222053527832,
      "activations/layer14_attention_weight_min": -32.113372802734375,
      "activations/layer15_attention_weight_max": 33.14945983886719,
      "activations/layer15_attention_weight_min": -31.413406372070312,
      "activations/layer16_attention_weight_max": 32.51533889770508,
      "activations/layer16_attention_weight_min": -29.5550594329834,
      "activations/layer17_attention_weight_max": 52.173248291015625,
      "activations/layer17_attention_weight_min": -44.90483093261719,
      "activations/layer18_attention_weight_max": 46.06884002685547,
      "activations/layer18_attention_weight_min": -38.57706069946289,
      "activations/layer19_attention_weight_max": 25.85386085510254,
      "activations/layer19_attention_weight_min": -22.51263427734375,
      "activations/layer1_attention_weight_max": 16.544017791748047,
      "activations/layer1_attention_weight_min": -15.026334762573242,
      "activations/layer20_attention_weight_max": 25.135562896728516,
      "activations/layer20_attention_weight_min": -20.618396759033203,
      "activations/layer21_attention_weight_max": 37.3445930480957,
      "activations/layer21_attention_weight_min": -21.35952377319336,
      "activations/layer22_attention_weight_max": 30.99079132080078,
      "activations/layer22_attention_weight_min": -25.77241325378418,
      "activations/layer23_attention_weight_max": 38.922462463378906,
      "activations/layer23_attention_weight_min": -22.785940170288086,
      "activations/layer2_attention_weight_max": 34.47600555419922,
      "activations/layer2_attention_weight_min": -29.79750633239746,
      "activations/layer3_attention_weight_max": 97.81963348388672,
      "activations/layer3_attention_weight_min": -99.77495574951172,
      "activations/layer4_attention_weight_max": 97.38423919677734,
      "activations/layer4_attention_weight_min": -93.38566589355469,
      "activations/layer5_attention_weight_max": 76.50029754638672,
      "activations/layer5_attention_weight_min": -81.19998168945312,
      "activations/layer6_attention_weight_max": 56.092742919921875,
      "activations/layer6_attention_weight_min": -55.10370635986328,
      "activations/layer7_attention_weight_max": 68.40591430664062,
      "activations/layer7_attention_weight_min": -67.69944763183594,
      "activations/layer8_attention_weight_max": 48.33303451538086,
      "activations/layer8_attention_weight_min": -50.10368728637695,
      "activations/layer9_attention_weight_max": 47.08808135986328,
      "activations/layer9_attention_weight_min": -45.84857940673828,
      "epoch": 13.91,
      "learning_rate": 6.092840909090909e-05,
      "loss": 2.7775,
      "step": 239350
    },
    {
      "activations/layer0_attention_weight_max": 16.339200973510742,
      "activations/layer0_attention_weight_min": -13.617837905883789,
      "activations/layer10_attention_weight_max": 36.01185607910156,
      "activations/layer10_attention_weight_min": -33.5938835144043,
      "activations/layer11_attention_weight_max": 30.849205017089844,
      "activations/layer11_attention_weight_min": -33.17789840698242,
      "activations/layer12_attention_weight_max": 19.804367065429688,
      "activations/layer12_attention_weight_min": -24.483707427978516,
      "activations/layer13_attention_weight_max": 37.07479476928711,
      "activations/layer13_attention_weight_min": -29.081867218017578,
      "activations/layer14_attention_weight_max": 36.288082122802734,
      "activations/layer14_attention_weight_min": -29.878564834594727,
      "activations/layer15_attention_weight_max": 32.598873138427734,
      "activations/layer15_attention_weight_min": -28.634479522705078,
      "activations/layer16_attention_weight_max": 32.88814163208008,
      "activations/layer16_attention_weight_min": -28.272193908691406,
      "activations/layer17_attention_weight_max": 53.8925666809082,
      "activations/layer17_attention_weight_min": -45.40852737426758,
      "activations/layer18_attention_weight_max": 45.30710983276367,
      "activations/layer18_attention_weight_min": -37.15610885620117,
      "activations/layer19_attention_weight_max": 24.108205795288086,
      "activations/layer19_attention_weight_min": -24.391746520996094,
      "activations/layer1_attention_weight_max": 16.479711532592773,
      "activations/layer1_attention_weight_min": -14.701698303222656,
      "activations/layer20_attention_weight_max": 22.12843894958496,
      "activations/layer20_attention_weight_min": -20.736478805541992,
      "activations/layer21_attention_weight_max": 34.548702239990234,
      "activations/layer21_attention_weight_min": -22.336639404296875,
      "activations/layer22_attention_weight_max": 30.741928100585938,
      "activations/layer22_attention_weight_min": -26.27783203125,
      "activations/layer23_attention_weight_max": 37.27423858642578,
      "activations/layer23_attention_weight_min": -25.459280014038086,
      "activations/layer2_attention_weight_max": 32.37492370605469,
      "activations/layer2_attention_weight_min": -31.96620750427246,
      "activations/layer3_attention_weight_max": 96.99324798583984,
      "activations/layer3_attention_weight_min": -93.40457916259766,
      "activations/layer4_attention_weight_max": 96.0561752319336,
      "activations/layer4_attention_weight_min": -91.40294647216797,
      "activations/layer5_attention_weight_max": 72.73561096191406,
      "activations/layer5_attention_weight_min": -75.84957885742188,
      "activations/layer6_attention_weight_max": 54.42430877685547,
      "activations/layer6_attention_weight_min": -53.52104949951172,
      "activations/layer7_attention_weight_max": 68.69303131103516,
      "activations/layer7_attention_weight_min": -67.3199691772461,
      "activations/layer8_attention_weight_max": 46.32732009887695,
      "activations/layer8_attention_weight_min": -45.966609954833984,
      "activations/layer9_attention_weight_max": 44.933815002441406,
      "activations/layer9_attention_weight_min": -43.51561737060547,
      "epoch": 13.91,
      "learning_rate": 6.0909469696969696e-05,
      "loss": 2.7529,
      "step": 239400
    },
    {
      "activations/layer0_attention_weight_max": 16.46258544921875,
      "activations/layer0_attention_weight_min": -14.009102821350098,
      "activations/layer10_attention_weight_max": 41.2025032043457,
      "activations/layer10_attention_weight_min": -40.28483200073242,
      "activations/layer11_attention_weight_max": 36.41259002685547,
      "activations/layer11_attention_weight_min": -37.18968200683594,
      "activations/layer12_attention_weight_max": 23.47722816467285,
      "activations/layer12_attention_weight_min": -28.217605590820312,
      "activations/layer13_attention_weight_max": 37.780738830566406,
      "activations/layer13_attention_weight_min": -32.10462951660156,
      "activations/layer14_attention_weight_max": 37.35655975341797,
      "activations/layer14_attention_weight_min": -30.854679107666016,
      "activations/layer15_attention_weight_max": 33.70368957519531,
      "activations/layer15_attention_weight_min": -28.8214054107666,
      "activations/layer16_attention_weight_max": 32.47195816040039,
      "activations/layer16_attention_weight_min": -31.00177764892578,
      "activations/layer17_attention_weight_max": 50.31153106689453,
      "activations/layer17_attention_weight_min": -43.98471450805664,
      "activations/layer18_attention_weight_max": 46.26431655883789,
      "activations/layer18_attention_weight_min": -37.66246795654297,
      "activations/layer19_attention_weight_max": 23.681406021118164,
      "activations/layer19_attention_weight_min": -23.09535026550293,
      "activations/layer1_attention_weight_max": 17.44222640991211,
      "activations/layer1_attention_weight_min": -14.00423526763916,
      "activations/layer20_attention_weight_max": 20.233081817626953,
      "activations/layer20_attention_weight_min": -23.222532272338867,
      "activations/layer21_attention_weight_max": 33.19810485839844,
      "activations/layer21_attention_weight_min": -28.198740005493164,
      "activations/layer22_attention_weight_max": 28.924837112426758,
      "activations/layer22_attention_weight_min": -25.67116355895996,
      "activations/layer23_attention_weight_max": 35.15095520019531,
      "activations/layer23_attention_weight_min": -24.928356170654297,
      "activations/layer2_attention_weight_max": 32.9366455078125,
      "activations/layer2_attention_weight_min": -31.994863510131836,
      "activations/layer3_attention_weight_max": 96.30631256103516,
      "activations/layer3_attention_weight_min": -97.4528579711914,
      "activations/layer4_attention_weight_max": 101.9719009399414,
      "activations/layer4_attention_weight_min": -98.1035385131836,
      "activations/layer5_attention_weight_max": 71.85758972167969,
      "activations/layer5_attention_weight_min": -77.408935546875,
      "activations/layer6_attention_weight_max": 51.925498962402344,
      "activations/layer6_attention_weight_min": -52.83857727050781,
      "activations/layer7_attention_weight_max": 70.69831085205078,
      "activations/layer7_attention_weight_min": -64.35633850097656,
      "activations/layer8_attention_weight_max": 47.1568603515625,
      "activations/layer8_attention_weight_min": -48.95412063598633,
      "activations/layer9_attention_weight_max": 46.4858512878418,
      "activations/layer9_attention_weight_min": -48.1121711730957,
      "epoch": 13.91,
      "learning_rate": 6.08905303030303e-05,
      "loss": 2.773,
      "step": 239450
    },
    {
      "activations/layer0_attention_weight_max": 16.078479766845703,
      "activations/layer0_attention_weight_min": -14.266000747680664,
      "activations/layer10_attention_weight_max": 36.593116760253906,
      "activations/layer10_attention_weight_min": -36.68000030517578,
      "activations/layer11_attention_weight_max": 32.161376953125,
      "activations/layer11_attention_weight_min": -34.62717056274414,
      "activations/layer12_attention_weight_max": 18.859663009643555,
      "activations/layer12_attention_weight_min": -26.87563705444336,
      "activations/layer13_attention_weight_max": 37.72748947143555,
      "activations/layer13_attention_weight_min": -27.90118980407715,
      "activations/layer14_attention_weight_max": 40.11784744262695,
      "activations/layer14_attention_weight_min": -30.337156295776367,
      "activations/layer15_attention_weight_max": 31.52022361755371,
      "activations/layer15_attention_weight_min": -30.04096221923828,
      "activations/layer16_attention_weight_max": 32.21120834350586,
      "activations/layer16_attention_weight_min": -30.691972732543945,
      "activations/layer17_attention_weight_max": 53.263145446777344,
      "activations/layer17_attention_weight_min": -43.61717987060547,
      "activations/layer18_attention_weight_max": 48.55533218383789,
      "activations/layer18_attention_weight_min": -39.40769958496094,
      "activations/layer19_attention_weight_max": 24.986806869506836,
      "activations/layer19_attention_weight_min": -25.278667449951172,
      "activations/layer1_attention_weight_max": 15.89393138885498,
      "activations/layer1_attention_weight_min": -14.767521858215332,
      "activations/layer20_attention_weight_max": 25.865942001342773,
      "activations/layer20_attention_weight_min": -22.98090171813965,
      "activations/layer21_attention_weight_max": 36.56232452392578,
      "activations/layer21_attention_weight_min": -22.311025619506836,
      "activations/layer22_attention_weight_max": 35.48391342163086,
      "activations/layer22_attention_weight_min": -25.983003616333008,
      "activations/layer23_attention_weight_max": 45.32221221923828,
      "activations/layer23_attention_weight_min": -23.962196350097656,
      "activations/layer2_attention_weight_max": 31.886234283447266,
      "activations/layer2_attention_weight_min": -30.155515670776367,
      "activations/layer3_attention_weight_max": 95.80664825439453,
      "activations/layer3_attention_weight_min": -98.44451141357422,
      "activations/layer4_attention_weight_max": 97.7210693359375,
      "activations/layer4_attention_weight_min": -94.18920135498047,
      "activations/layer5_attention_weight_max": 75.6281509399414,
      "activations/layer5_attention_weight_min": -78.78666687011719,
      "activations/layer6_attention_weight_max": 54.17597961425781,
      "activations/layer6_attention_weight_min": -55.0120964050293,
      "activations/layer7_attention_weight_max": 72.35635375976562,
      "activations/layer7_attention_weight_min": -64.9183578491211,
      "activations/layer8_attention_weight_max": 46.72789001464844,
      "activations/layer8_attention_weight_min": -48.56245422363281,
      "activations/layer9_attention_weight_max": 48.872398376464844,
      "activations/layer9_attention_weight_min": -48.318111419677734,
      "epoch": 13.92,
      "learning_rate": 6.08715909090909e-05,
      "loss": 2.7517,
      "step": 239500
    },
    {
      "activations/layer0_attention_weight_max": 15.764090538024902,
      "activations/layer0_attention_weight_min": -14.263483047485352,
      "activations/layer10_attention_weight_max": 34.972564697265625,
      "activations/layer10_attention_weight_min": -35.77975082397461,
      "activations/layer11_attention_weight_max": 30.99493408203125,
      "activations/layer11_attention_weight_min": -35.0440559387207,
      "activations/layer12_attention_weight_max": 18.562829971313477,
      "activations/layer12_attention_weight_min": -24.24720573425293,
      "activations/layer13_attention_weight_max": 40.58055877685547,
      "activations/layer13_attention_weight_min": -30.358964920043945,
      "activations/layer14_attention_weight_max": 35.99397659301758,
      "activations/layer14_attention_weight_min": -29.872045516967773,
      "activations/layer15_attention_weight_max": 34.84080123901367,
      "activations/layer15_attention_weight_min": -29.446504592895508,
      "activations/layer16_attention_weight_max": 34.95126724243164,
      "activations/layer16_attention_weight_min": -29.877546310424805,
      "activations/layer17_attention_weight_max": 53.072731018066406,
      "activations/layer17_attention_weight_min": -45.76115417480469,
      "activations/layer18_attention_weight_max": 49.90377426147461,
      "activations/layer18_attention_weight_min": -41.02266311645508,
      "activations/layer19_attention_weight_max": 25.097219467163086,
      "activations/layer19_attention_weight_min": -22.301799774169922,
      "activations/layer1_attention_weight_max": 15.988141059875488,
      "activations/layer1_attention_weight_min": -13.674635887145996,
      "activations/layer20_attention_weight_max": 23.332197189331055,
      "activations/layer20_attention_weight_min": -21.30331802368164,
      "activations/layer21_attention_weight_max": 36.15766143798828,
      "activations/layer21_attention_weight_min": -22.602825164794922,
      "activations/layer22_attention_weight_max": 32.78926086425781,
      "activations/layer22_attention_weight_min": -26.989343643188477,
      "activations/layer23_attention_weight_max": 39.87382125854492,
      "activations/layer23_attention_weight_min": -24.332746505737305,
      "activations/layer2_attention_weight_max": 30.281768798828125,
      "activations/layer2_attention_weight_min": -30.71255111694336,
      "activations/layer3_attention_weight_max": 95.78874206542969,
      "activations/layer3_attention_weight_min": -101.44931030273438,
      "activations/layer4_attention_weight_max": 101.73736572265625,
      "activations/layer4_attention_weight_min": -95.20301055908203,
      "activations/layer5_attention_weight_max": 72.53994750976562,
      "activations/layer5_attention_weight_min": -79.43617248535156,
      "activations/layer6_attention_weight_max": 54.0110969543457,
      "activations/layer6_attention_weight_min": -53.39731216430664,
      "activations/layer7_attention_weight_max": 69.60063934326172,
      "activations/layer7_attention_weight_min": -66.17189025878906,
      "activations/layer8_attention_weight_max": 48.05491638183594,
      "activations/layer8_attention_weight_min": -48.713687896728516,
      "activations/layer9_attention_weight_max": 46.47255325317383,
      "activations/layer9_attention_weight_min": -46.894100189208984,
      "epoch": 13.92,
      "learning_rate": 6.0852651515151514e-05,
      "loss": 2.7666,
      "step": 239550
    },
    {
      "activations/layer0_attention_weight_max": 16.490825653076172,
      "activations/layer0_attention_weight_min": -13.77426815032959,
      "activations/layer10_attention_weight_max": 39.39495849609375,
      "activations/layer10_attention_weight_min": -39.91518020629883,
      "activations/layer11_attention_weight_max": 34.427608489990234,
      "activations/layer11_attention_weight_min": -36.235618591308594,
      "activations/layer12_attention_weight_max": 26.550209045410156,
      "activations/layer12_attention_weight_min": -26.152938842773438,
      "activations/layer13_attention_weight_max": 43.31537628173828,
      "activations/layer13_attention_weight_min": -32.02828598022461,
      "activations/layer14_attention_weight_max": 39.1881217956543,
      "activations/layer14_attention_weight_min": -30.33466911315918,
      "activations/layer15_attention_weight_max": 35.41775131225586,
      "activations/layer15_attention_weight_min": -31.28185272216797,
      "activations/layer16_attention_weight_max": 38.96609878540039,
      "activations/layer16_attention_weight_min": -31.69749641418457,
      "activations/layer17_attention_weight_max": 59.89308166503906,
      "activations/layer17_attention_weight_min": -46.25611877441406,
      "activations/layer18_attention_weight_max": 51.02806854248047,
      "activations/layer18_attention_weight_min": -39.602943420410156,
      "activations/layer19_attention_weight_max": 24.690773010253906,
      "activations/layer19_attention_weight_min": -22.676687240600586,
      "activations/layer1_attention_weight_max": 16.436674118041992,
      "activations/layer1_attention_weight_min": -15.667243003845215,
      "activations/layer20_attention_weight_max": 26.162275314331055,
      "activations/layer20_attention_weight_min": -23.117141723632812,
      "activations/layer21_attention_weight_max": 36.00545120239258,
      "activations/layer21_attention_weight_min": -22.07222557067871,
      "activations/layer22_attention_weight_max": 32.326011657714844,
      "activations/layer22_attention_weight_min": -24.865856170654297,
      "activations/layer23_attention_weight_max": 39.998313903808594,
      "activations/layer23_attention_weight_min": -23.646678924560547,
      "activations/layer2_attention_weight_max": 33.542999267578125,
      "activations/layer2_attention_weight_min": -30.783138275146484,
      "activations/layer3_attention_weight_max": 97.63021087646484,
      "activations/layer3_attention_weight_min": -97.09455108642578,
      "activations/layer4_attention_weight_max": 100.58177947998047,
      "activations/layer4_attention_weight_min": -95.87569427490234,
      "activations/layer5_attention_weight_max": 74.12224578857422,
      "activations/layer5_attention_weight_min": -80.06631469726562,
      "activations/layer6_attention_weight_max": 54.342811584472656,
      "activations/layer6_attention_weight_min": -55.12220764160156,
      "activations/layer7_attention_weight_max": 70.67293548583984,
      "activations/layer7_attention_weight_min": -66.75116729736328,
      "activations/layer8_attention_weight_max": 50.16456985473633,
      "activations/layer8_attention_weight_min": -52.27478790283203,
      "activations/layer9_attention_weight_max": 49.261775970458984,
      "activations/layer9_attention_weight_min": -50.24890899658203,
      "epoch": 13.92,
      "learning_rate": 6.0833712121212115e-05,
      "loss": 2.7635,
      "step": 239600
    },
    {
      "activations/layer0_attention_weight_max": 16.000783920288086,
      "activations/layer0_attention_weight_min": -14.528166770935059,
      "activations/layer10_attention_weight_max": 36.512569427490234,
      "activations/layer10_attention_weight_min": -37.29059982299805,
      "activations/layer11_attention_weight_max": 34.1134033203125,
      "activations/layer11_attention_weight_min": -33.759033203125,
      "activations/layer12_attention_weight_max": 20.52670669555664,
      "activations/layer12_attention_weight_min": -26.656896591186523,
      "activations/layer13_attention_weight_max": 50.502708435058594,
      "activations/layer13_attention_weight_min": -39.47881317138672,
      "activations/layer14_attention_weight_max": 62.61403274536133,
      "activations/layer14_attention_weight_min": -35.0748291015625,
      "activations/layer15_attention_weight_max": 57.03744125366211,
      "activations/layer15_attention_weight_min": -35.55392074584961,
      "activations/layer16_attention_weight_max": 37.30480194091797,
      "activations/layer16_attention_weight_min": -34.08934020996094,
      "activations/layer17_attention_weight_max": 60.4908447265625,
      "activations/layer17_attention_weight_min": -55.04152297973633,
      "activations/layer18_attention_weight_max": 51.50897979736328,
      "activations/layer18_attention_weight_min": -44.190032958984375,
      "activations/layer19_attention_weight_max": 26.40860366821289,
      "activations/layer19_attention_weight_min": -25.388931274414062,
      "activations/layer1_attention_weight_max": 15.999197959899902,
      "activations/layer1_attention_weight_min": -15.632984161376953,
      "activations/layer20_attention_weight_max": 28.503971099853516,
      "activations/layer20_attention_weight_min": -21.52770233154297,
      "activations/layer21_attention_weight_max": 45.5919189453125,
      "activations/layer21_attention_weight_min": -28.867155075073242,
      "activations/layer22_attention_weight_max": 31.727983474731445,
      "activations/layer22_attention_weight_min": -25.174226760864258,
      "activations/layer23_attention_weight_max": 33.80280303955078,
      "activations/layer23_attention_weight_min": -24.509624481201172,
      "activations/layer2_attention_weight_max": 32.91215515136719,
      "activations/layer2_attention_weight_min": -30.703956604003906,
      "activations/layer3_attention_weight_max": 95.26018524169922,
      "activations/layer3_attention_weight_min": -100.14771270751953,
      "activations/layer4_attention_weight_max": 94.32662200927734,
      "activations/layer4_attention_weight_min": -90.53778839111328,
      "activations/layer5_attention_weight_max": 69.43080139160156,
      "activations/layer5_attention_weight_min": -75.25782775878906,
      "activations/layer6_attention_weight_max": 49.32858657836914,
      "activations/layer6_attention_weight_min": -51.30479431152344,
      "activations/layer7_attention_weight_max": 66.05545806884766,
      "activations/layer7_attention_weight_min": -64.32520294189453,
      "activations/layer8_attention_weight_max": 50.15470886230469,
      "activations/layer8_attention_weight_min": -47.08810806274414,
      "activations/layer9_attention_weight_max": 47.8914909362793,
      "activations/layer9_attention_weight_min": -45.626705169677734,
      "epoch": 13.92,
      "learning_rate": 6.081477272727272e-05,
      "loss": 2.7487,
      "step": 239650
    },
    {
      "activations/layer0_attention_weight_max": 16.26117706298828,
      "activations/layer0_attention_weight_min": -13.887688636779785,
      "activations/layer10_attention_weight_max": 35.27325439453125,
      "activations/layer10_attention_weight_min": -33.87956619262695,
      "activations/layer11_attention_weight_max": 32.6700439453125,
      "activations/layer11_attention_weight_min": -34.21260070800781,
      "activations/layer12_attention_weight_max": 19.801176071166992,
      "activations/layer12_attention_weight_min": -27.22909927368164,
      "activations/layer13_attention_weight_max": 41.658287048339844,
      "activations/layer13_attention_weight_min": -30.23349952697754,
      "activations/layer14_attention_weight_max": 33.8023681640625,
      "activations/layer14_attention_weight_min": -29.77469253540039,
      "activations/layer15_attention_weight_max": 34.241127014160156,
      "activations/layer15_attention_weight_min": -29.14468765258789,
      "activations/layer16_attention_weight_max": 32.78466033935547,
      "activations/layer16_attention_weight_min": -32.557857513427734,
      "activations/layer17_attention_weight_max": 48.99397659301758,
      "activations/layer17_attention_weight_min": -43.69124984741211,
      "activations/layer18_attention_weight_max": 44.01131820678711,
      "activations/layer18_attention_weight_min": -37.75785827636719,
      "activations/layer19_attention_weight_max": 22.053157806396484,
      "activations/layer19_attention_weight_min": -22.74061393737793,
      "activations/layer1_attention_weight_max": 15.802469253540039,
      "activations/layer1_attention_weight_min": -14.681291580200195,
      "activations/layer20_attention_weight_max": 22.06322479248047,
      "activations/layer20_attention_weight_min": -23.014392852783203,
      "activations/layer21_attention_weight_max": 36.52394485473633,
      "activations/layer21_attention_weight_min": -23.36807632446289,
      "activations/layer22_attention_weight_max": 29.959993362426758,
      "activations/layer22_attention_weight_min": -26.120012283325195,
      "activations/layer23_attention_weight_max": 38.03384780883789,
      "activations/layer23_attention_weight_min": -23.685558319091797,
      "activations/layer2_attention_weight_max": 33.35152816772461,
      "activations/layer2_attention_weight_min": -35.665870666503906,
      "activations/layer3_attention_weight_max": 91.74674224853516,
      "activations/layer3_attention_weight_min": -99.1539077758789,
      "activations/layer4_attention_weight_max": 92.86454772949219,
      "activations/layer4_attention_weight_min": -94.73429107666016,
      "activations/layer5_attention_weight_max": 72.08045959472656,
      "activations/layer5_attention_weight_min": -83.96771240234375,
      "activations/layer6_attention_weight_max": 53.75265121459961,
      "activations/layer6_attention_weight_min": -49.168392181396484,
      "activations/layer7_attention_weight_max": 64.4937973022461,
      "activations/layer7_attention_weight_min": -64.96260833740234,
      "activations/layer8_attention_weight_max": 43.49652862548828,
      "activations/layer8_attention_weight_min": -45.436546325683594,
      "activations/layer9_attention_weight_max": 45.6402702331543,
      "activations/layer9_attention_weight_min": -45.76095962524414,
      "epoch": 13.93,
      "learning_rate": 6.0795833333333325e-05,
      "loss": 2.7638,
      "step": 239700
    },
    {
      "activations/layer0_attention_weight_max": 15.906267166137695,
      "activations/layer0_attention_weight_min": -14.620899200439453,
      "activations/layer10_attention_weight_max": 37.00307083129883,
      "activations/layer10_attention_weight_min": -37.389957427978516,
      "activations/layer11_attention_weight_max": 34.41040802001953,
      "activations/layer11_attention_weight_min": -37.13698959350586,
      "activations/layer12_attention_weight_max": 19.34408187866211,
      "activations/layer12_attention_weight_min": -32.47710037231445,
      "activations/layer13_attention_weight_max": 38.85235595703125,
      "activations/layer13_attention_weight_min": -28.338720321655273,
      "activations/layer14_attention_weight_max": 40.44550323486328,
      "activations/layer14_attention_weight_min": -32.67470169067383,
      "activations/layer15_attention_weight_max": 37.538856506347656,
      "activations/layer15_attention_weight_min": -30.483715057373047,
      "activations/layer16_attention_weight_max": 32.49443435668945,
      "activations/layer16_attention_weight_min": -31.461849212646484,
      "activations/layer17_attention_weight_max": 49.31741714477539,
      "activations/layer17_attention_weight_min": -48.2639274597168,
      "activations/layer18_attention_weight_max": 42.71715545654297,
      "activations/layer18_attention_weight_min": -41.43768310546875,
      "activations/layer19_attention_weight_max": 23.74323081970215,
      "activations/layer19_attention_weight_min": -21.97113609313965,
      "activations/layer1_attention_weight_max": 17.15182876586914,
      "activations/layer1_attention_weight_min": -17.59564208984375,
      "activations/layer20_attention_weight_max": 23.077524185180664,
      "activations/layer20_attention_weight_min": -22.181798934936523,
      "activations/layer21_attention_weight_max": 36.407203674316406,
      "activations/layer21_attention_weight_min": -22.91472053527832,
      "activations/layer22_attention_weight_max": 28.71600341796875,
      "activations/layer22_attention_weight_min": -26.21431541442871,
      "activations/layer23_attention_weight_max": 36.814720153808594,
      "activations/layer23_attention_weight_min": -23.604846954345703,
      "activations/layer2_attention_weight_max": 32.121421813964844,
      "activations/layer2_attention_weight_min": -31.178754806518555,
      "activations/layer3_attention_weight_max": 92.43739318847656,
      "activations/layer3_attention_weight_min": -94.92217254638672,
      "activations/layer4_attention_weight_max": 99.05013275146484,
      "activations/layer4_attention_weight_min": -93.28923034667969,
      "activations/layer5_attention_weight_max": 74.07418060302734,
      "activations/layer5_attention_weight_min": -79.25053405761719,
      "activations/layer6_attention_weight_max": 56.39076614379883,
      "activations/layer6_attention_weight_min": -56.00684356689453,
      "activations/layer7_attention_weight_max": 66.93805694580078,
      "activations/layer7_attention_weight_min": -65.23876190185547,
      "activations/layer8_attention_weight_max": 48.591796875,
      "activations/layer8_attention_weight_min": -50.06589126586914,
      "activations/layer9_attention_weight_max": 50.574275970458984,
      "activations/layer9_attention_weight_min": -46.45500564575195,
      "epoch": 13.93,
      "learning_rate": 6.077689393939393e-05,
      "loss": 2.7687,
      "step": 239750
    },
    {
      "activations/layer0_attention_weight_max": 16.44571876525879,
      "activations/layer0_attention_weight_min": -14.49454402923584,
      "activations/layer10_attention_weight_max": 45.34804153442383,
      "activations/layer10_attention_weight_min": -43.1786003112793,
      "activations/layer11_attention_weight_max": 36.22957992553711,
      "activations/layer11_attention_weight_min": -37.439666748046875,
      "activations/layer12_attention_weight_max": 19.86992835998535,
      "activations/layer12_attention_weight_min": -25.998109817504883,
      "activations/layer13_attention_weight_max": 35.801361083984375,
      "activations/layer13_attention_weight_min": -28.212535858154297,
      "activations/layer14_attention_weight_max": 36.28101348876953,
      "activations/layer14_attention_weight_min": -30.915929794311523,
      "activations/layer15_attention_weight_max": 31.83751106262207,
      "activations/layer15_attention_weight_min": -30.826210021972656,
      "activations/layer16_attention_weight_max": 30.9595947265625,
      "activations/layer16_attention_weight_min": -29.194202423095703,
      "activations/layer17_attention_weight_max": 50.88404846191406,
      "activations/layer17_attention_weight_min": -45.77788162231445,
      "activations/layer18_attention_weight_max": 44.5484733581543,
      "activations/layer18_attention_weight_min": -37.820770263671875,
      "activations/layer19_attention_weight_max": 24.264925003051758,
      "activations/layer19_attention_weight_min": -21.009878158569336,
      "activations/layer1_attention_weight_max": 17.27769660949707,
      "activations/layer1_attention_weight_min": -12.81253719329834,
      "activations/layer20_attention_weight_max": 22.571542739868164,
      "activations/layer20_attention_weight_min": -21.014371871948242,
      "activations/layer21_attention_weight_max": 36.52729034423828,
      "activations/layer21_attention_weight_min": -21.572927474975586,
      "activations/layer22_attention_weight_max": 34.40260314941406,
      "activations/layer22_attention_weight_min": -24.7397518157959,
      "activations/layer23_attention_weight_max": 45.131526947021484,
      "activations/layer23_attention_weight_min": -23.59172821044922,
      "activations/layer2_attention_weight_max": 30.806753158569336,
      "activations/layer2_attention_weight_min": -28.787067413330078,
      "activations/layer3_attention_weight_max": 94.6050033569336,
      "activations/layer3_attention_weight_min": -97.5837631225586,
      "activations/layer4_attention_weight_max": 98.63352966308594,
      "activations/layer4_attention_weight_min": -92.7293930053711,
      "activations/layer5_attention_weight_max": 71.23756408691406,
      "activations/layer5_attention_weight_min": -80.9754409790039,
      "activations/layer6_attention_weight_max": 52.71607971191406,
      "activations/layer6_attention_weight_min": -55.39192199707031,
      "activations/layer7_attention_weight_max": 67.79214477539062,
      "activations/layer7_attention_weight_min": -66.13356018066406,
      "activations/layer8_attention_weight_max": 52.40333557128906,
      "activations/layer8_attention_weight_min": -56.058563232421875,
      "activations/layer9_attention_weight_max": 51.0892219543457,
      "activations/layer9_attention_weight_min": -51.54169464111328,
      "epoch": 13.93,
      "learning_rate": 6.075795454545454e-05,
      "loss": 2.752,
      "step": 239800
    },
    {
      "activations/layer0_attention_weight_max": 15.743270874023438,
      "activations/layer0_attention_weight_min": -14.6618070602417,
      "activations/layer10_attention_weight_max": 46.924163818359375,
      "activations/layer10_attention_weight_min": -42.5451545715332,
      "activations/layer11_attention_weight_max": 43.43020248413086,
      "activations/layer11_attention_weight_min": -39.074462890625,
      "activations/layer12_attention_weight_max": 19.09187889099121,
      "activations/layer12_attention_weight_min": -26.02436065673828,
      "activations/layer13_attention_weight_max": 69.9556884765625,
      "activations/layer13_attention_weight_min": -37.15641784667969,
      "activations/layer14_attention_weight_max": 62.43843078613281,
      "activations/layer14_attention_weight_min": -34.539207458496094,
      "activations/layer15_attention_weight_max": 51.5234260559082,
      "activations/layer15_attention_weight_min": -31.595802307128906,
      "activations/layer16_attention_weight_max": 34.88352966308594,
      "activations/layer16_attention_weight_min": -32.13758087158203,
      "activations/layer17_attention_weight_max": 53.75527572631836,
      "activations/layer17_attention_weight_min": -48.518348693847656,
      "activations/layer18_attention_weight_max": 50.441993713378906,
      "activations/layer18_attention_weight_min": -42.53371047973633,
      "activations/layer19_attention_weight_max": 25.340200424194336,
      "activations/layer19_attention_weight_min": -22.655073165893555,
      "activations/layer1_attention_weight_max": 16.2750244140625,
      "activations/layer1_attention_weight_min": -14.861946105957031,
      "activations/layer20_attention_weight_max": 24.30622100830078,
      "activations/layer20_attention_weight_min": -21.418771743774414,
      "activations/layer21_attention_weight_max": 48.6346549987793,
      "activations/layer21_attention_weight_min": -26.470565795898438,
      "activations/layer22_attention_weight_max": 30.832578659057617,
      "activations/layer22_attention_weight_min": -25.47991371154785,
      "activations/layer23_attention_weight_max": 42.91136169433594,
      "activations/layer23_attention_weight_min": -25.938190460205078,
      "activations/layer2_attention_weight_max": 32.37641525268555,
      "activations/layer2_attention_weight_min": -29.834026336669922,
      "activations/layer3_attention_weight_max": 95.2552261352539,
      "activations/layer3_attention_weight_min": -95.57298278808594,
      "activations/layer4_attention_weight_max": 96.23886108398438,
      "activations/layer4_attention_weight_min": -92.21558380126953,
      "activations/layer5_attention_weight_max": 68.63529205322266,
      "activations/layer5_attention_weight_min": -80.35307312011719,
      "activations/layer6_attention_weight_max": 57.572486877441406,
      "activations/layer6_attention_weight_min": -56.75871658325195,
      "activations/layer7_attention_weight_max": 76.1375732421875,
      "activations/layer7_attention_weight_min": -73.63445281982422,
      "activations/layer8_attention_weight_max": 57.21541213989258,
      "activations/layer8_attention_weight_min": -55.084449768066406,
      "activations/layer9_attention_weight_max": 65.77363586425781,
      "activations/layer9_attention_weight_min": -52.63429641723633,
      "epoch": 13.94,
      "learning_rate": 6.073901515151514e-05,
      "loss": 2.761,
      "step": 239850
    },
    {
      "activations/layer0_attention_weight_max": 16.031909942626953,
      "activations/layer0_attention_weight_min": -14.362039566040039,
      "activations/layer10_attention_weight_max": 35.45081329345703,
      "activations/layer10_attention_weight_min": -36.43012619018555,
      "activations/layer11_attention_weight_max": 32.430625915527344,
      "activations/layer11_attention_weight_min": -33.64406204223633,
      "activations/layer12_attention_weight_max": 19.92788314819336,
      "activations/layer12_attention_weight_min": -24.04601287841797,
      "activations/layer13_attention_weight_max": 38.137451171875,
      "activations/layer13_attention_weight_min": -28.026947021484375,
      "activations/layer14_attention_weight_max": 41.45762252807617,
      "activations/layer14_attention_weight_min": -31.20484733581543,
      "activations/layer15_attention_weight_max": 36.42930603027344,
      "activations/layer15_attention_weight_min": -29.07061004638672,
      "activations/layer16_attention_weight_max": 37.65927505493164,
      "activations/layer16_attention_weight_min": -31.949785232543945,
      "activations/layer17_attention_weight_max": 56.138587951660156,
      "activations/layer17_attention_weight_min": -45.020633697509766,
      "activations/layer18_attention_weight_max": 54.25253677368164,
      "activations/layer18_attention_weight_min": -44.474143981933594,
      "activations/layer19_attention_weight_max": 24.964017868041992,
      "activations/layer19_attention_weight_min": -27.936471939086914,
      "activations/layer1_attention_weight_max": 16.85869598388672,
      "activations/layer1_attention_weight_min": -14.223250389099121,
      "activations/layer20_attention_weight_max": 29.63909149169922,
      "activations/layer20_attention_weight_min": -24.16802215576172,
      "activations/layer21_attention_weight_max": 46.552398681640625,
      "activations/layer21_attention_weight_min": -29.075407028198242,
      "activations/layer22_attention_weight_max": 34.543113708496094,
      "activations/layer22_attention_weight_min": -26.77897834777832,
      "activations/layer23_attention_weight_max": 46.993553161621094,
      "activations/layer23_attention_weight_min": -24.566139221191406,
      "activations/layer2_attention_weight_max": 29.751815795898438,
      "activations/layer2_attention_weight_min": -28.37325096130371,
      "activations/layer3_attention_weight_max": 87.73432922363281,
      "activations/layer3_attention_weight_min": -86.55818939208984,
      "activations/layer4_attention_weight_max": 86.90544891357422,
      "activations/layer4_attention_weight_min": -88.57054901123047,
      "activations/layer5_attention_weight_max": 69.57012939453125,
      "activations/layer5_attention_weight_min": -74.40705108642578,
      "activations/layer6_attention_weight_max": 50.629661560058594,
      "activations/layer6_attention_weight_min": -51.91455841064453,
      "activations/layer7_attention_weight_max": 65.02478790283203,
      "activations/layer7_attention_weight_min": -63.694366455078125,
      "activations/layer8_attention_weight_max": 45.42924118041992,
      "activations/layer8_attention_weight_min": -49.15671920776367,
      "activations/layer9_attention_weight_max": 42.31733322143555,
      "activations/layer9_attention_weight_min": -45.80202865600586,
      "epoch": 13.94,
      "learning_rate": 6.072007575757576e-05,
      "loss": 2.7773,
      "step": 239900
    },
    {
      "activations/layer0_attention_weight_max": 16.69523048400879,
      "activations/layer0_attention_weight_min": -14.058388710021973,
      "activations/layer10_attention_weight_max": 36.502540588378906,
      "activations/layer10_attention_weight_min": -35.97514724731445,
      "activations/layer11_attention_weight_max": 31.58113670349121,
      "activations/layer11_attention_weight_min": -34.104923248291016,
      "activations/layer12_attention_weight_max": 18.310550689697266,
      "activations/layer12_attention_weight_min": -25.14364242553711,
      "activations/layer13_attention_weight_max": 33.96598815917969,
      "activations/layer13_attention_weight_min": -30.190011978149414,
      "activations/layer14_attention_weight_max": 40.13356018066406,
      "activations/layer14_attention_weight_min": -29.924306869506836,
      "activations/layer15_attention_weight_max": 38.618621826171875,
      "activations/layer15_attention_weight_min": -28.605745315551758,
      "activations/layer16_attention_weight_max": 32.30070495605469,
      "activations/layer16_attention_weight_min": -29.373798370361328,
      "activations/layer17_attention_weight_max": 54.60725021362305,
      "activations/layer17_attention_weight_min": -45.09393310546875,
      "activations/layer18_attention_weight_max": 51.926265716552734,
      "activations/layer18_attention_weight_min": -37.59754180908203,
      "activations/layer19_attention_weight_max": 23.022781372070312,
      "activations/layer19_attention_weight_min": -23.602956771850586,
      "activations/layer1_attention_weight_max": 16.6151065826416,
      "activations/layer1_attention_weight_min": -13.186423301696777,
      "activations/layer20_attention_weight_max": 21.388063430786133,
      "activations/layer20_attention_weight_min": -22.785863876342773,
      "activations/layer21_attention_weight_max": 36.663673400878906,
      "activations/layer21_attention_weight_min": -22.7016544342041,
      "activations/layer22_attention_weight_max": 32.587467193603516,
      "activations/layer22_attention_weight_min": -26.713666915893555,
      "activations/layer23_attention_weight_max": 37.78772735595703,
      "activations/layer23_attention_weight_min": -24.99903678894043,
      "activations/layer2_attention_weight_max": 33.65974426269531,
      "activations/layer2_attention_weight_min": -31.42947769165039,
      "activations/layer3_attention_weight_max": 93.2733383178711,
      "activations/layer3_attention_weight_min": -92.63021850585938,
      "activations/layer4_attention_weight_max": 89.70244598388672,
      "activations/layer4_attention_weight_min": -89.6052017211914,
      "activations/layer5_attention_weight_max": 68.21715545654297,
      "activations/layer5_attention_weight_min": -75.38705444335938,
      "activations/layer6_attention_weight_max": 49.62397003173828,
      "activations/layer6_attention_weight_min": -49.25971603393555,
      "activations/layer7_attention_weight_max": 65.87995147705078,
      "activations/layer7_attention_weight_min": -66.60379791259766,
      "activations/layer8_attention_weight_max": 46.08940505981445,
      "activations/layer8_attention_weight_min": -47.63603210449219,
      "activations/layer9_attention_weight_max": 49.81618881225586,
      "activations/layer9_attention_weight_min": -44.99314498901367,
      "epoch": 13.94,
      "learning_rate": 6.070113636363636e-05,
      "loss": 2.7752,
      "step": 239950
    },
    {
      "activations/layer0_attention_weight_max": 15.400590896606445,
      "activations/layer0_attention_weight_min": -14.214286804199219,
      "activations/layer10_attention_weight_max": 37.3083381652832,
      "activations/layer10_attention_weight_min": -36.216434478759766,
      "activations/layer11_attention_weight_max": 32.45695114135742,
      "activations/layer11_attention_weight_min": -31.78842544555664,
      "activations/layer12_attention_weight_max": 20.905364990234375,
      "activations/layer12_attention_weight_min": -24.495208740234375,
      "activations/layer13_attention_weight_max": 34.486328125,
      "activations/layer13_attention_weight_min": -28.677785873413086,
      "activations/layer14_attention_weight_max": 35.03068923950195,
      "activations/layer14_attention_weight_min": -29.258636474609375,
      "activations/layer15_attention_weight_max": 31.561986923217773,
      "activations/layer15_attention_weight_min": -29.94853401184082,
      "activations/layer16_attention_weight_max": 31.67943572998047,
      "activations/layer16_attention_weight_min": -31.967565536499023,
      "activations/layer17_attention_weight_max": 51.96003723144531,
      "activations/layer17_attention_weight_min": -47.07318878173828,
      "activations/layer18_attention_weight_max": 48.14530944824219,
      "activations/layer18_attention_weight_min": -39.287200927734375,
      "activations/layer19_attention_weight_max": 24.25299644470215,
      "activations/layer19_attention_weight_min": -22.07916259765625,
      "activations/layer1_attention_weight_max": 16.01185417175293,
      "activations/layer1_attention_weight_min": -15.542503356933594,
      "activations/layer20_attention_weight_max": 22.217758178710938,
      "activations/layer20_attention_weight_min": -23.489809036254883,
      "activations/layer21_attention_weight_max": 32.9650764465332,
      "activations/layer21_attention_weight_min": -21.10057830810547,
      "activations/layer22_attention_weight_max": 32.629981994628906,
      "activations/layer22_attention_weight_min": -30.264484405517578,
      "activations/layer23_attention_weight_max": 36.22554397583008,
      "activations/layer23_attention_weight_min": -24.37213706970215,
      "activations/layer2_attention_weight_max": 31.395687103271484,
      "activations/layer2_attention_weight_min": -30.61548614501953,
      "activations/layer3_attention_weight_max": 90.83220672607422,
      "activations/layer3_attention_weight_min": -90.23789978027344,
      "activations/layer4_attention_weight_max": 90.64681243896484,
      "activations/layer4_attention_weight_min": -86.17137145996094,
      "activations/layer5_attention_weight_max": 70.71236419677734,
      "activations/layer5_attention_weight_min": -76.82707214355469,
      "activations/layer6_attention_weight_max": 50.490333557128906,
      "activations/layer6_attention_weight_min": -52.04692077636719,
      "activations/layer7_attention_weight_max": 67.73651123046875,
      "activations/layer7_attention_weight_min": -63.77603530883789,
      "activations/layer8_attention_weight_max": 44.99622344970703,
      "activations/layer8_attention_weight_min": -47.41461181640625,
      "activations/layer9_attention_weight_max": 46.58964538574219,
      "activations/layer9_attention_weight_min": -44.33785629272461,
      "epoch": 13.95,
      "learning_rate": 6.068219696969696e-05,
      "loss": 2.7588,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.5272,
      "eval_samples_per_second": 503.566,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.5272,
      "eval_openwebtext_samples_per_second": 503.566,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_wikitext_loss": 2.943359375,
      "eval_wikitext_ppl": 18.979498589532987,
      "eval_wikitext_runtime": 1.9639,
      "eval_wikitext_samples_per_second": 232.186,
      "step": 240000
    },
    {
      "epoch": 13.95,
      "eval_lambada_loss": 2.509765625,
      "eval_lambada_ppl": 12.302046430466076,
      "eval_lambada_runtime": 9.5806,
      "eval_lambada_samples_per_second": 508.213,
      "step": 240000
    },
    {
      "activations/layer0_attention_weight_max": 16.2380428314209,
      "activations/layer0_attention_weight_min": -14.381732940673828,
      "activations/layer10_attention_weight_max": 38.76249694824219,
      "activations/layer10_attention_weight_min": -37.639373779296875,
      "activations/layer11_attention_weight_max": 33.099639892578125,
      "activations/layer11_attention_weight_min": -32.72658157348633,
      "activations/layer12_attention_weight_max": 19.04916763305664,
      "activations/layer12_attention_weight_min": -24.742013931274414,
      "activations/layer13_attention_weight_max": 41.10691833496094,
      "activations/layer13_attention_weight_min": -29.381359100341797,
      "activations/layer14_attention_weight_max": 37.17007827758789,
      "activations/layer14_attention_weight_min": -30.78590202331543,
      "activations/layer15_attention_weight_max": 33.83684158325195,
      "activations/layer15_attention_weight_min": -32.03300094604492,
      "activations/layer16_attention_weight_max": 36.277950286865234,
      "activations/layer16_attention_weight_min": -33.36874771118164,
      "activations/layer17_attention_weight_max": 56.40098190307617,
      "activations/layer17_attention_weight_min": -47.764862060546875,
      "activations/layer18_attention_weight_max": 53.91429901123047,
      "activations/layer18_attention_weight_min": -43.047882080078125,
      "activations/layer19_attention_weight_max": 26.425039291381836,
      "activations/layer19_attention_weight_min": -22.44289207458496,
      "activations/layer1_attention_weight_max": 17.620275497436523,
      "activations/layer1_attention_weight_min": -15.372227668762207,
      "activations/layer20_attention_weight_max": 26.7315616607666,
      "activations/layer20_attention_weight_min": -21.78917694091797,
      "activations/layer21_attention_weight_max": 40.051063537597656,
      "activations/layer21_attention_weight_min": -22.538721084594727,
      "activations/layer22_attention_weight_max": 31.118427276611328,
      "activations/layer22_attention_weight_min": -26.001855850219727,
      "activations/layer23_attention_weight_max": 36.993587493896484,
      "activations/layer23_attention_weight_min": -25.606603622436523,
      "activations/layer2_attention_weight_max": 34.12049865722656,
      "activations/layer2_attention_weight_min": -33.16590881347656,
      "activations/layer3_attention_weight_max": 95.05084228515625,
      "activations/layer3_attention_weight_min": -99.7651138305664,
      "activations/layer4_attention_weight_max": 97.6716537475586,
      "activations/layer4_attention_weight_min": -95.50444793701172,
      "activations/layer5_attention_weight_max": 76.54640197753906,
      "activations/layer5_attention_weight_min": -79.7084732055664,
      "activations/layer6_attention_weight_max": 56.316619873046875,
      "activations/layer6_attention_weight_min": -55.885520935058594,
      "activations/layer7_attention_weight_max": 67.81399536132812,
      "activations/layer7_attention_weight_min": -65.40129852294922,
      "activations/layer8_attention_weight_max": 46.93290710449219,
      "activations/layer8_attention_weight_min": -48.62648391723633,
      "activations/layer9_attention_weight_max": 48.169281005859375,
      "activations/layer9_attention_weight_min": -46.9028434753418,
      "epoch": 13.95,
      "learning_rate": 6.066325757575757e-05,
      "loss": 2.7698,
      "step": 240050
    },
    {
      "activations/layer0_attention_weight_max": 16.224529266357422,
      "activations/layer0_attention_weight_min": -14.537506103515625,
      "activations/layer10_attention_weight_max": 42.61773681640625,
      "activations/layer10_attention_weight_min": -40.89881134033203,
      "activations/layer11_attention_weight_max": 39.01676940917969,
      "activations/layer11_attention_weight_min": -36.87430953979492,
      "activations/layer12_attention_weight_max": 22.648149490356445,
      "activations/layer12_attention_weight_min": -24.949703216552734,
      "activations/layer13_attention_weight_max": 58.29458999633789,
      "activations/layer13_attention_weight_min": -35.563941955566406,
      "activations/layer14_attention_weight_max": 50.87039566040039,
      "activations/layer14_attention_weight_min": -35.11039352416992,
      "activations/layer15_attention_weight_max": 46.03221893310547,
      "activations/layer15_attention_weight_min": -31.00324058532715,
      "activations/layer16_attention_weight_max": 34.9202880859375,
      "activations/layer16_attention_weight_min": -33.07137680053711,
      "activations/layer17_attention_weight_max": 62.82552719116211,
      "activations/layer17_attention_weight_min": -53.9262809753418,
      "activations/layer18_attention_weight_max": 51.97379684448242,
      "activations/layer18_attention_weight_min": -45.73029708862305,
      "activations/layer19_attention_weight_max": 27.644628524780273,
      "activations/layer19_attention_weight_min": -25.1102294921875,
      "activations/layer1_attention_weight_max": 16.661571502685547,
      "activations/layer1_attention_weight_min": -15.595224380493164,
      "activations/layer20_attention_weight_max": 30.216625213623047,
      "activations/layer20_attention_weight_min": -23.530424118041992,
      "activations/layer21_attention_weight_max": 55.062339782714844,
      "activations/layer21_attention_weight_min": -28.14330291748047,
      "activations/layer22_attention_weight_max": 38.56825256347656,
      "activations/layer22_attention_weight_min": -28.067455291748047,
      "activations/layer23_attention_weight_max": 42.404720306396484,
      "activations/layer23_attention_weight_min": -29.39010238647461,
      "activations/layer2_attention_weight_max": 33.175567626953125,
      "activations/layer2_attention_weight_min": -32.204063415527344,
      "activations/layer3_attention_weight_max": 97.81002044677734,
      "activations/layer3_attention_weight_min": -98.95906829833984,
      "activations/layer4_attention_weight_max": 91.97771453857422,
      "activations/layer4_attention_weight_min": -96.4885025024414,
      "activations/layer5_attention_weight_max": 65.44960021972656,
      "activations/layer5_attention_weight_min": -76.25312805175781,
      "activations/layer6_attention_weight_max": 54.03062438964844,
      "activations/layer6_attention_weight_min": -51.64161682128906,
      "activations/layer7_attention_weight_max": 65.28651428222656,
      "activations/layer7_attention_weight_min": -66.23834228515625,
      "activations/layer8_attention_weight_max": 49.793556213378906,
      "activations/layer8_attention_weight_min": -49.99095916748047,
      "activations/layer9_attention_weight_max": 54.11250686645508,
      "activations/layer9_attention_weight_min": -48.55297088623047,
      "epoch": 13.95,
      "learning_rate": 6.064431818181818e-05,
      "loss": 2.762,
      "step": 240100
    },
    {
      "activations/layer0_attention_weight_max": 16.22990608215332,
      "activations/layer0_attention_weight_min": -13.731236457824707,
      "activations/layer10_attention_weight_max": 43.68156051635742,
      "activations/layer10_attention_weight_min": -40.622474670410156,
      "activations/layer11_attention_weight_max": 39.22013854980469,
      "activations/layer11_attention_weight_min": -35.81407928466797,
      "activations/layer12_attention_weight_max": 18.43231773376465,
      "activations/layer12_attention_weight_min": -24.84576416015625,
      "activations/layer13_attention_weight_max": 42.19559860229492,
      "activations/layer13_attention_weight_min": -27.669509887695312,
      "activations/layer14_attention_weight_max": 36.45869827270508,
      "activations/layer14_attention_weight_min": -29.20615005493164,
      "activations/layer15_attention_weight_max": 33.560550689697266,
      "activations/layer15_attention_weight_min": -30.18805503845215,
      "activations/layer16_attention_weight_max": 34.49580383300781,
      "activations/layer16_attention_weight_min": -33.27880859375,
      "activations/layer17_attention_weight_max": 55.305816650390625,
      "activations/layer17_attention_weight_min": -48.28204345703125,
      "activations/layer18_attention_weight_max": 48.481754302978516,
      "activations/layer18_attention_weight_min": -41.02455520629883,
      "activations/layer19_attention_weight_max": 23.290538787841797,
      "activations/layer19_attention_weight_min": -23.481372833251953,
      "activations/layer1_attention_weight_max": 17.278568267822266,
      "activations/layer1_attention_weight_min": -13.96641731262207,
      "activations/layer20_attention_weight_max": 22.715791702270508,
      "activations/layer20_attention_weight_min": -21.76722526550293,
      "activations/layer21_attention_weight_max": 34.58882522583008,
      "activations/layer21_attention_weight_min": -20.93878173828125,
      "activations/layer22_attention_weight_max": 32.369140625,
      "activations/layer22_attention_weight_min": -25.1827449798584,
      "activations/layer23_attention_weight_max": 38.192012786865234,
      "activations/layer23_attention_weight_min": -23.097171783447266,
      "activations/layer2_attention_weight_max": 35.437225341796875,
      "activations/layer2_attention_weight_min": -30.42995262145996,
      "activations/layer3_attention_weight_max": 96.81426239013672,
      "activations/layer3_attention_weight_min": -95.51622009277344,
      "activations/layer4_attention_weight_max": 93.22482299804688,
      "activations/layer4_attention_weight_min": -87.94451904296875,
      "activations/layer5_attention_weight_max": 73.80985260009766,
      "activations/layer5_attention_weight_min": -73.85546875,
      "activations/layer6_attention_weight_max": 52.720603942871094,
      "activations/layer6_attention_weight_min": -52.274715423583984,
      "activations/layer7_attention_weight_max": 72.0466537475586,
      "activations/layer7_attention_weight_min": -64.5431900024414,
      "activations/layer8_attention_weight_max": 51.07024383544922,
      "activations/layer8_attention_weight_min": -50.927513122558594,
      "activations/layer9_attention_weight_max": 59.67299270629883,
      "activations/layer9_attention_weight_min": -47.54487991333008,
      "epoch": 13.95,
      "learning_rate": 6.0625378787878786e-05,
      "loss": 2.7535,
      "step": 240150
    },
    {
      "activations/layer0_attention_weight_max": 15.962398529052734,
      "activations/layer0_attention_weight_min": -14.203069686889648,
      "activations/layer10_attention_weight_max": 35.75130081176758,
      "activations/layer10_attention_weight_min": -35.047088623046875,
      "activations/layer11_attention_weight_max": 33.45120620727539,
      "activations/layer11_attention_weight_min": -34.479061126708984,
      "activations/layer12_attention_weight_max": 18.376480102539062,
      "activations/layer12_attention_weight_min": -28.773456573486328,
      "activations/layer13_attention_weight_max": 37.57194519042969,
      "activations/layer13_attention_weight_min": -30.08120346069336,
      "activations/layer14_attention_weight_max": 35.5398063659668,
      "activations/layer14_attention_weight_min": -32.100189208984375,
      "activations/layer15_attention_weight_max": 34.7531852722168,
      "activations/layer15_attention_weight_min": -30.8486385345459,
      "activations/layer16_attention_weight_max": 34.112037658691406,
      "activations/layer16_attention_weight_min": -31.18819236755371,
      "activations/layer17_attention_weight_max": 51.640743255615234,
      "activations/layer17_attention_weight_min": -47.25661849975586,
      "activations/layer18_attention_weight_max": 47.776824951171875,
      "activations/layer18_attention_weight_min": -42.117671966552734,
      "activations/layer19_attention_weight_max": 25.771543502807617,
      "activations/layer19_attention_weight_min": -20.878032684326172,
      "activations/layer1_attention_weight_max": 15.619675636291504,
      "activations/layer1_attention_weight_min": -14.01761531829834,
      "activations/layer20_attention_weight_max": 21.60140037536621,
      "activations/layer20_attention_weight_min": -22.664175033569336,
      "activations/layer21_attention_weight_max": 36.7259407043457,
      "activations/layer21_attention_weight_min": -22.60532569885254,
      "activations/layer22_attention_weight_max": 29.218244552612305,
      "activations/layer22_attention_weight_min": -25.114303588867188,
      "activations/layer23_attention_weight_max": 36.12255859375,
      "activations/layer23_attention_weight_min": -22.986461639404297,
      "activations/layer2_attention_weight_max": 33.52763366699219,
      "activations/layer2_attention_weight_min": -32.54814529418945,
      "activations/layer3_attention_weight_max": 95.65416717529297,
      "activations/layer3_attention_weight_min": -90.12248992919922,
      "activations/layer4_attention_weight_max": 100.08817291259766,
      "activations/layer4_attention_weight_min": -92.0584945678711,
      "activations/layer5_attention_weight_max": 70.40172576904297,
      "activations/layer5_attention_weight_min": -78.88818359375,
      "activations/layer6_attention_weight_max": 53.65827941894531,
      "activations/layer6_attention_weight_min": -53.402652740478516,
      "activations/layer7_attention_weight_max": 71.05230712890625,
      "activations/layer7_attention_weight_min": -67.6272964477539,
      "activations/layer8_attention_weight_max": 48.64234924316406,
      "activations/layer8_attention_weight_min": -49.71027374267578,
      "activations/layer9_attention_weight_max": 44.72007369995117,
      "activations/layer9_attention_weight_min": -48.3916130065918,
      "epoch": 13.96,
      "learning_rate": 6.060643939393939e-05,
      "loss": 2.7711,
      "step": 240200
    },
    {
      "activations/layer0_attention_weight_max": 15.72459602355957,
      "activations/layer0_attention_weight_min": -14.514037132263184,
      "activations/layer10_attention_weight_max": 40.744937896728516,
      "activations/layer10_attention_weight_min": -36.6397819519043,
      "activations/layer11_attention_weight_max": 38.05060958862305,
      "activations/layer11_attention_weight_min": -34.26910400390625,
      "activations/layer12_attention_weight_max": 22.328998565673828,
      "activations/layer12_attention_weight_min": -23.86217498779297,
      "activations/layer13_attention_weight_max": 36.55221939086914,
      "activations/layer13_attention_weight_min": -30.692848205566406,
      "activations/layer14_attention_weight_max": 35.615413665771484,
      "activations/layer14_attention_weight_min": -31.365856170654297,
      "activations/layer15_attention_weight_max": 32.681427001953125,
      "activations/layer15_attention_weight_min": -32.550045013427734,
      "activations/layer16_attention_weight_max": 32.277992248535156,
      "activations/layer16_attention_weight_min": -30.271753311157227,
      "activations/layer17_attention_weight_max": 51.52980422973633,
      "activations/layer17_attention_weight_min": -46.07059860229492,
      "activations/layer18_attention_weight_max": 48.06413650512695,
      "activations/layer18_attention_weight_min": -38.90492630004883,
      "activations/layer19_attention_weight_max": 22.796186447143555,
      "activations/layer19_attention_weight_min": -21.48589515686035,
      "activations/layer1_attention_weight_max": 17.192399978637695,
      "activations/layer1_attention_weight_min": -16.296916961669922,
      "activations/layer20_attention_weight_max": 24.765356063842773,
      "activations/layer20_attention_weight_min": -22.86714744567871,
      "activations/layer21_attention_weight_max": 39.882667541503906,
      "activations/layer21_attention_weight_min": -23.434024810791016,
      "activations/layer22_attention_weight_max": 31.19028091430664,
      "activations/layer22_attention_weight_min": -27.93971824645996,
      "activations/layer23_attention_weight_max": 38.637271881103516,
      "activations/layer23_attention_weight_min": -24.934757232666016,
      "activations/layer2_attention_weight_max": 33.2545166015625,
      "activations/layer2_attention_weight_min": -32.840511322021484,
      "activations/layer3_attention_weight_max": 94.68029022216797,
      "activations/layer3_attention_weight_min": -91.38728332519531,
      "activations/layer4_attention_weight_max": 99.72900390625,
      "activations/layer4_attention_weight_min": -94.80902862548828,
      "activations/layer5_attention_weight_max": 73.25942993164062,
      "activations/layer5_attention_weight_min": -83.64994049072266,
      "activations/layer6_attention_weight_max": 55.569461822509766,
      "activations/layer6_attention_weight_min": -54.781009674072266,
      "activations/layer7_attention_weight_max": 70.6483154296875,
      "activations/layer7_attention_weight_min": -72.74276733398438,
      "activations/layer8_attention_weight_max": 47.897552490234375,
      "activations/layer8_attention_weight_min": -50.8891487121582,
      "activations/layer9_attention_weight_max": 46.74212646484375,
      "activations/layer9_attention_weight_min": -47.52696228027344,
      "epoch": 13.96,
      "learning_rate": 6.058749999999999e-05,
      "loss": 2.7748,
      "step": 240250
    },
    {
      "activations/layer0_attention_weight_max": 15.657244682312012,
      "activations/layer0_attention_weight_min": -14.592520713806152,
      "activations/layer10_attention_weight_max": 36.67759323120117,
      "activations/layer10_attention_weight_min": -36.090484619140625,
      "activations/layer11_attention_weight_max": 33.63648986816406,
      "activations/layer11_attention_weight_min": -34.87655258178711,
      "activations/layer12_attention_weight_max": 17.722936630249023,
      "activations/layer12_attention_weight_min": -26.011066436767578,
      "activations/layer13_attention_weight_max": 34.02386474609375,
      "activations/layer13_attention_weight_min": -28.595741271972656,
      "activations/layer14_attention_weight_max": 35.43782043457031,
      "activations/layer14_attention_weight_min": -29.222122192382812,
      "activations/layer15_attention_weight_max": 32.67482376098633,
      "activations/layer15_attention_weight_min": -28.30486488342285,
      "activations/layer16_attention_weight_max": 32.722721099853516,
      "activations/layer16_attention_weight_min": -30.818403244018555,
      "activations/layer17_attention_weight_max": 51.63542938232422,
      "activations/layer17_attention_weight_min": -43.48408889770508,
      "activations/layer18_attention_weight_max": 49.088504791259766,
      "activations/layer18_attention_weight_min": -39.0946159362793,
      "activations/layer19_attention_weight_max": 23.16343879699707,
      "activations/layer19_attention_weight_min": -22.653335571289062,
      "activations/layer1_attention_weight_max": 16.702390670776367,
      "activations/layer1_attention_weight_min": -15.08790397644043,
      "activations/layer20_attention_weight_max": 22.246191024780273,
      "activations/layer20_attention_weight_min": -19.4493408203125,
      "activations/layer21_attention_weight_max": 43.54887771606445,
      "activations/layer21_attention_weight_min": -26.12471580505371,
      "activations/layer22_attention_weight_max": 32.453243255615234,
      "activations/layer22_attention_weight_min": -24.027111053466797,
      "activations/layer23_attention_weight_max": 38.043312072753906,
      "activations/layer23_attention_weight_min": -25.77950668334961,
      "activations/layer2_attention_weight_max": 31.005325317382812,
      "activations/layer2_attention_weight_min": -29.83132553100586,
      "activations/layer3_attention_weight_max": 97.65418243408203,
      "activations/layer3_attention_weight_min": -96.53289794921875,
      "activations/layer4_attention_weight_max": 96.50069427490234,
      "activations/layer4_attention_weight_min": -94.95957946777344,
      "activations/layer5_attention_weight_max": 76.0052719116211,
      "activations/layer5_attention_weight_min": -80.65992736816406,
      "activations/layer6_attention_weight_max": 53.77229690551758,
      "activations/layer6_attention_weight_min": -56.01201629638672,
      "activations/layer7_attention_weight_max": 66.1973876953125,
      "activations/layer7_attention_weight_min": -67.8501968383789,
      "activations/layer8_attention_weight_max": 47.54622268676758,
      "activations/layer8_attention_weight_min": -51.219932556152344,
      "activations/layer9_attention_weight_max": 46.94268798828125,
      "activations/layer9_attention_weight_min": -46.01215362548828,
      "epoch": 13.96,
      "learning_rate": 6.0568560606060604e-05,
      "loss": 2.7707,
      "step": 240300
    },
    {
      "activations/layer0_attention_weight_max": 16.822811126708984,
      "activations/layer0_attention_weight_min": -14.411467552185059,
      "activations/layer10_attention_weight_max": 38.441436767578125,
      "activations/layer10_attention_weight_min": -36.91602325439453,
      "activations/layer11_attention_weight_max": 33.56421661376953,
      "activations/layer11_attention_weight_min": -33.33358383178711,
      "activations/layer12_attention_weight_max": 18.52147674560547,
      "activations/layer12_attention_weight_min": -24.758331298828125,
      "activations/layer13_attention_weight_max": 37.908382415771484,
      "activations/layer13_attention_weight_min": -27.841476440429688,
      "activations/layer14_attention_weight_max": 36.18157196044922,
      "activations/layer14_attention_weight_min": -29.938264846801758,
      "activations/layer15_attention_weight_max": 32.9059944152832,
      "activations/layer15_attention_weight_min": -30.164220809936523,
      "activations/layer16_attention_weight_max": 34.42647171020508,
      "activations/layer16_attention_weight_min": -30.26105499267578,
      "activations/layer17_attention_weight_max": 55.828033447265625,
      "activations/layer17_attention_weight_min": -44.030662536621094,
      "activations/layer18_attention_weight_max": 47.12712860107422,
      "activations/layer18_attention_weight_min": -37.348541259765625,
      "activations/layer19_attention_weight_max": 26.956369400024414,
      "activations/layer19_attention_weight_min": -22.67310333251953,
      "activations/layer1_attention_weight_max": 15.87988567352295,
      "activations/layer1_attention_weight_min": -14.631176948547363,
      "activations/layer20_attention_weight_max": 23.885149002075195,
      "activations/layer20_attention_weight_min": -20.735149383544922,
      "activations/layer21_attention_weight_max": 34.60694885253906,
      "activations/layer21_attention_weight_min": -24.653385162353516,
      "activations/layer22_attention_weight_max": 28.56682014465332,
      "activations/layer22_attention_weight_min": -25.38255500793457,
      "activations/layer23_attention_weight_max": 40.28569793701172,
      "activations/layer23_attention_weight_min": -24.30887222290039,
      "activations/layer2_attention_weight_max": 32.87466049194336,
      "activations/layer2_attention_weight_min": -28.947202682495117,
      "activations/layer3_attention_weight_max": 91.39981079101562,
      "activations/layer3_attention_weight_min": -91.38182830810547,
      "activations/layer4_attention_weight_max": 90.22779846191406,
      "activations/layer4_attention_weight_min": -88.80709838867188,
      "activations/layer5_attention_weight_max": 71.59213256835938,
      "activations/layer5_attention_weight_min": -79.08043670654297,
      "activations/layer6_attention_weight_max": 51.42618942260742,
      "activations/layer6_attention_weight_min": -50.7066535949707,
      "activations/layer7_attention_weight_max": 70.3512954711914,
      "activations/layer7_attention_weight_min": -66.23212432861328,
      "activations/layer8_attention_weight_max": 46.835365295410156,
      "activations/layer8_attention_weight_min": -50.11482620239258,
      "activations/layer9_attention_weight_max": 45.92647933959961,
      "activations/layer9_attention_weight_min": -49.31593322753906,
      "epoch": 13.97,
      "learning_rate": 6.0549621212121205e-05,
      "loss": 2.7656,
      "step": 240350
    },
    {
      "activations/layer0_attention_weight_max": 15.728952407836914,
      "activations/layer0_attention_weight_min": -14.078807830810547,
      "activations/layer10_attention_weight_max": 37.96392822265625,
      "activations/layer10_attention_weight_min": -38.12921905517578,
      "activations/layer11_attention_weight_max": 32.54730224609375,
      "activations/layer11_attention_weight_min": -32.120784759521484,
      "activations/layer12_attention_weight_max": 20.598133087158203,
      "activations/layer12_attention_weight_min": -24.232582092285156,
      "activations/layer13_attention_weight_max": 33.83270263671875,
      "activations/layer13_attention_weight_min": -28.97670555114746,
      "activations/layer14_attention_weight_max": 36.475154876708984,
      "activations/layer14_attention_weight_min": -30.35896110534668,
      "activations/layer15_attention_weight_max": 32.20722198486328,
      "activations/layer15_attention_weight_min": -30.461896896362305,
      "activations/layer16_attention_weight_max": 32.93460464477539,
      "activations/layer16_attention_weight_min": -30.702220916748047,
      "activations/layer17_attention_weight_max": 55.92683792114258,
      "activations/layer17_attention_weight_min": -46.054683685302734,
      "activations/layer18_attention_weight_max": 49.003746032714844,
      "activations/layer18_attention_weight_min": -40.207794189453125,
      "activations/layer19_attention_weight_max": 22.131288528442383,
      "activations/layer19_attention_weight_min": -22.871198654174805,
      "activations/layer1_attention_weight_max": 17.1943416595459,
      "activations/layer1_attention_weight_min": -15.586966514587402,
      "activations/layer20_attention_weight_max": 23.699033737182617,
      "activations/layer20_attention_weight_min": -21.695768356323242,
      "activations/layer21_attention_weight_max": 41.87908172607422,
      "activations/layer21_attention_weight_min": -24.24333381652832,
      "activations/layer22_attention_weight_max": 29.25374984741211,
      "activations/layer22_attention_weight_min": -24.35000228881836,
      "activations/layer23_attention_weight_max": 36.688453674316406,
      "activations/layer23_attention_weight_min": -22.612646102905273,
      "activations/layer2_attention_weight_max": 31.47876739501953,
      "activations/layer2_attention_weight_min": -28.514755249023438,
      "activations/layer3_attention_weight_max": 91.39261627197266,
      "activations/layer3_attention_weight_min": -90.58356475830078,
      "activations/layer4_attention_weight_max": 93.46744537353516,
      "activations/layer4_attention_weight_min": -89.1923599243164,
      "activations/layer5_attention_weight_max": 69.33270263671875,
      "activations/layer5_attention_weight_min": -76.73503112792969,
      "activations/layer6_attention_weight_max": 52.8456916809082,
      "activations/layer6_attention_weight_min": -51.4566535949707,
      "activations/layer7_attention_weight_max": 68.47502899169922,
      "activations/layer7_attention_weight_min": -65.28824615478516,
      "activations/layer8_attention_weight_max": 46.311248779296875,
      "activations/layer8_attention_weight_min": -50.80168533325195,
      "activations/layer9_attention_weight_max": 48.225765228271484,
      "activations/layer9_attention_weight_min": -46.278133392333984,
      "epoch": 13.97,
      "learning_rate": 6.0530681818181814e-05,
      "loss": 2.7603,
      "step": 240400
    },
    {
      "activations/layer0_attention_weight_max": 17.599424362182617,
      "activations/layer0_attention_weight_min": -14.283129692077637,
      "activations/layer10_attention_weight_max": 43.942359924316406,
      "activations/layer10_attention_weight_min": -43.0859260559082,
      "activations/layer11_attention_weight_max": 39.211883544921875,
      "activations/layer11_attention_weight_min": -39.90312194824219,
      "activations/layer12_attention_weight_max": 19.238216400146484,
      "activations/layer12_attention_weight_min": -27.05339241027832,
      "activations/layer13_attention_weight_max": 57.811302185058594,
      "activations/layer13_attention_weight_min": -36.84404373168945,
      "activations/layer14_attention_weight_max": 57.17621612548828,
      "activations/layer14_attention_weight_min": -35.30023193359375,
      "activations/layer15_attention_weight_max": 46.49892807006836,
      "activations/layer15_attention_weight_min": -37.333839416503906,
      "activations/layer16_attention_weight_max": 42.315528869628906,
      "activations/layer16_attention_weight_min": -37.481868743896484,
      "activations/layer17_attention_weight_max": 66.27261352539062,
      "activations/layer17_attention_weight_min": -52.34880447387695,
      "activations/layer18_attention_weight_max": 58.436683654785156,
      "activations/layer18_attention_weight_min": -47.82735061645508,
      "activations/layer19_attention_weight_max": 30.758697509765625,
      "activations/layer19_attention_weight_min": -26.868589401245117,
      "activations/layer1_attention_weight_max": 16.408870697021484,
      "activations/layer1_attention_weight_min": -13.911724090576172,
      "activations/layer20_attention_weight_max": 33.13528060913086,
      "activations/layer20_attention_weight_min": -23.512205123901367,
      "activations/layer21_attention_weight_max": 55.061973571777344,
      "activations/layer21_attention_weight_min": -24.351749420166016,
      "activations/layer22_attention_weight_max": 38.664398193359375,
      "activations/layer22_attention_weight_min": -26.663917541503906,
      "activations/layer23_attention_weight_max": 44.61726379394531,
      "activations/layer23_attention_weight_min": -24.055252075195312,
      "activations/layer2_attention_weight_max": 33.30324935913086,
      "activations/layer2_attention_weight_min": -31.512868881225586,
      "activations/layer3_attention_weight_max": 97.57247161865234,
      "activations/layer3_attention_weight_min": -96.01557159423828,
      "activations/layer4_attention_weight_max": 97.68255615234375,
      "activations/layer4_attention_weight_min": -93.56153869628906,
      "activations/layer5_attention_weight_max": 72.90185546875,
      "activations/layer5_attention_weight_min": -82.43798065185547,
      "activations/layer6_attention_weight_max": 55.02220153808594,
      "activations/layer6_attention_weight_min": -56.1978645324707,
      "activations/layer7_attention_weight_max": 71.1410903930664,
      "activations/layer7_attention_weight_min": -72.72965240478516,
      "activations/layer8_attention_weight_max": 55.72447204589844,
      "activations/layer8_attention_weight_min": -55.04254150390625,
      "activations/layer9_attention_weight_max": 56.41156768798828,
      "activations/layer9_attention_weight_min": -56.78145980834961,
      "epoch": 13.97,
      "learning_rate": 6.0511742424242415e-05,
      "loss": 2.7727,
      "step": 240450
    },
    {
      "activations/layer0_attention_weight_max": 16.284029006958008,
      "activations/layer0_attention_weight_min": -14.32929515838623,
      "activations/layer10_attention_weight_max": 39.670753479003906,
      "activations/layer10_attention_weight_min": -35.85908508300781,
      "activations/layer11_attention_weight_max": 34.28300476074219,
      "activations/layer11_attention_weight_min": -33.897342681884766,
      "activations/layer12_attention_weight_max": 18.329803466796875,
      "activations/layer12_attention_weight_min": -24.997028350830078,
      "activations/layer13_attention_weight_max": 36.04578399658203,
      "activations/layer13_attention_weight_min": -27.30713653564453,
      "activations/layer14_attention_weight_max": 34.45830154418945,
      "activations/layer14_attention_weight_min": -30.653446197509766,
      "activations/layer15_attention_weight_max": 32.933021545410156,
      "activations/layer15_attention_weight_min": -31.112783432006836,
      "activations/layer16_attention_weight_max": 30.87380027770996,
      "activations/layer16_attention_weight_min": -30.335643768310547,
      "activations/layer17_attention_weight_max": 48.47877883911133,
      "activations/layer17_attention_weight_min": -45.74323272705078,
      "activations/layer18_attention_weight_max": 42.2055778503418,
      "activations/layer18_attention_weight_min": -37.50142288208008,
      "activations/layer19_attention_weight_max": 21.9985294342041,
      "activations/layer19_attention_weight_min": -20.444761276245117,
      "activations/layer1_attention_weight_max": 16.433202743530273,
      "activations/layer1_attention_weight_min": -15.25831127166748,
      "activations/layer20_attention_weight_max": 21.73859214782715,
      "activations/layer20_attention_weight_min": -20.12375259399414,
      "activations/layer21_attention_weight_max": 35.80055618286133,
      "activations/layer21_attention_weight_min": -23.702590942382812,
      "activations/layer22_attention_weight_max": 30.77573585510254,
      "activations/layer22_attention_weight_min": -25.93468475341797,
      "activations/layer23_attention_weight_max": 35.87123107910156,
      "activations/layer23_attention_weight_min": -24.107303619384766,
      "activations/layer2_attention_weight_max": 32.14017105102539,
      "activations/layer2_attention_weight_min": -30.6372127532959,
      "activations/layer3_attention_weight_max": 97.83401489257812,
      "activations/layer3_attention_weight_min": -97.72012329101562,
      "activations/layer4_attention_weight_max": 97.49214935302734,
      "activations/layer4_attention_weight_min": -94.62787628173828,
      "activations/layer5_attention_weight_max": 73.24793243408203,
      "activations/layer5_attention_weight_min": -76.89781951904297,
      "activations/layer6_attention_weight_max": 56.09357833862305,
      "activations/layer6_attention_weight_min": -52.43756866455078,
      "activations/layer7_attention_weight_max": 65.72447967529297,
      "activations/layer7_attention_weight_min": -64.46688842773438,
      "activations/layer8_attention_weight_max": 48.18620681762695,
      "activations/layer8_attention_weight_min": -48.36709976196289,
      "activations/layer9_attention_weight_max": 45.281883239746094,
      "activations/layer9_attention_weight_min": -45.498043060302734,
      "epoch": 13.97,
      "learning_rate": 6.049280303030303e-05,
      "loss": 2.7558,
      "step": 240500
    },
    {
      "activations/layer0_attention_weight_max": 16.077667236328125,
      "activations/layer0_attention_weight_min": -14.126007080078125,
      "activations/layer10_attention_weight_max": 38.9708366394043,
      "activations/layer10_attention_weight_min": -37.17816925048828,
      "activations/layer11_attention_weight_max": 33.87147521972656,
      "activations/layer11_attention_weight_min": -34.779640197753906,
      "activations/layer12_attention_weight_max": 17.739368438720703,
      "activations/layer12_attention_weight_min": -26.7142276763916,
      "activations/layer13_attention_weight_max": 35.26793670654297,
      "activations/layer13_attention_weight_min": -27.45999526977539,
      "activations/layer14_attention_weight_max": 36.69426345825195,
      "activations/layer14_attention_weight_min": -32.728416442871094,
      "activations/layer15_attention_weight_max": 34.5987434387207,
      "activations/layer15_attention_weight_min": -32.41970443725586,
      "activations/layer16_attention_weight_max": 35.10025405883789,
      "activations/layer16_attention_weight_min": -35.596614837646484,
      "activations/layer17_attention_weight_max": 55.978302001953125,
      "activations/layer17_attention_weight_min": -51.124244689941406,
      "activations/layer18_attention_weight_max": 48.3037223815918,
      "activations/layer18_attention_weight_min": -41.54471206665039,
      "activations/layer19_attention_weight_max": 23.77663230895996,
      "activations/layer19_attention_weight_min": -26.825117111206055,
      "activations/layer1_attention_weight_max": 17.62708282470703,
      "activations/layer1_attention_weight_min": -15.941596984863281,
      "activations/layer20_attention_weight_max": 22.17070198059082,
      "activations/layer20_attention_weight_min": -22.861879348754883,
      "activations/layer21_attention_weight_max": 37.901065826416016,
      "activations/layer21_attention_weight_min": -25.208812713623047,
      "activations/layer22_attention_weight_max": 28.181074142456055,
      "activations/layer22_attention_weight_min": -27.526018142700195,
      "activations/layer23_attention_weight_max": 37.378089904785156,
      "activations/layer23_attention_weight_min": -25.137290954589844,
      "activations/layer2_attention_weight_max": 32.42305374145508,
      "activations/layer2_attention_weight_min": -31.018760681152344,
      "activations/layer3_attention_weight_max": 97.61805725097656,
      "activations/layer3_attention_weight_min": -100.46104431152344,
      "activations/layer4_attention_weight_max": 99.13773345947266,
      "activations/layer4_attention_weight_min": -95.61958312988281,
      "activations/layer5_attention_weight_max": 71.55894470214844,
      "activations/layer5_attention_weight_min": -80.39775085449219,
      "activations/layer6_attention_weight_max": 53.409305572509766,
      "activations/layer6_attention_weight_min": -50.42668914794922,
      "activations/layer7_attention_weight_max": 68.06964111328125,
      "activations/layer7_attention_weight_min": -63.82759475708008,
      "activations/layer8_attention_weight_max": 45.10580062866211,
      "activations/layer8_attention_weight_min": -46.2827033996582,
      "activations/layer9_attention_weight_max": 46.75177001953125,
      "activations/layer9_attention_weight_min": -46.464778900146484,
      "epoch": 13.98,
      "learning_rate": 6.047386363636363e-05,
      "loss": 2.7564,
      "step": 240550
    },
    {
      "activations/layer0_attention_weight_max": 17.198135375976562,
      "activations/layer0_attention_weight_min": -14.060073852539062,
      "activations/layer10_attention_weight_max": 35.61346435546875,
      "activations/layer10_attention_weight_min": -35.50951385498047,
      "activations/layer11_attention_weight_max": 32.59226989746094,
      "activations/layer11_attention_weight_min": -34.34736633300781,
      "activations/layer12_attention_weight_max": 19.689390182495117,
      "activations/layer12_attention_weight_min": -25.237899780273438,
      "activations/layer13_attention_weight_max": 35.03046798706055,
      "activations/layer13_attention_weight_min": -29.196441650390625,
      "activations/layer14_attention_weight_max": 37.607933044433594,
      "activations/layer14_attention_weight_min": -31.77487564086914,
      "activations/layer15_attention_weight_max": 36.7218017578125,
      "activations/layer15_attention_weight_min": -30.317245483398438,
      "activations/layer16_attention_weight_max": 34.57480239868164,
      "activations/layer16_attention_weight_min": -31.773828506469727,
      "activations/layer17_attention_weight_max": 51.376495361328125,
      "activations/layer17_attention_weight_min": -47.108577728271484,
      "activations/layer18_attention_weight_max": 47.07377243041992,
      "activations/layer18_attention_weight_min": -39.703704833984375,
      "activations/layer19_attention_weight_max": 23.72049331665039,
      "activations/layer19_attention_weight_min": -23.699462890625,
      "activations/layer1_attention_weight_max": 16.331260681152344,
      "activations/layer1_attention_weight_min": -16.860618591308594,
      "activations/layer20_attention_weight_max": 23.302766799926758,
      "activations/layer20_attention_weight_min": -22.140727996826172,
      "activations/layer21_attention_weight_max": 34.85212707519531,
      "activations/layer21_attention_weight_min": -22.979827880859375,
      "activations/layer22_attention_weight_max": 29.91507911682129,
      "activations/layer22_attention_weight_min": -29.103445053100586,
      "activations/layer23_attention_weight_max": 36.87670135498047,
      "activations/layer23_attention_weight_min": -24.963176727294922,
      "activations/layer2_attention_weight_max": 32.91342544555664,
      "activations/layer2_attention_weight_min": -30.847625732421875,
      "activations/layer3_attention_weight_max": 99.11852264404297,
      "activations/layer3_attention_weight_min": -99.91704559326172,
      "activations/layer4_attention_weight_max": 97.95512390136719,
      "activations/layer4_attention_weight_min": -93.3991928100586,
      "activations/layer5_attention_weight_max": 75.95968627929688,
      "activations/layer5_attention_weight_min": -77.05593872070312,
      "activations/layer6_attention_weight_max": 54.19124984741211,
      "activations/layer6_attention_weight_min": -53.08036804199219,
      "activations/layer7_attention_weight_max": 70.21487426757812,
      "activations/layer7_attention_weight_min": -66.87008666992188,
      "activations/layer8_attention_weight_max": 46.6469841003418,
      "activations/layer8_attention_weight_min": -50.694915771484375,
      "activations/layer9_attention_weight_max": 43.29059982299805,
      "activations/layer9_attention_weight_min": -46.900272369384766,
      "epoch": 13.98,
      "learning_rate": 6.045492424242423e-05,
      "loss": 2.7662,
      "step": 240600
    },
    {
      "activations/layer0_attention_weight_max": 16.209014892578125,
      "activations/layer0_attention_weight_min": -14.132665634155273,
      "activations/layer10_attention_weight_max": 35.88047790527344,
      "activations/layer10_attention_weight_min": -35.35044860839844,
      "activations/layer11_attention_weight_max": 31.0179500579834,
      "activations/layer11_attention_weight_min": -33.407711029052734,
      "activations/layer12_attention_weight_max": 19.017311096191406,
      "activations/layer12_attention_weight_min": -31.863792419433594,
      "activations/layer13_attention_weight_max": 35.85420227050781,
      "activations/layer13_attention_weight_min": -27.081214904785156,
      "activations/layer14_attention_weight_max": 35.293670654296875,
      "activations/layer14_attention_weight_min": -30.530397415161133,
      "activations/layer15_attention_weight_max": 33.775901794433594,
      "activations/layer15_attention_weight_min": -30.557960510253906,
      "activations/layer16_attention_weight_max": 31.846330642700195,
      "activations/layer16_attention_weight_min": -30.96225929260254,
      "activations/layer17_attention_weight_max": 51.85440444946289,
      "activations/layer17_attention_weight_min": -46.070247650146484,
      "activations/layer18_attention_weight_max": 47.429229736328125,
      "activations/layer18_attention_weight_min": -40.053443908691406,
      "activations/layer19_attention_weight_max": 23.432249069213867,
      "activations/layer19_attention_weight_min": -21.776033401489258,
      "activations/layer1_attention_weight_max": 17.067855834960938,
      "activations/layer1_attention_weight_min": -16.490131378173828,
      "activations/layer20_attention_weight_max": 21.91295623779297,
      "activations/layer20_attention_weight_min": -20.849090576171875,
      "activations/layer21_attention_weight_max": 38.467681884765625,
      "activations/layer21_attention_weight_min": -26.64188575744629,
      "activations/layer22_attention_weight_max": 31.29047966003418,
      "activations/layer22_attention_weight_min": -25.01885986328125,
      "activations/layer23_attention_weight_max": 39.25325012207031,
      "activations/layer23_attention_weight_min": -24.126270294189453,
      "activations/layer2_attention_weight_max": 33.7283821105957,
      "activations/layer2_attention_weight_min": -29.64505386352539,
      "activations/layer3_attention_weight_max": 98.2066879272461,
      "activations/layer3_attention_weight_min": -96.01751708984375,
      "activations/layer4_attention_weight_max": 97.82111358642578,
      "activations/layer4_attention_weight_min": -94.9011459350586,
      "activations/layer5_attention_weight_max": 71.42481994628906,
      "activations/layer5_attention_weight_min": -77.83782958984375,
      "activations/layer6_attention_weight_max": 52.779624938964844,
      "activations/layer6_attention_weight_min": -57.19853591918945,
      "activations/layer7_attention_weight_max": 69.78520965576172,
      "activations/layer7_attention_weight_min": -66.4347152709961,
      "activations/layer8_attention_weight_max": 45.9813346862793,
      "activations/layer8_attention_weight_min": -48.42145538330078,
      "activations/layer9_attention_weight_max": 44.562007904052734,
      "activations/layer9_attention_weight_min": -47.67441177368164,
      "epoch": 13.98,
      "learning_rate": 6.043598484848484e-05,
      "loss": 2.7696,
      "step": 240650
    },
    {
      "activations/layer0_attention_weight_max": 16.43245506286621,
      "activations/layer0_attention_weight_min": -14.106264114379883,
      "activations/layer10_attention_weight_max": 34.39067077636719,
      "activations/layer10_attention_weight_min": -36.18994140625,
      "activations/layer11_attention_weight_max": 31.696102142333984,
      "activations/layer11_attention_weight_min": -33.9876594543457,
      "activations/layer12_attention_weight_max": 17.94374656677246,
      "activations/layer12_attention_weight_min": -23.032913208007812,
      "activations/layer13_attention_weight_max": 35.85480499267578,
      "activations/layer13_attention_weight_min": -27.402278900146484,
      "activations/layer14_attention_weight_max": 35.58815383911133,
      "activations/layer14_attention_weight_min": -29.918664932250977,
      "activations/layer15_attention_weight_max": 30.544836044311523,
      "activations/layer15_attention_weight_min": -30.308372497558594,
      "activations/layer16_attention_weight_max": 29.8697509765625,
      "activations/layer16_attention_weight_min": -28.5314884185791,
      "activations/layer17_attention_weight_max": 51.092288970947266,
      "activations/layer17_attention_weight_min": -42.653106689453125,
      "activations/layer18_attention_weight_max": 44.56489944458008,
      "activations/layer18_attention_weight_min": -38.216983795166016,
      "activations/layer19_attention_weight_max": 23.41822624206543,
      "activations/layer19_attention_weight_min": -21.51448631286621,
      "activations/layer1_attention_weight_max": 18.678871154785156,
      "activations/layer1_attention_weight_min": -14.929163932800293,
      "activations/layer20_attention_weight_max": 20.786893844604492,
      "activations/layer20_attention_weight_min": -21.106462478637695,
      "activations/layer21_attention_weight_max": 31.794170379638672,
      "activations/layer21_attention_weight_min": -22.08220100402832,
      "activations/layer22_attention_weight_max": 31.033363342285156,
      "activations/layer22_attention_weight_min": -27.41895866394043,
      "activations/layer23_attention_weight_max": 35.39763641357422,
      "activations/layer23_attention_weight_min": -24.58977508544922,
      "activations/layer2_attention_weight_max": 32.733856201171875,
      "activations/layer2_attention_weight_min": -30.710065841674805,
      "activations/layer3_attention_weight_max": 96.27704620361328,
      "activations/layer3_attention_weight_min": -93.9114761352539,
      "activations/layer4_attention_weight_max": 94.37336730957031,
      "activations/layer4_attention_weight_min": -90.19034576416016,
      "activations/layer5_attention_weight_max": 71.33812713623047,
      "activations/layer5_attention_weight_min": -77.91704559326172,
      "activations/layer6_attention_weight_max": 54.49837112426758,
      "activations/layer6_attention_weight_min": -52.744300842285156,
      "activations/layer7_attention_weight_max": 64.58051300048828,
      "activations/layer7_attention_weight_min": -64.58012390136719,
      "activations/layer8_attention_weight_max": 44.16868209838867,
      "activations/layer8_attention_weight_min": -48.4387092590332,
      "activations/layer9_attention_weight_max": 44.9845085144043,
      "activations/layer9_attention_weight_min": -44.002933502197266,
      "epoch": 13.99,
      "learning_rate": 6.041704545454545e-05,
      "loss": 2.7757,
      "step": 240700
    },
    {
      "activations/layer0_attention_weight_max": 16.083498001098633,
      "activations/layer0_attention_weight_min": -13.91818618774414,
      "activations/layer10_attention_weight_max": 32.02812194824219,
      "activations/layer10_attention_weight_min": -32.98635482788086,
      "activations/layer11_attention_weight_max": 31.247821807861328,
      "activations/layer11_attention_weight_min": -30.912330627441406,
      "activations/layer12_attention_weight_max": 18.823026657104492,
      "activations/layer12_attention_weight_min": -25.054304122924805,
      "activations/layer13_attention_weight_max": 34.197349548339844,
      "activations/layer13_attention_weight_min": -28.609989166259766,
      "activations/layer14_attention_weight_max": 37.06206130981445,
      "activations/layer14_attention_weight_min": -29.456079483032227,
      "activations/layer15_attention_weight_max": 32.376914978027344,
      "activations/layer15_attention_weight_min": -30.110990524291992,
      "activations/layer16_attention_weight_max": 30.894603729248047,
      "activations/layer16_attention_weight_min": -29.183746337890625,
      "activations/layer17_attention_weight_max": 51.83737564086914,
      "activations/layer17_attention_weight_min": -42.69234085083008,
      "activations/layer18_attention_weight_max": 47.65260696411133,
      "activations/layer18_attention_weight_min": -37.237274169921875,
      "activations/layer19_attention_weight_max": 22.093183517456055,
      "activations/layer19_attention_weight_min": -22.466821670532227,
      "activations/layer1_attention_weight_max": 15.849076271057129,
      "activations/layer1_attention_weight_min": -16.744972229003906,
      "activations/layer20_attention_weight_max": 24.63274574279785,
      "activations/layer20_attention_weight_min": -21.915565490722656,
      "activations/layer21_attention_weight_max": 33.87510681152344,
      "activations/layer21_attention_weight_min": -19.914377212524414,
      "activations/layer22_attention_weight_max": 30.10638427734375,
      "activations/layer22_attention_weight_min": -24.95613670349121,
      "activations/layer23_attention_weight_max": 36.458953857421875,
      "activations/layer23_attention_weight_min": -22.98123550415039,
      "activations/layer2_attention_weight_max": 31.441213607788086,
      "activations/layer2_attention_weight_min": -30.408788681030273,
      "activations/layer3_attention_weight_max": 90.39543151855469,
      "activations/layer3_attention_weight_min": -94.17133331298828,
      "activations/layer4_attention_weight_max": 86.8431625366211,
      "activations/layer4_attention_weight_min": -91.44496154785156,
      "activations/layer5_attention_weight_max": 66.72454833984375,
      "activations/layer5_attention_weight_min": -78.26271057128906,
      "activations/layer6_attention_weight_max": 48.25132369995117,
      "activations/layer6_attention_weight_min": -49.56550598144531,
      "activations/layer7_attention_weight_max": 64.64998626708984,
      "activations/layer7_attention_weight_min": -64.44473266601562,
      "activations/layer8_attention_weight_max": 42.84657669067383,
      "activations/layer8_attention_weight_min": -44.95246505737305,
      "activations/layer9_attention_weight_max": 42.33803176879883,
      "activations/layer9_attention_weight_min": -42.611534118652344,
      "epoch": 13.99,
      "learning_rate": 6.039810606060606e-05,
      "loss": 2.7483,
      "step": 240750
    },
    {
      "activations/layer0_attention_weight_max": 16.288230895996094,
      "activations/layer0_attention_weight_min": -14.679924011230469,
      "activations/layer10_attention_weight_max": 35.464237213134766,
      "activations/layer10_attention_weight_min": -36.58782958984375,
      "activations/layer11_attention_weight_max": 31.690793991088867,
      "activations/layer11_attention_weight_min": -32.53662872314453,
      "activations/layer12_attention_weight_max": 16.825519561767578,
      "activations/layer12_attention_weight_min": -22.484420776367188,
      "activations/layer13_attention_weight_max": 31.211589813232422,
      "activations/layer13_attention_weight_min": -33.129058837890625,
      "activations/layer14_attention_weight_max": 32.50916290283203,
      "activations/layer14_attention_weight_min": -29.015832901000977,
      "activations/layer15_attention_weight_max": 30.07016944885254,
      "activations/layer15_attention_weight_min": -29.731538772583008,
      "activations/layer16_attention_weight_max": 28.40407371520996,
      "activations/layer16_attention_weight_min": -29.58612632751465,
      "activations/layer17_attention_weight_max": 48.451419830322266,
      "activations/layer17_attention_weight_min": -42.205806732177734,
      "activations/layer18_attention_weight_max": 44.1273193359375,
      "activations/layer18_attention_weight_min": -38.051788330078125,
      "activations/layer19_attention_weight_max": 21.895177841186523,
      "activations/layer19_attention_weight_min": -22.291288375854492,
      "activations/layer1_attention_weight_max": 15.362181663513184,
      "activations/layer1_attention_weight_min": -14.629255294799805,
      "activations/layer20_attention_weight_max": 21.589323043823242,
      "activations/layer20_attention_weight_min": -20.776517868041992,
      "activations/layer21_attention_weight_max": 35.788734436035156,
      "activations/layer21_attention_weight_min": -23.117551803588867,
      "activations/layer22_attention_weight_max": 29.07286834716797,
      "activations/layer22_attention_weight_min": -25.91372299194336,
      "activations/layer23_attention_weight_max": 35.086395263671875,
      "activations/layer23_attention_weight_min": -24.19025993347168,
      "activations/layer2_attention_weight_max": 31.799665451049805,
      "activations/layer2_attention_weight_min": -32.372859954833984,
      "activations/layer3_attention_weight_max": 94.7342758178711,
      "activations/layer3_attention_weight_min": -99.42926788330078,
      "activations/layer4_attention_weight_max": 94.8822250366211,
      "activations/layer4_attention_weight_min": -91.8862533569336,
      "activations/layer5_attention_weight_max": 72.70401000976562,
      "activations/layer5_attention_weight_min": -82.70796203613281,
      "activations/layer6_attention_weight_max": 53.61325454711914,
      "activations/layer6_attention_weight_min": -53.72367858886719,
      "activations/layer7_attention_weight_max": 67.89905548095703,
      "activations/layer7_attention_weight_min": -64.79900360107422,
      "activations/layer8_attention_weight_max": 45.25069046020508,
      "activations/layer8_attention_weight_min": -48.95900344848633,
      "activations/layer9_attention_weight_max": 49.40827178955078,
      "activations/layer9_attention_weight_min": -45.4589729309082,
      "epoch": 13.99,
      "learning_rate": 6.037916666666666e-05,
      "loss": 2.7732,
      "step": 240800
    },
    {
      "activations/layer0_attention_weight_max": 15.640571594238281,
      "activations/layer0_attention_weight_min": -14.276899337768555,
      "activations/layer10_attention_weight_max": 35.8282470703125,
      "activations/layer10_attention_weight_min": -36.829261779785156,
      "activations/layer11_attention_weight_max": 33.302894592285156,
      "activations/layer11_attention_weight_min": -33.4427604675293,
      "activations/layer12_attention_weight_max": 17.82107162475586,
      "activations/layer12_attention_weight_min": -27.076940536499023,
      "activations/layer13_attention_weight_max": 36.9780387878418,
      "activations/layer13_attention_weight_min": -30.34844970703125,
      "activations/layer14_attention_weight_max": 36.38758087158203,
      "activations/layer14_attention_weight_min": -30.47066307067871,
      "activations/layer15_attention_weight_max": 32.72923278808594,
      "activations/layer15_attention_weight_min": -30.498247146606445,
      "activations/layer16_attention_weight_max": 32.49749755859375,
      "activations/layer16_attention_weight_min": -30.798885345458984,
      "activations/layer17_attention_weight_max": 51.40857696533203,
      "activations/layer17_attention_weight_min": -43.98411560058594,
      "activations/layer18_attention_weight_max": 46.32744598388672,
      "activations/layer18_attention_weight_min": -38.3059196472168,
      "activations/layer19_attention_weight_max": 25.69091796875,
      "activations/layer19_attention_weight_min": -21.69681167602539,
      "activations/layer1_attention_weight_max": 16.156360626220703,
      "activations/layer1_attention_weight_min": -16.43077278137207,
      "activations/layer20_attention_weight_max": 21.951175689697266,
      "activations/layer20_attention_weight_min": -21.051450729370117,
      "activations/layer21_attention_weight_max": 35.564239501953125,
      "activations/layer21_attention_weight_min": -24.84795570373535,
      "activations/layer22_attention_weight_max": 29.988985061645508,
      "activations/layer22_attention_weight_min": -25.946821212768555,
      "activations/layer23_attention_weight_max": 36.19810485839844,
      "activations/layer23_attention_weight_min": -23.396121978759766,
      "activations/layer2_attention_weight_max": 32.98922348022461,
      "activations/layer2_attention_weight_min": -31.214332580566406,
      "activations/layer3_attention_weight_max": 99.74620819091797,
      "activations/layer3_attention_weight_min": -96.95475006103516,
      "activations/layer4_attention_weight_max": 98.40813446044922,
      "activations/layer4_attention_weight_min": -99.31417083740234,
      "activations/layer5_attention_weight_max": 75.07827758789062,
      "activations/layer5_attention_weight_min": -83.7314453125,
      "activations/layer6_attention_weight_max": 55.13483428955078,
      "activations/layer6_attention_weight_min": -57.15153121948242,
      "activations/layer7_attention_weight_max": 68.95748138427734,
      "activations/layer7_attention_weight_min": -67.01187896728516,
      "activations/layer8_attention_weight_max": 45.56916809082031,
      "activations/layer8_attention_weight_min": -47.91583251953125,
      "activations/layer9_attention_weight_max": 46.043575286865234,
      "activations/layer9_attention_weight_min": -46.84707260131836,
      "epoch": 13.99,
      "learning_rate": 6.0360227272727274e-05,
      "loss": 2.766,
      "step": 240850
    },
    {
      "activations/layer0_attention_weight_max": 17.05462074279785,
      "activations/layer0_attention_weight_min": -14.433145523071289,
      "activations/layer10_attention_weight_max": 36.82705307006836,
      "activations/layer10_attention_weight_min": -35.83565139770508,
      "activations/layer11_attention_weight_max": 31.716629028320312,
      "activations/layer11_attention_weight_min": -34.8161506652832,
      "activations/layer12_attention_weight_max": 20.085693359375,
      "activations/layer12_attention_weight_min": -25.096885681152344,
      "activations/layer13_attention_weight_max": 36.76897048950195,
      "activations/layer13_attention_weight_min": -28.76085662841797,
      "activations/layer14_attention_weight_max": 35.40315628051758,
      "activations/layer14_attention_weight_min": -31.189035415649414,
      "activations/layer15_attention_weight_max": 33.417022705078125,
      "activations/layer15_attention_weight_min": -30.97926139831543,
      "activations/layer16_attention_weight_max": 31.52637481689453,
      "activations/layer16_attention_weight_min": -30.124799728393555,
      "activations/layer17_attention_weight_max": 50.904701232910156,
      "activations/layer17_attention_weight_min": -43.28516387939453,
      "activations/layer18_attention_weight_max": 48.02344512939453,
      "activations/layer18_attention_weight_min": -41.38346481323242,
      "activations/layer19_attention_weight_max": 23.893787384033203,
      "activations/layer19_attention_weight_min": -23.887609481811523,
      "activations/layer1_attention_weight_max": 16.451000213623047,
      "activations/layer1_attention_weight_min": -15.319847106933594,
      "activations/layer20_attention_weight_max": 22.56063461303711,
      "activations/layer20_attention_weight_min": -22.70975685119629,
      "activations/layer21_attention_weight_max": 35.68024444580078,
      "activations/layer21_attention_weight_min": -21.942333221435547,
      "activations/layer22_attention_weight_max": 29.018526077270508,
      "activations/layer22_attention_weight_min": -26.097070693969727,
      "activations/layer23_attention_weight_max": 37.35221862792969,
      "activations/layer23_attention_weight_min": -23.030044555664062,
      "activations/layer2_attention_weight_max": 33.96800231933594,
      "activations/layer2_attention_weight_min": -32.13349533081055,
      "activations/layer3_attention_weight_max": 101.22473907470703,
      "activations/layer3_attention_weight_min": -99.77072143554688,
      "activations/layer4_attention_weight_max": 103.23394775390625,
      "activations/layer4_attention_weight_min": -93.92845153808594,
      "activations/layer5_attention_weight_max": 73.76914978027344,
      "activations/layer5_attention_weight_min": -81.90937805175781,
      "activations/layer6_attention_weight_max": 53.4244499206543,
      "activations/layer6_attention_weight_min": -54.4079704284668,
      "activations/layer7_attention_weight_max": 68.16748046875,
      "activations/layer7_attention_weight_min": -63.33555221557617,
      "activations/layer8_attention_weight_max": 46.68478775024414,
      "activations/layer8_attention_weight_min": -48.832313537597656,
      "activations/layer9_attention_weight_max": 45.88523483276367,
      "activations/layer9_attention_weight_min": -46.18367385864258,
      "epoch": 14.0,
      "learning_rate": 6.0341287878787876e-05,
      "loss": 2.7571,
      "step": 240900
    },
    {
      "activations/layer0_attention_weight_max": 15.559391975402832,
      "activations/layer0_attention_weight_min": -14.19907283782959,
      "activations/layer10_attention_weight_max": 39.389888763427734,
      "activations/layer10_attention_weight_min": -39.623634338378906,
      "activations/layer11_attention_weight_max": 33.425575256347656,
      "activations/layer11_attention_weight_min": -37.26875305175781,
      "activations/layer12_attention_weight_max": 20.833871841430664,
      "activations/layer12_attention_weight_min": -27.68964385986328,
      "activations/layer13_attention_weight_max": 39.77753448486328,
      "activations/layer13_attention_weight_min": -30.00281524658203,
      "activations/layer14_attention_weight_max": 36.243709564208984,
      "activations/layer14_attention_weight_min": -30.734052658081055,
      "activations/layer15_attention_weight_max": 34.528236389160156,
      "activations/layer15_attention_weight_min": -29.739906311035156,
      "activations/layer16_attention_weight_max": 32.9124870300293,
      "activations/layer16_attention_weight_min": -29.274900436401367,
      "activations/layer17_attention_weight_max": 54.8311767578125,
      "activations/layer17_attention_weight_min": -45.2559700012207,
      "activations/layer18_attention_weight_max": 49.955177307128906,
      "activations/layer18_attention_weight_min": -40.00749206542969,
      "activations/layer19_attention_weight_max": 21.508392333984375,
      "activations/layer19_attention_weight_min": -22.501976013183594,
      "activations/layer1_attention_weight_max": 16.706960678100586,
      "activations/layer1_attention_weight_min": -14.398093223571777,
      "activations/layer20_attention_weight_max": 23.278940200805664,
      "activations/layer20_attention_weight_min": -22.272127151489258,
      "activations/layer21_attention_weight_max": 39.384803771972656,
      "activations/layer21_attention_weight_min": -23.7550048828125,
      "activations/layer22_attention_weight_max": 28.474458694458008,
      "activations/layer22_attention_weight_min": -26.168725967407227,
      "activations/layer23_attention_weight_max": 38.415401458740234,
      "activations/layer23_attention_weight_min": -25.358840942382812,
      "activations/layer2_attention_weight_max": 33.56602478027344,
      "activations/layer2_attention_weight_min": -31.84328269958496,
      "activations/layer3_attention_weight_max": 100.5350341796875,
      "activations/layer3_attention_weight_min": -96.12555694580078,
      "activations/layer4_attention_weight_max": 98.73543548583984,
      "activations/layer4_attention_weight_min": -92.87507629394531,
      "activations/layer5_attention_weight_max": 76.10045623779297,
      "activations/layer5_attention_weight_min": -80.89627838134766,
      "activations/layer6_attention_weight_max": 53.69795227050781,
      "activations/layer6_attention_weight_min": -54.05620193481445,
      "activations/layer7_attention_weight_max": 67.91490936279297,
      "activations/layer7_attention_weight_min": -64.88294982910156,
      "activations/layer8_attention_weight_max": 46.395633697509766,
      "activations/layer8_attention_weight_min": -50.62088394165039,
      "activations/layer9_attention_weight_max": 51.017333984375,
      "activations/layer9_attention_weight_min": -46.80366516113281,
      "epoch": 14.0,
      "learning_rate": 6.032196969696969e-05,
      "loss": 2.8263,
      "step": 240950
    },
    {
      "activations/layer0_attention_weight_max": 16.972335815429688,
      "activations/layer0_attention_weight_min": -13.895759582519531,
      "activations/layer10_attention_weight_max": 38.00838088989258,
      "activations/layer10_attention_weight_min": -33.94562530517578,
      "activations/layer11_attention_weight_max": 32.80027770996094,
      "activations/layer11_attention_weight_min": -31.799053192138672,
      "activations/layer12_attention_weight_max": 18.130247116088867,
      "activations/layer12_attention_weight_min": -22.530012130737305,
      "activations/layer13_attention_weight_max": 36.48186111450195,
      "activations/layer13_attention_weight_min": -28.261384963989258,
      "activations/layer14_attention_weight_max": 35.24909210205078,
      "activations/layer14_attention_weight_min": -29.37080955505371,
      "activations/layer15_attention_weight_max": 29.37848472595215,
      "activations/layer15_attention_weight_min": -28.736774444580078,
      "activations/layer16_attention_weight_max": 30.540897369384766,
      "activations/layer16_attention_weight_min": -29.408239364624023,
      "activations/layer17_attention_weight_max": 47.81476974487305,
      "activations/layer17_attention_weight_min": -43.230770111083984,
      "activations/layer18_attention_weight_max": 43.845542907714844,
      "activations/layer18_attention_weight_min": -35.197021484375,
      "activations/layer19_attention_weight_max": 22.96204948425293,
      "activations/layer19_attention_weight_min": -21.29639434814453,
      "activations/layer1_attention_weight_max": 16.046751022338867,
      "activations/layer1_attention_weight_min": -16.40648651123047,
      "activations/layer20_attention_weight_max": 21.394102096557617,
      "activations/layer20_attention_weight_min": -25.20502281188965,
      "activations/layer21_attention_weight_max": 36.440773010253906,
      "activations/layer21_attention_weight_min": -24.289865493774414,
      "activations/layer22_attention_weight_max": 28.389083862304688,
      "activations/layer22_attention_weight_min": -24.415138244628906,
      "activations/layer23_attention_weight_max": 38.85438537597656,
      "activations/layer23_attention_weight_min": -25.108449935913086,
      "activations/layer2_attention_weight_max": 32.378990173339844,
      "activations/layer2_attention_weight_min": -29.88864517211914,
      "activations/layer3_attention_weight_max": 97.36722564697266,
      "activations/layer3_attention_weight_min": -98.0101089477539,
      "activations/layer4_attention_weight_max": 93.48213958740234,
      "activations/layer4_attention_weight_min": -92.29325103759766,
      "activations/layer5_attention_weight_max": 68.7657470703125,
      "activations/layer5_attention_weight_min": -77.8079833984375,
      "activations/layer6_attention_weight_max": 52.17194747924805,
      "activations/layer6_attention_weight_min": -52.54375457763672,
      "activations/layer7_attention_weight_max": 65.49364471435547,
      "activations/layer7_attention_weight_min": -65.9767074584961,
      "activations/layer8_attention_weight_max": 45.796478271484375,
      "activations/layer8_attention_weight_min": -47.78752136230469,
      "activations/layer9_attention_weight_max": 45.989356994628906,
      "activations/layer9_attention_weight_min": -45.1501350402832,
      "epoch": 14.0,
      "learning_rate": 6.0303030303030296e-05,
      "loss": 2.7538,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.5438,
      "eval_samples_per_second": 502.589,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.5438,
      "eval_openwebtext_samples_per_second": 502.589,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_wikitext_loss": 2.94921875,
      "eval_wikitext_ppl": 19.091033031037714,
      "eval_wikitext_runtime": 2.0235,
      "eval_wikitext_samples_per_second": 225.358,
      "step": 241000
    },
    {
      "epoch": 14.0,
      "eval_lambada_loss": 2.466796875,
      "eval_lambada_ppl": 11.784638661080912,
      "eval_lambada_runtime": 9.6374,
      "eval_lambada_samples_per_second": 505.221,
      "step": 241000
    },
    {
      "activations/layer0_attention_weight_max": 16.72932243347168,
      "activations/layer0_attention_weight_min": -14.11097240447998,
      "activations/layer10_attention_weight_max": 42.11163330078125,
      "activations/layer10_attention_weight_min": -41.28660583496094,
      "activations/layer11_attention_weight_max": 38.538570404052734,
      "activations/layer11_attention_weight_min": -36.54436492919922,
      "activations/layer12_attention_weight_max": 19.528820037841797,
      "activations/layer12_attention_weight_min": -23.003726959228516,
      "activations/layer13_attention_weight_max": 34.93061447143555,
      "activations/layer13_attention_weight_min": -28.895221710205078,
      "activations/layer14_attention_weight_max": 35.352577209472656,
      "activations/layer14_attention_weight_min": -32.32003402709961,
      "activations/layer15_attention_weight_max": 36.27101516723633,
      "activations/layer15_attention_weight_min": -29.45606231689453,
      "activations/layer16_attention_weight_max": 34.21016311645508,
      "activations/layer16_attention_weight_min": -33.15797805786133,
      "activations/layer17_attention_weight_max": 52.967918395996094,
      "activations/layer17_attention_weight_min": -46.914085388183594,
      "activations/layer18_attention_weight_max": 52.34549331665039,
      "activations/layer18_attention_weight_min": -41.03411102294922,
      "activations/layer19_attention_weight_max": 24.09112548828125,
      "activations/layer19_attention_weight_min": -23.080320358276367,
      "activations/layer1_attention_weight_max": 16.658039093017578,
      "activations/layer1_attention_weight_min": -16.42078399658203,
      "activations/layer20_attention_weight_max": 25.256633758544922,
      "activations/layer20_attention_weight_min": -22.830629348754883,
      "activations/layer21_attention_weight_max": 38.81528854370117,
      "activations/layer21_attention_weight_min": -22.885181427001953,
      "activations/layer22_attention_weight_max": 31.784259796142578,
      "activations/layer22_attention_weight_min": -26.28829002380371,
      "activations/layer23_attention_weight_max": 39.46187210083008,
      "activations/layer23_attention_weight_min": -22.210704803466797,
      "activations/layer2_attention_weight_max": 31.710201263427734,
      "activations/layer2_attention_weight_min": -30.556386947631836,
      "activations/layer3_attention_weight_max": 96.54031372070312,
      "activations/layer3_attention_weight_min": -100.02332305908203,
      "activations/layer4_attention_weight_max": 99.8311996459961,
      "activations/layer4_attention_weight_min": -100.0179443359375,
      "activations/layer5_attention_weight_max": 75.48324584960938,
      "activations/layer5_attention_weight_min": -83.64349365234375,
      "activations/layer6_attention_weight_max": 54.89695739746094,
      "activations/layer6_attention_weight_min": -56.84490966796875,
      "activations/layer7_attention_weight_max": 72.93439483642578,
      "activations/layer7_attention_weight_min": -69.99816131591797,
      "activations/layer8_attention_weight_max": 51.35866928100586,
      "activations/layer8_attention_weight_min": -55.48731231689453,
      "activations/layer9_attention_weight_max": 48.26493835449219,
      "activations/layer9_attention_weight_min": -49.595703125,
      "epoch": 14.01,
      "learning_rate": 6.0284090909090905e-05,
      "loss": 2.7492,
      "step": 241050
    },
    {
      "activations/layer0_attention_weight_max": 15.953483581542969,
      "activations/layer0_attention_weight_min": -14.477042198181152,
      "activations/layer10_attention_weight_max": 35.71690368652344,
      "activations/layer10_attention_weight_min": -33.95439910888672,
      "activations/layer11_attention_weight_max": 31.870046615600586,
      "activations/layer11_attention_weight_min": -32.94451904296875,
      "activations/layer12_attention_weight_max": 19.678821563720703,
      "activations/layer12_attention_weight_min": -22.4503116607666,
      "activations/layer13_attention_weight_max": 37.26408767700195,
      "activations/layer13_attention_weight_min": -30.784154891967773,
      "activations/layer14_attention_weight_max": 35.83298110961914,
      "activations/layer14_attention_weight_min": -31.748584747314453,
      "activations/layer15_attention_weight_max": 32.60264205932617,
      "activations/layer15_attention_weight_min": -30.92375946044922,
      "activations/layer16_attention_weight_max": 31.63520050048828,
      "activations/layer16_attention_weight_min": -30.597742080688477,
      "activations/layer17_attention_weight_max": 54.252342224121094,
      "activations/layer17_attention_weight_min": -45.9474983215332,
      "activations/layer18_attention_weight_max": 47.12848663330078,
      "activations/layer18_attention_weight_min": -39.303340911865234,
      "activations/layer19_attention_weight_max": 23.466697692871094,
      "activations/layer19_attention_weight_min": -22.279245376586914,
      "activations/layer1_attention_weight_max": 17.110427856445312,
      "activations/layer1_attention_weight_min": -18.893266677856445,
      "activations/layer20_attention_weight_max": 22.837987899780273,
      "activations/layer20_attention_weight_min": -24.782798767089844,
      "activations/layer21_attention_weight_max": 36.95256042480469,
      "activations/layer21_attention_weight_min": -26.20992088317871,
      "activations/layer22_attention_weight_max": 30.943450927734375,
      "activations/layer22_attention_weight_min": -29.518327713012695,
      "activations/layer23_attention_weight_max": 38.502960205078125,
      "activations/layer23_attention_weight_min": -25.988126754760742,
      "activations/layer2_attention_weight_max": 32.59290313720703,
      "activations/layer2_attention_weight_min": -29.740211486816406,
      "activations/layer3_attention_weight_max": 95.609375,
      "activations/layer3_attention_weight_min": -97.07652282714844,
      "activations/layer4_attention_weight_max": 94.35961151123047,
      "activations/layer4_attention_weight_min": -94.69109344482422,
      "activations/layer5_attention_weight_max": 74.97200775146484,
      "activations/layer5_attention_weight_min": -85.05550384521484,
      "activations/layer6_attention_weight_max": 52.675575256347656,
      "activations/layer6_attention_weight_min": -53.93446731567383,
      "activations/layer7_attention_weight_max": 71.70880126953125,
      "activations/layer7_attention_weight_min": -69.71553802490234,
      "activations/layer8_attention_weight_max": 45.95988845825195,
      "activations/layer8_attention_weight_min": -46.325347900390625,
      "activations/layer9_attention_weight_max": 44.371788024902344,
      "activations/layer9_attention_weight_min": -46.25078201293945,
      "epoch": 14.01,
      "learning_rate": 6.026515151515151e-05,
      "loss": 2.7679,
      "step": 241100
    },
    {
      "activations/layer0_attention_weight_max": 15.815852165222168,
      "activations/layer0_attention_weight_min": -14.420071601867676,
      "activations/layer10_attention_weight_max": 37.48240280151367,
      "activations/layer10_attention_weight_min": -37.7542839050293,
      "activations/layer11_attention_weight_max": 33.179725646972656,
      "activations/layer11_attention_weight_min": -34.93324279785156,
      "activations/layer12_attention_weight_max": 19.298654556274414,
      "activations/layer12_attention_weight_min": -24.20453643798828,
      "activations/layer13_attention_weight_max": 35.814353942871094,
      "activations/layer13_attention_weight_min": -28.01085090637207,
      "activations/layer14_attention_weight_max": 37.935855865478516,
      "activations/layer14_attention_weight_min": -30.270872116088867,
      "activations/layer15_attention_weight_max": 33.61750793457031,
      "activations/layer15_attention_weight_min": -29.69635772705078,
      "activations/layer16_attention_weight_max": 30.239946365356445,
      "activations/layer16_attention_weight_min": -30.369096755981445,
      "activations/layer17_attention_weight_max": 49.726261138916016,
      "activations/layer17_attention_weight_min": -42.3226203918457,
      "activations/layer18_attention_weight_max": 44.069889068603516,
      "activations/layer18_attention_weight_min": -37.84034729003906,
      "activations/layer19_attention_weight_max": 21.044918060302734,
      "activations/layer19_attention_weight_min": -20.61233901977539,
      "activations/layer1_attention_weight_max": 17.087833404541016,
      "activations/layer1_attention_weight_min": -14.224821090698242,
      "activations/layer20_attention_weight_max": 23.13214874267578,
      "activations/layer20_attention_weight_min": -20.80076789855957,
      "activations/layer21_attention_weight_max": 38.107635498046875,
      "activations/layer21_attention_weight_min": -24.94182777404785,
      "activations/layer22_attention_weight_max": 28.313461303710938,
      "activations/layer22_attention_weight_min": -25.996959686279297,
      "activations/layer23_attention_weight_max": 39.0272102355957,
      "activations/layer23_attention_weight_min": -23.906423568725586,
      "activations/layer2_attention_weight_max": 31.435497283935547,
      "activations/layer2_attention_weight_min": -29.23109245300293,
      "activations/layer3_attention_weight_max": 95.46549224853516,
      "activations/layer3_attention_weight_min": -93.70823669433594,
      "activations/layer4_attention_weight_max": 98.2283935546875,
      "activations/layer4_attention_weight_min": -100.03008270263672,
      "activations/layer5_attention_weight_max": 72.39229583740234,
      "activations/layer5_attention_weight_min": -80.87055969238281,
      "activations/layer6_attention_weight_max": 54.03867721557617,
      "activations/layer6_attention_weight_min": -53.17687225341797,
      "activations/layer7_attention_weight_max": 63.550865173339844,
      "activations/layer7_attention_weight_min": -63.61594772338867,
      "activations/layer8_attention_weight_max": 47.34607696533203,
      "activations/layer8_attention_weight_min": -50.82979202270508,
      "activations/layer9_attention_weight_max": 47.80832290649414,
      "activations/layer9_attention_weight_min": -48.641334533691406,
      "epoch": 14.01,
      "learning_rate": 6.0246212121212114e-05,
      "loss": 2.7576,
      "step": 241150
    },
    {
      "activations/layer0_attention_weight_max": 15.640413284301758,
      "activations/layer0_attention_weight_min": -14.355901718139648,
      "activations/layer10_attention_weight_max": 35.35896682739258,
      "activations/layer10_attention_weight_min": -35.677860260009766,
      "activations/layer11_attention_weight_max": 31.365537643432617,
      "activations/layer11_attention_weight_min": -32.991424560546875,
      "activations/layer12_attention_weight_max": 20.572681427001953,
      "activations/layer12_attention_weight_min": -23.149677276611328,
      "activations/layer13_attention_weight_max": 34.06060791015625,
      "activations/layer13_attention_weight_min": -31.709674835205078,
      "activations/layer14_attention_weight_max": 35.51782989501953,
      "activations/layer14_attention_weight_min": -31.584997177124023,
      "activations/layer15_attention_weight_max": 32.53094482421875,
      "activations/layer15_attention_weight_min": -29.236194610595703,
      "activations/layer16_attention_weight_max": 32.9598274230957,
      "activations/layer16_attention_weight_min": -30.460140228271484,
      "activations/layer17_attention_weight_max": 54.534950256347656,
      "activations/layer17_attention_weight_min": -45.42060470581055,
      "activations/layer18_attention_weight_max": 48.4744758605957,
      "activations/layer18_attention_weight_min": -41.4401741027832,
      "activations/layer19_attention_weight_max": 22.919809341430664,
      "activations/layer19_attention_weight_min": -24.22402000427246,
      "activations/layer1_attention_weight_max": 16.29062843322754,
      "activations/layer1_attention_weight_min": -16.062524795532227,
      "activations/layer20_attention_weight_max": 23.13475227355957,
      "activations/layer20_attention_weight_min": -22.382871627807617,
      "activations/layer21_attention_weight_max": 41.20034408569336,
      "activations/layer21_attention_weight_min": -27.177391052246094,
      "activations/layer22_attention_weight_max": 31.69259262084961,
      "activations/layer22_attention_weight_min": -26.441810607910156,
      "activations/layer23_attention_weight_max": 35.886253356933594,
      "activations/layer23_attention_weight_min": -25.167699813842773,
      "activations/layer2_attention_weight_max": 33.820011138916016,
      "activations/layer2_attention_weight_min": -31.556989669799805,
      "activations/layer3_attention_weight_max": 95.94869995117188,
      "activations/layer3_attention_weight_min": -91.22431945800781,
      "activations/layer4_attention_weight_max": 95.9006576538086,
      "activations/layer4_attention_weight_min": -90.82042694091797,
      "activations/layer5_attention_weight_max": 71.21401977539062,
      "activations/layer5_attention_weight_min": -78.42484283447266,
      "activations/layer6_attention_weight_max": 51.68434143066406,
      "activations/layer6_attention_weight_min": -50.55659103393555,
      "activations/layer7_attention_weight_max": 68.97127532958984,
      "activations/layer7_attention_weight_min": -64.56507110595703,
      "activations/layer8_attention_weight_max": 46.15485763549805,
      "activations/layer8_attention_weight_min": -47.23579788208008,
      "activations/layer9_attention_weight_max": 53.44429397583008,
      "activations/layer9_attention_weight_min": -48.64130783081055,
      "epoch": 14.02,
      "learning_rate": 6.0227272727272716e-05,
      "loss": 2.7613,
      "step": 241200
    },
    {
      "activations/layer0_attention_weight_max": 16.263647079467773,
      "activations/layer0_attention_weight_min": -13.900134086608887,
      "activations/layer10_attention_weight_max": 37.62895202636719,
      "activations/layer10_attention_weight_min": -34.5262565612793,
      "activations/layer11_attention_weight_max": 31.095470428466797,
      "activations/layer11_attention_weight_min": -32.27723693847656,
      "activations/layer12_attention_weight_max": 18.42452049255371,
      "activations/layer12_attention_weight_min": -26.24412727355957,
      "activations/layer13_attention_weight_max": 54.018306732177734,
      "activations/layer13_attention_weight_min": -29.6732120513916,
      "activations/layer14_attention_weight_max": 45.50141143798828,
      "activations/layer14_attention_weight_min": -29.567121505737305,
      "activations/layer15_attention_weight_max": 40.89396286010742,
      "activations/layer15_attention_weight_min": -27.817296981811523,
      "activations/layer16_attention_weight_max": 34.23740005493164,
      "activations/layer16_attention_weight_min": -30.339370727539062,
      "activations/layer17_attention_weight_max": 53.66120910644531,
      "activations/layer17_attention_weight_min": -45.4195442199707,
      "activations/layer18_attention_weight_max": 50.27043914794922,
      "activations/layer18_attention_weight_min": -39.87986755371094,
      "activations/layer19_attention_weight_max": 21.591352462768555,
      "activations/layer19_attention_weight_min": -22.404451370239258,
      "activations/layer1_attention_weight_max": 17.228515625,
      "activations/layer1_attention_weight_min": -15.551630973815918,
      "activations/layer20_attention_weight_max": 23.97704315185547,
      "activations/layer20_attention_weight_min": -22.096515655517578,
      "activations/layer21_attention_weight_max": 42.47092819213867,
      "activations/layer21_attention_weight_min": -23.707780838012695,
      "activations/layer22_attention_weight_max": 32.30574035644531,
      "activations/layer22_attention_weight_min": -26.95636749267578,
      "activations/layer23_attention_weight_max": 37.59303283691406,
      "activations/layer23_attention_weight_min": -26.223798751831055,
      "activations/layer2_attention_weight_max": 32.34909439086914,
      "activations/layer2_attention_weight_min": -32.33494567871094,
      "activations/layer3_attention_weight_max": 92.50619506835938,
      "activations/layer3_attention_weight_min": -94.4052734375,
      "activations/layer4_attention_weight_max": 90.29571533203125,
      "activations/layer4_attention_weight_min": -89.78350067138672,
      "activations/layer5_attention_weight_max": 70.47666931152344,
      "activations/layer5_attention_weight_min": -76.3317642211914,
      "activations/layer6_attention_weight_max": 51.450096130371094,
      "activations/layer6_attention_weight_min": -50.11604309082031,
      "activations/layer7_attention_weight_max": 63.53050231933594,
      "activations/layer7_attention_weight_min": -63.05293273925781,
      "activations/layer8_attention_weight_max": 46.05769729614258,
      "activations/layer8_attention_weight_min": -50.29308319091797,
      "activations/layer9_attention_weight_max": 47.48990249633789,
      "activations/layer9_attention_weight_min": -44.547428131103516,
      "epoch": 14.02,
      "learning_rate": 6.020871212121212e-05,
      "loss": 2.7607,
      "step": 241250
    },
    {
      "activations/layer0_attention_weight_max": 15.833890914916992,
      "activations/layer0_attention_weight_min": -13.815147399902344,
      "activations/layer10_attention_weight_max": 34.66884231567383,
      "activations/layer10_attention_weight_min": -33.23545455932617,
      "activations/layer11_attention_weight_max": 30.970060348510742,
      "activations/layer11_attention_weight_min": -31.493452072143555,
      "activations/layer12_attention_weight_max": 19.32963752746582,
      "activations/layer12_attention_weight_min": -23.643508911132812,
      "activations/layer13_attention_weight_max": 43.87409973144531,
      "activations/layer13_attention_weight_min": -27.754302978515625,
      "activations/layer14_attention_weight_max": 36.47062683105469,
      "activations/layer14_attention_weight_min": -31.3525447845459,
      "activations/layer15_attention_weight_max": 36.19411849975586,
      "activations/layer15_attention_weight_min": -28.98256492614746,
      "activations/layer16_attention_weight_max": 35.607948303222656,
      "activations/layer16_attention_weight_min": -33.26539993286133,
      "activations/layer17_attention_weight_max": 55.04496765136719,
      "activations/layer17_attention_weight_min": -46.0019416809082,
      "activations/layer18_attention_weight_max": 51.19213104248047,
      "activations/layer18_attention_weight_min": -40.81827926635742,
      "activations/layer19_attention_weight_max": 25.197072982788086,
      "activations/layer19_attention_weight_min": -25.216650009155273,
      "activations/layer1_attention_weight_max": 16.550371170043945,
      "activations/layer1_attention_weight_min": -13.519319534301758,
      "activations/layer20_attention_weight_max": 27.551156997680664,
      "activations/layer20_attention_weight_min": -23.650760650634766,
      "activations/layer21_attention_weight_max": 43.82470703125,
      "activations/layer21_attention_weight_min": -20.60553550720215,
      "activations/layer22_attention_weight_max": 34.973148345947266,
      "activations/layer22_attention_weight_min": -25.84614372253418,
      "activations/layer23_attention_weight_max": 45.4219970703125,
      "activations/layer23_attention_weight_min": -25.76858139038086,
      "activations/layer2_attention_weight_max": 32.630821228027344,
      "activations/layer2_attention_weight_min": -30.305744171142578,
      "activations/layer3_attention_weight_max": 92.62907409667969,
      "activations/layer3_attention_weight_min": -90.98007202148438,
      "activations/layer4_attention_weight_max": 90.6837387084961,
      "activations/layer4_attention_weight_min": -90.10868835449219,
      "activations/layer5_attention_weight_max": 68.28498077392578,
      "activations/layer5_attention_weight_min": -82.5453872680664,
      "activations/layer6_attention_weight_max": 49.090858459472656,
      "activations/layer6_attention_weight_min": -47.64212417602539,
      "activations/layer7_attention_weight_max": 61.155513763427734,
      "activations/layer7_attention_weight_min": -59.2935791015625,
      "activations/layer8_attention_weight_max": 43.8583869934082,
      "activations/layer8_attention_weight_min": -44.214210510253906,
      "activations/layer9_attention_weight_max": 43.20512390136719,
      "activations/layer9_attention_weight_min": -42.59473419189453,
      "epoch": 14.02,
      "learning_rate": 6.019015151515151e-05,
      "loss": 2.756,
      "step": 241300
    },
    {
      "activations/layer0_attention_weight_max": 16.332170486450195,
      "activations/layer0_attention_weight_min": -14.41553783416748,
      "activations/layer10_attention_weight_max": 42.969242095947266,
      "activations/layer10_attention_weight_min": -38.1869010925293,
      "activations/layer11_attention_weight_max": 37.63587951660156,
      "activations/layer11_attention_weight_min": -33.424739837646484,
      "activations/layer12_attention_weight_max": 18.31366539001465,
      "activations/layer12_attention_weight_min": -27.82462501525879,
      "activations/layer13_attention_weight_max": 46.07575988769531,
      "activations/layer13_attention_weight_min": -33.16022872924805,
      "activations/layer14_attention_weight_max": 42.359527587890625,
      "activations/layer14_attention_weight_min": -29.38902473449707,
      "activations/layer15_attention_weight_max": 36.47275924682617,
      "activations/layer15_attention_weight_min": -28.823745727539062,
      "activations/layer16_attention_weight_max": 34.745643615722656,
      "activations/layer16_attention_weight_min": -32.055877685546875,
      "activations/layer17_attention_weight_max": 60.87382888793945,
      "activations/layer17_attention_weight_min": -48.63261795043945,
      "activations/layer18_attention_weight_max": 52.694461822509766,
      "activations/layer18_attention_weight_min": -43.519500732421875,
      "activations/layer19_attention_weight_max": 23.861154556274414,
      "activations/layer19_attention_weight_min": -23.003835678100586,
      "activations/layer1_attention_weight_max": 16.804990768432617,
      "activations/layer1_attention_weight_min": -17.230321884155273,
      "activations/layer20_attention_weight_max": 22.67827796936035,
      "activations/layer20_attention_weight_min": -21.057191848754883,
      "activations/layer21_attention_weight_max": 37.66144943237305,
      "activations/layer21_attention_weight_min": -22.968826293945312,
      "activations/layer22_attention_weight_max": 33.08122634887695,
      "activations/layer22_attention_weight_min": -26.856401443481445,
      "activations/layer23_attention_weight_max": 37.59832763671875,
      "activations/layer23_attention_weight_min": -25.06109619140625,
      "activations/layer2_attention_weight_max": 38.160804748535156,
      "activations/layer2_attention_weight_min": -35.37575912475586,
      "activations/layer3_attention_weight_max": 93.12059020996094,
      "activations/layer3_attention_weight_min": -93.7269287109375,
      "activations/layer4_attention_weight_max": 93.19718170166016,
      "activations/layer4_attention_weight_min": -92.39139556884766,
      "activations/layer5_attention_weight_max": 71.71979522705078,
      "activations/layer5_attention_weight_min": -73.55956268310547,
      "activations/layer6_attention_weight_max": 55.320682525634766,
      "activations/layer6_attention_weight_min": -53.121524810791016,
      "activations/layer7_attention_weight_max": 76.50028991699219,
      "activations/layer7_attention_weight_min": -75.28007507324219,
      "activations/layer8_attention_weight_max": 53.7465705871582,
      "activations/layer8_attention_weight_min": -52.36174392700195,
      "activations/layer9_attention_weight_max": 57.786285400390625,
      "activations/layer9_attention_weight_min": -51.947479248046875,
      "epoch": 14.02,
      "learning_rate": 6.017121212121211e-05,
      "loss": 2.7429,
      "step": 241350
    },
    {
      "activations/layer0_attention_weight_max": 16.329893112182617,
      "activations/layer0_attention_weight_min": -14.01717758178711,
      "activations/layer10_attention_weight_max": 37.06767654418945,
      "activations/layer10_attention_weight_min": -35.614768981933594,
      "activations/layer11_attention_weight_max": 30.958816528320312,
      "activations/layer11_attention_weight_min": -33.86992645263672,
      "activations/layer12_attention_weight_max": 19.044897079467773,
      "activations/layer12_attention_weight_min": -25.217126846313477,
      "activations/layer13_attention_weight_max": 37.713478088378906,
      "activations/layer13_attention_weight_min": -31.012319564819336,
      "activations/layer14_attention_weight_max": 37.812259674072266,
      "activations/layer14_attention_weight_min": -32.42488098144531,
      "activations/layer15_attention_weight_max": 34.81703186035156,
      "activations/layer15_attention_weight_min": -31.426908493041992,
      "activations/layer16_attention_weight_max": 33.63505554199219,
      "activations/layer16_attention_weight_min": -30.793827056884766,
      "activations/layer17_attention_weight_max": 52.22480773925781,
      "activations/layer17_attention_weight_min": -43.04330062866211,
      "activations/layer18_attention_weight_max": 46.10771942138672,
      "activations/layer18_attention_weight_min": -40.673946380615234,
      "activations/layer19_attention_weight_max": 21.13749122619629,
      "activations/layer19_attention_weight_min": -24.254560470581055,
      "activations/layer1_attention_weight_max": 16.26291847229004,
      "activations/layer1_attention_weight_min": -14.395127296447754,
      "activations/layer20_attention_weight_max": 22.99480438232422,
      "activations/layer20_attention_weight_min": -23.686216354370117,
      "activations/layer21_attention_weight_max": 34.68634796142578,
      "activations/layer21_attention_weight_min": -22.528196334838867,
      "activations/layer22_attention_weight_max": 29.537582397460938,
      "activations/layer22_attention_weight_min": -29.243297576904297,
      "activations/layer23_attention_weight_max": 39.23916244506836,
      "activations/layer23_attention_weight_min": -25.392568588256836,
      "activations/layer2_attention_weight_max": 31.093517303466797,
      "activations/layer2_attention_weight_min": -29.674619674682617,
      "activations/layer3_attention_weight_max": 87.27108764648438,
      "activations/layer3_attention_weight_min": -89.9185791015625,
      "activations/layer4_attention_weight_max": 94.728515625,
      "activations/layer4_attention_weight_min": -91.45636749267578,
      "activations/layer5_attention_weight_max": 69.05906677246094,
      "activations/layer5_attention_weight_min": -76.53186798095703,
      "activations/layer6_attention_weight_max": 49.4672966003418,
      "activations/layer6_attention_weight_min": -51.16816329956055,
      "activations/layer7_attention_weight_max": 61.870121002197266,
      "activations/layer7_attention_weight_min": -68.5206298828125,
      "activations/layer8_attention_weight_max": 45.67457580566406,
      "activations/layer8_attention_weight_min": -48.370750427246094,
      "activations/layer9_attention_weight_max": 41.726318359375,
      "activations/layer9_attention_weight_min": -43.47045135498047,
      "epoch": 14.03,
      "learning_rate": 6.015227272727272e-05,
      "loss": 2.753,
      "step": 241400
    },
    {
      "activations/layer0_attention_weight_max": 16.334087371826172,
      "activations/layer0_attention_weight_min": -14.070021629333496,
      "activations/layer10_attention_weight_max": 45.757686614990234,
      "activations/layer10_attention_weight_min": -40.69452667236328,
      "activations/layer11_attention_weight_max": 40.62508010864258,
      "activations/layer11_attention_weight_min": -38.42991638183594,
      "activations/layer12_attention_weight_max": 21.664167404174805,
      "activations/layer12_attention_weight_min": -34.3376579284668,
      "activations/layer13_attention_weight_max": 51.621238708496094,
      "activations/layer13_attention_weight_min": -30.864763259887695,
      "activations/layer14_attention_weight_max": 46.063392639160156,
      "activations/layer14_attention_weight_min": -31.533781051635742,
      "activations/layer15_attention_weight_max": 43.90114974975586,
      "activations/layer15_attention_weight_min": -30.51268768310547,
      "activations/layer16_attention_weight_max": 37.50718307495117,
      "activations/layer16_attention_weight_min": -32.186458587646484,
      "activations/layer17_attention_weight_max": 59.35149383544922,
      "activations/layer17_attention_weight_min": -48.44239044189453,
      "activations/layer18_attention_weight_max": 58.66153335571289,
      "activations/layer18_attention_weight_min": -42.59926986694336,
      "activations/layer19_attention_weight_max": 29.810049057006836,
      "activations/layer19_attention_weight_min": -23.128787994384766,
      "activations/layer1_attention_weight_max": 16.63785171508789,
      "activations/layer1_attention_weight_min": -14.4866943359375,
      "activations/layer20_attention_weight_max": 29.391101837158203,
      "activations/layer20_attention_weight_min": -20.507076263427734,
      "activations/layer21_attention_weight_max": 47.532955169677734,
      "activations/layer21_attention_weight_min": -24.43851089477539,
      "activations/layer22_attention_weight_max": 33.76273727416992,
      "activations/layer22_attention_weight_min": -27.045883178710938,
      "activations/layer23_attention_weight_max": 41.785892486572266,
      "activations/layer23_attention_weight_min": -24.040334701538086,
      "activations/layer2_attention_weight_max": 32.90345764160156,
      "activations/layer2_attention_weight_min": -30.383800506591797,
      "activations/layer3_attention_weight_max": 97.33981323242188,
      "activations/layer3_attention_weight_min": -96.44933319091797,
      "activations/layer4_attention_weight_max": 98.4804458618164,
      "activations/layer4_attention_weight_min": -92.275146484375,
      "activations/layer5_attention_weight_max": 72.23619079589844,
      "activations/layer5_attention_weight_min": -75.94881439208984,
      "activations/layer6_attention_weight_max": 54.114933013916016,
      "activations/layer6_attention_weight_min": -54.43094253540039,
      "activations/layer7_attention_weight_max": 78.15254974365234,
      "activations/layer7_attention_weight_min": -72.13574981689453,
      "activations/layer8_attention_weight_max": 59.17679214477539,
      "activations/layer8_attention_weight_min": -58.21929931640625,
      "activations/layer9_attention_weight_max": 55.10932540893555,
      "activations/layer9_attention_weight_min": -55.483367919921875,
      "epoch": 14.03,
      "learning_rate": 6.013333333333333e-05,
      "loss": 2.7427,
      "step": 241450
    },
    {
      "activations/layer0_attention_weight_max": 16.961580276489258,
      "activations/layer0_attention_weight_min": -14.331930160522461,
      "activations/layer10_attention_weight_max": 40.834388732910156,
      "activations/layer10_attention_weight_min": -35.34931945800781,
      "activations/layer11_attention_weight_max": 37.657508850097656,
      "activations/layer11_attention_weight_min": -33.699317932128906,
      "activations/layer12_attention_weight_max": 17.295312881469727,
      "activations/layer12_attention_weight_min": -24.730932235717773,
      "activations/layer13_attention_weight_max": 39.86964416503906,
      "activations/layer13_attention_weight_min": -30.032718658447266,
      "activations/layer14_attention_weight_max": 35.32721710205078,
      "activations/layer14_attention_weight_min": -31.072399139404297,
      "activations/layer15_attention_weight_max": 34.8204460144043,
      "activations/layer15_attention_weight_min": -30.161983489990234,
      "activations/layer16_attention_weight_max": 33.01067352294922,
      "activations/layer16_attention_weight_min": -31.33574104309082,
      "activations/layer17_attention_weight_max": 52.583797454833984,
      "activations/layer17_attention_weight_min": -44.55339431762695,
      "activations/layer18_attention_weight_max": 51.47566604614258,
      "activations/layer18_attention_weight_min": -37.28726577758789,
      "activations/layer19_attention_weight_max": 26.554737091064453,
      "activations/layer19_attention_weight_min": -21.8087215423584,
      "activations/layer1_attention_weight_max": 17.258832931518555,
      "activations/layer1_attention_weight_min": -14.15845775604248,
      "activations/layer20_attention_weight_max": 25.569324493408203,
      "activations/layer20_attention_weight_min": -22.61559295654297,
      "activations/layer21_attention_weight_max": 39.148841857910156,
      "activations/layer21_attention_weight_min": -23.824186325073242,
      "activations/layer22_attention_weight_max": 32.69398498535156,
      "activations/layer22_attention_weight_min": -26.29610824584961,
      "activations/layer23_attention_weight_max": 38.453128814697266,
      "activations/layer23_attention_weight_min": -24.572622299194336,
      "activations/layer2_attention_weight_max": 34.42607116699219,
      "activations/layer2_attention_weight_min": -30.192411422729492,
      "activations/layer3_attention_weight_max": 99.62498474121094,
      "activations/layer3_attention_weight_min": -95.52835845947266,
      "activations/layer4_attention_weight_max": 99.1935043334961,
      "activations/layer4_attention_weight_min": -95.15857696533203,
      "activations/layer5_attention_weight_max": 75.9780044555664,
      "activations/layer5_attention_weight_min": -83.55966186523438,
      "activations/layer6_attention_weight_max": 53.12163543701172,
      "activations/layer6_attention_weight_min": -54.296974182128906,
      "activations/layer7_attention_weight_max": 66.37577819824219,
      "activations/layer7_attention_weight_min": -66.51087951660156,
      "activations/layer8_attention_weight_max": 48.86338424682617,
      "activations/layer8_attention_weight_min": -48.556339263916016,
      "activations/layer9_attention_weight_max": 47.58724594116211,
      "activations/layer9_attention_weight_min": -47.5255241394043,
      "epoch": 14.03,
      "learning_rate": 6.011439393939394e-05,
      "loss": 2.7538,
      "step": 241500
    },
    {
      "activations/layer0_attention_weight_max": 16.030277252197266,
      "activations/layer0_attention_weight_min": -14.184536933898926,
      "activations/layer10_attention_weight_max": 39.45470428466797,
      "activations/layer10_attention_weight_min": -36.72914123535156,
      "activations/layer11_attention_weight_max": 36.70365524291992,
      "activations/layer11_attention_weight_min": -33.25189208984375,
      "activations/layer12_attention_weight_max": 28.49622344970703,
      "activations/layer12_attention_weight_min": -21.956043243408203,
      "activations/layer13_attention_weight_max": 46.641502380371094,
      "activations/layer13_attention_weight_min": -31.219331741333008,
      "activations/layer14_attention_weight_max": 43.273048400878906,
      "activations/layer14_attention_weight_min": -29.5672664642334,
      "activations/layer15_attention_weight_max": 40.41627883911133,
      "activations/layer15_attention_weight_min": -28.267744064331055,
      "activations/layer16_attention_weight_max": 31.369152069091797,
      "activations/layer16_attention_weight_min": -29.66873550415039,
      "activations/layer17_attention_weight_max": 51.21661376953125,
      "activations/layer17_attention_weight_min": -42.891422271728516,
      "activations/layer18_attention_weight_max": 44.52511978149414,
      "activations/layer18_attention_weight_min": -38.118228912353516,
      "activations/layer19_attention_weight_max": 23.77619171142578,
      "activations/layer19_attention_weight_min": -20.772144317626953,
      "activations/layer1_attention_weight_max": 16.917226791381836,
      "activations/layer1_attention_weight_min": -15.349769592285156,
      "activations/layer20_attention_weight_max": 23.174894332885742,
      "activations/layer20_attention_weight_min": -20.769838333129883,
      "activations/layer21_attention_weight_max": 31.126005172729492,
      "activations/layer21_attention_weight_min": -21.75750160217285,
      "activations/layer22_attention_weight_max": 32.16636276245117,
      "activations/layer22_attention_weight_min": -25.36252784729004,
      "activations/layer23_attention_weight_max": 37.17455291748047,
      "activations/layer23_attention_weight_min": -23.776830673217773,
      "activations/layer2_attention_weight_max": 31.692581176757812,
      "activations/layer2_attention_weight_min": -29.40364646911621,
      "activations/layer3_attention_weight_max": 92.21977996826172,
      "activations/layer3_attention_weight_min": -93.4206771850586,
      "activations/layer4_attention_weight_max": 94.58405303955078,
      "activations/layer4_attention_weight_min": -92.57482147216797,
      "activations/layer5_attention_weight_max": 70.62666320800781,
      "activations/layer5_attention_weight_min": -84.34468078613281,
      "activations/layer6_attention_weight_max": 53.06241989135742,
      "activations/layer6_attention_weight_min": -52.91518020629883,
      "activations/layer7_attention_weight_max": 66.94253540039062,
      "activations/layer7_attention_weight_min": -65.35157775878906,
      "activations/layer8_attention_weight_max": 46.031532287597656,
      "activations/layer8_attention_weight_min": -47.6335334777832,
      "activations/layer9_attention_weight_max": 46.33510208129883,
      "activations/layer9_attention_weight_min": -45.0543098449707,
      "epoch": 14.04,
      "learning_rate": 6.009545454545454e-05,
      "loss": 2.7739,
      "step": 241550
    },
    {
      "activations/layer0_attention_weight_max": 17.270280838012695,
      "activations/layer0_attention_weight_min": -13.74976634979248,
      "activations/layer10_attention_weight_max": 36.63041687011719,
      "activations/layer10_attention_weight_min": -36.81147384643555,
      "activations/layer11_attention_weight_max": 33.04988098144531,
      "activations/layer11_attention_weight_min": -36.1685676574707,
      "activations/layer12_attention_weight_max": 19.52127456665039,
      "activations/layer12_attention_weight_min": -23.187015533447266,
      "activations/layer13_attention_weight_max": 39.093902587890625,
      "activations/layer13_attention_weight_min": -33.1172981262207,
      "activations/layer14_attention_weight_max": 36.882930755615234,
      "activations/layer14_attention_weight_min": -31.2137508392334,
      "activations/layer15_attention_weight_max": 34.33978271484375,
      "activations/layer15_attention_weight_min": -33.122901916503906,
      "activations/layer16_attention_weight_max": 35.16425323486328,
      "activations/layer16_attention_weight_min": -32.32034683227539,
      "activations/layer17_attention_weight_max": 54.81789779663086,
      "activations/layer17_attention_weight_min": -46.93831253051758,
      "activations/layer18_attention_weight_max": 48.04811096191406,
      "activations/layer18_attention_weight_min": -39.61247634887695,
      "activations/layer19_attention_weight_max": 23.9114990234375,
      "activations/layer19_attention_weight_min": -20.98839569091797,
      "activations/layer1_attention_weight_max": 16.433813095092773,
      "activations/layer1_attention_weight_min": -14.35188102722168,
      "activations/layer20_attention_weight_max": 22.68605613708496,
      "activations/layer20_attention_weight_min": -21.059783935546875,
      "activations/layer21_attention_weight_max": 38.331809997558594,
      "activations/layer21_attention_weight_min": -22.3740291595459,
      "activations/layer22_attention_weight_max": 32.52605056762695,
      "activations/layer22_attention_weight_min": -25.14344024658203,
      "activations/layer23_attention_weight_max": 41.082550048828125,
      "activations/layer23_attention_weight_min": -23.421527862548828,
      "activations/layer2_attention_weight_max": 32.605594635009766,
      "activations/layer2_attention_weight_min": -31.379758834838867,
      "activations/layer3_attention_weight_max": 93.64299011230469,
      "activations/layer3_attention_weight_min": -95.51614379882812,
      "activations/layer4_attention_weight_max": 97.56928253173828,
      "activations/layer4_attention_weight_min": -97.45096588134766,
      "activations/layer5_attention_weight_max": 71.12705993652344,
      "activations/layer5_attention_weight_min": -81.52470397949219,
      "activations/layer6_attention_weight_max": 52.98512268066406,
      "activations/layer6_attention_weight_min": -53.88746643066406,
      "activations/layer7_attention_weight_max": 65.74179077148438,
      "activations/layer7_attention_weight_min": -66.07726287841797,
      "activations/layer8_attention_weight_max": 45.74410629272461,
      "activations/layer8_attention_weight_min": -50.573486328125,
      "activations/layer9_attention_weight_max": 44.29706954956055,
      "activations/layer9_attention_weight_min": -46.35953140258789,
      "epoch": 14.04,
      "learning_rate": 6.007651515151514e-05,
      "loss": 2.7592,
      "step": 241600
    },
    {
      "activations/layer0_attention_weight_max": 15.756332397460938,
      "activations/layer0_attention_weight_min": -13.887569427490234,
      "activations/layer10_attention_weight_max": 36.51906204223633,
      "activations/layer10_attention_weight_min": -35.61811447143555,
      "activations/layer11_attention_weight_max": 32.12957000732422,
      "activations/layer11_attention_weight_min": -33.82567596435547,
      "activations/layer12_attention_weight_max": 19.094276428222656,
      "activations/layer12_attention_weight_min": -25.98084259033203,
      "activations/layer13_attention_weight_max": 41.83839797973633,
      "activations/layer13_attention_weight_min": -29.97715950012207,
      "activations/layer14_attention_weight_max": 36.31850051879883,
      "activations/layer14_attention_weight_min": -33.93122100830078,
      "activations/layer15_attention_weight_max": 34.5225830078125,
      "activations/layer15_attention_weight_min": -32.55170822143555,
      "activations/layer16_attention_weight_max": 32.92667770385742,
      "activations/layer16_attention_weight_min": -31.827547073364258,
      "activations/layer17_attention_weight_max": 53.860713958740234,
      "activations/layer17_attention_weight_min": -46.23414993286133,
      "activations/layer18_attention_weight_max": 49.078704833984375,
      "activations/layer18_attention_weight_min": -40.301841735839844,
      "activations/layer19_attention_weight_max": 23.221572875976562,
      "activations/layer19_attention_weight_min": -24.276670455932617,
      "activations/layer1_attention_weight_max": 16.054916381835938,
      "activations/layer1_attention_weight_min": -14.804631233215332,
      "activations/layer20_attention_weight_max": 22.03097152709961,
      "activations/layer20_attention_weight_min": -22.835649490356445,
      "activations/layer21_attention_weight_max": 41.28700637817383,
      "activations/layer21_attention_weight_min": -24.581140518188477,
      "activations/layer22_attention_weight_max": 31.66176414489746,
      "activations/layer22_attention_weight_min": -26.30971336364746,
      "activations/layer23_attention_weight_max": 38.66236877441406,
      "activations/layer23_attention_weight_min": -24.625030517578125,
      "activations/layer2_attention_weight_max": 32.12181091308594,
      "activations/layer2_attention_weight_min": -30.132305145263672,
      "activations/layer3_attention_weight_max": 95.36259460449219,
      "activations/layer3_attention_weight_min": -101.58048248291016,
      "activations/layer4_attention_weight_max": 94.0658950805664,
      "activations/layer4_attention_weight_min": -90.48344421386719,
      "activations/layer5_attention_weight_max": 70.3587417602539,
      "activations/layer5_attention_weight_min": -78.14786529541016,
      "activations/layer6_attention_weight_max": 50.933494567871094,
      "activations/layer6_attention_weight_min": -52.26927185058594,
      "activations/layer7_attention_weight_max": 65.67667388916016,
      "activations/layer7_attention_weight_min": -65.88067626953125,
      "activations/layer8_attention_weight_max": 44.552268981933594,
      "activations/layer8_attention_weight_min": -51.831390380859375,
      "activations/layer9_attention_weight_max": 47.507137298583984,
      "activations/layer9_attention_weight_min": -47.6942138671875,
      "epoch": 14.04,
      "learning_rate": 6.0057575757575755e-05,
      "loss": 2.7554,
      "step": 241650
    },
    {
      "activations/layer0_attention_weight_max": 16.14653778076172,
      "activations/layer0_attention_weight_min": -14.021793365478516,
      "activations/layer10_attention_weight_max": 35.87890625,
      "activations/layer10_attention_weight_min": -35.9875373840332,
      "activations/layer11_attention_weight_max": 30.66930389404297,
      "activations/layer11_attention_weight_min": -34.18507766723633,
      "activations/layer12_attention_weight_max": 18.239124298095703,
      "activations/layer12_attention_weight_min": -22.5494327545166,
      "activations/layer13_attention_weight_max": 36.4411735534668,
      "activations/layer13_attention_weight_min": -26.040979385375977,
      "activations/layer14_attention_weight_max": 36.166709899902344,
      "activations/layer14_attention_weight_min": -29.689573287963867,
      "activations/layer15_attention_weight_max": 31.737266540527344,
      "activations/layer15_attention_weight_min": -27.638700485229492,
      "activations/layer16_attention_weight_max": 32.83487319946289,
      "activations/layer16_attention_weight_min": -30.540552139282227,
      "activations/layer17_attention_weight_max": 52.96775436401367,
      "activations/layer17_attention_weight_min": -43.13401794433594,
      "activations/layer18_attention_weight_max": 46.62839126586914,
      "activations/layer18_attention_weight_min": -37.0198974609375,
      "activations/layer19_attention_weight_max": 21.10333251953125,
      "activations/layer19_attention_weight_min": -20.57814598083496,
      "activations/layer1_attention_weight_max": 16.11791229248047,
      "activations/layer1_attention_weight_min": -14.780753135681152,
      "activations/layer20_attention_weight_max": 21.043516159057617,
      "activations/layer20_attention_weight_min": -20.15127944946289,
      "activations/layer21_attention_weight_max": 34.451778411865234,
      "activations/layer21_attention_weight_min": -21.547607421875,
      "activations/layer22_attention_weight_max": 29.98887062072754,
      "activations/layer22_attention_weight_min": -24.668731689453125,
      "activations/layer23_attention_weight_max": 37.94232177734375,
      "activations/layer23_attention_weight_min": -24.25301742553711,
      "activations/layer2_attention_weight_max": 32.62255859375,
      "activations/layer2_attention_weight_min": -30.702421188354492,
      "activations/layer3_attention_weight_max": 94.30680847167969,
      "activations/layer3_attention_weight_min": -94.39805603027344,
      "activations/layer4_attention_weight_max": 92.76679229736328,
      "activations/layer4_attention_weight_min": -89.36189270019531,
      "activations/layer5_attention_weight_max": 72.07662963867188,
      "activations/layer5_attention_weight_min": -79.28052520751953,
      "activations/layer6_attention_weight_max": 52.36185836791992,
      "activations/layer6_attention_weight_min": -52.15957260131836,
      "activations/layer7_attention_weight_max": 66.90885162353516,
      "activations/layer7_attention_weight_min": -61.8967399597168,
      "activations/layer8_attention_weight_max": 45.784027099609375,
      "activations/layer8_attention_weight_min": -49.760379791259766,
      "activations/layer9_attention_weight_max": 46.0142822265625,
      "activations/layer9_attention_weight_min": -47.29026412963867,
      "epoch": 14.04,
      "learning_rate": 6.003863636363636e-05,
      "loss": 2.7435,
      "step": 241700
    },
    {
      "activations/layer0_attention_weight_max": 16.868728637695312,
      "activations/layer0_attention_weight_min": -13.521378517150879,
      "activations/layer10_attention_weight_max": 36.97157287597656,
      "activations/layer10_attention_weight_min": -36.61223220825195,
      "activations/layer11_attention_weight_max": 36.138275146484375,
      "activations/layer11_attention_weight_min": -35.158966064453125,
      "activations/layer12_attention_weight_max": 19.497108459472656,
      "activations/layer12_attention_weight_min": -26.369991302490234,
      "activations/layer13_attention_weight_max": 34.965171813964844,
      "activations/layer13_attention_weight_min": -26.670854568481445,
      "activations/layer14_attention_weight_max": 34.91425704956055,
      "activations/layer14_attention_weight_min": -29.263656616210938,
      "activations/layer15_attention_weight_max": 34.221317291259766,
      "activations/layer15_attention_weight_min": -28.34716796875,
      "activations/layer16_attention_weight_max": 30.83036994934082,
      "activations/layer16_attention_weight_min": -29.95459747314453,
      "activations/layer17_attention_weight_max": 52.1280403137207,
      "activations/layer17_attention_weight_min": -44.66078567504883,
      "activations/layer18_attention_weight_max": 46.57319259643555,
      "activations/layer18_attention_weight_min": -37.739505767822266,
      "activations/layer19_attention_weight_max": 22.431066513061523,
      "activations/layer19_attention_weight_min": -21.527084350585938,
      "activations/layer1_attention_weight_max": 15.734452247619629,
      "activations/layer1_attention_weight_min": -14.605817794799805,
      "activations/layer20_attention_weight_max": 21.947851181030273,
      "activations/layer20_attention_weight_min": -21.301912307739258,
      "activations/layer21_attention_weight_max": 37.39200973510742,
      "activations/layer21_attention_weight_min": -25.195390701293945,
      "activations/layer22_attention_weight_max": 30.449148178100586,
      "activations/layer22_attention_weight_min": -25.21379852294922,
      "activations/layer23_attention_weight_max": 38.76245880126953,
      "activations/layer23_attention_weight_min": -22.89833641052246,
      "activations/layer2_attention_weight_max": 31.252967834472656,
      "activations/layer2_attention_weight_min": -30.959877014160156,
      "activations/layer3_attention_weight_max": 91.83099365234375,
      "activations/layer3_attention_weight_min": -99.05039978027344,
      "activations/layer4_attention_weight_max": 94.5728759765625,
      "activations/layer4_attention_weight_min": -95.27764129638672,
      "activations/layer5_attention_weight_max": 70.59860229492188,
      "activations/layer5_attention_weight_min": -75.90320587158203,
      "activations/layer6_attention_weight_max": 50.5377311706543,
      "activations/layer6_attention_weight_min": -51.15848159790039,
      "activations/layer7_attention_weight_max": 68.68290710449219,
      "activations/layer7_attention_weight_min": -67.49604034423828,
      "activations/layer8_attention_weight_max": 46.21074295043945,
      "activations/layer8_attention_weight_min": -49.24007034301758,
      "activations/layer9_attention_weight_max": 43.67757797241211,
      "activations/layer9_attention_weight_min": -48.02180099487305,
      "epoch": 14.05,
      "learning_rate": 6.0019696969696965e-05,
      "loss": 2.7576,
      "step": 241750
    },
    {
      "activations/layer0_attention_weight_max": 17.019014358520508,
      "activations/layer0_attention_weight_min": -13.873953819274902,
      "activations/layer10_attention_weight_max": 34.52448272705078,
      "activations/layer10_attention_weight_min": -34.39226531982422,
      "activations/layer11_attention_weight_max": 30.630207061767578,
      "activations/layer11_attention_weight_min": -33.188385009765625,
      "activations/layer12_attention_weight_max": 17.625713348388672,
      "activations/layer12_attention_weight_min": -25.028865814208984,
      "activations/layer13_attention_weight_max": 33.535614013671875,
      "activations/layer13_attention_weight_min": -27.247758865356445,
      "activations/layer14_attention_weight_max": 32.57604217529297,
      "activations/layer14_attention_weight_min": -27.954113006591797,
      "activations/layer15_attention_weight_max": 29.947845458984375,
      "activations/layer15_attention_weight_min": -29.615802764892578,
      "activations/layer16_attention_weight_max": 30.694225311279297,
      "activations/layer16_attention_weight_min": -30.08479118347168,
      "activations/layer17_attention_weight_max": 47.69316864013672,
      "activations/layer17_attention_weight_min": -41.21044158935547,
      "activations/layer18_attention_weight_max": 41.482276916503906,
      "activations/layer18_attention_weight_min": -37.1019172668457,
      "activations/layer19_attention_weight_max": 22.528623580932617,
      "activations/layer19_attention_weight_min": -21.211345672607422,
      "activations/layer1_attention_weight_max": 16.684616088867188,
      "activations/layer1_attention_weight_min": -14.557330131530762,
      "activations/layer20_attention_weight_max": 21.707998275756836,
      "activations/layer20_attention_weight_min": -21.208988189697266,
      "activations/layer21_attention_weight_max": 32.09900665283203,
      "activations/layer21_attention_weight_min": -20.12746238708496,
      "activations/layer22_attention_weight_max": 28.941003799438477,
      "activations/layer22_attention_weight_min": -26.16983413696289,
      "activations/layer23_attention_weight_max": 40.82853317260742,
      "activations/layer23_attention_weight_min": -23.31915855407715,
      "activations/layer2_attention_weight_max": 31.131820678710938,
      "activations/layer2_attention_weight_min": -31.08837890625,
      "activations/layer3_attention_weight_max": 94.81732940673828,
      "activations/layer3_attention_weight_min": -93.55659484863281,
      "activations/layer4_attention_weight_max": 94.911865234375,
      "activations/layer4_attention_weight_min": -91.15608215332031,
      "activations/layer5_attention_weight_max": 70.1915283203125,
      "activations/layer5_attention_weight_min": -80.76595306396484,
      "activations/layer6_attention_weight_max": 50.828304290771484,
      "activations/layer6_attention_weight_min": -52.248661041259766,
      "activations/layer7_attention_weight_max": 68.87369537353516,
      "activations/layer7_attention_weight_min": -65.5877456665039,
      "activations/layer8_attention_weight_max": 44.59104537963867,
      "activations/layer8_attention_weight_min": -47.35348129272461,
      "activations/layer9_attention_weight_max": 43.76949691772461,
      "activations/layer9_attention_weight_min": -46.93618392944336,
      "epoch": 14.05,
      "learning_rate": 6.0000757575757573e-05,
      "loss": 2.7589,
      "step": 241800
    },
    {
      "activations/layer0_attention_weight_max": 16.370708465576172,
      "activations/layer0_attention_weight_min": -14.448756217956543,
      "activations/layer10_attention_weight_max": 39.12054443359375,
      "activations/layer10_attention_weight_min": -40.34121322631836,
      "activations/layer11_attention_weight_max": 35.43412399291992,
      "activations/layer11_attention_weight_min": -33.9862174987793,
      "activations/layer12_attention_weight_max": 18.660053253173828,
      "activations/layer12_attention_weight_min": -24.601667404174805,
      "activations/layer13_attention_weight_max": 37.8835334777832,
      "activations/layer13_attention_weight_min": -30.381229400634766,
      "activations/layer14_attention_weight_max": 36.009239196777344,
      "activations/layer14_attention_weight_min": -32.261497497558594,
      "activations/layer15_attention_weight_max": 33.929649353027344,
      "activations/layer15_attention_weight_min": -29.81877326965332,
      "activations/layer16_attention_weight_max": 36.70295333862305,
      "activations/layer16_attention_weight_min": -33.11448669433594,
      "activations/layer17_attention_weight_max": 52.918800354003906,
      "activations/layer17_attention_weight_min": -45.534400939941406,
      "activations/layer18_attention_weight_max": 48.4259147644043,
      "activations/layer18_attention_weight_min": -37.32599639892578,
      "activations/layer19_attention_weight_max": 22.41178321838379,
      "activations/layer19_attention_weight_min": -22.65425682067871,
      "activations/layer1_attention_weight_max": 16.723114013671875,
      "activations/layer1_attention_weight_min": -14.18206787109375,
      "activations/layer20_attention_weight_max": 21.826854705810547,
      "activations/layer20_attention_weight_min": -23.10004997253418,
      "activations/layer21_attention_weight_max": 34.7642707824707,
      "activations/layer21_attention_weight_min": -23.746408462524414,
      "activations/layer22_attention_weight_max": 30.022886276245117,
      "activations/layer22_attention_weight_min": -23.87694549560547,
      "activations/layer23_attention_weight_max": 37.230865478515625,
      "activations/layer23_attention_weight_min": -25.006591796875,
      "activations/layer2_attention_weight_max": 32.34966278076172,
      "activations/layer2_attention_weight_min": -31.71551513671875,
      "activations/layer3_attention_weight_max": 97.96282196044922,
      "activations/layer3_attention_weight_min": -104.36763763427734,
      "activations/layer4_attention_weight_max": 95.04505157470703,
      "activations/layer4_attention_weight_min": -90.77661895751953,
      "activations/layer5_attention_weight_max": 70.37890625,
      "activations/layer5_attention_weight_min": -80.55975341796875,
      "activations/layer6_attention_weight_max": 52.045509338378906,
      "activations/layer6_attention_weight_min": -52.84598922729492,
      "activations/layer7_attention_weight_max": 65.94583129882812,
      "activations/layer7_attention_weight_min": -63.462196350097656,
      "activations/layer8_attention_weight_max": 45.95163345336914,
      "activations/layer8_attention_weight_min": -47.66210174560547,
      "activations/layer9_attention_weight_max": 45.10468673706055,
      "activations/layer9_attention_weight_min": -46.54774856567383,
      "epoch": 14.05,
      "learning_rate": 5.9981818181818175e-05,
      "loss": 2.7608,
      "step": 241850
    },
    {
      "activations/layer0_attention_weight_max": 16.76773452758789,
      "activations/layer0_attention_weight_min": -14.274506568908691,
      "activations/layer10_attention_weight_max": 39.79049301147461,
      "activations/layer10_attention_weight_min": -40.28845977783203,
      "activations/layer11_attention_weight_max": 35.7119026184082,
      "activations/layer11_attention_weight_min": -34.24016189575195,
      "activations/layer12_attention_weight_max": 19.363351821899414,
      "activations/layer12_attention_weight_min": -25.376859664916992,
      "activations/layer13_attention_weight_max": 38.97559356689453,
      "activations/layer13_attention_weight_min": -31.105642318725586,
      "activations/layer14_attention_weight_max": 38.101627349853516,
      "activations/layer14_attention_weight_min": -32.18201446533203,
      "activations/layer15_attention_weight_max": 37.1204948425293,
      "activations/layer15_attention_weight_min": -31.57942008972168,
      "activations/layer16_attention_weight_max": 33.539615631103516,
      "activations/layer16_attention_weight_min": -31.34092903137207,
      "activations/layer17_attention_weight_max": 53.042171478271484,
      "activations/layer17_attention_weight_min": -49.65353012084961,
      "activations/layer18_attention_weight_max": 48.471134185791016,
      "activations/layer18_attention_weight_min": -44.83511734008789,
      "activations/layer19_attention_weight_max": 23.2519588470459,
      "activations/layer19_attention_weight_min": -23.08778190612793,
      "activations/layer1_attention_weight_max": 15.859277725219727,
      "activations/layer1_attention_weight_min": -15.134873390197754,
      "activations/layer20_attention_weight_max": 23.2866153717041,
      "activations/layer20_attention_weight_min": -22.382282257080078,
      "activations/layer21_attention_weight_max": 40.170162200927734,
      "activations/layer21_attention_weight_min": -24.849456787109375,
      "activations/layer22_attention_weight_max": 31.69934844970703,
      "activations/layer22_attention_weight_min": -27.29071807861328,
      "activations/layer23_attention_weight_max": 40.78221893310547,
      "activations/layer23_attention_weight_min": -26.87868881225586,
      "activations/layer2_attention_weight_max": 33.64361572265625,
      "activations/layer2_attention_weight_min": -31.955617904663086,
      "activations/layer3_attention_weight_max": 97.90451049804688,
      "activations/layer3_attention_weight_min": -97.32717895507812,
      "activations/layer4_attention_weight_max": 99.94877624511719,
      "activations/layer4_attention_weight_min": -97.05087280273438,
      "activations/layer5_attention_weight_max": 76.84306335449219,
      "activations/layer5_attention_weight_min": -78.36945343017578,
      "activations/layer6_attention_weight_max": 54.84242630004883,
      "activations/layer6_attention_weight_min": -55.728275299072266,
      "activations/layer7_attention_weight_max": 68.3343734741211,
      "activations/layer7_attention_weight_min": -66.27291107177734,
      "activations/layer8_attention_weight_max": 49.565345764160156,
      "activations/layer8_attention_weight_min": -50.41834259033203,
      "activations/layer9_attention_weight_max": 50.05325698852539,
      "activations/layer9_attention_weight_min": -49.943477630615234,
      "epoch": 14.06,
      "learning_rate": 5.996287878787878e-05,
      "loss": 2.7687,
      "step": 241900
    },
    {
      "activations/layer0_attention_weight_max": 15.596766471862793,
      "activations/layer0_attention_weight_min": -14.058028221130371,
      "activations/layer10_attention_weight_max": 36.64076614379883,
      "activations/layer10_attention_weight_min": -36.824153900146484,
      "activations/layer11_attention_weight_max": 31.936058044433594,
      "activations/layer11_attention_weight_min": -33.624610900878906,
      "activations/layer12_attention_weight_max": 18.30819320678711,
      "activations/layer12_attention_weight_min": -27.96579360961914,
      "activations/layer13_attention_weight_max": 34.23124313354492,
      "activations/layer13_attention_weight_min": -26.998727798461914,
      "activations/layer14_attention_weight_max": 37.73998260498047,
      "activations/layer14_attention_weight_min": -30.937049865722656,
      "activations/layer15_attention_weight_max": 31.985475540161133,
      "activations/layer15_attention_weight_min": -29.202171325683594,
      "activations/layer16_attention_weight_max": 30.876935958862305,
      "activations/layer16_attention_weight_min": -29.95567512512207,
      "activations/layer17_attention_weight_max": 48.727874755859375,
      "activations/layer17_attention_weight_min": -42.18248748779297,
      "activations/layer18_attention_weight_max": 44.9212532043457,
      "activations/layer18_attention_weight_min": -37.081703186035156,
      "activations/layer19_attention_weight_max": 22.43309211730957,
      "activations/layer19_attention_weight_min": -21.98036003112793,
      "activations/layer1_attention_weight_max": 17.30923080444336,
      "activations/layer1_attention_weight_min": -12.83210563659668,
      "activations/layer20_attention_weight_max": 21.417869567871094,
      "activations/layer20_attention_weight_min": -20.816438674926758,
      "activations/layer21_attention_weight_max": 33.96562957763672,
      "activations/layer21_attention_weight_min": -25.5490665435791,
      "activations/layer22_attention_weight_max": 30.983551025390625,
      "activations/layer22_attention_weight_min": -27.102521896362305,
      "activations/layer23_attention_weight_max": 35.7786865234375,
      "activations/layer23_attention_weight_min": -24.54066276550293,
      "activations/layer2_attention_weight_max": 33.257225036621094,
      "activations/layer2_attention_weight_min": -31.591571807861328,
      "activations/layer3_attention_weight_max": 91.05023193359375,
      "activations/layer3_attention_weight_min": -96.00009155273438,
      "activations/layer4_attention_weight_max": 96.10005950927734,
      "activations/layer4_attention_weight_min": -91.71754455566406,
      "activations/layer5_attention_weight_max": 72.48712921142578,
      "activations/layer5_attention_weight_min": -75.02439880371094,
      "activations/layer6_attention_weight_max": 52.03958511352539,
      "activations/layer6_attention_weight_min": -51.81978988647461,
      "activations/layer7_attention_weight_max": 66.96116638183594,
      "activations/layer7_attention_weight_min": -66.7672348022461,
      "activations/layer8_attention_weight_max": 43.39716720581055,
      "activations/layer8_attention_weight_min": -50.76790237426758,
      "activations/layer9_attention_weight_max": 46.48953628540039,
      "activations/layer9_attention_weight_min": -47.93780517578125,
      "epoch": 14.06,
      "learning_rate": 5.9943939393939385e-05,
      "loss": 2.7537,
      "step": 241950
    },
    {
      "activations/layer0_attention_weight_max": 16.253368377685547,
      "activations/layer0_attention_weight_min": -14.637781143188477,
      "activations/layer10_attention_weight_max": 37.169071197509766,
      "activations/layer10_attention_weight_min": -35.93989944458008,
      "activations/layer11_attention_weight_max": 34.39173126220703,
      "activations/layer11_attention_weight_min": -33.63996124267578,
      "activations/layer12_attention_weight_max": 17.903593063354492,
      "activations/layer12_attention_weight_min": -26.295703887939453,
      "activations/layer13_attention_weight_max": 33.87773513793945,
      "activations/layer13_attention_weight_min": -30.83468246459961,
      "activations/layer14_attention_weight_max": 33.2586669921875,
      "activations/layer14_attention_weight_min": -29.485647201538086,
      "activations/layer15_attention_weight_max": 30.279674530029297,
      "activations/layer15_attention_weight_min": -30.998746871948242,
      "activations/layer16_attention_weight_max": 30.273258209228516,
      "activations/layer16_attention_weight_min": -29.78853416442871,
      "activations/layer17_attention_weight_max": 47.21500778198242,
      "activations/layer17_attention_weight_min": -42.38264846801758,
      "activations/layer18_attention_weight_max": 44.9263801574707,
      "activations/layer18_attention_weight_min": -38.35630416870117,
      "activations/layer19_attention_weight_max": 23.986360549926758,
      "activations/layer19_attention_weight_min": -25.392017364501953,
      "activations/layer1_attention_weight_max": 15.884434700012207,
      "activations/layer1_attention_weight_min": -14.632749557495117,
      "activations/layer20_attention_weight_max": 25.58877182006836,
      "activations/layer20_attention_weight_min": -26.185245513916016,
      "activations/layer21_attention_weight_max": 38.17447280883789,
      "activations/layer21_attention_weight_min": -24.379735946655273,
      "activations/layer22_attention_weight_max": 30.691097259521484,
      "activations/layer22_attention_weight_min": -31.126544952392578,
      "activations/layer23_attention_weight_max": 37.65421676635742,
      "activations/layer23_attention_weight_min": -25.50478172302246,
      "activations/layer2_attention_weight_max": 31.37505340576172,
      "activations/layer2_attention_weight_min": -30.45801544189453,
      "activations/layer3_attention_weight_max": 90.70237731933594,
      "activations/layer3_attention_weight_min": -97.74281311035156,
      "activations/layer4_attention_weight_max": 90.15302276611328,
      "activations/layer4_attention_weight_min": -91.68358612060547,
      "activations/layer5_attention_weight_max": 72.98452758789062,
      "activations/layer5_attention_weight_min": -77.2059326171875,
      "activations/layer6_attention_weight_max": 49.88988494873047,
      "activations/layer6_attention_weight_min": -49.96485137939453,
      "activations/layer7_attention_weight_max": 64.05774688720703,
      "activations/layer7_attention_weight_min": -64.07259368896484,
      "activations/layer8_attention_weight_max": 47.86002731323242,
      "activations/layer8_attention_weight_min": -48.87958526611328,
      "activations/layer9_attention_weight_max": 44.71769332885742,
      "activations/layer9_attention_weight_min": -44.63705062866211,
      "epoch": 14.06,
      "learning_rate": 5.9925e-05,
      "loss": 2.7488,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_loss": 2.716796875,
      "eval_runtime": 8.4925,
      "eval_samples_per_second": 505.623,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_openwebtext_loss": 2.716796875,
      "eval_openwebtext_ppl": 15.131775567308885,
      "eval_openwebtext_runtime": 8.4925,
      "eval_openwebtext_samples_per_second": 505.623,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 2.0482,
      "eval_wikitext_samples_per_second": 222.635,
      "step": 242000
    },
    {
      "epoch": 14.06,
      "eval_lambada_loss": 2.501953125,
      "eval_lambada_ppl": 12.206311145618837,
      "eval_lambada_runtime": 9.5694,
      "eval_lambada_samples_per_second": 508.807,
      "step": 242000
    },
    {
      "activations/layer0_attention_weight_max": 16.37603759765625,
      "activations/layer0_attention_weight_min": -14.190887451171875,
      "activations/layer10_attention_weight_max": 38.346099853515625,
      "activations/layer10_attention_weight_min": -35.89034652709961,
      "activations/layer11_attention_weight_max": 34.710662841796875,
      "activations/layer11_attention_weight_min": -34.84433364868164,
      "activations/layer12_attention_weight_max": 19.479631423950195,
      "activations/layer12_attention_weight_min": -29.547664642333984,
      "activations/layer13_attention_weight_max": 40.2796630859375,
      "activations/layer13_attention_weight_min": -27.86049461364746,
      "activations/layer14_attention_weight_max": 36.48456954956055,
      "activations/layer14_attention_weight_min": -31.37925148010254,
      "activations/layer15_attention_weight_max": 33.548614501953125,
      "activations/layer15_attention_weight_min": -29.342042922973633,
      "activations/layer16_attention_weight_max": 33.5812873840332,
      "activations/layer16_attention_weight_min": -30.426151275634766,
      "activations/layer17_attention_weight_max": 55.368751525878906,
      "activations/layer17_attention_weight_min": -44.18914794921875,
      "activations/layer18_attention_weight_max": 47.90914535522461,
      "activations/layer18_attention_weight_min": -37.52214813232422,
      "activations/layer19_attention_weight_max": 24.442113876342773,
      "activations/layer19_attention_weight_min": -21.491107940673828,
      "activations/layer1_attention_weight_max": 16.387771606445312,
      "activations/layer1_attention_weight_min": -13.92871379852295,
      "activations/layer20_attention_weight_max": 23.09892463684082,
      "activations/layer20_attention_weight_min": -20.272018432617188,
      "activations/layer21_attention_weight_max": 36.29012680053711,
      "activations/layer21_attention_weight_min": -23.36874771118164,
      "activations/layer22_attention_weight_max": 30.227163314819336,
      "activations/layer22_attention_weight_min": -23.346961975097656,
      "activations/layer23_attention_weight_max": 37.7630615234375,
      "activations/layer23_attention_weight_min": -25.571517944335938,
      "activations/layer2_attention_weight_max": 31.22150993347168,
      "activations/layer2_attention_weight_min": -29.473142623901367,
      "activations/layer3_attention_weight_max": 97.44253540039062,
      "activations/layer3_attention_weight_min": -98.25029754638672,
      "activations/layer4_attention_weight_max": 96.93575286865234,
      "activations/layer4_attention_weight_min": -92.78512573242188,
      "activations/layer5_attention_weight_max": 69.73336791992188,
      "activations/layer5_attention_weight_min": -80.5321044921875,
      "activations/layer6_attention_weight_max": 53.58626174926758,
      "activations/layer6_attention_weight_min": -51.13844299316406,
      "activations/layer7_attention_weight_max": 68.38204956054688,
      "activations/layer7_attention_weight_min": -64.42814636230469,
      "activations/layer8_attention_weight_max": 46.8963508605957,
      "activations/layer8_attention_weight_min": -50.42274475097656,
      "activations/layer9_attention_weight_max": 46.325077056884766,
      "activations/layer9_attention_weight_min": -48.67165756225586,
      "epoch": 14.06,
      "learning_rate": 5.99060606060606e-05,
      "loss": 2.7538,
      "step": 242050
    },
    {
      "activations/layer0_attention_weight_max": 16.78273582458496,
      "activations/layer0_attention_weight_min": -15.760820388793945,
      "activations/layer10_attention_weight_max": 33.77455520629883,
      "activations/layer10_attention_weight_min": -34.37150955200195,
      "activations/layer11_attention_weight_max": 29.7647705078125,
      "activations/layer11_attention_weight_min": -32.7407112121582,
      "activations/layer12_attention_weight_max": 17.59969711303711,
      "activations/layer12_attention_weight_min": -26.67558479309082,
      "activations/layer13_attention_weight_max": 40.41167449951172,
      "activations/layer13_attention_weight_min": -27.93075180053711,
      "activations/layer14_attention_weight_max": 37.29370880126953,
      "activations/layer14_attention_weight_min": -30.92011260986328,
      "activations/layer15_attention_weight_max": 32.032752990722656,
      "activations/layer15_attention_weight_min": -31.185754776000977,
      "activations/layer16_attention_weight_max": 28.71869659423828,
      "activations/layer16_attention_weight_min": -29.44359588623047,
      "activations/layer17_attention_weight_max": 47.40059280395508,
      "activations/layer17_attention_weight_min": -43.81563186645508,
      "activations/layer18_attention_weight_max": 42.82052993774414,
      "activations/layer18_attention_weight_min": -37.608619689941406,
      "activations/layer19_attention_weight_max": 20.662540435791016,
      "activations/layer19_attention_weight_min": -22.830114364624023,
      "activations/layer1_attention_weight_max": 16.339923858642578,
      "activations/layer1_attention_weight_min": -14.38097095489502,
      "activations/layer20_attention_weight_max": 21.541166305541992,
      "activations/layer20_attention_weight_min": -20.638113021850586,
      "activations/layer21_attention_weight_max": 35.8045654296875,
      "activations/layer21_attention_weight_min": -23.919248580932617,
      "activations/layer22_attention_weight_max": 29.43519401550293,
      "activations/layer22_attention_weight_min": -24.25797462463379,
      "activations/layer23_attention_weight_max": 35.99325942993164,
      "activations/layer23_attention_weight_min": -23.232763290405273,
      "activations/layer2_attention_weight_max": 30.570974349975586,
      "activations/layer2_attention_weight_min": -29.681808471679688,
      "activations/layer3_attention_weight_max": 91.2775650024414,
      "activations/layer3_attention_weight_min": -99.34622955322266,
      "activations/layer4_attention_weight_max": 89.97783660888672,
      "activations/layer4_attention_weight_min": -89.14144134521484,
      "activations/layer5_attention_weight_max": 68.46778869628906,
      "activations/layer5_attention_weight_min": -77.0404052734375,
      "activations/layer6_attention_weight_max": 49.654762268066406,
      "activations/layer6_attention_weight_min": -51.76300811767578,
      "activations/layer7_attention_weight_max": 62.93330001831055,
      "activations/layer7_attention_weight_min": -63.17021560668945,
      "activations/layer8_attention_weight_max": 43.12761306762695,
      "activations/layer8_attention_weight_min": -48.3511848449707,
      "activations/layer9_attention_weight_max": 42.50843048095703,
      "activations/layer9_attention_weight_min": -48.257625579833984,
      "epoch": 14.07,
      "learning_rate": 5.98871212121212e-05,
      "loss": 2.7399,
      "step": 242100
    },
    {
      "activations/layer0_attention_weight_max": 17.458812713623047,
      "activations/layer0_attention_weight_min": -14.101984024047852,
      "activations/layer10_attention_weight_max": 36.439205169677734,
      "activations/layer10_attention_weight_min": -36.201202392578125,
      "activations/layer11_attention_weight_max": 31.90114974975586,
      "activations/layer11_attention_weight_min": -34.63760757446289,
      "activations/layer12_attention_weight_max": 18.24979019165039,
      "activations/layer12_attention_weight_min": -24.365509033203125,
      "activations/layer13_attention_weight_max": 39.377403259277344,
      "activations/layer13_attention_weight_min": -28.04326629638672,
      "activations/layer14_attention_weight_max": 37.001739501953125,
      "activations/layer14_attention_weight_min": -32.39572525024414,
      "activations/layer15_attention_weight_max": 33.433937072753906,
      "activations/layer15_attention_weight_min": -30.31159782409668,
      "activations/layer16_attention_weight_max": 32.73276138305664,
      "activations/layer16_attention_weight_min": -32.65950393676758,
      "activations/layer17_attention_weight_max": 55.3390998840332,
      "activations/layer17_attention_weight_min": -46.61323547363281,
      "activations/layer18_attention_weight_max": 49.06623840332031,
      "activations/layer18_attention_weight_min": -40.83589172363281,
      "activations/layer19_attention_weight_max": 27.244461059570312,
      "activations/layer19_attention_weight_min": -23.023841857910156,
      "activations/layer1_attention_weight_max": 17.212026596069336,
      "activations/layer1_attention_weight_min": -15.272865295410156,
      "activations/layer20_attention_weight_max": 23.55059051513672,
      "activations/layer20_attention_weight_min": -24.617834091186523,
      "activations/layer21_attention_weight_max": 32.12656784057617,
      "activations/layer21_attention_weight_min": -22.440967559814453,
      "activations/layer22_attention_weight_max": 32.822811126708984,
      "activations/layer22_attention_weight_min": -25.60342788696289,
      "activations/layer23_attention_weight_max": 35.15494155883789,
      "activations/layer23_attention_weight_min": -25.174161911010742,
      "activations/layer2_attention_weight_max": 31.629682540893555,
      "activations/layer2_attention_weight_min": -30.383275985717773,
      "activations/layer3_attention_weight_max": 95.67594146728516,
      "activations/layer3_attention_weight_min": -99.04216766357422,
      "activations/layer4_attention_weight_max": 92.85334014892578,
      "activations/layer4_attention_weight_min": -92.08100891113281,
      "activations/layer5_attention_weight_max": 70.27948760986328,
      "activations/layer5_attention_weight_min": -78.46359252929688,
      "activations/layer6_attention_weight_max": 52.62936782836914,
      "activations/layer6_attention_weight_min": -53.02132034301758,
      "activations/layer7_attention_weight_max": 65.16222381591797,
      "activations/layer7_attention_weight_min": -66.13746643066406,
      "activations/layer8_attention_weight_max": 44.3884162902832,
      "activations/layer8_attention_weight_min": -47.78266525268555,
      "activations/layer9_attention_weight_max": 43.71311950683594,
      "activations/layer9_attention_weight_min": -48.464962005615234,
      "epoch": 14.07,
      "learning_rate": 5.986818181818181e-05,
      "loss": 2.7546,
      "step": 242150
    },
    {
      "activations/layer0_attention_weight_max": 15.866545677185059,
      "activations/layer0_attention_weight_min": -14.546701431274414,
      "activations/layer10_attention_weight_max": 38.581016540527344,
      "activations/layer10_attention_weight_min": -39.31537628173828,
      "activations/layer11_attention_weight_max": 35.0499382019043,
      "activations/layer11_attention_weight_min": -37.53836441040039,
      "activations/layer12_attention_weight_max": 20.85772705078125,
      "activations/layer12_attention_weight_min": -24.827030181884766,
      "activations/layer13_attention_weight_max": 41.49482727050781,
      "activations/layer13_attention_weight_min": -32.532814025878906,
      "activations/layer14_attention_weight_max": 44.95657730102539,
      "activations/layer14_attention_weight_min": -31.154542922973633,
      "activations/layer15_attention_weight_max": 42.30024719238281,
      "activations/layer15_attention_weight_min": -31.657617568969727,
      "activations/layer16_attention_weight_max": 32.031436920166016,
      "activations/layer16_attention_weight_min": -31.145349502563477,
      "activations/layer17_attention_weight_max": 50.01921463012695,
      "activations/layer17_attention_weight_min": -43.32680130004883,
      "activations/layer18_attention_weight_max": 47.65492630004883,
      "activations/layer18_attention_weight_min": -38.497222900390625,
      "activations/layer19_attention_weight_max": 22.365991592407227,
      "activations/layer19_attention_weight_min": -23.768449783325195,
      "activations/layer1_attention_weight_max": 17.340051651000977,
      "activations/layer1_attention_weight_min": -15.762110710144043,
      "activations/layer20_attention_weight_max": 23.061729431152344,
      "activations/layer20_attention_weight_min": -24.51049041748047,
      "activations/layer21_attention_weight_max": 35.67291259765625,
      "activations/layer21_attention_weight_min": -24.044755935668945,
      "activations/layer22_attention_weight_max": 29.682941436767578,
      "activations/layer22_attention_weight_min": -26.11953353881836,
      "activations/layer23_attention_weight_max": 36.968055725097656,
      "activations/layer23_attention_weight_min": -27.257854461669922,
      "activations/layer2_attention_weight_max": 32.148040771484375,
      "activations/layer2_attention_weight_min": -29.823612213134766,
      "activations/layer3_attention_weight_max": 94.43010711669922,
      "activations/layer3_attention_weight_min": -99.33389282226562,
      "activations/layer4_attention_weight_max": 95.54373168945312,
      "activations/layer4_attention_weight_min": -94.97017669677734,
      "activations/layer5_attention_weight_max": 79.39881134033203,
      "activations/layer5_attention_weight_min": -81.60255432128906,
      "activations/layer6_attention_weight_max": 56.497413635253906,
      "activations/layer6_attention_weight_min": -58.407230377197266,
      "activations/layer7_attention_weight_max": 74.59620666503906,
      "activations/layer7_attention_weight_min": -68.96328735351562,
      "activations/layer8_attention_weight_max": 47.894474029541016,
      "activations/layer8_attention_weight_min": -50.654136657714844,
      "activations/layer9_attention_weight_max": 52.560420989990234,
      "activations/layer9_attention_weight_min": -49.004051208496094,
      "epoch": 14.07,
      "learning_rate": 5.984924242424242e-05,
      "loss": 2.7399,
      "step": 242200
    },
    {
      "activations/layer0_attention_weight_max": 15.794557571411133,
      "activations/layer0_attention_weight_min": -14.132359504699707,
      "activations/layer10_attention_weight_max": 39.586761474609375,
      "activations/layer10_attention_weight_min": -40.60511016845703,
      "activations/layer11_attention_weight_max": 33.370216369628906,
      "activations/layer11_attention_weight_min": -33.99000930786133,
      "activations/layer12_attention_weight_max": 20.1732177734375,
      "activations/layer12_attention_weight_min": -23.482505798339844,
      "activations/layer13_attention_weight_max": 45.357139587402344,
      "activations/layer13_attention_weight_min": -33.189186096191406,
      "activations/layer14_attention_weight_max": 41.7058219909668,
      "activations/layer14_attention_weight_min": -32.20800018310547,
      "activations/layer15_attention_weight_max": 32.95159912109375,
      "activations/layer15_attention_weight_min": -31.705724716186523,
      "activations/layer16_attention_weight_max": 31.959375381469727,
      "activations/layer16_attention_weight_min": -31.0521240234375,
      "activations/layer17_attention_weight_max": 53.127140045166016,
      "activations/layer17_attention_weight_min": -44.28953552246094,
      "activations/layer18_attention_weight_max": 46.982234954833984,
      "activations/layer18_attention_weight_min": -37.91456604003906,
      "activations/layer19_attention_weight_max": 25.900455474853516,
      "activations/layer19_attention_weight_min": -23.153852462768555,
      "activations/layer1_attention_weight_max": 16.057849884033203,
      "activations/layer1_attention_weight_min": -14.393115043640137,
      "activations/layer20_attention_weight_max": 22.803255081176758,
      "activations/layer20_attention_weight_min": -21.232816696166992,
      "activations/layer21_attention_weight_max": 34.62358856201172,
      "activations/layer21_attention_weight_min": -24.483123779296875,
      "activations/layer22_attention_weight_max": 32.348777770996094,
      "activations/layer22_attention_weight_min": -23.949399948120117,
      "activations/layer23_attention_weight_max": 37.97400665283203,
      "activations/layer23_attention_weight_min": -23.109745025634766,
      "activations/layer2_attention_weight_max": 30.621414184570312,
      "activations/layer2_attention_weight_min": -29.326568603515625,
      "activations/layer3_attention_weight_max": 91.51465606689453,
      "activations/layer3_attention_weight_min": -93.75439453125,
      "activations/layer4_attention_weight_max": 94.36722564697266,
      "activations/layer4_attention_weight_min": -97.43891906738281,
      "activations/layer5_attention_weight_max": 74.03288269042969,
      "activations/layer5_attention_weight_min": -79.05989837646484,
      "activations/layer6_attention_weight_max": 51.83644104003906,
      "activations/layer6_attention_weight_min": -55.32716369628906,
      "activations/layer7_attention_weight_max": 70.4691162109375,
      "activations/layer7_attention_weight_min": -66.90586853027344,
      "activations/layer8_attention_weight_max": 48.87046432495117,
      "activations/layer8_attention_weight_min": -50.422706604003906,
      "activations/layer9_attention_weight_max": 47.94000244140625,
      "activations/layer9_attention_weight_min": -49.10891342163086,
      "epoch": 14.08,
      "learning_rate": 5.983030303030303e-05,
      "loss": 2.7271,
      "step": 242250
    },
    {
      "activations/layer0_attention_weight_max": 15.843417167663574,
      "activations/layer0_attention_weight_min": -14.015530586242676,
      "activations/layer10_attention_weight_max": 36.73971939086914,
      "activations/layer10_attention_weight_min": -36.26332092285156,
      "activations/layer11_attention_weight_max": 32.31100845336914,
      "activations/layer11_attention_weight_min": -33.121761322021484,
      "activations/layer12_attention_weight_max": 18.329425811767578,
      "activations/layer12_attention_weight_min": -24.45032501220703,
      "activations/layer13_attention_weight_max": 37.790061950683594,
      "activations/layer13_attention_weight_min": -30.15768051147461,
      "activations/layer14_attention_weight_max": 34.039344787597656,
      "activations/layer14_attention_weight_min": -29.459205627441406,
      "activations/layer15_attention_weight_max": 33.24338912963867,
      "activations/layer15_attention_weight_min": -31.124052047729492,
      "activations/layer16_attention_weight_max": 32.65029525756836,
      "activations/layer16_attention_weight_min": -30.70693016052246,
      "activations/layer17_attention_weight_max": 50.6254768371582,
      "activations/layer17_attention_weight_min": -46.91996383666992,
      "activations/layer18_attention_weight_max": 47.47673034667969,
      "activations/layer18_attention_weight_min": -41.345340728759766,
      "activations/layer19_attention_weight_max": 22.23244285583496,
      "activations/layer19_attention_weight_min": -22.955015182495117,
      "activations/layer1_attention_weight_max": 16.16405487060547,
      "activations/layer1_attention_weight_min": -16.22242546081543,
      "activations/layer20_attention_weight_max": 21.192495346069336,
      "activations/layer20_attention_weight_min": -23.693307876586914,
      "activations/layer21_attention_weight_max": 33.634334564208984,
      "activations/layer21_attention_weight_min": -22.815959930419922,
      "activations/layer22_attention_weight_max": 29.692781448364258,
      "activations/layer22_attention_weight_min": -28.116628646850586,
      "activations/layer23_attention_weight_max": 35.972511291503906,
      "activations/layer23_attention_weight_min": -25.08453369140625,
      "activations/layer2_attention_weight_max": 31.439178466796875,
      "activations/layer2_attention_weight_min": -31.24846649169922,
      "activations/layer3_attention_weight_max": 94.16834259033203,
      "activations/layer3_attention_weight_min": -99.77888488769531,
      "activations/layer4_attention_weight_max": 97.52481079101562,
      "activations/layer4_attention_weight_min": -94.63175201416016,
      "activations/layer5_attention_weight_max": 73.37390899658203,
      "activations/layer5_attention_weight_min": -78.64912414550781,
      "activations/layer6_attention_weight_max": 52.79493713378906,
      "activations/layer6_attention_weight_min": -52.66044616699219,
      "activations/layer7_attention_weight_max": 65.37638092041016,
      "activations/layer7_attention_weight_min": -70.04328155517578,
      "activations/layer8_attention_weight_max": 45.75849914550781,
      "activations/layer8_attention_weight_min": -48.8275146484375,
      "activations/layer9_attention_weight_max": 45.38730239868164,
      "activations/layer9_attention_weight_min": -48.31123352050781,
      "epoch": 14.08,
      "learning_rate": 5.981136363636363e-05,
      "loss": 2.7394,
      "step": 242300
    },
    {
      "activations/layer0_attention_weight_max": 16.704145431518555,
      "activations/layer0_attention_weight_min": -13.825003623962402,
      "activations/layer10_attention_weight_max": 39.16669845581055,
      "activations/layer10_attention_weight_min": -38.09683609008789,
      "activations/layer11_attention_weight_max": 35.19407653808594,
      "activations/layer11_attention_weight_min": -34.51885223388672,
      "activations/layer12_attention_weight_max": 19.30413818359375,
      "activations/layer12_attention_weight_min": -25.828285217285156,
      "activations/layer13_attention_weight_max": 45.01469421386719,
      "activations/layer13_attention_weight_min": -30.378944396972656,
      "activations/layer14_attention_weight_max": 40.207191467285156,
      "activations/layer14_attention_weight_min": -31.537797927856445,
      "activations/layer15_attention_weight_max": 43.60280990600586,
      "activations/layer15_attention_weight_min": -30.769363403320312,
      "activations/layer16_attention_weight_max": 34.02184295654297,
      "activations/layer16_attention_weight_min": -30.857385635375977,
      "activations/layer17_attention_weight_max": 56.79015350341797,
      "activations/layer17_attention_weight_min": -45.70164489746094,
      "activations/layer18_attention_weight_max": 51.101505279541016,
      "activations/layer18_attention_weight_min": -40.18478775024414,
      "activations/layer19_attention_weight_max": 27.040212631225586,
      "activations/layer19_attention_weight_min": -25.83900260925293,
      "activations/layer1_attention_weight_max": 17.36237144470215,
      "activations/layer1_attention_weight_min": -14.786989212036133,
      "activations/layer20_attention_weight_max": 24.954368591308594,
      "activations/layer20_attention_weight_min": -24.193281173706055,
      "activations/layer21_attention_weight_max": 48.23675537109375,
      "activations/layer21_attention_weight_min": -28.29779624938965,
      "activations/layer22_attention_weight_max": 34.00688552856445,
      "activations/layer22_attention_weight_min": -28.613101959228516,
      "activations/layer23_attention_weight_max": 42.11524963378906,
      "activations/layer23_attention_weight_min": -27.07904815673828,
      "activations/layer2_attention_weight_max": 32.380374908447266,
      "activations/layer2_attention_weight_min": -31.424413681030273,
      "activations/layer3_attention_weight_max": 95.616943359375,
      "activations/layer3_attention_weight_min": -106.61334228515625,
      "activations/layer4_attention_weight_max": 94.50603485107422,
      "activations/layer4_attention_weight_min": -96.16661834716797,
      "activations/layer5_attention_weight_max": 70.87236785888672,
      "activations/layer5_attention_weight_min": -79.50872039794922,
      "activations/layer6_attention_weight_max": 51.70827102661133,
      "activations/layer6_attention_weight_min": -53.00537872314453,
      "activations/layer7_attention_weight_max": 67.8950424194336,
      "activations/layer7_attention_weight_min": -65.74964141845703,
      "activations/layer8_attention_weight_max": 47.99079895019531,
      "activations/layer8_attention_weight_min": -51.82209396362305,
      "activations/layer9_attention_weight_max": 47.42048263549805,
      "activations/layer9_attention_weight_min": -49.30415725708008,
      "epoch": 14.08,
      "learning_rate": 5.979242424242423e-05,
      "loss": 2.749,
      "step": 242350
    },
    {
      "activations/layer0_attention_weight_max": 15.317152976989746,
      "activations/layer0_attention_weight_min": -13.855186462402344,
      "activations/layer10_attention_weight_max": 38.46323776245117,
      "activations/layer10_attention_weight_min": -36.07976150512695,
      "activations/layer11_attention_weight_max": 31.699947357177734,
      "activations/layer11_attention_weight_min": -34.643760681152344,
      "activations/layer12_attention_weight_max": 18.56931495666504,
      "activations/layer12_attention_weight_min": -27.2725887298584,
      "activations/layer13_attention_weight_max": 32.41886901855469,
      "activations/layer13_attention_weight_min": -30.675689697265625,
      "activations/layer14_attention_weight_max": 37.442386627197266,
      "activations/layer14_attention_weight_min": -31.429216384887695,
      "activations/layer15_attention_weight_max": 32.067996978759766,
      "activations/layer15_attention_weight_min": -30.413482666015625,
      "activations/layer16_attention_weight_max": 31.716474533081055,
      "activations/layer16_attention_weight_min": -29.52496910095215,
      "activations/layer17_attention_weight_max": 49.47300720214844,
      "activations/layer17_attention_weight_min": -43.11833190917969,
      "activations/layer18_attention_weight_max": 44.52051544189453,
      "activations/layer18_attention_weight_min": -37.31315994262695,
      "activations/layer19_attention_weight_max": 22.898969650268555,
      "activations/layer19_attention_weight_min": -21.944416046142578,
      "activations/layer1_attention_weight_max": 16.608470916748047,
      "activations/layer1_attention_weight_min": -15.659513473510742,
      "activations/layer20_attention_weight_max": 21.783123016357422,
      "activations/layer20_attention_weight_min": -23.987186431884766,
      "activations/layer21_attention_weight_max": 30.88637351989746,
      "activations/layer21_attention_weight_min": -23.69207000732422,
      "activations/layer22_attention_weight_max": 27.559656143188477,
      "activations/layer22_attention_weight_min": -25.954635620117188,
      "activations/layer23_attention_weight_max": 34.446624755859375,
      "activations/layer23_attention_weight_min": -23.447330474853516,
      "activations/layer2_attention_weight_max": 31.351890563964844,
      "activations/layer2_attention_weight_min": -30.154621124267578,
      "activations/layer3_attention_weight_max": 92.20306396484375,
      "activations/layer3_attention_weight_min": -96.62158203125,
      "activations/layer4_attention_weight_max": 93.65541076660156,
      "activations/layer4_attention_weight_min": -90.82418060302734,
      "activations/layer5_attention_weight_max": 71.58592987060547,
      "activations/layer5_attention_weight_min": -78.87987518310547,
      "activations/layer6_attention_weight_max": 48.97848892211914,
      "activations/layer6_attention_weight_min": -49.803470611572266,
      "activations/layer7_attention_weight_max": 66.49728393554688,
      "activations/layer7_attention_weight_min": -63.610904693603516,
      "activations/layer8_attention_weight_max": 44.492618560791016,
      "activations/layer8_attention_weight_min": -46.286800384521484,
      "activations/layer9_attention_weight_max": 45.568111419677734,
      "activations/layer9_attention_weight_min": -46.18798828125,
      "epoch": 14.08,
      "learning_rate": 5.9773484848484846e-05,
      "loss": 2.7424,
      "step": 242400
    },
    {
      "activations/layer0_attention_weight_max": 16.416887283325195,
      "activations/layer0_attention_weight_min": -14.026732444763184,
      "activations/layer10_attention_weight_max": 40.163421630859375,
      "activations/layer10_attention_weight_min": -36.66603469848633,
      "activations/layer11_attention_weight_max": 32.70246124267578,
      "activations/layer11_attention_weight_min": -34.085296630859375,
      "activations/layer12_attention_weight_max": 18.926820755004883,
      "activations/layer12_attention_weight_min": -27.07093620300293,
      "activations/layer13_attention_weight_max": 33.499412536621094,
      "activations/layer13_attention_weight_min": -29.76646614074707,
      "activations/layer14_attention_weight_max": 36.154048919677734,
      "activations/layer14_attention_weight_min": -30.483993530273438,
      "activations/layer15_attention_weight_max": 32.223445892333984,
      "activations/layer15_attention_weight_min": -30.378562927246094,
      "activations/layer16_attention_weight_max": 33.13653564453125,
      "activations/layer16_attention_weight_min": -30.340221405029297,
      "activations/layer17_attention_weight_max": 51.64684295654297,
      "activations/layer17_attention_weight_min": -44.50126647949219,
      "activations/layer18_attention_weight_max": 43.910499572753906,
      "activations/layer18_attention_weight_min": -38.95494842529297,
      "activations/layer19_attention_weight_max": 21.6773624420166,
      "activations/layer19_attention_weight_min": -22.27073860168457,
      "activations/layer1_attention_weight_max": 16.630754470825195,
      "activations/layer1_attention_weight_min": -15.784868240356445,
      "activations/layer20_attention_weight_max": 20.958314895629883,
      "activations/layer20_attention_weight_min": -23.696767807006836,
      "activations/layer21_attention_weight_max": 37.10739517211914,
      "activations/layer21_attention_weight_min": -25.30776023864746,
      "activations/layer22_attention_weight_max": 26.970619201660156,
      "activations/layer22_attention_weight_min": -28.00726890563965,
      "activations/layer23_attention_weight_max": 39.49898147583008,
      "activations/layer23_attention_weight_min": -26.29210662841797,
      "activations/layer2_attention_weight_max": 31.814870834350586,
      "activations/layer2_attention_weight_min": -30.596031188964844,
      "activations/layer3_attention_weight_max": 96.73372650146484,
      "activations/layer3_attention_weight_min": -98.4834976196289,
      "activations/layer4_attention_weight_max": 100.59646606445312,
      "activations/layer4_attention_weight_min": -93.13229370117188,
      "activations/layer5_attention_weight_max": 72.16444396972656,
      "activations/layer5_attention_weight_min": -78.09147644042969,
      "activations/layer6_attention_weight_max": 52.86898422241211,
      "activations/layer6_attention_weight_min": -55.944862365722656,
      "activations/layer7_attention_weight_max": 68.10234832763672,
      "activations/layer7_attention_weight_min": -66.14546203613281,
      "activations/layer8_attention_weight_max": 49.957332611083984,
      "activations/layer8_attention_weight_min": -51.90858840942383,
      "activations/layer9_attention_weight_max": 44.67754364013672,
      "activations/layer9_attention_weight_min": -52.37146759033203,
      "epoch": 14.09,
      "learning_rate": 5.975454545454545e-05,
      "loss": 2.7605,
      "step": 242450
    },
    {
      "activations/layer0_attention_weight_max": 15.915680885314941,
      "activations/layer0_attention_weight_min": -13.687759399414062,
      "activations/layer10_attention_weight_max": 36.44338607788086,
      "activations/layer10_attention_weight_min": -36.14344024658203,
      "activations/layer11_attention_weight_max": 30.943866729736328,
      "activations/layer11_attention_weight_min": -33.390357971191406,
      "activations/layer12_attention_weight_max": 20.106616973876953,
      "activations/layer12_attention_weight_min": -25.945377349853516,
      "activations/layer13_attention_weight_max": 32.94902038574219,
      "activations/layer13_attention_weight_min": -29.207225799560547,
      "activations/layer14_attention_weight_max": 35.86793518066406,
      "activations/layer14_attention_weight_min": -31.33355140686035,
      "activations/layer15_attention_weight_max": 32.50142288208008,
      "activations/layer15_attention_weight_min": -30.25616455078125,
      "activations/layer16_attention_weight_max": 33.35312271118164,
      "activations/layer16_attention_weight_min": -31.194185256958008,
      "activations/layer17_attention_weight_max": 56.38700866699219,
      "activations/layer17_attention_weight_min": -47.90119552612305,
      "activations/layer18_attention_weight_max": 50.40666961669922,
      "activations/layer18_attention_weight_min": -39.45137023925781,
      "activations/layer19_attention_weight_max": 21.978137969970703,
      "activations/layer19_attention_weight_min": -21.936431884765625,
      "activations/layer1_attention_weight_max": 16.682043075561523,
      "activations/layer1_attention_weight_min": -14.511507034301758,
      "activations/layer20_attention_weight_max": 23.59625816345215,
      "activations/layer20_attention_weight_min": -21.713869094848633,
      "activations/layer21_attention_weight_max": 40.44539260864258,
      "activations/layer21_attention_weight_min": -21.080379486083984,
      "activations/layer22_attention_weight_max": 29.729415893554688,
      "activations/layer22_attention_weight_min": -26.182695388793945,
      "activations/layer23_attention_weight_max": 32.708763122558594,
      "activations/layer23_attention_weight_min": -22.584503173828125,
      "activations/layer2_attention_weight_max": 31.12521743774414,
      "activations/layer2_attention_weight_min": -29.740360260009766,
      "activations/layer3_attention_weight_max": 93.22889709472656,
      "activations/layer3_attention_weight_min": -96.23126220703125,
      "activations/layer4_attention_weight_max": 92.65172576904297,
      "activations/layer4_attention_weight_min": -93.36864471435547,
      "activations/layer5_attention_weight_max": 71.76219177246094,
      "activations/layer5_attention_weight_min": -76.39790344238281,
      "activations/layer6_attention_weight_max": 51.87688446044922,
      "activations/layer6_attention_weight_min": -50.44233322143555,
      "activations/layer7_attention_weight_max": 64.60579681396484,
      "activations/layer7_attention_weight_min": -61.80667495727539,
      "activations/layer8_attention_weight_max": 43.58491897583008,
      "activations/layer8_attention_weight_min": -49.41576385498047,
      "activations/layer9_attention_weight_max": 44.00774002075195,
      "activations/layer9_attention_weight_min": -46.32191467285156,
      "epoch": 14.09,
      "learning_rate": 5.9735606060606055e-05,
      "loss": 2.7514,
      "step": 242500
    },
    {
      "activations/layer0_attention_weight_max": 15.84182071685791,
      "activations/layer0_attention_weight_min": -13.894048690795898,
      "activations/layer10_attention_weight_max": 34.04866027832031,
      "activations/layer10_attention_weight_min": -35.22827911376953,
      "activations/layer11_attention_weight_max": 32.02167510986328,
      "activations/layer11_attention_weight_min": -30.46749496459961,
      "activations/layer12_attention_weight_max": 18.415348052978516,
      "activations/layer12_attention_weight_min": -27.333152770996094,
      "activations/layer13_attention_weight_max": 36.962646484375,
      "activations/layer13_attention_weight_min": -29.83835220336914,
      "activations/layer14_attention_weight_max": 34.910274505615234,
      "activations/layer14_attention_weight_min": -31.208602905273438,
      "activations/layer15_attention_weight_max": 31.72231101989746,
      "activations/layer15_attention_weight_min": -30.093488693237305,
      "activations/layer16_attention_weight_max": 31.481033325195312,
      "activations/layer16_attention_weight_min": -32.12489700317383,
      "activations/layer17_attention_weight_max": 49.97319030761719,
      "activations/layer17_attention_weight_min": -44.216949462890625,
      "activations/layer18_attention_weight_max": 45.26352310180664,
      "activations/layer18_attention_weight_min": -40.22998046875,
      "activations/layer19_attention_weight_max": 22.73029136657715,
      "activations/layer19_attention_weight_min": -22.987768173217773,
      "activations/layer1_attention_weight_max": 16.334917068481445,
      "activations/layer1_attention_weight_min": -15.238995552062988,
      "activations/layer20_attention_weight_max": 22.198402404785156,
      "activations/layer20_attention_weight_min": -22.866619110107422,
      "activations/layer21_attention_weight_max": 36.172096252441406,
      "activations/layer21_attention_weight_min": -22.489395141601562,
      "activations/layer22_attention_weight_max": 28.48615837097168,
      "activations/layer22_attention_weight_min": -26.35957908630371,
      "activations/layer23_attention_weight_max": 37.16367721557617,
      "activations/layer23_attention_weight_min": -24.32601547241211,
      "activations/layer2_attention_weight_max": 32.05269241333008,
      "activations/layer2_attention_weight_min": -31.00880241394043,
      "activations/layer3_attention_weight_max": 92.7378921508789,
      "activations/layer3_attention_weight_min": -95.97350311279297,
      "activations/layer4_attention_weight_max": 90.6457290649414,
      "activations/layer4_attention_weight_min": -90.01647186279297,
      "activations/layer5_attention_weight_max": 72.33867645263672,
      "activations/layer5_attention_weight_min": -79.19566345214844,
      "activations/layer6_attention_weight_max": 50.588401794433594,
      "activations/layer6_attention_weight_min": -49.48667526245117,
      "activations/layer7_attention_weight_max": 64.0063247680664,
      "activations/layer7_attention_weight_min": -61.23579025268555,
      "activations/layer8_attention_weight_max": 42.28815841674805,
      "activations/layer8_attention_weight_min": -45.17561721801758,
      "activations/layer9_attention_weight_max": 41.56279373168945,
      "activations/layer9_attention_weight_min": -44.77359390258789,
      "epoch": 14.09,
      "learning_rate": 5.9716666666666664e-05,
      "loss": 2.7499,
      "step": 242550
    },
    {
      "activations/layer0_attention_weight_max": 16.054702758789062,
      "activations/layer0_attention_weight_min": -14.442630767822266,
      "activations/layer10_attention_weight_max": 38.62317657470703,
      "activations/layer10_attention_weight_min": -36.2549934387207,
      "activations/layer11_attention_weight_max": 33.07221984863281,
      "activations/layer11_attention_weight_min": -32.36712646484375,
      "activations/layer12_attention_weight_max": 22.42837142944336,
      "activations/layer12_attention_weight_min": -27.445131301879883,
      "activations/layer13_attention_weight_max": 37.31728744506836,
      "activations/layer13_attention_weight_min": -30.171676635742188,
      "activations/layer14_attention_weight_max": 36.16357421875,
      "activations/layer14_attention_weight_min": -31.024883270263672,
      "activations/layer15_attention_weight_max": 35.00599670410156,
      "activations/layer15_attention_weight_min": -30.82000160217285,
      "activations/layer16_attention_weight_max": 31.834728240966797,
      "activations/layer16_attention_weight_min": -29.895463943481445,
      "activations/layer17_attention_weight_max": 48.728294372558594,
      "activations/layer17_attention_weight_min": -42.4174919128418,
      "activations/layer18_attention_weight_max": 46.89665985107422,
      "activations/layer18_attention_weight_min": -38.75362777709961,
      "activations/layer19_attention_weight_max": 27.559370040893555,
      "activations/layer19_attention_weight_min": -22.422727584838867,
      "activations/layer1_attention_weight_max": 16.095291137695312,
      "activations/layer1_attention_weight_min": -14.308775901794434,
      "activations/layer20_attention_weight_max": 24.05194854736328,
      "activations/layer20_attention_weight_min": -20.931121826171875,
      "activations/layer21_attention_weight_max": 36.41943359375,
      "activations/layer21_attention_weight_min": -22.458776473999023,
      "activations/layer22_attention_weight_max": 29.183591842651367,
      "activations/layer22_attention_weight_min": -26.532958984375,
      "activations/layer23_attention_weight_max": 37.610008239746094,
      "activations/layer23_attention_weight_min": -25.72241973876953,
      "activations/layer2_attention_weight_max": 31.605487823486328,
      "activations/layer2_attention_weight_min": -30.976417541503906,
      "activations/layer3_attention_weight_max": 98.23400115966797,
      "activations/layer3_attention_weight_min": -100.54261016845703,
      "activations/layer4_attention_weight_max": 96.03848266601562,
      "activations/layer4_attention_weight_min": -95.08541107177734,
      "activations/layer5_attention_weight_max": 71.92771911621094,
      "activations/layer5_attention_weight_min": -80.56501770019531,
      "activations/layer6_attention_weight_max": 52.74759292602539,
      "activations/layer6_attention_weight_min": -53.22178649902344,
      "activations/layer7_attention_weight_max": 74.96929931640625,
      "activations/layer7_attention_weight_min": -64.69619750976562,
      "activations/layer8_attention_weight_max": 49.38036346435547,
      "activations/layer8_attention_weight_min": -51.39958572387695,
      "activations/layer9_attention_weight_max": 46.69717025756836,
      "activations/layer9_attention_weight_min": -46.9912223815918,
      "epoch": 14.1,
      "learning_rate": 5.969772727272727e-05,
      "loss": 2.7441,
      "step": 242600
    },
    {
      "activations/layer0_attention_weight_max": 16.380146026611328,
      "activations/layer0_attention_weight_min": -13.853464126586914,
      "activations/layer10_attention_weight_max": 35.8701057434082,
      "activations/layer10_attention_weight_min": -36.966793060302734,
      "activations/layer11_attention_weight_max": 31.717086791992188,
      "activations/layer11_attention_weight_min": -33.43769454956055,
      "activations/layer12_attention_weight_max": 18.94266128540039,
      "activations/layer12_attention_weight_min": -27.582923889160156,
      "activations/layer13_attention_weight_max": 40.278072357177734,
      "activations/layer13_attention_weight_min": -30.578950881958008,
      "activations/layer14_attention_weight_max": 34.0969123840332,
      "activations/layer14_attention_weight_min": -30.11538314819336,
      "activations/layer15_attention_weight_max": 31.207609176635742,
      "activations/layer15_attention_weight_min": -30.87102699279785,
      "activations/layer16_attention_weight_max": 33.674583435058594,
      "activations/layer16_attention_weight_min": -30.789871215820312,
      "activations/layer17_attention_weight_max": 55.06142807006836,
      "activations/layer17_attention_weight_min": -47.50470733642578,
      "activations/layer18_attention_weight_max": 46.971214294433594,
      "activations/layer18_attention_weight_min": -38.79296875,
      "activations/layer19_attention_weight_max": 25.24641227722168,
      "activations/layer19_attention_weight_min": -21.303953170776367,
      "activations/layer1_attention_weight_max": 16.83708953857422,
      "activations/layer1_attention_weight_min": -15.35611343383789,
      "activations/layer20_attention_weight_max": 21.873523712158203,
      "activations/layer20_attention_weight_min": -20.685197830200195,
      "activations/layer21_attention_weight_max": 32.17586135864258,
      "activations/layer21_attention_weight_min": -22.477590560913086,
      "activations/layer22_attention_weight_max": 29.25381851196289,
      "activations/layer22_attention_weight_min": -25.295326232910156,
      "activations/layer23_attention_weight_max": 34.298675537109375,
      "activations/layer23_attention_weight_min": -24.36664390563965,
      "activations/layer2_attention_weight_max": 31.626272201538086,
      "activations/layer2_attention_weight_min": -30.550962448120117,
      "activations/layer3_attention_weight_max": 92.8956069946289,
      "activations/layer3_attention_weight_min": -94.27120971679688,
      "activations/layer4_attention_weight_max": 92.62068939208984,
      "activations/layer4_attention_weight_min": -91.1080551147461,
      "activations/layer5_attention_weight_max": 68.32067108154297,
      "activations/layer5_attention_weight_min": -81.98648071289062,
      "activations/layer6_attention_weight_max": 51.35580825805664,
      "activations/layer6_attention_weight_min": -49.9998893737793,
      "activations/layer7_attention_weight_max": 68.80540466308594,
      "activations/layer7_attention_weight_min": -67.37126922607422,
      "activations/layer8_attention_weight_max": 45.73804473876953,
      "activations/layer8_attention_weight_min": -49.24142074584961,
      "activations/layer9_attention_weight_max": 44.68455505371094,
      "activations/layer9_attention_weight_min": -49.23112487792969,
      "epoch": 14.1,
      "learning_rate": 5.967878787878787e-05,
      "loss": 2.7464,
      "step": 242650
    },
    {
      "activations/layer0_attention_weight_max": 16.60500717163086,
      "activations/layer0_attention_weight_min": -14.00454044342041,
      "activations/layer10_attention_weight_max": 34.859840393066406,
      "activations/layer10_attention_weight_min": -36.07332229614258,
      "activations/layer11_attention_weight_max": 32.189674377441406,
      "activations/layer11_attention_weight_min": -32.907310485839844,
      "activations/layer12_attention_weight_max": 16.555641174316406,
      "activations/layer12_attention_weight_min": -23.996570587158203,
      "activations/layer13_attention_weight_max": 38.44867706298828,
      "activations/layer13_attention_weight_min": -27.87747573852539,
      "activations/layer14_attention_weight_max": 33.8834114074707,
      "activations/layer14_attention_weight_min": -31.03338050842285,
      "activations/layer15_attention_weight_max": 31.908082962036133,
      "activations/layer15_attention_weight_min": -28.189647674560547,
      "activations/layer16_attention_weight_max": 30.026411056518555,
      "activations/layer16_attention_weight_min": -29.73950958251953,
      "activations/layer17_attention_weight_max": 52.2012825012207,
      "activations/layer17_attention_weight_min": -44.2480583190918,
      "activations/layer18_attention_weight_max": 46.124446868896484,
      "activations/layer18_attention_weight_min": -40.47077560424805,
      "activations/layer19_attention_weight_max": 20.869754791259766,
      "activations/layer19_attention_weight_min": -24.39305877685547,
      "activations/layer1_attention_weight_max": 16.37911605834961,
      "activations/layer1_attention_weight_min": -16.289051055908203,
      "activations/layer20_attention_weight_max": 22.9565486907959,
      "activations/layer20_attention_weight_min": -22.384912490844727,
      "activations/layer21_attention_weight_max": 32.25014114379883,
      "activations/layer21_attention_weight_min": -21.02983283996582,
      "activations/layer22_attention_weight_max": 28.414947509765625,
      "activations/layer22_attention_weight_min": -25.947771072387695,
      "activations/layer23_attention_weight_max": 34.52756881713867,
      "activations/layer23_attention_weight_min": -24.436132431030273,
      "activations/layer2_attention_weight_max": 31.423328399658203,
      "activations/layer2_attention_weight_min": -29.669965744018555,
      "activations/layer3_attention_weight_max": 93.95741271972656,
      "activations/layer3_attention_weight_min": -95.68185424804688,
      "activations/layer4_attention_weight_max": 92.16777801513672,
      "activations/layer4_attention_weight_min": -87.9805908203125,
      "activations/layer5_attention_weight_max": 68.05404663085938,
      "activations/layer5_attention_weight_min": -76.79979705810547,
      "activations/layer6_attention_weight_max": 52.23555374145508,
      "activations/layer6_attention_weight_min": -54.37572479248047,
      "activations/layer7_attention_weight_max": 66.12921142578125,
      "activations/layer7_attention_weight_min": -70.18145751953125,
      "activations/layer8_attention_weight_max": 45.28374099731445,
      "activations/layer8_attention_weight_min": -46.77891540527344,
      "activations/layer9_attention_weight_max": 42.301753997802734,
      "activations/layer9_attention_weight_min": -47.61688995361328,
      "epoch": 14.1,
      "learning_rate": 5.9659848484848475e-05,
      "loss": 2.7619,
      "step": 242700
    },
    {
      "activations/layer0_attention_weight_max": 16.51573371887207,
      "activations/layer0_attention_weight_min": -14.273000717163086,
      "activations/layer10_attention_weight_max": 35.70109558105469,
      "activations/layer10_attention_weight_min": -35.988128662109375,
      "activations/layer11_attention_weight_max": 32.5299072265625,
      "activations/layer11_attention_weight_min": -35.82758331298828,
      "activations/layer12_attention_weight_max": 18.640371322631836,
      "activations/layer12_attention_weight_min": -29.82420539855957,
      "activations/layer13_attention_weight_max": 39.07084655761719,
      "activations/layer13_attention_weight_min": -28.378223419189453,
      "activations/layer14_attention_weight_max": 39.47394943237305,
      "activations/layer14_attention_weight_min": -31.766416549682617,
      "activations/layer15_attention_weight_max": 36.22037124633789,
      "activations/layer15_attention_weight_min": -28.994796752929688,
      "activations/layer16_attention_weight_max": 32.938663482666016,
      "activations/layer16_attention_weight_min": -29.82438850402832,
      "activations/layer17_attention_weight_max": 55.568546295166016,
      "activations/layer17_attention_weight_min": -46.705928802490234,
      "activations/layer18_attention_weight_max": 50.061893463134766,
      "activations/layer18_attention_weight_min": -38.87883758544922,
      "activations/layer19_attention_weight_max": 24.247175216674805,
      "activations/layer19_attention_weight_min": -22.524168014526367,
      "activations/layer1_attention_weight_max": 16.596099853515625,
      "activations/layer1_attention_weight_min": -13.948826789855957,
      "activations/layer20_attention_weight_max": 27.025352478027344,
      "activations/layer20_attention_weight_min": -21.247974395751953,
      "activations/layer21_attention_weight_max": 41.063392639160156,
      "activations/layer21_attention_weight_min": -24.353832244873047,
      "activations/layer22_attention_weight_max": 30.266742706298828,
      "activations/layer22_attention_weight_min": -25.25897789001465,
      "activations/layer23_attention_weight_max": 38.820777893066406,
      "activations/layer23_attention_weight_min": -23.388771057128906,
      "activations/layer2_attention_weight_max": 33.08526611328125,
      "activations/layer2_attention_weight_min": -30.400611877441406,
      "activations/layer3_attention_weight_max": 94.41251373291016,
      "activations/layer3_attention_weight_min": -96.84890747070312,
      "activations/layer4_attention_weight_max": 93.25102233886719,
      "activations/layer4_attention_weight_min": -90.58118438720703,
      "activations/layer5_attention_weight_max": 72.0489501953125,
      "activations/layer5_attention_weight_min": -77.44415283203125,
      "activations/layer6_attention_weight_max": 51.13017654418945,
      "activations/layer6_attention_weight_min": -52.22457504272461,
      "activations/layer7_attention_weight_max": 64.45289611816406,
      "activations/layer7_attention_weight_min": -64.27792358398438,
      "activations/layer8_attention_weight_max": 46.15879821777344,
      "activations/layer8_attention_weight_min": -47.2647590637207,
      "activations/layer9_attention_weight_max": 45.248435974121094,
      "activations/layer9_attention_weight_min": -46.25920486450195,
      "epoch": 14.11,
      "learning_rate": 5.964090909090909e-05,
      "loss": 2.7548,
      "step": 242750
    },
    {
      "activations/layer0_attention_weight_max": 16.360275268554688,
      "activations/layer0_attention_weight_min": -14.155107498168945,
      "activations/layer10_attention_weight_max": 32.56465148925781,
      "activations/layer10_attention_weight_min": -33.89991760253906,
      "activations/layer11_attention_weight_max": 30.813016891479492,
      "activations/layer11_attention_weight_min": -34.045867919921875,
      "activations/layer12_attention_weight_max": 18.20624542236328,
      "activations/layer12_attention_weight_min": -25.441303253173828,
      "activations/layer13_attention_weight_max": 38.11741256713867,
      "activations/layer13_attention_weight_min": -30.861703872680664,
      "activations/layer14_attention_weight_max": 35.708900451660156,
      "activations/layer14_attention_weight_min": -30.15645980834961,
      "activations/layer15_attention_weight_max": 31.728984832763672,
      "activations/layer15_attention_weight_min": -29.82954978942871,
      "activations/layer16_attention_weight_max": 32.69395446777344,
      "activations/layer16_attention_weight_min": -29.72661590576172,
      "activations/layer17_attention_weight_max": 50.3141975402832,
      "activations/layer17_attention_weight_min": -43.06404113769531,
      "activations/layer18_attention_weight_max": 45.918113708496094,
      "activations/layer18_attention_weight_min": -35.667083740234375,
      "activations/layer19_attention_weight_max": 26.965784072875977,
      "activations/layer19_attention_weight_min": -20.765756607055664,
      "activations/layer1_attention_weight_max": 17.0000057220459,
      "activations/layer1_attention_weight_min": -14.240150451660156,
      "activations/layer20_attention_weight_max": 23.890043258666992,
      "activations/layer20_attention_weight_min": -20.761642456054688,
      "activations/layer21_attention_weight_max": 37.78309631347656,
      "activations/layer21_attention_weight_min": -24.686399459838867,
      "activations/layer22_attention_weight_max": 32.65312576293945,
      "activations/layer22_attention_weight_min": -25.420516967773438,
      "activations/layer23_attention_weight_max": 38.46894454956055,
      "activations/layer23_attention_weight_min": -28.42793846130371,
      "activations/layer2_attention_weight_max": 31.065589904785156,
      "activations/layer2_attention_weight_min": -31.238813400268555,
      "activations/layer3_attention_weight_max": 93.80004119873047,
      "activations/layer3_attention_weight_min": -100.88284301757812,
      "activations/layer4_attention_weight_max": 88.61132049560547,
      "activations/layer4_attention_weight_min": -88.6451187133789,
      "activations/layer5_attention_weight_max": 67.2078857421875,
      "activations/layer5_attention_weight_min": -76.69406127929688,
      "activations/layer6_attention_weight_max": 49.57769012451172,
      "activations/layer6_attention_weight_min": -48.81703186035156,
      "activations/layer7_attention_weight_max": 62.47227096557617,
      "activations/layer7_attention_weight_min": -63.944923400878906,
      "activations/layer8_attention_weight_max": 40.855140686035156,
      "activations/layer8_attention_weight_min": -45.876583099365234,
      "activations/layer9_attention_weight_max": 41.34073257446289,
      "activations/layer9_attention_weight_min": -44.48237228393555,
      "epoch": 14.11,
      "learning_rate": 5.962196969696969e-05,
      "loss": 2.7494,
      "step": 242800
    },
    {
      "activations/layer0_attention_weight_max": 16.40167236328125,
      "activations/layer0_attention_weight_min": -13.992907524108887,
      "activations/layer10_attention_weight_max": 35.400917053222656,
      "activations/layer10_attention_weight_min": -35.04104995727539,
      "activations/layer11_attention_weight_max": 32.77940368652344,
      "activations/layer11_attention_weight_min": -33.615394592285156,
      "activations/layer12_attention_weight_max": 19.718734741210938,
      "activations/layer12_attention_weight_min": -23.84581184387207,
      "activations/layer13_attention_weight_max": 36.65243148803711,
      "activations/layer13_attention_weight_min": -28.294836044311523,
      "activations/layer14_attention_weight_max": 33.740665435791016,
      "activations/layer14_attention_weight_min": -29.65153694152832,
      "activations/layer15_attention_weight_max": 31.022979736328125,
      "activations/layer15_attention_weight_min": -28.88089942932129,
      "activations/layer16_attention_weight_max": 29.32676887512207,
      "activations/layer16_attention_weight_min": -29.004268646240234,
      "activations/layer17_attention_weight_max": 48.33407211303711,
      "activations/layer17_attention_weight_min": -42.78704833984375,
      "activations/layer18_attention_weight_max": 42.64990997314453,
      "activations/layer18_attention_weight_min": -37.763526916503906,
      "activations/layer19_attention_weight_max": 22.942506790161133,
      "activations/layer19_attention_weight_min": -20.636213302612305,
      "activations/layer1_attention_weight_max": 16.146183013916016,
      "activations/layer1_attention_weight_min": -14.578100204467773,
      "activations/layer20_attention_weight_max": 20.244108200073242,
      "activations/layer20_attention_weight_min": -23.603445053100586,
      "activations/layer21_attention_weight_max": 37.79159927368164,
      "activations/layer21_attention_weight_min": -22.506649017333984,
      "activations/layer22_attention_weight_max": 31.905914306640625,
      "activations/layer22_attention_weight_min": -24.295347213745117,
      "activations/layer23_attention_weight_max": 34.93245315551758,
      "activations/layer23_attention_weight_min": -25.061416625976562,
      "activations/layer2_attention_weight_max": 31.59400749206543,
      "activations/layer2_attention_weight_min": -31.11151885986328,
      "activations/layer3_attention_weight_max": 97.80890655517578,
      "activations/layer3_attention_weight_min": -97.81908416748047,
      "activations/layer4_attention_weight_max": 95.78292083740234,
      "activations/layer4_attention_weight_min": -90.8716049194336,
      "activations/layer5_attention_weight_max": 69.3221435546875,
      "activations/layer5_attention_weight_min": -75.14608764648438,
      "activations/layer6_attention_weight_max": 51.692691802978516,
      "activations/layer6_attention_weight_min": -51.426124572753906,
      "activations/layer7_attention_weight_max": 63.99192428588867,
      "activations/layer7_attention_weight_min": -62.18892288208008,
      "activations/layer8_attention_weight_max": 44.6088752746582,
      "activations/layer8_attention_weight_min": -47.52626419067383,
      "activations/layer9_attention_weight_max": 43.41339111328125,
      "activations/layer9_attention_weight_min": -45.17914962768555,
      "epoch": 14.11,
      "learning_rate": 5.96030303030303e-05,
      "loss": 2.7552,
      "step": 242850
    },
    {
      "activations/layer0_attention_weight_max": 17.123750686645508,
      "activations/layer0_attention_weight_min": -14.098335266113281,
      "activations/layer10_attention_weight_max": 34.03042221069336,
      "activations/layer10_attention_weight_min": -35.6844367980957,
      "activations/layer11_attention_weight_max": 32.7751350402832,
      "activations/layer11_attention_weight_min": -35.572540283203125,
      "activations/layer12_attention_weight_max": 17.116474151611328,
      "activations/layer12_attention_weight_min": -23.961755752563477,
      "activations/layer13_attention_weight_max": 39.75947570800781,
      "activations/layer13_attention_weight_min": -29.782167434692383,
      "activations/layer14_attention_weight_max": 34.78806686401367,
      "activations/layer14_attention_weight_min": -30.124889373779297,
      "activations/layer15_attention_weight_max": 30.05905532836914,
      "activations/layer15_attention_weight_min": -29.83985137939453,
      "activations/layer16_attention_weight_max": 33.416561126708984,
      "activations/layer16_attention_weight_min": -34.28772735595703,
      "activations/layer17_attention_weight_max": 51.90795135498047,
      "activations/layer17_attention_weight_min": -47.03961944580078,
      "activations/layer18_attention_weight_max": 49.89692306518555,
      "activations/layer18_attention_weight_min": -38.59380340576172,
      "activations/layer19_attention_weight_max": 22.14948272705078,
      "activations/layer19_attention_weight_min": -21.627897262573242,
      "activations/layer1_attention_weight_max": 16.495813369750977,
      "activations/layer1_attention_weight_min": -15.932007789611816,
      "activations/layer20_attention_weight_max": 20.785104751586914,
      "activations/layer20_attention_weight_min": -19.98881721496582,
      "activations/layer21_attention_weight_max": 32.359107971191406,
      "activations/layer21_attention_weight_min": -25.52260398864746,
      "activations/layer22_attention_weight_max": 27.4290714263916,
      "activations/layer22_attention_weight_min": -25.78917121887207,
      "activations/layer23_attention_weight_max": 34.37760543823242,
      "activations/layer23_attention_weight_min": -25.002899169921875,
      "activations/layer2_attention_weight_max": 31.459896087646484,
      "activations/layer2_attention_weight_min": -31.39151382446289,
      "activations/layer3_attention_weight_max": 96.37649536132812,
      "activations/layer3_attention_weight_min": -97.5868911743164,
      "activations/layer4_attention_weight_max": 95.24740600585938,
      "activations/layer4_attention_weight_min": -93.5647201538086,
      "activations/layer5_attention_weight_max": 74.32768249511719,
      "activations/layer5_attention_weight_min": -77.6298828125,
      "activations/layer6_attention_weight_max": 56.81711196899414,
      "activations/layer6_attention_weight_min": -58.4107666015625,
      "activations/layer7_attention_weight_max": 72.86042785644531,
      "activations/layer7_attention_weight_min": -65.1754150390625,
      "activations/layer8_attention_weight_max": 48.07379150390625,
      "activations/layer8_attention_weight_min": -49.46267318725586,
      "activations/layer9_attention_weight_max": 43.77478790283203,
      "activations/layer9_attention_weight_min": -46.454952239990234,
      "epoch": 14.11,
      "learning_rate": 5.95840909090909e-05,
      "loss": 2.7533,
      "step": 242900
    },
    {
      "activations/layer0_attention_weight_max": 16.253875732421875,
      "activations/layer0_attention_weight_min": -13.882193565368652,
      "activations/layer10_attention_weight_max": 34.07571029663086,
      "activations/layer10_attention_weight_min": -35.3667106628418,
      "activations/layer11_attention_weight_max": 31.439697265625,
      "activations/layer11_attention_weight_min": -33.062156677246094,
      "activations/layer12_attention_weight_max": 17.962106704711914,
      "activations/layer12_attention_weight_min": -25.526548385620117,
      "activations/layer13_attention_weight_max": 44.25257110595703,
      "activations/layer13_attention_weight_min": -30.019874572753906,
      "activations/layer14_attention_weight_max": 31.872821807861328,
      "activations/layer14_attention_weight_min": -29.653162002563477,
      "activations/layer15_attention_weight_max": 29.356279373168945,
      "activations/layer15_attention_weight_min": -28.937925338745117,
      "activations/layer16_attention_weight_max": 28.944252014160156,
      "activations/layer16_attention_weight_min": -30.999231338500977,
      "activations/layer17_attention_weight_max": 47.49919891357422,
      "activations/layer17_attention_weight_min": -41.98828887939453,
      "activations/layer18_attention_weight_max": 41.620262145996094,
      "activations/layer18_attention_weight_min": -35.88816452026367,
      "activations/layer19_attention_weight_max": 23.489492416381836,
      "activations/layer19_attention_weight_min": -21.513992309570312,
      "activations/layer1_attention_weight_max": 16.64250373840332,
      "activations/layer1_attention_weight_min": -14.991005897521973,
      "activations/layer20_attention_weight_max": 20.460432052612305,
      "activations/layer20_attention_weight_min": -20.281965255737305,
      "activations/layer21_attention_weight_max": 32.439048767089844,
      "activations/layer21_attention_weight_min": -21.8997802734375,
      "activations/layer22_attention_weight_max": 26.5682373046875,
      "activations/layer22_attention_weight_min": -25.598665237426758,
      "activations/layer23_attention_weight_max": 33.78575897216797,
      "activations/layer23_attention_weight_min": -27.64084243774414,
      "activations/layer2_attention_weight_max": 32.955299377441406,
      "activations/layer2_attention_weight_min": -30.426700592041016,
      "activations/layer3_attention_weight_max": 94.63388061523438,
      "activations/layer3_attention_weight_min": -93.91226196289062,
      "activations/layer4_attention_weight_max": 93.53624725341797,
      "activations/layer4_attention_weight_min": -91.27083587646484,
      "activations/layer5_attention_weight_max": 73.75079345703125,
      "activations/layer5_attention_weight_min": -79.08522033691406,
      "activations/layer6_attention_weight_max": 53.70560073852539,
      "activations/layer6_attention_weight_min": -53.157474517822266,
      "activations/layer7_attention_weight_max": 65.78451538085938,
      "activations/layer7_attention_weight_min": -64.7962646484375,
      "activations/layer8_attention_weight_max": 47.021202087402344,
      "activations/layer8_attention_weight_min": -51.683631896972656,
      "activations/layer9_attention_weight_max": 48.836429595947266,
      "activations/layer9_attention_weight_min": -47.77430725097656,
      "epoch": 14.12,
      "learning_rate": 5.9565151515151516e-05,
      "loss": 2.7552,
      "step": 242950
    },
    {
      "activations/layer0_attention_weight_max": 16.19206428527832,
      "activations/layer0_attention_weight_min": -14.205792427062988,
      "activations/layer10_attention_weight_max": 38.32367706298828,
      "activations/layer10_attention_weight_min": -35.77052688598633,
      "activations/layer11_attention_weight_max": 33.59575653076172,
      "activations/layer11_attention_weight_min": -34.49956130981445,
      "activations/layer12_attention_weight_max": 19.368389129638672,
      "activations/layer12_attention_weight_min": -25.56966781616211,
      "activations/layer13_attention_weight_max": 36.01279830932617,
      "activations/layer13_attention_weight_min": -29.715190887451172,
      "activations/layer14_attention_weight_max": 36.71881866455078,
      "activations/layer14_attention_weight_min": -29.875282287597656,
      "activations/layer15_attention_weight_max": 31.77035903930664,
      "activations/layer15_attention_weight_min": -30.281734466552734,
      "activations/layer16_attention_weight_max": 32.388736724853516,
      "activations/layer16_attention_weight_min": -32.10300064086914,
      "activations/layer17_attention_weight_max": 50.69029998779297,
      "activations/layer17_attention_weight_min": -44.365745544433594,
      "activations/layer18_attention_weight_max": 49.1345329284668,
      "activations/layer18_attention_weight_min": -37.032344818115234,
      "activations/layer19_attention_weight_max": 26.2591552734375,
      "activations/layer19_attention_weight_min": -22.11134147644043,
      "activations/layer1_attention_weight_max": 16.647354125976562,
      "activations/layer1_attention_weight_min": -15.421589851379395,
      "activations/layer20_attention_weight_max": 23.66526985168457,
      "activations/layer20_attention_weight_min": -20.42528533935547,
      "activations/layer21_attention_weight_max": 41.50053024291992,
      "activations/layer21_attention_weight_min": -22.48357582092285,
      "activations/layer22_attention_weight_max": 32.968292236328125,
      "activations/layer22_attention_weight_min": -24.45396614074707,
      "activations/layer23_attention_weight_max": 41.30610275268555,
      "activations/layer23_attention_weight_min": -23.269010543823242,
      "activations/layer2_attention_weight_max": 32.20999526977539,
      "activations/layer2_attention_weight_min": -31.794837951660156,
      "activations/layer3_attention_weight_max": 103.07539367675781,
      "activations/layer3_attention_weight_min": -103.11397552490234,
      "activations/layer4_attention_weight_max": 101.1916275024414,
      "activations/layer4_attention_weight_min": -98.83324432373047,
      "activations/layer5_attention_weight_max": 73.97915649414062,
      "activations/layer5_attention_weight_min": -81.72865295410156,
      "activations/layer6_attention_weight_max": 54.39918518066406,
      "activations/layer6_attention_weight_min": -55.11094665527344,
      "activations/layer7_attention_weight_max": 67.67755126953125,
      "activations/layer7_attention_weight_min": -66.16202545166016,
      "activations/layer8_attention_weight_max": 50.48582077026367,
      "activations/layer8_attention_weight_min": -52.622154235839844,
      "activations/layer9_attention_weight_max": 48.39158248901367,
      "activations/layer9_attention_weight_min": -50.53986740112305,
      "epoch": 14.12,
      "learning_rate": 5.954621212121212e-05,
      "loss": 2.7378,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4545,
      "eval_samples_per_second": 507.893,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4545,
      "eval_openwebtext_samples_per_second": 507.893,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_wikitext_loss": 2.947265625,
      "eval_wikitext_ppl": 19.053782246709744,
      "eval_wikitext_runtime": 1.9468,
      "eval_wikitext_samples_per_second": 234.233,
      "step": 243000
    },
    {
      "epoch": 14.12,
      "eval_lambada_loss": 2.533203125,
      "eval_lambada_ppl": 12.593781054375055,
      "eval_lambada_runtime": 9.5522,
      "eval_lambada_samples_per_second": 509.725,
      "step": 243000
    },
    {
      "activations/layer0_attention_weight_max": 16.104808807373047,
      "activations/layer0_attention_weight_min": -13.815315246582031,
      "activations/layer10_attention_weight_max": 39.22400665283203,
      "activations/layer10_attention_weight_min": -37.970306396484375,
      "activations/layer11_attention_weight_max": 35.87762451171875,
      "activations/layer11_attention_weight_min": -36.355472564697266,
      "activations/layer12_attention_weight_max": 18.60376739501953,
      "activations/layer12_attention_weight_min": -27.69746208190918,
      "activations/layer13_attention_weight_max": 38.48380661010742,
      "activations/layer13_attention_weight_min": -29.598398208618164,
      "activations/layer14_attention_weight_max": 32.80541229248047,
      "activations/layer14_attention_weight_min": -30.229063034057617,
      "activations/layer15_attention_weight_max": 32.2881965637207,
      "activations/layer15_attention_weight_min": -30.02013397216797,
      "activations/layer16_attention_weight_max": 33.26956558227539,
      "activations/layer16_attention_weight_min": -31.76700782775879,
      "activations/layer17_attention_weight_max": 53.66930389404297,
      "activations/layer17_attention_weight_min": -48.048824310302734,
      "activations/layer18_attention_weight_max": 49.262813568115234,
      "activations/layer18_attention_weight_min": -39.97370529174805,
      "activations/layer19_attention_weight_max": 21.847867965698242,
      "activations/layer19_attention_weight_min": -21.74993896484375,
      "activations/layer1_attention_weight_max": 16.708660125732422,
      "activations/layer1_attention_weight_min": -15.796571731567383,
      "activations/layer20_attention_weight_max": 20.516887664794922,
      "activations/layer20_attention_weight_min": -20.001140594482422,
      "activations/layer21_attention_weight_max": 32.682926177978516,
      "activations/layer21_attention_weight_min": -23.46248435974121,
      "activations/layer22_attention_weight_max": 29.86855697631836,
      "activations/layer22_attention_weight_min": -28.704322814941406,
      "activations/layer23_attention_weight_max": 36.73584747314453,
      "activations/layer23_attention_weight_min": -23.648334503173828,
      "activations/layer2_attention_weight_max": 33.84512710571289,
      "activations/layer2_attention_weight_min": -31.78280258178711,
      "activations/layer3_attention_weight_max": 106.1346435546875,
      "activations/layer3_attention_weight_min": -102.82887268066406,
      "activations/layer4_attention_weight_max": 102.7712173461914,
      "activations/layer4_attention_weight_min": -100.81800842285156,
      "activations/layer5_attention_weight_max": 78.6661605834961,
      "activations/layer5_attention_weight_min": -80.55779266357422,
      "activations/layer6_attention_weight_max": 54.79587936401367,
      "activations/layer6_attention_weight_min": -57.081790924072266,
      "activations/layer7_attention_weight_max": 70.78877258300781,
      "activations/layer7_attention_weight_min": -68.47737121582031,
      "activations/layer8_attention_weight_max": 48.483787536621094,
      "activations/layer8_attention_weight_min": -53.1867790222168,
      "activations/layer9_attention_weight_max": 48.87611770629883,
      "activations/layer9_attention_weight_min": -49.88526916503906,
      "epoch": 14.12,
      "learning_rate": 5.952727272727272e-05,
      "loss": 2.7439,
      "step": 243050
    },
    {
      "activations/layer0_attention_weight_max": 16.924463272094727,
      "activations/layer0_attention_weight_min": -13.634952545166016,
      "activations/layer10_attention_weight_max": 39.788238525390625,
      "activations/layer10_attention_weight_min": -38.688533782958984,
      "activations/layer11_attention_weight_max": 36.09322738647461,
      "activations/layer11_attention_weight_min": -36.57361602783203,
      "activations/layer12_attention_weight_max": 19.679059982299805,
      "activations/layer12_attention_weight_min": -25.604129791259766,
      "activations/layer13_attention_weight_max": 43.161380767822266,
      "activations/layer13_attention_weight_min": -30.523744583129883,
      "activations/layer14_attention_weight_max": 39.71601104736328,
      "activations/layer14_attention_weight_min": -29.878854751586914,
      "activations/layer15_attention_weight_max": 34.21176528930664,
      "activations/layer15_attention_weight_min": -28.841773986816406,
      "activations/layer16_attention_weight_max": 34.45888900756836,
      "activations/layer16_attention_weight_min": -32.440670013427734,
      "activations/layer17_attention_weight_max": 55.308128356933594,
      "activations/layer17_attention_weight_min": -43.42469787597656,
      "activations/layer18_attention_weight_max": 46.79872512817383,
      "activations/layer18_attention_weight_min": -36.98184585571289,
      "activations/layer19_attention_weight_max": 24.84521484375,
      "activations/layer19_attention_weight_min": -22.457298278808594,
      "activations/layer1_attention_weight_max": 16.082550048828125,
      "activations/layer1_attention_weight_min": -14.167582511901855,
      "activations/layer20_attention_weight_max": 22.471189498901367,
      "activations/layer20_attention_weight_min": -22.192546844482422,
      "activations/layer21_attention_weight_max": 32.457855224609375,
      "activations/layer21_attention_weight_min": -22.34946060180664,
      "activations/layer22_attention_weight_max": 31.684900283813477,
      "activations/layer22_attention_weight_min": -26.01346206665039,
      "activations/layer23_attention_weight_max": 39.734222412109375,
      "activations/layer23_attention_weight_min": -23.260543823242188,
      "activations/layer2_attention_weight_max": 30.51373291015625,
      "activations/layer2_attention_weight_min": -29.181041717529297,
      "activations/layer3_attention_weight_max": 94.03971099853516,
      "activations/layer3_attention_weight_min": -95.30805206298828,
      "activations/layer4_attention_weight_max": 92.63359832763672,
      "activations/layer4_attention_weight_min": -91.72065734863281,
      "activations/layer5_attention_weight_max": 69.87800598144531,
      "activations/layer5_attention_weight_min": -77.54986572265625,
      "activations/layer6_attention_weight_max": 52.232662200927734,
      "activations/layer6_attention_weight_min": -53.685604095458984,
      "activations/layer7_attention_weight_max": 66.45038604736328,
      "activations/layer7_attention_weight_min": -62.54397964477539,
      "activations/layer8_attention_weight_max": 47.91810989379883,
      "activations/layer8_attention_weight_min": -47.0190544128418,
      "activations/layer9_attention_weight_max": 45.69047927856445,
      "activations/layer9_attention_weight_min": -45.38597869873047,
      "epoch": 14.13,
      "learning_rate": 5.950833333333333e-05,
      "loss": 2.744,
      "step": 243100
    },
    {
      "activations/layer0_attention_weight_max": 17.31431007385254,
      "activations/layer0_attention_weight_min": -14.076252937316895,
      "activations/layer10_attention_weight_max": 33.86967468261719,
      "activations/layer10_attention_weight_min": -33.25313949584961,
      "activations/layer11_attention_weight_max": 28.59676170349121,
      "activations/layer11_attention_weight_min": -30.42841148376465,
      "activations/layer12_attention_weight_max": 18.839282989501953,
      "activations/layer12_attention_weight_min": -30.708040237426758,
      "activations/layer13_attention_weight_max": 34.53651428222656,
      "activations/layer13_attention_weight_min": -27.21509552001953,
      "activations/layer14_attention_weight_max": 35.373836517333984,
      "activations/layer14_attention_weight_min": -27.524005889892578,
      "activations/layer15_attention_weight_max": 32.47954177856445,
      "activations/layer15_attention_weight_min": -28.534528732299805,
      "activations/layer16_attention_weight_max": 32.90290069580078,
      "activations/layer16_attention_weight_min": -30.59352684020996,
      "activations/layer17_attention_weight_max": 53.370243072509766,
      "activations/layer17_attention_weight_min": -45.35091781616211,
      "activations/layer18_attention_weight_max": 49.1498908996582,
      "activations/layer18_attention_weight_min": -36.99846649169922,
      "activations/layer19_attention_weight_max": 25.055429458618164,
      "activations/layer19_attention_weight_min": -21.897232055664062,
      "activations/layer1_attention_weight_max": 15.737350463867188,
      "activations/layer1_attention_weight_min": -15.724128723144531,
      "activations/layer20_attention_weight_max": 23.80573844909668,
      "activations/layer20_attention_weight_min": -21.47886085510254,
      "activations/layer21_attention_weight_max": 35.36033630371094,
      "activations/layer21_attention_weight_min": -21.922626495361328,
      "activations/layer22_attention_weight_max": 27.636106491088867,
      "activations/layer22_attention_weight_min": -27.623411178588867,
      "activations/layer23_attention_weight_max": 34.00968933105469,
      "activations/layer23_attention_weight_min": -22.69154930114746,
      "activations/layer2_attention_weight_max": 34.08338165283203,
      "activations/layer2_attention_weight_min": -30.068492889404297,
      "activations/layer3_attention_weight_max": 89.57611846923828,
      "activations/layer3_attention_weight_min": -94.06652069091797,
      "activations/layer4_attention_weight_max": 89.64537811279297,
      "activations/layer4_attention_weight_min": -91.43276977539062,
      "activations/layer5_attention_weight_max": 67.3133544921875,
      "activations/layer5_attention_weight_min": -78.00434112548828,
      "activations/layer6_attention_weight_max": 52.73137283325195,
      "activations/layer6_attention_weight_min": -51.48032760620117,
      "activations/layer7_attention_weight_max": 63.226749420166016,
      "activations/layer7_attention_weight_min": -62.583595275878906,
      "activations/layer8_attention_weight_max": 43.44172286987305,
      "activations/layer8_attention_weight_min": -46.298622131347656,
      "activations/layer9_attention_weight_max": 43.36779022216797,
      "activations/layer9_attention_weight_min": -43.90339660644531,
      "epoch": 14.13,
      "learning_rate": 5.9489393939393936e-05,
      "loss": 2.7464,
      "step": 243150
    },
    {
      "activations/layer0_attention_weight_max": 16.784095764160156,
      "activations/layer0_attention_weight_min": -13.97378921508789,
      "activations/layer10_attention_weight_max": 40.31964874267578,
      "activations/layer10_attention_weight_min": -40.23980712890625,
      "activations/layer11_attention_weight_max": 36.04182434082031,
      "activations/layer11_attention_weight_min": -35.11595916748047,
      "activations/layer12_attention_weight_max": 20.134109497070312,
      "activations/layer12_attention_weight_min": -25.09026527404785,
      "activations/layer13_attention_weight_max": 38.2574577331543,
      "activations/layer13_attention_weight_min": -28.879953384399414,
      "activations/layer14_attention_weight_max": 39.53483581542969,
      "activations/layer14_attention_weight_min": -29.543804168701172,
      "activations/layer15_attention_weight_max": 35.40053939819336,
      "activations/layer15_attention_weight_min": -29.20977020263672,
      "activations/layer16_attention_weight_max": 35.9456787109375,
      "activations/layer16_attention_weight_min": -34.142364501953125,
      "activations/layer17_attention_weight_max": 57.76210403442383,
      "activations/layer17_attention_weight_min": -49.450931549072266,
      "activations/layer18_attention_weight_max": 49.11388397216797,
      "activations/layer18_attention_weight_min": -41.19784927368164,
      "activations/layer19_attention_weight_max": 23.575380325317383,
      "activations/layer19_attention_weight_min": -22.667041778564453,
      "activations/layer1_attention_weight_max": 16.75720977783203,
      "activations/layer1_attention_weight_min": -14.996914863586426,
      "activations/layer20_attention_weight_max": 23.63460922241211,
      "activations/layer20_attention_weight_min": -21.55006980895996,
      "activations/layer21_attention_weight_max": 41.5174674987793,
      "activations/layer21_attention_weight_min": -23.72212791442871,
      "activations/layer22_attention_weight_max": 31.26945686340332,
      "activations/layer22_attention_weight_min": -27.717548370361328,
      "activations/layer23_attention_weight_max": 35.715187072753906,
      "activations/layer23_attention_weight_min": -24.24664306640625,
      "activations/layer2_attention_weight_max": 33.09687423706055,
      "activations/layer2_attention_weight_min": -31.89623260498047,
      "activations/layer3_attention_weight_max": 97.97853088378906,
      "activations/layer3_attention_weight_min": -98.68595886230469,
      "activations/layer4_attention_weight_max": 96.87767791748047,
      "activations/layer4_attention_weight_min": -93.38138580322266,
      "activations/layer5_attention_weight_max": 72.17228698730469,
      "activations/layer5_attention_weight_min": -77.19837951660156,
      "activations/layer6_attention_weight_max": 54.49299621582031,
      "activations/layer6_attention_weight_min": -54.776344299316406,
      "activations/layer7_attention_weight_max": 68.48604583740234,
      "activations/layer7_attention_weight_min": -65.21945190429688,
      "activations/layer8_attention_weight_max": 48.6745491027832,
      "activations/layer8_attention_weight_min": -50.03520202636719,
      "activations/layer9_attention_weight_max": 48.04623031616211,
      "activations/layer9_attention_weight_min": -50.73555374145508,
      "epoch": 14.13,
      "learning_rate": 5.9470454545454544e-05,
      "loss": 2.7535,
      "step": 243200
    },
    {
      "activations/layer0_attention_weight_max": 16.554412841796875,
      "activations/layer0_attention_weight_min": -14.177129745483398,
      "activations/layer10_attention_weight_max": 36.120880126953125,
      "activations/layer10_attention_weight_min": -36.88501739501953,
      "activations/layer11_attention_weight_max": 33.69354248046875,
      "activations/layer11_attention_weight_min": -34.600990295410156,
      "activations/layer12_attention_weight_max": 19.690343856811523,
      "activations/layer12_attention_weight_min": -22.371191024780273,
      "activations/layer13_attention_weight_max": 40.12080001831055,
      "activations/layer13_attention_weight_min": -29.62366485595703,
      "activations/layer14_attention_weight_max": 36.84035110473633,
      "activations/layer14_attention_weight_min": -31.25005531311035,
      "activations/layer15_attention_weight_max": 36.20082473754883,
      "activations/layer15_attention_weight_min": -30.8900203704834,
      "activations/layer16_attention_weight_max": 31.614131927490234,
      "activations/layer16_attention_weight_min": -31.214248657226562,
      "activations/layer17_attention_weight_max": 53.574012756347656,
      "activations/layer17_attention_weight_min": -45.12831497192383,
      "activations/layer18_attention_weight_max": 46.99757385253906,
      "activations/layer18_attention_weight_min": -39.121681213378906,
      "activations/layer19_attention_weight_max": 23.684484481811523,
      "activations/layer19_attention_weight_min": -23.676610946655273,
      "activations/layer1_attention_weight_max": 16.511110305786133,
      "activations/layer1_attention_weight_min": -15.181607246398926,
      "activations/layer20_attention_weight_max": 23.198158264160156,
      "activations/layer20_attention_weight_min": -20.481456756591797,
      "activations/layer21_attention_weight_max": 35.00270462036133,
      "activations/layer21_attention_weight_min": -20.112789154052734,
      "activations/layer22_attention_weight_max": 32.61036682128906,
      "activations/layer22_attention_weight_min": -27.964447021484375,
      "activations/layer23_attention_weight_max": 44.77340316772461,
      "activations/layer23_attention_weight_min": -23.34461784362793,
      "activations/layer2_attention_weight_max": 30.42644500732422,
      "activations/layer2_attention_weight_min": -29.796987533569336,
      "activations/layer3_attention_weight_max": 93.42728424072266,
      "activations/layer3_attention_weight_min": -95.87712097167969,
      "activations/layer4_attention_weight_max": 92.47750091552734,
      "activations/layer4_attention_weight_min": -88.71080780029297,
      "activations/layer5_attention_weight_max": 68.69702911376953,
      "activations/layer5_attention_weight_min": -76.37982940673828,
      "activations/layer6_attention_weight_max": 49.73200988769531,
      "activations/layer6_attention_weight_min": -51.88694381713867,
      "activations/layer7_attention_weight_max": 65.43714141845703,
      "activations/layer7_attention_weight_min": -63.44615936279297,
      "activations/layer8_attention_weight_max": 45.697265625,
      "activations/layer8_attention_weight_min": -47.334754943847656,
      "activations/layer9_attention_weight_max": 44.820472717285156,
      "activations/layer9_attention_weight_min": -46.37420654296875,
      "epoch": 14.13,
      "learning_rate": 5.9451515151515145e-05,
      "loss": 2.7586,
      "step": 243250
    },
    {
      "activations/layer0_attention_weight_max": 15.753796577453613,
      "activations/layer0_attention_weight_min": -14.042152404785156,
      "activations/layer10_attention_weight_max": 38.62364959716797,
      "activations/layer10_attention_weight_min": -35.47828674316406,
      "activations/layer11_attention_weight_max": 32.41690444946289,
      "activations/layer11_attention_weight_min": -32.754451751708984,
      "activations/layer12_attention_weight_max": 17.90687370300293,
      "activations/layer12_attention_weight_min": -24.50579261779785,
      "activations/layer13_attention_weight_max": 50.0346565246582,
      "activations/layer13_attention_weight_min": -35.52772521972656,
      "activations/layer14_attention_weight_max": 32.66923522949219,
      "activations/layer14_attention_weight_min": -27.31273651123047,
      "activations/layer15_attention_weight_max": 31.895463943481445,
      "activations/layer15_attention_weight_min": -28.765687942504883,
      "activations/layer16_attention_weight_max": 31.785388946533203,
      "activations/layer16_attention_weight_min": -27.845088958740234,
      "activations/layer17_attention_weight_max": 49.56581497192383,
      "activations/layer17_attention_weight_min": -41.994384765625,
      "activations/layer18_attention_weight_max": 42.57673263549805,
      "activations/layer18_attention_weight_min": -37.368736267089844,
      "activations/layer19_attention_weight_max": 21.768535614013672,
      "activations/layer19_attention_weight_min": -20.897754669189453,
      "activations/layer1_attention_weight_max": 16.684326171875,
      "activations/layer1_attention_weight_min": -15.839744567871094,
      "activations/layer20_attention_weight_max": 19.68185806274414,
      "activations/layer20_attention_weight_min": -21.736909866333008,
      "activations/layer21_attention_weight_max": 30.861595153808594,
      "activations/layer21_attention_weight_min": -23.058012008666992,
      "activations/layer22_attention_weight_max": 29.853656768798828,
      "activations/layer22_attention_weight_min": -26.35197639465332,
      "activations/layer23_attention_weight_max": 34.151588439941406,
      "activations/layer23_attention_weight_min": -22.711048126220703,
      "activations/layer2_attention_weight_max": 32.077064514160156,
      "activations/layer2_attention_weight_min": -29.88953399658203,
      "activations/layer3_attention_weight_max": 96.05968475341797,
      "activations/layer3_attention_weight_min": -93.78716278076172,
      "activations/layer4_attention_weight_max": 94.06707000732422,
      "activations/layer4_attention_weight_min": -91.7825698852539,
      "activations/layer5_attention_weight_max": 70.96678161621094,
      "activations/layer5_attention_weight_min": -77.49713134765625,
      "activations/layer6_attention_weight_max": 50.80600357055664,
      "activations/layer6_attention_weight_min": -53.19186782836914,
      "activations/layer7_attention_weight_max": 65.79609680175781,
      "activations/layer7_attention_weight_min": -62.30403137207031,
      "activations/layer8_attention_weight_max": 46.36254119873047,
      "activations/layer8_attention_weight_min": -49.85482406616211,
      "activations/layer9_attention_weight_max": 44.7974967956543,
      "activations/layer9_attention_weight_min": -46.457740783691406,
      "epoch": 14.14,
      "learning_rate": 5.943257575757575e-05,
      "loss": 2.7322,
      "step": 243300
    },
    {
      "activations/layer0_attention_weight_max": 15.872818946838379,
      "activations/layer0_attention_weight_min": -14.05025577545166,
      "activations/layer10_attention_weight_max": 36.494773864746094,
      "activations/layer10_attention_weight_min": -35.779090881347656,
      "activations/layer11_attention_weight_max": 33.245094299316406,
      "activations/layer11_attention_weight_min": -33.6842155456543,
      "activations/layer12_attention_weight_max": 18.617605209350586,
      "activations/layer12_attention_weight_min": -28.490467071533203,
      "activations/layer13_attention_weight_max": 37.571075439453125,
      "activations/layer13_attention_weight_min": -29.512248992919922,
      "activations/layer14_attention_weight_max": 32.83544921875,
      "activations/layer14_attention_weight_min": -28.060396194458008,
      "activations/layer15_attention_weight_max": 32.14011764526367,
      "activations/layer15_attention_weight_min": -28.454383850097656,
      "activations/layer16_attention_weight_max": 31.612958908081055,
      "activations/layer16_attention_weight_min": -30.93701934814453,
      "activations/layer17_attention_weight_max": 51.907833099365234,
      "activations/layer17_attention_weight_min": -46.37969207763672,
      "activations/layer18_attention_weight_max": 48.75346374511719,
      "activations/layer18_attention_weight_min": -39.70586013793945,
      "activations/layer19_attention_weight_max": 21.939407348632812,
      "activations/layer19_attention_weight_min": -22.358793258666992,
      "activations/layer1_attention_weight_max": 16.242582321166992,
      "activations/layer1_attention_weight_min": -14.651232719421387,
      "activations/layer20_attention_weight_max": 23.186643600463867,
      "activations/layer20_attention_weight_min": -21.300418853759766,
      "activations/layer21_attention_weight_max": 36.151180267333984,
      "activations/layer21_attention_weight_min": -22.594396591186523,
      "activations/layer22_attention_weight_max": 29.434797286987305,
      "activations/layer22_attention_weight_min": -25.842418670654297,
      "activations/layer23_attention_weight_max": 34.10826110839844,
      "activations/layer23_attention_weight_min": -23.169422149658203,
      "activations/layer2_attention_weight_max": 34.24970626831055,
      "activations/layer2_attention_weight_min": -32.19037628173828,
      "activations/layer3_attention_weight_max": 100.19860076904297,
      "activations/layer3_attention_weight_min": -97.71158599853516,
      "activations/layer4_attention_weight_max": 93.13300323486328,
      "activations/layer4_attention_weight_min": -90.31175994873047,
      "activations/layer5_attention_weight_max": 67.42047119140625,
      "activations/layer5_attention_weight_min": -72.696533203125,
      "activations/layer6_attention_weight_max": 52.52735900878906,
      "activations/layer6_attention_weight_min": -51.37837600708008,
      "activations/layer7_attention_weight_max": 61.50566864013672,
      "activations/layer7_attention_weight_min": -59.57297134399414,
      "activations/layer8_attention_weight_max": 44.02373123168945,
      "activations/layer8_attention_weight_min": -49.634010314941406,
      "activations/layer9_attention_weight_max": 45.70287322998047,
      "activations/layer9_attention_weight_min": -47.41606140136719,
      "epoch": 14.14,
      "learning_rate": 5.941363636363636e-05,
      "loss": 2.7528,
      "step": 243350
    },
    {
      "activations/layer0_attention_weight_max": 16.06355094909668,
      "activations/layer0_attention_weight_min": -13.973898887634277,
      "activations/layer10_attention_weight_max": 35.00914764404297,
      "activations/layer10_attention_weight_min": -34.897457122802734,
      "activations/layer11_attention_weight_max": 31.18868637084961,
      "activations/layer11_attention_weight_min": -31.095462799072266,
      "activations/layer12_attention_weight_max": 17.305723190307617,
      "activations/layer12_attention_weight_min": -25.15498161315918,
      "activations/layer13_attention_weight_max": 31.95999526977539,
      "activations/layer13_attention_weight_min": -26.933748245239258,
      "activations/layer14_attention_weight_max": 33.885440826416016,
      "activations/layer14_attention_weight_min": -30.731185913085938,
      "activations/layer15_attention_weight_max": 30.85693359375,
      "activations/layer15_attention_weight_min": -30.974428176879883,
      "activations/layer16_attention_weight_max": 32.17851638793945,
      "activations/layer16_attention_weight_min": -30.399816513061523,
      "activations/layer17_attention_weight_max": 48.5108528137207,
      "activations/layer17_attention_weight_min": -45.70060348510742,
      "activations/layer18_attention_weight_max": 44.95845413208008,
      "activations/layer18_attention_weight_min": -38.494537353515625,
      "activations/layer19_attention_weight_max": 22.799240112304688,
      "activations/layer19_attention_weight_min": -23.859582901000977,
      "activations/layer1_attention_weight_max": 16.307653427124023,
      "activations/layer1_attention_weight_min": -15.19896411895752,
      "activations/layer20_attention_weight_max": 23.74534797668457,
      "activations/layer20_attention_weight_min": -23.932476043701172,
      "activations/layer21_attention_weight_max": 36.33269119262695,
      "activations/layer21_attention_weight_min": -25.29970359802246,
      "activations/layer22_attention_weight_max": 29.51825523376465,
      "activations/layer22_attention_weight_min": -27.75409698486328,
      "activations/layer23_attention_weight_max": 41.118316650390625,
      "activations/layer23_attention_weight_min": -25.173561096191406,
      "activations/layer2_attention_weight_max": 33.824623107910156,
      "activations/layer2_attention_weight_min": -31.470176696777344,
      "activations/layer3_attention_weight_max": 97.66612243652344,
      "activations/layer3_attention_weight_min": -101.94013214111328,
      "activations/layer4_attention_weight_max": 90.47748565673828,
      "activations/layer4_attention_weight_min": -91.75032806396484,
      "activations/layer5_attention_weight_max": 70.56920623779297,
      "activations/layer5_attention_weight_min": -74.61936950683594,
      "activations/layer6_attention_weight_max": 49.289588928222656,
      "activations/layer6_attention_weight_min": -50.09764862060547,
      "activations/layer7_attention_weight_max": 63.07707214355469,
      "activations/layer7_attention_weight_min": -64.21126556396484,
      "activations/layer8_attention_weight_max": 43.490413665771484,
      "activations/layer8_attention_weight_min": -46.05119705200195,
      "activations/layer9_attention_weight_max": 43.09140396118164,
      "activations/layer9_attention_weight_min": -48.758094787597656,
      "epoch": 14.14,
      "learning_rate": 5.9394696969696963e-05,
      "loss": 2.743,
      "step": 243400
    },
    {
      "activations/layer0_attention_weight_max": 16.96941566467285,
      "activations/layer0_attention_weight_min": -14.10035228729248,
      "activations/layer10_attention_weight_max": 41.68050003051758,
      "activations/layer10_attention_weight_min": -41.5193977355957,
      "activations/layer11_attention_weight_max": 37.89911651611328,
      "activations/layer11_attention_weight_min": -38.64714431762695,
      "activations/layer12_attention_weight_max": 19.591047286987305,
      "activations/layer12_attention_weight_min": -25.853858947753906,
      "activations/layer13_attention_weight_max": 43.632450103759766,
      "activations/layer13_attention_weight_min": -30.774736404418945,
      "activations/layer14_attention_weight_max": 46.78750991821289,
      "activations/layer14_attention_weight_min": -33.20001983642578,
      "activations/layer15_attention_weight_max": 36.489532470703125,
      "activations/layer15_attention_weight_min": -34.087554931640625,
      "activations/layer16_attention_weight_max": 36.04371643066406,
      "activations/layer16_attention_weight_min": -33.264076232910156,
      "activations/layer17_attention_weight_max": 57.60074996948242,
      "activations/layer17_attention_weight_min": -46.44685363769531,
      "activations/layer18_attention_weight_max": 52.67601013183594,
      "activations/layer18_attention_weight_min": -40.29780197143555,
      "activations/layer19_attention_weight_max": 23.7230281829834,
      "activations/layer19_attention_weight_min": -21.872909545898438,
      "activations/layer1_attention_weight_max": 16.528409957885742,
      "activations/layer1_attention_weight_min": -15.052806854248047,
      "activations/layer20_attention_weight_max": 23.903112411499023,
      "activations/layer20_attention_weight_min": -24.780338287353516,
      "activations/layer21_attention_weight_max": 41.23457717895508,
      "activations/layer21_attention_weight_min": -23.540176391601562,
      "activations/layer22_attention_weight_max": 31.62643814086914,
      "activations/layer22_attention_weight_min": -26.16156005859375,
      "activations/layer23_attention_weight_max": 40.69744873046875,
      "activations/layer23_attention_weight_min": -24.921674728393555,
      "activations/layer2_attention_weight_max": 34.39519500732422,
      "activations/layer2_attention_weight_min": -33.03364562988281,
      "activations/layer3_attention_weight_max": 103.10223388671875,
      "activations/layer3_attention_weight_min": -106.2887954711914,
      "activations/layer4_attention_weight_max": 104.9057388305664,
      "activations/layer4_attention_weight_min": -103.64826965332031,
      "activations/layer5_attention_weight_max": 71.77288055419922,
      "activations/layer5_attention_weight_min": -79.21239471435547,
      "activations/layer6_attention_weight_max": 56.1158561706543,
      "activations/layer6_attention_weight_min": -59.50960159301758,
      "activations/layer7_attention_weight_max": 68.88249969482422,
      "activations/layer7_attention_weight_min": -70.61184692382812,
      "activations/layer8_attention_weight_max": 51.21394348144531,
      "activations/layer8_attention_weight_min": -52.41699981689453,
      "activations/layer9_attention_weight_max": 52.46367263793945,
      "activations/layer9_attention_weight_min": -52.15715408325195,
      "epoch": 14.15,
      "learning_rate": 5.9375757575757565e-05,
      "loss": 2.7584,
      "step": 243450
    },
    {
      "activations/layer0_attention_weight_max": 16.51299476623535,
      "activations/layer0_attention_weight_min": -13.971854209899902,
      "activations/layer10_attention_weight_max": 37.915714263916016,
      "activations/layer10_attention_weight_min": -35.45151138305664,
      "activations/layer11_attention_weight_max": 31.98131561279297,
      "activations/layer11_attention_weight_min": -33.58344650268555,
      "activations/layer12_attention_weight_max": 18.563928604125977,
      "activations/layer12_attention_weight_min": -24.347797393798828,
      "activations/layer13_attention_weight_max": 33.231422424316406,
      "activations/layer13_attention_weight_min": -28.371755599975586,
      "activations/layer14_attention_weight_max": 33.807167053222656,
      "activations/layer14_attention_weight_min": -28.729787826538086,
      "activations/layer15_attention_weight_max": 30.437097549438477,
      "activations/layer15_attention_weight_min": -28.320268630981445,
      "activations/layer16_attention_weight_max": 29.698232650756836,
      "activations/layer16_attention_weight_min": -28.667579650878906,
      "activations/layer17_attention_weight_max": 45.474246978759766,
      "activations/layer17_attention_weight_min": -43.80872344970703,
      "activations/layer18_attention_weight_max": 41.9668083190918,
      "activations/layer18_attention_weight_min": -36.87019348144531,
      "activations/layer19_attention_weight_max": 22.883485794067383,
      "activations/layer19_attention_weight_min": -22.986005783081055,
      "activations/layer1_attention_weight_max": 16.502580642700195,
      "activations/layer1_attention_weight_min": -14.197654724121094,
      "activations/layer20_attention_weight_max": 20.340274810791016,
      "activations/layer20_attention_weight_min": -22.04764747619629,
      "activations/layer21_attention_weight_max": 37.378211975097656,
      "activations/layer21_attention_weight_min": -23.680496215820312,
      "activations/layer22_attention_weight_max": 27.55908966064453,
      "activations/layer22_attention_weight_min": -25.33152961730957,
      "activations/layer23_attention_weight_max": 34.49747085571289,
      "activations/layer23_attention_weight_min": -25.02924346923828,
      "activations/layer2_attention_weight_max": 32.03786849975586,
      "activations/layer2_attention_weight_min": -31.44684600830078,
      "activations/layer3_attention_weight_max": 97.11563110351562,
      "activations/layer3_attention_weight_min": -98.80451202392578,
      "activations/layer4_attention_weight_max": 94.69854736328125,
      "activations/layer4_attention_weight_min": -92.05586242675781,
      "activations/layer5_attention_weight_max": 74.3179931640625,
      "activations/layer5_attention_weight_min": -80.81378936767578,
      "activations/layer6_attention_weight_max": 51.026432037353516,
      "activations/layer6_attention_weight_min": -52.340824127197266,
      "activations/layer7_attention_weight_max": 66.42948150634766,
      "activations/layer7_attention_weight_min": -64.28577423095703,
      "activations/layer8_attention_weight_max": 46.46824264526367,
      "activations/layer8_attention_weight_min": -49.30877685546875,
      "activations/layer9_attention_weight_max": 45.60580825805664,
      "activations/layer9_attention_weight_min": -51.03370666503906,
      "epoch": 14.15,
      "learning_rate": 5.935681818181818e-05,
      "loss": 2.7584,
      "step": 243500
    },
    {
      "activations/layer0_attention_weight_max": 16.772611618041992,
      "activations/layer0_attention_weight_min": -14.144139289855957,
      "activations/layer10_attention_weight_max": 37.36640548706055,
      "activations/layer10_attention_weight_min": -38.30237579345703,
      "activations/layer11_attention_weight_max": 34.60357666015625,
      "activations/layer11_attention_weight_min": -35.13199996948242,
      "activations/layer12_attention_weight_max": 20.89937973022461,
      "activations/layer12_attention_weight_min": -26.337081909179688,
      "activations/layer13_attention_weight_max": 38.36101150512695,
      "activations/layer13_attention_weight_min": -25.81538200378418,
      "activations/layer14_attention_weight_max": 46.97480773925781,
      "activations/layer14_attention_weight_min": -29.549911499023438,
      "activations/layer15_attention_weight_max": 36.11963653564453,
      "activations/layer15_attention_weight_min": -28.87370491027832,
      "activations/layer16_attention_weight_max": 30.60279083251953,
      "activations/layer16_attention_weight_min": -29.42438507080078,
      "activations/layer17_attention_weight_max": 50.19731521606445,
      "activations/layer17_attention_weight_min": -45.07526397705078,
      "activations/layer18_attention_weight_max": 42.08121871948242,
      "activations/layer18_attention_weight_min": -37.03976821899414,
      "activations/layer19_attention_weight_max": 20.931127548217773,
      "activations/layer19_attention_weight_min": -22.470088958740234,
      "activations/layer1_attention_weight_max": 17.50943374633789,
      "activations/layer1_attention_weight_min": -15.662374496459961,
      "activations/layer20_attention_weight_max": 21.716503143310547,
      "activations/layer20_attention_weight_min": -24.694334030151367,
      "activations/layer21_attention_weight_max": 32.65631866455078,
      "activations/layer21_attention_weight_min": -22.429588317871094,
      "activations/layer22_attention_weight_max": 28.55325698852539,
      "activations/layer22_attention_weight_min": -25.017990112304688,
      "activations/layer23_attention_weight_max": 33.95823287963867,
      "activations/layer23_attention_weight_min": -24.764007568359375,
      "activations/layer2_attention_weight_max": 33.53571319580078,
      "activations/layer2_attention_weight_min": -31.409530639648438,
      "activations/layer3_attention_weight_max": 102.53022003173828,
      "activations/layer3_attention_weight_min": -99.13459777832031,
      "activations/layer4_attention_weight_max": 100.35050201416016,
      "activations/layer4_attention_weight_min": -94.6191635131836,
      "activations/layer5_attention_weight_max": 75.2615737915039,
      "activations/layer5_attention_weight_min": -82.25559997558594,
      "activations/layer6_attention_weight_max": 56.91038513183594,
      "activations/layer6_attention_weight_min": -54.24073791503906,
      "activations/layer7_attention_weight_max": 70.07353210449219,
      "activations/layer7_attention_weight_min": -69.24755859375,
      "activations/layer8_attention_weight_max": 51.7674446105957,
      "activations/layer8_attention_weight_min": -53.73192596435547,
      "activations/layer9_attention_weight_max": 55.21527862548828,
      "activations/layer9_attention_weight_min": -47.341556549072266,
      "epoch": 14.15,
      "learning_rate": 5.933787878787878e-05,
      "loss": 2.7513,
      "step": 243550
    },
    {
      "activations/layer0_attention_weight_max": 16.353960037231445,
      "activations/layer0_attention_weight_min": -13.958816528320312,
      "activations/layer10_attention_weight_max": 37.02421188354492,
      "activations/layer10_attention_weight_min": -36.40190887451172,
      "activations/layer11_attention_weight_max": 33.211795806884766,
      "activations/layer11_attention_weight_min": -34.81293487548828,
      "activations/layer12_attention_weight_max": 20.124845504760742,
      "activations/layer12_attention_weight_min": -23.03746795654297,
      "activations/layer13_attention_weight_max": 41.6352653503418,
      "activations/layer13_attention_weight_min": -30.3157958984375,
      "activations/layer14_attention_weight_max": 36.470123291015625,
      "activations/layer14_attention_weight_min": -28.98298454284668,
      "activations/layer15_attention_weight_max": 33.055110931396484,
      "activations/layer15_attention_weight_min": -29.478893280029297,
      "activations/layer16_attention_weight_max": 35.512149810791016,
      "activations/layer16_attention_weight_min": -32.30160903930664,
      "activations/layer17_attention_weight_max": 51.9046630859375,
      "activations/layer17_attention_weight_min": -44.41861343383789,
      "activations/layer18_attention_weight_max": 44.85974884033203,
      "activations/layer18_attention_weight_min": -37.79586410522461,
      "activations/layer19_attention_weight_max": 21.631492614746094,
      "activations/layer19_attention_weight_min": -21.02809715270996,
      "activations/layer1_attention_weight_max": 17.34811019897461,
      "activations/layer1_attention_weight_min": -15.603052139282227,
      "activations/layer20_attention_weight_max": 22.253108978271484,
      "activations/layer20_attention_weight_min": -19.745464324951172,
      "activations/layer21_attention_weight_max": 33.98011016845703,
      "activations/layer21_attention_weight_min": -24.181884765625,
      "activations/layer22_attention_weight_max": 32.38840866088867,
      "activations/layer22_attention_weight_min": -25.37867546081543,
      "activations/layer23_attention_weight_max": 40.961944580078125,
      "activations/layer23_attention_weight_min": -24.30970573425293,
      "activations/layer2_attention_weight_max": 32.10115432739258,
      "activations/layer2_attention_weight_min": -29.923490524291992,
      "activations/layer3_attention_weight_max": 94.39665985107422,
      "activations/layer3_attention_weight_min": -99.57916259765625,
      "activations/layer4_attention_weight_max": 97.0528793334961,
      "activations/layer4_attention_weight_min": -94.6596908569336,
      "activations/layer5_attention_weight_max": 72.54580688476562,
      "activations/layer5_attention_weight_min": -80.50735473632812,
      "activations/layer6_attention_weight_max": 53.67473220825195,
      "activations/layer6_attention_weight_min": -51.55766296386719,
      "activations/layer7_attention_weight_max": 72.69245147705078,
      "activations/layer7_attention_weight_min": -69.14623260498047,
      "activations/layer8_attention_weight_max": 46.875213623046875,
      "activations/layer8_attention_weight_min": -48.48076629638672,
      "activations/layer9_attention_weight_max": 47.87197494506836,
      "activations/layer9_attention_weight_min": -46.64604949951172,
      "epoch": 14.15,
      "learning_rate": 5.931931818181818e-05,
      "loss": 2.7492,
      "step": 243600
    },
    {
      "activations/layer0_attention_weight_max": 17.728519439697266,
      "activations/layer0_attention_weight_min": -13.975626945495605,
      "activations/layer10_attention_weight_max": 33.95271301269531,
      "activations/layer10_attention_weight_min": -33.3519172668457,
      "activations/layer11_attention_weight_max": 30.844871520996094,
      "activations/layer11_attention_weight_min": -32.51123046875,
      "activations/layer12_attention_weight_max": 18.396907806396484,
      "activations/layer12_attention_weight_min": -22.91376495361328,
      "activations/layer13_attention_weight_max": 36.0704460144043,
      "activations/layer13_attention_weight_min": -28.316774368286133,
      "activations/layer14_attention_weight_max": 36.39816665649414,
      "activations/layer14_attention_weight_min": -29.906309127807617,
      "activations/layer15_attention_weight_max": 33.52297592163086,
      "activations/layer15_attention_weight_min": -30.484079360961914,
      "activations/layer16_attention_weight_max": 31.297603607177734,
      "activations/layer16_attention_weight_min": -31.727445602416992,
      "activations/layer17_attention_weight_max": 51.724388122558594,
      "activations/layer17_attention_weight_min": -47.17604064941406,
      "activations/layer18_attention_weight_max": 47.256168365478516,
      "activations/layer18_attention_weight_min": -42.99737548828125,
      "activations/layer19_attention_weight_max": 22.01856803894043,
      "activations/layer19_attention_weight_min": -25.082984924316406,
      "activations/layer1_attention_weight_max": 16.28828239440918,
      "activations/layer1_attention_weight_min": -14.487277030944824,
      "activations/layer20_attention_weight_max": 23.32542610168457,
      "activations/layer20_attention_weight_min": -22.297060012817383,
      "activations/layer21_attention_weight_max": 31.78765106201172,
      "activations/layer21_attention_weight_min": -22.054258346557617,
      "activations/layer22_attention_weight_max": 29.373775482177734,
      "activations/layer22_attention_weight_min": -25.77183723449707,
      "activations/layer23_attention_weight_max": 40.61084747314453,
      "activations/layer23_attention_weight_min": -25.757047653198242,
      "activations/layer2_attention_weight_max": 32.20875549316406,
      "activations/layer2_attention_weight_min": -32.987754821777344,
      "activations/layer3_attention_weight_max": 93.74462127685547,
      "activations/layer3_attention_weight_min": -95.5438232421875,
      "activations/layer4_attention_weight_max": 91.26766204833984,
      "activations/layer4_attention_weight_min": -89.09548950195312,
      "activations/layer5_attention_weight_max": 73.11614990234375,
      "activations/layer5_attention_weight_min": -74.9901123046875,
      "activations/layer6_attention_weight_max": 52.864017486572266,
      "activations/layer6_attention_weight_min": -51.17009735107422,
      "activations/layer7_attention_weight_max": 68.62355041503906,
      "activations/layer7_attention_weight_min": -63.76557922363281,
      "activations/layer8_attention_weight_max": 44.19547653198242,
      "activations/layer8_attention_weight_min": -47.3272590637207,
      "activations/layer9_attention_weight_max": 43.08138656616211,
      "activations/layer9_attention_weight_min": -45.504154205322266,
      "epoch": 14.16,
      "learning_rate": 5.930037878787878e-05,
      "loss": 2.7566,
      "step": 243650
    },
    {
      "activations/layer0_attention_weight_max": 16.831626892089844,
      "activations/layer0_attention_weight_min": -13.71415901184082,
      "activations/layer10_attention_weight_max": 36.42055130004883,
      "activations/layer10_attention_weight_min": -37.033817291259766,
      "activations/layer11_attention_weight_max": 31.90619659423828,
      "activations/layer11_attention_weight_min": -33.9754753112793,
      "activations/layer12_attention_weight_max": 19.54312515258789,
      "activations/layer12_attention_weight_min": -24.73724937438965,
      "activations/layer13_attention_weight_max": 37.88315963745117,
      "activations/layer13_attention_weight_min": -28.155864715576172,
      "activations/layer14_attention_weight_max": 38.33549880981445,
      "activations/layer14_attention_weight_min": -30.726947784423828,
      "activations/layer15_attention_weight_max": 34.3329963684082,
      "activations/layer15_attention_weight_min": -32.07386016845703,
      "activations/layer16_attention_weight_max": 34.59244918823242,
      "activations/layer16_attention_weight_min": -32.26021194458008,
      "activations/layer17_attention_weight_max": 53.66641616821289,
      "activations/layer17_attention_weight_min": -45.9924430847168,
      "activations/layer18_attention_weight_max": 50.946048736572266,
      "activations/layer18_attention_weight_min": -41.51677703857422,
      "activations/layer19_attention_weight_max": 22.810178756713867,
      "activations/layer19_attention_weight_min": -24.09679412841797,
      "activations/layer1_attention_weight_max": 16.61366081237793,
      "activations/layer1_attention_weight_min": -15.471569061279297,
      "activations/layer20_attention_weight_max": 23.255361557006836,
      "activations/layer20_attention_weight_min": -24.370262145996094,
      "activations/layer21_attention_weight_max": 34.139305114746094,
      "activations/layer21_attention_weight_min": -21.818466186523438,
      "activations/layer22_attention_weight_max": 31.318004608154297,
      "activations/layer22_attention_weight_min": -27.1517333984375,
      "activations/layer23_attention_weight_max": 35.75092315673828,
      "activations/layer23_attention_weight_min": -23.24505615234375,
      "activations/layer2_attention_weight_max": 32.23344421386719,
      "activations/layer2_attention_weight_min": -33.00074768066406,
      "activations/layer3_attention_weight_max": 97.90808868408203,
      "activations/layer3_attention_weight_min": -101.93260955810547,
      "activations/layer4_attention_weight_max": 94.6438217163086,
      "activations/layer4_attention_weight_min": -94.9878158569336,
      "activations/layer5_attention_weight_max": 69.04669189453125,
      "activations/layer5_attention_weight_min": -83.1263427734375,
      "activations/layer6_attention_weight_max": 53.00031280517578,
      "activations/layer6_attention_weight_min": -53.80607604980469,
      "activations/layer7_attention_weight_max": 64.1540756225586,
      "activations/layer7_attention_weight_min": -64.03790283203125,
      "activations/layer8_attention_weight_max": 45.23701858520508,
      "activations/layer8_attention_weight_min": -48.46738052368164,
      "activations/layer9_attention_weight_max": 45.289634704589844,
      "activations/layer9_attention_weight_min": -45.463600158691406,
      "epoch": 14.16,
      "learning_rate": 5.928143939393939e-05,
      "loss": 2.748,
      "step": 243700
    },
    {
      "activations/layer0_attention_weight_max": 18.023006439208984,
      "activations/layer0_attention_weight_min": -14.402141571044922,
      "activations/layer10_attention_weight_max": 34.41556167602539,
      "activations/layer10_attention_weight_min": -34.3702278137207,
      "activations/layer11_attention_weight_max": 31.879302978515625,
      "activations/layer11_attention_weight_min": -31.959693908691406,
      "activations/layer12_attention_weight_max": 19.32441520690918,
      "activations/layer12_attention_weight_min": -23.706830978393555,
      "activations/layer13_attention_weight_max": 38.824462890625,
      "activations/layer13_attention_weight_min": -28.652835845947266,
      "activations/layer14_attention_weight_max": 33.604347229003906,
      "activations/layer14_attention_weight_min": -28.43012046813965,
      "activations/layer15_attention_weight_max": 32.187950134277344,
      "activations/layer15_attention_weight_min": -29.614013671875,
      "activations/layer16_attention_weight_max": 31.318008422851562,
      "activations/layer16_attention_weight_min": -28.81780433654785,
      "activations/layer17_attention_weight_max": 49.853092193603516,
      "activations/layer17_attention_weight_min": -42.77578353881836,
      "activations/layer18_attention_weight_max": 45.48188018798828,
      "activations/layer18_attention_weight_min": -38.44468307495117,
      "activations/layer19_attention_weight_max": 22.5151309967041,
      "activations/layer19_attention_weight_min": -23.02552604675293,
      "activations/layer1_attention_weight_max": 16.88615608215332,
      "activations/layer1_attention_weight_min": -17.157896041870117,
      "activations/layer20_attention_weight_max": 24.21755027770996,
      "activations/layer20_attention_weight_min": -21.868009567260742,
      "activations/layer21_attention_weight_max": 37.22528076171875,
      "activations/layer21_attention_weight_min": -23.468059539794922,
      "activations/layer22_attention_weight_max": 33.102989196777344,
      "activations/layer22_attention_weight_min": -26.51190948486328,
      "activations/layer23_attention_weight_max": 40.16102981567383,
      "activations/layer23_attention_weight_min": -24.049449920654297,
      "activations/layer2_attention_weight_max": 33.90425109863281,
      "activations/layer2_attention_weight_min": -32.25585174560547,
      "activations/layer3_attention_weight_max": 98.82974243164062,
      "activations/layer3_attention_weight_min": -99.80010223388672,
      "activations/layer4_attention_weight_max": 95.21638488769531,
      "activations/layer4_attention_weight_min": -93.65471649169922,
      "activations/layer5_attention_weight_max": 72.14138793945312,
      "activations/layer5_attention_weight_min": -78.6378402709961,
      "activations/layer6_attention_weight_max": 49.584957122802734,
      "activations/layer6_attention_weight_min": -52.29557800292969,
      "activations/layer7_attention_weight_max": 63.61160659790039,
      "activations/layer7_attention_weight_min": -63.19154357910156,
      "activations/layer8_attention_weight_max": 44.11787796020508,
      "activations/layer8_attention_weight_min": -45.8680419921875,
      "activations/layer9_attention_weight_max": 45.088504791259766,
      "activations/layer9_attention_weight_min": -44.55511474609375,
      "epoch": 14.16,
      "learning_rate": 5.92625e-05,
      "loss": 2.7673,
      "step": 243750
    },
    {
      "activations/layer0_attention_weight_max": 16.845951080322266,
      "activations/layer0_attention_weight_min": -14.200228691101074,
      "activations/layer10_attention_weight_max": 33.95584487915039,
      "activations/layer10_attention_weight_min": -34.567054748535156,
      "activations/layer11_attention_weight_max": 30.879276275634766,
      "activations/layer11_attention_weight_min": -31.09796142578125,
      "activations/layer12_attention_weight_max": 18.983076095581055,
      "activations/layer12_attention_weight_min": -25.46474838256836,
      "activations/layer13_attention_weight_max": 47.31476593017578,
      "activations/layer13_attention_weight_min": -29.385921478271484,
      "activations/layer14_attention_weight_max": 42.19108581542969,
      "activations/layer14_attention_weight_min": -33.19096374511719,
      "activations/layer15_attention_weight_max": 38.40687942504883,
      "activations/layer15_attention_weight_min": -31.333599090576172,
      "activations/layer16_attention_weight_max": 33.8610954284668,
      "activations/layer16_attention_weight_min": -32.71556854248047,
      "activations/layer17_attention_weight_max": 62.10396194458008,
      "activations/layer17_attention_weight_min": -50.5357551574707,
      "activations/layer18_attention_weight_max": 54.74034881591797,
      "activations/layer18_attention_weight_min": -45.201358795166016,
      "activations/layer19_attention_weight_max": 23.646203994750977,
      "activations/layer19_attention_weight_min": -23.981016159057617,
      "activations/layer1_attention_weight_max": 16.10307502746582,
      "activations/layer1_attention_weight_min": -15.971689224243164,
      "activations/layer20_attention_weight_max": 23.612342834472656,
      "activations/layer20_attention_weight_min": -23.092771530151367,
      "activations/layer21_attention_weight_max": 38.489013671875,
      "activations/layer21_attention_weight_min": -25.207351684570312,
      "activations/layer22_attention_weight_max": 29.635616302490234,
      "activations/layer22_attention_weight_min": -26.65434455871582,
      "activations/layer23_attention_weight_max": 38.65785217285156,
      "activations/layer23_attention_weight_min": -24.20120620727539,
      "activations/layer2_attention_weight_max": 32.8936882019043,
      "activations/layer2_attention_weight_min": -30.84451675415039,
      "activations/layer3_attention_weight_max": 96.89915466308594,
      "activations/layer3_attention_weight_min": -97.48564910888672,
      "activations/layer4_attention_weight_max": 91.6692886352539,
      "activations/layer4_attention_weight_min": -90.63109588623047,
      "activations/layer5_attention_weight_max": 70.2769546508789,
      "activations/layer5_attention_weight_min": -76.11473083496094,
      "activations/layer6_attention_weight_max": 51.197513580322266,
      "activations/layer6_attention_weight_min": -50.4128532409668,
      "activations/layer7_attention_weight_max": 65.99089050292969,
      "activations/layer7_attention_weight_min": -63.39860153198242,
      "activations/layer8_attention_weight_max": 46.193397521972656,
      "activations/layer8_attention_weight_min": -48.762672424316406,
      "activations/layer9_attention_weight_max": 44.25094985961914,
      "activations/layer9_attention_weight_min": -44.52933883666992,
      "epoch": 14.17,
      "learning_rate": 5.92435606060606e-05,
      "loss": 2.7458,
      "step": 243800
    },
    {
      "activations/layer0_attention_weight_max": 17.067399978637695,
      "activations/layer0_attention_weight_min": -14.217906951904297,
      "activations/layer10_attention_weight_max": 35.46195983886719,
      "activations/layer10_attention_weight_min": -36.66672134399414,
      "activations/layer11_attention_weight_max": 33.315860748291016,
      "activations/layer11_attention_weight_min": -33.52177429199219,
      "activations/layer12_attention_weight_max": 18.685033798217773,
      "activations/layer12_attention_weight_min": -25.973737716674805,
      "activations/layer13_attention_weight_max": 35.204307556152344,
      "activations/layer13_attention_weight_min": -31.11808967590332,
      "activations/layer14_attention_weight_max": 35.56838607788086,
      "activations/layer14_attention_weight_min": -31.13030433654785,
      "activations/layer15_attention_weight_max": 32.24828338623047,
      "activations/layer15_attention_weight_min": -28.822288513183594,
      "activations/layer16_attention_weight_max": 31.049304962158203,
      "activations/layer16_attention_weight_min": -28.886734008789062,
      "activations/layer17_attention_weight_max": 51.528411865234375,
      "activations/layer17_attention_weight_min": -42.298095703125,
      "activations/layer18_attention_weight_max": 47.5278434753418,
      "activations/layer18_attention_weight_min": -38.65818405151367,
      "activations/layer19_attention_weight_max": 22.173362731933594,
      "activations/layer19_attention_weight_min": -21.83841896057129,
      "activations/layer1_attention_weight_max": 17.2054443359375,
      "activations/layer1_attention_weight_min": -15.256080627441406,
      "activations/layer20_attention_weight_max": 21.497238159179688,
      "activations/layer20_attention_weight_min": -25.301475524902344,
      "activations/layer21_attention_weight_max": 37.34149169921875,
      "activations/layer21_attention_weight_min": -23.28008460998535,
      "activations/layer22_attention_weight_max": 28.89759635925293,
      "activations/layer22_attention_weight_min": -24.36720085144043,
      "activations/layer23_attention_weight_max": 39.51067352294922,
      "activations/layer23_attention_weight_min": -27.725879669189453,
      "activations/layer2_attention_weight_max": 32.46392822265625,
      "activations/layer2_attention_weight_min": -30.911727905273438,
      "activations/layer3_attention_weight_max": 100.3292236328125,
      "activations/layer3_attention_weight_min": -93.60710906982422,
      "activations/layer4_attention_weight_max": 96.5328140258789,
      "activations/layer4_attention_weight_min": -92.06707763671875,
      "activations/layer5_attention_weight_max": 71.98027038574219,
      "activations/layer5_attention_weight_min": -77.96866607666016,
      "activations/layer6_attention_weight_max": 52.94668960571289,
      "activations/layer6_attention_weight_min": -52.40897750854492,
      "activations/layer7_attention_weight_max": 66.10728454589844,
      "activations/layer7_attention_weight_min": -66.44563293457031,
      "activations/layer8_attention_weight_max": 46.800811767578125,
      "activations/layer8_attention_weight_min": -48.327571868896484,
      "activations/layer9_attention_weight_max": 45.296138763427734,
      "activations/layer9_attention_weight_min": -45.83785629272461,
      "epoch": 14.17,
      "learning_rate": 5.922462121212121e-05,
      "loss": 2.7485,
      "step": 243850
    },
    {
      "activations/layer0_attention_weight_max": 16.770164489746094,
      "activations/layer0_attention_weight_min": -13.998083114624023,
      "activations/layer10_attention_weight_max": 33.820762634277344,
      "activations/layer10_attention_weight_min": -34.62749099731445,
      "activations/layer11_attention_weight_max": 30.479848861694336,
      "activations/layer11_attention_weight_min": -32.94487762451172,
      "activations/layer12_attention_weight_max": 17.193710327148438,
      "activations/layer12_attention_weight_min": -23.3519229888916,
      "activations/layer13_attention_weight_max": 34.69179916381836,
      "activations/layer13_attention_weight_min": -27.057405471801758,
      "activations/layer14_attention_weight_max": 32.64375305175781,
      "activations/layer14_attention_weight_min": -30.40780258178711,
      "activations/layer15_attention_weight_max": 31.349506378173828,
      "activations/layer15_attention_weight_min": -29.828943252563477,
      "activations/layer16_attention_weight_max": 29.685216903686523,
      "activations/layer16_attention_weight_min": -29.14640235900879,
      "activations/layer17_attention_weight_max": 49.469329833984375,
      "activations/layer17_attention_weight_min": -41.86796569824219,
      "activations/layer18_attention_weight_max": 46.008487701416016,
      "activations/layer18_attention_weight_min": -36.705379486083984,
      "activations/layer19_attention_weight_max": 21.5711727142334,
      "activations/layer19_attention_weight_min": -22.496458053588867,
      "activations/layer1_attention_weight_max": 16.2735652923584,
      "activations/layer1_attention_weight_min": -16.38211441040039,
      "activations/layer20_attention_weight_max": 20.246557235717773,
      "activations/layer20_attention_weight_min": -20.656654357910156,
      "activations/layer21_attention_weight_max": 36.33966064453125,
      "activations/layer21_attention_weight_min": -23.66166114807129,
      "activations/layer22_attention_weight_max": 28.5606689453125,
      "activations/layer22_attention_weight_min": -25.694007873535156,
      "activations/layer23_attention_weight_max": 36.75613784790039,
      "activations/layer23_attention_weight_min": -23.033981323242188,
      "activations/layer2_attention_weight_max": 34.91674041748047,
      "activations/layer2_attention_weight_min": -35.29315948486328,
      "activations/layer3_attention_weight_max": 94.01055908203125,
      "activations/layer3_attention_weight_min": -98.09114074707031,
      "activations/layer4_attention_weight_max": 96.5484848022461,
      "activations/layer4_attention_weight_min": -91.9012680053711,
      "activations/layer5_attention_weight_max": 70.62872314453125,
      "activations/layer5_attention_weight_min": -77.51213073730469,
      "activations/layer6_attention_weight_max": 51.68411636352539,
      "activations/layer6_attention_weight_min": -53.13375473022461,
      "activations/layer7_attention_weight_max": 65.93633270263672,
      "activations/layer7_attention_weight_min": -66.09980773925781,
      "activations/layer8_attention_weight_max": 45.8292350769043,
      "activations/layer8_attention_weight_min": -49.0042610168457,
      "activations/layer9_attention_weight_max": 44.85154342651367,
      "activations/layer9_attention_weight_min": -45.56977462768555,
      "epoch": 14.17,
      "learning_rate": 5.9205681818181815e-05,
      "loss": 2.7508,
      "step": 243900
    },
    {
      "activations/layer0_attention_weight_max": 16.603370666503906,
      "activations/layer0_attention_weight_min": -14.10009765625,
      "activations/layer10_attention_weight_max": 36.955650329589844,
      "activations/layer10_attention_weight_min": -35.67555618286133,
      "activations/layer11_attention_weight_max": 32.577171325683594,
      "activations/layer11_attention_weight_min": -35.534278869628906,
      "activations/layer12_attention_weight_max": 19.457935333251953,
      "activations/layer12_attention_weight_min": -24.30015754699707,
      "activations/layer13_attention_weight_max": 40.67829895019531,
      "activations/layer13_attention_weight_min": -28.963869094848633,
      "activations/layer14_attention_weight_max": 35.38652038574219,
      "activations/layer14_attention_weight_min": -29.998329162597656,
      "activations/layer15_attention_weight_max": 34.7863655090332,
      "activations/layer15_attention_weight_min": -29.775707244873047,
      "activations/layer16_attention_weight_max": 30.959854125976562,
      "activations/layer16_attention_weight_min": -29.95446014404297,
      "activations/layer17_attention_weight_max": 51.60926818847656,
      "activations/layer17_attention_weight_min": -44.506263732910156,
      "activations/layer18_attention_weight_max": 48.19948959350586,
      "activations/layer18_attention_weight_min": -36.808475494384766,
      "activations/layer19_attention_weight_max": 24.595272064208984,
      "activations/layer19_attention_weight_min": -21.195283889770508,
      "activations/layer1_attention_weight_max": 16.45383644104004,
      "activations/layer1_attention_weight_min": -16.425811767578125,
      "activations/layer20_attention_weight_max": 22.945322036743164,
      "activations/layer20_attention_weight_min": -22.247047424316406,
      "activations/layer21_attention_weight_max": 47.178401947021484,
      "activations/layer21_attention_weight_min": -27.481807708740234,
      "activations/layer22_attention_weight_max": 32.0565185546875,
      "activations/layer22_attention_weight_min": -25.603479385375977,
      "activations/layer23_attention_weight_max": 42.50188446044922,
      "activations/layer23_attention_weight_min": -24.29439353942871,
      "activations/layer2_attention_weight_max": 32.49085235595703,
      "activations/layer2_attention_weight_min": -32.04042053222656,
      "activations/layer3_attention_weight_max": 95.80224609375,
      "activations/layer3_attention_weight_min": -99.6266860961914,
      "activations/layer4_attention_weight_max": 94.84557342529297,
      "activations/layer4_attention_weight_min": -90.00830078125,
      "activations/layer5_attention_weight_max": 68.20133972167969,
      "activations/layer5_attention_weight_min": -75.2884750366211,
      "activations/layer6_attention_weight_max": 51.81815719604492,
      "activations/layer6_attention_weight_min": -51.347843170166016,
      "activations/layer7_attention_weight_max": 62.99497604370117,
      "activations/layer7_attention_weight_min": -64.37104034423828,
      "activations/layer8_attention_weight_max": 49.11623001098633,
      "activations/layer8_attention_weight_min": -51.409854888916016,
      "activations/layer9_attention_weight_max": 47.084678649902344,
      "activations/layer9_attention_weight_min": -46.03823471069336,
      "epoch": 14.17,
      "learning_rate": 5.918674242424242e-05,
      "loss": 2.7663,
      "step": 243950
    },
    {
      "activations/layer0_attention_weight_max": 17.275924682617188,
      "activations/layer0_attention_weight_min": -13.723484992980957,
      "activations/layer10_attention_weight_max": 39.317378997802734,
      "activations/layer10_attention_weight_min": -37.71727752685547,
      "activations/layer11_attention_weight_max": 36.29160690307617,
      "activations/layer11_attention_weight_min": -35.869651794433594,
      "activations/layer12_attention_weight_max": 20.53023910522461,
      "activations/layer12_attention_weight_min": -23.382688522338867,
      "activations/layer13_attention_weight_max": 45.896305084228516,
      "activations/layer13_attention_weight_min": -30.488067626953125,
      "activations/layer14_attention_weight_max": 48.55640411376953,
      "activations/layer14_attention_weight_min": -32.173377990722656,
      "activations/layer15_attention_weight_max": 39.921295166015625,
      "activations/layer15_attention_weight_min": -32.39470291137695,
      "activations/layer16_attention_weight_max": 38.64398193359375,
      "activations/layer16_attention_weight_min": -34.687782287597656,
      "activations/layer17_attention_weight_max": 62.55136489868164,
      "activations/layer17_attention_weight_min": -51.69166564941406,
      "activations/layer18_attention_weight_max": 58.737266540527344,
      "activations/layer18_attention_weight_min": -43.84101486206055,
      "activations/layer19_attention_weight_max": 26.177230834960938,
      "activations/layer19_attention_weight_min": -25.0810546875,
      "activations/layer1_attention_weight_max": 15.895158767700195,
      "activations/layer1_attention_weight_min": -15.377415657043457,
      "activations/layer20_attention_weight_max": 30.437026977539062,
      "activations/layer20_attention_weight_min": -21.921262741088867,
      "activations/layer21_attention_weight_max": 45.687679290771484,
      "activations/layer21_attention_weight_min": -24.93558692932129,
      "activations/layer22_attention_weight_max": 36.1693229675293,
      "activations/layer22_attention_weight_min": -27.82193946838379,
      "activations/layer23_attention_weight_max": 47.78971481323242,
      "activations/layer23_attention_weight_min": -26.02382469177246,
      "activations/layer2_attention_weight_max": 33.364784240722656,
      "activations/layer2_attention_weight_min": -33.19856643676758,
      "activations/layer3_attention_weight_max": 96.2928466796875,
      "activations/layer3_attention_weight_min": -94.90984344482422,
      "activations/layer4_attention_weight_max": 97.58130645751953,
      "activations/layer4_attention_weight_min": -91.28694915771484,
      "activations/layer5_attention_weight_max": 74.5028305053711,
      "activations/layer5_attention_weight_min": -80.97694396972656,
      "activations/layer6_attention_weight_max": 54.83842086791992,
      "activations/layer6_attention_weight_min": -54.03251647949219,
      "activations/layer7_attention_weight_max": 68.92619323730469,
      "activations/layer7_attention_weight_min": -65.59476470947266,
      "activations/layer8_attention_weight_max": 45.88142776489258,
      "activations/layer8_attention_weight_min": -51.13978576660156,
      "activations/layer9_attention_weight_max": 44.9820671081543,
      "activations/layer9_attention_weight_min": -45.38471603393555,
      "epoch": 14.18,
      "learning_rate": 5.9167803030303025e-05,
      "loss": 2.7415,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4276,
      "eval_samples_per_second": 509.514,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4276,
      "eval_openwebtext_samples_per_second": 509.514,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_wikitext_loss": 2.94140625,
      "eval_wikitext_ppl": 18.942465433314446,
      "eval_wikitext_runtime": 1.9303,
      "eval_wikitext_samples_per_second": 236.234,
      "step": 244000
    },
    {
      "epoch": 14.18,
      "eval_lambada_loss": 2.501953125,
      "eval_lambada_ppl": 12.206311145618837,
      "eval_lambada_runtime": 9.5208,
      "eval_lambada_samples_per_second": 511.404,
      "step": 244000
    },
    {
      "activations/layer0_attention_weight_max": 16.945419311523438,
      "activations/layer0_attention_weight_min": -14.132013320922852,
      "activations/layer10_attention_weight_max": 36.759971618652344,
      "activations/layer10_attention_weight_min": -37.44361114501953,
      "activations/layer11_attention_weight_max": 32.62038803100586,
      "activations/layer11_attention_weight_min": -34.59038162231445,
      "activations/layer12_attention_weight_max": 18.885692596435547,
      "activations/layer12_attention_weight_min": -24.53348159790039,
      "activations/layer13_attention_weight_max": 35.177642822265625,
      "activations/layer13_attention_weight_min": -27.55821990966797,
      "activations/layer14_attention_weight_max": 39.05399703979492,
      "activations/layer14_attention_weight_min": -29.96055030822754,
      "activations/layer15_attention_weight_max": 34.13750457763672,
      "activations/layer15_attention_weight_min": -28.38001823425293,
      "activations/layer16_attention_weight_max": 35.76931381225586,
      "activations/layer16_attention_weight_min": -30.31254005432129,
      "activations/layer17_attention_weight_max": 51.49727249145508,
      "activations/layer17_attention_weight_min": -44.48732376098633,
      "activations/layer18_attention_weight_max": 48.034828186035156,
      "activations/layer18_attention_weight_min": -37.4743537902832,
      "activations/layer19_attention_weight_max": 24.618865966796875,
      "activations/layer19_attention_weight_min": -21.533702850341797,
      "activations/layer1_attention_weight_max": 16.715961456298828,
      "activations/layer1_attention_weight_min": -17.094852447509766,
      "activations/layer20_attention_weight_max": 22.893268585205078,
      "activations/layer20_attention_weight_min": -20.063804626464844,
      "activations/layer21_attention_weight_max": 41.3712043762207,
      "activations/layer21_attention_weight_min": -26.90960693359375,
      "activations/layer22_attention_weight_max": 31.30455207824707,
      "activations/layer22_attention_weight_min": -26.091999053955078,
      "activations/layer23_attention_weight_max": 41.06219482421875,
      "activations/layer23_attention_weight_min": -23.321455001831055,
      "activations/layer2_attention_weight_max": 33.545928955078125,
      "activations/layer2_attention_weight_min": -32.16313552856445,
      "activations/layer3_attention_weight_max": 100.34729766845703,
      "activations/layer3_attention_weight_min": -99.18376922607422,
      "activations/layer4_attention_weight_max": 98.31394958496094,
      "activations/layer4_attention_weight_min": -93.40866088867188,
      "activations/layer5_attention_weight_max": 72.10397338867188,
      "activations/layer5_attention_weight_min": -81.46968841552734,
      "activations/layer6_attention_weight_max": 51.44240951538086,
      "activations/layer6_attention_weight_min": -55.63616180419922,
      "activations/layer7_attention_weight_max": 69.35539245605469,
      "activations/layer7_attention_weight_min": -70.82676696777344,
      "activations/layer8_attention_weight_max": 47.45852279663086,
      "activations/layer8_attention_weight_min": -51.267723083496094,
      "activations/layer9_attention_weight_max": 44.08195114135742,
      "activations/layer9_attention_weight_min": -46.787559509277344,
      "epoch": 14.18,
      "learning_rate": 5.9148863636363626e-05,
      "loss": 2.7634,
      "step": 244050
    },
    {
      "activations/layer0_attention_weight_max": 16.340492248535156,
      "activations/layer0_attention_weight_min": -13.914511680603027,
      "activations/layer10_attention_weight_max": 38.473388671875,
      "activations/layer10_attention_weight_min": -34.851009368896484,
      "activations/layer11_attention_weight_max": 32.177730560302734,
      "activations/layer11_attention_weight_min": -32.10742950439453,
      "activations/layer12_attention_weight_max": 18.884958267211914,
      "activations/layer12_attention_weight_min": -26.22450065612793,
      "activations/layer13_attention_weight_max": 39.808372497558594,
      "activations/layer13_attention_weight_min": -26.046180725097656,
      "activations/layer14_attention_weight_max": 37.75235366821289,
      "activations/layer14_attention_weight_min": -29.997406005859375,
      "activations/layer15_attention_weight_max": 34.471500396728516,
      "activations/layer15_attention_weight_min": -30.497394561767578,
      "activations/layer16_attention_weight_max": 34.74991989135742,
      "activations/layer16_attention_weight_min": -31.479740142822266,
      "activations/layer17_attention_weight_max": 53.846439361572266,
      "activations/layer17_attention_weight_min": -44.33289337158203,
      "activations/layer18_attention_weight_max": 49.77253723144531,
      "activations/layer18_attention_weight_min": -38.20707321166992,
      "activations/layer19_attention_weight_max": 24.51316261291504,
      "activations/layer19_attention_weight_min": -24.52923011779785,
      "activations/layer1_attention_weight_max": 17.56312370300293,
      "activations/layer1_attention_weight_min": -15.820972442626953,
      "activations/layer20_attention_weight_max": 26.259965896606445,
      "activations/layer20_attention_weight_min": -21.336851119995117,
      "activations/layer21_attention_weight_max": 43.90104293823242,
      "activations/layer21_attention_weight_min": -24.322208404541016,
      "activations/layer22_attention_weight_max": 34.65168762207031,
      "activations/layer22_attention_weight_min": -27.952117919921875,
      "activations/layer23_attention_weight_max": 40.596641540527344,
      "activations/layer23_attention_weight_min": -23.640783309936523,
      "activations/layer2_attention_weight_max": 32.32548141479492,
      "activations/layer2_attention_weight_min": -33.36378860473633,
      "activations/layer3_attention_weight_max": 91.32271575927734,
      "activations/layer3_attention_weight_min": -101.13243103027344,
      "activations/layer4_attention_weight_max": 93.15035247802734,
      "activations/layer4_attention_weight_min": -94.96134185791016,
      "activations/layer5_attention_weight_max": 69.34324645996094,
      "activations/layer5_attention_weight_min": -77.190673828125,
      "activations/layer6_attention_weight_max": 53.261314392089844,
      "activations/layer6_attention_weight_min": -50.56022262573242,
      "activations/layer7_attention_weight_max": 64.72010040283203,
      "activations/layer7_attention_weight_min": -60.14447784423828,
      "activations/layer8_attention_weight_max": 44.27099609375,
      "activations/layer8_attention_weight_min": -45.8261604309082,
      "activations/layer9_attention_weight_max": 42.89054870605469,
      "activations/layer9_attention_weight_min": -43.8690299987793,
      "epoch": 14.18,
      "learning_rate": 5.912992424242424e-05,
      "loss": 2.7581,
      "step": 244100
    },
    {
      "activations/layer0_attention_weight_max": 16.962820053100586,
      "activations/layer0_attention_weight_min": -14.617073059082031,
      "activations/layer10_attention_weight_max": 35.439910888671875,
      "activations/layer10_attention_weight_min": -36.317352294921875,
      "activations/layer11_attention_weight_max": 31.851634979248047,
      "activations/layer11_attention_weight_min": -34.19155502319336,
      "activations/layer12_attention_weight_max": 19.007863998413086,
      "activations/layer12_attention_weight_min": -27.08322525024414,
      "activations/layer13_attention_weight_max": 32.983985900878906,
      "activations/layer13_attention_weight_min": -29.703720092773438,
      "activations/layer14_attention_weight_max": 33.63597869873047,
      "activations/layer14_attention_weight_min": -30.305368423461914,
      "activations/layer15_attention_weight_max": 32.52968978881836,
      "activations/layer15_attention_weight_min": -31.713685989379883,
      "activations/layer16_attention_weight_max": 32.2898063659668,
      "activations/layer16_attention_weight_min": -31.431623458862305,
      "activations/layer17_attention_weight_max": 52.99044418334961,
      "activations/layer17_attention_weight_min": -47.63751220703125,
      "activations/layer18_attention_weight_max": 47.83075714111328,
      "activations/layer18_attention_weight_min": -42.525665283203125,
      "activations/layer19_attention_weight_max": 26.893850326538086,
      "activations/layer19_attention_weight_min": -23.94293785095215,
      "activations/layer1_attention_weight_max": 15.839231491088867,
      "activations/layer1_attention_weight_min": -14.530303955078125,
      "activations/layer20_attention_weight_max": 25.105117797851562,
      "activations/layer20_attention_weight_min": -22.7161865234375,
      "activations/layer21_attention_weight_max": 40.11796188354492,
      "activations/layer21_attention_weight_min": -24.448938369750977,
      "activations/layer22_attention_weight_max": 31.160797119140625,
      "activations/layer22_attention_weight_min": -25.467039108276367,
      "activations/layer23_attention_weight_max": 43.194252014160156,
      "activations/layer23_attention_weight_min": -24.543380737304688,
      "activations/layer2_attention_weight_max": 33.62896728515625,
      "activations/layer2_attention_weight_min": -33.32613754272461,
      "activations/layer3_attention_weight_max": 98.05810546875,
      "activations/layer3_attention_weight_min": -93.94499969482422,
      "activations/layer4_attention_weight_max": 93.62895965576172,
      "activations/layer4_attention_weight_min": -92.65833282470703,
      "activations/layer5_attention_weight_max": 70.10978698730469,
      "activations/layer5_attention_weight_min": -80.11473083496094,
      "activations/layer6_attention_weight_max": 52.1756706237793,
      "activations/layer6_attention_weight_min": -54.31838607788086,
      "activations/layer7_attention_weight_max": 64.95674896240234,
      "activations/layer7_attention_weight_min": -65.45390319824219,
      "activations/layer8_attention_weight_max": 46.7794075012207,
      "activations/layer8_attention_weight_min": -49.0623664855957,
      "activations/layer9_attention_weight_max": 42.313472747802734,
      "activations/layer9_attention_weight_min": -45.63429641723633,
      "epoch": 14.19,
      "learning_rate": 5.911098484848484e-05,
      "loss": 2.7569,
      "step": 244150
    },
    {
      "activations/layer0_attention_weight_max": 16.39605712890625,
      "activations/layer0_attention_weight_min": -14.116864204406738,
      "activations/layer10_attention_weight_max": 36.370506286621094,
      "activations/layer10_attention_weight_min": -36.342674255371094,
      "activations/layer11_attention_weight_max": 31.571121215820312,
      "activations/layer11_attention_weight_min": -33.08442687988281,
      "activations/layer12_attention_weight_max": 21.61281394958496,
      "activations/layer12_attention_weight_min": -23.585058212280273,
      "activations/layer13_attention_weight_max": 39.91368103027344,
      "activations/layer13_attention_weight_min": -30.14987564086914,
      "activations/layer14_attention_weight_max": 37.78263854980469,
      "activations/layer14_attention_weight_min": -30.830995559692383,
      "activations/layer15_attention_weight_max": 34.50925827026367,
      "activations/layer15_attention_weight_min": -30.256023406982422,
      "activations/layer16_attention_weight_max": 35.706119537353516,
      "activations/layer16_attention_weight_min": -32.53555679321289,
      "activations/layer17_attention_weight_max": 53.71885299682617,
      "activations/layer17_attention_weight_min": -45.257720947265625,
      "activations/layer18_attention_weight_max": 49.806976318359375,
      "activations/layer18_attention_weight_min": -40.75098419189453,
      "activations/layer19_attention_weight_max": 24.995994567871094,
      "activations/layer19_attention_weight_min": -21.43899154663086,
      "activations/layer1_attention_weight_max": 16.054399490356445,
      "activations/layer1_attention_weight_min": -14.197245597839355,
      "activations/layer20_attention_weight_max": 23.064128875732422,
      "activations/layer20_attention_weight_min": -20.65641975402832,
      "activations/layer21_attention_weight_max": 38.352081298828125,
      "activations/layer21_attention_weight_min": -23.500818252563477,
      "activations/layer22_attention_weight_max": 30.923738479614258,
      "activations/layer22_attention_weight_min": -26.79195785522461,
      "activations/layer23_attention_weight_max": 38.428199768066406,
      "activations/layer23_attention_weight_min": -25.528533935546875,
      "activations/layer2_attention_weight_max": 32.689273834228516,
      "activations/layer2_attention_weight_min": -32.78265380859375,
      "activations/layer3_attention_weight_max": 95.54105377197266,
      "activations/layer3_attention_weight_min": -101.14185333251953,
      "activations/layer4_attention_weight_max": 91.64827728271484,
      "activations/layer4_attention_weight_min": -94.6650161743164,
      "activations/layer5_attention_weight_max": 67.07537078857422,
      "activations/layer5_attention_weight_min": -82.24995422363281,
      "activations/layer6_attention_weight_max": 53.0610466003418,
      "activations/layer6_attention_weight_min": -50.68746566772461,
      "activations/layer7_attention_weight_max": 63.55540084838867,
      "activations/layer7_attention_weight_min": -65.78986358642578,
      "activations/layer8_attention_weight_max": 45.475685119628906,
      "activations/layer8_attention_weight_min": -48.42475891113281,
      "activations/layer9_attention_weight_max": 46.005435943603516,
      "activations/layer9_attention_weight_min": -48.23087692260742,
      "epoch": 14.19,
      "learning_rate": 5.9092045454545444e-05,
      "loss": 2.7526,
      "step": 244200
    },
    {
      "activations/layer0_attention_weight_max": 16.632747650146484,
      "activations/layer0_attention_weight_min": -13.984832763671875,
      "activations/layer10_attention_weight_max": 36.611907958984375,
      "activations/layer10_attention_weight_min": -33.85803985595703,
      "activations/layer11_attention_weight_max": 32.11636734008789,
      "activations/layer11_attention_weight_min": -32.2423095703125,
      "activations/layer12_attention_weight_max": 19.653827667236328,
      "activations/layer12_attention_weight_min": -26.722965240478516,
      "activations/layer13_attention_weight_max": 33.58629608154297,
      "activations/layer13_attention_weight_min": -27.03818702697754,
      "activations/layer14_attention_weight_max": 31.890644073486328,
      "activations/layer14_attention_weight_min": -31.030969619750977,
      "activations/layer15_attention_weight_max": 31.400684356689453,
      "activations/layer15_attention_weight_min": -29.642457962036133,
      "activations/layer16_attention_weight_max": 31.767200469970703,
      "activations/layer16_attention_weight_min": -30.198339462280273,
      "activations/layer17_attention_weight_max": 47.291629791259766,
      "activations/layer17_attention_weight_min": -41.864559173583984,
      "activations/layer18_attention_weight_max": 41.581634521484375,
      "activations/layer18_attention_weight_min": -35.546627044677734,
      "activations/layer19_attention_weight_max": 22.161731719970703,
      "activations/layer19_attention_weight_min": -21.259986877441406,
      "activations/layer1_attention_weight_max": 17.16924285888672,
      "activations/layer1_attention_weight_min": -15.146760940551758,
      "activations/layer20_attention_weight_max": 24.027294158935547,
      "activations/layer20_attention_weight_min": -23.556943893432617,
      "activations/layer21_attention_weight_max": 32.371395111083984,
      "activations/layer21_attention_weight_min": -25.554147720336914,
      "activations/layer22_attention_weight_max": 28.45531463623047,
      "activations/layer22_attention_weight_min": -26.03790283203125,
      "activations/layer23_attention_weight_max": 36.47229766845703,
      "activations/layer23_attention_weight_min": -23.133182525634766,
      "activations/layer2_attention_weight_max": 33.74458694458008,
      "activations/layer2_attention_weight_min": -32.7083740234375,
      "activations/layer3_attention_weight_max": 95.93463134765625,
      "activations/layer3_attention_weight_min": -98.12268829345703,
      "activations/layer4_attention_weight_max": 96.87626647949219,
      "activations/layer4_attention_weight_min": -91.43585968017578,
      "activations/layer5_attention_weight_max": 73.166259765625,
      "activations/layer5_attention_weight_min": -74.79700469970703,
      "activations/layer6_attention_weight_max": 51.834938049316406,
      "activations/layer6_attention_weight_min": -50.000003814697266,
      "activations/layer7_attention_weight_max": 64.60221099853516,
      "activations/layer7_attention_weight_min": -62.45368576049805,
      "activations/layer8_attention_weight_max": 45.413673400878906,
      "activations/layer8_attention_weight_min": -46.54808807373047,
      "activations/layer9_attention_weight_max": 44.340084075927734,
      "activations/layer9_attention_weight_min": -44.970726013183594,
      "epoch": 14.19,
      "learning_rate": 5.907310606060605e-05,
      "loss": 2.7625,
      "step": 244250
    },
    {
      "activations/layer0_attention_weight_max": 17.07921600341797,
      "activations/layer0_attention_weight_min": -14.299471855163574,
      "activations/layer10_attention_weight_max": 38.11419677734375,
      "activations/layer10_attention_weight_min": -38.1667594909668,
      "activations/layer11_attention_weight_max": 34.78964614868164,
      "activations/layer11_attention_weight_min": -35.4248161315918,
      "activations/layer12_attention_weight_max": 19.24370765686035,
      "activations/layer12_attention_weight_min": -25.05133819580078,
      "activations/layer13_attention_weight_max": 36.190711975097656,
      "activations/layer13_attention_weight_min": -28.851909637451172,
      "activations/layer14_attention_weight_max": 37.195213317871094,
      "activations/layer14_attention_weight_min": -29.868005752563477,
      "activations/layer15_attention_weight_max": 39.576377868652344,
      "activations/layer15_attention_weight_min": -29.636247634887695,
      "activations/layer16_attention_weight_max": 35.47840881347656,
      "activations/layer16_attention_weight_min": -31.38953399658203,
      "activations/layer17_attention_weight_max": 56.95878982543945,
      "activations/layer17_attention_weight_min": -44.07868576049805,
      "activations/layer18_attention_weight_max": 54.28662872314453,
      "activations/layer18_attention_weight_min": -38.58779525756836,
      "activations/layer19_attention_weight_max": 24.922306060791016,
      "activations/layer19_attention_weight_min": -23.37105369567871,
      "activations/layer1_attention_weight_max": 16.7750186920166,
      "activations/layer1_attention_weight_min": -16.371578216552734,
      "activations/layer20_attention_weight_max": 23.701107025146484,
      "activations/layer20_attention_weight_min": -22.72465705871582,
      "activations/layer21_attention_weight_max": 41.589813232421875,
      "activations/layer21_attention_weight_min": -25.892648696899414,
      "activations/layer22_attention_weight_max": 30.99778175354004,
      "activations/layer22_attention_weight_min": -26.617645263671875,
      "activations/layer23_attention_weight_max": 40.244625091552734,
      "activations/layer23_attention_weight_min": -26.388652801513672,
      "activations/layer2_attention_weight_max": 34.520206451416016,
      "activations/layer2_attention_weight_min": -33.426212310791016,
      "activations/layer3_attention_weight_max": 99.85828399658203,
      "activations/layer3_attention_weight_min": -99.5519790649414,
      "activations/layer4_attention_weight_max": 98.90592193603516,
      "activations/layer4_attention_weight_min": -95.76280975341797,
      "activations/layer5_attention_weight_max": 73.05747985839844,
      "activations/layer5_attention_weight_min": -83.4274673461914,
      "activations/layer6_attention_weight_max": 53.06755828857422,
      "activations/layer6_attention_weight_min": -54.260616302490234,
      "activations/layer7_attention_weight_max": 68.1545181274414,
      "activations/layer7_attention_weight_min": -68.4176254272461,
      "activations/layer8_attention_weight_max": 51.05805587768555,
      "activations/layer8_attention_weight_min": -52.58002471923828,
      "activations/layer9_attention_weight_max": 53.1029052734375,
      "activations/layer9_attention_weight_min": -48.59845733642578,
      "epoch": 14.2,
      "learning_rate": 5.905416666666666e-05,
      "loss": 2.7548,
      "step": 244300
    },
    {
      "activations/layer0_attention_weight_max": 17.795751571655273,
      "activations/layer0_attention_weight_min": -14.50046157836914,
      "activations/layer10_attention_weight_max": 39.77608871459961,
      "activations/layer10_attention_weight_min": -38.105892181396484,
      "activations/layer11_attention_weight_max": 34.52548599243164,
      "activations/layer11_attention_weight_min": -34.71409225463867,
      "activations/layer12_attention_weight_max": 21.595903396606445,
      "activations/layer12_attention_weight_min": -23.16498374938965,
      "activations/layer13_attention_weight_max": 35.210174560546875,
      "activations/layer13_attention_weight_min": -28.652759552001953,
      "activations/layer14_attention_weight_max": 35.89844512939453,
      "activations/layer14_attention_weight_min": -31.51422119140625,
      "activations/layer15_attention_weight_max": 34.01605987548828,
      "activations/layer15_attention_weight_min": -31.307254791259766,
      "activations/layer16_attention_weight_max": 34.760074615478516,
      "activations/layer16_attention_weight_min": -29.69656753540039,
      "activations/layer17_attention_weight_max": 51.20823669433594,
      "activations/layer17_attention_weight_min": -46.663169860839844,
      "activations/layer18_attention_weight_max": 49.9305419921875,
      "activations/layer18_attention_weight_min": -40.01740646362305,
      "activations/layer19_attention_weight_max": 21.93977165222168,
      "activations/layer19_attention_weight_min": -20.929794311523438,
      "activations/layer1_attention_weight_max": 16.760738372802734,
      "activations/layer1_attention_weight_min": -15.623409271240234,
      "activations/layer20_attention_weight_max": 24.654451370239258,
      "activations/layer20_attention_weight_min": -21.231897354125977,
      "activations/layer21_attention_weight_max": 43.220985412597656,
      "activations/layer21_attention_weight_min": -22.31633758544922,
      "activations/layer22_attention_weight_max": 31.07404327392578,
      "activations/layer22_attention_weight_min": -25.72905158996582,
      "activations/layer23_attention_weight_max": 40.05878829956055,
      "activations/layer23_attention_weight_min": -24.286685943603516,
      "activations/layer2_attention_weight_max": 33.98942565917969,
      "activations/layer2_attention_weight_min": -32.892723083496094,
      "activations/layer3_attention_weight_max": 99.81216430664062,
      "activations/layer3_attention_weight_min": -102.95150756835938,
      "activations/layer4_attention_weight_max": 100.56666564941406,
      "activations/layer4_attention_weight_min": -95.44638061523438,
      "activations/layer5_attention_weight_max": 69.07958984375,
      "activations/layer5_attention_weight_min": -82.27429962158203,
      "activations/layer6_attention_weight_max": 54.01218032836914,
      "activations/layer6_attention_weight_min": -51.93190383911133,
      "activations/layer7_attention_weight_max": 67.99617004394531,
      "activations/layer7_attention_weight_min": -67.12005615234375,
      "activations/layer8_attention_weight_max": 51.272159576416016,
      "activations/layer8_attention_weight_min": -49.13323974609375,
      "activations/layer9_attention_weight_max": 52.45077133178711,
      "activations/layer9_attention_weight_min": -46.113765716552734,
      "epoch": 14.2,
      "learning_rate": 5.903522727272727e-05,
      "loss": 2.739,
      "step": 244350
    },
    {
      "activations/layer0_attention_weight_max": 16.867311477661133,
      "activations/layer0_attention_weight_min": -14.145920753479004,
      "activations/layer10_attention_weight_max": 35.498207092285156,
      "activations/layer10_attention_weight_min": -33.5850715637207,
      "activations/layer11_attention_weight_max": 32.18645095825195,
      "activations/layer11_attention_weight_min": -30.597692489624023,
      "activations/layer12_attention_weight_max": 17.899106979370117,
      "activations/layer12_attention_weight_min": -28.49795150756836,
      "activations/layer13_attention_weight_max": 42.04609680175781,
      "activations/layer13_attention_weight_min": -27.74134063720703,
      "activations/layer14_attention_weight_max": 36.05133819580078,
      "activations/layer14_attention_weight_min": -31.004377365112305,
      "activations/layer15_attention_weight_max": 32.98545455932617,
      "activations/layer15_attention_weight_min": -30.847816467285156,
      "activations/layer16_attention_weight_max": 34.82491683959961,
      "activations/layer16_attention_weight_min": -32.74346160888672,
      "activations/layer17_attention_weight_max": 52.21352005004883,
      "activations/layer17_attention_weight_min": -43.02839660644531,
      "activations/layer18_attention_weight_max": 46.917381286621094,
      "activations/layer18_attention_weight_min": -37.777347564697266,
      "activations/layer19_attention_weight_max": 24.92171859741211,
      "activations/layer19_attention_weight_min": -21.99873924255371,
      "activations/layer1_attention_weight_max": 16.335668563842773,
      "activations/layer1_attention_weight_min": -16.375822067260742,
      "activations/layer20_attention_weight_max": 25.47130584716797,
      "activations/layer20_attention_weight_min": -22.805381774902344,
      "activations/layer21_attention_weight_max": 43.792327880859375,
      "activations/layer21_attention_weight_min": -26.760068893432617,
      "activations/layer22_attention_weight_max": 32.4416618347168,
      "activations/layer22_attention_weight_min": -25.879426956176758,
      "activations/layer23_attention_weight_max": 42.77442932128906,
      "activations/layer23_attention_weight_min": -25.89893913269043,
      "activations/layer2_attention_weight_max": 34.43363952636719,
      "activations/layer2_attention_weight_min": -31.75235366821289,
      "activations/layer3_attention_weight_max": 99.23055267333984,
      "activations/layer3_attention_weight_min": -96.06847381591797,
      "activations/layer4_attention_weight_max": 92.2140121459961,
      "activations/layer4_attention_weight_min": -89.1584701538086,
      "activations/layer5_attention_weight_max": 67.8428726196289,
      "activations/layer5_attention_weight_min": -79.45858764648438,
      "activations/layer6_attention_weight_max": 50.69277572631836,
      "activations/layer6_attention_weight_min": -50.01045227050781,
      "activations/layer7_attention_weight_max": 63.97473907470703,
      "activations/layer7_attention_weight_min": -60.94648361206055,
      "activations/layer8_attention_weight_max": 45.51401901245117,
      "activations/layer8_attention_weight_min": -45.08223342895508,
      "activations/layer9_attention_weight_max": 52.464820861816406,
      "activations/layer9_attention_weight_min": -43.966773986816406,
      "epoch": 14.2,
      "learning_rate": 5.901628787878787e-05,
      "loss": 2.7636,
      "step": 244400
    },
    {
      "activations/layer0_attention_weight_max": 16.651018142700195,
      "activations/layer0_attention_weight_min": -14.47494888305664,
      "activations/layer10_attention_weight_max": 35.646400451660156,
      "activations/layer10_attention_weight_min": -35.19480514526367,
      "activations/layer11_attention_weight_max": 31.86736297607422,
      "activations/layer11_attention_weight_min": -35.835411071777344,
      "activations/layer12_attention_weight_max": 19.880102157592773,
      "activations/layer12_attention_weight_min": -24.432275772094727,
      "activations/layer13_attention_weight_max": 36.481727600097656,
      "activations/layer13_attention_weight_min": -29.00864028930664,
      "activations/layer14_attention_weight_max": 35.271053314208984,
      "activations/layer14_attention_weight_min": -30.842058181762695,
      "activations/layer15_attention_weight_max": 34.54705810546875,
      "activations/layer15_attention_weight_min": -29.66000747680664,
      "activations/layer16_attention_weight_max": 35.41851043701172,
      "activations/layer16_attention_weight_min": -32.30429458618164,
      "activations/layer17_attention_weight_max": 52.862178802490234,
      "activations/layer17_attention_weight_min": -45.71170425415039,
      "activations/layer18_attention_weight_max": 47.62522506713867,
      "activations/layer18_attention_weight_min": -37.04003143310547,
      "activations/layer19_attention_weight_max": 22.167757034301758,
      "activations/layer19_attention_weight_min": -23.10687828063965,
      "activations/layer1_attention_weight_max": 17.526166915893555,
      "activations/layer1_attention_weight_min": -14.568912506103516,
      "activations/layer20_attention_weight_max": 22.98531150817871,
      "activations/layer20_attention_weight_min": -22.204065322875977,
      "activations/layer21_attention_weight_max": 40.76011657714844,
      "activations/layer21_attention_weight_min": -24.529539108276367,
      "activations/layer22_attention_weight_max": 31.235811233520508,
      "activations/layer22_attention_weight_min": -26.22443199157715,
      "activations/layer23_attention_weight_max": 39.52906036376953,
      "activations/layer23_attention_weight_min": -24.78156280517578,
      "activations/layer2_attention_weight_max": 33.79893493652344,
      "activations/layer2_attention_weight_min": -33.57969665527344,
      "activations/layer3_attention_weight_max": 99.73697662353516,
      "activations/layer3_attention_weight_min": -107.82088470458984,
      "activations/layer4_attention_weight_max": 97.39330291748047,
      "activations/layer4_attention_weight_min": -96.84209442138672,
      "activations/layer5_attention_weight_max": 70.13885498046875,
      "activations/layer5_attention_weight_min": -81.66020202636719,
      "activations/layer6_attention_weight_max": 53.89485168457031,
      "activations/layer6_attention_weight_min": -50.66994094848633,
      "activations/layer7_attention_weight_max": 68.71905517578125,
      "activations/layer7_attention_weight_min": -62.920631408691406,
      "activations/layer8_attention_weight_max": 45.28792953491211,
      "activations/layer8_attention_weight_min": -48.10622024536133,
      "activations/layer9_attention_weight_max": 44.55445861816406,
      "activations/layer9_attention_weight_min": -47.784786224365234,
      "epoch": 14.2,
      "learning_rate": 5.8997348484848486e-05,
      "loss": 2.7672,
      "step": 244450
    },
    {
      "activations/layer0_attention_weight_max": 17.009794235229492,
      "activations/layer0_attention_weight_min": -15.02754020690918,
      "activations/layer10_attention_weight_max": 35.04737091064453,
      "activations/layer10_attention_weight_min": -35.88237380981445,
      "activations/layer11_attention_weight_max": 32.14118957519531,
      "activations/layer11_attention_weight_min": -33.588111877441406,
      "activations/layer12_attention_weight_max": 18.265228271484375,
      "activations/layer12_attention_weight_min": -25.60080337524414,
      "activations/layer13_attention_weight_max": 33.97352600097656,
      "activations/layer13_attention_weight_min": -26.74088478088379,
      "activations/layer14_attention_weight_max": 37.27109909057617,
      "activations/layer14_attention_weight_min": -30.222497940063477,
      "activations/layer15_attention_weight_max": 32.017887115478516,
      "activations/layer15_attention_weight_min": -29.18077850341797,
      "activations/layer16_attention_weight_max": 32.58627700805664,
      "activations/layer16_attention_weight_min": -29.717634201049805,
      "activations/layer17_attention_weight_max": 50.291229248046875,
      "activations/layer17_attention_weight_min": -43.305355072021484,
      "activations/layer18_attention_weight_max": 46.13901901245117,
      "activations/layer18_attention_weight_min": -37.17812728881836,
      "activations/layer19_attention_weight_max": 21.89928436279297,
      "activations/layer19_attention_weight_min": -23.203258514404297,
      "activations/layer1_attention_weight_max": 16.370473861694336,
      "activations/layer1_attention_weight_min": -15.200164794921875,
      "activations/layer20_attention_weight_max": 22.970355987548828,
      "activations/layer20_attention_weight_min": -24.5672607421875,
      "activations/layer21_attention_weight_max": 37.110958099365234,
      "activations/layer21_attention_weight_min": -25.9230899810791,
      "activations/layer22_attention_weight_max": 28.70241355895996,
      "activations/layer22_attention_weight_min": -26.867279052734375,
      "activations/layer23_attention_weight_max": 37.250858306884766,
      "activations/layer23_attention_weight_min": -23.921422958374023,
      "activations/layer2_attention_weight_max": 33.43211364746094,
      "activations/layer2_attention_weight_min": -31.19294548034668,
      "activations/layer3_attention_weight_max": 95.75359344482422,
      "activations/layer3_attention_weight_min": -96.85991668701172,
      "activations/layer4_attention_weight_max": 93.7260971069336,
      "activations/layer4_attention_weight_min": -93.0866928100586,
      "activations/layer5_attention_weight_max": 70.8083724975586,
      "activations/layer5_attention_weight_min": -76.40161895751953,
      "activations/layer6_attention_weight_max": 51.97499084472656,
      "activations/layer6_attention_weight_min": -52.52860641479492,
      "activations/layer7_attention_weight_max": 68.6573715209961,
      "activations/layer7_attention_weight_min": -64.14241027832031,
      "activations/layer8_attention_weight_max": 50.41531753540039,
      "activations/layer8_attention_weight_min": -52.423343658447266,
      "activations/layer9_attention_weight_max": 46.1532096862793,
      "activations/layer9_attention_weight_min": -45.838932037353516,
      "epoch": 14.21,
      "learning_rate": 5.897840909090909e-05,
      "loss": 2.7659,
      "step": 244500
    },
    {
      "activations/layer0_attention_weight_max": 16.979310989379883,
      "activations/layer0_attention_weight_min": -14.708903312683105,
      "activations/layer10_attention_weight_max": 36.58430480957031,
      "activations/layer10_attention_weight_min": -36.80512237548828,
      "activations/layer11_attention_weight_max": 34.83614730834961,
      "activations/layer11_attention_weight_min": -36.997352600097656,
      "activations/layer12_attention_weight_max": 20.343353271484375,
      "activations/layer12_attention_weight_min": -30.240480422973633,
      "activations/layer13_attention_weight_max": 41.35783004760742,
      "activations/layer13_attention_weight_min": -30.946868896484375,
      "activations/layer14_attention_weight_max": 44.34454345703125,
      "activations/layer14_attention_weight_min": -30.37491226196289,
      "activations/layer15_attention_weight_max": 40.80387878417969,
      "activations/layer15_attention_weight_min": -32.01433563232422,
      "activations/layer16_attention_weight_max": 37.53323745727539,
      "activations/layer16_attention_weight_min": -32.484153747558594,
      "activations/layer17_attention_weight_max": 59.70370101928711,
      "activations/layer17_attention_weight_min": -45.56309509277344,
      "activations/layer18_attention_weight_max": 54.22861862182617,
      "activations/layer18_attention_weight_min": -40.88833236694336,
      "activations/layer19_attention_weight_max": 24.400981903076172,
      "activations/layer19_attention_weight_min": -24.560012817382812,
      "activations/layer1_attention_weight_max": 17.578529357910156,
      "activations/layer1_attention_weight_min": -19.866769790649414,
      "activations/layer20_attention_weight_max": 25.71415901184082,
      "activations/layer20_attention_weight_min": -22.269290924072266,
      "activations/layer21_attention_weight_max": 46.09238052368164,
      "activations/layer21_attention_weight_min": -25.360803604125977,
      "activations/layer22_attention_weight_max": 35.830867767333984,
      "activations/layer22_attention_weight_min": -25.2115421295166,
      "activations/layer23_attention_weight_max": 45.966949462890625,
      "activations/layer23_attention_weight_min": -22.909513473510742,
      "activations/layer2_attention_weight_max": 34.429222106933594,
      "activations/layer2_attention_weight_min": -33.0987663269043,
      "activations/layer3_attention_weight_max": 98.41458129882812,
      "activations/layer3_attention_weight_min": -101.5300521850586,
      "activations/layer4_attention_weight_max": 95.97611236572266,
      "activations/layer4_attention_weight_min": -94.05872344970703,
      "activations/layer5_attention_weight_max": 72.84005737304688,
      "activations/layer5_attention_weight_min": -84.32137298583984,
      "activations/layer6_attention_weight_max": 53.688289642333984,
      "activations/layer6_attention_weight_min": -53.89915466308594,
      "activations/layer7_attention_weight_max": 66.41507720947266,
      "activations/layer7_attention_weight_min": -64.73324584960938,
      "activations/layer8_attention_weight_max": 47.99937057495117,
      "activations/layer8_attention_weight_min": -49.16252517700195,
      "activations/layer9_attention_weight_max": 48.39592361450195,
      "activations/layer9_attention_weight_min": -51.41111373901367,
      "epoch": 14.21,
      "learning_rate": 5.895946969696969e-05,
      "loss": 2.7504,
      "step": 244550
    },
    {
      "activations/layer0_attention_weight_max": 17.084917068481445,
      "activations/layer0_attention_weight_min": -14.537453651428223,
      "activations/layer10_attention_weight_max": 35.463661193847656,
      "activations/layer10_attention_weight_min": -35.33926773071289,
      "activations/layer11_attention_weight_max": 30.94740104675293,
      "activations/layer11_attention_weight_min": -31.811603546142578,
      "activations/layer12_attention_weight_max": 18.15467071533203,
      "activations/layer12_attention_weight_min": -24.717857360839844,
      "activations/layer13_attention_weight_max": 33.73133850097656,
      "activations/layer13_attention_weight_min": -28.77664566040039,
      "activations/layer14_attention_weight_max": 33.87013244628906,
      "activations/layer14_attention_weight_min": -29.877717971801758,
      "activations/layer15_attention_weight_max": 31.348093032836914,
      "activations/layer15_attention_weight_min": -29.738759994506836,
      "activations/layer16_attention_weight_max": 30.147293090820312,
      "activations/layer16_attention_weight_min": -28.839696884155273,
      "activations/layer17_attention_weight_max": 48.9804573059082,
      "activations/layer17_attention_weight_min": -45.25210189819336,
      "activations/layer18_attention_weight_max": 43.501708984375,
      "activations/layer18_attention_weight_min": -37.61900329589844,
      "activations/layer19_attention_weight_max": 21.105222702026367,
      "activations/layer19_attention_weight_min": -20.672117233276367,
      "activations/layer1_attention_weight_max": 17.214391708374023,
      "activations/layer1_attention_weight_min": -17.44913101196289,
      "activations/layer20_attention_weight_max": 20.010700225830078,
      "activations/layer20_attention_weight_min": -20.368894577026367,
      "activations/layer21_attention_weight_max": 38.72554397583008,
      "activations/layer21_attention_weight_min": -24.121746063232422,
      "activations/layer22_attention_weight_max": 28.459518432617188,
      "activations/layer22_attention_weight_min": -27.02828025817871,
      "activations/layer23_attention_weight_max": 34.079315185546875,
      "activations/layer23_attention_weight_min": -24.671775817871094,
      "activations/layer2_attention_weight_max": 33.93745803833008,
      "activations/layer2_attention_weight_min": -33.54886245727539,
      "activations/layer3_attention_weight_max": 97.0325698852539,
      "activations/layer3_attention_weight_min": -101.49717712402344,
      "activations/layer4_attention_weight_max": 93.60169982910156,
      "activations/layer4_attention_weight_min": -90.93474578857422,
      "activations/layer5_attention_weight_max": 68.13631439208984,
      "activations/layer5_attention_weight_min": -82.56694793701172,
      "activations/layer6_attention_weight_max": 49.2617301940918,
      "activations/layer6_attention_weight_min": -49.50096893310547,
      "activations/layer7_attention_weight_max": 66.9368667602539,
      "activations/layer7_attention_weight_min": -63.93513488769531,
      "activations/layer8_attention_weight_max": 43.57440185546875,
      "activations/layer8_attention_weight_min": -48.05988311767578,
      "activations/layer9_attention_weight_max": 47.385135650634766,
      "activations/layer9_attention_weight_min": -44.68537139892578,
      "epoch": 14.21,
      "learning_rate": 5.8940909090909086e-05,
      "loss": 2.7608,
      "step": 244600
    },
    {
      "activations/layer0_attention_weight_max": 16.174909591674805,
      "activations/layer0_attention_weight_min": -14.483229637145996,
      "activations/layer10_attention_weight_max": 35.59680938720703,
      "activations/layer10_attention_weight_min": -34.49365234375,
      "activations/layer11_attention_weight_max": 34.028839111328125,
      "activations/layer11_attention_weight_min": -32.2588005065918,
      "activations/layer12_attention_weight_max": 18.714622497558594,
      "activations/layer12_attention_weight_min": -24.603347778320312,
      "activations/layer13_attention_weight_max": 37.693519592285156,
      "activations/layer13_attention_weight_min": -29.2193660736084,
      "activations/layer14_attention_weight_max": 35.51678466796875,
      "activations/layer14_attention_weight_min": -32.19099426269531,
      "activations/layer15_attention_weight_max": 34.06135559082031,
      "activations/layer15_attention_weight_min": -32.30915832519531,
      "activations/layer16_attention_weight_max": 33.932037353515625,
      "activations/layer16_attention_weight_min": -33.2308235168457,
      "activations/layer17_attention_weight_max": 58.87300491333008,
      "activations/layer17_attention_weight_min": -49.399723052978516,
      "activations/layer18_attention_weight_max": 49.79100799560547,
      "activations/layer18_attention_weight_min": -41.66046905517578,
      "activations/layer19_attention_weight_max": 22.248165130615234,
      "activations/layer19_attention_weight_min": -22.260799407958984,
      "activations/layer1_attention_weight_max": 17.068269729614258,
      "activations/layer1_attention_weight_min": -14.125738143920898,
      "activations/layer20_attention_weight_max": 21.526132583618164,
      "activations/layer20_attention_weight_min": -21.788297653198242,
      "activations/layer21_attention_weight_max": 38.503604888916016,
      "activations/layer21_attention_weight_min": -24.577499389648438,
      "activations/layer22_attention_weight_max": 32.31930923461914,
      "activations/layer22_attention_weight_min": -25.2128849029541,
      "activations/layer23_attention_weight_max": 36.60835266113281,
      "activations/layer23_attention_weight_min": -22.709510803222656,
      "activations/layer2_attention_weight_max": 30.2781982421875,
      "activations/layer2_attention_weight_min": -31.14870262145996,
      "activations/layer3_attention_weight_max": 93.23486328125,
      "activations/layer3_attention_weight_min": -89.4169921875,
      "activations/layer4_attention_weight_max": 96.93396759033203,
      "activations/layer4_attention_weight_min": -91.57786560058594,
      "activations/layer5_attention_weight_max": 66.71623229980469,
      "activations/layer5_attention_weight_min": -81.300048828125,
      "activations/layer6_attention_weight_max": 53.626399993896484,
      "activations/layer6_attention_weight_min": -52.57148742675781,
      "activations/layer7_attention_weight_max": 65.12285614013672,
      "activations/layer7_attention_weight_min": -64.68195343017578,
      "activations/layer8_attention_weight_max": 45.00788879394531,
      "activations/layer8_attention_weight_min": -45.24049377441406,
      "activations/layer9_attention_weight_max": 43.59129333496094,
      "activations/layer9_attention_weight_min": -45.13467788696289,
      "epoch": 14.22,
      "learning_rate": 5.8921969696969695e-05,
      "loss": 2.7672,
      "step": 244650
    },
    {
      "activations/layer0_attention_weight_max": 15.862936019897461,
      "activations/layer0_attention_weight_min": -14.74958324432373,
      "activations/layer10_attention_weight_max": 35.91869354248047,
      "activations/layer10_attention_weight_min": -34.8408088684082,
      "activations/layer11_attention_weight_max": 32.149627685546875,
      "activations/layer11_attention_weight_min": -33.01663589477539,
      "activations/layer12_attention_weight_max": 17.869176864624023,
      "activations/layer12_attention_weight_min": -27.51818084716797,
      "activations/layer13_attention_weight_max": 31.218706130981445,
      "activations/layer13_attention_weight_min": -26.049715042114258,
      "activations/layer14_attention_weight_max": 34.77299499511719,
      "activations/layer14_attention_weight_min": -29.822114944458008,
      "activations/layer15_attention_weight_max": 31.684717178344727,
      "activations/layer15_attention_weight_min": -28.82640266418457,
      "activations/layer16_attention_weight_max": 31.33324432373047,
      "activations/layer16_attention_weight_min": -28.579299926757812,
      "activations/layer17_attention_weight_max": 49.78445816040039,
      "activations/layer17_attention_weight_min": -41.775840759277344,
      "activations/layer18_attention_weight_max": 45.02611541748047,
      "activations/layer18_attention_weight_min": -36.66038131713867,
      "activations/layer19_attention_weight_max": 21.59003448486328,
      "activations/layer19_attention_weight_min": -21.73316764831543,
      "activations/layer1_attention_weight_max": 17.958391189575195,
      "activations/layer1_attention_weight_min": -15.057259559631348,
      "activations/layer20_attention_weight_max": 20.86046028137207,
      "activations/layer20_attention_weight_min": -22.752683639526367,
      "activations/layer21_attention_weight_max": 33.752830505371094,
      "activations/layer21_attention_weight_min": -19.64640998840332,
      "activations/layer22_attention_weight_max": 27.028730392456055,
      "activations/layer22_attention_weight_min": -24.240022659301758,
      "activations/layer23_attention_weight_max": 35.108665466308594,
      "activations/layer23_attention_weight_min": -23.40603256225586,
      "activations/layer2_attention_weight_max": 32.266334533691406,
      "activations/layer2_attention_weight_min": -31.22521209716797,
      "activations/layer3_attention_weight_max": 93.39444732666016,
      "activations/layer3_attention_weight_min": -99.24486541748047,
      "activations/layer4_attention_weight_max": 93.83977508544922,
      "activations/layer4_attention_weight_min": -96.31175231933594,
      "activations/layer5_attention_weight_max": 73.46035766601562,
      "activations/layer5_attention_weight_min": -79.89210510253906,
      "activations/layer6_attention_weight_max": 51.33821105957031,
      "activations/layer6_attention_weight_min": -54.06378936767578,
      "activations/layer7_attention_weight_max": 62.83899688720703,
      "activations/layer7_attention_weight_min": -67.6007080078125,
      "activations/layer8_attention_weight_max": 44.013912200927734,
      "activations/layer8_attention_weight_min": -46.98842239379883,
      "activations/layer9_attention_weight_max": 41.98319625854492,
      "activations/layer9_attention_weight_min": -47.373924255371094,
      "epoch": 14.22,
      "learning_rate": 5.8903030303030296e-05,
      "loss": 2.7657,
      "step": 244700
    },
    {
      "activations/layer0_attention_weight_max": 17.257848739624023,
      "activations/layer0_attention_weight_min": -14.533699989318848,
      "activations/layer10_attention_weight_max": 37.47304916381836,
      "activations/layer10_attention_weight_min": -37.938026428222656,
      "activations/layer11_attention_weight_max": 34.99199676513672,
      "activations/layer11_attention_weight_min": -34.94325637817383,
      "activations/layer12_attention_weight_max": 18.688901901245117,
      "activations/layer12_attention_weight_min": -23.1407470703125,
      "activations/layer13_attention_weight_max": 36.932369232177734,
      "activations/layer13_attention_weight_min": -27.90699005126953,
      "activations/layer14_attention_weight_max": 35.68503952026367,
      "activations/layer14_attention_weight_min": -31.465206146240234,
      "activations/layer15_attention_weight_max": 32.978214263916016,
      "activations/layer15_attention_weight_min": -32.369747161865234,
      "activations/layer16_attention_weight_max": 33.02349853515625,
      "activations/layer16_attention_weight_min": -30.479801177978516,
      "activations/layer17_attention_weight_max": 51.44963455200195,
      "activations/layer17_attention_weight_min": -44.429969787597656,
      "activations/layer18_attention_weight_max": 46.60462188720703,
      "activations/layer18_attention_weight_min": -36.24678039550781,
      "activations/layer19_attention_weight_max": 21.914608001708984,
      "activations/layer19_attention_weight_min": -22.0675106048584,
      "activations/layer1_attention_weight_max": 16.55525016784668,
      "activations/layer1_attention_weight_min": -15.391560554504395,
      "activations/layer20_attention_weight_max": 21.517141342163086,
      "activations/layer20_attention_weight_min": -19.913288116455078,
      "activations/layer21_attention_weight_max": 33.407039642333984,
      "activations/layer21_attention_weight_min": -22.4399471282959,
      "activations/layer22_attention_weight_max": 30.89015007019043,
      "activations/layer22_attention_weight_min": -26.29156494140625,
      "activations/layer23_attention_weight_max": 39.418174743652344,
      "activations/layer23_attention_weight_min": -25.811439514160156,
      "activations/layer2_attention_weight_max": 33.923851013183594,
      "activations/layer2_attention_weight_min": -30.83301544189453,
      "activations/layer3_attention_weight_max": 98.04092407226562,
      "activations/layer3_attention_weight_min": -99.84851837158203,
      "activations/layer4_attention_weight_max": 97.54109954833984,
      "activations/layer4_attention_weight_min": -92.51940155029297,
      "activations/layer5_attention_weight_max": 70.6298599243164,
      "activations/layer5_attention_weight_min": -79.10931396484375,
      "activations/layer6_attention_weight_max": 52.95714569091797,
      "activations/layer6_attention_weight_min": -52.817203521728516,
      "activations/layer7_attention_weight_max": 65.94422912597656,
      "activations/layer7_attention_weight_min": -65.9074935913086,
      "activations/layer8_attention_weight_max": 47.28800964355469,
      "activations/layer8_attention_weight_min": -49.57780075073242,
      "activations/layer9_attention_weight_max": 45.5195426940918,
      "activations/layer9_attention_weight_min": -47.1035041809082,
      "epoch": 14.22,
      "learning_rate": 5.8884090909090904e-05,
      "loss": 2.749,
      "step": 244750
    },
    {
      "activations/layer0_attention_weight_max": 16.911134719848633,
      "activations/layer0_attention_weight_min": -14.737833976745605,
      "activations/layer10_attention_weight_max": 34.01880645751953,
      "activations/layer10_attention_weight_min": -34.41525650024414,
      "activations/layer11_attention_weight_max": 31.683462142944336,
      "activations/layer11_attention_weight_min": -32.22784423828125,
      "activations/layer12_attention_weight_max": 18.02815818786621,
      "activations/layer12_attention_weight_min": -30.565113067626953,
      "activations/layer13_attention_weight_max": 38.10679626464844,
      "activations/layer13_attention_weight_min": -29.334571838378906,
      "activations/layer14_attention_weight_max": 37.261192321777344,
      "activations/layer14_attention_weight_min": -29.207653045654297,
      "activations/layer15_attention_weight_max": 31.647926330566406,
      "activations/layer15_attention_weight_min": -28.057201385498047,
      "activations/layer16_attention_weight_max": 32.59004211425781,
      "activations/layer16_attention_weight_min": -31.046972274780273,
      "activations/layer17_attention_weight_max": 52.13230514526367,
      "activations/layer17_attention_weight_min": -47.27742385864258,
      "activations/layer18_attention_weight_max": 49.10378646850586,
      "activations/layer18_attention_weight_min": -41.19318389892578,
      "activations/layer19_attention_weight_max": 21.958160400390625,
      "activations/layer19_attention_weight_min": -22.196327209472656,
      "activations/layer1_attention_weight_max": 16.427785873413086,
      "activations/layer1_attention_weight_min": -14.021397590637207,
      "activations/layer20_attention_weight_max": 22.516225814819336,
      "activations/layer20_attention_weight_min": -22.048171997070312,
      "activations/layer21_attention_weight_max": 33.885658264160156,
      "activations/layer21_attention_weight_min": -22.039356231689453,
      "activations/layer22_attention_weight_max": 29.197046279907227,
      "activations/layer22_attention_weight_min": -25.58649253845215,
      "activations/layer23_attention_weight_max": 39.76667022705078,
      "activations/layer23_attention_weight_min": -22.905235290527344,
      "activations/layer2_attention_weight_max": 34.86749267578125,
      "activations/layer2_attention_weight_min": -34.376285552978516,
      "activations/layer3_attention_weight_max": 99.35143280029297,
      "activations/layer3_attention_weight_min": -102.99315643310547,
      "activations/layer4_attention_weight_max": 96.3196029663086,
      "activations/layer4_attention_weight_min": -93.93255615234375,
      "activations/layer5_attention_weight_max": 71.82491302490234,
      "activations/layer5_attention_weight_min": -78.95640563964844,
      "activations/layer6_attention_weight_max": 51.62373733520508,
      "activations/layer6_attention_weight_min": -55.6700553894043,
      "activations/layer7_attention_weight_max": 66.09864807128906,
      "activations/layer7_attention_weight_min": -64.7027359008789,
      "activations/layer8_attention_weight_max": 45.60114669799805,
      "activations/layer8_attention_weight_min": -48.530738830566406,
      "activations/layer9_attention_weight_max": 44.74917984008789,
      "activations/layer9_attention_weight_min": -46.044960021972656,
      "epoch": 14.22,
      "learning_rate": 5.8865151515151506e-05,
      "loss": 2.7515,
      "step": 244800
    },
    {
      "activations/layer0_attention_weight_max": 16.047719955444336,
      "activations/layer0_attention_weight_min": -14.687797546386719,
      "activations/layer10_attention_weight_max": 33.4425048828125,
      "activations/layer10_attention_weight_min": -36.45947265625,
      "activations/layer11_attention_weight_max": 30.623699188232422,
      "activations/layer11_attention_weight_min": -31.860858917236328,
      "activations/layer12_attention_weight_max": 18.20047378540039,
      "activations/layer12_attention_weight_min": -24.933319091796875,
      "activations/layer13_attention_weight_max": 35.06522750854492,
      "activations/layer13_attention_weight_min": -29.561141967773438,
      "activations/layer14_attention_weight_max": 33.9096565246582,
      "activations/layer14_attention_weight_min": -28.79610824584961,
      "activations/layer15_attention_weight_max": 31.811908721923828,
      "activations/layer15_attention_weight_min": -28.585247039794922,
      "activations/layer16_attention_weight_max": 30.18424415588379,
      "activations/layer16_attention_weight_min": -28.411754608154297,
      "activations/layer17_attention_weight_max": 49.901981353759766,
      "activations/layer17_attention_weight_min": -41.99390411376953,
      "activations/layer18_attention_weight_max": 45.30298614501953,
      "activations/layer18_attention_weight_min": -36.39881134033203,
      "activations/layer19_attention_weight_max": 23.06764793395996,
      "activations/layer19_attention_weight_min": -21.860137939453125,
      "activations/layer1_attention_weight_max": 16.710693359375,
      "activations/layer1_attention_weight_min": -15.585927963256836,
      "activations/layer20_attention_weight_max": 20.404075622558594,
      "activations/layer20_attention_weight_min": -20.35703468322754,
      "activations/layer21_attention_weight_max": 36.63427734375,
      "activations/layer21_attention_weight_min": -23.816537857055664,
      "activations/layer22_attention_weight_max": 29.28382110595703,
      "activations/layer22_attention_weight_min": -24.183147430419922,
      "activations/layer23_attention_weight_max": 32.590431213378906,
      "activations/layer23_attention_weight_min": -25.973594665527344,
      "activations/layer2_attention_weight_max": 33.33530807495117,
      "activations/layer2_attention_weight_min": -31.915908813476562,
      "activations/layer3_attention_weight_max": 97.48507690429688,
      "activations/layer3_attention_weight_min": -97.62803649902344,
      "activations/layer4_attention_weight_max": 93.78528594970703,
      "activations/layer4_attention_weight_min": -90.7967300415039,
      "activations/layer5_attention_weight_max": 69.9326400756836,
      "activations/layer5_attention_weight_min": -76.07005310058594,
      "activations/layer6_attention_weight_max": 52.79072189331055,
      "activations/layer6_attention_weight_min": -51.41194152832031,
      "activations/layer7_attention_weight_max": 65.18753051757812,
      "activations/layer7_attention_weight_min": -63.01350784301758,
      "activations/layer8_attention_weight_max": 43.31326675415039,
      "activations/layer8_attention_weight_min": -46.81906509399414,
      "activations/layer9_attention_weight_max": 42.77683639526367,
      "activations/layer9_attention_weight_min": -44.56232833862305,
      "epoch": 14.23,
      "learning_rate": 5.884621212121212e-05,
      "loss": 2.7412,
      "step": 244850
    },
    {
      "activations/layer0_attention_weight_max": 17.235166549682617,
      "activations/layer0_attention_weight_min": -14.505998611450195,
      "activations/layer10_attention_weight_max": 34.93513870239258,
      "activations/layer10_attention_weight_min": -34.58089828491211,
      "activations/layer11_attention_weight_max": 31.93954849243164,
      "activations/layer11_attention_weight_min": -32.5511474609375,
      "activations/layer12_attention_weight_max": 17.964296340942383,
      "activations/layer12_attention_weight_min": -27.042261123657227,
      "activations/layer13_attention_weight_max": 35.93083572387695,
      "activations/layer13_attention_weight_min": -28.06627082824707,
      "activations/layer14_attention_weight_max": 36.66011428833008,
      "activations/layer14_attention_weight_min": -30.37750244140625,
      "activations/layer15_attention_weight_max": 31.749237060546875,
      "activations/layer15_attention_weight_min": -29.196430206298828,
      "activations/layer16_attention_weight_max": 30.721113204956055,
      "activations/layer16_attention_weight_min": -29.802757263183594,
      "activations/layer17_attention_weight_max": 52.51204299926758,
      "activations/layer17_attention_weight_min": -44.200843811035156,
      "activations/layer18_attention_weight_max": 46.0337028503418,
      "activations/layer18_attention_weight_min": -37.19268035888672,
      "activations/layer19_attention_weight_max": 24.09693145751953,
      "activations/layer19_attention_weight_min": -23.252111434936523,
      "activations/layer1_attention_weight_max": 16.218334197998047,
      "activations/layer1_attention_weight_min": -15.699997901916504,
      "activations/layer20_attention_weight_max": 22.955915451049805,
      "activations/layer20_attention_weight_min": -22.53537368774414,
      "activations/layer21_attention_weight_max": 40.13861083984375,
      "activations/layer21_attention_weight_min": -23.934770584106445,
      "activations/layer22_attention_weight_max": 29.333126068115234,
      "activations/layer22_attention_weight_min": -26.508317947387695,
      "activations/layer23_attention_weight_max": 35.87165832519531,
      "activations/layer23_attention_weight_min": -26.820873260498047,
      "activations/layer2_attention_weight_max": 32.98843002319336,
      "activations/layer2_attention_weight_min": -30.829055786132812,
      "activations/layer3_attention_weight_max": 97.30062103271484,
      "activations/layer3_attention_weight_min": -96.49267578125,
      "activations/layer4_attention_weight_max": 94.5052719116211,
      "activations/layer4_attention_weight_min": -92.01856994628906,
      "activations/layer5_attention_weight_max": 71.22296142578125,
      "activations/layer5_attention_weight_min": -81.42552947998047,
      "activations/layer6_attention_weight_max": 51.29692459106445,
      "activations/layer6_attention_weight_min": -51.6734733581543,
      "activations/layer7_attention_weight_max": 61.64432144165039,
      "activations/layer7_attention_weight_min": -63.49479293823242,
      "activations/layer8_attention_weight_max": 43.33554458618164,
      "activations/layer8_attention_weight_min": -47.06066131591797,
      "activations/layer9_attention_weight_max": 45.43761444091797,
      "activations/layer9_attention_weight_min": -44.81644058227539,
      "epoch": 14.23,
      "learning_rate": 5.882727272727272e-05,
      "loss": 2.7746,
      "step": 244900
    },
    {
      "activations/layer0_attention_weight_max": 17.985214233398438,
      "activations/layer0_attention_weight_min": -14.313186645507812,
      "activations/layer10_attention_weight_max": 34.823402404785156,
      "activations/layer10_attention_weight_min": -34.37921905517578,
      "activations/layer11_attention_weight_max": 32.08024978637695,
      "activations/layer11_attention_weight_min": -32.39219665527344,
      "activations/layer12_attention_weight_max": 18.688861846923828,
      "activations/layer12_attention_weight_min": -27.156612396240234,
      "activations/layer13_attention_weight_max": 34.62622833251953,
      "activations/layer13_attention_weight_min": -27.39472770690918,
      "activations/layer14_attention_weight_max": 36.71144485473633,
      "activations/layer14_attention_weight_min": -31.106517791748047,
      "activations/layer15_attention_weight_max": 31.840612411499023,
      "activations/layer15_attention_weight_min": -29.298460006713867,
      "activations/layer16_attention_weight_max": 32.48233413696289,
      "activations/layer16_attention_weight_min": -32.76500701904297,
      "activations/layer17_attention_weight_max": 51.10564422607422,
      "activations/layer17_attention_weight_min": -44.57239532470703,
      "activations/layer18_attention_weight_max": 49.49653244018555,
      "activations/layer18_attention_weight_min": -41.401939392089844,
      "activations/layer19_attention_weight_max": 23.294015884399414,
      "activations/layer19_attention_weight_min": -22.638750076293945,
      "activations/layer1_attention_weight_max": 16.064424514770508,
      "activations/layer1_attention_weight_min": -13.694673538208008,
      "activations/layer20_attention_weight_max": 23.037633895874023,
      "activations/layer20_attention_weight_min": -23.242795944213867,
      "activations/layer21_attention_weight_max": 36.67158889770508,
      "activations/layer21_attention_weight_min": -22.84192657470703,
      "activations/layer22_attention_weight_max": 32.85841369628906,
      "activations/layer22_attention_weight_min": -26.350204467773438,
      "activations/layer23_attention_weight_max": 41.12776184082031,
      "activations/layer23_attention_weight_min": -25.097393035888672,
      "activations/layer2_attention_weight_max": 32.727195739746094,
      "activations/layer2_attention_weight_min": -31.395292282104492,
      "activations/layer3_attention_weight_max": 94.0912094116211,
      "activations/layer3_attention_weight_min": -94.1084213256836,
      "activations/layer4_attention_weight_max": 94.12063598632812,
      "activations/layer4_attention_weight_min": -89.55828857421875,
      "activations/layer5_attention_weight_max": 69.15739440917969,
      "activations/layer5_attention_weight_min": -78.79888916015625,
      "activations/layer6_attention_weight_max": 48.26716995239258,
      "activations/layer6_attention_weight_min": -48.28478240966797,
      "activations/layer7_attention_weight_max": 64.04351806640625,
      "activations/layer7_attention_weight_min": -60.87482452392578,
      "activations/layer8_attention_weight_max": 41.99818420410156,
      "activations/layer8_attention_weight_min": -47.68583297729492,
      "activations/layer9_attention_weight_max": 41.6788330078125,
      "activations/layer9_attention_weight_min": -44.55217742919922,
      "epoch": 14.23,
      "learning_rate": 5.8808333333333324e-05,
      "loss": 2.7504,
      "step": 244950
    },
    {
      "activations/layer0_attention_weight_max": 16.853363037109375,
      "activations/layer0_attention_weight_min": -13.951160430908203,
      "activations/layer10_attention_weight_max": 41.01144027709961,
      "activations/layer10_attention_weight_min": -39.20986557006836,
      "activations/layer11_attention_weight_max": 36.76912307739258,
      "activations/layer11_attention_weight_min": -34.978450775146484,
      "activations/layer12_attention_weight_max": 23.516454696655273,
      "activations/layer12_attention_weight_min": -22.607542037963867,
      "activations/layer13_attention_weight_max": 45.44966506958008,
      "activations/layer13_attention_weight_min": -30.711875915527344,
      "activations/layer14_attention_weight_max": 49.536834716796875,
      "activations/layer14_attention_weight_min": -28.406898498535156,
      "activations/layer15_attention_weight_max": 46.87187576293945,
      "activations/layer15_attention_weight_min": -29.15193748474121,
      "activations/layer16_attention_weight_max": 33.7918586730957,
      "activations/layer16_attention_weight_min": -31.264432907104492,
      "activations/layer17_attention_weight_max": 56.42059326171875,
      "activations/layer17_attention_weight_min": -44.73490524291992,
      "activations/layer18_attention_weight_max": 53.20053482055664,
      "activations/layer18_attention_weight_min": -37.5577507019043,
      "activations/layer19_attention_weight_max": 24.925874710083008,
      "activations/layer19_attention_weight_min": -23.278717041015625,
      "activations/layer1_attention_weight_max": 15.68659782409668,
      "activations/layer1_attention_weight_min": -14.951742172241211,
      "activations/layer20_attention_weight_max": 26.05187225341797,
      "activations/layer20_attention_weight_min": -22.185222625732422,
      "activations/layer21_attention_weight_max": 46.149566650390625,
      "activations/layer21_attention_weight_min": -25.088115692138672,
      "activations/layer22_attention_weight_max": 33.72301483154297,
      "activations/layer22_attention_weight_min": -25.79041862487793,
      "activations/layer23_attention_weight_max": 46.46511459350586,
      "activations/layer23_attention_weight_min": -24.351261138916016,
      "activations/layer2_attention_weight_max": 34.09898376464844,
      "activations/layer2_attention_weight_min": -30.80886459350586,
      "activations/layer3_attention_weight_max": 95.46187591552734,
      "activations/layer3_attention_weight_min": -99.83360290527344,
      "activations/layer4_attention_weight_max": 93.14464569091797,
      "activations/layer4_attention_weight_min": -95.36992645263672,
      "activations/layer5_attention_weight_max": 71.94758605957031,
      "activations/layer5_attention_weight_min": -82.01951599121094,
      "activations/layer6_attention_weight_max": 51.56528854370117,
      "activations/layer6_attention_weight_min": -53.75562286376953,
      "activations/layer7_attention_weight_max": 79.976806640625,
      "activations/layer7_attention_weight_min": -67.72935485839844,
      "activations/layer8_attention_weight_max": 50.861732482910156,
      "activations/layer8_attention_weight_min": -52.406211853027344,
      "activations/layer9_attention_weight_max": 54.97042465209961,
      "activations/layer9_attention_weight_min": -51.6562614440918,
      "epoch": 14.24,
      "learning_rate": 5.878939393939393e-05,
      "loss": 2.7714,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4341,
      "eval_samples_per_second": 509.126,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4341,
      "eval_openwebtext_samples_per_second": 509.126,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9222,
      "eval_wikitext_samples_per_second": 237.229,
      "step": 245000
    },
    {
      "epoch": 14.24,
      "eval_lambada_loss": 2.49609375,
      "eval_lambada_ppl": 12.134998917818166,
      "eval_lambada_runtime": 9.4828,
      "eval_lambada_samples_per_second": 513.457,
      "step": 245000
    },
    {
      "activations/layer0_attention_weight_max": 18.0489501953125,
      "activations/layer0_attention_weight_min": -14.07697868347168,
      "activations/layer10_attention_weight_max": 36.596309661865234,
      "activations/layer10_attention_weight_min": -34.569862365722656,
      "activations/layer11_attention_weight_max": 34.5704460144043,
      "activations/layer11_attention_weight_min": -33.922332763671875,
      "activations/layer12_attention_weight_max": 20.31622886657715,
      "activations/layer12_attention_weight_min": -29.669445037841797,
      "activations/layer13_attention_weight_max": 34.03092575073242,
      "activations/layer13_attention_weight_min": -29.531190872192383,
      "activations/layer14_attention_weight_max": 34.687339782714844,
      "activations/layer14_attention_weight_min": -33.8648681640625,
      "activations/layer15_attention_weight_max": 33.86171340942383,
      "activations/layer15_attention_weight_min": -32.74891662597656,
      "activations/layer16_attention_weight_max": 33.523223876953125,
      "activations/layer16_attention_weight_min": -34.06838607788086,
      "activations/layer17_attention_weight_max": 51.72256851196289,
      "activations/layer17_attention_weight_min": -47.375728607177734,
      "activations/layer18_attention_weight_max": 47.31711196899414,
      "activations/layer18_attention_weight_min": -39.815982818603516,
      "activations/layer19_attention_weight_max": 26.611173629760742,
      "activations/layer19_attention_weight_min": -23.42207145690918,
      "activations/layer1_attention_weight_max": 17.383399963378906,
      "activations/layer1_attention_weight_min": -15.68384838104248,
      "activations/layer20_attention_weight_max": 21.522008895874023,
      "activations/layer20_attention_weight_min": -23.336395263671875,
      "activations/layer21_attention_weight_max": 31.40757942199707,
      "activations/layer21_attention_weight_min": -24.053537368774414,
      "activations/layer22_attention_weight_max": 35.85810852050781,
      "activations/layer22_attention_weight_min": -28.53265380859375,
      "activations/layer23_attention_weight_max": 40.087039947509766,
      "activations/layer23_attention_weight_min": -23.527219772338867,
      "activations/layer2_attention_weight_max": 34.110565185546875,
      "activations/layer2_attention_weight_min": -31.788127899169922,
      "activations/layer3_attention_weight_max": 94.0079574584961,
      "activations/layer3_attention_weight_min": -92.8017578125,
      "activations/layer4_attention_weight_max": 90.4156494140625,
      "activations/layer4_attention_weight_min": -89.13579559326172,
      "activations/layer5_attention_weight_max": 67.03652954101562,
      "activations/layer5_attention_weight_min": -76.03422546386719,
      "activations/layer6_attention_weight_max": 51.84307098388672,
      "activations/layer6_attention_weight_min": -49.43577194213867,
      "activations/layer7_attention_weight_max": 66.69383239746094,
      "activations/layer7_attention_weight_min": -62.353546142578125,
      "activations/layer8_attention_weight_max": 47.00297164916992,
      "activations/layer8_attention_weight_min": -46.659358978271484,
      "activations/layer9_attention_weight_max": 45.103450775146484,
      "activations/layer9_attention_weight_min": -47.24077606201172,
      "epoch": 14.24,
      "learning_rate": 5.877045454545454e-05,
      "loss": 2.7614,
      "step": 245050
    },
    {
      "activations/layer0_attention_weight_max": 17.475370407104492,
      "activations/layer0_attention_weight_min": -15.035816192626953,
      "activations/layer10_attention_weight_max": 35.80253219604492,
      "activations/layer10_attention_weight_min": -36.221866607666016,
      "activations/layer11_attention_weight_max": 32.97261047363281,
      "activations/layer11_attention_weight_min": -34.97845458984375,
      "activations/layer12_attention_weight_max": 18.494077682495117,
      "activations/layer12_attention_weight_min": -28.437868118286133,
      "activations/layer13_attention_weight_max": 41.923580169677734,
      "activations/layer13_attention_weight_min": -29.7847957611084,
      "activations/layer14_attention_weight_max": 35.69829559326172,
      "activations/layer14_attention_weight_min": -29.77823829650879,
      "activations/layer15_attention_weight_max": 32.88068389892578,
      "activations/layer15_attention_weight_min": -29.376096725463867,
      "activations/layer16_attention_weight_max": 31.751745223999023,
      "activations/layer16_attention_weight_min": -30.08580207824707,
      "activations/layer17_attention_weight_max": 47.00893020629883,
      "activations/layer17_attention_weight_min": -42.30937576293945,
      "activations/layer18_attention_weight_max": 46.49445724487305,
      "activations/layer18_attention_weight_min": -38.198204040527344,
      "activations/layer19_attention_weight_max": 24.563627243041992,
      "activations/layer19_attention_weight_min": -24.753862380981445,
      "activations/layer1_attention_weight_max": 16.956186294555664,
      "activations/layer1_attention_weight_min": -14.708178520202637,
      "activations/layer20_attention_weight_max": 22.975162506103516,
      "activations/layer20_attention_weight_min": -22.648067474365234,
      "activations/layer21_attention_weight_max": 35.54921340942383,
      "activations/layer21_attention_weight_min": -24.015356063842773,
      "activations/layer22_attention_weight_max": 29.299808502197266,
      "activations/layer22_attention_weight_min": -26.66549301147461,
      "activations/layer23_attention_weight_max": 34.52741622924805,
      "activations/layer23_attention_weight_min": -25.86738395690918,
      "activations/layer2_attention_weight_max": 34.325836181640625,
      "activations/layer2_attention_weight_min": -32.703094482421875,
      "activations/layer3_attention_weight_max": 101.51868438720703,
      "activations/layer3_attention_weight_min": -93.33885955810547,
      "activations/layer4_attention_weight_max": 95.28723907470703,
      "activations/layer4_attention_weight_min": -91.59705352783203,
      "activations/layer5_attention_weight_max": 71.15769958496094,
      "activations/layer5_attention_weight_min": -76.81856536865234,
      "activations/layer6_attention_weight_max": 53.01875686645508,
      "activations/layer6_attention_weight_min": -54.62783432006836,
      "activations/layer7_attention_weight_max": 68.06097412109375,
      "activations/layer7_attention_weight_min": -64.80657196044922,
      "activations/layer8_attention_weight_max": 49.12295150756836,
      "activations/layer8_attention_weight_min": -50.07061004638672,
      "activations/layer9_attention_weight_max": 45.67560577392578,
      "activations/layer9_attention_weight_min": -48.82135009765625,
      "epoch": 14.24,
      "learning_rate": 5.875151515151515e-05,
      "loss": 2.7739,
      "step": 245100
    },
    {
      "activations/layer0_attention_weight_max": 16.500520706176758,
      "activations/layer0_attention_weight_min": -14.563458442687988,
      "activations/layer10_attention_weight_max": 35.82730484008789,
      "activations/layer10_attention_weight_min": -35.624847412109375,
      "activations/layer11_attention_weight_max": 30.111652374267578,
      "activations/layer11_attention_weight_min": -32.68280029296875,
      "activations/layer12_attention_weight_max": 17.688688278198242,
      "activations/layer12_attention_weight_min": -24.11650276184082,
      "activations/layer13_attention_weight_max": 37.46918869018555,
      "activations/layer13_attention_weight_min": -30.779094696044922,
      "activations/layer14_attention_weight_max": 37.80302810668945,
      "activations/layer14_attention_weight_min": -32.135704040527344,
      "activations/layer15_attention_weight_max": 34.69768524169922,
      "activations/layer15_attention_weight_min": -29.90941619873047,
      "activations/layer16_attention_weight_max": 32.858280181884766,
      "activations/layer16_attention_weight_min": -31.106624603271484,
      "activations/layer17_attention_weight_max": 55.047607421875,
      "activations/layer17_attention_weight_min": -43.7206916809082,
      "activations/layer18_attention_weight_max": 49.95844650268555,
      "activations/layer18_attention_weight_min": -37.12852478027344,
      "activations/layer19_attention_weight_max": 21.899396896362305,
      "activations/layer19_attention_weight_min": -20.29680633544922,
      "activations/layer1_attention_weight_max": 17.54070472717285,
      "activations/layer1_attention_weight_min": -13.730856895446777,
      "activations/layer20_attention_weight_max": 22.209409713745117,
      "activations/layer20_attention_weight_min": -23.194231033325195,
      "activations/layer21_attention_weight_max": 36.332244873046875,
      "activations/layer21_attention_weight_min": -25.62876319885254,
      "activations/layer22_attention_weight_max": 29.50423812866211,
      "activations/layer22_attention_weight_min": -26.27696990966797,
      "activations/layer23_attention_weight_max": 38.574058532714844,
      "activations/layer23_attention_weight_min": -24.685691833496094,
      "activations/layer2_attention_weight_max": 31.4514217376709,
      "activations/layer2_attention_weight_min": -31.00836944580078,
      "activations/layer3_attention_weight_max": 88.44991302490234,
      "activations/layer3_attention_weight_min": -94.797119140625,
      "activations/layer4_attention_weight_max": 89.33385467529297,
      "activations/layer4_attention_weight_min": -90.47982025146484,
      "activations/layer5_attention_weight_max": 69.30606079101562,
      "activations/layer5_attention_weight_min": -80.88467407226562,
      "activations/layer6_attention_weight_max": 51.96961212158203,
      "activations/layer6_attention_weight_min": -52.70603561401367,
      "activations/layer7_attention_weight_max": 71.4426498413086,
      "activations/layer7_attention_weight_min": -65.0824203491211,
      "activations/layer8_attention_weight_max": 46.2871208190918,
      "activations/layer8_attention_weight_min": -47.380855560302734,
      "activations/layer9_attention_weight_max": 50.23966598510742,
      "activations/layer9_attention_weight_min": -47.511009216308594,
      "epoch": 14.24,
      "learning_rate": 5.873257575757575e-05,
      "loss": 2.7779,
      "step": 245150
    },
    {
      "activations/layer0_attention_weight_max": 16.225887298583984,
      "activations/layer0_attention_weight_min": -13.896770477294922,
      "activations/layer10_attention_weight_max": 34.70452117919922,
      "activations/layer10_attention_weight_min": -33.736366271972656,
      "activations/layer11_attention_weight_max": 31.223628997802734,
      "activations/layer11_attention_weight_min": -31.40378189086914,
      "activations/layer12_attention_weight_max": 17.492958068847656,
      "activations/layer12_attention_weight_min": -29.507837295532227,
      "activations/layer13_attention_weight_max": 31.64508056640625,
      "activations/layer13_attention_weight_min": -27.800628662109375,
      "activations/layer14_attention_weight_max": 36.28306198120117,
      "activations/layer14_attention_weight_min": -30.610050201416016,
      "activations/layer15_attention_weight_max": 32.060001373291016,
      "activations/layer15_attention_weight_min": -29.458784103393555,
      "activations/layer16_attention_weight_max": 34.25272750854492,
      "activations/layer16_attention_weight_min": -29.784530639648438,
      "activations/layer17_attention_weight_max": 52.0761604309082,
      "activations/layer17_attention_weight_min": -42.68486404418945,
      "activations/layer18_attention_weight_max": 42.13005447387695,
      "activations/layer18_attention_weight_min": -35.87055587768555,
      "activations/layer19_attention_weight_max": 20.34023094177246,
      "activations/layer19_attention_weight_min": -20.944263458251953,
      "activations/layer1_attention_weight_max": 17.111881256103516,
      "activations/layer1_attention_weight_min": -15.594557762145996,
      "activations/layer20_attention_weight_max": 20.677642822265625,
      "activations/layer20_attention_weight_min": -22.27703857421875,
      "activations/layer21_attention_weight_max": 34.09987258911133,
      "activations/layer21_attention_weight_min": -21.621458053588867,
      "activations/layer22_attention_weight_max": 29.150436401367188,
      "activations/layer22_attention_weight_min": -25.070215225219727,
      "activations/layer23_attention_weight_max": 36.95570373535156,
      "activations/layer23_attention_weight_min": -23.39919090270996,
      "activations/layer2_attention_weight_max": 33.406349182128906,
      "activations/layer2_attention_weight_min": -31.877500534057617,
      "activations/layer3_attention_weight_max": 94.17266082763672,
      "activations/layer3_attention_weight_min": -96.3485336303711,
      "activations/layer4_attention_weight_max": 86.70081329345703,
      "activations/layer4_attention_weight_min": -89.6259765625,
      "activations/layer5_attention_weight_max": 68.3779296875,
      "activations/layer5_attention_weight_min": -82.94490051269531,
      "activations/layer6_attention_weight_max": 47.05820846557617,
      "activations/layer6_attention_weight_min": -49.87834167480469,
      "activations/layer7_attention_weight_max": 62.96463394165039,
      "activations/layer7_attention_weight_min": -62.55126190185547,
      "activations/layer8_attention_weight_max": 41.595787048339844,
      "activations/layer8_attention_weight_min": -46.05738067626953,
      "activations/layer9_attention_weight_max": 43.97758102416992,
      "activations/layer9_attention_weight_min": -44.07798385620117,
      "epoch": 14.25,
      "learning_rate": 5.871363636363635e-05,
      "loss": 2.7622,
      "step": 245200
    },
    {
      "activations/layer0_attention_weight_max": 18.277009963989258,
      "activations/layer0_attention_weight_min": -14.34829330444336,
      "activations/layer10_attention_weight_max": 35.891944885253906,
      "activations/layer10_attention_weight_min": -35.6389274597168,
      "activations/layer11_attention_weight_max": 34.056026458740234,
      "activations/layer11_attention_weight_min": -34.50677490234375,
      "activations/layer12_attention_weight_max": 17.622451782226562,
      "activations/layer12_attention_weight_min": -25.57213592529297,
      "activations/layer13_attention_weight_max": 34.32188415527344,
      "activations/layer13_attention_weight_min": -26.44449234008789,
      "activations/layer14_attention_weight_max": 33.935325622558594,
      "activations/layer14_attention_weight_min": -30.470674514770508,
      "activations/layer15_attention_weight_max": 29.378110885620117,
      "activations/layer15_attention_weight_min": -30.964736938476562,
      "activations/layer16_attention_weight_max": 30.600786209106445,
      "activations/layer16_attention_weight_min": -30.138484954833984,
      "activations/layer17_attention_weight_max": 48.50664138793945,
      "activations/layer17_attention_weight_min": -46.812496185302734,
      "activations/layer18_attention_weight_max": 47.04593276977539,
      "activations/layer18_attention_weight_min": -41.04776382446289,
      "activations/layer19_attention_weight_max": 23.0745906829834,
      "activations/layer19_attention_weight_min": -22.744115829467773,
      "activations/layer1_attention_weight_max": 18.224445343017578,
      "activations/layer1_attention_weight_min": -15.310303688049316,
      "activations/layer20_attention_weight_max": 22.152376174926758,
      "activations/layer20_attention_weight_min": -23.359590530395508,
      "activations/layer21_attention_weight_max": 35.45658493041992,
      "activations/layer21_attention_weight_min": -22.7044620513916,
      "activations/layer22_attention_weight_max": 26.692251205444336,
      "activations/layer22_attention_weight_min": -26.403413772583008,
      "activations/layer23_attention_weight_max": 33.42939376831055,
      "activations/layer23_attention_weight_min": -23.125946044921875,
      "activations/layer2_attention_weight_max": 31.551589965820312,
      "activations/layer2_attention_weight_min": -30.383262634277344,
      "activations/layer3_attention_weight_max": 90.7008056640625,
      "activations/layer3_attention_weight_min": -90.71603393554688,
      "activations/layer4_attention_weight_max": 91.41554260253906,
      "activations/layer4_attention_weight_min": -87.97991180419922,
      "activations/layer5_attention_weight_max": 70.85883331298828,
      "activations/layer5_attention_weight_min": -76.17117309570312,
      "activations/layer6_attention_weight_max": 54.22993469238281,
      "activations/layer6_attention_weight_min": -54.04286575317383,
      "activations/layer7_attention_weight_max": 67.98548889160156,
      "activations/layer7_attention_weight_min": -63.683868408203125,
      "activations/layer8_attention_weight_max": 46.64188766479492,
      "activations/layer8_attention_weight_min": -47.89793395996094,
      "activations/layer9_attention_weight_max": 45.2739143371582,
      "activations/layer9_attention_weight_min": -46.488189697265625,
      "epoch": 14.25,
      "learning_rate": 5.869469696969697e-05,
      "loss": 2.7718,
      "step": 245250
    },
    {
      "activations/layer0_attention_weight_max": 17.565048217773438,
      "activations/layer0_attention_weight_min": -14.02606201171875,
      "activations/layer10_attention_weight_max": 36.30064392089844,
      "activations/layer10_attention_weight_min": -35.43448257446289,
      "activations/layer11_attention_weight_max": 33.13741683959961,
      "activations/layer11_attention_weight_min": -33.89209747314453,
      "activations/layer12_attention_weight_max": 17.906309127807617,
      "activations/layer12_attention_weight_min": -23.311704635620117,
      "activations/layer13_attention_weight_max": 36.23065185546875,
      "activations/layer13_attention_weight_min": -28.252431869506836,
      "activations/layer14_attention_weight_max": 39.59716033935547,
      "activations/layer14_attention_weight_min": -29.87889289855957,
      "activations/layer15_attention_weight_max": 35.2205810546875,
      "activations/layer15_attention_weight_min": -29.428447723388672,
      "activations/layer16_attention_weight_max": 34.10952377319336,
      "activations/layer16_attention_weight_min": -30.510026931762695,
      "activations/layer17_attention_weight_max": 54.85728073120117,
      "activations/layer17_attention_weight_min": -42.083560943603516,
      "activations/layer18_attention_weight_max": 48.5944938659668,
      "activations/layer18_attention_weight_min": -40.2744255065918,
      "activations/layer19_attention_weight_max": 21.972511291503906,
      "activations/layer19_attention_weight_min": -21.164688110351562,
      "activations/layer1_attention_weight_max": 16.28106689453125,
      "activations/layer1_attention_weight_min": -14.489389419555664,
      "activations/layer20_attention_weight_max": 21.720365524291992,
      "activations/layer20_attention_weight_min": -20.47151756286621,
      "activations/layer21_attention_weight_max": 34.940330505371094,
      "activations/layer21_attention_weight_min": -21.830520629882812,
      "activations/layer22_attention_weight_max": 30.390636444091797,
      "activations/layer22_attention_weight_min": -25.67835235595703,
      "activations/layer23_attention_weight_max": 38.24424743652344,
      "activations/layer23_attention_weight_min": -29.434043884277344,
      "activations/layer2_attention_weight_max": 30.448200225830078,
      "activations/layer2_attention_weight_min": -28.856006622314453,
      "activations/layer3_attention_weight_max": 90.4596939086914,
      "activations/layer3_attention_weight_min": -87.1559066772461,
      "activations/layer4_attention_weight_max": 87.27452087402344,
      "activations/layer4_attention_weight_min": -87.17286682128906,
      "activations/layer5_attention_weight_max": 67.26785278320312,
      "activations/layer5_attention_weight_min": -78.62570190429688,
      "activations/layer6_attention_weight_max": 50.81419372558594,
      "activations/layer6_attention_weight_min": -54.35350799560547,
      "activations/layer7_attention_weight_max": 62.214149475097656,
      "activations/layer7_attention_weight_min": -67.83901977539062,
      "activations/layer8_attention_weight_max": 44.366397857666016,
      "activations/layer8_attention_weight_min": -50.34278869628906,
      "activations/layer9_attention_weight_max": 46.31774139404297,
      "activations/layer9_attention_weight_min": -46.87681198120117,
      "epoch": 14.25,
      "learning_rate": 5.867575757575757e-05,
      "loss": 2.7692,
      "step": 245300
    },
    {
      "activations/layer0_attention_weight_max": 17.38720703125,
      "activations/layer0_attention_weight_min": -14.039334297180176,
      "activations/layer10_attention_weight_max": 41.08953857421875,
      "activations/layer10_attention_weight_min": -40.3390998840332,
      "activations/layer11_attention_weight_max": 34.37114715576172,
      "activations/layer11_attention_weight_min": -34.833221435546875,
      "activations/layer12_attention_weight_max": 21.81822395324707,
      "activations/layer12_attention_weight_min": -28.189279556274414,
      "activations/layer13_attention_weight_max": 35.65909957885742,
      "activations/layer13_attention_weight_min": -28.50935935974121,
      "activations/layer14_attention_weight_max": 39.35612106323242,
      "activations/layer14_attention_weight_min": -31.077415466308594,
      "activations/layer15_attention_weight_max": 36.860050201416016,
      "activations/layer15_attention_weight_min": -31.722122192382812,
      "activations/layer16_attention_weight_max": 33.42156219482422,
      "activations/layer16_attention_weight_min": -30.93467140197754,
      "activations/layer17_attention_weight_max": 55.327598571777344,
      "activations/layer17_attention_weight_min": -45.813167572021484,
      "activations/layer18_attention_weight_max": 50.234962463378906,
      "activations/layer18_attention_weight_min": -38.295799255371094,
      "activations/layer19_attention_weight_max": 24.49165916442871,
      "activations/layer19_attention_weight_min": -20.979585647583008,
      "activations/layer1_attention_weight_max": 15.790377616882324,
      "activations/layer1_attention_weight_min": -15.361802101135254,
      "activations/layer20_attention_weight_max": 24.549100875854492,
      "activations/layer20_attention_weight_min": -21.08218765258789,
      "activations/layer21_attention_weight_max": 41.81770706176758,
      "activations/layer21_attention_weight_min": -24.379507064819336,
      "activations/layer22_attention_weight_max": 31.043075561523438,
      "activations/layer22_attention_weight_min": -26.029315948486328,
      "activations/layer23_attention_weight_max": 41.571083068847656,
      "activations/layer23_attention_weight_min": -24.44934844970703,
      "activations/layer2_attention_weight_max": 30.575864791870117,
      "activations/layer2_attention_weight_min": -29.307506561279297,
      "activations/layer3_attention_weight_max": 89.06777954101562,
      "activations/layer3_attention_weight_min": -91.39398193359375,
      "activations/layer4_attention_weight_max": 88.36925506591797,
      "activations/layer4_attention_weight_min": -89.2441177368164,
      "activations/layer5_attention_weight_max": 66.89987182617188,
      "activations/layer5_attention_weight_min": -76.75585174560547,
      "activations/layer6_attention_weight_max": 51.70935821533203,
      "activations/layer6_attention_weight_min": -52.88575744628906,
      "activations/layer7_attention_weight_max": 65.08998107910156,
      "activations/layer7_attention_weight_min": -66.00944519042969,
      "activations/layer8_attention_weight_max": 46.59408187866211,
      "activations/layer8_attention_weight_min": -49.866188049316406,
      "activations/layer9_attention_weight_max": 47.57526779174805,
      "activations/layer9_attention_weight_min": -49.83412551879883,
      "epoch": 14.26,
      "learning_rate": 5.8656818181818177e-05,
      "loss": 2.7737,
      "step": 245350
    },
    {
      "activations/layer0_attention_weight_max": 17.62408447265625,
      "activations/layer0_attention_weight_min": -13.928192138671875,
      "activations/layer10_attention_weight_max": 35.03016662597656,
      "activations/layer10_attention_weight_min": -34.56385040283203,
      "activations/layer11_attention_weight_max": 31.332481384277344,
      "activations/layer11_attention_weight_min": -33.8006477355957,
      "activations/layer12_attention_weight_max": 18.058849334716797,
      "activations/layer12_attention_weight_min": -24.63926887512207,
      "activations/layer13_attention_weight_max": 34.325782775878906,
      "activations/layer13_attention_weight_min": -26.765098571777344,
      "activations/layer14_attention_weight_max": 34.49114990234375,
      "activations/layer14_attention_weight_min": -30.02403450012207,
      "activations/layer15_attention_weight_max": 32.09360885620117,
      "activations/layer15_attention_weight_min": -29.78876495361328,
      "activations/layer16_attention_weight_max": 29.225839614868164,
      "activations/layer16_attention_weight_min": -30.63358497619629,
      "activations/layer17_attention_weight_max": 50.96282196044922,
      "activations/layer17_attention_weight_min": -44.7050666809082,
      "activations/layer18_attention_weight_max": 46.253334045410156,
      "activations/layer18_attention_weight_min": -38.02021408081055,
      "activations/layer19_attention_weight_max": 24.51251792907715,
      "activations/layer19_attention_weight_min": -21.62603187561035,
      "activations/layer1_attention_weight_max": 15.812968254089355,
      "activations/layer1_attention_weight_min": -13.637175559997559,
      "activations/layer20_attention_weight_max": 21.619157791137695,
      "activations/layer20_attention_weight_min": -20.428037643432617,
      "activations/layer21_attention_weight_max": 34.64737319946289,
      "activations/layer21_attention_weight_min": -21.868871688842773,
      "activations/layer22_attention_weight_max": 29.61448097229004,
      "activations/layer22_attention_weight_min": -25.49471092224121,
      "activations/layer23_attention_weight_max": 37.21198654174805,
      "activations/layer23_attention_weight_min": -23.531063079833984,
      "activations/layer2_attention_weight_max": 31.09793472290039,
      "activations/layer2_attention_weight_min": -31.033905029296875,
      "activations/layer3_attention_weight_max": 88.46064758300781,
      "activations/layer3_attention_weight_min": -86.73157501220703,
      "activations/layer4_attention_weight_max": 89.39630126953125,
      "activations/layer4_attention_weight_min": -85.82393646240234,
      "activations/layer5_attention_weight_max": 67.2608413696289,
      "activations/layer5_attention_weight_min": -79.28190612792969,
      "activations/layer6_attention_weight_max": 50.2085075378418,
      "activations/layer6_attention_weight_min": -51.25517654418945,
      "activations/layer7_attention_weight_max": 66.01813507080078,
      "activations/layer7_attention_weight_min": -62.92052459716797,
      "activations/layer8_attention_weight_max": 43.30449295043945,
      "activations/layer8_attention_weight_min": -47.38240051269531,
      "activations/layer9_attention_weight_max": 44.61662292480469,
      "activations/layer9_attention_weight_min": -44.42131805419922,
      "epoch": 14.26,
      "learning_rate": 5.8637878787878785e-05,
      "loss": 2.7512,
      "step": 245400
    },
    {
      "activations/layer0_attention_weight_max": 17.54552459716797,
      "activations/layer0_attention_weight_min": -13.904807090759277,
      "activations/layer10_attention_weight_max": 34.61638259887695,
      "activations/layer10_attention_weight_min": -34.720123291015625,
      "activations/layer11_attention_weight_max": 30.711721420288086,
      "activations/layer11_attention_weight_min": -30.838924407958984,
      "activations/layer12_attention_weight_max": 18.7474422454834,
      "activations/layer12_attention_weight_min": -22.528562545776367,
      "activations/layer13_attention_weight_max": 36.06463623046875,
      "activations/layer13_attention_weight_min": -28.146739959716797,
      "activations/layer14_attention_weight_max": 35.57132339477539,
      "activations/layer14_attention_weight_min": -32.01980972290039,
      "activations/layer15_attention_weight_max": 33.599388122558594,
      "activations/layer15_attention_weight_min": -31.56528663635254,
      "activations/layer16_attention_weight_max": 33.15718460083008,
      "activations/layer16_attention_weight_min": -31.476329803466797,
      "activations/layer17_attention_weight_max": 53.54631423950195,
      "activations/layer17_attention_weight_min": -43.261688232421875,
      "activations/layer18_attention_weight_max": 51.74808120727539,
      "activations/layer18_attention_weight_min": -39.15087890625,
      "activations/layer19_attention_weight_max": 25.508514404296875,
      "activations/layer19_attention_weight_min": -23.270158767700195,
      "activations/layer1_attention_weight_max": 15.704445838928223,
      "activations/layer1_attention_weight_min": -14.257326126098633,
      "activations/layer20_attention_weight_max": 26.60664939880371,
      "activations/layer20_attention_weight_min": -20.99945640563965,
      "activations/layer21_attention_weight_max": 37.632015228271484,
      "activations/layer21_attention_weight_min": -23.36282730102539,
      "activations/layer22_attention_weight_max": 28.82280158996582,
      "activations/layer22_attention_weight_min": -25.98495864868164,
      "activations/layer23_attention_weight_max": 35.666046142578125,
      "activations/layer23_attention_weight_min": -23.745046615600586,
      "activations/layer2_attention_weight_max": 30.242572784423828,
      "activations/layer2_attention_weight_min": -28.418357849121094,
      "activations/layer3_attention_weight_max": 88.44515228271484,
      "activations/layer3_attention_weight_min": -87.30650329589844,
      "activations/layer4_attention_weight_max": 88.19160461425781,
      "activations/layer4_attention_weight_min": -84.7321548461914,
      "activations/layer5_attention_weight_max": 70.69766998291016,
      "activations/layer5_attention_weight_min": -74.41856384277344,
      "activations/layer6_attention_weight_max": 50.54574203491211,
      "activations/layer6_attention_weight_min": -49.08821487426758,
      "activations/layer7_attention_weight_max": 64.85563659667969,
      "activations/layer7_attention_weight_min": -62.18062973022461,
      "activations/layer8_attention_weight_max": 45.79429244995117,
      "activations/layer8_attention_weight_min": -46.98733139038086,
      "activations/layer9_attention_weight_max": 46.35590362548828,
      "activations/layer9_attention_weight_min": -43.86351776123047,
      "epoch": 14.26,
      "learning_rate": 5.861893939393939e-05,
      "loss": 2.7655,
      "step": 245450
    },
    {
      "activations/layer0_attention_weight_max": 16.144948959350586,
      "activations/layer0_attention_weight_min": -14.383052825927734,
      "activations/layer10_attention_weight_max": 40.01333999633789,
      "activations/layer10_attention_weight_min": -37.774147033691406,
      "activations/layer11_attention_weight_max": 36.306495666503906,
      "activations/layer11_attention_weight_min": -35.08763122558594,
      "activations/layer12_attention_weight_max": 18.94723892211914,
      "activations/layer12_attention_weight_min": -27.796546936035156,
      "activations/layer13_attention_weight_max": 37.714752197265625,
      "activations/layer13_attention_weight_min": -26.959890365600586,
      "activations/layer14_attention_weight_max": 41.711246490478516,
      "activations/layer14_attention_weight_min": -31.83177375793457,
      "activations/layer15_attention_weight_max": 37.01995086669922,
      "activations/layer15_attention_weight_min": -30.693395614624023,
      "activations/layer16_attention_weight_max": 33.107810974121094,
      "activations/layer16_attention_weight_min": -30.790605545043945,
      "activations/layer17_attention_weight_max": 54.9591064453125,
      "activations/layer17_attention_weight_min": -44.333229064941406,
      "activations/layer18_attention_weight_max": 45.553070068359375,
      "activations/layer18_attention_weight_min": -41.17424774169922,
      "activations/layer19_attention_weight_max": 22.657201766967773,
      "activations/layer19_attention_weight_min": -23.17134666442871,
      "activations/layer1_attention_weight_max": 16.17978286743164,
      "activations/layer1_attention_weight_min": -15.267043113708496,
      "activations/layer20_attention_weight_max": 23.306842803955078,
      "activations/layer20_attention_weight_min": -21.959625244140625,
      "activations/layer21_attention_weight_max": 39.17102813720703,
      "activations/layer21_attention_weight_min": -23.647611618041992,
      "activations/layer22_attention_weight_max": 31.360980987548828,
      "activations/layer22_attention_weight_min": -26.58804702758789,
      "activations/layer23_attention_weight_max": 36.92982864379883,
      "activations/layer23_attention_weight_min": -25.968727111816406,
      "activations/layer2_attention_weight_max": 32.32328414916992,
      "activations/layer2_attention_weight_min": -32.28458023071289,
      "activations/layer3_attention_weight_max": 96.5547866821289,
      "activations/layer3_attention_weight_min": -96.10810089111328,
      "activations/layer4_attention_weight_max": 92.50137329101562,
      "activations/layer4_attention_weight_min": -91.41545867919922,
      "activations/layer5_attention_weight_max": 72.44036865234375,
      "activations/layer5_attention_weight_min": -74.75279235839844,
      "activations/layer6_attention_weight_max": 51.10720443725586,
      "activations/layer6_attention_weight_min": -52.96726608276367,
      "activations/layer7_attention_weight_max": 70.31843566894531,
      "activations/layer7_attention_weight_min": -68.23645782470703,
      "activations/layer8_attention_weight_max": 47.853275299072266,
      "activations/layer8_attention_weight_min": -50.40446853637695,
      "activations/layer9_attention_weight_max": 48.34466552734375,
      "activations/layer9_attention_weight_min": -51.629920959472656,
      "epoch": 14.26,
      "learning_rate": 5.8599999999999995e-05,
      "loss": 2.7647,
      "step": 245500
    },
    {
      "activations/layer0_attention_weight_max": 16.181385040283203,
      "activations/layer0_attention_weight_min": -14.427305221557617,
      "activations/layer10_attention_weight_max": 34.81084442138672,
      "activations/layer10_attention_weight_min": -33.995361328125,
      "activations/layer11_attention_weight_max": 31.085426330566406,
      "activations/layer11_attention_weight_min": -33.97692108154297,
      "activations/layer12_attention_weight_max": 18.401962280273438,
      "activations/layer12_attention_weight_min": -25.558988571166992,
      "activations/layer13_attention_weight_max": 32.52308654785156,
      "activations/layer13_attention_weight_min": -28.44542121887207,
      "activations/layer14_attention_weight_max": 33.03876495361328,
      "activations/layer14_attention_weight_min": -30.02651023864746,
      "activations/layer15_attention_weight_max": 32.69263458251953,
      "activations/layer15_attention_weight_min": -29.61917495727539,
      "activations/layer16_attention_weight_max": 32.261878967285156,
      "activations/layer16_attention_weight_min": -29.49542236328125,
      "activations/layer17_attention_weight_max": 51.57810592651367,
      "activations/layer17_attention_weight_min": -45.0467414855957,
      "activations/layer18_attention_weight_max": 47.18732833862305,
      "activations/layer18_attention_weight_min": -38.11725997924805,
      "activations/layer19_attention_weight_max": 21.546783447265625,
      "activations/layer19_attention_weight_min": -21.103803634643555,
      "activations/layer1_attention_weight_max": 16.639020919799805,
      "activations/layer1_attention_weight_min": -16.087644577026367,
      "activations/layer20_attention_weight_max": 22.808807373046875,
      "activations/layer20_attention_weight_min": -21.30106544494629,
      "activations/layer21_attention_weight_max": 38.275665283203125,
      "activations/layer21_attention_weight_min": -22.831985473632812,
      "activations/layer22_attention_weight_max": 30.938798904418945,
      "activations/layer22_attention_weight_min": -26.202472686767578,
      "activations/layer23_attention_weight_max": 38.987937927246094,
      "activations/layer23_attention_weight_min": -23.652454376220703,
      "activations/layer2_attention_weight_max": 31.55284881591797,
      "activations/layer2_attention_weight_min": -30.431119918823242,
      "activations/layer3_attention_weight_max": 92.18329620361328,
      "activations/layer3_attention_weight_min": -94.7567138671875,
      "activations/layer4_attention_weight_max": 92.79535675048828,
      "activations/layer4_attention_weight_min": -85.2459716796875,
      "activations/layer5_attention_weight_max": 68.68144226074219,
      "activations/layer5_attention_weight_min": -76.42375946044922,
      "activations/layer6_attention_weight_max": 50.186588287353516,
      "activations/layer6_attention_weight_min": -52.349002838134766,
      "activations/layer7_attention_weight_max": 66.74575805664062,
      "activations/layer7_attention_weight_min": -63.6568717956543,
      "activations/layer8_attention_weight_max": 43.265777587890625,
      "activations/layer8_attention_weight_min": -45.657955169677734,
      "activations/layer9_attention_weight_max": 43.82395935058594,
      "activations/layer9_attention_weight_min": -45.016998291015625,
      "epoch": 14.27,
      "learning_rate": 5.8581060606060596e-05,
      "loss": 2.7631,
      "step": 245550
    },
    {
      "activations/layer0_attention_weight_max": 16.259286880493164,
      "activations/layer0_attention_weight_min": -14.640195846557617,
      "activations/layer10_attention_weight_max": 35.533870697021484,
      "activations/layer10_attention_weight_min": -33.94461441040039,
      "activations/layer11_attention_weight_max": 31.942859649658203,
      "activations/layer11_attention_weight_min": -32.467376708984375,
      "activations/layer12_attention_weight_max": 19.134984970092773,
      "activations/layer12_attention_weight_min": -26.38481903076172,
      "activations/layer13_attention_weight_max": 39.727535247802734,
      "activations/layer13_attention_weight_min": -29.030101776123047,
      "activations/layer14_attention_weight_max": 37.767730712890625,
      "activations/layer14_attention_weight_min": -31.177766799926758,
      "activations/layer15_attention_weight_max": 36.83357238769531,
      "activations/layer15_attention_weight_min": -30.97636604309082,
      "activations/layer16_attention_weight_max": 32.310604095458984,
      "activations/layer16_attention_weight_min": -30.25995635986328,
      "activations/layer17_attention_weight_max": 52.80150604248047,
      "activations/layer17_attention_weight_min": -45.09797668457031,
      "activations/layer18_attention_weight_max": 47.930015563964844,
      "activations/layer18_attention_weight_min": -39.17257308959961,
      "activations/layer19_attention_weight_max": 25.898971557617188,
      "activations/layer19_attention_weight_min": -24.18009376525879,
      "activations/layer1_attention_weight_max": 17.153076171875,
      "activations/layer1_attention_weight_min": -14.402084350585938,
      "activations/layer20_attention_weight_max": 24.565013885498047,
      "activations/layer20_attention_weight_min": -23.746641159057617,
      "activations/layer21_attention_weight_max": 36.38531494140625,
      "activations/layer21_attention_weight_min": -22.532028198242188,
      "activations/layer22_attention_weight_max": 32.55600357055664,
      "activations/layer22_attention_weight_min": -24.514123916625977,
      "activations/layer23_attention_weight_max": 34.138099670410156,
      "activations/layer23_attention_weight_min": -22.901784896850586,
      "activations/layer2_attention_weight_max": 31.47738265991211,
      "activations/layer2_attention_weight_min": -30.180160522460938,
      "activations/layer3_attention_weight_max": 97.38658905029297,
      "activations/layer3_attention_weight_min": -92.676025390625,
      "activations/layer4_attention_weight_max": 89.64775848388672,
      "activations/layer4_attention_weight_min": -89.22579956054688,
      "activations/layer5_attention_weight_max": 72.78350830078125,
      "activations/layer5_attention_weight_min": -75.78108215332031,
      "activations/layer6_attention_weight_max": 50.09075164794922,
      "activations/layer6_attention_weight_min": -50.50498962402344,
      "activations/layer7_attention_weight_max": 62.95170593261719,
      "activations/layer7_attention_weight_min": -63.22340393066406,
      "activations/layer8_attention_weight_max": 47.70920181274414,
      "activations/layer8_attention_weight_min": -46.2952880859375,
      "activations/layer9_attention_weight_max": 43.196044921875,
      "activations/layer9_attention_weight_min": -44.147796630859375,
      "epoch": 14.27,
      "learning_rate": 5.856212121212121e-05,
      "loss": 2.7669,
      "step": 245600
    },
    {
      "activations/layer0_attention_weight_max": 15.43885326385498,
      "activations/layer0_attention_weight_min": -14.309496879577637,
      "activations/layer10_attention_weight_max": 38.31135559082031,
      "activations/layer10_attention_weight_min": -34.0621223449707,
      "activations/layer11_attention_weight_max": 32.456756591796875,
      "activations/layer11_attention_weight_min": -32.32176971435547,
      "activations/layer12_attention_weight_max": 23.847835540771484,
      "activations/layer12_attention_weight_min": -24.15337562561035,
      "activations/layer13_attention_weight_max": 37.706844329833984,
      "activations/layer13_attention_weight_min": -28.761919021606445,
      "activations/layer14_attention_weight_max": 37.72932815551758,
      "activations/layer14_attention_weight_min": -29.914417266845703,
      "activations/layer15_attention_weight_max": 34.316673278808594,
      "activations/layer15_attention_weight_min": -31.2537899017334,
      "activations/layer16_attention_weight_max": 32.16514205932617,
      "activations/layer16_attention_weight_min": -31.896560668945312,
      "activations/layer17_attention_weight_max": 50.73387908935547,
      "activations/layer17_attention_weight_min": -43.8215446472168,
      "activations/layer18_attention_weight_max": 47.3297233581543,
      "activations/layer18_attention_weight_min": -39.174869537353516,
      "activations/layer19_attention_weight_max": 22.89086151123047,
      "activations/layer19_attention_weight_min": -21.761791229248047,
      "activations/layer1_attention_weight_max": 16.330049514770508,
      "activations/layer1_attention_weight_min": -14.03783130645752,
      "activations/layer20_attention_weight_max": 21.095497131347656,
      "activations/layer20_attention_weight_min": -21.367624282836914,
      "activations/layer21_attention_weight_max": 35.90715026855469,
      "activations/layer21_attention_weight_min": -23.828369140625,
      "activations/layer22_attention_weight_max": 31.993371963500977,
      "activations/layer22_attention_weight_min": -24.8486270904541,
      "activations/layer23_attention_weight_max": 36.92290496826172,
      "activations/layer23_attention_weight_min": -24.757980346679688,
      "activations/layer2_attention_weight_max": 32.17068099975586,
      "activations/layer2_attention_weight_min": -29.860126495361328,
      "activations/layer3_attention_weight_max": 94.4187240600586,
      "activations/layer3_attention_weight_min": -94.36219787597656,
      "activations/layer4_attention_weight_max": 94.04177856445312,
      "activations/layer4_attention_weight_min": -89.09978485107422,
      "activations/layer5_attention_weight_max": 68.27468872070312,
      "activations/layer5_attention_weight_min": -75.7328109741211,
      "activations/layer6_attention_weight_max": 50.095008850097656,
      "activations/layer6_attention_weight_min": -50.68880081176758,
      "activations/layer7_attention_weight_max": 64.95475769042969,
      "activations/layer7_attention_weight_min": -62.84940719604492,
      "activations/layer8_attention_weight_max": 45.09573745727539,
      "activations/layer8_attention_weight_min": -45.919952392578125,
      "activations/layer9_attention_weight_max": 45.462852478027344,
      "activations/layer9_attention_weight_min": -45.857078552246094,
      "epoch": 14.27,
      "learning_rate": 5.854318181818181e-05,
      "loss": 2.753,
      "step": 245650
    },
    {
      "activations/layer0_attention_weight_max": 16.508007049560547,
      "activations/layer0_attention_weight_min": -14.279634475708008,
      "activations/layer10_attention_weight_max": 35.380348205566406,
      "activations/layer10_attention_weight_min": -36.91859436035156,
      "activations/layer11_attention_weight_max": 30.30255699157715,
      "activations/layer11_attention_weight_min": -33.54671859741211,
      "activations/layer12_attention_weight_max": 18.948091506958008,
      "activations/layer12_attention_weight_min": -22.699583053588867,
      "activations/layer13_attention_weight_max": 38.55244064331055,
      "activations/layer13_attention_weight_min": -33.12060546875,
      "activations/layer14_attention_weight_max": 36.03835678100586,
      "activations/layer14_attention_weight_min": -30.322614669799805,
      "activations/layer15_attention_weight_max": 34.75912857055664,
      "activations/layer15_attention_weight_min": -30.676103591918945,
      "activations/layer16_attention_weight_max": 33.01087951660156,
      "activations/layer16_attention_weight_min": -30.568754196166992,
      "activations/layer17_attention_weight_max": 50.95172119140625,
      "activations/layer17_attention_weight_min": -45.6419563293457,
      "activations/layer18_attention_weight_max": 45.81412887573242,
      "activations/layer18_attention_weight_min": -40.277915954589844,
      "activations/layer19_attention_weight_max": 26.081174850463867,
      "activations/layer19_attention_weight_min": -21.787799835205078,
      "activations/layer1_attention_weight_max": 15.990538597106934,
      "activations/layer1_attention_weight_min": -15.064422607421875,
      "activations/layer20_attention_weight_max": 24.048358917236328,
      "activations/layer20_attention_weight_min": -22.125411987304688,
      "activations/layer21_attention_weight_max": 35.99983596801758,
      "activations/layer21_attention_weight_min": -23.14040184020996,
      "activations/layer22_attention_weight_max": 33.14167404174805,
      "activations/layer22_attention_weight_min": -26.523340225219727,
      "activations/layer23_attention_weight_max": 38.41585159301758,
      "activations/layer23_attention_weight_min": -24.5678768157959,
      "activations/layer2_attention_weight_max": 32.62786865234375,
      "activations/layer2_attention_weight_min": -32.05249786376953,
      "activations/layer3_attention_weight_max": 96.50855255126953,
      "activations/layer3_attention_weight_min": -94.67938232421875,
      "activations/layer4_attention_weight_max": 93.0367431640625,
      "activations/layer4_attention_weight_min": -87.35842895507812,
      "activations/layer5_attention_weight_max": 71.45820617675781,
      "activations/layer5_attention_weight_min": -75.71762084960938,
      "activations/layer6_attention_weight_max": 50.86271286010742,
      "activations/layer6_attention_weight_min": -50.379920959472656,
      "activations/layer7_attention_weight_max": 67.69409942626953,
      "activations/layer7_attention_weight_min": -65.08277893066406,
      "activations/layer8_attention_weight_max": 45.92586898803711,
      "activations/layer8_attention_weight_min": -46.509033203125,
      "activations/layer9_attention_weight_max": 47.209571838378906,
      "activations/layer9_attention_weight_min": -46.193702697753906,
      "epoch": 14.28,
      "learning_rate": 5.852424242424242e-05,
      "loss": 2.7595,
      "step": 245700
    },
    {
      "activations/layer0_attention_weight_max": 16.992307662963867,
      "activations/layer0_attention_weight_min": -14.222759246826172,
      "activations/layer10_attention_weight_max": 34.168861389160156,
      "activations/layer10_attention_weight_min": -34.4267578125,
      "activations/layer11_attention_weight_max": 31.055662155151367,
      "activations/layer11_attention_weight_min": -32.965240478515625,
      "activations/layer12_attention_weight_max": 18.096702575683594,
      "activations/layer12_attention_weight_min": -24.692691802978516,
      "activations/layer13_attention_weight_max": 34.37907409667969,
      "activations/layer13_attention_weight_min": -29.16494369506836,
      "activations/layer14_attention_weight_max": 34.05824661254883,
      "activations/layer14_attention_weight_min": -28.45871925354004,
      "activations/layer15_attention_weight_max": 30.16518783569336,
      "activations/layer15_attention_weight_min": -30.408559799194336,
      "activations/layer16_attention_weight_max": 28.472944259643555,
      "activations/layer16_attention_weight_min": -29.479827880859375,
      "activations/layer17_attention_weight_max": 48.74223709106445,
      "activations/layer17_attention_weight_min": -42.229373931884766,
      "activations/layer18_attention_weight_max": 42.7669677734375,
      "activations/layer18_attention_weight_min": -37.431312561035156,
      "activations/layer19_attention_weight_max": 21.67252540588379,
      "activations/layer19_attention_weight_min": -23.17891502380371,
      "activations/layer1_attention_weight_max": 16.81829261779785,
      "activations/layer1_attention_weight_min": -15.862070083618164,
      "activations/layer20_attention_weight_max": 22.777122497558594,
      "activations/layer20_attention_weight_min": -24.99840545654297,
      "activations/layer21_attention_weight_max": 30.119182586669922,
      "activations/layer21_attention_weight_min": -24.635894775390625,
      "activations/layer22_attention_weight_max": 29.398700714111328,
      "activations/layer22_attention_weight_min": -29.43378257751465,
      "activations/layer23_attention_weight_max": 33.183326721191406,
      "activations/layer23_attention_weight_min": -25.065162658691406,
      "activations/layer2_attention_weight_max": 34.733028411865234,
      "activations/layer2_attention_weight_min": -31.359289169311523,
      "activations/layer3_attention_weight_max": 93.51760864257812,
      "activations/layer3_attention_weight_min": -95.14818572998047,
      "activations/layer4_attention_weight_max": 89.9309310913086,
      "activations/layer4_attention_weight_min": -89.01583099365234,
      "activations/layer5_attention_weight_max": 66.81271362304688,
      "activations/layer5_attention_weight_min": -73.66920471191406,
      "activations/layer6_attention_weight_max": 47.84841537475586,
      "activations/layer6_attention_weight_min": -47.65980911254883,
      "activations/layer7_attention_weight_max": 63.957664489746094,
      "activations/layer7_attention_weight_min": -60.376686096191406,
      "activations/layer8_attention_weight_max": 43.797950744628906,
      "activations/layer8_attention_weight_min": -43.942874908447266,
      "activations/layer9_attention_weight_max": 42.6399040222168,
      "activations/layer9_attention_weight_min": -43.548954010009766,
      "epoch": 14.28,
      "learning_rate": 5.850530303030302e-05,
      "loss": 2.7628,
      "step": 245750
    },
    {
      "activations/layer0_attention_weight_max": 15.787800788879395,
      "activations/layer0_attention_weight_min": -14.200014114379883,
      "activations/layer10_attention_weight_max": 35.897193908691406,
      "activations/layer10_attention_weight_min": -35.16858673095703,
      "activations/layer11_attention_weight_max": 32.056297302246094,
      "activations/layer11_attention_weight_min": -32.97187042236328,
      "activations/layer12_attention_weight_max": 17.90781021118164,
      "activations/layer12_attention_weight_min": -26.801664352416992,
      "activations/layer13_attention_weight_max": 33.813899993896484,
      "activations/layer13_attention_weight_min": -27.941713333129883,
      "activations/layer14_attention_weight_max": 34.43061828613281,
      "activations/layer14_attention_weight_min": -30.22593116760254,
      "activations/layer15_attention_weight_max": 32.432098388671875,
      "activations/layer15_attention_weight_min": -30.721017837524414,
      "activations/layer16_attention_weight_max": 31.25012969970703,
      "activations/layer16_attention_weight_min": -30.241466522216797,
      "activations/layer17_attention_weight_max": 49.00068283081055,
      "activations/layer17_attention_weight_min": -44.803680419921875,
      "activations/layer18_attention_weight_max": 43.664955139160156,
      "activations/layer18_attention_weight_min": -35.62751388549805,
      "activations/layer19_attention_weight_max": 21.32729148864746,
      "activations/layer19_attention_weight_min": -22.41414451599121,
      "activations/layer1_attention_weight_max": 17.391338348388672,
      "activations/layer1_attention_weight_min": -13.917441368103027,
      "activations/layer20_attention_weight_max": 20.916770935058594,
      "activations/layer20_attention_weight_min": -20.753238677978516,
      "activations/layer21_attention_weight_max": 41.892066955566406,
      "activations/layer21_attention_weight_min": -23.317392349243164,
      "activations/layer22_attention_weight_max": 32.21348571777344,
      "activations/layer22_attention_weight_min": -24.742145538330078,
      "activations/layer23_attention_weight_max": 36.23423767089844,
      "activations/layer23_attention_weight_min": -22.81648063659668,
      "activations/layer2_attention_weight_max": 33.866424560546875,
      "activations/layer2_attention_weight_min": -32.05095672607422,
      "activations/layer3_attention_weight_max": 96.19275665283203,
      "activations/layer3_attention_weight_min": -97.87337493896484,
      "activations/layer4_attention_weight_max": 92.3482894897461,
      "activations/layer4_attention_weight_min": -93.25163269042969,
      "activations/layer5_attention_weight_max": 71.1583251953125,
      "activations/layer5_attention_weight_min": -74.74446105957031,
      "activations/layer6_attention_weight_max": 52.98993682861328,
      "activations/layer6_attention_weight_min": -52.18644714355469,
      "activations/layer7_attention_weight_max": 67.06694793701172,
      "activations/layer7_attention_weight_min": -62.12885284423828,
      "activations/layer8_attention_weight_max": 44.70757293701172,
      "activations/layer8_attention_weight_min": -46.42115783691406,
      "activations/layer9_attention_weight_max": 45.78322982788086,
      "activations/layer9_attention_weight_min": -44.315921783447266,
      "epoch": 14.28,
      "learning_rate": 5.848636363636363e-05,
      "loss": 2.7466,
      "step": 245800
    },
    {
      "activations/layer0_attention_weight_max": 15.304828643798828,
      "activations/layer0_attention_weight_min": -14.400066375732422,
      "activations/layer10_attention_weight_max": 38.86920928955078,
      "activations/layer10_attention_weight_min": -36.1348762512207,
      "activations/layer11_attention_weight_max": 33.1949348449707,
      "activations/layer11_attention_weight_min": -33.42420196533203,
      "activations/layer12_attention_weight_max": 18.9537296295166,
      "activations/layer12_attention_weight_min": -24.272010803222656,
      "activations/layer13_attention_weight_max": 37.102352142333984,
      "activations/layer13_attention_weight_min": -29.8348445892334,
      "activations/layer14_attention_weight_max": 35.947601318359375,
      "activations/layer14_attention_weight_min": -32.10482406616211,
      "activations/layer15_attention_weight_max": 34.074466705322266,
      "activations/layer15_attention_weight_min": -30.819705963134766,
      "activations/layer16_attention_weight_max": 33.49291229248047,
      "activations/layer16_attention_weight_min": -30.995433807373047,
      "activations/layer17_attention_weight_max": 51.047367095947266,
      "activations/layer17_attention_weight_min": -44.76417922973633,
      "activations/layer18_attention_weight_max": 48.69561767578125,
      "activations/layer18_attention_weight_min": -37.23949432373047,
      "activations/layer19_attention_weight_max": 22.41371726989746,
      "activations/layer19_attention_weight_min": -21.472639083862305,
      "activations/layer1_attention_weight_max": 16.240867614746094,
      "activations/layer1_attention_weight_min": -14.698568344116211,
      "activations/layer20_attention_weight_max": 23.57079315185547,
      "activations/layer20_attention_weight_min": -20.47730827331543,
      "activations/layer21_attention_weight_max": 39.81058883666992,
      "activations/layer21_attention_weight_min": -27.347076416015625,
      "activations/layer22_attention_weight_max": 30.6784725189209,
      "activations/layer22_attention_weight_min": -26.07176399230957,
      "activations/layer23_attention_weight_max": 41.212921142578125,
      "activations/layer23_attention_weight_min": -25.705734252929688,
      "activations/layer2_attention_weight_max": 35.109703063964844,
      "activations/layer2_attention_weight_min": -32.51618576049805,
      "activations/layer3_attention_weight_max": 101.9529037475586,
      "activations/layer3_attention_weight_min": -97.77510070800781,
      "activations/layer4_attention_weight_max": 93.96987915039062,
      "activations/layer4_attention_weight_min": -89.01325988769531,
      "activations/layer5_attention_weight_max": 71.26237487792969,
      "activations/layer5_attention_weight_min": -76.7538833618164,
      "activations/layer6_attention_weight_max": 52.55293273925781,
      "activations/layer6_attention_weight_min": -51.534908294677734,
      "activations/layer7_attention_weight_max": 67.59737396240234,
      "activations/layer7_attention_weight_min": -61.82563781738281,
      "activations/layer8_attention_weight_max": 49.33543014526367,
      "activations/layer8_attention_weight_min": -50.32625198364258,
      "activations/layer9_attention_weight_max": 45.92383575439453,
      "activations/layer9_attention_weight_min": -46.7684440612793,
      "epoch": 14.29,
      "learning_rate": 5.846742424242424e-05,
      "loss": 2.749,
      "step": 245850
    },
    {
      "activations/layer0_attention_weight_max": 15.152204513549805,
      "activations/layer0_attention_weight_min": -14.426697731018066,
      "activations/layer10_attention_weight_max": 37.58246612548828,
      "activations/layer10_attention_weight_min": -36.454132080078125,
      "activations/layer11_attention_weight_max": 33.610801696777344,
      "activations/layer11_attention_weight_min": -36.64955139160156,
      "activations/layer12_attention_weight_max": 18.198774337768555,
      "activations/layer12_attention_weight_min": -23.93787956237793,
      "activations/layer13_attention_weight_max": 33.83049774169922,
      "activations/layer13_attention_weight_min": -29.659372329711914,
      "activations/layer14_attention_weight_max": 32.81495666503906,
      "activations/layer14_attention_weight_min": -31.153121948242188,
      "activations/layer15_attention_weight_max": 30.602617263793945,
      "activations/layer15_attention_weight_min": -30.702112197875977,
      "activations/layer16_attention_weight_max": 29.90459632873535,
      "activations/layer16_attention_weight_min": -30.499235153198242,
      "activations/layer17_attention_weight_max": 49.95718765258789,
      "activations/layer17_attention_weight_min": -41.715309143066406,
      "activations/layer18_attention_weight_max": 46.70921325683594,
      "activations/layer18_attention_weight_min": -37.93125534057617,
      "activations/layer19_attention_weight_max": 22.49054527282715,
      "activations/layer19_attention_weight_min": -23.053874969482422,
      "activations/layer1_attention_weight_max": 16.310529708862305,
      "activations/layer1_attention_weight_min": -14.91734504699707,
      "activations/layer20_attention_weight_max": 20.325298309326172,
      "activations/layer20_attention_weight_min": -22.14492416381836,
      "activations/layer21_attention_weight_max": 35.776302337646484,
      "activations/layer21_attention_weight_min": -20.8968448638916,
      "activations/layer22_attention_weight_max": 29.245277404785156,
      "activations/layer22_attention_weight_min": -25.35979461669922,
      "activations/layer23_attention_weight_max": 35.59592819213867,
      "activations/layer23_attention_weight_min": -25.01900291442871,
      "activations/layer2_attention_weight_max": 33.6297492980957,
      "activations/layer2_attention_weight_min": -33.41892623901367,
      "activations/layer3_attention_weight_max": 95.1506576538086,
      "activations/layer3_attention_weight_min": -96.51316833496094,
      "activations/layer4_attention_weight_max": 91.3899917602539,
      "activations/layer4_attention_weight_min": -90.9267807006836,
      "activations/layer5_attention_weight_max": 74.72061157226562,
      "activations/layer5_attention_weight_min": -76.59388732910156,
      "activations/layer6_attention_weight_max": 49.898277282714844,
      "activations/layer6_attention_weight_min": -50.421199798583984,
      "activations/layer7_attention_weight_max": 64.07208251953125,
      "activations/layer7_attention_weight_min": -62.236610412597656,
      "activations/layer8_attention_weight_max": 48.012786865234375,
      "activations/layer8_attention_weight_min": -50.478485107421875,
      "activations/layer9_attention_weight_max": 44.904762268066406,
      "activations/layer9_attention_weight_min": -45.407962799072266,
      "epoch": 14.29,
      "learning_rate": 5.844848484848484e-05,
      "loss": 2.7552,
      "step": 245900
    },
    {
      "activations/layer0_attention_weight_max": 16.176799774169922,
      "activations/layer0_attention_weight_min": -14.230963706970215,
      "activations/layer10_attention_weight_max": 36.25385284423828,
      "activations/layer10_attention_weight_min": -35.70943069458008,
      "activations/layer11_attention_weight_max": 32.85858154296875,
      "activations/layer11_attention_weight_min": -33.765830993652344,
      "activations/layer12_attention_weight_max": 19.521862030029297,
      "activations/layer12_attention_weight_min": -24.51748275756836,
      "activations/layer13_attention_weight_max": 36.05831527709961,
      "activations/layer13_attention_weight_min": -28.879627227783203,
      "activations/layer14_attention_weight_max": 38.73298263549805,
      "activations/layer14_attention_weight_min": -29.717477798461914,
      "activations/layer15_attention_weight_max": 33.68423080444336,
      "activations/layer15_attention_weight_min": -29.679840087890625,
      "activations/layer16_attention_weight_max": 36.40742874145508,
      "activations/layer16_attention_weight_min": -31.758684158325195,
      "activations/layer17_attention_weight_max": 58.13667297363281,
      "activations/layer17_attention_weight_min": -45.79763412475586,
      "activations/layer18_attention_weight_max": 49.34873580932617,
      "activations/layer18_attention_weight_min": -38.20330810546875,
      "activations/layer19_attention_weight_max": 23.090322494506836,
      "activations/layer19_attention_weight_min": -21.63374137878418,
      "activations/layer1_attention_weight_max": 16.15110969543457,
      "activations/layer1_attention_weight_min": -15.751770973205566,
      "activations/layer20_attention_weight_max": 25.163280487060547,
      "activations/layer20_attention_weight_min": -21.28377342224121,
      "activations/layer21_attention_weight_max": 41.89992141723633,
      "activations/layer21_attention_weight_min": -22.599061965942383,
      "activations/layer22_attention_weight_max": 33.04453659057617,
      "activations/layer22_attention_weight_min": -23.88165855407715,
      "activations/layer23_attention_weight_max": 41.72825622558594,
      "activations/layer23_attention_weight_min": -25.564289093017578,
      "activations/layer2_attention_weight_max": 33.13286209106445,
      "activations/layer2_attention_weight_min": -32.386802673339844,
      "activations/layer3_attention_weight_max": 95.07534790039062,
      "activations/layer3_attention_weight_min": -92.92428588867188,
      "activations/layer4_attention_weight_max": 96.54048919677734,
      "activations/layer4_attention_weight_min": -93.6932373046875,
      "activations/layer5_attention_weight_max": 74.59256744384766,
      "activations/layer5_attention_weight_min": -77.38873291015625,
      "activations/layer6_attention_weight_max": 53.44377517700195,
      "activations/layer6_attention_weight_min": -52.47590255737305,
      "activations/layer7_attention_weight_max": 67.52742004394531,
      "activations/layer7_attention_weight_min": -64.44688415527344,
      "activations/layer8_attention_weight_max": 48.24959182739258,
      "activations/layer8_attention_weight_min": -49.9790153503418,
      "activations/layer9_attention_weight_max": 51.136924743652344,
      "activations/layer9_attention_weight_min": -46.56464767456055,
      "epoch": 14.29,
      "learning_rate": 5.842954545454545e-05,
      "loss": 2.7696,
      "step": 245950
    },
    {
      "activations/layer0_attention_weight_max": 15.977121353149414,
      "activations/layer0_attention_weight_min": -14.192741394042969,
      "activations/layer10_attention_weight_max": 35.73270034790039,
      "activations/layer10_attention_weight_min": -35.60215377807617,
      "activations/layer11_attention_weight_max": 31.909109115600586,
      "activations/layer11_attention_weight_min": -34.410926818847656,
      "activations/layer12_attention_weight_max": 19.017547607421875,
      "activations/layer12_attention_weight_min": -24.331682205200195,
      "activations/layer13_attention_weight_max": 35.072959899902344,
      "activations/layer13_attention_weight_min": -28.760845184326172,
      "activations/layer14_attention_weight_max": 35.3982048034668,
      "activations/layer14_attention_weight_min": -30.736539840698242,
      "activations/layer15_attention_weight_max": 32.75727844238281,
      "activations/layer15_attention_weight_min": -29.572404861450195,
      "activations/layer16_attention_weight_max": 31.44121551513672,
      "activations/layer16_attention_weight_min": -29.546005249023438,
      "activations/layer17_attention_weight_max": 49.7281379699707,
      "activations/layer17_attention_weight_min": -42.09756851196289,
      "activations/layer18_attention_weight_max": 46.760562896728516,
      "activations/layer18_attention_weight_min": -36.51128387451172,
      "activations/layer19_attention_weight_max": 22.464454650878906,
      "activations/layer19_attention_weight_min": -20.92351531982422,
      "activations/layer1_attention_weight_max": 17.72458267211914,
      "activations/layer1_attention_weight_min": -14.110919952392578,
      "activations/layer20_attention_weight_max": 19.59479331970215,
      "activations/layer20_attention_weight_min": -20.19650650024414,
      "activations/layer21_attention_weight_max": 30.967363357543945,
      "activations/layer21_attention_weight_min": -21.11557388305664,
      "activations/layer22_attention_weight_max": 28.03603172302246,
      "activations/layer22_attention_weight_min": -24.3049373626709,
      "activations/layer23_attention_weight_max": 37.046478271484375,
      "activations/layer23_attention_weight_min": -23.63733673095703,
      "activations/layer2_attention_weight_max": 33.44344711303711,
      "activations/layer2_attention_weight_min": -30.63701629638672,
      "activations/layer3_attention_weight_max": 95.09376525878906,
      "activations/layer3_attention_weight_min": -95.2522201538086,
      "activations/layer4_attention_weight_max": 92.13431549072266,
      "activations/layer4_attention_weight_min": -91.56456756591797,
      "activations/layer5_attention_weight_max": 71.22280883789062,
      "activations/layer5_attention_weight_min": -78.45990753173828,
      "activations/layer6_attention_weight_max": 50.93830490112305,
      "activations/layer6_attention_weight_min": -50.437400817871094,
      "activations/layer7_attention_weight_max": 65.5449447631836,
      "activations/layer7_attention_weight_min": -67.78446960449219,
      "activations/layer8_attention_weight_max": 45.090484619140625,
      "activations/layer8_attention_weight_min": -48.45171356201172,
      "activations/layer9_attention_weight_max": 45.70613479614258,
      "activations/layer9_attention_weight_min": -44.440711975097656,
      "epoch": 14.29,
      "learning_rate": 5.841060606060606e-05,
      "loss": 2.7719,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4233,
      "eval_samples_per_second": 509.778,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4233,
      "eval_openwebtext_samples_per_second": 509.778,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9017,
      "eval_wikitext_samples_per_second": 239.785,
      "step": 246000
    },
    {
      "epoch": 14.29,
      "eval_lambada_loss": 2.44921875,
      "eval_lambada_ppl": 11.57929685891098,
      "eval_lambada_runtime": 9.5411,
      "eval_lambada_samples_per_second": 510.318,
      "step": 246000
    },
    {
      "activations/layer0_attention_weight_max": 15.28207778930664,
      "activations/layer0_attention_weight_min": -14.060102462768555,
      "activations/layer10_attention_weight_max": 38.24637222290039,
      "activations/layer10_attention_weight_min": -34.47871398925781,
      "activations/layer11_attention_weight_max": 34.108455657958984,
      "activations/layer11_attention_weight_min": -33.27216339111328,
      "activations/layer12_attention_weight_max": 19.70212745666504,
      "activations/layer12_attention_weight_min": -23.724685668945312,
      "activations/layer13_attention_weight_max": 32.34820556640625,
      "activations/layer13_attention_weight_min": -27.868642807006836,
      "activations/layer14_attention_weight_max": 33.89628219604492,
      "activations/layer14_attention_weight_min": -29.48848533630371,
      "activations/layer15_attention_weight_max": 32.57683181762695,
      "activations/layer15_attention_weight_min": -29.36177635192871,
      "activations/layer16_attention_weight_max": 30.734619140625,
      "activations/layer16_attention_weight_min": -29.737754821777344,
      "activations/layer17_attention_weight_max": 52.620758056640625,
      "activations/layer17_attention_weight_min": -44.301116943359375,
      "activations/layer18_attention_weight_max": 48.335811614990234,
      "activations/layer18_attention_weight_min": -38.186031341552734,
      "activations/layer19_attention_weight_max": 24.493751525878906,
      "activations/layer19_attention_weight_min": -22.548778533935547,
      "activations/layer1_attention_weight_max": 15.870133399963379,
      "activations/layer1_attention_weight_min": -15.106532096862793,
      "activations/layer20_attention_weight_max": 23.770383834838867,
      "activations/layer20_attention_weight_min": -22.635902404785156,
      "activations/layer21_attention_weight_max": 35.49985885620117,
      "activations/layer21_attention_weight_min": -19.903553009033203,
      "activations/layer22_attention_weight_max": 28.988481521606445,
      "activations/layer22_attention_weight_min": -25.34379005432129,
      "activations/layer23_attention_weight_max": 35.92924880981445,
      "activations/layer23_attention_weight_min": -23.95111656188965,
      "activations/layer2_attention_weight_max": 32.544822692871094,
      "activations/layer2_attention_weight_min": -31.549232482910156,
      "activations/layer3_attention_weight_max": 98.49017333984375,
      "activations/layer3_attention_weight_min": -99.57975006103516,
      "activations/layer4_attention_weight_max": 96.58011627197266,
      "activations/layer4_attention_weight_min": -89.58304595947266,
      "activations/layer5_attention_weight_max": 70.0528793334961,
      "activations/layer5_attention_weight_min": -81.3731689453125,
      "activations/layer6_attention_weight_max": 50.669490814208984,
      "activations/layer6_attention_weight_min": -50.385047912597656,
      "activations/layer7_attention_weight_max": 63.83335494995117,
      "activations/layer7_attention_weight_min": -62.78388595581055,
      "activations/layer8_attention_weight_max": 45.91936492919922,
      "activations/layer8_attention_weight_min": -46.218475341796875,
      "activations/layer9_attention_weight_max": 47.3345832824707,
      "activations/layer9_attention_weight_min": -45.583988189697266,
      "epoch": 14.3,
      "learning_rate": 5.839166666666666e-05,
      "loss": 2.7655,
      "step": 246050
    },
    {
      "activations/layer0_attention_weight_max": 15.418961524963379,
      "activations/layer0_attention_weight_min": -13.865049362182617,
      "activations/layer10_attention_weight_max": 36.880210876464844,
      "activations/layer10_attention_weight_min": -35.45269775390625,
      "activations/layer11_attention_weight_max": 32.245452880859375,
      "activations/layer11_attention_weight_min": -34.366004943847656,
      "activations/layer12_attention_weight_max": 18.188152313232422,
      "activations/layer12_attention_weight_min": -24.413394927978516,
      "activations/layer13_attention_weight_max": 34.05388641357422,
      "activations/layer13_attention_weight_min": -29.95013999938965,
      "activations/layer14_attention_weight_max": 33.24995040893555,
      "activations/layer14_attention_weight_min": -31.263391494750977,
      "activations/layer15_attention_weight_max": 31.19243049621582,
      "activations/layer15_attention_weight_min": -29.153974533081055,
      "activations/layer16_attention_weight_max": 31.59869384765625,
      "activations/layer16_attention_weight_min": -30.039060592651367,
      "activations/layer17_attention_weight_max": 47.91708755493164,
      "activations/layer17_attention_weight_min": -45.19246292114258,
      "activations/layer18_attention_weight_max": 43.730133056640625,
      "activations/layer18_attention_weight_min": -39.95573806762695,
      "activations/layer19_attention_weight_max": 22.84120750427246,
      "activations/layer19_attention_weight_min": -23.68750762939453,
      "activations/layer1_attention_weight_max": 15.511540412902832,
      "activations/layer1_attention_weight_min": -13.059409141540527,
      "activations/layer20_attention_weight_max": 22.28066635131836,
      "activations/layer20_attention_weight_min": -21.131175994873047,
      "activations/layer21_attention_weight_max": 38.035221099853516,
      "activations/layer21_attention_weight_min": -22.022323608398438,
      "activations/layer22_attention_weight_max": 29.10350799560547,
      "activations/layer22_attention_weight_min": -26.2825984954834,
      "activations/layer23_attention_weight_max": 37.56089782714844,
      "activations/layer23_attention_weight_min": -23.88716697692871,
      "activations/layer2_attention_weight_max": 30.82167625427246,
      "activations/layer2_attention_weight_min": -28.39849853515625,
      "activations/layer3_attention_weight_max": 93.50836944580078,
      "activations/layer3_attention_weight_min": -96.04196166992188,
      "activations/layer4_attention_weight_max": 93.76046752929688,
      "activations/layer4_attention_weight_min": -95.27721405029297,
      "activations/layer5_attention_weight_max": 70.09437561035156,
      "activations/layer5_attention_weight_min": -81.79446411132812,
      "activations/layer6_attention_weight_max": 52.85149002075195,
      "activations/layer6_attention_weight_min": -54.84645080566406,
      "activations/layer7_attention_weight_max": 64.40219116210938,
      "activations/layer7_attention_weight_min": -61.426700592041016,
      "activations/layer8_attention_weight_max": 44.91411209106445,
      "activations/layer8_attention_weight_min": -46.28704833984375,
      "activations/layer9_attention_weight_max": 44.70830154418945,
      "activations/layer9_attention_weight_min": -46.012901306152344,
      "epoch": 14.3,
      "learning_rate": 5.837272727272727e-05,
      "loss": 2.7781,
      "step": 246100
    },
    {
      "activations/layer0_attention_weight_max": 15.932573318481445,
      "activations/layer0_attention_weight_min": -13.918214797973633,
      "activations/layer10_attention_weight_max": 34.553462982177734,
      "activations/layer10_attention_weight_min": -34.78830337524414,
      "activations/layer11_attention_weight_max": 30.36244010925293,
      "activations/layer11_attention_weight_min": -31.93043327331543,
      "activations/layer12_attention_weight_max": 18.9256649017334,
      "activations/layer12_attention_weight_min": -22.733903884887695,
      "activations/layer13_attention_weight_max": 36.12593078613281,
      "activations/layer13_attention_weight_min": -30.245887756347656,
      "activations/layer14_attention_weight_max": 34.81051254272461,
      "activations/layer14_attention_weight_min": -30.766447067260742,
      "activations/layer15_attention_weight_max": 33.129146575927734,
      "activations/layer15_attention_weight_min": -29.353708267211914,
      "activations/layer16_attention_weight_max": 31.38092803955078,
      "activations/layer16_attention_weight_min": -29.743268966674805,
      "activations/layer17_attention_weight_max": 49.8425178527832,
      "activations/layer17_attention_weight_min": -44.17436218261719,
      "activations/layer18_attention_weight_max": 45.33740234375,
      "activations/layer18_attention_weight_min": -37.163841247558594,
      "activations/layer19_attention_weight_max": 22.34577751159668,
      "activations/layer19_attention_weight_min": -20.5693416595459,
      "activations/layer1_attention_weight_max": 18.1279239654541,
      "activations/layer1_attention_weight_min": -14.292232513427734,
      "activations/layer20_attention_weight_max": 21.217782974243164,
      "activations/layer20_attention_weight_min": -20.66330337524414,
      "activations/layer21_attention_weight_max": 34.49818420410156,
      "activations/layer21_attention_weight_min": -23.840782165527344,
      "activations/layer22_attention_weight_max": 30.889404296875,
      "activations/layer22_attention_weight_min": -28.249788284301758,
      "activations/layer23_attention_weight_max": 37.60414505004883,
      "activations/layer23_attention_weight_min": -23.737468719482422,
      "activations/layer2_attention_weight_max": 33.87025833129883,
      "activations/layer2_attention_weight_min": -31.94599723815918,
      "activations/layer3_attention_weight_max": 92.96920013427734,
      "activations/layer3_attention_weight_min": -100.4273910522461,
      "activations/layer4_attention_weight_max": 90.31585693359375,
      "activations/layer4_attention_weight_min": -88.77900695800781,
      "activations/layer5_attention_weight_max": 70.7099609375,
      "activations/layer5_attention_weight_min": -76.59591674804688,
      "activations/layer6_attention_weight_max": 51.01229476928711,
      "activations/layer6_attention_weight_min": -54.14834213256836,
      "activations/layer7_attention_weight_max": 62.55543518066406,
      "activations/layer7_attention_weight_min": -63.80739974975586,
      "activations/layer8_attention_weight_max": 43.91508865356445,
      "activations/layer8_attention_weight_min": -48.88998794555664,
      "activations/layer9_attention_weight_max": 41.347713470458984,
      "activations/layer9_attention_weight_min": -45.039794921875,
      "epoch": 14.3,
      "learning_rate": 5.8353787878787875e-05,
      "loss": 2.7371,
      "step": 246150
    },
    {
      "activations/layer0_attention_weight_max": 15.75740909576416,
      "activations/layer0_attention_weight_min": -14.223102569580078,
      "activations/layer10_attention_weight_max": 36.25001907348633,
      "activations/layer10_attention_weight_min": -37.13020706176758,
      "activations/layer11_attention_weight_max": 33.945587158203125,
      "activations/layer11_attention_weight_min": -34.50891876220703,
      "activations/layer12_attention_weight_max": 21.047006607055664,
      "activations/layer12_attention_weight_min": -24.29111099243164,
      "activations/layer13_attention_weight_max": 32.59431457519531,
      "activations/layer13_attention_weight_min": -28.22538948059082,
      "activations/layer14_attention_weight_max": 35.16957092285156,
      "activations/layer14_attention_weight_min": -30.15866470336914,
      "activations/layer15_attention_weight_max": 33.62532043457031,
      "activations/layer15_attention_weight_min": -29.48720932006836,
      "activations/layer16_attention_weight_max": 32.597686767578125,
      "activations/layer16_attention_weight_min": -32.239280700683594,
      "activations/layer17_attention_weight_max": 53.038658142089844,
      "activations/layer17_attention_weight_min": -46.82101821899414,
      "activations/layer18_attention_weight_max": 44.54443359375,
      "activations/layer18_attention_weight_min": -39.713165283203125,
      "activations/layer19_attention_weight_max": 22.02998924255371,
      "activations/layer19_attention_weight_min": -22.692949295043945,
      "activations/layer1_attention_weight_max": 16.439577102661133,
      "activations/layer1_attention_weight_min": -15.346126556396484,
      "activations/layer20_attention_weight_max": 22.121244430541992,
      "activations/layer20_attention_weight_min": -26.874284744262695,
      "activations/layer21_attention_weight_max": 32.54767608642578,
      "activations/layer21_attention_weight_min": -19.829946517944336,
      "activations/layer22_attention_weight_max": 31.503578186035156,
      "activations/layer22_attention_weight_min": -26.208955764770508,
      "activations/layer23_attention_weight_max": 32.88392639160156,
      "activations/layer23_attention_weight_min": -25.29926872253418,
      "activations/layer2_attention_weight_max": 34.0832405090332,
      "activations/layer2_attention_weight_min": -32.589317321777344,
      "activations/layer3_attention_weight_max": 95.86421203613281,
      "activations/layer3_attention_weight_min": -97.16233825683594,
      "activations/layer4_attention_weight_max": 94.5001449584961,
      "activations/layer4_attention_weight_min": -90.17362976074219,
      "activations/layer5_attention_weight_max": 73.70429229736328,
      "activations/layer5_attention_weight_min": -80.3966064453125,
      "activations/layer6_attention_weight_max": 51.6755256652832,
      "activations/layer6_attention_weight_min": -51.35466003417969,
      "activations/layer7_attention_weight_max": 68.63217163085938,
      "activations/layer7_attention_weight_min": -66.36946868896484,
      "activations/layer8_attention_weight_max": 46.1418571472168,
      "activations/layer8_attention_weight_min": -47.98955154418945,
      "activations/layer9_attention_weight_max": 48.01482391357422,
      "activations/layer9_attention_weight_min": -47.00404357910156,
      "epoch": 14.31,
      "learning_rate": 5.833484848484848e-05,
      "loss": 2.7625,
      "step": 246200
    },
    {
      "activations/layer0_attention_weight_max": 15.798199653625488,
      "activations/layer0_attention_weight_min": -14.499578475952148,
      "activations/layer10_attention_weight_max": 34.822601318359375,
      "activations/layer10_attention_weight_min": -34.98084259033203,
      "activations/layer11_attention_weight_max": 30.95014190673828,
      "activations/layer11_attention_weight_min": -31.912155151367188,
      "activations/layer12_attention_weight_max": 18.98490333557129,
      "activations/layer12_attention_weight_min": -24.7755069732666,
      "activations/layer13_attention_weight_max": 36.69788360595703,
      "activations/layer13_attention_weight_min": -28.227951049804688,
      "activations/layer14_attention_weight_max": 35.92244338989258,
      "activations/layer14_attention_weight_min": -29.18185043334961,
      "activations/layer15_attention_weight_max": 32.372802734375,
      "activations/layer15_attention_weight_min": -29.188749313354492,
      "activations/layer16_attention_weight_max": 33.18532180786133,
      "activations/layer16_attention_weight_min": -30.161746978759766,
      "activations/layer17_attention_weight_max": 52.84060287475586,
      "activations/layer17_attention_weight_min": -43.469051361083984,
      "activations/layer18_attention_weight_max": 45.677669525146484,
      "activations/layer18_attention_weight_min": -38.27854919433594,
      "activations/layer19_attention_weight_max": 21.892480850219727,
      "activations/layer19_attention_weight_min": -22.498281478881836,
      "activations/layer1_attention_weight_max": 17.706817626953125,
      "activations/layer1_attention_weight_min": -15.181931495666504,
      "activations/layer20_attention_weight_max": 22.203649520874023,
      "activations/layer20_attention_weight_min": -21.31218910217285,
      "activations/layer21_attention_weight_max": 35.876564025878906,
      "activations/layer21_attention_weight_min": -22.2906494140625,
      "activations/layer22_attention_weight_max": 28.70060920715332,
      "activations/layer22_attention_weight_min": -24.391979217529297,
      "activations/layer23_attention_weight_max": 37.4107666015625,
      "activations/layer23_attention_weight_min": -23.25570297241211,
      "activations/layer2_attention_weight_max": 32.58663558959961,
      "activations/layer2_attention_weight_min": -32.026695251464844,
      "activations/layer3_attention_weight_max": 95.36618041992188,
      "activations/layer3_attention_weight_min": -104.9999008178711,
      "activations/layer4_attention_weight_max": 95.68138885498047,
      "activations/layer4_attention_weight_min": -93.78411102294922,
      "activations/layer5_attention_weight_max": 73.52157592773438,
      "activations/layer5_attention_weight_min": -75.71864318847656,
      "activations/layer6_attention_weight_max": 53.36119842529297,
      "activations/layer6_attention_weight_min": -53.67970657348633,
      "activations/layer7_attention_weight_max": 67.36414337158203,
      "activations/layer7_attention_weight_min": -67.42884063720703,
      "activations/layer8_attention_weight_max": 46.45623016357422,
      "activations/layer8_attention_weight_min": -48.45278549194336,
      "activations/layer9_attention_weight_max": 47.51885223388672,
      "activations/layer9_attention_weight_min": -48.38347625732422,
      "epoch": 14.31,
      "learning_rate": 5.8315909090909085e-05,
      "loss": 2.7652,
      "step": 246250
    },
    {
      "activations/layer0_attention_weight_max": 15.978219032287598,
      "activations/layer0_attention_weight_min": -14.174468994140625,
      "activations/layer10_attention_weight_max": 37.6549072265625,
      "activations/layer10_attention_weight_min": -37.260093688964844,
      "activations/layer11_attention_weight_max": 33.940948486328125,
      "activations/layer11_attention_weight_min": -36.21101379394531,
      "activations/layer12_attention_weight_max": 19.618371963500977,
      "activations/layer12_attention_weight_min": -27.086137771606445,
      "activations/layer13_attention_weight_max": 35.506561279296875,
      "activations/layer13_attention_weight_min": -29.94392967224121,
      "activations/layer14_attention_weight_max": 37.243690490722656,
      "activations/layer14_attention_weight_min": -30.273229598999023,
      "activations/layer15_attention_weight_max": 33.31792449951172,
      "activations/layer15_attention_weight_min": -28.466398239135742,
      "activations/layer16_attention_weight_max": 32.36326599121094,
      "activations/layer16_attention_weight_min": -29.07711410522461,
      "activations/layer17_attention_weight_max": 51.62273025512695,
      "activations/layer17_attention_weight_min": -44.5241584777832,
      "activations/layer18_attention_weight_max": 47.0120849609375,
      "activations/layer18_attention_weight_min": -37.97102737426758,
      "activations/layer19_attention_weight_max": 23.112123489379883,
      "activations/layer19_attention_weight_min": -19.870216369628906,
      "activations/layer1_attention_weight_max": 16.275203704833984,
      "activations/layer1_attention_weight_min": -13.984251976013184,
      "activations/layer20_attention_weight_max": 20.54631996154785,
      "activations/layer20_attention_weight_min": -21.01620864868164,
      "activations/layer21_attention_weight_max": 34.32586669921875,
      "activations/layer21_attention_weight_min": -22.057769775390625,
      "activations/layer22_attention_weight_max": 26.020187377929688,
      "activations/layer22_attention_weight_min": -24.246158599853516,
      "activations/layer23_attention_weight_max": 36.450775146484375,
      "activations/layer23_attention_weight_min": -24.07038116455078,
      "activations/layer2_attention_weight_max": 33.6934814453125,
      "activations/layer2_attention_weight_min": -30.620092391967773,
      "activations/layer3_attention_weight_max": 87.83964538574219,
      "activations/layer3_attention_weight_min": -87.54669189453125,
      "activations/layer4_attention_weight_max": 87.7224349975586,
      "activations/layer4_attention_weight_min": -86.37612915039062,
      "activations/layer5_attention_weight_max": 66.79727172851562,
      "activations/layer5_attention_weight_min": -72.25285339355469,
      "activations/layer6_attention_weight_max": 49.23855209350586,
      "activations/layer6_attention_weight_min": -52.85789108276367,
      "activations/layer7_attention_weight_max": 61.77589416503906,
      "activations/layer7_attention_weight_min": -63.49866485595703,
      "activations/layer8_attention_weight_max": 43.814056396484375,
      "activations/layer8_attention_weight_min": -47.504310607910156,
      "activations/layer9_attention_weight_max": 45.19770812988281,
      "activations/layer9_attention_weight_min": -46.5213623046875,
      "epoch": 14.31,
      "learning_rate": 5.8296969696969686e-05,
      "loss": 2.7517,
      "step": 246300
    },
    {
      "activations/layer0_attention_weight_max": 15.585479736328125,
      "activations/layer0_attention_weight_min": -14.420000076293945,
      "activations/layer10_attention_weight_max": 46.94612503051758,
      "activations/layer10_attention_weight_min": -43.40373992919922,
      "activations/layer11_attention_weight_max": 42.12000274658203,
      "activations/layer11_attention_weight_min": -38.75802230834961,
      "activations/layer12_attention_weight_max": 16.83248519897461,
      "activations/layer12_attention_weight_min": -27.123735427856445,
      "activations/layer13_attention_weight_max": 37.1129150390625,
      "activations/layer13_attention_weight_min": -29.76123809814453,
      "activations/layer14_attention_weight_max": 39.76536560058594,
      "activations/layer14_attention_weight_min": -29.368953704833984,
      "activations/layer15_attention_weight_max": 36.78471374511719,
      "activations/layer15_attention_weight_min": -29.88006591796875,
      "activations/layer16_attention_weight_max": 30.51454734802246,
      "activations/layer16_attention_weight_min": -29.79487419128418,
      "activations/layer17_attention_weight_max": 52.26441955566406,
      "activations/layer17_attention_weight_min": -44.785057067871094,
      "activations/layer18_attention_weight_max": 47.53024673461914,
      "activations/layer18_attention_weight_min": -41.526145935058594,
      "activations/layer19_attention_weight_max": 24.37172508239746,
      "activations/layer19_attention_weight_min": -22.66469383239746,
      "activations/layer1_attention_weight_max": 17.0154972076416,
      "activations/layer1_attention_weight_min": -13.938387870788574,
      "activations/layer20_attention_weight_max": 24.58794593811035,
      "activations/layer20_attention_weight_min": -21.291147232055664,
      "activations/layer21_attention_weight_max": 37.92219924926758,
      "activations/layer21_attention_weight_min": -24.24713706970215,
      "activations/layer22_attention_weight_max": 31.74842071533203,
      "activations/layer22_attention_weight_min": -26.73787498474121,
      "activations/layer23_attention_weight_max": 41.669532775878906,
      "activations/layer23_attention_weight_min": -26.636608123779297,
      "activations/layer2_attention_weight_max": 34.5062255859375,
      "activations/layer2_attention_weight_min": -31.889408111572266,
      "activations/layer3_attention_weight_max": 98.91814422607422,
      "activations/layer3_attention_weight_min": -98.34404754638672,
      "activations/layer4_attention_weight_max": 100.57781219482422,
      "activations/layer4_attention_weight_min": -92.26386260986328,
      "activations/layer5_attention_weight_max": 74.39329528808594,
      "activations/layer5_attention_weight_min": -73.81640625,
      "activations/layer6_attention_weight_max": 52.75922393798828,
      "activations/layer6_attention_weight_min": -52.48503112792969,
      "activations/layer7_attention_weight_max": 78.9199447631836,
      "activations/layer7_attention_weight_min": -71.89130401611328,
      "activations/layer8_attention_weight_max": 56.512691497802734,
      "activations/layer8_attention_weight_min": -52.600372314453125,
      "activations/layer9_attention_weight_max": 66.22167205810547,
      "activations/layer9_attention_weight_min": -56.069366455078125,
      "epoch": 14.31,
      "learning_rate": 5.827840909090909e-05,
      "loss": 2.7653,
      "step": 246350
    },
    {
      "activations/layer0_attention_weight_max": 16.604389190673828,
      "activations/layer0_attention_weight_min": -14.826486587524414,
      "activations/layer10_attention_weight_max": 33.92570877075195,
      "activations/layer10_attention_weight_min": -33.40673828125,
      "activations/layer11_attention_weight_max": 31.853370666503906,
      "activations/layer11_attention_weight_min": -30.157194137573242,
      "activations/layer12_attention_weight_max": 17.835859298706055,
      "activations/layer12_attention_weight_min": -23.621854782104492,
      "activations/layer13_attention_weight_max": 34.3625602722168,
      "activations/layer13_attention_weight_min": -31.338512420654297,
      "activations/layer14_attention_weight_max": 31.749298095703125,
      "activations/layer14_attention_weight_min": -29.44142723083496,
      "activations/layer15_attention_weight_max": 30.72237777709961,
      "activations/layer15_attention_weight_min": -29.694246292114258,
      "activations/layer16_attention_weight_max": 30.187211990356445,
      "activations/layer16_attention_weight_min": -29.162717819213867,
      "activations/layer17_attention_weight_max": 50.86715316772461,
      "activations/layer17_attention_weight_min": -43.39122772216797,
      "activations/layer18_attention_weight_max": 41.2959098815918,
      "activations/layer18_attention_weight_min": -38.67423629760742,
      "activations/layer19_attention_weight_max": 19.827198028564453,
      "activations/layer19_attention_weight_min": -20.999767303466797,
      "activations/layer1_attention_weight_max": 17.008005142211914,
      "activations/layer1_attention_weight_min": -17.50419807434082,
      "activations/layer20_attention_weight_max": 20.736953735351562,
      "activations/layer20_attention_weight_min": -21.096454620361328,
      "activations/layer21_attention_weight_max": 27.275571823120117,
      "activations/layer21_attention_weight_min": -22.227792739868164,
      "activations/layer22_attention_weight_max": 27.830923080444336,
      "activations/layer22_attention_weight_min": -25.344453811645508,
      "activations/layer23_attention_weight_max": 32.72174835205078,
      "activations/layer23_attention_weight_min": -21.96121597290039,
      "activations/layer2_attention_weight_max": 33.387451171875,
      "activations/layer2_attention_weight_min": -32.21769332885742,
      "activations/layer3_attention_weight_max": 90.19923400878906,
      "activations/layer3_attention_weight_min": -88.62773895263672,
      "activations/layer4_attention_weight_max": 92.52942657470703,
      "activations/layer4_attention_weight_min": -86.00485229492188,
      "activations/layer5_attention_weight_max": 68.28577423095703,
      "activations/layer5_attention_weight_min": -75.04209899902344,
      "activations/layer6_attention_weight_max": 50.50082778930664,
      "activations/layer6_attention_weight_min": -50.94974136352539,
      "activations/layer7_attention_weight_max": 61.8264045715332,
      "activations/layer7_attention_weight_min": -60.728580474853516,
      "activations/layer8_attention_weight_max": 40.99821853637695,
      "activations/layer8_attention_weight_min": -44.674278259277344,
      "activations/layer9_attention_weight_max": 42.64213180541992,
      "activations/layer9_attention_weight_min": -42.679195404052734,
      "epoch": 14.32,
      "learning_rate": 5.825946969696969e-05,
      "loss": 2.7465,
      "step": 246400
    },
    {
      "activations/layer0_attention_weight_max": 15.462175369262695,
      "activations/layer0_attention_weight_min": -14.248189926147461,
      "activations/layer10_attention_weight_max": 37.07722091674805,
      "activations/layer10_attention_weight_min": -35.74500274658203,
      "activations/layer11_attention_weight_max": 36.051048278808594,
      "activations/layer11_attention_weight_min": -32.99348449707031,
      "activations/layer12_attention_weight_max": 18.8850040435791,
      "activations/layer12_attention_weight_min": -23.37028694152832,
      "activations/layer13_attention_weight_max": 36.830318450927734,
      "activations/layer13_attention_weight_min": -30.97203826904297,
      "activations/layer14_attention_weight_max": 37.92972183227539,
      "activations/layer14_attention_weight_min": -32.47705841064453,
      "activations/layer15_attention_weight_max": 34.9428825378418,
      "activations/layer15_attention_weight_min": -32.20979309082031,
      "activations/layer16_attention_weight_max": 36.14285659790039,
      "activations/layer16_attention_weight_min": -33.38174819946289,
      "activations/layer17_attention_weight_max": 54.64632034301758,
      "activations/layer17_attention_weight_min": -49.15045928955078,
      "activations/layer18_attention_weight_max": 55.0610237121582,
      "activations/layer18_attention_weight_min": -42.58048629760742,
      "activations/layer19_attention_weight_max": 24.77172088623047,
      "activations/layer19_attention_weight_min": -22.13563346862793,
      "activations/layer1_attention_weight_max": 17.77065086364746,
      "activations/layer1_attention_weight_min": -15.824766159057617,
      "activations/layer20_attention_weight_max": 22.848121643066406,
      "activations/layer20_attention_weight_min": -20.997526168823242,
      "activations/layer21_attention_weight_max": 37.14179229736328,
      "activations/layer21_attention_weight_min": -23.348440170288086,
      "activations/layer22_attention_weight_max": 30.587785720825195,
      "activations/layer22_attention_weight_min": -27.06842613220215,
      "activations/layer23_attention_weight_max": 41.34715270996094,
      "activations/layer23_attention_weight_min": -26.05709457397461,
      "activations/layer2_attention_weight_max": 30.258502960205078,
      "activations/layer2_attention_weight_min": -28.92664337158203,
      "activations/layer3_attention_weight_max": 90.102783203125,
      "activations/layer3_attention_weight_min": -90.89458465576172,
      "activations/layer4_attention_weight_max": 89.11681365966797,
      "activations/layer4_attention_weight_min": -84.509033203125,
      "activations/layer5_attention_weight_max": 71.4342269897461,
      "activations/layer5_attention_weight_min": -75.62376403808594,
      "activations/layer6_attention_weight_max": 49.0360221862793,
      "activations/layer6_attention_weight_min": -49.17972946166992,
      "activations/layer7_attention_weight_max": 64.49454498291016,
      "activations/layer7_attention_weight_min": -62.347408294677734,
      "activations/layer8_attention_weight_max": 43.94340133666992,
      "activations/layer8_attention_weight_min": -47.62885284423828,
      "activations/layer9_attention_weight_max": 45.04830551147461,
      "activations/layer9_attention_weight_min": -45.78715133666992,
      "epoch": 14.32,
      "learning_rate": 5.82405303030303e-05,
      "loss": 2.751,
      "step": 246450
    },
    {
      "activations/layer0_attention_weight_max": 15.6032075881958,
      "activations/layer0_attention_weight_min": -14.203827857971191,
      "activations/layer10_attention_weight_max": 44.4686164855957,
      "activations/layer10_attention_weight_min": -42.942081451416016,
      "activations/layer11_attention_weight_max": 40.38301086425781,
      "activations/layer11_attention_weight_min": -38.09648895263672,
      "activations/layer12_attention_weight_max": 21.20661735534668,
      "activations/layer12_attention_weight_min": -31.637109756469727,
      "activations/layer13_attention_weight_max": 42.51201248168945,
      "activations/layer13_attention_weight_min": -33.528648376464844,
      "activations/layer14_attention_weight_max": 65.25042724609375,
      "activations/layer14_attention_weight_min": -38.23827362060547,
      "activations/layer15_attention_weight_max": 54.014217376708984,
      "activations/layer15_attention_weight_min": -30.758710861206055,
      "activations/layer16_attention_weight_max": 38.674827575683594,
      "activations/layer16_attention_weight_min": -32.73406219482422,
      "activations/layer17_attention_weight_max": 60.39576721191406,
      "activations/layer17_attention_weight_min": -50.3719596862793,
      "activations/layer18_attention_weight_max": 55.973228454589844,
      "activations/layer18_attention_weight_min": -46.35403060913086,
      "activations/layer19_attention_weight_max": 27.954669952392578,
      "activations/layer19_attention_weight_min": -25.966827392578125,
      "activations/layer1_attention_weight_max": 16.20446014404297,
      "activations/layer1_attention_weight_min": -17.254404067993164,
      "activations/layer20_attention_weight_max": 31.954118728637695,
      "activations/layer20_attention_weight_min": -22.789073944091797,
      "activations/layer21_attention_weight_max": 47.31425094604492,
      "activations/layer21_attention_weight_min": -24.578649520874023,
      "activations/layer22_attention_weight_max": 36.08053207397461,
      "activations/layer22_attention_weight_min": -26.39160919189453,
      "activations/layer23_attention_weight_max": 38.975547790527344,
      "activations/layer23_attention_weight_min": -23.980030059814453,
      "activations/layer2_attention_weight_max": 33.442169189453125,
      "activations/layer2_attention_weight_min": -29.269344329833984,
      "activations/layer3_attention_weight_max": 97.3973388671875,
      "activations/layer3_attention_weight_min": -92.74031829833984,
      "activations/layer4_attention_weight_max": 88.13544464111328,
      "activations/layer4_attention_weight_min": -88.3239974975586,
      "activations/layer5_attention_weight_max": 67.38511657714844,
      "activations/layer5_attention_weight_min": -73.10629272460938,
      "activations/layer6_attention_weight_max": 51.86355972290039,
      "activations/layer6_attention_weight_min": -52.251792907714844,
      "activations/layer7_attention_weight_max": 69.85862731933594,
      "activations/layer7_attention_weight_min": -69.42311096191406,
      "activations/layer8_attention_weight_max": 51.727508544921875,
      "activations/layer8_attention_weight_min": -51.70296859741211,
      "activations/layer9_attention_weight_max": 58.6612663269043,
      "activations/layer9_attention_weight_min": -51.20906448364258,
      "epoch": 14.32,
      "learning_rate": 5.82215909090909e-05,
      "loss": 2.7683,
      "step": 246500
    },
    {
      "activations/layer0_attention_weight_max": 15.93720531463623,
      "activations/layer0_attention_weight_min": -14.694321632385254,
      "activations/layer10_attention_weight_max": 33.584442138671875,
      "activations/layer10_attention_weight_min": -35.186767578125,
      "activations/layer11_attention_weight_max": 29.775203704833984,
      "activations/layer11_attention_weight_min": -33.31727600097656,
      "activations/layer12_attention_weight_max": 18.446439743041992,
      "activations/layer12_attention_weight_min": -24.953643798828125,
      "activations/layer13_attention_weight_max": 33.86699295043945,
      "activations/layer13_attention_weight_min": -28.12765121459961,
      "activations/layer14_attention_weight_max": 38.73310470581055,
      "activations/layer14_attention_weight_min": -30.759639739990234,
      "activations/layer15_attention_weight_max": 36.05705261230469,
      "activations/layer15_attention_weight_min": -29.919525146484375,
      "activations/layer16_attention_weight_max": 33.29913330078125,
      "activations/layer16_attention_weight_min": -31.7171630859375,
      "activations/layer17_attention_weight_max": 52.61396789550781,
      "activations/layer17_attention_weight_min": -47.22787094116211,
      "activations/layer18_attention_weight_max": 47.495975494384766,
      "activations/layer18_attention_weight_min": -39.145469665527344,
      "activations/layer19_attention_weight_max": 22.407745361328125,
      "activations/layer19_attention_weight_min": -22.303882598876953,
      "activations/layer1_attention_weight_max": 16.747756958007812,
      "activations/layer1_attention_weight_min": -16.499467849731445,
      "activations/layer20_attention_weight_max": 24.92537498474121,
      "activations/layer20_attention_weight_min": -22.532808303833008,
      "activations/layer21_attention_weight_max": 35.32768249511719,
      "activations/layer21_attention_weight_min": -22.537918090820312,
      "activations/layer22_attention_weight_max": 29.843904495239258,
      "activations/layer22_attention_weight_min": -26.042417526245117,
      "activations/layer23_attention_weight_max": 35.89134979248047,
      "activations/layer23_attention_weight_min": -25.794797897338867,
      "activations/layer2_attention_weight_max": 31.00186538696289,
      "activations/layer2_attention_weight_min": -29.17078399658203,
      "activations/layer3_attention_weight_max": 90.42829895019531,
      "activations/layer3_attention_weight_min": -90.56989288330078,
      "activations/layer4_attention_weight_max": 89.46942138671875,
      "activations/layer4_attention_weight_min": -87.00153350830078,
      "activations/layer5_attention_weight_max": 67.87998962402344,
      "activations/layer5_attention_weight_min": -80.0417251586914,
      "activations/layer6_attention_weight_max": 49.55431365966797,
      "activations/layer6_attention_weight_min": -53.10670852661133,
      "activations/layer7_attention_weight_max": 65.43682098388672,
      "activations/layer7_attention_weight_min": -60.5659065246582,
      "activations/layer8_attention_weight_max": 43.34520721435547,
      "activations/layer8_attention_weight_min": -49.008968353271484,
      "activations/layer9_attention_weight_max": 42.42719650268555,
      "activations/layer9_attention_weight_min": -44.988197326660156,
      "epoch": 14.33,
      "learning_rate": 5.820265151515151e-05,
      "loss": 2.7509,
      "step": 246550
    },
    {
      "activations/layer0_attention_weight_max": 15.842676162719727,
      "activations/layer0_attention_weight_min": -13.980295181274414,
      "activations/layer10_attention_weight_max": 31.63280487060547,
      "activations/layer10_attention_weight_min": -33.27264404296875,
      "activations/layer11_attention_weight_max": 30.7923583984375,
      "activations/layer11_attention_weight_min": -32.797935485839844,
      "activations/layer12_attention_weight_max": 17.035167694091797,
      "activations/layer12_attention_weight_min": -24.18449592590332,
      "activations/layer13_attention_weight_max": 30.20025062561035,
      "activations/layer13_attention_weight_min": -26.987388610839844,
      "activations/layer14_attention_weight_max": 33.8636589050293,
      "activations/layer14_attention_weight_min": -28.689247131347656,
      "activations/layer15_attention_weight_max": 33.98044204711914,
      "activations/layer15_attention_weight_min": -29.028196334838867,
      "activations/layer16_attention_weight_max": 30.702884674072266,
      "activations/layer16_attention_weight_min": -29.30624771118164,
      "activations/layer17_attention_weight_max": 48.2656364440918,
      "activations/layer17_attention_weight_min": -43.616310119628906,
      "activations/layer18_attention_weight_max": 44.238037109375,
      "activations/layer18_attention_weight_min": -38.13323974609375,
      "activations/layer19_attention_weight_max": 21.076473236083984,
      "activations/layer19_attention_weight_min": -20.075536727905273,
      "activations/layer1_attention_weight_max": 15.706178665161133,
      "activations/layer1_attention_weight_min": -14.814215660095215,
      "activations/layer20_attention_weight_max": 22.243375778198242,
      "activations/layer20_attention_weight_min": -24.08028793334961,
      "activations/layer21_attention_weight_max": 33.36781692504883,
      "activations/layer21_attention_weight_min": -22.069442749023438,
      "activations/layer22_attention_weight_max": 26.15842056274414,
      "activations/layer22_attention_weight_min": -23.33881378173828,
      "activations/layer23_attention_weight_max": 37.881717681884766,
      "activations/layer23_attention_weight_min": -26.406797409057617,
      "activations/layer2_attention_weight_max": 30.678550720214844,
      "activations/layer2_attention_weight_min": -28.94331169128418,
      "activations/layer3_attention_weight_max": 87.30978393554688,
      "activations/layer3_attention_weight_min": -90.10757446289062,
      "activations/layer4_attention_weight_max": 87.91744232177734,
      "activations/layer4_attention_weight_min": -86.56067657470703,
      "activations/layer5_attention_weight_max": 67.59902954101562,
      "activations/layer5_attention_weight_min": -80.6510009765625,
      "activations/layer6_attention_weight_max": 46.90271759033203,
      "activations/layer6_attention_weight_min": -51.76559066772461,
      "activations/layer7_attention_weight_max": 60.94205093383789,
      "activations/layer7_attention_weight_min": -61.05406951904297,
      "activations/layer8_attention_weight_max": 40.663997650146484,
      "activations/layer8_attention_weight_min": -46.43721389770508,
      "activations/layer9_attention_weight_max": 41.88447189331055,
      "activations/layer9_attention_weight_min": -44.72154998779297,
      "epoch": 14.33,
      "learning_rate": 5.818371212121212e-05,
      "loss": 2.7693,
      "step": 246600
    },
    {
      "activations/layer0_attention_weight_max": 16.881492614746094,
      "activations/layer0_attention_weight_min": -14.490898132324219,
      "activations/layer10_attention_weight_max": 36.175392150878906,
      "activations/layer10_attention_weight_min": -35.13207244873047,
      "activations/layer11_attention_weight_max": 32.654815673828125,
      "activations/layer11_attention_weight_min": -33.27312469482422,
      "activations/layer12_attention_weight_max": 20.416776657104492,
      "activations/layer12_attention_weight_min": -24.118507385253906,
      "activations/layer13_attention_weight_max": 36.34074020385742,
      "activations/layer13_attention_weight_min": -29.557327270507812,
      "activations/layer14_attention_weight_max": 39.114646911621094,
      "activations/layer14_attention_weight_min": -31.704679489135742,
      "activations/layer15_attention_weight_max": 35.97959518432617,
      "activations/layer15_attention_weight_min": -31.81684112548828,
      "activations/layer16_attention_weight_max": 34.76028060913086,
      "activations/layer16_attention_weight_min": -31.34269905090332,
      "activations/layer17_attention_weight_max": 54.98087692260742,
      "activations/layer17_attention_weight_min": -45.6435661315918,
      "activations/layer18_attention_weight_max": 47.83962631225586,
      "activations/layer18_attention_weight_min": -38.76286697387695,
      "activations/layer19_attention_weight_max": 23.2252197265625,
      "activations/layer19_attention_weight_min": -22.98143196105957,
      "activations/layer1_attention_weight_max": 16.574962615966797,
      "activations/layer1_attention_weight_min": -17.049190521240234,
      "activations/layer20_attention_weight_max": 22.978134155273438,
      "activations/layer20_attention_weight_min": -21.488218307495117,
      "activations/layer21_attention_weight_max": 39.47005081176758,
      "activations/layer21_attention_weight_min": -24.988426208496094,
      "activations/layer22_attention_weight_max": 31.82213592529297,
      "activations/layer22_attention_weight_min": -26.48328399658203,
      "activations/layer23_attention_weight_max": 40.187049865722656,
      "activations/layer23_attention_weight_min": -25.771892547607422,
      "activations/layer2_attention_weight_max": 31.720312118530273,
      "activations/layer2_attention_weight_min": -30.672016143798828,
      "activations/layer3_attention_weight_max": 91.37382507324219,
      "activations/layer3_attention_weight_min": -99.39813995361328,
      "activations/layer4_attention_weight_max": 90.07560729980469,
      "activations/layer4_attention_weight_min": -91.46393585205078,
      "activations/layer5_attention_weight_max": 69.22552490234375,
      "activations/layer5_attention_weight_min": -78.29718017578125,
      "activations/layer6_attention_weight_max": 52.21288299560547,
      "activations/layer6_attention_weight_min": -52.65547180175781,
      "activations/layer7_attention_weight_max": 64.39891052246094,
      "activations/layer7_attention_weight_min": -63.679603576660156,
      "activations/layer8_attention_weight_max": 43.91984939575195,
      "activations/layer8_attention_weight_min": -47.814476013183594,
      "activations/layer9_attention_weight_max": 43.65966033935547,
      "activations/layer9_attention_weight_min": -47.94612503051758,
      "epoch": 14.33,
      "learning_rate": 5.816515151515151e-05,
      "loss": 2.7537,
      "step": 246650
    },
    {
      "activations/layer0_attention_weight_max": 16.626638412475586,
      "activations/layer0_attention_weight_min": -14.700017929077148,
      "activations/layer10_attention_weight_max": 35.005889892578125,
      "activations/layer10_attention_weight_min": -36.41226577758789,
      "activations/layer11_attention_weight_max": 33.3055419921875,
      "activations/layer11_attention_weight_min": -33.588008880615234,
      "activations/layer12_attention_weight_max": 19.087186813354492,
      "activations/layer12_attention_weight_min": -25.87976837158203,
      "activations/layer13_attention_weight_max": 32.202205657958984,
      "activations/layer13_attention_weight_min": -28.47959327697754,
      "activations/layer14_attention_weight_max": 33.53553771972656,
      "activations/layer14_attention_weight_min": -30.249313354492188,
      "activations/layer15_attention_weight_max": 32.57176971435547,
      "activations/layer15_attention_weight_min": -28.879714965820312,
      "activations/layer16_attention_weight_max": 38.64618682861328,
      "activations/layer16_attention_weight_min": -29.769731521606445,
      "activations/layer17_attention_weight_max": 52.837223052978516,
      "activations/layer17_attention_weight_min": -41.6600227355957,
      "activations/layer18_attention_weight_max": 44.292724609375,
      "activations/layer18_attention_weight_min": -37.230403900146484,
      "activations/layer19_attention_weight_max": 20.544389724731445,
      "activations/layer19_attention_weight_min": -20.1199893951416,
      "activations/layer1_attention_weight_max": 16.550058364868164,
      "activations/layer1_attention_weight_min": -16.0662784576416,
      "activations/layer20_attention_weight_max": 20.967500686645508,
      "activations/layer20_attention_weight_min": -21.555082321166992,
      "activations/layer21_attention_weight_max": 35.44200134277344,
      "activations/layer21_attention_weight_min": -22.471960067749023,
      "activations/layer22_attention_weight_max": 32.22956085205078,
      "activations/layer22_attention_weight_min": -25.5062313079834,
      "activations/layer23_attention_weight_max": 44.15705871582031,
      "activations/layer23_attention_weight_min": -23.559097290039062,
      "activations/layer2_attention_weight_max": 32.10982131958008,
      "activations/layer2_attention_weight_min": -29.832874298095703,
      "activations/layer3_attention_weight_max": 92.98751068115234,
      "activations/layer3_attention_weight_min": -96.50768280029297,
      "activations/layer4_attention_weight_max": 89.42296600341797,
      "activations/layer4_attention_weight_min": -88.79346466064453,
      "activations/layer5_attention_weight_max": 68.81138610839844,
      "activations/layer5_attention_weight_min": -77.00350952148438,
      "activations/layer6_attention_weight_max": 49.169490814208984,
      "activations/layer6_attention_weight_min": -49.8983268737793,
      "activations/layer7_attention_weight_max": 64.39175415039062,
      "activations/layer7_attention_weight_min": -63.98733139038086,
      "activations/layer8_attention_weight_max": 45.225311279296875,
      "activations/layer8_attention_weight_min": -50.12357711791992,
      "activations/layer9_attention_weight_max": 47.10832214355469,
      "activations/layer9_attention_weight_min": -50.96992111206055,
      "epoch": 14.33,
      "learning_rate": 5.814621212121211e-05,
      "loss": 2.772,
      "step": 246700
    },
    {
      "activations/layer0_attention_weight_max": 16.417293548583984,
      "activations/layer0_attention_weight_min": -14.667719841003418,
      "activations/layer10_attention_weight_max": 34.93886184692383,
      "activations/layer10_attention_weight_min": -35.47414016723633,
      "activations/layer11_attention_weight_max": 32.35909652709961,
      "activations/layer11_attention_weight_min": -33.69892883300781,
      "activations/layer12_attention_weight_max": 18.87637710571289,
      "activations/layer12_attention_weight_min": -23.930774688720703,
      "activations/layer13_attention_weight_max": 37.115325927734375,
      "activations/layer13_attention_weight_min": -29.990610122680664,
      "activations/layer14_attention_weight_max": 36.92862319946289,
      "activations/layer14_attention_weight_min": -30.808734893798828,
      "activations/layer15_attention_weight_max": 32.867366790771484,
      "activations/layer15_attention_weight_min": -29.43427276611328,
      "activations/layer16_attention_weight_max": 31.118131637573242,
      "activations/layer16_attention_weight_min": -30.47349739074707,
      "activations/layer17_attention_weight_max": 51.09938049316406,
      "activations/layer17_attention_weight_min": -43.79851150512695,
      "activations/layer18_attention_weight_max": 48.26333999633789,
      "activations/layer18_attention_weight_min": -37.35770034790039,
      "activations/layer19_attention_weight_max": 22.422435760498047,
      "activations/layer19_attention_weight_min": -21.195510864257812,
      "activations/layer1_attention_weight_max": 16.91802215576172,
      "activations/layer1_attention_weight_min": -16.02593231201172,
      "activations/layer20_attention_weight_max": 23.729270935058594,
      "activations/layer20_attention_weight_min": -24.33154296875,
      "activations/layer21_attention_weight_max": 37.148048400878906,
      "activations/layer21_attention_weight_min": -24.174041748046875,
      "activations/layer22_attention_weight_max": 28.581085205078125,
      "activations/layer22_attention_weight_min": -25.150009155273438,
      "activations/layer23_attention_weight_max": 37.881866455078125,
      "activations/layer23_attention_weight_min": -25.557119369506836,
      "activations/layer2_attention_weight_max": 35.09911346435547,
      "activations/layer2_attention_weight_min": -31.843765258789062,
      "activations/layer3_attention_weight_max": 96.43486785888672,
      "activations/layer3_attention_weight_min": -96.08956909179688,
      "activations/layer4_attention_weight_max": 93.93675231933594,
      "activations/layer4_attention_weight_min": -89.24056243896484,
      "activations/layer5_attention_weight_max": 68.85892486572266,
      "activations/layer5_attention_weight_min": -77.95439910888672,
      "activations/layer6_attention_weight_max": 52.43756866455078,
      "activations/layer6_attention_weight_min": -53.171226501464844,
      "activations/layer7_attention_weight_max": 63.072776794433594,
      "activations/layer7_attention_weight_min": -66.36432647705078,
      "activations/layer8_attention_weight_max": 45.660911560058594,
      "activations/layer8_attention_weight_min": -46.680274963378906,
      "activations/layer9_attention_weight_max": 44.88996505737305,
      "activations/layer9_attention_weight_min": -47.39155578613281,
      "epoch": 14.34,
      "learning_rate": 5.8127272727272726e-05,
      "loss": 2.7678,
      "step": 246750
    },
    {
      "activations/layer0_attention_weight_max": 15.734637260437012,
      "activations/layer0_attention_weight_min": -13.949418067932129,
      "activations/layer10_attention_weight_max": 37.741355895996094,
      "activations/layer10_attention_weight_min": -35.95504379272461,
      "activations/layer11_attention_weight_max": 33.437744140625,
      "activations/layer11_attention_weight_min": -33.44205093383789,
      "activations/layer12_attention_weight_max": 18.60767936706543,
      "activations/layer12_attention_weight_min": -27.096542358398438,
      "activations/layer13_attention_weight_max": 40.01078414916992,
      "activations/layer13_attention_weight_min": -27.46393394470215,
      "activations/layer14_attention_weight_max": 37.533348083496094,
      "activations/layer14_attention_weight_min": -32.04079055786133,
      "activations/layer15_attention_weight_max": 33.57497787475586,
      "activations/layer15_attention_weight_min": -30.534032821655273,
      "activations/layer16_attention_weight_max": 33.35364532470703,
      "activations/layer16_attention_weight_min": -30.933752059936523,
      "activations/layer17_attention_weight_max": 51.81950759887695,
      "activations/layer17_attention_weight_min": -44.01717758178711,
      "activations/layer18_attention_weight_max": 44.76319122314453,
      "activations/layer18_attention_weight_min": -37.278900146484375,
      "activations/layer19_attention_weight_max": 21.348115921020508,
      "activations/layer19_attention_weight_min": -21.66385841369629,
      "activations/layer1_attention_weight_max": 16.991012573242188,
      "activations/layer1_attention_weight_min": -16.5540714263916,
      "activations/layer20_attention_weight_max": 23.27582550048828,
      "activations/layer20_attention_weight_min": -21.76051902770996,
      "activations/layer21_attention_weight_max": 40.494873046875,
      "activations/layer21_attention_weight_min": -22.1064453125,
      "activations/layer22_attention_weight_max": 28.489643096923828,
      "activations/layer22_attention_weight_min": -26.570072174072266,
      "activations/layer23_attention_weight_max": 39.53513717651367,
      "activations/layer23_attention_weight_min": -26.006994247436523,
      "activations/layer2_attention_weight_max": 32.15748596191406,
      "activations/layer2_attention_weight_min": -30.549396514892578,
      "activations/layer3_attention_weight_max": 91.98919677734375,
      "activations/layer3_attention_weight_min": -90.70148468017578,
      "activations/layer4_attention_weight_max": 91.36540985107422,
      "activations/layer4_attention_weight_min": -91.48772430419922,
      "activations/layer5_attention_weight_max": 72.52664947509766,
      "activations/layer5_attention_weight_min": -79.65292358398438,
      "activations/layer6_attention_weight_max": 51.9549446105957,
      "activations/layer6_attention_weight_min": -55.39719772338867,
      "activations/layer7_attention_weight_max": 68.3157958984375,
      "activations/layer7_attention_weight_min": -64.24291229248047,
      "activations/layer8_attention_weight_max": 46.14745330810547,
      "activations/layer8_attention_weight_min": -49.05336380004883,
      "activations/layer9_attention_weight_max": 45.21665573120117,
      "activations/layer9_attention_weight_min": -45.51252746582031,
      "epoch": 14.34,
      "learning_rate": 5.810833333333333e-05,
      "loss": 2.7686,
      "step": 246800
    },
    {
      "activations/layer0_attention_weight_max": 15.958063125610352,
      "activations/layer0_attention_weight_min": -14.07660961151123,
      "activations/layer10_attention_weight_max": 41.751708984375,
      "activations/layer10_attention_weight_min": -38.915435791015625,
      "activations/layer11_attention_weight_max": 34.00206756591797,
      "activations/layer11_attention_weight_min": -33.38941192626953,
      "activations/layer12_attention_weight_max": 18.234949111938477,
      "activations/layer12_attention_weight_min": -23.728513717651367,
      "activations/layer13_attention_weight_max": 38.86128616333008,
      "activations/layer13_attention_weight_min": -32.15790939331055,
      "activations/layer14_attention_weight_max": 36.487762451171875,
      "activations/layer14_attention_weight_min": -30.95140838623047,
      "activations/layer15_attention_weight_max": 34.972251892089844,
      "activations/layer15_attention_weight_min": -31.819875717163086,
      "activations/layer16_attention_weight_max": 32.76813888549805,
      "activations/layer16_attention_weight_min": -31.661054611206055,
      "activations/layer17_attention_weight_max": 53.6683235168457,
      "activations/layer17_attention_weight_min": -46.10493850708008,
      "activations/layer18_attention_weight_max": 49.21005630493164,
      "activations/layer18_attention_weight_min": -39.0942497253418,
      "activations/layer19_attention_weight_max": 23.34833335876465,
      "activations/layer19_attention_weight_min": -21.4510440826416,
      "activations/layer1_attention_weight_max": 15.901901245117188,
      "activations/layer1_attention_weight_min": -17.503355026245117,
      "activations/layer20_attention_weight_max": 22.29330062866211,
      "activations/layer20_attention_weight_min": -20.681095123291016,
      "activations/layer21_attention_weight_max": 32.889495849609375,
      "activations/layer21_attention_weight_min": -23.482206344604492,
      "activations/layer22_attention_weight_max": 30.605873107910156,
      "activations/layer22_attention_weight_min": -27.16051483154297,
      "activations/layer23_attention_weight_max": 38.66642761230469,
      "activations/layer23_attention_weight_min": -23.857711791992188,
      "activations/layer2_attention_weight_max": 33.67945098876953,
      "activations/layer2_attention_weight_min": -31.51983642578125,
      "activations/layer3_attention_weight_max": 102.71558380126953,
      "activations/layer3_attention_weight_min": -99.85932159423828,
      "activations/layer4_attention_weight_max": 99.62115478515625,
      "activations/layer4_attention_weight_min": -99.84807586669922,
      "activations/layer5_attention_weight_max": 74.03607177734375,
      "activations/layer5_attention_weight_min": -78.12969207763672,
      "activations/layer6_attention_weight_max": 56.95942687988281,
      "activations/layer6_attention_weight_min": -60.250389099121094,
      "activations/layer7_attention_weight_max": 66.3825454711914,
      "activations/layer7_attention_weight_min": -66.91661834716797,
      "activations/layer8_attention_weight_max": 50.127559661865234,
      "activations/layer8_attention_weight_min": -52.55009078979492,
      "activations/layer9_attention_weight_max": 52.4720458984375,
      "activations/layer9_attention_weight_min": -49.80603790283203,
      "epoch": 14.34,
      "learning_rate": 5.8089393939393936e-05,
      "loss": 2.7602,
      "step": 246850
    },
    {
      "activations/layer0_attention_weight_max": 15.611969947814941,
      "activations/layer0_attention_weight_min": -14.171345710754395,
      "activations/layer10_attention_weight_max": 36.983585357666016,
      "activations/layer10_attention_weight_min": -36.003963470458984,
      "activations/layer11_attention_weight_max": 33.19390106201172,
      "activations/layer11_attention_weight_min": -35.17030334472656,
      "activations/layer12_attention_weight_max": 16.95561981201172,
      "activations/layer12_attention_weight_min": -27.498531341552734,
      "activations/layer13_attention_weight_max": 34.840999603271484,
      "activations/layer13_attention_weight_min": -26.653820037841797,
      "activations/layer14_attention_weight_max": 36.78968048095703,
      "activations/layer14_attention_weight_min": -31.67337989807129,
      "activations/layer15_attention_weight_max": 34.29148864746094,
      "activations/layer15_attention_weight_min": -30.039222717285156,
      "activations/layer16_attention_weight_max": 29.655542373657227,
      "activations/layer16_attention_weight_min": -30.786706924438477,
      "activations/layer17_attention_weight_max": 51.287967681884766,
      "activations/layer17_attention_weight_min": -43.1390266418457,
      "activations/layer18_attention_weight_max": 44.713706970214844,
      "activations/layer18_attention_weight_min": -38.053348541259766,
      "activations/layer19_attention_weight_max": 22.369815826416016,
      "activations/layer19_attention_weight_min": -23.552091598510742,
      "activations/layer1_attention_weight_max": 16.13673210144043,
      "activations/layer1_attention_weight_min": -17.213109970092773,
      "activations/layer20_attention_weight_max": 22.18792724609375,
      "activations/layer20_attention_weight_min": -21.41203498840332,
      "activations/layer21_attention_weight_max": 35.84244918823242,
      "activations/layer21_attention_weight_min": -25.45090675354004,
      "activations/layer22_attention_weight_max": 31.169879913330078,
      "activations/layer22_attention_weight_min": -24.81903839111328,
      "activations/layer23_attention_weight_max": 39.14622497558594,
      "activations/layer23_attention_weight_min": -24.264326095581055,
      "activations/layer2_attention_weight_max": 31.22133445739746,
      "activations/layer2_attention_weight_min": -30.78002166748047,
      "activations/layer3_attention_weight_max": 93.28618621826172,
      "activations/layer3_attention_weight_min": -98.04376983642578,
      "activations/layer4_attention_weight_max": 93.35962677001953,
      "activations/layer4_attention_weight_min": -90.66598510742188,
      "activations/layer5_attention_weight_max": 70.38153076171875,
      "activations/layer5_attention_weight_min": -75.68719482421875,
      "activations/layer6_attention_weight_max": 49.758846282958984,
      "activations/layer6_attention_weight_min": -52.12027359008789,
      "activations/layer7_attention_weight_max": 66.81119537353516,
      "activations/layer7_attention_weight_min": -61.04011535644531,
      "activations/layer8_attention_weight_max": 44.77430725097656,
      "activations/layer8_attention_weight_min": -45.97919845581055,
      "activations/layer9_attention_weight_max": 46.03482437133789,
      "activations/layer9_attention_weight_min": -44.14505386352539,
      "epoch": 14.35,
      "learning_rate": 5.807045454545454e-05,
      "loss": 2.7472,
      "step": 246900
    },
    {
      "activations/layer0_attention_weight_max": 15.446582794189453,
      "activations/layer0_attention_weight_min": -14.271235466003418,
      "activations/layer10_attention_weight_max": 37.49788284301758,
      "activations/layer10_attention_weight_min": -38.30653381347656,
      "activations/layer11_attention_weight_max": 36.07130432128906,
      "activations/layer11_attention_weight_min": -35.381263732910156,
      "activations/layer12_attention_weight_max": 19.367053985595703,
      "activations/layer12_attention_weight_min": -27.110305786132812,
      "activations/layer13_attention_weight_max": 33.475975036621094,
      "activations/layer13_attention_weight_min": -30.058177947998047,
      "activations/layer14_attention_weight_max": 37.01350784301758,
      "activations/layer14_attention_weight_min": -29.672760009765625,
      "activations/layer15_attention_weight_max": 37.15897750854492,
      "activations/layer15_attention_weight_min": -30.43889045715332,
      "activations/layer16_attention_weight_max": 33.61093521118164,
      "activations/layer16_attention_weight_min": -32.714847564697266,
      "activations/layer17_attention_weight_max": 52.78606414794922,
      "activations/layer17_attention_weight_min": -45.94367980957031,
      "activations/layer18_attention_weight_max": 51.50635528564453,
      "activations/layer18_attention_weight_min": -39.292781829833984,
      "activations/layer19_attention_weight_max": 23.10392951965332,
      "activations/layer19_attention_weight_min": -22.81263542175293,
      "activations/layer1_attention_weight_max": 17.067554473876953,
      "activations/layer1_attention_weight_min": -15.086487770080566,
      "activations/layer20_attention_weight_max": 21.96930694580078,
      "activations/layer20_attention_weight_min": -25.01152992248535,
      "activations/layer21_attention_weight_max": 35.58484649658203,
      "activations/layer21_attention_weight_min": -21.776033401489258,
      "activations/layer22_attention_weight_max": 30.508975982666016,
      "activations/layer22_attention_weight_min": -29.222084045410156,
      "activations/layer23_attention_weight_max": 37.49530029296875,
      "activations/layer23_attention_weight_min": -23.836620330810547,
      "activations/layer2_attention_weight_max": 35.935211181640625,
      "activations/layer2_attention_weight_min": -34.63108825683594,
      "activations/layer3_attention_weight_max": 101.05059051513672,
      "activations/layer3_attention_weight_min": -103.15692138671875,
      "activations/layer4_attention_weight_max": 98.93141174316406,
      "activations/layer4_attention_weight_min": -98.7325439453125,
      "activations/layer5_attention_weight_max": 76.34274291992188,
      "activations/layer5_attention_weight_min": -82.58027648925781,
      "activations/layer6_attention_weight_max": 56.22393798828125,
      "activations/layer6_attention_weight_min": -56.68747329711914,
      "activations/layer7_attention_weight_max": 66.2525863647461,
      "activations/layer7_attention_weight_min": -68.28047180175781,
      "activations/layer8_attention_weight_max": 51.7173957824707,
      "activations/layer8_attention_weight_min": -49.9844970703125,
      "activations/layer9_attention_weight_max": 53.995643615722656,
      "activations/layer9_attention_weight_min": -50.2344856262207,
      "epoch": 14.35,
      "learning_rate": 5.805151515151515e-05,
      "loss": 2.7429,
      "step": 246950
    },
    {
      "activations/layer0_attention_weight_max": 16.015827178955078,
      "activations/layer0_attention_weight_min": -14.174040794372559,
      "activations/layer10_attention_weight_max": 36.68672180175781,
      "activations/layer10_attention_weight_min": -38.314205169677734,
      "activations/layer11_attention_weight_max": 35.61821746826172,
      "activations/layer11_attention_weight_min": -33.90011215209961,
      "activations/layer12_attention_weight_max": 18.584491729736328,
      "activations/layer12_attention_weight_min": -26.057540893554688,
      "activations/layer13_attention_weight_max": 43.89611053466797,
      "activations/layer13_attention_weight_min": -28.7424259185791,
      "activations/layer14_attention_weight_max": 46.23527145385742,
      "activations/layer14_attention_weight_min": -33.9136848449707,
      "activations/layer15_attention_weight_max": 37.98045349121094,
      "activations/layer15_attention_weight_min": -33.13086700439453,
      "activations/layer16_attention_weight_max": 35.64291000366211,
      "activations/layer16_attention_weight_min": -33.20161056518555,
      "activations/layer17_attention_weight_max": 55.30610656738281,
      "activations/layer17_attention_weight_min": -49.650753021240234,
      "activations/layer18_attention_weight_max": 52.993839263916016,
      "activations/layer18_attention_weight_min": -42.493648529052734,
      "activations/layer19_attention_weight_max": 23.4917049407959,
      "activations/layer19_attention_weight_min": -25.305330276489258,
      "activations/layer1_attention_weight_max": 17.682249069213867,
      "activations/layer1_attention_weight_min": -14.954575538635254,
      "activations/layer20_attention_weight_max": 26.895219802856445,
      "activations/layer20_attention_weight_min": -21.88951301574707,
      "activations/layer21_attention_weight_max": 42.97259521484375,
      "activations/layer21_attention_weight_min": -24.623180389404297,
      "activations/layer22_attention_weight_max": 32.222190856933594,
      "activations/layer22_attention_weight_min": -25.93083953857422,
      "activations/layer23_attention_weight_max": 39.82008361816406,
      "activations/layer23_attention_weight_min": -24.803958892822266,
      "activations/layer2_attention_weight_max": 33.367889404296875,
      "activations/layer2_attention_weight_min": -31.32294273376465,
      "activations/layer3_attention_weight_max": 94.69207000732422,
      "activations/layer3_attention_weight_min": -94.72998046875,
      "activations/layer4_attention_weight_max": 92.97430419921875,
      "activations/layer4_attention_weight_min": -86.7463607788086,
      "activations/layer5_attention_weight_max": 68.29094696044922,
      "activations/layer5_attention_weight_min": -73.55810546875,
      "activations/layer6_attention_weight_max": 52.03632736206055,
      "activations/layer6_attention_weight_min": -51.33043670654297,
      "activations/layer7_attention_weight_max": 61.08449172973633,
      "activations/layer7_attention_weight_min": -65.8694839477539,
      "activations/layer8_attention_weight_max": 46.73020935058594,
      "activations/layer8_attention_weight_min": -47.86146926879883,
      "activations/layer9_attention_weight_max": 47.83042526245117,
      "activations/layer9_attention_weight_min": -45.70214080810547,
      "epoch": 14.35,
      "learning_rate": 5.8032575757575754e-05,
      "loss": 2.7609,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4828,
      "eval_samples_per_second": 506.202,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4828,
      "eval_openwebtext_samples_per_second": 506.202,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9442,
      "eval_wikitext_samples_per_second": 234.541,
      "step": 247000
    },
    {
      "epoch": 14.35,
      "eval_lambada_loss": 2.564453125,
      "eval_lambada_ppl": 12.993550578338562,
      "eval_lambada_runtime": 9.5155,
      "eval_lambada_samples_per_second": 511.69,
      "step": 247000
    },
    {
      "activations/layer0_attention_weight_max": 15.586259841918945,
      "activations/layer0_attention_weight_min": -14.695982933044434,
      "activations/layer10_attention_weight_max": 36.358436584472656,
      "activations/layer10_attention_weight_min": -34.186458587646484,
      "activations/layer11_attention_weight_max": 32.64487075805664,
      "activations/layer11_attention_weight_min": -34.162742614746094,
      "activations/layer12_attention_weight_max": 18.644018173217773,
      "activations/layer12_attention_weight_min": -29.66332244873047,
      "activations/layer13_attention_weight_max": 33.883056640625,
      "activations/layer13_attention_weight_min": -30.23830223083496,
      "activations/layer14_attention_weight_max": 34.187862396240234,
      "activations/layer14_attention_weight_min": -29.15515899658203,
      "activations/layer15_attention_weight_max": 33.374794006347656,
      "activations/layer15_attention_weight_min": -27.488895416259766,
      "activations/layer16_attention_weight_max": 30.732860565185547,
      "activations/layer16_attention_weight_min": -29.52783966064453,
      "activations/layer17_attention_weight_max": 51.86471176147461,
      "activations/layer17_attention_weight_min": -44.98909378051758,
      "activations/layer18_attention_weight_max": 46.1987190246582,
      "activations/layer18_attention_weight_min": -37.56011962890625,
      "activations/layer19_attention_weight_max": 21.516868591308594,
      "activations/layer19_attention_weight_min": -21.493711471557617,
      "activations/layer1_attention_weight_max": 16.93338966369629,
      "activations/layer1_attention_weight_min": -15.676668167114258,
      "activations/layer20_attention_weight_max": 21.051145553588867,
      "activations/layer20_attention_weight_min": -23.841562271118164,
      "activations/layer21_attention_weight_max": 36.34423828125,
      "activations/layer21_attention_weight_min": -24.105161666870117,
      "activations/layer22_attention_weight_max": 28.24016761779785,
      "activations/layer22_attention_weight_min": -25.76874542236328,
      "activations/layer23_attention_weight_max": 37.832481384277344,
      "activations/layer23_attention_weight_min": -24.852569580078125,
      "activations/layer2_attention_weight_max": 32.89326095581055,
      "activations/layer2_attention_weight_min": -30.750751495361328,
      "activations/layer3_attention_weight_max": 92.55575561523438,
      "activations/layer3_attention_weight_min": -97.72814178466797,
      "activations/layer4_attention_weight_max": 91.17037963867188,
      "activations/layer4_attention_weight_min": -89.968505859375,
      "activations/layer5_attention_weight_max": 71.17354583740234,
      "activations/layer5_attention_weight_min": -77.33213806152344,
      "activations/layer6_attention_weight_max": 50.430118560791016,
      "activations/layer6_attention_weight_min": -53.838294982910156,
      "activations/layer7_attention_weight_max": 62.80060958862305,
      "activations/layer7_attention_weight_min": -62.68625259399414,
      "activations/layer8_attention_weight_max": 44.1901969909668,
      "activations/layer8_attention_weight_min": -45.92897415161133,
      "activations/layer9_attention_weight_max": 42.43523406982422,
      "activations/layer9_attention_weight_min": -45.823944091796875,
      "epoch": 14.36,
      "learning_rate": 5.8013636363636355e-05,
      "loss": 2.7429,
      "step": 247050
    },
    {
      "activations/layer0_attention_weight_max": 15.8525390625,
      "activations/layer0_attention_weight_min": -14.686463356018066,
      "activations/layer10_attention_weight_max": 35.272552490234375,
      "activations/layer10_attention_weight_min": -35.90324783325195,
      "activations/layer11_attention_weight_max": 31.93829345703125,
      "activations/layer11_attention_weight_min": -32.65707015991211,
      "activations/layer12_attention_weight_max": 17.194686889648438,
      "activations/layer12_attention_weight_min": -25.374475479125977,
      "activations/layer13_attention_weight_max": 35.851505279541016,
      "activations/layer13_attention_weight_min": -28.84881019592285,
      "activations/layer14_attention_weight_max": 33.10932540893555,
      "activations/layer14_attention_weight_min": -31.371442794799805,
      "activations/layer15_attention_weight_max": 30.506860733032227,
      "activations/layer15_attention_weight_min": -28.85468101501465,
      "activations/layer16_attention_weight_max": 30.104053497314453,
      "activations/layer16_attention_weight_min": -29.264480590820312,
      "activations/layer17_attention_weight_max": 47.828956604003906,
      "activations/layer17_attention_weight_min": -47.69527053833008,
      "activations/layer18_attention_weight_max": 45.47281265258789,
      "activations/layer18_attention_weight_min": -41.480255126953125,
      "activations/layer19_attention_weight_max": 21.084529876708984,
      "activations/layer19_attention_weight_min": -22.266061782836914,
      "activations/layer1_attention_weight_max": 16.746824264526367,
      "activations/layer1_attention_weight_min": -15.14601993560791,
      "activations/layer20_attention_weight_max": 20.421192169189453,
      "activations/layer20_attention_weight_min": -20.734140396118164,
      "activations/layer21_attention_weight_max": 35.04275894165039,
      "activations/layer21_attention_weight_min": -21.431442260742188,
      "activations/layer22_attention_weight_max": 27.973909378051758,
      "activations/layer22_attention_weight_min": -29.23597526550293,
      "activations/layer23_attention_weight_max": 34.10392761230469,
      "activations/layer23_attention_weight_min": -24.379505157470703,
      "activations/layer2_attention_weight_max": 33.86202621459961,
      "activations/layer2_attention_weight_min": -30.20022964477539,
      "activations/layer3_attention_weight_max": 96.96028137207031,
      "activations/layer3_attention_weight_min": -94.70381164550781,
      "activations/layer4_attention_weight_max": 96.33771514892578,
      "activations/layer4_attention_weight_min": -91.78108978271484,
      "activations/layer5_attention_weight_max": 69.32814025878906,
      "activations/layer5_attention_weight_min": -77.45803833007812,
      "activations/layer6_attention_weight_max": 52.85616683959961,
      "activations/layer6_attention_weight_min": -51.218101501464844,
      "activations/layer7_attention_weight_max": 66.10610961914062,
      "activations/layer7_attention_weight_min": -64.31281280517578,
      "activations/layer8_attention_weight_max": 46.13053894042969,
      "activations/layer8_attention_weight_min": -46.47169876098633,
      "activations/layer9_attention_weight_max": 45.50438690185547,
      "activations/layer9_attention_weight_min": -46.43268966674805,
      "epoch": 14.36,
      "learning_rate": 5.799469696969697e-05,
      "loss": 2.7588,
      "step": 247100
    },
    {
      "activations/layer0_attention_weight_max": 16.491727828979492,
      "activations/layer0_attention_weight_min": -14.136289596557617,
      "activations/layer10_attention_weight_max": 33.660614013671875,
      "activations/layer10_attention_weight_min": -32.974361419677734,
      "activations/layer11_attention_weight_max": 29.468351364135742,
      "activations/layer11_attention_weight_min": -30.31570816040039,
      "activations/layer12_attention_weight_max": 16.32233238220215,
      "activations/layer12_attention_weight_min": -24.650753021240234,
      "activations/layer13_attention_weight_max": 39.7695426940918,
      "activations/layer13_attention_weight_min": -30.860429763793945,
      "activations/layer14_attention_weight_max": 34.0930290222168,
      "activations/layer14_attention_weight_min": -29.128931045532227,
      "activations/layer15_attention_weight_max": 32.58803939819336,
      "activations/layer15_attention_weight_min": -28.95774269104004,
      "activations/layer16_attention_weight_max": 31.15416717529297,
      "activations/layer16_attention_weight_min": -29.67481803894043,
      "activations/layer17_attention_weight_max": 49.46418380737305,
      "activations/layer17_attention_weight_min": -43.55802917480469,
      "activations/layer18_attention_weight_max": 46.39247512817383,
      "activations/layer18_attention_weight_min": -36.89377975463867,
      "activations/layer19_attention_weight_max": 22.385665893554688,
      "activations/layer19_attention_weight_min": -22.607765197753906,
      "activations/layer1_attention_weight_max": 16.189321517944336,
      "activations/layer1_attention_weight_min": -13.219728469848633,
      "activations/layer20_attention_weight_max": 22.8112735748291,
      "activations/layer20_attention_weight_min": -22.258663177490234,
      "activations/layer21_attention_weight_max": 40.150291442871094,
      "activations/layer21_attention_weight_min": -25.48307228088379,
      "activations/layer22_attention_weight_max": 32.8677864074707,
      "activations/layer22_attention_weight_min": -25.3322696685791,
      "activations/layer23_attention_weight_max": 42.267967224121094,
      "activations/layer23_attention_weight_min": -26.18099021911621,
      "activations/layer2_attention_weight_max": 31.499271392822266,
      "activations/layer2_attention_weight_min": -29.325489044189453,
      "activations/layer3_attention_weight_max": 91.4156265258789,
      "activations/layer3_attention_weight_min": -100.31329345703125,
      "activations/layer4_attention_weight_max": 86.82968139648438,
      "activations/layer4_attention_weight_min": -88.30497741699219,
      "activations/layer5_attention_weight_max": 66.96003723144531,
      "activations/layer5_attention_weight_min": -79.23460388183594,
      "activations/layer6_attention_weight_max": 50.43146896362305,
      "activations/layer6_attention_weight_min": -49.884342193603516,
      "activations/layer7_attention_weight_max": 62.359622955322266,
      "activations/layer7_attention_weight_min": -64.45686340332031,
      "activations/layer8_attention_weight_max": 42.449546813964844,
      "activations/layer8_attention_weight_min": -42.24874496459961,
      "activations/layer9_attention_weight_max": 43.411617279052734,
      "activations/layer9_attention_weight_min": -44.64456558227539,
      "epoch": 14.36,
      "learning_rate": 5.797575757575757e-05,
      "loss": 2.7678,
      "step": 247150
    },
    {
      "activations/layer0_attention_weight_max": 15.914796829223633,
      "activations/layer0_attention_weight_min": -14.068260192871094,
      "activations/layer10_attention_weight_max": 44.2511100769043,
      "activations/layer10_attention_weight_min": -42.7539176940918,
      "activations/layer11_attention_weight_max": 42.835060119628906,
      "activations/layer11_attention_weight_min": -41.08335876464844,
      "activations/layer12_attention_weight_max": 19.161653518676758,
      "activations/layer12_attention_weight_min": -26.81558609008789,
      "activations/layer13_attention_weight_max": 37.51558303833008,
      "activations/layer13_attention_weight_min": -32.332550048828125,
      "activations/layer14_attention_weight_max": 35.204750061035156,
      "activations/layer14_attention_weight_min": -31.481521606445312,
      "activations/layer15_attention_weight_max": 33.28525924682617,
      "activations/layer15_attention_weight_min": -30.611024856567383,
      "activations/layer16_attention_weight_max": 32.65303039550781,
      "activations/layer16_attention_weight_min": -30.440698623657227,
      "activations/layer17_attention_weight_max": 52.284584045410156,
      "activations/layer17_attention_weight_min": -47.46539306640625,
      "activations/layer18_attention_weight_max": 46.75556564331055,
      "activations/layer18_attention_weight_min": -40.598365783691406,
      "activations/layer19_attention_weight_max": 24.736875534057617,
      "activations/layer19_attention_weight_min": -23.15009880065918,
      "activations/layer1_attention_weight_max": 16.697092056274414,
      "activations/layer1_attention_weight_min": -16.06891632080078,
      "activations/layer20_attention_weight_max": 22.38121795654297,
      "activations/layer20_attention_weight_min": -21.505582809448242,
      "activations/layer21_attention_weight_max": 41.312042236328125,
      "activations/layer21_attention_weight_min": -27.857250213623047,
      "activations/layer22_attention_weight_max": 31.58024787902832,
      "activations/layer22_attention_weight_min": -25.212867736816406,
      "activations/layer23_attention_weight_max": 40.49468231201172,
      "activations/layer23_attention_weight_min": -23.969947814941406,
      "activations/layer2_attention_weight_max": 36.64381790161133,
      "activations/layer2_attention_weight_min": -32.702247619628906,
      "activations/layer3_attention_weight_max": 115.79931640625,
      "activations/layer3_attention_weight_min": -109.19277954101562,
      "activations/layer4_attention_weight_max": 109.95096588134766,
      "activations/layer4_attention_weight_min": -101.78802490234375,
      "activations/layer5_attention_weight_max": 72.71932983398438,
      "activations/layer5_attention_weight_min": -77.73910522460938,
      "activations/layer6_attention_weight_max": 55.48982238769531,
      "activations/layer6_attention_weight_min": -55.2940559387207,
      "activations/layer7_attention_weight_max": 77.20144653320312,
      "activations/layer7_attention_weight_min": -69.38712310791016,
      "activations/layer8_attention_weight_max": 53.51071548461914,
      "activations/layer8_attention_weight_min": -51.2017822265625,
      "activations/layer9_attention_weight_max": 61.19446563720703,
      "activations/layer9_attention_weight_min": -54.08552169799805,
      "epoch": 14.36,
      "learning_rate": 5.795681818181818e-05,
      "loss": 2.7618,
      "step": 247200
    },
    {
      "activations/layer0_attention_weight_max": 16.028263092041016,
      "activations/layer0_attention_weight_min": -14.070795059204102,
      "activations/layer10_attention_weight_max": 33.253883361816406,
      "activations/layer10_attention_weight_min": -33.49702835083008,
      "activations/layer11_attention_weight_max": 29.18149185180664,
      "activations/layer11_attention_weight_min": -32.46226501464844,
      "activations/layer12_attention_weight_max": 17.993362426757812,
      "activations/layer12_attention_weight_min": -25.471506118774414,
      "activations/layer13_attention_weight_max": 39.15165710449219,
      "activations/layer13_attention_weight_min": -27.072505950927734,
      "activations/layer14_attention_weight_max": 35.494667053222656,
      "activations/layer14_attention_weight_min": -30.725919723510742,
      "activations/layer15_attention_weight_max": 35.16805648803711,
      "activations/layer15_attention_weight_min": -28.160236358642578,
      "activations/layer16_attention_weight_max": 31.236425399780273,
      "activations/layer16_attention_weight_min": -30.310155868530273,
      "activations/layer17_attention_weight_max": 51.70036697387695,
      "activations/layer17_attention_weight_min": -42.988975524902344,
      "activations/layer18_attention_weight_max": 44.25119400024414,
      "activations/layer18_attention_weight_min": -36.619083404541016,
      "activations/layer19_attention_weight_max": 22.266416549682617,
      "activations/layer19_attention_weight_min": -21.492591857910156,
      "activations/layer1_attention_weight_max": 15.76501750946045,
      "activations/layer1_attention_weight_min": -14.855212211608887,
      "activations/layer20_attention_weight_max": 22.237430572509766,
      "activations/layer20_attention_weight_min": -21.809001922607422,
      "activations/layer21_attention_weight_max": 35.422916412353516,
      "activations/layer21_attention_weight_min": -22.68096351623535,
      "activations/layer22_attention_weight_max": 30.617517471313477,
      "activations/layer22_attention_weight_min": -27.93362045288086,
      "activations/layer23_attention_weight_max": 38.87903594970703,
      "activations/layer23_attention_weight_min": -26.285024642944336,
      "activations/layer2_attention_weight_max": 31.45193099975586,
      "activations/layer2_attention_weight_min": -30.090023040771484,
      "activations/layer3_attention_weight_max": 90.5787353515625,
      "activations/layer3_attention_weight_min": -91.39366149902344,
      "activations/layer4_attention_weight_max": 93.24858856201172,
      "activations/layer4_attention_weight_min": -89.5306167602539,
      "activations/layer5_attention_weight_max": 75.82765197753906,
      "activations/layer5_attention_weight_min": -78.34576416015625,
      "activations/layer6_attention_weight_max": 52.984825134277344,
      "activations/layer6_attention_weight_min": -50.74652862548828,
      "activations/layer7_attention_weight_max": 66.62935638427734,
      "activations/layer7_attention_weight_min": -63.02289962768555,
      "activations/layer8_attention_weight_max": 44.70691680908203,
      "activations/layer8_attention_weight_min": -47.29072570800781,
      "activations/layer9_attention_weight_max": 43.7306022644043,
      "activations/layer9_attention_weight_min": -45.39060592651367,
      "epoch": 14.37,
      "learning_rate": 5.793787878787878e-05,
      "loss": 2.7695,
      "step": 247250
    },
    {
      "activations/layer0_attention_weight_max": 16.214807510375977,
      "activations/layer0_attention_weight_min": -14.234829902648926,
      "activations/layer10_attention_weight_max": 36.389984130859375,
      "activations/layer10_attention_weight_min": -35.899261474609375,
      "activations/layer11_attention_weight_max": 33.64228057861328,
      "activations/layer11_attention_weight_min": -34.775142669677734,
      "activations/layer12_attention_weight_max": 19.755390167236328,
      "activations/layer12_attention_weight_min": -24.951461791992188,
      "activations/layer13_attention_weight_max": 40.20856857299805,
      "activations/layer13_attention_weight_min": -33.53666687011719,
      "activations/layer14_attention_weight_max": 39.812007904052734,
      "activations/layer14_attention_weight_min": -30.552288055419922,
      "activations/layer15_attention_weight_max": 35.32866668701172,
      "activations/layer15_attention_weight_min": -28.838350296020508,
      "activations/layer16_attention_weight_max": 33.11532974243164,
      "activations/layer16_attention_weight_min": -29.636619567871094,
      "activations/layer17_attention_weight_max": 51.91316604614258,
      "activations/layer17_attention_weight_min": -42.107906341552734,
      "activations/layer18_attention_weight_max": 48.658851623535156,
      "activations/layer18_attention_weight_min": -37.26666259765625,
      "activations/layer19_attention_weight_max": 22.406728744506836,
      "activations/layer19_attention_weight_min": -20.514219284057617,
      "activations/layer1_attention_weight_max": 16.78752899169922,
      "activations/layer1_attention_weight_min": -14.802727699279785,
      "activations/layer20_attention_weight_max": 21.93800163269043,
      "activations/layer20_attention_weight_min": -20.802366256713867,
      "activations/layer21_attention_weight_max": 35.25678634643555,
      "activations/layer21_attention_weight_min": -21.361827850341797,
      "activations/layer22_attention_weight_max": 34.441864013671875,
      "activations/layer22_attention_weight_min": -26.065385818481445,
      "activations/layer23_attention_weight_max": 39.4721565246582,
      "activations/layer23_attention_weight_min": -24.430875778198242,
      "activations/layer2_attention_weight_max": 33.07728576660156,
      "activations/layer2_attention_weight_min": -29.29184341430664,
      "activations/layer3_attention_weight_max": 95.31913757324219,
      "activations/layer3_attention_weight_min": -92.56645965576172,
      "activations/layer4_attention_weight_max": 97.32567596435547,
      "activations/layer4_attention_weight_min": -91.25166320800781,
      "activations/layer5_attention_weight_max": 72.55921936035156,
      "activations/layer5_attention_weight_min": -73.09938049316406,
      "activations/layer6_attention_weight_max": 54.78385925292969,
      "activations/layer6_attention_weight_min": -54.33289337158203,
      "activations/layer7_attention_weight_max": 69.21491241455078,
      "activations/layer7_attention_weight_min": -63.58197784423828,
      "activations/layer8_attention_weight_max": 48.16122817993164,
      "activations/layer8_attention_weight_min": -48.13857650756836,
      "activations/layer9_attention_weight_max": 49.17026901245117,
      "activations/layer9_attention_weight_min": -46.11385726928711,
      "epoch": 14.37,
      "learning_rate": 5.791893939393939e-05,
      "loss": 2.7463,
      "step": 247300
    },
    {
      "activations/layer0_attention_weight_max": 15.780916213989258,
      "activations/layer0_attention_weight_min": -14.324701309204102,
      "activations/layer10_attention_weight_max": 33.946319580078125,
      "activations/layer10_attention_weight_min": -33.73799133300781,
      "activations/layer11_attention_weight_max": 30.5592041015625,
      "activations/layer11_attention_weight_min": -32.08925247192383,
      "activations/layer12_attention_weight_max": 17.209150314331055,
      "activations/layer12_attention_weight_min": -24.6905460357666,
      "activations/layer13_attention_weight_max": 34.72761535644531,
      "activations/layer13_attention_weight_min": -27.716079711914062,
      "activations/layer14_attention_weight_max": 33.37306213378906,
      "activations/layer14_attention_weight_min": -29.62112045288086,
      "activations/layer15_attention_weight_max": 31.05113983154297,
      "activations/layer15_attention_weight_min": -27.977670669555664,
      "activations/layer16_attention_weight_max": 30.221174240112305,
      "activations/layer16_attention_weight_min": -30.165115356445312,
      "activations/layer17_attention_weight_max": 51.644317626953125,
      "activations/layer17_attention_weight_min": -45.39460372924805,
      "activations/layer18_attention_weight_max": 47.41735076904297,
      "activations/layer18_attention_weight_min": -39.799556732177734,
      "activations/layer19_attention_weight_max": 22.403867721557617,
      "activations/layer19_attention_weight_min": -20.391767501831055,
      "activations/layer1_attention_weight_max": 16.817256927490234,
      "activations/layer1_attention_weight_min": -14.416984558105469,
      "activations/layer20_attention_weight_max": 24.110074996948242,
      "activations/layer20_attention_weight_min": -21.28276252746582,
      "activations/layer21_attention_weight_max": 33.41032791137695,
      "activations/layer21_attention_weight_min": -21.962568283081055,
      "activations/layer22_attention_weight_max": 29.679706573486328,
      "activations/layer22_attention_weight_min": -24.88125228881836,
      "activations/layer23_attention_weight_max": 36.671329498291016,
      "activations/layer23_attention_weight_min": -24.654422760009766,
      "activations/layer2_attention_weight_max": 29.83521842956543,
      "activations/layer2_attention_weight_min": -28.81476402282715,
      "activations/layer3_attention_weight_max": 85.91719818115234,
      "activations/layer3_attention_weight_min": -88.27080535888672,
      "activations/layer4_attention_weight_max": 84.03857421875,
      "activations/layer4_attention_weight_min": -82.26729583740234,
      "activations/layer5_attention_weight_max": 67.17414093017578,
      "activations/layer5_attention_weight_min": -68.94316101074219,
      "activations/layer6_attention_weight_max": 46.99098205566406,
      "activations/layer6_attention_weight_min": -48.118404388427734,
      "activations/layer7_attention_weight_max": 62.62045669555664,
      "activations/layer7_attention_weight_min": -57.29884719848633,
      "activations/layer8_attention_weight_max": 44.796539306640625,
      "activations/layer8_attention_weight_min": -46.12450408935547,
      "activations/layer9_attention_weight_max": 44.008399963378906,
      "activations/layer9_attention_weight_min": -44.18046951293945,
      "epoch": 14.37,
      "learning_rate": 5.79e-05,
      "loss": 2.7654,
      "step": 247350
    },
    {
      "activations/layer0_attention_weight_max": 16.527278900146484,
      "activations/layer0_attention_weight_min": -14.141946792602539,
      "activations/layer10_attention_weight_max": 39.451908111572266,
      "activations/layer10_attention_weight_min": -37.344932556152344,
      "activations/layer11_attention_weight_max": 32.84595489501953,
      "activations/layer11_attention_weight_min": -33.8362922668457,
      "activations/layer12_attention_weight_max": 18.75821876525879,
      "activations/layer12_attention_weight_min": -23.620397567749023,
      "activations/layer13_attention_weight_max": 35.84093475341797,
      "activations/layer13_attention_weight_min": -28.955223083496094,
      "activations/layer14_attention_weight_max": 35.528621673583984,
      "activations/layer14_attention_weight_min": -30.584482192993164,
      "activations/layer15_attention_weight_max": 32.49946975708008,
      "activations/layer15_attention_weight_min": -29.230484008789062,
      "activations/layer16_attention_weight_max": 31.148115158081055,
      "activations/layer16_attention_weight_min": -30.516008377075195,
      "activations/layer17_attention_weight_max": 50.010101318359375,
      "activations/layer17_attention_weight_min": -47.0069694519043,
      "activations/layer18_attention_weight_max": 44.357818603515625,
      "activations/layer18_attention_weight_min": -39.227386474609375,
      "activations/layer19_attention_weight_max": 22.357934951782227,
      "activations/layer19_attention_weight_min": -23.371509552001953,
      "activations/layer1_attention_weight_max": 16.721446990966797,
      "activations/layer1_attention_weight_min": -14.432638168334961,
      "activations/layer20_attention_weight_max": 20.503374099731445,
      "activations/layer20_attention_weight_min": -23.359115600585938,
      "activations/layer21_attention_weight_max": 34.55899429321289,
      "activations/layer21_attention_weight_min": -22.55747413635254,
      "activations/layer22_attention_weight_max": 27.07221794128418,
      "activations/layer22_attention_weight_min": -25.19577980041504,
      "activations/layer23_attention_weight_max": 36.01221466064453,
      "activations/layer23_attention_weight_min": -24.266521453857422,
      "activations/layer2_attention_weight_max": 32.77642822265625,
      "activations/layer2_attention_weight_min": -29.645206451416016,
      "activations/layer3_attention_weight_max": 95.29796600341797,
      "activations/layer3_attention_weight_min": -92.47219848632812,
      "activations/layer4_attention_weight_max": 90.73470306396484,
      "activations/layer4_attention_weight_min": -85.1204605102539,
      "activations/layer5_attention_weight_max": 71.86027526855469,
      "activations/layer5_attention_weight_min": -76.45409393310547,
      "activations/layer6_attention_weight_max": 53.944664001464844,
      "activations/layer6_attention_weight_min": -52.37846755981445,
      "activations/layer7_attention_weight_max": 65.2076416015625,
      "activations/layer7_attention_weight_min": -65.90824127197266,
      "activations/layer8_attention_weight_max": 47.49003601074219,
      "activations/layer8_attention_weight_min": -46.635414123535156,
      "activations/layer9_attention_weight_max": 46.94994354248047,
      "activations/layer9_attention_weight_min": -47.28616714477539,
      "epoch": 14.38,
      "learning_rate": 5.78810606060606e-05,
      "loss": 2.7613,
      "step": 247400
    },
    {
      "activations/layer0_attention_weight_max": 16.056671142578125,
      "activations/layer0_attention_weight_min": -14.567436218261719,
      "activations/layer10_attention_weight_max": 39.21840286254883,
      "activations/layer10_attention_weight_min": -37.84143829345703,
      "activations/layer11_attention_weight_max": 33.486083984375,
      "activations/layer11_attention_weight_min": -37.30879211425781,
      "activations/layer12_attention_weight_max": 18.285802841186523,
      "activations/layer12_attention_weight_min": -32.8869743347168,
      "activations/layer13_attention_weight_max": 33.53345489501953,
      "activations/layer13_attention_weight_min": -26.833362579345703,
      "activations/layer14_attention_weight_max": 34.65965270996094,
      "activations/layer14_attention_weight_min": -30.868247985839844,
      "activations/layer15_attention_weight_max": 32.272796630859375,
      "activations/layer15_attention_weight_min": -30.306177139282227,
      "activations/layer16_attention_weight_max": 33.12198257446289,
      "activations/layer16_attention_weight_min": -32.48210525512695,
      "activations/layer17_attention_weight_max": 51.21457290649414,
      "activations/layer17_attention_weight_min": -44.376590728759766,
      "activations/layer18_attention_weight_max": 43.93129348754883,
      "activations/layer18_attention_weight_min": -38.483421325683594,
      "activations/layer19_attention_weight_max": 23.244455337524414,
      "activations/layer19_attention_weight_min": -22.462316513061523,
      "activations/layer1_attention_weight_max": 17.85883331298828,
      "activations/layer1_attention_weight_min": -16.12894630432129,
      "activations/layer20_attention_weight_max": 22.82883644104004,
      "activations/layer20_attention_weight_min": -22.553462982177734,
      "activations/layer21_attention_weight_max": 33.5738410949707,
      "activations/layer21_attention_weight_min": -25.22171401977539,
      "activations/layer22_attention_weight_max": 30.42426300048828,
      "activations/layer22_attention_weight_min": -24.87268829345703,
      "activations/layer23_attention_weight_max": 35.21059799194336,
      "activations/layer23_attention_weight_min": -27.17562484741211,
      "activations/layer2_attention_weight_max": 33.11042785644531,
      "activations/layer2_attention_weight_min": -31.923994064331055,
      "activations/layer3_attention_weight_max": 95.98136901855469,
      "activations/layer3_attention_weight_min": -102.81610870361328,
      "activations/layer4_attention_weight_max": 94.43919372558594,
      "activations/layer4_attention_weight_min": -93.07217407226562,
      "activations/layer5_attention_weight_max": 70.10066223144531,
      "activations/layer5_attention_weight_min": -76.46400451660156,
      "activations/layer6_attention_weight_max": 54.35813522338867,
      "activations/layer6_attention_weight_min": -53.74338912963867,
      "activations/layer7_attention_weight_max": 62.81924819946289,
      "activations/layer7_attention_weight_min": -64.8995590209961,
      "activations/layer8_attention_weight_max": 44.63460159301758,
      "activations/layer8_attention_weight_min": -49.4029426574707,
      "activations/layer9_attention_weight_max": 47.38515853881836,
      "activations/layer9_attention_weight_min": -47.34128952026367,
      "epoch": 14.38,
      "learning_rate": 5.78621212121212e-05,
      "loss": 2.7697,
      "step": 247450
    },
    {
      "activations/layer0_attention_weight_max": 16.421409606933594,
      "activations/layer0_attention_weight_min": -14.220784187316895,
      "activations/layer10_attention_weight_max": 36.91712188720703,
      "activations/layer10_attention_weight_min": -33.732337951660156,
      "activations/layer11_attention_weight_max": 31.62301254272461,
      "activations/layer11_attention_weight_min": -32.50136947631836,
      "activations/layer12_attention_weight_max": 21.346166610717773,
      "activations/layer12_attention_weight_min": -24.127166748046875,
      "activations/layer13_attention_weight_max": 40.6693229675293,
      "activations/layer13_attention_weight_min": -27.701101303100586,
      "activations/layer14_attention_weight_max": 34.89361572265625,
      "activations/layer14_attention_weight_min": -28.908321380615234,
      "activations/layer15_attention_weight_max": 33.53065490722656,
      "activations/layer15_attention_weight_min": -27.419519424438477,
      "activations/layer16_attention_weight_max": 31.238008499145508,
      "activations/layer16_attention_weight_min": -28.951234817504883,
      "activations/layer17_attention_weight_max": 49.030059814453125,
      "activations/layer17_attention_weight_min": -42.972129821777344,
      "activations/layer18_attention_weight_max": 46.10530471801758,
      "activations/layer18_attention_weight_min": -38.71377182006836,
      "activations/layer19_attention_weight_max": 23.394268035888672,
      "activations/layer19_attention_weight_min": -24.00910758972168,
      "activations/layer1_attention_weight_max": 18.21293067932129,
      "activations/layer1_attention_weight_min": -16.29705047607422,
      "activations/layer20_attention_weight_max": 25.346208572387695,
      "activations/layer20_attention_weight_min": -21.26054573059082,
      "activations/layer21_attention_weight_max": 35.864986419677734,
      "activations/layer21_attention_weight_min": -21.7581844329834,
      "activations/layer22_attention_weight_max": 29.425336837768555,
      "activations/layer22_attention_weight_min": -25.55356216430664,
      "activations/layer23_attention_weight_max": 34.57396697998047,
      "activations/layer23_attention_weight_min": -23.434356689453125,
      "activations/layer2_attention_weight_max": 32.483726501464844,
      "activations/layer2_attention_weight_min": -30.63911247253418,
      "activations/layer3_attention_weight_max": 97.44686126708984,
      "activations/layer3_attention_weight_min": -93.51432800292969,
      "activations/layer4_attention_weight_max": 94.8650131225586,
      "activations/layer4_attention_weight_min": -87.59495544433594,
      "activations/layer5_attention_weight_max": 68.4699935913086,
      "activations/layer5_attention_weight_min": -77.13367462158203,
      "activations/layer6_attention_weight_max": 49.150882720947266,
      "activations/layer6_attention_weight_min": -49.105995178222656,
      "activations/layer7_attention_weight_max": 62.86233139038086,
      "activations/layer7_attention_weight_min": -64.08045959472656,
      "activations/layer8_attention_weight_max": 45.130245208740234,
      "activations/layer8_attention_weight_min": -47.80043411254883,
      "activations/layer9_attention_weight_max": 43.72893524169922,
      "activations/layer9_attention_weight_min": -42.83549118041992,
      "epoch": 14.38,
      "learning_rate": 5.7843181818181816e-05,
      "loss": 2.7542,
      "step": 247500
    },
    {
      "activations/layer0_attention_weight_max": 15.616872787475586,
      "activations/layer0_attention_weight_min": -14.15046215057373,
      "activations/layer10_attention_weight_max": 34.54339599609375,
      "activations/layer10_attention_weight_min": -32.72637176513672,
      "activations/layer11_attention_weight_max": 31.064777374267578,
      "activations/layer11_attention_weight_min": -32.012630462646484,
      "activations/layer12_attention_weight_max": 18.212324142456055,
      "activations/layer12_attention_weight_min": -25.388614654541016,
      "activations/layer13_attention_weight_max": 37.25078201293945,
      "activations/layer13_attention_weight_min": -27.633197784423828,
      "activations/layer14_attention_weight_max": 36.295013427734375,
      "activations/layer14_attention_weight_min": -28.382291793823242,
      "activations/layer15_attention_weight_max": 33.08940124511719,
      "activations/layer15_attention_weight_min": -28.578523635864258,
      "activations/layer16_attention_weight_max": 31.65345001220703,
      "activations/layer16_attention_weight_min": -29.347681045532227,
      "activations/layer17_attention_weight_max": 52.623470306396484,
      "activations/layer17_attention_weight_min": -43.042945861816406,
      "activations/layer18_attention_weight_max": 45.866539001464844,
      "activations/layer18_attention_weight_min": -37.728572845458984,
      "activations/layer19_attention_weight_max": 22.60051155090332,
      "activations/layer19_attention_weight_min": -23.076553344726562,
      "activations/layer1_attention_weight_max": 15.877918243408203,
      "activations/layer1_attention_weight_min": -14.818146705627441,
      "activations/layer20_attention_weight_max": 23.783706665039062,
      "activations/layer20_attention_weight_min": -23.87501335144043,
      "activations/layer21_attention_weight_max": 34.428131103515625,
      "activations/layer21_attention_weight_min": -21.273488998413086,
      "activations/layer22_attention_weight_max": 29.933536529541016,
      "activations/layer22_attention_weight_min": -27.231422424316406,
      "activations/layer23_attention_weight_max": 37.00191116333008,
      "activations/layer23_attention_weight_min": -24.296260833740234,
      "activations/layer2_attention_weight_max": 31.52728843688965,
      "activations/layer2_attention_weight_min": -31.05594253540039,
      "activations/layer3_attention_weight_max": 93.11174774169922,
      "activations/layer3_attention_weight_min": -96.77037048339844,
      "activations/layer4_attention_weight_max": 88.3619155883789,
      "activations/layer4_attention_weight_min": -85.30052947998047,
      "activations/layer5_attention_weight_max": 71.78202819824219,
      "activations/layer5_attention_weight_min": -78.83245849609375,
      "activations/layer6_attention_weight_max": 49.05946731567383,
      "activations/layer6_attention_weight_min": -51.041419982910156,
      "activations/layer7_attention_weight_max": 62.82143020629883,
      "activations/layer7_attention_weight_min": -61.02416229248047,
      "activations/layer8_attention_weight_max": 44.66263198852539,
      "activations/layer8_attention_weight_min": -47.83597183227539,
      "activations/layer9_attention_weight_max": 42.42628860473633,
      "activations/layer9_attention_weight_min": -44.261165618896484,
      "epoch": 14.38,
      "learning_rate": 5.782424242424242e-05,
      "loss": 2.7543,
      "step": 247550
    },
    {
      "activations/layer0_attention_weight_max": 16.309799194335938,
      "activations/layer0_attention_weight_min": -14.186134338378906,
      "activations/layer10_attention_weight_max": 35.89384460449219,
      "activations/layer10_attention_weight_min": -35.35930252075195,
      "activations/layer11_attention_weight_max": 30.74250602722168,
      "activations/layer11_attention_weight_min": -32.732139587402344,
      "activations/layer12_attention_weight_max": 18.114809036254883,
      "activations/layer12_attention_weight_min": -22.20405387878418,
      "activations/layer13_attention_weight_max": 34.601375579833984,
      "activations/layer13_attention_weight_min": -26.964529037475586,
      "activations/layer14_attention_weight_max": 34.38230895996094,
      "activations/layer14_attention_weight_min": -29.868423461914062,
      "activations/layer15_attention_weight_max": 32.085079193115234,
      "activations/layer15_attention_weight_min": -29.525897979736328,
      "activations/layer16_attention_weight_max": 31.657852172851562,
      "activations/layer16_attention_weight_min": -30.60144805908203,
      "activations/layer17_attention_weight_max": 52.42304992675781,
      "activations/layer17_attention_weight_min": -44.893699645996094,
      "activations/layer18_attention_weight_max": 44.407135009765625,
      "activations/layer18_attention_weight_min": -38.99543762207031,
      "activations/layer19_attention_weight_max": 23.53139305114746,
      "activations/layer19_attention_weight_min": -24.106006622314453,
      "activations/layer1_attention_weight_max": 16.31070327758789,
      "activations/layer1_attention_weight_min": -15.471877098083496,
      "activations/layer20_attention_weight_max": 23.12816619873047,
      "activations/layer20_attention_weight_min": -23.655691146850586,
      "activations/layer21_attention_weight_max": 36.20087432861328,
      "activations/layer21_attention_weight_min": -23.889028549194336,
      "activations/layer22_attention_weight_max": 33.01591110229492,
      "activations/layer22_attention_weight_min": -27.31899642944336,
      "activations/layer23_attention_weight_max": 37.05672836303711,
      "activations/layer23_attention_weight_min": -23.8179931640625,
      "activations/layer2_attention_weight_max": 31.521549224853516,
      "activations/layer2_attention_weight_min": -28.906843185424805,
      "activations/layer3_attention_weight_max": 92.103759765625,
      "activations/layer3_attention_weight_min": -91.86742401123047,
      "activations/layer4_attention_weight_max": 91.45801544189453,
      "activations/layer4_attention_weight_min": -84.4729232788086,
      "activations/layer5_attention_weight_max": 69.23590087890625,
      "activations/layer5_attention_weight_min": -75.78443908691406,
      "activations/layer6_attention_weight_max": 50.0428581237793,
      "activations/layer6_attention_weight_min": -50.325931549072266,
      "activations/layer7_attention_weight_max": 62.974822998046875,
      "activations/layer7_attention_weight_min": -66.09981536865234,
      "activations/layer8_attention_weight_max": 44.054439544677734,
      "activations/layer8_attention_weight_min": -45.967525482177734,
      "activations/layer9_attention_weight_max": 44.023624420166016,
      "activations/layer9_attention_weight_min": -45.9466438293457,
      "epoch": 14.39,
      "learning_rate": 5.7805303030303026e-05,
      "loss": 2.7497,
      "step": 247600
    },
    {
      "activations/layer0_attention_weight_max": 16.57637596130371,
      "activations/layer0_attention_weight_min": -14.30753231048584,
      "activations/layer10_attention_weight_max": 36.77848815917969,
      "activations/layer10_attention_weight_min": -38.9829216003418,
      "activations/layer11_attention_weight_max": 32.516136169433594,
      "activations/layer11_attention_weight_min": -33.81434631347656,
      "activations/layer12_attention_weight_max": 20.228378295898438,
      "activations/layer12_attention_weight_min": -25.24416732788086,
      "activations/layer13_attention_weight_max": 36.49453353881836,
      "activations/layer13_attention_weight_min": -27.063663482666016,
      "activations/layer14_attention_weight_max": 34.67414855957031,
      "activations/layer14_attention_weight_min": -34.039512634277344,
      "activations/layer15_attention_weight_max": 33.569461822509766,
      "activations/layer15_attention_weight_min": -31.23154640197754,
      "activations/layer16_attention_weight_max": 31.552209854125977,
      "activations/layer16_attention_weight_min": -32.19935607910156,
      "activations/layer17_attention_weight_max": 51.51385498046875,
      "activations/layer17_attention_weight_min": -46.81170654296875,
      "activations/layer18_attention_weight_max": 46.423221588134766,
      "activations/layer18_attention_weight_min": -42.39790344238281,
      "activations/layer19_attention_weight_max": 25.351959228515625,
      "activations/layer19_attention_weight_min": -23.805646896362305,
      "activations/layer1_attention_weight_max": 16.489356994628906,
      "activations/layer1_attention_weight_min": -14.859743118286133,
      "activations/layer20_attention_weight_max": 23.700397491455078,
      "activations/layer20_attention_weight_min": -23.21502113342285,
      "activations/layer21_attention_weight_max": 46.7113151550293,
      "activations/layer21_attention_weight_min": -25.02695655822754,
      "activations/layer22_attention_weight_max": 32.45177459716797,
      "activations/layer22_attention_weight_min": -26.51499366760254,
      "activations/layer23_attention_weight_max": 39.87697219848633,
      "activations/layer23_attention_weight_min": -24.726383209228516,
      "activations/layer2_attention_weight_max": 32.63022232055664,
      "activations/layer2_attention_weight_min": -30.640615463256836,
      "activations/layer3_attention_weight_max": 97.95948028564453,
      "activations/layer3_attention_weight_min": -98.41427612304688,
      "activations/layer4_attention_weight_max": 93.56060028076172,
      "activations/layer4_attention_weight_min": -89.10736846923828,
      "activations/layer5_attention_weight_max": 69.72921752929688,
      "activations/layer5_attention_weight_min": -70.99674987792969,
      "activations/layer6_attention_weight_max": 49.644893646240234,
      "activations/layer6_attention_weight_min": -51.996334075927734,
      "activations/layer7_attention_weight_max": 63.52082061767578,
      "activations/layer7_attention_weight_min": -63.65874099731445,
      "activations/layer8_attention_weight_max": 45.73784255981445,
      "activations/layer8_attention_weight_min": -47.32101821899414,
      "activations/layer9_attention_weight_max": 45.22385025024414,
      "activations/layer9_attention_weight_min": -44.479122161865234,
      "epoch": 14.39,
      "learning_rate": 5.778636363636363e-05,
      "loss": 2.7704,
      "step": 247650
    },
    {
      "activations/layer0_attention_weight_max": 16.575395584106445,
      "activations/layer0_attention_weight_min": -14.2880220413208,
      "activations/layer10_attention_weight_max": 36.17390060424805,
      "activations/layer10_attention_weight_min": -34.44546890258789,
      "activations/layer11_attention_weight_max": 33.19677734375,
      "activations/layer11_attention_weight_min": -33.383811950683594,
      "activations/layer12_attention_weight_max": 18.098346710205078,
      "activations/layer12_attention_weight_min": -24.28044319152832,
      "activations/layer13_attention_weight_max": 33.434959411621094,
      "activations/layer13_attention_weight_min": -26.753313064575195,
      "activations/layer14_attention_weight_max": 32.890586853027344,
      "activations/layer14_attention_weight_min": -30.05609130859375,
      "activations/layer15_attention_weight_max": 30.105485916137695,
      "activations/layer15_attention_weight_min": -28.930198669433594,
      "activations/layer16_attention_weight_max": 30.69472312927246,
      "activations/layer16_attention_weight_min": -30.460309982299805,
      "activations/layer17_attention_weight_max": 48.45289993286133,
      "activations/layer17_attention_weight_min": -42.4202880859375,
      "activations/layer18_attention_weight_max": 43.57689666748047,
      "activations/layer18_attention_weight_min": -36.285972595214844,
      "activations/layer19_attention_weight_max": 23.575843811035156,
      "activations/layer19_attention_weight_min": -23.16965675354004,
      "activations/layer1_attention_weight_max": 17.49687385559082,
      "activations/layer1_attention_weight_min": -15.352534294128418,
      "activations/layer20_attention_weight_max": 21.508764266967773,
      "activations/layer20_attention_weight_min": -21.996807098388672,
      "activations/layer21_attention_weight_max": 33.27024459838867,
      "activations/layer21_attention_weight_min": -20.21360206604004,
      "activations/layer22_attention_weight_max": 31.032211303710938,
      "activations/layer22_attention_weight_min": -25.030332565307617,
      "activations/layer23_attention_weight_max": 40.049400329589844,
      "activations/layer23_attention_weight_min": -25.608152389526367,
      "activations/layer2_attention_weight_max": 32.492252349853516,
      "activations/layer2_attention_weight_min": -31.123395919799805,
      "activations/layer3_attention_weight_max": 94.96215057373047,
      "activations/layer3_attention_weight_min": -93.89175415039062,
      "activations/layer4_attention_weight_max": 90.13927459716797,
      "activations/layer4_attention_weight_min": -88.13493347167969,
      "activations/layer5_attention_weight_max": 72.83392333984375,
      "activations/layer5_attention_weight_min": -78.0938720703125,
      "activations/layer6_attention_weight_max": 49.44486999511719,
      "activations/layer6_attention_weight_min": -51.305747985839844,
      "activations/layer7_attention_weight_max": 66.77828979492188,
      "activations/layer7_attention_weight_min": -63.90011978149414,
      "activations/layer8_attention_weight_max": 45.214820861816406,
      "activations/layer8_attention_weight_min": -46.5349006652832,
      "activations/layer9_attention_weight_max": 44.02404022216797,
      "activations/layer9_attention_weight_min": -44.211463928222656,
      "epoch": 14.39,
      "learning_rate": 5.776742424242424e-05,
      "loss": 2.7607,
      "step": 247700
    },
    {
      "activations/layer0_attention_weight_max": 16.096965789794922,
      "activations/layer0_attention_weight_min": -14.122631072998047,
      "activations/layer10_attention_weight_max": 35.25542068481445,
      "activations/layer10_attention_weight_min": -34.1016960144043,
      "activations/layer11_attention_weight_max": 31.834976196289062,
      "activations/layer11_attention_weight_min": -32.240543365478516,
      "activations/layer12_attention_weight_max": 18.141477584838867,
      "activations/layer12_attention_weight_min": -24.0258731842041,
      "activations/layer13_attention_weight_max": 37.9914436340332,
      "activations/layer13_attention_weight_min": -30.77506446838379,
      "activations/layer14_attention_weight_max": 35.93937301635742,
      "activations/layer14_attention_weight_min": -30.122440338134766,
      "activations/layer15_attention_weight_max": 33.56601333618164,
      "activations/layer15_attention_weight_min": -28.762516021728516,
      "activations/layer16_attention_weight_max": 32.607887268066406,
      "activations/layer16_attention_weight_min": -30.7943115234375,
      "activations/layer17_attention_weight_max": 53.18114471435547,
      "activations/layer17_attention_weight_min": -45.66778564453125,
      "activations/layer18_attention_weight_max": 48.846588134765625,
      "activations/layer18_attention_weight_min": -41.684043884277344,
      "activations/layer19_attention_weight_max": 23.774465560913086,
      "activations/layer19_attention_weight_min": -23.93395233154297,
      "activations/layer1_attention_weight_max": 15.895719528198242,
      "activations/layer1_attention_weight_min": -15.648436546325684,
      "activations/layer20_attention_weight_max": 23.318178176879883,
      "activations/layer20_attention_weight_min": -22.435226440429688,
      "activations/layer21_attention_weight_max": 36.49250793457031,
      "activations/layer21_attention_weight_min": -24.249414443969727,
      "activations/layer22_attention_weight_max": 27.74888801574707,
      "activations/layer22_attention_weight_min": -26.20426368713379,
      "activations/layer23_attention_weight_max": 38.27484130859375,
      "activations/layer23_attention_weight_min": -22.88134002685547,
      "activations/layer2_attention_weight_max": 30.25311279296875,
      "activations/layer2_attention_weight_min": -28.574756622314453,
      "activations/layer3_attention_weight_max": 94.52143096923828,
      "activations/layer3_attention_weight_min": -98.8262710571289,
      "activations/layer4_attention_weight_max": 93.1202392578125,
      "activations/layer4_attention_weight_min": -87.7178955078125,
      "activations/layer5_attention_weight_max": 69.68476867675781,
      "activations/layer5_attention_weight_min": -77.93941497802734,
      "activations/layer6_attention_weight_max": 51.725547790527344,
      "activations/layer6_attention_weight_min": -55.16122817993164,
      "activations/layer7_attention_weight_max": 62.78946304321289,
      "activations/layer7_attention_weight_min": -66.38914489746094,
      "activations/layer8_attention_weight_max": 42.91232681274414,
      "activations/layer8_attention_weight_min": -46.48749542236328,
      "activations/layer9_attention_weight_max": 42.6814079284668,
      "activations/layer9_attention_weight_min": -51.085479736328125,
      "epoch": 14.4,
      "learning_rate": 5.7748484848484844e-05,
      "loss": 2.7618,
      "step": 247750
    },
    {
      "activations/layer0_attention_weight_max": 15.461429595947266,
      "activations/layer0_attention_weight_min": -14.299781799316406,
      "activations/layer10_attention_weight_max": 36.31836700439453,
      "activations/layer10_attention_weight_min": -36.160430908203125,
      "activations/layer11_attention_weight_max": 33.709991455078125,
      "activations/layer11_attention_weight_min": -32.78338623046875,
      "activations/layer12_attention_weight_max": 22.49909782409668,
      "activations/layer12_attention_weight_min": -25.592254638671875,
      "activations/layer13_attention_weight_max": 37.655540466308594,
      "activations/layer13_attention_weight_min": -30.631444931030273,
      "activations/layer14_attention_weight_max": 39.22642517089844,
      "activations/layer14_attention_weight_min": -34.73065948486328,
      "activations/layer15_attention_weight_max": 36.14008331298828,
      "activations/layer15_attention_weight_min": -36.231536865234375,
      "activations/layer16_attention_weight_max": 34.01206970214844,
      "activations/layer16_attention_weight_min": -31.600496292114258,
      "activations/layer17_attention_weight_max": 53.799476623535156,
      "activations/layer17_attention_weight_min": -46.090362548828125,
      "activations/layer18_attention_weight_max": 47.3349723815918,
      "activations/layer18_attention_weight_min": -38.507606506347656,
      "activations/layer19_attention_weight_max": 27.690969467163086,
      "activations/layer19_attention_weight_min": -23.47539710998535,
      "activations/layer1_attention_weight_max": 15.879990577697754,
      "activations/layer1_attention_weight_min": -16.64047622680664,
      "activations/layer20_attention_weight_max": 23.062646865844727,
      "activations/layer20_attention_weight_min": -25.437875747680664,
      "activations/layer21_attention_weight_max": 36.73727035522461,
      "activations/layer21_attention_weight_min": -20.840099334716797,
      "activations/layer22_attention_weight_max": 28.66136932373047,
      "activations/layer22_attention_weight_min": -26.64154624938965,
      "activations/layer23_attention_weight_max": 34.67573547363281,
      "activations/layer23_attention_weight_min": -22.587982177734375,
      "activations/layer2_attention_weight_max": 31.743261337280273,
      "activations/layer2_attention_weight_min": -28.91903305053711,
      "activations/layer3_attention_weight_max": 94.5127182006836,
      "activations/layer3_attention_weight_min": -96.71454620361328,
      "activations/layer4_attention_weight_max": 92.06051635742188,
      "activations/layer4_attention_weight_min": -88.84397888183594,
      "activations/layer5_attention_weight_max": 66.6405029296875,
      "activations/layer5_attention_weight_min": -75.1402816772461,
      "activations/layer6_attention_weight_max": 52.216861724853516,
      "activations/layer6_attention_weight_min": -51.65815734863281,
      "activations/layer7_attention_weight_max": 62.58699035644531,
      "activations/layer7_attention_weight_min": -63.98204040527344,
      "activations/layer8_attention_weight_max": 48.53915786743164,
      "activations/layer8_attention_weight_min": -49.647865295410156,
      "activations/layer9_attention_weight_max": 45.91515350341797,
      "activations/layer9_attention_weight_min": -47.019073486328125,
      "epoch": 14.4,
      "learning_rate": 5.7729545454545445e-05,
      "loss": 2.7487,
      "step": 247800
    },
    {
      "activations/layer0_attention_weight_max": 15.206968307495117,
      "activations/layer0_attention_weight_min": -14.377479553222656,
      "activations/layer10_attention_weight_max": 37.53560256958008,
      "activations/layer10_attention_weight_min": -37.944740295410156,
      "activations/layer11_attention_weight_max": 34.593475341796875,
      "activations/layer11_attention_weight_min": -35.412933349609375,
      "activations/layer12_attention_weight_max": 18.313331604003906,
      "activations/layer12_attention_weight_min": -22.90328598022461,
      "activations/layer13_attention_weight_max": 35.43229675292969,
      "activations/layer13_attention_weight_min": -32.36385726928711,
      "activations/layer14_attention_weight_max": 37.62705612182617,
      "activations/layer14_attention_weight_min": -30.13864517211914,
      "activations/layer15_attention_weight_max": 32.56645202636719,
      "activations/layer15_attention_weight_min": -28.43130111694336,
      "activations/layer16_attention_weight_max": 34.142539978027344,
      "activations/layer16_attention_weight_min": -30.60778045654297,
      "activations/layer17_attention_weight_max": 52.06471633911133,
      "activations/layer17_attention_weight_min": -44.520973205566406,
      "activations/layer18_attention_weight_max": 46.72206497192383,
      "activations/layer18_attention_weight_min": -38.1936149597168,
      "activations/layer19_attention_weight_max": 24.235509872436523,
      "activations/layer19_attention_weight_min": -20.901111602783203,
      "activations/layer1_attention_weight_max": 16.012866973876953,
      "activations/layer1_attention_weight_min": -15.277865409851074,
      "activations/layer20_attention_weight_max": 23.08538055419922,
      "activations/layer20_attention_weight_min": -22.93422508239746,
      "activations/layer21_attention_weight_max": 42.258609771728516,
      "activations/layer21_attention_weight_min": -23.50377082824707,
      "activations/layer22_attention_weight_max": 30.19120979309082,
      "activations/layer22_attention_weight_min": -25.240686416625977,
      "activations/layer23_attention_weight_max": 36.31086730957031,
      "activations/layer23_attention_weight_min": -24.243911743164062,
      "activations/layer2_attention_weight_max": 32.997535705566406,
      "activations/layer2_attention_weight_min": -31.887662887573242,
      "activations/layer3_attention_weight_max": 98.74687194824219,
      "activations/layer3_attention_weight_min": -101.2123031616211,
      "activations/layer4_attention_weight_max": 96.10724639892578,
      "activations/layer4_attention_weight_min": -98.67710876464844,
      "activations/layer5_attention_weight_max": 66.68975830078125,
      "activations/layer5_attention_weight_min": -78.98660278320312,
      "activations/layer6_attention_weight_max": 52.06443786621094,
      "activations/layer6_attention_weight_min": -52.89875030517578,
      "activations/layer7_attention_weight_max": 66.98980712890625,
      "activations/layer7_attention_weight_min": -66.51285552978516,
      "activations/layer8_attention_weight_max": 49.30912399291992,
      "activations/layer8_attention_weight_min": -49.15875244140625,
      "activations/layer9_attention_weight_max": 47.25484848022461,
      "activations/layer9_attention_weight_min": -46.89619064331055,
      "epoch": 14.4,
      "learning_rate": 5.7710606060606054e-05,
      "loss": 2.7367,
      "step": 247850
    },
    {
      "activations/layer0_attention_weight_max": 15.319108009338379,
      "activations/layer0_attention_weight_min": -14.02113151550293,
      "activations/layer10_attention_weight_max": 47.137054443359375,
      "activations/layer10_attention_weight_min": -44.871585845947266,
      "activations/layer11_attention_weight_max": 42.21736145019531,
      "activations/layer11_attention_weight_min": -38.79446792602539,
      "activations/layer12_attention_weight_max": 19.5323429107666,
      "activations/layer12_attention_weight_min": -30.836044311523438,
      "activations/layer13_attention_weight_max": 42.26472091674805,
      "activations/layer13_attention_weight_min": -29.792470932006836,
      "activations/layer14_attention_weight_max": 38.361541748046875,
      "activations/layer14_attention_weight_min": -31.376708984375,
      "activations/layer15_attention_weight_max": 41.95330810546875,
      "activations/layer15_attention_weight_min": -31.528406143188477,
      "activations/layer16_attention_weight_max": 32.516578674316406,
      "activations/layer16_attention_weight_min": -31.873157501220703,
      "activations/layer17_attention_weight_max": 53.63407897949219,
      "activations/layer17_attention_weight_min": -49.461341857910156,
      "activations/layer18_attention_weight_max": 46.64226531982422,
      "activations/layer18_attention_weight_min": -44.27849197387695,
      "activations/layer19_attention_weight_max": 27.964765548706055,
      "activations/layer19_attention_weight_min": -23.38214683532715,
      "activations/layer1_attention_weight_max": 16.633926391601562,
      "activations/layer1_attention_weight_min": -15.206168174743652,
      "activations/layer20_attention_weight_max": 27.049055099487305,
      "activations/layer20_attention_weight_min": -21.787492752075195,
      "activations/layer21_attention_weight_max": 38.35748291015625,
      "activations/layer21_attention_weight_min": -23.99669075012207,
      "activations/layer22_attention_weight_max": 32.86334991455078,
      "activations/layer22_attention_weight_min": -25.054155349731445,
      "activations/layer23_attention_weight_max": 36.95947265625,
      "activations/layer23_attention_weight_min": -25.201915740966797,
      "activations/layer2_attention_weight_max": 34.30213928222656,
      "activations/layer2_attention_weight_min": -31.78204345703125,
      "activations/layer3_attention_weight_max": 95.35848999023438,
      "activations/layer3_attention_weight_min": -99.30841064453125,
      "activations/layer4_attention_weight_max": 94.58404541015625,
      "activations/layer4_attention_weight_min": -92.98058319091797,
      "activations/layer5_attention_weight_max": 72.34616088867188,
      "activations/layer5_attention_weight_min": -76.1198501586914,
      "activations/layer6_attention_weight_max": 52.09980773925781,
      "activations/layer6_attention_weight_min": -52.53053283691406,
      "activations/layer7_attention_weight_max": 76.5213623046875,
      "activations/layer7_attention_weight_min": -70.69612884521484,
      "activations/layer8_attention_weight_max": 57.4835205078125,
      "activations/layer8_attention_weight_min": -51.29855728149414,
      "activations/layer9_attention_weight_max": 61.32746505737305,
      "activations/layer9_attention_weight_min": -54.72861862182617,
      "epoch": 14.4,
      "learning_rate": 5.769166666666666e-05,
      "loss": 2.7642,
      "step": 247900
    },
    {
      "activations/layer0_attention_weight_max": 16.43416404724121,
      "activations/layer0_attention_weight_min": -14.177083015441895,
      "activations/layer10_attention_weight_max": 32.767425537109375,
      "activations/layer10_attention_weight_min": -32.27503204345703,
      "activations/layer11_attention_weight_max": 28.445310592651367,
      "activations/layer11_attention_weight_min": -31.093276977539062,
      "activations/layer12_attention_weight_max": 16.563907623291016,
      "activations/layer12_attention_weight_min": -26.043434143066406,
      "activations/layer13_attention_weight_max": 32.842628479003906,
      "activations/layer13_attention_weight_min": -26.74655532836914,
      "activations/layer14_attention_weight_max": 32.96919631958008,
      "activations/layer14_attention_weight_min": -30.422183990478516,
      "activations/layer15_attention_weight_max": 30.60972023010254,
      "activations/layer15_attention_weight_min": -28.352128982543945,
      "activations/layer16_attention_weight_max": 30.541263580322266,
      "activations/layer16_attention_weight_min": -29.628536224365234,
      "activations/layer17_attention_weight_max": 49.26395034790039,
      "activations/layer17_attention_weight_min": -43.647621154785156,
      "activations/layer18_attention_weight_max": 42.94484329223633,
      "activations/layer18_attention_weight_min": -38.12444305419922,
      "activations/layer19_attention_weight_max": 21.628551483154297,
      "activations/layer19_attention_weight_min": -21.51463508605957,
      "activations/layer1_attention_weight_max": 16.58928871154785,
      "activations/layer1_attention_weight_min": -17.16197967529297,
      "activations/layer20_attention_weight_max": 20.411624908447266,
      "activations/layer20_attention_weight_min": -23.558696746826172,
      "activations/layer21_attention_weight_max": 31.86407470703125,
      "activations/layer21_attention_weight_min": -19.85127830505371,
      "activations/layer22_attention_weight_max": 26.655229568481445,
      "activations/layer22_attention_weight_min": -25.735607147216797,
      "activations/layer23_attention_weight_max": 32.12282943725586,
      "activations/layer23_attention_weight_min": -26.20047378540039,
      "activations/layer2_attention_weight_max": 33.382205963134766,
      "activations/layer2_attention_weight_min": -32.14362335205078,
      "activations/layer3_attention_weight_max": 94.19288635253906,
      "activations/layer3_attention_weight_min": -98.79847717285156,
      "activations/layer4_attention_weight_max": 91.98265838623047,
      "activations/layer4_attention_weight_min": -88.68891906738281,
      "activations/layer5_attention_weight_max": 71.14029693603516,
      "activations/layer5_attention_weight_min": -81.33146667480469,
      "activations/layer6_attention_weight_max": 49.76675796508789,
      "activations/layer6_attention_weight_min": -49.165306091308594,
      "activations/layer7_attention_weight_max": 62.44096374511719,
      "activations/layer7_attention_weight_min": -59.774574279785156,
      "activations/layer8_attention_weight_max": 42.27423858642578,
      "activations/layer8_attention_weight_min": -43.9544792175293,
      "activations/layer9_attention_weight_max": 41.43342971801758,
      "activations/layer9_attention_weight_min": -43.544857025146484,
      "epoch": 14.41,
      "learning_rate": 5.767272727272727e-05,
      "loss": 2.7575,
      "step": 247950
    },
    {
      "activations/layer0_attention_weight_max": 14.785577774047852,
      "activations/layer0_attention_weight_min": -14.315031051635742,
      "activations/layer10_attention_weight_max": 33.7469367980957,
      "activations/layer10_attention_weight_min": -33.85011672973633,
      "activations/layer11_attention_weight_max": 32.14282989501953,
      "activations/layer11_attention_weight_min": -32.68217849731445,
      "activations/layer12_attention_weight_max": 17.475391387939453,
      "activations/layer12_attention_weight_min": -24.15800666809082,
      "activations/layer13_attention_weight_max": 34.538814544677734,
      "activations/layer13_attention_weight_min": -26.819177627563477,
      "activations/layer14_attention_weight_max": 32.800167083740234,
      "activations/layer14_attention_weight_min": -29.18971061706543,
      "activations/layer15_attention_weight_max": 31.383047103881836,
      "activations/layer15_attention_weight_min": -28.82513427734375,
      "activations/layer16_attention_weight_max": 30.459192276000977,
      "activations/layer16_attention_weight_min": -30.04248046875,
      "activations/layer17_attention_weight_max": 48.1216926574707,
      "activations/layer17_attention_weight_min": -44.3883056640625,
      "activations/layer18_attention_weight_max": 42.07205581665039,
      "activations/layer18_attention_weight_min": -38.231285095214844,
      "activations/layer19_attention_weight_max": 20.536069869995117,
      "activations/layer19_attention_weight_min": -22.31416893005371,
      "activations/layer1_attention_weight_max": 16.098649978637695,
      "activations/layer1_attention_weight_min": -14.682048797607422,
      "activations/layer20_attention_weight_max": 23.332334518432617,
      "activations/layer20_attention_weight_min": -20.699758529663086,
      "activations/layer21_attention_weight_max": 36.84426498413086,
      "activations/layer21_attention_weight_min": -25.515756607055664,
      "activations/layer22_attention_weight_max": 27.833446502685547,
      "activations/layer22_attention_weight_min": -27.06043815612793,
      "activations/layer23_attention_weight_max": 32.95079803466797,
      "activations/layer23_attention_weight_min": -23.749122619628906,
      "activations/layer2_attention_weight_max": 35.157772064208984,
      "activations/layer2_attention_weight_min": -31.025554656982422,
      "activations/layer3_attention_weight_max": 93.56966400146484,
      "activations/layer3_attention_weight_min": -94.75877380371094,
      "activations/layer4_attention_weight_max": 90.59772491455078,
      "activations/layer4_attention_weight_min": -87.72206115722656,
      "activations/layer5_attention_weight_max": 71.33967590332031,
      "activations/layer5_attention_weight_min": -74.95834350585938,
      "activations/layer6_attention_weight_max": 49.409385681152344,
      "activations/layer6_attention_weight_min": -50.543521881103516,
      "activations/layer7_attention_weight_max": 68.37205505371094,
      "activations/layer7_attention_weight_min": -60.282188415527344,
      "activations/layer8_attention_weight_max": 43.70970153808594,
      "activations/layer8_attention_weight_min": -46.07111358642578,
      "activations/layer9_attention_weight_max": 46.397850036621094,
      "activations/layer9_attention_weight_min": -43.40933609008789,
      "epoch": 14.41,
      "learning_rate": 5.765378787878787e-05,
      "loss": 2.7564,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4631,
      "eval_samples_per_second": 507.377,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4631,
      "eval_openwebtext_samples_per_second": 507.377,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.9493,
      "eval_wikitext_samples_per_second": 233.924,
      "step": 248000
    },
    {
      "epoch": 14.41,
      "eval_lambada_loss": 2.525390625,
      "eval_lambada_ppl": 12.495775472672934,
      "eval_lambada_runtime": 9.548,
      "eval_lambada_samples_per_second": 509.952,
      "step": 248000
    },
    {
      "activations/layer0_attention_weight_max": 15.138911247253418,
      "activations/layer0_attention_weight_min": -14.90095043182373,
      "activations/layer10_attention_weight_max": 35.57122802734375,
      "activations/layer10_attention_weight_min": -34.743289947509766,
      "activations/layer11_attention_weight_max": 31.581693649291992,
      "activations/layer11_attention_weight_min": -32.28461837768555,
      "activations/layer12_attention_weight_max": 20.04574203491211,
      "activations/layer12_attention_weight_min": -23.235319137573242,
      "activations/layer13_attention_weight_max": 36.3370361328125,
      "activations/layer13_attention_weight_min": -31.665977478027344,
      "activations/layer14_attention_weight_max": 35.021080017089844,
      "activations/layer14_attention_weight_min": -30.146163940429688,
      "activations/layer15_attention_weight_max": 31.3533992767334,
      "activations/layer15_attention_weight_min": -29.324214935302734,
      "activations/layer16_attention_weight_max": 34.26992416381836,
      "activations/layer16_attention_weight_min": -31.556489944458008,
      "activations/layer17_attention_weight_max": 52.488529205322266,
      "activations/layer17_attention_weight_min": -46.518856048583984,
      "activations/layer18_attention_weight_max": 44.71576690673828,
      "activations/layer18_attention_weight_min": -39.194122314453125,
      "activations/layer19_attention_weight_max": 22.879465103149414,
      "activations/layer19_attention_weight_min": -24.11659049987793,
      "activations/layer1_attention_weight_max": 15.972820281982422,
      "activations/layer1_attention_weight_min": -14.388814926147461,
      "activations/layer20_attention_weight_max": 20.766925811767578,
      "activations/layer20_attention_weight_min": -21.205347061157227,
      "activations/layer21_attention_weight_max": 34.74689483642578,
      "activations/layer21_attention_weight_min": -22.349817276000977,
      "activations/layer22_attention_weight_max": 29.188610076904297,
      "activations/layer22_attention_weight_min": -26.956377029418945,
      "activations/layer23_attention_weight_max": 35.952545166015625,
      "activations/layer23_attention_weight_min": -27.092304229736328,
      "activations/layer2_attention_weight_max": 32.87508010864258,
      "activations/layer2_attention_weight_min": -31.570837020874023,
      "activations/layer3_attention_weight_max": 90.91425323486328,
      "activations/layer3_attention_weight_min": -96.16019439697266,
      "activations/layer4_attention_weight_max": 98.74266815185547,
      "activations/layer4_attention_weight_min": -94.5502700805664,
      "activations/layer5_attention_weight_max": 68.7408447265625,
      "activations/layer5_attention_weight_min": -79.6253433227539,
      "activations/layer6_attention_weight_max": 53.994686126708984,
      "activations/layer6_attention_weight_min": -51.73689651489258,
      "activations/layer7_attention_weight_max": 65.38044738769531,
      "activations/layer7_attention_weight_min": -62.264122009277344,
      "activations/layer8_attention_weight_max": 46.64102554321289,
      "activations/layer8_attention_weight_min": -46.67128372192383,
      "activations/layer9_attention_weight_max": 42.329158782958984,
      "activations/layer9_attention_weight_min": -43.83349609375,
      "epoch": 14.41,
      "learning_rate": 5.7634848484848487e-05,
      "loss": 2.7347,
      "step": 248050
    },
    {
      "activations/layer0_attention_weight_max": 16.21236228942871,
      "activations/layer0_attention_weight_min": -14.596035957336426,
      "activations/layer10_attention_weight_max": 36.68299102783203,
      "activations/layer10_attention_weight_min": -35.29097366333008,
      "activations/layer11_attention_weight_max": 35.565589904785156,
      "activations/layer11_attention_weight_min": -33.35235595703125,
      "activations/layer12_attention_weight_max": 17.910573959350586,
      "activations/layer12_attention_weight_min": -25.20722198486328,
      "activations/layer13_attention_weight_max": 37.89837646484375,
      "activations/layer13_attention_weight_min": -27.870351791381836,
      "activations/layer14_attention_weight_max": 45.559242248535156,
      "activations/layer14_attention_weight_min": -31.82620620727539,
      "activations/layer15_attention_weight_max": 35.98539352416992,
      "activations/layer15_attention_weight_min": -29.5400390625,
      "activations/layer16_attention_weight_max": 31.79332160949707,
      "activations/layer16_attention_weight_min": -30.772903442382812,
      "activations/layer17_attention_weight_max": 49.5838737487793,
      "activations/layer17_attention_weight_min": -49.40443801879883,
      "activations/layer18_attention_weight_max": 44.381378173828125,
      "activations/layer18_attention_weight_min": -44.32081985473633,
      "activations/layer19_attention_weight_max": 22.262317657470703,
      "activations/layer19_attention_weight_min": -24.042097091674805,
      "activations/layer1_attention_weight_max": 15.92098331451416,
      "activations/layer1_attention_weight_min": -15.474528312683105,
      "activations/layer20_attention_weight_max": 21.747440338134766,
      "activations/layer20_attention_weight_min": -22.785789489746094,
      "activations/layer21_attention_weight_max": 34.57692337036133,
      "activations/layer21_attention_weight_min": -24.406442642211914,
      "activations/layer22_attention_weight_max": 28.913944244384766,
      "activations/layer22_attention_weight_min": -28.784990310668945,
      "activations/layer23_attention_weight_max": 32.57289123535156,
      "activations/layer23_attention_weight_min": -23.423742294311523,
      "activations/layer2_attention_weight_max": 32.38996887207031,
      "activations/layer2_attention_weight_min": -31.292966842651367,
      "activations/layer3_attention_weight_max": 96.86872863769531,
      "activations/layer3_attention_weight_min": -98.93988037109375,
      "activations/layer4_attention_weight_max": 91.6789321899414,
      "activations/layer4_attention_weight_min": -87.4864730834961,
      "activations/layer5_attention_weight_max": 70.23499298095703,
      "activations/layer5_attention_weight_min": -73.28923797607422,
      "activations/layer6_attention_weight_max": 48.31771469116211,
      "activations/layer6_attention_weight_min": -50.26288986206055,
      "activations/layer7_attention_weight_max": 69.4579849243164,
      "activations/layer7_attention_weight_min": -65.74207305908203,
      "activations/layer8_attention_weight_max": 48.45476150512695,
      "activations/layer8_attention_weight_min": -47.05659484863281,
      "activations/layer9_attention_weight_max": 55.86027145385742,
      "activations/layer9_attention_weight_min": -44.58642578125,
      "epoch": 14.42,
      "learning_rate": 5.761590909090909e-05,
      "loss": 2.7581,
      "step": 248100
    },
    {
      "activations/layer0_attention_weight_max": 17.21966552734375,
      "activations/layer0_attention_weight_min": -14.857070922851562,
      "activations/layer10_attention_weight_max": 36.17533493041992,
      "activations/layer10_attention_weight_min": -35.2984619140625,
      "activations/layer11_attention_weight_max": 33.22737503051758,
      "activations/layer11_attention_weight_min": -35.6400146484375,
      "activations/layer12_attention_weight_max": 18.977933883666992,
      "activations/layer12_attention_weight_min": -24.59758949279785,
      "activations/layer13_attention_weight_max": 35.550392150878906,
      "activations/layer13_attention_weight_min": -27.798669815063477,
      "activations/layer14_attention_weight_max": 36.36946105957031,
      "activations/layer14_attention_weight_min": -28.838825225830078,
      "activations/layer15_attention_weight_max": 32.32427215576172,
      "activations/layer15_attention_weight_min": -28.543962478637695,
      "activations/layer16_attention_weight_max": 30.74700164794922,
      "activations/layer16_attention_weight_min": -29.080846786499023,
      "activations/layer17_attention_weight_max": 48.27695846557617,
      "activations/layer17_attention_weight_min": -42.54505157470703,
      "activations/layer18_attention_weight_max": 45.52861404418945,
      "activations/layer18_attention_weight_min": -37.594547271728516,
      "activations/layer19_attention_weight_max": 21.3212833404541,
      "activations/layer19_attention_weight_min": -21.903512954711914,
      "activations/layer1_attention_weight_max": 15.734169960021973,
      "activations/layer1_attention_weight_min": -18.030324935913086,
      "activations/layer20_attention_weight_max": 21.41417694091797,
      "activations/layer20_attention_weight_min": -19.925676345825195,
      "activations/layer21_attention_weight_max": 33.13181686401367,
      "activations/layer21_attention_weight_min": -21.459590911865234,
      "activations/layer22_attention_weight_max": 26.53571128845215,
      "activations/layer22_attention_weight_min": -23.567808151245117,
      "activations/layer23_attention_weight_max": 33.445125579833984,
      "activations/layer23_attention_weight_min": -25.904666900634766,
      "activations/layer2_attention_weight_max": 32.014556884765625,
      "activations/layer2_attention_weight_min": -30.942829132080078,
      "activations/layer3_attention_weight_max": 89.97518920898438,
      "activations/layer3_attention_weight_min": -94.78486633300781,
      "activations/layer4_attention_weight_max": 91.65230560302734,
      "activations/layer4_attention_weight_min": -88.6023178100586,
      "activations/layer5_attention_weight_max": 74.3852767944336,
      "activations/layer5_attention_weight_min": -77.62982177734375,
      "activations/layer6_attention_weight_max": 49.327308654785156,
      "activations/layer6_attention_weight_min": -50.363582611083984,
      "activations/layer7_attention_weight_max": 63.51338195800781,
      "activations/layer7_attention_weight_min": -60.83806228637695,
      "activations/layer8_attention_weight_max": 43.53846740722656,
      "activations/layer8_attention_weight_min": -46.3438720703125,
      "activations/layer9_attention_weight_max": 43.18137741088867,
      "activations/layer9_attention_weight_min": -46.12370681762695,
      "epoch": 14.42,
      "learning_rate": 5.759696969696969e-05,
      "loss": 2.7638,
      "step": 248150
    },
    {
      "activations/layer0_attention_weight_max": 15.209637641906738,
      "activations/layer0_attention_weight_min": -14.42996883392334,
      "activations/layer10_attention_weight_max": 36.155120849609375,
      "activations/layer10_attention_weight_min": -33.81047058105469,
      "activations/layer11_attention_weight_max": 31.83715057373047,
      "activations/layer11_attention_weight_min": -30.90829086303711,
      "activations/layer12_attention_weight_max": 19.458633422851562,
      "activations/layer12_attention_weight_min": -24.0010986328125,
      "activations/layer13_attention_weight_max": 33.00959777832031,
      "activations/layer13_attention_weight_min": -26.95240020751953,
      "activations/layer14_attention_weight_max": 36.321861267089844,
      "activations/layer14_attention_weight_min": -32.60387420654297,
      "activations/layer15_attention_weight_max": 33.71311569213867,
      "activations/layer15_attention_weight_min": -33.00872802734375,
      "activations/layer16_attention_weight_max": 33.85356140136719,
      "activations/layer16_attention_weight_min": -34.05594253540039,
      "activations/layer17_attention_weight_max": 54.20813751220703,
      "activations/layer17_attention_weight_min": -48.44341278076172,
      "activations/layer18_attention_weight_max": 48.31633758544922,
      "activations/layer18_attention_weight_min": -42.97074890136719,
      "activations/layer19_attention_weight_max": 24.409225463867188,
      "activations/layer19_attention_weight_min": -23.513893127441406,
      "activations/layer1_attention_weight_max": 16.85792350769043,
      "activations/layer1_attention_weight_min": -15.968937873840332,
      "activations/layer20_attention_weight_max": 23.159080505371094,
      "activations/layer20_attention_weight_min": -20.50408363342285,
      "activations/layer21_attention_weight_max": 44.67835998535156,
      "activations/layer21_attention_weight_min": -26.845821380615234,
      "activations/layer22_attention_weight_max": 31.85428237915039,
      "activations/layer22_attention_weight_min": -26.17218780517578,
      "activations/layer23_attention_weight_max": 37.586830139160156,
      "activations/layer23_attention_weight_min": -22.660541534423828,
      "activations/layer2_attention_weight_max": 32.79319763183594,
      "activations/layer2_attention_weight_min": -31.277069091796875,
      "activations/layer3_attention_weight_max": 93.87250518798828,
      "activations/layer3_attention_weight_min": -99.01361083984375,
      "activations/layer4_attention_weight_max": 91.01559448242188,
      "activations/layer4_attention_weight_min": -88.31832122802734,
      "activations/layer5_attention_weight_max": 67.8475341796875,
      "activations/layer5_attention_weight_min": -75.0352783203125,
      "activations/layer6_attention_weight_max": 49.54579162597656,
      "activations/layer6_attention_weight_min": -50.8358268737793,
      "activations/layer7_attention_weight_max": 65.97750854492188,
      "activations/layer7_attention_weight_min": -60.31256866455078,
      "activations/layer8_attention_weight_max": 42.22589874267578,
      "activations/layer8_attention_weight_min": -45.274505615234375,
      "activations/layer9_attention_weight_max": 43.11223220825195,
      "activations/layer9_attention_weight_min": -44.09601593017578,
      "epoch": 14.42,
      "learning_rate": 5.75780303030303e-05,
      "loss": 2.7483,
      "step": 248200
    },
    {
      "activations/layer0_attention_weight_max": 15.566414833068848,
      "activations/layer0_attention_weight_min": -14.464083671569824,
      "activations/layer10_attention_weight_max": 43.882476806640625,
      "activations/layer10_attention_weight_min": -41.31132507324219,
      "activations/layer11_attention_weight_max": 39.08710861206055,
      "activations/layer11_attention_weight_min": -35.777713775634766,
      "activations/layer12_attention_weight_max": 19.35592269897461,
      "activations/layer12_attention_weight_min": -25.104328155517578,
      "activations/layer13_attention_weight_max": 54.589786529541016,
      "activations/layer13_attention_weight_min": -34.516761779785156,
      "activations/layer14_attention_weight_max": 56.94755172729492,
      "activations/layer14_attention_weight_min": -31.664491653442383,
      "activations/layer15_attention_weight_max": 49.03945541381836,
      "activations/layer15_attention_weight_min": -30.75066375732422,
      "activations/layer16_attention_weight_max": 33.068603515625,
      "activations/layer16_attention_weight_min": -30.141870498657227,
      "activations/layer17_attention_weight_max": 53.48552703857422,
      "activations/layer17_attention_weight_min": -45.156978607177734,
      "activations/layer18_attention_weight_max": 48.75251007080078,
      "activations/layer18_attention_weight_min": -40.831275939941406,
      "activations/layer19_attention_weight_max": 21.89625358581543,
      "activations/layer19_attention_weight_min": -23.26415252685547,
      "activations/layer1_attention_weight_max": 16.599245071411133,
      "activations/layer1_attention_weight_min": -16.086986541748047,
      "activations/layer20_attention_weight_max": 22.836288452148438,
      "activations/layer20_attention_weight_min": -21.649160385131836,
      "activations/layer21_attention_weight_max": 35.84379196166992,
      "activations/layer21_attention_weight_min": -26.230712890625,
      "activations/layer22_attention_weight_max": 30.79612922668457,
      "activations/layer22_attention_weight_min": -25.37116050720215,
      "activations/layer23_attention_weight_max": 38.21234893798828,
      "activations/layer23_attention_weight_min": -24.420217514038086,
      "activations/layer2_attention_weight_max": 32.175071716308594,
      "activations/layer2_attention_weight_min": -33.403724670410156,
      "activations/layer3_attention_weight_max": 92.69405364990234,
      "activations/layer3_attention_weight_min": -97.16936492919922,
      "activations/layer4_attention_weight_max": 90.72982025146484,
      "activations/layer4_attention_weight_min": -87.8957748413086,
      "activations/layer5_attention_weight_max": 70.00283813476562,
      "activations/layer5_attention_weight_min": -72.09501647949219,
      "activations/layer6_attention_weight_max": 49.103599548339844,
      "activations/layer6_attention_weight_min": -50.66029739379883,
      "activations/layer7_attention_weight_max": 78.42449951171875,
      "activations/layer7_attention_weight_min": -63.93028259277344,
      "activations/layer8_attention_weight_max": 54.08272171020508,
      "activations/layer8_attention_weight_min": -48.30042266845703,
      "activations/layer9_attention_weight_max": 62.38376998901367,
      "activations/layer9_attention_weight_min": -48.81766128540039,
      "epoch": 14.42,
      "learning_rate": 5.7559090909090906e-05,
      "loss": 2.7521,
      "step": 248250
    },
    {
      "activations/layer0_attention_weight_max": 15.681578636169434,
      "activations/layer0_attention_weight_min": -14.3319091796875,
      "activations/layer10_attention_weight_max": 35.54627990722656,
      "activations/layer10_attention_weight_min": -34.88747024536133,
      "activations/layer11_attention_weight_max": 30.998584747314453,
      "activations/layer11_attention_weight_min": -33.842857360839844,
      "activations/layer12_attention_weight_max": 18.352294921875,
      "activations/layer12_attention_weight_min": -27.070571899414062,
      "activations/layer13_attention_weight_max": 41.072227478027344,
      "activations/layer13_attention_weight_min": -28.800315856933594,
      "activations/layer14_attention_weight_max": 44.08943176269531,
      "activations/layer14_attention_weight_min": -32.842830657958984,
      "activations/layer15_attention_weight_max": 42.529483795166016,
      "activations/layer15_attention_weight_min": -30.095537185668945,
      "activations/layer16_attention_weight_max": 34.56587219238281,
      "activations/layer16_attention_weight_min": -31.801040649414062,
      "activations/layer17_attention_weight_max": 55.90752410888672,
      "activations/layer17_attention_weight_min": -48.22407913208008,
      "activations/layer18_attention_weight_max": 53.30779266357422,
      "activations/layer18_attention_weight_min": -42.7525520324707,
      "activations/layer19_attention_weight_max": 27.296072006225586,
      "activations/layer19_attention_weight_min": -24.56753921508789,
      "activations/layer1_attention_weight_max": 16.829614639282227,
      "activations/layer1_attention_weight_min": -15.10344123840332,
      "activations/layer20_attention_weight_max": 26.4224910736084,
      "activations/layer20_attention_weight_min": -22.716001510620117,
      "activations/layer21_attention_weight_max": 41.70427322387695,
      "activations/layer21_attention_weight_min": -21.47202491760254,
      "activations/layer22_attention_weight_max": 31.794031143188477,
      "activations/layer22_attention_weight_min": -26.74980926513672,
      "activations/layer23_attention_weight_max": 39.108245849609375,
      "activations/layer23_attention_weight_min": -24.510562896728516,
      "activations/layer2_attention_weight_max": 33.10923767089844,
      "activations/layer2_attention_weight_min": -30.798465728759766,
      "activations/layer3_attention_weight_max": 94.6468734741211,
      "activations/layer3_attention_weight_min": -91.57304382324219,
      "activations/layer4_attention_weight_max": 91.01571655273438,
      "activations/layer4_attention_weight_min": -85.19063568115234,
      "activations/layer5_attention_weight_max": 67.64191436767578,
      "activations/layer5_attention_weight_min": -76.6900634765625,
      "activations/layer6_attention_weight_max": 50.34069061279297,
      "activations/layer6_attention_weight_min": -51.316898345947266,
      "activations/layer7_attention_weight_max": 65.72992706298828,
      "activations/layer7_attention_weight_min": -62.21669006347656,
      "activations/layer8_attention_weight_max": 44.29237747192383,
      "activations/layer8_attention_weight_min": -48.2747917175293,
      "activations/layer9_attention_weight_max": 43.73999786376953,
      "activations/layer9_attention_weight_min": -44.98982238769531,
      "epoch": 14.43,
      "learning_rate": 5.7540151515151514e-05,
      "loss": 2.7647,
      "step": 248300
    },
    {
      "activations/layer0_attention_weight_max": 16.382272720336914,
      "activations/layer0_attention_weight_min": -14.314678192138672,
      "activations/layer10_attention_weight_max": 36.094696044921875,
      "activations/layer10_attention_weight_min": -33.931732177734375,
      "activations/layer11_attention_weight_max": 31.905996322631836,
      "activations/layer11_attention_weight_min": -32.32284927368164,
      "activations/layer12_attention_weight_max": 17.909881591796875,
      "activations/layer12_attention_weight_min": -30.581315994262695,
      "activations/layer13_attention_weight_max": 42.02241516113281,
      "activations/layer13_attention_weight_min": -28.086320877075195,
      "activations/layer14_attention_weight_max": 50.48566436767578,
      "activations/layer14_attention_weight_min": -30.074987411499023,
      "activations/layer15_attention_weight_max": 42.92323684692383,
      "activations/layer15_attention_weight_min": -29.280969619750977,
      "activations/layer16_attention_weight_max": 36.3084716796875,
      "activations/layer16_attention_weight_min": -31.476533889770508,
      "activations/layer17_attention_weight_max": 59.17183303833008,
      "activations/layer17_attention_weight_min": -47.243560791015625,
      "activations/layer18_attention_weight_max": 59.22092819213867,
      "activations/layer18_attention_weight_min": -42.511451721191406,
      "activations/layer19_attention_weight_max": 29.69192886352539,
      "activations/layer19_attention_weight_min": -27.93012809753418,
      "activations/layer1_attention_weight_max": 15.916542053222656,
      "activations/layer1_attention_weight_min": -14.609405517578125,
      "activations/layer20_attention_weight_max": 33.25159454345703,
      "activations/layer20_attention_weight_min": -22.19446563720703,
      "activations/layer21_attention_weight_max": 58.34904098510742,
      "activations/layer21_attention_weight_min": -27.45975112915039,
      "activations/layer22_attention_weight_max": 37.05467224121094,
      "activations/layer22_attention_weight_min": -26.84075164794922,
      "activations/layer23_attention_weight_max": 48.4322509765625,
      "activations/layer23_attention_weight_min": -24.366392135620117,
      "activations/layer2_attention_weight_max": 32.274925231933594,
      "activations/layer2_attention_weight_min": -30.39728355407715,
      "activations/layer3_attention_weight_max": 93.7754898071289,
      "activations/layer3_attention_weight_min": -93.03981018066406,
      "activations/layer4_attention_weight_max": 87.36231231689453,
      "activations/layer4_attention_weight_min": -84.64034271240234,
      "activations/layer5_attention_weight_max": 67.56343078613281,
      "activations/layer5_attention_weight_min": -73.96582794189453,
      "activations/layer6_attention_weight_max": 51.71804428100586,
      "activations/layer6_attention_weight_min": -52.715293884277344,
      "activations/layer7_attention_weight_max": 61.50340270996094,
      "activations/layer7_attention_weight_min": -65.12347412109375,
      "activations/layer8_attention_weight_max": 43.81829833984375,
      "activations/layer8_attention_weight_min": -48.00727844238281,
      "activations/layer9_attention_weight_max": 44.61116027832031,
      "activations/layer9_attention_weight_min": -47.705623626708984,
      "epoch": 14.43,
      "learning_rate": 5.7521212121212116e-05,
      "loss": 2.7346,
      "step": 248350
    },
    {
      "activations/layer0_attention_weight_max": 15.784427642822266,
      "activations/layer0_attention_weight_min": -14.451528549194336,
      "activations/layer10_attention_weight_max": 38.0883903503418,
      "activations/layer10_attention_weight_min": -36.57097244262695,
      "activations/layer11_attention_weight_max": 32.79914093017578,
      "activations/layer11_attention_weight_min": -32.94448471069336,
      "activations/layer12_attention_weight_max": 17.192201614379883,
      "activations/layer12_attention_weight_min": -26.323978424072266,
      "activations/layer13_attention_weight_max": 35.50025177001953,
      "activations/layer13_attention_weight_min": -30.367626190185547,
      "activations/layer14_attention_weight_max": 36.190185546875,
      "activations/layer14_attention_weight_min": -30.530771255493164,
      "activations/layer15_attention_weight_max": 34.59388732910156,
      "activations/layer15_attention_weight_min": -31.373825073242188,
      "activations/layer16_attention_weight_max": 32.88600158691406,
      "activations/layer16_attention_weight_min": -29.897613525390625,
      "activations/layer17_attention_weight_max": 50.13554763793945,
      "activations/layer17_attention_weight_min": -46.49070358276367,
      "activations/layer18_attention_weight_max": 47.885772705078125,
      "activations/layer18_attention_weight_min": -40.90274429321289,
      "activations/layer19_attention_weight_max": 20.94362449645996,
      "activations/layer19_attention_weight_min": -22.25972557067871,
      "activations/layer1_attention_weight_max": 15.767735481262207,
      "activations/layer1_attention_weight_min": -15.77988052368164,
      "activations/layer20_attention_weight_max": 21.26412010192871,
      "activations/layer20_attention_weight_min": -20.420040130615234,
      "activations/layer21_attention_weight_max": 35.39731979370117,
      "activations/layer21_attention_weight_min": -24.986398696899414,
      "activations/layer22_attention_weight_max": 28.041419982910156,
      "activations/layer22_attention_weight_min": -24.66185188293457,
      "activations/layer23_attention_weight_max": 35.68004608154297,
      "activations/layer23_attention_weight_min": -25.556232452392578,
      "activations/layer2_attention_weight_max": 33.24003601074219,
      "activations/layer2_attention_weight_min": -30.75081443786621,
      "activations/layer3_attention_weight_max": 93.432373046875,
      "activations/layer3_attention_weight_min": -96.3663330078125,
      "activations/layer4_attention_weight_max": 88.61284637451172,
      "activations/layer4_attention_weight_min": -88.01263427734375,
      "activations/layer5_attention_weight_max": 67.28787231445312,
      "activations/layer5_attention_weight_min": -77.20133209228516,
      "activations/layer6_attention_weight_max": 50.42100143432617,
      "activations/layer6_attention_weight_min": -51.57857131958008,
      "activations/layer7_attention_weight_max": 66.03484344482422,
      "activations/layer7_attention_weight_min": -62.0406608581543,
      "activations/layer8_attention_weight_max": 47.39698028564453,
      "activations/layer8_attention_weight_min": -47.55904769897461,
      "activations/layer9_attention_weight_max": 44.986324310302734,
      "activations/layer9_attention_weight_min": -44.64439010620117,
      "epoch": 14.43,
      "learning_rate": 5.750227272727272e-05,
      "loss": 2.7498,
      "step": 248400
    },
    {
      "activations/layer0_attention_weight_max": 15.198883056640625,
      "activations/layer0_attention_weight_min": -14.329854965209961,
      "activations/layer10_attention_weight_max": 36.5528678894043,
      "activations/layer10_attention_weight_min": -35.84876251220703,
      "activations/layer11_attention_weight_max": 32.569297790527344,
      "activations/layer11_attention_weight_min": -33.84264373779297,
      "activations/layer12_attention_weight_max": 17.466732025146484,
      "activations/layer12_attention_weight_min": -27.2994441986084,
      "activations/layer13_attention_weight_max": 33.36649703979492,
      "activations/layer13_attention_weight_min": -26.621225357055664,
      "activations/layer14_attention_weight_max": 37.697086334228516,
      "activations/layer14_attention_weight_min": -29.156513214111328,
      "activations/layer15_attention_weight_max": 34.14799880981445,
      "activations/layer15_attention_weight_min": -29.398393630981445,
      "activations/layer16_attention_weight_max": 31.772964477539062,
      "activations/layer16_attention_weight_min": -31.40989112854004,
      "activations/layer17_attention_weight_max": 53.560726165771484,
      "activations/layer17_attention_weight_min": -47.79142761230469,
      "activations/layer18_attention_weight_max": 45.06140899658203,
      "activations/layer18_attention_weight_min": -40.91961669921875,
      "activations/layer19_attention_weight_max": 22.102638244628906,
      "activations/layer19_attention_weight_min": -22.052980422973633,
      "activations/layer1_attention_weight_max": 17.406068801879883,
      "activations/layer1_attention_weight_min": -16.003890991210938,
      "activations/layer20_attention_weight_max": 22.060958862304688,
      "activations/layer20_attention_weight_min": -21.983701705932617,
      "activations/layer21_attention_weight_max": 34.97042465209961,
      "activations/layer21_attention_weight_min": -21.80812644958496,
      "activations/layer22_attention_weight_max": 28.711565017700195,
      "activations/layer22_attention_weight_min": -24.84219741821289,
      "activations/layer23_attention_weight_max": 36.82137680053711,
      "activations/layer23_attention_weight_min": -22.963010787963867,
      "activations/layer2_attention_weight_max": 33.446678161621094,
      "activations/layer2_attention_weight_min": -31.060239791870117,
      "activations/layer3_attention_weight_max": 96.23351287841797,
      "activations/layer3_attention_weight_min": -95.42333984375,
      "activations/layer4_attention_weight_max": 93.57882690429688,
      "activations/layer4_attention_weight_min": -86.86791229248047,
      "activations/layer5_attention_weight_max": 70.79837036132812,
      "activations/layer5_attention_weight_min": -72.91736602783203,
      "activations/layer6_attention_weight_max": 51.991981506347656,
      "activations/layer6_attention_weight_min": -54.52970504760742,
      "activations/layer7_attention_weight_max": 65.35236358642578,
      "activations/layer7_attention_weight_min": -66.86163330078125,
      "activations/layer8_attention_weight_max": 47.61249923706055,
      "activations/layer8_attention_weight_min": -49.414329528808594,
      "activations/layer9_attention_weight_max": 48.59136199951172,
      "activations/layer9_attention_weight_min": -50.35405349731445,
      "epoch": 14.44,
      "learning_rate": 5.748333333333333e-05,
      "loss": 2.7515,
      "step": 248450
    },
    {
      "activations/layer0_attention_weight_max": 15.44222354888916,
      "activations/layer0_attention_weight_min": -13.794553756713867,
      "activations/layer10_attention_weight_max": 36.31117630004883,
      "activations/layer10_attention_weight_min": -35.54994583129883,
      "activations/layer11_attention_weight_max": 33.42427062988281,
      "activations/layer11_attention_weight_min": -35.251197814941406,
      "activations/layer12_attention_weight_max": 22.45108985900879,
      "activations/layer12_attention_weight_min": -31.091529846191406,
      "activations/layer13_attention_weight_max": 35.872947692871094,
      "activations/layer13_attention_weight_min": -28.800127029418945,
      "activations/layer14_attention_weight_max": 37.898277282714844,
      "activations/layer14_attention_weight_min": -30.93833351135254,
      "activations/layer15_attention_weight_max": 34.89458084106445,
      "activations/layer15_attention_weight_min": -29.901527404785156,
      "activations/layer16_attention_weight_max": 32.4481201171875,
      "activations/layer16_attention_weight_min": -30.312780380249023,
      "activations/layer17_attention_weight_max": 52.492881774902344,
      "activations/layer17_attention_weight_min": -41.966670989990234,
      "activations/layer18_attention_weight_max": 50.70024871826172,
      "activations/layer18_attention_weight_min": -37.55943298339844,
      "activations/layer19_attention_weight_max": 22.0518856048584,
      "activations/layer19_attention_weight_min": -21.656827926635742,
      "activations/layer1_attention_weight_max": 17.318038940429688,
      "activations/layer1_attention_weight_min": -14.322566032409668,
      "activations/layer20_attention_weight_max": 24.357086181640625,
      "activations/layer20_attention_weight_min": -22.047178268432617,
      "activations/layer21_attention_weight_max": 43.40061569213867,
      "activations/layer21_attention_weight_min": -22.642839431762695,
      "activations/layer22_attention_weight_max": 30.11298370361328,
      "activations/layer22_attention_weight_min": -24.229394912719727,
      "activations/layer23_attention_weight_max": 41.42536926269531,
      "activations/layer23_attention_weight_min": -22.684452056884766,
      "activations/layer2_attention_weight_max": 33.78828048706055,
      "activations/layer2_attention_weight_min": -33.157073974609375,
      "activations/layer3_attention_weight_max": 94.06893920898438,
      "activations/layer3_attention_weight_min": -95.40374755859375,
      "activations/layer4_attention_weight_max": 92.9527816772461,
      "activations/layer4_attention_weight_min": -90.35347747802734,
      "activations/layer5_attention_weight_max": 72.29589080810547,
      "activations/layer5_attention_weight_min": -82.40298461914062,
      "activations/layer6_attention_weight_max": 47.8632698059082,
      "activations/layer6_attention_weight_min": -49.20842742919922,
      "activations/layer7_attention_weight_max": 66.74864196777344,
      "activations/layer7_attention_weight_min": -62.75894546508789,
      "activations/layer8_attention_weight_max": 45.07393264770508,
      "activations/layer8_attention_weight_min": -46.59608840942383,
      "activations/layer9_attention_weight_max": 55.377960205078125,
      "activations/layer9_attention_weight_min": -48.10137939453125,
      "epoch": 14.44,
      "learning_rate": 5.7464393939393934e-05,
      "loss": 2.7593,
      "step": 248500
    },
    {
      "activations/layer0_attention_weight_max": 16.3045711517334,
      "activations/layer0_attention_weight_min": -13.746450424194336,
      "activations/layer10_attention_weight_max": 37.78948974609375,
      "activations/layer10_attention_weight_min": -35.13878631591797,
      "activations/layer11_attention_weight_max": 33.868553161621094,
      "activations/layer11_attention_weight_min": -31.81899642944336,
      "activations/layer12_attention_weight_max": 22.471521377563477,
      "activations/layer12_attention_weight_min": -34.713661193847656,
      "activations/layer13_attention_weight_max": 36.385276794433594,
      "activations/layer13_attention_weight_min": -28.2318058013916,
      "activations/layer14_attention_weight_max": 33.84580612182617,
      "activations/layer14_attention_weight_min": -29.14410972595215,
      "activations/layer15_attention_weight_max": 31.791095733642578,
      "activations/layer15_attention_weight_min": -29.68252182006836,
      "activations/layer16_attention_weight_max": 35.78776168823242,
      "activations/layer16_attention_weight_min": -31.194408416748047,
      "activations/layer17_attention_weight_max": 49.18313980102539,
      "activations/layer17_attention_weight_min": -43.56110382080078,
      "activations/layer18_attention_weight_max": 42.88134002685547,
      "activations/layer18_attention_weight_min": -37.58340835571289,
      "activations/layer19_attention_weight_max": 19.943479537963867,
      "activations/layer19_attention_weight_min": -21.387357711791992,
      "activations/layer1_attention_weight_max": 16.494937896728516,
      "activations/layer1_attention_weight_min": -14.198099136352539,
      "activations/layer20_attention_weight_max": 20.674236297607422,
      "activations/layer20_attention_weight_min": -22.29582405090332,
      "activations/layer21_attention_weight_max": 34.901920318603516,
      "activations/layer21_attention_weight_min": -21.126184463500977,
      "activations/layer22_attention_weight_max": 29.61646842956543,
      "activations/layer22_attention_weight_min": -26.36936378479004,
      "activations/layer23_attention_weight_max": 39.44983673095703,
      "activations/layer23_attention_weight_min": -22.096006393432617,
      "activations/layer2_attention_weight_max": 30.12201690673828,
      "activations/layer2_attention_weight_min": -30.70836639404297,
      "activations/layer3_attention_weight_max": 95.20726013183594,
      "activations/layer3_attention_weight_min": -97.15816497802734,
      "activations/layer4_attention_weight_max": 90.22374725341797,
      "activations/layer4_attention_weight_min": -87.23078155517578,
      "activations/layer5_attention_weight_max": 71.11833953857422,
      "activations/layer5_attention_weight_min": -75.1396484375,
      "activations/layer6_attention_weight_max": 50.62809371948242,
      "activations/layer6_attention_weight_min": -50.48923873901367,
      "activations/layer7_attention_weight_max": 68.1656494140625,
      "activations/layer7_attention_weight_min": -60.563438415527344,
      "activations/layer8_attention_weight_max": 47.12645721435547,
      "activations/layer8_attention_weight_min": -49.96377182006836,
      "activations/layer9_attention_weight_max": 50.89321517944336,
      "activations/layer9_attention_weight_min": -46.77848434448242,
      "epoch": 14.44,
      "learning_rate": 5.7445833333333325e-05,
      "loss": 2.7425,
      "step": 248550
    },
    {
      "activations/layer0_attention_weight_max": 17.608064651489258,
      "activations/layer0_attention_weight_min": -14.260604858398438,
      "activations/layer10_attention_weight_max": 37.40757369995117,
      "activations/layer10_attention_weight_min": -37.16802215576172,
      "activations/layer11_attention_weight_max": 31.473934173583984,
      "activations/layer11_attention_weight_min": -34.464874267578125,
      "activations/layer12_attention_weight_max": 18.006906509399414,
      "activations/layer12_attention_weight_min": -24.217668533325195,
      "activations/layer13_attention_weight_max": 35.06914138793945,
      "activations/layer13_attention_weight_min": -28.420177459716797,
      "activations/layer14_attention_weight_max": 34.00966262817383,
      "activations/layer14_attention_weight_min": -29.556398391723633,
      "activations/layer15_attention_weight_max": 31.600778579711914,
      "activations/layer15_attention_weight_min": -28.580446243286133,
      "activations/layer16_attention_weight_max": 32.810054779052734,
      "activations/layer16_attention_weight_min": -29.735807418823242,
      "activations/layer17_attention_weight_max": 54.423770904541016,
      "activations/layer17_attention_weight_min": -42.97171401977539,
      "activations/layer18_attention_weight_max": 50.27332305908203,
      "activations/layer18_attention_weight_min": -37.29375076293945,
      "activations/layer19_attention_weight_max": 23.239213943481445,
      "activations/layer19_attention_weight_min": -23.540250778198242,
      "activations/layer1_attention_weight_max": 17.675901412963867,
      "activations/layer1_attention_weight_min": -17.70963478088379,
      "activations/layer20_attention_weight_max": 24.219724655151367,
      "activations/layer20_attention_weight_min": -21.231698989868164,
      "activations/layer21_attention_weight_max": 39.42429733276367,
      "activations/layer21_attention_weight_min": -23.768163681030273,
      "activations/layer22_attention_weight_max": 30.561080932617188,
      "activations/layer22_attention_weight_min": -25.3341121673584,
      "activations/layer23_attention_weight_max": 34.66659164428711,
      "activations/layer23_attention_weight_min": -24.135879516601562,
      "activations/layer2_attention_weight_max": 31.8731689453125,
      "activations/layer2_attention_weight_min": -30.551223754882812,
      "activations/layer3_attention_weight_max": 99.10612487792969,
      "activations/layer3_attention_weight_min": -94.38583374023438,
      "activations/layer4_attention_weight_max": 90.41413116455078,
      "activations/layer4_attention_weight_min": -91.3346176147461,
      "activations/layer5_attention_weight_max": 68.76797485351562,
      "activations/layer5_attention_weight_min": -75.63722229003906,
      "activations/layer6_attention_weight_max": 52.14304733276367,
      "activations/layer6_attention_weight_min": -50.77061080932617,
      "activations/layer7_attention_weight_max": 66.92584228515625,
      "activations/layer7_attention_weight_min": -63.20038986206055,
      "activations/layer8_attention_weight_max": 45.076263427734375,
      "activations/layer8_attention_weight_min": -47.96467971801758,
      "activations/layer9_attention_weight_max": 44.30629348754883,
      "activations/layer9_attention_weight_min": -46.3223991394043,
      "epoch": 14.45,
      "learning_rate": 5.742689393939393e-05,
      "loss": 2.7548,
      "step": 248600
    },
    {
      "activations/layer0_attention_weight_max": 16.721878051757812,
      "activations/layer0_attention_weight_min": -14.043888092041016,
      "activations/layer10_attention_weight_max": 34.93315124511719,
      "activations/layer10_attention_weight_min": -34.84504318237305,
      "activations/layer11_attention_weight_max": 30.914709091186523,
      "activations/layer11_attention_weight_min": -34.35235595703125,
      "activations/layer12_attention_weight_max": 19.85127830505371,
      "activations/layer12_attention_weight_min": -24.312936782836914,
      "activations/layer13_attention_weight_max": 33.95298385620117,
      "activations/layer13_attention_weight_min": -27.511573791503906,
      "activations/layer14_attention_weight_max": 35.01980209350586,
      "activations/layer14_attention_weight_min": -29.687347412109375,
      "activations/layer15_attention_weight_max": 34.43892288208008,
      "activations/layer15_attention_weight_min": -31.566341400146484,
      "activations/layer16_attention_weight_max": 30.678735733032227,
      "activations/layer16_attention_weight_min": -32.09034729003906,
      "activations/layer17_attention_weight_max": 51.021995544433594,
      "activations/layer17_attention_weight_min": -44.32307815551758,
      "activations/layer18_attention_weight_max": 45.214759826660156,
      "activations/layer18_attention_weight_min": -38.2216682434082,
      "activations/layer19_attention_weight_max": 23.45635414123535,
      "activations/layer19_attention_weight_min": -21.89180564880371,
      "activations/layer1_attention_weight_max": 17.466814041137695,
      "activations/layer1_attention_weight_min": -15.07465934753418,
      "activations/layer20_attention_weight_max": 23.918798446655273,
      "activations/layer20_attention_weight_min": -19.93097686767578,
      "activations/layer21_attention_weight_max": 34.858829498291016,
      "activations/layer21_attention_weight_min": -22.885726928710938,
      "activations/layer22_attention_weight_max": 29.616451263427734,
      "activations/layer22_attention_weight_min": -23.848976135253906,
      "activations/layer23_attention_weight_max": 35.39882278442383,
      "activations/layer23_attention_weight_min": -24.653335571289062,
      "activations/layer2_attention_weight_max": 32.66025924682617,
      "activations/layer2_attention_weight_min": -32.74730682373047,
      "activations/layer3_attention_weight_max": 96.94036102294922,
      "activations/layer3_attention_weight_min": -101.9759750366211,
      "activations/layer4_attention_weight_max": 86.0224609375,
      "activations/layer4_attention_weight_min": -89.084716796875,
      "activations/layer5_attention_weight_max": 68.39998626708984,
      "activations/layer5_attention_weight_min": -77.2628173828125,
      "activations/layer6_attention_weight_max": 49.49897003173828,
      "activations/layer6_attention_weight_min": -50.96369171142578,
      "activations/layer7_attention_weight_max": 67.39795684814453,
      "activations/layer7_attention_weight_min": -62.57383728027344,
      "activations/layer8_attention_weight_max": 43.23686599731445,
      "activations/layer8_attention_weight_min": -46.73378372192383,
      "activations/layer9_attention_weight_max": 43.55408477783203,
      "activations/layer9_attention_weight_min": -45.1342887878418,
      "epoch": 14.45,
      "learning_rate": 5.740795454545454e-05,
      "loss": 2.758,
      "step": 248650
    },
    {
      "activations/layer0_attention_weight_max": 15.976954460144043,
      "activations/layer0_attention_weight_min": -14.139355659484863,
      "activations/layer10_attention_weight_max": 37.01697540283203,
      "activations/layer10_attention_weight_min": -34.724830627441406,
      "activations/layer11_attention_weight_max": 31.855648040771484,
      "activations/layer11_attention_weight_min": -33.24031066894531,
      "activations/layer12_attention_weight_max": 17.8207950592041,
      "activations/layer12_attention_weight_min": -24.068159103393555,
      "activations/layer13_attention_weight_max": 36.52132797241211,
      "activations/layer13_attention_weight_min": -30.09833526611328,
      "activations/layer14_attention_weight_max": 37.214569091796875,
      "activations/layer14_attention_weight_min": -29.6318302154541,
      "activations/layer15_attention_weight_max": 33.80195617675781,
      "activations/layer15_attention_weight_min": -29.778095245361328,
      "activations/layer16_attention_weight_max": 34.08354568481445,
      "activations/layer16_attention_weight_min": -31.228303909301758,
      "activations/layer17_attention_weight_max": 56.83026885986328,
      "activations/layer17_attention_weight_min": -45.23140335083008,
      "activations/layer18_attention_weight_max": 47.2514762878418,
      "activations/layer18_attention_weight_min": -36.34457778930664,
      "activations/layer19_attention_weight_max": 22.57625389099121,
      "activations/layer19_attention_weight_min": -21.613615036010742,
      "activations/layer1_attention_weight_max": 16.850563049316406,
      "activations/layer1_attention_weight_min": -16.001760482788086,
      "activations/layer20_attention_weight_max": 23.592313766479492,
      "activations/layer20_attention_weight_min": -21.917373657226562,
      "activations/layer21_attention_weight_max": 41.37041091918945,
      "activations/layer21_attention_weight_min": -24.219390869140625,
      "activations/layer22_attention_weight_max": 31.36353302001953,
      "activations/layer22_attention_weight_min": -24.65857696533203,
      "activations/layer23_attention_weight_max": 44.3106689453125,
      "activations/layer23_attention_weight_min": -23.8205623626709,
      "activations/layer2_attention_weight_max": 30.610820770263672,
      "activations/layer2_attention_weight_min": -29.43987274169922,
      "activations/layer3_attention_weight_max": 93.32067108154297,
      "activations/layer3_attention_weight_min": -91.54891967773438,
      "activations/layer4_attention_weight_max": 88.69444274902344,
      "activations/layer4_attention_weight_min": -85.67504119873047,
      "activations/layer5_attention_weight_max": 72.82260131835938,
      "activations/layer5_attention_weight_min": -75.95748901367188,
      "activations/layer6_attention_weight_max": 49.129981994628906,
      "activations/layer6_attention_weight_min": -52.421504974365234,
      "activations/layer7_attention_weight_max": 69.3151626586914,
      "activations/layer7_attention_weight_min": -64.64305114746094,
      "activations/layer8_attention_weight_max": 43.958499908447266,
      "activations/layer8_attention_weight_min": -48.41267013549805,
      "activations/layer9_attention_weight_max": 47.035675048828125,
      "activations/layer9_attention_weight_min": -46.758628845214844,
      "epoch": 14.45,
      "learning_rate": 5.738901515151515e-05,
      "loss": 2.7553,
      "step": 248700
    },
    {
      "activations/layer0_attention_weight_max": 16.16954803466797,
      "activations/layer0_attention_weight_min": -14.378114700317383,
      "activations/layer10_attention_weight_max": 37.9223518371582,
      "activations/layer10_attention_weight_min": -36.047298431396484,
      "activations/layer11_attention_weight_max": 33.52549743652344,
      "activations/layer11_attention_weight_min": -34.02547836303711,
      "activations/layer12_attention_weight_max": 17.0356388092041,
      "activations/layer12_attention_weight_min": -26.91889190673828,
      "activations/layer13_attention_weight_max": 32.19886016845703,
      "activations/layer13_attention_weight_min": -27.851552963256836,
      "activations/layer14_attention_weight_max": 33.4903564453125,
      "activations/layer14_attention_weight_min": -30.011198043823242,
      "activations/layer15_attention_weight_max": 31.224376678466797,
      "activations/layer15_attention_weight_min": -29.738723754882812,
      "activations/layer16_attention_weight_max": 32.05862045288086,
      "activations/layer16_attention_weight_min": -30.28620147705078,
      "activations/layer17_attention_weight_max": 49.211483001708984,
      "activations/layer17_attention_weight_min": -43.48637771606445,
      "activations/layer18_attention_weight_max": 47.55314636230469,
      "activations/layer18_attention_weight_min": -38.22806930541992,
      "activations/layer19_attention_weight_max": 21.8536434173584,
      "activations/layer19_attention_weight_min": -21.11945343017578,
      "activations/layer1_attention_weight_max": 16.652111053466797,
      "activations/layer1_attention_weight_min": -14.09780216217041,
      "activations/layer20_attention_weight_max": 20.509092330932617,
      "activations/layer20_attention_weight_min": -23.107297897338867,
      "activations/layer21_attention_weight_max": 37.14821243286133,
      "activations/layer21_attention_weight_min": -22.860605239868164,
      "activations/layer22_attention_weight_max": 30.838939666748047,
      "activations/layer22_attention_weight_min": -25.40351676940918,
      "activations/layer23_attention_weight_max": 35.269676208496094,
      "activations/layer23_attention_weight_min": -22.97171401977539,
      "activations/layer2_attention_weight_max": 33.06707763671875,
      "activations/layer2_attention_weight_min": -30.14443588256836,
      "activations/layer3_attention_weight_max": 91.29788970947266,
      "activations/layer3_attention_weight_min": -94.03816223144531,
      "activations/layer4_attention_weight_max": 86.41981506347656,
      "activations/layer4_attention_weight_min": -87.70650482177734,
      "activations/layer5_attention_weight_max": 67.22367858886719,
      "activations/layer5_attention_weight_min": -74.25301361083984,
      "activations/layer6_attention_weight_max": 52.23444747924805,
      "activations/layer6_attention_weight_min": -51.4192008972168,
      "activations/layer7_attention_weight_max": 66.03423309326172,
      "activations/layer7_attention_weight_min": -63.586429595947266,
      "activations/layer8_attention_weight_max": 46.96027755737305,
      "activations/layer8_attention_weight_min": -49.32880401611328,
      "activations/layer9_attention_weight_max": 45.192047119140625,
      "activations/layer9_attention_weight_min": -45.459930419921875,
      "epoch": 14.45,
      "learning_rate": 5.737007575757575e-05,
      "loss": 2.7564,
      "step": 248750
    },
    {
      "activations/layer0_attention_weight_max": 16.16059112548828,
      "activations/layer0_attention_weight_min": -13.771195411682129,
      "activations/layer10_attention_weight_max": 37.800052642822266,
      "activations/layer10_attention_weight_min": -35.921875,
      "activations/layer11_attention_weight_max": 32.636878967285156,
      "activations/layer11_attention_weight_min": -34.245574951171875,
      "activations/layer12_attention_weight_max": 18.33469581604004,
      "activations/layer12_attention_weight_min": -23.14288902282715,
      "activations/layer13_attention_weight_max": 36.88031768798828,
      "activations/layer13_attention_weight_min": -29.364986419677734,
      "activations/layer14_attention_weight_max": 34.72930908203125,
      "activations/layer14_attention_weight_min": -29.549028396606445,
      "activations/layer15_attention_weight_max": 32.400875091552734,
      "activations/layer15_attention_weight_min": -28.84603500366211,
      "activations/layer16_attention_weight_max": 29.94541358947754,
      "activations/layer16_attention_weight_min": -29.536422729492188,
      "activations/layer17_attention_weight_max": 49.17313003540039,
      "activations/layer17_attention_weight_min": -42.388038635253906,
      "activations/layer18_attention_weight_max": 44.91355895996094,
      "activations/layer18_attention_weight_min": -37.178768157958984,
      "activations/layer19_attention_weight_max": 24.58800506591797,
      "activations/layer19_attention_weight_min": -23.632089614868164,
      "activations/layer1_attention_weight_max": 16.737289428710938,
      "activations/layer1_attention_weight_min": -15.734842300415039,
      "activations/layer20_attention_weight_max": 21.648231506347656,
      "activations/layer20_attention_weight_min": -21.995872497558594,
      "activations/layer21_attention_weight_max": 33.89989471435547,
      "activations/layer21_attention_weight_min": -22.293638229370117,
      "activations/layer22_attention_weight_max": 27.478803634643555,
      "activations/layer22_attention_weight_min": -24.991331100463867,
      "activations/layer23_attention_weight_max": 35.955345153808594,
      "activations/layer23_attention_weight_min": -25.187801361083984,
      "activations/layer2_attention_weight_max": 32.10429382324219,
      "activations/layer2_attention_weight_min": -34.8797721862793,
      "activations/layer3_attention_weight_max": 93.80354309082031,
      "activations/layer3_attention_weight_min": -96.43714141845703,
      "activations/layer4_attention_weight_max": 90.40579986572266,
      "activations/layer4_attention_weight_min": -89.4147720336914,
      "activations/layer5_attention_weight_max": 70.35404968261719,
      "activations/layer5_attention_weight_min": -76.86866760253906,
      "activations/layer6_attention_weight_max": 52.27665328979492,
      "activations/layer6_attention_weight_min": -51.603450775146484,
      "activations/layer7_attention_weight_max": 65.37957763671875,
      "activations/layer7_attention_weight_min": -64.0556869506836,
      "activations/layer8_attention_weight_max": 46.55030059814453,
      "activations/layer8_attention_weight_min": -51.70976638793945,
      "activations/layer9_attention_weight_max": 46.003597259521484,
      "activations/layer9_attention_weight_min": -49.566619873046875,
      "epoch": 14.46,
      "learning_rate": 5.735113636363635e-05,
      "loss": 2.7536,
      "step": 248800
    },
    {
      "activations/layer0_attention_weight_max": 15.61851978302002,
      "activations/layer0_attention_weight_min": -14.278416633605957,
      "activations/layer10_attention_weight_max": 35.224884033203125,
      "activations/layer10_attention_weight_min": -34.19455337524414,
      "activations/layer11_attention_weight_max": 31.703725814819336,
      "activations/layer11_attention_weight_min": -32.14148712158203,
      "activations/layer12_attention_weight_max": 18.24425506591797,
      "activations/layer12_attention_weight_min": -27.174985885620117,
      "activations/layer13_attention_weight_max": 37.853057861328125,
      "activations/layer13_attention_weight_min": -31.890796661376953,
      "activations/layer14_attention_weight_max": 35.407772064208984,
      "activations/layer14_attention_weight_min": -30.454790115356445,
      "activations/layer15_attention_weight_max": 32.19330596923828,
      "activations/layer15_attention_weight_min": -30.27592658996582,
      "activations/layer16_attention_weight_max": 33.87643814086914,
      "activations/layer16_attention_weight_min": -32.36691665649414,
      "activations/layer17_attention_weight_max": 51.06892013549805,
      "activations/layer17_attention_weight_min": -44.408660888671875,
      "activations/layer18_attention_weight_max": 48.411529541015625,
      "activations/layer18_attention_weight_min": -39.87389373779297,
      "activations/layer19_attention_weight_max": 23.215255737304688,
      "activations/layer19_attention_weight_min": -21.3603515625,
      "activations/layer1_attention_weight_max": 16.094942092895508,
      "activations/layer1_attention_weight_min": -14.69688892364502,
      "activations/layer20_attention_weight_max": 21.513919830322266,
      "activations/layer20_attention_weight_min": -22.11833381652832,
      "activations/layer21_attention_weight_max": 37.49396514892578,
      "activations/layer21_attention_weight_min": -21.515838623046875,
      "activations/layer22_attention_weight_max": 27.733057022094727,
      "activations/layer22_attention_weight_min": -24.147689819335938,
      "activations/layer23_attention_weight_max": 36.696861267089844,
      "activations/layer23_attention_weight_min": -24.974454879760742,
      "activations/layer2_attention_weight_max": 31.96352195739746,
      "activations/layer2_attention_weight_min": -30.721839904785156,
      "activations/layer3_attention_weight_max": 93.92023468017578,
      "activations/layer3_attention_weight_min": -94.57898712158203,
      "activations/layer4_attention_weight_max": 88.26410675048828,
      "activations/layer4_attention_weight_min": -86.84912109375,
      "activations/layer5_attention_weight_max": 68.494384765625,
      "activations/layer5_attention_weight_min": -76.74716186523438,
      "activations/layer6_attention_weight_max": 49.15859603881836,
      "activations/layer6_attention_weight_min": -48.36452865600586,
      "activations/layer7_attention_weight_max": 65.50869750976562,
      "activations/layer7_attention_weight_min": -63.80626678466797,
      "activations/layer8_attention_weight_max": 44.58461380004883,
      "activations/layer8_attention_weight_min": -46.7745475769043,
      "activations/layer9_attention_weight_max": 47.765159606933594,
      "activations/layer9_attention_weight_min": -44.24856948852539,
      "epoch": 14.46,
      "learning_rate": 5.733219696969697e-05,
      "loss": 2.747,
      "step": 248850
    },
    {
      "activations/layer0_attention_weight_max": 15.513161659240723,
      "activations/layer0_attention_weight_min": -14.2471284866333,
      "activations/layer10_attention_weight_max": 35.350738525390625,
      "activations/layer10_attention_weight_min": -34.81019973754883,
      "activations/layer11_attention_weight_max": 33.15203094482422,
      "activations/layer11_attention_weight_min": -32.971946716308594,
      "activations/layer12_attention_weight_max": 17.547603607177734,
      "activations/layer12_attention_weight_min": -23.970375061035156,
      "activations/layer13_attention_weight_max": 40.21767044067383,
      "activations/layer13_attention_weight_min": -29.68889617919922,
      "activations/layer14_attention_weight_max": 37.54584503173828,
      "activations/layer14_attention_weight_min": -31.949975967407227,
      "activations/layer15_attention_weight_max": 36.16696548461914,
      "activations/layer15_attention_weight_min": -30.939102172851562,
      "activations/layer16_attention_weight_max": 34.83725357055664,
      "activations/layer16_attention_weight_min": -31.868579864501953,
      "activations/layer17_attention_weight_max": 55.65085220336914,
      "activations/layer17_attention_weight_min": -47.24586868286133,
      "activations/layer18_attention_weight_max": 50.35859298706055,
      "activations/layer18_attention_weight_min": -40.767398834228516,
      "activations/layer19_attention_weight_max": 26.90105628967285,
      "activations/layer19_attention_weight_min": -22.008155822753906,
      "activations/layer1_attention_weight_max": 17.42015266418457,
      "activations/layer1_attention_weight_min": -13.872420310974121,
      "activations/layer20_attention_weight_max": 23.993783950805664,
      "activations/layer20_attention_weight_min": -21.53072738647461,
      "activations/layer21_attention_weight_max": 36.49168014526367,
      "activations/layer21_attention_weight_min": -20.710744857788086,
      "activations/layer22_attention_weight_max": 35.08425521850586,
      "activations/layer22_attention_weight_min": -27.67917823791504,
      "activations/layer23_attention_weight_max": 36.185768127441406,
      "activations/layer23_attention_weight_min": -22.94087028503418,
      "activations/layer2_attention_weight_max": 32.687618255615234,
      "activations/layer2_attention_weight_min": -32.23193359375,
      "activations/layer3_attention_weight_max": 96.4994888305664,
      "activations/layer3_attention_weight_min": -105.99202728271484,
      "activations/layer4_attention_weight_max": 95.42768859863281,
      "activations/layer4_attention_weight_min": -93.2957534790039,
      "activations/layer5_attention_weight_max": 72.71672058105469,
      "activations/layer5_attention_weight_min": -81.99274444580078,
      "activations/layer6_attention_weight_max": 55.89176940917969,
      "activations/layer6_attention_weight_min": -54.24552536010742,
      "activations/layer7_attention_weight_max": 66.30223846435547,
      "activations/layer7_attention_weight_min": -66.22782897949219,
      "activations/layer8_attention_weight_max": 48.34926986694336,
      "activations/layer8_attention_weight_min": -48.93379592895508,
      "activations/layer9_attention_weight_max": 46.7854118347168,
      "activations/layer9_attention_weight_min": -46.53598403930664,
      "epoch": 14.46,
      "learning_rate": 5.731325757575757e-05,
      "loss": 2.7461,
      "step": 248900
    },
    {
      "activations/layer0_attention_weight_max": 16.020084381103516,
      "activations/layer0_attention_weight_min": -13.900534629821777,
      "activations/layer10_attention_weight_max": 34.985595703125,
      "activations/layer10_attention_weight_min": -34.714359283447266,
      "activations/layer11_attention_weight_max": 31.76329803466797,
      "activations/layer11_attention_weight_min": -33.88929748535156,
      "activations/layer12_attention_weight_max": 20.47372055053711,
      "activations/layer12_attention_weight_min": -28.425519943237305,
      "activations/layer13_attention_weight_max": 31.869966506958008,
      "activations/layer13_attention_weight_min": -28.506380081176758,
      "activations/layer14_attention_weight_max": 37.000980377197266,
      "activations/layer14_attention_weight_min": -32.13138961791992,
      "activations/layer15_attention_weight_max": 34.3908805847168,
      "activations/layer15_attention_weight_min": -29.020971298217773,
      "activations/layer16_attention_weight_max": 35.93983459472656,
      "activations/layer16_attention_weight_min": -34.690895080566406,
      "activations/layer17_attention_weight_max": 57.03682327270508,
      "activations/layer17_attention_weight_min": -51.022216796875,
      "activations/layer18_attention_weight_max": 49.00528335571289,
      "activations/layer18_attention_weight_min": -43.33404541015625,
      "activations/layer19_attention_weight_max": 24.826488494873047,
      "activations/layer19_attention_weight_min": -25.730039596557617,
      "activations/layer1_attention_weight_max": 16.123998641967773,
      "activations/layer1_attention_weight_min": -16.026966094970703,
      "activations/layer20_attention_weight_max": 22.95206642150879,
      "activations/layer20_attention_weight_min": -22.846853256225586,
      "activations/layer21_attention_weight_max": 39.80995178222656,
      "activations/layer21_attention_weight_min": -25.258018493652344,
      "activations/layer22_attention_weight_max": 33.664337158203125,
      "activations/layer22_attention_weight_min": -25.475341796875,
      "activations/layer23_attention_weight_max": 35.729209899902344,
      "activations/layer23_attention_weight_min": -25.77729606628418,
      "activations/layer2_attention_weight_max": 32.21481704711914,
      "activations/layer2_attention_weight_min": -31.90133285522461,
      "activations/layer3_attention_weight_max": 94.64308166503906,
      "activations/layer3_attention_weight_min": -96.7312240600586,
      "activations/layer4_attention_weight_max": 91.89594268798828,
      "activations/layer4_attention_weight_min": -90.28002166748047,
      "activations/layer5_attention_weight_max": 67.39051055908203,
      "activations/layer5_attention_weight_min": -74.79397583007812,
      "activations/layer6_attention_weight_max": 50.78335189819336,
      "activations/layer6_attention_weight_min": -49.10651779174805,
      "activations/layer7_attention_weight_max": 62.57775115966797,
      "activations/layer7_attention_weight_min": -63.22661209106445,
      "activations/layer8_attention_weight_max": 45.134307861328125,
      "activations/layer8_attention_weight_min": -47.249061584472656,
      "activations/layer9_attention_weight_max": 44.906978607177734,
      "activations/layer9_attention_weight_min": -44.25336456298828,
      "epoch": 14.47,
      "learning_rate": 5.729431818181818e-05,
      "loss": 2.7664,
      "step": 248950
    },
    {
      "activations/layer0_attention_weight_max": 14.767626762390137,
      "activations/layer0_attention_weight_min": -13.890338897705078,
      "activations/layer10_attention_weight_max": 38.666507720947266,
      "activations/layer10_attention_weight_min": -40.64302062988281,
      "activations/layer11_attention_weight_max": 33.48288345336914,
      "activations/layer11_attention_weight_min": -36.80134582519531,
      "activations/layer12_attention_weight_max": 18.910144805908203,
      "activations/layer12_attention_weight_min": -30.66868782043457,
      "activations/layer13_attention_weight_max": 35.65785598754883,
      "activations/layer13_attention_weight_min": -29.16478729248047,
      "activations/layer14_attention_weight_max": 35.628562927246094,
      "activations/layer14_attention_weight_min": -29.92110252380371,
      "activations/layer15_attention_weight_max": 32.62062072753906,
      "activations/layer15_attention_weight_min": -31.09939193725586,
      "activations/layer16_attention_weight_max": 34.006927490234375,
      "activations/layer16_attention_weight_min": -31.87293243408203,
      "activations/layer17_attention_weight_max": 50.061458587646484,
      "activations/layer17_attention_weight_min": -42.778846740722656,
      "activations/layer18_attention_weight_max": 43.459903717041016,
      "activations/layer18_attention_weight_min": -38.58137130737305,
      "activations/layer19_attention_weight_max": 23.146045684814453,
      "activations/layer19_attention_weight_min": -22.853540420532227,
      "activations/layer1_attention_weight_max": 16.411819458007812,
      "activations/layer1_attention_weight_min": -15.71590518951416,
      "activations/layer20_attention_weight_max": 22.222618103027344,
      "activations/layer20_attention_weight_min": -22.970911026000977,
      "activations/layer21_attention_weight_max": 40.289588928222656,
      "activations/layer21_attention_weight_min": -23.623098373413086,
      "activations/layer22_attention_weight_max": 31.01366424560547,
      "activations/layer22_attention_weight_min": -27.91287612915039,
      "activations/layer23_attention_weight_max": 36.572174072265625,
      "activations/layer23_attention_weight_min": -25.882474899291992,
      "activations/layer2_attention_weight_max": 33.72957229614258,
      "activations/layer2_attention_weight_min": -30.144948959350586,
      "activations/layer3_attention_weight_max": 94.15386962890625,
      "activations/layer3_attention_weight_min": -95.92041015625,
      "activations/layer4_attention_weight_max": 94.24024200439453,
      "activations/layer4_attention_weight_min": -91.66215515136719,
      "activations/layer5_attention_weight_max": 68.95541381835938,
      "activations/layer5_attention_weight_min": -77.76741790771484,
      "activations/layer6_attention_weight_max": 51.70499801635742,
      "activations/layer6_attention_weight_min": -51.58975601196289,
      "activations/layer7_attention_weight_max": 67.619140625,
      "activations/layer7_attention_weight_min": -66.3602294921875,
      "activations/layer8_attention_weight_max": 47.01784133911133,
      "activations/layer8_attention_weight_min": -48.8698616027832,
      "activations/layer9_attention_weight_max": 48.75107192993164,
      "activations/layer9_attention_weight_min": -50.99312210083008,
      "epoch": 14.47,
      "learning_rate": 5.7275378787878786e-05,
      "loss": 2.7654,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_loss": 2.71484375,
      "eval_runtime": 8.4814,
      "eval_samples_per_second": 506.286,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_openwebtext_loss": 2.71484375,
      "eval_openwebtext_ppl": 15.10225016094451,
      "eval_openwebtext_runtime": 8.4814,
      "eval_openwebtext_samples_per_second": 506.286,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9211,
      "eval_wikitext_samples_per_second": 237.359,
      "step": 249000
    },
    {
      "epoch": 14.47,
      "eval_lambada_loss": 2.564453125,
      "eval_lambada_ppl": 12.993550578338562,
      "eval_lambada_runtime": 9.5239,
      "eval_lambada_samples_per_second": 511.24,
      "step": 249000
    },
    {
      "activations/layer0_attention_weight_max": 15.76870346069336,
      "activations/layer0_attention_weight_min": -13.759613037109375,
      "activations/layer10_attention_weight_max": 36.62591552734375,
      "activations/layer10_attention_weight_min": -35.25395202636719,
      "activations/layer11_attention_weight_max": 32.02389144897461,
      "activations/layer11_attention_weight_min": -32.28178405761719,
      "activations/layer12_attention_weight_max": 23.797847747802734,
      "activations/layer12_attention_weight_min": -22.822158813476562,
      "activations/layer13_attention_weight_max": 41.85834503173828,
      "activations/layer13_attention_weight_min": -30.02444839477539,
      "activations/layer14_attention_weight_max": 38.36472702026367,
      "activations/layer14_attention_weight_min": -30.92925453186035,
      "activations/layer15_attention_weight_max": 34.5372428894043,
      "activations/layer15_attention_weight_min": -30.93965721130371,
      "activations/layer16_attention_weight_max": 31.867916107177734,
      "activations/layer16_attention_weight_min": -29.793214797973633,
      "activations/layer17_attention_weight_max": 52.57912826538086,
      "activations/layer17_attention_weight_min": -42.98151397705078,
      "activations/layer18_attention_weight_max": 48.553733825683594,
      "activations/layer18_attention_weight_min": -37.515541076660156,
      "activations/layer19_attention_weight_max": 25.994998931884766,
      "activations/layer19_attention_weight_min": -22.706134796142578,
      "activations/layer1_attention_weight_max": 16.722352981567383,
      "activations/layer1_attention_weight_min": -14.785117149353027,
      "activations/layer20_attention_weight_max": 22.272321701049805,
      "activations/layer20_attention_weight_min": -22.554685592651367,
      "activations/layer21_attention_weight_max": 38.74773025512695,
      "activations/layer21_attention_weight_min": -26.13518524169922,
      "activations/layer22_attention_weight_max": 30.987239837646484,
      "activations/layer22_attention_weight_min": -27.05061912536621,
      "activations/layer23_attention_weight_max": 41.93537139892578,
      "activations/layer23_attention_weight_min": -24.258214950561523,
      "activations/layer2_attention_weight_max": 31.637603759765625,
      "activations/layer2_attention_weight_min": -31.033920288085938,
      "activations/layer3_attention_weight_max": 92.23995208740234,
      "activations/layer3_attention_weight_min": -94.42684173583984,
      "activations/layer4_attention_weight_max": 87.79071807861328,
      "activations/layer4_attention_weight_min": -83.78477478027344,
      "activations/layer5_attention_weight_max": 69.98905181884766,
      "activations/layer5_attention_weight_min": -79.24398803710938,
      "activations/layer6_attention_weight_max": 48.62834548950195,
      "activations/layer6_attention_weight_min": -50.110469818115234,
      "activations/layer7_attention_weight_max": 59.35245132446289,
      "activations/layer7_attention_weight_min": -62.48866271972656,
      "activations/layer8_attention_weight_max": 45.92416000366211,
      "activations/layer8_attention_weight_min": -48.71162796020508,
      "activations/layer9_attention_weight_max": 44.36825942993164,
      "activations/layer9_attention_weight_min": -45.68046188354492,
      "epoch": 14.47,
      "learning_rate": 5.7256439393939394e-05,
      "loss": 2.7493,
      "step": 249050
    },
    {
      "activations/layer0_attention_weight_max": 16.0465087890625,
      "activations/layer0_attention_weight_min": -13.89147663116455,
      "activations/layer10_attention_weight_max": 35.50755310058594,
      "activations/layer10_attention_weight_min": -34.55448532104492,
      "activations/layer11_attention_weight_max": 32.862640380859375,
      "activations/layer11_attention_weight_min": -33.85675048828125,
      "activations/layer12_attention_weight_max": 16.88951301574707,
      "activations/layer12_attention_weight_min": -23.66704750061035,
      "activations/layer13_attention_weight_max": 35.55943298339844,
      "activations/layer13_attention_weight_min": -28.028766632080078,
      "activations/layer14_attention_weight_max": 34.87552261352539,
      "activations/layer14_attention_weight_min": -30.270322799682617,
      "activations/layer15_attention_weight_max": 33.53044128417969,
      "activations/layer15_attention_weight_min": -29.350584030151367,
      "activations/layer16_attention_weight_max": 31.171695709228516,
      "activations/layer16_attention_weight_min": -30.240266799926758,
      "activations/layer17_attention_weight_max": 49.00627517700195,
      "activations/layer17_attention_weight_min": -44.601139068603516,
      "activations/layer18_attention_weight_max": 43.2000732421875,
      "activations/layer18_attention_weight_min": -37.46889877319336,
      "activations/layer19_attention_weight_max": 23.372093200683594,
      "activations/layer19_attention_weight_min": -23.446819305419922,
      "activations/layer1_attention_weight_max": 15.911745071411133,
      "activations/layer1_attention_weight_min": -18.183128356933594,
      "activations/layer20_attention_weight_max": 22.757360458374023,
      "activations/layer20_attention_weight_min": -21.42231559753418,
      "activations/layer21_attention_weight_max": 34.173526763916016,
      "activations/layer21_attention_weight_min": -26.414249420166016,
      "activations/layer22_attention_weight_max": 30.046342849731445,
      "activations/layer22_attention_weight_min": -25.48361587524414,
      "activations/layer23_attention_weight_max": 35.300384521484375,
      "activations/layer23_attention_weight_min": -24.986583709716797,
      "activations/layer2_attention_weight_max": 31.595260620117188,
      "activations/layer2_attention_weight_min": -30.61983871459961,
      "activations/layer3_attention_weight_max": 97.15907287597656,
      "activations/layer3_attention_weight_min": -96.83487701416016,
      "activations/layer4_attention_weight_max": 91.01181030273438,
      "activations/layer4_attention_weight_min": -91.0380630493164,
      "activations/layer5_attention_weight_max": 70.5163803100586,
      "activations/layer5_attention_weight_min": -74.4581069946289,
      "activations/layer6_attention_weight_max": 48.93620681762695,
      "activations/layer6_attention_weight_min": -51.50608444213867,
      "activations/layer7_attention_weight_max": 62.84811019897461,
      "activations/layer7_attention_weight_min": -63.32840347290039,
      "activations/layer8_attention_weight_max": 45.58409118652344,
      "activations/layer8_attention_weight_min": -48.50192642211914,
      "activations/layer9_attention_weight_max": 43.510765075683594,
      "activations/layer9_attention_weight_min": -45.46419906616211,
      "epoch": 14.47,
      "learning_rate": 5.7237499999999995e-05,
      "loss": 2.7563,
      "step": 249100
    },
    {
      "activations/layer0_attention_weight_max": 16.456439971923828,
      "activations/layer0_attention_weight_min": -13.696874618530273,
      "activations/layer10_attention_weight_max": 35.721866607666016,
      "activations/layer10_attention_weight_min": -36.161373138427734,
      "activations/layer11_attention_weight_max": 35.07476043701172,
      "activations/layer11_attention_weight_min": -34.03071975708008,
      "activations/layer12_attention_weight_max": 17.094228744506836,
      "activations/layer12_attention_weight_min": -25.417356491088867,
      "activations/layer13_attention_weight_max": 34.94306182861328,
      "activations/layer13_attention_weight_min": -27.35947608947754,
      "activations/layer14_attention_weight_max": 36.67743682861328,
      "activations/layer14_attention_weight_min": -30.570621490478516,
      "activations/layer15_attention_weight_max": 31.180484771728516,
      "activations/layer15_attention_weight_min": -30.279619216918945,
      "activations/layer16_attention_weight_max": 30.230073928833008,
      "activations/layer16_attention_weight_min": -29.689828872680664,
      "activations/layer17_attention_weight_max": 49.07920837402344,
      "activations/layer17_attention_weight_min": -41.04698181152344,
      "activations/layer18_attention_weight_max": 43.770751953125,
      "activations/layer18_attention_weight_min": -37.605857849121094,
      "activations/layer19_attention_weight_max": 26.018253326416016,
      "activations/layer19_attention_weight_min": -22.839847564697266,
      "activations/layer1_attention_weight_max": 16.016441345214844,
      "activations/layer1_attention_weight_min": -14.085254669189453,
      "activations/layer20_attention_weight_max": 21.840566635131836,
      "activations/layer20_attention_weight_min": -23.40623664855957,
      "activations/layer21_attention_weight_max": 37.13118362426758,
      "activations/layer21_attention_weight_min": -27.211837768554688,
      "activations/layer22_attention_weight_max": 29.29532241821289,
      "activations/layer22_attention_weight_min": -25.811243057250977,
      "activations/layer23_attention_weight_max": 35.477699279785156,
      "activations/layer23_attention_weight_min": -26.876638412475586,
      "activations/layer2_attention_weight_max": 31.94247055053711,
      "activations/layer2_attention_weight_min": -31.102115631103516,
      "activations/layer3_attention_weight_max": 97.53858947753906,
      "activations/layer3_attention_weight_min": -102.61671447753906,
      "activations/layer4_attention_weight_max": 95.09278106689453,
      "activations/layer4_attention_weight_min": -93.49832916259766,
      "activations/layer5_attention_weight_max": 71.80305480957031,
      "activations/layer5_attention_weight_min": -79.50711059570312,
      "activations/layer6_attention_weight_max": 51.954124450683594,
      "activations/layer6_attention_weight_min": -53.46403121948242,
      "activations/layer7_attention_weight_max": 60.01218795776367,
      "activations/layer7_attention_weight_min": -63.69263458251953,
      "activations/layer8_attention_weight_max": 46.10206604003906,
      "activations/layer8_attention_weight_min": -50.901973724365234,
      "activations/layer9_attention_weight_max": 44.046260833740234,
      "activations/layer9_attention_weight_min": -49.436519622802734,
      "epoch": 14.48,
      "learning_rate": 5.72185606060606e-05,
      "loss": 2.7515,
      "step": 249150
    },
    {
      "activations/layer0_attention_weight_max": 15.28376579284668,
      "activations/layer0_attention_weight_min": -13.451916694641113,
      "activations/layer10_attention_weight_max": 35.38587951660156,
      "activations/layer10_attention_weight_min": -36.43116760253906,
      "activations/layer11_attention_weight_max": 30.442420959472656,
      "activations/layer11_attention_weight_min": -32.66156005859375,
      "activations/layer12_attention_weight_max": 19.97068977355957,
      "activations/layer12_attention_weight_min": -24.308982849121094,
      "activations/layer13_attention_weight_max": 34.97611999511719,
      "activations/layer13_attention_weight_min": -25.64542007446289,
      "activations/layer14_attention_weight_max": 37.950172424316406,
      "activations/layer14_attention_weight_min": -30.45657730102539,
      "activations/layer15_attention_weight_max": 32.86528396606445,
      "activations/layer15_attention_weight_min": -30.608369827270508,
      "activations/layer16_attention_weight_max": 31.281213760375977,
      "activations/layer16_attention_weight_min": -31.375261306762695,
      "activations/layer17_attention_weight_max": 49.695091247558594,
      "activations/layer17_attention_weight_min": -43.067161560058594,
      "activations/layer18_attention_weight_max": 44.06953811645508,
      "activations/layer18_attention_weight_min": -36.57836151123047,
      "activations/layer19_attention_weight_max": 24.141895294189453,
      "activations/layer19_attention_weight_min": -20.91300392150879,
      "activations/layer1_attention_weight_max": 17.995006561279297,
      "activations/layer1_attention_weight_min": -14.735762596130371,
      "activations/layer20_attention_weight_max": 24.6845703125,
      "activations/layer20_attention_weight_min": -20.475313186645508,
      "activations/layer21_attention_weight_max": 36.56156921386719,
      "activations/layer21_attention_weight_min": -23.59691619873047,
      "activations/layer22_attention_weight_max": 30.75265121459961,
      "activations/layer22_attention_weight_min": -25.383407592773438,
      "activations/layer23_attention_weight_max": 36.09714126586914,
      "activations/layer23_attention_weight_min": -24.147907257080078,
      "activations/layer2_attention_weight_max": 33.73328399658203,
      "activations/layer2_attention_weight_min": -31.472129821777344,
      "activations/layer3_attention_weight_max": 94.70561981201172,
      "activations/layer3_attention_weight_min": -97.91262817382812,
      "activations/layer4_attention_weight_max": 89.04454803466797,
      "activations/layer4_attention_weight_min": -88.24208068847656,
      "activations/layer5_attention_weight_max": 68.28004455566406,
      "activations/layer5_attention_weight_min": -74.63321685791016,
      "activations/layer6_attention_weight_max": 50.724185943603516,
      "activations/layer6_attention_weight_min": -50.72596740722656,
      "activations/layer7_attention_weight_max": 68.62589263916016,
      "activations/layer7_attention_weight_min": -62.75362014770508,
      "activations/layer8_attention_weight_max": 43.43532943725586,
      "activations/layer8_attention_weight_min": -47.82245635986328,
      "activations/layer9_attention_weight_max": 40.91116714477539,
      "activations/layer9_attention_weight_min": -45.25493621826172,
      "epoch": 14.48,
      "learning_rate": 5.719962121212121e-05,
      "loss": 2.7603,
      "step": 249200
    },
    {
      "activations/layer0_attention_weight_max": 15.251928329467773,
      "activations/layer0_attention_weight_min": -13.823031425476074,
      "activations/layer10_attention_weight_max": 38.5459098815918,
      "activations/layer10_attention_weight_min": -36.155723571777344,
      "activations/layer11_attention_weight_max": 33.96905517578125,
      "activations/layer11_attention_weight_min": -32.503822326660156,
      "activations/layer12_attention_weight_max": 18.320566177368164,
      "activations/layer12_attention_weight_min": -26.032445907592773,
      "activations/layer13_attention_weight_max": 37.28050231933594,
      "activations/layer13_attention_weight_min": -28.553424835205078,
      "activations/layer14_attention_weight_max": 37.43500900268555,
      "activations/layer14_attention_weight_min": -30.870803833007812,
      "activations/layer15_attention_weight_max": 34.05870056152344,
      "activations/layer15_attention_weight_min": -29.572202682495117,
      "activations/layer16_attention_weight_max": 31.14610481262207,
      "activations/layer16_attention_weight_min": -30.30614471435547,
      "activations/layer17_attention_weight_max": 50.76442337036133,
      "activations/layer17_attention_weight_min": -43.73369216918945,
      "activations/layer18_attention_weight_max": 47.2608757019043,
      "activations/layer18_attention_weight_min": -37.636749267578125,
      "activations/layer19_attention_weight_max": 24.652463912963867,
      "activations/layer19_attention_weight_min": -23.459707260131836,
      "activations/layer1_attention_weight_max": 15.617159843444824,
      "activations/layer1_attention_weight_min": -15.094923973083496,
      "activations/layer20_attention_weight_max": 24.332307815551758,
      "activations/layer20_attention_weight_min": -22.7460994720459,
      "activations/layer21_attention_weight_max": 40.24148941040039,
      "activations/layer21_attention_weight_min": -28.939056396484375,
      "activations/layer22_attention_weight_max": 29.50723648071289,
      "activations/layer22_attention_weight_min": -24.55421257019043,
      "activations/layer23_attention_weight_max": 38.34741973876953,
      "activations/layer23_attention_weight_min": -28.109729766845703,
      "activations/layer2_attention_weight_max": 34.036773681640625,
      "activations/layer2_attention_weight_min": -32.31437683105469,
      "activations/layer3_attention_weight_max": 98.82877349853516,
      "activations/layer3_attention_weight_min": -106.01459503173828,
      "activations/layer4_attention_weight_max": 98.12191009521484,
      "activations/layer4_attention_weight_min": -91.9582290649414,
      "activations/layer5_attention_weight_max": 73.16049194335938,
      "activations/layer5_attention_weight_min": -78.81765747070312,
      "activations/layer6_attention_weight_max": 54.86452102661133,
      "activations/layer6_attention_weight_min": -54.78092575073242,
      "activations/layer7_attention_weight_max": 64.75142669677734,
      "activations/layer7_attention_weight_min": -65.59545135498047,
      "activations/layer8_attention_weight_max": 46.753658294677734,
      "activations/layer8_attention_weight_min": -50.68277359008789,
      "activations/layer9_attention_weight_max": 46.29220199584961,
      "activations/layer9_attention_weight_min": -46.79927062988281,
      "epoch": 14.48,
      "learning_rate": 5.7180681818181813e-05,
      "loss": 2.7589,
      "step": 249250
    },
    {
      "activations/layer0_attention_weight_max": 15.484842300415039,
      "activations/layer0_attention_weight_min": -13.873567581176758,
      "activations/layer10_attention_weight_max": 35.25788116455078,
      "activations/layer10_attention_weight_min": -34.98737335205078,
      "activations/layer11_attention_weight_max": 33.46887969970703,
      "activations/layer11_attention_weight_min": -33.308475494384766,
      "activations/layer12_attention_weight_max": 17.86838150024414,
      "activations/layer12_attention_weight_min": -27.271299362182617,
      "activations/layer13_attention_weight_max": 36.893455505371094,
      "activations/layer13_attention_weight_min": -27.36273765563965,
      "activations/layer14_attention_weight_max": 36.51448440551758,
      "activations/layer14_attention_weight_min": -29.891067504882812,
      "activations/layer15_attention_weight_max": 30.600059509277344,
      "activations/layer15_attention_weight_min": -28.96636199951172,
      "activations/layer16_attention_weight_max": 35.346290588378906,
      "activations/layer16_attention_weight_min": -32.04634475708008,
      "activations/layer17_attention_weight_max": 52.66142272949219,
      "activations/layer17_attention_weight_min": -50.209720611572266,
      "activations/layer18_attention_weight_max": 44.35062026977539,
      "activations/layer18_attention_weight_min": -40.402706146240234,
      "activations/layer19_attention_weight_max": 23.301250457763672,
      "activations/layer19_attention_weight_min": -24.010541915893555,
      "activations/layer1_attention_weight_max": 17.02804946899414,
      "activations/layer1_attention_weight_min": -14.480035781860352,
      "activations/layer20_attention_weight_max": 22.981990814208984,
      "activations/layer20_attention_weight_min": -23.188720703125,
      "activations/layer21_attention_weight_max": 31.750347137451172,
      "activations/layer21_attention_weight_min": -23.51665687561035,
      "activations/layer22_attention_weight_max": 29.496198654174805,
      "activations/layer22_attention_weight_min": -25.19550323486328,
      "activations/layer23_attention_weight_max": 36.781558990478516,
      "activations/layer23_attention_weight_min": -25.60564422607422,
      "activations/layer2_attention_weight_max": 32.82504653930664,
      "activations/layer2_attention_weight_min": -28.902618408203125,
      "activations/layer3_attention_weight_max": 99.72210693359375,
      "activations/layer3_attention_weight_min": -96.44113159179688,
      "activations/layer4_attention_weight_max": 94.76859283447266,
      "activations/layer4_attention_weight_min": -91.18279266357422,
      "activations/layer5_attention_weight_max": 71.296142578125,
      "activations/layer5_attention_weight_min": -76.53988647460938,
      "activations/layer6_attention_weight_max": 52.2157096862793,
      "activations/layer6_attention_weight_min": -52.08784484863281,
      "activations/layer7_attention_weight_max": 65.90629577636719,
      "activations/layer7_attention_weight_min": -62.2923583984375,
      "activations/layer8_attention_weight_max": 43.53065872192383,
      "activations/layer8_attention_weight_min": -46.05159378051758,
      "activations/layer9_attention_weight_max": 48.621070861816406,
      "activations/layer9_attention_weight_min": -45.8454704284668,
      "epoch": 14.49,
      "learning_rate": 5.7161742424242415e-05,
      "loss": 2.7665,
      "step": 249300
    },
    {
      "activations/layer0_attention_weight_max": 15.056912422180176,
      "activations/layer0_attention_weight_min": -13.9614896774292,
      "activations/layer10_attention_weight_max": 39.23016357421875,
      "activations/layer10_attention_weight_min": -40.41822814941406,
      "activations/layer11_attention_weight_max": 33.457969665527344,
      "activations/layer11_attention_weight_min": -34.22990417480469,
      "activations/layer12_attention_weight_max": 18.617101669311523,
      "activations/layer12_attention_weight_min": -31.491273880004883,
      "activations/layer13_attention_weight_max": 38.17298126220703,
      "activations/layer13_attention_weight_min": -29.28850746154785,
      "activations/layer14_attention_weight_max": 36.97324752807617,
      "activations/layer14_attention_weight_min": -30.70245361328125,
      "activations/layer15_attention_weight_max": 33.02677917480469,
      "activations/layer15_attention_weight_min": -29.352542877197266,
      "activations/layer16_attention_weight_max": 31.907520294189453,
      "activations/layer16_attention_weight_min": -29.861024856567383,
      "activations/layer17_attention_weight_max": 53.815311431884766,
      "activations/layer17_attention_weight_min": -45.005794525146484,
      "activations/layer18_attention_weight_max": 46.297359466552734,
      "activations/layer18_attention_weight_min": -39.42087173461914,
      "activations/layer19_attention_weight_max": 25.583377838134766,
      "activations/layer19_attention_weight_min": -21.72136688232422,
      "activations/layer1_attention_weight_max": 16.706214904785156,
      "activations/layer1_attention_weight_min": -15.429656982421875,
      "activations/layer20_attention_weight_max": 24.85374641418457,
      "activations/layer20_attention_weight_min": -22.522350311279297,
      "activations/layer21_attention_weight_max": 42.570640563964844,
      "activations/layer21_attention_weight_min": -23.286998748779297,
      "activations/layer22_attention_weight_max": 32.493961334228516,
      "activations/layer22_attention_weight_min": -26.646766662597656,
      "activations/layer23_attention_weight_max": 38.421844482421875,
      "activations/layer23_attention_weight_min": -23.5601863861084,
      "activations/layer2_attention_weight_max": 29.496694564819336,
      "activations/layer2_attention_weight_min": -28.275495529174805,
      "activations/layer3_attention_weight_max": 95.72357177734375,
      "activations/layer3_attention_weight_min": -97.4978256225586,
      "activations/layer4_attention_weight_max": 97.18612670898438,
      "activations/layer4_attention_weight_min": -97.43482208251953,
      "activations/layer5_attention_weight_max": 69.47621154785156,
      "activations/layer5_attention_weight_min": -80.37541198730469,
      "activations/layer6_attention_weight_max": 52.246681213378906,
      "activations/layer6_attention_weight_min": -54.57965850830078,
      "activations/layer7_attention_weight_max": 64.05998229980469,
      "activations/layer7_attention_weight_min": -67.43600463867188,
      "activations/layer8_attention_weight_max": 48.275856018066406,
      "activations/layer8_attention_weight_min": -52.7813606262207,
      "activations/layer9_attention_weight_max": 50.26185607910156,
      "activations/layer9_attention_weight_min": -50.79513931274414,
      "epoch": 14.49,
      "learning_rate": 5.714280303030302e-05,
      "loss": 2.7694,
      "step": 249350
    },
    {
      "activations/layer0_attention_weight_max": 15.679009437561035,
      "activations/layer0_attention_weight_min": -13.515799522399902,
      "activations/layer10_attention_weight_max": 32.45355987548828,
      "activations/layer10_attention_weight_min": -33.103126525878906,
      "activations/layer11_attention_weight_max": 31.257537841796875,
      "activations/layer11_attention_weight_min": -31.4316463470459,
      "activations/layer12_attention_weight_max": 16.781879425048828,
      "activations/layer12_attention_weight_min": -25.276351928710938,
      "activations/layer13_attention_weight_max": 29.82794952392578,
      "activations/layer13_attention_weight_min": -26.375200271606445,
      "activations/layer14_attention_weight_max": 33.21126937866211,
      "activations/layer14_attention_weight_min": -28.47092056274414,
      "activations/layer15_attention_weight_max": 30.188140869140625,
      "activations/layer15_attention_weight_min": -30.93271827697754,
      "activations/layer16_attention_weight_max": 28.863130569458008,
      "activations/layer16_attention_weight_min": -29.03213882446289,
      "activations/layer17_attention_weight_max": 49.264034271240234,
      "activations/layer17_attention_weight_min": -43.021663665771484,
      "activations/layer18_attention_weight_max": 42.34433364868164,
      "activations/layer18_attention_weight_min": -37.782440185546875,
      "activations/layer19_attention_weight_max": 22.606761932373047,
      "activations/layer19_attention_weight_min": -20.686256408691406,
      "activations/layer1_attention_weight_max": 17.08113670349121,
      "activations/layer1_attention_weight_min": -15.80319595336914,
      "activations/layer20_attention_weight_max": 21.449169158935547,
      "activations/layer20_attention_weight_min": -22.049230575561523,
      "activations/layer21_attention_weight_max": 31.48916244506836,
      "activations/layer21_attention_weight_min": -20.217714309692383,
      "activations/layer22_attention_weight_max": 28.338512420654297,
      "activations/layer22_attention_weight_min": -25.40943145751953,
      "activations/layer23_attention_weight_max": 34.09434509277344,
      "activations/layer23_attention_weight_min": -24.508214950561523,
      "activations/layer2_attention_weight_max": 31.976316452026367,
      "activations/layer2_attention_weight_min": -31.547481536865234,
      "activations/layer3_attention_weight_max": 92.83535766601562,
      "activations/layer3_attention_weight_min": -94.7271499633789,
      "activations/layer4_attention_weight_max": 91.41876983642578,
      "activations/layer4_attention_weight_min": -87.10413360595703,
      "activations/layer5_attention_weight_max": 66.32281494140625,
      "activations/layer5_attention_weight_min": -78.68260192871094,
      "activations/layer6_attention_weight_max": 49.55741500854492,
      "activations/layer6_attention_weight_min": -51.26179885864258,
      "activations/layer7_attention_weight_max": 66.72008514404297,
      "activations/layer7_attention_weight_min": -60.384517669677734,
      "activations/layer8_attention_weight_max": 44.13942337036133,
      "activations/layer8_attention_weight_min": -44.718040466308594,
      "activations/layer9_attention_weight_max": 42.16917037963867,
      "activations/layer9_attention_weight_min": -43.642452239990234,
      "epoch": 14.49,
      "learning_rate": 5.712386363636363e-05,
      "loss": 2.7749,
      "step": 249400
    },
    {
      "activations/layer0_attention_weight_max": 15.6892728805542,
      "activations/layer0_attention_weight_min": -13.52354907989502,
      "activations/layer10_attention_weight_max": 34.55554962158203,
      "activations/layer10_attention_weight_min": -34.607994079589844,
      "activations/layer11_attention_weight_max": 32.96002960205078,
      "activations/layer11_attention_weight_min": -33.81629943847656,
      "activations/layer12_attention_weight_max": 19.30259895324707,
      "activations/layer12_attention_weight_min": -22.989463806152344,
      "activations/layer13_attention_weight_max": 32.54499816894531,
      "activations/layer13_attention_weight_min": -27.39561653137207,
      "activations/layer14_attention_weight_max": 32.46681594848633,
      "activations/layer14_attention_weight_min": -30.58055305480957,
      "activations/layer15_attention_weight_max": 30.294296264648438,
      "activations/layer15_attention_weight_min": -28.92522430419922,
      "activations/layer16_attention_weight_max": 28.892553329467773,
      "activations/layer16_attention_weight_min": -28.799394607543945,
      "activations/layer17_attention_weight_max": 46.34110641479492,
      "activations/layer17_attention_weight_min": -41.68061065673828,
      "activations/layer18_attention_weight_max": 40.65855407714844,
      "activations/layer18_attention_weight_min": -35.91941833496094,
      "activations/layer19_attention_weight_max": 20.13301658630371,
      "activations/layer19_attention_weight_min": -23.058940887451172,
      "activations/layer1_attention_weight_max": 17.509796142578125,
      "activations/layer1_attention_weight_min": -15.690159797668457,
      "activations/layer20_attention_weight_max": 19.853429794311523,
      "activations/layer20_attention_weight_min": -26.675430297851562,
      "activations/layer21_attention_weight_max": 28.47411346435547,
      "activations/layer21_attention_weight_min": -19.385168075561523,
      "activations/layer22_attention_weight_max": 27.15232276916504,
      "activations/layer22_attention_weight_min": -24.647174835205078,
      "activations/layer23_attention_weight_max": 32.789695739746094,
      "activations/layer23_attention_weight_min": -24.033546447753906,
      "activations/layer2_attention_weight_max": 32.05727005004883,
      "activations/layer2_attention_weight_min": -30.45758056640625,
      "activations/layer3_attention_weight_max": 93.29143524169922,
      "activations/layer3_attention_weight_min": -95.18004608154297,
      "activations/layer4_attention_weight_max": 92.99720001220703,
      "activations/layer4_attention_weight_min": -94.9794692993164,
      "activations/layer5_attention_weight_max": 70.03453063964844,
      "activations/layer5_attention_weight_min": -79.73876190185547,
      "activations/layer6_attention_weight_max": 53.02616500854492,
      "activations/layer6_attention_weight_min": -56.013572692871094,
      "activations/layer7_attention_weight_max": 66.03937530517578,
      "activations/layer7_attention_weight_min": -67.6576919555664,
      "activations/layer8_attention_weight_max": 46.478302001953125,
      "activations/layer8_attention_weight_min": -49.98563766479492,
      "activations/layer9_attention_weight_max": 43.238468170166016,
      "activations/layer9_attention_weight_min": -46.23497772216797,
      "epoch": 14.49,
      "learning_rate": 5.710492424242424e-05,
      "loss": 2.7602,
      "step": 249450
    },
    {
      "activations/layer0_attention_weight_max": 17.610563278198242,
      "activations/layer0_attention_weight_min": -13.57530403137207,
      "activations/layer10_attention_weight_max": 34.04310989379883,
      "activations/layer10_attention_weight_min": -34.343841552734375,
      "activations/layer11_attention_weight_max": 29.74950408935547,
      "activations/layer11_attention_weight_min": -31.41513442993164,
      "activations/layer12_attention_weight_max": 18.323749542236328,
      "activations/layer12_attention_weight_min": -31.737512588500977,
      "activations/layer13_attention_weight_max": 39.91373825073242,
      "activations/layer13_attention_weight_min": -29.205286026000977,
      "activations/layer14_attention_weight_max": 40.69123840332031,
      "activations/layer14_attention_weight_min": -31.31797981262207,
      "activations/layer15_attention_weight_max": 31.47562599182129,
      "activations/layer15_attention_weight_min": -29.582298278808594,
      "activations/layer16_attention_weight_max": 32.50508499145508,
      "activations/layer16_attention_weight_min": -30.029632568359375,
      "activations/layer17_attention_weight_max": 48.79238510131836,
      "activations/layer17_attention_weight_min": -45.41902542114258,
      "activations/layer18_attention_weight_max": 45.76852035522461,
      "activations/layer18_attention_weight_min": -40.463905334472656,
      "activations/layer19_attention_weight_max": 22.410863876342773,
      "activations/layer19_attention_weight_min": -25.106433868408203,
      "activations/layer1_attention_weight_max": 15.793509483337402,
      "activations/layer1_attention_weight_min": -14.023208618164062,
      "activations/layer20_attention_weight_max": 23.18462371826172,
      "activations/layer20_attention_weight_min": -23.73998260498047,
      "activations/layer21_attention_weight_max": 35.30075454711914,
      "activations/layer21_attention_weight_min": -23.99513816833496,
      "activations/layer22_attention_weight_max": 32.18577194213867,
      "activations/layer22_attention_weight_min": -26.00211524963379,
      "activations/layer23_attention_weight_max": 39.12635040283203,
      "activations/layer23_attention_weight_min": -26.273908615112305,
      "activations/layer2_attention_weight_max": 29.92197036743164,
      "activations/layer2_attention_weight_min": -28.682964324951172,
      "activations/layer3_attention_weight_max": 88.8424301147461,
      "activations/layer3_attention_weight_min": -89.90560150146484,
      "activations/layer4_attention_weight_max": 93.65056610107422,
      "activations/layer4_attention_weight_min": -87.19670867919922,
      "activations/layer5_attention_weight_max": 69.64169311523438,
      "activations/layer5_attention_weight_min": -73.85424041748047,
      "activations/layer6_attention_weight_max": 51.315269470214844,
      "activations/layer6_attention_weight_min": -51.23112869262695,
      "activations/layer7_attention_weight_max": 68.96796417236328,
      "activations/layer7_attention_weight_min": -61.0413818359375,
      "activations/layer8_attention_weight_max": 46.51852798461914,
      "activations/layer8_attention_weight_min": -48.79530715942383,
      "activations/layer9_attention_weight_max": 45.52275085449219,
      "activations/layer9_attention_weight_min": -46.56574630737305,
      "epoch": 14.5,
      "learning_rate": 5.708598484848484e-05,
      "loss": 2.7668,
      "step": 249500
    },
    {
      "activations/layer0_attention_weight_max": 15.741595268249512,
      "activations/layer0_attention_weight_min": -13.477082252502441,
      "activations/layer10_attention_weight_max": 40.12496566772461,
      "activations/layer10_attention_weight_min": -37.4300537109375,
      "activations/layer11_attention_weight_max": 34.826175689697266,
      "activations/layer11_attention_weight_min": -34.055179595947266,
      "activations/layer12_attention_weight_max": 18.963409423828125,
      "activations/layer12_attention_weight_min": -23.578351974487305,
      "activations/layer13_attention_weight_max": 34.30243682861328,
      "activations/layer13_attention_weight_min": -29.38776206970215,
      "activations/layer14_attention_weight_max": 31.88859748840332,
      "activations/layer14_attention_weight_min": -30.447860717773438,
      "activations/layer15_attention_weight_max": 30.446731567382812,
      "activations/layer15_attention_weight_min": -29.996318817138672,
      "activations/layer16_attention_weight_max": 30.97621726989746,
      "activations/layer16_attention_weight_min": -30.27533721923828,
      "activations/layer17_attention_weight_max": 51.92559814453125,
      "activations/layer17_attention_weight_min": -43.183570861816406,
      "activations/layer18_attention_weight_max": 44.3654899597168,
      "activations/layer18_attention_weight_min": -38.019081115722656,
      "activations/layer19_attention_weight_max": 22.92034149169922,
      "activations/layer19_attention_weight_min": -20.284162521362305,
      "activations/layer1_attention_weight_max": 16.059696197509766,
      "activations/layer1_attention_weight_min": -15.573087692260742,
      "activations/layer20_attention_weight_max": 20.258825302124023,
      "activations/layer20_attention_weight_min": -19.554729461669922,
      "activations/layer21_attention_weight_max": 30.497617721557617,
      "activations/layer21_attention_weight_min": -22.787445068359375,
      "activations/layer22_attention_weight_max": 26.19488525390625,
      "activations/layer22_attention_weight_min": -24.920652389526367,
      "activations/layer23_attention_weight_max": 33.80784225463867,
      "activations/layer23_attention_weight_min": -26.89484405517578,
      "activations/layer2_attention_weight_max": 31.006223678588867,
      "activations/layer2_attention_weight_min": -28.628782272338867,
      "activations/layer3_attention_weight_max": 92.67911529541016,
      "activations/layer3_attention_weight_min": -93.31739807128906,
      "activations/layer4_attention_weight_max": 91.9738540649414,
      "activations/layer4_attention_weight_min": -86.6822280883789,
      "activations/layer5_attention_weight_max": 69.41879272460938,
      "activations/layer5_attention_weight_min": -78.3471908569336,
      "activations/layer6_attention_weight_max": 53.078285217285156,
      "activations/layer6_attention_weight_min": -53.247249603271484,
      "activations/layer7_attention_weight_max": 65.36710357666016,
      "activations/layer7_attention_weight_min": -63.95505905151367,
      "activations/layer8_attention_weight_max": 48.17761993408203,
      "activations/layer8_attention_weight_min": -51.08199691772461,
      "activations/layer9_attention_weight_max": 47.604095458984375,
      "activations/layer9_attention_weight_min": -47.91770553588867,
      "epoch": 14.5,
      "learning_rate": 5.706704545454544e-05,
      "loss": 2.7589,
      "step": 249550
    },
    {
      "activations/layer0_attention_weight_max": 16.732635498046875,
      "activations/layer0_attention_weight_min": -13.756710052490234,
      "activations/layer10_attention_weight_max": 33.23443603515625,
      "activations/layer10_attention_weight_min": -33.49153518676758,
      "activations/layer11_attention_weight_max": 33.04409408569336,
      "activations/layer11_attention_weight_min": -32.20276641845703,
      "activations/layer12_attention_weight_max": 25.960025787353516,
      "activations/layer12_attention_weight_min": -28.886844635009766,
      "activations/layer13_attention_weight_max": 32.005271911621094,
      "activations/layer13_attention_weight_min": -26.2121639251709,
      "activations/layer14_attention_weight_max": 32.17573547363281,
      "activations/layer14_attention_weight_min": -29.12306022644043,
      "activations/layer15_attention_weight_max": 29.691070556640625,
      "activations/layer15_attention_weight_min": -29.07571029663086,
      "activations/layer16_attention_weight_max": 31.71309471130371,
      "activations/layer16_attention_weight_min": -32.052032470703125,
      "activations/layer17_attention_weight_max": 45.912193298339844,
      "activations/layer17_attention_weight_min": -42.19633865356445,
      "activations/layer18_attention_weight_max": 42.11551284790039,
      "activations/layer18_attention_weight_min": -38.72509002685547,
      "activations/layer19_attention_weight_max": 24.207658767700195,
      "activations/layer19_attention_weight_min": -20.72242546081543,
      "activations/layer1_attention_weight_max": 17.128799438476562,
      "activations/layer1_attention_weight_min": -14.234378814697266,
      "activations/layer20_attention_weight_max": 22.468158721923828,
      "activations/layer20_attention_weight_min": -20.27297592163086,
      "activations/layer21_attention_weight_max": 31.111120223999023,
      "activations/layer21_attention_weight_min": -21.249448776245117,
      "activations/layer22_attention_weight_max": 29.37820816040039,
      "activations/layer22_attention_weight_min": -24.21360206604004,
      "activations/layer23_attention_weight_max": 30.404523849487305,
      "activations/layer23_attention_weight_min": -22.81866455078125,
      "activations/layer2_attention_weight_max": 30.61337661743164,
      "activations/layer2_attention_weight_min": -29.37224578857422,
      "activations/layer3_attention_weight_max": 92.43033599853516,
      "activations/layer3_attention_weight_min": -90.79402923583984,
      "activations/layer4_attention_weight_max": 85.65006256103516,
      "activations/layer4_attention_weight_min": -82.94054412841797,
      "activations/layer5_attention_weight_max": 65.87518310546875,
      "activations/layer5_attention_weight_min": -70.19552612304688,
      "activations/layer6_attention_weight_max": 46.277347564697266,
      "activations/layer6_attention_weight_min": -48.929542541503906,
      "activations/layer7_attention_weight_max": 59.67571258544922,
      "activations/layer7_attention_weight_min": -56.679603576660156,
      "activations/layer8_attention_weight_max": 42.61060333251953,
      "activations/layer8_attention_weight_min": -44.76045608520508,
      "activations/layer9_attention_weight_max": 50.30718231201172,
      "activations/layer9_attention_weight_min": -45.12169647216797,
      "epoch": 14.5,
      "learning_rate": 5.704810606060606e-05,
      "loss": 2.7581,
      "step": 249600
    },
    {
      "activations/layer0_attention_weight_max": 15.163297653198242,
      "activations/layer0_attention_weight_min": -13.895814895629883,
      "activations/layer10_attention_weight_max": 36.4876708984375,
      "activations/layer10_attention_weight_min": -35.42604064941406,
      "activations/layer11_attention_weight_max": 34.634063720703125,
      "activations/layer11_attention_weight_min": -34.767276763916016,
      "activations/layer12_attention_weight_max": 18.605979919433594,
      "activations/layer12_attention_weight_min": -29.29351043701172,
      "activations/layer13_attention_weight_max": 32.901023864746094,
      "activations/layer13_attention_weight_min": -29.917503356933594,
      "activations/layer14_attention_weight_max": 37.66819381713867,
      "activations/layer14_attention_weight_min": -31.63526725769043,
      "activations/layer15_attention_weight_max": 39.92051315307617,
      "activations/layer15_attention_weight_min": -32.27056884765625,
      "activations/layer16_attention_weight_max": 36.72828674316406,
      "activations/layer16_attention_weight_min": -33.51811981201172,
      "activations/layer17_attention_weight_max": 61.70507049560547,
      "activations/layer17_attention_weight_min": -48.58356857299805,
      "activations/layer18_attention_weight_max": 50.14480972290039,
      "activations/layer18_attention_weight_min": -43.5582275390625,
      "activations/layer19_attention_weight_max": 23.846805572509766,
      "activations/layer19_attention_weight_min": -25.809417724609375,
      "activations/layer1_attention_weight_max": 18.038875579833984,
      "activations/layer1_attention_weight_min": -14.780776977539062,
      "activations/layer20_attention_weight_max": 27.222434997558594,
      "activations/layer20_attention_weight_min": -26.456098556518555,
      "activations/layer21_attention_weight_max": 48.2486572265625,
      "activations/layer21_attention_weight_min": -28.752986907958984,
      "activations/layer22_attention_weight_max": 33.59504699707031,
      "activations/layer22_attention_weight_min": -26.981124877929688,
      "activations/layer23_attention_weight_max": 36.90032958984375,
      "activations/layer23_attention_weight_min": -24.63454246520996,
      "activations/layer2_attention_weight_max": 32.83634567260742,
      "activations/layer2_attention_weight_min": -31.042631149291992,
      "activations/layer3_attention_weight_max": 91.1930160522461,
      "activations/layer3_attention_weight_min": -95.0783462524414,
      "activations/layer4_attention_weight_max": 90.13111114501953,
      "activations/layer4_attention_weight_min": -86.95944213867188,
      "activations/layer5_attention_weight_max": 71.34471893310547,
      "activations/layer5_attention_weight_min": -73.83805847167969,
      "activations/layer6_attention_weight_max": 54.36567306518555,
      "activations/layer6_attention_weight_min": -49.505348205566406,
      "activations/layer7_attention_weight_max": 64.64558410644531,
      "activations/layer7_attention_weight_min": -59.750572204589844,
      "activations/layer8_attention_weight_max": 43.7651481628418,
      "activations/layer8_attention_weight_min": -46.28028106689453,
      "activations/layer9_attention_weight_max": 44.2811279296875,
      "activations/layer9_attention_weight_min": -47.63380432128906,
      "epoch": 14.51,
      "learning_rate": 5.702916666666666e-05,
      "loss": 2.7707,
      "step": 249650
    },
    {
      "activations/layer0_attention_weight_max": 16.47020149230957,
      "activations/layer0_attention_weight_min": -13.20618724822998,
      "activations/layer10_attention_weight_max": 34.085289001464844,
      "activations/layer10_attention_weight_min": -34.140071868896484,
      "activations/layer11_attention_weight_max": 31.69192886352539,
      "activations/layer11_attention_weight_min": -32.694000244140625,
      "activations/layer12_attention_weight_max": 19.59182357788086,
      "activations/layer12_attention_weight_min": -24.08991241455078,
      "activations/layer13_attention_weight_max": 33.5225944519043,
      "activations/layer13_attention_weight_min": -27.558122634887695,
      "activations/layer14_attention_weight_max": 34.5877685546875,
      "activations/layer14_attention_weight_min": -29.297367095947266,
      "activations/layer15_attention_weight_max": 32.0799674987793,
      "activations/layer15_attention_weight_min": -30.078968048095703,
      "activations/layer16_attention_weight_max": 34.41388702392578,
      "activations/layer16_attention_weight_min": -31.327693939208984,
      "activations/layer17_attention_weight_max": 49.8658447265625,
      "activations/layer17_attention_weight_min": -46.584617614746094,
      "activations/layer18_attention_weight_max": 43.799415588378906,
      "activations/layer18_attention_weight_min": -38.056026458740234,
      "activations/layer19_attention_weight_max": 21.14406394958496,
      "activations/layer19_attention_weight_min": -22.601154327392578,
      "activations/layer1_attention_weight_max": 15.953279495239258,
      "activations/layer1_attention_weight_min": -15.936501502990723,
      "activations/layer20_attention_weight_max": 21.56608772277832,
      "activations/layer20_attention_weight_min": -22.3797664642334,
      "activations/layer21_attention_weight_max": 36.78334426879883,
      "activations/layer21_attention_weight_min": -22.092824935913086,
      "activations/layer22_attention_weight_max": 28.062381744384766,
      "activations/layer22_attention_weight_min": -24.433427810668945,
      "activations/layer23_attention_weight_max": 38.03934097290039,
      "activations/layer23_attention_weight_min": -22.54476547241211,
      "activations/layer2_attention_weight_max": 31.82559585571289,
      "activations/layer2_attention_weight_min": -32.129547119140625,
      "activations/layer3_attention_weight_max": 88.90888977050781,
      "activations/layer3_attention_weight_min": -91.27389526367188,
      "activations/layer4_attention_weight_max": 82.03225708007812,
      "activations/layer4_attention_weight_min": -81.96106719970703,
      "activations/layer5_attention_weight_max": 65.85799407958984,
      "activations/layer5_attention_weight_min": -77.54682922363281,
      "activations/layer6_attention_weight_max": 48.06403350830078,
      "activations/layer6_attention_weight_min": -47.64436340332031,
      "activations/layer7_attention_weight_max": 63.5507698059082,
      "activations/layer7_attention_weight_min": -60.64280700683594,
      "activations/layer8_attention_weight_max": 42.32615280151367,
      "activations/layer8_attention_weight_min": -45.46157455444336,
      "activations/layer9_attention_weight_max": 47.5129280090332,
      "activations/layer9_attention_weight_min": -43.38808822631836,
      "epoch": 14.51,
      "learning_rate": 5.701022727272727e-05,
      "loss": 2.756,
      "step": 249700
    },
    {
      "activations/layer0_attention_weight_max": 15.557293891906738,
      "activations/layer0_attention_weight_min": -13.434653282165527,
      "activations/layer10_attention_weight_max": 36.26232147216797,
      "activations/layer10_attention_weight_min": -35.190452575683594,
      "activations/layer11_attention_weight_max": 33.7132682800293,
      "activations/layer11_attention_weight_min": -32.67915344238281,
      "activations/layer12_attention_weight_max": 19.559680938720703,
      "activations/layer12_attention_weight_min": -26.061979293823242,
      "activations/layer13_attention_weight_max": 38.43955612182617,
      "activations/layer13_attention_weight_min": -27.76245880126953,
      "activations/layer14_attention_weight_max": 38.441184997558594,
      "activations/layer14_attention_weight_min": -33.86237716674805,
      "activations/layer15_attention_weight_max": 34.22178268432617,
      "activations/layer15_attention_weight_min": -30.253562927246094,
      "activations/layer16_attention_weight_max": 33.63125991821289,
      "activations/layer16_attention_weight_min": -32.08818435668945,
      "activations/layer17_attention_weight_max": 53.51188659667969,
      "activations/layer17_attention_weight_min": -46.45738983154297,
      "activations/layer18_attention_weight_max": 48.69550323486328,
      "activations/layer18_attention_weight_min": -41.16020584106445,
      "activations/layer19_attention_weight_max": 22.802061080932617,
      "activations/layer19_attention_weight_min": -25.056285858154297,
      "activations/layer1_attention_weight_max": 16.863544464111328,
      "activations/layer1_attention_weight_min": -15.890192031860352,
      "activations/layer20_attention_weight_max": 22.298147201538086,
      "activations/layer20_attention_weight_min": -22.298851013183594,
      "activations/layer21_attention_weight_max": 41.50531005859375,
      "activations/layer21_attention_weight_min": -24.36795997619629,
      "activations/layer22_attention_weight_max": 30.28946876525879,
      "activations/layer22_attention_weight_min": -26.81563377380371,
      "activations/layer23_attention_weight_max": 38.523468017578125,
      "activations/layer23_attention_weight_min": -23.4188232421875,
      "activations/layer2_attention_weight_max": 31.569190979003906,
      "activations/layer2_attention_weight_min": -30.34493064880371,
      "activations/layer3_attention_weight_max": 89.5297622680664,
      "activations/layer3_attention_weight_min": -86.24806213378906,
      "activations/layer4_attention_weight_max": 85.0138168334961,
      "activations/layer4_attention_weight_min": -82.20545196533203,
      "activations/layer5_attention_weight_max": 66.47166442871094,
      "activations/layer5_attention_weight_min": -73.60638427734375,
      "activations/layer6_attention_weight_max": 49.40861892700195,
      "activations/layer6_attention_weight_min": -49.417598724365234,
      "activations/layer7_attention_weight_max": 63.1100959777832,
      "activations/layer7_attention_weight_min": -60.410587310791016,
      "activations/layer8_attention_weight_max": 45.3105354309082,
      "activations/layer8_attention_weight_min": -46.74187469482422,
      "activations/layer9_attention_weight_max": 43.720516204833984,
      "activations/layer9_attention_weight_min": -44.12528610229492,
      "epoch": 14.51,
      "learning_rate": 5.6991287878787876e-05,
      "loss": 2.7539,
      "step": 249750
    },
    {
      "activations/layer0_attention_weight_max": 15.604710578918457,
      "activations/layer0_attention_weight_min": -13.640796661376953,
      "activations/layer10_attention_weight_max": 37.941192626953125,
      "activations/layer10_attention_weight_min": -34.9638557434082,
      "activations/layer11_attention_weight_max": 33.65513610839844,
      "activations/layer11_attention_weight_min": -33.57288360595703,
      "activations/layer12_attention_weight_max": 17.770408630371094,
      "activations/layer12_attention_weight_min": -27.3676700592041,
      "activations/layer13_attention_weight_max": 35.38320541381836,
      "activations/layer13_attention_weight_min": -27.88179588317871,
      "activations/layer14_attention_weight_max": 35.447078704833984,
      "activations/layer14_attention_weight_min": -32.91169738769531,
      "activations/layer15_attention_weight_max": 31.19147491455078,
      "activations/layer15_attention_weight_min": -30.434463500976562,
      "activations/layer16_attention_weight_max": 31.294910430908203,
      "activations/layer16_attention_weight_min": -28.778831481933594,
      "activations/layer17_attention_weight_max": 51.0645637512207,
      "activations/layer17_attention_weight_min": -44.89371109008789,
      "activations/layer18_attention_weight_max": 44.194183349609375,
      "activations/layer18_attention_weight_min": -39.0984992980957,
      "activations/layer19_attention_weight_max": 24.143198013305664,
      "activations/layer19_attention_weight_min": -22.60590171813965,
      "activations/layer1_attention_weight_max": 16.17959213256836,
      "activations/layer1_attention_weight_min": -16.790809631347656,
      "activations/layer20_attention_weight_max": 22.605222702026367,
      "activations/layer20_attention_weight_min": -23.111488342285156,
      "activations/layer21_attention_weight_max": 36.180824279785156,
      "activations/layer21_attention_weight_min": -25.182451248168945,
      "activations/layer22_attention_weight_max": 28.18316078186035,
      "activations/layer22_attention_weight_min": -27.20470428466797,
      "activations/layer23_attention_weight_max": 34.25543975830078,
      "activations/layer23_attention_weight_min": -24.804288864135742,
      "activations/layer2_attention_weight_max": 32.508270263671875,
      "activations/layer2_attention_weight_min": -30.584659576416016,
      "activations/layer3_attention_weight_max": 95.3392105102539,
      "activations/layer3_attention_weight_min": -99.29788208007812,
      "activations/layer4_attention_weight_max": 89.79536437988281,
      "activations/layer4_attention_weight_min": -93.05874633789062,
      "activations/layer5_attention_weight_max": 67.23585510253906,
      "activations/layer5_attention_weight_min": -75.9913330078125,
      "activations/layer6_attention_weight_max": 50.151119232177734,
      "activations/layer6_attention_weight_min": -50.942508697509766,
      "activations/layer7_attention_weight_max": 63.034568786621094,
      "activations/layer7_attention_weight_min": -62.677589416503906,
      "activations/layer8_attention_weight_max": 48.505924224853516,
      "activations/layer8_attention_weight_min": -52.26799392700195,
      "activations/layer9_attention_weight_max": 45.55826950073242,
      "activations/layer9_attention_weight_min": -47.529170989990234,
      "epoch": 14.51,
      "learning_rate": 5.6972348484848484e-05,
      "loss": 2.7532,
      "step": 249800
    },
    {
      "activations/layer0_attention_weight_max": 16.6522216796875,
      "activations/layer0_attention_weight_min": -13.448448181152344,
      "activations/layer10_attention_weight_max": 37.87031173706055,
      "activations/layer10_attention_weight_min": -36.93583297729492,
      "activations/layer11_attention_weight_max": 32.824424743652344,
      "activations/layer11_attention_weight_min": -33.42396545410156,
      "activations/layer12_attention_weight_max": 17.696949005126953,
      "activations/layer12_attention_weight_min": -27.433080673217773,
      "activations/layer13_attention_weight_max": 34.912322998046875,
      "activations/layer13_attention_weight_min": -30.38959312438965,
      "activations/layer14_attention_weight_max": 35.45423889160156,
      "activations/layer14_attention_weight_min": -30.785140991210938,
      "activations/layer15_attention_weight_max": 34.071678161621094,
      "activations/layer15_attention_weight_min": -32.10554504394531,
      "activations/layer16_attention_weight_max": 32.02153396606445,
      "activations/layer16_attention_weight_min": -31.256635665893555,
      "activations/layer17_attention_weight_max": 49.82689666748047,
      "activations/layer17_attention_weight_min": -46.058204650878906,
      "activations/layer18_attention_weight_max": 43.14389419555664,
      "activations/layer18_attention_weight_min": -35.91678237915039,
      "activations/layer19_attention_weight_max": 20.42223358154297,
      "activations/layer19_attention_weight_min": -20.290250778198242,
      "activations/layer1_attention_weight_max": 16.943008422851562,
      "activations/layer1_attention_weight_min": -15.993432998657227,
      "activations/layer20_attention_weight_max": 21.700599670410156,
      "activations/layer20_attention_weight_min": -22.785551071166992,
      "activations/layer21_attention_weight_max": 37.19709014892578,
      "activations/layer21_attention_weight_min": -23.125944137573242,
      "activations/layer22_attention_weight_max": 29.49673843383789,
      "activations/layer22_attention_weight_min": -25.73969268798828,
      "activations/layer23_attention_weight_max": 35.803932189941406,
      "activations/layer23_attention_weight_min": -25.58892059326172,
      "activations/layer2_attention_weight_max": 34.111717224121094,
      "activations/layer2_attention_weight_min": -29.805036544799805,
      "activations/layer3_attention_weight_max": 93.20414733886719,
      "activations/layer3_attention_weight_min": -91.22117614746094,
      "activations/layer4_attention_weight_max": 88.3212661743164,
      "activations/layer4_attention_weight_min": -88.19853210449219,
      "activations/layer5_attention_weight_max": 70.20407104492188,
      "activations/layer5_attention_weight_min": -71.12702178955078,
      "activations/layer6_attention_weight_max": 51.690731048583984,
      "activations/layer6_attention_weight_min": -51.39070129394531,
      "activations/layer7_attention_weight_max": 65.11918640136719,
      "activations/layer7_attention_weight_min": -63.706825256347656,
      "activations/layer8_attention_weight_max": 45.57069778442383,
      "activations/layer8_attention_weight_min": -48.83876037597656,
      "activations/layer9_attention_weight_max": 46.775028228759766,
      "activations/layer9_attention_weight_min": -46.68548583984375,
      "epoch": 14.52,
      "learning_rate": 5.6953409090909086e-05,
      "loss": 2.7481,
      "step": 249850
    },
    {
      "activations/layer0_attention_weight_max": 16.051191329956055,
      "activations/layer0_attention_weight_min": -13.585033416748047,
      "activations/layer10_attention_weight_max": 38.43384552001953,
      "activations/layer10_attention_weight_min": -38.77938461303711,
      "activations/layer11_attention_weight_max": 33.71225357055664,
      "activations/layer11_attention_weight_min": -34.0400505065918,
      "activations/layer12_attention_weight_max": 19.609403610229492,
      "activations/layer12_attention_weight_min": -23.966766357421875,
      "activations/layer13_attention_weight_max": 42.709434509277344,
      "activations/layer13_attention_weight_min": -31.03028106689453,
      "activations/layer14_attention_weight_max": 37.85015106201172,
      "activations/layer14_attention_weight_min": -31.815324783325195,
      "activations/layer15_attention_weight_max": 34.48638916015625,
      "activations/layer15_attention_weight_min": -29.708600997924805,
      "activations/layer16_attention_weight_max": 34.68288040161133,
      "activations/layer16_attention_weight_min": -31.9146728515625,
      "activations/layer17_attention_weight_max": 55.02922439575195,
      "activations/layer17_attention_weight_min": -46.423370361328125,
      "activations/layer18_attention_weight_max": 49.51539611816406,
      "activations/layer18_attention_weight_min": -40.36656951904297,
      "activations/layer19_attention_weight_max": 22.971664428710938,
      "activations/layer19_attention_weight_min": -22.556272506713867,
      "activations/layer1_attention_weight_max": 17.508386611938477,
      "activations/layer1_attention_weight_min": -15.74161434173584,
      "activations/layer20_attention_weight_max": 23.19138526916504,
      "activations/layer20_attention_weight_min": -21.37547492980957,
      "activations/layer21_attention_weight_max": 39.07047653198242,
      "activations/layer21_attention_weight_min": -26.901456832885742,
      "activations/layer22_attention_weight_max": 28.3568172454834,
      "activations/layer22_attention_weight_min": -26.62120819091797,
      "activations/layer23_attention_weight_max": 36.473419189453125,
      "activations/layer23_attention_weight_min": -24.478063583374023,
      "activations/layer2_attention_weight_max": 29.46108627319336,
      "activations/layer2_attention_weight_min": -28.52669906616211,
      "activations/layer3_attention_weight_max": 88.35916900634766,
      "activations/layer3_attention_weight_min": -90.86125946044922,
      "activations/layer4_attention_weight_max": 87.7868881225586,
      "activations/layer4_attention_weight_min": -84.7578125,
      "activations/layer5_attention_weight_max": 66.05278015136719,
      "activations/layer5_attention_weight_min": -72.55371856689453,
      "activations/layer6_attention_weight_max": 50.513423919677734,
      "activations/layer6_attention_weight_min": -50.89186096191406,
      "activations/layer7_attention_weight_max": 63.422218322753906,
      "activations/layer7_attention_weight_min": -62.89917755126953,
      "activations/layer8_attention_weight_max": 48.654293060302734,
      "activations/layer8_attention_weight_min": -55.49431228637695,
      "activations/layer9_attention_weight_max": 48.39857482910156,
      "activations/layer9_attention_weight_min": -48.39784240722656,
      "epoch": 14.52,
      "learning_rate": 5.693446969696969e-05,
      "loss": 2.7529,
      "step": 249900
    },
    {
      "activations/layer0_attention_weight_max": 15.184511184692383,
      "activations/layer0_attention_weight_min": -13.634446144104004,
      "activations/layer10_attention_weight_max": 35.733741760253906,
      "activations/layer10_attention_weight_min": -33.35400390625,
      "activations/layer11_attention_weight_max": 32.64069366455078,
      "activations/layer11_attention_weight_min": -32.692230224609375,
      "activations/layer12_attention_weight_max": 16.61040687561035,
      "activations/layer12_attention_weight_min": -24.66104507446289,
      "activations/layer13_attention_weight_max": 31.985979080200195,
      "activations/layer13_attention_weight_min": -26.37824249267578,
      "activations/layer14_attention_weight_max": 34.69074630737305,
      "activations/layer14_attention_weight_min": -29.95186424255371,
      "activations/layer15_attention_weight_max": 32.49275207519531,
      "activations/layer15_attention_weight_min": -31.254756927490234,
      "activations/layer16_attention_weight_max": 29.718992233276367,
      "activations/layer16_attention_weight_min": -29.025747299194336,
      "activations/layer17_attention_weight_max": 49.94971466064453,
      "activations/layer17_attention_weight_min": -42.78303527832031,
      "activations/layer18_attention_weight_max": 45.563758850097656,
      "activations/layer18_attention_weight_min": -38.00893783569336,
      "activations/layer19_attention_weight_max": 23.79030418395996,
      "activations/layer19_attention_weight_min": -22.323680877685547,
      "activations/layer1_attention_weight_max": 16.02303695678711,
      "activations/layer1_attention_weight_min": -14.722954750061035,
      "activations/layer20_attention_weight_max": 23.91969871520996,
      "activations/layer20_attention_weight_min": -21.943038940429688,
      "activations/layer21_attention_weight_max": 36.82374572753906,
      "activations/layer21_attention_weight_min": -20.902755737304688,
      "activations/layer22_attention_weight_max": 28.7829647064209,
      "activations/layer22_attention_weight_min": -24.423809051513672,
      "activations/layer23_attention_weight_max": 36.10727310180664,
      "activations/layer23_attention_weight_min": -23.89398765563965,
      "activations/layer2_attention_weight_max": 32.18803787231445,
      "activations/layer2_attention_weight_min": -30.047107696533203,
      "activations/layer3_attention_weight_max": 96.16957092285156,
      "activations/layer3_attention_weight_min": -96.1029281616211,
      "activations/layer4_attention_weight_max": 87.62831115722656,
      "activations/layer4_attention_weight_min": -87.16390228271484,
      "activations/layer5_attention_weight_max": 66.84588623046875,
      "activations/layer5_attention_weight_min": -75.44610595703125,
      "activations/layer6_attention_weight_max": 51.65327835083008,
      "activations/layer6_attention_weight_min": -49.77510070800781,
      "activations/layer7_attention_weight_max": 63.115867614746094,
      "activations/layer7_attention_weight_min": -62.21275329589844,
      "activations/layer8_attention_weight_max": 43.5738525390625,
      "activations/layer8_attention_weight_min": -45.94696807861328,
      "activations/layer9_attention_weight_max": 44.12667465209961,
      "activations/layer9_attention_weight_min": -44.06880187988281,
      "epoch": 14.52,
      "learning_rate": 5.69155303030303e-05,
      "loss": 2.781,
      "step": 249950
    },
    {
      "activations/layer0_attention_weight_max": 15.703338623046875,
      "activations/layer0_attention_weight_min": -13.26290225982666,
      "activations/layer10_attention_weight_max": 36.45962142944336,
      "activations/layer10_attention_weight_min": -34.74825668334961,
      "activations/layer11_attention_weight_max": 32.30298614501953,
      "activations/layer11_attention_weight_min": -33.50148010253906,
      "activations/layer12_attention_weight_max": 17.76662826538086,
      "activations/layer12_attention_weight_min": -26.451290130615234,
      "activations/layer13_attention_weight_max": 30.990266799926758,
      "activations/layer13_attention_weight_min": -26.857959747314453,
      "activations/layer14_attention_weight_max": 33.79515075683594,
      "activations/layer14_attention_weight_min": -30.298879623413086,
      "activations/layer15_attention_weight_max": 31.25335121154785,
      "activations/layer15_attention_weight_min": -29.945770263671875,
      "activations/layer16_attention_weight_max": 31.8731689453125,
      "activations/layer16_attention_weight_min": -29.769664764404297,
      "activations/layer17_attention_weight_max": 51.16102981567383,
      "activations/layer17_attention_weight_min": -43.73204040527344,
      "activations/layer18_attention_weight_max": 46.16344451904297,
      "activations/layer18_attention_weight_min": -37.87548828125,
      "activations/layer19_attention_weight_max": 23.66594886779785,
      "activations/layer19_attention_weight_min": -21.7099666595459,
      "activations/layer1_attention_weight_max": 17.463115692138672,
      "activations/layer1_attention_weight_min": -15.974213600158691,
      "activations/layer20_attention_weight_max": 21.48558235168457,
      "activations/layer20_attention_weight_min": -20.40694808959961,
      "activations/layer21_attention_weight_max": 33.94750213623047,
      "activations/layer21_attention_weight_min": -20.587921142578125,
      "activations/layer22_attention_weight_max": 28.99618911743164,
      "activations/layer22_attention_weight_min": -25.26397132873535,
      "activations/layer23_attention_weight_max": 33.1792106628418,
      "activations/layer23_attention_weight_min": -24.515546798706055,
      "activations/layer2_attention_weight_max": 31.579547882080078,
      "activations/layer2_attention_weight_min": -29.413074493408203,
      "activations/layer3_attention_weight_max": 96.17316436767578,
      "activations/layer3_attention_weight_min": -96.80235290527344,
      "activations/layer4_attention_weight_max": 91.38666534423828,
      "activations/layer4_attention_weight_min": -91.62635803222656,
      "activations/layer5_attention_weight_max": 74.38182830810547,
      "activations/layer5_attention_weight_min": -82.86346435546875,
      "activations/layer6_attention_weight_max": 51.89779281616211,
      "activations/layer6_attention_weight_min": -53.850990295410156,
      "activations/layer7_attention_weight_max": 66.0106430053711,
      "activations/layer7_attention_weight_min": -64.00275421142578,
      "activations/layer8_attention_weight_max": 46.695648193359375,
      "activations/layer8_attention_weight_min": -51.02722930908203,
      "activations/layer9_attention_weight_max": 47.133052825927734,
      "activations/layer9_attention_weight_min": -47.07234191894531,
      "epoch": 14.53,
      "learning_rate": 5.6896590909090904e-05,
      "loss": 2.7613,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4582,
      "eval_samples_per_second": 507.673,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4582,
      "eval_openwebtext_samples_per_second": 507.673,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9323,
      "eval_wikitext_samples_per_second": 235.983,
      "step": 250000
    },
    {
      "epoch": 14.53,
      "eval_lambada_loss": 2.564453125,
      "eval_lambada_ppl": 12.993550578338562,
      "eval_lambada_runtime": 9.5594,
      "eval_lambada_samples_per_second": 509.343,
      "step": 250000
    },
    {
      "activations/layer0_attention_weight_max": 17.276762008666992,
      "activations/layer0_attention_weight_min": -13.307988166809082,
      "activations/layer10_attention_weight_max": 40.77215576171875,
      "activations/layer10_attention_weight_min": -36.62824630737305,
      "activations/layer11_attention_weight_max": 35.397396087646484,
      "activations/layer11_attention_weight_min": -37.024620056152344,
      "activations/layer12_attention_weight_max": 19.260557174682617,
      "activations/layer12_attention_weight_min": -24.70006561279297,
      "activations/layer13_attention_weight_max": 39.986671447753906,
      "activations/layer13_attention_weight_min": -28.01551055908203,
      "activations/layer14_attention_weight_max": 36.52201843261719,
      "activations/layer14_attention_weight_min": -30.629526138305664,
      "activations/layer15_attention_weight_max": 35.15858840942383,
      "activations/layer15_attention_weight_min": -31.052242279052734,
      "activations/layer16_attention_weight_max": 33.54030990600586,
      "activations/layer16_attention_weight_min": -31.685991287231445,
      "activations/layer17_attention_weight_max": 53.6900520324707,
      "activations/layer17_attention_weight_min": -45.286109924316406,
      "activations/layer18_attention_weight_max": 49.223670959472656,
      "activations/layer18_attention_weight_min": -39.326446533203125,
      "activations/layer19_attention_weight_max": 24.13289451599121,
      "activations/layer19_attention_weight_min": -21.963258743286133,
      "activations/layer1_attention_weight_max": 15.879056930541992,
      "activations/layer1_attention_weight_min": -13.553448677062988,
      "activations/layer20_attention_weight_max": 23.164710998535156,
      "activations/layer20_attention_weight_min": -21.195030212402344,
      "activations/layer21_attention_weight_max": 37.54021453857422,
      "activations/layer21_attention_weight_min": -21.213464736938477,
      "activations/layer22_attention_weight_max": 30.313976287841797,
      "activations/layer22_attention_weight_min": -26.60690689086914,
      "activations/layer23_attention_weight_max": 38.58279800415039,
      "activations/layer23_attention_weight_min": -24.257429122924805,
      "activations/layer2_attention_weight_max": 31.643592834472656,
      "activations/layer2_attention_weight_min": -29.45509910583496,
      "activations/layer3_attention_weight_max": 95.4366683959961,
      "activations/layer3_attention_weight_min": -100.44950866699219,
      "activations/layer4_attention_weight_max": 94.14968872070312,
      "activations/layer4_attention_weight_min": -91.98018646240234,
      "activations/layer5_attention_weight_max": 72.41065979003906,
      "activations/layer5_attention_weight_min": -78.79072570800781,
      "activations/layer6_attention_weight_max": 55.023258209228516,
      "activations/layer6_attention_weight_min": -52.60554885864258,
      "activations/layer7_attention_weight_max": 66.10511779785156,
      "activations/layer7_attention_weight_min": -64.6107406616211,
      "activations/layer8_attention_weight_max": 48.89393997192383,
      "activations/layer8_attention_weight_min": -51.45293426513672,
      "activations/layer9_attention_weight_max": 47.57356643676758,
      "activations/layer9_attention_weight_min": -48.22431564331055,
      "epoch": 14.53,
      "learning_rate": 5.687765151515151e-05,
      "loss": 2.7581,
      "step": 250050
    },
    {
      "activations/layer0_attention_weight_max": 16.824140548706055,
      "activations/layer0_attention_weight_min": -13.6483154296875,
      "activations/layer10_attention_weight_max": 42.75200271606445,
      "activations/layer10_attention_weight_min": -39.58064270019531,
      "activations/layer11_attention_weight_max": 38.94658660888672,
      "activations/layer11_attention_weight_min": -36.18025207519531,
      "activations/layer12_attention_weight_max": 20.793689727783203,
      "activations/layer12_attention_weight_min": -24.937244415283203,
      "activations/layer13_attention_weight_max": 68.27042388916016,
      "activations/layer13_attention_weight_min": -34.68867492675781,
      "activations/layer14_attention_weight_max": 72.91804504394531,
      "activations/layer14_attention_weight_min": -36.02405548095703,
      "activations/layer15_attention_weight_max": 63.2065315246582,
      "activations/layer15_attention_weight_min": -32.20657730102539,
      "activations/layer16_attention_weight_max": 35.40642547607422,
      "activations/layer16_attention_weight_min": -31.056840896606445,
      "activations/layer17_attention_weight_max": 54.68370056152344,
      "activations/layer17_attention_weight_min": -46.54756164550781,
      "activations/layer18_attention_weight_max": 48.72378158569336,
      "activations/layer18_attention_weight_min": -40.54880142211914,
      "activations/layer19_attention_weight_max": 24.023109436035156,
      "activations/layer19_attention_weight_min": -23.905475616455078,
      "activations/layer1_attention_weight_max": 17.648168563842773,
      "activations/layer1_attention_weight_min": -15.541444778442383,
      "activations/layer20_attention_weight_max": 23.094036102294922,
      "activations/layer20_attention_weight_min": -25.50897789001465,
      "activations/layer21_attention_weight_max": 35.76944351196289,
      "activations/layer21_attention_weight_min": -27.14634895324707,
      "activations/layer22_attention_weight_max": 32.17915344238281,
      "activations/layer22_attention_weight_min": -26.31818199157715,
      "activations/layer23_attention_weight_max": 38.03904724121094,
      "activations/layer23_attention_weight_min": -23.282285690307617,
      "activations/layer2_attention_weight_max": 33.34097671508789,
      "activations/layer2_attention_weight_min": -29.894323348999023,
      "activations/layer3_attention_weight_max": 95.02043151855469,
      "activations/layer3_attention_weight_min": -98.37885284423828,
      "activations/layer4_attention_weight_max": 93.54109954833984,
      "activations/layer4_attention_weight_min": -93.78831481933594,
      "activations/layer5_attention_weight_max": 75.12191772460938,
      "activations/layer5_attention_weight_min": -72.97361755371094,
      "activations/layer6_attention_weight_max": 53.611534118652344,
      "activations/layer6_attention_weight_min": -54.32431411743164,
      "activations/layer7_attention_weight_max": 70.60836029052734,
      "activations/layer7_attention_weight_min": -70.93777465820312,
      "activations/layer8_attention_weight_max": 53.62028884887695,
      "activations/layer8_attention_weight_min": -53.51250076293945,
      "activations/layer9_attention_weight_max": 57.50657272338867,
      "activations/layer9_attention_weight_min": -52.673458099365234,
      "epoch": 14.53,
      "learning_rate": 5.685871212121211e-05,
      "loss": 2.7706,
      "step": 250100
    },
    {
      "activations/layer0_attention_weight_max": 16.28310775756836,
      "activations/layer0_attention_weight_min": -13.507226943969727,
      "activations/layer10_attention_weight_max": 35.690059661865234,
      "activations/layer10_attention_weight_min": -37.041358947753906,
      "activations/layer11_attention_weight_max": 31.00490379333496,
      "activations/layer11_attention_weight_min": -34.35258865356445,
      "activations/layer12_attention_weight_max": 17.08244514465332,
      "activations/layer12_attention_weight_min": -25.143112182617188,
      "activations/layer13_attention_weight_max": 29.144649505615234,
      "activations/layer13_attention_weight_min": -30.2011661529541,
      "activations/layer14_attention_weight_max": 32.129520416259766,
      "activations/layer14_attention_weight_min": -29.04644203186035,
      "activations/layer15_attention_weight_max": 28.086681365966797,
      "activations/layer15_attention_weight_min": -28.797571182250977,
      "activations/layer16_attention_weight_max": 28.65635871887207,
      "activations/layer16_attention_weight_min": -30.593631744384766,
      "activations/layer17_attention_weight_max": 47.7449836730957,
      "activations/layer17_attention_weight_min": -41.937660217285156,
      "activations/layer18_attention_weight_max": 42.58427429199219,
      "activations/layer18_attention_weight_min": -37.05473327636719,
      "activations/layer19_attention_weight_max": 22.016361236572266,
      "activations/layer19_attention_weight_min": -20.689804077148438,
      "activations/layer1_attention_weight_max": 15.608222961425781,
      "activations/layer1_attention_weight_min": -15.444234848022461,
      "activations/layer20_attention_weight_max": 22.028457641601562,
      "activations/layer20_attention_weight_min": -21.47111701965332,
      "activations/layer21_attention_weight_max": 31.237533569335938,
      "activations/layer21_attention_weight_min": -20.988933563232422,
      "activations/layer22_attention_weight_max": 29.80365562438965,
      "activations/layer22_attention_weight_min": -25.09376335144043,
      "activations/layer23_attention_weight_max": 35.43877410888672,
      "activations/layer23_attention_weight_min": -22.989402770996094,
      "activations/layer2_attention_weight_max": 32.04616165161133,
      "activations/layer2_attention_weight_min": -30.912748336791992,
      "activations/layer3_attention_weight_max": 98.7553939819336,
      "activations/layer3_attention_weight_min": -100.55712127685547,
      "activations/layer4_attention_weight_max": 90.5569076538086,
      "activations/layer4_attention_weight_min": -89.67120361328125,
      "activations/layer5_attention_weight_max": 65.69441986083984,
      "activations/layer5_attention_weight_min": -73.71427917480469,
      "activations/layer6_attention_weight_max": 50.04357147216797,
      "activations/layer6_attention_weight_min": -54.53541946411133,
      "activations/layer7_attention_weight_max": 62.78639221191406,
      "activations/layer7_attention_weight_min": -61.806602478027344,
      "activations/layer8_attention_weight_max": 44.997127532958984,
      "activations/layer8_attention_weight_min": -49.53186798095703,
      "activations/layer9_attention_weight_max": 44.246402740478516,
      "activations/layer9_attention_weight_min": -47.88764953613281,
      "epoch": 14.54,
      "learning_rate": 5.683977272727273e-05,
      "loss": 2.7462,
      "step": 250150
    },
    {
      "activations/layer0_attention_weight_max": 15.252107620239258,
      "activations/layer0_attention_weight_min": -13.291996955871582,
      "activations/layer10_attention_weight_max": 39.83637237548828,
      "activations/layer10_attention_weight_min": -36.38922882080078,
      "activations/layer11_attention_weight_max": 36.04176712036133,
      "activations/layer11_attention_weight_min": -33.04090118408203,
      "activations/layer12_attention_weight_max": 19.6263427734375,
      "activations/layer12_attention_weight_min": -23.50391960144043,
      "activations/layer13_attention_weight_max": 39.475196838378906,
      "activations/layer13_attention_weight_min": -29.872102737426758,
      "activations/layer14_attention_weight_max": 36.26699447631836,
      "activations/layer14_attention_weight_min": -30.744932174682617,
      "activations/layer15_attention_weight_max": 33.055416107177734,
      "activations/layer15_attention_weight_min": -30.48667335510254,
      "activations/layer16_attention_weight_max": 33.54973220825195,
      "activations/layer16_attention_weight_min": -31.98378562927246,
      "activations/layer17_attention_weight_max": 54.29106903076172,
      "activations/layer17_attention_weight_min": -44.83134460449219,
      "activations/layer18_attention_weight_max": 49.18340301513672,
      "activations/layer18_attention_weight_min": -39.47916793823242,
      "activations/layer19_attention_weight_max": 25.072734832763672,
      "activations/layer19_attention_weight_min": -22.883108139038086,
      "activations/layer1_attention_weight_max": 17.19356346130371,
      "activations/layer1_attention_weight_min": -16.830734252929688,
      "activations/layer20_attention_weight_max": 24.19228172302246,
      "activations/layer20_attention_weight_min": -22.326616287231445,
      "activations/layer21_attention_weight_max": 37.39933776855469,
      "activations/layer21_attention_weight_min": -22.019432067871094,
      "activations/layer22_attention_weight_max": 31.195520401000977,
      "activations/layer22_attention_weight_min": -25.98993682861328,
      "activations/layer23_attention_weight_max": 40.005977630615234,
      "activations/layer23_attention_weight_min": -23.861221313476562,
      "activations/layer2_attention_weight_max": 32.18216323852539,
      "activations/layer2_attention_weight_min": -30.257938385009766,
      "activations/layer3_attention_weight_max": 93.71552276611328,
      "activations/layer3_attention_weight_min": -95.46078491210938,
      "activations/layer4_attention_weight_max": 90.98628997802734,
      "activations/layer4_attention_weight_min": -88.28961181640625,
      "activations/layer5_attention_weight_max": 72.16334533691406,
      "activations/layer5_attention_weight_min": -73.01439666748047,
      "activations/layer6_attention_weight_max": 52.49699401855469,
      "activations/layer6_attention_weight_min": -53.63412857055664,
      "activations/layer7_attention_weight_max": 66.26734161376953,
      "activations/layer7_attention_weight_min": -61.81199645996094,
      "activations/layer8_attention_weight_max": 48.95539855957031,
      "activations/layer8_attention_weight_min": -48.24468231201172,
      "activations/layer9_attention_weight_max": 45.42295455932617,
      "activations/layer9_attention_weight_min": -45.86566925048828,
      "epoch": 14.54,
      "learning_rate": 5.682083333333333e-05,
      "loss": 2.7791,
      "step": 250200
    },
    {
      "activations/layer0_attention_weight_max": 16.752147674560547,
      "activations/layer0_attention_weight_min": -13.573290824890137,
      "activations/layer10_attention_weight_max": 39.09218215942383,
      "activations/layer10_attention_weight_min": -37.24851989746094,
      "activations/layer11_attention_weight_max": 33.16862487792969,
      "activations/layer11_attention_weight_min": -33.49806213378906,
      "activations/layer12_attention_weight_max": 17.795276641845703,
      "activations/layer12_attention_weight_min": -30.14362144470215,
      "activations/layer13_attention_weight_max": 34.18870544433594,
      "activations/layer13_attention_weight_min": -26.63094711303711,
      "activations/layer14_attention_weight_max": 33.104759216308594,
      "activations/layer14_attention_weight_min": -27.92084312438965,
      "activations/layer15_attention_weight_max": 32.50576400756836,
      "activations/layer15_attention_weight_min": -27.230146408081055,
      "activations/layer16_attention_weight_max": 31.476150512695312,
      "activations/layer16_attention_weight_min": -28.87627601623535,
      "activations/layer17_attention_weight_max": 50.070396423339844,
      "activations/layer17_attention_weight_min": -41.13079833984375,
      "activations/layer18_attention_weight_max": 43.88154983520508,
      "activations/layer18_attention_weight_min": -34.66136169433594,
      "activations/layer19_attention_weight_max": 21.119895935058594,
      "activations/layer19_attention_weight_min": -22.449758529663086,
      "activations/layer1_attention_weight_max": 17.208654403686523,
      "activations/layer1_attention_weight_min": -13.775629997253418,
      "activations/layer20_attention_weight_max": 20.796785354614258,
      "activations/layer20_attention_weight_min": -20.13390350341797,
      "activations/layer21_attention_weight_max": 31.458364486694336,
      "activations/layer21_attention_weight_min": -19.569406509399414,
      "activations/layer22_attention_weight_max": 26.303911209106445,
      "activations/layer22_attention_weight_min": -25.50144386291504,
      "activations/layer23_attention_weight_max": 34.94455337524414,
      "activations/layer23_attention_weight_min": -21.591487884521484,
      "activations/layer2_attention_weight_max": 30.446880340576172,
      "activations/layer2_attention_weight_min": -29.866134643554688,
      "activations/layer3_attention_weight_max": 91.44682312011719,
      "activations/layer3_attention_weight_min": -96.01192474365234,
      "activations/layer4_attention_weight_max": 84.26271057128906,
      "activations/layer4_attention_weight_min": -86.87162017822266,
      "activations/layer5_attention_weight_max": 66.86097717285156,
      "activations/layer5_attention_weight_min": -73.71041107177734,
      "activations/layer6_attention_weight_max": 49.46864700317383,
      "activations/layer6_attention_weight_min": -50.40779113769531,
      "activations/layer7_attention_weight_max": 62.72962951660156,
      "activations/layer7_attention_weight_min": -60.88328170776367,
      "activations/layer8_attention_weight_max": 47.993080139160156,
      "activations/layer8_attention_weight_min": -48.638511657714844,
      "activations/layer9_attention_weight_max": 46.88603210449219,
      "activations/layer9_attention_weight_min": -49.786746978759766,
      "epoch": 14.54,
      "learning_rate": 5.680189393939393e-05,
      "loss": 2.7722,
      "step": 250250
    },
    {
      "activations/layer0_attention_weight_max": 15.97790813446045,
      "activations/layer0_attention_weight_min": -13.402229309082031,
      "activations/layer10_attention_weight_max": 35.57283401489258,
      "activations/layer10_attention_weight_min": -35.1135139465332,
      "activations/layer11_attention_weight_max": 32.1547966003418,
      "activations/layer11_attention_weight_min": -33.95629119873047,
      "activations/layer12_attention_weight_max": 24.036420822143555,
      "activations/layer12_attention_weight_min": -25.47911262512207,
      "activations/layer13_attention_weight_max": 38.570350646972656,
      "activations/layer13_attention_weight_min": -29.770164489746094,
      "activations/layer14_attention_weight_max": 36.478790283203125,
      "activations/layer14_attention_weight_min": -30.49677276611328,
      "activations/layer15_attention_weight_max": 33.317169189453125,
      "activations/layer15_attention_weight_min": -30.075084686279297,
      "activations/layer16_attention_weight_max": 34.332122802734375,
      "activations/layer16_attention_weight_min": -29.941818237304688,
      "activations/layer17_attention_weight_max": 54.4537239074707,
      "activations/layer17_attention_weight_min": -45.3221435546875,
      "activations/layer18_attention_weight_max": 48.4320182800293,
      "activations/layer18_attention_weight_min": -40.54503631591797,
      "activations/layer19_attention_weight_max": 23.59986686706543,
      "activations/layer19_attention_weight_min": -21.53377342224121,
      "activations/layer1_attention_weight_max": 16.576976776123047,
      "activations/layer1_attention_weight_min": -14.561027526855469,
      "activations/layer20_attention_weight_max": 25.806795120239258,
      "activations/layer20_attention_weight_min": -23.670654296875,
      "activations/layer21_attention_weight_max": 36.35658645629883,
      "activations/layer21_attention_weight_min": -23.960649490356445,
      "activations/layer22_attention_weight_max": 27.96916389465332,
      "activations/layer22_attention_weight_min": -26.157991409301758,
      "activations/layer23_attention_weight_max": 36.626800537109375,
      "activations/layer23_attention_weight_min": -24.267372131347656,
      "activations/layer2_attention_weight_max": 31.211782455444336,
      "activations/layer2_attention_weight_min": -30.720386505126953,
      "activations/layer3_attention_weight_max": 89.71916961669922,
      "activations/layer3_attention_weight_min": -92.68596649169922,
      "activations/layer4_attention_weight_max": 85.8029556274414,
      "activations/layer4_attention_weight_min": -85.2381591796875,
      "activations/layer5_attention_weight_max": 71.74446105957031,
      "activations/layer5_attention_weight_min": -75.6322021484375,
      "activations/layer6_attention_weight_max": 52.09303283691406,
      "activations/layer6_attention_weight_min": -52.99215316772461,
      "activations/layer7_attention_weight_max": 63.73491668701172,
      "activations/layer7_attention_weight_min": -64.4573745727539,
      "activations/layer8_attention_weight_max": 44.55971145629883,
      "activations/layer8_attention_weight_min": -47.02829360961914,
      "activations/layer9_attention_weight_max": 43.47376251220703,
      "activations/layer9_attention_weight_min": -44.036407470703125,
      "epoch": 14.54,
      "learning_rate": 5.678295454545454e-05,
      "loss": 2.7621,
      "step": 250300
    },
    {
      "activations/layer0_attention_weight_max": 16.09225845336914,
      "activations/layer0_attention_weight_min": -13.42160415649414,
      "activations/layer10_attention_weight_max": 34.77803039550781,
      "activations/layer10_attention_weight_min": -34.32258224487305,
      "activations/layer11_attention_weight_max": 30.496719360351562,
      "activations/layer11_attention_weight_min": -31.765850067138672,
      "activations/layer12_attention_weight_max": 17.320579528808594,
      "activations/layer12_attention_weight_min": -26.544471740722656,
      "activations/layer13_attention_weight_max": 36.57469177246094,
      "activations/layer13_attention_weight_min": -28.10342025756836,
      "activations/layer14_attention_weight_max": 32.17570877075195,
      "activations/layer14_attention_weight_min": -28.786296844482422,
      "activations/layer15_attention_weight_max": 29.993738174438477,
      "activations/layer15_attention_weight_min": -28.692203521728516,
      "activations/layer16_attention_weight_max": 28.50865364074707,
      "activations/layer16_attention_weight_min": -28.403575897216797,
      "activations/layer17_attention_weight_max": 43.5394172668457,
      "activations/layer17_attention_weight_min": -42.36629104614258,
      "activations/layer18_attention_weight_max": 40.74415969848633,
      "activations/layer18_attention_weight_min": -36.64215850830078,
      "activations/layer19_attention_weight_max": 21.315095901489258,
      "activations/layer19_attention_weight_min": -21.639982223510742,
      "activations/layer1_attention_weight_max": 16.97385597229004,
      "activations/layer1_attention_weight_min": -15.128479957580566,
      "activations/layer20_attention_weight_max": 23.390811920166016,
      "activations/layer20_attention_weight_min": -21.907020568847656,
      "activations/layer21_attention_weight_max": 31.421539306640625,
      "activations/layer21_attention_weight_min": -22.142038345336914,
      "activations/layer22_attention_weight_max": 26.27677345275879,
      "activations/layer22_attention_weight_min": -24.47796058654785,
      "activations/layer23_attention_weight_max": 34.99815368652344,
      "activations/layer23_attention_weight_min": -23.76714324951172,
      "activations/layer2_attention_weight_max": 32.13308334350586,
      "activations/layer2_attention_weight_min": -30.808151245117188,
      "activations/layer3_attention_weight_max": 92.83660125732422,
      "activations/layer3_attention_weight_min": -96.6292953491211,
      "activations/layer4_attention_weight_max": 90.5652084350586,
      "activations/layer4_attention_weight_min": -85.87901306152344,
      "activations/layer5_attention_weight_max": 67.35414123535156,
      "activations/layer5_attention_weight_min": -72.81719207763672,
      "activations/layer6_attention_weight_max": 49.61128234863281,
      "activations/layer6_attention_weight_min": -51.25063705444336,
      "activations/layer7_attention_weight_max": 61.267826080322266,
      "activations/layer7_attention_weight_min": -62.84081268310547,
      "activations/layer8_attention_weight_max": 43.1657600402832,
      "activations/layer8_attention_weight_min": -46.477256774902344,
      "activations/layer9_attention_weight_max": 45.058597564697266,
      "activations/layer9_attention_weight_min": -45.50782775878906,
      "epoch": 14.55,
      "learning_rate": 5.676401515151515e-05,
      "loss": 2.7737,
      "step": 250350
    },
    {
      "activations/layer0_attention_weight_max": 17.033000946044922,
      "activations/layer0_attention_weight_min": -13.596637725830078,
      "activations/layer10_attention_weight_max": 33.78748321533203,
      "activations/layer10_attention_weight_min": -33.71168899536133,
      "activations/layer11_attention_weight_max": 32.011390686035156,
      "activations/layer11_attention_weight_min": -33.799827575683594,
      "activations/layer12_attention_weight_max": 18.01181983947754,
      "activations/layer12_attention_weight_min": -24.824445724487305,
      "activations/layer13_attention_weight_max": 37.56241989135742,
      "activations/layer13_attention_weight_min": -31.578105926513672,
      "activations/layer14_attention_weight_max": 36.113182067871094,
      "activations/layer14_attention_weight_min": -29.80536460876465,
      "activations/layer15_attention_weight_max": 34.20222473144531,
      "activations/layer15_attention_weight_min": -28.871362686157227,
      "activations/layer16_attention_weight_max": 33.69666290283203,
      "activations/layer16_attention_weight_min": -30.701004028320312,
      "activations/layer17_attention_weight_max": 50.21916961669922,
      "activations/layer17_attention_weight_min": -44.1273078918457,
      "activations/layer18_attention_weight_max": 47.0096435546875,
      "activations/layer18_attention_weight_min": -38.100852966308594,
      "activations/layer19_attention_weight_max": 23.413576126098633,
      "activations/layer19_attention_weight_min": -22.028121948242188,
      "activations/layer1_attention_weight_max": 15.878087043762207,
      "activations/layer1_attention_weight_min": -14.177242279052734,
      "activations/layer20_attention_weight_max": 23.880247116088867,
      "activations/layer20_attention_weight_min": -20.702234268188477,
      "activations/layer21_attention_weight_max": 35.69767379760742,
      "activations/layer21_attention_weight_min": -23.565275192260742,
      "activations/layer22_attention_weight_max": 31.50701904296875,
      "activations/layer22_attention_weight_min": -25.35767936706543,
      "activations/layer23_attention_weight_max": 38.74603271484375,
      "activations/layer23_attention_weight_min": -23.45687484741211,
      "activations/layer2_attention_weight_max": 31.620208740234375,
      "activations/layer2_attention_weight_min": -30.856151580810547,
      "activations/layer3_attention_weight_max": 96.29412078857422,
      "activations/layer3_attention_weight_min": -97.4337158203125,
      "activations/layer4_attention_weight_max": 87.75284576416016,
      "activations/layer4_attention_weight_min": -84.48918914794922,
      "activations/layer5_attention_weight_max": 68.606689453125,
      "activations/layer5_attention_weight_min": -73.18836212158203,
      "activations/layer6_attention_weight_max": 52.45012283325195,
      "activations/layer6_attention_weight_min": -48.27239227294922,
      "activations/layer7_attention_weight_max": 64.478271484375,
      "activations/layer7_attention_weight_min": -62.71112060546875,
      "activations/layer8_attention_weight_max": 43.30491256713867,
      "activations/layer8_attention_weight_min": -46.08845520019531,
      "activations/layer9_attention_weight_max": 45.52544021606445,
      "activations/layer9_attention_weight_min": -46.6879997253418,
      "epoch": 14.55,
      "learning_rate": 5.6745075757575756e-05,
      "loss": 2.7513,
      "step": 250400
    },
    {
      "activations/layer0_attention_weight_max": 16.10540199279785,
      "activations/layer0_attention_weight_min": -13.129725456237793,
      "activations/layer10_attention_weight_max": 41.90711975097656,
      "activations/layer10_attention_weight_min": -40.93092346191406,
      "activations/layer11_attention_weight_max": 42.55743408203125,
      "activations/layer11_attention_weight_min": -37.47030258178711,
      "activations/layer12_attention_weight_max": 19.929819107055664,
      "activations/layer12_attention_weight_min": -27.95669174194336,
      "activations/layer13_attention_weight_max": 43.137882232666016,
      "activations/layer13_attention_weight_min": -30.564287185668945,
      "activations/layer14_attention_weight_max": 50.269752502441406,
      "activations/layer14_attention_weight_min": -32.69239044189453,
      "activations/layer15_attention_weight_max": 52.03264617919922,
      "activations/layer15_attention_weight_min": -32.525333404541016,
      "activations/layer16_attention_weight_max": 34.40312194824219,
      "activations/layer16_attention_weight_min": -34.31240463256836,
      "activations/layer17_attention_weight_max": 58.59556198120117,
      "activations/layer17_attention_weight_min": -53.30742263793945,
      "activations/layer18_attention_weight_max": 51.21391296386719,
      "activations/layer18_attention_weight_min": -41.67436981201172,
      "activations/layer19_attention_weight_max": 24.475996017456055,
      "activations/layer19_attention_weight_min": -24.671396255493164,
      "activations/layer1_attention_weight_max": 16.22882843017578,
      "activations/layer1_attention_weight_min": -16.701026916503906,
      "activations/layer20_attention_weight_max": 23.89519691467285,
      "activations/layer20_attention_weight_min": -22.6935977935791,
      "activations/layer21_attention_weight_max": 36.726051330566406,
      "activations/layer21_attention_weight_min": -23.011363983154297,
      "activations/layer22_attention_weight_max": 32.76082229614258,
      "activations/layer22_attention_weight_min": -26.596967697143555,
      "activations/layer23_attention_weight_max": 37.703575134277344,
      "activations/layer23_attention_weight_min": -23.19876480102539,
      "activations/layer2_attention_weight_max": 33.25580596923828,
      "activations/layer2_attention_weight_min": -31.993309020996094,
      "activations/layer3_attention_weight_max": 98.8919448852539,
      "activations/layer3_attention_weight_min": -101.22779083251953,
      "activations/layer4_attention_weight_max": 100.4183578491211,
      "activations/layer4_attention_weight_min": -94.21862030029297,
      "activations/layer5_attention_weight_max": 78.25466918945312,
      "activations/layer5_attention_weight_min": -80.4288330078125,
      "activations/layer6_attention_weight_max": 53.22972106933594,
      "activations/layer6_attention_weight_min": -54.11623001098633,
      "activations/layer7_attention_weight_max": 75.9133071899414,
      "activations/layer7_attention_weight_min": -70.77055358886719,
      "activations/layer8_attention_weight_max": 54.13783645629883,
      "activations/layer8_attention_weight_min": -52.16621780395508,
      "activations/layer9_attention_weight_max": 62.33264923095703,
      "activations/layer9_attention_weight_min": -51.202117919921875,
      "epoch": 14.55,
      "learning_rate": 5.672613636363636e-05,
      "loss": 2.7487,
      "step": 250450
    },
    {
      "activations/layer0_attention_weight_max": 16.486373901367188,
      "activations/layer0_attention_weight_min": -13.415849685668945,
      "activations/layer10_attention_weight_max": 35.90888214111328,
      "activations/layer10_attention_weight_min": -36.06303787231445,
      "activations/layer11_attention_weight_max": 32.94982147216797,
      "activations/layer11_attention_weight_min": -33.1730842590332,
      "activations/layer12_attention_weight_max": 18.057157516479492,
      "activations/layer12_attention_weight_min": -24.472476959228516,
      "activations/layer13_attention_weight_max": 33.23444366455078,
      "activations/layer13_attention_weight_min": -27.627357482910156,
      "activations/layer14_attention_weight_max": 36.09947967529297,
      "activations/layer14_attention_weight_min": -30.54171371459961,
      "activations/layer15_attention_weight_max": 32.33245086669922,
      "activations/layer15_attention_weight_min": -30.15899085998535,
      "activations/layer16_attention_weight_max": 30.44342041015625,
      "activations/layer16_attention_weight_min": -30.821455001831055,
      "activations/layer17_attention_weight_max": 47.716285705566406,
      "activations/layer17_attention_weight_min": -41.74278259277344,
      "activations/layer18_attention_weight_max": 43.637081146240234,
      "activations/layer18_attention_weight_min": -38.119956970214844,
      "activations/layer19_attention_weight_max": 23.41466522216797,
      "activations/layer19_attention_weight_min": -24.32667350769043,
      "activations/layer1_attention_weight_max": 16.663972854614258,
      "activations/layer1_attention_weight_min": -14.915157318115234,
      "activations/layer20_attention_weight_max": 22.9204158782959,
      "activations/layer20_attention_weight_min": -25.094837188720703,
      "activations/layer21_attention_weight_max": 34.197208404541016,
      "activations/layer21_attention_weight_min": -25.280868530273438,
      "activations/layer22_attention_weight_max": 29.001178741455078,
      "activations/layer22_attention_weight_min": -27.344743728637695,
      "activations/layer23_attention_weight_max": 37.49786376953125,
      "activations/layer23_attention_weight_min": -25.108915328979492,
      "activations/layer2_attention_weight_max": 31.32418441772461,
      "activations/layer2_attention_weight_min": -28.779027938842773,
      "activations/layer3_attention_weight_max": 91.31077575683594,
      "activations/layer3_attention_weight_min": -92.26191711425781,
      "activations/layer4_attention_weight_max": 88.40169525146484,
      "activations/layer4_attention_weight_min": -84.66943359375,
      "activations/layer5_attention_weight_max": 67.35318756103516,
      "activations/layer5_attention_weight_min": -72.00164794921875,
      "activations/layer6_attention_weight_max": 49.332664489746094,
      "activations/layer6_attention_weight_min": -50.209861755371094,
      "activations/layer7_attention_weight_max": 60.83594512939453,
      "activations/layer7_attention_weight_min": -61.304752349853516,
      "activations/layer8_attention_weight_max": 45.635009765625,
      "activations/layer8_attention_weight_min": -47.20814895629883,
      "activations/layer9_attention_weight_max": 42.90070724487305,
      "activations/layer9_attention_weight_min": -42.18551254272461,
      "epoch": 14.56,
      "learning_rate": 5.670719696969696e-05,
      "loss": 2.7524,
      "step": 250500
    },
    {
      "activations/layer0_attention_weight_max": 15.477088928222656,
      "activations/layer0_attention_weight_min": -12.977696418762207,
      "activations/layer10_attention_weight_max": 37.876949310302734,
      "activations/layer10_attention_weight_min": -38.313270568847656,
      "activations/layer11_attention_weight_max": 33.40592575073242,
      "activations/layer11_attention_weight_min": -33.358131408691406,
      "activations/layer12_attention_weight_max": 18.3184871673584,
      "activations/layer12_attention_weight_min": -24.44188117980957,
      "activations/layer13_attention_weight_max": 39.3323974609375,
      "activations/layer13_attention_weight_min": -31.04096031188965,
      "activations/layer14_attention_weight_max": 40.618167877197266,
      "activations/layer14_attention_weight_min": -31.15563201904297,
      "activations/layer15_attention_weight_max": 36.185150146484375,
      "activations/layer15_attention_weight_min": -31.2198543548584,
      "activations/layer16_attention_weight_max": 31.368200302124023,
      "activations/layer16_attention_weight_min": -30.673477172851562,
      "activations/layer17_attention_weight_max": 51.60590362548828,
      "activations/layer17_attention_weight_min": -45.29628372192383,
      "activations/layer18_attention_weight_max": 45.606712341308594,
      "activations/layer18_attention_weight_min": -39.80421829223633,
      "activations/layer19_attention_weight_max": 20.828275680541992,
      "activations/layer19_attention_weight_min": -22.104658126831055,
      "activations/layer1_attention_weight_max": 17.208194732666016,
      "activations/layer1_attention_weight_min": -13.960578918457031,
      "activations/layer20_attention_weight_max": 23.24657440185547,
      "activations/layer20_attention_weight_min": -24.86642074584961,
      "activations/layer21_attention_weight_max": 39.82435989379883,
      "activations/layer21_attention_weight_min": -24.903274536132812,
      "activations/layer22_attention_weight_max": 28.326801300048828,
      "activations/layer22_attention_weight_min": -25.30866050720215,
      "activations/layer23_attention_weight_max": 35.57947540283203,
      "activations/layer23_attention_weight_min": -23.460952758789062,
      "activations/layer2_attention_weight_max": 31.268850326538086,
      "activations/layer2_attention_weight_min": -27.81940460205078,
      "activations/layer3_attention_weight_max": 89.15409088134766,
      "activations/layer3_attention_weight_min": -89.31491088867188,
      "activations/layer4_attention_weight_max": 86.08522033691406,
      "activations/layer4_attention_weight_min": -86.51378631591797,
      "activations/layer5_attention_weight_max": 68.51204681396484,
      "activations/layer5_attention_weight_min": -80.71095275878906,
      "activations/layer6_attention_weight_max": 50.31376647949219,
      "activations/layer6_attention_weight_min": -49.864376068115234,
      "activations/layer7_attention_weight_max": 66.02816009521484,
      "activations/layer7_attention_weight_min": -61.404415130615234,
      "activations/layer8_attention_weight_max": 45.03478240966797,
      "activations/layer8_attention_weight_min": -49.12477493286133,
      "activations/layer9_attention_weight_max": 53.60433578491211,
      "activations/layer9_attention_weight_min": -49.13077926635742,
      "epoch": 14.56,
      "learning_rate": 5.6688257575757574e-05,
      "loss": 2.7665,
      "step": 250550
    },
    {
      "activations/layer0_attention_weight_max": 15.726226806640625,
      "activations/layer0_attention_weight_min": -13.420340538024902,
      "activations/layer10_attention_weight_max": 33.763328552246094,
      "activations/layer10_attention_weight_min": -33.92129898071289,
      "activations/layer11_attention_weight_max": 30.94202423095703,
      "activations/layer11_attention_weight_min": -32.02912902832031,
      "activations/layer12_attention_weight_max": 19.43009376525879,
      "activations/layer12_attention_weight_min": -28.84719467163086,
      "activations/layer13_attention_weight_max": 34.41633605957031,
      "activations/layer13_attention_weight_min": -31.33428955078125,
      "activations/layer14_attention_weight_max": 32.55888366699219,
      "activations/layer14_attention_weight_min": -30.27461814880371,
      "activations/layer15_attention_weight_max": 31.62664222717285,
      "activations/layer15_attention_weight_min": -28.437477111816406,
      "activations/layer16_attention_weight_max": 33.05726623535156,
      "activations/layer16_attention_weight_min": -29.917404174804688,
      "activations/layer17_attention_weight_max": 54.5357666015625,
      "activations/layer17_attention_weight_min": -45.974918365478516,
      "activations/layer18_attention_weight_max": 48.29653549194336,
      "activations/layer18_attention_weight_min": -40.49629592895508,
      "activations/layer19_attention_weight_max": 21.71455955505371,
      "activations/layer19_attention_weight_min": -21.25823974609375,
      "activations/layer1_attention_weight_max": 16.4768123626709,
      "activations/layer1_attention_weight_min": -14.612550735473633,
      "activations/layer20_attention_weight_max": 22.559593200683594,
      "activations/layer20_attention_weight_min": -23.9150390625,
      "activations/layer21_attention_weight_max": 30.342702865600586,
      "activations/layer21_attention_weight_min": -21.85032081604004,
      "activations/layer22_attention_weight_max": 29.61078453063965,
      "activations/layer22_attention_weight_min": -24.745441436767578,
      "activations/layer23_attention_weight_max": 34.626121520996094,
      "activations/layer23_attention_weight_min": -21.673818588256836,
      "activations/layer2_attention_weight_max": 31.617656707763672,
      "activations/layer2_attention_weight_min": -28.984281539916992,
      "activations/layer3_attention_weight_max": 88.853271484375,
      "activations/layer3_attention_weight_min": -88.7308349609375,
      "activations/layer4_attention_weight_max": 85.47334289550781,
      "activations/layer4_attention_weight_min": -82.00865936279297,
      "activations/layer5_attention_weight_max": 66.57473754882812,
      "activations/layer5_attention_weight_min": -76.68864440917969,
      "activations/layer6_attention_weight_max": 48.51979446411133,
      "activations/layer6_attention_weight_min": -48.91048812866211,
      "activations/layer7_attention_weight_max": 60.459205627441406,
      "activations/layer7_attention_weight_min": -60.47444534301758,
      "activations/layer8_attention_weight_max": 44.44902420043945,
      "activations/layer8_attention_weight_min": -46.35958480834961,
      "activations/layer9_attention_weight_max": 42.147945404052734,
      "activations/layer9_attention_weight_min": -45.52330780029297,
      "epoch": 14.56,
      "learning_rate": 5.6669318181818176e-05,
      "loss": 2.7586,
      "step": 250600
    },
    {
      "activations/layer0_attention_weight_max": 14.902806282043457,
      "activations/layer0_attention_weight_min": -13.20632553100586,
      "activations/layer10_attention_weight_max": 33.404441833496094,
      "activations/layer10_attention_weight_min": -34.981075286865234,
      "activations/layer11_attention_weight_max": 31.64058494567871,
      "activations/layer11_attention_weight_min": -33.78932189941406,
      "activations/layer12_attention_weight_max": 17.96880531311035,
      "activations/layer12_attention_weight_min": -26.79597282409668,
      "activations/layer13_attention_weight_max": 38.49279022216797,
      "activations/layer13_attention_weight_min": -30.66757583618164,
      "activations/layer14_attention_weight_max": 34.41756820678711,
      "activations/layer14_attention_weight_min": -31.51218605041504,
      "activations/layer15_attention_weight_max": 32.492218017578125,
      "activations/layer15_attention_weight_min": -31.400163650512695,
      "activations/layer16_attention_weight_max": 34.25825881958008,
      "activations/layer16_attention_weight_min": -33.41017532348633,
      "activations/layer17_attention_weight_max": 54.35553741455078,
      "activations/layer17_attention_weight_min": -49.17158889770508,
      "activations/layer18_attention_weight_max": 49.0400505065918,
      "activations/layer18_attention_weight_min": -40.82180404663086,
      "activations/layer19_attention_weight_max": 25.88922691345215,
      "activations/layer19_attention_weight_min": -23.887731552124023,
      "activations/layer1_attention_weight_max": 16.190793991088867,
      "activations/layer1_attention_weight_min": -16.186424255371094,
      "activations/layer20_attention_weight_max": 23.11800193786621,
      "activations/layer20_attention_weight_min": -21.90220832824707,
      "activations/layer21_attention_weight_max": 36.757232666015625,
      "activations/layer21_attention_weight_min": -22.851057052612305,
      "activations/layer22_attention_weight_max": 29.61457633972168,
      "activations/layer22_attention_weight_min": -25.882890701293945,
      "activations/layer23_attention_weight_max": 36.955230712890625,
      "activations/layer23_attention_weight_min": -22.908451080322266,
      "activations/layer2_attention_weight_max": 34.18680953979492,
      "activations/layer2_attention_weight_min": -31.922834396362305,
      "activations/layer3_attention_weight_max": 93.0799560546875,
      "activations/layer3_attention_weight_min": -88.84008026123047,
      "activations/layer4_attention_weight_max": 87.68840789794922,
      "activations/layer4_attention_weight_min": -85.10054016113281,
      "activations/layer5_attention_weight_max": 66.43243408203125,
      "activations/layer5_attention_weight_min": -74.95675659179688,
      "activations/layer6_attention_weight_max": 49.4843864440918,
      "activations/layer6_attention_weight_min": -54.84169387817383,
      "activations/layer7_attention_weight_max": 60.98814010620117,
      "activations/layer7_attention_weight_min": -66.6808090209961,
      "activations/layer8_attention_weight_max": 44.99399948120117,
      "activations/layer8_attention_weight_min": -47.63325500488281,
      "activations/layer9_attention_weight_max": 43.290740966796875,
      "activations/layer9_attention_weight_min": -46.22788619995117,
      "epoch": 14.56,
      "learning_rate": 5.6650378787878784e-05,
      "loss": 2.7687,
      "step": 250650
    },
    {
      "activations/layer0_attention_weight_max": 15.283021926879883,
      "activations/layer0_attention_weight_min": -13.147612571716309,
      "activations/layer10_attention_weight_max": 36.054054260253906,
      "activations/layer10_attention_weight_min": -36.36688232421875,
      "activations/layer11_attention_weight_max": 32.24951171875,
      "activations/layer11_attention_weight_min": -32.839942932128906,
      "activations/layer12_attention_weight_max": 21.09035301208496,
      "activations/layer12_attention_weight_min": -25.369667053222656,
      "activations/layer13_attention_weight_max": 42.79578399658203,
      "activations/layer13_attention_weight_min": -32.55889129638672,
      "activations/layer14_attention_weight_max": 38.11170196533203,
      "activations/layer14_attention_weight_min": -31.134048461914062,
      "activations/layer15_attention_weight_max": 35.64397430419922,
      "activations/layer15_attention_weight_min": -30.419452667236328,
      "activations/layer16_attention_weight_max": 36.63929748535156,
      "activations/layer16_attention_weight_min": -32.82780838012695,
      "activations/layer17_attention_weight_max": 54.53376007080078,
      "activations/layer17_attention_weight_min": -48.57676696777344,
      "activations/layer18_attention_weight_max": 51.360008239746094,
      "activations/layer18_attention_weight_min": -39.99891662597656,
      "activations/layer19_attention_weight_max": 24.23891258239746,
      "activations/layer19_attention_weight_min": -24.117116928100586,
      "activations/layer1_attention_weight_max": 16.026025772094727,
      "activations/layer1_attention_weight_min": -15.976768493652344,
      "activations/layer20_attention_weight_max": 22.759050369262695,
      "activations/layer20_attention_weight_min": -22.963401794433594,
      "activations/layer21_attention_weight_max": 38.102413177490234,
      "activations/layer21_attention_weight_min": -23.4676570892334,
      "activations/layer22_attention_weight_max": 33.50160598754883,
      "activations/layer22_attention_weight_min": -27.82938575744629,
      "activations/layer23_attention_weight_max": 41.91656494140625,
      "activations/layer23_attention_weight_min": -24.66695785522461,
      "activations/layer2_attention_weight_max": 30.315540313720703,
      "activations/layer2_attention_weight_min": -29.57246208190918,
      "activations/layer3_attention_weight_max": 94.06460571289062,
      "activations/layer3_attention_weight_min": -93.67841339111328,
      "activations/layer4_attention_weight_max": 89.1489486694336,
      "activations/layer4_attention_weight_min": -89.12310028076172,
      "activations/layer5_attention_weight_max": 67.13795471191406,
      "activations/layer5_attention_weight_min": -78.3327407836914,
      "activations/layer6_attention_weight_max": 51.82998275756836,
      "activations/layer6_attention_weight_min": -51.30186080932617,
      "activations/layer7_attention_weight_max": 62.96335220336914,
      "activations/layer7_attention_weight_min": -62.004547119140625,
      "activations/layer8_attention_weight_max": 44.46957015991211,
      "activations/layer8_attention_weight_min": -47.2658576965332,
      "activations/layer9_attention_weight_max": 44.69898223876953,
      "activations/layer9_attention_weight_min": -46.9941520690918,
      "epoch": 14.57,
      "learning_rate": 5.663143939393939e-05,
      "loss": 2.7561,
      "step": 250700
    },
    {
      "activations/layer0_attention_weight_max": 15.216353416442871,
      "activations/layer0_attention_weight_min": -13.215615272521973,
      "activations/layer10_attention_weight_max": 37.25157928466797,
      "activations/layer10_attention_weight_min": -36.16789245605469,
      "activations/layer11_attention_weight_max": 32.93461608886719,
      "activations/layer11_attention_weight_min": -32.25848388671875,
      "activations/layer12_attention_weight_max": 19.017534255981445,
      "activations/layer12_attention_weight_min": -24.695247650146484,
      "activations/layer13_attention_weight_max": 42.73326110839844,
      "activations/layer13_attention_weight_min": -27.581371307373047,
      "activations/layer14_attention_weight_max": 36.1620979309082,
      "activations/layer14_attention_weight_min": -28.515565872192383,
      "activations/layer15_attention_weight_max": 33.48002624511719,
      "activations/layer15_attention_weight_min": -26.959671020507812,
      "activations/layer16_attention_weight_max": 33.44565963745117,
      "activations/layer16_attention_weight_min": -27.435897827148438,
      "activations/layer17_attention_weight_max": 47.46943664550781,
      "activations/layer17_attention_weight_min": -40.31059646606445,
      "activations/layer18_attention_weight_max": 42.88813781738281,
      "activations/layer18_attention_weight_min": -34.6467399597168,
      "activations/layer19_attention_weight_max": 22.974525451660156,
      "activations/layer19_attention_weight_min": -20.12506675720215,
      "activations/layer1_attention_weight_max": 16.58538246154785,
      "activations/layer1_attention_weight_min": -13.452017784118652,
      "activations/layer20_attention_weight_max": 20.769472122192383,
      "activations/layer20_attention_weight_min": -19.311643600463867,
      "activations/layer21_attention_weight_max": 29.685501098632812,
      "activations/layer21_attention_weight_min": -22.019702911376953,
      "activations/layer22_attention_weight_max": 25.730756759643555,
      "activations/layer22_attention_weight_min": -23.95738983154297,
      "activations/layer23_attention_weight_max": 38.12487030029297,
      "activations/layer23_attention_weight_min": -23.72614097595215,
      "activations/layer2_attention_weight_max": 31.498146057128906,
      "activations/layer2_attention_weight_min": -30.559783935546875,
      "activations/layer3_attention_weight_max": 92.19102478027344,
      "activations/layer3_attention_weight_min": -92.26686096191406,
      "activations/layer4_attention_weight_max": 87.71062469482422,
      "activations/layer4_attention_weight_min": -84.46123504638672,
      "activations/layer5_attention_weight_max": 68.9507064819336,
      "activations/layer5_attention_weight_min": -74.7562255859375,
      "activations/layer6_attention_weight_max": 51.719730377197266,
      "activations/layer6_attention_weight_min": -50.73816680908203,
      "activations/layer7_attention_weight_max": 62.26517868041992,
      "activations/layer7_attention_weight_min": -60.53892135620117,
      "activations/layer8_attention_weight_max": 45.5526237487793,
      "activations/layer8_attention_weight_min": -47.183597564697266,
      "activations/layer9_attention_weight_max": 45.6922492980957,
      "activations/layer9_attention_weight_min": -47.08210372924805,
      "epoch": 14.57,
      "learning_rate": 5.6612499999999994e-05,
      "loss": 2.7543,
      "step": 250750
    },
    {
      "activations/layer0_attention_weight_max": 15.143723487854004,
      "activations/layer0_attention_weight_min": -13.533138275146484,
      "activations/layer10_attention_weight_max": 37.3725471496582,
      "activations/layer10_attention_weight_min": -34.81760025024414,
      "activations/layer11_attention_weight_max": 35.09852600097656,
      "activations/layer11_attention_weight_min": -34.050689697265625,
      "activations/layer12_attention_weight_max": 18.405427932739258,
      "activations/layer12_attention_weight_min": -26.986665725708008,
      "activations/layer13_attention_weight_max": 34.98560333251953,
      "activations/layer13_attention_weight_min": -27.27147102355957,
      "activations/layer14_attention_weight_max": 35.922630310058594,
      "activations/layer14_attention_weight_min": -29.488143920898438,
      "activations/layer15_attention_weight_max": 31.86246109008789,
      "activations/layer15_attention_weight_min": -30.43236541748047,
      "activations/layer16_attention_weight_max": 32.5853385925293,
      "activations/layer16_attention_weight_min": -32.373199462890625,
      "activations/layer17_attention_weight_max": 54.170440673828125,
      "activations/layer17_attention_weight_min": -45.54762649536133,
      "activations/layer18_attention_weight_max": 46.8905029296875,
      "activations/layer18_attention_weight_min": -39.870853424072266,
      "activations/layer19_attention_weight_max": 22.87080192565918,
      "activations/layer19_attention_weight_min": -23.52931022644043,
      "activations/layer1_attention_weight_max": 16.44637680053711,
      "activations/layer1_attention_weight_min": -14.788076400756836,
      "activations/layer20_attention_weight_max": 22.762826919555664,
      "activations/layer20_attention_weight_min": -23.169849395751953,
      "activations/layer21_attention_weight_max": 38.276432037353516,
      "activations/layer21_attention_weight_min": -22.58768653869629,
      "activations/layer22_attention_weight_max": 32.27892303466797,
      "activations/layer22_attention_weight_min": -28.25501251220703,
      "activations/layer23_attention_weight_max": 36.38234329223633,
      "activations/layer23_attention_weight_min": -24.566152572631836,
      "activations/layer2_attention_weight_max": 33.364952087402344,
      "activations/layer2_attention_weight_min": -29.785892486572266,
      "activations/layer3_attention_weight_max": 94.97889709472656,
      "activations/layer3_attention_weight_min": -92.21012878417969,
      "activations/layer4_attention_weight_max": 91.95590209960938,
      "activations/layer4_attention_weight_min": -87.98916625976562,
      "activations/layer5_attention_weight_max": 67.06067657470703,
      "activations/layer5_attention_weight_min": -74.94361877441406,
      "activations/layer6_attention_weight_max": 51.461910247802734,
      "activations/layer6_attention_weight_min": -49.627017974853516,
      "activations/layer7_attention_weight_max": 64.7048110961914,
      "activations/layer7_attention_weight_min": -63.267642974853516,
      "activations/layer8_attention_weight_max": 44.11836624145508,
      "activations/layer8_attention_weight_min": -46.18757629394531,
      "activations/layer9_attention_weight_max": 44.446128845214844,
      "activations/layer9_attention_weight_min": -44.86853790283203,
      "epoch": 14.57,
      "learning_rate": 5.65935606060606e-05,
      "loss": 2.754,
      "step": 250800
    },
    {
      "activations/layer0_attention_weight_max": 14.516837120056152,
      "activations/layer0_attention_weight_min": -12.938008308410645,
      "activations/layer10_attention_weight_max": 35.52177047729492,
      "activations/layer10_attention_weight_min": -34.815189361572266,
      "activations/layer11_attention_weight_max": 32.291053771972656,
      "activations/layer11_attention_weight_min": -33.78173065185547,
      "activations/layer12_attention_weight_max": 19.265554428100586,
      "activations/layer12_attention_weight_min": -28.471851348876953,
      "activations/layer13_attention_weight_max": 30.780277252197266,
      "activations/layer13_attention_weight_min": -29.762929916381836,
      "activations/layer14_attention_weight_max": 33.50936508178711,
      "activations/layer14_attention_weight_min": -30.90635871887207,
      "activations/layer15_attention_weight_max": 30.34296226501465,
      "activations/layer15_attention_weight_min": -33.031375885009766,
      "activations/layer16_attention_weight_max": 31.46403694152832,
      "activations/layer16_attention_weight_min": -29.972326278686523,
      "activations/layer17_attention_weight_max": 50.19389343261719,
      "activations/layer17_attention_weight_min": -46.75979232788086,
      "activations/layer18_attention_weight_max": 47.81726837158203,
      "activations/layer18_attention_weight_min": -40.26964569091797,
      "activations/layer19_attention_weight_max": 24.92221450805664,
      "activations/layer19_attention_weight_min": -25.26580810546875,
      "activations/layer1_attention_weight_max": 16.538469314575195,
      "activations/layer1_attention_weight_min": -15.279454231262207,
      "activations/layer20_attention_weight_max": 21.37890625,
      "activations/layer20_attention_weight_min": -22.602556228637695,
      "activations/layer21_attention_weight_max": 34.18722152709961,
      "activations/layer21_attention_weight_min": -23.010478973388672,
      "activations/layer22_attention_weight_max": 28.135772705078125,
      "activations/layer22_attention_weight_min": -29.183408737182617,
      "activations/layer23_attention_weight_max": 36.905052185058594,
      "activations/layer23_attention_weight_min": -24.349889755249023,
      "activations/layer2_attention_weight_max": 33.52195358276367,
      "activations/layer2_attention_weight_min": -32.73895263671875,
      "activations/layer3_attention_weight_max": 89.03668975830078,
      "activations/layer3_attention_weight_min": -93.95641326904297,
      "activations/layer4_attention_weight_max": 88.92977142333984,
      "activations/layer4_attention_weight_min": -85.4038314819336,
      "activations/layer5_attention_weight_max": 69.686279296875,
      "activations/layer5_attention_weight_min": -74.95718383789062,
      "activations/layer6_attention_weight_max": 52.15487289428711,
      "activations/layer6_attention_weight_min": -50.37760925292969,
      "activations/layer7_attention_weight_max": 65.41021728515625,
      "activations/layer7_attention_weight_min": -61.68092346191406,
      "activations/layer8_attention_weight_max": 44.131351470947266,
      "activations/layer8_attention_weight_min": -47.09010696411133,
      "activations/layer9_attention_weight_max": 44.51364517211914,
      "activations/layer9_attention_weight_min": -44.875526428222656,
      "epoch": 14.58,
      "learning_rate": 5.657499999999999e-05,
      "loss": 2.7818,
      "step": 250850
    },
    {
      "activations/layer0_attention_weight_max": 16.048419952392578,
      "activations/layer0_attention_weight_min": -13.761630058288574,
      "activations/layer10_attention_weight_max": 34.77238464355469,
      "activations/layer10_attention_weight_min": -34.66204833984375,
      "activations/layer11_attention_weight_max": 32.53358459472656,
      "activations/layer11_attention_weight_min": -33.88903045654297,
      "activations/layer12_attention_weight_max": 17.908010482788086,
      "activations/layer12_attention_weight_min": -25.873456954956055,
      "activations/layer13_attention_weight_max": 33.29803466796875,
      "activations/layer13_attention_weight_min": -28.906505584716797,
      "activations/layer14_attention_weight_max": 36.87705993652344,
      "activations/layer14_attention_weight_min": -31.108945846557617,
      "activations/layer15_attention_weight_max": 31.01569938659668,
      "activations/layer15_attention_weight_min": -31.12677764892578,
      "activations/layer16_attention_weight_max": 33.708072662353516,
      "activations/layer16_attention_weight_min": -32.86113357543945,
      "activations/layer17_attention_weight_max": 49.03239822387695,
      "activations/layer17_attention_weight_min": -45.10236740112305,
      "activations/layer18_attention_weight_max": 44.64895248413086,
      "activations/layer18_attention_weight_min": -38.72191619873047,
      "activations/layer19_attention_weight_max": 22.579605102539062,
      "activations/layer19_attention_weight_min": -22.88855743408203,
      "activations/layer1_attention_weight_max": 17.30027961730957,
      "activations/layer1_attention_weight_min": -13.569896697998047,
      "activations/layer20_attention_weight_max": 24.235876083374023,
      "activations/layer20_attention_weight_min": -22.48767852783203,
      "activations/layer21_attention_weight_max": 40.91762161254883,
      "activations/layer21_attention_weight_min": -22.086715698242188,
      "activations/layer22_attention_weight_max": 30.317626953125,
      "activations/layer22_attention_weight_min": -26.513547897338867,
      "activations/layer23_attention_weight_max": 37.48017883300781,
      "activations/layer23_attention_weight_min": -22.931745529174805,
      "activations/layer2_attention_weight_max": 29.767589569091797,
      "activations/layer2_attention_weight_min": -28.234100341796875,
      "activations/layer3_attention_weight_max": 90.04180145263672,
      "activations/layer3_attention_weight_min": -89.6136245727539,
      "activations/layer4_attention_weight_max": 88.10501098632812,
      "activations/layer4_attention_weight_min": -84.7105941772461,
      "activations/layer5_attention_weight_max": 70.78428649902344,
      "activations/layer5_attention_weight_min": -77.66685485839844,
      "activations/layer6_attention_weight_max": 49.73579025268555,
      "activations/layer6_attention_weight_min": -49.06661605834961,
      "activations/layer7_attention_weight_max": 63.03147506713867,
      "activations/layer7_attention_weight_min": -60.15435028076172,
      "activations/layer8_attention_weight_max": 43.44722366333008,
      "activations/layer8_attention_weight_min": -48.14068603515625,
      "activations/layer9_attention_weight_max": 41.277828216552734,
      "activations/layer9_attention_weight_min": -46.287208557128906,
      "epoch": 14.58,
      "learning_rate": 5.655606060606061e-05,
      "loss": 2.7658,
      "step": 250900
    },
    {
      "activations/layer0_attention_weight_max": 15.12116527557373,
      "activations/layer0_attention_weight_min": -13.2305326461792,
      "activations/layer10_attention_weight_max": 32.490516662597656,
      "activations/layer10_attention_weight_min": -33.64248275756836,
      "activations/layer11_attention_weight_max": 28.466522216796875,
      "activations/layer11_attention_weight_min": -31.270246505737305,
      "activations/layer12_attention_weight_max": 18.171228408813477,
      "activations/layer12_attention_weight_min": -28.074193954467773,
      "activations/layer13_attention_weight_max": 33.86792755126953,
      "activations/layer13_attention_weight_min": -27.22715950012207,
      "activations/layer14_attention_weight_max": 33.70652770996094,
      "activations/layer14_attention_weight_min": -28.61854362487793,
      "activations/layer15_attention_weight_max": 31.180316925048828,
      "activations/layer15_attention_weight_min": -28.54665184020996,
      "activations/layer16_attention_weight_max": 31.58154296875,
      "activations/layer16_attention_weight_min": -31.267412185668945,
      "activations/layer17_attention_weight_max": 54.14654541015625,
      "activations/layer17_attention_weight_min": -43.2624626159668,
      "activations/layer18_attention_weight_max": 42.835594177246094,
      "activations/layer18_attention_weight_min": -39.2259521484375,
      "activations/layer19_attention_weight_max": 20.902446746826172,
      "activations/layer19_attention_weight_min": -23.53090476989746,
      "activations/layer1_attention_weight_max": 17.349834442138672,
      "activations/layer1_attention_weight_min": -15.655582427978516,
      "activations/layer20_attention_weight_max": 20.11895179748535,
      "activations/layer20_attention_weight_min": -22.381818771362305,
      "activations/layer21_attention_weight_max": 33.05561065673828,
      "activations/layer21_attention_weight_min": -24.13865089416504,
      "activations/layer22_attention_weight_max": 28.89537811279297,
      "activations/layer22_attention_weight_min": -25.964767456054688,
      "activations/layer23_attention_weight_max": 36.62421417236328,
      "activations/layer23_attention_weight_min": -23.687034606933594,
      "activations/layer2_attention_weight_max": 31.968006134033203,
      "activations/layer2_attention_weight_min": -28.095144271850586,
      "activations/layer3_attention_weight_max": 91.89004516601562,
      "activations/layer3_attention_weight_min": -84.51608276367188,
      "activations/layer4_attention_weight_max": 82.5087661743164,
      "activations/layer4_attention_weight_min": -81.06562805175781,
      "activations/layer5_attention_weight_max": 63.33000183105469,
      "activations/layer5_attention_weight_min": -71.9979019165039,
      "activations/layer6_attention_weight_max": 45.52053451538086,
      "activations/layer6_attention_weight_min": -46.694862365722656,
      "activations/layer7_attention_weight_max": 60.34367752075195,
      "activations/layer7_attention_weight_min": -58.08251190185547,
      "activations/layer8_attention_weight_max": 38.8917121887207,
      "activations/layer8_attention_weight_min": -44.18453598022461,
      "activations/layer9_attention_weight_max": 40.57699203491211,
      "activations/layer9_attention_weight_min": -43.03404998779297,
      "epoch": 14.58,
      "learning_rate": 5.653712121212121e-05,
      "loss": 2.7748,
      "step": 250950
    },
    {
      "activations/layer0_attention_weight_max": 17.1171875,
      "activations/layer0_attention_weight_min": -13.592050552368164,
      "activations/layer10_attention_weight_max": 34.423370361328125,
      "activations/layer10_attention_weight_min": -35.97706604003906,
      "activations/layer11_attention_weight_max": 30.795705795288086,
      "activations/layer11_attention_weight_min": -32.22236633300781,
      "activations/layer12_attention_weight_max": 18.086286544799805,
      "activations/layer12_attention_weight_min": -25.768415451049805,
      "activations/layer13_attention_weight_max": 34.29938888549805,
      "activations/layer13_attention_weight_min": -29.230905532836914,
      "activations/layer14_attention_weight_max": 33.286312103271484,
      "activations/layer14_attention_weight_min": -28.077484130859375,
      "activations/layer15_attention_weight_max": 29.967226028442383,
      "activations/layer15_attention_weight_min": -27.791013717651367,
      "activations/layer16_attention_weight_max": 30.389204025268555,
      "activations/layer16_attention_weight_min": -29.28556251525879,
      "activations/layer17_attention_weight_max": 47.11186981201172,
      "activations/layer17_attention_weight_min": -42.297279357910156,
      "activations/layer18_attention_weight_max": 43.07661437988281,
      "activations/layer18_attention_weight_min": -37.45455551147461,
      "activations/layer19_attention_weight_max": 22.243085861206055,
      "activations/layer19_attention_weight_min": -20.86465072631836,
      "activations/layer1_attention_weight_max": 16.764694213867188,
      "activations/layer1_attention_weight_min": -16.26764488220215,
      "activations/layer20_attention_weight_max": 21.94023895263672,
      "activations/layer20_attention_weight_min": -23.042068481445312,
      "activations/layer21_attention_weight_max": 36.10125732421875,
      "activations/layer21_attention_weight_min": -24.199172973632812,
      "activations/layer22_attention_weight_max": 28.62228012084961,
      "activations/layer22_attention_weight_min": -24.550880432128906,
      "activations/layer23_attention_weight_max": 35.03770446777344,
      "activations/layer23_attention_weight_min": -25.440412521362305,
      "activations/layer2_attention_weight_max": 32.25709915161133,
      "activations/layer2_attention_weight_min": -30.828596115112305,
      "activations/layer3_attention_weight_max": 91.14385986328125,
      "activations/layer3_attention_weight_min": -93.27516174316406,
      "activations/layer4_attention_weight_max": 89.12332916259766,
      "activations/layer4_attention_weight_min": -84.68885040283203,
      "activations/layer5_attention_weight_max": 68.31265258789062,
      "activations/layer5_attention_weight_min": -79.48841857910156,
      "activations/layer6_attention_weight_max": 50.45248794555664,
      "activations/layer6_attention_weight_min": -51.02631759643555,
      "activations/layer7_attention_weight_max": 62.87572479248047,
      "activations/layer7_attention_weight_min": -63.55345916748047,
      "activations/layer8_attention_weight_max": 42.19821548461914,
      "activations/layer8_attention_weight_min": -45.971351623535156,
      "activations/layer9_attention_weight_max": 47.10165023803711,
      "activations/layer9_attention_weight_min": -47.76986312866211,
      "epoch": 14.58,
      "learning_rate": 5.651818181818181e-05,
      "loss": 2.7552,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4612,
      "eval_samples_per_second": 507.496,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4612,
      "eval_openwebtext_samples_per_second": 507.496,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_wikitext_loss": 2.9375,
      "eval_wikitext_ppl": 18.868615759264884,
      "eval_wikitext_runtime": 1.9121,
      "eval_wikitext_samples_per_second": 238.476,
      "step": 251000
    },
    {
      "epoch": 14.58,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.528,
      "eval_lambada_samples_per_second": 511.019,
      "step": 251000
    },
    {
      "activations/layer0_attention_weight_max": 15.261205673217773,
      "activations/layer0_attention_weight_min": -13.565693855285645,
      "activations/layer10_attention_weight_max": 35.79179763793945,
      "activations/layer10_attention_weight_min": -34.453155517578125,
      "activations/layer11_attention_weight_max": 32.50513458251953,
      "activations/layer11_attention_weight_min": -32.91997528076172,
      "activations/layer12_attention_weight_max": 18.83115577697754,
      "activations/layer12_attention_weight_min": -24.57619857788086,
      "activations/layer13_attention_weight_max": 45.87859344482422,
      "activations/layer13_attention_weight_min": -29.28321075439453,
      "activations/layer14_attention_weight_max": 37.877384185791016,
      "activations/layer14_attention_weight_min": -29.40363121032715,
      "activations/layer15_attention_weight_max": 35.61276626586914,
      "activations/layer15_attention_weight_min": -30.103559494018555,
      "activations/layer16_attention_weight_max": 34.96659851074219,
      "activations/layer16_attention_weight_min": -30.264820098876953,
      "activations/layer17_attention_weight_max": 54.668739318847656,
      "activations/layer17_attention_weight_min": -42.99003601074219,
      "activations/layer18_attention_weight_max": 46.22924041748047,
      "activations/layer18_attention_weight_min": -35.16393280029297,
      "activations/layer19_attention_weight_max": 21.0726375579834,
      "activations/layer19_attention_weight_min": -20.85340690612793,
      "activations/layer1_attention_weight_max": 16.51301383972168,
      "activations/layer1_attention_weight_min": -13.612202644348145,
      "activations/layer20_attention_weight_max": 20.88362693786621,
      "activations/layer20_attention_weight_min": -21.48193359375,
      "activations/layer21_attention_weight_max": 42.85955810546875,
      "activations/layer21_attention_weight_min": -23.5487060546875,
      "activations/layer22_attention_weight_max": 30.41657066345215,
      "activations/layer22_attention_weight_min": -25.9202938079834,
      "activations/layer23_attention_weight_max": 37.72667694091797,
      "activations/layer23_attention_weight_min": -26.81021499633789,
      "activations/layer2_attention_weight_max": 29.80989646911621,
      "activations/layer2_attention_weight_min": -28.165241241455078,
      "activations/layer3_attention_weight_max": 90.83057403564453,
      "activations/layer3_attention_weight_min": -90.57122039794922,
      "activations/layer4_attention_weight_max": 90.52544403076172,
      "activations/layer4_attention_weight_min": -88.25389862060547,
      "activations/layer5_attention_weight_max": 70.67327880859375,
      "activations/layer5_attention_weight_min": -73.61199188232422,
      "activations/layer6_attention_weight_max": 50.16864776611328,
      "activations/layer6_attention_weight_min": -48.10944747924805,
      "activations/layer7_attention_weight_max": 61.886112213134766,
      "activations/layer7_attention_weight_min": -61.482177734375,
      "activations/layer8_attention_weight_max": 43.25740432739258,
      "activations/layer8_attention_weight_min": -48.33235549926758,
      "activations/layer9_attention_weight_max": 42.53862762451172,
      "activations/layer9_attention_weight_min": -42.66825866699219,
      "epoch": 14.59,
      "learning_rate": 5.649924242424242e-05,
      "loss": 2.7563,
      "step": 251050
    },
    {
      "activations/layer0_attention_weight_max": 16.368154525756836,
      "activations/layer0_attention_weight_min": -13.856508255004883,
      "activations/layer10_attention_weight_max": 48.77593994140625,
      "activations/layer10_attention_weight_min": -48.162742614746094,
      "activations/layer11_attention_weight_max": 43.93169403076172,
      "activations/layer11_attention_weight_min": -41.40862274169922,
      "activations/layer12_attention_weight_max": 19.066429138183594,
      "activations/layer12_attention_weight_min": -26.752321243286133,
      "activations/layer13_attention_weight_max": 43.900997161865234,
      "activations/layer13_attention_weight_min": -29.665842056274414,
      "activations/layer14_attention_weight_max": 55.790977478027344,
      "activations/layer14_attention_weight_min": -31.42281723022461,
      "activations/layer15_attention_weight_max": 53.852298736572266,
      "activations/layer15_attention_weight_min": -30.945220947265625,
      "activations/layer16_attention_weight_max": 31.34752082824707,
      "activations/layer16_attention_weight_min": -29.938154220581055,
      "activations/layer17_attention_weight_max": 47.14443588256836,
      "activations/layer17_attention_weight_min": -49.359275817871094,
      "activations/layer18_attention_weight_max": 43.19629669189453,
      "activations/layer18_attention_weight_min": -41.864742279052734,
      "activations/layer19_attention_weight_max": 23.733875274658203,
      "activations/layer19_attention_weight_min": -22.1961727142334,
      "activations/layer1_attention_weight_max": 17.43921661376953,
      "activations/layer1_attention_weight_min": -14.135750770568848,
      "activations/layer20_attention_weight_max": 20.508495330810547,
      "activations/layer20_attention_weight_min": -19.31553840637207,
      "activations/layer21_attention_weight_max": 37.572608947753906,
      "activations/layer21_attention_weight_min": -23.534177780151367,
      "activations/layer22_attention_weight_max": 29.677310943603516,
      "activations/layer22_attention_weight_min": -24.056671142578125,
      "activations/layer23_attention_weight_max": 37.530517578125,
      "activations/layer23_attention_weight_min": -24.810222625732422,
      "activations/layer2_attention_weight_max": 32.75931167602539,
      "activations/layer2_attention_weight_min": -32.47419738769531,
      "activations/layer3_attention_weight_max": 93.66058349609375,
      "activations/layer3_attention_weight_min": -96.58063507080078,
      "activations/layer4_attention_weight_max": 92.34310150146484,
      "activations/layer4_attention_weight_min": -84.46185302734375,
      "activations/layer5_attention_weight_max": 69.72875213623047,
      "activations/layer5_attention_weight_min": -76.97065734863281,
      "activations/layer6_attention_weight_max": 49.09028244018555,
      "activations/layer6_attention_weight_min": -49.1555290222168,
      "activations/layer7_attention_weight_max": 83.09584045410156,
      "activations/layer7_attention_weight_min": -64.39747619628906,
      "activations/layer8_attention_weight_max": 58.56589126586914,
      "activations/layer8_attention_weight_min": -53.087093353271484,
      "activations/layer9_attention_weight_max": 67.7922592163086,
      "activations/layer9_attention_weight_min": -55.03035354614258,
      "epoch": 14.59,
      "learning_rate": 5.648030303030303e-05,
      "loss": 2.7664,
      "step": 251100
    },
    {
      "activations/layer0_attention_weight_max": 15.00544261932373,
      "activations/layer0_attention_weight_min": -13.975525856018066,
      "activations/layer10_attention_weight_max": 45.7285270690918,
      "activations/layer10_attention_weight_min": -42.14408493041992,
      "activations/layer11_attention_weight_max": 41.38654327392578,
      "activations/layer11_attention_weight_min": -39.64665222167969,
      "activations/layer12_attention_weight_max": 19.232210159301758,
      "activations/layer12_attention_weight_min": -31.365379333496094,
      "activations/layer13_attention_weight_max": 51.12318801879883,
      "activations/layer13_attention_weight_min": -28.921777725219727,
      "activations/layer14_attention_weight_max": 49.04582214355469,
      "activations/layer14_attention_weight_min": -31.990388870239258,
      "activations/layer15_attention_weight_max": 46.56581115722656,
      "activations/layer15_attention_weight_min": -31.28923225402832,
      "activations/layer16_attention_weight_max": 37.16368865966797,
      "activations/layer16_attention_weight_min": -32.9512825012207,
      "activations/layer17_attention_weight_max": 57.61489486694336,
      "activations/layer17_attention_weight_min": -50.412071228027344,
      "activations/layer18_attention_weight_max": 55.69036102294922,
      "activations/layer18_attention_weight_min": -46.68801498413086,
      "activations/layer19_attention_weight_max": 26.673765182495117,
      "activations/layer19_attention_weight_min": -26.747739791870117,
      "activations/layer1_attention_weight_max": 15.761940002441406,
      "activations/layer1_attention_weight_min": -14.696248054504395,
      "activations/layer20_attention_weight_max": 25.59944725036621,
      "activations/layer20_attention_weight_min": -23.609594345092773,
      "activations/layer21_attention_weight_max": 48.0620002746582,
      "activations/layer21_attention_weight_min": -23.82205581665039,
      "activations/layer22_attention_weight_max": 34.11050796508789,
      "activations/layer22_attention_weight_min": -25.56881332397461,
      "activations/layer23_attention_weight_max": 40.71006393432617,
      "activations/layer23_attention_weight_min": -24.000730514526367,
      "activations/layer2_attention_weight_max": 31.369338989257812,
      "activations/layer2_attention_weight_min": -29.29036521911621,
      "activations/layer3_attention_weight_max": 90.96794891357422,
      "activations/layer3_attention_weight_min": -93.2225341796875,
      "activations/layer4_attention_weight_max": 88.44451904296875,
      "activations/layer4_attention_weight_min": -87.16522216796875,
      "activations/layer5_attention_weight_max": 66.22773742675781,
      "activations/layer5_attention_weight_min": -76.77701568603516,
      "activations/layer6_attention_weight_max": 51.16037368774414,
      "activations/layer6_attention_weight_min": -49.5539665222168,
      "activations/layer7_attention_weight_max": 74.65391540527344,
      "activations/layer7_attention_weight_min": -67.17847442626953,
      "activations/layer8_attention_weight_max": 55.022369384765625,
      "activations/layer8_attention_weight_min": -53.153072357177734,
      "activations/layer9_attention_weight_max": 63.366390228271484,
      "activations/layer9_attention_weight_min": -49.24208068847656,
      "epoch": 14.59,
      "learning_rate": 5.6461363636363636e-05,
      "loss": 2.7571,
      "step": 251150
    },
    {
      "activations/layer0_attention_weight_max": 15.513577461242676,
      "activations/layer0_attention_weight_min": -13.671030044555664,
      "activations/layer10_attention_weight_max": 39.929046630859375,
      "activations/layer10_attention_weight_min": -39.335575103759766,
      "activations/layer11_attention_weight_max": 33.771339416503906,
      "activations/layer11_attention_weight_min": -34.750823974609375,
      "activations/layer12_attention_weight_max": 18.647117614746094,
      "activations/layer12_attention_weight_min": -26.25040626525879,
      "activations/layer13_attention_weight_max": 34.533172607421875,
      "activations/layer13_attention_weight_min": -29.24029541015625,
      "activations/layer14_attention_weight_max": 36.80443572998047,
      "activations/layer14_attention_weight_min": -30.8970947265625,
      "activations/layer15_attention_weight_max": 32.23396301269531,
      "activations/layer15_attention_weight_min": -30.831851959228516,
      "activations/layer16_attention_weight_max": 34.23747253417969,
      "activations/layer16_attention_weight_min": -30.623104095458984,
      "activations/layer17_attention_weight_max": 53.13867950439453,
      "activations/layer17_attention_weight_min": -46.73533248901367,
      "activations/layer18_attention_weight_max": 48.1766471862793,
      "activations/layer18_attention_weight_min": -38.67113494873047,
      "activations/layer19_attention_weight_max": 22.09958267211914,
      "activations/layer19_attention_weight_min": -23.427248001098633,
      "activations/layer1_attention_weight_max": 16.911996841430664,
      "activations/layer1_attention_weight_min": -15.826047897338867,
      "activations/layer20_attention_weight_max": 20.415794372558594,
      "activations/layer20_attention_weight_min": -22.96194839477539,
      "activations/layer21_attention_weight_max": 33.046775817871094,
      "activations/layer21_attention_weight_min": -24.88348388671875,
      "activations/layer22_attention_weight_max": 28.201576232910156,
      "activations/layer22_attention_weight_min": -26.13434600830078,
      "activations/layer23_attention_weight_max": 38.06550598144531,
      "activations/layer23_attention_weight_min": -24.622211456298828,
      "activations/layer2_attention_weight_max": 33.54523849487305,
      "activations/layer2_attention_weight_min": -30.529403686523438,
      "activations/layer3_attention_weight_max": 91.50946044921875,
      "activations/layer3_attention_weight_min": -91.16808319091797,
      "activations/layer4_attention_weight_max": 88.46044158935547,
      "activations/layer4_attention_weight_min": -87.99874114990234,
      "activations/layer5_attention_weight_max": 71.13014221191406,
      "activations/layer5_attention_weight_min": -71.99549865722656,
      "activations/layer6_attention_weight_max": 52.919498443603516,
      "activations/layer6_attention_weight_min": -51.40852737426758,
      "activations/layer7_attention_weight_max": 63.9163932800293,
      "activations/layer7_attention_weight_min": -64.07958221435547,
      "activations/layer8_attention_weight_max": 47.2322883605957,
      "activations/layer8_attention_weight_min": -49.83429718017578,
      "activations/layer9_attention_weight_max": 47.0764274597168,
      "activations/layer9_attention_weight_min": -47.13683319091797,
      "epoch": 14.6,
      "learning_rate": 5.644242424242424e-05,
      "loss": 2.7626,
      "step": 251200
    },
    {
      "activations/layer0_attention_weight_max": 15.705516815185547,
      "activations/layer0_attention_weight_min": -14.027142524719238,
      "activations/layer10_attention_weight_max": 35.30501174926758,
      "activations/layer10_attention_weight_min": -34.98883056640625,
      "activations/layer11_attention_weight_max": 32.5203742980957,
      "activations/layer11_attention_weight_min": -33.70981979370117,
      "activations/layer12_attention_weight_max": 18.79180145263672,
      "activations/layer12_attention_weight_min": -23.5733699798584,
      "activations/layer13_attention_weight_max": 35.321510314941406,
      "activations/layer13_attention_weight_min": -28.64005470275879,
      "activations/layer14_attention_weight_max": 36.95573043823242,
      "activations/layer14_attention_weight_min": -30.970855712890625,
      "activations/layer15_attention_weight_max": 33.51473617553711,
      "activations/layer15_attention_weight_min": -29.68094825744629,
      "activations/layer16_attention_weight_max": 35.03421401977539,
      "activations/layer16_attention_weight_min": -32.76570129394531,
      "activations/layer17_attention_weight_max": 51.6909294128418,
      "activations/layer17_attention_weight_min": -42.82596206665039,
      "activations/layer18_attention_weight_max": 47.88980484008789,
      "activations/layer18_attention_weight_min": -36.890045166015625,
      "activations/layer19_attention_weight_max": 21.52406120300293,
      "activations/layer19_attention_weight_min": -22.7789306640625,
      "activations/layer1_attention_weight_max": 16.838964462280273,
      "activations/layer1_attention_weight_min": -14.698891639709473,
      "activations/layer20_attention_weight_max": 21.540006637573242,
      "activations/layer20_attention_weight_min": -21.17633056640625,
      "activations/layer21_attention_weight_max": 31.98333740234375,
      "activations/layer21_attention_weight_min": -22.137956619262695,
      "activations/layer22_attention_weight_max": 28.65447235107422,
      "activations/layer22_attention_weight_min": -25.416080474853516,
      "activations/layer23_attention_weight_max": 33.93821716308594,
      "activations/layer23_attention_weight_min": -25.00389862060547,
      "activations/layer2_attention_weight_max": 31.600797653198242,
      "activations/layer2_attention_weight_min": -29.006725311279297,
      "activations/layer3_attention_weight_max": 90.91334533691406,
      "activations/layer3_attention_weight_min": -93.92677307128906,
      "activations/layer4_attention_weight_max": 88.86205291748047,
      "activations/layer4_attention_weight_min": -85.1240463256836,
      "activations/layer5_attention_weight_max": 66.46539306640625,
      "activations/layer5_attention_weight_min": -77.47701263427734,
      "activations/layer6_attention_weight_max": 48.22478485107422,
      "activations/layer6_attention_weight_min": -49.601322174072266,
      "activations/layer7_attention_weight_max": 63.25826644897461,
      "activations/layer7_attention_weight_min": -63.366050720214844,
      "activations/layer8_attention_weight_max": 45.1517448425293,
      "activations/layer8_attention_weight_min": -51.278560638427734,
      "activations/layer9_attention_weight_max": 41.49396514892578,
      "activations/layer9_attention_weight_min": -50.10320281982422,
      "epoch": 14.6,
      "learning_rate": 5.642348484848484e-05,
      "loss": 2.7724,
      "step": 251250
    },
    {
      "activations/layer0_attention_weight_max": 15.336483001708984,
      "activations/layer0_attention_weight_min": -13.525135040283203,
      "activations/layer10_attention_weight_max": 39.326873779296875,
      "activations/layer10_attention_weight_min": -35.62882995605469,
      "activations/layer11_attention_weight_max": 34.24370574951172,
      "activations/layer11_attention_weight_min": -35.29570770263672,
      "activations/layer12_attention_weight_max": 18.51150894165039,
      "activations/layer12_attention_weight_min": -27.48084831237793,
      "activations/layer13_attention_weight_max": 36.432247161865234,
      "activations/layer13_attention_weight_min": -28.276016235351562,
      "activations/layer14_attention_weight_max": 36.43560028076172,
      "activations/layer14_attention_weight_min": -30.002397537231445,
      "activations/layer15_attention_weight_max": 33.37957763671875,
      "activations/layer15_attention_weight_min": -30.019229888916016,
      "activations/layer16_attention_weight_max": 33.957157135009766,
      "activations/layer16_attention_weight_min": -31.443265914916992,
      "activations/layer17_attention_weight_max": 51.82905578613281,
      "activations/layer17_attention_weight_min": -46.385986328125,
      "activations/layer18_attention_weight_max": 46.1533317565918,
      "activations/layer18_attention_weight_min": -36.13308334350586,
      "activations/layer19_attention_weight_max": 22.472944259643555,
      "activations/layer19_attention_weight_min": -20.53467559814453,
      "activations/layer1_attention_weight_max": 15.929996490478516,
      "activations/layer1_attention_weight_min": -14.328654289245605,
      "activations/layer20_attention_weight_max": 21.79819679260254,
      "activations/layer20_attention_weight_min": -22.6671085357666,
      "activations/layer21_attention_weight_max": 36.51762008666992,
      "activations/layer21_attention_weight_min": -22.98673439025879,
      "activations/layer22_attention_weight_max": 31.725509643554688,
      "activations/layer22_attention_weight_min": -27.584877014160156,
      "activations/layer23_attention_weight_max": 36.5851936340332,
      "activations/layer23_attention_weight_min": -25.69762420654297,
      "activations/layer2_attention_weight_max": 31.59828758239746,
      "activations/layer2_attention_weight_min": -29.817798614501953,
      "activations/layer3_attention_weight_max": 96.18476104736328,
      "activations/layer3_attention_weight_min": -97.58306884765625,
      "activations/layer4_attention_weight_max": 92.16829681396484,
      "activations/layer4_attention_weight_min": -87.84378814697266,
      "activations/layer5_attention_weight_max": 67.27948760986328,
      "activations/layer5_attention_weight_min": -78.81832885742188,
      "activations/layer6_attention_weight_max": 51.63188552856445,
      "activations/layer6_attention_weight_min": -51.950599670410156,
      "activations/layer7_attention_weight_max": 65.04841613769531,
      "activations/layer7_attention_weight_min": -66.21549224853516,
      "activations/layer8_attention_weight_max": 48.1875114440918,
      "activations/layer8_attention_weight_min": -49.10658645629883,
      "activations/layer9_attention_weight_max": 48.3502082824707,
      "activations/layer9_attention_weight_min": -46.70166015625,
      "epoch": 14.6,
      "learning_rate": 5.6404545454545454e-05,
      "loss": 2.769,
      "step": 251300
    },
    {
      "activations/layer0_attention_weight_max": 16.924379348754883,
      "activations/layer0_attention_weight_min": -13.776727676391602,
      "activations/layer10_attention_weight_max": 35.28554153442383,
      "activations/layer10_attention_weight_min": -36.33974075317383,
      "activations/layer11_attention_weight_max": 31.482501983642578,
      "activations/layer11_attention_weight_min": -35.19172668457031,
      "activations/layer12_attention_weight_max": 19.03188133239746,
      "activations/layer12_attention_weight_min": -28.846458435058594,
      "activations/layer13_attention_weight_max": 35.62954330444336,
      "activations/layer13_attention_weight_min": -30.588382720947266,
      "activations/layer14_attention_weight_max": 37.40045928955078,
      "activations/layer14_attention_weight_min": -32.14689254760742,
      "activations/layer15_attention_weight_max": 33.38870620727539,
      "activations/layer15_attention_weight_min": -32.12683868408203,
      "activations/layer16_attention_weight_max": 34.87456512451172,
      "activations/layer16_attention_weight_min": -32.04698181152344,
      "activations/layer17_attention_weight_max": 55.540164947509766,
      "activations/layer17_attention_weight_min": -46.5110969543457,
      "activations/layer18_attention_weight_max": 48.38358688354492,
      "activations/layer18_attention_weight_min": -42.096370697021484,
      "activations/layer19_attention_weight_max": 22.728544235229492,
      "activations/layer19_attention_weight_min": -23.151905059814453,
      "activations/layer1_attention_weight_max": 16.000389099121094,
      "activations/layer1_attention_weight_min": -17.305294036865234,
      "activations/layer20_attention_weight_max": 24.713382720947266,
      "activations/layer20_attention_weight_min": -20.29246711730957,
      "activations/layer21_attention_weight_max": 41.903221130371094,
      "activations/layer21_attention_weight_min": -21.993118286132812,
      "activations/layer22_attention_weight_max": 31.455781936645508,
      "activations/layer22_attention_weight_min": -27.96414566040039,
      "activations/layer23_attention_weight_max": 37.126976013183594,
      "activations/layer23_attention_weight_min": -23.112682342529297,
      "activations/layer2_attention_weight_max": 30.493009567260742,
      "activations/layer2_attention_weight_min": -29.333242416381836,
      "activations/layer3_attention_weight_max": 91.81334686279297,
      "activations/layer3_attention_weight_min": -94.28295135498047,
      "activations/layer4_attention_weight_max": 89.72784423828125,
      "activations/layer4_attention_weight_min": -91.51946258544922,
      "activations/layer5_attention_weight_max": 67.46420288085938,
      "activations/layer5_attention_weight_min": -72.59666442871094,
      "activations/layer6_attention_weight_max": 49.57936477661133,
      "activations/layer6_attention_weight_min": -51.29560470581055,
      "activations/layer7_attention_weight_max": 63.89510726928711,
      "activations/layer7_attention_weight_min": -63.94466018676758,
      "activations/layer8_attention_weight_max": 44.292991638183594,
      "activations/layer8_attention_weight_min": -46.61918640136719,
      "activations/layer9_attention_weight_max": 44.98163604736328,
      "activations/layer9_attention_weight_min": -45.87152862548828,
      "epoch": 14.6,
      "learning_rate": 5.6385606060606055e-05,
      "loss": 2.7398,
      "step": 251350
    },
    {
      "activations/layer0_attention_weight_max": 15.653959274291992,
      "activations/layer0_attention_weight_min": -13.463371276855469,
      "activations/layer10_attention_weight_max": 36.051918029785156,
      "activations/layer10_attention_weight_min": -34.8741340637207,
      "activations/layer11_attention_weight_max": 31.646963119506836,
      "activations/layer11_attention_weight_min": -34.20381164550781,
      "activations/layer12_attention_weight_max": 17.730594635009766,
      "activations/layer12_attention_weight_min": -26.082246780395508,
      "activations/layer13_attention_weight_max": 34.79527282714844,
      "activations/layer13_attention_weight_min": -29.190141677856445,
      "activations/layer14_attention_weight_max": 35.28288650512695,
      "activations/layer14_attention_weight_min": -29.984121322631836,
      "activations/layer15_attention_weight_max": 31.980356216430664,
      "activations/layer15_attention_weight_min": -29.918546676635742,
      "activations/layer16_attention_weight_max": 31.88452911376953,
      "activations/layer16_attention_weight_min": -30.252944946289062,
      "activations/layer17_attention_weight_max": 52.17005157470703,
      "activations/layer17_attention_weight_min": -46.702938079833984,
      "activations/layer18_attention_weight_max": 46.4635124206543,
      "activations/layer18_attention_weight_min": -39.7522087097168,
      "activations/layer19_attention_weight_max": 23.553789138793945,
      "activations/layer19_attention_weight_min": -22.010297775268555,
      "activations/layer1_attention_weight_max": 16.779869079589844,
      "activations/layer1_attention_weight_min": -13.284403800964355,
      "activations/layer20_attention_weight_max": 26.032976150512695,
      "activations/layer20_attention_weight_min": -21.224042892456055,
      "activations/layer21_attention_weight_max": 34.695655822753906,
      "activations/layer21_attention_weight_min": -20.995859146118164,
      "activations/layer22_attention_weight_max": 28.55138397216797,
      "activations/layer22_attention_weight_min": -24.87495231628418,
      "activations/layer23_attention_weight_max": 38.37852096557617,
      "activations/layer23_attention_weight_min": -22.71023941040039,
      "activations/layer2_attention_weight_max": 30.91543197631836,
      "activations/layer2_attention_weight_min": -29.422550201416016,
      "activations/layer3_attention_weight_max": 89.2837905883789,
      "activations/layer3_attention_weight_min": -93.5740966796875,
      "activations/layer4_attention_weight_max": 93.07030487060547,
      "activations/layer4_attention_weight_min": -90.61392974853516,
      "activations/layer5_attention_weight_max": 71.85639953613281,
      "activations/layer5_attention_weight_min": -81.70295715332031,
      "activations/layer6_attention_weight_max": 51.473487854003906,
      "activations/layer6_attention_weight_min": -50.838233947753906,
      "activations/layer7_attention_weight_max": 67.15750122070312,
      "activations/layer7_attention_weight_min": -66.03301239013672,
      "activations/layer8_attention_weight_max": 44.12838363647461,
      "activations/layer8_attention_weight_min": -46.598121643066406,
      "activations/layer9_attention_weight_max": 45.856624603271484,
      "activations/layer9_attention_weight_min": -49.41008758544922,
      "epoch": 14.61,
      "learning_rate": 5.636666666666666e-05,
      "loss": 2.7513,
      "step": 251400
    },
    {
      "activations/layer0_attention_weight_max": 16.9270076751709,
      "activations/layer0_attention_weight_min": -13.390134811401367,
      "activations/layer10_attention_weight_max": 35.65062713623047,
      "activations/layer10_attention_weight_min": -34.91497039794922,
      "activations/layer11_attention_weight_max": 31.496219635009766,
      "activations/layer11_attention_weight_min": -30.88648796081543,
      "activations/layer12_attention_weight_max": 19.10273551940918,
      "activations/layer12_attention_weight_min": -25.53239631652832,
      "activations/layer13_attention_weight_max": 34.29589080810547,
      "activations/layer13_attention_weight_min": -26.92500877380371,
      "activations/layer14_attention_weight_max": 36.45609664916992,
      "activations/layer14_attention_weight_min": -28.7188720703125,
      "activations/layer15_attention_weight_max": 32.985260009765625,
      "activations/layer15_attention_weight_min": -27.52046775817871,
      "activations/layer16_attention_weight_max": 31.96656036376953,
      "activations/layer16_attention_weight_min": -29.84973907470703,
      "activations/layer17_attention_weight_max": 53.82908630371094,
      "activations/layer17_attention_weight_min": -42.2187385559082,
      "activations/layer18_attention_weight_max": 48.12092971801758,
      "activations/layer18_attention_weight_min": -35.56965255737305,
      "activations/layer19_attention_weight_max": 22.986873626708984,
      "activations/layer19_attention_weight_min": -20.36578941345215,
      "activations/layer1_attention_weight_max": 16.5324764251709,
      "activations/layer1_attention_weight_min": -15.22864055633545,
      "activations/layer20_attention_weight_max": 27.719764709472656,
      "activations/layer20_attention_weight_min": -20.11721420288086,
      "activations/layer21_attention_weight_max": 41.323917388916016,
      "activations/layer21_attention_weight_min": -21.313364028930664,
      "activations/layer22_attention_weight_max": 31.52899169921875,
      "activations/layer22_attention_weight_min": -25.715730667114258,
      "activations/layer23_attention_weight_max": 40.92606735229492,
      "activations/layer23_attention_weight_min": -22.790748596191406,
      "activations/layer2_attention_weight_max": 32.694087982177734,
      "activations/layer2_attention_weight_min": -31.31667709350586,
      "activations/layer3_attention_weight_max": 96.94235229492188,
      "activations/layer3_attention_weight_min": -94.31266784667969,
      "activations/layer4_attention_weight_max": 90.80278778076172,
      "activations/layer4_attention_weight_min": -94.38822174072266,
      "activations/layer5_attention_weight_max": 69.74967956542969,
      "activations/layer5_attention_weight_min": -79.08232116699219,
      "activations/layer6_attention_weight_max": 52.43095779418945,
      "activations/layer6_attention_weight_min": -53.767738342285156,
      "activations/layer7_attention_weight_max": 65.76174926757812,
      "activations/layer7_attention_weight_min": -65.50221252441406,
      "activations/layer8_attention_weight_max": 44.425941467285156,
      "activations/layer8_attention_weight_min": -48.15872573852539,
      "activations/layer9_attention_weight_max": 44.28128433227539,
      "activations/layer9_attention_weight_min": -45.55255126953125,
      "epoch": 14.61,
      "learning_rate": 5.6347727272727265e-05,
      "loss": 2.7599,
      "step": 251450
    },
    {
      "activations/layer0_attention_weight_max": 15.727850914001465,
      "activations/layer0_attention_weight_min": -13.8329496383667,
      "activations/layer10_attention_weight_max": 36.16962432861328,
      "activations/layer10_attention_weight_min": -36.548282623291016,
      "activations/layer11_attention_weight_max": 32.062381744384766,
      "activations/layer11_attention_weight_min": -34.965545654296875,
      "activations/layer12_attention_weight_max": 17.931983947753906,
      "activations/layer12_attention_weight_min": -23.682889938354492,
      "activations/layer13_attention_weight_max": 34.536399841308594,
      "activations/layer13_attention_weight_min": -28.76164436340332,
      "activations/layer14_attention_weight_max": 35.01469421386719,
      "activations/layer14_attention_weight_min": -30.009960174560547,
      "activations/layer15_attention_weight_max": 30.453950881958008,
      "activations/layer15_attention_weight_min": -31.993343353271484,
      "activations/layer16_attention_weight_max": 30.708255767822266,
      "activations/layer16_attention_weight_min": -29.762760162353516,
      "activations/layer17_attention_weight_max": 49.3620491027832,
      "activations/layer17_attention_weight_min": -47.264007568359375,
      "activations/layer18_attention_weight_max": 41.64181900024414,
      "activations/layer18_attention_weight_min": -38.887813568115234,
      "activations/layer19_attention_weight_max": 20.8747501373291,
      "activations/layer19_attention_weight_min": -22.033321380615234,
      "activations/layer1_attention_weight_max": 16.4443359375,
      "activations/layer1_attention_weight_min": -14.54925537109375,
      "activations/layer20_attention_weight_max": 20.15909767150879,
      "activations/layer20_attention_weight_min": -23.004371643066406,
      "activations/layer21_attention_weight_max": 35.06730270385742,
      "activations/layer21_attention_weight_min": -23.53644371032715,
      "activations/layer22_attention_weight_max": 27.691207885742188,
      "activations/layer22_attention_weight_min": -25.88892364501953,
      "activations/layer23_attention_weight_max": 37.21503448486328,
      "activations/layer23_attention_weight_min": -22.433853149414062,
      "activations/layer2_attention_weight_max": 30.99184799194336,
      "activations/layer2_attention_weight_min": -30.617385864257812,
      "activations/layer3_attention_weight_max": 84.27203369140625,
      "activations/layer3_attention_weight_min": -86.8275146484375,
      "activations/layer4_attention_weight_max": 85.9438705444336,
      "activations/layer4_attention_weight_min": -85.13678741455078,
      "activations/layer5_attention_weight_max": 65.35969543457031,
      "activations/layer5_attention_weight_min": -74.75234985351562,
      "activations/layer6_attention_weight_max": 46.679134368896484,
      "activations/layer6_attention_weight_min": -47.73995590209961,
      "activations/layer7_attention_weight_max": 58.80533218383789,
      "activations/layer7_attention_weight_min": -59.70901107788086,
      "activations/layer8_attention_weight_max": 43.006412506103516,
      "activations/layer8_attention_weight_min": -47.63423538208008,
      "activations/layer9_attention_weight_max": 45.450618743896484,
      "activations/layer9_attention_weight_min": -45.662715911865234,
      "epoch": 14.61,
      "learning_rate": 5.632878787878787e-05,
      "loss": 2.7601,
      "step": 251500
    },
    {
      "activations/layer0_attention_weight_max": 15.576257705688477,
      "activations/layer0_attention_weight_min": -13.524123191833496,
      "activations/layer10_attention_weight_max": 35.15968704223633,
      "activations/layer10_attention_weight_min": -35.40491485595703,
      "activations/layer11_attention_weight_max": 32.50053787231445,
      "activations/layer11_attention_weight_min": -32.89375686645508,
      "activations/layer12_attention_weight_max": 17.703758239746094,
      "activations/layer12_attention_weight_min": -35.43065643310547,
      "activations/layer13_attention_weight_max": 35.47773742675781,
      "activations/layer13_attention_weight_min": -30.64401626586914,
      "activations/layer14_attention_weight_max": 39.07329177856445,
      "activations/layer14_attention_weight_min": -31.70966148376465,
      "activations/layer15_attention_weight_max": 34.90407180786133,
      "activations/layer15_attention_weight_min": -31.714183807373047,
      "activations/layer16_attention_weight_max": 35.981258392333984,
      "activations/layer16_attention_weight_min": -33.13408660888672,
      "activations/layer17_attention_weight_max": 54.07112121582031,
      "activations/layer17_attention_weight_min": -49.120338439941406,
      "activations/layer18_attention_weight_max": 48.43893814086914,
      "activations/layer18_attention_weight_min": -43.57236099243164,
      "activations/layer19_attention_weight_max": 22.965452194213867,
      "activations/layer19_attention_weight_min": -23.81917381286621,
      "activations/layer1_attention_weight_max": 16.523582458496094,
      "activations/layer1_attention_weight_min": -13.890256881713867,
      "activations/layer20_attention_weight_max": 22.166236877441406,
      "activations/layer20_attention_weight_min": -24.874126434326172,
      "activations/layer21_attention_weight_max": 36.51319122314453,
      "activations/layer21_attention_weight_min": -22.15461540222168,
      "activations/layer22_attention_weight_max": 31.26204490661621,
      "activations/layer22_attention_weight_min": -28.898950576782227,
      "activations/layer23_attention_weight_max": 38.689815521240234,
      "activations/layer23_attention_weight_min": -24.072086334228516,
      "activations/layer2_attention_weight_max": 32.860260009765625,
      "activations/layer2_attention_weight_min": -31.91033935546875,
      "activations/layer3_attention_weight_max": 90.66703033447266,
      "activations/layer3_attention_weight_min": -92.59269714355469,
      "activations/layer4_attention_weight_max": 90.72893524169922,
      "activations/layer4_attention_weight_min": -87.5455551147461,
      "activations/layer5_attention_weight_max": 69.35120391845703,
      "activations/layer5_attention_weight_min": -79.23680114746094,
      "activations/layer6_attention_weight_max": 49.69376754760742,
      "activations/layer6_attention_weight_min": -51.26261901855469,
      "activations/layer7_attention_weight_max": 62.61091613769531,
      "activations/layer7_attention_weight_min": -61.65473937988281,
      "activations/layer8_attention_weight_max": 44.33015823364258,
      "activations/layer8_attention_weight_min": -46.3712272644043,
      "activations/layer9_attention_weight_max": 49.6448860168457,
      "activations/layer9_attention_weight_min": -45.27988815307617,
      "epoch": 14.62,
      "learning_rate": 5.630984848484848e-05,
      "loss": 2.7527,
      "step": 251550
    },
    {
      "activations/layer0_attention_weight_max": 16.05734634399414,
      "activations/layer0_attention_weight_min": -14.224395751953125,
      "activations/layer10_attention_weight_max": 35.993682861328125,
      "activations/layer10_attention_weight_min": -35.18828582763672,
      "activations/layer11_attention_weight_max": 32.29304504394531,
      "activations/layer11_attention_weight_min": -33.60432815551758,
      "activations/layer12_attention_weight_max": 17.19411277770996,
      "activations/layer12_attention_weight_min": -26.81014633178711,
      "activations/layer13_attention_weight_max": 34.595062255859375,
      "activations/layer13_attention_weight_min": -31.063432693481445,
      "activations/layer14_attention_weight_max": 34.86201095581055,
      "activations/layer14_attention_weight_min": -29.013957977294922,
      "activations/layer15_attention_weight_max": 29.95884132385254,
      "activations/layer15_attention_weight_min": -29.11260986328125,
      "activations/layer16_attention_weight_max": 32.86907958984375,
      "activations/layer16_attention_weight_min": -30.0576171875,
      "activations/layer17_attention_weight_max": 49.29804229736328,
      "activations/layer17_attention_weight_min": -45.74917221069336,
      "activations/layer18_attention_weight_max": 44.58600997924805,
      "activations/layer18_attention_weight_min": -39.20731735229492,
      "activations/layer19_attention_weight_max": 20.960033416748047,
      "activations/layer19_attention_weight_min": -21.60250473022461,
      "activations/layer1_attention_weight_max": 16.68060874938965,
      "activations/layer1_attention_weight_min": -14.820201873779297,
      "activations/layer20_attention_weight_max": 20.811426162719727,
      "activations/layer20_attention_weight_min": -21.84676742553711,
      "activations/layer21_attention_weight_max": 38.19546127319336,
      "activations/layer21_attention_weight_min": -24.071378707885742,
      "activations/layer22_attention_weight_max": 30.71642303466797,
      "activations/layer22_attention_weight_min": -25.702985763549805,
      "activations/layer23_attention_weight_max": 36.99263000488281,
      "activations/layer23_attention_weight_min": -24.80140495300293,
      "activations/layer2_attention_weight_max": 35.08242416381836,
      "activations/layer2_attention_weight_min": -31.364866256713867,
      "activations/layer3_attention_weight_max": 93.29193115234375,
      "activations/layer3_attention_weight_min": -91.10214233398438,
      "activations/layer4_attention_weight_max": 92.21282958984375,
      "activations/layer4_attention_weight_min": -86.49408721923828,
      "activations/layer5_attention_weight_max": 68.73587036132812,
      "activations/layer5_attention_weight_min": -75.83267974853516,
      "activations/layer6_attention_weight_max": 50.9115104675293,
      "activations/layer6_attention_weight_min": -49.24891662597656,
      "activations/layer7_attention_weight_max": 61.64448928833008,
      "activations/layer7_attention_weight_min": -61.375301361083984,
      "activations/layer8_attention_weight_max": 46.067745208740234,
      "activations/layer8_attention_weight_min": -46.1505012512207,
      "activations/layer9_attention_weight_max": 43.665809631347656,
      "activations/layer9_attention_weight_min": -45.65040588378906,
      "epoch": 14.62,
      "learning_rate": 5.629090909090908e-05,
      "loss": 2.7601,
      "step": 251600
    },
    {
      "activations/layer0_attention_weight_max": 15.708966255187988,
      "activations/layer0_attention_weight_min": -14.022178649902344,
      "activations/layer10_attention_weight_max": 39.02008056640625,
      "activations/layer10_attention_weight_min": -37.224178314208984,
      "activations/layer11_attention_weight_max": 34.12565612792969,
      "activations/layer11_attention_weight_min": -34.748992919921875,
      "activations/layer12_attention_weight_max": 18.56841278076172,
      "activations/layer12_attention_weight_min": -26.172000885009766,
      "activations/layer13_attention_weight_max": 35.90371322631836,
      "activations/layer13_attention_weight_min": -31.400646209716797,
      "activations/layer14_attention_weight_max": 36.39277267456055,
      "activations/layer14_attention_weight_min": -30.777132034301758,
      "activations/layer15_attention_weight_max": 33.676334381103516,
      "activations/layer15_attention_weight_min": -31.383272171020508,
      "activations/layer16_attention_weight_max": 34.8363037109375,
      "activations/layer16_attention_weight_min": -31.917129516601562,
      "activations/layer17_attention_weight_max": 54.86149215698242,
      "activations/layer17_attention_weight_min": -45.803409576416016,
      "activations/layer18_attention_weight_max": 50.05529022216797,
      "activations/layer18_attention_weight_min": -41.50305938720703,
      "activations/layer19_attention_weight_max": 23.672033309936523,
      "activations/layer19_attention_weight_min": -21.958791732788086,
      "activations/layer1_attention_weight_max": 17.100278854370117,
      "activations/layer1_attention_weight_min": -15.029717445373535,
      "activations/layer20_attention_weight_max": 24.327322006225586,
      "activations/layer20_attention_weight_min": -22.98967170715332,
      "activations/layer21_attention_weight_max": 34.477718353271484,
      "activations/layer21_attention_weight_min": -23.842283248901367,
      "activations/layer22_attention_weight_max": 30.77165985107422,
      "activations/layer22_attention_weight_min": -26.72218132019043,
      "activations/layer23_attention_weight_max": 40.038330078125,
      "activations/layer23_attention_weight_min": -25.284330368041992,
      "activations/layer2_attention_weight_max": 33.94070816040039,
      "activations/layer2_attention_weight_min": -33.51316452026367,
      "activations/layer3_attention_weight_max": 101.93195343017578,
      "activations/layer3_attention_weight_min": -103.91181945800781,
      "activations/layer4_attention_weight_max": 97.24150848388672,
      "activations/layer4_attention_weight_min": -96.4248046875,
      "activations/layer5_attention_weight_max": 67.29818725585938,
      "activations/layer5_attention_weight_min": -79.62813568115234,
      "activations/layer6_attention_weight_max": 52.07996368408203,
      "activations/layer6_attention_weight_min": -54.820316314697266,
      "activations/layer7_attention_weight_max": 65.16653442382812,
      "activations/layer7_attention_weight_min": -65.238037109375,
      "activations/layer8_attention_weight_max": 49.81824493408203,
      "activations/layer8_attention_weight_min": -49.74976348876953,
      "activations/layer9_attention_weight_max": 48.931968688964844,
      "activations/layer9_attention_weight_min": -49.39799499511719,
      "epoch": 14.62,
      "learning_rate": 5.62719696969697e-05,
      "loss": 2.7704,
      "step": 251650
    },
    {
      "activations/layer0_attention_weight_max": 16.069395065307617,
      "activations/layer0_attention_weight_min": -14.123485565185547,
      "activations/layer10_attention_weight_max": 45.21147537231445,
      "activations/layer10_attention_weight_min": -40.82666015625,
      "activations/layer11_attention_weight_max": 41.1786994934082,
      "activations/layer11_attention_weight_min": -37.37946319580078,
      "activations/layer12_attention_weight_max": 22.358184814453125,
      "activations/layer12_attention_weight_min": -27.773588180541992,
      "activations/layer13_attention_weight_max": 39.72318649291992,
      "activations/layer13_attention_weight_min": -31.78377342224121,
      "activations/layer14_attention_weight_max": 35.995635986328125,
      "activations/layer14_attention_weight_min": -33.39442443847656,
      "activations/layer15_attention_weight_max": 33.4376335144043,
      "activations/layer15_attention_weight_min": -31.61319923400879,
      "activations/layer16_attention_weight_max": 32.26085662841797,
      "activations/layer16_attention_weight_min": -32.07465362548828,
      "activations/layer17_attention_weight_max": 49.201072692871094,
      "activations/layer17_attention_weight_min": -44.21123123168945,
      "activations/layer18_attention_weight_max": 45.77412796020508,
      "activations/layer18_attention_weight_min": -37.93621063232422,
      "activations/layer19_attention_weight_max": 22.604475021362305,
      "activations/layer19_attention_weight_min": -21.438749313354492,
      "activations/layer1_attention_weight_max": 16.83669090270996,
      "activations/layer1_attention_weight_min": -18.699148178100586,
      "activations/layer20_attention_weight_max": 24.966875076293945,
      "activations/layer20_attention_weight_min": -21.338773727416992,
      "activations/layer21_attention_weight_max": 34.168609619140625,
      "activations/layer21_attention_weight_min": -22.300100326538086,
      "activations/layer22_attention_weight_max": 30.334152221679688,
      "activations/layer22_attention_weight_min": -24.798887252807617,
      "activations/layer23_attention_weight_max": 35.85774230957031,
      "activations/layer23_attention_weight_min": -26.266128540039062,
      "activations/layer2_attention_weight_max": 32.52030944824219,
      "activations/layer2_attention_weight_min": -28.60194969177246,
      "activations/layer3_attention_weight_max": 86.3604736328125,
      "activations/layer3_attention_weight_min": -90.2378158569336,
      "activations/layer4_attention_weight_max": 86.35921478271484,
      "activations/layer4_attention_weight_min": -89.40355682373047,
      "activations/layer5_attention_weight_max": 70.38551330566406,
      "activations/layer5_attention_weight_min": -75.58583068847656,
      "activations/layer6_attention_weight_max": 51.015689849853516,
      "activations/layer6_attention_weight_min": -50.376766204833984,
      "activations/layer7_attention_weight_max": 73.01502227783203,
      "activations/layer7_attention_weight_min": -66.03325653076172,
      "activations/layer8_attention_weight_max": 50.495052337646484,
      "activations/layer8_attention_weight_min": -51.10462188720703,
      "activations/layer9_attention_weight_max": 49.2442512512207,
      "activations/layer9_attention_weight_min": -48.93781661987305,
      "epoch": 14.63,
      "learning_rate": 5.62530303030303e-05,
      "loss": 2.7516,
      "step": 251700
    },
    {
      "activations/layer0_attention_weight_max": 15.384532928466797,
      "activations/layer0_attention_weight_min": -13.688639640808105,
      "activations/layer10_attention_weight_max": 34.8123664855957,
      "activations/layer10_attention_weight_min": -33.9987907409668,
      "activations/layer11_attention_weight_max": 32.30185317993164,
      "activations/layer11_attention_weight_min": -33.49711608886719,
      "activations/layer12_attention_weight_max": 20.146888732910156,
      "activations/layer12_attention_weight_min": -27.290302276611328,
      "activations/layer13_attention_weight_max": 47.80257034301758,
      "activations/layer13_attention_weight_min": -36.800865173339844,
      "activations/layer14_attention_weight_max": 32.58750915527344,
      "activations/layer14_attention_weight_min": -29.031919479370117,
      "activations/layer15_attention_weight_max": 29.16224479675293,
      "activations/layer15_attention_weight_min": -29.176969528198242,
      "activations/layer16_attention_weight_max": 29.75982093811035,
      "activations/layer16_attention_weight_min": -29.7419376373291,
      "activations/layer17_attention_weight_max": 46.626102447509766,
      "activations/layer17_attention_weight_min": -42.75526428222656,
      "activations/layer18_attention_weight_max": 41.604461669921875,
      "activations/layer18_attention_weight_min": -37.66934585571289,
      "activations/layer19_attention_weight_max": 20.99162483215332,
      "activations/layer19_attention_weight_min": -22.13351058959961,
      "activations/layer1_attention_weight_max": 15.454950332641602,
      "activations/layer1_attention_weight_min": -16.5714054107666,
      "activations/layer20_attention_weight_max": 20.30105972290039,
      "activations/layer20_attention_weight_min": -23.285619735717773,
      "activations/layer21_attention_weight_max": 34.33835983276367,
      "activations/layer21_attention_weight_min": -23.3736572265625,
      "activations/layer22_attention_weight_max": 26.732812881469727,
      "activations/layer22_attention_weight_min": -26.534456253051758,
      "activations/layer23_attention_weight_max": 34.45494079589844,
      "activations/layer23_attention_weight_min": -25.017311096191406,
      "activations/layer2_attention_weight_max": 32.29462432861328,
      "activations/layer2_attention_weight_min": -29.98432159423828,
      "activations/layer3_attention_weight_max": 92.48777770996094,
      "activations/layer3_attention_weight_min": -93.29824829101562,
      "activations/layer4_attention_weight_max": 86.2220458984375,
      "activations/layer4_attention_weight_min": -85.95674133300781,
      "activations/layer5_attention_weight_max": 68.43978881835938,
      "activations/layer5_attention_weight_min": -72.93949890136719,
      "activations/layer6_attention_weight_max": 48.20265197753906,
      "activations/layer6_attention_weight_min": -47.96529006958008,
      "activations/layer7_attention_weight_max": 62.89623260498047,
      "activations/layer7_attention_weight_min": -63.8041877746582,
      "activations/layer8_attention_weight_max": 48.056095123291016,
      "activations/layer8_attention_weight_min": -51.577049255371094,
      "activations/layer9_attention_weight_max": 47.02027130126953,
      "activations/layer9_attention_weight_min": -44.87662887573242,
      "epoch": 14.63,
      "learning_rate": 5.62340909090909e-05,
      "loss": 2.7492,
      "step": 251750
    },
    {
      "activations/layer0_attention_weight_max": 15.239012718200684,
      "activations/layer0_attention_weight_min": -13.83244800567627,
      "activations/layer10_attention_weight_max": 38.50434875488281,
      "activations/layer10_attention_weight_min": -36.74291229248047,
      "activations/layer11_attention_weight_max": 33.64169692993164,
      "activations/layer11_attention_weight_min": -35.21600341796875,
      "activations/layer12_attention_weight_max": 19.9443302154541,
      "activations/layer12_attention_weight_min": -30.12340545654297,
      "activations/layer13_attention_weight_max": 38.34120178222656,
      "activations/layer13_attention_weight_min": -27.95804214477539,
      "activations/layer14_attention_weight_max": 47.80924987792969,
      "activations/layer14_attention_weight_min": -31.84895133972168,
      "activations/layer15_attention_weight_max": 45.32881164550781,
      "activations/layer15_attention_weight_min": -30.531238555908203,
      "activations/layer16_attention_weight_max": 39.28839874267578,
      "activations/layer16_attention_weight_min": -32.14971160888672,
      "activations/layer17_attention_weight_max": 57.50790786743164,
      "activations/layer17_attention_weight_min": -47.44742202758789,
      "activations/layer18_attention_weight_max": 57.458072662353516,
      "activations/layer18_attention_weight_min": -44.96732711791992,
      "activations/layer19_attention_weight_max": 29.621082305908203,
      "activations/layer19_attention_weight_min": -24.95787239074707,
      "activations/layer1_attention_weight_max": 16.44575309753418,
      "activations/layer1_attention_weight_min": -14.25075626373291,
      "activations/layer20_attention_weight_max": 30.803611755371094,
      "activations/layer20_attention_weight_min": -24.9158935546875,
      "activations/layer21_attention_weight_max": 50.74241256713867,
      "activations/layer21_attention_weight_min": -24.761394500732422,
      "activations/layer22_attention_weight_max": 34.172996520996094,
      "activations/layer22_attention_weight_min": -25.4444637298584,
      "activations/layer23_attention_weight_max": 40.122764587402344,
      "activations/layer23_attention_weight_min": -26.4310359954834,
      "activations/layer2_attention_weight_max": 31.529144287109375,
      "activations/layer2_attention_weight_min": -30.711423873901367,
      "activations/layer3_attention_weight_max": 96.77684020996094,
      "activations/layer3_attention_weight_min": -98.72750854492188,
      "activations/layer4_attention_weight_max": 93.89259338378906,
      "activations/layer4_attention_weight_min": -87.65831756591797,
      "activations/layer5_attention_weight_max": 69.09420013427734,
      "activations/layer5_attention_weight_min": -79.19259643554688,
      "activations/layer6_attention_weight_max": 53.2933464050293,
      "activations/layer6_attention_weight_min": -50.087833404541016,
      "activations/layer7_attention_weight_max": 66.34339904785156,
      "activations/layer7_attention_weight_min": -64.23933410644531,
      "activations/layer8_attention_weight_max": 50.5318603515625,
      "activations/layer8_attention_weight_min": -48.4885139465332,
      "activations/layer9_attention_weight_max": 49.375587463378906,
      "activations/layer9_attention_weight_min": -48.056461334228516,
      "epoch": 14.63,
      "learning_rate": 5.621515151515151e-05,
      "loss": 2.771,
      "step": 251800
    },
    {
      "activations/layer0_attention_weight_max": 17.43894386291504,
      "activations/layer0_attention_weight_min": -13.454598426818848,
      "activations/layer10_attention_weight_max": 35.03462219238281,
      "activations/layer10_attention_weight_min": -36.576576232910156,
      "activations/layer11_attention_weight_max": 33.4402961730957,
      "activations/layer11_attention_weight_min": -34.32537078857422,
      "activations/layer12_attention_weight_max": 19.39790153503418,
      "activations/layer12_attention_weight_min": -21.839170455932617,
      "activations/layer13_attention_weight_max": 32.032562255859375,
      "activations/layer13_attention_weight_min": -27.380178451538086,
      "activations/layer14_attention_weight_max": 37.219093322753906,
      "activations/layer14_attention_weight_min": -31.84545135498047,
      "activations/layer15_attention_weight_max": 32.35865783691406,
      "activations/layer15_attention_weight_min": -31.719036102294922,
      "activations/layer16_attention_weight_max": 32.903079986572266,
      "activations/layer16_attention_weight_min": -32.16670227050781,
      "activations/layer17_attention_weight_max": 55.8283576965332,
      "activations/layer17_attention_weight_min": -48.695796966552734,
      "activations/layer18_attention_weight_max": 48.17669677734375,
      "activations/layer18_attention_weight_min": -42.91478729248047,
      "activations/layer19_attention_weight_max": 23.07509994506836,
      "activations/layer19_attention_weight_min": -23.055728912353516,
      "activations/layer1_attention_weight_max": 16.278871536254883,
      "activations/layer1_attention_weight_min": -14.970071792602539,
      "activations/layer20_attention_weight_max": 24.45351219177246,
      "activations/layer20_attention_weight_min": -23.869123458862305,
      "activations/layer21_attention_weight_max": 36.209083557128906,
      "activations/layer21_attention_weight_min": -23.432872772216797,
      "activations/layer22_attention_weight_max": 29.97603988647461,
      "activations/layer22_attention_weight_min": -29.550493240356445,
      "activations/layer23_attention_weight_max": 34.040977478027344,
      "activations/layer23_attention_weight_min": -28.784629821777344,
      "activations/layer2_attention_weight_max": 31.149381637573242,
      "activations/layer2_attention_weight_min": -29.95197296142578,
      "activations/layer3_attention_weight_max": 85.29486083984375,
      "activations/layer3_attention_weight_min": -90.19239044189453,
      "activations/layer4_attention_weight_max": 85.98125457763672,
      "activations/layer4_attention_weight_min": -86.40754699707031,
      "activations/layer5_attention_weight_max": 70.87539672851562,
      "activations/layer5_attention_weight_min": -79.468017578125,
      "activations/layer6_attention_weight_max": 51.25300598144531,
      "activations/layer6_attention_weight_min": -51.635555267333984,
      "activations/layer7_attention_weight_max": 62.453487396240234,
      "activations/layer7_attention_weight_min": -61.959529876708984,
      "activations/layer8_attention_weight_max": 46.176231384277344,
      "activations/layer8_attention_weight_min": -48.51853561401367,
      "activations/layer9_attention_weight_max": 43.64332962036133,
      "activations/layer9_attention_weight_min": -47.14015579223633,
      "epoch": 14.63,
      "learning_rate": 5.619621212121212e-05,
      "loss": 2.769,
      "step": 251850
    },
    {
      "activations/layer0_attention_weight_max": 16.735492706298828,
      "activations/layer0_attention_weight_min": -12.812798500061035,
      "activations/layer10_attention_weight_max": 40.07794189453125,
      "activations/layer10_attention_weight_min": -37.8091926574707,
      "activations/layer11_attention_weight_max": 35.79368591308594,
      "activations/layer11_attention_weight_min": -32.885963439941406,
      "activations/layer12_attention_weight_max": 19.825895309448242,
      "activations/layer12_attention_weight_min": -26.509653091430664,
      "activations/layer13_attention_weight_max": 42.800594329833984,
      "activations/layer13_attention_weight_min": -29.218406677246094,
      "activations/layer14_attention_weight_max": 46.9069709777832,
      "activations/layer14_attention_weight_min": -29.844018936157227,
      "activations/layer15_attention_weight_max": 39.536685943603516,
      "activations/layer15_attention_weight_min": -29.727632522583008,
      "activations/layer16_attention_weight_max": 36.51997375488281,
      "activations/layer16_attention_weight_min": -31.415721893310547,
      "activations/layer17_attention_weight_max": 55.96419143676758,
      "activations/layer17_attention_weight_min": -45.3726692199707,
      "activations/layer18_attention_weight_max": 56.40115737915039,
      "activations/layer18_attention_weight_min": -40.63192367553711,
      "activations/layer19_attention_weight_max": 31.821165084838867,
      "activations/layer19_attention_weight_min": -25.729598999023438,
      "activations/layer1_attention_weight_max": 16.529525756835938,
      "activations/layer1_attention_weight_min": -15.493415832519531,
      "activations/layer20_attention_weight_max": 32.96989059448242,
      "activations/layer20_attention_weight_min": -23.177885055541992,
      "activations/layer21_attention_weight_max": 56.528045654296875,
      "activations/layer21_attention_weight_min": -27.516220092773438,
      "activations/layer22_attention_weight_max": 41.53190994262695,
      "activations/layer22_attention_weight_min": -29.54474639892578,
      "activations/layer23_attention_weight_max": 49.56261444091797,
      "activations/layer23_attention_weight_min": -27.04684829711914,
      "activations/layer2_attention_weight_max": 33.0546875,
      "activations/layer2_attention_weight_min": -32.992774963378906,
      "activations/layer3_attention_weight_max": 92.47830200195312,
      "activations/layer3_attention_weight_min": -101.9651107788086,
      "activations/layer4_attention_weight_max": 88.93977355957031,
      "activations/layer4_attention_weight_min": -89.62237548828125,
      "activations/layer5_attention_weight_max": 68.66421508789062,
      "activations/layer5_attention_weight_min": -78.45564270019531,
      "activations/layer6_attention_weight_max": 52.579288482666016,
      "activations/layer6_attention_weight_min": -54.236183166503906,
      "activations/layer7_attention_weight_max": 66.20404052734375,
      "activations/layer7_attention_weight_min": -71.9933853149414,
      "activations/layer8_attention_weight_max": 48.87300491333008,
      "activations/layer8_attention_weight_min": -54.322200775146484,
      "activations/layer9_attention_weight_max": 51.01008605957031,
      "activations/layer9_attention_weight_min": -47.98829650878906,
      "epoch": 14.64,
      "learning_rate": 5.617765151515151e-05,
      "loss": 2.7653,
      "step": 251900
    },
    {
      "activations/layer0_attention_weight_max": 16.905485153198242,
      "activations/layer0_attention_weight_min": -12.784070014953613,
      "activations/layer10_attention_weight_max": 37.727447509765625,
      "activations/layer10_attention_weight_min": -41.152252197265625,
      "activations/layer11_attention_weight_max": 37.49909591674805,
      "activations/layer11_attention_weight_min": -36.309776306152344,
      "activations/layer12_attention_weight_max": 18.53428077697754,
      "activations/layer12_attention_weight_min": -24.609867095947266,
      "activations/layer13_attention_weight_max": 37.075889587402344,
      "activations/layer13_attention_weight_min": -29.89189910888672,
      "activations/layer14_attention_weight_max": 37.934120178222656,
      "activations/layer14_attention_weight_min": -30.879451751708984,
      "activations/layer15_attention_weight_max": 34.71122741699219,
      "activations/layer15_attention_weight_min": -30.027849197387695,
      "activations/layer16_attention_weight_max": 34.448238372802734,
      "activations/layer16_attention_weight_min": -33.62004852294922,
      "activations/layer17_attention_weight_max": 51.87356948852539,
      "activations/layer17_attention_weight_min": -49.14143371582031,
      "activations/layer18_attention_weight_max": 49.317100524902344,
      "activations/layer18_attention_weight_min": -43.974239349365234,
      "activations/layer19_attention_weight_max": 25.604511260986328,
      "activations/layer19_attention_weight_min": -24.632680892944336,
      "activations/layer1_attention_weight_max": 16.20484161376953,
      "activations/layer1_attention_weight_min": -13.532404899597168,
      "activations/layer20_attention_weight_max": 24.47606086730957,
      "activations/layer20_attention_weight_min": -20.531206130981445,
      "activations/layer21_attention_weight_max": 39.41862487792969,
      "activations/layer21_attention_weight_min": -28.108993530273438,
      "activations/layer22_attention_weight_max": 31.73676872253418,
      "activations/layer22_attention_weight_min": -26.57753562927246,
      "activations/layer23_attention_weight_max": 41.643836975097656,
      "activations/layer23_attention_weight_min": -25.482227325439453,
      "activations/layer2_attention_weight_max": 31.421356201171875,
      "activations/layer2_attention_weight_min": -30.428062438964844,
      "activations/layer3_attention_weight_max": 93.35970306396484,
      "activations/layer3_attention_weight_min": -90.602783203125,
      "activations/layer4_attention_weight_max": 93.3659439086914,
      "activations/layer4_attention_weight_min": -85.2567138671875,
      "activations/layer5_attention_weight_max": 68.51268005371094,
      "activations/layer5_attention_weight_min": -74.79646301269531,
      "activations/layer6_attention_weight_max": 51.68784713745117,
      "activations/layer6_attention_weight_min": -52.3308219909668,
      "activations/layer7_attention_weight_max": 72.23616027832031,
      "activations/layer7_attention_weight_min": -64.69884490966797,
      "activations/layer8_attention_weight_max": 48.953670501708984,
      "activations/layer8_attention_weight_min": -52.665008544921875,
      "activations/layer9_attention_weight_max": 58.479095458984375,
      "activations/layer9_attention_weight_min": -48.220359802246094,
      "epoch": 14.64,
      "learning_rate": 5.615871212121212e-05,
      "loss": 2.7645,
      "step": 251950
    },
    {
      "activations/layer0_attention_weight_max": 16.338411331176758,
      "activations/layer0_attention_weight_min": -14.03171157836914,
      "activations/layer10_attention_weight_max": 37.86722946166992,
      "activations/layer10_attention_weight_min": -38.48257064819336,
      "activations/layer11_attention_weight_max": 32.75436782836914,
      "activations/layer11_attention_weight_min": -34.840728759765625,
      "activations/layer12_attention_weight_max": 17.8147029876709,
      "activations/layer12_attention_weight_min": -22.889623641967773,
      "activations/layer13_attention_weight_max": 46.881839752197266,
      "activations/layer13_attention_weight_min": -28.037540435791016,
      "activations/layer14_attention_weight_max": 42.90135192871094,
      "activations/layer14_attention_weight_min": -30.674192428588867,
      "activations/layer15_attention_weight_max": 33.032779693603516,
      "activations/layer15_attention_weight_min": -30.187198638916016,
      "activations/layer16_attention_weight_max": 32.825252532958984,
      "activations/layer16_attention_weight_min": -31.347246170043945,
      "activations/layer17_attention_weight_max": 50.973724365234375,
      "activations/layer17_attention_weight_min": -45.70905685424805,
      "activations/layer18_attention_weight_max": 45.80598068237305,
      "activations/layer18_attention_weight_min": -40.06330490112305,
      "activations/layer19_attention_weight_max": 23.204975128173828,
      "activations/layer19_attention_weight_min": -22.02672004699707,
      "activations/layer1_attention_weight_max": 16.413835525512695,
      "activations/layer1_attention_weight_min": -14.681572914123535,
      "activations/layer20_attention_weight_max": 22.767179489135742,
      "activations/layer20_attention_weight_min": -21.689910888671875,
      "activations/layer21_attention_weight_max": 37.801483154296875,
      "activations/layer21_attention_weight_min": -24.10700035095215,
      "activations/layer22_attention_weight_max": 30.72208023071289,
      "activations/layer22_attention_weight_min": -24.996999740600586,
      "activations/layer23_attention_weight_max": 42.20140838623047,
      "activations/layer23_attention_weight_min": -26.658992767333984,
      "activations/layer2_attention_weight_max": 33.95138931274414,
      "activations/layer2_attention_weight_min": -32.85221481323242,
      "activations/layer3_attention_weight_max": 96.19666290283203,
      "activations/layer3_attention_weight_min": -91.2493896484375,
      "activations/layer4_attention_weight_max": 92.09026336669922,
      "activations/layer4_attention_weight_min": -91.68754577636719,
      "activations/layer5_attention_weight_max": 66.78889465332031,
      "activations/layer5_attention_weight_min": -78.34640502929688,
      "activations/layer6_attention_weight_max": 50.08300018310547,
      "activations/layer6_attention_weight_min": -51.71473693847656,
      "activations/layer7_attention_weight_max": 65.68293762207031,
      "activations/layer7_attention_weight_min": -66.29643249511719,
      "activations/layer8_attention_weight_max": 47.219337463378906,
      "activations/layer8_attention_weight_min": -52.887184143066406,
      "activations/layer9_attention_weight_max": 47.382171630859375,
      "activations/layer9_attention_weight_min": -46.637081146240234,
      "epoch": 14.64,
      "learning_rate": 5.613977272727272e-05,
      "loss": 2.7625,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4581,
      "eval_samples_per_second": 507.678,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4581,
      "eval_openwebtext_samples_per_second": 507.678,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9262,
      "eval_wikitext_samples_per_second": 236.733,
      "step": 252000
    },
    {
      "epoch": 14.64,
      "eval_lambada_loss": 2.576171875,
      "eval_lambada_ppl": 13.146714440868186,
      "eval_lambada_runtime": 9.5112,
      "eval_lambada_samples_per_second": 511.921,
      "step": 252000
    },
    {
      "activations/layer0_attention_weight_max": 16.131555557250977,
      "activations/layer0_attention_weight_min": -13.227611541748047,
      "activations/layer10_attention_weight_max": 37.728302001953125,
      "activations/layer10_attention_weight_min": -35.633724212646484,
      "activations/layer11_attention_weight_max": 33.633445739746094,
      "activations/layer11_attention_weight_min": -34.96133804321289,
      "activations/layer12_attention_weight_max": 18.89523696899414,
      "activations/layer12_attention_weight_min": -26.403459548950195,
      "activations/layer13_attention_weight_max": 36.97639083862305,
      "activations/layer13_attention_weight_min": -29.32123374938965,
      "activations/layer14_attention_weight_max": 35.99306869506836,
      "activations/layer14_attention_weight_min": -32.26902389526367,
      "activations/layer15_attention_weight_max": 33.923858642578125,
      "activations/layer15_attention_weight_min": -31.04671859741211,
      "activations/layer16_attention_weight_max": 31.86811065673828,
      "activations/layer16_attention_weight_min": -30.48897361755371,
      "activations/layer17_attention_weight_max": 52.61466598510742,
      "activations/layer17_attention_weight_min": -46.019222259521484,
      "activations/layer18_attention_weight_max": 49.6321907043457,
      "activations/layer18_attention_weight_min": -39.77546310424805,
      "activations/layer19_attention_weight_max": 20.114397048950195,
      "activations/layer19_attention_weight_min": -21.162841796875,
      "activations/layer1_attention_weight_max": 18.052474975585938,
      "activations/layer1_attention_weight_min": -17.327274322509766,
      "activations/layer20_attention_weight_max": 21.917116165161133,
      "activations/layer20_attention_weight_min": -22.045875549316406,
      "activations/layer21_attention_weight_max": 35.46822738647461,
      "activations/layer21_attention_weight_min": -23.030019760131836,
      "activations/layer22_attention_weight_max": 29.647119522094727,
      "activations/layer22_attention_weight_min": -27.294986724853516,
      "activations/layer23_attention_weight_max": 37.172237396240234,
      "activations/layer23_attention_weight_min": -24.22085189819336,
      "activations/layer2_attention_weight_max": 31.675031661987305,
      "activations/layer2_attention_weight_min": -29.85239028930664,
      "activations/layer3_attention_weight_max": 96.70277404785156,
      "activations/layer3_attention_weight_min": -94.30219268798828,
      "activations/layer4_attention_weight_max": 94.21263885498047,
      "activations/layer4_attention_weight_min": -87.57088470458984,
      "activations/layer5_attention_weight_max": 69.72834777832031,
      "activations/layer5_attention_weight_min": -77.45631408691406,
      "activations/layer6_attention_weight_max": 51.138893127441406,
      "activations/layer6_attention_weight_min": -49.81576156616211,
      "activations/layer7_attention_weight_max": 64.86939239501953,
      "activations/layer7_attention_weight_min": -62.725440979003906,
      "activations/layer8_attention_weight_max": 44.40062713623047,
      "activations/layer8_attention_weight_min": -45.14979553222656,
      "activations/layer9_attention_weight_max": 49.31007766723633,
      "activations/layer9_attention_weight_min": -45.74337387084961,
      "epoch": 14.65,
      "learning_rate": 5.612083333333333e-05,
      "loss": 2.7485,
      "step": 252050
    },
    {
      "activations/layer0_attention_weight_max": 15.5099515914917,
      "activations/layer0_attention_weight_min": -13.063569068908691,
      "activations/layer10_attention_weight_max": 37.32597732543945,
      "activations/layer10_attention_weight_min": -35.868255615234375,
      "activations/layer11_attention_weight_max": 35.169979095458984,
      "activations/layer11_attention_weight_min": -32.21400451660156,
      "activations/layer12_attention_weight_max": 18.06450653076172,
      "activations/layer12_attention_weight_min": -24.42258071899414,
      "activations/layer13_attention_weight_max": 34.91765594482422,
      "activations/layer13_attention_weight_min": -28.079673767089844,
      "activations/layer14_attention_weight_max": 36.82637405395508,
      "activations/layer14_attention_weight_min": -30.7321720123291,
      "activations/layer15_attention_weight_max": 32.31676483154297,
      "activations/layer15_attention_weight_min": -28.956708908081055,
      "activations/layer16_attention_weight_max": 31.712255477905273,
      "activations/layer16_attention_weight_min": -30.630752563476562,
      "activations/layer17_attention_weight_max": 50.38887023925781,
      "activations/layer17_attention_weight_min": -42.50755310058594,
      "activations/layer18_attention_weight_max": 46.30887222290039,
      "activations/layer18_attention_weight_min": -37.46119689941406,
      "activations/layer19_attention_weight_max": 24.957921981811523,
      "activations/layer19_attention_weight_min": -22.131141662597656,
      "activations/layer1_attention_weight_max": 17.037120819091797,
      "activations/layer1_attention_weight_min": -14.293401718139648,
      "activations/layer20_attention_weight_max": 24.54869270324707,
      "activations/layer20_attention_weight_min": -24.184383392333984,
      "activations/layer21_attention_weight_max": 39.31770706176758,
      "activations/layer21_attention_weight_min": -23.5273380279541,
      "activations/layer22_attention_weight_max": 29.427082061767578,
      "activations/layer22_attention_weight_min": -25.94137191772461,
      "activations/layer23_attention_weight_max": 34.83799362182617,
      "activations/layer23_attention_weight_min": -24.780935287475586,
      "activations/layer2_attention_weight_max": 31.341291427612305,
      "activations/layer2_attention_weight_min": -29.516033172607422,
      "activations/layer3_attention_weight_max": 90.26807403564453,
      "activations/layer3_attention_weight_min": -95.62760925292969,
      "activations/layer4_attention_weight_max": 83.37128448486328,
      "activations/layer4_attention_weight_min": -82.2978286743164,
      "activations/layer5_attention_weight_max": 68.76404571533203,
      "activations/layer5_attention_weight_min": -75.5956802368164,
      "activations/layer6_attention_weight_max": 47.40150451660156,
      "activations/layer6_attention_weight_min": -48.62848663330078,
      "activations/layer7_attention_weight_max": 60.99763107299805,
      "activations/layer7_attention_weight_min": -61.11428451538086,
      "activations/layer8_attention_weight_max": 46.8346061706543,
      "activations/layer8_attention_weight_min": -50.69060516357422,
      "activations/layer9_attention_weight_max": 41.80647659301758,
      "activations/layer9_attention_weight_min": -45.88387680053711,
      "epoch": 14.65,
      "learning_rate": 5.6101893939393935e-05,
      "loss": 2.7571,
      "step": 252100
    },
    {
      "activations/layer0_attention_weight_max": 15.150683403015137,
      "activations/layer0_attention_weight_min": -13.738543510437012,
      "activations/layer10_attention_weight_max": 38.099708557128906,
      "activations/layer10_attention_weight_min": -35.90643310546875,
      "activations/layer11_attention_weight_max": 31.362197875976562,
      "activations/layer11_attention_weight_min": -35.39996337890625,
      "activations/layer12_attention_weight_max": 18.256254196166992,
      "activations/layer12_attention_weight_min": -25.976999282836914,
      "activations/layer13_attention_weight_max": 38.26374053955078,
      "activations/layer13_attention_weight_min": -27.454975128173828,
      "activations/layer14_attention_weight_max": 36.95022201538086,
      "activations/layer14_attention_weight_min": -31.364601135253906,
      "activations/layer15_attention_weight_max": 33.56562042236328,
      "activations/layer15_attention_weight_min": -29.300050735473633,
      "activations/layer16_attention_weight_max": 33.88969802856445,
      "activations/layer16_attention_weight_min": -32.86771011352539,
      "activations/layer17_attention_weight_max": 52.118019104003906,
      "activations/layer17_attention_weight_min": -44.16324234008789,
      "activations/layer18_attention_weight_max": 48.14745330810547,
      "activations/layer18_attention_weight_min": -39.58918380737305,
      "activations/layer19_attention_weight_max": 22.85749626159668,
      "activations/layer19_attention_weight_min": -21.61370277404785,
      "activations/layer1_attention_weight_max": 17.035884857177734,
      "activations/layer1_attention_weight_min": -16.37308692932129,
      "activations/layer20_attention_weight_max": 23.959810256958008,
      "activations/layer20_attention_weight_min": -20.895910263061523,
      "activations/layer21_attention_weight_max": 38.71895217895508,
      "activations/layer21_attention_weight_min": -22.32929801940918,
      "activations/layer22_attention_weight_max": 31.269060134887695,
      "activations/layer22_attention_weight_min": -26.161035537719727,
      "activations/layer23_attention_weight_max": 38.23741149902344,
      "activations/layer23_attention_weight_min": -23.172237396240234,
      "activations/layer2_attention_weight_max": 33.2338752746582,
      "activations/layer2_attention_weight_min": -29.794124603271484,
      "activations/layer3_attention_weight_max": 89.39395141601562,
      "activations/layer3_attention_weight_min": -89.3503189086914,
      "activations/layer4_attention_weight_max": 87.0165786743164,
      "activations/layer4_attention_weight_min": -86.01786804199219,
      "activations/layer5_attention_weight_max": 69.06729125976562,
      "activations/layer5_attention_weight_min": -75.9259033203125,
      "activations/layer6_attention_weight_max": 51.08338165283203,
      "activations/layer6_attention_weight_min": -50.16526412963867,
      "activations/layer7_attention_weight_max": 65.56156158447266,
      "activations/layer7_attention_weight_min": -64.57796478271484,
      "activations/layer8_attention_weight_max": 48.70012664794922,
      "activations/layer8_attention_weight_min": -49.646522521972656,
      "activations/layer9_attention_weight_max": 43.51274490356445,
      "activations/layer9_attention_weight_min": -48.19443893432617,
      "epoch": 14.65,
      "learning_rate": 5.6082954545454536e-05,
      "loss": 2.7586,
      "step": 252150
    },
    {
      "activations/layer0_attention_weight_max": 15.20627498626709,
      "activations/layer0_attention_weight_min": -13.283171653747559,
      "activations/layer10_attention_weight_max": 38.204620361328125,
      "activations/layer10_attention_weight_min": -37.181396484375,
      "activations/layer11_attention_weight_max": 35.10052490234375,
      "activations/layer11_attention_weight_min": -34.85155487060547,
      "activations/layer12_attention_weight_max": 19.210487365722656,
      "activations/layer12_attention_weight_min": -24.2774658203125,
      "activations/layer13_attention_weight_max": 39.7005500793457,
      "activations/layer13_attention_weight_min": -30.612770080566406,
      "activations/layer14_attention_weight_max": 33.96786117553711,
      "activations/layer14_attention_weight_min": -30.580717086791992,
      "activations/layer15_attention_weight_max": 32.76222229003906,
      "activations/layer15_attention_weight_min": -29.850656509399414,
      "activations/layer16_attention_weight_max": 33.995487213134766,
      "activations/layer16_attention_weight_min": -31.357152938842773,
      "activations/layer17_attention_weight_max": 57.64128875732422,
      "activations/layer17_attention_weight_min": -48.22999572753906,
      "activations/layer18_attention_weight_max": 53.09455490112305,
      "activations/layer18_attention_weight_min": -43.14427185058594,
      "activations/layer19_attention_weight_max": 22.840993881225586,
      "activations/layer19_attention_weight_min": -22.44348907470703,
      "activations/layer1_attention_weight_max": 15.19576358795166,
      "activations/layer1_attention_weight_min": -13.348489761352539,
      "activations/layer20_attention_weight_max": 23.146873474121094,
      "activations/layer20_attention_weight_min": -21.92222785949707,
      "activations/layer21_attention_weight_max": 36.541603088378906,
      "activations/layer21_attention_weight_min": -22.962366104125977,
      "activations/layer22_attention_weight_max": 33.092674255371094,
      "activations/layer22_attention_weight_min": -24.919879913330078,
      "activations/layer23_attention_weight_max": 33.724952697753906,
      "activations/layer23_attention_weight_min": -27.470584869384766,
      "activations/layer2_attention_weight_max": 32.37800598144531,
      "activations/layer2_attention_weight_min": -31.896390914916992,
      "activations/layer3_attention_weight_max": 88.34540557861328,
      "activations/layer3_attention_weight_min": -91.90472412109375,
      "activations/layer4_attention_weight_max": 91.19430541992188,
      "activations/layer4_attention_weight_min": -94.00611114501953,
      "activations/layer5_attention_weight_max": 67.66828918457031,
      "activations/layer5_attention_weight_min": -78.51922607421875,
      "activations/layer6_attention_weight_max": 52.98139572143555,
      "activations/layer6_attention_weight_min": -51.44404602050781,
      "activations/layer7_attention_weight_max": 63.65933609008789,
      "activations/layer7_attention_weight_min": -64.88319396972656,
      "activations/layer8_attention_weight_max": 47.48146438598633,
      "activations/layer8_attention_weight_min": -49.028594970703125,
      "activations/layer9_attention_weight_max": 50.4781494140625,
      "activations/layer9_attention_weight_min": -47.97532272338867,
      "epoch": 14.65,
      "learning_rate": 5.6064015151515144e-05,
      "loss": 2.7658,
      "step": 252200
    },
    {
      "activations/layer0_attention_weight_max": 15.936117172241211,
      "activations/layer0_attention_weight_min": -13.78534984588623,
      "activations/layer10_attention_weight_max": 42.68207931518555,
      "activations/layer10_attention_weight_min": -40.87028884887695,
      "activations/layer11_attention_weight_max": 38.84361267089844,
      "activations/layer11_attention_weight_min": -36.52579116821289,
      "activations/layer12_attention_weight_max": 17.692703247070312,
      "activations/layer12_attention_weight_min": -25.510215759277344,
      "activations/layer13_attention_weight_max": 35.27322769165039,
      "activations/layer13_attention_weight_min": -29.171754837036133,
      "activations/layer14_attention_weight_max": 34.79596710205078,
      "activations/layer14_attention_weight_min": -30.812358856201172,
      "activations/layer15_attention_weight_max": 31.162492752075195,
      "activations/layer15_attention_weight_min": -29.857816696166992,
      "activations/layer16_attention_weight_max": 30.98557472229004,
      "activations/layer16_attention_weight_min": -30.838171005249023,
      "activations/layer17_attention_weight_max": 49.30009078979492,
      "activations/layer17_attention_weight_min": -43.73493194580078,
      "activations/layer18_attention_weight_max": 44.1428337097168,
      "activations/layer18_attention_weight_min": -38.5429801940918,
      "activations/layer19_attention_weight_max": 23.439254760742188,
      "activations/layer19_attention_weight_min": -23.141878128051758,
      "activations/layer1_attention_weight_max": 16.61383628845215,
      "activations/layer1_attention_weight_min": -13.737435340881348,
      "activations/layer20_attention_weight_max": 23.330455780029297,
      "activations/layer20_attention_weight_min": -21.426298141479492,
      "activations/layer21_attention_weight_max": 34.012916564941406,
      "activations/layer21_attention_weight_min": -23.588560104370117,
      "activations/layer22_attention_weight_max": 30.29038429260254,
      "activations/layer22_attention_weight_min": -26.065948486328125,
      "activations/layer23_attention_weight_max": 42.90617370605469,
      "activations/layer23_attention_weight_min": -24.017982482910156,
      "activations/layer2_attention_weight_max": 32.644691467285156,
      "activations/layer2_attention_weight_min": -29.87136459350586,
      "activations/layer3_attention_weight_max": 94.97119903564453,
      "activations/layer3_attention_weight_min": -96.41342163085938,
      "activations/layer4_attention_weight_max": 91.23626708984375,
      "activations/layer4_attention_weight_min": -98.92534637451172,
      "activations/layer5_attention_weight_max": 69.4913558959961,
      "activations/layer5_attention_weight_min": -79.038818359375,
      "activations/layer6_attention_weight_max": 52.36317443847656,
      "activations/layer6_attention_weight_min": -56.450565338134766,
      "activations/layer7_attention_weight_max": 65.79090881347656,
      "activations/layer7_attention_weight_min": -68.99610137939453,
      "activations/layer8_attention_weight_max": 49.19983673095703,
      "activations/layer8_attention_weight_min": -50.40459442138672,
      "activations/layer9_attention_weight_max": 46.630184173583984,
      "activations/layer9_attention_weight_min": -47.18764877319336,
      "epoch": 14.66,
      "learning_rate": 5.604507575757575e-05,
      "loss": 2.7469,
      "step": 252250
    },
    {
      "activations/layer0_attention_weight_max": 15.633428573608398,
      "activations/layer0_attention_weight_min": -13.812024116516113,
      "activations/layer10_attention_weight_max": 38.6531982421875,
      "activations/layer10_attention_weight_min": -38.00071716308594,
      "activations/layer11_attention_weight_max": 35.941261291503906,
      "activations/layer11_attention_weight_min": -36.630062103271484,
      "activations/layer12_attention_weight_max": 17.682849884033203,
      "activations/layer12_attention_weight_min": -26.551048278808594,
      "activations/layer13_attention_weight_max": 32.05949401855469,
      "activations/layer13_attention_weight_min": -27.7703914642334,
      "activations/layer14_attention_weight_max": 35.08745193481445,
      "activations/layer14_attention_weight_min": -29.49294662475586,
      "activations/layer15_attention_weight_max": 31.66851043701172,
      "activations/layer15_attention_weight_min": -30.17308235168457,
      "activations/layer16_attention_weight_max": 32.73821258544922,
      "activations/layer16_attention_weight_min": -31.70922088623047,
      "activations/layer17_attention_weight_max": 51.80535888671875,
      "activations/layer17_attention_weight_min": -44.48748016357422,
      "activations/layer18_attention_weight_max": 44.701324462890625,
      "activations/layer18_attention_weight_min": -38.57685470581055,
      "activations/layer19_attention_weight_max": 22.94798469543457,
      "activations/layer19_attention_weight_min": -22.82268714904785,
      "activations/layer1_attention_weight_max": 16.489381790161133,
      "activations/layer1_attention_weight_min": -14.246481895446777,
      "activations/layer20_attention_weight_max": 22.306509017944336,
      "activations/layer20_attention_weight_min": -21.91271209716797,
      "activations/layer21_attention_weight_max": 40.224456787109375,
      "activations/layer21_attention_weight_min": -23.029682159423828,
      "activations/layer22_attention_weight_max": 26.804006576538086,
      "activations/layer22_attention_weight_min": -24.25275421142578,
      "activations/layer23_attention_weight_max": 36.98554611206055,
      "activations/layer23_attention_weight_min": -24.501676559448242,
      "activations/layer2_attention_weight_max": 31.638036727905273,
      "activations/layer2_attention_weight_min": -33.394920349121094,
      "activations/layer3_attention_weight_max": 93.94171142578125,
      "activations/layer3_attention_weight_min": -97.72357940673828,
      "activations/layer4_attention_weight_max": 94.45858001708984,
      "activations/layer4_attention_weight_min": -91.15642547607422,
      "activations/layer5_attention_weight_max": 73.65022277832031,
      "activations/layer5_attention_weight_min": -83.09052276611328,
      "activations/layer6_attention_weight_max": 53.6132698059082,
      "activations/layer6_attention_weight_min": -53.00772476196289,
      "activations/layer7_attention_weight_max": 68.74375915527344,
      "activations/layer7_attention_weight_min": -69.83536529541016,
      "activations/layer8_attention_weight_max": 48.17558670043945,
      "activations/layer8_attention_weight_min": -51.60764694213867,
      "activations/layer9_attention_weight_max": 49.00307083129883,
      "activations/layer9_attention_weight_min": -51.05789566040039,
      "epoch": 14.66,
      "learning_rate": 5.602613636363636e-05,
      "loss": 2.7652,
      "step": 252300
    },
    {
      "activations/layer0_attention_weight_max": 16.56187629699707,
      "activations/layer0_attention_weight_min": -13.747730255126953,
      "activations/layer10_attention_weight_max": 38.18412780761719,
      "activations/layer10_attention_weight_min": -37.10171890258789,
      "activations/layer11_attention_weight_max": 35.000770568847656,
      "activations/layer11_attention_weight_min": -34.571922302246094,
      "activations/layer12_attention_weight_max": 17.61925506591797,
      "activations/layer12_attention_weight_min": -23.41587257385254,
      "activations/layer13_attention_weight_max": 32.232154846191406,
      "activations/layer13_attention_weight_min": -26.917381286621094,
      "activations/layer14_attention_weight_max": 36.599666595458984,
      "activations/layer14_attention_weight_min": -30.663854598999023,
      "activations/layer15_attention_weight_max": 32.211605072021484,
      "activations/layer15_attention_weight_min": -30.039470672607422,
      "activations/layer16_attention_weight_max": 33.393402099609375,
      "activations/layer16_attention_weight_min": -32.7169189453125,
      "activations/layer17_attention_weight_max": 56.706703186035156,
      "activations/layer17_attention_weight_min": -47.256736755371094,
      "activations/layer18_attention_weight_max": 47.8914680480957,
      "activations/layer18_attention_weight_min": -41.0709114074707,
      "activations/layer19_attention_weight_max": 22.176454544067383,
      "activations/layer19_attention_weight_min": -23.2320613861084,
      "activations/layer1_attention_weight_max": 16.559656143188477,
      "activations/layer1_attention_weight_min": -14.623347282409668,
      "activations/layer20_attention_weight_max": 22.835369110107422,
      "activations/layer20_attention_weight_min": -24.222557067871094,
      "activations/layer21_attention_weight_max": 37.77180862426758,
      "activations/layer21_attention_weight_min": -21.633760452270508,
      "activations/layer22_attention_weight_max": 28.141998291015625,
      "activations/layer22_attention_weight_min": -25.3314151763916,
      "activations/layer23_attention_weight_max": 39.21331787109375,
      "activations/layer23_attention_weight_min": -24.731090545654297,
      "activations/layer2_attention_weight_max": 32.94746780395508,
      "activations/layer2_attention_weight_min": -31.898977279663086,
      "activations/layer3_attention_weight_max": 94.86384582519531,
      "activations/layer3_attention_weight_min": -98.44535827636719,
      "activations/layer4_attention_weight_max": 95.38284301757812,
      "activations/layer4_attention_weight_min": -92.54241180419922,
      "activations/layer5_attention_weight_max": 71.48799896240234,
      "activations/layer5_attention_weight_min": -81.8189697265625,
      "activations/layer6_attention_weight_max": 52.85586929321289,
      "activations/layer6_attention_weight_min": -54.68173599243164,
      "activations/layer7_attention_weight_max": 66.04126739501953,
      "activations/layer7_attention_weight_min": -62.93785095214844,
      "activations/layer8_attention_weight_max": 46.45732116699219,
      "activations/layer8_attention_weight_min": -50.12443923950195,
      "activations/layer9_attention_weight_max": 47.60611343383789,
      "activations/layer9_attention_weight_min": -48.41510772705078,
      "epoch": 14.66,
      "learning_rate": 5.600719696969696e-05,
      "loss": 2.7729,
      "step": 252350
    },
    {
      "activations/layer0_attention_weight_max": 15.275152206420898,
      "activations/layer0_attention_weight_min": -13.633365631103516,
      "activations/layer10_attention_weight_max": 36.09211349487305,
      "activations/layer10_attention_weight_min": -37.01250076293945,
      "activations/layer11_attention_weight_max": 32.41883850097656,
      "activations/layer11_attention_weight_min": -34.89404296875,
      "activations/layer12_attention_weight_max": 17.94873046875,
      "activations/layer12_attention_weight_min": -25.5213565826416,
      "activations/layer13_attention_weight_max": 34.878360748291016,
      "activations/layer13_attention_weight_min": -29.75752830505371,
      "activations/layer14_attention_weight_max": 37.85335159301758,
      "activations/layer14_attention_weight_min": -31.56299591064453,
      "activations/layer15_attention_weight_max": 32.911468505859375,
      "activations/layer15_attention_weight_min": -32.204017639160156,
      "activations/layer16_attention_weight_max": 32.484092712402344,
      "activations/layer16_attention_weight_min": -31.169605255126953,
      "activations/layer17_attention_weight_max": 52.44124221801758,
      "activations/layer17_attention_weight_min": -47.11152648925781,
      "activations/layer18_attention_weight_max": 45.949100494384766,
      "activations/layer18_attention_weight_min": -40.314666748046875,
      "activations/layer19_attention_weight_max": 22.497438430786133,
      "activations/layer19_attention_weight_min": -24.12220001220703,
      "activations/layer1_attention_weight_max": 16.383811950683594,
      "activations/layer1_attention_weight_min": -14.48231315612793,
      "activations/layer20_attention_weight_max": 21.98033332824707,
      "activations/layer20_attention_weight_min": -22.417457580566406,
      "activations/layer21_attention_weight_max": 34.67197799682617,
      "activations/layer21_attention_weight_min": -21.164031982421875,
      "activations/layer22_attention_weight_max": 30.84197235107422,
      "activations/layer22_attention_weight_min": -25.252155303955078,
      "activations/layer23_attention_weight_max": 36.826995849609375,
      "activations/layer23_attention_weight_min": -23.10552215576172,
      "activations/layer2_attention_weight_max": 31.71966552734375,
      "activations/layer2_attention_weight_min": -30.938283920288086,
      "activations/layer3_attention_weight_max": 88.90142822265625,
      "activations/layer3_attention_weight_min": -95.6003646850586,
      "activations/layer4_attention_weight_max": 86.71443939208984,
      "activations/layer4_attention_weight_min": -89.00601196289062,
      "activations/layer5_attention_weight_max": 66.81665802001953,
      "activations/layer5_attention_weight_min": -73.16787719726562,
      "activations/layer6_attention_weight_max": 48.17365264892578,
      "activations/layer6_attention_weight_min": -51.95655822753906,
      "activations/layer7_attention_weight_max": 66.3967056274414,
      "activations/layer7_attention_weight_min": -60.560279846191406,
      "activations/layer8_attention_weight_max": 42.74802780151367,
      "activations/layer8_attention_weight_min": -46.825809478759766,
      "activations/layer9_attention_weight_max": 45.1624755859375,
      "activations/layer9_attention_weight_min": -45.52460479736328,
      "epoch": 14.67,
      "learning_rate": 5.5988257575757564e-05,
      "loss": 2.7739,
      "step": 252400
    },
    {
      "activations/layer0_attention_weight_max": 15.597075462341309,
      "activations/layer0_attention_weight_min": -13.219115257263184,
      "activations/layer10_attention_weight_max": 38.83864974975586,
      "activations/layer10_attention_weight_min": -39.27128982543945,
      "activations/layer11_attention_weight_max": 34.826141357421875,
      "activations/layer11_attention_weight_min": -33.931182861328125,
      "activations/layer12_attention_weight_max": 18.867950439453125,
      "activations/layer12_attention_weight_min": -25.634143829345703,
      "activations/layer13_attention_weight_max": 37.6345100402832,
      "activations/layer13_attention_weight_min": -30.48773765563965,
      "activations/layer14_attention_weight_max": 34.54905700683594,
      "activations/layer14_attention_weight_min": -30.89727210998535,
      "activations/layer15_attention_weight_max": 33.03678512573242,
      "activations/layer15_attention_weight_min": -30.62887954711914,
      "activations/layer16_attention_weight_max": 32.390724182128906,
      "activations/layer16_attention_weight_min": -32.375301361083984,
      "activations/layer17_attention_weight_max": 48.91053771972656,
      "activations/layer17_attention_weight_min": -44.74690628051758,
      "activations/layer18_attention_weight_max": 45.550148010253906,
      "activations/layer18_attention_weight_min": -38.90916442871094,
      "activations/layer19_attention_weight_max": 23.752042770385742,
      "activations/layer19_attention_weight_min": -24.301834106445312,
      "activations/layer1_attention_weight_max": 15.457420349121094,
      "activations/layer1_attention_weight_min": -14.56792163848877,
      "activations/layer20_attention_weight_max": 24.20270538330078,
      "activations/layer20_attention_weight_min": -22.684091567993164,
      "activations/layer21_attention_weight_max": 34.42753219604492,
      "activations/layer21_attention_weight_min": -22.378671646118164,
      "activations/layer22_attention_weight_max": 30.78799057006836,
      "activations/layer22_attention_weight_min": -26.250234603881836,
      "activations/layer23_attention_weight_max": 40.39705276489258,
      "activations/layer23_attention_weight_min": -24.60184097290039,
      "activations/layer2_attention_weight_max": 32.381561279296875,
      "activations/layer2_attention_weight_min": -30.753570556640625,
      "activations/layer3_attention_weight_max": 94.93074035644531,
      "activations/layer3_attention_weight_min": -99.54464721679688,
      "activations/layer4_attention_weight_max": 94.2704849243164,
      "activations/layer4_attention_weight_min": -92.27124786376953,
      "activations/layer5_attention_weight_max": 69.95319366455078,
      "activations/layer5_attention_weight_min": -79.80606842041016,
      "activations/layer6_attention_weight_max": 54.229713439941406,
      "activations/layer6_attention_weight_min": -53.933719635009766,
      "activations/layer7_attention_weight_max": 64.87346649169922,
      "activations/layer7_attention_weight_min": -66.6326904296875,
      "activations/layer8_attention_weight_max": 48.30331039428711,
      "activations/layer8_attention_weight_min": -52.684043884277344,
      "activations/layer9_attention_weight_max": 48.5989875793457,
      "activations/layer9_attention_weight_min": -48.855186462402344,
      "epoch": 14.67,
      "learning_rate": 5.596931818181818e-05,
      "loss": 2.7372,
      "step": 252450
    },
    {
      "activations/layer0_attention_weight_max": 16.30295181274414,
      "activations/layer0_attention_weight_min": -13.754488945007324,
      "activations/layer10_attention_weight_max": 34.93436050415039,
      "activations/layer10_attention_weight_min": -35.9003791809082,
      "activations/layer11_attention_weight_max": 34.70594787597656,
      "activations/layer11_attention_weight_min": -34.352813720703125,
      "activations/layer12_attention_weight_max": 17.291091918945312,
      "activations/layer12_attention_weight_min": -25.36359405517578,
      "activations/layer13_attention_weight_max": 35.15532684326172,
      "activations/layer13_attention_weight_min": -27.067358016967773,
      "activations/layer14_attention_weight_max": 36.016204833984375,
      "activations/layer14_attention_weight_min": -29.56721305847168,
      "activations/layer15_attention_weight_max": 30.953472137451172,
      "activations/layer15_attention_weight_min": -29.14672088623047,
      "activations/layer16_attention_weight_max": 31.88069725036621,
      "activations/layer16_attention_weight_min": -31.618253707885742,
      "activations/layer17_attention_weight_max": 49.87407302856445,
      "activations/layer17_attention_weight_min": -47.99432373046875,
      "activations/layer18_attention_weight_max": 48.66361999511719,
      "activations/layer18_attention_weight_min": -39.59539031982422,
      "activations/layer19_attention_weight_max": 23.59969139099121,
      "activations/layer19_attention_weight_min": -23.511098861694336,
      "activations/layer1_attention_weight_max": 16.561765670776367,
      "activations/layer1_attention_weight_min": -14.1505765914917,
      "activations/layer20_attention_weight_max": 25.472267150878906,
      "activations/layer20_attention_weight_min": -20.75291633605957,
      "activations/layer21_attention_weight_max": 37.50624084472656,
      "activations/layer21_attention_weight_min": -22.852285385131836,
      "activations/layer22_attention_weight_max": 29.99527931213379,
      "activations/layer22_attention_weight_min": -26.613698959350586,
      "activations/layer23_attention_weight_max": 36.5438232421875,
      "activations/layer23_attention_weight_min": -24.565948486328125,
      "activations/layer2_attention_weight_max": 33.00423812866211,
      "activations/layer2_attention_weight_min": -31.301158905029297,
      "activations/layer3_attention_weight_max": 95.34620666503906,
      "activations/layer3_attention_weight_min": -93.97222900390625,
      "activations/layer4_attention_weight_max": 93.87596893310547,
      "activations/layer4_attention_weight_min": -87.16566467285156,
      "activations/layer5_attention_weight_max": 67.39964294433594,
      "activations/layer5_attention_weight_min": -76.40744018554688,
      "activations/layer6_attention_weight_max": 52.46754455566406,
      "activations/layer6_attention_weight_min": -54.14977264404297,
      "activations/layer7_attention_weight_max": 63.65049362182617,
      "activations/layer7_attention_weight_min": -62.67667007446289,
      "activations/layer8_attention_weight_max": 46.35669708251953,
      "activations/layer8_attention_weight_min": -51.71848678588867,
      "activations/layer9_attention_weight_max": 42.957881927490234,
      "activations/layer9_attention_weight_min": -48.023433685302734,
      "epoch": 14.67,
      "learning_rate": 5.595037878787878e-05,
      "loss": 2.7518,
      "step": 252500
    },
    {
      "activations/layer0_attention_weight_max": 15.709144592285156,
      "activations/layer0_attention_weight_min": -13.4157075881958,
      "activations/layer10_attention_weight_max": 34.11069107055664,
      "activations/layer10_attention_weight_min": -33.994441986083984,
      "activations/layer11_attention_weight_max": 31.0208740234375,
      "activations/layer11_attention_weight_min": -32.421382904052734,
      "activations/layer12_attention_weight_max": 20.301406860351562,
      "activations/layer12_attention_weight_min": -25.421993255615234,
      "activations/layer13_attention_weight_max": 37.69115447998047,
      "activations/layer13_attention_weight_min": -27.52361488342285,
      "activations/layer14_attention_weight_max": 41.84248352050781,
      "activations/layer14_attention_weight_min": -29.014747619628906,
      "activations/layer15_attention_weight_max": 34.45098876953125,
      "activations/layer15_attention_weight_min": -29.61197853088379,
      "activations/layer16_attention_weight_max": 32.77079391479492,
      "activations/layer16_attention_weight_min": -30.125911712646484,
      "activations/layer17_attention_weight_max": 52.4817008972168,
      "activations/layer17_attention_weight_min": -43.4915885925293,
      "activations/layer18_attention_weight_max": 49.44087219238281,
      "activations/layer18_attention_weight_min": -40.31940460205078,
      "activations/layer19_attention_weight_max": 26.06910514831543,
      "activations/layer19_attention_weight_min": -24.655099868774414,
      "activations/layer1_attention_weight_max": 16.656652450561523,
      "activations/layer1_attention_weight_min": -13.012311935424805,
      "activations/layer20_attention_weight_max": 24.39252281188965,
      "activations/layer20_attention_weight_min": -23.61025047302246,
      "activations/layer21_attention_weight_max": 40.027618408203125,
      "activations/layer21_attention_weight_min": -26.78759765625,
      "activations/layer22_attention_weight_max": 29.3035888671875,
      "activations/layer22_attention_weight_min": -27.078144073486328,
      "activations/layer23_attention_weight_max": 36.61834716796875,
      "activations/layer23_attention_weight_min": -25.4034423828125,
      "activations/layer2_attention_weight_max": 33.00238037109375,
      "activations/layer2_attention_weight_min": -32.230133056640625,
      "activations/layer3_attention_weight_max": 93.70557403564453,
      "activations/layer3_attention_weight_min": -93.49604797363281,
      "activations/layer4_attention_weight_max": 88.22704315185547,
      "activations/layer4_attention_weight_min": -87.1598129272461,
      "activations/layer5_attention_weight_max": 68.56636047363281,
      "activations/layer5_attention_weight_min": -77.65233612060547,
      "activations/layer6_attention_weight_max": 48.55508804321289,
      "activations/layer6_attention_weight_min": -49.105655670166016,
      "activations/layer7_attention_weight_max": 61.28076934814453,
      "activations/layer7_attention_weight_min": -57.98015213012695,
      "activations/layer8_attention_weight_max": 43.87065505981445,
      "activations/layer8_attention_weight_min": -49.09580612182617,
      "activations/layer9_attention_weight_max": 42.88149642944336,
      "activations/layer9_attention_weight_min": -44.07534408569336,
      "epoch": 14.67,
      "learning_rate": 5.593143939393939e-05,
      "loss": 2.7531,
      "step": 252550
    },
    {
      "activations/layer0_attention_weight_max": 16.568134307861328,
      "activations/layer0_attention_weight_min": -13.687152862548828,
      "activations/layer10_attention_weight_max": 35.908973693847656,
      "activations/layer10_attention_weight_min": -34.96524429321289,
      "activations/layer11_attention_weight_max": 33.5574951171875,
      "activations/layer11_attention_weight_min": -31.842344284057617,
      "activations/layer12_attention_weight_max": 20.221757888793945,
      "activations/layer12_attention_weight_min": -28.828353881835938,
      "activations/layer13_attention_weight_max": 38.54758834838867,
      "activations/layer13_attention_weight_min": -27.510400772094727,
      "activations/layer14_attention_weight_max": 37.79936218261719,
      "activations/layer14_attention_weight_min": -29.18021583557129,
      "activations/layer15_attention_weight_max": 34.46455764770508,
      "activations/layer15_attention_weight_min": -29.735082626342773,
      "activations/layer16_attention_weight_max": 34.82442092895508,
      "activations/layer16_attention_weight_min": -29.84457015991211,
      "activations/layer17_attention_weight_max": 53.64590835571289,
      "activations/layer17_attention_weight_min": -46.92688751220703,
      "activations/layer18_attention_weight_max": 47.25119400024414,
      "activations/layer18_attention_weight_min": -37.72283172607422,
      "activations/layer19_attention_weight_max": 22.37343406677246,
      "activations/layer19_attention_weight_min": -21.363656997680664,
      "activations/layer1_attention_weight_max": 15.954985618591309,
      "activations/layer1_attention_weight_min": -14.532405853271484,
      "activations/layer20_attention_weight_max": 24.638925552368164,
      "activations/layer20_attention_weight_min": -22.7049617767334,
      "activations/layer21_attention_weight_max": 44.5833854675293,
      "activations/layer21_attention_weight_min": -22.68661880493164,
      "activations/layer22_attention_weight_max": 30.398042678833008,
      "activations/layer22_attention_weight_min": -28.736021041870117,
      "activations/layer23_attention_weight_max": 39.722679138183594,
      "activations/layer23_attention_weight_min": -27.970836639404297,
      "activations/layer2_attention_weight_max": 35.01866149902344,
      "activations/layer2_attention_weight_min": -30.269804000854492,
      "activations/layer3_attention_weight_max": 96.77165222167969,
      "activations/layer3_attention_weight_min": -91.6384048461914,
      "activations/layer4_attention_weight_max": 90.86622619628906,
      "activations/layer4_attention_weight_min": -84.3627700805664,
      "activations/layer5_attention_weight_max": 65.73693084716797,
      "activations/layer5_attention_weight_min": -69.9002456665039,
      "activations/layer6_attention_weight_max": 51.0890998840332,
      "activations/layer6_attention_weight_min": -47.82974624633789,
      "activations/layer7_attention_weight_max": 67.02146911621094,
      "activations/layer7_attention_weight_min": -61.139469146728516,
      "activations/layer8_attention_weight_max": 47.200279235839844,
      "activations/layer8_attention_weight_min": -46.65485382080078,
      "activations/layer9_attention_weight_max": 49.185272216796875,
      "activations/layer9_attention_weight_min": -44.481666564941406,
      "epoch": 14.68,
      "learning_rate": 5.59125e-05,
      "loss": 2.7628,
      "step": 252600
    },
    {
      "activations/layer0_attention_weight_max": 17.205459594726562,
      "activations/layer0_attention_weight_min": -13.840960502624512,
      "activations/layer10_attention_weight_max": 32.89767074584961,
      "activations/layer10_attention_weight_min": -35.0259895324707,
      "activations/layer11_attention_weight_max": 30.528522491455078,
      "activations/layer11_attention_weight_min": -33.56383514404297,
      "activations/layer12_attention_weight_max": 17.641530990600586,
      "activations/layer12_attention_weight_min": -24.88530158996582,
      "activations/layer13_attention_weight_max": 33.62158966064453,
      "activations/layer13_attention_weight_min": -30.278812408447266,
      "activations/layer14_attention_weight_max": 35.84593963623047,
      "activations/layer14_attention_weight_min": -30.442642211914062,
      "activations/layer15_attention_weight_max": 32.477386474609375,
      "activations/layer15_attention_weight_min": -29.114776611328125,
      "activations/layer16_attention_weight_max": 32.24179458618164,
      "activations/layer16_attention_weight_min": -29.397966384887695,
      "activations/layer17_attention_weight_max": 51.87678909301758,
      "activations/layer17_attention_weight_min": -42.921104431152344,
      "activations/layer18_attention_weight_max": 45.68486785888672,
      "activations/layer18_attention_weight_min": -39.440616607666016,
      "activations/layer19_attention_weight_max": 22.068323135375977,
      "activations/layer19_attention_weight_min": -23.157325744628906,
      "activations/layer1_attention_weight_max": 16.935705184936523,
      "activations/layer1_attention_weight_min": -13.738042831420898,
      "activations/layer20_attention_weight_max": 21.56537437438965,
      "activations/layer20_attention_weight_min": -20.769060134887695,
      "activations/layer21_attention_weight_max": 34.447357177734375,
      "activations/layer21_attention_weight_min": -21.96038818359375,
      "activations/layer22_attention_weight_max": 27.54161262512207,
      "activations/layer22_attention_weight_min": -24.765209197998047,
      "activations/layer23_attention_weight_max": 37.527713775634766,
      "activations/layer23_attention_weight_min": -22.953237533569336,
      "activations/layer2_attention_weight_max": 34.43684768676758,
      "activations/layer2_attention_weight_min": -31.191165924072266,
      "activations/layer3_attention_weight_max": 95.03099822998047,
      "activations/layer3_attention_weight_min": -91.80207824707031,
      "activations/layer4_attention_weight_max": 86.82075500488281,
      "activations/layer4_attention_weight_min": -86.39682006835938,
      "activations/layer5_attention_weight_max": 63.992652893066406,
      "activations/layer5_attention_weight_min": -73.34004974365234,
      "activations/layer6_attention_weight_max": 47.918209075927734,
      "activations/layer6_attention_weight_min": -49.137874603271484,
      "activations/layer7_attention_weight_max": 59.03889846801758,
      "activations/layer7_attention_weight_min": -63.709228515625,
      "activations/layer8_attention_weight_max": 41.82820129394531,
      "activations/layer8_attention_weight_min": -45.3915901184082,
      "activations/layer9_attention_weight_max": 47.86183166503906,
      "activations/layer9_attention_weight_min": -44.24637222290039,
      "epoch": 14.68,
      "learning_rate": 5.5893560606060605e-05,
      "loss": 2.7649,
      "step": 252650
    },
    {
      "activations/layer0_attention_weight_max": 15.795659065246582,
      "activations/layer0_attention_weight_min": -13.735461235046387,
      "activations/layer10_attention_weight_max": 37.348533630371094,
      "activations/layer10_attention_weight_min": -36.402565002441406,
      "activations/layer11_attention_weight_max": 32.92823028564453,
      "activations/layer11_attention_weight_min": -33.535160064697266,
      "activations/layer12_attention_weight_max": 19.09089469909668,
      "activations/layer12_attention_weight_min": -28.57980728149414,
      "activations/layer13_attention_weight_max": 41.94765090942383,
      "activations/layer13_attention_weight_min": -33.52122116088867,
      "activations/layer14_attention_weight_max": 43.70140838623047,
      "activations/layer14_attention_weight_min": -31.676651000976562,
      "activations/layer15_attention_weight_max": 40.56313705444336,
      "activations/layer15_attention_weight_min": -30.679792404174805,
      "activations/layer16_attention_weight_max": 31.26862335205078,
      "activations/layer16_attention_weight_min": -31.653011322021484,
      "activations/layer17_attention_weight_max": 52.09028244018555,
      "activations/layer17_attention_weight_min": -45.8211555480957,
      "activations/layer18_attention_weight_max": 47.64317321777344,
      "activations/layer18_attention_weight_min": -39.528717041015625,
      "activations/layer19_attention_weight_max": 25.187400817871094,
      "activations/layer19_attention_weight_min": -23.843217849731445,
      "activations/layer1_attention_weight_max": 16.18198585510254,
      "activations/layer1_attention_weight_min": -14.621561050415039,
      "activations/layer20_attention_weight_max": 25.310546875,
      "activations/layer20_attention_weight_min": -21.629535675048828,
      "activations/layer21_attention_weight_max": 46.75198745727539,
      "activations/layer21_attention_weight_min": -23.66261863708496,
      "activations/layer22_attention_weight_max": 31.179019927978516,
      "activations/layer22_attention_weight_min": -26.80777931213379,
      "activations/layer23_attention_weight_max": 39.485504150390625,
      "activations/layer23_attention_weight_min": -26.114368438720703,
      "activations/layer2_attention_weight_max": 34.145843505859375,
      "activations/layer2_attention_weight_min": -31.687458038330078,
      "activations/layer3_attention_weight_max": 92.67198181152344,
      "activations/layer3_attention_weight_min": -93.94149780273438,
      "activations/layer4_attention_weight_max": 91.75406646728516,
      "activations/layer4_attention_weight_min": -89.64774322509766,
      "activations/layer5_attention_weight_max": 67.79450988769531,
      "activations/layer5_attention_weight_min": -80.80995178222656,
      "activations/layer6_attention_weight_max": 51.072940826416016,
      "activations/layer6_attention_weight_min": -51.7790412902832,
      "activations/layer7_attention_weight_max": 67.47705841064453,
      "activations/layer7_attention_weight_min": -63.32004165649414,
      "activations/layer8_attention_weight_max": 46.15873336791992,
      "activations/layer8_attention_weight_min": -47.54097366333008,
      "activations/layer9_attention_weight_max": 48.12200927734375,
      "activations/layer9_attention_weight_min": -45.21951675415039,
      "epoch": 14.68,
      "learning_rate": 5.587462121212121e-05,
      "loss": 2.7613,
      "step": 252700
    },
    {
      "activations/layer0_attention_weight_max": 15.65172004699707,
      "activations/layer0_attention_weight_min": -13.449374198913574,
      "activations/layer10_attention_weight_max": 33.78801345825195,
      "activations/layer10_attention_weight_min": -33.381587982177734,
      "activations/layer11_attention_weight_max": 31.35832977294922,
      "activations/layer11_attention_weight_min": -31.539342880249023,
      "activations/layer12_attention_weight_max": 16.7402400970459,
      "activations/layer12_attention_weight_min": -23.75136947631836,
      "activations/layer13_attention_weight_max": 32.74483871459961,
      "activations/layer13_attention_weight_min": -25.21501350402832,
      "activations/layer14_attention_weight_max": 34.65975570678711,
      "activations/layer14_attention_weight_min": -28.51953887939453,
      "activations/layer15_attention_weight_max": 31.034719467163086,
      "activations/layer15_attention_weight_min": -27.599773406982422,
      "activations/layer16_attention_weight_max": 30.215612411499023,
      "activations/layer16_attention_weight_min": -30.2899169921875,
      "activations/layer17_attention_weight_max": 46.91835021972656,
      "activations/layer17_attention_weight_min": -41.2822265625,
      "activations/layer18_attention_weight_max": 42.28953552246094,
      "activations/layer18_attention_weight_min": -35.12495803833008,
      "activations/layer19_attention_weight_max": 21.8509578704834,
      "activations/layer19_attention_weight_min": -22.378658294677734,
      "activations/layer1_attention_weight_max": 16.26624298095703,
      "activations/layer1_attention_weight_min": -14.062766075134277,
      "activations/layer20_attention_weight_max": 21.123689651489258,
      "activations/layer20_attention_weight_min": -21.614505767822266,
      "activations/layer21_attention_weight_max": 31.727371215820312,
      "activations/layer21_attention_weight_min": -20.45309829711914,
      "activations/layer22_attention_weight_max": 31.000057220458984,
      "activations/layer22_attention_weight_min": -25.020160675048828,
      "activations/layer23_attention_weight_max": 35.79753112792969,
      "activations/layer23_attention_weight_min": -24.047544479370117,
      "activations/layer2_attention_weight_max": 32.72510528564453,
      "activations/layer2_attention_weight_min": -30.817134857177734,
      "activations/layer3_attention_weight_max": 91.98184204101562,
      "activations/layer3_attention_weight_min": -95.61283874511719,
      "activations/layer4_attention_weight_max": 89.8637924194336,
      "activations/layer4_attention_weight_min": -88.92842864990234,
      "activations/layer5_attention_weight_max": 66.7966079711914,
      "activations/layer5_attention_weight_min": -78.76427459716797,
      "activations/layer6_attention_weight_max": 50.03940963745117,
      "activations/layer6_attention_weight_min": -51.87825012207031,
      "activations/layer7_attention_weight_max": 62.40655517578125,
      "activations/layer7_attention_weight_min": -61.570072174072266,
      "activations/layer8_attention_weight_max": 44.373382568359375,
      "activations/layer8_attention_weight_min": -47.576759338378906,
      "activations/layer9_attention_weight_max": 43.26812744140625,
      "activations/layer9_attention_weight_min": -44.38534164428711,
      "epoch": 14.69,
      "learning_rate": 5.585568181818181e-05,
      "loss": 2.7607,
      "step": 252750
    },
    {
      "activations/layer0_attention_weight_max": 15.542356491088867,
      "activations/layer0_attention_weight_min": -14.204105377197266,
      "activations/layer10_attention_weight_max": 37.824974060058594,
      "activations/layer10_attention_weight_min": -38.041358947753906,
      "activations/layer11_attention_weight_max": 35.187904357910156,
      "activations/layer11_attention_weight_min": -35.31139373779297,
      "activations/layer12_attention_weight_max": 20.208908081054688,
      "activations/layer12_attention_weight_min": -28.16497039794922,
      "activations/layer13_attention_weight_max": 40.01787567138672,
      "activations/layer13_attention_weight_min": -28.915233612060547,
      "activations/layer14_attention_weight_max": 36.48590087890625,
      "activations/layer14_attention_weight_min": -29.84108543395996,
      "activations/layer15_attention_weight_max": 33.76906204223633,
      "activations/layer15_attention_weight_min": -30.74989128112793,
      "activations/layer16_attention_weight_max": 33.7262077331543,
      "activations/layer16_attention_weight_min": -30.73453712463379,
      "activations/layer17_attention_weight_max": 51.7597541809082,
      "activations/layer17_attention_weight_min": -47.6820068359375,
      "activations/layer18_attention_weight_max": 50.264808654785156,
      "activations/layer18_attention_weight_min": -40.34113311767578,
      "activations/layer19_attention_weight_max": 24.22076988220215,
      "activations/layer19_attention_weight_min": -24.8948974609375,
      "activations/layer1_attention_weight_max": 16.2753849029541,
      "activations/layer1_attention_weight_min": -14.242262840270996,
      "activations/layer20_attention_weight_max": 24.51714324951172,
      "activations/layer20_attention_weight_min": -23.278356552124023,
      "activations/layer21_attention_weight_max": 39.45199203491211,
      "activations/layer21_attention_weight_min": -24.164566040039062,
      "activations/layer22_attention_weight_max": 31.80415153503418,
      "activations/layer22_attention_weight_min": -27.53786277770996,
      "activations/layer23_attention_weight_max": 38.824398040771484,
      "activations/layer23_attention_weight_min": -24.838911056518555,
      "activations/layer2_attention_weight_max": 31.94363021850586,
      "activations/layer2_attention_weight_min": -33.063079833984375,
      "activations/layer3_attention_weight_max": 89.66148376464844,
      "activations/layer3_attention_weight_min": -98.8727035522461,
      "activations/layer4_attention_weight_max": 89.52335357666016,
      "activations/layer4_attention_weight_min": -89.92536926269531,
      "activations/layer5_attention_weight_max": 68.56511688232422,
      "activations/layer5_attention_weight_min": -75.55924224853516,
      "activations/layer6_attention_weight_max": 50.495357513427734,
      "activations/layer6_attention_weight_min": -55.387542724609375,
      "activations/layer7_attention_weight_max": 63.688533782958984,
      "activations/layer7_attention_weight_min": -68.13414764404297,
      "activations/layer8_attention_weight_max": 47.4981689453125,
      "activations/layer8_attention_weight_min": -54.1005859375,
      "activations/layer9_attention_weight_max": 50.56752395629883,
      "activations/layer9_attention_weight_min": -49.63673782348633,
      "epoch": 14.69,
      "learning_rate": 5.583674242424242e-05,
      "loss": 2.7554,
      "step": 252800
    },
    {
      "activations/layer0_attention_weight_max": 16.19672393798828,
      "activations/layer0_attention_weight_min": -13.342011451721191,
      "activations/layer10_attention_weight_max": 34.32339859008789,
      "activations/layer10_attention_weight_min": -33.71678924560547,
      "activations/layer11_attention_weight_max": 30.129024505615234,
      "activations/layer11_attention_weight_min": -33.471099853515625,
      "activations/layer12_attention_weight_max": 16.892391204833984,
      "activations/layer12_attention_weight_min": -25.83393096923828,
      "activations/layer13_attention_weight_max": 32.98748779296875,
      "activations/layer13_attention_weight_min": -26.48369598388672,
      "activations/layer14_attention_weight_max": 33.77346420288086,
      "activations/layer14_attention_weight_min": -29.356143951416016,
      "activations/layer15_attention_weight_max": 31.10577964782715,
      "activations/layer15_attention_weight_min": -28.70530128479004,
      "activations/layer16_attention_weight_max": 31.427486419677734,
      "activations/layer16_attention_weight_min": -30.11310386657715,
      "activations/layer17_attention_weight_max": 50.72979736328125,
      "activations/layer17_attention_weight_min": -43.274818420410156,
      "activations/layer18_attention_weight_max": 45.57770538330078,
      "activations/layer18_attention_weight_min": -40.40453338623047,
      "activations/layer19_attention_weight_max": 22.926166534423828,
      "activations/layer19_attention_weight_min": -21.658658981323242,
      "activations/layer1_attention_weight_max": 15.778634071350098,
      "activations/layer1_attention_weight_min": -14.911110877990723,
      "activations/layer20_attention_weight_max": 22.240447998046875,
      "activations/layer20_attention_weight_min": -21.824432373046875,
      "activations/layer21_attention_weight_max": 34.35573959350586,
      "activations/layer21_attention_weight_min": -24.220115661621094,
      "activations/layer22_attention_weight_max": 31.055688858032227,
      "activations/layer22_attention_weight_min": -26.428037643432617,
      "activations/layer23_attention_weight_max": 36.299949645996094,
      "activations/layer23_attention_weight_min": -29.151573181152344,
      "activations/layer2_attention_weight_max": 32.18994903564453,
      "activations/layer2_attention_weight_min": -30.41785430908203,
      "activations/layer3_attention_weight_max": 91.97793579101562,
      "activations/layer3_attention_weight_min": -90.44217681884766,
      "activations/layer4_attention_weight_max": 86.7016830444336,
      "activations/layer4_attention_weight_min": -83.71602630615234,
      "activations/layer5_attention_weight_max": 70.22281646728516,
      "activations/layer5_attention_weight_min": -77.97833251953125,
      "activations/layer6_attention_weight_max": 50.38867950439453,
      "activations/layer6_attention_weight_min": -50.25984191894531,
      "activations/layer7_attention_weight_max": 64.00948333740234,
      "activations/layer7_attention_weight_min": -64.15130615234375,
      "activations/layer8_attention_weight_max": 45.49699783325195,
      "activations/layer8_attention_weight_min": -49.482723236083984,
      "activations/layer9_attention_weight_max": 44.252376556396484,
      "activations/layer9_attention_weight_min": -43.57401657104492,
      "epoch": 14.69,
      "learning_rate": 5.5817803030303025e-05,
      "loss": 2.753,
      "step": 252850
    },
    {
      "activations/layer0_attention_weight_max": 15.398826599121094,
      "activations/layer0_attention_weight_min": -12.855058670043945,
      "activations/layer10_attention_weight_max": 36.3532600402832,
      "activations/layer10_attention_weight_min": -37.209320068359375,
      "activations/layer11_attention_weight_max": 31.591724395751953,
      "activations/layer11_attention_weight_min": -34.324546813964844,
      "activations/layer12_attention_weight_max": 17.897138595581055,
      "activations/layer12_attention_weight_min": -34.6783332824707,
      "activations/layer13_attention_weight_max": 35.77001953125,
      "activations/layer13_attention_weight_min": -28.676898956298828,
      "activations/layer14_attention_weight_max": 33.27583694458008,
      "activations/layer14_attention_weight_min": -30.4731388092041,
      "activations/layer15_attention_weight_max": 30.661895751953125,
      "activations/layer15_attention_weight_min": -29.208051681518555,
      "activations/layer16_attention_weight_max": 30.824111938476562,
      "activations/layer16_attention_weight_min": -30.12591552734375,
      "activations/layer17_attention_weight_max": 51.31476974487305,
      "activations/layer17_attention_weight_min": -46.170143127441406,
      "activations/layer18_attention_weight_max": 45.11669921875,
      "activations/layer18_attention_weight_min": -39.966827392578125,
      "activations/layer19_attention_weight_max": 21.796064376831055,
      "activations/layer19_attention_weight_min": -23.604650497436523,
      "activations/layer1_attention_weight_max": 17.132572174072266,
      "activations/layer1_attention_weight_min": -14.365518569946289,
      "activations/layer20_attention_weight_max": 22.93937110900879,
      "activations/layer20_attention_weight_min": -21.367229461669922,
      "activations/layer21_attention_weight_max": 36.93605041503906,
      "activations/layer21_attention_weight_min": -25.731626510620117,
      "activations/layer22_attention_weight_max": 27.63347053527832,
      "activations/layer22_attention_weight_min": -27.47553253173828,
      "activations/layer23_attention_weight_max": 34.51384353637695,
      "activations/layer23_attention_weight_min": -27.92950439453125,
      "activations/layer2_attention_weight_max": 32.06085968017578,
      "activations/layer2_attention_weight_min": -30.416606903076172,
      "activations/layer3_attention_weight_max": 89.63546752929688,
      "activations/layer3_attention_weight_min": -90.64289093017578,
      "activations/layer4_attention_weight_max": 86.62129974365234,
      "activations/layer4_attention_weight_min": -84.09044647216797,
      "activations/layer5_attention_weight_max": 66.1659164428711,
      "activations/layer5_attention_weight_min": -77.41949462890625,
      "activations/layer6_attention_weight_max": 51.92885971069336,
      "activations/layer6_attention_weight_min": -51.27005386352539,
      "activations/layer7_attention_weight_max": 68.47740173339844,
      "activations/layer7_attention_weight_min": -63.09584045410156,
      "activations/layer8_attention_weight_max": 45.632667541503906,
      "activations/layer8_attention_weight_min": -53.06911849975586,
      "activations/layer9_attention_weight_max": 51.06211853027344,
      "activations/layer9_attention_weight_min": -46.923580169677734,
      "epoch": 14.69,
      "learning_rate": 5.579886363636363e-05,
      "loss": 2.7613,
      "step": 252900
    },
    {
      "activations/layer0_attention_weight_max": 15.59254264831543,
      "activations/layer0_attention_weight_min": -13.215873718261719,
      "activations/layer10_attention_weight_max": 35.07878494262695,
      "activations/layer10_attention_weight_min": -35.17371368408203,
      "activations/layer11_attention_weight_max": 33.2108154296875,
      "activations/layer11_attention_weight_min": -34.752708435058594,
      "activations/layer12_attention_weight_max": 20.67949104309082,
      "activations/layer12_attention_weight_min": -25.79977035522461,
      "activations/layer13_attention_weight_max": 38.802886962890625,
      "activations/layer13_attention_weight_min": -27.876785278320312,
      "activations/layer14_attention_weight_max": 36.75619125366211,
      "activations/layer14_attention_weight_min": -29.543230056762695,
      "activations/layer15_attention_weight_max": 32.31010818481445,
      "activations/layer15_attention_weight_min": -29.50848388671875,
      "activations/layer16_attention_weight_max": 34.00859069824219,
      "activations/layer16_attention_weight_min": -31.62001609802246,
      "activations/layer17_attention_weight_max": 53.574256896972656,
      "activations/layer17_attention_weight_min": -46.841304779052734,
      "activations/layer18_attention_weight_max": 50.881675720214844,
      "activations/layer18_attention_weight_min": -40.552886962890625,
      "activations/layer19_attention_weight_max": 23.870561599731445,
      "activations/layer19_attention_weight_min": -23.078815460205078,
      "activations/layer1_attention_weight_max": 16.449445724487305,
      "activations/layer1_attention_weight_min": -14.456598281860352,
      "activations/layer20_attention_weight_max": 21.443008422851562,
      "activations/layer20_attention_weight_min": -20.834108352661133,
      "activations/layer21_attention_weight_max": 34.331756591796875,
      "activations/layer21_attention_weight_min": -23.332706451416016,
      "activations/layer22_attention_weight_max": 28.586763381958008,
      "activations/layer22_attention_weight_min": -26.530162811279297,
      "activations/layer23_attention_weight_max": 35.59772491455078,
      "activations/layer23_attention_weight_min": -22.985258102416992,
      "activations/layer2_attention_weight_max": 31.169952392578125,
      "activations/layer2_attention_weight_min": -30.908004760742188,
      "activations/layer3_attention_weight_max": 90.34918975830078,
      "activations/layer3_attention_weight_min": -89.67329406738281,
      "activations/layer4_attention_weight_max": 87.18582153320312,
      "activations/layer4_attention_weight_min": -84.63388061523438,
      "activations/layer5_attention_weight_max": 65.39379119873047,
      "activations/layer5_attention_weight_min": -75.5965576171875,
      "activations/layer6_attention_weight_max": 49.50560760498047,
      "activations/layer6_attention_weight_min": -52.50267791748047,
      "activations/layer7_attention_weight_max": 62.07722854614258,
      "activations/layer7_attention_weight_min": -61.90380096435547,
      "activations/layer8_attention_weight_max": 44.81449508666992,
      "activations/layer8_attention_weight_min": -49.40299606323242,
      "activations/layer9_attention_weight_max": 44.53877639770508,
      "activations/layer9_attention_weight_min": -48.35485076904297,
      "epoch": 14.7,
      "learning_rate": 5.5779924242424235e-05,
      "loss": 2.7672,
      "step": 252950
    },
    {
      "activations/layer0_attention_weight_max": 15.418655395507812,
      "activations/layer0_attention_weight_min": -13.49251937866211,
      "activations/layer10_attention_weight_max": 40.097835540771484,
      "activations/layer10_attention_weight_min": -40.51905059814453,
      "activations/layer11_attention_weight_max": 36.923954010009766,
      "activations/layer11_attention_weight_min": -36.159664154052734,
      "activations/layer12_attention_weight_max": 20.925312042236328,
      "activations/layer12_attention_weight_min": -29.466846466064453,
      "activations/layer13_attention_weight_max": 36.56537628173828,
      "activations/layer13_attention_weight_min": -27.67061424255371,
      "activations/layer14_attention_weight_max": 34.8942985534668,
      "activations/layer14_attention_weight_min": -29.541414260864258,
      "activations/layer15_attention_weight_max": 32.677589416503906,
      "activations/layer15_attention_weight_min": -28.51160430908203,
      "activations/layer16_attention_weight_max": 31.044694900512695,
      "activations/layer16_attention_weight_min": -30.661670684814453,
      "activations/layer17_attention_weight_max": 49.6511116027832,
      "activations/layer17_attention_weight_min": -46.017032623291016,
      "activations/layer18_attention_weight_max": 43.833003997802734,
      "activations/layer18_attention_weight_min": -41.11225128173828,
      "activations/layer19_attention_weight_max": 23.296411514282227,
      "activations/layer19_attention_weight_min": -21.52354621887207,
      "activations/layer1_attention_weight_max": 16.587980270385742,
      "activations/layer1_attention_weight_min": -14.36170768737793,
      "activations/layer20_attention_weight_max": 24.623939514160156,
      "activations/layer20_attention_weight_min": -23.18610382080078,
      "activations/layer21_attention_weight_max": 32.65095138549805,
      "activations/layer21_attention_weight_min": -20.64719581604004,
      "activations/layer22_attention_weight_max": 28.233131408691406,
      "activations/layer22_attention_weight_min": -25.319381713867188,
      "activations/layer23_attention_weight_max": 38.157859802246094,
      "activations/layer23_attention_weight_min": -25.102262496948242,
      "activations/layer2_attention_weight_max": 33.39480972290039,
      "activations/layer2_attention_weight_min": -31.39572525024414,
      "activations/layer3_attention_weight_max": 99.31175994873047,
      "activations/layer3_attention_weight_min": -93.77740478515625,
      "activations/layer4_attention_weight_max": 94.06270599365234,
      "activations/layer4_attention_weight_min": -90.86305236816406,
      "activations/layer5_attention_weight_max": 70.63372039794922,
      "activations/layer5_attention_weight_min": -84.97218322753906,
      "activations/layer6_attention_weight_max": 56.15760040283203,
      "activations/layer6_attention_weight_min": -55.34212112426758,
      "activations/layer7_attention_weight_max": 66.9957275390625,
      "activations/layer7_attention_weight_min": -70.58113098144531,
      "activations/layer8_attention_weight_max": 50.62263870239258,
      "activations/layer8_attention_weight_min": -52.445594787597656,
      "activations/layer9_attention_weight_max": 50.14609909057617,
      "activations/layer9_attention_weight_min": -52.75712966918945,
      "epoch": 14.7,
      "learning_rate": 5.576098484848485e-05,
      "loss": 2.7651,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4417,
      "eval_samples_per_second": 508.663,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4417,
      "eval_openwebtext_samples_per_second": 508.663,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9278,
      "eval_wikitext_samples_per_second": 236.541,
      "step": 253000
    },
    {
      "epoch": 14.7,
      "eval_lambada_loss": 2.51171875,
      "eval_lambada_ppl": 12.326097344475713,
      "eval_lambada_runtime": 9.5013,
      "eval_lambada_samples_per_second": 512.455,
      "step": 253000
    },
    {
      "activations/layer0_attention_weight_max": 15.623003959655762,
      "activations/layer0_attention_weight_min": -13.502758979797363,
      "activations/layer10_attention_weight_max": 38.86749267578125,
      "activations/layer10_attention_weight_min": -37.96007537841797,
      "activations/layer11_attention_weight_max": 37.665767669677734,
      "activations/layer11_attention_weight_min": -38.2109375,
      "activations/layer12_attention_weight_max": 20.95083999633789,
      "activations/layer12_attention_weight_min": -22.39665985107422,
      "activations/layer13_attention_weight_max": 40.631622314453125,
      "activations/layer13_attention_weight_min": -31.189607620239258,
      "activations/layer14_attention_weight_max": 40.75957489013672,
      "activations/layer14_attention_weight_min": -32.7124137878418,
      "activations/layer15_attention_weight_max": 37.23078155517578,
      "activations/layer15_attention_weight_min": -31.968994140625,
      "activations/layer16_attention_weight_max": 37.94907760620117,
      "activations/layer16_attention_weight_min": -33.71393966674805,
      "activations/layer17_attention_weight_max": 57.45199966430664,
      "activations/layer17_attention_weight_min": -50.2847900390625,
      "activations/layer18_attention_weight_max": 56.76585006713867,
      "activations/layer18_attention_weight_min": -43.10533142089844,
      "activations/layer19_attention_weight_max": 26.994680404663086,
      "activations/layer19_attention_weight_min": -24.084461212158203,
      "activations/layer1_attention_weight_max": 16.079952239990234,
      "activations/layer1_attention_weight_min": -14.449311256408691,
      "activations/layer20_attention_weight_max": 27.820348739624023,
      "activations/layer20_attention_weight_min": -20.263771057128906,
      "activations/layer21_attention_weight_max": 46.1596794128418,
      "activations/layer21_attention_weight_min": -23.998571395874023,
      "activations/layer22_attention_weight_max": 32.228858947753906,
      "activations/layer22_attention_weight_min": -26.303714752197266,
      "activations/layer23_attention_weight_max": 44.676815032958984,
      "activations/layer23_attention_weight_min": -22.96315574645996,
      "activations/layer2_attention_weight_max": 30.413755416870117,
      "activations/layer2_attention_weight_min": -30.333227157592773,
      "activations/layer3_attention_weight_max": 88.8290023803711,
      "activations/layer3_attention_weight_min": -89.35176849365234,
      "activations/layer4_attention_weight_max": 86.66500091552734,
      "activations/layer4_attention_weight_min": -85.60646057128906,
      "activations/layer5_attention_weight_max": 68.51618957519531,
      "activations/layer5_attention_weight_min": -77.60223388671875,
      "activations/layer6_attention_weight_max": 50.15333557128906,
      "activations/layer6_attention_weight_min": -51.24783706665039,
      "activations/layer7_attention_weight_max": 62.20524597167969,
      "activations/layer7_attention_weight_min": -66.60295104980469,
      "activations/layer8_attention_weight_max": 48.559425354003906,
      "activations/layer8_attention_weight_min": -52.01648712158203,
      "activations/layer9_attention_weight_max": 46.51321029663086,
      "activations/layer9_attention_weight_min": -49.209232330322266,
      "epoch": 14.7,
      "learning_rate": 5.574204545454545e-05,
      "loss": 2.7483,
      "step": 253050
    },
    {
      "activations/layer0_attention_weight_max": 15.196134567260742,
      "activations/layer0_attention_weight_min": -13.400891304016113,
      "activations/layer10_attention_weight_max": 37.031890869140625,
      "activations/layer10_attention_weight_min": -35.34355926513672,
      "activations/layer11_attention_weight_max": 32.00949478149414,
      "activations/layer11_attention_weight_min": -33.115074157714844,
      "activations/layer12_attention_weight_max": 22.43846321105957,
      "activations/layer12_attention_weight_min": -22.81824493408203,
      "activations/layer13_attention_weight_max": 37.19392395019531,
      "activations/layer13_attention_weight_min": -29.962175369262695,
      "activations/layer14_attention_weight_max": 37.00574493408203,
      "activations/layer14_attention_weight_min": -30.93407440185547,
      "activations/layer15_attention_weight_max": 34.021881103515625,
      "activations/layer15_attention_weight_min": -32.19464111328125,
      "activations/layer16_attention_weight_max": 35.71091079711914,
      "activations/layer16_attention_weight_min": -32.38655090332031,
      "activations/layer17_attention_weight_max": 58.67304992675781,
      "activations/layer17_attention_weight_min": -52.21390914916992,
      "activations/layer18_attention_weight_max": 53.68622970581055,
      "activations/layer18_attention_weight_min": -42.37548065185547,
      "activations/layer19_attention_weight_max": 23.191373825073242,
      "activations/layer19_attention_weight_min": -22.214187622070312,
      "activations/layer1_attention_weight_max": 16.813688278198242,
      "activations/layer1_attention_weight_min": -14.674365997314453,
      "activations/layer20_attention_weight_max": 25.18780517578125,
      "activations/layer20_attention_weight_min": -20.25878143310547,
      "activations/layer21_attention_weight_max": 35.32575607299805,
      "activations/layer21_attention_weight_min": -22.33931541442871,
      "activations/layer22_attention_weight_max": 31.949569702148438,
      "activations/layer22_attention_weight_min": -24.96026611328125,
      "activations/layer23_attention_weight_max": 41.724754333496094,
      "activations/layer23_attention_weight_min": -26.74195098876953,
      "activations/layer2_attention_weight_max": 31.411312103271484,
      "activations/layer2_attention_weight_min": -30.930152893066406,
      "activations/layer3_attention_weight_max": 88.4417724609375,
      "activations/layer3_attention_weight_min": -90.82991790771484,
      "activations/layer4_attention_weight_max": 86.68879699707031,
      "activations/layer4_attention_weight_min": -86.9417495727539,
      "activations/layer5_attention_weight_max": 67.559814453125,
      "activations/layer5_attention_weight_min": -76.91383361816406,
      "activations/layer6_attention_weight_max": 49.994686126708984,
      "activations/layer6_attention_weight_min": -50.332984924316406,
      "activations/layer7_attention_weight_max": 61.39640426635742,
      "activations/layer7_attention_weight_min": -63.511783599853516,
      "activations/layer8_attention_weight_max": 44.43934631347656,
      "activations/layer8_attention_weight_min": -49.49931716918945,
      "activations/layer9_attention_weight_max": 47.102088928222656,
      "activations/layer9_attention_weight_min": -45.11957931518555,
      "epoch": 14.71,
      "learning_rate": 5.572310606060605e-05,
      "loss": 2.7659,
      "step": 253100
    },
    {
      "activations/layer0_attention_weight_max": 15.17751693725586,
      "activations/layer0_attention_weight_min": -13.421418190002441,
      "activations/layer10_attention_weight_max": 35.79636001586914,
      "activations/layer10_attention_weight_min": -35.77884292602539,
      "activations/layer11_attention_weight_max": 33.32965087890625,
      "activations/layer11_attention_weight_min": -34.40459442138672,
      "activations/layer12_attention_weight_max": 17.7232666015625,
      "activations/layer12_attention_weight_min": -27.12224578857422,
      "activations/layer13_attention_weight_max": 38.63493728637695,
      "activations/layer13_attention_weight_min": -29.848928451538086,
      "activations/layer14_attention_weight_max": 38.10812759399414,
      "activations/layer14_attention_weight_min": -30.179250717163086,
      "activations/layer15_attention_weight_max": 34.7037239074707,
      "activations/layer15_attention_weight_min": -30.046369552612305,
      "activations/layer16_attention_weight_max": 34.72809982299805,
      "activations/layer16_attention_weight_min": -32.080078125,
      "activations/layer17_attention_weight_max": 53.38657760620117,
      "activations/layer17_attention_weight_min": -45.70629119873047,
      "activations/layer18_attention_weight_max": 48.707305908203125,
      "activations/layer18_attention_weight_min": -38.12289047241211,
      "activations/layer19_attention_weight_max": 24.042421340942383,
      "activations/layer19_attention_weight_min": -22.194364547729492,
      "activations/layer1_attention_weight_max": 17.07317543029785,
      "activations/layer1_attention_weight_min": -14.213494300842285,
      "activations/layer20_attention_weight_max": 24.7863712310791,
      "activations/layer20_attention_weight_min": -22.26786994934082,
      "activations/layer21_attention_weight_max": 40.90926742553711,
      "activations/layer21_attention_weight_min": -21.710315704345703,
      "activations/layer22_attention_weight_max": 31.55260467529297,
      "activations/layer22_attention_weight_min": -24.89013671875,
      "activations/layer23_attention_weight_max": 47.6721305847168,
      "activations/layer23_attention_weight_min": -24.054706573486328,
      "activations/layer2_attention_weight_max": 30.775772094726562,
      "activations/layer2_attention_weight_min": -31.851882934570312,
      "activations/layer3_attention_weight_max": 90.37438201904297,
      "activations/layer3_attention_weight_min": -90.82412719726562,
      "activations/layer4_attention_weight_max": 89.18923950195312,
      "activations/layer4_attention_weight_min": -86.93556213378906,
      "activations/layer5_attention_weight_max": 65.5770492553711,
      "activations/layer5_attention_weight_min": -76.33463287353516,
      "activations/layer6_attention_weight_max": 50.67582702636719,
      "activations/layer6_attention_weight_min": -50.23598098754883,
      "activations/layer7_attention_weight_max": 64.31062316894531,
      "activations/layer7_attention_weight_min": -64.82205963134766,
      "activations/layer8_attention_weight_max": 45.107688903808594,
      "activations/layer8_attention_weight_min": -47.912574768066406,
      "activations/layer9_attention_weight_max": 45.235836029052734,
      "activations/layer9_attention_weight_min": -47.20066452026367,
      "epoch": 14.71,
      "learning_rate": 5.570416666666666e-05,
      "loss": 2.7545,
      "step": 253150
    },
    {
      "activations/layer0_attention_weight_max": 15.02309799194336,
      "activations/layer0_attention_weight_min": -13.53000545501709,
      "activations/layer10_attention_weight_max": 37.00518798828125,
      "activations/layer10_attention_weight_min": -36.095375061035156,
      "activations/layer11_attention_weight_max": 34.220394134521484,
      "activations/layer11_attention_weight_min": -34.998416900634766,
      "activations/layer12_attention_weight_max": 19.30515480041504,
      "activations/layer12_attention_weight_min": -26.226572036743164,
      "activations/layer13_attention_weight_max": 33.832115173339844,
      "activations/layer13_attention_weight_min": -27.70029640197754,
      "activations/layer14_attention_weight_max": 35.61842346191406,
      "activations/layer14_attention_weight_min": -32.04880142211914,
      "activations/layer15_attention_weight_max": 32.947235107421875,
      "activations/layer15_attention_weight_min": -30.535139083862305,
      "activations/layer16_attention_weight_max": 34.68049240112305,
      "activations/layer16_attention_weight_min": -33.63750076293945,
      "activations/layer17_attention_weight_max": 52.378089904785156,
      "activations/layer17_attention_weight_min": -47.996826171875,
      "activations/layer18_attention_weight_max": 46.147560119628906,
      "activations/layer18_attention_weight_min": -39.356693267822266,
      "activations/layer19_attention_weight_max": 21.671375274658203,
      "activations/layer19_attention_weight_min": -21.188255310058594,
      "activations/layer1_attention_weight_max": 16.14057159423828,
      "activations/layer1_attention_weight_min": -14.489357948303223,
      "activations/layer20_attention_weight_max": 22.292844772338867,
      "activations/layer20_attention_weight_min": -20.846105575561523,
      "activations/layer21_attention_weight_max": 39.21419143676758,
      "activations/layer21_attention_weight_min": -22.188739776611328,
      "activations/layer22_attention_weight_max": 30.956188201904297,
      "activations/layer22_attention_weight_min": -26.3867130279541,
      "activations/layer23_attention_weight_max": 34.639862060546875,
      "activations/layer23_attention_weight_min": -25.09683609008789,
      "activations/layer2_attention_weight_max": 33.49531555175781,
      "activations/layer2_attention_weight_min": -31.05695915222168,
      "activations/layer3_attention_weight_max": 97.36003112792969,
      "activations/layer3_attention_weight_min": -93.68353271484375,
      "activations/layer4_attention_weight_max": 95.17162322998047,
      "activations/layer4_attention_weight_min": -91.98009490966797,
      "activations/layer5_attention_weight_max": 69.13754272460938,
      "activations/layer5_attention_weight_min": -75.88014221191406,
      "activations/layer6_attention_weight_max": 52.607505798339844,
      "activations/layer6_attention_weight_min": -56.085411071777344,
      "activations/layer7_attention_weight_max": 65.21894073486328,
      "activations/layer7_attention_weight_min": -65.19298553466797,
      "activations/layer8_attention_weight_max": 50.38221740722656,
      "activations/layer8_attention_weight_min": -51.32211685180664,
      "activations/layer9_attention_weight_max": 46.18952178955078,
      "activations/layer9_attention_weight_min": -49.70313262939453,
      "epoch": 14.71,
      "learning_rate": 5.568522727272727e-05,
      "loss": 2.7657,
      "step": 253200
    },
    {
      "activations/layer0_attention_weight_max": 16.44867706298828,
      "activations/layer0_attention_weight_min": -13.350974082946777,
      "activations/layer10_attention_weight_max": 37.77345657348633,
      "activations/layer10_attention_weight_min": -35.81850814819336,
      "activations/layer11_attention_weight_max": 33.14342498779297,
      "activations/layer11_attention_weight_min": -33.15444564819336,
      "activations/layer12_attention_weight_max": 19.050491333007812,
      "activations/layer12_attention_weight_min": -26.96674919128418,
      "activations/layer13_attention_weight_max": 38.068572998046875,
      "activations/layer13_attention_weight_min": -31.47393798828125,
      "activations/layer14_attention_weight_max": 34.73994445800781,
      "activations/layer14_attention_weight_min": -29.38932228088379,
      "activations/layer15_attention_weight_max": 31.58025360107422,
      "activations/layer15_attention_weight_min": -28.85725975036621,
      "activations/layer16_attention_weight_max": 31.914636611938477,
      "activations/layer16_attention_weight_min": -30.793758392333984,
      "activations/layer17_attention_weight_max": 57.7862434387207,
      "activations/layer17_attention_weight_min": -46.59408187866211,
      "activations/layer18_attention_weight_max": 45.43533706665039,
      "activations/layer18_attention_weight_min": -37.88751220703125,
      "activations/layer19_attention_weight_max": 22.600788116455078,
      "activations/layer19_attention_weight_min": -22.1854190826416,
      "activations/layer1_attention_weight_max": 15.525529861450195,
      "activations/layer1_attention_weight_min": -15.131475448608398,
      "activations/layer20_attention_weight_max": 22.784940719604492,
      "activations/layer20_attention_weight_min": -22.027280807495117,
      "activations/layer21_attention_weight_max": 34.27285385131836,
      "activations/layer21_attention_weight_min": -22.396373748779297,
      "activations/layer22_attention_weight_max": 33.52288818359375,
      "activations/layer22_attention_weight_min": -25.488054275512695,
      "activations/layer23_attention_weight_max": 34.73588943481445,
      "activations/layer23_attention_weight_min": -24.145437240600586,
      "activations/layer2_attention_weight_max": 32.908355712890625,
      "activations/layer2_attention_weight_min": -33.767494201660156,
      "activations/layer3_attention_weight_max": 96.7679443359375,
      "activations/layer3_attention_weight_min": -103.51201629638672,
      "activations/layer4_attention_weight_max": 95.94537353515625,
      "activations/layer4_attention_weight_min": -92.49653625488281,
      "activations/layer5_attention_weight_max": 67.86093139648438,
      "activations/layer5_attention_weight_min": -77.14350891113281,
      "activations/layer6_attention_weight_max": 52.61172103881836,
      "activations/layer6_attention_weight_min": -51.511756896972656,
      "activations/layer7_attention_weight_max": 62.74734878540039,
      "activations/layer7_attention_weight_min": -66.94699096679688,
      "activations/layer8_attention_weight_max": 46.163265228271484,
      "activations/layer8_attention_weight_min": -51.51057052612305,
      "activations/layer9_attention_weight_max": 45.79483413696289,
      "activations/layer9_attention_weight_min": -47.21488571166992,
      "epoch": 14.72,
      "learning_rate": 5.566628787878788e-05,
      "loss": 2.7507,
      "step": 253250
    },
    {
      "activations/layer0_attention_weight_max": 15.479756355285645,
      "activations/layer0_attention_weight_min": -13.171919822692871,
      "activations/layer10_attention_weight_max": 37.014041900634766,
      "activations/layer10_attention_weight_min": -35.975730895996094,
      "activations/layer11_attention_weight_max": 31.95882797241211,
      "activations/layer11_attention_weight_min": -32.94725799560547,
      "activations/layer12_attention_weight_max": 17.917999267578125,
      "activations/layer12_attention_weight_min": -23.771507263183594,
      "activations/layer13_attention_weight_max": 37.591285705566406,
      "activations/layer13_attention_weight_min": -29.421680450439453,
      "activations/layer14_attention_weight_max": 35.30593490600586,
      "activations/layer14_attention_weight_min": -30.334285736083984,
      "activations/layer15_attention_weight_max": 33.07753372192383,
      "activations/layer15_attention_weight_min": -30.79540252685547,
      "activations/layer16_attention_weight_max": 32.55817794799805,
      "activations/layer16_attention_weight_min": -31.662906646728516,
      "activations/layer17_attention_weight_max": 53.708126068115234,
      "activations/layer17_attention_weight_min": -45.08673095703125,
      "activations/layer18_attention_weight_max": 46.74348831176758,
      "activations/layer18_attention_weight_min": -37.4499626159668,
      "activations/layer19_attention_weight_max": 22.16814613342285,
      "activations/layer19_attention_weight_min": -21.839637756347656,
      "activations/layer1_attention_weight_max": 16.522768020629883,
      "activations/layer1_attention_weight_min": -13.649136543273926,
      "activations/layer20_attention_weight_max": 22.658191680908203,
      "activations/layer20_attention_weight_min": -20.937246322631836,
      "activations/layer21_attention_weight_max": 31.68216896057129,
      "activations/layer21_attention_weight_min": -22.909746170043945,
      "activations/layer22_attention_weight_max": 28.43221664428711,
      "activations/layer22_attention_weight_min": -24.47366714477539,
      "activations/layer23_attention_weight_max": 37.5474739074707,
      "activations/layer23_attention_weight_min": -23.288448333740234,
      "activations/layer2_attention_weight_max": 31.208297729492188,
      "activations/layer2_attention_weight_min": -29.73391342163086,
      "activations/layer3_attention_weight_max": 88.2782974243164,
      "activations/layer3_attention_weight_min": -92.39836120605469,
      "activations/layer4_attention_weight_max": 86.72618865966797,
      "activations/layer4_attention_weight_min": -88.19285583496094,
      "activations/layer5_attention_weight_max": 68.90164947509766,
      "activations/layer5_attention_weight_min": -74.43401336669922,
      "activations/layer6_attention_weight_max": 49.657100677490234,
      "activations/layer6_attention_weight_min": -50.790809631347656,
      "activations/layer7_attention_weight_max": 65.17630767822266,
      "activations/layer7_attention_weight_min": -61.321434020996094,
      "activations/layer8_attention_weight_max": 48.333621978759766,
      "activations/layer8_attention_weight_min": -50.906394958496094,
      "activations/layer9_attention_weight_max": 45.19068908691406,
      "activations/layer9_attention_weight_min": -47.443260192871094,
      "epoch": 14.72,
      "learning_rate": 5.564734848484848e-05,
      "loss": 2.7439,
      "step": 253300
    },
    {
      "activations/layer0_attention_weight_max": 16.67371368408203,
      "activations/layer0_attention_weight_min": -13.086942672729492,
      "activations/layer10_attention_weight_max": 34.03337478637695,
      "activations/layer10_attention_weight_min": -34.62692642211914,
      "activations/layer11_attention_weight_max": 33.18085479736328,
      "activations/layer11_attention_weight_min": -31.653867721557617,
      "activations/layer12_attention_weight_max": 17.775861740112305,
      "activations/layer12_attention_weight_min": -25.317626953125,
      "activations/layer13_attention_weight_max": 40.340965270996094,
      "activations/layer13_attention_weight_min": -28.51225471496582,
      "activations/layer14_attention_weight_max": 32.936920166015625,
      "activations/layer14_attention_weight_min": -29.798112869262695,
      "activations/layer15_attention_weight_max": 33.97098922729492,
      "activations/layer15_attention_weight_min": -28.63717269897461,
      "activations/layer16_attention_weight_max": 31.22315216064453,
      "activations/layer16_attention_weight_min": -29.7171573638916,
      "activations/layer17_attention_weight_max": 50.84037399291992,
      "activations/layer17_attention_weight_min": -43.94434356689453,
      "activations/layer18_attention_weight_max": 41.949459075927734,
      "activations/layer18_attention_weight_min": -37.90534973144531,
      "activations/layer19_attention_weight_max": 22.619409561157227,
      "activations/layer19_attention_weight_min": -21.239891052246094,
      "activations/layer1_attention_weight_max": 15.910077095031738,
      "activations/layer1_attention_weight_min": -14.332805633544922,
      "activations/layer20_attention_weight_max": 22.853588104248047,
      "activations/layer20_attention_weight_min": -22.134126663208008,
      "activations/layer21_attention_weight_max": 34.514137268066406,
      "activations/layer21_attention_weight_min": -20.552162170410156,
      "activations/layer22_attention_weight_max": 29.039684295654297,
      "activations/layer22_attention_weight_min": -25.918460845947266,
      "activations/layer23_attention_weight_max": 35.45198440551758,
      "activations/layer23_attention_weight_min": -24.49527359008789,
      "activations/layer2_attention_weight_max": 33.698211669921875,
      "activations/layer2_attention_weight_min": -32.28489685058594,
      "activations/layer3_attention_weight_max": 87.5265884399414,
      "activations/layer3_attention_weight_min": -93.27141571044922,
      "activations/layer4_attention_weight_max": 84.80972290039062,
      "activations/layer4_attention_weight_min": -84.96812438964844,
      "activations/layer5_attention_weight_max": 66.98304748535156,
      "activations/layer5_attention_weight_min": -73.34027099609375,
      "activations/layer6_attention_weight_max": 46.974544525146484,
      "activations/layer6_attention_weight_min": -49.422855377197266,
      "activations/layer7_attention_weight_max": 60.276336669921875,
      "activations/layer7_attention_weight_min": -58.9616584777832,
      "activations/layer8_attention_weight_max": 43.832279205322266,
      "activations/layer8_attention_weight_min": -46.14467239379883,
      "activations/layer9_attention_weight_max": 45.30290985107422,
      "activations/layer9_attention_weight_min": -44.72370910644531,
      "epoch": 14.72,
      "learning_rate": 5.562840909090909e-05,
      "loss": 2.7613,
      "step": 253350
    },
    {
      "activations/layer0_attention_weight_max": 16.758943557739258,
      "activations/layer0_attention_weight_min": -13.255687713623047,
      "activations/layer10_attention_weight_max": 38.99691390991211,
      "activations/layer10_attention_weight_min": -37.09156036376953,
      "activations/layer11_attention_weight_max": 34.4647102355957,
      "activations/layer11_attention_weight_min": -33.36888885498047,
      "activations/layer12_attention_weight_max": 19.259241104125977,
      "activations/layer12_attention_weight_min": -26.719022750854492,
      "activations/layer13_attention_weight_max": 36.58018112182617,
      "activations/layer13_attention_weight_min": -28.12090492248535,
      "activations/layer14_attention_weight_max": 33.722042083740234,
      "activations/layer14_attention_weight_min": -31.309099197387695,
      "activations/layer15_attention_weight_max": 31.965452194213867,
      "activations/layer15_attention_weight_min": -31.148359298706055,
      "activations/layer16_attention_weight_max": 32.79994583129883,
      "activations/layer16_attention_weight_min": -31.823633193969727,
      "activations/layer17_attention_weight_max": 54.133033752441406,
      "activations/layer17_attention_weight_min": -47.51603317260742,
      "activations/layer18_attention_weight_max": 49.96518325805664,
      "activations/layer18_attention_weight_min": -41.55603790283203,
      "activations/layer19_attention_weight_max": 24.19474220275879,
      "activations/layer19_attention_weight_min": -23.56510353088379,
      "activations/layer1_attention_weight_max": 16.903535842895508,
      "activations/layer1_attention_weight_min": -13.347567558288574,
      "activations/layer20_attention_weight_max": 22.41164207458496,
      "activations/layer20_attention_weight_min": -26.27324867248535,
      "activations/layer21_attention_weight_max": 34.32697296142578,
      "activations/layer21_attention_weight_min": -23.46954345703125,
      "activations/layer22_attention_weight_max": 30.792070388793945,
      "activations/layer22_attention_weight_min": -28.087373733520508,
      "activations/layer23_attention_weight_max": 35.168914794921875,
      "activations/layer23_attention_weight_min": -29.02264404296875,
      "activations/layer2_attention_weight_max": 33.64221954345703,
      "activations/layer2_attention_weight_min": -32.172393798828125,
      "activations/layer3_attention_weight_max": 98.80903625488281,
      "activations/layer3_attention_weight_min": -93.5791015625,
      "activations/layer4_attention_weight_max": 95.51277923583984,
      "activations/layer4_attention_weight_min": -94.09282684326172,
      "activations/layer5_attention_weight_max": 68.41869354248047,
      "activations/layer5_attention_weight_min": -76.82579803466797,
      "activations/layer6_attention_weight_max": 50.81938171386719,
      "activations/layer6_attention_weight_min": -52.405738830566406,
      "activations/layer7_attention_weight_max": 61.119998931884766,
      "activations/layer7_attention_weight_min": -62.907196044921875,
      "activations/layer8_attention_weight_max": 50.03179168701172,
      "activations/layer8_attention_weight_min": -53.00923538208008,
      "activations/layer9_attention_weight_max": 47.56943893432617,
      "activations/layer9_attention_weight_min": -47.816585540771484,
      "epoch": 14.72,
      "learning_rate": 5.5609469696969695e-05,
      "loss": 2.7808,
      "step": 253400
    },
    {
      "activations/layer0_attention_weight_max": 15.380024909973145,
      "activations/layer0_attention_weight_min": -13.086982727050781,
      "activations/layer10_attention_weight_max": 39.238338470458984,
      "activations/layer10_attention_weight_min": -36.07374954223633,
      "activations/layer11_attention_weight_max": 34.91669464111328,
      "activations/layer11_attention_weight_min": -34.67643737792969,
      "activations/layer12_attention_weight_max": 17.9699764251709,
      "activations/layer12_attention_weight_min": -26.744569778442383,
      "activations/layer13_attention_weight_max": 36.8304328918457,
      "activations/layer13_attention_weight_min": -29.415559768676758,
      "activations/layer14_attention_weight_max": 37.4996223449707,
      "activations/layer14_attention_weight_min": -29.130300521850586,
      "activations/layer15_attention_weight_max": 32.796138763427734,
      "activations/layer15_attention_weight_min": -29.441320419311523,
      "activations/layer16_attention_weight_max": 31.555740356445312,
      "activations/layer16_attention_weight_min": -30.343868255615234,
      "activations/layer17_attention_weight_max": 52.01906204223633,
      "activations/layer17_attention_weight_min": -43.72429275512695,
      "activations/layer18_attention_weight_max": 47.6956787109375,
      "activations/layer18_attention_weight_min": -37.40902328491211,
      "activations/layer19_attention_weight_max": 23.860815048217773,
      "activations/layer19_attention_weight_min": -20.87407875061035,
      "activations/layer1_attention_weight_max": 16.196575164794922,
      "activations/layer1_attention_weight_min": -14.736512184143066,
      "activations/layer20_attention_weight_max": 25.292329788208008,
      "activations/layer20_attention_weight_min": -21.209754943847656,
      "activations/layer21_attention_weight_max": 35.71807861328125,
      "activations/layer21_attention_weight_min": -23.834035873413086,
      "activations/layer22_attention_weight_max": 29.488109588623047,
      "activations/layer22_attention_weight_min": -24.790674209594727,
      "activations/layer23_attention_weight_max": 37.05638122558594,
      "activations/layer23_attention_weight_min": -25.954936981201172,
      "activations/layer2_attention_weight_max": 32.417327880859375,
      "activations/layer2_attention_weight_min": -31.541156768798828,
      "activations/layer3_attention_weight_max": 89.86634063720703,
      "activations/layer3_attention_weight_min": -92.89734649658203,
      "activations/layer4_attention_weight_max": 87.44558715820312,
      "activations/layer4_attention_weight_min": -88.10842895507812,
      "activations/layer5_attention_weight_max": 64.83524322509766,
      "activations/layer5_attention_weight_min": -74.0157470703125,
      "activations/layer6_attention_weight_max": 49.80044174194336,
      "activations/layer6_attention_weight_min": -50.49911880493164,
      "activations/layer7_attention_weight_max": 62.63859558105469,
      "activations/layer7_attention_weight_min": -63.06791687011719,
      "activations/layer8_attention_weight_max": 46.33229446411133,
      "activations/layer8_attention_weight_min": -49.02206039428711,
      "activations/layer9_attention_weight_max": 47.16425704956055,
      "activations/layer9_attention_weight_min": -45.9256591796875,
      "epoch": 14.73,
      "learning_rate": 5.5590909090909086e-05,
      "loss": 2.764,
      "step": 253450
    },
    {
      "activations/layer0_attention_weight_max": 15.656736373901367,
      "activations/layer0_attention_weight_min": -12.977604866027832,
      "activations/layer10_attention_weight_max": 36.836273193359375,
      "activations/layer10_attention_weight_min": -35.52293014526367,
      "activations/layer11_attention_weight_max": 32.219505310058594,
      "activations/layer11_attention_weight_min": -35.82530212402344,
      "activations/layer12_attention_weight_max": 17.151987075805664,
      "activations/layer12_attention_weight_min": -26.085603713989258,
      "activations/layer13_attention_weight_max": 36.95582580566406,
      "activations/layer13_attention_weight_min": -28.199541091918945,
      "activations/layer14_attention_weight_max": 35.37284469604492,
      "activations/layer14_attention_weight_min": -30.274721145629883,
      "activations/layer15_attention_weight_max": 36.873069763183594,
      "activations/layer15_attention_weight_min": -30.405752182006836,
      "activations/layer16_attention_weight_max": 35.222007751464844,
      "activations/layer16_attention_weight_min": -31.423221588134766,
      "activations/layer17_attention_weight_max": 53.55526351928711,
      "activations/layer17_attention_weight_min": -44.48143768310547,
      "activations/layer18_attention_weight_max": 50.09043884277344,
      "activations/layer18_attention_weight_min": -40.1566047668457,
      "activations/layer19_attention_weight_max": 24.124982833862305,
      "activations/layer19_attention_weight_min": -21.53022575378418,
      "activations/layer1_attention_weight_max": 16.182083129882812,
      "activations/layer1_attention_weight_min": -13.810856819152832,
      "activations/layer20_attention_weight_max": 23.855897903442383,
      "activations/layer20_attention_weight_min": -20.77798080444336,
      "activations/layer21_attention_weight_max": 36.68962478637695,
      "activations/layer21_attention_weight_min": -22.37560272216797,
      "activations/layer22_attention_weight_max": 31.227386474609375,
      "activations/layer22_attention_weight_min": -26.20824432373047,
      "activations/layer23_attention_weight_max": 36.63621520996094,
      "activations/layer23_attention_weight_min": -23.228954315185547,
      "activations/layer2_attention_weight_max": 30.596271514892578,
      "activations/layer2_attention_weight_min": -28.74273681640625,
      "activations/layer3_attention_weight_max": 88.37830352783203,
      "activations/layer3_attention_weight_min": -93.66345977783203,
      "activations/layer4_attention_weight_max": 86.85247039794922,
      "activations/layer4_attention_weight_min": -88.83036804199219,
      "activations/layer5_attention_weight_max": 68.59349060058594,
      "activations/layer5_attention_weight_min": -75.2567367553711,
      "activations/layer6_attention_weight_max": 50.80880355834961,
      "activations/layer6_attention_weight_min": -53.57096481323242,
      "activations/layer7_attention_weight_max": 61.07493591308594,
      "activations/layer7_attention_weight_min": -63.759185791015625,
      "activations/layer8_attention_weight_max": 45.46986389160156,
      "activations/layer8_attention_weight_min": -50.90299987792969,
      "activations/layer9_attention_weight_max": 45.18889617919922,
      "activations/layer9_attention_weight_min": -46.55793380737305,
      "epoch": 14.73,
      "learning_rate": 5.557196969696969e-05,
      "loss": 2.7508,
      "step": 253500
    },
    {
      "activations/layer0_attention_weight_max": 15.880889892578125,
      "activations/layer0_attention_weight_min": -13.12442684173584,
      "activations/layer10_attention_weight_max": 34.65699005126953,
      "activations/layer10_attention_weight_min": -34.33260726928711,
      "activations/layer11_attention_weight_max": 31.28005599975586,
      "activations/layer11_attention_weight_min": -31.342365264892578,
      "activations/layer12_attention_weight_max": 18.454910278320312,
      "activations/layer12_attention_weight_min": -25.30982208251953,
      "activations/layer13_attention_weight_max": 39.37781524658203,
      "activations/layer13_attention_weight_min": -27.69475555419922,
      "activations/layer14_attention_weight_max": 33.525360107421875,
      "activations/layer14_attention_weight_min": -30.142236709594727,
      "activations/layer15_attention_weight_max": 33.4718017578125,
      "activations/layer15_attention_weight_min": -29.67206382751465,
      "activations/layer16_attention_weight_max": 32.421871185302734,
      "activations/layer16_attention_weight_min": -30.24277687072754,
      "activations/layer17_attention_weight_max": 52.03187942504883,
      "activations/layer17_attention_weight_min": -43.18152618408203,
      "activations/layer18_attention_weight_max": 50.10731887817383,
      "activations/layer18_attention_weight_min": -38.323394775390625,
      "activations/layer19_attention_weight_max": 26.394590377807617,
      "activations/layer19_attention_weight_min": -23.323955535888672,
      "activations/layer1_attention_weight_max": 17.99065589904785,
      "activations/layer1_attention_weight_min": -13.03486156463623,
      "activations/layer20_attention_weight_max": 25.238107681274414,
      "activations/layer20_attention_weight_min": -23.514564514160156,
      "activations/layer21_attention_weight_max": 37.83961868286133,
      "activations/layer21_attention_weight_min": -22.196428298950195,
      "activations/layer22_attention_weight_max": 33.28857421875,
      "activations/layer22_attention_weight_min": -26.88727569580078,
      "activations/layer23_attention_weight_max": 42.057777404785156,
      "activations/layer23_attention_weight_min": -24.71377182006836,
      "activations/layer2_attention_weight_max": 33.918304443359375,
      "activations/layer2_attention_weight_min": -30.039854049682617,
      "activations/layer3_attention_weight_max": 85.91226196289062,
      "activations/layer3_attention_weight_min": -89.88029479980469,
      "activations/layer4_attention_weight_max": 90.95738983154297,
      "activations/layer4_attention_weight_min": -92.3097152709961,
      "activations/layer5_attention_weight_max": 70.33045196533203,
      "activations/layer5_attention_weight_min": -73.35103607177734,
      "activations/layer6_attention_weight_max": 49.33597183227539,
      "activations/layer6_attention_weight_min": -52.35205841064453,
      "activations/layer7_attention_weight_max": 60.063323974609375,
      "activations/layer7_attention_weight_min": -62.51604080200195,
      "activations/layer8_attention_weight_max": 45.31819534301758,
      "activations/layer8_attention_weight_min": -48.74017333984375,
      "activations/layer9_attention_weight_max": 47.78042221069336,
      "activations/layer9_attention_weight_min": -47.338619232177734,
      "epoch": 14.73,
      "learning_rate": 5.55530303030303e-05,
      "loss": 2.7522,
      "step": 253550
    },
    {
      "activations/layer0_attention_weight_max": 15.891656875610352,
      "activations/layer0_attention_weight_min": -13.335564613342285,
      "activations/layer10_attention_weight_max": 35.52631378173828,
      "activations/layer10_attention_weight_min": -37.570186614990234,
      "activations/layer11_attention_weight_max": 33.35178756713867,
      "activations/layer11_attention_weight_min": -34.1055908203125,
      "activations/layer12_attention_weight_max": 18.491941452026367,
      "activations/layer12_attention_weight_min": -23.40298843383789,
      "activations/layer13_attention_weight_max": 35.02766799926758,
      "activations/layer13_attention_weight_min": -30.1610050201416,
      "activations/layer14_attention_weight_max": 36.391780853271484,
      "activations/layer14_attention_weight_min": -31.819629669189453,
      "activations/layer15_attention_weight_max": 31.488025665283203,
      "activations/layer15_attention_weight_min": -29.546239852905273,
      "activations/layer16_attention_weight_max": 32.44514465332031,
      "activations/layer16_attention_weight_min": -33.21267318725586,
      "activations/layer17_attention_weight_max": 48.60319137573242,
      "activations/layer17_attention_weight_min": -47.911773681640625,
      "activations/layer18_attention_weight_max": 45.17911148071289,
      "activations/layer18_attention_weight_min": -41.052059173583984,
      "activations/layer19_attention_weight_max": 24.436477661132812,
      "activations/layer19_attention_weight_min": -22.695598602294922,
      "activations/layer1_attention_weight_max": 17.03189468383789,
      "activations/layer1_attention_weight_min": -14.832925796508789,
      "activations/layer20_attention_weight_max": 21.688297271728516,
      "activations/layer20_attention_weight_min": -23.405170440673828,
      "activations/layer21_attention_weight_max": 37.20428466796875,
      "activations/layer21_attention_weight_min": -23.164838790893555,
      "activations/layer22_attention_weight_max": 31.982406616210938,
      "activations/layer22_attention_weight_min": -27.57098960876465,
      "activations/layer23_attention_weight_max": 39.575439453125,
      "activations/layer23_attention_weight_min": -26.375904083251953,
      "activations/layer2_attention_weight_max": 32.496334075927734,
      "activations/layer2_attention_weight_min": -30.93079376220703,
      "activations/layer3_attention_weight_max": 90.7869873046875,
      "activations/layer3_attention_weight_min": -93.95854949951172,
      "activations/layer4_attention_weight_max": 89.20140838623047,
      "activations/layer4_attention_weight_min": -97.08403778076172,
      "activations/layer5_attention_weight_max": 66.9726333618164,
      "activations/layer5_attention_weight_min": -74.7703857421875,
      "activations/layer6_attention_weight_max": 53.27724838256836,
      "activations/layer6_attention_weight_min": -50.81534194946289,
      "activations/layer7_attention_weight_max": 64.08817291259766,
      "activations/layer7_attention_weight_min": -63.804019927978516,
      "activations/layer8_attention_weight_max": 47.19083786010742,
      "activations/layer8_attention_weight_min": -48.80424880981445,
      "activations/layer9_attention_weight_max": 46.826072692871094,
      "activations/layer9_attention_weight_min": -49.593116760253906,
      "epoch": 14.74,
      "learning_rate": 5.5534090909090904e-05,
      "loss": 2.7726,
      "step": 253600
    },
    {
      "activations/layer0_attention_weight_max": 15.673951148986816,
      "activations/layer0_attention_weight_min": -13.3177490234375,
      "activations/layer10_attention_weight_max": 37.93306350708008,
      "activations/layer10_attention_weight_min": -35.879180908203125,
      "activations/layer11_attention_weight_max": 33.88789367675781,
      "activations/layer11_attention_weight_min": -34.94502639770508,
      "activations/layer12_attention_weight_max": 17.99464225769043,
      "activations/layer12_attention_weight_min": -24.54088020324707,
      "activations/layer13_attention_weight_max": 35.550331115722656,
      "activations/layer13_attention_weight_min": -31.53944969177246,
      "activations/layer14_attention_weight_max": 33.534095764160156,
      "activations/layer14_attention_weight_min": -31.12177276611328,
      "activations/layer15_attention_weight_max": 30.81376838684082,
      "activations/layer15_attention_weight_min": -29.728490829467773,
      "activations/layer16_attention_weight_max": 30.335893630981445,
      "activations/layer16_attention_weight_min": -29.91413688659668,
      "activations/layer17_attention_weight_max": 46.983238220214844,
      "activations/layer17_attention_weight_min": -43.64938735961914,
      "activations/layer18_attention_weight_max": 44.058982849121094,
      "activations/layer18_attention_weight_min": -38.555179595947266,
      "activations/layer19_attention_weight_max": 20.485647201538086,
      "activations/layer19_attention_weight_min": -23.621654510498047,
      "activations/layer1_attention_weight_max": 15.848275184631348,
      "activations/layer1_attention_weight_min": -14.059504508972168,
      "activations/layer20_attention_weight_max": 21.60205078125,
      "activations/layer20_attention_weight_min": -20.275163650512695,
      "activations/layer21_attention_weight_max": 32.98141098022461,
      "activations/layer21_attention_weight_min": -21.758054733276367,
      "activations/layer22_attention_weight_max": 28.84511375427246,
      "activations/layer22_attention_weight_min": -27.454208374023438,
      "activations/layer23_attention_weight_max": 39.163421630859375,
      "activations/layer23_attention_weight_min": -26.68379783630371,
      "activations/layer2_attention_weight_max": 30.07386589050293,
      "activations/layer2_attention_weight_min": -28.809852600097656,
      "activations/layer3_attention_weight_max": 90.39127349853516,
      "activations/layer3_attention_weight_min": -90.04279327392578,
      "activations/layer4_attention_weight_max": 88.6280746459961,
      "activations/layer4_attention_weight_min": -93.20864868164062,
      "activations/layer5_attention_weight_max": 66.16188049316406,
      "activations/layer5_attention_weight_min": -77.2286376953125,
      "activations/layer6_attention_weight_max": 53.208133697509766,
      "activations/layer6_attention_weight_min": -57.15119934082031,
      "activations/layer7_attention_weight_max": 64.24896240234375,
      "activations/layer7_attention_weight_min": -65.34074401855469,
      "activations/layer8_attention_weight_max": 48.406978607177734,
      "activations/layer8_attention_weight_min": -52.016029357910156,
      "activations/layer9_attention_weight_max": 45.441341400146484,
      "activations/layer9_attention_weight_min": -46.370113372802734,
      "epoch": 14.74,
      "learning_rate": 5.551515151515151e-05,
      "loss": 2.7516,
      "step": 253650
    },
    {
      "activations/layer0_attention_weight_max": 15.503914833068848,
      "activations/layer0_attention_weight_min": -12.98508071899414,
      "activations/layer10_attention_weight_max": 35.697933197021484,
      "activations/layer10_attention_weight_min": -33.19715118408203,
      "activations/layer11_attention_weight_max": 32.58418655395508,
      "activations/layer11_attention_weight_min": -32.97589874267578,
      "activations/layer12_attention_weight_max": 19.032390594482422,
      "activations/layer12_attention_weight_min": -24.051841735839844,
      "activations/layer13_attention_weight_max": 35.17171096801758,
      "activations/layer13_attention_weight_min": -27.756921768188477,
      "activations/layer14_attention_weight_max": 34.77570343017578,
      "activations/layer14_attention_weight_min": -29.324556350708008,
      "activations/layer15_attention_weight_max": 32.126426696777344,
      "activations/layer15_attention_weight_min": -29.035112380981445,
      "activations/layer16_attention_weight_max": 31.462539672851562,
      "activations/layer16_attention_weight_min": -30.044225692749023,
      "activations/layer17_attention_weight_max": 50.91330337524414,
      "activations/layer17_attention_weight_min": -43.86171340942383,
      "activations/layer18_attention_weight_max": 46.3067512512207,
      "activations/layer18_attention_weight_min": -38.01765060424805,
      "activations/layer19_attention_weight_max": 22.264142990112305,
      "activations/layer19_attention_weight_min": -21.379308700561523,
      "activations/layer1_attention_weight_max": 15.619114875793457,
      "activations/layer1_attention_weight_min": -13.902952194213867,
      "activations/layer20_attention_weight_max": 21.260509490966797,
      "activations/layer20_attention_weight_min": -21.490888595581055,
      "activations/layer21_attention_weight_max": 32.304683685302734,
      "activations/layer21_attention_weight_min": -20.339317321777344,
      "activations/layer22_attention_weight_max": 26.998395919799805,
      "activations/layer22_attention_weight_min": -25.706806182861328,
      "activations/layer23_attention_weight_max": 39.70393753051758,
      "activations/layer23_attention_weight_min": -25.1405029296875,
      "activations/layer2_attention_weight_max": 31.257253646850586,
      "activations/layer2_attention_weight_min": -30.662879943847656,
      "activations/layer3_attention_weight_max": 93.07064819335938,
      "activations/layer3_attention_weight_min": -91.71886444091797,
      "activations/layer4_attention_weight_max": 90.298095703125,
      "activations/layer4_attention_weight_min": -90.90347290039062,
      "activations/layer5_attention_weight_max": 69.43081665039062,
      "activations/layer5_attention_weight_min": -78.56195068359375,
      "activations/layer6_attention_weight_max": 50.45076370239258,
      "activations/layer6_attention_weight_min": -50.93132400512695,
      "activations/layer7_attention_weight_max": 63.143394470214844,
      "activations/layer7_attention_weight_min": -65.54043579101562,
      "activations/layer8_attention_weight_max": 43.40017318725586,
      "activations/layer8_attention_weight_min": -45.924720764160156,
      "activations/layer9_attention_weight_max": 43.573814392089844,
      "activations/layer9_attention_weight_min": -44.928253173828125,
      "epoch": 14.74,
      "learning_rate": 5.5496212121212114e-05,
      "loss": 2.7513,
      "step": 253700
    },
    {
      "activations/layer0_attention_weight_max": 15.353781700134277,
      "activations/layer0_attention_weight_min": -12.762818336486816,
      "activations/layer10_attention_weight_max": 33.83866500854492,
      "activations/layer10_attention_weight_min": -34.027957916259766,
      "activations/layer11_attention_weight_max": 30.87871551513672,
      "activations/layer11_attention_weight_min": -32.58415985107422,
      "activations/layer12_attention_weight_max": 18.551101684570312,
      "activations/layer12_attention_weight_min": -28.18392562866211,
      "activations/layer13_attention_weight_max": 34.21299362182617,
      "activations/layer13_attention_weight_min": -27.661333084106445,
      "activations/layer14_attention_weight_max": 38.081974029541016,
      "activations/layer14_attention_weight_min": -30.75107192993164,
      "activations/layer15_attention_weight_max": 36.2436637878418,
      "activations/layer15_attention_weight_min": -30.77092170715332,
      "activations/layer16_attention_weight_max": 34.038150787353516,
      "activations/layer16_attention_weight_min": -30.817920684814453,
      "activations/layer17_attention_weight_max": 51.06079864501953,
      "activations/layer17_attention_weight_min": -44.60654067993164,
      "activations/layer18_attention_weight_max": 44.38212203979492,
      "activations/layer18_attention_weight_min": -39.4727897644043,
      "activations/layer19_attention_weight_max": 22.150192260742188,
      "activations/layer19_attention_weight_min": -23.084829330444336,
      "activations/layer1_attention_weight_max": 16.173553466796875,
      "activations/layer1_attention_weight_min": -13.669631004333496,
      "activations/layer20_attention_weight_max": 22.99789810180664,
      "activations/layer20_attention_weight_min": -23.86887550354004,
      "activations/layer21_attention_weight_max": 36.046875,
      "activations/layer21_attention_weight_min": -22.935625076293945,
      "activations/layer22_attention_weight_max": 33.62664794921875,
      "activations/layer22_attention_weight_min": -28.36540985107422,
      "activations/layer23_attention_weight_max": 35.385475158691406,
      "activations/layer23_attention_weight_min": -25.770042419433594,
      "activations/layer2_attention_weight_max": 30.203960418701172,
      "activations/layer2_attention_weight_min": -27.974472045898438,
      "activations/layer3_attention_weight_max": 87.88793182373047,
      "activations/layer3_attention_weight_min": -88.1382827758789,
      "activations/layer4_attention_weight_max": 81.6842041015625,
      "activations/layer4_attention_weight_min": -83.37055969238281,
      "activations/layer5_attention_weight_max": 64.5882568359375,
      "activations/layer5_attention_weight_min": -77.7127685546875,
      "activations/layer6_attention_weight_max": 47.815914154052734,
      "activations/layer6_attention_weight_min": -50.62461471557617,
      "activations/layer7_attention_weight_max": 63.490745544433594,
      "activations/layer7_attention_weight_min": -59.368778228759766,
      "activations/layer8_attention_weight_max": 46.1922721862793,
      "activations/layer8_attention_weight_min": -49.14596176147461,
      "activations/layer9_attention_weight_max": 42.578121185302734,
      "activations/layer9_attention_weight_min": -45.97024917602539,
      "epoch": 14.74,
      "learning_rate": 5.547727272727273e-05,
      "loss": 2.745,
      "step": 253750
    },
    {
      "activations/layer0_attention_weight_max": 15.88418197631836,
      "activations/layer0_attention_weight_min": -13.047160148620605,
      "activations/layer10_attention_weight_max": 36.03983688354492,
      "activations/layer10_attention_weight_min": -36.38517379760742,
      "activations/layer11_attention_weight_max": 31.351667404174805,
      "activations/layer11_attention_weight_min": -32.09734344482422,
      "activations/layer12_attention_weight_max": 20.578243255615234,
      "activations/layer12_attention_weight_min": -25.494840621948242,
      "activations/layer13_attention_weight_max": 31.882604598999023,
      "activations/layer13_attention_weight_min": -27.67576026916504,
      "activations/layer14_attention_weight_max": 39.83070373535156,
      "activations/layer14_attention_weight_min": -33.80349349975586,
      "activations/layer15_attention_weight_max": 36.21961212158203,
      "activations/layer15_attention_weight_min": -33.32857894897461,
      "activations/layer16_attention_weight_max": 37.09208297729492,
      "activations/layer16_attention_weight_min": -34.353759765625,
      "activations/layer17_attention_weight_max": 59.63444900512695,
      "activations/layer17_attention_weight_min": -52.491355895996094,
      "activations/layer18_attention_weight_max": 56.130706787109375,
      "activations/layer18_attention_weight_min": -43.811912536621094,
      "activations/layer19_attention_weight_max": 25.424165725708008,
      "activations/layer19_attention_weight_min": -23.461950302124023,
      "activations/layer1_attention_weight_max": 16.6947021484375,
      "activations/layer1_attention_weight_min": -14.53847599029541,
      "activations/layer20_attention_weight_max": 23.504858016967773,
      "activations/layer20_attention_weight_min": -22.81028175354004,
      "activations/layer21_attention_weight_max": 36.23185729980469,
      "activations/layer21_attention_weight_min": -20.419189453125,
      "activations/layer22_attention_weight_max": 31.19034767150879,
      "activations/layer22_attention_weight_min": -27.16438865661621,
      "activations/layer23_attention_weight_max": 41.57659149169922,
      "activations/layer23_attention_weight_min": -24.07455825805664,
      "activations/layer2_attention_weight_max": 30.156566619873047,
      "activations/layer2_attention_weight_min": -27.925830841064453,
      "activations/layer3_attention_weight_max": 88.02095794677734,
      "activations/layer3_attention_weight_min": -88.83474731445312,
      "activations/layer4_attention_weight_max": 92.63607788085938,
      "activations/layer4_attention_weight_min": -85.18451690673828,
      "activations/layer5_attention_weight_max": 67.06101989746094,
      "activations/layer5_attention_weight_min": -79.35506439208984,
      "activations/layer6_attention_weight_max": 49.19819259643555,
      "activations/layer6_attention_weight_min": -49.85897445678711,
      "activations/layer7_attention_weight_max": 67.40086364746094,
      "activations/layer7_attention_weight_min": -60.50148010253906,
      "activations/layer8_attention_weight_max": 43.787620544433594,
      "activations/layer8_attention_weight_min": -47.1901741027832,
      "activations/layer9_attention_weight_max": 43.45428466796875,
      "activations/layer9_attention_weight_min": -44.0318603515625,
      "epoch": 14.75,
      "learning_rate": 5.545833333333333e-05,
      "loss": 2.7673,
      "step": 253800
    },
    {
      "activations/layer0_attention_weight_max": 15.454300880432129,
      "activations/layer0_attention_weight_min": -12.905684471130371,
      "activations/layer10_attention_weight_max": 36.4471435546875,
      "activations/layer10_attention_weight_min": -34.571720123291016,
      "activations/layer11_attention_weight_max": 33.06937789916992,
      "activations/layer11_attention_weight_min": -34.04781723022461,
      "activations/layer12_attention_weight_max": 19.85822105407715,
      "activations/layer12_attention_weight_min": -24.74216079711914,
      "activations/layer13_attention_weight_max": 37.57130813598633,
      "activations/layer13_attention_weight_min": -28.5903263092041,
      "activations/layer14_attention_weight_max": 37.66609573364258,
      "activations/layer14_attention_weight_min": -30.787233352661133,
      "activations/layer15_attention_weight_max": 36.03282928466797,
      "activations/layer15_attention_weight_min": -30.586776733398438,
      "activations/layer16_attention_weight_max": 36.70154571533203,
      "activations/layer16_attention_weight_min": -31.481021881103516,
      "activations/layer17_attention_weight_max": 56.16844177246094,
      "activations/layer17_attention_weight_min": -46.197757720947266,
      "activations/layer18_attention_weight_max": 53.47615051269531,
      "activations/layer18_attention_weight_min": -42.7962760925293,
      "activations/layer19_attention_weight_max": 28.60188865661621,
      "activations/layer19_attention_weight_min": -24.96710205078125,
      "activations/layer1_attention_weight_max": 16.373546600341797,
      "activations/layer1_attention_weight_min": -15.605830192565918,
      "activations/layer20_attention_weight_max": 28.86088752746582,
      "activations/layer20_attention_weight_min": -22.89684295654297,
      "activations/layer21_attention_weight_max": 44.747310638427734,
      "activations/layer21_attention_weight_min": -22.826688766479492,
      "activations/layer22_attention_weight_max": 33.20066833496094,
      "activations/layer22_attention_weight_min": -25.4073543548584,
      "activations/layer23_attention_weight_max": 43.746917724609375,
      "activations/layer23_attention_weight_min": -27.804092407226562,
      "activations/layer2_attention_weight_max": 30.24237632751465,
      "activations/layer2_attention_weight_min": -28.441246032714844,
      "activations/layer3_attention_weight_max": 87.01467895507812,
      "activations/layer3_attention_weight_min": -88.51665496826172,
      "activations/layer4_attention_weight_max": 86.92562103271484,
      "activations/layer4_attention_weight_min": -85.35253143310547,
      "activations/layer5_attention_weight_max": 65.31053161621094,
      "activations/layer5_attention_weight_min": -69.16118621826172,
      "activations/layer6_attention_weight_max": 51.15285110473633,
      "activations/layer6_attention_weight_min": -50.97065353393555,
      "activations/layer7_attention_weight_max": 62.46006774902344,
      "activations/layer7_attention_weight_min": -62.752960205078125,
      "activations/layer8_attention_weight_max": 43.68169403076172,
      "activations/layer8_attention_weight_min": -48.04164123535156,
      "activations/layer9_attention_weight_max": 42.66766357421875,
      "activations/layer9_attention_weight_min": -46.463253021240234,
      "epoch": 14.75,
      "learning_rate": 5.543939393939393e-05,
      "loss": 2.7555,
      "step": 253850
    },
    {
      "activations/layer0_attention_weight_max": 16.200706481933594,
      "activations/layer0_attention_weight_min": -12.948156356811523,
      "activations/layer10_attention_weight_max": 36.73170852661133,
      "activations/layer10_attention_weight_min": -36.914024353027344,
      "activations/layer11_attention_weight_max": 34.56128692626953,
      "activations/layer11_attention_weight_min": -35.145408630371094,
      "activations/layer12_attention_weight_max": 18.44869613647461,
      "activations/layer12_attention_weight_min": -24.748165130615234,
      "activations/layer13_attention_weight_max": 33.77857208251953,
      "activations/layer13_attention_weight_min": -25.642967224121094,
      "activations/layer14_attention_weight_max": 46.38038635253906,
      "activations/layer14_attention_weight_min": -29.725982666015625,
      "activations/layer15_attention_weight_max": 36.42279815673828,
      "activations/layer15_attention_weight_min": -30.039649963378906,
      "activations/layer16_attention_weight_max": 31.696199417114258,
      "activations/layer16_attention_weight_min": -31.22342872619629,
      "activations/layer17_attention_weight_max": 50.04448318481445,
      "activations/layer17_attention_weight_min": -44.192806243896484,
      "activations/layer18_attention_weight_max": 47.64921951293945,
      "activations/layer18_attention_weight_min": -38.42935562133789,
      "activations/layer19_attention_weight_max": 24.098188400268555,
      "activations/layer19_attention_weight_min": -23.76922035217285,
      "activations/layer1_attention_weight_max": 16.808799743652344,
      "activations/layer1_attention_weight_min": -13.974757194519043,
      "activations/layer20_attention_weight_max": 23.932064056396484,
      "activations/layer20_attention_weight_min": -23.212799072265625,
      "activations/layer21_attention_weight_max": 36.50347137451172,
      "activations/layer21_attention_weight_min": -23.468631744384766,
      "activations/layer22_attention_weight_max": 31.700477600097656,
      "activations/layer22_attention_weight_min": -26.311569213867188,
      "activations/layer23_attention_weight_max": 37.75059509277344,
      "activations/layer23_attention_weight_min": -23.388587951660156,
      "activations/layer2_attention_weight_max": 32.70758056640625,
      "activations/layer2_attention_weight_min": -31.126976013183594,
      "activations/layer3_attention_weight_max": 94.55362701416016,
      "activations/layer3_attention_weight_min": -95.83969116210938,
      "activations/layer4_attention_weight_max": 92.52678680419922,
      "activations/layer4_attention_weight_min": -88.15352630615234,
      "activations/layer5_attention_weight_max": 68.22457885742188,
      "activations/layer5_attention_weight_min": -73.0373306274414,
      "activations/layer6_attention_weight_max": 52.81596374511719,
      "activations/layer6_attention_weight_min": -53.80004119873047,
      "activations/layer7_attention_weight_max": 67.3702392578125,
      "activations/layer7_attention_weight_min": -62.655208587646484,
      "activations/layer8_attention_weight_max": 51.52516555786133,
      "activations/layer8_attention_weight_min": -54.16316604614258,
      "activations/layer9_attention_weight_max": 49.97579574584961,
      "activations/layer9_attention_weight_min": -49.55385208129883,
      "epoch": 14.75,
      "learning_rate": 5.542045454545454e-05,
      "loss": 2.7661,
      "step": 253900
    },
    {
      "activations/layer0_attention_weight_max": 15.414993286132812,
      "activations/layer0_attention_weight_min": -13.129866600036621,
      "activations/layer10_attention_weight_max": 37.22334671020508,
      "activations/layer10_attention_weight_min": -37.93091583251953,
      "activations/layer11_attention_weight_max": 33.127830505371094,
      "activations/layer11_attention_weight_min": -32.33137512207031,
      "activations/layer12_attention_weight_max": 18.51145362854004,
      "activations/layer12_attention_weight_min": -22.13675880432129,
      "activations/layer13_attention_weight_max": 34.38656234741211,
      "activations/layer13_attention_weight_min": -35.18610382080078,
      "activations/layer14_attention_weight_max": 36.05704116821289,
      "activations/layer14_attention_weight_min": -31.63512420654297,
      "activations/layer15_attention_weight_max": 34.75408935546875,
      "activations/layer15_attention_weight_min": -29.314939498901367,
      "activations/layer16_attention_weight_max": 33.90974426269531,
      "activations/layer16_attention_weight_min": -31.671680450439453,
      "activations/layer17_attention_weight_max": 53.05015563964844,
      "activations/layer17_attention_weight_min": -44.178627014160156,
      "activations/layer18_attention_weight_max": 49.183231353759766,
      "activations/layer18_attention_weight_min": -38.861080169677734,
      "activations/layer19_attention_weight_max": 25.79471206665039,
      "activations/layer19_attention_weight_min": -24.12713623046875,
      "activations/layer1_attention_weight_max": 16.60123634338379,
      "activations/layer1_attention_weight_min": -15.45964241027832,
      "activations/layer20_attention_weight_max": 28.19649887084961,
      "activations/layer20_attention_weight_min": -22.027097702026367,
      "activations/layer21_attention_weight_max": 43.24008560180664,
      "activations/layer21_attention_weight_min": -25.03986167907715,
      "activations/layer22_attention_weight_max": 31.88115692138672,
      "activations/layer22_attention_weight_min": -26.27002716064453,
      "activations/layer23_attention_weight_max": 39.04934310913086,
      "activations/layer23_attention_weight_min": -23.49830436706543,
      "activations/layer2_attention_weight_max": 31.32620620727539,
      "activations/layer2_attention_weight_min": -31.30188751220703,
      "activations/layer3_attention_weight_max": 95.16424560546875,
      "activations/layer3_attention_weight_min": -96.94732666015625,
      "activations/layer4_attention_weight_max": 88.381103515625,
      "activations/layer4_attention_weight_min": -85.5599136352539,
      "activations/layer5_attention_weight_max": 69.52068328857422,
      "activations/layer5_attention_weight_min": -74.97908020019531,
      "activations/layer6_attention_weight_max": 48.26790237426758,
      "activations/layer6_attention_weight_min": -50.18934631347656,
      "activations/layer7_attention_weight_max": 63.177330017089844,
      "activations/layer7_attention_weight_min": -60.72368621826172,
      "activations/layer8_attention_weight_max": 46.736331939697266,
      "activations/layer8_attention_weight_min": -47.6899299621582,
      "activations/layer9_attention_weight_max": 48.204586029052734,
      "activations/layer9_attention_weight_min": -46.65342330932617,
      "epoch": 14.76,
      "learning_rate": 5.540151515151515e-05,
      "loss": 2.7598,
      "step": 253950
    },
    {
      "activations/layer0_attention_weight_max": 15.812256813049316,
      "activations/layer0_attention_weight_min": -13.158374786376953,
      "activations/layer10_attention_weight_max": 35.27119064331055,
      "activations/layer10_attention_weight_min": -34.73948287963867,
      "activations/layer11_attention_weight_max": 33.405799865722656,
      "activations/layer11_attention_weight_min": -33.476715087890625,
      "activations/layer12_attention_weight_max": 18.353668212890625,
      "activations/layer12_attention_weight_min": -33.71977233886719,
      "activations/layer13_attention_weight_max": 40.970802307128906,
      "activations/layer13_attention_weight_min": -28.925006866455078,
      "activations/layer14_attention_weight_max": 45.15583419799805,
      "activations/layer14_attention_weight_min": -29.74646759033203,
      "activations/layer15_attention_weight_max": 36.03030014038086,
      "activations/layer15_attention_weight_min": -28.645456314086914,
      "activations/layer16_attention_weight_max": 32.10169219970703,
      "activations/layer16_attention_weight_min": -30.62397003173828,
      "activations/layer17_attention_weight_max": 50.8758544921875,
      "activations/layer17_attention_weight_min": -48.11232376098633,
      "activations/layer18_attention_weight_max": 45.90110778808594,
      "activations/layer18_attention_weight_min": -39.36924743652344,
      "activations/layer19_attention_weight_max": 23.897411346435547,
      "activations/layer19_attention_weight_min": -22.017152786254883,
      "activations/layer1_attention_weight_max": 16.371973037719727,
      "activations/layer1_attention_weight_min": -15.131707191467285,
      "activations/layer20_attention_weight_max": 25.90493392944336,
      "activations/layer20_attention_weight_min": -21.45404052734375,
      "activations/layer21_attention_weight_max": 35.35945510864258,
      "activations/layer21_attention_weight_min": -22.119760513305664,
      "activations/layer22_attention_weight_max": 31.111371994018555,
      "activations/layer22_attention_weight_min": -24.26763153076172,
      "activations/layer23_attention_weight_max": 36.48002624511719,
      "activations/layer23_attention_weight_min": -24.681297302246094,
      "activations/layer2_attention_weight_max": 32.971435546875,
      "activations/layer2_attention_weight_min": -32.02435302734375,
      "activations/layer3_attention_weight_max": 89.0411148071289,
      "activations/layer3_attention_weight_min": -91.00699615478516,
      "activations/layer4_attention_weight_max": 91.61734008789062,
      "activations/layer4_attention_weight_min": -88.7051010131836,
      "activations/layer5_attention_weight_max": 68.7693862915039,
      "activations/layer5_attention_weight_min": -71.99197387695312,
      "activations/layer6_attention_weight_max": 52.71287155151367,
      "activations/layer6_attention_weight_min": -51.102359771728516,
      "activations/layer7_attention_weight_max": 68.67813873291016,
      "activations/layer7_attention_weight_min": -63.71076965332031,
      "activations/layer8_attention_weight_max": 46.7082405090332,
      "activations/layer8_attention_weight_min": -48.97332763671875,
      "activations/layer9_attention_weight_max": 50.2133903503418,
      "activations/layer9_attention_weight_min": -45.54588317871094,
      "epoch": 14.76,
      "learning_rate": 5.538257575757575e-05,
      "loss": 2.771,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4951,
      "eval_samples_per_second": 505.467,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4951,
      "eval_openwebtext_samples_per_second": 505.467,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9327,
      "eval_wikitext_samples_per_second": 235.944,
      "step": 254000
    },
    {
      "epoch": 14.76,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.5239,
      "eval_lambada_samples_per_second": 511.241,
      "step": 254000
    },
    {
      "activations/layer0_attention_weight_max": 15.93535327911377,
      "activations/layer0_attention_weight_min": -13.054529190063477,
      "activations/layer10_attention_weight_max": 34.24085235595703,
      "activations/layer10_attention_weight_min": -36.12453842163086,
      "activations/layer11_attention_weight_max": 31.862449645996094,
      "activations/layer11_attention_weight_min": -32.41796875,
      "activations/layer12_attention_weight_max": 18.850963592529297,
      "activations/layer12_attention_weight_min": -25.47450065612793,
      "activations/layer13_attention_weight_max": 37.16417694091797,
      "activations/layer13_attention_weight_min": -27.355918884277344,
      "activations/layer14_attention_weight_max": 36.812137603759766,
      "activations/layer14_attention_weight_min": -29.60365867614746,
      "activations/layer15_attention_weight_max": 31.62709617614746,
      "activations/layer15_attention_weight_min": -29.226356506347656,
      "activations/layer16_attention_weight_max": 33.40366744995117,
      "activations/layer16_attention_weight_min": -30.63056182861328,
      "activations/layer17_attention_weight_max": 51.77864074707031,
      "activations/layer17_attention_weight_min": -43.796478271484375,
      "activations/layer18_attention_weight_max": 46.47517395019531,
      "activations/layer18_attention_weight_min": -37.17290496826172,
      "activations/layer19_attention_weight_max": 23.888389587402344,
      "activations/layer19_attention_weight_min": -23.18379783630371,
      "activations/layer1_attention_weight_max": 16.305137634277344,
      "activations/layer1_attention_weight_min": -13.593658447265625,
      "activations/layer20_attention_weight_max": 21.185806274414062,
      "activations/layer20_attention_weight_min": -20.94041633605957,
      "activations/layer21_attention_weight_max": 32.64764404296875,
      "activations/layer21_attention_weight_min": -20.816957473754883,
      "activations/layer22_attention_weight_max": 29.727155685424805,
      "activations/layer22_attention_weight_min": -27.37087631225586,
      "activations/layer23_attention_weight_max": 37.31359100341797,
      "activations/layer23_attention_weight_min": -26.302291870117188,
      "activations/layer2_attention_weight_max": 29.765287399291992,
      "activations/layer2_attention_weight_min": -27.73131561279297,
      "activations/layer3_attention_weight_max": 84.07646179199219,
      "activations/layer3_attention_weight_min": -87.07646179199219,
      "activations/layer4_attention_weight_max": 87.51945495605469,
      "activations/layer4_attention_weight_min": -86.80961608886719,
      "activations/layer5_attention_weight_max": 67.96345520019531,
      "activations/layer5_attention_weight_min": -77.42291259765625,
      "activations/layer6_attention_weight_max": 50.73171615600586,
      "activations/layer6_attention_weight_min": -51.21672821044922,
      "activations/layer7_attention_weight_max": 62.33559799194336,
      "activations/layer7_attention_weight_min": -62.5223503112793,
      "activations/layer8_attention_weight_max": 44.63669204711914,
      "activations/layer8_attention_weight_min": -49.10182571411133,
      "activations/layer9_attention_weight_max": 44.19612121582031,
      "activations/layer9_attention_weight_min": -47.94868087768555,
      "epoch": 14.76,
      "learning_rate": 5.536363636363636e-05,
      "loss": 2.7728,
      "step": 254050
    },
    {
      "activations/layer0_attention_weight_max": 16.44231414794922,
      "activations/layer0_attention_weight_min": -13.389294624328613,
      "activations/layer10_attention_weight_max": 36.368377685546875,
      "activations/layer10_attention_weight_min": -35.111488342285156,
      "activations/layer11_attention_weight_max": 32.29685974121094,
      "activations/layer11_attention_weight_min": -34.85559844970703,
      "activations/layer12_attention_weight_max": 17.871551513671875,
      "activations/layer12_attention_weight_min": -24.07588005065918,
      "activations/layer13_attention_weight_max": 32.313045501708984,
      "activations/layer13_attention_weight_min": -30.732479095458984,
      "activations/layer14_attention_weight_max": 32.10667419433594,
      "activations/layer14_attention_weight_min": -32.15204620361328,
      "activations/layer15_attention_weight_max": 30.727079391479492,
      "activations/layer15_attention_weight_min": -32.24571990966797,
      "activations/layer16_attention_weight_max": 30.154443740844727,
      "activations/layer16_attention_weight_min": -30.015064239501953,
      "activations/layer17_attention_weight_max": 47.293983459472656,
      "activations/layer17_attention_weight_min": -42.10595703125,
      "activations/layer18_attention_weight_max": 41.601871490478516,
      "activations/layer18_attention_weight_min": -37.5175895690918,
      "activations/layer19_attention_weight_max": 21.509490966796875,
      "activations/layer19_attention_weight_min": -20.518495559692383,
      "activations/layer1_attention_weight_max": 16.85249900817871,
      "activations/layer1_attention_weight_min": -15.332956314086914,
      "activations/layer20_attention_weight_max": 20.930482864379883,
      "activations/layer20_attention_weight_min": -21.58464813232422,
      "activations/layer21_attention_weight_max": 36.89259338378906,
      "activations/layer21_attention_weight_min": -23.007953643798828,
      "activations/layer22_attention_weight_max": 29.520322799682617,
      "activations/layer22_attention_weight_min": -25.71363067626953,
      "activations/layer23_attention_weight_max": 37.86446762084961,
      "activations/layer23_attention_weight_min": -28.543704986572266,
      "activations/layer2_attention_weight_max": 30.470027923583984,
      "activations/layer2_attention_weight_min": -29.520339965820312,
      "activations/layer3_attention_weight_max": 91.89166259765625,
      "activations/layer3_attention_weight_min": -87.16802978515625,
      "activations/layer4_attention_weight_max": 92.42186737060547,
      "activations/layer4_attention_weight_min": -88.03048706054688,
      "activations/layer5_attention_weight_max": 65.9971923828125,
      "activations/layer5_attention_weight_min": -72.28339385986328,
      "activations/layer6_attention_weight_max": 50.81437683105469,
      "activations/layer6_attention_weight_min": -51.53406524658203,
      "activations/layer7_attention_weight_max": 63.05558395385742,
      "activations/layer7_attention_weight_min": -63.786808013916016,
      "activations/layer8_attention_weight_max": 44.88621139526367,
      "activations/layer8_attention_weight_min": -48.290061950683594,
      "activations/layer9_attention_weight_max": 45.738094329833984,
      "activations/layer9_attention_weight_min": -46.57366943359375,
      "epoch": 14.76,
      "learning_rate": 5.534469696969696e-05,
      "loss": 2.7623,
      "step": 254100
    },
    {
      "activations/layer0_attention_weight_max": 16.207616806030273,
      "activations/layer0_attention_weight_min": -13.221588134765625,
      "activations/layer10_attention_weight_max": 35.350311279296875,
      "activations/layer10_attention_weight_min": -34.495338439941406,
      "activations/layer11_attention_weight_max": 31.5114803314209,
      "activations/layer11_attention_weight_min": -34.38104248046875,
      "activations/layer12_attention_weight_max": 17.454158782958984,
      "activations/layer12_attention_weight_min": -25.69091033935547,
      "activations/layer13_attention_weight_max": 31.24264144897461,
      "activations/layer13_attention_weight_min": -26.988637924194336,
      "activations/layer14_attention_weight_max": 34.69678497314453,
      "activations/layer14_attention_weight_min": -30.792943954467773,
      "activations/layer15_attention_weight_max": 34.56343460083008,
      "activations/layer15_attention_weight_min": -31.134571075439453,
      "activations/layer16_attention_weight_max": 33.28717803955078,
      "activations/layer16_attention_weight_min": -33.00215148925781,
      "activations/layer17_attention_weight_max": 48.29115676879883,
      "activations/layer17_attention_weight_min": -45.467079162597656,
      "activations/layer18_attention_weight_max": 44.55203628540039,
      "activations/layer18_attention_weight_min": -39.602420806884766,
      "activations/layer19_attention_weight_max": 24.49623680114746,
      "activations/layer19_attention_weight_min": -23.645719528198242,
      "activations/layer1_attention_weight_max": 16.500898361206055,
      "activations/layer1_attention_weight_min": -14.54726505279541,
      "activations/layer20_attention_weight_max": 21.29157066345215,
      "activations/layer20_attention_weight_min": -21.968162536621094,
      "activations/layer21_attention_weight_max": 35.02796173095703,
      "activations/layer21_attention_weight_min": -22.612749099731445,
      "activations/layer22_attention_weight_max": 26.94930076599121,
      "activations/layer22_attention_weight_min": -26.96953773498535,
      "activations/layer23_attention_weight_max": 33.60820770263672,
      "activations/layer23_attention_weight_min": -23.74182891845703,
      "activations/layer2_attention_weight_max": 31.037830352783203,
      "activations/layer2_attention_weight_min": -29.61559295654297,
      "activations/layer3_attention_weight_max": 90.16374206542969,
      "activations/layer3_attention_weight_min": -92.6295166015625,
      "activations/layer4_attention_weight_max": 85.39021301269531,
      "activations/layer4_attention_weight_min": -87.29431915283203,
      "activations/layer5_attention_weight_max": 66.19293975830078,
      "activations/layer5_attention_weight_min": -72.63101196289062,
      "activations/layer6_attention_weight_max": 50.78788757324219,
      "activations/layer6_attention_weight_min": -49.248634338378906,
      "activations/layer7_attention_weight_max": 59.95498275756836,
      "activations/layer7_attention_weight_min": -62.05503463745117,
      "activations/layer8_attention_weight_max": 44.0068244934082,
      "activations/layer8_attention_weight_min": -50.7923469543457,
      "activations/layer9_attention_weight_max": 43.21002197265625,
      "activations/layer9_attention_weight_min": -44.38780212402344,
      "epoch": 14.77,
      "learning_rate": 5.5325757575757575e-05,
      "loss": 2.7673,
      "step": 254150
    },
    {
      "activations/layer0_attention_weight_max": 15.632647514343262,
      "activations/layer0_attention_weight_min": -13.059127807617188,
      "activations/layer10_attention_weight_max": 37.7190055847168,
      "activations/layer10_attention_weight_min": -38.74842834472656,
      "activations/layer11_attention_weight_max": 36.42807388305664,
      "activations/layer11_attention_weight_min": -33.433143615722656,
      "activations/layer12_attention_weight_max": 17.89223289489746,
      "activations/layer12_attention_weight_min": -29.223709106445312,
      "activations/layer13_attention_weight_max": 34.12553024291992,
      "activations/layer13_attention_weight_min": -27.064481735229492,
      "activations/layer14_attention_weight_max": 38.90597152709961,
      "activations/layer14_attention_weight_min": -31.040416717529297,
      "activations/layer15_attention_weight_max": 33.50966262817383,
      "activations/layer15_attention_weight_min": -29.731853485107422,
      "activations/layer16_attention_weight_max": 33.931453704833984,
      "activations/layer16_attention_weight_min": -32.834503173828125,
      "activations/layer17_attention_weight_max": 53.66238021850586,
      "activations/layer17_attention_weight_min": -47.332061767578125,
      "activations/layer18_attention_weight_max": 44.57261276245117,
      "activations/layer18_attention_weight_min": -38.881690979003906,
      "activations/layer19_attention_weight_max": 22.025480270385742,
      "activations/layer19_attention_weight_min": -21.59750747680664,
      "activations/layer1_attention_weight_max": 17.051284790039062,
      "activations/layer1_attention_weight_min": -14.330875396728516,
      "activations/layer20_attention_weight_max": 22.977964401245117,
      "activations/layer20_attention_weight_min": -22.2535457611084,
      "activations/layer21_attention_weight_max": 41.2911376953125,
      "activations/layer21_attention_weight_min": -27.235355377197266,
      "activations/layer22_attention_weight_max": 30.737398147583008,
      "activations/layer22_attention_weight_min": -27.02467918395996,
      "activations/layer23_attention_weight_max": 35.564979553222656,
      "activations/layer23_attention_weight_min": -27.157791137695312,
      "activations/layer2_attention_weight_max": 32.26115798950195,
      "activations/layer2_attention_weight_min": -27.810644149780273,
      "activations/layer3_attention_weight_max": 95.6605453491211,
      "activations/layer3_attention_weight_min": -92.51278686523438,
      "activations/layer4_attention_weight_max": 91.40921020507812,
      "activations/layer4_attention_weight_min": -87.85987854003906,
      "activations/layer5_attention_weight_max": 67.11453247070312,
      "activations/layer5_attention_weight_min": -72.01331329345703,
      "activations/layer6_attention_weight_max": 51.88827896118164,
      "activations/layer6_attention_weight_min": -49.526859283447266,
      "activations/layer7_attention_weight_max": 63.79108428955078,
      "activations/layer7_attention_weight_min": -62.54907989501953,
      "activations/layer8_attention_weight_max": 46.98166275024414,
      "activations/layer8_attention_weight_min": -48.50310134887695,
      "activations/layer9_attention_weight_max": 51.24308395385742,
      "activations/layer9_attention_weight_min": -47.04681396484375,
      "epoch": 14.77,
      "learning_rate": 5.5306818181818176e-05,
      "loss": 2.7539,
      "step": 254200
    },
    {
      "activations/layer0_attention_weight_max": 15.685237884521484,
      "activations/layer0_attention_weight_min": -13.309402465820312,
      "activations/layer10_attention_weight_max": 38.74262237548828,
      "activations/layer10_attention_weight_min": -38.859283447265625,
      "activations/layer11_attention_weight_max": 35.97869110107422,
      "activations/layer11_attention_weight_min": -34.9976692199707,
      "activations/layer12_attention_weight_max": 18.573522567749023,
      "activations/layer12_attention_weight_min": -27.386398315429688,
      "activations/layer13_attention_weight_max": 39.21242904663086,
      "activations/layer13_attention_weight_min": -29.50420570373535,
      "activations/layer14_attention_weight_max": 36.69450378417969,
      "activations/layer14_attention_weight_min": -32.18455505371094,
      "activations/layer15_attention_weight_max": 31.564516067504883,
      "activations/layer15_attention_weight_min": -30.78078842163086,
      "activations/layer16_attention_weight_max": 33.50357437133789,
      "activations/layer16_attention_weight_min": -32.73324203491211,
      "activations/layer17_attention_weight_max": 50.06606674194336,
      "activations/layer17_attention_weight_min": -47.05570983886719,
      "activations/layer18_attention_weight_max": 45.1930046081543,
      "activations/layer18_attention_weight_min": -42.54369354248047,
      "activations/layer19_attention_weight_max": 22.863862991333008,
      "activations/layer19_attention_weight_min": -23.95778465270996,
      "activations/layer1_attention_weight_max": 16.187469482421875,
      "activations/layer1_attention_weight_min": -15.816497802734375,
      "activations/layer20_attention_weight_max": 23.188222885131836,
      "activations/layer20_attention_weight_min": -21.549667358398438,
      "activations/layer21_attention_weight_max": 45.04192352294922,
      "activations/layer21_attention_weight_min": -24.643863677978516,
      "activations/layer22_attention_weight_max": 29.947099685668945,
      "activations/layer22_attention_weight_min": -25.157381057739258,
      "activations/layer23_attention_weight_max": 39.26904296875,
      "activations/layer23_attention_weight_min": -24.651296615600586,
      "activations/layer2_attention_weight_max": 32.03356170654297,
      "activations/layer2_attention_weight_min": -29.62776756286621,
      "activations/layer3_attention_weight_max": 90.49948120117188,
      "activations/layer3_attention_weight_min": -87.84358978271484,
      "activations/layer4_attention_weight_max": 90.20214080810547,
      "activations/layer4_attention_weight_min": -87.585693359375,
      "activations/layer5_attention_weight_max": 67.40582275390625,
      "activations/layer5_attention_weight_min": -71.7838134765625,
      "activations/layer6_attention_weight_max": 50.543556213378906,
      "activations/layer6_attention_weight_min": -51.15725326538086,
      "activations/layer7_attention_weight_max": 66.18922424316406,
      "activations/layer7_attention_weight_min": -64.28239440917969,
      "activations/layer8_attention_weight_max": 46.79241180419922,
      "activations/layer8_attention_weight_min": -50.42671203613281,
      "activations/layer9_attention_weight_max": 46.077857971191406,
      "activations/layer9_attention_weight_min": -47.67209243774414,
      "epoch": 14.77,
      "learning_rate": 5.528787878787878e-05,
      "loss": 2.7629,
      "step": 254250
    },
    {
      "activations/layer0_attention_weight_max": 15.364340782165527,
      "activations/layer0_attention_weight_min": -13.005949020385742,
      "activations/layer10_attention_weight_max": 38.18559646606445,
      "activations/layer10_attention_weight_min": -33.77783966064453,
      "activations/layer11_attention_weight_max": 34.74153518676758,
      "activations/layer11_attention_weight_min": -32.248565673828125,
      "activations/layer12_attention_weight_max": 19.78008460998535,
      "activations/layer12_attention_weight_min": -29.349533081054688,
      "activations/layer13_attention_weight_max": 38.628692626953125,
      "activations/layer13_attention_weight_min": -28.758249282836914,
      "activations/layer14_attention_weight_max": 38.29685592651367,
      "activations/layer14_attention_weight_min": -32.305545806884766,
      "activations/layer15_attention_weight_max": 34.43492889404297,
      "activations/layer15_attention_weight_min": -30.8553524017334,
      "activations/layer16_attention_weight_max": 34.46718215942383,
      "activations/layer16_attention_weight_min": -30.393508911132812,
      "activations/layer17_attention_weight_max": 57.79086685180664,
      "activations/layer17_attention_weight_min": -47.9962158203125,
      "activations/layer18_attention_weight_max": 50.591461181640625,
      "activations/layer18_attention_weight_min": -44.215450286865234,
      "activations/layer19_attention_weight_max": 23.846302032470703,
      "activations/layer19_attention_weight_min": -22.913740158081055,
      "activations/layer1_attention_weight_max": 16.580944061279297,
      "activations/layer1_attention_weight_min": -16.649816513061523,
      "activations/layer20_attention_weight_max": 24.653541564941406,
      "activations/layer20_attention_weight_min": -20.938148498535156,
      "activations/layer21_attention_weight_max": 38.317657470703125,
      "activations/layer21_attention_weight_min": -22.136640548706055,
      "activations/layer22_attention_weight_max": 30.528011322021484,
      "activations/layer22_attention_weight_min": -25.30620574951172,
      "activations/layer23_attention_weight_max": 43.29218673706055,
      "activations/layer23_attention_weight_min": -24.2944393157959,
      "activations/layer2_attention_weight_max": 32.241825103759766,
      "activations/layer2_attention_weight_min": -30.772138595581055,
      "activations/layer3_attention_weight_max": 93.37747192382812,
      "activations/layer3_attention_weight_min": -96.25563049316406,
      "activations/layer4_attention_weight_max": 92.92852020263672,
      "activations/layer4_attention_weight_min": -89.9395523071289,
      "activations/layer5_attention_weight_max": 71.1971206665039,
      "activations/layer5_attention_weight_min": -78.03642272949219,
      "activations/layer6_attention_weight_max": 52.50752639770508,
      "activations/layer6_attention_weight_min": -51.490135192871094,
      "activations/layer7_attention_weight_max": 63.08793640136719,
      "activations/layer7_attention_weight_min": -61.898658752441406,
      "activations/layer8_attention_weight_max": 45.927284240722656,
      "activations/layer8_attention_weight_min": -45.95637512207031,
      "activations/layer9_attention_weight_max": 46.107975006103516,
      "activations/layer9_attention_weight_min": -46.25301742553711,
      "epoch": 14.78,
      "learning_rate": 5.526893939393939e-05,
      "loss": 2.759,
      "step": 254300
    },
    {
      "activations/layer0_attention_weight_max": 15.581315040588379,
      "activations/layer0_attention_weight_min": -12.713808059692383,
      "activations/layer10_attention_weight_max": 34.46614074707031,
      "activations/layer10_attention_weight_min": -35.00712966918945,
      "activations/layer11_attention_weight_max": 31.63727569580078,
      "activations/layer11_attention_weight_min": -32.590980529785156,
      "activations/layer12_attention_weight_max": 19.33814811706543,
      "activations/layer12_attention_weight_min": -22.60091781616211,
      "activations/layer13_attention_weight_max": 39.813140869140625,
      "activations/layer13_attention_weight_min": -28.85333824157715,
      "activations/layer14_attention_weight_max": 39.20314025878906,
      "activations/layer14_attention_weight_min": -30.359134674072266,
      "activations/layer15_attention_weight_max": 36.351104736328125,
      "activations/layer15_attention_weight_min": -30.517301559448242,
      "activations/layer16_attention_weight_max": 34.252220153808594,
      "activations/layer16_attention_weight_min": -33.52848815917969,
      "activations/layer17_attention_weight_max": 55.94753646850586,
      "activations/layer17_attention_weight_min": -48.03292465209961,
      "activations/layer18_attention_weight_max": 52.23948669433594,
      "activations/layer18_attention_weight_min": -40.45791244506836,
      "activations/layer19_attention_weight_max": 24.248449325561523,
      "activations/layer19_attention_weight_min": -23.693376541137695,
      "activations/layer1_attention_weight_max": 15.892091751098633,
      "activations/layer1_attention_weight_min": -15.099404335021973,
      "activations/layer20_attention_weight_max": 25.23592185974121,
      "activations/layer20_attention_weight_min": -22.375246047973633,
      "activations/layer21_attention_weight_max": 38.4913330078125,
      "activations/layer21_attention_weight_min": -21.886098861694336,
      "activations/layer22_attention_weight_max": 34.71354675292969,
      "activations/layer22_attention_weight_min": -25.12994384765625,
      "activations/layer23_attention_weight_max": 37.43661117553711,
      "activations/layer23_attention_weight_min": -25.743934631347656,
      "activations/layer2_attention_weight_max": 27.209896087646484,
      "activations/layer2_attention_weight_min": -27.174163818359375,
      "activations/layer3_attention_weight_max": 85.77574157714844,
      "activations/layer3_attention_weight_min": -83.86473083496094,
      "activations/layer4_attention_weight_max": 89.28697967529297,
      "activations/layer4_attention_weight_min": -88.14735412597656,
      "activations/layer5_attention_weight_max": 66.356689453125,
      "activations/layer5_attention_weight_min": -78.17390441894531,
      "activations/layer6_attention_weight_max": 51.43785858154297,
      "activations/layer6_attention_weight_min": -49.13883972167969,
      "activations/layer7_attention_weight_max": 65.24928283691406,
      "activations/layer7_attention_weight_min": -61.458370208740234,
      "activations/layer8_attention_weight_max": 47.81864929199219,
      "activations/layer8_attention_weight_min": -49.179420471191406,
      "activations/layer9_attention_weight_max": 45.108482360839844,
      "activations/layer9_attention_weight_min": -44.35149383544922,
      "epoch": 14.78,
      "learning_rate": 5.5249999999999994e-05,
      "loss": 2.7735,
      "step": 254350
    },
    {
      "activations/layer0_attention_weight_max": 15.862759590148926,
      "activations/layer0_attention_weight_min": -13.085742950439453,
      "activations/layer10_attention_weight_max": 36.68643569946289,
      "activations/layer10_attention_weight_min": -36.23426055908203,
      "activations/layer11_attention_weight_max": 32.95154571533203,
      "activations/layer11_attention_weight_min": -33.49715805053711,
      "activations/layer12_attention_weight_max": 19.929784774780273,
      "activations/layer12_attention_weight_min": -32.1564826965332,
      "activations/layer13_attention_weight_max": 30.989206314086914,
      "activations/layer13_attention_weight_min": -31.22430419921875,
      "activations/layer14_attention_weight_max": 37.24416732788086,
      "activations/layer14_attention_weight_min": -29.825775146484375,
      "activations/layer15_attention_weight_max": 33.13189697265625,
      "activations/layer15_attention_weight_min": -30.253061294555664,
      "activations/layer16_attention_weight_max": 35.71237564086914,
      "activations/layer16_attention_weight_min": -33.65225601196289,
      "activations/layer17_attention_weight_max": 60.57757568359375,
      "activations/layer17_attention_weight_min": -50.96598815917969,
      "activations/layer18_attention_weight_max": 47.979698181152344,
      "activations/layer18_attention_weight_min": -43.203067779541016,
      "activations/layer19_attention_weight_max": 24.0900821685791,
      "activations/layer19_attention_weight_min": -25.302534103393555,
      "activations/layer1_attention_weight_max": 16.701236724853516,
      "activations/layer1_attention_weight_min": -15.256525039672852,
      "activations/layer20_attention_weight_max": 24.100242614746094,
      "activations/layer20_attention_weight_min": -23.1722412109375,
      "activations/layer21_attention_weight_max": 36.58209991455078,
      "activations/layer21_attention_weight_min": -24.23586082458496,
      "activations/layer22_attention_weight_max": 32.69540023803711,
      "activations/layer22_attention_weight_min": -27.38079261779785,
      "activations/layer23_attention_weight_max": 38.19773864746094,
      "activations/layer23_attention_weight_min": -23.61608123779297,
      "activations/layer2_attention_weight_max": 30.337003707885742,
      "activations/layer2_attention_weight_min": -29.845539093017578,
      "activations/layer3_attention_weight_max": 85.80073547363281,
      "activations/layer3_attention_weight_min": -90.72093200683594,
      "activations/layer4_attention_weight_max": 87.50074005126953,
      "activations/layer4_attention_weight_min": -83.96922302246094,
      "activations/layer5_attention_weight_max": 65.03287506103516,
      "activations/layer5_attention_weight_min": -73.8046875,
      "activations/layer6_attention_weight_max": 47.69344711303711,
      "activations/layer6_attention_weight_min": -50.18211364746094,
      "activations/layer7_attention_weight_max": 64.14273834228516,
      "activations/layer7_attention_weight_min": -64.12562561035156,
      "activations/layer8_attention_weight_max": 46.66683578491211,
      "activations/layer8_attention_weight_min": -49.83242416381836,
      "activations/layer9_attention_weight_max": 47.92876052856445,
      "activations/layer9_attention_weight_min": -45.48097229003906,
      "epoch": 14.78,
      "learning_rate": 5.52310606060606e-05,
      "loss": 2.7553,
      "step": 254400
    },
    {
      "activations/layer0_attention_weight_max": 16.330617904663086,
      "activations/layer0_attention_weight_min": -12.787697792053223,
      "activations/layer10_attention_weight_max": 38.90456008911133,
      "activations/layer10_attention_weight_min": -39.09002685546875,
      "activations/layer11_attention_weight_max": 36.40415954589844,
      "activations/layer11_attention_weight_min": -36.445560455322266,
      "activations/layer12_attention_weight_max": 19.762250900268555,
      "activations/layer12_attention_weight_min": -27.53929901123047,
      "activations/layer13_attention_weight_max": 36.27642059326172,
      "activations/layer13_attention_weight_min": -28.81096839904785,
      "activations/layer14_attention_weight_max": 38.75865936279297,
      "activations/layer14_attention_weight_min": -31.372920989990234,
      "activations/layer15_attention_weight_max": 36.407413482666016,
      "activations/layer15_attention_weight_min": -30.09699249267578,
      "activations/layer16_attention_weight_max": 34.68966293334961,
      "activations/layer16_attention_weight_min": -31.963159561157227,
      "activations/layer17_attention_weight_max": 56.43437957763672,
      "activations/layer17_attention_weight_min": -48.57265090942383,
      "activations/layer18_attention_weight_max": 50.26065444946289,
      "activations/layer18_attention_weight_min": -40.0516357421875,
      "activations/layer19_attention_weight_max": 24.540590286254883,
      "activations/layer19_attention_weight_min": -23.372812271118164,
      "activations/layer1_attention_weight_max": 16.67498779296875,
      "activations/layer1_attention_weight_min": -14.765439987182617,
      "activations/layer20_attention_weight_max": 26.974443435668945,
      "activations/layer20_attention_weight_min": -22.885774612426758,
      "activations/layer21_attention_weight_max": 48.499168395996094,
      "activations/layer21_attention_weight_min": -26.07958221435547,
      "activations/layer22_attention_weight_max": 32.64912414550781,
      "activations/layer22_attention_weight_min": -26.86927604675293,
      "activations/layer23_attention_weight_max": 41.85630416870117,
      "activations/layer23_attention_weight_min": -24.395164489746094,
      "activations/layer2_attention_weight_max": 32.7867431640625,
      "activations/layer2_attention_weight_min": -32.075557708740234,
      "activations/layer3_attention_weight_max": 95.39862823486328,
      "activations/layer3_attention_weight_min": -100.18326568603516,
      "activations/layer4_attention_weight_max": 98.68328094482422,
      "activations/layer4_attention_weight_min": -95.55115509033203,
      "activations/layer5_attention_weight_max": 70.16336059570312,
      "activations/layer5_attention_weight_min": -80.62784576416016,
      "activations/layer6_attention_weight_max": 55.31647872924805,
      "activations/layer6_attention_weight_min": -56.102420806884766,
      "activations/layer7_attention_weight_max": 66.27643585205078,
      "activations/layer7_attention_weight_min": -73.92994689941406,
      "activations/layer8_attention_weight_max": 48.77244567871094,
      "activations/layer8_attention_weight_min": -52.946659088134766,
      "activations/layer9_attention_weight_max": 49.18744659423828,
      "activations/layer9_attention_weight_min": -51.492042541503906,
      "epoch": 14.78,
      "learning_rate": 5.5212121212121204e-05,
      "loss": 2.7764,
      "step": 254450
    },
    {
      "activations/layer0_attention_weight_max": 16.722932815551758,
      "activations/layer0_attention_weight_min": -12.678720474243164,
      "activations/layer10_attention_weight_max": 33.44704818725586,
      "activations/layer10_attention_weight_min": -33.94786834716797,
      "activations/layer11_attention_weight_max": 30.185386657714844,
      "activations/layer11_attention_weight_min": -31.706165313720703,
      "activations/layer12_attention_weight_max": 18.981470108032227,
      "activations/layer12_attention_weight_min": -35.6499137878418,
      "activations/layer13_attention_weight_max": 38.38615798950195,
      "activations/layer13_attention_weight_min": -28.967605590820312,
      "activations/layer14_attention_weight_max": 34.85529327392578,
      "activations/layer14_attention_weight_min": -30.325132369995117,
      "activations/layer15_attention_weight_max": 30.395505905151367,
      "activations/layer15_attention_weight_min": -29.885250091552734,
      "activations/layer16_attention_weight_max": 30.861799240112305,
      "activations/layer16_attention_weight_min": -29.715404510498047,
      "activations/layer17_attention_weight_max": 46.12754440307617,
      "activations/layer17_attention_weight_min": -42.62889099121094,
      "activations/layer18_attention_weight_max": 41.57173156738281,
      "activations/layer18_attention_weight_min": -38.376365661621094,
      "activations/layer19_attention_weight_max": 21.238330841064453,
      "activations/layer19_attention_weight_min": -22.223012924194336,
      "activations/layer1_attention_weight_max": 18.436491012573242,
      "activations/layer1_attention_weight_min": -15.269513130187988,
      "activations/layer20_attention_weight_max": 19.849018096923828,
      "activations/layer20_attention_weight_min": -22.516036987304688,
      "activations/layer21_attention_weight_max": 32.00175476074219,
      "activations/layer21_attention_weight_min": -22.82217788696289,
      "activations/layer22_attention_weight_max": 27.800352096557617,
      "activations/layer22_attention_weight_min": -25.608797073364258,
      "activations/layer23_attention_weight_max": 34.28723907470703,
      "activations/layer23_attention_weight_min": -24.263084411621094,
      "activations/layer2_attention_weight_max": 30.416641235351562,
      "activations/layer2_attention_weight_min": -29.389759063720703,
      "activations/layer3_attention_weight_max": 89.2121810913086,
      "activations/layer3_attention_weight_min": -96.5812759399414,
      "activations/layer4_attention_weight_max": 84.48686981201172,
      "activations/layer4_attention_weight_min": -90.02661895751953,
      "activations/layer5_attention_weight_max": 66.88014221191406,
      "activations/layer5_attention_weight_min": -72.68223571777344,
      "activations/layer6_attention_weight_max": 50.16918182373047,
      "activations/layer6_attention_weight_min": -49.493003845214844,
      "activations/layer7_attention_weight_max": 60.842681884765625,
      "activations/layer7_attention_weight_min": -63.29297637939453,
      "activations/layer8_attention_weight_max": 44.06923294067383,
      "activations/layer8_attention_weight_min": -49.12813949584961,
      "activations/layer9_attention_weight_max": 43.17310333251953,
      "activations/layer9_attention_weight_min": -45.26092529296875,
      "epoch": 14.79,
      "learning_rate": 5.519318181818182e-05,
      "loss": 2.7419,
      "step": 254500
    },
    {
      "activations/layer0_attention_weight_max": 16.399686813354492,
      "activations/layer0_attention_weight_min": -12.818909645080566,
      "activations/layer10_attention_weight_max": 37.22554397583008,
      "activations/layer10_attention_weight_min": -37.004390716552734,
      "activations/layer11_attention_weight_max": 33.192317962646484,
      "activations/layer11_attention_weight_min": -34.16999053955078,
      "activations/layer12_attention_weight_max": 19.835975646972656,
      "activations/layer12_attention_weight_min": -24.849815368652344,
      "activations/layer13_attention_weight_max": 36.33487319946289,
      "activations/layer13_attention_weight_min": -28.990875244140625,
      "activations/layer14_attention_weight_max": 33.255985260009766,
      "activations/layer14_attention_weight_min": -29.130205154418945,
      "activations/layer15_attention_weight_max": 31.691608428955078,
      "activations/layer15_attention_weight_min": -29.16926383972168,
      "activations/layer16_attention_weight_max": 34.753414154052734,
      "activations/layer16_attention_weight_min": -31.948156356811523,
      "activations/layer17_attention_weight_max": 54.87156295776367,
      "activations/layer17_attention_weight_min": -50.1562385559082,
      "activations/layer18_attention_weight_max": 49.35953140258789,
      "activations/layer18_attention_weight_min": -38.044681549072266,
      "activations/layer19_attention_weight_max": 23.36655044555664,
      "activations/layer19_attention_weight_min": -20.370553970336914,
      "activations/layer1_attention_weight_max": 17.418209075927734,
      "activations/layer1_attention_weight_min": -16.58667755126953,
      "activations/layer20_attention_weight_max": 22.42276382446289,
      "activations/layer20_attention_weight_min": -25.850231170654297,
      "activations/layer21_attention_weight_max": 32.31218338012695,
      "activations/layer21_attention_weight_min": -20.219837188720703,
      "activations/layer22_attention_weight_max": 31.53048324584961,
      "activations/layer22_attention_weight_min": -24.543455123901367,
      "activations/layer23_attention_weight_max": 38.596805572509766,
      "activations/layer23_attention_weight_min": -24.5875244140625,
      "activations/layer2_attention_weight_max": 28.579349517822266,
      "activations/layer2_attention_weight_min": -29.961090087890625,
      "activations/layer3_attention_weight_max": 86.28360748291016,
      "activations/layer3_attention_weight_min": -90.96501922607422,
      "activations/layer4_attention_weight_max": 89.31896209716797,
      "activations/layer4_attention_weight_min": -91.08760833740234,
      "activations/layer5_attention_weight_max": 66.26657104492188,
      "activations/layer5_attention_weight_min": -78.82472229003906,
      "activations/layer6_attention_weight_max": 50.05617904663086,
      "activations/layer6_attention_weight_min": -51.37968063354492,
      "activations/layer7_attention_weight_max": 63.74725341796875,
      "activations/layer7_attention_weight_min": -64.5036392211914,
      "activations/layer8_attention_weight_max": 47.580657958984375,
      "activations/layer8_attention_weight_min": -49.4994010925293,
      "activations/layer9_attention_weight_max": 46.134071350097656,
      "activations/layer9_attention_weight_min": -46.008323669433594,
      "epoch": 14.79,
      "learning_rate": 5.517424242424242e-05,
      "loss": 2.7524,
      "step": 254550
    },
    {
      "activations/layer0_attention_weight_max": 16.046125411987305,
      "activations/layer0_attention_weight_min": -12.806732177734375,
      "activations/layer10_attention_weight_max": 36.451759338378906,
      "activations/layer10_attention_weight_min": -33.706459045410156,
      "activations/layer11_attention_weight_max": 33.02377700805664,
      "activations/layer11_attention_weight_min": -32.5326042175293,
      "activations/layer12_attention_weight_max": 18.381982803344727,
      "activations/layer12_attention_weight_min": -34.28125,
      "activations/layer13_attention_weight_max": 38.498226165771484,
      "activations/layer13_attention_weight_min": -31.868732452392578,
      "activations/layer14_attention_weight_max": 32.39002990722656,
      "activations/layer14_attention_weight_min": -30.370786666870117,
      "activations/layer15_attention_weight_max": 30.65709686279297,
      "activations/layer15_attention_weight_min": -28.433494567871094,
      "activations/layer16_attention_weight_max": 29.76951789855957,
      "activations/layer16_attention_weight_min": -30.074033737182617,
      "activations/layer17_attention_weight_max": 49.00095748901367,
      "activations/layer17_attention_weight_min": -45.44606399536133,
      "activations/layer18_attention_weight_max": 41.89442443847656,
      "activations/layer18_attention_weight_min": -39.17647171020508,
      "activations/layer19_attention_weight_max": 23.44982147216797,
      "activations/layer19_attention_weight_min": -21.597923278808594,
      "activations/layer1_attention_weight_max": 16.8758602142334,
      "activations/layer1_attention_weight_min": -16.856369018554688,
      "activations/layer20_attention_weight_max": 20.262361526489258,
      "activations/layer20_attention_weight_min": -21.34473991394043,
      "activations/layer21_attention_weight_max": 32.54852294921875,
      "activations/layer21_attention_weight_min": -22.690067291259766,
      "activations/layer22_attention_weight_max": 27.25617218017578,
      "activations/layer22_attention_weight_min": -26.414657592773438,
      "activations/layer23_attention_weight_max": 34.39302062988281,
      "activations/layer23_attention_weight_min": -25.50893211364746,
      "activations/layer2_attention_weight_max": 31.087871551513672,
      "activations/layer2_attention_weight_min": -32.39616394042969,
      "activations/layer3_attention_weight_max": 93.17866516113281,
      "activations/layer3_attention_weight_min": -90.73849487304688,
      "activations/layer4_attention_weight_max": 90.55834197998047,
      "activations/layer4_attention_weight_min": -84.59517669677734,
      "activations/layer5_attention_weight_max": 68.02352142333984,
      "activations/layer5_attention_weight_min": -71.61434173583984,
      "activations/layer6_attention_weight_max": 48.5855827331543,
      "activations/layer6_attention_weight_min": -49.78170394897461,
      "activations/layer7_attention_weight_max": 65.38862609863281,
      "activations/layer7_attention_weight_min": -61.939544677734375,
      "activations/layer8_attention_weight_max": 46.22812271118164,
      "activations/layer8_attention_weight_min": -48.98881149291992,
      "activations/layer9_attention_weight_max": 49.05840301513672,
      "activations/layer9_attention_weight_min": -46.722557067871094,
      "epoch": 14.79,
      "learning_rate": 5.515530303030302e-05,
      "loss": 2.7539,
      "step": 254600
    },
    {
      "activations/layer0_attention_weight_max": 16.546436309814453,
      "activations/layer0_attention_weight_min": -13.047161102294922,
      "activations/layer10_attention_weight_max": 36.697547912597656,
      "activations/layer10_attention_weight_min": -34.9852409362793,
      "activations/layer11_attention_weight_max": 34.11212158203125,
      "activations/layer11_attention_weight_min": -32.561309814453125,
      "activations/layer12_attention_weight_max": 20.832412719726562,
      "activations/layer12_attention_weight_min": -25.517398834228516,
      "activations/layer13_attention_weight_max": 33.88063049316406,
      "activations/layer13_attention_weight_min": -29.07428550720215,
      "activations/layer14_attention_weight_max": 35.6702995300293,
      "activations/layer14_attention_weight_min": -30.2926082611084,
      "activations/layer15_attention_weight_max": 32.1812629699707,
      "activations/layer15_attention_weight_min": -30.291553497314453,
      "activations/layer16_attention_weight_max": 30.949214935302734,
      "activations/layer16_attention_weight_min": -31.99884033203125,
      "activations/layer17_attention_weight_max": 50.32715606689453,
      "activations/layer17_attention_weight_min": -48.14936828613281,
      "activations/layer18_attention_weight_max": 42.45148468017578,
      "activations/layer18_attention_weight_min": -40.01189422607422,
      "activations/layer19_attention_weight_max": 21.251806259155273,
      "activations/layer19_attention_weight_min": -22.798336029052734,
      "activations/layer1_attention_weight_max": 15.6539945602417,
      "activations/layer1_attention_weight_min": -15.92776870727539,
      "activations/layer20_attention_weight_max": 21.764833450317383,
      "activations/layer20_attention_weight_min": -20.973134994506836,
      "activations/layer21_attention_weight_max": 35.87994384765625,
      "activations/layer21_attention_weight_min": -20.22197151184082,
      "activations/layer22_attention_weight_max": 27.55116844177246,
      "activations/layer22_attention_weight_min": -27.66999053955078,
      "activations/layer23_attention_weight_max": 37.20731735229492,
      "activations/layer23_attention_weight_min": -25.030414581298828,
      "activations/layer2_attention_weight_max": 31.69293212890625,
      "activations/layer2_attention_weight_min": -30.67426109313965,
      "activations/layer3_attention_weight_max": 88.98941802978516,
      "activations/layer3_attention_weight_min": -89.02742767333984,
      "activations/layer4_attention_weight_max": 90.28260040283203,
      "activations/layer4_attention_weight_min": -83.14649963378906,
      "activations/layer5_attention_weight_max": 65.57740020751953,
      "activations/layer5_attention_weight_min": -69.57034301757812,
      "activations/layer6_attention_weight_max": 50.29824447631836,
      "activations/layer6_attention_weight_min": -48.19059371948242,
      "activations/layer7_attention_weight_max": 62.25584030151367,
      "activations/layer7_attention_weight_min": -60.97124481201172,
      "activations/layer8_attention_weight_max": 45.71076965332031,
      "activations/layer8_attention_weight_min": -47.50350570678711,
      "activations/layer9_attention_weight_max": 47.28670120239258,
      "activations/layer9_attention_weight_min": -43.89968490600586,
      "epoch": 14.8,
      "learning_rate": 5.513674242424242e-05,
      "loss": 2.7554,
      "step": 254650
    },
    {
      "activations/layer0_attention_weight_max": 16.221607208251953,
      "activations/layer0_attention_weight_min": -13.454739570617676,
      "activations/layer10_attention_weight_max": 37.40610885620117,
      "activations/layer10_attention_weight_min": -33.68843460083008,
      "activations/layer11_attention_weight_max": 32.446136474609375,
      "activations/layer11_attention_weight_min": -32.579261779785156,
      "activations/layer12_attention_weight_max": 18.03863525390625,
      "activations/layer12_attention_weight_min": -23.87342071533203,
      "activations/layer13_attention_weight_max": 36.382450103759766,
      "activations/layer13_attention_weight_min": -26.74077796936035,
      "activations/layer14_attention_weight_max": 35.72972869873047,
      "activations/layer14_attention_weight_min": -29.912988662719727,
      "activations/layer15_attention_weight_max": 31.714134216308594,
      "activations/layer15_attention_weight_min": -31.41328239440918,
      "activations/layer16_attention_weight_max": 32.97919845581055,
      "activations/layer16_attention_weight_min": -29.975807189941406,
      "activations/layer17_attention_weight_max": 49.14848709106445,
      "activations/layer17_attention_weight_min": -44.10675048828125,
      "activations/layer18_attention_weight_max": 46.54928207397461,
      "activations/layer18_attention_weight_min": -39.54602813720703,
      "activations/layer19_attention_weight_max": 23.862960815429688,
      "activations/layer19_attention_weight_min": -24.097270965576172,
      "activations/layer1_attention_weight_max": 18.433345794677734,
      "activations/layer1_attention_weight_min": -17.72193145751953,
      "activations/layer20_attention_weight_max": 21.96900749206543,
      "activations/layer20_attention_weight_min": -23.17170524597168,
      "activations/layer21_attention_weight_max": 37.61345291137695,
      "activations/layer21_attention_weight_min": -22.675119400024414,
      "activations/layer22_attention_weight_max": 31.264402389526367,
      "activations/layer22_attention_weight_min": -25.05300521850586,
      "activations/layer23_attention_weight_max": 37.782203674316406,
      "activations/layer23_attention_weight_min": -23.198165893554688,
      "activations/layer2_attention_weight_max": 32.6002197265625,
      "activations/layer2_attention_weight_min": -31.30841064453125,
      "activations/layer3_attention_weight_max": 89.99897766113281,
      "activations/layer3_attention_weight_min": -92.73701477050781,
      "activations/layer4_attention_weight_max": 89.41989135742188,
      "activations/layer4_attention_weight_min": -87.5515365600586,
      "activations/layer5_attention_weight_max": 69.740234375,
      "activations/layer5_attention_weight_min": -73.39520263671875,
      "activations/layer6_attention_weight_max": 52.61380386352539,
      "activations/layer6_attention_weight_min": -51.822940826416016,
      "activations/layer7_attention_weight_max": 64.18751525878906,
      "activations/layer7_attention_weight_min": -65.78567504882812,
      "activations/layer8_attention_weight_max": 46.89065170288086,
      "activations/layer8_attention_weight_min": -48.9853515625,
      "activations/layer9_attention_weight_max": 44.50099182128906,
      "activations/layer9_attention_weight_min": -45.076812744140625,
      "epoch": 14.8,
      "learning_rate": 5.511780303030303e-05,
      "loss": 2.7711,
      "step": 254700
    },
    {
      "activations/layer0_attention_weight_max": 16.226224899291992,
      "activations/layer0_attention_weight_min": -12.955358505249023,
      "activations/layer10_attention_weight_max": 40.329566955566406,
      "activations/layer10_attention_weight_min": -37.49095916748047,
      "activations/layer11_attention_weight_max": 35.99468994140625,
      "activations/layer11_attention_weight_min": -36.10394287109375,
      "activations/layer12_attention_weight_max": 18.89947509765625,
      "activations/layer12_attention_weight_min": -28.275880813598633,
      "activations/layer13_attention_weight_max": 34.2036247253418,
      "activations/layer13_attention_weight_min": -27.292219161987305,
      "activations/layer14_attention_weight_max": 35.30829620361328,
      "activations/layer14_attention_weight_min": -30.584300994873047,
      "activations/layer15_attention_weight_max": 31.174428939819336,
      "activations/layer15_attention_weight_min": -30.287290573120117,
      "activations/layer16_attention_weight_max": 30.36045265197754,
      "activations/layer16_attention_weight_min": -28.627527236938477,
      "activations/layer17_attention_weight_max": 47.32735061645508,
      "activations/layer17_attention_weight_min": -41.60891342163086,
      "activations/layer18_attention_weight_max": 42.88813400268555,
      "activations/layer18_attention_weight_min": -39.065982818603516,
      "activations/layer19_attention_weight_max": 20.187002182006836,
      "activations/layer19_attention_weight_min": -20.97674560546875,
      "activations/layer1_attention_weight_max": 17.303361892700195,
      "activations/layer1_attention_weight_min": -16.399858474731445,
      "activations/layer20_attention_weight_max": 22.271995544433594,
      "activations/layer20_attention_weight_min": -21.856971740722656,
      "activations/layer21_attention_weight_max": 34.390968322753906,
      "activations/layer21_attention_weight_min": -25.093839645385742,
      "activations/layer22_attention_weight_max": 29.53059196472168,
      "activations/layer22_attention_weight_min": -25.913816452026367,
      "activations/layer23_attention_weight_max": 36.671478271484375,
      "activations/layer23_attention_weight_min": -23.41358184814453,
      "activations/layer2_attention_weight_max": 33.044044494628906,
      "activations/layer2_attention_weight_min": -29.704387664794922,
      "activations/layer3_attention_weight_max": 90.05628204345703,
      "activations/layer3_attention_weight_min": -93.47393798828125,
      "activations/layer4_attention_weight_max": 91.24467468261719,
      "activations/layer4_attention_weight_min": -88.34660339355469,
      "activations/layer5_attention_weight_max": 68.640380859375,
      "activations/layer5_attention_weight_min": -74.90816497802734,
      "activations/layer6_attention_weight_max": 52.99955749511719,
      "activations/layer6_attention_weight_min": -51.74769592285156,
      "activations/layer7_attention_weight_max": 66.10456085205078,
      "activations/layer7_attention_weight_min": -61.97478485107422,
      "activations/layer8_attention_weight_max": 46.36100387573242,
      "activations/layer8_attention_weight_min": -47.75724411010742,
      "activations/layer9_attention_weight_max": 50.39286422729492,
      "activations/layer9_attention_weight_min": -46.174461364746094,
      "epoch": 14.8,
      "learning_rate": 5.509886363636363e-05,
      "loss": 2.7562,
      "step": 254750
    },
    {
      "activations/layer0_attention_weight_max": 16.342227935791016,
      "activations/layer0_attention_weight_min": -13.209515571594238,
      "activations/layer10_attention_weight_max": 35.65068054199219,
      "activations/layer10_attention_weight_min": -35.765316009521484,
      "activations/layer11_attention_weight_max": 31.42978286743164,
      "activations/layer11_attention_weight_min": -33.19158172607422,
      "activations/layer12_attention_weight_max": 19.6857852935791,
      "activations/layer12_attention_weight_min": -25.480478286743164,
      "activations/layer13_attention_weight_max": 33.12786102294922,
      "activations/layer13_attention_weight_min": -26.875925064086914,
      "activations/layer14_attention_weight_max": 36.705318450927734,
      "activations/layer14_attention_weight_min": -32.880008697509766,
      "activations/layer15_attention_weight_max": 34.40177917480469,
      "activations/layer15_attention_weight_min": -29.84088897705078,
      "activations/layer16_attention_weight_max": 33.55595779418945,
      "activations/layer16_attention_weight_min": -32.30465316772461,
      "activations/layer17_attention_weight_max": 52.145572662353516,
      "activations/layer17_attention_weight_min": -46.36989212036133,
      "activations/layer18_attention_weight_max": 48.2349853515625,
      "activations/layer18_attention_weight_min": -41.11093521118164,
      "activations/layer19_attention_weight_max": 22.67658233642578,
      "activations/layer19_attention_weight_min": -24.05733299255371,
      "activations/layer1_attention_weight_max": 17.192535400390625,
      "activations/layer1_attention_weight_min": -16.3732852935791,
      "activations/layer20_attention_weight_max": 25.2932186126709,
      "activations/layer20_attention_weight_min": -20.525331497192383,
      "activations/layer21_attention_weight_max": 33.29510498046875,
      "activations/layer21_attention_weight_min": -24.632841110229492,
      "activations/layer22_attention_weight_max": 30.092859268188477,
      "activations/layer22_attention_weight_min": -26.498212814331055,
      "activations/layer23_attention_weight_max": 36.91417694091797,
      "activations/layer23_attention_weight_min": -26.58129119873047,
      "activations/layer2_attention_weight_max": 30.111572265625,
      "activations/layer2_attention_weight_min": -29.011093139648438,
      "activations/layer3_attention_weight_max": 87.50892639160156,
      "activations/layer3_attention_weight_min": -88.01933288574219,
      "activations/layer4_attention_weight_max": 87.81415557861328,
      "activations/layer4_attention_weight_min": -89.49263000488281,
      "activations/layer5_attention_weight_max": 64.29623413085938,
      "activations/layer5_attention_weight_min": -76.31615447998047,
      "activations/layer6_attention_weight_max": 48.62238311767578,
      "activations/layer6_attention_weight_min": -49.48088836669922,
      "activations/layer7_attention_weight_max": 63.57135009765625,
      "activations/layer7_attention_weight_min": -61.698604583740234,
      "activations/layer8_attention_weight_max": 43.01921081542969,
      "activations/layer8_attention_weight_min": -49.27976989746094,
      "activations/layer9_attention_weight_max": 45.2784423828125,
      "activations/layer9_attention_weight_min": -46.152099609375,
      "epoch": 14.81,
      "learning_rate": 5.507992424242424e-05,
      "loss": 2.7572,
      "step": 254800
    },
    {
      "activations/layer0_attention_weight_max": 16.058700561523438,
      "activations/layer0_attention_weight_min": -13.300750732421875,
      "activations/layer10_attention_weight_max": 38.591854095458984,
      "activations/layer10_attention_weight_min": -34.594947814941406,
      "activations/layer11_attention_weight_max": 33.342185974121094,
      "activations/layer11_attention_weight_min": -34.16934585571289,
      "activations/layer12_attention_weight_max": 20.614887237548828,
      "activations/layer12_attention_weight_min": -24.62208366394043,
      "activations/layer13_attention_weight_max": 36.81331253051758,
      "activations/layer13_attention_weight_min": -31.82286262512207,
      "activations/layer14_attention_weight_max": 38.14281463623047,
      "activations/layer14_attention_weight_min": -33.8268928527832,
      "activations/layer15_attention_weight_max": 34.69917678833008,
      "activations/layer15_attention_weight_min": -32.06947708129883,
      "activations/layer16_attention_weight_max": 33.34212112426758,
      "activations/layer16_attention_weight_min": -32.64188766479492,
      "activations/layer17_attention_weight_max": 55.03300476074219,
      "activations/layer17_attention_weight_min": -48.20121383666992,
      "activations/layer18_attention_weight_max": 51.10990905761719,
      "activations/layer18_attention_weight_min": -43.724308013916016,
      "activations/layer19_attention_weight_max": 23.317670822143555,
      "activations/layer19_attention_weight_min": -25.023046493530273,
      "activations/layer1_attention_weight_max": 17.02675437927246,
      "activations/layer1_attention_weight_min": -15.680624008178711,
      "activations/layer20_attention_weight_max": 23.289949417114258,
      "activations/layer20_attention_weight_min": -24.869773864746094,
      "activations/layer21_attention_weight_max": 41.86525344848633,
      "activations/layer21_attention_weight_min": -24.0305233001709,
      "activations/layer22_attention_weight_max": 31.368637084960938,
      "activations/layer22_attention_weight_min": -27.63047981262207,
      "activations/layer23_attention_weight_max": 39.26322937011719,
      "activations/layer23_attention_weight_min": -26.19683074951172,
      "activations/layer2_attention_weight_max": 31.482994079589844,
      "activations/layer2_attention_weight_min": -30.876239776611328,
      "activations/layer3_attention_weight_max": 91.12782287597656,
      "activations/layer3_attention_weight_min": -94.24658203125,
      "activations/layer4_attention_weight_max": 92.94867706298828,
      "activations/layer4_attention_weight_min": -90.06160736083984,
      "activations/layer5_attention_weight_max": 68.79485321044922,
      "activations/layer5_attention_weight_min": -73.70178985595703,
      "activations/layer6_attention_weight_max": 53.53102111816406,
      "activations/layer6_attention_weight_min": -52.79826354980469,
      "activations/layer7_attention_weight_max": 65.58845520019531,
      "activations/layer7_attention_weight_min": -63.478294372558594,
      "activations/layer8_attention_weight_max": 49.07819747924805,
      "activations/layer8_attention_weight_min": -52.35564422607422,
      "activations/layer9_attention_weight_max": 45.70223617553711,
      "activations/layer9_attention_weight_min": -49.97530746459961,
      "epoch": 14.81,
      "learning_rate": 5.506098484848484e-05,
      "loss": 2.7497,
      "step": 254850
    },
    {
      "activations/layer0_attention_weight_max": 16.034652709960938,
      "activations/layer0_attention_weight_min": -13.403215408325195,
      "activations/layer10_attention_weight_max": 35.98069381713867,
      "activations/layer10_attention_weight_min": -38.105712890625,
      "activations/layer11_attention_weight_max": 32.94691467285156,
      "activations/layer11_attention_weight_min": -33.707054138183594,
      "activations/layer12_attention_weight_max": 20.260700225830078,
      "activations/layer12_attention_weight_min": -28.860923767089844,
      "activations/layer13_attention_weight_max": 37.49327087402344,
      "activations/layer13_attention_weight_min": -30.487991333007812,
      "activations/layer14_attention_weight_max": 36.859764099121094,
      "activations/layer14_attention_weight_min": -34.84912872314453,
      "activations/layer15_attention_weight_max": 32.82453536987305,
      "activations/layer15_attention_weight_min": -31.644901275634766,
      "activations/layer16_attention_weight_max": 37.57890319824219,
      "activations/layer16_attention_weight_min": -36.70688247680664,
      "activations/layer17_attention_weight_max": 53.33469772338867,
      "activations/layer17_attention_weight_min": -50.929893493652344,
      "activations/layer18_attention_weight_max": 48.12531280517578,
      "activations/layer18_attention_weight_min": -45.02842330932617,
      "activations/layer19_attention_weight_max": 25.798282623291016,
      "activations/layer19_attention_weight_min": -26.879581451416016,
      "activations/layer1_attention_weight_max": 16.941370010375977,
      "activations/layer1_attention_weight_min": -18.94845199584961,
      "activations/layer20_attention_weight_max": 22.260412216186523,
      "activations/layer20_attention_weight_min": -23.976625442504883,
      "activations/layer21_attention_weight_max": 36.18599319458008,
      "activations/layer21_attention_weight_min": -23.905784606933594,
      "activations/layer22_attention_weight_max": 28.27812385559082,
      "activations/layer22_attention_weight_min": -29.76213836669922,
      "activations/layer23_attention_weight_max": 34.90330505371094,
      "activations/layer23_attention_weight_min": -26.265243530273438,
      "activations/layer2_attention_weight_max": 34.096553802490234,
      "activations/layer2_attention_weight_min": -33.297611236572266,
      "activations/layer3_attention_weight_max": 94.5419921875,
      "activations/layer3_attention_weight_min": -94.13240814208984,
      "activations/layer4_attention_weight_max": 96.05521392822266,
      "activations/layer4_attention_weight_min": -94.03459930419922,
      "activations/layer5_attention_weight_max": 72.46357727050781,
      "activations/layer5_attention_weight_min": -76.5785140991211,
      "activations/layer6_attention_weight_max": 51.284034729003906,
      "activations/layer6_attention_weight_min": -51.828487396240234,
      "activations/layer7_attention_weight_max": 66.52033996582031,
      "activations/layer7_attention_weight_min": -65.41100311279297,
      "activations/layer8_attention_weight_max": 47.549095153808594,
      "activations/layer8_attention_weight_min": -50.52827072143555,
      "activations/layer9_attention_weight_max": 46.92255783081055,
      "activations/layer9_attention_weight_min": -48.08549118041992,
      "epoch": 14.81,
      "learning_rate": 5.5042045454545454e-05,
      "loss": 2.7357,
      "step": 254900
    },
    {
      "activations/layer0_attention_weight_max": 15.87907600402832,
      "activations/layer0_attention_weight_min": -13.337432861328125,
      "activations/layer10_attention_weight_max": 40.690696716308594,
      "activations/layer10_attention_weight_min": -39.17877960205078,
      "activations/layer11_attention_weight_max": 34.56793975830078,
      "activations/layer11_attention_weight_min": -35.55012512207031,
      "activations/layer12_attention_weight_max": 19.538414001464844,
      "activations/layer12_attention_weight_min": -25.058868408203125,
      "activations/layer13_attention_weight_max": 45.41196060180664,
      "activations/layer13_attention_weight_min": -31.149127960205078,
      "activations/layer14_attention_weight_max": 35.79313659667969,
      "activations/layer14_attention_weight_min": -32.09229278564453,
      "activations/layer15_attention_weight_max": 32.758277893066406,
      "activations/layer15_attention_weight_min": -31.484233856201172,
      "activations/layer16_attention_weight_max": 33.29645538330078,
      "activations/layer16_attention_weight_min": -30.832630157470703,
      "activations/layer17_attention_weight_max": 50.408348083496094,
      "activations/layer17_attention_weight_min": -48.01214599609375,
      "activations/layer18_attention_weight_max": 48.191715240478516,
      "activations/layer18_attention_weight_min": -42.74387741088867,
      "activations/layer19_attention_weight_max": 23.874126434326172,
      "activations/layer19_attention_weight_min": -23.12737274169922,
      "activations/layer1_attention_weight_max": 17.338462829589844,
      "activations/layer1_attention_weight_min": -17.81391716003418,
      "activations/layer20_attention_weight_max": 24.25926399230957,
      "activations/layer20_attention_weight_min": -26.954416275024414,
      "activations/layer21_attention_weight_max": 39.515010833740234,
      "activations/layer21_attention_weight_min": -24.50603485107422,
      "activations/layer22_attention_weight_max": 29.438251495361328,
      "activations/layer22_attention_weight_min": -28.020954132080078,
      "activations/layer23_attention_weight_max": 41.26821517944336,
      "activations/layer23_attention_weight_min": -26.81169891357422,
      "activations/layer2_attention_weight_max": 32.09929656982422,
      "activations/layer2_attention_weight_min": -31.540935516357422,
      "activations/layer3_attention_weight_max": 92.1610107421875,
      "activations/layer3_attention_weight_min": -94.51812744140625,
      "activations/layer4_attention_weight_max": 94.0561752319336,
      "activations/layer4_attention_weight_min": -92.55093383789062,
      "activations/layer5_attention_weight_max": 66.18164825439453,
      "activations/layer5_attention_weight_min": -72.86123657226562,
      "activations/layer6_attention_weight_max": 49.69956970214844,
      "activations/layer6_attention_weight_min": -52.02082443237305,
      "activations/layer7_attention_weight_max": 61.85648727416992,
      "activations/layer7_attention_weight_min": -63.83623123168945,
      "activations/layer8_attention_weight_max": 48.74311447143555,
      "activations/layer8_attention_weight_min": -51.37350845336914,
      "activations/layer9_attention_weight_max": 45.3199577331543,
      "activations/layer9_attention_weight_min": -45.418357849121094,
      "epoch": 14.81,
      "learning_rate": 5.5023106060606056e-05,
      "loss": 2.7627,
      "step": 254950
    },
    {
      "activations/layer0_attention_weight_max": 16.10557746887207,
      "activations/layer0_attention_weight_min": -13.264567375183105,
      "activations/layer10_attention_weight_max": 36.40883255004883,
      "activations/layer10_attention_weight_min": -36.54637145996094,
      "activations/layer11_attention_weight_max": 32.74014663696289,
      "activations/layer11_attention_weight_min": -35.56033706665039,
      "activations/layer12_attention_weight_max": 19.614168167114258,
      "activations/layer12_attention_weight_min": -26.421546936035156,
      "activations/layer13_attention_weight_max": 46.99301528930664,
      "activations/layer13_attention_weight_min": -30.53395652770996,
      "activations/layer14_attention_weight_max": 44.062721252441406,
      "activations/layer14_attention_weight_min": -33.24741744995117,
      "activations/layer15_attention_weight_max": 42.3853645324707,
      "activations/layer15_attention_weight_min": -31.95880699157715,
      "activations/layer16_attention_weight_max": 35.16874313354492,
      "activations/layer16_attention_weight_min": -33.525611877441406,
      "activations/layer17_attention_weight_max": 53.56294250488281,
      "activations/layer17_attention_weight_min": -47.77173614501953,
      "activations/layer18_attention_weight_max": 50.26840591430664,
      "activations/layer18_attention_weight_min": -41.48418045043945,
      "activations/layer19_attention_weight_max": 24.503950119018555,
      "activations/layer19_attention_weight_min": -22.653169631958008,
      "activations/layer1_attention_weight_max": 16.53693962097168,
      "activations/layer1_attention_weight_min": -15.15846061706543,
      "activations/layer20_attention_weight_max": 26.393985748291016,
      "activations/layer20_attention_weight_min": -21.316335678100586,
      "activations/layer21_attention_weight_max": 44.796024322509766,
      "activations/layer21_attention_weight_min": -23.409435272216797,
      "activations/layer22_attention_weight_max": 33.19651794433594,
      "activations/layer22_attention_weight_min": -27.860471725463867,
      "activations/layer23_attention_weight_max": 43.074337005615234,
      "activations/layer23_attention_weight_min": -25.585119247436523,
      "activations/layer2_attention_weight_max": 31.037837982177734,
      "activations/layer2_attention_weight_min": -30.683223724365234,
      "activations/layer3_attention_weight_max": 96.13846588134766,
      "activations/layer3_attention_weight_min": -96.30509948730469,
      "activations/layer4_attention_weight_max": 93.14726257324219,
      "activations/layer4_attention_weight_min": -90.02339172363281,
      "activations/layer5_attention_weight_max": 67.77584838867188,
      "activations/layer5_attention_weight_min": -77.69397735595703,
      "activations/layer6_attention_weight_max": 51.555965423583984,
      "activations/layer6_attention_weight_min": -52.63642883300781,
      "activations/layer7_attention_weight_max": 64.3534164428711,
      "activations/layer7_attention_weight_min": -62.504371643066406,
      "activations/layer8_attention_weight_max": 46.10757827758789,
      "activations/layer8_attention_weight_min": -49.31550598144531,
      "activations/layer9_attention_weight_max": 45.25636291503906,
      "activations/layer9_attention_weight_min": -45.42148971557617,
      "epoch": 14.82,
      "learning_rate": 5.500416666666666e-05,
      "loss": 2.7572,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_loss": 2.712890625,
      "eval_runtime": 8.4996,
      "eval_samples_per_second": 505.203,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_openwebtext_loss": 2.712890625,
      "eval_openwebtext_ppl": 15.072782365110845,
      "eval_openwebtext_runtime": 8.4996,
      "eval_openwebtext_samples_per_second": 505.203,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9385,
      "eval_wikitext_samples_per_second": 235.229,
      "step": 255000
    },
    {
      "epoch": 14.82,
      "eval_lambada_loss": 2.55859375,
      "eval_lambada_ppl": 12.917639106991658,
      "eval_lambada_runtime": 9.5206,
      "eval_lambada_samples_per_second": 511.415,
      "step": 255000
    },
    {
      "activations/layer0_attention_weight_max": 15.563358306884766,
      "activations/layer0_attention_weight_min": -13.400407791137695,
      "activations/layer10_attention_weight_max": 36.33075714111328,
      "activations/layer10_attention_weight_min": -35.2309455871582,
      "activations/layer11_attention_weight_max": 33.24253845214844,
      "activations/layer11_attention_weight_min": -32.29230499267578,
      "activations/layer12_attention_weight_max": 18.79578971862793,
      "activations/layer12_attention_weight_min": -23.967382431030273,
      "activations/layer13_attention_weight_max": 33.180870056152344,
      "activations/layer13_attention_weight_min": -29.79010009765625,
      "activations/layer14_attention_weight_max": 34.606712341308594,
      "activations/layer14_attention_weight_min": -32.26542663574219,
      "activations/layer15_attention_weight_max": 33.8365364074707,
      "activations/layer15_attention_weight_min": -30.508100509643555,
      "activations/layer16_attention_weight_max": 33.954986572265625,
      "activations/layer16_attention_weight_min": -32.63983917236328,
      "activations/layer17_attention_weight_max": 54.673152923583984,
      "activations/layer17_attention_weight_min": -48.57465744018555,
      "activations/layer18_attention_weight_max": 48.98717498779297,
      "activations/layer18_attention_weight_min": -39.73468780517578,
      "activations/layer19_attention_weight_max": 23.39021110534668,
      "activations/layer19_attention_weight_min": -22.910125732421875,
      "activations/layer1_attention_weight_max": 15.636040687561035,
      "activations/layer1_attention_weight_min": -14.135584831237793,
      "activations/layer20_attention_weight_max": 21.517637252807617,
      "activations/layer20_attention_weight_min": -22.241064071655273,
      "activations/layer21_attention_weight_max": 33.02302169799805,
      "activations/layer21_attention_weight_min": -22.892961502075195,
      "activations/layer22_attention_weight_max": 27.472900390625,
      "activations/layer22_attention_weight_min": -27.089496612548828,
      "activations/layer23_attention_weight_max": 36.07017517089844,
      "activations/layer23_attention_weight_min": -25.275907516479492,
      "activations/layer2_attention_weight_max": 30.242015838623047,
      "activations/layer2_attention_weight_min": -30.332189559936523,
      "activations/layer3_attention_weight_max": 88.48707580566406,
      "activations/layer3_attention_weight_min": -94.76765441894531,
      "activations/layer4_attention_weight_max": 90.36238861083984,
      "activations/layer4_attention_weight_min": -88.4945068359375,
      "activations/layer5_attention_weight_max": 68.55657196044922,
      "activations/layer5_attention_weight_min": -74.91864776611328,
      "activations/layer6_attention_weight_max": 50.15688705444336,
      "activations/layer6_attention_weight_min": -52.928199768066406,
      "activations/layer7_attention_weight_max": 61.59196090698242,
      "activations/layer7_attention_weight_min": -67.539794921875,
      "activations/layer8_attention_weight_max": 47.23613739013672,
      "activations/layer8_attention_weight_min": -48.39152145385742,
      "activations/layer9_attention_weight_max": 43.60813522338867,
      "activations/layer9_attention_weight_min": -45.738441467285156,
      "epoch": 14.82,
      "learning_rate": 5.4985227272727266e-05,
      "loss": 2.7432,
      "step": 255050
    },
    {
      "activations/layer0_attention_weight_max": 15.462743759155273,
      "activations/layer0_attention_weight_min": -13.1985502243042,
      "activations/layer10_attention_weight_max": 34.1963005065918,
      "activations/layer10_attention_weight_min": -35.1884651184082,
      "activations/layer11_attention_weight_max": 32.4815559387207,
      "activations/layer11_attention_weight_min": -32.79917907714844,
      "activations/layer12_attention_weight_max": 18.63441276550293,
      "activations/layer12_attention_weight_min": -25.082565307617188,
      "activations/layer13_attention_weight_max": 30.796104431152344,
      "activations/layer13_attention_weight_min": -24.68317985534668,
      "activations/layer14_attention_weight_max": 33.093666076660156,
      "activations/layer14_attention_weight_min": -29.052587509155273,
      "activations/layer15_attention_weight_max": 28.713882446289062,
      "activations/layer15_attention_weight_min": -28.79423713684082,
      "activations/layer16_attention_weight_max": 30.670324325561523,
      "activations/layer16_attention_weight_min": -29.752262115478516,
      "activations/layer17_attention_weight_max": 44.6845588684082,
      "activations/layer17_attention_weight_min": -40.5760498046875,
      "activations/layer18_attention_weight_max": 41.655609130859375,
      "activations/layer18_attention_weight_min": -36.5937385559082,
      "activations/layer19_attention_weight_max": 22.752073287963867,
      "activations/layer19_attention_weight_min": -22.024497985839844,
      "activations/layer1_attention_weight_max": 15.993526458740234,
      "activations/layer1_attention_weight_min": -13.95688247680664,
      "activations/layer20_attention_weight_max": 19.89329719543457,
      "activations/layer20_attention_weight_min": -23.206745147705078,
      "activations/layer21_attention_weight_max": 31.324819564819336,
      "activations/layer21_attention_weight_min": -21.971424102783203,
      "activations/layer22_attention_weight_max": 27.804059982299805,
      "activations/layer22_attention_weight_min": -25.75417709350586,
      "activations/layer23_attention_weight_max": 34.1681022644043,
      "activations/layer23_attention_weight_min": -24.46259117126465,
      "activations/layer2_attention_weight_max": 31.23921012878418,
      "activations/layer2_attention_weight_min": -29.422374725341797,
      "activations/layer3_attention_weight_max": 87.14591979980469,
      "activations/layer3_attention_weight_min": -87.99800872802734,
      "activations/layer4_attention_weight_max": 85.2326889038086,
      "activations/layer4_attention_weight_min": -81.56761932373047,
      "activations/layer5_attention_weight_max": 63.71805953979492,
      "activations/layer5_attention_weight_min": -74.35366821289062,
      "activations/layer6_attention_weight_max": 47.06760025024414,
      "activations/layer6_attention_weight_min": -46.63719177246094,
      "activations/layer7_attention_weight_max": 61.02019500732422,
      "activations/layer7_attention_weight_min": -59.57786560058594,
      "activations/layer8_attention_weight_max": 45.737640380859375,
      "activations/layer8_attention_weight_min": -50.726383209228516,
      "activations/layer9_attention_weight_max": 42.122413635253906,
      "activations/layer9_attention_weight_min": -44.592132568359375,
      "epoch": 14.82,
      "learning_rate": 5.4966287878787874e-05,
      "loss": 2.7711,
      "step": 255100
    },
    {
      "activations/layer0_attention_weight_max": 16.64920425415039,
      "activations/layer0_attention_weight_min": -13.304553031921387,
      "activations/layer10_attention_weight_max": 34.991233825683594,
      "activations/layer10_attention_weight_min": -36.34766387939453,
      "activations/layer11_attention_weight_max": 31.224044799804688,
      "activations/layer11_attention_weight_min": -33.91278076171875,
      "activations/layer12_attention_weight_max": 19.540699005126953,
      "activations/layer12_attention_weight_min": -27.036441802978516,
      "activations/layer13_attention_weight_max": 37.47215270996094,
      "activations/layer13_attention_weight_min": -26.156129837036133,
      "activations/layer14_attention_weight_max": 36.89614486694336,
      "activations/layer14_attention_weight_min": -30.398256301879883,
      "activations/layer15_attention_weight_max": 36.18043899536133,
      "activations/layer15_attention_weight_min": -31.785381317138672,
      "activations/layer16_attention_weight_max": 33.18503952026367,
      "activations/layer16_attention_weight_min": -29.882850646972656,
      "activations/layer17_attention_weight_max": 49.319984436035156,
      "activations/layer17_attention_weight_min": -42.654273986816406,
      "activations/layer18_attention_weight_max": 43.49634552001953,
      "activations/layer18_attention_weight_min": -37.42516326904297,
      "activations/layer19_attention_weight_max": 22.171052932739258,
      "activations/layer19_attention_weight_min": -22.559585571289062,
      "activations/layer1_attention_weight_max": 16.668718338012695,
      "activations/layer1_attention_weight_min": -13.291574478149414,
      "activations/layer20_attention_weight_max": 21.742252349853516,
      "activations/layer20_attention_weight_min": -21.28823471069336,
      "activations/layer21_attention_weight_max": 37.092811584472656,
      "activations/layer21_attention_weight_min": -22.92368507385254,
      "activations/layer22_attention_weight_max": 28.908105850219727,
      "activations/layer22_attention_weight_min": -23.515161514282227,
      "activations/layer23_attention_weight_max": 37.55469512939453,
      "activations/layer23_attention_weight_min": -25.171390533447266,
      "activations/layer2_attention_weight_max": 30.639915466308594,
      "activations/layer2_attention_weight_min": -29.724475860595703,
      "activations/layer3_attention_weight_max": 84.36638641357422,
      "activations/layer3_attention_weight_min": -87.49446105957031,
      "activations/layer4_attention_weight_max": 85.65900421142578,
      "activations/layer4_attention_weight_min": -84.44025421142578,
      "activations/layer5_attention_weight_max": 67.47607421875,
      "activations/layer5_attention_weight_min": -72.81404113769531,
      "activations/layer6_attention_weight_max": 46.91758346557617,
      "activations/layer6_attention_weight_min": -48.816585540771484,
      "activations/layer7_attention_weight_max": 63.46269226074219,
      "activations/layer7_attention_weight_min": -59.54060745239258,
      "activations/layer8_attention_weight_max": 43.74091339111328,
      "activations/layer8_attention_weight_min": -47.88407897949219,
      "activations/layer9_attention_weight_max": 45.32355499267578,
      "activations/layer9_attention_weight_min": -45.063255310058594,
      "epoch": 14.83,
      "learning_rate": 5.494734848484848e-05,
      "loss": 2.7641,
      "step": 255150
    },
    {
      "activations/layer0_attention_weight_max": 15.930328369140625,
      "activations/layer0_attention_weight_min": -13.275348663330078,
      "activations/layer10_attention_weight_max": 37.5508918762207,
      "activations/layer10_attention_weight_min": -36.08149719238281,
      "activations/layer11_attention_weight_max": 33.536861419677734,
      "activations/layer11_attention_weight_min": -35.051143646240234,
      "activations/layer12_attention_weight_max": 17.577571868896484,
      "activations/layer12_attention_weight_min": -24.499927520751953,
      "activations/layer13_attention_weight_max": 42.7215690612793,
      "activations/layer13_attention_weight_min": -29.639734268188477,
      "activations/layer14_attention_weight_max": 35.543209075927734,
      "activations/layer14_attention_weight_min": -30.419429779052734,
      "activations/layer15_attention_weight_max": 33.94463348388672,
      "activations/layer15_attention_weight_min": -30.69606590270996,
      "activations/layer16_attention_weight_max": 32.99296951293945,
      "activations/layer16_attention_weight_min": -30.475805282592773,
      "activations/layer17_attention_weight_max": 51.796630859375,
      "activations/layer17_attention_weight_min": -44.72016525268555,
      "activations/layer18_attention_weight_max": 45.92927169799805,
      "activations/layer18_attention_weight_min": -38.578182220458984,
      "activations/layer19_attention_weight_max": 20.955411911010742,
      "activations/layer19_attention_weight_min": -22.472124099731445,
      "activations/layer1_attention_weight_max": 15.997300148010254,
      "activations/layer1_attention_weight_min": -14.07927131652832,
      "activations/layer20_attention_weight_max": 21.44809341430664,
      "activations/layer20_attention_weight_min": -21.994762420654297,
      "activations/layer21_attention_weight_max": 34.563087463378906,
      "activations/layer21_attention_weight_min": -24.31368064880371,
      "activations/layer22_attention_weight_max": 29.89434051513672,
      "activations/layer22_attention_weight_min": -24.83409881591797,
      "activations/layer23_attention_weight_max": 34.07291793823242,
      "activations/layer23_attention_weight_min": -23.824453353881836,
      "activations/layer2_attention_weight_max": 30.490455627441406,
      "activations/layer2_attention_weight_min": -30.240476608276367,
      "activations/layer3_attention_weight_max": 90.96885681152344,
      "activations/layer3_attention_weight_min": -98.01251220703125,
      "activations/layer4_attention_weight_max": 92.42901611328125,
      "activations/layer4_attention_weight_min": -92.83934783935547,
      "activations/layer5_attention_weight_max": 65.17134857177734,
      "activations/layer5_attention_weight_min": -75.82337951660156,
      "activations/layer6_attention_weight_max": 51.03097915649414,
      "activations/layer6_attention_weight_min": -50.10549545288086,
      "activations/layer7_attention_weight_max": 68.53175354003906,
      "activations/layer7_attention_weight_min": -64.27481842041016,
      "activations/layer8_attention_weight_max": 46.45287322998047,
      "activations/layer8_attention_weight_min": -50.74729537963867,
      "activations/layer9_attention_weight_max": 43.72933578491211,
      "activations/layer9_attention_weight_min": -47.39628982543945,
      "epoch": 14.83,
      "learning_rate": 5.4928409090909084e-05,
      "loss": 2.7358,
      "step": 255200
    },
    {
      "activations/layer0_attention_weight_max": 15.710983276367188,
      "activations/layer0_attention_weight_min": -13.136672973632812,
      "activations/layer10_attention_weight_max": 33.706878662109375,
      "activations/layer10_attention_weight_min": -33.91720199584961,
      "activations/layer11_attention_weight_max": 29.87594985961914,
      "activations/layer11_attention_weight_min": -32.92790222167969,
      "activations/layer12_attention_weight_max": 17.563159942626953,
      "activations/layer12_attention_weight_min": -26.380252838134766,
      "activations/layer13_attention_weight_max": 38.40617752075195,
      "activations/layer13_attention_weight_min": -29.139728546142578,
      "activations/layer14_attention_weight_max": 35.511253356933594,
      "activations/layer14_attention_weight_min": -30.60334014892578,
      "activations/layer15_attention_weight_max": 32.58674621582031,
      "activations/layer15_attention_weight_min": -30.77558708190918,
      "activations/layer16_attention_weight_max": 31.150835037231445,
      "activations/layer16_attention_weight_min": -30.483613967895508,
      "activations/layer17_attention_weight_max": 50.14836883544922,
      "activations/layer17_attention_weight_min": -42.8881721496582,
      "activations/layer18_attention_weight_max": 45.3427619934082,
      "activations/layer18_attention_weight_min": -36.979583740234375,
      "activations/layer19_attention_weight_max": 24.940961837768555,
      "activations/layer19_attention_weight_min": -20.94510841369629,
      "activations/layer1_attention_weight_max": 15.757074356079102,
      "activations/layer1_attention_weight_min": -15.079347610473633,
      "activations/layer20_attention_weight_max": 22.872507095336914,
      "activations/layer20_attention_weight_min": -20.89540672302246,
      "activations/layer21_attention_weight_max": 36.30507278442383,
      "activations/layer21_attention_weight_min": -26.4328670501709,
      "activations/layer22_attention_weight_max": 29.94176483154297,
      "activations/layer22_attention_weight_min": -24.976919174194336,
      "activations/layer23_attention_weight_max": 38.30839538574219,
      "activations/layer23_attention_weight_min": -23.936399459838867,
      "activations/layer2_attention_weight_max": 30.197628021240234,
      "activations/layer2_attention_weight_min": -28.025428771972656,
      "activations/layer3_attention_weight_max": 86.31591033935547,
      "activations/layer3_attention_weight_min": -90.52608489990234,
      "activations/layer4_attention_weight_max": 86.97077178955078,
      "activations/layer4_attention_weight_min": -85.77301788330078,
      "activations/layer5_attention_weight_max": 65.67969512939453,
      "activations/layer5_attention_weight_min": -75.46136474609375,
      "activations/layer6_attention_weight_max": 50.2897834777832,
      "activations/layer6_attention_weight_min": -50.40690231323242,
      "activations/layer7_attention_weight_max": 68.12785339355469,
      "activations/layer7_attention_weight_min": -61.27000427246094,
      "activations/layer8_attention_weight_max": 43.33797836303711,
      "activations/layer8_attention_weight_min": -46.492774963378906,
      "activations/layer9_attention_weight_max": 42.174678802490234,
      "activations/layer9_attention_weight_min": -44.698524475097656,
      "epoch": 14.83,
      "learning_rate": 5.49094696969697e-05,
      "loss": 2.7754,
      "step": 255250
    },
    {
      "activations/layer0_attention_weight_max": 16.176342010498047,
      "activations/layer0_attention_weight_min": -12.991082191467285,
      "activations/layer10_attention_weight_max": 36.76866149902344,
      "activations/layer10_attention_weight_min": -35.242828369140625,
      "activations/layer11_attention_weight_max": 31.043296813964844,
      "activations/layer11_attention_weight_min": -33.232032775878906,
      "activations/layer12_attention_weight_max": 18.9700927734375,
      "activations/layer12_attention_weight_min": -25.277551651000977,
      "activations/layer13_attention_weight_max": 36.7191047668457,
      "activations/layer13_attention_weight_min": -27.52152442932129,
      "activations/layer14_attention_weight_max": 40.63395690917969,
      "activations/layer14_attention_weight_min": -31.137609481811523,
      "activations/layer15_attention_weight_max": 36.33885192871094,
      "activations/layer15_attention_weight_min": -30.601343154907227,
      "activations/layer16_attention_weight_max": 35.36082458496094,
      "activations/layer16_attention_weight_min": -30.465221405029297,
      "activations/layer17_attention_weight_max": 53.45088577270508,
      "activations/layer17_attention_weight_min": -46.05119705200195,
      "activations/layer18_attention_weight_max": 45.95330047607422,
      "activations/layer18_attention_weight_min": -42.001285552978516,
      "activations/layer19_attention_weight_max": 23.589719772338867,
      "activations/layer19_attention_weight_min": -23.874921798706055,
      "activations/layer1_attention_weight_max": 15.812959671020508,
      "activations/layer1_attention_weight_min": -15.0274019241333,
      "activations/layer20_attention_weight_max": 25.42593002319336,
      "activations/layer20_attention_weight_min": -21.811063766479492,
      "activations/layer21_attention_weight_max": 39.991390228271484,
      "activations/layer21_attention_weight_min": -25.18132972717285,
      "activations/layer22_attention_weight_max": 30.15648078918457,
      "activations/layer22_attention_weight_min": -26.684324264526367,
      "activations/layer23_attention_weight_max": 39.756874084472656,
      "activations/layer23_attention_weight_min": -26.9892578125,
      "activations/layer2_attention_weight_max": 32.85114669799805,
      "activations/layer2_attention_weight_min": -31.283687591552734,
      "activations/layer3_attention_weight_max": 90.44772338867188,
      "activations/layer3_attention_weight_min": -99.74628448486328,
      "activations/layer4_attention_weight_max": 91.46434020996094,
      "activations/layer4_attention_weight_min": -91.70671844482422,
      "activations/layer5_attention_weight_max": 67.67329406738281,
      "activations/layer5_attention_weight_min": -81.64051818847656,
      "activations/layer6_attention_weight_max": 49.40982437133789,
      "activations/layer6_attention_weight_min": -49.93486785888672,
      "activations/layer7_attention_weight_max": 63.11716079711914,
      "activations/layer7_attention_weight_min": -63.728782653808594,
      "activations/layer8_attention_weight_max": 45.11381530761719,
      "activations/layer8_attention_weight_min": -46.952579498291016,
      "activations/layer9_attention_weight_max": 45.67892074584961,
      "activations/layer9_attention_weight_min": -47.51687240600586,
      "epoch": 14.83,
      "learning_rate": 5.48905303030303e-05,
      "loss": 2.7832,
      "step": 255300
    },
    {
      "activations/layer0_attention_weight_max": 15.884840965270996,
      "activations/layer0_attention_weight_min": -13.007580757141113,
      "activations/layer10_attention_weight_max": 36.0279426574707,
      "activations/layer10_attention_weight_min": -35.491817474365234,
      "activations/layer11_attention_weight_max": 32.778892517089844,
      "activations/layer11_attention_weight_min": -34.10664367675781,
      "activations/layer12_attention_weight_max": 18.802305221557617,
      "activations/layer12_attention_weight_min": -26.63380241394043,
      "activations/layer13_attention_weight_max": 36.85700988769531,
      "activations/layer13_attention_weight_min": -26.68082046508789,
      "activations/layer14_attention_weight_max": 37.64655685424805,
      "activations/layer14_attention_weight_min": -33.259098052978516,
      "activations/layer15_attention_weight_max": 34.29940414428711,
      "activations/layer15_attention_weight_min": -30.103839874267578,
      "activations/layer16_attention_weight_max": 33.68257141113281,
      "activations/layer16_attention_weight_min": -30.22587013244629,
      "activations/layer17_attention_weight_max": 55.09063720703125,
      "activations/layer17_attention_weight_min": -45.58696746826172,
      "activations/layer18_attention_weight_max": 47.87099838256836,
      "activations/layer18_attention_weight_min": -37.72053527832031,
      "activations/layer19_attention_weight_max": 24.842344284057617,
      "activations/layer19_attention_weight_min": -22.409326553344727,
      "activations/layer1_attention_weight_max": 16.009233474731445,
      "activations/layer1_attention_weight_min": -16.684707641601562,
      "activations/layer20_attention_weight_max": 23.17934799194336,
      "activations/layer20_attention_weight_min": -24.05818748474121,
      "activations/layer21_attention_weight_max": 38.27827453613281,
      "activations/layer21_attention_weight_min": -22.30286407470703,
      "activations/layer22_attention_weight_max": 30.445392608642578,
      "activations/layer22_attention_weight_min": -27.19877052307129,
      "activations/layer23_attention_weight_max": 39.0270881652832,
      "activations/layer23_attention_weight_min": -24.530250549316406,
      "activations/layer2_attention_weight_max": 33.490718841552734,
      "activations/layer2_attention_weight_min": -32.86293411254883,
      "activations/layer3_attention_weight_max": 95.35457611083984,
      "activations/layer3_attention_weight_min": -96.63704681396484,
      "activations/layer4_attention_weight_max": 90.63416290283203,
      "activations/layer4_attention_weight_min": -91.39867401123047,
      "activations/layer5_attention_weight_max": 68.78482055664062,
      "activations/layer5_attention_weight_min": -77.31778717041016,
      "activations/layer6_attention_weight_max": 50.66697692871094,
      "activations/layer6_attention_weight_min": -52.33350372314453,
      "activations/layer7_attention_weight_max": 68.85834503173828,
      "activations/layer7_attention_weight_min": -64.57284545898438,
      "activations/layer8_attention_weight_max": 47.32246780395508,
      "activations/layer8_attention_weight_min": -49.1767692565918,
      "activations/layer9_attention_weight_max": 55.736080169677734,
      "activations/layer9_attention_weight_min": -46.18098068237305,
      "epoch": 14.84,
      "learning_rate": 5.48715909090909e-05,
      "loss": 2.7459,
      "step": 255350
    },
    {
      "activations/layer0_attention_weight_max": 15.790321350097656,
      "activations/layer0_attention_weight_min": -13.037834167480469,
      "activations/layer10_attention_weight_max": 35.73323440551758,
      "activations/layer10_attention_weight_min": -35.6693229675293,
      "activations/layer11_attention_weight_max": 33.99386978149414,
      "activations/layer11_attention_weight_min": -35.41046905517578,
      "activations/layer12_attention_weight_max": 18.715360641479492,
      "activations/layer12_attention_weight_min": -23.061786651611328,
      "activations/layer13_attention_weight_max": 32.597660064697266,
      "activations/layer13_attention_weight_min": -27.242544174194336,
      "activations/layer14_attention_weight_max": 37.555580139160156,
      "activations/layer14_attention_weight_min": -28.955215454101562,
      "activations/layer15_attention_weight_max": 34.54462432861328,
      "activations/layer15_attention_weight_min": -28.900163650512695,
      "activations/layer16_attention_weight_max": 35.13892364501953,
      "activations/layer16_attention_weight_min": -32.01654815673828,
      "activations/layer17_attention_weight_max": 51.28696823120117,
      "activations/layer17_attention_weight_min": -42.899986267089844,
      "activations/layer18_attention_weight_max": 50.881919860839844,
      "activations/layer18_attention_weight_min": -39.140472412109375,
      "activations/layer19_attention_weight_max": 24.56862449645996,
      "activations/layer19_attention_weight_min": -21.819297790527344,
      "activations/layer1_attention_weight_max": 16.85896873474121,
      "activations/layer1_attention_weight_min": -13.470342636108398,
      "activations/layer20_attention_weight_max": 24.412046432495117,
      "activations/layer20_attention_weight_min": -20.989158630371094,
      "activations/layer21_attention_weight_max": 41.5421142578125,
      "activations/layer21_attention_weight_min": -25.35407066345215,
      "activations/layer22_attention_weight_max": 27.422069549560547,
      "activations/layer22_attention_weight_min": -26.169837951660156,
      "activations/layer23_attention_weight_max": 40.750057220458984,
      "activations/layer23_attention_weight_min": -26.378131866455078,
      "activations/layer2_attention_weight_max": 28.810598373413086,
      "activations/layer2_attention_weight_min": -29.52210235595703,
      "activations/layer3_attention_weight_max": 88.13501739501953,
      "activations/layer3_attention_weight_min": -91.32760620117188,
      "activations/layer4_attention_weight_max": 86.68621063232422,
      "activations/layer4_attention_weight_min": -86.77191925048828,
      "activations/layer5_attention_weight_max": 65.53463745117188,
      "activations/layer5_attention_weight_min": -79.39497375488281,
      "activations/layer6_attention_weight_max": 48.22811508178711,
      "activations/layer6_attention_weight_min": -50.095237731933594,
      "activations/layer7_attention_weight_max": 65.9232406616211,
      "activations/layer7_attention_weight_min": -61.35637664794922,
      "activations/layer8_attention_weight_max": 50.34050369262695,
      "activations/layer8_attention_weight_min": -49.12813186645508,
      "activations/layer9_attention_weight_max": 44.06417465209961,
      "activations/layer9_attention_weight_min": -45.077632904052734,
      "epoch": 14.84,
      "learning_rate": 5.485265151515151e-05,
      "loss": 2.7786,
      "step": 255400
    },
    {
      "activations/layer0_attention_weight_max": 14.891031265258789,
      "activations/layer0_attention_weight_min": -12.969378471374512,
      "activations/layer10_attention_weight_max": 34.42384719848633,
      "activations/layer10_attention_weight_min": -35.78392791748047,
      "activations/layer11_attention_weight_max": 32.97235870361328,
      "activations/layer11_attention_weight_min": -34.12396240234375,
      "activations/layer12_attention_weight_max": 18.599435806274414,
      "activations/layer12_attention_weight_min": -23.553104400634766,
      "activations/layer13_attention_weight_max": 37.74443817138672,
      "activations/layer13_attention_weight_min": -27.986642837524414,
      "activations/layer14_attention_weight_max": 39.728233337402344,
      "activations/layer14_attention_weight_min": -30.40317726135254,
      "activations/layer15_attention_weight_max": 34.962284088134766,
      "activations/layer15_attention_weight_min": -30.131303787231445,
      "activations/layer16_attention_weight_max": 34.373355865478516,
      "activations/layer16_attention_weight_min": -31.651147842407227,
      "activations/layer17_attention_weight_max": 51.4698371887207,
      "activations/layer17_attention_weight_min": -45.57500076293945,
      "activations/layer18_attention_weight_max": 48.41122055053711,
      "activations/layer18_attention_weight_min": -38.114627838134766,
      "activations/layer19_attention_weight_max": 26.478010177612305,
      "activations/layer19_attention_weight_min": -23.955968856811523,
      "activations/layer1_attention_weight_max": 15.81761360168457,
      "activations/layer1_attention_weight_min": -14.344097137451172,
      "activations/layer20_attention_weight_max": 25.481916427612305,
      "activations/layer20_attention_weight_min": -20.77975845336914,
      "activations/layer21_attention_weight_max": 46.19742202758789,
      "activations/layer21_attention_weight_min": -24.290403366088867,
      "activations/layer22_attention_weight_max": 32.63386154174805,
      "activations/layer22_attention_weight_min": -26.29184913635254,
      "activations/layer23_attention_weight_max": 40.92173767089844,
      "activations/layer23_attention_weight_min": -24.509971618652344,
      "activations/layer2_attention_weight_max": 31.175899505615234,
      "activations/layer2_attention_weight_min": -29.178451538085938,
      "activations/layer3_attention_weight_max": 89.9534683227539,
      "activations/layer3_attention_weight_min": -93.55026245117188,
      "activations/layer4_attention_weight_max": 91.67479705810547,
      "activations/layer4_attention_weight_min": -87.81957244873047,
      "activations/layer5_attention_weight_max": 67.94428253173828,
      "activations/layer5_attention_weight_min": -74.35154724121094,
      "activations/layer6_attention_weight_max": 49.69020080566406,
      "activations/layer6_attention_weight_min": -50.9070930480957,
      "activations/layer7_attention_weight_max": 65.87020874023438,
      "activations/layer7_attention_weight_min": -61.24156188964844,
      "activations/layer8_attention_weight_max": 43.7864875793457,
      "activations/layer8_attention_weight_min": -46.56148910522461,
      "activations/layer9_attention_weight_max": 43.02248764038086,
      "activations/layer9_attention_weight_min": -44.80420684814453,
      "epoch": 14.84,
      "learning_rate": 5.483371212121212e-05,
      "loss": 2.7541,
      "step": 255450
    },
    {
      "activations/layer0_attention_weight_max": 16.535188674926758,
      "activations/layer0_attention_weight_min": -13.331585884094238,
      "activations/layer10_attention_weight_max": 35.757083892822266,
      "activations/layer10_attention_weight_min": -35.421913146972656,
      "activations/layer11_attention_weight_max": 33.8419189453125,
      "activations/layer11_attention_weight_min": -33.641944885253906,
      "activations/layer12_attention_weight_max": 19.50583839416504,
      "activations/layer12_attention_weight_min": -24.55376434326172,
      "activations/layer13_attention_weight_max": 40.035118103027344,
      "activations/layer13_attention_weight_min": -27.17745018005371,
      "activations/layer14_attention_weight_max": 40.56020736694336,
      "activations/layer14_attention_weight_min": -33.606929779052734,
      "activations/layer15_attention_weight_max": 39.61873245239258,
      "activations/layer15_attention_weight_min": -30.950172424316406,
      "activations/layer16_attention_weight_max": 35.711669921875,
      "activations/layer16_attention_weight_min": -34.07523727416992,
      "activations/layer17_attention_weight_max": 63.29944610595703,
      "activations/layer17_attention_weight_min": -52.104652404785156,
      "activations/layer18_attention_weight_max": 55.751747131347656,
      "activations/layer18_attention_weight_min": -41.92336654663086,
      "activations/layer19_attention_weight_max": 30.457021713256836,
      "activations/layer19_attention_weight_min": -24.91604232788086,
      "activations/layer1_attention_weight_max": 16.810014724731445,
      "activations/layer1_attention_weight_min": -15.461637496948242,
      "activations/layer20_attention_weight_max": 31.33447265625,
      "activations/layer20_attention_weight_min": -21.61620330810547,
      "activations/layer21_attention_weight_max": 45.0167350769043,
      "activations/layer21_attention_weight_min": -22.66912078857422,
      "activations/layer22_attention_weight_max": 37.974220275878906,
      "activations/layer22_attention_weight_min": -26.23638153076172,
      "activations/layer23_attention_weight_max": 45.83806610107422,
      "activations/layer23_attention_weight_min": -26.04364776611328,
      "activations/layer2_attention_weight_max": 31.08139419555664,
      "activations/layer2_attention_weight_min": -31.90572738647461,
      "activations/layer3_attention_weight_max": 90.1915283203125,
      "activations/layer3_attention_weight_min": -88.94066619873047,
      "activations/layer4_attention_weight_max": 91.47696685791016,
      "activations/layer4_attention_weight_min": -87.36433410644531,
      "activations/layer5_attention_weight_max": 68.45126342773438,
      "activations/layer5_attention_weight_min": -71.30184936523438,
      "activations/layer6_attention_weight_max": 51.878971099853516,
      "activations/layer6_attention_weight_min": -48.63978958129883,
      "activations/layer7_attention_weight_max": 73.31895446777344,
      "activations/layer7_attention_weight_min": -60.726402282714844,
      "activations/layer8_attention_weight_max": 46.3113899230957,
      "activations/layer8_attention_weight_min": -50.11106491088867,
      "activations/layer9_attention_weight_max": 47.11213684082031,
      "activations/layer9_attention_weight_min": -45.004295349121094,
      "epoch": 14.85,
      "learning_rate": 5.4814772727272727e-05,
      "loss": 2.7669,
      "step": 255500
    },
    {
      "activations/layer0_attention_weight_max": 15.416190147399902,
      "activations/layer0_attention_weight_min": -12.984755516052246,
      "activations/layer10_attention_weight_max": 38.987579345703125,
      "activations/layer10_attention_weight_min": -35.080360412597656,
      "activations/layer11_attention_weight_max": 33.56405258178711,
      "activations/layer11_attention_weight_min": -32.796512603759766,
      "activations/layer12_attention_weight_max": 21.11537742614746,
      "activations/layer12_attention_weight_min": -22.308916091918945,
      "activations/layer13_attention_weight_max": 38.0617561340332,
      "activations/layer13_attention_weight_min": -28.451431274414062,
      "activations/layer14_attention_weight_max": 40.1574592590332,
      "activations/layer14_attention_weight_min": -34.48408889770508,
      "activations/layer15_attention_weight_max": 36.333953857421875,
      "activations/layer15_attention_weight_min": -32.47800827026367,
      "activations/layer16_attention_weight_max": 36.702449798583984,
      "activations/layer16_attention_weight_min": -34.673583984375,
      "activations/layer17_attention_weight_max": 58.01891326904297,
      "activations/layer17_attention_weight_min": -49.534358978271484,
      "activations/layer18_attention_weight_max": 53.91443634033203,
      "activations/layer18_attention_weight_min": -44.04821014404297,
      "activations/layer19_attention_weight_max": 27.171600341796875,
      "activations/layer19_attention_weight_min": -25.33861541748047,
      "activations/layer1_attention_weight_max": 16.398828506469727,
      "activations/layer1_attention_weight_min": -15.928901672363281,
      "activations/layer20_attention_weight_max": 27.569395065307617,
      "activations/layer20_attention_weight_min": -23.01715660095215,
      "activations/layer21_attention_weight_max": 46.18950653076172,
      "activations/layer21_attention_weight_min": -21.625144958496094,
      "activations/layer22_attention_weight_max": 33.126121520996094,
      "activations/layer22_attention_weight_min": -26.005632400512695,
      "activations/layer23_attention_weight_max": 40.601837158203125,
      "activations/layer23_attention_weight_min": -24.57679557800293,
      "activations/layer2_attention_weight_max": 32.525917053222656,
      "activations/layer2_attention_weight_min": -31.35859489440918,
      "activations/layer3_attention_weight_max": 92.09840393066406,
      "activations/layer3_attention_weight_min": -91.25025177001953,
      "activations/layer4_attention_weight_max": 92.71417999267578,
      "activations/layer4_attention_weight_min": -86.5127182006836,
      "activations/layer5_attention_weight_max": 67.5156478881836,
      "activations/layer5_attention_weight_min": -74.078857421875,
      "activations/layer6_attention_weight_max": 53.15796661376953,
      "activations/layer6_attention_weight_min": -55.7235221862793,
      "activations/layer7_attention_weight_max": 68.8443374633789,
      "activations/layer7_attention_weight_min": -65.20636749267578,
      "activations/layer8_attention_weight_max": 48.010223388671875,
      "activations/layer8_attention_weight_min": -48.09465408325195,
      "activations/layer9_attention_weight_max": 50.70149230957031,
      "activations/layer9_attention_weight_min": -44.32990646362305,
      "epoch": 14.85,
      "learning_rate": 5.479583333333333e-05,
      "loss": 2.7415,
      "step": 255550
    },
    {
      "activations/layer0_attention_weight_max": 17.699329376220703,
      "activations/layer0_attention_weight_min": -13.263128280639648,
      "activations/layer10_attention_weight_max": 36.006927490234375,
      "activations/layer10_attention_weight_min": -34.78432083129883,
      "activations/layer11_attention_weight_max": 32.6870002746582,
      "activations/layer11_attention_weight_min": -32.76869201660156,
      "activations/layer12_attention_weight_max": 17.852684020996094,
      "activations/layer12_attention_weight_min": -24.60399055480957,
      "activations/layer13_attention_weight_max": 32.826271057128906,
      "activations/layer13_attention_weight_min": -30.680992126464844,
      "activations/layer14_attention_weight_max": 36.966312408447266,
      "activations/layer14_attention_weight_min": -33.38436508178711,
      "activations/layer15_attention_weight_max": 30.659631729125977,
      "activations/layer15_attention_weight_min": -30.921974182128906,
      "activations/layer16_attention_weight_max": 31.72861671447754,
      "activations/layer16_attention_weight_min": -32.164005279541016,
      "activations/layer17_attention_weight_max": 48.1395263671875,
      "activations/layer17_attention_weight_min": -44.50056838989258,
      "activations/layer18_attention_weight_max": 42.89177322387695,
      "activations/layer18_attention_weight_min": -42.26436233520508,
      "activations/layer19_attention_weight_max": 23.88666534423828,
      "activations/layer19_attention_weight_min": -22.629819869995117,
      "activations/layer1_attention_weight_max": 15.760600090026855,
      "activations/layer1_attention_weight_min": -15.028271675109863,
      "activations/layer20_attention_weight_max": 23.257083892822266,
      "activations/layer20_attention_weight_min": -22.351762771606445,
      "activations/layer21_attention_weight_max": 37.20751190185547,
      "activations/layer21_attention_weight_min": -25.61338233947754,
      "activations/layer22_attention_weight_max": 29.209157943725586,
      "activations/layer22_attention_weight_min": -26.200111389160156,
      "activations/layer23_attention_weight_max": 37.2758903503418,
      "activations/layer23_attention_weight_min": -22.459815979003906,
      "activations/layer2_attention_weight_max": 31.293155670166016,
      "activations/layer2_attention_weight_min": -30.382736206054688,
      "activations/layer3_attention_weight_max": 103.46357727050781,
      "activations/layer3_attention_weight_min": -100.89791107177734,
      "activations/layer4_attention_weight_max": 92.09882354736328,
      "activations/layer4_attention_weight_min": -93.09363555908203,
      "activations/layer5_attention_weight_max": 67.58807373046875,
      "activations/layer5_attention_weight_min": -77.09031677246094,
      "activations/layer6_attention_weight_max": 52.51457214355469,
      "activations/layer6_attention_weight_min": -54.50999450683594,
      "activations/layer7_attention_weight_max": 63.62229919433594,
      "activations/layer7_attention_weight_min": -62.55472183227539,
      "activations/layer8_attention_weight_max": 47.736717224121094,
      "activations/layer8_attention_weight_min": -47.18495559692383,
      "activations/layer9_attention_weight_max": 48.97236633300781,
      "activations/layer9_attention_weight_min": -46.14271926879883,
      "epoch": 14.85,
      "learning_rate": 5.477689393939393e-05,
      "loss": 2.7637,
      "step": 255600
    },
    {
      "activations/layer0_attention_weight_max": 15.850626945495605,
      "activations/layer0_attention_weight_min": -13.387399673461914,
      "activations/layer10_attention_weight_max": 34.41999435424805,
      "activations/layer10_attention_weight_min": -34.25141143798828,
      "activations/layer11_attention_weight_max": 33.725257873535156,
      "activations/layer11_attention_weight_min": -31.8524227142334,
      "activations/layer12_attention_weight_max": 19.049076080322266,
      "activations/layer12_attention_weight_min": -38.42107009887695,
      "activations/layer13_attention_weight_max": 37.284095764160156,
      "activations/layer13_attention_weight_min": -32.01249694824219,
      "activations/layer14_attention_weight_max": 36.558021545410156,
      "activations/layer14_attention_weight_min": -30.956764221191406,
      "activations/layer15_attention_weight_max": 34.51426696777344,
      "activations/layer15_attention_weight_min": -29.792360305786133,
      "activations/layer16_attention_weight_max": 32.72916030883789,
      "activations/layer16_attention_weight_min": -29.37197494506836,
      "activations/layer17_attention_weight_max": 46.70698165893555,
      "activations/layer17_attention_weight_min": -42.536678314208984,
      "activations/layer18_attention_weight_max": 43.96359634399414,
      "activations/layer18_attention_weight_min": -38.81096267700195,
      "activations/layer19_attention_weight_max": 24.398523330688477,
      "activations/layer19_attention_weight_min": -23.50133514404297,
      "activations/layer1_attention_weight_max": 16.790481567382812,
      "activations/layer1_attention_weight_min": -14.64979362487793,
      "activations/layer20_attention_weight_max": 23.782573699951172,
      "activations/layer20_attention_weight_min": -21.65165138244629,
      "activations/layer21_attention_weight_max": 43.46624755859375,
      "activations/layer21_attention_weight_min": -25.119516372680664,
      "activations/layer22_attention_weight_max": 28.76249122619629,
      "activations/layer22_attention_weight_min": -27.65322494506836,
      "activations/layer23_attention_weight_max": 37.795352935791016,
      "activations/layer23_attention_weight_min": -23.84676742553711,
      "activations/layer2_attention_weight_max": 30.86873435974121,
      "activations/layer2_attention_weight_min": -30.46234703063965,
      "activations/layer3_attention_weight_max": 89.20176696777344,
      "activations/layer3_attention_weight_min": -93.55303192138672,
      "activations/layer4_attention_weight_max": 92.79387664794922,
      "activations/layer4_attention_weight_min": -93.24945068359375,
      "activations/layer5_attention_weight_max": 66.1998291015625,
      "activations/layer5_attention_weight_min": -73.198974609375,
      "activations/layer6_attention_weight_max": 50.794010162353516,
      "activations/layer6_attention_weight_min": -48.521705627441406,
      "activations/layer7_attention_weight_max": 65.2847671508789,
      "activations/layer7_attention_weight_min": -61.77788162231445,
      "activations/layer8_attention_weight_max": 46.64303207397461,
      "activations/layer8_attention_weight_min": -50.99174880981445,
      "activations/layer9_attention_weight_max": 52.47499465942383,
      "activations/layer9_attention_weight_min": -50.306026458740234,
      "epoch": 14.85,
      "learning_rate": 5.4757954545454545e-05,
      "loss": 2.7658,
      "step": 255650
    },
    {
      "activations/layer0_attention_weight_max": 16.015268325805664,
      "activations/layer0_attention_weight_min": -13.10848331451416,
      "activations/layer10_attention_weight_max": 34.83642578125,
      "activations/layer10_attention_weight_min": -34.12933349609375,
      "activations/layer11_attention_weight_max": 30.480037689208984,
      "activations/layer11_attention_weight_min": -32.280704498291016,
      "activations/layer12_attention_weight_max": 17.632326126098633,
      "activations/layer12_attention_weight_min": -27.807727813720703,
      "activations/layer13_attention_weight_max": 34.56911849975586,
      "activations/layer13_attention_weight_min": -26.004667282104492,
      "activations/layer14_attention_weight_max": 35.34767150878906,
      "activations/layer14_attention_weight_min": -30.135818481445312,
      "activations/layer15_attention_weight_max": 37.86762237548828,
      "activations/layer15_attention_weight_min": -31.45847511291504,
      "activations/layer16_attention_weight_max": 30.92658233642578,
      "activations/layer16_attention_weight_min": -30.650039672851562,
      "activations/layer17_attention_weight_max": 55.25127029418945,
      "activations/layer17_attention_weight_min": -43.9711799621582,
      "activations/layer18_attention_weight_max": 46.200531005859375,
      "activations/layer18_attention_weight_min": -38.830623626708984,
      "activations/layer19_attention_weight_max": 25.342756271362305,
      "activations/layer19_attention_weight_min": -23.182836532592773,
      "activations/layer1_attention_weight_max": 17.34505271911621,
      "activations/layer1_attention_weight_min": -12.804010391235352,
      "activations/layer20_attention_weight_max": 21.503360748291016,
      "activations/layer20_attention_weight_min": -22.46835708618164,
      "activations/layer21_attention_weight_max": 33.58320999145508,
      "activations/layer21_attention_weight_min": -22.77116584777832,
      "activations/layer22_attention_weight_max": 29.97398567199707,
      "activations/layer22_attention_weight_min": -25.97250747680664,
      "activations/layer23_attention_weight_max": 35.13428497314453,
      "activations/layer23_attention_weight_min": -25.417362213134766,
      "activations/layer2_attention_weight_max": 30.608556747436523,
      "activations/layer2_attention_weight_min": -29.638694763183594,
      "activations/layer3_attention_weight_max": 86.87476348876953,
      "activations/layer3_attention_weight_min": -92.04924774169922,
      "activations/layer4_attention_weight_max": 85.07740783691406,
      "activations/layer4_attention_weight_min": -84.34102630615234,
      "activations/layer5_attention_weight_max": 66.77548217773438,
      "activations/layer5_attention_weight_min": -75.25743865966797,
      "activations/layer6_attention_weight_max": 46.97898864746094,
      "activations/layer6_attention_weight_min": -50.343101501464844,
      "activations/layer7_attention_weight_max": 60.17558670043945,
      "activations/layer7_attention_weight_min": -62.91529083251953,
      "activations/layer8_attention_weight_max": 43.31966018676758,
      "activations/layer8_attention_weight_min": -48.184444427490234,
      "activations/layer9_attention_weight_max": 43.550106048583984,
      "activations/layer9_attention_weight_min": -44.138179779052734,
      "epoch": 14.86,
      "learning_rate": 5.4739015151515146e-05,
      "loss": 2.7733,
      "step": 255700
    },
    {
      "activations/layer0_attention_weight_max": 16.784500122070312,
      "activations/layer0_attention_weight_min": -13.032361030578613,
      "activations/layer10_attention_weight_max": 31.527618408203125,
      "activations/layer10_attention_weight_min": -32.670387268066406,
      "activations/layer11_attention_weight_max": 29.356006622314453,
      "activations/layer11_attention_weight_min": -30.721487045288086,
      "activations/layer12_attention_weight_max": 18.29033660888672,
      "activations/layer12_attention_weight_min": -28.420616149902344,
      "activations/layer13_attention_weight_max": 31.53314208984375,
      "activations/layer13_attention_weight_min": -27.070669174194336,
      "activations/layer14_attention_weight_max": 33.574764251708984,
      "activations/layer14_attention_weight_min": -28.2825927734375,
      "activations/layer15_attention_weight_max": 32.97794723510742,
      "activations/layer15_attention_weight_min": -28.529653549194336,
      "activations/layer16_attention_weight_max": 31.998884201049805,
      "activations/layer16_attention_weight_min": -27.869726181030273,
      "activations/layer17_attention_weight_max": 48.32585906982422,
      "activations/layer17_attention_weight_min": -42.655330657958984,
      "activations/layer18_attention_weight_max": 44.33865737915039,
      "activations/layer18_attention_weight_min": -37.4680290222168,
      "activations/layer19_attention_weight_max": 22.502765655517578,
      "activations/layer19_attention_weight_min": -25.184284210205078,
      "activations/layer1_attention_weight_max": 16.285064697265625,
      "activations/layer1_attention_weight_min": -15.324573516845703,
      "activations/layer20_attention_weight_max": 20.52544403076172,
      "activations/layer20_attention_weight_min": -22.454927444458008,
      "activations/layer21_attention_weight_max": 32.34046173095703,
      "activations/layer21_attention_weight_min": -20.75556755065918,
      "activations/layer22_attention_weight_max": 30.851560592651367,
      "activations/layer22_attention_weight_min": -26.178464889526367,
      "activations/layer23_attention_weight_max": 38.72928237915039,
      "activations/layer23_attention_weight_min": -24.007579803466797,
      "activations/layer2_attention_weight_max": 30.726926803588867,
      "activations/layer2_attention_weight_min": -30.665695190429688,
      "activations/layer3_attention_weight_max": 87.02851867675781,
      "activations/layer3_attention_weight_min": -91.8990249633789,
      "activations/layer4_attention_weight_max": 88.4834976196289,
      "activations/layer4_attention_weight_min": -84.50004577636719,
      "activations/layer5_attention_weight_max": 63.58111572265625,
      "activations/layer5_attention_weight_min": -74.02268981933594,
      "activations/layer6_attention_weight_max": 48.2337532043457,
      "activations/layer6_attention_weight_min": -47.996437072753906,
      "activations/layer7_attention_weight_max": 60.41786193847656,
      "activations/layer7_attention_weight_min": -58.00819778442383,
      "activations/layer8_attention_weight_max": 43.4112434387207,
      "activations/layer8_attention_weight_min": -46.522361755371094,
      "activations/layer9_attention_weight_max": 40.48270797729492,
      "activations/layer9_attention_weight_min": -43.77013397216797,
      "epoch": 14.86,
      "learning_rate": 5.4720075757575754e-05,
      "loss": 2.751,
      "step": 255750
    },
    {
      "activations/layer0_attention_weight_max": 15.809615135192871,
      "activations/layer0_attention_weight_min": -13.026485443115234,
      "activations/layer10_attention_weight_max": 37.02827835083008,
      "activations/layer10_attention_weight_min": -36.3873405456543,
      "activations/layer11_attention_weight_max": 34.523414611816406,
      "activations/layer11_attention_weight_min": -33.745304107666016,
      "activations/layer12_attention_weight_max": 20.209976196289062,
      "activations/layer12_attention_weight_min": -27.11309242248535,
      "activations/layer13_attention_weight_max": 35.324134826660156,
      "activations/layer13_attention_weight_min": -26.526798248291016,
      "activations/layer14_attention_weight_max": 36.91618728637695,
      "activations/layer14_attention_weight_min": -31.468076705932617,
      "activations/layer15_attention_weight_max": 33.070003509521484,
      "activations/layer15_attention_weight_min": -29.525177001953125,
      "activations/layer16_attention_weight_max": 33.12147903442383,
      "activations/layer16_attention_weight_min": -29.980634689331055,
      "activations/layer17_attention_weight_max": 50.5751838684082,
      "activations/layer17_attention_weight_min": -41.837120056152344,
      "activations/layer18_attention_weight_max": 47.578399658203125,
      "activations/layer18_attention_weight_min": -38.27617645263672,
      "activations/layer19_attention_weight_max": 22.915586471557617,
      "activations/layer19_attention_weight_min": -25.164384841918945,
      "activations/layer1_attention_weight_max": 16.40810203552246,
      "activations/layer1_attention_weight_min": -14.140055656433105,
      "activations/layer20_attention_weight_max": 23.159292221069336,
      "activations/layer20_attention_weight_min": -25.356285095214844,
      "activations/layer21_attention_weight_max": 32.86839294433594,
      "activations/layer21_attention_weight_min": -25.552759170532227,
      "activations/layer22_attention_weight_max": 31.584733963012695,
      "activations/layer22_attention_weight_min": -26.97305679321289,
      "activations/layer23_attention_weight_max": 36.449737548828125,
      "activations/layer23_attention_weight_min": -23.48019027709961,
      "activations/layer2_attention_weight_max": 31.076847076416016,
      "activations/layer2_attention_weight_min": -29.677322387695312,
      "activations/layer3_attention_weight_max": 87.17166900634766,
      "activations/layer3_attention_weight_min": -89.32373809814453,
      "activations/layer4_attention_weight_max": 90.29491424560547,
      "activations/layer4_attention_weight_min": -89.06920623779297,
      "activations/layer5_attention_weight_max": 73.77999114990234,
      "activations/layer5_attention_weight_min": -73.91204833984375,
      "activations/layer6_attention_weight_max": 53.84346008300781,
      "activations/layer6_attention_weight_min": -51.48902893066406,
      "activations/layer7_attention_weight_max": 71.48857116699219,
      "activations/layer7_attention_weight_min": -65.16307830810547,
      "activations/layer8_attention_weight_max": 47.997650146484375,
      "activations/layer8_attention_weight_min": -49.051902770996094,
      "activations/layer9_attention_weight_max": 46.50798416137695,
      "activations/layer9_attention_weight_min": -47.26010513305664,
      "epoch": 14.86,
      "learning_rate": 5.4701136363636356e-05,
      "loss": 2.7603,
      "step": 255800
    },
    {
      "activations/layer0_attention_weight_max": 17.091459274291992,
      "activations/layer0_attention_weight_min": -13.363173484802246,
      "activations/layer10_attention_weight_max": 32.736507415771484,
      "activations/layer10_attention_weight_min": -34.034263610839844,
      "activations/layer11_attention_weight_max": 30.240718841552734,
      "activations/layer11_attention_weight_min": -34.17336654663086,
      "activations/layer12_attention_weight_max": 18.48680877685547,
      "activations/layer12_attention_weight_min": -27.277502059936523,
      "activations/layer13_attention_weight_max": 33.80529022216797,
      "activations/layer13_attention_weight_min": -28.13627815246582,
      "activations/layer14_attention_weight_max": 33.79603576660156,
      "activations/layer14_attention_weight_min": -30.484878540039062,
      "activations/layer15_attention_weight_max": 31.509319305419922,
      "activations/layer15_attention_weight_min": -29.076059341430664,
      "activations/layer16_attention_weight_max": 31.23912239074707,
      "activations/layer16_attention_weight_min": -30.393342971801758,
      "activations/layer17_attention_weight_max": 49.0660285949707,
      "activations/layer17_attention_weight_min": -43.15570831298828,
      "activations/layer18_attention_weight_max": 44.70625686645508,
      "activations/layer18_attention_weight_min": -36.67741775512695,
      "activations/layer19_attention_weight_max": 20.729413986206055,
      "activations/layer19_attention_weight_min": -21.212251663208008,
      "activations/layer1_attention_weight_max": 15.744112014770508,
      "activations/layer1_attention_weight_min": -14.306608200073242,
      "activations/layer20_attention_weight_max": 22.221940994262695,
      "activations/layer20_attention_weight_min": -23.586929321289062,
      "activations/layer21_attention_weight_max": 35.73580551147461,
      "activations/layer21_attention_weight_min": -22.255084991455078,
      "activations/layer22_attention_weight_max": 28.22362518310547,
      "activations/layer22_attention_weight_min": -25.091630935668945,
      "activations/layer23_attention_weight_max": 35.96061706542969,
      "activations/layer23_attention_weight_min": -26.248764038085938,
      "activations/layer2_attention_weight_max": 30.149625778198242,
      "activations/layer2_attention_weight_min": -29.368139266967773,
      "activations/layer3_attention_weight_max": 81.90824127197266,
      "activations/layer3_attention_weight_min": -85.12950897216797,
      "activations/layer4_attention_weight_max": 84.32194519042969,
      "activations/layer4_attention_weight_min": -86.47513580322266,
      "activations/layer5_attention_weight_max": 65.72332763671875,
      "activations/layer5_attention_weight_min": -70.6949691772461,
      "activations/layer6_attention_weight_max": 49.60024642944336,
      "activations/layer6_attention_weight_min": -46.75068283081055,
      "activations/layer7_attention_weight_max": 60.318389892578125,
      "activations/layer7_attention_weight_min": -59.62718963623047,
      "activations/layer8_attention_weight_max": 42.87651443481445,
      "activations/layer8_attention_weight_min": -48.5334358215332,
      "activations/layer9_attention_weight_max": 40.373382568359375,
      "activations/layer9_attention_weight_min": -44.002384185791016,
      "epoch": 14.87,
      "learning_rate": 5.4682196969696964e-05,
      "loss": 2.7665,
      "step": 255850
    },
    {
      "activations/layer0_attention_weight_max": 16.697519302368164,
      "activations/layer0_attention_weight_min": -13.345100402832031,
      "activations/layer10_attention_weight_max": 36.6230354309082,
      "activations/layer10_attention_weight_min": -36.34833526611328,
      "activations/layer11_attention_weight_max": 32.895782470703125,
      "activations/layer11_attention_weight_min": -35.36323547363281,
      "activations/layer12_attention_weight_max": 17.82086944580078,
      "activations/layer12_attention_weight_min": -25.461896896362305,
      "activations/layer13_attention_weight_max": 32.03614044189453,
      "activations/layer13_attention_weight_min": -27.87761878967285,
      "activations/layer14_attention_weight_max": 35.95803451538086,
      "activations/layer14_attention_weight_min": -30.559633255004883,
      "activations/layer15_attention_weight_max": 31.335514068603516,
      "activations/layer15_attention_weight_min": -29.845443725585938,
      "activations/layer16_attention_weight_max": 32.33782196044922,
      "activations/layer16_attention_weight_min": -30.613704681396484,
      "activations/layer17_attention_weight_max": 49.05745315551758,
      "activations/layer17_attention_weight_min": -47.36296463012695,
      "activations/layer18_attention_weight_max": 48.83672332763672,
      "activations/layer18_attention_weight_min": -41.26464080810547,
      "activations/layer19_attention_weight_max": 20.7386531829834,
      "activations/layer19_attention_weight_min": -23.230283737182617,
      "activations/layer1_attention_weight_max": 15.93427848815918,
      "activations/layer1_attention_weight_min": -15.567462921142578,
      "activations/layer20_attention_weight_max": 21.13755989074707,
      "activations/layer20_attention_weight_min": -20.888498306274414,
      "activations/layer21_attention_weight_max": 37.15266036987305,
      "activations/layer21_attention_weight_min": -22.638397216796875,
      "activations/layer22_attention_weight_max": 30.020526885986328,
      "activations/layer22_attention_weight_min": -26.82779884338379,
      "activations/layer23_attention_weight_max": 35.08345031738281,
      "activations/layer23_attention_weight_min": -22.705900192260742,
      "activations/layer2_attention_weight_max": 30.21826171875,
      "activations/layer2_attention_weight_min": -29.027311325073242,
      "activations/layer3_attention_weight_max": 86.22986602783203,
      "activations/layer3_attention_weight_min": -88.87007904052734,
      "activations/layer4_attention_weight_max": 85.24119567871094,
      "activations/layer4_attention_weight_min": -89.06708526611328,
      "activations/layer5_attention_weight_max": 68.78034973144531,
      "activations/layer5_attention_weight_min": -82.04720306396484,
      "activations/layer6_attention_weight_max": 50.507545471191406,
      "activations/layer6_attention_weight_min": -52.3619270324707,
      "activations/layer7_attention_weight_max": 67.41688537597656,
      "activations/layer7_attention_weight_min": -62.66004180908203,
      "activations/layer8_attention_weight_max": 45.729068756103516,
      "activations/layer8_attention_weight_min": -48.87796401977539,
      "activations/layer9_attention_weight_max": 46.53068923950195,
      "activations/layer9_attention_weight_min": -46.2995719909668,
      "epoch": 14.87,
      "learning_rate": 5.466325757575757e-05,
      "loss": 2.7603,
      "step": 255900
    },
    {
      "activations/layer0_attention_weight_max": 15.671186447143555,
      "activations/layer0_attention_weight_min": -13.28120231628418,
      "activations/layer10_attention_weight_max": 35.67727279663086,
      "activations/layer10_attention_weight_min": -35.0794563293457,
      "activations/layer11_attention_weight_max": 32.486351013183594,
      "activations/layer11_attention_weight_min": -33.60237121582031,
      "activations/layer12_attention_weight_max": 19.565473556518555,
      "activations/layer12_attention_weight_min": -25.583539962768555,
      "activations/layer13_attention_weight_max": 42.51913070678711,
      "activations/layer13_attention_weight_min": -29.99781036376953,
      "activations/layer14_attention_weight_max": 37.05614471435547,
      "activations/layer14_attention_weight_min": -30.8410587310791,
      "activations/layer15_attention_weight_max": 32.98887252807617,
      "activations/layer15_attention_weight_min": -29.928325653076172,
      "activations/layer16_attention_weight_max": 32.81190872192383,
      "activations/layer16_attention_weight_min": -30.88749122619629,
      "activations/layer17_attention_weight_max": 50.475460052490234,
      "activations/layer17_attention_weight_min": -44.70843505859375,
      "activations/layer18_attention_weight_max": 44.106082916259766,
      "activations/layer18_attention_weight_min": -39.35072708129883,
      "activations/layer19_attention_weight_max": 22.9935302734375,
      "activations/layer19_attention_weight_min": -21.43442153930664,
      "activations/layer1_attention_weight_max": 16.246414184570312,
      "activations/layer1_attention_weight_min": -13.506864547729492,
      "activations/layer20_attention_weight_max": 21.90471076965332,
      "activations/layer20_attention_weight_min": -20.132436752319336,
      "activations/layer21_attention_weight_max": 37.09071350097656,
      "activations/layer21_attention_weight_min": -22.44305992126465,
      "activations/layer22_attention_weight_max": 30.928266525268555,
      "activations/layer22_attention_weight_min": -24.725156784057617,
      "activations/layer23_attention_weight_max": 42.48393249511719,
      "activations/layer23_attention_weight_min": -23.237899780273438,
      "activations/layer2_attention_weight_max": 29.16223907470703,
      "activations/layer2_attention_weight_min": -30.321659088134766,
      "activations/layer3_attention_weight_max": 81.97675323486328,
      "activations/layer3_attention_weight_min": -90.00666046142578,
      "activations/layer4_attention_weight_max": 82.88946533203125,
      "activations/layer4_attention_weight_min": -82.77989959716797,
      "activations/layer5_attention_weight_max": 64.39152526855469,
      "activations/layer5_attention_weight_min": -72.39818572998047,
      "activations/layer6_attention_weight_max": 47.89448547363281,
      "activations/layer6_attention_weight_min": -47.22122573852539,
      "activations/layer7_attention_weight_max": 60.15737533569336,
      "activations/layer7_attention_weight_min": -58.67302322387695,
      "activations/layer8_attention_weight_max": 44.415889739990234,
      "activations/layer8_attention_weight_min": -45.63547897338867,
      "activations/layer9_attention_weight_max": 44.78498077392578,
      "activations/layer9_attention_weight_min": -48.07987594604492,
      "epoch": 14.87,
      "learning_rate": 5.4644318181818174e-05,
      "loss": 2.7572,
      "step": 255950
    },
    {
      "activations/layer0_attention_weight_max": 15.761661529541016,
      "activations/layer0_attention_weight_min": -13.013068199157715,
      "activations/layer10_attention_weight_max": 35.12265396118164,
      "activations/layer10_attention_weight_min": -34.56904220581055,
      "activations/layer11_attention_weight_max": 33.83056640625,
      "activations/layer11_attention_weight_min": -33.80902099609375,
      "activations/layer12_attention_weight_max": 18.12453269958496,
      "activations/layer12_attention_weight_min": -29.888517379760742,
      "activations/layer13_attention_weight_max": 34.16516876220703,
      "activations/layer13_attention_weight_min": -26.24648666381836,
      "activations/layer14_attention_weight_max": 53.01145553588867,
      "activations/layer14_attention_weight_min": -27.797880172729492,
      "activations/layer15_attention_weight_max": 43.76478576660156,
      "activations/layer15_attention_weight_min": -28.925628662109375,
      "activations/layer16_attention_weight_max": 31.197404861450195,
      "activations/layer16_attention_weight_min": -28.273895263671875,
      "activations/layer17_attention_weight_max": 54.33320617675781,
      "activations/layer17_attention_weight_min": -44.27009201049805,
      "activations/layer18_attention_weight_max": 44.82560729980469,
      "activations/layer18_attention_weight_min": -41.3962287902832,
      "activations/layer19_attention_weight_max": 23.932331085205078,
      "activations/layer19_attention_weight_min": -24.595409393310547,
      "activations/layer1_attention_weight_max": 16.65324592590332,
      "activations/layer1_attention_weight_min": -14.782504081726074,
      "activations/layer20_attention_weight_max": 28.0762882232666,
      "activations/layer20_attention_weight_min": -24.166973114013672,
      "activations/layer21_attention_weight_max": 46.138336181640625,
      "activations/layer21_attention_weight_min": -24.08323097229004,
      "activations/layer22_attention_weight_max": 32.739383697509766,
      "activations/layer22_attention_weight_min": -24.87339210510254,
      "activations/layer23_attention_weight_max": 40.958091735839844,
      "activations/layer23_attention_weight_min": -29.370344161987305,
      "activations/layer2_attention_weight_max": 31.952733993530273,
      "activations/layer2_attention_weight_min": -31.68836784362793,
      "activations/layer3_attention_weight_max": 95.62760925292969,
      "activations/layer3_attention_weight_min": -95.58869171142578,
      "activations/layer4_attention_weight_max": 89.82379913330078,
      "activations/layer4_attention_weight_min": -93.26402282714844,
      "activations/layer5_attention_weight_max": 64.62471008300781,
      "activations/layer5_attention_weight_min": -73.58830261230469,
      "activations/layer6_attention_weight_max": 49.203670501708984,
      "activations/layer6_attention_weight_min": -48.62674331665039,
      "activations/layer7_attention_weight_max": 64.85087585449219,
      "activations/layer7_attention_weight_min": -62.134483337402344,
      "activations/layer8_attention_weight_max": 44.05318832397461,
      "activations/layer8_attention_weight_min": -44.02842330932617,
      "activations/layer9_attention_weight_max": 53.63898468017578,
      "activations/layer9_attention_weight_min": -45.266845703125,
      "epoch": 14.88,
      "learning_rate": 5.462537878787878e-05,
      "loss": 2.7473,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.483,
      "eval_samples_per_second": 506.189,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.483,
      "eval_openwebtext_samples_per_second": 506.189,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9121,
      "eval_wikitext_samples_per_second": 238.476,
      "step": 256000
    },
    {
      "epoch": 14.88,
      "eval_lambada_loss": 2.548828125,
      "eval_lambada_ppl": 12.792104248423877,
      "eval_lambada_runtime": 9.4919,
      "eval_lambada_samples_per_second": 512.962,
      "step": 256000
    },
    {
      "activations/layer0_attention_weight_max": 16.626508712768555,
      "activations/layer0_attention_weight_min": -13.146268844604492,
      "activations/layer10_attention_weight_max": 34.96112823486328,
      "activations/layer10_attention_weight_min": -34.925682067871094,
      "activations/layer11_attention_weight_max": 31.286588668823242,
      "activations/layer11_attention_weight_min": -33.450225830078125,
      "activations/layer12_attention_weight_max": 17.34925651550293,
      "activations/layer12_attention_weight_min": -27.251596450805664,
      "activations/layer13_attention_weight_max": 31.097951889038086,
      "activations/layer13_attention_weight_min": -27.90227699279785,
      "activations/layer14_attention_weight_max": 35.47711944580078,
      "activations/layer14_attention_weight_min": -30.469985961914062,
      "activations/layer15_attention_weight_max": 32.30598068237305,
      "activations/layer15_attention_weight_min": -32.265132904052734,
      "activations/layer16_attention_weight_max": 33.609619140625,
      "activations/layer16_attention_weight_min": -30.738008499145508,
      "activations/layer17_attention_weight_max": 53.02711868286133,
      "activations/layer17_attention_weight_min": -44.17368698120117,
      "activations/layer18_attention_weight_max": 45.221824645996094,
      "activations/layer18_attention_weight_min": -39.50879669189453,
      "activations/layer19_attention_weight_max": 23.257469177246094,
      "activations/layer19_attention_weight_min": -24.002779006958008,
      "activations/layer1_attention_weight_max": 16.81163215637207,
      "activations/layer1_attention_weight_min": -15.539979934692383,
      "activations/layer20_attention_weight_max": 22.284610748291016,
      "activations/layer20_attention_weight_min": -23.9247989654541,
      "activations/layer21_attention_weight_max": 39.1060676574707,
      "activations/layer21_attention_weight_min": -29.96238899230957,
      "activations/layer22_attention_weight_max": 29.178728103637695,
      "activations/layer22_attention_weight_min": -27.171751022338867,
      "activations/layer23_attention_weight_max": 34.8697509765625,
      "activations/layer23_attention_weight_min": -26.879077911376953,
      "activations/layer2_attention_weight_max": 31.729488372802734,
      "activations/layer2_attention_weight_min": -30.616374969482422,
      "activations/layer3_attention_weight_max": 90.93205261230469,
      "activations/layer3_attention_weight_min": -92.0711669921875,
      "activations/layer4_attention_weight_max": 93.57323455810547,
      "activations/layer4_attention_weight_min": -91.14989471435547,
      "activations/layer5_attention_weight_max": 71.41203308105469,
      "activations/layer5_attention_weight_min": -75.22244262695312,
      "activations/layer6_attention_weight_max": 51.48220443725586,
      "activations/layer6_attention_weight_min": -52.597007751464844,
      "activations/layer7_attention_weight_max": 61.01695251464844,
      "activations/layer7_attention_weight_min": -63.232330322265625,
      "activations/layer8_attention_weight_max": 44.30310821533203,
      "activations/layer8_attention_weight_min": -52.040672302246094,
      "activations/layer9_attention_weight_max": 42.52802658081055,
      "activations/layer9_attention_weight_min": -49.461727142333984,
      "epoch": 14.88,
      "learning_rate": 5.460643939393939e-05,
      "loss": 2.7607,
      "step": 256050
    },
    {
      "activations/layer0_attention_weight_max": 16.388134002685547,
      "activations/layer0_attention_weight_min": -13.663812637329102,
      "activations/layer10_attention_weight_max": 37.22036361694336,
      "activations/layer10_attention_weight_min": -36.558841705322266,
      "activations/layer11_attention_weight_max": 33.768558502197266,
      "activations/layer11_attention_weight_min": -33.5567626953125,
      "activations/layer12_attention_weight_max": 22.21166229248047,
      "activations/layer12_attention_weight_min": -27.538963317871094,
      "activations/layer13_attention_weight_max": 37.6786994934082,
      "activations/layer13_attention_weight_min": -28.257984161376953,
      "activations/layer14_attention_weight_max": 36.88539505004883,
      "activations/layer14_attention_weight_min": -28.755828857421875,
      "activations/layer15_attention_weight_max": 34.38116455078125,
      "activations/layer15_attention_weight_min": -28.399673461914062,
      "activations/layer16_attention_weight_max": 33.20210266113281,
      "activations/layer16_attention_weight_min": -31.360795974731445,
      "activations/layer17_attention_weight_max": 52.706851959228516,
      "activations/layer17_attention_weight_min": -46.41999053955078,
      "activations/layer18_attention_weight_max": 49.580047607421875,
      "activations/layer18_attention_weight_min": -39.73708724975586,
      "activations/layer19_attention_weight_max": 24.108095169067383,
      "activations/layer19_attention_weight_min": -20.50596046447754,
      "activations/layer1_attention_weight_max": 16.212270736694336,
      "activations/layer1_attention_weight_min": -14.204963684082031,
      "activations/layer20_attention_weight_max": 24.46832847595215,
      "activations/layer20_attention_weight_min": -20.075273513793945,
      "activations/layer21_attention_weight_max": 42.40012741088867,
      "activations/layer21_attention_weight_min": -22.923540115356445,
      "activations/layer22_attention_weight_max": 33.45102310180664,
      "activations/layer22_attention_weight_min": -25.44288444519043,
      "activations/layer23_attention_weight_max": 39.900604248046875,
      "activations/layer23_attention_weight_min": -23.91280174255371,
      "activations/layer2_attention_weight_max": 31.846759796142578,
      "activations/layer2_attention_weight_min": -31.6098690032959,
      "activations/layer3_attention_weight_max": 90.69776916503906,
      "activations/layer3_attention_weight_min": -93.10726165771484,
      "activations/layer4_attention_weight_max": 89.5569839477539,
      "activations/layer4_attention_weight_min": -86.35555267333984,
      "activations/layer5_attention_weight_max": 70.29867553710938,
      "activations/layer5_attention_weight_min": -71.59556579589844,
      "activations/layer6_attention_weight_max": 51.250789642333984,
      "activations/layer6_attention_weight_min": -51.90046310424805,
      "activations/layer7_attention_weight_max": 68.92048645019531,
      "activations/layer7_attention_weight_min": -63.3791618347168,
      "activations/layer8_attention_weight_max": 49.947059631347656,
      "activations/layer8_attention_weight_min": -52.147483825683594,
      "activations/layer9_attention_weight_max": 47.47827911376953,
      "activations/layer9_attention_weight_min": -50.778926849365234,
      "epoch": 14.88,
      "learning_rate": 5.458749999999999e-05,
      "loss": 2.7603,
      "step": 256100
    },
    {
      "activations/layer0_attention_weight_max": 16.295574188232422,
      "activations/layer0_attention_weight_min": -13.162821769714355,
      "activations/layer10_attention_weight_max": 35.15049362182617,
      "activations/layer10_attention_weight_min": -37.10884094238281,
      "activations/layer11_attention_weight_max": 32.88604736328125,
      "activations/layer11_attention_weight_min": -32.628143310546875,
      "activations/layer12_attention_weight_max": 17.991178512573242,
      "activations/layer12_attention_weight_min": -26.027772903442383,
      "activations/layer13_attention_weight_max": 37.974422454833984,
      "activations/layer13_attention_weight_min": -28.830957412719727,
      "activations/layer14_attention_weight_max": 41.61093521118164,
      "activations/layer14_attention_weight_min": -31.63157081604004,
      "activations/layer15_attention_weight_max": 42.306793212890625,
      "activations/layer15_attention_weight_min": -30.00107192993164,
      "activations/layer16_attention_weight_max": 32.75385665893555,
      "activations/layer16_attention_weight_min": -31.576854705810547,
      "activations/layer17_attention_weight_max": 49.90663528442383,
      "activations/layer17_attention_weight_min": -47.59917068481445,
      "activations/layer18_attention_weight_max": 44.352073669433594,
      "activations/layer18_attention_weight_min": -41.4759521484375,
      "activations/layer19_attention_weight_max": 22.578136444091797,
      "activations/layer19_attention_weight_min": -24.217628479003906,
      "activations/layer1_attention_weight_max": 16.60210418701172,
      "activations/layer1_attention_weight_min": -15.418437957763672,
      "activations/layer20_attention_weight_max": 25.205167770385742,
      "activations/layer20_attention_weight_min": -25.1179256439209,
      "activations/layer21_attention_weight_max": 37.766029357910156,
      "activations/layer21_attention_weight_min": -24.731534957885742,
      "activations/layer22_attention_weight_max": 30.212007522583008,
      "activations/layer22_attention_weight_min": -26.789161682128906,
      "activations/layer23_attention_weight_max": 33.151344299316406,
      "activations/layer23_attention_weight_min": -24.093442916870117,
      "activations/layer2_attention_weight_max": 31.193449020385742,
      "activations/layer2_attention_weight_min": -29.406089782714844,
      "activations/layer3_attention_weight_max": 90.50096130371094,
      "activations/layer3_attention_weight_min": -90.67131805419922,
      "activations/layer4_attention_weight_max": 87.20970916748047,
      "activations/layer4_attention_weight_min": -84.45015716552734,
      "activations/layer5_attention_weight_max": 64.34651184082031,
      "activations/layer5_attention_weight_min": -72.61526489257812,
      "activations/layer6_attention_weight_max": 47.8065071105957,
      "activations/layer6_attention_weight_min": -48.433006286621094,
      "activations/layer7_attention_weight_max": 62.4517936706543,
      "activations/layer7_attention_weight_min": -59.71061325073242,
      "activations/layer8_attention_weight_max": 46.254154205322266,
      "activations/layer8_attention_weight_min": -50.04205322265625,
      "activations/layer9_attention_weight_max": 45.2789421081543,
      "activations/layer9_attention_weight_min": -47.16754150390625,
      "epoch": 14.88,
      "learning_rate": 5.45685606060606e-05,
      "loss": 2.767,
      "step": 256150
    },
    {
      "activations/layer0_attention_weight_max": 16.452526092529297,
      "activations/layer0_attention_weight_min": -13.58932876586914,
      "activations/layer10_attention_weight_max": 34.2119026184082,
      "activations/layer10_attention_weight_min": -33.65811538696289,
      "activations/layer11_attention_weight_max": 30.34943199157715,
      "activations/layer11_attention_weight_min": -31.666187286376953,
      "activations/layer12_attention_weight_max": 18.510536193847656,
      "activations/layer12_attention_weight_min": -23.855913162231445,
      "activations/layer13_attention_weight_max": 32.64169692993164,
      "activations/layer13_attention_weight_min": -27.335355758666992,
      "activations/layer14_attention_weight_max": 36.77726745605469,
      "activations/layer14_attention_weight_min": -29.150646209716797,
      "activations/layer15_attention_weight_max": 33.587100982666016,
      "activations/layer15_attention_weight_min": -28.12190818786621,
      "activations/layer16_attention_weight_max": 33.20710372924805,
      "activations/layer16_attention_weight_min": -30.04981803894043,
      "activations/layer17_attention_weight_max": 56.72133255004883,
      "activations/layer17_attention_weight_min": -46.18141555786133,
      "activations/layer18_attention_weight_max": 52.08751678466797,
      "activations/layer18_attention_weight_min": -39.84859085083008,
      "activations/layer19_attention_weight_max": 25.078704833984375,
      "activations/layer19_attention_weight_min": -23.00108528137207,
      "activations/layer1_attention_weight_max": 16.167869567871094,
      "activations/layer1_attention_weight_min": -15.71298885345459,
      "activations/layer20_attention_weight_max": 26.821575164794922,
      "activations/layer20_attention_weight_min": -21.99443817138672,
      "activations/layer21_attention_weight_max": 39.39028549194336,
      "activations/layer21_attention_weight_min": -24.022436141967773,
      "activations/layer22_attention_weight_max": 31.63159942626953,
      "activations/layer22_attention_weight_min": -26.802562713623047,
      "activations/layer23_attention_weight_max": 39.779380798339844,
      "activations/layer23_attention_weight_min": -24.150169372558594,
      "activations/layer2_attention_weight_max": 31.09083366394043,
      "activations/layer2_attention_weight_min": -31.25864028930664,
      "activations/layer3_attention_weight_max": 90.53450775146484,
      "activations/layer3_attention_weight_min": -93.05035400390625,
      "activations/layer4_attention_weight_max": 89.7076416015625,
      "activations/layer4_attention_weight_min": -89.05314636230469,
      "activations/layer5_attention_weight_max": 68.15138244628906,
      "activations/layer5_attention_weight_min": -72.585693359375,
      "activations/layer6_attention_weight_max": 47.83326721191406,
      "activations/layer6_attention_weight_min": -48.26585388183594,
      "activations/layer7_attention_weight_max": 61.81163787841797,
      "activations/layer7_attention_weight_min": -59.03162384033203,
      "activations/layer8_attention_weight_max": 45.58594512939453,
      "activations/layer8_attention_weight_min": -46.95428466796875,
      "activations/layer9_attention_weight_max": 42.58607482910156,
      "activations/layer9_attention_weight_min": -46.39746856689453,
      "epoch": 14.89,
      "learning_rate": 5.454962121212121e-05,
      "loss": 2.7487,
      "step": 256200
    },
    {
      "activations/layer0_attention_weight_max": 16.013320922851562,
      "activations/layer0_attention_weight_min": -13.349967002868652,
      "activations/layer10_attention_weight_max": 36.01140213012695,
      "activations/layer10_attention_weight_min": -36.15949249267578,
      "activations/layer11_attention_weight_max": 32.718345642089844,
      "activations/layer11_attention_weight_min": -32.960914611816406,
      "activations/layer12_attention_weight_max": 19.86146354675293,
      "activations/layer12_attention_weight_min": -26.222305297851562,
      "activations/layer13_attention_weight_max": 34.68757629394531,
      "activations/layer13_attention_weight_min": -26.507427215576172,
      "activations/layer14_attention_weight_max": 35.98415756225586,
      "activations/layer14_attention_weight_min": -28.886444091796875,
      "activations/layer15_attention_weight_max": 34.85771560668945,
      "activations/layer15_attention_weight_min": -29.61260223388672,
      "activations/layer16_attention_weight_max": 33.291202545166016,
      "activations/layer16_attention_weight_min": -30.712574005126953,
      "activations/layer17_attention_weight_max": 53.97715377807617,
      "activations/layer17_attention_weight_min": -45.81488037109375,
      "activations/layer18_attention_weight_max": 49.957645416259766,
      "activations/layer18_attention_weight_min": -39.58319091796875,
      "activations/layer19_attention_weight_max": 23.23705291748047,
      "activations/layer19_attention_weight_min": -23.96000862121582,
      "activations/layer1_attention_weight_max": 16.967832565307617,
      "activations/layer1_attention_weight_min": -15.941522598266602,
      "activations/layer20_attention_weight_max": 23.864749908447266,
      "activations/layer20_attention_weight_min": -20.060754776000977,
      "activations/layer21_attention_weight_max": 38.048858642578125,
      "activations/layer21_attention_weight_min": -20.431468963623047,
      "activations/layer22_attention_weight_max": 32.48741912841797,
      "activations/layer22_attention_weight_min": -25.7145938873291,
      "activations/layer23_attention_weight_max": 35.75898361206055,
      "activations/layer23_attention_weight_min": -23.995473861694336,
      "activations/layer2_attention_weight_max": 31.163681030273438,
      "activations/layer2_attention_weight_min": -30.367530822753906,
      "activations/layer3_attention_weight_max": 95.11772155761719,
      "activations/layer3_attention_weight_min": -96.60453796386719,
      "activations/layer4_attention_weight_max": 90.66796112060547,
      "activations/layer4_attention_weight_min": -94.88945770263672,
      "activations/layer5_attention_weight_max": 67.13296508789062,
      "activations/layer5_attention_weight_min": -74.42033386230469,
      "activations/layer6_attention_weight_max": 49.75558090209961,
      "activations/layer6_attention_weight_min": -49.260398864746094,
      "activations/layer7_attention_weight_max": 61.5365104675293,
      "activations/layer7_attention_weight_min": -61.90888214111328,
      "activations/layer8_attention_weight_max": 44.433624267578125,
      "activations/layer8_attention_weight_min": -48.03733825683594,
      "activations/layer9_attention_weight_max": 43.14931869506836,
      "activations/layer9_attention_weight_min": -44.718345642089844,
      "epoch": 14.89,
      "learning_rate": 5.453068181818182e-05,
      "loss": 2.7686,
      "step": 256250
    },
    {
      "activations/layer0_attention_weight_max": 15.861702919006348,
      "activations/layer0_attention_weight_min": -13.462908744812012,
      "activations/layer10_attention_weight_max": 39.292091369628906,
      "activations/layer10_attention_weight_min": -36.22327423095703,
      "activations/layer11_attention_weight_max": 37.8152961730957,
      "activations/layer11_attention_weight_min": -33.629554748535156,
      "activations/layer12_attention_weight_max": 17.954483032226562,
      "activations/layer12_attention_weight_min": -32.59425735473633,
      "activations/layer13_attention_weight_max": 38.30671310424805,
      "activations/layer13_attention_weight_min": -27.682119369506836,
      "activations/layer14_attention_weight_max": 37.76108932495117,
      "activations/layer14_attention_weight_min": -29.013492584228516,
      "activations/layer15_attention_weight_max": 31.464841842651367,
      "activations/layer15_attention_weight_min": -27.55699920654297,
      "activations/layer16_attention_weight_max": 30.01297950744629,
      "activations/layer16_attention_weight_min": -29.87136459350586,
      "activations/layer17_attention_weight_max": 50.00766372680664,
      "activations/layer17_attention_weight_min": -43.7335205078125,
      "activations/layer18_attention_weight_max": 44.38565444946289,
      "activations/layer18_attention_weight_min": -36.67346954345703,
      "activations/layer19_attention_weight_max": 22.959604263305664,
      "activations/layer19_attention_weight_min": -21.95775032043457,
      "activations/layer1_attention_weight_max": 17.096120834350586,
      "activations/layer1_attention_weight_min": -13.285969734191895,
      "activations/layer20_attention_weight_max": 21.01512908935547,
      "activations/layer20_attention_weight_min": -20.523948669433594,
      "activations/layer21_attention_weight_max": 32.431190490722656,
      "activations/layer21_attention_weight_min": -23.36348533630371,
      "activations/layer22_attention_weight_max": 28.14057159423828,
      "activations/layer22_attention_weight_min": -24.89090919494629,
      "activations/layer23_attention_weight_max": 33.6124382019043,
      "activations/layer23_attention_weight_min": -25.342899322509766,
      "activations/layer2_attention_weight_max": 32.76187515258789,
      "activations/layer2_attention_weight_min": -30.595752716064453,
      "activations/layer3_attention_weight_max": 92.09595489501953,
      "activations/layer3_attention_weight_min": -91.34634399414062,
      "activations/layer4_attention_weight_max": 90.16756439208984,
      "activations/layer4_attention_weight_min": -84.33416748046875,
      "activations/layer5_attention_weight_max": 66.20700073242188,
      "activations/layer5_attention_weight_min": -78.39873504638672,
      "activations/layer6_attention_weight_max": 49.87855911254883,
      "activations/layer6_attention_weight_min": -49.313846588134766,
      "activations/layer7_attention_weight_max": 72.1203384399414,
      "activations/layer7_attention_weight_min": -58.4545783996582,
      "activations/layer8_attention_weight_max": 53.66011428833008,
      "activations/layer8_attention_weight_min": -46.69091796875,
      "activations/layer9_attention_weight_max": 58.32244873046875,
      "activations/layer9_attention_weight_min": -51.13692855834961,
      "epoch": 14.89,
      "learning_rate": 5.451174242424242e-05,
      "loss": 2.7417,
      "step": 256300
    },
    {
      "activations/layer0_attention_weight_max": 16.62851905822754,
      "activations/layer0_attention_weight_min": -13.416359901428223,
      "activations/layer10_attention_weight_max": 36.05071258544922,
      "activations/layer10_attention_weight_min": -36.02720642089844,
      "activations/layer11_attention_weight_max": 34.231685638427734,
      "activations/layer11_attention_weight_min": -34.11187744140625,
      "activations/layer12_attention_weight_max": 18.251134872436523,
      "activations/layer12_attention_weight_min": -24.782636642456055,
      "activations/layer13_attention_weight_max": 36.03683090209961,
      "activations/layer13_attention_weight_min": -29.00887680053711,
      "activations/layer14_attention_weight_max": 37.740535736083984,
      "activations/layer14_attention_weight_min": -33.46601867675781,
      "activations/layer15_attention_weight_max": 33.18848419189453,
      "activations/layer15_attention_weight_min": -30.887393951416016,
      "activations/layer16_attention_weight_max": 31.27736473083496,
      "activations/layer16_attention_weight_min": -33.04050064086914,
      "activations/layer17_attention_weight_max": 50.37755584716797,
      "activations/layer17_attention_weight_min": -45.870487213134766,
      "activations/layer18_attention_weight_max": 45.93617248535156,
      "activations/layer18_attention_weight_min": -41.85173797607422,
      "activations/layer19_attention_weight_max": 25.02373504638672,
      "activations/layer19_attention_weight_min": -21.957069396972656,
      "activations/layer1_attention_weight_max": 17.099353790283203,
      "activations/layer1_attention_weight_min": -15.18939208984375,
      "activations/layer20_attention_weight_max": 23.1854190826416,
      "activations/layer20_attention_weight_min": -21.605876922607422,
      "activations/layer21_attention_weight_max": 40.716156005859375,
      "activations/layer21_attention_weight_min": -26.699865341186523,
      "activations/layer22_attention_weight_max": 28.12468719482422,
      "activations/layer22_attention_weight_min": -29.49753189086914,
      "activations/layer23_attention_weight_max": 36.736934661865234,
      "activations/layer23_attention_weight_min": -28.826902389526367,
      "activations/layer2_attention_weight_max": 32.62077331542969,
      "activations/layer2_attention_weight_min": -32.79936599731445,
      "activations/layer3_attention_weight_max": 92.26979064941406,
      "activations/layer3_attention_weight_min": -94.73692321777344,
      "activations/layer4_attention_weight_max": 89.0451889038086,
      "activations/layer4_attention_weight_min": -83.94983673095703,
      "activations/layer5_attention_weight_max": 65.73945617675781,
      "activations/layer5_attention_weight_min": -79.00851440429688,
      "activations/layer6_attention_weight_max": 51.32460403442383,
      "activations/layer6_attention_weight_min": -48.50926208496094,
      "activations/layer7_attention_weight_max": 63.3951301574707,
      "activations/layer7_attention_weight_min": -62.20220947265625,
      "activations/layer8_attention_weight_max": 49.71394348144531,
      "activations/layer8_attention_weight_min": -48.9932746887207,
      "activations/layer9_attention_weight_max": 47.15571212768555,
      "activations/layer9_attention_weight_min": -44.18317794799805,
      "epoch": 14.9,
      "learning_rate": 5.449280303030302e-05,
      "loss": 2.7513,
      "step": 256350
    },
    {
      "activations/layer0_attention_weight_max": 15.525014877319336,
      "activations/layer0_attention_weight_min": -13.514097213745117,
      "activations/layer10_attention_weight_max": 39.202964782714844,
      "activations/layer10_attention_weight_min": -38.24763107299805,
      "activations/layer11_attention_weight_max": 33.82252883911133,
      "activations/layer11_attention_weight_min": -34.0045166015625,
      "activations/layer12_attention_weight_max": 18.989282608032227,
      "activations/layer12_attention_weight_min": -27.226078033447266,
      "activations/layer13_attention_weight_max": 35.92034912109375,
      "activations/layer13_attention_weight_min": -28.071847915649414,
      "activations/layer14_attention_weight_max": 35.608062744140625,
      "activations/layer14_attention_weight_min": -31.461631774902344,
      "activations/layer15_attention_weight_max": 31.11932373046875,
      "activations/layer15_attention_weight_min": -30.97367286682129,
      "activations/layer16_attention_weight_max": 33.99399185180664,
      "activations/layer16_attention_weight_min": -32.253395080566406,
      "activations/layer17_attention_weight_max": 53.34996032714844,
      "activations/layer17_attention_weight_min": -47.02755355834961,
      "activations/layer18_attention_weight_max": 45.93007278442383,
      "activations/layer18_attention_weight_min": -38.3026237487793,
      "activations/layer19_attention_weight_max": 22.075170516967773,
      "activations/layer19_attention_weight_min": -21.430301666259766,
      "activations/layer1_attention_weight_max": 16.036190032958984,
      "activations/layer1_attention_weight_min": -13.90120792388916,
      "activations/layer20_attention_weight_max": 21.929058074951172,
      "activations/layer20_attention_weight_min": -20.73055648803711,
      "activations/layer21_attention_weight_max": 32.985347747802734,
      "activations/layer21_attention_weight_min": -21.06935691833496,
      "activations/layer22_attention_weight_max": 29.900592803955078,
      "activations/layer22_attention_weight_min": -25.300188064575195,
      "activations/layer23_attention_weight_max": 35.646514892578125,
      "activations/layer23_attention_weight_min": -23.464757919311523,
      "activations/layer2_attention_weight_max": 31.018505096435547,
      "activations/layer2_attention_weight_min": -29.930408477783203,
      "activations/layer3_attention_weight_max": 96.76313018798828,
      "activations/layer3_attention_weight_min": -91.71591186523438,
      "activations/layer4_attention_weight_max": 92.85250854492188,
      "activations/layer4_attention_weight_min": -91.29671478271484,
      "activations/layer5_attention_weight_max": 64.84761810302734,
      "activations/layer5_attention_weight_min": -72.10514831542969,
      "activations/layer6_attention_weight_max": 48.852821350097656,
      "activations/layer6_attention_weight_min": -50.811641693115234,
      "activations/layer7_attention_weight_max": 63.396060943603516,
      "activations/layer7_attention_weight_min": -59.00111770629883,
      "activations/layer8_attention_weight_max": 45.798118591308594,
      "activations/layer8_attention_weight_min": -47.01838302612305,
      "activations/layer9_attention_weight_max": 46.43024826049805,
      "activations/layer9_attention_weight_min": -45.39521026611328,
      "epoch": 14.9,
      "learning_rate": 5.4473863636363635e-05,
      "loss": 2.7696,
      "step": 256400
    },
    {
      "activations/layer0_attention_weight_max": 15.65606689453125,
      "activations/layer0_attention_weight_min": -13.648855209350586,
      "activations/layer10_attention_weight_max": 34.95803451538086,
      "activations/layer10_attention_weight_min": -34.56559753417969,
      "activations/layer11_attention_weight_max": 31.679431915283203,
      "activations/layer11_attention_weight_min": -33.68770217895508,
      "activations/layer12_attention_weight_max": 20.065433502197266,
      "activations/layer12_attention_weight_min": -22.351720809936523,
      "activations/layer13_attention_weight_max": 37.73232650756836,
      "activations/layer13_attention_weight_min": -27.42778205871582,
      "activations/layer14_attention_weight_max": 38.971031188964844,
      "activations/layer14_attention_weight_min": -33.576927185058594,
      "activations/layer15_attention_weight_max": 35.171905517578125,
      "activations/layer15_attention_weight_min": -31.096284866333008,
      "activations/layer16_attention_weight_max": 36.17530822753906,
      "activations/layer16_attention_weight_min": -33.63185119628906,
      "activations/layer17_attention_weight_max": 54.718204498291016,
      "activations/layer17_attention_weight_min": -45.950965881347656,
      "activations/layer18_attention_weight_max": 51.117855072021484,
      "activations/layer18_attention_weight_min": -40.62892532348633,
      "activations/layer19_attention_weight_max": 25.154834747314453,
      "activations/layer19_attention_weight_min": -22.636388778686523,
      "activations/layer1_attention_weight_max": 16.019678115844727,
      "activations/layer1_attention_weight_min": -15.5753812789917,
      "activations/layer20_attention_weight_max": 27.77354621887207,
      "activations/layer20_attention_weight_min": -22.412588119506836,
      "activations/layer21_attention_weight_max": 42.36333084106445,
      "activations/layer21_attention_weight_min": -22.5897274017334,
      "activations/layer22_attention_weight_max": 34.3375129699707,
      "activations/layer22_attention_weight_min": -27.383203506469727,
      "activations/layer23_attention_weight_max": 48.219573974609375,
      "activations/layer23_attention_weight_min": -26.042903900146484,
      "activations/layer2_attention_weight_max": 32.857154846191406,
      "activations/layer2_attention_weight_min": -32.63795471191406,
      "activations/layer3_attention_weight_max": 87.09541320800781,
      "activations/layer3_attention_weight_min": -95.62855529785156,
      "activations/layer4_attention_weight_max": 86.98668670654297,
      "activations/layer4_attention_weight_min": -85.54386901855469,
      "activations/layer5_attention_weight_max": 62.2347412109375,
      "activations/layer5_attention_weight_min": -79.69320678710938,
      "activations/layer6_attention_weight_max": 47.13205337524414,
      "activations/layer6_attention_weight_min": -46.719459533691406,
      "activations/layer7_attention_weight_max": 62.25602722167969,
      "activations/layer7_attention_weight_min": -59.6632194519043,
      "activations/layer8_attention_weight_max": 43.95478439331055,
      "activations/layer8_attention_weight_min": -47.16048812866211,
      "activations/layer9_attention_weight_max": 43.21529769897461,
      "activations/layer9_attention_weight_min": -45.603248596191406,
      "epoch": 14.9,
      "learning_rate": 5.4454924242424236e-05,
      "loss": 2.7396,
      "step": 256450
    },
    {
      "activations/layer0_attention_weight_max": 16.318309783935547,
      "activations/layer0_attention_weight_min": -13.710190773010254,
      "activations/layer10_attention_weight_max": 40.16379165649414,
      "activations/layer10_attention_weight_min": -42.53696823120117,
      "activations/layer11_attention_weight_max": 34.044883728027344,
      "activations/layer11_attention_weight_min": -34.25613021850586,
      "activations/layer12_attention_weight_max": 18.127981185913086,
      "activations/layer12_attention_weight_min": -25.264951705932617,
      "activations/layer13_attention_weight_max": 36.37288284301758,
      "activations/layer13_attention_weight_min": -29.607789993286133,
      "activations/layer14_attention_weight_max": 39.45131301879883,
      "activations/layer14_attention_weight_min": -29.959104537963867,
      "activations/layer15_attention_weight_max": 33.87678527832031,
      "activations/layer15_attention_weight_min": -31.453773498535156,
      "activations/layer16_attention_weight_max": 30.69182014465332,
      "activations/layer16_attention_weight_min": -31.40753936767578,
      "activations/layer17_attention_weight_max": 50.593570709228516,
      "activations/layer17_attention_weight_min": -46.07716369628906,
      "activations/layer18_attention_weight_max": 45.8295783996582,
      "activations/layer18_attention_weight_min": -39.71807098388672,
      "activations/layer19_attention_weight_max": 21.390180587768555,
      "activations/layer19_attention_weight_min": -22.908193588256836,
      "activations/layer1_attention_weight_max": 15.645862579345703,
      "activations/layer1_attention_weight_min": -18.27130889892578,
      "activations/layer20_attention_weight_max": 22.82242774963379,
      "activations/layer20_attention_weight_min": -21.959970474243164,
      "activations/layer21_attention_weight_max": 35.8175048828125,
      "activations/layer21_attention_weight_min": -23.543718338012695,
      "activations/layer22_attention_weight_max": 29.08883285522461,
      "activations/layer22_attention_weight_min": -26.046648025512695,
      "activations/layer23_attention_weight_max": 33.7471923828125,
      "activations/layer23_attention_weight_min": -23.80652618408203,
      "activations/layer2_attention_weight_max": 32.793800354003906,
      "activations/layer2_attention_weight_min": -34.90492630004883,
      "activations/layer3_attention_weight_max": 92.66218566894531,
      "activations/layer3_attention_weight_min": -103.58232116699219,
      "activations/layer4_attention_weight_max": 92.64280700683594,
      "activations/layer4_attention_weight_min": -91.98113250732422,
      "activations/layer5_attention_weight_max": 70.12149047851562,
      "activations/layer5_attention_weight_min": -74.47039031982422,
      "activations/layer6_attention_weight_max": 53.30022048950195,
      "activations/layer6_attention_weight_min": -52.33444595336914,
      "activations/layer7_attention_weight_max": 66.5674057006836,
      "activations/layer7_attention_weight_min": -67.43418884277344,
      "activations/layer8_attention_weight_max": 49.55717086791992,
      "activations/layer8_attention_weight_min": -50.233856201171875,
      "activations/layer9_attention_weight_max": 49.397705078125,
      "activations/layer9_attention_weight_min": -46.292442321777344,
      "epoch": 14.9,
      "learning_rate": 5.4435984848484845e-05,
      "loss": 2.7595,
      "step": 256500
    },
    {
      "activations/layer0_attention_weight_max": 16.139022827148438,
      "activations/layer0_attention_weight_min": -13.619976043701172,
      "activations/layer10_attention_weight_max": 36.3806037902832,
      "activations/layer10_attention_weight_min": -34.52299499511719,
      "activations/layer11_attention_weight_max": 34.62104797363281,
      "activations/layer11_attention_weight_min": -33.903968811035156,
      "activations/layer12_attention_weight_max": 19.665586471557617,
      "activations/layer12_attention_weight_min": -23.123559951782227,
      "activations/layer13_attention_weight_max": 35.119667053222656,
      "activations/layer13_attention_weight_min": -26.79619789123535,
      "activations/layer14_attention_weight_max": 36.52957534790039,
      "activations/layer14_attention_weight_min": -32.71714401245117,
      "activations/layer15_attention_weight_max": 33.373714447021484,
      "activations/layer15_attention_weight_min": -29.84029769897461,
      "activations/layer16_attention_weight_max": 35.4221076965332,
      "activations/layer16_attention_weight_min": -31.45514678955078,
      "activations/layer17_attention_weight_max": 54.8210334777832,
      "activations/layer17_attention_weight_min": -47.53010940551758,
      "activations/layer18_attention_weight_max": 48.164695739746094,
      "activations/layer18_attention_weight_min": -39.209442138671875,
      "activations/layer19_attention_weight_max": 22.5014705657959,
      "activations/layer19_attention_weight_min": -23.13450050354004,
      "activations/layer1_attention_weight_max": 15.764365196228027,
      "activations/layer1_attention_weight_min": -14.805719375610352,
      "activations/layer20_attention_weight_max": 22.748119354248047,
      "activations/layer20_attention_weight_min": -21.499467849731445,
      "activations/layer21_attention_weight_max": 35.42731475830078,
      "activations/layer21_attention_weight_min": -22.976959228515625,
      "activations/layer22_attention_weight_max": 29.855743408203125,
      "activations/layer22_attention_weight_min": -27.02250862121582,
      "activations/layer23_attention_weight_max": 36.12264633178711,
      "activations/layer23_attention_weight_min": -22.43659019470215,
      "activations/layer2_attention_weight_max": 32.089263916015625,
      "activations/layer2_attention_weight_min": -30.33283233642578,
      "activations/layer3_attention_weight_max": 91.2950668334961,
      "activations/layer3_attention_weight_min": -91.40788269042969,
      "activations/layer4_attention_weight_max": 97.61285400390625,
      "activations/layer4_attention_weight_min": -93.97908782958984,
      "activations/layer5_attention_weight_max": 69.85618591308594,
      "activations/layer5_attention_weight_min": -75.54911804199219,
      "activations/layer6_attention_weight_max": 55.28709411621094,
      "activations/layer6_attention_weight_min": -52.71988296508789,
      "activations/layer7_attention_weight_max": 69.79981994628906,
      "activations/layer7_attention_weight_min": -62.87348937988281,
      "activations/layer8_attention_weight_max": 47.2326774597168,
      "activations/layer8_attention_weight_min": -48.345516204833984,
      "activations/layer9_attention_weight_max": 52.65308380126953,
      "activations/layer9_attention_weight_min": -48.185054779052734,
      "epoch": 14.91,
      "learning_rate": 5.4417045454545446e-05,
      "loss": 2.7637,
      "step": 256550
    },
    {
      "activations/layer0_attention_weight_max": 15.822273254394531,
      "activations/layer0_attention_weight_min": -13.450409889221191,
      "activations/layer10_attention_weight_max": 38.06425094604492,
      "activations/layer10_attention_weight_min": -39.96764373779297,
      "activations/layer11_attention_weight_max": 35.42040252685547,
      "activations/layer11_attention_weight_min": -37.22886276245117,
      "activations/layer12_attention_weight_max": 20.185941696166992,
      "activations/layer12_attention_weight_min": -25.581323623657227,
      "activations/layer13_attention_weight_max": 44.80052947998047,
      "activations/layer13_attention_weight_min": -32.31130599975586,
      "activations/layer14_attention_weight_max": 40.827083587646484,
      "activations/layer14_attention_weight_min": -31.314701080322266,
      "activations/layer15_attention_weight_max": 35.27141189575195,
      "activations/layer15_attention_weight_min": -30.67681884765625,
      "activations/layer16_attention_weight_max": 35.96229553222656,
      "activations/layer16_attention_weight_min": -33.37674331665039,
      "activations/layer17_attention_weight_max": 55.397865295410156,
      "activations/layer17_attention_weight_min": -47.23540496826172,
      "activations/layer18_attention_weight_max": 51.20571517944336,
      "activations/layer18_attention_weight_min": -43.12474822998047,
      "activations/layer19_attention_weight_max": 24.714632034301758,
      "activations/layer19_attention_weight_min": -26.507490158081055,
      "activations/layer1_attention_weight_max": 17.11284637451172,
      "activations/layer1_attention_weight_min": -15.496819496154785,
      "activations/layer20_attention_weight_max": 23.9627685546875,
      "activations/layer20_attention_weight_min": -28.24089813232422,
      "activations/layer21_attention_weight_max": 37.656856536865234,
      "activations/layer21_attention_weight_min": -23.8199405670166,
      "activations/layer22_attention_weight_max": 31.564193725585938,
      "activations/layer22_attention_weight_min": -27.59191131591797,
      "activations/layer23_attention_weight_max": 39.654842376708984,
      "activations/layer23_attention_weight_min": -25.43488311767578,
      "activations/layer2_attention_weight_max": 33.78235626220703,
      "activations/layer2_attention_weight_min": -32.03348922729492,
      "activations/layer3_attention_weight_max": 94.7412338256836,
      "activations/layer3_attention_weight_min": -97.36007690429688,
      "activations/layer4_attention_weight_max": 92.76177215576172,
      "activations/layer4_attention_weight_min": -91.79876708984375,
      "activations/layer5_attention_weight_max": 66.53350067138672,
      "activations/layer5_attention_weight_min": -79.52738952636719,
      "activations/layer6_attention_weight_max": 52.513099670410156,
      "activations/layer6_attention_weight_min": -49.9323616027832,
      "activations/layer7_attention_weight_max": 61.07133102416992,
      "activations/layer7_attention_weight_min": -63.695655822753906,
      "activations/layer8_attention_weight_max": 47.6444091796875,
      "activations/layer8_attention_weight_min": -48.69465637207031,
      "activations/layer9_attention_weight_max": 45.931575775146484,
      "activations/layer9_attention_weight_min": -49.16516876220703,
      "epoch": 14.91,
      "learning_rate": 5.439810606060606e-05,
      "loss": 2.7599,
      "step": 256600
    },
    {
      "activations/layer0_attention_weight_max": 16.43260955810547,
      "activations/layer0_attention_weight_min": -13.457756996154785,
      "activations/layer10_attention_weight_max": 35.66862487792969,
      "activations/layer10_attention_weight_min": -35.79058837890625,
      "activations/layer11_attention_weight_max": 30.694705963134766,
      "activations/layer11_attention_weight_min": -34.455047607421875,
      "activations/layer12_attention_weight_max": 19.7774658203125,
      "activations/layer12_attention_weight_min": -25.69532585144043,
      "activations/layer13_attention_weight_max": 37.125423431396484,
      "activations/layer13_attention_weight_min": -30.235065460205078,
      "activations/layer14_attention_weight_max": 36.067115783691406,
      "activations/layer14_attention_weight_min": -31.213029861450195,
      "activations/layer15_attention_weight_max": 32.91883087158203,
      "activations/layer15_attention_weight_min": -29.622188568115234,
      "activations/layer16_attention_weight_max": 34.86687469482422,
      "activations/layer16_attention_weight_min": -33.0083122253418,
      "activations/layer17_attention_weight_max": 54.95978927612305,
      "activations/layer17_attention_weight_min": -47.08128356933594,
      "activations/layer18_attention_weight_max": 47.168212890625,
      "activations/layer18_attention_weight_min": -40.65361022949219,
      "activations/layer19_attention_weight_max": 23.260066986083984,
      "activations/layer19_attention_weight_min": -25.358375549316406,
      "activations/layer1_attention_weight_max": 16.596477508544922,
      "activations/layer1_attention_weight_min": -14.88580322265625,
      "activations/layer20_attention_weight_max": 22.377605438232422,
      "activations/layer20_attention_weight_min": -22.813079833984375,
      "activations/layer21_attention_weight_max": 38.66450881958008,
      "activations/layer21_attention_weight_min": -23.59416389465332,
      "activations/layer22_attention_weight_max": 29.67719841003418,
      "activations/layer22_attention_weight_min": -26.95806312561035,
      "activations/layer23_attention_weight_max": 39.081092834472656,
      "activations/layer23_attention_weight_min": -26.33161735534668,
      "activations/layer2_attention_weight_max": 31.43576431274414,
      "activations/layer2_attention_weight_min": -29.063411712646484,
      "activations/layer3_attention_weight_max": 89.9258041381836,
      "activations/layer3_attention_weight_min": -90.08671569824219,
      "activations/layer4_attention_weight_max": 90.87913513183594,
      "activations/layer4_attention_weight_min": -86.6838607788086,
      "activations/layer5_attention_weight_max": 64.98054504394531,
      "activations/layer5_attention_weight_min": -75.5787124633789,
      "activations/layer6_attention_weight_max": 50.330196380615234,
      "activations/layer6_attention_weight_min": -50.84823226928711,
      "activations/layer7_attention_weight_max": 64.38568115234375,
      "activations/layer7_attention_weight_min": -60.40456771850586,
      "activations/layer8_attention_weight_max": 48.848052978515625,
      "activations/layer8_attention_weight_min": -49.17961120605469,
      "activations/layer9_attention_weight_max": 47.762908935546875,
      "activations/layer9_attention_weight_min": -46.415828704833984,
      "epoch": 14.91,
      "learning_rate": 5.437916666666666e-05,
      "loss": 2.7684,
      "step": 256650
    },
    {
      "activations/layer0_attention_weight_max": 16.266321182250977,
      "activations/layer0_attention_weight_min": -13.53200912475586,
      "activations/layer10_attention_weight_max": 38.210384368896484,
      "activations/layer10_attention_weight_min": -38.65665817260742,
      "activations/layer11_attention_weight_max": 33.440895080566406,
      "activations/layer11_attention_weight_min": -34.70771789550781,
      "activations/layer12_attention_weight_max": 17.75018882751465,
      "activations/layer12_attention_weight_min": -25.24857521057129,
      "activations/layer13_attention_weight_max": 34.34743881225586,
      "activations/layer13_attention_weight_min": -28.715744018554688,
      "activations/layer14_attention_weight_max": 34.797401428222656,
      "activations/layer14_attention_weight_min": -29.812559127807617,
      "activations/layer15_attention_weight_max": 32.10451126098633,
      "activations/layer15_attention_weight_min": -29.849014282226562,
      "activations/layer16_attention_weight_max": 30.393489837646484,
      "activations/layer16_attention_weight_min": -29.41578483581543,
      "activations/layer17_attention_weight_max": 52.08774185180664,
      "activations/layer17_attention_weight_min": -42.682254791259766,
      "activations/layer18_attention_weight_max": 42.60239028930664,
      "activations/layer18_attention_weight_min": -41.033512115478516,
      "activations/layer19_attention_weight_max": 20.980478286743164,
      "activations/layer19_attention_weight_min": -22.768844604492188,
      "activations/layer1_attention_weight_max": 17.13665008544922,
      "activations/layer1_attention_weight_min": -14.576271057128906,
      "activations/layer20_attention_weight_max": 23.876256942749023,
      "activations/layer20_attention_weight_min": -22.10691261291504,
      "activations/layer21_attention_weight_max": 34.837276458740234,
      "activations/layer21_attention_weight_min": -22.487646102905273,
      "activations/layer22_attention_weight_max": 27.999698638916016,
      "activations/layer22_attention_weight_min": -25.865734100341797,
      "activations/layer23_attention_weight_max": 33.1158447265625,
      "activations/layer23_attention_weight_min": -24.58383560180664,
      "activations/layer2_attention_weight_max": 32.449405670166016,
      "activations/layer2_attention_weight_min": -30.920761108398438,
      "activations/layer3_attention_weight_max": 97.05979919433594,
      "activations/layer3_attention_weight_min": -94.91864776611328,
      "activations/layer4_attention_weight_max": 94.90776062011719,
      "activations/layer4_attention_weight_min": -91.77758026123047,
      "activations/layer5_attention_weight_max": 69.25856018066406,
      "activations/layer5_attention_weight_min": -72.42613983154297,
      "activations/layer6_attention_weight_max": 50.58203887939453,
      "activations/layer6_attention_weight_min": -50.860130310058594,
      "activations/layer7_attention_weight_max": 62.401458740234375,
      "activations/layer7_attention_weight_min": -63.78007888793945,
      "activations/layer8_attention_weight_max": 46.974395751953125,
      "activations/layer8_attention_weight_min": -51.3327751159668,
      "activations/layer9_attention_weight_max": 46.12238693237305,
      "activations/layer9_attention_weight_min": -49.2011833190918,
      "epoch": 14.92,
      "learning_rate": 5.4360227272727264e-05,
      "loss": 2.766,
      "step": 256700
    },
    {
      "activations/layer0_attention_weight_max": 16.63572120666504,
      "activations/layer0_attention_weight_min": -13.512369155883789,
      "activations/layer10_attention_weight_max": 36.332862854003906,
      "activations/layer10_attention_weight_min": -35.48992919921875,
      "activations/layer11_attention_weight_max": 30.954532623291016,
      "activations/layer11_attention_weight_min": -30.784870147705078,
      "activations/layer12_attention_weight_max": 17.72589874267578,
      "activations/layer12_attention_weight_min": -25.92576026916504,
      "activations/layer13_attention_weight_max": 30.496740341186523,
      "activations/layer13_attention_weight_min": -28.403095245361328,
      "activations/layer14_attention_weight_max": 33.16987228393555,
      "activations/layer14_attention_weight_min": -29.164344787597656,
      "activations/layer15_attention_weight_max": 33.12058639526367,
      "activations/layer15_attention_weight_min": -28.97063636779785,
      "activations/layer16_attention_weight_max": 30.02336311340332,
      "activations/layer16_attention_weight_min": -29.107688903808594,
      "activations/layer17_attention_weight_max": 50.73084259033203,
      "activations/layer17_attention_weight_min": -44.498207092285156,
      "activations/layer18_attention_weight_max": 44.99440002441406,
      "activations/layer18_attention_weight_min": -38.312076568603516,
      "activations/layer19_attention_weight_max": 21.898632049560547,
      "activations/layer19_attention_weight_min": -23.605031967163086,
      "activations/layer1_attention_weight_max": 15.783326148986816,
      "activations/layer1_attention_weight_min": -15.640299797058105,
      "activations/layer20_attention_weight_max": 20.171056747436523,
      "activations/layer20_attention_weight_min": -24.52167320251465,
      "activations/layer21_attention_weight_max": 36.963443756103516,
      "activations/layer21_attention_weight_min": -24.597753524780273,
      "activations/layer22_attention_weight_max": 28.4671630859375,
      "activations/layer22_attention_weight_min": -24.709444046020508,
      "activations/layer23_attention_weight_max": 35.0110969543457,
      "activations/layer23_attention_weight_min": -23.222888946533203,
      "activations/layer2_attention_weight_max": 32.20441436767578,
      "activations/layer2_attention_weight_min": -31.656570434570312,
      "activations/layer3_attention_weight_max": 90.42949676513672,
      "activations/layer3_attention_weight_min": -97.9217758178711,
      "activations/layer4_attention_weight_max": 91.74349212646484,
      "activations/layer4_attention_weight_min": -88.0429916381836,
      "activations/layer5_attention_weight_max": 65.38807678222656,
      "activations/layer5_attention_weight_min": -71.200927734375,
      "activations/layer6_attention_weight_max": 51.1365852355957,
      "activations/layer6_attention_weight_min": -50.62364196777344,
      "activations/layer7_attention_weight_max": 62.034690856933594,
      "activations/layer7_attention_weight_min": -60.96464920043945,
      "activations/layer8_attention_weight_max": 45.31791305541992,
      "activations/layer8_attention_weight_min": -47.630699157714844,
      "activations/layer9_attention_weight_max": 44.9186897277832,
      "activations/layer9_attention_weight_min": -44.37623977661133,
      "epoch": 14.92,
      "learning_rate": 5.434128787878787e-05,
      "loss": 2.751,
      "step": 256750
    },
    {
      "activations/layer0_attention_weight_max": 15.722855567932129,
      "activations/layer0_attention_weight_min": -13.791582107543945,
      "activations/layer10_attention_weight_max": 35.785797119140625,
      "activations/layer10_attention_weight_min": -36.720977783203125,
      "activations/layer11_attention_weight_max": 31.158063888549805,
      "activations/layer11_attention_weight_min": -32.489418029785156,
      "activations/layer12_attention_weight_max": 18.912006378173828,
      "activations/layer12_attention_weight_min": -24.5954647064209,
      "activations/layer13_attention_weight_max": 36.58564758300781,
      "activations/layer13_attention_weight_min": -28.27377700805664,
      "activations/layer14_attention_weight_max": 37.04170227050781,
      "activations/layer14_attention_weight_min": -29.92807388305664,
      "activations/layer15_attention_weight_max": 32.95069885253906,
      "activations/layer15_attention_weight_min": -29.82098960876465,
      "activations/layer16_attention_weight_max": 33.124629974365234,
      "activations/layer16_attention_weight_min": -31.90210723876953,
      "activations/layer17_attention_weight_max": 52.068363189697266,
      "activations/layer17_attention_weight_min": -46.93306350708008,
      "activations/layer18_attention_weight_max": 47.17399978637695,
      "activations/layer18_attention_weight_min": -41.36805725097656,
      "activations/layer19_attention_weight_max": 23.46785545349121,
      "activations/layer19_attention_weight_min": -23.167394638061523,
      "activations/layer1_attention_weight_max": 16.34588050842285,
      "activations/layer1_attention_weight_min": -14.831663131713867,
      "activations/layer20_attention_weight_max": 23.700910568237305,
      "activations/layer20_attention_weight_min": -21.841081619262695,
      "activations/layer21_attention_weight_max": 44.5942268371582,
      "activations/layer21_attention_weight_min": -26.80312728881836,
      "activations/layer22_attention_weight_max": 29.379371643066406,
      "activations/layer22_attention_weight_min": -26.507593154907227,
      "activations/layer23_attention_weight_max": 35.18926239013672,
      "activations/layer23_attention_weight_min": -23.82190704345703,
      "activations/layer2_attention_weight_max": 33.16591262817383,
      "activations/layer2_attention_weight_min": -31.682106018066406,
      "activations/layer3_attention_weight_max": 94.57221984863281,
      "activations/layer3_attention_weight_min": -96.89006042480469,
      "activations/layer4_attention_weight_max": 93.50122833251953,
      "activations/layer4_attention_weight_min": -92.6624755859375,
      "activations/layer5_attention_weight_max": 69.60926818847656,
      "activations/layer5_attention_weight_min": -78.90846252441406,
      "activations/layer6_attention_weight_max": 49.98149490356445,
      "activations/layer6_attention_weight_min": -53.33061218261719,
      "activations/layer7_attention_weight_max": 65.26825714111328,
      "activations/layer7_attention_weight_min": -60.46636199951172,
      "activations/layer8_attention_weight_max": 45.34933090209961,
      "activations/layer8_attention_weight_min": -50.06320571899414,
      "activations/layer9_attention_weight_max": 45.14316177368164,
      "activations/layer9_attention_weight_min": -48.969547271728516,
      "epoch": 14.92,
      "learning_rate": 5.432234848484848e-05,
      "loss": 2.7627,
      "step": 256800
    },
    {
      "activations/layer0_attention_weight_max": 16.492305755615234,
      "activations/layer0_attention_weight_min": -13.580263137817383,
      "activations/layer10_attention_weight_max": 37.46543884277344,
      "activations/layer10_attention_weight_min": -33.6826057434082,
      "activations/layer11_attention_weight_max": 32.66267776489258,
      "activations/layer11_attention_weight_min": -31.649351119995117,
      "activations/layer12_attention_weight_max": 19.221853256225586,
      "activations/layer12_attention_weight_min": -26.241622924804688,
      "activations/layer13_attention_weight_max": 43.612648010253906,
      "activations/layer13_attention_weight_min": -33.05882263183594,
      "activations/layer14_attention_weight_max": 40.148284912109375,
      "activations/layer14_attention_weight_min": -30.752153396606445,
      "activations/layer15_attention_weight_max": 34.94654083251953,
      "activations/layer15_attention_weight_min": -30.07054901123047,
      "activations/layer16_attention_weight_max": 34.14369201660156,
      "activations/layer16_attention_weight_min": -30.963146209716797,
      "activations/layer17_attention_weight_max": 55.34830856323242,
      "activations/layer17_attention_weight_min": -43.306251525878906,
      "activations/layer18_attention_weight_max": 51.17741012573242,
      "activations/layer18_attention_weight_min": -38.795188903808594,
      "activations/layer19_attention_weight_max": 24.408597946166992,
      "activations/layer19_attention_weight_min": -21.179651260375977,
      "activations/layer1_attention_weight_max": 15.877305030822754,
      "activations/layer1_attention_weight_min": -14.750540733337402,
      "activations/layer20_attention_weight_max": 21.919925689697266,
      "activations/layer20_attention_weight_min": -21.277244567871094,
      "activations/layer21_attention_weight_max": 35.812774658203125,
      "activations/layer21_attention_weight_min": -22.046323776245117,
      "activations/layer22_attention_weight_max": 36.17363357543945,
      "activations/layer22_attention_weight_min": -25.18975830078125,
      "activations/layer23_attention_weight_max": 45.69596481323242,
      "activations/layer23_attention_weight_min": -24.313804626464844,
      "activations/layer2_attention_weight_max": 32.2072868347168,
      "activations/layer2_attention_weight_min": -31.556058883666992,
      "activations/layer3_attention_weight_max": 88.87101745605469,
      "activations/layer3_attention_weight_min": -94.39064025878906,
      "activations/layer4_attention_weight_max": 89.04890441894531,
      "activations/layer4_attention_weight_min": -86.2059555053711,
      "activations/layer5_attention_weight_max": 69.56350708007812,
      "activations/layer5_attention_weight_min": -71.8455581665039,
      "activations/layer6_attention_weight_max": 50.345741271972656,
      "activations/layer6_attention_weight_min": -49.118080139160156,
      "activations/layer7_attention_weight_max": 65.20175170898438,
      "activations/layer7_attention_weight_min": -61.522037506103516,
      "activations/layer8_attention_weight_max": 46.35683059692383,
      "activations/layer8_attention_weight_min": -46.0287971496582,
      "activations/layer9_attention_weight_max": 44.91437911987305,
      "activations/layer9_attention_weight_min": -43.885990142822266,
      "epoch": 14.92,
      "learning_rate": 5.430340909090909e-05,
      "loss": 2.7586,
      "step": 256850
    },
    {
      "activations/layer0_attention_weight_max": 16.046167373657227,
      "activations/layer0_attention_weight_min": -14.386579513549805,
      "activations/layer10_attention_weight_max": 36.824012756347656,
      "activations/layer10_attention_weight_min": -36.194129943847656,
      "activations/layer11_attention_weight_max": 32.71921157836914,
      "activations/layer11_attention_weight_min": -34.59716796875,
      "activations/layer12_attention_weight_max": 19.49041175842285,
      "activations/layer12_attention_weight_min": -31.632099151611328,
      "activations/layer13_attention_weight_max": 33.859458923339844,
      "activations/layer13_attention_weight_min": -30.033973693847656,
      "activations/layer14_attention_weight_max": 34.175777435302734,
      "activations/layer14_attention_weight_min": -30.587814331054688,
      "activations/layer15_attention_weight_max": 31.08595085144043,
      "activations/layer15_attention_weight_min": -30.04722023010254,
      "activations/layer16_attention_weight_max": 30.54306983947754,
      "activations/layer16_attention_weight_min": -31.063257217407227,
      "activations/layer17_attention_weight_max": 48.39910888671875,
      "activations/layer17_attention_weight_min": -43.28019714355469,
      "activations/layer18_attention_weight_max": 42.5948486328125,
      "activations/layer18_attention_weight_min": -37.737361907958984,
      "activations/layer19_attention_weight_max": 21.482229232788086,
      "activations/layer19_attention_weight_min": -23.923364639282227,
      "activations/layer1_attention_weight_max": 16.18830680847168,
      "activations/layer1_attention_weight_min": -16.817607879638672,
      "activations/layer20_attention_weight_max": 19.713525772094727,
      "activations/layer20_attention_weight_min": -23.667402267456055,
      "activations/layer21_attention_weight_max": 30.264066696166992,
      "activations/layer21_attention_weight_min": -20.70345115661621,
      "activations/layer22_attention_weight_max": 27.179462432861328,
      "activations/layer22_attention_weight_min": -26.3889102935791,
      "activations/layer23_attention_weight_max": 32.967323303222656,
      "activations/layer23_attention_weight_min": -22.459991455078125,
      "activations/layer2_attention_weight_max": 33.20109558105469,
      "activations/layer2_attention_weight_min": -31.295377731323242,
      "activations/layer3_attention_weight_max": 96.72136688232422,
      "activations/layer3_attention_weight_min": -95.23319244384766,
      "activations/layer4_attention_weight_max": 95.31908416748047,
      "activations/layer4_attention_weight_min": -89.36970520019531,
      "activations/layer5_attention_weight_max": 66.91957092285156,
      "activations/layer5_attention_weight_min": -75.63249206542969,
      "activations/layer6_attention_weight_max": 49.62923049926758,
      "activations/layer6_attention_weight_min": -50.068538665771484,
      "activations/layer7_attention_weight_max": 64.98096466064453,
      "activations/layer7_attention_weight_min": -64.46778869628906,
      "activations/layer8_attention_weight_max": 45.46857833862305,
      "activations/layer8_attention_weight_min": -47.26131057739258,
      "activations/layer9_attention_weight_max": 45.599727630615234,
      "activations/layer9_attention_weight_min": -47.47420883178711,
      "epoch": 14.93,
      "learning_rate": 5.428446969696969e-05,
      "loss": 2.7657,
      "step": 256900
    },
    {
      "activations/layer0_attention_weight_max": 15.163390159606934,
      "activations/layer0_attention_weight_min": -13.771109580993652,
      "activations/layer10_attention_weight_max": 35.85979080200195,
      "activations/layer10_attention_weight_min": -35.514617919921875,
      "activations/layer11_attention_weight_max": 33.081417083740234,
      "activations/layer11_attention_weight_min": -33.117218017578125,
      "activations/layer12_attention_weight_max": 18.003652572631836,
      "activations/layer12_attention_weight_min": -24.355953216552734,
      "activations/layer13_attention_weight_max": 31.920076370239258,
      "activations/layer13_attention_weight_min": -27.27488899230957,
      "activations/layer14_attention_weight_max": 39.00754165649414,
      "activations/layer14_attention_weight_min": -30.00177001953125,
      "activations/layer15_attention_weight_max": 32.80231475830078,
      "activations/layer15_attention_weight_min": -29.481618881225586,
      "activations/layer16_attention_weight_max": 32.88938522338867,
      "activations/layer16_attention_weight_min": -29.645647048950195,
      "activations/layer17_attention_weight_max": 53.579612731933594,
      "activations/layer17_attention_weight_min": -44.751136779785156,
      "activations/layer18_attention_weight_max": 47.761383056640625,
      "activations/layer18_attention_weight_min": -39.329280853271484,
      "activations/layer19_attention_weight_max": 22.95949363708496,
      "activations/layer19_attention_weight_min": -23.9809627532959,
      "activations/layer1_attention_weight_max": 15.90675163269043,
      "activations/layer1_attention_weight_min": -16.70103645324707,
      "activations/layer20_attention_weight_max": 22.802824020385742,
      "activations/layer20_attention_weight_min": -22.771947860717773,
      "activations/layer21_attention_weight_max": 37.88411331176758,
      "activations/layer21_attention_weight_min": -24.003875732421875,
      "activations/layer22_attention_weight_max": 29.591583251953125,
      "activations/layer22_attention_weight_min": -26.066240310668945,
      "activations/layer23_attention_weight_max": 38.777156829833984,
      "activations/layer23_attention_weight_min": -24.228986740112305,
      "activations/layer2_attention_weight_max": 34.01348876953125,
      "activations/layer2_attention_weight_min": -31.324630737304688,
      "activations/layer3_attention_weight_max": 92.43773651123047,
      "activations/layer3_attention_weight_min": -91.27071380615234,
      "activations/layer4_attention_weight_max": 89.66051483154297,
      "activations/layer4_attention_weight_min": -88.70731353759766,
      "activations/layer5_attention_weight_max": 65.91380310058594,
      "activations/layer5_attention_weight_min": -72.51910400390625,
      "activations/layer6_attention_weight_max": 50.55044174194336,
      "activations/layer6_attention_weight_min": -47.52587127685547,
      "activations/layer7_attention_weight_max": 58.56121826171875,
      "activations/layer7_attention_weight_min": -59.64669418334961,
      "activations/layer8_attention_weight_max": 47.56657409667969,
      "activations/layer8_attention_weight_min": -45.244205474853516,
      "activations/layer9_attention_weight_max": 43.39200973510742,
      "activations/layer9_attention_weight_min": -44.71883010864258,
      "epoch": 14.93,
      "learning_rate": 5.4265530303030305e-05,
      "loss": 2.7662,
      "step": 256950
    },
    {
      "activations/layer0_attention_weight_max": 15.931130409240723,
      "activations/layer0_attention_weight_min": -13.905725479125977,
      "activations/layer10_attention_weight_max": 39.47111129760742,
      "activations/layer10_attention_weight_min": -34.70709228515625,
      "activations/layer11_attention_weight_max": 36.5238037109375,
      "activations/layer11_attention_weight_min": -33.98758316040039,
      "activations/layer12_attention_weight_max": 19.44768524169922,
      "activations/layer12_attention_weight_min": -38.191017150878906,
      "activations/layer13_attention_weight_max": 44.94822311401367,
      "activations/layer13_attention_weight_min": -29.90158462524414,
      "activations/layer14_attention_weight_max": 50.28841781616211,
      "activations/layer14_attention_weight_min": -32.214508056640625,
      "activations/layer15_attention_weight_max": 40.48421096801758,
      "activations/layer15_attention_weight_min": -30.084148406982422,
      "activations/layer16_attention_weight_max": 36.520626068115234,
      "activations/layer16_attention_weight_min": -32.72084426879883,
      "activations/layer17_attention_weight_max": 56.60529327392578,
      "activations/layer17_attention_weight_min": -46.61707305908203,
      "activations/layer18_attention_weight_max": 52.17778015136719,
      "activations/layer18_attention_weight_min": -42.28926086425781,
      "activations/layer19_attention_weight_max": 28.29960060119629,
      "activations/layer19_attention_weight_min": -22.7525577545166,
      "activations/layer1_attention_weight_max": 16.956459045410156,
      "activations/layer1_attention_weight_min": -14.867661476135254,
      "activations/layer20_attention_weight_max": 29.7341251373291,
      "activations/layer20_attention_weight_min": -23.72045135498047,
      "activations/layer21_attention_weight_max": 61.287437438964844,
      "activations/layer21_attention_weight_min": -30.193307876586914,
      "activations/layer22_attention_weight_max": 32.35182189941406,
      "activations/layer22_attention_weight_min": -25.16628646850586,
      "activations/layer23_attention_weight_max": 45.61060333251953,
      "activations/layer23_attention_weight_min": -26.781463623046875,
      "activations/layer2_attention_weight_max": 33.24205780029297,
      "activations/layer2_attention_weight_min": -31.054256439208984,
      "activations/layer3_attention_weight_max": 96.68568420410156,
      "activations/layer3_attention_weight_min": -95.9970703125,
      "activations/layer4_attention_weight_max": 93.08150482177734,
      "activations/layer4_attention_weight_min": -89.23638916015625,
      "activations/layer5_attention_weight_max": 70.71189880371094,
      "activations/layer5_attention_weight_min": -74.32110595703125,
      "activations/layer6_attention_weight_max": 52.17830276489258,
      "activations/layer6_attention_weight_min": -47.26622772216797,
      "activations/layer7_attention_weight_max": 62.26836395263672,
      "activations/layer7_attention_weight_min": -61.13419723510742,
      "activations/layer8_attention_weight_max": 47.72466278076172,
      "activations/layer8_attention_weight_min": -46.76508331298828,
      "activations/layer9_attention_weight_max": 48.67654037475586,
      "activations/layer9_attention_weight_min": -47.107357025146484,
      "epoch": 14.93,
      "learning_rate": 5.424659090909091e-05,
      "loss": 2.7514,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4456,
      "eval_samples_per_second": 508.433,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4456,
      "eval_openwebtext_samples_per_second": 508.433,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9366,
      "eval_wikitext_samples_per_second": 235.469,
      "step": 257000
    },
    {
      "epoch": 14.93,
      "eval_lambada_loss": 2.48828125,
      "eval_lambada_ppl": 12.040563607027826,
      "eval_lambada_runtime": 9.513,
      "eval_lambada_samples_per_second": 511.828,
      "step": 257000
    },
    {
      "activations/layer0_attention_weight_max": 16.198474884033203,
      "activations/layer0_attention_weight_min": -14.058960914611816,
      "activations/layer10_attention_weight_max": 35.14514923095703,
      "activations/layer10_attention_weight_min": -34.55058670043945,
      "activations/layer11_attention_weight_max": 31.497089385986328,
      "activations/layer11_attention_weight_min": -32.37682342529297,
      "activations/layer12_attention_weight_max": 18.96170425415039,
      "activations/layer12_attention_weight_min": -22.840085983276367,
      "activations/layer13_attention_weight_max": 34.57398986816406,
      "activations/layer13_attention_weight_min": -26.699464797973633,
      "activations/layer14_attention_weight_max": 36.876590728759766,
      "activations/layer14_attention_weight_min": -30.599836349487305,
      "activations/layer15_attention_weight_max": 34.384395599365234,
      "activations/layer15_attention_weight_min": -31.09992790222168,
      "activations/layer16_attention_weight_max": 35.09036636352539,
      "activations/layer16_attention_weight_min": -32.19004821777344,
      "activations/layer17_attention_weight_max": 57.95998001098633,
      "activations/layer17_attention_weight_min": -45.49433898925781,
      "activations/layer18_attention_weight_max": 50.266963958740234,
      "activations/layer18_attention_weight_min": -40.348060607910156,
      "activations/layer19_attention_weight_max": 24.184940338134766,
      "activations/layer19_attention_weight_min": -21.93284797668457,
      "activations/layer1_attention_weight_max": 15.978043556213379,
      "activations/layer1_attention_weight_min": -14.34609603881836,
      "activations/layer20_attention_weight_max": 23.919933319091797,
      "activations/layer20_attention_weight_min": -22.21454429626465,
      "activations/layer21_attention_weight_max": 35.49922561645508,
      "activations/layer21_attention_weight_min": -24.01732063293457,
      "activations/layer22_attention_weight_max": 29.46470069885254,
      "activations/layer22_attention_weight_min": -25.58858299255371,
      "activations/layer23_attention_weight_max": 37.70228576660156,
      "activations/layer23_attention_weight_min": -26.859214782714844,
      "activations/layer2_attention_weight_max": 33.07298278808594,
      "activations/layer2_attention_weight_min": -30.49341583251953,
      "activations/layer3_attention_weight_max": 88.76463317871094,
      "activations/layer3_attention_weight_min": -90.68647766113281,
      "activations/layer4_attention_weight_max": 88.45465087890625,
      "activations/layer4_attention_weight_min": -86.74000549316406,
      "activations/layer5_attention_weight_max": 66.76964569091797,
      "activations/layer5_attention_weight_min": -79.85002899169922,
      "activations/layer6_attention_weight_max": 49.50715255737305,
      "activations/layer6_attention_weight_min": -53.11130905151367,
      "activations/layer7_attention_weight_max": 64.42405700683594,
      "activations/layer7_attention_weight_min": -59.71403121948242,
      "activations/layer8_attention_weight_max": 43.594573974609375,
      "activations/layer8_attention_weight_min": -49.98463821411133,
      "activations/layer9_attention_weight_max": 42.306880950927734,
      "activations/layer9_attention_weight_min": -45.333126068115234,
      "epoch": 14.94,
      "learning_rate": 5.422765151515151e-05,
      "loss": 2.7832,
      "step": 257050
    },
    {
      "activations/layer0_attention_weight_max": 15.84878921508789,
      "activations/layer0_attention_weight_min": -14.128594398498535,
      "activations/layer10_attention_weight_max": 36.77252960205078,
      "activations/layer10_attention_weight_min": -36.78007507324219,
      "activations/layer11_attention_weight_max": 31.355838775634766,
      "activations/layer11_attention_weight_min": -32.31270217895508,
      "activations/layer12_attention_weight_max": 18.186050415039062,
      "activations/layer12_attention_weight_min": -24.85874366760254,
      "activations/layer13_attention_weight_max": 49.757320404052734,
      "activations/layer13_attention_weight_min": -36.35808563232422,
      "activations/layer14_attention_weight_max": 33.38960647583008,
      "activations/layer14_attention_weight_min": -30.105749130249023,
      "activations/layer15_attention_weight_max": 31.00448226928711,
      "activations/layer15_attention_weight_min": -29.94849395751953,
      "activations/layer16_attention_weight_max": 31.182409286499023,
      "activations/layer16_attention_weight_min": -28.470317840576172,
      "activations/layer17_attention_weight_max": 50.71458435058594,
      "activations/layer17_attention_weight_min": -42.82777786254883,
      "activations/layer18_attention_weight_max": 45.917755126953125,
      "activations/layer18_attention_weight_min": -37.225303649902344,
      "activations/layer19_attention_weight_max": 22.391889572143555,
      "activations/layer19_attention_weight_min": -21.113494873046875,
      "activations/layer1_attention_weight_max": 15.654850959777832,
      "activations/layer1_attention_weight_min": -14.451544761657715,
      "activations/layer20_attention_weight_max": 21.73514175415039,
      "activations/layer20_attention_weight_min": -20.118907928466797,
      "activations/layer21_attention_weight_max": 36.08195114135742,
      "activations/layer21_attention_weight_min": -22.127857208251953,
      "activations/layer22_attention_weight_max": 27.859041213989258,
      "activations/layer22_attention_weight_min": -25.5345458984375,
      "activations/layer23_attention_weight_max": 35.49932861328125,
      "activations/layer23_attention_weight_min": -24.389305114746094,
      "activations/layer2_attention_weight_max": 31.606544494628906,
      "activations/layer2_attention_weight_min": -30.715354919433594,
      "activations/layer3_attention_weight_max": 91.5973129272461,
      "activations/layer3_attention_weight_min": -97.06636047363281,
      "activations/layer4_attention_weight_max": 91.39313507080078,
      "activations/layer4_attention_weight_min": -92.41438293457031,
      "activations/layer5_attention_weight_max": 66.12867736816406,
      "activations/layer5_attention_weight_min": -75.02249145507812,
      "activations/layer6_attention_weight_max": 50.67304992675781,
      "activations/layer6_attention_weight_min": -52.42478561401367,
      "activations/layer7_attention_weight_max": 62.44460678100586,
      "activations/layer7_attention_weight_min": -64.33106994628906,
      "activations/layer8_attention_weight_max": 44.89210891723633,
      "activations/layer8_attention_weight_min": -48.30521011352539,
      "activations/layer9_attention_weight_max": 52.09519577026367,
      "activations/layer9_attention_weight_min": -47.824989318847656,
      "epoch": 14.94,
      "learning_rate": 5.420871212121212e-05,
      "loss": 2.7622,
      "step": 257100
    },
    {
      "activations/layer0_attention_weight_max": 16.26449203491211,
      "activations/layer0_attention_weight_min": -13.360029220581055,
      "activations/layer10_attention_weight_max": 34.134578704833984,
      "activations/layer10_attention_weight_min": -33.07398223876953,
      "activations/layer11_attention_weight_max": 30.288127899169922,
      "activations/layer11_attention_weight_min": -32.229339599609375,
      "activations/layer12_attention_weight_max": 19.079761505126953,
      "activations/layer12_attention_weight_min": -25.135547637939453,
      "activations/layer13_attention_weight_max": 35.39718246459961,
      "activations/layer13_attention_weight_min": -27.254940032958984,
      "activations/layer14_attention_weight_max": 36.61399841308594,
      "activations/layer14_attention_weight_min": -30.785343170166016,
      "activations/layer15_attention_weight_max": 34.439292907714844,
      "activations/layer15_attention_weight_min": -29.987682342529297,
      "activations/layer16_attention_weight_max": 31.792497634887695,
      "activations/layer16_attention_weight_min": -33.23755645751953,
      "activations/layer17_attention_weight_max": 48.40089416503906,
      "activations/layer17_attention_weight_min": -45.69426727294922,
      "activations/layer18_attention_weight_max": 43.0941162109375,
      "activations/layer18_attention_weight_min": -40.48185348510742,
      "activations/layer19_attention_weight_max": 21.207061767578125,
      "activations/layer19_attention_weight_min": -20.632234573364258,
      "activations/layer1_attention_weight_max": 16.58656120300293,
      "activations/layer1_attention_weight_min": -14.631781578063965,
      "activations/layer20_attention_weight_max": 20.12786102294922,
      "activations/layer20_attention_weight_min": -22.472623825073242,
      "activations/layer21_attention_weight_max": 35.263641357421875,
      "activations/layer21_attention_weight_min": -24.99361228942871,
      "activations/layer22_attention_weight_max": 27.768526077270508,
      "activations/layer22_attention_weight_min": -25.0449161529541,
      "activations/layer23_attention_weight_max": 34.57086181640625,
      "activations/layer23_attention_weight_min": -25.661176681518555,
      "activations/layer2_attention_weight_max": 30.5625,
      "activations/layer2_attention_weight_min": -30.285207748413086,
      "activations/layer3_attention_weight_max": 88.65320587158203,
      "activations/layer3_attention_weight_min": -86.13916015625,
      "activations/layer4_attention_weight_max": 90.53536987304688,
      "activations/layer4_attention_weight_min": -86.73552703857422,
      "activations/layer5_attention_weight_max": 65.82626342773438,
      "activations/layer5_attention_weight_min": -79.9464111328125,
      "activations/layer6_attention_weight_max": 50.000099182128906,
      "activations/layer6_attention_weight_min": -48.313453674316406,
      "activations/layer7_attention_weight_max": 59.41489028930664,
      "activations/layer7_attention_weight_min": -59.82484817504883,
      "activations/layer8_attention_weight_max": 42.016170501708984,
      "activations/layer8_attention_weight_min": -45.15280532836914,
      "activations/layer9_attention_weight_max": 42.685359954833984,
      "activations/layer9_attention_weight_min": -43.27880859375,
      "epoch": 14.94,
      "learning_rate": 5.4190151515151514e-05,
      "loss": 2.7504,
      "step": 257150
    },
    {
      "activations/layer0_attention_weight_max": 15.465100288391113,
      "activations/layer0_attention_weight_min": -13.8193998336792,
      "activations/layer10_attention_weight_max": 33.98103713989258,
      "activations/layer10_attention_weight_min": -33.8981819152832,
      "activations/layer11_attention_weight_max": 29.992671966552734,
      "activations/layer11_attention_weight_min": -31.97049331665039,
      "activations/layer12_attention_weight_max": 18.88385772705078,
      "activations/layer12_attention_weight_min": -25.35936164855957,
      "activations/layer13_attention_weight_max": 40.53440856933594,
      "activations/layer13_attention_weight_min": -29.222614288330078,
      "activations/layer14_attention_weight_max": 41.3006477355957,
      "activations/layer14_attention_weight_min": -35.26688003540039,
      "activations/layer15_attention_weight_max": 43.314823150634766,
      "activations/layer15_attention_weight_min": -34.86096954345703,
      "activations/layer16_attention_weight_max": 36.016326904296875,
      "activations/layer16_attention_weight_min": -36.443058013916016,
      "activations/layer17_attention_weight_max": 54.092552185058594,
      "activations/layer17_attention_weight_min": -54.46571731567383,
      "activations/layer18_attention_weight_max": 45.36840057373047,
      "activations/layer18_attention_weight_min": -45.74960708618164,
      "activations/layer19_attention_weight_max": 23.14486312866211,
      "activations/layer19_attention_weight_min": -28.066116333007812,
      "activations/layer1_attention_weight_max": 16.178380966186523,
      "activations/layer1_attention_weight_min": -14.191465377807617,
      "activations/layer20_attention_weight_max": 21.01540756225586,
      "activations/layer20_attention_weight_min": -23.554792404174805,
      "activations/layer21_attention_weight_max": 39.93169403076172,
      "activations/layer21_attention_weight_min": -26.559335708618164,
      "activations/layer22_attention_weight_max": 27.311033248901367,
      "activations/layer22_attention_weight_min": -27.914419174194336,
      "activations/layer23_attention_weight_max": 35.61400604248047,
      "activations/layer23_attention_weight_min": -30.994375228881836,
      "activations/layer2_attention_weight_max": 33.49086380004883,
      "activations/layer2_attention_weight_min": -31.02770233154297,
      "activations/layer3_attention_weight_max": 94.00064086914062,
      "activations/layer3_attention_weight_min": -93.81040954589844,
      "activations/layer4_attention_weight_max": 89.96636199951172,
      "activations/layer4_attention_weight_min": -88.83585357666016,
      "activations/layer5_attention_weight_max": 68.69287109375,
      "activations/layer5_attention_weight_min": -75.99696350097656,
      "activations/layer6_attention_weight_max": 50.05878829956055,
      "activations/layer6_attention_weight_min": -51.45894241333008,
      "activations/layer7_attention_weight_max": 62.90738296508789,
      "activations/layer7_attention_weight_min": -59.711158752441406,
      "activations/layer8_attention_weight_max": 45.346832275390625,
      "activations/layer8_attention_weight_min": -47.71080780029297,
      "activations/layer9_attention_weight_max": 46.00517272949219,
      "activations/layer9_attention_weight_min": -45.32589340209961,
      "epoch": 14.94,
      "learning_rate": 5.4171212121212116e-05,
      "loss": 2.7805,
      "step": 257200
    },
    {
      "activations/layer0_attention_weight_max": 16.490829467773438,
      "activations/layer0_attention_weight_min": -13.65064525604248,
      "activations/layer10_attention_weight_max": 34.88033676147461,
      "activations/layer10_attention_weight_min": -34.758522033691406,
      "activations/layer11_attention_weight_max": 30.599197387695312,
      "activations/layer11_attention_weight_min": -31.677738189697266,
      "activations/layer12_attention_weight_max": 20.20065689086914,
      "activations/layer12_attention_weight_min": -26.434303283691406,
      "activations/layer13_attention_weight_max": 35.70173645019531,
      "activations/layer13_attention_weight_min": -28.487924575805664,
      "activations/layer14_attention_weight_max": 37.15393829345703,
      "activations/layer14_attention_weight_min": -31.079381942749023,
      "activations/layer15_attention_weight_max": 35.12289047241211,
      "activations/layer15_attention_weight_min": -29.89067840576172,
      "activations/layer16_attention_weight_max": 35.59540557861328,
      "activations/layer16_attention_weight_min": -31.537235260009766,
      "activations/layer17_attention_weight_max": 54.71944808959961,
      "activations/layer17_attention_weight_min": -45.30560302734375,
      "activations/layer18_attention_weight_max": 49.26910400390625,
      "activations/layer18_attention_weight_min": -40.46722412109375,
      "activations/layer19_attention_weight_max": 24.883718490600586,
      "activations/layer19_attention_weight_min": -24.791074752807617,
      "activations/layer1_attention_weight_max": 17.33342933654785,
      "activations/layer1_attention_weight_min": -14.801384925842285,
      "activations/layer20_attention_weight_max": 23.999547958374023,
      "activations/layer20_attention_weight_min": -24.61409568786621,
      "activations/layer21_attention_weight_max": 36.832637786865234,
      "activations/layer21_attention_weight_min": -21.71219825744629,
      "activations/layer22_attention_weight_max": 29.843692779541016,
      "activations/layer22_attention_weight_min": -27.039262771606445,
      "activations/layer23_attention_weight_max": 40.45885467529297,
      "activations/layer23_attention_weight_min": -23.77692985534668,
      "activations/layer2_attention_weight_max": 26.08704376220703,
      "activations/layer2_attention_weight_min": -28.37325668334961,
      "activations/layer3_attention_weight_max": 78.37003326416016,
      "activations/layer3_attention_weight_min": -84.91045379638672,
      "activations/layer4_attention_weight_max": 82.6740951538086,
      "activations/layer4_attention_weight_min": -84.74546813964844,
      "activations/layer5_attention_weight_max": 59.24830627441406,
      "activations/layer5_attention_weight_min": -69.68102264404297,
      "activations/layer6_attention_weight_max": 43.612911224365234,
      "activations/layer6_attention_weight_min": -44.241146087646484,
      "activations/layer7_attention_weight_max": 58.12389373779297,
      "activations/layer7_attention_weight_min": -56.62407302856445,
      "activations/layer8_attention_weight_max": 40.676177978515625,
      "activations/layer8_attention_weight_min": -46.099151611328125,
      "activations/layer9_attention_weight_max": 43.55195236206055,
      "activations/layer9_attention_weight_min": -44.37483215332031,
      "epoch": 14.95,
      "learning_rate": 5.4152272727272724e-05,
      "loss": 2.765,
      "step": 257250
    },
    {
      "activations/layer0_attention_weight_max": 16.314599990844727,
      "activations/layer0_attention_weight_min": -13.493617057800293,
      "activations/layer10_attention_weight_max": 37.0135383605957,
      "activations/layer10_attention_weight_min": -39.595340728759766,
      "activations/layer11_attention_weight_max": 32.264862060546875,
      "activations/layer11_attention_weight_min": -33.95634460449219,
      "activations/layer12_attention_weight_max": 19.95643424987793,
      "activations/layer12_attention_weight_min": -24.26728630065918,
      "activations/layer13_attention_weight_max": 34.43671798706055,
      "activations/layer13_attention_weight_min": -30.27313995361328,
      "activations/layer14_attention_weight_max": 38.96760559082031,
      "activations/layer14_attention_weight_min": -30.560142517089844,
      "activations/layer15_attention_weight_max": 36.100284576416016,
      "activations/layer15_attention_weight_min": -29.39760398864746,
      "activations/layer16_attention_weight_max": 33.92030334472656,
      "activations/layer16_attention_weight_min": -30.585859298706055,
      "activations/layer17_attention_weight_max": 57.09464645385742,
      "activations/layer17_attention_weight_min": -44.3947639465332,
      "activations/layer18_attention_weight_max": 51.99956512451172,
      "activations/layer18_attention_weight_min": -40.12494659423828,
      "activations/layer19_attention_weight_max": 25.07924461364746,
      "activations/layer19_attention_weight_min": -25.924604415893555,
      "activations/layer1_attention_weight_max": 16.104957580566406,
      "activations/layer1_attention_weight_min": -13.883334159851074,
      "activations/layer20_attention_weight_max": 23.528949737548828,
      "activations/layer20_attention_weight_min": -22.177404403686523,
      "activations/layer21_attention_weight_max": 41.443931579589844,
      "activations/layer21_attention_weight_min": -24.13927459716797,
      "activations/layer22_attention_weight_max": 30.97713851928711,
      "activations/layer22_attention_weight_min": -25.554521560668945,
      "activations/layer23_attention_weight_max": 37.72962951660156,
      "activations/layer23_attention_weight_min": -24.697471618652344,
      "activations/layer2_attention_weight_max": 33.662803649902344,
      "activations/layer2_attention_weight_min": -31.138532638549805,
      "activations/layer3_attention_weight_max": 87.73685455322266,
      "activations/layer3_attention_weight_min": -92.00076293945312,
      "activations/layer4_attention_weight_max": 86.70586395263672,
      "activations/layer4_attention_weight_min": -90.89857482910156,
      "activations/layer5_attention_weight_max": 66.83297729492188,
      "activations/layer5_attention_weight_min": -73.06689453125,
      "activations/layer6_attention_weight_max": 48.724632263183594,
      "activations/layer6_attention_weight_min": -48.09809875488281,
      "activations/layer7_attention_weight_max": 63.921329498291016,
      "activations/layer7_attention_weight_min": -63.64427185058594,
      "activations/layer8_attention_weight_max": 45.857330322265625,
      "activations/layer8_attention_weight_min": -50.490936279296875,
      "activations/layer9_attention_weight_max": 46.8260612487793,
      "activations/layer9_attention_weight_min": -49.08378982543945,
      "epoch": 14.95,
      "learning_rate": 5.4133712121212115e-05,
      "loss": 2.7627,
      "step": 257300
    },
    {
      "activations/layer0_attention_weight_max": 16.304004669189453,
      "activations/layer0_attention_weight_min": -13.653970718383789,
      "activations/layer10_attention_weight_max": 36.125160217285156,
      "activations/layer10_attention_weight_min": -35.7706184387207,
      "activations/layer11_attention_weight_max": 32.263145446777344,
      "activations/layer11_attention_weight_min": -33.17825698852539,
      "activations/layer12_attention_weight_max": 20.54667854309082,
      "activations/layer12_attention_weight_min": -25.957124710083008,
      "activations/layer13_attention_weight_max": 35.239990234375,
      "activations/layer13_attention_weight_min": -28.829177856445312,
      "activations/layer14_attention_weight_max": 35.425743103027344,
      "activations/layer14_attention_weight_min": -31.663890838623047,
      "activations/layer15_attention_weight_max": 33.87739181518555,
      "activations/layer15_attention_weight_min": -30.8374080657959,
      "activations/layer16_attention_weight_max": 31.82017707824707,
      "activations/layer16_attention_weight_min": -31.7537841796875,
      "activations/layer17_attention_weight_max": 49.47145080566406,
      "activations/layer17_attention_weight_min": -48.77410888671875,
      "activations/layer18_attention_weight_max": 48.83864974975586,
      "activations/layer18_attention_weight_min": -40.14978790283203,
      "activations/layer19_attention_weight_max": 23.73468017578125,
      "activations/layer19_attention_weight_min": -23.938602447509766,
      "activations/layer1_attention_weight_max": 16.211034774780273,
      "activations/layer1_attention_weight_min": -13.835051536560059,
      "activations/layer20_attention_weight_max": 21.92967987060547,
      "activations/layer20_attention_weight_min": -25.585952758789062,
      "activations/layer21_attention_weight_max": 38.92934036254883,
      "activations/layer21_attention_weight_min": -22.00691795349121,
      "activations/layer22_attention_weight_max": 28.88861846923828,
      "activations/layer22_attention_weight_min": -27.14257049560547,
      "activations/layer23_attention_weight_max": 35.32514190673828,
      "activations/layer23_attention_weight_min": -24.65863037109375,
      "activations/layer2_attention_weight_max": 30.36440086364746,
      "activations/layer2_attention_weight_min": -30.56043243408203,
      "activations/layer3_attention_weight_max": 90.01273345947266,
      "activations/layer3_attention_weight_min": -92.29154968261719,
      "activations/layer4_attention_weight_max": 90.99198150634766,
      "activations/layer4_attention_weight_min": -87.77281951904297,
      "activations/layer5_attention_weight_max": 63.74778366088867,
      "activations/layer5_attention_weight_min": -71.39263153076172,
      "activations/layer6_attention_weight_max": 49.89862823486328,
      "activations/layer6_attention_weight_min": -50.38380813598633,
      "activations/layer7_attention_weight_max": 60.09058380126953,
      "activations/layer7_attention_weight_min": -60.730106353759766,
      "activations/layer8_attention_weight_max": 45.25041580200195,
      "activations/layer8_attention_weight_min": -46.78105545043945,
      "activations/layer9_attention_weight_max": 43.20923614501953,
      "activations/layer9_attention_weight_min": -44.179317474365234,
      "epoch": 14.95,
      "learning_rate": 5.411477272727272e-05,
      "loss": 2.7521,
      "step": 257350
    },
    {
      "activations/layer0_attention_weight_max": 16.305057525634766,
      "activations/layer0_attention_weight_min": -13.536711692810059,
      "activations/layer10_attention_weight_max": 37.19639205932617,
      "activations/layer10_attention_weight_min": -37.23334503173828,
      "activations/layer11_attention_weight_max": 32.99230194091797,
      "activations/layer11_attention_weight_min": -34.4440803527832,
      "activations/layer12_attention_weight_max": 19.271615982055664,
      "activations/layer12_attention_weight_min": -26.91089630126953,
      "activations/layer13_attention_weight_max": 38.928077697753906,
      "activations/layer13_attention_weight_min": -27.44519805908203,
      "activations/layer14_attention_weight_max": 38.723731994628906,
      "activations/layer14_attention_weight_min": -30.174407958984375,
      "activations/layer15_attention_weight_max": 34.58590316772461,
      "activations/layer15_attention_weight_min": -31.65072250366211,
      "activations/layer16_attention_weight_max": 35.75160598754883,
      "activations/layer16_attention_weight_min": -34.01992416381836,
      "activations/layer17_attention_weight_max": 56.37876892089844,
      "activations/layer17_attention_weight_min": -47.126007080078125,
      "activations/layer18_attention_weight_max": 48.07012176513672,
      "activations/layer18_attention_weight_min": -39.85706329345703,
      "activations/layer19_attention_weight_max": 23.87613868713379,
      "activations/layer19_attention_weight_min": -25.003684997558594,
      "activations/layer1_attention_weight_max": 15.55313777923584,
      "activations/layer1_attention_weight_min": -13.126300811767578,
      "activations/layer20_attention_weight_max": 25.579496383666992,
      "activations/layer20_attention_weight_min": -22.50553321838379,
      "activations/layer21_attention_weight_max": 35.53973388671875,
      "activations/layer21_attention_weight_min": -24.80575942993164,
      "activations/layer22_attention_weight_max": 30.80712127685547,
      "activations/layer22_attention_weight_min": -28.14978790283203,
      "activations/layer23_attention_weight_max": 41.11613845825195,
      "activations/layer23_attention_weight_min": -27.72473907470703,
      "activations/layer2_attention_weight_max": 31.027210235595703,
      "activations/layer2_attention_weight_min": -29.400402069091797,
      "activations/layer3_attention_weight_max": 85.94373321533203,
      "activations/layer3_attention_weight_min": -88.9706039428711,
      "activations/layer4_attention_weight_max": 89.21849822998047,
      "activations/layer4_attention_weight_min": -87.34612274169922,
      "activations/layer5_attention_weight_max": 68.00709533691406,
      "activations/layer5_attention_weight_min": -75.09390258789062,
      "activations/layer6_attention_weight_max": 50.49764633178711,
      "activations/layer6_attention_weight_min": -49.372867584228516,
      "activations/layer7_attention_weight_max": 65.8377456665039,
      "activations/layer7_attention_weight_min": -63.16551208496094,
      "activations/layer8_attention_weight_max": 45.48417282104492,
      "activations/layer8_attention_weight_min": -48.961055755615234,
      "activations/layer9_attention_weight_max": 44.573768615722656,
      "activations/layer9_attention_weight_min": -47.97860336303711,
      "epoch": 14.96,
      "learning_rate": 5.409583333333333e-05,
      "loss": 2.7519,
      "step": 257400
    },
    {
      "activations/layer0_attention_weight_max": 16.272092819213867,
      "activations/layer0_attention_weight_min": -13.611943244934082,
      "activations/layer10_attention_weight_max": 34.7605094909668,
      "activations/layer10_attention_weight_min": -34.383548736572266,
      "activations/layer11_attention_weight_max": 31.103178024291992,
      "activations/layer11_attention_weight_min": -34.3038215637207,
      "activations/layer12_attention_weight_max": 17.90852928161621,
      "activations/layer12_attention_weight_min": -22.192106246948242,
      "activations/layer13_attention_weight_max": 33.42662811279297,
      "activations/layer13_attention_weight_min": -28.610742568969727,
      "activations/layer14_attention_weight_max": 34.50330352783203,
      "activations/layer14_attention_weight_min": -30.466325759887695,
      "activations/layer15_attention_weight_max": 29.704397201538086,
      "activations/layer15_attention_weight_min": -30.468441009521484,
      "activations/layer16_attention_weight_max": 29.874128341674805,
      "activations/layer16_attention_weight_min": -31.324357986450195,
      "activations/layer17_attention_weight_max": 46.51974105834961,
      "activations/layer17_attention_weight_min": -46.70832443237305,
      "activations/layer18_attention_weight_max": 40.395835876464844,
      "activations/layer18_attention_weight_min": -39.212833404541016,
      "activations/layer19_attention_weight_max": 22.13617515563965,
      "activations/layer19_attention_weight_min": -23.786001205444336,
      "activations/layer1_attention_weight_max": 16.90703773498535,
      "activations/layer1_attention_weight_min": -15.088361740112305,
      "activations/layer20_attention_weight_max": 21.663667678833008,
      "activations/layer20_attention_weight_min": -22.2085018157959,
      "activations/layer21_attention_weight_max": 33.3858642578125,
      "activations/layer21_attention_weight_min": -20.32377052307129,
      "activations/layer22_attention_weight_max": 29.65884017944336,
      "activations/layer22_attention_weight_min": -27.830551147460938,
      "activations/layer23_attention_weight_max": 39.990909576416016,
      "activations/layer23_attention_weight_min": -22.81257438659668,
      "activations/layer2_attention_weight_max": 30.732086181640625,
      "activations/layer2_attention_weight_min": -30.00404930114746,
      "activations/layer3_attention_weight_max": 84.13819122314453,
      "activations/layer3_attention_weight_min": -83.59799194335938,
      "activations/layer4_attention_weight_max": 86.5294189453125,
      "activations/layer4_attention_weight_min": -85.09095001220703,
      "activations/layer5_attention_weight_max": 65.83956146240234,
      "activations/layer5_attention_weight_min": -74.58309173583984,
      "activations/layer6_attention_weight_max": 47.13408660888672,
      "activations/layer6_attention_weight_min": -50.821998596191406,
      "activations/layer7_attention_weight_max": 61.37923812866211,
      "activations/layer7_attention_weight_min": -62.96649169921875,
      "activations/layer8_attention_weight_max": 43.510719299316406,
      "activations/layer8_attention_weight_min": -47.5299072265625,
      "activations/layer9_attention_weight_max": 42.694053649902344,
      "activations/layer9_attention_weight_min": -43.664669036865234,
      "epoch": 14.96,
      "learning_rate": 5.407689393939393e-05,
      "loss": 2.7515,
      "step": 257450
    },
    {
      "activations/layer0_attention_weight_max": 16.155929565429688,
      "activations/layer0_attention_weight_min": -13.651745796203613,
      "activations/layer10_attention_weight_max": 37.816707611083984,
      "activations/layer10_attention_weight_min": -37.156314849853516,
      "activations/layer11_attention_weight_max": 33.60267639160156,
      "activations/layer11_attention_weight_min": -33.991783142089844,
      "activations/layer12_attention_weight_max": 19.222370147705078,
      "activations/layer12_attention_weight_min": -25.763551712036133,
      "activations/layer13_attention_weight_max": 42.10992431640625,
      "activations/layer13_attention_weight_min": -28.425861358642578,
      "activations/layer14_attention_weight_max": 38.92112731933594,
      "activations/layer14_attention_weight_min": -32.73771286010742,
      "activations/layer15_attention_weight_max": 34.36635208129883,
      "activations/layer15_attention_weight_min": -31.584394454956055,
      "activations/layer16_attention_weight_max": 34.67236328125,
      "activations/layer16_attention_weight_min": -33.18740463256836,
      "activations/layer17_attention_weight_max": 52.696468353271484,
      "activations/layer17_attention_weight_min": -50.82347869873047,
      "activations/layer18_attention_weight_max": 46.92354965209961,
      "activations/layer18_attention_weight_min": -44.24833297729492,
      "activations/layer19_attention_weight_max": 23.98853874206543,
      "activations/layer19_attention_weight_min": -24.673789978027344,
      "activations/layer1_attention_weight_max": 17.93075180053711,
      "activations/layer1_attention_weight_min": -14.920452117919922,
      "activations/layer20_attention_weight_max": 23.185482025146484,
      "activations/layer20_attention_weight_min": -23.56090545654297,
      "activations/layer21_attention_weight_max": 42.70430374145508,
      "activations/layer21_attention_weight_min": -22.193161010742188,
      "activations/layer22_attention_weight_max": 31.157880783081055,
      "activations/layer22_attention_weight_min": -26.399635314941406,
      "activations/layer23_attention_weight_max": 41.55915832519531,
      "activations/layer23_attention_weight_min": -26.6577205657959,
      "activations/layer2_attention_weight_max": 32.56562805175781,
      "activations/layer2_attention_weight_min": -29.747787475585938,
      "activations/layer3_attention_weight_max": 90.5312271118164,
      "activations/layer3_attention_weight_min": -90.85881805419922,
      "activations/layer4_attention_weight_max": 92.31987762451172,
      "activations/layer4_attention_weight_min": -89.31603240966797,
      "activations/layer5_attention_weight_max": 65.49462127685547,
      "activations/layer5_attention_weight_min": -74.74443054199219,
      "activations/layer6_attention_weight_max": 51.278018951416016,
      "activations/layer6_attention_weight_min": -51.64667892456055,
      "activations/layer7_attention_weight_max": 62.34446334838867,
      "activations/layer7_attention_weight_min": -65.76490020751953,
      "activations/layer8_attention_weight_max": 52.36101150512695,
      "activations/layer8_attention_weight_min": -49.532657623291016,
      "activations/layer9_attention_weight_max": 51.03411865234375,
      "activations/layer9_attention_weight_min": -47.570472717285156,
      "epoch": 14.96,
      "learning_rate": 5.4057954545454534e-05,
      "loss": 2.758,
      "step": 257500
    },
    {
      "activations/layer0_attention_weight_max": 17.399700164794922,
      "activations/layer0_attention_weight_min": -13.778429985046387,
      "activations/layer10_attention_weight_max": 40.72230911254883,
      "activations/layer10_attention_weight_min": -36.75037384033203,
      "activations/layer11_attention_weight_max": 33.75893783569336,
      "activations/layer11_attention_weight_min": -33.26596450805664,
      "activations/layer12_attention_weight_max": 18.256452560424805,
      "activations/layer12_attention_weight_min": -23.23755645751953,
      "activations/layer13_attention_weight_max": 35.76662826538086,
      "activations/layer13_attention_weight_min": -27.598939895629883,
      "activations/layer14_attention_weight_max": 35.12783432006836,
      "activations/layer14_attention_weight_min": -29.058473587036133,
      "activations/layer15_attention_weight_max": 31.492136001586914,
      "activations/layer15_attention_weight_min": -28.560680389404297,
      "activations/layer16_attention_weight_max": 31.298717498779297,
      "activations/layer16_attention_weight_min": -29.21649169921875,
      "activations/layer17_attention_weight_max": 47.64228057861328,
      "activations/layer17_attention_weight_min": -44.82707214355469,
      "activations/layer18_attention_weight_max": 42.96306610107422,
      "activations/layer18_attention_weight_min": -38.137298583984375,
      "activations/layer19_attention_weight_max": 23.402463912963867,
      "activations/layer19_attention_weight_min": -22.36713218688965,
      "activations/layer1_attention_weight_max": 16.53470230102539,
      "activations/layer1_attention_weight_min": -14.703593254089355,
      "activations/layer20_attention_weight_max": 21.140029907226562,
      "activations/layer20_attention_weight_min": -20.808259963989258,
      "activations/layer21_attention_weight_max": 37.50663375854492,
      "activations/layer21_attention_weight_min": -21.498367309570312,
      "activations/layer22_attention_weight_max": 30.205896377563477,
      "activations/layer22_attention_weight_min": -25.94574737548828,
      "activations/layer23_attention_weight_max": 36.793006896972656,
      "activations/layer23_attention_weight_min": -23.555797576904297,
      "activations/layer2_attention_weight_max": 31.255985260009766,
      "activations/layer2_attention_weight_min": -30.321996688842773,
      "activations/layer3_attention_weight_max": 91.33267974853516,
      "activations/layer3_attention_weight_min": -92.64898681640625,
      "activations/layer4_attention_weight_max": 93.36322784423828,
      "activations/layer4_attention_weight_min": -90.61388397216797,
      "activations/layer5_attention_weight_max": 65.87706756591797,
      "activations/layer5_attention_weight_min": -76.34245300292969,
      "activations/layer6_attention_weight_max": 52.78635787963867,
      "activations/layer6_attention_weight_min": -52.733421325683594,
      "activations/layer7_attention_weight_max": 64.13658905029297,
      "activations/layer7_attention_weight_min": -65.76493835449219,
      "activations/layer8_attention_weight_max": 47.53818893432617,
      "activations/layer8_attention_weight_min": -49.42723083496094,
      "activations/layer9_attention_weight_max": 46.804569244384766,
      "activations/layer9_attention_weight_min": -48.60686492919922,
      "epoch": 14.97,
      "learning_rate": 5.403901515151515e-05,
      "loss": 2.743,
      "step": 257550
    },
    {
      "activations/layer0_attention_weight_max": 16.12189292907715,
      "activations/layer0_attention_weight_min": -13.961982727050781,
      "activations/layer10_attention_weight_max": 36.49140548706055,
      "activations/layer10_attention_weight_min": -36.36249542236328,
      "activations/layer11_attention_weight_max": 31.970035552978516,
      "activations/layer11_attention_weight_min": -34.503021240234375,
      "activations/layer12_attention_weight_max": 19.4753475189209,
      "activations/layer12_attention_weight_min": -24.869157791137695,
      "activations/layer13_attention_weight_max": 33.44032669067383,
      "activations/layer13_attention_weight_min": -29.211793899536133,
      "activations/layer14_attention_weight_max": 35.01887893676758,
      "activations/layer14_attention_weight_min": -30.561599731445312,
      "activations/layer15_attention_weight_max": 32.184268951416016,
      "activations/layer15_attention_weight_min": -29.703479766845703,
      "activations/layer16_attention_weight_max": 31.91908836364746,
      "activations/layer16_attention_weight_min": -30.622425079345703,
      "activations/layer17_attention_weight_max": 49.85945510864258,
      "activations/layer17_attention_weight_min": -44.95610427856445,
      "activations/layer18_attention_weight_max": 44.08249282836914,
      "activations/layer18_attention_weight_min": -41.53020095825195,
      "activations/layer19_attention_weight_max": 22.573427200317383,
      "activations/layer19_attention_weight_min": -23.487703323364258,
      "activations/layer1_attention_weight_max": 15.177949905395508,
      "activations/layer1_attention_weight_min": -13.148613929748535,
      "activations/layer20_attention_weight_max": 22.26088523864746,
      "activations/layer20_attention_weight_min": -21.076452255249023,
      "activations/layer21_attention_weight_max": 38.547542572021484,
      "activations/layer21_attention_weight_min": -21.057065963745117,
      "activations/layer22_attention_weight_max": 32.47865676879883,
      "activations/layer22_attention_weight_min": -27.61945152282715,
      "activations/layer23_attention_weight_max": 34.25674819946289,
      "activations/layer23_attention_weight_min": -26.026222229003906,
      "activations/layer2_attention_weight_max": 30.26083755493164,
      "activations/layer2_attention_weight_min": -30.448837280273438,
      "activations/layer3_attention_weight_max": 90.13555145263672,
      "activations/layer3_attention_weight_min": -94.31598663330078,
      "activations/layer4_attention_weight_max": 91.52034759521484,
      "activations/layer4_attention_weight_min": -90.43314361572266,
      "activations/layer5_attention_weight_max": 63.956626892089844,
      "activations/layer5_attention_weight_min": -73.70210266113281,
      "activations/layer6_attention_weight_max": 47.69712448120117,
      "activations/layer6_attention_weight_min": -48.86267852783203,
      "activations/layer7_attention_weight_max": 63.01284408569336,
      "activations/layer7_attention_weight_min": -59.50987243652344,
      "activations/layer8_attention_weight_max": 46.21134567260742,
      "activations/layer8_attention_weight_min": -51.69094467163086,
      "activations/layer9_attention_weight_max": 44.65558624267578,
      "activations/layer9_attention_weight_min": -46.803401947021484,
      "epoch": 14.97,
      "learning_rate": 5.402007575757575e-05,
      "loss": 2.7656,
      "step": 257600
    },
    {
      "activations/layer0_attention_weight_max": 15.676470756530762,
      "activations/layer0_attention_weight_min": -14.053935050964355,
      "activations/layer10_attention_weight_max": 34.96733093261719,
      "activations/layer10_attention_weight_min": -33.962127685546875,
      "activations/layer11_attention_weight_max": 30.9492244720459,
      "activations/layer11_attention_weight_min": -32.858497619628906,
      "activations/layer12_attention_weight_max": 19.09415626525879,
      "activations/layer12_attention_weight_min": -26.447471618652344,
      "activations/layer13_attention_weight_max": 33.386558532714844,
      "activations/layer13_attention_weight_min": -26.30730628967285,
      "activations/layer14_attention_weight_max": 35.5136604309082,
      "activations/layer14_attention_weight_min": -30.469663619995117,
      "activations/layer15_attention_weight_max": 33.6949577331543,
      "activations/layer15_attention_weight_min": -29.50670051574707,
      "activations/layer16_attention_weight_max": 34.727088928222656,
      "activations/layer16_attention_weight_min": -31.801958084106445,
      "activations/layer17_attention_weight_max": 52.90010070800781,
      "activations/layer17_attention_weight_min": -46.77133560180664,
      "activations/layer18_attention_weight_max": 48.653934478759766,
      "activations/layer18_attention_weight_min": -41.876773834228516,
      "activations/layer19_attention_weight_max": 23.378494262695312,
      "activations/layer19_attention_weight_min": -22.759092330932617,
      "activations/layer1_attention_weight_max": 16.45686149597168,
      "activations/layer1_attention_weight_min": -14.916003227233887,
      "activations/layer20_attention_weight_max": 21.989879608154297,
      "activations/layer20_attention_weight_min": -21.41314125061035,
      "activations/layer21_attention_weight_max": 33.1953125,
      "activations/layer21_attention_weight_min": -22.44877052307129,
      "activations/layer22_attention_weight_max": 30.067794799804688,
      "activations/layer22_attention_weight_min": -27.908905029296875,
      "activations/layer23_attention_weight_max": 34.804283142089844,
      "activations/layer23_attention_weight_min": -23.76711082458496,
      "activations/layer2_attention_weight_max": 31.73108673095703,
      "activations/layer2_attention_weight_min": -31.4425106048584,
      "activations/layer3_attention_weight_max": 89.66178894042969,
      "activations/layer3_attention_weight_min": -86.4466323852539,
      "activations/layer4_attention_weight_max": 88.58916473388672,
      "activations/layer4_attention_weight_min": -85.56143188476562,
      "activations/layer5_attention_weight_max": 65.0138168334961,
      "activations/layer5_attention_weight_min": -70.55575561523438,
      "activations/layer6_attention_weight_max": 51.358619689941406,
      "activations/layer6_attention_weight_min": -48.45468521118164,
      "activations/layer7_attention_weight_max": 60.004024505615234,
      "activations/layer7_attention_weight_min": -57.1258659362793,
      "activations/layer8_attention_weight_max": 47.16094970703125,
      "activations/layer8_attention_weight_min": -46.680702209472656,
      "activations/layer9_attention_weight_max": 42.37482833862305,
      "activations/layer9_attention_weight_min": -44.20952224731445,
      "epoch": 14.97,
      "learning_rate": 5.400113636363636e-05,
      "loss": 2.76,
      "step": 257650
    },
    {
      "activations/layer0_attention_weight_max": 16.121015548706055,
      "activations/layer0_attention_weight_min": -13.456886291503906,
      "activations/layer10_attention_weight_max": 42.935585021972656,
      "activations/layer10_attention_weight_min": -40.822479248046875,
      "activations/layer11_attention_weight_max": 39.72931671142578,
      "activations/layer11_attention_weight_min": -37.921287536621094,
      "activations/layer12_attention_weight_max": 19.699838638305664,
      "activations/layer12_attention_weight_min": -25.08707046508789,
      "activations/layer13_attention_weight_max": 39.36937713623047,
      "activations/layer13_attention_weight_min": -31.528892517089844,
      "activations/layer14_attention_weight_max": 40.16676330566406,
      "activations/layer14_attention_weight_min": -31.51102066040039,
      "activations/layer15_attention_weight_max": 37.465641021728516,
      "activations/layer15_attention_weight_min": -31.057964324951172,
      "activations/layer16_attention_weight_max": 34.09653091430664,
      "activations/layer16_attention_weight_min": -32.25837326049805,
      "activations/layer17_attention_weight_max": 54.55420684814453,
      "activations/layer17_attention_weight_min": -44.38987731933594,
      "activations/layer18_attention_weight_max": 54.368011474609375,
      "activations/layer18_attention_weight_min": -41.85297393798828,
      "activations/layer19_attention_weight_max": 24.871740341186523,
      "activations/layer19_attention_weight_min": -23.295438766479492,
      "activations/layer1_attention_weight_max": 16.529876708984375,
      "activations/layer1_attention_weight_min": -14.159425735473633,
      "activations/layer20_attention_weight_max": 23.231670379638672,
      "activations/layer20_attention_weight_min": -21.228328704833984,
      "activations/layer21_attention_weight_max": 36.31889724731445,
      "activations/layer21_attention_weight_min": -21.824203491210938,
      "activations/layer22_attention_weight_max": 29.783588409423828,
      "activations/layer22_attention_weight_min": -25.655887603759766,
      "activations/layer23_attention_weight_max": 37.904605865478516,
      "activations/layer23_attention_weight_min": -24.15801239013672,
      "activations/layer2_attention_weight_max": 33.0283203125,
      "activations/layer2_attention_weight_min": -31.093814849853516,
      "activations/layer3_attention_weight_max": 95.68714141845703,
      "activations/layer3_attention_weight_min": -94.4468002319336,
      "activations/layer4_attention_weight_max": 100.8160171508789,
      "activations/layer4_attention_weight_min": -96.03147888183594,
      "activations/layer5_attention_weight_max": 70.00454711914062,
      "activations/layer5_attention_weight_min": -78.40719604492188,
      "activations/layer6_attention_weight_max": 54.80766677856445,
      "activations/layer6_attention_weight_min": -56.408935546875,
      "activations/layer7_attention_weight_max": 67.29229736328125,
      "activations/layer7_attention_weight_min": -68.82949829101562,
      "activations/layer8_attention_weight_max": 53.11357116699219,
      "activations/layer8_attention_weight_min": -52.365962982177734,
      "activations/layer9_attention_weight_max": 52.21034622192383,
      "activations/layer9_attention_weight_min": -49.49443435668945,
      "epoch": 14.97,
      "learning_rate": 5.398219696969696e-05,
      "loss": 2.7553,
      "step": 257700
    },
    {
      "activations/layer0_attention_weight_max": 16.48971176147461,
      "activations/layer0_attention_weight_min": -13.402640342712402,
      "activations/layer10_attention_weight_max": 35.01790237426758,
      "activations/layer10_attention_weight_min": -35.87864303588867,
      "activations/layer11_attention_weight_max": 32.472007751464844,
      "activations/layer11_attention_weight_min": -34.071563720703125,
      "activations/layer12_attention_weight_max": 18.3673095703125,
      "activations/layer12_attention_weight_min": -23.31680679321289,
      "activations/layer13_attention_weight_max": 33.3480339050293,
      "activations/layer13_attention_weight_min": -26.488256454467773,
      "activations/layer14_attention_weight_max": 36.25396728515625,
      "activations/layer14_attention_weight_min": -33.3759651184082,
      "activations/layer15_attention_weight_max": 34.62712478637695,
      "activations/layer15_attention_weight_min": -31.690229415893555,
      "activations/layer16_attention_weight_max": 34.43431091308594,
      "activations/layer16_attention_weight_min": -31.644725799560547,
      "activations/layer17_attention_weight_max": 51.045013427734375,
      "activations/layer17_attention_weight_min": -47.07864761352539,
      "activations/layer18_attention_weight_max": 46.18517303466797,
      "activations/layer18_attention_weight_min": -39.637481689453125,
      "activations/layer19_attention_weight_max": 22.59019660949707,
      "activations/layer19_attention_weight_min": -24.92609214782715,
      "activations/layer1_attention_weight_max": 16.889345169067383,
      "activations/layer1_attention_weight_min": -15.945144653320312,
      "activations/layer20_attention_weight_max": 22.860576629638672,
      "activations/layer20_attention_weight_min": -23.905614852905273,
      "activations/layer21_attention_weight_max": 35.5189094543457,
      "activations/layer21_attention_weight_min": -25.32699966430664,
      "activations/layer22_attention_weight_max": 29.84592628479004,
      "activations/layer22_attention_weight_min": -27.626312255859375,
      "activations/layer23_attention_weight_max": 38.73250961303711,
      "activations/layer23_attention_weight_min": -23.760089874267578,
      "activations/layer2_attention_weight_max": 31.45285415649414,
      "activations/layer2_attention_weight_min": -31.959007263183594,
      "activations/layer3_attention_weight_max": 90.5559310913086,
      "activations/layer3_attention_weight_min": -92.7396240234375,
      "activations/layer4_attention_weight_max": 90.26380920410156,
      "activations/layer4_attention_weight_min": -90.06996154785156,
      "activations/layer5_attention_weight_max": 67.38525390625,
      "activations/layer5_attention_weight_min": -82.12416076660156,
      "activations/layer6_attention_weight_max": 51.401710510253906,
      "activations/layer6_attention_weight_min": -50.47985076904297,
      "activations/layer7_attention_weight_max": 63.5914192199707,
      "activations/layer7_attention_weight_min": -66.60391235351562,
      "activations/layer8_attention_weight_max": 46.257347106933594,
      "activations/layer8_attention_weight_min": -49.44777297973633,
      "activations/layer9_attention_weight_max": 47.99423599243164,
      "activations/layer9_attention_weight_min": -44.49860763549805,
      "epoch": 14.98,
      "learning_rate": 5.3963257575757576e-05,
      "loss": 2.755,
      "step": 257750
    },
    {
      "activations/layer0_attention_weight_max": 15.912993431091309,
      "activations/layer0_attention_weight_min": -13.79155445098877,
      "activations/layer10_attention_weight_max": 38.69711685180664,
      "activations/layer10_attention_weight_min": -38.28694534301758,
      "activations/layer11_attention_weight_max": 33.8885612487793,
      "activations/layer11_attention_weight_min": -34.82743453979492,
      "activations/layer12_attention_weight_max": 20.22515296936035,
      "activations/layer12_attention_weight_min": -22.816354751586914,
      "activations/layer13_attention_weight_max": 39.46757888793945,
      "activations/layer13_attention_weight_min": -28.629526138305664,
      "activations/layer14_attention_weight_max": 37.67656707763672,
      "activations/layer14_attention_weight_min": -31.402620315551758,
      "activations/layer15_attention_weight_max": 36.268150329589844,
      "activations/layer15_attention_weight_min": -30.536680221557617,
      "activations/layer16_attention_weight_max": 33.681884765625,
      "activations/layer16_attention_weight_min": -31.801280975341797,
      "activations/layer17_attention_weight_max": 53.70397186279297,
      "activations/layer17_attention_weight_min": -45.980655670166016,
      "activations/layer18_attention_weight_max": 54.42045593261719,
      "activations/layer18_attention_weight_min": -39.84922790527344,
      "activations/layer19_attention_weight_max": 24.29213523864746,
      "activations/layer19_attention_weight_min": -22.382381439208984,
      "activations/layer1_attention_weight_max": 15.512431144714355,
      "activations/layer1_attention_weight_min": -13.958916664123535,
      "activations/layer20_attention_weight_max": 25.202163696289062,
      "activations/layer20_attention_weight_min": -21.241867065429688,
      "activations/layer21_attention_weight_max": 45.1219596862793,
      "activations/layer21_attention_weight_min": -26.075063705444336,
      "activations/layer22_attention_weight_max": 30.74653434753418,
      "activations/layer22_attention_weight_min": -26.645586013793945,
      "activations/layer23_attention_weight_max": 40.75019836425781,
      "activations/layer23_attention_weight_min": -25.28274917602539,
      "activations/layer2_attention_weight_max": 32.528968811035156,
      "activations/layer2_attention_weight_min": -31.947622299194336,
      "activations/layer3_attention_weight_max": 92.18646240234375,
      "activations/layer3_attention_weight_min": -95.68852233886719,
      "activations/layer4_attention_weight_max": 93.43441009521484,
      "activations/layer4_attention_weight_min": -94.11727905273438,
      "activations/layer5_attention_weight_max": 66.95060729980469,
      "activations/layer5_attention_weight_min": -74.70784759521484,
      "activations/layer6_attention_weight_max": 51.25702667236328,
      "activations/layer6_attention_weight_min": -50.62466049194336,
      "activations/layer7_attention_weight_max": 66.05567169189453,
      "activations/layer7_attention_weight_min": -64.31146240234375,
      "activations/layer8_attention_weight_max": 47.572940826416016,
      "activations/layer8_attention_weight_min": -49.570701599121094,
      "activations/layer9_attention_weight_max": 45.202945709228516,
      "activations/layer9_attention_weight_min": -47.791011810302734,
      "epoch": 14.98,
      "learning_rate": 5.394431818181818e-05,
      "loss": 2.7685,
      "step": 257800
    },
    {
      "activations/layer0_attention_weight_max": 16.3129825592041,
      "activations/layer0_attention_weight_min": -13.535628318786621,
      "activations/layer10_attention_weight_max": 38.41241455078125,
      "activations/layer10_attention_weight_min": -38.648193359375,
      "activations/layer11_attention_weight_max": 36.39698028564453,
      "activations/layer11_attention_weight_min": -38.32777404785156,
      "activations/layer12_attention_weight_max": 19.19135856628418,
      "activations/layer12_attention_weight_min": -29.972143173217773,
      "activations/layer13_attention_weight_max": 41.993995666503906,
      "activations/layer13_attention_weight_min": -32.971920013427734,
      "activations/layer14_attention_weight_max": 41.958404541015625,
      "activations/layer14_attention_weight_min": -36.573577880859375,
      "activations/layer15_attention_weight_max": 39.15291213989258,
      "activations/layer15_attention_weight_min": -35.88045120239258,
      "activations/layer16_attention_weight_max": 38.69695281982422,
      "activations/layer16_attention_weight_min": -36.82488250732422,
      "activations/layer17_attention_weight_max": 60.76239776611328,
      "activations/layer17_attention_weight_min": -52.723087310791016,
      "activations/layer18_attention_weight_max": 54.62070083618164,
      "activations/layer18_attention_weight_min": -44.27383804321289,
      "activations/layer19_attention_weight_max": 27.799636840820312,
      "activations/layer19_attention_weight_min": -24.71388053894043,
      "activations/layer1_attention_weight_max": 16.507923126220703,
      "activations/layer1_attention_weight_min": -14.87867546081543,
      "activations/layer20_attention_weight_max": 31.0643367767334,
      "activations/layer20_attention_weight_min": -22.8580265045166,
      "activations/layer21_attention_weight_max": 50.4025764465332,
      "activations/layer21_attention_weight_min": -24.66701316833496,
      "activations/layer22_attention_weight_max": 37.65718460083008,
      "activations/layer22_attention_weight_min": -26.731714248657227,
      "activations/layer23_attention_weight_max": 43.94710922241211,
      "activations/layer23_attention_weight_min": -26.582271575927734,
      "activations/layer2_attention_weight_max": 32.714622497558594,
      "activations/layer2_attention_weight_min": -30.707138061523438,
      "activations/layer3_attention_weight_max": 89.40946197509766,
      "activations/layer3_attention_weight_min": -93.05465698242188,
      "activations/layer4_attention_weight_max": 90.20256042480469,
      "activations/layer4_attention_weight_min": -88.5626220703125,
      "activations/layer5_attention_weight_max": 67.94599914550781,
      "activations/layer5_attention_weight_min": -74.48579406738281,
      "activations/layer6_attention_weight_max": 50.70802307128906,
      "activations/layer6_attention_weight_min": -52.45791244506836,
      "activations/layer7_attention_weight_max": 63.93532180786133,
      "activations/layer7_attention_weight_min": -64.35881805419922,
      "activations/layer8_attention_weight_max": 46.70524215698242,
      "activations/layer8_attention_weight_min": -51.07988357543945,
      "activations/layer9_attention_weight_max": 51.046566009521484,
      "activations/layer9_attention_weight_min": -46.196834564208984,
      "epoch": 14.98,
      "learning_rate": 5.392537878787878e-05,
      "loss": 2.761,
      "step": 257850
    },
    {
      "activations/layer0_attention_weight_max": 15.858664512634277,
      "activations/layer0_attention_weight_min": -13.887528419494629,
      "activations/layer10_attention_weight_max": 38.22807693481445,
      "activations/layer10_attention_weight_min": -37.66475296020508,
      "activations/layer11_attention_weight_max": 33.93740463256836,
      "activations/layer11_attention_weight_min": -36.34070587158203,
      "activations/layer12_attention_weight_max": 20.32220458984375,
      "activations/layer12_attention_weight_min": -22.91550064086914,
      "activations/layer13_attention_weight_max": 43.24300003051758,
      "activations/layer13_attention_weight_min": -29.644092559814453,
      "activations/layer14_attention_weight_max": 47.24601745605469,
      "activations/layer14_attention_weight_min": -30.51299285888672,
      "activations/layer15_attention_weight_max": 39.91851806640625,
      "activations/layer15_attention_weight_min": -30.17586898803711,
      "activations/layer16_attention_weight_max": 35.592315673828125,
      "activations/layer16_attention_weight_min": -32.41368865966797,
      "activations/layer17_attention_weight_max": 58.03590774536133,
      "activations/layer17_attention_weight_min": -46.78716278076172,
      "activations/layer18_attention_weight_max": 51.514854431152344,
      "activations/layer18_attention_weight_min": -41.631656646728516,
      "activations/layer19_attention_weight_max": 28.114246368408203,
      "activations/layer19_attention_weight_min": -24.34061050415039,
      "activations/layer1_attention_weight_max": 15.535859107971191,
      "activations/layer1_attention_weight_min": -13.570345878601074,
      "activations/layer20_attention_weight_max": 25.801462173461914,
      "activations/layer20_attention_weight_min": -25.485427856445312,
      "activations/layer21_attention_weight_max": 39.04632568359375,
      "activations/layer21_attention_weight_min": -22.7213191986084,
      "activations/layer22_attention_weight_max": 31.553512573242188,
      "activations/layer22_attention_weight_min": -27.933746337890625,
      "activations/layer23_attention_weight_max": 37.62921905517578,
      "activations/layer23_attention_weight_min": -23.945798873901367,
      "activations/layer2_attention_weight_max": 32.17439270019531,
      "activations/layer2_attention_weight_min": -30.801496505737305,
      "activations/layer3_attention_weight_max": 97.33783721923828,
      "activations/layer3_attention_weight_min": -96.42491912841797,
      "activations/layer4_attention_weight_max": 95.04029083251953,
      "activations/layer4_attention_weight_min": -92.0178451538086,
      "activations/layer5_attention_weight_max": 69.06205749511719,
      "activations/layer5_attention_weight_min": -74.49056243896484,
      "activations/layer6_attention_weight_max": 52.345054626464844,
      "activations/layer6_attention_weight_min": -52.383121490478516,
      "activations/layer7_attention_weight_max": 67.34252166748047,
      "activations/layer7_attention_weight_min": -66.7547378540039,
      "activations/layer8_attention_weight_max": 47.68144607543945,
      "activations/layer8_attention_weight_min": -48.761775970458984,
      "activations/layer9_attention_weight_max": 45.11553192138672,
      "activations/layer9_attention_weight_min": -47.67683410644531,
      "epoch": 14.99,
      "learning_rate": 5.390643939393939e-05,
      "loss": 2.7566,
      "step": 257900
    },
    {
      "activations/layer0_attention_weight_max": 17.14218521118164,
      "activations/layer0_attention_weight_min": -13.54531478881836,
      "activations/layer10_attention_weight_max": 40.5423698425293,
      "activations/layer10_attention_weight_min": -40.54278564453125,
      "activations/layer11_attention_weight_max": 34.953678131103516,
      "activations/layer11_attention_weight_min": -35.918174743652344,
      "activations/layer12_attention_weight_max": 19.80733299255371,
      "activations/layer12_attention_weight_min": -26.02540397644043,
      "activations/layer13_attention_weight_max": 33.84183120727539,
      "activations/layer13_attention_weight_min": -28.20536994934082,
      "activations/layer14_attention_weight_max": 36.078304290771484,
      "activations/layer14_attention_weight_min": -30.333515167236328,
      "activations/layer15_attention_weight_max": 31.57618522644043,
      "activations/layer15_attention_weight_min": -30.868120193481445,
      "activations/layer16_attention_weight_max": 32.50931167602539,
      "activations/layer16_attention_weight_min": -31.14689064025879,
      "activations/layer17_attention_weight_max": 50.93107223510742,
      "activations/layer17_attention_weight_min": -43.54854202270508,
      "activations/layer18_attention_weight_max": 47.33833312988281,
      "activations/layer18_attention_weight_min": -38.6597900390625,
      "activations/layer19_attention_weight_max": 22.782161712646484,
      "activations/layer19_attention_weight_min": -23.258930206298828,
      "activations/layer1_attention_weight_max": 16.55368995666504,
      "activations/layer1_attention_weight_min": -15.645393371582031,
      "activations/layer20_attention_weight_max": 22.33716583251953,
      "activations/layer20_attention_weight_min": -23.182676315307617,
      "activations/layer21_attention_weight_max": 38.77846908569336,
      "activations/layer21_attention_weight_min": -27.403879165649414,
      "activations/layer22_attention_weight_max": 30.180328369140625,
      "activations/layer22_attention_weight_min": -26.01382827758789,
      "activations/layer23_attention_weight_max": 33.90650939941406,
      "activations/layer23_attention_weight_min": -25.50881576538086,
      "activations/layer2_attention_weight_max": 33.67741775512695,
      "activations/layer2_attention_weight_min": -33.06404113769531,
      "activations/layer3_attention_weight_max": 98.83707427978516,
      "activations/layer3_attention_weight_min": -102.32003021240234,
      "activations/layer4_attention_weight_max": 99.48363494873047,
      "activations/layer4_attention_weight_min": -91.40377807617188,
      "activations/layer5_attention_weight_max": 66.57315063476562,
      "activations/layer5_attention_weight_min": -75.6572265625,
      "activations/layer6_attention_weight_max": 52.8339958190918,
      "activations/layer6_attention_weight_min": -49.75672149658203,
      "activations/layer7_attention_weight_max": 64.25152587890625,
      "activations/layer7_attention_weight_min": -62.34163284301758,
      "activations/layer8_attention_weight_max": 47.01408767700195,
      "activations/layer8_attention_weight_min": -48.33216857910156,
      "activations/layer9_attention_weight_max": 48.56740951538086,
      "activations/layer9_attention_weight_min": -45.16095733642578,
      "epoch": 14.99,
      "learning_rate": 5.3887499999999995e-05,
      "loss": 2.7634,
      "step": 257950
    },
    {
      "activations/layer0_attention_weight_max": 15.119176864624023,
      "activations/layer0_attention_weight_min": -13.257545471191406,
      "activations/layer10_attention_weight_max": 36.26789093017578,
      "activations/layer10_attention_weight_min": -34.66130065917969,
      "activations/layer11_attention_weight_max": 32.9763298034668,
      "activations/layer11_attention_weight_min": -32.38581848144531,
      "activations/layer12_attention_weight_max": 17.24103546142578,
      "activations/layer12_attention_weight_min": -24.547359466552734,
      "activations/layer13_attention_weight_max": 32.90920639038086,
      "activations/layer13_attention_weight_min": -28.810319900512695,
      "activations/layer14_attention_weight_max": 33.48374557495117,
      "activations/layer14_attention_weight_min": -30.02362823486328,
      "activations/layer15_attention_weight_max": 30.403818130493164,
      "activations/layer15_attention_weight_min": -29.654863357543945,
      "activations/layer16_attention_weight_max": 30.466405868530273,
      "activations/layer16_attention_weight_min": -29.369285583496094,
      "activations/layer17_attention_weight_max": 47.43586730957031,
      "activations/layer17_attention_weight_min": -44.167049407958984,
      "activations/layer18_attention_weight_max": 42.695472717285156,
      "activations/layer18_attention_weight_min": -38.17424011230469,
      "activations/layer19_attention_weight_max": 21.01325035095215,
      "activations/layer19_attention_weight_min": -23.38058853149414,
      "activations/layer1_attention_weight_max": 16.095176696777344,
      "activations/layer1_attention_weight_min": -15.352920532226562,
      "activations/layer20_attention_weight_max": 21.031042098999023,
      "activations/layer20_attention_weight_min": -21.979597091674805,
      "activations/layer21_attention_weight_max": 35.415428161621094,
      "activations/layer21_attention_weight_min": -24.228391647338867,
      "activations/layer22_attention_weight_max": 31.342350006103516,
      "activations/layer22_attention_weight_min": -28.144699096679688,
      "activations/layer23_attention_weight_max": 35.956424713134766,
      "activations/layer23_attention_weight_min": -23.390201568603516,
      "activations/layer2_attention_weight_max": 31.153518676757812,
      "activations/layer2_attention_weight_min": -30.727590560913086,
      "activations/layer3_attention_weight_max": 89.15765380859375,
      "activations/layer3_attention_weight_min": -97.35147094726562,
      "activations/layer4_attention_weight_max": 90.2230224609375,
      "activations/layer4_attention_weight_min": -88.65209197998047,
      "activations/layer5_attention_weight_max": 69.19298553466797,
      "activations/layer5_attention_weight_min": -74.41807556152344,
      "activations/layer6_attention_weight_max": 48.77132797241211,
      "activations/layer6_attention_weight_min": -50.813392639160156,
      "activations/layer7_attention_weight_max": 61.83639907836914,
      "activations/layer7_attention_weight_min": -60.26864242553711,
      "activations/layer8_attention_weight_max": 47.155914306640625,
      "activations/layer8_attention_weight_min": -48.30723571777344,
      "activations/layer9_attention_weight_max": 45.693565368652344,
      "activations/layer9_attention_weight_min": -45.51896286010742,
      "epoch": 14.99,
      "learning_rate": 5.3868560606060604e-05,
      "loss": 2.7524,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.461,
      "eval_samples_per_second": 507.506,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.461,
      "eval_openwebtext_samples_per_second": 507.506,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_wikitext_loss": 2.939453125,
      "eval_wikitext_ppl": 18.90550453688997,
      "eval_wikitext_runtime": 1.9415,
      "eval_wikitext_samples_per_second": 234.87,
      "step": 258000
    },
    {
      "epoch": 14.99,
      "eval_lambada_loss": 2.521484375,
      "eval_lambada_ppl": 12.447059060920614,
      "eval_lambada_runtime": 9.5379,
      "eval_lambada_samples_per_second": 510.492,
      "step": 258000
    },
    {
      "activations/layer0_attention_weight_max": 15.261981010437012,
      "activations/layer0_attention_weight_min": -13.830324172973633,
      "activations/layer10_attention_weight_max": 38.62849807739258,
      "activations/layer10_attention_weight_min": -41.05797576904297,
      "activations/layer11_attention_weight_max": 36.112918853759766,
      "activations/layer11_attention_weight_min": -37.04633331298828,
      "activations/layer12_attention_weight_max": 18.603260040283203,
      "activations/layer12_attention_weight_min": -24.944019317626953,
      "activations/layer13_attention_weight_max": 36.731292724609375,
      "activations/layer13_attention_weight_min": -29.03522300720215,
      "activations/layer14_attention_weight_max": 35.80313491821289,
      "activations/layer14_attention_weight_min": -34.893821716308594,
      "activations/layer15_attention_weight_max": 35.0295295715332,
      "activations/layer15_attention_weight_min": -33.30305099487305,
      "activations/layer16_attention_weight_max": 35.81778335571289,
      "activations/layer16_attention_weight_min": -35.311927795410156,
      "activations/layer17_attention_weight_max": 56.06447219848633,
      "activations/layer17_attention_weight_min": -48.10399627685547,
      "activations/layer18_attention_weight_max": 49.81039810180664,
      "activations/layer18_attention_weight_min": -42.241703033447266,
      "activations/layer19_attention_weight_max": 25.221668243408203,
      "activations/layer19_attention_weight_min": -22.93014144897461,
      "activations/layer1_attention_weight_max": 16.316377639770508,
      "activations/layer1_attention_weight_min": -14.449991226196289,
      "activations/layer20_attention_weight_max": 24.295879364013672,
      "activations/layer20_attention_weight_min": -23.03885269165039,
      "activations/layer21_attention_weight_max": 40.93698501586914,
      "activations/layer21_attention_weight_min": -25.68767738342285,
      "activations/layer22_attention_weight_max": 29.7835693359375,
      "activations/layer22_attention_weight_min": -28.726715087890625,
      "activations/layer23_attention_weight_max": 37.9636116027832,
      "activations/layer23_attention_weight_min": -23.318944931030273,
      "activations/layer2_attention_weight_max": 31.939315795898438,
      "activations/layer2_attention_weight_min": -31.018911361694336,
      "activations/layer3_attention_weight_max": 92.46170043945312,
      "activations/layer3_attention_weight_min": -91.35848999023438,
      "activations/layer4_attention_weight_max": 92.67253875732422,
      "activations/layer4_attention_weight_min": -90.57167053222656,
      "activations/layer5_attention_weight_max": 70.26324462890625,
      "activations/layer5_attention_weight_min": -79.094970703125,
      "activations/layer6_attention_weight_max": 52.84668731689453,
      "activations/layer6_attention_weight_min": -51.38620376586914,
      "activations/layer7_attention_weight_max": 67.33446502685547,
      "activations/layer7_attention_weight_min": -65.20883178710938,
      "activations/layer8_attention_weight_max": 48.36315155029297,
      "activations/layer8_attention_weight_min": -50.71037292480469,
      "activations/layer9_attention_weight_max": 47.71561813354492,
      "activations/layer9_attention_weight_min": -48.998939514160156,
      "epoch": 14.99,
      "learning_rate": 5.3849621212121205e-05,
      "loss": 2.7559,
      "step": 258050
    },
    {
      "activations/layer0_attention_weight_max": 16.484527587890625,
      "activations/layer0_attention_weight_min": -13.413324356079102,
      "activations/layer10_attention_weight_max": 38.30131912231445,
      "activations/layer10_attention_weight_min": -36.35763168334961,
      "activations/layer11_attention_weight_max": 36.30467224121094,
      "activations/layer11_attention_weight_min": -33.598480224609375,
      "activations/layer12_attention_weight_max": 18.093889236450195,
      "activations/layer12_attention_weight_min": -24.94962501525879,
      "activations/layer13_attention_weight_max": 34.25811004638672,
      "activations/layer13_attention_weight_min": -27.66180419921875,
      "activations/layer14_attention_weight_max": 38.540653228759766,
      "activations/layer14_attention_weight_min": -30.745765686035156,
      "activations/layer15_attention_weight_max": 35.50883483886719,
      "activations/layer15_attention_weight_min": -31.587032318115234,
      "activations/layer16_attention_weight_max": 35.18108367919922,
      "activations/layer16_attention_weight_min": -32.37977600097656,
      "activations/layer17_attention_weight_max": 54.21468734741211,
      "activations/layer17_attention_weight_min": -46.77067184448242,
      "activations/layer18_attention_weight_max": 49.0545654296875,
      "activations/layer18_attention_weight_min": -41.51840591430664,
      "activations/layer19_attention_weight_max": 22.676990509033203,
      "activations/layer19_attention_weight_min": -23.91956901550293,
      "activations/layer1_attention_weight_max": 17.294355392456055,
      "activations/layer1_attention_weight_min": -16.454683303833008,
      "activations/layer20_attention_weight_max": 22.518714904785156,
      "activations/layer20_attention_weight_min": -22.94537925720215,
      "activations/layer21_attention_weight_max": 38.29918670654297,
      "activations/layer21_attention_weight_min": -22.454360961914062,
      "activations/layer22_attention_weight_max": 31.91061782836914,
      "activations/layer22_attention_weight_min": -27.80426788330078,
      "activations/layer23_attention_weight_max": 36.038204193115234,
      "activations/layer23_attention_weight_min": -23.101078033447266,
      "activations/layer2_attention_weight_max": 33.51932907104492,
      "activations/layer2_attention_weight_min": -31.07333755493164,
      "activations/layer3_attention_weight_max": 96.25801086425781,
      "activations/layer3_attention_weight_min": -97.34088897705078,
      "activations/layer4_attention_weight_max": 95.54978942871094,
      "activations/layer4_attention_weight_min": -88.57186126708984,
      "activations/layer5_attention_weight_max": 67.77167510986328,
      "activations/layer5_attention_weight_min": -72.4568099975586,
      "activations/layer6_attention_weight_max": 51.331233978271484,
      "activations/layer6_attention_weight_min": -49.11189651489258,
      "activations/layer7_attention_weight_max": 66.88542938232422,
      "activations/layer7_attention_weight_min": -63.38566589355469,
      "activations/layer8_attention_weight_max": 48.532630920410156,
      "activations/layer8_attention_weight_min": -47.4289665222168,
      "activations/layer9_attention_weight_max": 51.486812591552734,
      "activations/layer9_attention_weight_min": -46.67875289916992,
      "epoch": 15.0,
      "learning_rate": 5.383068181818182e-05,
      "loss": 2.7629,
      "step": 258100
    },
    {
      "activations/layer0_attention_weight_max": 14.980682373046875,
      "activations/layer0_attention_weight_min": -13.797350883483887,
      "activations/layer10_attention_weight_max": 36.49762725830078,
      "activations/layer10_attention_weight_min": -34.695220947265625,
      "activations/layer11_attention_weight_max": 33.74017333984375,
      "activations/layer11_attention_weight_min": -33.01530456542969,
      "activations/layer12_attention_weight_max": 17.619579315185547,
      "activations/layer12_attention_weight_min": -28.111835479736328,
      "activations/layer13_attention_weight_max": 37.41627883911133,
      "activations/layer13_attention_weight_min": -29.434314727783203,
      "activations/layer14_attention_weight_max": 34.96626663208008,
      "activations/layer14_attention_weight_min": -30.17449378967285,
      "activations/layer15_attention_weight_max": 31.565603256225586,
      "activations/layer15_attention_weight_min": -28.04141616821289,
      "activations/layer16_attention_weight_max": 30.56303596496582,
      "activations/layer16_attention_weight_min": -31.27079963684082,
      "activations/layer17_attention_weight_max": 49.030574798583984,
      "activations/layer17_attention_weight_min": -45.50020217895508,
      "activations/layer18_attention_weight_max": 41.94346237182617,
      "activations/layer18_attention_weight_min": -37.619049072265625,
      "activations/layer19_attention_weight_max": 21.45931053161621,
      "activations/layer19_attention_weight_min": -21.2144832611084,
      "activations/layer1_attention_weight_max": 16.191682815551758,
      "activations/layer1_attention_weight_min": -16.098247528076172,
      "activations/layer20_attention_weight_max": 23.444561004638672,
      "activations/layer20_attention_weight_min": -22.079627990722656,
      "activations/layer21_attention_weight_max": 39.49163818359375,
      "activations/layer21_attention_weight_min": -21.887327194213867,
      "activations/layer22_attention_weight_max": 30.152557373046875,
      "activations/layer22_attention_weight_min": -25.236949920654297,
      "activations/layer23_attention_weight_max": 38.19365692138672,
      "activations/layer23_attention_weight_min": -26.358015060424805,
      "activations/layer2_attention_weight_max": 32.56460952758789,
      "activations/layer2_attention_weight_min": -31.17437744140625,
      "activations/layer3_attention_weight_max": 91.1575698852539,
      "activations/layer3_attention_weight_min": -95.23771667480469,
      "activations/layer4_attention_weight_max": 90.83739471435547,
      "activations/layer4_attention_weight_min": -89.50502014160156,
      "activations/layer5_attention_weight_max": 68.47528076171875,
      "activations/layer5_attention_weight_min": -78.4249038696289,
      "activations/layer6_attention_weight_max": 47.54315948486328,
      "activations/layer6_attention_weight_min": -49.01339340209961,
      "activations/layer7_attention_weight_max": 64.22972106933594,
      "activations/layer7_attention_weight_min": -62.44662094116211,
      "activations/layer8_attention_weight_max": 48.00627517700195,
      "activations/layer8_attention_weight_min": -46.2560920715332,
      "activations/layer9_attention_weight_max": 45.64830017089844,
      "activations/layer9_attention_weight_min": -44.00371551513672,
      "epoch": 15.0,
      "learning_rate": 5.381174242424242e-05,
      "loss": 2.7437,
      "step": 258150
    },
    {
      "activations/layer0_attention_weight_max": 16.41356658935547,
      "activations/layer0_attention_weight_min": -13.583992004394531,
      "activations/layer10_attention_weight_max": 34.36759567260742,
      "activations/layer10_attention_weight_min": -35.11982345581055,
      "activations/layer11_attention_weight_max": 31.947216033935547,
      "activations/layer11_attention_weight_min": -34.52173614501953,
      "activations/layer12_attention_weight_max": 18.07188606262207,
      "activations/layer12_attention_weight_min": -22.193708419799805,
      "activations/layer13_attention_weight_max": 37.03117752075195,
      "activations/layer13_attention_weight_min": -26.026777267456055,
      "activations/layer14_attention_weight_max": 34.104225158691406,
      "activations/layer14_attention_weight_min": -30.108688354492188,
      "activations/layer15_attention_weight_max": 33.581119537353516,
      "activations/layer15_attention_weight_min": -30.6081600189209,
      "activations/layer16_attention_weight_max": 36.5206413269043,
      "activations/layer16_attention_weight_min": -32.159053802490234,
      "activations/layer17_attention_weight_max": 57.02737808227539,
      "activations/layer17_attention_weight_min": -49.163028717041016,
      "activations/layer18_attention_weight_max": 49.37162780761719,
      "activations/layer18_attention_weight_min": -41.33055877685547,
      "activations/layer19_attention_weight_max": 29.055490493774414,
      "activations/layer19_attention_weight_min": -23.010303497314453,
      "activations/layer1_attention_weight_max": 15.780570983886719,
      "activations/layer1_attention_weight_min": -14.823348999023438,
      "activations/layer20_attention_weight_max": 22.954830169677734,
      "activations/layer20_attention_weight_min": -21.512798309326172,
      "activations/layer21_attention_weight_max": 38.35331344604492,
      "activations/layer21_attention_weight_min": -23.027145385742188,
      "activations/layer22_attention_weight_max": 33.68360900878906,
      "activations/layer22_attention_weight_min": -28.277435302734375,
      "activations/layer23_attention_weight_max": 37.33567428588867,
      "activations/layer23_attention_weight_min": -24.426422119140625,
      "activations/layer2_attention_weight_max": 33.286468505859375,
      "activations/layer2_attention_weight_min": -31.753543853759766,
      "activations/layer3_attention_weight_max": 89.71922302246094,
      "activations/layer3_attention_weight_min": -94.42086029052734,
      "activations/layer4_attention_weight_max": 89.13976287841797,
      "activations/layer4_attention_weight_min": -92.10222625732422,
      "activations/layer5_attention_weight_max": 65.1943588256836,
      "activations/layer5_attention_weight_min": -74.6129150390625,
      "activations/layer6_attention_weight_max": 49.63297653198242,
      "activations/layer6_attention_weight_min": -51.47268295288086,
      "activations/layer7_attention_weight_max": 65.07364654541016,
      "activations/layer7_attention_weight_min": -60.895565032958984,
      "activations/layer8_attention_weight_max": 44.634517669677734,
      "activations/layer8_attention_weight_min": -47.167545318603516,
      "activations/layer9_attention_weight_max": 47.72080612182617,
      "activations/layer9_attention_weight_min": -44.42721939086914,
      "epoch": 15.0,
      "learning_rate": 5.379280303030302e-05,
      "loss": 2.8034,
      "step": 258200
    },
    {
      "activations/layer0_attention_weight_max": 15.789542198181152,
      "activations/layer0_attention_weight_min": -13.352523803710938,
      "activations/layer10_attention_weight_max": 35.978084564208984,
      "activations/layer10_attention_weight_min": -35.974769592285156,
      "activations/layer11_attention_weight_max": 31.55108642578125,
      "activations/layer11_attention_weight_min": -32.943504333496094,
      "activations/layer12_attention_weight_max": 19.747831344604492,
      "activations/layer12_attention_weight_min": -26.61286163330078,
      "activations/layer13_attention_weight_max": 38.56755828857422,
      "activations/layer13_attention_weight_min": -27.07657814025879,
      "activations/layer14_attention_weight_max": 36.56143569946289,
      "activations/layer14_attention_weight_min": -33.025421142578125,
      "activations/layer15_attention_weight_max": 34.377952575683594,
      "activations/layer15_attention_weight_min": -31.04800033569336,
      "activations/layer16_attention_weight_max": 35.138458251953125,
      "activations/layer16_attention_weight_min": -31.92082977294922,
      "activations/layer17_attention_weight_max": 56.81420135498047,
      "activations/layer17_attention_weight_min": -44.39821243286133,
      "activations/layer18_attention_weight_max": 51.84191131591797,
      "activations/layer18_attention_weight_min": -39.877288818359375,
      "activations/layer19_attention_weight_max": 26.37629508972168,
      "activations/layer19_attention_weight_min": -23.69664192199707,
      "activations/layer1_attention_weight_max": 17.61568832397461,
      "activations/layer1_attention_weight_min": -14.274810791015625,
      "activations/layer20_attention_weight_max": 25.49782943725586,
      "activations/layer20_attention_weight_min": -21.635074615478516,
      "activations/layer21_attention_weight_max": 42.6248893737793,
      "activations/layer21_attention_weight_min": -22.512208938598633,
      "activations/layer22_attention_weight_max": 34.11411666870117,
      "activations/layer22_attention_weight_min": -26.22704315185547,
      "activations/layer23_attention_weight_max": 44.300350189208984,
      "activations/layer23_attention_weight_min": -22.501516342163086,
      "activations/layer2_attention_weight_max": 31.212865829467773,
      "activations/layer2_attention_weight_min": -31.142133712768555,
      "activations/layer3_attention_weight_max": 89.9122314453125,
      "activations/layer3_attention_weight_min": -93.77055358886719,
      "activations/layer4_attention_weight_max": 92.46729278564453,
      "activations/layer4_attention_weight_min": -88.60794830322266,
      "activations/layer5_attention_weight_max": 71.75748443603516,
      "activations/layer5_attention_weight_min": -74.42505645751953,
      "activations/layer6_attention_weight_max": 51.31277847290039,
      "activations/layer6_attention_weight_min": -51.798152923583984,
      "activations/layer7_attention_weight_max": 64.04852294921875,
      "activations/layer7_attention_weight_min": -58.22693634033203,
      "activations/layer8_attention_weight_max": 46.66156768798828,
      "activations/layer8_attention_weight_min": -47.96297836303711,
      "activations/layer9_attention_weight_max": 45.298519134521484,
      "activations/layer9_attention_weight_min": -46.211463928222656,
      "epoch": 15.01,
      "learning_rate": 5.377386363636363e-05,
      "loss": 2.7699,
      "step": 258250
    },
    {
      "activations/layer0_attention_weight_max": 15.490911483764648,
      "activations/layer0_attention_weight_min": -13.51893138885498,
      "activations/layer10_attention_weight_max": 35.914581298828125,
      "activations/layer10_attention_weight_min": -36.54045486450195,
      "activations/layer11_attention_weight_max": 33.470428466796875,
      "activations/layer11_attention_weight_min": -33.341190338134766,
      "activations/layer12_attention_weight_max": 19.395450592041016,
      "activations/layer12_attention_weight_min": -23.028879165649414,
      "activations/layer13_attention_weight_max": 38.06816864013672,
      "activations/layer13_attention_weight_min": -28.384157180786133,
      "activations/layer14_attention_weight_max": 37.26768493652344,
      "activations/layer14_attention_weight_min": -35.3210334777832,
      "activations/layer15_attention_weight_max": 34.035770416259766,
      "activations/layer15_attention_weight_min": -33.3546257019043,
      "activations/layer16_attention_weight_max": 35.54863739013672,
      "activations/layer16_attention_weight_min": -34.02405548095703,
      "activations/layer17_attention_weight_max": 58.35954666137695,
      "activations/layer17_attention_weight_min": -50.98646545410156,
      "activations/layer18_attention_weight_max": 53.48032760620117,
      "activations/layer18_attention_weight_min": -44.97151184082031,
      "activations/layer19_attention_weight_max": 27.395238876342773,
      "activations/layer19_attention_weight_min": -26.465778350830078,
      "activations/layer1_attention_weight_max": 16.47913360595703,
      "activations/layer1_attention_weight_min": -15.628040313720703,
      "activations/layer20_attention_weight_max": 26.210058212280273,
      "activations/layer20_attention_weight_min": -23.491506576538086,
      "activations/layer21_attention_weight_max": 41.001251220703125,
      "activations/layer21_attention_weight_min": -23.605337142944336,
      "activations/layer22_attention_weight_max": 34.11778259277344,
      "activations/layer22_attention_weight_min": -29.257139205932617,
      "activations/layer23_attention_weight_max": 44.61407470703125,
      "activations/layer23_attention_weight_min": -23.409706115722656,
      "activations/layer2_attention_weight_max": 32.22247314453125,
      "activations/layer2_attention_weight_min": -30.905942916870117,
      "activations/layer3_attention_weight_max": 92.33894348144531,
      "activations/layer3_attention_weight_min": -93.72735595703125,
      "activations/layer4_attention_weight_max": 92.84053802490234,
      "activations/layer4_attention_weight_min": -88.48531341552734,
      "activations/layer5_attention_weight_max": 66.1763687133789,
      "activations/layer5_attention_weight_min": -72.46098327636719,
      "activations/layer6_attention_weight_max": 50.72761535644531,
      "activations/layer6_attention_weight_min": -50.61054992675781,
      "activations/layer7_attention_weight_max": 66.34463500976562,
      "activations/layer7_attention_weight_min": -61.955055236816406,
      "activations/layer8_attention_weight_max": 44.42985534667969,
      "activations/layer8_attention_weight_min": -48.84379196166992,
      "activations/layer9_attention_weight_max": 44.04846954345703,
      "activations/layer9_attention_weight_min": -46.805076599121094,
      "epoch": 15.01,
      "learning_rate": 5.375492424242424e-05,
      "loss": 2.745,
      "step": 258300
    },
    {
      "activations/layer0_attention_weight_max": 16.23583984375,
      "activations/layer0_attention_weight_min": -13.168681144714355,
      "activations/layer10_attention_weight_max": 32.38994598388672,
      "activations/layer10_attention_weight_min": -35.281951904296875,
      "activations/layer11_attention_weight_max": 32.94294738769531,
      "activations/layer11_attention_weight_min": -31.661039352416992,
      "activations/layer12_attention_weight_max": 18.50247573852539,
      "activations/layer12_attention_weight_min": -32.050201416015625,
      "activations/layer13_attention_weight_max": 36.54995346069336,
      "activations/layer13_attention_weight_min": -30.990493774414062,
      "activations/layer14_attention_weight_max": 36.6902961730957,
      "activations/layer14_attention_weight_min": -30.689159393310547,
      "activations/layer15_attention_weight_max": 35.48506164550781,
      "activations/layer15_attention_weight_min": -29.87538719177246,
      "activations/layer16_attention_weight_max": 33.16099166870117,
      "activations/layer16_attention_weight_min": -30.103485107421875,
      "activations/layer17_attention_weight_max": 55.55058670043945,
      "activations/layer17_attention_weight_min": -46.62700271606445,
      "activations/layer18_attention_weight_max": 49.4116325378418,
      "activations/layer18_attention_weight_min": -40.28278732299805,
      "activations/layer19_attention_weight_max": 24.794330596923828,
      "activations/layer19_attention_weight_min": -20.4499568939209,
      "activations/layer1_attention_weight_max": 18.463361740112305,
      "activations/layer1_attention_weight_min": -14.891404151916504,
      "activations/layer20_attention_weight_max": 21.588462829589844,
      "activations/layer20_attention_weight_min": -22.178691864013672,
      "activations/layer21_attention_weight_max": 41.640045166015625,
      "activations/layer21_attention_weight_min": -29.275646209716797,
      "activations/layer22_attention_weight_max": 30.249286651611328,
      "activations/layer22_attention_weight_min": -24.19424819946289,
      "activations/layer23_attention_weight_max": 41.068565368652344,
      "activations/layer23_attention_weight_min": -23.82012939453125,
      "activations/layer2_attention_weight_max": 32.268924713134766,
      "activations/layer2_attention_weight_min": -33.66807556152344,
      "activations/layer3_attention_weight_max": 90.66392517089844,
      "activations/layer3_attention_weight_min": -91.2152099609375,
      "activations/layer4_attention_weight_max": 88.56108856201172,
      "activations/layer4_attention_weight_min": -87.29430389404297,
      "activations/layer5_attention_weight_max": 65.32632446289062,
      "activations/layer5_attention_weight_min": -69.32847595214844,
      "activations/layer6_attention_weight_max": 48.66655731201172,
      "activations/layer6_attention_weight_min": -48.25886154174805,
      "activations/layer7_attention_weight_max": 59.09358215332031,
      "activations/layer7_attention_weight_min": -58.31718444824219,
      "activations/layer8_attention_weight_max": 42.796302795410156,
      "activations/layer8_attention_weight_min": -41.7355842590332,
      "activations/layer9_attention_weight_max": 48.972530364990234,
      "activations/layer9_attention_weight_min": -51.04548645019531,
      "epoch": 15.01,
      "learning_rate": 5.373598484848485e-05,
      "loss": 2.7473,
      "step": 258350
    },
    {
      "activations/layer0_attention_weight_max": 16.00887680053711,
      "activations/layer0_attention_weight_min": -13.369321823120117,
      "activations/layer10_attention_weight_max": 34.930335998535156,
      "activations/layer10_attention_weight_min": -35.057403564453125,
      "activations/layer11_attention_weight_max": 31.427345275878906,
      "activations/layer11_attention_weight_min": -34.522403717041016,
      "activations/layer12_attention_weight_max": 18.21781349182129,
      "activations/layer12_attention_weight_min": -23.782649993896484,
      "activations/layer13_attention_weight_max": 36.95042037963867,
      "activations/layer13_attention_weight_min": -29.898056030273438,
      "activations/layer14_attention_weight_max": 38.050968170166016,
      "activations/layer14_attention_weight_min": -31.375381469726562,
      "activations/layer15_attention_weight_max": 30.803966522216797,
      "activations/layer15_attention_weight_min": -31.003273010253906,
      "activations/layer16_attention_weight_max": 32.64114761352539,
      "activations/layer16_attention_weight_min": -31.693679809570312,
      "activations/layer17_attention_weight_max": 51.69325256347656,
      "activations/layer17_attention_weight_min": -45.25638198852539,
      "activations/layer18_attention_weight_max": 46.96578598022461,
      "activations/layer18_attention_weight_min": -38.05099868774414,
      "activations/layer19_attention_weight_max": 25.545913696289062,
      "activations/layer19_attention_weight_min": -22.78292465209961,
      "activations/layer1_attention_weight_max": 17.784772872924805,
      "activations/layer1_attention_weight_min": -15.001590728759766,
      "activations/layer20_attention_weight_max": 24.007762908935547,
      "activations/layer20_attention_weight_min": -21.173799514770508,
      "activations/layer21_attention_weight_max": 40.33848190307617,
      "activations/layer21_attention_weight_min": -22.728736877441406,
      "activations/layer22_attention_weight_max": 29.201839447021484,
      "activations/layer22_attention_weight_min": -25.543365478515625,
      "activations/layer23_attention_weight_max": 35.67288589477539,
      "activations/layer23_attention_weight_min": -24.292675018310547,
      "activations/layer2_attention_weight_max": 31.849647521972656,
      "activations/layer2_attention_weight_min": -32.751670837402344,
      "activations/layer3_attention_weight_max": 90.26288604736328,
      "activations/layer3_attention_weight_min": -102.27099609375,
      "activations/layer4_attention_weight_max": 88.47455596923828,
      "activations/layer4_attention_weight_min": -89.60602569580078,
      "activations/layer5_attention_weight_max": 63.34457015991211,
      "activations/layer5_attention_weight_min": -76.82304382324219,
      "activations/layer6_attention_weight_max": 49.186180114746094,
      "activations/layer6_attention_weight_min": -51.097408294677734,
      "activations/layer7_attention_weight_max": 63.29243087768555,
      "activations/layer7_attention_weight_min": -60.03884506225586,
      "activations/layer8_attention_weight_max": 42.26262283325195,
      "activations/layer8_attention_weight_min": -46.827735900878906,
      "activations/layer9_attention_weight_max": 43.683467864990234,
      "activations/layer9_attention_weight_min": -42.91598129272461,
      "epoch": 15.01,
      "learning_rate": 5.371704545454545e-05,
      "loss": 2.7358,
      "step": 258400
    },
    {
      "activations/layer0_attention_weight_max": 16.737991333007812,
      "activations/layer0_attention_weight_min": -13.358719825744629,
      "activations/layer10_attention_weight_max": 34.935546875,
      "activations/layer10_attention_weight_min": -35.991851806640625,
      "activations/layer11_attention_weight_max": 31.15108871459961,
      "activations/layer11_attention_weight_min": -33.197853088378906,
      "activations/layer12_attention_weight_max": 18.461383819580078,
      "activations/layer12_attention_weight_min": -24.34086799621582,
      "activations/layer13_attention_weight_max": 37.231910705566406,
      "activations/layer13_attention_weight_min": -28.711212158203125,
      "activations/layer14_attention_weight_max": 35.65392303466797,
      "activations/layer14_attention_weight_min": -30.94280433654785,
      "activations/layer15_attention_weight_max": 31.925403594970703,
      "activations/layer15_attention_weight_min": -32.09944152832031,
      "activations/layer16_attention_weight_max": 32.45454025268555,
      "activations/layer16_attention_weight_min": -31.02536964416504,
      "activations/layer17_attention_weight_max": 50.43998718261719,
      "activations/layer17_attention_weight_min": -46.84083938598633,
      "activations/layer18_attention_weight_max": 46.26317596435547,
      "activations/layer18_attention_weight_min": -39.547760009765625,
      "activations/layer19_attention_weight_max": 24.029037475585938,
      "activations/layer19_attention_weight_min": -22.20557975769043,
      "activations/layer1_attention_weight_max": 16.74622344970703,
      "activations/layer1_attention_weight_min": -17.588010787963867,
      "activations/layer20_attention_weight_max": 22.029085159301758,
      "activations/layer20_attention_weight_min": -21.313705444335938,
      "activations/layer21_attention_weight_max": 39.283287048339844,
      "activations/layer21_attention_weight_min": -25.851959228515625,
      "activations/layer22_attention_weight_max": 30.14004135131836,
      "activations/layer22_attention_weight_min": -28.887794494628906,
      "activations/layer23_attention_weight_max": 41.83665466308594,
      "activations/layer23_attention_weight_min": -25.979320526123047,
      "activations/layer2_attention_weight_max": 33.93327331542969,
      "activations/layer2_attention_weight_min": -30.1290283203125,
      "activations/layer3_attention_weight_max": 93.22203826904297,
      "activations/layer3_attention_weight_min": -92.43868255615234,
      "activations/layer4_attention_weight_max": 90.07183074951172,
      "activations/layer4_attention_weight_min": -88.3363037109375,
      "activations/layer5_attention_weight_max": 66.60716247558594,
      "activations/layer5_attention_weight_min": -72.86636352539062,
      "activations/layer6_attention_weight_max": 48.60783004760742,
      "activations/layer6_attention_weight_min": -49.27302551269531,
      "activations/layer7_attention_weight_max": 59.90995407104492,
      "activations/layer7_attention_weight_min": -59.64231491088867,
      "activations/layer8_attention_weight_max": 46.222286224365234,
      "activations/layer8_attention_weight_min": -47.16206741333008,
      "activations/layer9_attention_weight_max": 44.63727569580078,
      "activations/layer9_attention_weight_min": -46.29671096801758,
      "epoch": 15.02,
      "learning_rate": 5.369810606060605e-05,
      "loss": 2.74,
      "step": 258450
    },
    {
      "activations/layer0_attention_weight_max": 15.977958679199219,
      "activations/layer0_attention_weight_min": -13.481037139892578,
      "activations/layer10_attention_weight_max": 39.790771484375,
      "activations/layer10_attention_weight_min": -35.382347106933594,
      "activations/layer11_attention_weight_max": 34.67222595214844,
      "activations/layer11_attention_weight_min": -32.91150665283203,
      "activations/layer12_attention_weight_max": 24.67742919921875,
      "activations/layer12_attention_weight_min": -28.72270965576172,
      "activations/layer13_attention_weight_max": 41.14760208129883,
      "activations/layer13_attention_weight_min": -33.36162567138672,
      "activations/layer14_attention_weight_max": 37.68621063232422,
      "activations/layer14_attention_weight_min": -30.937824249267578,
      "activations/layer15_attention_weight_max": 34.17525100708008,
      "activations/layer15_attention_weight_min": -29.122053146362305,
      "activations/layer16_attention_weight_max": 33.47903823852539,
      "activations/layer16_attention_weight_min": -31.130809783935547,
      "activations/layer17_attention_weight_max": 56.480567932128906,
      "activations/layer17_attention_weight_min": -45.245216369628906,
      "activations/layer18_attention_weight_max": 47.9326057434082,
      "activations/layer18_attention_weight_min": -38.45382308959961,
      "activations/layer19_attention_weight_max": 25.274356842041016,
      "activations/layer19_attention_weight_min": -24.100627899169922,
      "activations/layer1_attention_weight_max": 17.246036529541016,
      "activations/layer1_attention_weight_min": -16.127164840698242,
      "activations/layer20_attention_weight_max": 21.312957763671875,
      "activations/layer20_attention_weight_min": -21.47218132019043,
      "activations/layer21_attention_weight_max": 36.00281524658203,
      "activations/layer21_attention_weight_min": -22.79585838317871,
      "activations/layer22_attention_weight_max": 28.615421295166016,
      "activations/layer22_attention_weight_min": -27.06235122680664,
      "activations/layer23_attention_weight_max": 36.20361328125,
      "activations/layer23_attention_weight_min": -24.259674072265625,
      "activations/layer2_attention_weight_max": 33.3964729309082,
      "activations/layer2_attention_weight_min": -32.241336822509766,
      "activations/layer3_attention_weight_max": 94.44916534423828,
      "activations/layer3_attention_weight_min": -97.51739501953125,
      "activations/layer4_attention_weight_max": 92.55445098876953,
      "activations/layer4_attention_weight_min": -94.44817352294922,
      "activations/layer5_attention_weight_max": 65.92772674560547,
      "activations/layer5_attention_weight_min": -71.37884521484375,
      "activations/layer6_attention_weight_max": 54.69648361206055,
      "activations/layer6_attention_weight_min": -54.25294494628906,
      "activations/layer7_attention_weight_max": 60.9643440246582,
      "activations/layer7_attention_weight_min": -59.96751403808594,
      "activations/layer8_attention_weight_max": 45.60857391357422,
      "activations/layer8_attention_weight_min": -46.32815933227539,
      "activations/layer9_attention_weight_max": 48.88331604003906,
      "activations/layer9_attention_weight_min": -47.26121139526367,
      "epoch": 15.02,
      "learning_rate": 5.3679166666666666e-05,
      "loss": 2.7621,
      "step": 258500
    },
    {
      "activations/layer0_attention_weight_max": 15.910504341125488,
      "activations/layer0_attention_weight_min": -14.163460731506348,
      "activations/layer10_attention_weight_max": 34.70869827270508,
      "activations/layer10_attention_weight_min": -34.570457458496094,
      "activations/layer11_attention_weight_max": 32.51807403564453,
      "activations/layer11_attention_weight_min": -34.932777404785156,
      "activations/layer12_attention_weight_max": 17.657093048095703,
      "activations/layer12_attention_weight_min": -26.35450553894043,
      "activations/layer13_attention_weight_max": 28.55778694152832,
      "activations/layer13_attention_weight_min": -27.434865951538086,
      "activations/layer14_attention_weight_max": 31.576208114624023,
      "activations/layer14_attention_weight_min": -30.151897430419922,
      "activations/layer15_attention_weight_max": 29.3808536529541,
      "activations/layer15_attention_weight_min": -28.170438766479492,
      "activations/layer16_attention_weight_max": 29.67058563232422,
      "activations/layer16_attention_weight_min": -29.722013473510742,
      "activations/layer17_attention_weight_max": 48.14850616455078,
      "activations/layer17_attention_weight_min": -45.0786247253418,
      "activations/layer18_attention_weight_max": 42.524112701416016,
      "activations/layer18_attention_weight_min": -38.05706024169922,
      "activations/layer19_attention_weight_max": 21.222055435180664,
      "activations/layer19_attention_weight_min": -24.65987205505371,
      "activations/layer1_attention_weight_max": 16.734159469604492,
      "activations/layer1_attention_weight_min": -15.043408393859863,
      "activations/layer20_attention_weight_max": 20.0335750579834,
      "activations/layer20_attention_weight_min": -24.708288192749023,
      "activations/layer21_attention_weight_max": 32.193450927734375,
      "activations/layer21_attention_weight_min": -22.923263549804688,
      "activations/layer22_attention_weight_max": 26.039968490600586,
      "activations/layer22_attention_weight_min": -27.29218292236328,
      "activations/layer23_attention_weight_max": 32.3070182800293,
      "activations/layer23_attention_weight_min": -25.188167572021484,
      "activations/layer2_attention_weight_max": 33.15624237060547,
      "activations/layer2_attention_weight_min": -31.299314498901367,
      "activations/layer3_attention_weight_max": 90.01188659667969,
      "activations/layer3_attention_weight_min": -89.85674285888672,
      "activations/layer4_attention_weight_max": 93.06364440917969,
      "activations/layer4_attention_weight_min": -95.92674255371094,
      "activations/layer5_attention_weight_max": 64.25916290283203,
      "activations/layer5_attention_weight_min": -75.79523468017578,
      "activations/layer6_attention_weight_max": 49.36015319824219,
      "activations/layer6_attention_weight_min": -48.67158126831055,
      "activations/layer7_attention_weight_max": 64.58407592773438,
      "activations/layer7_attention_weight_min": -57.70674514770508,
      "activations/layer8_attention_weight_max": 46.676292419433594,
      "activations/layer8_attention_weight_min": -47.11766052246094,
      "activations/layer9_attention_weight_max": 45.44244384765625,
      "activations/layer9_attention_weight_min": -45.35349655151367,
      "epoch": 15.02,
      "learning_rate": 5.366022727272727e-05,
      "loss": 2.7459,
      "step": 258550
    },
    {
      "activations/layer0_attention_weight_max": 16.357900619506836,
      "activations/layer0_attention_weight_min": -13.782499313354492,
      "activations/layer10_attention_weight_max": 37.65544128417969,
      "activations/layer10_attention_weight_min": -35.68683624267578,
      "activations/layer11_attention_weight_max": 33.8740234375,
      "activations/layer11_attention_weight_min": -33.296783447265625,
      "activations/layer12_attention_weight_max": 19.420015335083008,
      "activations/layer12_attention_weight_min": -25.788583755493164,
      "activations/layer13_attention_weight_max": 32.46125793457031,
      "activations/layer13_attention_weight_min": -27.400936126708984,
      "activations/layer14_attention_weight_max": 39.427852630615234,
      "activations/layer14_attention_weight_min": -31.532297134399414,
      "activations/layer15_attention_weight_max": 33.99722671508789,
      "activations/layer15_attention_weight_min": -31.30335235595703,
      "activations/layer16_attention_weight_max": 33.45354461669922,
      "activations/layer16_attention_weight_min": -31.716739654541016,
      "activations/layer17_attention_weight_max": 52.0688362121582,
      "activations/layer17_attention_weight_min": -45.32566452026367,
      "activations/layer18_attention_weight_max": 50.08612060546875,
      "activations/layer18_attention_weight_min": -41.06301498413086,
      "activations/layer19_attention_weight_max": 23.35761833190918,
      "activations/layer19_attention_weight_min": -23.57491683959961,
      "activations/layer1_attention_weight_max": 16.379592895507812,
      "activations/layer1_attention_weight_min": -14.928571701049805,
      "activations/layer20_attention_weight_max": 23.231950759887695,
      "activations/layer20_attention_weight_min": -21.397281646728516,
      "activations/layer21_attention_weight_max": 39.846256256103516,
      "activations/layer21_attention_weight_min": -27.09227180480957,
      "activations/layer22_attention_weight_max": 31.039209365844727,
      "activations/layer22_attention_weight_min": -26.83267593383789,
      "activations/layer23_attention_weight_max": 42.93254852294922,
      "activations/layer23_attention_weight_min": -25.658740997314453,
      "activations/layer2_attention_weight_max": 32.5433349609375,
      "activations/layer2_attention_weight_min": -30.847614288330078,
      "activations/layer3_attention_weight_max": 89.0584945678711,
      "activations/layer3_attention_weight_min": -95.28213500976562,
      "activations/layer4_attention_weight_max": 86.40340423583984,
      "activations/layer4_attention_weight_min": -86.21794128417969,
      "activations/layer5_attention_weight_max": 65.78785705566406,
      "activations/layer5_attention_weight_min": -75.80059814453125,
      "activations/layer6_attention_weight_max": 47.886714935302734,
      "activations/layer6_attention_weight_min": -50.51750183105469,
      "activations/layer7_attention_weight_max": 59.79026412963867,
      "activations/layer7_attention_weight_min": -60.84797286987305,
      "activations/layer8_attention_weight_max": 45.5189094543457,
      "activations/layer8_attention_weight_min": -48.852142333984375,
      "activations/layer9_attention_weight_max": 44.328121185302734,
      "activations/layer9_attention_weight_min": -44.96993637084961,
      "epoch": 15.03,
      "learning_rate": 5.3641287878787876e-05,
      "loss": 2.7533,
      "step": 258600
    },
    {
      "activations/layer0_attention_weight_max": 16.043319702148438,
      "activations/layer0_attention_weight_min": -13.915871620178223,
      "activations/layer10_attention_weight_max": 33.83388900756836,
      "activations/layer10_attention_weight_min": -33.508140563964844,
      "activations/layer11_attention_weight_max": 30.300613403320312,
      "activations/layer11_attention_weight_min": -32.78084945678711,
      "activations/layer12_attention_weight_max": 17.853168487548828,
      "activations/layer12_attention_weight_min": -25.389036178588867,
      "activations/layer13_attention_weight_max": 33.78590393066406,
      "activations/layer13_attention_weight_min": -27.62188148498535,
      "activations/layer14_attention_weight_max": 33.3370361328125,
      "activations/layer14_attention_weight_min": -28.761171340942383,
      "activations/layer15_attention_weight_max": 29.732995986938477,
      "activations/layer15_attention_weight_min": -28.292400360107422,
      "activations/layer16_attention_weight_max": 30.957433700561523,
      "activations/layer16_attention_weight_min": -29.086973190307617,
      "activations/layer17_attention_weight_max": 47.2460823059082,
      "activations/layer17_attention_weight_min": -40.98689651489258,
      "activations/layer18_attention_weight_max": 41.59064865112305,
      "activations/layer18_attention_weight_min": -35.931461334228516,
      "activations/layer19_attention_weight_max": 22.047801971435547,
      "activations/layer19_attention_weight_min": -21.810203552246094,
      "activations/layer1_attention_weight_max": 16.452392578125,
      "activations/layer1_attention_weight_min": -14.788625717163086,
      "activations/layer20_attention_weight_max": 20.09082794189453,
      "activations/layer20_attention_weight_min": -22.660932540893555,
      "activations/layer21_attention_weight_max": 29.18326759338379,
      "activations/layer21_attention_weight_min": -20.434816360473633,
      "activations/layer22_attention_weight_max": 29.028549194335938,
      "activations/layer22_attention_weight_min": -26.42071533203125,
      "activations/layer23_attention_weight_max": 32.84126663208008,
      "activations/layer23_attention_weight_min": -23.557109832763672,
      "activations/layer2_attention_weight_max": 31.711956024169922,
      "activations/layer2_attention_weight_min": -34.98487854003906,
      "activations/layer3_attention_weight_max": 91.32819366455078,
      "activations/layer3_attention_weight_min": -104.1537857055664,
      "activations/layer4_attention_weight_max": 89.6683578491211,
      "activations/layer4_attention_weight_min": -94.43827056884766,
      "activations/layer5_attention_weight_max": 62.502960205078125,
      "activations/layer5_attention_weight_min": -74.82830810546875,
      "activations/layer6_attention_weight_max": 48.050865173339844,
      "activations/layer6_attention_weight_min": -48.92531967163086,
      "activations/layer7_attention_weight_max": 60.48186492919922,
      "activations/layer7_attention_weight_min": -63.12371063232422,
      "activations/layer8_attention_weight_max": 42.745540618896484,
      "activations/layer8_attention_weight_min": -49.26231384277344,
      "activations/layer9_attention_weight_max": 43.53004455566406,
      "activations/layer9_attention_weight_min": -47.03833770751953,
      "epoch": 15.03,
      "learning_rate": 5.3622727272727266e-05,
      "loss": 2.7439,
      "step": 258650
    },
    {
      "activations/layer0_attention_weight_max": 15.825947761535645,
      "activations/layer0_attention_weight_min": -13.874753952026367,
      "activations/layer10_attention_weight_max": 36.89134979248047,
      "activations/layer10_attention_weight_min": -36.318641662597656,
      "activations/layer11_attention_weight_max": 33.6850471496582,
      "activations/layer11_attention_weight_min": -33.179931640625,
      "activations/layer12_attention_weight_max": 19.167274475097656,
      "activations/layer12_attention_weight_min": -21.965295791625977,
      "activations/layer13_attention_weight_max": 39.85429000854492,
      "activations/layer13_attention_weight_min": -29.948841094970703,
      "activations/layer14_attention_weight_max": 36.9709587097168,
      "activations/layer14_attention_weight_min": -29.6317138671875,
      "activations/layer15_attention_weight_max": 31.000574111938477,
      "activations/layer15_attention_weight_min": -29.10458755493164,
      "activations/layer16_attention_weight_max": 31.06315040588379,
      "activations/layer16_attention_weight_min": -30.760957717895508,
      "activations/layer17_attention_weight_max": 51.76844787597656,
      "activations/layer17_attention_weight_min": -44.512264251708984,
      "activations/layer18_attention_weight_max": 46.90143966674805,
      "activations/layer18_attention_weight_min": -40.102073669433594,
      "activations/layer19_attention_weight_max": 22.687040328979492,
      "activations/layer19_attention_weight_min": -21.984663009643555,
      "activations/layer1_attention_weight_max": 16.45084571838379,
      "activations/layer1_attention_weight_min": -15.964394569396973,
      "activations/layer20_attention_weight_max": 23.711118698120117,
      "activations/layer20_attention_weight_min": -21.567546844482422,
      "activations/layer21_attention_weight_max": 34.8353271484375,
      "activations/layer21_attention_weight_min": -21.172780990600586,
      "activations/layer22_attention_weight_max": 28.690916061401367,
      "activations/layer22_attention_weight_min": -25.51422882080078,
      "activations/layer23_attention_weight_max": 34.62599563598633,
      "activations/layer23_attention_weight_min": -22.990741729736328,
      "activations/layer2_attention_weight_max": 32.128597259521484,
      "activations/layer2_attention_weight_min": -33.39372634887695,
      "activations/layer3_attention_weight_max": 95.37073516845703,
      "activations/layer3_attention_weight_min": -101.32565307617188,
      "activations/layer4_attention_weight_max": 92.44927215576172,
      "activations/layer4_attention_weight_min": -92.63143157958984,
      "activations/layer5_attention_weight_max": 66.65802001953125,
      "activations/layer5_attention_weight_min": -74.56541442871094,
      "activations/layer6_attention_weight_max": 49.63031768798828,
      "activations/layer6_attention_weight_min": -49.868289947509766,
      "activations/layer7_attention_weight_max": 61.964500427246094,
      "activations/layer7_attention_weight_min": -62.33540344238281,
      "activations/layer8_attention_weight_max": 47.468650817871094,
      "activations/layer8_attention_weight_min": -49.63014221191406,
      "activations/layer9_attention_weight_max": 46.33122253417969,
      "activations/layer9_attention_weight_min": -47.9796028137207,
      "epoch": 15.03,
      "learning_rate": 5.3603787878787875e-05,
      "loss": 2.751,
      "step": 258700
    },
    {
      "activations/layer0_attention_weight_max": 15.464567184448242,
      "activations/layer0_attention_weight_min": -13.471766471862793,
      "activations/layer10_attention_weight_max": 34.1595458984375,
      "activations/layer10_attention_weight_min": -34.70256423950195,
      "activations/layer11_attention_weight_max": 33.220001220703125,
      "activations/layer11_attention_weight_min": -34.31640625,
      "activations/layer12_attention_weight_max": 17.90162467956543,
      "activations/layer12_attention_weight_min": -25.108890533447266,
      "activations/layer13_attention_weight_max": 32.16297149658203,
      "activations/layer13_attention_weight_min": -27.404470443725586,
      "activations/layer14_attention_weight_max": 34.64386749267578,
      "activations/layer14_attention_weight_min": -28.952251434326172,
      "activations/layer15_attention_weight_max": 33.13712692260742,
      "activations/layer15_attention_weight_min": -29.16950035095215,
      "activations/layer16_attention_weight_max": 34.530582427978516,
      "activations/layer16_attention_weight_min": -30.881128311157227,
      "activations/layer17_attention_weight_max": 53.57374954223633,
      "activations/layer17_attention_weight_min": -46.02424240112305,
      "activations/layer18_attention_weight_max": 46.275020599365234,
      "activations/layer18_attention_weight_min": -39.40088653564453,
      "activations/layer19_attention_weight_max": 22.022380828857422,
      "activations/layer19_attention_weight_min": -21.428560256958008,
      "activations/layer1_attention_weight_max": 16.392427444458008,
      "activations/layer1_attention_weight_min": -15.049491882324219,
      "activations/layer20_attention_weight_max": 20.593469619750977,
      "activations/layer20_attention_weight_min": -20.997825622558594,
      "activations/layer21_attention_weight_max": 39.259742736816406,
      "activations/layer21_attention_weight_min": -24.591337203979492,
      "activations/layer22_attention_weight_max": 29.32663345336914,
      "activations/layer22_attention_weight_min": -25.87094497680664,
      "activations/layer23_attention_weight_max": 35.060691833496094,
      "activations/layer23_attention_weight_min": -24.605838775634766,
      "activations/layer2_attention_weight_max": 30.376935958862305,
      "activations/layer2_attention_weight_min": -29.857177734375,
      "activations/layer3_attention_weight_max": 90.52067565917969,
      "activations/layer3_attention_weight_min": -92.49980926513672,
      "activations/layer4_attention_weight_max": 88.588134765625,
      "activations/layer4_attention_weight_min": -89.56214904785156,
      "activations/layer5_attention_weight_max": 63.92904281616211,
      "activations/layer5_attention_weight_min": -69.25959777832031,
      "activations/layer6_attention_weight_max": 48.05287170410156,
      "activations/layer6_attention_weight_min": -50.60334777832031,
      "activations/layer7_attention_weight_max": 62.098388671875,
      "activations/layer7_attention_weight_min": -63.20750045776367,
      "activations/layer8_attention_weight_max": 44.46223831176758,
      "activations/layer8_attention_weight_min": -45.46669387817383,
      "activations/layer9_attention_weight_max": 43.56208038330078,
      "activations/layer9_attention_weight_min": -47.0159797668457,
      "epoch": 15.03,
      "learning_rate": 5.358484848484848e-05,
      "loss": 2.7603,
      "step": 258750
    },
    {
      "activations/layer0_attention_weight_max": 16.090009689331055,
      "activations/layer0_attention_weight_min": -13.634671211242676,
      "activations/layer10_attention_weight_max": 34.77423858642578,
      "activations/layer10_attention_weight_min": -34.68849563598633,
      "activations/layer11_attention_weight_max": 31.194255828857422,
      "activations/layer11_attention_weight_min": -32.53813934326172,
      "activations/layer12_attention_weight_max": 19.34612274169922,
      "activations/layer12_attention_weight_min": -30.130800247192383,
      "activations/layer13_attention_weight_max": 33.97489929199219,
      "activations/layer13_attention_weight_min": -29.228050231933594,
      "activations/layer14_attention_weight_max": 37.85383605957031,
      "activations/layer14_attention_weight_min": -32.515804290771484,
      "activations/layer15_attention_weight_max": 33.98321533203125,
      "activations/layer15_attention_weight_min": -29.943580627441406,
      "activations/layer16_attention_weight_max": 33.32429504394531,
      "activations/layer16_attention_weight_min": -31.075159072875977,
      "activations/layer17_attention_weight_max": 53.94453430175781,
      "activations/layer17_attention_weight_min": -47.048667907714844,
      "activations/layer18_attention_weight_max": 48.725341796875,
      "activations/layer18_attention_weight_min": -40.34828186035156,
      "activations/layer19_attention_weight_max": 23.44497299194336,
      "activations/layer19_attention_weight_min": -23.49637222290039,
      "activations/layer1_attention_weight_max": 16.368125915527344,
      "activations/layer1_attention_weight_min": -14.483952522277832,
      "activations/layer20_attention_weight_max": 23.051713943481445,
      "activations/layer20_attention_weight_min": -23.27220916748047,
      "activations/layer21_attention_weight_max": 44.9062614440918,
      "activations/layer21_attention_weight_min": -23.250043869018555,
      "activations/layer22_attention_weight_max": 30.51462745666504,
      "activations/layer22_attention_weight_min": -27.941173553466797,
      "activations/layer23_attention_weight_max": 38.01170349121094,
      "activations/layer23_attention_weight_min": -23.924894332885742,
      "activations/layer2_attention_weight_max": 31.49390411376953,
      "activations/layer2_attention_weight_min": -30.04574966430664,
      "activations/layer3_attention_weight_max": 91.2962646484375,
      "activations/layer3_attention_weight_min": -90.44053649902344,
      "activations/layer4_attention_weight_max": 86.36841583251953,
      "activations/layer4_attention_weight_min": -88.80817413330078,
      "activations/layer5_attention_weight_max": 65.35487365722656,
      "activations/layer5_attention_weight_min": -73.0162582397461,
      "activations/layer6_attention_weight_max": 48.15868377685547,
      "activations/layer6_attention_weight_min": -48.86750793457031,
      "activations/layer7_attention_weight_max": 60.1973991394043,
      "activations/layer7_attention_weight_min": -61.19795608520508,
      "activations/layer8_attention_weight_max": 47.83853530883789,
      "activations/layer8_attention_weight_min": -49.243309020996094,
      "activations/layer9_attention_weight_max": 45.31460189819336,
      "activations/layer9_attention_weight_min": -44.6229362487793,
      "epoch": 15.04,
      "learning_rate": 5.3565909090909084e-05,
      "loss": 2.7451,
      "step": 258800
    },
    {
      "activations/layer0_attention_weight_max": 15.940595626831055,
      "activations/layer0_attention_weight_min": -13.706297874450684,
      "activations/layer10_attention_weight_max": 36.53321075439453,
      "activations/layer10_attention_weight_min": -36.708953857421875,
      "activations/layer11_attention_weight_max": 32.466217041015625,
      "activations/layer11_attention_weight_min": -33.05274200439453,
      "activations/layer12_attention_weight_max": 17.51357650756836,
      "activations/layer12_attention_weight_min": -25.592819213867188,
      "activations/layer13_attention_weight_max": 29.7466983795166,
      "activations/layer13_attention_weight_min": -27.998062133789062,
      "activations/layer14_attention_weight_max": 34.988399505615234,
      "activations/layer14_attention_weight_min": -28.785964965820312,
      "activations/layer15_attention_weight_max": 31.823959350585938,
      "activations/layer15_attention_weight_min": -29.26679039001465,
      "activations/layer16_attention_weight_max": 29.942113876342773,
      "activations/layer16_attention_weight_min": -29.762794494628906,
      "activations/layer17_attention_weight_max": 49.685787200927734,
      "activations/layer17_attention_weight_min": -42.7806510925293,
      "activations/layer18_attention_weight_max": 46.679481506347656,
      "activations/layer18_attention_weight_min": -37.742061614990234,
      "activations/layer19_attention_weight_max": 23.553205490112305,
      "activations/layer19_attention_weight_min": -22.378000259399414,
      "activations/layer1_attention_weight_max": 16.65875244140625,
      "activations/layer1_attention_weight_min": -15.256054878234863,
      "activations/layer20_attention_weight_max": 23.311250686645508,
      "activations/layer20_attention_weight_min": -24.622777938842773,
      "activations/layer21_attention_weight_max": 36.138710021972656,
      "activations/layer21_attention_weight_min": -22.423330307006836,
      "activations/layer22_attention_weight_max": 30.41359519958496,
      "activations/layer22_attention_weight_min": -25.951250076293945,
      "activations/layer23_attention_weight_max": 37.2414665222168,
      "activations/layer23_attention_weight_min": -23.361766815185547,
      "activations/layer2_attention_weight_max": 33.485137939453125,
      "activations/layer2_attention_weight_min": -31.769977569580078,
      "activations/layer3_attention_weight_max": 92.49484252929688,
      "activations/layer3_attention_weight_min": -98.32249450683594,
      "activations/layer4_attention_weight_max": 93.23079681396484,
      "activations/layer4_attention_weight_min": -91.87101745605469,
      "activations/layer5_attention_weight_max": 63.56486511230469,
      "activations/layer5_attention_weight_min": -72.68404388427734,
      "activations/layer6_attention_weight_max": 49.07777786254883,
      "activations/layer6_attention_weight_min": -48.20155715942383,
      "activations/layer7_attention_weight_max": 63.384212493896484,
      "activations/layer7_attention_weight_min": -56.13706970214844,
      "activations/layer8_attention_weight_max": 45.27764129638672,
      "activations/layer8_attention_weight_min": -50.472286224365234,
      "activations/layer9_attention_weight_max": 46.71987533569336,
      "activations/layer9_attention_weight_min": -48.291481018066406,
      "epoch": 15.04,
      "learning_rate": 5.3546969696969686e-05,
      "loss": 2.7502,
      "step": 258850
    },
    {
      "activations/layer0_attention_weight_max": 15.535065650939941,
      "activations/layer0_attention_weight_min": -13.506202697753906,
      "activations/layer10_attention_weight_max": 35.3703498840332,
      "activations/layer10_attention_weight_min": -34.59740447998047,
      "activations/layer11_attention_weight_max": 31.772205352783203,
      "activations/layer11_attention_weight_min": -34.14464569091797,
      "activations/layer12_attention_weight_max": 20.042131423950195,
      "activations/layer12_attention_weight_min": -24.60698127746582,
      "activations/layer13_attention_weight_max": 36.09259033203125,
      "activations/layer13_attention_weight_min": -26.893033981323242,
      "activations/layer14_attention_weight_max": 37.41727828979492,
      "activations/layer14_attention_weight_min": -30.40682601928711,
      "activations/layer15_attention_weight_max": 33.89162826538086,
      "activations/layer15_attention_weight_min": -30.399578094482422,
      "activations/layer16_attention_weight_max": 34.52849578857422,
      "activations/layer16_attention_weight_min": -32.88954162597656,
      "activations/layer17_attention_weight_max": 55.10892105102539,
      "activations/layer17_attention_weight_min": -48.156768798828125,
      "activations/layer18_attention_weight_max": 50.66616439819336,
      "activations/layer18_attention_weight_min": -39.194583892822266,
      "activations/layer19_attention_weight_max": 22.345142364501953,
      "activations/layer19_attention_weight_min": -24.36600112915039,
      "activations/layer1_attention_weight_max": 16.815092086791992,
      "activations/layer1_attention_weight_min": -13.97801399230957,
      "activations/layer20_attention_weight_max": 23.074684143066406,
      "activations/layer20_attention_weight_min": -22.468427658081055,
      "activations/layer21_attention_weight_max": 39.03871536254883,
      "activations/layer21_attention_weight_min": -22.513097763061523,
      "activations/layer22_attention_weight_max": 29.404327392578125,
      "activations/layer22_attention_weight_min": -25.535175323486328,
      "activations/layer23_attention_weight_max": 36.91148376464844,
      "activations/layer23_attention_weight_min": -24.2148494720459,
      "activations/layer2_attention_weight_max": 30.35141372680664,
      "activations/layer2_attention_weight_min": -30.139060974121094,
      "activations/layer3_attention_weight_max": 93.55634307861328,
      "activations/layer3_attention_weight_min": -96.1408462524414,
      "activations/layer4_attention_weight_max": 89.87342071533203,
      "activations/layer4_attention_weight_min": -87.23604583740234,
      "activations/layer5_attention_weight_max": 66.12140655517578,
      "activations/layer5_attention_weight_min": -72.29056549072266,
      "activations/layer6_attention_weight_max": 48.48893737792969,
      "activations/layer6_attention_weight_min": -48.51028060913086,
      "activations/layer7_attention_weight_max": 61.9500846862793,
      "activations/layer7_attention_weight_min": -60.04771423339844,
      "activations/layer8_attention_weight_max": 42.891319274902344,
      "activations/layer8_attention_weight_min": -47.6507682800293,
      "activations/layer9_attention_weight_max": 43.815673828125,
      "activations/layer9_attention_weight_min": -44.24042892456055,
      "epoch": 15.04,
      "learning_rate": 5.35280303030303e-05,
      "loss": 2.7401,
      "step": 258900
    },
    {
      "activations/layer0_attention_weight_max": 15.54641342163086,
      "activations/layer0_attention_weight_min": -13.881342887878418,
      "activations/layer10_attention_weight_max": 36.036231994628906,
      "activations/layer10_attention_weight_min": -35.36785888671875,
      "activations/layer11_attention_weight_max": 33.12318420410156,
      "activations/layer11_attention_weight_min": -33.351566314697266,
      "activations/layer12_attention_weight_max": 18.20783805847168,
      "activations/layer12_attention_weight_min": -23.223716735839844,
      "activations/layer13_attention_weight_max": 38.58195877075195,
      "activations/layer13_attention_weight_min": -30.467010498046875,
      "activations/layer14_attention_weight_max": 33.74254608154297,
      "activations/layer14_attention_weight_min": -30.260149002075195,
      "activations/layer15_attention_weight_max": 30.26076316833496,
      "activations/layer15_attention_weight_min": -30.4432430267334,
      "activations/layer16_attention_weight_max": 30.398393630981445,
      "activations/layer16_attention_weight_min": -30.065786361694336,
      "activations/layer17_attention_weight_max": 48.95572280883789,
      "activations/layer17_attention_weight_min": -48.16957473754883,
      "activations/layer18_attention_weight_max": 44.52033996582031,
      "activations/layer18_attention_weight_min": -42.057796478271484,
      "activations/layer19_attention_weight_max": 22.822792053222656,
      "activations/layer19_attention_weight_min": -25.37175941467285,
      "activations/layer1_attention_weight_max": 15.755167961120605,
      "activations/layer1_attention_weight_min": -14.458951950073242,
      "activations/layer20_attention_weight_max": 21.400789260864258,
      "activations/layer20_attention_weight_min": -26.637975692749023,
      "activations/layer21_attention_weight_max": 36.4129524230957,
      "activations/layer21_attention_weight_min": -23.788375854492188,
      "activations/layer22_attention_weight_max": 28.801593780517578,
      "activations/layer22_attention_weight_min": -29.437959671020508,
      "activations/layer23_attention_weight_max": 39.502708435058594,
      "activations/layer23_attention_weight_min": -26.183897018432617,
      "activations/layer2_attention_weight_max": 30.249988555908203,
      "activations/layer2_attention_weight_min": -28.7485408782959,
      "activations/layer3_attention_weight_max": 85.4966812133789,
      "activations/layer3_attention_weight_min": -89.02593994140625,
      "activations/layer4_attention_weight_max": 88.19255828857422,
      "activations/layer4_attention_weight_min": -85.8671875,
      "activations/layer5_attention_weight_max": 65.63652038574219,
      "activations/layer5_attention_weight_min": -73.95804595947266,
      "activations/layer6_attention_weight_max": 46.99013900756836,
      "activations/layer6_attention_weight_min": -46.642005920410156,
      "activations/layer7_attention_weight_max": 64.20439910888672,
      "activations/layer7_attention_weight_min": -60.06359100341797,
      "activations/layer8_attention_weight_max": 46.02048873901367,
      "activations/layer8_attention_weight_min": -49.859561920166016,
      "activations/layer9_attention_weight_max": 46.50286865234375,
      "activations/layer9_attention_weight_min": -48.283084869384766,
      "epoch": 15.05,
      "learning_rate": 5.35090909090909e-05,
      "loss": 2.7591,
      "step": 258950
    },
    {
      "activations/layer0_attention_weight_max": 16.921215057373047,
      "activations/layer0_attention_weight_min": -13.54182243347168,
      "activations/layer10_attention_weight_max": 41.01516342163086,
      "activations/layer10_attention_weight_min": -41.3848876953125,
      "activations/layer11_attention_weight_max": 38.073089599609375,
      "activations/layer11_attention_weight_min": -34.298561096191406,
      "activations/layer12_attention_weight_max": 16.7056827545166,
      "activations/layer12_attention_weight_min": -28.0618953704834,
      "activations/layer13_attention_weight_max": 41.59569549560547,
      "activations/layer13_attention_weight_min": -27.641956329345703,
      "activations/layer14_attention_weight_max": 34.05662536621094,
      "activations/layer14_attention_weight_min": -30.20530891418457,
      "activations/layer15_attention_weight_max": 30.843175888061523,
      "activations/layer15_attention_weight_min": -29.842531204223633,
      "activations/layer16_attention_weight_max": 31.210803985595703,
      "activations/layer16_attention_weight_min": -31.24953269958496,
      "activations/layer17_attention_weight_max": 46.75358200073242,
      "activations/layer17_attention_weight_min": -42.27312088012695,
      "activations/layer18_attention_weight_max": 42.205299377441406,
      "activations/layer18_attention_weight_min": -39.575923919677734,
      "activations/layer19_attention_weight_max": 22.480710983276367,
      "activations/layer19_attention_weight_min": -23.843036651611328,
      "activations/layer1_attention_weight_max": 17.68079376220703,
      "activations/layer1_attention_weight_min": -15.339860916137695,
      "activations/layer20_attention_weight_max": 25.58211898803711,
      "activations/layer20_attention_weight_min": -21.45276641845703,
      "activations/layer21_attention_weight_max": 34.147300720214844,
      "activations/layer21_attention_weight_min": -25.4573917388916,
      "activations/layer22_attention_weight_max": 27.941572189331055,
      "activations/layer22_attention_weight_min": -25.77190399169922,
      "activations/layer23_attention_weight_max": 37.146644592285156,
      "activations/layer23_attention_weight_min": -25.383445739746094,
      "activations/layer2_attention_weight_max": 33.55101776123047,
      "activations/layer2_attention_weight_min": -31.37902069091797,
      "activations/layer3_attention_weight_max": 93.89933776855469,
      "activations/layer3_attention_weight_min": -94.3904037475586,
      "activations/layer4_attention_weight_max": 90.04003143310547,
      "activations/layer4_attention_weight_min": -88.49410247802734,
      "activations/layer5_attention_weight_max": 70.78337860107422,
      "activations/layer5_attention_weight_min": -71.3418197631836,
      "activations/layer6_attention_weight_max": 46.58761215209961,
      "activations/layer6_attention_weight_min": -48.51203155517578,
      "activations/layer7_attention_weight_max": 76.82823944091797,
      "activations/layer7_attention_weight_min": -62.42386245727539,
      "activations/layer8_attention_weight_max": 51.84701156616211,
      "activations/layer8_attention_weight_min": -45.87412643432617,
      "activations/layer9_attention_weight_max": 63.50032424926758,
      "activations/layer9_attention_weight_min": -48.158058166503906,
      "epoch": 15.05,
      "learning_rate": 5.349015151515151e-05,
      "loss": 2.7505,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.5129,
      "eval_samples_per_second": 504.412,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.5129,
      "eval_openwebtext_samples_per_second": 504.412,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9794,
      "eval_wikitext_samples_per_second": 230.373,
      "step": 259000
    },
    {
      "epoch": 15.05,
      "eval_lambada_loss": 2.5546875,
      "eval_lambada_ppl": 12.867278004795136,
      "eval_lambada_runtime": 9.5442,
      "eval_lambada_samples_per_second": 510.152,
      "step": 259000
    },
    {
      "activations/layer0_attention_weight_max": 15.989495277404785,
      "activations/layer0_attention_weight_min": -13.409651756286621,
      "activations/layer10_attention_weight_max": 35.18427276611328,
      "activations/layer10_attention_weight_min": -35.03166580200195,
      "activations/layer11_attention_weight_max": 34.497535705566406,
      "activations/layer11_attention_weight_min": -35.95079803466797,
      "activations/layer12_attention_weight_max": 18.38579559326172,
      "activations/layer12_attention_weight_min": -28.409774780273438,
      "activations/layer13_attention_weight_max": 37.6949348449707,
      "activations/layer13_attention_weight_min": -28.22458839416504,
      "activations/layer14_attention_weight_max": 35.36296081542969,
      "activations/layer14_attention_weight_min": -30.774728775024414,
      "activations/layer15_attention_weight_max": 31.684497833251953,
      "activations/layer15_attention_weight_min": -28.6561336517334,
      "activations/layer16_attention_weight_max": 31.066450119018555,
      "activations/layer16_attention_weight_min": -29.051395416259766,
      "activations/layer17_attention_weight_max": 48.15016174316406,
      "activations/layer17_attention_weight_min": -43.500614166259766,
      "activations/layer18_attention_weight_max": 43.544456481933594,
      "activations/layer18_attention_weight_min": -37.861053466796875,
      "activations/layer19_attention_weight_max": 21.698471069335938,
      "activations/layer19_attention_weight_min": -23.38899803161621,
      "activations/layer1_attention_weight_max": 16.207412719726562,
      "activations/layer1_attention_weight_min": -14.69057846069336,
      "activations/layer20_attention_weight_max": 20.876487731933594,
      "activations/layer20_attention_weight_min": -22.217632293701172,
      "activations/layer21_attention_weight_max": 40.194923400878906,
      "activations/layer21_attention_weight_min": -23.204191207885742,
      "activations/layer22_attention_weight_max": 27.52262306213379,
      "activations/layer22_attention_weight_min": -25.56763458251953,
      "activations/layer23_attention_weight_max": 34.097923278808594,
      "activations/layer23_attention_weight_min": -25.757673263549805,
      "activations/layer2_attention_weight_max": 30.75841522216797,
      "activations/layer2_attention_weight_min": -29.105817794799805,
      "activations/layer3_attention_weight_max": 89.662109375,
      "activations/layer3_attention_weight_min": -94.92859649658203,
      "activations/layer4_attention_weight_max": 92.41939544677734,
      "activations/layer4_attention_weight_min": -85.85692596435547,
      "activations/layer5_attention_weight_max": 67.76483917236328,
      "activations/layer5_attention_weight_min": -71.19219207763672,
      "activations/layer6_attention_weight_max": 51.783443450927734,
      "activations/layer6_attention_weight_min": -50.42707061767578,
      "activations/layer7_attention_weight_max": 65.8471450805664,
      "activations/layer7_attention_weight_min": -60.24524688720703,
      "activations/layer8_attention_weight_max": 44.25814437866211,
      "activations/layer8_attention_weight_min": -47.89093780517578,
      "activations/layer9_attention_weight_max": 45.55564880371094,
      "activations/layer9_attention_weight_min": -47.74563980102539,
      "epoch": 15.05,
      "learning_rate": 5.347121212121212e-05,
      "loss": 2.7587,
      "step": 259050
    },
    {
      "activations/layer0_attention_weight_max": 15.874015808105469,
      "activations/layer0_attention_weight_min": -13.960826873779297,
      "activations/layer10_attention_weight_max": 37.420867919921875,
      "activations/layer10_attention_weight_min": -37.466827392578125,
      "activations/layer11_attention_weight_max": 34.22560119628906,
      "activations/layer11_attention_weight_min": -35.15257263183594,
      "activations/layer12_attention_weight_max": 20.458797454833984,
      "activations/layer12_attention_weight_min": -26.257221221923828,
      "activations/layer13_attention_weight_max": 39.52635192871094,
      "activations/layer13_attention_weight_min": -30.396577835083008,
      "activations/layer14_attention_weight_max": 35.561622619628906,
      "activations/layer14_attention_weight_min": -28.707923889160156,
      "activations/layer15_attention_weight_max": 33.78263854980469,
      "activations/layer15_attention_weight_min": -28.692659378051758,
      "activations/layer16_attention_weight_max": 32.08805847167969,
      "activations/layer16_attention_weight_min": -29.997838973999023,
      "activations/layer17_attention_weight_max": 52.283363342285156,
      "activations/layer17_attention_weight_min": -45.14413833618164,
      "activations/layer18_attention_weight_max": 46.12871551513672,
      "activations/layer18_attention_weight_min": -39.31318283081055,
      "activations/layer19_attention_weight_max": 23.566436767578125,
      "activations/layer19_attention_weight_min": -21.87944793701172,
      "activations/layer1_attention_weight_max": 16.38368034362793,
      "activations/layer1_attention_weight_min": -14.521346092224121,
      "activations/layer20_attention_weight_max": 25.4608211517334,
      "activations/layer20_attention_weight_min": -21.8452205657959,
      "activations/layer21_attention_weight_max": 40.92263412475586,
      "activations/layer21_attention_weight_min": -24.878990173339844,
      "activations/layer22_attention_weight_max": 29.58382225036621,
      "activations/layer22_attention_weight_min": -26.335494995117188,
      "activations/layer23_attention_weight_max": 39.344276428222656,
      "activations/layer23_attention_weight_min": -25.09360122680664,
      "activations/layer2_attention_weight_max": 32.53061294555664,
      "activations/layer2_attention_weight_min": -32.50007629394531,
      "activations/layer3_attention_weight_max": 96.5181655883789,
      "activations/layer3_attention_weight_min": -97.95657348632812,
      "activations/layer4_attention_weight_max": 94.26842498779297,
      "activations/layer4_attention_weight_min": -88.94750213623047,
      "activations/layer5_attention_weight_max": 67.76346588134766,
      "activations/layer5_attention_weight_min": -77.81519317626953,
      "activations/layer6_attention_weight_max": 49.739688873291016,
      "activations/layer6_attention_weight_min": -50.29124069213867,
      "activations/layer7_attention_weight_max": 62.66014099121094,
      "activations/layer7_attention_weight_min": -61.316688537597656,
      "activations/layer8_attention_weight_max": 47.34559631347656,
      "activations/layer8_attention_weight_min": -46.49893569946289,
      "activations/layer9_attention_weight_max": 46.30587387084961,
      "activations/layer9_attention_weight_min": -47.25282669067383,
      "epoch": 15.06,
      "learning_rate": 5.345227272727273e-05,
      "loss": 2.7518,
      "step": 259100
    },
    {
      "activations/layer0_attention_weight_max": 16.06485939025879,
      "activations/layer0_attention_weight_min": -13.750179290771484,
      "activations/layer10_attention_weight_max": 35.2515869140625,
      "activations/layer10_attention_weight_min": -33.3421630859375,
      "activations/layer11_attention_weight_max": 30.909881591796875,
      "activations/layer11_attention_weight_min": -30.466039657592773,
      "activations/layer12_attention_weight_max": 18.391183853149414,
      "activations/layer12_attention_weight_min": -26.769563674926758,
      "activations/layer13_attention_weight_max": 34.47472381591797,
      "activations/layer13_attention_weight_min": -27.88945960998535,
      "activations/layer14_attention_weight_max": 36.437347412109375,
      "activations/layer14_attention_weight_min": -31.51866340637207,
      "activations/layer15_attention_weight_max": 33.036678314208984,
      "activations/layer15_attention_weight_min": -30.55055046081543,
      "activations/layer16_attention_weight_max": 33.461490631103516,
      "activations/layer16_attention_weight_min": -30.23897361755371,
      "activations/layer17_attention_weight_max": 51.1489143371582,
      "activations/layer17_attention_weight_min": -45.73158645629883,
      "activations/layer18_attention_weight_max": 48.47345733642578,
      "activations/layer18_attention_weight_min": -38.302555084228516,
      "activations/layer19_attention_weight_max": 22.73673439025879,
      "activations/layer19_attention_weight_min": -22.04987335205078,
      "activations/layer1_attention_weight_max": 16.16093635559082,
      "activations/layer1_attention_weight_min": -15.35767650604248,
      "activations/layer20_attention_weight_max": 25.772600173950195,
      "activations/layer20_attention_weight_min": -22.311809539794922,
      "activations/layer21_attention_weight_max": 35.16911315917969,
      "activations/layer21_attention_weight_min": -23.221145629882812,
      "activations/layer22_attention_weight_max": 32.42173767089844,
      "activations/layer22_attention_weight_min": -26.119815826416016,
      "activations/layer23_attention_weight_max": 41.097496032714844,
      "activations/layer23_attention_weight_min": -26.405643463134766,
      "activations/layer2_attention_weight_max": 30.719585418701172,
      "activations/layer2_attention_weight_min": -31.582799911499023,
      "activations/layer3_attention_weight_max": 88.42893981933594,
      "activations/layer3_attention_weight_min": -97.78685760498047,
      "activations/layer4_attention_weight_max": 86.1837158203125,
      "activations/layer4_attention_weight_min": -87.68168640136719,
      "activations/layer5_attention_weight_max": 68.7508316040039,
      "activations/layer5_attention_weight_min": -75.78669738769531,
      "activations/layer6_attention_weight_max": 46.79368209838867,
      "activations/layer6_attention_weight_min": -49.746604919433594,
      "activations/layer7_attention_weight_max": 61.63642120361328,
      "activations/layer7_attention_weight_min": -62.921424865722656,
      "activations/layer8_attention_weight_max": 42.65860366821289,
      "activations/layer8_attention_weight_min": -45.42154312133789,
      "activations/layer9_attention_weight_max": 41.43339538574219,
      "activations/layer9_attention_weight_min": -43.51328659057617,
      "epoch": 15.06,
      "learning_rate": 5.343333333333333e-05,
      "loss": 2.7552,
      "step": 259150
    },
    {
      "activations/layer0_attention_weight_max": 16.233074188232422,
      "activations/layer0_attention_weight_min": -13.809808731079102,
      "activations/layer10_attention_weight_max": 37.141963958740234,
      "activations/layer10_attention_weight_min": -35.76693344116211,
      "activations/layer11_attention_weight_max": 32.01045227050781,
      "activations/layer11_attention_weight_min": -33.68926239013672,
      "activations/layer12_attention_weight_max": 17.447437286376953,
      "activations/layer12_attention_weight_min": -25.583749771118164,
      "activations/layer13_attention_weight_max": 31.357929229736328,
      "activations/layer13_attention_weight_min": -27.627155303955078,
      "activations/layer14_attention_weight_max": 36.14058303833008,
      "activations/layer14_attention_weight_min": -31.183441162109375,
      "activations/layer15_attention_weight_max": 33.796836853027344,
      "activations/layer15_attention_weight_min": -29.44184112548828,
      "activations/layer16_attention_weight_max": 35.05424118041992,
      "activations/layer16_attention_weight_min": -32.277992248535156,
      "activations/layer17_attention_weight_max": 55.0306282043457,
      "activations/layer17_attention_weight_min": -46.47856521606445,
      "activations/layer18_attention_weight_max": 50.35612106323242,
      "activations/layer18_attention_weight_min": -38.5707893371582,
      "activations/layer19_attention_weight_max": 25.6903018951416,
      "activations/layer19_attention_weight_min": -20.87969398498535,
      "activations/layer1_attention_weight_max": 15.603121757507324,
      "activations/layer1_attention_weight_min": -15.676847457885742,
      "activations/layer20_attention_weight_max": 23.433719635009766,
      "activations/layer20_attention_weight_min": -24.945960998535156,
      "activations/layer21_attention_weight_max": 39.91798782348633,
      "activations/layer21_attention_weight_min": -22.9886474609375,
      "activations/layer22_attention_weight_max": 31.058170318603516,
      "activations/layer22_attention_weight_min": -26.490522384643555,
      "activations/layer23_attention_weight_max": 37.55902099609375,
      "activations/layer23_attention_weight_min": -23.589582443237305,
      "activations/layer2_attention_weight_max": 30.92396354675293,
      "activations/layer2_attention_weight_min": -30.168943405151367,
      "activations/layer3_attention_weight_max": 92.0691909790039,
      "activations/layer3_attention_weight_min": -91.08747863769531,
      "activations/layer4_attention_weight_max": 93.99617767333984,
      "activations/layer4_attention_weight_min": -93.06343841552734,
      "activations/layer5_attention_weight_max": 66.06218719482422,
      "activations/layer5_attention_weight_min": -78.5287857055664,
      "activations/layer6_attention_weight_max": 50.45083999633789,
      "activations/layer6_attention_weight_min": -49.48060989379883,
      "activations/layer7_attention_weight_max": 63.27761459350586,
      "activations/layer7_attention_weight_min": -59.8200798034668,
      "activations/layer8_attention_weight_max": 45.89564514160156,
      "activations/layer8_attention_weight_min": -47.42704391479492,
      "activations/layer9_attention_weight_max": 42.89236831665039,
      "activations/layer9_attention_weight_min": -44.35565185546875,
      "epoch": 15.06,
      "learning_rate": 5.341439393939393e-05,
      "loss": 2.7302,
      "step": 259200
    },
    {
      "activations/layer0_attention_weight_max": 15.353878021240234,
      "activations/layer0_attention_weight_min": -13.799178123474121,
      "activations/layer10_attention_weight_max": 36.095890045166016,
      "activations/layer10_attention_weight_min": -35.55446243286133,
      "activations/layer11_attention_weight_max": 32.190155029296875,
      "activations/layer11_attention_weight_min": -32.580867767333984,
      "activations/layer12_attention_weight_max": 19.005287170410156,
      "activations/layer12_attention_weight_min": -26.666996002197266,
      "activations/layer13_attention_weight_max": 47.03231430053711,
      "activations/layer13_attention_weight_min": -31.9780216217041,
      "activations/layer14_attention_weight_max": 36.11738204956055,
      "activations/layer14_attention_weight_min": -29.496747970581055,
      "activations/layer15_attention_weight_max": 33.953670501708984,
      "activations/layer15_attention_weight_min": -32.04591369628906,
      "activations/layer16_attention_weight_max": 30.962011337280273,
      "activations/layer16_attention_weight_min": -33.07850646972656,
      "activations/layer17_attention_weight_max": 51.14143753051758,
      "activations/layer17_attention_weight_min": -46.713985443115234,
      "activations/layer18_attention_weight_max": 50.204063415527344,
      "activations/layer18_attention_weight_min": -38.11448287963867,
      "activations/layer19_attention_weight_max": 24.551305770874023,
      "activations/layer19_attention_weight_min": -22.264028549194336,
      "activations/layer1_attention_weight_max": 17.772754669189453,
      "activations/layer1_attention_weight_min": -15.056666374206543,
      "activations/layer20_attention_weight_max": 23.12546157836914,
      "activations/layer20_attention_weight_min": -26.136333465576172,
      "activations/layer21_attention_weight_max": 35.72450637817383,
      "activations/layer21_attention_weight_min": -22.83867835998535,
      "activations/layer22_attention_weight_max": 27.08690643310547,
      "activations/layer22_attention_weight_min": -26.2087345123291,
      "activations/layer23_attention_weight_max": 36.598228454589844,
      "activations/layer23_attention_weight_min": -23.223339080810547,
      "activations/layer2_attention_weight_max": 31.24224090576172,
      "activations/layer2_attention_weight_min": -29.626968383789062,
      "activations/layer3_attention_weight_max": 86.14252471923828,
      "activations/layer3_attention_weight_min": -85.2430419921875,
      "activations/layer4_attention_weight_max": 85.8653793334961,
      "activations/layer4_attention_weight_min": -85.18795013427734,
      "activations/layer5_attention_weight_max": 66.4708023071289,
      "activations/layer5_attention_weight_min": -76.13555908203125,
      "activations/layer6_attention_weight_max": 50.07734680175781,
      "activations/layer6_attention_weight_min": -49.323421478271484,
      "activations/layer7_attention_weight_max": 64.01398468017578,
      "activations/layer7_attention_weight_min": -59.6365966796875,
      "activations/layer8_attention_weight_max": 46.200130462646484,
      "activations/layer8_attention_weight_min": -47.419456481933594,
      "activations/layer9_attention_weight_max": 44.6248893737793,
      "activations/layer9_attention_weight_min": -45.6652946472168,
      "epoch": 15.06,
      "learning_rate": 5.3395454545454545e-05,
      "loss": 2.7629,
      "step": 259250
    },
    {
      "activations/layer0_attention_weight_max": 14.871902465820312,
      "activations/layer0_attention_weight_min": -13.901266098022461,
      "activations/layer10_attention_weight_max": 35.61223220825195,
      "activations/layer10_attention_weight_min": -35.0059700012207,
      "activations/layer11_attention_weight_max": 32.59743881225586,
      "activations/layer11_attention_weight_min": -33.59562683105469,
      "activations/layer12_attention_weight_max": 19.475618362426758,
      "activations/layer12_attention_weight_min": -27.346885681152344,
      "activations/layer13_attention_weight_max": 37.73761749267578,
      "activations/layer13_attention_weight_min": -26.840757369995117,
      "activations/layer14_attention_weight_max": 42.661598205566406,
      "activations/layer14_attention_weight_min": -32.07261657714844,
      "activations/layer15_attention_weight_max": 40.3120002746582,
      "activations/layer15_attention_weight_min": -28.917600631713867,
      "activations/layer16_attention_weight_max": 40.74395751953125,
      "activations/layer16_attention_weight_min": -34.25314712524414,
      "activations/layer17_attention_weight_max": 60.49991226196289,
      "activations/layer17_attention_weight_min": -49.04063415527344,
      "activations/layer18_attention_weight_max": 55.217864990234375,
      "activations/layer18_attention_weight_min": -41.5407600402832,
      "activations/layer19_attention_weight_max": 27.37384605407715,
      "activations/layer19_attention_weight_min": -23.582632064819336,
      "activations/layer1_attention_weight_max": 16.539045333862305,
      "activations/layer1_attention_weight_min": -16.186962127685547,
      "activations/layer20_attention_weight_max": 29.009746551513672,
      "activations/layer20_attention_weight_min": -22.190683364868164,
      "activations/layer21_attention_weight_max": 47.71745681762695,
      "activations/layer21_attention_weight_min": -24.17303466796875,
      "activations/layer22_attention_weight_max": 35.23688507080078,
      "activations/layer22_attention_weight_min": -26.422950744628906,
      "activations/layer23_attention_weight_max": 48.096412658691406,
      "activations/layer23_attention_weight_min": -26.904312133789062,
      "activations/layer2_attention_weight_max": 31.09984588623047,
      "activations/layer2_attention_weight_min": -30.77176284790039,
      "activations/layer3_attention_weight_max": 92.77613067626953,
      "activations/layer3_attention_weight_min": -93.36528015136719,
      "activations/layer4_attention_weight_max": 93.2658462524414,
      "activations/layer4_attention_weight_min": -91.25103759765625,
      "activations/layer5_attention_weight_max": 67.61590576171875,
      "activations/layer5_attention_weight_min": -81.32717895507812,
      "activations/layer6_attention_weight_max": 50.391658782958984,
      "activations/layer6_attention_weight_min": -49.634178161621094,
      "activations/layer7_attention_weight_max": 61.85138702392578,
      "activations/layer7_attention_weight_min": -64.84693145751953,
      "activations/layer8_attention_weight_max": 45.860172271728516,
      "activations/layer8_attention_weight_min": -48.46624755859375,
      "activations/layer9_attention_weight_max": 50.0653190612793,
      "activations/layer9_attention_weight_min": -47.68254852294922,
      "epoch": 15.07,
      "learning_rate": 5.337651515151515e-05,
      "loss": 2.7513,
      "step": 259300
    },
    {
      "activations/layer0_attention_weight_max": 16.41946792602539,
      "activations/layer0_attention_weight_min": -13.891854286193848,
      "activations/layer10_attention_weight_max": 35.17724609375,
      "activations/layer10_attention_weight_min": -36.99489212036133,
      "activations/layer11_attention_weight_max": 32.11024475097656,
      "activations/layer11_attention_weight_min": -34.0051383972168,
      "activations/layer12_attention_weight_max": 19.58517837524414,
      "activations/layer12_attention_weight_min": -23.666868209838867,
      "activations/layer13_attention_weight_max": 34.80042266845703,
      "activations/layer13_attention_weight_min": -27.745718002319336,
      "activations/layer14_attention_weight_max": 36.75083541870117,
      "activations/layer14_attention_weight_min": -31.51433563232422,
      "activations/layer15_attention_weight_max": 33.09844207763672,
      "activations/layer15_attention_weight_min": -31.368181228637695,
      "activations/layer16_attention_weight_max": 32.68964385986328,
      "activations/layer16_attention_weight_min": -31.448745727539062,
      "activations/layer17_attention_weight_max": 50.840126037597656,
      "activations/layer17_attention_weight_min": -45.4114875793457,
      "activations/layer18_attention_weight_max": 46.03986740112305,
      "activations/layer18_attention_weight_min": -39.389610290527344,
      "activations/layer19_attention_weight_max": 21.93903923034668,
      "activations/layer19_attention_weight_min": -22.844858169555664,
      "activations/layer1_attention_weight_max": 16.89687728881836,
      "activations/layer1_attention_weight_min": -14.646576881408691,
      "activations/layer20_attention_weight_max": 23.817943572998047,
      "activations/layer20_attention_weight_min": -22.827054977416992,
      "activations/layer21_attention_weight_max": 33.747047424316406,
      "activations/layer21_attention_weight_min": -24.74086570739746,
      "activations/layer22_attention_weight_max": 29.54621696472168,
      "activations/layer22_attention_weight_min": -28.546939849853516,
      "activations/layer23_attention_weight_max": 38.31243896484375,
      "activations/layer23_attention_weight_min": -23.955982208251953,
      "activations/layer2_attention_weight_max": 31.148090362548828,
      "activations/layer2_attention_weight_min": -31.3001708984375,
      "activations/layer3_attention_weight_max": 89.27507781982422,
      "activations/layer3_attention_weight_min": -101.74443054199219,
      "activations/layer4_attention_weight_max": 91.3658447265625,
      "activations/layer4_attention_weight_min": -88.52623748779297,
      "activations/layer5_attention_weight_max": 66.9902572631836,
      "activations/layer5_attention_weight_min": -75.24488067626953,
      "activations/layer6_attention_weight_max": 51.84810256958008,
      "activations/layer6_attention_weight_min": -50.53880310058594,
      "activations/layer7_attention_weight_max": 69.22956848144531,
      "activations/layer7_attention_weight_min": -57.79090881347656,
      "activations/layer8_attention_weight_max": 45.29834747314453,
      "activations/layer8_attention_weight_min": -47.24189758300781,
      "activations/layer9_attention_weight_max": 45.85154724121094,
      "activations/layer9_attention_weight_min": -45.827632904052734,
      "epoch": 15.07,
      "learning_rate": 5.3357575757575755e-05,
      "loss": 2.7572,
      "step": 259350
    },
    {
      "activations/layer0_attention_weight_max": 15.516887664794922,
      "activations/layer0_attention_weight_min": -13.68876838684082,
      "activations/layer10_attention_weight_max": 36.68901824951172,
      "activations/layer10_attention_weight_min": -34.311767578125,
      "activations/layer11_attention_weight_max": 33.67207336425781,
      "activations/layer11_attention_weight_min": -33.91766357421875,
      "activations/layer12_attention_weight_max": 17.618576049804688,
      "activations/layer12_attention_weight_min": -23.720138549804688,
      "activations/layer13_attention_weight_max": 32.99394989013672,
      "activations/layer13_attention_weight_min": -28.063425064086914,
      "activations/layer14_attention_weight_max": 33.02193832397461,
      "activations/layer14_attention_weight_min": -29.55483055114746,
      "activations/layer15_attention_weight_max": 30.38370132446289,
      "activations/layer15_attention_weight_min": -27.438451766967773,
      "activations/layer16_attention_weight_max": 30.155309677124023,
      "activations/layer16_attention_weight_min": -29.56501007080078,
      "activations/layer17_attention_weight_max": 47.319435119628906,
      "activations/layer17_attention_weight_min": -41.12004089355469,
      "activations/layer18_attention_weight_max": 43.140872955322266,
      "activations/layer18_attention_weight_min": -39.65627670288086,
      "activations/layer19_attention_weight_max": 21.67635726928711,
      "activations/layer19_attention_weight_min": -21.901472091674805,
      "activations/layer1_attention_weight_max": 15.649309158325195,
      "activations/layer1_attention_weight_min": -14.741619110107422,
      "activations/layer20_attention_weight_max": 20.886255264282227,
      "activations/layer20_attention_weight_min": -22.046886444091797,
      "activations/layer21_attention_weight_max": 32.54459762573242,
      "activations/layer21_attention_weight_min": -23.95138931274414,
      "activations/layer22_attention_weight_max": 27.090662002563477,
      "activations/layer22_attention_weight_min": -27.82462501525879,
      "activations/layer23_attention_weight_max": 34.77130889892578,
      "activations/layer23_attention_weight_min": -27.120147705078125,
      "activations/layer2_attention_weight_max": 33.19534683227539,
      "activations/layer2_attention_weight_min": -32.70508575439453,
      "activations/layer3_attention_weight_max": 94.26560974121094,
      "activations/layer3_attention_weight_min": -98.09398651123047,
      "activations/layer4_attention_weight_max": 89.73975372314453,
      "activations/layer4_attention_weight_min": -91.34011840820312,
      "activations/layer5_attention_weight_max": 65.0490951538086,
      "activations/layer5_attention_weight_min": -74.95841979980469,
      "activations/layer6_attention_weight_max": 48.687171936035156,
      "activations/layer6_attention_weight_min": -52.22577667236328,
      "activations/layer7_attention_weight_max": 63.52444076538086,
      "activations/layer7_attention_weight_min": -59.143333435058594,
      "activations/layer8_attention_weight_max": 46.03809356689453,
      "activations/layer8_attention_weight_min": -52.73210144042969,
      "activations/layer9_attention_weight_max": 43.22831344604492,
      "activations/layer9_attention_weight_min": -47.74402618408203,
      "epoch": 15.07,
      "learning_rate": 5.333863636363636e-05,
      "loss": 2.759,
      "step": 259400
    },
    {
      "activations/layer0_attention_weight_max": 15.698148727416992,
      "activations/layer0_attention_weight_min": -13.553234100341797,
      "activations/layer10_attention_weight_max": 34.49683380126953,
      "activations/layer10_attention_weight_min": -33.3802375793457,
      "activations/layer11_attention_weight_max": 31.911212921142578,
      "activations/layer11_attention_weight_min": -33.47358703613281,
      "activations/layer12_attention_weight_max": 18.770221710205078,
      "activations/layer12_attention_weight_min": -24.59835433959961,
      "activations/layer13_attention_weight_max": 41.69316864013672,
      "activations/layer13_attention_weight_min": -31.077478408813477,
      "activations/layer14_attention_weight_max": 42.80625534057617,
      "activations/layer14_attention_weight_min": -32.27641296386719,
      "activations/layer15_attention_weight_max": 36.752288818359375,
      "activations/layer15_attention_weight_min": -31.474151611328125,
      "activations/layer16_attention_weight_max": 36.166072845458984,
      "activations/layer16_attention_weight_min": -32.77704620361328,
      "activations/layer17_attention_weight_max": 56.81566619873047,
      "activations/layer17_attention_weight_min": -47.21609878540039,
      "activations/layer18_attention_weight_max": 50.06840896606445,
      "activations/layer18_attention_weight_min": -42.948001861572266,
      "activations/layer19_attention_weight_max": 25.44339370727539,
      "activations/layer19_attention_weight_min": -22.756271362304688,
      "activations/layer1_attention_weight_max": 16.972312927246094,
      "activations/layer1_attention_weight_min": -14.735544204711914,
      "activations/layer20_attention_weight_max": 25.85344696044922,
      "activations/layer20_attention_weight_min": -20.88029670715332,
      "activations/layer21_attention_weight_max": 41.24305725097656,
      "activations/layer21_attention_weight_min": -25.544780731201172,
      "activations/layer22_attention_weight_max": 30.84013557434082,
      "activations/layer22_attention_weight_min": -24.73781394958496,
      "activations/layer23_attention_weight_max": 39.39274597167969,
      "activations/layer23_attention_weight_min": -26.35841178894043,
      "activations/layer2_attention_weight_max": 30.651527404785156,
      "activations/layer2_attention_weight_min": -29.334274291992188,
      "activations/layer3_attention_weight_max": 89.56888580322266,
      "activations/layer3_attention_weight_min": -88.4134292602539,
      "activations/layer4_attention_weight_max": 90.61865234375,
      "activations/layer4_attention_weight_min": -90.1203384399414,
      "activations/layer5_attention_weight_max": 65.49830627441406,
      "activations/layer5_attention_weight_min": -71.27094268798828,
      "activations/layer6_attention_weight_max": 49.50058364868164,
      "activations/layer6_attention_weight_min": -48.11386489868164,
      "activations/layer7_attention_weight_max": 62.220558166503906,
      "activations/layer7_attention_weight_min": -56.34553909301758,
      "activations/layer8_attention_weight_max": 43.540504455566406,
      "activations/layer8_attention_weight_min": -45.91652297973633,
      "activations/layer9_attention_weight_max": 42.2908821105957,
      "activations/layer9_attention_weight_min": -42.8713493347168,
      "epoch": 15.08,
      "learning_rate": 5.3319696969696965e-05,
      "loss": 2.7399,
      "step": 259450
    },
    {
      "activations/layer0_attention_weight_max": 16.268041610717773,
      "activations/layer0_attention_weight_min": -13.821091651916504,
      "activations/layer10_attention_weight_max": 36.10149383544922,
      "activations/layer10_attention_weight_min": -36.12898254394531,
      "activations/layer11_attention_weight_max": 35.55963134765625,
      "activations/layer11_attention_weight_min": -37.39100646972656,
      "activations/layer12_attention_weight_max": 17.930084228515625,
      "activations/layer12_attention_weight_min": -23.32710075378418,
      "activations/layer13_attention_weight_max": 35.81781768798828,
      "activations/layer13_attention_weight_min": -26.64497184753418,
      "activations/layer14_attention_weight_max": 36.076751708984375,
      "activations/layer14_attention_weight_min": -29.865386962890625,
      "activations/layer15_attention_weight_max": 34.267948150634766,
      "activations/layer15_attention_weight_min": -30.657855987548828,
      "activations/layer16_attention_weight_max": 32.64181137084961,
      "activations/layer16_attention_weight_min": -31.21696662902832,
      "activations/layer17_attention_weight_max": 51.29549026489258,
      "activations/layer17_attention_weight_min": -41.295654296875,
      "activations/layer18_attention_weight_max": 46.33478546142578,
      "activations/layer18_attention_weight_min": -38.337371826171875,
      "activations/layer19_attention_weight_max": 24.825422286987305,
      "activations/layer19_attention_weight_min": -21.71748924255371,
      "activations/layer1_attention_weight_max": 15.672320365905762,
      "activations/layer1_attention_weight_min": -14.25333309173584,
      "activations/layer20_attention_weight_max": 24.638090133666992,
      "activations/layer20_attention_weight_min": -21.320737838745117,
      "activations/layer21_attention_weight_max": 35.616703033447266,
      "activations/layer21_attention_weight_min": -21.738271713256836,
      "activations/layer22_attention_weight_max": 31.08828353881836,
      "activations/layer22_attention_weight_min": -26.732006072998047,
      "activations/layer23_attention_weight_max": 38.73411560058594,
      "activations/layer23_attention_weight_min": -31.053075790405273,
      "activations/layer2_attention_weight_max": 32.68925857543945,
      "activations/layer2_attention_weight_min": -31.52770233154297,
      "activations/layer3_attention_weight_max": 92.54633331298828,
      "activations/layer3_attention_weight_min": -97.53964233398438,
      "activations/layer4_attention_weight_max": 94.80248260498047,
      "activations/layer4_attention_weight_min": -91.10594940185547,
      "activations/layer5_attention_weight_max": 70.43696594238281,
      "activations/layer5_attention_weight_min": -77.00880432128906,
      "activations/layer6_attention_weight_max": 50.801204681396484,
      "activations/layer6_attention_weight_min": -52.69480514526367,
      "activations/layer7_attention_weight_max": 64.63391876220703,
      "activations/layer7_attention_weight_min": -63.9947395324707,
      "activations/layer8_attention_weight_max": 47.94802474975586,
      "activations/layer8_attention_weight_min": -49.102962493896484,
      "activations/layer9_attention_weight_max": 46.25936508178711,
      "activations/layer9_attention_weight_min": -46.54460144042969,
      "epoch": 15.08,
      "learning_rate": 5.330075757575757e-05,
      "loss": 2.746,
      "step": 259500
    },
    {
      "activations/layer0_attention_weight_max": 16.0039119720459,
      "activations/layer0_attention_weight_min": -13.786368370056152,
      "activations/layer10_attention_weight_max": 36.232078552246094,
      "activations/layer10_attention_weight_min": -35.44984817504883,
      "activations/layer11_attention_weight_max": 32.95008850097656,
      "activations/layer11_attention_weight_min": -33.14338684082031,
      "activations/layer12_attention_weight_max": 18.774246215820312,
      "activations/layer12_attention_weight_min": -26.56477165222168,
      "activations/layer13_attention_weight_max": 41.24100875854492,
      "activations/layer13_attention_weight_min": -29.610790252685547,
      "activations/layer14_attention_weight_max": 37.97565841674805,
      "activations/layer14_attention_weight_min": -31.07994842529297,
      "activations/layer15_attention_weight_max": 34.10320281982422,
      "activations/layer15_attention_weight_min": -31.056522369384766,
      "activations/layer16_attention_weight_max": 31.399524688720703,
      "activations/layer16_attention_weight_min": -29.343568801879883,
      "activations/layer17_attention_weight_max": 49.80421829223633,
      "activations/layer17_attention_weight_min": -44.38317108154297,
      "activations/layer18_attention_weight_max": 47.208560943603516,
      "activations/layer18_attention_weight_min": -41.38077163696289,
      "activations/layer19_attention_weight_max": 22.972057342529297,
      "activations/layer19_attention_weight_min": -23.63746452331543,
      "activations/layer1_attention_weight_max": 15.81228256225586,
      "activations/layer1_attention_weight_min": -15.064254760742188,
      "activations/layer20_attention_weight_max": 21.206865310668945,
      "activations/layer20_attention_weight_min": -20.969449996948242,
      "activations/layer21_attention_weight_max": 38.26526641845703,
      "activations/layer21_attention_weight_min": -22.572235107421875,
      "activations/layer22_attention_weight_max": 29.258134841918945,
      "activations/layer22_attention_weight_min": -27.065183639526367,
      "activations/layer23_attention_weight_max": 38.413002014160156,
      "activations/layer23_attention_weight_min": -26.180866241455078,
      "activations/layer2_attention_weight_max": 30.841693878173828,
      "activations/layer2_attention_weight_min": -31.686222076416016,
      "activations/layer3_attention_weight_max": 91.27359771728516,
      "activations/layer3_attention_weight_min": -89.72798919677734,
      "activations/layer4_attention_weight_max": 88.41551208496094,
      "activations/layer4_attention_weight_min": -83.2578353881836,
      "activations/layer5_attention_weight_max": 68.47710418701172,
      "activations/layer5_attention_weight_min": -70.73873901367188,
      "activations/layer6_attention_weight_max": 50.988624572753906,
      "activations/layer6_attention_weight_min": -50.14747619628906,
      "activations/layer7_attention_weight_max": 59.04160690307617,
      "activations/layer7_attention_weight_min": -58.294944763183594,
      "activations/layer8_attention_weight_max": 45.078670501708984,
      "activations/layer8_attention_weight_min": -45.592811584472656,
      "activations/layer9_attention_weight_max": 43.8505744934082,
      "activations/layer9_attention_weight_min": -44.813385009765625,
      "epoch": 15.08,
      "learning_rate": 5.3281818181818175e-05,
      "loss": 2.7621,
      "step": 259550
    },
    {
      "activations/layer0_attention_weight_max": 15.593079566955566,
      "activations/layer0_attention_weight_min": -13.156567573547363,
      "activations/layer10_attention_weight_max": 33.14162063598633,
      "activations/layer10_attention_weight_min": -35.27742385864258,
      "activations/layer11_attention_weight_max": 28.60999298095703,
      "activations/layer11_attention_weight_min": -31.12714385986328,
      "activations/layer12_attention_weight_max": 18.27973747253418,
      "activations/layer12_attention_weight_min": -28.1945858001709,
      "activations/layer13_attention_weight_max": 35.07202911376953,
      "activations/layer13_attention_weight_min": -28.41724395751953,
      "activations/layer14_attention_weight_max": 36.08977508544922,
      "activations/layer14_attention_weight_min": -30.711162567138672,
      "activations/layer15_attention_weight_max": 35.464439392089844,
      "activations/layer15_attention_weight_min": -30.368877410888672,
      "activations/layer16_attention_weight_max": 35.52296829223633,
      "activations/layer16_attention_weight_min": -33.1716423034668,
      "activations/layer17_attention_weight_max": 57.97198486328125,
      "activations/layer17_attention_weight_min": -49.007450103759766,
      "activations/layer18_attention_weight_max": 49.691261291503906,
      "activations/layer18_attention_weight_min": -42.25016784667969,
      "activations/layer19_attention_weight_max": 25.2250919342041,
      "activations/layer19_attention_weight_min": -25.429960250854492,
      "activations/layer1_attention_weight_max": 16.57536506652832,
      "activations/layer1_attention_weight_min": -15.152124404907227,
      "activations/layer20_attention_weight_max": 25.06422996520996,
      "activations/layer20_attention_weight_min": -23.169706344604492,
      "activations/layer21_attention_weight_max": 39.75396728515625,
      "activations/layer21_attention_weight_min": -22.68418312072754,
      "activations/layer22_attention_weight_max": 33.18797302246094,
      "activations/layer22_attention_weight_min": -27.252153396606445,
      "activations/layer23_attention_weight_max": 39.186553955078125,
      "activations/layer23_attention_weight_min": -25.723501205444336,
      "activations/layer2_attention_weight_max": 32.56480026245117,
      "activations/layer2_attention_weight_min": -29.893253326416016,
      "activations/layer3_attention_weight_max": 84.18163299560547,
      "activations/layer3_attention_weight_min": -82.47746276855469,
      "activations/layer4_attention_weight_max": 82.080810546875,
      "activations/layer4_attention_weight_min": -82.60931396484375,
      "activations/layer5_attention_weight_max": 61.488521575927734,
      "activations/layer5_attention_weight_min": -72.89015197753906,
      "activations/layer6_attention_weight_max": 49.5207405090332,
      "activations/layer6_attention_weight_min": -49.060970306396484,
      "activations/layer7_attention_weight_max": 56.586708068847656,
      "activations/layer7_attention_weight_min": -58.37722396850586,
      "activations/layer8_attention_weight_max": 41.79619598388672,
      "activations/layer8_attention_weight_min": -46.580162048339844,
      "activations/layer9_attention_weight_max": 43.095760345458984,
      "activations/layer9_attention_weight_min": -46.51482009887695,
      "epoch": 15.08,
      "learning_rate": 5.3262878787878776e-05,
      "loss": 2.7559,
      "step": 259600
    },
    {
      "activations/layer0_attention_weight_max": 16.146211624145508,
      "activations/layer0_attention_weight_min": -13.413727760314941,
      "activations/layer10_attention_weight_max": 36.689613342285156,
      "activations/layer10_attention_weight_min": -37.43853759765625,
      "activations/layer11_attention_weight_max": 33.54419708251953,
      "activations/layer11_attention_weight_min": -33.88408660888672,
      "activations/layer12_attention_weight_max": 18.469694137573242,
      "activations/layer12_attention_weight_min": -27.128049850463867,
      "activations/layer13_attention_weight_max": 31.362260818481445,
      "activations/layer13_attention_weight_min": -27.076412200927734,
      "activations/layer14_attention_weight_max": 39.16524124145508,
      "activations/layer14_attention_weight_min": -31.85703468322754,
      "activations/layer15_attention_weight_max": 35.80420684814453,
      "activations/layer15_attention_weight_min": -31.17926597595215,
      "activations/layer16_attention_weight_max": 32.80351638793945,
      "activations/layer16_attention_weight_min": -34.551612854003906,
      "activations/layer17_attention_weight_max": 50.49681091308594,
      "activations/layer17_attention_weight_min": -50.17464065551758,
      "activations/layer18_attention_weight_max": 46.14216232299805,
      "activations/layer18_attention_weight_min": -41.14570999145508,
      "activations/layer19_attention_weight_max": 23.266448974609375,
      "activations/layer19_attention_weight_min": -23.696958541870117,
      "activations/layer1_attention_weight_max": 17.378767013549805,
      "activations/layer1_attention_weight_min": -15.466651916503906,
      "activations/layer20_attention_weight_max": 22.945695877075195,
      "activations/layer20_attention_weight_min": -24.833646774291992,
      "activations/layer21_attention_weight_max": 37.30139923095703,
      "activations/layer21_attention_weight_min": -25.215158462524414,
      "activations/layer22_attention_weight_max": 28.584035873413086,
      "activations/layer22_attention_weight_min": -29.317543029785156,
      "activations/layer23_attention_weight_max": 36.2921142578125,
      "activations/layer23_attention_weight_min": -26.282894134521484,
      "activations/layer2_attention_weight_max": 32.989112854003906,
      "activations/layer2_attention_weight_min": -35.73604965209961,
      "activations/layer3_attention_weight_max": 92.10030364990234,
      "activations/layer3_attention_weight_min": -96.09886169433594,
      "activations/layer4_attention_weight_max": 93.06011199951172,
      "activations/layer4_attention_weight_min": -86.13314819335938,
      "activations/layer5_attention_weight_max": 68.01813507080078,
      "activations/layer5_attention_weight_min": -78.08784484863281,
      "activations/layer6_attention_weight_max": 52.35007095336914,
      "activations/layer6_attention_weight_min": -51.27239227294922,
      "activations/layer7_attention_weight_max": 65.74431610107422,
      "activations/layer7_attention_weight_min": -62.66262435913086,
      "activations/layer8_attention_weight_max": 46.59575271606445,
      "activations/layer8_attention_weight_min": -49.10346984863281,
      "activations/layer9_attention_weight_max": 50.49905776977539,
      "activations/layer9_attention_weight_min": -46.467018127441406,
      "epoch": 15.09,
      "learning_rate": 5.324393939393939e-05,
      "loss": 2.7482,
      "step": 259650
    },
    {
      "activations/layer0_attention_weight_max": 15.920342445373535,
      "activations/layer0_attention_weight_min": -13.581095695495605,
      "activations/layer10_attention_weight_max": 33.3102912902832,
      "activations/layer10_attention_weight_min": -33.28145217895508,
      "activations/layer11_attention_weight_max": 30.329099655151367,
      "activations/layer11_attention_weight_min": -33.28620529174805,
      "activations/layer12_attention_weight_max": 18.797924041748047,
      "activations/layer12_attention_weight_min": -23.50823211669922,
      "activations/layer13_attention_weight_max": 31.56048011779785,
      "activations/layer13_attention_weight_min": -26.956472396850586,
      "activations/layer14_attention_weight_max": 35.16313171386719,
      "activations/layer14_attention_weight_min": -29.66921615600586,
      "activations/layer15_attention_weight_max": 34.254600524902344,
      "activations/layer15_attention_weight_min": -32.489166259765625,
      "activations/layer16_attention_weight_max": 33.78019714355469,
      "activations/layer16_attention_weight_min": -31.448347091674805,
      "activations/layer17_attention_weight_max": 55.62358474731445,
      "activations/layer17_attention_weight_min": -48.18526840209961,
      "activations/layer18_attention_weight_max": 50.53337860107422,
      "activations/layer18_attention_weight_min": -40.28806686401367,
      "activations/layer19_attention_weight_max": 24.94129180908203,
      "activations/layer19_attention_weight_min": -23.562421798706055,
      "activations/layer1_attention_weight_max": 15.834356307983398,
      "activations/layer1_attention_weight_min": -14.675036430358887,
      "activations/layer20_attention_weight_max": 24.216354370117188,
      "activations/layer20_attention_weight_min": -24.20446014404297,
      "activations/layer21_attention_weight_max": 35.05615234375,
      "activations/layer21_attention_weight_min": -22.080718994140625,
      "activations/layer22_attention_weight_max": 32.372501373291016,
      "activations/layer22_attention_weight_min": -28.29241371154785,
      "activations/layer23_attention_weight_max": 39.25257873535156,
      "activations/layer23_attention_weight_min": -25.711238861083984,
      "activations/layer2_attention_weight_max": 33.208499908447266,
      "activations/layer2_attention_weight_min": -30.49945068359375,
      "activations/layer3_attention_weight_max": 88.11385345458984,
      "activations/layer3_attention_weight_min": -90.504150390625,
      "activations/layer4_attention_weight_max": 89.05620574951172,
      "activations/layer4_attention_weight_min": -86.90214538574219,
      "activations/layer5_attention_weight_max": 66.45186614990234,
      "activations/layer5_attention_weight_min": -74.253173828125,
      "activations/layer6_attention_weight_max": 49.805965423583984,
      "activations/layer6_attention_weight_min": -48.823936462402344,
      "activations/layer7_attention_weight_max": 61.45762634277344,
      "activations/layer7_attention_weight_min": -60.09652328491211,
      "activations/layer8_attention_weight_max": 42.44390106201172,
      "activations/layer8_attention_weight_min": -46.0693359375,
      "activations/layer9_attention_weight_max": 41.04392623901367,
      "activations/layer9_attention_weight_min": -44.783145904541016,
      "epoch": 15.09,
      "learning_rate": 5.322499999999999e-05,
      "loss": 2.7513,
      "step": 259700
    },
    {
      "activations/layer0_attention_weight_max": 15.716588020324707,
      "activations/layer0_attention_weight_min": -13.994722366333008,
      "activations/layer10_attention_weight_max": 38.08666229248047,
      "activations/layer10_attention_weight_min": -36.74293518066406,
      "activations/layer11_attention_weight_max": 33.37274932861328,
      "activations/layer11_attention_weight_min": -34.66813278198242,
      "activations/layer12_attention_weight_max": 19.462764739990234,
      "activations/layer12_attention_weight_min": -22.481718063354492,
      "activations/layer13_attention_weight_max": 46.474327087402344,
      "activations/layer13_attention_weight_min": -31.670259475708008,
      "activations/layer14_attention_weight_max": 37.352333068847656,
      "activations/layer14_attention_weight_min": -33.4813117980957,
      "activations/layer15_attention_weight_max": 36.27220916748047,
      "activations/layer15_attention_weight_min": -31.96653938293457,
      "activations/layer16_attention_weight_max": 38.23251724243164,
      "activations/layer16_attention_weight_min": -36.92805480957031,
      "activations/layer17_attention_weight_max": 54.204898834228516,
      "activations/layer17_attention_weight_min": -50.44134521484375,
      "activations/layer18_attention_weight_max": 51.23762893676758,
      "activations/layer18_attention_weight_min": -41.06421661376953,
      "activations/layer19_attention_weight_max": 22.950559616088867,
      "activations/layer19_attention_weight_min": -21.787817001342773,
      "activations/layer1_attention_weight_max": 16.654733657836914,
      "activations/layer1_attention_weight_min": -14.504170417785645,
      "activations/layer20_attention_weight_max": 24.276927947998047,
      "activations/layer20_attention_weight_min": -23.237464904785156,
      "activations/layer21_attention_weight_max": 39.86258316040039,
      "activations/layer21_attention_weight_min": -20.757898330688477,
      "activations/layer22_attention_weight_max": 31.708770751953125,
      "activations/layer22_attention_weight_min": -25.297367095947266,
      "activations/layer23_attention_weight_max": 40.816505432128906,
      "activations/layer23_attention_weight_min": -20.637863159179688,
      "activations/layer2_attention_weight_max": 32.27229690551758,
      "activations/layer2_attention_weight_min": -31.104393005371094,
      "activations/layer3_attention_weight_max": 90.10770416259766,
      "activations/layer3_attention_weight_min": -94.9502944946289,
      "activations/layer4_attention_weight_max": 92.67665100097656,
      "activations/layer4_attention_weight_min": -89.39676666259766,
      "activations/layer5_attention_weight_max": 65.57781982421875,
      "activations/layer5_attention_weight_min": -75.31768035888672,
      "activations/layer6_attention_weight_max": 50.580631256103516,
      "activations/layer6_attention_weight_min": -52.61117172241211,
      "activations/layer7_attention_weight_max": 62.63078308105469,
      "activations/layer7_attention_weight_min": -63.722320556640625,
      "activations/layer8_attention_weight_max": 45.08180618286133,
      "activations/layer8_attention_weight_min": -50.7684440612793,
      "activations/layer9_attention_weight_max": 45.54780578613281,
      "activations/layer9_attention_weight_min": -46.06088638305664,
      "epoch": 15.09,
      "learning_rate": 5.320643939393939e-05,
      "loss": 2.7518,
      "step": 259750
    },
    {
      "activations/layer0_attention_weight_max": 14.753706932067871,
      "activations/layer0_attention_weight_min": -14.718523025512695,
      "activations/layer10_attention_weight_max": 32.94160461425781,
      "activations/layer10_attention_weight_min": -32.88361358642578,
      "activations/layer11_attention_weight_max": 30.654651641845703,
      "activations/layer11_attention_weight_min": -33.23348617553711,
      "activations/layer12_attention_weight_max": 18.25335693359375,
      "activations/layer12_attention_weight_min": -26.278438568115234,
      "activations/layer13_attention_weight_max": 32.682796478271484,
      "activations/layer13_attention_weight_min": -26.076988220214844,
      "activations/layer14_attention_weight_max": 34.54021072387695,
      "activations/layer14_attention_weight_min": -29.027524948120117,
      "activations/layer15_attention_weight_max": 32.987548828125,
      "activations/layer15_attention_weight_min": -29.197751998901367,
      "activations/layer16_attention_weight_max": 34.13264465332031,
      "activations/layer16_attention_weight_min": -30.631994247436523,
      "activations/layer17_attention_weight_max": 49.1826171875,
      "activations/layer17_attention_weight_min": -43.46845245361328,
      "activations/layer18_attention_weight_max": 46.94441223144531,
      "activations/layer18_attention_weight_min": -38.90949630737305,
      "activations/layer19_attention_weight_max": 21.352216720581055,
      "activations/layer19_attention_weight_min": -22.546932220458984,
      "activations/layer1_attention_weight_max": 16.393234252929688,
      "activations/layer1_attention_weight_min": -14.413091659545898,
      "activations/layer20_attention_weight_max": 24.0284423828125,
      "activations/layer20_attention_weight_min": -19.353567123413086,
      "activations/layer21_attention_weight_max": 35.25254821777344,
      "activations/layer21_attention_weight_min": -22.422494888305664,
      "activations/layer22_attention_weight_max": 34.08625030517578,
      "activations/layer22_attention_weight_min": -23.917253494262695,
      "activations/layer23_attention_weight_max": 37.62098693847656,
      "activations/layer23_attention_weight_min": -25.047176361083984,
      "activations/layer2_attention_weight_max": 32.767059326171875,
      "activations/layer2_attention_weight_min": -32.72123718261719,
      "activations/layer3_attention_weight_max": 94.28385925292969,
      "activations/layer3_attention_weight_min": -96.482177734375,
      "activations/layer4_attention_weight_max": 89.4942855834961,
      "activations/layer4_attention_weight_min": -88.16567993164062,
      "activations/layer5_attention_weight_max": 70.29171752929688,
      "activations/layer5_attention_weight_min": -78.66828155517578,
      "activations/layer6_attention_weight_max": 49.30874252319336,
      "activations/layer6_attention_weight_min": -51.07992935180664,
      "activations/layer7_attention_weight_max": 62.69783020019531,
      "activations/layer7_attention_weight_min": -61.58964157104492,
      "activations/layer8_attention_weight_max": 44.41386795043945,
      "activations/layer8_attention_weight_min": -47.242103576660156,
      "activations/layer9_attention_weight_max": 43.222801208496094,
      "activations/layer9_attention_weight_min": -46.58907699584961,
      "epoch": 15.1,
      "learning_rate": 5.318749999999999e-05,
      "loss": 2.7442,
      "step": 259800
    },
    {
      "activations/layer0_attention_weight_max": 15.642889976501465,
      "activations/layer0_attention_weight_min": -13.799346923828125,
      "activations/layer10_attention_weight_max": 34.81751251220703,
      "activations/layer10_attention_weight_min": -35.603431701660156,
      "activations/layer11_attention_weight_max": 32.239776611328125,
      "activations/layer11_attention_weight_min": -33.535804748535156,
      "activations/layer12_attention_weight_max": 19.028024673461914,
      "activations/layer12_attention_weight_min": -25.518930435180664,
      "activations/layer13_attention_weight_max": 35.22908401489258,
      "activations/layer13_attention_weight_min": -29.829479217529297,
      "activations/layer14_attention_weight_max": 31.60567855834961,
      "activations/layer14_attention_weight_min": -30.230573654174805,
      "activations/layer15_attention_weight_max": 30.136594772338867,
      "activations/layer15_attention_weight_min": -29.95320701599121,
      "activations/layer16_attention_weight_max": 30.42523193359375,
      "activations/layer16_attention_weight_min": -30.402685165405273,
      "activations/layer17_attention_weight_max": 46.66896438598633,
      "activations/layer17_attention_weight_min": -45.00511169433594,
      "activations/layer18_attention_weight_max": 41.611629486083984,
      "activations/layer18_attention_weight_min": -37.66511154174805,
      "activations/layer19_attention_weight_max": 22.205333709716797,
      "activations/layer19_attention_weight_min": -22.87689781188965,
      "activations/layer1_attention_weight_max": 16.86431121826172,
      "activations/layer1_attention_weight_min": -15.032220840454102,
      "activations/layer20_attention_weight_max": 20.99842643737793,
      "activations/layer20_attention_weight_min": -21.795127868652344,
      "activations/layer21_attention_weight_max": 34.507076263427734,
      "activations/layer21_attention_weight_min": -21.8160343170166,
      "activations/layer22_attention_weight_max": 27.21616554260254,
      "activations/layer22_attention_weight_min": -26.181421279907227,
      "activations/layer23_attention_weight_max": 34.218204498291016,
      "activations/layer23_attention_weight_min": -26.277915954589844,
      "activations/layer2_attention_weight_max": 32.86656951904297,
      "activations/layer2_attention_weight_min": -30.75400161743164,
      "activations/layer3_attention_weight_max": 96.10037231445312,
      "activations/layer3_attention_weight_min": -94.3415756225586,
      "activations/layer4_attention_weight_max": 92.72173309326172,
      "activations/layer4_attention_weight_min": -89.31717681884766,
      "activations/layer5_attention_weight_max": 69.57625579833984,
      "activations/layer5_attention_weight_min": -74.3663101196289,
      "activations/layer6_attention_weight_max": 51.15101623535156,
      "activations/layer6_attention_weight_min": -51.60446548461914,
      "activations/layer7_attention_weight_max": 63.40269470214844,
      "activations/layer7_attention_weight_min": -61.08556365966797,
      "activations/layer8_attention_weight_max": 47.49888610839844,
      "activations/layer8_attention_weight_min": -51.14360427856445,
      "activations/layer9_attention_weight_max": 43.66422653198242,
      "activations/layer9_attention_weight_min": -44.96933364868164,
      "epoch": 15.1,
      "learning_rate": 5.31685606060606e-05,
      "loss": 2.7626,
      "step": 259850
    },
    {
      "activations/layer0_attention_weight_max": 16.519010543823242,
      "activations/layer0_attention_weight_min": -13.791014671325684,
      "activations/layer10_attention_weight_max": 35.11772537231445,
      "activations/layer10_attention_weight_min": -37.5159912109375,
      "activations/layer11_attention_weight_max": 31.810104370117188,
      "activations/layer11_attention_weight_min": -33.008460998535156,
      "activations/layer12_attention_weight_max": 18.28596305847168,
      "activations/layer12_attention_weight_min": -25.615018844604492,
      "activations/layer13_attention_weight_max": 34.365264892578125,
      "activations/layer13_attention_weight_min": -29.591075897216797,
      "activations/layer14_attention_weight_max": 35.9517936706543,
      "activations/layer14_attention_weight_min": -29.185590744018555,
      "activations/layer15_attention_weight_max": 32.06477355957031,
      "activations/layer15_attention_weight_min": -27.295602798461914,
      "activations/layer16_attention_weight_max": 31.965303421020508,
      "activations/layer16_attention_weight_min": -28.978992462158203,
      "activations/layer17_attention_weight_max": 51.63090133666992,
      "activations/layer17_attention_weight_min": -43.4179573059082,
      "activations/layer18_attention_weight_max": 45.370304107666016,
      "activations/layer18_attention_weight_min": -41.11747741699219,
      "activations/layer19_attention_weight_max": 23.021448135375977,
      "activations/layer19_attention_weight_min": -23.68509864807129,
      "activations/layer1_attention_weight_max": 17.509916305541992,
      "activations/layer1_attention_weight_min": -14.527328491210938,
      "activations/layer20_attention_weight_max": 22.180585861206055,
      "activations/layer20_attention_weight_min": -23.107379913330078,
      "activations/layer21_attention_weight_max": 34.582176208496094,
      "activations/layer21_attention_weight_min": -24.359416961669922,
      "activations/layer22_attention_weight_max": 25.970666885375977,
      "activations/layer22_attention_weight_min": -24.94943618774414,
      "activations/layer23_attention_weight_max": 33.38753128051758,
      "activations/layer23_attention_weight_min": -24.27991485595703,
      "activations/layer2_attention_weight_max": 34.680198669433594,
      "activations/layer2_attention_weight_min": -33.317813873291016,
      "activations/layer3_attention_weight_max": 94.86146545410156,
      "activations/layer3_attention_weight_min": -98.9803237915039,
      "activations/layer4_attention_weight_max": 94.60013580322266,
      "activations/layer4_attention_weight_min": -89.8742904663086,
      "activations/layer5_attention_weight_max": 63.97016525268555,
      "activations/layer5_attention_weight_min": -74.25308227539062,
      "activations/layer6_attention_weight_max": 48.714820861816406,
      "activations/layer6_attention_weight_min": -54.32307052612305,
      "activations/layer7_attention_weight_max": 58.71317672729492,
      "activations/layer7_attention_weight_min": -66.55001068115234,
      "activations/layer8_attention_weight_max": 47.29457473754883,
      "activations/layer8_attention_weight_min": -52.63302230834961,
      "activations/layer9_attention_weight_max": 47.45335006713867,
      "activations/layer9_attention_weight_min": -53.21612548828125,
      "epoch": 15.1,
      "learning_rate": 5.314962121212121e-05,
      "loss": 2.736,
      "step": 259900
    },
    {
      "activations/layer0_attention_weight_max": 15.364686012268066,
      "activations/layer0_attention_weight_min": -13.79391098022461,
      "activations/layer10_attention_weight_max": 36.70711135864258,
      "activations/layer10_attention_weight_min": -35.273162841796875,
      "activations/layer11_attention_weight_max": 33.79502487182617,
      "activations/layer11_attention_weight_min": -32.31319808959961,
      "activations/layer12_attention_weight_max": 18.233449935913086,
      "activations/layer12_attention_weight_min": -24.74634552001953,
      "activations/layer13_attention_weight_max": 36.4117431640625,
      "activations/layer13_attention_weight_min": -28.48157501220703,
      "activations/layer14_attention_weight_max": 36.210411071777344,
      "activations/layer14_attention_weight_min": -29.702869415283203,
      "activations/layer15_attention_weight_max": 31.93669891357422,
      "activations/layer15_attention_weight_min": -28.788618087768555,
      "activations/layer16_attention_weight_max": 33.951011657714844,
      "activations/layer16_attention_weight_min": -29.335308074951172,
      "activations/layer17_attention_weight_max": 52.926387786865234,
      "activations/layer17_attention_weight_min": -41.53861618041992,
      "activations/layer18_attention_weight_max": 50.835731506347656,
      "activations/layer18_attention_weight_min": -36.66410446166992,
      "activations/layer19_attention_weight_max": 23.04128074645996,
      "activations/layer19_attention_weight_min": -22.021562576293945,
      "activations/layer1_attention_weight_max": 16.580646514892578,
      "activations/layer1_attention_weight_min": -14.339030265808105,
      "activations/layer20_attention_weight_max": 22.78399085998535,
      "activations/layer20_attention_weight_min": -23.29888153076172,
      "activations/layer21_attention_weight_max": 36.69585037231445,
      "activations/layer21_attention_weight_min": -23.95428466796875,
      "activations/layer22_attention_weight_max": 31.33814239501953,
      "activations/layer22_attention_weight_min": -25.712244033813477,
      "activations/layer23_attention_weight_max": 44.58396530151367,
      "activations/layer23_attention_weight_min": -29.89954948425293,
      "activations/layer2_attention_weight_max": 31.381671905517578,
      "activations/layer2_attention_weight_min": -30.994619369506836,
      "activations/layer3_attention_weight_max": 91.36373901367188,
      "activations/layer3_attention_weight_min": -93.47500610351562,
      "activations/layer4_attention_weight_max": 90.21172332763672,
      "activations/layer4_attention_weight_min": -91.65587615966797,
      "activations/layer5_attention_weight_max": 66.76457977294922,
      "activations/layer5_attention_weight_min": -77.60334777832031,
      "activations/layer6_attention_weight_max": 52.740047454833984,
      "activations/layer6_attention_weight_min": -52.03221893310547,
      "activations/layer7_attention_weight_max": 63.77946090698242,
      "activations/layer7_attention_weight_min": -63.142845153808594,
      "activations/layer8_attention_weight_max": 46.97768020629883,
      "activations/layer8_attention_weight_min": -48.83574295043945,
      "activations/layer9_attention_weight_max": 45.745826721191406,
      "activations/layer9_attention_weight_min": -46.817138671875,
      "epoch": 15.1,
      "learning_rate": 5.313068181818181e-05,
      "loss": 2.7385,
      "step": 259950
    },
    {
      "activations/layer0_attention_weight_max": 15.005632400512695,
      "activations/layer0_attention_weight_min": -13.851737022399902,
      "activations/layer10_attention_weight_max": 40.60041427612305,
      "activations/layer10_attention_weight_min": -41.923728942871094,
      "activations/layer11_attention_weight_max": 35.288414001464844,
      "activations/layer11_attention_weight_min": -38.27254867553711,
      "activations/layer12_attention_weight_max": 20.05885124206543,
      "activations/layer12_attention_weight_min": -26.114412307739258,
      "activations/layer13_attention_weight_max": 47.60824203491211,
      "activations/layer13_attention_weight_min": -33.01516342163086,
      "activations/layer14_attention_weight_max": 54.189701080322266,
      "activations/layer14_attention_weight_min": -35.528343200683594,
      "activations/layer15_attention_weight_max": 53.001766204833984,
      "activations/layer15_attention_weight_min": -32.35612106323242,
      "activations/layer16_attention_weight_max": 37.19243240356445,
      "activations/layer16_attention_weight_min": -36.128440856933594,
      "activations/layer17_attention_weight_max": 55.52811813354492,
      "activations/layer17_attention_weight_min": -57.06380844116211,
      "activations/layer18_attention_weight_max": 51.61130905151367,
      "activations/layer18_attention_weight_min": -51.63807678222656,
      "activations/layer19_attention_weight_max": 27.721881866455078,
      "activations/layer19_attention_weight_min": -29.269662857055664,
      "activations/layer1_attention_weight_max": 17.391008377075195,
      "activations/layer1_attention_weight_min": -17.120622634887695,
      "activations/layer20_attention_weight_max": 26.666385650634766,
      "activations/layer20_attention_weight_min": -25.067150115966797,
      "activations/layer21_attention_weight_max": 45.820701599121094,
      "activations/layer21_attention_weight_min": -29.577285766601562,
      "activations/layer22_attention_weight_max": 32.683650970458984,
      "activations/layer22_attention_weight_min": -28.44179916381836,
      "activations/layer23_attention_weight_max": 39.35818862915039,
      "activations/layer23_attention_weight_min": -26.03498077392578,
      "activations/layer2_attention_weight_max": 33.51494216918945,
      "activations/layer2_attention_weight_min": -33.121368408203125,
      "activations/layer3_attention_weight_max": 93.88105773925781,
      "activations/layer3_attention_weight_min": -95.89117431640625,
      "activations/layer4_attention_weight_max": 96.48643493652344,
      "activations/layer4_attention_weight_min": -96.57067108154297,
      "activations/layer5_attention_weight_max": 64.55703735351562,
      "activations/layer5_attention_weight_min": -77.69728088378906,
      "activations/layer6_attention_weight_max": 52.13713836669922,
      "activations/layer6_attention_weight_min": -54.669071197509766,
      "activations/layer7_attention_weight_max": 63.62521743774414,
      "activations/layer7_attention_weight_min": -65.0776138305664,
      "activations/layer8_attention_weight_max": 49.91133499145508,
      "activations/layer8_attention_weight_min": -52.82176971435547,
      "activations/layer9_attention_weight_max": 53.148468017578125,
      "activations/layer9_attention_weight_min": -51.8587646484375,
      "epoch": 15.11,
      "learning_rate": 5.3111742424242425e-05,
      "loss": 2.754,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.5334,
      "eval_samples_per_second": 503.197,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.5334,
      "eval_openwebtext_samples_per_second": 503.197,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_wikitext_loss": 2.9453125,
      "eval_wikitext_ppl": 19.016604146815915,
      "eval_wikitext_runtime": 1.9339,
      "eval_wikitext_samples_per_second": 235.794,
      "step": 260000
    },
    {
      "epoch": 15.11,
      "eval_lambada_loss": 2.478515625,
      "eval_lambada_ppl": 11.923552252478798,
      "eval_lambada_runtime": 9.5262,
      "eval_lambada_samples_per_second": 511.119,
      "step": 260000
    },
    {
      "activations/layer0_attention_weight_max": 15.462268829345703,
      "activations/layer0_attention_weight_min": -13.754716873168945,
      "activations/layer10_attention_weight_max": 35.57664108276367,
      "activations/layer10_attention_weight_min": -32.43482971191406,
      "activations/layer11_attention_weight_max": 31.212650299072266,
      "activations/layer11_attention_weight_min": -31.128629684448242,
      "activations/layer12_attention_weight_max": 18.36065101623535,
      "activations/layer12_attention_weight_min": -25.333646774291992,
      "activations/layer13_attention_weight_max": 40.092796325683594,
      "activations/layer13_attention_weight_min": -28.774761199951172,
      "activations/layer14_attention_weight_max": 43.81184005737305,
      "activations/layer14_attention_weight_min": -29.973234176635742,
      "activations/layer15_attention_weight_max": 39.06144332885742,
      "activations/layer15_attention_weight_min": -29.82513999938965,
      "activations/layer16_attention_weight_max": 36.46909713745117,
      "activations/layer16_attention_weight_min": -32.03380584716797,
      "activations/layer17_attention_weight_max": 60.16392135620117,
      "activations/layer17_attention_weight_min": -47.76008605957031,
      "activations/layer18_attention_weight_max": 55.255916595458984,
      "activations/layer18_attention_weight_min": -42.32026290893555,
      "activations/layer19_attention_weight_max": 26.84554100036621,
      "activations/layer19_attention_weight_min": -27.685998916625977,
      "activations/layer1_attention_weight_max": 16.486848831176758,
      "activations/layer1_attention_weight_min": -14.69930648803711,
      "activations/layer20_attention_weight_max": 31.24992561340332,
      "activations/layer20_attention_weight_min": -22.698585510253906,
      "activations/layer21_attention_weight_max": 40.72602081298828,
      "activations/layer21_attention_weight_min": -24.631912231445312,
      "activations/layer22_attention_weight_max": 36.70352554321289,
      "activations/layer22_attention_weight_min": -24.66367530822754,
      "activations/layer23_attention_weight_max": 39.67778778076172,
      "activations/layer23_attention_weight_min": -23.502273559570312,
      "activations/layer2_attention_weight_max": 34.213558197021484,
      "activations/layer2_attention_weight_min": -32.849212646484375,
      "activations/layer3_attention_weight_max": 93.46873474121094,
      "activations/layer3_attention_weight_min": -89.43128967285156,
      "activations/layer4_attention_weight_max": 94.0401840209961,
      "activations/layer4_attention_weight_min": -85.99414825439453,
      "activations/layer5_attention_weight_max": 62.65346908569336,
      "activations/layer5_attention_weight_min": -75.73109436035156,
      "activations/layer6_attention_weight_max": 48.541446685791016,
      "activations/layer6_attention_weight_min": -47.047977447509766,
      "activations/layer7_attention_weight_max": 63.09738540649414,
      "activations/layer7_attention_weight_min": -56.272422790527344,
      "activations/layer8_attention_weight_max": 43.8922004699707,
      "activations/layer8_attention_weight_min": -43.83658218383789,
      "activations/layer9_attention_weight_max": 47.164634704589844,
      "activations/layer9_attention_weight_min": -43.630252838134766,
      "epoch": 15.11,
      "learning_rate": 5.3092803030303026e-05,
      "loss": 2.7519,
      "step": 260050
    },
    {
      "activations/layer0_attention_weight_max": 15.099713325500488,
      "activations/layer0_attention_weight_min": -14.152828216552734,
      "activations/layer10_attention_weight_max": 36.89943313598633,
      "activations/layer10_attention_weight_min": -37.096866607666016,
      "activations/layer11_attention_weight_max": 33.68897247314453,
      "activations/layer11_attention_weight_min": -32.844947814941406,
      "activations/layer12_attention_weight_max": 19.252347946166992,
      "activations/layer12_attention_weight_min": -39.33353805541992,
      "activations/layer13_attention_weight_max": 34.73466110229492,
      "activations/layer13_attention_weight_min": -28.46819496154785,
      "activations/layer14_attention_weight_max": 35.65656280517578,
      "activations/layer14_attention_weight_min": -30.956565856933594,
      "activations/layer15_attention_weight_max": 32.414546966552734,
      "activations/layer15_attention_weight_min": -29.642770767211914,
      "activations/layer16_attention_weight_max": 33.39052200317383,
      "activations/layer16_attention_weight_min": -29.89751625061035,
      "activations/layer17_attention_weight_max": 52.74848175048828,
      "activations/layer17_attention_weight_min": -42.402427673339844,
      "activations/layer18_attention_weight_max": 46.37202835083008,
      "activations/layer18_attention_weight_min": -39.294036865234375,
      "activations/layer19_attention_weight_max": 25.419599533081055,
      "activations/layer19_attention_weight_min": -21.5922908782959,
      "activations/layer1_attention_weight_max": 17.91437339782715,
      "activations/layer1_attention_weight_min": -16.266441345214844,
      "activations/layer20_attention_weight_max": 24.49492073059082,
      "activations/layer20_attention_weight_min": -24.44964599609375,
      "activations/layer21_attention_weight_max": 35.924171447753906,
      "activations/layer21_attention_weight_min": -22.559345245361328,
      "activations/layer22_attention_weight_max": 31.12118911743164,
      "activations/layer22_attention_weight_min": -27.25423240661621,
      "activations/layer23_attention_weight_max": 39.56560134887695,
      "activations/layer23_attention_weight_min": -25.292739868164062,
      "activations/layer2_attention_weight_max": 32.41695022583008,
      "activations/layer2_attention_weight_min": -31.516799926757812,
      "activations/layer3_attention_weight_max": 94.52293395996094,
      "activations/layer3_attention_weight_min": -92.97554016113281,
      "activations/layer4_attention_weight_max": 90.55184173583984,
      "activations/layer4_attention_weight_min": -88.7137451171875,
      "activations/layer5_attention_weight_max": 69.01570129394531,
      "activations/layer5_attention_weight_min": -74.58042907714844,
      "activations/layer6_attention_weight_max": 49.42586135864258,
      "activations/layer6_attention_weight_min": -51.44493865966797,
      "activations/layer7_attention_weight_max": 68.61703491210938,
      "activations/layer7_attention_weight_min": -58.47969436645508,
      "activations/layer8_attention_weight_max": 45.132869720458984,
      "activations/layer8_attention_weight_min": -47.812076568603516,
      "activations/layer9_attention_weight_max": 52.922027587890625,
      "activations/layer9_attention_weight_min": -48.36957931518555,
      "epoch": 15.11,
      "learning_rate": 5.307386363636363e-05,
      "loss": 2.7458,
      "step": 260100
    },
    {
      "activations/layer0_attention_weight_max": 15.288381576538086,
      "activations/layer0_attention_weight_min": -13.840933799743652,
      "activations/layer10_attention_weight_max": 39.648075103759766,
      "activations/layer10_attention_weight_min": -39.387306213378906,
      "activations/layer11_attention_weight_max": 39.55689239501953,
      "activations/layer11_attention_weight_min": -35.9132080078125,
      "activations/layer12_attention_weight_max": 20.008134841918945,
      "activations/layer12_attention_weight_min": -25.4189453125,
      "activations/layer13_attention_weight_max": 56.07686233520508,
      "activations/layer13_attention_weight_min": -32.32750701904297,
      "activations/layer14_attention_weight_max": 48.37942123413086,
      "activations/layer14_attention_weight_min": -33.18803024291992,
      "activations/layer15_attention_weight_max": 35.83049774169922,
      "activations/layer15_attention_weight_min": -31.1789493560791,
      "activations/layer16_attention_weight_max": 31.05497169494629,
      "activations/layer16_attention_weight_min": -30.38013458251953,
      "activations/layer17_attention_weight_max": 50.25847244262695,
      "activations/layer17_attention_weight_min": -45.48939895629883,
      "activations/layer18_attention_weight_max": 48.6248664855957,
      "activations/layer18_attention_weight_min": -43.186519622802734,
      "activations/layer19_attention_weight_max": 21.507909774780273,
      "activations/layer19_attention_weight_min": -21.93905258178711,
      "activations/layer1_attention_weight_max": 15.613240242004395,
      "activations/layer1_attention_weight_min": -16.668739318847656,
      "activations/layer20_attention_weight_max": 22.512229919433594,
      "activations/layer20_attention_weight_min": -22.390769958496094,
      "activations/layer21_attention_weight_max": 33.96284484863281,
      "activations/layer21_attention_weight_min": -22.61341094970703,
      "activations/layer22_attention_weight_max": 28.7701473236084,
      "activations/layer22_attention_weight_min": -26.670549392700195,
      "activations/layer23_attention_weight_max": 37.665313720703125,
      "activations/layer23_attention_weight_min": -24.67196273803711,
      "activations/layer2_attention_weight_max": 31.40912437438965,
      "activations/layer2_attention_weight_min": -31.86061668395996,
      "activations/layer3_attention_weight_max": 94.12835693359375,
      "activations/layer3_attention_weight_min": -95.20295715332031,
      "activations/layer4_attention_weight_max": 92.71154022216797,
      "activations/layer4_attention_weight_min": -89.57149505615234,
      "activations/layer5_attention_weight_max": 65.7332763671875,
      "activations/layer5_attention_weight_min": -75.84204864501953,
      "activations/layer6_attention_weight_max": 47.81052017211914,
      "activations/layer6_attention_weight_min": -48.93254470825195,
      "activations/layer7_attention_weight_max": 69.17930603027344,
      "activations/layer7_attention_weight_min": -64.00824737548828,
      "activations/layer8_attention_weight_max": 50.78310775756836,
      "activations/layer8_attention_weight_min": -46.85221862792969,
      "activations/layer9_attention_weight_max": 57.711421966552734,
      "activations/layer9_attention_weight_min": -45.74470138549805,
      "epoch": 15.12,
      "learning_rate": 5.3055303030303025e-05,
      "loss": 2.7335,
      "step": 260150
    },
    {
      "activations/layer0_attention_weight_max": 16.014522552490234,
      "activations/layer0_attention_weight_min": -13.735830307006836,
      "activations/layer10_attention_weight_max": 33.87723159790039,
      "activations/layer10_attention_weight_min": -33.702049255371094,
      "activations/layer11_attention_weight_max": 29.452342987060547,
      "activations/layer11_attention_weight_min": -33.405487060546875,
      "activations/layer12_attention_weight_max": 18.65141487121582,
      "activations/layer12_attention_weight_min": -27.551467895507812,
      "activations/layer13_attention_weight_max": 33.49409103393555,
      "activations/layer13_attention_weight_min": -27.128767013549805,
      "activations/layer14_attention_weight_max": 34.360206604003906,
      "activations/layer14_attention_weight_min": -30.72524642944336,
      "activations/layer15_attention_weight_max": 31.509601593017578,
      "activations/layer15_attention_weight_min": -28.615060806274414,
      "activations/layer16_attention_weight_max": 32.62616729736328,
      "activations/layer16_attention_weight_min": -32.12266540527344,
      "activations/layer17_attention_weight_max": 48.945274353027344,
      "activations/layer17_attention_weight_min": -43.568115234375,
      "activations/layer18_attention_weight_max": 45.53995895385742,
      "activations/layer18_attention_weight_min": -37.790767669677734,
      "activations/layer19_attention_weight_max": 22.06531524658203,
      "activations/layer19_attention_weight_min": -22.748451232910156,
      "activations/layer1_attention_weight_max": 17.461448669433594,
      "activations/layer1_attention_weight_min": -14.732348442077637,
      "activations/layer20_attention_weight_max": 22.035552978515625,
      "activations/layer20_attention_weight_min": -22.771360397338867,
      "activations/layer21_attention_weight_max": 33.8563117980957,
      "activations/layer21_attention_weight_min": -23.778959274291992,
      "activations/layer22_attention_weight_max": 30.098018646240234,
      "activations/layer22_attention_weight_min": -25.8519229888916,
      "activations/layer23_attention_weight_max": 35.12120819091797,
      "activations/layer23_attention_weight_min": -24.43661880493164,
      "activations/layer2_attention_weight_max": 34.801204681396484,
      "activations/layer2_attention_weight_min": -31.68840789794922,
      "activations/layer3_attention_weight_max": 91.15101623535156,
      "activations/layer3_attention_weight_min": -97.2404556274414,
      "activations/layer4_attention_weight_max": 91.3871841430664,
      "activations/layer4_attention_weight_min": -89.0515365600586,
      "activations/layer5_attention_weight_max": 65.37434387207031,
      "activations/layer5_attention_weight_min": -72.26499938964844,
      "activations/layer6_attention_weight_max": 51.791053771972656,
      "activations/layer6_attention_weight_min": -50.18800735473633,
      "activations/layer7_attention_weight_max": 61.188026428222656,
      "activations/layer7_attention_weight_min": -59.248077392578125,
      "activations/layer8_attention_weight_max": 41.2426872253418,
      "activations/layer8_attention_weight_min": -46.61531448364258,
      "activations/layer9_attention_weight_max": 41.78672790527344,
      "activations/layer9_attention_weight_min": -43.643310546875,
      "epoch": 15.12,
      "learning_rate": 5.3036363636363634e-05,
      "loss": 2.7457,
      "step": 260200
    },
    {
      "activations/layer0_attention_weight_max": 16.491018295288086,
      "activations/layer0_attention_weight_min": -14.10417652130127,
      "activations/layer10_attention_weight_max": 35.12849426269531,
      "activations/layer10_attention_weight_min": -34.401710510253906,
      "activations/layer11_attention_weight_max": 33.50857925415039,
      "activations/layer11_attention_weight_min": -33.02429962158203,
      "activations/layer12_attention_weight_max": 17.5302677154541,
      "activations/layer12_attention_weight_min": -24.632261276245117,
      "activations/layer13_attention_weight_max": 41.82476806640625,
      "activations/layer13_attention_weight_min": -27.85546875,
      "activations/layer14_attention_weight_max": 42.875755310058594,
      "activations/layer14_attention_weight_min": -29.018718719482422,
      "activations/layer15_attention_weight_max": 37.43171310424805,
      "activations/layer15_attention_weight_min": -27.804729461669922,
      "activations/layer16_attention_weight_max": 33.9698600769043,
      "activations/layer16_attention_weight_min": -31.387508392333984,
      "activations/layer17_attention_weight_max": 51.33491134643555,
      "activations/layer17_attention_weight_min": -49.70639419555664,
      "activations/layer18_attention_weight_max": 48.22529602050781,
      "activations/layer18_attention_weight_min": -43.59531021118164,
      "activations/layer19_attention_weight_max": 24.49500274658203,
      "activations/layer19_attention_weight_min": -25.556623458862305,
      "activations/layer1_attention_weight_max": 16.360170364379883,
      "activations/layer1_attention_weight_min": -14.516843795776367,
      "activations/layer20_attention_weight_max": 24.884370803833008,
      "activations/layer20_attention_weight_min": -23.214941024780273,
      "activations/layer21_attention_weight_max": 38.55360794067383,
      "activations/layer21_attention_weight_min": -22.123703002929688,
      "activations/layer22_attention_weight_max": 32.39377975463867,
      "activations/layer22_attention_weight_min": -26.960725784301758,
      "activations/layer23_attention_weight_max": 35.0728759765625,
      "activations/layer23_attention_weight_min": -27.663101196289062,
      "activations/layer2_attention_weight_max": 32.43537902832031,
      "activations/layer2_attention_weight_min": -30.877742767333984,
      "activations/layer3_attention_weight_max": 94.89295959472656,
      "activations/layer3_attention_weight_min": -92.6072006225586,
      "activations/layer4_attention_weight_max": 93.10794067382812,
      "activations/layer4_attention_weight_min": -85.11225128173828,
      "activations/layer5_attention_weight_max": 61.91459655761719,
      "activations/layer5_attention_weight_min": -71.67684936523438,
      "activations/layer6_attention_weight_max": 46.42319107055664,
      "activations/layer6_attention_weight_min": -47.605247497558594,
      "activations/layer7_attention_weight_max": 60.91428756713867,
      "activations/layer7_attention_weight_min": -57.29022979736328,
      "activations/layer8_attention_weight_max": 45.85757827758789,
      "activations/layer8_attention_weight_min": -45.9975700378418,
      "activations/layer9_attention_weight_max": 51.92971420288086,
      "activations/layer9_attention_weight_min": -43.309539794921875,
      "epoch": 15.12,
      "learning_rate": 5.301742424242424e-05,
      "loss": 2.7349,
      "step": 260250
    },
    {
      "activations/layer0_attention_weight_max": 15.315299987792969,
      "activations/layer0_attention_weight_min": -13.754583358764648,
      "activations/layer10_attention_weight_max": 37.22264099121094,
      "activations/layer10_attention_weight_min": -36.67975616455078,
      "activations/layer11_attention_weight_max": 31.364179611206055,
      "activations/layer11_attention_weight_min": -34.979957580566406,
      "activations/layer12_attention_weight_max": 19.445737838745117,
      "activations/layer12_attention_weight_min": -23.86810874938965,
      "activations/layer13_attention_weight_max": 33.712806701660156,
      "activations/layer13_attention_weight_min": -27.404682159423828,
      "activations/layer14_attention_weight_max": 33.93976593017578,
      "activations/layer14_attention_weight_min": -30.219141006469727,
      "activations/layer15_attention_weight_max": 30.41538429260254,
      "activations/layer15_attention_weight_min": -29.37787437438965,
      "activations/layer16_attention_weight_max": 32.676692962646484,
      "activations/layer16_attention_weight_min": -31.84497833251953,
      "activations/layer17_attention_weight_max": 51.19821548461914,
      "activations/layer17_attention_weight_min": -42.297611236572266,
      "activations/layer18_attention_weight_max": 44.216163635253906,
      "activations/layer18_attention_weight_min": -37.28775405883789,
      "activations/layer19_attention_weight_max": 22.738683700561523,
      "activations/layer19_attention_weight_min": -22.335874557495117,
      "activations/layer1_attention_weight_max": 16.347488403320312,
      "activations/layer1_attention_weight_min": -14.390326499938965,
      "activations/layer20_attention_weight_max": 22.15353012084961,
      "activations/layer20_attention_weight_min": -22.154882431030273,
      "activations/layer21_attention_weight_max": 37.053497314453125,
      "activations/layer21_attention_weight_min": -21.95766258239746,
      "activations/layer22_attention_weight_max": 29.5150203704834,
      "activations/layer22_attention_weight_min": -26.509572982788086,
      "activations/layer23_attention_weight_max": 34.417762756347656,
      "activations/layer23_attention_weight_min": -24.467388153076172,
      "activations/layer2_attention_weight_max": 29.7392635345459,
      "activations/layer2_attention_weight_min": -29.070499420166016,
      "activations/layer3_attention_weight_max": 89.0929183959961,
      "activations/layer3_attention_weight_min": -90.98714447021484,
      "activations/layer4_attention_weight_max": 94.4047622680664,
      "activations/layer4_attention_weight_min": -90.56368255615234,
      "activations/layer5_attention_weight_max": 68.83931732177734,
      "activations/layer5_attention_weight_min": -82.18841552734375,
      "activations/layer6_attention_weight_max": 52.91428756713867,
      "activations/layer6_attention_weight_min": -55.50482177734375,
      "activations/layer7_attention_weight_max": 66.77401733398438,
      "activations/layer7_attention_weight_min": -62.71162796020508,
      "activations/layer8_attention_weight_max": 45.91969680786133,
      "activations/layer8_attention_weight_min": -51.297706604003906,
      "activations/layer9_attention_weight_max": 45.87095642089844,
      "activations/layer9_attention_weight_min": -45.97154235839844,
      "epoch": 15.12,
      "learning_rate": 5.2998484848484843e-05,
      "loss": 2.7515,
      "step": 260300
    },
    {
      "activations/layer0_attention_weight_max": 15.531770706176758,
      "activations/layer0_attention_weight_min": -13.606959342956543,
      "activations/layer10_attention_weight_max": 37.193336486816406,
      "activations/layer10_attention_weight_min": -38.494903564453125,
      "activations/layer11_attention_weight_max": 34.44524383544922,
      "activations/layer11_attention_weight_min": -33.86511993408203,
      "activations/layer12_attention_weight_max": 17.646631240844727,
      "activations/layer12_attention_weight_min": -25.408714294433594,
      "activations/layer13_attention_weight_max": 35.58331298828125,
      "activations/layer13_attention_weight_min": -27.839982986450195,
      "activations/layer14_attention_weight_max": 40.484458923339844,
      "activations/layer14_attention_weight_min": -28.293670654296875,
      "activations/layer15_attention_weight_max": 33.85091018676758,
      "activations/layer15_attention_weight_min": -27.85387420654297,
      "activations/layer16_attention_weight_max": 33.72251892089844,
      "activations/layer16_attention_weight_min": -30.494518280029297,
      "activations/layer17_attention_weight_max": 54.94072341918945,
      "activations/layer17_attention_weight_min": -42.55363845825195,
      "activations/layer18_attention_weight_max": 50.45996856689453,
      "activations/layer18_attention_weight_min": -37.036415100097656,
      "activations/layer19_attention_weight_max": 24.38145637512207,
      "activations/layer19_attention_weight_min": -22.322431564331055,
      "activations/layer1_attention_weight_max": 18.3579158782959,
      "activations/layer1_attention_weight_min": -15.761561393737793,
      "activations/layer20_attention_weight_max": 28.142009735107422,
      "activations/layer20_attention_weight_min": -22.96196174621582,
      "activations/layer21_attention_weight_max": 49.15232849121094,
      "activations/layer21_attention_weight_min": -21.541101455688477,
      "activations/layer22_attention_weight_max": 35.51002883911133,
      "activations/layer22_attention_weight_min": -26.199434280395508,
      "activations/layer23_attention_weight_max": 42.11353302001953,
      "activations/layer23_attention_weight_min": -22.864776611328125,
      "activations/layer2_attention_weight_max": 30.712797164916992,
      "activations/layer2_attention_weight_min": -30.773120880126953,
      "activations/layer3_attention_weight_max": 96.4873275756836,
      "activations/layer3_attention_weight_min": -94.6360855102539,
      "activations/layer4_attention_weight_max": 90.89512634277344,
      "activations/layer4_attention_weight_min": -90.8062744140625,
      "activations/layer5_attention_weight_max": 67.21442413330078,
      "activations/layer5_attention_weight_min": -70.67268371582031,
      "activations/layer6_attention_weight_max": 48.940650939941406,
      "activations/layer6_attention_weight_min": -52.22761917114258,
      "activations/layer7_attention_weight_max": 62.670982360839844,
      "activations/layer7_attention_weight_min": -63.859893798828125,
      "activations/layer8_attention_weight_max": 45.3325080871582,
      "activations/layer8_attention_weight_min": -49.351810455322266,
      "activations/layer9_attention_weight_max": 52.2943115234375,
      "activations/layer9_attention_weight_min": -49.47602844238281,
      "epoch": 15.13,
      "learning_rate": 5.2979545454545445e-05,
      "loss": 2.738,
      "step": 260350
    },
    {
      "activations/layer0_attention_weight_max": 17.145597457885742,
      "activations/layer0_attention_weight_min": -13.852092742919922,
      "activations/layer10_attention_weight_max": 33.360172271728516,
      "activations/layer10_attention_weight_min": -35.61058044433594,
      "activations/layer11_attention_weight_max": 32.820960998535156,
      "activations/layer11_attention_weight_min": -33.4698371887207,
      "activations/layer12_attention_weight_max": 17.942712783813477,
      "activations/layer12_attention_weight_min": -24.66172218322754,
      "activations/layer13_attention_weight_max": 37.534786224365234,
      "activations/layer13_attention_weight_min": -31.415618896484375,
      "activations/layer14_attention_weight_max": 36.901275634765625,
      "activations/layer14_attention_weight_min": -31.23493003845215,
      "activations/layer15_attention_weight_max": 33.250152587890625,
      "activations/layer15_attention_weight_min": -31.410964965820312,
      "activations/layer16_attention_weight_max": 32.90311050415039,
      "activations/layer16_attention_weight_min": -32.13819885253906,
      "activations/layer17_attention_weight_max": 51.2890625,
      "activations/layer17_attention_weight_min": -47.65767288208008,
      "activations/layer18_attention_weight_max": 44.26861572265625,
      "activations/layer18_attention_weight_min": -42.02015686035156,
      "activations/layer19_attention_weight_max": 22.534902572631836,
      "activations/layer19_attention_weight_min": -22.919384002685547,
      "activations/layer1_attention_weight_max": 16.631216049194336,
      "activations/layer1_attention_weight_min": -16.059741973876953,
      "activations/layer20_attention_weight_max": 23.100902557373047,
      "activations/layer20_attention_weight_min": -20.974172592163086,
      "activations/layer21_attention_weight_max": 36.35750198364258,
      "activations/layer21_attention_weight_min": -24.132230758666992,
      "activations/layer22_attention_weight_max": 29.794872283935547,
      "activations/layer22_attention_weight_min": -25.37989616394043,
      "activations/layer23_attention_weight_max": 39.297264099121094,
      "activations/layer23_attention_weight_min": -24.977336883544922,
      "activations/layer2_attention_weight_max": 31.72353172302246,
      "activations/layer2_attention_weight_min": -31.879316329956055,
      "activations/layer3_attention_weight_max": 93.95943450927734,
      "activations/layer3_attention_weight_min": -98.30632781982422,
      "activations/layer4_attention_weight_max": 94.90892791748047,
      "activations/layer4_attention_weight_min": -93.90311431884766,
      "activations/layer5_attention_weight_max": 70.04877471923828,
      "activations/layer5_attention_weight_min": -73.4417495727539,
      "activations/layer6_attention_weight_max": 54.64802551269531,
      "activations/layer6_attention_weight_min": -50.31149673461914,
      "activations/layer7_attention_weight_max": 69.48094940185547,
      "activations/layer7_attention_weight_min": -62.12766647338867,
      "activations/layer8_attention_weight_max": 45.83082580566406,
      "activations/layer8_attention_weight_min": -50.66524124145508,
      "activations/layer9_attention_weight_max": 49.22993087768555,
      "activations/layer9_attention_weight_min": -48.5430908203125,
      "epoch": 15.13,
      "learning_rate": 5.296060606060606e-05,
      "loss": 2.7379,
      "step": 260400
    },
    {
      "activations/layer0_attention_weight_max": 15.840644836425781,
      "activations/layer0_attention_weight_min": -13.413015365600586,
      "activations/layer10_attention_weight_max": 39.237125396728516,
      "activations/layer10_attention_weight_min": -37.3631591796875,
      "activations/layer11_attention_weight_max": 34.316932678222656,
      "activations/layer11_attention_weight_min": -33.45492172241211,
      "activations/layer12_attention_weight_max": 17.508634567260742,
      "activations/layer12_attention_weight_min": -29.57573890686035,
      "activations/layer13_attention_weight_max": 32.97371292114258,
      "activations/layer13_attention_weight_min": -26.105283737182617,
      "activations/layer14_attention_weight_max": 34.56281280517578,
      "activations/layer14_attention_weight_min": -28.304054260253906,
      "activations/layer15_attention_weight_max": 32.25020980834961,
      "activations/layer15_attention_weight_min": -28.958341598510742,
      "activations/layer16_attention_weight_max": 32.35672378540039,
      "activations/layer16_attention_weight_min": -31.43741798400879,
      "activations/layer17_attention_weight_max": 52.942596435546875,
      "activations/layer17_attention_weight_min": -43.651058197021484,
      "activations/layer18_attention_weight_max": 51.61237335205078,
      "activations/layer18_attention_weight_min": -39.7443962097168,
      "activations/layer19_attention_weight_max": 22.6958065032959,
      "activations/layer19_attention_weight_min": -20.5896053314209,
      "activations/layer1_attention_weight_max": 16.634449005126953,
      "activations/layer1_attention_weight_min": -14.654693603515625,
      "activations/layer20_attention_weight_max": 23.529800415039062,
      "activations/layer20_attention_weight_min": -23.792789459228516,
      "activations/layer21_attention_weight_max": 35.100189208984375,
      "activations/layer21_attention_weight_min": -26.8769588470459,
      "activations/layer22_attention_weight_max": 31.582721710205078,
      "activations/layer22_attention_weight_min": -26.54659080505371,
      "activations/layer23_attention_weight_max": 38.204933166503906,
      "activations/layer23_attention_weight_min": -28.28510093688965,
      "activations/layer2_attention_weight_max": 32.057769775390625,
      "activations/layer2_attention_weight_min": -31.274486541748047,
      "activations/layer3_attention_weight_max": 91.78215026855469,
      "activations/layer3_attention_weight_min": -96.22040557861328,
      "activations/layer4_attention_weight_max": 92.92666625976562,
      "activations/layer4_attention_weight_min": -90.92897033691406,
      "activations/layer5_attention_weight_max": 65.26264953613281,
      "activations/layer5_attention_weight_min": -78.83187103271484,
      "activations/layer6_attention_weight_max": 49.65170669555664,
      "activations/layer6_attention_weight_min": -51.633907318115234,
      "activations/layer7_attention_weight_max": 63.242191314697266,
      "activations/layer7_attention_weight_min": -60.7303466796875,
      "activations/layer8_attention_weight_max": 49.11442184448242,
      "activations/layer8_attention_weight_min": -48.54587936401367,
      "activations/layer9_attention_weight_max": 46.314659118652344,
      "activations/layer9_attention_weight_min": -46.74560546875,
      "epoch": 15.13,
      "learning_rate": 5.294166666666666e-05,
      "loss": 2.7485,
      "step": 260450
    },
    {
      "activations/layer0_attention_weight_max": 15.996502876281738,
      "activations/layer0_attention_weight_min": -13.642621994018555,
      "activations/layer10_attention_weight_max": 37.702972412109375,
      "activations/layer10_attention_weight_min": -39.304969787597656,
      "activations/layer11_attention_weight_max": 34.433815002441406,
      "activations/layer11_attention_weight_min": -36.93450164794922,
      "activations/layer12_attention_weight_max": 24.89263916015625,
      "activations/layer12_attention_weight_min": -23.330299377441406,
      "activations/layer13_attention_weight_max": 41.61705780029297,
      "activations/layer13_attention_weight_min": -27.74850845336914,
      "activations/layer14_attention_weight_max": 38.99906539916992,
      "activations/layer14_attention_weight_min": -29.474740982055664,
      "activations/layer15_attention_weight_max": 37.198665618896484,
      "activations/layer15_attention_weight_min": -29.20125389099121,
      "activations/layer16_attention_weight_max": 36.917842864990234,
      "activations/layer16_attention_weight_min": -31.75640869140625,
      "activations/layer17_attention_weight_max": 59.5527229309082,
      "activations/layer17_attention_weight_min": -45.54620361328125,
      "activations/layer18_attention_weight_max": 52.77709197998047,
      "activations/layer18_attention_weight_min": -41.71247482299805,
      "activations/layer19_attention_weight_max": 25.59787368774414,
      "activations/layer19_attention_weight_min": -23.855802536010742,
      "activations/layer1_attention_weight_max": 16.421905517578125,
      "activations/layer1_attention_weight_min": -15.217817306518555,
      "activations/layer20_attention_weight_max": 24.367942810058594,
      "activations/layer20_attention_weight_min": -22.45207977294922,
      "activations/layer21_attention_weight_max": 46.739036560058594,
      "activations/layer21_attention_weight_min": -25.091604232788086,
      "activations/layer22_attention_weight_max": 30.92302703857422,
      "activations/layer22_attention_weight_min": -23.92483901977539,
      "activations/layer23_attention_weight_max": 43.181297302246094,
      "activations/layer23_attention_weight_min": -24.965322494506836,
      "activations/layer2_attention_weight_max": 31.85402488708496,
      "activations/layer2_attention_weight_min": -30.421916961669922,
      "activations/layer3_attention_weight_max": 96.89429473876953,
      "activations/layer3_attention_weight_min": -95.47288513183594,
      "activations/layer4_attention_weight_max": 95.16602325439453,
      "activations/layer4_attention_weight_min": -99.90292358398438,
      "activations/layer5_attention_weight_max": 66.75753784179688,
      "activations/layer5_attention_weight_min": -76.89202880859375,
      "activations/layer6_attention_weight_max": 52.22966766357422,
      "activations/layer6_attention_weight_min": -54.797462463378906,
      "activations/layer7_attention_weight_max": 65.6247329711914,
      "activations/layer7_attention_weight_min": -64.04871368408203,
      "activations/layer8_attention_weight_max": 50.68503189086914,
      "activations/layer8_attention_weight_min": -52.70226287841797,
      "activations/layer9_attention_weight_max": 46.18050765991211,
      "activations/layer9_attention_weight_min": -48.419429779052734,
      "epoch": 15.14,
      "learning_rate": 5.292272727272727e-05,
      "loss": 2.742,
      "step": 260500
    },
    {
      "activations/layer0_attention_weight_max": 14.61305046081543,
      "activations/layer0_attention_weight_min": -13.711647987365723,
      "activations/layer10_attention_weight_max": 34.882076263427734,
      "activations/layer10_attention_weight_min": -33.5556640625,
      "activations/layer11_attention_weight_max": 32.76318359375,
      "activations/layer11_attention_weight_min": -32.640419006347656,
      "activations/layer12_attention_weight_max": 18.614957809448242,
      "activations/layer12_attention_weight_min": -24.17205238342285,
      "activations/layer13_attention_weight_max": 32.74411392211914,
      "activations/layer13_attention_weight_min": -29.629112243652344,
      "activations/layer14_attention_weight_max": 34.5130615234375,
      "activations/layer14_attention_weight_min": -31.70380210876465,
      "activations/layer15_attention_weight_max": 31.0520076751709,
      "activations/layer15_attention_weight_min": -30.747543334960938,
      "activations/layer16_attention_weight_max": 31.783544540405273,
      "activations/layer16_attention_weight_min": -32.27561569213867,
      "activations/layer17_attention_weight_max": 50.13792419433594,
      "activations/layer17_attention_weight_min": -43.92156219482422,
      "activations/layer18_attention_weight_max": 45.574195861816406,
      "activations/layer18_attention_weight_min": -38.21010971069336,
      "activations/layer19_attention_weight_max": 22.227977752685547,
      "activations/layer19_attention_weight_min": -23.204925537109375,
      "activations/layer1_attention_weight_max": 19.06707191467285,
      "activations/layer1_attention_weight_min": -14.337038040161133,
      "activations/layer20_attention_weight_max": 23.54007339477539,
      "activations/layer20_attention_weight_min": -23.788400650024414,
      "activations/layer21_attention_weight_max": 42.47096252441406,
      "activations/layer21_attention_weight_min": -27.405349731445312,
      "activations/layer22_attention_weight_max": 28.08627700805664,
      "activations/layer22_attention_weight_min": -25.4064998626709,
      "activations/layer23_attention_weight_max": 35.95477294921875,
      "activations/layer23_attention_weight_min": -24.27155113220215,
      "activations/layer2_attention_weight_max": 30.5983943939209,
      "activations/layer2_attention_weight_min": -30.26592254638672,
      "activations/layer3_attention_weight_max": 88.38684844970703,
      "activations/layer3_attention_weight_min": -88.14189147949219,
      "activations/layer4_attention_weight_max": 89.3340072631836,
      "activations/layer4_attention_weight_min": -86.7673568725586,
      "activations/layer5_attention_weight_max": 66.24797058105469,
      "activations/layer5_attention_weight_min": -74.52098083496094,
      "activations/layer6_attention_weight_max": 47.908302307128906,
      "activations/layer6_attention_weight_min": -49.281639099121094,
      "activations/layer7_attention_weight_max": 59.59107208251953,
      "activations/layer7_attention_weight_min": -61.36544418334961,
      "activations/layer8_attention_weight_max": 44.91692352294922,
      "activations/layer8_attention_weight_min": -48.45119857788086,
      "activations/layer9_attention_weight_max": 42.367431640625,
      "activations/layer9_attention_weight_min": -46.90667724609375,
      "epoch": 15.14,
      "learning_rate": 5.290378787878787e-05,
      "loss": 2.759,
      "step": 260550
    },
    {
      "activations/layer0_attention_weight_max": 15.38418197631836,
      "activations/layer0_attention_weight_min": -13.793240547180176,
      "activations/layer10_attention_weight_max": 38.48438262939453,
      "activations/layer10_attention_weight_min": -36.483943939208984,
      "activations/layer11_attention_weight_max": 36.060752868652344,
      "activations/layer11_attention_weight_min": -36.30271911621094,
      "activations/layer12_attention_weight_max": 22.533241271972656,
      "activations/layer12_attention_weight_min": -28.06618881225586,
      "activations/layer13_attention_weight_max": 39.09514617919922,
      "activations/layer13_attention_weight_min": -31.597187042236328,
      "activations/layer14_attention_weight_max": 39.19115447998047,
      "activations/layer14_attention_weight_min": -31.0609188079834,
      "activations/layer15_attention_weight_max": 35.32556915283203,
      "activations/layer15_attention_weight_min": -30.285367965698242,
      "activations/layer16_attention_weight_max": 35.586021423339844,
      "activations/layer16_attention_weight_min": -32.16634750366211,
      "activations/layer17_attention_weight_max": 55.44328689575195,
      "activations/layer17_attention_weight_min": -48.824989318847656,
      "activations/layer18_attention_weight_max": 50.31438446044922,
      "activations/layer18_attention_weight_min": -42.70063018798828,
      "activations/layer19_attention_weight_max": 24.933189392089844,
      "activations/layer19_attention_weight_min": -23.561386108398438,
      "activations/layer1_attention_weight_max": 15.944276809692383,
      "activations/layer1_attention_weight_min": -15.81532096862793,
      "activations/layer20_attention_weight_max": 23.347902297973633,
      "activations/layer20_attention_weight_min": -23.27360725402832,
      "activations/layer21_attention_weight_max": 34.80949783325195,
      "activations/layer21_attention_weight_min": -23.842296600341797,
      "activations/layer22_attention_weight_max": 30.72545051574707,
      "activations/layer22_attention_weight_min": -26.83006477355957,
      "activations/layer23_attention_weight_max": 38.49489974975586,
      "activations/layer23_attention_weight_min": -25.67249298095703,
      "activations/layer2_attention_weight_max": 32.382225036621094,
      "activations/layer2_attention_weight_min": -32.9018669128418,
      "activations/layer3_attention_weight_max": 96.97589874267578,
      "activations/layer3_attention_weight_min": -96.484130859375,
      "activations/layer4_attention_weight_max": 98.41893768310547,
      "activations/layer4_attention_weight_min": -97.56966400146484,
      "activations/layer5_attention_weight_max": 70.6403579711914,
      "activations/layer5_attention_weight_min": -75.26741027832031,
      "activations/layer6_attention_weight_max": 56.52145767211914,
      "activations/layer6_attention_weight_min": -53.59254837036133,
      "activations/layer7_attention_weight_max": 66.03199005126953,
      "activations/layer7_attention_weight_min": -64.09864044189453,
      "activations/layer8_attention_weight_max": 49.996063232421875,
      "activations/layer8_attention_weight_min": -52.35112380981445,
      "activations/layer9_attention_weight_max": 48.761390686035156,
      "activations/layer9_attention_weight_min": -49.45096969604492,
      "epoch": 15.14,
      "learning_rate": 5.288484848484848e-05,
      "loss": 2.7587,
      "step": 260600
    },
    {
      "activations/layer0_attention_weight_max": 15.594291687011719,
      "activations/layer0_attention_weight_min": -13.687138557434082,
      "activations/layer10_attention_weight_max": 37.85472869873047,
      "activations/layer10_attention_weight_min": -36.4032096862793,
      "activations/layer11_attention_weight_max": 34.02808380126953,
      "activations/layer11_attention_weight_min": -33.48597717285156,
      "activations/layer12_attention_weight_max": 17.806365966796875,
      "activations/layer12_attention_weight_min": -24.326948165893555,
      "activations/layer13_attention_weight_max": 34.490989685058594,
      "activations/layer13_attention_weight_min": -27.143945693969727,
      "activations/layer14_attention_weight_max": 35.242645263671875,
      "activations/layer14_attention_weight_min": -31.965042114257812,
      "activations/layer15_attention_weight_max": 31.533689498901367,
      "activations/layer15_attention_weight_min": -29.976367950439453,
      "activations/layer16_attention_weight_max": 32.466495513916016,
      "activations/layer16_attention_weight_min": -31.368595123291016,
      "activations/layer17_attention_weight_max": 50.279624938964844,
      "activations/layer17_attention_weight_min": -46.3152961730957,
      "activations/layer18_attention_weight_max": 45.607154846191406,
      "activations/layer18_attention_weight_min": -42.43778991699219,
      "activations/layer19_attention_weight_max": 22.065776824951172,
      "activations/layer19_attention_weight_min": -24.70689582824707,
      "activations/layer1_attention_weight_max": 17.18537139892578,
      "activations/layer1_attention_weight_min": -16.826276779174805,
      "activations/layer20_attention_weight_max": 21.777456283569336,
      "activations/layer20_attention_weight_min": -22.316892623901367,
      "activations/layer21_attention_weight_max": 33.247066497802734,
      "activations/layer21_attention_weight_min": -22.883108139038086,
      "activations/layer22_attention_weight_max": 28.72911262512207,
      "activations/layer22_attention_weight_min": -27.88167381286621,
      "activations/layer23_attention_weight_max": 39.83916091918945,
      "activations/layer23_attention_weight_min": -25.02051544189453,
      "activations/layer2_attention_weight_max": 32.46527862548828,
      "activations/layer2_attention_weight_min": -31.445106506347656,
      "activations/layer3_attention_weight_max": 90.04328918457031,
      "activations/layer3_attention_weight_min": -90.40370178222656,
      "activations/layer4_attention_weight_max": 89.69762420654297,
      "activations/layer4_attention_weight_min": -90.08921813964844,
      "activations/layer5_attention_weight_max": 66.0341567993164,
      "activations/layer5_attention_weight_min": -73.5351333618164,
      "activations/layer6_attention_weight_max": 48.19136428833008,
      "activations/layer6_attention_weight_min": -49.556419372558594,
      "activations/layer7_attention_weight_max": 61.63142395019531,
      "activations/layer7_attention_weight_min": -58.665164947509766,
      "activations/layer8_attention_weight_max": 47.56110382080078,
      "activations/layer8_attention_weight_min": -50.857177734375,
      "activations/layer9_attention_weight_max": 56.69346237182617,
      "activations/layer9_attention_weight_min": -48.00075149536133,
      "epoch": 15.15,
      "learning_rate": 5.286590909090909e-05,
      "loss": 2.7539,
      "step": 260650
    },
    {
      "activations/layer0_attention_weight_max": 15.709659576416016,
      "activations/layer0_attention_weight_min": -13.607246398925781,
      "activations/layer10_attention_weight_max": 35.496063232421875,
      "activations/layer10_attention_weight_min": -35.47624588012695,
      "activations/layer11_attention_weight_max": 35.14644241333008,
      "activations/layer11_attention_weight_min": -33.198848724365234,
      "activations/layer12_attention_weight_max": 18.859922409057617,
      "activations/layer12_attention_weight_min": -31.354875564575195,
      "activations/layer13_attention_weight_max": 38.75659942626953,
      "activations/layer13_attention_weight_min": -27.013954162597656,
      "activations/layer14_attention_weight_max": 35.92537307739258,
      "activations/layer14_attention_weight_min": -30.628711700439453,
      "activations/layer15_attention_weight_max": 34.41232681274414,
      "activations/layer15_attention_weight_min": -29.775733947753906,
      "activations/layer16_attention_weight_max": 33.61407470703125,
      "activations/layer16_attention_weight_min": -31.93965721130371,
      "activations/layer17_attention_weight_max": 53.21927261352539,
      "activations/layer17_attention_weight_min": -51.40890884399414,
      "activations/layer18_attention_weight_max": 47.039676666259766,
      "activations/layer18_attention_weight_min": -42.55006790161133,
      "activations/layer19_attention_weight_max": 26.81013298034668,
      "activations/layer19_attention_weight_min": -23.006086349487305,
      "activations/layer1_attention_weight_max": 16.80092430114746,
      "activations/layer1_attention_weight_min": -14.138829231262207,
      "activations/layer20_attention_weight_max": 26.235300064086914,
      "activations/layer20_attention_weight_min": -21.78702163696289,
      "activations/layer21_attention_weight_max": 38.2116813659668,
      "activations/layer21_attention_weight_min": -22.19703483581543,
      "activations/layer22_attention_weight_max": 30.48625946044922,
      "activations/layer22_attention_weight_min": -25.098453521728516,
      "activations/layer23_attention_weight_max": 37.64629364013672,
      "activations/layer23_attention_weight_min": -26.658050537109375,
      "activations/layer2_attention_weight_max": 32.365272521972656,
      "activations/layer2_attention_weight_min": -29.634681701660156,
      "activations/layer3_attention_weight_max": 98.3246078491211,
      "activations/layer3_attention_weight_min": -98.48409271240234,
      "activations/layer4_attention_weight_max": 93.22622680664062,
      "activations/layer4_attention_weight_min": -90.17890167236328,
      "activations/layer5_attention_weight_max": 65.95983123779297,
      "activations/layer5_attention_weight_min": -75.38066101074219,
      "activations/layer6_attention_weight_max": 47.99862289428711,
      "activations/layer6_attention_weight_min": -48.69804000854492,
      "activations/layer7_attention_weight_max": 64.1875991821289,
      "activations/layer7_attention_weight_min": -58.867576599121094,
      "activations/layer8_attention_weight_max": 46.87545394897461,
      "activations/layer8_attention_weight_min": -49.854034423828125,
      "activations/layer9_attention_weight_max": 52.24734115600586,
      "activations/layer9_attention_weight_min": -49.00661849975586,
      "epoch": 15.15,
      "learning_rate": 5.284696969696969e-05,
      "loss": 2.7422,
      "step": 260700
    },
    {
      "activations/layer0_attention_weight_max": 16.140947341918945,
      "activations/layer0_attention_weight_min": -14.011545181274414,
      "activations/layer10_attention_weight_max": 36.7915153503418,
      "activations/layer10_attention_weight_min": -36.0759162902832,
      "activations/layer11_attention_weight_max": 32.954498291015625,
      "activations/layer11_attention_weight_min": -32.149864196777344,
      "activations/layer12_attention_weight_max": 21.455001831054688,
      "activations/layer12_attention_weight_min": -23.45635223388672,
      "activations/layer13_attention_weight_max": 34.505157470703125,
      "activations/layer13_attention_weight_min": -29.042787551879883,
      "activations/layer14_attention_weight_max": 36.28501892089844,
      "activations/layer14_attention_weight_min": -28.815258026123047,
      "activations/layer15_attention_weight_max": 33.98935317993164,
      "activations/layer15_attention_weight_min": -28.742883682250977,
      "activations/layer16_attention_weight_max": 33.230655670166016,
      "activations/layer16_attention_weight_min": -29.12948226928711,
      "activations/layer17_attention_weight_max": 55.403656005859375,
      "activations/layer17_attention_weight_min": -43.80801773071289,
      "activations/layer18_attention_weight_max": 46.94377899169922,
      "activations/layer18_attention_weight_min": -38.52751159667969,
      "activations/layer19_attention_weight_max": 23.3570613861084,
      "activations/layer19_attention_weight_min": -21.358449935913086,
      "activations/layer1_attention_weight_max": 17.662498474121094,
      "activations/layer1_attention_weight_min": -15.49721622467041,
      "activations/layer20_attention_weight_max": 21.489885330200195,
      "activations/layer20_attention_weight_min": -21.814048767089844,
      "activations/layer21_attention_weight_max": 32.034820556640625,
      "activations/layer21_attention_weight_min": -21.5173282623291,
      "activations/layer22_attention_weight_max": 26.9962100982666,
      "activations/layer22_attention_weight_min": -25.33378028869629,
      "activations/layer23_attention_weight_max": 37.194942474365234,
      "activations/layer23_attention_weight_min": -24.451025009155273,
      "activations/layer2_attention_weight_max": 31.23681640625,
      "activations/layer2_attention_weight_min": -30.168628692626953,
      "activations/layer3_attention_weight_max": 90.74718475341797,
      "activations/layer3_attention_weight_min": -90.79733276367188,
      "activations/layer4_attention_weight_max": 90.6978530883789,
      "activations/layer4_attention_weight_min": -91.61096954345703,
      "activations/layer5_attention_weight_max": 64.40616607666016,
      "activations/layer5_attention_weight_min": -75.739501953125,
      "activations/layer6_attention_weight_max": 46.09489440917969,
      "activations/layer6_attention_weight_min": -50.56753921508789,
      "activations/layer7_attention_weight_max": 63.658294677734375,
      "activations/layer7_attention_weight_min": -60.154808044433594,
      "activations/layer8_attention_weight_max": 46.78081130981445,
      "activations/layer8_attention_weight_min": -47.5845832824707,
      "activations/layer9_attention_weight_max": 55.2736930847168,
      "activations/layer9_attention_weight_min": -47.80965805053711,
      "epoch": 15.15,
      "learning_rate": 5.28280303030303e-05,
      "loss": 2.7773,
      "step": 260750
    },
    {
      "activations/layer0_attention_weight_max": 15.555413246154785,
      "activations/layer0_attention_weight_min": -13.73513126373291,
      "activations/layer10_attention_weight_max": 33.984588623046875,
      "activations/layer10_attention_weight_min": -35.90171432495117,
      "activations/layer11_attention_weight_max": 31.244823455810547,
      "activations/layer11_attention_weight_min": -33.36494445800781,
      "activations/layer12_attention_weight_max": 18.769895553588867,
      "activations/layer12_attention_weight_min": -24.300939559936523,
      "activations/layer13_attention_weight_max": 31.33990478515625,
      "activations/layer13_attention_weight_min": -24.64293098449707,
      "activations/layer14_attention_weight_max": 32.9078483581543,
      "activations/layer14_attention_weight_min": -29.927570343017578,
      "activations/layer15_attention_weight_max": 28.654964447021484,
      "activations/layer15_attention_weight_min": -29.712955474853516,
      "activations/layer16_attention_weight_max": 29.932252883911133,
      "activations/layer16_attention_weight_min": -29.263978958129883,
      "activations/layer17_attention_weight_max": 55.92621612548828,
      "activations/layer17_attention_weight_min": -43.42686080932617,
      "activations/layer18_attention_weight_max": 42.544593811035156,
      "activations/layer18_attention_weight_min": -38.41258239746094,
      "activations/layer19_attention_weight_max": 19.67180633544922,
      "activations/layer19_attention_weight_min": -22.880800247192383,
      "activations/layer1_attention_weight_max": 16.964555740356445,
      "activations/layer1_attention_weight_min": -14.917328834533691,
      "activations/layer20_attention_weight_max": 18.415019989013672,
      "activations/layer20_attention_weight_min": -23.422441482543945,
      "activations/layer21_attention_weight_max": 35.9769287109375,
      "activations/layer21_attention_weight_min": -21.293899536132812,
      "activations/layer22_attention_weight_max": 31.245901107788086,
      "activations/layer22_attention_weight_min": -24.19045639038086,
      "activations/layer23_attention_weight_max": 35.370643615722656,
      "activations/layer23_attention_weight_min": -24.396778106689453,
      "activations/layer2_attention_weight_max": 31.809215545654297,
      "activations/layer2_attention_weight_min": -30.111587524414062,
      "activations/layer3_attention_weight_max": 86.26445007324219,
      "activations/layer3_attention_weight_min": -94.47128295898438,
      "activations/layer4_attention_weight_max": 87.07923889160156,
      "activations/layer4_attention_weight_min": -84.50250244140625,
      "activations/layer5_attention_weight_max": 66.73056030273438,
      "activations/layer5_attention_weight_min": -71.00737762451172,
      "activations/layer6_attention_weight_max": 48.654232025146484,
      "activations/layer6_attention_weight_min": -49.88304901123047,
      "activations/layer7_attention_weight_max": 63.24247360229492,
      "activations/layer7_attention_weight_min": -61.416419982910156,
      "activations/layer8_attention_weight_max": 44.835166931152344,
      "activations/layer8_attention_weight_min": -48.88808059692383,
      "activations/layer9_attention_weight_max": 44.87715530395508,
      "activations/layer9_attention_weight_min": -43.90424346923828,
      "epoch": 15.15,
      "learning_rate": 5.2809090909090906e-05,
      "loss": 2.7652,
      "step": 260800
    },
    {
      "activations/layer0_attention_weight_max": 15.07419204711914,
      "activations/layer0_attention_weight_min": -13.293700218200684,
      "activations/layer10_attention_weight_max": 41.52845001220703,
      "activations/layer10_attention_weight_min": -40.7519416809082,
      "activations/layer11_attention_weight_max": 34.64170837402344,
      "activations/layer11_attention_weight_min": -35.10911560058594,
      "activations/layer12_attention_weight_max": 18.636655807495117,
      "activations/layer12_attention_weight_min": -26.444921493530273,
      "activations/layer13_attention_weight_max": 37.40609359741211,
      "activations/layer13_attention_weight_min": -30.051738739013672,
      "activations/layer14_attention_weight_max": 37.9254264831543,
      "activations/layer14_attention_weight_min": -33.0720329284668,
      "activations/layer15_attention_weight_max": 35.389747619628906,
      "activations/layer15_attention_weight_min": -31.25247573852539,
      "activations/layer16_attention_weight_max": 33.87321090698242,
      "activations/layer16_attention_weight_min": -31.35502052307129,
      "activations/layer17_attention_weight_max": 56.313289642333984,
      "activations/layer17_attention_weight_min": -45.622432708740234,
      "activations/layer18_attention_weight_max": 52.1690788269043,
      "activations/layer18_attention_weight_min": -39.88789749145508,
      "activations/layer19_attention_weight_max": 27.111419677734375,
      "activations/layer19_attention_weight_min": -22.284090042114258,
      "activations/layer1_attention_weight_max": 16.555309295654297,
      "activations/layer1_attention_weight_min": -18.367109298706055,
      "activations/layer20_attention_weight_max": 22.881120681762695,
      "activations/layer20_attention_weight_min": -21.502483367919922,
      "activations/layer21_attention_weight_max": 37.363372802734375,
      "activations/layer21_attention_weight_min": -22.919002532958984,
      "activations/layer22_attention_weight_max": 27.519081115722656,
      "activations/layer22_attention_weight_min": -26.036754608154297,
      "activations/layer23_attention_weight_max": 34.771484375,
      "activations/layer23_attention_weight_min": -24.719253540039062,
      "activations/layer2_attention_weight_max": 31.191486358642578,
      "activations/layer2_attention_weight_min": -32.05149459838867,
      "activations/layer3_attention_weight_max": 96.41768646240234,
      "activations/layer3_attention_weight_min": -97.3924560546875,
      "activations/layer4_attention_weight_max": 87.4208984375,
      "activations/layer4_attention_weight_min": -87.8710708618164,
      "activations/layer5_attention_weight_max": 65.8134765625,
      "activations/layer5_attention_weight_min": -76.74515533447266,
      "activations/layer6_attention_weight_max": 49.69231414794922,
      "activations/layer6_attention_weight_min": -54.76787567138672,
      "activations/layer7_attention_weight_max": 62.3127555847168,
      "activations/layer7_attention_weight_min": -63.88361740112305,
      "activations/layer8_attention_weight_max": 50.04728698730469,
      "activations/layer8_attention_weight_min": -53.61214065551758,
      "activations/layer9_attention_weight_max": 47.71758270263672,
      "activations/layer9_attention_weight_min": -51.474334716796875,
      "epoch": 15.16,
      "learning_rate": 5.279015151515151e-05,
      "loss": 2.7429,
      "step": 260850
    },
    {
      "activations/layer0_attention_weight_max": 15.627264976501465,
      "activations/layer0_attention_weight_min": -13.741500854492188,
      "activations/layer10_attention_weight_max": 33.11348342895508,
      "activations/layer10_attention_weight_min": -33.42449951171875,
      "activations/layer11_attention_weight_max": 30.288846969604492,
      "activations/layer11_attention_weight_min": -31.79330825805664,
      "activations/layer12_attention_weight_max": 17.64092445373535,
      "activations/layer12_attention_weight_min": -28.7172908782959,
      "activations/layer13_attention_weight_max": 31.231176376342773,
      "activations/layer13_attention_weight_min": -26.14946746826172,
      "activations/layer14_attention_weight_max": 34.032501220703125,
      "activations/layer14_attention_weight_min": -27.40972900390625,
      "activations/layer15_attention_weight_max": 31.653718948364258,
      "activations/layer15_attention_weight_min": -29.203275680541992,
      "activations/layer16_attention_weight_max": 32.036155700683594,
      "activations/layer16_attention_weight_min": -30.171049118041992,
      "activations/layer17_attention_weight_max": 48.127193450927734,
      "activations/layer17_attention_weight_min": -43.61030578613281,
      "activations/layer18_attention_weight_max": 43.5695686340332,
      "activations/layer18_attention_weight_min": -36.71992111206055,
      "activations/layer19_attention_weight_max": 22.969703674316406,
      "activations/layer19_attention_weight_min": -22.659841537475586,
      "activations/layer1_attention_weight_max": 17.581140518188477,
      "activations/layer1_attention_weight_min": -17.213497161865234,
      "activations/layer20_attention_weight_max": 21.477378845214844,
      "activations/layer20_attention_weight_min": -25.95726776123047,
      "activations/layer21_attention_weight_max": 36.74135208129883,
      "activations/layer21_attention_weight_min": -22.944555282592773,
      "activations/layer22_attention_weight_max": 28.857757568359375,
      "activations/layer22_attention_weight_min": -27.33437728881836,
      "activations/layer23_attention_weight_max": 36.416988372802734,
      "activations/layer23_attention_weight_min": -26.900222778320312,
      "activations/layer2_attention_weight_max": 34.84234619140625,
      "activations/layer2_attention_weight_min": -32.347774505615234,
      "activations/layer3_attention_weight_max": 97.54596710205078,
      "activations/layer3_attention_weight_min": -94.60417938232422,
      "activations/layer4_attention_weight_max": 89.9623794555664,
      "activations/layer4_attention_weight_min": -85.7185287475586,
      "activations/layer5_attention_weight_max": 65.64588928222656,
      "activations/layer5_attention_weight_min": -73.05977630615234,
      "activations/layer6_attention_weight_max": 48.414546966552734,
      "activations/layer6_attention_weight_min": -49.28825759887695,
      "activations/layer7_attention_weight_max": 62.12181854248047,
      "activations/layer7_attention_weight_min": -59.693153381347656,
      "activations/layer8_attention_weight_max": 43.529109954833984,
      "activations/layer8_attention_weight_min": -46.84661865234375,
      "activations/layer9_attention_weight_max": 48.908687591552734,
      "activations/layer9_attention_weight_min": -43.23812484741211,
      "epoch": 15.16,
      "learning_rate": 5.2771212121212116e-05,
      "loss": 2.749,
      "step": 260900
    },
    {
      "activations/layer0_attention_weight_max": 15.147430419921875,
      "activations/layer0_attention_weight_min": -13.765175819396973,
      "activations/layer10_attention_weight_max": 34.96364212036133,
      "activations/layer10_attention_weight_min": -33.96765899658203,
      "activations/layer11_attention_weight_max": 31.907676696777344,
      "activations/layer11_attention_weight_min": -32.926673889160156,
      "activations/layer12_attention_weight_max": 19.094388961791992,
      "activations/layer12_attention_weight_min": -22.858728408813477,
      "activations/layer13_attention_weight_max": 32.02901077270508,
      "activations/layer13_attention_weight_min": -26.68254280090332,
      "activations/layer14_attention_weight_max": 31.412240982055664,
      "activations/layer14_attention_weight_min": -31.591949462890625,
      "activations/layer15_attention_weight_max": 29.529809951782227,
      "activations/layer15_attention_weight_min": -29.858369827270508,
      "activations/layer16_attention_weight_max": 30.71581268310547,
      "activations/layer16_attention_weight_min": -28.86271858215332,
      "activations/layer17_attention_weight_max": 47.28081512451172,
      "activations/layer17_attention_weight_min": -43.1709098815918,
      "activations/layer18_attention_weight_max": 43.91779708862305,
      "activations/layer18_attention_weight_min": -37.207271575927734,
      "activations/layer19_attention_weight_max": 24.18524742126465,
      "activations/layer19_attention_weight_min": -24.367706298828125,
      "activations/layer1_attention_weight_max": 15.6337308883667,
      "activations/layer1_attention_weight_min": -15.83582878112793,
      "activations/layer20_attention_weight_max": 22.226701736450195,
      "activations/layer20_attention_weight_min": -25.483678817749023,
      "activations/layer21_attention_weight_max": 33.93221664428711,
      "activations/layer21_attention_weight_min": -22.05239486694336,
      "activations/layer22_attention_weight_max": 28.47586441040039,
      "activations/layer22_attention_weight_min": -30.27758026123047,
      "activations/layer23_attention_weight_max": 39.759761810302734,
      "activations/layer23_attention_weight_min": -26.352561950683594,
      "activations/layer2_attention_weight_max": 31.38701629638672,
      "activations/layer2_attention_weight_min": -32.377906799316406,
      "activations/layer3_attention_weight_max": 92.73267364501953,
      "activations/layer3_attention_weight_min": -97.10436248779297,
      "activations/layer4_attention_weight_max": 84.57616424560547,
      "activations/layer4_attention_weight_min": -83.96157836914062,
      "activations/layer5_attention_weight_max": 61.81147384643555,
      "activations/layer5_attention_weight_min": -71.27288818359375,
      "activations/layer6_attention_weight_max": 49.02408981323242,
      "activations/layer6_attention_weight_min": -48.79730987548828,
      "activations/layer7_attention_weight_max": 59.22599411010742,
      "activations/layer7_attention_weight_min": -59.06549835205078,
      "activations/layer8_attention_weight_max": 43.473609924316406,
      "activations/layer8_attention_weight_min": -47.41642761230469,
      "activations/layer9_attention_weight_max": 43.442626953125,
      "activations/layer9_attention_weight_min": -45.9657096862793,
      "epoch": 15.16,
      "learning_rate": 5.2752272727272724e-05,
      "loss": 2.7563,
      "step": 260950
    },
    {
      "activations/layer0_attention_weight_max": 15.394194602966309,
      "activations/layer0_attention_weight_min": -13.323297500610352,
      "activations/layer10_attention_weight_max": 35.96160888671875,
      "activations/layer10_attention_weight_min": -35.93193435668945,
      "activations/layer11_attention_weight_max": 33.6052360534668,
      "activations/layer11_attention_weight_min": -33.784725189208984,
      "activations/layer12_attention_weight_max": 19.29528045654297,
      "activations/layer12_attention_weight_min": -26.50210189819336,
      "activations/layer13_attention_weight_max": 37.174476623535156,
      "activations/layer13_attention_weight_min": -29.22552490234375,
      "activations/layer14_attention_weight_max": 36.795658111572266,
      "activations/layer14_attention_weight_min": -31.072887420654297,
      "activations/layer15_attention_weight_max": 33.647010803222656,
      "activations/layer15_attention_weight_min": -29.496070861816406,
      "activations/layer16_attention_weight_max": 31.702844619750977,
      "activations/layer16_attention_weight_min": -29.240129470825195,
      "activations/layer17_attention_weight_max": 52.2177619934082,
      "activations/layer17_attention_weight_min": -43.12885665893555,
      "activations/layer18_attention_weight_max": 44.79228973388672,
      "activations/layer18_attention_weight_min": -37.69377517700195,
      "activations/layer19_attention_weight_max": 25.1912899017334,
      "activations/layer19_attention_weight_min": -24.598480224609375,
      "activations/layer1_attention_weight_max": 16.207393646240234,
      "activations/layer1_attention_weight_min": -15.32349681854248,
      "activations/layer20_attention_weight_max": 24.28907012939453,
      "activations/layer20_attention_weight_min": -25.238601684570312,
      "activations/layer21_attention_weight_max": 35.08013916015625,
      "activations/layer21_attention_weight_min": -25.154388427734375,
      "activations/layer22_attention_weight_max": 32.39950942993164,
      "activations/layer22_attention_weight_min": -25.158254623413086,
      "activations/layer23_attention_weight_max": 37.7220573425293,
      "activations/layer23_attention_weight_min": -24.34613800048828,
      "activations/layer2_attention_weight_max": 31.799911499023438,
      "activations/layer2_attention_weight_min": -30.587905883789062,
      "activations/layer3_attention_weight_max": 88.78658294677734,
      "activations/layer3_attention_weight_min": -93.20244598388672,
      "activations/layer4_attention_weight_max": 88.0030288696289,
      "activations/layer4_attention_weight_min": -88.27534484863281,
      "activations/layer5_attention_weight_max": 65.34068298339844,
      "activations/layer5_attention_weight_min": -72.71965789794922,
      "activations/layer6_attention_weight_max": 47.81637191772461,
      "activations/layer6_attention_weight_min": -50.41122817993164,
      "activations/layer7_attention_weight_max": 62.91021728515625,
      "activations/layer7_attention_weight_min": -62.42589569091797,
      "activations/layer8_attention_weight_max": 46.4186897277832,
      "activations/layer8_attention_weight_min": -49.22164535522461,
      "activations/layer9_attention_weight_max": 44.95397186279297,
      "activations/layer9_attention_weight_min": -48.632606506347656,
      "epoch": 15.17,
      "learning_rate": 5.273333333333333e-05,
      "loss": 2.7558,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_loss": 2.7109375,
      "eval_runtime": 8.4846,
      "eval_samples_per_second": 506.092,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_openwebtext_loss": 2.7109375,
      "eval_openwebtext_ppl": 15.04337206739713,
      "eval_openwebtext_runtime": 8.4846,
      "eval_openwebtext_samples_per_second": 506.092,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9764,
      "eval_wikitext_samples_per_second": 230.718,
      "step": 261000
    },
    {
      "epoch": 15.17,
      "eval_lambada_loss": 2.5234375,
      "eval_lambada_ppl": 12.471393479493909,
      "eval_lambada_runtime": 9.5511,
      "eval_lambada_samples_per_second": 509.784,
      "step": 261000
    },
    {
      "activations/layer0_attention_weight_max": 15.582989692687988,
      "activations/layer0_attention_weight_min": -13.60376262664795,
      "activations/layer10_attention_weight_max": 41.5216178894043,
      "activations/layer10_attention_weight_min": -39.41230010986328,
      "activations/layer11_attention_weight_max": 36.383575439453125,
      "activations/layer11_attention_weight_min": -35.569679260253906,
      "activations/layer12_attention_weight_max": 19.149959564208984,
      "activations/layer12_attention_weight_min": -23.708919525146484,
      "activations/layer13_attention_weight_max": 37.23725509643555,
      "activations/layer13_attention_weight_min": -28.81942367553711,
      "activations/layer14_attention_weight_max": 38.46306228637695,
      "activations/layer14_attention_weight_min": -30.388383865356445,
      "activations/layer15_attention_weight_max": 34.559200286865234,
      "activations/layer15_attention_weight_min": -29.635211944580078,
      "activations/layer16_attention_weight_max": 34.83995056152344,
      "activations/layer16_attention_weight_min": -33.18158721923828,
      "activations/layer17_attention_weight_max": 55.52769088745117,
      "activations/layer17_attention_weight_min": -46.78923416137695,
      "activations/layer18_attention_weight_max": 50.61396408081055,
      "activations/layer18_attention_weight_min": -39.965763092041016,
      "activations/layer19_attention_weight_max": 25.297481536865234,
      "activations/layer19_attention_weight_min": -23.317790985107422,
      "activations/layer1_attention_weight_max": 15.778379440307617,
      "activations/layer1_attention_weight_min": -16.195602416992188,
      "activations/layer20_attention_weight_max": 25.458580017089844,
      "activations/layer20_attention_weight_min": -21.490346908569336,
      "activations/layer21_attention_weight_max": 41.82963180541992,
      "activations/layer21_attention_weight_min": -22.497753143310547,
      "activations/layer22_attention_weight_max": 33.39845657348633,
      "activations/layer22_attention_weight_min": -25.91522979736328,
      "activations/layer23_attention_weight_max": 40.525508880615234,
      "activations/layer23_attention_weight_min": -24.760265350341797,
      "activations/layer2_attention_weight_max": 33.30437469482422,
      "activations/layer2_attention_weight_min": -32.594940185546875,
      "activations/layer3_attention_weight_max": 92.37641143798828,
      "activations/layer3_attention_weight_min": -94.95855712890625,
      "activations/layer4_attention_weight_max": 93.6537857055664,
      "activations/layer4_attention_weight_min": -92.10961151123047,
      "activations/layer5_attention_weight_max": 73.6935806274414,
      "activations/layer5_attention_weight_min": -77.9514389038086,
      "activations/layer6_attention_weight_max": 53.96202850341797,
      "activations/layer6_attention_weight_min": -54.401119232177734,
      "activations/layer7_attention_weight_max": 62.87856674194336,
      "activations/layer7_attention_weight_min": -59.47063446044922,
      "activations/layer8_attention_weight_max": 49.41759490966797,
      "activations/layer8_attention_weight_min": -51.89228820800781,
      "activations/layer9_attention_weight_max": 47.88236618041992,
      "activations/layer9_attention_weight_min": -48.65163803100586,
      "epoch": 15.17,
      "learning_rate": 5.2714393939393934e-05,
      "loss": 2.7736,
      "step": 261050
    },
    {
      "activations/layer0_attention_weight_max": 15.331624984741211,
      "activations/layer0_attention_weight_min": -13.796262741088867,
      "activations/layer10_attention_weight_max": 42.570499420166016,
      "activations/layer10_attention_weight_min": -41.397525787353516,
      "activations/layer11_attention_weight_max": 38.86737823486328,
      "activations/layer11_attention_weight_min": -37.894813537597656,
      "activations/layer12_attention_weight_max": 18.548858642578125,
      "activations/layer12_attention_weight_min": -25.41401481628418,
      "activations/layer13_attention_weight_max": 42.31797409057617,
      "activations/layer13_attention_weight_min": -26.872034072875977,
      "activations/layer14_attention_weight_max": 38.560707092285156,
      "activations/layer14_attention_weight_min": -30.39775276184082,
      "activations/layer15_attention_weight_max": 36.715293884277344,
      "activations/layer15_attention_weight_min": -32.96482467651367,
      "activations/layer16_attention_weight_max": 34.97102737426758,
      "activations/layer16_attention_weight_min": -31.29997444152832,
      "activations/layer17_attention_weight_max": 54.0377311706543,
      "activations/layer17_attention_weight_min": -48.446590423583984,
      "activations/layer18_attention_weight_max": 51.62682342529297,
      "activations/layer18_attention_weight_min": -40.9609260559082,
      "activations/layer19_attention_weight_max": 26.845224380493164,
      "activations/layer19_attention_weight_min": -22.283336639404297,
      "activations/layer1_attention_weight_max": 17.42136001586914,
      "activations/layer1_attention_weight_min": -20.477113723754883,
      "activations/layer20_attention_weight_max": 27.219148635864258,
      "activations/layer20_attention_weight_min": -20.737106323242188,
      "activations/layer21_attention_weight_max": 41.17231369018555,
      "activations/layer21_attention_weight_min": -21.858142852783203,
      "activations/layer22_attention_weight_max": 31.047645568847656,
      "activations/layer22_attention_weight_min": -25.08791160583496,
      "activations/layer23_attention_weight_max": 35.70897674560547,
      "activations/layer23_attention_weight_min": -24.469131469726562,
      "activations/layer2_attention_weight_max": 34.46798324584961,
      "activations/layer2_attention_weight_min": -34.73760986328125,
      "activations/layer3_attention_weight_max": 98.38864135742188,
      "activations/layer3_attention_weight_min": -98.81576538085938,
      "activations/layer4_attention_weight_max": 95.826904296875,
      "activations/layer4_attention_weight_min": -96.07463073730469,
      "activations/layer5_attention_weight_max": 69.41001892089844,
      "activations/layer5_attention_weight_min": -77.34040069580078,
      "activations/layer6_attention_weight_max": 55.61522674560547,
      "activations/layer6_attention_weight_min": -56.69175338745117,
      "activations/layer7_attention_weight_max": 71.1536865234375,
      "activations/layer7_attention_weight_min": -68.07281494140625,
      "activations/layer8_attention_weight_max": 58.54587936401367,
      "activations/layer8_attention_weight_min": -56.1795539855957,
      "activations/layer9_attention_weight_max": 59.17340087890625,
      "activations/layer9_attention_weight_min": -56.70405960083008,
      "epoch": 15.17,
      "learning_rate": 5.2695454545454535e-05,
      "loss": 2.7418,
      "step": 261100
    },
    {
      "activations/layer0_attention_weight_max": 14.50549602508545,
      "activations/layer0_attention_weight_min": -13.9345121383667,
      "activations/layer10_attention_weight_max": 40.21733093261719,
      "activations/layer10_attention_weight_min": -38.23694610595703,
      "activations/layer11_attention_weight_max": 36.68605041503906,
      "activations/layer11_attention_weight_min": -34.387916564941406,
      "activations/layer12_attention_weight_max": 19.51437759399414,
      "activations/layer12_attention_weight_min": -25.265869140625,
      "activations/layer13_attention_weight_max": 42.73573684692383,
      "activations/layer13_attention_weight_min": -30.057851791381836,
      "activations/layer14_attention_weight_max": 40.219852447509766,
      "activations/layer14_attention_weight_min": -29.0592098236084,
      "activations/layer15_attention_weight_max": 35.57781982421875,
      "activations/layer15_attention_weight_min": -28.971193313598633,
      "activations/layer16_attention_weight_max": 34.571067810058594,
      "activations/layer16_attention_weight_min": -29.53536605834961,
      "activations/layer17_attention_weight_max": 54.30104446411133,
      "activations/layer17_attention_weight_min": -44.454246520996094,
      "activations/layer18_attention_weight_max": 50.435699462890625,
      "activations/layer18_attention_weight_min": -38.54237365722656,
      "activations/layer19_attention_weight_max": 25.70292091369629,
      "activations/layer19_attention_weight_min": -22.32911491394043,
      "activations/layer1_attention_weight_max": 16.858163833618164,
      "activations/layer1_attention_weight_min": -17.287328720092773,
      "activations/layer20_attention_weight_max": 23.918733596801758,
      "activations/layer20_attention_weight_min": -21.58051872253418,
      "activations/layer21_attention_weight_max": 37.60283279418945,
      "activations/layer21_attention_weight_min": -22.94338035583496,
      "activations/layer22_attention_weight_max": 30.144521713256836,
      "activations/layer22_attention_weight_min": -24.756956100463867,
      "activations/layer23_attention_weight_max": 41.59598922729492,
      "activations/layer23_attention_weight_min": -22.775054931640625,
      "activations/layer2_attention_weight_max": 33.853477478027344,
      "activations/layer2_attention_weight_min": -33.62546157836914,
      "activations/layer3_attention_weight_max": 98.85599517822266,
      "activations/layer3_attention_weight_min": -106.53340911865234,
      "activations/layer4_attention_weight_max": 96.73823547363281,
      "activations/layer4_attention_weight_min": -93.36758422851562,
      "activations/layer5_attention_weight_max": 67.71931457519531,
      "activations/layer5_attention_weight_min": -71.44497680664062,
      "activations/layer6_attention_weight_max": 49.12141418457031,
      "activations/layer6_attention_weight_min": -48.99047088623047,
      "activations/layer7_attention_weight_max": 67.15521240234375,
      "activations/layer7_attention_weight_min": -63.27965545654297,
      "activations/layer8_attention_weight_max": 49.47458267211914,
      "activations/layer8_attention_weight_min": -50.46442413330078,
      "activations/layer9_attention_weight_max": 51.14126968383789,
      "activations/layer9_attention_weight_min": -47.250179290771484,
      "epoch": 15.17,
      "learning_rate": 5.267651515151515e-05,
      "loss": 2.74,
      "step": 261150
    },
    {
      "activations/layer0_attention_weight_max": 16.25333023071289,
      "activations/layer0_attention_weight_min": -13.694003105163574,
      "activations/layer10_attention_weight_max": 36.10212707519531,
      "activations/layer10_attention_weight_min": -35.680450439453125,
      "activations/layer11_attention_weight_max": 32.479339599609375,
      "activations/layer11_attention_weight_min": -33.63650131225586,
      "activations/layer12_attention_weight_max": 19.069494247436523,
      "activations/layer12_attention_weight_min": -24.566776275634766,
      "activations/layer13_attention_weight_max": 34.42924118041992,
      "activations/layer13_attention_weight_min": -29.14996337890625,
      "activations/layer14_attention_weight_max": 34.5887451171875,
      "activations/layer14_attention_weight_min": -30.007095336914062,
      "activations/layer15_attention_weight_max": 29.48044776916504,
      "activations/layer15_attention_weight_min": -29.57476043701172,
      "activations/layer16_attention_weight_max": 31.916507720947266,
      "activations/layer16_attention_weight_min": -32.29677963256836,
      "activations/layer17_attention_weight_max": 47.582054138183594,
      "activations/layer17_attention_weight_min": -45.42442321777344,
      "activations/layer18_attention_weight_max": 42.85684585571289,
      "activations/layer18_attention_weight_min": -39.3298225402832,
      "activations/layer19_attention_weight_max": 23.23969841003418,
      "activations/layer19_attention_weight_min": -22.052719116210938,
      "activations/layer1_attention_weight_max": 17.545146942138672,
      "activations/layer1_attention_weight_min": -15.97300910949707,
      "activations/layer20_attention_weight_max": 23.30418586730957,
      "activations/layer20_attention_weight_min": -20.860424041748047,
      "activations/layer21_attention_weight_max": 35.00179672241211,
      "activations/layer21_attention_weight_min": -23.953466415405273,
      "activations/layer22_attention_weight_max": 29.194509506225586,
      "activations/layer22_attention_weight_min": -24.59282684326172,
      "activations/layer23_attention_weight_max": 34.218528747558594,
      "activations/layer23_attention_weight_min": -24.812551498413086,
      "activations/layer2_attention_weight_max": 30.83336639404297,
      "activations/layer2_attention_weight_min": -29.501943588256836,
      "activations/layer3_attention_weight_max": 94.62913513183594,
      "activations/layer3_attention_weight_min": -88.78206634521484,
      "activations/layer4_attention_weight_max": 91.13670349121094,
      "activations/layer4_attention_weight_min": -85.63594055175781,
      "activations/layer5_attention_weight_max": 66.47776794433594,
      "activations/layer5_attention_weight_min": -73.11015319824219,
      "activations/layer6_attention_weight_max": 49.52934265136719,
      "activations/layer6_attention_weight_min": -49.69651794433594,
      "activations/layer7_attention_weight_max": 63.382389068603516,
      "activations/layer7_attention_weight_min": -60.70391845703125,
      "activations/layer8_attention_weight_max": 45.368003845214844,
      "activations/layer8_attention_weight_min": -46.386688232421875,
      "activations/layer9_attention_weight_max": 45.81758117675781,
      "activations/layer9_attention_weight_min": -46.638484954833984,
      "epoch": 15.18,
      "learning_rate": 5.265757575757575e-05,
      "loss": 2.756,
      "step": 261200
    },
    {
      "activations/layer0_attention_weight_max": 14.794758796691895,
      "activations/layer0_attention_weight_min": -13.632232666015625,
      "activations/layer10_attention_weight_max": 32.594154357910156,
      "activations/layer10_attention_weight_min": -33.31571960449219,
      "activations/layer11_attention_weight_max": 30.617788314819336,
      "activations/layer11_attention_weight_min": -32.36747360229492,
      "activations/layer12_attention_weight_max": 18.254989624023438,
      "activations/layer12_attention_weight_min": -22.9962158203125,
      "activations/layer13_attention_weight_max": 33.59044647216797,
      "activations/layer13_attention_weight_min": -26.533449172973633,
      "activations/layer14_attention_weight_max": 38.55234909057617,
      "activations/layer14_attention_weight_min": -30.623003005981445,
      "activations/layer15_attention_weight_max": 31.26387596130371,
      "activations/layer15_attention_weight_min": -30.388484954833984,
      "activations/layer16_attention_weight_max": 31.599576950073242,
      "activations/layer16_attention_weight_min": -30.752290725708008,
      "activations/layer17_attention_weight_max": 53.45813751220703,
      "activations/layer17_attention_weight_min": -45.80205154418945,
      "activations/layer18_attention_weight_max": 47.46055221557617,
      "activations/layer18_attention_weight_min": -41.0977783203125,
      "activations/layer19_attention_weight_max": 23.96274757385254,
      "activations/layer19_attention_weight_min": -23.675094604492188,
      "activations/layer1_attention_weight_max": 16.39019012451172,
      "activations/layer1_attention_weight_min": -15.300931930541992,
      "activations/layer20_attention_weight_max": 27.075571060180664,
      "activations/layer20_attention_weight_min": -24.16826057434082,
      "activations/layer21_attention_weight_max": 34.92223358154297,
      "activations/layer21_attention_weight_min": -25.30389404296875,
      "activations/layer22_attention_weight_max": 30.872100830078125,
      "activations/layer22_attention_weight_min": -26.0830078125,
      "activations/layer23_attention_weight_max": 39.271366119384766,
      "activations/layer23_attention_weight_min": -28.073471069335938,
      "activations/layer2_attention_weight_max": 29.273067474365234,
      "activations/layer2_attention_weight_min": -27.55545425415039,
      "activations/layer3_attention_weight_max": 87.75399780273438,
      "activations/layer3_attention_weight_min": -87.21625518798828,
      "activations/layer4_attention_weight_max": 84.67337036132812,
      "activations/layer4_attention_weight_min": -83.4145278930664,
      "activations/layer5_attention_weight_max": 64.53158569335938,
      "activations/layer5_attention_weight_min": -71.8109130859375,
      "activations/layer6_attention_weight_max": 48.39668273925781,
      "activations/layer6_attention_weight_min": -48.48088455200195,
      "activations/layer7_attention_weight_max": 62.30989456176758,
      "activations/layer7_attention_weight_min": -59.138946533203125,
      "activations/layer8_attention_weight_max": 44.55037307739258,
      "activations/layer8_attention_weight_min": -46.68769073486328,
      "activations/layer9_attention_weight_max": 44.01748275756836,
      "activations/layer9_attention_weight_min": -43.6601676940918,
      "epoch": 15.18,
      "learning_rate": 5.263863636363636e-05,
      "loss": 2.7572,
      "step": 261250
    },
    {
      "activations/layer0_attention_weight_max": 14.822476387023926,
      "activations/layer0_attention_weight_min": -13.67269515991211,
      "activations/layer10_attention_weight_max": 32.20957946777344,
      "activations/layer10_attention_weight_min": -34.276878356933594,
      "activations/layer11_attention_weight_max": 30.681156158447266,
      "activations/layer11_attention_weight_min": -31.723651885986328,
      "activations/layer12_attention_weight_max": 19.44370460510254,
      "activations/layer12_attention_weight_min": -24.838865280151367,
      "activations/layer13_attention_weight_max": 38.19013214111328,
      "activations/layer13_attention_weight_min": -27.796428680419922,
      "activations/layer14_attention_weight_max": 32.85207748413086,
      "activations/layer14_attention_weight_min": -27.911073684692383,
      "activations/layer15_attention_weight_max": 31.529891967773438,
      "activations/layer15_attention_weight_min": -27.585264205932617,
      "activations/layer16_attention_weight_max": 31.303409576416016,
      "activations/layer16_attention_weight_min": -28.77998924255371,
      "activations/layer17_attention_weight_max": 47.0645751953125,
      "activations/layer17_attention_weight_min": -42.60389709472656,
      "activations/layer18_attention_weight_max": 46.51472091674805,
      "activations/layer18_attention_weight_min": -38.392608642578125,
      "activations/layer19_attention_weight_max": 23.206932067871094,
      "activations/layer19_attention_weight_min": -22.631553649902344,
      "activations/layer1_attention_weight_max": 16.684484481811523,
      "activations/layer1_attention_weight_min": -14.754521369934082,
      "activations/layer20_attention_weight_max": 22.323348999023438,
      "activations/layer20_attention_weight_min": -20.984891891479492,
      "activations/layer21_attention_weight_max": 37.496795654296875,
      "activations/layer21_attention_weight_min": -25.0920352935791,
      "activations/layer22_attention_weight_max": 34.90188980102539,
      "activations/layer22_attention_weight_min": -26.22019386291504,
      "activations/layer23_attention_weight_max": 35.190284729003906,
      "activations/layer23_attention_weight_min": -25.276264190673828,
      "activations/layer2_attention_weight_max": 32.35997009277344,
      "activations/layer2_attention_weight_min": -29.808731079101562,
      "activations/layer3_attention_weight_max": 96.12443542480469,
      "activations/layer3_attention_weight_min": -93.19145965576172,
      "activations/layer4_attention_weight_max": 95.0203628540039,
      "activations/layer4_attention_weight_min": -89.21675872802734,
      "activations/layer5_attention_weight_max": 65.63792419433594,
      "activations/layer5_attention_weight_min": -69.84052276611328,
      "activations/layer6_attention_weight_max": 48.82062911987305,
      "activations/layer6_attention_weight_min": -50.086891174316406,
      "activations/layer7_attention_weight_max": 65.28030395507812,
      "activations/layer7_attention_weight_min": -57.5711555480957,
      "activations/layer8_attention_weight_max": 43.76094055175781,
      "activations/layer8_attention_weight_min": -47.85499954223633,
      "activations/layer9_attention_weight_max": 51.273773193359375,
      "activations/layer9_attention_weight_min": -43.91459655761719,
      "epoch": 15.18,
      "learning_rate": 5.261969696969696e-05,
      "loss": 2.7501,
      "step": 261300
    },
    {
      "activations/layer0_attention_weight_max": 15.542664527893066,
      "activations/layer0_attention_weight_min": -14.355237007141113,
      "activations/layer10_attention_weight_max": 35.796295166015625,
      "activations/layer10_attention_weight_min": -35.81495666503906,
      "activations/layer11_attention_weight_max": 32.591304779052734,
      "activations/layer11_attention_weight_min": -33.79761505126953,
      "activations/layer12_attention_weight_max": 17.571823120117188,
      "activations/layer12_attention_weight_min": -24.75729751586914,
      "activations/layer13_attention_weight_max": 38.56858825683594,
      "activations/layer13_attention_weight_min": -27.655580520629883,
      "activations/layer14_attention_weight_max": 32.82244110107422,
      "activations/layer14_attention_weight_min": -29.590755462646484,
      "activations/layer15_attention_weight_max": 32.161468505859375,
      "activations/layer15_attention_weight_min": -30.79039192199707,
      "activations/layer16_attention_weight_max": 32.45182418823242,
      "activations/layer16_attention_weight_min": -29.96813201904297,
      "activations/layer17_attention_weight_max": 48.47207260131836,
      "activations/layer17_attention_weight_min": -44.07162094116211,
      "activations/layer18_attention_weight_max": 43.822898864746094,
      "activations/layer18_attention_weight_min": -37.975337982177734,
      "activations/layer19_attention_weight_max": 21.670942306518555,
      "activations/layer19_attention_weight_min": -21.831356048583984,
      "activations/layer1_attention_weight_max": 17.189966201782227,
      "activations/layer1_attention_weight_min": -15.292206764221191,
      "activations/layer20_attention_weight_max": 20.450899124145508,
      "activations/layer20_attention_weight_min": -23.42089080810547,
      "activations/layer21_attention_weight_max": 33.89988327026367,
      "activations/layer21_attention_weight_min": -24.591306686401367,
      "activations/layer22_attention_weight_max": 29.140182495117188,
      "activations/layer22_attention_weight_min": -27.076929092407227,
      "activations/layer23_attention_weight_max": 34.57190704345703,
      "activations/layer23_attention_weight_min": -25.538570404052734,
      "activations/layer2_attention_weight_max": 31.56171226501465,
      "activations/layer2_attention_weight_min": -31.315868377685547,
      "activations/layer3_attention_weight_max": 89.11589050292969,
      "activations/layer3_attention_weight_min": -89.83195495605469,
      "activations/layer4_attention_weight_max": 91.76895141601562,
      "activations/layer4_attention_weight_min": -87.51180267333984,
      "activations/layer5_attention_weight_max": 63.295082092285156,
      "activations/layer5_attention_weight_min": -76.84769439697266,
      "activations/layer6_attention_weight_max": 50.557579040527344,
      "activations/layer6_attention_weight_min": -50.5107536315918,
      "activations/layer7_attention_weight_max": 56.17338943481445,
      "activations/layer7_attention_weight_min": -61.90464782714844,
      "activations/layer8_attention_weight_max": 44.30507278442383,
      "activations/layer8_attention_weight_min": -50.324790954589844,
      "activations/layer9_attention_weight_max": 43.53213119506836,
      "activations/layer9_attention_weight_min": -44.09990310668945,
      "epoch": 15.19,
      "learning_rate": 5.2600757575757576e-05,
      "loss": 2.7649,
      "step": 261350
    },
    {
      "activations/layer0_attention_weight_max": 16.354251861572266,
      "activations/layer0_attention_weight_min": -13.892496109008789,
      "activations/layer10_attention_weight_max": 37.577701568603516,
      "activations/layer10_attention_weight_min": -36.866546630859375,
      "activations/layer11_attention_weight_max": 34.5260009765625,
      "activations/layer11_attention_weight_min": -36.34693145751953,
      "activations/layer12_attention_weight_max": 21.126773834228516,
      "activations/layer12_attention_weight_min": -28.81045913696289,
      "activations/layer13_attention_weight_max": 42.014915466308594,
      "activations/layer13_attention_weight_min": -30.435457229614258,
      "activations/layer14_attention_weight_max": 41.28404998779297,
      "activations/layer14_attention_weight_min": -30.430675506591797,
      "activations/layer15_attention_weight_max": 38.87666320800781,
      "activations/layer15_attention_weight_min": -29.449968338012695,
      "activations/layer16_attention_weight_max": 35.12431716918945,
      "activations/layer16_attention_weight_min": -31.50566291809082,
      "activations/layer17_attention_weight_max": 55.11833953857422,
      "activations/layer17_attention_weight_min": -45.979583740234375,
      "activations/layer18_attention_weight_max": 54.15945053100586,
      "activations/layer18_attention_weight_min": -40.44356155395508,
      "activations/layer19_attention_weight_max": 25.329696655273438,
      "activations/layer19_attention_weight_min": -24.671995162963867,
      "activations/layer1_attention_weight_max": 15.967985153198242,
      "activations/layer1_attention_weight_min": -14.881209373474121,
      "activations/layer20_attention_weight_max": 28.53857421875,
      "activations/layer20_attention_weight_min": -21.494747161865234,
      "activations/layer21_attention_weight_max": 44.400177001953125,
      "activations/layer21_attention_weight_min": -23.1811580657959,
      "activations/layer22_attention_weight_max": 34.74772262573242,
      "activations/layer22_attention_weight_min": -26.688934326171875,
      "activations/layer23_attention_weight_max": 42.30842208862305,
      "activations/layer23_attention_weight_min": -24.05510711669922,
      "activations/layer2_attention_weight_max": 32.112770080566406,
      "activations/layer2_attention_weight_min": -31.041461944580078,
      "activations/layer3_attention_weight_max": 92.30635070800781,
      "activations/layer3_attention_weight_min": -99.31917572021484,
      "activations/layer4_attention_weight_max": 92.30205535888672,
      "activations/layer4_attention_weight_min": -92.26346588134766,
      "activations/layer5_attention_weight_max": 68.43098449707031,
      "activations/layer5_attention_weight_min": -76.59174346923828,
      "activations/layer6_attention_weight_max": 51.70048141479492,
      "activations/layer6_attention_weight_min": -52.80380630493164,
      "activations/layer7_attention_weight_max": 58.633766174316406,
      "activations/layer7_attention_weight_min": -61.85967254638672,
      "activations/layer8_attention_weight_max": 47.28388214111328,
      "activations/layer8_attention_weight_min": -50.43306350708008,
      "activations/layer9_attention_weight_max": 43.904052734375,
      "activations/layer9_attention_weight_min": -47.677452087402344,
      "epoch": 15.19,
      "learning_rate": 5.258181818181818e-05,
      "loss": 2.7566,
      "step": 261400
    },
    {
      "activations/layer0_attention_weight_max": 15.543923377990723,
      "activations/layer0_attention_weight_min": -13.879920959472656,
      "activations/layer10_attention_weight_max": 34.66900634765625,
      "activations/layer10_attention_weight_min": -34.04195022583008,
      "activations/layer11_attention_weight_max": 32.25946807861328,
      "activations/layer11_attention_weight_min": -32.75734329223633,
      "activations/layer12_attention_weight_max": 18.15990447998047,
      "activations/layer12_attention_weight_min": -30.521879196166992,
      "activations/layer13_attention_weight_max": 29.820178985595703,
      "activations/layer13_attention_weight_min": -28.815237045288086,
      "activations/layer14_attention_weight_max": 36.7642707824707,
      "activations/layer14_attention_weight_min": -30.39658546447754,
      "activations/layer15_attention_weight_max": 33.150848388671875,
      "activations/layer15_attention_weight_min": -28.72794532775879,
      "activations/layer16_attention_weight_max": 33.61853790283203,
      "activations/layer16_attention_weight_min": -32.36058807373047,
      "activations/layer17_attention_weight_max": 52.776283264160156,
      "activations/layer17_attention_weight_min": -45.950775146484375,
      "activations/layer18_attention_weight_max": 45.2461051940918,
      "activations/layer18_attention_weight_min": -40.861576080322266,
      "activations/layer19_attention_weight_max": 26.3347225189209,
      "activations/layer19_attention_weight_min": -22.87902069091797,
      "activations/layer1_attention_weight_max": 16.197622299194336,
      "activations/layer1_attention_weight_min": -13.493138313293457,
      "activations/layer20_attention_weight_max": 26.692100524902344,
      "activations/layer20_attention_weight_min": -21.591455459594727,
      "activations/layer21_attention_weight_max": 39.7764778137207,
      "activations/layer21_attention_weight_min": -27.201738357543945,
      "activations/layer22_attention_weight_max": 31.735092163085938,
      "activations/layer22_attention_weight_min": -26.615234375,
      "activations/layer23_attention_weight_max": 36.526466369628906,
      "activations/layer23_attention_weight_min": -25.36995506286621,
      "activations/layer2_attention_weight_max": 35.6110725402832,
      "activations/layer2_attention_weight_min": -32.092018127441406,
      "activations/layer3_attention_weight_max": 93.16350555419922,
      "activations/layer3_attention_weight_min": -89.42849731445312,
      "activations/layer4_attention_weight_max": 93.02046966552734,
      "activations/layer4_attention_weight_min": -88.8564224243164,
      "activations/layer5_attention_weight_max": 68.89449310302734,
      "activations/layer5_attention_weight_min": -72.46315002441406,
      "activations/layer6_attention_weight_max": 52.37556838989258,
      "activations/layer6_attention_weight_min": -51.79195022583008,
      "activations/layer7_attention_weight_max": 62.127994537353516,
      "activations/layer7_attention_weight_min": -64.1214599609375,
      "activations/layer8_attention_weight_max": 43.96735763549805,
      "activations/layer8_attention_weight_min": -46.1982307434082,
      "activations/layer9_attention_weight_max": 46.83451461791992,
      "activations/layer9_attention_weight_min": -47.11723327636719,
      "epoch": 15.19,
      "learning_rate": 5.256287878787878e-05,
      "loss": 2.749,
      "step": 261450
    },
    {
      "activations/layer0_attention_weight_max": 15.62084674835205,
      "activations/layer0_attention_weight_min": -13.72241497039795,
      "activations/layer10_attention_weight_max": 33.30683135986328,
      "activations/layer10_attention_weight_min": -33.59035110473633,
      "activations/layer11_attention_weight_max": 30.47164535522461,
      "activations/layer11_attention_weight_min": -33.479610443115234,
      "activations/layer12_attention_weight_max": 17.337646484375,
      "activations/layer12_attention_weight_min": -27.397621154785156,
      "activations/layer13_attention_weight_max": 32.112850189208984,
      "activations/layer13_attention_weight_min": -27.985864639282227,
      "activations/layer14_attention_weight_max": 33.198612213134766,
      "activations/layer14_attention_weight_min": -29.126379013061523,
      "activations/layer15_attention_weight_max": 30.951021194458008,
      "activations/layer15_attention_weight_min": -28.876834869384766,
      "activations/layer16_attention_weight_max": 31.039228439331055,
      "activations/layer16_attention_weight_min": -31.518436431884766,
      "activations/layer17_attention_weight_max": 47.505767822265625,
      "activations/layer17_attention_weight_min": -42.905479431152344,
      "activations/layer18_attention_weight_max": 44.0676383972168,
      "activations/layer18_attention_weight_min": -37.931968688964844,
      "activations/layer19_attention_weight_max": 23.094966888427734,
      "activations/layer19_attention_weight_min": -21.492692947387695,
      "activations/layer1_attention_weight_max": 16.86064910888672,
      "activations/layer1_attention_weight_min": -14.413741111755371,
      "activations/layer20_attention_weight_max": 23.606035232543945,
      "activations/layer20_attention_weight_min": -21.784934997558594,
      "activations/layer21_attention_weight_max": 35.58528137207031,
      "activations/layer21_attention_weight_min": -22.91562843322754,
      "activations/layer22_attention_weight_max": 28.47069549560547,
      "activations/layer22_attention_weight_min": -27.253618240356445,
      "activations/layer23_attention_weight_max": 37.4301643371582,
      "activations/layer23_attention_weight_min": -26.135509490966797,
      "activations/layer2_attention_weight_max": 31.708005905151367,
      "activations/layer2_attention_weight_min": -31.079086303710938,
      "activations/layer3_attention_weight_max": 90.19609832763672,
      "activations/layer3_attention_weight_min": -93.31739044189453,
      "activations/layer4_attention_weight_max": 88.57794952392578,
      "activations/layer4_attention_weight_min": -88.21232604980469,
      "activations/layer5_attention_weight_max": 64.74324798583984,
      "activations/layer5_attention_weight_min": -73.78485107421875,
      "activations/layer6_attention_weight_max": 49.09172821044922,
      "activations/layer6_attention_weight_min": -52.714576721191406,
      "activations/layer7_attention_weight_max": 60.00918960571289,
      "activations/layer7_attention_weight_min": -59.47079849243164,
      "activations/layer8_attention_weight_max": 44.26991653442383,
      "activations/layer8_attention_weight_min": -47.15459060668945,
      "activations/layer9_attention_weight_max": 43.70820999145508,
      "activations/layer9_attention_weight_min": -44.12009811401367,
      "epoch": 15.19,
      "learning_rate": 5.254393939393939e-05,
      "loss": 2.7371,
      "step": 261500
    },
    {
      "activations/layer0_attention_weight_max": 16.170862197875977,
      "activations/layer0_attention_weight_min": -13.848073959350586,
      "activations/layer10_attention_weight_max": 41.64848709106445,
      "activations/layer10_attention_weight_min": -41.07814025878906,
      "activations/layer11_attention_weight_max": 37.80467224121094,
      "activations/layer11_attention_weight_min": -35.710609436035156,
      "activations/layer12_attention_weight_max": 20.636125564575195,
      "activations/layer12_attention_weight_min": -26.392370223999023,
      "activations/layer13_attention_weight_max": 38.73130798339844,
      "activations/layer13_attention_weight_min": -29.659873962402344,
      "activations/layer14_attention_weight_max": 39.034629821777344,
      "activations/layer14_attention_weight_min": -29.98649787902832,
      "activations/layer15_attention_weight_max": 36.33829116821289,
      "activations/layer15_attention_weight_min": -31.01490592956543,
      "activations/layer16_attention_weight_max": 36.74808883666992,
      "activations/layer16_attention_weight_min": -32.86670684814453,
      "activations/layer17_attention_weight_max": 54.88460922241211,
      "activations/layer17_attention_weight_min": -48.16475296020508,
      "activations/layer18_attention_weight_max": 49.87237548828125,
      "activations/layer18_attention_weight_min": -39.66999435424805,
      "activations/layer19_attention_weight_max": 27.500686645507812,
      "activations/layer19_attention_weight_min": -23.107807159423828,
      "activations/layer1_attention_weight_max": 16.649656295776367,
      "activations/layer1_attention_weight_min": -14.986496925354004,
      "activations/layer20_attention_weight_max": 26.17680549621582,
      "activations/layer20_attention_weight_min": -23.314769744873047,
      "activations/layer21_attention_weight_max": 44.7834358215332,
      "activations/layer21_attention_weight_min": -23.189348220825195,
      "activations/layer22_attention_weight_max": 34.71565246582031,
      "activations/layer22_attention_weight_min": -28.242815017700195,
      "activations/layer23_attention_weight_max": 41.948211669921875,
      "activations/layer23_attention_weight_min": -26.369062423706055,
      "activations/layer2_attention_weight_max": 32.38213348388672,
      "activations/layer2_attention_weight_min": -33.514671325683594,
      "activations/layer3_attention_weight_max": 95.93746948242188,
      "activations/layer3_attention_weight_min": -99.70393371582031,
      "activations/layer4_attention_weight_max": 92.40776824951172,
      "activations/layer4_attention_weight_min": -99.71658325195312,
      "activations/layer5_attention_weight_max": 65.63313293457031,
      "activations/layer5_attention_weight_min": -74.39122772216797,
      "activations/layer6_attention_weight_max": 53.662654876708984,
      "activations/layer6_attention_weight_min": -57.710350036621094,
      "activations/layer7_attention_weight_max": 71.03972625732422,
      "activations/layer7_attention_weight_min": -74.45757293701172,
      "activations/layer8_attention_weight_max": 54.861019134521484,
      "activations/layer8_attention_weight_min": -55.0247917175293,
      "activations/layer9_attention_weight_max": 59.764095306396484,
      "activations/layer9_attention_weight_min": -53.79907989501953,
      "epoch": 15.2,
      "learning_rate": 5.2524999999999996e-05,
      "loss": 2.7543,
      "step": 261550
    },
    {
      "activations/layer0_attention_weight_max": 15.030553817749023,
      "activations/layer0_attention_weight_min": -13.690078735351562,
      "activations/layer10_attention_weight_max": 39.714996337890625,
      "activations/layer10_attention_weight_min": -40.2690315246582,
      "activations/layer11_attention_weight_max": 36.80537414550781,
      "activations/layer11_attention_weight_min": -34.913177490234375,
      "activations/layer12_attention_weight_max": 20.04347038269043,
      "activations/layer12_attention_weight_min": -25.06365966796875,
      "activations/layer13_attention_weight_max": 41.71794891357422,
      "activations/layer13_attention_weight_min": -29.168926239013672,
      "activations/layer14_attention_weight_max": 43.18110656738281,
      "activations/layer14_attention_weight_min": -30.116504669189453,
      "activations/layer15_attention_weight_max": 36.913429260253906,
      "activations/layer15_attention_weight_min": -28.759815216064453,
      "activations/layer16_attention_weight_max": 38.18753433227539,
      "activations/layer16_attention_weight_min": -34.55223846435547,
      "activations/layer17_attention_weight_max": 58.96367263793945,
      "activations/layer17_attention_weight_min": -48.082176208496094,
      "activations/layer18_attention_weight_max": 56.61967468261719,
      "activations/layer18_attention_weight_min": -42.2816047668457,
      "activations/layer19_attention_weight_max": 25.21082878112793,
      "activations/layer19_attention_weight_min": -23.126008987426758,
      "activations/layer1_attention_weight_max": 18.4210147857666,
      "activations/layer1_attention_weight_min": -15.900538444519043,
      "activations/layer20_attention_weight_max": 28.894140243530273,
      "activations/layer20_attention_weight_min": -20.45162582397461,
      "activations/layer21_attention_weight_max": 44.4953498840332,
      "activations/layer21_attention_weight_min": -24.13820457458496,
      "activations/layer22_attention_weight_max": 34.976375579833984,
      "activations/layer22_attention_weight_min": -26.768613815307617,
      "activations/layer23_attention_weight_max": 47.047950744628906,
      "activations/layer23_attention_weight_min": -25.019927978515625,
      "activations/layer2_attention_weight_max": 32.78639602661133,
      "activations/layer2_attention_weight_min": -31.692203521728516,
      "activations/layer3_attention_weight_max": 91.99132537841797,
      "activations/layer3_attention_weight_min": -91.46009826660156,
      "activations/layer4_attention_weight_max": 90.70756530761719,
      "activations/layer4_attention_weight_min": -88.3828353881836,
      "activations/layer5_attention_weight_max": 68.42247009277344,
      "activations/layer5_attention_weight_min": -71.19590759277344,
      "activations/layer6_attention_weight_max": 50.40354919433594,
      "activations/layer6_attention_weight_min": -51.110469818115234,
      "activations/layer7_attention_weight_max": 73.56803894042969,
      "activations/layer7_attention_weight_min": -69.44303131103516,
      "activations/layer8_attention_weight_max": 55.854976654052734,
      "activations/layer8_attention_weight_min": -51.1608772277832,
      "activations/layer9_attention_weight_max": 57.64164352416992,
      "activations/layer9_attention_weight_min": -50.67810821533203,
      "epoch": 15.2,
      "learning_rate": 5.2506060606060604e-05,
      "loss": 2.7631,
      "step": 261600
    },
    {
      "activations/layer0_attention_weight_max": 15.156028747558594,
      "activations/layer0_attention_weight_min": -13.512955665588379,
      "activations/layer10_attention_weight_max": 36.004364013671875,
      "activations/layer10_attention_weight_min": -34.998451232910156,
      "activations/layer11_attention_weight_max": 31.885826110839844,
      "activations/layer11_attention_weight_min": -32.28502655029297,
      "activations/layer12_attention_weight_max": 19.397085189819336,
      "activations/layer12_attention_weight_min": -26.631452560424805,
      "activations/layer13_attention_weight_max": 37.28773880004883,
      "activations/layer13_attention_weight_min": -27.120113372802734,
      "activations/layer14_attention_weight_max": 36.68632888793945,
      "activations/layer14_attention_weight_min": -31.507061004638672,
      "activations/layer15_attention_weight_max": 31.718957901000977,
      "activations/layer15_attention_weight_min": -31.174955368041992,
      "activations/layer16_attention_weight_max": 35.335689544677734,
      "activations/layer16_attention_weight_min": -32.471160888671875,
      "activations/layer17_attention_weight_max": 55.274627685546875,
      "activations/layer17_attention_weight_min": -47.2696647644043,
      "activations/layer18_attention_weight_max": 52.61614227294922,
      "activations/layer18_attention_weight_min": -44.68309783935547,
      "activations/layer19_attention_weight_max": 25.38621711730957,
      "activations/layer19_attention_weight_min": -26.232868194580078,
      "activations/layer1_attention_weight_max": 16.615751266479492,
      "activations/layer1_attention_weight_min": -15.199151039123535,
      "activations/layer20_attention_weight_max": 27.029041290283203,
      "activations/layer20_attention_weight_min": -23.197954177856445,
      "activations/layer21_attention_weight_max": 37.3336181640625,
      "activations/layer21_attention_weight_min": -24.867883682250977,
      "activations/layer22_attention_weight_max": 31.845687866210938,
      "activations/layer22_attention_weight_min": -30.11546516418457,
      "activations/layer23_attention_weight_max": 38.24150085449219,
      "activations/layer23_attention_weight_min": -26.380725860595703,
      "activations/layer2_attention_weight_max": 31.187103271484375,
      "activations/layer2_attention_weight_min": -30.512691497802734,
      "activations/layer3_attention_weight_max": 90.90367889404297,
      "activations/layer3_attention_weight_min": -92.9322509765625,
      "activations/layer4_attention_weight_max": 87.85810089111328,
      "activations/layer4_attention_weight_min": -88.99813842773438,
      "activations/layer5_attention_weight_max": 63.87881088256836,
      "activations/layer5_attention_weight_min": -74.18644714355469,
      "activations/layer6_attention_weight_max": 48.2901725769043,
      "activations/layer6_attention_weight_min": -51.76323318481445,
      "activations/layer7_attention_weight_max": 63.21669387817383,
      "activations/layer7_attention_weight_min": -58.950035095214844,
      "activations/layer8_attention_weight_max": 44.04865264892578,
      "activations/layer8_attention_weight_min": -46.119171142578125,
      "activations/layer9_attention_weight_max": 44.6351432800293,
      "activations/layer9_attention_weight_min": -44.5057258605957,
      "epoch": 15.2,
      "learning_rate": 5.2487121212121206e-05,
      "loss": 2.7536,
      "step": 261650
    },
    {
      "activations/layer0_attention_weight_max": 15.951605796813965,
      "activations/layer0_attention_weight_min": -13.82174015045166,
      "activations/layer10_attention_weight_max": 37.80598831176758,
      "activations/layer10_attention_weight_min": -36.67214584350586,
      "activations/layer11_attention_weight_max": 35.84309768676758,
      "activations/layer11_attention_weight_min": -36.22332763671875,
      "activations/layer12_attention_weight_max": 18.390625,
      "activations/layer12_attention_weight_min": -25.930524826049805,
      "activations/layer13_attention_weight_max": 38.113250732421875,
      "activations/layer13_attention_weight_min": -30.166261672973633,
      "activations/layer14_attention_weight_max": 37.971397399902344,
      "activations/layer14_attention_weight_min": -29.490869522094727,
      "activations/layer15_attention_weight_max": 33.94386291503906,
      "activations/layer15_attention_weight_min": -30.000988006591797,
      "activations/layer16_attention_weight_max": 34.787261962890625,
      "activations/layer16_attention_weight_min": -32.417442321777344,
      "activations/layer17_attention_weight_max": 54.87517166137695,
      "activations/layer17_attention_weight_min": -45.92993927001953,
      "activations/layer18_attention_weight_max": 51.069889068603516,
      "activations/layer18_attention_weight_min": -41.7859992980957,
      "activations/layer19_attention_weight_max": 23.26679039001465,
      "activations/layer19_attention_weight_min": -22.62144660949707,
      "activations/layer1_attention_weight_max": 15.659671783447266,
      "activations/layer1_attention_weight_min": -15.395322799682617,
      "activations/layer20_attention_weight_max": 23.397769927978516,
      "activations/layer20_attention_weight_min": -21.6894588470459,
      "activations/layer21_attention_weight_max": 36.2526969909668,
      "activations/layer21_attention_weight_min": -22.9913387298584,
      "activations/layer22_attention_weight_max": 32.07809829711914,
      "activations/layer22_attention_weight_min": -25.8818302154541,
      "activations/layer23_attention_weight_max": 34.81364059448242,
      "activations/layer23_attention_weight_min": -22.229217529296875,
      "activations/layer2_attention_weight_max": 32.84773254394531,
      "activations/layer2_attention_weight_min": -31.564952850341797,
      "activations/layer3_attention_weight_max": 95.13407897949219,
      "activations/layer3_attention_weight_min": -96.9249496459961,
      "activations/layer4_attention_weight_max": 94.31383514404297,
      "activations/layer4_attention_weight_min": -89.66948699951172,
      "activations/layer5_attention_weight_max": 64.6217269897461,
      "activations/layer5_attention_weight_min": -76.19407653808594,
      "activations/layer6_attention_weight_max": 50.711360931396484,
      "activations/layer6_attention_weight_min": -52.79767608642578,
      "activations/layer7_attention_weight_max": 59.80343246459961,
      "activations/layer7_attention_weight_min": -58.61311340332031,
      "activations/layer8_attention_weight_max": 45.83920669555664,
      "activations/layer8_attention_weight_min": -48.97724151611328,
      "activations/layer9_attention_weight_max": 46.79232406616211,
      "activations/layer9_attention_weight_min": -45.195709228515625,
      "epoch": 15.21,
      "learning_rate": 5.246818181818182e-05,
      "loss": 2.7616,
      "step": 261700
    },
    {
      "activations/layer0_attention_weight_max": 14.843128204345703,
      "activations/layer0_attention_weight_min": -13.680495262145996,
      "activations/layer10_attention_weight_max": 34.430625915527344,
      "activations/layer10_attention_weight_min": -34.23785400390625,
      "activations/layer11_attention_weight_max": 31.378482818603516,
      "activations/layer11_attention_weight_min": -32.876434326171875,
      "activations/layer12_attention_weight_max": 20.259260177612305,
      "activations/layer12_attention_weight_min": -24.248456954956055,
      "activations/layer13_attention_weight_max": 34.982452392578125,
      "activations/layer13_attention_weight_min": -27.360340118408203,
      "activations/layer14_attention_weight_max": 34.944129943847656,
      "activations/layer14_attention_weight_min": -33.86349105834961,
      "activations/layer15_attention_weight_max": 34.26239776611328,
      "activations/layer15_attention_weight_min": -31.738126754760742,
      "activations/layer16_attention_weight_max": 32.29043960571289,
      "activations/layer16_attention_weight_min": -31.281408309936523,
      "activations/layer17_attention_weight_max": 50.480960845947266,
      "activations/layer17_attention_weight_min": -44.64822006225586,
      "activations/layer18_attention_weight_max": 45.251304626464844,
      "activations/layer18_attention_weight_min": -38.294586181640625,
      "activations/layer19_attention_weight_max": 22.680295944213867,
      "activations/layer19_attention_weight_min": -21.85563087463379,
      "activations/layer1_attention_weight_max": 16.25784683227539,
      "activations/layer1_attention_weight_min": -13.810802459716797,
      "activations/layer20_attention_weight_max": 22.4354248046875,
      "activations/layer20_attention_weight_min": -23.43189239501953,
      "activations/layer21_attention_weight_max": 35.3637580871582,
      "activations/layer21_attention_weight_min": -22.295650482177734,
      "activations/layer22_attention_weight_max": 28.721683502197266,
      "activations/layer22_attention_weight_min": -29.29248809814453,
      "activations/layer23_attention_weight_max": 39.86900329589844,
      "activations/layer23_attention_weight_min": -24.414527893066406,
      "activations/layer2_attention_weight_max": 31.12401580810547,
      "activations/layer2_attention_weight_min": -30.59493064880371,
      "activations/layer3_attention_weight_max": 93.73268127441406,
      "activations/layer3_attention_weight_min": -92.44749450683594,
      "activations/layer4_attention_weight_max": 91.92459869384766,
      "activations/layer4_attention_weight_min": -87.15902709960938,
      "activations/layer5_attention_weight_max": 66.0252685546875,
      "activations/layer5_attention_weight_min": -70.61881256103516,
      "activations/layer6_attention_weight_max": 48.2713508605957,
      "activations/layer6_attention_weight_min": -49.03632736206055,
      "activations/layer7_attention_weight_max": 60.057273864746094,
      "activations/layer7_attention_weight_min": -62.02210998535156,
      "activations/layer8_attention_weight_max": 43.442413330078125,
      "activations/layer8_attention_weight_min": -46.682247161865234,
      "activations/layer9_attention_weight_max": 45.10780334472656,
      "activations/layer9_attention_weight_min": -43.59296798706055,
      "epoch": 15.21,
      "learning_rate": 5.244924242424242e-05,
      "loss": 2.7527,
      "step": 261750
    },
    {
      "activations/layer0_attention_weight_max": 16.1707820892334,
      "activations/layer0_attention_weight_min": -13.599279403686523,
      "activations/layer10_attention_weight_max": 37.11200714111328,
      "activations/layer10_attention_weight_min": -35.53772735595703,
      "activations/layer11_attention_weight_max": 33.17631530761719,
      "activations/layer11_attention_weight_min": -34.76970291137695,
      "activations/layer12_attention_weight_max": 19.986160278320312,
      "activations/layer12_attention_weight_min": -27.75835609436035,
      "activations/layer13_attention_weight_max": 46.44504165649414,
      "activations/layer13_attention_weight_min": -30.277915954589844,
      "activations/layer14_attention_weight_max": 35.337440490722656,
      "activations/layer14_attention_weight_min": -31.723506927490234,
      "activations/layer15_attention_weight_max": 34.233497619628906,
      "activations/layer15_attention_weight_min": -29.755666732788086,
      "activations/layer16_attention_weight_max": 33.72751998901367,
      "activations/layer16_attention_weight_min": -32.191368103027344,
      "activations/layer17_attention_weight_max": 53.94461441040039,
      "activations/layer17_attention_weight_min": -48.97502517700195,
      "activations/layer18_attention_weight_max": 48.65645980834961,
      "activations/layer18_attention_weight_min": -42.34624481201172,
      "activations/layer19_attention_weight_max": 23.66672706604004,
      "activations/layer19_attention_weight_min": -23.471107482910156,
      "activations/layer1_attention_weight_max": 16.2983455657959,
      "activations/layer1_attention_weight_min": -15.4077787399292,
      "activations/layer20_attention_weight_max": 25.08241844177246,
      "activations/layer20_attention_weight_min": -22.82581329345703,
      "activations/layer21_attention_weight_max": 41.934635162353516,
      "activations/layer21_attention_weight_min": -22.276432037353516,
      "activations/layer22_attention_weight_max": 30.417537689208984,
      "activations/layer22_attention_weight_min": -25.235567092895508,
      "activations/layer23_attention_weight_max": 37.6524772644043,
      "activations/layer23_attention_weight_min": -25.178451538085938,
      "activations/layer2_attention_weight_max": 33.78938674926758,
      "activations/layer2_attention_weight_min": -31.332548141479492,
      "activations/layer3_attention_weight_max": 96.0708236694336,
      "activations/layer3_attention_weight_min": -96.580810546875,
      "activations/layer4_attention_weight_max": 90.50676727294922,
      "activations/layer4_attention_weight_min": -86.71564483642578,
      "activations/layer5_attention_weight_max": 64.73176574707031,
      "activations/layer5_attention_weight_min": -74.71501159667969,
      "activations/layer6_attention_weight_max": 49.69776153564453,
      "activations/layer6_attention_weight_min": -49.96481704711914,
      "activations/layer7_attention_weight_max": 63.50532150268555,
      "activations/layer7_attention_weight_min": -64.13895416259766,
      "activations/layer8_attention_weight_max": 45.14419174194336,
      "activations/layer8_attention_weight_min": -51.71751022338867,
      "activations/layer9_attention_weight_max": 47.549476623535156,
      "activations/layer9_attention_weight_min": -48.116844177246094,
      "epoch": 15.21,
      "learning_rate": 5.2430303030303024e-05,
      "loss": 2.7673,
      "step": 261800
    },
    {
      "activations/layer0_attention_weight_max": 15.261198997497559,
      "activations/layer0_attention_weight_min": -13.391236305236816,
      "activations/layer10_attention_weight_max": 38.33829879760742,
      "activations/layer10_attention_weight_min": -37.87932586669922,
      "activations/layer11_attention_weight_max": 32.849910736083984,
      "activations/layer11_attention_weight_min": -33.41864776611328,
      "activations/layer12_attention_weight_max": 18.606748580932617,
      "activations/layer12_attention_weight_min": -26.186695098876953,
      "activations/layer13_attention_weight_max": 42.52152633666992,
      "activations/layer13_attention_weight_min": -31.29526138305664,
      "activations/layer14_attention_weight_max": 36.75187301635742,
      "activations/layer14_attention_weight_min": -31.668922424316406,
      "activations/layer15_attention_weight_max": 33.95362854003906,
      "activations/layer15_attention_weight_min": -31.00592613220215,
      "activations/layer16_attention_weight_max": 35.620201110839844,
      "activations/layer16_attention_weight_min": -33.86128616333008,
      "activations/layer17_attention_weight_max": 53.80327224731445,
      "activations/layer17_attention_weight_min": -46.073890686035156,
      "activations/layer18_attention_weight_max": 49.63264083862305,
      "activations/layer18_attention_weight_min": -39.74944305419922,
      "activations/layer19_attention_weight_max": 24.076303482055664,
      "activations/layer19_attention_weight_min": -22.778778076171875,
      "activations/layer1_attention_weight_max": 16.30025863647461,
      "activations/layer1_attention_weight_min": -15.453960418701172,
      "activations/layer20_attention_weight_max": 22.536773681640625,
      "activations/layer20_attention_weight_min": -20.995655059814453,
      "activations/layer21_attention_weight_max": 36.09912872314453,
      "activations/layer21_attention_weight_min": -24.742443084716797,
      "activations/layer22_attention_weight_max": 29.795549392700195,
      "activations/layer22_attention_weight_min": -26.044675827026367,
      "activations/layer23_attention_weight_max": 37.02410125732422,
      "activations/layer23_attention_weight_min": -23.977750778198242,
      "activations/layer2_attention_weight_max": 30.823246002197266,
      "activations/layer2_attention_weight_min": -29.897377014160156,
      "activations/layer3_attention_weight_max": 91.18896484375,
      "activations/layer3_attention_weight_min": -91.16927337646484,
      "activations/layer4_attention_weight_max": 94.8826904296875,
      "activations/layer4_attention_weight_min": -87.3545913696289,
      "activations/layer5_attention_weight_max": 66.02006530761719,
      "activations/layer5_attention_weight_min": -72.69017028808594,
      "activations/layer6_attention_weight_max": 49.77173614501953,
      "activations/layer6_attention_weight_min": -50.526634216308594,
      "activations/layer7_attention_weight_max": 63.41273498535156,
      "activations/layer7_attention_weight_min": -62.64154052734375,
      "activations/layer8_attention_weight_max": 47.5633659362793,
      "activations/layer8_attention_weight_min": -49.34518814086914,
      "activations/layer9_attention_weight_max": 48.74306869506836,
      "activations/layer9_attention_weight_min": -43.67854690551758,
      "epoch": 15.21,
      "learning_rate": 5.241136363636363e-05,
      "loss": 2.766,
      "step": 261850
    },
    {
      "activations/layer0_attention_weight_max": 15.148834228515625,
      "activations/layer0_attention_weight_min": -13.527301788330078,
      "activations/layer10_attention_weight_max": 39.611270904541016,
      "activations/layer10_attention_weight_min": -40.64168930053711,
      "activations/layer11_attention_weight_max": 33.844940185546875,
      "activations/layer11_attention_weight_min": -36.7750129699707,
      "activations/layer12_attention_weight_max": 20.850996017456055,
      "activations/layer12_attention_weight_min": -27.772565841674805,
      "activations/layer13_attention_weight_max": 38.2698860168457,
      "activations/layer13_attention_weight_min": -29.998119354248047,
      "activations/layer14_attention_weight_max": 35.95210266113281,
      "activations/layer14_attention_weight_min": -30.74341583251953,
      "activations/layer15_attention_weight_max": 34.2548713684082,
      "activations/layer15_attention_weight_min": -29.61927604675293,
      "activations/layer16_attention_weight_max": 33.37953186035156,
      "activations/layer16_attention_weight_min": -30.68340301513672,
      "activations/layer17_attention_weight_max": 54.4473991394043,
      "activations/layer17_attention_weight_min": -44.95985794067383,
      "activations/layer18_attention_weight_max": 50.767974853515625,
      "activations/layer18_attention_weight_min": -39.57896041870117,
      "activations/layer19_attention_weight_max": 23.745098114013672,
      "activations/layer19_attention_weight_min": -21.739065170288086,
      "activations/layer1_attention_weight_max": 16.129892349243164,
      "activations/layer1_attention_weight_min": -15.123079299926758,
      "activations/layer20_attention_weight_max": 22.4532527923584,
      "activations/layer20_attention_weight_min": -22.408031463623047,
      "activations/layer21_attention_weight_max": 36.75763702392578,
      "activations/layer21_attention_weight_min": -23.3116455078125,
      "activations/layer22_attention_weight_max": 28.851776123046875,
      "activations/layer22_attention_weight_min": -24.24346351623535,
      "activations/layer23_attention_weight_max": 42.25778579711914,
      "activations/layer23_attention_weight_min": -29.903369903564453,
      "activations/layer2_attention_weight_max": 33.118080139160156,
      "activations/layer2_attention_weight_min": -30.814268112182617,
      "activations/layer3_attention_weight_max": 93.29551696777344,
      "activations/layer3_attention_weight_min": -94.79859161376953,
      "activations/layer4_attention_weight_max": 92.23760223388672,
      "activations/layer4_attention_weight_min": -92.44234466552734,
      "activations/layer5_attention_weight_max": 67.01210021972656,
      "activations/layer5_attention_weight_min": -73.53700256347656,
      "activations/layer6_attention_weight_max": 49.35232925415039,
      "activations/layer6_attention_weight_min": -51.1230583190918,
      "activations/layer7_attention_weight_max": 62.69658660888672,
      "activations/layer7_attention_weight_min": -60.6138916015625,
      "activations/layer8_attention_weight_max": 49.758602142333984,
      "activations/layer8_attention_weight_min": -52.759613037109375,
      "activations/layer9_attention_weight_max": 46.78682327270508,
      "activations/layer9_attention_weight_min": -49.503692626953125,
      "epoch": 15.22,
      "learning_rate": 5.239242424242424e-05,
      "loss": 2.7655,
      "step": 261900
    },
    {
      "activations/layer0_attention_weight_max": 15.581754684448242,
      "activations/layer0_attention_weight_min": -13.85167121887207,
      "activations/layer10_attention_weight_max": 38.162803649902344,
      "activations/layer10_attention_weight_min": -37.991973876953125,
      "activations/layer11_attention_weight_max": 35.088321685791016,
      "activations/layer11_attention_weight_min": -34.280147552490234,
      "activations/layer12_attention_weight_max": 18.751523971557617,
      "activations/layer12_attention_weight_min": -30.312419891357422,
      "activations/layer13_attention_weight_max": 39.061767578125,
      "activations/layer13_attention_weight_min": -27.650102615356445,
      "activations/layer14_attention_weight_max": 37.981266021728516,
      "activations/layer14_attention_weight_min": -30.966609954833984,
      "activations/layer15_attention_weight_max": 34.250911712646484,
      "activations/layer15_attention_weight_min": -29.967945098876953,
      "activations/layer16_attention_weight_max": 35.703369140625,
      "activations/layer16_attention_weight_min": -31.183162689208984,
      "activations/layer17_attention_weight_max": 57.96807861328125,
      "activations/layer17_attention_weight_min": -46.34738540649414,
      "activations/layer18_attention_weight_max": 54.37485122680664,
      "activations/layer18_attention_weight_min": -39.70519256591797,
      "activations/layer19_attention_weight_max": 23.306745529174805,
      "activations/layer19_attention_weight_min": -24.109567642211914,
      "activations/layer1_attention_weight_max": 18.233373641967773,
      "activations/layer1_attention_weight_min": -15.092096328735352,
      "activations/layer20_attention_weight_max": 24.860570907592773,
      "activations/layer20_attention_weight_min": -24.904752731323242,
      "activations/layer21_attention_weight_max": 37.81364822387695,
      "activations/layer21_attention_weight_min": -23.765886306762695,
      "activations/layer22_attention_weight_max": 29.52477264404297,
      "activations/layer22_attention_weight_min": -26.784414291381836,
      "activations/layer23_attention_weight_max": 41.30727767944336,
      "activations/layer23_attention_weight_min": -24.658109664916992,
      "activations/layer2_attention_weight_max": 31.443906784057617,
      "activations/layer2_attention_weight_min": -30.501880645751953,
      "activations/layer3_attention_weight_max": 93.59175872802734,
      "activations/layer3_attention_weight_min": -96.16634368896484,
      "activations/layer4_attention_weight_max": 92.73992919921875,
      "activations/layer4_attention_weight_min": -89.65069580078125,
      "activations/layer5_attention_weight_max": 67.79256439208984,
      "activations/layer5_attention_weight_min": -73.33419799804688,
      "activations/layer6_attention_weight_max": 50.765464782714844,
      "activations/layer6_attention_weight_min": -49.9199104309082,
      "activations/layer7_attention_weight_max": 64.34674072265625,
      "activations/layer7_attention_weight_min": -61.42118835449219,
      "activations/layer8_attention_weight_max": 46.08943176269531,
      "activations/layer8_attention_weight_min": -46.51496505737305,
      "activations/layer9_attention_weight_max": 46.6352653503418,
      "activations/layer9_attention_weight_min": -46.871910095214844,
      "epoch": 15.22,
      "learning_rate": 5.237348484848484e-05,
      "loss": 2.7443,
      "step": 261950
    },
    {
      "activations/layer0_attention_weight_max": 15.787349700927734,
      "activations/layer0_attention_weight_min": -13.86962890625,
      "activations/layer10_attention_weight_max": 37.63917541503906,
      "activations/layer10_attention_weight_min": -36.2568473815918,
      "activations/layer11_attention_weight_max": 31.731678009033203,
      "activations/layer11_attention_weight_min": -33.367279052734375,
      "activations/layer12_attention_weight_max": 19.31355094909668,
      "activations/layer12_attention_weight_min": -24.602088928222656,
      "activations/layer13_attention_weight_max": 34.804420471191406,
      "activations/layer13_attention_weight_min": -30.68412208557129,
      "activations/layer14_attention_weight_max": 36.52129364013672,
      "activations/layer14_attention_weight_min": -32.56694030761719,
      "activations/layer15_attention_weight_max": 32.492584228515625,
      "activations/layer15_attention_weight_min": -31.29400634765625,
      "activations/layer16_attention_weight_max": 33.82323455810547,
      "activations/layer16_attention_weight_min": -31.85520362854004,
      "activations/layer17_attention_weight_max": 58.26996612548828,
      "activations/layer17_attention_weight_min": -48.15132522583008,
      "activations/layer18_attention_weight_max": 52.78143310546875,
      "activations/layer18_attention_weight_min": -43.230499267578125,
      "activations/layer19_attention_weight_max": 24.04071617126465,
      "activations/layer19_attention_weight_min": -22.69196128845215,
      "activations/layer1_attention_weight_max": 16.549589157104492,
      "activations/layer1_attention_weight_min": -16.0596981048584,
      "activations/layer20_attention_weight_max": 24.9949951171875,
      "activations/layer20_attention_weight_min": -24.97088050842285,
      "activations/layer21_attention_weight_max": 39.7312126159668,
      "activations/layer21_attention_weight_min": -21.0782470703125,
      "activations/layer22_attention_weight_max": 29.35456085205078,
      "activations/layer22_attention_weight_min": -25.941484451293945,
      "activations/layer23_attention_weight_max": 39.635074615478516,
      "activations/layer23_attention_weight_min": -23.81848907470703,
      "activations/layer2_attention_weight_max": 31.615055084228516,
      "activations/layer2_attention_weight_min": -30.029037475585938,
      "activations/layer3_attention_weight_max": 94.28570556640625,
      "activations/layer3_attention_weight_min": -92.9690933227539,
      "activations/layer4_attention_weight_max": 93.9078598022461,
      "activations/layer4_attention_weight_min": -90.43251037597656,
      "activations/layer5_attention_weight_max": 70.15298461914062,
      "activations/layer5_attention_weight_min": -80.4906997680664,
      "activations/layer6_attention_weight_max": 51.91861343383789,
      "activations/layer6_attention_weight_min": -52.78707504272461,
      "activations/layer7_attention_weight_max": 67.09624481201172,
      "activations/layer7_attention_weight_min": -61.58891296386719,
      "activations/layer8_attention_weight_max": 46.24427032470703,
      "activations/layer8_attention_weight_min": -47.97896194458008,
      "activations/layer9_attention_weight_max": 44.2126350402832,
      "activations/layer9_attention_weight_min": -44.95455551147461,
      "epoch": 15.22,
      "learning_rate": 5.235454545454545e-05,
      "loss": 2.7601,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4744,
      "eval_samples_per_second": 506.7,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4744,
      "eval_openwebtext_samples_per_second": 506.7,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9297,
      "eval_wikitext_samples_per_second": 236.307,
      "step": 262000
    },
    {
      "epoch": 15.22,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.5348,
      "eval_lambada_samples_per_second": 510.655,
      "step": 262000
    },
    {
      "activations/layer0_attention_weight_max": 14.759618759155273,
      "activations/layer0_attention_weight_min": -13.590240478515625,
      "activations/layer10_attention_weight_max": 35.38619613647461,
      "activations/layer10_attention_weight_min": -36.80815505981445,
      "activations/layer11_attention_weight_max": 32.1220817565918,
      "activations/layer11_attention_weight_min": -34.446693420410156,
      "activations/layer12_attention_weight_max": 17.196481704711914,
      "activations/layer12_attention_weight_min": -25.44947052001953,
      "activations/layer13_attention_weight_max": 33.480159759521484,
      "activations/layer13_attention_weight_min": -28.69601058959961,
      "activations/layer14_attention_weight_max": 32.355777740478516,
      "activations/layer14_attention_weight_min": -31.610546112060547,
      "activations/layer15_attention_weight_max": 31.438488006591797,
      "activations/layer15_attention_weight_min": -30.92547607421875,
      "activations/layer16_attention_weight_max": 31.550853729248047,
      "activations/layer16_attention_weight_min": -33.3880729675293,
      "activations/layer17_attention_weight_max": 50.843536376953125,
      "activations/layer17_attention_weight_min": -49.79738998413086,
      "activations/layer18_attention_weight_max": 45.755409240722656,
      "activations/layer18_attention_weight_min": -40.297889709472656,
      "activations/layer19_attention_weight_max": 21.53506851196289,
      "activations/layer19_attention_weight_min": -21.851850509643555,
      "activations/layer1_attention_weight_max": 16.66085433959961,
      "activations/layer1_attention_weight_min": -16.20651626586914,
      "activations/layer20_attention_weight_max": 20.6088809967041,
      "activations/layer20_attention_weight_min": -24.334718704223633,
      "activations/layer21_attention_weight_max": 33.51913070678711,
      "activations/layer21_attention_weight_min": -21.774280548095703,
      "activations/layer22_attention_weight_max": 29.14897918701172,
      "activations/layer22_attention_weight_min": -25.5308895111084,
      "activations/layer23_attention_weight_max": 34.941192626953125,
      "activations/layer23_attention_weight_min": -25.11121940612793,
      "activations/layer2_attention_weight_max": 32.72098159790039,
      "activations/layer2_attention_weight_min": -29.922231674194336,
      "activations/layer3_attention_weight_max": 91.24077606201172,
      "activations/layer3_attention_weight_min": -92.71509552001953,
      "activations/layer4_attention_weight_max": 87.23755645751953,
      "activations/layer4_attention_weight_min": -89.65381622314453,
      "activations/layer5_attention_weight_max": 64.1094970703125,
      "activations/layer5_attention_weight_min": -75.16618347167969,
      "activations/layer6_attention_weight_max": 49.06354904174805,
      "activations/layer6_attention_weight_min": -51.35686492919922,
      "activations/layer7_attention_weight_max": 60.231388092041016,
      "activations/layer7_attention_weight_min": -61.114742279052734,
      "activations/layer8_attention_weight_max": 42.62272262573242,
      "activations/layer8_attention_weight_min": -46.41630172729492,
      "activations/layer9_attention_weight_max": 43.21982955932617,
      "activations/layer9_attention_weight_min": -44.893619537353516,
      "epoch": 15.23,
      "learning_rate": 5.233560606060605e-05,
      "loss": 2.7637,
      "step": 262050
    },
    {
      "activations/layer0_attention_weight_max": 15.590197563171387,
      "activations/layer0_attention_weight_min": -13.361977577209473,
      "activations/layer10_attention_weight_max": 37.20885467529297,
      "activations/layer10_attention_weight_min": -37.19377517700195,
      "activations/layer11_attention_weight_max": 34.158416748046875,
      "activations/layer11_attention_weight_min": -34.73292541503906,
      "activations/layer12_attention_weight_max": 20.772560119628906,
      "activations/layer12_attention_weight_min": -24.08981704711914,
      "activations/layer13_attention_weight_max": 34.65827560424805,
      "activations/layer13_attention_weight_min": -28.02232551574707,
      "activations/layer14_attention_weight_max": 35.257972717285156,
      "activations/layer14_attention_weight_min": -32.34288024902344,
      "activations/layer15_attention_weight_max": 35.62907791137695,
      "activations/layer15_attention_weight_min": -30.962926864624023,
      "activations/layer16_attention_weight_max": 33.92290496826172,
      "activations/layer16_attention_weight_min": -31.953508377075195,
      "activations/layer17_attention_weight_max": 54.52652359008789,
      "activations/layer17_attention_weight_min": -45.45600128173828,
      "activations/layer18_attention_weight_max": 50.23546600341797,
      "activations/layer18_attention_weight_min": -40.34939193725586,
      "activations/layer19_attention_weight_max": 24.919212341308594,
      "activations/layer19_attention_weight_min": -23.2545108795166,
      "activations/layer1_attention_weight_max": 16.163785934448242,
      "activations/layer1_attention_weight_min": -13.71005630493164,
      "activations/layer20_attention_weight_max": 24.75811195373535,
      "activations/layer20_attention_weight_min": -22.045345306396484,
      "activations/layer21_attention_weight_max": 37.2309684753418,
      "activations/layer21_attention_weight_min": -20.216270446777344,
      "activations/layer22_attention_weight_max": 30.405494689941406,
      "activations/layer22_attention_weight_min": -27.244844436645508,
      "activations/layer23_attention_weight_max": 37.93473434448242,
      "activations/layer23_attention_weight_min": -25.029571533203125,
      "activations/layer2_attention_weight_max": 30.82114028930664,
      "activations/layer2_attention_weight_min": -29.801185607910156,
      "activations/layer3_attention_weight_max": 93.90347290039062,
      "activations/layer3_attention_weight_min": -97.86166381835938,
      "activations/layer4_attention_weight_max": 92.61572265625,
      "activations/layer4_attention_weight_min": -89.73625183105469,
      "activations/layer5_attention_weight_max": 69.19758605957031,
      "activations/layer5_attention_weight_min": -73.54969787597656,
      "activations/layer6_attention_weight_max": 52.854530334472656,
      "activations/layer6_attention_weight_min": -52.54096984863281,
      "activations/layer7_attention_weight_max": 65.13728332519531,
      "activations/layer7_attention_weight_min": -60.269832611083984,
      "activations/layer8_attention_weight_max": 48.29988098144531,
      "activations/layer8_attention_weight_min": -49.46193313598633,
      "activations/layer9_attention_weight_max": 46.62448501586914,
      "activations/layer9_attention_weight_min": -48.45045852661133,
      "epoch": 15.23,
      "learning_rate": 5.231666666666667e-05,
      "loss": 2.7398,
      "step": 262100
    },
    {
      "activations/layer0_attention_weight_max": 15.059879302978516,
      "activations/layer0_attention_weight_min": -13.767695426940918,
      "activations/layer10_attention_weight_max": 35.632835388183594,
      "activations/layer10_attention_weight_min": -35.45869445800781,
      "activations/layer11_attention_weight_max": 32.378849029541016,
      "activations/layer11_attention_weight_min": -34.09016418457031,
      "activations/layer12_attention_weight_max": 22.058752059936523,
      "activations/layer12_attention_weight_min": -23.5982666015625,
      "activations/layer13_attention_weight_max": 35.30270004272461,
      "activations/layer13_attention_weight_min": -27.8454532623291,
      "activations/layer14_attention_weight_max": 35.80863952636719,
      "activations/layer14_attention_weight_min": -30.535404205322266,
      "activations/layer15_attention_weight_max": 33.15861511230469,
      "activations/layer15_attention_weight_min": -32.16616439819336,
      "activations/layer16_attention_weight_max": 32.44182205200195,
      "activations/layer16_attention_weight_min": -31.214574813842773,
      "activations/layer17_attention_weight_max": 52.570411682128906,
      "activations/layer17_attention_weight_min": -49.77631378173828,
      "activations/layer18_attention_weight_max": 48.472694396972656,
      "activations/layer18_attention_weight_min": -41.68758773803711,
      "activations/layer19_attention_weight_max": 24.26338005065918,
      "activations/layer19_attention_weight_min": -23.1238956451416,
      "activations/layer1_attention_weight_max": 16.14414405822754,
      "activations/layer1_attention_weight_min": -14.476618766784668,
      "activations/layer20_attention_weight_max": 23.366975784301758,
      "activations/layer20_attention_weight_min": -24.101953506469727,
      "activations/layer21_attention_weight_max": 34.23963165283203,
      "activations/layer21_attention_weight_min": -25.88347053527832,
      "activations/layer22_attention_weight_max": 28.776575088500977,
      "activations/layer22_attention_weight_min": -27.185733795166016,
      "activations/layer23_attention_weight_max": 35.70166778564453,
      "activations/layer23_attention_weight_min": -23.547332763671875,
      "activations/layer2_attention_weight_max": 30.135536193847656,
      "activations/layer2_attention_weight_min": -29.742277145385742,
      "activations/layer3_attention_weight_max": 85.77705383300781,
      "activations/layer3_attention_weight_min": -95.78707122802734,
      "activations/layer4_attention_weight_max": 89.56136322021484,
      "activations/layer4_attention_weight_min": -85.87786865234375,
      "activations/layer5_attention_weight_max": 68.01104736328125,
      "activations/layer5_attention_weight_min": -73.89291381835938,
      "activations/layer6_attention_weight_max": 50.30577087402344,
      "activations/layer6_attention_weight_min": -51.74116897583008,
      "activations/layer7_attention_weight_max": 59.035972595214844,
      "activations/layer7_attention_weight_min": -60.30946350097656,
      "activations/layer8_attention_weight_max": 45.491943359375,
      "activations/layer8_attention_weight_min": -49.549312591552734,
      "activations/layer9_attention_weight_max": 45.209476470947266,
      "activations/layer9_attention_weight_min": -45.899169921875,
      "epoch": 15.23,
      "learning_rate": 5.229772727272727e-05,
      "loss": 2.7541,
      "step": 262150
    },
    {
      "activations/layer0_attention_weight_max": 15.414515495300293,
      "activations/layer0_attention_weight_min": -13.103031158447266,
      "activations/layer10_attention_weight_max": 41.366573333740234,
      "activations/layer10_attention_weight_min": -38.27379608154297,
      "activations/layer11_attention_weight_max": 38.059303283691406,
      "activations/layer11_attention_weight_min": -34.10917282104492,
      "activations/layer12_attention_weight_max": 17.328289031982422,
      "activations/layer12_attention_weight_min": -25.628602981567383,
      "activations/layer13_attention_weight_max": 44.54085922241211,
      "activations/layer13_attention_weight_min": -31.268421173095703,
      "activations/layer14_attention_weight_max": 44.33916473388672,
      "activations/layer14_attention_weight_min": -31.030818939208984,
      "activations/layer15_attention_weight_max": 38.82706069946289,
      "activations/layer15_attention_weight_min": -30.580965042114258,
      "activations/layer16_attention_weight_max": 35.969356536865234,
      "activations/layer16_attention_weight_min": -32.558998107910156,
      "activations/layer17_attention_weight_max": 56.52664566040039,
      "activations/layer17_attention_weight_min": -47.57994079589844,
      "activations/layer18_attention_weight_max": 54.83916473388672,
      "activations/layer18_attention_weight_min": -43.141231536865234,
      "activations/layer19_attention_weight_max": 26.00693130493164,
      "activations/layer19_attention_weight_min": -23.540897369384766,
      "activations/layer1_attention_weight_max": 16.729286193847656,
      "activations/layer1_attention_weight_min": -15.612726211547852,
      "activations/layer20_attention_weight_max": 26.102842330932617,
      "activations/layer20_attention_weight_min": -21.9329776763916,
      "activations/layer21_attention_weight_max": 47.69089126586914,
      "activations/layer21_attention_weight_min": -21.036550521850586,
      "activations/layer22_attention_weight_max": 29.18436050415039,
      "activations/layer22_attention_weight_min": -25.34686851501465,
      "activations/layer23_attention_weight_max": 35.491371154785156,
      "activations/layer23_attention_weight_min": -23.1448974609375,
      "activations/layer2_attention_weight_max": 34.16156005859375,
      "activations/layer2_attention_weight_min": -32.3764533996582,
      "activations/layer3_attention_weight_max": 95.7322998046875,
      "activations/layer3_attention_weight_min": -98.09757995605469,
      "activations/layer4_attention_weight_max": 95.85259246826172,
      "activations/layer4_attention_weight_min": -91.16861724853516,
      "activations/layer5_attention_weight_max": 70.56440734863281,
      "activations/layer5_attention_weight_min": -71.60430908203125,
      "activations/layer6_attention_weight_max": 55.307098388671875,
      "activations/layer6_attention_weight_min": -48.385677337646484,
      "activations/layer7_attention_weight_max": 63.57209396362305,
      "activations/layer7_attention_weight_min": -61.59723663330078,
      "activations/layer8_attention_weight_max": 51.02989959716797,
      "activations/layer8_attention_weight_min": -48.87858963012695,
      "activations/layer9_attention_weight_max": 50.583587646484375,
      "activations/layer9_attention_weight_min": -47.16971969604492,
      "epoch": 15.24,
      "learning_rate": 5.227878787878787e-05,
      "loss": 2.7537,
      "step": 262200
    },
    {
      "activations/layer0_attention_weight_max": 16.039758682250977,
      "activations/layer0_attention_weight_min": -13.492431640625,
      "activations/layer10_attention_weight_max": 38.19627380371094,
      "activations/layer10_attention_weight_min": -36.63703918457031,
      "activations/layer11_attention_weight_max": 33.59015655517578,
      "activations/layer11_attention_weight_min": -33.86299514770508,
      "activations/layer12_attention_weight_max": 17.188522338867188,
      "activations/layer12_attention_weight_min": -26.619930267333984,
      "activations/layer13_attention_weight_max": 43.22024917602539,
      "activations/layer13_attention_weight_min": -29.85188865661621,
      "activations/layer14_attention_weight_max": 38.05213928222656,
      "activations/layer14_attention_weight_min": -29.19348907470703,
      "activations/layer15_attention_weight_max": 35.73017501831055,
      "activations/layer15_attention_weight_min": -29.83100128173828,
      "activations/layer16_attention_weight_max": 35.99710464477539,
      "activations/layer16_attention_weight_min": -34.73966979980469,
      "activations/layer17_attention_weight_max": 52.281036376953125,
      "activations/layer17_attention_weight_min": -48.8691520690918,
      "activations/layer18_attention_weight_max": 51.720890045166016,
      "activations/layer18_attention_weight_min": -38.7766227722168,
      "activations/layer19_attention_weight_max": 30.946210861206055,
      "activations/layer19_attention_weight_min": -20.225183486938477,
      "activations/layer1_attention_weight_max": 15.915364265441895,
      "activations/layer1_attention_weight_min": -17.130252838134766,
      "activations/layer20_attention_weight_max": 28.53697967529297,
      "activations/layer20_attention_weight_min": -19.512022018432617,
      "activations/layer21_attention_weight_max": 47.010555267333984,
      "activations/layer21_attention_weight_min": -24.106351852416992,
      "activations/layer22_attention_weight_max": 31.585676193237305,
      "activations/layer22_attention_weight_min": -24.08680534362793,
      "activations/layer23_attention_weight_max": 40.006874084472656,
      "activations/layer23_attention_weight_min": -26.0339412689209,
      "activations/layer2_attention_weight_max": 32.28395462036133,
      "activations/layer2_attention_weight_min": -32.496063232421875,
      "activations/layer3_attention_weight_max": 98.67158508300781,
      "activations/layer3_attention_weight_min": -100.92072296142578,
      "activations/layer4_attention_weight_max": 91.46416473388672,
      "activations/layer4_attention_weight_min": -90.0907211303711,
      "activations/layer5_attention_weight_max": 68.60298156738281,
      "activations/layer5_attention_weight_min": -79.90560913085938,
      "activations/layer6_attention_weight_max": 52.521873474121094,
      "activations/layer6_attention_weight_min": -52.07543182373047,
      "activations/layer7_attention_weight_max": 62.40751647949219,
      "activations/layer7_attention_weight_min": -66.25532531738281,
      "activations/layer8_attention_weight_max": 49.48085403442383,
      "activations/layer8_attention_weight_min": -51.1004753112793,
      "activations/layer9_attention_weight_max": 48.611549377441406,
      "activations/layer9_attention_weight_min": -46.752132415771484,
      "epoch": 15.24,
      "learning_rate": 5.225984848484848e-05,
      "loss": 2.7542,
      "step": 262250
    },
    {
      "activations/layer0_attention_weight_max": 16.41791534423828,
      "activations/layer0_attention_weight_min": -14.026485443115234,
      "activations/layer10_attention_weight_max": 36.38710403442383,
      "activations/layer10_attention_weight_min": -36.23970413208008,
      "activations/layer11_attention_weight_max": 34.09129333496094,
      "activations/layer11_attention_weight_min": -35.40868377685547,
      "activations/layer12_attention_weight_max": 17.990869522094727,
      "activations/layer12_attention_weight_min": -29.92733383178711,
      "activations/layer13_attention_weight_max": 35.01461410522461,
      "activations/layer13_attention_weight_min": -29.810466766357422,
      "activations/layer14_attention_weight_max": 40.24320983886719,
      "activations/layer14_attention_weight_min": -27.969083786010742,
      "activations/layer15_attention_weight_max": 36.07959747314453,
      "activations/layer15_attention_weight_min": -28.132932662963867,
      "activations/layer16_attention_weight_max": 35.536705017089844,
      "activations/layer16_attention_weight_min": -31.323368072509766,
      "activations/layer17_attention_weight_max": 51.42816925048828,
      "activations/layer17_attention_weight_min": -45.46807861328125,
      "activations/layer18_attention_weight_max": 45.17361068725586,
      "activations/layer18_attention_weight_min": -41.68399429321289,
      "activations/layer19_attention_weight_max": 23.6634578704834,
      "activations/layer19_attention_weight_min": -24.979732513427734,
      "activations/layer1_attention_weight_max": 16.88215446472168,
      "activations/layer1_attention_weight_min": -16.499996185302734,
      "activations/layer20_attention_weight_max": 25.83956527709961,
      "activations/layer20_attention_weight_min": -26.504566192626953,
      "activations/layer21_attention_weight_max": 42.67361831665039,
      "activations/layer21_attention_weight_min": -24.04851531982422,
      "activations/layer22_attention_weight_max": 30.186019897460938,
      "activations/layer22_attention_weight_min": -27.26059913635254,
      "activations/layer23_attention_weight_max": 39.18423843383789,
      "activations/layer23_attention_weight_min": -25.735946655273438,
      "activations/layer2_attention_weight_max": 33.622371673583984,
      "activations/layer2_attention_weight_min": -33.74288558959961,
      "activations/layer3_attention_weight_max": 99.26334381103516,
      "activations/layer3_attention_weight_min": -101.88677215576172,
      "activations/layer4_attention_weight_max": 93.33751678466797,
      "activations/layer4_attention_weight_min": -91.80583953857422,
      "activations/layer5_attention_weight_max": 66.52757263183594,
      "activations/layer5_attention_weight_min": -72.33429718017578,
      "activations/layer6_attention_weight_max": 54.588951110839844,
      "activations/layer6_attention_weight_min": -55.18625259399414,
      "activations/layer7_attention_weight_max": 66.84806823730469,
      "activations/layer7_attention_weight_min": -68.9695053100586,
      "activations/layer8_attention_weight_max": 49.710853576660156,
      "activations/layer8_attention_weight_min": -48.70425033569336,
      "activations/layer9_attention_weight_max": 54.67757797241211,
      "activations/layer9_attention_weight_min": -48.98125076293945,
      "epoch": 15.24,
      "learning_rate": 5.2240909090909086e-05,
      "loss": 2.7497,
      "step": 262300
    },
    {
      "activations/layer0_attention_weight_max": 14.774410247802734,
      "activations/layer0_attention_weight_min": -13.276341438293457,
      "activations/layer10_attention_weight_max": 37.808990478515625,
      "activations/layer10_attention_weight_min": -35.868743896484375,
      "activations/layer11_attention_weight_max": 32.56908416748047,
      "activations/layer11_attention_weight_min": -35.940494537353516,
      "activations/layer12_attention_weight_max": 18.807424545288086,
      "activations/layer12_attention_weight_min": -23.800186157226562,
      "activations/layer13_attention_weight_max": 38.71394729614258,
      "activations/layer13_attention_weight_min": -31.34716796875,
      "activations/layer14_attention_weight_max": 35.642120361328125,
      "activations/layer14_attention_weight_min": -30.442249298095703,
      "activations/layer15_attention_weight_max": 38.88252258300781,
      "activations/layer15_attention_weight_min": -31.148609161376953,
      "activations/layer16_attention_weight_max": 33.876075744628906,
      "activations/layer16_attention_weight_min": -32.311073303222656,
      "activations/layer17_attention_weight_max": 53.7220458984375,
      "activations/layer17_attention_weight_min": -46.11940383911133,
      "activations/layer18_attention_weight_max": 51.344295501708984,
      "activations/layer18_attention_weight_min": -40.7962760925293,
      "activations/layer19_attention_weight_max": 25.37442970275879,
      "activations/layer19_attention_weight_min": -22.711069107055664,
      "activations/layer1_attention_weight_max": 16.067716598510742,
      "activations/layer1_attention_weight_min": -16.362075805664062,
      "activations/layer20_attention_weight_max": 24.9789981842041,
      "activations/layer20_attention_weight_min": -22.254011154174805,
      "activations/layer21_attention_weight_max": 46.15837097167969,
      "activations/layer21_attention_weight_min": -25.6727237701416,
      "activations/layer22_attention_weight_max": 34.16603088378906,
      "activations/layer22_attention_weight_min": -27.13041114807129,
      "activations/layer23_attention_weight_max": 39.73399353027344,
      "activations/layer23_attention_weight_min": -24.586118698120117,
      "activations/layer2_attention_weight_max": 32.834503173828125,
      "activations/layer2_attention_weight_min": -32.032264709472656,
      "activations/layer3_attention_weight_max": 92.74369049072266,
      "activations/layer3_attention_weight_min": -99.02734375,
      "activations/layer4_attention_weight_max": 93.2832260131836,
      "activations/layer4_attention_weight_min": -89.7252197265625,
      "activations/layer5_attention_weight_max": 67.41488647460938,
      "activations/layer5_attention_weight_min": -73.7248764038086,
      "activations/layer6_attention_weight_max": 50.91750717163086,
      "activations/layer6_attention_weight_min": -49.86323547363281,
      "activations/layer7_attention_weight_max": 61.87800216674805,
      "activations/layer7_attention_weight_min": -60.00208282470703,
      "activations/layer8_attention_weight_max": 45.92679977416992,
      "activations/layer8_attention_weight_min": -46.591922760009766,
      "activations/layer9_attention_weight_max": 46.03744125366211,
      "activations/layer9_attention_weight_min": -45.994041442871094,
      "epoch": 15.24,
      "learning_rate": 5.2221969696969694e-05,
      "loss": 2.7555,
      "step": 262350
    },
    {
      "activations/layer0_attention_weight_max": 14.531218528747559,
      "activations/layer0_attention_weight_min": -13.840494155883789,
      "activations/layer10_attention_weight_max": 35.855072021484375,
      "activations/layer10_attention_weight_min": -34.08605194091797,
      "activations/layer11_attention_weight_max": 33.30555725097656,
      "activations/layer11_attention_weight_min": -32.3053092956543,
      "activations/layer12_attention_weight_max": 19.501962661743164,
      "activations/layer12_attention_weight_min": -27.345949172973633,
      "activations/layer13_attention_weight_max": 39.252418518066406,
      "activations/layer13_attention_weight_min": -27.564834594726562,
      "activations/layer14_attention_weight_max": 37.868404388427734,
      "activations/layer14_attention_weight_min": -29.257160186767578,
      "activations/layer15_attention_weight_max": 34.589141845703125,
      "activations/layer15_attention_weight_min": -28.725252151489258,
      "activations/layer16_attention_weight_max": 32.90582275390625,
      "activations/layer16_attention_weight_min": -31.611759185791016,
      "activations/layer17_attention_weight_max": 52.476654052734375,
      "activations/layer17_attention_weight_min": -46.54754638671875,
      "activations/layer18_attention_weight_max": 46.94316864013672,
      "activations/layer18_attention_weight_min": -38.320919036865234,
      "activations/layer19_attention_weight_max": 23.72303581237793,
      "activations/layer19_attention_weight_min": -24.71059799194336,
      "activations/layer1_attention_weight_max": 16.416906356811523,
      "activations/layer1_attention_weight_min": -16.413223266601562,
      "activations/layer20_attention_weight_max": 22.120990753173828,
      "activations/layer20_attention_weight_min": -21.863113403320312,
      "activations/layer21_attention_weight_max": 34.462833404541016,
      "activations/layer21_attention_weight_min": -23.090660095214844,
      "activations/layer22_attention_weight_max": 33.23237991333008,
      "activations/layer22_attention_weight_min": -26.18562889099121,
      "activations/layer23_attention_weight_max": 35.71920394897461,
      "activations/layer23_attention_weight_min": -25.17927360534668,
      "activations/layer2_attention_weight_max": 33.934608459472656,
      "activations/layer2_attention_weight_min": -30.959033966064453,
      "activations/layer3_attention_weight_max": 92.09611511230469,
      "activations/layer3_attention_weight_min": -89.2100601196289,
      "activations/layer4_attention_weight_max": 88.70061492919922,
      "activations/layer4_attention_weight_min": -85.60921478271484,
      "activations/layer5_attention_weight_max": 65.83987426757812,
      "activations/layer5_attention_weight_min": -74.31331634521484,
      "activations/layer6_attention_weight_max": 49.606300354003906,
      "activations/layer6_attention_weight_min": -48.36930847167969,
      "activations/layer7_attention_weight_max": 60.38602066040039,
      "activations/layer7_attention_weight_min": -57.90608596801758,
      "activations/layer8_attention_weight_max": 46.6140022277832,
      "activations/layer8_attention_weight_min": -45.90699768066406,
      "activations/layer9_attention_weight_max": 48.27537155151367,
      "activations/layer9_attention_weight_min": -44.011810302734375,
      "epoch": 15.25,
      "learning_rate": 5.2203030303030296e-05,
      "loss": 2.7651,
      "step": 262400
    },
    {
      "activations/layer0_attention_weight_max": 15.73348331451416,
      "activations/layer0_attention_weight_min": -13.563560485839844,
      "activations/layer10_attention_weight_max": 35.31492614746094,
      "activations/layer10_attention_weight_min": -34.87855529785156,
      "activations/layer11_attention_weight_max": 32.68003845214844,
      "activations/layer11_attention_weight_min": -32.64046096801758,
      "activations/layer12_attention_weight_max": 20.040626525878906,
      "activations/layer12_attention_weight_min": -24.180864334106445,
      "activations/layer13_attention_weight_max": 36.558101654052734,
      "activations/layer13_attention_weight_min": -28.272727966308594,
      "activations/layer14_attention_weight_max": 35.275733947753906,
      "activations/layer14_attention_weight_min": -30.276920318603516,
      "activations/layer15_attention_weight_max": 31.855627059936523,
      "activations/layer15_attention_weight_min": -29.785755157470703,
      "activations/layer16_attention_weight_max": 32.834800720214844,
      "activations/layer16_attention_weight_min": -32.27206802368164,
      "activations/layer17_attention_weight_max": 52.48637771606445,
      "activations/layer17_attention_weight_min": -46.77449417114258,
      "activations/layer18_attention_weight_max": 45.76584243774414,
      "activations/layer18_attention_weight_min": -41.64934539794922,
      "activations/layer19_attention_weight_max": 23.11165428161621,
      "activations/layer19_attention_weight_min": -24.800817489624023,
      "activations/layer1_attention_weight_max": 15.921198844909668,
      "activations/layer1_attention_weight_min": -17.245595932006836,
      "activations/layer20_attention_weight_max": 23.452104568481445,
      "activations/layer20_attention_weight_min": -24.222003936767578,
      "activations/layer21_attention_weight_max": 34.61655807495117,
      "activations/layer21_attention_weight_min": -22.105548858642578,
      "activations/layer22_attention_weight_max": 29.728546142578125,
      "activations/layer22_attention_weight_min": -27.565725326538086,
      "activations/layer23_attention_weight_max": 38.3337287902832,
      "activations/layer23_attention_weight_min": -24.713804244995117,
      "activations/layer2_attention_weight_max": 32.410423278808594,
      "activations/layer2_attention_weight_min": -30.85588836669922,
      "activations/layer3_attention_weight_max": 92.72515869140625,
      "activations/layer3_attention_weight_min": -93.6507568359375,
      "activations/layer4_attention_weight_max": 92.38911437988281,
      "activations/layer4_attention_weight_min": -86.81376647949219,
      "activations/layer5_attention_weight_max": 65.39991760253906,
      "activations/layer5_attention_weight_min": -74.61702728271484,
      "activations/layer6_attention_weight_max": 52.0377311706543,
      "activations/layer6_attention_weight_min": -50.6447868347168,
      "activations/layer7_attention_weight_max": 61.1713981628418,
      "activations/layer7_attention_weight_min": -59.380252838134766,
      "activations/layer8_attention_weight_max": 45.44527816772461,
      "activations/layer8_attention_weight_min": -47.488189697265625,
      "activations/layer9_attention_weight_max": 45.011722564697266,
      "activations/layer9_attention_weight_min": -46.11235809326172,
      "epoch": 15.25,
      "learning_rate": 5.21840909090909e-05,
      "loss": 2.72,
      "step": 262450
    },
    {
      "activations/layer0_attention_weight_max": 16.167274475097656,
      "activations/layer0_attention_weight_min": -13.863540649414062,
      "activations/layer10_attention_weight_max": 38.30592727661133,
      "activations/layer10_attention_weight_min": -36.56302261352539,
      "activations/layer11_attention_weight_max": 34.81517791748047,
      "activations/layer11_attention_weight_min": -34.63990783691406,
      "activations/layer12_attention_weight_max": 19.78131103515625,
      "activations/layer12_attention_weight_min": -25.99004364013672,
      "activations/layer13_attention_weight_max": 36.10464096069336,
      "activations/layer13_attention_weight_min": -30.7232608795166,
      "activations/layer14_attention_weight_max": 35.37204360961914,
      "activations/layer14_attention_weight_min": -31.703771591186523,
      "activations/layer15_attention_weight_max": 30.762800216674805,
      "activations/layer15_attention_weight_min": -31.0651912689209,
      "activations/layer16_attention_weight_max": 32.048622131347656,
      "activations/layer16_attention_weight_min": -30.41465950012207,
      "activations/layer17_attention_weight_max": 51.40038299560547,
      "activations/layer17_attention_weight_min": -46.83743667602539,
      "activations/layer18_attention_weight_max": 45.28278732299805,
      "activations/layer18_attention_weight_min": -41.96311950683594,
      "activations/layer19_attention_weight_max": 23.60486602783203,
      "activations/layer19_attention_weight_min": -23.08086585998535,
      "activations/layer1_attention_weight_max": 16.62750244140625,
      "activations/layer1_attention_weight_min": -16.352542877197266,
      "activations/layer20_attention_weight_max": 21.24175453186035,
      "activations/layer20_attention_weight_min": -25.522586822509766,
      "activations/layer21_attention_weight_max": 35.451908111572266,
      "activations/layer21_attention_weight_min": -21.527324676513672,
      "activations/layer22_attention_weight_max": 28.124711990356445,
      "activations/layer22_attention_weight_min": -28.75957679748535,
      "activations/layer23_attention_weight_max": 34.06867980957031,
      "activations/layer23_attention_weight_min": -26.15540313720703,
      "activations/layer2_attention_weight_max": 33.142974853515625,
      "activations/layer2_attention_weight_min": -30.385135650634766,
      "activations/layer3_attention_weight_max": 94.46170043945312,
      "activations/layer3_attention_weight_min": -92.39654541015625,
      "activations/layer4_attention_weight_max": 94.08446502685547,
      "activations/layer4_attention_weight_min": -92.3657455444336,
      "activations/layer5_attention_weight_max": 68.36347961425781,
      "activations/layer5_attention_weight_min": -75.32127380371094,
      "activations/layer6_attention_weight_max": 49.935523986816406,
      "activations/layer6_attention_weight_min": -50.311119079589844,
      "activations/layer7_attention_weight_max": 60.05852508544922,
      "activations/layer7_attention_weight_min": -59.000518798828125,
      "activations/layer8_attention_weight_max": 49.950565338134766,
      "activations/layer8_attention_weight_min": -49.91606521606445,
      "activations/layer9_attention_weight_max": 46.62177276611328,
      "activations/layer9_attention_weight_min": -46.4498176574707,
      "epoch": 15.25,
      "learning_rate": 5.216515151515151e-05,
      "loss": 2.7459,
      "step": 262500
    },
    {
      "activations/layer0_attention_weight_max": 15.187921524047852,
      "activations/layer0_attention_weight_min": -13.628727912902832,
      "activations/layer10_attention_weight_max": 34.85551071166992,
      "activations/layer10_attention_weight_min": -38.37448501586914,
      "activations/layer11_attention_weight_max": 32.7785530090332,
      "activations/layer11_attention_weight_min": -34.951480865478516,
      "activations/layer12_attention_weight_max": 17.401317596435547,
      "activations/layer12_attention_weight_min": -28.961515426635742,
      "activations/layer13_attention_weight_max": 38.90700912475586,
      "activations/layer13_attention_weight_min": -30.456079483032227,
      "activations/layer14_attention_weight_max": 33.53567123413086,
      "activations/layer14_attention_weight_min": -30.8431396484375,
      "activations/layer15_attention_weight_max": 32.32971954345703,
      "activations/layer15_attention_weight_min": -30.305625915527344,
      "activations/layer16_attention_weight_max": 35.71410369873047,
      "activations/layer16_attention_weight_min": -33.63229751586914,
      "activations/layer17_attention_weight_max": 50.98115921020508,
      "activations/layer17_attention_weight_min": -47.63545608520508,
      "activations/layer18_attention_weight_max": 49.19028091430664,
      "activations/layer18_attention_weight_min": -39.53739547729492,
      "activations/layer19_attention_weight_max": 24.89974021911621,
      "activations/layer19_attention_weight_min": -22.979642868041992,
      "activations/layer1_attention_weight_max": 16.748437881469727,
      "activations/layer1_attention_weight_min": -15.657538414001465,
      "activations/layer20_attention_weight_max": 24.473594665527344,
      "activations/layer20_attention_weight_min": -19.760988235473633,
      "activations/layer21_attention_weight_max": 36.503421783447266,
      "activations/layer21_attention_weight_min": -20.967418670654297,
      "activations/layer22_attention_weight_max": 30.728683471679688,
      "activations/layer22_attention_weight_min": -25.802719116210938,
      "activations/layer23_attention_weight_max": 34.7137565612793,
      "activations/layer23_attention_weight_min": -25.17866325378418,
      "activations/layer2_attention_weight_max": 32.685813903808594,
      "activations/layer2_attention_weight_min": -32.27867126464844,
      "activations/layer3_attention_weight_max": 92.37767028808594,
      "activations/layer3_attention_weight_min": -95.3689193725586,
      "activations/layer4_attention_weight_max": 88.89714050292969,
      "activations/layer4_attention_weight_min": -92.9365463256836,
      "activations/layer5_attention_weight_max": 67.85087585449219,
      "activations/layer5_attention_weight_min": -71.849609375,
      "activations/layer6_attention_weight_max": 51.06185531616211,
      "activations/layer6_attention_weight_min": -50.19890213012695,
      "activations/layer7_attention_weight_max": 59.88145065307617,
      "activations/layer7_attention_weight_min": -60.541160583496094,
      "activations/layer8_attention_weight_max": 43.97248458862305,
      "activations/layer8_attention_weight_min": -50.19403839111328,
      "activations/layer9_attention_weight_max": 43.68889617919922,
      "activations/layer9_attention_weight_min": -48.16078567504883,
      "epoch": 15.26,
      "learning_rate": 5.2146212121212114e-05,
      "loss": 2.7577,
      "step": 262550
    },
    {
      "activations/layer0_attention_weight_max": 15.258119583129883,
      "activations/layer0_attention_weight_min": -13.665304183959961,
      "activations/layer10_attention_weight_max": 38.900360107421875,
      "activations/layer10_attention_weight_min": -36.33781814575195,
      "activations/layer11_attention_weight_max": 37.62542724609375,
      "activations/layer11_attention_weight_min": -35.05584716796875,
      "activations/layer12_attention_weight_max": 19.280160903930664,
      "activations/layer12_attention_weight_min": -24.24114418029785,
      "activations/layer13_attention_weight_max": 34.9652214050293,
      "activations/layer13_attention_weight_min": -28.51546287536621,
      "activations/layer14_attention_weight_max": 35.804901123046875,
      "activations/layer14_attention_weight_min": -32.41361999511719,
      "activations/layer15_attention_weight_max": 31.759111404418945,
      "activations/layer15_attention_weight_min": -30.406787872314453,
      "activations/layer16_attention_weight_max": 32.12546157836914,
      "activations/layer16_attention_weight_min": -31.636137008666992,
      "activations/layer17_attention_weight_max": 52.192962646484375,
      "activations/layer17_attention_weight_min": -45.607261657714844,
      "activations/layer18_attention_weight_max": 46.38043212890625,
      "activations/layer18_attention_weight_min": -40.05179977416992,
      "activations/layer19_attention_weight_max": 22.164833068847656,
      "activations/layer19_attention_weight_min": -22.037710189819336,
      "activations/layer1_attention_weight_max": 17.04491424560547,
      "activations/layer1_attention_weight_min": -16.506301879882812,
      "activations/layer20_attention_weight_max": 21.23546600341797,
      "activations/layer20_attention_weight_min": -19.73634910583496,
      "activations/layer21_attention_weight_max": 42.294918060302734,
      "activations/layer21_attention_weight_min": -23.053552627563477,
      "activations/layer22_attention_weight_max": 28.480920791625977,
      "activations/layer22_attention_weight_min": -27.200490951538086,
      "activations/layer23_attention_weight_max": 34.63853454589844,
      "activations/layer23_attention_weight_min": -21.653282165527344,
      "activations/layer2_attention_weight_max": 33.812408447265625,
      "activations/layer2_attention_weight_min": -31.735288619995117,
      "activations/layer3_attention_weight_max": 101.40196990966797,
      "activations/layer3_attention_weight_min": -101.1529541015625,
      "activations/layer4_attention_weight_max": 99.78996276855469,
      "activations/layer4_attention_weight_min": -90.90547943115234,
      "activations/layer5_attention_weight_max": 69.56978607177734,
      "activations/layer5_attention_weight_min": -69.05925750732422,
      "activations/layer6_attention_weight_max": 50.138267517089844,
      "activations/layer6_attention_weight_min": -50.00601577758789,
      "activations/layer7_attention_weight_max": 65.87355041503906,
      "activations/layer7_attention_weight_min": -64.73258209228516,
      "activations/layer8_attention_weight_max": 50.156333923339844,
      "activations/layer8_attention_weight_min": -48.430564880371094,
      "activations/layer9_attention_weight_max": 57.39027786254883,
      "activations/layer9_attention_weight_min": -47.130699157714844,
      "epoch": 15.26,
      "learning_rate": 5.212727272727272e-05,
      "loss": 2.7576,
      "step": 262600
    },
    {
      "activations/layer0_attention_weight_max": 16.106281280517578,
      "activations/layer0_attention_weight_min": -13.941696166992188,
      "activations/layer10_attention_weight_max": 37.065635681152344,
      "activations/layer10_attention_weight_min": -34.96028518676758,
      "activations/layer11_attention_weight_max": 32.545555114746094,
      "activations/layer11_attention_weight_min": -33.44948196411133,
      "activations/layer12_attention_weight_max": 18.300491333007812,
      "activations/layer12_attention_weight_min": -24.37896728515625,
      "activations/layer13_attention_weight_max": 35.94371795654297,
      "activations/layer13_attention_weight_min": -29.275531768798828,
      "activations/layer14_attention_weight_max": 36.738372802734375,
      "activations/layer14_attention_weight_min": -29.25046157836914,
      "activations/layer15_attention_weight_max": 33.28779602050781,
      "activations/layer15_attention_weight_min": -28.96147346496582,
      "activations/layer16_attention_weight_max": 33.72488021850586,
      "activations/layer16_attention_weight_min": -32.607322692871094,
      "activations/layer17_attention_weight_max": 51.762264251708984,
      "activations/layer17_attention_weight_min": -45.6349983215332,
      "activations/layer18_attention_weight_max": 46.278236389160156,
      "activations/layer18_attention_weight_min": -40.58705520629883,
      "activations/layer19_attention_weight_max": 25.270734786987305,
      "activations/layer19_attention_weight_min": -23.576353073120117,
      "activations/layer1_attention_weight_max": 16.985567092895508,
      "activations/layer1_attention_weight_min": -15.567817687988281,
      "activations/layer20_attention_weight_max": 22.926733016967773,
      "activations/layer20_attention_weight_min": -22.735448837280273,
      "activations/layer21_attention_weight_max": 36.30565643310547,
      "activations/layer21_attention_weight_min": -23.42001724243164,
      "activations/layer22_attention_weight_max": 29.360212326049805,
      "activations/layer22_attention_weight_min": -28.413909912109375,
      "activations/layer23_attention_weight_max": 37.71930694580078,
      "activations/layer23_attention_weight_min": -23.28005027770996,
      "activations/layer2_attention_weight_max": 33.09467315673828,
      "activations/layer2_attention_weight_min": -30.999671936035156,
      "activations/layer3_attention_weight_max": 89.15491485595703,
      "activations/layer3_attention_weight_min": -93.96550750732422,
      "activations/layer4_attention_weight_max": 85.63970184326172,
      "activations/layer4_attention_weight_min": -86.24751281738281,
      "activations/layer5_attention_weight_max": 65.741943359375,
      "activations/layer5_attention_weight_min": -79.02496337890625,
      "activations/layer6_attention_weight_max": 51.19541549682617,
      "activations/layer6_attention_weight_min": -50.447566986083984,
      "activations/layer7_attention_weight_max": 58.74874496459961,
      "activations/layer7_attention_weight_min": -58.88580322265625,
      "activations/layer8_attention_weight_max": 45.447200775146484,
      "activations/layer8_attention_weight_min": -47.70802688598633,
      "activations/layer9_attention_weight_max": 45.19071578979492,
      "activations/layer9_attention_weight_min": -44.25050735473633,
      "epoch": 15.26,
      "learning_rate": 5.210833333333333e-05,
      "loss": 2.7559,
      "step": 262650
    },
    {
      "activations/layer0_attention_weight_max": 16.454984664916992,
      "activations/layer0_attention_weight_min": -13.722463607788086,
      "activations/layer10_attention_weight_max": 36.45779037475586,
      "activations/layer10_attention_weight_min": -39.123748779296875,
      "activations/layer11_attention_weight_max": 34.91468811035156,
      "activations/layer11_attention_weight_min": -34.20371627807617,
      "activations/layer12_attention_weight_max": 19.239246368408203,
      "activations/layer12_attention_weight_min": -26.14322280883789,
      "activations/layer13_attention_weight_max": 35.042903900146484,
      "activations/layer13_attention_weight_min": -28.83213996887207,
      "activations/layer14_attention_weight_max": 36.67048645019531,
      "activations/layer14_attention_weight_min": -29.99672508239746,
      "activations/layer15_attention_weight_max": 33.86540222167969,
      "activations/layer15_attention_weight_min": -29.502899169921875,
      "activations/layer16_attention_weight_max": 33.790584564208984,
      "activations/layer16_attention_weight_min": -33.1900520324707,
      "activations/layer17_attention_weight_max": 51.789180755615234,
      "activations/layer17_attention_weight_min": -44.933921813964844,
      "activations/layer18_attention_weight_max": 46.50022506713867,
      "activations/layer18_attention_weight_min": -37.315826416015625,
      "activations/layer19_attention_weight_max": 23.341751098632812,
      "activations/layer19_attention_weight_min": -23.303375244140625,
      "activations/layer1_attention_weight_max": 16.33222770690918,
      "activations/layer1_attention_weight_min": -14.919721603393555,
      "activations/layer20_attention_weight_max": 21.276718139648438,
      "activations/layer20_attention_weight_min": -23.18634605407715,
      "activations/layer21_attention_weight_max": 35.758480072021484,
      "activations/layer21_attention_weight_min": -21.76459503173828,
      "activations/layer22_attention_weight_max": 28.714075088500977,
      "activations/layer22_attention_weight_min": -26.24135398864746,
      "activations/layer23_attention_weight_max": 44.65420150756836,
      "activations/layer23_attention_weight_min": -25.095558166503906,
      "activations/layer2_attention_weight_max": 29.41072654724121,
      "activations/layer2_attention_weight_min": -30.145614624023438,
      "activations/layer3_attention_weight_max": 90.45342254638672,
      "activations/layer3_attention_weight_min": -93.1075210571289,
      "activations/layer4_attention_weight_max": 88.0748062133789,
      "activations/layer4_attention_weight_min": -91.65569305419922,
      "activations/layer5_attention_weight_max": 64.15699005126953,
      "activations/layer5_attention_weight_min": -79.00176239013672,
      "activations/layer6_attention_weight_max": 47.93699645996094,
      "activations/layer6_attention_weight_min": -51.38021469116211,
      "activations/layer7_attention_weight_max": 60.24063491821289,
      "activations/layer7_attention_weight_min": -66.641357421875,
      "activations/layer8_attention_weight_max": 46.75520324707031,
      "activations/layer8_attention_weight_min": -54.336692810058594,
      "activations/layer9_attention_weight_max": 47.04791259765625,
      "activations/layer9_attention_weight_min": -50.81057357788086,
      "epoch": 15.26,
      "learning_rate": 5.208939393939394e-05,
      "loss": 2.7606,
      "step": 262700
    },
    {
      "activations/layer0_attention_weight_max": 14.96895980834961,
      "activations/layer0_attention_weight_min": -13.519765853881836,
      "activations/layer10_attention_weight_max": 38.021644592285156,
      "activations/layer10_attention_weight_min": -38.6436767578125,
      "activations/layer11_attention_weight_max": 34.719825744628906,
      "activations/layer11_attention_weight_min": -34.83601379394531,
      "activations/layer12_attention_weight_max": 18.819141387939453,
      "activations/layer12_attention_weight_min": -27.550146102905273,
      "activations/layer13_attention_weight_max": 32.8394889831543,
      "activations/layer13_attention_weight_min": -29.747316360473633,
      "activations/layer14_attention_weight_max": 34.33272933959961,
      "activations/layer14_attention_weight_min": -29.313610076904297,
      "activations/layer15_attention_weight_max": 31.25936508178711,
      "activations/layer15_attention_weight_min": -31.280057907104492,
      "activations/layer16_attention_weight_max": 31.507474899291992,
      "activations/layer16_attention_weight_min": -29.86330795288086,
      "activations/layer17_attention_weight_max": 49.63141632080078,
      "activations/layer17_attention_weight_min": -42.65005874633789,
      "activations/layer18_attention_weight_max": 44.80992126464844,
      "activations/layer18_attention_weight_min": -40.15026092529297,
      "activations/layer19_attention_weight_max": 21.211570739746094,
      "activations/layer19_attention_weight_min": -21.946218490600586,
      "activations/layer1_attention_weight_max": 17.033489227294922,
      "activations/layer1_attention_weight_min": -16.507495880126953,
      "activations/layer20_attention_weight_max": 21.203310012817383,
      "activations/layer20_attention_weight_min": -21.391019821166992,
      "activations/layer21_attention_weight_max": 34.50802993774414,
      "activations/layer21_attention_weight_min": -19.109500885009766,
      "activations/layer22_attention_weight_max": 28.690162658691406,
      "activations/layer22_attention_weight_min": -27.524290084838867,
      "activations/layer23_attention_weight_max": 37.86125946044922,
      "activations/layer23_attention_weight_min": -23.658931732177734,
      "activations/layer2_attention_weight_max": 31.481380462646484,
      "activations/layer2_attention_weight_min": -31.482030868530273,
      "activations/layer3_attention_weight_max": 96.28340148925781,
      "activations/layer3_attention_weight_min": -97.37318420410156,
      "activations/layer4_attention_weight_max": 89.47212982177734,
      "activations/layer4_attention_weight_min": -90.9694595336914,
      "activations/layer5_attention_weight_max": 67.94599914550781,
      "activations/layer5_attention_weight_min": -71.6813735961914,
      "activations/layer6_attention_weight_max": 54.16057205200195,
      "activations/layer6_attention_weight_min": -52.691707611083984,
      "activations/layer7_attention_weight_max": 60.78340530395508,
      "activations/layer7_attention_weight_min": -59.766231536865234,
      "activations/layer8_attention_weight_max": 47.90930938720703,
      "activations/layer8_attention_weight_min": -52.77045440673828,
      "activations/layer9_attention_weight_max": 49.89234161376953,
      "activations/layer9_attention_weight_min": -47.142024993896484,
      "epoch": 15.27,
      "learning_rate": 5.207045454545454e-05,
      "loss": 2.7566,
      "step": 262750
    },
    {
      "activations/layer0_attention_weight_max": 14.975852966308594,
      "activations/layer0_attention_weight_min": -13.728224754333496,
      "activations/layer10_attention_weight_max": 37.747093200683594,
      "activations/layer10_attention_weight_min": -36.329566955566406,
      "activations/layer11_attention_weight_max": 34.98774719238281,
      "activations/layer11_attention_weight_min": -34.10182571411133,
      "activations/layer12_attention_weight_max": 18.454587936401367,
      "activations/layer12_attention_weight_min": -31.09650230407715,
      "activations/layer13_attention_weight_max": 38.095848083496094,
      "activations/layer13_attention_weight_min": -28.99226951599121,
      "activations/layer14_attention_weight_max": 35.55319595336914,
      "activations/layer14_attention_weight_min": -30.774131774902344,
      "activations/layer15_attention_weight_max": 34.740047454833984,
      "activations/layer15_attention_weight_min": -30.116928100585938,
      "activations/layer16_attention_weight_max": 32.686092376708984,
      "activations/layer16_attention_weight_min": -31.893360137939453,
      "activations/layer17_attention_weight_max": 55.922706604003906,
      "activations/layer17_attention_weight_min": -50.545318603515625,
      "activations/layer18_attention_weight_max": 46.223045349121094,
      "activations/layer18_attention_weight_min": -44.1781005859375,
      "activations/layer19_attention_weight_max": 21.47047233581543,
      "activations/layer19_attention_weight_min": -21.175518035888672,
      "activations/layer1_attention_weight_max": 16.356197357177734,
      "activations/layer1_attention_weight_min": -16.409103393554688,
      "activations/layer20_attention_weight_max": 21.379411697387695,
      "activations/layer20_attention_weight_min": -22.00344467163086,
      "activations/layer21_attention_weight_max": 36.37797546386719,
      "activations/layer21_attention_weight_min": -24.400089263916016,
      "activations/layer22_attention_weight_max": 31.96677017211914,
      "activations/layer22_attention_weight_min": -28.4483699798584,
      "activations/layer23_attention_weight_max": 37.0302734375,
      "activations/layer23_attention_weight_min": -28.25802993774414,
      "activations/layer2_attention_weight_max": 33.61703109741211,
      "activations/layer2_attention_weight_min": -31.314620971679688,
      "activations/layer3_attention_weight_max": 96.58103942871094,
      "activations/layer3_attention_weight_min": -92.66047668457031,
      "activations/layer4_attention_weight_max": 91.1848373413086,
      "activations/layer4_attention_weight_min": -87.95150756835938,
      "activations/layer5_attention_weight_max": 68.80157470703125,
      "activations/layer5_attention_weight_min": -73.36458587646484,
      "activations/layer6_attention_weight_max": 49.9035758972168,
      "activations/layer6_attention_weight_min": -49.72172927856445,
      "activations/layer7_attention_weight_max": 64.22122192382812,
      "activations/layer7_attention_weight_min": -63.70084762573242,
      "activations/layer8_attention_weight_max": 46.09334182739258,
      "activations/layer8_attention_weight_min": -47.93452453613281,
      "activations/layer9_attention_weight_max": 51.05696487426758,
      "activations/layer9_attention_weight_min": -44.63283920288086,
      "epoch": 15.27,
      "learning_rate": 5.205151515151514e-05,
      "loss": 2.7501,
      "step": 262800
    },
    {
      "activations/layer0_attention_weight_max": 14.705166816711426,
      "activations/layer0_attention_weight_min": -13.465951919555664,
      "activations/layer10_attention_weight_max": 36.7474479675293,
      "activations/layer10_attention_weight_min": -37.37009811401367,
      "activations/layer11_attention_weight_max": 33.13622283935547,
      "activations/layer11_attention_weight_min": -34.5827522277832,
      "activations/layer12_attention_weight_max": 17.849843978881836,
      "activations/layer12_attention_weight_min": -27.294166564941406,
      "activations/layer13_attention_weight_max": 33.27326965332031,
      "activations/layer13_attention_weight_min": -29.29075813293457,
      "activations/layer14_attention_weight_max": 34.23285675048828,
      "activations/layer14_attention_weight_min": -32.57356262207031,
      "activations/layer15_attention_weight_max": 30.159685134887695,
      "activations/layer15_attention_weight_min": -30.72007942199707,
      "activations/layer16_attention_weight_max": 30.918352127075195,
      "activations/layer16_attention_weight_min": -31.2574462890625,
      "activations/layer17_attention_weight_max": 50.12862014770508,
      "activations/layer17_attention_weight_min": -46.10649490356445,
      "activations/layer18_attention_weight_max": 48.54664611816406,
      "activations/layer18_attention_weight_min": -41.96432876586914,
      "activations/layer19_attention_weight_max": 23.14632225036621,
      "activations/layer19_attention_weight_min": -23.531198501586914,
      "activations/layer1_attention_weight_max": 17.95953369140625,
      "activations/layer1_attention_weight_min": -16.45577049255371,
      "activations/layer20_attention_weight_max": 24.17205047607422,
      "activations/layer20_attention_weight_min": -21.81941795349121,
      "activations/layer21_attention_weight_max": 42.56291961669922,
      "activations/layer21_attention_weight_min": -24.255313873291016,
      "activations/layer22_attention_weight_max": 28.604848861694336,
      "activations/layer22_attention_weight_min": -25.26691436767578,
      "activations/layer23_attention_weight_max": 35.06293487548828,
      "activations/layer23_attention_weight_min": -24.39344024658203,
      "activations/layer2_attention_weight_max": 33.00657272338867,
      "activations/layer2_attention_weight_min": -30.016468048095703,
      "activations/layer3_attention_weight_max": 98.67233276367188,
      "activations/layer3_attention_weight_min": -87.6446533203125,
      "activations/layer4_attention_weight_max": 91.5746078491211,
      "activations/layer4_attention_weight_min": -89.0482406616211,
      "activations/layer5_attention_weight_max": 69.25714874267578,
      "activations/layer5_attention_weight_min": -71.70651245117188,
      "activations/layer6_attention_weight_max": 52.576717376708984,
      "activations/layer6_attention_weight_min": -49.587074279785156,
      "activations/layer7_attention_weight_max": 64.87532806396484,
      "activations/layer7_attention_weight_min": -60.98743438720703,
      "activations/layer8_attention_weight_max": 46.28290939331055,
      "activations/layer8_attention_weight_min": -48.39372253417969,
      "activations/layer9_attention_weight_max": 46.44677734375,
      "activations/layer9_attention_weight_min": -44.40608596801758,
      "epoch": 15.27,
      "learning_rate": 5.203257575757576e-05,
      "loss": 2.7791,
      "step": 262850
    },
    {
      "activations/layer0_attention_weight_max": 16.07010269165039,
      "activations/layer0_attention_weight_min": -13.949686050415039,
      "activations/layer10_attention_weight_max": 34.0349006652832,
      "activations/layer10_attention_weight_min": -37.1420783996582,
      "activations/layer11_attention_weight_max": 32.81797790527344,
      "activations/layer11_attention_weight_min": -35.37169647216797,
      "activations/layer12_attention_weight_max": 16.297828674316406,
      "activations/layer12_attention_weight_min": -24.439653396606445,
      "activations/layer13_attention_weight_max": 36.42102813720703,
      "activations/layer13_attention_weight_min": -29.006412506103516,
      "activations/layer14_attention_weight_max": 32.79709243774414,
      "activations/layer14_attention_weight_min": -28.303621292114258,
      "activations/layer15_attention_weight_max": 31.49230194091797,
      "activations/layer15_attention_weight_min": -29.771331787109375,
      "activations/layer16_attention_weight_max": 30.494535446166992,
      "activations/layer16_attention_weight_min": -29.000791549682617,
      "activations/layer17_attention_weight_max": 46.39179611206055,
      "activations/layer17_attention_weight_min": -43.116455078125,
      "activations/layer18_attention_weight_max": 41.029563903808594,
      "activations/layer18_attention_weight_min": -39.13420867919922,
      "activations/layer19_attention_weight_max": 22.297611236572266,
      "activations/layer19_attention_weight_min": -20.534828186035156,
      "activations/layer1_attention_weight_max": 16.384754180908203,
      "activations/layer1_attention_weight_min": -15.449485778808594,
      "activations/layer20_attention_weight_max": 20.054731369018555,
      "activations/layer20_attention_weight_min": -22.762767791748047,
      "activations/layer21_attention_weight_max": 31.92244529724121,
      "activations/layer21_attention_weight_min": -22.363998413085938,
      "activations/layer22_attention_weight_max": 26.951005935668945,
      "activations/layer22_attention_weight_min": -26.670461654663086,
      "activations/layer23_attention_weight_max": 31.834716796875,
      "activations/layer23_attention_weight_min": -24.810558319091797,
      "activations/layer2_attention_weight_max": 33.10606384277344,
      "activations/layer2_attention_weight_min": -31.856163024902344,
      "activations/layer3_attention_weight_max": 92.02168273925781,
      "activations/layer3_attention_weight_min": -92.74432373046875,
      "activations/layer4_attention_weight_max": 89.1488265991211,
      "activations/layer4_attention_weight_min": -87.09059143066406,
      "activations/layer5_attention_weight_max": 66.65440368652344,
      "activations/layer5_attention_weight_min": -74.7133560180664,
      "activations/layer6_attention_weight_max": 51.65801239013672,
      "activations/layer6_attention_weight_min": -53.025291442871094,
      "activations/layer7_attention_weight_max": 64.11637115478516,
      "activations/layer7_attention_weight_min": -62.14075469970703,
      "activations/layer8_attention_weight_max": 46.65413284301758,
      "activations/layer8_attention_weight_min": -52.80095291137695,
      "activations/layer9_attention_weight_max": 45.34098815917969,
      "activations/layer9_attention_weight_min": -46.579524993896484,
      "epoch": 15.28,
      "learning_rate": 5.201363636363636e-05,
      "loss": 2.7408,
      "step": 262900
    },
    {
      "activations/layer0_attention_weight_max": 15.301732063293457,
      "activations/layer0_attention_weight_min": -13.570515632629395,
      "activations/layer10_attention_weight_max": 37.22721862792969,
      "activations/layer10_attention_weight_min": -36.439857482910156,
      "activations/layer11_attention_weight_max": 35.623348236083984,
      "activations/layer11_attention_weight_min": -34.36814880371094,
      "activations/layer12_attention_weight_max": 19.11107635498047,
      "activations/layer12_attention_weight_min": -23.82500648498535,
      "activations/layer13_attention_weight_max": 36.032203674316406,
      "activations/layer13_attention_weight_min": -30.243959426879883,
      "activations/layer14_attention_weight_max": 34.06160354614258,
      "activations/layer14_attention_weight_min": -31.185077667236328,
      "activations/layer15_attention_weight_max": 31.091581344604492,
      "activations/layer15_attention_weight_min": -29.88491439819336,
      "activations/layer16_attention_weight_max": 31.79973030090332,
      "activations/layer16_attention_weight_min": -30.23857307434082,
      "activations/layer17_attention_weight_max": 55.78434371948242,
      "activations/layer17_attention_weight_min": -44.74870681762695,
      "activations/layer18_attention_weight_max": 46.245460510253906,
      "activations/layer18_attention_weight_min": -39.183685302734375,
      "activations/layer19_attention_weight_max": 22.195999145507812,
      "activations/layer19_attention_weight_min": -21.032634735107422,
      "activations/layer1_attention_weight_max": 16.744733810424805,
      "activations/layer1_attention_weight_min": -14.414387702941895,
      "activations/layer20_attention_weight_max": 23.454540252685547,
      "activations/layer20_attention_weight_min": -20.093406677246094,
      "activations/layer21_attention_weight_max": 29.03677749633789,
      "activations/layer21_attention_weight_min": -19.387292861938477,
      "activations/layer22_attention_weight_max": 26.90278434753418,
      "activations/layer22_attention_weight_min": -23.35616683959961,
      "activations/layer23_attention_weight_max": 35.212127685546875,
      "activations/layer23_attention_weight_min": -23.354530334472656,
      "activations/layer2_attention_weight_max": 30.382352828979492,
      "activations/layer2_attention_weight_min": -30.331647872924805,
      "activations/layer3_attention_weight_max": 91.96063232421875,
      "activations/layer3_attention_weight_min": -96.94361114501953,
      "activations/layer4_attention_weight_max": 88.44151306152344,
      "activations/layer4_attention_weight_min": -90.42549133300781,
      "activations/layer5_attention_weight_max": 63.97304916381836,
      "activations/layer5_attention_weight_min": -75.65797424316406,
      "activations/layer6_attention_weight_max": 52.364036560058594,
      "activations/layer6_attention_weight_min": -51.15803527832031,
      "activations/layer7_attention_weight_max": 58.03537368774414,
      "activations/layer7_attention_weight_min": -60.283790588378906,
      "activations/layer8_attention_weight_max": 46.64072036743164,
      "activations/layer8_attention_weight_min": -47.965248107910156,
      "activations/layer9_attention_weight_max": 46.84675979614258,
      "activations/layer9_attention_weight_min": -47.45439910888672,
      "epoch": 15.28,
      "learning_rate": 5.1994696969696967e-05,
      "loss": 2.7515,
      "step": 262950
    },
    {
      "activations/layer0_attention_weight_max": 16.37855339050293,
      "activations/layer0_attention_weight_min": -13.622849464416504,
      "activations/layer10_attention_weight_max": 39.79993438720703,
      "activations/layer10_attention_weight_min": -36.063446044921875,
      "activations/layer11_attention_weight_max": 32.94613265991211,
      "activations/layer11_attention_weight_min": -33.452049255371094,
      "activations/layer12_attention_weight_max": 17.838224411010742,
      "activations/layer12_attention_weight_min": -28.15597152709961,
      "activations/layer13_attention_weight_max": 34.95024871826172,
      "activations/layer13_attention_weight_min": -28.912019729614258,
      "activations/layer14_attention_weight_max": 34.46699523925781,
      "activations/layer14_attention_weight_min": -29.053627014160156,
      "activations/layer15_attention_weight_max": 32.49412536621094,
      "activations/layer15_attention_weight_min": -30.489273071289062,
      "activations/layer16_attention_weight_max": 32.15798568725586,
      "activations/layer16_attention_weight_min": -30.757217407226562,
      "activations/layer17_attention_weight_max": 54.3525276184082,
      "activations/layer17_attention_weight_min": -44.54553985595703,
      "activations/layer18_attention_weight_max": 46.912506103515625,
      "activations/layer18_attention_weight_min": -39.00162124633789,
      "activations/layer19_attention_weight_max": 23.904150009155273,
      "activations/layer19_attention_weight_min": -22.492740631103516,
      "activations/layer1_attention_weight_max": 16.913583755493164,
      "activations/layer1_attention_weight_min": -16.311708450317383,
      "activations/layer20_attention_weight_max": 22.846277236938477,
      "activations/layer20_attention_weight_min": -22.091218948364258,
      "activations/layer21_attention_weight_max": 37.86954879760742,
      "activations/layer21_attention_weight_min": -23.992111206054688,
      "activations/layer22_attention_weight_max": 29.438703536987305,
      "activations/layer22_attention_weight_min": -26.573333740234375,
      "activations/layer23_attention_weight_max": 36.328041076660156,
      "activations/layer23_attention_weight_min": -22.82721710205078,
      "activations/layer2_attention_weight_max": 32.0356330871582,
      "activations/layer2_attention_weight_min": -33.15641784667969,
      "activations/layer3_attention_weight_max": 93.49412536621094,
      "activations/layer3_attention_weight_min": -99.95269775390625,
      "activations/layer4_attention_weight_max": 91.53231811523438,
      "activations/layer4_attention_weight_min": -92.30220794677734,
      "activations/layer5_attention_weight_max": 66.62908935546875,
      "activations/layer5_attention_weight_min": -74.50477600097656,
      "activations/layer6_attention_weight_max": 51.16786193847656,
      "activations/layer6_attention_weight_min": -51.51887893676758,
      "activations/layer7_attention_weight_max": 61.020511627197266,
      "activations/layer7_attention_weight_min": -62.02356719970703,
      "activations/layer8_attention_weight_max": 46.20802307128906,
      "activations/layer8_attention_weight_min": -51.00539016723633,
      "activations/layer9_attention_weight_max": 45.673179626464844,
      "activations/layer9_attention_weight_min": -46.43773651123047,
      "epoch": 15.28,
      "learning_rate": 5.197575757575757e-05,
      "loss": 2.7362,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4841,
      "eval_samples_per_second": 506.124,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4841,
      "eval_openwebtext_samples_per_second": 506.124,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.953,
      "eval_wikitext_samples_per_second": 233.483,
      "step": 263000
    },
    {
      "epoch": 15.28,
      "eval_lambada_loss": 2.515625,
      "eval_lambada_ppl": 12.374340325455691,
      "eval_lambada_runtime": 9.5416,
      "eval_lambada_samples_per_second": 510.29,
      "step": 263000
    },
    {
      "activations/layer0_attention_weight_max": 14.948927879333496,
      "activations/layer0_attention_weight_min": -14.125167846679688,
      "activations/layer10_attention_weight_max": 34.83313751220703,
      "activations/layer10_attention_weight_min": -32.99210739135742,
      "activations/layer11_attention_weight_max": 33.837547302246094,
      "activations/layer11_attention_weight_min": -31.21216583251953,
      "activations/layer12_attention_weight_max": 19.345043182373047,
      "activations/layer12_attention_weight_min": -24.100284576416016,
      "activations/layer13_attention_weight_max": 33.87867736816406,
      "activations/layer13_attention_weight_min": -27.83975601196289,
      "activations/layer14_attention_weight_max": 35.81471633911133,
      "activations/layer14_attention_weight_min": -29.362808227539062,
      "activations/layer15_attention_weight_max": 32.615211486816406,
      "activations/layer15_attention_weight_min": -29.660816192626953,
      "activations/layer16_attention_weight_max": 31.063793182373047,
      "activations/layer16_attention_weight_min": -29.43315887451172,
      "activations/layer17_attention_weight_max": 51.54452896118164,
      "activations/layer17_attention_weight_min": -43.027427673339844,
      "activations/layer18_attention_weight_max": 44.391685485839844,
      "activations/layer18_attention_weight_min": -35.98687744140625,
      "activations/layer19_attention_weight_max": 25.741474151611328,
      "activations/layer19_attention_weight_min": -22.55910301208496,
      "activations/layer1_attention_weight_max": 16.4122257232666,
      "activations/layer1_attention_weight_min": -14.289331436157227,
      "activations/layer20_attention_weight_max": 21.39332389831543,
      "activations/layer20_attention_weight_min": -21.607391357421875,
      "activations/layer21_attention_weight_max": 36.990753173828125,
      "activations/layer21_attention_weight_min": -27.00155258178711,
      "activations/layer22_attention_weight_max": 30.982677459716797,
      "activations/layer22_attention_weight_min": -25.071645736694336,
      "activations/layer23_attention_weight_max": 38.53903579711914,
      "activations/layer23_attention_weight_min": -24.645505905151367,
      "activations/layer2_attention_weight_max": 29.600116729736328,
      "activations/layer2_attention_weight_min": -28.690570831298828,
      "activations/layer3_attention_weight_max": 89.43050384521484,
      "activations/layer3_attention_weight_min": -89.92007446289062,
      "activations/layer4_attention_weight_max": 89.09709930419922,
      "activations/layer4_attention_weight_min": -85.7677230834961,
      "activations/layer5_attention_weight_max": 65.65692138671875,
      "activations/layer5_attention_weight_min": -70.71119689941406,
      "activations/layer6_attention_weight_max": 49.03003692626953,
      "activations/layer6_attention_weight_min": -48.10369110107422,
      "activations/layer7_attention_weight_max": 61.4331169128418,
      "activations/layer7_attention_weight_min": -59.453983306884766,
      "activations/layer8_attention_weight_max": 43.43109893798828,
      "activations/layer8_attention_weight_min": -48.10844421386719,
      "activations/layer9_attention_weight_max": 42.059043884277344,
      "activations/layer9_attention_weight_min": -43.40403366088867,
      "epoch": 15.28,
      "learning_rate": 5.195681818181818e-05,
      "loss": 2.7387,
      "step": 263050
    },
    {
      "activations/layer0_attention_weight_max": 15.295610427856445,
      "activations/layer0_attention_weight_min": -13.69803237915039,
      "activations/layer10_attention_weight_max": 34.52411651611328,
      "activations/layer10_attention_weight_min": -35.0277099609375,
      "activations/layer11_attention_weight_max": 32.84228515625,
      "activations/layer11_attention_weight_min": -34.12774658203125,
      "activations/layer12_attention_weight_max": 18.9422664642334,
      "activations/layer12_attention_weight_min": -23.110260009765625,
      "activations/layer13_attention_weight_max": 34.982154846191406,
      "activations/layer13_attention_weight_min": -28.188615798950195,
      "activations/layer14_attention_weight_max": 36.921722412109375,
      "activations/layer14_attention_weight_min": -29.64397621154785,
      "activations/layer15_attention_weight_max": 32.701263427734375,
      "activations/layer15_attention_weight_min": -28.872211456298828,
      "activations/layer16_attention_weight_max": 31.70520782470703,
      "activations/layer16_attention_weight_min": -28.431198120117188,
      "activations/layer17_attention_weight_max": 50.847503662109375,
      "activations/layer17_attention_weight_min": -41.57160186767578,
      "activations/layer18_attention_weight_max": 45.94136047363281,
      "activations/layer18_attention_weight_min": -36.476200103759766,
      "activations/layer19_attention_weight_max": 22.558425903320312,
      "activations/layer19_attention_weight_min": -21.393442153930664,
      "activations/layer1_attention_weight_max": 16.134262084960938,
      "activations/layer1_attention_weight_min": -15.70128059387207,
      "activations/layer20_attention_weight_max": 23.618000030517578,
      "activations/layer20_attention_weight_min": -20.14269256591797,
      "activations/layer21_attention_weight_max": 37.769805908203125,
      "activations/layer21_attention_weight_min": -25.159282684326172,
      "activations/layer22_attention_weight_max": 29.6235408782959,
      "activations/layer22_attention_weight_min": -27.426788330078125,
      "activations/layer23_attention_weight_max": 37.19715881347656,
      "activations/layer23_attention_weight_min": -26.571453094482422,
      "activations/layer2_attention_weight_max": 32.10294723510742,
      "activations/layer2_attention_weight_min": -31.439672470092773,
      "activations/layer3_attention_weight_max": 88.329833984375,
      "activations/layer3_attention_weight_min": -94.26397705078125,
      "activations/layer4_attention_weight_max": 88.45426940917969,
      "activations/layer4_attention_weight_min": -89.36273956298828,
      "activations/layer5_attention_weight_max": 68.28087615966797,
      "activations/layer5_attention_weight_min": -80.2835693359375,
      "activations/layer6_attention_weight_max": 50.70731735229492,
      "activations/layer6_attention_weight_min": -51.35234069824219,
      "activations/layer7_attention_weight_max": 64.96430969238281,
      "activations/layer7_attention_weight_min": -63.702362060546875,
      "activations/layer8_attention_weight_max": 45.756858825683594,
      "activations/layer8_attention_weight_min": -50.44283676147461,
      "activations/layer9_attention_weight_max": 43.525482177734375,
      "activations/layer9_attention_weight_min": -47.03781509399414,
      "epoch": 15.29,
      "learning_rate": 5.1937878787878785e-05,
      "loss": 2.749,
      "step": 263100
    },
    {
      "activations/layer0_attention_weight_max": 15.295988082885742,
      "activations/layer0_attention_weight_min": -13.92546558380127,
      "activations/layer10_attention_weight_max": 39.49823760986328,
      "activations/layer10_attention_weight_min": -38.2060546875,
      "activations/layer11_attention_weight_max": 33.790340423583984,
      "activations/layer11_attention_weight_min": -35.50008010864258,
      "activations/layer12_attention_weight_max": 20.258724212646484,
      "activations/layer12_attention_weight_min": -25.23921012878418,
      "activations/layer13_attention_weight_max": 41.26508331298828,
      "activations/layer13_attention_weight_min": -28.2028751373291,
      "activations/layer14_attention_weight_max": 40.48827362060547,
      "activations/layer14_attention_weight_min": -29.545551300048828,
      "activations/layer15_attention_weight_max": 37.105369567871094,
      "activations/layer15_attention_weight_min": -30.908517837524414,
      "activations/layer16_attention_weight_max": 38.086647033691406,
      "activations/layer16_attention_weight_min": -32.89318084716797,
      "activations/layer17_attention_weight_max": 57.23588180541992,
      "activations/layer17_attention_weight_min": -44.692352294921875,
      "activations/layer18_attention_weight_max": 51.609745025634766,
      "activations/layer18_attention_weight_min": -38.90235137939453,
      "activations/layer19_attention_weight_max": 24.64754295349121,
      "activations/layer19_attention_weight_min": -22.51826286315918,
      "activations/layer1_attention_weight_max": 16.598684310913086,
      "activations/layer1_attention_weight_min": -14.446629524230957,
      "activations/layer20_attention_weight_max": 25.756202697753906,
      "activations/layer20_attention_weight_min": -22.2592716217041,
      "activations/layer21_attention_weight_max": 38.13225555419922,
      "activations/layer21_attention_weight_min": -21.34210205078125,
      "activations/layer22_attention_weight_max": 29.005512237548828,
      "activations/layer22_attention_weight_min": -23.65104866027832,
      "activations/layer23_attention_weight_max": 44.852745056152344,
      "activations/layer23_attention_weight_min": -24.794654846191406,
      "activations/layer2_attention_weight_max": 30.186100006103516,
      "activations/layer2_attention_weight_min": -30.90206527709961,
      "activations/layer3_attention_weight_max": 91.98899841308594,
      "activations/layer3_attention_weight_min": -100.4205322265625,
      "activations/layer4_attention_weight_max": 91.1314926147461,
      "activations/layer4_attention_weight_min": -93.08373260498047,
      "activations/layer5_attention_weight_max": 69.85713195800781,
      "activations/layer5_attention_weight_min": -80.10035705566406,
      "activations/layer6_attention_weight_max": 54.34262466430664,
      "activations/layer6_attention_weight_min": -53.61892318725586,
      "activations/layer7_attention_weight_max": 65.13101196289062,
      "activations/layer7_attention_weight_min": -66.45724487304688,
      "activations/layer8_attention_weight_max": 50.578407287597656,
      "activations/layer8_attention_weight_min": -55.41032028198242,
      "activations/layer9_attention_weight_max": 49.534664154052734,
      "activations/layer9_attention_weight_min": -50.009490966796875,
      "epoch": 15.29,
      "learning_rate": 5.1919318181818175e-05,
      "loss": 2.7516,
      "step": 263150
    },
    {
      "activations/layer0_attention_weight_max": 15.518929481506348,
      "activations/layer0_attention_weight_min": -13.841002464294434,
      "activations/layer10_attention_weight_max": 35.799476623535156,
      "activations/layer10_attention_weight_min": -35.770206451416016,
      "activations/layer11_attention_weight_max": 32.62800216674805,
      "activations/layer11_attention_weight_min": -33.84747314453125,
      "activations/layer12_attention_weight_max": 17.70575523376465,
      "activations/layer12_attention_weight_min": -26.059782028198242,
      "activations/layer13_attention_weight_max": 29.69141387939453,
      "activations/layer13_attention_weight_min": -31.210256576538086,
      "activations/layer14_attention_weight_max": 35.07854461669922,
      "activations/layer14_attention_weight_min": -30.691999435424805,
      "activations/layer15_attention_weight_max": 33.22865676879883,
      "activations/layer15_attention_weight_min": -29.668216705322266,
      "activations/layer16_attention_weight_max": 32.11840057373047,
      "activations/layer16_attention_weight_min": -31.24663543701172,
      "activations/layer17_attention_weight_max": 52.05967330932617,
      "activations/layer17_attention_weight_min": -44.863651275634766,
      "activations/layer18_attention_weight_max": 47.56471252441406,
      "activations/layer18_attention_weight_min": -38.39940643310547,
      "activations/layer19_attention_weight_max": 23.11310577392578,
      "activations/layer19_attention_weight_min": -23.31218147277832,
      "activations/layer1_attention_weight_max": 17.912012100219727,
      "activations/layer1_attention_weight_min": -15.45063591003418,
      "activations/layer20_attention_weight_max": 21.51844024658203,
      "activations/layer20_attention_weight_min": -22.730419158935547,
      "activations/layer21_attention_weight_max": 34.98386001586914,
      "activations/layer21_attention_weight_min": -23.75490379333496,
      "activations/layer22_attention_weight_max": 29.054481506347656,
      "activations/layer22_attention_weight_min": -25.678667068481445,
      "activations/layer23_attention_weight_max": 38.33353042602539,
      "activations/layer23_attention_weight_min": -24.742694854736328,
      "activations/layer2_attention_weight_max": 30.63471794128418,
      "activations/layer2_attention_weight_min": -30.112380981445312,
      "activations/layer3_attention_weight_max": 88.82879638671875,
      "activations/layer3_attention_weight_min": -98.59951782226562,
      "activations/layer4_attention_weight_max": 87.59088134765625,
      "activations/layer4_attention_weight_min": -84.99382019042969,
      "activations/layer5_attention_weight_max": 63.94850158691406,
      "activations/layer5_attention_weight_min": -77.55099487304688,
      "activations/layer6_attention_weight_max": 51.034305572509766,
      "activations/layer6_attention_weight_min": -47.78367233276367,
      "activations/layer7_attention_weight_max": 60.919185638427734,
      "activations/layer7_attention_weight_min": -57.794166564941406,
      "activations/layer8_attention_weight_max": 45.204429626464844,
      "activations/layer8_attention_weight_min": -48.35319900512695,
      "activations/layer9_attention_weight_max": 42.68605422973633,
      "activations/layer9_attention_weight_min": -44.102962493896484,
      "epoch": 15.29,
      "learning_rate": 5.190037878787878e-05,
      "loss": 2.739,
      "step": 263200
    },
    {
      "activations/layer0_attention_weight_max": 15.716294288635254,
      "activations/layer0_attention_weight_min": -14.053969383239746,
      "activations/layer10_attention_weight_max": 35.86935043334961,
      "activations/layer10_attention_weight_min": -36.445472717285156,
      "activations/layer11_attention_weight_max": 31.495464324951172,
      "activations/layer11_attention_weight_min": -34.687870025634766,
      "activations/layer12_attention_weight_max": 19.72248077392578,
      "activations/layer12_attention_weight_min": -25.03492546081543,
      "activations/layer13_attention_weight_max": 37.23925018310547,
      "activations/layer13_attention_weight_min": -28.7018985748291,
      "activations/layer14_attention_weight_max": 35.63560485839844,
      "activations/layer14_attention_weight_min": -31.89659881591797,
      "activations/layer15_attention_weight_max": 35.8592643737793,
      "activations/layer15_attention_weight_min": -31.50925064086914,
      "activations/layer16_attention_weight_max": 34.7845344543457,
      "activations/layer16_attention_weight_min": -32.622886657714844,
      "activations/layer17_attention_weight_max": 54.922611236572266,
      "activations/layer17_attention_weight_min": -45.29157257080078,
      "activations/layer18_attention_weight_max": 48.64739990234375,
      "activations/layer18_attention_weight_min": -40.03201675415039,
      "activations/layer19_attention_weight_max": 23.90939712524414,
      "activations/layer19_attention_weight_min": -22.596742630004883,
      "activations/layer1_attention_weight_max": 17.103588104248047,
      "activations/layer1_attention_weight_min": -14.398236274719238,
      "activations/layer20_attention_weight_max": 23.503040313720703,
      "activations/layer20_attention_weight_min": -22.05711555480957,
      "activations/layer21_attention_weight_max": 38.87224578857422,
      "activations/layer21_attention_weight_min": -21.936555862426758,
      "activations/layer22_attention_weight_max": 32.34248352050781,
      "activations/layer22_attention_weight_min": -27.266525268554688,
      "activations/layer23_attention_weight_max": 39.41242980957031,
      "activations/layer23_attention_weight_min": -23.548513412475586,
      "activations/layer2_attention_weight_max": 31.12456512451172,
      "activations/layer2_attention_weight_min": -30.648496627807617,
      "activations/layer3_attention_weight_max": 93.08623504638672,
      "activations/layer3_attention_weight_min": -97.0820083618164,
      "activations/layer4_attention_weight_max": 91.01541137695312,
      "activations/layer4_attention_weight_min": -89.32154846191406,
      "activations/layer5_attention_weight_max": 68.55950927734375,
      "activations/layer5_attention_weight_min": -74.70516204833984,
      "activations/layer6_attention_weight_max": 52.358333587646484,
      "activations/layer6_attention_weight_min": -53.98134231567383,
      "activations/layer7_attention_weight_max": 64.41743469238281,
      "activations/layer7_attention_weight_min": -62.98088836669922,
      "activations/layer8_attention_weight_max": 46.97673416137695,
      "activations/layer8_attention_weight_min": -52.87643814086914,
      "activations/layer9_attention_weight_max": 46.63541030883789,
      "activations/layer9_attention_weight_min": -45.97370910644531,
      "epoch": 15.3,
      "learning_rate": 5.188143939393939e-05,
      "loss": 2.7399,
      "step": 263250
    },
    {
      "activations/layer0_attention_weight_max": 15.006084442138672,
      "activations/layer0_attention_weight_min": -13.929667472839355,
      "activations/layer10_attention_weight_max": 34.49813461303711,
      "activations/layer10_attention_weight_min": -34.35809326171875,
      "activations/layer11_attention_weight_max": 31.644649505615234,
      "activations/layer11_attention_weight_min": -33.41870880126953,
      "activations/layer12_attention_weight_max": 18.911300659179688,
      "activations/layer12_attention_weight_min": -27.175687789916992,
      "activations/layer13_attention_weight_max": 35.63404846191406,
      "activations/layer13_attention_weight_min": -29.562576293945312,
      "activations/layer14_attention_weight_max": 33.23102951049805,
      "activations/layer14_attention_weight_min": -31.15570640563965,
      "activations/layer15_attention_weight_max": 30.85392189025879,
      "activations/layer15_attention_weight_min": -29.985437393188477,
      "activations/layer16_attention_weight_max": 31.587114334106445,
      "activations/layer16_attention_weight_min": -32.412025451660156,
      "activations/layer17_attention_weight_max": 47.87942886352539,
      "activations/layer17_attention_weight_min": -46.17749786376953,
      "activations/layer18_attention_weight_max": 44.912899017333984,
      "activations/layer18_attention_weight_min": -39.420799255371094,
      "activations/layer19_attention_weight_max": 24.73188591003418,
      "activations/layer19_attention_weight_min": -24.496522903442383,
      "activations/layer1_attention_weight_max": 16.206388473510742,
      "activations/layer1_attention_weight_min": -14.044001579284668,
      "activations/layer20_attention_weight_max": 22.599849700927734,
      "activations/layer20_attention_weight_min": -22.60209846496582,
      "activations/layer21_attention_weight_max": 32.94502639770508,
      "activations/layer21_attention_weight_min": -21.43947410583496,
      "activations/layer22_attention_weight_max": 27.388595581054688,
      "activations/layer22_attention_weight_min": -24.846242904663086,
      "activations/layer23_attention_weight_max": 33.999820709228516,
      "activations/layer23_attention_weight_min": -27.359983444213867,
      "activations/layer2_attention_weight_max": 31.17215347290039,
      "activations/layer2_attention_weight_min": -30.682239532470703,
      "activations/layer3_attention_weight_max": 87.3170394897461,
      "activations/layer3_attention_weight_min": -96.41541290283203,
      "activations/layer4_attention_weight_max": 86.67918395996094,
      "activations/layer4_attention_weight_min": -88.94170379638672,
      "activations/layer5_attention_weight_max": 66.7724609375,
      "activations/layer5_attention_weight_min": -76.71148681640625,
      "activations/layer6_attention_weight_max": 48.076210021972656,
      "activations/layer6_attention_weight_min": -51.69392776489258,
      "activations/layer7_attention_weight_max": 61.23500061035156,
      "activations/layer7_attention_weight_min": -61.2894172668457,
      "activations/layer8_attention_weight_max": 45.083946228027344,
      "activations/layer8_attention_weight_min": -52.60710525512695,
      "activations/layer9_attention_weight_max": 42.16143798828125,
      "activations/layer9_attention_weight_min": -45.25169372558594,
      "epoch": 15.3,
      "learning_rate": 5.186287878787878e-05,
      "loss": 2.7347,
      "step": 263300
    },
    {
      "activations/layer0_attention_weight_max": 15.029579162597656,
      "activations/layer0_attention_weight_min": -13.778968811035156,
      "activations/layer10_attention_weight_max": 46.21637725830078,
      "activations/layer10_attention_weight_min": -44.51081085205078,
      "activations/layer11_attention_weight_max": 39.76298522949219,
      "activations/layer11_attention_weight_min": -40.204837799072266,
      "activations/layer12_attention_weight_max": 19.493026733398438,
      "activations/layer12_attention_weight_min": -26.593223571777344,
      "activations/layer13_attention_weight_max": 41.83788299560547,
      "activations/layer13_attention_weight_min": -30.358417510986328,
      "activations/layer14_attention_weight_max": 40.85122299194336,
      "activations/layer14_attention_weight_min": -31.193273544311523,
      "activations/layer15_attention_weight_max": 42.865718841552734,
      "activations/layer15_attention_weight_min": -29.05663299560547,
      "activations/layer16_attention_weight_max": 37.55144119262695,
      "activations/layer16_attention_weight_min": -32.08926773071289,
      "activations/layer17_attention_weight_max": 52.566444396972656,
      "activations/layer17_attention_weight_min": -43.570411682128906,
      "activations/layer18_attention_weight_max": 51.040828704833984,
      "activations/layer18_attention_weight_min": -43.34520721435547,
      "activations/layer19_attention_weight_max": 27.981184005737305,
      "activations/layer19_attention_weight_min": -26.249439239501953,
      "activations/layer1_attention_weight_max": 16.576305389404297,
      "activations/layer1_attention_weight_min": -14.578080177307129,
      "activations/layer20_attention_weight_max": 27.43512535095215,
      "activations/layer20_attention_weight_min": -23.608976364135742,
      "activations/layer21_attention_weight_max": 45.66046142578125,
      "activations/layer21_attention_weight_min": -24.94479751586914,
      "activations/layer22_attention_weight_max": 34.650386810302734,
      "activations/layer22_attention_weight_min": -25.738319396972656,
      "activations/layer23_attention_weight_max": 53.491573333740234,
      "activations/layer23_attention_weight_min": -25.473663330078125,
      "activations/layer2_attention_weight_max": 31.59967041015625,
      "activations/layer2_attention_weight_min": -30.547536849975586,
      "activations/layer3_attention_weight_max": 88.14144897460938,
      "activations/layer3_attention_weight_min": -91.89045715332031,
      "activations/layer4_attention_weight_max": 91.43548583984375,
      "activations/layer4_attention_weight_min": -87.73802947998047,
      "activations/layer5_attention_weight_max": 68.18646240234375,
      "activations/layer5_attention_weight_min": -75.09384155273438,
      "activations/layer6_attention_weight_max": 52.211063385009766,
      "activations/layer6_attention_weight_min": -51.15834045410156,
      "activations/layer7_attention_weight_max": 68.0545654296875,
      "activations/layer7_attention_weight_min": -60.931861877441406,
      "activations/layer8_attention_weight_max": 56.88410186767578,
      "activations/layer8_attention_weight_min": -53.25824737548828,
      "activations/layer9_attention_weight_max": 59.412899017333984,
      "activations/layer9_attention_weight_min": -51.67659378051758,
      "epoch": 15.3,
      "learning_rate": 5.184393939393939e-05,
      "loss": 2.7456,
      "step": 263350
    },
    {
      "activations/layer0_attention_weight_max": 15.93924331665039,
      "activations/layer0_attention_weight_min": -14.029647827148438,
      "activations/layer10_attention_weight_max": 40.03364944458008,
      "activations/layer10_attention_weight_min": -37.33108139038086,
      "activations/layer11_attention_weight_max": 34.69944381713867,
      "activations/layer11_attention_weight_min": -34.69325637817383,
      "activations/layer12_attention_weight_max": 19.41682243347168,
      "activations/layer12_attention_weight_min": -24.432783126831055,
      "activations/layer13_attention_weight_max": 33.63385009765625,
      "activations/layer13_attention_weight_min": -26.693439483642578,
      "activations/layer14_attention_weight_max": 33.84423065185547,
      "activations/layer14_attention_weight_min": -31.39435577392578,
      "activations/layer15_attention_weight_max": 31.045148849487305,
      "activations/layer15_attention_weight_min": -29.779340744018555,
      "activations/layer16_attention_weight_max": 31.330121994018555,
      "activations/layer16_attention_weight_min": -29.360551834106445,
      "activations/layer17_attention_weight_max": 46.80207824707031,
      "activations/layer17_attention_weight_min": -43.81620788574219,
      "activations/layer18_attention_weight_max": 43.7861213684082,
      "activations/layer18_attention_weight_min": -40.1276969909668,
      "activations/layer19_attention_weight_max": 23.651174545288086,
      "activations/layer19_attention_weight_min": -22.290700912475586,
      "activations/layer1_attention_weight_max": 16.058387756347656,
      "activations/layer1_attention_weight_min": -15.131600379943848,
      "activations/layer20_attention_weight_max": 21.6475830078125,
      "activations/layer20_attention_weight_min": -22.82731819152832,
      "activations/layer21_attention_weight_max": 37.709693908691406,
      "activations/layer21_attention_weight_min": -27.454160690307617,
      "activations/layer22_attention_weight_max": 30.58538055419922,
      "activations/layer22_attention_weight_min": -24.770673751831055,
      "activations/layer23_attention_weight_max": 36.13300323486328,
      "activations/layer23_attention_weight_min": -24.158023834228516,
      "activations/layer2_attention_weight_max": 32.54020690917969,
      "activations/layer2_attention_weight_min": -31.0665283203125,
      "activations/layer3_attention_weight_max": 95.72364807128906,
      "activations/layer3_attention_weight_min": -93.79136657714844,
      "activations/layer4_attention_weight_max": 93.27898406982422,
      "activations/layer4_attention_weight_min": -92.13524627685547,
      "activations/layer5_attention_weight_max": 68.51933288574219,
      "activations/layer5_attention_weight_min": -71.47328186035156,
      "activations/layer6_attention_weight_max": 55.37248992919922,
      "activations/layer6_attention_weight_min": -52.77018356323242,
      "activations/layer7_attention_weight_max": 62.1898078918457,
      "activations/layer7_attention_weight_min": -60.70100402832031,
      "activations/layer8_attention_weight_max": 47.23441696166992,
      "activations/layer8_attention_weight_min": -49.53183364868164,
      "activations/layer9_attention_weight_max": 47.51215744018555,
      "activations/layer9_attention_weight_min": -46.955867767333984,
      "epoch": 15.31,
      "learning_rate": 5.182499999999999e-05,
      "loss": 2.7361,
      "step": 263400
    },
    {
      "activations/layer0_attention_weight_max": 16.062313079833984,
      "activations/layer0_attention_weight_min": -14.352540969848633,
      "activations/layer10_attention_weight_max": 35.39033508300781,
      "activations/layer10_attention_weight_min": -34.23724365234375,
      "activations/layer11_attention_weight_max": 30.591384887695312,
      "activations/layer11_attention_weight_min": -33.040889739990234,
      "activations/layer12_attention_weight_max": 17.696199417114258,
      "activations/layer12_attention_weight_min": -25.2230167388916,
      "activations/layer13_attention_weight_max": 31.175790786743164,
      "activations/layer13_attention_weight_min": -28.114948272705078,
      "activations/layer14_attention_weight_max": 32.54474639892578,
      "activations/layer14_attention_weight_min": -31.165616989135742,
      "activations/layer15_attention_weight_max": 29.344240188598633,
      "activations/layer15_attention_weight_min": -30.648447036743164,
      "activations/layer16_attention_weight_max": 31.63446044921875,
      "activations/layer16_attention_weight_min": -31.27480125427246,
      "activations/layer17_attention_weight_max": 48.3063850402832,
      "activations/layer17_attention_weight_min": -48.69514846801758,
      "activations/layer18_attention_weight_max": 44.28437042236328,
      "activations/layer18_attention_weight_min": -41.14394760131836,
      "activations/layer19_attention_weight_max": 23.11863899230957,
      "activations/layer19_attention_weight_min": -21.968748092651367,
      "activations/layer1_attention_weight_max": 17.36928939819336,
      "activations/layer1_attention_weight_min": -15.300917625427246,
      "activations/layer20_attention_weight_max": 21.323955535888672,
      "activations/layer20_attention_weight_min": -23.037351608276367,
      "activations/layer21_attention_weight_max": 32.66472625732422,
      "activations/layer21_attention_weight_min": -21.821975708007812,
      "activations/layer22_attention_weight_max": 27.221229553222656,
      "activations/layer22_attention_weight_min": -29.573123931884766,
      "activations/layer23_attention_weight_max": 34.27833938598633,
      "activations/layer23_attention_weight_min": -25.157217025756836,
      "activations/layer2_attention_weight_max": 30.48333740234375,
      "activations/layer2_attention_weight_min": -29.769874572753906,
      "activations/layer3_attention_weight_max": 87.93431854248047,
      "activations/layer3_attention_weight_min": -89.28177642822266,
      "activations/layer4_attention_weight_max": 88.7727279663086,
      "activations/layer4_attention_weight_min": -85.17937469482422,
      "activations/layer5_attention_weight_max": 62.56439208984375,
      "activations/layer5_attention_weight_min": -72.91162109375,
      "activations/layer6_attention_weight_max": 49.993587493896484,
      "activations/layer6_attention_weight_min": -49.76945495605469,
      "activations/layer7_attention_weight_max": 56.28879928588867,
      "activations/layer7_attention_weight_min": -58.01823806762695,
      "activations/layer8_attention_weight_max": 44.10563659667969,
      "activations/layer8_attention_weight_min": -46.21101760864258,
      "activations/layer9_attention_weight_max": 42.582828521728516,
      "activations/layer9_attention_weight_min": -42.4515495300293,
      "epoch": 15.31,
      "learning_rate": 5.18060606060606e-05,
      "loss": 2.7692,
      "step": 263450
    },
    {
      "activations/layer0_attention_weight_max": 15.32607364654541,
      "activations/layer0_attention_weight_min": -13.532549858093262,
      "activations/layer10_attention_weight_max": 33.310604095458984,
      "activations/layer10_attention_weight_min": -33.97734069824219,
      "activations/layer11_attention_weight_max": 30.50690269470215,
      "activations/layer11_attention_weight_min": -31.758914947509766,
      "activations/layer12_attention_weight_max": 19.261018753051758,
      "activations/layer12_attention_weight_min": -22.760452270507812,
      "activations/layer13_attention_weight_max": 37.29232406616211,
      "activations/layer13_attention_weight_min": -29.54033660888672,
      "activations/layer14_attention_weight_max": 38.44294738769531,
      "activations/layer14_attention_weight_min": -29.770299911499023,
      "activations/layer15_attention_weight_max": 30.86540985107422,
      "activations/layer15_attention_weight_min": -29.70183753967285,
      "activations/layer16_attention_weight_max": 30.869686126708984,
      "activations/layer16_attention_weight_min": -30.500402450561523,
      "activations/layer17_attention_weight_max": 49.66714859008789,
      "activations/layer17_attention_weight_min": -45.153194427490234,
      "activations/layer18_attention_weight_max": 43.6036262512207,
      "activations/layer18_attention_weight_min": -37.52740478515625,
      "activations/layer19_attention_weight_max": 23.465837478637695,
      "activations/layer19_attention_weight_min": -22.428607940673828,
      "activations/layer1_attention_weight_max": 16.978334426879883,
      "activations/layer1_attention_weight_min": -13.959257125854492,
      "activations/layer20_attention_weight_max": 20.841140747070312,
      "activations/layer20_attention_weight_min": -23.33131217956543,
      "activations/layer21_attention_weight_max": 32.72513198852539,
      "activations/layer21_attention_weight_min": -21.875513076782227,
      "activations/layer22_attention_weight_max": 27.26058578491211,
      "activations/layer22_attention_weight_min": -25.186609268188477,
      "activations/layer23_attention_weight_max": 34.30803680419922,
      "activations/layer23_attention_weight_min": -24.990726470947266,
      "activations/layer2_attention_weight_max": 30.79947280883789,
      "activations/layer2_attention_weight_min": -29.925390243530273,
      "activations/layer3_attention_weight_max": 90.66639709472656,
      "activations/layer3_attention_weight_min": -90.78138732910156,
      "activations/layer4_attention_weight_max": 86.08746337890625,
      "activations/layer4_attention_weight_min": -85.09417724609375,
      "activations/layer5_attention_weight_max": 62.13739013671875,
      "activations/layer5_attention_weight_min": -69.37591552734375,
      "activations/layer6_attention_weight_max": 47.7738151550293,
      "activations/layer6_attention_weight_min": -47.31184387207031,
      "activations/layer7_attention_weight_max": 56.6877326965332,
      "activations/layer7_attention_weight_min": -56.16577911376953,
      "activations/layer8_attention_weight_max": 42.61408615112305,
      "activations/layer8_attention_weight_min": -49.71458053588867,
      "activations/layer9_attention_weight_max": 41.04386520385742,
      "activations/layer9_attention_weight_min": -46.05532455444336,
      "epoch": 15.31,
      "learning_rate": 5.178712121212121e-05,
      "loss": 2.7501,
      "step": 263500
    },
    {
      "activations/layer0_attention_weight_max": 16.802644729614258,
      "activations/layer0_attention_weight_min": -14.055730819702148,
      "activations/layer10_attention_weight_max": 40.309974670410156,
      "activations/layer10_attention_weight_min": -36.87166213989258,
      "activations/layer11_attention_weight_max": 35.00166320800781,
      "activations/layer11_attention_weight_min": -35.81607437133789,
      "activations/layer12_attention_weight_max": 21.270944595336914,
      "activations/layer12_attention_weight_min": -28.506370544433594,
      "activations/layer13_attention_weight_max": 34.91143798828125,
      "activations/layer13_attention_weight_min": -29.08169937133789,
      "activations/layer14_attention_weight_max": 36.3814697265625,
      "activations/layer14_attention_weight_min": -32.11676788330078,
      "activations/layer15_attention_weight_max": 32.812477111816406,
      "activations/layer15_attention_weight_min": -30.401636123657227,
      "activations/layer16_attention_weight_max": 33.201141357421875,
      "activations/layer16_attention_weight_min": -33.83011245727539,
      "activations/layer17_attention_weight_max": 52.072933197021484,
      "activations/layer17_attention_weight_min": -47.03036117553711,
      "activations/layer18_attention_weight_max": 44.86174774169922,
      "activations/layer18_attention_weight_min": -40.062198638916016,
      "activations/layer19_attention_weight_max": 22.579608917236328,
      "activations/layer19_attention_weight_min": -22.96162223815918,
      "activations/layer1_attention_weight_max": 16.174882888793945,
      "activations/layer1_attention_weight_min": -14.525982856750488,
      "activations/layer20_attention_weight_max": 22.76797103881836,
      "activations/layer20_attention_weight_min": -23.101947784423828,
      "activations/layer21_attention_weight_max": 40.204280853271484,
      "activations/layer21_attention_weight_min": -25.593608856201172,
      "activations/layer22_attention_weight_max": 28.966732025146484,
      "activations/layer22_attention_weight_min": -27.654687881469727,
      "activations/layer23_attention_weight_max": 37.87529373168945,
      "activations/layer23_attention_weight_min": -28.19643211364746,
      "activations/layer2_attention_weight_max": 31.018096923828125,
      "activations/layer2_attention_weight_min": -31.537254333496094,
      "activations/layer3_attention_weight_max": 88.31290435791016,
      "activations/layer3_attention_weight_min": -94.07059478759766,
      "activations/layer4_attention_weight_max": 88.41383361816406,
      "activations/layer4_attention_weight_min": -88.8269271850586,
      "activations/layer5_attention_weight_max": 67.56863403320312,
      "activations/layer5_attention_weight_min": -73.88522338867188,
      "activations/layer6_attention_weight_max": 53.28852462768555,
      "activations/layer6_attention_weight_min": -50.938236236572266,
      "activations/layer7_attention_weight_max": 62.31884002685547,
      "activations/layer7_attention_weight_min": -61.06150817871094,
      "activations/layer8_attention_weight_max": 48.47873306274414,
      "activations/layer8_attention_weight_min": -52.38683319091797,
      "activations/layer9_attention_weight_max": 46.93303298950195,
      "activations/layer9_attention_weight_min": -45.91367721557617,
      "epoch": 15.31,
      "learning_rate": 5.176818181818181e-05,
      "loss": 2.7612,
      "step": 263550
    },
    {
      "activations/layer0_attention_weight_max": 15.814496040344238,
      "activations/layer0_attention_weight_min": -13.575539588928223,
      "activations/layer10_attention_weight_max": 35.937076568603516,
      "activations/layer10_attention_weight_min": -36.16346740722656,
      "activations/layer11_attention_weight_max": 32.85285186767578,
      "activations/layer11_attention_weight_min": -33.79787063598633,
      "activations/layer12_attention_weight_max": 18.888080596923828,
      "activations/layer12_attention_weight_min": -26.45139503479004,
      "activations/layer13_attention_weight_max": 37.99144744873047,
      "activations/layer13_attention_weight_min": -29.09678077697754,
      "activations/layer14_attention_weight_max": 43.0694580078125,
      "activations/layer14_attention_weight_min": -34.34208679199219,
      "activations/layer15_attention_weight_max": 36.10955810546875,
      "activations/layer15_attention_weight_min": -32.34541702270508,
      "activations/layer16_attention_weight_max": 35.21268081665039,
      "activations/layer16_attention_weight_min": -34.55626678466797,
      "activations/layer17_attention_weight_max": 53.755958557128906,
      "activations/layer17_attention_weight_min": -51.63447570800781,
      "activations/layer18_attention_weight_max": 52.39426803588867,
      "activations/layer18_attention_weight_min": -43.85403823852539,
      "activations/layer19_attention_weight_max": 22.763933181762695,
      "activations/layer19_attention_weight_min": -25.52776336669922,
      "activations/layer1_attention_weight_max": 16.872312545776367,
      "activations/layer1_attention_weight_min": -15.875226020812988,
      "activations/layer20_attention_weight_max": 24.031450271606445,
      "activations/layer20_attention_weight_min": -21.05803680419922,
      "activations/layer21_attention_weight_max": 49.9603271484375,
      "activations/layer21_attention_weight_min": -25.45553970336914,
      "activations/layer22_attention_weight_max": 34.05540466308594,
      "activations/layer22_attention_weight_min": -26.324209213256836,
      "activations/layer23_attention_weight_max": 45.50953674316406,
      "activations/layer23_attention_weight_min": -25.09865379333496,
      "activations/layer2_attention_weight_max": 30.468399047851562,
      "activations/layer2_attention_weight_min": -29.303791046142578,
      "activations/layer3_attention_weight_max": 86.34752655029297,
      "activations/layer3_attention_weight_min": -93.2550048828125,
      "activations/layer4_attention_weight_max": 89.0627670288086,
      "activations/layer4_attention_weight_min": -88.72969055175781,
      "activations/layer5_attention_weight_max": 64.5794677734375,
      "activations/layer5_attention_weight_min": -75.90447998046875,
      "activations/layer6_attention_weight_max": 50.10331344604492,
      "activations/layer6_attention_weight_min": -51.6348762512207,
      "activations/layer7_attention_weight_max": 59.75819778442383,
      "activations/layer7_attention_weight_min": -59.88401412963867,
      "activations/layer8_attention_weight_max": 44.63063049316406,
      "activations/layer8_attention_weight_min": -52.07340621948242,
      "activations/layer9_attention_weight_max": 45.717430114746094,
      "activations/layer9_attention_weight_min": -46.858680725097656,
      "epoch": 15.32,
      "learning_rate": 5.1749242424242426e-05,
      "loss": 2.7446,
      "step": 263600
    },
    {
      "activations/layer0_attention_weight_max": 15.929753303527832,
      "activations/layer0_attention_weight_min": -13.675681114196777,
      "activations/layer10_attention_weight_max": 38.16306686401367,
      "activations/layer10_attention_weight_min": -38.43220520019531,
      "activations/layer11_attention_weight_max": 33.83087921142578,
      "activations/layer11_attention_weight_min": -34.67803192138672,
      "activations/layer12_attention_weight_max": 19.114295959472656,
      "activations/layer12_attention_weight_min": -23.208717346191406,
      "activations/layer13_attention_weight_max": 41.475364685058594,
      "activations/layer13_attention_weight_min": -31.68088150024414,
      "activations/layer14_attention_weight_max": 42.368534088134766,
      "activations/layer14_attention_weight_min": -30.196422576904297,
      "activations/layer15_attention_weight_max": 37.98171615600586,
      "activations/layer15_attention_weight_min": -29.400882720947266,
      "activations/layer16_attention_weight_max": 36.3167724609375,
      "activations/layer16_attention_weight_min": -32.671688079833984,
      "activations/layer17_attention_weight_max": 54.29617691040039,
      "activations/layer17_attention_weight_min": -45.88522720336914,
      "activations/layer18_attention_weight_max": 49.18785095214844,
      "activations/layer18_attention_weight_min": -37.85029220581055,
      "activations/layer19_attention_weight_max": 22.15028953552246,
      "activations/layer19_attention_weight_min": -23.37250328063965,
      "activations/layer1_attention_weight_max": 17.417280197143555,
      "activations/layer1_attention_weight_min": -13.637947082519531,
      "activations/layer20_attention_weight_max": 22.533357620239258,
      "activations/layer20_attention_weight_min": -21.150876998901367,
      "activations/layer21_attention_weight_max": 40.67847442626953,
      "activations/layer21_attention_weight_min": -23.338577270507812,
      "activations/layer22_attention_weight_max": 33.15330123901367,
      "activations/layer22_attention_weight_min": -28.70409393310547,
      "activations/layer23_attention_weight_max": 37.16189193725586,
      "activations/layer23_attention_weight_min": -24.524885177612305,
      "activations/layer2_attention_weight_max": 31.096843719482422,
      "activations/layer2_attention_weight_min": -31.216161727905273,
      "activations/layer3_attention_weight_max": 92.18231964111328,
      "activations/layer3_attention_weight_min": -96.64923858642578,
      "activations/layer4_attention_weight_max": 88.78520965576172,
      "activations/layer4_attention_weight_min": -89.42125701904297,
      "activations/layer5_attention_weight_max": 64.63968658447266,
      "activations/layer5_attention_weight_min": -71.88902282714844,
      "activations/layer6_attention_weight_max": 48.542259216308594,
      "activations/layer6_attention_weight_min": -49.88799285888672,
      "activations/layer7_attention_weight_max": 58.62535858154297,
      "activations/layer7_attention_weight_min": -62.2678337097168,
      "activations/layer8_attention_weight_max": 46.79813003540039,
      "activations/layer8_attention_weight_min": -51.06007385253906,
      "activations/layer9_attention_weight_max": 47.57750701904297,
      "activations/layer9_attention_weight_min": -50.004451751708984,
      "epoch": 15.32,
      "learning_rate": 5.173030303030303e-05,
      "loss": 2.7484,
      "step": 263650
    },
    {
      "activations/layer0_attention_weight_max": 16.33034896850586,
      "activations/layer0_attention_weight_min": -13.71108341217041,
      "activations/layer10_attention_weight_max": 32.57196807861328,
      "activations/layer10_attention_weight_min": -33.62202453613281,
      "activations/layer11_attention_weight_max": 31.077260971069336,
      "activations/layer11_attention_weight_min": -32.9585075378418,
      "activations/layer12_attention_weight_max": 17.563838958740234,
      "activations/layer12_attention_weight_min": -25.1299991607666,
      "activations/layer13_attention_weight_max": 30.732524871826172,
      "activations/layer13_attention_weight_min": -27.521154403686523,
      "activations/layer14_attention_weight_max": 31.283443450927734,
      "activations/layer14_attention_weight_min": -28.510501861572266,
      "activations/layer15_attention_weight_max": 28.24707794189453,
      "activations/layer15_attention_weight_min": -29.336719512939453,
      "activations/layer16_attention_weight_max": 28.72292137145996,
      "activations/layer16_attention_weight_min": -29.520755767822266,
      "activations/layer17_attention_weight_max": 45.01304626464844,
      "activations/layer17_attention_weight_min": -42.037601470947266,
      "activations/layer18_attention_weight_max": 42.04268264770508,
      "activations/layer18_attention_weight_min": -36.338069915771484,
      "activations/layer19_attention_weight_max": 22.34331703186035,
      "activations/layer19_attention_weight_min": -22.41168975830078,
      "activations/layer1_attention_weight_max": 15.718926429748535,
      "activations/layer1_attention_weight_min": -13.703721046447754,
      "activations/layer20_attention_weight_max": 20.17868423461914,
      "activations/layer20_attention_weight_min": -21.037763595581055,
      "activations/layer21_attention_weight_max": 31.07758140563965,
      "activations/layer21_attention_weight_min": -23.834951400756836,
      "activations/layer22_attention_weight_max": 29.75496482849121,
      "activations/layer22_attention_weight_min": -27.13283920288086,
      "activations/layer23_attention_weight_max": 40.051422119140625,
      "activations/layer23_attention_weight_min": -24.71345329284668,
      "activations/layer2_attention_weight_max": 32.38938903808594,
      "activations/layer2_attention_weight_min": -32.542415618896484,
      "activations/layer3_attention_weight_max": 88.00012969970703,
      "activations/layer3_attention_weight_min": -95.92247772216797,
      "activations/layer4_attention_weight_max": 87.97913360595703,
      "activations/layer4_attention_weight_min": -90.95730590820312,
      "activations/layer5_attention_weight_max": 66.51376342773438,
      "activations/layer5_attention_weight_min": -74.7918701171875,
      "activations/layer6_attention_weight_max": 49.33884048461914,
      "activations/layer6_attention_weight_min": -51.04159164428711,
      "activations/layer7_attention_weight_max": 59.81757736206055,
      "activations/layer7_attention_weight_min": -59.67342758178711,
      "activations/layer8_attention_weight_max": 43.75172424316406,
      "activations/layer8_attention_weight_min": -48.10749053955078,
      "activations/layer9_attention_weight_max": 43.541725158691406,
      "activations/layer9_attention_weight_min": -45.836219787597656,
      "epoch": 15.32,
      "learning_rate": 5.171136363636363e-05,
      "loss": 2.751,
      "step": 263700
    },
    {
      "activations/layer0_attention_weight_max": 15.732613563537598,
      "activations/layer0_attention_weight_min": -13.631304740905762,
      "activations/layer10_attention_weight_max": 35.66069412231445,
      "activations/layer10_attention_weight_min": -37.08007049560547,
      "activations/layer11_attention_weight_max": 31.91265869140625,
      "activations/layer11_attention_weight_min": -34.044029235839844,
      "activations/layer12_attention_weight_max": 18.60055923461914,
      "activations/layer12_attention_weight_min": -26.377277374267578,
      "activations/layer13_attention_weight_max": 35.916587829589844,
      "activations/layer13_attention_weight_min": -30.93010139465332,
      "activations/layer14_attention_weight_max": 36.82381057739258,
      "activations/layer14_attention_weight_min": -30.774736404418945,
      "activations/layer15_attention_weight_max": 32.76597213745117,
      "activations/layer15_attention_weight_min": -31.375192642211914,
      "activations/layer16_attention_weight_max": 32.50850296020508,
      "activations/layer16_attention_weight_min": -31.37962532043457,
      "activations/layer17_attention_weight_max": 50.365386962890625,
      "activations/layer17_attention_weight_min": -45.60256576538086,
      "activations/layer18_attention_weight_max": 44.44281005859375,
      "activations/layer18_attention_weight_min": -39.71044921875,
      "activations/layer19_attention_weight_max": 24.41248321533203,
      "activations/layer19_attention_weight_min": -23.160316467285156,
      "activations/layer1_attention_weight_max": 15.806234359741211,
      "activations/layer1_attention_weight_min": -14.55639362335205,
      "activations/layer20_attention_weight_max": 22.836166381835938,
      "activations/layer20_attention_weight_min": -23.675321578979492,
      "activations/layer21_attention_weight_max": 32.753578186035156,
      "activations/layer21_attention_weight_min": -22.30255889892578,
      "activations/layer22_attention_weight_max": 31.619661331176758,
      "activations/layer22_attention_weight_min": -26.394826889038086,
      "activations/layer23_attention_weight_max": 37.51496124267578,
      "activations/layer23_attention_weight_min": -23.146007537841797,
      "activations/layer2_attention_weight_max": 30.60869598388672,
      "activations/layer2_attention_weight_min": -30.618228912353516,
      "activations/layer3_attention_weight_max": 87.96875,
      "activations/layer3_attention_weight_min": -94.06617736816406,
      "activations/layer4_attention_weight_max": 91.49456024169922,
      "activations/layer4_attention_weight_min": -91.38899993896484,
      "activations/layer5_attention_weight_max": 67.00076293945312,
      "activations/layer5_attention_weight_min": -77.4372329711914,
      "activations/layer6_attention_weight_max": 49.57835006713867,
      "activations/layer6_attention_weight_min": -52.34619903564453,
      "activations/layer7_attention_weight_max": 62.30519104003906,
      "activations/layer7_attention_weight_min": -64.41627502441406,
      "activations/layer8_attention_weight_max": 45.13112258911133,
      "activations/layer8_attention_weight_min": -48.7863655090332,
      "activations/layer9_attention_weight_max": 46.863521575927734,
      "activations/layer9_attention_weight_min": -48.94306564331055,
      "epoch": 15.33,
      "learning_rate": 5.169242424242424e-05,
      "loss": 2.7557,
      "step": 263750
    },
    {
      "activations/layer0_attention_weight_max": 16.046579360961914,
      "activations/layer0_attention_weight_min": -13.777534484863281,
      "activations/layer10_attention_weight_max": 36.96558380126953,
      "activations/layer10_attention_weight_min": -36.049644470214844,
      "activations/layer11_attention_weight_max": 36.05094909667969,
      "activations/layer11_attention_weight_min": -34.48577117919922,
      "activations/layer12_attention_weight_max": 19.09701156616211,
      "activations/layer12_attention_weight_min": -24.50326919555664,
      "activations/layer13_attention_weight_max": 30.350624084472656,
      "activations/layer13_attention_weight_min": -27.55916976928711,
      "activations/layer14_attention_weight_max": 35.040279388427734,
      "activations/layer14_attention_weight_min": -29.693058013916016,
      "activations/layer15_attention_weight_max": 33.59009552001953,
      "activations/layer15_attention_weight_min": -29.338403701782227,
      "activations/layer16_attention_weight_max": 32.61080551147461,
      "activations/layer16_attention_weight_min": -30.648880004882812,
      "activations/layer17_attention_weight_max": 49.30319595336914,
      "activations/layer17_attention_weight_min": -43.44344711303711,
      "activations/layer18_attention_weight_max": 45.49555969238281,
      "activations/layer18_attention_weight_min": -38.521270751953125,
      "activations/layer19_attention_weight_max": 23.67285919189453,
      "activations/layer19_attention_weight_min": -24.77715492248535,
      "activations/layer1_attention_weight_max": 15.559823036193848,
      "activations/layer1_attention_weight_min": -13.78246784210205,
      "activations/layer20_attention_weight_max": 23.24510383605957,
      "activations/layer20_attention_weight_min": -22.784759521484375,
      "activations/layer21_attention_weight_max": 41.597354888916016,
      "activations/layer21_attention_weight_min": -25.583599090576172,
      "activations/layer22_attention_weight_max": 29.373125076293945,
      "activations/layer22_attention_weight_min": -29.599424362182617,
      "activations/layer23_attention_weight_max": 35.844120025634766,
      "activations/layer23_attention_weight_min": -26.396276473999023,
      "activations/layer2_attention_weight_max": 30.272930145263672,
      "activations/layer2_attention_weight_min": -30.161710739135742,
      "activations/layer3_attention_weight_max": 91.4525146484375,
      "activations/layer3_attention_weight_min": -96.12447357177734,
      "activations/layer4_attention_weight_max": 90.9411849975586,
      "activations/layer4_attention_weight_min": -90.97349548339844,
      "activations/layer5_attention_weight_max": 72.34697723388672,
      "activations/layer5_attention_weight_min": -78.76496124267578,
      "activations/layer6_attention_weight_max": 54.66440963745117,
      "activations/layer6_attention_weight_min": -52.97588348388672,
      "activations/layer7_attention_weight_max": 59.953819274902344,
      "activations/layer7_attention_weight_min": -61.253414154052734,
      "activations/layer8_attention_weight_max": 45.18532943725586,
      "activations/layer8_attention_weight_min": -47.652015686035156,
      "activations/layer9_attention_weight_max": 44.567134857177734,
      "activations/layer9_attention_weight_min": -46.62001419067383,
      "epoch": 15.33,
      "learning_rate": 5.1673484848484845e-05,
      "loss": 2.7544,
      "step": 263800
    },
    {
      "activations/layer0_attention_weight_max": 16.178604125976562,
      "activations/layer0_attention_weight_min": -13.82625961303711,
      "activations/layer10_attention_weight_max": 36.33600616455078,
      "activations/layer10_attention_weight_min": -37.816444396972656,
      "activations/layer11_attention_weight_max": 33.19449996948242,
      "activations/layer11_attention_weight_min": -35.4217529296875,
      "activations/layer12_attention_weight_max": 17.282625198364258,
      "activations/layer12_attention_weight_min": -23.476985931396484,
      "activations/layer13_attention_weight_max": 36.813682556152344,
      "activations/layer13_attention_weight_min": -27.682199478149414,
      "activations/layer14_attention_weight_max": 38.93221664428711,
      "activations/layer14_attention_weight_min": -29.824071884155273,
      "activations/layer15_attention_weight_max": 36.6226692199707,
      "activations/layer15_attention_weight_min": -29.142841339111328,
      "activations/layer16_attention_weight_max": 33.12863540649414,
      "activations/layer16_attention_weight_min": -31.66133689880371,
      "activations/layer17_attention_weight_max": 53.431087493896484,
      "activations/layer17_attention_weight_min": -45.22515106201172,
      "activations/layer18_attention_weight_max": 44.786781311035156,
      "activations/layer18_attention_weight_min": -38.52747344970703,
      "activations/layer19_attention_weight_max": 23.00767707824707,
      "activations/layer19_attention_weight_min": -23.233572006225586,
      "activations/layer1_attention_weight_max": 16.81833267211914,
      "activations/layer1_attention_weight_min": -15.30544662475586,
      "activations/layer20_attention_weight_max": 24.785594940185547,
      "activations/layer20_attention_weight_min": -21.171728134155273,
      "activations/layer21_attention_weight_max": 37.01384353637695,
      "activations/layer21_attention_weight_min": -21.0235595703125,
      "activations/layer22_attention_weight_max": 30.86092185974121,
      "activations/layer22_attention_weight_min": -24.514205932617188,
      "activations/layer23_attention_weight_max": 38.327171325683594,
      "activations/layer23_attention_weight_min": -23.02823257446289,
      "activations/layer2_attention_weight_max": 31.40517234802246,
      "activations/layer2_attention_weight_min": -30.48062515258789,
      "activations/layer3_attention_weight_max": 88.85106658935547,
      "activations/layer3_attention_weight_min": -92.01052856445312,
      "activations/layer4_attention_weight_max": 93.00371551513672,
      "activations/layer4_attention_weight_min": -91.52816772460938,
      "activations/layer5_attention_weight_max": 68.86651611328125,
      "activations/layer5_attention_weight_min": -77.12106323242188,
      "activations/layer6_attention_weight_max": 50.99623107910156,
      "activations/layer6_attention_weight_min": -52.971168518066406,
      "activations/layer7_attention_weight_max": 61.50722122192383,
      "activations/layer7_attention_weight_min": -58.49543380737305,
      "activations/layer8_attention_weight_max": 45.65376663208008,
      "activations/layer8_attention_weight_min": -48.15935134887695,
      "activations/layer9_attention_weight_max": 45.57463073730469,
      "activations/layer9_attention_weight_min": -45.36675262451172,
      "epoch": 15.33,
      "learning_rate": 5.1654545454545453e-05,
      "loss": 2.768,
      "step": 263850
    },
    {
      "activations/layer0_attention_weight_max": 15.093758583068848,
      "activations/layer0_attention_weight_min": -13.717726707458496,
      "activations/layer10_attention_weight_max": 42.68434143066406,
      "activations/layer10_attention_weight_min": -40.59487533569336,
      "activations/layer11_attention_weight_max": 36.47159194946289,
      "activations/layer11_attention_weight_min": -37.27039337158203,
      "activations/layer12_attention_weight_max": 18.72226333618164,
      "activations/layer12_attention_weight_min": -25.578622817993164,
      "activations/layer13_attention_weight_max": 35.615272521972656,
      "activations/layer13_attention_weight_min": -31.411264419555664,
      "activations/layer14_attention_weight_max": 37.42579650878906,
      "activations/layer14_attention_weight_min": -29.14408302307129,
      "activations/layer15_attention_weight_max": 34.75265121459961,
      "activations/layer15_attention_weight_min": -30.130422592163086,
      "activations/layer16_attention_weight_max": 33.9481086730957,
      "activations/layer16_attention_weight_min": -32.69009017944336,
      "activations/layer17_attention_weight_max": 52.36213302612305,
      "activations/layer17_attention_weight_min": -47.177364349365234,
      "activations/layer18_attention_weight_max": 45.1511344909668,
      "activations/layer18_attention_weight_min": -38.96099853515625,
      "activations/layer19_attention_weight_max": 24.017080307006836,
      "activations/layer19_attention_weight_min": -21.364486694335938,
      "activations/layer1_attention_weight_max": 18.215078353881836,
      "activations/layer1_attention_weight_min": -14.52112102508545,
      "activations/layer20_attention_weight_max": 21.88458251953125,
      "activations/layer20_attention_weight_min": -20.828411102294922,
      "activations/layer21_attention_weight_max": 34.197505950927734,
      "activations/layer21_attention_weight_min": -21.199260711669922,
      "activations/layer22_attention_weight_max": 29.20348358154297,
      "activations/layer22_attention_weight_min": -23.995582580566406,
      "activations/layer23_attention_weight_max": 36.70204162597656,
      "activations/layer23_attention_weight_min": -24.847068786621094,
      "activations/layer2_attention_weight_max": 34.10606002807617,
      "activations/layer2_attention_weight_min": -32.63105773925781,
      "activations/layer3_attention_weight_max": 92.3517074584961,
      "activations/layer3_attention_weight_min": -97.74030303955078,
      "activations/layer4_attention_weight_max": 94.48045349121094,
      "activations/layer4_attention_weight_min": -92.3369369506836,
      "activations/layer5_attention_weight_max": 70.19454193115234,
      "activations/layer5_attention_weight_min": -73.58746337890625,
      "activations/layer6_attention_weight_max": 55.19636535644531,
      "activations/layer6_attention_weight_min": -55.76618957519531,
      "activations/layer7_attention_weight_max": 64.1531753540039,
      "activations/layer7_attention_weight_min": -58.39490509033203,
      "activations/layer8_attention_weight_max": 53.29298782348633,
      "activations/layer8_attention_weight_min": -54.01423263549805,
      "activations/layer9_attention_weight_max": 51.698768615722656,
      "activations/layer9_attention_weight_min": -49.1816520690918,
      "epoch": 15.33,
      "learning_rate": 5.1635606060606055e-05,
      "loss": 2.7533,
      "step": 263900
    },
    {
      "activations/layer0_attention_weight_max": 15.37216854095459,
      "activations/layer0_attention_weight_min": -13.49140739440918,
      "activations/layer10_attention_weight_max": 36.7803955078125,
      "activations/layer10_attention_weight_min": -34.9858283996582,
      "activations/layer11_attention_weight_max": 33.13459014892578,
      "activations/layer11_attention_weight_min": -35.052268981933594,
      "activations/layer12_attention_weight_max": 18.274484634399414,
      "activations/layer12_attention_weight_min": -33.192649841308594,
      "activations/layer13_attention_weight_max": 32.95121765136719,
      "activations/layer13_attention_weight_min": -26.524370193481445,
      "activations/layer14_attention_weight_max": 39.48122024536133,
      "activations/layer14_attention_weight_min": -31.307313919067383,
      "activations/layer15_attention_weight_max": 35.99584197998047,
      "activations/layer15_attention_weight_min": -30.0682373046875,
      "activations/layer16_attention_weight_max": 34.779510498046875,
      "activations/layer16_attention_weight_min": -32.9931526184082,
      "activations/layer17_attention_weight_max": 51.07616424560547,
      "activations/layer17_attention_weight_min": -45.19746780395508,
      "activations/layer18_attention_weight_max": 49.00895690917969,
      "activations/layer18_attention_weight_min": -38.338287353515625,
      "activations/layer19_attention_weight_max": 25.614774703979492,
      "activations/layer19_attention_weight_min": -23.52989387512207,
      "activations/layer1_attention_weight_max": 15.86437702178955,
      "activations/layer1_attention_weight_min": -14.041045188903809,
      "activations/layer20_attention_weight_max": 23.3541316986084,
      "activations/layer20_attention_weight_min": -21.170597076416016,
      "activations/layer21_attention_weight_max": 38.88053894042969,
      "activations/layer21_attention_weight_min": -22.78139305114746,
      "activations/layer22_attention_weight_max": 30.71591567993164,
      "activations/layer22_attention_weight_min": -25.751375198364258,
      "activations/layer23_attention_weight_max": 40.503562927246094,
      "activations/layer23_attention_weight_min": -28.70589828491211,
      "activations/layer2_attention_weight_max": 31.796232223510742,
      "activations/layer2_attention_weight_min": -31.572235107421875,
      "activations/layer3_attention_weight_max": 89.1661376953125,
      "activations/layer3_attention_weight_min": -93.85889434814453,
      "activations/layer4_attention_weight_max": 93.22362518310547,
      "activations/layer4_attention_weight_min": -93.1891098022461,
      "activations/layer5_attention_weight_max": 68.2425537109375,
      "activations/layer5_attention_weight_min": -75.34175109863281,
      "activations/layer6_attention_weight_max": 53.586524963378906,
      "activations/layer6_attention_weight_min": -53.31620407104492,
      "activations/layer7_attention_weight_max": 64.13274383544922,
      "activations/layer7_attention_weight_min": -62.66730880737305,
      "activations/layer8_attention_weight_max": 45.14904022216797,
      "activations/layer8_attention_weight_min": -47.65633010864258,
      "activations/layer9_attention_weight_max": 46.57427978515625,
      "activations/layer9_attention_weight_min": -47.968936920166016,
      "epoch": 15.34,
      "learning_rate": 5.1616666666666656e-05,
      "loss": 2.7445,
      "step": 263950
    },
    {
      "activations/layer0_attention_weight_max": 15.344741821289062,
      "activations/layer0_attention_weight_min": -14.152032852172852,
      "activations/layer10_attention_weight_max": 38.43525695800781,
      "activations/layer10_attention_weight_min": -37.8398551940918,
      "activations/layer11_attention_weight_max": 34.60367202758789,
      "activations/layer11_attention_weight_min": -34.417205810546875,
      "activations/layer12_attention_weight_max": 22.07180404663086,
      "activations/layer12_attention_weight_min": -26.33643913269043,
      "activations/layer13_attention_weight_max": 34.19245529174805,
      "activations/layer13_attention_weight_min": -29.640893936157227,
      "activations/layer14_attention_weight_max": 37.175601959228516,
      "activations/layer14_attention_weight_min": -30.57646942138672,
      "activations/layer15_attention_weight_max": 31.403812408447266,
      "activations/layer15_attention_weight_min": -28.155012130737305,
      "activations/layer16_attention_weight_max": 32.400901794433594,
      "activations/layer16_attention_weight_min": -31.22991180419922,
      "activations/layer17_attention_weight_max": 49.42618179321289,
      "activations/layer17_attention_weight_min": -44.07759475708008,
      "activations/layer18_attention_weight_max": 47.21988296508789,
      "activations/layer18_attention_weight_min": -39.76407241821289,
      "activations/layer19_attention_weight_max": 23.751195907592773,
      "activations/layer19_attention_weight_min": -21.731075286865234,
      "activations/layer1_attention_weight_max": 16.601892471313477,
      "activations/layer1_attention_weight_min": -15.372614860534668,
      "activations/layer20_attention_weight_max": 22.686586380004883,
      "activations/layer20_attention_weight_min": -23.6042537689209,
      "activations/layer21_attention_weight_max": 34.172176361083984,
      "activations/layer21_attention_weight_min": -22.652542114257812,
      "activations/layer22_attention_weight_max": 33.4772834777832,
      "activations/layer22_attention_weight_min": -25.05150604248047,
      "activations/layer23_attention_weight_max": 39.091854095458984,
      "activations/layer23_attention_weight_min": -24.234169006347656,
      "activations/layer2_attention_weight_max": 32.551422119140625,
      "activations/layer2_attention_weight_min": -32.27994918823242,
      "activations/layer3_attention_weight_max": 92.81057739257812,
      "activations/layer3_attention_weight_min": -93.30149841308594,
      "activations/layer4_attention_weight_max": 91.68195343017578,
      "activations/layer4_attention_weight_min": -90.42473602294922,
      "activations/layer5_attention_weight_max": 68.93962097167969,
      "activations/layer5_attention_weight_min": -70.72291564941406,
      "activations/layer6_attention_weight_max": 50.957740783691406,
      "activations/layer6_attention_weight_min": -51.517967224121094,
      "activations/layer7_attention_weight_max": 63.161834716796875,
      "activations/layer7_attention_weight_min": -57.980255126953125,
      "activations/layer8_attention_weight_max": 45.310340881347656,
      "activations/layer8_attention_weight_min": -48.60142517089844,
      "activations/layer9_attention_weight_max": 45.67708969116211,
      "activations/layer9_attention_weight_min": -47.18562316894531,
      "epoch": 15.34,
      "learning_rate": 5.159772727272727e-05,
      "loss": 2.7455,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4498,
      "eval_samples_per_second": 508.181,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4498,
      "eval_openwebtext_samples_per_second": 508.181,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9193,
      "eval_wikitext_samples_per_second": 237.587,
      "step": 264000
    },
    {
      "epoch": 15.34,
      "eval_lambada_loss": 2.4296875,
      "eval_lambada_ppl": 11.3553329839266,
      "eval_lambada_runtime": 9.5498,
      "eval_lambada_samples_per_second": 509.856,
      "step": 264000
    },
    {
      "activations/layer0_attention_weight_max": 14.864705085754395,
      "activations/layer0_attention_weight_min": -13.853449821472168,
      "activations/layer10_attention_weight_max": 35.97115707397461,
      "activations/layer10_attention_weight_min": -35.948486328125,
      "activations/layer11_attention_weight_max": 32.24469757080078,
      "activations/layer11_attention_weight_min": -33.321434020996094,
      "activations/layer12_attention_weight_max": 19.231021881103516,
      "activations/layer12_attention_weight_min": -28.342384338378906,
      "activations/layer13_attention_weight_max": 32.68803787231445,
      "activations/layer13_attention_weight_min": -28.248987197875977,
      "activations/layer14_attention_weight_max": 34.72238540649414,
      "activations/layer14_attention_weight_min": -32.91166305541992,
      "activations/layer15_attention_weight_max": 35.30240249633789,
      "activations/layer15_attention_weight_min": -33.12766647338867,
      "activations/layer16_attention_weight_max": 37.661338806152344,
      "activations/layer16_attention_weight_min": -35.87395477294922,
      "activations/layer17_attention_weight_max": 58.6630859375,
      "activations/layer17_attention_weight_min": -57.24034881591797,
      "activations/layer18_attention_weight_max": 45.4293212890625,
      "activations/layer18_attention_weight_min": -46.46005630493164,
      "activations/layer19_attention_weight_max": 24.135427474975586,
      "activations/layer19_attention_weight_min": -25.357318878173828,
      "activations/layer1_attention_weight_max": 17.14913558959961,
      "activations/layer1_attention_weight_min": -16.146738052368164,
      "activations/layer20_attention_weight_max": 21.736892700195312,
      "activations/layer20_attention_weight_min": -25.2874813079834,
      "activations/layer21_attention_weight_max": 35.14557647705078,
      "activations/layer21_attention_weight_min": -25.991031646728516,
      "activations/layer22_attention_weight_max": 28.443283081054688,
      "activations/layer22_attention_weight_min": -27.355628967285156,
      "activations/layer23_attention_weight_max": 36.35511779785156,
      "activations/layer23_attention_weight_min": -26.545530319213867,
      "activations/layer2_attention_weight_max": 31.944477081298828,
      "activations/layer2_attention_weight_min": -30.04369354248047,
      "activations/layer3_attention_weight_max": 88.61600494384766,
      "activations/layer3_attention_weight_min": -90.22534942626953,
      "activations/layer4_attention_weight_max": 85.99583435058594,
      "activations/layer4_attention_weight_min": -83.0739517211914,
      "activations/layer5_attention_weight_max": 64.70986938476562,
      "activations/layer5_attention_weight_min": -70.77741241455078,
      "activations/layer6_attention_weight_max": 49.393699645996094,
      "activations/layer6_attention_weight_min": -49.77694320678711,
      "activations/layer7_attention_weight_max": 57.70360565185547,
      "activations/layer7_attention_weight_min": -58.85545349121094,
      "activations/layer8_attention_weight_max": 47.60575485229492,
      "activations/layer8_attention_weight_min": -50.216182708740234,
      "activations/layer9_attention_weight_max": 44.06858444213867,
      "activations/layer9_attention_weight_min": -44.56808853149414,
      "epoch": 15.34,
      "learning_rate": 5.157878787878787e-05,
      "loss": 2.7479,
      "step": 264050
    },
    {
      "activations/layer0_attention_weight_max": 15.0289945602417,
      "activations/layer0_attention_weight_min": -14.026473999023438,
      "activations/layer10_attention_weight_max": 35.92323303222656,
      "activations/layer10_attention_weight_min": -36.96039581298828,
      "activations/layer11_attention_weight_max": 32.72029113769531,
      "activations/layer11_attention_weight_min": -33.75819778442383,
      "activations/layer12_attention_weight_max": 18.322650909423828,
      "activations/layer12_attention_weight_min": -25.10002326965332,
      "activations/layer13_attention_weight_max": 40.227447509765625,
      "activations/layer13_attention_weight_min": -32.2675666809082,
      "activations/layer14_attention_weight_max": 37.2251091003418,
      "activations/layer14_attention_weight_min": -32.294822692871094,
      "activations/layer15_attention_weight_max": 32.677921295166016,
      "activations/layer15_attention_weight_min": -30.208585739135742,
      "activations/layer16_attention_weight_max": 33.51945114135742,
      "activations/layer16_attention_weight_min": -31.37801170349121,
      "activations/layer17_attention_weight_max": 51.662254333496094,
      "activations/layer17_attention_weight_min": -48.39747619628906,
      "activations/layer18_attention_weight_max": 43.60301208496094,
      "activations/layer18_attention_weight_min": -45.88226318359375,
      "activations/layer19_attention_weight_max": 23.636234283447266,
      "activations/layer19_attention_weight_min": -26.646732330322266,
      "activations/layer1_attention_weight_max": 16.761865615844727,
      "activations/layer1_attention_weight_min": -15.416849136352539,
      "activations/layer20_attention_weight_max": 22.419572830200195,
      "activations/layer20_attention_weight_min": -23.8218936920166,
      "activations/layer21_attention_weight_max": 36.85506820678711,
      "activations/layer21_attention_weight_min": -25.8131046295166,
      "activations/layer22_attention_weight_max": 28.563859939575195,
      "activations/layer22_attention_weight_min": -25.0767822265625,
      "activations/layer23_attention_weight_max": 34.2382926940918,
      "activations/layer23_attention_weight_min": -26.529373168945312,
      "activations/layer2_attention_weight_max": 30.47126579284668,
      "activations/layer2_attention_weight_min": -31.92966079711914,
      "activations/layer3_attention_weight_max": 83.7332763671875,
      "activations/layer3_attention_weight_min": -88.84882354736328,
      "activations/layer4_attention_weight_max": 81.81712341308594,
      "activations/layer4_attention_weight_min": -83.1866683959961,
      "activations/layer5_attention_weight_max": 65.66173553466797,
      "activations/layer5_attention_weight_min": -68.87572479248047,
      "activations/layer6_attention_weight_max": 48.181575775146484,
      "activations/layer6_attention_weight_min": -46.10317611694336,
      "activations/layer7_attention_weight_max": 59.119140625,
      "activations/layer7_attention_weight_min": -57.55561828613281,
      "activations/layer8_attention_weight_max": 42.85992431640625,
      "activations/layer8_attention_weight_min": -44.5418815612793,
      "activations/layer9_attention_weight_max": 42.272666931152344,
      "activations/layer9_attention_weight_min": -44.72825241088867,
      "epoch": 15.35,
      "learning_rate": 5.155984848484848e-05,
      "loss": 2.7592,
      "step": 264100
    },
    {
      "activations/layer0_attention_weight_max": 15.25353717803955,
      "activations/layer0_attention_weight_min": -14.068927764892578,
      "activations/layer10_attention_weight_max": 36.865901947021484,
      "activations/layer10_attention_weight_min": -36.563148498535156,
      "activations/layer11_attention_weight_max": 31.90765380859375,
      "activations/layer11_attention_weight_min": -34.32463455200195,
      "activations/layer12_attention_weight_max": 19.372303009033203,
      "activations/layer12_attention_weight_min": -31.262741088867188,
      "activations/layer13_attention_weight_max": 34.3880615234375,
      "activations/layer13_attention_weight_min": -28.53347396850586,
      "activations/layer14_attention_weight_max": 37.120262145996094,
      "activations/layer14_attention_weight_min": -30.13442611694336,
      "activations/layer15_attention_weight_max": 35.238426208496094,
      "activations/layer15_attention_weight_min": -30.696184158325195,
      "activations/layer16_attention_weight_max": 35.33354949951172,
      "activations/layer16_attention_weight_min": -32.12797164916992,
      "activations/layer17_attention_weight_max": 56.21016311645508,
      "activations/layer17_attention_weight_min": -45.53754425048828,
      "activations/layer18_attention_weight_max": 52.370052337646484,
      "activations/layer18_attention_weight_min": -40.33320236206055,
      "activations/layer19_attention_weight_max": 23.017576217651367,
      "activations/layer19_attention_weight_min": -24.907503128051758,
      "activations/layer1_attention_weight_max": 15.37905216217041,
      "activations/layer1_attention_weight_min": -16.76779556274414,
      "activations/layer20_attention_weight_max": 23.537071228027344,
      "activations/layer20_attention_weight_min": -22.94212532043457,
      "activations/layer21_attention_weight_max": 42.34381103515625,
      "activations/layer21_attention_weight_min": -30.21402931213379,
      "activations/layer22_attention_weight_max": 29.03341293334961,
      "activations/layer22_attention_weight_min": -26.060150146484375,
      "activations/layer23_attention_weight_max": 37.6838493347168,
      "activations/layer23_attention_weight_min": -25.572128295898438,
      "activations/layer2_attention_weight_max": 32.564125061035156,
      "activations/layer2_attention_weight_min": -31.204387664794922,
      "activations/layer3_attention_weight_max": 90.22358703613281,
      "activations/layer3_attention_weight_min": -92.56466674804688,
      "activations/layer4_attention_weight_max": 90.54039001464844,
      "activations/layer4_attention_weight_min": -90.34706115722656,
      "activations/layer5_attention_weight_max": 65.29891967773438,
      "activations/layer5_attention_weight_min": -72.87890625,
      "activations/layer6_attention_weight_max": 53.37354278564453,
      "activations/layer6_attention_weight_min": -57.12565231323242,
      "activations/layer7_attention_weight_max": 59.85361099243164,
      "activations/layer7_attention_weight_min": -61.45023727416992,
      "activations/layer8_attention_weight_max": 49.79951095581055,
      "activations/layer8_attention_weight_min": -53.75563430786133,
      "activations/layer9_attention_weight_max": 46.15561294555664,
      "activations/layer9_attention_weight_min": -49.54970932006836,
      "epoch": 15.35,
      "learning_rate": 5.154090909090908e-05,
      "loss": 2.7448,
      "step": 264150
    },
    {
      "activations/layer0_attention_weight_max": 15.740435600280762,
      "activations/layer0_attention_weight_min": -13.863312721252441,
      "activations/layer10_attention_weight_max": 38.84237289428711,
      "activations/layer10_attention_weight_min": -38.90634536743164,
      "activations/layer11_attention_weight_max": 32.20643997192383,
      "activations/layer11_attention_weight_min": -35.35243225097656,
      "activations/layer12_attention_weight_max": 20.405364990234375,
      "activations/layer12_attention_weight_min": -27.64874839782715,
      "activations/layer13_attention_weight_max": 36.95176315307617,
      "activations/layer13_attention_weight_min": -29.15664291381836,
      "activations/layer14_attention_weight_max": 37.012229919433594,
      "activations/layer14_attention_weight_min": -30.1453857421875,
      "activations/layer15_attention_weight_max": 34.44929504394531,
      "activations/layer15_attention_weight_min": -31.458019256591797,
      "activations/layer16_attention_weight_max": 34.73129653930664,
      "activations/layer16_attention_weight_min": -31.663143157958984,
      "activations/layer17_attention_weight_max": 53.59187698364258,
      "activations/layer17_attention_weight_min": -44.708248138427734,
      "activations/layer18_attention_weight_max": 47.90155029296875,
      "activations/layer18_attention_weight_min": -40.023338317871094,
      "activations/layer19_attention_weight_max": 24.378707885742188,
      "activations/layer19_attention_weight_min": -22.72142791748047,
      "activations/layer1_attention_weight_max": 16.628154754638672,
      "activations/layer1_attention_weight_min": -17.754220962524414,
      "activations/layer20_attention_weight_max": 24.534128189086914,
      "activations/layer20_attention_weight_min": -22.718101501464844,
      "activations/layer21_attention_weight_max": 47.55827713012695,
      "activations/layer21_attention_weight_min": -24.021465301513672,
      "activations/layer22_attention_weight_max": 30.198110580444336,
      "activations/layer22_attention_weight_min": -25.656452178955078,
      "activations/layer23_attention_weight_max": 38.32827377319336,
      "activations/layer23_attention_weight_min": -24.10224151611328,
      "activations/layer2_attention_weight_max": 31.3398494720459,
      "activations/layer2_attention_weight_min": -29.908428192138672,
      "activations/layer3_attention_weight_max": 94.94695281982422,
      "activations/layer3_attention_weight_min": -91.65701293945312,
      "activations/layer4_attention_weight_max": 90.7247543334961,
      "activations/layer4_attention_weight_min": -89.0876693725586,
      "activations/layer5_attention_weight_max": 66.36723327636719,
      "activations/layer5_attention_weight_min": -76.57227325439453,
      "activations/layer6_attention_weight_max": 50.6529655456543,
      "activations/layer6_attention_weight_min": -50.03577423095703,
      "activations/layer7_attention_weight_max": 58.46794509887695,
      "activations/layer7_attention_weight_min": -57.931640625,
      "activations/layer8_attention_weight_max": 46.21202850341797,
      "activations/layer8_attention_weight_min": -46.764041900634766,
      "activations/layer9_attention_weight_max": 45.64669418334961,
      "activations/layer9_attention_weight_min": -44.598182678222656,
      "epoch": 15.35,
      "learning_rate": 5.15219696969697e-05,
      "loss": 2.7536,
      "step": 264200
    },
    {
      "activations/layer0_attention_weight_max": 15.020857810974121,
      "activations/layer0_attention_weight_min": -14.050885200500488,
      "activations/layer10_attention_weight_max": 39.697853088378906,
      "activations/layer10_attention_weight_min": -36.223087310791016,
      "activations/layer11_attention_weight_max": 30.81397819519043,
      "activations/layer11_attention_weight_min": -31.921863555908203,
      "activations/layer12_attention_weight_max": 18.033859252929688,
      "activations/layer12_attention_weight_min": -24.253047943115234,
      "activations/layer13_attention_weight_max": 37.789344787597656,
      "activations/layer13_attention_weight_min": -31.38854217529297,
      "activations/layer14_attention_weight_max": 36.30813217163086,
      "activations/layer14_attention_weight_min": -29.926218032836914,
      "activations/layer15_attention_weight_max": 33.8317985534668,
      "activations/layer15_attention_weight_min": -29.545026779174805,
      "activations/layer16_attention_weight_max": 34.329776763916016,
      "activations/layer16_attention_weight_min": -33.27290725708008,
      "activations/layer17_attention_weight_max": 56.81050491333008,
      "activations/layer17_attention_weight_min": -47.36684036254883,
      "activations/layer18_attention_weight_max": 47.90357208251953,
      "activations/layer18_attention_weight_min": -40.67235565185547,
      "activations/layer19_attention_weight_max": 23.036006927490234,
      "activations/layer19_attention_weight_min": -21.940092086791992,
      "activations/layer1_attention_weight_max": 16.291227340698242,
      "activations/layer1_attention_weight_min": -15.523062705993652,
      "activations/layer20_attention_weight_max": 24.62383460998535,
      "activations/layer20_attention_weight_min": -22.02086639404297,
      "activations/layer21_attention_weight_max": 36.18107986450195,
      "activations/layer21_attention_weight_min": -23.29392433166504,
      "activations/layer22_attention_weight_max": 32.90617752075195,
      "activations/layer22_attention_weight_min": -27.16619110107422,
      "activations/layer23_attention_weight_max": 40.380027770996094,
      "activations/layer23_attention_weight_min": -24.874244689941406,
      "activations/layer2_attention_weight_max": 29.980253219604492,
      "activations/layer2_attention_weight_min": -31.023277282714844,
      "activations/layer3_attention_weight_max": 89.4111099243164,
      "activations/layer3_attention_weight_min": -93.41683959960938,
      "activations/layer4_attention_weight_max": 91.50267791748047,
      "activations/layer4_attention_weight_min": -93.1799545288086,
      "activations/layer5_attention_weight_max": 66.84220123291016,
      "activations/layer5_attention_weight_min": -76.17232513427734,
      "activations/layer6_attention_weight_max": 50.7181282043457,
      "activations/layer6_attention_weight_min": -54.12637710571289,
      "activations/layer7_attention_weight_max": 62.64939498901367,
      "activations/layer7_attention_weight_min": -62.638240814208984,
      "activations/layer8_attention_weight_max": 45.706485748291016,
      "activations/layer8_attention_weight_min": -50.03187942504883,
      "activations/layer9_attention_weight_max": 45.70950698852539,
      "activations/layer9_attention_weight_min": -50.06648635864258,
      "epoch": 15.35,
      "learning_rate": 5.15030303030303e-05,
      "loss": 2.7412,
      "step": 264250
    },
    {
      "activations/layer0_attention_weight_max": 15.932226181030273,
      "activations/layer0_attention_weight_min": -14.081767082214355,
      "activations/layer10_attention_weight_max": 37.366275787353516,
      "activations/layer10_attention_weight_min": -37.601661682128906,
      "activations/layer11_attention_weight_max": 34.58799362182617,
      "activations/layer11_attention_weight_min": -35.660552978515625,
      "activations/layer12_attention_weight_max": 19.462270736694336,
      "activations/layer12_attention_weight_min": -21.69253158569336,
      "activations/layer13_attention_weight_max": 35.02384948730469,
      "activations/layer13_attention_weight_min": -27.343873977661133,
      "activations/layer14_attention_weight_max": 35.38011932373047,
      "activations/layer14_attention_weight_min": -31.062511444091797,
      "activations/layer15_attention_weight_max": 32.80400848388672,
      "activations/layer15_attention_weight_min": -30.766990661621094,
      "activations/layer16_attention_weight_max": 33.945011138916016,
      "activations/layer16_attention_weight_min": -29.838340759277344,
      "activations/layer17_attention_weight_max": 56.75641632080078,
      "activations/layer17_attention_weight_min": -44.823204040527344,
      "activations/layer18_attention_weight_max": 50.18448257446289,
      "activations/layer18_attention_weight_min": -42.466670989990234,
      "activations/layer19_attention_weight_max": 23.691226959228516,
      "activations/layer19_attention_weight_min": -22.465547561645508,
      "activations/layer1_attention_weight_max": 16.320777893066406,
      "activations/layer1_attention_weight_min": -16.22562026977539,
      "activations/layer20_attention_weight_max": 24.04373550415039,
      "activations/layer20_attention_weight_min": -19.929771423339844,
      "activations/layer21_attention_weight_max": 41.95712661743164,
      "activations/layer21_attention_weight_min": -22.56088638305664,
      "activations/layer22_attention_weight_max": 31.24530029296875,
      "activations/layer22_attention_weight_min": -24.51095199584961,
      "activations/layer23_attention_weight_max": 37.96533203125,
      "activations/layer23_attention_weight_min": -23.917877197265625,
      "activations/layer2_attention_weight_max": 31.2260799407959,
      "activations/layer2_attention_weight_min": -30.62415313720703,
      "activations/layer3_attention_weight_max": 95.1383056640625,
      "activations/layer3_attention_weight_min": -91.26286315917969,
      "activations/layer4_attention_weight_max": 97.27413940429688,
      "activations/layer4_attention_weight_min": -93.39253234863281,
      "activations/layer5_attention_weight_max": 68.10797119140625,
      "activations/layer5_attention_weight_min": -71.17520141601562,
      "activations/layer6_attention_weight_max": 53.815914154052734,
      "activations/layer6_attention_weight_min": -54.096431732177734,
      "activations/layer7_attention_weight_max": 59.688690185546875,
      "activations/layer7_attention_weight_min": -61.40143585205078,
      "activations/layer8_attention_weight_max": 46.60008239746094,
      "activations/layer8_attention_weight_min": -50.163455963134766,
      "activations/layer9_attention_weight_max": 47.891483306884766,
      "activations/layer9_attention_weight_min": -49.065155029296875,
      "epoch": 15.36,
      "learning_rate": 5.14840909090909e-05,
      "loss": 2.7605,
      "step": 264300
    },
    {
      "activations/layer0_attention_weight_max": 16.689830780029297,
      "activations/layer0_attention_weight_min": -13.880766868591309,
      "activations/layer10_attention_weight_max": 36.0329475402832,
      "activations/layer10_attention_weight_min": -37.495826721191406,
      "activations/layer11_attention_weight_max": 32.51111602783203,
      "activations/layer11_attention_weight_min": -32.367225646972656,
      "activations/layer12_attention_weight_max": 18.3704891204834,
      "activations/layer12_attention_weight_min": -22.84771156311035,
      "activations/layer13_attention_weight_max": 36.67332077026367,
      "activations/layer13_attention_weight_min": -29.201725006103516,
      "activations/layer14_attention_weight_max": 37.3647575378418,
      "activations/layer14_attention_weight_min": -29.04051399230957,
      "activations/layer15_attention_weight_max": 35.547115325927734,
      "activations/layer15_attention_weight_min": -30.1977481842041,
      "activations/layer16_attention_weight_max": 33.137393951416016,
      "activations/layer16_attention_weight_min": -30.585371017456055,
      "activations/layer17_attention_weight_max": 55.16981887817383,
      "activations/layer17_attention_weight_min": -45.01864242553711,
      "activations/layer18_attention_weight_max": 51.256343841552734,
      "activations/layer18_attention_weight_min": -40.26364517211914,
      "activations/layer19_attention_weight_max": 22.604930877685547,
      "activations/layer19_attention_weight_min": -22.639835357666016,
      "activations/layer1_attention_weight_max": 17.165668487548828,
      "activations/layer1_attention_weight_min": -17.598533630371094,
      "activations/layer20_attention_weight_max": 24.155860900878906,
      "activations/layer20_attention_weight_min": -21.35173225402832,
      "activations/layer21_attention_weight_max": 42.64771270751953,
      "activations/layer21_attention_weight_min": -22.756324768066406,
      "activations/layer22_attention_weight_max": 29.256752014160156,
      "activations/layer22_attention_weight_min": -24.81920623779297,
      "activations/layer23_attention_weight_max": 38.08164978027344,
      "activations/layer23_attention_weight_min": -25.200469970703125,
      "activations/layer2_attention_weight_max": 30.080486297607422,
      "activations/layer2_attention_weight_min": -29.24338150024414,
      "activations/layer3_attention_weight_max": 93.76947784423828,
      "activations/layer3_attention_weight_min": -95.81204223632812,
      "activations/layer4_attention_weight_max": 94.42088317871094,
      "activations/layer4_attention_weight_min": -94.24540710449219,
      "activations/layer5_attention_weight_max": 69.16606140136719,
      "activations/layer5_attention_weight_min": -74.548583984375,
      "activations/layer6_attention_weight_max": 53.2760124206543,
      "activations/layer6_attention_weight_min": -52.388675689697266,
      "activations/layer7_attention_weight_max": 59.1432991027832,
      "activations/layer7_attention_weight_min": -60.022396087646484,
      "activations/layer8_attention_weight_max": 46.34086990356445,
      "activations/layer8_attention_weight_min": -48.39156723022461,
      "activations/layer9_attention_weight_max": 46.0583610534668,
      "activations/layer9_attention_weight_min": -47.28177261352539,
      "epoch": 15.36,
      "learning_rate": 5.1465151515151516e-05,
      "loss": 2.7334,
      "step": 264350
    },
    {
      "activations/layer0_attention_weight_max": 15.454798698425293,
      "activations/layer0_attention_weight_min": -14.197529792785645,
      "activations/layer10_attention_weight_max": 38.39841079711914,
      "activations/layer10_attention_weight_min": -35.099647521972656,
      "activations/layer11_attention_weight_max": 32.87504959106445,
      "activations/layer11_attention_weight_min": -32.643741607666016,
      "activations/layer12_attention_weight_max": 18.648141860961914,
      "activations/layer12_attention_weight_min": -23.416257858276367,
      "activations/layer13_attention_weight_max": 33.65214157104492,
      "activations/layer13_attention_weight_min": -27.35898208618164,
      "activations/layer14_attention_weight_max": 34.13063430786133,
      "activations/layer14_attention_weight_min": -30.99622344970703,
      "activations/layer15_attention_weight_max": 32.45738220214844,
      "activations/layer15_attention_weight_min": -30.122112274169922,
      "activations/layer16_attention_weight_max": 31.350736618041992,
      "activations/layer16_attention_weight_min": -30.284488677978516,
      "activations/layer17_attention_weight_max": 50.12684631347656,
      "activations/layer17_attention_weight_min": -44.77012252807617,
      "activations/layer18_attention_weight_max": 45.07711410522461,
      "activations/layer18_attention_weight_min": -41.104061126708984,
      "activations/layer19_attention_weight_max": 23.18696403503418,
      "activations/layer19_attention_weight_min": -22.42137908935547,
      "activations/layer1_attention_weight_max": 16.366647720336914,
      "activations/layer1_attention_weight_min": -14.619315147399902,
      "activations/layer20_attention_weight_max": 25.266700744628906,
      "activations/layer20_attention_weight_min": -20.602371215820312,
      "activations/layer21_attention_weight_max": 38.7597770690918,
      "activations/layer21_attention_weight_min": -20.31081771850586,
      "activations/layer22_attention_weight_max": 29.265439987182617,
      "activations/layer22_attention_weight_min": -24.86227035522461,
      "activations/layer23_attention_weight_max": 37.05318069458008,
      "activations/layer23_attention_weight_min": -24.534276962280273,
      "activations/layer2_attention_weight_max": 32.613929748535156,
      "activations/layer2_attention_weight_min": -32.018798828125,
      "activations/layer3_attention_weight_max": 90.07145690917969,
      "activations/layer3_attention_weight_min": -95.8287353515625,
      "activations/layer4_attention_weight_max": 94.92720794677734,
      "activations/layer4_attention_weight_min": -89.5103530883789,
      "activations/layer5_attention_weight_max": 66.33894348144531,
      "activations/layer5_attention_weight_min": -74.76419067382812,
      "activations/layer6_attention_weight_max": 53.84873962402344,
      "activations/layer6_attention_weight_min": -53.36531448364258,
      "activations/layer7_attention_weight_max": 64.14562225341797,
      "activations/layer7_attention_weight_min": -64.53629302978516,
      "activations/layer8_attention_weight_max": 47.8555908203125,
      "activations/layer8_attention_weight_min": -48.83660125732422,
      "activations/layer9_attention_weight_max": 47.086116790771484,
      "activations/layer9_attention_weight_min": -47.24778747558594,
      "epoch": 15.36,
      "learning_rate": 5.144621212121212e-05,
      "loss": 2.7429,
      "step": 264400
    },
    {
      "activations/layer0_attention_weight_max": 15.731380462646484,
      "activations/layer0_attention_weight_min": -14.177743911743164,
      "activations/layer10_attention_weight_max": 35.82645797729492,
      "activations/layer10_attention_weight_min": -33.58952713012695,
      "activations/layer11_attention_weight_max": 32.73004150390625,
      "activations/layer11_attention_weight_min": -33.235591888427734,
      "activations/layer12_attention_weight_max": 18.567838668823242,
      "activations/layer12_attention_weight_min": -24.54994010925293,
      "activations/layer13_attention_weight_max": 35.18769836425781,
      "activations/layer13_attention_weight_min": -26.302757263183594,
      "activations/layer14_attention_weight_max": 33.25715637207031,
      "activations/layer14_attention_weight_min": -30.40374183654785,
      "activations/layer15_attention_weight_max": 30.170921325683594,
      "activations/layer15_attention_weight_min": -28.43178367614746,
      "activations/layer16_attention_weight_max": 29.82887840270996,
      "activations/layer16_attention_weight_min": -28.967517852783203,
      "activations/layer17_attention_weight_max": 50.54818344116211,
      "activations/layer17_attention_weight_min": -43.754276275634766,
      "activations/layer18_attention_weight_max": 42.590999603271484,
      "activations/layer18_attention_weight_min": -36.048301696777344,
      "activations/layer19_attention_weight_max": 21.381811141967773,
      "activations/layer19_attention_weight_min": -20.745332717895508,
      "activations/layer1_attention_weight_max": 15.693840980529785,
      "activations/layer1_attention_weight_min": -14.304184913635254,
      "activations/layer20_attention_weight_max": 22.514787673950195,
      "activations/layer20_attention_weight_min": -19.84058952331543,
      "activations/layer21_attention_weight_max": 29.582901000976562,
      "activations/layer21_attention_weight_min": -22.452959060668945,
      "activations/layer22_attention_weight_max": 26.383691787719727,
      "activations/layer22_attention_weight_min": -26.12668228149414,
      "activations/layer23_attention_weight_max": 35.09337615966797,
      "activations/layer23_attention_weight_min": -25.087085723876953,
      "activations/layer2_attention_weight_max": 32.92668914794922,
      "activations/layer2_attention_weight_min": -31.19970703125,
      "activations/layer3_attention_weight_max": 99.24253845214844,
      "activations/layer3_attention_weight_min": -95.65862274169922,
      "activations/layer4_attention_weight_max": 97.22871398925781,
      "activations/layer4_attention_weight_min": -92.8448257446289,
      "activations/layer5_attention_weight_max": 70.29788208007812,
      "activations/layer5_attention_weight_min": -78.21760559082031,
      "activations/layer6_attention_weight_max": 54.25534439086914,
      "activations/layer6_attention_weight_min": -53.63215637207031,
      "activations/layer7_attention_weight_max": 60.589073181152344,
      "activations/layer7_attention_weight_min": -60.914337158203125,
      "activations/layer8_attention_weight_max": 46.467159271240234,
      "activations/layer8_attention_weight_min": -49.04191970825195,
      "activations/layer9_attention_weight_max": 45.63764190673828,
      "activations/layer9_attention_weight_min": -45.6291618347168,
      "epoch": 15.37,
      "learning_rate": 5.1427272727272726e-05,
      "loss": 2.7443,
      "step": 264450
    },
    {
      "activations/layer0_attention_weight_max": 16.093488693237305,
      "activations/layer0_attention_weight_min": -13.79454517364502,
      "activations/layer10_attention_weight_max": 35.50840759277344,
      "activations/layer10_attention_weight_min": -36.64817810058594,
      "activations/layer11_attention_weight_max": 31.541946411132812,
      "activations/layer11_attention_weight_min": -33.61041259765625,
      "activations/layer12_attention_weight_max": 18.832822799682617,
      "activations/layer12_attention_weight_min": -24.30129623413086,
      "activations/layer13_attention_weight_max": 36.40052795410156,
      "activations/layer13_attention_weight_min": -28.438365936279297,
      "activations/layer14_attention_weight_max": 37.00027084350586,
      "activations/layer14_attention_weight_min": -31.35637855529785,
      "activations/layer15_attention_weight_max": 31.303550720214844,
      "activations/layer15_attention_weight_min": -29.611536026000977,
      "activations/layer16_attention_weight_max": 32.69135284423828,
      "activations/layer16_attention_weight_min": -33.081642150878906,
      "activations/layer17_attention_weight_max": 52.72382736206055,
      "activations/layer17_attention_weight_min": -45.35507583618164,
      "activations/layer18_attention_weight_max": 47.18434143066406,
      "activations/layer18_attention_weight_min": -39.288692474365234,
      "activations/layer19_attention_weight_max": 22.996742248535156,
      "activations/layer19_attention_weight_min": -22.04319190979004,
      "activations/layer1_attention_weight_max": 17.09843635559082,
      "activations/layer1_attention_weight_min": -15.172086715698242,
      "activations/layer20_attention_weight_max": 21.36357879638672,
      "activations/layer20_attention_weight_min": -23.294414520263672,
      "activations/layer21_attention_weight_max": 38.321083068847656,
      "activations/layer21_attention_weight_min": -22.183521270751953,
      "activations/layer22_attention_weight_max": 27.394636154174805,
      "activations/layer22_attention_weight_min": -25.091617584228516,
      "activations/layer23_attention_weight_max": 37.06002426147461,
      "activations/layer23_attention_weight_min": -24.705059051513672,
      "activations/layer2_attention_weight_max": 31.718351364135742,
      "activations/layer2_attention_weight_min": -30.93529510498047,
      "activations/layer3_attention_weight_max": 91.61365509033203,
      "activations/layer3_attention_weight_min": -88.4520492553711,
      "activations/layer4_attention_weight_max": 89.92471313476562,
      "activations/layer4_attention_weight_min": -86.25055694580078,
      "activations/layer5_attention_weight_max": 65.70370483398438,
      "activations/layer5_attention_weight_min": -71.30972290039062,
      "activations/layer6_attention_weight_max": 49.61781311035156,
      "activations/layer6_attention_weight_min": -50.66312789916992,
      "activations/layer7_attention_weight_max": 62.8715705871582,
      "activations/layer7_attention_weight_min": -58.72526168823242,
      "activations/layer8_attention_weight_max": 44.7535285949707,
      "activations/layer8_attention_weight_min": -50.33454513549805,
      "activations/layer9_attention_weight_max": 47.972991943359375,
      "activations/layer9_attention_weight_min": -45.39414978027344,
      "epoch": 15.37,
      "learning_rate": 5.140833333333333e-05,
      "loss": 2.7453,
      "step": 264500
    },
    {
      "activations/layer0_attention_weight_max": 16.316631317138672,
      "activations/layer0_attention_weight_min": -13.73389720916748,
      "activations/layer10_attention_weight_max": 37.70763397216797,
      "activations/layer10_attention_weight_min": -38.260597229003906,
      "activations/layer11_attention_weight_max": 34.345863342285156,
      "activations/layer11_attention_weight_min": -34.672786712646484,
      "activations/layer12_attention_weight_max": 20.254072189331055,
      "activations/layer12_attention_weight_min": -23.193796157836914,
      "activations/layer13_attention_weight_max": 33.774818420410156,
      "activations/layer13_attention_weight_min": -28.598543167114258,
      "activations/layer14_attention_weight_max": 33.49332809448242,
      "activations/layer14_attention_weight_min": -29.781774520874023,
      "activations/layer15_attention_weight_max": 31.779094696044922,
      "activations/layer15_attention_weight_min": -30.76947593688965,
      "activations/layer16_attention_weight_max": 33.0427131652832,
      "activations/layer16_attention_weight_min": -30.937511444091797,
      "activations/layer17_attention_weight_max": 52.3565788269043,
      "activations/layer17_attention_weight_min": -45.20402908325195,
      "activations/layer18_attention_weight_max": 45.88406753540039,
      "activations/layer18_attention_weight_min": -38.74238967895508,
      "activations/layer19_attention_weight_max": 22.681371688842773,
      "activations/layer19_attention_weight_min": -23.265592575073242,
      "activations/layer1_attention_weight_max": 15.172382354736328,
      "activations/layer1_attention_weight_min": -15.034205436706543,
      "activations/layer20_attention_weight_max": 22.651506423950195,
      "activations/layer20_attention_weight_min": -23.16434097290039,
      "activations/layer21_attention_weight_max": 32.84450149536133,
      "activations/layer21_attention_weight_min": -20.0355281829834,
      "activations/layer22_attention_weight_max": 28.27800178527832,
      "activations/layer22_attention_weight_min": -26.863889694213867,
      "activations/layer23_attention_weight_max": 35.33551025390625,
      "activations/layer23_attention_weight_min": -22.4222412109375,
      "activations/layer2_attention_weight_max": 32.1477165222168,
      "activations/layer2_attention_weight_min": -30.854137420654297,
      "activations/layer3_attention_weight_max": 85.77171325683594,
      "activations/layer3_attention_weight_min": -87.09397888183594,
      "activations/layer4_attention_weight_max": 89.5681381225586,
      "activations/layer4_attention_weight_min": -89.6734848022461,
      "activations/layer5_attention_weight_max": 69.12933349609375,
      "activations/layer5_attention_weight_min": -79.50991821289062,
      "activations/layer6_attention_weight_max": 50.72673416137695,
      "activations/layer6_attention_weight_min": -52.45138168334961,
      "activations/layer7_attention_weight_max": 64.59313201904297,
      "activations/layer7_attention_weight_min": -60.54999542236328,
      "activations/layer8_attention_weight_max": 46.16170120239258,
      "activations/layer8_attention_weight_min": -48.99700927734375,
      "activations/layer9_attention_weight_max": 45.00590133666992,
      "activations/layer9_attention_weight_min": -45.92796325683594,
      "epoch": 15.37,
      "learning_rate": 5.1389393939393935e-05,
      "loss": 2.7562,
      "step": 264550
    },
    {
      "activations/layer0_attention_weight_max": 14.858691215515137,
      "activations/layer0_attention_weight_min": -14.049591064453125,
      "activations/layer10_attention_weight_max": 34.12744140625,
      "activations/layer10_attention_weight_min": -34.21018600463867,
      "activations/layer11_attention_weight_max": 30.15213394165039,
      "activations/layer11_attention_weight_min": -30.902999877929688,
      "activations/layer12_attention_weight_max": 19.299867630004883,
      "activations/layer12_attention_weight_min": -27.60002326965332,
      "activations/layer13_attention_weight_max": 33.842994689941406,
      "activations/layer13_attention_weight_min": -28.618228912353516,
      "activations/layer14_attention_weight_max": 32.853328704833984,
      "activations/layer14_attention_weight_min": -32.1566047668457,
      "activations/layer15_attention_weight_max": 31.407318115234375,
      "activations/layer15_attention_weight_min": -30.163055419921875,
      "activations/layer16_attention_weight_max": 34.48637771606445,
      "activations/layer16_attention_weight_min": -34.2025032043457,
      "activations/layer17_attention_weight_max": 49.3222541809082,
      "activations/layer17_attention_weight_min": -47.16999053955078,
      "activations/layer18_attention_weight_max": 42.92797088623047,
      "activations/layer18_attention_weight_min": -38.50846862792969,
      "activations/layer19_attention_weight_max": 25.26653289794922,
      "activations/layer19_attention_weight_min": -23.231313705444336,
      "activations/layer1_attention_weight_max": 16.247314453125,
      "activations/layer1_attention_weight_min": -14.664569854736328,
      "activations/layer20_attention_weight_max": 22.292787551879883,
      "activations/layer20_attention_weight_min": -23.62735939025879,
      "activations/layer21_attention_weight_max": 33.3820686340332,
      "activations/layer21_attention_weight_min": -21.868270874023438,
      "activations/layer22_attention_weight_max": 29.355541229248047,
      "activations/layer22_attention_weight_min": -26.2235164642334,
      "activations/layer23_attention_weight_max": 34.78315734863281,
      "activations/layer23_attention_weight_min": -24.791372299194336,
      "activations/layer2_attention_weight_max": 32.983970642089844,
      "activations/layer2_attention_weight_min": -30.502704620361328,
      "activations/layer3_attention_weight_max": 88.35826110839844,
      "activations/layer3_attention_weight_min": -91.74945831298828,
      "activations/layer4_attention_weight_max": 84.28755950927734,
      "activations/layer4_attention_weight_min": -84.90817260742188,
      "activations/layer5_attention_weight_max": 67.98632049560547,
      "activations/layer5_attention_weight_min": -73.32365417480469,
      "activations/layer6_attention_weight_max": 47.76260757446289,
      "activations/layer6_attention_weight_min": -49.64519500732422,
      "activations/layer7_attention_weight_max": 64.01700592041016,
      "activations/layer7_attention_weight_min": -58.81812286376953,
      "activations/layer8_attention_weight_max": 42.321651458740234,
      "activations/layer8_attention_weight_min": -46.338436126708984,
      "activations/layer9_attention_weight_max": 43.75089645385742,
      "activations/layer9_attention_weight_min": -45.03662109375,
      "epoch": 15.37,
      "learning_rate": 5.1370454545454544e-05,
      "loss": 2.7564,
      "step": 264600
    },
    {
      "activations/layer0_attention_weight_max": 15.016740798950195,
      "activations/layer0_attention_weight_min": -14.365995407104492,
      "activations/layer10_attention_weight_max": 35.49507522583008,
      "activations/layer10_attention_weight_min": -34.252586364746094,
      "activations/layer11_attention_weight_max": 30.960508346557617,
      "activations/layer11_attention_weight_min": -32.858978271484375,
      "activations/layer12_attention_weight_max": 18.586719512939453,
      "activations/layer12_attention_weight_min": -24.98220443725586,
      "activations/layer13_attention_weight_max": 35.987754821777344,
      "activations/layer13_attention_weight_min": -30.096546173095703,
      "activations/layer14_attention_weight_max": 34.04961395263672,
      "activations/layer14_attention_weight_min": -29.822080612182617,
      "activations/layer15_attention_weight_max": 32.19842529296875,
      "activations/layer15_attention_weight_min": -28.958518981933594,
      "activations/layer16_attention_weight_max": 34.67160415649414,
      "activations/layer16_attention_weight_min": -31.61777687072754,
      "activations/layer17_attention_weight_max": 55.33818817138672,
      "activations/layer17_attention_weight_min": -45.27833557128906,
      "activations/layer18_attention_weight_max": 48.705440521240234,
      "activations/layer18_attention_weight_min": -41.86442947387695,
      "activations/layer19_attention_weight_max": 23.049652099609375,
      "activations/layer19_attention_weight_min": -24.27687644958496,
      "activations/layer1_attention_weight_max": 16.15826416015625,
      "activations/layer1_attention_weight_min": -15.748345375061035,
      "activations/layer20_attention_weight_max": 22.936519622802734,
      "activations/layer20_attention_weight_min": -22.825172424316406,
      "activations/layer21_attention_weight_max": 38.44954299926758,
      "activations/layer21_attention_weight_min": -26.758920669555664,
      "activations/layer22_attention_weight_max": 29.36957359313965,
      "activations/layer22_attention_weight_min": -25.201616287231445,
      "activations/layer23_attention_weight_max": 36.9083137512207,
      "activations/layer23_attention_weight_min": -23.329303741455078,
      "activations/layer2_attention_weight_max": 30.13348388671875,
      "activations/layer2_attention_weight_min": -29.122081756591797,
      "activations/layer3_attention_weight_max": 87.53534698486328,
      "activations/layer3_attention_weight_min": -93.62846374511719,
      "activations/layer4_attention_weight_max": 88.75547790527344,
      "activations/layer4_attention_weight_min": -88.29085540771484,
      "activations/layer5_attention_weight_max": 68.14561462402344,
      "activations/layer5_attention_weight_min": -76.47225952148438,
      "activations/layer6_attention_weight_max": 50.79185104370117,
      "activations/layer6_attention_weight_min": -50.216365814208984,
      "activations/layer7_attention_weight_max": 62.494686126708984,
      "activations/layer7_attention_weight_min": -61.01461410522461,
      "activations/layer8_attention_weight_max": 43.529972076416016,
      "activations/layer8_attention_weight_min": -45.868621826171875,
      "activations/layer9_attention_weight_max": 44.322303771972656,
      "activations/layer9_attention_weight_min": -44.51644515991211,
      "epoch": 15.38,
      "learning_rate": 5.1351515151515145e-05,
      "loss": 2.7565,
      "step": 264650
    },
    {
      "activations/layer0_attention_weight_max": 15.496563911437988,
      "activations/layer0_attention_weight_min": -13.707907676696777,
      "activations/layer10_attention_weight_max": 34.10508728027344,
      "activations/layer10_attention_weight_min": -35.7391242980957,
      "activations/layer11_attention_weight_max": 32.69965744018555,
      "activations/layer11_attention_weight_min": -34.40203857421875,
      "activations/layer12_attention_weight_max": 21.380157470703125,
      "activations/layer12_attention_weight_min": -23.502662658691406,
      "activations/layer13_attention_weight_max": 36.78605270385742,
      "activations/layer13_attention_weight_min": -29.764312744140625,
      "activations/layer14_attention_weight_max": 34.2893180847168,
      "activations/layer14_attention_weight_min": -30.11774444580078,
      "activations/layer15_attention_weight_max": 34.21921157836914,
      "activations/layer15_attention_weight_min": -29.952993392944336,
      "activations/layer16_attention_weight_max": 34.36235046386719,
      "activations/layer16_attention_weight_min": -30.543054580688477,
      "activations/layer17_attention_weight_max": 52.1556282043457,
      "activations/layer17_attention_weight_min": -46.95779037475586,
      "activations/layer18_attention_weight_max": 47.44672775268555,
      "activations/layer18_attention_weight_min": -42.2911262512207,
      "activations/layer19_attention_weight_max": 22.255666732788086,
      "activations/layer19_attention_weight_min": -23.463165283203125,
      "activations/layer1_attention_weight_max": 17.26536750793457,
      "activations/layer1_attention_weight_min": -14.790813446044922,
      "activations/layer20_attention_weight_max": 23.24005126953125,
      "activations/layer20_attention_weight_min": -23.648130416870117,
      "activations/layer21_attention_weight_max": 34.924800872802734,
      "activations/layer21_attention_weight_min": -22.576675415039062,
      "activations/layer22_attention_weight_max": 30.551666259765625,
      "activations/layer22_attention_weight_min": -27.266042709350586,
      "activations/layer23_attention_weight_max": 36.42238235473633,
      "activations/layer23_attention_weight_min": -24.557811737060547,
      "activations/layer2_attention_weight_max": 33.42852783203125,
      "activations/layer2_attention_weight_min": -30.939373016357422,
      "activations/layer3_attention_weight_max": 86.33811950683594,
      "activations/layer3_attention_weight_min": -88.63675689697266,
      "activations/layer4_attention_weight_max": 87.39965057373047,
      "activations/layer4_attention_weight_min": -85.20337677001953,
      "activations/layer5_attention_weight_max": 65.18753051757812,
      "activations/layer5_attention_weight_min": -72.42158508300781,
      "activations/layer6_attention_weight_max": 47.85574722290039,
      "activations/layer6_attention_weight_min": -49.80335998535156,
      "activations/layer7_attention_weight_max": 57.35458755493164,
      "activations/layer7_attention_weight_min": -56.905574798583984,
      "activations/layer8_attention_weight_max": 43.30721664428711,
      "activations/layer8_attention_weight_min": -45.42052459716797,
      "activations/layer9_attention_weight_max": 48.144874572753906,
      "activations/layer9_attention_weight_min": -45.77241134643555,
      "epoch": 15.38,
      "learning_rate": 5.133257575757575e-05,
      "loss": 2.7472,
      "step": 264700
    },
    {
      "activations/layer0_attention_weight_max": 15.467947006225586,
      "activations/layer0_attention_weight_min": -13.631397247314453,
      "activations/layer10_attention_weight_max": 37.901397705078125,
      "activations/layer10_attention_weight_min": -36.193443298339844,
      "activations/layer11_attention_weight_max": 36.03498458862305,
      "activations/layer11_attention_weight_min": -36.547637939453125,
      "activations/layer12_attention_weight_max": 17.200653076171875,
      "activations/layer12_attention_weight_min": -27.722084045410156,
      "activations/layer13_attention_weight_max": 44.59244155883789,
      "activations/layer13_attention_weight_min": -29.042587280273438,
      "activations/layer14_attention_weight_max": 36.483211517333984,
      "activations/layer14_attention_weight_min": -30.624351501464844,
      "activations/layer15_attention_weight_max": 33.91351318359375,
      "activations/layer15_attention_weight_min": -31.554611206054688,
      "activations/layer16_attention_weight_max": 32.8013801574707,
      "activations/layer16_attention_weight_min": -30.354114532470703,
      "activations/layer17_attention_weight_max": 50.68159103393555,
      "activations/layer17_attention_weight_min": -46.7716064453125,
      "activations/layer18_attention_weight_max": 43.3460693359375,
      "activations/layer18_attention_weight_min": -41.66953659057617,
      "activations/layer19_attention_weight_max": 20.40511131286621,
      "activations/layer19_attention_weight_min": -21.287212371826172,
      "activations/layer1_attention_weight_max": 16.582616806030273,
      "activations/layer1_attention_weight_min": -13.896178245544434,
      "activations/layer20_attention_weight_max": 21.639591217041016,
      "activations/layer20_attention_weight_min": -23.364465713500977,
      "activations/layer21_attention_weight_max": 40.20911407470703,
      "activations/layer21_attention_weight_min": -20.343950271606445,
      "activations/layer22_attention_weight_max": 28.73880958557129,
      "activations/layer22_attention_weight_min": -27.05440902709961,
      "activations/layer23_attention_weight_max": 35.36784362792969,
      "activations/layer23_attention_weight_min": -24.182498931884766,
      "activations/layer2_attention_weight_max": 33.87278366088867,
      "activations/layer2_attention_weight_min": -30.817729949951172,
      "activations/layer3_attention_weight_max": 89.58322143554688,
      "activations/layer3_attention_weight_min": -91.73133087158203,
      "activations/layer4_attention_weight_max": 87.2162094116211,
      "activations/layer4_attention_weight_min": -85.41340637207031,
      "activations/layer5_attention_weight_max": 62.24147033691406,
      "activations/layer5_attention_weight_min": -70.76896667480469,
      "activations/layer6_attention_weight_max": 46.871517181396484,
      "activations/layer6_attention_weight_min": -47.00210952758789,
      "activations/layer7_attention_weight_max": 64.58605194091797,
      "activations/layer7_attention_weight_min": -63.817752838134766,
      "activations/layer8_attention_weight_max": 49.7325439453125,
      "activations/layer8_attention_weight_min": -48.00617218017578,
      "activations/layer9_attention_weight_max": 53.75320053100586,
      "activations/layer9_attention_weight_min": -47.68000030517578,
      "epoch": 15.38,
      "learning_rate": 5.131363636363636e-05,
      "loss": 2.7576,
      "step": 264750
    },
    {
      "activations/layer0_attention_weight_max": 14.989169120788574,
      "activations/layer0_attention_weight_min": -14.106179237365723,
      "activations/layer10_attention_weight_max": 34.929779052734375,
      "activations/layer10_attention_weight_min": -35.99141311645508,
      "activations/layer11_attention_weight_max": 31.732860565185547,
      "activations/layer11_attention_weight_min": -34.36481475830078,
      "activations/layer12_attention_weight_max": 18.783750534057617,
      "activations/layer12_attention_weight_min": -23.704620361328125,
      "activations/layer13_attention_weight_max": 34.06306457519531,
      "activations/layer13_attention_weight_min": -29.279590606689453,
      "activations/layer14_attention_weight_max": 34.97810745239258,
      "activations/layer14_attention_weight_min": -29.61665916442871,
      "activations/layer15_attention_weight_max": 29.754854202270508,
      "activations/layer15_attention_weight_min": -30.18597412109375,
      "activations/layer16_attention_weight_max": 33.03302764892578,
      "activations/layer16_attention_weight_min": -31.78180694580078,
      "activations/layer17_attention_weight_max": 50.9720344543457,
      "activations/layer17_attention_weight_min": -45.27974319458008,
      "activations/layer18_attention_weight_max": 45.125736236572266,
      "activations/layer18_attention_weight_min": -39.554969787597656,
      "activations/layer19_attention_weight_max": 25.291662216186523,
      "activations/layer19_attention_weight_min": -24.015338897705078,
      "activations/layer1_attention_weight_max": 15.211365699768066,
      "activations/layer1_attention_weight_min": -15.292828559875488,
      "activations/layer20_attention_weight_max": 22.566869735717773,
      "activations/layer20_attention_weight_min": -22.800058364868164,
      "activations/layer21_attention_weight_max": 33.786048889160156,
      "activations/layer21_attention_weight_min": -27.28713607788086,
      "activations/layer22_attention_weight_max": 28.465595245361328,
      "activations/layer22_attention_weight_min": -26.49447250366211,
      "activations/layer23_attention_weight_max": 35.21482849121094,
      "activations/layer23_attention_weight_min": -24.61452293395996,
      "activations/layer2_attention_weight_max": 32.74151611328125,
      "activations/layer2_attention_weight_min": -30.755870819091797,
      "activations/layer3_attention_weight_max": 94.22325134277344,
      "activations/layer3_attention_weight_min": -91.58393096923828,
      "activations/layer4_attention_weight_max": 92.43729400634766,
      "activations/layer4_attention_weight_min": -91.48621368408203,
      "activations/layer5_attention_weight_max": 68.96053314208984,
      "activations/layer5_attention_weight_min": -74.34705352783203,
      "activations/layer6_attention_weight_max": 53.96968078613281,
      "activations/layer6_attention_weight_min": -52.75816345214844,
      "activations/layer7_attention_weight_max": 60.467430114746094,
      "activations/layer7_attention_weight_min": -60.7752571105957,
      "activations/layer8_attention_weight_max": 47.8115234375,
      "activations/layer8_attention_weight_min": -49.453269958496094,
      "activations/layer9_attention_weight_max": 47.93598937988281,
      "activations/layer9_attention_weight_min": -47.15097427368164,
      "epoch": 15.39,
      "learning_rate": 5.129469696969696e-05,
      "loss": 2.7448,
      "step": 264800
    },
    {
      "activations/layer0_attention_weight_max": 15.2622709274292,
      "activations/layer0_attention_weight_min": -13.776361465454102,
      "activations/layer10_attention_weight_max": 34.227813720703125,
      "activations/layer10_attention_weight_min": -33.21119689941406,
      "activations/layer11_attention_weight_max": 32.792049407958984,
      "activations/layer11_attention_weight_min": -32.534767150878906,
      "activations/layer12_attention_weight_max": 17.69487953186035,
      "activations/layer12_attention_weight_min": -29.884035110473633,
      "activations/layer13_attention_weight_max": 43.352054595947266,
      "activations/layer13_attention_weight_min": -29.8814754486084,
      "activations/layer14_attention_weight_max": 34.0462646484375,
      "activations/layer14_attention_weight_min": -30.99897575378418,
      "activations/layer15_attention_weight_max": 33.007938385009766,
      "activations/layer15_attention_weight_min": -29.19385528564453,
      "activations/layer16_attention_weight_max": 34.11552810668945,
      "activations/layer16_attention_weight_min": -30.564428329467773,
      "activations/layer17_attention_weight_max": 51.02137756347656,
      "activations/layer17_attention_weight_min": -44.600406646728516,
      "activations/layer18_attention_weight_max": 46.67323303222656,
      "activations/layer18_attention_weight_min": -41.23273849487305,
      "activations/layer19_attention_weight_max": 24.752479553222656,
      "activations/layer19_attention_weight_min": -22.10133934020996,
      "activations/layer1_attention_weight_max": 16.87896728515625,
      "activations/layer1_attention_weight_min": -14.79393482208252,
      "activations/layer20_attention_weight_max": 23.630638122558594,
      "activations/layer20_attention_weight_min": -21.970598220825195,
      "activations/layer21_attention_weight_max": 38.22423553466797,
      "activations/layer21_attention_weight_min": -23.838659286499023,
      "activations/layer22_attention_weight_max": 31.356815338134766,
      "activations/layer22_attention_weight_min": -27.495630264282227,
      "activations/layer23_attention_weight_max": 40.392608642578125,
      "activations/layer23_attention_weight_min": -25.2974910736084,
      "activations/layer2_attention_weight_max": 32.16218566894531,
      "activations/layer2_attention_weight_min": -30.461116790771484,
      "activations/layer3_attention_weight_max": 93.35661315917969,
      "activations/layer3_attention_weight_min": -92.080322265625,
      "activations/layer4_attention_weight_max": 86.99320220947266,
      "activations/layer4_attention_weight_min": -85.080322265625,
      "activations/layer5_attention_weight_max": 63.42259216308594,
      "activations/layer5_attention_weight_min": -66.52346801757812,
      "activations/layer6_attention_weight_max": 47.656822204589844,
      "activations/layer6_attention_weight_min": -47.2375373840332,
      "activations/layer7_attention_weight_max": 59.35031509399414,
      "activations/layer7_attention_weight_min": -56.581878662109375,
      "activations/layer8_attention_weight_max": 45.43806076049805,
      "activations/layer8_attention_weight_min": -46.89033889770508,
      "activations/layer9_attention_weight_max": 49.87457275390625,
      "activations/layer9_attention_weight_min": -44.35020446777344,
      "epoch": 15.39,
      "learning_rate": 5.127575757575757e-05,
      "loss": 2.7405,
      "step": 264850
    },
    {
      "activations/layer0_attention_weight_max": 16.21341323852539,
      "activations/layer0_attention_weight_min": -13.728342056274414,
      "activations/layer10_attention_weight_max": 36.79828643798828,
      "activations/layer10_attention_weight_min": -35.32945251464844,
      "activations/layer11_attention_weight_max": 37.101131439208984,
      "activations/layer11_attention_weight_min": -34.192508697509766,
      "activations/layer12_attention_weight_max": 17.636028289794922,
      "activations/layer12_attention_weight_min": -29.511472702026367,
      "activations/layer13_attention_weight_max": 36.270118713378906,
      "activations/layer13_attention_weight_min": -32.577266693115234,
      "activations/layer14_attention_weight_max": 33.530033111572266,
      "activations/layer14_attention_weight_min": -29.06569480895996,
      "activations/layer15_attention_weight_max": 30.302196502685547,
      "activations/layer15_attention_weight_min": -30.01003646850586,
      "activations/layer16_attention_weight_max": 32.544830322265625,
      "activations/layer16_attention_weight_min": -30.642423629760742,
      "activations/layer17_attention_weight_max": 48.402889251708984,
      "activations/layer17_attention_weight_min": -44.20439529418945,
      "activations/layer18_attention_weight_max": 46.666839599609375,
      "activations/layer18_attention_weight_min": -38.61567306518555,
      "activations/layer19_attention_weight_max": 23.301733016967773,
      "activations/layer19_attention_weight_min": -24.777366638183594,
      "activations/layer1_attention_weight_max": 15.624235153198242,
      "activations/layer1_attention_weight_min": -15.248079299926758,
      "activations/layer20_attention_weight_max": 24.67193603515625,
      "activations/layer20_attention_weight_min": -23.132795333862305,
      "activations/layer21_attention_weight_max": 36.061065673828125,
      "activations/layer21_attention_weight_min": -22.424781799316406,
      "activations/layer22_attention_weight_max": 29.326337814331055,
      "activations/layer22_attention_weight_min": -26.214679718017578,
      "activations/layer23_attention_weight_max": 33.07616424560547,
      "activations/layer23_attention_weight_min": -25.63155174255371,
      "activations/layer2_attention_weight_max": 31.536169052124023,
      "activations/layer2_attention_weight_min": -29.778011322021484,
      "activations/layer3_attention_weight_max": 94.88416290283203,
      "activations/layer3_attention_weight_min": -98.39791107177734,
      "activations/layer4_attention_weight_max": 88.5747299194336,
      "activations/layer4_attention_weight_min": -89.56331634521484,
      "activations/layer5_attention_weight_max": 64.19677734375,
      "activations/layer5_attention_weight_min": -71.53948211669922,
      "activations/layer6_attention_weight_max": 47.11080551147461,
      "activations/layer6_attention_weight_min": -47.35960388183594,
      "activations/layer7_attention_weight_max": 58.54299545288086,
      "activations/layer7_attention_weight_min": -57.144126892089844,
      "activations/layer8_attention_weight_max": 45.76137161254883,
      "activations/layer8_attention_weight_min": -46.330936431884766,
      "activations/layer9_attention_weight_max": 50.24418258666992,
      "activations/layer9_attention_weight_min": -48.37192916870117,
      "epoch": 15.39,
      "learning_rate": 5.125719696969696e-05,
      "loss": 2.7456,
      "step": 264900
    },
    {
      "activations/layer0_attention_weight_max": 15.408638954162598,
      "activations/layer0_attention_weight_min": -13.97381591796875,
      "activations/layer10_attention_weight_max": 34.53840637207031,
      "activations/layer10_attention_weight_min": -33.9007682800293,
      "activations/layer11_attention_weight_max": 32.47248840332031,
      "activations/layer11_attention_weight_min": -33.5545654296875,
      "activations/layer12_attention_weight_max": 17.48014259338379,
      "activations/layer12_attention_weight_min": -27.091602325439453,
      "activations/layer13_attention_weight_max": 35.861698150634766,
      "activations/layer13_attention_weight_min": -27.12325096130371,
      "activations/layer14_attention_weight_max": 36.46647644042969,
      "activations/layer14_attention_weight_min": -29.49346923828125,
      "activations/layer15_attention_weight_max": 32.76373291015625,
      "activations/layer15_attention_weight_min": -28.91029930114746,
      "activations/layer16_attention_weight_max": 32.40620803833008,
      "activations/layer16_attention_weight_min": -32.70789337158203,
      "activations/layer17_attention_weight_max": 50.09857177734375,
      "activations/layer17_attention_weight_min": -44.190128326416016,
      "activations/layer18_attention_weight_max": 49.08180618286133,
      "activations/layer18_attention_weight_min": -38.80341720581055,
      "activations/layer19_attention_weight_max": 24.901174545288086,
      "activations/layer19_attention_weight_min": -22.021299362182617,
      "activations/layer1_attention_weight_max": 16.20685386657715,
      "activations/layer1_attention_weight_min": -14.977768898010254,
      "activations/layer20_attention_weight_max": 25.777666091918945,
      "activations/layer20_attention_weight_min": -23.970869064331055,
      "activations/layer21_attention_weight_max": 42.817081451416016,
      "activations/layer21_attention_weight_min": -27.930845260620117,
      "activations/layer22_attention_weight_max": 30.462646484375,
      "activations/layer22_attention_weight_min": -25.1232852935791,
      "activations/layer23_attention_weight_max": 42.57515335083008,
      "activations/layer23_attention_weight_min": -26.378101348876953,
      "activations/layer2_attention_weight_max": 32.951087951660156,
      "activations/layer2_attention_weight_min": -30.96295738220215,
      "activations/layer3_attention_weight_max": 91.99879455566406,
      "activations/layer3_attention_weight_min": -93.98457336425781,
      "activations/layer4_attention_weight_max": 88.96431732177734,
      "activations/layer4_attention_weight_min": -86.50825500488281,
      "activations/layer5_attention_weight_max": 61.671241760253906,
      "activations/layer5_attention_weight_min": -72.59214782714844,
      "activations/layer6_attention_weight_max": 45.83883285522461,
      "activations/layer6_attention_weight_min": -46.906166076660156,
      "activations/layer7_attention_weight_max": 58.4105110168457,
      "activations/layer7_attention_weight_min": -55.75664138793945,
      "activations/layer8_attention_weight_max": 43.27534484863281,
      "activations/layer8_attention_weight_min": -45.18560028076172,
      "activations/layer9_attention_weight_max": 41.02801513671875,
      "activations/layer9_attention_weight_min": -43.98542022705078,
      "epoch": 15.4,
      "learning_rate": 5.123825757575758e-05,
      "loss": 2.7548,
      "step": 264950
    },
    {
      "activations/layer0_attention_weight_max": 14.623048782348633,
      "activations/layer0_attention_weight_min": -13.857791900634766,
      "activations/layer10_attention_weight_max": 36.943050384521484,
      "activations/layer10_attention_weight_min": -38.438472747802734,
      "activations/layer11_attention_weight_max": 32.08979797363281,
      "activations/layer11_attention_weight_min": -32.9609260559082,
      "activations/layer12_attention_weight_max": 18.881746292114258,
      "activations/layer12_attention_weight_min": -25.286121368408203,
      "activations/layer13_attention_weight_max": 32.11129379272461,
      "activations/layer13_attention_weight_min": -28.10811996459961,
      "activations/layer14_attention_weight_max": 33.573936462402344,
      "activations/layer14_attention_weight_min": -32.25543212890625,
      "activations/layer15_attention_weight_max": 31.301734924316406,
      "activations/layer15_attention_weight_min": -30.50417709350586,
      "activations/layer16_attention_weight_max": 31.225080490112305,
      "activations/layer16_attention_weight_min": -31.803258895874023,
      "activations/layer17_attention_weight_max": 51.1744499206543,
      "activations/layer17_attention_weight_min": -45.82664489746094,
      "activations/layer18_attention_weight_max": 45.85723114013672,
      "activations/layer18_attention_weight_min": -42.023040771484375,
      "activations/layer19_attention_weight_max": 23.916303634643555,
      "activations/layer19_attention_weight_min": -22.699371337890625,
      "activations/layer1_attention_weight_max": 16.197608947753906,
      "activations/layer1_attention_weight_min": -14.483397483825684,
      "activations/layer20_attention_weight_max": 22.908838272094727,
      "activations/layer20_attention_weight_min": -22.743457794189453,
      "activations/layer21_attention_weight_max": 33.573883056640625,
      "activations/layer21_attention_weight_min": -22.65476417541504,
      "activations/layer22_attention_weight_max": 33.02695083618164,
      "activations/layer22_attention_weight_min": -26.515554428100586,
      "activations/layer23_attention_weight_max": 36.012943267822266,
      "activations/layer23_attention_weight_min": -26.337745666503906,
      "activations/layer2_attention_weight_max": 31.845905303955078,
      "activations/layer2_attention_weight_min": -30.81366539001465,
      "activations/layer3_attention_weight_max": 86.9052734375,
      "activations/layer3_attention_weight_min": -92.60285949707031,
      "activations/layer4_attention_weight_max": 86.24238586425781,
      "activations/layer4_attention_weight_min": -88.35362243652344,
      "activations/layer5_attention_weight_max": 66.64216613769531,
      "activations/layer5_attention_weight_min": -73.82290649414062,
      "activations/layer6_attention_weight_max": 52.698097229003906,
      "activations/layer6_attention_weight_min": -51.90992736816406,
      "activations/layer7_attention_weight_max": 61.799468994140625,
      "activations/layer7_attention_weight_min": -60.7535285949707,
      "activations/layer8_attention_weight_max": 45.43203353881836,
      "activations/layer8_attention_weight_min": -49.674312591552734,
      "activations/layer9_attention_weight_max": 44.77811050415039,
      "activations/layer9_attention_weight_min": -46.7011833190918,
      "epoch": 15.4,
      "learning_rate": 5.121931818181818e-05,
      "loss": 2.7671,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4957,
      "eval_samples_per_second": 505.434,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4957,
      "eval_openwebtext_samples_per_second": 505.434,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9679,
      "eval_wikitext_samples_per_second": 231.722,
      "step": 265000
    },
    {
      "epoch": 15.4,
      "eval_lambada_loss": 2.517578125,
      "eval_lambada_ppl": 12.398532576458457,
      "eval_lambada_runtime": 9.5834,
      "eval_lambada_samples_per_second": 508.065,
      "step": 265000
    },
    {
      "activations/layer0_attention_weight_max": 16.209577560424805,
      "activations/layer0_attention_weight_min": -14.241667747497559,
      "activations/layer10_attention_weight_max": 33.59846115112305,
      "activations/layer10_attention_weight_min": -36.79290771484375,
      "activations/layer11_attention_weight_max": 29.761417388916016,
      "activations/layer11_attention_weight_min": -33.07123947143555,
      "activations/layer12_attention_weight_max": 18.355350494384766,
      "activations/layer12_attention_weight_min": -30.99859046936035,
      "activations/layer13_attention_weight_max": 34.7642707824707,
      "activations/layer13_attention_weight_min": -27.596033096313477,
      "activations/layer14_attention_weight_max": 32.2719841003418,
      "activations/layer14_attention_weight_min": -32.42528533935547,
      "activations/layer15_attention_weight_max": 29.996051788330078,
      "activations/layer15_attention_weight_min": -29.98506736755371,
      "activations/layer16_attention_weight_max": 31.73874855041504,
      "activations/layer16_attention_weight_min": -30.379776000976562,
      "activations/layer17_attention_weight_max": 46.59431838989258,
      "activations/layer17_attention_weight_min": -41.46144104003906,
      "activations/layer18_attention_weight_max": 42.67558288574219,
      "activations/layer18_attention_weight_min": -38.12091827392578,
      "activations/layer19_attention_weight_max": 20.206661224365234,
      "activations/layer19_attention_weight_min": -22.188257217407227,
      "activations/layer1_attention_weight_max": 15.731283187866211,
      "activations/layer1_attention_weight_min": -15.012733459472656,
      "activations/layer20_attention_weight_max": 20.180912017822266,
      "activations/layer20_attention_weight_min": -22.025779724121094,
      "activations/layer21_attention_weight_max": 33.458892822265625,
      "activations/layer21_attention_weight_min": -23.950674057006836,
      "activations/layer22_attention_weight_max": 27.123594284057617,
      "activations/layer22_attention_weight_min": -27.09817886352539,
      "activations/layer23_attention_weight_max": 33.338558197021484,
      "activations/layer23_attention_weight_min": -27.392601013183594,
      "activations/layer2_attention_weight_max": 30.710712432861328,
      "activations/layer2_attention_weight_min": -31.042722702026367,
      "activations/layer3_attention_weight_max": 89.02144622802734,
      "activations/layer3_attention_weight_min": -92.47434997558594,
      "activations/layer4_attention_weight_max": 83.52558898925781,
      "activations/layer4_attention_weight_min": -81.25362396240234,
      "activations/layer5_attention_weight_max": 62.3935661315918,
      "activations/layer5_attention_weight_min": -71.47750091552734,
      "activations/layer6_attention_weight_max": 48.190250396728516,
      "activations/layer6_attention_weight_min": -49.90119552612305,
      "activations/layer7_attention_weight_max": 57.68589782714844,
      "activations/layer7_attention_weight_min": -57.57484817504883,
      "activations/layer8_attention_weight_max": 43.2913932800293,
      "activations/layer8_attention_weight_min": -45.682289123535156,
      "activations/layer9_attention_weight_max": 43.65298843383789,
      "activations/layer9_attention_weight_min": -45.542911529541016,
      "epoch": 15.4,
      "learning_rate": 5.120037878787878e-05,
      "loss": 2.7562,
      "step": 265050
    },
    {
      "activations/layer0_attention_weight_max": 15.941839218139648,
      "activations/layer0_attention_weight_min": -13.557205200195312,
      "activations/layer10_attention_weight_max": 41.20060729980469,
      "activations/layer10_attention_weight_min": -38.68492126464844,
      "activations/layer11_attention_weight_max": 37.067832946777344,
      "activations/layer11_attention_weight_min": -35.998565673828125,
      "activations/layer12_attention_weight_max": 18.505836486816406,
      "activations/layer12_attention_weight_min": -23.8225040435791,
      "activations/layer13_attention_weight_max": 46.804141998291016,
      "activations/layer13_attention_weight_min": -31.299203872680664,
      "activations/layer14_attention_weight_max": 39.27511978149414,
      "activations/layer14_attention_weight_min": -31.585330963134766,
      "activations/layer15_attention_weight_max": 39.157493591308594,
      "activations/layer15_attention_weight_min": -32.089839935302734,
      "activations/layer16_attention_weight_max": 36.16607666015625,
      "activations/layer16_attention_weight_min": -32.68198776245117,
      "activations/layer17_attention_weight_max": 55.11438751220703,
      "activations/layer17_attention_weight_min": -47.20783233642578,
      "activations/layer18_attention_weight_max": 49.23598861694336,
      "activations/layer18_attention_weight_min": -41.20290756225586,
      "activations/layer19_attention_weight_max": 24.001338958740234,
      "activations/layer19_attention_weight_min": -24.123979568481445,
      "activations/layer1_attention_weight_max": 16.422344207763672,
      "activations/layer1_attention_weight_min": -14.96615982055664,
      "activations/layer20_attention_weight_max": 24.575590133666992,
      "activations/layer20_attention_weight_min": -22.746427536010742,
      "activations/layer21_attention_weight_max": 40.82120895385742,
      "activations/layer21_attention_weight_min": -23.471946716308594,
      "activations/layer22_attention_weight_max": 29.8846435546875,
      "activations/layer22_attention_weight_min": -27.98008155822754,
      "activations/layer23_attention_weight_max": 38.303184509277344,
      "activations/layer23_attention_weight_min": -25.88238525390625,
      "activations/layer2_attention_weight_max": 32.5154914855957,
      "activations/layer2_attention_weight_min": -29.390052795410156,
      "activations/layer3_attention_weight_max": 92.40497589111328,
      "activations/layer3_attention_weight_min": -90.26360321044922,
      "activations/layer4_attention_weight_max": 91.5665283203125,
      "activations/layer4_attention_weight_min": -89.5736312866211,
      "activations/layer5_attention_weight_max": 67.10429382324219,
      "activations/layer5_attention_weight_min": -74.26233673095703,
      "activations/layer6_attention_weight_max": 50.66410827636719,
      "activations/layer6_attention_weight_min": -53.85523986816406,
      "activations/layer7_attention_weight_max": 59.657569885253906,
      "activations/layer7_attention_weight_min": -64.23908233642578,
      "activations/layer8_attention_weight_max": 51.40089797973633,
      "activations/layer8_attention_weight_min": -51.25687789916992,
      "activations/layer9_attention_weight_max": 51.632057189941406,
      "activations/layer9_attention_weight_min": -47.2574577331543,
      "epoch": 15.4,
      "learning_rate": 5.118143939393939e-05,
      "loss": 2.7723,
      "step": 265100
    },
    {
      "activations/layer0_attention_weight_max": 15.82833194732666,
      "activations/layer0_attention_weight_min": -13.407821655273438,
      "activations/layer10_attention_weight_max": 36.93693923950195,
      "activations/layer10_attention_weight_min": -35.8321533203125,
      "activations/layer11_attention_weight_max": 35.2456169128418,
      "activations/layer11_attention_weight_min": -35.14060592651367,
      "activations/layer12_attention_weight_max": 19.154125213623047,
      "activations/layer12_attention_weight_min": -25.501724243164062,
      "activations/layer13_attention_weight_max": 42.756534576416016,
      "activations/layer13_attention_weight_min": -27.577457427978516,
      "activations/layer14_attention_weight_max": 36.19709777832031,
      "activations/layer14_attention_weight_min": -31.449796676635742,
      "activations/layer15_attention_weight_max": 35.27693176269531,
      "activations/layer15_attention_weight_min": -30.08463478088379,
      "activations/layer16_attention_weight_max": 33.19331359863281,
      "activations/layer16_attention_weight_min": -31.741573333740234,
      "activations/layer17_attention_weight_max": 53.66904830932617,
      "activations/layer17_attention_weight_min": -45.279998779296875,
      "activations/layer18_attention_weight_max": 48.318233489990234,
      "activations/layer18_attention_weight_min": -39.13144302368164,
      "activations/layer19_attention_weight_max": 22.125295639038086,
      "activations/layer19_attention_weight_min": -22.328998565673828,
      "activations/layer1_attention_weight_max": 16.36326026916504,
      "activations/layer1_attention_weight_min": -14.428298950195312,
      "activations/layer20_attention_weight_max": 24.091777801513672,
      "activations/layer20_attention_weight_min": -22.057109832763672,
      "activations/layer21_attention_weight_max": 34.268531799316406,
      "activations/layer21_attention_weight_min": -23.134130477905273,
      "activations/layer22_attention_weight_max": 27.346317291259766,
      "activations/layer22_attention_weight_min": -24.657611846923828,
      "activations/layer23_attention_weight_max": 36.420196533203125,
      "activations/layer23_attention_weight_min": -24.552040100097656,
      "activations/layer2_attention_weight_max": 33.40937423706055,
      "activations/layer2_attention_weight_min": -33.097259521484375,
      "activations/layer3_attention_weight_max": 95.39363098144531,
      "activations/layer3_attention_weight_min": -93.99427795410156,
      "activations/layer4_attention_weight_max": 85.66195678710938,
      "activations/layer4_attention_weight_min": -83.93768310546875,
      "activations/layer5_attention_weight_max": 62.481842041015625,
      "activations/layer5_attention_weight_min": -79.10028076171875,
      "activations/layer6_attention_weight_max": 49.886905670166016,
      "activations/layer6_attention_weight_min": -50.68962860107422,
      "activations/layer7_attention_weight_max": 60.32145309448242,
      "activations/layer7_attention_weight_min": -60.855873107910156,
      "activations/layer8_attention_weight_max": 46.20088577270508,
      "activations/layer8_attention_weight_min": -50.710784912109375,
      "activations/layer9_attention_weight_max": 45.71847152709961,
      "activations/layer9_attention_weight_min": -46.39078903198242,
      "epoch": 15.41,
      "learning_rate": 5.11625e-05,
      "loss": 2.7634,
      "step": 265150
    },
    {
      "activations/layer0_attention_weight_max": 15.550958633422852,
      "activations/layer0_attention_weight_min": -13.690088272094727,
      "activations/layer10_attention_weight_max": 38.06538391113281,
      "activations/layer10_attention_weight_min": -41.34202575683594,
      "activations/layer11_attention_weight_max": 34.03044509887695,
      "activations/layer11_attention_weight_min": -34.98780059814453,
      "activations/layer12_attention_weight_max": 19.007667541503906,
      "activations/layer12_attention_weight_min": -23.28960418701172,
      "activations/layer13_attention_weight_max": 36.29137420654297,
      "activations/layer13_attention_weight_min": -29.13907814025879,
      "activations/layer14_attention_weight_max": 35.27112579345703,
      "activations/layer14_attention_weight_min": -31.14017105102539,
      "activations/layer15_attention_weight_max": 32.08482360839844,
      "activations/layer15_attention_weight_min": -31.343524932861328,
      "activations/layer16_attention_weight_max": 33.054405212402344,
      "activations/layer16_attention_weight_min": -31.234773635864258,
      "activations/layer17_attention_weight_max": 52.10147476196289,
      "activations/layer17_attention_weight_min": -44.62637710571289,
      "activations/layer18_attention_weight_max": 44.905609130859375,
      "activations/layer18_attention_weight_min": -39.14161682128906,
      "activations/layer19_attention_weight_max": 23.942045211791992,
      "activations/layer19_attention_weight_min": -24.211238861083984,
      "activations/layer1_attention_weight_max": 16.25802230834961,
      "activations/layer1_attention_weight_min": -14.236004829406738,
      "activations/layer20_attention_weight_max": 22.719356536865234,
      "activations/layer20_attention_weight_min": -23.26864242553711,
      "activations/layer21_attention_weight_max": 33.668033599853516,
      "activations/layer21_attention_weight_min": -24.8925724029541,
      "activations/layer22_attention_weight_max": 30.99373435974121,
      "activations/layer22_attention_weight_min": -27.812700271606445,
      "activations/layer23_attention_weight_max": 36.27306365966797,
      "activations/layer23_attention_weight_min": -24.71053695678711,
      "activations/layer2_attention_weight_max": 33.719078063964844,
      "activations/layer2_attention_weight_min": -34.3546142578125,
      "activations/layer3_attention_weight_max": 93.63555145263672,
      "activations/layer3_attention_weight_min": -99.3200912475586,
      "activations/layer4_attention_weight_max": 88.08175659179688,
      "activations/layer4_attention_weight_min": -89.75574493408203,
      "activations/layer5_attention_weight_max": 68.93807983398438,
      "activations/layer5_attention_weight_min": -76.75901794433594,
      "activations/layer6_attention_weight_max": 54.618141174316406,
      "activations/layer6_attention_weight_min": -53.179019927978516,
      "activations/layer7_attention_weight_max": 61.57220458984375,
      "activations/layer7_attention_weight_min": -62.643104553222656,
      "activations/layer8_attention_weight_max": 49.93351364135742,
      "activations/layer8_attention_weight_min": -50.74555969238281,
      "activations/layer9_attention_weight_max": 49.301490783691406,
      "activations/layer9_attention_weight_min": -49.22706604003906,
      "epoch": 15.41,
      "learning_rate": 5.1143560606060605e-05,
      "loss": 2.7565,
      "step": 265200
    },
    {
      "activations/layer0_attention_weight_max": 15.944524765014648,
      "activations/layer0_attention_weight_min": -14.024602890014648,
      "activations/layer10_attention_weight_max": 36.00189971923828,
      "activations/layer10_attention_weight_min": -36.90412139892578,
      "activations/layer11_attention_weight_max": 32.652252197265625,
      "activations/layer11_attention_weight_min": -33.18329620361328,
      "activations/layer12_attention_weight_max": 17.698633193969727,
      "activations/layer12_attention_weight_min": -25.64834976196289,
      "activations/layer13_attention_weight_max": 35.736515045166016,
      "activations/layer13_attention_weight_min": -26.644128799438477,
      "activations/layer14_attention_weight_max": 35.88597869873047,
      "activations/layer14_attention_weight_min": -30.42768096923828,
      "activations/layer15_attention_weight_max": 31.85427474975586,
      "activations/layer15_attention_weight_min": -28.328710556030273,
      "activations/layer16_attention_weight_max": 32.30758285522461,
      "activations/layer16_attention_weight_min": -30.63074493408203,
      "activations/layer17_attention_weight_max": 51.66714096069336,
      "activations/layer17_attention_weight_min": -44.611881256103516,
      "activations/layer18_attention_weight_max": 46.630104064941406,
      "activations/layer18_attention_weight_min": -37.32402420043945,
      "activations/layer19_attention_weight_max": 24.177167892456055,
      "activations/layer19_attention_weight_min": -20.81867218017578,
      "activations/layer1_attention_weight_max": 17.268657684326172,
      "activations/layer1_attention_weight_min": -13.973522186279297,
      "activations/layer20_attention_weight_max": 23.033281326293945,
      "activations/layer20_attention_weight_min": -21.233182907104492,
      "activations/layer21_attention_weight_max": 33.17872619628906,
      "activations/layer21_attention_weight_min": -23.062108993530273,
      "activations/layer22_attention_weight_max": 31.891620635986328,
      "activations/layer22_attention_weight_min": -24.947431564331055,
      "activations/layer23_attention_weight_max": 35.946014404296875,
      "activations/layer23_attention_weight_min": -23.237293243408203,
      "activations/layer2_attention_weight_max": 32.13086700439453,
      "activations/layer2_attention_weight_min": -31.846149444580078,
      "activations/layer3_attention_weight_max": 89.82648468017578,
      "activations/layer3_attention_weight_min": -99.20947265625,
      "activations/layer4_attention_weight_max": 92.76433563232422,
      "activations/layer4_attention_weight_min": -92.39412689208984,
      "activations/layer5_attention_weight_max": 67.45970153808594,
      "activations/layer5_attention_weight_min": -73.12648010253906,
      "activations/layer6_attention_weight_max": 51.87186050415039,
      "activations/layer6_attention_weight_min": -54.65470504760742,
      "activations/layer7_attention_weight_max": 61.24948501586914,
      "activations/layer7_attention_weight_min": -61.31266784667969,
      "activations/layer8_attention_weight_max": 45.807891845703125,
      "activations/layer8_attention_weight_min": -52.14915466308594,
      "activations/layer9_attention_weight_max": 42.277774810791016,
      "activations/layer9_attention_weight_min": -49.19377899169922,
      "epoch": 15.41,
      "learning_rate": 5.1124621212121207e-05,
      "loss": 2.7629,
      "step": 265250
    },
    {
      "activations/layer0_attention_weight_max": 15.744038581848145,
      "activations/layer0_attention_weight_min": -13.513646125793457,
      "activations/layer10_attention_weight_max": 40.28351593017578,
      "activations/layer10_attention_weight_min": -37.98035430908203,
      "activations/layer11_attention_weight_max": 34.376686096191406,
      "activations/layer11_attention_weight_min": -34.67007064819336,
      "activations/layer12_attention_weight_max": 18.146390914916992,
      "activations/layer12_attention_weight_min": -23.815595626831055,
      "activations/layer13_attention_weight_max": 33.3066520690918,
      "activations/layer13_attention_weight_min": -28.356414794921875,
      "activations/layer14_attention_weight_max": 35.77779006958008,
      "activations/layer14_attention_weight_min": -31.612674713134766,
      "activations/layer15_attention_weight_max": 31.381441116333008,
      "activations/layer15_attention_weight_min": -31.176305770874023,
      "activations/layer16_attention_weight_max": 31.89910125732422,
      "activations/layer16_attention_weight_min": -31.198389053344727,
      "activations/layer17_attention_weight_max": 50.06718826293945,
      "activations/layer17_attention_weight_min": -45.03474426269531,
      "activations/layer18_attention_weight_max": 47.87824630737305,
      "activations/layer18_attention_weight_min": -39.37494659423828,
      "activations/layer19_attention_weight_max": 22.873708724975586,
      "activations/layer19_attention_weight_min": -23.69527244567871,
      "activations/layer1_attention_weight_max": 16.84632682800293,
      "activations/layer1_attention_weight_min": -13.518818855285645,
      "activations/layer20_attention_weight_max": 21.47646713256836,
      "activations/layer20_attention_weight_min": -21.615741729736328,
      "activations/layer21_attention_weight_max": 44.7765998840332,
      "activations/layer21_attention_weight_min": -25.126733779907227,
      "activations/layer22_attention_weight_max": 27.398542404174805,
      "activations/layer22_attention_weight_min": -25.45595359802246,
      "activations/layer23_attention_weight_max": 39.86168670654297,
      "activations/layer23_attention_weight_min": -23.572803497314453,
      "activations/layer2_attention_weight_max": 32.24681091308594,
      "activations/layer2_attention_weight_min": -30.106367111206055,
      "activations/layer3_attention_weight_max": 92.18553924560547,
      "activations/layer3_attention_weight_min": -93.33965301513672,
      "activations/layer4_attention_weight_max": 90.77178192138672,
      "activations/layer4_attention_weight_min": -86.57342529296875,
      "activations/layer5_attention_weight_max": 65.55303955078125,
      "activations/layer5_attention_weight_min": -70.33895874023438,
      "activations/layer6_attention_weight_max": 53.214271545410156,
      "activations/layer6_attention_weight_min": -50.88530731201172,
      "activations/layer7_attention_weight_max": 60.174476623535156,
      "activations/layer7_attention_weight_min": -61.90196990966797,
      "activations/layer8_attention_weight_max": 46.71392822265625,
      "activations/layer8_attention_weight_min": -50.00071334838867,
      "activations/layer9_attention_weight_max": 47.68018341064453,
      "activations/layer9_attention_weight_min": -45.48640823364258,
      "epoch": 15.42,
      "learning_rate": 5.1105681818181815e-05,
      "loss": 2.7606,
      "step": 265300
    },
    {
      "activations/layer0_attention_weight_max": 15.072874069213867,
      "activations/layer0_attention_weight_min": -13.52725601196289,
      "activations/layer10_attention_weight_max": 40.487342834472656,
      "activations/layer10_attention_weight_min": -38.331581115722656,
      "activations/layer11_attention_weight_max": 40.09775924682617,
      "activations/layer11_attention_weight_min": -38.49896240234375,
      "activations/layer12_attention_weight_max": 19.050708770751953,
      "activations/layer12_attention_weight_min": -26.369306564331055,
      "activations/layer13_attention_weight_max": 35.355350494384766,
      "activations/layer13_attention_weight_min": -29.093856811523438,
      "activations/layer14_attention_weight_max": 35.52828598022461,
      "activations/layer14_attention_weight_min": -31.077402114868164,
      "activations/layer15_attention_weight_max": 32.464412689208984,
      "activations/layer15_attention_weight_min": -30.54608726501465,
      "activations/layer16_attention_weight_max": 32.84797286987305,
      "activations/layer16_attention_weight_min": -31.4884033203125,
      "activations/layer17_attention_weight_max": 54.070831298828125,
      "activations/layer17_attention_weight_min": -45.696876525878906,
      "activations/layer18_attention_weight_max": 45.020790100097656,
      "activations/layer18_attention_weight_min": -41.052616119384766,
      "activations/layer19_attention_weight_max": 23.940534591674805,
      "activations/layer19_attention_weight_min": -24.510746002197266,
      "activations/layer1_attention_weight_max": 17.454172134399414,
      "activations/layer1_attention_weight_min": -14.582111358642578,
      "activations/layer20_attention_weight_max": 22.308311462402344,
      "activations/layer20_attention_weight_min": -20.529645919799805,
      "activations/layer21_attention_weight_max": 35.97382354736328,
      "activations/layer21_attention_weight_min": -21.42313575744629,
      "activations/layer22_attention_weight_max": 29.606782913208008,
      "activations/layer22_attention_weight_min": -26.33616065979004,
      "activations/layer23_attention_weight_max": 38.325435638427734,
      "activations/layer23_attention_weight_min": -25.076200485229492,
      "activations/layer2_attention_weight_max": 33.20759201049805,
      "activations/layer2_attention_weight_min": -31.569730758666992,
      "activations/layer3_attention_weight_max": 91.10344696044922,
      "activations/layer3_attention_weight_min": -96.66607666015625,
      "activations/layer4_attention_weight_max": 91.0562973022461,
      "activations/layer4_attention_weight_min": -90.84764099121094,
      "activations/layer5_attention_weight_max": 66.62257385253906,
      "activations/layer5_attention_weight_min": -73.61553192138672,
      "activations/layer6_attention_weight_max": 53.17422103881836,
      "activations/layer6_attention_weight_min": -53.39900588989258,
      "activations/layer7_attention_weight_max": 61.97145462036133,
      "activations/layer7_attention_weight_min": -58.88380432128906,
      "activations/layer8_attention_weight_max": 51.583282470703125,
      "activations/layer8_attention_weight_min": -51.38478469848633,
      "activations/layer9_attention_weight_max": 52.52839279174805,
      "activations/layer9_attention_weight_min": -46.42435073852539,
      "epoch": 15.42,
      "learning_rate": 5.108674242424242e-05,
      "loss": 2.7548,
      "step": 265350
    },
    {
      "activations/layer0_attention_weight_max": 15.70394229888916,
      "activations/layer0_attention_weight_min": -13.658429145812988,
      "activations/layer10_attention_weight_max": 34.71802520751953,
      "activations/layer10_attention_weight_min": -33.44817352294922,
      "activations/layer11_attention_weight_max": 33.87168884277344,
      "activations/layer11_attention_weight_min": -33.339202880859375,
      "activations/layer12_attention_weight_max": 19.426990509033203,
      "activations/layer12_attention_weight_min": -27.529909133911133,
      "activations/layer13_attention_weight_max": 35.803741455078125,
      "activations/layer13_attention_weight_min": -27.532705307006836,
      "activations/layer14_attention_weight_max": 38.65719223022461,
      "activations/layer14_attention_weight_min": -30.31254768371582,
      "activations/layer15_attention_weight_max": 35.7956657409668,
      "activations/layer15_attention_weight_min": -30.405071258544922,
      "activations/layer16_attention_weight_max": 34.030216217041016,
      "activations/layer16_attention_weight_min": -31.038021087646484,
      "activations/layer17_attention_weight_max": 52.77531433105469,
      "activations/layer17_attention_weight_min": -46.007408142089844,
      "activations/layer18_attention_weight_max": 49.5938835144043,
      "activations/layer18_attention_weight_min": -37.15862274169922,
      "activations/layer19_attention_weight_max": 23.152326583862305,
      "activations/layer19_attention_weight_min": -21.379131317138672,
      "activations/layer1_attention_weight_max": 15.76057243347168,
      "activations/layer1_attention_weight_min": -13.70021915435791,
      "activations/layer20_attention_weight_max": 23.955429077148438,
      "activations/layer20_attention_weight_min": -21.854389190673828,
      "activations/layer21_attention_weight_max": 34.775428771972656,
      "activations/layer21_attention_weight_min": -22.360334396362305,
      "activations/layer22_attention_weight_max": 32.622947692871094,
      "activations/layer22_attention_weight_min": -26.816635131835938,
      "activations/layer23_attention_weight_max": 43.433982849121094,
      "activations/layer23_attention_weight_min": -23.7591552734375,
      "activations/layer2_attention_weight_max": 30.369138717651367,
      "activations/layer2_attention_weight_min": -29.067405700683594,
      "activations/layer3_attention_weight_max": 90.26666259765625,
      "activations/layer3_attention_weight_min": -95.05397033691406,
      "activations/layer4_attention_weight_max": 85.16535186767578,
      "activations/layer4_attention_weight_min": -89.05612182617188,
      "activations/layer5_attention_weight_max": 63.764976501464844,
      "activations/layer5_attention_weight_min": -75.88910675048828,
      "activations/layer6_attention_weight_max": 48.27248001098633,
      "activations/layer6_attention_weight_min": -48.89929962158203,
      "activations/layer7_attention_weight_max": 64.65861511230469,
      "activations/layer7_attention_weight_min": -60.78835678100586,
      "activations/layer8_attention_weight_max": 43.56142044067383,
      "activations/layer8_attention_weight_min": -45.729286193847656,
      "activations/layer9_attention_weight_max": 44.04268264770508,
      "activations/layer9_attention_weight_min": -46.479373931884766,
      "epoch": 15.42,
      "learning_rate": 5.1068181818181814e-05,
      "loss": 2.7655,
      "step": 265400
    },
    {
      "activations/layer0_attention_weight_max": 16.037080764770508,
      "activations/layer0_attention_weight_min": -13.639484405517578,
      "activations/layer10_attention_weight_max": 33.57051086425781,
      "activations/layer10_attention_weight_min": -33.2442512512207,
      "activations/layer11_attention_weight_max": 34.90876388549805,
      "activations/layer11_attention_weight_min": -35.44580078125,
      "activations/layer12_attention_weight_max": 18.37765884399414,
      "activations/layer12_attention_weight_min": -25.047834396362305,
      "activations/layer13_attention_weight_max": 32.982574462890625,
      "activations/layer13_attention_weight_min": -28.0324764251709,
      "activations/layer14_attention_weight_max": 32.66120910644531,
      "activations/layer14_attention_weight_min": -30.540618896484375,
      "activations/layer15_attention_weight_max": 29.66163444519043,
      "activations/layer15_attention_weight_min": -28.00144386291504,
      "activations/layer16_attention_weight_max": 32.05162048339844,
      "activations/layer16_attention_weight_min": -29.913204193115234,
      "activations/layer17_attention_weight_max": 49.114776611328125,
      "activations/layer17_attention_weight_min": -43.920806884765625,
      "activations/layer18_attention_weight_max": 46.611114501953125,
      "activations/layer18_attention_weight_min": -37.5007438659668,
      "activations/layer19_attention_weight_max": 22.1434383392334,
      "activations/layer19_attention_weight_min": -23.363889694213867,
      "activations/layer1_attention_weight_max": 17.291812896728516,
      "activations/layer1_attention_weight_min": -15.59781551361084,
      "activations/layer20_attention_weight_max": 20.50961685180664,
      "activations/layer20_attention_weight_min": -23.200756072998047,
      "activations/layer21_attention_weight_max": 35.607601165771484,
      "activations/layer21_attention_weight_min": -21.855361938476562,
      "activations/layer22_attention_weight_max": 29.172515869140625,
      "activations/layer22_attention_weight_min": -28.721662521362305,
      "activations/layer23_attention_weight_max": 35.25814437866211,
      "activations/layer23_attention_weight_min": -25.209646224975586,
      "activations/layer2_attention_weight_max": 32.26332092285156,
      "activations/layer2_attention_weight_min": -30.047286987304688,
      "activations/layer3_attention_weight_max": 86.77800750732422,
      "activations/layer3_attention_weight_min": -92.79157257080078,
      "activations/layer4_attention_weight_max": 87.30445098876953,
      "activations/layer4_attention_weight_min": -83.55859375,
      "activations/layer5_attention_weight_max": 64.03070068359375,
      "activations/layer5_attention_weight_min": -72.83926391601562,
      "activations/layer6_attention_weight_max": 49.36847686767578,
      "activations/layer6_attention_weight_min": -50.91575241088867,
      "activations/layer7_attention_weight_max": 60.2664794921875,
      "activations/layer7_attention_weight_min": -57.968017578125,
      "activations/layer8_attention_weight_max": 42.757572174072266,
      "activations/layer8_attention_weight_min": -44.638084411621094,
      "activations/layer9_attention_weight_max": 42.60249710083008,
      "activations/layer9_attention_weight_min": -44.322227478027344,
      "epoch": 15.42,
      "learning_rate": 5.1049242424242415e-05,
      "loss": 2.7485,
      "step": 265450
    },
    {
      "activations/layer0_attention_weight_max": 15.536376953125,
      "activations/layer0_attention_weight_min": -13.464329719543457,
      "activations/layer10_attention_weight_max": 33.85688018798828,
      "activations/layer10_attention_weight_min": -32.99267578125,
      "activations/layer11_attention_weight_max": 31.81937026977539,
      "activations/layer11_attention_weight_min": -32.56624984741211,
      "activations/layer12_attention_weight_max": 19.394285202026367,
      "activations/layer12_attention_weight_min": -23.792699813842773,
      "activations/layer13_attention_weight_max": 36.05109786987305,
      "activations/layer13_attention_weight_min": -27.85101318359375,
      "activations/layer14_attention_weight_max": 36.699466705322266,
      "activations/layer14_attention_weight_min": -33.27881622314453,
      "activations/layer15_attention_weight_max": 32.602901458740234,
      "activations/layer15_attention_weight_min": -32.163177490234375,
      "activations/layer16_attention_weight_max": 33.7442741394043,
      "activations/layer16_attention_weight_min": -32.51205825805664,
      "activations/layer17_attention_weight_max": 49.330291748046875,
      "activations/layer17_attention_weight_min": -46.32820129394531,
      "activations/layer18_attention_weight_max": 47.701839447021484,
      "activations/layer18_attention_weight_min": -40.321205139160156,
      "activations/layer19_attention_weight_max": 23.548826217651367,
      "activations/layer19_attention_weight_min": -23.144699096679688,
      "activations/layer1_attention_weight_max": 15.885873794555664,
      "activations/layer1_attention_weight_min": -14.511960983276367,
      "activations/layer20_attention_weight_max": 23.945341110229492,
      "activations/layer20_attention_weight_min": -21.653535842895508,
      "activations/layer21_attention_weight_max": 35.41309356689453,
      "activations/layer21_attention_weight_min": -22.84623146057129,
      "activations/layer22_attention_weight_max": 28.171403884887695,
      "activations/layer22_attention_weight_min": -26.913068771362305,
      "activations/layer23_attention_weight_max": 35.552772521972656,
      "activations/layer23_attention_weight_min": -25.478851318359375,
      "activations/layer2_attention_weight_max": 31.379138946533203,
      "activations/layer2_attention_weight_min": -29.09212875366211,
      "activations/layer3_attention_weight_max": 85.75021362304688,
      "activations/layer3_attention_weight_min": -86.06476593017578,
      "activations/layer4_attention_weight_max": 84.41461181640625,
      "activations/layer4_attention_weight_min": -84.20329284667969,
      "activations/layer5_attention_weight_max": 65.32156372070312,
      "activations/layer5_attention_weight_min": -74.5461196899414,
      "activations/layer6_attention_weight_max": 48.08625411987305,
      "activations/layer6_attention_weight_min": -48.61811828613281,
      "activations/layer7_attention_weight_max": 59.967437744140625,
      "activations/layer7_attention_weight_min": -56.908653259277344,
      "activations/layer8_attention_weight_max": 42.925621032714844,
      "activations/layer8_attention_weight_min": -45.05524826049805,
      "activations/layer9_attention_weight_max": 43.49892807006836,
      "activations/layer9_attention_weight_min": -44.182655334472656,
      "epoch": 15.43,
      "learning_rate": 5.103030303030303e-05,
      "loss": 2.7557,
      "step": 265500
    },
    {
      "activations/layer0_attention_weight_max": 14.798383712768555,
      "activations/layer0_attention_weight_min": -13.779576301574707,
      "activations/layer10_attention_weight_max": 38.617549896240234,
      "activations/layer10_attention_weight_min": -38.557552337646484,
      "activations/layer11_attention_weight_max": 35.51386642456055,
      "activations/layer11_attention_weight_min": -36.87501907348633,
      "activations/layer12_attention_weight_max": 20.004913330078125,
      "activations/layer12_attention_weight_min": -26.88378143310547,
      "activations/layer13_attention_weight_max": 36.104217529296875,
      "activations/layer13_attention_weight_min": -28.11681365966797,
      "activations/layer14_attention_weight_max": 36.0682373046875,
      "activations/layer14_attention_weight_min": -29.945226669311523,
      "activations/layer15_attention_weight_max": 35.08210372924805,
      "activations/layer15_attention_weight_min": -31.062589645385742,
      "activations/layer16_attention_weight_max": 36.14693069458008,
      "activations/layer16_attention_weight_min": -35.162132263183594,
      "activations/layer17_attention_weight_max": 55.81340789794922,
      "activations/layer17_attention_weight_min": -47.29585647583008,
      "activations/layer18_attention_weight_max": 49.36384201049805,
      "activations/layer18_attention_weight_min": -39.92444610595703,
      "activations/layer19_attention_weight_max": 22.79768180847168,
      "activations/layer19_attention_weight_min": -21.779157638549805,
      "activations/layer1_attention_weight_max": 15.305870056152344,
      "activations/layer1_attention_weight_min": -14.731254577636719,
      "activations/layer20_attention_weight_max": 21.203847885131836,
      "activations/layer20_attention_weight_min": -21.256010055541992,
      "activations/layer21_attention_weight_max": 35.87839126586914,
      "activations/layer21_attention_weight_min": -21.75029182434082,
      "activations/layer22_attention_weight_max": 29.643321990966797,
      "activations/layer22_attention_weight_min": -24.49955177307129,
      "activations/layer23_attention_weight_max": 39.34331512451172,
      "activations/layer23_attention_weight_min": -25.398971557617188,
      "activations/layer2_attention_weight_max": 31.988666534423828,
      "activations/layer2_attention_weight_min": -31.32184600830078,
      "activations/layer3_attention_weight_max": 92.7327880859375,
      "activations/layer3_attention_weight_min": -91.59786224365234,
      "activations/layer4_attention_weight_max": 94.61739349365234,
      "activations/layer4_attention_weight_min": -91.50603485107422,
      "activations/layer5_attention_weight_max": 67.81309509277344,
      "activations/layer5_attention_weight_min": -71.87034606933594,
      "activations/layer6_attention_weight_max": 52.65467071533203,
      "activations/layer6_attention_weight_min": -54.60551071166992,
      "activations/layer7_attention_weight_max": 62.57353591918945,
      "activations/layer7_attention_weight_min": -62.062255859375,
      "activations/layer8_attention_weight_max": 51.37287139892578,
      "activations/layer8_attention_weight_min": -55.45205307006836,
      "activations/layer9_attention_weight_max": 46.79357147216797,
      "activations/layer9_attention_weight_min": -51.53731918334961,
      "epoch": 15.43,
      "learning_rate": 5.101136363636363e-05,
      "loss": 2.7441,
      "step": 265550
    },
    {
      "activations/layer0_attention_weight_max": 14.68253231048584,
      "activations/layer0_attention_weight_min": -13.814193725585938,
      "activations/layer10_attention_weight_max": 38.32079315185547,
      "activations/layer10_attention_weight_min": -35.85613250732422,
      "activations/layer11_attention_weight_max": 34.60383987426758,
      "activations/layer11_attention_weight_min": -34.334197998046875,
      "activations/layer12_attention_weight_max": 19.724842071533203,
      "activations/layer12_attention_weight_min": -29.512744903564453,
      "activations/layer13_attention_weight_max": 37.0836067199707,
      "activations/layer13_attention_weight_min": -30.38421630859375,
      "activations/layer14_attention_weight_max": 36.064613342285156,
      "activations/layer14_attention_weight_min": -30.054885864257812,
      "activations/layer15_attention_weight_max": 35.73297882080078,
      "activations/layer15_attention_weight_min": -31.673751831054688,
      "activations/layer16_attention_weight_max": 35.719749450683594,
      "activations/layer16_attention_weight_min": -32.78742980957031,
      "activations/layer17_attention_weight_max": 57.13652801513672,
      "activations/layer17_attention_weight_min": -47.450950622558594,
      "activations/layer18_attention_weight_max": 50.477901458740234,
      "activations/layer18_attention_weight_min": -42.92670822143555,
      "activations/layer19_attention_weight_max": 27.307708740234375,
      "activations/layer19_attention_weight_min": -22.728992462158203,
      "activations/layer1_attention_weight_max": 17.796104431152344,
      "activations/layer1_attention_weight_min": -15.12830924987793,
      "activations/layer20_attention_weight_max": 25.05159568786621,
      "activations/layer20_attention_weight_min": -20.59772491455078,
      "activations/layer21_attention_weight_max": 41.347816467285156,
      "activations/layer21_attention_weight_min": -24.565532684326172,
      "activations/layer22_attention_weight_max": 28.760833740234375,
      "activations/layer22_attention_weight_min": -25.76190948486328,
      "activations/layer23_attention_weight_max": 39.09471130371094,
      "activations/layer23_attention_weight_min": -24.311758041381836,
      "activations/layer2_attention_weight_max": 29.0891170501709,
      "activations/layer2_attention_weight_min": -27.605802536010742,
      "activations/layer3_attention_weight_max": 85.27969360351562,
      "activations/layer3_attention_weight_min": -83.88176727294922,
      "activations/layer4_attention_weight_max": 82.72164916992188,
      "activations/layer4_attention_weight_min": -80.3480453491211,
      "activations/layer5_attention_weight_max": 65.91668701171875,
      "activations/layer5_attention_weight_min": -70.9706039428711,
      "activations/layer6_attention_weight_max": 48.07796096801758,
      "activations/layer6_attention_weight_min": -49.16408157348633,
      "activations/layer7_attention_weight_max": 58.924190521240234,
      "activations/layer7_attention_weight_min": -56.35255432128906,
      "activations/layer8_attention_weight_max": 46.794429779052734,
      "activations/layer8_attention_weight_min": -49.44118881225586,
      "activations/layer9_attention_weight_max": 44.4978141784668,
      "activations/layer9_attention_weight_min": -48.171443939208984,
      "epoch": 15.43,
      "learning_rate": 5.099242424242424e-05,
      "loss": 2.7661,
      "step": 265600
    },
    {
      "activations/layer0_attention_weight_max": 15.264044761657715,
      "activations/layer0_attention_weight_min": -13.953958511352539,
      "activations/layer10_attention_weight_max": 36.395912170410156,
      "activations/layer10_attention_weight_min": -36.32785415649414,
      "activations/layer11_attention_weight_max": 33.63488006591797,
      "activations/layer11_attention_weight_min": -34.182899475097656,
      "activations/layer12_attention_weight_max": 19.880605697631836,
      "activations/layer12_attention_weight_min": -25.090965270996094,
      "activations/layer13_attention_weight_max": 35.51668167114258,
      "activations/layer13_attention_weight_min": -31.584182739257812,
      "activations/layer14_attention_weight_max": 36.63134002685547,
      "activations/layer14_attention_weight_min": -29.916866302490234,
      "activations/layer15_attention_weight_max": 36.54553985595703,
      "activations/layer15_attention_weight_min": -31.435970306396484,
      "activations/layer16_attention_weight_max": 34.60755157470703,
      "activations/layer16_attention_weight_min": -32.66598892211914,
      "activations/layer17_attention_weight_max": 54.58363723754883,
      "activations/layer17_attention_weight_min": -50.41080093383789,
      "activations/layer18_attention_weight_max": 49.6911735534668,
      "activations/layer18_attention_weight_min": -41.537269592285156,
      "activations/layer19_attention_weight_max": 24.90575408935547,
      "activations/layer19_attention_weight_min": -25.796588897705078,
      "activations/layer1_attention_weight_max": 15.512423515319824,
      "activations/layer1_attention_weight_min": -15.088099479675293,
      "activations/layer20_attention_weight_max": 24.070232391357422,
      "activations/layer20_attention_weight_min": -27.109285354614258,
      "activations/layer21_attention_weight_max": 41.24699783325195,
      "activations/layer21_attention_weight_min": -21.876220703125,
      "activations/layer22_attention_weight_max": 29.930906295776367,
      "activations/layer22_attention_weight_min": -25.543624877929688,
      "activations/layer23_attention_weight_max": 43.02234649658203,
      "activations/layer23_attention_weight_min": -25.38934898376465,
      "activations/layer2_attention_weight_max": 32.17306900024414,
      "activations/layer2_attention_weight_min": -32.278480529785156,
      "activations/layer3_attention_weight_max": 90.84027862548828,
      "activations/layer3_attention_weight_min": -91.54691314697266,
      "activations/layer4_attention_weight_max": 90.74019622802734,
      "activations/layer4_attention_weight_min": -87.64273834228516,
      "activations/layer5_attention_weight_max": 63.92316436767578,
      "activations/layer5_attention_weight_min": -76.51288604736328,
      "activations/layer6_attention_weight_max": 51.72715377807617,
      "activations/layer6_attention_weight_min": -50.9715690612793,
      "activations/layer7_attention_weight_max": 60.24833679199219,
      "activations/layer7_attention_weight_min": -57.161006927490234,
      "activations/layer8_attention_weight_max": 47.70540237426758,
      "activations/layer8_attention_weight_min": -48.25127029418945,
      "activations/layer9_attention_weight_max": 44.408573150634766,
      "activations/layer9_attention_weight_min": -46.202266693115234,
      "epoch": 15.44,
      "learning_rate": 5.097348484848484e-05,
      "loss": 2.7546,
      "step": 265650
    },
    {
      "activations/layer0_attention_weight_max": 15.025737762451172,
      "activations/layer0_attention_weight_min": -13.462514877319336,
      "activations/layer10_attention_weight_max": 34.2701301574707,
      "activations/layer10_attention_weight_min": -32.82558822631836,
      "activations/layer11_attention_weight_max": 30.90169906616211,
      "activations/layer11_attention_weight_min": -31.572673797607422,
      "activations/layer12_attention_weight_max": 19.10744857788086,
      "activations/layer12_attention_weight_min": -28.454212188720703,
      "activations/layer13_attention_weight_max": 34.572017669677734,
      "activations/layer13_attention_weight_min": -26.88195037841797,
      "activations/layer14_attention_weight_max": 36.21164321899414,
      "activations/layer14_attention_weight_min": -32.022186279296875,
      "activations/layer15_attention_weight_max": 32.90104293823242,
      "activations/layer15_attention_weight_min": -30.033266067504883,
      "activations/layer16_attention_weight_max": 34.50300598144531,
      "activations/layer16_attention_weight_min": -32.60322952270508,
      "activations/layer17_attention_weight_max": 52.82030487060547,
      "activations/layer17_attention_weight_min": -45.950592041015625,
      "activations/layer18_attention_weight_max": 47.8055534362793,
      "activations/layer18_attention_weight_min": -40.615028381347656,
      "activations/layer19_attention_weight_max": 27.766759872436523,
      "activations/layer19_attention_weight_min": -23.577381134033203,
      "activations/layer1_attention_weight_max": 18.052453994750977,
      "activations/layer1_attention_weight_min": -13.704673767089844,
      "activations/layer20_attention_weight_max": 24.8799991607666,
      "activations/layer20_attention_weight_min": -21.535701751708984,
      "activations/layer21_attention_weight_max": 37.407318115234375,
      "activations/layer21_attention_weight_min": -22.882923126220703,
      "activations/layer22_attention_weight_max": 35.693180084228516,
      "activations/layer22_attention_weight_min": -27.09067153930664,
      "activations/layer23_attention_weight_max": 37.728599548339844,
      "activations/layer23_attention_weight_min": -25.036128997802734,
      "activations/layer2_attention_weight_max": 29.678586959838867,
      "activations/layer2_attention_weight_min": -28.43999481201172,
      "activations/layer3_attention_weight_max": 84.21430206298828,
      "activations/layer3_attention_weight_min": -92.75778198242188,
      "activations/layer4_attention_weight_max": 85.40872192382812,
      "activations/layer4_attention_weight_min": -86.82575225830078,
      "activations/layer5_attention_weight_max": 65.08172607421875,
      "activations/layer5_attention_weight_min": -66.18840026855469,
      "activations/layer6_attention_weight_max": 45.96046829223633,
      "activations/layer6_attention_weight_min": -48.6640739440918,
      "activations/layer7_attention_weight_max": 54.876564025878906,
      "activations/layer7_attention_weight_min": -56.17720031738281,
      "activations/layer8_attention_weight_max": 41.567298889160156,
      "activations/layer8_attention_weight_min": -46.547691345214844,
      "activations/layer9_attention_weight_max": 42.622039794921875,
      "activations/layer9_attention_weight_min": -43.48440170288086,
      "epoch": 15.44,
      "learning_rate": 5.095454545454546e-05,
      "loss": 2.7466,
      "step": 265700
    },
    {
      "activations/layer0_attention_weight_max": 15.091591835021973,
      "activations/layer0_attention_weight_min": -13.829084396362305,
      "activations/layer10_attention_weight_max": 39.06969451904297,
      "activations/layer10_attention_weight_min": -38.390838623046875,
      "activations/layer11_attention_weight_max": 41.631378173828125,
      "activations/layer11_attention_weight_min": -36.501224517822266,
      "activations/layer12_attention_weight_max": 17.474197387695312,
      "activations/layer12_attention_weight_min": -26.08806800842285,
      "activations/layer13_attention_weight_max": 33.76528549194336,
      "activations/layer13_attention_weight_min": -30.467336654663086,
      "activations/layer14_attention_weight_max": 32.21776580810547,
      "activations/layer14_attention_weight_min": -29.772066116333008,
      "activations/layer15_attention_weight_max": 29.004846572875977,
      "activations/layer15_attention_weight_min": -29.2769832611084,
      "activations/layer16_attention_weight_max": 29.61503791809082,
      "activations/layer16_attention_weight_min": -28.55324363708496,
      "activations/layer17_attention_weight_max": 47.2987060546875,
      "activations/layer17_attention_weight_min": -41.87580490112305,
      "activations/layer18_attention_weight_max": 42.889732360839844,
      "activations/layer18_attention_weight_min": -37.33995056152344,
      "activations/layer19_attention_weight_max": 21.279186248779297,
      "activations/layer19_attention_weight_min": -23.10125160217285,
      "activations/layer1_attention_weight_max": 17.668806076049805,
      "activations/layer1_attention_weight_min": -15.287895202636719,
      "activations/layer20_attention_weight_max": 20.056962966918945,
      "activations/layer20_attention_weight_min": -21.719932556152344,
      "activations/layer21_attention_weight_max": 30.857147216796875,
      "activations/layer21_attention_weight_min": -25.274621963500977,
      "activations/layer22_attention_weight_max": 28.07209587097168,
      "activations/layer22_attention_weight_min": -25.550798416137695,
      "activations/layer23_attention_weight_max": 36.72785186767578,
      "activations/layer23_attention_weight_min": -23.41701316833496,
      "activations/layer2_attention_weight_max": 33.40814971923828,
      "activations/layer2_attention_weight_min": -31.644025802612305,
      "activations/layer3_attention_weight_max": 93.58914947509766,
      "activations/layer3_attention_weight_min": -95.59003448486328,
      "activations/layer4_attention_weight_max": 96.00566864013672,
      "activations/layer4_attention_weight_min": -85.15290832519531,
      "activations/layer5_attention_weight_max": 71.58281707763672,
      "activations/layer5_attention_weight_min": -71.50545501708984,
      "activations/layer6_attention_weight_max": 50.23591995239258,
      "activations/layer6_attention_weight_min": -50.12724685668945,
      "activations/layer7_attention_weight_max": 72.36219787597656,
      "activations/layer7_attention_weight_min": -61.206512451171875,
      "activations/layer8_attention_weight_max": 51.572479248046875,
      "activations/layer8_attention_weight_min": -51.0799446105957,
      "activations/layer9_attention_weight_max": 60.80326461791992,
      "activations/layer9_attention_weight_min": -51.580177307128906,
      "epoch": 15.44,
      "learning_rate": 5.093560606060606e-05,
      "loss": 2.7481,
      "step": 265750
    },
    {
      "activations/layer0_attention_weight_max": 16.07521629333496,
      "activations/layer0_attention_weight_min": -13.844979286193848,
      "activations/layer10_attention_weight_max": 40.57490539550781,
      "activations/layer10_attention_weight_min": -36.808837890625,
      "activations/layer11_attention_weight_max": 35.33867263793945,
      "activations/layer11_attention_weight_min": -34.04535675048828,
      "activations/layer12_attention_weight_max": 19.14779281616211,
      "activations/layer12_attention_weight_min": -24.825048446655273,
      "activations/layer13_attention_weight_max": 34.40306091308594,
      "activations/layer13_attention_weight_min": -27.414064407348633,
      "activations/layer14_attention_weight_max": 36.1944580078125,
      "activations/layer14_attention_weight_min": -31.377391815185547,
      "activations/layer15_attention_weight_max": 33.76382064819336,
      "activations/layer15_attention_weight_min": -32.004390716552734,
      "activations/layer16_attention_weight_max": 32.83216857910156,
      "activations/layer16_attention_weight_min": -31.624372482299805,
      "activations/layer17_attention_weight_max": 53.008365631103516,
      "activations/layer17_attention_weight_min": -45.12556838989258,
      "activations/layer18_attention_weight_max": 49.33102798461914,
      "activations/layer18_attention_weight_min": -40.08031463623047,
      "activations/layer19_attention_weight_max": 23.052112579345703,
      "activations/layer19_attention_weight_min": -23.743074417114258,
      "activations/layer1_attention_weight_max": 16.500429153442383,
      "activations/layer1_attention_weight_min": -14.166973114013672,
      "activations/layer20_attention_weight_max": 24.75444221496582,
      "activations/layer20_attention_weight_min": -22.137277603149414,
      "activations/layer21_attention_weight_max": 38.68025588989258,
      "activations/layer21_attention_weight_min": -26.277252197265625,
      "activations/layer22_attention_weight_max": 32.20857238769531,
      "activations/layer22_attention_weight_min": -25.867904663085938,
      "activations/layer23_attention_weight_max": 37.99838638305664,
      "activations/layer23_attention_weight_min": -25.010862350463867,
      "activations/layer2_attention_weight_max": 30.162490844726562,
      "activations/layer2_attention_weight_min": -30.759395599365234,
      "activations/layer3_attention_weight_max": 94.01673126220703,
      "activations/layer3_attention_weight_min": -96.22775268554688,
      "activations/layer4_attention_weight_max": 91.09197235107422,
      "activations/layer4_attention_weight_min": -89.5129165649414,
      "activations/layer5_attention_weight_max": 65.79205322265625,
      "activations/layer5_attention_weight_min": -73.04496765136719,
      "activations/layer6_attention_weight_max": 52.004737854003906,
      "activations/layer6_attention_weight_min": -53.24851608276367,
      "activations/layer7_attention_weight_max": 59.89118576049805,
      "activations/layer7_attention_weight_min": -59.540924072265625,
      "activations/layer8_attention_weight_max": 47.59062194824219,
      "activations/layer8_attention_weight_min": -52.69505310058594,
      "activations/layer9_attention_weight_max": 46.57334518432617,
      "activations/layer9_attention_weight_min": -49.487003326416016,
      "epoch": 15.44,
      "learning_rate": 5.091666666666666e-05,
      "loss": 2.7651,
      "step": 265800
    },
    {
      "activations/layer0_attention_weight_max": 15.401022911071777,
      "activations/layer0_attention_weight_min": -13.43662166595459,
      "activations/layer10_attention_weight_max": 38.84257507324219,
      "activations/layer10_attention_weight_min": -38.65839767456055,
      "activations/layer11_attention_weight_max": 39.08304977416992,
      "activations/layer11_attention_weight_min": -36.654685974121094,
      "activations/layer12_attention_weight_max": 21.61732292175293,
      "activations/layer12_attention_weight_min": -22.838056564331055,
      "activations/layer13_attention_weight_max": 44.558109283447266,
      "activations/layer13_attention_weight_min": -37.00966262817383,
      "activations/layer14_attention_weight_max": 44.73288345336914,
      "activations/layer14_attention_weight_min": -32.24984359741211,
      "activations/layer15_attention_weight_max": 35.175148010253906,
      "activations/layer15_attention_weight_min": -31.572616577148438,
      "activations/layer16_attention_weight_max": 33.51155090332031,
      "activations/layer16_attention_weight_min": -33.416622161865234,
      "activations/layer17_attention_weight_max": 50.73088455200195,
      "activations/layer17_attention_weight_min": -48.33216857910156,
      "activations/layer18_attention_weight_max": 47.3551139831543,
      "activations/layer18_attention_weight_min": -39.124629974365234,
      "activations/layer19_attention_weight_max": 23.81494140625,
      "activations/layer19_attention_weight_min": -22.508569717407227,
      "activations/layer1_attention_weight_max": 16.481782913208008,
      "activations/layer1_attention_weight_min": -15.065312385559082,
      "activations/layer20_attention_weight_max": 22.715835571289062,
      "activations/layer20_attention_weight_min": -22.18834114074707,
      "activations/layer21_attention_weight_max": 33.94786071777344,
      "activations/layer21_attention_weight_min": -21.604162216186523,
      "activations/layer22_attention_weight_max": 30.25602149963379,
      "activations/layer22_attention_weight_min": -25.13212776184082,
      "activations/layer23_attention_weight_max": 37.629756927490234,
      "activations/layer23_attention_weight_min": -23.957744598388672,
      "activations/layer2_attention_weight_max": 32.05674743652344,
      "activations/layer2_attention_weight_min": -31.19611167907715,
      "activations/layer3_attention_weight_max": 94.2735824584961,
      "activations/layer3_attention_weight_min": -92.77859497070312,
      "activations/layer4_attention_weight_max": 97.16124725341797,
      "activations/layer4_attention_weight_min": -94.82081604003906,
      "activations/layer5_attention_weight_max": 71.59809875488281,
      "activations/layer5_attention_weight_min": -72.7190933227539,
      "activations/layer6_attention_weight_max": 51.296714782714844,
      "activations/layer6_attention_weight_min": -54.30137252807617,
      "activations/layer7_attention_weight_max": 65.29744720458984,
      "activations/layer7_attention_weight_min": -65.64881896972656,
      "activations/layer8_attention_weight_max": 49.597843170166016,
      "activations/layer8_attention_weight_min": -53.445716857910156,
      "activations/layer9_attention_weight_max": 49.1056022644043,
      "activations/layer9_attention_weight_min": -47.64942169189453,
      "epoch": 15.45,
      "learning_rate": 5.089772727272727e-05,
      "loss": 2.7554,
      "step": 265850
    },
    {
      "activations/layer0_attention_weight_max": 16.014812469482422,
      "activations/layer0_attention_weight_min": -13.767598152160645,
      "activations/layer10_attention_weight_max": 33.967803955078125,
      "activations/layer10_attention_weight_min": -34.36634063720703,
      "activations/layer11_attention_weight_max": 29.78347396850586,
      "activations/layer11_attention_weight_min": -32.50383758544922,
      "activations/layer12_attention_weight_max": 18.90280532836914,
      "activations/layer12_attention_weight_min": -26.01258087158203,
      "activations/layer13_attention_weight_max": 33.86652374267578,
      "activations/layer13_attention_weight_min": -28.379253387451172,
      "activations/layer14_attention_weight_max": 33.840267181396484,
      "activations/layer14_attention_weight_min": -29.54494857788086,
      "activations/layer15_attention_weight_max": 30.555566787719727,
      "activations/layer15_attention_weight_min": -30.00497055053711,
      "activations/layer16_attention_weight_max": 32.80748748779297,
      "activations/layer16_attention_weight_min": -31.216503143310547,
      "activations/layer17_attention_weight_max": 51.01101303100586,
      "activations/layer17_attention_weight_min": -43.7188835144043,
      "activations/layer18_attention_weight_max": 43.55975341796875,
      "activations/layer18_attention_weight_min": -37.4666862487793,
      "activations/layer19_attention_weight_max": 23.109888076782227,
      "activations/layer19_attention_weight_min": -22.52858543395996,
      "activations/layer1_attention_weight_max": 15.621200561523438,
      "activations/layer1_attention_weight_min": -12.67707347869873,
      "activations/layer20_attention_weight_max": 22.1147518157959,
      "activations/layer20_attention_weight_min": -23.6782283782959,
      "activations/layer21_attention_weight_max": 36.26546859741211,
      "activations/layer21_attention_weight_min": -23.118322372436523,
      "activations/layer22_attention_weight_max": 29.931396484375,
      "activations/layer22_attention_weight_min": -27.256534576416016,
      "activations/layer23_attention_weight_max": 36.52558898925781,
      "activations/layer23_attention_weight_min": -24.11267852783203,
      "activations/layer2_attention_weight_max": 27.6451473236084,
      "activations/layer2_attention_weight_min": -24.033042907714844,
      "activations/layer3_attention_weight_max": 74.51964569091797,
      "activations/layer3_attention_weight_min": -78.51465606689453,
      "activations/layer4_attention_weight_max": 78.908935546875,
      "activations/layer4_attention_weight_min": -74.95143127441406,
      "activations/layer5_attention_weight_max": 58.80739974975586,
      "activations/layer5_attention_weight_min": -68.75702667236328,
      "activations/layer6_attention_weight_max": 44.340885162353516,
      "activations/layer6_attention_weight_min": -47.173912048339844,
      "activations/layer7_attention_weight_max": 55.88026809692383,
      "activations/layer7_attention_weight_min": -53.23196029663086,
      "activations/layer8_attention_weight_max": 41.77410125732422,
      "activations/layer8_attention_weight_min": -43.558170318603516,
      "activations/layer9_attention_weight_max": 41.67543029785156,
      "activations/layer9_attention_weight_min": -45.07249069213867,
      "epoch": 15.45,
      "learning_rate": 5.0878787878787876e-05,
      "loss": 2.7474,
      "step": 265900
    },
    {
      "activations/layer0_attention_weight_max": 15.905317306518555,
      "activations/layer0_attention_weight_min": -13.771458625793457,
      "activations/layer10_attention_weight_max": 36.41825485229492,
      "activations/layer10_attention_weight_min": -38.09512710571289,
      "activations/layer11_attention_weight_max": 32.28533172607422,
      "activations/layer11_attention_weight_min": -33.95570373535156,
      "activations/layer12_attention_weight_max": 19.038780212402344,
      "activations/layer12_attention_weight_min": -27.10407257080078,
      "activations/layer13_attention_weight_max": 34.099918365478516,
      "activations/layer13_attention_weight_min": -29.124483108520508,
      "activations/layer14_attention_weight_max": 37.214019775390625,
      "activations/layer14_attention_weight_min": -31.11882781982422,
      "activations/layer15_attention_weight_max": 32.2354621887207,
      "activations/layer15_attention_weight_min": -30.31031608581543,
      "activations/layer16_attention_weight_max": 34.05669021606445,
      "activations/layer16_attention_weight_min": -31.104093551635742,
      "activations/layer17_attention_weight_max": 54.27363204956055,
      "activations/layer17_attention_weight_min": -46.49278259277344,
      "activations/layer18_attention_weight_max": 45.6712646484375,
      "activations/layer18_attention_weight_min": -40.27238464355469,
      "activations/layer19_attention_weight_max": 22.973852157592773,
      "activations/layer19_attention_weight_min": -23.704755783081055,
      "activations/layer1_attention_weight_max": 17.380813598632812,
      "activations/layer1_attention_weight_min": -16.07973289489746,
      "activations/layer20_attention_weight_max": 22.199472427368164,
      "activations/layer20_attention_weight_min": -24.9393367767334,
      "activations/layer21_attention_weight_max": 35.4716911315918,
      "activations/layer21_attention_weight_min": -21.7579288482666,
      "activations/layer22_attention_weight_max": 30.53270721435547,
      "activations/layer22_attention_weight_min": -25.908418655395508,
      "activations/layer23_attention_weight_max": 36.61366271972656,
      "activations/layer23_attention_weight_min": -23.125797271728516,
      "activations/layer2_attention_weight_max": 28.474395751953125,
      "activations/layer2_attention_weight_min": -27.88930320739746,
      "activations/layer3_attention_weight_max": 78.17388916015625,
      "activations/layer3_attention_weight_min": -82.79965209960938,
      "activations/layer4_attention_weight_max": 86.2861099243164,
      "activations/layer4_attention_weight_min": -80.37097930908203,
      "activations/layer5_attention_weight_max": 67.49604797363281,
      "activations/layer5_attention_weight_min": -67.53170013427734,
      "activations/layer6_attention_weight_max": 46.73360824584961,
      "activations/layer6_attention_weight_min": -48.35621643066406,
      "activations/layer7_attention_weight_max": 57.994049072265625,
      "activations/layer7_attention_weight_min": -57.973602294921875,
      "activations/layer8_attention_weight_max": 46.98902893066406,
      "activations/layer8_attention_weight_min": -47.36760330200195,
      "activations/layer9_attention_weight_max": 50.236854553222656,
      "activations/layer9_attention_weight_min": -47.30888748168945,
      "epoch": 15.45,
      "learning_rate": 5.085984848484848e-05,
      "loss": 2.7673,
      "step": 265950
    },
    {
      "activations/layer0_attention_weight_max": 16.229114532470703,
      "activations/layer0_attention_weight_min": -13.67409896850586,
      "activations/layer10_attention_weight_max": 35.952911376953125,
      "activations/layer10_attention_weight_min": -37.388607025146484,
      "activations/layer11_attention_weight_max": 36.73406982421875,
      "activations/layer11_attention_weight_min": -34.31064987182617,
      "activations/layer12_attention_weight_max": 19.371339797973633,
      "activations/layer12_attention_weight_min": -27.043685913085938,
      "activations/layer13_attention_weight_max": 31.688634872436523,
      "activations/layer13_attention_weight_min": -29.938610076904297,
      "activations/layer14_attention_weight_max": 35.200984954833984,
      "activations/layer14_attention_weight_min": -30.802425384521484,
      "activations/layer15_attention_weight_max": 33.30232238769531,
      "activations/layer15_attention_weight_min": -31.289243698120117,
      "activations/layer16_attention_weight_max": 29.762136459350586,
      "activations/layer16_attention_weight_min": -29.774799346923828,
      "activations/layer17_attention_weight_max": 47.300994873046875,
      "activations/layer17_attention_weight_min": -44.9212532043457,
      "activations/layer18_attention_weight_max": 44.31257629394531,
      "activations/layer18_attention_weight_min": -39.15531921386719,
      "activations/layer19_attention_weight_max": 20.66480255126953,
      "activations/layer19_attention_weight_min": -25.6887149810791,
      "activations/layer1_attention_weight_max": 17.241565704345703,
      "activations/layer1_attention_weight_min": -16.291860580444336,
      "activations/layer20_attention_weight_max": 22.092449188232422,
      "activations/layer20_attention_weight_min": -23.388853073120117,
      "activations/layer21_attention_weight_max": 31.857967376708984,
      "activations/layer21_attention_weight_min": -22.20479393005371,
      "activations/layer22_attention_weight_max": 28.03068733215332,
      "activations/layer22_attention_weight_min": -24.88549041748047,
      "activations/layer23_attention_weight_max": 33.74924850463867,
      "activations/layer23_attention_weight_min": -22.679149627685547,
      "activations/layer2_attention_weight_max": 30.210599899291992,
      "activations/layer2_attention_weight_min": -29.34140396118164,
      "activations/layer3_attention_weight_max": 84.87871551513672,
      "activations/layer3_attention_weight_min": -89.48126220703125,
      "activations/layer4_attention_weight_max": 84.0315170288086,
      "activations/layer4_attention_weight_min": -86.22631072998047,
      "activations/layer5_attention_weight_max": 62.51394271850586,
      "activations/layer5_attention_weight_min": -77.034423828125,
      "activations/layer6_attention_weight_max": 48.72432327270508,
      "activations/layer6_attention_weight_min": -52.35447311401367,
      "activations/layer7_attention_weight_max": 65.74153137207031,
      "activations/layer7_attention_weight_min": -58.93442916870117,
      "activations/layer8_attention_weight_max": 48.64711380004883,
      "activations/layer8_attention_weight_min": -49.25644302368164,
      "activations/layer9_attention_weight_max": 55.8094367980957,
      "activations/layer9_attention_weight_min": -47.0301513671875,
      "epoch": 15.46,
      "learning_rate": 5.0840909090909086e-05,
      "loss": 2.7477,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4735,
      "eval_samples_per_second": 506.755,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4735,
      "eval_openwebtext_samples_per_second": 506.755,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9383,
      "eval_wikitext_samples_per_second": 235.264,
      "step": 266000
    },
    {
      "epoch": 15.46,
      "eval_lambada_loss": 2.537109375,
      "eval_lambada_ppl": 12.643071719773257,
      "eval_lambada_runtime": 9.5907,
      "eval_lambada_samples_per_second": 507.677,
      "step": 266000
    },
    {
      "activations/layer0_attention_weight_max": 15.87977409362793,
      "activations/layer0_attention_weight_min": -13.857463836669922,
      "activations/layer10_attention_weight_max": 36.46276092529297,
      "activations/layer10_attention_weight_min": -35.30903244018555,
      "activations/layer11_attention_weight_max": 32.54530715942383,
      "activations/layer11_attention_weight_min": -32.755882263183594,
      "activations/layer12_attention_weight_max": 18.263092041015625,
      "activations/layer12_attention_weight_min": -26.616844177246094,
      "activations/layer13_attention_weight_max": 32.94780731201172,
      "activations/layer13_attention_weight_min": -27.08646011352539,
      "activations/layer14_attention_weight_max": 34.99345016479492,
      "activations/layer14_attention_weight_min": -30.07459259033203,
      "activations/layer15_attention_weight_max": 32.638427734375,
      "activations/layer15_attention_weight_min": -29.393661499023438,
      "activations/layer16_attention_weight_max": 32.251922607421875,
      "activations/layer16_attention_weight_min": -30.84046173095703,
      "activations/layer17_attention_weight_max": 52.70302200317383,
      "activations/layer17_attention_weight_min": -45.38517761230469,
      "activations/layer18_attention_weight_max": 48.41234588623047,
      "activations/layer18_attention_weight_min": -41.269676208496094,
      "activations/layer19_attention_weight_max": 23.222684860229492,
      "activations/layer19_attention_weight_min": -22.98193359375,
      "activations/layer1_attention_weight_max": 16.027973175048828,
      "activations/layer1_attention_weight_min": -15.078158378601074,
      "activations/layer20_attention_weight_max": 22.801998138427734,
      "activations/layer20_attention_weight_min": -22.331602096557617,
      "activations/layer21_attention_weight_max": 39.391387939453125,
      "activations/layer21_attention_weight_min": -22.648448944091797,
      "activations/layer22_attention_weight_max": 34.0323600769043,
      "activations/layer22_attention_weight_min": -25.707319259643555,
      "activations/layer23_attention_weight_max": 38.329673767089844,
      "activations/layer23_attention_weight_min": -24.233991622924805,
      "activations/layer2_attention_weight_max": 30.52655792236328,
      "activations/layer2_attention_weight_min": -30.664077758789062,
      "activations/layer3_attention_weight_max": 86.97566223144531,
      "activations/layer3_attention_weight_min": -85.5811538696289,
      "activations/layer4_attention_weight_max": 86.8951187133789,
      "activations/layer4_attention_weight_min": -82.50667572021484,
      "activations/layer5_attention_weight_max": 64.24665832519531,
      "activations/layer5_attention_weight_min": -71.02142333984375,
      "activations/layer6_attention_weight_max": 48.97838592529297,
      "activations/layer6_attention_weight_min": -50.93153381347656,
      "activations/layer7_attention_weight_max": 61.08123779296875,
      "activations/layer7_attention_weight_min": -60.88825225830078,
      "activations/layer8_attention_weight_max": 47.18657302856445,
      "activations/layer8_attention_weight_min": -49.10538101196289,
      "activations/layer9_attention_weight_max": 46.350914001464844,
      "activations/layer9_attention_weight_min": -45.86558151245117,
      "epoch": 15.46,
      "learning_rate": 5.082196969696969e-05,
      "loss": 2.7466,
      "step": 266050
    },
    {
      "activations/layer0_attention_weight_max": 15.589502334594727,
      "activations/layer0_attention_weight_min": -13.515636444091797,
      "activations/layer10_attention_weight_max": 34.67531204223633,
      "activations/layer10_attention_weight_min": -35.00050735473633,
      "activations/layer11_attention_weight_max": 31.66736602783203,
      "activations/layer11_attention_weight_min": -32.772117614746094,
      "activations/layer12_attention_weight_max": 17.330575942993164,
      "activations/layer12_attention_weight_min": -27.88414192199707,
      "activations/layer13_attention_weight_max": 32.36284637451172,
      "activations/layer13_attention_weight_min": -26.369768142700195,
      "activations/layer14_attention_weight_max": 33.27133560180664,
      "activations/layer14_attention_weight_min": -30.22906494140625,
      "activations/layer15_attention_weight_max": 32.06730651855469,
      "activations/layer15_attention_weight_min": -30.76028060913086,
      "activations/layer16_attention_weight_max": 31.736324310302734,
      "activations/layer16_attention_weight_min": -31.88634490966797,
      "activations/layer17_attention_weight_max": 48.472381591796875,
      "activations/layer17_attention_weight_min": -45.44957733154297,
      "activations/layer18_attention_weight_max": 43.50698471069336,
      "activations/layer18_attention_weight_min": -38.576904296875,
      "activations/layer19_attention_weight_max": 21.193716049194336,
      "activations/layer19_attention_weight_min": -21.346771240234375,
      "activations/layer1_attention_weight_max": 17.60500717163086,
      "activations/layer1_attention_weight_min": -16.024545669555664,
      "activations/layer20_attention_weight_max": 20.107513427734375,
      "activations/layer20_attention_weight_min": -23.114887237548828,
      "activations/layer21_attention_weight_max": 34.2789306640625,
      "activations/layer21_attention_weight_min": -23.553966522216797,
      "activations/layer22_attention_weight_max": 29.232894897460938,
      "activations/layer22_attention_weight_min": -27.95425033569336,
      "activations/layer23_attention_weight_max": 35.22572708129883,
      "activations/layer23_attention_weight_min": -24.378738403320312,
      "activations/layer2_attention_weight_max": 31.307270050048828,
      "activations/layer2_attention_weight_min": -29.22345733642578,
      "activations/layer3_attention_weight_max": 86.80329132080078,
      "activations/layer3_attention_weight_min": -87.486572265625,
      "activations/layer4_attention_weight_max": 87.50251007080078,
      "activations/layer4_attention_weight_min": -88.08306121826172,
      "activations/layer5_attention_weight_max": 64.71682739257812,
      "activations/layer5_attention_weight_min": -73.8858871459961,
      "activations/layer6_attention_weight_max": 51.4991569519043,
      "activations/layer6_attention_weight_min": -49.789405822753906,
      "activations/layer7_attention_weight_max": 63.06300354003906,
      "activations/layer7_attention_weight_min": -59.368282318115234,
      "activations/layer8_attention_weight_max": 45.04780578613281,
      "activations/layer8_attention_weight_min": -46.81676483154297,
      "activations/layer9_attention_weight_max": 44.26736068725586,
      "activations/layer9_attention_weight_min": -44.11822509765625,
      "epoch": 15.46,
      "learning_rate": 5.08030303030303e-05,
      "loss": 2.7545,
      "step": 266100
    },
    {
      "activations/layer0_attention_weight_max": 15.032809257507324,
      "activations/layer0_attention_weight_min": -13.077506065368652,
      "activations/layer10_attention_weight_max": 38.66414260864258,
      "activations/layer10_attention_weight_min": -35.76626968383789,
      "activations/layer11_attention_weight_max": 38.20936965942383,
      "activations/layer11_attention_weight_min": -35.5162353515625,
      "activations/layer12_attention_weight_max": 18.69517707824707,
      "activations/layer12_attention_weight_min": -26.589601516723633,
      "activations/layer13_attention_weight_max": 40.0814094543457,
      "activations/layer13_attention_weight_min": -30.18722915649414,
      "activations/layer14_attention_weight_max": 51.4781608581543,
      "activations/layer14_attention_weight_min": -32.18415832519531,
      "activations/layer15_attention_weight_max": 47.95808792114258,
      "activations/layer15_attention_weight_min": -32.29518508911133,
      "activations/layer16_attention_weight_max": 39.713104248046875,
      "activations/layer16_attention_weight_min": -35.60767364501953,
      "activations/layer17_attention_weight_max": 63.63924026489258,
      "activations/layer17_attention_weight_min": -50.154624938964844,
      "activations/layer18_attention_weight_max": 59.86415481567383,
      "activations/layer18_attention_weight_min": -43.8156852722168,
      "activations/layer19_attention_weight_max": 29.165861129760742,
      "activations/layer19_attention_weight_min": -25.463483810424805,
      "activations/layer1_attention_weight_max": 16.66383171081543,
      "activations/layer1_attention_weight_min": -16.6701602935791,
      "activations/layer20_attention_weight_max": 28.883066177368164,
      "activations/layer20_attention_weight_min": -24.3602352142334,
      "activations/layer21_attention_weight_max": 51.106197357177734,
      "activations/layer21_attention_weight_min": -26.745702743530273,
      "activations/layer22_attention_weight_max": 35.93533706665039,
      "activations/layer22_attention_weight_min": -28.155773162841797,
      "activations/layer23_attention_weight_max": 47.8808708190918,
      "activations/layer23_attention_weight_min": -25.42574119567871,
      "activations/layer2_attention_weight_max": 35.12022399902344,
      "activations/layer2_attention_weight_min": -29.95148277282715,
      "activations/layer3_attention_weight_max": 105.82852935791016,
      "activations/layer3_attention_weight_min": -90.89495849609375,
      "activations/layer4_attention_weight_max": 94.93343353271484,
      "activations/layer4_attention_weight_min": -89.6774673461914,
      "activations/layer5_attention_weight_max": 66.60142517089844,
      "activations/layer5_attention_weight_min": -72.51350402832031,
      "activations/layer6_attention_weight_max": 49.104881286621094,
      "activations/layer6_attention_weight_min": -50.2102165222168,
      "activations/layer7_attention_weight_max": 61.639137268066406,
      "activations/layer7_attention_weight_min": -58.44978713989258,
      "activations/layer8_attention_weight_max": 46.09895706176758,
      "activations/layer8_attention_weight_min": -47.94435501098633,
      "activations/layer9_attention_weight_max": 50.5926513671875,
      "activations/layer9_attention_weight_min": -45.923545837402344,
      "epoch": 15.46,
      "learning_rate": 5.0784090909090904e-05,
      "loss": 2.7518,
      "step": 266150
    },
    {
      "activations/layer0_attention_weight_max": 15.114167213439941,
      "activations/layer0_attention_weight_min": -13.725934028625488,
      "activations/layer10_attention_weight_max": 35.61008071899414,
      "activations/layer10_attention_weight_min": -34.35277557373047,
      "activations/layer11_attention_weight_max": 31.640308380126953,
      "activations/layer11_attention_weight_min": -32.79888916015625,
      "activations/layer12_attention_weight_max": 19.744779586791992,
      "activations/layer12_attention_weight_min": -25.249099731445312,
      "activations/layer13_attention_weight_max": 36.334808349609375,
      "activations/layer13_attention_weight_min": -26.688932418823242,
      "activations/layer14_attention_weight_max": 37.69636154174805,
      "activations/layer14_attention_weight_min": -32.624610900878906,
      "activations/layer15_attention_weight_max": 33.46855545043945,
      "activations/layer15_attention_weight_min": -30.705066680908203,
      "activations/layer16_attention_weight_max": 33.35170364379883,
      "activations/layer16_attention_weight_min": -30.614994049072266,
      "activations/layer17_attention_weight_max": 52.276344299316406,
      "activations/layer17_attention_weight_min": -46.607391357421875,
      "activations/layer18_attention_weight_max": 50.75177764892578,
      "activations/layer18_attention_weight_min": -41.08196258544922,
      "activations/layer19_attention_weight_max": 24.682226181030273,
      "activations/layer19_attention_weight_min": -23.841161727905273,
      "activations/layer1_attention_weight_max": 19.2643985748291,
      "activations/layer1_attention_weight_min": -14.836256980895996,
      "activations/layer20_attention_weight_max": 25.396909713745117,
      "activations/layer20_attention_weight_min": -24.2179012298584,
      "activations/layer21_attention_weight_max": 38.412620544433594,
      "activations/layer21_attention_weight_min": -23.94065284729004,
      "activations/layer22_attention_weight_max": 31.061927795410156,
      "activations/layer22_attention_weight_min": -27.389265060424805,
      "activations/layer23_attention_weight_max": 42.90694808959961,
      "activations/layer23_attention_weight_min": -24.072216033935547,
      "activations/layer2_attention_weight_max": 28.506546020507812,
      "activations/layer2_attention_weight_min": -29.568626403808594,
      "activations/layer3_attention_weight_max": 81.13550567626953,
      "activations/layer3_attention_weight_min": -87.32772827148438,
      "activations/layer4_attention_weight_max": 83.0971450805664,
      "activations/layer4_attention_weight_min": -84.33673858642578,
      "activations/layer5_attention_weight_max": 63.89997100830078,
      "activations/layer5_attention_weight_min": -70.36947631835938,
      "activations/layer6_attention_weight_max": 49.60609817504883,
      "activations/layer6_attention_weight_min": -49.63972091674805,
      "activations/layer7_attention_weight_max": 62.373382568359375,
      "activations/layer7_attention_weight_min": -62.06494140625,
      "activations/layer8_attention_weight_max": 45.40200424194336,
      "activations/layer8_attention_weight_min": -47.077919006347656,
      "activations/layer9_attention_weight_max": 46.7718505859375,
      "activations/layer9_attention_weight_min": -47.335357666015625,
      "epoch": 15.47,
      "learning_rate": 5.0765151515151506e-05,
      "loss": 2.7511,
      "step": 266200
    },
    {
      "activations/layer0_attention_weight_max": 15.726091384887695,
      "activations/layer0_attention_weight_min": -13.602513313293457,
      "activations/layer10_attention_weight_max": 33.301795959472656,
      "activations/layer10_attention_weight_min": -32.9941291809082,
      "activations/layer11_attention_weight_max": 30.854290008544922,
      "activations/layer11_attention_weight_min": -32.44669723510742,
      "activations/layer12_attention_weight_max": 17.980907440185547,
      "activations/layer12_attention_weight_min": -23.456323623657227,
      "activations/layer13_attention_weight_max": 33.294769287109375,
      "activations/layer13_attention_weight_min": -29.449020385742188,
      "activations/layer14_attention_weight_max": 35.16884994506836,
      "activations/layer14_attention_weight_min": -30.432899475097656,
      "activations/layer15_attention_weight_max": 31.658763885498047,
      "activations/layer15_attention_weight_min": -29.607421875,
      "activations/layer16_attention_weight_max": 31.082447052001953,
      "activations/layer16_attention_weight_min": -30.954195022583008,
      "activations/layer17_attention_weight_max": 49.93828201293945,
      "activations/layer17_attention_weight_min": -43.923282623291016,
      "activations/layer18_attention_weight_max": 43.9962043762207,
      "activations/layer18_attention_weight_min": -39.556602478027344,
      "activations/layer19_attention_weight_max": 21.866676330566406,
      "activations/layer19_attention_weight_min": -22.800704956054688,
      "activations/layer1_attention_weight_max": 17.66459083557129,
      "activations/layer1_attention_weight_min": -16.17643165588379,
      "activations/layer20_attention_weight_max": 23.2769718170166,
      "activations/layer20_attention_weight_min": -22.186826705932617,
      "activations/layer21_attention_weight_max": 33.022125244140625,
      "activations/layer21_attention_weight_min": -24.1571044921875,
      "activations/layer22_attention_weight_max": 28.445533752441406,
      "activations/layer22_attention_weight_min": -25.508588790893555,
      "activations/layer23_attention_weight_max": 33.164859771728516,
      "activations/layer23_attention_weight_min": -23.17770004272461,
      "activations/layer2_attention_weight_max": 29.050518035888672,
      "activations/layer2_attention_weight_min": -28.39315414428711,
      "activations/layer3_attention_weight_max": 81.27619171142578,
      "activations/layer3_attention_weight_min": -86.81971740722656,
      "activations/layer4_attention_weight_max": 85.24288940429688,
      "activations/layer4_attention_weight_min": -87.47962951660156,
      "activations/layer5_attention_weight_max": 63.38806915283203,
      "activations/layer5_attention_weight_min": -74.55906677246094,
      "activations/layer6_attention_weight_max": 49.55953598022461,
      "activations/layer6_attention_weight_min": -50.18074035644531,
      "activations/layer7_attention_weight_max": 59.81331253051758,
      "activations/layer7_attention_weight_min": -58.74201965332031,
      "activations/layer8_attention_weight_max": 45.312339782714844,
      "activations/layer8_attention_weight_min": -49.91604232788086,
      "activations/layer9_attention_weight_max": 42.829429626464844,
      "activations/layer9_attention_weight_min": -45.39670944213867,
      "epoch": 15.47,
      "learning_rate": 5.074621212121212e-05,
      "loss": 2.7564,
      "step": 266250
    },
    {
      "activations/layer0_attention_weight_max": 16.657398223876953,
      "activations/layer0_attention_weight_min": -13.230141639709473,
      "activations/layer10_attention_weight_max": 34.306880950927734,
      "activations/layer10_attention_weight_min": -34.30908203125,
      "activations/layer11_attention_weight_max": 30.765783309936523,
      "activations/layer11_attention_weight_min": -34.07371520996094,
      "activations/layer12_attention_weight_max": 18.98758316040039,
      "activations/layer12_attention_weight_min": -23.65593147277832,
      "activations/layer13_attention_weight_max": 37.943260192871094,
      "activations/layer13_attention_weight_min": -27.982118606567383,
      "activations/layer14_attention_weight_max": 32.482418060302734,
      "activations/layer14_attention_weight_min": -29.59897804260254,
      "activations/layer15_attention_weight_max": 29.576927185058594,
      "activations/layer15_attention_weight_min": -30.91679573059082,
      "activations/layer16_attention_weight_max": 31.934329986572266,
      "activations/layer16_attention_weight_min": -30.522371292114258,
      "activations/layer17_attention_weight_max": 48.8904914855957,
      "activations/layer17_attention_weight_min": -45.397308349609375,
      "activations/layer18_attention_weight_max": 43.304115295410156,
      "activations/layer18_attention_weight_min": -38.657657623291016,
      "activations/layer19_attention_weight_max": 22.280689239501953,
      "activations/layer19_attention_weight_min": -21.757038116455078,
      "activations/layer1_attention_weight_max": 17.695138931274414,
      "activations/layer1_attention_weight_min": -16.509065628051758,
      "activations/layer20_attention_weight_max": 20.62285041809082,
      "activations/layer20_attention_weight_min": -23.035884857177734,
      "activations/layer21_attention_weight_max": 32.90098190307617,
      "activations/layer21_attention_weight_min": -22.287734985351562,
      "activations/layer22_attention_weight_max": 28.083908081054688,
      "activations/layer22_attention_weight_min": -26.808244705200195,
      "activations/layer23_attention_weight_max": 34.54121398925781,
      "activations/layer23_attention_weight_min": -23.560688018798828,
      "activations/layer2_attention_weight_max": 29.39907455444336,
      "activations/layer2_attention_weight_min": -27.683128356933594,
      "activations/layer3_attention_weight_max": 81.8883285522461,
      "activations/layer3_attention_weight_min": -86.98397827148438,
      "activations/layer4_attention_weight_max": 86.14207458496094,
      "activations/layer4_attention_weight_min": -82.07537078857422,
      "activations/layer5_attention_weight_max": 65.1420669555664,
      "activations/layer5_attention_weight_min": -77.37179565429688,
      "activations/layer6_attention_weight_max": 49.42736053466797,
      "activations/layer6_attention_weight_min": -49.24995422363281,
      "activations/layer7_attention_weight_max": 59.67213439941406,
      "activations/layer7_attention_weight_min": -58.001094818115234,
      "activations/layer8_attention_weight_max": 43.389320373535156,
      "activations/layer8_attention_weight_min": -46.91033172607422,
      "activations/layer9_attention_weight_max": 41.96886444091797,
      "activations/layer9_attention_weight_min": -45.4798698425293,
      "epoch": 15.47,
      "learning_rate": 5.072727272727272e-05,
      "loss": 2.7543,
      "step": 266300
    },
    {
      "activations/layer0_attention_weight_max": 15.36396312713623,
      "activations/layer0_attention_weight_min": -13.438200950622559,
      "activations/layer10_attention_weight_max": 33.68955993652344,
      "activations/layer10_attention_weight_min": -34.56224822998047,
      "activations/layer11_attention_weight_max": 30.672985076904297,
      "activations/layer11_attention_weight_min": -33.050785064697266,
      "activations/layer12_attention_weight_max": 21.150041580200195,
      "activations/layer12_attention_weight_min": -28.287569046020508,
      "activations/layer13_attention_weight_max": 35.42900466918945,
      "activations/layer13_attention_weight_min": -29.557411193847656,
      "activations/layer14_attention_weight_max": 36.301937103271484,
      "activations/layer14_attention_weight_min": -32.172454833984375,
      "activations/layer15_attention_weight_max": 33.209617614746094,
      "activations/layer15_attention_weight_min": -30.706090927124023,
      "activations/layer16_attention_weight_max": 35.833255767822266,
      "activations/layer16_attention_weight_min": -33.18669891357422,
      "activations/layer17_attention_weight_max": 57.14085006713867,
      "activations/layer17_attention_weight_min": -49.80873489379883,
      "activations/layer18_attention_weight_max": 51.90250778198242,
      "activations/layer18_attention_weight_min": -44.00923538208008,
      "activations/layer19_attention_weight_max": 25.403270721435547,
      "activations/layer19_attention_weight_min": -23.527273178100586,
      "activations/layer1_attention_weight_max": 18.995201110839844,
      "activations/layer1_attention_weight_min": -13.324237823486328,
      "activations/layer20_attention_weight_max": 27.172338485717773,
      "activations/layer20_attention_weight_min": -22.936885833740234,
      "activations/layer21_attention_weight_max": 48.95655059814453,
      "activations/layer21_attention_weight_min": -30.172515869140625,
      "activations/layer22_attention_weight_max": 31.300561904907227,
      "activations/layer22_attention_weight_min": -27.48003578186035,
      "activations/layer23_attention_weight_max": 35.71267318725586,
      "activations/layer23_attention_weight_min": -24.151662826538086,
      "activations/layer2_attention_weight_max": 30.2941951751709,
      "activations/layer2_attention_weight_min": -26.987812042236328,
      "activations/layer3_attention_weight_max": 76.22513580322266,
      "activations/layer3_attention_weight_min": -79.42198181152344,
      "activations/layer4_attention_weight_max": 80.28093719482422,
      "activations/layer4_attention_weight_min": -82.18499755859375,
      "activations/layer5_attention_weight_max": 60.32786560058594,
      "activations/layer5_attention_weight_min": -69.1890640258789,
      "activations/layer6_attention_weight_max": 47.50899887084961,
      "activations/layer6_attention_weight_min": -49.164241790771484,
      "activations/layer7_attention_weight_max": 58.3444709777832,
      "activations/layer7_attention_weight_min": -59.93198013305664,
      "activations/layer8_attention_weight_max": 41.80654525756836,
      "activations/layer8_attention_weight_min": -45.45341491699219,
      "activations/layer9_attention_weight_max": 43.43619155883789,
      "activations/layer9_attention_weight_min": -42.846805572509766,
      "epoch": 15.48,
      "learning_rate": 5.070833333333333e-05,
      "loss": 2.7482,
      "step": 266350
    },
    {
      "activations/layer0_attention_weight_max": 16.08422088623047,
      "activations/layer0_attention_weight_min": -13.296231269836426,
      "activations/layer10_attention_weight_max": 43.59064865112305,
      "activations/layer10_attention_weight_min": -44.64982223510742,
      "activations/layer11_attention_weight_max": 42.22655487060547,
      "activations/layer11_attention_weight_min": -42.709930419921875,
      "activations/layer12_attention_weight_max": 25.064815521240234,
      "activations/layer12_attention_weight_min": -31.500288009643555,
      "activations/layer13_attention_weight_max": 40.09536361694336,
      "activations/layer13_attention_weight_min": -29.824254989624023,
      "activations/layer14_attention_weight_max": 39.9756965637207,
      "activations/layer14_attention_weight_min": -32.44984436035156,
      "activations/layer15_attention_weight_max": 34.327945709228516,
      "activations/layer15_attention_weight_min": -30.096315383911133,
      "activations/layer16_attention_weight_max": 34.389488220214844,
      "activations/layer16_attention_weight_min": -31.852880477905273,
      "activations/layer17_attention_weight_max": 56.94510269165039,
      "activations/layer17_attention_weight_min": -47.63097381591797,
      "activations/layer18_attention_weight_max": 48.37680435180664,
      "activations/layer18_attention_weight_min": -40.3504753112793,
      "activations/layer19_attention_weight_max": 23.464689254760742,
      "activations/layer19_attention_weight_min": -22.36259651184082,
      "activations/layer1_attention_weight_max": 16.736093521118164,
      "activations/layer1_attention_weight_min": -14.906730651855469,
      "activations/layer20_attention_weight_max": 23.767942428588867,
      "activations/layer20_attention_weight_min": -22.655912399291992,
      "activations/layer21_attention_weight_max": 35.226158142089844,
      "activations/layer21_attention_weight_min": -27.382652282714844,
      "activations/layer22_attention_weight_max": 28.373987197875977,
      "activations/layer22_attention_weight_min": -25.356605529785156,
      "activations/layer23_attention_weight_max": 31.697927474975586,
      "activations/layer23_attention_weight_min": -24.60702896118164,
      "activations/layer2_attention_weight_max": 29.4547119140625,
      "activations/layer2_attention_weight_min": -27.611190795898438,
      "activations/layer3_attention_weight_max": 85.40233612060547,
      "activations/layer3_attention_weight_min": -90.99938201904297,
      "activations/layer4_attention_weight_max": 86.12007141113281,
      "activations/layer4_attention_weight_min": -84.54183197021484,
      "activations/layer5_attention_weight_max": 61.11080551147461,
      "activations/layer5_attention_weight_min": -67.75483703613281,
      "activations/layer6_attention_weight_max": 51.164268493652344,
      "activations/layer6_attention_weight_min": -52.39851379394531,
      "activations/layer7_attention_weight_max": 66.89765167236328,
      "activations/layer7_attention_weight_min": -65.46885681152344,
      "activations/layer8_attention_weight_max": 51.291587829589844,
      "activations/layer8_attention_weight_min": -55.93428421020508,
      "activations/layer9_attention_weight_max": 54.09634017944336,
      "activations/layer9_attention_weight_min": -50.5380859375,
      "epoch": 15.48,
      "learning_rate": 5.068939393939393e-05,
      "loss": 2.7626,
      "step": 266400
    },
    {
      "activations/layer0_attention_weight_max": 15.579436302185059,
      "activations/layer0_attention_weight_min": -13.600066184997559,
      "activations/layer10_attention_weight_max": 37.02326202392578,
      "activations/layer10_attention_weight_min": -37.21236038208008,
      "activations/layer11_attention_weight_max": 32.99595642089844,
      "activations/layer11_attention_weight_min": -34.44089889526367,
      "activations/layer12_attention_weight_max": 18.411020278930664,
      "activations/layer12_attention_weight_min": -25.087553024291992,
      "activations/layer13_attention_weight_max": 41.60700607299805,
      "activations/layer13_attention_weight_min": -26.835485458374023,
      "activations/layer14_attention_weight_max": 39.30301284790039,
      "activations/layer14_attention_weight_min": -30.721338272094727,
      "activations/layer15_attention_weight_max": 33.08718490600586,
      "activations/layer15_attention_weight_min": -31.504419326782227,
      "activations/layer16_attention_weight_max": 32.82674789428711,
      "activations/layer16_attention_weight_min": -31.354291915893555,
      "activations/layer17_attention_weight_max": 49.84406661987305,
      "activations/layer17_attention_weight_min": -45.66261672973633,
      "activations/layer18_attention_weight_max": 45.42251968383789,
      "activations/layer18_attention_weight_min": -39.5379524230957,
      "activations/layer19_attention_weight_max": 22.18138313293457,
      "activations/layer19_attention_weight_min": -23.561321258544922,
      "activations/layer1_attention_weight_max": 16.738344192504883,
      "activations/layer1_attention_weight_min": -14.674015045166016,
      "activations/layer20_attention_weight_max": 20.99427604675293,
      "activations/layer20_attention_weight_min": -22.906208038330078,
      "activations/layer21_attention_weight_max": 32.87160110473633,
      "activations/layer21_attention_weight_min": -22.116960525512695,
      "activations/layer22_attention_weight_max": 27.915775299072266,
      "activations/layer22_attention_weight_min": -27.317468643188477,
      "activations/layer23_attention_weight_max": 36.845314025878906,
      "activations/layer23_attention_weight_min": -24.072294235229492,
      "activations/layer2_attention_weight_max": 28.89339256286621,
      "activations/layer2_attention_weight_min": -28.858617782592773,
      "activations/layer3_attention_weight_max": 85.11566925048828,
      "activations/layer3_attention_weight_min": -88.4758529663086,
      "activations/layer4_attention_weight_max": 80.71731567382812,
      "activations/layer4_attention_weight_min": -79.42672729492188,
      "activations/layer5_attention_weight_max": 64.02423858642578,
      "activations/layer5_attention_weight_min": -68.53038024902344,
      "activations/layer6_attention_weight_max": 46.580928802490234,
      "activations/layer6_attention_weight_min": -49.537837982177734,
      "activations/layer7_attention_weight_max": 59.42902374267578,
      "activations/layer7_attention_weight_min": -60.05575942993164,
      "activations/layer8_attention_weight_max": 47.02779006958008,
      "activations/layer8_attention_weight_min": -48.047298431396484,
      "activations/layer9_attention_weight_max": 45.62574005126953,
      "activations/layer9_attention_weight_min": -45.6982307434082,
      "epoch": 15.48,
      "learning_rate": 5.067045454545455e-05,
      "loss": 2.7472,
      "step": 266450
    },
    {
      "activations/layer0_attention_weight_max": 16.881549835205078,
      "activations/layer0_attention_weight_min": -13.037140846252441,
      "activations/layer10_attention_weight_max": 35.37559127807617,
      "activations/layer10_attention_weight_min": -35.817996978759766,
      "activations/layer11_attention_weight_max": 30.716537475585938,
      "activations/layer11_attention_weight_min": -33.26808166503906,
      "activations/layer12_attention_weight_max": 21.302907943725586,
      "activations/layer12_attention_weight_min": -27.537525177001953,
      "activations/layer13_attention_weight_max": 41.126094818115234,
      "activations/layer13_attention_weight_min": -28.979251861572266,
      "activations/layer14_attention_weight_max": 37.51316452026367,
      "activations/layer14_attention_weight_min": -30.662025451660156,
      "activations/layer15_attention_weight_max": 36.08013916015625,
      "activations/layer15_attention_weight_min": -31.590747833251953,
      "activations/layer16_attention_weight_max": 36.243385314941406,
      "activations/layer16_attention_weight_min": -33.65638732910156,
      "activations/layer17_attention_weight_max": 56.20158004760742,
      "activations/layer17_attention_weight_min": -47.92284393310547,
      "activations/layer18_attention_weight_max": 49.67640686035156,
      "activations/layer18_attention_weight_min": -40.092105865478516,
      "activations/layer19_attention_weight_max": 25.916275024414062,
      "activations/layer19_attention_weight_min": -22.0317440032959,
      "activations/layer1_attention_weight_max": 16.002437591552734,
      "activations/layer1_attention_weight_min": -14.419919967651367,
      "activations/layer20_attention_weight_max": 24.282882690429688,
      "activations/layer20_attention_weight_min": -21.152738571166992,
      "activations/layer21_attention_weight_max": 41.84480667114258,
      "activations/layer21_attention_weight_min": -21.379886627197266,
      "activations/layer22_attention_weight_max": 31.205461502075195,
      "activations/layer22_attention_weight_min": -25.980485916137695,
      "activations/layer23_attention_weight_max": 38.97266387939453,
      "activations/layer23_attention_weight_min": -25.38600730895996,
      "activations/layer2_attention_weight_max": 29.234394073486328,
      "activations/layer2_attention_weight_min": -27.845073699951172,
      "activations/layer3_attention_weight_max": 85.39037322998047,
      "activations/layer3_attention_weight_min": -86.11051940917969,
      "activations/layer4_attention_weight_max": 85.8656997680664,
      "activations/layer4_attention_weight_min": -86.71517181396484,
      "activations/layer5_attention_weight_max": 62.18014144897461,
      "activations/layer5_attention_weight_min": -70.98040008544922,
      "activations/layer6_attention_weight_max": 51.32646560668945,
      "activations/layer6_attention_weight_min": -52.90998840332031,
      "activations/layer7_attention_weight_max": 62.11457824707031,
      "activations/layer7_attention_weight_min": -60.59307098388672,
      "activations/layer8_attention_weight_max": 44.44268035888672,
      "activations/layer8_attention_weight_min": -47.691165924072266,
      "activations/layer9_attention_weight_max": 47.77939224243164,
      "activations/layer9_attention_weight_min": -44.92865753173828,
      "epoch": 15.49,
      "learning_rate": 5.065151515151515e-05,
      "loss": 2.7471,
      "step": 266500
    },
    {
      "activations/layer0_attention_weight_max": 16.543153762817383,
      "activations/layer0_attention_weight_min": -13.288883209228516,
      "activations/layer10_attention_weight_max": 34.330810546875,
      "activations/layer10_attention_weight_min": -34.72111129760742,
      "activations/layer11_attention_weight_max": 29.904117584228516,
      "activations/layer11_attention_weight_min": -32.06284713745117,
      "activations/layer12_attention_weight_max": 17.059797286987305,
      "activations/layer12_attention_weight_min": -24.148351669311523,
      "activations/layer13_attention_weight_max": 31.877763748168945,
      "activations/layer13_attention_weight_min": -26.879234313964844,
      "activations/layer14_attention_weight_max": 34.542518615722656,
      "activations/layer14_attention_weight_min": -29.45215606689453,
      "activations/layer15_attention_weight_max": 31.458955764770508,
      "activations/layer15_attention_weight_min": -29.5303897857666,
      "activations/layer16_attention_weight_max": 31.465803146362305,
      "activations/layer16_attention_weight_min": -30.75072479248047,
      "activations/layer17_attention_weight_max": 48.19031524658203,
      "activations/layer17_attention_weight_min": -41.33589553833008,
      "activations/layer18_attention_weight_max": 43.905941009521484,
      "activations/layer18_attention_weight_min": -37.269126892089844,
      "activations/layer19_attention_weight_max": 21.334270477294922,
      "activations/layer19_attention_weight_min": -21.075271606445312,
      "activations/layer1_attention_weight_max": 16.815811157226562,
      "activations/layer1_attention_weight_min": -14.611846923828125,
      "activations/layer20_attention_weight_max": 19.960294723510742,
      "activations/layer20_attention_weight_min": -21.47857093811035,
      "activations/layer21_attention_weight_max": 30.531187057495117,
      "activations/layer21_attention_weight_min": -22.025461196899414,
      "activations/layer22_attention_weight_max": 26.29231071472168,
      "activations/layer22_attention_weight_min": -26.48075294494629,
      "activations/layer23_attention_weight_max": 32.79800796508789,
      "activations/layer23_attention_weight_min": -24.36092758178711,
      "activations/layer2_attention_weight_max": 28.595947265625,
      "activations/layer2_attention_weight_min": -27.760404586791992,
      "activations/layer3_attention_weight_max": 86.42523193359375,
      "activations/layer3_attention_weight_min": -86.76351165771484,
      "activations/layer4_attention_weight_max": 85.71150207519531,
      "activations/layer4_attention_weight_min": -84.45396423339844,
      "activations/layer5_attention_weight_max": 63.33351135253906,
      "activations/layer5_attention_weight_min": -69.66151428222656,
      "activations/layer6_attention_weight_max": 50.60694885253906,
      "activations/layer6_attention_weight_min": -50.29396438598633,
      "activations/layer7_attention_weight_max": 62.63737487792969,
      "activations/layer7_attention_weight_min": -58.17985534667969,
      "activations/layer8_attention_weight_max": 43.65839385986328,
      "activations/layer8_attention_weight_min": -46.26462936401367,
      "activations/layer9_attention_weight_max": 43.79568099975586,
      "activations/layer9_attention_weight_min": -45.638736724853516,
      "epoch": 15.49,
      "learning_rate": 5.063257575757575e-05,
      "loss": 2.7652,
      "step": 266550
    },
    {
      "activations/layer0_attention_weight_max": 15.628466606140137,
      "activations/layer0_attention_weight_min": -13.434114456176758,
      "activations/layer10_attention_weight_max": 33.86335372924805,
      "activations/layer10_attention_weight_min": -34.557289123535156,
      "activations/layer11_attention_weight_max": 32.45858383178711,
      "activations/layer11_attention_weight_min": -31.9559326171875,
      "activations/layer12_attention_weight_max": 19.91973304748535,
      "activations/layer12_attention_weight_min": -26.895856857299805,
      "activations/layer13_attention_weight_max": 40.62846755981445,
      "activations/layer13_attention_weight_min": -30.46722412109375,
      "activations/layer14_attention_weight_max": 35.18812942504883,
      "activations/layer14_attention_weight_min": -31.031957626342773,
      "activations/layer15_attention_weight_max": 33.4328727722168,
      "activations/layer15_attention_weight_min": -29.344892501831055,
      "activations/layer16_attention_weight_max": 33.321311950683594,
      "activations/layer16_attention_weight_min": -32.640830993652344,
      "activations/layer17_attention_weight_max": 54.82353591918945,
      "activations/layer17_attention_weight_min": -48.9440803527832,
      "activations/layer18_attention_weight_max": 49.27608108520508,
      "activations/layer18_attention_weight_min": -45.05167770385742,
      "activations/layer19_attention_weight_max": 21.84082794189453,
      "activations/layer19_attention_weight_min": -22.152236938476562,
      "activations/layer1_attention_weight_max": 16.23921012878418,
      "activations/layer1_attention_weight_min": -14.516756057739258,
      "activations/layer20_attention_weight_max": 22.69612693786621,
      "activations/layer20_attention_weight_min": -21.77195930480957,
      "activations/layer21_attention_weight_max": 36.01830291748047,
      "activations/layer21_attention_weight_min": -27.52981948852539,
      "activations/layer22_attention_weight_max": 28.844377517700195,
      "activations/layer22_attention_weight_min": -25.656436920166016,
      "activations/layer23_attention_weight_max": 38.27312088012695,
      "activations/layer23_attention_weight_min": -24.02025604248047,
      "activations/layer2_attention_weight_max": 27.1771240234375,
      "activations/layer2_attention_weight_min": -26.67715835571289,
      "activations/layer3_attention_weight_max": 83.40261840820312,
      "activations/layer3_attention_weight_min": -86.04203033447266,
      "activations/layer4_attention_weight_max": 84.01148223876953,
      "activations/layer4_attention_weight_min": -82.06945037841797,
      "activations/layer5_attention_weight_max": 62.66426086425781,
      "activations/layer5_attention_weight_min": -75.4130630493164,
      "activations/layer6_attention_weight_max": 52.55038833618164,
      "activations/layer6_attention_weight_min": -52.23344039916992,
      "activations/layer7_attention_weight_max": 59.79123306274414,
      "activations/layer7_attention_weight_min": -58.766746520996094,
      "activations/layer8_attention_weight_max": 44.68064880371094,
      "activations/layer8_attention_weight_min": -51.69176483154297,
      "activations/layer9_attention_weight_max": 43.43964767456055,
      "activations/layer9_attention_weight_min": -45.639522552490234,
      "epoch": 15.49,
      "learning_rate": 5.061363636363636e-05,
      "loss": 2.7517,
      "step": 266600
    },
    {
      "activations/layer0_attention_weight_max": 15.244872093200684,
      "activations/layer0_attention_weight_min": -13.319555282592773,
      "activations/layer10_attention_weight_max": 33.643653869628906,
      "activations/layer10_attention_weight_min": -36.561126708984375,
      "activations/layer11_attention_weight_max": 31.274978637695312,
      "activations/layer11_attention_weight_min": -33.00682830810547,
      "activations/layer12_attention_weight_max": 20.1005859375,
      "activations/layer12_attention_weight_min": -28.463956832885742,
      "activations/layer13_attention_weight_max": 34.933494567871094,
      "activations/layer13_attention_weight_min": -27.05486297607422,
      "activations/layer14_attention_weight_max": 37.46540069580078,
      "activations/layer14_attention_weight_min": -29.13987159729004,
      "activations/layer15_attention_weight_max": 32.87709045410156,
      "activations/layer15_attention_weight_min": -31.958696365356445,
      "activations/layer16_attention_weight_max": 34.13459777832031,
      "activations/layer16_attention_weight_min": -32.02490234375,
      "activations/layer17_attention_weight_max": 56.14094924926758,
      "activations/layer17_attention_weight_min": -47.92052459716797,
      "activations/layer18_attention_weight_max": 48.462867736816406,
      "activations/layer18_attention_weight_min": -42.34970474243164,
      "activations/layer19_attention_weight_max": 23.133522033691406,
      "activations/layer19_attention_weight_min": -22.34854507446289,
      "activations/layer1_attention_weight_max": 16.607576370239258,
      "activations/layer1_attention_weight_min": -14.686558723449707,
      "activations/layer20_attention_weight_max": 22.77220344543457,
      "activations/layer20_attention_weight_min": -22.202072143554688,
      "activations/layer21_attention_weight_max": 37.525394439697266,
      "activations/layer21_attention_weight_min": -21.233619689941406,
      "activations/layer22_attention_weight_max": 33.49091720581055,
      "activations/layer22_attention_weight_min": -26.129968643188477,
      "activations/layer23_attention_weight_max": 37.1264762878418,
      "activations/layer23_attention_weight_min": -24.17316436767578,
      "activations/layer2_attention_weight_max": 30.702924728393555,
      "activations/layer2_attention_weight_min": -28.137344360351562,
      "activations/layer3_attention_weight_max": 87.27835083007812,
      "activations/layer3_attention_weight_min": -89.56411743164062,
      "activations/layer4_attention_weight_max": 88.02703857421875,
      "activations/layer4_attention_weight_min": -84.75850677490234,
      "activations/layer5_attention_weight_max": 69.76691436767578,
      "activations/layer5_attention_weight_min": -72.83039855957031,
      "activations/layer6_attention_weight_max": 50.85026931762695,
      "activations/layer6_attention_weight_min": -51.05591583251953,
      "activations/layer7_attention_weight_max": 63.23618698120117,
      "activations/layer7_attention_weight_min": -58.62150955200195,
      "activations/layer8_attention_weight_max": 43.673095703125,
      "activations/layer8_attention_weight_min": -47.78606033325195,
      "activations/layer9_attention_weight_max": 42.640777587890625,
      "activations/layer9_attention_weight_min": -46.9787483215332,
      "epoch": 15.49,
      "learning_rate": 5.0594696969696966e-05,
      "loss": 2.761,
      "step": 266650
    },
    {
      "activations/layer0_attention_weight_max": 14.356468200683594,
      "activations/layer0_attention_weight_min": -13.533341407775879,
      "activations/layer10_attention_weight_max": 38.56598663330078,
      "activations/layer10_attention_weight_min": -37.889892578125,
      "activations/layer11_attention_weight_max": 33.90176010131836,
      "activations/layer11_attention_weight_min": -35.242515563964844,
      "activations/layer12_attention_weight_max": 18.93977928161621,
      "activations/layer12_attention_weight_min": -27.023557662963867,
      "activations/layer13_attention_weight_max": 37.202125549316406,
      "activations/layer13_attention_weight_min": -28.049156188964844,
      "activations/layer14_attention_weight_max": 36.68046188354492,
      "activations/layer14_attention_weight_min": -31.62090301513672,
      "activations/layer15_attention_weight_max": 34.26106262207031,
      "activations/layer15_attention_weight_min": -30.045570373535156,
      "activations/layer16_attention_weight_max": 33.21954345703125,
      "activations/layer16_attention_weight_min": -32.01634216308594,
      "activations/layer17_attention_weight_max": 52.9538459777832,
      "activations/layer17_attention_weight_min": -44.89812088012695,
      "activations/layer18_attention_weight_max": 47.82583999633789,
      "activations/layer18_attention_weight_min": -39.85258865356445,
      "activations/layer19_attention_weight_max": 25.6577091217041,
      "activations/layer19_attention_weight_min": -21.836978912353516,
      "activations/layer1_attention_weight_max": 16.808006286621094,
      "activations/layer1_attention_weight_min": -15.658023834228516,
      "activations/layer20_attention_weight_max": 22.853910446166992,
      "activations/layer20_attention_weight_min": -21.566810607910156,
      "activations/layer21_attention_weight_max": 34.76953125,
      "activations/layer21_attention_weight_min": -24.23247718811035,
      "activations/layer22_attention_weight_max": 28.812646865844727,
      "activations/layer22_attention_weight_min": -25.547903060913086,
      "activations/layer23_attention_weight_max": 36.07061767578125,
      "activations/layer23_attention_weight_min": -29.630767822265625,
      "activations/layer2_attention_weight_max": 29.252681732177734,
      "activations/layer2_attention_weight_min": -28.89196014404297,
      "activations/layer3_attention_weight_max": 89.55262756347656,
      "activations/layer3_attention_weight_min": -91.11327362060547,
      "activations/layer4_attention_weight_max": 93.02947235107422,
      "activations/layer4_attention_weight_min": -89.58975982666016,
      "activations/layer5_attention_weight_max": 69.08065795898438,
      "activations/layer5_attention_weight_min": -78.08783721923828,
      "activations/layer6_attention_weight_max": 52.16252517700195,
      "activations/layer6_attention_weight_min": -52.88589859008789,
      "activations/layer7_attention_weight_max": 61.53013610839844,
      "activations/layer7_attention_weight_min": -64.9915542602539,
      "activations/layer8_attention_weight_max": 48.58713912963867,
      "activations/layer8_attention_weight_min": -50.80043411254883,
      "activations/layer9_attention_weight_max": 47.60495376586914,
      "activations/layer9_attention_weight_min": -48.73699188232422,
      "epoch": 15.5,
      "learning_rate": 5.0575757575757575e-05,
      "loss": 2.7599,
      "step": 266700
    },
    {
      "activations/layer0_attention_weight_max": 15.328207015991211,
      "activations/layer0_attention_weight_min": -13.851338386535645,
      "activations/layer10_attention_weight_max": 37.00789260864258,
      "activations/layer10_attention_weight_min": -36.665462493896484,
      "activations/layer11_attention_weight_max": 34.02577590942383,
      "activations/layer11_attention_weight_min": -32.97478103637695,
      "activations/layer12_attention_weight_max": 18.539857864379883,
      "activations/layer12_attention_weight_min": -27.018430709838867,
      "activations/layer13_attention_weight_max": 32.222381591796875,
      "activations/layer13_attention_weight_min": -28.02512550354004,
      "activations/layer14_attention_weight_max": 33.72203063964844,
      "activations/layer14_attention_weight_min": -30.67052459716797,
      "activations/layer15_attention_weight_max": 30.97130012512207,
      "activations/layer15_attention_weight_min": -29.774473190307617,
      "activations/layer16_attention_weight_max": 32.34378433227539,
      "activations/layer16_attention_weight_min": -31.72487449645996,
      "activations/layer17_attention_weight_max": 49.02116394042969,
      "activations/layer17_attention_weight_min": -46.4471321105957,
      "activations/layer18_attention_weight_max": 42.1839485168457,
      "activations/layer18_attention_weight_min": -39.23109436035156,
      "activations/layer19_attention_weight_max": 22.280075073242188,
      "activations/layer19_attention_weight_min": -23.024124145507812,
      "activations/layer1_attention_weight_max": 16.310409545898438,
      "activations/layer1_attention_weight_min": -15.553973197937012,
      "activations/layer20_attention_weight_max": 20.777244567871094,
      "activations/layer20_attention_weight_min": -21.410486221313477,
      "activations/layer21_attention_weight_max": 32.359920501708984,
      "activations/layer21_attention_weight_min": -22.158620834350586,
      "activations/layer22_attention_weight_max": 30.29550552368164,
      "activations/layer22_attention_weight_min": -26.074501037597656,
      "activations/layer23_attention_weight_max": 35.540977478027344,
      "activations/layer23_attention_weight_min": -23.414047241210938,
      "activations/layer2_attention_weight_max": 30.23494529724121,
      "activations/layer2_attention_weight_min": -29.096343994140625,
      "activations/layer3_attention_weight_max": 85.67510986328125,
      "activations/layer3_attention_weight_min": -90.13275146484375,
      "activations/layer4_attention_weight_max": 90.19645690917969,
      "activations/layer4_attention_weight_min": -86.50133514404297,
      "activations/layer5_attention_weight_max": 65.95153045654297,
      "activations/layer5_attention_weight_min": -70.17916107177734,
      "activations/layer6_attention_weight_max": 50.182945251464844,
      "activations/layer6_attention_weight_min": -48.79372787475586,
      "activations/layer7_attention_weight_max": 62.35549545288086,
      "activations/layer7_attention_weight_min": -58.377315521240234,
      "activations/layer8_attention_weight_max": 46.39419937133789,
      "activations/layer8_attention_weight_min": -48.394264221191406,
      "activations/layer9_attention_weight_max": 44.90018844604492,
      "activations/layer9_attention_weight_min": -44.37409210205078,
      "epoch": 15.5,
      "learning_rate": 5.0556818181818176e-05,
      "loss": 2.7569,
      "step": 266750
    },
    {
      "activations/layer0_attention_weight_max": 15.764508247375488,
      "activations/layer0_attention_weight_min": -13.487828254699707,
      "activations/layer10_attention_weight_max": 38.11639404296875,
      "activations/layer10_attention_weight_min": -37.43379592895508,
      "activations/layer11_attention_weight_max": 33.86330032348633,
      "activations/layer11_attention_weight_min": -34.94379806518555,
      "activations/layer12_attention_weight_max": 20.682697296142578,
      "activations/layer12_attention_weight_min": -25.159088134765625,
      "activations/layer13_attention_weight_max": 35.12934112548828,
      "activations/layer13_attention_weight_min": -31.706790924072266,
      "activations/layer14_attention_weight_max": 37.46310806274414,
      "activations/layer14_attention_weight_min": -31.52958106994629,
      "activations/layer15_attention_weight_max": 33.67106246948242,
      "activations/layer15_attention_weight_min": -29.896705627441406,
      "activations/layer16_attention_weight_max": 33.62056350708008,
      "activations/layer16_attention_weight_min": -31.651945114135742,
      "activations/layer17_attention_weight_max": 53.43864822387695,
      "activations/layer17_attention_weight_min": -47.14912414550781,
      "activations/layer18_attention_weight_max": 48.82805252075195,
      "activations/layer18_attention_weight_min": -39.59186553955078,
      "activations/layer19_attention_weight_max": 25.12548065185547,
      "activations/layer19_attention_weight_min": -22.159305572509766,
      "activations/layer1_attention_weight_max": 16.65542221069336,
      "activations/layer1_attention_weight_min": -16.234695434570312,
      "activations/layer20_attention_weight_max": 25.009004592895508,
      "activations/layer20_attention_weight_min": -24.24242401123047,
      "activations/layer21_attention_weight_max": 41.7642822265625,
      "activations/layer21_attention_weight_min": -23.656951904296875,
      "activations/layer22_attention_weight_max": 31.74921226501465,
      "activations/layer22_attention_weight_min": -27.829357147216797,
      "activations/layer23_attention_weight_max": 38.415279388427734,
      "activations/layer23_attention_weight_min": -26.349172592163086,
      "activations/layer2_attention_weight_max": 32.160972595214844,
      "activations/layer2_attention_weight_min": -29.965255737304688,
      "activations/layer3_attention_weight_max": 94.12377166748047,
      "activations/layer3_attention_weight_min": -91.23738098144531,
      "activations/layer4_attention_weight_max": 87.35684967041016,
      "activations/layer4_attention_weight_min": -86.40604400634766,
      "activations/layer5_attention_weight_max": 65.10285186767578,
      "activations/layer5_attention_weight_min": -74.9732437133789,
      "activations/layer6_attention_weight_max": 51.49943542480469,
      "activations/layer6_attention_weight_min": -49.28664779663086,
      "activations/layer7_attention_weight_max": 60.619651794433594,
      "activations/layer7_attention_weight_min": -58.3203125,
      "activations/layer8_attention_weight_max": 50.931095123291016,
      "activations/layer8_attention_weight_min": -48.45893859863281,
      "activations/layer9_attention_weight_max": 44.79711151123047,
      "activations/layer9_attention_weight_min": -43.776763916015625,
      "epoch": 15.5,
      "learning_rate": 5.053787878787878e-05,
      "loss": 2.7519,
      "step": 266800
    },
    {
      "activations/layer0_attention_weight_max": 15.546042442321777,
      "activations/layer0_attention_weight_min": -13.312824249267578,
      "activations/layer10_attention_weight_max": 38.18562698364258,
      "activations/layer10_attention_weight_min": -38.309478759765625,
      "activations/layer11_attention_weight_max": 35.2381477355957,
      "activations/layer11_attention_weight_min": -36.14795684814453,
      "activations/layer12_attention_weight_max": 20.26986312866211,
      "activations/layer12_attention_weight_min": -24.52835464477539,
      "activations/layer13_attention_weight_max": 36.91041946411133,
      "activations/layer13_attention_weight_min": -27.059104919433594,
      "activations/layer14_attention_weight_max": 42.991641998291016,
      "activations/layer14_attention_weight_min": -31.85215187072754,
      "activations/layer15_attention_weight_max": 37.701744079589844,
      "activations/layer15_attention_weight_min": -30.865489959716797,
      "activations/layer16_attention_weight_max": 39.019187927246094,
      "activations/layer16_attention_weight_min": -34.498626708984375,
      "activations/layer17_attention_weight_max": 57.09027862548828,
      "activations/layer17_attention_weight_min": -48.04037857055664,
      "activations/layer18_attention_weight_max": 51.53856658935547,
      "activations/layer18_attention_weight_min": -40.29237747192383,
      "activations/layer19_attention_weight_max": 24.686878204345703,
      "activations/layer19_attention_weight_min": -23.659961700439453,
      "activations/layer1_attention_weight_max": 19.135835647583008,
      "activations/layer1_attention_weight_min": -14.690336227416992,
      "activations/layer20_attention_weight_max": 24.080570220947266,
      "activations/layer20_attention_weight_min": -20.924108505249023,
      "activations/layer21_attention_weight_max": 35.52455520629883,
      "activations/layer21_attention_weight_min": -19.97490882873535,
      "activations/layer22_attention_weight_max": 29.40404510498047,
      "activations/layer22_attention_weight_min": -24.49190902709961,
      "activations/layer23_attention_weight_max": 37.47659683227539,
      "activations/layer23_attention_weight_min": -23.02246856689453,
      "activations/layer2_attention_weight_max": 29.733623504638672,
      "activations/layer2_attention_weight_min": -28.40729522705078,
      "activations/layer3_attention_weight_max": 85.80581665039062,
      "activations/layer3_attention_weight_min": -89.65778350830078,
      "activations/layer4_attention_weight_max": 90.9604263305664,
      "activations/layer4_attention_weight_min": -91.2403793334961,
      "activations/layer5_attention_weight_max": 70.40592956542969,
      "activations/layer5_attention_weight_min": -74.34769439697266,
      "activations/layer6_attention_weight_max": 50.71614074707031,
      "activations/layer6_attention_weight_min": -53.27000045776367,
      "activations/layer7_attention_weight_max": 64.78389739990234,
      "activations/layer7_attention_weight_min": -65.80912017822266,
      "activations/layer8_attention_weight_max": 52.69913864135742,
      "activations/layer8_attention_weight_min": -54.639163970947266,
      "activations/layer9_attention_weight_max": 51.4302978515625,
      "activations/layer9_attention_weight_min": -49.60105895996094,
      "epoch": 15.51,
      "learning_rate": 5.051893939393939e-05,
      "loss": 2.7457,
      "step": 266850
    },
    {
      "activations/layer0_attention_weight_max": 16.833425521850586,
      "activations/layer0_attention_weight_min": -13.3545560836792,
      "activations/layer10_attention_weight_max": 37.320404052734375,
      "activations/layer10_attention_weight_min": -36.13242721557617,
      "activations/layer11_attention_weight_max": 33.007957458496094,
      "activations/layer11_attention_weight_min": -33.35091781616211,
      "activations/layer12_attention_weight_max": 18.58016014099121,
      "activations/layer12_attention_weight_min": -25.698928833007812,
      "activations/layer13_attention_weight_max": 36.00409698486328,
      "activations/layer13_attention_weight_min": -28.818300247192383,
      "activations/layer14_attention_weight_max": 37.51443099975586,
      "activations/layer14_attention_weight_min": -30.679262161254883,
      "activations/layer15_attention_weight_max": 32.97428512573242,
      "activations/layer15_attention_weight_min": -30.719636917114258,
      "activations/layer16_attention_weight_max": 33.51725769042969,
      "activations/layer16_attention_weight_min": -31.08219337463379,
      "activations/layer17_attention_weight_max": 57.5098876953125,
      "activations/layer17_attention_weight_min": -46.052040100097656,
      "activations/layer18_attention_weight_max": 45.457096099853516,
      "activations/layer18_attention_weight_min": -40.70277404785156,
      "activations/layer19_attention_weight_max": 21.55116844177246,
      "activations/layer19_attention_weight_min": -21.445735931396484,
      "activations/layer1_attention_weight_max": 17.745119094848633,
      "activations/layer1_attention_weight_min": -14.847492218017578,
      "activations/layer20_attention_weight_max": 19.780820846557617,
      "activations/layer20_attention_weight_min": -22.72202491760254,
      "activations/layer21_attention_weight_max": 30.97787094116211,
      "activations/layer21_attention_weight_min": -20.498605728149414,
      "activations/layer22_attention_weight_max": 28.685152053833008,
      "activations/layer22_attention_weight_min": -25.324216842651367,
      "activations/layer23_attention_weight_max": 34.86201095581055,
      "activations/layer23_attention_weight_min": -23.172760009765625,
      "activations/layer2_attention_weight_max": 31.172277450561523,
      "activations/layer2_attention_weight_min": -29.77822494506836,
      "activations/layer3_attention_weight_max": 87.28530883789062,
      "activations/layer3_attention_weight_min": -89.69772338867188,
      "activations/layer4_attention_weight_max": 89.0869369506836,
      "activations/layer4_attention_weight_min": -86.8570785522461,
      "activations/layer5_attention_weight_max": 67.39344787597656,
      "activations/layer5_attention_weight_min": -72.42356872558594,
      "activations/layer6_attention_weight_max": 53.71974563598633,
      "activations/layer6_attention_weight_min": -49.74079513549805,
      "activations/layer7_attention_weight_max": 60.9703369140625,
      "activations/layer7_attention_weight_min": -58.62541580200195,
      "activations/layer8_attention_weight_max": 47.51396560668945,
      "activations/layer8_attention_weight_min": -47.77191925048828,
      "activations/layer9_attention_weight_max": 46.182491302490234,
      "activations/layer9_attention_weight_min": -46.289791107177734,
      "epoch": 15.51,
      "learning_rate": 5.0499999999999994e-05,
      "loss": 2.7537,
      "step": 266900
    },
    {
      "activations/layer0_attention_weight_max": 15.350428581237793,
      "activations/layer0_attention_weight_min": -13.339137077331543,
      "activations/layer10_attention_weight_max": 35.326385498046875,
      "activations/layer10_attention_weight_min": -33.9949836730957,
      "activations/layer11_attention_weight_max": 29.822105407714844,
      "activations/layer11_attention_weight_min": -30.693273544311523,
      "activations/layer12_attention_weight_max": 17.976661682128906,
      "activations/layer12_attention_weight_min": -24.37508773803711,
      "activations/layer13_attention_weight_max": 35.56019592285156,
      "activations/layer13_attention_weight_min": -28.79848289489746,
      "activations/layer14_attention_weight_max": 36.23094177246094,
      "activations/layer14_attention_weight_min": -30.738014221191406,
      "activations/layer15_attention_weight_max": 33.95563507080078,
      "activations/layer15_attention_weight_min": -30.981304168701172,
      "activations/layer16_attention_weight_max": 33.73014450073242,
      "activations/layer16_attention_weight_min": -31.3651123046875,
      "activations/layer17_attention_weight_max": 53.1414794921875,
      "activations/layer17_attention_weight_min": -44.554439544677734,
      "activations/layer18_attention_weight_max": 45.64252853393555,
      "activations/layer18_attention_weight_min": -38.066925048828125,
      "activations/layer19_attention_weight_max": 24.46869468688965,
      "activations/layer19_attention_weight_min": -23.78157615661621,
      "activations/layer1_attention_weight_max": 17.65765953063965,
      "activations/layer1_attention_weight_min": -15.264599800109863,
      "activations/layer20_attention_weight_max": 23.64716911315918,
      "activations/layer20_attention_weight_min": -22.21209716796875,
      "activations/layer21_attention_weight_max": 33.951839447021484,
      "activations/layer21_attention_weight_min": -27.530920028686523,
      "activations/layer22_attention_weight_max": 28.86406135559082,
      "activations/layer22_attention_weight_min": -24.785383224487305,
      "activations/layer23_attention_weight_max": 37.15549850463867,
      "activations/layer23_attention_weight_min": -24.571636199951172,
      "activations/layer2_attention_weight_max": 30.161123275756836,
      "activations/layer2_attention_weight_min": -28.617895126342773,
      "activations/layer3_attention_weight_max": 84.90921783447266,
      "activations/layer3_attention_weight_min": -86.84500122070312,
      "activations/layer4_attention_weight_max": 85.93961334228516,
      "activations/layer4_attention_weight_min": -84.33080291748047,
      "activations/layer5_attention_weight_max": 64.3239974975586,
      "activations/layer5_attention_weight_min": -76.38710021972656,
      "activations/layer6_attention_weight_max": 47.68280792236328,
      "activations/layer6_attention_weight_min": -49.86244201660156,
      "activations/layer7_attention_weight_max": 57.316566467285156,
      "activations/layer7_attention_weight_min": -54.153743743896484,
      "activations/layer8_attention_weight_max": 42.041481018066406,
      "activations/layer8_attention_weight_min": -48.436126708984375,
      "activations/layer9_attention_weight_max": 41.34009552001953,
      "activations/layer9_attention_weight_min": -43.29952621459961,
      "epoch": 15.51,
      "learning_rate": 5.04810606060606e-05,
      "loss": 2.7509,
      "step": 266950
    },
    {
      "activations/layer0_attention_weight_max": 16.408573150634766,
      "activations/layer0_attention_weight_min": -13.331835746765137,
      "activations/layer10_attention_weight_max": 36.29655075073242,
      "activations/layer10_attention_weight_min": -34.00468444824219,
      "activations/layer11_attention_weight_max": 33.22074890136719,
      "activations/layer11_attention_weight_min": -33.39171600341797,
      "activations/layer12_attention_weight_max": 17.4842472076416,
      "activations/layer12_attention_weight_min": -29.355838775634766,
      "activations/layer13_attention_weight_max": 33.43012237548828,
      "activations/layer13_attention_weight_min": -27.250524520874023,
      "activations/layer14_attention_weight_max": 33.489906311035156,
      "activations/layer14_attention_weight_min": -29.089313507080078,
      "activations/layer15_attention_weight_max": 32.91645812988281,
      "activations/layer15_attention_weight_min": -29.101974487304688,
      "activations/layer16_attention_weight_max": 31.964807510375977,
      "activations/layer16_attention_weight_min": -29.199440002441406,
      "activations/layer17_attention_weight_max": 50.40480041503906,
      "activations/layer17_attention_weight_min": -41.6981201171875,
      "activations/layer18_attention_weight_max": 44.70342254638672,
      "activations/layer18_attention_weight_min": -38.31185531616211,
      "activations/layer19_attention_weight_max": 24.33173942565918,
      "activations/layer19_attention_weight_min": -22.488059997558594,
      "activations/layer1_attention_weight_max": 16.847726821899414,
      "activations/layer1_attention_weight_min": -16.074481964111328,
      "activations/layer20_attention_weight_max": 24.844892501831055,
      "activations/layer20_attention_weight_min": -21.4337158203125,
      "activations/layer21_attention_weight_max": 42.670928955078125,
      "activations/layer21_attention_weight_min": -22.600154876708984,
      "activations/layer22_attention_weight_max": 30.107345581054688,
      "activations/layer22_attention_weight_min": -25.879257202148438,
      "activations/layer23_attention_weight_max": 36.86796569824219,
      "activations/layer23_attention_weight_min": -25.601247787475586,
      "activations/layer2_attention_weight_max": 30.961637496948242,
      "activations/layer2_attention_weight_min": -29.842388153076172,
      "activations/layer3_attention_weight_max": 85.0719223022461,
      "activations/layer3_attention_weight_min": -92.50010681152344,
      "activations/layer4_attention_weight_max": 86.07139587402344,
      "activations/layer4_attention_weight_min": -88.00667572021484,
      "activations/layer5_attention_weight_max": 65.72522735595703,
      "activations/layer5_attention_weight_min": -74.53520965576172,
      "activations/layer6_attention_weight_max": 50.18543243408203,
      "activations/layer6_attention_weight_min": -50.88289260864258,
      "activations/layer7_attention_weight_max": 58.644527435302734,
      "activations/layer7_attention_weight_min": -59.354679107666016,
      "activations/layer8_attention_weight_max": 44.22786331176758,
      "activations/layer8_attention_weight_min": -47.75179672241211,
      "activations/layer9_attention_weight_max": 43.082672119140625,
      "activations/layer9_attention_weight_min": -47.03933334350586,
      "epoch": 15.51,
      "learning_rate": 5.0462121212121204e-05,
      "loss": 2.7387,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.4981,
      "eval_samples_per_second": 505.292,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.4981,
      "eval_openwebtext_samples_per_second": 505.292,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9414,
      "eval_wikitext_samples_per_second": 234.881,
      "step": 267000
    },
    {
      "epoch": 15.51,
      "eval_lambada_loss": 2.478515625,
      "eval_lambada_ppl": 11.923552252478798,
      "eval_lambada_runtime": 9.5538,
      "eval_lambada_samples_per_second": 509.643,
      "step": 267000
    },
    {
      "activations/layer0_attention_weight_max": 15.7584867477417,
      "activations/layer0_attention_weight_min": -13.328816413879395,
      "activations/layer10_attention_weight_max": 34.72300720214844,
      "activations/layer10_attention_weight_min": -33.85340881347656,
      "activations/layer11_attention_weight_max": 30.106040954589844,
      "activations/layer11_attention_weight_min": -31.030471801757812,
      "activations/layer12_attention_weight_max": 17.93501091003418,
      "activations/layer12_attention_weight_min": -25.58993911743164,
      "activations/layer13_attention_weight_max": 34.72623062133789,
      "activations/layer13_attention_weight_min": -30.48247718811035,
      "activations/layer14_attention_weight_max": 32.46015930175781,
      "activations/layer14_attention_weight_min": -29.622976303100586,
      "activations/layer15_attention_weight_max": 31.602439880371094,
      "activations/layer15_attention_weight_min": -29.487266540527344,
      "activations/layer16_attention_weight_max": 32.30030059814453,
      "activations/layer16_attention_weight_min": -30.211034774780273,
      "activations/layer17_attention_weight_max": 51.12727355957031,
      "activations/layer17_attention_weight_min": -42.532840728759766,
      "activations/layer18_attention_weight_max": 44.528114318847656,
      "activations/layer18_attention_weight_min": -36.95259094238281,
      "activations/layer19_attention_weight_max": 26.590757369995117,
      "activations/layer19_attention_weight_min": -22.136194229125977,
      "activations/layer1_attention_weight_max": 15.9985933303833,
      "activations/layer1_attention_weight_min": -14.669537544250488,
      "activations/layer20_attention_weight_max": 22.408721923828125,
      "activations/layer20_attention_weight_min": -23.249361038208008,
      "activations/layer21_attention_weight_max": 35.49286651611328,
      "activations/layer21_attention_weight_min": -25.69663429260254,
      "activations/layer22_attention_weight_max": 33.11874771118164,
      "activations/layer22_attention_weight_min": -25.77853012084961,
      "activations/layer23_attention_weight_max": 37.64356231689453,
      "activations/layer23_attention_weight_min": -24.65741729736328,
      "activations/layer2_attention_weight_max": 29.086212158203125,
      "activations/layer2_attention_weight_min": -27.98664093017578,
      "activations/layer3_attention_weight_max": 83.49917602539062,
      "activations/layer3_attention_weight_min": -86.1230697631836,
      "activations/layer4_attention_weight_max": 86.86094665527344,
      "activations/layer4_attention_weight_min": -84.66547393798828,
      "activations/layer5_attention_weight_max": 69.51813507080078,
      "activations/layer5_attention_weight_min": -78.64707946777344,
      "activations/layer6_attention_weight_max": 49.16169738769531,
      "activations/layer6_attention_weight_min": -49.456783294677734,
      "activations/layer7_attention_weight_max": 59.95186233520508,
      "activations/layer7_attention_weight_min": -59.155555725097656,
      "activations/layer8_attention_weight_max": 45.22861862182617,
      "activations/layer8_attention_weight_min": -47.0617561340332,
      "activations/layer9_attention_weight_max": 46.005836486816406,
      "activations/layer9_attention_weight_min": -43.523109436035156,
      "epoch": 15.52,
      "learning_rate": 5.044318181818182e-05,
      "loss": 2.7597,
      "step": 267050
    },
    {
      "activations/layer0_attention_weight_max": 15.234333992004395,
      "activations/layer0_attention_weight_min": -13.34406852722168,
      "activations/layer10_attention_weight_max": 39.242332458496094,
      "activations/layer10_attention_weight_min": -36.72117614746094,
      "activations/layer11_attention_weight_max": 34.987335205078125,
      "activations/layer11_attention_weight_min": -34.23382568359375,
      "activations/layer12_attention_weight_max": 19.34238052368164,
      "activations/layer12_attention_weight_min": -24.21404457092285,
      "activations/layer13_attention_weight_max": 35.56132507324219,
      "activations/layer13_attention_weight_min": -28.568655014038086,
      "activations/layer14_attention_weight_max": 36.337459564208984,
      "activations/layer14_attention_weight_min": -32.50453567504883,
      "activations/layer15_attention_weight_max": 32.44186782836914,
      "activations/layer15_attention_weight_min": -32.26857376098633,
      "activations/layer16_attention_weight_max": 33.63531494140625,
      "activations/layer16_attention_weight_min": -31.748506546020508,
      "activations/layer17_attention_weight_max": 55.869293212890625,
      "activations/layer17_attention_weight_min": -49.53598403930664,
      "activations/layer18_attention_weight_max": 47.23616409301758,
      "activations/layer18_attention_weight_min": -39.686134338378906,
      "activations/layer19_attention_weight_max": 23.465314865112305,
      "activations/layer19_attention_weight_min": -24.101415634155273,
      "activations/layer1_attention_weight_max": 16.280771255493164,
      "activations/layer1_attention_weight_min": -14.245630264282227,
      "activations/layer20_attention_weight_max": 22.362089157104492,
      "activations/layer20_attention_weight_min": -28.220796585083008,
      "activations/layer21_attention_weight_max": 33.73646926879883,
      "activations/layer21_attention_weight_min": -23.88594627380371,
      "activations/layer22_attention_weight_max": 26.232271194458008,
      "activations/layer22_attention_weight_min": -27.966569900512695,
      "activations/layer23_attention_weight_max": 33.031517028808594,
      "activations/layer23_attention_weight_min": -25.161605834960938,
      "activations/layer2_attention_weight_max": 30.691543579101562,
      "activations/layer2_attention_weight_min": -28.527206420898438,
      "activations/layer3_attention_weight_max": 86.9645767211914,
      "activations/layer3_attention_weight_min": -88.27249908447266,
      "activations/layer4_attention_weight_max": 87.25906372070312,
      "activations/layer4_attention_weight_min": -83.8868408203125,
      "activations/layer5_attention_weight_max": 65.18196868896484,
      "activations/layer5_attention_weight_min": -73.4314956665039,
      "activations/layer6_attention_weight_max": 48.87418746948242,
      "activations/layer6_attention_weight_min": -49.19334411621094,
      "activations/layer7_attention_weight_max": 59.46856689453125,
      "activations/layer7_attention_weight_min": -55.73069381713867,
      "activations/layer8_attention_weight_max": 47.2462158203125,
      "activations/layer8_attention_weight_min": -50.091468811035156,
      "activations/layer9_attention_weight_max": 45.4325065612793,
      "activations/layer9_attention_weight_min": -46.49308776855469,
      "epoch": 15.52,
      "learning_rate": 5.042424242424242e-05,
      "loss": 2.7536,
      "step": 267100
    },
    {
      "activations/layer0_attention_weight_max": 15.970344543457031,
      "activations/layer0_attention_weight_min": -13.793156623840332,
      "activations/layer10_attention_weight_max": 35.41543960571289,
      "activations/layer10_attention_weight_min": -35.40435791015625,
      "activations/layer11_attention_weight_max": 31.89932632446289,
      "activations/layer11_attention_weight_min": -33.187721252441406,
      "activations/layer12_attention_weight_max": 18.802560806274414,
      "activations/layer12_attention_weight_min": -25.068601608276367,
      "activations/layer13_attention_weight_max": 29.603050231933594,
      "activations/layer13_attention_weight_min": -28.23895835876465,
      "activations/layer14_attention_weight_max": 35.0162239074707,
      "activations/layer14_attention_weight_min": -31.871047973632812,
      "activations/layer15_attention_weight_max": 31.418664932250977,
      "activations/layer15_attention_weight_min": -30.575002670288086,
      "activations/layer16_attention_weight_max": 31.58055877685547,
      "activations/layer16_attention_weight_min": -30.377822875976562,
      "activations/layer17_attention_weight_max": 48.8509635925293,
      "activations/layer17_attention_weight_min": -42.56709671020508,
      "activations/layer18_attention_weight_max": 42.46273422241211,
      "activations/layer18_attention_weight_min": -37.65464782714844,
      "activations/layer19_attention_weight_max": 21.7345027923584,
      "activations/layer19_attention_weight_min": -22.018877029418945,
      "activations/layer1_attention_weight_max": 16.54690170288086,
      "activations/layer1_attention_weight_min": -14.883085250854492,
      "activations/layer20_attention_weight_max": 20.863187789916992,
      "activations/layer20_attention_weight_min": -25.045896530151367,
      "activations/layer21_attention_weight_max": 33.60393142700195,
      "activations/layer21_attention_weight_min": -22.386009216308594,
      "activations/layer22_attention_weight_max": 26.32282257080078,
      "activations/layer22_attention_weight_min": -25.4495906829834,
      "activations/layer23_attention_weight_max": 35.78852844238281,
      "activations/layer23_attention_weight_min": -25.93830680847168,
      "activations/layer2_attention_weight_max": 30.26312255859375,
      "activations/layer2_attention_weight_min": -29.118419647216797,
      "activations/layer3_attention_weight_max": 89.1707992553711,
      "activations/layer3_attention_weight_min": -84.24897766113281,
      "activations/layer4_attention_weight_max": 91.1051254272461,
      "activations/layer4_attention_weight_min": -84.12654876708984,
      "activations/layer5_attention_weight_max": 65.69149780273438,
      "activations/layer5_attention_weight_min": -76.13390350341797,
      "activations/layer6_attention_weight_max": 52.07057571411133,
      "activations/layer6_attention_weight_min": -50.40760803222656,
      "activations/layer7_attention_weight_max": 60.29938507080078,
      "activations/layer7_attention_weight_min": -61.255558013916016,
      "activations/layer8_attention_weight_max": 46.506649017333984,
      "activations/layer8_attention_weight_min": -49.084041595458984,
      "activations/layer9_attention_weight_max": 42.37763595581055,
      "activations/layer9_attention_weight_min": -46.13300704956055,
      "epoch": 15.52,
      "learning_rate": 5.040568181818181e-05,
      "loss": 2.7529,
      "step": 267150
    },
    {
      "activations/layer0_attention_weight_max": 16.59271812438965,
      "activations/layer0_attention_weight_min": -13.20033073425293,
      "activations/layer10_attention_weight_max": 39.28351593017578,
      "activations/layer10_attention_weight_min": -36.621246337890625,
      "activations/layer11_attention_weight_max": 35.109466552734375,
      "activations/layer11_attention_weight_min": -36.78903579711914,
      "activations/layer12_attention_weight_max": 20.37732696533203,
      "activations/layer12_attention_weight_min": -24.277118682861328,
      "activations/layer13_attention_weight_max": 37.048789978027344,
      "activations/layer13_attention_weight_min": -27.917285919189453,
      "activations/layer14_attention_weight_max": 34.74951171875,
      "activations/layer14_attention_weight_min": -30.71056365966797,
      "activations/layer15_attention_weight_max": 31.30319595336914,
      "activations/layer15_attention_weight_min": -31.739965438842773,
      "activations/layer16_attention_weight_max": 34.60527801513672,
      "activations/layer16_attention_weight_min": -33.22412109375,
      "activations/layer17_attention_weight_max": 53.69693374633789,
      "activations/layer17_attention_weight_min": -47.682621002197266,
      "activations/layer18_attention_weight_max": 47.41487121582031,
      "activations/layer18_attention_weight_min": -43.090858459472656,
      "activations/layer19_attention_weight_max": 25.56780242919922,
      "activations/layer19_attention_weight_min": -23.848875045776367,
      "activations/layer1_attention_weight_max": 16.31717300415039,
      "activations/layer1_attention_weight_min": -14.707372665405273,
      "activations/layer20_attention_weight_max": 21.233383178710938,
      "activations/layer20_attention_weight_min": -23.659027099609375,
      "activations/layer21_attention_weight_max": 34.12440872192383,
      "activations/layer21_attention_weight_min": -24.181705474853516,
      "activations/layer22_attention_weight_max": 31.08889389038086,
      "activations/layer22_attention_weight_min": -28.16029167175293,
      "activations/layer23_attention_weight_max": 35.757713317871094,
      "activations/layer23_attention_weight_min": -25.079265594482422,
      "activations/layer2_attention_weight_max": 29.93019676208496,
      "activations/layer2_attention_weight_min": -29.788436889648438,
      "activations/layer3_attention_weight_max": 84.5872573852539,
      "activations/layer3_attention_weight_min": -86.74581909179688,
      "activations/layer4_attention_weight_max": 90.86934661865234,
      "activations/layer4_attention_weight_min": -87.76030731201172,
      "activations/layer5_attention_weight_max": 71.05857849121094,
      "activations/layer5_attention_weight_min": -75.250244140625,
      "activations/layer6_attention_weight_max": 52.13973617553711,
      "activations/layer6_attention_weight_min": -49.9467658996582,
      "activations/layer7_attention_weight_max": 60.1955680847168,
      "activations/layer7_attention_weight_min": -60.92353057861328,
      "activations/layer8_attention_weight_max": 45.25188064575195,
      "activations/layer8_attention_weight_min": -49.13880920410156,
      "activations/layer9_attention_weight_max": 44.448604583740234,
      "activations/layer9_attention_weight_min": -45.53046798706055,
      "epoch": 15.53,
      "learning_rate": 5.0386742424242426e-05,
      "loss": 2.7651,
      "step": 267200
    },
    {
      "activations/layer0_attention_weight_max": 15.738845825195312,
      "activations/layer0_attention_weight_min": -12.92110538482666,
      "activations/layer10_attention_weight_max": 34.031314849853516,
      "activations/layer10_attention_weight_min": -34.670997619628906,
      "activations/layer11_attention_weight_max": 31.708152770996094,
      "activations/layer11_attention_weight_min": -33.04631042480469,
      "activations/layer12_attention_weight_max": 19.009748458862305,
      "activations/layer12_attention_weight_min": -23.003141403198242,
      "activations/layer13_attention_weight_max": 31.374134063720703,
      "activations/layer13_attention_weight_min": -26.36308479309082,
      "activations/layer14_attention_weight_max": 35.84901428222656,
      "activations/layer14_attention_weight_min": -30.09031105041504,
      "activations/layer15_attention_weight_max": 33.678810119628906,
      "activations/layer15_attention_weight_min": -30.944969177246094,
      "activations/layer16_attention_weight_max": 34.40812301635742,
      "activations/layer16_attention_weight_min": -30.82806396484375,
      "activations/layer17_attention_weight_max": 51.826969146728516,
      "activations/layer17_attention_weight_min": -46.12815856933594,
      "activations/layer18_attention_weight_max": 48.99290466308594,
      "activations/layer18_attention_weight_min": -40.42628479003906,
      "activations/layer19_attention_weight_max": 23.3042049407959,
      "activations/layer19_attention_weight_min": -24.03022575378418,
      "activations/layer1_attention_weight_max": 16.011320114135742,
      "activations/layer1_attention_weight_min": -16.31617546081543,
      "activations/layer20_attention_weight_max": 23.12517547607422,
      "activations/layer20_attention_weight_min": -24.026939392089844,
      "activations/layer21_attention_weight_max": 35.42113494873047,
      "activations/layer21_attention_weight_min": -26.506816864013672,
      "activations/layer22_attention_weight_max": 29.849193572998047,
      "activations/layer22_attention_weight_min": -28.230337142944336,
      "activations/layer23_attention_weight_max": 39.12343215942383,
      "activations/layer23_attention_weight_min": -26.15810203552246,
      "activations/layer2_attention_weight_max": 29.327054977416992,
      "activations/layer2_attention_weight_min": -28.20833396911621,
      "activations/layer3_attention_weight_max": 83.29480743408203,
      "activations/layer3_attention_weight_min": -84.68704223632812,
      "activations/layer4_attention_weight_max": 86.5186767578125,
      "activations/layer4_attention_weight_min": -83.94567108154297,
      "activations/layer5_attention_weight_max": 63.45838928222656,
      "activations/layer5_attention_weight_min": -73.26828002929688,
      "activations/layer6_attention_weight_max": 49.79676818847656,
      "activations/layer6_attention_weight_min": -47.90309524536133,
      "activations/layer7_attention_weight_max": 63.036441802978516,
      "activations/layer7_attention_weight_min": -56.846473693847656,
      "activations/layer8_attention_weight_max": 42.10121536254883,
      "activations/layer8_attention_weight_min": -44.84869384765625,
      "activations/layer9_attention_weight_max": 43.31270980834961,
      "activations/layer9_attention_weight_min": -46.55729293823242,
      "epoch": 15.53,
      "learning_rate": 5.036780303030303e-05,
      "loss": 2.7525,
      "step": 267250
    },
    {
      "activations/layer0_attention_weight_max": 15.680020332336426,
      "activations/layer0_attention_weight_min": -12.89831829071045,
      "activations/layer10_attention_weight_max": 36.9210090637207,
      "activations/layer10_attention_weight_min": -35.202735900878906,
      "activations/layer11_attention_weight_max": 32.305423736572266,
      "activations/layer11_attention_weight_min": -33.33512878417969,
      "activations/layer12_attention_weight_max": 21.06665802001953,
      "activations/layer12_attention_weight_min": -29.830480575561523,
      "activations/layer13_attention_weight_max": 42.461585998535156,
      "activations/layer13_attention_weight_min": -32.38713836669922,
      "activations/layer14_attention_weight_max": 41.408546447753906,
      "activations/layer14_attention_weight_min": -33.4703483581543,
      "activations/layer15_attention_weight_max": 36.97493362426758,
      "activations/layer15_attention_weight_min": -33.1321907043457,
      "activations/layer16_attention_weight_max": 38.21212387084961,
      "activations/layer16_attention_weight_min": -31.254348754882812,
      "activations/layer17_attention_weight_max": 56.55001449584961,
      "activations/layer17_attention_weight_min": -44.878379821777344,
      "activations/layer18_attention_weight_max": 53.27603530883789,
      "activations/layer18_attention_weight_min": -45.089027404785156,
      "activations/layer19_attention_weight_max": 30.144540786743164,
      "activations/layer19_attention_weight_min": -28.346078872680664,
      "activations/layer1_attention_weight_max": 16.119182586669922,
      "activations/layer1_attention_weight_min": -16.699859619140625,
      "activations/layer20_attention_weight_max": 30.3751220703125,
      "activations/layer20_attention_weight_min": -26.419231414794922,
      "activations/layer21_attention_weight_max": 54.125938415527344,
      "activations/layer21_attention_weight_min": -28.642263412475586,
      "activations/layer22_attention_weight_max": 34.56906509399414,
      "activations/layer22_attention_weight_min": -27.45903968811035,
      "activations/layer23_attention_weight_max": 50.65070343017578,
      "activations/layer23_attention_weight_min": -25.783336639404297,
      "activations/layer2_attention_weight_max": 30.05805015563965,
      "activations/layer2_attention_weight_min": -30.417434692382812,
      "activations/layer3_attention_weight_max": 87.41534423828125,
      "activations/layer3_attention_weight_min": -96.1910400390625,
      "activations/layer4_attention_weight_max": 90.35884094238281,
      "activations/layer4_attention_weight_min": -89.3390884399414,
      "activations/layer5_attention_weight_max": 64.87713623046875,
      "activations/layer5_attention_weight_min": -73.107421875,
      "activations/layer6_attention_weight_max": 50.51443862915039,
      "activations/layer6_attention_weight_min": -50.06459426879883,
      "activations/layer7_attention_weight_max": 58.91685485839844,
      "activations/layer7_attention_weight_min": -61.501529693603516,
      "activations/layer8_attention_weight_max": 45.502620697021484,
      "activations/layer8_attention_weight_min": -48.15443420410156,
      "activations/layer9_attention_weight_max": 43.364810943603516,
      "activations/layer9_attention_weight_min": -45.21774673461914,
      "epoch": 15.53,
      "learning_rate": 5.034886363636363e-05,
      "loss": 2.7582,
      "step": 267300
    },
    {
      "activations/layer0_attention_weight_max": 15.58816146850586,
      "activations/layer0_attention_weight_min": -13.182111740112305,
      "activations/layer10_attention_weight_max": 34.109928131103516,
      "activations/layer10_attention_weight_min": -39.10944366455078,
      "activations/layer11_attention_weight_max": 31.977981567382812,
      "activations/layer11_attention_weight_min": -36.14917755126953,
      "activations/layer12_attention_weight_max": 20.93720054626465,
      "activations/layer12_attention_weight_min": -22.754337310791016,
      "activations/layer13_attention_weight_max": 33.435848236083984,
      "activations/layer13_attention_weight_min": -27.74058723449707,
      "activations/layer14_attention_weight_max": 31.238475799560547,
      "activations/layer14_attention_weight_min": -30.41606330871582,
      "activations/layer15_attention_weight_max": 30.06680679321289,
      "activations/layer15_attention_weight_min": -30.251249313354492,
      "activations/layer16_attention_weight_max": 30.87398338317871,
      "activations/layer16_attention_weight_min": -30.248136520385742,
      "activations/layer17_attention_weight_max": 47.531829833984375,
      "activations/layer17_attention_weight_min": -43.67250061035156,
      "activations/layer18_attention_weight_max": 42.051422119140625,
      "activations/layer18_attention_weight_min": -38.30797576904297,
      "activations/layer19_attention_weight_max": 21.667407989501953,
      "activations/layer19_attention_weight_min": -23.369731903076172,
      "activations/layer1_attention_weight_max": 15.943675994873047,
      "activations/layer1_attention_weight_min": -15.902854919433594,
      "activations/layer20_attention_weight_max": 20.849384307861328,
      "activations/layer20_attention_weight_min": -21.391000747680664,
      "activations/layer21_attention_weight_max": 29.25592613220215,
      "activations/layer21_attention_weight_min": -23.3158016204834,
      "activations/layer22_attention_weight_max": 25.97298812866211,
      "activations/layer22_attention_weight_min": -28.475372314453125,
      "activations/layer23_attention_weight_max": 33.97944641113281,
      "activations/layer23_attention_weight_min": -24.605331420898438,
      "activations/layer2_attention_weight_max": 30.69415283203125,
      "activations/layer2_attention_weight_min": -29.97683334350586,
      "activations/layer3_attention_weight_max": 91.1441421508789,
      "activations/layer3_attention_weight_min": -96.04727935791016,
      "activations/layer4_attention_weight_max": 92.01520538330078,
      "activations/layer4_attention_weight_min": -93.42890167236328,
      "activations/layer5_attention_weight_max": 68.11013793945312,
      "activations/layer5_attention_weight_min": -73.59407806396484,
      "activations/layer6_attention_weight_max": 52.52530288696289,
      "activations/layer6_attention_weight_min": -50.330074310302734,
      "activations/layer7_attention_weight_max": 65.08981323242188,
      "activations/layer7_attention_weight_min": -62.000633239746094,
      "activations/layer8_attention_weight_max": 44.01701736450195,
      "activations/layer8_attention_weight_min": -49.086578369140625,
      "activations/layer9_attention_weight_max": 43.51200485229492,
      "activations/layer9_attention_weight_min": -49.56252670288086,
      "epoch": 15.53,
      "learning_rate": 5.032992424242424e-05,
      "loss": 2.7659,
      "step": 267350
    },
    {
      "activations/layer0_attention_weight_max": 16.671524047851562,
      "activations/layer0_attention_weight_min": -12.925850868225098,
      "activations/layer10_attention_weight_max": 35.810237884521484,
      "activations/layer10_attention_weight_min": -35.009151458740234,
      "activations/layer11_attention_weight_max": 30.26441192626953,
      "activations/layer11_attention_weight_min": -33.01653289794922,
      "activations/layer12_attention_weight_max": 18.54037857055664,
      "activations/layer12_attention_weight_min": -25.941011428833008,
      "activations/layer13_attention_weight_max": 30.966354370117188,
      "activations/layer13_attention_weight_min": -30.417739868164062,
      "activations/layer14_attention_weight_max": 35.266319274902344,
      "activations/layer14_attention_weight_min": -32.14262390136719,
      "activations/layer15_attention_weight_max": 32.86454772949219,
      "activations/layer15_attention_weight_min": -30.64670181274414,
      "activations/layer16_attention_weight_max": 33.04881286621094,
      "activations/layer16_attention_weight_min": -32.03251266479492,
      "activations/layer17_attention_weight_max": 49.08943176269531,
      "activations/layer17_attention_weight_min": -45.93684387207031,
      "activations/layer18_attention_weight_max": 44.12115478515625,
      "activations/layer18_attention_weight_min": -40.1051139831543,
      "activations/layer19_attention_weight_max": 23.453161239624023,
      "activations/layer19_attention_weight_min": -22.576906204223633,
      "activations/layer1_attention_weight_max": 16.65045928955078,
      "activations/layer1_attention_weight_min": -15.761750221252441,
      "activations/layer20_attention_weight_max": 23.668643951416016,
      "activations/layer20_attention_weight_min": -22.13890266418457,
      "activations/layer21_attention_weight_max": 36.6497917175293,
      "activations/layer21_attention_weight_min": -26.295528411865234,
      "activations/layer22_attention_weight_max": 26.11190414428711,
      "activations/layer22_attention_weight_min": -27.265607833862305,
      "activations/layer23_attention_weight_max": 32.899375915527344,
      "activations/layer23_attention_weight_min": -25.255661010742188,
      "activations/layer2_attention_weight_max": 30.564990997314453,
      "activations/layer2_attention_weight_min": -30.424930572509766,
      "activations/layer3_attention_weight_max": 89.39913940429688,
      "activations/layer3_attention_weight_min": -90.88087463378906,
      "activations/layer4_attention_weight_max": 88.21277618408203,
      "activations/layer4_attention_weight_min": -86.63776397705078,
      "activations/layer5_attention_weight_max": 65.48771667480469,
      "activations/layer5_attention_weight_min": -75.36927795410156,
      "activations/layer6_attention_weight_max": 48.05885696411133,
      "activations/layer6_attention_weight_min": -53.39621353149414,
      "activations/layer7_attention_weight_max": 58.40103530883789,
      "activations/layer7_attention_weight_min": -57.87681198120117,
      "activations/layer8_attention_weight_max": 46.87379455566406,
      "activations/layer8_attention_weight_min": -47.0537223815918,
      "activations/layer9_attention_weight_max": 42.972591400146484,
      "activations/layer9_attention_weight_min": -43.3672981262207,
      "epoch": 15.54,
      "learning_rate": 5.0310984848484846e-05,
      "loss": 2.7361,
      "step": 267400
    },
    {
      "activations/layer0_attention_weight_max": 14.710417747497559,
      "activations/layer0_attention_weight_min": -13.192879676818848,
      "activations/layer10_attention_weight_max": 37.014556884765625,
      "activations/layer10_attention_weight_min": -37.21565628051758,
      "activations/layer11_attention_weight_max": 33.08673858642578,
      "activations/layer11_attention_weight_min": -35.59284591674805,
      "activations/layer12_attention_weight_max": 19.06563377380371,
      "activations/layer12_attention_weight_min": -27.391633987426758,
      "activations/layer13_attention_weight_max": 34.53902053833008,
      "activations/layer13_attention_weight_min": -32.28186798095703,
      "activations/layer14_attention_weight_max": 36.42800521850586,
      "activations/layer14_attention_weight_min": -33.37659454345703,
      "activations/layer15_attention_weight_max": 34.809814453125,
      "activations/layer15_attention_weight_min": -33.203739166259766,
      "activations/layer16_attention_weight_max": 33.297481536865234,
      "activations/layer16_attention_weight_min": -32.947425842285156,
      "activations/layer17_attention_weight_max": 53.958003997802734,
      "activations/layer17_attention_weight_min": -45.83858108520508,
      "activations/layer18_attention_weight_max": 47.981781005859375,
      "activations/layer18_attention_weight_min": -43.82103729248047,
      "activations/layer19_attention_weight_max": 23.877477645874023,
      "activations/layer19_attention_weight_min": -24.952295303344727,
      "activations/layer1_attention_weight_max": 17.369693756103516,
      "activations/layer1_attention_weight_min": -14.349952697753906,
      "activations/layer20_attention_weight_max": 25.103832244873047,
      "activations/layer20_attention_weight_min": -22.30976676940918,
      "activations/layer21_attention_weight_max": 39.582366943359375,
      "activations/layer21_attention_weight_min": -21.11341094970703,
      "activations/layer22_attention_weight_max": 28.557729721069336,
      "activations/layer22_attention_weight_min": -27.134119033813477,
      "activations/layer23_attention_weight_max": 38.19442367553711,
      "activations/layer23_attention_weight_min": -24.600780487060547,
      "activations/layer2_attention_weight_max": 31.657323837280273,
      "activations/layer2_attention_weight_min": -29.187049865722656,
      "activations/layer3_attention_weight_max": 89.58722686767578,
      "activations/layer3_attention_weight_min": -87.47872161865234,
      "activations/layer4_attention_weight_max": 92.99884033203125,
      "activations/layer4_attention_weight_min": -90.40364837646484,
      "activations/layer5_attention_weight_max": 68.71589660644531,
      "activations/layer5_attention_weight_min": -73.67047119140625,
      "activations/layer6_attention_weight_max": 51.23964309692383,
      "activations/layer6_attention_weight_min": -52.22018814086914,
      "activations/layer7_attention_weight_max": 61.46683883666992,
      "activations/layer7_attention_weight_min": -62.04717254638672,
      "activations/layer8_attention_weight_max": 48.002777099609375,
      "activations/layer8_attention_weight_min": -48.82322692871094,
      "activations/layer9_attention_weight_max": 45.132747650146484,
      "activations/layer9_attention_weight_min": -49.04400634765625,
      "epoch": 15.54,
      "learning_rate": 5.0292045454545454e-05,
      "loss": 2.7477,
      "step": 267450
    },
    {
      "activations/layer0_attention_weight_max": 15.332169532775879,
      "activations/layer0_attention_weight_min": -13.552934646606445,
      "activations/layer10_attention_weight_max": 36.440643310546875,
      "activations/layer10_attention_weight_min": -38.2211799621582,
      "activations/layer11_attention_weight_max": 31.86199188232422,
      "activations/layer11_attention_weight_min": -35.48657989501953,
      "activations/layer12_attention_weight_max": 18.500415802001953,
      "activations/layer12_attention_weight_min": -23.327877044677734,
      "activations/layer13_attention_weight_max": 30.062850952148438,
      "activations/layer13_attention_weight_min": -27.927927017211914,
      "activations/layer14_attention_weight_max": 32.9919548034668,
      "activations/layer14_attention_weight_min": -30.20428466796875,
      "activations/layer15_attention_weight_max": 29.922182083129883,
      "activations/layer15_attention_weight_min": -29.799945831298828,
      "activations/layer16_attention_weight_max": 33.90317153930664,
      "activations/layer16_attention_weight_min": -30.55666160583496,
      "activations/layer17_attention_weight_max": 46.53480529785156,
      "activations/layer17_attention_weight_min": -43.437217712402344,
      "activations/layer18_attention_weight_max": 42.364845275878906,
      "activations/layer18_attention_weight_min": -37.447532653808594,
      "activations/layer19_attention_weight_max": 22.533010482788086,
      "activations/layer19_attention_weight_min": -23.249977111816406,
      "activations/layer1_attention_weight_max": 16.075515747070312,
      "activations/layer1_attention_weight_min": -14.534282684326172,
      "activations/layer20_attention_weight_max": 20.10567283630371,
      "activations/layer20_attention_weight_min": -24.419719696044922,
      "activations/layer21_attention_weight_max": 35.78559112548828,
      "activations/layer21_attention_weight_min": -23.91617774963379,
      "activations/layer22_attention_weight_max": 26.974227905273438,
      "activations/layer22_attention_weight_min": -26.666934967041016,
      "activations/layer23_attention_weight_max": 34.65623474121094,
      "activations/layer23_attention_weight_min": -25.076276779174805,
      "activations/layer2_attention_weight_max": 31.566650390625,
      "activations/layer2_attention_weight_min": -30.1818904876709,
      "activations/layer3_attention_weight_max": 87.8134536743164,
      "activations/layer3_attention_weight_min": -89.01332092285156,
      "activations/layer4_attention_weight_max": 90.5444564819336,
      "activations/layer4_attention_weight_min": -87.00262451171875,
      "activations/layer5_attention_weight_max": 66.92691040039062,
      "activations/layer5_attention_weight_min": -74.62335205078125,
      "activations/layer6_attention_weight_max": 54.19049835205078,
      "activations/layer6_attention_weight_min": -51.503395080566406,
      "activations/layer7_attention_weight_max": 61.14696502685547,
      "activations/layer7_attention_weight_min": -60.14848709106445,
      "activations/layer8_attention_weight_max": 47.047813415527344,
      "activations/layer8_attention_weight_min": -49.96102523803711,
      "activations/layer9_attention_weight_max": 44.331748962402344,
      "activations/layer9_attention_weight_min": -47.17565155029297,
      "epoch": 15.54,
      "learning_rate": 5.0273106060606056e-05,
      "loss": 2.7569,
      "step": 267500
    },
    {
      "activations/layer0_attention_weight_max": 16.69037628173828,
      "activations/layer0_attention_weight_min": -13.524353981018066,
      "activations/layer10_attention_weight_max": 34.70789337158203,
      "activations/layer10_attention_weight_min": -36.6407470703125,
      "activations/layer11_attention_weight_max": 31.05916976928711,
      "activations/layer11_attention_weight_min": -34.84710693359375,
      "activations/layer12_attention_weight_max": 19.53611946105957,
      "activations/layer12_attention_weight_min": -24.503812789916992,
      "activations/layer13_attention_weight_max": 34.47325134277344,
      "activations/layer13_attention_weight_min": -30.98366355895996,
      "activations/layer14_attention_weight_max": 39.00646209716797,
      "activations/layer14_attention_weight_min": -36.061248779296875,
      "activations/layer15_attention_weight_max": 34.088077545166016,
      "activations/layer15_attention_weight_min": -37.26106262207031,
      "activations/layer16_attention_weight_max": 33.42808532714844,
      "activations/layer16_attention_weight_min": -32.08039855957031,
      "activations/layer17_attention_weight_max": 49.972957611083984,
      "activations/layer17_attention_weight_min": -47.428958892822266,
      "activations/layer18_attention_weight_max": 46.111358642578125,
      "activations/layer18_attention_weight_min": -42.12685775756836,
      "activations/layer19_attention_weight_max": 24.698530197143555,
      "activations/layer19_attention_weight_min": -25.765432357788086,
      "activations/layer1_attention_weight_max": 15.48598575592041,
      "activations/layer1_attention_weight_min": -13.40331745147705,
      "activations/layer20_attention_weight_max": 25.298194885253906,
      "activations/layer20_attention_weight_min": -24.4754695892334,
      "activations/layer21_attention_weight_max": 40.87348556518555,
      "activations/layer21_attention_weight_min": -24.81756591796875,
      "activations/layer22_attention_weight_max": 29.686519622802734,
      "activations/layer22_attention_weight_min": -28.414737701416016,
      "activations/layer23_attention_weight_max": 37.56142807006836,
      "activations/layer23_attention_weight_min": -26.96991729736328,
      "activations/layer2_attention_weight_max": 30.1904296875,
      "activations/layer2_attention_weight_min": -28.89179229736328,
      "activations/layer3_attention_weight_max": 89.3960952758789,
      "activations/layer3_attention_weight_min": -89.149169921875,
      "activations/layer4_attention_weight_max": 87.35330200195312,
      "activations/layer4_attention_weight_min": -84.70382690429688,
      "activations/layer5_attention_weight_max": 65.19078063964844,
      "activations/layer5_attention_weight_min": -72.45526123046875,
      "activations/layer6_attention_weight_max": 50.30900955200195,
      "activations/layer6_attention_weight_min": -49.33202362060547,
      "activations/layer7_attention_weight_max": 60.40563201904297,
      "activations/layer7_attention_weight_min": -58.953697204589844,
      "activations/layer8_attention_weight_max": 43.81018829345703,
      "activations/layer8_attention_weight_min": -47.84221267700195,
      "activations/layer9_attention_weight_max": 43.18949508666992,
      "activations/layer9_attention_weight_min": -43.17097091674805,
      "epoch": 15.55,
      "learning_rate": 5.025416666666666e-05,
      "loss": 2.7513,
      "step": 267550
    },
    {
      "activations/layer0_attention_weight_max": 14.9998140335083,
      "activations/layer0_attention_weight_min": -13.560352325439453,
      "activations/layer10_attention_weight_max": 36.978851318359375,
      "activations/layer10_attention_weight_min": -38.4212532043457,
      "activations/layer11_attention_weight_max": 33.18763732910156,
      "activations/layer11_attention_weight_min": -36.205345153808594,
      "activations/layer12_attention_weight_max": 20.17268180847168,
      "activations/layer12_attention_weight_min": -24.251663208007812,
      "activations/layer13_attention_weight_max": 32.536590576171875,
      "activations/layer13_attention_weight_min": -28.296127319335938,
      "activations/layer14_attention_weight_max": 35.25385665893555,
      "activations/layer14_attention_weight_min": -32.23429489135742,
      "activations/layer15_attention_weight_max": 31.7844181060791,
      "activations/layer15_attention_weight_min": -32.42756271362305,
      "activations/layer16_attention_weight_max": 34.50390625,
      "activations/layer16_attention_weight_min": -31.841995239257812,
      "activations/layer17_attention_weight_max": 53.38895797729492,
      "activations/layer17_attention_weight_min": -48.37101364135742,
      "activations/layer18_attention_weight_max": 49.53972244262695,
      "activations/layer18_attention_weight_min": -44.1302375793457,
      "activations/layer19_attention_weight_max": 22.47087287902832,
      "activations/layer19_attention_weight_min": -23.083221435546875,
      "activations/layer1_attention_weight_max": 16.95316505432129,
      "activations/layer1_attention_weight_min": -14.237292289733887,
      "activations/layer20_attention_weight_max": 21.966703414916992,
      "activations/layer20_attention_weight_min": -24.064672470092773,
      "activations/layer21_attention_weight_max": 35.32597732543945,
      "activations/layer21_attention_weight_min": -25.581066131591797,
      "activations/layer22_attention_weight_max": 32.403839111328125,
      "activations/layer22_attention_weight_min": -27.70623779296875,
      "activations/layer23_attention_weight_max": 37.641883850097656,
      "activations/layer23_attention_weight_min": -27.083614349365234,
      "activations/layer2_attention_weight_max": 30.90473747253418,
      "activations/layer2_attention_weight_min": -30.200389862060547,
      "activations/layer3_attention_weight_max": 90.54286193847656,
      "activations/layer3_attention_weight_min": -86.78185272216797,
      "activations/layer4_attention_weight_max": 91.39149475097656,
      "activations/layer4_attention_weight_min": -90.5949935913086,
      "activations/layer5_attention_weight_max": 64.03411102294922,
      "activations/layer5_attention_weight_min": -73.75663757324219,
      "activations/layer6_attention_weight_max": 49.998111724853516,
      "activations/layer6_attention_weight_min": -52.178504943847656,
      "activations/layer7_attention_weight_max": 64.1768798828125,
      "activations/layer7_attention_weight_min": -59.44402313232422,
      "activations/layer8_attention_weight_max": 45.44083786010742,
      "activations/layer8_attention_weight_min": -49.51527786254883,
      "activations/layer9_attention_weight_max": 44.71940231323242,
      "activations/layer9_attention_weight_min": -47.141075134277344,
      "epoch": 15.55,
      "learning_rate": 5.023522727272727e-05,
      "loss": 2.7732,
      "step": 267600
    },
    {
      "activations/layer0_attention_weight_max": 16.2301025390625,
      "activations/layer0_attention_weight_min": -13.680012702941895,
      "activations/layer10_attention_weight_max": 37.14807891845703,
      "activations/layer10_attention_weight_min": -36.54155731201172,
      "activations/layer11_attention_weight_max": 32.868865966796875,
      "activations/layer11_attention_weight_min": -34.63569641113281,
      "activations/layer12_attention_weight_max": 20.153427124023438,
      "activations/layer12_attention_weight_min": -23.782392501831055,
      "activations/layer13_attention_weight_max": 37.16195297241211,
      "activations/layer13_attention_weight_min": -29.2539005279541,
      "activations/layer14_attention_weight_max": 42.149723052978516,
      "activations/layer14_attention_weight_min": -33.077484130859375,
      "activations/layer15_attention_weight_max": 39.417091369628906,
      "activations/layer15_attention_weight_min": -32.274932861328125,
      "activations/layer16_attention_weight_max": 40.12145233154297,
      "activations/layer16_attention_weight_min": -33.34799575805664,
      "activations/layer17_attention_weight_max": 62.1674690246582,
      "activations/layer17_attention_weight_min": -48.96693420410156,
      "activations/layer18_attention_weight_max": 53.18165969848633,
      "activations/layer18_attention_weight_min": -40.99836349487305,
      "activations/layer19_attention_weight_max": 24.2852783203125,
      "activations/layer19_attention_weight_min": -24.27179527282715,
      "activations/layer1_attention_weight_max": 16.366653442382812,
      "activations/layer1_attention_weight_min": -13.378290176391602,
      "activations/layer20_attention_weight_max": 25.120084762573242,
      "activations/layer20_attention_weight_min": -21.916637420654297,
      "activations/layer21_attention_weight_max": 44.54289245605469,
      "activations/layer21_attention_weight_min": -23.357425689697266,
      "activations/layer22_attention_weight_max": 30.583616256713867,
      "activations/layer22_attention_weight_min": -25.18052101135254,
      "activations/layer23_attention_weight_max": 42.25382614135742,
      "activations/layer23_attention_weight_min": -25.372051239013672,
      "activations/layer2_attention_weight_max": 31.240570068359375,
      "activations/layer2_attention_weight_min": -30.002164840698242,
      "activations/layer3_attention_weight_max": 89.95309448242188,
      "activations/layer3_attention_weight_min": -96.2320327758789,
      "activations/layer4_attention_weight_max": 94.13047790527344,
      "activations/layer4_attention_weight_min": -94.26535034179688,
      "activations/layer5_attention_weight_max": 69.66162872314453,
      "activations/layer5_attention_weight_min": -77.65092468261719,
      "activations/layer6_attention_weight_max": 54.04561233520508,
      "activations/layer6_attention_weight_min": -52.9306755065918,
      "activations/layer7_attention_weight_max": 62.49983215332031,
      "activations/layer7_attention_weight_min": -58.3948860168457,
      "activations/layer8_attention_weight_max": 48.81893539428711,
      "activations/layer8_attention_weight_min": -49.01637649536133,
      "activations/layer9_attention_weight_max": 46.352928161621094,
      "activations/layer9_attention_weight_min": -46.93073654174805,
      "epoch": 15.55,
      "learning_rate": 5.0216287878787874e-05,
      "loss": 2.7448,
      "step": 267650
    },
    {
      "activations/layer0_attention_weight_max": 16.76198387145996,
      "activations/layer0_attention_weight_min": -13.516698837280273,
      "activations/layer10_attention_weight_max": 34.88983917236328,
      "activations/layer10_attention_weight_min": -34.94272232055664,
      "activations/layer11_attention_weight_max": 31.028648376464844,
      "activations/layer11_attention_weight_min": -33.9715576171875,
      "activations/layer12_attention_weight_max": 17.707605361938477,
      "activations/layer12_attention_weight_min": -28.359148025512695,
      "activations/layer13_attention_weight_max": 31.968685150146484,
      "activations/layer13_attention_weight_min": -25.17108917236328,
      "activations/layer14_attention_weight_max": 34.10972595214844,
      "activations/layer14_attention_weight_min": -29.374353408813477,
      "activations/layer15_attention_weight_max": 31.221567153930664,
      "activations/layer15_attention_weight_min": -28.87584686279297,
      "activations/layer16_attention_weight_max": 30.97593879699707,
      "activations/layer16_attention_weight_min": -29.380035400390625,
      "activations/layer17_attention_weight_max": 51.22720718383789,
      "activations/layer17_attention_weight_min": -43.03593444824219,
      "activations/layer18_attention_weight_max": 42.574981689453125,
      "activations/layer18_attention_weight_min": -39.57265090942383,
      "activations/layer19_attention_weight_max": 23.49964714050293,
      "activations/layer19_attention_weight_min": -23.56352424621582,
      "activations/layer1_attention_weight_max": 16.329618453979492,
      "activations/layer1_attention_weight_min": -13.68393611907959,
      "activations/layer20_attention_weight_max": 21.061738967895508,
      "activations/layer20_attention_weight_min": -22.13326644897461,
      "activations/layer21_attention_weight_max": 38.56355285644531,
      "activations/layer21_attention_weight_min": -22.89569664001465,
      "activations/layer22_attention_weight_max": 27.746267318725586,
      "activations/layer22_attention_weight_min": -23.69533920288086,
      "activations/layer23_attention_weight_max": 35.57445526123047,
      "activations/layer23_attention_weight_min": -22.25849723815918,
      "activations/layer2_attention_weight_max": 28.880565643310547,
      "activations/layer2_attention_weight_min": -28.450931549072266,
      "activations/layer3_attention_weight_max": 85.78911590576172,
      "activations/layer3_attention_weight_min": -88.06201171875,
      "activations/layer4_attention_weight_max": 87.23866271972656,
      "activations/layer4_attention_weight_min": -88.31705474853516,
      "activations/layer5_attention_weight_max": 63.013145446777344,
      "activations/layer5_attention_weight_min": -75.16065979003906,
      "activations/layer6_attention_weight_max": 50.419681549072266,
      "activations/layer6_attention_weight_min": -49.3984260559082,
      "activations/layer7_attention_weight_max": 60.419063568115234,
      "activations/layer7_attention_weight_min": -59.221580505371094,
      "activations/layer8_attention_weight_max": 43.53561019897461,
      "activations/layer8_attention_weight_min": -48.35637283325195,
      "activations/layer9_attention_weight_max": 42.424747467041016,
      "activations/layer9_attention_weight_min": -45.961605072021484,
      "epoch": 15.55,
      "learning_rate": 5.019734848484848e-05,
      "loss": 2.7498,
      "step": 267700
    },
    {
      "activations/layer0_attention_weight_max": 15.674273490905762,
      "activations/layer0_attention_weight_min": -13.683523178100586,
      "activations/layer10_attention_weight_max": 37.933231353759766,
      "activations/layer10_attention_weight_min": -38.168514251708984,
      "activations/layer11_attention_weight_max": 36.69807815551758,
      "activations/layer11_attention_weight_min": -35.4213752746582,
      "activations/layer12_attention_weight_max": 19.281496047973633,
      "activations/layer12_attention_weight_min": -27.512989044189453,
      "activations/layer13_attention_weight_max": 32.868568420410156,
      "activations/layer13_attention_weight_min": -28.429529190063477,
      "activations/layer14_attention_weight_max": 37.260528564453125,
      "activations/layer14_attention_weight_min": -30.094654083251953,
      "activations/layer15_attention_weight_max": 33.79963684082031,
      "activations/layer15_attention_weight_min": -30.044410705566406,
      "activations/layer16_attention_weight_max": 33.676300048828125,
      "activations/layer16_attention_weight_min": -30.816089630126953,
      "activations/layer17_attention_weight_max": 55.24308776855469,
      "activations/layer17_attention_weight_min": -43.58546447753906,
      "activations/layer18_attention_weight_max": 48.409759521484375,
      "activations/layer18_attention_weight_min": -39.783119201660156,
      "activations/layer19_attention_weight_max": 22.56184959411621,
      "activations/layer19_attention_weight_min": -24.255908966064453,
      "activations/layer1_attention_weight_max": 16.158655166625977,
      "activations/layer1_attention_weight_min": -13.85774040222168,
      "activations/layer20_attention_weight_max": 22.52158546447754,
      "activations/layer20_attention_weight_min": -21.61026954650879,
      "activations/layer21_attention_weight_max": 33.27870559692383,
      "activations/layer21_attention_weight_min": -23.98443031311035,
      "activations/layer22_attention_weight_max": 29.596696853637695,
      "activations/layer22_attention_weight_min": -25.11424446105957,
      "activations/layer23_attention_weight_max": 37.225799560546875,
      "activations/layer23_attention_weight_min": -25.334745407104492,
      "activations/layer2_attention_weight_max": 28.92914581298828,
      "activations/layer2_attention_weight_min": -29.35033416748047,
      "activations/layer3_attention_weight_max": 89.30361938476562,
      "activations/layer3_attention_weight_min": -95.09445190429688,
      "activations/layer4_attention_weight_max": 89.62406158447266,
      "activations/layer4_attention_weight_min": -88.76780700683594,
      "activations/layer5_attention_weight_max": 65.49116516113281,
      "activations/layer5_attention_weight_min": -77.09935760498047,
      "activations/layer6_attention_weight_max": 53.67847442626953,
      "activations/layer6_attention_weight_min": -54.50288009643555,
      "activations/layer7_attention_weight_max": 64.8626480102539,
      "activations/layer7_attention_weight_min": -62.75740432739258,
      "activations/layer8_attention_weight_max": 45.42573165893555,
      "activations/layer8_attention_weight_min": -52.963531494140625,
      "activations/layer9_attention_weight_max": 45.894901275634766,
      "activations/layer9_attention_weight_min": -47.743072509765625,
      "epoch": 15.56,
      "learning_rate": 5.0178409090909083e-05,
      "loss": 2.7598,
      "step": 267750
    },
    {
      "activations/layer0_attention_weight_max": 15.807608604431152,
      "activations/layer0_attention_weight_min": -13.771260261535645,
      "activations/layer10_attention_weight_max": 37.87677764892578,
      "activations/layer10_attention_weight_min": -35.80249786376953,
      "activations/layer11_attention_weight_max": 31.356359481811523,
      "activations/layer11_attention_weight_min": -34.12265396118164,
      "activations/layer12_attention_weight_max": 19.074140548706055,
      "activations/layer12_attention_weight_min": -26.697240829467773,
      "activations/layer13_attention_weight_max": 31.514686584472656,
      "activations/layer13_attention_weight_min": -26.514598846435547,
      "activations/layer14_attention_weight_max": 30.93136978149414,
      "activations/layer14_attention_weight_min": -28.313514709472656,
      "activations/layer15_attention_weight_max": 27.18735122680664,
      "activations/layer15_attention_weight_min": -28.39279556274414,
      "activations/layer16_attention_weight_max": 30.701831817626953,
      "activations/layer16_attention_weight_min": -29.432971954345703,
      "activations/layer17_attention_weight_max": 47.21160125732422,
      "activations/layer17_attention_weight_min": -41.66548156738281,
      "activations/layer18_attention_weight_max": 43.6922721862793,
      "activations/layer18_attention_weight_min": -38.253074645996094,
      "activations/layer19_attention_weight_max": 22.572195053100586,
      "activations/layer19_attention_weight_min": -23.914175033569336,
      "activations/layer1_attention_weight_max": 16.48175811767578,
      "activations/layer1_attention_weight_min": -15.680773735046387,
      "activations/layer20_attention_weight_max": 22.881771087646484,
      "activations/layer20_attention_weight_min": -21.73208236694336,
      "activations/layer21_attention_weight_max": 32.43482208251953,
      "activations/layer21_attention_weight_min": -21.06917381286621,
      "activations/layer22_attention_weight_max": 28.6144962310791,
      "activations/layer22_attention_weight_min": -27.189722061157227,
      "activations/layer23_attention_weight_max": 34.49256134033203,
      "activations/layer23_attention_weight_min": -23.21510124206543,
      "activations/layer2_attention_weight_max": 31.364500045776367,
      "activations/layer2_attention_weight_min": -30.111751556396484,
      "activations/layer3_attention_weight_max": 87.16604614257812,
      "activations/layer3_attention_weight_min": -91.52838897705078,
      "activations/layer4_attention_weight_max": 86.54791259765625,
      "activations/layer4_attention_weight_min": -88.62773895263672,
      "activations/layer5_attention_weight_max": 68.38419342041016,
      "activations/layer5_attention_weight_min": -78.83255004882812,
      "activations/layer6_attention_weight_max": 49.61082458496094,
      "activations/layer6_attention_weight_min": -53.037723541259766,
      "activations/layer7_attention_weight_max": 62.97746658325195,
      "activations/layer7_attention_weight_min": -60.92264175415039,
      "activations/layer8_attention_weight_max": 44.487937927246094,
      "activations/layer8_attention_weight_min": -52.44758224487305,
      "activations/layer9_attention_weight_max": 46.25171661376953,
      "activations/layer9_attention_weight_min": -47.295650482177734,
      "epoch": 15.56,
      "learning_rate": 5.01594696969697e-05,
      "loss": 2.7498,
      "step": 267800
    },
    {
      "activations/layer0_attention_weight_max": 16.007450103759766,
      "activations/layer0_attention_weight_min": -13.628925323486328,
      "activations/layer10_attention_weight_max": 37.94382095336914,
      "activations/layer10_attention_weight_min": -36.62507629394531,
      "activations/layer11_attention_weight_max": 32.76479721069336,
      "activations/layer11_attention_weight_min": -36.76388931274414,
      "activations/layer12_attention_weight_max": 18.50305938720703,
      "activations/layer12_attention_weight_min": -24.745023727416992,
      "activations/layer13_attention_weight_max": 38.26753616333008,
      "activations/layer13_attention_weight_min": -28.472396850585938,
      "activations/layer14_attention_weight_max": 40.57980728149414,
      "activations/layer14_attention_weight_min": -32.71501922607422,
      "activations/layer15_attention_weight_max": 36.93937301635742,
      "activations/layer15_attention_weight_min": -33.519351959228516,
      "activations/layer16_attention_weight_max": 37.16324234008789,
      "activations/layer16_attention_weight_min": -31.98819923400879,
      "activations/layer17_attention_weight_max": 56.98356246948242,
      "activations/layer17_attention_weight_min": -46.203086853027344,
      "activations/layer18_attention_weight_max": 54.99016571044922,
      "activations/layer18_attention_weight_min": -40.76293182373047,
      "activations/layer19_attention_weight_max": 27.008909225463867,
      "activations/layer19_attention_weight_min": -25.188234329223633,
      "activations/layer1_attention_weight_max": 17.248472213745117,
      "activations/layer1_attention_weight_min": -15.178319931030273,
      "activations/layer20_attention_weight_max": 28.828813552856445,
      "activations/layer20_attention_weight_min": -22.29123306274414,
      "activations/layer21_attention_weight_max": 41.79258346557617,
      "activations/layer21_attention_weight_min": -24.678245544433594,
      "activations/layer22_attention_weight_max": 33.11552047729492,
      "activations/layer22_attention_weight_min": -28.554590225219727,
      "activations/layer23_attention_weight_max": 39.82341766357422,
      "activations/layer23_attention_weight_min": -25.078323364257812,
      "activations/layer2_attention_weight_max": 31.102127075195312,
      "activations/layer2_attention_weight_min": -31.880847930908203,
      "activations/layer3_attention_weight_max": 91.72369384765625,
      "activations/layer3_attention_weight_min": -95.84817504882812,
      "activations/layer4_attention_weight_max": 93.49525451660156,
      "activations/layer4_attention_weight_min": -93.3280029296875,
      "activations/layer5_attention_weight_max": 72.17454528808594,
      "activations/layer5_attention_weight_min": -80.29767608642578,
      "activations/layer6_attention_weight_max": 53.991092681884766,
      "activations/layer6_attention_weight_min": -54.33473587036133,
      "activations/layer7_attention_weight_max": 65.2023696899414,
      "activations/layer7_attention_weight_min": -65.03145599365234,
      "activations/layer8_attention_weight_max": 48.39329528808594,
      "activations/layer8_attention_weight_min": -51.4677734375,
      "activations/layer9_attention_weight_max": 47.782474517822266,
      "activations/layer9_attention_weight_min": -49.0822868347168,
      "epoch": 15.56,
      "learning_rate": 5.01405303030303e-05,
      "loss": 2.7535,
      "step": 267850
    },
    {
      "activations/layer0_attention_weight_max": 16.294294357299805,
      "activations/layer0_attention_weight_min": -13.784759521484375,
      "activations/layer10_attention_weight_max": 37.57548522949219,
      "activations/layer10_attention_weight_min": -35.683658599853516,
      "activations/layer11_attention_weight_max": 34.309547424316406,
      "activations/layer11_attention_weight_min": -34.97431182861328,
      "activations/layer12_attention_weight_max": 18.5916690826416,
      "activations/layer12_attention_weight_min": -25.891603469848633,
      "activations/layer13_attention_weight_max": 34.58940887451172,
      "activations/layer13_attention_weight_min": -28.570100784301758,
      "activations/layer14_attention_weight_max": 38.111915588378906,
      "activations/layer14_attention_weight_min": -33.72144317626953,
      "activations/layer15_attention_weight_max": 38.267452239990234,
      "activations/layer15_attention_weight_min": -34.0208854675293,
      "activations/layer16_attention_weight_max": 37.07115936279297,
      "activations/layer16_attention_weight_min": -35.27033615112305,
      "activations/layer17_attention_weight_max": 55.51240921020508,
      "activations/layer17_attention_weight_min": -50.05669021606445,
      "activations/layer18_attention_weight_max": 53.821754455566406,
      "activations/layer18_attention_weight_min": -44.2514533996582,
      "activations/layer19_attention_weight_max": 28.381643295288086,
      "activations/layer19_attention_weight_min": -26.027301788330078,
      "activations/layer1_attention_weight_max": 16.887004852294922,
      "activations/layer1_attention_weight_min": -16.00238609313965,
      "activations/layer20_attention_weight_max": 28.2535400390625,
      "activations/layer20_attention_weight_min": -23.81842613220215,
      "activations/layer21_attention_weight_max": 41.56294631958008,
      "activations/layer21_attention_weight_min": -24.364532470703125,
      "activations/layer22_attention_weight_max": 37.80080795288086,
      "activations/layer22_attention_weight_min": -26.708786010742188,
      "activations/layer23_attention_weight_max": 42.22701644897461,
      "activations/layer23_attention_weight_min": -24.44154167175293,
      "activations/layer2_attention_weight_max": 33.237525939941406,
      "activations/layer2_attention_weight_min": -30.945348739624023,
      "activations/layer3_attention_weight_max": 90.87525177001953,
      "activations/layer3_attention_weight_min": -94.13469696044922,
      "activations/layer4_attention_weight_max": 89.63995361328125,
      "activations/layer4_attention_weight_min": -87.6150131225586,
      "activations/layer5_attention_weight_max": 70.60051727294922,
      "activations/layer5_attention_weight_min": -72.45140838623047,
      "activations/layer6_attention_weight_max": 50.1105842590332,
      "activations/layer6_attention_weight_min": -51.3944206237793,
      "activations/layer7_attention_weight_max": 64.21648406982422,
      "activations/layer7_attention_weight_min": -59.462181091308594,
      "activations/layer8_attention_weight_max": 45.886898040771484,
      "activations/layer8_attention_weight_min": -48.05257034301758,
      "activations/layer9_attention_weight_max": 42.6527214050293,
      "activations/layer9_attention_weight_min": -47.75477981567383,
      "epoch": 15.57,
      "learning_rate": 5.01215909090909e-05,
      "loss": 2.7609,
      "step": 267900
    },
    {
      "activations/layer0_attention_weight_max": 15.735920906066895,
      "activations/layer0_attention_weight_min": -13.987251281738281,
      "activations/layer10_attention_weight_max": 36.560298919677734,
      "activations/layer10_attention_weight_min": -36.133583068847656,
      "activations/layer11_attention_weight_max": 32.63936233520508,
      "activations/layer11_attention_weight_min": -35.644927978515625,
      "activations/layer12_attention_weight_max": 19.26321792602539,
      "activations/layer12_attention_weight_min": -25.124637603759766,
      "activations/layer13_attention_weight_max": 35.88780212402344,
      "activations/layer13_attention_weight_min": -27.910602569580078,
      "activations/layer14_attention_weight_max": 36.04200744628906,
      "activations/layer14_attention_weight_min": -31.594852447509766,
      "activations/layer15_attention_weight_max": 32.20524978637695,
      "activations/layer15_attention_weight_min": -28.183612823486328,
      "activations/layer16_attention_weight_max": 33.012481689453125,
      "activations/layer16_attention_weight_min": -28.5895938873291,
      "activations/layer17_attention_weight_max": 50.9656982421875,
      "activations/layer17_attention_weight_min": -42.390933990478516,
      "activations/layer18_attention_weight_max": 45.0723991394043,
      "activations/layer18_attention_weight_min": -37.617862701416016,
      "activations/layer19_attention_weight_max": 23.35099220275879,
      "activations/layer19_attention_weight_min": -22.625301361083984,
      "activations/layer1_attention_weight_max": 15.33456039428711,
      "activations/layer1_attention_weight_min": -14.23614501953125,
      "activations/layer20_attention_weight_max": 23.17452621459961,
      "activations/layer20_attention_weight_min": -23.479341506958008,
      "activations/layer21_attention_weight_max": 35.14522171020508,
      "activations/layer21_attention_weight_min": -24.76486587524414,
      "activations/layer22_attention_weight_max": 29.857921600341797,
      "activations/layer22_attention_weight_min": -27.810565948486328,
      "activations/layer23_attention_weight_max": 39.26907730102539,
      "activations/layer23_attention_weight_min": -27.352407455444336,
      "activations/layer2_attention_weight_max": 30.60708236694336,
      "activations/layer2_attention_weight_min": -29.807233810424805,
      "activations/layer3_attention_weight_max": 87.69254302978516,
      "activations/layer3_attention_weight_min": -89.44661712646484,
      "activations/layer4_attention_weight_max": 88.49415588378906,
      "activations/layer4_attention_weight_min": -89.94812774658203,
      "activations/layer5_attention_weight_max": 64.88861083984375,
      "activations/layer5_attention_weight_min": -70.81796264648438,
      "activations/layer6_attention_weight_max": 52.78181838989258,
      "activations/layer6_attention_weight_min": -52.44501876831055,
      "activations/layer7_attention_weight_max": 62.182918548583984,
      "activations/layer7_attention_weight_min": -60.13655471801758,
      "activations/layer8_attention_weight_max": 46.865413665771484,
      "activations/layer8_attention_weight_min": -50.53095245361328,
      "activations/layer9_attention_weight_max": 45.42108917236328,
      "activations/layer9_attention_weight_min": -46.4076042175293,
      "epoch": 15.57,
      "learning_rate": 5.010265151515151e-05,
      "loss": 2.7609,
      "step": 267950
    },
    {
      "activations/layer0_attention_weight_max": 16.37465476989746,
      "activations/layer0_attention_weight_min": -13.906991004943848,
      "activations/layer10_attention_weight_max": 35.924957275390625,
      "activations/layer10_attention_weight_min": -35.96604537963867,
      "activations/layer11_attention_weight_max": 34.75157165527344,
      "activations/layer11_attention_weight_min": -34.9119873046875,
      "activations/layer12_attention_weight_max": 18.479650497436523,
      "activations/layer12_attention_weight_min": -24.170745849609375,
      "activations/layer13_attention_weight_max": 47.13091278076172,
      "activations/layer13_attention_weight_min": -30.588598251342773,
      "activations/layer14_attention_weight_max": 50.858131408691406,
      "activations/layer14_attention_weight_min": -34.81966781616211,
      "activations/layer15_attention_weight_max": 39.01673889160156,
      "activations/layer15_attention_weight_min": -31.147794723510742,
      "activations/layer16_attention_weight_max": 34.90357208251953,
      "activations/layer16_attention_weight_min": -36.21640396118164,
      "activations/layer17_attention_weight_max": 52.7493896484375,
      "activations/layer17_attention_weight_min": -51.48720932006836,
      "activations/layer18_attention_weight_max": 49.647438049316406,
      "activations/layer18_attention_weight_min": -44.34900665283203,
      "activations/layer19_attention_weight_max": 27.622777938842773,
      "activations/layer19_attention_weight_min": -26.432218551635742,
      "activations/layer1_attention_weight_max": 16.74615478515625,
      "activations/layer1_attention_weight_min": -15.116665840148926,
      "activations/layer20_attention_weight_max": 27.768630981445312,
      "activations/layer20_attention_weight_min": -26.43341636657715,
      "activations/layer21_attention_weight_max": 41.58511734008789,
      "activations/layer21_attention_weight_min": -28.10988426208496,
      "activations/layer22_attention_weight_max": 31.61285400390625,
      "activations/layer22_attention_weight_min": -28.97258758544922,
      "activations/layer23_attention_weight_max": 41.21185302734375,
      "activations/layer23_attention_weight_min": -26.38368797302246,
      "activations/layer2_attention_weight_max": 32.17816162109375,
      "activations/layer2_attention_weight_min": -31.59527587890625,
      "activations/layer3_attention_weight_max": 92.30160522460938,
      "activations/layer3_attention_weight_min": -92.63384246826172,
      "activations/layer4_attention_weight_max": 88.44414520263672,
      "activations/layer4_attention_weight_min": -87.15618896484375,
      "activations/layer5_attention_weight_max": 63.908477783203125,
      "activations/layer5_attention_weight_min": -72.6868667602539,
      "activations/layer6_attention_weight_max": 50.21393966674805,
      "activations/layer6_attention_weight_min": -49.436641693115234,
      "activations/layer7_attention_weight_max": 61.77000427246094,
      "activations/layer7_attention_weight_min": -57.043556213378906,
      "activations/layer8_attention_weight_max": 45.50599670410156,
      "activations/layer8_attention_weight_min": -46.490028381347656,
      "activations/layer9_attention_weight_max": 46.30706024169922,
      "activations/layer9_attention_weight_min": -44.89762496948242,
      "epoch": 15.57,
      "learning_rate": 5.008371212121212e-05,
      "loss": 2.746,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.4752,
      "eval_samples_per_second": 506.656,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.4752,
      "eval_openwebtext_samples_per_second": 506.656,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9376,
      "eval_wikitext_samples_per_second": 235.347,
      "step": 268000
    },
    {
      "epoch": 15.57,
      "eval_lambada_loss": 2.572265625,
      "eval_lambada_ppl": 13.095460258581369,
      "eval_lambada_runtime": 9.6094,
      "eval_lambada_samples_per_second": 506.693,
      "step": 268000
    },
    {
      "activations/layer0_attention_weight_max": 15.989795684814453,
      "activations/layer0_attention_weight_min": -13.776671409606934,
      "activations/layer10_attention_weight_max": 34.04423904418945,
      "activations/layer10_attention_weight_min": -35.83158874511719,
      "activations/layer11_attention_weight_max": 31.488285064697266,
      "activations/layer11_attention_weight_min": -32.57164764404297,
      "activations/layer12_attention_weight_max": 19.08698272705078,
      "activations/layer12_attention_weight_min": -26.900650024414062,
      "activations/layer13_attention_weight_max": 37.930503845214844,
      "activations/layer13_attention_weight_min": -27.89128875732422,
      "activations/layer14_attention_weight_max": 33.78968811035156,
      "activations/layer14_attention_weight_min": -30.595651626586914,
      "activations/layer15_attention_weight_max": 31.455402374267578,
      "activations/layer15_attention_weight_min": -30.116548538208008,
      "activations/layer16_attention_weight_max": 33.933326721191406,
      "activations/layer16_attention_weight_min": -31.725467681884766,
      "activations/layer17_attention_weight_max": 54.94308090209961,
      "activations/layer17_attention_weight_min": -46.83750534057617,
      "activations/layer18_attention_weight_max": 48.218902587890625,
      "activations/layer18_attention_weight_min": -42.896175384521484,
      "activations/layer19_attention_weight_max": 23.84263038635254,
      "activations/layer19_attention_weight_min": -23.868276596069336,
      "activations/layer1_attention_weight_max": 16.039325714111328,
      "activations/layer1_attention_weight_min": -15.13916301727295,
      "activations/layer20_attention_weight_max": 21.60369110107422,
      "activations/layer20_attention_weight_min": -23.509613037109375,
      "activations/layer21_attention_weight_max": 38.52165603637695,
      "activations/layer21_attention_weight_min": -20.99543571472168,
      "activations/layer22_attention_weight_max": 31.90614128112793,
      "activations/layer22_attention_weight_min": -28.02220344543457,
      "activations/layer23_attention_weight_max": 43.37543869018555,
      "activations/layer23_attention_weight_min": -23.991867065429688,
      "activations/layer2_attention_weight_max": 29.909360885620117,
      "activations/layer2_attention_weight_min": -29.545028686523438,
      "activations/layer3_attention_weight_max": 88.17223358154297,
      "activations/layer3_attention_weight_min": -87.69146728515625,
      "activations/layer4_attention_weight_max": 85.20946502685547,
      "activations/layer4_attention_weight_min": -83.80902099609375,
      "activations/layer5_attention_weight_max": 65.30856323242188,
      "activations/layer5_attention_weight_min": -78.84281158447266,
      "activations/layer6_attention_weight_max": 51.276493072509766,
      "activations/layer6_attention_weight_min": -53.19045639038086,
      "activations/layer7_attention_weight_max": 58.66213607788086,
      "activations/layer7_attention_weight_min": -63.46744155883789,
      "activations/layer8_attention_weight_max": 45.60582733154297,
      "activations/layer8_attention_weight_min": -51.04475402832031,
      "activations/layer9_attention_weight_max": 44.43110275268555,
      "activations/layer9_attention_weight_min": -49.37955093383789,
      "epoch": 15.58,
      "learning_rate": 5.006477272727272e-05,
      "loss": 2.7333,
      "step": 268050
    },
    {
      "activations/layer0_attention_weight_max": 15.43091106414795,
      "activations/layer0_attention_weight_min": -14.101958274841309,
      "activations/layer10_attention_weight_max": 37.11711502075195,
      "activations/layer10_attention_weight_min": -38.572818756103516,
      "activations/layer11_attention_weight_max": 31.927833557128906,
      "activations/layer11_attention_weight_min": -34.39073181152344,
      "activations/layer12_attention_weight_max": 18.490575790405273,
      "activations/layer12_attention_weight_min": -24.86302375793457,
      "activations/layer13_attention_weight_max": 31.471744537353516,
      "activations/layer13_attention_weight_min": -29.567007064819336,
      "activations/layer14_attention_weight_max": 33.673709869384766,
      "activations/layer14_attention_weight_min": -30.270832061767578,
      "activations/layer15_attention_weight_max": 30.608135223388672,
      "activations/layer15_attention_weight_min": -31.89190101623535,
      "activations/layer16_attention_weight_max": 32.80329895019531,
      "activations/layer16_attention_weight_min": -31.80078887939453,
      "activations/layer17_attention_weight_max": 50.324703216552734,
      "activations/layer17_attention_weight_min": -47.4079475402832,
      "activations/layer18_attention_weight_max": 47.25040054321289,
      "activations/layer18_attention_weight_min": -40.25153732299805,
      "activations/layer19_attention_weight_max": 21.796720504760742,
      "activations/layer19_attention_weight_min": -22.235736846923828,
      "activations/layer1_attention_weight_max": 17.315080642700195,
      "activations/layer1_attention_weight_min": -15.840230941772461,
      "activations/layer20_attention_weight_max": 20.916933059692383,
      "activations/layer20_attention_weight_min": -21.93414878845215,
      "activations/layer21_attention_weight_max": 35.439754486083984,
      "activations/layer21_attention_weight_min": -21.59690284729004,
      "activations/layer22_attention_weight_max": 28.21363639831543,
      "activations/layer22_attention_weight_min": -26.15966796875,
      "activations/layer23_attention_weight_max": 35.66120910644531,
      "activations/layer23_attention_weight_min": -28.84560775756836,
      "activations/layer2_attention_weight_max": 31.980655670166016,
      "activations/layer2_attention_weight_min": -30.345443725585938,
      "activations/layer3_attention_weight_max": 93.37605285644531,
      "activations/layer3_attention_weight_min": -92.74806213378906,
      "activations/layer4_attention_weight_max": 90.28074645996094,
      "activations/layer4_attention_weight_min": -87.02842712402344,
      "activations/layer5_attention_weight_max": 65.77970886230469,
      "activations/layer5_attention_weight_min": -71.66022491455078,
      "activations/layer6_attention_weight_max": 50.98225021362305,
      "activations/layer6_attention_weight_min": -52.8113899230957,
      "activations/layer7_attention_weight_max": 61.254817962646484,
      "activations/layer7_attention_weight_min": -58.00384521484375,
      "activations/layer8_attention_weight_max": 45.01666259765625,
      "activations/layer8_attention_weight_min": -47.78691482543945,
      "activations/layer9_attention_weight_max": 40.97797775268555,
      "activations/layer9_attention_weight_min": -44.126644134521484,
      "epoch": 15.58,
      "learning_rate": 5.004583333333333e-05,
      "loss": 2.7464,
      "step": 268100
    },
    {
      "activations/layer0_attention_weight_max": 15.69036865234375,
      "activations/layer0_attention_weight_min": -14.121074676513672,
      "activations/layer10_attention_weight_max": 41.993473052978516,
      "activations/layer10_attention_weight_min": -41.614356994628906,
      "activations/layer11_attention_weight_max": 37.05567169189453,
      "activations/layer11_attention_weight_min": -35.42484664916992,
      "activations/layer12_attention_weight_max": 21.477270126342773,
      "activations/layer12_attention_weight_min": -25.214229583740234,
      "activations/layer13_attention_weight_max": 34.44902801513672,
      "activations/layer13_attention_weight_min": -29.10380744934082,
      "activations/layer14_attention_weight_max": 34.747779846191406,
      "activations/layer14_attention_weight_min": -34.387298583984375,
      "activations/layer15_attention_weight_max": 33.123878479003906,
      "activations/layer15_attention_weight_min": -34.568634033203125,
      "activations/layer16_attention_weight_max": 34.3358268737793,
      "activations/layer16_attention_weight_min": -34.48659896850586,
      "activations/layer17_attention_weight_max": 53.04986572265625,
      "activations/layer17_attention_weight_min": -48.681976318359375,
      "activations/layer18_attention_weight_max": 45.67287826538086,
      "activations/layer18_attention_weight_min": -42.566650390625,
      "activations/layer19_attention_weight_max": 23.031099319458008,
      "activations/layer19_attention_weight_min": -23.038352966308594,
      "activations/layer1_attention_weight_max": 16.641651153564453,
      "activations/layer1_attention_weight_min": -14.621232032775879,
      "activations/layer20_attention_weight_max": 22.473045349121094,
      "activations/layer20_attention_weight_min": -23.24652671813965,
      "activations/layer21_attention_weight_max": 37.65460205078125,
      "activations/layer21_attention_weight_min": -25.423797607421875,
      "activations/layer22_attention_weight_max": 33.55354309082031,
      "activations/layer22_attention_weight_min": -28.13108253479004,
      "activations/layer23_attention_weight_max": 35.278663635253906,
      "activations/layer23_attention_weight_min": -23.838939666748047,
      "activations/layer2_attention_weight_max": 30.549619674682617,
      "activations/layer2_attention_weight_min": -28.863859176635742,
      "activations/layer3_attention_weight_max": 88.48052978515625,
      "activations/layer3_attention_weight_min": -88.262939453125,
      "activations/layer4_attention_weight_max": 90.63792419433594,
      "activations/layer4_attention_weight_min": -89.36872863769531,
      "activations/layer5_attention_weight_max": 70.76275634765625,
      "activations/layer5_attention_weight_min": -75.8248291015625,
      "activations/layer6_attention_weight_max": 50.303836822509766,
      "activations/layer6_attention_weight_min": -53.80137634277344,
      "activations/layer7_attention_weight_max": 63.309959411621094,
      "activations/layer7_attention_weight_min": -64.4052505493164,
      "activations/layer8_attention_weight_max": 48.818233489990234,
      "activations/layer8_attention_weight_min": -52.723899841308594,
      "activations/layer9_attention_weight_max": 48.67782211303711,
      "activations/layer9_attention_weight_min": -53.136566162109375,
      "epoch": 15.58,
      "learning_rate": 5.0026893939393936e-05,
      "loss": 2.7561,
      "step": 268150
    },
    {
      "activations/layer0_attention_weight_max": 15.96479606628418,
      "activations/layer0_attention_weight_min": -13.795652389526367,
      "activations/layer10_attention_weight_max": 35.97415542602539,
      "activations/layer10_attention_weight_min": -38.566741943359375,
      "activations/layer11_attention_weight_max": 32.71941375732422,
      "activations/layer11_attention_weight_min": -37.52047348022461,
      "activations/layer12_attention_weight_max": 18.25133514404297,
      "activations/layer12_attention_weight_min": -27.72681427001953,
      "activations/layer13_attention_weight_max": 34.99201965332031,
      "activations/layer13_attention_weight_min": -29.701709747314453,
      "activations/layer14_attention_weight_max": 34.83197784423828,
      "activations/layer14_attention_weight_min": -29.819747924804688,
      "activations/layer15_attention_weight_max": 31.474437713623047,
      "activations/layer15_attention_weight_min": -31.112855911254883,
      "activations/layer16_attention_weight_max": 34.43992233276367,
      "activations/layer16_attention_weight_min": -31.246549606323242,
      "activations/layer17_attention_weight_max": 50.150333404541016,
      "activations/layer17_attention_weight_min": -46.20941162109375,
      "activations/layer18_attention_weight_max": 45.86592102050781,
      "activations/layer18_attention_weight_min": -38.55949783325195,
      "activations/layer19_attention_weight_max": 24.854421615600586,
      "activations/layer19_attention_weight_min": -24.158676147460938,
      "activations/layer1_attention_weight_max": 15.990838050842285,
      "activations/layer1_attention_weight_min": -15.62182903289795,
      "activations/layer20_attention_weight_max": 25.443490982055664,
      "activations/layer20_attention_weight_min": -21.733470916748047,
      "activations/layer21_attention_weight_max": 34.271087646484375,
      "activations/layer21_attention_weight_min": -22.493892669677734,
      "activations/layer22_attention_weight_max": 27.499250411987305,
      "activations/layer22_attention_weight_min": -26.132184982299805,
      "activations/layer23_attention_weight_max": 37.578426361083984,
      "activations/layer23_attention_weight_min": -22.758460998535156,
      "activations/layer2_attention_weight_max": 32.2620964050293,
      "activations/layer2_attention_weight_min": -33.892459869384766,
      "activations/layer3_attention_weight_max": 91.05175018310547,
      "activations/layer3_attention_weight_min": -92.30695343017578,
      "activations/layer4_attention_weight_max": 89.6228256225586,
      "activations/layer4_attention_weight_min": -88.08456420898438,
      "activations/layer5_attention_weight_max": 67.55758666992188,
      "activations/layer5_attention_weight_min": -75.556640625,
      "activations/layer6_attention_weight_max": 51.178932189941406,
      "activations/layer6_attention_weight_min": -51.394920349121094,
      "activations/layer7_attention_weight_max": 60.687477111816406,
      "activations/layer7_attention_weight_min": -57.65985870361328,
      "activations/layer8_attention_weight_max": 46.925174713134766,
      "activations/layer8_attention_weight_min": -49.51309585571289,
      "activations/layer9_attention_weight_max": 47.10725784301758,
      "activations/layer9_attention_weight_min": -46.969947814941406,
      "epoch": 15.58,
      "learning_rate": 5.0007954545454544e-05,
      "loss": 2.7335,
      "step": 268200
    },
    {
      "activations/layer0_attention_weight_max": 16.02539825439453,
      "activations/layer0_attention_weight_min": -13.640109062194824,
      "activations/layer10_attention_weight_max": 35.47142791748047,
      "activations/layer10_attention_weight_min": -35.6096305847168,
      "activations/layer11_attention_weight_max": 31.316221237182617,
      "activations/layer11_attention_weight_min": -32.8260612487793,
      "activations/layer12_attention_weight_max": 18.807905197143555,
      "activations/layer12_attention_weight_min": -23.88302993774414,
      "activations/layer13_attention_weight_max": 34.052276611328125,
      "activations/layer13_attention_weight_min": -31.08362579345703,
      "activations/layer14_attention_weight_max": 35.60161590576172,
      "activations/layer14_attention_weight_min": -31.332468032836914,
      "activations/layer15_attention_weight_max": 34.65983963012695,
      "activations/layer15_attention_weight_min": -31.92453956604004,
      "activations/layer16_attention_weight_max": 35.168949127197266,
      "activations/layer16_attention_weight_min": -32.32661056518555,
      "activations/layer17_attention_weight_max": 57.51047134399414,
      "activations/layer17_attention_weight_min": -47.23445510864258,
      "activations/layer18_attention_weight_max": 52.4321403503418,
      "activations/layer18_attention_weight_min": -42.24772644042969,
      "activations/layer19_attention_weight_max": 27.721837997436523,
      "activations/layer19_attention_weight_min": -26.046960830688477,
      "activations/layer1_attention_weight_max": 16.110267639160156,
      "activations/layer1_attention_weight_min": -14.824563026428223,
      "activations/layer20_attention_weight_max": 25.926136016845703,
      "activations/layer20_attention_weight_min": -25.081626892089844,
      "activations/layer21_attention_weight_max": 40.41023635864258,
      "activations/layer21_attention_weight_min": -23.877164840698242,
      "activations/layer22_attention_weight_max": 32.855220794677734,
      "activations/layer22_attention_weight_min": -27.093231201171875,
      "activations/layer23_attention_weight_max": 40.03184509277344,
      "activations/layer23_attention_weight_min": -24.70907974243164,
      "activations/layer2_attention_weight_max": 31.7627010345459,
      "activations/layer2_attention_weight_min": -30.69685935974121,
      "activations/layer3_attention_weight_max": 90.08824920654297,
      "activations/layer3_attention_weight_min": -92.79637908935547,
      "activations/layer4_attention_weight_max": 87.79240417480469,
      "activations/layer4_attention_weight_min": -92.44059753417969,
      "activations/layer5_attention_weight_max": 67.4229965209961,
      "activations/layer5_attention_weight_min": -74.67430114746094,
      "activations/layer6_attention_weight_max": 49.04241180419922,
      "activations/layer6_attention_weight_min": -51.646217346191406,
      "activations/layer7_attention_weight_max": 63.24940490722656,
      "activations/layer7_attention_weight_min": -59.6361083984375,
      "activations/layer8_attention_weight_max": 47.38003921508789,
      "activations/layer8_attention_weight_min": -51.69831466674805,
      "activations/layer9_attention_weight_max": 44.418155670166016,
      "activations/layer9_attention_weight_min": -47.293636322021484,
      "epoch": 15.59,
      "learning_rate": 4.9989015151515146e-05,
      "loss": 2.7386,
      "step": 268250
    },
    {
      "activations/layer0_attention_weight_max": 16.674236297607422,
      "activations/layer0_attention_weight_min": -14.131341934204102,
      "activations/layer10_attention_weight_max": 35.669532775878906,
      "activations/layer10_attention_weight_min": -35.16493606567383,
      "activations/layer11_attention_weight_max": 33.729278564453125,
      "activations/layer11_attention_weight_min": -33.28358459472656,
      "activations/layer12_attention_weight_max": 17.358755111694336,
      "activations/layer12_attention_weight_min": -25.077301025390625,
      "activations/layer13_attention_weight_max": 31.77792739868164,
      "activations/layer13_attention_weight_min": -26.32598304748535,
      "activations/layer14_attention_weight_max": 34.24436950683594,
      "activations/layer14_attention_weight_min": -29.404205322265625,
      "activations/layer15_attention_weight_max": 33.78410339355469,
      "activations/layer15_attention_weight_min": -29.167194366455078,
      "activations/layer16_attention_weight_max": 34.40758514404297,
      "activations/layer16_attention_weight_min": -29.98618507385254,
      "activations/layer17_attention_weight_max": 51.381675720214844,
      "activations/layer17_attention_weight_min": -44.35435104370117,
      "activations/layer18_attention_weight_max": 52.46784973144531,
      "activations/layer18_attention_weight_min": -38.67692947387695,
      "activations/layer19_attention_weight_max": 23.78717041015625,
      "activations/layer19_attention_weight_min": -20.791793823242188,
      "activations/layer1_attention_weight_max": 16.71939468383789,
      "activations/layer1_attention_weight_min": -16.43825912475586,
      "activations/layer20_attention_weight_max": 22.273366928100586,
      "activations/layer20_attention_weight_min": -23.906259536743164,
      "activations/layer21_attention_weight_max": 42.93238830566406,
      "activations/layer21_attention_weight_min": -23.262216567993164,
      "activations/layer22_attention_weight_max": 33.9271125793457,
      "activations/layer22_attention_weight_min": -24.01414680480957,
      "activations/layer23_attention_weight_max": 37.808509826660156,
      "activations/layer23_attention_weight_min": -25.198902130126953,
      "activations/layer2_attention_weight_max": 33.9636116027832,
      "activations/layer2_attention_weight_min": -33.51129913330078,
      "activations/layer3_attention_weight_max": 93.2108154296875,
      "activations/layer3_attention_weight_min": -95.1606674194336,
      "activations/layer4_attention_weight_max": 93.26268005371094,
      "activations/layer4_attention_weight_min": -91.46896362304688,
      "activations/layer5_attention_weight_max": 65.35678100585938,
      "activations/layer5_attention_weight_min": -73.41490173339844,
      "activations/layer6_attention_weight_max": 49.66184997558594,
      "activations/layer6_attention_weight_min": -51.20176315307617,
      "activations/layer7_attention_weight_max": 67.75456237792969,
      "activations/layer7_attention_weight_min": -59.87990188598633,
      "activations/layer8_attention_weight_max": 45.41987991333008,
      "activations/layer8_attention_weight_min": -47.603721618652344,
      "activations/layer9_attention_weight_max": 50.33576583862305,
      "activations/layer9_attention_weight_min": -45.824214935302734,
      "epoch": 15.59,
      "learning_rate": 4.997007575757575e-05,
      "loss": 2.7577,
      "step": 268300
    },
    {
      "activations/layer0_attention_weight_max": 15.796418190002441,
      "activations/layer0_attention_weight_min": -14.340751647949219,
      "activations/layer10_attention_weight_max": 39.064449310302734,
      "activations/layer10_attention_weight_min": -36.871437072753906,
      "activations/layer11_attention_weight_max": 35.89988327026367,
      "activations/layer11_attention_weight_min": -34.94110107421875,
      "activations/layer12_attention_weight_max": 19.80303382873535,
      "activations/layer12_attention_weight_min": -26.99277687072754,
      "activations/layer13_attention_weight_max": 45.07383346557617,
      "activations/layer13_attention_weight_min": -30.761333465576172,
      "activations/layer14_attention_weight_max": 37.41557693481445,
      "activations/layer14_attention_weight_min": -33.265926361083984,
      "activations/layer15_attention_weight_max": 34.30819320678711,
      "activations/layer15_attention_weight_min": -32.03203201293945,
      "activations/layer16_attention_weight_max": 37.284603118896484,
      "activations/layer16_attention_weight_min": -32.20321273803711,
      "activations/layer17_attention_weight_max": 55.145816802978516,
      "activations/layer17_attention_weight_min": -48.68872833251953,
      "activations/layer18_attention_weight_max": 51.73451614379883,
      "activations/layer18_attention_weight_min": -44.38146209716797,
      "activations/layer19_attention_weight_max": 25.92963218688965,
      "activations/layer19_attention_weight_min": -26.393512725830078,
      "activations/layer1_attention_weight_max": 16.719736099243164,
      "activations/layer1_attention_weight_min": -13.962932586669922,
      "activations/layer20_attention_weight_max": 28.159406661987305,
      "activations/layer20_attention_weight_min": -22.779977798461914,
      "activations/layer21_attention_weight_max": 42.177940368652344,
      "activations/layer21_attention_weight_min": -26.213420867919922,
      "activations/layer22_attention_weight_max": 30.686656951904297,
      "activations/layer22_attention_weight_min": -27.615171432495117,
      "activations/layer23_attention_weight_max": 39.59406280517578,
      "activations/layer23_attention_weight_min": -24.16783905029297,
      "activations/layer2_attention_weight_max": 31.51825523376465,
      "activations/layer2_attention_weight_min": -30.421005249023438,
      "activations/layer3_attention_weight_max": 89.24237060546875,
      "activations/layer3_attention_weight_min": -88.65802764892578,
      "activations/layer4_attention_weight_max": 90.3309326171875,
      "activations/layer4_attention_weight_min": -87.49060821533203,
      "activations/layer5_attention_weight_max": 67.95413208007812,
      "activations/layer5_attention_weight_min": -73.81773376464844,
      "activations/layer6_attention_weight_max": 51.85703659057617,
      "activations/layer6_attention_weight_min": -50.806644439697266,
      "activations/layer7_attention_weight_max": 60.59298324584961,
      "activations/layer7_attention_weight_min": -60.585792541503906,
      "activations/layer8_attention_weight_max": 45.779197692871094,
      "activations/layer8_attention_weight_min": -49.63602066040039,
      "activations/layer9_attention_weight_max": 45.67622756958008,
      "activations/layer9_attention_weight_min": -46.08692169189453,
      "epoch": 15.59,
      "learning_rate": 4.995113636363636e-05,
      "loss": 2.747,
      "step": 268350
    },
    {
      "activations/layer0_attention_weight_max": 15.296217918395996,
      "activations/layer0_attention_weight_min": -14.039791107177734,
      "activations/layer10_attention_weight_max": 35.705413818359375,
      "activations/layer10_attention_weight_min": -37.104278564453125,
      "activations/layer11_attention_weight_max": 32.39082336425781,
      "activations/layer11_attention_weight_min": -35.20356369018555,
      "activations/layer12_attention_weight_max": 19.560596466064453,
      "activations/layer12_attention_weight_min": -27.689367294311523,
      "activations/layer13_attention_weight_max": 32.9283447265625,
      "activations/layer13_attention_weight_min": -27.148029327392578,
      "activations/layer14_attention_weight_max": 35.404232025146484,
      "activations/layer14_attention_weight_min": -30.77881622314453,
      "activations/layer15_attention_weight_max": 32.43605422973633,
      "activations/layer15_attention_weight_min": -31.250925064086914,
      "activations/layer16_attention_weight_max": 35.71809387207031,
      "activations/layer16_attention_weight_min": -33.24867248535156,
      "activations/layer17_attention_weight_max": 52.56270980834961,
      "activations/layer17_attention_weight_min": -45.93464660644531,
      "activations/layer18_attention_weight_max": 47.9655876159668,
      "activations/layer18_attention_weight_min": -40.277259826660156,
      "activations/layer19_attention_weight_max": 22.891088485717773,
      "activations/layer19_attention_weight_min": -23.89084815979004,
      "activations/layer1_attention_weight_max": 17.08376693725586,
      "activations/layer1_attention_weight_min": -15.181794166564941,
      "activations/layer20_attention_weight_max": 22.24811363220215,
      "activations/layer20_attention_weight_min": -27.395610809326172,
      "activations/layer21_attention_weight_max": 33.43796920776367,
      "activations/layer21_attention_weight_min": -24.88341522216797,
      "activations/layer22_attention_weight_max": 27.47454833984375,
      "activations/layer22_attention_weight_min": -27.8052921295166,
      "activations/layer23_attention_weight_max": 35.41458511352539,
      "activations/layer23_attention_weight_min": -25.355430603027344,
      "activations/layer2_attention_weight_max": 31.519773483276367,
      "activations/layer2_attention_weight_min": -31.26795196533203,
      "activations/layer3_attention_weight_max": 90.92491912841797,
      "activations/layer3_attention_weight_min": -90.72289276123047,
      "activations/layer4_attention_weight_max": 87.66144561767578,
      "activations/layer4_attention_weight_min": -85.51549530029297,
      "activations/layer5_attention_weight_max": 67.12615966796875,
      "activations/layer5_attention_weight_min": -71.9468994140625,
      "activations/layer6_attention_weight_max": 50.6234245300293,
      "activations/layer6_attention_weight_min": -49.8471565246582,
      "activations/layer7_attention_weight_max": 57.65773391723633,
      "activations/layer7_attention_weight_min": -58.15328598022461,
      "activations/layer8_attention_weight_max": 41.79557418823242,
      "activations/layer8_attention_weight_min": -46.265869140625,
      "activations/layer9_attention_weight_max": 43.4144401550293,
      "activations/layer9_attention_weight_min": -45.732505798339844,
      "epoch": 15.6,
      "learning_rate": 4.9932196969696964e-05,
      "loss": 2.7579,
      "step": 268400
    },
    {
      "activations/layer0_attention_weight_max": 16.3568172454834,
      "activations/layer0_attention_weight_min": -14.096668243408203,
      "activations/layer10_attention_weight_max": 37.23023986816406,
      "activations/layer10_attention_weight_min": -36.49420928955078,
      "activations/layer11_attention_weight_max": 33.49889373779297,
      "activations/layer11_attention_weight_min": -35.47825622558594,
      "activations/layer12_attention_weight_max": 19.623931884765625,
      "activations/layer12_attention_weight_min": -25.259071350097656,
      "activations/layer13_attention_weight_max": 38.433990478515625,
      "activations/layer13_attention_weight_min": -29.00460433959961,
      "activations/layer14_attention_weight_max": 37.45880889892578,
      "activations/layer14_attention_weight_min": -30.110889434814453,
      "activations/layer15_attention_weight_max": 33.333614349365234,
      "activations/layer15_attention_weight_min": -31.163955688476562,
      "activations/layer16_attention_weight_max": 33.438751220703125,
      "activations/layer16_attention_weight_min": -32.14833450317383,
      "activations/layer17_attention_weight_max": 55.24773025512695,
      "activations/layer17_attention_weight_min": -46.33113098144531,
      "activations/layer18_attention_weight_max": 45.59698486328125,
      "activations/layer18_attention_weight_min": -41.20395278930664,
      "activations/layer19_attention_weight_max": 21.73216438293457,
      "activations/layer19_attention_weight_min": -21.894607543945312,
      "activations/layer1_attention_weight_max": 16.226930618286133,
      "activations/layer1_attention_weight_min": -16.283313751220703,
      "activations/layer20_attention_weight_max": 22.883617401123047,
      "activations/layer20_attention_weight_min": -21.08841896057129,
      "activations/layer21_attention_weight_max": 37.07255935668945,
      "activations/layer21_attention_weight_min": -21.866779327392578,
      "activations/layer22_attention_weight_max": 28.196842193603516,
      "activations/layer22_attention_weight_min": -29.040592193603516,
      "activations/layer23_attention_weight_max": 36.396751403808594,
      "activations/layer23_attention_weight_min": -24.44537353515625,
      "activations/layer2_attention_weight_max": 33.00560760498047,
      "activations/layer2_attention_weight_min": -31.62380599975586,
      "activations/layer3_attention_weight_max": 93.29581451416016,
      "activations/layer3_attention_weight_min": -93.92382049560547,
      "activations/layer4_attention_weight_max": 90.6907958984375,
      "activations/layer4_attention_weight_min": -87.6742172241211,
      "activations/layer5_attention_weight_max": 67.55818939208984,
      "activations/layer5_attention_weight_min": -76.13449096679688,
      "activations/layer6_attention_weight_max": 51.48427963256836,
      "activations/layer6_attention_weight_min": -51.21497344970703,
      "activations/layer7_attention_weight_max": 63.36937713623047,
      "activations/layer7_attention_weight_min": -61.72084045410156,
      "activations/layer8_attention_weight_max": 46.61038589477539,
      "activations/layer8_attention_weight_min": -50.289649963378906,
      "activations/layer9_attention_weight_max": 47.013118743896484,
      "activations/layer9_attention_weight_min": -47.396934509277344,
      "epoch": 15.6,
      "learning_rate": 4.991325757575757e-05,
      "loss": 2.7668,
      "step": 268450
    },
    {
      "activations/layer0_attention_weight_max": 15.392951011657715,
      "activations/layer0_attention_weight_min": -14.057146072387695,
      "activations/layer10_attention_weight_max": 35.977577209472656,
      "activations/layer10_attention_weight_min": -35.83111572265625,
      "activations/layer11_attention_weight_max": 30.408763885498047,
      "activations/layer11_attention_weight_min": -35.52586364746094,
      "activations/layer12_attention_weight_max": 17.60979461669922,
      "activations/layer12_attention_weight_min": -29.06316375732422,
      "activations/layer13_attention_weight_max": 28.916854858398438,
      "activations/layer13_attention_weight_min": -27.49066925048828,
      "activations/layer14_attention_weight_max": 34.34769058227539,
      "activations/layer14_attention_weight_min": -29.171192169189453,
      "activations/layer15_attention_weight_max": 31.69057273864746,
      "activations/layer15_attention_weight_min": -29.154476165771484,
      "activations/layer16_attention_weight_max": 32.435752868652344,
      "activations/layer16_attention_weight_min": -29.17059898376465,
      "activations/layer17_attention_weight_max": 49.88080596923828,
      "activations/layer17_attention_weight_min": -42.692203521728516,
      "activations/layer18_attention_weight_max": 43.45400619506836,
      "activations/layer18_attention_weight_min": -35.74367141723633,
      "activations/layer19_attention_weight_max": 22.695425033569336,
      "activations/layer19_attention_weight_min": -21.22041130065918,
      "activations/layer1_attention_weight_max": 16.283143997192383,
      "activations/layer1_attention_weight_min": -14.974932670593262,
      "activations/layer20_attention_weight_max": 19.872087478637695,
      "activations/layer20_attention_weight_min": -22.454753875732422,
      "activations/layer21_attention_weight_max": 34.21965408325195,
      "activations/layer21_attention_weight_min": -22.41514015197754,
      "activations/layer22_attention_weight_max": 26.602441787719727,
      "activations/layer22_attention_weight_min": -26.255159378051758,
      "activations/layer23_attention_weight_max": 32.65943908691406,
      "activations/layer23_attention_weight_min": -26.10969352722168,
      "activations/layer2_attention_weight_max": 31.89906883239746,
      "activations/layer2_attention_weight_min": -32.69557189941406,
      "activations/layer3_attention_weight_max": 90.338623046875,
      "activations/layer3_attention_weight_min": -88.22615051269531,
      "activations/layer4_attention_weight_max": 89.78174591064453,
      "activations/layer4_attention_weight_min": -88.54214477539062,
      "activations/layer5_attention_weight_max": 65.53375244140625,
      "activations/layer5_attention_weight_min": -74.76869201660156,
      "activations/layer6_attention_weight_max": 49.67190933227539,
      "activations/layer6_attention_weight_min": -50.789485931396484,
      "activations/layer7_attention_weight_max": 60.67867660522461,
      "activations/layer7_attention_weight_min": -59.57027053833008,
      "activations/layer8_attention_weight_max": 43.40528869628906,
      "activations/layer8_attention_weight_min": -47.94453811645508,
      "activations/layer9_attention_weight_max": 45.10456848144531,
      "activations/layer9_attention_weight_min": -44.43836212158203,
      "epoch": 15.6,
      "learning_rate": 4.9894318181818174e-05,
      "loss": 2.7419,
      "step": 268500
    },
    {
      "activations/layer0_attention_weight_max": 16.173892974853516,
      "activations/layer0_attention_weight_min": -14.223953247070312,
      "activations/layer10_attention_weight_max": 37.13858413696289,
      "activations/layer10_attention_weight_min": -36.309547424316406,
      "activations/layer11_attention_weight_max": 33.41106414794922,
      "activations/layer11_attention_weight_min": -33.617271423339844,
      "activations/layer12_attention_weight_max": 18.762920379638672,
      "activations/layer12_attention_weight_min": -25.974916458129883,
      "activations/layer13_attention_weight_max": 31.911226272583008,
      "activations/layer13_attention_weight_min": -28.878952026367188,
      "activations/layer14_attention_weight_max": 36.01599884033203,
      "activations/layer14_attention_weight_min": -30.29572105407715,
      "activations/layer15_attention_weight_max": 31.28540802001953,
      "activations/layer15_attention_weight_min": -29.994596481323242,
      "activations/layer16_attention_weight_max": 31.798383712768555,
      "activations/layer16_attention_weight_min": -30.769136428833008,
      "activations/layer17_attention_weight_max": 49.53519058227539,
      "activations/layer17_attention_weight_min": -44.19180679321289,
      "activations/layer18_attention_weight_max": 43.543190002441406,
      "activations/layer18_attention_weight_min": -37.63174057006836,
      "activations/layer19_attention_weight_max": 22.069082260131836,
      "activations/layer19_attention_weight_min": -22.848255157470703,
      "activations/layer1_attention_weight_max": 16.211013793945312,
      "activations/layer1_attention_weight_min": -14.594819068908691,
      "activations/layer20_attention_weight_max": 22.528196334838867,
      "activations/layer20_attention_weight_min": -24.788827896118164,
      "activations/layer21_attention_weight_max": 35.67970657348633,
      "activations/layer21_attention_weight_min": -24.6070556640625,
      "activations/layer22_attention_weight_max": 25.982995986938477,
      "activations/layer22_attention_weight_min": -25.452289581298828,
      "activations/layer23_attention_weight_max": 34.230560302734375,
      "activations/layer23_attention_weight_min": -23.95934295654297,
      "activations/layer2_attention_weight_max": 32.008541107177734,
      "activations/layer2_attention_weight_min": -30.98575210571289,
      "activations/layer3_attention_weight_max": 88.89888763427734,
      "activations/layer3_attention_weight_min": -88.24542236328125,
      "activations/layer4_attention_weight_max": 90.38622283935547,
      "activations/layer4_attention_weight_min": -88.74517822265625,
      "activations/layer5_attention_weight_max": 64.78529357910156,
      "activations/layer5_attention_weight_min": -74.05528259277344,
      "activations/layer6_attention_weight_max": 50.143592834472656,
      "activations/layer6_attention_weight_min": -50.27275848388672,
      "activations/layer7_attention_weight_max": 62.40458679199219,
      "activations/layer7_attention_weight_min": -60.207244873046875,
      "activations/layer8_attention_weight_max": 43.28861618041992,
      "activations/layer8_attention_weight_min": -46.96175765991211,
      "activations/layer9_attention_weight_max": 43.808372497558594,
      "activations/layer9_attention_weight_min": -47.95178985595703,
      "epoch": 15.6,
      "learning_rate": 4.987537878787879e-05,
      "loss": 2.7447,
      "step": 268550
    },
    {
      "activations/layer0_attention_weight_max": 15.107855796813965,
      "activations/layer0_attention_weight_min": -14.412568092346191,
      "activations/layer10_attention_weight_max": 34.47963333129883,
      "activations/layer10_attention_weight_min": -33.594482421875,
      "activations/layer11_attention_weight_max": 30.72576141357422,
      "activations/layer11_attention_weight_min": -32.43974304199219,
      "activations/layer12_attention_weight_max": 18.832473754882812,
      "activations/layer12_attention_weight_min": -23.85797691345215,
      "activations/layer13_attention_weight_max": 34.88594436645508,
      "activations/layer13_attention_weight_min": -27.14496421813965,
      "activations/layer14_attention_weight_max": 32.53234100341797,
      "activations/layer14_attention_weight_min": -30.860429763793945,
      "activations/layer15_attention_weight_max": 31.664134979248047,
      "activations/layer15_attention_weight_min": -30.965307235717773,
      "activations/layer16_attention_weight_max": 30.385791778564453,
      "activations/layer16_attention_weight_min": -30.735401153564453,
      "activations/layer17_attention_weight_max": 48.31307601928711,
      "activations/layer17_attention_weight_min": -44.863040924072266,
      "activations/layer18_attention_weight_max": 44.81898880004883,
      "activations/layer18_attention_weight_min": -39.299373626708984,
      "activations/layer19_attention_weight_max": 22.182647705078125,
      "activations/layer19_attention_weight_min": -23.25979232788086,
      "activations/layer1_attention_weight_max": 16.727249145507812,
      "activations/layer1_attention_weight_min": -16.3551025390625,
      "activations/layer20_attention_weight_max": 20.336130142211914,
      "activations/layer20_attention_weight_min": -23.088394165039062,
      "activations/layer21_attention_weight_max": 34.36044692993164,
      "activations/layer21_attention_weight_min": -23.834701538085938,
      "activations/layer22_attention_weight_max": 29.465364456176758,
      "activations/layer22_attention_weight_min": -28.532739639282227,
      "activations/layer23_attention_weight_max": 37.077064514160156,
      "activations/layer23_attention_weight_min": -23.877927780151367,
      "activations/layer2_attention_weight_max": 33.31325149536133,
      "activations/layer2_attention_weight_min": -31.11583709716797,
      "activations/layer3_attention_weight_max": 90.11603546142578,
      "activations/layer3_attention_weight_min": -91.98722076416016,
      "activations/layer4_attention_weight_max": 88.54212188720703,
      "activations/layer4_attention_weight_min": -85.11727905273438,
      "activations/layer5_attention_weight_max": 63.349647521972656,
      "activations/layer5_attention_weight_min": -71.75009155273438,
      "activations/layer6_attention_weight_max": 49.82597351074219,
      "activations/layer6_attention_weight_min": -49.495121002197266,
      "activations/layer7_attention_weight_max": 59.67885208129883,
      "activations/layer7_attention_weight_min": -56.72323226928711,
      "activations/layer8_attention_weight_max": 44.24922180175781,
      "activations/layer8_attention_weight_min": -47.627662658691406,
      "activations/layer9_attention_weight_max": 43.41566467285156,
      "activations/layer9_attention_weight_min": -46.082767486572266,
      "epoch": 15.61,
      "learning_rate": 4.985643939393939e-05,
      "loss": 2.7534,
      "step": 268600
    },
    {
      "activations/layer0_attention_weight_max": 15.552814483642578,
      "activations/layer0_attention_weight_min": -14.32512378692627,
      "activations/layer10_attention_weight_max": 38.932918548583984,
      "activations/layer10_attention_weight_min": -36.607906341552734,
      "activations/layer11_attention_weight_max": 33.47487258911133,
      "activations/layer11_attention_weight_min": -34.70082092285156,
      "activations/layer12_attention_weight_max": 19.80887794494629,
      "activations/layer12_attention_weight_min": -25.55906105041504,
      "activations/layer13_attention_weight_max": 38.54667663574219,
      "activations/layer13_attention_weight_min": -27.955490112304688,
      "activations/layer14_attention_weight_max": 38.265342712402344,
      "activations/layer14_attention_weight_min": -31.253150939941406,
      "activations/layer15_attention_weight_max": 35.00844955444336,
      "activations/layer15_attention_weight_min": -29.16238021850586,
      "activations/layer16_attention_weight_max": 34.585365295410156,
      "activations/layer16_attention_weight_min": -30.416759490966797,
      "activations/layer17_attention_weight_max": 54.53593444824219,
      "activations/layer17_attention_weight_min": -43.9583625793457,
      "activations/layer18_attention_weight_max": 50.512115478515625,
      "activations/layer18_attention_weight_min": -38.69572448730469,
      "activations/layer19_attention_weight_max": 25.374330520629883,
      "activations/layer19_attention_weight_min": -24.168630599975586,
      "activations/layer1_attention_weight_max": 16.495874404907227,
      "activations/layer1_attention_weight_min": -14.934345245361328,
      "activations/layer20_attention_weight_max": 23.52643394470215,
      "activations/layer20_attention_weight_min": -22.740304946899414,
      "activations/layer21_attention_weight_max": 38.113311767578125,
      "activations/layer21_attention_weight_min": -23.751220703125,
      "activations/layer22_attention_weight_max": 32.274085998535156,
      "activations/layer22_attention_weight_min": -25.821788787841797,
      "activations/layer23_attention_weight_max": 41.925350189208984,
      "activations/layer23_attention_weight_min": -25.286649703979492,
      "activations/layer2_attention_weight_max": 32.240264892578125,
      "activations/layer2_attention_weight_min": -30.551572799682617,
      "activations/layer3_attention_weight_max": 92.07125854492188,
      "activations/layer3_attention_weight_min": -88.86754608154297,
      "activations/layer4_attention_weight_max": 89.14485931396484,
      "activations/layer4_attention_weight_min": -91.50648498535156,
      "activations/layer5_attention_weight_max": 68.59974670410156,
      "activations/layer5_attention_weight_min": -76.42853546142578,
      "activations/layer6_attention_weight_max": 49.76277542114258,
      "activations/layer6_attention_weight_min": -50.41499710083008,
      "activations/layer7_attention_weight_max": 63.63937759399414,
      "activations/layer7_attention_weight_min": -56.36269760131836,
      "activations/layer8_attention_weight_max": 48.70412063598633,
      "activations/layer8_attention_weight_min": -48.15883255004883,
      "activations/layer9_attention_weight_max": 49.29315185546875,
      "activations/layer9_attention_weight_min": -45.79197311401367,
      "epoch": 15.61,
      "learning_rate": 4.983749999999999e-05,
      "loss": 2.7565,
      "step": 268650
    },
    {
      "activations/layer0_attention_weight_max": 15.286346435546875,
      "activations/layer0_attention_weight_min": -14.50073528289795,
      "activations/layer10_attention_weight_max": 41.783424377441406,
      "activations/layer10_attention_weight_min": -41.041229248046875,
      "activations/layer11_attention_weight_max": 38.86916732788086,
      "activations/layer11_attention_weight_min": -37.34675598144531,
      "activations/layer12_attention_weight_max": 20.34610939025879,
      "activations/layer12_attention_weight_min": -28.07575798034668,
      "activations/layer13_attention_weight_max": 52.90605163574219,
      "activations/layer13_attention_weight_min": -33.02701187133789,
      "activations/layer14_attention_weight_max": 52.85855484008789,
      "activations/layer14_attention_weight_min": -33.16773223876953,
      "activations/layer15_attention_weight_max": 43.820926666259766,
      "activations/layer15_attention_weight_min": -33.57099914550781,
      "activations/layer16_attention_weight_max": 38.52790832519531,
      "activations/layer16_attention_weight_min": -34.53909683227539,
      "activations/layer17_attention_weight_max": 63.94879913330078,
      "activations/layer17_attention_weight_min": -55.0334587097168,
      "activations/layer18_attention_weight_max": 56.4406623840332,
      "activations/layer18_attention_weight_min": -51.31892013549805,
      "activations/layer19_attention_weight_max": 32.862979888916016,
      "activations/layer19_attention_weight_min": -26.35822868347168,
      "activations/layer1_attention_weight_max": 16.84663200378418,
      "activations/layer1_attention_weight_min": -14.203997611999512,
      "activations/layer20_attention_weight_max": 31.4566593170166,
      "activations/layer20_attention_weight_min": -24.29289436340332,
      "activations/layer21_attention_weight_max": 47.19401931762695,
      "activations/layer21_attention_weight_min": -25.055356979370117,
      "activations/layer22_attention_weight_max": 34.89438247680664,
      "activations/layer22_attention_weight_min": -28.42745018005371,
      "activations/layer23_attention_weight_max": 38.24420928955078,
      "activations/layer23_attention_weight_min": -25.796417236328125,
      "activations/layer2_attention_weight_max": 30.90057945251465,
      "activations/layer2_attention_weight_min": -30.298587799072266,
      "activations/layer3_attention_weight_max": 86.50792694091797,
      "activations/layer3_attention_weight_min": -87.6852035522461,
      "activations/layer4_attention_weight_max": 88.97520446777344,
      "activations/layer4_attention_weight_min": -91.12728118896484,
      "activations/layer5_attention_weight_max": 65.9001693725586,
      "activations/layer5_attention_weight_min": -72.67671203613281,
      "activations/layer6_attention_weight_max": 50.465145111083984,
      "activations/layer6_attention_weight_min": -50.50215148925781,
      "activations/layer7_attention_weight_max": 65.4245834350586,
      "activations/layer7_attention_weight_min": -64.08413696289062,
      "activations/layer8_attention_weight_max": 51.8062629699707,
      "activations/layer8_attention_weight_min": -51.59352493286133,
      "activations/layer9_attention_weight_max": 53.78103256225586,
      "activations/layer9_attention_weight_min": -48.28512191772461,
      "epoch": 15.61,
      "learning_rate": 4.98185606060606e-05,
      "loss": 2.7553,
      "step": 268700
    },
    {
      "activations/layer0_attention_weight_max": 15.3714017868042,
      "activations/layer0_attention_weight_min": -14.477355003356934,
      "activations/layer10_attention_weight_max": 34.308692932128906,
      "activations/layer10_attention_weight_min": -35.002471923828125,
      "activations/layer11_attention_weight_max": 29.976787567138672,
      "activations/layer11_attention_weight_min": -33.30419158935547,
      "activations/layer12_attention_weight_max": 18.968536376953125,
      "activations/layer12_attention_weight_min": -26.13414764404297,
      "activations/layer13_attention_weight_max": 33.35205841064453,
      "activations/layer13_attention_weight_min": -28.732492446899414,
      "activations/layer14_attention_weight_max": 35.87480545043945,
      "activations/layer14_attention_weight_min": -34.39349365234375,
      "activations/layer15_attention_weight_max": 33.30799102783203,
      "activations/layer15_attention_weight_min": -32.179603576660156,
      "activations/layer16_attention_weight_max": 34.41092300415039,
      "activations/layer16_attention_weight_min": -33.02703857421875,
      "activations/layer17_attention_weight_max": 52.51945114135742,
      "activations/layer17_attention_weight_min": -46.62711715698242,
      "activations/layer18_attention_weight_max": 50.43437957763672,
      "activations/layer18_attention_weight_min": -40.56053924560547,
      "activations/layer19_attention_weight_max": 23.140439987182617,
      "activations/layer19_attention_weight_min": -21.762113571166992,
      "activations/layer1_attention_weight_max": 17.553543090820312,
      "activations/layer1_attention_weight_min": -15.890636444091797,
      "activations/layer20_attention_weight_max": 24.5006103515625,
      "activations/layer20_attention_weight_min": -22.43433952331543,
      "activations/layer21_attention_weight_max": 45.12353515625,
      "activations/layer21_attention_weight_min": -23.384714126586914,
      "activations/layer22_attention_weight_max": 30.959205627441406,
      "activations/layer22_attention_weight_min": -27.732446670532227,
      "activations/layer23_attention_weight_max": 41.540557861328125,
      "activations/layer23_attention_weight_min": -22.34548568725586,
      "activations/layer2_attention_weight_max": 30.9649658203125,
      "activations/layer2_attention_weight_min": -29.918441772460938,
      "activations/layer3_attention_weight_max": 92.07837677001953,
      "activations/layer3_attention_weight_min": -88.94525146484375,
      "activations/layer4_attention_weight_max": 88.68233489990234,
      "activations/layer4_attention_weight_min": -84.09362030029297,
      "activations/layer5_attention_weight_max": 61.99705123901367,
      "activations/layer5_attention_weight_min": -78.73724365234375,
      "activations/layer6_attention_weight_max": 50.22883605957031,
      "activations/layer6_attention_weight_min": -47.135074615478516,
      "activations/layer7_attention_weight_max": 60.07727813720703,
      "activations/layer7_attention_weight_min": -56.86532211303711,
      "activations/layer8_attention_weight_max": 42.58625411987305,
      "activations/layer8_attention_weight_min": -45.094520568847656,
      "activations/layer9_attention_weight_max": 40.87812805175781,
      "activations/layer9_attention_weight_min": -43.20711898803711,
      "epoch": 15.62,
      "learning_rate": 4.979962121212121e-05,
      "loss": 2.7393,
      "step": 268750
    },
    {
      "activations/layer0_attention_weight_max": 16.692846298217773,
      "activations/layer0_attention_weight_min": -15.364723205566406,
      "activations/layer10_attention_weight_max": 41.95304489135742,
      "activations/layer10_attention_weight_min": -41.25455856323242,
      "activations/layer11_attention_weight_max": 39.687782287597656,
      "activations/layer11_attention_weight_min": -36.9893684387207,
      "activations/layer12_attention_weight_max": 20.887720108032227,
      "activations/layer12_attention_weight_min": -26.93614387512207,
      "activations/layer13_attention_weight_max": 49.06703186035156,
      "activations/layer13_attention_weight_min": -33.39884948730469,
      "activations/layer14_attention_weight_max": 50.29362487792969,
      "activations/layer14_attention_weight_min": -33.85822677612305,
      "activations/layer15_attention_weight_max": 47.02632522583008,
      "activations/layer15_attention_weight_min": -32.86891174316406,
      "activations/layer16_attention_weight_max": 40.78446578979492,
      "activations/layer16_attention_weight_min": -32.398292541503906,
      "activations/layer17_attention_weight_max": 60.46436309814453,
      "activations/layer17_attention_weight_min": -47.79258728027344,
      "activations/layer18_attention_weight_max": 56.3515510559082,
      "activations/layer18_attention_weight_min": -43.3561897277832,
      "activations/layer19_attention_weight_max": 32.45155334472656,
      "activations/layer19_attention_weight_min": -26.968017578125,
      "activations/layer1_attention_weight_max": 17.360761642456055,
      "activations/layer1_attention_weight_min": -15.487685203552246,
      "activations/layer20_attention_weight_max": 33.049591064453125,
      "activations/layer20_attention_weight_min": -25.513904571533203,
      "activations/layer21_attention_weight_max": 66.48613739013672,
      "activations/layer21_attention_weight_min": -28.03969383239746,
      "activations/layer22_attention_weight_max": 34.10068130493164,
      "activations/layer22_attention_weight_min": -26.16901969909668,
      "activations/layer23_attention_weight_max": 40.300689697265625,
      "activations/layer23_attention_weight_min": -24.710126876831055,
      "activations/layer2_attention_weight_max": 32.817604064941406,
      "activations/layer2_attention_weight_min": -33.05091857910156,
      "activations/layer3_attention_weight_max": 89.88985443115234,
      "activations/layer3_attention_weight_min": -92.31170654296875,
      "activations/layer4_attention_weight_max": 87.89165496826172,
      "activations/layer4_attention_weight_min": -88.0848159790039,
      "activations/layer5_attention_weight_max": 65.5245132446289,
      "activations/layer5_attention_weight_min": -70.06805419921875,
      "activations/layer6_attention_weight_max": 50.933902740478516,
      "activations/layer6_attention_weight_min": -50.011756896972656,
      "activations/layer7_attention_weight_max": 64.6611099243164,
      "activations/layer7_attention_weight_min": -64.36483764648438,
      "activations/layer8_attention_weight_max": 49.62862777709961,
      "activations/layer8_attention_weight_min": -49.82356643676758,
      "activations/layer9_attention_weight_max": 54.5322380065918,
      "activations/layer9_attention_weight_min": -52.83384323120117,
      "epoch": 15.62,
      "learning_rate": 4.9780681818181816e-05,
      "loss": 2.7474,
      "step": 268800
    },
    {
      "activations/layer0_attention_weight_max": 15.460993766784668,
      "activations/layer0_attention_weight_min": -14.70512580871582,
      "activations/layer10_attention_weight_max": 35.077491760253906,
      "activations/layer10_attention_weight_min": -34.532623291015625,
      "activations/layer11_attention_weight_max": 31.819950103759766,
      "activations/layer11_attention_weight_min": -33.326377868652344,
      "activations/layer12_attention_weight_max": 17.74444580078125,
      "activations/layer12_attention_weight_min": -22.813940048217773,
      "activations/layer13_attention_weight_max": 29.507522583007812,
      "activations/layer13_attention_weight_min": -27.539575576782227,
      "activations/layer14_attention_weight_max": 33.90625,
      "activations/layer14_attention_weight_min": -29.85575294494629,
      "activations/layer15_attention_weight_max": 29.842538833618164,
      "activations/layer15_attention_weight_min": -30.246295928955078,
      "activations/layer16_attention_weight_max": 30.976484298706055,
      "activations/layer16_attention_weight_min": -29.58599281311035,
      "activations/layer17_attention_weight_max": 48.46638107299805,
      "activations/layer17_attention_weight_min": -43.35246276855469,
      "activations/layer18_attention_weight_max": 44.71839904785156,
      "activations/layer18_attention_weight_min": -39.432682037353516,
      "activations/layer19_attention_weight_max": 24.4227352142334,
      "activations/layer19_attention_weight_min": -23.200031280517578,
      "activations/layer1_attention_weight_max": 16.518444061279297,
      "activations/layer1_attention_weight_min": -14.567342758178711,
      "activations/layer20_attention_weight_max": 26.151317596435547,
      "activations/layer20_attention_weight_min": -22.934873580932617,
      "activations/layer21_attention_weight_max": 31.995840072631836,
      "activations/layer21_attention_weight_min": -24.991872787475586,
      "activations/layer22_attention_weight_max": 25.39394760131836,
      "activations/layer22_attention_weight_min": -27.066783905029297,
      "activations/layer23_attention_weight_max": 35.889888763427734,
      "activations/layer23_attention_weight_min": -23.542654037475586,
      "activations/layer2_attention_weight_max": 29.84724235534668,
      "activations/layer2_attention_weight_min": -29.372169494628906,
      "activations/layer3_attention_weight_max": 91.59620666503906,
      "activations/layer3_attention_weight_min": -90.26532745361328,
      "activations/layer4_attention_weight_max": 90.64110565185547,
      "activations/layer4_attention_weight_min": -89.0813217163086,
      "activations/layer5_attention_weight_max": 67.91380310058594,
      "activations/layer5_attention_weight_min": -75.94271087646484,
      "activations/layer6_attention_weight_max": 50.66924285888672,
      "activations/layer6_attention_weight_min": -51.51726150512695,
      "activations/layer7_attention_weight_max": 61.00326919555664,
      "activations/layer7_attention_weight_min": -60.56018829345703,
      "activations/layer8_attention_weight_max": 44.8851318359375,
      "activations/layer8_attention_weight_min": -47.50209426879883,
      "activations/layer9_attention_weight_max": 46.295955657958984,
      "activations/layer9_attention_weight_min": -44.399723052978516,
      "epoch": 15.62,
      "learning_rate": 4.976174242424242e-05,
      "loss": 2.7522,
      "step": 268850
    },
    {
      "activations/layer0_attention_weight_max": 15.98092269897461,
      "activations/layer0_attention_weight_min": -14.602363586425781,
      "activations/layer10_attention_weight_max": 34.84917449951172,
      "activations/layer10_attention_weight_min": -37.08945846557617,
      "activations/layer11_attention_weight_max": 31.735172271728516,
      "activations/layer11_attention_weight_min": -34.542518615722656,
      "activations/layer12_attention_weight_max": 18.259244918823242,
      "activations/layer12_attention_weight_min": -25.719614028930664,
      "activations/layer13_attention_weight_max": 29.780498504638672,
      "activations/layer13_attention_weight_min": -26.793529510498047,
      "activations/layer14_attention_weight_max": 32.477413177490234,
      "activations/layer14_attention_weight_min": -30.653343200683594,
      "activations/layer15_attention_weight_max": 30.005207061767578,
      "activations/layer15_attention_weight_min": -28.89325714111328,
      "activations/layer16_attention_weight_max": 32.634864807128906,
      "activations/layer16_attention_weight_min": -29.716520309448242,
      "activations/layer17_attention_weight_max": 48.42926788330078,
      "activations/layer17_attention_weight_min": -45.145687103271484,
      "activations/layer18_attention_weight_max": 44.34599304199219,
      "activations/layer18_attention_weight_min": -39.80891036987305,
      "activations/layer19_attention_weight_max": 21.595762252807617,
      "activations/layer19_attention_weight_min": -23.431015014648438,
      "activations/layer1_attention_weight_max": 16.973196029663086,
      "activations/layer1_attention_weight_min": -12.667108535766602,
      "activations/layer20_attention_weight_max": 20.238786697387695,
      "activations/layer20_attention_weight_min": -24.31865692138672,
      "activations/layer21_attention_weight_max": 36.65092849731445,
      "activations/layer21_attention_weight_min": -24.50314712524414,
      "activations/layer22_attention_weight_max": 27.830121994018555,
      "activations/layer22_attention_weight_min": -26.29631996154785,
      "activations/layer23_attention_weight_max": 37.13513946533203,
      "activations/layer23_attention_weight_min": -29.50251579284668,
      "activations/layer2_attention_weight_max": 30.70044708251953,
      "activations/layer2_attention_weight_min": -32.63259506225586,
      "activations/layer3_attention_weight_max": 91.68561553955078,
      "activations/layer3_attention_weight_min": -93.5440444946289,
      "activations/layer4_attention_weight_max": 90.78134155273438,
      "activations/layer4_attention_weight_min": -89.93196105957031,
      "activations/layer5_attention_weight_max": 67.6173095703125,
      "activations/layer5_attention_weight_min": -76.05162048339844,
      "activations/layer6_attention_weight_max": 50.568180084228516,
      "activations/layer6_attention_weight_min": -50.67377471923828,
      "activations/layer7_attention_weight_max": 62.787113189697266,
      "activations/layer7_attention_weight_min": -59.073211669921875,
      "activations/layer8_attention_weight_max": 44.9675178527832,
      "activations/layer8_attention_weight_min": -46.09135055541992,
      "activations/layer9_attention_weight_max": 45.066246032714844,
      "activations/layer9_attention_weight_min": -48.188236236572266,
      "epoch": 15.62,
      "learning_rate": 4.974280303030303e-05,
      "loss": 2.7641,
      "step": 268900
    },
    {
      "activations/layer0_attention_weight_max": 15.839629173278809,
      "activations/layer0_attention_weight_min": -14.316071510314941,
      "activations/layer10_attention_weight_max": 41.55180358886719,
      "activations/layer10_attention_weight_min": -43.943397521972656,
      "activations/layer11_attention_weight_max": 37.59883117675781,
      "activations/layer11_attention_weight_min": -37.223228454589844,
      "activations/layer12_attention_weight_max": 18.584339141845703,
      "activations/layer12_attention_weight_min": -24.036258697509766,
      "activations/layer13_attention_weight_max": 33.92809295654297,
      "activations/layer13_attention_weight_min": -30.42665672302246,
      "activations/layer14_attention_weight_max": 36.59950256347656,
      "activations/layer14_attention_weight_min": -30.970699310302734,
      "activations/layer15_attention_weight_max": 33.26558303833008,
      "activations/layer15_attention_weight_min": -30.42849349975586,
      "activations/layer16_attention_weight_max": 33.70250701904297,
      "activations/layer16_attention_weight_min": -33.86780548095703,
      "activations/layer17_attention_weight_max": 51.48213577270508,
      "activations/layer17_attention_weight_min": -45.780704498291016,
      "activations/layer18_attention_weight_max": 47.38756561279297,
      "activations/layer18_attention_weight_min": -39.073734283447266,
      "activations/layer19_attention_weight_max": 23.595796585083008,
      "activations/layer19_attention_weight_min": -21.3072566986084,
      "activations/layer1_attention_weight_max": 16.01457977294922,
      "activations/layer1_attention_weight_min": -13.132067680358887,
      "activations/layer20_attention_weight_max": 22.372291564941406,
      "activations/layer20_attention_weight_min": -23.64596176147461,
      "activations/layer21_attention_weight_max": 35.25118637084961,
      "activations/layer21_attention_weight_min": -20.460939407348633,
      "activations/layer22_attention_weight_max": 28.689661026000977,
      "activations/layer22_attention_weight_min": -27.18857192993164,
      "activations/layer23_attention_weight_max": 41.3888053894043,
      "activations/layer23_attention_weight_min": -23.95893669128418,
      "activations/layer2_attention_weight_max": 31.310882568359375,
      "activations/layer2_attention_weight_min": -31.13440704345703,
      "activations/layer3_attention_weight_max": 96.79117584228516,
      "activations/layer3_attention_weight_min": -102.30111694335938,
      "activations/layer4_attention_weight_max": 91.8475570678711,
      "activations/layer4_attention_weight_min": -89.91696166992188,
      "activations/layer5_attention_weight_max": 67.042236328125,
      "activations/layer5_attention_weight_min": -75.00767517089844,
      "activations/layer6_attention_weight_max": 51.60377502441406,
      "activations/layer6_attention_weight_min": -54.061988830566406,
      "activations/layer7_attention_weight_max": 62.54945755004883,
      "activations/layer7_attention_weight_min": -62.63986587524414,
      "activations/layer8_attention_weight_max": 48.913360595703125,
      "activations/layer8_attention_weight_min": -53.234283447265625,
      "activations/layer9_attention_weight_max": 50.35911560058594,
      "activations/layer9_attention_weight_min": -51.698028564453125,
      "epoch": 15.63,
      "learning_rate": 4.9724242424242424e-05,
      "loss": 2.759,
      "step": 268950
    },
    {
      "activations/layer0_attention_weight_max": 15.696843147277832,
      "activations/layer0_attention_weight_min": -14.92579174041748,
      "activations/layer10_attention_weight_max": 36.384788513183594,
      "activations/layer10_attention_weight_min": -37.6377067565918,
      "activations/layer11_attention_weight_max": 31.362197875976562,
      "activations/layer11_attention_weight_min": -33.77691650390625,
      "activations/layer12_attention_weight_max": 19.650712966918945,
      "activations/layer12_attention_weight_min": -24.777118682861328,
      "activations/layer13_attention_weight_max": 37.70158767700195,
      "activations/layer13_attention_weight_min": -28.895198822021484,
      "activations/layer14_attention_weight_max": 37.35844802856445,
      "activations/layer14_attention_weight_min": -31.827028274536133,
      "activations/layer15_attention_weight_max": 32.87862014770508,
      "activations/layer15_attention_weight_min": -31.577253341674805,
      "activations/layer16_attention_weight_max": 37.28604507446289,
      "activations/layer16_attention_weight_min": -32.08127212524414,
      "activations/layer17_attention_weight_max": 53.83879470825195,
      "activations/layer17_attention_weight_min": -45.90449523925781,
      "activations/layer18_attention_weight_max": 53.6867790222168,
      "activations/layer18_attention_weight_min": -41.24718475341797,
      "activations/layer19_attention_weight_max": 27.759033203125,
      "activations/layer19_attention_weight_min": -23.563222885131836,
      "activations/layer1_attention_weight_max": 16.089677810668945,
      "activations/layer1_attention_weight_min": -13.478793144226074,
      "activations/layer20_attention_weight_max": 28.22332763671875,
      "activations/layer20_attention_weight_min": -23.39268684387207,
      "activations/layer21_attention_weight_max": 50.49314880371094,
      "activations/layer21_attention_weight_min": -24.096080780029297,
      "activations/layer22_attention_weight_max": 33.183189392089844,
      "activations/layer22_attention_weight_min": -28.495393753051758,
      "activations/layer23_attention_weight_max": 43.65754318237305,
      "activations/layer23_attention_weight_min": -23.426856994628906,
      "activations/layer2_attention_weight_max": 31.36495018005371,
      "activations/layer2_attention_weight_min": -30.87259292602539,
      "activations/layer3_attention_weight_max": 90.33203887939453,
      "activations/layer3_attention_weight_min": -91.24490356445312,
      "activations/layer4_attention_weight_max": 90.3399887084961,
      "activations/layer4_attention_weight_min": -91.2733383178711,
      "activations/layer5_attention_weight_max": 67.61121368408203,
      "activations/layer5_attention_weight_min": -74.99050903320312,
      "activations/layer6_attention_weight_max": 53.37791442871094,
      "activations/layer6_attention_weight_min": -54.045284271240234,
      "activations/layer7_attention_weight_max": 63.48745346069336,
      "activations/layer7_attention_weight_min": -63.781982421875,
      "activations/layer8_attention_weight_max": 47.65471267700195,
      "activations/layer8_attention_weight_min": -52.52626419067383,
      "activations/layer9_attention_weight_max": 44.52995681762695,
      "activations/layer9_attention_weight_min": -47.91099166870117,
      "epoch": 15.63,
      "learning_rate": 4.9705303030303025e-05,
      "loss": 2.7626,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.4667,
      "eval_samples_per_second": 507.163,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.4667,
      "eval_openwebtext_samples_per_second": 507.163,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9249,
      "eval_wikitext_samples_per_second": 236.901,
      "step": 269000
    },
    {
      "epoch": 15.63,
      "eval_lambada_loss": 2.60546875,
      "eval_lambada_ppl": 13.537569551425428,
      "eval_lambada_runtime": 9.5904,
      "eval_lambada_samples_per_second": 507.695,
      "step": 269000
    },
    {
      "activations/layer0_attention_weight_max": 16.748722076416016,
      "activations/layer0_attention_weight_min": -14.982818603515625,
      "activations/layer10_attention_weight_max": 33.90083312988281,
      "activations/layer10_attention_weight_min": -34.81525421142578,
      "activations/layer11_attention_weight_max": 30.910186767578125,
      "activations/layer11_attention_weight_min": -33.657630920410156,
      "activations/layer12_attention_weight_max": 20.436079025268555,
      "activations/layer12_attention_weight_min": -24.99321937561035,
      "activations/layer13_attention_weight_max": 31.26778793334961,
      "activations/layer13_attention_weight_min": -28.522998809814453,
      "activations/layer14_attention_weight_max": 35.79216384887695,
      "activations/layer14_attention_weight_min": -32.20374298095703,
      "activations/layer15_attention_weight_max": 31.838464736938477,
      "activations/layer15_attention_weight_min": -33.44121170043945,
      "activations/layer16_attention_weight_max": 34.346473693847656,
      "activations/layer16_attention_weight_min": -31.938236236572266,
      "activations/layer17_attention_weight_max": 53.36919021606445,
      "activations/layer17_attention_weight_min": -44.92013931274414,
      "activations/layer18_attention_weight_max": 45.91437530517578,
      "activations/layer18_attention_weight_min": -39.9990348815918,
      "activations/layer19_attention_weight_max": 22.97490119934082,
      "activations/layer19_attention_weight_min": -24.238080978393555,
      "activations/layer1_attention_weight_max": 16.125524520874023,
      "activations/layer1_attention_weight_min": -18.011152267456055,
      "activations/layer20_attention_weight_max": 23.182788848876953,
      "activations/layer20_attention_weight_min": -22.964624404907227,
      "activations/layer21_attention_weight_max": 40.64630889892578,
      "activations/layer21_attention_weight_min": -23.119768142700195,
      "activations/layer22_attention_weight_max": 28.46502113342285,
      "activations/layer22_attention_weight_min": -26.39113998413086,
      "activations/layer23_attention_weight_max": 35.38683319091797,
      "activations/layer23_attention_weight_min": -24.019193649291992,
      "activations/layer2_attention_weight_max": 32.119346618652344,
      "activations/layer2_attention_weight_min": -31.31964874267578,
      "activations/layer3_attention_weight_max": 89.94844818115234,
      "activations/layer3_attention_weight_min": -95.63134765625,
      "activations/layer4_attention_weight_max": 85.4468002319336,
      "activations/layer4_attention_weight_min": -85.06710052490234,
      "activations/layer5_attention_weight_max": 61.52103805541992,
      "activations/layer5_attention_weight_min": -70.67314147949219,
      "activations/layer6_attention_weight_max": 47.74190902709961,
      "activations/layer6_attention_weight_min": -49.32909393310547,
      "activations/layer7_attention_weight_max": 56.46336364746094,
      "activations/layer7_attention_weight_min": -56.49570846557617,
      "activations/layer8_attention_weight_max": 43.54677200317383,
      "activations/layer8_attention_weight_min": -44.87256622314453,
      "activations/layer9_attention_weight_max": 40.08032989501953,
      "activations/layer9_attention_weight_min": -43.54471206665039,
      "epoch": 15.63,
      "learning_rate": 4.968636363636363e-05,
      "loss": 2.7529,
      "step": 269050
    },
    {
      "activations/layer0_attention_weight_max": 16.1353702545166,
      "activations/layer0_attention_weight_min": -14.899662971496582,
      "activations/layer10_attention_weight_max": 35.99341583251953,
      "activations/layer10_attention_weight_min": -39.10222244262695,
      "activations/layer11_attention_weight_max": 32.48487091064453,
      "activations/layer11_attention_weight_min": -36.65675354003906,
      "activations/layer12_attention_weight_max": 19.535842895507812,
      "activations/layer12_attention_weight_min": -26.954256057739258,
      "activations/layer13_attention_weight_max": 33.34626007080078,
      "activations/layer13_attention_weight_min": -26.283504486083984,
      "activations/layer14_attention_weight_max": 34.982269287109375,
      "activations/layer14_attention_weight_min": -30.244094848632812,
      "activations/layer15_attention_weight_max": 31.016372680664062,
      "activations/layer15_attention_weight_min": -30.98163414001465,
      "activations/layer16_attention_weight_max": 34.01723861694336,
      "activations/layer16_attention_weight_min": -32.700443267822266,
      "activations/layer17_attention_weight_max": 52.129981994628906,
      "activations/layer17_attention_weight_min": -48.366512298583984,
      "activations/layer18_attention_weight_max": 48.160179138183594,
      "activations/layer18_attention_weight_min": -41.17708969116211,
      "activations/layer19_attention_weight_max": 22.695249557495117,
      "activations/layer19_attention_weight_min": -22.805278778076172,
      "activations/layer1_attention_weight_max": 16.605836868286133,
      "activations/layer1_attention_weight_min": -14.41882038116455,
      "activations/layer20_attention_weight_max": 22.104087829589844,
      "activations/layer20_attention_weight_min": -22.942060470581055,
      "activations/layer21_attention_weight_max": 34.626564025878906,
      "activations/layer21_attention_weight_min": -22.964941024780273,
      "activations/layer22_attention_weight_max": 28.475936889648438,
      "activations/layer22_attention_weight_min": -28.58949851989746,
      "activations/layer23_attention_weight_max": 37.839393615722656,
      "activations/layer23_attention_weight_min": -26.79058837890625,
      "activations/layer2_attention_weight_max": 28.529865264892578,
      "activations/layer2_attention_weight_min": -30.064559936523438,
      "activations/layer3_attention_weight_max": 93.20832061767578,
      "activations/layer3_attention_weight_min": -89.16633605957031,
      "activations/layer4_attention_weight_max": 90.04808807373047,
      "activations/layer4_attention_weight_min": -91.1533432006836,
      "activations/layer5_attention_weight_max": 64.68975067138672,
      "activations/layer5_attention_weight_min": -82.32786560058594,
      "activations/layer6_attention_weight_max": 51.27253341674805,
      "activations/layer6_attention_weight_min": -52.52638244628906,
      "activations/layer7_attention_weight_max": 63.4860954284668,
      "activations/layer7_attention_weight_min": -62.36777114868164,
      "activations/layer8_attention_weight_max": 47.20484161376953,
      "activations/layer8_attention_weight_min": -53.5182991027832,
      "activations/layer9_attention_weight_max": 45.98977279663086,
      "activations/layer9_attention_weight_min": -51.6070442199707,
      "epoch": 15.64,
      "learning_rate": 4.966742424242424e-05,
      "loss": 2.7704,
      "step": 269100
    },
    {
      "activations/layer0_attention_weight_max": 15.351860046386719,
      "activations/layer0_attention_weight_min": -15.303451538085938,
      "activations/layer10_attention_weight_max": 35.723880767822266,
      "activations/layer10_attention_weight_min": -34.01809310913086,
      "activations/layer11_attention_weight_max": 30.76327133178711,
      "activations/layer11_attention_weight_min": -31.39251708984375,
      "activations/layer12_attention_weight_max": 20.216413497924805,
      "activations/layer12_attention_weight_min": -24.913082122802734,
      "activations/layer13_attention_weight_max": 30.418792724609375,
      "activations/layer13_attention_weight_min": -28.508010864257812,
      "activations/layer14_attention_weight_max": 33.58679962158203,
      "activations/layer14_attention_weight_min": -29.311941146850586,
      "activations/layer15_attention_weight_max": 34.16884231567383,
      "activations/layer15_attention_weight_min": -28.217267990112305,
      "activations/layer16_attention_weight_max": 30.77082061767578,
      "activations/layer16_attention_weight_min": -30.51453399658203,
      "activations/layer17_attention_weight_max": 47.51443099975586,
      "activations/layer17_attention_weight_min": -41.73386764526367,
      "activations/layer18_attention_weight_max": 44.03993606567383,
      "activations/layer18_attention_weight_min": -38.48854446411133,
      "activations/layer19_attention_weight_max": 22.811412811279297,
      "activations/layer19_attention_weight_min": -24.413118362426758,
      "activations/layer1_attention_weight_max": 15.929707527160645,
      "activations/layer1_attention_weight_min": -13.40610408782959,
      "activations/layer20_attention_weight_max": 23.008649826049805,
      "activations/layer20_attention_weight_min": -24.111263275146484,
      "activations/layer21_attention_weight_max": 33.8912353515625,
      "activations/layer21_attention_weight_min": -22.318923950195312,
      "activations/layer22_attention_weight_max": 27.474470138549805,
      "activations/layer22_attention_weight_min": -28.3561954498291,
      "activations/layer23_attention_weight_max": 33.19480895996094,
      "activations/layer23_attention_weight_min": -28.034223556518555,
      "activations/layer2_attention_weight_max": 30.142974853515625,
      "activations/layer2_attention_weight_min": -30.72907257080078,
      "activations/layer3_attention_weight_max": 85.7873306274414,
      "activations/layer3_attention_weight_min": -87.82818603515625,
      "activations/layer4_attention_weight_max": 90.25593566894531,
      "activations/layer4_attention_weight_min": -87.35408782958984,
      "activations/layer5_attention_weight_max": 61.524471282958984,
      "activations/layer5_attention_weight_min": -76.66873168945312,
      "activations/layer6_attention_weight_max": 49.118507385253906,
      "activations/layer6_attention_weight_min": -51.22378158569336,
      "activations/layer7_attention_weight_max": 57.6085205078125,
      "activations/layer7_attention_weight_min": -58.19418716430664,
      "activations/layer8_attention_weight_max": 45.08682632446289,
      "activations/layer8_attention_weight_min": -47.535186767578125,
      "activations/layer9_attention_weight_max": 41.34122848510742,
      "activations/layer9_attention_weight_min": -43.4480094909668,
      "epoch": 15.64,
      "learning_rate": 4.964848484848484e-05,
      "loss": 2.7647,
      "step": 269150
    },
    {
      "activations/layer0_attention_weight_max": 15.880908012390137,
      "activations/layer0_attention_weight_min": -14.756598472595215,
      "activations/layer10_attention_weight_max": 41.25632858276367,
      "activations/layer10_attention_weight_min": -39.04179382324219,
      "activations/layer11_attention_weight_max": 37.4757194519043,
      "activations/layer11_attention_weight_min": -34.17030715942383,
      "activations/layer12_attention_weight_max": 17.12129020690918,
      "activations/layer12_attention_weight_min": -27.548158645629883,
      "activations/layer13_attention_weight_max": 36.79285430908203,
      "activations/layer13_attention_weight_min": -27.82352066040039,
      "activations/layer14_attention_weight_max": 33.11736297607422,
      "activations/layer14_attention_weight_min": -32.598262786865234,
      "activations/layer15_attention_weight_max": 29.31505012512207,
      "activations/layer15_attention_weight_min": -29.886398315429688,
      "activations/layer16_attention_weight_max": 31.606531143188477,
      "activations/layer16_attention_weight_min": -32.43263626098633,
      "activations/layer17_attention_weight_max": 48.20082092285156,
      "activations/layer17_attention_weight_min": -46.21230697631836,
      "activations/layer18_attention_weight_max": 42.22895050048828,
      "activations/layer18_attention_weight_min": -41.324886322021484,
      "activations/layer19_attention_weight_max": 22.067060470581055,
      "activations/layer19_attention_weight_min": -23.38367462158203,
      "activations/layer1_attention_weight_max": 16.634809494018555,
      "activations/layer1_attention_weight_min": -13.91063404083252,
      "activations/layer20_attention_weight_max": 20.28566551208496,
      "activations/layer20_attention_weight_min": -21.990446090698242,
      "activations/layer21_attention_weight_max": 32.56562423706055,
      "activations/layer21_attention_weight_min": -23.587297439575195,
      "activations/layer22_attention_weight_max": 27.386947631835938,
      "activations/layer22_attention_weight_min": -27.4051570892334,
      "activations/layer23_attention_weight_max": 32.353206634521484,
      "activations/layer23_attention_weight_min": -22.712797164916992,
      "activations/layer2_attention_weight_max": 32.719810485839844,
      "activations/layer2_attention_weight_min": -32.56107711791992,
      "activations/layer3_attention_weight_max": 89.34777069091797,
      "activations/layer3_attention_weight_min": -93.59715270996094,
      "activations/layer4_attention_weight_max": 92.83956146240234,
      "activations/layer4_attention_weight_min": -87.37905883789062,
      "activations/layer5_attention_weight_max": 70.66249084472656,
      "activations/layer5_attention_weight_min": -70.4508056640625,
      "activations/layer6_attention_weight_max": 49.979576110839844,
      "activations/layer6_attention_weight_min": -48.986900329589844,
      "activations/layer7_attention_weight_max": 73.22965240478516,
      "activations/layer7_attention_weight_min": -64.82654571533203,
      "activations/layer8_attention_weight_max": 51.092857360839844,
      "activations/layer8_attention_weight_min": -47.253177642822266,
      "activations/layer9_attention_weight_max": 57.33660888671875,
      "activations/layer9_attention_weight_min": -50.02496337890625,
      "epoch": 15.64,
      "learning_rate": 4.962954545454545e-05,
      "loss": 2.7502,
      "step": 269200
    },
    {
      "activations/layer0_attention_weight_max": 16.66936492919922,
      "activations/layer0_attention_weight_min": -15.201271057128906,
      "activations/layer10_attention_weight_max": 39.27467727661133,
      "activations/layer10_attention_weight_min": -38.40748977661133,
      "activations/layer11_attention_weight_max": 33.89142608642578,
      "activations/layer11_attention_weight_min": -34.70423889160156,
      "activations/layer12_attention_weight_max": 22.532913208007812,
      "activations/layer12_attention_weight_min": -26.794330596923828,
      "activations/layer13_attention_weight_max": 38.77451705932617,
      "activations/layer13_attention_weight_min": -28.82527732849121,
      "activations/layer14_attention_weight_max": 33.115264892578125,
      "activations/layer14_attention_weight_min": -31.68804931640625,
      "activations/layer15_attention_weight_max": 33.548099517822266,
      "activations/layer15_attention_weight_min": -31.682722091674805,
      "activations/layer16_attention_weight_max": 31.650272369384766,
      "activations/layer16_attention_weight_min": -30.176897048950195,
      "activations/layer17_attention_weight_max": 48.3800048828125,
      "activations/layer17_attention_weight_min": -41.7486572265625,
      "activations/layer18_attention_weight_max": 43.098995208740234,
      "activations/layer18_attention_weight_min": -38.02104949951172,
      "activations/layer19_attention_weight_max": 21.46192741394043,
      "activations/layer19_attention_weight_min": -22.042869567871094,
      "activations/layer1_attention_weight_max": 15.426031112670898,
      "activations/layer1_attention_weight_min": -13.286033630371094,
      "activations/layer20_attention_weight_max": 21.785429000854492,
      "activations/layer20_attention_weight_min": -20.632545471191406,
      "activations/layer21_attention_weight_max": 31.01718521118164,
      "activations/layer21_attention_weight_min": -19.432851791381836,
      "activations/layer22_attention_weight_max": 26.325450897216797,
      "activations/layer22_attention_weight_min": -23.86332893371582,
      "activations/layer23_attention_weight_max": 38.02977752685547,
      "activations/layer23_attention_weight_min": -24.043067932128906,
      "activations/layer2_attention_weight_max": 32.270469665527344,
      "activations/layer2_attention_weight_min": -29.968730926513672,
      "activations/layer3_attention_weight_max": 90.8552017211914,
      "activations/layer3_attention_weight_min": -93.2189712524414,
      "activations/layer4_attention_weight_max": 86.58856201171875,
      "activations/layer4_attention_weight_min": -86.42619323730469,
      "activations/layer5_attention_weight_max": 62.47547149658203,
      "activations/layer5_attention_weight_min": -75.06898498535156,
      "activations/layer6_attention_weight_max": 49.95756530761719,
      "activations/layer6_attention_weight_min": -49.45560836791992,
      "activations/layer7_attention_weight_max": 59.74126434326172,
      "activations/layer7_attention_weight_min": -60.23293685913086,
      "activations/layer8_attention_weight_max": 48.00835037231445,
      "activations/layer8_attention_weight_min": -50.28316116333008,
      "activations/layer9_attention_weight_max": 45.39284896850586,
      "activations/layer9_attention_weight_min": -48.76290512084961,
      "epoch": 15.64,
      "learning_rate": 4.961060606060605e-05,
      "loss": 2.7685,
      "step": 269250
    },
    {
      "activations/layer0_attention_weight_max": 15.920405387878418,
      "activations/layer0_attention_weight_min": -14.607888221740723,
      "activations/layer10_attention_weight_max": 34.478633880615234,
      "activations/layer10_attention_weight_min": -34.20655059814453,
      "activations/layer11_attention_weight_max": 32.88572311401367,
      "activations/layer11_attention_weight_min": -33.17436981201172,
      "activations/layer12_attention_weight_max": 19.424287796020508,
      "activations/layer12_attention_weight_min": -26.428808212280273,
      "activations/layer13_attention_weight_max": 33.311851501464844,
      "activations/layer13_attention_weight_min": -28.23208236694336,
      "activations/layer14_attention_weight_max": 35.8192024230957,
      "activations/layer14_attention_weight_min": -29.925649642944336,
      "activations/layer15_attention_weight_max": 32.830684661865234,
      "activations/layer15_attention_weight_min": -29.80982208251953,
      "activations/layer16_attention_weight_max": 34.813987731933594,
      "activations/layer16_attention_weight_min": -31.28550910949707,
      "activations/layer17_attention_weight_max": 50.66780471801758,
      "activations/layer17_attention_weight_min": -45.952003479003906,
      "activations/layer18_attention_weight_max": 49.75782775878906,
      "activations/layer18_attention_weight_min": -39.695865631103516,
      "activations/layer19_attention_weight_max": 23.21120834350586,
      "activations/layer19_attention_weight_min": -21.40937614440918,
      "activations/layer1_attention_weight_max": 16.071319580078125,
      "activations/layer1_attention_weight_min": -13.061922073364258,
      "activations/layer20_attention_weight_max": 21.65328025817871,
      "activations/layer20_attention_weight_min": -21.255252838134766,
      "activations/layer21_attention_weight_max": 41.90918731689453,
      "activations/layer21_attention_weight_min": -22.174091339111328,
      "activations/layer22_attention_weight_max": 31.283512115478516,
      "activations/layer22_attention_weight_min": -27.366668701171875,
      "activations/layer23_attention_weight_max": 41.42819595336914,
      "activations/layer23_attention_weight_min": -23.005817413330078,
      "activations/layer2_attention_weight_max": 30.458938598632812,
      "activations/layer2_attention_weight_min": -28.946361541748047,
      "activations/layer3_attention_weight_max": 85.4637680053711,
      "activations/layer3_attention_weight_min": -83.01193237304688,
      "activations/layer4_attention_weight_max": 85.21916961669922,
      "activations/layer4_attention_weight_min": -84.56476593017578,
      "activations/layer5_attention_weight_max": 63.895416259765625,
      "activations/layer5_attention_weight_min": -76.03350830078125,
      "activations/layer6_attention_weight_max": 50.84518814086914,
      "activations/layer6_attention_weight_min": -49.56709289550781,
      "activations/layer7_attention_weight_max": 61.89826965332031,
      "activations/layer7_attention_weight_min": -57.539466857910156,
      "activations/layer8_attention_weight_max": 45.7166633605957,
      "activations/layer8_attention_weight_min": -47.28133010864258,
      "activations/layer9_attention_weight_max": 46.05007553100586,
      "activations/layer9_attention_weight_min": -44.38004684448242,
      "epoch": 15.65,
      "learning_rate": 4.959166666666667e-05,
      "loss": 2.7658,
      "step": 269300
    },
    {
      "activations/layer0_attention_weight_max": 15.57912826538086,
      "activations/layer0_attention_weight_min": -15.021992683410645,
      "activations/layer10_attention_weight_max": 36.3443717956543,
      "activations/layer10_attention_weight_min": -35.50016403198242,
      "activations/layer11_attention_weight_max": 33.088375091552734,
      "activations/layer11_attention_weight_min": -34.43635559082031,
      "activations/layer12_attention_weight_max": 20.40988540649414,
      "activations/layer12_attention_weight_min": -24.04416275024414,
      "activations/layer13_attention_weight_max": 36.29820251464844,
      "activations/layer13_attention_weight_min": -26.654455184936523,
      "activations/layer14_attention_weight_max": 41.94062805175781,
      "activations/layer14_attention_weight_min": -30.19753646850586,
      "activations/layer15_attention_weight_max": 34.00606155395508,
      "activations/layer15_attention_weight_min": -29.858978271484375,
      "activations/layer16_attention_weight_max": 33.62718200683594,
      "activations/layer16_attention_weight_min": -30.771879196166992,
      "activations/layer17_attention_weight_max": 54.0181999206543,
      "activations/layer17_attention_weight_min": -47.56498336791992,
      "activations/layer18_attention_weight_max": 49.21901321411133,
      "activations/layer18_attention_weight_min": -40.34087371826172,
      "activations/layer19_attention_weight_max": 24.619691848754883,
      "activations/layer19_attention_weight_min": -22.798734664916992,
      "activations/layer1_attention_weight_max": 16.031330108642578,
      "activations/layer1_attention_weight_min": -13.99322509765625,
      "activations/layer20_attention_weight_max": 24.994312286376953,
      "activations/layer20_attention_weight_min": -21.837560653686523,
      "activations/layer21_attention_weight_max": 47.56343460083008,
      "activations/layer21_attention_weight_min": -25.366043090820312,
      "activations/layer22_attention_weight_max": 31.764705657958984,
      "activations/layer22_attention_weight_min": -25.19554901123047,
      "activations/layer23_attention_weight_max": 41.44700622558594,
      "activations/layer23_attention_weight_min": -24.091060638427734,
      "activations/layer2_attention_weight_max": 31.317035675048828,
      "activations/layer2_attention_weight_min": -30.522443771362305,
      "activations/layer3_attention_weight_max": 87.58013153076172,
      "activations/layer3_attention_weight_min": -89.32042694091797,
      "activations/layer4_attention_weight_max": 90.50704956054688,
      "activations/layer4_attention_weight_min": -91.04912567138672,
      "activations/layer5_attention_weight_max": 63.88099670410156,
      "activations/layer5_attention_weight_min": -75.67588806152344,
      "activations/layer6_attention_weight_max": 53.45406723022461,
      "activations/layer6_attention_weight_min": -53.4421501159668,
      "activations/layer7_attention_weight_max": 61.920166015625,
      "activations/layer7_attention_weight_min": -60.88751220703125,
      "activations/layer8_attention_weight_max": 48.91484069824219,
      "activations/layer8_attention_weight_min": -50.93204116821289,
      "activations/layer9_attention_weight_max": 45.32500076293945,
      "activations/layer9_attention_weight_min": -45.025875091552734,
      "epoch": 15.65,
      "learning_rate": 4.957272727272727e-05,
      "loss": 2.7523,
      "step": 269350
    },
    {
      "activations/layer0_attention_weight_max": 15.736518859863281,
      "activations/layer0_attention_weight_min": -14.73012638092041,
      "activations/layer10_attention_weight_max": 37.46820831298828,
      "activations/layer10_attention_weight_min": -36.92630386352539,
      "activations/layer11_attention_weight_max": 35.13023376464844,
      "activations/layer11_attention_weight_min": -36.53128433227539,
      "activations/layer12_attention_weight_max": 19.11115074157715,
      "activations/layer12_attention_weight_min": -26.696962356567383,
      "activations/layer13_attention_weight_max": 37.93052291870117,
      "activations/layer13_attention_weight_min": -32.112770080566406,
      "activations/layer14_attention_weight_max": 36.15492248535156,
      "activations/layer14_attention_weight_min": -31.79808235168457,
      "activations/layer15_attention_weight_max": 31.5400390625,
      "activations/layer15_attention_weight_min": -30.95690155029297,
      "activations/layer16_attention_weight_max": 33.38779067993164,
      "activations/layer16_attention_weight_min": -30.028533935546875,
      "activations/layer17_attention_weight_max": 50.761775970458984,
      "activations/layer17_attention_weight_min": -42.50308609008789,
      "activations/layer18_attention_weight_max": 49.5079231262207,
      "activations/layer18_attention_weight_min": -37.44633102416992,
      "activations/layer19_attention_weight_max": 24.384639739990234,
      "activations/layer19_attention_weight_min": -21.083084106445312,
      "activations/layer1_attention_weight_max": 15.573020935058594,
      "activations/layer1_attention_weight_min": -13.893746376037598,
      "activations/layer20_attention_weight_max": 23.22551727294922,
      "activations/layer20_attention_weight_min": -21.134628295898438,
      "activations/layer21_attention_weight_max": 44.8097038269043,
      "activations/layer21_attention_weight_min": -24.566152572631836,
      "activations/layer22_attention_weight_max": 30.116180419921875,
      "activations/layer22_attention_weight_min": -26.61025619506836,
      "activations/layer23_attention_weight_max": 40.147823333740234,
      "activations/layer23_attention_weight_min": -25.62348175048828,
      "activations/layer2_attention_weight_max": 32.783531188964844,
      "activations/layer2_attention_weight_min": -30.16858673095703,
      "activations/layer3_attention_weight_max": 95.52597045898438,
      "activations/layer3_attention_weight_min": -92.6897964477539,
      "activations/layer4_attention_weight_max": 92.75239562988281,
      "activations/layer4_attention_weight_min": -92.8298568725586,
      "activations/layer5_attention_weight_max": 71.29600524902344,
      "activations/layer5_attention_weight_min": -77.37709045410156,
      "activations/layer6_attention_weight_max": 50.50908279418945,
      "activations/layer6_attention_weight_min": -51.27864456176758,
      "activations/layer7_attention_weight_max": 63.71061325073242,
      "activations/layer7_attention_weight_min": -60.33749771118164,
      "activations/layer8_attention_weight_max": 50.04710388183594,
      "activations/layer8_attention_weight_min": -50.251277923583984,
      "activations/layer9_attention_weight_max": 51.6964111328125,
      "activations/layer9_attention_weight_min": -49.30855178833008,
      "epoch": 15.65,
      "learning_rate": 4.955378787878787e-05,
      "loss": 2.7711,
      "step": 269400
    },
    {
      "activations/layer0_attention_weight_max": 15.754644393920898,
      "activations/layer0_attention_weight_min": -14.266304016113281,
      "activations/layer10_attention_weight_max": 36.69107437133789,
      "activations/layer10_attention_weight_min": -35.415035247802734,
      "activations/layer11_attention_weight_max": 32.281333923339844,
      "activations/layer11_attention_weight_min": -34.06760025024414,
      "activations/layer12_attention_weight_max": 18.304845809936523,
      "activations/layer12_attention_weight_min": -25.903791427612305,
      "activations/layer13_attention_weight_max": 33.197261810302734,
      "activations/layer13_attention_weight_min": -27.665695190429688,
      "activations/layer14_attention_weight_max": 35.06509017944336,
      "activations/layer14_attention_weight_min": -31.230201721191406,
      "activations/layer15_attention_weight_max": 32.127159118652344,
      "activations/layer15_attention_weight_min": -31.907468795776367,
      "activations/layer16_attention_weight_max": 33.50014877319336,
      "activations/layer16_attention_weight_min": -32.66328811645508,
      "activations/layer17_attention_weight_max": 54.418514251708984,
      "activations/layer17_attention_weight_min": -49.385093688964844,
      "activations/layer18_attention_weight_max": 48.20213317871094,
      "activations/layer18_attention_weight_min": -39.96942138671875,
      "activations/layer19_attention_weight_max": 24.92407989501953,
      "activations/layer19_attention_weight_min": -23.056711196899414,
      "activations/layer1_attention_weight_max": 17.065982818603516,
      "activations/layer1_attention_weight_min": -14.580286026000977,
      "activations/layer20_attention_weight_max": 23.821447372436523,
      "activations/layer20_attention_weight_min": -21.554763793945312,
      "activations/layer21_attention_weight_max": 39.15262222290039,
      "activations/layer21_attention_weight_min": -22.125415802001953,
      "activations/layer22_attention_weight_max": 30.87308692932129,
      "activations/layer22_attention_weight_min": -24.787378311157227,
      "activations/layer23_attention_weight_max": 43.87957763671875,
      "activations/layer23_attention_weight_min": -23.719900131225586,
      "activations/layer2_attention_weight_max": 32.00586700439453,
      "activations/layer2_attention_weight_min": -30.43090057373047,
      "activations/layer3_attention_weight_max": 86.75492858886719,
      "activations/layer3_attention_weight_min": -91.33424377441406,
      "activations/layer4_attention_weight_max": 90.8361587524414,
      "activations/layer4_attention_weight_min": -87.24201202392578,
      "activations/layer5_attention_weight_max": 68.49832153320312,
      "activations/layer5_attention_weight_min": -77.08307647705078,
      "activations/layer6_attention_weight_max": 51.34791564941406,
      "activations/layer6_attention_weight_min": -53.194400787353516,
      "activations/layer7_attention_weight_max": 66.09911346435547,
      "activations/layer7_attention_weight_min": -59.43110275268555,
      "activations/layer8_attention_weight_max": 44.953617095947266,
      "activations/layer8_attention_weight_min": -45.925941467285156,
      "activations/layer9_attention_weight_max": 46.146034240722656,
      "activations/layer9_attention_weight_min": -44.60090637207031,
      "epoch": 15.66,
      "learning_rate": 4.953484848484848e-05,
      "loss": 2.7571,
      "step": 269450
    },
    {
      "activations/layer0_attention_weight_max": 15.23111629486084,
      "activations/layer0_attention_weight_min": -14.44450855255127,
      "activations/layer10_attention_weight_max": 40.5058479309082,
      "activations/layer10_attention_weight_min": -36.71809387207031,
      "activations/layer11_attention_weight_max": 37.12786865234375,
      "activations/layer11_attention_weight_min": -35.19507598876953,
      "activations/layer12_attention_weight_max": 20.234619140625,
      "activations/layer12_attention_weight_min": -23.62028694152832,
      "activations/layer13_attention_weight_max": 38.40815734863281,
      "activations/layer13_attention_weight_min": -27.13437271118164,
      "activations/layer14_attention_weight_max": 38.05599594116211,
      "activations/layer14_attention_weight_min": -30.09107208251953,
      "activations/layer15_attention_weight_max": 32.20021057128906,
      "activations/layer15_attention_weight_min": -29.57094383239746,
      "activations/layer16_attention_weight_max": 32.70669937133789,
      "activations/layer16_attention_weight_min": -31.130586624145508,
      "activations/layer17_attention_weight_max": 49.652462005615234,
      "activations/layer17_attention_weight_min": -45.23321533203125,
      "activations/layer18_attention_weight_max": 45.070762634277344,
      "activations/layer18_attention_weight_min": -38.10601806640625,
      "activations/layer19_attention_weight_max": 22.810516357421875,
      "activations/layer19_attention_weight_min": -21.7761173248291,
      "activations/layer1_attention_weight_max": 16.82558822631836,
      "activations/layer1_attention_weight_min": -15.3327054977417,
      "activations/layer20_attention_weight_max": 21.923593521118164,
      "activations/layer20_attention_weight_min": -22.576251983642578,
      "activations/layer21_attention_weight_max": 32.82724380493164,
      "activations/layer21_attention_weight_min": -21.112064361572266,
      "activations/layer22_attention_weight_max": 28.09991455078125,
      "activations/layer22_attention_weight_min": -26.036897659301758,
      "activations/layer23_attention_weight_max": 36.90299606323242,
      "activations/layer23_attention_weight_min": -23.7891902923584,
      "activations/layer2_attention_weight_max": 31.389320373535156,
      "activations/layer2_attention_weight_min": -31.633460998535156,
      "activations/layer3_attention_weight_max": 92.9229965209961,
      "activations/layer3_attention_weight_min": -92.25993347167969,
      "activations/layer4_attention_weight_max": 90.62281799316406,
      "activations/layer4_attention_weight_min": -90.1779556274414,
      "activations/layer5_attention_weight_max": 66.82392883300781,
      "activations/layer5_attention_weight_min": -74.74898529052734,
      "activations/layer6_attention_weight_max": 50.51032257080078,
      "activations/layer6_attention_weight_min": -53.141334533691406,
      "activations/layer7_attention_weight_max": 61.983036041259766,
      "activations/layer7_attention_weight_min": -64.07452392578125,
      "activations/layer8_attention_weight_max": 50.54655838012695,
      "activations/layer8_attention_weight_min": -55.04844284057617,
      "activations/layer9_attention_weight_max": 51.45307540893555,
      "activations/layer9_attention_weight_min": -46.925994873046875,
      "epoch": 15.66,
      "learning_rate": 4.951590909090909e-05,
      "loss": 2.7522,
      "step": 269500
    },
    {
      "activations/layer0_attention_weight_max": 15.408607482910156,
      "activations/layer0_attention_weight_min": -14.812360763549805,
      "activations/layer10_attention_weight_max": 33.06584548950195,
      "activations/layer10_attention_weight_min": -34.61524963378906,
      "activations/layer11_attention_weight_max": 30.003087997436523,
      "activations/layer11_attention_weight_min": -31.04102325439453,
      "activations/layer12_attention_weight_max": 19.31149673461914,
      "activations/layer12_attention_weight_min": -25.928409576416016,
      "activations/layer13_attention_weight_max": 32.14551544189453,
      "activations/layer13_attention_weight_min": -28.081588745117188,
      "activations/layer14_attention_weight_max": 33.60780715942383,
      "activations/layer14_attention_weight_min": -29.514759063720703,
      "activations/layer15_attention_weight_max": 30.761377334594727,
      "activations/layer15_attention_weight_min": -29.46746253967285,
      "activations/layer16_attention_weight_max": 31.678503036499023,
      "activations/layer16_attention_weight_min": -32.62465286254883,
      "activations/layer17_attention_weight_max": 50.14189529418945,
      "activations/layer17_attention_weight_min": -45.85447692871094,
      "activations/layer18_attention_weight_max": 45.36070251464844,
      "activations/layer18_attention_weight_min": -39.9127197265625,
      "activations/layer19_attention_weight_max": 23.55223846435547,
      "activations/layer19_attention_weight_min": -24.267675399780273,
      "activations/layer1_attention_weight_max": 15.398895263671875,
      "activations/layer1_attention_weight_min": -12.424752235412598,
      "activations/layer20_attention_weight_max": 22.98467254638672,
      "activations/layer20_attention_weight_min": -21.291465759277344,
      "activations/layer21_attention_weight_max": 34.513458251953125,
      "activations/layer21_attention_weight_min": -21.472822189331055,
      "activations/layer22_attention_weight_max": 27.507532119750977,
      "activations/layer22_attention_weight_min": -26.827985763549805,
      "activations/layer23_attention_weight_max": 34.937767028808594,
      "activations/layer23_attention_weight_min": -25.299091339111328,
      "activations/layer2_attention_weight_max": 28.88907241821289,
      "activations/layer2_attention_weight_min": -28.589553833007812,
      "activations/layer3_attention_weight_max": 84.97056579589844,
      "activations/layer3_attention_weight_min": -83.03907012939453,
      "activations/layer4_attention_weight_max": 86.56613159179688,
      "activations/layer4_attention_weight_min": -82.8928451538086,
      "activations/layer5_attention_weight_max": 65.86563110351562,
      "activations/layer5_attention_weight_min": -68.92324829101562,
      "activations/layer6_attention_weight_max": 49.19252014160156,
      "activations/layer6_attention_weight_min": -48.594825744628906,
      "activations/layer7_attention_weight_max": 58.22542953491211,
      "activations/layer7_attention_weight_min": -56.366641998291016,
      "activations/layer8_attention_weight_max": 43.04299545288086,
      "activations/layer8_attention_weight_min": -48.00215530395508,
      "activations/layer9_attention_weight_max": 42.927425384521484,
      "activations/layer9_attention_weight_min": -42.72811508178711,
      "epoch": 15.66,
      "learning_rate": 4.9496969696969696e-05,
      "loss": 2.7549,
      "step": 269550
    },
    {
      "activations/layer0_attention_weight_max": 15.676618576049805,
      "activations/layer0_attention_weight_min": -14.163374900817871,
      "activations/layer10_attention_weight_max": 37.40242004394531,
      "activations/layer10_attention_weight_min": -35.93728256225586,
      "activations/layer11_attention_weight_max": 31.40692138671875,
      "activations/layer11_attention_weight_min": -33.673248291015625,
      "activations/layer12_attention_weight_max": 17.968914031982422,
      "activations/layer12_attention_weight_min": -23.89764976501465,
      "activations/layer13_attention_weight_max": 35.79609298706055,
      "activations/layer13_attention_weight_min": -26.448286056518555,
      "activations/layer14_attention_weight_max": 34.881874084472656,
      "activations/layer14_attention_weight_min": -32.32480239868164,
      "activations/layer15_attention_weight_max": 32.98308563232422,
      "activations/layer15_attention_weight_min": -30.96996307373047,
      "activations/layer16_attention_weight_max": 33.11050796508789,
      "activations/layer16_attention_weight_min": -32.04487991333008,
      "activations/layer17_attention_weight_max": 51.83955001831055,
      "activations/layer17_attention_weight_min": -44.57460403442383,
      "activations/layer18_attention_weight_max": 44.67306137084961,
      "activations/layer18_attention_weight_min": -38.248992919921875,
      "activations/layer19_attention_weight_max": 23.51256561279297,
      "activations/layer19_attention_weight_min": -24.045166015625,
      "activations/layer1_attention_weight_max": 15.984861373901367,
      "activations/layer1_attention_weight_min": -12.538466453552246,
      "activations/layer20_attention_weight_max": 22.961830139160156,
      "activations/layer20_attention_weight_min": -20.93206024169922,
      "activations/layer21_attention_weight_max": 33.58179473876953,
      "activations/layer21_attention_weight_min": -22.23314666748047,
      "activations/layer22_attention_weight_max": 29.42859649658203,
      "activations/layer22_attention_weight_min": -25.31083106994629,
      "activations/layer23_attention_weight_max": 37.39356231689453,
      "activations/layer23_attention_weight_min": -22.465923309326172,
      "activations/layer2_attention_weight_max": 31.694509506225586,
      "activations/layer2_attention_weight_min": -29.00777816772461,
      "activations/layer3_attention_weight_max": 85.66806030273438,
      "activations/layer3_attention_weight_min": -87.6547622680664,
      "activations/layer4_attention_weight_max": 86.21746063232422,
      "activations/layer4_attention_weight_min": -84.56060028076172,
      "activations/layer5_attention_weight_max": 63.61449432373047,
      "activations/layer5_attention_weight_min": -76.5871810913086,
      "activations/layer6_attention_weight_max": 48.47737503051758,
      "activations/layer6_attention_weight_min": -48.769248962402344,
      "activations/layer7_attention_weight_max": 58.13322448730469,
      "activations/layer7_attention_weight_min": -58.28336715698242,
      "activations/layer8_attention_weight_max": 45.20387268066406,
      "activations/layer8_attention_weight_min": -45.99373245239258,
      "activations/layer9_attention_weight_max": 41.394466400146484,
      "activations/layer9_attention_weight_min": -44.294898986816406,
      "epoch": 15.67,
      "learning_rate": 4.94780303030303e-05,
      "loss": 2.758,
      "step": 269600
    },
    {
      "activations/layer0_attention_weight_max": 16.5523624420166,
      "activations/layer0_attention_weight_min": -14.0787353515625,
      "activations/layer10_attention_weight_max": 36.04983901977539,
      "activations/layer10_attention_weight_min": -36.95521545410156,
      "activations/layer11_attention_weight_max": 33.7313232421875,
      "activations/layer11_attention_weight_min": -34.19060134887695,
      "activations/layer12_attention_weight_max": 18.905637741088867,
      "activations/layer12_attention_weight_min": -22.909305572509766,
      "activations/layer13_attention_weight_max": 40.68817138671875,
      "activations/layer13_attention_weight_min": -28.80582618713379,
      "activations/layer14_attention_weight_max": 37.12854766845703,
      "activations/layer14_attention_weight_min": -29.044939041137695,
      "activations/layer15_attention_weight_max": 38.35078430175781,
      "activations/layer15_attention_weight_min": -29.391969680786133,
      "activations/layer16_attention_weight_max": 39.07228469848633,
      "activations/layer16_attention_weight_min": -35.50431442260742,
      "activations/layer17_attention_weight_max": 56.703758239746094,
      "activations/layer17_attention_weight_min": -48.34672927856445,
      "activations/layer18_attention_weight_max": 51.487762451171875,
      "activations/layer18_attention_weight_min": -42.243507385253906,
      "activations/layer19_attention_weight_max": 27.610334396362305,
      "activations/layer19_attention_weight_min": -24.462299346923828,
      "activations/layer1_attention_weight_max": 16.715381622314453,
      "activations/layer1_attention_weight_min": -13.4537992477417,
      "activations/layer20_attention_weight_max": 27.49413299560547,
      "activations/layer20_attention_weight_min": -21.742443084716797,
      "activations/layer21_attention_weight_max": 37.76091384887695,
      "activations/layer21_attention_weight_min": -22.2206974029541,
      "activations/layer22_attention_weight_max": 32.489620208740234,
      "activations/layer22_attention_weight_min": -27.012285232543945,
      "activations/layer23_attention_weight_max": 41.6906623840332,
      "activations/layer23_attention_weight_min": -23.57611846923828,
      "activations/layer2_attention_weight_max": 33.33995819091797,
      "activations/layer2_attention_weight_min": -32.156280517578125,
      "activations/layer3_attention_weight_max": 92.43948364257812,
      "activations/layer3_attention_weight_min": -90.24390411376953,
      "activations/layer4_attention_weight_max": 91.17916107177734,
      "activations/layer4_attention_weight_min": -89.5234603881836,
      "activations/layer5_attention_weight_max": 65.51127624511719,
      "activations/layer5_attention_weight_min": -76.89117431640625,
      "activations/layer6_attention_weight_max": 52.311885833740234,
      "activations/layer6_attention_weight_min": -54.61040496826172,
      "activations/layer7_attention_weight_max": 63.07666778564453,
      "activations/layer7_attention_weight_min": -62.90753936767578,
      "activations/layer8_attention_weight_max": 48.89051055908203,
      "activations/layer8_attention_weight_min": -47.8326416015625,
      "activations/layer9_attention_weight_max": 53.6016960144043,
      "activations/layer9_attention_weight_min": -47.98335647583008,
      "epoch": 15.67,
      "learning_rate": 4.94590909090909e-05,
      "loss": 2.7415,
      "step": 269650
    },
    {
      "activations/layer0_attention_weight_max": 15.531437873840332,
      "activations/layer0_attention_weight_min": -14.564517974853516,
      "activations/layer10_attention_weight_max": 31.904325485229492,
      "activations/layer10_attention_weight_min": -33.23006057739258,
      "activations/layer11_attention_weight_max": 29.58184051513672,
      "activations/layer11_attention_weight_min": -31.13754653930664,
      "activations/layer12_attention_weight_max": 18.67862892150879,
      "activations/layer12_attention_weight_min": -28.406095504760742,
      "activations/layer13_attention_weight_max": 30.54596519470215,
      "activations/layer13_attention_weight_min": -28.56158447265625,
      "activations/layer14_attention_weight_max": 35.33867263793945,
      "activations/layer14_attention_weight_min": -28.9465389251709,
      "activations/layer15_attention_weight_max": 32.404029846191406,
      "activations/layer15_attention_weight_min": -29.633968353271484,
      "activations/layer16_attention_weight_max": 30.362075805664062,
      "activations/layer16_attention_weight_min": -29.181779861450195,
      "activations/layer17_attention_weight_max": 47.801727294921875,
      "activations/layer17_attention_weight_min": -43.455753326416016,
      "activations/layer18_attention_weight_max": 43.28085708618164,
      "activations/layer18_attention_weight_min": -39.023590087890625,
      "activations/layer19_attention_weight_max": 25.169599533081055,
      "activations/layer19_attention_weight_min": -21.57941246032715,
      "activations/layer1_attention_weight_max": 15.990738868713379,
      "activations/layer1_attention_weight_min": -14.388307571411133,
      "activations/layer20_attention_weight_max": 23.050878524780273,
      "activations/layer20_attention_weight_min": -22.649335861206055,
      "activations/layer21_attention_weight_max": 33.07100296020508,
      "activations/layer21_attention_weight_min": -22.254791259765625,
      "activations/layer22_attention_weight_max": 30.308156967163086,
      "activations/layer22_attention_weight_min": -25.159650802612305,
      "activations/layer23_attention_weight_max": 38.53375244140625,
      "activations/layer23_attention_weight_min": -23.247802734375,
      "activations/layer2_attention_weight_max": 31.728160858154297,
      "activations/layer2_attention_weight_min": -30.83069610595703,
      "activations/layer3_attention_weight_max": 87.04403686523438,
      "activations/layer3_attention_weight_min": -89.64672088623047,
      "activations/layer4_attention_weight_max": 86.29700469970703,
      "activations/layer4_attention_weight_min": -85.90715789794922,
      "activations/layer5_attention_weight_max": 62.01765060424805,
      "activations/layer5_attention_weight_min": -70.6278076171875,
      "activations/layer6_attention_weight_max": 46.84028244018555,
      "activations/layer6_attention_weight_min": -47.817134857177734,
      "activations/layer7_attention_weight_max": 57.58787536621094,
      "activations/layer7_attention_weight_min": -56.52155685424805,
      "activations/layer8_attention_weight_max": 41.76596450805664,
      "activations/layer8_attention_weight_min": -45.039424896240234,
      "activations/layer9_attention_weight_max": 41.04508590698242,
      "activations/layer9_attention_weight_min": -41.20954132080078,
      "epoch": 15.67,
      "learning_rate": 4.9440151515151514e-05,
      "loss": 2.7786,
      "step": 269700
    },
    {
      "activations/layer0_attention_weight_max": 15.860563278198242,
      "activations/layer0_attention_weight_min": -13.744070053100586,
      "activations/layer10_attention_weight_max": 46.33142852783203,
      "activations/layer10_attention_weight_min": -43.45462417602539,
      "activations/layer11_attention_weight_max": 43.38311004638672,
      "activations/layer11_attention_weight_min": -40.42426300048828,
      "activations/layer12_attention_weight_max": 19.49959373474121,
      "activations/layer12_attention_weight_min": -28.26118278503418,
      "activations/layer13_attention_weight_max": 44.68619155883789,
      "activations/layer13_attention_weight_min": -31.8087215423584,
      "activations/layer14_attention_weight_max": 48.93991470336914,
      "activations/layer14_attention_weight_min": -32.37752914428711,
      "activations/layer15_attention_weight_max": 47.069942474365234,
      "activations/layer15_attention_weight_min": -32.12626647949219,
      "activations/layer16_attention_weight_max": 41.89086151123047,
      "activations/layer16_attention_weight_min": -35.17650604248047,
      "activations/layer17_attention_weight_max": 63.29582595825195,
      "activations/layer17_attention_weight_min": -50.44458770751953,
      "activations/layer18_attention_weight_max": 57.44374465942383,
      "activations/layer18_attention_weight_min": -44.780338287353516,
      "activations/layer19_attention_weight_max": 27.528461456298828,
      "activations/layer19_attention_weight_min": -23.762662887573242,
      "activations/layer1_attention_weight_max": 18.086591720581055,
      "activations/layer1_attention_weight_min": -16.889354705810547,
      "activations/layer20_attention_weight_max": 28.288299560546875,
      "activations/layer20_attention_weight_min": -22.063840866088867,
      "activations/layer21_attention_weight_max": 43.655975341796875,
      "activations/layer21_attention_weight_min": -25.37151336669922,
      "activations/layer22_attention_weight_max": 31.39131736755371,
      "activations/layer22_attention_weight_min": -25.09511375427246,
      "activations/layer23_attention_weight_max": 42.06352233886719,
      "activations/layer23_attention_weight_min": -23.549121856689453,
      "activations/layer2_attention_weight_max": 35.54802703857422,
      "activations/layer2_attention_weight_min": -35.81999206542969,
      "activations/layer3_attention_weight_max": 106.49870300292969,
      "activations/layer3_attention_weight_min": -106.81712341308594,
      "activations/layer4_attention_weight_max": 101.04685974121094,
      "activations/layer4_attention_weight_min": -97.64627838134766,
      "activations/layer5_attention_weight_max": 66.30220031738281,
      "activations/layer5_attention_weight_min": -72.656005859375,
      "activations/layer6_attention_weight_max": 55.24910354614258,
      "activations/layer6_attention_weight_min": -54.526023864746094,
      "activations/layer7_attention_weight_max": 65.73283386230469,
      "activations/layer7_attention_weight_min": -63.93056869506836,
      "activations/layer8_attention_weight_max": 54.254432678222656,
      "activations/layer8_attention_weight_min": -54.95912551879883,
      "activations/layer9_attention_weight_max": 56.0484504699707,
      "activations/layer9_attention_weight_min": -54.693145751953125,
      "epoch": 15.67,
      "learning_rate": 4.9421212121212115e-05,
      "loss": 2.7593,
      "step": 269750
    },
    {
      "activations/layer0_attention_weight_max": 15.507258415222168,
      "activations/layer0_attention_weight_min": -13.732490539550781,
      "activations/layer10_attention_weight_max": 34.546268463134766,
      "activations/layer10_attention_weight_min": -36.31817626953125,
      "activations/layer11_attention_weight_max": 32.8914680480957,
      "activations/layer11_attention_weight_min": -34.97136688232422,
      "activations/layer12_attention_weight_max": 19.415769577026367,
      "activations/layer12_attention_weight_min": -26.914993286132812,
      "activations/layer13_attention_weight_max": 28.994470596313477,
      "activations/layer13_attention_weight_min": -31.0259952545166,
      "activations/layer14_attention_weight_max": 31.930360794067383,
      "activations/layer14_attention_weight_min": -31.87558937072754,
      "activations/layer15_attention_weight_max": 28.975645065307617,
      "activations/layer15_attention_weight_min": -31.39274024963379,
      "activations/layer16_attention_weight_max": 31.24822235107422,
      "activations/layer16_attention_weight_min": -30.407169342041016,
      "activations/layer17_attention_weight_max": 48.84079360961914,
      "activations/layer17_attention_weight_min": -45.112266540527344,
      "activations/layer18_attention_weight_max": 43.19383239746094,
      "activations/layer18_attention_weight_min": -39.114540100097656,
      "activations/layer19_attention_weight_max": 22.277973175048828,
      "activations/layer19_attention_weight_min": -24.13526725769043,
      "activations/layer1_attention_weight_max": 19.368942260742188,
      "activations/layer1_attention_weight_min": -14.00577163696289,
      "activations/layer20_attention_weight_max": 19.17227554321289,
      "activations/layer20_attention_weight_min": -25.182998657226562,
      "activations/layer21_attention_weight_max": 31.549320220947266,
      "activations/layer21_attention_weight_min": -22.6839542388916,
      "activations/layer22_attention_weight_max": 25.505884170532227,
      "activations/layer22_attention_weight_min": -27.882062911987305,
      "activations/layer23_attention_weight_max": 33.651344299316406,
      "activations/layer23_attention_weight_min": -25.61640739440918,
      "activations/layer2_attention_weight_max": 31.71587371826172,
      "activations/layer2_attention_weight_min": -30.73996353149414,
      "activations/layer3_attention_weight_max": 87.1015625,
      "activations/layer3_attention_weight_min": -84.44200897216797,
      "activations/layer4_attention_weight_max": 91.40605926513672,
      "activations/layer4_attention_weight_min": -90.95428466796875,
      "activations/layer5_attention_weight_max": 68.11116027832031,
      "activations/layer5_attention_weight_min": -75.18196105957031,
      "activations/layer6_attention_weight_max": 54.884220123291016,
      "activations/layer6_attention_weight_min": -55.50516891479492,
      "activations/layer7_attention_weight_max": 61.68418884277344,
      "activations/layer7_attention_weight_min": -64.55711364746094,
      "activations/layer8_attention_weight_max": 47.63166046142578,
      "activations/layer8_attention_weight_min": -49.45857620239258,
      "activations/layer9_attention_weight_max": 45.09407043457031,
      "activations/layer9_attention_weight_min": -47.079322814941406,
      "epoch": 15.68,
      "learning_rate": 4.9402272727272724e-05,
      "loss": 2.7645,
      "step": 269800
    },
    {
      "activations/layer0_attention_weight_max": 16.233165740966797,
      "activations/layer0_attention_weight_min": -13.402167320251465,
      "activations/layer10_attention_weight_max": 36.46802520751953,
      "activations/layer10_attention_weight_min": -36.1868896484375,
      "activations/layer11_attention_weight_max": 32.759185791015625,
      "activations/layer11_attention_weight_min": -32.86338806152344,
      "activations/layer12_attention_weight_max": 16.879201889038086,
      "activations/layer12_attention_weight_min": -25.587303161621094,
      "activations/layer13_attention_weight_max": 33.52047348022461,
      "activations/layer13_attention_weight_min": -29.227462768554688,
      "activations/layer14_attention_weight_max": 32.62852096557617,
      "activations/layer14_attention_weight_min": -32.0098762512207,
      "activations/layer15_attention_weight_max": 30.297229766845703,
      "activations/layer15_attention_weight_min": -31.929962158203125,
      "activations/layer16_attention_weight_max": 29.080530166625977,
      "activations/layer16_attention_weight_min": -32.812156677246094,
      "activations/layer17_attention_weight_max": 48.47962188720703,
      "activations/layer17_attention_weight_min": -45.05838394165039,
      "activations/layer18_attention_weight_max": 42.40359878540039,
      "activations/layer18_attention_weight_min": -39.34843444824219,
      "activations/layer19_attention_weight_max": 21.524784088134766,
      "activations/layer19_attention_weight_min": -22.6585750579834,
      "activations/layer1_attention_weight_max": 16.64518165588379,
      "activations/layer1_attention_weight_min": -13.832761764526367,
      "activations/layer20_attention_weight_max": 21.32735252380371,
      "activations/layer20_attention_weight_min": -23.27773666381836,
      "activations/layer21_attention_weight_max": 31.814945220947266,
      "activations/layer21_attention_weight_min": -22.220754623413086,
      "activations/layer22_attention_weight_max": 27.530988693237305,
      "activations/layer22_attention_weight_min": -26.267047882080078,
      "activations/layer23_attention_weight_max": 37.570777893066406,
      "activations/layer23_attention_weight_min": -26.827892303466797,
      "activations/layer2_attention_weight_max": 33.24464416503906,
      "activations/layer2_attention_weight_min": -30.52813720703125,
      "activations/layer3_attention_weight_max": 84.11973571777344,
      "activations/layer3_attention_weight_min": -85.72191619873047,
      "activations/layer4_attention_weight_max": 82.5029296875,
      "activations/layer4_attention_weight_min": -82.0093994140625,
      "activations/layer5_attention_weight_max": 63.443626403808594,
      "activations/layer5_attention_weight_min": -67.93885803222656,
      "activations/layer6_attention_weight_max": 47.6494026184082,
      "activations/layer6_attention_weight_min": -48.14632797241211,
      "activations/layer7_attention_weight_max": 55.493045806884766,
      "activations/layer7_attention_weight_min": -56.856651306152344,
      "activations/layer8_attention_weight_max": 44.01839065551758,
      "activations/layer8_attention_weight_min": -47.5004997253418,
      "activations/layer9_attention_weight_max": 44.07206726074219,
      "activations/layer9_attention_weight_min": -43.43388366699219,
      "epoch": 15.68,
      "learning_rate": 4.938333333333333e-05,
      "loss": 2.743,
      "step": 269850
    },
    {
      "activations/layer0_attention_weight_max": 16.508235931396484,
      "activations/layer0_attention_weight_min": -13.596855163574219,
      "activations/layer10_attention_weight_max": 36.98551940917969,
      "activations/layer10_attention_weight_min": -35.951515197753906,
      "activations/layer11_attention_weight_max": 33.64986801147461,
      "activations/layer11_attention_weight_min": -34.75138854980469,
      "activations/layer12_attention_weight_max": 20.322790145874023,
      "activations/layer12_attention_weight_min": -25.102853775024414,
      "activations/layer13_attention_weight_max": 31.690898895263672,
      "activations/layer13_attention_weight_min": -28.071950912475586,
      "activations/layer14_attention_weight_max": 33.90093231201172,
      "activations/layer14_attention_weight_min": -30.538942337036133,
      "activations/layer15_attention_weight_max": 32.44512176513672,
      "activations/layer15_attention_weight_min": -31.743871688842773,
      "activations/layer16_attention_weight_max": 33.82979202270508,
      "activations/layer16_attention_weight_min": -30.27814483642578,
      "activations/layer17_attention_weight_max": 54.69129180908203,
      "activations/layer17_attention_weight_min": -45.15550994873047,
      "activations/layer18_attention_weight_max": 48.84672164916992,
      "activations/layer18_attention_weight_min": -41.05193328857422,
      "activations/layer19_attention_weight_max": 23.124753952026367,
      "activations/layer19_attention_weight_min": -21.981861114501953,
      "activations/layer1_attention_weight_max": 18.21306037902832,
      "activations/layer1_attention_weight_min": -17.299087524414062,
      "activations/layer20_attention_weight_max": 23.39903450012207,
      "activations/layer20_attention_weight_min": -23.344209671020508,
      "activations/layer21_attention_weight_max": 41.52967071533203,
      "activations/layer21_attention_weight_min": -23.07179069519043,
      "activations/layer22_attention_weight_max": 32.118289947509766,
      "activations/layer22_attention_weight_min": -27.15020751953125,
      "activations/layer23_attention_weight_max": 42.35353088378906,
      "activations/layer23_attention_weight_min": -22.846181869506836,
      "activations/layer2_attention_weight_max": 34.17976760864258,
      "activations/layer2_attention_weight_min": -31.7220458984375,
      "activations/layer3_attention_weight_max": 85.5636215209961,
      "activations/layer3_attention_weight_min": -85.43983459472656,
      "activations/layer4_attention_weight_max": 87.08767700195312,
      "activations/layer4_attention_weight_min": -83.49481201171875,
      "activations/layer5_attention_weight_max": 66.69828033447266,
      "activations/layer5_attention_weight_min": -73.91156768798828,
      "activations/layer6_attention_weight_max": 54.628929138183594,
      "activations/layer6_attention_weight_min": -52.05213165283203,
      "activations/layer7_attention_weight_max": 62.870574951171875,
      "activations/layer7_attention_weight_min": -61.686973571777344,
      "activations/layer8_attention_weight_max": 48.58393478393555,
      "activations/layer8_attention_weight_min": -51.68806838989258,
      "activations/layer9_attention_weight_max": 46.495941162109375,
      "activations/layer9_attention_weight_min": -47.24663543701172,
      "epoch": 15.68,
      "learning_rate": 4.936439393939394e-05,
      "loss": 2.7559,
      "step": 269900
    },
    {
      "activations/layer0_attention_weight_max": 15.605157852172852,
      "activations/layer0_attention_weight_min": -14.318180084228516,
      "activations/layer10_attention_weight_max": 34.97552490234375,
      "activations/layer10_attention_weight_min": -35.17824172973633,
      "activations/layer11_attention_weight_max": 29.353660583496094,
      "activations/layer11_attention_weight_min": -33.51836395263672,
      "activations/layer12_attention_weight_max": 19.490833282470703,
      "activations/layer12_attention_weight_min": -22.05769157409668,
      "activations/layer13_attention_weight_max": 28.757081985473633,
      "activations/layer13_attention_weight_min": -29.005144119262695,
      "activations/layer14_attention_weight_max": 33.695701599121094,
      "activations/layer14_attention_weight_min": -29.87248420715332,
      "activations/layer15_attention_weight_max": 29.788328170776367,
      "activations/layer15_attention_weight_min": -31.087459564208984,
      "activations/layer16_attention_weight_max": 32.19614791870117,
      "activations/layer16_attention_weight_min": -33.531219482421875,
      "activations/layer17_attention_weight_max": 47.56477355957031,
      "activations/layer17_attention_weight_min": -45.150455474853516,
      "activations/layer18_attention_weight_max": 41.83788299560547,
      "activations/layer18_attention_weight_min": -37.8869514465332,
      "activations/layer19_attention_weight_max": 22.451904296875,
      "activations/layer19_attention_weight_min": -23.074996948242188,
      "activations/layer1_attention_weight_max": 16.77948760986328,
      "activations/layer1_attention_weight_min": -16.377033233642578,
      "activations/layer20_attention_weight_max": 21.661842346191406,
      "activations/layer20_attention_weight_min": -23.164154052734375,
      "activations/layer21_attention_weight_max": 32.18756866455078,
      "activations/layer21_attention_weight_min": -22.08302879333496,
      "activations/layer22_attention_weight_max": 30.385114669799805,
      "activations/layer22_attention_weight_min": -26.85284423828125,
      "activations/layer23_attention_weight_max": 39.20633316040039,
      "activations/layer23_attention_weight_min": -25.09723472595215,
      "activations/layer2_attention_weight_max": 31.492694854736328,
      "activations/layer2_attention_weight_min": -29.82248306274414,
      "activations/layer3_attention_weight_max": 92.2378158569336,
      "activations/layer3_attention_weight_min": -89.53362274169922,
      "activations/layer4_attention_weight_max": 89.06368255615234,
      "activations/layer4_attention_weight_min": -87.09484100341797,
      "activations/layer5_attention_weight_max": 66.16979217529297,
      "activations/layer5_attention_weight_min": -72.47382354736328,
      "activations/layer6_attention_weight_max": 48.863346099853516,
      "activations/layer6_attention_weight_min": -50.12393569946289,
      "activations/layer7_attention_weight_max": 60.455963134765625,
      "activations/layer7_attention_weight_min": -56.583457946777344,
      "activations/layer8_attention_weight_max": 41.03102111816406,
      "activations/layer8_attention_weight_min": -45.664085388183594,
      "activations/layer9_attention_weight_max": 41.858238220214844,
      "activations/layer9_attention_weight_min": -46.47400665283203,
      "epoch": 15.69,
      "learning_rate": 4.934545454545454e-05,
      "loss": 2.7471,
      "step": 269950
    },
    {
      "activations/layer0_attention_weight_max": 16.468265533447266,
      "activations/layer0_attention_weight_min": -13.80671215057373,
      "activations/layer10_attention_weight_max": 35.7657470703125,
      "activations/layer10_attention_weight_min": -34.47761154174805,
      "activations/layer11_attention_weight_max": 34.11711120605469,
      "activations/layer11_attention_weight_min": -33.871524810791016,
      "activations/layer12_attention_weight_max": 19.22316551208496,
      "activations/layer12_attention_weight_min": -25.739437103271484,
      "activations/layer13_attention_weight_max": 35.35346221923828,
      "activations/layer13_attention_weight_min": -29.55744171142578,
      "activations/layer14_attention_weight_max": 34.11363983154297,
      "activations/layer14_attention_weight_min": -31.34760856628418,
      "activations/layer15_attention_weight_max": 33.07734680175781,
      "activations/layer15_attention_weight_min": -30.29485511779785,
      "activations/layer16_attention_weight_max": 31.349781036376953,
      "activations/layer16_attention_weight_min": -30.57984161376953,
      "activations/layer17_attention_weight_max": 47.30103302001953,
      "activations/layer17_attention_weight_min": -43.31050491333008,
      "activations/layer18_attention_weight_max": 44.262901306152344,
      "activations/layer18_attention_weight_min": -38.06077194213867,
      "activations/layer19_attention_weight_max": 22.9249324798584,
      "activations/layer19_attention_weight_min": -20.2337703704834,
      "activations/layer1_attention_weight_max": 16.60086441040039,
      "activations/layer1_attention_weight_min": -13.357461929321289,
      "activations/layer20_attention_weight_max": 21.958913803100586,
      "activations/layer20_attention_weight_min": -21.125614166259766,
      "activations/layer21_attention_weight_max": 34.235740661621094,
      "activations/layer21_attention_weight_min": -21.92270278930664,
      "activations/layer22_attention_weight_max": 29.5614013671875,
      "activations/layer22_attention_weight_min": -25.795808792114258,
      "activations/layer23_attention_weight_max": 38.539981842041016,
      "activations/layer23_attention_weight_min": -29.02533531188965,
      "activations/layer2_attention_weight_max": 28.197837829589844,
      "activations/layer2_attention_weight_min": -29.503524780273438,
      "activations/layer3_attention_weight_max": 86.8986587524414,
      "activations/layer3_attention_weight_min": -87.9615707397461,
      "activations/layer4_attention_weight_max": 94.2138900756836,
      "activations/layer4_attention_weight_min": -90.5339584350586,
      "activations/layer5_attention_weight_max": 66.17461395263672,
      "activations/layer5_attention_weight_min": -75.27214050292969,
      "activations/layer6_attention_weight_max": 50.54970932006836,
      "activations/layer6_attention_weight_min": -51.56511306762695,
      "activations/layer7_attention_weight_max": 60.5272331237793,
      "activations/layer7_attention_weight_min": -62.806243896484375,
      "activations/layer8_attention_weight_max": 45.569236755371094,
      "activations/layer8_attention_weight_min": -47.37995147705078,
      "activations/layer9_attention_weight_max": 45.23544692993164,
      "activations/layer9_attention_weight_min": -47.733863830566406,
      "epoch": 15.69,
      "learning_rate": 4.932651515151514e-05,
      "loss": 2.7669,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_loss": 2.708984375,
      "eval_runtime": 8.4492,
      "eval_samples_per_second": 508.217,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_openwebtext_loss": 2.708984375,
      "eval_openwebtext_ppl": 15.01401915561195,
      "eval_openwebtext_runtime": 8.4492,
      "eval_openwebtext_samples_per_second": 508.217,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9741,
      "eval_wikitext_samples_per_second": 230.993,
      "step": 270000
    },
    {
      "epoch": 15.69,
      "eval_lambada_loss": 2.619140625,
      "eval_lambada_ppl": 13.72392451825968,
      "eval_lambada_runtime": 9.578,
      "eval_lambada_samples_per_second": 508.351,
      "step": 270000
    },
    {
      "activations/layer0_attention_weight_max": 16.978717803955078,
      "activations/layer0_attention_weight_min": -13.746376991271973,
      "activations/layer10_attention_weight_max": 32.69069290161133,
      "activations/layer10_attention_weight_min": -33.47309875488281,
      "activations/layer11_attention_weight_max": 30.050825119018555,
      "activations/layer11_attention_weight_min": -32.07252502441406,
      "activations/layer12_attention_weight_max": 18.864456176757812,
      "activations/layer12_attention_weight_min": -25.539291381835938,
      "activations/layer13_attention_weight_max": 34.79678726196289,
      "activations/layer13_attention_weight_min": -27.972143173217773,
      "activations/layer14_attention_weight_max": 35.91069412231445,
      "activations/layer14_attention_weight_min": -31.08675765991211,
      "activations/layer15_attention_weight_max": 31.70588493347168,
      "activations/layer15_attention_weight_min": -29.25726318359375,
      "activations/layer16_attention_weight_max": 32.03263473510742,
      "activations/layer16_attention_weight_min": -31.564428329467773,
      "activations/layer17_attention_weight_max": 50.33694076538086,
      "activations/layer17_attention_weight_min": -45.08436584472656,
      "activations/layer18_attention_weight_max": 43.37702560424805,
      "activations/layer18_attention_weight_min": -41.131263732910156,
      "activations/layer19_attention_weight_max": 26.056278228759766,
      "activations/layer19_attention_weight_min": -24.603242874145508,
      "activations/layer1_attention_weight_max": 16.380596160888672,
      "activations/layer1_attention_weight_min": -13.287466049194336,
      "activations/layer20_attention_weight_max": 22.441240310668945,
      "activations/layer20_attention_weight_min": -23.955053329467773,
      "activations/layer21_attention_weight_max": 36.11737823486328,
      "activations/layer21_attention_weight_min": -22.26837158203125,
      "activations/layer22_attention_weight_max": 29.92262840270996,
      "activations/layer22_attention_weight_min": -28.933977127075195,
      "activations/layer23_attention_weight_max": 37.12615966796875,
      "activations/layer23_attention_weight_min": -25.968856811523438,
      "activations/layer2_attention_weight_max": 29.31453514099121,
      "activations/layer2_attention_weight_min": -28.774574279785156,
      "activations/layer3_attention_weight_max": 89.82032012939453,
      "activations/layer3_attention_weight_min": -86.36978912353516,
      "activations/layer4_attention_weight_max": 86.46885681152344,
      "activations/layer4_attention_weight_min": -86.72386932373047,
      "activations/layer5_attention_weight_max": 63.97871017456055,
      "activations/layer5_attention_weight_min": -75.75840759277344,
      "activations/layer6_attention_weight_max": 51.30335235595703,
      "activations/layer6_attention_weight_min": -51.3858757019043,
      "activations/layer7_attention_weight_max": 61.8757209777832,
      "activations/layer7_attention_weight_min": -61.580745697021484,
      "activations/layer8_attention_weight_max": 44.8304328918457,
      "activations/layer8_attention_weight_min": -45.773887634277344,
      "activations/layer9_attention_weight_max": 42.656166076660156,
      "activations/layer9_attention_weight_min": -44.65933609008789,
      "epoch": 15.69,
      "learning_rate": 4.930757575757576e-05,
      "loss": 2.7692,
      "step": 270050
    },
    {
      "activations/layer0_attention_weight_max": 15.189518928527832,
      "activations/layer0_attention_weight_min": -13.99761962890625,
      "activations/layer10_attention_weight_max": 36.63713836669922,
      "activations/layer10_attention_weight_min": -36.0445442199707,
      "activations/layer11_attention_weight_max": 33.114410400390625,
      "activations/layer11_attention_weight_min": -34.901885986328125,
      "activations/layer12_attention_weight_max": 19.349166870117188,
      "activations/layer12_attention_weight_min": -25.46627426147461,
      "activations/layer13_attention_weight_max": 36.20356369018555,
      "activations/layer13_attention_weight_min": -26.91087532043457,
      "activations/layer14_attention_weight_max": 35.83174514770508,
      "activations/layer14_attention_weight_min": -30.453332901000977,
      "activations/layer15_attention_weight_max": 33.977378845214844,
      "activations/layer15_attention_weight_min": -31.035110473632812,
      "activations/layer16_attention_weight_max": 37.04772186279297,
      "activations/layer16_attention_weight_min": -32.37218475341797,
      "activations/layer17_attention_weight_max": 55.39531326293945,
      "activations/layer17_attention_weight_min": -46.40985870361328,
      "activations/layer18_attention_weight_max": 48.934906005859375,
      "activations/layer18_attention_weight_min": -40.66727066040039,
      "activations/layer19_attention_weight_max": 25.67525291442871,
      "activations/layer19_attention_weight_min": -24.2047061920166,
      "activations/layer1_attention_weight_max": 17.49163055419922,
      "activations/layer1_attention_weight_min": -14.102460861206055,
      "activations/layer20_attention_weight_max": 23.44792366027832,
      "activations/layer20_attention_weight_min": -22.647645950317383,
      "activations/layer21_attention_weight_max": 37.7476692199707,
      "activations/layer21_attention_weight_min": -23.417644500732422,
      "activations/layer22_attention_weight_max": 28.655195236206055,
      "activations/layer22_attention_weight_min": -28.149723052978516,
      "activations/layer23_attention_weight_max": 35.88903045654297,
      "activations/layer23_attention_weight_min": -24.00166130065918,
      "activations/layer2_attention_weight_max": 29.29338836669922,
      "activations/layer2_attention_weight_min": -29.009489059448242,
      "activations/layer3_attention_weight_max": 86.09271240234375,
      "activations/layer3_attention_weight_min": -82.59357452392578,
      "activations/layer4_attention_weight_max": 83.94453430175781,
      "activations/layer4_attention_weight_min": -86.9847412109375,
      "activations/layer5_attention_weight_max": 62.11794662475586,
      "activations/layer5_attention_weight_min": -70.43975067138672,
      "activations/layer6_attention_weight_max": 49.265235900878906,
      "activations/layer6_attention_weight_min": -50.82364273071289,
      "activations/layer7_attention_weight_max": 59.16736602783203,
      "activations/layer7_attention_weight_min": -58.589927673339844,
      "activations/layer8_attention_weight_max": 45.87213134765625,
      "activations/layer8_attention_weight_min": -48.96746826171875,
      "activations/layer9_attention_weight_max": 44.47982406616211,
      "activations/layer9_attention_weight_min": -46.088035583496094,
      "epoch": 15.69,
      "learning_rate": 4.928863636363636e-05,
      "loss": 2.7395,
      "step": 270100
    },
    {
      "activations/layer0_attention_weight_max": 15.850011825561523,
      "activations/layer0_attention_weight_min": -14.03382682800293,
      "activations/layer10_attention_weight_max": 35.71999740600586,
      "activations/layer10_attention_weight_min": -35.379364013671875,
      "activations/layer11_attention_weight_max": 32.221187591552734,
      "activations/layer11_attention_weight_min": -34.008975982666016,
      "activations/layer12_attention_weight_max": 18.0513916015625,
      "activations/layer12_attention_weight_min": -25.602710723876953,
      "activations/layer13_attention_weight_max": 39.5005989074707,
      "activations/layer13_attention_weight_min": -27.635927200317383,
      "activations/layer14_attention_weight_max": 36.55331802368164,
      "activations/layer14_attention_weight_min": -29.9215030670166,
      "activations/layer15_attention_weight_max": 33.08464431762695,
      "activations/layer15_attention_weight_min": -29.178325653076172,
      "activations/layer16_attention_weight_max": 31.631792068481445,
      "activations/layer16_attention_weight_min": -31.260534286499023,
      "activations/layer17_attention_weight_max": 48.865745544433594,
      "activations/layer17_attention_weight_min": -44.278404235839844,
      "activations/layer18_attention_weight_max": 44.45083999633789,
      "activations/layer18_attention_weight_min": -38.74198532104492,
      "activations/layer19_attention_weight_max": 24.1292667388916,
      "activations/layer19_attention_weight_min": -23.705537796020508,
      "activations/layer1_attention_weight_max": 16.897724151611328,
      "activations/layer1_attention_weight_min": -13.829806327819824,
      "activations/layer20_attention_weight_max": 22.581506729125977,
      "activations/layer20_attention_weight_min": -23.426855087280273,
      "activations/layer21_attention_weight_max": 38.02827835083008,
      "activations/layer21_attention_weight_min": -23.77688980102539,
      "activations/layer22_attention_weight_max": 29.33019256591797,
      "activations/layer22_attention_weight_min": -24.590322494506836,
      "activations/layer23_attention_weight_max": 45.994117736816406,
      "activations/layer23_attention_weight_min": -27.414793014526367,
      "activations/layer2_attention_weight_max": 29.949115753173828,
      "activations/layer2_attention_weight_min": -29.09912872314453,
      "activations/layer3_attention_weight_max": 89.51644134521484,
      "activations/layer3_attention_weight_min": -87.51875305175781,
      "activations/layer4_attention_weight_max": 86.4793930053711,
      "activations/layer4_attention_weight_min": -87.40444946289062,
      "activations/layer5_attention_weight_max": 64.54071044921875,
      "activations/layer5_attention_weight_min": -72.47483825683594,
      "activations/layer6_attention_weight_max": 48.68915557861328,
      "activations/layer6_attention_weight_min": -50.35554885864258,
      "activations/layer7_attention_weight_max": 62.3369140625,
      "activations/layer7_attention_weight_min": -59.37993621826172,
      "activations/layer8_attention_weight_max": 44.2916145324707,
      "activations/layer8_attention_weight_min": -45.925254821777344,
      "activations/layer9_attention_weight_max": 44.639286041259766,
      "activations/layer9_attention_weight_min": -44.48662567138672,
      "epoch": 15.7,
      "learning_rate": 4.926969696969696e-05,
      "loss": 2.757,
      "step": 270150
    },
    {
      "activations/layer0_attention_weight_max": 14.886791229248047,
      "activations/layer0_attention_weight_min": -14.115534782409668,
      "activations/layer10_attention_weight_max": 37.84941864013672,
      "activations/layer10_attention_weight_min": -35.98225402832031,
      "activations/layer11_attention_weight_max": 34.287086486816406,
      "activations/layer11_attention_weight_min": -34.08583068847656,
      "activations/layer12_attention_weight_max": 19.390905380249023,
      "activations/layer12_attention_weight_min": -25.827529907226562,
      "activations/layer13_attention_weight_max": 38.69853591918945,
      "activations/layer13_attention_weight_min": -31.380739212036133,
      "activations/layer14_attention_weight_max": 37.08419418334961,
      "activations/layer14_attention_weight_min": -31.8657169342041,
      "activations/layer15_attention_weight_max": 32.94718551635742,
      "activations/layer15_attention_weight_min": -30.359722137451172,
      "activations/layer16_attention_weight_max": 32.2359619140625,
      "activations/layer16_attention_weight_min": -31.53936767578125,
      "activations/layer17_attention_weight_max": 50.539493560791016,
      "activations/layer17_attention_weight_min": -47.922119140625,
      "activations/layer18_attention_weight_max": 47.56373977661133,
      "activations/layer18_attention_weight_min": -42.05443572998047,
      "activations/layer19_attention_weight_max": 27.10126304626465,
      "activations/layer19_attention_weight_min": -23.872188568115234,
      "activations/layer1_attention_weight_max": 15.811046600341797,
      "activations/layer1_attention_weight_min": -13.776698112487793,
      "activations/layer20_attention_weight_max": 25.55906867980957,
      "activations/layer20_attention_weight_min": -22.436315536499023,
      "activations/layer21_attention_weight_max": 36.699615478515625,
      "activations/layer21_attention_weight_min": -25.500316619873047,
      "activations/layer22_attention_weight_max": 29.72612190246582,
      "activations/layer22_attention_weight_min": -26.355791091918945,
      "activations/layer23_attention_weight_max": 39.08919143676758,
      "activations/layer23_attention_weight_min": -27.16472625732422,
      "activations/layer2_attention_weight_max": 28.46802520751953,
      "activations/layer2_attention_weight_min": -27.300506591796875,
      "activations/layer3_attention_weight_max": 86.67967987060547,
      "activations/layer3_attention_weight_min": -84.8465805053711,
      "activations/layer4_attention_weight_max": 90.3534164428711,
      "activations/layer4_attention_weight_min": -89.26115417480469,
      "activations/layer5_attention_weight_max": 66.3614501953125,
      "activations/layer5_attention_weight_min": -72.91870880126953,
      "activations/layer6_attention_weight_max": 50.931068420410156,
      "activations/layer6_attention_weight_min": -50.99454116821289,
      "activations/layer7_attention_weight_max": 65.40068817138672,
      "activations/layer7_attention_weight_min": -61.84910583496094,
      "activations/layer8_attention_weight_max": 47.49806594848633,
      "activations/layer8_attention_weight_min": -50.10376739501953,
      "activations/layer9_attention_weight_max": 45.86024856567383,
      "activations/layer9_attention_weight_min": -47.09885787963867,
      "epoch": 15.7,
      "learning_rate": 4.925075757575757e-05,
      "loss": 2.7444,
      "step": 270200
    },
    {
      "activations/layer0_attention_weight_max": 16.009092330932617,
      "activations/layer0_attention_weight_min": -13.84182357788086,
      "activations/layer10_attention_weight_max": 38.16567611694336,
      "activations/layer10_attention_weight_min": -36.06477737426758,
      "activations/layer11_attention_weight_max": 34.35770797729492,
      "activations/layer11_attention_weight_min": -33.55097198486328,
      "activations/layer12_attention_weight_max": 20.738351821899414,
      "activations/layer12_attention_weight_min": -26.75973129272461,
      "activations/layer13_attention_weight_max": 42.09376525878906,
      "activations/layer13_attention_weight_min": -30.43368911743164,
      "activations/layer14_attention_weight_max": 50.436614990234375,
      "activations/layer14_attention_weight_min": -30.825241088867188,
      "activations/layer15_attention_weight_max": 46.88277053833008,
      "activations/layer15_attention_weight_min": -30.91794204711914,
      "activations/layer16_attention_weight_max": 36.690608978271484,
      "activations/layer16_attention_weight_min": -34.12315368652344,
      "activations/layer17_attention_weight_max": 56.711891174316406,
      "activations/layer17_attention_weight_min": -50.443302154541016,
      "activations/layer18_attention_weight_max": 49.98808670043945,
      "activations/layer18_attention_weight_min": -45.140018463134766,
      "activations/layer19_attention_weight_max": 25.508480072021484,
      "activations/layer19_attention_weight_min": -23.990610122680664,
      "activations/layer1_attention_weight_max": 17.973976135253906,
      "activations/layer1_attention_weight_min": -15.83562183380127,
      "activations/layer20_attention_weight_max": 26.50808334350586,
      "activations/layer20_attention_weight_min": -21.857616424560547,
      "activations/layer21_attention_weight_max": 39.988807678222656,
      "activations/layer21_attention_weight_min": -22.297073364257812,
      "activations/layer22_attention_weight_max": 32.01668930053711,
      "activations/layer22_attention_weight_min": -27.01271629333496,
      "activations/layer23_attention_weight_max": 42.418548583984375,
      "activations/layer23_attention_weight_min": -26.22115707397461,
      "activations/layer2_attention_weight_max": 33.8343505859375,
      "activations/layer2_attention_weight_min": -30.972087860107422,
      "activations/layer3_attention_weight_max": 91.31456756591797,
      "activations/layer3_attention_weight_min": -91.56974792480469,
      "activations/layer4_attention_weight_max": 92.03343963623047,
      "activations/layer4_attention_weight_min": -88.79776763916016,
      "activations/layer5_attention_weight_max": 65.2634048461914,
      "activations/layer5_attention_weight_min": -73.31767272949219,
      "activations/layer6_attention_weight_max": 51.37760543823242,
      "activations/layer6_attention_weight_min": -53.023292541503906,
      "activations/layer7_attention_weight_max": 63.585899353027344,
      "activations/layer7_attention_weight_min": -65.80552673339844,
      "activations/layer8_attention_weight_max": 51.226070404052734,
      "activations/layer8_attention_weight_min": -49.6330680847168,
      "activations/layer9_attention_weight_max": 53.69950485229492,
      "activations/layer9_attention_weight_min": -45.551570892333984,
      "epoch": 15.7,
      "learning_rate": 4.923181818181818e-05,
      "loss": 2.7688,
      "step": 270250
    },
    {
      "activations/layer0_attention_weight_max": 15.487691879272461,
      "activations/layer0_attention_weight_min": -13.83496379852295,
      "activations/layer10_attention_weight_max": 34.18174743652344,
      "activations/layer10_attention_weight_min": -35.12895965576172,
      "activations/layer11_attention_weight_max": 29.01355743408203,
      "activations/layer11_attention_weight_min": -32.888641357421875,
      "activations/layer12_attention_weight_max": 18.131155014038086,
      "activations/layer12_attention_weight_min": -24.87159538269043,
      "activations/layer13_attention_weight_max": 29.366653442382812,
      "activations/layer13_attention_weight_min": -28.863868713378906,
      "activations/layer14_attention_weight_max": 32.46012878417969,
      "activations/layer14_attention_weight_min": -29.967845916748047,
      "activations/layer15_attention_weight_max": 29.305648803710938,
      "activations/layer15_attention_weight_min": -28.45476722717285,
      "activations/layer16_attention_weight_max": 29.676361083984375,
      "activations/layer16_attention_weight_min": -28.866371154785156,
      "activations/layer17_attention_weight_max": 47.9310302734375,
      "activations/layer17_attention_weight_min": -42.172149658203125,
      "activations/layer18_attention_weight_max": 40.26265335083008,
      "activations/layer18_attention_weight_min": -39.39119338989258,
      "activations/layer19_attention_weight_max": 22.704910278320312,
      "activations/layer19_attention_weight_min": -24.23845100402832,
      "activations/layer1_attention_weight_max": 16.036401748657227,
      "activations/layer1_attention_weight_min": -13.821958541870117,
      "activations/layer20_attention_weight_max": 20.505056381225586,
      "activations/layer20_attention_weight_min": -20.995847702026367,
      "activations/layer21_attention_weight_max": 30.046382904052734,
      "activations/layer21_attention_weight_min": -24.933658599853516,
      "activations/layer22_attention_weight_max": 27.23587989807129,
      "activations/layer22_attention_weight_min": -25.414522171020508,
      "activations/layer23_attention_weight_max": 34.38114547729492,
      "activations/layer23_attention_weight_min": -25.940168380737305,
      "activations/layer2_attention_weight_max": 29.25233268737793,
      "activations/layer2_attention_weight_min": -27.58027458190918,
      "activations/layer3_attention_weight_max": 86.0764389038086,
      "activations/layer3_attention_weight_min": -87.94757843017578,
      "activations/layer4_attention_weight_max": 83.29937744140625,
      "activations/layer4_attention_weight_min": -84.98329162597656,
      "activations/layer5_attention_weight_max": 62.51909637451172,
      "activations/layer5_attention_weight_min": -75.75727844238281,
      "activations/layer6_attention_weight_max": 45.17519760131836,
      "activations/layer6_attention_weight_min": -48.79610061645508,
      "activations/layer7_attention_weight_max": 59.54810333251953,
      "activations/layer7_attention_weight_min": -57.217254638671875,
      "activations/layer8_attention_weight_max": 42.563941955566406,
      "activations/layer8_attention_weight_min": -45.22976303100586,
      "activations/layer9_attention_weight_max": 40.120609283447266,
      "activations/layer9_attention_weight_min": -43.98362731933594,
      "epoch": 15.71,
      "learning_rate": 4.9212878787878786e-05,
      "loss": 2.7505,
      "step": 270300
    },
    {
      "activations/layer0_attention_weight_max": 15.54931354522705,
      "activations/layer0_attention_weight_min": -14.088217735290527,
      "activations/layer10_attention_weight_max": 36.56697463989258,
      "activations/layer10_attention_weight_min": -39.03681945800781,
      "activations/layer11_attention_weight_max": 33.705406188964844,
      "activations/layer11_attention_weight_min": -35.80476760864258,
      "activations/layer12_attention_weight_max": 19.739377975463867,
      "activations/layer12_attention_weight_min": -22.95265007019043,
      "activations/layer13_attention_weight_max": 37.429176330566406,
      "activations/layer13_attention_weight_min": -27.340801239013672,
      "activations/layer14_attention_weight_max": 35.04783248901367,
      "activations/layer14_attention_weight_min": -31.593442916870117,
      "activations/layer15_attention_weight_max": 32.952579498291016,
      "activations/layer15_attention_weight_min": -30.69186019897461,
      "activations/layer16_attention_weight_max": 33.83554458618164,
      "activations/layer16_attention_weight_min": -31.512975692749023,
      "activations/layer17_attention_weight_max": 51.054691314697266,
      "activations/layer17_attention_weight_min": -44.97957992553711,
      "activations/layer18_attention_weight_max": 48.90433883666992,
      "activations/layer18_attention_weight_min": -38.8372917175293,
      "activations/layer19_attention_weight_max": 24.48113250732422,
      "activations/layer19_attention_weight_min": -21.258472442626953,
      "activations/layer1_attention_weight_max": 16.90628433227539,
      "activations/layer1_attention_weight_min": -14.00748062133789,
      "activations/layer20_attention_weight_max": 25.042112350463867,
      "activations/layer20_attention_weight_min": -21.382884979248047,
      "activations/layer21_attention_weight_max": 35.3890266418457,
      "activations/layer21_attention_weight_min": -21.246665954589844,
      "activations/layer22_attention_weight_max": 32.357688903808594,
      "activations/layer22_attention_weight_min": -26.646833419799805,
      "activations/layer23_attention_weight_max": 38.87671661376953,
      "activations/layer23_attention_weight_min": -23.017919540405273,
      "activations/layer2_attention_weight_max": 28.358314514160156,
      "activations/layer2_attention_weight_min": -28.188793182373047,
      "activations/layer3_attention_weight_max": 90.86750793457031,
      "activations/layer3_attention_weight_min": -89.19722747802734,
      "activations/layer4_attention_weight_max": 91.50723266601562,
      "activations/layer4_attention_weight_min": -88.3648910522461,
      "activations/layer5_attention_weight_max": 72.50321960449219,
      "activations/layer5_attention_weight_min": -71.63200378417969,
      "activations/layer6_attention_weight_max": 56.5958251953125,
      "activations/layer6_attention_weight_min": -55.04542922973633,
      "activations/layer7_attention_weight_max": 71.31847381591797,
      "activations/layer7_attention_weight_min": -65.27511596679688,
      "activations/layer8_attention_weight_max": 47.84718704223633,
      "activations/layer8_attention_weight_min": -50.91939163208008,
      "activations/layer9_attention_weight_max": 44.24700164794922,
      "activations/layer9_attention_weight_min": -48.24633026123047,
      "epoch": 15.71,
      "learning_rate": 4.919393939393939e-05,
      "loss": 2.7609,
      "step": 270350
    },
    {
      "activations/layer0_attention_weight_max": 16.161296844482422,
      "activations/layer0_attention_weight_min": -14.281915664672852,
      "activations/layer10_attention_weight_max": 33.95968246459961,
      "activations/layer10_attention_weight_min": -36.516998291015625,
      "activations/layer11_attention_weight_max": 31.55178451538086,
      "activations/layer11_attention_weight_min": -33.14106750488281,
      "activations/layer12_attention_weight_max": 18.301042556762695,
      "activations/layer12_attention_weight_min": -26.708393096923828,
      "activations/layer13_attention_weight_max": 34.23601531982422,
      "activations/layer13_attention_weight_min": -28.57924461364746,
      "activations/layer14_attention_weight_max": 36.33646011352539,
      "activations/layer14_attention_weight_min": -29.883712768554688,
      "activations/layer15_attention_weight_max": 34.28208923339844,
      "activations/layer15_attention_weight_min": -28.770540237426758,
      "activations/layer16_attention_weight_max": 33.20625686645508,
      "activations/layer16_attention_weight_min": -31.20567512512207,
      "activations/layer17_attention_weight_max": 52.77383804321289,
      "activations/layer17_attention_weight_min": -46.250877380371094,
      "activations/layer18_attention_weight_max": 49.14711380004883,
      "activations/layer18_attention_weight_min": -40.216556549072266,
      "activations/layer19_attention_weight_max": 24.527605056762695,
      "activations/layer19_attention_weight_min": -22.844892501831055,
      "activations/layer1_attention_weight_max": 16.595409393310547,
      "activations/layer1_attention_weight_min": -13.402682304382324,
      "activations/layer20_attention_weight_max": 21.05314826965332,
      "activations/layer20_attention_weight_min": -21.218807220458984,
      "activations/layer21_attention_weight_max": 39.05864334106445,
      "activations/layer21_attention_weight_min": -20.283876419067383,
      "activations/layer22_attention_weight_max": 28.306991577148438,
      "activations/layer22_attention_weight_min": -27.314790725708008,
      "activations/layer23_attention_weight_max": 31.852767944335938,
      "activations/layer23_attention_weight_min": -23.416845321655273,
      "activations/layer2_attention_weight_max": 30.368576049804688,
      "activations/layer2_attention_weight_min": -29.979825973510742,
      "activations/layer3_attention_weight_max": 91.30745697021484,
      "activations/layer3_attention_weight_min": -89.4800796508789,
      "activations/layer4_attention_weight_max": 96.1989974975586,
      "activations/layer4_attention_weight_min": -90.46797943115234,
      "activations/layer5_attention_weight_max": 68.41098022460938,
      "activations/layer5_attention_weight_min": -74.6624755859375,
      "activations/layer6_attention_weight_max": 50.92519760131836,
      "activations/layer6_attention_weight_min": -52.744930267333984,
      "activations/layer7_attention_weight_max": 60.26203918457031,
      "activations/layer7_attention_weight_min": -62.22386932373047,
      "activations/layer8_attention_weight_max": 44.891563415527344,
      "activations/layer8_attention_weight_min": -47.7468147277832,
      "activations/layer9_attention_weight_max": 42.81516647338867,
      "activations/layer9_attention_weight_min": -43.68368148803711,
      "epoch": 15.71,
      "learning_rate": 4.917499999999999e-05,
      "loss": 2.7383,
      "step": 270400
    },
    {
      "activations/layer0_attention_weight_max": 17.109914779663086,
      "activations/layer0_attention_weight_min": -13.991084098815918,
      "activations/layer10_attention_weight_max": 34.321388244628906,
      "activations/layer10_attention_weight_min": -33.96204376220703,
      "activations/layer11_attention_weight_max": 32.38413619995117,
      "activations/layer11_attention_weight_min": -34.30854797363281,
      "activations/layer12_attention_weight_max": 18.678556442260742,
      "activations/layer12_attention_weight_min": -23.04082679748535,
      "activations/layer13_attention_weight_max": 35.95449447631836,
      "activations/layer13_attention_weight_min": -31.17753028869629,
      "activations/layer14_attention_weight_max": 37.413963317871094,
      "activations/layer14_attention_weight_min": -32.40419387817383,
      "activations/layer15_attention_weight_max": 34.32768630981445,
      "activations/layer15_attention_weight_min": -29.590782165527344,
      "activations/layer16_attention_weight_max": 33.418121337890625,
      "activations/layer16_attention_weight_min": -31.777311325073242,
      "activations/layer17_attention_weight_max": 51.94181823730469,
      "activations/layer17_attention_weight_min": -44.31515884399414,
      "activations/layer18_attention_weight_max": 48.495975494384766,
      "activations/layer18_attention_weight_min": -37.652156829833984,
      "activations/layer19_attention_weight_max": 22.69111442565918,
      "activations/layer19_attention_weight_min": -23.032127380371094,
      "activations/layer1_attention_weight_max": 16.634462356567383,
      "activations/layer1_attention_weight_min": -13.7262544631958,
      "activations/layer20_attention_weight_max": 22.92581558227539,
      "activations/layer20_attention_weight_min": -21.24492835998535,
      "activations/layer21_attention_weight_max": 38.47969055175781,
      "activations/layer21_attention_weight_min": -23.919017791748047,
      "activations/layer22_attention_weight_max": 28.303802490234375,
      "activations/layer22_attention_weight_min": -26.754106521606445,
      "activations/layer23_attention_weight_max": 40.88334655761719,
      "activations/layer23_attention_weight_min": -23.94611358642578,
      "activations/layer2_attention_weight_max": 30.157730102539062,
      "activations/layer2_attention_weight_min": -30.341779708862305,
      "activations/layer3_attention_weight_max": 88.0644760131836,
      "activations/layer3_attention_weight_min": -90.42146301269531,
      "activations/layer4_attention_weight_max": 88.67208099365234,
      "activations/layer4_attention_weight_min": -85.25003051757812,
      "activations/layer5_attention_weight_max": 63.76044464111328,
      "activations/layer5_attention_weight_min": -69.44940185546875,
      "activations/layer6_attention_weight_max": 48.83579635620117,
      "activations/layer6_attention_weight_min": -48.599910736083984,
      "activations/layer7_attention_weight_max": 59.82438278198242,
      "activations/layer7_attention_weight_min": -58.154624938964844,
      "activations/layer8_attention_weight_max": 43.11520767211914,
      "activations/layer8_attention_weight_min": -47.3309326171875,
      "activations/layer9_attention_weight_max": 41.7106819152832,
      "activations/layer9_attention_weight_min": -42.73536682128906,
      "epoch": 15.71,
      "learning_rate": 4.9156060606060604e-05,
      "loss": 2.7525,
      "step": 270450
    },
    {
      "activations/layer0_attention_weight_max": 17.847328186035156,
      "activations/layer0_attention_weight_min": -13.63852596282959,
      "activations/layer10_attention_weight_max": 38.694480895996094,
      "activations/layer10_attention_weight_min": -40.247257232666016,
      "activations/layer11_attention_weight_max": 39.23638153076172,
      "activations/layer11_attention_weight_min": -39.00906753540039,
      "activations/layer12_attention_weight_max": 18.987180709838867,
      "activations/layer12_attention_weight_min": -25.468732833862305,
      "activations/layer13_attention_weight_max": 34.697269439697266,
      "activations/layer13_attention_weight_min": -26.453065872192383,
      "activations/layer14_attention_weight_max": 42.02800750732422,
      "activations/layer14_attention_weight_min": -29.815431594848633,
      "activations/layer15_attention_weight_max": 37.06516647338867,
      "activations/layer15_attention_weight_min": -32.02202606201172,
      "activations/layer16_attention_weight_max": 36.792789459228516,
      "activations/layer16_attention_weight_min": -34.02445983886719,
      "activations/layer17_attention_weight_max": 55.98915100097656,
      "activations/layer17_attention_weight_min": -49.85222625732422,
      "activations/layer18_attention_weight_max": 53.96451187133789,
      "activations/layer18_attention_weight_min": -44.62371063232422,
      "activations/layer19_attention_weight_max": 29.36203956604004,
      "activations/layer19_attention_weight_min": -27.227310180664062,
      "activations/layer1_attention_weight_max": 16.381912231445312,
      "activations/layer1_attention_weight_min": -13.580768585205078,
      "activations/layer20_attention_weight_max": 28.905221939086914,
      "activations/layer20_attention_weight_min": -23.607276916503906,
      "activations/layer21_attention_weight_max": 43.47346878051758,
      "activations/layer21_attention_weight_min": -27.236923217773438,
      "activations/layer22_attention_weight_max": 34.76894760131836,
      "activations/layer22_attention_weight_min": -27.41710090637207,
      "activations/layer23_attention_weight_max": 36.320343017578125,
      "activations/layer23_attention_weight_min": -25.70370101928711,
      "activations/layer2_attention_weight_max": 31.621152877807617,
      "activations/layer2_attention_weight_min": -30.607702255249023,
      "activations/layer3_attention_weight_max": 90.89128875732422,
      "activations/layer3_attention_weight_min": -91.8184585571289,
      "activations/layer4_attention_weight_max": 89.02996063232422,
      "activations/layer4_attention_weight_min": -88.15899658203125,
      "activations/layer5_attention_weight_max": 68.77623748779297,
      "activations/layer5_attention_weight_min": -74.44417572021484,
      "activations/layer6_attention_weight_max": 50.15345001220703,
      "activations/layer6_attention_weight_min": -53.86308670043945,
      "activations/layer7_attention_weight_max": 63.25117492675781,
      "activations/layer7_attention_weight_min": -63.811519622802734,
      "activations/layer8_attention_weight_max": 50.493873596191406,
      "activations/layer8_attention_weight_min": -49.7070198059082,
      "activations/layer9_attention_weight_max": 53.9176139831543,
      "activations/layer9_attention_weight_min": -46.468868255615234,
      "epoch": 15.72,
      "learning_rate": 4.9137121212121206e-05,
      "loss": 2.7671,
      "step": 270500
    },
    {
      "activations/layer0_attention_weight_max": 16.302274703979492,
      "activations/layer0_attention_weight_min": -13.932406425476074,
      "activations/layer10_attention_weight_max": 39.39949417114258,
      "activations/layer10_attention_weight_min": -37.28337478637695,
      "activations/layer11_attention_weight_max": 34.9154052734375,
      "activations/layer11_attention_weight_min": -36.77240753173828,
      "activations/layer12_attention_weight_max": 20.192792892456055,
      "activations/layer12_attention_weight_min": -29.141773223876953,
      "activations/layer13_attention_weight_max": 34.055503845214844,
      "activations/layer13_attention_weight_min": -29.636821746826172,
      "activations/layer14_attention_weight_max": 36.745487213134766,
      "activations/layer14_attention_weight_min": -32.065879821777344,
      "activations/layer15_attention_weight_max": 35.11930465698242,
      "activations/layer15_attention_weight_min": -33.40960693359375,
      "activations/layer16_attention_weight_max": 36.38158416748047,
      "activations/layer16_attention_weight_min": -32.29434585571289,
      "activations/layer17_attention_weight_max": 54.55629348754883,
      "activations/layer17_attention_weight_min": -47.51445007324219,
      "activations/layer18_attention_weight_max": 54.94995880126953,
      "activations/layer18_attention_weight_min": -42.46122741699219,
      "activations/layer19_attention_weight_max": 29.034149169921875,
      "activations/layer19_attention_weight_min": -23.59659194946289,
      "activations/layer1_attention_weight_max": 16.017839431762695,
      "activations/layer1_attention_weight_min": -13.59273624420166,
      "activations/layer20_attention_weight_max": 26.194114685058594,
      "activations/layer20_attention_weight_min": -22.10359764099121,
      "activations/layer21_attention_weight_max": 39.7856330871582,
      "activations/layer21_attention_weight_min": -21.035444259643555,
      "activations/layer22_attention_weight_max": 33.00651931762695,
      "activations/layer22_attention_weight_min": -26.601736068725586,
      "activations/layer23_attention_weight_max": 35.30960464477539,
      "activations/layer23_attention_weight_min": -24.232662200927734,
      "activations/layer2_attention_weight_max": 31.826950073242188,
      "activations/layer2_attention_weight_min": -30.095947265625,
      "activations/layer3_attention_weight_max": 92.17771911621094,
      "activations/layer3_attention_weight_min": -95.67829132080078,
      "activations/layer4_attention_weight_max": 90.42806243896484,
      "activations/layer4_attention_weight_min": -89.07591247558594,
      "activations/layer5_attention_weight_max": 67.54862213134766,
      "activations/layer5_attention_weight_min": -72.57003784179688,
      "activations/layer6_attention_weight_max": 53.85306930541992,
      "activations/layer6_attention_weight_min": -54.19882583618164,
      "activations/layer7_attention_weight_max": 65.17610931396484,
      "activations/layer7_attention_weight_min": -61.22236251831055,
      "activations/layer8_attention_weight_max": 49.20811080932617,
      "activations/layer8_attention_weight_min": -51.339534759521484,
      "activations/layer9_attention_weight_max": 48.71108627319336,
      "activations/layer9_attention_weight_min": -47.9975700378418,
      "epoch": 15.72,
      "learning_rate": 4.9118181818181814e-05,
      "loss": 2.7526,
      "step": 270550
    },
    {
      "activations/layer0_attention_weight_max": 16.54857635498047,
      "activations/layer0_attention_weight_min": -13.696783065795898,
      "activations/layer10_attention_weight_max": 34.781646728515625,
      "activations/layer10_attention_weight_min": -35.6595573425293,
      "activations/layer11_attention_weight_max": 33.40094757080078,
      "activations/layer11_attention_weight_min": -32.995277404785156,
      "activations/layer12_attention_weight_max": 18.0028133392334,
      "activations/layer12_attention_weight_min": -26.581254959106445,
      "activations/layer13_attention_weight_max": 33.923702239990234,
      "activations/layer13_attention_weight_min": -30.483491897583008,
      "activations/layer14_attention_weight_max": 35.096858978271484,
      "activations/layer14_attention_weight_min": -29.923067092895508,
      "activations/layer15_attention_weight_max": 32.886940002441406,
      "activations/layer15_attention_weight_min": -29.48281478881836,
      "activations/layer16_attention_weight_max": 32.711585998535156,
      "activations/layer16_attention_weight_min": -31.401391983032227,
      "activations/layer17_attention_weight_max": 54.46308135986328,
      "activations/layer17_attention_weight_min": -44.6000862121582,
      "activations/layer18_attention_weight_max": 46.158416748046875,
      "activations/layer18_attention_weight_min": -38.69172286987305,
      "activations/layer19_attention_weight_max": 24.64373207092285,
      "activations/layer19_attention_weight_min": -22.571191787719727,
      "activations/layer1_attention_weight_max": 18.100669860839844,
      "activations/layer1_attention_weight_min": -12.664868354797363,
      "activations/layer20_attention_weight_max": 23.973068237304688,
      "activations/layer20_attention_weight_min": -22.143951416015625,
      "activations/layer21_attention_weight_max": 35.6412239074707,
      "activations/layer21_attention_weight_min": -21.247718811035156,
      "activations/layer22_attention_weight_max": 29.388792037963867,
      "activations/layer22_attention_weight_min": -28.212055206298828,
      "activations/layer23_attention_weight_max": 38.742393493652344,
      "activations/layer23_attention_weight_min": -24.20336151123047,
      "activations/layer2_attention_weight_max": 30.13799285888672,
      "activations/layer2_attention_weight_min": -29.047569274902344,
      "activations/layer3_attention_weight_max": 90.10997772216797,
      "activations/layer3_attention_weight_min": -87.93827056884766,
      "activations/layer4_attention_weight_max": 92.13912200927734,
      "activations/layer4_attention_weight_min": -86.29015350341797,
      "activations/layer5_attention_weight_max": 65.45576477050781,
      "activations/layer5_attention_weight_min": -72.32308197021484,
      "activations/layer6_attention_weight_max": 51.16341018676758,
      "activations/layer6_attention_weight_min": -54.18903350830078,
      "activations/layer7_attention_weight_max": 63.866310119628906,
      "activations/layer7_attention_weight_min": -58.945987701416016,
      "activations/layer8_attention_weight_max": 42.92732620239258,
      "activations/layer8_attention_weight_min": -46.833824157714844,
      "activations/layer9_attention_weight_max": 51.45231246948242,
      "activations/layer9_attention_weight_min": -50.26298141479492,
      "epoch": 15.72,
      "learning_rate": 4.9099242424242415e-05,
      "loss": 2.7563,
      "step": 270600
    },
    {
      "activations/layer0_attention_weight_max": 15.130534172058105,
      "activations/layer0_attention_weight_min": -14.010839462280273,
      "activations/layer10_attention_weight_max": 34.556339263916016,
      "activations/layer10_attention_weight_min": -34.732852935791016,
      "activations/layer11_attention_weight_max": 31.719533920288086,
      "activations/layer11_attention_weight_min": -34.119361877441406,
      "activations/layer12_attention_weight_max": 18.32999038696289,
      "activations/layer12_attention_weight_min": -23.08189582824707,
      "activations/layer13_attention_weight_max": 46.28971862792969,
      "activations/layer13_attention_weight_min": -29.218399047851562,
      "activations/layer14_attention_weight_max": 35.20042037963867,
      "activations/layer14_attention_weight_min": -30.3624267578125,
      "activations/layer15_attention_weight_max": 36.56395721435547,
      "activations/layer15_attention_weight_min": -29.290306091308594,
      "activations/layer16_attention_weight_max": 32.93527603149414,
      "activations/layer16_attention_weight_min": -30.773481369018555,
      "activations/layer17_attention_weight_max": 49.433876037597656,
      "activations/layer17_attention_weight_min": -43.77408218383789,
      "activations/layer18_attention_weight_max": 51.58431625366211,
      "activations/layer18_attention_weight_min": -39.62935256958008,
      "activations/layer19_attention_weight_max": 22.739561080932617,
      "activations/layer19_attention_weight_min": -22.843671798706055,
      "activations/layer1_attention_weight_max": 15.666784286499023,
      "activations/layer1_attention_weight_min": -14.1345796585083,
      "activations/layer20_attention_weight_max": 23.363588333129883,
      "activations/layer20_attention_weight_min": -21.801008224487305,
      "activations/layer21_attention_weight_max": 36.29730224609375,
      "activations/layer21_attention_weight_min": -23.555776596069336,
      "activations/layer22_attention_weight_max": 28.49616050720215,
      "activations/layer22_attention_weight_min": -24.219263076782227,
      "activations/layer23_attention_weight_max": 38.415061950683594,
      "activations/layer23_attention_weight_min": -29.748395919799805,
      "activations/layer2_attention_weight_max": 29.646682739257812,
      "activations/layer2_attention_weight_min": -29.4241943359375,
      "activations/layer3_attention_weight_max": 85.02595520019531,
      "activations/layer3_attention_weight_min": -82.97309875488281,
      "activations/layer4_attention_weight_max": 86.8656234741211,
      "activations/layer4_attention_weight_min": -83.92002868652344,
      "activations/layer5_attention_weight_max": 62.5954704284668,
      "activations/layer5_attention_weight_min": -71.87496948242188,
      "activations/layer6_attention_weight_max": 49.41762161254883,
      "activations/layer6_attention_weight_min": -49.927059173583984,
      "activations/layer7_attention_weight_max": 58.18206787109375,
      "activations/layer7_attention_weight_min": -57.369049072265625,
      "activations/layer8_attention_weight_max": 44.913970947265625,
      "activations/layer8_attention_weight_min": -50.5447883605957,
      "activations/layer9_attention_weight_max": 43.856746673583984,
      "activations/layer9_attention_weight_min": -44.43312454223633,
      "epoch": 15.73,
      "learning_rate": 4.908030303030303e-05,
      "loss": 2.7543,
      "step": 270650
    },
    {
      "activations/layer0_attention_weight_max": 15.267741203308105,
      "activations/layer0_attention_weight_min": -14.141255378723145,
      "activations/layer10_attention_weight_max": 37.70599365234375,
      "activations/layer10_attention_weight_min": -38.09724807739258,
      "activations/layer11_attention_weight_max": 33.383689880371094,
      "activations/layer11_attention_weight_min": -34.51348114013672,
      "activations/layer12_attention_weight_max": 23.283382415771484,
      "activations/layer12_attention_weight_min": -24.448944091796875,
      "activations/layer13_attention_weight_max": 35.61391830444336,
      "activations/layer13_attention_weight_min": -31.032955169677734,
      "activations/layer14_attention_weight_max": 35.0772590637207,
      "activations/layer14_attention_weight_min": -31.025522232055664,
      "activations/layer15_attention_weight_max": 32.580039978027344,
      "activations/layer15_attention_weight_min": -30.457412719726562,
      "activations/layer16_attention_weight_max": 32.58053970336914,
      "activations/layer16_attention_weight_min": -30.95246124267578,
      "activations/layer17_attention_weight_max": 51.30491256713867,
      "activations/layer17_attention_weight_min": -47.38933181762695,
      "activations/layer18_attention_weight_max": 45.344417572021484,
      "activations/layer18_attention_weight_min": -41.779151916503906,
      "activations/layer19_attention_weight_max": 23.396770477294922,
      "activations/layer19_attention_weight_min": -24.70331573486328,
      "activations/layer1_attention_weight_max": 16.54338836669922,
      "activations/layer1_attention_weight_min": -15.272282600402832,
      "activations/layer20_attention_weight_max": 25.812559127807617,
      "activations/layer20_attention_weight_min": -25.098787307739258,
      "activations/layer21_attention_weight_max": 32.95894241333008,
      "activations/layer21_attention_weight_min": -26.641250610351562,
      "activations/layer22_attention_weight_max": 28.9268741607666,
      "activations/layer22_attention_weight_min": -27.412479400634766,
      "activations/layer23_attention_weight_max": 36.479896545410156,
      "activations/layer23_attention_weight_min": -24.071247100830078,
      "activations/layer2_attention_weight_max": 30.761859893798828,
      "activations/layer2_attention_weight_min": -31.829864501953125,
      "activations/layer3_attention_weight_max": 94.80725860595703,
      "activations/layer3_attention_weight_min": -97.30189514160156,
      "activations/layer4_attention_weight_max": 89.68498992919922,
      "activations/layer4_attention_weight_min": -92.56096649169922,
      "activations/layer5_attention_weight_max": 65.53018188476562,
      "activations/layer5_attention_weight_min": -72.78704071044922,
      "activations/layer6_attention_weight_max": 50.67617416381836,
      "activations/layer6_attention_weight_min": -51.126930236816406,
      "activations/layer7_attention_weight_max": 63.20027542114258,
      "activations/layer7_attention_weight_min": -61.111881256103516,
      "activations/layer8_attention_weight_max": 47.7653694152832,
      "activations/layer8_attention_weight_min": -48.221736907958984,
      "activations/layer9_attention_weight_max": 47.540157318115234,
      "activations/layer9_attention_weight_min": -45.97921371459961,
      "epoch": 15.73,
      "learning_rate": 4.906136363636363e-05,
      "loss": 2.7653,
      "step": 270700
    },
    {
      "activations/layer0_attention_weight_max": 16.65329933166504,
      "activations/layer0_attention_weight_min": -13.697479248046875,
      "activations/layer10_attention_weight_max": 36.452083587646484,
      "activations/layer10_attention_weight_min": -37.9853515625,
      "activations/layer11_attention_weight_max": 31.573705673217773,
      "activations/layer11_attention_weight_min": -32.73469924926758,
      "activations/layer12_attention_weight_max": 18.24666404724121,
      "activations/layer12_attention_weight_min": -26.542707443237305,
      "activations/layer13_attention_weight_max": 35.70735549926758,
      "activations/layer13_attention_weight_min": -26.8005428314209,
      "activations/layer14_attention_weight_max": 40.18153762817383,
      "activations/layer14_attention_weight_min": -30.966991424560547,
      "activations/layer15_attention_weight_max": 33.655372619628906,
      "activations/layer15_attention_weight_min": -29.927072525024414,
      "activations/layer16_attention_weight_max": 35.10613250732422,
      "activations/layer16_attention_weight_min": -31.936840057373047,
      "activations/layer17_attention_weight_max": 54.14637756347656,
      "activations/layer17_attention_weight_min": -46.067726135253906,
      "activations/layer18_attention_weight_max": 46.2749137878418,
      "activations/layer18_attention_weight_min": -39.34686279296875,
      "activations/layer19_attention_weight_max": 21.620899200439453,
      "activations/layer19_attention_weight_min": -23.41718101501465,
      "activations/layer1_attention_weight_max": 16.412866592407227,
      "activations/layer1_attention_weight_min": -13.753987312316895,
      "activations/layer20_attention_weight_max": 26.018220901489258,
      "activations/layer20_attention_weight_min": -22.572416305541992,
      "activations/layer21_attention_weight_max": 39.0423469543457,
      "activations/layer21_attention_weight_min": -27.645549774169922,
      "activations/layer22_attention_weight_max": 27.27293586730957,
      "activations/layer22_attention_weight_min": -25.568899154663086,
      "activations/layer23_attention_weight_max": 35.72288513183594,
      "activations/layer23_attention_weight_min": -24.192859649658203,
      "activations/layer2_attention_weight_max": 30.788671493530273,
      "activations/layer2_attention_weight_min": -31.304248809814453,
      "activations/layer3_attention_weight_max": 92.0073013305664,
      "activations/layer3_attention_weight_min": -92.93714904785156,
      "activations/layer4_attention_weight_max": 91.56795501708984,
      "activations/layer4_attention_weight_min": -86.77716827392578,
      "activations/layer5_attention_weight_max": 66.73331451416016,
      "activations/layer5_attention_weight_min": -83.83753204345703,
      "activations/layer6_attention_weight_max": 49.68211364746094,
      "activations/layer6_attention_weight_min": -52.24481964111328,
      "activations/layer7_attention_weight_max": 59.20924758911133,
      "activations/layer7_attention_weight_min": -58.4012336730957,
      "activations/layer8_attention_weight_max": 47.13774108886719,
      "activations/layer8_attention_weight_min": -51.64918518066406,
      "activations/layer9_attention_weight_max": 48.35425567626953,
      "activations/layer9_attention_weight_min": -48.83187484741211,
      "epoch": 15.73,
      "learning_rate": 4.9042424242424233e-05,
      "loss": 2.75,
      "step": 270750
    },
    {
      "activations/layer0_attention_weight_max": 15.910140037536621,
      "activations/layer0_attention_weight_min": -13.935276985168457,
      "activations/layer10_attention_weight_max": 33.75647735595703,
      "activations/layer10_attention_weight_min": -34.33277130126953,
      "activations/layer11_attention_weight_max": 31.59705924987793,
      "activations/layer11_attention_weight_min": -32.75566864013672,
      "activations/layer12_attention_weight_max": 19.48008155822754,
      "activations/layer12_attention_weight_min": -26.97892951965332,
      "activations/layer13_attention_weight_max": 36.668663024902344,
      "activations/layer13_attention_weight_min": -28.133981704711914,
      "activations/layer14_attention_weight_max": 34.53981399536133,
      "activations/layer14_attention_weight_min": -29.03693962097168,
      "activations/layer15_attention_weight_max": 31.54470443725586,
      "activations/layer15_attention_weight_min": -28.702220916748047,
      "activations/layer16_attention_weight_max": 31.118432998657227,
      "activations/layer16_attention_weight_min": -30.13735008239746,
      "activations/layer17_attention_weight_max": 52.48759841918945,
      "activations/layer17_attention_weight_min": -46.43315505981445,
      "activations/layer18_attention_weight_max": 45.52543258666992,
      "activations/layer18_attention_weight_min": -41.782196044921875,
      "activations/layer19_attention_weight_max": 26.87137222290039,
      "activations/layer19_attention_weight_min": -22.11638069152832,
      "activations/layer1_attention_weight_max": 17.490585327148438,
      "activations/layer1_attention_weight_min": -13.18479061126709,
      "activations/layer20_attention_weight_max": 25.376569747924805,
      "activations/layer20_attention_weight_min": -20.90719985961914,
      "activations/layer21_attention_weight_max": 35.685176849365234,
      "activations/layer21_attention_weight_min": -21.322500228881836,
      "activations/layer22_attention_weight_max": 30.420413970947266,
      "activations/layer22_attention_weight_min": -24.657779693603516,
      "activations/layer23_attention_weight_max": 40.94162368774414,
      "activations/layer23_attention_weight_min": -25.26785659790039,
      "activations/layer2_attention_weight_max": 30.181644439697266,
      "activations/layer2_attention_weight_min": -26.928844451904297,
      "activations/layer3_attention_weight_max": 88.96218872070312,
      "activations/layer3_attention_weight_min": -86.48638153076172,
      "activations/layer4_attention_weight_max": 86.53661346435547,
      "activations/layer4_attention_weight_min": -86.92694854736328,
      "activations/layer5_attention_weight_max": 61.59430694580078,
      "activations/layer5_attention_weight_min": -70.77322387695312,
      "activations/layer6_attention_weight_max": 49.82060241699219,
      "activations/layer6_attention_weight_min": -48.14543914794922,
      "activations/layer7_attention_weight_max": 61.02033615112305,
      "activations/layer7_attention_weight_min": -65.40115356445312,
      "activations/layer8_attention_weight_max": 44.50822448730469,
      "activations/layer8_attention_weight_min": -45.926612854003906,
      "activations/layer9_attention_weight_max": 46.21003341674805,
      "activations/layer9_attention_weight_min": -44.357051849365234,
      "epoch": 15.74,
      "learning_rate": 4.902348484848485e-05,
      "loss": 2.7475,
      "step": 270800
    },
    {
      "activations/layer0_attention_weight_max": 17.408239364624023,
      "activations/layer0_attention_weight_min": -13.85877513885498,
      "activations/layer10_attention_weight_max": 39.27993392944336,
      "activations/layer10_attention_weight_min": -38.101051330566406,
      "activations/layer11_attention_weight_max": 36.21220016479492,
      "activations/layer11_attention_weight_min": -33.45391082763672,
      "activations/layer12_attention_weight_max": 19.236356735229492,
      "activations/layer12_attention_weight_min": -31.626506805419922,
      "activations/layer13_attention_weight_max": 34.27445983886719,
      "activations/layer13_attention_weight_min": -29.225513458251953,
      "activations/layer14_attention_weight_max": 37.248008728027344,
      "activations/layer14_attention_weight_min": -28.963085174560547,
      "activations/layer15_attention_weight_max": 33.11078643798828,
      "activations/layer15_attention_weight_min": -30.68113136291504,
      "activations/layer16_attention_weight_max": 32.136634826660156,
      "activations/layer16_attention_weight_min": -31.42072296142578,
      "activations/layer17_attention_weight_max": 51.48965835571289,
      "activations/layer17_attention_weight_min": -47.13173294067383,
      "activations/layer18_attention_weight_max": 46.739803314208984,
      "activations/layer18_attention_weight_min": -40.7103385925293,
      "activations/layer19_attention_weight_max": 25.43808937072754,
      "activations/layer19_attention_weight_min": -22.890193939208984,
      "activations/layer1_attention_weight_max": 16.216285705566406,
      "activations/layer1_attention_weight_min": -14.819310188293457,
      "activations/layer20_attention_weight_max": 22.001028060913086,
      "activations/layer20_attention_weight_min": -22.3096866607666,
      "activations/layer21_attention_weight_max": 38.22098159790039,
      "activations/layer21_attention_weight_min": -26.49349594116211,
      "activations/layer22_attention_weight_max": 26.92386817932129,
      "activations/layer22_attention_weight_min": -24.770736694335938,
      "activations/layer23_attention_weight_max": 36.201194763183594,
      "activations/layer23_attention_weight_min": -26.401060104370117,
      "activations/layer2_attention_weight_max": 31.61927032470703,
      "activations/layer2_attention_weight_min": -32.64173126220703,
      "activations/layer3_attention_weight_max": 95.76612854003906,
      "activations/layer3_attention_weight_min": -90.9229507446289,
      "activations/layer4_attention_weight_max": 98.65805053710938,
      "activations/layer4_attention_weight_min": -90.32913208007812,
      "activations/layer5_attention_weight_max": 69.11338806152344,
      "activations/layer5_attention_weight_min": -77.51081848144531,
      "activations/layer6_attention_weight_max": 55.757511138916016,
      "activations/layer6_attention_weight_min": -54.391109466552734,
      "activations/layer7_attention_weight_max": 63.22868728637695,
      "activations/layer7_attention_weight_min": -62.351287841796875,
      "activations/layer8_attention_weight_max": 50.4533576965332,
      "activations/layer8_attention_weight_min": -50.3906364440918,
      "activations/layer9_attention_weight_max": 47.6954460144043,
      "activations/layer9_attention_weight_min": -48.75287628173828,
      "epoch": 15.74,
      "learning_rate": 4.900454545454545e-05,
      "loss": 2.751,
      "step": 270850
    },
    {
      "activations/layer0_attention_weight_max": 15.701066970825195,
      "activations/layer0_attention_weight_min": -13.864278793334961,
      "activations/layer10_attention_weight_max": 32.389305114746094,
      "activations/layer10_attention_weight_min": -36.4864387512207,
      "activations/layer11_attention_weight_max": 32.70283508300781,
      "activations/layer11_attention_weight_min": -32.870025634765625,
      "activations/layer12_attention_weight_max": 17.178081512451172,
      "activations/layer12_attention_weight_min": -23.53969955444336,
      "activations/layer13_attention_weight_max": 45.1651496887207,
      "activations/layer13_attention_weight_min": -33.18720626831055,
      "activations/layer14_attention_weight_max": 55.56739807128906,
      "activations/layer14_attention_weight_min": -30.37237548828125,
      "activations/layer15_attention_weight_max": 50.1866455078125,
      "activations/layer15_attention_weight_min": -30.08207893371582,
      "activations/layer16_attention_weight_max": 35.00996017456055,
      "activations/layer16_attention_weight_min": -30.30584716796875,
      "activations/layer17_attention_weight_max": 63.11955642700195,
      "activations/layer17_attention_weight_min": -48.72454833984375,
      "activations/layer18_attention_weight_max": 46.657649993896484,
      "activations/layer18_attention_weight_min": -40.88263702392578,
      "activations/layer19_attention_weight_max": 22.833316802978516,
      "activations/layer19_attention_weight_min": -23.871305465698242,
      "activations/layer1_attention_weight_max": 15.48084545135498,
      "activations/layer1_attention_weight_min": -13.636754035949707,
      "activations/layer20_attention_weight_max": 25.420337677001953,
      "activations/layer20_attention_weight_min": -23.087987899780273,
      "activations/layer21_attention_weight_max": 46.11647033691406,
      "activations/layer21_attention_weight_min": -21.996986389160156,
      "activations/layer22_attention_weight_max": 32.151187896728516,
      "activations/layer22_attention_weight_min": -25.853256225585938,
      "activations/layer23_attention_weight_max": 34.653106689453125,
      "activations/layer23_attention_weight_min": -23.245182037353516,
      "activations/layer2_attention_weight_max": 31.41151237487793,
      "activations/layer2_attention_weight_min": -29.650798797607422,
      "activations/layer3_attention_weight_max": 92.5655288696289,
      "activations/layer3_attention_weight_min": -87.56272888183594,
      "activations/layer4_attention_weight_max": 95.11543273925781,
      "activations/layer4_attention_weight_min": -89.46041870117188,
      "activations/layer5_attention_weight_max": 65.64764404296875,
      "activations/layer5_attention_weight_min": -72.58296203613281,
      "activations/layer6_attention_weight_max": 49.19334411621094,
      "activations/layer6_attention_weight_min": -49.03946304321289,
      "activations/layer7_attention_weight_max": 63.305782318115234,
      "activations/layer7_attention_weight_min": -64.934814453125,
      "activations/layer8_attention_weight_max": 45.44275665283203,
      "activations/layer8_attention_weight_min": -48.92317199707031,
      "activations/layer9_attention_weight_max": 53.306514739990234,
      "activations/layer9_attention_weight_min": -44.6252555847168,
      "epoch": 15.74,
      "learning_rate": 4.898560606060606e-05,
      "loss": 2.7491,
      "step": 270900
    },
    {
      "activations/layer0_attention_weight_max": 17.072921752929688,
      "activations/layer0_attention_weight_min": -14.231759071350098,
      "activations/layer10_attention_weight_max": 37.31917190551758,
      "activations/layer10_attention_weight_min": -37.742225646972656,
      "activations/layer11_attention_weight_max": 35.986324310302734,
      "activations/layer11_attention_weight_min": -35.29936981201172,
      "activations/layer12_attention_weight_max": 18.558856964111328,
      "activations/layer12_attention_weight_min": -23.45852279663086,
      "activations/layer13_attention_weight_max": 39.797542572021484,
      "activations/layer13_attention_weight_min": -30.22540283203125,
      "activations/layer14_attention_weight_max": 36.768218994140625,
      "activations/layer14_attention_weight_min": -33.12874984741211,
      "activations/layer15_attention_weight_max": 34.0436897277832,
      "activations/layer15_attention_weight_min": -31.384231567382812,
      "activations/layer16_attention_weight_max": 32.396297454833984,
      "activations/layer16_attention_weight_min": -31.846818923950195,
      "activations/layer17_attention_weight_max": 53.577186584472656,
      "activations/layer17_attention_weight_min": -44.795562744140625,
      "activations/layer18_attention_weight_max": 45.49973678588867,
      "activations/layer18_attention_weight_min": -38.60917282104492,
      "activations/layer19_attention_weight_max": 25.158859252929688,
      "activations/layer19_attention_weight_min": -24.586109161376953,
      "activations/layer1_attention_weight_max": 15.799775123596191,
      "activations/layer1_attention_weight_min": -13.39722728729248,
      "activations/layer20_attention_weight_max": 24.754215240478516,
      "activations/layer20_attention_weight_min": -22.260141372680664,
      "activations/layer21_attention_weight_max": 39.45156478881836,
      "activations/layer21_attention_weight_min": -22.970155715942383,
      "activations/layer22_attention_weight_max": 29.52316665649414,
      "activations/layer22_attention_weight_min": -25.42679214477539,
      "activations/layer23_attention_weight_max": 37.894264221191406,
      "activations/layer23_attention_weight_min": -23.97777557373047,
      "activations/layer2_attention_weight_max": 31.6463680267334,
      "activations/layer2_attention_weight_min": -30.009597778320312,
      "activations/layer3_attention_weight_max": 93.67909240722656,
      "activations/layer3_attention_weight_min": -89.19418334960938,
      "activations/layer4_attention_weight_max": 96.6580581665039,
      "activations/layer4_attention_weight_min": -94.98528289794922,
      "activations/layer5_attention_weight_max": 73.10906219482422,
      "activations/layer5_attention_weight_min": -75.07452392578125,
      "activations/layer6_attention_weight_max": 53.79325866699219,
      "activations/layer6_attention_weight_min": -55.81077575683594,
      "activations/layer7_attention_weight_max": 66.98699188232422,
      "activations/layer7_attention_weight_min": -63.8629150390625,
      "activations/layer8_attention_weight_max": 49.30433654785156,
      "activations/layer8_attention_weight_min": -52.71196365356445,
      "activations/layer9_attention_weight_max": 47.86539077758789,
      "activations/layer9_attention_weight_min": -49.57781982421875,
      "epoch": 15.74,
      "learning_rate": 4.896666666666666e-05,
      "loss": 2.7377,
      "step": 270950
    },
    {
      "activations/layer0_attention_weight_max": 16.007034301757812,
      "activations/layer0_attention_weight_min": -14.252076148986816,
      "activations/layer10_attention_weight_max": 33.7571907043457,
      "activations/layer10_attention_weight_min": -33.240936279296875,
      "activations/layer11_attention_weight_max": 29.98386001586914,
      "activations/layer11_attention_weight_min": -30.998044967651367,
      "activations/layer12_attention_weight_max": 18.550146102905273,
      "activations/layer12_attention_weight_min": -29.602802276611328,
      "activations/layer13_attention_weight_max": 35.60061264038086,
      "activations/layer13_attention_weight_min": -30.597023010253906,
      "activations/layer14_attention_weight_max": 36.47138977050781,
      "activations/layer14_attention_weight_min": -28.54682731628418,
      "activations/layer15_attention_weight_max": 31.55320930480957,
      "activations/layer15_attention_weight_min": -30.082077026367188,
      "activations/layer16_attention_weight_max": 31.98169708251953,
      "activations/layer16_attention_weight_min": -30.153711318969727,
      "activations/layer17_attention_weight_max": 48.17277145385742,
      "activations/layer17_attention_weight_min": -41.48796463012695,
      "activations/layer18_attention_weight_max": 46.19099426269531,
      "activations/layer18_attention_weight_min": -37.28052520751953,
      "activations/layer19_attention_weight_max": 22.319660186767578,
      "activations/layer19_attention_weight_min": -22.29252052307129,
      "activations/layer1_attention_weight_max": 16.238910675048828,
      "activations/layer1_attention_weight_min": -13.221345901489258,
      "activations/layer20_attention_weight_max": 21.050113677978516,
      "activations/layer20_attention_weight_min": -20.438154220581055,
      "activations/layer21_attention_weight_max": 40.2928466796875,
      "activations/layer21_attention_weight_min": -23.388132095336914,
      "activations/layer22_attention_weight_max": 27.869762420654297,
      "activations/layer22_attention_weight_min": -26.42363739013672,
      "activations/layer23_attention_weight_max": 37.69611358642578,
      "activations/layer23_attention_weight_min": -23.958065032958984,
      "activations/layer2_attention_weight_max": 29.342308044433594,
      "activations/layer2_attention_weight_min": -30.05832290649414,
      "activations/layer3_attention_weight_max": 86.7047348022461,
      "activations/layer3_attention_weight_min": -92.72898864746094,
      "activations/layer4_attention_weight_max": 82.6124038696289,
      "activations/layer4_attention_weight_min": -83.23702239990234,
      "activations/layer5_attention_weight_max": 63.141380310058594,
      "activations/layer5_attention_weight_min": -71.24319458007812,
      "activations/layer6_attention_weight_max": 46.639305114746094,
      "activations/layer6_attention_weight_min": -48.46989059448242,
      "activations/layer7_attention_weight_max": 61.690269470214844,
      "activations/layer7_attention_weight_min": -56.33191680908203,
      "activations/layer8_attention_weight_max": 42.33367156982422,
      "activations/layer8_attention_weight_min": -46.792579650878906,
      "activations/layer9_attention_weight_max": 45.971923828125,
      "activations/layer9_attention_weight_min": -43.29509353637695,
      "epoch": 15.75,
      "learning_rate": 4.8947727272727275e-05,
      "loss": 2.7455,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_loss": 2.70703125,
      "eval_runtime": 8.4263,
      "eval_samples_per_second": 509.597,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_openwebtext_loss": 2.70703125,
      "eval_openwebtext_ppl": 14.984723517782793,
      "eval_openwebtext_runtime": 8.4263,
      "eval_openwebtext_samples_per_second": 509.597,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9407,
      "eval_wikitext_samples_per_second": 234.961,
      "step": 271000
    },
    {
      "epoch": 15.75,
      "eval_lambada_loss": 2.544921875,
      "eval_lambada_ppl": 12.742232560259533,
      "eval_lambada_runtime": 9.6198,
      "eval_lambada_samples_per_second": 506.145,
      "step": 271000
    },
    {
      "activations/layer0_attention_weight_max": 15.320658683776855,
      "activations/layer0_attention_weight_min": -14.514820098876953,
      "activations/layer10_attention_weight_max": 38.86063766479492,
      "activations/layer10_attention_weight_min": -39.79066467285156,
      "activations/layer11_attention_weight_max": 34.19063949584961,
      "activations/layer11_attention_weight_min": -35.33434295654297,
      "activations/layer12_attention_weight_max": 19.05218505859375,
      "activations/layer12_attention_weight_min": -27.330734252929688,
      "activations/layer13_attention_weight_max": 36.112388610839844,
      "activations/layer13_attention_weight_min": -28.395288467407227,
      "activations/layer14_attention_weight_max": 36.52901077270508,
      "activations/layer14_attention_weight_min": -31.018312454223633,
      "activations/layer15_attention_weight_max": 37.4804801940918,
      "activations/layer15_attention_weight_min": -30.77787971496582,
      "activations/layer16_attention_weight_max": 35.76919937133789,
      "activations/layer16_attention_weight_min": -32.230255126953125,
      "activations/layer17_attention_weight_max": 54.39848327636719,
      "activations/layer17_attention_weight_min": -51.122161865234375,
      "activations/layer18_attention_weight_max": 51.82231903076172,
      "activations/layer18_attention_weight_min": -43.59442901611328,
      "activations/layer19_attention_weight_max": 24.725500106811523,
      "activations/layer19_attention_weight_min": -23.508346557617188,
      "activations/layer1_attention_weight_max": 16.89457893371582,
      "activations/layer1_attention_weight_min": -13.610406875610352,
      "activations/layer20_attention_weight_max": 24.72447967529297,
      "activations/layer20_attention_weight_min": -22.089021682739258,
      "activations/layer21_attention_weight_max": 35.546531677246094,
      "activations/layer21_attention_weight_min": -24.075084686279297,
      "activations/layer22_attention_weight_max": 30.197744369506836,
      "activations/layer22_attention_weight_min": -27.09273338317871,
      "activations/layer23_attention_weight_max": 37.6208610534668,
      "activations/layer23_attention_weight_min": -23.313941955566406,
      "activations/layer2_attention_weight_max": 30.464370727539062,
      "activations/layer2_attention_weight_min": -29.378414154052734,
      "activations/layer3_attention_weight_max": 92.96111297607422,
      "activations/layer3_attention_weight_min": -87.28321075439453,
      "activations/layer4_attention_weight_max": 86.61231994628906,
      "activations/layer4_attention_weight_min": -89.2734603881836,
      "activations/layer5_attention_weight_max": 66.57157135009766,
      "activations/layer5_attention_weight_min": -74.60450744628906,
      "activations/layer6_attention_weight_max": 51.89747619628906,
      "activations/layer6_attention_weight_min": -54.8256950378418,
      "activations/layer7_attention_weight_max": 63.5257568359375,
      "activations/layer7_attention_weight_min": -59.899658203125,
      "activations/layer8_attention_weight_max": 49.29158020019531,
      "activations/layer8_attention_weight_min": -52.32010269165039,
      "activations/layer9_attention_weight_max": 48.49544906616211,
      "activations/layer9_attention_weight_min": -47.238101959228516,
      "epoch": 15.75,
      "learning_rate": 4.8928787878787876e-05,
      "loss": 2.739,
      "step": 271050
    },
    {
      "activations/layer0_attention_weight_max": 15.873434066772461,
      "activations/layer0_attention_weight_min": -13.918378829956055,
      "activations/layer10_attention_weight_max": 35.14985275268555,
      "activations/layer10_attention_weight_min": -36.99940872192383,
      "activations/layer11_attention_weight_max": 32.75791931152344,
      "activations/layer11_attention_weight_min": -33.12061309814453,
      "activations/layer12_attention_weight_max": 18.684946060180664,
      "activations/layer12_attention_weight_min": -28.69267463684082,
      "activations/layer13_attention_weight_max": 34.682891845703125,
      "activations/layer13_attention_weight_min": -28.609577178955078,
      "activations/layer14_attention_weight_max": 34.73687744140625,
      "activations/layer14_attention_weight_min": -29.605140686035156,
      "activations/layer15_attention_weight_max": 32.05104446411133,
      "activations/layer15_attention_weight_min": -30.704944610595703,
      "activations/layer16_attention_weight_max": 33.782196044921875,
      "activations/layer16_attention_weight_min": -32.11147689819336,
      "activations/layer17_attention_weight_max": 52.48168182373047,
      "activations/layer17_attention_weight_min": -45.0448112487793,
      "activations/layer18_attention_weight_max": 42.49531173706055,
      "activations/layer18_attention_weight_min": -39.27800750732422,
      "activations/layer19_attention_weight_max": 21.319259643554688,
      "activations/layer19_attention_weight_min": -22.05130386352539,
      "activations/layer1_attention_weight_max": 15.903759956359863,
      "activations/layer1_attention_weight_min": -13.139328002929688,
      "activations/layer20_attention_weight_max": 19.994159698486328,
      "activations/layer20_attention_weight_min": -20.875978469848633,
      "activations/layer21_attention_weight_max": 34.42784881591797,
      "activations/layer21_attention_weight_min": -22.680763244628906,
      "activations/layer22_attention_weight_max": 29.0372314453125,
      "activations/layer22_attention_weight_min": -27.092309951782227,
      "activations/layer23_attention_weight_max": 39.791831970214844,
      "activations/layer23_attention_weight_min": -23.183807373046875,
      "activations/layer2_attention_weight_max": 29.94148826599121,
      "activations/layer2_attention_weight_min": -29.93147087097168,
      "activations/layer3_attention_weight_max": 90.43539428710938,
      "activations/layer3_attention_weight_min": -92.05668640136719,
      "activations/layer4_attention_weight_max": 88.79803466796875,
      "activations/layer4_attention_weight_min": -88.2559585571289,
      "activations/layer5_attention_weight_max": 65.570556640625,
      "activations/layer5_attention_weight_min": -73.48115539550781,
      "activations/layer6_attention_weight_max": 48.82133483886719,
      "activations/layer6_attention_weight_min": -51.42787551879883,
      "activations/layer7_attention_weight_max": 58.90636444091797,
      "activations/layer7_attention_weight_min": -60.52861022949219,
      "activations/layer8_attention_weight_max": 43.58000183105469,
      "activations/layer8_attention_weight_min": -47.07676696777344,
      "activations/layer9_attention_weight_max": 44.196475982666016,
      "activations/layer9_attention_weight_min": -45.93720626831055,
      "epoch": 15.75,
      "learning_rate": 4.890984848484848e-05,
      "loss": 2.7452,
      "step": 271100
    },
    {
      "activations/layer0_attention_weight_max": 16.222318649291992,
      "activations/layer0_attention_weight_min": -14.202375411987305,
      "activations/layer10_attention_weight_max": 36.710601806640625,
      "activations/layer10_attention_weight_min": -35.599761962890625,
      "activations/layer11_attention_weight_max": 33.20653533935547,
      "activations/layer11_attention_weight_min": -33.29841995239258,
      "activations/layer12_attention_weight_max": 19.490642547607422,
      "activations/layer12_attention_weight_min": -28.964962005615234,
      "activations/layer13_attention_weight_max": 32.56306457519531,
      "activations/layer13_attention_weight_min": -27.133657455444336,
      "activations/layer14_attention_weight_max": 35.33450698852539,
      "activations/layer14_attention_weight_min": -29.917036056518555,
      "activations/layer15_attention_weight_max": 32.894901275634766,
      "activations/layer15_attention_weight_min": -30.504491806030273,
      "activations/layer16_attention_weight_max": 34.199127197265625,
      "activations/layer16_attention_weight_min": -33.94609832763672,
      "activations/layer17_attention_weight_max": 49.58217239379883,
      "activations/layer17_attention_weight_min": -47.31734848022461,
      "activations/layer18_attention_weight_max": 44.15577697753906,
      "activations/layer18_attention_weight_min": -39.714698791503906,
      "activations/layer19_attention_weight_max": 21.37384605407715,
      "activations/layer19_attention_weight_min": -22.11320686340332,
      "activations/layer1_attention_weight_max": 16.284908294677734,
      "activations/layer1_attention_weight_min": -13.922301292419434,
      "activations/layer20_attention_weight_max": 21.049116134643555,
      "activations/layer20_attention_weight_min": -22.523412704467773,
      "activations/layer21_attention_weight_max": 35.692779541015625,
      "activations/layer21_attention_weight_min": -25.37984275817871,
      "activations/layer22_attention_weight_max": 29.32962417602539,
      "activations/layer22_attention_weight_min": -26.887807846069336,
      "activations/layer23_attention_weight_max": 36.25825500488281,
      "activations/layer23_attention_weight_min": -22.718441009521484,
      "activations/layer2_attention_weight_max": 31.436201095581055,
      "activations/layer2_attention_weight_min": -31.186721801757812,
      "activations/layer3_attention_weight_max": 97.68302917480469,
      "activations/layer3_attention_weight_min": -94.29451751708984,
      "activations/layer4_attention_weight_max": 90.41534423828125,
      "activations/layer4_attention_weight_min": -90.83256530761719,
      "activations/layer5_attention_weight_max": 63.442100524902344,
      "activations/layer5_attention_weight_min": -72.59780883789062,
      "activations/layer6_attention_weight_max": 52.389381408691406,
      "activations/layer6_attention_weight_min": -52.22732925415039,
      "activations/layer7_attention_weight_max": 59.751548767089844,
      "activations/layer7_attention_weight_min": -59.705753326416016,
      "activations/layer8_attention_weight_max": 46.861663818359375,
      "activations/layer8_attention_weight_min": -50.11781692504883,
      "activations/layer9_attention_weight_max": 44.69810104370117,
      "activations/layer9_attention_weight_min": -46.86425018310547,
      "epoch": 15.76,
      "learning_rate": 4.8890909090909086e-05,
      "loss": 2.7233,
      "step": 271150
    },
    {
      "activations/layer0_attention_weight_max": 15.987412452697754,
      "activations/layer0_attention_weight_min": -14.085262298583984,
      "activations/layer10_attention_weight_max": 37.22062301635742,
      "activations/layer10_attention_weight_min": -35.56558609008789,
      "activations/layer11_attention_weight_max": 33.808876037597656,
      "activations/layer11_attention_weight_min": -35.39344787597656,
      "activations/layer12_attention_weight_max": 20.01358413696289,
      "activations/layer12_attention_weight_min": -29.715055465698242,
      "activations/layer13_attention_weight_max": 35.82878875732422,
      "activations/layer13_attention_weight_min": -31.789413452148438,
      "activations/layer14_attention_weight_max": 40.30208206176758,
      "activations/layer14_attention_weight_min": -33.786373138427734,
      "activations/layer15_attention_weight_max": 37.8367805480957,
      "activations/layer15_attention_weight_min": -31.875837326049805,
      "activations/layer16_attention_weight_max": 36.94679260253906,
      "activations/layer16_attention_weight_min": -34.440006256103516,
      "activations/layer17_attention_weight_max": 58.7468147277832,
      "activations/layer17_attention_weight_min": -48.95523452758789,
      "activations/layer18_attention_weight_max": 52.41764450073242,
      "activations/layer18_attention_weight_min": -46.51816177368164,
      "activations/layer19_attention_weight_max": 23.962575912475586,
      "activations/layer19_attention_weight_min": -23.269033432006836,
      "activations/layer1_attention_weight_max": 16.601787567138672,
      "activations/layer1_attention_weight_min": -16.111576080322266,
      "activations/layer20_attention_weight_max": 23.457908630371094,
      "activations/layer20_attention_weight_min": -22.5904598236084,
      "activations/layer21_attention_weight_max": 41.6233024597168,
      "activations/layer21_attention_weight_min": -23.559080123901367,
      "activations/layer22_attention_weight_max": 29.58588218688965,
      "activations/layer22_attention_weight_min": -25.57834243774414,
      "activations/layer23_attention_weight_max": 37.438865661621094,
      "activations/layer23_attention_weight_min": -22.952011108398438,
      "activations/layer2_attention_weight_max": 30.8782958984375,
      "activations/layer2_attention_weight_min": -29.674541473388672,
      "activations/layer3_attention_weight_max": 91.34841918945312,
      "activations/layer3_attention_weight_min": -91.8310317993164,
      "activations/layer4_attention_weight_max": 88.19908142089844,
      "activations/layer4_attention_weight_min": -87.66234588623047,
      "activations/layer5_attention_weight_max": 63.728023529052734,
      "activations/layer5_attention_weight_min": -73.61988830566406,
      "activations/layer6_attention_weight_max": 51.47100067138672,
      "activations/layer6_attention_weight_min": -52.0772590637207,
      "activations/layer7_attention_weight_max": 60.445159912109375,
      "activations/layer7_attention_weight_min": -62.87066650390625,
      "activations/layer8_attention_weight_max": 47.920894622802734,
      "activations/layer8_attention_weight_min": -49.47686004638672,
      "activations/layer9_attention_weight_max": 46.537105560302734,
      "activations/layer9_attention_weight_min": -45.94597625732422,
      "epoch": 15.76,
      "learning_rate": 4.8871969696969694e-05,
      "loss": 2.7388,
      "step": 271200
    },
    {
      "activations/layer0_attention_weight_max": 15.312714576721191,
      "activations/layer0_attention_weight_min": -14.23023796081543,
      "activations/layer10_attention_weight_max": 34.35820388793945,
      "activations/layer10_attention_weight_min": -35.465919494628906,
      "activations/layer11_attention_weight_max": 31.70931625366211,
      "activations/layer11_attention_weight_min": -32.06983947753906,
      "activations/layer12_attention_weight_max": 19.017135620117188,
      "activations/layer12_attention_weight_min": -26.244043350219727,
      "activations/layer13_attention_weight_max": 36.65098190307617,
      "activations/layer13_attention_weight_min": -28.50265884399414,
      "activations/layer14_attention_weight_max": 39.8496208190918,
      "activations/layer14_attention_weight_min": -30.8929443359375,
      "activations/layer15_attention_weight_max": 40.54354476928711,
      "activations/layer15_attention_weight_min": -32.61095428466797,
      "activations/layer16_attention_weight_max": 35.08159255981445,
      "activations/layer16_attention_weight_min": -33.08713150024414,
      "activations/layer17_attention_weight_max": 53.862998962402344,
      "activations/layer17_attention_weight_min": -45.44967269897461,
      "activations/layer18_attention_weight_max": 44.73396682739258,
      "activations/layer18_attention_weight_min": -41.91059112548828,
      "activations/layer19_attention_weight_max": 23.761503219604492,
      "activations/layer19_attention_weight_min": -23.49176788330078,
      "activations/layer1_attention_weight_max": 16.178184509277344,
      "activations/layer1_attention_weight_min": -13.47390365600586,
      "activations/layer20_attention_weight_max": 24.211040496826172,
      "activations/layer20_attention_weight_min": -23.665403366088867,
      "activations/layer21_attention_weight_max": 43.1999397277832,
      "activations/layer21_attention_weight_min": -23.8487548828125,
      "activations/layer22_attention_weight_max": 28.29010772705078,
      "activations/layer22_attention_weight_min": -27.816953659057617,
      "activations/layer23_attention_weight_max": 36.18718338012695,
      "activations/layer23_attention_weight_min": -26.312931060791016,
      "activations/layer2_attention_weight_max": 30.6469783782959,
      "activations/layer2_attention_weight_min": -29.236099243164062,
      "activations/layer3_attention_weight_max": 89.8819808959961,
      "activations/layer3_attention_weight_min": -88.90862274169922,
      "activations/layer4_attention_weight_max": 92.76830291748047,
      "activations/layer4_attention_weight_min": -90.78340148925781,
      "activations/layer5_attention_weight_max": 67.19929504394531,
      "activations/layer5_attention_weight_min": -72.79713439941406,
      "activations/layer6_attention_weight_max": 52.42915725708008,
      "activations/layer6_attention_weight_min": -51.92350769042969,
      "activations/layer7_attention_weight_max": 65.6559829711914,
      "activations/layer7_attention_weight_min": -61.36602020263672,
      "activations/layer8_attention_weight_max": 44.961585998535156,
      "activations/layer8_attention_weight_min": -48.87712860107422,
      "activations/layer9_attention_weight_max": 47.3662109375,
      "activations/layer9_attention_weight_min": -44.841819763183594,
      "epoch": 15.76,
      "learning_rate": 4.88530303030303e-05,
      "loss": 2.7348,
      "step": 271250
    },
    {
      "activations/layer0_attention_weight_max": 15.754889488220215,
      "activations/layer0_attention_weight_min": -14.267590522766113,
      "activations/layer10_attention_weight_max": 39.76713562011719,
      "activations/layer10_attention_weight_min": -36.91705322265625,
      "activations/layer11_attention_weight_max": 35.24883270263672,
      "activations/layer11_attention_weight_min": -35.285125732421875,
      "activations/layer12_attention_weight_max": 18.189218521118164,
      "activations/layer12_attention_weight_min": -23.953935623168945,
      "activations/layer13_attention_weight_max": 33.3680534362793,
      "activations/layer13_attention_weight_min": -28.883167266845703,
      "activations/layer14_attention_weight_max": 33.57167053222656,
      "activations/layer14_attention_weight_min": -30.474977493286133,
      "activations/layer15_attention_weight_max": 31.148448944091797,
      "activations/layer15_attention_weight_min": -31.03203582763672,
      "activations/layer16_attention_weight_max": 31.73008155822754,
      "activations/layer16_attention_weight_min": -29.9805965423584,
      "activations/layer17_attention_weight_max": 52.82060241699219,
      "activations/layer17_attention_weight_min": -44.93789291381836,
      "activations/layer18_attention_weight_max": 43.08063888549805,
      "activations/layer18_attention_weight_min": -37.32575225830078,
      "activations/layer19_attention_weight_max": 22.523157119750977,
      "activations/layer19_attention_weight_min": -21.9008846282959,
      "activations/layer1_attention_weight_max": 16.049123764038086,
      "activations/layer1_attention_weight_min": -15.379002571105957,
      "activations/layer20_attention_weight_max": 22.917020797729492,
      "activations/layer20_attention_weight_min": -21.266422271728516,
      "activations/layer21_attention_weight_max": 31.906429290771484,
      "activations/layer21_attention_weight_min": -23.054170608520508,
      "activations/layer22_attention_weight_max": 28.309412002563477,
      "activations/layer22_attention_weight_min": -23.48625946044922,
      "activations/layer23_attention_weight_max": 38.14846420288086,
      "activations/layer23_attention_weight_min": -25.522090911865234,
      "activations/layer2_attention_weight_max": 30.373523712158203,
      "activations/layer2_attention_weight_min": -31.215314865112305,
      "activations/layer3_attention_weight_max": 90.1723403930664,
      "activations/layer3_attention_weight_min": -88.0313491821289,
      "activations/layer4_attention_weight_max": 93.61066436767578,
      "activations/layer4_attention_weight_min": -90.68378448486328,
      "activations/layer5_attention_weight_max": 72.34679412841797,
      "activations/layer5_attention_weight_min": -74.46949768066406,
      "activations/layer6_attention_weight_max": 57.486572265625,
      "activations/layer6_attention_weight_min": -53.61589813232422,
      "activations/layer7_attention_weight_max": 65.87028503417969,
      "activations/layer7_attention_weight_min": -63.93336868286133,
      "activations/layer8_attention_weight_max": 49.4073371887207,
      "activations/layer8_attention_weight_min": -53.80049514770508,
      "activations/layer9_attention_weight_max": 49.21669387817383,
      "activations/layer9_attention_weight_min": -49.365169525146484,
      "epoch": 15.76,
      "learning_rate": 4.8834090909090904e-05,
      "loss": 2.7427,
      "step": 271300
    },
    {
      "activations/layer0_attention_weight_max": 15.147465705871582,
      "activations/layer0_attention_weight_min": -14.11609935760498,
      "activations/layer10_attention_weight_max": 39.31914520263672,
      "activations/layer10_attention_weight_min": -37.1363525390625,
      "activations/layer11_attention_weight_max": 35.24794006347656,
      "activations/layer11_attention_weight_min": -35.26539611816406,
      "activations/layer12_attention_weight_max": 19.770437240600586,
      "activations/layer12_attention_weight_min": -27.260290145874023,
      "activations/layer13_attention_weight_max": 33.659332275390625,
      "activations/layer13_attention_weight_min": -29.818788528442383,
      "activations/layer14_attention_weight_max": 35.99338150024414,
      "activations/layer14_attention_weight_min": -31.1243953704834,
      "activations/layer15_attention_weight_max": 33.40563201904297,
      "activations/layer15_attention_weight_min": -30.29828643798828,
      "activations/layer16_attention_weight_max": 31.379451751708984,
      "activations/layer16_attention_weight_min": -31.834278106689453,
      "activations/layer17_attention_weight_max": 50.812374114990234,
      "activations/layer17_attention_weight_min": -46.5407600402832,
      "activations/layer18_attention_weight_max": 43.96748352050781,
      "activations/layer18_attention_weight_min": -39.67671585083008,
      "activations/layer19_attention_weight_max": 21.41444969177246,
      "activations/layer19_attention_weight_min": -22.03874969482422,
      "activations/layer1_attention_weight_max": 17.023296356201172,
      "activations/layer1_attention_weight_min": -14.701221466064453,
      "activations/layer20_attention_weight_max": 22.3646240234375,
      "activations/layer20_attention_weight_min": -20.48305320739746,
      "activations/layer21_attention_weight_max": 34.45869827270508,
      "activations/layer21_attention_weight_min": -21.76787757873535,
      "activations/layer22_attention_weight_max": 31.165966033935547,
      "activations/layer22_attention_weight_min": -24.86750030517578,
      "activations/layer23_attention_weight_max": 37.11625289916992,
      "activations/layer23_attention_weight_min": -24.199539184570312,
      "activations/layer2_attention_weight_max": 30.370498657226562,
      "activations/layer2_attention_weight_min": -28.537015914916992,
      "activations/layer3_attention_weight_max": 90.62471771240234,
      "activations/layer3_attention_weight_min": -87.4424057006836,
      "activations/layer4_attention_weight_max": 95.36515808105469,
      "activations/layer4_attention_weight_min": -92.77052307128906,
      "activations/layer5_attention_weight_max": 70.55032348632812,
      "activations/layer5_attention_weight_min": -72.4544677734375,
      "activations/layer6_attention_weight_max": 54.59815216064453,
      "activations/layer6_attention_weight_min": -52.6803092956543,
      "activations/layer7_attention_weight_max": 67.61002349853516,
      "activations/layer7_attention_weight_min": -61.4334716796875,
      "activations/layer8_attention_weight_max": 48.657684326171875,
      "activations/layer8_attention_weight_min": -51.77861404418945,
      "activations/layer9_attention_weight_max": 50.765316009521484,
      "activations/layer9_attention_weight_min": -48.541282653808594,
      "epoch": 15.77,
      "learning_rate": 4.8815151515151506e-05,
      "loss": 2.7422,
      "step": 271350
    },
    {
      "activations/layer0_attention_weight_max": 15.996654510498047,
      "activations/layer0_attention_weight_min": -13.667844772338867,
      "activations/layer10_attention_weight_max": 36.7813606262207,
      "activations/layer10_attention_weight_min": -37.227447509765625,
      "activations/layer11_attention_weight_max": 33.19123840332031,
      "activations/layer11_attention_weight_min": -33.88629150390625,
      "activations/layer12_attention_weight_max": 19.3608341217041,
      "activations/layer12_attention_weight_min": -24.953020095825195,
      "activations/layer13_attention_weight_max": 36.421504974365234,
      "activations/layer13_attention_weight_min": -27.280433654785156,
      "activations/layer14_attention_weight_max": 37.1788215637207,
      "activations/layer14_attention_weight_min": -29.165006637573242,
      "activations/layer15_attention_weight_max": 33.059722900390625,
      "activations/layer15_attention_weight_min": -29.014432907104492,
      "activations/layer16_attention_weight_max": 33.879364013671875,
      "activations/layer16_attention_weight_min": -30.77915382385254,
      "activations/layer17_attention_weight_max": 52.67997360229492,
      "activations/layer17_attention_weight_min": -42.69533920288086,
      "activations/layer18_attention_weight_max": 49.397308349609375,
      "activations/layer18_attention_weight_min": -37.28975296020508,
      "activations/layer19_attention_weight_max": 25.450332641601562,
      "activations/layer19_attention_weight_min": -22.855545043945312,
      "activations/layer1_attention_weight_max": 15.808650970458984,
      "activations/layer1_attention_weight_min": -14.355597496032715,
      "activations/layer20_attention_weight_max": 25.260297775268555,
      "activations/layer20_attention_weight_min": -20.591381072998047,
      "activations/layer21_attention_weight_max": 50.149898529052734,
      "activations/layer21_attention_weight_min": -27.426767349243164,
      "activations/layer22_attention_weight_max": 32.154396057128906,
      "activations/layer22_attention_weight_min": -24.96619415283203,
      "activations/layer23_attention_weight_max": 38.4333381652832,
      "activations/layer23_attention_weight_min": -27.308887481689453,
      "activations/layer2_attention_weight_max": 31.699264526367188,
      "activations/layer2_attention_weight_min": -30.028507232666016,
      "activations/layer3_attention_weight_max": 90.78060150146484,
      "activations/layer3_attention_weight_min": -94.9490966796875,
      "activations/layer4_attention_weight_max": 93.46353149414062,
      "activations/layer4_attention_weight_min": -91.86666870117188,
      "activations/layer5_attention_weight_max": 67.43573760986328,
      "activations/layer5_attention_weight_min": -74.82221984863281,
      "activations/layer6_attention_weight_max": 53.90857696533203,
      "activations/layer6_attention_weight_min": -51.617469787597656,
      "activations/layer7_attention_weight_max": 65.0029296875,
      "activations/layer7_attention_weight_min": -61.98354721069336,
      "activations/layer8_attention_weight_max": 49.11442565917969,
      "activations/layer8_attention_weight_min": -50.35532760620117,
      "activations/layer9_attention_weight_max": 45.75300216674805,
      "activations/layer9_attention_weight_min": -46.85383605957031,
      "epoch": 15.77,
      "learning_rate": 4.879621212121212e-05,
      "loss": 2.7449,
      "step": 271400
    },
    {
      "activations/layer0_attention_weight_max": 15.822893142700195,
      "activations/layer0_attention_weight_min": -14.12932300567627,
      "activations/layer10_attention_weight_max": 36.285945892333984,
      "activations/layer10_attention_weight_min": -37.5908203125,
      "activations/layer11_attention_weight_max": 33.544368743896484,
      "activations/layer11_attention_weight_min": -36.14727783203125,
      "activations/layer12_attention_weight_max": 19.33536720275879,
      "activations/layer12_attention_weight_min": -22.689146041870117,
      "activations/layer13_attention_weight_max": 37.046749114990234,
      "activations/layer13_attention_weight_min": -28.5472412109375,
      "activations/layer14_attention_weight_max": 42.06608200073242,
      "activations/layer14_attention_weight_min": -33.559669494628906,
      "activations/layer15_attention_weight_max": 37.980648040771484,
      "activations/layer15_attention_weight_min": -32.266414642333984,
      "activations/layer16_attention_weight_max": 38.15953826904297,
      "activations/layer16_attention_weight_min": -36.23835372924805,
      "activations/layer17_attention_weight_max": 57.59307861328125,
      "activations/layer17_attention_weight_min": -53.0107307434082,
      "activations/layer18_attention_weight_max": 51.71382522583008,
      "activations/layer18_attention_weight_min": -43.113304138183594,
      "activations/layer19_attention_weight_max": 25.58728790283203,
      "activations/layer19_attention_weight_min": -23.1575927734375,
      "activations/layer1_attention_weight_max": 15.421101570129395,
      "activations/layer1_attention_weight_min": -13.419347763061523,
      "activations/layer20_attention_weight_max": 24.474943161010742,
      "activations/layer20_attention_weight_min": -21.926998138427734,
      "activations/layer21_attention_weight_max": 38.75913619995117,
      "activations/layer21_attention_weight_min": -21.32156753540039,
      "activations/layer22_attention_weight_max": 32.674827575683594,
      "activations/layer22_attention_weight_min": -27.957523345947266,
      "activations/layer23_attention_weight_max": 43.85102081298828,
      "activations/layer23_attention_weight_min": -22.816207885742188,
      "activations/layer2_attention_weight_max": 31.253524780273438,
      "activations/layer2_attention_weight_min": -29.319847106933594,
      "activations/layer3_attention_weight_max": 92.3857650756836,
      "activations/layer3_attention_weight_min": -88.33232879638672,
      "activations/layer4_attention_weight_max": 90.81922149658203,
      "activations/layer4_attention_weight_min": -89.48760223388672,
      "activations/layer5_attention_weight_max": 67.36500549316406,
      "activations/layer5_attention_weight_min": -72.75642395019531,
      "activations/layer6_attention_weight_max": 52.49658966064453,
      "activations/layer6_attention_weight_min": -52.58406448364258,
      "activations/layer7_attention_weight_max": 64.64292907714844,
      "activations/layer7_attention_weight_min": -63.9776725769043,
      "activations/layer8_attention_weight_max": 46.635868072509766,
      "activations/layer8_attention_weight_min": -53.0774040222168,
      "activations/layer9_attention_weight_max": 44.849281311035156,
      "activations/layer9_attention_weight_min": -48.279850006103516,
      "epoch": 15.77,
      "learning_rate": 4.877727272727272e-05,
      "loss": 2.7435,
      "step": 271450
    },
    {
      "activations/layer0_attention_weight_max": 15.512856483459473,
      "activations/layer0_attention_weight_min": -13.768379211425781,
      "activations/layer10_attention_weight_max": 33.254512786865234,
      "activations/layer10_attention_weight_min": -35.400821685791016,
      "activations/layer11_attention_weight_max": 31.599515914916992,
      "activations/layer11_attention_weight_min": -32.53683853149414,
      "activations/layer12_attention_weight_max": 18.647655487060547,
      "activations/layer12_attention_weight_min": -27.313831329345703,
      "activations/layer13_attention_weight_max": 38.80665588378906,
      "activations/layer13_attention_weight_min": -28.582984924316406,
      "activations/layer14_attention_weight_max": 34.915489196777344,
      "activations/layer14_attention_weight_min": -30.830673217773438,
      "activations/layer15_attention_weight_max": 33.91380310058594,
      "activations/layer15_attention_weight_min": -31.344629287719727,
      "activations/layer16_attention_weight_max": 32.00455093383789,
      "activations/layer16_attention_weight_min": -33.23215103149414,
      "activations/layer17_attention_weight_max": 49.47051239013672,
      "activations/layer17_attention_weight_min": -45.829647064208984,
      "activations/layer18_attention_weight_max": 45.4326286315918,
      "activations/layer18_attention_weight_min": -41.576011657714844,
      "activations/layer19_attention_weight_max": 23.65807342529297,
      "activations/layer19_attention_weight_min": -22.70047378540039,
      "activations/layer1_attention_weight_max": 16.759279251098633,
      "activations/layer1_attention_weight_min": -12.881064414978027,
      "activations/layer20_attention_weight_max": 21.196805953979492,
      "activations/layer20_attention_weight_min": -20.349090576171875,
      "activations/layer21_attention_weight_max": 38.20967483520508,
      "activations/layer21_attention_weight_min": -22.94407844543457,
      "activations/layer22_attention_weight_max": 27.979537963867188,
      "activations/layer22_attention_weight_min": -23.617177963256836,
      "activations/layer23_attention_weight_max": 35.2701416015625,
      "activations/layer23_attention_weight_min": -25.582130432128906,
      "activations/layer2_attention_weight_max": 29.69064712524414,
      "activations/layer2_attention_weight_min": -28.72388458251953,
      "activations/layer3_attention_weight_max": 89.62086486816406,
      "activations/layer3_attention_weight_min": -87.82559204101562,
      "activations/layer4_attention_weight_max": 88.09276580810547,
      "activations/layer4_attention_weight_min": -88.01453399658203,
      "activations/layer5_attention_weight_max": 63.090328216552734,
      "activations/layer5_attention_weight_min": -68.98607635498047,
      "activations/layer6_attention_weight_max": 51.29609680175781,
      "activations/layer6_attention_weight_min": -48.06532669067383,
      "activations/layer7_attention_weight_max": 58.624446868896484,
      "activations/layer7_attention_weight_min": -58.988739013671875,
      "activations/layer8_attention_weight_max": 43.08943557739258,
      "activations/layer8_attention_weight_min": -44.9838752746582,
      "activations/layer9_attention_weight_max": 42.261207580566406,
      "activations/layer9_attention_weight_min": -45.45732498168945,
      "epoch": 15.78,
      "learning_rate": 4.875871212121211e-05,
      "loss": 2.7386,
      "step": 271500
    },
    {
      "activations/layer0_attention_weight_max": 15.552081108093262,
      "activations/layer0_attention_weight_min": -13.724481582641602,
      "activations/layer10_attention_weight_max": 36.785491943359375,
      "activations/layer10_attention_weight_min": -36.50364303588867,
      "activations/layer11_attention_weight_max": 33.2755126953125,
      "activations/layer11_attention_weight_min": -33.662044525146484,
      "activations/layer12_attention_weight_max": 18.378738403320312,
      "activations/layer12_attention_weight_min": -25.63239860534668,
      "activations/layer13_attention_weight_max": 43.94375991821289,
      "activations/layer13_attention_weight_min": -30.801172256469727,
      "activations/layer14_attention_weight_max": 35.451332092285156,
      "activations/layer14_attention_weight_min": -33.10911178588867,
      "activations/layer15_attention_weight_max": 32.05173873901367,
      "activations/layer15_attention_weight_min": -31.78981590270996,
      "activations/layer16_attention_weight_max": 32.85483169555664,
      "activations/layer16_attention_weight_min": -30.96144676208496,
      "activations/layer17_attention_weight_max": 52.39078903198242,
      "activations/layer17_attention_weight_min": -46.55623245239258,
      "activations/layer18_attention_weight_max": 45.77131652832031,
      "activations/layer18_attention_weight_min": -41.4582633972168,
      "activations/layer19_attention_weight_max": 23.541257858276367,
      "activations/layer19_attention_weight_min": -23.960283279418945,
      "activations/layer1_attention_weight_max": 16.244504928588867,
      "activations/layer1_attention_weight_min": -14.095869064331055,
      "activations/layer20_attention_weight_max": 23.554174423217773,
      "activations/layer20_attention_weight_min": -24.384532928466797,
      "activations/layer21_attention_weight_max": 35.32440948486328,
      "activations/layer21_attention_weight_min": -25.467466354370117,
      "activations/layer22_attention_weight_max": 28.514875411987305,
      "activations/layer22_attention_weight_min": -28.756568908691406,
      "activations/layer23_attention_weight_max": 34.81732940673828,
      "activations/layer23_attention_weight_min": -29.35296630859375,
      "activations/layer2_attention_weight_max": 30.005699157714844,
      "activations/layer2_attention_weight_min": -28.98162841796875,
      "activations/layer3_attention_weight_max": 86.16555786132812,
      "activations/layer3_attention_weight_min": -85.39115142822266,
      "activations/layer4_attention_weight_max": 86.05269622802734,
      "activations/layer4_attention_weight_min": -93.37980651855469,
      "activations/layer5_attention_weight_max": 65.46903228759766,
      "activations/layer5_attention_weight_min": -74.01292419433594,
      "activations/layer6_attention_weight_max": 52.40853500366211,
      "activations/layer6_attention_weight_min": -54.42267990112305,
      "activations/layer7_attention_weight_max": 61.11945724487305,
      "activations/layer7_attention_weight_min": -59.238075256347656,
      "activations/layer8_attention_weight_max": 44.87211990356445,
      "activations/layer8_attention_weight_min": -48.70622253417969,
      "activations/layer9_attention_weight_max": 44.22085189819336,
      "activations/layer9_attention_weight_min": -46.01920700073242,
      "epoch": 15.78,
      "learning_rate": 4.873977272727272e-05,
      "loss": 2.7502,
      "step": 271550
    },
    {
      "activations/layer0_attention_weight_max": 15.64088249206543,
      "activations/layer0_attention_weight_min": -13.90362548828125,
      "activations/layer10_attention_weight_max": 43.85506057739258,
      "activations/layer10_attention_weight_min": -42.050418853759766,
      "activations/layer11_attention_weight_max": 42.358978271484375,
      "activations/layer11_attention_weight_min": -38.666107177734375,
      "activations/layer12_attention_weight_max": 18.656681060791016,
      "activations/layer12_attention_weight_min": -24.985252380371094,
      "activations/layer13_attention_weight_max": 37.4853515625,
      "activations/layer13_attention_weight_min": -30.115398406982422,
      "activations/layer14_attention_weight_max": 54.05178451538086,
      "activations/layer14_attention_weight_min": -31.446256637573242,
      "activations/layer15_attention_weight_max": 49.46952819824219,
      "activations/layer15_attention_weight_min": -31.174270629882812,
      "activations/layer16_attention_weight_max": 33.92481231689453,
      "activations/layer16_attention_weight_min": -32.43097686767578,
      "activations/layer17_attention_weight_max": 54.657962799072266,
      "activations/layer17_attention_weight_min": -52.81311798095703,
      "activations/layer18_attention_weight_max": 47.921363830566406,
      "activations/layer18_attention_weight_min": -47.87156677246094,
      "activations/layer19_attention_weight_max": 27.784421920776367,
      "activations/layer19_attention_weight_min": -28.611616134643555,
      "activations/layer1_attention_weight_max": 17.101116180419922,
      "activations/layer1_attention_weight_min": -14.25562858581543,
      "activations/layer20_attention_weight_max": 30.52416229248047,
      "activations/layer20_attention_weight_min": -25.442209243774414,
      "activations/layer21_attention_weight_max": 42.932037353515625,
      "activations/layer21_attention_weight_min": -25.330669403076172,
      "activations/layer22_attention_weight_max": 37.19341278076172,
      "activations/layer22_attention_weight_min": -28.001754760742188,
      "activations/layer23_attention_weight_max": 35.30834197998047,
      "activations/layer23_attention_weight_min": -23.075416564941406,
      "activations/layer2_attention_weight_max": 31.67110252380371,
      "activations/layer2_attention_weight_min": -30.05846405029297,
      "activations/layer3_attention_weight_max": 102.66453552246094,
      "activations/layer3_attention_weight_min": -92.1393051147461,
      "activations/layer4_attention_weight_max": 96.9032211303711,
      "activations/layer4_attention_weight_min": -89.2965087890625,
      "activations/layer5_attention_weight_max": 68.42182159423828,
      "activations/layer5_attention_weight_min": -74.41629791259766,
      "activations/layer6_attention_weight_max": 52.73442840576172,
      "activations/layer6_attention_weight_min": -50.73590850830078,
      "activations/layer7_attention_weight_max": 71.9976577758789,
      "activations/layer7_attention_weight_min": -63.378204345703125,
      "activations/layer8_attention_weight_max": 52.713706970214844,
      "activations/layer8_attention_weight_min": -49.15386199951172,
      "activations/layer9_attention_weight_max": 59.95429229736328,
      "activations/layer9_attention_weight_min": -52.944759368896484,
      "epoch": 15.78,
      "learning_rate": 4.872083333333333e-05,
      "loss": 2.7412,
      "step": 271600
    },
    {
      "activations/layer0_attention_weight_max": 16.11866569519043,
      "activations/layer0_attention_weight_min": -13.872654914855957,
      "activations/layer10_attention_weight_max": 40.185813903808594,
      "activations/layer10_attention_weight_min": -37.115516662597656,
      "activations/layer11_attention_weight_max": 33.5115966796875,
      "activations/layer11_attention_weight_min": -34.76464080810547,
      "activations/layer12_attention_weight_max": 18.576168060302734,
      "activations/layer12_attention_weight_min": -28.534000396728516,
      "activations/layer13_attention_weight_max": 40.48615646362305,
      "activations/layer13_attention_weight_min": -29.46706199645996,
      "activations/layer14_attention_weight_max": 33.290283203125,
      "activations/layer14_attention_weight_min": -29.883745193481445,
      "activations/layer15_attention_weight_max": 33.370880126953125,
      "activations/layer15_attention_weight_min": -28.690427780151367,
      "activations/layer16_attention_weight_max": 33.058021545410156,
      "activations/layer16_attention_weight_min": -31.409427642822266,
      "activations/layer17_attention_weight_max": 51.62877655029297,
      "activations/layer17_attention_weight_min": -46.73237609863281,
      "activations/layer18_attention_weight_max": 44.34033966064453,
      "activations/layer18_attention_weight_min": -39.39595413208008,
      "activations/layer19_attention_weight_max": 25.254247665405273,
      "activations/layer19_attention_weight_min": -24.94285774230957,
      "activations/layer1_attention_weight_max": 15.939281463623047,
      "activations/layer1_attention_weight_min": -12.56689167022705,
      "activations/layer20_attention_weight_max": 22.00000762939453,
      "activations/layer20_attention_weight_min": -24.707136154174805,
      "activations/layer21_attention_weight_max": 39.80597686767578,
      "activations/layer21_attention_weight_min": -28.40599250793457,
      "activations/layer22_attention_weight_max": 29.722259521484375,
      "activations/layer22_attention_weight_min": -27.373506546020508,
      "activations/layer23_attention_weight_max": 38.995323181152344,
      "activations/layer23_attention_weight_min": -25.95855712890625,
      "activations/layer2_attention_weight_max": 30.23456573486328,
      "activations/layer2_attention_weight_min": -28.588193893432617,
      "activations/layer3_attention_weight_max": 92.77920532226562,
      "activations/layer3_attention_weight_min": -92.96739196777344,
      "activations/layer4_attention_weight_max": 91.32344818115234,
      "activations/layer4_attention_weight_min": -89.59270477294922,
      "activations/layer5_attention_weight_max": 63.868736267089844,
      "activations/layer5_attention_weight_min": -81.26411437988281,
      "activations/layer6_attention_weight_max": 49.873416900634766,
      "activations/layer6_attention_weight_min": -49.504451751708984,
      "activations/layer7_attention_weight_max": 60.4675407409668,
      "activations/layer7_attention_weight_min": -57.95672607421875,
      "activations/layer8_attention_weight_max": 49.624961853027344,
      "activations/layer8_attention_weight_min": -50.033058166503906,
      "activations/layer9_attention_weight_max": 46.66925048828125,
      "activations/layer9_attention_weight_min": -45.46527862548828,
      "epoch": 15.78,
      "learning_rate": 4.870189393939394e-05,
      "loss": 2.7396,
      "step": 271650
    },
    {
      "activations/layer0_attention_weight_max": 15.939208030700684,
      "activations/layer0_attention_weight_min": -14.215988159179688,
      "activations/layer10_attention_weight_max": 35.94730758666992,
      "activations/layer10_attention_weight_min": -35.20310974121094,
      "activations/layer11_attention_weight_max": 32.60657501220703,
      "activations/layer11_attention_weight_min": -33.356666564941406,
      "activations/layer12_attention_weight_max": 20.860183715820312,
      "activations/layer12_attention_weight_min": -23.684106826782227,
      "activations/layer13_attention_weight_max": 35.11130142211914,
      "activations/layer13_attention_weight_min": -28.563701629638672,
      "activations/layer14_attention_weight_max": 35.4538459777832,
      "activations/layer14_attention_weight_min": -31.53729248046875,
      "activations/layer15_attention_weight_max": 33.032161712646484,
      "activations/layer15_attention_weight_min": -30.80994987487793,
      "activations/layer16_attention_weight_max": 32.63043975830078,
      "activations/layer16_attention_weight_min": -32.03718566894531,
      "activations/layer17_attention_weight_max": 50.66646194458008,
      "activations/layer17_attention_weight_min": -43.6166877746582,
      "activations/layer18_attention_weight_max": 45.69400405883789,
      "activations/layer18_attention_weight_min": -36.66715621948242,
      "activations/layer19_attention_weight_max": 21.83417320251465,
      "activations/layer19_attention_weight_min": -21.243675231933594,
      "activations/layer1_attention_weight_max": 17.167800903320312,
      "activations/layer1_attention_weight_min": -13.017675399780273,
      "activations/layer20_attention_weight_max": 21.403419494628906,
      "activations/layer20_attention_weight_min": -20.298070907592773,
      "activations/layer21_attention_weight_max": 34.59978485107422,
      "activations/layer21_attention_weight_min": -21.647430419921875,
      "activations/layer22_attention_weight_max": 30.660364151000977,
      "activations/layer22_attention_weight_min": -25.3680419921875,
      "activations/layer23_attention_weight_max": 34.41090774536133,
      "activations/layer23_attention_weight_min": -21.70081329345703,
      "activations/layer2_attention_weight_max": 29.345684051513672,
      "activations/layer2_attention_weight_min": -28.97783660888672,
      "activations/layer3_attention_weight_max": 97.48323059082031,
      "activations/layer3_attention_weight_min": -93.40369415283203,
      "activations/layer4_attention_weight_max": 93.85913848876953,
      "activations/layer4_attention_weight_min": -89.16619873046875,
      "activations/layer5_attention_weight_max": 70.24903869628906,
      "activations/layer5_attention_weight_min": -73.37071990966797,
      "activations/layer6_attention_weight_max": 51.12166976928711,
      "activations/layer6_attention_weight_min": -52.68053436279297,
      "activations/layer7_attention_weight_max": 68.87889099121094,
      "activations/layer7_attention_weight_min": -64.90914916992188,
      "activations/layer8_attention_weight_max": 44.76740264892578,
      "activations/layer8_attention_weight_min": -49.140621185302734,
      "activations/layer9_attention_weight_max": 48.212921142578125,
      "activations/layer9_attention_weight_min": -45.60276412963867,
      "epoch": 15.79,
      "learning_rate": 4.868295454545454e-05,
      "loss": 2.7482,
      "step": 271700
    },
    {
      "activations/layer0_attention_weight_max": 15.795119285583496,
      "activations/layer0_attention_weight_min": -13.545948028564453,
      "activations/layer10_attention_weight_max": 40.1671257019043,
      "activations/layer10_attention_weight_min": -38.55318832397461,
      "activations/layer11_attention_weight_max": 34.02526092529297,
      "activations/layer11_attention_weight_min": -34.914100646972656,
      "activations/layer12_attention_weight_max": 18.338327407836914,
      "activations/layer12_attention_weight_min": -24.005916595458984,
      "activations/layer13_attention_weight_max": 34.906978607177734,
      "activations/layer13_attention_weight_min": -26.316770553588867,
      "activations/layer14_attention_weight_max": 37.064964294433594,
      "activations/layer14_attention_weight_min": -32.02353286743164,
      "activations/layer15_attention_weight_max": 34.32804870605469,
      "activations/layer15_attention_weight_min": -30.012920379638672,
      "activations/layer16_attention_weight_max": 33.70323944091797,
      "activations/layer16_attention_weight_min": -30.491670608520508,
      "activations/layer17_attention_weight_max": 51.649662017822266,
      "activations/layer17_attention_weight_min": -41.8304443359375,
      "activations/layer18_attention_weight_max": 47.665653228759766,
      "activations/layer18_attention_weight_min": -37.69568634033203,
      "activations/layer19_attention_weight_max": 25.162975311279297,
      "activations/layer19_attention_weight_min": -22.57308006286621,
      "activations/layer1_attention_weight_max": 17.37272834777832,
      "activations/layer1_attention_weight_min": -12.630929946899414,
      "activations/layer20_attention_weight_max": 22.416053771972656,
      "activations/layer20_attention_weight_min": -21.768831253051758,
      "activations/layer21_attention_weight_max": 30.817794799804688,
      "activations/layer21_attention_weight_min": -20.20061683654785,
      "activations/layer22_attention_weight_max": 29.701690673828125,
      "activations/layer22_attention_weight_min": -24.838315963745117,
      "activations/layer23_attention_weight_max": 37.00818634033203,
      "activations/layer23_attention_weight_min": -22.769020080566406,
      "activations/layer2_attention_weight_max": 30.44940185546875,
      "activations/layer2_attention_weight_min": -30.370662689208984,
      "activations/layer3_attention_weight_max": 92.24456024169922,
      "activations/layer3_attention_weight_min": -89.99491882324219,
      "activations/layer4_attention_weight_max": 98.38909912109375,
      "activations/layer4_attention_weight_min": -94.92208099365234,
      "activations/layer5_attention_weight_max": 73.31440734863281,
      "activations/layer5_attention_weight_min": -76.17552947998047,
      "activations/layer6_attention_weight_max": 58.72947311401367,
      "activations/layer6_attention_weight_min": -55.66393280029297,
      "activations/layer7_attention_weight_max": 65.98706817626953,
      "activations/layer7_attention_weight_min": -64.48828887939453,
      "activations/layer8_attention_weight_max": 48.86800003051758,
      "activations/layer8_attention_weight_min": -52.58159255981445,
      "activations/layer9_attention_weight_max": 49.7783317565918,
      "activations/layer9_attention_weight_min": -48.74250411987305,
      "epoch": 15.79,
      "learning_rate": 4.866439393939394e-05,
      "loss": 2.7466,
      "step": 271750
    },
    {
      "activations/layer0_attention_weight_max": 15.58659553527832,
      "activations/layer0_attention_weight_min": -13.602910041809082,
      "activations/layer10_attention_weight_max": 37.03461456298828,
      "activations/layer10_attention_weight_min": -35.493473052978516,
      "activations/layer11_attention_weight_max": 32.96521759033203,
      "activations/layer11_attention_weight_min": -33.28340530395508,
      "activations/layer12_attention_weight_max": 20.025222778320312,
      "activations/layer12_attention_weight_min": -28.47443962097168,
      "activations/layer13_attention_weight_max": 40.02730941772461,
      "activations/layer13_attention_weight_min": -27.3619441986084,
      "activations/layer14_attention_weight_max": 37.478519439697266,
      "activations/layer14_attention_weight_min": -31.188861846923828,
      "activations/layer15_attention_weight_max": 30.471080780029297,
      "activations/layer15_attention_weight_min": -30.087251663208008,
      "activations/layer16_attention_weight_max": 34.17266845703125,
      "activations/layer16_attention_weight_min": -30.740657806396484,
      "activations/layer17_attention_weight_max": 54.29515075683594,
      "activations/layer17_attention_weight_min": -44.50215530395508,
      "activations/layer18_attention_weight_max": 52.67835235595703,
      "activations/layer18_attention_weight_min": -39.472740173339844,
      "activations/layer19_attention_weight_max": 25.113771438598633,
      "activations/layer19_attention_weight_min": -23.959741592407227,
      "activations/layer1_attention_weight_max": 15.330866813659668,
      "activations/layer1_attention_weight_min": -15.621299743652344,
      "activations/layer20_attention_weight_max": 24.588476181030273,
      "activations/layer20_attention_weight_min": -21.59861946105957,
      "activations/layer21_attention_weight_max": 40.55575942993164,
      "activations/layer21_attention_weight_min": -26.031320571899414,
      "activations/layer22_attention_weight_max": 29.523162841796875,
      "activations/layer22_attention_weight_min": -26.40502166748047,
      "activations/layer23_attention_weight_max": 41.667884826660156,
      "activations/layer23_attention_weight_min": -24.47928237915039,
      "activations/layer2_attention_weight_max": 29.669294357299805,
      "activations/layer2_attention_weight_min": -29.675247192382812,
      "activations/layer3_attention_weight_max": 87.82069396972656,
      "activations/layer3_attention_weight_min": -89.76776123046875,
      "activations/layer4_attention_weight_max": 91.01842498779297,
      "activations/layer4_attention_weight_min": -89.73578643798828,
      "activations/layer5_attention_weight_max": 64.88261413574219,
      "activations/layer5_attention_weight_min": -73.54684448242188,
      "activations/layer6_attention_weight_max": 52.00403594970703,
      "activations/layer6_attention_weight_min": -51.42792892456055,
      "activations/layer7_attention_weight_max": 63.29030227661133,
      "activations/layer7_attention_weight_min": -59.50949478149414,
      "activations/layer8_attention_weight_max": 45.97679901123047,
      "activations/layer8_attention_weight_min": -48.6657600402832,
      "activations/layer9_attention_weight_max": 43.66103744506836,
      "activations/layer9_attention_weight_min": -46.71445083618164,
      "epoch": 15.79,
      "learning_rate": 4.8645454545454545e-05,
      "loss": 2.7411,
      "step": 271800
    },
    {
      "activations/layer0_attention_weight_max": 17.34400177001953,
      "activations/layer0_attention_weight_min": -13.564779281616211,
      "activations/layer10_attention_weight_max": 45.35374069213867,
      "activations/layer10_attention_weight_min": -40.40860366821289,
      "activations/layer11_attention_weight_max": 44.76713180541992,
      "activations/layer11_attention_weight_min": -40.03315353393555,
      "activations/layer12_attention_weight_max": 21.950407028198242,
      "activations/layer12_attention_weight_min": -28.970552444458008,
      "activations/layer13_attention_weight_max": 58.474586486816406,
      "activations/layer13_attention_weight_min": -32.34319305419922,
      "activations/layer14_attention_weight_max": 75.47964477539062,
      "activations/layer14_attention_weight_min": -35.73641586303711,
      "activations/layer15_attention_weight_max": 67.7468490600586,
      "activations/layer15_attention_weight_min": -33.43088150024414,
      "activations/layer16_attention_weight_max": 39.8962516784668,
      "activations/layer16_attention_weight_min": -33.24018478393555,
      "activations/layer17_attention_weight_max": 57.12590789794922,
      "activations/layer17_attention_weight_min": -53.42176818847656,
      "activations/layer18_attention_weight_max": 53.34052658081055,
      "activations/layer18_attention_weight_min": -43.352352142333984,
      "activations/layer19_attention_weight_max": 27.130252838134766,
      "activations/layer19_attention_weight_min": -25.390430450439453,
      "activations/layer1_attention_weight_max": 15.612347602844238,
      "activations/layer1_attention_weight_min": -14.477805137634277,
      "activations/layer20_attention_weight_max": 29.932443618774414,
      "activations/layer20_attention_weight_min": -23.736772537231445,
      "activations/layer21_attention_weight_max": 52.148651123046875,
      "activations/layer21_attention_weight_min": -28.272212982177734,
      "activations/layer22_attention_weight_max": 32.8137321472168,
      "activations/layer22_attention_weight_min": -26.294092178344727,
      "activations/layer23_attention_weight_max": 45.19896697998047,
      "activations/layer23_attention_weight_min": -26.138103485107422,
      "activations/layer2_attention_weight_max": 37.138553619384766,
      "activations/layer2_attention_weight_min": -30.98779296875,
      "activations/layer3_attention_weight_max": 107.43152618408203,
      "activations/layer3_attention_weight_min": -97.44768524169922,
      "activations/layer4_attention_weight_max": 111.87688446044922,
      "activations/layer4_attention_weight_min": -93.55738067626953,
      "activations/layer5_attention_weight_max": 72.43275451660156,
      "activations/layer5_attention_weight_min": -72.49407196044922,
      "activations/layer6_attention_weight_max": 54.72502899169922,
      "activations/layer6_attention_weight_min": -54.0131721496582,
      "activations/layer7_attention_weight_max": 67.58280944824219,
      "activations/layer7_attention_weight_min": -63.21257781982422,
      "activations/layer8_attention_weight_max": 52.960182189941406,
      "activations/layer8_attention_weight_min": -49.08720779418945,
      "activations/layer9_attention_weight_max": 54.333717346191406,
      "activations/layer9_attention_weight_min": -52.37122344970703,
      "epoch": 15.8,
      "learning_rate": 4.8626515151515147e-05,
      "loss": 2.7548,
      "step": 271850
    },
    {
      "activations/layer0_attention_weight_max": 15.617441177368164,
      "activations/layer0_attention_weight_min": -13.682882308959961,
      "activations/layer10_attention_weight_max": 33.38652038574219,
      "activations/layer10_attention_weight_min": -33.100257873535156,
      "activations/layer11_attention_weight_max": 32.65027618408203,
      "activations/layer11_attention_weight_min": -31.82074737548828,
      "activations/layer12_attention_weight_max": 19.774459838867188,
      "activations/layer12_attention_weight_min": -24.990018844604492,
      "activations/layer13_attention_weight_max": 35.275428771972656,
      "activations/layer13_attention_weight_min": -27.51418113708496,
      "activations/layer14_attention_weight_max": 36.9111213684082,
      "activations/layer14_attention_weight_min": -31.029281616210938,
      "activations/layer15_attention_weight_max": 34.279930114746094,
      "activations/layer15_attention_weight_min": -31.053573608398438,
      "activations/layer16_attention_weight_max": 35.489990234375,
      "activations/layer16_attention_weight_min": -32.902286529541016,
      "activations/layer17_attention_weight_max": 53.49577713012695,
      "activations/layer17_attention_weight_min": -48.2010383605957,
      "activations/layer18_attention_weight_max": 48.64716339111328,
      "activations/layer18_attention_weight_min": -42.977806091308594,
      "activations/layer19_attention_weight_max": 23.608488082885742,
      "activations/layer19_attention_weight_min": -23.21550178527832,
      "activations/layer1_attention_weight_max": 16.21318817138672,
      "activations/layer1_attention_weight_min": -14.163025856018066,
      "activations/layer20_attention_weight_max": 23.511688232421875,
      "activations/layer20_attention_weight_min": -20.473241806030273,
      "activations/layer21_attention_weight_max": 40.68445587158203,
      "activations/layer21_attention_weight_min": -24.759403228759766,
      "activations/layer22_attention_weight_max": 31.49175262451172,
      "activations/layer22_attention_weight_min": -25.327199935913086,
      "activations/layer23_attention_weight_max": 38.95768737792969,
      "activations/layer23_attention_weight_min": -25.26508140563965,
      "activations/layer2_attention_weight_max": 28.545927047729492,
      "activations/layer2_attention_weight_min": -28.758047103881836,
      "activations/layer3_attention_weight_max": 91.1540298461914,
      "activations/layer3_attention_weight_min": -86.89502716064453,
      "activations/layer4_attention_weight_max": 86.59213256835938,
      "activations/layer4_attention_weight_min": -83.9217758178711,
      "activations/layer5_attention_weight_max": 60.74134063720703,
      "activations/layer5_attention_weight_min": -78.62478637695312,
      "activations/layer6_attention_weight_max": 49.88933181762695,
      "activations/layer6_attention_weight_min": -49.91969299316406,
      "activations/layer7_attention_weight_max": 58.49703598022461,
      "activations/layer7_attention_weight_min": -58.93819808959961,
      "activations/layer8_attention_weight_max": 43.27275466918945,
      "activations/layer8_attention_weight_min": -46.83599853515625,
      "activations/layer9_attention_weight_max": 42.12290573120117,
      "activations/layer9_attention_weight_min": -46.06454086303711,
      "epoch": 15.8,
      "learning_rate": 4.860757575757575e-05,
      "loss": 2.7453,
      "step": 271900
    },
    {
      "activations/layer0_attention_weight_max": 16.799163818359375,
      "activations/layer0_attention_weight_min": -13.926385879516602,
      "activations/layer10_attention_weight_max": 38.33208084106445,
      "activations/layer10_attention_weight_min": -36.8170051574707,
      "activations/layer11_attention_weight_max": 33.750553131103516,
      "activations/layer11_attention_weight_min": -34.5408935546875,
      "activations/layer12_attention_weight_max": 18.68339729309082,
      "activations/layer12_attention_weight_min": -24.82468605041504,
      "activations/layer13_attention_weight_max": 34.387657165527344,
      "activations/layer13_attention_weight_min": -29.884204864501953,
      "activations/layer14_attention_weight_max": 35.15132522583008,
      "activations/layer14_attention_weight_min": -32.386871337890625,
      "activations/layer15_attention_weight_max": 32.09817123413086,
      "activations/layer15_attention_weight_min": -32.828208923339844,
      "activations/layer16_attention_weight_max": 32.940513610839844,
      "activations/layer16_attention_weight_min": -32.416412353515625,
      "activations/layer17_attention_weight_max": 53.43501281738281,
      "activations/layer17_attention_weight_min": -48.221458435058594,
      "activations/layer18_attention_weight_max": 49.12456512451172,
      "activations/layer18_attention_weight_min": -40.98678970336914,
      "activations/layer19_attention_weight_max": 23.986770629882812,
      "activations/layer19_attention_weight_min": -23.358266830444336,
      "activations/layer1_attention_weight_max": 15.684585571289062,
      "activations/layer1_attention_weight_min": -14.284089088439941,
      "activations/layer20_attention_weight_max": 25.136850357055664,
      "activations/layer20_attention_weight_min": -22.870206832885742,
      "activations/layer21_attention_weight_max": 35.05010223388672,
      "activations/layer21_attention_weight_min": -24.108016967773438,
      "activations/layer22_attention_weight_max": 30.256107330322266,
      "activations/layer22_attention_weight_min": -26.777050018310547,
      "activations/layer23_attention_weight_max": 39.495765686035156,
      "activations/layer23_attention_weight_min": -25.91829490661621,
      "activations/layer2_attention_weight_max": 31.883609771728516,
      "activations/layer2_attention_weight_min": -31.075979232788086,
      "activations/layer3_attention_weight_max": 89.65879821777344,
      "activations/layer3_attention_weight_min": -89.95893096923828,
      "activations/layer4_attention_weight_max": 92.1438217163086,
      "activations/layer4_attention_weight_min": -90.83330535888672,
      "activations/layer5_attention_weight_max": 67.96147155761719,
      "activations/layer5_attention_weight_min": -73.86227416992188,
      "activations/layer6_attention_weight_max": 53.14421463012695,
      "activations/layer6_attention_weight_min": -52.647579193115234,
      "activations/layer7_attention_weight_max": 60.46333694458008,
      "activations/layer7_attention_weight_min": -61.409576416015625,
      "activations/layer8_attention_weight_max": 46.94037628173828,
      "activations/layer8_attention_weight_min": -53.10643768310547,
      "activations/layer9_attention_weight_max": 46.8902702331543,
      "activations/layer9_attention_weight_min": -49.072078704833984,
      "epoch": 15.8,
      "learning_rate": 4.858863636363636e-05,
      "loss": 2.7436,
      "step": 271950
    },
    {
      "activations/layer0_attention_weight_max": 15.711490631103516,
      "activations/layer0_attention_weight_min": -13.755498886108398,
      "activations/layer10_attention_weight_max": 39.657554626464844,
      "activations/layer10_attention_weight_min": -38.2115364074707,
      "activations/layer11_attention_weight_max": 35.36689376831055,
      "activations/layer11_attention_weight_min": -36.09177017211914,
      "activations/layer12_attention_weight_max": 19.00444793701172,
      "activations/layer12_attention_weight_min": -26.383522033691406,
      "activations/layer13_attention_weight_max": 38.84906005859375,
      "activations/layer13_attention_weight_min": -33.24016571044922,
      "activations/layer14_attention_weight_max": 34.62855529785156,
      "activations/layer14_attention_weight_min": -30.866172790527344,
      "activations/layer15_attention_weight_max": 31.624122619628906,
      "activations/layer15_attention_weight_min": -29.81572723388672,
      "activations/layer16_attention_weight_max": 32.803466796875,
      "activations/layer16_attention_weight_min": -31.099483489990234,
      "activations/layer17_attention_weight_max": 48.73143768310547,
      "activations/layer17_attention_weight_min": -44.09211730957031,
      "activations/layer18_attention_weight_max": 45.077816009521484,
      "activations/layer18_attention_weight_min": -39.83195877075195,
      "activations/layer19_attention_weight_max": 23.233251571655273,
      "activations/layer19_attention_weight_min": -22.482284545898438,
      "activations/layer1_attention_weight_max": 17.2412109375,
      "activations/layer1_attention_weight_min": -12.903107643127441,
      "activations/layer20_attention_weight_max": 22.784788131713867,
      "activations/layer20_attention_weight_min": -21.011070251464844,
      "activations/layer21_attention_weight_max": 33.972145080566406,
      "activations/layer21_attention_weight_min": -21.463783264160156,
      "activations/layer22_attention_weight_max": 29.98258399963379,
      "activations/layer22_attention_weight_min": -25.31859016418457,
      "activations/layer23_attention_weight_max": 38.587005615234375,
      "activations/layer23_attention_weight_min": -25.93633270263672,
      "activations/layer2_attention_weight_max": 29.62953758239746,
      "activations/layer2_attention_weight_min": -28.930320739746094,
      "activations/layer3_attention_weight_max": 93.68901824951172,
      "activations/layer3_attention_weight_min": -86.95266723632812,
      "activations/layer4_attention_weight_max": 92.3036117553711,
      "activations/layer4_attention_weight_min": -90.50701141357422,
      "activations/layer5_attention_weight_max": 62.0487060546875,
      "activations/layer5_attention_weight_min": -78.94769287109375,
      "activations/layer6_attention_weight_max": 51.555885314941406,
      "activations/layer6_attention_weight_min": -52.57872009277344,
      "activations/layer7_attention_weight_max": 60.52595520019531,
      "activations/layer7_attention_weight_min": -60.68561553955078,
      "activations/layer8_attention_weight_max": 47.32854461669922,
      "activations/layer8_attention_weight_min": -52.04317855834961,
      "activations/layer9_attention_weight_max": 47.365055084228516,
      "activations/layer9_attention_weight_min": -50.026084899902344,
      "epoch": 15.8,
      "learning_rate": 4.8569696969696965e-05,
      "loss": 2.7432,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4418,
      "eval_samples_per_second": 508.657,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4418,
      "eval_openwebtext_samples_per_second": 508.657,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9234,
      "eval_wikitext_samples_per_second": 237.084,
      "step": 272000
    },
    {
      "epoch": 15.8,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5175,
      "eval_lambada_samples_per_second": 511.584,
      "step": 272000
    },
    {
      "activations/layer0_attention_weight_max": 15.985344886779785,
      "activations/layer0_attention_weight_min": -13.798967361450195,
      "activations/layer10_attention_weight_max": 35.33089828491211,
      "activations/layer10_attention_weight_min": -34.98426055908203,
      "activations/layer11_attention_weight_max": 33.293190002441406,
      "activations/layer11_attention_weight_min": -31.75882339477539,
      "activations/layer12_attention_weight_max": 18.674457550048828,
      "activations/layer12_attention_weight_min": -27.67027473449707,
      "activations/layer13_attention_weight_max": 32.44948959350586,
      "activations/layer13_attention_weight_min": -28.820146560668945,
      "activations/layer14_attention_weight_max": 37.645389556884766,
      "activations/layer14_attention_weight_min": -30.36598014831543,
      "activations/layer15_attention_weight_max": 41.14796829223633,
      "activations/layer15_attention_weight_min": -29.905384063720703,
      "activations/layer16_attention_weight_max": 34.86469268798828,
      "activations/layer16_attention_weight_min": -31.1214599609375,
      "activations/layer17_attention_weight_max": 55.13383483886719,
      "activations/layer17_attention_weight_min": -46.11070251464844,
      "activations/layer18_attention_weight_max": 51.19571304321289,
      "activations/layer18_attention_weight_min": -40.56436538696289,
      "activations/layer19_attention_weight_max": 25.223913192749023,
      "activations/layer19_attention_weight_min": -25.413183212280273,
      "activations/layer1_attention_weight_max": 16.612606048583984,
      "activations/layer1_attention_weight_min": -14.924537658691406,
      "activations/layer20_attention_weight_max": 25.022361755371094,
      "activations/layer20_attention_weight_min": -23.00746726989746,
      "activations/layer21_attention_weight_max": 44.30445098876953,
      "activations/layer21_attention_weight_min": -23.45085334777832,
      "activations/layer22_attention_weight_max": 38.86369705200195,
      "activations/layer22_attention_weight_min": -26.36598014831543,
      "activations/layer23_attention_weight_max": 42.284263610839844,
      "activations/layer23_attention_weight_min": -24.710716247558594,
      "activations/layer2_attention_weight_max": 31.651046752929688,
      "activations/layer2_attention_weight_min": -30.180416107177734,
      "activations/layer3_attention_weight_max": 89.93817138671875,
      "activations/layer3_attention_weight_min": -90.85767364501953,
      "activations/layer4_attention_weight_max": 86.89176940917969,
      "activations/layer4_attention_weight_min": -87.62914276123047,
      "activations/layer5_attention_weight_max": 61.806148529052734,
      "activations/layer5_attention_weight_min": -69.9439697265625,
      "activations/layer6_attention_weight_max": 47.45266342163086,
      "activations/layer6_attention_weight_min": -48.177757263183594,
      "activations/layer7_attention_weight_max": 63.047882080078125,
      "activations/layer7_attention_weight_min": -57.50482940673828,
      "activations/layer8_attention_weight_max": 47.38829803466797,
      "activations/layer8_attention_weight_min": -44.77840805053711,
      "activations/layer9_attention_weight_max": 51.97504806518555,
      "activations/layer9_attention_weight_min": -45.14726257324219,
      "epoch": 15.81,
      "learning_rate": 4.855075757575757e-05,
      "loss": 2.7455,
      "step": 272050
    },
    {
      "activations/layer0_attention_weight_max": 14.81240177154541,
      "activations/layer0_attention_weight_min": -13.7103910446167,
      "activations/layer10_attention_weight_max": 37.687068939208984,
      "activations/layer10_attention_weight_min": -36.92621612548828,
      "activations/layer11_attention_weight_max": 32.64582061767578,
      "activations/layer11_attention_weight_min": -33.75041961669922,
      "activations/layer12_attention_weight_max": 20.410314559936523,
      "activations/layer12_attention_weight_min": -22.98896598815918,
      "activations/layer13_attention_weight_max": 34.456329345703125,
      "activations/layer13_attention_weight_min": -27.334115982055664,
      "activations/layer14_attention_weight_max": 36.98728561401367,
      "activations/layer14_attention_weight_min": -29.527219772338867,
      "activations/layer15_attention_weight_max": 33.849151611328125,
      "activations/layer15_attention_weight_min": -29.86508560180664,
      "activations/layer16_attention_weight_max": 35.120670318603516,
      "activations/layer16_attention_weight_min": -31.836885452270508,
      "activations/layer17_attention_weight_max": 54.90468215942383,
      "activations/layer17_attention_weight_min": -45.35267639160156,
      "activations/layer18_attention_weight_max": 45.36270523071289,
      "activations/layer18_attention_weight_min": -41.092491149902344,
      "activations/layer19_attention_weight_max": 24.83509635925293,
      "activations/layer19_attention_weight_min": -24.105844497680664,
      "activations/layer1_attention_weight_max": 15.982855796813965,
      "activations/layer1_attention_weight_min": -14.532064437866211,
      "activations/layer20_attention_weight_max": 21.52298355102539,
      "activations/layer20_attention_weight_min": -21.86907196044922,
      "activations/layer21_attention_weight_max": 45.62773132324219,
      "activations/layer21_attention_weight_min": -30.948606491088867,
      "activations/layer22_attention_weight_max": 29.353458404541016,
      "activations/layer22_attention_weight_min": -28.61136817932129,
      "activations/layer23_attention_weight_max": 36.168121337890625,
      "activations/layer23_attention_weight_min": -25.143310546875,
      "activations/layer2_attention_weight_max": 32.14940643310547,
      "activations/layer2_attention_weight_min": -30.996627807617188,
      "activations/layer3_attention_weight_max": 92.27644348144531,
      "activations/layer3_attention_weight_min": -97.63264465332031,
      "activations/layer4_attention_weight_max": 94.7569351196289,
      "activations/layer4_attention_weight_min": -91.84072875976562,
      "activations/layer5_attention_weight_max": 68.96426391601562,
      "activations/layer5_attention_weight_min": -77.36963653564453,
      "activations/layer6_attention_weight_max": 51.51602554321289,
      "activations/layer6_attention_weight_min": -52.62018966674805,
      "activations/layer7_attention_weight_max": 61.52867126464844,
      "activations/layer7_attention_weight_min": -65.6128158569336,
      "activations/layer8_attention_weight_max": 47.97011184692383,
      "activations/layer8_attention_weight_min": -50.04575729370117,
      "activations/layer9_attention_weight_max": 46.932735443115234,
      "activations/layer9_attention_weight_min": -47.289817810058594,
      "epoch": 15.81,
      "learning_rate": 4.8531818181818174e-05,
      "loss": 2.7546,
      "step": 272100
    },
    {
      "activations/layer0_attention_weight_max": 14.853639602661133,
      "activations/layer0_attention_weight_min": -13.719576835632324,
      "activations/layer10_attention_weight_max": 35.264060974121094,
      "activations/layer10_attention_weight_min": -36.77201461791992,
      "activations/layer11_attention_weight_max": 31.580251693725586,
      "activations/layer11_attention_weight_min": -37.227760314941406,
      "activations/layer12_attention_weight_max": 18.246803283691406,
      "activations/layer12_attention_weight_min": -26.99397087097168,
      "activations/layer13_attention_weight_max": 30.233644485473633,
      "activations/layer13_attention_weight_min": -29.37116050720215,
      "activations/layer14_attention_weight_max": 33.29618835449219,
      "activations/layer14_attention_weight_min": -29.15974235534668,
      "activations/layer15_attention_weight_max": 30.931318283081055,
      "activations/layer15_attention_weight_min": -28.378826141357422,
      "activations/layer16_attention_weight_max": 29.81346321105957,
      "activations/layer16_attention_weight_min": -28.763671875,
      "activations/layer17_attention_weight_max": 49.79682159423828,
      "activations/layer17_attention_weight_min": -43.4844856262207,
      "activations/layer18_attention_weight_max": 45.1106071472168,
      "activations/layer18_attention_weight_min": -39.44427490234375,
      "activations/layer19_attention_weight_max": 20.911170959472656,
      "activations/layer19_attention_weight_min": -22.787595748901367,
      "activations/layer1_attention_weight_max": 14.954085350036621,
      "activations/layer1_attention_weight_min": -14.92435073852539,
      "activations/layer20_attention_weight_max": 21.81910514831543,
      "activations/layer20_attention_weight_min": -22.217124938964844,
      "activations/layer21_attention_weight_max": 33.75447463989258,
      "activations/layer21_attention_weight_min": -23.900310516357422,
      "activations/layer22_attention_weight_max": 27.201244354248047,
      "activations/layer22_attention_weight_min": -27.563602447509766,
      "activations/layer23_attention_weight_max": 33.0492057800293,
      "activations/layer23_attention_weight_min": -22.653297424316406,
      "activations/layer2_attention_weight_max": 31.10422134399414,
      "activations/layer2_attention_weight_min": -29.069501876831055,
      "activations/layer3_attention_weight_max": 88.43608093261719,
      "activations/layer3_attention_weight_min": -86.3958511352539,
      "activations/layer4_attention_weight_max": 89.39788055419922,
      "activations/layer4_attention_weight_min": -88.47045135498047,
      "activations/layer5_attention_weight_max": 65.20309448242188,
      "activations/layer5_attention_weight_min": -72.90808868408203,
      "activations/layer6_attention_weight_max": 50.50861740112305,
      "activations/layer6_attention_weight_min": -51.69959259033203,
      "activations/layer7_attention_weight_max": 60.92289733886719,
      "activations/layer7_attention_weight_min": -59.813961029052734,
      "activations/layer8_attention_weight_max": 48.44657897949219,
      "activations/layer8_attention_weight_min": -49.3019905090332,
      "activations/layer9_attention_weight_max": 44.3631477355957,
      "activations/layer9_attention_weight_min": -46.52252197265625,
      "epoch": 15.81,
      "learning_rate": 4.851287878787879e-05,
      "loss": 2.7468,
      "step": 272150
    },
    {
      "activations/layer0_attention_weight_max": 16.2573184967041,
      "activations/layer0_attention_weight_min": -13.73389720916748,
      "activations/layer10_attention_weight_max": 36.46708679199219,
      "activations/layer10_attention_weight_min": -37.177852630615234,
      "activations/layer11_attention_weight_max": 37.346710205078125,
      "activations/layer11_attention_weight_min": -34.57896041870117,
      "activations/layer12_attention_weight_max": 19.83700942993164,
      "activations/layer12_attention_weight_min": -34.03932571411133,
      "activations/layer13_attention_weight_max": 36.00536346435547,
      "activations/layer13_attention_weight_min": -31.018152236938477,
      "activations/layer14_attention_weight_max": 35.869564056396484,
      "activations/layer14_attention_weight_min": -31.97396469116211,
      "activations/layer15_attention_weight_max": 34.671600341796875,
      "activations/layer15_attention_weight_min": -30.44448471069336,
      "activations/layer16_attention_weight_max": 36.85519027709961,
      "activations/layer16_attention_weight_min": -34.0507698059082,
      "activations/layer17_attention_weight_max": 55.42372512817383,
      "activations/layer17_attention_weight_min": -48.83167266845703,
      "activations/layer18_attention_weight_max": 53.01131820678711,
      "activations/layer18_attention_weight_min": -41.23701858520508,
      "activations/layer19_attention_weight_max": 27.369184494018555,
      "activations/layer19_attention_weight_min": -23.391740798950195,
      "activations/layer1_attention_weight_max": 15.834939956665039,
      "activations/layer1_attention_weight_min": -12.951229095458984,
      "activations/layer20_attention_weight_max": 26.759502410888672,
      "activations/layer20_attention_weight_min": -22.567720413208008,
      "activations/layer21_attention_weight_max": 37.18964767456055,
      "activations/layer21_attention_weight_min": -23.026315689086914,
      "activations/layer22_attention_weight_max": 30.276020050048828,
      "activations/layer22_attention_weight_min": -25.748790740966797,
      "activations/layer23_attention_weight_max": 39.203826904296875,
      "activations/layer23_attention_weight_min": -24.23470115661621,
      "activations/layer2_attention_weight_max": 30.186065673828125,
      "activations/layer2_attention_weight_min": -29.37824058532715,
      "activations/layer3_attention_weight_max": 92.4944076538086,
      "activations/layer3_attention_weight_min": -89.97179412841797,
      "activations/layer4_attention_weight_max": 94.7634506225586,
      "activations/layer4_attention_weight_min": -92.94881439208984,
      "activations/layer5_attention_weight_max": 68.0223159790039,
      "activations/layer5_attention_weight_min": -77.37043762207031,
      "activations/layer6_attention_weight_max": 50.87460708618164,
      "activations/layer6_attention_weight_min": -52.11569595336914,
      "activations/layer7_attention_weight_max": 61.670867919921875,
      "activations/layer7_attention_weight_min": -62.94767761230469,
      "activations/layer8_attention_weight_max": 48.17888259887695,
      "activations/layer8_attention_weight_min": -51.31965637207031,
      "activations/layer9_attention_weight_max": 51.999908447265625,
      "activations/layer9_attention_weight_min": -48.72294616699219,
      "epoch": 15.82,
      "learning_rate": 4.849393939393939e-05,
      "loss": 2.7368,
      "step": 272200
    },
    {
      "activations/layer0_attention_weight_max": 15.598282814025879,
      "activations/layer0_attention_weight_min": -14.251015663146973,
      "activations/layer10_attention_weight_max": 36.19257354736328,
      "activations/layer10_attention_weight_min": -36.34608840942383,
      "activations/layer11_attention_weight_max": 34.18046569824219,
      "activations/layer11_attention_weight_min": -36.40177917480469,
      "activations/layer12_attention_weight_max": 18.263492584228516,
      "activations/layer12_attention_weight_min": -30.310209274291992,
      "activations/layer13_attention_weight_max": 31.987733840942383,
      "activations/layer13_attention_weight_min": -28.925655364990234,
      "activations/layer14_attention_weight_max": 35.58516311645508,
      "activations/layer14_attention_weight_min": -31.940475463867188,
      "activations/layer15_attention_weight_max": 32.26102066040039,
      "activations/layer15_attention_weight_min": -34.48145294189453,
      "activations/layer16_attention_weight_max": 30.448055267333984,
      "activations/layer16_attention_weight_min": -33.5787353515625,
      "activations/layer17_attention_weight_max": 45.771602630615234,
      "activations/layer17_attention_weight_min": -47.600379943847656,
      "activations/layer18_attention_weight_max": 44.748172760009766,
      "activations/layer18_attention_weight_min": -39.604671478271484,
      "activations/layer19_attention_weight_max": 24.814485549926758,
      "activations/layer19_attention_weight_min": -23.642454147338867,
      "activations/layer1_attention_weight_max": 16.315532684326172,
      "activations/layer1_attention_weight_min": -14.046738624572754,
      "activations/layer20_attention_weight_max": 23.406017303466797,
      "activations/layer20_attention_weight_min": -23.666847229003906,
      "activations/layer21_attention_weight_max": 31.87139892578125,
      "activations/layer21_attention_weight_min": -25.40013885498047,
      "activations/layer22_attention_weight_max": 28.374740600585938,
      "activations/layer22_attention_weight_min": -27.104251861572266,
      "activations/layer23_attention_weight_max": 31.43787956237793,
      "activations/layer23_attention_weight_min": -25.285884857177734,
      "activations/layer2_attention_weight_max": 32.49830627441406,
      "activations/layer2_attention_weight_min": -30.047374725341797,
      "activations/layer3_attention_weight_max": 99.07861328125,
      "activations/layer3_attention_weight_min": -99.83902740478516,
      "activations/layer4_attention_weight_max": 98.62495422363281,
      "activations/layer4_attention_weight_min": -92.02978515625,
      "activations/layer5_attention_weight_max": 63.206417083740234,
      "activations/layer5_attention_weight_min": -68.96764373779297,
      "activations/layer6_attention_weight_max": 47.35219192504883,
      "activations/layer6_attention_weight_min": -50.55214309692383,
      "activations/layer7_attention_weight_max": 64.62847900390625,
      "activations/layer7_attention_weight_min": -59.914859771728516,
      "activations/layer8_attention_weight_max": 44.17982482910156,
      "activations/layer8_attention_weight_min": -50.91688919067383,
      "activations/layer9_attention_weight_max": 51.53265380859375,
      "activations/layer9_attention_weight_min": -47.3414421081543,
      "epoch": 15.82,
      "learning_rate": 4.847499999999999e-05,
      "loss": 2.748,
      "step": 272250
    },
    {
      "activations/layer0_attention_weight_max": 15.898092269897461,
      "activations/layer0_attention_weight_min": -14.177276611328125,
      "activations/layer10_attention_weight_max": 34.24053192138672,
      "activations/layer10_attention_weight_min": -34.11333465576172,
      "activations/layer11_attention_weight_max": 32.847747802734375,
      "activations/layer11_attention_weight_min": -33.10000991821289,
      "activations/layer12_attention_weight_max": 19.245925903320312,
      "activations/layer12_attention_weight_min": -25.58797836303711,
      "activations/layer13_attention_weight_max": 35.107139587402344,
      "activations/layer13_attention_weight_min": -30.608917236328125,
      "activations/layer14_attention_weight_max": 34.427146911621094,
      "activations/layer14_attention_weight_min": -30.548690795898438,
      "activations/layer15_attention_weight_max": 32.52621841430664,
      "activations/layer15_attention_weight_min": -32.24897766113281,
      "activations/layer16_attention_weight_max": 33.073360443115234,
      "activations/layer16_attention_weight_min": -32.21589660644531,
      "activations/layer17_attention_weight_max": 51.09743118286133,
      "activations/layer17_attention_weight_min": -44.080623626708984,
      "activations/layer18_attention_weight_max": 45.48231506347656,
      "activations/layer18_attention_weight_min": -38.85900115966797,
      "activations/layer19_attention_weight_max": 23.174379348754883,
      "activations/layer19_attention_weight_min": -24.22338104248047,
      "activations/layer1_attention_weight_max": 15.50416088104248,
      "activations/layer1_attention_weight_min": -13.988828659057617,
      "activations/layer20_attention_weight_max": 21.116443634033203,
      "activations/layer20_attention_weight_min": -24.07757568359375,
      "activations/layer21_attention_weight_max": 36.39601135253906,
      "activations/layer21_attention_weight_min": -27.0401611328125,
      "activations/layer22_attention_weight_max": 30.26595687866211,
      "activations/layer22_attention_weight_min": -27.694501876831055,
      "activations/layer23_attention_weight_max": 37.537879943847656,
      "activations/layer23_attention_weight_min": -24.93861961364746,
      "activations/layer2_attention_weight_max": 30.34490203857422,
      "activations/layer2_attention_weight_min": -29.464345932006836,
      "activations/layer3_attention_weight_max": 86.8263168334961,
      "activations/layer3_attention_weight_min": -85.4324951171875,
      "activations/layer4_attention_weight_max": 87.57540130615234,
      "activations/layer4_attention_weight_min": -86.03097534179688,
      "activations/layer5_attention_weight_max": 63.852169036865234,
      "activations/layer5_attention_weight_min": -72.21502685546875,
      "activations/layer6_attention_weight_max": 49.45783615112305,
      "activations/layer6_attention_weight_min": -51.7244987487793,
      "activations/layer7_attention_weight_max": 62.55352020263672,
      "activations/layer7_attention_weight_min": -60.25749969482422,
      "activations/layer8_attention_weight_max": 45.159217834472656,
      "activations/layer8_attention_weight_min": -45.57398223876953,
      "activations/layer9_attention_weight_max": 43.76533126831055,
      "activations/layer9_attention_weight_min": -44.37635040283203,
      "epoch": 15.82,
      "learning_rate": 4.84560606060606e-05,
      "loss": 2.7535,
      "step": 272300
    },
    {
      "activations/layer0_attention_weight_max": 15.24142837524414,
      "activations/layer0_attention_weight_min": -14.449020385742188,
      "activations/layer10_attention_weight_max": 34.57027053833008,
      "activations/layer10_attention_weight_min": -34.839298248291016,
      "activations/layer11_attention_weight_max": 31.437747955322266,
      "activations/layer11_attention_weight_min": -33.055274963378906,
      "activations/layer12_attention_weight_max": 17.228708267211914,
      "activations/layer12_attention_weight_min": -25.966215133666992,
      "activations/layer13_attention_weight_max": 32.44489288330078,
      "activations/layer13_attention_weight_min": -27.194643020629883,
      "activations/layer14_attention_weight_max": 32.72785949707031,
      "activations/layer14_attention_weight_min": -29.9584903717041,
      "activations/layer15_attention_weight_max": 30.635339736938477,
      "activations/layer15_attention_weight_min": -29.624069213867188,
      "activations/layer16_attention_weight_max": 31.517377853393555,
      "activations/layer16_attention_weight_min": -30.880428314208984,
      "activations/layer17_attention_weight_max": 54.321434020996094,
      "activations/layer17_attention_weight_min": -48.172447204589844,
      "activations/layer18_attention_weight_max": 45.64387512207031,
      "activations/layer18_attention_weight_min": -43.30830001831055,
      "activations/layer19_attention_weight_max": 22.446823120117188,
      "activations/layer19_attention_weight_min": -23.74825668334961,
      "activations/layer1_attention_weight_max": 16.11754608154297,
      "activations/layer1_attention_weight_min": -12.514238357543945,
      "activations/layer20_attention_weight_max": 21.249300003051758,
      "activations/layer20_attention_weight_min": -24.270036697387695,
      "activations/layer21_attention_weight_max": 32.82544708251953,
      "activations/layer21_attention_weight_min": -24.042509078979492,
      "activations/layer22_attention_weight_max": 26.69647979736328,
      "activations/layer22_attention_weight_min": -28.591169357299805,
      "activations/layer23_attention_weight_max": 34.67365264892578,
      "activations/layer23_attention_weight_min": -25.732755661010742,
      "activations/layer2_attention_weight_max": 29.22960662841797,
      "activations/layer2_attention_weight_min": -28.95306396484375,
      "activations/layer3_attention_weight_max": 89.48167419433594,
      "activations/layer3_attention_weight_min": -90.88162231445312,
      "activations/layer4_attention_weight_max": 87.7794418334961,
      "activations/layer4_attention_weight_min": -87.73320007324219,
      "activations/layer5_attention_weight_max": 65.7464599609375,
      "activations/layer5_attention_weight_min": -73.62635803222656,
      "activations/layer6_attention_weight_max": 50.80891418457031,
      "activations/layer6_attention_weight_min": -53.3195915222168,
      "activations/layer7_attention_weight_max": 63.0379524230957,
      "activations/layer7_attention_weight_min": -59.28771209716797,
      "activations/layer8_attention_weight_max": 44.76533508300781,
      "activations/layer8_attention_weight_min": -48.5496711730957,
      "activations/layer9_attention_weight_max": 45.11893844604492,
      "activations/layer9_attention_weight_min": -45.465003967285156,
      "epoch": 15.83,
      "learning_rate": 4.843712121212121e-05,
      "loss": 2.7547,
      "step": 272350
    },
    {
      "activations/layer0_attention_weight_max": 15.336217880249023,
      "activations/layer0_attention_weight_min": -14.142577171325684,
      "activations/layer10_attention_weight_max": 36.58246612548828,
      "activations/layer10_attention_weight_min": -35.54758834838867,
      "activations/layer11_attention_weight_max": 33.136695861816406,
      "activations/layer11_attention_weight_min": -34.63031005859375,
      "activations/layer12_attention_weight_max": 18.927549362182617,
      "activations/layer12_attention_weight_min": -29.426767349243164,
      "activations/layer13_attention_weight_max": 35.787353515625,
      "activations/layer13_attention_weight_min": -27.306303024291992,
      "activations/layer14_attention_weight_max": 42.49787521362305,
      "activations/layer14_attention_weight_min": -31.40049171447754,
      "activations/layer15_attention_weight_max": 36.98869323730469,
      "activations/layer15_attention_weight_min": -30.913137435913086,
      "activations/layer16_attention_weight_max": 35.64225387573242,
      "activations/layer16_attention_weight_min": -34.766841888427734,
      "activations/layer17_attention_weight_max": 58.25278091430664,
      "activations/layer17_attention_weight_min": -49.26435089111328,
      "activations/layer18_attention_weight_max": 50.93910217285156,
      "activations/layer18_attention_weight_min": -40.391456604003906,
      "activations/layer19_attention_weight_max": 26.54398536682129,
      "activations/layer19_attention_weight_min": -23.74123764038086,
      "activations/layer1_attention_weight_max": 17.08020782470703,
      "activations/layer1_attention_weight_min": -13.620573043823242,
      "activations/layer20_attention_weight_max": 24.822362899780273,
      "activations/layer20_attention_weight_min": -23.086530685424805,
      "activations/layer21_attention_weight_max": 40.4937858581543,
      "activations/layer21_attention_weight_min": -22.979778289794922,
      "activations/layer22_attention_weight_max": 30.698076248168945,
      "activations/layer22_attention_weight_min": -24.81497573852539,
      "activations/layer23_attention_weight_max": 40.73908233642578,
      "activations/layer23_attention_weight_min": -27.943464279174805,
      "activations/layer2_attention_weight_max": 32.59641647338867,
      "activations/layer2_attention_weight_min": -31.18408203125,
      "activations/layer3_attention_weight_max": 95.15239715576172,
      "activations/layer3_attention_weight_min": -96.52395629882812,
      "activations/layer4_attention_weight_max": 94.0577621459961,
      "activations/layer4_attention_weight_min": -91.3926773071289,
      "activations/layer5_attention_weight_max": 63.84791564941406,
      "activations/layer5_attention_weight_min": -77.55394744873047,
      "activations/layer6_attention_weight_max": 55.53422927856445,
      "activations/layer6_attention_weight_min": -53.55345153808594,
      "activations/layer7_attention_weight_max": 60.15330505371094,
      "activations/layer7_attention_weight_min": -59.75656509399414,
      "activations/layer8_attention_weight_max": 48.05668640136719,
      "activations/layer8_attention_weight_min": -52.15272521972656,
      "activations/layer9_attention_weight_max": 44.15760803222656,
      "activations/layer9_attention_weight_min": -45.13211441040039,
      "epoch": 15.83,
      "learning_rate": 4.841818181818182e-05,
      "loss": 2.7461,
      "step": 272400
    },
    {
      "activations/layer0_attention_weight_max": 16.162595748901367,
      "activations/layer0_attention_weight_min": -14.122875213623047,
      "activations/layer10_attention_weight_max": 37.09845733642578,
      "activations/layer10_attention_weight_min": -35.30171585083008,
      "activations/layer11_attention_weight_max": 33.35528564453125,
      "activations/layer11_attention_weight_min": -34.16753387451172,
      "activations/layer12_attention_weight_max": 18.147432327270508,
      "activations/layer12_attention_weight_min": -29.24671173095703,
      "activations/layer13_attention_weight_max": 30.94428062438965,
      "activations/layer13_attention_weight_min": -25.978361129760742,
      "activations/layer14_attention_weight_max": 33.29903793334961,
      "activations/layer14_attention_weight_min": -29.65140724182129,
      "activations/layer15_attention_weight_max": 31.394338607788086,
      "activations/layer15_attention_weight_min": -28.559465408325195,
      "activations/layer16_attention_weight_max": 32.09507751464844,
      "activations/layer16_attention_weight_min": -30.210527420043945,
      "activations/layer17_attention_weight_max": 49.806278228759766,
      "activations/layer17_attention_weight_min": -43.072998046875,
      "activations/layer18_attention_weight_max": 42.180850982666016,
      "activations/layer18_attention_weight_min": -37.10425567626953,
      "activations/layer19_attention_weight_max": 21.8054256439209,
      "activations/layer19_attention_weight_min": -22.461740493774414,
      "activations/layer1_attention_weight_max": 15.908136367797852,
      "activations/layer1_attention_weight_min": -14.33457088470459,
      "activations/layer20_attention_weight_max": 19.966747283935547,
      "activations/layer20_attention_weight_min": -21.666276931762695,
      "activations/layer21_attention_weight_max": 36.239501953125,
      "activations/layer21_attention_weight_min": -22.86519432067871,
      "activations/layer22_attention_weight_max": 29.084375381469727,
      "activations/layer22_attention_weight_min": -25.266342163085938,
      "activations/layer23_attention_weight_max": 32.841163635253906,
      "activations/layer23_attention_weight_min": -25.779762268066406,
      "activations/layer2_attention_weight_max": 30.176063537597656,
      "activations/layer2_attention_weight_min": -29.460620880126953,
      "activations/layer3_attention_weight_max": 83.23200225830078,
      "activations/layer3_attention_weight_min": -85.1314697265625,
      "activations/layer4_attention_weight_max": 86.05138397216797,
      "activations/layer4_attention_weight_min": -86.94309997558594,
      "activations/layer5_attention_weight_max": 61.26380920410156,
      "activations/layer5_attention_weight_min": -71.25836181640625,
      "activations/layer6_attention_weight_max": 49.630741119384766,
      "activations/layer6_attention_weight_min": -50.45262145996094,
      "activations/layer7_attention_weight_max": 58.35603332519531,
      "activations/layer7_attention_weight_min": -56.69816207885742,
      "activations/layer8_attention_weight_max": 46.4163703918457,
      "activations/layer8_attention_weight_min": -51.116580963134766,
      "activations/layer9_attention_weight_max": 43.945518493652344,
      "activations/layer9_attention_weight_min": -44.43598556518555,
      "epoch": 15.83,
      "learning_rate": 4.839924242424242e-05,
      "loss": 2.7568,
      "step": 272450
    },
    {
      "activations/layer0_attention_weight_max": 16.290695190429688,
      "activations/layer0_attention_weight_min": -14.033998489379883,
      "activations/layer10_attention_weight_max": 42.12895202636719,
      "activations/layer10_attention_weight_min": -42.01243209838867,
      "activations/layer11_attention_weight_max": 42.49069595336914,
      "activations/layer11_attention_weight_min": -41.4130973815918,
      "activations/layer12_attention_weight_max": 19.541046142578125,
      "activations/layer12_attention_weight_min": -26.840044021606445,
      "activations/layer13_attention_weight_max": 32.97409439086914,
      "activations/layer13_attention_weight_min": -30.853734970092773,
      "activations/layer14_attention_weight_max": 37.074134826660156,
      "activations/layer14_attention_weight_min": -31.505645751953125,
      "activations/layer15_attention_weight_max": 32.165679931640625,
      "activations/layer15_attention_weight_min": -30.787372589111328,
      "activations/layer16_attention_weight_max": 33.94256591796875,
      "activations/layer16_attention_weight_min": -33.659912109375,
      "activations/layer17_attention_weight_max": 51.68525314331055,
      "activations/layer17_attention_weight_min": -50.320281982421875,
      "activations/layer18_attention_weight_max": 46.777984619140625,
      "activations/layer18_attention_weight_min": -44.326595306396484,
      "activations/layer19_attention_weight_max": 23.491085052490234,
      "activations/layer19_attention_weight_min": -26.676603317260742,
      "activations/layer1_attention_weight_max": 15.679325103759766,
      "activations/layer1_attention_weight_min": -14.032252311706543,
      "activations/layer20_attention_weight_max": 21.078454971313477,
      "activations/layer20_attention_weight_min": -23.090713500976562,
      "activations/layer21_attention_weight_max": 36.35662841796875,
      "activations/layer21_attention_weight_min": -24.39678382873535,
      "activations/layer22_attention_weight_max": 27.474273681640625,
      "activations/layer22_attention_weight_min": -27.915891647338867,
      "activations/layer23_attention_weight_max": 34.262054443359375,
      "activations/layer23_attention_weight_min": -26.59339714050293,
      "activations/layer2_attention_weight_max": 32.099151611328125,
      "activations/layer2_attention_weight_min": -32.316246032714844,
      "activations/layer3_attention_weight_max": 89.4320297241211,
      "activations/layer3_attention_weight_min": -96.89397430419922,
      "activations/layer4_attention_weight_max": 90.02556610107422,
      "activations/layer4_attention_weight_min": -91.11614227294922,
      "activations/layer5_attention_weight_max": 62.922645568847656,
      "activations/layer5_attention_weight_min": -75.1545639038086,
      "activations/layer6_attention_weight_max": 53.16847229003906,
      "activations/layer6_attention_weight_min": -56.75318908691406,
      "activations/layer7_attention_weight_max": 67.4183120727539,
      "activations/layer7_attention_weight_min": -67.838623046875,
      "activations/layer8_attention_weight_max": 52.94588088989258,
      "activations/layer8_attention_weight_min": -54.704071044921875,
      "activations/layer9_attention_weight_max": 60.083282470703125,
      "activations/layer9_attention_weight_min": -55.55904006958008,
      "epoch": 15.83,
      "learning_rate": 4.838030303030302e-05,
      "loss": 2.7383,
      "step": 272500
    },
    {
      "activations/layer0_attention_weight_max": 16.02507781982422,
      "activations/layer0_attention_weight_min": -14.114792823791504,
      "activations/layer10_attention_weight_max": 41.53995895385742,
      "activations/layer10_attention_weight_min": -40.62754821777344,
      "activations/layer11_attention_weight_max": 36.14521026611328,
      "activations/layer11_attention_weight_min": -36.89189910888672,
      "activations/layer12_attention_weight_max": 19.364728927612305,
      "activations/layer12_attention_weight_min": -26.06393051147461,
      "activations/layer13_attention_weight_max": 35.33672332763672,
      "activations/layer13_attention_weight_min": -31.39206314086914,
      "activations/layer14_attention_weight_max": 38.27548599243164,
      "activations/layer14_attention_weight_min": -31.810340881347656,
      "activations/layer15_attention_weight_max": 32.83943176269531,
      "activations/layer15_attention_weight_min": -31.15812873840332,
      "activations/layer16_attention_weight_max": 35.23579025268555,
      "activations/layer16_attention_weight_min": -33.21034240722656,
      "activations/layer17_attention_weight_max": 51.28182601928711,
      "activations/layer17_attention_weight_min": -44.51962661743164,
      "activations/layer18_attention_weight_max": 46.97372817993164,
      "activations/layer18_attention_weight_min": -39.68859100341797,
      "activations/layer19_attention_weight_max": 23.811777114868164,
      "activations/layer19_attention_weight_min": -22.378915786743164,
      "activations/layer1_attention_weight_max": 15.909966468811035,
      "activations/layer1_attention_weight_min": -14.591123580932617,
      "activations/layer20_attention_weight_max": 22.532611846923828,
      "activations/layer20_attention_weight_min": -23.678861618041992,
      "activations/layer21_attention_weight_max": 35.99309158325195,
      "activations/layer21_attention_weight_min": -24.51303482055664,
      "activations/layer22_attention_weight_max": 31.522933959960938,
      "activations/layer22_attention_weight_min": -28.16951560974121,
      "activations/layer23_attention_weight_max": 38.10958480834961,
      "activations/layer23_attention_weight_min": -28.48225975036621,
      "activations/layer2_attention_weight_max": 32.48722457885742,
      "activations/layer2_attention_weight_min": -31.01144790649414,
      "activations/layer3_attention_weight_max": 93.0215835571289,
      "activations/layer3_attention_weight_min": -94.24427032470703,
      "activations/layer4_attention_weight_max": 96.37727355957031,
      "activations/layer4_attention_weight_min": -93.16311645507812,
      "activations/layer5_attention_weight_max": 68.60530090332031,
      "activations/layer5_attention_weight_min": -76.65863037109375,
      "activations/layer6_attention_weight_max": 56.85417938232422,
      "activations/layer6_attention_weight_min": -58.153709411621094,
      "activations/layer7_attention_weight_max": 67.68157958984375,
      "activations/layer7_attention_weight_min": -67.32829284667969,
      "activations/layer8_attention_weight_max": 53.56332778930664,
      "activations/layer8_attention_weight_min": -56.87803268432617,
      "activations/layer9_attention_weight_max": 52.759437561035156,
      "activations/layer9_attention_weight_min": -50.172115325927734,
      "epoch": 15.84,
      "learning_rate": 4.8361363636363635e-05,
      "loss": 2.7462,
      "step": 272550
    },
    {
      "activations/layer0_attention_weight_max": 15.9088773727417,
      "activations/layer0_attention_weight_min": -13.952007293701172,
      "activations/layer10_attention_weight_max": 35.62550735473633,
      "activations/layer10_attention_weight_min": -35.649208068847656,
      "activations/layer11_attention_weight_max": 31.649547576904297,
      "activations/layer11_attention_weight_min": -33.941646575927734,
      "activations/layer12_attention_weight_max": 21.498266220092773,
      "activations/layer12_attention_weight_min": -23.644805908203125,
      "activations/layer13_attention_weight_max": 37.805843353271484,
      "activations/layer13_attention_weight_min": -27.25290298461914,
      "activations/layer14_attention_weight_max": 36.09663772583008,
      "activations/layer14_attention_weight_min": -30.093482971191406,
      "activations/layer15_attention_weight_max": 33.069828033447266,
      "activations/layer15_attention_weight_min": -28.50825309753418,
      "activations/layer16_attention_weight_max": 32.76386642456055,
      "activations/layer16_attention_weight_min": -32.10065841674805,
      "activations/layer17_attention_weight_max": 54.6461181640625,
      "activations/layer17_attention_weight_min": -45.11396408081055,
      "activations/layer18_attention_weight_max": 46.79119110107422,
      "activations/layer18_attention_weight_min": -39.067771911621094,
      "activations/layer19_attention_weight_max": 24.162275314331055,
      "activations/layer19_attention_weight_min": -21.628873825073242,
      "activations/layer1_attention_weight_max": 14.738750457763672,
      "activations/layer1_attention_weight_min": -15.143482208251953,
      "activations/layer20_attention_weight_max": 24.71569061279297,
      "activations/layer20_attention_weight_min": -22.036027908325195,
      "activations/layer21_attention_weight_max": 39.47681427001953,
      "activations/layer21_attention_weight_min": -25.07567024230957,
      "activations/layer22_attention_weight_max": 30.627906799316406,
      "activations/layer22_attention_weight_min": -26.49040412902832,
      "activations/layer23_attention_weight_max": 41.406795501708984,
      "activations/layer23_attention_weight_min": -23.304405212402344,
      "activations/layer2_attention_weight_max": 29.83033561706543,
      "activations/layer2_attention_weight_min": -29.207843780517578,
      "activations/layer3_attention_weight_max": 90.81808471679688,
      "activations/layer3_attention_weight_min": -91.17485046386719,
      "activations/layer4_attention_weight_max": 91.23126983642578,
      "activations/layer4_attention_weight_min": -88.90015411376953,
      "activations/layer5_attention_weight_max": 64.856689453125,
      "activations/layer5_attention_weight_min": -76.61996459960938,
      "activations/layer6_attention_weight_max": 49.354713439941406,
      "activations/layer6_attention_weight_min": -50.70085906982422,
      "activations/layer7_attention_weight_max": 64.65171813964844,
      "activations/layer7_attention_weight_min": -57.970706939697266,
      "activations/layer8_attention_weight_max": 44.545127868652344,
      "activations/layer8_attention_weight_min": -49.45589828491211,
      "activations/layer9_attention_weight_max": 45.49166488647461,
      "activations/layer9_attention_weight_min": -47.131752014160156,
      "epoch": 15.84,
      "learning_rate": 4.834242424242424e-05,
      "loss": 2.7648,
      "step": 272600
    },
    {
      "activations/layer0_attention_weight_max": 15.54978084564209,
      "activations/layer0_attention_weight_min": -13.939408302307129,
      "activations/layer10_attention_weight_max": 34.435585021972656,
      "activations/layer10_attention_weight_min": -35.367733001708984,
      "activations/layer11_attention_weight_max": 32.092987060546875,
      "activations/layer11_attention_weight_min": -33.516963958740234,
      "activations/layer12_attention_weight_max": 18.0948543548584,
      "activations/layer12_attention_weight_min": -24.23170280456543,
      "activations/layer13_attention_weight_max": 39.79457092285156,
      "activations/layer13_attention_weight_min": -29.28215217590332,
      "activations/layer14_attention_weight_max": 38.144378662109375,
      "activations/layer14_attention_weight_min": -30.880416870117188,
      "activations/layer15_attention_weight_max": 35.05224609375,
      "activations/layer15_attention_weight_min": -31.531944274902344,
      "activations/layer16_attention_weight_max": 32.645355224609375,
      "activations/layer16_attention_weight_min": -32.79651641845703,
      "activations/layer17_attention_weight_max": 52.41232681274414,
      "activations/layer17_attention_weight_min": -48.404056549072266,
      "activations/layer18_attention_weight_max": 45.42012023925781,
      "activations/layer18_attention_weight_min": -44.514434814453125,
      "activations/layer19_attention_weight_max": 21.97467803955078,
      "activations/layer19_attention_weight_min": -24.346256256103516,
      "activations/layer1_attention_weight_max": 16.3845157623291,
      "activations/layer1_attention_weight_min": -14.053000450134277,
      "activations/layer20_attention_weight_max": 21.770244598388672,
      "activations/layer20_attention_weight_min": -22.829538345336914,
      "activations/layer21_attention_weight_max": 37.42454528808594,
      "activations/layer21_attention_weight_min": -24.27800178527832,
      "activations/layer22_attention_weight_max": 28.663238525390625,
      "activations/layer22_attention_weight_min": -25.6779842376709,
      "activations/layer23_attention_weight_max": 35.47137451171875,
      "activations/layer23_attention_weight_min": -25.964038848876953,
      "activations/layer2_attention_weight_max": 30.712600708007812,
      "activations/layer2_attention_weight_min": -30.349872589111328,
      "activations/layer3_attention_weight_max": 82.56712341308594,
      "activations/layer3_attention_weight_min": -83.4146957397461,
      "activations/layer4_attention_weight_max": 84.50455474853516,
      "activations/layer4_attention_weight_min": -84.61450958251953,
      "activations/layer5_attention_weight_max": 61.46112060546875,
      "activations/layer5_attention_weight_min": -72.98368835449219,
      "activations/layer6_attention_weight_max": 47.09210968017578,
      "activations/layer6_attention_weight_min": -48.89018249511719,
      "activations/layer7_attention_weight_max": 63.948509216308594,
      "activations/layer7_attention_weight_min": -60.27115249633789,
      "activations/layer8_attention_weight_max": 44.975189208984375,
      "activations/layer8_attention_weight_min": -48.72669219970703,
      "activations/layer9_attention_weight_max": 47.94895935058594,
      "activations/layer9_attention_weight_min": -44.85122299194336,
      "epoch": 15.84,
      "learning_rate": 4.8323484848484845e-05,
      "loss": 2.7506,
      "step": 272650
    },
    {
      "activations/layer0_attention_weight_max": 16.065364837646484,
      "activations/layer0_attention_weight_min": -13.856542587280273,
      "activations/layer10_attention_weight_max": 38.7279052734375,
      "activations/layer10_attention_weight_min": -38.827735900878906,
      "activations/layer11_attention_weight_max": 34.79974365234375,
      "activations/layer11_attention_weight_min": -33.833194732666016,
      "activations/layer12_attention_weight_max": 18.346166610717773,
      "activations/layer12_attention_weight_min": -26.843408584594727,
      "activations/layer13_attention_weight_max": 36.59534454345703,
      "activations/layer13_attention_weight_min": -27.728809356689453,
      "activations/layer14_attention_weight_max": 35.76627731323242,
      "activations/layer14_attention_weight_min": -30.71086311340332,
      "activations/layer15_attention_weight_max": 30.83488655090332,
      "activations/layer15_attention_weight_min": -29.928564071655273,
      "activations/layer16_attention_weight_max": 33.03513717651367,
      "activations/layer16_attention_weight_min": -30.757888793945312,
      "activations/layer17_attention_weight_max": 54.10696029663086,
      "activations/layer17_attention_weight_min": -46.00649642944336,
      "activations/layer18_attention_weight_max": 47.571712493896484,
      "activations/layer18_attention_weight_min": -40.01972579956055,
      "activations/layer19_attention_weight_max": 23.28434944152832,
      "activations/layer19_attention_weight_min": -24.218769073486328,
      "activations/layer1_attention_weight_max": 16.86991310119629,
      "activations/layer1_attention_weight_min": -15.532255172729492,
      "activations/layer20_attention_weight_max": 23.355066299438477,
      "activations/layer20_attention_weight_min": -22.485122680664062,
      "activations/layer21_attention_weight_max": 34.894962310791016,
      "activations/layer21_attention_weight_min": -22.339155197143555,
      "activations/layer22_attention_weight_max": 27.369157791137695,
      "activations/layer22_attention_weight_min": -24.73021125793457,
      "activations/layer23_attention_weight_max": 37.533042907714844,
      "activations/layer23_attention_weight_min": -24.060338973999023,
      "activations/layer2_attention_weight_max": 33.78832244873047,
      "activations/layer2_attention_weight_min": -32.466392517089844,
      "activations/layer3_attention_weight_max": 94.92585754394531,
      "activations/layer3_attention_weight_min": -93.98863983154297,
      "activations/layer4_attention_weight_max": 94.54373168945312,
      "activations/layer4_attention_weight_min": -92.02162170410156,
      "activations/layer5_attention_weight_max": 67.5838851928711,
      "activations/layer5_attention_weight_min": -79.21173095703125,
      "activations/layer6_attention_weight_max": 53.27334976196289,
      "activations/layer6_attention_weight_min": -52.2583122253418,
      "activations/layer7_attention_weight_max": 66.55745697021484,
      "activations/layer7_attention_weight_min": -66.44888305664062,
      "activations/layer8_attention_weight_max": 50.16609573364258,
      "activations/layer8_attention_weight_min": -51.15629959106445,
      "activations/layer9_attention_weight_max": 55.03416061401367,
      "activations/layer9_attention_weight_min": -46.61598587036133,
      "epoch": 15.85,
      "learning_rate": 4.830454545454545e-05,
      "loss": 2.755,
      "step": 272700
    },
    {
      "activations/layer0_attention_weight_max": 16.164283752441406,
      "activations/layer0_attention_weight_min": -13.84633731842041,
      "activations/layer10_attention_weight_max": 36.016639709472656,
      "activations/layer10_attention_weight_min": -36.7894401550293,
      "activations/layer11_attention_weight_max": 33.23371505737305,
      "activations/layer11_attention_weight_min": -32.73120880126953,
      "activations/layer12_attention_weight_max": 19.953351974487305,
      "activations/layer12_attention_weight_min": -27.446348190307617,
      "activations/layer13_attention_weight_max": 34.53075408935547,
      "activations/layer13_attention_weight_min": -27.34882164001465,
      "activations/layer14_attention_weight_max": 34.83961868286133,
      "activations/layer14_attention_weight_min": -30.799530029296875,
      "activations/layer15_attention_weight_max": 32.343788146972656,
      "activations/layer15_attention_weight_min": -29.27398109436035,
      "activations/layer16_attention_weight_max": 32.225257873535156,
      "activations/layer16_attention_weight_min": -30.561853408813477,
      "activations/layer17_attention_weight_max": 50.45582962036133,
      "activations/layer17_attention_weight_min": -43.97127151489258,
      "activations/layer18_attention_weight_max": 44.07658767700195,
      "activations/layer18_attention_weight_min": -40.172279357910156,
      "activations/layer19_attention_weight_max": 22.28737449645996,
      "activations/layer19_attention_weight_min": -21.519086837768555,
      "activations/layer1_attention_weight_max": 15.949751853942871,
      "activations/layer1_attention_weight_min": -15.032354354858398,
      "activations/layer20_attention_weight_max": 22.99106788635254,
      "activations/layer20_attention_weight_min": -20.85342788696289,
      "activations/layer21_attention_weight_max": 33.75102233886719,
      "activations/layer21_attention_weight_min": -24.28404426574707,
      "activations/layer22_attention_weight_max": 27.857465744018555,
      "activations/layer22_attention_weight_min": -27.846696853637695,
      "activations/layer23_attention_weight_max": 34.78158187866211,
      "activations/layer23_attention_weight_min": -23.945798873901367,
      "activations/layer2_attention_weight_max": 32.366485595703125,
      "activations/layer2_attention_weight_min": -33.582489013671875,
      "activations/layer3_attention_weight_max": 93.84293365478516,
      "activations/layer3_attention_weight_min": -97.4496078491211,
      "activations/layer4_attention_weight_max": 92.07463073730469,
      "activations/layer4_attention_weight_min": -94.9030990600586,
      "activations/layer5_attention_weight_max": 65.76020812988281,
      "activations/layer5_attention_weight_min": -80.8649673461914,
      "activations/layer6_attention_weight_max": 51.93317794799805,
      "activations/layer6_attention_weight_min": -52.53345489501953,
      "activations/layer7_attention_weight_max": 59.74596405029297,
      "activations/layer7_attention_weight_min": -60.9190673828125,
      "activations/layer8_attention_weight_max": 47.07492446899414,
      "activations/layer8_attention_weight_min": -50.27287673950195,
      "activations/layer9_attention_weight_max": 44.025390625,
      "activations/layer9_attention_weight_min": -46.678062438964844,
      "epoch": 15.85,
      "learning_rate": 4.8285606060606055e-05,
      "loss": 2.7416,
      "step": 272750
    },
    {
      "activations/layer0_attention_weight_max": 16.23183822631836,
      "activations/layer0_attention_weight_min": -13.549920082092285,
      "activations/layer10_attention_weight_max": 48.743255615234375,
      "activations/layer10_attention_weight_min": -48.307029724121094,
      "activations/layer11_attention_weight_max": 48.03131866455078,
      "activations/layer11_attention_weight_min": -44.992794036865234,
      "activations/layer12_attention_weight_max": 19.36347007751465,
      "activations/layer12_attention_weight_min": -25.618770599365234,
      "activations/layer13_attention_weight_max": 63.66220474243164,
      "activations/layer13_attention_weight_min": -35.1596565246582,
      "activations/layer14_attention_weight_max": 58.87239456176758,
      "activations/layer14_attention_weight_min": -39.07484436035156,
      "activations/layer15_attention_weight_max": 60.85537338256836,
      "activations/layer15_attention_weight_min": -42.18528747558594,
      "activations/layer16_attention_weight_max": 43.43983840942383,
      "activations/layer16_attention_weight_min": -42.097267150878906,
      "activations/layer17_attention_weight_max": 63.58687973022461,
      "activations/layer17_attention_weight_min": -60.95680236816406,
      "activations/layer18_attention_weight_max": 59.17102813720703,
      "activations/layer18_attention_weight_min": -57.59010696411133,
      "activations/layer19_attention_weight_max": 30.165924072265625,
      "activations/layer19_attention_weight_min": -33.59115219116211,
      "activations/layer1_attention_weight_max": 16.54690170288086,
      "activations/layer1_attention_weight_min": -14.477356910705566,
      "activations/layer20_attention_weight_max": 34.268795013427734,
      "activations/layer20_attention_weight_min": -28.682411193847656,
      "activations/layer21_attention_weight_max": 60.018951416015625,
      "activations/layer21_attention_weight_min": -38.66376495361328,
      "activations/layer22_attention_weight_max": 39.86552047729492,
      "activations/layer22_attention_weight_min": -31.978975296020508,
      "activations/layer23_attention_weight_max": 43.091064453125,
      "activations/layer23_attention_weight_min": -29.685216903686523,
      "activations/layer2_attention_weight_max": 31.66968536376953,
      "activations/layer2_attention_weight_min": -31.161727905273438,
      "activations/layer3_attention_weight_max": 93.77196502685547,
      "activations/layer3_attention_weight_min": -94.98900604248047,
      "activations/layer4_attention_weight_max": 97.66783905029297,
      "activations/layer4_attention_weight_min": -95.43734741210938,
      "activations/layer5_attention_weight_max": 69.95394897460938,
      "activations/layer5_attention_weight_min": -73.07418823242188,
      "activations/layer6_attention_weight_max": 56.867591857910156,
      "activations/layer6_attention_weight_min": -57.70020294189453,
      "activations/layer7_attention_weight_max": 73.9136734008789,
      "activations/layer7_attention_weight_min": -66.29603576660156,
      "activations/layer8_attention_weight_max": 57.54303741455078,
      "activations/layer8_attention_weight_min": -54.31462478637695,
      "activations/layer9_attention_weight_max": 60.39677810668945,
      "activations/layer9_attention_weight_min": -51.99628829956055,
      "epoch": 15.85,
      "learning_rate": 4.826666666666666e-05,
      "loss": 2.7451,
      "step": 272800
    },
    {
      "activations/layer0_attention_weight_max": 16.57661247253418,
      "activations/layer0_attention_weight_min": -13.667206764221191,
      "activations/layer10_attention_weight_max": 33.84431838989258,
      "activations/layer10_attention_weight_min": -34.46950912475586,
      "activations/layer11_attention_weight_max": 33.901126861572266,
      "activations/layer11_attention_weight_min": -32.4990119934082,
      "activations/layer12_attention_weight_max": 19.35597801208496,
      "activations/layer12_attention_weight_min": -24.352493286132812,
      "activations/layer13_attention_weight_max": 35.431583404541016,
      "activations/layer13_attention_weight_min": -27.80939292907715,
      "activations/layer14_attention_weight_max": 35.37947082519531,
      "activations/layer14_attention_weight_min": -30.665462493896484,
      "activations/layer15_attention_weight_max": 32.597137451171875,
      "activations/layer15_attention_weight_min": -30.47134780883789,
      "activations/layer16_attention_weight_max": 32.36404037475586,
      "activations/layer16_attention_weight_min": -32.59391784667969,
      "activations/layer17_attention_weight_max": 49.16265106201172,
      "activations/layer17_attention_weight_min": -43.21548843383789,
      "activations/layer18_attention_weight_max": 45.32051467895508,
      "activations/layer18_attention_weight_min": -37.860130310058594,
      "activations/layer19_attention_weight_max": 22.098302841186523,
      "activations/layer19_attention_weight_min": -22.238088607788086,
      "activations/layer1_attention_weight_max": 17.874671936035156,
      "activations/layer1_attention_weight_min": -13.10515022277832,
      "activations/layer20_attention_weight_max": 21.171581268310547,
      "activations/layer20_attention_weight_min": -21.100309371948242,
      "activations/layer21_attention_weight_max": 35.705867767333984,
      "activations/layer21_attention_weight_min": -21.767425537109375,
      "activations/layer22_attention_weight_max": 28.696531295776367,
      "activations/layer22_attention_weight_min": -27.878063201904297,
      "activations/layer23_attention_weight_max": 33.540523529052734,
      "activations/layer23_attention_weight_min": -23.203237533569336,
      "activations/layer2_attention_weight_max": 28.034942626953125,
      "activations/layer2_attention_weight_min": -26.62957763671875,
      "activations/layer3_attention_weight_max": 80.31336975097656,
      "activations/layer3_attention_weight_min": -78.15511322021484,
      "activations/layer4_attention_weight_max": 79.1246109008789,
      "activations/layer4_attention_weight_min": -80.77641296386719,
      "activations/layer5_attention_weight_max": 63.355438232421875,
      "activations/layer5_attention_weight_min": -70.52628326416016,
      "activations/layer6_attention_weight_max": 47.79825973510742,
      "activations/layer6_attention_weight_min": -49.42240524291992,
      "activations/layer7_attention_weight_max": 61.679283142089844,
      "activations/layer7_attention_weight_min": -59.084651947021484,
      "activations/layer8_attention_weight_max": 44.56365203857422,
      "activations/layer8_attention_weight_min": -46.59012222290039,
      "activations/layer9_attention_weight_max": 43.67362976074219,
      "activations/layer9_attention_weight_min": -43.20022201538086,
      "epoch": 15.85,
      "learning_rate": 4.8247727272727265e-05,
      "loss": 2.7499,
      "step": 272850
    },
    {
      "activations/layer0_attention_weight_max": 15.84089469909668,
      "activations/layer0_attention_weight_min": -13.96462345123291,
      "activations/layer10_attention_weight_max": 34.5461311340332,
      "activations/layer10_attention_weight_min": -33.811614990234375,
      "activations/layer11_attention_weight_max": 31.434375762939453,
      "activations/layer11_attention_weight_min": -32.901763916015625,
      "activations/layer12_attention_weight_max": 18.310144424438477,
      "activations/layer12_attention_weight_min": -25.377647399902344,
      "activations/layer13_attention_weight_max": 37.239532470703125,
      "activations/layer13_attention_weight_min": -26.41823959350586,
      "activations/layer14_attention_weight_max": 34.68278884887695,
      "activations/layer14_attention_weight_min": -29.22665786743164,
      "activations/layer15_attention_weight_max": 31.960987091064453,
      "activations/layer15_attention_weight_min": -28.666250228881836,
      "activations/layer16_attention_weight_max": 32.503517150878906,
      "activations/layer16_attention_weight_min": -30.432979583740234,
      "activations/layer17_attention_weight_max": 51.15983963012695,
      "activations/layer17_attention_weight_min": -43.66643142700195,
      "activations/layer18_attention_weight_max": 43.02037048339844,
      "activations/layer18_attention_weight_min": -38.39527893066406,
      "activations/layer19_attention_weight_max": 22.66486167907715,
      "activations/layer19_attention_weight_min": -22.229616165161133,
      "activations/layer1_attention_weight_max": 16.5848331451416,
      "activations/layer1_attention_weight_min": -13.695380210876465,
      "activations/layer20_attention_weight_max": 20.6542911529541,
      "activations/layer20_attention_weight_min": -20.310791015625,
      "activations/layer21_attention_weight_max": 33.492523193359375,
      "activations/layer21_attention_weight_min": -20.430221557617188,
      "activations/layer22_attention_weight_max": 26.832763671875,
      "activations/layer22_attention_weight_min": -25.7486515045166,
      "activations/layer23_attention_weight_max": 36.40849304199219,
      "activations/layer23_attention_weight_min": -27.25879669189453,
      "activations/layer2_attention_weight_max": 29.89084243774414,
      "activations/layer2_attention_weight_min": -29.112211227416992,
      "activations/layer3_attention_weight_max": 80.75382995605469,
      "activations/layer3_attention_weight_min": -83.4540786743164,
      "activations/layer4_attention_weight_max": 82.84414672851562,
      "activations/layer4_attention_weight_min": -81.37914276123047,
      "activations/layer5_attention_weight_max": 60.692169189453125,
      "activations/layer5_attention_weight_min": -69.29962921142578,
      "activations/layer6_attention_weight_max": 52.11262893676758,
      "activations/layer6_attention_weight_min": -49.48320007324219,
      "activations/layer7_attention_weight_max": 60.825252532958984,
      "activations/layer7_attention_weight_min": -60.03628158569336,
      "activations/layer8_attention_weight_max": 45.037269592285156,
      "activations/layer8_attention_weight_min": -50.13267135620117,
      "activations/layer9_attention_weight_max": 43.45460891723633,
      "activations/layer9_attention_weight_min": -45.6506462097168,
      "epoch": 15.86,
      "learning_rate": 4.822878787878788e-05,
      "loss": 2.7498,
      "step": 272900
    },
    {
      "activations/layer0_attention_weight_max": 15.858242988586426,
      "activations/layer0_attention_weight_min": -13.70383358001709,
      "activations/layer10_attention_weight_max": 34.86659240722656,
      "activations/layer10_attention_weight_min": -34.5667610168457,
      "activations/layer11_attention_weight_max": 31.888286590576172,
      "activations/layer11_attention_weight_min": -32.740631103515625,
      "activations/layer12_attention_weight_max": 18.401166915893555,
      "activations/layer12_attention_weight_min": -26.53301429748535,
      "activations/layer13_attention_weight_max": 34.548828125,
      "activations/layer13_attention_weight_min": -29.220417022705078,
      "activations/layer14_attention_weight_max": 33.657745361328125,
      "activations/layer14_attention_weight_min": -30.792808532714844,
      "activations/layer15_attention_weight_max": 32.17747497558594,
      "activations/layer15_attention_weight_min": -30.662311553955078,
      "activations/layer16_attention_weight_max": 32.60390853881836,
      "activations/layer16_attention_weight_min": -31.208341598510742,
      "activations/layer17_attention_weight_max": 49.522300720214844,
      "activations/layer17_attention_weight_min": -43.82232666015625,
      "activations/layer18_attention_weight_max": 45.2661247253418,
      "activations/layer18_attention_weight_min": -39.53041076660156,
      "activations/layer19_attention_weight_max": 24.311946868896484,
      "activations/layer19_attention_weight_min": -22.582881927490234,
      "activations/layer1_attention_weight_max": 15.85666561126709,
      "activations/layer1_attention_weight_min": -14.165922164916992,
      "activations/layer20_attention_weight_max": 23.776527404785156,
      "activations/layer20_attention_weight_min": -24.155986785888672,
      "activations/layer21_attention_weight_max": 37.79706954956055,
      "activations/layer21_attention_weight_min": -23.484207153320312,
      "activations/layer22_attention_weight_max": 29.53883171081543,
      "activations/layer22_attention_weight_min": -29.97926902770996,
      "activations/layer23_attention_weight_max": 37.003204345703125,
      "activations/layer23_attention_weight_min": -24.33216094970703,
      "activations/layer2_attention_weight_max": 32.14521789550781,
      "activations/layer2_attention_weight_min": -30.162817001342773,
      "activations/layer3_attention_weight_max": 85.7681655883789,
      "activations/layer3_attention_weight_min": -84.22594451904297,
      "activations/layer4_attention_weight_max": 88.96576690673828,
      "activations/layer4_attention_weight_min": -82.73833465576172,
      "activations/layer5_attention_weight_max": 61.240928649902344,
      "activations/layer5_attention_weight_min": -68.30935668945312,
      "activations/layer6_attention_weight_max": 48.05356216430664,
      "activations/layer6_attention_weight_min": -48.238826751708984,
      "activations/layer7_attention_weight_max": 56.465274810791016,
      "activations/layer7_attention_weight_min": -56.47250747680664,
      "activations/layer8_attention_weight_max": 43.617027282714844,
      "activations/layer8_attention_weight_min": -44.26274871826172,
      "activations/layer9_attention_weight_max": 42.98414611816406,
      "activations/layer9_attention_weight_min": -46.228946685791016,
      "epoch": 15.86,
      "learning_rate": 4.820984848484848e-05,
      "loss": 2.7495,
      "step": 272950
    },
    {
      "activations/layer0_attention_weight_max": 15.756178855895996,
      "activations/layer0_attention_weight_min": -13.92518138885498,
      "activations/layer10_attention_weight_max": 34.56552505493164,
      "activations/layer10_attention_weight_min": -35.24677276611328,
      "activations/layer11_attention_weight_max": 31.32785415649414,
      "activations/layer11_attention_weight_min": -33.043643951416016,
      "activations/layer12_attention_weight_max": 18.368484497070312,
      "activations/layer12_attention_weight_min": -23.04410743713379,
      "activations/layer13_attention_weight_max": 33.9935417175293,
      "activations/layer13_attention_weight_min": -26.903278350830078,
      "activations/layer14_attention_weight_max": 37.226829528808594,
      "activations/layer14_attention_weight_min": -29.937280654907227,
      "activations/layer15_attention_weight_max": 34.83699035644531,
      "activations/layer15_attention_weight_min": -29.658267974853516,
      "activations/layer16_attention_weight_max": 35.333251953125,
      "activations/layer16_attention_weight_min": -32.10710144042969,
      "activations/layer17_attention_weight_max": 52.32509994506836,
      "activations/layer17_attention_weight_min": -44.969444274902344,
      "activations/layer18_attention_weight_max": 47.57890319824219,
      "activations/layer18_attention_weight_min": -39.46883773803711,
      "activations/layer19_attention_weight_max": 22.286523818969727,
      "activations/layer19_attention_weight_min": -23.35333824157715,
      "activations/layer1_attention_weight_max": 16.84055519104004,
      "activations/layer1_attention_weight_min": -15.715051651000977,
      "activations/layer20_attention_weight_max": 24.632854461669922,
      "activations/layer20_attention_weight_min": -20.13756561279297,
      "activations/layer21_attention_weight_max": 36.214351654052734,
      "activations/layer21_attention_weight_min": -21.846851348876953,
      "activations/layer22_attention_weight_max": 30.370311737060547,
      "activations/layer22_attention_weight_min": -23.79231834411621,
      "activations/layer23_attention_weight_max": 38.588932037353516,
      "activations/layer23_attention_weight_min": -24.785375595092773,
      "activations/layer2_attention_weight_max": 31.891067504882812,
      "activations/layer2_attention_weight_min": -30.97079086303711,
      "activations/layer3_attention_weight_max": 87.90277099609375,
      "activations/layer3_attention_weight_min": -90.7068862915039,
      "activations/layer4_attention_weight_max": 86.95260620117188,
      "activations/layer4_attention_weight_min": -84.2761001586914,
      "activations/layer5_attention_weight_max": 61.67615509033203,
      "activations/layer5_attention_weight_min": -70.37198638916016,
      "activations/layer6_attention_weight_max": 47.09674835205078,
      "activations/layer6_attention_weight_min": -48.22203826904297,
      "activations/layer7_attention_weight_max": 56.68544387817383,
      "activations/layer7_attention_weight_min": -58.277732849121094,
      "activations/layer8_attention_weight_max": 45.3344841003418,
      "activations/layer8_attention_weight_min": -50.12276077270508,
      "activations/layer9_attention_weight_max": 43.843570709228516,
      "activations/layer9_attention_weight_min": -44.19572830200195,
      "epoch": 15.86,
      "learning_rate": 4.819090909090908e-05,
      "loss": 2.7437,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4381,
      "eval_samples_per_second": 508.883,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4381,
      "eval_openwebtext_samples_per_second": 508.883,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9062,
      "eval_wikitext_samples_per_second": 239.218,
      "step": 273000
    },
    {
      "epoch": 15.86,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.5324,
      "eval_lambada_samples_per_second": 510.787,
      "step": 273000
    },
    {
      "activations/layer0_attention_weight_max": 15.761488914489746,
      "activations/layer0_attention_weight_min": -13.783855438232422,
      "activations/layer10_attention_weight_max": 34.5189208984375,
      "activations/layer10_attention_weight_min": -34.271488189697266,
      "activations/layer11_attention_weight_max": 31.593603134155273,
      "activations/layer11_attention_weight_min": -33.893951416015625,
      "activations/layer12_attention_weight_max": 18.30772590637207,
      "activations/layer12_attention_weight_min": -25.332721710205078,
      "activations/layer13_attention_weight_max": 31.770471572875977,
      "activations/layer13_attention_weight_min": -27.37354850769043,
      "activations/layer14_attention_weight_max": 34.50214385986328,
      "activations/layer14_attention_weight_min": -33.43173599243164,
      "activations/layer15_attention_weight_max": 32.57699966430664,
      "activations/layer15_attention_weight_min": -30.45923614501953,
      "activations/layer16_attention_weight_max": 33.49951934814453,
      "activations/layer16_attention_weight_min": -32.029293060302734,
      "activations/layer17_attention_weight_max": 52.70983123779297,
      "activations/layer17_attention_weight_min": -44.142696380615234,
      "activations/layer18_attention_weight_max": 46.27122116088867,
      "activations/layer18_attention_weight_min": -37.80080032348633,
      "activations/layer19_attention_weight_max": 22.2086238861084,
      "activations/layer19_attention_weight_min": -21.39434051513672,
      "activations/layer1_attention_weight_max": 16.239988327026367,
      "activations/layer1_attention_weight_min": -14.811871528625488,
      "activations/layer20_attention_weight_max": 22.917936325073242,
      "activations/layer20_attention_weight_min": -19.45047378540039,
      "activations/layer21_attention_weight_max": 32.94618606567383,
      "activations/layer21_attention_weight_min": -20.805810928344727,
      "activations/layer22_attention_weight_max": 29.617605209350586,
      "activations/layer22_attention_weight_min": -23.496702194213867,
      "activations/layer23_attention_weight_max": 35.52770233154297,
      "activations/layer23_attention_weight_min": -22.644805908203125,
      "activations/layer2_attention_weight_max": 31.283023834228516,
      "activations/layer2_attention_weight_min": -29.039413452148438,
      "activations/layer3_attention_weight_max": 83.88044738769531,
      "activations/layer3_attention_weight_min": -81.59320831298828,
      "activations/layer4_attention_weight_max": 84.10899353027344,
      "activations/layer4_attention_weight_min": -84.00043487548828,
      "activations/layer5_attention_weight_max": 66.53336334228516,
      "activations/layer5_attention_weight_min": -77.3541259765625,
      "activations/layer6_attention_weight_max": 51.19040298461914,
      "activations/layer6_attention_weight_min": -51.896671295166016,
      "activations/layer7_attention_weight_max": 60.872886657714844,
      "activations/layer7_attention_weight_min": -60.013240814208984,
      "activations/layer8_attention_weight_max": 44.043548583984375,
      "activations/layer8_attention_weight_min": -47.90726089477539,
      "activations/layer9_attention_weight_max": 44.14894104003906,
      "activations/layer9_attention_weight_min": -46.57624435424805,
      "epoch": 15.87,
      "learning_rate": 4.817196969696969e-05,
      "loss": 2.7485,
      "step": 273050
    },
    {
      "activations/layer0_attention_weight_max": 15.190934181213379,
      "activations/layer0_attention_weight_min": -13.863760948181152,
      "activations/layer10_attention_weight_max": 35.330039978027344,
      "activations/layer10_attention_weight_min": -35.17426300048828,
      "activations/layer11_attention_weight_max": 32.384864807128906,
      "activations/layer11_attention_weight_min": -34.249298095703125,
      "activations/layer12_attention_weight_max": 18.077173233032227,
      "activations/layer12_attention_weight_min": -30.485191345214844,
      "activations/layer13_attention_weight_max": 31.537227630615234,
      "activations/layer13_attention_weight_min": -28.134496688842773,
      "activations/layer14_attention_weight_max": 34.14278030395508,
      "activations/layer14_attention_weight_min": -30.919509887695312,
      "activations/layer15_attention_weight_max": 31.12760353088379,
      "activations/layer15_attention_weight_min": -31.1157283782959,
      "activations/layer16_attention_weight_max": 32.131805419921875,
      "activations/layer16_attention_weight_min": -30.34336280822754,
      "activations/layer17_attention_weight_max": 49.10874557495117,
      "activations/layer17_attention_weight_min": -45.93182373046875,
      "activations/layer18_attention_weight_max": 44.53816223144531,
      "activations/layer18_attention_weight_min": -37.930633544921875,
      "activations/layer19_attention_weight_max": 22.481006622314453,
      "activations/layer19_attention_weight_min": -24.753833770751953,
      "activations/layer1_attention_weight_max": 15.586396217346191,
      "activations/layer1_attention_weight_min": -15.109511375427246,
      "activations/layer20_attention_weight_max": 21.884281158447266,
      "activations/layer20_attention_weight_min": -25.106582641601562,
      "activations/layer21_attention_weight_max": 34.870208740234375,
      "activations/layer21_attention_weight_min": -22.565610885620117,
      "activations/layer22_attention_weight_max": 32.6335563659668,
      "activations/layer22_attention_weight_min": -27.35209846496582,
      "activations/layer23_attention_weight_max": 35.12236785888672,
      "activations/layer23_attention_weight_min": -25.2613582611084,
      "activations/layer2_attention_weight_max": 30.87181854248047,
      "activations/layer2_attention_weight_min": -29.578147888183594,
      "activations/layer3_attention_weight_max": 90.47137451171875,
      "activations/layer3_attention_weight_min": -91.21612548828125,
      "activations/layer4_attention_weight_max": 88.77135467529297,
      "activations/layer4_attention_weight_min": -88.43274688720703,
      "activations/layer5_attention_weight_max": 63.16254425048828,
      "activations/layer5_attention_weight_min": -72.97410583496094,
      "activations/layer6_attention_weight_max": 49.87031555175781,
      "activations/layer6_attention_weight_min": -53.63248062133789,
      "activations/layer7_attention_weight_max": 61.30678939819336,
      "activations/layer7_attention_weight_min": -61.915748596191406,
      "activations/layer8_attention_weight_max": 45.21553421020508,
      "activations/layer8_attention_weight_min": -48.896358489990234,
      "activations/layer9_attention_weight_max": 43.41201400756836,
      "activations/layer9_attention_weight_min": -45.89263916015625,
      "epoch": 15.87,
      "learning_rate": 4.81530303030303e-05,
      "loss": 2.7543,
      "step": 273100
    },
    {
      "activations/layer0_attention_weight_max": 15.919038772583008,
      "activations/layer0_attention_weight_min": -14.599803924560547,
      "activations/layer10_attention_weight_max": 35.02383041381836,
      "activations/layer10_attention_weight_min": -34.065059661865234,
      "activations/layer11_attention_weight_max": 33.73863983154297,
      "activations/layer11_attention_weight_min": -32.41694641113281,
      "activations/layer12_attention_weight_max": 19.678495407104492,
      "activations/layer12_attention_weight_min": -24.254179000854492,
      "activations/layer13_attention_weight_max": 36.33748245239258,
      "activations/layer13_attention_weight_min": -29.627853393554688,
      "activations/layer14_attention_weight_max": 37.4576301574707,
      "activations/layer14_attention_weight_min": -31.885080337524414,
      "activations/layer15_attention_weight_max": 32.95756149291992,
      "activations/layer15_attention_weight_min": -31.144248962402344,
      "activations/layer16_attention_weight_max": 35.50435256958008,
      "activations/layer16_attention_weight_min": -33.48725128173828,
      "activations/layer17_attention_weight_max": 54.4851188659668,
      "activations/layer17_attention_weight_min": -47.607444763183594,
      "activations/layer18_attention_weight_max": 47.55347442626953,
      "activations/layer18_attention_weight_min": -39.95322036743164,
      "activations/layer19_attention_weight_max": 24.01122283935547,
      "activations/layer19_attention_weight_min": -24.70604133605957,
      "activations/layer1_attention_weight_max": 15.437071800231934,
      "activations/layer1_attention_weight_min": -13.836200714111328,
      "activations/layer20_attention_weight_max": 23.02492332458496,
      "activations/layer20_attention_weight_min": -22.444683074951172,
      "activations/layer21_attention_weight_max": 34.350162506103516,
      "activations/layer21_attention_weight_min": -23.30327033996582,
      "activations/layer22_attention_weight_max": 27.820716857910156,
      "activations/layer22_attention_weight_min": -27.837936401367188,
      "activations/layer23_attention_weight_max": 38.705047607421875,
      "activations/layer23_attention_weight_min": -23.471607208251953,
      "activations/layer2_attention_weight_max": 29.992284774780273,
      "activations/layer2_attention_weight_min": -30.04066276550293,
      "activations/layer3_attention_weight_max": 85.49546813964844,
      "activations/layer3_attention_weight_min": -87.81947326660156,
      "activations/layer4_attention_weight_max": 85.21773529052734,
      "activations/layer4_attention_weight_min": -82.9704818725586,
      "activations/layer5_attention_weight_max": 66.47201538085938,
      "activations/layer5_attention_weight_min": -69.82038116455078,
      "activations/layer6_attention_weight_max": 48.78451156616211,
      "activations/layer6_attention_weight_min": -48.32326126098633,
      "activations/layer7_attention_weight_max": 59.86956024169922,
      "activations/layer7_attention_weight_min": -57.35978317260742,
      "activations/layer8_attention_weight_max": 43.97591018676758,
      "activations/layer8_attention_weight_min": -48.335750579833984,
      "activations/layer9_attention_weight_max": 42.318973541259766,
      "activations/layer9_attention_weight_min": -44.248836517333984,
      "epoch": 15.87,
      "learning_rate": 4.813409090909091e-05,
      "loss": 2.7469,
      "step": 273150
    },
    {
      "activations/layer0_attention_weight_max": 16.143890380859375,
      "activations/layer0_attention_weight_min": -14.23105239868164,
      "activations/layer10_attention_weight_max": 37.291481018066406,
      "activations/layer10_attention_weight_min": -37.13268280029297,
      "activations/layer11_attention_weight_max": 34.980369567871094,
      "activations/layer11_attention_weight_min": -33.986968994140625,
      "activations/layer12_attention_weight_max": 17.994409561157227,
      "activations/layer12_attention_weight_min": -29.183530807495117,
      "activations/layer13_attention_weight_max": 33.289241790771484,
      "activations/layer13_attention_weight_min": -27.080095291137695,
      "activations/layer14_attention_weight_max": 35.948570251464844,
      "activations/layer14_attention_weight_min": -29.67527198791504,
      "activations/layer15_attention_weight_max": 33.45549774169922,
      "activations/layer15_attention_weight_min": -29.301448822021484,
      "activations/layer16_attention_weight_max": 34.873355865478516,
      "activations/layer16_attention_weight_min": -32.685546875,
      "activations/layer17_attention_weight_max": 54.26127243041992,
      "activations/layer17_attention_weight_min": -46.976985931396484,
      "activations/layer18_attention_weight_max": 46.636470794677734,
      "activations/layer18_attention_weight_min": -40.86722946166992,
      "activations/layer19_attention_weight_max": 21.453996658325195,
      "activations/layer19_attention_weight_min": -22.554410934448242,
      "activations/layer1_attention_weight_max": 16.365957260131836,
      "activations/layer1_attention_weight_min": -13.736053466796875,
      "activations/layer20_attention_weight_max": 20.18370246887207,
      "activations/layer20_attention_weight_min": -22.228382110595703,
      "activations/layer21_attention_weight_max": 29.479557037353516,
      "activations/layer21_attention_weight_min": -19.269376754760742,
      "activations/layer22_attention_weight_max": 26.017358779907227,
      "activations/layer22_attention_weight_min": -26.008251190185547,
      "activations/layer23_attention_weight_max": 32.51275634765625,
      "activations/layer23_attention_weight_min": -24.310226440429688,
      "activations/layer2_attention_weight_max": 32.79069519042969,
      "activations/layer2_attention_weight_min": -31.26962661743164,
      "activations/layer3_attention_weight_max": 82.74080657958984,
      "activations/layer3_attention_weight_min": -86.82487487792969,
      "activations/layer4_attention_weight_max": 89.15152740478516,
      "activations/layer4_attention_weight_min": -86.59685516357422,
      "activations/layer5_attention_weight_max": 64.95816802978516,
      "activations/layer5_attention_weight_min": -72.83975219726562,
      "activations/layer6_attention_weight_max": 51.11935806274414,
      "activations/layer6_attention_weight_min": -52.18595504760742,
      "activations/layer7_attention_weight_max": 60.307674407958984,
      "activations/layer7_attention_weight_min": -61.55045700073242,
      "activations/layer8_attention_weight_max": 46.35478591918945,
      "activations/layer8_attention_weight_min": -48.407798767089844,
      "activations/layer9_attention_weight_max": 45.11318588256836,
      "activations/layer9_attention_weight_min": -44.843780517578125,
      "epoch": 15.87,
      "learning_rate": 4.811515151515151e-05,
      "loss": 2.7522,
      "step": 273200
    },
    {
      "activations/layer0_attention_weight_max": 16.099693298339844,
      "activations/layer0_attention_weight_min": -14.21732234954834,
      "activations/layer10_attention_weight_max": 42.41707229614258,
      "activations/layer10_attention_weight_min": -36.823238372802734,
      "activations/layer11_attention_weight_max": 40.4105339050293,
      "activations/layer11_attention_weight_min": -35.192176818847656,
      "activations/layer12_attention_weight_max": 18.201017379760742,
      "activations/layer12_attention_weight_min": -27.14392852783203,
      "activations/layer13_attention_weight_max": 52.831153869628906,
      "activations/layer13_attention_weight_min": -32.55613327026367,
      "activations/layer14_attention_weight_max": 44.84953308105469,
      "activations/layer14_attention_weight_min": -29.297096252441406,
      "activations/layer15_attention_weight_max": 41.30574035644531,
      "activations/layer15_attention_weight_min": -29.779150009155273,
      "activations/layer16_attention_weight_max": 34.03571319580078,
      "activations/layer16_attention_weight_min": -31.387325286865234,
      "activations/layer17_attention_weight_max": 52.39419174194336,
      "activations/layer17_attention_weight_min": -45.579612731933594,
      "activations/layer18_attention_weight_max": 46.50191116333008,
      "activations/layer18_attention_weight_min": -42.44778823852539,
      "activations/layer19_attention_weight_max": 23.146047592163086,
      "activations/layer19_attention_weight_min": -22.600072860717773,
      "activations/layer1_attention_weight_max": 15.741894721984863,
      "activations/layer1_attention_weight_min": -17.25523567199707,
      "activations/layer20_attention_weight_max": 24.63597297668457,
      "activations/layer20_attention_weight_min": -22.157495498657227,
      "activations/layer21_attention_weight_max": 48.190818786621094,
      "activations/layer21_attention_weight_min": -24.160003662109375,
      "activations/layer22_attention_weight_max": 31.741493225097656,
      "activations/layer22_attention_weight_min": -25.647539138793945,
      "activations/layer23_attention_weight_max": 38.89362335205078,
      "activations/layer23_attention_weight_min": -22.655902862548828,
      "activations/layer2_attention_weight_max": 33.54242706298828,
      "activations/layer2_attention_weight_min": -30.79958724975586,
      "activations/layer3_attention_weight_max": 92.37489318847656,
      "activations/layer3_attention_weight_min": -97.01618194580078,
      "activations/layer4_attention_weight_max": 91.90895080566406,
      "activations/layer4_attention_weight_min": -91.08319091796875,
      "activations/layer5_attention_weight_max": 71.60921478271484,
      "activations/layer5_attention_weight_min": -77.27348327636719,
      "activations/layer6_attention_weight_max": 52.177608489990234,
      "activations/layer6_attention_weight_min": -54.857666015625,
      "activations/layer7_attention_weight_max": 71.70503234863281,
      "activations/layer7_attention_weight_min": -63.08778381347656,
      "activations/layer8_attention_weight_max": 52.59013748168945,
      "activations/layer8_attention_weight_min": -51.124725341796875,
      "activations/layer9_attention_weight_max": 59.36457443237305,
      "activations/layer9_attention_weight_min": -51.41057205200195,
      "epoch": 15.88,
      "learning_rate": 4.809621212121211e-05,
      "loss": 2.7471,
      "step": 273250
    },
    {
      "activations/layer0_attention_weight_max": 14.9942626953125,
      "activations/layer0_attention_weight_min": -14.247554779052734,
      "activations/layer10_attention_weight_max": 38.942481994628906,
      "activations/layer10_attention_weight_min": -37.200775146484375,
      "activations/layer11_attention_weight_max": 33.83666229248047,
      "activations/layer11_attention_weight_min": -33.63054275512695,
      "activations/layer12_attention_weight_max": 17.414358139038086,
      "activations/layer12_attention_weight_min": -23.88994598388672,
      "activations/layer13_attention_weight_max": 31.99582290649414,
      "activations/layer13_attention_weight_min": -27.296049118041992,
      "activations/layer14_attention_weight_max": 32.836910247802734,
      "activations/layer14_attention_weight_min": -29.177698135375977,
      "activations/layer15_attention_weight_max": 30.930482864379883,
      "activations/layer15_attention_weight_min": -28.807401657104492,
      "activations/layer16_attention_weight_max": 32.302120208740234,
      "activations/layer16_attention_weight_min": -32.40280532836914,
      "activations/layer17_attention_weight_max": 49.03046417236328,
      "activations/layer17_attention_weight_min": -45.418174743652344,
      "activations/layer18_attention_weight_max": 47.21407699584961,
      "activations/layer18_attention_weight_min": -38.36444091796875,
      "activations/layer19_attention_weight_max": 23.246946334838867,
      "activations/layer19_attention_weight_min": -21.082443237304688,
      "activations/layer1_attention_weight_max": 16.234214782714844,
      "activations/layer1_attention_weight_min": -14.143006324768066,
      "activations/layer20_attention_weight_max": 21.000246047973633,
      "activations/layer20_attention_weight_min": -21.72283172607422,
      "activations/layer21_attention_weight_max": 33.94154739379883,
      "activations/layer21_attention_weight_min": -22.28312110900879,
      "activations/layer22_attention_weight_max": 29.512454986572266,
      "activations/layer22_attention_weight_min": -26.9664306640625,
      "activations/layer23_attention_weight_max": 34.36923599243164,
      "activations/layer23_attention_weight_min": -23.563453674316406,
      "activations/layer2_attention_weight_max": 30.794157028198242,
      "activations/layer2_attention_weight_min": -31.740446090698242,
      "activations/layer3_attention_weight_max": 84.02862548828125,
      "activations/layer3_attention_weight_min": -86.42884826660156,
      "activations/layer4_attention_weight_max": 87.90172576904297,
      "activations/layer4_attention_weight_min": -88.12140655517578,
      "activations/layer5_attention_weight_max": 68.9681167602539,
      "activations/layer5_attention_weight_min": -79.15681457519531,
      "activations/layer6_attention_weight_max": 49.149497985839844,
      "activations/layer6_attention_weight_min": -53.41926193237305,
      "activations/layer7_attention_weight_max": 59.95318603515625,
      "activations/layer7_attention_weight_min": -63.50757598876953,
      "activations/layer8_attention_weight_max": 48.7236328125,
      "activations/layer8_attention_weight_min": -53.28506851196289,
      "activations/layer9_attention_weight_max": 45.76876449584961,
      "activations/layer9_attention_weight_min": -48.43682861328125,
      "epoch": 15.88,
      "learning_rate": 4.8077272727272725e-05,
      "loss": 2.7551,
      "step": 273300
    },
    {
      "activations/layer0_attention_weight_max": 15.080924034118652,
      "activations/layer0_attention_weight_min": -14.452434539794922,
      "activations/layer10_attention_weight_max": 37.78853988647461,
      "activations/layer10_attention_weight_min": -37.324363708496094,
      "activations/layer11_attention_weight_max": 32.63014221191406,
      "activations/layer11_attention_weight_min": -35.42033386230469,
      "activations/layer12_attention_weight_max": 18.888154983520508,
      "activations/layer12_attention_weight_min": -26.678586959838867,
      "activations/layer13_attention_weight_max": 43.85881042480469,
      "activations/layer13_attention_weight_min": -31.057655334472656,
      "activations/layer14_attention_weight_max": 32.62936782836914,
      "activations/layer14_attention_weight_min": -27.94191551208496,
      "activations/layer15_attention_weight_max": 30.792028427124023,
      "activations/layer15_attention_weight_min": -28.96511459350586,
      "activations/layer16_attention_weight_max": 31.913990020751953,
      "activations/layer16_attention_weight_min": -30.798627853393555,
      "activations/layer17_attention_weight_max": 47.567928314208984,
      "activations/layer17_attention_weight_min": -42.08782196044922,
      "activations/layer18_attention_weight_max": 44.84017562866211,
      "activations/layer18_attention_weight_min": -37.29426193237305,
      "activations/layer19_attention_weight_max": 25.989599227905273,
      "activations/layer19_attention_weight_min": -21.563100814819336,
      "activations/layer1_attention_weight_max": 15.576972961425781,
      "activations/layer1_attention_weight_min": -14.818137168884277,
      "activations/layer20_attention_weight_max": 24.712434768676758,
      "activations/layer20_attention_weight_min": -22.92537498474121,
      "activations/layer21_attention_weight_max": 34.57731628417969,
      "activations/layer21_attention_weight_min": -21.800071716308594,
      "activations/layer22_attention_weight_max": 27.766891479492188,
      "activations/layer22_attention_weight_min": -25.71468162536621,
      "activations/layer23_attention_weight_max": 36.663299560546875,
      "activations/layer23_attention_weight_min": -24.093618392944336,
      "activations/layer2_attention_weight_max": 31.321020126342773,
      "activations/layer2_attention_weight_min": -31.37677764892578,
      "activations/layer3_attention_weight_max": 91.71009826660156,
      "activations/layer3_attention_weight_min": -94.75819396972656,
      "activations/layer4_attention_weight_max": 87.93936157226562,
      "activations/layer4_attention_weight_min": -89.4999008178711,
      "activations/layer5_attention_weight_max": 65.34825134277344,
      "activations/layer5_attention_weight_min": -76.33489990234375,
      "activations/layer6_attention_weight_max": 50.337398529052734,
      "activations/layer6_attention_weight_min": -52.81657409667969,
      "activations/layer7_attention_weight_max": 60.929508209228516,
      "activations/layer7_attention_weight_min": -63.27800750732422,
      "activations/layer8_attention_weight_max": 46.48976135253906,
      "activations/layer8_attention_weight_min": -49.82126235961914,
      "activations/layer9_attention_weight_max": 43.02412033081055,
      "activations/layer9_attention_weight_min": -48.08070373535156,
      "epoch": 15.88,
      "learning_rate": 4.805833333333333e-05,
      "loss": 2.7485,
      "step": 273350
    },
    {
      "activations/layer0_attention_weight_max": 16.0649471282959,
      "activations/layer0_attention_weight_min": -14.035863876342773,
      "activations/layer10_attention_weight_max": 35.421173095703125,
      "activations/layer10_attention_weight_min": -34.11627960205078,
      "activations/layer11_attention_weight_max": 33.130313873291016,
      "activations/layer11_attention_weight_min": -32.567142486572266,
      "activations/layer12_attention_weight_max": 18.773746490478516,
      "activations/layer12_attention_weight_min": -25.017478942871094,
      "activations/layer13_attention_weight_max": 39.26148986816406,
      "activations/layer13_attention_weight_min": -30.926498413085938,
      "activations/layer14_attention_weight_max": 34.02871322631836,
      "activations/layer14_attention_weight_min": -30.289899826049805,
      "activations/layer15_attention_weight_max": 32.74298095703125,
      "activations/layer15_attention_weight_min": -29.108627319335938,
      "activations/layer16_attention_weight_max": 33.779632568359375,
      "activations/layer16_attention_weight_min": -32.64575958251953,
      "activations/layer17_attention_weight_max": 48.45152282714844,
      "activations/layer17_attention_weight_min": -44.0106315612793,
      "activations/layer18_attention_weight_max": 43.79180145263672,
      "activations/layer18_attention_weight_min": -39.13749313354492,
      "activations/layer19_attention_weight_max": 24.502309799194336,
      "activations/layer19_attention_weight_min": -23.72814178466797,
      "activations/layer1_attention_weight_max": 15.861030578613281,
      "activations/layer1_attention_weight_min": -13.54030704498291,
      "activations/layer20_attention_weight_max": 21.58357810974121,
      "activations/layer20_attention_weight_min": -22.78468894958496,
      "activations/layer21_attention_weight_max": 34.449432373046875,
      "activations/layer21_attention_weight_min": -24.15110206604004,
      "activations/layer22_attention_weight_max": 28.79444694519043,
      "activations/layer22_attention_weight_min": -26.456214904785156,
      "activations/layer23_attention_weight_max": 36.716392517089844,
      "activations/layer23_attention_weight_min": -24.83721923828125,
      "activations/layer2_attention_weight_max": 30.803749084472656,
      "activations/layer2_attention_weight_min": -28.486356735229492,
      "activations/layer3_attention_weight_max": 83.69226837158203,
      "activations/layer3_attention_weight_min": -81.0445785522461,
      "activations/layer4_attention_weight_max": 86.86068725585938,
      "activations/layer4_attention_weight_min": -84.73314666748047,
      "activations/layer5_attention_weight_max": 63.744163513183594,
      "activations/layer5_attention_weight_min": -70.41238403320312,
      "activations/layer6_attention_weight_max": 52.63151931762695,
      "activations/layer6_attention_weight_min": -49.140953063964844,
      "activations/layer7_attention_weight_max": 60.35342788696289,
      "activations/layer7_attention_weight_min": -61.56568145751953,
      "activations/layer8_attention_weight_max": 47.27254867553711,
      "activations/layer8_attention_weight_min": -48.130191802978516,
      "activations/layer9_attention_weight_max": 43.90581512451172,
      "activations/layer9_attention_weight_min": -45.96953201293945,
      "epoch": 15.89,
      "learning_rate": 4.8039393939393935e-05,
      "loss": 2.7505,
      "step": 273400
    },
    {
      "activations/layer0_attention_weight_max": 16.171972274780273,
      "activations/layer0_attention_weight_min": -13.556529998779297,
      "activations/layer10_attention_weight_max": 35.99267578125,
      "activations/layer10_attention_weight_min": -34.44977951049805,
      "activations/layer11_attention_weight_max": 33.14899826049805,
      "activations/layer11_attention_weight_min": -32.906272888183594,
      "activations/layer12_attention_weight_max": 18.206527709960938,
      "activations/layer12_attention_weight_min": -23.446924209594727,
      "activations/layer13_attention_weight_max": 36.30504608154297,
      "activations/layer13_attention_weight_min": -27.040767669677734,
      "activations/layer14_attention_weight_max": 35.208187103271484,
      "activations/layer14_attention_weight_min": -29.599985122680664,
      "activations/layer15_attention_weight_max": 32.694618225097656,
      "activations/layer15_attention_weight_min": -29.60664939880371,
      "activations/layer16_attention_weight_max": 32.59345626831055,
      "activations/layer16_attention_weight_min": -30.220462799072266,
      "activations/layer17_attention_weight_max": 52.26433563232422,
      "activations/layer17_attention_weight_min": -44.83776092529297,
      "activations/layer18_attention_weight_max": 48.56127166748047,
      "activations/layer18_attention_weight_min": -38.823951721191406,
      "activations/layer19_attention_weight_max": 23.7918758392334,
      "activations/layer19_attention_weight_min": -22.783184051513672,
      "activations/layer1_attention_weight_max": 17.038652420043945,
      "activations/layer1_attention_weight_min": -14.89509391784668,
      "activations/layer20_attention_weight_max": 24.847143173217773,
      "activations/layer20_attention_weight_min": -22.276342391967773,
      "activations/layer21_attention_weight_max": 34.0169677734375,
      "activations/layer21_attention_weight_min": -20.882307052612305,
      "activations/layer22_attention_weight_max": 30.434953689575195,
      "activations/layer22_attention_weight_min": -25.909570693969727,
      "activations/layer23_attention_weight_max": 37.86007308959961,
      "activations/layer23_attention_weight_min": -23.630970001220703,
      "activations/layer2_attention_weight_max": 31.545886993408203,
      "activations/layer2_attention_weight_min": -31.148670196533203,
      "activations/layer3_attention_weight_max": 89.34810638427734,
      "activations/layer3_attention_weight_min": -87.46479797363281,
      "activations/layer4_attention_weight_max": 89.61689758300781,
      "activations/layer4_attention_weight_min": -88.84639739990234,
      "activations/layer5_attention_weight_max": 65.0992660522461,
      "activations/layer5_attention_weight_min": -72.677734375,
      "activations/layer6_attention_weight_max": 50.15719223022461,
      "activations/layer6_attention_weight_min": -51.85283660888672,
      "activations/layer7_attention_weight_max": 59.71158218383789,
      "activations/layer7_attention_weight_min": -57.85209274291992,
      "activations/layer8_attention_weight_max": 45.29838562011719,
      "activations/layer8_attention_weight_min": -48.67697525024414,
      "activations/layer9_attention_weight_max": 42.792118072509766,
      "activations/layer9_attention_weight_min": -44.04463195800781,
      "epoch": 15.89,
      "learning_rate": 4.8020454545454543e-05,
      "loss": 2.7624,
      "step": 273450
    },
    {
      "activations/layer0_attention_weight_max": 16.12241554260254,
      "activations/layer0_attention_weight_min": -13.933450698852539,
      "activations/layer10_attention_weight_max": 38.970252990722656,
      "activations/layer10_attention_weight_min": -38.875431060791016,
      "activations/layer11_attention_weight_max": 36.108131408691406,
      "activations/layer11_attention_weight_min": -36.49064636230469,
      "activations/layer12_attention_weight_max": 19.805898666381836,
      "activations/layer12_attention_weight_min": -23.71184730529785,
      "activations/layer13_attention_weight_max": 34.63198471069336,
      "activations/layer13_attention_weight_min": -25.7640438079834,
      "activations/layer14_attention_weight_max": 36.971893310546875,
      "activations/layer14_attention_weight_min": -30.729875564575195,
      "activations/layer15_attention_weight_max": 32.446632385253906,
      "activations/layer15_attention_weight_min": -30.653810501098633,
      "activations/layer16_attention_weight_max": 34.18234634399414,
      "activations/layer16_attention_weight_min": -32.210662841796875,
      "activations/layer17_attention_weight_max": 51.69487380981445,
      "activations/layer17_attention_weight_min": -45.67146682739258,
      "activations/layer18_attention_weight_max": 47.06133270263672,
      "activations/layer18_attention_weight_min": -37.60946273803711,
      "activations/layer19_attention_weight_max": 21.980443954467773,
      "activations/layer19_attention_weight_min": -20.980117797851562,
      "activations/layer1_attention_weight_max": 18.164281845092773,
      "activations/layer1_attention_weight_min": -13.541353225708008,
      "activations/layer20_attention_weight_max": 21.836597442626953,
      "activations/layer20_attention_weight_min": -21.258441925048828,
      "activations/layer21_attention_weight_max": 38.826473236083984,
      "activations/layer21_attention_weight_min": -21.845731735229492,
      "activations/layer22_attention_weight_max": 26.694286346435547,
      "activations/layer22_attention_weight_min": -25.04991912841797,
      "activations/layer23_attention_weight_max": 35.855079650878906,
      "activations/layer23_attention_weight_min": -25.73202896118164,
      "activations/layer2_attention_weight_max": 30.472900390625,
      "activations/layer2_attention_weight_min": -29.050071716308594,
      "activations/layer3_attention_weight_max": 87.25030517578125,
      "activations/layer3_attention_weight_min": -89.65264129638672,
      "activations/layer4_attention_weight_max": 88.76329040527344,
      "activations/layer4_attention_weight_min": -88.76685333251953,
      "activations/layer5_attention_weight_max": 63.608978271484375,
      "activations/layer5_attention_weight_min": -76.24202728271484,
      "activations/layer6_attention_weight_max": 49.17708969116211,
      "activations/layer6_attention_weight_min": -51.284786224365234,
      "activations/layer7_attention_weight_max": 62.30591583251953,
      "activations/layer7_attention_weight_min": -60.95759963989258,
      "activations/layer8_attention_weight_max": 47.57299041748047,
      "activations/layer8_attention_weight_min": -50.72665786743164,
      "activations/layer9_attention_weight_max": 46.224430084228516,
      "activations/layer9_attention_weight_min": -50.85493850708008,
      "epoch": 15.89,
      "learning_rate": 4.800151515151515e-05,
      "loss": 2.7541,
      "step": 273500
    },
    {
      "activations/layer0_attention_weight_max": 15.909112930297852,
      "activations/layer0_attention_weight_min": -13.55262279510498,
      "activations/layer10_attention_weight_max": 36.118682861328125,
      "activations/layer10_attention_weight_min": -36.30974578857422,
      "activations/layer11_attention_weight_max": 31.38020133972168,
      "activations/layer11_attention_weight_min": -32.90913391113281,
      "activations/layer12_attention_weight_max": 18.84286880493164,
      "activations/layer12_attention_weight_min": -24.132368087768555,
      "activations/layer13_attention_weight_max": 33.1389045715332,
      "activations/layer13_attention_weight_min": -28.20207405090332,
      "activations/layer14_attention_weight_max": 34.025875091552734,
      "activations/layer14_attention_weight_min": -30.83504867553711,
      "activations/layer15_attention_weight_max": 32.22111892700195,
      "activations/layer15_attention_weight_min": -29.37734031677246,
      "activations/layer16_attention_weight_max": 32.173831939697266,
      "activations/layer16_attention_weight_min": -32.10776138305664,
      "activations/layer17_attention_weight_max": 49.9296760559082,
      "activations/layer17_attention_weight_min": -45.809574127197266,
      "activations/layer18_attention_weight_max": 42.615867614746094,
      "activations/layer18_attention_weight_min": -41.8797607421875,
      "activations/layer19_attention_weight_max": 22.318756103515625,
      "activations/layer19_attention_weight_min": -25.11479377746582,
      "activations/layer1_attention_weight_max": 15.842874526977539,
      "activations/layer1_attention_weight_min": -14.119361877441406,
      "activations/layer20_attention_weight_max": 23.529369354248047,
      "activations/layer20_attention_weight_min": -23.0341854095459,
      "activations/layer21_attention_weight_max": 33.74928665161133,
      "activations/layer21_attention_weight_min": -24.4083309173584,
      "activations/layer22_attention_weight_max": 27.948646545410156,
      "activations/layer22_attention_weight_min": -28.45517349243164,
      "activations/layer23_attention_weight_max": 33.16353225708008,
      "activations/layer23_attention_weight_min": -23.648983001708984,
      "activations/layer2_attention_weight_max": 30.778989791870117,
      "activations/layer2_attention_weight_min": -28.628379821777344,
      "activations/layer3_attention_weight_max": 85.64313507080078,
      "activations/layer3_attention_weight_min": -88.80896759033203,
      "activations/layer4_attention_weight_max": 83.6446533203125,
      "activations/layer4_attention_weight_min": -84.38341522216797,
      "activations/layer5_attention_weight_max": 64.23743438720703,
      "activations/layer5_attention_weight_min": -69.6494140625,
      "activations/layer6_attention_weight_max": 46.88515853881836,
      "activations/layer6_attention_weight_min": -50.72026062011719,
      "activations/layer7_attention_weight_max": 59.718204498291016,
      "activations/layer7_attention_weight_min": -61.700660705566406,
      "activations/layer8_attention_weight_max": 43.946624755859375,
      "activations/layer8_attention_weight_min": -49.78865051269531,
      "activations/layer9_attention_weight_max": 45.056556701660156,
      "activations/layer9_attention_weight_min": -46.97504806518555,
      "epoch": 15.89,
      "learning_rate": 4.798257575757575e-05,
      "loss": 2.7471,
      "step": 273550
    },
    {
      "activations/layer0_attention_weight_max": 16.54007339477539,
      "activations/layer0_attention_weight_min": -13.54298210144043,
      "activations/layer10_attention_weight_max": 36.922298431396484,
      "activations/layer10_attention_weight_min": -36.099769592285156,
      "activations/layer11_attention_weight_max": 33.440799713134766,
      "activations/layer11_attention_weight_min": -33.309898376464844,
      "activations/layer12_attention_weight_max": 18.747177124023438,
      "activations/layer12_attention_weight_min": -25.813669204711914,
      "activations/layer13_attention_weight_max": 38.49132537841797,
      "activations/layer13_attention_weight_min": -26.667814254760742,
      "activations/layer14_attention_weight_max": 36.90131378173828,
      "activations/layer14_attention_weight_min": -31.144378662109375,
      "activations/layer15_attention_weight_max": 33.22434616088867,
      "activations/layer15_attention_weight_min": -31.070363998413086,
      "activations/layer16_attention_weight_max": 34.89476776123047,
      "activations/layer16_attention_weight_min": -32.91939926147461,
      "activations/layer17_attention_weight_max": 51.58603286743164,
      "activations/layer17_attention_weight_min": -45.50790023803711,
      "activations/layer18_attention_weight_max": 46.77431869506836,
      "activations/layer18_attention_weight_min": -41.696903228759766,
      "activations/layer19_attention_weight_max": 23.39044761657715,
      "activations/layer19_attention_weight_min": -23.974050521850586,
      "activations/layer1_attention_weight_max": 16.24529266357422,
      "activations/layer1_attention_weight_min": -15.65671443939209,
      "activations/layer20_attention_weight_max": 23.3061580657959,
      "activations/layer20_attention_weight_min": -21.476194381713867,
      "activations/layer21_attention_weight_max": 37.48594665527344,
      "activations/layer21_attention_weight_min": -23.0889949798584,
      "activations/layer22_attention_weight_max": 31.344921112060547,
      "activations/layer22_attention_weight_min": -27.37447738647461,
      "activations/layer23_attention_weight_max": 36.89681625366211,
      "activations/layer23_attention_weight_min": -22.888469696044922,
      "activations/layer2_attention_weight_max": 32.65053176879883,
      "activations/layer2_attention_weight_min": -30.43572235107422,
      "activations/layer3_attention_weight_max": 86.35108947753906,
      "activations/layer3_attention_weight_min": -84.01803588867188,
      "activations/layer4_attention_weight_max": 87.41963958740234,
      "activations/layer4_attention_weight_min": -87.43050384521484,
      "activations/layer5_attention_weight_max": 64.96715545654297,
      "activations/layer5_attention_weight_min": -72.75033569335938,
      "activations/layer6_attention_weight_max": 51.835323333740234,
      "activations/layer6_attention_weight_min": -50.947689056396484,
      "activations/layer7_attention_weight_max": 61.665245056152344,
      "activations/layer7_attention_weight_min": -60.17070388793945,
      "activations/layer8_attention_weight_max": 47.827449798583984,
      "activations/layer8_attention_weight_min": -47.5876350402832,
      "activations/layer9_attention_weight_max": 44.933921813964844,
      "activations/layer9_attention_weight_min": -46.899837493896484,
      "epoch": 15.9,
      "learning_rate": 4.7963636363636355e-05,
      "loss": 2.7501,
      "step": 273600
    },
    {
      "activations/layer0_attention_weight_max": 15.989731788635254,
      "activations/layer0_attention_weight_min": -14.117015838623047,
      "activations/layer10_attention_weight_max": 35.787776947021484,
      "activations/layer10_attention_weight_min": -34.93048095703125,
      "activations/layer11_attention_weight_max": 30.234882354736328,
      "activations/layer11_attention_weight_min": -32.65678787231445,
      "activations/layer12_attention_weight_max": 18.2082462310791,
      "activations/layer12_attention_weight_min": -29.391849517822266,
      "activations/layer13_attention_weight_max": 34.18852233886719,
      "activations/layer13_attention_weight_min": -29.019838333129883,
      "activations/layer14_attention_weight_max": 31.60721206665039,
      "activations/layer14_attention_weight_min": -31.872079849243164,
      "activations/layer15_attention_weight_max": 29.162702560424805,
      "activations/layer15_attention_weight_min": -29.45237159729004,
      "activations/layer16_attention_weight_max": 31.205705642700195,
      "activations/layer16_attention_weight_min": -31.41792869567871,
      "activations/layer17_attention_weight_max": 47.12203598022461,
      "activations/layer17_attention_weight_min": -41.72882080078125,
      "activations/layer18_attention_weight_max": 42.143070220947266,
      "activations/layer18_attention_weight_min": -38.94350051879883,
      "activations/layer19_attention_weight_max": 21.66639518737793,
      "activations/layer19_attention_weight_min": -24.149559020996094,
      "activations/layer1_attention_weight_max": 17.060422897338867,
      "activations/layer1_attention_weight_min": -15.937644958496094,
      "activations/layer20_attention_weight_max": 21.81570816040039,
      "activations/layer20_attention_weight_min": -23.054290771484375,
      "activations/layer21_attention_weight_max": 32.27720642089844,
      "activations/layer21_attention_weight_min": -21.607891082763672,
      "activations/layer22_attention_weight_max": 29.338703155517578,
      "activations/layer22_attention_weight_min": -30.832990646362305,
      "activations/layer23_attention_weight_max": 39.25393295288086,
      "activations/layer23_attention_weight_min": -27.792041778564453,
      "activations/layer2_attention_weight_max": 33.31572723388672,
      "activations/layer2_attention_weight_min": -31.05855369567871,
      "activations/layer3_attention_weight_max": 87.82791900634766,
      "activations/layer3_attention_weight_min": -91.26625061035156,
      "activations/layer4_attention_weight_max": 84.26465606689453,
      "activations/layer4_attention_weight_min": -84.93671417236328,
      "activations/layer5_attention_weight_max": 63.53852844238281,
      "activations/layer5_attention_weight_min": -75.2802734375,
      "activations/layer6_attention_weight_max": 49.2758674621582,
      "activations/layer6_attention_weight_min": -48.3132209777832,
      "activations/layer7_attention_weight_max": 61.8556022644043,
      "activations/layer7_attention_weight_min": -59.428672790527344,
      "activations/layer8_attention_weight_max": 44.8780632019043,
      "activations/layer8_attention_weight_min": -47.63896560668945,
      "activations/layer9_attention_weight_max": 44.36648941040039,
      "activations/layer9_attention_weight_min": -43.4483757019043,
      "epoch": 15.9,
      "learning_rate": 4.794469696969697e-05,
      "loss": 2.7506,
      "step": 273650
    },
    {
      "activations/layer0_attention_weight_max": 15.709921836853027,
      "activations/layer0_attention_weight_min": -14.049966812133789,
      "activations/layer10_attention_weight_max": 37.706260681152344,
      "activations/layer10_attention_weight_min": -36.19316101074219,
      "activations/layer11_attention_weight_max": 31.814754486083984,
      "activations/layer11_attention_weight_min": -33.98388671875,
      "activations/layer12_attention_weight_max": 17.62556266784668,
      "activations/layer12_attention_weight_min": -25.43527603149414,
      "activations/layer13_attention_weight_max": 29.50580596923828,
      "activations/layer13_attention_weight_min": -25.506635665893555,
      "activations/layer14_attention_weight_max": 34.626060485839844,
      "activations/layer14_attention_weight_min": -31.997255325317383,
      "activations/layer15_attention_weight_max": 32.522216796875,
      "activations/layer15_attention_weight_min": -30.8976993560791,
      "activations/layer16_attention_weight_max": 31.811283111572266,
      "activations/layer16_attention_weight_min": -32.84706497192383,
      "activations/layer17_attention_weight_max": 51.2100830078125,
      "activations/layer17_attention_weight_min": -45.2084846496582,
      "activations/layer18_attention_weight_max": 43.39591979980469,
      "activations/layer18_attention_weight_min": -43.20327377319336,
      "activations/layer19_attention_weight_max": 24.12779426574707,
      "activations/layer19_attention_weight_min": -23.46304702758789,
      "activations/layer1_attention_weight_max": 15.843475341796875,
      "activations/layer1_attention_weight_min": -14.533089637756348,
      "activations/layer20_attention_weight_max": 22.193681716918945,
      "activations/layer20_attention_weight_min": -20.763763427734375,
      "activations/layer21_attention_weight_max": 36.19997024536133,
      "activations/layer21_attention_weight_min": -22.10283660888672,
      "activations/layer22_attention_weight_max": 27.7109375,
      "activations/layer22_attention_weight_min": -23.995006561279297,
      "activations/layer23_attention_weight_max": 31.880111694335938,
      "activations/layer23_attention_weight_min": -23.751766204833984,
      "activations/layer2_attention_weight_max": 34.86156463623047,
      "activations/layer2_attention_weight_min": -33.69807434082031,
      "activations/layer3_attention_weight_max": 88.7689437866211,
      "activations/layer3_attention_weight_min": -89.56736755371094,
      "activations/layer4_attention_weight_max": 89.83097839355469,
      "activations/layer4_attention_weight_min": -87.48685455322266,
      "activations/layer5_attention_weight_max": 67.4756088256836,
      "activations/layer5_attention_weight_min": -76.3670883178711,
      "activations/layer6_attention_weight_max": 51.79335021972656,
      "activations/layer6_attention_weight_min": -49.91666793823242,
      "activations/layer7_attention_weight_max": 63.48798751831055,
      "activations/layer7_attention_weight_min": -59.34798049926758,
      "activations/layer8_attention_weight_max": 46.19621276855469,
      "activations/layer8_attention_weight_min": -50.733360290527344,
      "activations/layer9_attention_weight_max": 43.963401794433594,
      "activations/layer9_attention_weight_min": -46.47254180908203,
      "epoch": 15.9,
      "learning_rate": 4.792575757575757e-05,
      "loss": 2.7546,
      "step": 273700
    },
    {
      "activations/layer0_attention_weight_max": 16.18680763244629,
      "activations/layer0_attention_weight_min": -13.923867225646973,
      "activations/layer10_attention_weight_max": 34.4696044921875,
      "activations/layer10_attention_weight_min": -34.06691360473633,
      "activations/layer11_attention_weight_max": 31.633893966674805,
      "activations/layer11_attention_weight_min": -32.183284759521484,
      "activations/layer12_attention_weight_max": 18.165586471557617,
      "activations/layer12_attention_weight_min": -26.281129837036133,
      "activations/layer13_attention_weight_max": 35.289894104003906,
      "activations/layer13_attention_weight_min": -27.814165115356445,
      "activations/layer14_attention_weight_max": 34.6256217956543,
      "activations/layer14_attention_weight_min": -30.0252742767334,
      "activations/layer15_attention_weight_max": 32.97190475463867,
      "activations/layer15_attention_weight_min": -28.798402786254883,
      "activations/layer16_attention_weight_max": 32.50308609008789,
      "activations/layer16_attention_weight_min": -30.70884895324707,
      "activations/layer17_attention_weight_max": 53.676753997802734,
      "activations/layer17_attention_weight_min": -47.0595588684082,
      "activations/layer18_attention_weight_max": 52.828392028808594,
      "activations/layer18_attention_weight_min": -43.35268783569336,
      "activations/layer19_attention_weight_max": 24.1074275970459,
      "activations/layer19_attention_weight_min": -24.994117736816406,
      "activations/layer1_attention_weight_max": 15.620790481567383,
      "activations/layer1_attention_weight_min": -13.07987117767334,
      "activations/layer20_attention_weight_max": 22.891246795654297,
      "activations/layer20_attention_weight_min": -21.317825317382812,
      "activations/layer21_attention_weight_max": 35.68385696411133,
      "activations/layer21_attention_weight_min": -24.279644012451172,
      "activations/layer22_attention_weight_max": 29.725204467773438,
      "activations/layer22_attention_weight_min": -26.813026428222656,
      "activations/layer23_attention_weight_max": 38.099788665771484,
      "activations/layer23_attention_weight_min": -26.9179744720459,
      "activations/layer2_attention_weight_max": 31.473167419433594,
      "activations/layer2_attention_weight_min": -29.39543914794922,
      "activations/layer3_attention_weight_max": 89.48490905761719,
      "activations/layer3_attention_weight_min": -86.93095397949219,
      "activations/layer4_attention_weight_max": 92.49159240722656,
      "activations/layer4_attention_weight_min": -87.18727111816406,
      "activations/layer5_attention_weight_max": 68.3713150024414,
      "activations/layer5_attention_weight_min": -72.02073669433594,
      "activations/layer6_attention_weight_max": 53.82273864746094,
      "activations/layer6_attention_weight_min": -49.89640808105469,
      "activations/layer7_attention_weight_max": 59.540504455566406,
      "activations/layer7_attention_weight_min": -62.231964111328125,
      "activations/layer8_attention_weight_max": 46.10969924926758,
      "activations/layer8_attention_weight_min": -49.17009353637695,
      "activations/layer9_attention_weight_max": 45.193851470947266,
      "activations/layer9_attention_weight_min": -45.357173919677734,
      "epoch": 15.91,
      "learning_rate": 4.790681818181818e-05,
      "loss": 2.7511,
      "step": 273750
    },
    {
      "activations/layer0_attention_weight_max": 14.90770149230957,
      "activations/layer0_attention_weight_min": -13.790926933288574,
      "activations/layer10_attention_weight_max": 36.53163528442383,
      "activations/layer10_attention_weight_min": -35.53252410888672,
      "activations/layer11_attention_weight_max": 32.91516876220703,
      "activations/layer11_attention_weight_min": -33.568519592285156,
      "activations/layer12_attention_weight_max": 17.891372680664062,
      "activations/layer12_attention_weight_min": -25.05169677734375,
      "activations/layer13_attention_weight_max": 35.69771957397461,
      "activations/layer13_attention_weight_min": -25.258115768432617,
      "activations/layer14_attention_weight_max": 34.35849380493164,
      "activations/layer14_attention_weight_min": -31.99191665649414,
      "activations/layer15_attention_weight_max": 32.33689498901367,
      "activations/layer15_attention_weight_min": -29.108320236206055,
      "activations/layer16_attention_weight_max": 34.62404251098633,
      "activations/layer16_attention_weight_min": -30.89589500427246,
      "activations/layer17_attention_weight_max": 55.13826370239258,
      "activations/layer17_attention_weight_min": -47.62959671020508,
      "activations/layer18_attention_weight_max": 52.25672149658203,
      "activations/layer18_attention_weight_min": -43.312923431396484,
      "activations/layer19_attention_weight_max": 27.06294059753418,
      "activations/layer19_attention_weight_min": -23.234479904174805,
      "activations/layer1_attention_weight_max": 16.15692901611328,
      "activations/layer1_attention_weight_min": -16.26314353942871,
      "activations/layer20_attention_weight_max": 26.853647232055664,
      "activations/layer20_attention_weight_min": -20.866649627685547,
      "activations/layer21_attention_weight_max": 40.23947525024414,
      "activations/layer21_attention_weight_min": -23.00469207763672,
      "activations/layer22_attention_weight_max": 35.67571258544922,
      "activations/layer22_attention_weight_min": -26.61353874206543,
      "activations/layer23_attention_weight_max": 39.431156158447266,
      "activations/layer23_attention_weight_min": -25.037107467651367,
      "activations/layer2_attention_weight_max": 31.431289672851562,
      "activations/layer2_attention_weight_min": -30.144145965576172,
      "activations/layer3_attention_weight_max": 94.13670349121094,
      "activations/layer3_attention_weight_min": -91.9834213256836,
      "activations/layer4_attention_weight_max": 88.32628631591797,
      "activations/layer4_attention_weight_min": -85.8656234741211,
      "activations/layer5_attention_weight_max": 63.84431076049805,
      "activations/layer5_attention_weight_min": -72.37922668457031,
      "activations/layer6_attention_weight_max": 47.825260162353516,
      "activations/layer6_attention_weight_min": -50.10945129394531,
      "activations/layer7_attention_weight_max": 59.37689208984375,
      "activations/layer7_attention_weight_min": -57.93510437011719,
      "activations/layer8_attention_weight_max": 45.874366760253906,
      "activations/layer8_attention_weight_min": -47.1653938293457,
      "activations/layer9_attention_weight_max": 44.52431869506836,
      "activations/layer9_attention_weight_min": -43.08458709716797,
      "epoch": 15.91,
      "learning_rate": 4.788787878787878e-05,
      "loss": 2.757,
      "step": 273800
    },
    {
      "activations/layer0_attention_weight_max": 14.950916290283203,
      "activations/layer0_attention_weight_min": -13.546370506286621,
      "activations/layer10_attention_weight_max": 35.004417419433594,
      "activations/layer10_attention_weight_min": -34.291587829589844,
      "activations/layer11_attention_weight_max": 35.676177978515625,
      "activations/layer11_attention_weight_min": -35.490535736083984,
      "activations/layer12_attention_weight_max": 17.56744384765625,
      "activations/layer12_attention_weight_min": -25.468158721923828,
      "activations/layer13_attention_weight_max": 41.90788269042969,
      "activations/layer13_attention_weight_min": -32.90945053100586,
      "activations/layer14_attention_weight_max": 35.45305252075195,
      "activations/layer14_attention_weight_min": -28.396190643310547,
      "activations/layer15_attention_weight_max": 32.610374450683594,
      "activations/layer15_attention_weight_min": -28.266786575317383,
      "activations/layer16_attention_weight_max": 31.583362579345703,
      "activations/layer16_attention_weight_min": -30.022851943969727,
      "activations/layer17_attention_weight_max": 48.46013259887695,
      "activations/layer17_attention_weight_min": -44.450889587402344,
      "activations/layer18_attention_weight_max": 46.225074768066406,
      "activations/layer18_attention_weight_min": -39.9446907043457,
      "activations/layer19_attention_weight_max": 24.19304847717285,
      "activations/layer19_attention_weight_min": -20.199811935424805,
      "activations/layer1_attention_weight_max": 15.474942207336426,
      "activations/layer1_attention_weight_min": -14.616325378417969,
      "activations/layer20_attention_weight_max": 23.573118209838867,
      "activations/layer20_attention_weight_min": -20.653284072875977,
      "activations/layer21_attention_weight_max": 35.16451644897461,
      "activations/layer21_attention_weight_min": -22.074100494384766,
      "activations/layer22_attention_weight_max": 30.04979133605957,
      "activations/layer22_attention_weight_min": -26.143098831176758,
      "activations/layer23_attention_weight_max": 37.102500915527344,
      "activations/layer23_attention_weight_min": -27.1458797454834,
      "activations/layer2_attention_weight_max": 32.176841735839844,
      "activations/layer2_attention_weight_min": -29.720861434936523,
      "activations/layer3_attention_weight_max": 90.47306060791016,
      "activations/layer3_attention_weight_min": -83.79444885253906,
      "activations/layer4_attention_weight_max": 88.42838287353516,
      "activations/layer4_attention_weight_min": -83.11235046386719,
      "activations/layer5_attention_weight_max": 64.68571472167969,
      "activations/layer5_attention_weight_min": -74.98694610595703,
      "activations/layer6_attention_weight_max": 49.58290100097656,
      "activations/layer6_attention_weight_min": -49.250572204589844,
      "activations/layer7_attention_weight_max": 68.23321533203125,
      "activations/layer7_attention_weight_min": -59.510894775390625,
      "activations/layer8_attention_weight_max": 47.57418441772461,
      "activations/layer8_attention_weight_min": -44.859214782714844,
      "activations/layer9_attention_weight_max": 56.25088119506836,
      "activations/layer9_attention_weight_min": -44.4101448059082,
      "epoch": 15.91,
      "learning_rate": 4.7868939393939396e-05,
      "loss": 2.7483,
      "step": 273850
    },
    {
      "activations/layer0_attention_weight_max": 15.652429580688477,
      "activations/layer0_attention_weight_min": -14.009833335876465,
      "activations/layer10_attention_weight_max": 37.997169494628906,
      "activations/layer10_attention_weight_min": -38.46091842651367,
      "activations/layer11_attention_weight_max": 33.38005065917969,
      "activations/layer11_attention_weight_min": -34.25117874145508,
      "activations/layer12_attention_weight_max": 18.898462295532227,
      "activations/layer12_attention_weight_min": -25.19196891784668,
      "activations/layer13_attention_weight_max": 45.14558029174805,
      "activations/layer13_attention_weight_min": -31.290456771850586,
      "activations/layer14_attention_weight_max": 45.46406936645508,
      "activations/layer14_attention_weight_min": -32.03367614746094,
      "activations/layer15_attention_weight_max": 41.11531448364258,
      "activations/layer15_attention_weight_min": -32.73583221435547,
      "activations/layer16_attention_weight_max": 37.23685073852539,
      "activations/layer16_attention_weight_min": -32.82294464111328,
      "activations/layer17_attention_weight_max": 60.03661346435547,
      "activations/layer17_attention_weight_min": -49.18769073486328,
      "activations/layer18_attention_weight_max": 55.16452407836914,
      "activations/layer18_attention_weight_min": -41.456180572509766,
      "activations/layer19_attention_weight_max": 27.085494995117188,
      "activations/layer19_attention_weight_min": -22.96539306640625,
      "activations/layer1_attention_weight_max": 15.357447624206543,
      "activations/layer1_attention_weight_min": -13.862913131713867,
      "activations/layer20_attention_weight_max": 25.578229904174805,
      "activations/layer20_attention_weight_min": -22.736417770385742,
      "activations/layer21_attention_weight_max": 50.5901985168457,
      "activations/layer21_attention_weight_min": -29.536056518554688,
      "activations/layer22_attention_weight_max": 32.520538330078125,
      "activations/layer22_attention_weight_min": -25.93754005432129,
      "activations/layer23_attention_weight_max": 47.77748107910156,
      "activations/layer23_attention_weight_min": -25.503276824951172,
      "activations/layer2_attention_weight_max": 32.08654022216797,
      "activations/layer2_attention_weight_min": -31.429443359375,
      "activations/layer3_attention_weight_max": 91.2795639038086,
      "activations/layer3_attention_weight_min": -93.99443817138672,
      "activations/layer4_attention_weight_max": 95.97370147705078,
      "activations/layer4_attention_weight_min": -91.71050262451172,
      "activations/layer5_attention_weight_max": 66.90522766113281,
      "activations/layer5_attention_weight_min": -72.8331527709961,
      "activations/layer6_attention_weight_max": 53.927188873291016,
      "activations/layer6_attention_weight_min": -54.563392639160156,
      "activations/layer7_attention_weight_max": 60.75402069091797,
      "activations/layer7_attention_weight_min": -64.59375762939453,
      "activations/layer8_attention_weight_max": 48.44057846069336,
      "activations/layer8_attention_weight_min": -51.40896224975586,
      "activations/layer9_attention_weight_max": 47.23958969116211,
      "activations/layer9_attention_weight_min": -50.13755416870117,
      "epoch": 15.92,
      "learning_rate": 4.785e-05,
      "loss": 2.754,
      "step": 273900
    },
    {
      "activations/layer0_attention_weight_max": 15.464357376098633,
      "activations/layer0_attention_weight_min": -13.732409477233887,
      "activations/layer10_attention_weight_max": 34.8553581237793,
      "activations/layer10_attention_weight_min": -36.9501838684082,
      "activations/layer11_attention_weight_max": 32.77164077758789,
      "activations/layer11_attention_weight_min": -36.670040130615234,
      "activations/layer12_attention_weight_max": 19.291494369506836,
      "activations/layer12_attention_weight_min": -22.977039337158203,
      "activations/layer13_attention_weight_max": 37.676918029785156,
      "activations/layer13_attention_weight_min": -29.591392517089844,
      "activations/layer14_attention_weight_max": 37.20832443237305,
      "activations/layer14_attention_weight_min": -32.23180389404297,
      "activations/layer15_attention_weight_max": 34.521427154541016,
      "activations/layer15_attention_weight_min": -30.859519958496094,
      "activations/layer16_attention_weight_max": 35.8739128112793,
      "activations/layer16_attention_weight_min": -31.23811912536621,
      "activations/layer17_attention_weight_max": 58.12617492675781,
      "activations/layer17_attention_weight_min": -45.79084396362305,
      "activations/layer18_attention_weight_max": 49.04142379760742,
      "activations/layer18_attention_weight_min": -38.005191802978516,
      "activations/layer19_attention_weight_max": 21.46678352355957,
      "activations/layer19_attention_weight_min": -22.358264923095703,
      "activations/layer1_attention_weight_max": 15.39816951751709,
      "activations/layer1_attention_weight_min": -14.803147315979004,
      "activations/layer20_attention_weight_max": 20.686368942260742,
      "activations/layer20_attention_weight_min": -22.942516326904297,
      "activations/layer21_attention_weight_max": 34.333152770996094,
      "activations/layer21_attention_weight_min": -20.504613876342773,
      "activations/layer22_attention_weight_max": 29.627880096435547,
      "activations/layer22_attention_weight_min": -25.99746322631836,
      "activations/layer23_attention_weight_max": 40.7912483215332,
      "activations/layer23_attention_weight_min": -25.7030086517334,
      "activations/layer2_attention_weight_max": 31.504676818847656,
      "activations/layer2_attention_weight_min": -28.52187156677246,
      "activations/layer3_attention_weight_max": 85.47621154785156,
      "activations/layer3_attention_weight_min": -85.21515655517578,
      "activations/layer4_attention_weight_max": 87.6311264038086,
      "activations/layer4_attention_weight_min": -83.27909088134766,
      "activations/layer5_attention_weight_max": 67.49082946777344,
      "activations/layer5_attention_weight_min": -70.25363159179688,
      "activations/layer6_attention_weight_max": 50.57243347167969,
      "activations/layer6_attention_weight_min": -50.58998107910156,
      "activations/layer7_attention_weight_max": 57.59580993652344,
      "activations/layer7_attention_weight_min": -58.84879684448242,
      "activations/layer8_attention_weight_max": 45.82514572143555,
      "activations/layer8_attention_weight_min": -51.971397399902344,
      "activations/layer9_attention_weight_max": 42.53767395019531,
      "activations/layer9_attention_weight_min": -45.65444564819336,
      "epoch": 15.92,
      "learning_rate": 4.78310606060606e-05,
      "loss": 2.7538,
      "step": 273950
    },
    {
      "activations/layer0_attention_weight_max": 15.87312126159668,
      "activations/layer0_attention_weight_min": -13.765578269958496,
      "activations/layer10_attention_weight_max": 37.641143798828125,
      "activations/layer10_attention_weight_min": -36.31292724609375,
      "activations/layer11_attention_weight_max": 33.17568588256836,
      "activations/layer11_attention_weight_min": -33.786476135253906,
      "activations/layer12_attention_weight_max": 19.512685775756836,
      "activations/layer12_attention_weight_min": -24.189374923706055,
      "activations/layer13_attention_weight_max": 38.90147018432617,
      "activations/layer13_attention_weight_min": -30.63711929321289,
      "activations/layer14_attention_weight_max": 36.564369201660156,
      "activations/layer14_attention_weight_min": -33.456214904785156,
      "activations/layer15_attention_weight_max": 35.04684066772461,
      "activations/layer15_attention_weight_min": -33.85313034057617,
      "activations/layer16_attention_weight_max": 35.14115524291992,
      "activations/layer16_attention_weight_min": -32.589900970458984,
      "activations/layer17_attention_weight_max": 55.58346939086914,
      "activations/layer17_attention_weight_min": -47.58968734741211,
      "activations/layer18_attention_weight_max": 51.40652084350586,
      "activations/layer18_attention_weight_min": -40.64143371582031,
      "activations/layer19_attention_weight_max": 24.280580520629883,
      "activations/layer19_attention_weight_min": -23.704679489135742,
      "activations/layer1_attention_weight_max": 16.481124877929688,
      "activations/layer1_attention_weight_min": -13.84687614440918,
      "activations/layer20_attention_weight_max": 23.073436737060547,
      "activations/layer20_attention_weight_min": -21.346952438354492,
      "activations/layer21_attention_weight_max": 32.75900650024414,
      "activations/layer21_attention_weight_min": -20.9180850982666,
      "activations/layer22_attention_weight_max": 29.782760620117188,
      "activations/layer22_attention_weight_min": -26.131305694580078,
      "activations/layer23_attention_weight_max": 39.37214279174805,
      "activations/layer23_attention_weight_min": -25.48332977294922,
      "activations/layer2_attention_weight_max": 29.4521484375,
      "activations/layer2_attention_weight_min": -28.812597274780273,
      "activations/layer3_attention_weight_max": 87.12290954589844,
      "activations/layer3_attention_weight_min": -86.71558380126953,
      "activations/layer4_attention_weight_max": 90.09545135498047,
      "activations/layer4_attention_weight_min": -84.1074447631836,
      "activations/layer5_attention_weight_max": 64.32915496826172,
      "activations/layer5_attention_weight_min": -71.71033477783203,
      "activations/layer6_attention_weight_max": 50.398136138916016,
      "activations/layer6_attention_weight_min": -49.13350296020508,
      "activations/layer7_attention_weight_max": 57.19896697998047,
      "activations/layer7_attention_weight_min": -59.14111328125,
      "activations/layer8_attention_weight_max": 47.79719543457031,
      "activations/layer8_attention_weight_min": -46.35979080200195,
      "activations/layer9_attention_weight_max": 43.79320526123047,
      "activations/layer9_attention_weight_min": -44.406490325927734,
      "epoch": 15.92,
      "learning_rate": 4.781212121212121e-05,
      "loss": 2.7464,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4124,
      "eval_samples_per_second": 510.439,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4124,
      "eval_openwebtext_samples_per_second": 510.439,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9082,
      "eval_wikitext_samples_per_second": 238.964,
      "step": 274000
    },
    {
      "epoch": 15.92,
      "eval_lambada_loss": 2.48828125,
      "eval_lambada_ppl": 12.040563607027826,
      "eval_lambada_runtime": 9.5149,
      "eval_lambada_samples_per_second": 511.725,
      "step": 274000
    },
    {
      "activations/layer0_attention_weight_max": 14.756416320800781,
      "activations/layer0_attention_weight_min": -13.648445129394531,
      "activations/layer10_attention_weight_max": 37.609596252441406,
      "activations/layer10_attention_weight_min": -35.14205551147461,
      "activations/layer11_attention_weight_max": 38.184654235839844,
      "activations/layer11_attention_weight_min": -35.82673645019531,
      "activations/layer12_attention_weight_max": 18.41266441345215,
      "activations/layer12_attention_weight_min": -26.64035987854004,
      "activations/layer13_attention_weight_max": 36.693206787109375,
      "activations/layer13_attention_weight_min": -28.0101261138916,
      "activations/layer14_attention_weight_max": 42.20570755004883,
      "activations/layer14_attention_weight_min": -30.966346740722656,
      "activations/layer15_attention_weight_max": 38.54053497314453,
      "activations/layer15_attention_weight_min": -30.714921951293945,
      "activations/layer16_attention_weight_max": 33.77912902832031,
      "activations/layer16_attention_weight_min": -30.21642303466797,
      "activations/layer17_attention_weight_max": 51.41047668457031,
      "activations/layer17_attention_weight_min": -46.171791076660156,
      "activations/layer18_attention_weight_max": 49.99098205566406,
      "activations/layer18_attention_weight_min": -45.15483856201172,
      "activations/layer19_attention_weight_max": 25.281564712524414,
      "activations/layer19_attention_weight_min": -24.575929641723633,
      "activations/layer1_attention_weight_max": 16.561880111694336,
      "activations/layer1_attention_weight_min": -16.849689483642578,
      "activations/layer20_attention_weight_max": 28.348949432373047,
      "activations/layer20_attention_weight_min": -23.262693405151367,
      "activations/layer21_attention_weight_max": 40.99519348144531,
      "activations/layer21_attention_weight_min": -23.83319091796875,
      "activations/layer22_attention_weight_max": 32.49540328979492,
      "activations/layer22_attention_weight_min": -26.999584197998047,
      "activations/layer23_attention_weight_max": 35.783321380615234,
      "activations/layer23_attention_weight_min": -23.74431610107422,
      "activations/layer2_attention_weight_max": 32.23925018310547,
      "activations/layer2_attention_weight_min": -30.574111938476562,
      "activations/layer3_attention_weight_max": 91.33256530761719,
      "activations/layer3_attention_weight_min": -93.61222839355469,
      "activations/layer4_attention_weight_max": 90.57440185546875,
      "activations/layer4_attention_weight_min": -91.48668670654297,
      "activations/layer5_attention_weight_max": 68.7335433959961,
      "activations/layer5_attention_weight_min": -73.65101623535156,
      "activations/layer6_attention_weight_max": 53.17211151123047,
      "activations/layer6_attention_weight_min": -54.69162368774414,
      "activations/layer7_attention_weight_max": 64.86124420166016,
      "activations/layer7_attention_weight_min": -62.83997344970703,
      "activations/layer8_attention_weight_max": 47.26399612426758,
      "activations/layer8_attention_weight_min": -50.31129837036133,
      "activations/layer9_attention_weight_max": 56.240169525146484,
      "activations/layer9_attention_weight_min": -45.545501708984375,
      "epoch": 15.92,
      "learning_rate": 4.7793181818181816e-05,
      "loss": 2.7489,
      "step": 274050
    },
    {
      "activations/layer0_attention_weight_max": 15.386652946472168,
      "activations/layer0_attention_weight_min": -13.77082347869873,
      "activations/layer10_attention_weight_max": 36.09197998046875,
      "activations/layer10_attention_weight_min": -37.07578659057617,
      "activations/layer11_attention_weight_max": 32.61957550048828,
      "activations/layer11_attention_weight_min": -34.66606903076172,
      "activations/layer12_attention_weight_max": 18.822975158691406,
      "activations/layer12_attention_weight_min": -24.577959060668945,
      "activations/layer13_attention_weight_max": 32.61394500732422,
      "activations/layer13_attention_weight_min": -29.180490493774414,
      "activations/layer14_attention_weight_max": 39.29867172241211,
      "activations/layer14_attention_weight_min": -30.457977294921875,
      "activations/layer15_attention_weight_max": 33.179962158203125,
      "activations/layer15_attention_weight_min": -29.959909439086914,
      "activations/layer16_attention_weight_max": 33.39884948730469,
      "activations/layer16_attention_weight_min": -31.06128692626953,
      "activations/layer17_attention_weight_max": 52.474334716796875,
      "activations/layer17_attention_weight_min": -45.46181869506836,
      "activations/layer18_attention_weight_max": 45.69171142578125,
      "activations/layer18_attention_weight_min": -38.0371208190918,
      "activations/layer19_attention_weight_max": 23.867568969726562,
      "activations/layer19_attention_weight_min": -23.15645980834961,
      "activations/layer1_attention_weight_max": 15.853693008422852,
      "activations/layer1_attention_weight_min": -14.385188102722168,
      "activations/layer20_attention_weight_max": 25.410444259643555,
      "activations/layer20_attention_weight_min": -21.49907112121582,
      "activations/layer21_attention_weight_max": 34.414031982421875,
      "activations/layer21_attention_weight_min": -20.114519119262695,
      "activations/layer22_attention_weight_max": 27.76128578186035,
      "activations/layer22_attention_weight_min": -26.003259658813477,
      "activations/layer23_attention_weight_max": 41.198387145996094,
      "activations/layer23_attention_weight_min": -23.799114227294922,
      "activations/layer2_attention_weight_max": 29.923202514648438,
      "activations/layer2_attention_weight_min": -29.94753646850586,
      "activations/layer3_attention_weight_max": 87.63270568847656,
      "activations/layer3_attention_weight_min": -93.98834228515625,
      "activations/layer4_attention_weight_max": 84.38337707519531,
      "activations/layer4_attention_weight_min": -89.35211181640625,
      "activations/layer5_attention_weight_max": 60.90046691894531,
      "activations/layer5_attention_weight_min": -75.56642150878906,
      "activations/layer6_attention_weight_max": 49.54356384277344,
      "activations/layer6_attention_weight_min": -49.22370147705078,
      "activations/layer7_attention_weight_max": 56.95203399658203,
      "activations/layer7_attention_weight_min": -60.90679168701172,
      "activations/layer8_attention_weight_max": 45.7739143371582,
      "activations/layer8_attention_weight_min": -49.276065826416016,
      "activations/layer9_attention_weight_max": 44.46744918823242,
      "activations/layer9_attention_weight_min": -48.37308120727539,
      "epoch": 15.93,
      "learning_rate": 4.777424242424242e-05,
      "loss": 2.7541,
      "step": 274100
    },
    {
      "activations/layer0_attention_weight_max": 15.334317207336426,
      "activations/layer0_attention_weight_min": -13.595705032348633,
      "activations/layer10_attention_weight_max": 40.51723861694336,
      "activations/layer10_attention_weight_min": -36.299842834472656,
      "activations/layer11_attention_weight_max": 34.49800109863281,
      "activations/layer11_attention_weight_min": -34.601802825927734,
      "activations/layer12_attention_weight_max": 22.505069732666016,
      "activations/layer12_attention_weight_min": -25.266448974609375,
      "activations/layer13_attention_weight_max": 34.773521423339844,
      "activations/layer13_attention_weight_min": -32.79018020629883,
      "activations/layer14_attention_weight_max": 38.916770935058594,
      "activations/layer14_attention_weight_min": -32.420249938964844,
      "activations/layer15_attention_weight_max": 34.314720153808594,
      "activations/layer15_attention_weight_min": -32.713478088378906,
      "activations/layer16_attention_weight_max": 34.77939987182617,
      "activations/layer16_attention_weight_min": -31.92522430419922,
      "activations/layer17_attention_weight_max": 53.462337493896484,
      "activations/layer17_attention_weight_min": -46.87567138671875,
      "activations/layer18_attention_weight_max": 50.852272033691406,
      "activations/layer18_attention_weight_min": -40.546104431152344,
      "activations/layer19_attention_weight_max": 24.586339950561523,
      "activations/layer19_attention_weight_min": -23.352033615112305,
      "activations/layer1_attention_weight_max": 16.347444534301758,
      "activations/layer1_attention_weight_min": -15.801066398620605,
      "activations/layer20_attention_weight_max": 25.432832717895508,
      "activations/layer20_attention_weight_min": -21.71229362487793,
      "activations/layer21_attention_weight_max": 35.94590377807617,
      "activations/layer21_attention_weight_min": -26.28531265258789,
      "activations/layer22_attention_weight_max": 28.04973602294922,
      "activations/layer22_attention_weight_min": -26.07559585571289,
      "activations/layer23_attention_weight_max": 35.38427734375,
      "activations/layer23_attention_weight_min": -23.34982681274414,
      "activations/layer2_attention_weight_max": 30.75243377685547,
      "activations/layer2_attention_weight_min": -29.032346725463867,
      "activations/layer3_attention_weight_max": 90.38886260986328,
      "activations/layer3_attention_weight_min": -90.16107940673828,
      "activations/layer4_attention_weight_max": 92.35447692871094,
      "activations/layer4_attention_weight_min": -92.39244842529297,
      "activations/layer5_attention_weight_max": 70.83116912841797,
      "activations/layer5_attention_weight_min": -72.29212951660156,
      "activations/layer6_attention_weight_max": 52.31590270996094,
      "activations/layer6_attention_weight_min": -51.323001861572266,
      "activations/layer7_attention_weight_max": 65.48486328125,
      "activations/layer7_attention_weight_min": -60.94971466064453,
      "activations/layer8_attention_weight_max": 47.714473724365234,
      "activations/layer8_attention_weight_min": -50.16459274291992,
      "activations/layer9_attention_weight_max": 45.34295654296875,
      "activations/layer9_attention_weight_min": -47.06687927246094,
      "epoch": 15.93,
      "learning_rate": 4.7755303030303025e-05,
      "loss": 2.7483,
      "step": 274150
    },
    {
      "activations/layer0_attention_weight_max": 15.535014152526855,
      "activations/layer0_attention_weight_min": -13.61244010925293,
      "activations/layer10_attention_weight_max": 37.417510986328125,
      "activations/layer10_attention_weight_min": -36.970924377441406,
      "activations/layer11_attention_weight_max": 34.34722137451172,
      "activations/layer11_attention_weight_min": -35.42121887207031,
      "activations/layer12_attention_weight_max": 19.701231002807617,
      "activations/layer12_attention_weight_min": -25.475936889648438,
      "activations/layer13_attention_weight_max": 35.60723114013672,
      "activations/layer13_attention_weight_min": -28.192745208740234,
      "activations/layer14_attention_weight_max": 37.47511291503906,
      "activations/layer14_attention_weight_min": -30.262182235717773,
      "activations/layer15_attention_weight_max": 34.847896575927734,
      "activations/layer15_attention_weight_min": -31.53066635131836,
      "activations/layer16_attention_weight_max": 35.29212188720703,
      "activations/layer16_attention_weight_min": -31.839599609375,
      "activations/layer17_attention_weight_max": 57.65299606323242,
      "activations/layer17_attention_weight_min": -48.967994689941406,
      "activations/layer18_attention_weight_max": 49.86640548706055,
      "activations/layer18_attention_weight_min": -42.50511169433594,
      "activations/layer19_attention_weight_max": 21.508827209472656,
      "activations/layer19_attention_weight_min": -22.88636016845703,
      "activations/layer1_attention_weight_max": 15.64887523651123,
      "activations/layer1_attention_weight_min": -13.723036766052246,
      "activations/layer20_attention_weight_max": 21.965951919555664,
      "activations/layer20_attention_weight_min": -22.877775192260742,
      "activations/layer21_attention_weight_max": 33.949623107910156,
      "activations/layer21_attention_weight_min": -20.627304077148438,
      "activations/layer22_attention_weight_max": 28.5028133392334,
      "activations/layer22_attention_weight_min": -25.709728240966797,
      "activations/layer23_attention_weight_max": 37.02349090576172,
      "activations/layer23_attention_weight_min": -25.020553588867188,
      "activations/layer2_attention_weight_max": 32.44160461425781,
      "activations/layer2_attention_weight_min": -29.005889892578125,
      "activations/layer3_attention_weight_max": 83.3219985961914,
      "activations/layer3_attention_weight_min": -85.20269012451172,
      "activations/layer4_attention_weight_max": 86.05540466308594,
      "activations/layer4_attention_weight_min": -87.08955383300781,
      "activations/layer5_attention_weight_max": 63.748924255371094,
      "activations/layer5_attention_weight_min": -73.46710205078125,
      "activations/layer6_attention_weight_max": 49.116214752197266,
      "activations/layer6_attention_weight_min": -52.345096588134766,
      "activations/layer7_attention_weight_max": 62.40601348876953,
      "activations/layer7_attention_weight_min": -59.658103942871094,
      "activations/layer8_attention_weight_max": 46.22429656982422,
      "activations/layer8_attention_weight_min": -50.815879821777344,
      "activations/layer9_attention_weight_max": 46.365013122558594,
      "activations/layer9_attention_weight_min": -48.70917510986328,
      "epoch": 15.93,
      "learning_rate": 4.773636363636363e-05,
      "loss": 2.7622,
      "step": 274200
    },
    {
      "activations/layer0_attention_weight_max": 16.099044799804688,
      "activations/layer0_attention_weight_min": -13.465561866760254,
      "activations/layer10_attention_weight_max": 36.1129035949707,
      "activations/layer10_attention_weight_min": -34.49755859375,
      "activations/layer11_attention_weight_max": 32.73698043823242,
      "activations/layer11_attention_weight_min": -33.35622787475586,
      "activations/layer12_attention_weight_max": 18.749670028686523,
      "activations/layer12_attention_weight_min": -22.51526641845703,
      "activations/layer13_attention_weight_max": 33.12357711791992,
      "activations/layer13_attention_weight_min": -29.873220443725586,
      "activations/layer14_attention_weight_max": 36.28473663330078,
      "activations/layer14_attention_weight_min": -33.80754852294922,
      "activations/layer15_attention_weight_max": 35.47236251831055,
      "activations/layer15_attention_weight_min": -34.19256591796875,
      "activations/layer16_attention_weight_max": 34.62744903564453,
      "activations/layer16_attention_weight_min": -32.06425094604492,
      "activations/layer17_attention_weight_max": 54.38729476928711,
      "activations/layer17_attention_weight_min": -50.690975189208984,
      "activations/layer18_attention_weight_max": 48.91318130493164,
      "activations/layer18_attention_weight_min": -45.38741683959961,
      "activations/layer19_attention_weight_max": 26.009384155273438,
      "activations/layer19_attention_weight_min": -25.469161987304688,
      "activations/layer1_attention_weight_max": 15.828591346740723,
      "activations/layer1_attention_weight_min": -13.895462036132812,
      "activations/layer20_attention_weight_max": 24.251379013061523,
      "activations/layer20_attention_weight_min": -24.643150329589844,
      "activations/layer21_attention_weight_max": 33.59280014038086,
      "activations/layer21_attention_weight_min": -23.388097763061523,
      "activations/layer22_attention_weight_max": 31.52591323852539,
      "activations/layer22_attention_weight_min": -31.54926109313965,
      "activations/layer23_attention_weight_max": 42.95814514160156,
      "activations/layer23_attention_weight_min": -25.503995895385742,
      "activations/layer2_attention_weight_max": 32.697364807128906,
      "activations/layer2_attention_weight_min": -30.396535873413086,
      "activations/layer3_attention_weight_max": 91.53250885009766,
      "activations/layer3_attention_weight_min": -92.45848846435547,
      "activations/layer4_attention_weight_max": 88.83553314208984,
      "activations/layer4_attention_weight_min": -89.95397186279297,
      "activations/layer5_attention_weight_max": 66.62662506103516,
      "activations/layer5_attention_weight_min": -76.65812683105469,
      "activations/layer6_attention_weight_max": 51.79697036743164,
      "activations/layer6_attention_weight_min": -51.03248596191406,
      "activations/layer7_attention_weight_max": 60.69271469116211,
      "activations/layer7_attention_weight_min": -61.406883239746094,
      "activations/layer8_attention_weight_max": 46.586917877197266,
      "activations/layer8_attention_weight_min": -51.3111686706543,
      "activations/layer9_attention_weight_max": 43.26423263549805,
      "activations/layer9_attention_weight_min": -47.87728500366211,
      "epoch": 15.94,
      "learning_rate": 4.771742424242424e-05,
      "loss": 2.7594,
      "step": 274250
    },
    {
      "activations/layer0_attention_weight_max": 15.921615600585938,
      "activations/layer0_attention_weight_min": -14.074316024780273,
      "activations/layer10_attention_weight_max": 36.79036331176758,
      "activations/layer10_attention_weight_min": -36.63985061645508,
      "activations/layer11_attention_weight_max": 35.01499557495117,
      "activations/layer11_attention_weight_min": -34.61781311035156,
      "activations/layer12_attention_weight_max": 20.570749282836914,
      "activations/layer12_attention_weight_min": -23.514036178588867,
      "activations/layer13_attention_weight_max": 35.61715316772461,
      "activations/layer13_attention_weight_min": -29.187280654907227,
      "activations/layer14_attention_weight_max": 37.50708770751953,
      "activations/layer14_attention_weight_min": -33.120506286621094,
      "activations/layer15_attention_weight_max": 32.96889114379883,
      "activations/layer15_attention_weight_min": -32.51411437988281,
      "activations/layer16_attention_weight_max": 34.80128479003906,
      "activations/layer16_attention_weight_min": -33.528953552246094,
      "activations/layer17_attention_weight_max": 57.6176643371582,
      "activations/layer17_attention_weight_min": -50.8823127746582,
      "activations/layer18_attention_weight_max": 47.86140060424805,
      "activations/layer18_attention_weight_min": -41.9696159362793,
      "activations/layer19_attention_weight_max": 24.132369995117188,
      "activations/layer19_attention_weight_min": -23.525766372680664,
      "activations/layer1_attention_weight_max": 16.56987762451172,
      "activations/layer1_attention_weight_min": -14.805907249450684,
      "activations/layer20_attention_weight_max": 22.144596099853516,
      "activations/layer20_attention_weight_min": -22.484209060668945,
      "activations/layer21_attention_weight_max": 37.23915481567383,
      "activations/layer21_attention_weight_min": -23.76995849609375,
      "activations/layer22_attention_weight_max": 27.51822280883789,
      "activations/layer22_attention_weight_min": -29.35839080810547,
      "activations/layer23_attention_weight_max": 35.31488037109375,
      "activations/layer23_attention_weight_min": -25.333538055419922,
      "activations/layer2_attention_weight_max": 32.86941909790039,
      "activations/layer2_attention_weight_min": -30.957077026367188,
      "activations/layer3_attention_weight_max": 89.58576965332031,
      "activations/layer3_attention_weight_min": -95.4919662475586,
      "activations/layer4_attention_weight_max": 94.33982849121094,
      "activations/layer4_attention_weight_min": -93.31246185302734,
      "activations/layer5_attention_weight_max": 65.7012939453125,
      "activations/layer5_attention_weight_min": -73.83831024169922,
      "activations/layer6_attention_weight_max": 53.19829559326172,
      "activations/layer6_attention_weight_min": -51.60935974121094,
      "activations/layer7_attention_weight_max": 62.555171966552734,
      "activations/layer7_attention_weight_min": -61.20891189575195,
      "activations/layer8_attention_weight_max": 48.3001708984375,
      "activations/layer8_attention_weight_min": -51.678714752197266,
      "activations/layer9_attention_weight_max": 45.0294075012207,
      "activations/layer9_attention_weight_min": -47.07644271850586,
      "epoch": 15.94,
      "learning_rate": 4.7698484848484843e-05,
      "loss": 2.7479,
      "step": 274300
    },
    {
      "activations/layer0_attention_weight_max": 16.17788314819336,
      "activations/layer0_attention_weight_min": -13.685802459716797,
      "activations/layer10_attention_weight_max": 35.48834228515625,
      "activations/layer10_attention_weight_min": -35.229827880859375,
      "activations/layer11_attention_weight_max": 32.409812927246094,
      "activations/layer11_attention_weight_min": -34.087310791015625,
      "activations/layer12_attention_weight_max": 18.18044662475586,
      "activations/layer12_attention_weight_min": -31.024208068847656,
      "activations/layer13_attention_weight_max": 29.81603240966797,
      "activations/layer13_attention_weight_min": -26.28574562072754,
      "activations/layer14_attention_weight_max": 33.587867736816406,
      "activations/layer14_attention_weight_min": -28.995241165161133,
      "activations/layer15_attention_weight_max": 30.006330490112305,
      "activations/layer15_attention_weight_min": -30.11105728149414,
      "activations/layer16_attention_weight_max": 29.76181983947754,
      "activations/layer16_attention_weight_min": -30.29022216796875,
      "activations/layer17_attention_weight_max": 49.0638542175293,
      "activations/layer17_attention_weight_min": -43.683528900146484,
      "activations/layer18_attention_weight_max": 39.9378776550293,
      "activations/layer18_attention_weight_min": -37.34955978393555,
      "activations/layer19_attention_weight_max": 19.854019165039062,
      "activations/layer19_attention_weight_min": -20.54366111755371,
      "activations/layer1_attention_weight_max": 15.924151420593262,
      "activations/layer1_attention_weight_min": -12.643776893615723,
      "activations/layer20_attention_weight_max": 17.850252151489258,
      "activations/layer20_attention_weight_min": -22.933122634887695,
      "activations/layer21_attention_weight_max": 30.06003189086914,
      "activations/layer21_attention_weight_min": -22.70972442626953,
      "activations/layer22_attention_weight_max": 27.601058959960938,
      "activations/layer22_attention_weight_min": -25.034685134887695,
      "activations/layer23_attention_weight_max": 33.491580963134766,
      "activations/layer23_attention_weight_min": -22.999950408935547,
      "activations/layer2_attention_weight_max": 30.611648559570312,
      "activations/layer2_attention_weight_min": -28.235456466674805,
      "activations/layer3_attention_weight_max": 83.87255096435547,
      "activations/layer3_attention_weight_min": -79.65445709228516,
      "activations/layer4_attention_weight_max": 83.64669799804688,
      "activations/layer4_attention_weight_min": -82.78105926513672,
      "activations/layer5_attention_weight_max": 62.78739547729492,
      "activations/layer5_attention_weight_min": -70.67205047607422,
      "activations/layer6_attention_weight_max": 46.10918045043945,
      "activations/layer6_attention_weight_min": -49.89579772949219,
      "activations/layer7_attention_weight_max": 60.81982421875,
      "activations/layer7_attention_weight_min": -65.43264770507812,
      "activations/layer8_attention_weight_max": 45.52558898925781,
      "activations/layer8_attention_weight_min": -47.101253509521484,
      "activations/layer9_attention_weight_max": 50.75993347167969,
      "activations/layer9_attention_weight_min": -45.79884719848633,
      "epoch": 15.94,
      "learning_rate": 4.7679545454545445e-05,
      "loss": 2.7442,
      "step": 274350
    },
    {
      "activations/layer0_attention_weight_max": 16.897275924682617,
      "activations/layer0_attention_weight_min": -13.632441520690918,
      "activations/layer10_attention_weight_max": 33.79095458984375,
      "activations/layer10_attention_weight_min": -34.86831283569336,
      "activations/layer11_attention_weight_max": 31.069555282592773,
      "activations/layer11_attention_weight_min": -35.18916320800781,
      "activations/layer12_attention_weight_max": 17.693695068359375,
      "activations/layer12_attention_weight_min": -27.60401153564453,
      "activations/layer13_attention_weight_max": 34.50415802001953,
      "activations/layer13_attention_weight_min": -27.966541290283203,
      "activations/layer14_attention_weight_max": 33.91925048828125,
      "activations/layer14_attention_weight_min": -34.37571334838867,
      "activations/layer15_attention_weight_max": 33.54268264770508,
      "activations/layer15_attention_weight_min": -32.76089096069336,
      "activations/layer16_attention_weight_max": 32.910606384277344,
      "activations/layer16_attention_weight_min": -32.500057220458984,
      "activations/layer17_attention_weight_max": 51.813140869140625,
      "activations/layer17_attention_weight_min": -46.72479248046875,
      "activations/layer18_attention_weight_max": 48.4284782409668,
      "activations/layer18_attention_weight_min": -39.827884674072266,
      "activations/layer19_attention_weight_max": 21.52265739440918,
      "activations/layer19_attention_weight_min": -21.76200294494629,
      "activations/layer1_attention_weight_max": 15.865053176879883,
      "activations/layer1_attention_weight_min": -13.7944974899292,
      "activations/layer20_attention_weight_max": 22.861995697021484,
      "activations/layer20_attention_weight_min": -20.345134735107422,
      "activations/layer21_attention_weight_max": 30.125577926635742,
      "activations/layer21_attention_weight_min": -22.523019790649414,
      "activations/layer22_attention_weight_max": 27.260597229003906,
      "activations/layer22_attention_weight_min": -27.14822006225586,
      "activations/layer23_attention_weight_max": 32.937652587890625,
      "activations/layer23_attention_weight_min": -22.656970977783203,
      "activations/layer2_attention_weight_max": 30.416807174682617,
      "activations/layer2_attention_weight_min": -29.09593963623047,
      "activations/layer3_attention_weight_max": 90.2003173828125,
      "activations/layer3_attention_weight_min": -87.58016204833984,
      "activations/layer4_attention_weight_max": 89.3908462524414,
      "activations/layer4_attention_weight_min": -85.35929107666016,
      "activations/layer5_attention_weight_max": 63.88959884643555,
      "activations/layer5_attention_weight_min": -74.09617614746094,
      "activations/layer6_attention_weight_max": 49.7035026550293,
      "activations/layer6_attention_weight_min": -50.79104995727539,
      "activations/layer7_attention_weight_max": 63.87574768066406,
      "activations/layer7_attention_weight_min": -59.93178176879883,
      "activations/layer8_attention_weight_max": 44.42707443237305,
      "activations/layer8_attention_weight_min": -49.55561065673828,
      "activations/layer9_attention_weight_max": 45.038421630859375,
      "activations/layer9_attention_weight_min": -44.75733947753906,
      "epoch": 15.94,
      "learning_rate": 4.766060606060606e-05,
      "loss": 2.7502,
      "step": 274400
    },
    {
      "activations/layer0_attention_weight_max": 16.007343292236328,
      "activations/layer0_attention_weight_min": -13.733723640441895,
      "activations/layer10_attention_weight_max": 42.591094970703125,
      "activations/layer10_attention_weight_min": -43.010711669921875,
      "activations/layer11_attention_weight_max": 43.205135345458984,
      "activations/layer11_attention_weight_min": -38.53845977783203,
      "activations/layer12_attention_weight_max": 17.691539764404297,
      "activations/layer12_attention_weight_min": -27.519365310668945,
      "activations/layer13_attention_weight_max": 34.4867057800293,
      "activations/layer13_attention_weight_min": -28.376319885253906,
      "activations/layer14_attention_weight_max": 41.027740478515625,
      "activations/layer14_attention_weight_min": -30.454179763793945,
      "activations/layer15_attention_weight_max": 41.30636215209961,
      "activations/layer15_attention_weight_min": -29.52143096923828,
      "activations/layer16_attention_weight_max": 31.408666610717773,
      "activations/layer16_attention_weight_min": -31.828153610229492,
      "activations/layer17_attention_weight_max": 51.1350212097168,
      "activations/layer17_attention_weight_min": -46.73384475708008,
      "activations/layer18_attention_weight_max": 45.4072265625,
      "activations/layer18_attention_weight_min": -42.06626892089844,
      "activations/layer19_attention_weight_max": 21.132810592651367,
      "activations/layer19_attention_weight_min": -22.05501365661621,
      "activations/layer1_attention_weight_max": 16.680177688598633,
      "activations/layer1_attention_weight_min": -14.099443435668945,
      "activations/layer20_attention_weight_max": 21.646575927734375,
      "activations/layer20_attention_weight_min": -23.71901512145996,
      "activations/layer21_attention_weight_max": 31.829771041870117,
      "activations/layer21_attention_weight_min": -22.229726791381836,
      "activations/layer22_attention_weight_max": 27.953983306884766,
      "activations/layer22_attention_weight_min": -25.705638885498047,
      "activations/layer23_attention_weight_max": 36.95515060424805,
      "activations/layer23_attention_weight_min": -23.618690490722656,
      "activations/layer2_attention_weight_max": 30.986915588378906,
      "activations/layer2_attention_weight_min": -28.03375816345215,
      "activations/layer3_attention_weight_max": 87.23625183105469,
      "activations/layer3_attention_weight_min": -88.34627532958984,
      "activations/layer4_attention_weight_max": 95.52974700927734,
      "activations/layer4_attention_weight_min": -85.48869323730469,
      "activations/layer5_attention_weight_max": 71.17463684082031,
      "activations/layer5_attention_weight_min": -77.66465759277344,
      "activations/layer6_attention_weight_max": 53.001590728759766,
      "activations/layer6_attention_weight_min": -54.241004943847656,
      "activations/layer7_attention_weight_max": 77.10968017578125,
      "activations/layer7_attention_weight_min": -63.15312576293945,
      "activations/layer8_attention_weight_max": 56.709957122802734,
      "activations/layer8_attention_weight_min": -50.93836212158203,
      "activations/layer9_attention_weight_max": 60.49216842651367,
      "activations/layer9_attention_weight_min": -50.12053680419922,
      "epoch": 15.95,
      "learning_rate": 4.764204545454545e-05,
      "loss": 2.7377,
      "step": 274450
    },
    {
      "activations/layer0_attention_weight_max": 16.140623092651367,
      "activations/layer0_attention_weight_min": -13.70962142944336,
      "activations/layer10_attention_weight_max": 34.31500244140625,
      "activations/layer10_attention_weight_min": -35.83223342895508,
      "activations/layer11_attention_weight_max": 33.029029846191406,
      "activations/layer11_attention_weight_min": -34.0395622253418,
      "activations/layer12_attention_weight_max": 19.356149673461914,
      "activations/layer12_attention_weight_min": -27.313796997070312,
      "activations/layer13_attention_weight_max": 35.13536071777344,
      "activations/layer13_attention_weight_min": -28.626989364624023,
      "activations/layer14_attention_weight_max": 32.842533111572266,
      "activations/layer14_attention_weight_min": -29.843612670898438,
      "activations/layer15_attention_weight_max": 30.660930633544922,
      "activations/layer15_attention_weight_min": -29.962322235107422,
      "activations/layer16_attention_weight_max": 32.63625717163086,
      "activations/layer16_attention_weight_min": -31.320911407470703,
      "activations/layer17_attention_weight_max": 50.023040771484375,
      "activations/layer17_attention_weight_min": -44.55109405517578,
      "activations/layer18_attention_weight_max": 44.345680236816406,
      "activations/layer18_attention_weight_min": -38.80085372924805,
      "activations/layer19_attention_weight_max": 24.352949142456055,
      "activations/layer19_attention_weight_min": -23.720746994018555,
      "activations/layer1_attention_weight_max": 16.440061569213867,
      "activations/layer1_attention_weight_min": -14.697619438171387,
      "activations/layer20_attention_weight_max": 24.94091796875,
      "activations/layer20_attention_weight_min": -22.199663162231445,
      "activations/layer21_attention_weight_max": 37.06239700317383,
      "activations/layer21_attention_weight_min": -22.75721549987793,
      "activations/layer22_attention_weight_max": 27.495820999145508,
      "activations/layer22_attention_weight_min": -28.59589385986328,
      "activations/layer23_attention_weight_max": 35.09771728515625,
      "activations/layer23_attention_weight_min": -25.166662216186523,
      "activations/layer2_attention_weight_max": 33.38951873779297,
      "activations/layer2_attention_weight_min": -30.270484924316406,
      "activations/layer3_attention_weight_max": 87.877685546875,
      "activations/layer3_attention_weight_min": -89.48458862304688,
      "activations/layer4_attention_weight_max": 91.13380432128906,
      "activations/layer4_attention_weight_min": -89.65325927734375,
      "activations/layer5_attention_weight_max": 61.707550048828125,
      "activations/layer5_attention_weight_min": -67.93072509765625,
      "activations/layer6_attention_weight_max": 50.402976989746094,
      "activations/layer6_attention_weight_min": -49.22473907470703,
      "activations/layer7_attention_weight_max": 62.23522186279297,
      "activations/layer7_attention_weight_min": -62.15840530395508,
      "activations/layer8_attention_weight_max": 46.1373405456543,
      "activations/layer8_attention_weight_min": -47.287879943847656,
      "activations/layer9_attention_weight_max": 48.211814880371094,
      "activations/layer9_attention_weight_min": -47.124820709228516,
      "epoch": 15.95,
      "learning_rate": 4.762310606060606e-05,
      "loss": 2.752,
      "step": 274500
    },
    {
      "activations/layer0_attention_weight_max": 16.290515899658203,
      "activations/layer0_attention_weight_min": -14.031024932861328,
      "activations/layer10_attention_weight_max": 40.24503707885742,
      "activations/layer10_attention_weight_min": -38.80189514160156,
      "activations/layer11_attention_weight_max": 36.27783966064453,
      "activations/layer11_attention_weight_min": -33.9407958984375,
      "activations/layer12_attention_weight_max": 23.888059616088867,
      "activations/layer12_attention_weight_min": -24.115873336791992,
      "activations/layer13_attention_weight_max": 37.940338134765625,
      "activations/layer13_attention_weight_min": -31.599807739257812,
      "activations/layer14_attention_weight_max": 39.43763732910156,
      "activations/layer14_attention_weight_min": -29.948848724365234,
      "activations/layer15_attention_weight_max": 34.315982818603516,
      "activations/layer15_attention_weight_min": -29.231645584106445,
      "activations/layer16_attention_weight_max": 34.609867095947266,
      "activations/layer16_attention_weight_min": -30.981563568115234,
      "activations/layer17_attention_weight_max": 55.685123443603516,
      "activations/layer17_attention_weight_min": -46.0681266784668,
      "activations/layer18_attention_weight_max": 45.57960510253906,
      "activations/layer18_attention_weight_min": -38.08849334716797,
      "activations/layer19_attention_weight_max": 23.127798080444336,
      "activations/layer19_attention_weight_min": -22.80892562866211,
      "activations/layer1_attention_weight_max": 16.3021240234375,
      "activations/layer1_attention_weight_min": -13.27807331085205,
      "activations/layer20_attention_weight_max": 21.606876373291016,
      "activations/layer20_attention_weight_min": -22.816587448120117,
      "activations/layer21_attention_weight_max": 32.104488372802734,
      "activations/layer21_attention_weight_min": -22.4565372467041,
      "activations/layer22_attention_weight_max": 27.86894416809082,
      "activations/layer22_attention_weight_min": -26.10521697998047,
      "activations/layer23_attention_weight_max": 32.93010711669922,
      "activations/layer23_attention_weight_min": -22.628982543945312,
      "activations/layer2_attention_weight_max": 31.934673309326172,
      "activations/layer2_attention_weight_min": -29.909008026123047,
      "activations/layer3_attention_weight_max": 86.91111755371094,
      "activations/layer3_attention_weight_min": -97.04439544677734,
      "activations/layer4_attention_weight_max": 94.51268768310547,
      "activations/layer4_attention_weight_min": -90.65333557128906,
      "activations/layer5_attention_weight_max": 64.35419464111328,
      "activations/layer5_attention_weight_min": -73.5186538696289,
      "activations/layer6_attention_weight_max": 50.81006622314453,
      "activations/layer6_attention_weight_min": -50.37431335449219,
      "activations/layer7_attention_weight_max": 62.43837356567383,
      "activations/layer7_attention_weight_min": -60.20153045654297,
      "activations/layer8_attention_weight_max": 48.68791580200195,
      "activations/layer8_attention_weight_min": -47.217933654785156,
      "activations/layer9_attention_weight_max": 49.05902099609375,
      "activations/layer9_attention_weight_min": -46.14463424682617,
      "epoch": 15.95,
      "learning_rate": 4.760454545454545e-05,
      "loss": 2.7452,
      "step": 274550
    },
    {
      "activations/layer0_attention_weight_max": 18.193870544433594,
      "activations/layer0_attention_weight_min": -14.273721694946289,
      "activations/layer10_attention_weight_max": 33.297359466552734,
      "activations/layer10_attention_weight_min": -34.705562591552734,
      "activations/layer11_attention_weight_max": 31.222272872924805,
      "activations/layer11_attention_weight_min": -34.449893951416016,
      "activations/layer12_attention_weight_max": 20.321277618408203,
      "activations/layer12_attention_weight_min": -22.887691497802734,
      "activations/layer13_attention_weight_max": 31.88553237915039,
      "activations/layer13_attention_weight_min": -27.079565048217773,
      "activations/layer14_attention_weight_max": 37.67639923095703,
      "activations/layer14_attention_weight_min": -29.85822868347168,
      "activations/layer15_attention_weight_max": 34.268333435058594,
      "activations/layer15_attention_weight_min": -30.54625701904297,
      "activations/layer16_attention_weight_max": 33.40478515625,
      "activations/layer16_attention_weight_min": -30.842639923095703,
      "activations/layer17_attention_weight_max": 53.0042610168457,
      "activations/layer17_attention_weight_min": -44.2497673034668,
      "activations/layer18_attention_weight_max": 45.40248489379883,
      "activations/layer18_attention_weight_min": -39.767822265625,
      "activations/layer19_attention_weight_max": 25.81208610534668,
      "activations/layer19_attention_weight_min": -22.94676971435547,
      "activations/layer1_attention_weight_max": 16.09175682067871,
      "activations/layer1_attention_weight_min": -14.724586486816406,
      "activations/layer20_attention_weight_max": 25.260326385498047,
      "activations/layer20_attention_weight_min": -25.200637817382812,
      "activations/layer21_attention_weight_max": 36.793800354003906,
      "activations/layer21_attention_weight_min": -23.809539794921875,
      "activations/layer22_attention_weight_max": 30.111530303955078,
      "activations/layer22_attention_weight_min": -29.149934768676758,
      "activations/layer23_attention_weight_max": 42.721527099609375,
      "activations/layer23_attention_weight_min": -26.61873435974121,
      "activations/layer2_attention_weight_max": 31.45376968383789,
      "activations/layer2_attention_weight_min": -29.331050872802734,
      "activations/layer3_attention_weight_max": 81.82837677001953,
      "activations/layer3_attention_weight_min": -79.8941421508789,
      "activations/layer4_attention_weight_max": 80.40362548828125,
      "activations/layer4_attention_weight_min": -81.70891571044922,
      "activations/layer5_attention_weight_max": 58.76886749267578,
      "activations/layer5_attention_weight_min": -68.16070556640625,
      "activations/layer6_attention_weight_max": 46.941654205322266,
      "activations/layer6_attention_weight_min": -49.153568267822266,
      "activations/layer7_attention_weight_max": 56.225807189941406,
      "activations/layer7_attention_weight_min": -58.626426696777344,
      "activations/layer8_attention_weight_max": 42.75529479980469,
      "activations/layer8_attention_weight_min": -46.5568962097168,
      "activations/layer9_attention_weight_max": 44.44623565673828,
      "activations/layer9_attention_weight_min": -45.06535720825195,
      "epoch": 15.96,
      "learning_rate": 4.758560606060606e-05,
      "loss": 2.7475,
      "step": 274600
    },
    {
      "activations/layer0_attention_weight_max": 15.30131721496582,
      "activations/layer0_attention_weight_min": -14.097521781921387,
      "activations/layer10_attention_weight_max": 35.29013442993164,
      "activations/layer10_attention_weight_min": -36.564064025878906,
      "activations/layer11_attention_weight_max": 31.705322265625,
      "activations/layer11_attention_weight_min": -33.87126922607422,
      "activations/layer12_attention_weight_max": 18.927766799926758,
      "activations/layer12_attention_weight_min": -27.322912216186523,
      "activations/layer13_attention_weight_max": 30.884706497192383,
      "activations/layer13_attention_weight_min": -28.58734893798828,
      "activations/layer14_attention_weight_max": 34.626007080078125,
      "activations/layer14_attention_weight_min": -30.14899253845215,
      "activations/layer15_attention_weight_max": 31.79572296142578,
      "activations/layer15_attention_weight_min": -30.280006408691406,
      "activations/layer16_attention_weight_max": 35.17390823364258,
      "activations/layer16_attention_weight_min": -32.35335159301758,
      "activations/layer17_attention_weight_max": 50.833473205566406,
      "activations/layer17_attention_weight_min": -44.58041763305664,
      "activations/layer18_attention_weight_max": 47.820098876953125,
      "activations/layer18_attention_weight_min": -40.32735824584961,
      "activations/layer19_attention_weight_max": 21.86393928527832,
      "activations/layer19_attention_weight_min": -22.600492477416992,
      "activations/layer1_attention_weight_max": 17.250566482543945,
      "activations/layer1_attention_weight_min": -18.233638763427734,
      "activations/layer20_attention_weight_max": 19.25929069519043,
      "activations/layer20_attention_weight_min": -27.4686222076416,
      "activations/layer21_attention_weight_max": 34.8421516418457,
      "activations/layer21_attention_weight_min": -20.206937789916992,
      "activations/layer22_attention_weight_max": 27.1868896484375,
      "activations/layer22_attention_weight_min": -25.675392150878906,
      "activations/layer23_attention_weight_max": 32.684730529785156,
      "activations/layer23_attention_weight_min": -25.020675659179688,
      "activations/layer2_attention_weight_max": 30.34430694580078,
      "activations/layer2_attention_weight_min": -29.926742553710938,
      "activations/layer3_attention_weight_max": 88.91168212890625,
      "activations/layer3_attention_weight_min": -92.0152359008789,
      "activations/layer4_attention_weight_max": 90.89543914794922,
      "activations/layer4_attention_weight_min": -91.01709747314453,
      "activations/layer5_attention_weight_max": 64.88137817382812,
      "activations/layer5_attention_weight_min": -74.85979461669922,
      "activations/layer6_attention_weight_max": 53.71354675292969,
      "activations/layer6_attention_weight_min": -50.419185638427734,
      "activations/layer7_attention_weight_max": 62.89759063720703,
      "activations/layer7_attention_weight_min": -59.80411911010742,
      "activations/layer8_attention_weight_max": 47.28889083862305,
      "activations/layer8_attention_weight_min": -52.25071334838867,
      "activations/layer9_attention_weight_max": 46.04634094238281,
      "activations/layer9_attention_weight_min": -46.25518035888672,
      "epoch": 15.96,
      "learning_rate": 4.7566666666666666e-05,
      "loss": 2.7678,
      "step": 274650
    },
    {
      "activations/layer0_attention_weight_max": 15.941163063049316,
      "activations/layer0_attention_weight_min": -14.156978607177734,
      "activations/layer10_attention_weight_max": 35.7678108215332,
      "activations/layer10_attention_weight_min": -34.51640701293945,
      "activations/layer11_attention_weight_max": 32.09046936035156,
      "activations/layer11_attention_weight_min": -33.184326171875,
      "activations/layer12_attention_weight_max": 19.1151065826416,
      "activations/layer12_attention_weight_min": -24.36515235900879,
      "activations/layer13_attention_weight_max": 37.16506576538086,
      "activations/layer13_attention_weight_min": -27.014545440673828,
      "activations/layer14_attention_weight_max": 36.911582946777344,
      "activations/layer14_attention_weight_min": -30.07541847229004,
      "activations/layer15_attention_weight_max": 33.983848571777344,
      "activations/layer15_attention_weight_min": -28.22894859313965,
      "activations/layer16_attention_weight_max": 35.82984924316406,
      "activations/layer16_attention_weight_min": -31.198974609375,
      "activations/layer17_attention_weight_max": 56.03996658325195,
      "activations/layer17_attention_weight_min": -44.242130279541016,
      "activations/layer18_attention_weight_max": 48.99315643310547,
      "activations/layer18_attention_weight_min": -38.99476623535156,
      "activations/layer19_attention_weight_max": 23.218856811523438,
      "activations/layer19_attention_weight_min": -22.277042388916016,
      "activations/layer1_attention_weight_max": 15.640238761901855,
      "activations/layer1_attention_weight_min": -14.763569831848145,
      "activations/layer20_attention_weight_max": 23.845319747924805,
      "activations/layer20_attention_weight_min": -21.361940383911133,
      "activations/layer21_attention_weight_max": 36.867591857910156,
      "activations/layer21_attention_weight_min": -21.390085220336914,
      "activations/layer22_attention_weight_max": 27.447059631347656,
      "activations/layer22_attention_weight_min": -27.169513702392578,
      "activations/layer23_attention_weight_max": 34.97319030761719,
      "activations/layer23_attention_weight_min": -23.965599060058594,
      "activations/layer2_attention_weight_max": 30.193639755249023,
      "activations/layer2_attention_weight_min": -29.582767486572266,
      "activations/layer3_attention_weight_max": 90.94255828857422,
      "activations/layer3_attention_weight_min": -88.04991149902344,
      "activations/layer4_attention_weight_max": 87.78936004638672,
      "activations/layer4_attention_weight_min": -88.0256576538086,
      "activations/layer5_attention_weight_max": 63.56731414794922,
      "activations/layer5_attention_weight_min": -72.79438781738281,
      "activations/layer6_attention_weight_max": 49.46922302246094,
      "activations/layer6_attention_weight_min": -50.353736877441406,
      "activations/layer7_attention_weight_max": 60.487815856933594,
      "activations/layer7_attention_weight_min": -58.95625686645508,
      "activations/layer8_attention_weight_max": 47.06739044189453,
      "activations/layer8_attention_weight_min": -47.0262336730957,
      "activations/layer9_attention_weight_max": 42.3034553527832,
      "activations/layer9_attention_weight_min": -44.27964401245117,
      "epoch": 15.96,
      "learning_rate": 4.754772727272727e-05,
      "loss": 2.7488,
      "step": 274700
    },
    {
      "activations/layer0_attention_weight_max": 15.326123237609863,
      "activations/layer0_attention_weight_min": -14.150094985961914,
      "activations/layer10_attention_weight_max": 36.62233352661133,
      "activations/layer10_attention_weight_min": -36.74159622192383,
      "activations/layer11_attention_weight_max": 37.063785552978516,
      "activations/layer11_attention_weight_min": -34.05659484863281,
      "activations/layer12_attention_weight_max": 19.325937271118164,
      "activations/layer12_attention_weight_min": -27.889423370361328,
      "activations/layer13_attention_weight_max": 34.86099624633789,
      "activations/layer13_attention_weight_min": -26.35943031311035,
      "activations/layer14_attention_weight_max": 38.96049118041992,
      "activations/layer14_attention_weight_min": -30.7554988861084,
      "activations/layer15_attention_weight_max": 34.92751693725586,
      "activations/layer15_attention_weight_min": -29.74070930480957,
      "activations/layer16_attention_weight_max": 36.013370513916016,
      "activations/layer16_attention_weight_min": -32.012733459472656,
      "activations/layer17_attention_weight_max": 54.78053283691406,
      "activations/layer17_attention_weight_min": -46.144344329833984,
      "activations/layer18_attention_weight_max": 47.28801727294922,
      "activations/layer18_attention_weight_min": -40.761077880859375,
      "activations/layer19_attention_weight_max": 23.617202758789062,
      "activations/layer19_attention_weight_min": -26.09754753112793,
      "activations/layer1_attention_weight_max": 15.73983097076416,
      "activations/layer1_attention_weight_min": -13.794265747070312,
      "activations/layer20_attention_weight_max": 23.511808395385742,
      "activations/layer20_attention_weight_min": -25.047441482543945,
      "activations/layer21_attention_weight_max": 41.076900482177734,
      "activations/layer21_attention_weight_min": -25.43320655822754,
      "activations/layer22_attention_weight_max": 32.96306228637695,
      "activations/layer22_attention_weight_min": -27.56352996826172,
      "activations/layer23_attention_weight_max": 35.34027099609375,
      "activations/layer23_attention_weight_min": -25.585437774658203,
      "activations/layer2_attention_weight_max": 31.52521514892578,
      "activations/layer2_attention_weight_min": -29.94655990600586,
      "activations/layer3_attention_weight_max": 89.45442962646484,
      "activations/layer3_attention_weight_min": -89.95057678222656,
      "activations/layer4_attention_weight_max": 90.81700134277344,
      "activations/layer4_attention_weight_min": -87.03511047363281,
      "activations/layer5_attention_weight_max": 64.72532653808594,
      "activations/layer5_attention_weight_min": -72.62677001953125,
      "activations/layer6_attention_weight_max": 50.69332504272461,
      "activations/layer6_attention_weight_min": -50.2859001159668,
      "activations/layer7_attention_weight_max": 61.37187576293945,
      "activations/layer7_attention_weight_min": -60.26198959350586,
      "activations/layer8_attention_weight_max": 46.13357162475586,
      "activations/layer8_attention_weight_min": -50.90330123901367,
      "activations/layer9_attention_weight_max": 46.54777526855469,
      "activations/layer9_attention_weight_min": -47.98347091674805,
      "epoch": 15.96,
      "learning_rate": 4.752878787878787e-05,
      "loss": 2.7597,
      "step": 274750
    },
    {
      "activations/layer0_attention_weight_max": 15.986995697021484,
      "activations/layer0_attention_weight_min": -13.954880714416504,
      "activations/layer10_attention_weight_max": 36.29508972167969,
      "activations/layer10_attention_weight_min": -33.69461441040039,
      "activations/layer11_attention_weight_max": 32.952880859375,
      "activations/layer11_attention_weight_min": -31.048208236694336,
      "activations/layer12_attention_weight_max": 16.612558364868164,
      "activations/layer12_attention_weight_min": -25.36753273010254,
      "activations/layer13_attention_weight_max": 35.56107711791992,
      "activations/layer13_attention_weight_min": -27.85079574584961,
      "activations/layer14_attention_weight_max": 33.24938201904297,
      "activations/layer14_attention_weight_min": -31.4472599029541,
      "activations/layer15_attention_weight_max": 32.235595703125,
      "activations/layer15_attention_weight_min": -29.993301391601562,
      "activations/layer16_attention_weight_max": 33.30827331542969,
      "activations/layer16_attention_weight_min": -32.69264602661133,
      "activations/layer17_attention_weight_max": 51.45441818237305,
      "activations/layer17_attention_weight_min": -44.593971252441406,
      "activations/layer18_attention_weight_max": 44.39619064331055,
      "activations/layer18_attention_weight_min": -40.815486907958984,
      "activations/layer19_attention_weight_max": 25.517976760864258,
      "activations/layer19_attention_weight_min": -23.292205810546875,
      "activations/layer1_attention_weight_max": 16.855077743530273,
      "activations/layer1_attention_weight_min": -14.012276649475098,
      "activations/layer20_attention_weight_max": 21.96466827392578,
      "activations/layer20_attention_weight_min": -19.87761878967285,
      "activations/layer21_attention_weight_max": 34.49251174926758,
      "activations/layer21_attention_weight_min": -22.243192672729492,
      "activations/layer22_attention_weight_max": 25.429059982299805,
      "activations/layer22_attention_weight_min": -25.79818344116211,
      "activations/layer23_attention_weight_max": 32.1552734375,
      "activations/layer23_attention_weight_min": -24.467193603515625,
      "activations/layer2_attention_weight_max": 30.60053253173828,
      "activations/layer2_attention_weight_min": -28.604522705078125,
      "activations/layer3_attention_weight_max": 89.2793960571289,
      "activations/layer3_attention_weight_min": -87.24462890625,
      "activations/layer4_attention_weight_max": 89.9592514038086,
      "activations/layer4_attention_weight_min": -87.13966369628906,
      "activations/layer5_attention_weight_max": 64.4813461303711,
      "activations/layer5_attention_weight_min": -76.61407470703125,
      "activations/layer6_attention_weight_max": 49.17829513549805,
      "activations/layer6_attention_weight_min": -50.620304107666016,
      "activations/layer7_attention_weight_max": 57.5420036315918,
      "activations/layer7_attention_weight_min": -63.54641342163086,
      "activations/layer8_attention_weight_max": 47.29590606689453,
      "activations/layer8_attention_weight_min": -47.803985595703125,
      "activations/layer9_attention_weight_max": 42.92306137084961,
      "activations/layer9_attention_weight_min": -45.32309341430664,
      "epoch": 15.97,
      "learning_rate": 4.7509848484848484e-05,
      "loss": 2.7651,
      "step": 274800
    },
    {
      "activations/layer0_attention_weight_max": 15.273042678833008,
      "activations/layer0_attention_weight_min": -14.021464347839355,
      "activations/layer10_attention_weight_max": 35.996246337890625,
      "activations/layer10_attention_weight_min": -37.824649810791016,
      "activations/layer11_attention_weight_max": 33.32707977294922,
      "activations/layer11_attention_weight_min": -34.245811462402344,
      "activations/layer12_attention_weight_max": 19.834827423095703,
      "activations/layer12_attention_weight_min": -25.688241958618164,
      "activations/layer13_attention_weight_max": 37.089683532714844,
      "activations/layer13_attention_weight_min": -28.945880889892578,
      "activations/layer14_attention_weight_max": 33.05735397338867,
      "activations/layer14_attention_weight_min": -31.10088348388672,
      "activations/layer15_attention_weight_max": 29.784582138061523,
      "activations/layer15_attention_weight_min": -30.260459899902344,
      "activations/layer16_attention_weight_max": 31.735139846801758,
      "activations/layer16_attention_weight_min": -30.789005279541016,
      "activations/layer17_attention_weight_max": 48.27384567260742,
      "activations/layer17_attention_weight_min": -44.99495315551758,
      "activations/layer18_attention_weight_max": 43.8858757019043,
      "activations/layer18_attention_weight_min": -41.46231460571289,
      "activations/layer19_attention_weight_max": 23.034351348876953,
      "activations/layer19_attention_weight_min": -22.97608184814453,
      "activations/layer1_attention_weight_max": 16.17441749572754,
      "activations/layer1_attention_weight_min": -14.911809921264648,
      "activations/layer20_attention_weight_max": 21.564098358154297,
      "activations/layer20_attention_weight_min": -20.932037353515625,
      "activations/layer21_attention_weight_max": 31.026626586914062,
      "activations/layer21_attention_weight_min": -20.683794021606445,
      "activations/layer22_attention_weight_max": 29.05830955505371,
      "activations/layer22_attention_weight_min": -25.162702560424805,
      "activations/layer23_attention_weight_max": 39.46598815917969,
      "activations/layer23_attention_weight_min": -22.610349655151367,
      "activations/layer2_attention_weight_max": 31.936534881591797,
      "activations/layer2_attention_weight_min": -28.82473373413086,
      "activations/layer3_attention_weight_max": 81.98263549804688,
      "activations/layer3_attention_weight_min": -83.55347442626953,
      "activations/layer4_attention_weight_max": 88.62064361572266,
      "activations/layer4_attention_weight_min": -86.5212631225586,
      "activations/layer5_attention_weight_max": 61.819793701171875,
      "activations/layer5_attention_weight_min": -73.14071655273438,
      "activations/layer6_attention_weight_max": 48.67942428588867,
      "activations/layer6_attention_weight_min": -51.00653839111328,
      "activations/layer7_attention_weight_max": 61.84363555908203,
      "activations/layer7_attention_weight_min": -65.14716339111328,
      "activations/layer8_attention_weight_max": 45.98923873901367,
      "activations/layer8_attention_weight_min": -50.28484344482422,
      "activations/layer9_attention_weight_max": 45.30475997924805,
      "activations/layer9_attention_weight_min": -48.23446273803711,
      "epoch": 15.97,
      "learning_rate": 4.7490909090909086e-05,
      "loss": 2.762,
      "step": 274850
    },
    {
      "activations/layer0_attention_weight_max": 16.289676666259766,
      "activations/layer0_attention_weight_min": -13.734992980957031,
      "activations/layer10_attention_weight_max": 35.29449462890625,
      "activations/layer10_attention_weight_min": -37.0795783996582,
      "activations/layer11_attention_weight_max": 31.165538787841797,
      "activations/layer11_attention_weight_min": -32.30415344238281,
      "activations/layer12_attention_weight_max": 21.102035522460938,
      "activations/layer12_attention_weight_min": -27.70117950439453,
      "activations/layer13_attention_weight_max": 36.54810333251953,
      "activations/layer13_attention_weight_min": -28.674480438232422,
      "activations/layer14_attention_weight_max": 36.55341339111328,
      "activations/layer14_attention_weight_min": -32.6636848449707,
      "activations/layer15_attention_weight_max": 32.920562744140625,
      "activations/layer15_attention_weight_min": -30.68814468383789,
      "activations/layer16_attention_weight_max": 31.780858993530273,
      "activations/layer16_attention_weight_min": -32.4469108581543,
      "activations/layer17_attention_weight_max": 48.59946060180664,
      "activations/layer17_attention_weight_min": -45.383949279785156,
      "activations/layer18_attention_weight_max": 44.20004653930664,
      "activations/layer18_attention_weight_min": -39.957862854003906,
      "activations/layer19_attention_weight_max": 23.2092342376709,
      "activations/layer19_attention_weight_min": -24.52620506286621,
      "activations/layer1_attention_weight_max": 16.28036880493164,
      "activations/layer1_attention_weight_min": -13.102089881896973,
      "activations/layer20_attention_weight_max": 23.69719886779785,
      "activations/layer20_attention_weight_min": -23.378894805908203,
      "activations/layer21_attention_weight_max": 38.22632598876953,
      "activations/layer21_attention_weight_min": -24.67525291442871,
      "activations/layer22_attention_weight_max": 28.82874298095703,
      "activations/layer22_attention_weight_min": -27.38068199157715,
      "activations/layer23_attention_weight_max": 34.35329055786133,
      "activations/layer23_attention_weight_min": -23.418540954589844,
      "activations/layer2_attention_weight_max": 29.346464157104492,
      "activations/layer2_attention_weight_min": -29.618431091308594,
      "activations/layer3_attention_weight_max": 86.45146942138672,
      "activations/layer3_attention_weight_min": -85.01837158203125,
      "activations/layer4_attention_weight_max": 88.21326446533203,
      "activations/layer4_attention_weight_min": -82.67382049560547,
      "activations/layer5_attention_weight_max": 62.3066520690918,
      "activations/layer5_attention_weight_min": -69.39875793457031,
      "activations/layer6_attention_weight_max": 51.0946159362793,
      "activations/layer6_attention_weight_min": -50.9913444519043,
      "activations/layer7_attention_weight_max": 57.567134857177734,
      "activations/layer7_attention_weight_min": -60.49238204956055,
      "activations/layer8_attention_weight_max": 45.7369384765625,
      "activations/layer8_attention_weight_min": -48.77300262451172,
      "activations/layer9_attention_weight_max": 43.219852447509766,
      "activations/layer9_attention_weight_min": -47.36360549926758,
      "epoch": 15.97,
      "learning_rate": 4.7471969696969694e-05,
      "loss": 2.7638,
      "step": 274900
    },
    {
      "activations/layer0_attention_weight_max": 15.47924518585205,
      "activations/layer0_attention_weight_min": -14.170717239379883,
      "activations/layer10_attention_weight_max": 40.61439514160156,
      "activations/layer10_attention_weight_min": -38.65407180786133,
      "activations/layer11_attention_weight_max": 35.68310546875,
      "activations/layer11_attention_weight_min": -35.849761962890625,
      "activations/layer12_attention_weight_max": 19.387680053710938,
      "activations/layer12_attention_weight_min": -25.870223999023438,
      "activations/layer13_attention_weight_max": 39.281795501708984,
      "activations/layer13_attention_weight_min": -28.29213523864746,
      "activations/layer14_attention_weight_max": 37.198692321777344,
      "activations/layer14_attention_weight_min": -31.65538787841797,
      "activations/layer15_attention_weight_max": 32.857234954833984,
      "activations/layer15_attention_weight_min": -31.3714656829834,
      "activations/layer16_attention_weight_max": 36.182212829589844,
      "activations/layer16_attention_weight_min": -35.43890380859375,
      "activations/layer17_attention_weight_max": 60.28628921508789,
      "activations/layer17_attention_weight_min": -47.76223373413086,
      "activations/layer18_attention_weight_max": 51.84999465942383,
      "activations/layer18_attention_weight_min": -42.87763595581055,
      "activations/layer19_attention_weight_max": 24.230985641479492,
      "activations/layer19_attention_weight_min": -23.69835090637207,
      "activations/layer1_attention_weight_max": 15.68422794342041,
      "activations/layer1_attention_weight_min": -14.029194831848145,
      "activations/layer20_attention_weight_max": 25.23332977294922,
      "activations/layer20_attention_weight_min": -21.784343719482422,
      "activations/layer21_attention_weight_max": 39.5692253112793,
      "activations/layer21_attention_weight_min": -24.660547256469727,
      "activations/layer22_attention_weight_max": 30.112485885620117,
      "activations/layer22_attention_weight_min": -25.451257705688477,
      "activations/layer23_attention_weight_max": 39.724098205566406,
      "activations/layer23_attention_weight_min": -23.84979820251465,
      "activations/layer2_attention_weight_max": 30.03987693786621,
      "activations/layer2_attention_weight_min": -29.5518856048584,
      "activations/layer3_attention_weight_max": 88.50226593017578,
      "activations/layer3_attention_weight_min": -87.09285736083984,
      "activations/layer4_attention_weight_max": 91.94606018066406,
      "activations/layer4_attention_weight_min": -92.29582977294922,
      "activations/layer5_attention_weight_max": 63.71887969970703,
      "activations/layer5_attention_weight_min": -75.83959197998047,
      "activations/layer6_attention_weight_max": 53.775882720947266,
      "activations/layer6_attention_weight_min": -53.73851776123047,
      "activations/layer7_attention_weight_max": 61.938507080078125,
      "activations/layer7_attention_weight_min": -63.29472732543945,
      "activations/layer8_attention_weight_max": 47.179237365722656,
      "activations/layer8_attention_weight_min": -52.57524490356445,
      "activations/layer9_attention_weight_max": 49.0018424987793,
      "activations/layer9_attention_weight_min": -49.90614700317383,
      "epoch": 15.98,
      "learning_rate": 4.7453030303030296e-05,
      "loss": 2.7632,
      "step": 274950
    },
    {
      "activations/layer0_attention_weight_max": 16.585302352905273,
      "activations/layer0_attention_weight_min": -14.095056533813477,
      "activations/layer10_attention_weight_max": 35.553680419921875,
      "activations/layer10_attention_weight_min": -35.476078033447266,
      "activations/layer11_attention_weight_max": 31.46558952331543,
      "activations/layer11_attention_weight_min": -32.790714263916016,
      "activations/layer12_attention_weight_max": 18.31113624572754,
      "activations/layer12_attention_weight_min": -24.33020782470703,
      "activations/layer13_attention_weight_max": 31.198684692382812,
      "activations/layer13_attention_weight_min": -25.95626449584961,
      "activations/layer14_attention_weight_max": 33.80169677734375,
      "activations/layer14_attention_weight_min": -28.3106689453125,
      "activations/layer15_attention_weight_max": 28.446300506591797,
      "activations/layer15_attention_weight_min": -28.174230575561523,
      "activations/layer16_attention_weight_max": 29.27212142944336,
      "activations/layer16_attention_weight_min": -29.44651985168457,
      "activations/layer17_attention_weight_max": 47.31584548950195,
      "activations/layer17_attention_weight_min": -40.46846008300781,
      "activations/layer18_attention_weight_max": 42.58208084106445,
      "activations/layer18_attention_weight_min": -37.0859489440918,
      "activations/layer19_attention_weight_max": 19.753705978393555,
      "activations/layer19_attention_weight_min": -21.434741973876953,
      "activations/layer1_attention_weight_max": 15.923619270324707,
      "activations/layer1_attention_weight_min": -14.524419784545898,
      "activations/layer20_attention_weight_max": 19.550060272216797,
      "activations/layer20_attention_weight_min": -23.007427215576172,
      "activations/layer21_attention_weight_max": 28.432750701904297,
      "activations/layer21_attention_weight_min": -20.0695858001709,
      "activations/layer22_attention_weight_max": 26.223556518554688,
      "activations/layer22_attention_weight_min": -23.47283935546875,
      "activations/layer23_attention_weight_max": 32.85409164428711,
      "activations/layer23_attention_weight_min": -23.885587692260742,
      "activations/layer2_attention_weight_max": 30.677383422851562,
      "activations/layer2_attention_weight_min": -31.252233505249023,
      "activations/layer3_attention_weight_max": 81.7326431274414,
      "activations/layer3_attention_weight_min": -87.37330627441406,
      "activations/layer4_attention_weight_max": 86.92378997802734,
      "activations/layer4_attention_weight_min": -91.96475982666016,
      "activations/layer5_attention_weight_max": 61.74479675292969,
      "activations/layer5_attention_weight_min": -73.62174224853516,
      "activations/layer6_attention_weight_max": 48.18367004394531,
      "activations/layer6_attention_weight_min": -49.38621520996094,
      "activations/layer7_attention_weight_max": 59.89935302734375,
      "activations/layer7_attention_weight_min": -58.46833801269531,
      "activations/layer8_attention_weight_max": 43.45442581176758,
      "activations/layer8_attention_weight_min": -46.21464157104492,
      "activations/layer9_attention_weight_max": 42.4087028503418,
      "activations/layer9_attention_weight_min": -44.50851821899414,
      "epoch": 15.98,
      "learning_rate": 4.743409090909091e-05,
      "loss": 2.7447,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4489,
      "eval_samples_per_second": 508.233,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4489,
      "eval_openwebtext_samples_per_second": 508.233,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.924,
      "eval_wikitext_samples_per_second": 237.004,
      "step": 275000
    },
    {
      "epoch": 15.98,
      "eval_lambada_loss": 2.509765625,
      "eval_lambada_ppl": 12.302046430466076,
      "eval_lambada_runtime": 9.5139,
      "eval_lambada_samples_per_second": 511.775,
      "step": 275000
    },
    {
      "activations/layer0_attention_weight_max": 15.896632194519043,
      "activations/layer0_attention_weight_min": -14.030653953552246,
      "activations/layer10_attention_weight_max": 33.578834533691406,
      "activations/layer10_attention_weight_min": -35.49408721923828,
      "activations/layer11_attention_weight_max": 30.672353744506836,
      "activations/layer11_attention_weight_min": -31.901321411132812,
      "activations/layer12_attention_weight_max": 17.78574562072754,
      "activations/layer12_attention_weight_min": -22.523303985595703,
      "activations/layer13_attention_weight_max": 31.35685920715332,
      "activations/layer13_attention_weight_min": -26.68259048461914,
      "activations/layer14_attention_weight_max": 32.92423629760742,
      "activations/layer14_attention_weight_min": -29.72251319885254,
      "activations/layer15_attention_weight_max": 30.38496971130371,
      "activations/layer15_attention_weight_min": -28.522594451904297,
      "activations/layer16_attention_weight_max": 31.632705688476562,
      "activations/layer16_attention_weight_min": -30.855791091918945,
      "activations/layer17_attention_weight_max": 49.34724426269531,
      "activations/layer17_attention_weight_min": -45.06074523925781,
      "activations/layer18_attention_weight_max": 45.78980255126953,
      "activations/layer18_attention_weight_min": -39.69664001464844,
      "activations/layer19_attention_weight_max": 21.50177574157715,
      "activations/layer19_attention_weight_min": -23.093061447143555,
      "activations/layer1_attention_weight_max": 15.641355514526367,
      "activations/layer1_attention_weight_min": -15.30950927734375,
      "activations/layer20_attention_weight_max": 22.377965927124023,
      "activations/layer20_attention_weight_min": -24.429092407226562,
      "activations/layer21_attention_weight_max": 32.44990158081055,
      "activations/layer21_attention_weight_min": -21.257061004638672,
      "activations/layer22_attention_weight_max": 30.24828338623047,
      "activations/layer22_attention_weight_min": -26.98601722717285,
      "activations/layer23_attention_weight_max": 33.45355987548828,
      "activations/layer23_attention_weight_min": -23.641836166381836,
      "activations/layer2_attention_weight_max": 29.68306541442871,
      "activations/layer2_attention_weight_min": -28.214780807495117,
      "activations/layer3_attention_weight_max": 78.6323471069336,
      "activations/layer3_attention_weight_min": -77.69630432128906,
      "activations/layer4_attention_weight_max": 81.30939483642578,
      "activations/layer4_attention_weight_min": -80.18102264404297,
      "activations/layer5_attention_weight_max": 61.97239303588867,
      "activations/layer5_attention_weight_min": -67.962890625,
      "activations/layer6_attention_weight_max": 47.520015716552734,
      "activations/layer6_attention_weight_min": -49.254886627197266,
      "activations/layer7_attention_weight_max": 62.722801208496094,
      "activations/layer7_attention_weight_min": -59.43396759033203,
      "activations/layer8_attention_weight_max": 41.358585357666016,
      "activations/layer8_attention_weight_min": -45.26387405395508,
      "activations/layer9_attention_weight_max": 39.93838119506836,
      "activations/layer9_attention_weight_min": -42.720706939697266,
      "epoch": 15.98,
      "learning_rate": 4.741515151515151e-05,
      "loss": 2.7689,
      "step": 275050
    },
    {
      "activations/layer0_attention_weight_max": 15.272136688232422,
      "activations/layer0_attention_weight_min": -14.13318920135498,
      "activations/layer10_attention_weight_max": 36.644874572753906,
      "activations/layer10_attention_weight_min": -37.4870719909668,
      "activations/layer11_attention_weight_max": 31.730573654174805,
      "activations/layer11_attention_weight_min": -34.325828552246094,
      "activations/layer12_attention_weight_max": 17.05809211730957,
      "activations/layer12_attention_weight_min": -25.64824867248535,
      "activations/layer13_attention_weight_max": 30.47958755493164,
      "activations/layer13_attention_weight_min": -27.761951446533203,
      "activations/layer14_attention_weight_max": 31.821651458740234,
      "activations/layer14_attention_weight_min": -30.614534378051758,
      "activations/layer15_attention_weight_max": 30.810035705566406,
      "activations/layer15_attention_weight_min": -29.847488403320312,
      "activations/layer16_attention_weight_max": 33.855621337890625,
      "activations/layer16_attention_weight_min": -34.23537063598633,
      "activations/layer17_attention_weight_max": 48.617027282714844,
      "activations/layer17_attention_weight_min": -46.6926155090332,
      "activations/layer18_attention_weight_max": 43.34400939941406,
      "activations/layer18_attention_weight_min": -42.15861892700195,
      "activations/layer19_attention_weight_max": 22.22764778137207,
      "activations/layer19_attention_weight_min": -22.554182052612305,
      "activations/layer1_attention_weight_max": 16.408798217773438,
      "activations/layer1_attention_weight_min": -13.77632999420166,
      "activations/layer20_attention_weight_max": 20.870975494384766,
      "activations/layer20_attention_weight_min": -23.089487075805664,
      "activations/layer21_attention_weight_max": 37.001651763916016,
      "activations/layer21_attention_weight_min": -25.416147232055664,
      "activations/layer22_attention_weight_max": 27.570114135742188,
      "activations/layer22_attention_weight_min": -24.172470092773438,
      "activations/layer23_attention_weight_max": 32.99095916748047,
      "activations/layer23_attention_weight_min": -23.288970947265625,
      "activations/layer2_attention_weight_max": 28.567668914794922,
      "activations/layer2_attention_weight_min": -26.995960235595703,
      "activations/layer3_attention_weight_max": 78.78449249267578,
      "activations/layer3_attention_weight_min": -83.26997375488281,
      "activations/layer4_attention_weight_max": 84.8638687133789,
      "activations/layer4_attention_weight_min": -86.2143325805664,
      "activations/layer5_attention_weight_max": 63.51392364501953,
      "activations/layer5_attention_weight_min": -75.1521987915039,
      "activations/layer6_attention_weight_max": 49.61365509033203,
      "activations/layer6_attention_weight_min": -52.853126525878906,
      "activations/layer7_attention_weight_max": 63.74979782104492,
      "activations/layer7_attention_weight_min": -64.98676300048828,
      "activations/layer8_attention_weight_max": 43.32455825805664,
      "activations/layer8_attention_weight_min": -50.12645721435547,
      "activations/layer9_attention_weight_max": 43.86250305175781,
      "activations/layer9_attention_weight_min": -47.63814926147461,
      "epoch": 15.98,
      "learning_rate": 4.7396212121212114e-05,
      "loss": 2.7519,
      "step": 275100
    },
    {
      "activations/layer0_attention_weight_max": 15.383265495300293,
      "activations/layer0_attention_weight_min": -14.184043884277344,
      "activations/layer10_attention_weight_max": 34.94092559814453,
      "activations/layer10_attention_weight_min": -34.59046173095703,
      "activations/layer11_attention_weight_max": 32.18744659423828,
      "activations/layer11_attention_weight_min": -33.73551940917969,
      "activations/layer12_attention_weight_max": 19.073028564453125,
      "activations/layer12_attention_weight_min": -25.752893447875977,
      "activations/layer13_attention_weight_max": 31.683279037475586,
      "activations/layer13_attention_weight_min": -26.983808517456055,
      "activations/layer14_attention_weight_max": 32.65538024902344,
      "activations/layer14_attention_weight_min": -28.385683059692383,
      "activations/layer15_attention_weight_max": 29.86699104309082,
      "activations/layer15_attention_weight_min": -31.10797119140625,
      "activations/layer16_attention_weight_max": 31.91503143310547,
      "activations/layer16_attention_weight_min": -32.652591705322266,
      "activations/layer17_attention_weight_max": 48.63036346435547,
      "activations/layer17_attention_weight_min": -45.316715240478516,
      "activations/layer18_attention_weight_max": 41.445762634277344,
      "activations/layer18_attention_weight_min": -37.320194244384766,
      "activations/layer19_attention_weight_max": 21.985897064208984,
      "activations/layer19_attention_weight_min": -25.86618995666504,
      "activations/layer1_attention_weight_max": 16.316381454467773,
      "activations/layer1_attention_weight_min": -15.262727737426758,
      "activations/layer20_attention_weight_max": 18.961156845092773,
      "activations/layer20_attention_weight_min": -21.690845489501953,
      "activations/layer21_attention_weight_max": 32.7491455078125,
      "activations/layer21_attention_weight_min": -22.369205474853516,
      "activations/layer22_attention_weight_max": 28.334192276000977,
      "activations/layer22_attention_weight_min": -29.449848175048828,
      "activations/layer23_attention_weight_max": 34.91657638549805,
      "activations/layer23_attention_weight_min": -25.273597717285156,
      "activations/layer2_attention_weight_max": 30.84296226501465,
      "activations/layer2_attention_weight_min": -28.692014694213867,
      "activations/layer3_attention_weight_max": 86.94759368896484,
      "activations/layer3_attention_weight_min": -87.2592544555664,
      "activations/layer4_attention_weight_max": 86.85602569580078,
      "activations/layer4_attention_weight_min": -88.6009750366211,
      "activations/layer5_attention_weight_max": 64.99049377441406,
      "activations/layer5_attention_weight_min": -72.70527648925781,
      "activations/layer6_attention_weight_max": 49.03242111206055,
      "activations/layer6_attention_weight_min": -50.07579803466797,
      "activations/layer7_attention_weight_max": 64.73987579345703,
      "activations/layer7_attention_weight_min": -58.645843505859375,
      "activations/layer8_attention_weight_max": 47.2707633972168,
      "activations/layer8_attention_weight_min": -47.46754837036133,
      "activations/layer9_attention_weight_max": 45.16501998901367,
      "activations/layer9_attention_weight_min": -43.796897888183594,
      "epoch": 15.99,
      "learning_rate": 4.737727272727272e-05,
      "loss": 2.732,
      "step": 275150
    },
    {
      "activations/layer0_attention_weight_max": 15.859026908874512,
      "activations/layer0_attention_weight_min": -13.952402114868164,
      "activations/layer10_attention_weight_max": 35.99659729003906,
      "activations/layer10_attention_weight_min": -36.730228424072266,
      "activations/layer11_attention_weight_max": 31.393600463867188,
      "activations/layer11_attention_weight_min": -35.416114807128906,
      "activations/layer12_attention_weight_max": 19.483474731445312,
      "activations/layer12_attention_weight_min": -25.05491065979004,
      "activations/layer13_attention_weight_max": 33.3209342956543,
      "activations/layer13_attention_weight_min": -31.350534439086914,
      "activations/layer14_attention_weight_max": 35.41521453857422,
      "activations/layer14_attention_weight_min": -34.32565689086914,
      "activations/layer15_attention_weight_max": 33.48625946044922,
      "activations/layer15_attention_weight_min": -36.987266540527344,
      "activations/layer16_attention_weight_max": 34.155555725097656,
      "activations/layer16_attention_weight_min": -34.397682189941406,
      "activations/layer17_attention_weight_max": 53.726654052734375,
      "activations/layer17_attention_weight_min": -49.519100189208984,
      "activations/layer18_attention_weight_max": 46.478511810302734,
      "activations/layer18_attention_weight_min": -45.2651481628418,
      "activations/layer19_attention_weight_max": 22.677976608276367,
      "activations/layer19_attention_weight_min": -24.58770179748535,
      "activations/layer1_attention_weight_max": 16.137529373168945,
      "activations/layer1_attention_weight_min": -15.21855640411377,
      "activations/layer20_attention_weight_max": 21.219133377075195,
      "activations/layer20_attention_weight_min": -22.798810958862305,
      "activations/layer21_attention_weight_max": 34.394100189208984,
      "activations/layer21_attention_weight_min": -22.52119255065918,
      "activations/layer22_attention_weight_max": 28.760726928710938,
      "activations/layer22_attention_weight_min": -25.10185432434082,
      "activations/layer23_attention_weight_max": 36.287662506103516,
      "activations/layer23_attention_weight_min": -23.49439811706543,
      "activations/layer2_attention_weight_max": 28.069583892822266,
      "activations/layer2_attention_weight_min": -27.48712730407715,
      "activations/layer3_attention_weight_max": 80.32787322998047,
      "activations/layer3_attention_weight_min": -80.20751953125,
      "activations/layer4_attention_weight_max": 87.37898254394531,
      "activations/layer4_attention_weight_min": -85.605712890625,
      "activations/layer5_attention_weight_max": 65.40745544433594,
      "activations/layer5_attention_weight_min": -73.13346862792969,
      "activations/layer6_attention_weight_max": 50.053653717041016,
      "activations/layer6_attention_weight_min": -52.196598052978516,
      "activations/layer7_attention_weight_max": 62.42197036743164,
      "activations/layer7_attention_weight_min": -60.150634765625,
      "activations/layer8_attention_weight_max": 45.68730545043945,
      "activations/layer8_attention_weight_min": -52.86161422729492,
      "activations/layer9_attention_weight_max": 45.07390213012695,
      "activations/layer9_attention_weight_min": -47.15140914916992,
      "epoch": 15.99,
      "learning_rate": 4.735833333333333e-05,
      "loss": 2.7287,
      "step": 275200
    },
    {
      "activations/layer0_attention_weight_max": 16.543970108032227,
      "activations/layer0_attention_weight_min": -13.91230583190918,
      "activations/layer10_attention_weight_max": 35.98149871826172,
      "activations/layer10_attention_weight_min": -34.5198860168457,
      "activations/layer11_attention_weight_max": 31.351072311401367,
      "activations/layer11_attention_weight_min": -30.8308162689209,
      "activations/layer12_attention_weight_max": 16.492319107055664,
      "activations/layer12_attention_weight_min": -28.82898712158203,
      "activations/layer13_attention_weight_max": 30.92412757873535,
      "activations/layer13_attention_weight_min": -25.932233810424805,
      "activations/layer14_attention_weight_max": 33.10862731933594,
      "activations/layer14_attention_weight_min": -29.21216583251953,
      "activations/layer15_attention_weight_max": 31.069684982299805,
      "activations/layer15_attention_weight_min": -27.70525360107422,
      "activations/layer16_attention_weight_max": 30.832508087158203,
      "activations/layer16_attention_weight_min": -30.112138748168945,
      "activations/layer17_attention_weight_max": 48.701087951660156,
      "activations/layer17_attention_weight_min": -44.87990188598633,
      "activations/layer18_attention_weight_max": 42.23702621459961,
      "activations/layer18_attention_weight_min": -37.8376579284668,
      "activations/layer19_attention_weight_max": 21.765409469604492,
      "activations/layer19_attention_weight_min": -21.22062110900879,
      "activations/layer1_attention_weight_max": 16.854393005371094,
      "activations/layer1_attention_weight_min": -15.607216835021973,
      "activations/layer20_attention_weight_max": 22.203014373779297,
      "activations/layer20_attention_weight_min": -23.102237701416016,
      "activations/layer21_attention_weight_max": 34.04924392700195,
      "activations/layer21_attention_weight_min": -22.66082763671875,
      "activations/layer22_attention_weight_max": 26.151948928833008,
      "activations/layer22_attention_weight_min": -26.85013771057129,
      "activations/layer23_attention_weight_max": 34.79157257080078,
      "activations/layer23_attention_weight_min": -27.502166748046875,
      "activations/layer2_attention_weight_max": 31.974599838256836,
      "activations/layer2_attention_weight_min": -31.985828399658203,
      "activations/layer3_attention_weight_max": 86.75621032714844,
      "activations/layer3_attention_weight_min": -86.24365997314453,
      "activations/layer4_attention_weight_max": 83.27021789550781,
      "activations/layer4_attention_weight_min": -85.5501480102539,
      "activations/layer5_attention_weight_max": 63.181884765625,
      "activations/layer5_attention_weight_min": -64.49058532714844,
      "activations/layer6_attention_weight_max": 48.832035064697266,
      "activations/layer6_attention_weight_min": -47.2529411315918,
      "activations/layer7_attention_weight_max": 61.06800079345703,
      "activations/layer7_attention_weight_min": -61.21060562133789,
      "activations/layer8_attention_weight_max": 46.9662971496582,
      "activations/layer8_attention_weight_min": -51.1573600769043,
      "activations/layer9_attention_weight_max": 48.27734375,
      "activations/layer9_attention_weight_min": -43.5841064453125,
      "epoch": 15.99,
      "learning_rate": 4.733939393939394e-05,
      "loss": 2.7699,
      "step": 275250
    },
    {
      "activations/layer0_attention_weight_max": 15.502058029174805,
      "activations/layer0_attention_weight_min": -13.774505615234375,
      "activations/layer10_attention_weight_max": 36.81953811645508,
      "activations/layer10_attention_weight_min": -36.814422607421875,
      "activations/layer11_attention_weight_max": 34.11198043823242,
      "activations/layer11_attention_weight_min": -36.54265594482422,
      "activations/layer12_attention_weight_max": 19.425186157226562,
      "activations/layer12_attention_weight_min": -23.363584518432617,
      "activations/layer13_attention_weight_max": 35.16268539428711,
      "activations/layer13_attention_weight_min": -27.343971252441406,
      "activations/layer14_attention_weight_max": 35.94921875,
      "activations/layer14_attention_weight_min": -30.33920669555664,
      "activations/layer15_attention_weight_max": 33.16875076293945,
      "activations/layer15_attention_weight_min": -29.87185287475586,
      "activations/layer16_attention_weight_max": 31.94359588623047,
      "activations/layer16_attention_weight_min": -31.533525466918945,
      "activations/layer17_attention_weight_max": 53.47267532348633,
      "activations/layer17_attention_weight_min": -43.85953903198242,
      "activations/layer18_attention_weight_max": 46.70475387573242,
      "activations/layer18_attention_weight_min": -38.17292404174805,
      "activations/layer19_attention_weight_max": 22.954242706298828,
      "activations/layer19_attention_weight_min": -21.182613372802734,
      "activations/layer1_attention_weight_max": 16.169330596923828,
      "activations/layer1_attention_weight_min": -13.430262565612793,
      "activations/layer20_attention_weight_max": 22.468761444091797,
      "activations/layer20_attention_weight_min": -20.65281105041504,
      "activations/layer21_attention_weight_max": 31.94432830810547,
      "activations/layer21_attention_weight_min": -21.173452377319336,
      "activations/layer22_attention_weight_max": 29.625703811645508,
      "activations/layer22_attention_weight_min": -24.98079490661621,
      "activations/layer23_attention_weight_max": 35.39638900756836,
      "activations/layer23_attention_weight_min": -26.91389274597168,
      "activations/layer2_attention_weight_max": 28.009977340698242,
      "activations/layer2_attention_weight_min": -28.065641403198242,
      "activations/layer3_attention_weight_max": 85.47809600830078,
      "activations/layer3_attention_weight_min": -87.98444366455078,
      "activations/layer4_attention_weight_max": 85.90099334716797,
      "activations/layer4_attention_weight_min": -86.12129974365234,
      "activations/layer5_attention_weight_max": 65.62210845947266,
      "activations/layer5_attention_weight_min": -75.36329650878906,
      "activations/layer6_attention_weight_max": 51.3398323059082,
      "activations/layer6_attention_weight_min": -53.54610061645508,
      "activations/layer7_attention_weight_max": 61.806453704833984,
      "activations/layer7_attention_weight_min": -66.39161682128906,
      "activations/layer8_attention_weight_max": 47.60100555419922,
      "activations/layer8_attention_weight_min": -53.27397918701172,
      "activations/layer9_attention_weight_max": 45.73550796508789,
      "activations/layer9_attention_weight_min": -48.2034797668457,
      "epoch": 16.0,
      "learning_rate": 4.732045454545454e-05,
      "loss": 2.7606,
      "step": 275300
    },
    {
      "activations/layer0_attention_weight_max": 16.082975387573242,
      "activations/layer0_attention_weight_min": -13.84851360321045,
      "activations/layer10_attention_weight_max": 34.95983123779297,
      "activations/layer10_attention_weight_min": -35.102821350097656,
      "activations/layer11_attention_weight_max": 31.90308380126953,
      "activations/layer11_attention_weight_min": -34.903202056884766,
      "activations/layer12_attention_weight_max": 18.511432647705078,
      "activations/layer12_attention_weight_min": -24.4217529296875,
      "activations/layer13_attention_weight_max": 33.44214630126953,
      "activations/layer13_attention_weight_min": -26.100656509399414,
      "activations/layer14_attention_weight_max": 33.05682373046875,
      "activations/layer14_attention_weight_min": -30.10464096069336,
      "activations/layer15_attention_weight_max": 35.646568298339844,
      "activations/layer15_attention_weight_min": -28.53135108947754,
      "activations/layer16_attention_weight_max": 32.37517547607422,
      "activations/layer16_attention_weight_min": -30.63704490661621,
      "activations/layer17_attention_weight_max": 50.78266143798828,
      "activations/layer17_attention_weight_min": -48.03215026855469,
      "activations/layer18_attention_weight_max": 46.03194808959961,
      "activations/layer18_attention_weight_min": -40.0815544128418,
      "activations/layer19_attention_weight_max": 23.886837005615234,
      "activations/layer19_attention_weight_min": -22.507665634155273,
      "activations/layer1_attention_weight_max": 17.74753761291504,
      "activations/layer1_attention_weight_min": -15.687034606933594,
      "activations/layer20_attention_weight_max": 21.30910873413086,
      "activations/layer20_attention_weight_min": -22.074180603027344,
      "activations/layer21_attention_weight_max": 36.090362548828125,
      "activations/layer21_attention_weight_min": -25.208837509155273,
      "activations/layer22_attention_weight_max": 28.176485061645508,
      "activations/layer22_attention_weight_min": -28.367523193359375,
      "activations/layer23_attention_weight_max": 34.891754150390625,
      "activations/layer23_attention_weight_min": -23.56782341003418,
      "activations/layer2_attention_weight_max": 32.17185974121094,
      "activations/layer2_attention_weight_min": -29.459854125976562,
      "activations/layer3_attention_weight_max": 82.90597534179688,
      "activations/layer3_attention_weight_min": -87.56321716308594,
      "activations/layer4_attention_weight_max": 88.09809875488281,
      "activations/layer4_attention_weight_min": -88.06658172607422,
      "activations/layer5_attention_weight_max": 65.12274932861328,
      "activations/layer5_attention_weight_min": -78.4095458984375,
      "activations/layer6_attention_weight_max": 50.34882354736328,
      "activations/layer6_attention_weight_min": -51.08162307739258,
      "activations/layer7_attention_weight_max": 62.246456146240234,
      "activations/layer7_attention_weight_min": -59.750083923339844,
      "activations/layer8_attention_weight_max": 47.1583137512207,
      "activations/layer8_attention_weight_min": -50.0056037902832,
      "activations/layer9_attention_weight_max": 44.54856872558594,
      "activations/layer9_attention_weight_min": -47.17588424682617,
      "epoch": 16.0,
      "learning_rate": 4.730151515151515e-05,
      "loss": 2.7677,
      "step": 275350
    },
    {
      "activations/layer0_attention_weight_max": 15.515875816345215,
      "activations/layer0_attention_weight_min": -14.1421537399292,
      "activations/layer10_attention_weight_max": 38.10783767700195,
      "activations/layer10_attention_weight_min": -40.1746940612793,
      "activations/layer11_attention_weight_max": 32.10213088989258,
      "activations/layer11_attention_weight_min": -33.5692138671875,
      "activations/layer12_attention_weight_max": 19.427228927612305,
      "activations/layer12_attention_weight_min": -24.600019454956055,
      "activations/layer13_attention_weight_max": 38.124629974365234,
      "activations/layer13_attention_weight_min": -27.02755355834961,
      "activations/layer14_attention_weight_max": 35.9729118347168,
      "activations/layer14_attention_weight_min": -30.505508422851562,
      "activations/layer15_attention_weight_max": 34.17540740966797,
      "activations/layer15_attention_weight_min": -28.47770118713379,
      "activations/layer16_attention_weight_max": 32.30309295654297,
      "activations/layer16_attention_weight_min": -31.203821182250977,
      "activations/layer17_attention_weight_max": 51.120601654052734,
      "activations/layer17_attention_weight_min": -43.92769241333008,
      "activations/layer18_attention_weight_max": 48.406890869140625,
      "activations/layer18_attention_weight_min": -37.77417755126953,
      "activations/layer19_attention_weight_max": 23.478946685791016,
      "activations/layer19_attention_weight_min": -22.729633331298828,
      "activations/layer1_attention_weight_max": 16.11135482788086,
      "activations/layer1_attention_weight_min": -12.992900848388672,
      "activations/layer20_attention_weight_max": 24.416675567626953,
      "activations/layer20_attention_weight_min": -22.283599853515625,
      "activations/layer21_attention_weight_max": 31.041179656982422,
      "activations/layer21_attention_weight_min": -21.09710121154785,
      "activations/layer22_attention_weight_max": 28.2554988861084,
      "activations/layer22_attention_weight_min": -24.36528778076172,
      "activations/layer23_attention_weight_max": 38.120689392089844,
      "activations/layer23_attention_weight_min": -26.297286987304688,
      "activations/layer2_attention_weight_max": 29.78081512451172,
      "activations/layer2_attention_weight_min": -28.18914794921875,
      "activations/layer3_attention_weight_max": 81.22344970703125,
      "activations/layer3_attention_weight_min": -82.36548614501953,
      "activations/layer4_attention_weight_max": 86.61505126953125,
      "activations/layer4_attention_weight_min": -87.68973541259766,
      "activations/layer5_attention_weight_max": 64.99951171875,
      "activations/layer5_attention_weight_min": -72.97003173828125,
      "activations/layer6_attention_weight_max": 49.12394714355469,
      "activations/layer6_attention_weight_min": -50.22604751586914,
      "activations/layer7_attention_weight_max": 64.05172729492188,
      "activations/layer7_attention_weight_min": -57.507503509521484,
      "activations/layer8_attention_weight_max": 45.54478454589844,
      "activations/layer8_attention_weight_min": -46.86142349243164,
      "activations/layer9_attention_weight_max": 42.5105094909668,
      "activations/layer9_attention_weight_min": -46.80425262451172,
      "epoch": 16.0,
      "learning_rate": 4.7282575757575757e-05,
      "loss": 2.78,
      "step": 275400
    },
    {
      "activations/layer0_attention_weight_max": 16.929738998413086,
      "activations/layer0_attention_weight_min": -14.039229393005371,
      "activations/layer10_attention_weight_max": 40.06281661987305,
      "activations/layer10_attention_weight_min": -37.637481689453125,
      "activations/layer11_attention_weight_max": 36.003997802734375,
      "activations/layer11_attention_weight_min": -36.515594482421875,
      "activations/layer12_attention_weight_max": 17.99791145324707,
      "activations/layer12_attention_weight_min": -24.778491973876953,
      "activations/layer13_attention_weight_max": 32.41463851928711,
      "activations/layer13_attention_weight_min": -28.335285186767578,
      "activations/layer14_attention_weight_max": 36.121726989746094,
      "activations/layer14_attention_weight_min": -30.219806671142578,
      "activations/layer15_attention_weight_max": 32.641845703125,
      "activations/layer15_attention_weight_min": -30.262035369873047,
      "activations/layer16_attention_weight_max": 32.25636672973633,
      "activations/layer16_attention_weight_min": -30.77269744873047,
      "activations/layer17_attention_weight_max": 49.17913055419922,
      "activations/layer17_attention_weight_min": -44.6391716003418,
      "activations/layer18_attention_weight_max": 45.549232482910156,
      "activations/layer18_attention_weight_min": -40.259578704833984,
      "activations/layer19_attention_weight_max": 21.980777740478516,
      "activations/layer19_attention_weight_min": -22.578140258789062,
      "activations/layer1_attention_weight_max": 15.896635055541992,
      "activations/layer1_attention_weight_min": -14.099565505981445,
      "activations/layer20_attention_weight_max": 24.04054832458496,
      "activations/layer20_attention_weight_min": -22.656105041503906,
      "activations/layer21_attention_weight_max": 32.16938400268555,
      "activations/layer21_attention_weight_min": -24.5352840423584,
      "activations/layer22_attention_weight_max": 30.092065811157227,
      "activations/layer22_attention_weight_min": -27.324779510498047,
      "activations/layer23_attention_weight_max": 37.108253479003906,
      "activations/layer23_attention_weight_min": -24.83661651611328,
      "activations/layer2_attention_weight_max": 28.25843048095703,
      "activations/layer2_attention_weight_min": -27.485776901245117,
      "activations/layer3_attention_weight_max": 82.06919860839844,
      "activations/layer3_attention_weight_min": -84.21800994873047,
      "activations/layer4_attention_weight_max": 85.78339385986328,
      "activations/layer4_attention_weight_min": -88.17534637451172,
      "activations/layer5_attention_weight_max": 64.65736389160156,
      "activations/layer5_attention_weight_min": -83.37393951416016,
      "activations/layer6_attention_weight_max": 50.22177505493164,
      "activations/layer6_attention_weight_min": -52.411808013916016,
      "activations/layer7_attention_weight_max": 61.77360534667969,
      "activations/layer7_attention_weight_min": -61.247440338134766,
      "activations/layer8_attention_weight_max": 46.256717681884766,
      "activations/layer8_attention_weight_min": -48.193275451660156,
      "activations/layer9_attention_weight_max": 45.52603530883789,
      "activations/layer9_attention_weight_min": -46.79568099975586,
      "epoch": 16.01,
      "learning_rate": 4.726363636363636e-05,
      "loss": 2.743,
      "step": 275450
    },
    {
      "activations/layer0_attention_weight_max": 16.508516311645508,
      "activations/layer0_attention_weight_min": -13.920907020568848,
      "activations/layer10_attention_weight_max": 43.44279098510742,
      "activations/layer10_attention_weight_min": -41.90460968017578,
      "activations/layer11_attention_weight_max": 40.23031997680664,
      "activations/layer11_attention_weight_min": -38.192718505859375,
      "activations/layer12_attention_weight_max": 19.460968017578125,
      "activations/layer12_attention_weight_min": -25.22213363647461,
      "activations/layer13_attention_weight_max": 44.064510345458984,
      "activations/layer13_attention_weight_min": -26.91530418395996,
      "activations/layer14_attention_weight_max": 51.913818359375,
      "activations/layer14_attention_weight_min": -29.797182083129883,
      "activations/layer15_attention_weight_max": 43.21492385864258,
      "activations/layer15_attention_weight_min": -29.429597854614258,
      "activations/layer16_attention_weight_max": 32.95655059814453,
      "activations/layer16_attention_weight_min": -30.994287490844727,
      "activations/layer17_attention_weight_max": 49.742530822753906,
      "activations/layer17_attention_weight_min": -44.76898956298828,
      "activations/layer18_attention_weight_max": 44.675296783447266,
      "activations/layer18_attention_weight_min": -40.063758850097656,
      "activations/layer19_attention_weight_max": 21.687498092651367,
      "activations/layer19_attention_weight_min": -21.86116600036621,
      "activations/layer1_attention_weight_max": 17.11863136291504,
      "activations/layer1_attention_weight_min": -14.604679107666016,
      "activations/layer20_attention_weight_max": 21.739177703857422,
      "activations/layer20_attention_weight_min": -23.420501708984375,
      "activations/layer21_attention_weight_max": 37.59684753417969,
      "activations/layer21_attention_weight_min": -23.76897430419922,
      "activations/layer22_attention_weight_max": 25.769067764282227,
      "activations/layer22_attention_weight_min": -25.51361656188965,
      "activations/layer23_attention_weight_max": 34.707855224609375,
      "activations/layer23_attention_weight_min": -24.648908615112305,
      "activations/layer2_attention_weight_max": 32.963993072509766,
      "activations/layer2_attention_weight_min": -28.717975616455078,
      "activations/layer3_attention_weight_max": 91.80518341064453,
      "activations/layer3_attention_weight_min": -89.59478759765625,
      "activations/layer4_attention_weight_max": 92.61724853515625,
      "activations/layer4_attention_weight_min": -85.5217514038086,
      "activations/layer5_attention_weight_max": 62.43199157714844,
      "activations/layer5_attention_weight_min": -75.92281341552734,
      "activations/layer6_attention_weight_max": 48.77684783935547,
      "activations/layer6_attention_weight_min": -49.08628463745117,
      "activations/layer7_attention_weight_max": 70.47016143798828,
      "activations/layer7_attention_weight_min": -58.453556060791016,
      "activations/layer8_attention_weight_max": 54.9807014465332,
      "activations/layer8_attention_weight_min": -46.9277229309082,
      "activations/layer9_attention_weight_max": 56.24177169799805,
      "activations/layer9_attention_weight_min": -48.60797882080078,
      "epoch": 16.01,
      "learning_rate": 4.7244696969696966e-05,
      "loss": 2.7424,
      "step": 275500
    },
    {
      "activations/layer0_attention_weight_max": 16.88312339782715,
      "activations/layer0_attention_weight_min": -13.714326858520508,
      "activations/layer10_attention_weight_max": 39.511329650878906,
      "activations/layer10_attention_weight_min": -40.642494201660156,
      "activations/layer11_attention_weight_max": 36.52283477783203,
      "activations/layer11_attention_weight_min": -38.954620361328125,
      "activations/layer12_attention_weight_max": 25.62614631652832,
      "activations/layer12_attention_weight_min": -21.558767318725586,
      "activations/layer13_attention_weight_max": 41.79511260986328,
      "activations/layer13_attention_weight_min": -29.04721450805664,
      "activations/layer14_attention_weight_max": 38.7333869934082,
      "activations/layer14_attention_weight_min": -32.9621696472168,
      "activations/layer15_attention_weight_max": 33.57225036621094,
      "activations/layer15_attention_weight_min": -33.9400749206543,
      "activations/layer16_attention_weight_max": 35.41538619995117,
      "activations/layer16_attention_weight_min": -32.024085998535156,
      "activations/layer17_attention_weight_max": 55.66056442260742,
      "activations/layer17_attention_weight_min": -50.403568267822266,
      "activations/layer18_attention_weight_max": 51.534271240234375,
      "activations/layer18_attention_weight_min": -40.97848892211914,
      "activations/layer19_attention_weight_max": 24.140235900878906,
      "activations/layer19_attention_weight_min": -23.850858688354492,
      "activations/layer1_attention_weight_max": 15.960575103759766,
      "activations/layer1_attention_weight_min": -15.1478910446167,
      "activations/layer20_attention_weight_max": 22.903703689575195,
      "activations/layer20_attention_weight_min": -24.22481346130371,
      "activations/layer21_attention_weight_max": 39.00233840942383,
      "activations/layer21_attention_weight_min": -24.233972549438477,
      "activations/layer22_attention_weight_max": 29.805742263793945,
      "activations/layer22_attention_weight_min": -27.1522216796875,
      "activations/layer23_attention_weight_max": 41.07814025878906,
      "activations/layer23_attention_weight_min": -25.127796173095703,
      "activations/layer2_attention_weight_max": 32.935298919677734,
      "activations/layer2_attention_weight_min": -31.935043334960938,
      "activations/layer3_attention_weight_max": 86.93061065673828,
      "activations/layer3_attention_weight_min": -93.2028579711914,
      "activations/layer4_attention_weight_max": 90.02389526367188,
      "activations/layer4_attention_weight_min": -90.54659271240234,
      "activations/layer5_attention_weight_max": 66.15846252441406,
      "activations/layer5_attention_weight_min": -75.46983337402344,
      "activations/layer6_attention_weight_max": 51.334815979003906,
      "activations/layer6_attention_weight_min": -52.9060173034668,
      "activations/layer7_attention_weight_max": 62.84486770629883,
      "activations/layer7_attention_weight_min": -60.85162353515625,
      "activations/layer8_attention_weight_max": 48.521636962890625,
      "activations/layer8_attention_weight_min": -50.13088607788086,
      "activations/layer9_attention_weight_max": 47.34272384643555,
      "activations/layer9_attention_weight_min": -46.18518829345703,
      "epoch": 16.01,
      "learning_rate": 4.7225757575757575e-05,
      "loss": 2.7517,
      "step": 275550
    },
    {
      "activations/layer0_attention_weight_max": 15.500717163085938,
      "activations/layer0_attention_weight_min": -13.997802734375,
      "activations/layer10_attention_weight_max": 37.957489013671875,
      "activations/layer10_attention_weight_min": -36.53178787231445,
      "activations/layer11_attention_weight_max": 33.499855041503906,
      "activations/layer11_attention_weight_min": -34.40156936645508,
      "activations/layer12_attention_weight_max": 19.652164459228516,
      "activations/layer12_attention_weight_min": -25.91875457763672,
      "activations/layer13_attention_weight_max": 38.0562629699707,
      "activations/layer13_attention_weight_min": -29.20403289794922,
      "activations/layer14_attention_weight_max": 36.24092483520508,
      "activations/layer14_attention_weight_min": -29.457683563232422,
      "activations/layer15_attention_weight_max": 32.62259292602539,
      "activations/layer15_attention_weight_min": -29.033185958862305,
      "activations/layer16_attention_weight_max": 33.063026428222656,
      "activations/layer16_attention_weight_min": -31.305408477783203,
      "activations/layer17_attention_weight_max": 51.8343505859375,
      "activations/layer17_attention_weight_min": -44.86698532104492,
      "activations/layer18_attention_weight_max": 46.81928253173828,
      "activations/layer18_attention_weight_min": -36.9076042175293,
      "activations/layer19_attention_weight_max": 22.31595230102539,
      "activations/layer19_attention_weight_min": -20.409515380859375,
      "activations/layer1_attention_weight_max": 17.01154899597168,
      "activations/layer1_attention_weight_min": -13.596830368041992,
      "activations/layer20_attention_weight_max": 23.269756317138672,
      "activations/layer20_attention_weight_min": -19.920263290405273,
      "activations/layer21_attention_weight_max": 38.31517791748047,
      "activations/layer21_attention_weight_min": -22.556137084960938,
      "activations/layer22_attention_weight_max": 29.594818115234375,
      "activations/layer22_attention_weight_min": -26.0927791595459,
      "activations/layer23_attention_weight_max": 39.922664642333984,
      "activations/layer23_attention_weight_min": -25.702579498291016,
      "activations/layer2_attention_weight_max": 29.711889266967773,
      "activations/layer2_attention_weight_min": -30.33447265625,
      "activations/layer3_attention_weight_max": 85.75455474853516,
      "activations/layer3_attention_weight_min": -88.07945251464844,
      "activations/layer4_attention_weight_max": 88.88748168945312,
      "activations/layer4_attention_weight_min": -89.75048065185547,
      "activations/layer5_attention_weight_max": 63.138519287109375,
      "activations/layer5_attention_weight_min": -74.64373016357422,
      "activations/layer6_attention_weight_max": 50.58158874511719,
      "activations/layer6_attention_weight_min": -51.58650588989258,
      "activations/layer7_attention_weight_max": 60.6657600402832,
      "activations/layer7_attention_weight_min": -62.253665924072266,
      "activations/layer8_attention_weight_max": 45.8969612121582,
      "activations/layer8_attention_weight_min": -50.502559661865234,
      "activations/layer9_attention_weight_max": 45.199398040771484,
      "activations/layer9_attention_weight_min": -45.08613967895508,
      "epoch": 16.01,
      "learning_rate": 4.7206818181818176e-05,
      "loss": 2.7392,
      "step": 275600
    },
    {
      "activations/layer0_attention_weight_max": 15.599358558654785,
      "activations/layer0_attention_weight_min": -13.998916625976562,
      "activations/layer10_attention_weight_max": 37.31364059448242,
      "activations/layer10_attention_weight_min": -35.702171325683594,
      "activations/layer11_attention_weight_max": 33.29475021362305,
      "activations/layer11_attention_weight_min": -34.252357482910156,
      "activations/layer12_attention_weight_max": 18.5081844329834,
      "activations/layer12_attention_weight_min": -23.372241973876953,
      "activations/layer13_attention_weight_max": 35.210899353027344,
      "activations/layer13_attention_weight_min": -26.33363914489746,
      "activations/layer14_attention_weight_max": 37.05735778808594,
      "activations/layer14_attention_weight_min": -30.215927124023438,
      "activations/layer15_attention_weight_max": 34.03723907470703,
      "activations/layer15_attention_weight_min": -28.046571731567383,
      "activations/layer16_attention_weight_max": 35.02460479736328,
      "activations/layer16_attention_weight_min": -33.122859954833984,
      "activations/layer17_attention_weight_max": 52.47590637207031,
      "activations/layer17_attention_weight_min": -42.76852035522461,
      "activations/layer18_attention_weight_max": 47.66923141479492,
      "activations/layer18_attention_weight_min": -39.39033126831055,
      "activations/layer19_attention_weight_max": 23.318157196044922,
      "activations/layer19_attention_weight_min": -24.9882755279541,
      "activations/layer1_attention_weight_max": 17.974782943725586,
      "activations/layer1_attention_weight_min": -15.699604034423828,
      "activations/layer20_attention_weight_max": 22.93372917175293,
      "activations/layer20_attention_weight_min": -22.26239585876465,
      "activations/layer21_attention_weight_max": 37.09645462036133,
      "activations/layer21_attention_weight_min": -23.177404403686523,
      "activations/layer22_attention_weight_max": 30.015472412109375,
      "activations/layer22_attention_weight_min": -24.18226432800293,
      "activations/layer23_attention_weight_max": 35.832855224609375,
      "activations/layer23_attention_weight_min": -22.591808319091797,
      "activations/layer2_attention_weight_max": 33.61125183105469,
      "activations/layer2_attention_weight_min": -30.64186668395996,
      "activations/layer3_attention_weight_max": 85.5269775390625,
      "activations/layer3_attention_weight_min": -84.2305679321289,
      "activations/layer4_attention_weight_max": 85.10262298583984,
      "activations/layer4_attention_weight_min": -85.77765655517578,
      "activations/layer5_attention_weight_max": 68.11260986328125,
      "activations/layer5_attention_weight_min": -74.6724853515625,
      "activations/layer6_attention_weight_max": 48.305538177490234,
      "activations/layer6_attention_weight_min": -50.877357482910156,
      "activations/layer7_attention_weight_max": 61.679317474365234,
      "activations/layer7_attention_weight_min": -58.06107711791992,
      "activations/layer8_attention_weight_max": 45.72172927856445,
      "activations/layer8_attention_weight_min": -48.16161346435547,
      "activations/layer9_attention_weight_max": 45.731239318847656,
      "activations/layer9_attention_weight_min": -45.447208404541016,
      "epoch": 16.02,
      "learning_rate": 4.7187878787878784e-05,
      "loss": 2.7378,
      "step": 275650
    },
    {
      "activations/layer0_attention_weight_max": 16.972900390625,
      "activations/layer0_attention_weight_min": -14.030004501342773,
      "activations/layer10_attention_weight_max": 34.28273010253906,
      "activations/layer10_attention_weight_min": -37.05501174926758,
      "activations/layer11_attention_weight_max": 34.51904296875,
      "activations/layer11_attention_weight_min": -35.727684020996094,
      "activations/layer12_attention_weight_max": 18.685930252075195,
      "activations/layer12_attention_weight_min": -26.257108688354492,
      "activations/layer13_attention_weight_max": 36.2947998046875,
      "activations/layer13_attention_weight_min": -27.051095962524414,
      "activations/layer14_attention_weight_max": 33.28813934326172,
      "activations/layer14_attention_weight_min": -29.417951583862305,
      "activations/layer15_attention_weight_max": 33.29988479614258,
      "activations/layer15_attention_weight_min": -28.219667434692383,
      "activations/layer16_attention_weight_max": 30.408809661865234,
      "activations/layer16_attention_weight_min": -30.459712982177734,
      "activations/layer17_attention_weight_max": 46.24298095703125,
      "activations/layer17_attention_weight_min": -42.88167190551758,
      "activations/layer18_attention_weight_max": 43.42082595825195,
      "activations/layer18_attention_weight_min": -36.881534576416016,
      "activations/layer19_attention_weight_max": 21.7012939453125,
      "activations/layer19_attention_weight_min": -20.888853073120117,
      "activations/layer1_attention_weight_max": 15.932684898376465,
      "activations/layer1_attention_weight_min": -14.890273094177246,
      "activations/layer20_attention_weight_max": 20.672700881958008,
      "activations/layer20_attention_weight_min": -21.791996002197266,
      "activations/layer21_attention_weight_max": 30.989397048950195,
      "activations/layer21_attention_weight_min": -23.563644409179688,
      "activations/layer22_attention_weight_max": 26.744586944580078,
      "activations/layer22_attention_weight_min": -25.378997802734375,
      "activations/layer23_attention_weight_max": 34.24760818481445,
      "activations/layer23_attention_weight_min": -24.08228874206543,
      "activations/layer2_attention_weight_max": 30.718486785888672,
      "activations/layer2_attention_weight_min": -28.459821701049805,
      "activations/layer3_attention_weight_max": 83.65713500976562,
      "activations/layer3_attention_weight_min": -84.5582275390625,
      "activations/layer4_attention_weight_max": 87.5244369506836,
      "activations/layer4_attention_weight_min": -84.27552032470703,
      "activations/layer5_attention_weight_max": 63.10685729980469,
      "activations/layer5_attention_weight_min": -71.9211196899414,
      "activations/layer6_attention_weight_max": 50.301151275634766,
      "activations/layer6_attention_weight_min": -52.34699630737305,
      "activations/layer7_attention_weight_max": 58.222232818603516,
      "activations/layer7_attention_weight_min": -57.6147575378418,
      "activations/layer8_attention_weight_max": 44.79018020629883,
      "activations/layer8_attention_weight_min": -46.6036262512207,
      "activations/layer9_attention_weight_max": 43.42489242553711,
      "activations/layer9_attention_weight_min": -43.87128829956055,
      "epoch": 16.02,
      "learning_rate": 4.7168939393939386e-05,
      "loss": 2.7293,
      "step": 275700
    },
    {
      "activations/layer0_attention_weight_max": 15.480657577514648,
      "activations/layer0_attention_weight_min": -14.168679237365723,
      "activations/layer10_attention_weight_max": 33.35791015625,
      "activations/layer10_attention_weight_min": -33.71841812133789,
      "activations/layer11_attention_weight_max": 31.1055965423584,
      "activations/layer11_attention_weight_min": -33.746009826660156,
      "activations/layer12_attention_weight_max": 15.982952117919922,
      "activations/layer12_attention_weight_min": -29.35211944580078,
      "activations/layer13_attention_weight_max": 30.22857093811035,
      "activations/layer13_attention_weight_min": -25.158239364624023,
      "activations/layer14_attention_weight_max": 33.32558059692383,
      "activations/layer14_attention_weight_min": -27.27880096435547,
      "activations/layer15_attention_weight_max": 29.580474853515625,
      "activations/layer15_attention_weight_min": -29.216293334960938,
      "activations/layer16_attention_weight_max": 29.83516502380371,
      "activations/layer16_attention_weight_min": -30.0618839263916,
      "activations/layer17_attention_weight_max": 45.66984558105469,
      "activations/layer17_attention_weight_min": -40.588645935058594,
      "activations/layer18_attention_weight_max": 42.23970413208008,
      "activations/layer18_attention_weight_min": -35.35239028930664,
      "activations/layer19_attention_weight_max": 21.502513885498047,
      "activations/layer19_attention_weight_min": -22.265066146850586,
      "activations/layer1_attention_weight_max": 16.607702255249023,
      "activations/layer1_attention_weight_min": -13.002474784851074,
      "activations/layer20_attention_weight_max": 19.09935188293457,
      "activations/layer20_attention_weight_min": -20.627307891845703,
      "activations/layer21_attention_weight_max": 27.397171020507812,
      "activations/layer21_attention_weight_min": -21.46494483947754,
      "activations/layer22_attention_weight_max": 29.349538803100586,
      "activations/layer22_attention_weight_min": -22.92884063720703,
      "activations/layer23_attention_weight_max": 36.2808837890625,
      "activations/layer23_attention_weight_min": -23.873394012451172,
      "activations/layer2_attention_weight_max": 29.61249542236328,
      "activations/layer2_attention_weight_min": -29.970134735107422,
      "activations/layer3_attention_weight_max": 85.91706085205078,
      "activations/layer3_attention_weight_min": -85.99314880371094,
      "activations/layer4_attention_weight_max": 86.50878143310547,
      "activations/layer4_attention_weight_min": -85.54730224609375,
      "activations/layer5_attention_weight_max": 61.09773254394531,
      "activations/layer5_attention_weight_min": -68.60884857177734,
      "activations/layer6_attention_weight_max": 47.55213928222656,
      "activations/layer6_attention_weight_min": -48.72328567504883,
      "activations/layer7_attention_weight_max": 57.32376480102539,
      "activations/layer7_attention_weight_min": -58.363182067871094,
      "activations/layer8_attention_weight_max": 44.07182693481445,
      "activations/layer8_attention_weight_min": -45.72734069824219,
      "activations/layer9_attention_weight_max": 46.34627151489258,
      "activations/layer9_attention_weight_min": -44.202877044677734,
      "epoch": 16.02,
      "learning_rate": 4.715e-05,
      "loss": 2.7323,
      "step": 275750
    },
    {
      "activations/layer0_attention_weight_max": 15.57676887512207,
      "activations/layer0_attention_weight_min": -14.073396682739258,
      "activations/layer10_attention_weight_max": 33.92448806762695,
      "activations/layer10_attention_weight_min": -34.81633377075195,
      "activations/layer11_attention_weight_max": 31.848461151123047,
      "activations/layer11_attention_weight_min": -33.26681137084961,
      "activations/layer12_attention_weight_max": 19.147380828857422,
      "activations/layer12_attention_weight_min": -28.084476470947266,
      "activations/layer13_attention_weight_max": 41.7275276184082,
      "activations/layer13_attention_weight_min": -30.14113426208496,
      "activations/layer14_attention_weight_max": 34.76704025268555,
      "activations/layer14_attention_weight_min": -31.835264205932617,
      "activations/layer15_attention_weight_max": 31.70903205871582,
      "activations/layer15_attention_weight_min": -31.349363327026367,
      "activations/layer16_attention_weight_max": 32.87808609008789,
      "activations/layer16_attention_weight_min": -32.05574417114258,
      "activations/layer17_attention_weight_max": 52.21425247192383,
      "activations/layer17_attention_weight_min": -48.533931732177734,
      "activations/layer18_attention_weight_max": 47.805423736572266,
      "activations/layer18_attention_weight_min": -42.75604248046875,
      "activations/layer19_attention_weight_max": 23.269201278686523,
      "activations/layer19_attention_weight_min": -25.08755874633789,
      "activations/layer1_attention_weight_max": 17.41334342956543,
      "activations/layer1_attention_weight_min": -13.302459716796875,
      "activations/layer20_attention_weight_max": 22.683591842651367,
      "activations/layer20_attention_weight_min": -24.787675857543945,
      "activations/layer21_attention_weight_max": 34.9334602355957,
      "activations/layer21_attention_weight_min": -25.13759422302246,
      "activations/layer22_attention_weight_max": 28.43062973022461,
      "activations/layer22_attention_weight_min": -26.802947998046875,
      "activations/layer23_attention_weight_max": 34.22019958496094,
      "activations/layer23_attention_weight_min": -26.791547775268555,
      "activations/layer2_attention_weight_max": 30.044830322265625,
      "activations/layer2_attention_weight_min": -28.29994010925293,
      "activations/layer3_attention_weight_max": 84.5940170288086,
      "activations/layer3_attention_weight_min": -85.9463882446289,
      "activations/layer4_attention_weight_max": 86.34054565429688,
      "activations/layer4_attention_weight_min": -82.70830535888672,
      "activations/layer5_attention_weight_max": 66.66861724853516,
      "activations/layer5_attention_weight_min": -73.2366714477539,
      "activations/layer6_attention_weight_max": 50.16421890258789,
      "activations/layer6_attention_weight_min": -47.17966842651367,
      "activations/layer7_attention_weight_max": 63.35127258300781,
      "activations/layer7_attention_weight_min": -59.90340042114258,
      "activations/layer8_attention_weight_max": 47.30997848510742,
      "activations/layer8_attention_weight_min": -47.09101104736328,
      "activations/layer9_attention_weight_max": 47.51371383666992,
      "activations/layer9_attention_weight_min": -43.67454528808594,
      "epoch": 16.03,
      "learning_rate": 4.71310606060606e-05,
      "loss": 2.7484,
      "step": 275800
    },
    {
      "activations/layer0_attention_weight_max": 15.70052719116211,
      "activations/layer0_attention_weight_min": -14.256049156188965,
      "activations/layer10_attention_weight_max": 33.7153434753418,
      "activations/layer10_attention_weight_min": -34.82527542114258,
      "activations/layer11_attention_weight_max": 33.156864166259766,
      "activations/layer11_attention_weight_min": -36.57738494873047,
      "activations/layer12_attention_weight_max": 18.02536964416504,
      "activations/layer12_attention_weight_min": -21.760229110717773,
      "activations/layer13_attention_weight_max": 35.40058517456055,
      "activations/layer13_attention_weight_min": -25.6143856048584,
      "activations/layer14_attention_weight_max": 32.41551971435547,
      "activations/layer14_attention_weight_min": -30.40505027770996,
      "activations/layer15_attention_weight_max": 34.17791748046875,
      "activations/layer15_attention_weight_min": -29.8857421875,
      "activations/layer16_attention_weight_max": 30.976455688476562,
      "activations/layer16_attention_weight_min": -29.073169708251953,
      "activations/layer17_attention_weight_max": 46.626190185546875,
      "activations/layer17_attention_weight_min": -40.3482666015625,
      "activations/layer18_attention_weight_max": 42.299800872802734,
      "activations/layer18_attention_weight_min": -35.41704177856445,
      "activations/layer19_attention_weight_max": 22.0362606048584,
      "activations/layer19_attention_weight_min": -21.642932891845703,
      "activations/layer1_attention_weight_max": 15.68220329284668,
      "activations/layer1_attention_weight_min": -13.969075202941895,
      "activations/layer20_attention_weight_max": 21.7628116607666,
      "activations/layer20_attention_weight_min": -23.7230167388916,
      "activations/layer21_attention_weight_max": 31.80925750732422,
      "activations/layer21_attention_weight_min": -25.048526763916016,
      "activations/layer22_attention_weight_max": 26.846900939941406,
      "activations/layer22_attention_weight_min": -24.665836334228516,
      "activations/layer23_attention_weight_max": 37.160797119140625,
      "activations/layer23_attention_weight_min": -24.611194610595703,
      "activations/layer2_attention_weight_max": 29.219396591186523,
      "activations/layer2_attention_weight_min": -29.046659469604492,
      "activations/layer3_attention_weight_max": 85.1852798461914,
      "activations/layer3_attention_weight_min": -90.08574676513672,
      "activations/layer4_attention_weight_max": 88.8078842163086,
      "activations/layer4_attention_weight_min": -90.83100891113281,
      "activations/layer5_attention_weight_max": 62.67134094238281,
      "activations/layer5_attention_weight_min": -71.124267578125,
      "activations/layer6_attention_weight_max": 48.17605972290039,
      "activations/layer6_attention_weight_min": -49.10951232910156,
      "activations/layer7_attention_weight_max": 61.44978332519531,
      "activations/layer7_attention_weight_min": -58.948524475097656,
      "activations/layer8_attention_weight_max": 44.707305908203125,
      "activations/layer8_attention_weight_min": -47.34017562866211,
      "activations/layer9_attention_weight_max": 43.125938415527344,
      "activations/layer9_attention_weight_min": -44.174495697021484,
      "epoch": 16.03,
      "learning_rate": 4.7112121212121204e-05,
      "loss": 2.7228,
      "step": 275850
    },
    {
      "activations/layer0_attention_weight_max": 15.713123321533203,
      "activations/layer0_attention_weight_min": -13.931692123413086,
      "activations/layer10_attention_weight_max": 34.76778793334961,
      "activations/layer10_attention_weight_min": -35.00908660888672,
      "activations/layer11_attention_weight_max": 31.667675018310547,
      "activations/layer11_attention_weight_min": -32.8951416015625,
      "activations/layer12_attention_weight_max": 19.66192626953125,
      "activations/layer12_attention_weight_min": -25.415180206298828,
      "activations/layer13_attention_weight_max": 31.216140747070312,
      "activations/layer13_attention_weight_min": -31.53398323059082,
      "activations/layer14_attention_weight_max": 34.00896072387695,
      "activations/layer14_attention_weight_min": -29.04191017150879,
      "activations/layer15_attention_weight_max": 31.83133316040039,
      "activations/layer15_attention_weight_min": -28.76192855834961,
      "activations/layer16_attention_weight_max": 34.22394561767578,
      "activations/layer16_attention_weight_min": -31.638561248779297,
      "activations/layer17_attention_weight_max": 52.11991882324219,
      "activations/layer17_attention_weight_min": -45.572269439697266,
      "activations/layer18_attention_weight_max": 47.89749526977539,
      "activations/layer18_attention_weight_min": -39.680267333984375,
      "activations/layer19_attention_weight_max": 24.031442642211914,
      "activations/layer19_attention_weight_min": -24.334402084350586,
      "activations/layer1_attention_weight_max": 16.40989875793457,
      "activations/layer1_attention_weight_min": -17.351776123046875,
      "activations/layer20_attention_weight_max": 22.016313552856445,
      "activations/layer20_attention_weight_min": -23.06998634338379,
      "activations/layer21_attention_weight_max": 39.35089111328125,
      "activations/layer21_attention_weight_min": -26.992431640625,
      "activations/layer22_attention_weight_max": 30.16429901123047,
      "activations/layer22_attention_weight_min": -26.52529525756836,
      "activations/layer23_attention_weight_max": 34.58208465576172,
      "activations/layer23_attention_weight_min": -25.493682861328125,
      "activations/layer2_attention_weight_max": 29.983510971069336,
      "activations/layer2_attention_weight_min": -28.21717643737793,
      "activations/layer3_attention_weight_max": 85.35684204101562,
      "activations/layer3_attention_weight_min": -89.09037017822266,
      "activations/layer4_attention_weight_max": 87.32698822021484,
      "activations/layer4_attention_weight_min": -85.78485870361328,
      "activations/layer5_attention_weight_max": 63.70473861694336,
      "activations/layer5_attention_weight_min": -70.50827026367188,
      "activations/layer6_attention_weight_max": 49.504356384277344,
      "activations/layer6_attention_weight_min": -48.9310417175293,
      "activations/layer7_attention_weight_max": 61.97829055786133,
      "activations/layer7_attention_weight_min": -58.410682678222656,
      "activations/layer8_attention_weight_max": 44.1812629699707,
      "activations/layer8_attention_weight_min": -46.621517181396484,
      "activations/layer9_attention_weight_max": 44.94546127319336,
      "activations/layer9_attention_weight_min": -41.96649932861328,
      "epoch": 16.03,
      "learning_rate": 4.709318181818181e-05,
      "loss": 2.7383,
      "step": 275900
    },
    {
      "activations/layer0_attention_weight_max": 16.618684768676758,
      "activations/layer0_attention_weight_min": -14.246819496154785,
      "activations/layer10_attention_weight_max": 34.103851318359375,
      "activations/layer10_attention_weight_min": -35.513099670410156,
      "activations/layer11_attention_weight_max": 30.145103454589844,
      "activations/layer11_attention_weight_min": -33.08684539794922,
      "activations/layer12_attention_weight_max": 18.658607482910156,
      "activations/layer12_attention_weight_min": -27.352792739868164,
      "activations/layer13_attention_weight_max": 35.68082809448242,
      "activations/layer13_attention_weight_min": -29.203420639038086,
      "activations/layer14_attention_weight_max": 33.654842376708984,
      "activations/layer14_attention_weight_min": -31.122514724731445,
      "activations/layer15_attention_weight_max": 32.003517150878906,
      "activations/layer15_attention_weight_min": -31.258413314819336,
      "activations/layer16_attention_weight_max": 32.607627868652344,
      "activations/layer16_attention_weight_min": -30.263643264770508,
      "activations/layer17_attention_weight_max": 49.08843994140625,
      "activations/layer17_attention_weight_min": -45.47287368774414,
      "activations/layer18_attention_weight_max": 43.12157440185547,
      "activations/layer18_attention_weight_min": -40.178009033203125,
      "activations/layer19_attention_weight_max": 23.48164176940918,
      "activations/layer19_attention_weight_min": -24.78197479248047,
      "activations/layer1_attention_weight_max": 16.238496780395508,
      "activations/layer1_attention_weight_min": -15.786116600036621,
      "activations/layer20_attention_weight_max": 22.70878791809082,
      "activations/layer20_attention_weight_min": -25.54332733154297,
      "activations/layer21_attention_weight_max": 38.332271575927734,
      "activations/layer21_attention_weight_min": -23.26807403564453,
      "activations/layer22_attention_weight_max": 31.89602279663086,
      "activations/layer22_attention_weight_min": -30.112812042236328,
      "activations/layer23_attention_weight_max": 35.74082565307617,
      "activations/layer23_attention_weight_min": -26.170089721679688,
      "activations/layer2_attention_weight_max": 29.088829040527344,
      "activations/layer2_attention_weight_min": -28.93230438232422,
      "activations/layer3_attention_weight_max": 83.42761993408203,
      "activations/layer3_attention_weight_min": -83.10941314697266,
      "activations/layer4_attention_weight_max": 86.66796112060547,
      "activations/layer4_attention_weight_min": -83.17366027832031,
      "activations/layer5_attention_weight_max": 62.471832275390625,
      "activations/layer5_attention_weight_min": -71.21569061279297,
      "activations/layer6_attention_weight_max": 47.50001907348633,
      "activations/layer6_attention_weight_min": -50.25101089477539,
      "activations/layer7_attention_weight_max": 58.146240234375,
      "activations/layer7_attention_weight_min": -59.89995574951172,
      "activations/layer8_attention_weight_max": 39.7138557434082,
      "activations/layer8_attention_weight_min": -43.99012756347656,
      "activations/layer9_attention_weight_max": 45.46443557739258,
      "activations/layer9_attention_weight_min": -43.17301559448242,
      "epoch": 16.03,
      "learning_rate": 4.707424242424242e-05,
      "loss": 2.7445,
      "step": 275950
    },
    {
      "activations/layer0_attention_weight_max": 15.599289894104004,
      "activations/layer0_attention_weight_min": -13.737136840820312,
      "activations/layer10_attention_weight_max": 37.092674255371094,
      "activations/layer10_attention_weight_min": -37.63027572631836,
      "activations/layer11_attention_weight_max": 33.68284225463867,
      "activations/layer11_attention_weight_min": -34.91576385498047,
      "activations/layer12_attention_weight_max": 22.46902847290039,
      "activations/layer12_attention_weight_min": -24.023033142089844,
      "activations/layer13_attention_weight_max": 36.41745376586914,
      "activations/layer13_attention_weight_min": -28.6444149017334,
      "activations/layer14_attention_weight_max": 35.12458801269531,
      "activations/layer14_attention_weight_min": -30.698266983032227,
      "activations/layer15_attention_weight_max": 31.54997444152832,
      "activations/layer15_attention_weight_min": -29.920427322387695,
      "activations/layer16_attention_weight_max": 33.57109069824219,
      "activations/layer16_attention_weight_min": -31.317153930664062,
      "activations/layer17_attention_weight_max": 53.84591293334961,
      "activations/layer17_attention_weight_min": -45.29179763793945,
      "activations/layer18_attention_weight_max": 45.67966079711914,
      "activations/layer18_attention_weight_min": -38.2628059387207,
      "activations/layer19_attention_weight_max": 23.872888565063477,
      "activations/layer19_attention_weight_min": -23.55414581298828,
      "activations/layer1_attention_weight_max": 17.115646362304688,
      "activations/layer1_attention_weight_min": -13.230693817138672,
      "activations/layer20_attention_weight_max": 21.57501792907715,
      "activations/layer20_attention_weight_min": -21.81826400756836,
      "activations/layer21_attention_weight_max": 34.171844482421875,
      "activations/layer21_attention_weight_min": -23.508697509765625,
      "activations/layer22_attention_weight_max": 28.780744552612305,
      "activations/layer22_attention_weight_min": -26.491455078125,
      "activations/layer23_attention_weight_max": 38.526493072509766,
      "activations/layer23_attention_weight_min": -23.43408966064453,
      "activations/layer2_attention_weight_max": 29.565519332885742,
      "activations/layer2_attention_weight_min": -27.891752243041992,
      "activations/layer3_attention_weight_max": 88.37289428710938,
      "activations/layer3_attention_weight_min": -82.74139404296875,
      "activations/layer4_attention_weight_max": 86.58600616455078,
      "activations/layer4_attention_weight_min": -84.55384063720703,
      "activations/layer5_attention_weight_max": 63.58041000366211,
      "activations/layer5_attention_weight_min": -70.77838134765625,
      "activations/layer6_attention_weight_max": 49.90257263183594,
      "activations/layer6_attention_weight_min": -50.32875442504883,
      "activations/layer7_attention_weight_max": 62.993404388427734,
      "activations/layer7_attention_weight_min": -57.700862884521484,
      "activations/layer8_attention_weight_max": 45.502445220947266,
      "activations/layer8_attention_weight_min": -49.3625373840332,
      "activations/layer9_attention_weight_max": 43.82539749145508,
      "activations/layer9_attention_weight_min": -43.90338134765625,
      "epoch": 16.04,
      "learning_rate": 4.705568181818181e-05,
      "loss": 2.7422,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4696,
      "eval_samples_per_second": 506.993,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4696,
      "eval_openwebtext_samples_per_second": 506.993,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9566,
      "eval_wikitext_samples_per_second": 233.06,
      "step": 276000
    },
    {
      "epoch": 16.04,
      "eval_lambada_loss": 2.53125,
      "eval_lambada_ppl": 12.569207830853442,
      "eval_lambada_runtime": 9.5135,
      "eval_lambada_samples_per_second": 511.798,
      "step": 276000
    },
    {
      "activations/layer0_attention_weight_max": 16.608999252319336,
      "activations/layer0_attention_weight_min": -14.153299331665039,
      "activations/layer10_attention_weight_max": 37.21510696411133,
      "activations/layer10_attention_weight_min": -35.9918327331543,
      "activations/layer11_attention_weight_max": 33.15168380737305,
      "activations/layer11_attention_weight_min": -33.076107025146484,
      "activations/layer12_attention_weight_max": 18.67966079711914,
      "activations/layer12_attention_weight_min": -24.714385986328125,
      "activations/layer13_attention_weight_max": 32.42204284667969,
      "activations/layer13_attention_weight_min": -27.842592239379883,
      "activations/layer14_attention_weight_max": 35.41203689575195,
      "activations/layer14_attention_weight_min": -30.562786102294922,
      "activations/layer15_attention_weight_max": 34.09938049316406,
      "activations/layer15_attention_weight_min": -29.637258529663086,
      "activations/layer16_attention_weight_max": 35.54751205444336,
      "activations/layer16_attention_weight_min": -34.12900161743164,
      "activations/layer17_attention_weight_max": 54.006805419921875,
      "activations/layer17_attention_weight_min": -47.377174377441406,
      "activations/layer18_attention_weight_max": 50.09580612182617,
      "activations/layer18_attention_weight_min": -42.51363754272461,
      "activations/layer19_attention_weight_max": 23.317516326904297,
      "activations/layer19_attention_weight_min": -24.227397918701172,
      "activations/layer1_attention_weight_max": 16.625560760498047,
      "activations/layer1_attention_weight_min": -13.983699798583984,
      "activations/layer20_attention_weight_max": 21.977006912231445,
      "activations/layer20_attention_weight_min": -22.619287490844727,
      "activations/layer21_attention_weight_max": 34.0837287902832,
      "activations/layer21_attention_weight_min": -22.62802505493164,
      "activations/layer22_attention_weight_max": 28.9783992767334,
      "activations/layer22_attention_weight_min": -28.216136932373047,
      "activations/layer23_attention_weight_max": 36.171241760253906,
      "activations/layer23_attention_weight_min": -26.95395278930664,
      "activations/layer2_attention_weight_max": 29.05096435546875,
      "activations/layer2_attention_weight_min": -28.63748550415039,
      "activations/layer3_attention_weight_max": 79.75477600097656,
      "activations/layer3_attention_weight_min": -82.17430114746094,
      "activations/layer4_attention_weight_max": 84.62391662597656,
      "activations/layer4_attention_weight_min": -84.72640228271484,
      "activations/layer5_attention_weight_max": 60.65576171875,
      "activations/layer5_attention_weight_min": -72.46881866455078,
      "activations/layer6_attention_weight_max": 49.31154251098633,
      "activations/layer6_attention_weight_min": -49.08723831176758,
      "activations/layer7_attention_weight_max": 59.95216751098633,
      "activations/layer7_attention_weight_min": -59.93610382080078,
      "activations/layer8_attention_weight_max": 44.57570266723633,
      "activations/layer8_attention_weight_min": -50.161930084228516,
      "activations/layer9_attention_weight_max": 44.99580001831055,
      "activations/layer9_attention_weight_min": -44.865234375,
      "epoch": 16.04,
      "learning_rate": 4.703674242424242e-05,
      "loss": 2.7648,
      "step": 276050
    },
    {
      "activations/layer0_attention_weight_max": 15.79987907409668,
      "activations/layer0_attention_weight_min": -13.635587692260742,
      "activations/layer10_attention_weight_max": 36.398860931396484,
      "activations/layer10_attention_weight_min": -36.05707550048828,
      "activations/layer11_attention_weight_max": 31.883525848388672,
      "activations/layer11_attention_weight_min": -32.12611389160156,
      "activations/layer12_attention_weight_max": 18.38020133972168,
      "activations/layer12_attention_weight_min": -25.672584533691406,
      "activations/layer13_attention_weight_max": 33.25457763671875,
      "activations/layer13_attention_weight_min": -27.409934997558594,
      "activations/layer14_attention_weight_max": 31.98149871826172,
      "activations/layer14_attention_weight_min": -30.168203353881836,
      "activations/layer15_attention_weight_max": 30.73675537109375,
      "activations/layer15_attention_weight_min": -28.374935150146484,
      "activations/layer16_attention_weight_max": 31.868911743164062,
      "activations/layer16_attention_weight_min": -31.484643936157227,
      "activations/layer17_attention_weight_max": 50.39471435546875,
      "activations/layer17_attention_weight_min": -44.66154479980469,
      "activations/layer18_attention_weight_max": 43.715057373046875,
      "activations/layer18_attention_weight_min": -38.50884246826172,
      "activations/layer19_attention_weight_max": 21.60218620300293,
      "activations/layer19_attention_weight_min": -22.578062057495117,
      "activations/layer1_attention_weight_max": 16.213476181030273,
      "activations/layer1_attention_weight_min": -12.100308418273926,
      "activations/layer20_attention_weight_max": 20.67180061340332,
      "activations/layer20_attention_weight_min": -23.058568954467773,
      "activations/layer21_attention_weight_max": 33.90077209472656,
      "activations/layer21_attention_weight_min": -23.744945526123047,
      "activations/layer22_attention_weight_max": 29.035242080688477,
      "activations/layer22_attention_weight_min": -28.1011962890625,
      "activations/layer23_attention_weight_max": 35.907745361328125,
      "activations/layer23_attention_weight_min": -26.40774917602539,
      "activations/layer2_attention_weight_max": 30.130233764648438,
      "activations/layer2_attention_weight_min": -29.53195571899414,
      "activations/layer3_attention_weight_max": 82.98517608642578,
      "activations/layer3_attention_weight_min": -84.72042846679688,
      "activations/layer4_attention_weight_max": 86.52758026123047,
      "activations/layer4_attention_weight_min": -87.73477935791016,
      "activations/layer5_attention_weight_max": 65.05746459960938,
      "activations/layer5_attention_weight_min": -73.6993179321289,
      "activations/layer6_attention_weight_max": 50.1204948425293,
      "activations/layer6_attention_weight_min": -51.277687072753906,
      "activations/layer7_attention_weight_max": 59.31897735595703,
      "activations/layer7_attention_weight_min": -58.933128356933594,
      "activations/layer8_attention_weight_max": 45.0029182434082,
      "activations/layer8_attention_weight_min": -49.67177200317383,
      "activations/layer9_attention_weight_max": 42.70499801635742,
      "activations/layer9_attention_weight_min": -47.37638473510742,
      "epoch": 16.04,
      "learning_rate": 4.701780303030302e-05,
      "loss": 2.7559,
      "step": 276100
    },
    {
      "activations/layer0_attention_weight_max": 15.84378719329834,
      "activations/layer0_attention_weight_min": -13.669737815856934,
      "activations/layer10_attention_weight_max": 38.98181915283203,
      "activations/layer10_attention_weight_min": -36.871421813964844,
      "activations/layer11_attention_weight_max": 34.57963562011719,
      "activations/layer11_attention_weight_min": -36.0760498046875,
      "activations/layer12_attention_weight_max": 18.183475494384766,
      "activations/layer12_attention_weight_min": -24.68315315246582,
      "activations/layer13_attention_weight_max": 31.32000160217285,
      "activations/layer13_attention_weight_min": -28.363826751708984,
      "activations/layer14_attention_weight_max": 34.2913932800293,
      "activations/layer14_attention_weight_min": -32.696205139160156,
      "activations/layer15_attention_weight_max": 31.570655822753906,
      "activations/layer15_attention_weight_min": -31.021312713623047,
      "activations/layer16_attention_weight_max": 31.656570434570312,
      "activations/layer16_attention_weight_min": -30.581022262573242,
      "activations/layer17_attention_weight_max": 50.34175491333008,
      "activations/layer17_attention_weight_min": -43.619873046875,
      "activations/layer18_attention_weight_max": 45.850425720214844,
      "activations/layer18_attention_weight_min": -39.34919738769531,
      "activations/layer19_attention_weight_max": 22.900707244873047,
      "activations/layer19_attention_weight_min": -22.61637306213379,
      "activations/layer1_attention_weight_max": 16.40918731689453,
      "activations/layer1_attention_weight_min": -15.334016799926758,
      "activations/layer20_attention_weight_max": 21.625486373901367,
      "activations/layer20_attention_weight_min": -22.51935386657715,
      "activations/layer21_attention_weight_max": 33.34747314453125,
      "activations/layer21_attention_weight_min": -29.98821258544922,
      "activations/layer22_attention_weight_max": 29.57286262512207,
      "activations/layer22_attention_weight_min": -27.308910369873047,
      "activations/layer23_attention_weight_max": 37.173397064208984,
      "activations/layer23_attention_weight_min": -25.04450035095215,
      "activations/layer2_attention_weight_max": 31.249088287353516,
      "activations/layer2_attention_weight_min": -29.397563934326172,
      "activations/layer3_attention_weight_max": 82.92440032958984,
      "activations/layer3_attention_weight_min": -82.30824279785156,
      "activations/layer4_attention_weight_max": 90.68326568603516,
      "activations/layer4_attention_weight_min": -88.51522827148438,
      "activations/layer5_attention_weight_max": 67.49703216552734,
      "activations/layer5_attention_weight_min": -73.82268524169922,
      "activations/layer6_attention_weight_max": 50.6435661315918,
      "activations/layer6_attention_weight_min": -52.66017150878906,
      "activations/layer7_attention_weight_max": 65.30767822265625,
      "activations/layer7_attention_weight_min": -61.60300827026367,
      "activations/layer8_attention_weight_max": 48.9201545715332,
      "activations/layer8_attention_weight_min": -53.304229736328125,
      "activations/layer9_attention_weight_max": 43.65830612182617,
      "activations/layer9_attention_weight_min": -47.83457565307617,
      "epoch": 16.05,
      "learning_rate": 4.6998863636363636e-05,
      "loss": 2.749,
      "step": 276150
    },
    {
      "activations/layer0_attention_weight_max": 16.257890701293945,
      "activations/layer0_attention_weight_min": -13.791159629821777,
      "activations/layer10_attention_weight_max": 34.507965087890625,
      "activations/layer10_attention_weight_min": -36.01938247680664,
      "activations/layer11_attention_weight_max": 31.019432067871094,
      "activations/layer11_attention_weight_min": -34.035091400146484,
      "activations/layer12_attention_weight_max": 17.02711296081543,
      "activations/layer12_attention_weight_min": -27.815404891967773,
      "activations/layer13_attention_weight_max": 29.971893310546875,
      "activations/layer13_attention_weight_min": -27.14313507080078,
      "activations/layer14_attention_weight_max": 32.66149139404297,
      "activations/layer14_attention_weight_min": -29.38417625427246,
      "activations/layer15_attention_weight_max": 28.769432067871094,
      "activations/layer15_attention_weight_min": -29.056718826293945,
      "activations/layer16_attention_weight_max": 31.002473831176758,
      "activations/layer16_attention_weight_min": -30.165014266967773,
      "activations/layer17_attention_weight_max": 48.51913070678711,
      "activations/layer17_attention_weight_min": -43.65082550048828,
      "activations/layer18_attention_weight_max": 41.316436767578125,
      "activations/layer18_attention_weight_min": -38.40174865722656,
      "activations/layer19_attention_weight_max": 22.574045181274414,
      "activations/layer19_attention_weight_min": -22.93050193786621,
      "activations/layer1_attention_weight_max": 16.930742263793945,
      "activations/layer1_attention_weight_min": -15.224448204040527,
      "activations/layer20_attention_weight_max": 20.07090187072754,
      "activations/layer20_attention_weight_min": -20.756181716918945,
      "activations/layer21_attention_weight_max": 30.62701988220215,
      "activations/layer21_attention_weight_min": -21.723291397094727,
      "activations/layer22_attention_weight_max": 28.28143310546875,
      "activations/layer22_attention_weight_min": -26.859052658081055,
      "activations/layer23_attention_weight_max": 37.96572494506836,
      "activations/layer23_attention_weight_min": -25.36757469177246,
      "activations/layer2_attention_weight_max": 31.673839569091797,
      "activations/layer2_attention_weight_min": -29.378028869628906,
      "activations/layer3_attention_weight_max": 89.83240509033203,
      "activations/layer3_attention_weight_min": -93.21063232421875,
      "activations/layer4_attention_weight_max": 91.12051391601562,
      "activations/layer4_attention_weight_min": -89.55105590820312,
      "activations/layer5_attention_weight_max": 64.27783203125,
      "activations/layer5_attention_weight_min": -70.48100280761719,
      "activations/layer6_attention_weight_max": 49.42062759399414,
      "activations/layer6_attention_weight_min": -50.24848175048828,
      "activations/layer7_attention_weight_max": 60.55764389038086,
      "activations/layer7_attention_weight_min": -60.305240631103516,
      "activations/layer8_attention_weight_max": 46.27640914916992,
      "activations/layer8_attention_weight_min": -48.83452606201172,
      "activations/layer9_attention_weight_max": 42.110504150390625,
      "activations/layer9_attention_weight_min": -46.42189025878906,
      "epoch": 16.05,
      "learning_rate": 4.697992424242424e-05,
      "loss": 2.7498,
      "step": 276200
    },
    {
      "activations/layer0_attention_weight_max": 15.646434783935547,
      "activations/layer0_attention_weight_min": -13.670815467834473,
      "activations/layer10_attention_weight_max": 36.01694107055664,
      "activations/layer10_attention_weight_min": -35.64250564575195,
      "activations/layer11_attention_weight_max": 31.486175537109375,
      "activations/layer11_attention_weight_min": -32.128501892089844,
      "activations/layer12_attention_weight_max": 18.412601470947266,
      "activations/layer12_attention_weight_min": -22.707090377807617,
      "activations/layer13_attention_weight_max": 33.56428146362305,
      "activations/layer13_attention_weight_min": -29.918916702270508,
      "activations/layer14_attention_weight_max": 35.9013557434082,
      "activations/layer14_attention_weight_min": -30.623804092407227,
      "activations/layer15_attention_weight_max": 31.677736282348633,
      "activations/layer15_attention_weight_min": -29.658855438232422,
      "activations/layer16_attention_weight_max": 30.73307228088379,
      "activations/layer16_attention_weight_min": -31.035701751708984,
      "activations/layer17_attention_weight_max": 47.60472869873047,
      "activations/layer17_attention_weight_min": -42.21843338012695,
      "activations/layer18_attention_weight_max": 45.79624938964844,
      "activations/layer18_attention_weight_min": -37.63209533691406,
      "activations/layer19_attention_weight_max": 23.009218215942383,
      "activations/layer19_attention_weight_min": -22.19780731201172,
      "activations/layer1_attention_weight_max": 16.130817413330078,
      "activations/layer1_attention_weight_min": -14.52227783203125,
      "activations/layer20_attention_weight_max": 22.155765533447266,
      "activations/layer20_attention_weight_min": -21.18210220336914,
      "activations/layer21_attention_weight_max": 33.48149490356445,
      "activations/layer21_attention_weight_min": -21.911808013916016,
      "activations/layer22_attention_weight_max": 28.17560577392578,
      "activations/layer22_attention_weight_min": -25.3247127532959,
      "activations/layer23_attention_weight_max": 36.53630828857422,
      "activations/layer23_attention_weight_min": -27.47786521911621,
      "activations/layer2_attention_weight_max": 29.831584930419922,
      "activations/layer2_attention_weight_min": -28.950401306152344,
      "activations/layer3_attention_weight_max": 83.72117614746094,
      "activations/layer3_attention_weight_min": -84.37727355957031,
      "activations/layer4_attention_weight_max": 89.06404113769531,
      "activations/layer4_attention_weight_min": -88.31224060058594,
      "activations/layer5_attention_weight_max": 65.59105682373047,
      "activations/layer5_attention_weight_min": -77.08692932128906,
      "activations/layer6_attention_weight_max": 50.448787689208984,
      "activations/layer6_attention_weight_min": -50.07591247558594,
      "activations/layer7_attention_weight_max": 64.2761459350586,
      "activations/layer7_attention_weight_min": -60.71974182128906,
      "activations/layer8_attention_weight_max": 47.155799865722656,
      "activations/layer8_attention_weight_min": -47.36317825317383,
      "activations/layer9_attention_weight_max": 43.63093948364258,
      "activations/layer9_attention_weight_min": -44.70068359375,
      "epoch": 16.05,
      "learning_rate": 4.696098484848484e-05,
      "loss": 2.7428,
      "step": 276250
    },
    {
      "activations/layer0_attention_weight_max": 15.283360481262207,
      "activations/layer0_attention_weight_min": -14.17171859741211,
      "activations/layer10_attention_weight_max": 37.57373046875,
      "activations/layer10_attention_weight_min": -37.107215881347656,
      "activations/layer11_attention_weight_max": 32.691001892089844,
      "activations/layer11_attention_weight_min": -33.91867446899414,
      "activations/layer12_attention_weight_max": 19.78700828552246,
      "activations/layer12_attention_weight_min": -26.146116256713867,
      "activations/layer13_attention_weight_max": 41.13322830200195,
      "activations/layer13_attention_weight_min": -28.0262451171875,
      "activations/layer14_attention_weight_max": 36.8629035949707,
      "activations/layer14_attention_weight_min": -30.68622589111328,
      "activations/layer15_attention_weight_max": 35.947288513183594,
      "activations/layer15_attention_weight_min": -30.535717010498047,
      "activations/layer16_attention_weight_max": 34.72978591918945,
      "activations/layer16_attention_weight_min": -30.64844512939453,
      "activations/layer17_attention_weight_max": 51.05536651611328,
      "activations/layer17_attention_weight_min": -44.554351806640625,
      "activations/layer18_attention_weight_max": 52.19086837768555,
      "activations/layer18_attention_weight_min": -38.16790771484375,
      "activations/layer19_attention_weight_max": 26.289926528930664,
      "activations/layer19_attention_weight_min": -23.12464714050293,
      "activations/layer1_attention_weight_max": 15.712252616882324,
      "activations/layer1_attention_weight_min": -15.680948257446289,
      "activations/layer20_attention_weight_max": 24.871143341064453,
      "activations/layer20_attention_weight_min": -20.082988739013672,
      "activations/layer21_attention_weight_max": 35.926387786865234,
      "activations/layer21_attention_weight_min": -20.953567504882812,
      "activations/layer22_attention_weight_max": 29.92704963684082,
      "activations/layer22_attention_weight_min": -28.62474250793457,
      "activations/layer23_attention_weight_max": 42.61296844482422,
      "activations/layer23_attention_weight_min": -25.220909118652344,
      "activations/layer2_attention_weight_max": 30.55879783630371,
      "activations/layer2_attention_weight_min": -30.71419334411621,
      "activations/layer3_attention_weight_max": 82.96261596679688,
      "activations/layer3_attention_weight_min": -82.84295654296875,
      "activations/layer4_attention_weight_max": 88.50865936279297,
      "activations/layer4_attention_weight_min": -85.77076721191406,
      "activations/layer5_attention_weight_max": 66.9755859375,
      "activations/layer5_attention_weight_min": -74.23535919189453,
      "activations/layer6_attention_weight_max": 50.14982604980469,
      "activations/layer6_attention_weight_min": -51.72050857543945,
      "activations/layer7_attention_weight_max": 63.26036071777344,
      "activations/layer7_attention_weight_min": -62.49993133544922,
      "activations/layer8_attention_weight_max": 49.89970779418945,
      "activations/layer8_attention_weight_min": -50.53596878051758,
      "activations/layer9_attention_weight_max": 46.8232536315918,
      "activations/layer9_attention_weight_min": -47.42377471923828,
      "epoch": 16.05,
      "learning_rate": 4.6942045454545454e-05,
      "loss": 2.7282,
      "step": 276300
    },
    {
      "activations/layer0_attention_weight_max": 15.194252014160156,
      "activations/layer0_attention_weight_min": -13.51895809173584,
      "activations/layer10_attention_weight_max": 37.48221969604492,
      "activations/layer10_attention_weight_min": -38.18289566040039,
      "activations/layer11_attention_weight_max": 34.984371185302734,
      "activations/layer11_attention_weight_min": -36.44805908203125,
      "activations/layer12_attention_weight_max": 17.558147430419922,
      "activations/layer12_attention_weight_min": -24.438034057617188,
      "activations/layer13_attention_weight_max": 36.5937614440918,
      "activations/layer13_attention_weight_min": -32.03935241699219,
      "activations/layer14_attention_weight_max": 34.600154876708984,
      "activations/layer14_attention_weight_min": -28.86582374572754,
      "activations/layer15_attention_weight_max": 35.4827880859375,
      "activations/layer15_attention_weight_min": -30.761520385742188,
      "activations/layer16_attention_weight_max": 32.21349334716797,
      "activations/layer16_attention_weight_min": -32.09152603149414,
      "activations/layer17_attention_weight_max": 49.35521697998047,
      "activations/layer17_attention_weight_min": -47.46699523925781,
      "activations/layer18_attention_weight_max": 47.75355911254883,
      "activations/layer18_attention_weight_min": -40.5634651184082,
      "activations/layer19_attention_weight_max": 21.47199058532715,
      "activations/layer19_attention_weight_min": -25.668399810791016,
      "activations/layer1_attention_weight_max": 16.1342830657959,
      "activations/layer1_attention_weight_min": -14.770681381225586,
      "activations/layer20_attention_weight_max": 21.583160400390625,
      "activations/layer20_attention_weight_min": -22.928813934326172,
      "activations/layer21_attention_weight_max": 31.95040512084961,
      "activations/layer21_attention_weight_min": -21.250709533691406,
      "activations/layer22_attention_weight_max": 26.97924041748047,
      "activations/layer22_attention_weight_min": -26.923730850219727,
      "activations/layer23_attention_weight_max": 31.04608154296875,
      "activations/layer23_attention_weight_min": -24.470989227294922,
      "activations/layer2_attention_weight_max": 31.298215866088867,
      "activations/layer2_attention_weight_min": -30.83979034423828,
      "activations/layer3_attention_weight_max": 86.7011489868164,
      "activations/layer3_attention_weight_min": -88.43436431884766,
      "activations/layer4_attention_weight_max": 91.03640747070312,
      "activations/layer4_attention_weight_min": -88.25984954833984,
      "activations/layer5_attention_weight_max": 64.47396850585938,
      "activations/layer5_attention_weight_min": -76.71576690673828,
      "activations/layer6_attention_weight_max": 49.992862701416016,
      "activations/layer6_attention_weight_min": -53.3599739074707,
      "activations/layer7_attention_weight_max": 61.619503021240234,
      "activations/layer7_attention_weight_min": -63.13592529296875,
      "activations/layer8_attention_weight_max": 48.30696487426758,
      "activations/layer8_attention_weight_min": -50.931739807128906,
      "activations/layer9_attention_weight_max": 47.63972091674805,
      "activations/layer9_attention_weight_min": -47.41512680053711,
      "epoch": 16.06,
      "learning_rate": 4.6923106060606056e-05,
      "loss": 2.7424,
      "step": 276350
    },
    {
      "activations/layer0_attention_weight_max": 15.71819019317627,
      "activations/layer0_attention_weight_min": -13.816911697387695,
      "activations/layer10_attention_weight_max": 36.1570930480957,
      "activations/layer10_attention_weight_min": -34.79927444458008,
      "activations/layer11_attention_weight_max": 34.10527038574219,
      "activations/layer11_attention_weight_min": -34.103919982910156,
      "activations/layer12_attention_weight_max": 19.8400936126709,
      "activations/layer12_attention_weight_min": -25.943933486938477,
      "activations/layer13_attention_weight_max": 33.60084533691406,
      "activations/layer13_attention_weight_min": -27.186006546020508,
      "activations/layer14_attention_weight_max": 35.69883728027344,
      "activations/layer14_attention_weight_min": -28.025314331054688,
      "activations/layer15_attention_weight_max": 33.608802795410156,
      "activations/layer15_attention_weight_min": -28.859867095947266,
      "activations/layer16_attention_weight_max": 32.448402404785156,
      "activations/layer16_attention_weight_min": -30.959623336791992,
      "activations/layer17_attention_weight_max": 49.659393310546875,
      "activations/layer17_attention_weight_min": -42.96971893310547,
      "activations/layer18_attention_weight_max": 46.88829040527344,
      "activations/layer18_attention_weight_min": -40.69071960449219,
      "activations/layer19_attention_weight_max": 22.553569793701172,
      "activations/layer19_attention_weight_min": -22.842180252075195,
      "activations/layer1_attention_weight_max": 15.636448860168457,
      "activations/layer1_attention_weight_min": -14.228117942810059,
      "activations/layer20_attention_weight_max": 21.64398193359375,
      "activations/layer20_attention_weight_min": -21.64200782775879,
      "activations/layer21_attention_weight_max": 35.31843948364258,
      "activations/layer21_attention_weight_min": -22.04212188720703,
      "activations/layer22_attention_weight_max": 28.606321334838867,
      "activations/layer22_attention_weight_min": -24.114017486572266,
      "activations/layer23_attention_weight_max": 36.8244514465332,
      "activations/layer23_attention_weight_min": -23.948575973510742,
      "activations/layer2_attention_weight_max": 33.30426788330078,
      "activations/layer2_attention_weight_min": -31.473508834838867,
      "activations/layer3_attention_weight_max": 85.83612823486328,
      "activations/layer3_attention_weight_min": -92.72742462158203,
      "activations/layer4_attention_weight_max": 91.84197235107422,
      "activations/layer4_attention_weight_min": -89.45122528076172,
      "activations/layer5_attention_weight_max": 67.60621643066406,
      "activations/layer5_attention_weight_min": -73.56275177001953,
      "activations/layer6_attention_weight_max": 49.52678680419922,
      "activations/layer6_attention_weight_min": -51.6058235168457,
      "activations/layer7_attention_weight_max": 64.13856506347656,
      "activations/layer7_attention_weight_min": -59.65986251831055,
      "activations/layer8_attention_weight_max": 44.110904693603516,
      "activations/layer8_attention_weight_min": -48.2237663269043,
      "activations/layer9_attention_weight_max": 43.80647277832031,
      "activations/layer9_attention_weight_min": -45.90278244018555,
      "epoch": 16.06,
      "learning_rate": 4.6904166666666664e-05,
      "loss": 2.7312,
      "step": 276400
    },
    {
      "activations/layer0_attention_weight_max": 15.819718360900879,
      "activations/layer0_attention_weight_min": -13.621281623840332,
      "activations/layer10_attention_weight_max": 39.006160736083984,
      "activations/layer10_attention_weight_min": -35.446128845214844,
      "activations/layer11_attention_weight_max": 32.57442092895508,
      "activations/layer11_attention_weight_min": -33.16973876953125,
      "activations/layer12_attention_weight_max": 18.470808029174805,
      "activations/layer12_attention_weight_min": -34.44126510620117,
      "activations/layer13_attention_weight_max": 38.748470306396484,
      "activations/layer13_attention_weight_min": -28.058238983154297,
      "activations/layer14_attention_weight_max": 33.727752685546875,
      "activations/layer14_attention_weight_min": -29.930070877075195,
      "activations/layer15_attention_weight_max": 29.349567413330078,
      "activations/layer15_attention_weight_min": -29.479766845703125,
      "activations/layer16_attention_weight_max": 29.726184844970703,
      "activations/layer16_attention_weight_min": -30.339488983154297,
      "activations/layer17_attention_weight_max": 46.38597106933594,
      "activations/layer17_attention_weight_min": -43.580345153808594,
      "activations/layer18_attention_weight_max": 44.43052291870117,
      "activations/layer18_attention_weight_min": -38.53362274169922,
      "activations/layer19_attention_weight_max": 22.226852416992188,
      "activations/layer19_attention_weight_min": -21.63953971862793,
      "activations/layer1_attention_weight_max": 15.694783210754395,
      "activations/layer1_attention_weight_min": -13.73887825012207,
      "activations/layer20_attention_weight_max": 21.710376739501953,
      "activations/layer20_attention_weight_min": -20.920429229736328,
      "activations/layer21_attention_weight_max": 31.317289352416992,
      "activations/layer21_attention_weight_min": -20.32796287536621,
      "activations/layer22_attention_weight_max": 28.57955551147461,
      "activations/layer22_attention_weight_min": -29.781044006347656,
      "activations/layer23_attention_weight_max": 33.86888885498047,
      "activations/layer23_attention_weight_min": -24.00478172302246,
      "activations/layer2_attention_weight_max": 32.15364074707031,
      "activations/layer2_attention_weight_min": -29.841114044189453,
      "activations/layer3_attention_weight_max": 87.5775146484375,
      "activations/layer3_attention_weight_min": -87.35356903076172,
      "activations/layer4_attention_weight_max": 92.45857238769531,
      "activations/layer4_attention_weight_min": -86.33512878417969,
      "activations/layer5_attention_weight_max": 64.0982437133789,
      "activations/layer5_attention_weight_min": -73.97174072265625,
      "activations/layer6_attention_weight_max": 49.3195686340332,
      "activations/layer6_attention_weight_min": -51.10567092895508,
      "activations/layer7_attention_weight_max": 60.3682975769043,
      "activations/layer7_attention_weight_min": -60.64045715332031,
      "activations/layer8_attention_weight_max": 48.1703987121582,
      "activations/layer8_attention_weight_min": -48.71712875366211,
      "activations/layer9_attention_weight_max": 48.87217330932617,
      "activations/layer9_attention_weight_min": -50.74184036254883,
      "epoch": 16.06,
      "learning_rate": 4.6885227272727265e-05,
      "loss": 2.7651,
      "step": 276450
    },
    {
      "activations/layer0_attention_weight_max": 14.98598861694336,
      "activations/layer0_attention_weight_min": -14.411396980285645,
      "activations/layer10_attention_weight_max": 40.42498779296875,
      "activations/layer10_attention_weight_min": -36.664371490478516,
      "activations/layer11_attention_weight_max": 34.86320877075195,
      "activations/layer11_attention_weight_min": -34.60752868652344,
      "activations/layer12_attention_weight_max": 17.671228408813477,
      "activations/layer12_attention_weight_min": -24.7255802154541,
      "activations/layer13_attention_weight_max": 33.41600799560547,
      "activations/layer13_attention_weight_min": -29.819856643676758,
      "activations/layer14_attention_weight_max": 34.839393615722656,
      "activations/layer14_attention_weight_min": -29.346065521240234,
      "activations/layer15_attention_weight_max": 35.1911506652832,
      "activations/layer15_attention_weight_min": -29.128137588500977,
      "activations/layer16_attention_weight_max": 35.07167434692383,
      "activations/layer16_attention_weight_min": -33.44181823730469,
      "activations/layer17_attention_weight_max": 54.8358268737793,
      "activations/layer17_attention_weight_min": -46.85630416870117,
      "activations/layer18_attention_weight_max": 47.508785247802734,
      "activations/layer18_attention_weight_min": -40.27880859375,
      "activations/layer19_attention_weight_max": 22.745712280273438,
      "activations/layer19_attention_weight_min": -21.277301788330078,
      "activations/layer1_attention_weight_max": 16.76521873474121,
      "activations/layer1_attention_weight_min": -15.206478118896484,
      "activations/layer20_attention_weight_max": 23.082719802856445,
      "activations/layer20_attention_weight_min": -20.51112937927246,
      "activations/layer21_attention_weight_max": 32.425987243652344,
      "activations/layer21_attention_weight_min": -22.21323013305664,
      "activations/layer22_attention_weight_max": 29.397926330566406,
      "activations/layer22_attention_weight_min": -25.896223068237305,
      "activations/layer23_attention_weight_max": 34.95729064941406,
      "activations/layer23_attention_weight_min": -22.903188705444336,
      "activations/layer2_attention_weight_max": 30.690168380737305,
      "activations/layer2_attention_weight_min": -30.50457763671875,
      "activations/layer3_attention_weight_max": 90.09600067138672,
      "activations/layer3_attention_weight_min": -91.95458984375,
      "activations/layer4_attention_weight_max": 90.41510772705078,
      "activations/layer4_attention_weight_min": -86.29827880859375,
      "activations/layer5_attention_weight_max": 68.42168426513672,
      "activations/layer5_attention_weight_min": -71.29200744628906,
      "activations/layer6_attention_weight_max": 51.97727966308594,
      "activations/layer6_attention_weight_min": -53.139347076416016,
      "activations/layer7_attention_weight_max": 64.12969207763672,
      "activations/layer7_attention_weight_min": -60.00188446044922,
      "activations/layer8_attention_weight_max": 49.2576904296875,
      "activations/layer8_attention_weight_min": -51.442626953125,
      "activations/layer9_attention_weight_max": 44.62596893310547,
      "activations/layer9_attention_weight_min": -45.84865951538086,
      "epoch": 16.07,
      "learning_rate": 4.686628787878788e-05,
      "loss": 2.7449,
      "step": 276500
    },
    {
      "activations/layer0_attention_weight_max": 16.273893356323242,
      "activations/layer0_attention_weight_min": -14.214506149291992,
      "activations/layer10_attention_weight_max": 43.635250091552734,
      "activations/layer10_attention_weight_min": -41.89194107055664,
      "activations/layer11_attention_weight_max": 40.456871032714844,
      "activations/layer11_attention_weight_min": -37.419708251953125,
      "activations/layer12_attention_weight_max": 20.26246452331543,
      "activations/layer12_attention_weight_min": -28.26050567626953,
      "activations/layer13_attention_weight_max": 45.222896575927734,
      "activations/layer13_attention_weight_min": -31.047752380371094,
      "activations/layer14_attention_weight_max": 49.111289978027344,
      "activations/layer14_attention_weight_min": -32.47428512573242,
      "activations/layer15_attention_weight_max": 39.36853790283203,
      "activations/layer15_attention_weight_min": -28.956764221191406,
      "activations/layer16_attention_weight_max": 37.95664978027344,
      "activations/layer16_attention_weight_min": -33.317237854003906,
      "activations/layer17_attention_weight_max": 58.57651901245117,
      "activations/layer17_attention_weight_min": -46.124122619628906,
      "activations/layer18_attention_weight_max": 55.8519172668457,
      "activations/layer18_attention_weight_min": -41.2604866027832,
      "activations/layer19_attention_weight_max": 28.613096237182617,
      "activations/layer19_attention_weight_min": -23.50495719909668,
      "activations/layer1_attention_weight_max": 16.120281219482422,
      "activations/layer1_attention_weight_min": -13.734403610229492,
      "activations/layer20_attention_weight_max": 30.414289474487305,
      "activations/layer20_attention_weight_min": -23.010578155517578,
      "activations/layer21_attention_weight_max": 49.50711441040039,
      "activations/layer21_attention_weight_min": -22.160688400268555,
      "activations/layer22_attention_weight_max": 32.50462341308594,
      "activations/layer22_attention_weight_min": -27.24314308166504,
      "activations/layer23_attention_weight_max": 43.90541076660156,
      "activations/layer23_attention_weight_min": -22.66445541381836,
      "activations/layer2_attention_weight_max": 30.3951416015625,
      "activations/layer2_attention_weight_min": -30.73445701599121,
      "activations/layer3_attention_weight_max": 85.78890228271484,
      "activations/layer3_attention_weight_min": -89.96495819091797,
      "activations/layer4_attention_weight_max": 92.35098266601562,
      "activations/layer4_attention_weight_min": -90.59942626953125,
      "activations/layer5_attention_weight_max": 64.98782348632812,
      "activations/layer5_attention_weight_min": -73.89166259765625,
      "activations/layer6_attention_weight_max": 54.948509216308594,
      "activations/layer6_attention_weight_min": -53.21096420288086,
      "activations/layer7_attention_weight_max": 63.94268798828125,
      "activations/layer7_attention_weight_min": -67.70860290527344,
      "activations/layer8_attention_weight_max": 55.06210708618164,
      "activations/layer8_attention_weight_min": -53.981319427490234,
      "activations/layer9_attention_weight_max": 50.5066032409668,
      "activations/layer9_attention_weight_min": -48.79368209838867,
      "epoch": 16.07,
      "learning_rate": 4.684734848484848e-05,
      "loss": 2.7444,
      "step": 276550
    },
    {
      "activations/layer0_attention_weight_max": 15.956701278686523,
      "activations/layer0_attention_weight_min": -13.861262321472168,
      "activations/layer10_attention_weight_max": 40.01881790161133,
      "activations/layer10_attention_weight_min": -37.20001220703125,
      "activations/layer11_attention_weight_max": 33.68115234375,
      "activations/layer11_attention_weight_min": -33.98521041870117,
      "activations/layer12_attention_weight_max": 18.720991134643555,
      "activations/layer12_attention_weight_min": -25.6661434173584,
      "activations/layer13_attention_weight_max": 32.134254455566406,
      "activations/layer13_attention_weight_min": -29.633249282836914,
      "activations/layer14_attention_weight_max": 36.663185119628906,
      "activations/layer14_attention_weight_min": -30.401670455932617,
      "activations/layer15_attention_weight_max": 32.94649124145508,
      "activations/layer15_attention_weight_min": -30.01319122314453,
      "activations/layer16_attention_weight_max": 33.46979522705078,
      "activations/layer16_attention_weight_min": -33.313743591308594,
      "activations/layer17_attention_weight_max": 48.50800323486328,
      "activations/layer17_attention_weight_min": -47.01349639892578,
      "activations/layer18_attention_weight_max": 48.833072662353516,
      "activations/layer18_attention_weight_min": -40.75679016113281,
      "activations/layer19_attention_weight_max": 22.592506408691406,
      "activations/layer19_attention_weight_min": -23.29794692993164,
      "activations/layer1_attention_weight_max": 15.89083194732666,
      "activations/layer1_attention_weight_min": -15.003040313720703,
      "activations/layer20_attention_weight_max": 20.628437042236328,
      "activations/layer20_attention_weight_min": -21.58318328857422,
      "activations/layer21_attention_weight_max": 32.796356201171875,
      "activations/layer21_attention_weight_min": -21.555917739868164,
      "activations/layer22_attention_weight_max": 29.13072395324707,
      "activations/layer22_attention_weight_min": -27.136844635009766,
      "activations/layer23_attention_weight_max": 36.30305480957031,
      "activations/layer23_attention_weight_min": -26.336952209472656,
      "activations/layer2_attention_weight_max": 30.243101119995117,
      "activations/layer2_attention_weight_min": -29.776365280151367,
      "activations/layer3_attention_weight_max": 87.10205078125,
      "activations/layer3_attention_weight_min": -87.9942398071289,
      "activations/layer4_attention_weight_max": 91.09146881103516,
      "activations/layer4_attention_weight_min": -85.9168701171875,
      "activations/layer5_attention_weight_max": 65.08193969726562,
      "activations/layer5_attention_weight_min": -75.2459945678711,
      "activations/layer6_attention_weight_max": 52.52594757080078,
      "activations/layer6_attention_weight_min": -54.52890396118164,
      "activations/layer7_attention_weight_max": 63.74455642700195,
      "activations/layer7_attention_weight_min": -61.35625457763672,
      "activations/layer8_attention_weight_max": 52.16210174560547,
      "activations/layer8_attention_weight_min": -56.67768859863281,
      "activations/layer9_attention_weight_max": 47.0251579284668,
      "activations/layer9_attention_weight_min": -49.001705169677734,
      "epoch": 16.07,
      "learning_rate": 4.682840909090908e-05,
      "loss": 2.7534,
      "step": 276600
    },
    {
      "activations/layer0_attention_weight_max": 15.710711479187012,
      "activations/layer0_attention_weight_min": -14.024930953979492,
      "activations/layer10_attention_weight_max": 36.47100830078125,
      "activations/layer10_attention_weight_min": -37.84124755859375,
      "activations/layer11_attention_weight_max": 32.35464859008789,
      "activations/layer11_attention_weight_min": -35.93280792236328,
      "activations/layer12_attention_weight_max": 18.913259506225586,
      "activations/layer12_attention_weight_min": -31.054906845092773,
      "activations/layer13_attention_weight_max": 32.827571868896484,
      "activations/layer13_attention_weight_min": -28.41098403930664,
      "activations/layer14_attention_weight_max": 36.16567611694336,
      "activations/layer14_attention_weight_min": -30.35308265686035,
      "activations/layer15_attention_weight_max": 35.87868881225586,
      "activations/layer15_attention_weight_min": -29.12945556640625,
      "activations/layer16_attention_weight_max": 34.01801300048828,
      "activations/layer16_attention_weight_min": -34.10127639770508,
      "activations/layer17_attention_weight_max": 52.30564498901367,
      "activations/layer17_attention_weight_min": -50.538368225097656,
      "activations/layer18_attention_weight_max": 49.18460464477539,
      "activations/layer18_attention_weight_min": -47.456790924072266,
      "activations/layer19_attention_weight_max": 23.36972999572754,
      "activations/layer19_attention_weight_min": -26.378204345703125,
      "activations/layer1_attention_weight_max": 15.489896774291992,
      "activations/layer1_attention_weight_min": -14.188990592956543,
      "activations/layer20_attention_weight_max": 23.485347747802734,
      "activations/layer20_attention_weight_min": -23.4495849609375,
      "activations/layer21_attention_weight_max": 39.301822662353516,
      "activations/layer21_attention_weight_min": -24.960403442382812,
      "activations/layer22_attention_weight_max": 28.61772918701172,
      "activations/layer22_attention_weight_min": -25.934534072875977,
      "activations/layer23_attention_weight_max": 37.18075180053711,
      "activations/layer23_attention_weight_min": -24.550861358642578,
      "activations/layer2_attention_weight_max": 30.05345916748047,
      "activations/layer2_attention_weight_min": -29.451452255249023,
      "activations/layer3_attention_weight_max": 83.98045349121094,
      "activations/layer3_attention_weight_min": -83.55371856689453,
      "activations/layer4_attention_weight_max": 87.80968475341797,
      "activations/layer4_attention_weight_min": -86.51968383789062,
      "activations/layer5_attention_weight_max": 69.30467224121094,
      "activations/layer5_attention_weight_min": -70.80014038085938,
      "activations/layer6_attention_weight_max": 50.21881866455078,
      "activations/layer6_attention_weight_min": -51.1587028503418,
      "activations/layer7_attention_weight_max": 62.93483352661133,
      "activations/layer7_attention_weight_min": -61.888580322265625,
      "activations/layer8_attention_weight_max": 46.15602493286133,
      "activations/layer8_attention_weight_min": -48.6357307434082,
      "activations/layer9_attention_weight_max": 43.927425384521484,
      "activations/layer9_attention_weight_min": -47.803367614746094,
      "epoch": 16.07,
      "learning_rate": 4.680946969696969e-05,
      "loss": 2.7474,
      "step": 276650
    },
    {
      "activations/layer0_attention_weight_max": 15.696548461914062,
      "activations/layer0_attention_weight_min": -14.304832458496094,
      "activations/layer10_attention_weight_max": 35.84619903564453,
      "activations/layer10_attention_weight_min": -33.300758361816406,
      "activations/layer11_attention_weight_max": 32.43822479248047,
      "activations/layer11_attention_weight_min": -33.00544738769531,
      "activations/layer12_attention_weight_max": 18.826444625854492,
      "activations/layer12_attention_weight_min": -24.597103118896484,
      "activations/layer13_attention_weight_max": 29.87840461730957,
      "activations/layer13_attention_weight_min": -26.497783660888672,
      "activations/layer14_attention_weight_max": 34.37644958496094,
      "activations/layer14_attention_weight_min": -32.39576721191406,
      "activations/layer15_attention_weight_max": 29.884252548217773,
      "activations/layer15_attention_weight_min": -29.121206283569336,
      "activations/layer16_attention_weight_max": 34.04509735107422,
      "activations/layer16_attention_weight_min": -33.83892822265625,
      "activations/layer17_attention_weight_max": 49.567283630371094,
      "activations/layer17_attention_weight_min": -52.29248809814453,
      "activations/layer18_attention_weight_max": 47.63547897338867,
      "activations/layer18_attention_weight_min": -46.990482330322266,
      "activations/layer19_attention_weight_max": 24.881620407104492,
      "activations/layer19_attention_weight_min": -27.662933349609375,
      "activations/layer1_attention_weight_max": 16.539276123046875,
      "activations/layer1_attention_weight_min": -14.297100067138672,
      "activations/layer20_attention_weight_max": 25.143964767456055,
      "activations/layer20_attention_weight_min": -25.635175704956055,
      "activations/layer21_attention_weight_max": 34.68466567993164,
      "activations/layer21_attention_weight_min": -25.099077224731445,
      "activations/layer22_attention_weight_max": 27.910356521606445,
      "activations/layer22_attention_weight_min": -27.6639404296875,
      "activations/layer23_attention_weight_max": 33.23242950439453,
      "activations/layer23_attention_weight_min": -26.991613388061523,
      "activations/layer2_attention_weight_max": 31.30794906616211,
      "activations/layer2_attention_weight_min": -30.280662536621094,
      "activations/layer3_attention_weight_max": 85.6698226928711,
      "activations/layer3_attention_weight_min": -85.69601440429688,
      "activations/layer4_attention_weight_max": 91.48828887939453,
      "activations/layer4_attention_weight_min": -85.89508819580078,
      "activations/layer5_attention_weight_max": 65.7688980102539,
      "activations/layer5_attention_weight_min": -71.41123962402344,
      "activations/layer6_attention_weight_max": 52.16957092285156,
      "activations/layer6_attention_weight_min": -52.72834396362305,
      "activations/layer7_attention_weight_max": 60.022945404052734,
      "activations/layer7_attention_weight_min": -60.483673095703125,
      "activations/layer8_attention_weight_max": 44.45045852661133,
      "activations/layer8_attention_weight_min": -46.40693283081055,
      "activations/layer9_attention_weight_max": 42.72834777832031,
      "activations/layer9_attention_weight_min": -44.97098922729492,
      "epoch": 16.08,
      "learning_rate": 4.67905303030303e-05,
      "loss": 2.7394,
      "step": 276700
    },
    {
      "activations/layer0_attention_weight_max": 15.768256187438965,
      "activations/layer0_attention_weight_min": -13.930041313171387,
      "activations/layer10_attention_weight_max": 38.609798431396484,
      "activations/layer10_attention_weight_min": -36.49753189086914,
      "activations/layer11_attention_weight_max": 34.758148193359375,
      "activations/layer11_attention_weight_min": -34.25860595703125,
      "activations/layer12_attention_weight_max": 19.051401138305664,
      "activations/layer12_attention_weight_min": -24.266067504882812,
      "activations/layer13_attention_weight_max": 39.46018600463867,
      "activations/layer13_attention_weight_min": -29.095970153808594,
      "activations/layer14_attention_weight_max": 35.38264083862305,
      "activations/layer14_attention_weight_min": -33.49856185913086,
      "activations/layer15_attention_weight_max": 33.24347686767578,
      "activations/layer15_attention_weight_min": -33.95540237426758,
      "activations/layer16_attention_weight_max": 34.23716354370117,
      "activations/layer16_attention_weight_min": -33.297264099121094,
      "activations/layer17_attention_weight_max": 51.503456115722656,
      "activations/layer17_attention_weight_min": -49.411216735839844,
      "activations/layer18_attention_weight_max": 47.63835906982422,
      "activations/layer18_attention_weight_min": -40.099830627441406,
      "activations/layer19_attention_weight_max": 27.157682418823242,
      "activations/layer19_attention_weight_min": -22.531591415405273,
      "activations/layer1_attention_weight_max": 16.03727149963379,
      "activations/layer1_attention_weight_min": -16.66844940185547,
      "activations/layer20_attention_weight_max": 27.458232879638672,
      "activations/layer20_attention_weight_min": -21.13553810119629,
      "activations/layer21_attention_weight_max": 39.208534240722656,
      "activations/layer21_attention_weight_min": -23.207239151000977,
      "activations/layer22_attention_weight_max": 29.749799728393555,
      "activations/layer22_attention_weight_min": -24.39679527282715,
      "activations/layer23_attention_weight_max": 40.34925842285156,
      "activations/layer23_attention_weight_min": -23.8480224609375,
      "activations/layer2_attention_weight_max": 30.17501449584961,
      "activations/layer2_attention_weight_min": -30.41956901550293,
      "activations/layer3_attention_weight_max": 87.48817443847656,
      "activations/layer3_attention_weight_min": -86.17124938964844,
      "activations/layer4_attention_weight_max": 89.15019989013672,
      "activations/layer4_attention_weight_min": -88.65889739990234,
      "activations/layer5_attention_weight_max": 63.563331604003906,
      "activations/layer5_attention_weight_min": -74.98707580566406,
      "activations/layer6_attention_weight_max": 48.55543899536133,
      "activations/layer6_attention_weight_min": -50.41181564331055,
      "activations/layer7_attention_weight_max": 62.5050163269043,
      "activations/layer7_attention_weight_min": -59.143211364746094,
      "activations/layer8_attention_weight_max": 48.387977600097656,
      "activations/layer8_attention_weight_min": -46.809932708740234,
      "activations/layer9_attention_weight_max": 49.18915939331055,
      "activations/layer9_attention_weight_min": -44.03846740722656,
      "epoch": 16.08,
      "learning_rate": 4.677159090909091e-05,
      "loss": 2.7562,
      "step": 276750
    },
    {
      "activations/layer0_attention_weight_max": 15.62611198425293,
      "activations/layer0_attention_weight_min": -14.117931365966797,
      "activations/layer10_attention_weight_max": 34.92829513549805,
      "activations/layer10_attention_weight_min": -37.21034622192383,
      "activations/layer11_attention_weight_max": 34.605743408203125,
      "activations/layer11_attention_weight_min": -35.38792419433594,
      "activations/layer12_attention_weight_max": 17.13223648071289,
      "activations/layer12_attention_weight_min": -31.60316276550293,
      "activations/layer13_attention_weight_max": 30.806859970092773,
      "activations/layer13_attention_weight_min": -24.997114181518555,
      "activations/layer14_attention_weight_max": 32.240135192871094,
      "activations/layer14_attention_weight_min": -28.291051864624023,
      "activations/layer15_attention_weight_max": 30.321496963500977,
      "activations/layer15_attention_weight_min": -27.47321128845215,
      "activations/layer16_attention_weight_max": 29.36153221130371,
      "activations/layer16_attention_weight_min": -29.205957412719727,
      "activations/layer17_attention_weight_max": 47.25885772705078,
      "activations/layer17_attention_weight_min": -41.2641716003418,
      "activations/layer18_attention_weight_max": 40.998329162597656,
      "activations/layer18_attention_weight_min": -37.7720832824707,
      "activations/layer19_attention_weight_max": 22.01624298095703,
      "activations/layer19_attention_weight_min": -20.12433624267578,
      "activations/layer1_attention_weight_max": 16.138019561767578,
      "activations/layer1_attention_weight_min": -14.361479759216309,
      "activations/layer20_attention_weight_max": 21.097209930419922,
      "activations/layer20_attention_weight_min": -20.87421226501465,
      "activations/layer21_attention_weight_max": 29.501983642578125,
      "activations/layer21_attention_weight_min": -20.826364517211914,
      "activations/layer22_attention_weight_max": 27.99183464050293,
      "activations/layer22_attention_weight_min": -24.56451988220215,
      "activations/layer23_attention_weight_max": 36.56449890136719,
      "activations/layer23_attention_weight_min": -24.279844284057617,
      "activations/layer2_attention_weight_max": 32.5411376953125,
      "activations/layer2_attention_weight_min": -29.955379486083984,
      "activations/layer3_attention_weight_max": 92.76116943359375,
      "activations/layer3_attention_weight_min": -92.13837432861328,
      "activations/layer4_attention_weight_max": 95.93643951416016,
      "activations/layer4_attention_weight_min": -86.75761413574219,
      "activations/layer5_attention_weight_max": 64.70069885253906,
      "activations/layer5_attention_weight_min": -68.46495056152344,
      "activations/layer6_attention_weight_max": 48.175636291503906,
      "activations/layer6_attention_weight_min": -48.84791946411133,
      "activations/layer7_attention_weight_max": 62.13816833496094,
      "activations/layer7_attention_weight_min": -56.127567291259766,
      "activations/layer8_attention_weight_max": 44.65693664550781,
      "activations/layer8_attention_weight_min": -47.99370193481445,
      "activations/layer9_attention_weight_max": 53.75088119506836,
      "activations/layer9_attention_weight_min": -44.176761627197266,
      "epoch": 16.08,
      "learning_rate": 4.675265151515151e-05,
      "loss": 2.7406,
      "step": 276800
    },
    {
      "activations/layer0_attention_weight_max": 15.328670501708984,
      "activations/layer0_attention_weight_min": -13.997759819030762,
      "activations/layer10_attention_weight_max": 38.27294158935547,
      "activations/layer10_attention_weight_min": -38.51927947998047,
      "activations/layer11_attention_weight_max": 35.501380920410156,
      "activations/layer11_attention_weight_min": -34.457096099853516,
      "activations/layer12_attention_weight_max": 18.429059982299805,
      "activations/layer12_attention_weight_min": -24.4412784576416,
      "activations/layer13_attention_weight_max": 34.8486328125,
      "activations/layer13_attention_weight_min": -28.670963287353516,
      "activations/layer14_attention_weight_max": 36.343284606933594,
      "activations/layer14_attention_weight_min": -28.97358512878418,
      "activations/layer15_attention_weight_max": 33.42351531982422,
      "activations/layer15_attention_weight_min": -30.174312591552734,
      "activations/layer16_attention_weight_max": 34.49625015258789,
      "activations/layer16_attention_weight_min": -30.64512062072754,
      "activations/layer17_attention_weight_max": 51.480140686035156,
      "activations/layer17_attention_weight_min": -42.33317947387695,
      "activations/layer18_attention_weight_max": 47.27545166015625,
      "activations/layer18_attention_weight_min": -38.91514205932617,
      "activations/layer19_attention_weight_max": 23.64024543762207,
      "activations/layer19_attention_weight_min": -22.08889389038086,
      "activations/layer1_attention_weight_max": 16.00313949584961,
      "activations/layer1_attention_weight_min": -14.332769393920898,
      "activations/layer20_attention_weight_max": 22.045757293701172,
      "activations/layer20_attention_weight_min": -22.00352668762207,
      "activations/layer21_attention_weight_max": 33.42320251464844,
      "activations/layer21_attention_weight_min": -20.447790145874023,
      "activations/layer22_attention_weight_max": 33.11290740966797,
      "activations/layer22_attention_weight_min": -25.49012565612793,
      "activations/layer23_attention_weight_max": 36.07080841064453,
      "activations/layer23_attention_weight_min": -23.78323745727539,
      "activations/layer2_attention_weight_max": 30.1228084564209,
      "activations/layer2_attention_weight_min": -30.481117248535156,
      "activations/layer3_attention_weight_max": 84.58055877685547,
      "activations/layer3_attention_weight_min": -84.96530151367188,
      "activations/layer4_attention_weight_max": 89.12592315673828,
      "activations/layer4_attention_weight_min": -87.38510131835938,
      "activations/layer5_attention_weight_max": 62.9069938659668,
      "activations/layer5_attention_weight_min": -77.41143798828125,
      "activations/layer6_attention_weight_max": 50.773502349853516,
      "activations/layer6_attention_weight_min": -53.81283950805664,
      "activations/layer7_attention_weight_max": 63.38746643066406,
      "activations/layer7_attention_weight_min": -66.93553161621094,
      "activations/layer8_attention_weight_max": 49.93710708618164,
      "activations/layer8_attention_weight_min": -51.00926971435547,
      "activations/layer9_attention_weight_max": 50.48021697998047,
      "activations/layer9_attention_weight_min": -49.46543502807617,
      "epoch": 16.09,
      "learning_rate": 4.673371212121211e-05,
      "loss": 2.744,
      "step": 276850
    },
    {
      "activations/layer0_attention_weight_max": 15.54060173034668,
      "activations/layer0_attention_weight_min": -14.462546348571777,
      "activations/layer10_attention_weight_max": 37.16831588745117,
      "activations/layer10_attention_weight_min": -37.531715393066406,
      "activations/layer11_attention_weight_max": 34.18834686279297,
      "activations/layer11_attention_weight_min": -36.09339141845703,
      "activations/layer12_attention_weight_max": 19.137290954589844,
      "activations/layer12_attention_weight_min": -24.777488708496094,
      "activations/layer13_attention_weight_max": 38.813880920410156,
      "activations/layer13_attention_weight_min": -28.637727737426758,
      "activations/layer14_attention_weight_max": 42.56871795654297,
      "activations/layer14_attention_weight_min": -34.089412689208984,
      "activations/layer15_attention_weight_max": 41.867374420166016,
      "activations/layer15_attention_weight_min": -31.527748107910156,
      "activations/layer16_attention_weight_max": 36.739105224609375,
      "activations/layer16_attention_weight_min": -35.21339416503906,
      "activations/layer17_attention_weight_max": 63.5703010559082,
      "activations/layer17_attention_weight_min": -50.18205642700195,
      "activations/layer18_attention_weight_max": 53.862953186035156,
      "activations/layer18_attention_weight_min": -42.63459777832031,
      "activations/layer19_attention_weight_max": 24.878416061401367,
      "activations/layer19_attention_weight_min": -25.22247886657715,
      "activations/layer1_attention_weight_max": 16.364730834960938,
      "activations/layer1_attention_weight_min": -15.580299377441406,
      "activations/layer20_attention_weight_max": 24.098575592041016,
      "activations/layer20_attention_weight_min": -21.318954467773438,
      "activations/layer21_attention_weight_max": 40.8297004699707,
      "activations/layer21_attention_weight_min": -23.47173309326172,
      "activations/layer22_attention_weight_max": 34.907440185546875,
      "activations/layer22_attention_weight_min": -25.981536865234375,
      "activations/layer23_attention_weight_max": 40.32344436645508,
      "activations/layer23_attention_weight_min": -23.07339859008789,
      "activations/layer2_attention_weight_max": 30.380783081054688,
      "activations/layer2_attention_weight_min": -28.91387939453125,
      "activations/layer3_attention_weight_max": 86.98174285888672,
      "activations/layer3_attention_weight_min": -87.95508575439453,
      "activations/layer4_attention_weight_max": 93.96481323242188,
      "activations/layer4_attention_weight_min": -90.46468353271484,
      "activations/layer5_attention_weight_max": 67.13188171386719,
      "activations/layer5_attention_weight_min": -74.1009750366211,
      "activations/layer6_attention_weight_max": 52.840919494628906,
      "activations/layer6_attention_weight_min": -51.259742736816406,
      "activations/layer7_attention_weight_max": 62.801265716552734,
      "activations/layer7_attention_weight_min": -62.50962448120117,
      "activations/layer8_attention_weight_max": 49.866539001464844,
      "activations/layer8_attention_weight_min": -51.15370559692383,
      "activations/layer9_attention_weight_max": 46.445552825927734,
      "activations/layer9_attention_weight_min": -46.495540618896484,
      "epoch": 16.09,
      "learning_rate": 4.6714772727272726e-05,
      "loss": 2.7375,
      "step": 276900
    },
    {
      "activations/layer0_attention_weight_max": 16.140424728393555,
      "activations/layer0_attention_weight_min": -14.044657707214355,
      "activations/layer10_attention_weight_max": 35.232818603515625,
      "activations/layer10_attention_weight_min": -35.771484375,
      "activations/layer11_attention_weight_max": 32.29920959472656,
      "activations/layer11_attention_weight_min": -32.36621856689453,
      "activations/layer12_attention_weight_max": 18.16611099243164,
      "activations/layer12_attention_weight_min": -22.185672760009766,
      "activations/layer13_attention_weight_max": 35.43246841430664,
      "activations/layer13_attention_weight_min": -29.10607147216797,
      "activations/layer14_attention_weight_max": 33.262176513671875,
      "activations/layer14_attention_weight_min": -28.51406478881836,
      "activations/layer15_attention_weight_max": 31.669391632080078,
      "activations/layer15_attention_weight_min": -29.557371139526367,
      "activations/layer16_attention_weight_max": 30.077272415161133,
      "activations/layer16_attention_weight_min": -30.33170509338379,
      "activations/layer17_attention_weight_max": 48.71846008300781,
      "activations/layer17_attention_weight_min": -42.51036071777344,
      "activations/layer18_attention_weight_max": 43.302032470703125,
      "activations/layer18_attention_weight_min": -37.153770446777344,
      "activations/layer19_attention_weight_max": 21.5233154296875,
      "activations/layer19_attention_weight_min": -21.626909255981445,
      "activations/layer1_attention_weight_max": 16.234384536743164,
      "activations/layer1_attention_weight_min": -13.756604194641113,
      "activations/layer20_attention_weight_max": 21.529855728149414,
      "activations/layer20_attention_weight_min": -22.281110763549805,
      "activations/layer21_attention_weight_max": 29.951566696166992,
      "activations/layer21_attention_weight_min": -23.584325790405273,
      "activations/layer22_attention_weight_max": 27.706071853637695,
      "activations/layer22_attention_weight_min": -25.91805648803711,
      "activations/layer23_attention_weight_max": 32.92034912109375,
      "activations/layer23_attention_weight_min": -25.98870849609375,
      "activations/layer2_attention_weight_max": 31.585676193237305,
      "activations/layer2_attention_weight_min": -32.018795013427734,
      "activations/layer3_attention_weight_max": 89.02188110351562,
      "activations/layer3_attention_weight_min": -94.8166275024414,
      "activations/layer4_attention_weight_max": 91.75801849365234,
      "activations/layer4_attention_weight_min": -88.72364807128906,
      "activations/layer5_attention_weight_max": 63.630714416503906,
      "activations/layer5_attention_weight_min": -72.03389739990234,
      "activations/layer6_attention_weight_max": 48.44341278076172,
      "activations/layer6_attention_weight_min": -50.34431457519531,
      "activations/layer7_attention_weight_max": 64.69019317626953,
      "activations/layer7_attention_weight_min": -61.23820114135742,
      "activations/layer8_attention_weight_max": 42.244319915771484,
      "activations/layer8_attention_weight_min": -48.2138557434082,
      "activations/layer9_attention_weight_max": 41.8703727722168,
      "activations/layer9_attention_weight_min": -44.05460739135742,
      "epoch": 16.09,
      "learning_rate": 4.669583333333333e-05,
      "loss": 2.7603,
      "step": 276950
    },
    {
      "activations/layer0_attention_weight_max": 15.134733200073242,
      "activations/layer0_attention_weight_min": -14.064319610595703,
      "activations/layer10_attention_weight_max": 34.39940643310547,
      "activations/layer10_attention_weight_min": -33.57953643798828,
      "activations/layer11_attention_weight_max": 30.899518966674805,
      "activations/layer11_attention_weight_min": -31.70387077331543,
      "activations/layer12_attention_weight_max": 18.813093185424805,
      "activations/layer12_attention_weight_min": -27.00112533569336,
      "activations/layer13_attention_weight_max": 28.276325225830078,
      "activations/layer13_attention_weight_min": -25.72210121154785,
      "activations/layer14_attention_weight_max": 30.838298797607422,
      "activations/layer14_attention_weight_min": -28.960180282592773,
      "activations/layer15_attention_weight_max": 29.783628463745117,
      "activations/layer15_attention_weight_min": -27.63939094543457,
      "activations/layer16_attention_weight_max": 27.872716903686523,
      "activations/layer16_attention_weight_min": -29.099388122558594,
      "activations/layer17_attention_weight_max": 44.050323486328125,
      "activations/layer17_attention_weight_min": -40.241764068603516,
      "activations/layer18_attention_weight_max": 39.98091506958008,
      "activations/layer18_attention_weight_min": -37.013954162597656,
      "activations/layer19_attention_weight_max": 20.479183197021484,
      "activations/layer19_attention_weight_min": -24.784841537475586,
      "activations/layer1_attention_weight_max": 15.412251472473145,
      "activations/layer1_attention_weight_min": -13.205756187438965,
      "activations/layer20_attention_weight_max": 19.204654693603516,
      "activations/layer20_attention_weight_min": -21.477815628051758,
      "activations/layer21_attention_weight_max": 29.774456024169922,
      "activations/layer21_attention_weight_min": -22.492612838745117,
      "activations/layer22_attention_weight_max": 25.81756019592285,
      "activations/layer22_attention_weight_min": -24.52352523803711,
      "activations/layer23_attention_weight_max": 33.84725570678711,
      "activations/layer23_attention_weight_min": -22.951396942138672,
      "activations/layer2_attention_weight_max": 30.742454528808594,
      "activations/layer2_attention_weight_min": -30.43586540222168,
      "activations/layer3_attention_weight_max": 85.2696304321289,
      "activations/layer3_attention_weight_min": -92.25226593017578,
      "activations/layer4_attention_weight_max": 87.13121032714844,
      "activations/layer4_attention_weight_min": -88.25594329833984,
      "activations/layer5_attention_weight_max": 68.61917114257812,
      "activations/layer5_attention_weight_min": -72.47450256347656,
      "activations/layer6_attention_weight_max": 47.78575897216797,
      "activations/layer6_attention_weight_min": -49.976749420166016,
      "activations/layer7_attention_weight_max": 58.357215881347656,
      "activations/layer7_attention_weight_min": -58.36979675292969,
      "activations/layer8_attention_weight_max": 44.108795166015625,
      "activations/layer8_attention_weight_min": -49.882789611816406,
      "activations/layer9_attention_weight_max": 41.933868408203125,
      "activations/layer9_attention_weight_min": -44.875240325927734,
      "epoch": 16.1,
      "learning_rate": 4.6676893939393936e-05,
      "loss": 2.7358,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.4038,
      "eval_samples_per_second": 510.958,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.4038,
      "eval_openwebtext_samples_per_second": 510.958,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9521,
      "eval_wikitext_samples_per_second": 233.598,
      "step": 277000
    },
    {
      "epoch": 16.1,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.5239,
      "eval_lambada_samples_per_second": 511.241,
      "step": 277000
    },
    {
      "activations/layer0_attention_weight_max": 15.11489486694336,
      "activations/layer0_attention_weight_min": -14.263877868652344,
      "activations/layer10_attention_weight_max": 36.99995040893555,
      "activations/layer10_attention_weight_min": -39.9796142578125,
      "activations/layer11_attention_weight_max": 34.567317962646484,
      "activations/layer11_attention_weight_min": -35.580997467041016,
      "activations/layer12_attention_weight_max": 18.637691497802734,
      "activations/layer12_attention_weight_min": -26.396076202392578,
      "activations/layer13_attention_weight_max": 36.59191131591797,
      "activations/layer13_attention_weight_min": -29.771886825561523,
      "activations/layer14_attention_weight_max": 34.47854232788086,
      "activations/layer14_attention_weight_min": -28.60539436340332,
      "activations/layer15_attention_weight_max": 29.809497833251953,
      "activations/layer15_attention_weight_min": -29.623781204223633,
      "activations/layer16_attention_weight_max": 32.325035095214844,
      "activations/layer16_attention_weight_min": -30.414356231689453,
      "activations/layer17_attention_weight_max": 47.30236053466797,
      "activations/layer17_attention_weight_min": -42.2539176940918,
      "activations/layer18_attention_weight_max": 41.406002044677734,
      "activations/layer18_attention_weight_min": -37.26777648925781,
      "activations/layer19_attention_weight_max": 23.191246032714844,
      "activations/layer19_attention_weight_min": -22.62944984436035,
      "activations/layer1_attention_weight_max": 16.91117286682129,
      "activations/layer1_attention_weight_min": -14.870879173278809,
      "activations/layer20_attention_weight_max": 21.481863021850586,
      "activations/layer20_attention_weight_min": -23.220598220825195,
      "activations/layer21_attention_weight_max": 35.71666717529297,
      "activations/layer21_attention_weight_min": -23.273056030273438,
      "activations/layer22_attention_weight_max": 28.831892013549805,
      "activations/layer22_attention_weight_min": -25.098876953125,
      "activations/layer23_attention_weight_max": 34.443885803222656,
      "activations/layer23_attention_weight_min": -24.762149810791016,
      "activations/layer2_attention_weight_max": 29.610013961791992,
      "activations/layer2_attention_weight_min": -28.705764770507812,
      "activations/layer3_attention_weight_max": 88.4847412109375,
      "activations/layer3_attention_weight_min": -88.53189849853516,
      "activations/layer4_attention_weight_max": 92.92607879638672,
      "activations/layer4_attention_weight_min": -90.62218475341797,
      "activations/layer5_attention_weight_max": 70.28848266601562,
      "activations/layer5_attention_weight_min": -76.55484008789062,
      "activations/layer6_attention_weight_max": 53.11917495727539,
      "activations/layer6_attention_weight_min": -54.96413040161133,
      "activations/layer7_attention_weight_max": 61.50708770751953,
      "activations/layer7_attention_weight_min": -64.42481994628906,
      "activations/layer8_attention_weight_max": 48.598880767822266,
      "activations/layer8_attention_weight_min": -53.02440643310547,
      "activations/layer9_attention_weight_max": 46.20869827270508,
      "activations/layer9_attention_weight_min": -49.8816032409668,
      "epoch": 16.1,
      "learning_rate": 4.665795454545454e-05,
      "loss": 2.7463,
      "step": 277050
    },
    {
      "activations/layer0_attention_weight_max": 16.014739990234375,
      "activations/layer0_attention_weight_min": -13.85035228729248,
      "activations/layer10_attention_weight_max": 34.46038818359375,
      "activations/layer10_attention_weight_min": -35.05805206298828,
      "activations/layer11_attention_weight_max": 31.628597259521484,
      "activations/layer11_attention_weight_min": -35.29219055175781,
      "activations/layer12_attention_weight_max": 18.06080436706543,
      "activations/layer12_attention_weight_min": -26.30084800720215,
      "activations/layer13_attention_weight_max": 32.35134506225586,
      "activations/layer13_attention_weight_min": -25.98927879333496,
      "activations/layer14_attention_weight_max": 36.017269134521484,
      "activations/layer14_attention_weight_min": -29.14155387878418,
      "activations/layer15_attention_weight_max": 31.548723220825195,
      "activations/layer15_attention_weight_min": -27.11029815673828,
      "activations/layer16_attention_weight_max": 30.968889236450195,
      "activations/layer16_attention_weight_min": -28.888206481933594,
      "activations/layer17_attention_weight_max": 47.078731536865234,
      "activations/layer17_attention_weight_min": -41.71385955810547,
      "activations/layer18_attention_weight_max": 45.770423889160156,
      "activations/layer18_attention_weight_min": -39.11577606201172,
      "activations/layer19_attention_weight_max": 22.59682846069336,
      "activations/layer19_attention_weight_min": -22.26751136779785,
      "activations/layer1_attention_weight_max": 18.134626388549805,
      "activations/layer1_attention_weight_min": -14.933886528015137,
      "activations/layer20_attention_weight_max": 23.296939849853516,
      "activations/layer20_attention_weight_min": -21.02086639404297,
      "activations/layer21_attention_weight_max": 33.56276321411133,
      "activations/layer21_attention_weight_min": -20.49501609802246,
      "activations/layer22_attention_weight_max": 33.58687210083008,
      "activations/layer22_attention_weight_min": -26.060056686401367,
      "activations/layer23_attention_weight_max": 37.13416290283203,
      "activations/layer23_attention_weight_min": -23.305580139160156,
      "activations/layer2_attention_weight_max": 30.503467559814453,
      "activations/layer2_attention_weight_min": -29.84420394897461,
      "activations/layer3_attention_weight_max": 89.18063354492188,
      "activations/layer3_attention_weight_min": -87.0735855102539,
      "activations/layer4_attention_weight_max": 89.8511962890625,
      "activations/layer4_attention_weight_min": -88.74751281738281,
      "activations/layer5_attention_weight_max": 66.87837219238281,
      "activations/layer5_attention_weight_min": -75.85089874267578,
      "activations/layer6_attention_weight_max": 50.32856369018555,
      "activations/layer6_attention_weight_min": -53.161869049072266,
      "activations/layer7_attention_weight_max": 58.19239807128906,
      "activations/layer7_attention_weight_min": -65.18599700927734,
      "activations/layer8_attention_weight_max": 46.46107482910156,
      "activations/layer8_attention_weight_min": -49.953369140625,
      "activations/layer9_attention_weight_max": 44.82423400878906,
      "activations/layer9_attention_weight_min": -46.77325439453125,
      "epoch": 16.1,
      "learning_rate": 4.663901515151515e-05,
      "loss": 2.7473,
      "step": 277100
    },
    {
      "activations/layer0_attention_weight_max": 15.917677879333496,
      "activations/layer0_attention_weight_min": -14.020973205566406,
      "activations/layer10_attention_weight_max": 34.87285232543945,
      "activations/layer10_attention_weight_min": -36.67759704589844,
      "activations/layer11_attention_weight_max": 30.067228317260742,
      "activations/layer11_attention_weight_min": -32.08129119873047,
      "activations/layer12_attention_weight_max": 17.806011199951172,
      "activations/layer12_attention_weight_min": -23.526363372802734,
      "activations/layer13_attention_weight_max": 31.732166290283203,
      "activations/layer13_attention_weight_min": -26.482614517211914,
      "activations/layer14_attention_weight_max": 34.18730545043945,
      "activations/layer14_attention_weight_min": -28.570083618164062,
      "activations/layer15_attention_weight_max": 31.288232803344727,
      "activations/layer15_attention_weight_min": -28.943994522094727,
      "activations/layer16_attention_weight_max": 30.723669052124023,
      "activations/layer16_attention_weight_min": -29.182043075561523,
      "activations/layer17_attention_weight_max": 47.987892150878906,
      "activations/layer17_attention_weight_min": -40.43369674682617,
      "activations/layer18_attention_weight_max": 43.97295379638672,
      "activations/layer18_attention_weight_min": -39.102657318115234,
      "activations/layer19_attention_weight_max": 23.70131492614746,
      "activations/layer19_attention_weight_min": -22.708181381225586,
      "activations/layer1_attention_weight_max": 17.79340362548828,
      "activations/layer1_attention_weight_min": -14.77935791015625,
      "activations/layer20_attention_weight_max": 22.405405044555664,
      "activations/layer20_attention_weight_min": -22.211776733398438,
      "activations/layer21_attention_weight_max": 30.674528121948242,
      "activations/layer21_attention_weight_min": -20.511444091796875,
      "activations/layer22_attention_weight_max": 27.15450668334961,
      "activations/layer22_attention_weight_min": -24.41969108581543,
      "activations/layer23_attention_weight_max": 33.81330108642578,
      "activations/layer23_attention_weight_min": -23.581472396850586,
      "activations/layer2_attention_weight_max": 30.837841033935547,
      "activations/layer2_attention_weight_min": -28.922094345092773,
      "activations/layer3_attention_weight_max": 87.39729309082031,
      "activations/layer3_attention_weight_min": -85.74079132080078,
      "activations/layer4_attention_weight_max": 89.0628662109375,
      "activations/layer4_attention_weight_min": -86.30343627929688,
      "activations/layer5_attention_weight_max": 64.59325408935547,
      "activations/layer5_attention_weight_min": -76.2269287109375,
      "activations/layer6_attention_weight_max": 48.582035064697266,
      "activations/layer6_attention_weight_min": -49.39476013183594,
      "activations/layer7_attention_weight_max": 63.4613037109375,
      "activations/layer7_attention_weight_min": -61.14749526977539,
      "activations/layer8_attention_weight_max": 44.20429992675781,
      "activations/layer8_attention_weight_min": -47.22388458251953,
      "activations/layer9_attention_weight_max": 43.51760482788086,
      "activations/layer9_attention_weight_min": -43.04513168334961,
      "epoch": 16.1,
      "learning_rate": 4.6620075757575754e-05,
      "loss": 2.7297,
      "step": 277150
    },
    {
      "activations/layer0_attention_weight_max": 15.476778030395508,
      "activations/layer0_attention_weight_min": -14.105098724365234,
      "activations/layer10_attention_weight_max": 37.92131423950195,
      "activations/layer10_attention_weight_min": -37.72657775878906,
      "activations/layer11_attention_weight_max": 33.830116271972656,
      "activations/layer11_attention_weight_min": -33.65898513793945,
      "activations/layer12_attention_weight_max": 17.65693473815918,
      "activations/layer12_attention_weight_min": -24.37549591064453,
      "activations/layer13_attention_weight_max": 32.516990661621094,
      "activations/layer13_attention_weight_min": -26.999631881713867,
      "activations/layer14_attention_weight_max": 33.870643615722656,
      "activations/layer14_attention_weight_min": -31.413005828857422,
      "activations/layer15_attention_weight_max": 31.870729446411133,
      "activations/layer15_attention_weight_min": -30.048404693603516,
      "activations/layer16_attention_weight_max": 33.87883758544922,
      "activations/layer16_attention_weight_min": -33.89432144165039,
      "activations/layer17_attention_weight_max": 48.85813522338867,
      "activations/layer17_attention_weight_min": -46.20566940307617,
      "activations/layer18_attention_weight_max": 44.94620132446289,
      "activations/layer18_attention_weight_min": -39.85407638549805,
      "activations/layer19_attention_weight_max": 23.90939712524414,
      "activations/layer19_attention_weight_min": -23.04072380065918,
      "activations/layer1_attention_weight_max": 16.47446060180664,
      "activations/layer1_attention_weight_min": -15.676876068115234,
      "activations/layer20_attention_weight_max": 22.922962188720703,
      "activations/layer20_attention_weight_min": -21.302152633666992,
      "activations/layer21_attention_weight_max": 33.51478958129883,
      "activations/layer21_attention_weight_min": -22.973102569580078,
      "activations/layer22_attention_weight_max": 30.286897659301758,
      "activations/layer22_attention_weight_min": -26.830808639526367,
      "activations/layer23_attention_weight_max": 36.96735382080078,
      "activations/layer23_attention_weight_min": -25.02880096435547,
      "activations/layer2_attention_weight_max": 31.563488006591797,
      "activations/layer2_attention_weight_min": -29.41578483581543,
      "activations/layer3_attention_weight_max": 88.41934967041016,
      "activations/layer3_attention_weight_min": -96.60150146484375,
      "activations/layer4_attention_weight_max": 90.58245086669922,
      "activations/layer4_attention_weight_min": -87.68083953857422,
      "activations/layer5_attention_weight_max": 64.3559799194336,
      "activations/layer5_attention_weight_min": -71.15770721435547,
      "activations/layer6_attention_weight_max": 51.36894989013672,
      "activations/layer6_attention_weight_min": -50.64481735229492,
      "activations/layer7_attention_weight_max": 62.91027069091797,
      "activations/layer7_attention_weight_min": -59.76930236816406,
      "activations/layer8_attention_weight_max": 46.579315185546875,
      "activations/layer8_attention_weight_min": -49.1942138671875,
      "activations/layer9_attention_weight_max": 44.03120422363281,
      "activations/layer9_attention_weight_min": -49.34385299682617,
      "epoch": 16.11,
      "learning_rate": 4.6601136363636355e-05,
      "loss": 2.7449,
      "step": 277200
    },
    {
      "activations/layer0_attention_weight_max": 15.555021286010742,
      "activations/layer0_attention_weight_min": -13.902761459350586,
      "activations/layer10_attention_weight_max": 35.694366455078125,
      "activations/layer10_attention_weight_min": -36.56205749511719,
      "activations/layer11_attention_weight_max": 32.70225143432617,
      "activations/layer11_attention_weight_min": -36.89664840698242,
      "activations/layer12_attention_weight_max": 18.25442123413086,
      "activations/layer12_attention_weight_min": -29.859054565429688,
      "activations/layer13_attention_weight_max": 32.88179016113281,
      "activations/layer13_attention_weight_min": -26.284156799316406,
      "activations/layer14_attention_weight_max": 33.60671615600586,
      "activations/layer14_attention_weight_min": -31.71168327331543,
      "activations/layer15_attention_weight_max": 32.52101135253906,
      "activations/layer15_attention_weight_min": -30.96345329284668,
      "activations/layer16_attention_weight_max": 34.25260925292969,
      "activations/layer16_attention_weight_min": -33.029273986816406,
      "activations/layer17_attention_weight_max": 47.889427185058594,
      "activations/layer17_attention_weight_min": -44.91664123535156,
      "activations/layer18_attention_weight_max": 44.095218658447266,
      "activations/layer18_attention_weight_min": -40.13825607299805,
      "activations/layer19_attention_weight_max": 23.110803604125977,
      "activations/layer19_attention_weight_min": -22.508941650390625,
      "activations/layer1_attention_weight_max": 16.976388931274414,
      "activations/layer1_attention_weight_min": -14.886387825012207,
      "activations/layer20_attention_weight_max": 22.200666427612305,
      "activations/layer20_attention_weight_min": -20.980459213256836,
      "activations/layer21_attention_weight_max": 33.63111877441406,
      "activations/layer21_attention_weight_min": -22.665191650390625,
      "activations/layer22_attention_weight_max": 27.6341609954834,
      "activations/layer22_attention_weight_min": -26.74142837524414,
      "activations/layer23_attention_weight_max": 35.23155975341797,
      "activations/layer23_attention_weight_min": -26.954450607299805,
      "activations/layer2_attention_weight_max": 33.09736633300781,
      "activations/layer2_attention_weight_min": -29.19253921508789,
      "activations/layer3_attention_weight_max": 89.72914123535156,
      "activations/layer3_attention_weight_min": -87.7574234008789,
      "activations/layer4_attention_weight_max": 88.53478240966797,
      "activations/layer4_attention_weight_min": -89.96949005126953,
      "activations/layer5_attention_weight_max": 63.00498580932617,
      "activations/layer5_attention_weight_min": -72.39875793457031,
      "activations/layer6_attention_weight_max": 49.3259391784668,
      "activations/layer6_attention_weight_min": -51.25341033935547,
      "activations/layer7_attention_weight_max": 65.27169799804688,
      "activations/layer7_attention_weight_min": -63.881507873535156,
      "activations/layer8_attention_weight_max": 47.50560760498047,
      "activations/layer8_attention_weight_min": -48.35219192504883,
      "activations/layer9_attention_weight_max": 50.79479217529297,
      "activations/layer9_attention_weight_min": -46.27021789550781,
      "epoch": 16.11,
      "learning_rate": 4.658219696969697e-05,
      "loss": 2.7597,
      "step": 277250
    },
    {
      "activations/layer0_attention_weight_max": 16.947219848632812,
      "activations/layer0_attention_weight_min": -14.096548080444336,
      "activations/layer10_attention_weight_max": 36.35822296142578,
      "activations/layer10_attention_weight_min": -35.346168518066406,
      "activations/layer11_attention_weight_max": 32.14377975463867,
      "activations/layer11_attention_weight_min": -33.8482551574707,
      "activations/layer12_attention_weight_max": 17.55763053894043,
      "activations/layer12_attention_weight_min": -25.152061462402344,
      "activations/layer13_attention_weight_max": 36.00844192504883,
      "activations/layer13_attention_weight_min": -30.26239013671875,
      "activations/layer14_attention_weight_max": 38.27159881591797,
      "activations/layer14_attention_weight_min": -33.507320404052734,
      "activations/layer15_attention_weight_max": 31.914180755615234,
      "activations/layer15_attention_weight_min": -31.653242111206055,
      "activations/layer16_attention_weight_max": 35.23086166381836,
      "activations/layer16_attention_weight_min": -33.01625442504883,
      "activations/layer17_attention_weight_max": 55.69595718383789,
      "activations/layer17_attention_weight_min": -46.9706916809082,
      "activations/layer18_attention_weight_max": 53.160465240478516,
      "activations/layer18_attention_weight_min": -40.942237854003906,
      "activations/layer19_attention_weight_max": 24.92108726501465,
      "activations/layer19_attention_weight_min": -26.070337295532227,
      "activations/layer1_attention_weight_max": 17.228731155395508,
      "activations/layer1_attention_weight_min": -16.040624618530273,
      "activations/layer20_attention_weight_max": 27.512910842895508,
      "activations/layer20_attention_weight_min": -23.38530158996582,
      "activations/layer21_attention_weight_max": 40.64918899536133,
      "activations/layer21_attention_weight_min": -26.141794204711914,
      "activations/layer22_attention_weight_max": 30.73223876953125,
      "activations/layer22_attention_weight_min": -27.76313591003418,
      "activations/layer23_attention_weight_max": 34.81922912597656,
      "activations/layer23_attention_weight_min": -26.794475555419922,
      "activations/layer2_attention_weight_max": 32.4802360534668,
      "activations/layer2_attention_weight_min": -30.369792938232422,
      "activations/layer3_attention_weight_max": 91.25785064697266,
      "activations/layer3_attention_weight_min": -92.04682922363281,
      "activations/layer4_attention_weight_max": 89.94188690185547,
      "activations/layer4_attention_weight_min": -88.16497039794922,
      "activations/layer5_attention_weight_max": 64.91593170166016,
      "activations/layer5_attention_weight_min": -72.49939727783203,
      "activations/layer6_attention_weight_max": 50.7208137512207,
      "activations/layer6_attention_weight_min": -50.7922248840332,
      "activations/layer7_attention_weight_max": 64.64495849609375,
      "activations/layer7_attention_weight_min": -62.5214958190918,
      "activations/layer8_attention_weight_max": 45.262916564941406,
      "activations/layer8_attention_weight_min": -47.11411666870117,
      "activations/layer9_attention_weight_max": 45.6176643371582,
      "activations/layer9_attention_weight_min": -44.8423957824707,
      "epoch": 16.11,
      "learning_rate": 4.656325757575757e-05,
      "loss": 2.7374,
      "step": 277300
    },
    {
      "activations/layer0_attention_weight_max": 14.73602294921875,
      "activations/layer0_attention_weight_min": -14.01036548614502,
      "activations/layer10_attention_weight_max": 33.943885803222656,
      "activations/layer10_attention_weight_min": -33.53617858886719,
      "activations/layer11_attention_weight_max": 31.913288116455078,
      "activations/layer11_attention_weight_min": -33.46625518798828,
      "activations/layer12_attention_weight_max": 20.547338485717773,
      "activations/layer12_attention_weight_min": -26.813657760620117,
      "activations/layer13_attention_weight_max": 42.543731689453125,
      "activations/layer13_attention_weight_min": -28.11880111694336,
      "activations/layer14_attention_weight_max": 38.7996711730957,
      "activations/layer14_attention_weight_min": -30.771589279174805,
      "activations/layer15_attention_weight_max": 37.14710235595703,
      "activations/layer15_attention_weight_min": -30.148014068603516,
      "activations/layer16_attention_weight_max": 37.25056838989258,
      "activations/layer16_attention_weight_min": -33.12713623046875,
      "activations/layer17_attention_weight_max": 58.80475616455078,
      "activations/layer17_attention_weight_min": -48.561065673828125,
      "activations/layer18_attention_weight_max": 55.63113784790039,
      "activations/layer18_attention_weight_min": -41.71908950805664,
      "activations/layer19_attention_weight_max": 28.256322860717773,
      "activations/layer19_attention_weight_min": -22.581451416015625,
      "activations/layer1_attention_weight_max": 16.511550903320312,
      "activations/layer1_attention_weight_min": -15.704164505004883,
      "activations/layer20_attention_weight_max": 24.86556053161621,
      "activations/layer20_attention_weight_min": -21.082666397094727,
      "activations/layer21_attention_weight_max": 42.43989562988281,
      "activations/layer21_attention_weight_min": -23.710779190063477,
      "activations/layer22_attention_weight_max": 34.80096435546875,
      "activations/layer22_attention_weight_min": -23.96471405029297,
      "activations/layer23_attention_weight_max": 41.75914764404297,
      "activations/layer23_attention_weight_min": -28.46314811706543,
      "activations/layer2_attention_weight_max": 31.13745880126953,
      "activations/layer2_attention_weight_min": -30.52724838256836,
      "activations/layer3_attention_weight_max": 87.4334487915039,
      "activations/layer3_attention_weight_min": -86.1168441772461,
      "activations/layer4_attention_weight_max": 89.9988784790039,
      "activations/layer4_attention_weight_min": -86.75459289550781,
      "activations/layer5_attention_weight_max": 66.12159729003906,
      "activations/layer5_attention_weight_min": -74.27031707763672,
      "activations/layer6_attention_weight_max": 51.66596984863281,
      "activations/layer6_attention_weight_min": -53.782264709472656,
      "activations/layer7_attention_weight_max": 67.21385955810547,
      "activations/layer7_attention_weight_min": -58.92622375488281,
      "activations/layer8_attention_weight_max": 44.44014358520508,
      "activations/layer8_attention_weight_min": -49.20261001586914,
      "activations/layer9_attention_weight_max": 43.51777648925781,
      "activations/layer9_attention_weight_min": -44.34785079956055,
      "epoch": 16.12,
      "learning_rate": 4.654431818181818e-05,
      "loss": 2.7405,
      "step": 277350
    },
    {
      "activations/layer0_attention_weight_max": 15.310998916625977,
      "activations/layer0_attention_weight_min": -14.339503288269043,
      "activations/layer10_attention_weight_max": 36.16485595703125,
      "activations/layer10_attention_weight_min": -35.06392288208008,
      "activations/layer11_attention_weight_max": 33.6280632019043,
      "activations/layer11_attention_weight_min": -33.15898132324219,
      "activations/layer12_attention_weight_max": 18.803157806396484,
      "activations/layer12_attention_weight_min": -25.802839279174805,
      "activations/layer13_attention_weight_max": 35.6584587097168,
      "activations/layer13_attention_weight_min": -27.30146026611328,
      "activations/layer14_attention_weight_max": 33.78105926513672,
      "activations/layer14_attention_weight_min": -29.866662979125977,
      "activations/layer15_attention_weight_max": 32.376075744628906,
      "activations/layer15_attention_weight_min": -28.73575210571289,
      "activations/layer16_attention_weight_max": 33.867889404296875,
      "activations/layer16_attention_weight_min": -32.735633850097656,
      "activations/layer17_attention_weight_max": 52.10759353637695,
      "activations/layer17_attention_weight_min": -46.962242126464844,
      "activations/layer18_attention_weight_max": 47.93510437011719,
      "activations/layer18_attention_weight_min": -39.717262268066406,
      "activations/layer19_attention_weight_max": 21.92926788330078,
      "activations/layer19_attention_weight_min": -22.850011825561523,
      "activations/layer1_attention_weight_max": 16.524824142456055,
      "activations/layer1_attention_weight_min": -14.834884643554688,
      "activations/layer20_attention_weight_max": 22.495141983032227,
      "activations/layer20_attention_weight_min": -19.962522506713867,
      "activations/layer21_attention_weight_max": 34.912166595458984,
      "activations/layer21_attention_weight_min": -21.429527282714844,
      "activations/layer22_attention_weight_max": 28.894609451293945,
      "activations/layer22_attention_weight_min": -24.05556297302246,
      "activations/layer23_attention_weight_max": 38.26609420776367,
      "activations/layer23_attention_weight_min": -24.126510620117188,
      "activations/layer2_attention_weight_max": 32.46976089477539,
      "activations/layer2_attention_weight_min": -31.448322296142578,
      "activations/layer3_attention_weight_max": 90.62261199951172,
      "activations/layer3_attention_weight_min": -94.1979751586914,
      "activations/layer4_attention_weight_max": 91.43721771240234,
      "activations/layer4_attention_weight_min": -87.1502685546875,
      "activations/layer5_attention_weight_max": 62.92976379394531,
      "activations/layer5_attention_weight_min": -72.92469787597656,
      "activations/layer6_attention_weight_max": 50.26993942260742,
      "activations/layer6_attention_weight_min": -49.63351821899414,
      "activations/layer7_attention_weight_max": 64.53162384033203,
      "activations/layer7_attention_weight_min": -60.095703125,
      "activations/layer8_attention_weight_max": 46.82579040527344,
      "activations/layer8_attention_weight_min": -47.851200103759766,
      "activations/layer9_attention_weight_max": 44.98311996459961,
      "activations/layer9_attention_weight_min": -44.33110427856445,
      "epoch": 16.12,
      "learning_rate": 4.652537878787878e-05,
      "loss": 2.7225,
      "step": 277400
    },
    {
      "activations/layer0_attention_weight_max": 15.758858680725098,
      "activations/layer0_attention_weight_min": -14.028806686401367,
      "activations/layer10_attention_weight_max": 35.03265380859375,
      "activations/layer10_attention_weight_min": -35.34856414794922,
      "activations/layer11_attention_weight_max": 32.12041473388672,
      "activations/layer11_attention_weight_min": -32.71144104003906,
      "activations/layer12_attention_weight_max": 21.4971981048584,
      "activations/layer12_attention_weight_min": -23.273059844970703,
      "activations/layer13_attention_weight_max": 37.67704391479492,
      "activations/layer13_attention_weight_min": -27.708553314208984,
      "activations/layer14_attention_weight_max": 36.34806442260742,
      "activations/layer14_attention_weight_min": -32.461082458496094,
      "activations/layer15_attention_weight_max": 36.35125732421875,
      "activations/layer15_attention_weight_min": -32.02989196777344,
      "activations/layer16_attention_weight_max": 37.750274658203125,
      "activations/layer16_attention_weight_min": -33.14899826049805,
      "activations/layer17_attention_weight_max": 54.58537673950195,
      "activations/layer17_attention_weight_min": -46.928009033203125,
      "activations/layer18_attention_weight_max": 50.556365966796875,
      "activations/layer18_attention_weight_min": -41.17046356201172,
      "activations/layer19_attention_weight_max": 24.579181671142578,
      "activations/layer19_attention_weight_min": -23.470212936401367,
      "activations/layer1_attention_weight_max": 16.003097534179688,
      "activations/layer1_attention_weight_min": -15.448589324951172,
      "activations/layer20_attention_weight_max": 23.737504959106445,
      "activations/layer20_attention_weight_min": -21.76238441467285,
      "activations/layer21_attention_weight_max": 34.08159255981445,
      "activations/layer21_attention_weight_min": -21.54617691040039,
      "activations/layer22_attention_weight_max": 28.688745498657227,
      "activations/layer22_attention_weight_min": -23.51679229736328,
      "activations/layer23_attention_weight_max": 38.23213195800781,
      "activations/layer23_attention_weight_min": -23.70824432373047,
      "activations/layer2_attention_weight_max": 31.84437370300293,
      "activations/layer2_attention_weight_min": -29.724014282226562,
      "activations/layer3_attention_weight_max": 85.37629699707031,
      "activations/layer3_attention_weight_min": -84.02516174316406,
      "activations/layer4_attention_weight_max": 88.60167694091797,
      "activations/layer4_attention_weight_min": -86.46385955810547,
      "activations/layer5_attention_weight_max": 60.54308319091797,
      "activations/layer5_attention_weight_min": -73.2533950805664,
      "activations/layer6_attention_weight_max": 49.87162780761719,
      "activations/layer6_attention_weight_min": -48.82840347290039,
      "activations/layer7_attention_weight_max": 60.499351501464844,
      "activations/layer7_attention_weight_min": -60.79651641845703,
      "activations/layer8_attention_weight_max": 45.15938186645508,
      "activations/layer8_attention_weight_min": -46.52810287475586,
      "activations/layer9_attention_weight_max": 42.2071647644043,
      "activations/layer9_attention_weight_min": -43.577178955078125,
      "epoch": 16.12,
      "learning_rate": 4.650643939393939e-05,
      "loss": 2.734,
      "step": 277450
    },
    {
      "activations/layer0_attention_weight_max": 15.01617431640625,
      "activations/layer0_attention_weight_min": -13.682906150817871,
      "activations/layer10_attention_weight_max": 40.125465393066406,
      "activations/layer10_attention_weight_min": -39.039527893066406,
      "activations/layer11_attention_weight_max": 36.98954772949219,
      "activations/layer11_attention_weight_min": -34.77249526977539,
      "activations/layer12_attention_weight_max": 19.957496643066406,
      "activations/layer12_attention_weight_min": -25.92340660095215,
      "activations/layer13_attention_weight_max": 41.57108688354492,
      "activations/layer13_attention_weight_min": -32.47581481933594,
      "activations/layer14_attention_weight_max": 53.71028137207031,
      "activations/layer14_attention_weight_min": -33.24956512451172,
      "activations/layer15_attention_weight_max": 42.023109436035156,
      "activations/layer15_attention_weight_min": -35.68150329589844,
      "activations/layer16_attention_weight_max": 36.95265197753906,
      "activations/layer16_attention_weight_min": -35.58345413208008,
      "activations/layer17_attention_weight_max": 57.11559295654297,
      "activations/layer17_attention_weight_min": -53.55119705200195,
      "activations/layer18_attention_weight_max": 54.3603401184082,
      "activations/layer18_attention_weight_min": -49.33548355102539,
      "activations/layer19_attention_weight_max": 29.52471351623535,
      "activations/layer19_attention_weight_min": -29.48308753967285,
      "activations/layer1_attention_weight_max": 15.804937362670898,
      "activations/layer1_attention_weight_min": -13.465546607971191,
      "activations/layer20_attention_weight_max": 31.184722900390625,
      "activations/layer20_attention_weight_min": -25.726333618164062,
      "activations/layer21_attention_weight_max": 45.444793701171875,
      "activations/layer21_attention_weight_min": -28.515050888061523,
      "activations/layer22_attention_weight_max": 32.12455368041992,
      "activations/layer22_attention_weight_min": -29.165456771850586,
      "activations/layer23_attention_weight_max": 42.82781982421875,
      "activations/layer23_attention_weight_min": -25.90776824951172,
      "activations/layer2_attention_weight_max": 33.92295837402344,
      "activations/layer2_attention_weight_min": -30.321670532226562,
      "activations/layer3_attention_weight_max": 88.92689514160156,
      "activations/layer3_attention_weight_min": -90.34296417236328,
      "activations/layer4_attention_weight_max": 89.52125549316406,
      "activations/layer4_attention_weight_min": -89.26168060302734,
      "activations/layer5_attention_weight_max": 65.4245834350586,
      "activations/layer5_attention_weight_min": -75.27598571777344,
      "activations/layer6_attention_weight_max": 52.27647018432617,
      "activations/layer6_attention_weight_min": -52.915706634521484,
      "activations/layer7_attention_weight_max": 65.67459106445312,
      "activations/layer7_attention_weight_min": -69.13705444335938,
      "activations/layer8_attention_weight_max": 50.58676528930664,
      "activations/layer8_attention_weight_min": -51.0072135925293,
      "activations/layer9_attention_weight_max": 56.21793746948242,
      "activations/layer9_attention_weight_min": -45.64568328857422,
      "epoch": 16.12,
      "learning_rate": 4.64875e-05,
      "loss": 2.7548,
      "step": 277500
    },
    {
      "activations/layer0_attention_weight_max": 15.944575309753418,
      "activations/layer0_attention_weight_min": -13.69582462310791,
      "activations/layer10_attention_weight_max": 37.61354446411133,
      "activations/layer10_attention_weight_min": -35.49617004394531,
      "activations/layer11_attention_weight_max": 34.08152770996094,
      "activations/layer11_attention_weight_min": -32.83736038208008,
      "activations/layer12_attention_weight_max": 18.94911766052246,
      "activations/layer12_attention_weight_min": -26.044471740722656,
      "activations/layer13_attention_weight_max": 33.47246551513672,
      "activations/layer13_attention_weight_min": -29.464723587036133,
      "activations/layer14_attention_weight_max": 34.86476516723633,
      "activations/layer14_attention_weight_min": -30.24469566345215,
      "activations/layer15_attention_weight_max": 32.671329498291016,
      "activations/layer15_attention_weight_min": -29.09141731262207,
      "activations/layer16_attention_weight_max": 31.65445899963379,
      "activations/layer16_attention_weight_min": -29.785951614379883,
      "activations/layer17_attention_weight_max": 48.9599609375,
      "activations/layer17_attention_weight_min": -44.20549392700195,
      "activations/layer18_attention_weight_max": 43.83235168457031,
      "activations/layer18_attention_weight_min": -39.80970764160156,
      "activations/layer19_attention_weight_max": 22.51154899597168,
      "activations/layer19_attention_weight_min": -21.65151023864746,
      "activations/layer1_attention_weight_max": 15.31743335723877,
      "activations/layer1_attention_weight_min": -15.963916778564453,
      "activations/layer20_attention_weight_max": 23.778783798217773,
      "activations/layer20_attention_weight_min": -23.32364845275879,
      "activations/layer21_attention_weight_max": 34.98878860473633,
      "activations/layer21_attention_weight_min": -24.612796783447266,
      "activations/layer22_attention_weight_max": 29.975624084472656,
      "activations/layer22_attention_weight_min": -26.41339683532715,
      "activations/layer23_attention_weight_max": 33.77347183227539,
      "activations/layer23_attention_weight_min": -27.911338806152344,
      "activations/layer2_attention_weight_max": 30.234859466552734,
      "activations/layer2_attention_weight_min": -30.555736541748047,
      "activations/layer3_attention_weight_max": 88.91978454589844,
      "activations/layer3_attention_weight_min": -86.89985656738281,
      "activations/layer4_attention_weight_max": 91.93678283691406,
      "activations/layer4_attention_weight_min": -86.8238754272461,
      "activations/layer5_attention_weight_max": 65.47688293457031,
      "activations/layer5_attention_weight_min": -72.70333862304688,
      "activations/layer6_attention_weight_max": 50.790748596191406,
      "activations/layer6_attention_weight_min": -50.98374557495117,
      "activations/layer7_attention_weight_max": 61.51374435424805,
      "activations/layer7_attention_weight_min": -62.47477340698242,
      "activations/layer8_attention_weight_max": 47.52899169921875,
      "activations/layer8_attention_weight_min": -48.79640197753906,
      "activations/layer9_attention_weight_max": 47.104976654052734,
      "activations/layer9_attention_weight_min": -46.112831115722656,
      "epoch": 16.13,
      "learning_rate": 4.64685606060606e-05,
      "loss": 2.7406,
      "step": 277550
    },
    {
      "activations/layer0_attention_weight_max": 15.831892013549805,
      "activations/layer0_attention_weight_min": -13.954339027404785,
      "activations/layer10_attention_weight_max": 33.54454803466797,
      "activations/layer10_attention_weight_min": -35.228370666503906,
      "activations/layer11_attention_weight_max": 32.906578063964844,
      "activations/layer11_attention_weight_min": -32.65522384643555,
      "activations/layer12_attention_weight_max": 18.214929580688477,
      "activations/layer12_attention_weight_min": -24.96388053894043,
      "activations/layer13_attention_weight_max": 33.51005554199219,
      "activations/layer13_attention_weight_min": -30.665828704833984,
      "activations/layer14_attention_weight_max": 34.535621643066406,
      "activations/layer14_attention_weight_min": -29.22764778137207,
      "activations/layer15_attention_weight_max": 31.78915023803711,
      "activations/layer15_attention_weight_min": -28.472978591918945,
      "activations/layer16_attention_weight_max": 32.44269943237305,
      "activations/layer16_attention_weight_min": -29.339237213134766,
      "activations/layer17_attention_weight_max": 55.92837142944336,
      "activations/layer17_attention_weight_min": -44.919517517089844,
      "activations/layer18_attention_weight_max": 45.94882583618164,
      "activations/layer18_attention_weight_min": -37.63967514038086,
      "activations/layer19_attention_weight_max": 23.339323043823242,
      "activations/layer19_attention_weight_min": -21.52570915222168,
      "activations/layer1_attention_weight_max": 15.335537910461426,
      "activations/layer1_attention_weight_min": -14.656800270080566,
      "activations/layer20_attention_weight_max": 23.53931999206543,
      "activations/layer20_attention_weight_min": -24.480030059814453,
      "activations/layer21_attention_weight_max": 34.42146682739258,
      "activations/layer21_attention_weight_min": -21.395633697509766,
      "activations/layer22_attention_weight_max": 28.43708610534668,
      "activations/layer22_attention_weight_min": -27.0736026763916,
      "activations/layer23_attention_weight_max": 36.48990249633789,
      "activations/layer23_attention_weight_min": -24.96259117126465,
      "activations/layer2_attention_weight_max": 33.78816223144531,
      "activations/layer2_attention_weight_min": -28.755168914794922,
      "activations/layer3_attention_weight_max": 87.08295440673828,
      "activations/layer3_attention_weight_min": -85.1369400024414,
      "activations/layer4_attention_weight_max": 87.26702880859375,
      "activations/layer4_attention_weight_min": -84.38651275634766,
      "activations/layer5_attention_weight_max": 67.80427551269531,
      "activations/layer5_attention_weight_min": -69.21524047851562,
      "activations/layer6_attention_weight_max": 50.85564422607422,
      "activations/layer6_attention_weight_min": -50.83236312866211,
      "activations/layer7_attention_weight_max": 62.48317337036133,
      "activations/layer7_attention_weight_min": -60.134559631347656,
      "activations/layer8_attention_weight_max": 43.00766372680664,
      "activations/layer8_attention_weight_min": -45.93300247192383,
      "activations/layer9_attention_weight_max": 45.24177551269531,
      "activations/layer9_attention_weight_min": -45.05430221557617,
      "epoch": 16.13,
      "learning_rate": 4.64496212121212e-05,
      "loss": 2.741,
      "step": 277600
    },
    {
      "activations/layer0_attention_weight_max": 15.508142471313477,
      "activations/layer0_attention_weight_min": -14.312579154968262,
      "activations/layer10_attention_weight_max": 36.975379943847656,
      "activations/layer10_attention_weight_min": -34.827423095703125,
      "activations/layer11_attention_weight_max": 34.25132751464844,
      "activations/layer11_attention_weight_min": -33.46377182006836,
      "activations/layer12_attention_weight_max": 18.47684097290039,
      "activations/layer12_attention_weight_min": -25.11932373046875,
      "activations/layer13_attention_weight_max": 45.983577728271484,
      "activations/layer13_attention_weight_min": -28.72955894470215,
      "activations/layer14_attention_weight_max": 37.959991455078125,
      "activations/layer14_attention_weight_min": -32.54788589477539,
      "activations/layer15_attention_weight_max": 36.25981903076172,
      "activations/layer15_attention_weight_min": -33.318634033203125,
      "activations/layer16_attention_weight_max": 37.23564910888672,
      "activations/layer16_attention_weight_min": -33.05953598022461,
      "activations/layer17_attention_weight_max": 58.6349983215332,
      "activations/layer17_attention_weight_min": -50.6569709777832,
      "activations/layer18_attention_weight_max": 52.84026336669922,
      "activations/layer18_attention_weight_min": -43.59956741333008,
      "activations/layer19_attention_weight_max": 24.826932907104492,
      "activations/layer19_attention_weight_min": -23.828046798706055,
      "activations/layer1_attention_weight_max": 16.439912796020508,
      "activations/layer1_attention_weight_min": -15.770166397094727,
      "activations/layer20_attention_weight_max": 24.173248291015625,
      "activations/layer20_attention_weight_min": -24.373428344726562,
      "activations/layer21_attention_weight_max": 45.057762145996094,
      "activations/layer21_attention_weight_min": -21.839075088500977,
      "activations/layer22_attention_weight_max": 29.186201095581055,
      "activations/layer22_attention_weight_min": -27.469839096069336,
      "activations/layer23_attention_weight_max": 36.08879852294922,
      "activations/layer23_attention_weight_min": -26.332029342651367,
      "activations/layer2_attention_weight_max": 31.381755828857422,
      "activations/layer2_attention_weight_min": -29.7397518157959,
      "activations/layer3_attention_weight_max": 87.4335708618164,
      "activations/layer3_attention_weight_min": -86.29108428955078,
      "activations/layer4_attention_weight_max": 87.01834106445312,
      "activations/layer4_attention_weight_min": -87.02442169189453,
      "activations/layer5_attention_weight_max": 62.26760482788086,
      "activations/layer5_attention_weight_min": -72.49736022949219,
      "activations/layer6_attention_weight_max": 51.695274353027344,
      "activations/layer6_attention_weight_min": -49.74913024902344,
      "activations/layer7_attention_weight_max": 60.261756896972656,
      "activations/layer7_attention_weight_min": -58.964111328125,
      "activations/layer8_attention_weight_max": 48.291194915771484,
      "activations/layer8_attention_weight_min": -46.7514533996582,
      "activations/layer9_attention_weight_max": 44.71209716796875,
      "activations/layer9_attention_weight_min": -46.52529525756836,
      "epoch": 16.13,
      "learning_rate": 4.6430681818181816e-05,
      "loss": 2.7441,
      "step": 277650
    },
    {
      "activations/layer0_attention_weight_max": 16.517148971557617,
      "activations/layer0_attention_weight_min": -13.625927925109863,
      "activations/layer10_attention_weight_max": 40.244991302490234,
      "activations/layer10_attention_weight_min": -40.21073913574219,
      "activations/layer11_attention_weight_max": 34.01241683959961,
      "activations/layer11_attention_weight_min": -35.4254150390625,
      "activations/layer12_attention_weight_max": 19.830902099609375,
      "activations/layer12_attention_weight_min": -28.38661003112793,
      "activations/layer13_attention_weight_max": 33.02519989013672,
      "activations/layer13_attention_weight_min": -30.49224853515625,
      "activations/layer14_attention_weight_max": 39.88666534423828,
      "activations/layer14_attention_weight_min": -34.3814811706543,
      "activations/layer15_attention_weight_max": 36.446502685546875,
      "activations/layer15_attention_weight_min": -32.145469665527344,
      "activations/layer16_attention_weight_max": 36.80453872680664,
      "activations/layer16_attention_weight_min": -32.01765441894531,
      "activations/layer17_attention_weight_max": 56.344173431396484,
      "activations/layer17_attention_weight_min": -48.80049133300781,
      "activations/layer18_attention_weight_max": 50.7078742980957,
      "activations/layer18_attention_weight_min": -42.87834548950195,
      "activations/layer19_attention_weight_max": 22.925996780395508,
      "activations/layer19_attention_weight_min": -24.27666664123535,
      "activations/layer1_attention_weight_max": 15.520345687866211,
      "activations/layer1_attention_weight_min": -13.055736541748047,
      "activations/layer20_attention_weight_max": 23.088781356811523,
      "activations/layer20_attention_weight_min": -22.399011611938477,
      "activations/layer21_attention_weight_max": 36.76277160644531,
      "activations/layer21_attention_weight_min": -23.162216186523438,
      "activations/layer22_attention_weight_max": 29.40384292602539,
      "activations/layer22_attention_weight_min": -27.824087142944336,
      "activations/layer23_attention_weight_max": 38.88884735107422,
      "activations/layer23_attention_weight_min": -26.106014251708984,
      "activations/layer2_attention_weight_max": 31.44099235534668,
      "activations/layer2_attention_weight_min": -31.238609313964844,
      "activations/layer3_attention_weight_max": 91.59941864013672,
      "activations/layer3_attention_weight_min": -91.05329132080078,
      "activations/layer4_attention_weight_max": 92.97415924072266,
      "activations/layer4_attention_weight_min": -95.86543273925781,
      "activations/layer5_attention_weight_max": 67.18279266357422,
      "activations/layer5_attention_weight_min": -76.82723236083984,
      "activations/layer6_attention_weight_max": 52.686832427978516,
      "activations/layer6_attention_weight_min": -56.13130569458008,
      "activations/layer7_attention_weight_max": 64.03799438476562,
      "activations/layer7_attention_weight_min": -66.9256591796875,
      "activations/layer8_attention_weight_max": 47.27522659301758,
      "activations/layer8_attention_weight_min": -51.027164459228516,
      "activations/layer9_attention_weight_max": 47.88410186767578,
      "activations/layer9_attention_weight_min": -51.46742630004883,
      "epoch": 16.14,
      "learning_rate": 4.641174242424242e-05,
      "loss": 2.7245,
      "step": 277700
    },
    {
      "activations/layer0_attention_weight_max": 16.011411666870117,
      "activations/layer0_attention_weight_min": -14.103038787841797,
      "activations/layer10_attention_weight_max": 39.607872009277344,
      "activations/layer10_attention_weight_min": -38.4830436706543,
      "activations/layer11_attention_weight_max": 34.98846435546875,
      "activations/layer11_attention_weight_min": -36.093563079833984,
      "activations/layer12_attention_weight_max": 17.876150131225586,
      "activations/layer12_attention_weight_min": -26.120887756347656,
      "activations/layer13_attention_weight_max": 35.45833969116211,
      "activations/layer13_attention_weight_min": -28.658601760864258,
      "activations/layer14_attention_weight_max": 33.71117401123047,
      "activations/layer14_attention_weight_min": -29.368671417236328,
      "activations/layer15_attention_weight_max": 32.72289276123047,
      "activations/layer15_attention_weight_min": -29.86628532409668,
      "activations/layer16_attention_weight_max": 32.870384216308594,
      "activations/layer16_attention_weight_min": -30.51653289794922,
      "activations/layer17_attention_weight_max": 48.466102600097656,
      "activations/layer17_attention_weight_min": -43.60719680786133,
      "activations/layer18_attention_weight_max": 44.11197280883789,
      "activations/layer18_attention_weight_min": -38.62478256225586,
      "activations/layer19_attention_weight_max": 21.241952896118164,
      "activations/layer19_attention_weight_min": -20.71882438659668,
      "activations/layer1_attention_weight_max": 16.194076538085938,
      "activations/layer1_attention_weight_min": -15.282354354858398,
      "activations/layer20_attention_weight_max": 21.1785888671875,
      "activations/layer20_attention_weight_min": -20.23743438720703,
      "activations/layer21_attention_weight_max": 33.79277420043945,
      "activations/layer21_attention_weight_min": -23.303354263305664,
      "activations/layer22_attention_weight_max": 27.499353408813477,
      "activations/layer22_attention_weight_min": -26.166839599609375,
      "activations/layer23_attention_weight_max": 37.047019958496094,
      "activations/layer23_attention_weight_min": -25.695066452026367,
      "activations/layer2_attention_weight_max": 33.110958099365234,
      "activations/layer2_attention_weight_min": -31.243331909179688,
      "activations/layer3_attention_weight_max": 93.02543640136719,
      "activations/layer3_attention_weight_min": -94.08415222167969,
      "activations/layer4_attention_weight_max": 94.52416229248047,
      "activations/layer4_attention_weight_min": -95.20821380615234,
      "activations/layer5_attention_weight_max": 68.70440673828125,
      "activations/layer5_attention_weight_min": -71.89334869384766,
      "activations/layer6_attention_weight_max": 52.80681610107422,
      "activations/layer6_attention_weight_min": -53.40972137451172,
      "activations/layer7_attention_weight_max": 68.60511779785156,
      "activations/layer7_attention_weight_min": -61.59531784057617,
      "activations/layer8_attention_weight_max": 48.63801574707031,
      "activations/layer8_attention_weight_min": -52.09952926635742,
      "activations/layer9_attention_weight_max": 49.436668395996094,
      "activations/layer9_attention_weight_min": -49.06433868408203,
      "epoch": 16.14,
      "learning_rate": 4.6392803030303026e-05,
      "loss": 2.7542,
      "step": 277750
    },
    {
      "activations/layer0_attention_weight_max": 15.902704238891602,
      "activations/layer0_attention_weight_min": -14.096260070800781,
      "activations/layer10_attention_weight_max": 38.00873565673828,
      "activations/layer10_attention_weight_min": -35.854774475097656,
      "activations/layer11_attention_weight_max": 36.30543518066406,
      "activations/layer11_attention_weight_min": -33.618568420410156,
      "activations/layer12_attention_weight_max": 18.869770050048828,
      "activations/layer12_attention_weight_min": -24.853904724121094,
      "activations/layer13_attention_weight_max": 32.950618743896484,
      "activations/layer13_attention_weight_min": -26.148563385009766,
      "activations/layer14_attention_weight_max": 36.818443298339844,
      "activations/layer14_attention_weight_min": -31.96651840209961,
      "activations/layer15_attention_weight_max": 33.488365173339844,
      "activations/layer15_attention_weight_min": -32.96607971191406,
      "activations/layer16_attention_weight_max": 38.81740188598633,
      "activations/layer16_attention_weight_min": -35.78138732910156,
      "activations/layer17_attention_weight_max": 54.648014068603516,
      "activations/layer17_attention_weight_min": -50.91961669921875,
      "activations/layer18_attention_weight_max": 46.336509704589844,
      "activations/layer18_attention_weight_min": -40.813499450683594,
      "activations/layer19_attention_weight_max": 23.686368942260742,
      "activations/layer19_attention_weight_min": -24.72650718688965,
      "activations/layer1_attention_weight_max": 15.908075332641602,
      "activations/layer1_attention_weight_min": -16.009906768798828,
      "activations/layer20_attention_weight_max": 23.692045211791992,
      "activations/layer20_attention_weight_min": -24.666370391845703,
      "activations/layer21_attention_weight_max": 35.64369201660156,
      "activations/layer21_attention_weight_min": -20.670364379882812,
      "activations/layer22_attention_weight_max": 30.195968627929688,
      "activations/layer22_attention_weight_min": -25.94062042236328,
      "activations/layer23_attention_weight_max": 35.4598503112793,
      "activations/layer23_attention_weight_min": -23.71162223815918,
      "activations/layer2_attention_weight_max": 33.66085433959961,
      "activations/layer2_attention_weight_min": -31.090011596679688,
      "activations/layer3_attention_weight_max": 96.933837890625,
      "activations/layer3_attention_weight_min": -93.35842895507812,
      "activations/layer4_attention_weight_max": 93.29617309570312,
      "activations/layer4_attention_weight_min": -88.30057525634766,
      "activations/layer5_attention_weight_max": 64.61624145507812,
      "activations/layer5_attention_weight_min": -69.68280029296875,
      "activations/layer6_attention_weight_max": 53.0208854675293,
      "activations/layer6_attention_weight_min": -54.01493453979492,
      "activations/layer7_attention_weight_max": 65.49242401123047,
      "activations/layer7_attention_weight_min": -60.776878356933594,
      "activations/layer8_attention_weight_max": 47.46736526489258,
      "activations/layer8_attention_weight_min": -47.73596954345703,
      "activations/layer9_attention_weight_max": 44.64553451538086,
      "activations/layer9_attention_weight_min": -46.385074615478516,
      "epoch": 16.14,
      "learning_rate": 4.637386363636363e-05,
      "loss": 2.7427,
      "step": 277800
    },
    {
      "activations/layer0_attention_weight_max": 16.525959014892578,
      "activations/layer0_attention_weight_min": -14.098471641540527,
      "activations/layer10_attention_weight_max": 48.68841552734375,
      "activations/layer10_attention_weight_min": -47.39158248901367,
      "activations/layer11_attention_weight_max": 47.62445068359375,
      "activations/layer11_attention_weight_min": -44.911415100097656,
      "activations/layer12_attention_weight_max": 23.6192569732666,
      "activations/layer12_attention_weight_min": -29.57443618774414,
      "activations/layer13_attention_weight_max": 59.6053352355957,
      "activations/layer13_attention_weight_min": -38.32667541503906,
      "activations/layer14_attention_weight_max": 69.9283447265625,
      "activations/layer14_attention_weight_min": -41.00354766845703,
      "activations/layer15_attention_weight_max": 66.66077423095703,
      "activations/layer15_attention_weight_min": -40.311500549316406,
      "activations/layer16_attention_weight_max": 50.554264068603516,
      "activations/layer16_attention_weight_min": -43.958168029785156,
      "activations/layer17_attention_weight_max": 79.2434310913086,
      "activations/layer17_attention_weight_min": -69.25325775146484,
      "activations/layer18_attention_weight_max": 77.16184997558594,
      "activations/layer18_attention_weight_min": -62.242374420166016,
      "activations/layer19_attention_weight_max": 37.514930725097656,
      "activations/layer19_attention_weight_min": -33.38924789428711,
      "activations/layer1_attention_weight_max": 17.53413963317871,
      "activations/layer1_attention_weight_min": -14.505049705505371,
      "activations/layer20_attention_weight_max": 39.463714599609375,
      "activations/layer20_attention_weight_min": -25.69476890563965,
      "activations/layer21_attention_weight_max": 60.864105224609375,
      "activations/layer21_attention_weight_min": -28.162803649902344,
      "activations/layer22_attention_weight_max": 46.35310745239258,
      "activations/layer22_attention_weight_min": -27.832265853881836,
      "activations/layer23_attention_weight_max": 47.94915008544922,
      "activations/layer23_attention_weight_min": -23.28583526611328,
      "activations/layer2_attention_weight_max": 36.376407623291016,
      "activations/layer2_attention_weight_min": -33.406494140625,
      "activations/layer3_attention_weight_max": 99.79434967041016,
      "activations/layer3_attention_weight_min": -96.6847152709961,
      "activations/layer4_attention_weight_max": 100.53167724609375,
      "activations/layer4_attention_weight_min": -93.04133605957031,
      "activations/layer5_attention_weight_max": 66.80225372314453,
      "activations/layer5_attention_weight_min": -78.93517303466797,
      "activations/layer6_attention_weight_max": 54.681453704833984,
      "activations/layer6_attention_weight_min": -53.59584426879883,
      "activations/layer7_attention_weight_max": 78.79110717773438,
      "activations/layer7_attention_weight_min": -64.00619506835938,
      "activations/layer8_attention_weight_max": 61.92903518676758,
      "activations/layer8_attention_weight_min": -54.56462097167969,
      "activations/layer9_attention_weight_max": 65.0086898803711,
      "activations/layer9_attention_weight_min": -51.400875091552734,
      "epoch": 16.14,
      "learning_rate": 4.635492424242424e-05,
      "loss": 2.7411,
      "step": 277850
    },
    {
      "activations/layer0_attention_weight_max": 16.014463424682617,
      "activations/layer0_attention_weight_min": -13.800687789916992,
      "activations/layer10_attention_weight_max": 36.51630401611328,
      "activations/layer10_attention_weight_min": -35.21622085571289,
      "activations/layer11_attention_weight_max": 32.012718200683594,
      "activations/layer11_attention_weight_min": -32.80723571777344,
      "activations/layer12_attention_weight_max": 20.624357223510742,
      "activations/layer12_attention_weight_min": -25.98227310180664,
      "activations/layer13_attention_weight_max": 38.99696731567383,
      "activations/layer13_attention_weight_min": -27.536945343017578,
      "activations/layer14_attention_weight_max": 35.50528335571289,
      "activations/layer14_attention_weight_min": -28.558013916015625,
      "activations/layer15_attention_weight_max": 34.17890167236328,
      "activations/layer15_attention_weight_min": -29.379751205444336,
      "activations/layer16_attention_weight_max": 32.81516647338867,
      "activations/layer16_attention_weight_min": -30.36608123779297,
      "activations/layer17_attention_weight_max": 52.57300567626953,
      "activations/layer17_attention_weight_min": -44.322296142578125,
      "activations/layer18_attention_weight_max": 47.507537841796875,
      "activations/layer18_attention_weight_min": -42.48935317993164,
      "activations/layer19_attention_weight_max": 23.776142120361328,
      "activations/layer19_attention_weight_min": -23.61809730529785,
      "activations/layer1_attention_weight_max": 15.80536937713623,
      "activations/layer1_attention_weight_min": -13.31050968170166,
      "activations/layer20_attention_weight_max": 24.987138748168945,
      "activations/layer20_attention_weight_min": -22.50919532775879,
      "activations/layer21_attention_weight_max": 36.1981201171875,
      "activations/layer21_attention_weight_min": -24.013378143310547,
      "activations/layer22_attention_weight_max": 33.493072509765625,
      "activations/layer22_attention_weight_min": -27.6640682220459,
      "activations/layer23_attention_weight_max": 41.45915222167969,
      "activations/layer23_attention_weight_min": -25.2397518157959,
      "activations/layer2_attention_weight_max": 30.720853805541992,
      "activations/layer2_attention_weight_min": -29.75728988647461,
      "activations/layer3_attention_weight_max": 86.60331726074219,
      "activations/layer3_attention_weight_min": -86.164306640625,
      "activations/layer4_attention_weight_max": 91.2164535522461,
      "activations/layer4_attention_weight_min": -87.87562561035156,
      "activations/layer5_attention_weight_max": 63.830623626708984,
      "activations/layer5_attention_weight_min": -76.39189147949219,
      "activations/layer6_attention_weight_max": 49.089900970458984,
      "activations/layer6_attention_weight_min": -50.06148147583008,
      "activations/layer7_attention_weight_max": 62.55998229980469,
      "activations/layer7_attention_weight_min": -58.707210540771484,
      "activations/layer8_attention_weight_max": 47.1140251159668,
      "activations/layer8_attention_weight_min": -50.5473518371582,
      "activations/layer9_attention_weight_max": 46.1455192565918,
      "activations/layer9_attention_weight_min": -45.01682662963867,
      "epoch": 16.15,
      "learning_rate": 4.6335984848484844e-05,
      "loss": 2.7439,
      "step": 277900
    },
    {
      "activations/layer0_attention_weight_max": 15.502157211303711,
      "activations/layer0_attention_weight_min": -14.126249313354492,
      "activations/layer10_attention_weight_max": 38.28948974609375,
      "activations/layer10_attention_weight_min": -37.31294631958008,
      "activations/layer11_attention_weight_max": 35.595855712890625,
      "activations/layer11_attention_weight_min": -36.778564453125,
      "activations/layer12_attention_weight_max": 20.035953521728516,
      "activations/layer12_attention_weight_min": -26.93150520324707,
      "activations/layer13_attention_weight_max": 37.534141540527344,
      "activations/layer13_attention_weight_min": -28.030860900878906,
      "activations/layer14_attention_weight_max": 46.136863708496094,
      "activations/layer14_attention_weight_min": -31.489179611206055,
      "activations/layer15_attention_weight_max": 41.15400695800781,
      "activations/layer15_attention_weight_min": -30.666757583618164,
      "activations/layer16_attention_weight_max": 39.056880950927734,
      "activations/layer16_attention_weight_min": -38.10961151123047,
      "activations/layer17_attention_weight_max": 56.491355895996094,
      "activations/layer17_attention_weight_min": -55.112545013427734,
      "activations/layer18_attention_weight_max": 50.128334045410156,
      "activations/layer18_attention_weight_min": -49.682804107666016,
      "activations/layer19_attention_weight_max": 23.626108169555664,
      "activations/layer19_attention_weight_min": -28.854080200195312,
      "activations/layer1_attention_weight_max": 18.049053192138672,
      "activations/layer1_attention_weight_min": -14.234861373901367,
      "activations/layer20_attention_weight_max": 23.743541717529297,
      "activations/layer20_attention_weight_min": -23.44950294494629,
      "activations/layer21_attention_weight_max": 43.10353469848633,
      "activations/layer21_attention_weight_min": -24.716581344604492,
      "activations/layer22_attention_weight_max": 29.835737228393555,
      "activations/layer22_attention_weight_min": -26.130836486816406,
      "activations/layer23_attention_weight_max": 38.57353210449219,
      "activations/layer23_attention_weight_min": -26.436330795288086,
      "activations/layer2_attention_weight_max": 32.484703063964844,
      "activations/layer2_attention_weight_min": -31.892921447753906,
      "activations/layer3_attention_weight_max": 91.98944091796875,
      "activations/layer3_attention_weight_min": -96.30866241455078,
      "activations/layer4_attention_weight_max": 89.85587310791016,
      "activations/layer4_attention_weight_min": -89.73532104492188,
      "activations/layer5_attention_weight_max": 67.01774597167969,
      "activations/layer5_attention_weight_min": -77.7624740600586,
      "activations/layer6_attention_weight_max": 50.76725387573242,
      "activations/layer6_attention_weight_min": -53.70891571044922,
      "activations/layer7_attention_weight_max": 63.55437469482422,
      "activations/layer7_attention_weight_min": -64.24878692626953,
      "activations/layer8_attention_weight_max": 50.38722610473633,
      "activations/layer8_attention_weight_min": -52.4033317565918,
      "activations/layer9_attention_weight_max": 46.2469596862793,
      "activations/layer9_attention_weight_min": -50.08811569213867,
      "epoch": 16.15,
      "learning_rate": 4.6317045454545446e-05,
      "loss": 2.7546,
      "step": 277950
    },
    {
      "activations/layer0_attention_weight_max": 16.0210018157959,
      "activations/layer0_attention_weight_min": -14.071263313293457,
      "activations/layer10_attention_weight_max": 36.158748626708984,
      "activations/layer10_attention_weight_min": -35.29270935058594,
      "activations/layer11_attention_weight_max": 33.83932113647461,
      "activations/layer11_attention_weight_min": -34.77864074707031,
      "activations/layer12_attention_weight_max": 18.879240036010742,
      "activations/layer12_attention_weight_min": -24.065393447875977,
      "activations/layer13_attention_weight_max": 33.79520034790039,
      "activations/layer13_attention_weight_min": -29.29724884033203,
      "activations/layer14_attention_weight_max": 39.00999069213867,
      "activations/layer14_attention_weight_min": -29.086959838867188,
      "activations/layer15_attention_weight_max": 42.56941223144531,
      "activations/layer15_attention_weight_min": -29.018789291381836,
      "activations/layer16_attention_weight_max": 33.431636810302734,
      "activations/layer16_attention_weight_min": -30.35909080505371,
      "activations/layer17_attention_weight_max": 53.14098358154297,
      "activations/layer17_attention_weight_min": -42.96905517578125,
      "activations/layer18_attention_weight_max": 46.87167739868164,
      "activations/layer18_attention_weight_min": -38.708885192871094,
      "activations/layer19_attention_weight_max": 24.686128616333008,
      "activations/layer19_attention_weight_min": -22.348196029663086,
      "activations/layer1_attention_weight_max": 15.561640739440918,
      "activations/layer1_attention_weight_min": -13.641609191894531,
      "activations/layer20_attention_weight_max": 23.51812744140625,
      "activations/layer20_attention_weight_min": -23.419103622436523,
      "activations/layer21_attention_weight_max": 38.94310760498047,
      "activations/layer21_attention_weight_min": -24.69684410095215,
      "activations/layer22_attention_weight_max": 32.72766876220703,
      "activations/layer22_attention_weight_min": -27.163103103637695,
      "activations/layer23_attention_weight_max": 42.45995330810547,
      "activations/layer23_attention_weight_min": -25.27147674560547,
      "activations/layer2_attention_weight_max": 31.097137451171875,
      "activations/layer2_attention_weight_min": -30.020732879638672,
      "activations/layer3_attention_weight_max": 89.79042053222656,
      "activations/layer3_attention_weight_min": -85.82479858398438,
      "activations/layer4_attention_weight_max": 92.9489517211914,
      "activations/layer4_attention_weight_min": -88.36962127685547,
      "activations/layer5_attention_weight_max": 65.85355377197266,
      "activations/layer5_attention_weight_min": -74.2022705078125,
      "activations/layer6_attention_weight_max": 50.6699104309082,
      "activations/layer6_attention_weight_min": -50.834835052490234,
      "activations/layer7_attention_weight_max": 64.8606185913086,
      "activations/layer7_attention_weight_min": -60.23899459838867,
      "activations/layer8_attention_weight_max": 43.97161102294922,
      "activations/layer8_attention_weight_min": -44.97920608520508,
      "activations/layer9_attention_weight_max": 45.558380126953125,
      "activations/layer9_attention_weight_min": -45.83859634399414,
      "epoch": 16.15,
      "learning_rate": 4.629848484848484e-05,
      "loss": 2.7503,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4219,
      "eval_samples_per_second": 509.86,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4219,
      "eval_openwebtext_samples_per_second": 509.86,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.921,
      "eval_wikitext_samples_per_second": 237.378,
      "step": 278000
    },
    {
      "epoch": 16.15,
      "eval_lambada_loss": 2.46484375,
      "eval_lambada_ppl": 11.761644251483835,
      "eval_lambada_runtime": 9.5184,
      "eval_lambada_samples_per_second": 511.538,
      "step": 278000
    },
    {
      "activations/layer0_attention_weight_max": 15.588592529296875,
      "activations/layer0_attention_weight_min": -13.64263916015625,
      "activations/layer10_attention_weight_max": 36.055572509765625,
      "activations/layer10_attention_weight_min": -33.912811279296875,
      "activations/layer11_attention_weight_max": 32.331298828125,
      "activations/layer11_attention_weight_min": -34.68464660644531,
      "activations/layer12_attention_weight_max": 18.150720596313477,
      "activations/layer12_attention_weight_min": -22.454017639160156,
      "activations/layer13_attention_weight_max": 46.25160217285156,
      "activations/layer13_attention_weight_min": -32.23088836669922,
      "activations/layer14_attention_weight_max": 34.59767150878906,
      "activations/layer14_attention_weight_min": -30.57752227783203,
      "activations/layer15_attention_weight_max": 32.958526611328125,
      "activations/layer15_attention_weight_min": -29.117483139038086,
      "activations/layer16_attention_weight_max": 33.11798858642578,
      "activations/layer16_attention_weight_min": -32.86613845825195,
      "activations/layer17_attention_weight_max": 54.84797286987305,
      "activations/layer17_attention_weight_min": -46.95732879638672,
      "activations/layer18_attention_weight_max": 47.83513641357422,
      "activations/layer18_attention_weight_min": -41.95084762573242,
      "activations/layer19_attention_weight_max": 23.6734676361084,
      "activations/layer19_attention_weight_min": -22.90277671813965,
      "activations/layer1_attention_weight_max": 16.224641799926758,
      "activations/layer1_attention_weight_min": -14.770344734191895,
      "activations/layer20_attention_weight_max": 22.887937545776367,
      "activations/layer20_attention_weight_min": -22.781658172607422,
      "activations/layer21_attention_weight_max": 36.78700256347656,
      "activations/layer21_attention_weight_min": -22.53096580505371,
      "activations/layer22_attention_weight_max": 29.805416107177734,
      "activations/layer22_attention_weight_min": -26.83739471435547,
      "activations/layer23_attention_weight_max": 39.03368377685547,
      "activations/layer23_attention_weight_min": -24.347280502319336,
      "activations/layer2_attention_weight_max": 30.784133911132812,
      "activations/layer2_attention_weight_min": -30.569412231445312,
      "activations/layer3_attention_weight_max": 83.6046371459961,
      "activations/layer3_attention_weight_min": -81.37178802490234,
      "activations/layer4_attention_weight_max": 89.85145568847656,
      "activations/layer4_attention_weight_min": -85.11544036865234,
      "activations/layer5_attention_weight_max": 67.2125015258789,
      "activations/layer5_attention_weight_min": -78.79744720458984,
      "activations/layer6_attention_weight_max": 51.18038558959961,
      "activations/layer6_attention_weight_min": -52.91592788696289,
      "activations/layer7_attention_weight_max": 62.545230865478516,
      "activations/layer7_attention_weight_min": -60.03192138671875,
      "activations/layer8_attention_weight_max": 42.57967758178711,
      "activations/layer8_attention_weight_min": -47.15101623535156,
      "activations/layer9_attention_weight_max": 41.791954040527344,
      "activations/layer9_attention_weight_min": -44.0094108581543,
      "epoch": 16.16,
      "learning_rate": 4.627954545454545e-05,
      "loss": 2.7511,
      "step": 278050
    },
    {
      "activations/layer0_attention_weight_max": 16.06336784362793,
      "activations/layer0_attention_weight_min": -13.424495697021484,
      "activations/layer10_attention_weight_max": 35.19757080078125,
      "activations/layer10_attention_weight_min": -36.979915618896484,
      "activations/layer11_attention_weight_max": 32.58265686035156,
      "activations/layer11_attention_weight_min": -34.55640411376953,
      "activations/layer12_attention_weight_max": 17.86771011352539,
      "activations/layer12_attention_weight_min": -25.241622924804688,
      "activations/layer13_attention_weight_max": 34.158058166503906,
      "activations/layer13_attention_weight_min": -29.435100555419922,
      "activations/layer14_attention_weight_max": 35.45396041870117,
      "activations/layer14_attention_weight_min": -32.53717041015625,
      "activations/layer15_attention_weight_max": 32.924747467041016,
      "activations/layer15_attention_weight_min": -30.816495895385742,
      "activations/layer16_attention_weight_max": 33.665584564208984,
      "activations/layer16_attention_weight_min": -31.75486946105957,
      "activations/layer17_attention_weight_max": 50.02024459838867,
      "activations/layer17_attention_weight_min": -44.54127883911133,
      "activations/layer18_attention_weight_max": 48.285926818847656,
      "activations/layer18_attention_weight_min": -40.978965759277344,
      "activations/layer19_attention_weight_max": 26.2227725982666,
      "activations/layer19_attention_weight_min": -27.703720092773438,
      "activations/layer1_attention_weight_max": 16.685718536376953,
      "activations/layer1_attention_weight_min": -15.962855339050293,
      "activations/layer20_attention_weight_max": 26.145793914794922,
      "activations/layer20_attention_weight_min": -24.357641220092773,
      "activations/layer21_attention_weight_max": 38.988319396972656,
      "activations/layer21_attention_weight_min": -25.217931747436523,
      "activations/layer22_attention_weight_max": 31.92422866821289,
      "activations/layer22_attention_weight_min": -27.39063262939453,
      "activations/layer23_attention_weight_max": 42.467437744140625,
      "activations/layer23_attention_weight_min": -27.195770263671875,
      "activations/layer2_attention_weight_max": 33.67649841308594,
      "activations/layer2_attention_weight_min": -31.144163131713867,
      "activations/layer3_attention_weight_max": 90.1633071899414,
      "activations/layer3_attention_weight_min": -94.00010681152344,
      "activations/layer4_attention_weight_max": 95.60999298095703,
      "activations/layer4_attention_weight_min": -89.6411361694336,
      "activations/layer5_attention_weight_max": 65.91294860839844,
      "activations/layer5_attention_weight_min": -74.23466491699219,
      "activations/layer6_attention_weight_max": 55.02467727661133,
      "activations/layer6_attention_weight_min": -55.136138916015625,
      "activations/layer7_attention_weight_max": 64.56838989257812,
      "activations/layer7_attention_weight_min": -63.442832946777344,
      "activations/layer8_attention_weight_max": 46.247432708740234,
      "activations/layer8_attention_weight_min": -48.32691192626953,
      "activations/layer9_attention_weight_max": 43.61651611328125,
      "activations/layer9_attention_weight_min": -44.89108657836914,
      "epoch": 16.16,
      "learning_rate": 4.626060606060605e-05,
      "loss": 2.7379,
      "step": 278100
    },
    {
      "activations/layer0_attention_weight_max": 15.704741477966309,
      "activations/layer0_attention_weight_min": -13.81688117980957,
      "activations/layer10_attention_weight_max": 33.7698974609375,
      "activations/layer10_attention_weight_min": -35.626319885253906,
      "activations/layer11_attention_weight_max": 34.038185119628906,
      "activations/layer11_attention_weight_min": -32.696205139160156,
      "activations/layer12_attention_weight_max": 17.463573455810547,
      "activations/layer12_attention_weight_min": -23.119667053222656,
      "activations/layer13_attention_weight_max": 33.053802490234375,
      "activations/layer13_attention_weight_min": -27.499948501586914,
      "activations/layer14_attention_weight_max": 35.12396240234375,
      "activations/layer14_attention_weight_min": -29.716997146606445,
      "activations/layer15_attention_weight_max": 32.87548828125,
      "activations/layer15_attention_weight_min": -31.34878921508789,
      "activations/layer16_attention_weight_max": 34.750526428222656,
      "activations/layer16_attention_weight_min": -30.54645538330078,
      "activations/layer17_attention_weight_max": 55.8107795715332,
      "activations/layer17_attention_weight_min": -44.191436767578125,
      "activations/layer18_attention_weight_max": 50.63959884643555,
      "activations/layer18_attention_weight_min": -38.553077697753906,
      "activations/layer19_attention_weight_max": 24.2287540435791,
      "activations/layer19_attention_weight_min": -21.991329193115234,
      "activations/layer1_attention_weight_max": 15.714591979980469,
      "activations/layer1_attention_weight_min": -14.656623840332031,
      "activations/layer20_attention_weight_max": 24.244033813476562,
      "activations/layer20_attention_weight_min": -21.63316535949707,
      "activations/layer21_attention_weight_max": 40.195098876953125,
      "activations/layer21_attention_weight_min": -22.553558349609375,
      "activations/layer22_attention_weight_max": 28.427501678466797,
      "activations/layer22_attention_weight_min": -25.06893539428711,
      "activations/layer23_attention_weight_max": 36.15460205078125,
      "activations/layer23_attention_weight_min": -22.802831649780273,
      "activations/layer2_attention_weight_max": 30.354299545288086,
      "activations/layer2_attention_weight_min": -29.114912033081055,
      "activations/layer3_attention_weight_max": 84.49286651611328,
      "activations/layer3_attention_weight_min": -86.01021575927734,
      "activations/layer4_attention_weight_max": 87.51915740966797,
      "activations/layer4_attention_weight_min": -86.49308776855469,
      "activations/layer5_attention_weight_max": 65.03697204589844,
      "activations/layer5_attention_weight_min": -72.354736328125,
      "activations/layer6_attention_weight_max": 50.00901412963867,
      "activations/layer6_attention_weight_min": -51.14385223388672,
      "activations/layer7_attention_weight_max": 64.48564147949219,
      "activations/layer7_attention_weight_min": -62.56396484375,
      "activations/layer8_attention_weight_max": 45.447689056396484,
      "activations/layer8_attention_weight_min": -49.83574295043945,
      "activations/layer9_attention_weight_max": 43.116329193115234,
      "activations/layer9_attention_weight_min": -44.3492546081543,
      "epoch": 16.16,
      "learning_rate": 4.624166666666666e-05,
      "loss": 2.743,
      "step": 278150
    },
    {
      "activations/layer0_attention_weight_max": 15.29507064819336,
      "activations/layer0_attention_weight_min": -13.562490463256836,
      "activations/layer10_attention_weight_max": 36.54408264160156,
      "activations/layer10_attention_weight_min": -36.731910705566406,
      "activations/layer11_attention_weight_max": 31.705381393432617,
      "activations/layer11_attention_weight_min": -33.38276672363281,
      "activations/layer12_attention_weight_max": 16.674222946166992,
      "activations/layer12_attention_weight_min": -30.6472225189209,
      "activations/layer13_attention_weight_max": 36.05202102661133,
      "activations/layer13_attention_weight_min": -25.91505241394043,
      "activations/layer14_attention_weight_max": 39.24406051635742,
      "activations/layer14_attention_weight_min": -28.3880672454834,
      "activations/layer15_attention_weight_max": 38.258811950683594,
      "activations/layer15_attention_weight_min": -28.915821075439453,
      "activations/layer16_attention_weight_max": 31.582719802856445,
      "activations/layer16_attention_weight_min": -29.99891471862793,
      "activations/layer17_attention_weight_max": 50.78386306762695,
      "activations/layer17_attention_weight_min": -42.97747802734375,
      "activations/layer18_attention_weight_max": 44.92764663696289,
      "activations/layer18_attention_weight_min": -37.56763458251953,
      "activations/layer19_attention_weight_max": 22.816503524780273,
      "activations/layer19_attention_weight_min": -21.93752098083496,
      "activations/layer1_attention_weight_max": 16.417644500732422,
      "activations/layer1_attention_weight_min": -15.061641693115234,
      "activations/layer20_attention_weight_max": 24.32126808166504,
      "activations/layer20_attention_weight_min": -23.661479949951172,
      "activations/layer21_attention_weight_max": 37.52278518676758,
      "activations/layer21_attention_weight_min": -24.590526580810547,
      "activations/layer22_attention_weight_max": 26.831729888916016,
      "activations/layer22_attention_weight_min": -24.20416259765625,
      "activations/layer23_attention_weight_max": 35.13754653930664,
      "activations/layer23_attention_weight_min": -29.77286148071289,
      "activations/layer2_attention_weight_max": 32.85647201538086,
      "activations/layer2_attention_weight_min": -31.218338012695312,
      "activations/layer3_attention_weight_max": 91.81481170654297,
      "activations/layer3_attention_weight_min": -89.36504364013672,
      "activations/layer4_attention_weight_max": 91.01915740966797,
      "activations/layer4_attention_weight_min": -90.01744842529297,
      "activations/layer5_attention_weight_max": 66.74055480957031,
      "activations/layer5_attention_weight_min": -75.7315673828125,
      "activations/layer6_attention_weight_max": 51.53779983520508,
      "activations/layer6_attention_weight_min": -52.888179779052734,
      "activations/layer7_attention_weight_max": 61.273075103759766,
      "activations/layer7_attention_weight_min": -64.81498718261719,
      "activations/layer8_attention_weight_max": 46.595924377441406,
      "activations/layer8_attention_weight_min": -54.13631057739258,
      "activations/layer9_attention_weight_max": 47.52102279663086,
      "activations/layer9_attention_weight_min": -48.33478927612305,
      "epoch": 16.17,
      "learning_rate": 4.622272727272727e-05,
      "loss": 2.7454,
      "step": 278200
    },
    {
      "activations/layer0_attention_weight_max": 15.5996675491333,
      "activations/layer0_attention_weight_min": -13.960671424865723,
      "activations/layer10_attention_weight_max": 39.074134826660156,
      "activations/layer10_attention_weight_min": -36.99111557006836,
      "activations/layer11_attention_weight_max": 33.37199401855469,
      "activations/layer11_attention_weight_min": -34.25292205810547,
      "activations/layer12_attention_weight_max": 19.569915771484375,
      "activations/layer12_attention_weight_min": -26.949094772338867,
      "activations/layer13_attention_weight_max": 36.40863037109375,
      "activations/layer13_attention_weight_min": -28.736209869384766,
      "activations/layer14_attention_weight_max": 33.34068298339844,
      "activations/layer14_attention_weight_min": -30.093299865722656,
      "activations/layer15_attention_weight_max": 31.561769485473633,
      "activations/layer15_attention_weight_min": -30.38905906677246,
      "activations/layer16_attention_weight_max": 30.958097457885742,
      "activations/layer16_attention_weight_min": -30.589567184448242,
      "activations/layer17_attention_weight_max": 50.11924362182617,
      "activations/layer17_attention_weight_min": -43.887916564941406,
      "activations/layer18_attention_weight_max": 44.132965087890625,
      "activations/layer18_attention_weight_min": -38.07819747924805,
      "activations/layer19_attention_weight_max": 21.02427864074707,
      "activations/layer19_attention_weight_min": -21.16819953918457,
      "activations/layer1_attention_weight_max": 15.33695125579834,
      "activations/layer1_attention_weight_min": -14.357851028442383,
      "activations/layer20_attention_weight_max": 19.592952728271484,
      "activations/layer20_attention_weight_min": -22.813701629638672,
      "activations/layer21_attention_weight_max": 37.71448516845703,
      "activations/layer21_attention_weight_min": -25.64042091369629,
      "activations/layer22_attention_weight_max": 29.61239242553711,
      "activations/layer22_attention_weight_min": -30.21487808227539,
      "activations/layer23_attention_weight_max": 34.26179885864258,
      "activations/layer23_attention_weight_min": -25.357254028320312,
      "activations/layer2_attention_weight_max": 31.396453857421875,
      "activations/layer2_attention_weight_min": -30.753498077392578,
      "activations/layer3_attention_weight_max": 83.52191925048828,
      "activations/layer3_attention_weight_min": -84.35101318359375,
      "activations/layer4_attention_weight_max": 87.21407318115234,
      "activations/layer4_attention_weight_min": -81.13485717773438,
      "activations/layer5_attention_weight_max": 65.46174621582031,
      "activations/layer5_attention_weight_min": -75.23023986816406,
      "activations/layer6_attention_weight_max": 51.05620574951172,
      "activations/layer6_attention_weight_min": -49.42638397216797,
      "activations/layer7_attention_weight_max": 64.6945571899414,
      "activations/layer7_attention_weight_min": -62.52130126953125,
      "activations/layer8_attention_weight_max": 44.72492599487305,
      "activations/layer8_attention_weight_min": -49.327117919921875,
      "activations/layer9_attention_weight_max": 46.53127670288086,
      "activations/layer9_attention_weight_min": -45.10853576660156,
      "epoch": 16.17,
      "learning_rate": 4.620378787878788e-05,
      "loss": 2.7303,
      "step": 278250
    },
    {
      "activations/layer0_attention_weight_max": 15.92279052734375,
      "activations/layer0_attention_weight_min": -13.391281127929688,
      "activations/layer10_attention_weight_max": 36.5190315246582,
      "activations/layer10_attention_weight_min": -36.09928894042969,
      "activations/layer11_attention_weight_max": 34.4361572265625,
      "activations/layer11_attention_weight_min": -33.18739318847656,
      "activations/layer12_attention_weight_max": 16.607309341430664,
      "activations/layer12_attention_weight_min": -30.81493377685547,
      "activations/layer13_attention_weight_max": 31.793832778930664,
      "activations/layer13_attention_weight_min": -27.884506225585938,
      "activations/layer14_attention_weight_max": 33.39704132080078,
      "activations/layer14_attention_weight_min": -28.85699462890625,
      "activations/layer15_attention_weight_max": 32.395851135253906,
      "activations/layer15_attention_weight_min": -28.66002082824707,
      "activations/layer16_attention_weight_max": 32.04372024536133,
      "activations/layer16_attention_weight_min": -29.141605377197266,
      "activations/layer17_attention_weight_max": 47.75434494018555,
      "activations/layer17_attention_weight_min": -40.21071243286133,
      "activations/layer18_attention_weight_max": 46.2893180847168,
      "activations/layer18_attention_weight_min": -36.592674255371094,
      "activations/layer19_attention_weight_max": 25.961759567260742,
      "activations/layer19_attention_weight_min": -23.173572540283203,
      "activations/layer1_attention_weight_max": 17.349456787109375,
      "activations/layer1_attention_weight_min": -13.566605567932129,
      "activations/layer20_attention_weight_max": 26.234619140625,
      "activations/layer20_attention_weight_min": -20.821645736694336,
      "activations/layer21_attention_weight_max": 40.5137939453125,
      "activations/layer21_attention_weight_min": -23.004751205444336,
      "activations/layer22_attention_weight_max": 30.639644622802734,
      "activations/layer22_attention_weight_min": -24.92983055114746,
      "activations/layer23_attention_weight_max": 40.674720764160156,
      "activations/layer23_attention_weight_min": -26.517253875732422,
      "activations/layer2_attention_weight_max": 31.86139678955078,
      "activations/layer2_attention_weight_min": -29.475448608398438,
      "activations/layer3_attention_weight_max": 89.16639709472656,
      "activations/layer3_attention_weight_min": -84.64747619628906,
      "activations/layer4_attention_weight_max": 93.1076889038086,
      "activations/layer4_attention_weight_min": -83.90914916992188,
      "activations/layer5_attention_weight_max": 67.70169830322266,
      "activations/layer5_attention_weight_min": -70.07413482666016,
      "activations/layer6_attention_weight_max": 49.95138931274414,
      "activations/layer6_attention_weight_min": -51.63325500488281,
      "activations/layer7_attention_weight_max": 62.156761169433594,
      "activations/layer7_attention_weight_min": -62.86919021606445,
      "activations/layer8_attention_weight_max": 48.27848434448242,
      "activations/layer8_attention_weight_min": -48.520687103271484,
      "activations/layer9_attention_weight_max": 52.16925048828125,
      "activations/layer9_attention_weight_min": -47.892906188964844,
      "epoch": 16.17,
      "learning_rate": 4.618484848484848e-05,
      "loss": 2.748,
      "step": 278300
    },
    {
      "activations/layer0_attention_weight_max": 16.675966262817383,
      "activations/layer0_attention_weight_min": -13.852829933166504,
      "activations/layer10_attention_weight_max": 36.59688186645508,
      "activations/layer10_attention_weight_min": -37.56893539428711,
      "activations/layer11_attention_weight_max": 33.1240234375,
      "activations/layer11_attention_weight_min": -37.53947067260742,
      "activations/layer12_attention_weight_max": 18.306631088256836,
      "activations/layer12_attention_weight_min": -24.092737197875977,
      "activations/layer13_attention_weight_max": 36.214134216308594,
      "activations/layer13_attention_weight_min": -28.94877815246582,
      "activations/layer14_attention_weight_max": 36.70320129394531,
      "activations/layer14_attention_weight_min": -31.666505813598633,
      "activations/layer15_attention_weight_max": 31.703153610229492,
      "activations/layer15_attention_weight_min": -29.901405334472656,
      "activations/layer16_attention_weight_max": 34.28921890258789,
      "activations/layer16_attention_weight_min": -32.074405670166016,
      "activations/layer17_attention_weight_max": 51.54161834716797,
      "activations/layer17_attention_weight_min": -49.7845573425293,
      "activations/layer18_attention_weight_max": 44.45220947265625,
      "activations/layer18_attention_weight_min": -40.22245407104492,
      "activations/layer19_attention_weight_max": 22.571880340576172,
      "activations/layer19_attention_weight_min": -25.263835906982422,
      "activations/layer1_attention_weight_max": 14.914017677307129,
      "activations/layer1_attention_weight_min": -14.050113677978516,
      "activations/layer20_attention_weight_max": 24.352876663208008,
      "activations/layer20_attention_weight_min": -23.222063064575195,
      "activations/layer21_attention_weight_max": 30.954374313354492,
      "activations/layer21_attention_weight_min": -23.893484115600586,
      "activations/layer22_attention_weight_max": 27.4738826751709,
      "activations/layer22_attention_weight_min": -27.610261917114258,
      "activations/layer23_attention_weight_max": 38.146080017089844,
      "activations/layer23_attention_weight_min": -26.648216247558594,
      "activations/layer2_attention_weight_max": 32.8402214050293,
      "activations/layer2_attention_weight_min": -32.218902587890625,
      "activations/layer3_attention_weight_max": 97.08894348144531,
      "activations/layer3_attention_weight_min": -93.64395141601562,
      "activations/layer4_attention_weight_max": 94.68177032470703,
      "activations/layer4_attention_weight_min": -93.00730895996094,
      "activations/layer5_attention_weight_max": 69.32158660888672,
      "activations/layer5_attention_weight_min": -78.8194580078125,
      "activations/layer6_attention_weight_max": 53.290428161621094,
      "activations/layer6_attention_weight_min": -54.94769287109375,
      "activations/layer7_attention_weight_max": 65.7254409790039,
      "activations/layer7_attention_weight_min": -68.24143981933594,
      "activations/layer8_attention_weight_max": 47.40000915527344,
      "activations/layer8_attention_weight_min": -49.27138900756836,
      "activations/layer9_attention_weight_max": 46.25247573852539,
      "activations/layer9_attention_weight_min": -48.75355911254883,
      "epoch": 16.17,
      "learning_rate": 4.616590909090908e-05,
      "loss": 2.7437,
      "step": 278350
    },
    {
      "activations/layer0_attention_weight_max": 16.01784324645996,
      "activations/layer0_attention_weight_min": -13.588387489318848,
      "activations/layer10_attention_weight_max": 34.64173126220703,
      "activations/layer10_attention_weight_min": -35.262245178222656,
      "activations/layer11_attention_weight_max": 31.578611373901367,
      "activations/layer11_attention_weight_min": -32.65248489379883,
      "activations/layer12_attention_weight_max": 17.55234718322754,
      "activations/layer12_attention_weight_min": -24.113054275512695,
      "activations/layer13_attention_weight_max": 33.429603576660156,
      "activations/layer13_attention_weight_min": -26.31462287902832,
      "activations/layer14_attention_weight_max": 33.33475875854492,
      "activations/layer14_attention_weight_min": -29.920808792114258,
      "activations/layer15_attention_weight_max": 31.31545639038086,
      "activations/layer15_attention_weight_min": -29.50494956970215,
      "activations/layer16_attention_weight_max": 30.37327766418457,
      "activations/layer16_attention_weight_min": -31.860929489135742,
      "activations/layer17_attention_weight_max": 50.0723762512207,
      "activations/layer17_attention_weight_min": -44.4619026184082,
      "activations/layer18_attention_weight_max": 43.76398849487305,
      "activations/layer18_attention_weight_min": -37.753814697265625,
      "activations/layer19_attention_weight_max": 24.041275024414062,
      "activations/layer19_attention_weight_min": -21.22835350036621,
      "activations/layer1_attention_weight_max": 15.392780303955078,
      "activations/layer1_attention_weight_min": -14.19008731842041,
      "activations/layer20_attention_weight_max": 20.11217498779297,
      "activations/layer20_attention_weight_min": -22.50222396850586,
      "activations/layer21_attention_weight_max": 33.872901916503906,
      "activations/layer21_attention_weight_min": -23.986309051513672,
      "activations/layer22_attention_weight_max": 25.954362869262695,
      "activations/layer22_attention_weight_min": -25.35955810546875,
      "activations/layer23_attention_weight_max": 35.61144256591797,
      "activations/layer23_attention_weight_min": -24.253925323486328,
      "activations/layer2_attention_weight_max": 32.81380844116211,
      "activations/layer2_attention_weight_min": -29.42572021484375,
      "activations/layer3_attention_weight_max": 88.84091186523438,
      "activations/layer3_attention_weight_min": -87.10831451416016,
      "activations/layer4_attention_weight_max": 91.79177856445312,
      "activations/layer4_attention_weight_min": -90.79016876220703,
      "activations/layer5_attention_weight_max": 63.66683578491211,
      "activations/layer5_attention_weight_min": -74.56049346923828,
      "activations/layer6_attention_weight_max": 51.112449645996094,
      "activations/layer6_attention_weight_min": -51.7583122253418,
      "activations/layer7_attention_weight_max": 60.15188980102539,
      "activations/layer7_attention_weight_min": -60.632659912109375,
      "activations/layer8_attention_weight_max": 44.32034683227539,
      "activations/layer8_attention_weight_min": -48.69172286987305,
      "activations/layer9_attention_weight_max": 42.431915283203125,
      "activations/layer9_attention_weight_min": -46.3033561706543,
      "epoch": 16.18,
      "learning_rate": 4.6146969696969696e-05,
      "loss": 2.7371,
      "step": 278400
    },
    {
      "activations/layer0_attention_weight_max": 15.973546028137207,
      "activations/layer0_attention_weight_min": -14.139723777770996,
      "activations/layer10_attention_weight_max": 38.873355865478516,
      "activations/layer10_attention_weight_min": -35.81489944458008,
      "activations/layer11_attention_weight_max": 31.904033660888672,
      "activations/layer11_attention_weight_min": -33.448974609375,
      "activations/layer12_attention_weight_max": 17.173355102539062,
      "activations/layer12_attention_weight_min": -28.135663986206055,
      "activations/layer13_attention_weight_max": 36.01955795288086,
      "activations/layer13_attention_weight_min": -26.104761123657227,
      "activations/layer14_attention_weight_max": 36.509765625,
      "activations/layer14_attention_weight_min": -29.058141708374023,
      "activations/layer15_attention_weight_max": 31.8464412689209,
      "activations/layer15_attention_weight_min": -28.613412857055664,
      "activations/layer16_attention_weight_max": 33.69727325439453,
      "activations/layer16_attention_weight_min": -33.139122009277344,
      "activations/layer17_attention_weight_max": 48.54207229614258,
      "activations/layer17_attention_weight_min": -44.78902053833008,
      "activations/layer18_attention_weight_max": 44.660255432128906,
      "activations/layer18_attention_weight_min": -38.575252532958984,
      "activations/layer19_attention_weight_max": 22.773475646972656,
      "activations/layer19_attention_weight_min": -23.25012969970703,
      "activations/layer1_attention_weight_max": 17.940282821655273,
      "activations/layer1_attention_weight_min": -12.778889656066895,
      "activations/layer20_attention_weight_max": 23.326553344726562,
      "activations/layer20_attention_weight_min": -21.2989559173584,
      "activations/layer21_attention_weight_max": 35.83686065673828,
      "activations/layer21_attention_weight_min": -26.729110717773438,
      "activations/layer22_attention_weight_max": 29.373092651367188,
      "activations/layer22_attention_weight_min": -25.26152229309082,
      "activations/layer23_attention_weight_max": 35.634559631347656,
      "activations/layer23_attention_weight_min": -29.7939510345459,
      "activations/layer2_attention_weight_max": 30.71142578125,
      "activations/layer2_attention_weight_min": -29.488645553588867,
      "activations/layer3_attention_weight_max": 89.47209167480469,
      "activations/layer3_attention_weight_min": -91.62589263916016,
      "activations/layer4_attention_weight_max": 90.78297424316406,
      "activations/layer4_attention_weight_min": -91.23099517822266,
      "activations/layer5_attention_weight_max": 67.19022369384766,
      "activations/layer5_attention_weight_min": -73.00017547607422,
      "activations/layer6_attention_weight_max": 51.013675689697266,
      "activations/layer6_attention_weight_min": -52.86618423461914,
      "activations/layer7_attention_weight_max": 61.36235427856445,
      "activations/layer7_attention_weight_min": -63.15312957763672,
      "activations/layer8_attention_weight_max": 45.163639068603516,
      "activations/layer8_attention_weight_min": -51.8576545715332,
      "activations/layer9_attention_weight_max": 43.760982513427734,
      "activations/layer9_attention_weight_min": -46.71904754638672,
      "epoch": 16.18,
      "learning_rate": 4.61280303030303e-05,
      "loss": 2.7451,
      "step": 278450
    },
    {
      "activations/layer0_attention_weight_max": 16.070764541625977,
      "activations/layer0_attention_weight_min": -13.645772933959961,
      "activations/layer10_attention_weight_max": 36.88642120361328,
      "activations/layer10_attention_weight_min": -35.614471435546875,
      "activations/layer11_attention_weight_max": 34.110382080078125,
      "activations/layer11_attention_weight_min": -34.10956573486328,
      "activations/layer12_attention_weight_max": 18.38895606994629,
      "activations/layer12_attention_weight_min": -23.48465919494629,
      "activations/layer13_attention_weight_max": 35.38996124267578,
      "activations/layer13_attention_weight_min": -28.41433334350586,
      "activations/layer14_attention_weight_max": 34.91293716430664,
      "activations/layer14_attention_weight_min": -28.906206130981445,
      "activations/layer15_attention_weight_max": 31.815515518188477,
      "activations/layer15_attention_weight_min": -28.41856575012207,
      "activations/layer16_attention_weight_max": 31.84540557861328,
      "activations/layer16_attention_weight_min": -31.75889015197754,
      "activations/layer17_attention_weight_max": 52.1092643737793,
      "activations/layer17_attention_weight_min": -43.917869567871094,
      "activations/layer18_attention_weight_max": 41.96174621582031,
      "activations/layer18_attention_weight_min": -38.600189208984375,
      "activations/layer19_attention_weight_max": 23.82735252380371,
      "activations/layer19_attention_weight_min": -22.810537338256836,
      "activations/layer1_attention_weight_max": 15.569121360778809,
      "activations/layer1_attention_weight_min": -13.781785011291504,
      "activations/layer20_attention_weight_max": 23.322372436523438,
      "activations/layer20_attention_weight_min": -21.16558265686035,
      "activations/layer21_attention_weight_max": 37.76821517944336,
      "activations/layer21_attention_weight_min": -21.004932403564453,
      "activations/layer22_attention_weight_max": 29.23090934753418,
      "activations/layer22_attention_weight_min": -24.625905990600586,
      "activations/layer23_attention_weight_max": 36.08869552612305,
      "activations/layer23_attention_weight_min": -23.242115020751953,
      "activations/layer2_attention_weight_max": 31.40137481689453,
      "activations/layer2_attention_weight_min": -29.918285369873047,
      "activations/layer3_attention_weight_max": 88.41899871826172,
      "activations/layer3_attention_weight_min": -85.11717224121094,
      "activations/layer4_attention_weight_max": 89.82195281982422,
      "activations/layer4_attention_weight_min": -88.36272430419922,
      "activations/layer5_attention_weight_max": 66.09518432617188,
      "activations/layer5_attention_weight_min": -73.11207580566406,
      "activations/layer6_attention_weight_max": 52.5765266418457,
      "activations/layer6_attention_weight_min": -50.71116638183594,
      "activations/layer7_attention_weight_max": 64.63829803466797,
      "activations/layer7_attention_weight_min": -60.849212646484375,
      "activations/layer8_attention_weight_max": 45.054443359375,
      "activations/layer8_attention_weight_min": -48.10981750488281,
      "activations/layer9_attention_weight_max": 44.23844528198242,
      "activations/layer9_attention_weight_min": -46.130855560302734,
      "epoch": 16.18,
      "learning_rate": 4.6109090909090906e-05,
      "loss": 2.7477,
      "step": 278500
    },
    {
      "activations/layer0_attention_weight_max": 15.493266105651855,
      "activations/layer0_attention_weight_min": -13.930255889892578,
      "activations/layer10_attention_weight_max": 43.37727737426758,
      "activations/layer10_attention_weight_min": -41.779109954833984,
      "activations/layer11_attention_weight_max": 37.813385009765625,
      "activations/layer11_attention_weight_min": -38.90202713012695,
      "activations/layer12_attention_weight_max": 19.966407775878906,
      "activations/layer12_attention_weight_min": -25.252904891967773,
      "activations/layer13_attention_weight_max": 33.55091857910156,
      "activations/layer13_attention_weight_min": -31.482580184936523,
      "activations/layer14_attention_weight_max": 37.4722785949707,
      "activations/layer14_attention_weight_min": -32.46370315551758,
      "activations/layer15_attention_weight_max": 35.21785354614258,
      "activations/layer15_attention_weight_min": -31.85831069946289,
      "activations/layer16_attention_weight_max": 36.38861083984375,
      "activations/layer16_attention_weight_min": -33.94336700439453,
      "activations/layer17_attention_weight_max": 54.86754608154297,
      "activations/layer17_attention_weight_min": -48.33863067626953,
      "activations/layer18_attention_weight_max": 50.440155029296875,
      "activations/layer18_attention_weight_min": -40.814857482910156,
      "activations/layer19_attention_weight_max": 25.3006649017334,
      "activations/layer19_attention_weight_min": -26.16855812072754,
      "activations/layer1_attention_weight_max": 18.125883102416992,
      "activations/layer1_attention_weight_min": -15.202286720275879,
      "activations/layer20_attention_weight_max": 23.270917892456055,
      "activations/layer20_attention_weight_min": -23.69240379333496,
      "activations/layer21_attention_weight_max": 36.64410400390625,
      "activations/layer21_attention_weight_min": -26.368194580078125,
      "activations/layer22_attention_weight_max": 27.4589786529541,
      "activations/layer22_attention_weight_min": -28.694889068603516,
      "activations/layer23_attention_weight_max": 36.90066146850586,
      "activations/layer23_attention_weight_min": -27.013484954833984,
      "activations/layer2_attention_weight_max": 34.961708068847656,
      "activations/layer2_attention_weight_min": -33.88037109375,
      "activations/layer3_attention_weight_max": 100.92931365966797,
      "activations/layer3_attention_weight_min": -98.24887084960938,
      "activations/layer4_attention_weight_max": 100.05821990966797,
      "activations/layer4_attention_weight_min": -101.90943145751953,
      "activations/layer5_attention_weight_max": 68.69935607910156,
      "activations/layer5_attention_weight_min": -76.93138122558594,
      "activations/layer6_attention_weight_max": 57.75061798095703,
      "activations/layer6_attention_weight_min": -56.59375762939453,
      "activations/layer7_attention_weight_max": 69.53399658203125,
      "activations/layer7_attention_weight_min": -67.46320343017578,
      "activations/layer8_attention_weight_max": 51.65607833862305,
      "activations/layer8_attention_weight_min": -52.96018981933594,
      "activations/layer9_attention_weight_max": 51.205413818359375,
      "activations/layer9_attention_weight_min": -52.548439025878906,
      "epoch": 16.19,
      "learning_rate": 4.609015151515151e-05,
      "loss": 2.7572,
      "step": 278550
    },
    {
      "activations/layer0_attention_weight_max": 16.437097549438477,
      "activations/layer0_attention_weight_min": -13.84636116027832,
      "activations/layer10_attention_weight_max": 38.542442321777344,
      "activations/layer10_attention_weight_min": -39.71417236328125,
      "activations/layer11_attention_weight_max": 35.07514572143555,
      "activations/layer11_attention_weight_min": -35.95342254638672,
      "activations/layer12_attention_weight_max": 19.83836555480957,
      "activations/layer12_attention_weight_min": -24.86241912841797,
      "activations/layer13_attention_weight_max": 35.13786315917969,
      "activations/layer13_attention_weight_min": -28.502199172973633,
      "activations/layer14_attention_weight_max": 37.02592849731445,
      "activations/layer14_attention_weight_min": -30.449382781982422,
      "activations/layer15_attention_weight_max": 34.488460540771484,
      "activations/layer15_attention_weight_min": -32.631263732910156,
      "activations/layer16_attention_weight_max": 37.92127227783203,
      "activations/layer16_attention_weight_min": -34.83741760253906,
      "activations/layer17_attention_weight_max": 59.42759323120117,
      "activations/layer17_attention_weight_min": -52.9053840637207,
      "activations/layer18_attention_weight_max": 55.44900894165039,
      "activations/layer18_attention_weight_min": -47.813446044921875,
      "activations/layer19_attention_weight_max": 29.12799644470215,
      "activations/layer19_attention_weight_min": -28.03571128845215,
      "activations/layer1_attention_weight_max": 16.384109497070312,
      "activations/layer1_attention_weight_min": -14.338117599487305,
      "activations/layer20_attention_weight_max": 26.01055335998535,
      "activations/layer20_attention_weight_min": -26.2089900970459,
      "activations/layer21_attention_weight_max": 43.722137451171875,
      "activations/layer21_attention_weight_min": -25.789499282836914,
      "activations/layer22_attention_weight_max": 32.43658447265625,
      "activations/layer22_attention_weight_min": -29.99235725402832,
      "activations/layer23_attention_weight_max": 42.655517578125,
      "activations/layer23_attention_weight_min": -27.43189239501953,
      "activations/layer2_attention_weight_max": 30.208290100097656,
      "activations/layer2_attention_weight_min": -30.196407318115234,
      "activations/layer3_attention_weight_max": 83.5943374633789,
      "activations/layer3_attention_weight_min": -82.78870391845703,
      "activations/layer4_attention_weight_max": 87.79469299316406,
      "activations/layer4_attention_weight_min": -88.47574615478516,
      "activations/layer5_attention_weight_max": 63.29862976074219,
      "activations/layer5_attention_weight_min": -75.76742553710938,
      "activations/layer6_attention_weight_max": 50.818485260009766,
      "activations/layer6_attention_weight_min": -51.45124816894531,
      "activations/layer7_attention_weight_max": 60.522117614746094,
      "activations/layer7_attention_weight_min": -63.59107208251953,
      "activations/layer8_attention_weight_max": 46.40335464477539,
      "activations/layer8_attention_weight_min": -50.09897232055664,
      "activations/layer9_attention_weight_max": 45.7551383972168,
      "activations/layer9_attention_weight_min": -46.53343200683594,
      "epoch": 16.19,
      "learning_rate": 4.607121212121212e-05,
      "loss": 2.7488,
      "step": 278600
    },
    {
      "activations/layer0_attention_weight_max": 15.133201599121094,
      "activations/layer0_attention_weight_min": -13.983988761901855,
      "activations/layer10_attention_weight_max": 33.7191162109375,
      "activations/layer10_attention_weight_min": -34.20888900756836,
      "activations/layer11_attention_weight_max": 31.409622192382812,
      "activations/layer11_attention_weight_min": -33.930870056152344,
      "activations/layer12_attention_weight_max": 17.137496948242188,
      "activations/layer12_attention_weight_min": -27.730016708374023,
      "activations/layer13_attention_weight_max": 37.571990966796875,
      "activations/layer13_attention_weight_min": -29.86893081665039,
      "activations/layer14_attention_weight_max": 35.35702133178711,
      "activations/layer14_attention_weight_min": -30.679059982299805,
      "activations/layer15_attention_weight_max": 34.26807403564453,
      "activations/layer15_attention_weight_min": -31.834659576416016,
      "activations/layer16_attention_weight_max": 30.378202438354492,
      "activations/layer16_attention_weight_min": -29.241884231567383,
      "activations/layer17_attention_weight_max": 50.5865364074707,
      "activations/layer17_attention_weight_min": -40.721458435058594,
      "activations/layer18_attention_weight_max": 43.143428802490234,
      "activations/layer18_attention_weight_min": -36.55574035644531,
      "activations/layer19_attention_weight_max": 21.22090721130371,
      "activations/layer19_attention_weight_min": -22.274307250976562,
      "activations/layer1_attention_weight_max": 17.340635299682617,
      "activations/layer1_attention_weight_min": -13.684563636779785,
      "activations/layer20_attention_weight_max": 19.307632446289062,
      "activations/layer20_attention_weight_min": -21.67572021484375,
      "activations/layer21_attention_weight_max": 31.212387084960938,
      "activations/layer21_attention_weight_min": -21.49332618713379,
      "activations/layer22_attention_weight_max": 28.4656982421875,
      "activations/layer22_attention_weight_min": -24.969242095947266,
      "activations/layer23_attention_weight_max": 32.9631462097168,
      "activations/layer23_attention_weight_min": -23.744298934936523,
      "activations/layer2_attention_weight_max": 29.82282257080078,
      "activations/layer2_attention_weight_min": -27.840139389038086,
      "activations/layer3_attention_weight_max": 86.09981536865234,
      "activations/layer3_attention_weight_min": -84.90015411376953,
      "activations/layer4_attention_weight_max": 89.80123138427734,
      "activations/layer4_attention_weight_min": -87.26941680908203,
      "activations/layer5_attention_weight_max": 66.47611236572266,
      "activations/layer5_attention_weight_min": -75.21623992919922,
      "activations/layer6_attention_weight_max": 49.39511489868164,
      "activations/layer6_attention_weight_min": -49.409358978271484,
      "activations/layer7_attention_weight_max": 65.07210540771484,
      "activations/layer7_attention_weight_min": -59.324440002441406,
      "activations/layer8_attention_weight_max": 45.701114654541016,
      "activations/layer8_attention_weight_min": -52.43585968017578,
      "activations/layer9_attention_weight_max": 46.74199295043945,
      "activations/layer9_attention_weight_min": -45.07319259643555,
      "epoch": 16.19,
      "learning_rate": 4.6052272727272724e-05,
      "loss": 2.7508,
      "step": 278650
    },
    {
      "activations/layer0_attention_weight_max": 15.639945983886719,
      "activations/layer0_attention_weight_min": -13.684532165527344,
      "activations/layer10_attention_weight_max": 37.127967834472656,
      "activations/layer10_attention_weight_min": -37.578460693359375,
      "activations/layer11_attention_weight_max": 33.224090576171875,
      "activations/layer11_attention_weight_min": -33.92497253417969,
      "activations/layer12_attention_weight_max": 19.611398696899414,
      "activations/layer12_attention_weight_min": -25.078651428222656,
      "activations/layer13_attention_weight_max": 37.71856689453125,
      "activations/layer13_attention_weight_min": -30.334854125976562,
      "activations/layer14_attention_weight_max": 34.5380973815918,
      "activations/layer14_attention_weight_min": -29.594762802124023,
      "activations/layer15_attention_weight_max": 31.565656661987305,
      "activations/layer15_attention_weight_min": -28.590343475341797,
      "activations/layer16_attention_weight_max": 33.71543502807617,
      "activations/layer16_attention_weight_min": -31.48006248474121,
      "activations/layer17_attention_weight_max": 52.88612747192383,
      "activations/layer17_attention_weight_min": -44.418582916259766,
      "activations/layer18_attention_weight_max": 47.853328704833984,
      "activations/layer18_attention_weight_min": -38.41130828857422,
      "activations/layer19_attention_weight_max": 24.664154052734375,
      "activations/layer19_attention_weight_min": -21.258716583251953,
      "activations/layer1_attention_weight_max": 15.898683547973633,
      "activations/layer1_attention_weight_min": -15.776106834411621,
      "activations/layer20_attention_weight_max": 24.229869842529297,
      "activations/layer20_attention_weight_min": -20.06078338623047,
      "activations/layer21_attention_weight_max": 36.332645416259766,
      "activations/layer21_attention_weight_min": -22.812774658203125,
      "activations/layer22_attention_weight_max": 30.739099502563477,
      "activations/layer22_attention_weight_min": -28.50299644470215,
      "activations/layer23_attention_weight_max": 37.601951599121094,
      "activations/layer23_attention_weight_min": -25.108295440673828,
      "activations/layer2_attention_weight_max": 32.15364074707031,
      "activations/layer2_attention_weight_min": -31.275543212890625,
      "activations/layer3_attention_weight_max": 88.94085693359375,
      "activations/layer3_attention_weight_min": -89.91217041015625,
      "activations/layer4_attention_weight_max": 90.70787811279297,
      "activations/layer4_attention_weight_min": -87.78475189208984,
      "activations/layer5_attention_weight_max": 67.92472839355469,
      "activations/layer5_attention_weight_min": -75.51721954345703,
      "activations/layer6_attention_weight_max": 52.8287239074707,
      "activations/layer6_attention_weight_min": -51.752708435058594,
      "activations/layer7_attention_weight_max": 64.41020202636719,
      "activations/layer7_attention_weight_min": -61.71649169921875,
      "activations/layer8_attention_weight_max": 45.90968704223633,
      "activations/layer8_attention_weight_min": -48.85078811645508,
      "activations/layer9_attention_weight_max": 46.65458297729492,
      "activations/layer9_attention_weight_min": -47.04264450073242,
      "epoch": 16.19,
      "learning_rate": 4.6033333333333325e-05,
      "loss": 2.7364,
      "step": 278700
    },
    {
      "activations/layer0_attention_weight_max": 15.892443656921387,
      "activations/layer0_attention_weight_min": -13.761054039001465,
      "activations/layer10_attention_weight_max": 35.49919509887695,
      "activations/layer10_attention_weight_min": -35.423423767089844,
      "activations/layer11_attention_weight_max": 31.87740135192871,
      "activations/layer11_attention_weight_min": -33.27336502075195,
      "activations/layer12_attention_weight_max": 19.44693374633789,
      "activations/layer12_attention_weight_min": -29.06802749633789,
      "activations/layer13_attention_weight_max": 44.15480422973633,
      "activations/layer13_attention_weight_min": -30.55169677734375,
      "activations/layer14_attention_weight_max": 35.29048538208008,
      "activations/layer14_attention_weight_min": -28.928314208984375,
      "activations/layer15_attention_weight_max": 33.321598052978516,
      "activations/layer15_attention_weight_min": -29.262008666992188,
      "activations/layer16_attention_weight_max": 33.310394287109375,
      "activations/layer16_attention_weight_min": -29.681230545043945,
      "activations/layer17_attention_weight_max": 51.74411392211914,
      "activations/layer17_attention_weight_min": -41.95410919189453,
      "activations/layer18_attention_weight_max": 44.980655670166016,
      "activations/layer18_attention_weight_min": -37.801597595214844,
      "activations/layer19_attention_weight_max": 23.6162109375,
      "activations/layer19_attention_weight_min": -21.959699630737305,
      "activations/layer1_attention_weight_max": 15.48136043548584,
      "activations/layer1_attention_weight_min": -13.333805084228516,
      "activations/layer20_attention_weight_max": 23.927570343017578,
      "activations/layer20_attention_weight_min": -24.325105667114258,
      "activations/layer21_attention_weight_max": 35.267215728759766,
      "activations/layer21_attention_weight_min": -21.68235206604004,
      "activations/layer22_attention_weight_max": 28.20748519897461,
      "activations/layer22_attention_weight_min": -26.39035987854004,
      "activations/layer23_attention_weight_max": 35.879302978515625,
      "activations/layer23_attention_weight_min": -24.77642059326172,
      "activations/layer2_attention_weight_max": 30.954769134521484,
      "activations/layer2_attention_weight_min": -29.587512969970703,
      "activations/layer3_attention_weight_max": 87.77276611328125,
      "activations/layer3_attention_weight_min": -89.48495483398438,
      "activations/layer4_attention_weight_max": 87.08558654785156,
      "activations/layer4_attention_weight_min": -87.71758270263672,
      "activations/layer5_attention_weight_max": 65.89278411865234,
      "activations/layer5_attention_weight_min": -81.87161254882812,
      "activations/layer6_attention_weight_max": 51.83726119995117,
      "activations/layer6_attention_weight_min": -52.69179153442383,
      "activations/layer7_attention_weight_max": 60.822288513183594,
      "activations/layer7_attention_weight_min": -64.19456481933594,
      "activations/layer8_attention_weight_max": 48.2499885559082,
      "activations/layer8_attention_weight_min": -49.67180252075195,
      "activations/layer9_attention_weight_max": 42.89773941040039,
      "activations/layer9_attention_weight_min": -50.00480270385742,
      "epoch": 16.2,
      "learning_rate": 4.6014393939393933e-05,
      "loss": 2.753,
      "step": 278750
    },
    {
      "activations/layer0_attention_weight_max": 15.696480751037598,
      "activations/layer0_attention_weight_min": -13.983699798583984,
      "activations/layer10_attention_weight_max": 36.715980529785156,
      "activations/layer10_attention_weight_min": -35.87482452392578,
      "activations/layer11_attention_weight_max": 32.701332092285156,
      "activations/layer11_attention_weight_min": -34.31295394897461,
      "activations/layer12_attention_weight_max": 18.412782669067383,
      "activations/layer12_attention_weight_min": -24.667224884033203,
      "activations/layer13_attention_weight_max": 37.48263168334961,
      "activations/layer13_attention_weight_min": -26.74455451965332,
      "activations/layer14_attention_weight_max": 36.50810623168945,
      "activations/layer14_attention_weight_min": -30.53874397277832,
      "activations/layer15_attention_weight_max": 32.346900939941406,
      "activations/layer15_attention_weight_min": -29.15015411376953,
      "activations/layer16_attention_weight_max": 32.03988265991211,
      "activations/layer16_attention_weight_min": -32.39262771606445,
      "activations/layer17_attention_weight_max": 50.280338287353516,
      "activations/layer17_attention_weight_min": -49.01702117919922,
      "activations/layer18_attention_weight_max": 45.781463623046875,
      "activations/layer18_attention_weight_min": -41.56002426147461,
      "activations/layer19_attention_weight_max": 23.168392181396484,
      "activations/layer19_attention_weight_min": -24.418415069580078,
      "activations/layer1_attention_weight_max": 15.635506629943848,
      "activations/layer1_attention_weight_min": -13.890952110290527,
      "activations/layer20_attention_weight_max": 23.55601692199707,
      "activations/layer20_attention_weight_min": -20.617412567138672,
      "activations/layer21_attention_weight_max": 38.89635467529297,
      "activations/layer21_attention_weight_min": -24.53763198852539,
      "activations/layer22_attention_weight_max": 32.25263595581055,
      "activations/layer22_attention_weight_min": -26.55377769470215,
      "activations/layer23_attention_weight_max": 34.18986511230469,
      "activations/layer23_attention_weight_min": -25.402084350585938,
      "activations/layer2_attention_weight_max": 30.16020965576172,
      "activations/layer2_attention_weight_min": -28.092025756835938,
      "activations/layer3_attention_weight_max": 88.6862564086914,
      "activations/layer3_attention_weight_min": -84.47882843017578,
      "activations/layer4_attention_weight_max": 89.26432037353516,
      "activations/layer4_attention_weight_min": -87.97508239746094,
      "activations/layer5_attention_weight_max": 64.26387023925781,
      "activations/layer5_attention_weight_min": -74.64641571044922,
      "activations/layer6_attention_weight_max": 47.95384216308594,
      "activations/layer6_attention_weight_min": -52.08833312988281,
      "activations/layer7_attention_weight_max": 59.1828498840332,
      "activations/layer7_attention_weight_min": -64.48497009277344,
      "activations/layer8_attention_weight_max": 43.34098815917969,
      "activations/layer8_attention_weight_min": -46.6961784362793,
      "activations/layer9_attention_weight_max": 45.14567565917969,
      "activations/layer9_attention_weight_min": -46.617469787597656,
      "epoch": 16.2,
      "learning_rate": 4.599545454545454e-05,
      "loss": 2.7356,
      "step": 278800
    },
    {
      "activations/layer0_attention_weight_max": 15.896919250488281,
      "activations/layer0_attention_weight_min": -13.838698387145996,
      "activations/layer10_attention_weight_max": 33.83831787109375,
      "activations/layer10_attention_weight_min": -34.6647834777832,
      "activations/layer11_attention_weight_max": 30.46487045288086,
      "activations/layer11_attention_weight_min": -33.85710144042969,
      "activations/layer12_attention_weight_max": 19.019254684448242,
      "activations/layer12_attention_weight_min": -24.70329475402832,
      "activations/layer13_attention_weight_max": 34.82212829589844,
      "activations/layer13_attention_weight_min": -29.672792434692383,
      "activations/layer14_attention_weight_max": 36.63514709472656,
      "activations/layer14_attention_weight_min": -33.931846618652344,
      "activations/layer15_attention_weight_max": 34.486446380615234,
      "activations/layer15_attention_weight_min": -32.40546798706055,
      "activations/layer16_attention_weight_max": 36.7885627746582,
      "activations/layer16_attention_weight_min": -33.577816009521484,
      "activations/layer17_attention_weight_max": 56.46368408203125,
      "activations/layer17_attention_weight_min": -52.38087463378906,
      "activations/layer18_attention_weight_max": 50.93769073486328,
      "activations/layer18_attention_weight_min": -43.53572082519531,
      "activations/layer19_attention_weight_max": 25.619840621948242,
      "activations/layer19_attention_weight_min": -24.869543075561523,
      "activations/layer1_attention_weight_max": 16.477130889892578,
      "activations/layer1_attention_weight_min": -15.177356719970703,
      "activations/layer20_attention_weight_max": 25.44707489013672,
      "activations/layer20_attention_weight_min": -23.815013885498047,
      "activations/layer21_attention_weight_max": 43.66024398803711,
      "activations/layer21_attention_weight_min": -24.922685623168945,
      "activations/layer22_attention_weight_max": 29.400495529174805,
      "activations/layer22_attention_weight_min": -25.976743698120117,
      "activations/layer23_attention_weight_max": 36.63703918457031,
      "activations/layer23_attention_weight_min": -26.64706039428711,
      "activations/layer2_attention_weight_max": 29.739517211914062,
      "activations/layer2_attention_weight_min": -28.674541473388672,
      "activations/layer3_attention_weight_max": 86.72176361083984,
      "activations/layer3_attention_weight_min": -86.46063995361328,
      "activations/layer4_attention_weight_max": 86.85354614257812,
      "activations/layer4_attention_weight_min": -87.43892669677734,
      "activations/layer5_attention_weight_max": 62.810333251953125,
      "activations/layer5_attention_weight_min": -72.64008331298828,
      "activations/layer6_attention_weight_max": 48.46384048461914,
      "activations/layer6_attention_weight_min": -48.319602966308594,
      "activations/layer7_attention_weight_max": 64.20553588867188,
      "activations/layer7_attention_weight_min": -59.93199920654297,
      "activations/layer8_attention_weight_max": 44.02627944946289,
      "activations/layer8_attention_weight_min": -47.9930419921875,
      "activations/layer9_attention_weight_max": 42.566558837890625,
      "activations/layer9_attention_weight_min": -45.93232345581055,
      "epoch": 16.2,
      "learning_rate": 4.597651515151515e-05,
      "loss": 2.7512,
      "step": 278850
    },
    {
      "activations/layer0_attention_weight_max": 14.947277069091797,
      "activations/layer0_attention_weight_min": -13.649007797241211,
      "activations/layer10_attention_weight_max": 37.743133544921875,
      "activations/layer10_attention_weight_min": -39.2018928527832,
      "activations/layer11_attention_weight_max": 34.295440673828125,
      "activations/layer11_attention_weight_min": -36.557071685791016,
      "activations/layer12_attention_weight_max": 20.153018951416016,
      "activations/layer12_attention_weight_min": -24.873397827148438,
      "activations/layer13_attention_weight_max": 35.53205108642578,
      "activations/layer13_attention_weight_min": -27.674976348876953,
      "activations/layer14_attention_weight_max": 39.161312103271484,
      "activations/layer14_attention_weight_min": -34.315467834472656,
      "activations/layer15_attention_weight_max": 33.245933532714844,
      "activations/layer15_attention_weight_min": -31.683448791503906,
      "activations/layer16_attention_weight_max": 35.658870697021484,
      "activations/layer16_attention_weight_min": -33.1854248046875,
      "activations/layer17_attention_weight_max": 53.72762680053711,
      "activations/layer17_attention_weight_min": -48.44755554199219,
      "activations/layer18_attention_weight_max": 46.240203857421875,
      "activations/layer18_attention_weight_min": -40.585819244384766,
      "activations/layer19_attention_weight_max": 23.479564666748047,
      "activations/layer19_attention_weight_min": -22.688669204711914,
      "activations/layer1_attention_weight_max": 16.007169723510742,
      "activations/layer1_attention_weight_min": -15.030086517333984,
      "activations/layer20_attention_weight_max": 24.23929786682129,
      "activations/layer20_attention_weight_min": -22.93395233154297,
      "activations/layer21_attention_weight_max": 35.374759674072266,
      "activations/layer21_attention_weight_min": -21.201942443847656,
      "activations/layer22_attention_weight_max": 28.983470916748047,
      "activations/layer22_attention_weight_min": -25.67046356201172,
      "activations/layer23_attention_weight_max": 35.264678955078125,
      "activations/layer23_attention_weight_min": -23.92110252380371,
      "activations/layer2_attention_weight_max": 32.52336120605469,
      "activations/layer2_attention_weight_min": -31.745176315307617,
      "activations/layer3_attention_weight_max": 88.77238464355469,
      "activations/layer3_attention_weight_min": -88.94715118408203,
      "activations/layer4_attention_weight_max": 92.93069458007812,
      "activations/layer4_attention_weight_min": -85.84996795654297,
      "activations/layer5_attention_weight_max": 63.87731170654297,
      "activations/layer5_attention_weight_min": -69.80332946777344,
      "activations/layer6_attention_weight_max": 52.26566696166992,
      "activations/layer6_attention_weight_min": -55.373077392578125,
      "activations/layer7_attention_weight_max": 62.40221405029297,
      "activations/layer7_attention_weight_min": -64.8653564453125,
      "activations/layer8_attention_weight_max": 48.17072677612305,
      "activations/layer8_attention_weight_min": -53.438785552978516,
      "activations/layer9_attention_weight_max": 46.34171676635742,
      "activations/layer9_attention_weight_min": -49.54378128051758,
      "epoch": 16.21,
      "learning_rate": 4.595757575757575e-05,
      "loss": 2.7402,
      "step": 278900
    },
    {
      "activations/layer0_attention_weight_max": 17.121042251586914,
      "activations/layer0_attention_weight_min": -13.575700759887695,
      "activations/layer10_attention_weight_max": 34.98372268676758,
      "activations/layer10_attention_weight_min": -34.91545104980469,
      "activations/layer11_attention_weight_max": 34.04230499267578,
      "activations/layer11_attention_weight_min": -33.003055572509766,
      "activations/layer12_attention_weight_max": 19.328372955322266,
      "activations/layer12_attention_weight_min": -26.266868591308594,
      "activations/layer13_attention_weight_max": 34.47712707519531,
      "activations/layer13_attention_weight_min": -27.803688049316406,
      "activations/layer14_attention_weight_max": 37.69768524169922,
      "activations/layer14_attention_weight_min": -30.375741958618164,
      "activations/layer15_attention_weight_max": 34.09996032714844,
      "activations/layer15_attention_weight_min": -29.047866821289062,
      "activations/layer16_attention_weight_max": 32.92699432373047,
      "activations/layer16_attention_weight_min": -31.202009201049805,
      "activations/layer17_attention_weight_max": 51.32988357543945,
      "activations/layer17_attention_weight_min": -44.41493606567383,
      "activations/layer18_attention_weight_max": 50.41227722167969,
      "activations/layer18_attention_weight_min": -40.33965301513672,
      "activations/layer19_attention_weight_max": 22.652408599853516,
      "activations/layer19_attention_weight_min": -20.81659698486328,
      "activations/layer1_attention_weight_max": 15.822179794311523,
      "activations/layer1_attention_weight_min": -13.148032188415527,
      "activations/layer20_attention_weight_max": 21.34415054321289,
      "activations/layer20_attention_weight_min": -20.265867233276367,
      "activations/layer21_attention_weight_max": 33.32771682739258,
      "activations/layer21_attention_weight_min": -23.23212432861328,
      "activations/layer22_attention_weight_max": 26.306570053100586,
      "activations/layer22_attention_weight_min": -24.666828155517578,
      "activations/layer23_attention_weight_max": 35.98827362060547,
      "activations/layer23_attention_weight_min": -24.648723602294922,
      "activations/layer2_attention_weight_max": 31.74873161315918,
      "activations/layer2_attention_weight_min": -29.325969696044922,
      "activations/layer3_attention_weight_max": 84.9654541015625,
      "activations/layer3_attention_weight_min": -85.46134185791016,
      "activations/layer4_attention_weight_max": 85.06771087646484,
      "activations/layer4_attention_weight_min": -86.17070770263672,
      "activations/layer5_attention_weight_max": 63.983577728271484,
      "activations/layer5_attention_weight_min": -74.2287368774414,
      "activations/layer6_attention_weight_max": 48.5644416809082,
      "activations/layer6_attention_weight_min": -53.151737213134766,
      "activations/layer7_attention_weight_max": 60.99327087402344,
      "activations/layer7_attention_weight_min": -63.01490783691406,
      "activations/layer8_attention_weight_max": 44.38765335083008,
      "activations/layer8_attention_weight_min": -47.66859436035156,
      "activations/layer9_attention_weight_max": 41.54848861694336,
      "activations/layer9_attention_weight_min": -44.1849365234375,
      "epoch": 16.21,
      "learning_rate": 4.5938636363636366e-05,
      "loss": 2.7481,
      "step": 278950
    },
    {
      "activations/layer0_attention_weight_max": 17.091157913208008,
      "activations/layer0_attention_weight_min": -14.102912902832031,
      "activations/layer10_attention_weight_max": 35.39720153808594,
      "activations/layer10_attention_weight_min": -35.25258255004883,
      "activations/layer11_attention_weight_max": 33.92350387573242,
      "activations/layer11_attention_weight_min": -33.9878044128418,
      "activations/layer12_attention_weight_max": 19.141403198242188,
      "activations/layer12_attention_weight_min": -30.863208770751953,
      "activations/layer13_attention_weight_max": 42.22593307495117,
      "activations/layer13_attention_weight_min": -27.954965591430664,
      "activations/layer14_attention_weight_max": 40.53345489501953,
      "activations/layer14_attention_weight_min": -29.119586944580078,
      "activations/layer15_attention_weight_max": 40.158119201660156,
      "activations/layer15_attention_weight_min": -29.02540397644043,
      "activations/layer16_attention_weight_max": 32.58720397949219,
      "activations/layer16_attention_weight_min": -31.529544830322266,
      "activations/layer17_attention_weight_max": 50.61555099487305,
      "activations/layer17_attention_weight_min": -46.16842269897461,
      "activations/layer18_attention_weight_max": 45.69542694091797,
      "activations/layer18_attention_weight_min": -40.11376190185547,
      "activations/layer19_attention_weight_max": 24.161855697631836,
      "activations/layer19_attention_weight_min": -22.624101638793945,
      "activations/layer1_attention_weight_max": 16.286779403686523,
      "activations/layer1_attention_weight_min": -14.794075012207031,
      "activations/layer20_attention_weight_max": 23.500085830688477,
      "activations/layer20_attention_weight_min": -21.8543758392334,
      "activations/layer21_attention_weight_max": 36.089599609375,
      "activations/layer21_attention_weight_min": -23.681838989257812,
      "activations/layer22_attention_weight_max": 31.289941787719727,
      "activations/layer22_attention_weight_min": -25.890832901000977,
      "activations/layer23_attention_weight_max": 38.66535186767578,
      "activations/layer23_attention_weight_min": -24.123790740966797,
      "activations/layer2_attention_weight_max": 34.781578063964844,
      "activations/layer2_attention_weight_min": -29.609699249267578,
      "activations/layer3_attention_weight_max": 91.9158706665039,
      "activations/layer3_attention_weight_min": -94.73806762695312,
      "activations/layer4_attention_weight_max": 90.08587646484375,
      "activations/layer4_attention_weight_min": -92.69116973876953,
      "activations/layer5_attention_weight_max": 63.76774215698242,
      "activations/layer5_attention_weight_min": -77.8175048828125,
      "activations/layer6_attention_weight_max": 50.75178909301758,
      "activations/layer6_attention_weight_min": -53.524147033691406,
      "activations/layer7_attention_weight_max": 62.29413986206055,
      "activations/layer7_attention_weight_min": -62.26145553588867,
      "activations/layer8_attention_weight_max": 46.12028884887695,
      "activations/layer8_attention_weight_min": -50.925052642822266,
      "activations/layer9_attention_weight_max": 44.5769157409668,
      "activations/layer9_attention_weight_min": -47.82077407836914,
      "epoch": 16.21,
      "learning_rate": 4.591969696969697e-05,
      "loss": 2.7416,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4609,
      "eval_samples_per_second": 507.508,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4609,
      "eval_openwebtext_samples_per_second": 507.508,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9727,
      "eval_wikitext_samples_per_second": 231.154,
      "step": 279000
    },
    {
      "epoch": 16.21,
      "eval_lambada_loss": 2.466796875,
      "eval_lambada_ppl": 11.784638661080912,
      "eval_lambada_runtime": 9.5061,
      "eval_lambada_samples_per_second": 512.197,
      "step": 279000
    },
    {
      "activations/layer0_attention_weight_max": 16.3043270111084,
      "activations/layer0_attention_weight_min": -13.433619499206543,
      "activations/layer10_attention_weight_max": 35.36598205566406,
      "activations/layer10_attention_weight_min": -35.890865325927734,
      "activations/layer11_attention_weight_max": 32.22541046142578,
      "activations/layer11_attention_weight_min": -33.307708740234375,
      "activations/layer12_attention_weight_max": 18.521299362182617,
      "activations/layer12_attention_weight_min": -29.500045776367188,
      "activations/layer13_attention_weight_max": 36.79502487182617,
      "activations/layer13_attention_weight_min": -27.437898635864258,
      "activations/layer14_attention_weight_max": 32.63821029663086,
      "activations/layer14_attention_weight_min": -29.91693115234375,
      "activations/layer15_attention_weight_max": 30.41757583618164,
      "activations/layer15_attention_weight_min": -29.639665603637695,
      "activations/layer16_attention_weight_max": 32.14319610595703,
      "activations/layer16_attention_weight_min": -30.199995040893555,
      "activations/layer17_attention_weight_max": 47.95718002319336,
      "activations/layer17_attention_weight_min": -43.642555236816406,
      "activations/layer18_attention_weight_max": 45.102752685546875,
      "activations/layer18_attention_weight_min": -36.97484588623047,
      "activations/layer19_attention_weight_max": 20.655099868774414,
      "activations/layer19_attention_weight_min": -22.49538803100586,
      "activations/layer1_attention_weight_max": 15.84315013885498,
      "activations/layer1_attention_weight_min": -14.59646987915039,
      "activations/layer20_attention_weight_max": 22.101354598999023,
      "activations/layer20_attention_weight_min": -21.019468307495117,
      "activations/layer21_attention_weight_max": 34.306644439697266,
      "activations/layer21_attention_weight_min": -21.86958885192871,
      "activations/layer22_attention_weight_max": 32.19450759887695,
      "activations/layer22_attention_weight_min": -27.371402740478516,
      "activations/layer23_attention_weight_max": 36.17167663574219,
      "activations/layer23_attention_weight_min": -25.04157066345215,
      "activations/layer2_attention_weight_max": 29.652088165283203,
      "activations/layer2_attention_weight_min": -29.983638763427734,
      "activations/layer3_attention_weight_max": 89.73223114013672,
      "activations/layer3_attention_weight_min": -90.51314544677734,
      "activations/layer4_attention_weight_max": 91.07090759277344,
      "activations/layer4_attention_weight_min": -89.75484466552734,
      "activations/layer5_attention_weight_max": 70.66952514648438,
      "activations/layer5_attention_weight_min": -73.34449768066406,
      "activations/layer6_attention_weight_max": 51.3206672668457,
      "activations/layer6_attention_weight_min": -52.015316009521484,
      "activations/layer7_attention_weight_max": 60.18779373168945,
      "activations/layer7_attention_weight_min": -62.17924880981445,
      "activations/layer8_attention_weight_max": 44.02480697631836,
      "activations/layer8_attention_weight_min": -47.323673248291016,
      "activations/layer9_attention_weight_max": 44.74003601074219,
      "activations/layer9_attention_weight_min": -44.67204284667969,
      "epoch": 16.21,
      "learning_rate": 4.590075757575757e-05,
      "loss": 2.7427,
      "step": 279050
    },
    {
      "activations/layer0_attention_weight_max": 15.718921661376953,
      "activations/layer0_attention_weight_min": -13.297637939453125,
      "activations/layer10_attention_weight_max": 35.018943786621094,
      "activations/layer10_attention_weight_min": -36.254066467285156,
      "activations/layer11_attention_weight_max": 33.303314208984375,
      "activations/layer11_attention_weight_min": -35.692142486572266,
      "activations/layer12_attention_weight_max": 17.951662063598633,
      "activations/layer12_attention_weight_min": -22.750341415405273,
      "activations/layer13_attention_weight_max": 38.191368103027344,
      "activations/layer13_attention_weight_min": -28.161470413208008,
      "activations/layer14_attention_weight_max": 38.657588958740234,
      "activations/layer14_attention_weight_min": -30.85075569152832,
      "activations/layer15_attention_weight_max": 34.92990493774414,
      "activations/layer15_attention_weight_min": -30.983217239379883,
      "activations/layer16_attention_weight_max": 32.444271087646484,
      "activations/layer16_attention_weight_min": -30.85820198059082,
      "activations/layer17_attention_weight_max": 54.56612777709961,
      "activations/layer17_attention_weight_min": -46.09883499145508,
      "activations/layer18_attention_weight_max": 50.32150650024414,
      "activations/layer18_attention_weight_min": -41.249046325683594,
      "activations/layer19_attention_weight_max": 22.73407745361328,
      "activations/layer19_attention_weight_min": -24.162792205810547,
      "activations/layer1_attention_weight_max": 16.803207397460938,
      "activations/layer1_attention_weight_min": -13.780569076538086,
      "activations/layer20_attention_weight_max": 22.088457107543945,
      "activations/layer20_attention_weight_min": -21.49993896484375,
      "activations/layer21_attention_weight_max": 35.2073974609375,
      "activations/layer21_attention_weight_min": -23.236650466918945,
      "activations/layer22_attention_weight_max": 30.447452545166016,
      "activations/layer22_attention_weight_min": -26.739715576171875,
      "activations/layer23_attention_weight_max": 38.040645599365234,
      "activations/layer23_attention_weight_min": -24.386940002441406,
      "activations/layer2_attention_weight_max": 32.4089469909668,
      "activations/layer2_attention_weight_min": -30.06041717529297,
      "activations/layer3_attention_weight_max": 91.29017639160156,
      "activations/layer3_attention_weight_min": -92.71207427978516,
      "activations/layer4_attention_weight_max": 87.64379119873047,
      "activations/layer4_attention_weight_min": -91.4859390258789,
      "activations/layer5_attention_weight_max": 66.2883071899414,
      "activations/layer5_attention_weight_min": -76.69215393066406,
      "activations/layer6_attention_weight_max": 50.43122482299805,
      "activations/layer6_attention_weight_min": -49.771644592285156,
      "activations/layer7_attention_weight_max": 59.66584777832031,
      "activations/layer7_attention_weight_min": -59.829200744628906,
      "activations/layer8_attention_weight_max": 45.66621780395508,
      "activations/layer8_attention_weight_min": -48.94719314575195,
      "activations/layer9_attention_weight_max": 46.29642868041992,
      "activations/layer9_attention_weight_min": -46.7248649597168,
      "epoch": 16.22,
      "learning_rate": 4.588181818181818e-05,
      "loss": 2.7635,
      "step": 279100
    },
    {
      "activations/layer0_attention_weight_max": 17.1932430267334,
      "activations/layer0_attention_weight_min": -13.399029731750488,
      "activations/layer10_attention_weight_max": 35.74010467529297,
      "activations/layer10_attention_weight_min": -35.91696548461914,
      "activations/layer11_attention_weight_max": 32.40563201904297,
      "activations/layer11_attention_weight_min": -33.91598892211914,
      "activations/layer12_attention_weight_max": 20.04935646057129,
      "activations/layer12_attention_weight_min": -22.84225082397461,
      "activations/layer13_attention_weight_max": 36.24538040161133,
      "activations/layer13_attention_weight_min": -26.35610008239746,
      "activations/layer14_attention_weight_max": 38.61338424682617,
      "activations/layer14_attention_weight_min": -28.81117057800293,
      "activations/layer15_attention_weight_max": 35.06158447265625,
      "activations/layer15_attention_weight_min": -29.597745895385742,
      "activations/layer16_attention_weight_max": 34.498756408691406,
      "activations/layer16_attention_weight_min": -32.08798599243164,
      "activations/layer17_attention_weight_max": 55.174991607666016,
      "activations/layer17_attention_weight_min": -45.5073127746582,
      "activations/layer18_attention_weight_max": 50.958030700683594,
      "activations/layer18_attention_weight_min": -40.44231414794922,
      "activations/layer19_attention_weight_max": 26.919784545898438,
      "activations/layer19_attention_weight_min": -22.92275619506836,
      "activations/layer1_attention_weight_max": 16.87120819091797,
      "activations/layer1_attention_weight_min": -13.082520484924316,
      "activations/layer20_attention_weight_max": 25.434309005737305,
      "activations/layer20_attention_weight_min": -22.578344345092773,
      "activations/layer21_attention_weight_max": 36.81520462036133,
      "activations/layer21_attention_weight_min": -20.972135543823242,
      "activations/layer22_attention_weight_max": 31.273958206176758,
      "activations/layer22_attention_weight_min": -25.23180389404297,
      "activations/layer23_attention_weight_max": 36.742103576660156,
      "activations/layer23_attention_weight_min": -22.890974044799805,
      "activations/layer2_attention_weight_max": 30.451248168945312,
      "activations/layer2_attention_weight_min": -28.668500900268555,
      "activations/layer3_attention_weight_max": 81.4539794921875,
      "activations/layer3_attention_weight_min": -81.88356018066406,
      "activations/layer4_attention_weight_max": 82.4227523803711,
      "activations/layer4_attention_weight_min": -83.54100799560547,
      "activations/layer5_attention_weight_max": 62.80550765991211,
      "activations/layer5_attention_weight_min": -71.96859741210938,
      "activations/layer6_attention_weight_max": 47.950843811035156,
      "activations/layer6_attention_weight_min": -50.267398834228516,
      "activations/layer7_attention_weight_max": 59.47793960571289,
      "activations/layer7_attention_weight_min": -61.482059478759766,
      "activations/layer8_attention_weight_max": 42.11553192138672,
      "activations/layer8_attention_weight_min": -46.73173904418945,
      "activations/layer9_attention_weight_max": 41.29647445678711,
      "activations/layer9_attention_weight_min": -48.78141784667969,
      "epoch": 16.22,
      "learning_rate": 4.5862878787878786e-05,
      "loss": 2.763,
      "step": 279150
    },
    {
      "activations/layer0_attention_weight_max": 15.92225170135498,
      "activations/layer0_attention_weight_min": -13.106218338012695,
      "activations/layer10_attention_weight_max": 33.82083511352539,
      "activations/layer10_attention_weight_min": -34.02989959716797,
      "activations/layer11_attention_weight_max": 31.645278930664062,
      "activations/layer11_attention_weight_min": -32.459041595458984,
      "activations/layer12_attention_weight_max": 20.395137786865234,
      "activations/layer12_attention_weight_min": -26.951709747314453,
      "activations/layer13_attention_weight_max": 34.789241790771484,
      "activations/layer13_attention_weight_min": -28.772109985351562,
      "activations/layer14_attention_weight_max": 36.34745407104492,
      "activations/layer14_attention_weight_min": -30.960742950439453,
      "activations/layer15_attention_weight_max": 34.69127655029297,
      "activations/layer15_attention_weight_min": -30.47096824645996,
      "activations/layer16_attention_weight_max": 35.185585021972656,
      "activations/layer16_attention_weight_min": -32.85979461669922,
      "activations/layer17_attention_weight_max": 57.44667434692383,
      "activations/layer17_attention_weight_min": -45.844886779785156,
      "activations/layer18_attention_weight_max": 52.23644256591797,
      "activations/layer18_attention_weight_min": -42.76145553588867,
      "activations/layer19_attention_weight_max": 27.826751708984375,
      "activations/layer19_attention_weight_min": -25.81794548034668,
      "activations/layer1_attention_weight_max": 16.72149085998535,
      "activations/layer1_attention_weight_min": -14.385725021362305,
      "activations/layer20_attention_weight_max": 28.014463424682617,
      "activations/layer20_attention_weight_min": -22.470083236694336,
      "activations/layer21_attention_weight_max": 39.685340881347656,
      "activations/layer21_attention_weight_min": -25.141571044921875,
      "activations/layer22_attention_weight_max": 31.771326065063477,
      "activations/layer22_attention_weight_min": -29.47356414794922,
      "activations/layer23_attention_weight_max": 38.469932556152344,
      "activations/layer23_attention_weight_min": -25.72170066833496,
      "activations/layer2_attention_weight_max": 31.233707427978516,
      "activations/layer2_attention_weight_min": -28.937963485717773,
      "activations/layer3_attention_weight_max": 87.77078247070312,
      "activations/layer3_attention_weight_min": -87.30503845214844,
      "activations/layer4_attention_weight_max": 85.61203002929688,
      "activations/layer4_attention_weight_min": -83.27510070800781,
      "activations/layer5_attention_weight_max": 62.89183807373047,
      "activations/layer5_attention_weight_min": -72.47752380371094,
      "activations/layer6_attention_weight_max": 49.74732971191406,
      "activations/layer6_attention_weight_min": -53.302818298339844,
      "activations/layer7_attention_weight_max": 65.31289672851562,
      "activations/layer7_attention_weight_min": -60.99830627441406,
      "activations/layer8_attention_weight_max": 46.83066940307617,
      "activations/layer8_attention_weight_min": -49.6895637512207,
      "activations/layer9_attention_weight_max": 43.710792541503906,
      "activations/layer9_attention_weight_min": -46.493003845214844,
      "epoch": 16.22,
      "learning_rate": 4.5843939393939394e-05,
      "loss": 2.7494,
      "step": 279200
    },
    {
      "activations/layer0_attention_weight_max": 15.499754905700684,
      "activations/layer0_attention_weight_min": -13.124250411987305,
      "activations/layer10_attention_weight_max": 34.3629035949707,
      "activations/layer10_attention_weight_min": -36.45508575439453,
      "activations/layer11_attention_weight_max": 31.94611167907715,
      "activations/layer11_attention_weight_min": -33.49729537963867,
      "activations/layer12_attention_weight_max": 18.887746810913086,
      "activations/layer12_attention_weight_min": -23.646961212158203,
      "activations/layer13_attention_weight_max": 31.111370086669922,
      "activations/layer13_attention_weight_min": -28.563079833984375,
      "activations/layer14_attention_weight_max": 33.64237594604492,
      "activations/layer14_attention_weight_min": -28.945858001708984,
      "activations/layer15_attention_weight_max": 30.72899627685547,
      "activations/layer15_attention_weight_min": -30.366079330444336,
      "activations/layer16_attention_weight_max": 31.898513793945312,
      "activations/layer16_attention_weight_min": -30.437238693237305,
      "activations/layer17_attention_weight_max": 49.76703643798828,
      "activations/layer17_attention_weight_min": -45.490478515625,
      "activations/layer18_attention_weight_max": 45.38875198364258,
      "activations/layer18_attention_weight_min": -39.018741607666016,
      "activations/layer19_attention_weight_max": 29.015512466430664,
      "activations/layer19_attention_weight_min": -24.879377365112305,
      "activations/layer1_attention_weight_max": 16.043624877929688,
      "activations/layer1_attention_weight_min": -14.87994384765625,
      "activations/layer20_attention_weight_max": 27.727996826171875,
      "activations/layer20_attention_weight_min": -22.150911331176758,
      "activations/layer21_attention_weight_max": 40.55756378173828,
      "activations/layer21_attention_weight_min": -24.283716201782227,
      "activations/layer22_attention_weight_max": 29.495075225830078,
      "activations/layer22_attention_weight_min": -26.700225830078125,
      "activations/layer23_attention_weight_max": 33.74593734741211,
      "activations/layer23_attention_weight_min": -23.60101318359375,
      "activations/layer2_attention_weight_max": 33.95109558105469,
      "activations/layer2_attention_weight_min": -32.80548858642578,
      "activations/layer3_attention_weight_max": 93.21221923828125,
      "activations/layer3_attention_weight_min": -96.44097137451172,
      "activations/layer4_attention_weight_max": 93.89250946044922,
      "activations/layer4_attention_weight_min": -91.04051971435547,
      "activations/layer5_attention_weight_max": 63.913063049316406,
      "activations/layer5_attention_weight_min": -72.95895385742188,
      "activations/layer6_attention_weight_max": 52.26892852783203,
      "activations/layer6_attention_weight_min": -52.91971206665039,
      "activations/layer7_attention_weight_max": 63.89413833618164,
      "activations/layer7_attention_weight_min": -61.470455169677734,
      "activations/layer8_attention_weight_max": 45.29441833496094,
      "activations/layer8_attention_weight_min": -52.44001770019531,
      "activations/layer9_attention_weight_max": 42.2330436706543,
      "activations/layer9_attention_weight_min": -47.45156478881836,
      "epoch": 16.23,
      "learning_rate": 4.5824999999999996e-05,
      "loss": 2.7465,
      "step": 279250
    },
    {
      "activations/layer0_attention_weight_max": 15.712225914001465,
      "activations/layer0_attention_weight_min": -13.635987281799316,
      "activations/layer10_attention_weight_max": 32.98570251464844,
      "activations/layer10_attention_weight_min": -33.634925842285156,
      "activations/layer11_attention_weight_max": 31.68695068359375,
      "activations/layer11_attention_weight_min": -33.642337799072266,
      "activations/layer12_attention_weight_max": 18.516510009765625,
      "activations/layer12_attention_weight_min": -23.828012466430664,
      "activations/layer13_attention_weight_max": 33.57689666748047,
      "activations/layer13_attention_weight_min": -26.45458221435547,
      "activations/layer14_attention_weight_max": 36.576751708984375,
      "activations/layer14_attention_weight_min": -31.81303596496582,
      "activations/layer15_attention_weight_max": 33.31208038330078,
      "activations/layer15_attention_weight_min": -30.435569763183594,
      "activations/layer16_attention_weight_max": 33.23959732055664,
      "activations/layer16_attention_weight_min": -31.191837310791016,
      "activations/layer17_attention_weight_max": 51.40561294555664,
      "activations/layer17_attention_weight_min": -44.73698043823242,
      "activations/layer18_attention_weight_max": 46.92287063598633,
      "activations/layer18_attention_weight_min": -39.79155349731445,
      "activations/layer19_attention_weight_max": 24.536787033081055,
      "activations/layer19_attention_weight_min": -22.854801177978516,
      "activations/layer1_attention_weight_max": 15.966118812561035,
      "activations/layer1_attention_weight_min": -16.340911865234375,
      "activations/layer20_attention_weight_max": 24.241046905517578,
      "activations/layer20_attention_weight_min": -22.222963333129883,
      "activations/layer21_attention_weight_max": 35.437461853027344,
      "activations/layer21_attention_weight_min": -24.871379852294922,
      "activations/layer22_attention_weight_max": 29.112064361572266,
      "activations/layer22_attention_weight_min": -25.563989639282227,
      "activations/layer23_attention_weight_max": 40.67293167114258,
      "activations/layer23_attention_weight_min": -24.922428131103516,
      "activations/layer2_attention_weight_max": 30.934185028076172,
      "activations/layer2_attention_weight_min": -29.655685424804688,
      "activations/layer3_attention_weight_max": 85.60816955566406,
      "activations/layer3_attention_weight_min": -85.73553466796875,
      "activations/layer4_attention_weight_max": 87.7093276977539,
      "activations/layer4_attention_weight_min": -86.59120178222656,
      "activations/layer5_attention_weight_max": 61.94512176513672,
      "activations/layer5_attention_weight_min": -72.1971664428711,
      "activations/layer6_attention_weight_max": 50.235164642333984,
      "activations/layer6_attention_weight_min": -50.174293518066406,
      "activations/layer7_attention_weight_max": 59.814388275146484,
      "activations/layer7_attention_weight_min": -60.26575469970703,
      "activations/layer8_attention_weight_max": 43.6143913269043,
      "activations/layer8_attention_weight_min": -45.29719924926758,
      "activations/layer9_attention_weight_max": 40.17932891845703,
      "activations/layer9_attention_weight_min": -43.35047149658203,
      "epoch": 16.23,
      "learning_rate": 4.58060606060606e-05,
      "loss": 2.7525,
      "step": 279300
    },
    {
      "activations/layer0_attention_weight_max": 16.177711486816406,
      "activations/layer0_attention_weight_min": -13.043022155761719,
      "activations/layer10_attention_weight_max": 36.79804229736328,
      "activations/layer10_attention_weight_min": -34.863922119140625,
      "activations/layer11_attention_weight_max": 32.4785270690918,
      "activations/layer11_attention_weight_min": -34.43901443481445,
      "activations/layer12_attention_weight_max": 19.012332916259766,
      "activations/layer12_attention_weight_min": -24.58687973022461,
      "activations/layer13_attention_weight_max": 34.726409912109375,
      "activations/layer13_attention_weight_min": -27.536928176879883,
      "activations/layer14_attention_weight_max": 35.39664077758789,
      "activations/layer14_attention_weight_min": -30.598508834838867,
      "activations/layer15_attention_weight_max": 32.810752868652344,
      "activations/layer15_attention_weight_min": -30.0430850982666,
      "activations/layer16_attention_weight_max": 34.19773483276367,
      "activations/layer16_attention_weight_min": -33.69974136352539,
      "activations/layer17_attention_weight_max": 54.42961883544922,
      "activations/layer17_attention_weight_min": -49.92460250854492,
      "activations/layer18_attention_weight_max": 50.047237396240234,
      "activations/layer18_attention_weight_min": -46.99684143066406,
      "activations/layer19_attention_weight_max": 23.582128524780273,
      "activations/layer19_attention_weight_min": -26.178979873657227,
      "activations/layer1_attention_weight_max": 17.05196189880371,
      "activations/layer1_attention_weight_min": -13.29667854309082,
      "activations/layer20_attention_weight_max": 22.820697784423828,
      "activations/layer20_attention_weight_min": -23.073680877685547,
      "activations/layer21_attention_weight_max": 32.54793167114258,
      "activations/layer21_attention_weight_min": -25.378238677978516,
      "activations/layer22_attention_weight_max": 28.992408752441406,
      "activations/layer22_attention_weight_min": -24.95370101928711,
      "activations/layer23_attention_weight_max": 38.82244110107422,
      "activations/layer23_attention_weight_min": -23.8596248626709,
      "activations/layer2_attention_weight_max": 28.266477584838867,
      "activations/layer2_attention_weight_min": -28.818801879882812,
      "activations/layer3_attention_weight_max": 85.39399719238281,
      "activations/layer3_attention_weight_min": -87.28714752197266,
      "activations/layer4_attention_weight_max": 89.27481079101562,
      "activations/layer4_attention_weight_min": -88.7044906616211,
      "activations/layer5_attention_weight_max": 60.6497802734375,
      "activations/layer5_attention_weight_min": -70.42295837402344,
      "activations/layer6_attention_weight_max": 47.96705627441406,
      "activations/layer6_attention_weight_min": -48.83743667602539,
      "activations/layer7_attention_weight_max": 59.88722610473633,
      "activations/layer7_attention_weight_min": -57.68825149536133,
      "activations/layer8_attention_weight_max": 44.2724609375,
      "activations/layer8_attention_weight_min": -45.323909759521484,
      "activations/layer9_attention_weight_max": 42.477474212646484,
      "activations/layer9_attention_weight_min": -44.80147933959961,
      "epoch": 16.23,
      "learning_rate": 4.578712121212121e-05,
      "loss": 2.7328,
      "step": 279350
    },
    {
      "activations/layer0_attention_weight_max": 16.61849594116211,
      "activations/layer0_attention_weight_min": -13.528419494628906,
      "activations/layer10_attention_weight_max": 36.3051643371582,
      "activations/layer10_attention_weight_min": -36.34861373901367,
      "activations/layer11_attention_weight_max": 35.44681930541992,
      "activations/layer11_attention_weight_min": -32.49724197387695,
      "activations/layer12_attention_weight_max": 18.10786247253418,
      "activations/layer12_attention_weight_min": -23.843088150024414,
      "activations/layer13_attention_weight_max": 36.715057373046875,
      "activations/layer13_attention_weight_min": -27.498586654663086,
      "activations/layer14_attention_weight_max": 38.427398681640625,
      "activations/layer14_attention_weight_min": -30.943788528442383,
      "activations/layer15_attention_weight_max": 30.386686325073242,
      "activations/layer15_attention_weight_min": -29.142473220825195,
      "activations/layer16_attention_weight_max": 31.70244026184082,
      "activations/layer16_attention_weight_min": -31.252290725708008,
      "activations/layer17_attention_weight_max": 50.76173400878906,
      "activations/layer17_attention_weight_min": -42.90758514404297,
      "activations/layer18_attention_weight_max": 45.29742431640625,
      "activations/layer18_attention_weight_min": -37.19240188598633,
      "activations/layer19_attention_weight_max": 21.55951690673828,
      "activations/layer19_attention_weight_min": -22.797767639160156,
      "activations/layer1_attention_weight_max": 15.919134140014648,
      "activations/layer1_attention_weight_min": -14.458963394165039,
      "activations/layer20_attention_weight_max": 21.80801773071289,
      "activations/layer20_attention_weight_min": -23.190288543701172,
      "activations/layer21_attention_weight_max": 33.70946502685547,
      "activations/layer21_attention_weight_min": -23.613359451293945,
      "activations/layer22_attention_weight_max": 29.839847564697266,
      "activations/layer22_attention_weight_min": -25.86347007751465,
      "activations/layer23_attention_weight_max": 36.74705505371094,
      "activations/layer23_attention_weight_min": -24.78598403930664,
      "activations/layer2_attention_weight_max": 31.7850399017334,
      "activations/layer2_attention_weight_min": -27.916000366210938,
      "activations/layer3_attention_weight_max": 83.04109954833984,
      "activations/layer3_attention_weight_min": -87.34286499023438,
      "activations/layer4_attention_weight_max": 90.2872543334961,
      "activations/layer4_attention_weight_min": -88.38482666015625,
      "activations/layer5_attention_weight_max": 65.96405029296875,
      "activations/layer5_attention_weight_min": -75.93901062011719,
      "activations/layer6_attention_weight_max": 51.1577033996582,
      "activations/layer6_attention_weight_min": -52.183223724365234,
      "activations/layer7_attention_weight_max": 64.66175842285156,
      "activations/layer7_attention_weight_min": -64.25997161865234,
      "activations/layer8_attention_weight_max": 46.3056640625,
      "activations/layer8_attention_weight_min": -47.860877990722656,
      "activations/layer9_attention_weight_max": 50.38314437866211,
      "activations/layer9_attention_weight_min": -44.53190994262695,
      "epoch": 16.23,
      "learning_rate": 4.5768181818181814e-05,
      "loss": 2.7358,
      "step": 279400
    },
    {
      "activations/layer0_attention_weight_max": 15.592409133911133,
      "activations/layer0_attention_weight_min": -13.656010627746582,
      "activations/layer10_attention_weight_max": 35.031150817871094,
      "activations/layer10_attention_weight_min": -37.20216369628906,
      "activations/layer11_attention_weight_max": 31.97327995300293,
      "activations/layer11_attention_weight_min": -35.750308990478516,
      "activations/layer12_attention_weight_max": 19.273157119750977,
      "activations/layer12_attention_weight_min": -25.15986442565918,
      "activations/layer13_attention_weight_max": 43.13479995727539,
      "activations/layer13_attention_weight_min": -31.977943420410156,
      "activations/layer14_attention_weight_max": 35.24787139892578,
      "activations/layer14_attention_weight_min": -32.043949127197266,
      "activations/layer15_attention_weight_max": 32.25372314453125,
      "activations/layer15_attention_weight_min": -30.378929138183594,
      "activations/layer16_attention_weight_max": 32.80413055419922,
      "activations/layer16_attention_weight_min": -30.303266525268555,
      "activations/layer17_attention_weight_max": 51.55083465576172,
      "activations/layer17_attention_weight_min": -46.324913024902344,
      "activations/layer18_attention_weight_max": 45.51974868774414,
      "activations/layer18_attention_weight_min": -40.78582763671875,
      "activations/layer19_attention_weight_max": 23.17513656616211,
      "activations/layer19_attention_weight_min": -25.050329208374023,
      "activations/layer1_attention_weight_max": 16.36147117614746,
      "activations/layer1_attention_weight_min": -15.615278244018555,
      "activations/layer20_attention_weight_max": 23.512773513793945,
      "activations/layer20_attention_weight_min": -22.527034759521484,
      "activations/layer21_attention_weight_max": 34.1999626159668,
      "activations/layer21_attention_weight_min": -23.08662223815918,
      "activations/layer22_attention_weight_max": 28.97559928894043,
      "activations/layer22_attention_weight_min": -25.769630432128906,
      "activations/layer23_attention_weight_max": 40.321842193603516,
      "activations/layer23_attention_weight_min": -24.30573272705078,
      "activations/layer2_attention_weight_max": 30.06121063232422,
      "activations/layer2_attention_weight_min": -29.616682052612305,
      "activations/layer3_attention_weight_max": 83.837890625,
      "activations/layer3_attention_weight_min": -83.85908508300781,
      "activations/layer4_attention_weight_max": 87.77986907958984,
      "activations/layer4_attention_weight_min": -88.21966552734375,
      "activations/layer5_attention_weight_max": 62.887779235839844,
      "activations/layer5_attention_weight_min": -73.95354461669922,
      "activations/layer6_attention_weight_max": 48.071876525878906,
      "activations/layer6_attention_weight_min": -50.664649963378906,
      "activations/layer7_attention_weight_max": 63.22771072387695,
      "activations/layer7_attention_weight_min": -62.8358039855957,
      "activations/layer8_attention_weight_max": 43.50197219848633,
      "activations/layer8_attention_weight_min": -48.04938507080078,
      "activations/layer9_attention_weight_max": 42.818790435791016,
      "activations/layer9_attention_weight_min": -46.25657272338867,
      "epoch": 16.24,
      "learning_rate": 4.574924242424242e-05,
      "loss": 2.7448,
      "step": 279450
    },
    {
      "activations/layer0_attention_weight_max": 15.579751968383789,
      "activations/layer0_attention_weight_min": -13.525341987609863,
      "activations/layer10_attention_weight_max": 35.159141540527344,
      "activations/layer10_attention_weight_min": -35.342498779296875,
      "activations/layer11_attention_weight_max": 33.56287384033203,
      "activations/layer11_attention_weight_min": -32.82990646362305,
      "activations/layer12_attention_weight_max": 17.988454818725586,
      "activations/layer12_attention_weight_min": -26.348365783691406,
      "activations/layer13_attention_weight_max": 34.45869064331055,
      "activations/layer13_attention_weight_min": -26.842744827270508,
      "activations/layer14_attention_weight_max": 37.03359603881836,
      "activations/layer14_attention_weight_min": -28.28980827331543,
      "activations/layer15_attention_weight_max": 34.145362854003906,
      "activations/layer15_attention_weight_min": -30.1451358795166,
      "activations/layer16_attention_weight_max": 33.00349807739258,
      "activations/layer16_attention_weight_min": -33.589637756347656,
      "activations/layer17_attention_weight_max": 50.962093353271484,
      "activations/layer17_attention_weight_min": -48.59634780883789,
      "activations/layer18_attention_weight_max": 43.84843444824219,
      "activations/layer18_attention_weight_min": -41.431880950927734,
      "activations/layer19_attention_weight_max": 22.388341903686523,
      "activations/layer19_attention_weight_min": -21.494516372680664,
      "activations/layer1_attention_weight_max": 16.403146743774414,
      "activations/layer1_attention_weight_min": -16.250511169433594,
      "activations/layer20_attention_weight_max": 22.61001968383789,
      "activations/layer20_attention_weight_min": -21.849546432495117,
      "activations/layer21_attention_weight_max": 31.028757095336914,
      "activations/layer21_attention_weight_min": -20.889869689941406,
      "activations/layer22_attention_weight_max": 28.68497657775879,
      "activations/layer22_attention_weight_min": -25.024215698242188,
      "activations/layer23_attention_weight_max": 33.08638000488281,
      "activations/layer23_attention_weight_min": -23.7299861907959,
      "activations/layer2_attention_weight_max": 31.51636505126953,
      "activations/layer2_attention_weight_min": -30.283767700195312,
      "activations/layer3_attention_weight_max": 88.8743896484375,
      "activations/layer3_attention_weight_min": -90.59626770019531,
      "activations/layer4_attention_weight_max": 88.6242446899414,
      "activations/layer4_attention_weight_min": -86.28630828857422,
      "activations/layer5_attention_weight_max": 61.75832748413086,
      "activations/layer5_attention_weight_min": -75.81272888183594,
      "activations/layer6_attention_weight_max": 51.88041687011719,
      "activations/layer6_attention_weight_min": -52.14094924926758,
      "activations/layer7_attention_weight_max": 63.117008209228516,
      "activations/layer7_attention_weight_min": -61.925811767578125,
      "activations/layer8_attention_weight_max": 48.503944396972656,
      "activations/layer8_attention_weight_min": -50.34208679199219,
      "activations/layer9_attention_weight_max": 43.148738861083984,
      "activations/layer9_attention_weight_min": -47.323543548583984,
      "epoch": 16.24,
      "learning_rate": 4.5730303030303024e-05,
      "loss": 2.7423,
      "step": 279500
    },
    {
      "activations/layer0_attention_weight_max": 15.610261917114258,
      "activations/layer0_attention_weight_min": -13.750561714172363,
      "activations/layer10_attention_weight_max": 38.33116149902344,
      "activations/layer10_attention_weight_min": -38.19732666015625,
      "activations/layer11_attention_weight_max": 33.120506286621094,
      "activations/layer11_attention_weight_min": -33.72397232055664,
      "activations/layer12_attention_weight_max": 20.417251586914062,
      "activations/layer12_attention_weight_min": -27.149246215820312,
      "activations/layer13_attention_weight_max": 32.911251068115234,
      "activations/layer13_attention_weight_min": -26.20406723022461,
      "activations/layer14_attention_weight_max": 34.26923370361328,
      "activations/layer14_attention_weight_min": -31.259435653686523,
      "activations/layer15_attention_weight_max": 30.3721981048584,
      "activations/layer15_attention_weight_min": -30.662090301513672,
      "activations/layer16_attention_weight_max": 31.691516876220703,
      "activations/layer16_attention_weight_min": -30.48442840576172,
      "activations/layer17_attention_weight_max": 49.375858306884766,
      "activations/layer17_attention_weight_min": -44.394466400146484,
      "activations/layer18_attention_weight_max": 45.80931854248047,
      "activations/layer18_attention_weight_min": -40.14528274536133,
      "activations/layer19_attention_weight_max": 23.4993953704834,
      "activations/layer19_attention_weight_min": -22.868061065673828,
      "activations/layer1_attention_weight_max": 17.293760299682617,
      "activations/layer1_attention_weight_min": -14.942458152770996,
      "activations/layer20_attention_weight_max": 23.0218563079834,
      "activations/layer20_attention_weight_min": -22.089765548706055,
      "activations/layer21_attention_weight_max": 36.84005355834961,
      "activations/layer21_attention_weight_min": -23.778085708618164,
      "activations/layer22_attention_weight_max": 27.19782066345215,
      "activations/layer22_attention_weight_min": -24.86296272277832,
      "activations/layer23_attention_weight_max": 33.748046875,
      "activations/layer23_attention_weight_min": -23.144498825073242,
      "activations/layer2_attention_weight_max": 30.327014923095703,
      "activations/layer2_attention_weight_min": -27.729022979736328,
      "activations/layer3_attention_weight_max": 84.0322036743164,
      "activations/layer3_attention_weight_min": -82.73638916015625,
      "activations/layer4_attention_weight_max": 91.89580535888672,
      "activations/layer4_attention_weight_min": -86.14960479736328,
      "activations/layer5_attention_weight_max": 63.61914825439453,
      "activations/layer5_attention_weight_min": -77.56183624267578,
      "activations/layer6_attention_weight_max": 54.63957595825195,
      "activations/layer6_attention_weight_min": -52.623680114746094,
      "activations/layer7_attention_weight_max": 65.76174926757812,
      "activations/layer7_attention_weight_min": -61.32658386230469,
      "activations/layer8_attention_weight_max": 49.493377685546875,
      "activations/layer8_attention_weight_min": -49.832435607910156,
      "activations/layer9_attention_weight_max": 45.145816802978516,
      "activations/layer9_attention_weight_min": -45.97506332397461,
      "epoch": 16.24,
      "learning_rate": 4.571136363636363e-05,
      "loss": 2.7515,
      "step": 279550
    },
    {
      "activations/layer0_attention_weight_max": 15.865166664123535,
      "activations/layer0_attention_weight_min": -13.57291316986084,
      "activations/layer10_attention_weight_max": 38.61316680908203,
      "activations/layer10_attention_weight_min": -36.56825637817383,
      "activations/layer11_attention_weight_max": 37.98798751831055,
      "activations/layer11_attention_weight_min": -35.84940719604492,
      "activations/layer12_attention_weight_max": 17.792905807495117,
      "activations/layer12_attention_weight_min": -27.41326141357422,
      "activations/layer13_attention_weight_max": 38.071258544921875,
      "activations/layer13_attention_weight_min": -33.0684928894043,
      "activations/layer14_attention_weight_max": 43.789249420166016,
      "activations/layer14_attention_weight_min": -31.7456111907959,
      "activations/layer15_attention_weight_max": 44.2298583984375,
      "activations/layer15_attention_weight_min": -31.968050003051758,
      "activations/layer16_attention_weight_max": 33.312381744384766,
      "activations/layer16_attention_weight_min": -32.62973403930664,
      "activations/layer17_attention_weight_max": 52.20047378540039,
      "activations/layer17_attention_weight_min": -45.1109619140625,
      "activations/layer18_attention_weight_max": 45.924964904785156,
      "activations/layer18_attention_weight_min": -41.766815185546875,
      "activations/layer19_attention_weight_max": 21.994321823120117,
      "activations/layer19_attention_weight_min": -24.365385055541992,
      "activations/layer1_attention_weight_max": 15.654858589172363,
      "activations/layer1_attention_weight_min": -14.78591537475586,
      "activations/layer20_attention_weight_max": 24.244417190551758,
      "activations/layer20_attention_weight_min": -21.812274932861328,
      "activations/layer21_attention_weight_max": 42.71101760864258,
      "activations/layer21_attention_weight_min": -26.630950927734375,
      "activations/layer22_attention_weight_max": 29.86823081970215,
      "activations/layer22_attention_weight_min": -23.597654342651367,
      "activations/layer23_attention_weight_max": 37.327842712402344,
      "activations/layer23_attention_weight_min": -23.45696258544922,
      "activations/layer2_attention_weight_max": 31.730083465576172,
      "activations/layer2_attention_weight_min": -30.15949058532715,
      "activations/layer3_attention_weight_max": 88.04082489013672,
      "activations/layer3_attention_weight_min": -89.75379943847656,
      "activations/layer4_attention_weight_max": 91.95230865478516,
      "activations/layer4_attention_weight_min": -90.80953216552734,
      "activations/layer5_attention_weight_max": 64.27838897705078,
      "activations/layer5_attention_weight_min": -84.85714721679688,
      "activations/layer6_attention_weight_max": 52.58146667480469,
      "activations/layer6_attention_weight_min": -52.3649787902832,
      "activations/layer7_attention_weight_max": 71.63513946533203,
      "activations/layer7_attention_weight_min": -63.4245491027832,
      "activations/layer8_attention_weight_max": 50.44919967651367,
      "activations/layer8_attention_weight_min": -46.828243255615234,
      "activations/layer9_attention_weight_max": 53.10110092163086,
      "activations/layer9_attention_weight_min": -47.49790573120117,
      "epoch": 16.25,
      "learning_rate": 4.569242424242424e-05,
      "loss": 2.7542,
      "step": 279600
    },
    {
      "activations/layer0_attention_weight_max": 16.259342193603516,
      "activations/layer0_attention_weight_min": -13.54259204864502,
      "activations/layer10_attention_weight_max": 36.82195281982422,
      "activations/layer10_attention_weight_min": -36.337032318115234,
      "activations/layer11_attention_weight_max": 34.29793167114258,
      "activations/layer11_attention_weight_min": -32.907806396484375,
      "activations/layer12_attention_weight_max": 18.553817749023438,
      "activations/layer12_attention_weight_min": -30.21245002746582,
      "activations/layer13_attention_weight_max": 44.550472259521484,
      "activations/layer13_attention_weight_min": -34.22132873535156,
      "activations/layer14_attention_weight_max": 40.8544807434082,
      "activations/layer14_attention_weight_min": -29.780899047851562,
      "activations/layer15_attention_weight_max": 39.47965621948242,
      "activations/layer15_attention_weight_min": -33.3504524230957,
      "activations/layer16_attention_weight_max": 29.373186111450195,
      "activations/layer16_attention_weight_min": -30.52071189880371,
      "activations/layer17_attention_weight_max": 49.50934600830078,
      "activations/layer17_attention_weight_min": -45.57876205444336,
      "activations/layer18_attention_weight_max": 42.58058547973633,
      "activations/layer18_attention_weight_min": -41.16968536376953,
      "activations/layer19_attention_weight_max": 20.71546745300293,
      "activations/layer19_attention_weight_min": -21.4222469329834,
      "activations/layer1_attention_weight_max": 16.71492576599121,
      "activations/layer1_attention_weight_min": -15.797219276428223,
      "activations/layer20_attention_weight_max": 19.40996742248535,
      "activations/layer20_attention_weight_min": -21.22395133972168,
      "activations/layer21_attention_weight_max": 33.32631301879883,
      "activations/layer21_attention_weight_min": -20.64515495300293,
      "activations/layer22_attention_weight_max": 24.56365203857422,
      "activations/layer22_attention_weight_min": -23.548730850219727,
      "activations/layer23_attention_weight_max": 30.96947479248047,
      "activations/layer23_attention_weight_min": -23.21456527709961,
      "activations/layer2_attention_weight_max": 31.160484313964844,
      "activations/layer2_attention_weight_min": -30.738386154174805,
      "activations/layer3_attention_weight_max": 84.93534851074219,
      "activations/layer3_attention_weight_min": -84.9673080444336,
      "activations/layer4_attention_weight_max": 89.52748107910156,
      "activations/layer4_attention_weight_min": -87.74031829833984,
      "activations/layer5_attention_weight_max": 63.65827560424805,
      "activations/layer5_attention_weight_min": -78.80245971679688,
      "activations/layer6_attention_weight_max": 49.10391616821289,
      "activations/layer6_attention_weight_min": -50.596492767333984,
      "activations/layer7_attention_weight_max": 67.71597290039062,
      "activations/layer7_attention_weight_min": -58.873695373535156,
      "activations/layer8_attention_weight_max": 46.93291091918945,
      "activations/layer8_attention_weight_min": -46.22041320800781,
      "activations/layer9_attention_weight_max": 52.68009567260742,
      "activations/layer9_attention_weight_min": -50.04199981689453,
      "epoch": 16.25,
      "learning_rate": 4.567348484848484e-05,
      "loss": 2.7349,
      "step": 279650
    },
    {
      "activations/layer0_attention_weight_max": 15.257692337036133,
      "activations/layer0_attention_weight_min": -13.229395866394043,
      "activations/layer10_attention_weight_max": 38.218299865722656,
      "activations/layer10_attention_weight_min": -36.83311080932617,
      "activations/layer11_attention_weight_max": 32.904052734375,
      "activations/layer11_attention_weight_min": -35.35271453857422,
      "activations/layer12_attention_weight_max": 19.193557739257812,
      "activations/layer12_attention_weight_min": -25.625532150268555,
      "activations/layer13_attention_weight_max": 34.50969314575195,
      "activations/layer13_attention_weight_min": -27.82680892944336,
      "activations/layer14_attention_weight_max": 36.18537139892578,
      "activations/layer14_attention_weight_min": -29.595619201660156,
      "activations/layer15_attention_weight_max": 33.21199035644531,
      "activations/layer15_attention_weight_min": -29.93361473083496,
      "activations/layer16_attention_weight_max": 33.23124694824219,
      "activations/layer16_attention_weight_min": -31.659589767456055,
      "activations/layer17_attention_weight_max": 52.6487922668457,
      "activations/layer17_attention_weight_min": -43.61716842651367,
      "activations/layer18_attention_weight_max": 44.8227653503418,
      "activations/layer18_attention_weight_min": -38.88888931274414,
      "activations/layer19_attention_weight_max": 25.43779754638672,
      "activations/layer19_attention_weight_min": -22.26453971862793,
      "activations/layer1_attention_weight_max": 15.217300415039062,
      "activations/layer1_attention_weight_min": -14.078170776367188,
      "activations/layer20_attention_weight_max": 22.720706939697266,
      "activations/layer20_attention_weight_min": -25.269359588623047,
      "activations/layer21_attention_weight_max": 36.46763229370117,
      "activations/layer21_attention_weight_min": -21.38276481628418,
      "activations/layer22_attention_weight_max": 28.40998649597168,
      "activations/layer22_attention_weight_min": -28.11949920654297,
      "activations/layer23_attention_weight_max": 39.31819152832031,
      "activations/layer23_attention_weight_min": -23.25465202331543,
      "activations/layer2_attention_weight_max": 29.361312866210938,
      "activations/layer2_attention_weight_min": -28.258026123046875,
      "activations/layer3_attention_weight_max": 86.48381042480469,
      "activations/layer3_attention_weight_min": -83.10187530517578,
      "activations/layer4_attention_weight_max": 87.86966705322266,
      "activations/layer4_attention_weight_min": -88.75724792480469,
      "activations/layer5_attention_weight_max": 66.45802307128906,
      "activations/layer5_attention_weight_min": -71.8929672241211,
      "activations/layer6_attention_weight_max": 51.418941497802734,
      "activations/layer6_attention_weight_min": -52.735538482666016,
      "activations/layer7_attention_weight_max": 62.24332046508789,
      "activations/layer7_attention_weight_min": -60.95726776123047,
      "activations/layer8_attention_weight_max": 48.6447639465332,
      "activations/layer8_attention_weight_min": -52.19850540161133,
      "activations/layer9_attention_weight_max": 47.77149963378906,
      "activations/layer9_attention_weight_min": -47.92753601074219,
      "epoch": 16.25,
      "learning_rate": 4.565454545454544e-05,
      "loss": 2.7413,
      "step": 279700
    },
    {
      "activations/layer0_attention_weight_max": 15.339076042175293,
      "activations/layer0_attention_weight_min": -13.89919662475586,
      "activations/layer10_attention_weight_max": 35.2984504699707,
      "activations/layer10_attention_weight_min": -35.80707550048828,
      "activations/layer11_attention_weight_max": 31.639602661132812,
      "activations/layer11_attention_weight_min": -34.370079040527344,
      "activations/layer12_attention_weight_max": 18.284351348876953,
      "activations/layer12_attention_weight_min": -23.959959030151367,
      "activations/layer13_attention_weight_max": 30.16732406616211,
      "activations/layer13_attention_weight_min": -27.934526443481445,
      "activations/layer14_attention_weight_max": 33.67003631591797,
      "activations/layer14_attention_weight_min": -29.50827980041504,
      "activations/layer15_attention_weight_max": 30.5485897064209,
      "activations/layer15_attention_weight_min": -29.59772300720215,
      "activations/layer16_attention_weight_max": 31.146194458007812,
      "activations/layer16_attention_weight_min": -29.88450813293457,
      "activations/layer17_attention_weight_max": 47.20296096801758,
      "activations/layer17_attention_weight_min": -40.45980453491211,
      "activations/layer18_attention_weight_max": 43.64188003540039,
      "activations/layer18_attention_weight_min": -37.970619201660156,
      "activations/layer19_attention_weight_max": 22.268510818481445,
      "activations/layer19_attention_weight_min": -24.553579330444336,
      "activations/layer1_attention_weight_max": 16.174156188964844,
      "activations/layer1_attention_weight_min": -15.241215705871582,
      "activations/layer20_attention_weight_max": 21.883712768554688,
      "activations/layer20_attention_weight_min": -23.71357536315918,
      "activations/layer21_attention_weight_max": 35.55070495605469,
      "activations/layer21_attention_weight_min": -23.519573211669922,
      "activations/layer22_attention_weight_max": 28.09324836730957,
      "activations/layer22_attention_weight_min": -26.93644905090332,
      "activations/layer23_attention_weight_max": 34.993526458740234,
      "activations/layer23_attention_weight_min": -25.023822784423828,
      "activations/layer2_attention_weight_max": 31.278667449951172,
      "activations/layer2_attention_weight_min": -29.897361755371094,
      "activations/layer3_attention_weight_max": 87.64861297607422,
      "activations/layer3_attention_weight_min": -88.2774658203125,
      "activations/layer4_attention_weight_max": 90.73199462890625,
      "activations/layer4_attention_weight_min": -93.4087142944336,
      "activations/layer5_attention_weight_max": 66.73725891113281,
      "activations/layer5_attention_weight_min": -76.22428894042969,
      "activations/layer6_attention_weight_max": 48.884090423583984,
      "activations/layer6_attention_weight_min": -50.3563346862793,
      "activations/layer7_attention_weight_max": 64.01496124267578,
      "activations/layer7_attention_weight_min": -60.79209899902344,
      "activations/layer8_attention_weight_max": 45.393104553222656,
      "activations/layer8_attention_weight_min": -45.96955490112305,
      "activations/layer9_attention_weight_max": 42.480804443359375,
      "activations/layer9_attention_weight_min": -43.64030075073242,
      "epoch": 16.26,
      "learning_rate": 4.563560606060606e-05,
      "loss": 2.7411,
      "step": 279750
    },
    {
      "activations/layer0_attention_weight_max": 16.267045974731445,
      "activations/layer0_attention_weight_min": -13.771015167236328,
      "activations/layer10_attention_weight_max": 32.7232551574707,
      "activations/layer10_attention_weight_min": -34.22722625732422,
      "activations/layer11_attention_weight_max": 29.82162094116211,
      "activations/layer11_attention_weight_min": -32.7244987487793,
      "activations/layer12_attention_weight_max": 18.627439498901367,
      "activations/layer12_attention_weight_min": -24.996906280517578,
      "activations/layer13_attention_weight_max": 32.17256164550781,
      "activations/layer13_attention_weight_min": -29.269014358520508,
      "activations/layer14_attention_weight_max": 34.10874557495117,
      "activations/layer14_attention_weight_min": -30.505197525024414,
      "activations/layer15_attention_weight_max": 32.80595779418945,
      "activations/layer15_attention_weight_min": -28.883466720581055,
      "activations/layer16_attention_weight_max": 31.579740524291992,
      "activations/layer16_attention_weight_min": -30.369958877563477,
      "activations/layer17_attention_weight_max": 49.222469329833984,
      "activations/layer17_attention_weight_min": -43.4922981262207,
      "activations/layer18_attention_weight_max": 46.22164535522461,
      "activations/layer18_attention_weight_min": -38.60900115966797,
      "activations/layer19_attention_weight_max": 24.503915786743164,
      "activations/layer19_attention_weight_min": -21.470298767089844,
      "activations/layer1_attention_weight_max": 16.045856475830078,
      "activations/layer1_attention_weight_min": -15.122262954711914,
      "activations/layer20_attention_weight_max": 22.0520076751709,
      "activations/layer20_attention_weight_min": -20.457090377807617,
      "activations/layer21_attention_weight_max": 35.26139450073242,
      "activations/layer21_attention_weight_min": -24.87956428527832,
      "activations/layer22_attention_weight_max": 30.583894729614258,
      "activations/layer22_attention_weight_min": -24.367029190063477,
      "activations/layer23_attention_weight_max": 35.134559631347656,
      "activations/layer23_attention_weight_min": -24.684288024902344,
      "activations/layer2_attention_weight_max": 31.334447860717773,
      "activations/layer2_attention_weight_min": -29.815593719482422,
      "activations/layer3_attention_weight_max": 88.70030212402344,
      "activations/layer3_attention_weight_min": -85.42029571533203,
      "activations/layer4_attention_weight_max": 91.09852600097656,
      "activations/layer4_attention_weight_min": -86.8561782836914,
      "activations/layer5_attention_weight_max": 66.31768035888672,
      "activations/layer5_attention_weight_min": -74.16727447509766,
      "activations/layer6_attention_weight_max": 50.15877151489258,
      "activations/layer6_attention_weight_min": -49.74037170410156,
      "activations/layer7_attention_weight_max": 63.203487396240234,
      "activations/layer7_attention_weight_min": -58.04765701293945,
      "activations/layer8_attention_weight_max": 42.944026947021484,
      "activations/layer8_attention_weight_min": -50.63078689575195,
      "activations/layer9_attention_weight_max": 44.40462875366211,
      "activations/layer9_attention_weight_min": -45.5354118347168,
      "epoch": 16.26,
      "learning_rate": 4.561666666666666e-05,
      "loss": 2.7378,
      "step": 279800
    },
    {
      "activations/layer0_attention_weight_max": 16.48169708251953,
      "activations/layer0_attention_weight_min": -13.520625114440918,
      "activations/layer10_attention_weight_max": 35.15432357788086,
      "activations/layer10_attention_weight_min": -36.13526916503906,
      "activations/layer11_attention_weight_max": 31.50813102722168,
      "activations/layer11_attention_weight_min": -32.00102615356445,
      "activations/layer12_attention_weight_max": 17.492584228515625,
      "activations/layer12_attention_weight_min": -25.964412689208984,
      "activations/layer13_attention_weight_max": 35.228538513183594,
      "activations/layer13_attention_weight_min": -26.771669387817383,
      "activations/layer14_attention_weight_max": 32.52943801879883,
      "activations/layer14_attention_weight_min": -31.375621795654297,
      "activations/layer15_attention_weight_max": 31.197372436523438,
      "activations/layer15_attention_weight_min": -30.02769660949707,
      "activations/layer16_attention_weight_max": 30.856725692749023,
      "activations/layer16_attention_weight_min": -30.685956954956055,
      "activations/layer17_attention_weight_max": 49.31754684448242,
      "activations/layer17_attention_weight_min": -44.37116241455078,
      "activations/layer18_attention_weight_max": 43.12106704711914,
      "activations/layer18_attention_weight_min": -39.91872024536133,
      "activations/layer19_attention_weight_max": 21.891386032104492,
      "activations/layer19_attention_weight_min": -20.812393188476562,
      "activations/layer1_attention_weight_max": 15.66238021850586,
      "activations/layer1_attention_weight_min": -16.37592315673828,
      "activations/layer20_attention_weight_max": 20.850107192993164,
      "activations/layer20_attention_weight_min": -23.539466857910156,
      "activations/layer21_attention_weight_max": 33.35466003417969,
      "activations/layer21_attention_weight_min": -22.577346801757812,
      "activations/layer22_attention_weight_max": 28.299848556518555,
      "activations/layer22_attention_weight_min": -25.42212677001953,
      "activations/layer23_attention_weight_max": 37.830326080322266,
      "activations/layer23_attention_weight_min": -23.072723388671875,
      "activations/layer2_attention_weight_max": 31.08580780029297,
      "activations/layer2_attention_weight_min": -30.47431182861328,
      "activations/layer3_attention_weight_max": 87.35679626464844,
      "activations/layer3_attention_weight_min": -90.43270111083984,
      "activations/layer4_attention_weight_max": 90.10987091064453,
      "activations/layer4_attention_weight_min": -92.73912811279297,
      "activations/layer5_attention_weight_max": 62.08392333984375,
      "activations/layer5_attention_weight_min": -73.17190551757812,
      "activations/layer6_attention_weight_max": 49.79564666748047,
      "activations/layer6_attention_weight_min": -51.791831970214844,
      "activations/layer7_attention_weight_max": 58.48248291015625,
      "activations/layer7_attention_weight_min": -58.312286376953125,
      "activations/layer8_attention_weight_max": 47.81004333496094,
      "activations/layer8_attention_weight_min": -47.24602127075195,
      "activations/layer9_attention_weight_max": 45.484519958496094,
      "activations/layer9_attention_weight_min": -45.35950469970703,
      "epoch": 16.26,
      "learning_rate": 4.559772727272727e-05,
      "loss": 2.7639,
      "step": 279850
    },
    {
      "activations/layer0_attention_weight_max": 15.495282173156738,
      "activations/layer0_attention_weight_min": -13.447509765625,
      "activations/layer10_attention_weight_max": 36.20705795288086,
      "activations/layer10_attention_weight_min": -37.81035614013672,
      "activations/layer11_attention_weight_max": 33.53750991821289,
      "activations/layer11_attention_weight_min": -34.81805419921875,
      "activations/layer12_attention_weight_max": 19.852739334106445,
      "activations/layer12_attention_weight_min": -23.984786987304688,
      "activations/layer13_attention_weight_max": 50.36588668823242,
      "activations/layer13_attention_weight_min": -36.266239166259766,
      "activations/layer14_attention_weight_max": 37.1671028137207,
      "activations/layer14_attention_weight_min": -28.75177001953125,
      "activations/layer15_attention_weight_max": 30.6352596282959,
      "activations/layer15_attention_weight_min": -29.537546157836914,
      "activations/layer16_attention_weight_max": 32.73128890991211,
      "activations/layer16_attention_weight_min": -31.44495391845703,
      "activations/layer17_attention_weight_max": 51.541500091552734,
      "activations/layer17_attention_weight_min": -46.54501724243164,
      "activations/layer18_attention_weight_max": 46.61276626586914,
      "activations/layer18_attention_weight_min": -42.09698486328125,
      "activations/layer19_attention_weight_max": 22.149564743041992,
      "activations/layer19_attention_weight_min": -23.141584396362305,
      "activations/layer1_attention_weight_max": 16.528894424438477,
      "activations/layer1_attention_weight_min": -15.996650695800781,
      "activations/layer20_attention_weight_max": 22.694602966308594,
      "activations/layer20_attention_weight_min": -21.695585250854492,
      "activations/layer21_attention_weight_max": 35.38642501831055,
      "activations/layer21_attention_weight_min": -23.52556610107422,
      "activations/layer22_attention_weight_max": 28.200098037719727,
      "activations/layer22_attention_weight_min": -27.080238342285156,
      "activations/layer23_attention_weight_max": 38.1578483581543,
      "activations/layer23_attention_weight_min": -25.99200439453125,
      "activations/layer2_attention_weight_max": 33.17267990112305,
      "activations/layer2_attention_weight_min": -28.89888572692871,
      "activations/layer3_attention_weight_max": 86.26836395263672,
      "activations/layer3_attention_weight_min": -85.4439697265625,
      "activations/layer4_attention_weight_max": 89.63665008544922,
      "activations/layer4_attention_weight_min": -85.3834457397461,
      "activations/layer5_attention_weight_max": 67.20350646972656,
      "activations/layer5_attention_weight_min": -71.87944030761719,
      "activations/layer6_attention_weight_max": 49.248680114746094,
      "activations/layer6_attention_weight_min": -50.03642272949219,
      "activations/layer7_attention_weight_max": 64.30378723144531,
      "activations/layer7_attention_weight_min": -63.25563049316406,
      "activations/layer8_attention_weight_max": 46.60370635986328,
      "activations/layer8_attention_weight_min": -49.718658447265625,
      "activations/layer9_attention_weight_max": 47.2117919921875,
      "activations/layer9_attention_weight_min": -47.825565338134766,
      "epoch": 16.26,
      "learning_rate": 4.5578787878787876e-05,
      "loss": 2.7513,
      "step": 279900
    },
    {
      "activations/layer0_attention_weight_max": 15.47531795501709,
      "activations/layer0_attention_weight_min": -13.071465492248535,
      "activations/layer10_attention_weight_max": 35.1884765625,
      "activations/layer10_attention_weight_min": -35.74948501586914,
      "activations/layer11_attention_weight_max": 33.08360290527344,
      "activations/layer11_attention_weight_min": -33.600006103515625,
      "activations/layer12_attention_weight_max": 20.224720001220703,
      "activations/layer12_attention_weight_min": -32.759098052978516,
      "activations/layer13_attention_weight_max": 34.807491302490234,
      "activations/layer13_attention_weight_min": -28.68204689025879,
      "activations/layer14_attention_weight_max": 35.680572509765625,
      "activations/layer14_attention_weight_min": -33.5440673828125,
      "activations/layer15_attention_weight_max": 32.687564849853516,
      "activations/layer15_attention_weight_min": -33.2088623046875,
      "activations/layer16_attention_weight_max": 35.79853057861328,
      "activations/layer16_attention_weight_min": -34.19676971435547,
      "activations/layer17_attention_weight_max": 51.53523635864258,
      "activations/layer17_attention_weight_min": -48.58938217163086,
      "activations/layer18_attention_weight_max": 46.84252166748047,
      "activations/layer18_attention_weight_min": -44.59047317504883,
      "activations/layer19_attention_weight_max": 23.756223678588867,
      "activations/layer19_attention_weight_min": -25.219993591308594,
      "activations/layer1_attention_weight_max": 16.599035263061523,
      "activations/layer1_attention_weight_min": -14.56129264831543,
      "activations/layer20_attention_weight_max": 22.623260498046875,
      "activations/layer20_attention_weight_min": -22.80030632019043,
      "activations/layer21_attention_weight_max": 36.844482421875,
      "activations/layer21_attention_weight_min": -23.95977020263672,
      "activations/layer22_attention_weight_max": 28.272138595581055,
      "activations/layer22_attention_weight_min": -26.86697769165039,
      "activations/layer23_attention_weight_max": 32.963661193847656,
      "activations/layer23_attention_weight_min": -26.631534576416016,
      "activations/layer2_attention_weight_max": 31.33755874633789,
      "activations/layer2_attention_weight_min": -29.219276428222656,
      "activations/layer3_attention_weight_max": 86.40181732177734,
      "activations/layer3_attention_weight_min": -87.5262680053711,
      "activations/layer4_attention_weight_max": 88.53319549560547,
      "activations/layer4_attention_weight_min": -82.48249053955078,
      "activations/layer5_attention_weight_max": 59.7711296081543,
      "activations/layer5_attention_weight_min": -67.24223327636719,
      "activations/layer6_attention_weight_max": 47.90425109863281,
      "activations/layer6_attention_weight_min": -47.501888275146484,
      "activations/layer7_attention_weight_max": 58.676509857177734,
      "activations/layer7_attention_weight_min": -57.60594940185547,
      "activations/layer8_attention_weight_max": 45.71770477294922,
      "activations/layer8_attention_weight_min": -47.79557800292969,
      "activations/layer9_attention_weight_max": 46.5564079284668,
      "activations/layer9_attention_weight_min": -45.56708908081055,
      "epoch": 16.27,
      "learning_rate": 4.5559848484848484e-05,
      "loss": 2.739,
      "step": 279950
    },
    {
      "activations/layer0_attention_weight_max": 15.16532039642334,
      "activations/layer0_attention_weight_min": -13.211410522460938,
      "activations/layer10_attention_weight_max": 40.266292572021484,
      "activations/layer10_attention_weight_min": -37.22900390625,
      "activations/layer11_attention_weight_max": 33.84916687011719,
      "activations/layer11_attention_weight_min": -34.120201110839844,
      "activations/layer12_attention_weight_max": 19.526548385620117,
      "activations/layer12_attention_weight_min": -22.929641723632812,
      "activations/layer13_attention_weight_max": 44.161903381347656,
      "activations/layer13_attention_weight_min": -31.60061264038086,
      "activations/layer14_attention_weight_max": 36.5301628112793,
      "activations/layer14_attention_weight_min": -30.786630630493164,
      "activations/layer15_attention_weight_max": 35.240875244140625,
      "activations/layer15_attention_weight_min": -30.736635208129883,
      "activations/layer16_attention_weight_max": 36.40039825439453,
      "activations/layer16_attention_weight_min": -32.921539306640625,
      "activations/layer17_attention_weight_max": 56.862060546875,
      "activations/layer17_attention_weight_min": -44.901824951171875,
      "activations/layer18_attention_weight_max": 53.06268310546875,
      "activations/layer18_attention_weight_min": -40.48997497558594,
      "activations/layer19_attention_weight_max": 23.282987594604492,
      "activations/layer19_attention_weight_min": -22.305696487426758,
      "activations/layer1_attention_weight_max": 16.30373764038086,
      "activations/layer1_attention_weight_min": -13.913228034973145,
      "activations/layer20_attention_weight_max": 23.09092903137207,
      "activations/layer20_attention_weight_min": -22.515533447265625,
      "activations/layer21_attention_weight_max": 36.79948806762695,
      "activations/layer21_attention_weight_min": -27.075389862060547,
      "activations/layer22_attention_weight_max": 30.43463706970215,
      "activations/layer22_attention_weight_min": -25.94708824157715,
      "activations/layer23_attention_weight_max": 37.42162322998047,
      "activations/layer23_attention_weight_min": -27.644256591796875,
      "activations/layer2_attention_weight_max": 32.459014892578125,
      "activations/layer2_attention_weight_min": -30.767536163330078,
      "activations/layer3_attention_weight_max": 88.20125579833984,
      "activations/layer3_attention_weight_min": -88.6829833984375,
      "activations/layer4_attention_weight_max": 90.77809143066406,
      "activations/layer4_attention_weight_min": -89.92164611816406,
      "activations/layer5_attention_weight_max": 65.78541564941406,
      "activations/layer5_attention_weight_min": -78.30438232421875,
      "activations/layer6_attention_weight_max": 51.73890686035156,
      "activations/layer6_attention_weight_min": -54.28693771362305,
      "activations/layer7_attention_weight_max": 63.615699768066406,
      "activations/layer7_attention_weight_min": -63.029762268066406,
      "activations/layer8_attention_weight_max": 48.613521575927734,
      "activations/layer8_attention_weight_min": -50.6367301940918,
      "activations/layer9_attention_weight_max": 45.121742248535156,
      "activations/layer9_attention_weight_min": -45.36037063598633,
      "epoch": 16.27,
      "learning_rate": 4.5540909090909086e-05,
      "loss": 2.7457,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4576,
      "eval_samples_per_second": 507.709,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4576,
      "eval_openwebtext_samples_per_second": 507.709,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9274,
      "eval_wikitext_samples_per_second": 236.587,
      "step": 280000
    },
    {
      "epoch": 16.27,
      "eval_lambada_loss": 2.50390625,
      "eval_lambada_ppl": 12.230174893930755,
      "eval_lambada_runtime": 9.5105,
      "eval_lambada_samples_per_second": 511.959,
      "step": 280000
    },
    {
      "activations/layer0_attention_weight_max": 15.592367172241211,
      "activations/layer0_attention_weight_min": -13.005983352661133,
      "activations/layer10_attention_weight_max": 33.91484451293945,
      "activations/layer10_attention_weight_min": -34.439510345458984,
      "activations/layer11_attention_weight_max": 31.71211814880371,
      "activations/layer11_attention_weight_min": -34.064605712890625,
      "activations/layer12_attention_weight_max": 19.053390502929688,
      "activations/layer12_attention_weight_min": -25.16643524169922,
      "activations/layer13_attention_weight_max": 33.25621032714844,
      "activations/layer13_attention_weight_min": -26.534513473510742,
      "activations/layer14_attention_weight_max": 35.0848274230957,
      "activations/layer14_attention_weight_min": -30.929977416992188,
      "activations/layer15_attention_weight_max": 30.83746337890625,
      "activations/layer15_attention_weight_min": -30.233535766601562,
      "activations/layer16_attention_weight_max": 31.59153175354004,
      "activations/layer16_attention_weight_min": -30.98328399658203,
      "activations/layer17_attention_weight_max": 48.76830291748047,
      "activations/layer17_attention_weight_min": -43.78264236450195,
      "activations/layer18_attention_weight_max": 44.82434844970703,
      "activations/layer18_attention_weight_min": -39.06694412231445,
      "activations/layer19_attention_weight_max": 23.040006637573242,
      "activations/layer19_attention_weight_min": -22.875524520874023,
      "activations/layer1_attention_weight_max": 16.387428283691406,
      "activations/layer1_attention_weight_min": -15.342605590820312,
      "activations/layer20_attention_weight_max": 21.12993049621582,
      "activations/layer20_attention_weight_min": -24.286273956298828,
      "activations/layer21_attention_weight_max": 31.317745208740234,
      "activations/layer21_attention_weight_min": -22.971237182617188,
      "activations/layer22_attention_weight_max": 26.80744743347168,
      "activations/layer22_attention_weight_min": -26.482421875,
      "activations/layer23_attention_weight_max": 33.468788146972656,
      "activations/layer23_attention_weight_min": -26.21477508544922,
      "activations/layer2_attention_weight_max": 31.406736373901367,
      "activations/layer2_attention_weight_min": -29.480510711669922,
      "activations/layer3_attention_weight_max": 84.22623443603516,
      "activations/layer3_attention_weight_min": -87.54666137695312,
      "activations/layer4_attention_weight_max": 86.69414520263672,
      "activations/layer4_attention_weight_min": -85.56608581542969,
      "activations/layer5_attention_weight_max": 68.9166030883789,
      "activations/layer5_attention_weight_min": -68.85124206542969,
      "activations/layer6_attention_weight_max": 50.283206939697266,
      "activations/layer6_attention_weight_min": -52.63404846191406,
      "activations/layer7_attention_weight_max": 65.0389404296875,
      "activations/layer7_attention_weight_min": -59.28258514404297,
      "activations/layer8_attention_weight_max": 43.2275390625,
      "activations/layer8_attention_weight_min": -45.300113677978516,
      "activations/layer9_attention_weight_max": 43.002742767333984,
      "activations/layer9_attention_weight_min": -43.071292877197266,
      "epoch": 16.27,
      "learning_rate": 4.552196969696969e-05,
      "loss": 2.7493,
      "step": 280050
    },
    {
      "activations/layer0_attention_weight_max": 16.275066375732422,
      "activations/layer0_attention_weight_min": -13.578460693359375,
      "activations/layer10_attention_weight_max": 36.72283935546875,
      "activations/layer10_attention_weight_min": -36.67301940917969,
      "activations/layer11_attention_weight_max": 31.873462677001953,
      "activations/layer11_attention_weight_min": -35.28141403198242,
      "activations/layer12_attention_weight_max": 18.33122444152832,
      "activations/layer12_attention_weight_min": -26.36286163330078,
      "activations/layer13_attention_weight_max": 36.45718002319336,
      "activations/layer13_attention_weight_min": -27.415523529052734,
      "activations/layer14_attention_weight_max": 35.10037612915039,
      "activations/layer14_attention_weight_min": -29.268529891967773,
      "activations/layer15_attention_weight_max": 33.94290542602539,
      "activations/layer15_attention_weight_min": -29.562082290649414,
      "activations/layer16_attention_weight_max": 30.76573944091797,
      "activations/layer16_attention_weight_min": -30.866857528686523,
      "activations/layer17_attention_weight_max": 49.88052749633789,
      "activations/layer17_attention_weight_min": -44.335693359375,
      "activations/layer18_attention_weight_max": 44.46709442138672,
      "activations/layer18_attention_weight_min": -39.04198455810547,
      "activations/layer19_attention_weight_max": 22.703006744384766,
      "activations/layer19_attention_weight_min": -22.791120529174805,
      "activations/layer1_attention_weight_max": 15.920700073242188,
      "activations/layer1_attention_weight_min": -13.765555381774902,
      "activations/layer20_attention_weight_max": 23.236743927001953,
      "activations/layer20_attention_weight_min": -23.6951961517334,
      "activations/layer21_attention_weight_max": 35.33504104614258,
      "activations/layer21_attention_weight_min": -20.90287208557129,
      "activations/layer22_attention_weight_max": 28.13432502746582,
      "activations/layer22_attention_weight_min": -26.174470901489258,
      "activations/layer23_attention_weight_max": 34.07817840576172,
      "activations/layer23_attention_weight_min": -25.226280212402344,
      "activations/layer2_attention_weight_max": 32.05018997192383,
      "activations/layer2_attention_weight_min": -30.416959762573242,
      "activations/layer3_attention_weight_max": 88.30425262451172,
      "activations/layer3_attention_weight_min": -90.39022827148438,
      "activations/layer4_attention_weight_max": 85.9641342163086,
      "activations/layer4_attention_weight_min": -87.4094009399414,
      "activations/layer5_attention_weight_max": 60.24989318847656,
      "activations/layer5_attention_weight_min": -70.76263427734375,
      "activations/layer6_attention_weight_max": 52.0034065246582,
      "activations/layer6_attention_weight_min": -54.55434799194336,
      "activations/layer7_attention_weight_max": 59.32728958129883,
      "activations/layer7_attention_weight_min": -59.375877380371094,
      "activations/layer8_attention_weight_max": 46.02625274658203,
      "activations/layer8_attention_weight_min": -50.744327545166016,
      "activations/layer9_attention_weight_max": 43.375389099121094,
      "activations/layer9_attention_weight_min": -46.61086654663086,
      "epoch": 16.28,
      "learning_rate": 4.55030303030303e-05,
      "loss": 2.7387,
      "step": 280100
    },
    {
      "activations/layer0_attention_weight_max": 15.62543773651123,
      "activations/layer0_attention_weight_min": -13.206344604492188,
      "activations/layer10_attention_weight_max": 35.72154998779297,
      "activations/layer10_attention_weight_min": -35.11988830566406,
      "activations/layer11_attention_weight_max": 31.800405502319336,
      "activations/layer11_attention_weight_min": -34.421363830566406,
      "activations/layer12_attention_weight_max": 18.03315544128418,
      "activations/layer12_attention_weight_min": -26.134220123291016,
      "activations/layer13_attention_weight_max": 33.00764083862305,
      "activations/layer13_attention_weight_min": -27.02581024169922,
      "activations/layer14_attention_weight_max": 31.503395080566406,
      "activations/layer14_attention_weight_min": -29.977176666259766,
      "activations/layer15_attention_weight_max": 30.598247528076172,
      "activations/layer15_attention_weight_min": -29.75625991821289,
      "activations/layer16_attention_weight_max": 30.782869338989258,
      "activations/layer16_attention_weight_min": -30.61981964111328,
      "activations/layer17_attention_weight_max": 49.8072624206543,
      "activations/layer17_attention_weight_min": -44.50566482543945,
      "activations/layer18_attention_weight_max": 43.05953598022461,
      "activations/layer18_attention_weight_min": -37.90727996826172,
      "activations/layer19_attention_weight_max": 23.9881591796875,
      "activations/layer19_attention_weight_min": -21.957399368286133,
      "activations/layer1_attention_weight_max": 16.961360931396484,
      "activations/layer1_attention_weight_min": -13.688077926635742,
      "activations/layer20_attention_weight_max": 21.371686935424805,
      "activations/layer20_attention_weight_min": -23.257709503173828,
      "activations/layer21_attention_weight_max": 32.804935455322266,
      "activations/layer21_attention_weight_min": -21.140151977539062,
      "activations/layer22_attention_weight_max": 30.068939208984375,
      "activations/layer22_attention_weight_min": -26.25495147705078,
      "activations/layer23_attention_weight_max": 33.308372497558594,
      "activations/layer23_attention_weight_min": -22.487274169921875,
      "activations/layer2_attention_weight_max": 31.949567794799805,
      "activations/layer2_attention_weight_min": -28.69776725769043,
      "activations/layer3_attention_weight_max": 85.90919494628906,
      "activations/layer3_attention_weight_min": -81.9742431640625,
      "activations/layer4_attention_weight_max": 87.05354309082031,
      "activations/layer4_attention_weight_min": -83.52413177490234,
      "activations/layer5_attention_weight_max": 63.530784606933594,
      "activations/layer5_attention_weight_min": -68.97332763671875,
      "activations/layer6_attention_weight_max": 51.00941467285156,
      "activations/layer6_attention_weight_min": -50.85252380371094,
      "activations/layer7_attention_weight_max": 63.125762939453125,
      "activations/layer7_attention_weight_min": -59.166114807128906,
      "activations/layer8_attention_weight_max": 43.416114807128906,
      "activations/layer8_attention_weight_min": -45.50761795043945,
      "activations/layer9_attention_weight_max": 41.0882682800293,
      "activations/layer9_attention_weight_min": -42.6533317565918,
      "epoch": 16.28,
      "learning_rate": 4.5484090909090904e-05,
      "loss": 2.754,
      "step": 280150
    },
    {
      "activations/layer0_attention_weight_max": 16.790674209594727,
      "activations/layer0_attention_weight_min": -13.52055835723877,
      "activations/layer10_attention_weight_max": 38.19392013549805,
      "activations/layer10_attention_weight_min": -37.32423400878906,
      "activations/layer11_attention_weight_max": 35.10206604003906,
      "activations/layer11_attention_weight_min": -33.74174118041992,
      "activations/layer12_attention_weight_max": 20.669071197509766,
      "activations/layer12_attention_weight_min": -26.31251335144043,
      "activations/layer13_attention_weight_max": 41.1606330871582,
      "activations/layer13_attention_weight_min": -27.618188858032227,
      "activations/layer14_attention_weight_max": 41.28008270263672,
      "activations/layer14_attention_weight_min": -30.758935928344727,
      "activations/layer15_attention_weight_max": 37.809940338134766,
      "activations/layer15_attention_weight_min": -30.337812423706055,
      "activations/layer16_attention_weight_max": 35.84160232543945,
      "activations/layer16_attention_weight_min": -33.19171905517578,
      "activations/layer17_attention_weight_max": 55.08803939819336,
      "activations/layer17_attention_weight_min": -46.19425582885742,
      "activations/layer18_attention_weight_max": 51.81387710571289,
      "activations/layer18_attention_weight_min": -41.231040954589844,
      "activations/layer19_attention_weight_max": 28.69962501525879,
      "activations/layer19_attention_weight_min": -24.577190399169922,
      "activations/layer1_attention_weight_max": 16.2840633392334,
      "activations/layer1_attention_weight_min": -14.812326431274414,
      "activations/layer20_attention_weight_max": 25.981420516967773,
      "activations/layer20_attention_weight_min": -22.99376106262207,
      "activations/layer21_attention_weight_max": 41.758888244628906,
      "activations/layer21_attention_weight_min": -24.45427703857422,
      "activations/layer22_attention_weight_max": 39.0247917175293,
      "activations/layer22_attention_weight_min": -26.554935455322266,
      "activations/layer23_attention_weight_max": 39.302616119384766,
      "activations/layer23_attention_weight_min": -24.33169937133789,
      "activations/layer2_attention_weight_max": 29.423316955566406,
      "activations/layer2_attention_weight_min": -28.724782943725586,
      "activations/layer3_attention_weight_max": 85.18799591064453,
      "activations/layer3_attention_weight_min": -87.3879623413086,
      "activations/layer4_attention_weight_max": 87.08619689941406,
      "activations/layer4_attention_weight_min": -92.38348388671875,
      "activations/layer5_attention_weight_max": 64.11786651611328,
      "activations/layer5_attention_weight_min": -68.23114013671875,
      "activations/layer6_attention_weight_max": 48.26873016357422,
      "activations/layer6_attention_weight_min": -50.76036071777344,
      "activations/layer7_attention_weight_max": 62.066165924072266,
      "activations/layer7_attention_weight_min": -64.2861099243164,
      "activations/layer8_attention_weight_max": 45.3157958984375,
      "activations/layer8_attention_weight_min": -48.914451599121094,
      "activations/layer9_attention_weight_max": 44.87946319580078,
      "activations/layer9_attention_weight_min": -46.071781158447266,
      "epoch": 16.28,
      "learning_rate": 4.546515151515151e-05,
      "loss": 2.7491,
      "step": 280200
    },
    {
      "activations/layer0_attention_weight_max": 14.977204322814941,
      "activations/layer0_attention_weight_min": -13.100406646728516,
      "activations/layer10_attention_weight_max": 35.2958984375,
      "activations/layer10_attention_weight_min": -35.27100372314453,
      "activations/layer11_attention_weight_max": 31.85405731201172,
      "activations/layer11_attention_weight_min": -32.21965408325195,
      "activations/layer12_attention_weight_max": 20.018430709838867,
      "activations/layer12_attention_weight_min": -24.941537857055664,
      "activations/layer13_attention_weight_max": 31.00010108947754,
      "activations/layer13_attention_weight_min": -27.022706985473633,
      "activations/layer14_attention_weight_max": 34.188777923583984,
      "activations/layer14_attention_weight_min": -31.7545223236084,
      "activations/layer15_attention_weight_max": 31.794586181640625,
      "activations/layer15_attention_weight_min": -31.06476402282715,
      "activations/layer16_attention_weight_max": 31.95182228088379,
      "activations/layer16_attention_weight_min": -31.054264068603516,
      "activations/layer17_attention_weight_max": 51.272762298583984,
      "activations/layer17_attention_weight_min": -44.69498825073242,
      "activations/layer18_attention_weight_max": 48.78648376464844,
      "activations/layer18_attention_weight_min": -40.67856979370117,
      "activations/layer19_attention_weight_max": 26.386137008666992,
      "activations/layer19_attention_weight_min": -24.575910568237305,
      "activations/layer1_attention_weight_max": 16.318883895874023,
      "activations/layer1_attention_weight_min": -15.815560340881348,
      "activations/layer20_attention_weight_max": 26.0150089263916,
      "activations/layer20_attention_weight_min": -20.876617431640625,
      "activations/layer21_attention_weight_max": 36.64093780517578,
      "activations/layer21_attention_weight_min": -22.879919052124023,
      "activations/layer22_attention_weight_max": 31.897207260131836,
      "activations/layer22_attention_weight_min": -29.362852096557617,
      "activations/layer23_attention_weight_max": 38.049015045166016,
      "activations/layer23_attention_weight_min": -23.176612854003906,
      "activations/layer2_attention_weight_max": 31.375194549560547,
      "activations/layer2_attention_weight_min": -29.452428817749023,
      "activations/layer3_attention_weight_max": 86.23102569580078,
      "activations/layer3_attention_weight_min": -86.00885009765625,
      "activations/layer4_attention_weight_max": 88.97691345214844,
      "activations/layer4_attention_weight_min": -85.00121307373047,
      "activations/layer5_attention_weight_max": 64.8611068725586,
      "activations/layer5_attention_weight_min": -75.02426147460938,
      "activations/layer6_attention_weight_max": 49.838077545166016,
      "activations/layer6_attention_weight_min": -51.053462982177734,
      "activations/layer7_attention_weight_max": 62.7850341796875,
      "activations/layer7_attention_weight_min": -60.57150650024414,
      "activations/layer8_attention_weight_max": 42.749267578125,
      "activations/layer8_attention_weight_min": -47.965274810791016,
      "activations/layer9_attention_weight_max": 42.494300842285156,
      "activations/layer9_attention_weight_min": -43.24626922607422,
      "epoch": 16.28,
      "learning_rate": 4.5446212121212114e-05,
      "loss": 2.7629,
      "step": 280250
    },
    {
      "activations/layer0_attention_weight_max": 15.622004508972168,
      "activations/layer0_attention_weight_min": -13.515914916992188,
      "activations/layer10_attention_weight_max": 36.431884765625,
      "activations/layer10_attention_weight_min": -37.4395637512207,
      "activations/layer11_attention_weight_max": 33.99574661254883,
      "activations/layer11_attention_weight_min": -34.662174224853516,
      "activations/layer12_attention_weight_max": 17.427539825439453,
      "activations/layer12_attention_weight_min": -27.285131454467773,
      "activations/layer13_attention_weight_max": 32.52180480957031,
      "activations/layer13_attention_weight_min": -27.915077209472656,
      "activations/layer14_attention_weight_max": 34.84650421142578,
      "activations/layer14_attention_weight_min": -29.462053298950195,
      "activations/layer15_attention_weight_max": 31.284635543823242,
      "activations/layer15_attention_weight_min": -29.131528854370117,
      "activations/layer16_attention_weight_max": 31.454069137573242,
      "activations/layer16_attention_weight_min": -31.22334861755371,
      "activations/layer17_attention_weight_max": 52.909549713134766,
      "activations/layer17_attention_weight_min": -43.83760452270508,
      "activations/layer18_attention_weight_max": 43.02621841430664,
      "activations/layer18_attention_weight_min": -38.57653045654297,
      "activations/layer19_attention_weight_max": 21.744014739990234,
      "activations/layer19_attention_weight_min": -22.198698043823242,
      "activations/layer1_attention_weight_max": 16.799354553222656,
      "activations/layer1_attention_weight_min": -14.985664367675781,
      "activations/layer20_attention_weight_max": 20.580766677856445,
      "activations/layer20_attention_weight_min": -21.794620513916016,
      "activations/layer21_attention_weight_max": 33.734466552734375,
      "activations/layer21_attention_weight_min": -26.851329803466797,
      "activations/layer22_attention_weight_max": 29.44896697998047,
      "activations/layer22_attention_weight_min": -26.43594741821289,
      "activations/layer23_attention_weight_max": 36.787628173828125,
      "activations/layer23_attention_weight_min": -24.67487144470215,
      "activations/layer2_attention_weight_max": 31.07040023803711,
      "activations/layer2_attention_weight_min": -31.10886001586914,
      "activations/layer3_attention_weight_max": 89.19168090820312,
      "activations/layer3_attention_weight_min": -92.04579162597656,
      "activations/layer4_attention_weight_max": 89.77471160888672,
      "activations/layer4_attention_weight_min": -87.40543365478516,
      "activations/layer5_attention_weight_max": 62.46263122558594,
      "activations/layer5_attention_weight_min": -71.20146179199219,
      "activations/layer6_attention_weight_max": 48.80856704711914,
      "activations/layer6_attention_weight_min": -51.69635772705078,
      "activations/layer7_attention_weight_max": 63.29440689086914,
      "activations/layer7_attention_weight_min": -61.390045166015625,
      "activations/layer8_attention_weight_max": 45.25627517700195,
      "activations/layer8_attention_weight_min": -48.877567291259766,
      "activations/layer9_attention_weight_max": 42.69182205200195,
      "activations/layer9_attention_weight_min": -44.648719787597656,
      "epoch": 16.29,
      "learning_rate": 4.542727272727273e-05,
      "loss": 2.7474,
      "step": 280300
    },
    {
      "activations/layer0_attention_weight_max": 15.0120267868042,
      "activations/layer0_attention_weight_min": -12.873764038085938,
      "activations/layer10_attention_weight_max": 33.35749435424805,
      "activations/layer10_attention_weight_min": -32.67325210571289,
      "activations/layer11_attention_weight_max": 31.58934783935547,
      "activations/layer11_attention_weight_min": -32.79155349731445,
      "activations/layer12_attention_weight_max": 19.713682174682617,
      "activations/layer12_attention_weight_min": -24.3807315826416,
      "activations/layer13_attention_weight_max": 33.827816009521484,
      "activations/layer13_attention_weight_min": -32.175071716308594,
      "activations/layer14_attention_weight_max": 37.393741607666016,
      "activations/layer14_attention_weight_min": -31.88251495361328,
      "activations/layer15_attention_weight_max": 37.639549255371094,
      "activations/layer15_attention_weight_min": -31.940797805786133,
      "activations/layer16_attention_weight_max": 34.434661865234375,
      "activations/layer16_attention_weight_min": -32.60847854614258,
      "activations/layer17_attention_weight_max": 54.7021369934082,
      "activations/layer17_attention_weight_min": -48.96668243408203,
      "activations/layer18_attention_weight_max": 53.39540481567383,
      "activations/layer18_attention_weight_min": -44.22956085205078,
      "activations/layer19_attention_weight_max": 28.284948348999023,
      "activations/layer19_attention_weight_min": -25.353952407836914,
      "activations/layer1_attention_weight_max": 15.3991117477417,
      "activations/layer1_attention_weight_min": -13.475031852722168,
      "activations/layer20_attention_weight_max": 27.19573211669922,
      "activations/layer20_attention_weight_min": -24.07304573059082,
      "activations/layer21_attention_weight_max": 41.35845184326172,
      "activations/layer21_attention_weight_min": -23.387802124023438,
      "activations/layer22_attention_weight_max": 33.435359954833984,
      "activations/layer22_attention_weight_min": -28.53962516784668,
      "activations/layer23_attention_weight_max": 40.62611389160156,
      "activations/layer23_attention_weight_min": -27.10848045349121,
      "activations/layer2_attention_weight_max": 29.31976318359375,
      "activations/layer2_attention_weight_min": -27.545902252197266,
      "activations/layer3_attention_weight_max": 83.26113891601562,
      "activations/layer3_attention_weight_min": -85.36145782470703,
      "activations/layer4_attention_weight_max": 85.25226593017578,
      "activations/layer4_attention_weight_min": -84.27023315429688,
      "activations/layer5_attention_weight_max": 61.174827575683594,
      "activations/layer5_attention_weight_min": -71.53472137451172,
      "activations/layer6_attention_weight_max": 50.24533462524414,
      "activations/layer6_attention_weight_min": -47.68016815185547,
      "activations/layer7_attention_weight_max": 61.96441650390625,
      "activations/layer7_attention_weight_min": -64.60597229003906,
      "activations/layer8_attention_weight_max": 42.12771224975586,
      "activations/layer8_attention_weight_min": -45.59004592895508,
      "activations/layer9_attention_weight_max": 41.38296127319336,
      "activations/layer9_attention_weight_min": -43.49711608886719,
      "epoch": 16.29,
      "learning_rate": 4.540833333333333e-05,
      "loss": 2.7431,
      "step": 280350
    },
    {
      "activations/layer0_attention_weight_max": 14.604900360107422,
      "activations/layer0_attention_weight_min": -13.052489280700684,
      "activations/layer10_attention_weight_max": 34.168861389160156,
      "activations/layer10_attention_weight_min": -33.9824333190918,
      "activations/layer11_attention_weight_max": 31.307266235351562,
      "activations/layer11_attention_weight_min": -32.8693962097168,
      "activations/layer12_attention_weight_max": 17.65411376953125,
      "activations/layer12_attention_weight_min": -21.171655654907227,
      "activations/layer13_attention_weight_max": 31.60527992248535,
      "activations/layer13_attention_weight_min": -27.50828742980957,
      "activations/layer14_attention_weight_max": 36.167015075683594,
      "activations/layer14_attention_weight_min": -29.45402717590332,
      "activations/layer15_attention_weight_max": 32.64006423950195,
      "activations/layer15_attention_weight_min": -30.134977340698242,
      "activations/layer16_attention_weight_max": 33.15415573120117,
      "activations/layer16_attention_weight_min": -31.209156036376953,
      "activations/layer17_attention_weight_max": 49.53904342651367,
      "activations/layer17_attention_weight_min": -43.527496337890625,
      "activations/layer18_attention_weight_max": 47.48567199707031,
      "activations/layer18_attention_weight_min": -37.82081985473633,
      "activations/layer19_attention_weight_max": 25.102558135986328,
      "activations/layer19_attention_weight_min": -21.17551612854004,
      "activations/layer1_attention_weight_max": 15.662409782409668,
      "activations/layer1_attention_weight_min": -15.259678840637207,
      "activations/layer20_attention_weight_max": 24.35028839111328,
      "activations/layer20_attention_weight_min": -20.123268127441406,
      "activations/layer21_attention_weight_max": 34.41348648071289,
      "activations/layer21_attention_weight_min": -22.861413955688477,
      "activations/layer22_attention_weight_max": 29.192651748657227,
      "activations/layer22_attention_weight_min": -24.19259262084961,
      "activations/layer23_attention_weight_max": 40.56413650512695,
      "activations/layer23_attention_weight_min": -24.74237823486328,
      "activations/layer2_attention_weight_max": 30.18254852294922,
      "activations/layer2_attention_weight_min": -28.22755241394043,
      "activations/layer3_attention_weight_max": 85.45674133300781,
      "activations/layer3_attention_weight_min": -84.17339324951172,
      "activations/layer4_attention_weight_max": 89.45884704589844,
      "activations/layer4_attention_weight_min": -89.09113311767578,
      "activations/layer5_attention_weight_max": 63.25498580932617,
      "activations/layer5_attention_weight_min": -74.11096954345703,
      "activations/layer6_attention_weight_max": 49.51050567626953,
      "activations/layer6_attention_weight_min": -51.19289016723633,
      "activations/layer7_attention_weight_max": 61.85392761230469,
      "activations/layer7_attention_weight_min": -63.19634246826172,
      "activations/layer8_attention_weight_max": 44.64331817626953,
      "activations/layer8_attention_weight_min": -45.721763610839844,
      "activations/layer9_attention_weight_max": 42.19468307495117,
      "activations/layer9_attention_weight_min": -43.841678619384766,
      "epoch": 16.29,
      "learning_rate": 4.538977272727272e-05,
      "loss": 2.7564,
      "step": 280400
    },
    {
      "activations/layer0_attention_weight_max": 15.79145622253418,
      "activations/layer0_attention_weight_min": -12.99866771697998,
      "activations/layer10_attention_weight_max": 35.747135162353516,
      "activations/layer10_attention_weight_min": -36.32953643798828,
      "activations/layer11_attention_weight_max": 31.896915435791016,
      "activations/layer11_attention_weight_min": -32.286170959472656,
      "activations/layer12_attention_weight_max": 18.14912986755371,
      "activations/layer12_attention_weight_min": -25.435461044311523,
      "activations/layer13_attention_weight_max": 32.891963958740234,
      "activations/layer13_attention_weight_min": -27.795869827270508,
      "activations/layer14_attention_weight_max": 36.82377243041992,
      "activations/layer14_attention_weight_min": -32.2896614074707,
      "activations/layer15_attention_weight_max": 33.933013916015625,
      "activations/layer15_attention_weight_min": -31.008567810058594,
      "activations/layer16_attention_weight_max": 35.547359466552734,
      "activations/layer16_attention_weight_min": -35.531131744384766,
      "activations/layer17_attention_weight_max": 59.24260330200195,
      "activations/layer17_attention_weight_min": -48.76701736450195,
      "activations/layer18_attention_weight_max": 52.11299514770508,
      "activations/layer18_attention_weight_min": -41.7555046081543,
      "activations/layer19_attention_weight_max": 23.427595138549805,
      "activations/layer19_attention_weight_min": -22.500106811523438,
      "activations/layer1_attention_weight_max": 16.29773712158203,
      "activations/layer1_attention_weight_min": -14.537981986999512,
      "activations/layer20_attention_weight_max": 22.965221405029297,
      "activations/layer20_attention_weight_min": -20.490873336791992,
      "activations/layer21_attention_weight_max": 39.26511001586914,
      "activations/layer21_attention_weight_min": -21.87953758239746,
      "activations/layer22_attention_weight_max": 31.739255905151367,
      "activations/layer22_attention_weight_min": -26.95463752746582,
      "activations/layer23_attention_weight_max": 34.893524169921875,
      "activations/layer23_attention_weight_min": -24.569454193115234,
      "activations/layer2_attention_weight_max": 29.845809936523438,
      "activations/layer2_attention_weight_min": -29.143795013427734,
      "activations/layer3_attention_weight_max": 84.68014526367188,
      "activations/layer3_attention_weight_min": -84.5805892944336,
      "activations/layer4_attention_weight_max": 88.0301742553711,
      "activations/layer4_attention_weight_min": -88.07127380371094,
      "activations/layer5_attention_weight_max": 69.10244750976562,
      "activations/layer5_attention_weight_min": -74.2874755859375,
      "activations/layer6_attention_weight_max": 49.82973098754883,
      "activations/layer6_attention_weight_min": -51.71393585205078,
      "activations/layer7_attention_weight_max": 63.441444396972656,
      "activations/layer7_attention_weight_min": -59.50731658935547,
      "activations/layer8_attention_weight_max": 46.82915115356445,
      "activations/layer8_attention_weight_min": -49.34310531616211,
      "activations/layer9_attention_weight_max": 44.02054214477539,
      "activations/layer9_attention_weight_min": -45.35051345825195,
      "epoch": 16.3,
      "learning_rate": 4.537083333333332e-05,
      "loss": 2.7481,
      "step": 280450
    },
    {
      "activations/layer0_attention_weight_max": 14.655661582946777,
      "activations/layer0_attention_weight_min": -13.244089126586914,
      "activations/layer10_attention_weight_max": 34.533592224121094,
      "activations/layer10_attention_weight_min": -37.1526985168457,
      "activations/layer11_attention_weight_max": 30.103919982910156,
      "activations/layer11_attention_weight_min": -34.438316345214844,
      "activations/layer12_attention_weight_max": 18.37996482849121,
      "activations/layer12_attention_weight_min": -25.943885803222656,
      "activations/layer13_attention_weight_max": 32.41288757324219,
      "activations/layer13_attention_weight_min": -28.627092361450195,
      "activations/layer14_attention_weight_max": 33.13591766357422,
      "activations/layer14_attention_weight_min": -30.713401794433594,
      "activations/layer15_attention_weight_max": 30.64845085144043,
      "activations/layer15_attention_weight_min": -31.80931282043457,
      "activations/layer16_attention_weight_max": 31.63707160949707,
      "activations/layer16_attention_weight_min": -31.426481246948242,
      "activations/layer17_attention_weight_max": 49.66153335571289,
      "activations/layer17_attention_weight_min": -44.28696060180664,
      "activations/layer18_attention_weight_max": 46.297725677490234,
      "activations/layer18_attention_weight_min": -39.221832275390625,
      "activations/layer19_attention_weight_max": 21.834022521972656,
      "activations/layer19_attention_weight_min": -23.4561824798584,
      "activations/layer1_attention_weight_max": 15.933494567871094,
      "activations/layer1_attention_weight_min": -12.74782657623291,
      "activations/layer20_attention_weight_max": 22.196971893310547,
      "activations/layer20_attention_weight_min": -23.229841232299805,
      "activations/layer21_attention_weight_max": 36.56538772583008,
      "activations/layer21_attention_weight_min": -23.746971130371094,
      "activations/layer22_attention_weight_max": 29.669593811035156,
      "activations/layer22_attention_weight_min": -28.401941299438477,
      "activations/layer23_attention_weight_max": 36.36820602416992,
      "activations/layer23_attention_weight_min": -24.99877166748047,
      "activations/layer2_attention_weight_max": 30.51137924194336,
      "activations/layer2_attention_weight_min": -30.625545501708984,
      "activations/layer3_attention_weight_max": 89.12763977050781,
      "activations/layer3_attention_weight_min": -93.22752380371094,
      "activations/layer4_attention_weight_max": 91.5119857788086,
      "activations/layer4_attention_weight_min": -86.62609100341797,
      "activations/layer5_attention_weight_max": 64.97317504882812,
      "activations/layer5_attention_weight_min": -76.66123962402344,
      "activations/layer6_attention_weight_max": 51.939857482910156,
      "activations/layer6_attention_weight_min": -49.53285217285156,
      "activations/layer7_attention_weight_max": 66.14458465576172,
      "activations/layer7_attention_weight_min": -60.74372482299805,
      "activations/layer8_attention_weight_max": 50.420738220214844,
      "activations/layer8_attention_weight_min": -48.117252349853516,
      "activations/layer9_attention_weight_max": 43.037498474121094,
      "activations/layer9_attention_weight_min": -44.43852615356445,
      "epoch": 16.3,
      "learning_rate": 4.535189393939394e-05,
      "loss": 2.7498,
      "step": 280500
    },
    {
      "activations/layer0_attention_weight_max": 14.933982849121094,
      "activations/layer0_attention_weight_min": -13.353282928466797,
      "activations/layer10_attention_weight_max": 33.55971145629883,
      "activations/layer10_attention_weight_min": -33.64407730102539,
      "activations/layer11_attention_weight_max": 33.10485076904297,
      "activations/layer11_attention_weight_min": -32.14014434814453,
      "activations/layer12_attention_weight_max": 16.94980239868164,
      "activations/layer12_attention_weight_min": -24.002283096313477,
      "activations/layer13_attention_weight_max": 33.14244842529297,
      "activations/layer13_attention_weight_min": -28.27305030822754,
      "activations/layer14_attention_weight_max": 32.105796813964844,
      "activations/layer14_attention_weight_min": -30.238981246948242,
      "activations/layer15_attention_weight_max": 28.875028610229492,
      "activations/layer15_attention_weight_min": -31.089696884155273,
      "activations/layer16_attention_weight_max": 29.471349716186523,
      "activations/layer16_attention_weight_min": -29.137863159179688,
      "activations/layer17_attention_weight_max": 47.49064636230469,
      "activations/layer17_attention_weight_min": -42.91798782348633,
      "activations/layer18_attention_weight_max": 44.2315788269043,
      "activations/layer18_attention_weight_min": -38.10460662841797,
      "activations/layer19_attention_weight_max": 22.625015258789062,
      "activations/layer19_attention_weight_min": -23.847291946411133,
      "activations/layer1_attention_weight_max": 15.24096393585205,
      "activations/layer1_attention_weight_min": -13.083681106567383,
      "activations/layer20_attention_weight_max": 22.724319458007812,
      "activations/layer20_attention_weight_min": -27.134536743164062,
      "activations/layer21_attention_weight_max": 36.19427490234375,
      "activations/layer21_attention_weight_min": -24.017602920532227,
      "activations/layer22_attention_weight_max": 26.538850784301758,
      "activations/layer22_attention_weight_min": -24.96225929260254,
      "activations/layer23_attention_weight_max": 33.3045654296875,
      "activations/layer23_attention_weight_min": -25.489002227783203,
      "activations/layer2_attention_weight_max": 29.715755462646484,
      "activations/layer2_attention_weight_min": -28.370290756225586,
      "activations/layer3_attention_weight_max": 85.1872329711914,
      "activations/layer3_attention_weight_min": -84.85069274902344,
      "activations/layer4_attention_weight_max": 86.07067108154297,
      "activations/layer4_attention_weight_min": -86.15345764160156,
      "activations/layer5_attention_weight_max": 61.73325729370117,
      "activations/layer5_attention_weight_min": -71.51948547363281,
      "activations/layer6_attention_weight_max": 51.12629318237305,
      "activations/layer6_attention_weight_min": -50.79431915283203,
      "activations/layer7_attention_weight_max": 61.029117584228516,
      "activations/layer7_attention_weight_min": -60.52770233154297,
      "activations/layer8_attention_weight_max": 45.436397552490234,
      "activations/layer8_attention_weight_min": -47.190582275390625,
      "activations/layer9_attention_weight_max": 43.26626205444336,
      "activations/layer9_attention_weight_min": -44.1690788269043,
      "epoch": 16.3,
      "learning_rate": 4.533295454545454e-05,
      "loss": 2.7347,
      "step": 280550
    },
    {
      "activations/layer0_attention_weight_max": 15.431143760681152,
      "activations/layer0_attention_weight_min": -13.543716430664062,
      "activations/layer10_attention_weight_max": 35.910743713378906,
      "activations/layer10_attention_weight_min": -33.655277252197266,
      "activations/layer11_attention_weight_max": 32.279449462890625,
      "activations/layer11_attention_weight_min": -31.604902267456055,
      "activations/layer12_attention_weight_max": 17.675718307495117,
      "activations/layer12_attention_weight_min": -27.413362503051758,
      "activations/layer13_attention_weight_max": 37.683631896972656,
      "activations/layer13_attention_weight_min": -27.05459213256836,
      "activations/layer14_attention_weight_max": 34.14302062988281,
      "activations/layer14_attention_weight_min": -29.291292190551758,
      "activations/layer15_attention_weight_max": 31.44959259033203,
      "activations/layer15_attention_weight_min": -29.184680938720703,
      "activations/layer16_attention_weight_max": 31.69829559326172,
      "activations/layer16_attention_weight_min": -31.13513946533203,
      "activations/layer17_attention_weight_max": 50.533538818359375,
      "activations/layer17_attention_weight_min": -43.7735481262207,
      "activations/layer18_attention_weight_max": 43.840553283691406,
      "activations/layer18_attention_weight_min": -37.23124694824219,
      "activations/layer19_attention_weight_max": 23.78798484802246,
      "activations/layer19_attention_weight_min": -22.032522201538086,
      "activations/layer1_attention_weight_max": 15.542415618896484,
      "activations/layer1_attention_weight_min": -17.021875381469727,
      "activations/layer20_attention_weight_max": 22.03941535949707,
      "activations/layer20_attention_weight_min": -21.267044067382812,
      "activations/layer21_attention_weight_max": 33.32178497314453,
      "activations/layer21_attention_weight_min": -21.9683780670166,
      "activations/layer22_attention_weight_max": 29.994850158691406,
      "activations/layer22_attention_weight_min": -24.403860092163086,
      "activations/layer23_attention_weight_max": 39.30567169189453,
      "activations/layer23_attention_weight_min": -26.0083065032959,
      "activations/layer2_attention_weight_max": 30.742511749267578,
      "activations/layer2_attention_weight_min": -29.281280517578125,
      "activations/layer3_attention_weight_max": 88.49158477783203,
      "activations/layer3_attention_weight_min": -87.22820281982422,
      "activations/layer4_attention_weight_max": 92.78780364990234,
      "activations/layer4_attention_weight_min": -86.31719970703125,
      "activations/layer5_attention_weight_max": 66.55464172363281,
      "activations/layer5_attention_weight_min": -69.75297546386719,
      "activations/layer6_attention_weight_max": 51.6389274597168,
      "activations/layer6_attention_weight_min": -50.53232192993164,
      "activations/layer7_attention_weight_max": 65.5259780883789,
      "activations/layer7_attention_weight_min": -64.24385833740234,
      "activations/layer8_attention_weight_max": 45.73065948486328,
      "activations/layer8_attention_weight_min": -49.51725387573242,
      "activations/layer9_attention_weight_max": 52.713478088378906,
      "activations/layer9_attention_weight_min": -43.96878433227539,
      "epoch": 16.3,
      "learning_rate": 4.531401515151515e-05,
      "loss": 2.7309,
      "step": 280600
    },
    {
      "activations/layer0_attention_weight_max": 15.171025276184082,
      "activations/layer0_attention_weight_min": -13.554287910461426,
      "activations/layer10_attention_weight_max": 37.38213348388672,
      "activations/layer10_attention_weight_min": -35.86088943481445,
      "activations/layer11_attention_weight_max": 34.42155456542969,
      "activations/layer11_attention_weight_min": -34.680519104003906,
      "activations/layer12_attention_weight_max": 21.26260757446289,
      "activations/layer12_attention_weight_min": -25.044296264648438,
      "activations/layer13_attention_weight_max": 35.538612365722656,
      "activations/layer13_attention_weight_min": -31.56128692626953,
      "activations/layer14_attention_weight_max": 34.62091064453125,
      "activations/layer14_attention_weight_min": -31.856975555419922,
      "activations/layer15_attention_weight_max": 31.999292373657227,
      "activations/layer15_attention_weight_min": -31.16695785522461,
      "activations/layer16_attention_weight_max": 32.00906753540039,
      "activations/layer16_attention_weight_min": -30.87406349182129,
      "activations/layer17_attention_weight_max": 50.29829788208008,
      "activations/layer17_attention_weight_min": -42.00857162475586,
      "activations/layer18_attention_weight_max": 45.24333953857422,
      "activations/layer18_attention_weight_min": -36.80339431762695,
      "activations/layer19_attention_weight_max": 22.90407943725586,
      "activations/layer19_attention_weight_min": -22.04026985168457,
      "activations/layer1_attention_weight_max": 17.47926139831543,
      "activations/layer1_attention_weight_min": -14.664701461791992,
      "activations/layer20_attention_weight_max": 21.874027252197266,
      "activations/layer20_attention_weight_min": -22.174938201904297,
      "activations/layer21_attention_weight_max": 33.16108322143555,
      "activations/layer21_attention_weight_min": -21.935888290405273,
      "activations/layer22_attention_weight_max": 26.70521354675293,
      "activations/layer22_attention_weight_min": -28.27153205871582,
      "activations/layer23_attention_weight_max": 36.435951232910156,
      "activations/layer23_attention_weight_min": -24.298992156982422,
      "activations/layer2_attention_weight_max": 30.224082946777344,
      "activations/layer2_attention_weight_min": -29.69928741455078,
      "activations/layer3_attention_weight_max": 87.42993927001953,
      "activations/layer3_attention_weight_min": -89.24321746826172,
      "activations/layer4_attention_weight_max": 91.70955657958984,
      "activations/layer4_attention_weight_min": -91.06407928466797,
      "activations/layer5_attention_weight_max": 66.67878723144531,
      "activations/layer5_attention_weight_min": -75.74996948242188,
      "activations/layer6_attention_weight_max": 51.72658157348633,
      "activations/layer6_attention_weight_min": -54.2856330871582,
      "activations/layer7_attention_weight_max": 66.25552368164062,
      "activations/layer7_attention_weight_min": -64.27342987060547,
      "activations/layer8_attention_weight_max": 47.79261016845703,
      "activations/layer8_attention_weight_min": -51.0117073059082,
      "activations/layer9_attention_weight_max": 46.70266342163086,
      "activations/layer9_attention_weight_min": -47.97045135498047,
      "epoch": 16.31,
      "learning_rate": 4.529507575757575e-05,
      "loss": 2.7435,
      "step": 280650
    },
    {
      "activations/layer0_attention_weight_max": 16.06475830078125,
      "activations/layer0_attention_weight_min": -12.907832145690918,
      "activations/layer10_attention_weight_max": 36.4236946105957,
      "activations/layer10_attention_weight_min": -35.55246353149414,
      "activations/layer11_attention_weight_max": 32.11768341064453,
      "activations/layer11_attention_weight_min": -34.90533447265625,
      "activations/layer12_attention_weight_max": 19.01754379272461,
      "activations/layer12_attention_weight_min": -28.055622100830078,
      "activations/layer13_attention_weight_max": 33.76176452636719,
      "activations/layer13_attention_weight_min": -26.37018585205078,
      "activations/layer14_attention_weight_max": 37.86779022216797,
      "activations/layer14_attention_weight_min": -29.537811279296875,
      "activations/layer15_attention_weight_max": 34.27053451538086,
      "activations/layer15_attention_weight_min": -30.290834426879883,
      "activations/layer16_attention_weight_max": 36.31480026245117,
      "activations/layer16_attention_weight_min": -34.03055191040039,
      "activations/layer17_attention_weight_max": 52.580078125,
      "activations/layer17_attention_weight_min": -43.55972671508789,
      "activations/layer18_attention_weight_max": 51.02422332763672,
      "activations/layer18_attention_weight_min": -38.62739944458008,
      "activations/layer19_attention_weight_max": 24.617904663085938,
      "activations/layer19_attention_weight_min": -22.22029685974121,
      "activations/layer1_attention_weight_max": 18.138025283813477,
      "activations/layer1_attention_weight_min": -13.980900764465332,
      "activations/layer20_attention_weight_max": 24.311885833740234,
      "activations/layer20_attention_weight_min": -20.847349166870117,
      "activations/layer21_attention_weight_max": 40.48457717895508,
      "activations/layer21_attention_weight_min": -23.04659652709961,
      "activations/layer22_attention_weight_max": 32.545570373535156,
      "activations/layer22_attention_weight_min": -26.23130226135254,
      "activations/layer23_attention_weight_max": 32.577911376953125,
      "activations/layer23_attention_weight_min": -23.041112899780273,
      "activations/layer2_attention_weight_max": 29.97137451171875,
      "activations/layer2_attention_weight_min": -28.4658203125,
      "activations/layer3_attention_weight_max": 83.36842346191406,
      "activations/layer3_attention_weight_min": -81.88796997070312,
      "activations/layer4_attention_weight_max": 86.7828369140625,
      "activations/layer4_attention_weight_min": -89.01704406738281,
      "activations/layer5_attention_weight_max": 62.56743621826172,
      "activations/layer5_attention_weight_min": -75.07373809814453,
      "activations/layer6_attention_weight_max": 54.646812438964844,
      "activations/layer6_attention_weight_min": -52.65824508666992,
      "activations/layer7_attention_weight_max": 65.68999481201172,
      "activations/layer7_attention_weight_min": -63.4857177734375,
      "activations/layer8_attention_weight_max": 42.44691467285156,
      "activations/layer8_attention_weight_min": -48.36137008666992,
      "activations/layer9_attention_weight_max": 42.36713790893555,
      "activations/layer9_attention_weight_min": -45.3064079284668,
      "epoch": 16.31,
      "learning_rate": 4.5276136363636364e-05,
      "loss": 2.7417,
      "step": 280700
    },
    {
      "activations/layer0_attention_weight_max": 15.829808235168457,
      "activations/layer0_attention_weight_min": -13.186140060424805,
      "activations/layer10_attention_weight_max": 33.6752815246582,
      "activations/layer10_attention_weight_min": -33.757354736328125,
      "activations/layer11_attention_weight_max": 30.713455200195312,
      "activations/layer11_attention_weight_min": -32.10557556152344,
      "activations/layer12_attention_weight_max": 17.62289047241211,
      "activations/layer12_attention_weight_min": -25.662643432617188,
      "activations/layer13_attention_weight_max": 37.941951751708984,
      "activations/layer13_attention_weight_min": -28.54520606994629,
      "activations/layer14_attention_weight_max": 33.781009674072266,
      "activations/layer14_attention_weight_min": -29.6241512298584,
      "activations/layer15_attention_weight_max": 30.510379791259766,
      "activations/layer15_attention_weight_min": -29.37104606628418,
      "activations/layer16_attention_weight_max": 31.806238174438477,
      "activations/layer16_attention_weight_min": -30.227306365966797,
      "activations/layer17_attention_weight_max": 48.4220085144043,
      "activations/layer17_attention_weight_min": -42.587467193603516,
      "activations/layer18_attention_weight_max": 44.72582244873047,
      "activations/layer18_attention_weight_min": -43.44196701049805,
      "activations/layer19_attention_weight_max": 23.0037899017334,
      "activations/layer19_attention_weight_min": -24.065576553344727,
      "activations/layer1_attention_weight_max": 18.29686164855957,
      "activations/layer1_attention_weight_min": -14.383599281311035,
      "activations/layer20_attention_weight_max": 22.029499053955078,
      "activations/layer20_attention_weight_min": -22.907943725585938,
      "activations/layer21_attention_weight_max": 29.737577438354492,
      "activations/layer21_attention_weight_min": -23.208335876464844,
      "activations/layer22_attention_weight_max": 26.968917846679688,
      "activations/layer22_attention_weight_min": -26.11025047302246,
      "activations/layer23_attention_weight_max": 33.49192810058594,
      "activations/layer23_attention_weight_min": -23.599576950073242,
      "activations/layer2_attention_weight_max": 28.365278244018555,
      "activations/layer2_attention_weight_min": -27.006664276123047,
      "activations/layer3_attention_weight_max": 82.17170715332031,
      "activations/layer3_attention_weight_min": -80.41205596923828,
      "activations/layer4_attention_weight_max": 84.695068359375,
      "activations/layer4_attention_weight_min": -82.86714935302734,
      "activations/layer5_attention_weight_max": 63.85963439941406,
      "activations/layer5_attention_weight_min": -77.21515655517578,
      "activations/layer6_attention_weight_max": 49.41252899169922,
      "activations/layer6_attention_weight_min": -48.32120132446289,
      "activations/layer7_attention_weight_max": 65.1078872680664,
      "activations/layer7_attention_weight_min": -60.28976821899414,
      "activations/layer8_attention_weight_max": 43.2656135559082,
      "activations/layer8_attention_weight_min": -47.1369514465332,
      "activations/layer9_attention_weight_max": 43.256832122802734,
      "activations/layer9_attention_weight_min": -45.15479278564453,
      "epoch": 16.31,
      "learning_rate": 4.5257196969696965e-05,
      "loss": 2.7493,
      "step": 280750
    },
    {
      "activations/layer0_attention_weight_max": 16.950895309448242,
      "activations/layer0_attention_weight_min": -13.025982856750488,
      "activations/layer10_attention_weight_max": 35.2879524230957,
      "activations/layer10_attention_weight_min": -35.1083984375,
      "activations/layer11_attention_weight_max": 32.94441604614258,
      "activations/layer11_attention_weight_min": -32.52367401123047,
      "activations/layer12_attention_weight_max": 19.477643966674805,
      "activations/layer12_attention_weight_min": -27.445724487304688,
      "activations/layer13_attention_weight_max": 33.28726577758789,
      "activations/layer13_attention_weight_min": -28.208690643310547,
      "activations/layer14_attention_weight_max": 36.44203186035156,
      "activations/layer14_attention_weight_min": -31.17881965637207,
      "activations/layer15_attention_weight_max": 36.79944610595703,
      "activations/layer15_attention_weight_min": -29.7598934173584,
      "activations/layer16_attention_weight_max": 36.45640182495117,
      "activations/layer16_attention_weight_min": -31.137771606445312,
      "activations/layer17_attention_weight_max": 55.146461486816406,
      "activations/layer17_attention_weight_min": -45.53071975708008,
      "activations/layer18_attention_weight_max": 47.86358642578125,
      "activations/layer18_attention_weight_min": -38.27452087402344,
      "activations/layer19_attention_weight_max": 24.0887451171875,
      "activations/layer19_attention_weight_min": -22.124361038208008,
      "activations/layer1_attention_weight_max": 16.553180694580078,
      "activations/layer1_attention_weight_min": -15.78117847442627,
      "activations/layer20_attention_weight_max": 22.440410614013672,
      "activations/layer20_attention_weight_min": -20.371524810791016,
      "activations/layer21_attention_weight_max": 38.3499870300293,
      "activations/layer21_attention_weight_min": -20.119735717773438,
      "activations/layer22_attention_weight_max": 31.4209041595459,
      "activations/layer22_attention_weight_min": -24.84078025817871,
      "activations/layer23_attention_weight_max": 38.87866973876953,
      "activations/layer23_attention_weight_min": -22.745716094970703,
      "activations/layer2_attention_weight_max": 30.288129806518555,
      "activations/layer2_attention_weight_min": -27.987869262695312,
      "activations/layer3_attention_weight_max": 88.1513900756836,
      "activations/layer3_attention_weight_min": -88.06043243408203,
      "activations/layer4_attention_weight_max": 89.6739730834961,
      "activations/layer4_attention_weight_min": -85.68152618408203,
      "activations/layer5_attention_weight_max": 62.35746383666992,
      "activations/layer5_attention_weight_min": -68.83660888671875,
      "activations/layer6_attention_weight_max": 48.53130340576172,
      "activations/layer6_attention_weight_min": -48.67142105102539,
      "activations/layer7_attention_weight_max": 61.47144317626953,
      "activations/layer7_attention_weight_min": -59.5311279296875,
      "activations/layer8_attention_weight_max": 47.33052062988281,
      "activations/layer8_attention_weight_min": -48.99080276489258,
      "activations/layer9_attention_weight_max": 44.23603439331055,
      "activations/layer9_attention_weight_min": -45.73024368286133,
      "epoch": 16.32,
      "learning_rate": 4.523825757575757e-05,
      "loss": 2.7428,
      "step": 280800
    },
    {
      "activations/layer0_attention_weight_max": 15.668521881103516,
      "activations/layer0_attention_weight_min": -13.167619705200195,
      "activations/layer10_attention_weight_max": 37.66230773925781,
      "activations/layer10_attention_weight_min": -36.62166213989258,
      "activations/layer11_attention_weight_max": 36.26911544799805,
      "activations/layer11_attention_weight_min": -35.463748931884766,
      "activations/layer12_attention_weight_max": 18.384052276611328,
      "activations/layer12_attention_weight_min": -23.860538482666016,
      "activations/layer13_attention_weight_max": 36.88648223876953,
      "activations/layer13_attention_weight_min": -30.431711196899414,
      "activations/layer14_attention_weight_max": 41.886688232421875,
      "activations/layer14_attention_weight_min": -30.731292724609375,
      "activations/layer15_attention_weight_max": 40.829586029052734,
      "activations/layer15_attention_weight_min": -30.17765998840332,
      "activations/layer16_attention_weight_max": 34.28158950805664,
      "activations/layer16_attention_weight_min": -31.469575881958008,
      "activations/layer17_attention_weight_max": 52.75248336791992,
      "activations/layer17_attention_weight_min": -47.757225036621094,
      "activations/layer18_attention_weight_max": 50.56456756591797,
      "activations/layer18_attention_weight_min": -41.49360275268555,
      "activations/layer19_attention_weight_max": 26.48334312438965,
      "activations/layer19_attention_weight_min": -24.038631439208984,
      "activations/layer1_attention_weight_max": 16.22627067565918,
      "activations/layer1_attention_weight_min": -15.64124870300293,
      "activations/layer20_attention_weight_max": 23.726848602294922,
      "activations/layer20_attention_weight_min": -21.710573196411133,
      "activations/layer21_attention_weight_max": 36.880592346191406,
      "activations/layer21_attention_weight_min": -24.866985321044922,
      "activations/layer22_attention_weight_max": 32.33197784423828,
      "activations/layer22_attention_weight_min": -25.882673263549805,
      "activations/layer23_attention_weight_max": 37.300655364990234,
      "activations/layer23_attention_weight_min": -24.36542320251465,
      "activations/layer2_attention_weight_max": 31.859291076660156,
      "activations/layer2_attention_weight_min": -31.330833435058594,
      "activations/layer3_attention_weight_max": 91.95435333251953,
      "activations/layer3_attention_weight_min": -95.17930603027344,
      "activations/layer4_attention_weight_max": 90.03797912597656,
      "activations/layer4_attention_weight_min": -90.34932708740234,
      "activations/layer5_attention_weight_max": 64.3037338256836,
      "activations/layer5_attention_weight_min": -76.67982482910156,
      "activations/layer6_attention_weight_max": 51.54169845581055,
      "activations/layer6_attention_weight_min": -53.53669738769531,
      "activations/layer7_attention_weight_max": 61.24852752685547,
      "activations/layer7_attention_weight_min": -64.3755111694336,
      "activations/layer8_attention_weight_max": 47.57748031616211,
      "activations/layer8_attention_weight_min": -47.85883712768555,
      "activations/layer9_attention_weight_max": 52.890254974365234,
      "activations/layer9_attention_weight_min": -45.446598052978516,
      "epoch": 16.32,
      "learning_rate": 4.521931818181818e-05,
      "loss": 2.7442,
      "step": 280850
    },
    {
      "activations/layer0_attention_weight_max": 15.224209785461426,
      "activations/layer0_attention_weight_min": -13.044173240661621,
      "activations/layer10_attention_weight_max": 34.28607177734375,
      "activations/layer10_attention_weight_min": -32.6995964050293,
      "activations/layer11_attention_weight_max": 32.22885513305664,
      "activations/layer11_attention_weight_min": -32.11270523071289,
      "activations/layer12_attention_weight_max": 17.86290740966797,
      "activations/layer12_attention_weight_min": -24.187326431274414,
      "activations/layer13_attention_weight_max": 34.45734786987305,
      "activations/layer13_attention_weight_min": -27.051984786987305,
      "activations/layer14_attention_weight_max": 38.6575813293457,
      "activations/layer14_attention_weight_min": -30.121423721313477,
      "activations/layer15_attention_weight_max": 32.08808898925781,
      "activations/layer15_attention_weight_min": -29.47140884399414,
      "activations/layer16_attention_weight_max": 31.65732765197754,
      "activations/layer16_attention_weight_min": -31.348819732666016,
      "activations/layer17_attention_weight_max": 50.66069412231445,
      "activations/layer17_attention_weight_min": -43.663516998291016,
      "activations/layer18_attention_weight_max": 45.79217529296875,
      "activations/layer18_attention_weight_min": -39.58367156982422,
      "activations/layer19_attention_weight_max": 22.76209259033203,
      "activations/layer19_attention_weight_min": -20.742637634277344,
      "activations/layer1_attention_weight_max": 16.015933990478516,
      "activations/layer1_attention_weight_min": -13.684993743896484,
      "activations/layer20_attention_weight_max": 21.23263931274414,
      "activations/layer20_attention_weight_min": -21.565349578857422,
      "activations/layer21_attention_weight_max": 35.845733642578125,
      "activations/layer21_attention_weight_min": -20.579456329345703,
      "activations/layer22_attention_weight_max": 28.46514129638672,
      "activations/layer22_attention_weight_min": -26.560611724853516,
      "activations/layer23_attention_weight_max": 34.07419967651367,
      "activations/layer23_attention_weight_min": -22.416568756103516,
      "activations/layer2_attention_weight_max": 29.592273712158203,
      "activations/layer2_attention_weight_min": -28.7713623046875,
      "activations/layer3_attention_weight_max": 84.81739044189453,
      "activations/layer3_attention_weight_min": -82.41423797607422,
      "activations/layer4_attention_weight_max": 85.82881164550781,
      "activations/layer4_attention_weight_min": -84.1376724243164,
      "activations/layer5_attention_weight_max": 66.43435668945312,
      "activations/layer5_attention_weight_min": -76.08958435058594,
      "activations/layer6_attention_weight_max": 48.52865219116211,
      "activations/layer6_attention_weight_min": -51.05744552612305,
      "activations/layer7_attention_weight_max": 62.940650939941406,
      "activations/layer7_attention_weight_min": -61.093875885009766,
      "activations/layer8_attention_weight_max": 42.06949996948242,
      "activations/layer8_attention_weight_min": -45.847110748291016,
      "activations/layer9_attention_weight_max": 41.9193229675293,
      "activations/layer9_attention_weight_min": -42.43682861328125,
      "epoch": 16.32,
      "learning_rate": 4.5200378787878783e-05,
      "loss": 2.745,
      "step": 280900
    },
    {
      "activations/layer0_attention_weight_max": 15.55776309967041,
      "activations/layer0_attention_weight_min": -13.118417739868164,
      "activations/layer10_attention_weight_max": 33.39976501464844,
      "activations/layer10_attention_weight_min": -33.60254669189453,
      "activations/layer11_attention_weight_max": 29.592594146728516,
      "activations/layer11_attention_weight_min": -32.48558807373047,
      "activations/layer12_attention_weight_max": 18.31428337097168,
      "activations/layer12_attention_weight_min": -22.328453063964844,
      "activations/layer13_attention_weight_max": 36.573516845703125,
      "activations/layer13_attention_weight_min": -27.077009201049805,
      "activations/layer14_attention_weight_max": 32.905914306640625,
      "activations/layer14_attention_weight_min": -32.30314636230469,
      "activations/layer15_attention_weight_max": 33.28812026977539,
      "activations/layer15_attention_weight_min": -32.42205047607422,
      "activations/layer16_attention_weight_max": 34.75048828125,
      "activations/layer16_attention_weight_min": -31.44212532043457,
      "activations/layer17_attention_weight_max": 52.46928787231445,
      "activations/layer17_attention_weight_min": -45.84016036987305,
      "activations/layer18_attention_weight_max": 46.53934097290039,
      "activations/layer18_attention_weight_min": -39.13514709472656,
      "activations/layer19_attention_weight_max": 23.428373336791992,
      "activations/layer19_attention_weight_min": -22.48941421508789,
      "activations/layer1_attention_weight_max": 16.33266258239746,
      "activations/layer1_attention_weight_min": -14.635295867919922,
      "activations/layer20_attention_weight_max": 23.82416343688965,
      "activations/layer20_attention_weight_min": -23.450328826904297,
      "activations/layer21_attention_weight_max": 34.235843658447266,
      "activations/layer21_attention_weight_min": -22.40082359313965,
      "activations/layer22_attention_weight_max": 31.754194259643555,
      "activations/layer22_attention_weight_min": -28.403535842895508,
      "activations/layer23_attention_weight_max": 38.887451171875,
      "activations/layer23_attention_weight_min": -26.971927642822266,
      "activations/layer2_attention_weight_max": 30.628759384155273,
      "activations/layer2_attention_weight_min": -29.194740295410156,
      "activations/layer3_attention_weight_max": 89.48471069335938,
      "activations/layer3_attention_weight_min": -90.93560028076172,
      "activations/layer4_attention_weight_max": 89.19938659667969,
      "activations/layer4_attention_weight_min": -89.26213073730469,
      "activations/layer5_attention_weight_max": 66.30961608886719,
      "activations/layer5_attention_weight_min": -73.38374328613281,
      "activations/layer6_attention_weight_max": 50.81144332885742,
      "activations/layer6_attention_weight_min": -53.23488235473633,
      "activations/layer7_attention_weight_max": 62.06505584716797,
      "activations/layer7_attention_weight_min": -64.0208969116211,
      "activations/layer8_attention_weight_max": 42.239036560058594,
      "activations/layer8_attention_weight_min": -48.94485092163086,
      "activations/layer9_attention_weight_max": 43.421791076660156,
      "activations/layer9_attention_weight_min": -45.93471908569336,
      "epoch": 16.32,
      "learning_rate": 4.518143939393939e-05,
      "loss": 2.7575,
      "step": 280950
    },
    {
      "activations/layer0_attention_weight_max": 15.755058288574219,
      "activations/layer0_attention_weight_min": -13.1355619430542,
      "activations/layer10_attention_weight_max": 35.012874603271484,
      "activations/layer10_attention_weight_min": -34.82560348510742,
      "activations/layer11_attention_weight_max": 30.71056365966797,
      "activations/layer11_attention_weight_min": -32.90959548950195,
      "activations/layer12_attention_weight_max": 19.665603637695312,
      "activations/layer12_attention_weight_min": -24.957599639892578,
      "activations/layer13_attention_weight_max": 37.95775604248047,
      "activations/layer13_attention_weight_min": -28.838348388671875,
      "activations/layer14_attention_weight_max": 36.79471969604492,
      "activations/layer14_attention_weight_min": -32.314002990722656,
      "activations/layer15_attention_weight_max": 33.076969146728516,
      "activations/layer15_attention_weight_min": -31.212127685546875,
      "activations/layer16_attention_weight_max": 36.03567886352539,
      "activations/layer16_attention_weight_min": -32.59451675415039,
      "activations/layer17_attention_weight_max": 51.0067138671875,
      "activations/layer17_attention_weight_min": -44.728878021240234,
      "activations/layer18_attention_weight_max": 46.737205505371094,
      "activations/layer18_attention_weight_min": -41.74738693237305,
      "activations/layer19_attention_weight_max": 24.53155517578125,
      "activations/layer19_attention_weight_min": -22.26454734802246,
      "activations/layer1_attention_weight_max": 15.833577156066895,
      "activations/layer1_attention_weight_min": -13.695554733276367,
      "activations/layer20_attention_weight_max": 22.85393714904785,
      "activations/layer20_attention_weight_min": -21.382823944091797,
      "activations/layer21_attention_weight_max": 36.38022994995117,
      "activations/layer21_attention_weight_min": -21.582172393798828,
      "activations/layer22_attention_weight_max": 28.51504135131836,
      "activations/layer22_attention_weight_min": -25.92266082763672,
      "activations/layer23_attention_weight_max": 38.172088623046875,
      "activations/layer23_attention_weight_min": -23.918556213378906,
      "activations/layer2_attention_weight_max": 27.65839195251465,
      "activations/layer2_attention_weight_min": -27.083717346191406,
      "activations/layer3_attention_weight_max": 81.67616271972656,
      "activations/layer3_attention_weight_min": -80.01577758789062,
      "activations/layer4_attention_weight_max": 84.52253723144531,
      "activations/layer4_attention_weight_min": -84.18470001220703,
      "activations/layer5_attention_weight_max": 60.91974639892578,
      "activations/layer5_attention_weight_min": -78.60304260253906,
      "activations/layer6_attention_weight_max": 48.293983459472656,
      "activations/layer6_attention_weight_min": -52.040950775146484,
      "activations/layer7_attention_weight_max": 64.11519622802734,
      "activations/layer7_attention_weight_min": -61.78729248046875,
      "activations/layer8_attention_weight_max": 46.25993347167969,
      "activations/layer8_attention_weight_min": -50.01609420776367,
      "activations/layer9_attention_weight_max": 44.330291748046875,
      "activations/layer9_attention_weight_min": -46.283348083496094,
      "epoch": 16.33,
      "learning_rate": 4.516249999999999e-05,
      "loss": 2.7389,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4793,
      "eval_samples_per_second": 506.411,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4793,
      "eval_openwebtext_samples_per_second": 506.411,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9621,
      "eval_wikitext_samples_per_second": 232.405,
      "step": 281000
    },
    {
      "epoch": 16.33,
      "eval_lambada_loss": 2.509765625,
      "eval_lambada_ppl": 12.302046430466076,
      "eval_lambada_runtime": 9.5616,
      "eval_lambada_samples_per_second": 509.226,
      "step": 281000
    },
    {
      "activations/layer0_attention_weight_max": 15.899580001831055,
      "activations/layer0_attention_weight_min": -13.278508186340332,
      "activations/layer10_attention_weight_max": 37.43254089355469,
      "activations/layer10_attention_weight_min": -38.79350662231445,
      "activations/layer11_attention_weight_max": 32.726402282714844,
      "activations/layer11_attention_weight_min": -35.24545669555664,
      "activations/layer12_attention_weight_max": 19.719186782836914,
      "activations/layer12_attention_weight_min": -24.995737075805664,
      "activations/layer13_attention_weight_max": 34.444190979003906,
      "activations/layer13_attention_weight_min": -28.950511932373047,
      "activations/layer14_attention_weight_max": 36.91468048095703,
      "activations/layer14_attention_weight_min": -30.965469360351562,
      "activations/layer15_attention_weight_max": 32.85665512084961,
      "activations/layer15_attention_weight_min": -30.598955154418945,
      "activations/layer16_attention_weight_max": 33.87363052368164,
      "activations/layer16_attention_weight_min": -31.18434715270996,
      "activations/layer17_attention_weight_max": 51.57249450683594,
      "activations/layer17_attention_weight_min": -44.36931610107422,
      "activations/layer18_attention_weight_max": 49.23110580444336,
      "activations/layer18_attention_weight_min": -41.29207992553711,
      "activations/layer19_attention_weight_max": 23.834793090820312,
      "activations/layer19_attention_weight_min": -23.726072311401367,
      "activations/layer1_attention_weight_max": 15.86129093170166,
      "activations/layer1_attention_weight_min": -13.817743301391602,
      "activations/layer20_attention_weight_max": 24.641149520874023,
      "activations/layer20_attention_weight_min": -21.284587860107422,
      "activations/layer21_attention_weight_max": 34.68070602416992,
      "activations/layer21_attention_weight_min": -27.501487731933594,
      "activations/layer22_attention_weight_max": 30.62519645690918,
      "activations/layer22_attention_weight_min": -26.130159378051758,
      "activations/layer23_attention_weight_max": 37.02558135986328,
      "activations/layer23_attention_weight_min": -24.721227645874023,
      "activations/layer2_attention_weight_max": 29.742374420166016,
      "activations/layer2_attention_weight_min": -29.119731903076172,
      "activations/layer3_attention_weight_max": 87.29957580566406,
      "activations/layer3_attention_weight_min": -88.13104248046875,
      "activations/layer4_attention_weight_max": 91.30848693847656,
      "activations/layer4_attention_weight_min": -89.73871612548828,
      "activations/layer5_attention_weight_max": 64.96431732177734,
      "activations/layer5_attention_weight_min": -77.78182983398438,
      "activations/layer6_attention_weight_max": 53.06733703613281,
      "activations/layer6_attention_weight_min": -53.897586822509766,
      "activations/layer7_attention_weight_max": 68.81525421142578,
      "activations/layer7_attention_weight_min": -67.90921783447266,
      "activations/layer8_attention_weight_max": 46.90201187133789,
      "activations/layer8_attention_weight_min": -52.2000617980957,
      "activations/layer9_attention_weight_max": 45.25415802001953,
      "activations/layer9_attention_weight_min": -48.23688888549805,
      "epoch": 16.33,
      "learning_rate": 4.514356060606061e-05,
      "loss": 2.7515,
      "step": 281050
    },
    {
      "activations/layer0_attention_weight_max": 16.02125358581543,
      "activations/layer0_attention_weight_min": -13.611418724060059,
      "activations/layer10_attention_weight_max": 37.39608383178711,
      "activations/layer10_attention_weight_min": -37.76932907104492,
      "activations/layer11_attention_weight_max": 33.407875061035156,
      "activations/layer11_attention_weight_min": -34.69428253173828,
      "activations/layer12_attention_weight_max": 18.37563133239746,
      "activations/layer12_attention_weight_min": -26.93419075012207,
      "activations/layer13_attention_weight_max": 34.179683685302734,
      "activations/layer13_attention_weight_min": -29.742334365844727,
      "activations/layer14_attention_weight_max": 39.30113983154297,
      "activations/layer14_attention_weight_min": -33.04878234863281,
      "activations/layer15_attention_weight_max": 34.185367584228516,
      "activations/layer15_attention_weight_min": -30.10696029663086,
      "activations/layer16_attention_weight_max": 33.35265350341797,
      "activations/layer16_attention_weight_min": -33.32239532470703,
      "activations/layer17_attention_weight_max": 50.355018615722656,
      "activations/layer17_attention_weight_min": -49.40755081176758,
      "activations/layer18_attention_weight_max": 48.88974380493164,
      "activations/layer18_attention_weight_min": -47.09516143798828,
      "activations/layer19_attention_weight_max": 22.452465057373047,
      "activations/layer19_attention_weight_min": -25.886449813842773,
      "activations/layer1_attention_weight_max": 16.757667541503906,
      "activations/layer1_attention_weight_min": -15.237892150878906,
      "activations/layer20_attention_weight_max": 25.748151779174805,
      "activations/layer20_attention_weight_min": -22.173585891723633,
      "activations/layer21_attention_weight_max": 35.666160583496094,
      "activations/layer21_attention_weight_min": -27.785722732543945,
      "activations/layer22_attention_weight_max": 28.55040168762207,
      "activations/layer22_attention_weight_min": -25.006059646606445,
      "activations/layer23_attention_weight_max": 40.38875961303711,
      "activations/layer23_attention_weight_min": -25.379806518554688,
      "activations/layer2_attention_weight_max": 31.667966842651367,
      "activations/layer2_attention_weight_min": -29.58038330078125,
      "activations/layer3_attention_weight_max": 89.06982421875,
      "activations/layer3_attention_weight_min": -94.37570190429688,
      "activations/layer4_attention_weight_max": 91.0438003540039,
      "activations/layer4_attention_weight_min": -88.48796844482422,
      "activations/layer5_attention_weight_max": 66.63668823242188,
      "activations/layer5_attention_weight_min": -77.07278442382812,
      "activations/layer6_attention_weight_max": 51.57645034790039,
      "activations/layer6_attention_weight_min": -52.28800964355469,
      "activations/layer7_attention_weight_max": 62.39643859863281,
      "activations/layer7_attention_weight_min": -62.77461242675781,
      "activations/layer8_attention_weight_max": 47.75306701660156,
      "activations/layer8_attention_weight_min": -52.33441925048828,
      "activations/layer9_attention_weight_max": 46.17755126953125,
      "activations/layer9_attention_weight_min": -47.725196838378906,
      "epoch": 16.33,
      "learning_rate": 4.512462121212121e-05,
      "loss": 2.7545,
      "step": 281100
    },
    {
      "activations/layer0_attention_weight_max": 15.632959365844727,
      "activations/layer0_attention_weight_min": -13.190510749816895,
      "activations/layer10_attention_weight_max": 35.885311126708984,
      "activations/layer10_attention_weight_min": -36.78493118286133,
      "activations/layer11_attention_weight_max": 33.816070556640625,
      "activations/layer11_attention_weight_min": -33.320281982421875,
      "activations/layer12_attention_weight_max": 17.79146957397461,
      "activations/layer12_attention_weight_min": -24.77984619140625,
      "activations/layer13_attention_weight_max": 35.781558990478516,
      "activations/layer13_attention_weight_min": -30.3643856048584,
      "activations/layer14_attention_weight_max": 41.93000793457031,
      "activations/layer14_attention_weight_min": -29.054521560668945,
      "activations/layer15_attention_weight_max": 37.28483200073242,
      "activations/layer15_attention_weight_min": -28.282541275024414,
      "activations/layer16_attention_weight_max": 34.71349334716797,
      "activations/layer16_attention_weight_min": -29.6737060546875,
      "activations/layer17_attention_weight_max": 55.870521545410156,
      "activations/layer17_attention_weight_min": -48.903682708740234,
      "activations/layer18_attention_weight_max": 43.41336441040039,
      "activations/layer18_attention_weight_min": -38.229244232177734,
      "activations/layer19_attention_weight_max": 23.543315887451172,
      "activations/layer19_attention_weight_min": -24.01222801208496,
      "activations/layer1_attention_weight_max": 16.041507720947266,
      "activations/layer1_attention_weight_min": -14.684617042541504,
      "activations/layer20_attention_weight_max": 21.756145477294922,
      "activations/layer20_attention_weight_min": -22.672792434692383,
      "activations/layer21_attention_weight_max": 37.886871337890625,
      "activations/layer21_attention_weight_min": -19.822425842285156,
      "activations/layer22_attention_weight_max": 27.875247955322266,
      "activations/layer22_attention_weight_min": -25.446815490722656,
      "activations/layer23_attention_weight_max": 36.740447998046875,
      "activations/layer23_attention_weight_min": -23.026708602905273,
      "activations/layer2_attention_weight_max": 31.599472045898438,
      "activations/layer2_attention_weight_min": -31.243488311767578,
      "activations/layer3_attention_weight_max": 86.24716186523438,
      "activations/layer3_attention_weight_min": -83.62971496582031,
      "activations/layer4_attention_weight_max": 90.17229461669922,
      "activations/layer4_attention_weight_min": -87.40510559082031,
      "activations/layer5_attention_weight_max": 65.74908447265625,
      "activations/layer5_attention_weight_min": -70.57206726074219,
      "activations/layer6_attention_weight_max": 49.636661529541016,
      "activations/layer6_attention_weight_min": -49.937808990478516,
      "activations/layer7_attention_weight_max": 62.68195724487305,
      "activations/layer7_attention_weight_min": -63.598060607910156,
      "activations/layer8_attention_weight_max": 45.14728546142578,
      "activations/layer8_attention_weight_min": -46.68946838378906,
      "activations/layer9_attention_weight_max": 48.502079010009766,
      "activations/layer9_attention_weight_min": -45.44267272949219,
      "epoch": 16.34,
      "learning_rate": 4.510568181818181e-05,
      "loss": 2.7576,
      "step": 281150
    },
    {
      "activations/layer0_attention_weight_max": 16.011432647705078,
      "activations/layer0_attention_weight_min": -13.474993705749512,
      "activations/layer10_attention_weight_max": 35.360809326171875,
      "activations/layer10_attention_weight_min": -34.45817565917969,
      "activations/layer11_attention_weight_max": 31.671119689941406,
      "activations/layer11_attention_weight_min": -35.17394256591797,
      "activations/layer12_attention_weight_max": 19.366424560546875,
      "activations/layer12_attention_weight_min": -24.119596481323242,
      "activations/layer13_attention_weight_max": 35.22585678100586,
      "activations/layer13_attention_weight_min": -27.662944793701172,
      "activations/layer14_attention_weight_max": 36.38856887817383,
      "activations/layer14_attention_weight_min": -29.71513557434082,
      "activations/layer15_attention_weight_max": 34.47978973388672,
      "activations/layer15_attention_weight_min": -29.988073348999023,
      "activations/layer16_attention_weight_max": 33.90970230102539,
      "activations/layer16_attention_weight_min": -33.15912628173828,
      "activations/layer17_attention_weight_max": 52.400691986083984,
      "activations/layer17_attention_weight_min": -45.825782775878906,
      "activations/layer18_attention_weight_max": 51.38321304321289,
      "activations/layer18_attention_weight_min": -41.10437774658203,
      "activations/layer19_attention_weight_max": 24.63149070739746,
      "activations/layer19_attention_weight_min": -22.110002517700195,
      "activations/layer1_attention_weight_max": 16.093395233154297,
      "activations/layer1_attention_weight_min": -13.810507774353027,
      "activations/layer20_attention_weight_max": 22.99176597595215,
      "activations/layer20_attention_weight_min": -22.819107055664062,
      "activations/layer21_attention_weight_max": 41.58315658569336,
      "activations/layer21_attention_weight_min": -23.597612380981445,
      "activations/layer22_attention_weight_max": 31.149513244628906,
      "activations/layer22_attention_weight_min": -25.882097244262695,
      "activations/layer23_attention_weight_max": 38.19866180419922,
      "activations/layer23_attention_weight_min": -26.631200790405273,
      "activations/layer2_attention_weight_max": 27.096942901611328,
      "activations/layer2_attention_weight_min": -26.17949676513672,
      "activations/layer3_attention_weight_max": 84.45600891113281,
      "activations/layer3_attention_weight_min": -88.51126861572266,
      "activations/layer4_attention_weight_max": 87.53001403808594,
      "activations/layer4_attention_weight_min": -85.79485321044922,
      "activations/layer5_attention_weight_max": 65.69805908203125,
      "activations/layer5_attention_weight_min": -70.63079071044922,
      "activations/layer6_attention_weight_max": 49.7529182434082,
      "activations/layer6_attention_weight_min": -49.811546325683594,
      "activations/layer7_attention_weight_max": 59.9656867980957,
      "activations/layer7_attention_weight_min": -62.329124450683594,
      "activations/layer8_attention_weight_max": 43.77189254760742,
      "activations/layer8_attention_weight_min": -48.83692932128906,
      "activations/layer9_attention_weight_max": 46.12320327758789,
      "activations/layer9_attention_weight_min": -45.90005111694336,
      "epoch": 16.34,
      "learning_rate": 4.508674242424242e-05,
      "loss": 2.7591,
      "step": 281200
    },
    {
      "activations/layer0_attention_weight_max": 15.027628898620605,
      "activations/layer0_attention_weight_min": -13.395133972167969,
      "activations/layer10_attention_weight_max": 37.61193084716797,
      "activations/layer10_attention_weight_min": -37.29282760620117,
      "activations/layer11_attention_weight_max": 33.20655822753906,
      "activations/layer11_attention_weight_min": -35.73424530029297,
      "activations/layer12_attention_weight_max": 21.6315860748291,
      "activations/layer12_attention_weight_min": -24.184812545776367,
      "activations/layer13_attention_weight_max": 36.56459426879883,
      "activations/layer13_attention_weight_min": -28.367355346679688,
      "activations/layer14_attention_weight_max": 39.2971305847168,
      "activations/layer14_attention_weight_min": -30.27016258239746,
      "activations/layer15_attention_weight_max": 34.87656021118164,
      "activations/layer15_attention_weight_min": -29.45676612854004,
      "activations/layer16_attention_weight_max": 36.5687141418457,
      "activations/layer16_attention_weight_min": -32.58686447143555,
      "activations/layer17_attention_weight_max": 53.918155670166016,
      "activations/layer17_attention_weight_min": -45.59959411621094,
      "activations/layer18_attention_weight_max": 49.73994827270508,
      "activations/layer18_attention_weight_min": -38.45295333862305,
      "activations/layer19_attention_weight_max": 22.748498916625977,
      "activations/layer19_attention_weight_min": -22.151752471923828,
      "activations/layer1_attention_weight_max": 16.07309341430664,
      "activations/layer1_attention_weight_min": -12.4556884765625,
      "activations/layer20_attention_weight_max": 23.10287094116211,
      "activations/layer20_attention_weight_min": -22.480409622192383,
      "activations/layer21_attention_weight_max": 33.260616302490234,
      "activations/layer21_attention_weight_min": -22.356719970703125,
      "activations/layer22_attention_weight_max": 30.304346084594727,
      "activations/layer22_attention_weight_min": -25.90444564819336,
      "activations/layer23_attention_weight_max": 35.310577392578125,
      "activations/layer23_attention_weight_min": -25.339069366455078,
      "activations/layer2_attention_weight_max": 30.2945613861084,
      "activations/layer2_attention_weight_min": -29.343151092529297,
      "activations/layer3_attention_weight_max": 85.80289459228516,
      "activations/layer3_attention_weight_min": -87.4954605102539,
      "activations/layer4_attention_weight_max": 86.09444427490234,
      "activations/layer4_attention_weight_min": -84.54607391357422,
      "activations/layer5_attention_weight_max": 65.65031433105469,
      "activations/layer5_attention_weight_min": -73.84367370605469,
      "activations/layer6_attention_weight_max": 52.504844665527344,
      "activations/layer6_attention_weight_min": -52.44300079345703,
      "activations/layer7_attention_weight_max": 70.57498931884766,
      "activations/layer7_attention_weight_min": -64.3023452758789,
      "activations/layer8_attention_weight_max": 46.94102478027344,
      "activations/layer8_attention_weight_min": -49.32869338989258,
      "activations/layer9_attention_weight_max": 45.80377197265625,
      "activations/layer9_attention_weight_min": -47.748008728027344,
      "epoch": 16.34,
      "learning_rate": 4.506780303030303e-05,
      "loss": 2.7464,
      "step": 281250
    },
    {
      "activations/layer0_attention_weight_max": 16.40673828125,
      "activations/layer0_attention_weight_min": -13.138111114501953,
      "activations/layer10_attention_weight_max": 35.66431427001953,
      "activations/layer10_attention_weight_min": -35.853294372558594,
      "activations/layer11_attention_weight_max": 31.664478302001953,
      "activations/layer11_attention_weight_min": -34.60900115966797,
      "activations/layer12_attention_weight_max": 18.634014129638672,
      "activations/layer12_attention_weight_min": -27.775293350219727,
      "activations/layer13_attention_weight_max": 32.55221176147461,
      "activations/layer13_attention_weight_min": -26.623706817626953,
      "activations/layer14_attention_weight_max": 33.98589324951172,
      "activations/layer14_attention_weight_min": -28.663585662841797,
      "activations/layer15_attention_weight_max": 32.01426696777344,
      "activations/layer15_attention_weight_min": -29.703649520874023,
      "activations/layer16_attention_weight_max": 33.8255729675293,
      "activations/layer16_attention_weight_min": -29.552284240722656,
      "activations/layer17_attention_weight_max": 51.96171951293945,
      "activations/layer17_attention_weight_min": -43.544471740722656,
      "activations/layer18_attention_weight_max": 48.59687805175781,
      "activations/layer18_attention_weight_min": -40.04015350341797,
      "activations/layer19_attention_weight_max": 24.360586166381836,
      "activations/layer19_attention_weight_min": -23.024946212768555,
      "activations/layer1_attention_weight_max": 16.024433135986328,
      "activations/layer1_attention_weight_min": -15.04058837890625,
      "activations/layer20_attention_weight_max": 24.445188522338867,
      "activations/layer20_attention_weight_min": -22.58734893798828,
      "activations/layer21_attention_weight_max": 40.38568115234375,
      "activations/layer21_attention_weight_min": -25.874420166015625,
      "activations/layer22_attention_weight_max": 30.090986251831055,
      "activations/layer22_attention_weight_min": -27.678356170654297,
      "activations/layer23_attention_weight_max": 37.393516540527344,
      "activations/layer23_attention_weight_min": -26.03443145751953,
      "activations/layer2_attention_weight_max": 31.374156951904297,
      "activations/layer2_attention_weight_min": -29.368053436279297,
      "activations/layer3_attention_weight_max": 84.8901138305664,
      "activations/layer3_attention_weight_min": -85.04241180419922,
      "activations/layer4_attention_weight_max": 88.82730102539062,
      "activations/layer4_attention_weight_min": -84.95077514648438,
      "activations/layer5_attention_weight_max": 66.09578704833984,
      "activations/layer5_attention_weight_min": -74.2591552734375,
      "activations/layer6_attention_weight_max": 51.127193450927734,
      "activations/layer6_attention_weight_min": -52.00708770751953,
      "activations/layer7_attention_weight_max": 63.797183990478516,
      "activations/layer7_attention_weight_min": -62.687225341796875,
      "activations/layer8_attention_weight_max": 45.73250961303711,
      "activations/layer8_attention_weight_min": -49.00467300415039,
      "activations/layer9_attention_weight_max": 43.648658752441406,
      "activations/layer9_attention_weight_min": -46.50028610229492,
      "epoch": 16.35,
      "learning_rate": 4.5048863636363636e-05,
      "loss": 2.7531,
      "step": 281300
    },
    {
      "activations/layer0_attention_weight_max": 15.066843032836914,
      "activations/layer0_attention_weight_min": -13.11745834350586,
      "activations/layer10_attention_weight_max": 39.55952835083008,
      "activations/layer10_attention_weight_min": -36.444705963134766,
      "activations/layer11_attention_weight_max": 32.570892333984375,
      "activations/layer11_attention_weight_min": -34.35545349121094,
      "activations/layer12_attention_weight_max": 17.994510650634766,
      "activations/layer12_attention_weight_min": -29.623435974121094,
      "activations/layer13_attention_weight_max": 32.68930435180664,
      "activations/layer13_attention_weight_min": -26.672388076782227,
      "activations/layer14_attention_weight_max": 32.418846130371094,
      "activations/layer14_attention_weight_min": -28.650768280029297,
      "activations/layer15_attention_weight_max": 30.348342895507812,
      "activations/layer15_attention_weight_min": -29.217742919921875,
      "activations/layer16_attention_weight_max": 32.42271041870117,
      "activations/layer16_attention_weight_min": -31.999095916748047,
      "activations/layer17_attention_weight_max": 48.9080696105957,
      "activations/layer17_attention_weight_min": -43.14224624633789,
      "activations/layer18_attention_weight_max": 43.56064224243164,
      "activations/layer18_attention_weight_min": -38.414554595947266,
      "activations/layer19_attention_weight_max": 23.27621078491211,
      "activations/layer19_attention_weight_min": -22.11408042907715,
      "activations/layer1_attention_weight_max": 16.46664047241211,
      "activations/layer1_attention_weight_min": -14.114789962768555,
      "activations/layer20_attention_weight_max": 23.562395095825195,
      "activations/layer20_attention_weight_min": -22.858205795288086,
      "activations/layer21_attention_weight_max": 36.7935905456543,
      "activations/layer21_attention_weight_min": -23.71111488342285,
      "activations/layer22_attention_weight_max": 29.48655128479004,
      "activations/layer22_attention_weight_min": -26.236610412597656,
      "activations/layer23_attention_weight_max": 38.17408752441406,
      "activations/layer23_attention_weight_min": -23.434659957885742,
      "activations/layer2_attention_weight_max": 31.448883056640625,
      "activations/layer2_attention_weight_min": -28.91534423828125,
      "activations/layer3_attention_weight_max": 85.71410369873047,
      "activations/layer3_attention_weight_min": -88.12198638916016,
      "activations/layer4_attention_weight_max": 91.82024383544922,
      "activations/layer4_attention_weight_min": -89.45709228515625,
      "activations/layer5_attention_weight_max": 67.2633056640625,
      "activations/layer5_attention_weight_min": -76.6500244140625,
      "activations/layer6_attention_weight_max": 52.9995002746582,
      "activations/layer6_attention_weight_min": -53.61147689819336,
      "activations/layer7_attention_weight_max": 66.43451690673828,
      "activations/layer7_attention_weight_min": -67.24665832519531,
      "activations/layer8_attention_weight_max": 46.99079132080078,
      "activations/layer8_attention_weight_min": -51.92509841918945,
      "activations/layer9_attention_weight_max": 46.69602584838867,
      "activations/layer9_attention_weight_min": -47.452659606933594,
      "epoch": 16.35,
      "learning_rate": 4.502992424242424e-05,
      "loss": 2.7597,
      "step": 281350
    },
    {
      "activations/layer0_attention_weight_max": 15.46780014038086,
      "activations/layer0_attention_weight_min": -13.305335998535156,
      "activations/layer10_attention_weight_max": 36.378387451171875,
      "activations/layer10_attention_weight_min": -35.035064697265625,
      "activations/layer11_attention_weight_max": 31.226818084716797,
      "activations/layer11_attention_weight_min": -32.792354583740234,
      "activations/layer12_attention_weight_max": 18.0795841217041,
      "activations/layer12_attention_weight_min": -24.1995849609375,
      "activations/layer13_attention_weight_max": 33.52936553955078,
      "activations/layer13_attention_weight_min": -26.024927139282227,
      "activations/layer14_attention_weight_max": 34.2778205871582,
      "activations/layer14_attention_weight_min": -29.898956298828125,
      "activations/layer15_attention_weight_max": 31.179216384887695,
      "activations/layer15_attention_weight_min": -29.27701187133789,
      "activations/layer16_attention_weight_max": 34.572166442871094,
      "activations/layer16_attention_weight_min": -32.06676483154297,
      "activations/layer17_attention_weight_max": 53.32753372192383,
      "activations/layer17_attention_weight_min": -45.72245407104492,
      "activations/layer18_attention_weight_max": 47.52375793457031,
      "activations/layer18_attention_weight_min": -40.80976486206055,
      "activations/layer19_attention_weight_max": 22.983854293823242,
      "activations/layer19_attention_weight_min": -24.980627059936523,
      "activations/layer1_attention_weight_max": 16.469669342041016,
      "activations/layer1_attention_weight_min": -14.939020156860352,
      "activations/layer20_attention_weight_max": 24.293718338012695,
      "activations/layer20_attention_weight_min": -23.555849075317383,
      "activations/layer21_attention_weight_max": 33.09263229370117,
      "activations/layer21_attention_weight_min": -26.553178787231445,
      "activations/layer22_attention_weight_max": 29.37684440612793,
      "activations/layer22_attention_weight_min": -25.79164695739746,
      "activations/layer23_attention_weight_max": 36.808677673339844,
      "activations/layer23_attention_weight_min": -25.59755516052246,
      "activations/layer2_attention_weight_max": 33.454322814941406,
      "activations/layer2_attention_weight_min": -31.42724609375,
      "activations/layer3_attention_weight_max": 88.564208984375,
      "activations/layer3_attention_weight_min": -91.36212158203125,
      "activations/layer4_attention_weight_max": 90.60546112060547,
      "activations/layer4_attention_weight_min": -91.30552673339844,
      "activations/layer5_attention_weight_max": 65.17730712890625,
      "activations/layer5_attention_weight_min": -74.23316192626953,
      "activations/layer6_attention_weight_max": 49.512271881103516,
      "activations/layer6_attention_weight_min": -52.82210159301758,
      "activations/layer7_attention_weight_max": 63.17924499511719,
      "activations/layer7_attention_weight_min": -64.41384887695312,
      "activations/layer8_attention_weight_max": 47.20633316040039,
      "activations/layer8_attention_weight_min": -49.028289794921875,
      "activations/layer9_attention_weight_max": 43.97528076171875,
      "activations/layer9_attention_weight_min": -46.14207458496094,
      "epoch": 16.35,
      "learning_rate": 4.501098484848484e-05,
      "loss": 2.7553,
      "step": 281400
    },
    {
      "activations/layer0_attention_weight_max": 15.526562690734863,
      "activations/layer0_attention_weight_min": -13.165492057800293,
      "activations/layer10_attention_weight_max": 38.597774505615234,
      "activations/layer10_attention_weight_min": -41.19746780395508,
      "activations/layer11_attention_weight_max": 32.1889533996582,
      "activations/layer11_attention_weight_min": -36.45130920410156,
      "activations/layer12_attention_weight_max": 20.107694625854492,
      "activations/layer12_attention_weight_min": -27.0633487701416,
      "activations/layer13_attention_weight_max": 36.12189865112305,
      "activations/layer13_attention_weight_min": -33.876708984375,
      "activations/layer14_attention_weight_max": 36.619239807128906,
      "activations/layer14_attention_weight_min": -31.105815887451172,
      "activations/layer15_attention_weight_max": 34.771728515625,
      "activations/layer15_attention_weight_min": -29.603710174560547,
      "activations/layer16_attention_weight_max": 36.163421630859375,
      "activations/layer16_attention_weight_min": -33.5777587890625,
      "activations/layer17_attention_weight_max": 54.81328201293945,
      "activations/layer17_attention_weight_min": -48.726478576660156,
      "activations/layer18_attention_weight_max": 48.181976318359375,
      "activations/layer18_attention_weight_min": -42.69085693359375,
      "activations/layer19_attention_weight_max": 26.316091537475586,
      "activations/layer19_attention_weight_min": -22.759008407592773,
      "activations/layer1_attention_weight_max": 16.203227996826172,
      "activations/layer1_attention_weight_min": -14.045019149780273,
      "activations/layer20_attention_weight_max": 25.27486801147461,
      "activations/layer20_attention_weight_min": -22.237060546875,
      "activations/layer21_attention_weight_max": 44.522369384765625,
      "activations/layer21_attention_weight_min": -23.45305061340332,
      "activations/layer22_attention_weight_max": 36.28708267211914,
      "activations/layer22_attention_weight_min": -25.56222915649414,
      "activations/layer23_attention_weight_max": 44.26627731323242,
      "activations/layer23_attention_weight_min": -24.87612533569336,
      "activations/layer2_attention_weight_max": 33.832008361816406,
      "activations/layer2_attention_weight_min": -30.879161834716797,
      "activations/layer3_attention_weight_max": 89.36361694335938,
      "activations/layer3_attention_weight_min": -88.4059066772461,
      "activations/layer4_attention_weight_max": 93.45713806152344,
      "activations/layer4_attention_weight_min": -92.63560485839844,
      "activations/layer5_attention_weight_max": 64.2241439819336,
      "activations/layer5_attention_weight_min": -76.0277099609375,
      "activations/layer6_attention_weight_max": 55.74977493286133,
      "activations/layer6_attention_weight_min": -61.1372184753418,
      "activations/layer7_attention_weight_max": 64.43738555908203,
      "activations/layer7_attention_weight_min": -69.97898864746094,
      "activations/layer8_attention_weight_max": 46.98357009887695,
      "activations/layer8_attention_weight_min": -52.80303955078125,
      "activations/layer9_attention_weight_max": 47.66879653930664,
      "activations/layer9_attention_weight_min": -50.34727478027344,
      "epoch": 16.35,
      "learning_rate": 4.4992045454545454e-05,
      "loss": 2.7424,
      "step": 281450
    },
    {
      "activations/layer0_attention_weight_max": 15.216703414916992,
      "activations/layer0_attention_weight_min": -13.536961555480957,
      "activations/layer10_attention_weight_max": 40.334781646728516,
      "activations/layer10_attention_weight_min": -39.14079666137695,
      "activations/layer11_attention_weight_max": 35.894737243652344,
      "activations/layer11_attention_weight_min": -37.39573669433594,
      "activations/layer12_attention_weight_max": 18.742841720581055,
      "activations/layer12_attention_weight_min": -27.02588653564453,
      "activations/layer13_attention_weight_max": 35.7962646484375,
      "activations/layer13_attention_weight_min": -30.707292556762695,
      "activations/layer14_attention_weight_max": 34.427528381347656,
      "activations/layer14_attention_weight_min": -32.1783447265625,
      "activations/layer15_attention_weight_max": 30.056867599487305,
      "activations/layer15_attention_weight_min": -30.498830795288086,
      "activations/layer16_attention_weight_max": 30.8299560546875,
      "activations/layer16_attention_weight_min": -30.53410530090332,
      "activations/layer17_attention_weight_max": 50.321624755859375,
      "activations/layer17_attention_weight_min": -44.59236526489258,
      "activations/layer18_attention_weight_max": 44.64659881591797,
      "activations/layer18_attention_weight_min": -40.38949966430664,
      "activations/layer19_attention_weight_max": 21.923749923706055,
      "activations/layer19_attention_weight_min": -21.921154022216797,
      "activations/layer1_attention_weight_max": 17.269811630249023,
      "activations/layer1_attention_weight_min": -16.22553062438965,
      "activations/layer20_attention_weight_max": 23.00284767150879,
      "activations/layer20_attention_weight_min": -21.571475982666016,
      "activations/layer21_attention_weight_max": 33.21070861816406,
      "activations/layer21_attention_weight_min": -24.264694213867188,
      "activations/layer22_attention_weight_max": 27.216596603393555,
      "activations/layer22_attention_weight_min": -26.28857421875,
      "activations/layer23_attention_weight_max": 33.886573791503906,
      "activations/layer23_attention_weight_min": -26.50881576538086,
      "activations/layer2_attention_weight_max": 32.011600494384766,
      "activations/layer2_attention_weight_min": -30.234329223632812,
      "activations/layer3_attention_weight_max": 92.54345703125,
      "activations/layer3_attention_weight_min": -92.77605438232422,
      "activations/layer4_attention_weight_max": 92.04336547851562,
      "activations/layer4_attention_weight_min": -89.88700866699219,
      "activations/layer5_attention_weight_max": 66.02994537353516,
      "activations/layer5_attention_weight_min": -71.91523742675781,
      "activations/layer6_attention_weight_max": 54.22863006591797,
      "activations/layer6_attention_weight_min": -55.1798210144043,
      "activations/layer7_attention_weight_max": 64.37303161621094,
      "activations/layer7_attention_weight_min": -66.89344024658203,
      "activations/layer8_attention_weight_max": 51.014591217041016,
      "activations/layer8_attention_weight_min": -54.64616012573242,
      "activations/layer9_attention_weight_max": 51.175655364990234,
      "activations/layer9_attention_weight_min": -49.172142028808594,
      "epoch": 16.36,
      "learning_rate": 4.4973106060606056e-05,
      "loss": 2.7492,
      "step": 281500
    },
    {
      "activations/layer0_attention_weight_max": 15.85187816619873,
      "activations/layer0_attention_weight_min": -13.856460571289062,
      "activations/layer10_attention_weight_max": 35.18859100341797,
      "activations/layer10_attention_weight_min": -34.99296569824219,
      "activations/layer11_attention_weight_max": 32.884010314941406,
      "activations/layer11_attention_weight_min": -33.73746109008789,
      "activations/layer12_attention_weight_max": 19.00532341003418,
      "activations/layer12_attention_weight_min": -24.759992599487305,
      "activations/layer13_attention_weight_max": 38.49592971801758,
      "activations/layer13_attention_weight_min": -28.015750885009766,
      "activations/layer14_attention_weight_max": 37.838966369628906,
      "activations/layer14_attention_weight_min": -29.89671516418457,
      "activations/layer15_attention_weight_max": 35.15203094482422,
      "activations/layer15_attention_weight_min": -29.032054901123047,
      "activations/layer16_attention_weight_max": 34.56973648071289,
      "activations/layer16_attention_weight_min": -30.901578903198242,
      "activations/layer17_attention_weight_max": 55.03190994262695,
      "activations/layer17_attention_weight_min": -43.2816047668457,
      "activations/layer18_attention_weight_max": 47.51260757446289,
      "activations/layer18_attention_weight_min": -39.26191329956055,
      "activations/layer19_attention_weight_max": 24.944883346557617,
      "activations/layer19_attention_weight_min": -22.975339889526367,
      "activations/layer1_attention_weight_max": 16.027515411376953,
      "activations/layer1_attention_weight_min": -14.178940773010254,
      "activations/layer20_attention_weight_max": 23.598751068115234,
      "activations/layer20_attention_weight_min": -23.353553771972656,
      "activations/layer21_attention_weight_max": 41.58852005004883,
      "activations/layer21_attention_weight_min": -24.17422866821289,
      "activations/layer22_attention_weight_max": 28.488510131835938,
      "activations/layer22_attention_weight_min": -26.519786834716797,
      "activations/layer23_attention_weight_max": 38.45124816894531,
      "activations/layer23_attention_weight_min": -24.341108322143555,
      "activations/layer2_attention_weight_max": 30.323850631713867,
      "activations/layer2_attention_weight_min": -28.4653377532959,
      "activations/layer3_attention_weight_max": 86.2458267211914,
      "activations/layer3_attention_weight_min": -84.52217102050781,
      "activations/layer4_attention_weight_max": 85.517822265625,
      "activations/layer4_attention_weight_min": -84.59304809570312,
      "activations/layer5_attention_weight_max": 64.22157287597656,
      "activations/layer5_attention_weight_min": -76.35501098632812,
      "activations/layer6_attention_weight_max": 49.31721496582031,
      "activations/layer6_attention_weight_min": -48.34883117675781,
      "activations/layer7_attention_weight_max": 62.50627899169922,
      "activations/layer7_attention_weight_min": -67.23628234863281,
      "activations/layer8_attention_weight_max": 45.75436782836914,
      "activations/layer8_attention_weight_min": -49.22285461425781,
      "activations/layer9_attention_weight_max": 46.56414031982422,
      "activations/layer9_attention_weight_min": -45.85295867919922,
      "epoch": 16.36,
      "learning_rate": 4.4954166666666664e-05,
      "loss": 2.7694,
      "step": 281550
    },
    {
      "activations/layer0_attention_weight_max": 15.169746398925781,
      "activations/layer0_attention_weight_min": -13.247862815856934,
      "activations/layer10_attention_weight_max": 36.96729278564453,
      "activations/layer10_attention_weight_min": -37.147621154785156,
      "activations/layer11_attention_weight_max": 32.00499725341797,
      "activations/layer11_attention_weight_min": -34.836708068847656,
      "activations/layer12_attention_weight_max": 19.746007919311523,
      "activations/layer12_attention_weight_min": -25.193578720092773,
      "activations/layer13_attention_weight_max": 36.05564880371094,
      "activations/layer13_attention_weight_min": -28.041893005371094,
      "activations/layer14_attention_weight_max": 33.949676513671875,
      "activations/layer14_attention_weight_min": -29.45389175415039,
      "activations/layer15_attention_weight_max": 33.29759979248047,
      "activations/layer15_attention_weight_min": -28.666391372680664,
      "activations/layer16_attention_weight_max": 32.93060302734375,
      "activations/layer16_attention_weight_min": -31.147510528564453,
      "activations/layer17_attention_weight_max": 54.35222625732422,
      "activations/layer17_attention_weight_min": -44.6492805480957,
      "activations/layer18_attention_weight_max": 48.17898941040039,
      "activations/layer18_attention_weight_min": -41.27840042114258,
      "activations/layer19_attention_weight_max": 23.380247116088867,
      "activations/layer19_attention_weight_min": -23.218242645263672,
      "activations/layer1_attention_weight_max": 15.987492561340332,
      "activations/layer1_attention_weight_min": -15.746354103088379,
      "activations/layer20_attention_weight_max": 22.827499389648438,
      "activations/layer20_attention_weight_min": -22.291955947875977,
      "activations/layer21_attention_weight_max": 37.94967269897461,
      "activations/layer21_attention_weight_min": -23.58348274230957,
      "activations/layer22_attention_weight_max": 31.844486236572266,
      "activations/layer22_attention_weight_min": -27.32335090637207,
      "activations/layer23_attention_weight_max": 37.67259216308594,
      "activations/layer23_attention_weight_min": -26.18877410888672,
      "activations/layer2_attention_weight_max": 30.14785385131836,
      "activations/layer2_attention_weight_min": -28.553192138671875,
      "activations/layer3_attention_weight_max": 89.48175811767578,
      "activations/layer3_attention_weight_min": -84.79998016357422,
      "activations/layer4_attention_weight_max": 89.9861068725586,
      "activations/layer4_attention_weight_min": -87.28215789794922,
      "activations/layer5_attention_weight_max": 66.70774841308594,
      "activations/layer5_attention_weight_min": -76.37667846679688,
      "activations/layer6_attention_weight_max": 50.714908599853516,
      "activations/layer6_attention_weight_min": -51.4110221862793,
      "activations/layer7_attention_weight_max": 63.332000732421875,
      "activations/layer7_attention_weight_min": -65.7664794921875,
      "activations/layer8_attention_weight_max": 46.181396484375,
      "activations/layer8_attention_weight_min": -48.23347091674805,
      "activations/layer9_attention_weight_max": 45.26332473754883,
      "activations/layer9_attention_weight_min": -44.40253829956055,
      "epoch": 16.36,
      "learning_rate": 4.493522727272727e-05,
      "loss": 2.7517,
      "step": 281600
    },
    {
      "activations/layer0_attention_weight_max": 15.303232192993164,
      "activations/layer0_attention_weight_min": -13.092365264892578,
      "activations/layer10_attention_weight_max": 35.3170166015625,
      "activations/layer10_attention_weight_min": -34.485870361328125,
      "activations/layer11_attention_weight_max": 30.3785457611084,
      "activations/layer11_attention_weight_min": -32.45146942138672,
      "activations/layer12_attention_weight_max": 18.815204620361328,
      "activations/layer12_attention_weight_min": -25.23997688293457,
      "activations/layer13_attention_weight_max": 36.52233123779297,
      "activations/layer13_attention_weight_min": -28.94965362548828,
      "activations/layer14_attention_weight_max": 35.24182891845703,
      "activations/layer14_attention_weight_min": -30.643329620361328,
      "activations/layer15_attention_weight_max": 32.16082763671875,
      "activations/layer15_attention_weight_min": -30.5303897857666,
      "activations/layer16_attention_weight_max": 32.35681915283203,
      "activations/layer16_attention_weight_min": -32.628963470458984,
      "activations/layer17_attention_weight_max": 52.59660720825195,
      "activations/layer17_attention_weight_min": -46.21805191040039,
      "activations/layer18_attention_weight_max": 44.75430679321289,
      "activations/layer18_attention_weight_min": -39.75554656982422,
      "activations/layer19_attention_weight_max": 23.000459671020508,
      "activations/layer19_attention_weight_min": -24.27587127685547,
      "activations/layer1_attention_weight_max": 15.5862455368042,
      "activations/layer1_attention_weight_min": -13.743255615234375,
      "activations/layer20_attention_weight_max": 22.981815338134766,
      "activations/layer20_attention_weight_min": -22.212989807128906,
      "activations/layer21_attention_weight_max": 32.47417449951172,
      "activations/layer21_attention_weight_min": -24.861265182495117,
      "activations/layer22_attention_weight_max": 28.483179092407227,
      "activations/layer22_attention_weight_min": -26.127506256103516,
      "activations/layer23_attention_weight_max": 36.20572280883789,
      "activations/layer23_attention_weight_min": -25.79895782470703,
      "activations/layer2_attention_weight_max": 31.249099731445312,
      "activations/layer2_attention_weight_min": -30.170127868652344,
      "activations/layer3_attention_weight_max": 87.77157592773438,
      "activations/layer3_attention_weight_min": -83.8976821899414,
      "activations/layer4_attention_weight_max": 85.74127960205078,
      "activations/layer4_attention_weight_min": -83.48783874511719,
      "activations/layer5_attention_weight_max": 68.39186096191406,
      "activations/layer5_attention_weight_min": -74.00959777832031,
      "activations/layer6_attention_weight_max": 50.66376495361328,
      "activations/layer6_attention_weight_min": -50.03294372558594,
      "activations/layer7_attention_weight_max": 65.33121490478516,
      "activations/layer7_attention_weight_min": -63.66104507446289,
      "activations/layer8_attention_weight_max": 44.83170700073242,
      "activations/layer8_attention_weight_min": -48.778568267822266,
      "activations/layer9_attention_weight_max": 44.809261322021484,
      "activations/layer9_attention_weight_min": -44.14761734008789,
      "epoch": 16.37,
      "learning_rate": 4.4916287878787874e-05,
      "loss": 2.7588,
      "step": 281650
    },
    {
      "activations/layer0_attention_weight_max": 15.035465240478516,
      "activations/layer0_attention_weight_min": -13.629875183105469,
      "activations/layer10_attention_weight_max": 34.331939697265625,
      "activations/layer10_attention_weight_min": -33.94171142578125,
      "activations/layer11_attention_weight_max": 33.119876861572266,
      "activations/layer11_attention_weight_min": -33.52082824707031,
      "activations/layer12_attention_weight_max": 18.966503143310547,
      "activations/layer12_attention_weight_min": -23.35169219970703,
      "activations/layer13_attention_weight_max": 43.91907501220703,
      "activations/layer13_attention_weight_min": -28.374622344970703,
      "activations/layer14_attention_weight_max": 35.591217041015625,
      "activations/layer14_attention_weight_min": -29.04083251953125,
      "activations/layer15_attention_weight_max": 29.393169403076172,
      "activations/layer15_attention_weight_min": -30.04755973815918,
      "activations/layer16_attention_weight_max": 30.024356842041016,
      "activations/layer16_attention_weight_min": -30.35477638244629,
      "activations/layer17_attention_weight_max": 46.3641242980957,
      "activations/layer17_attention_weight_min": -43.69286346435547,
      "activations/layer18_attention_weight_max": 43.82964324951172,
      "activations/layer18_attention_weight_min": -38.39376449584961,
      "activations/layer19_attention_weight_max": 20.530012130737305,
      "activations/layer19_attention_weight_min": -22.021259307861328,
      "activations/layer1_attention_weight_max": 16.103239059448242,
      "activations/layer1_attention_weight_min": -13.581647872924805,
      "activations/layer20_attention_weight_max": 22.430845260620117,
      "activations/layer20_attention_weight_min": -22.689926147460938,
      "activations/layer21_attention_weight_max": 29.96503257751465,
      "activations/layer21_attention_weight_min": -20.612707138061523,
      "activations/layer22_attention_weight_max": 28.62180519104004,
      "activations/layer22_attention_weight_min": -23.23116683959961,
      "activations/layer23_attention_weight_max": 36.563663482666016,
      "activations/layer23_attention_weight_min": -21.113527297973633,
      "activations/layer2_attention_weight_max": 29.30902862548828,
      "activations/layer2_attention_weight_min": -27.27503204345703,
      "activations/layer3_attention_weight_max": 87.43873596191406,
      "activations/layer3_attention_weight_min": -86.65962219238281,
      "activations/layer4_attention_weight_max": 90.02942657470703,
      "activations/layer4_attention_weight_min": -88.53417205810547,
      "activations/layer5_attention_weight_max": 66.0781021118164,
      "activations/layer5_attention_weight_min": -77.62185668945312,
      "activations/layer6_attention_weight_max": 50.19612121582031,
      "activations/layer6_attention_weight_min": -50.47389221191406,
      "activations/layer7_attention_weight_max": 64.56068420410156,
      "activations/layer7_attention_weight_min": -63.08391571044922,
      "activations/layer8_attention_weight_max": 45.94733428955078,
      "activations/layer8_attention_weight_min": -46.91680908203125,
      "activations/layer9_attention_weight_max": 47.51827621459961,
      "activations/layer9_attention_weight_min": -46.07144546508789,
      "epoch": 16.37,
      "learning_rate": 4.489734848484848e-05,
      "loss": 2.7386,
      "step": 281700
    },
    {
      "activations/layer0_attention_weight_max": 16.983074188232422,
      "activations/layer0_attention_weight_min": -13.719621658325195,
      "activations/layer10_attention_weight_max": 34.18033981323242,
      "activations/layer10_attention_weight_min": -34.745574951171875,
      "activations/layer11_attention_weight_max": 30.904020309448242,
      "activations/layer11_attention_weight_min": -32.071144104003906,
      "activations/layer12_attention_weight_max": 18.389467239379883,
      "activations/layer12_attention_weight_min": -26.582176208496094,
      "activations/layer13_attention_weight_max": 39.93694305419922,
      "activations/layer13_attention_weight_min": -28.890024185180664,
      "activations/layer14_attention_weight_max": 37.051666259765625,
      "activations/layer14_attention_weight_min": -30.073087692260742,
      "activations/layer15_attention_weight_max": 32.02436447143555,
      "activations/layer15_attention_weight_min": -29.770370483398438,
      "activations/layer16_attention_weight_max": 34.2844352722168,
      "activations/layer16_attention_weight_min": -31.62512969970703,
      "activations/layer17_attention_weight_max": 53.068511962890625,
      "activations/layer17_attention_weight_min": -46.49612808227539,
      "activations/layer18_attention_weight_max": 50.38730239868164,
      "activations/layer18_attention_weight_min": -39.34748458862305,
      "activations/layer19_attention_weight_max": 26.401992797851562,
      "activations/layer19_attention_weight_min": -24.0537166595459,
      "activations/layer1_attention_weight_max": 16.037723541259766,
      "activations/layer1_attention_weight_min": -14.552038192749023,
      "activations/layer20_attention_weight_max": 24.793928146362305,
      "activations/layer20_attention_weight_min": -22.8994083404541,
      "activations/layer21_attention_weight_max": 38.150962829589844,
      "activations/layer21_attention_weight_min": -23.234079360961914,
      "activations/layer22_attention_weight_max": 32.094810485839844,
      "activations/layer22_attention_weight_min": -25.67435646057129,
      "activations/layer23_attention_weight_max": 39.31890869140625,
      "activations/layer23_attention_weight_min": -27.621810913085938,
      "activations/layer2_attention_weight_max": 29.28351593017578,
      "activations/layer2_attention_weight_min": -27.554107666015625,
      "activations/layer3_attention_weight_max": 80.50960540771484,
      "activations/layer3_attention_weight_min": -80.67144775390625,
      "activations/layer4_attention_weight_max": 81.67413330078125,
      "activations/layer4_attention_weight_min": -80.96250915527344,
      "activations/layer5_attention_weight_max": 67.1993408203125,
      "activations/layer5_attention_weight_min": -69.91215515136719,
      "activations/layer6_attention_weight_max": 47.59827423095703,
      "activations/layer6_attention_weight_min": -47.63071060180664,
      "activations/layer7_attention_weight_max": 60.191749572753906,
      "activations/layer7_attention_weight_min": -60.20805358886719,
      "activations/layer8_attention_weight_max": 44.7147216796875,
      "activations/layer8_attention_weight_min": -51.355587005615234,
      "activations/layer9_attention_weight_max": 41.4099006652832,
      "activations/layer9_attention_weight_min": -44.54294967651367,
      "epoch": 16.37,
      "learning_rate": 4.487878787878787e-05,
      "loss": 2.7306,
      "step": 281750
    },
    {
      "activations/layer0_attention_weight_max": 16.53211784362793,
      "activations/layer0_attention_weight_min": -13.785036087036133,
      "activations/layer10_attention_weight_max": 36.222721099853516,
      "activations/layer10_attention_weight_min": -37.34999465942383,
      "activations/layer11_attention_weight_max": 31.765287399291992,
      "activations/layer11_attention_weight_min": -35.188819885253906,
      "activations/layer12_attention_weight_max": 18.16486930847168,
      "activations/layer12_attention_weight_min": -27.033781051635742,
      "activations/layer13_attention_weight_max": 34.23188400268555,
      "activations/layer13_attention_weight_min": -27.333234786987305,
      "activations/layer14_attention_weight_max": 41.15939712524414,
      "activations/layer14_attention_weight_min": -30.414453506469727,
      "activations/layer15_attention_weight_max": 35.59189987182617,
      "activations/layer15_attention_weight_min": -29.795820236206055,
      "activations/layer16_attention_weight_max": 34.70363998413086,
      "activations/layer16_attention_weight_min": -31.655529022216797,
      "activations/layer17_attention_weight_max": 50.64145278930664,
      "activations/layer17_attention_weight_min": -43.944602966308594,
      "activations/layer18_attention_weight_max": 49.032596588134766,
      "activations/layer18_attention_weight_min": -39.4229850769043,
      "activations/layer19_attention_weight_max": 23.632003784179688,
      "activations/layer19_attention_weight_min": -22.45800018310547,
      "activations/layer1_attention_weight_max": 15.79155158996582,
      "activations/layer1_attention_weight_min": -16.269439697265625,
      "activations/layer20_attention_weight_max": 24.140764236450195,
      "activations/layer20_attention_weight_min": -22.188886642456055,
      "activations/layer21_attention_weight_max": 38.145545959472656,
      "activations/layer21_attention_weight_min": -22.9411678314209,
      "activations/layer22_attention_weight_max": 32.20771789550781,
      "activations/layer22_attention_weight_min": -25.0355224609375,
      "activations/layer23_attention_weight_max": 37.65928649902344,
      "activations/layer23_attention_weight_min": -24.001516342163086,
      "activations/layer2_attention_weight_max": 30.072484970092773,
      "activations/layer2_attention_weight_min": -29.74200439453125,
      "activations/layer3_attention_weight_max": 84.96200561523438,
      "activations/layer3_attention_weight_min": -89.25311279296875,
      "activations/layer4_attention_weight_max": 84.44125366210938,
      "activations/layer4_attention_weight_min": -90.84402465820312,
      "activations/layer5_attention_weight_max": 63.0984001159668,
      "activations/layer5_attention_weight_min": -72.85264587402344,
      "activations/layer6_attention_weight_max": 49.40934371948242,
      "activations/layer6_attention_weight_min": -51.424049377441406,
      "activations/layer7_attention_weight_max": 69.30261993408203,
      "activations/layer7_attention_weight_min": -62.670528411865234,
      "activations/layer8_attention_weight_max": 46.01093673706055,
      "activations/layer8_attention_weight_min": -51.48274612426758,
      "activations/layer9_attention_weight_max": 43.49018478393555,
      "activations/layer9_attention_weight_min": -48.307594299316406,
      "epoch": 16.37,
      "learning_rate": 4.485984848484849e-05,
      "loss": 2.7497,
      "step": 281800
    },
    {
      "activations/layer0_attention_weight_max": 16.371999740600586,
      "activations/layer0_attention_weight_min": -13.659333229064941,
      "activations/layer10_attention_weight_max": 37.64980697631836,
      "activations/layer10_attention_weight_min": -35.5318717956543,
      "activations/layer11_attention_weight_max": 33.208621978759766,
      "activations/layer11_attention_weight_min": -32.77045440673828,
      "activations/layer12_attention_weight_max": 19.46670913696289,
      "activations/layer12_attention_weight_min": -27.289169311523438,
      "activations/layer13_attention_weight_max": 36.88086700439453,
      "activations/layer13_attention_weight_min": -27.155786514282227,
      "activations/layer14_attention_weight_max": 37.8630256652832,
      "activations/layer14_attention_weight_min": -29.063735961914062,
      "activations/layer15_attention_weight_max": 35.34984588623047,
      "activations/layer15_attention_weight_min": -28.344144821166992,
      "activations/layer16_attention_weight_max": 31.99143409729004,
      "activations/layer16_attention_weight_min": -30.361169815063477,
      "activations/layer17_attention_weight_max": 50.8321418762207,
      "activations/layer17_attention_weight_min": -42.31413650512695,
      "activations/layer18_attention_weight_max": 46.63555908203125,
      "activations/layer18_attention_weight_min": -37.72529220581055,
      "activations/layer19_attention_weight_max": 24.956485748291016,
      "activations/layer19_attention_weight_min": -20.749528884887695,
      "activations/layer1_attention_weight_max": 15.028406143188477,
      "activations/layer1_attention_weight_min": -12.283205032348633,
      "activations/layer20_attention_weight_max": 22.567697525024414,
      "activations/layer20_attention_weight_min": -21.526514053344727,
      "activations/layer21_attention_weight_max": 34.385231018066406,
      "activations/layer21_attention_weight_min": -22.28557777404785,
      "activations/layer22_attention_weight_max": 31.86789894104004,
      "activations/layer22_attention_weight_min": -26.59701919555664,
      "activations/layer23_attention_weight_max": 37.13604736328125,
      "activations/layer23_attention_weight_min": -24.84369659423828,
      "activations/layer2_attention_weight_max": 30.58877944946289,
      "activations/layer2_attention_weight_min": -28.6793270111084,
      "activations/layer3_attention_weight_max": 88.21857452392578,
      "activations/layer3_attention_weight_min": -83.87638092041016,
      "activations/layer4_attention_weight_max": 89.54145050048828,
      "activations/layer4_attention_weight_min": -87.44770050048828,
      "activations/layer5_attention_weight_max": 64.69971466064453,
      "activations/layer5_attention_weight_min": -81.4713134765625,
      "activations/layer6_attention_weight_max": 49.059425354003906,
      "activations/layer6_attention_weight_min": -49.81877517700195,
      "activations/layer7_attention_weight_max": 66.78334045410156,
      "activations/layer7_attention_weight_min": -68.37779235839844,
      "activations/layer8_attention_weight_max": 46.570892333984375,
      "activations/layer8_attention_weight_min": -48.71419143676758,
      "activations/layer9_attention_weight_max": 46.78193664550781,
      "activations/layer9_attention_weight_min": -45.99999237060547,
      "epoch": 16.38,
      "learning_rate": 4.484090909090909e-05,
      "loss": 2.7505,
      "step": 281850
    },
    {
      "activations/layer0_attention_weight_max": 16.693628311157227,
      "activations/layer0_attention_weight_min": -14.524599075317383,
      "activations/layer10_attention_weight_max": 33.213932037353516,
      "activations/layer10_attention_weight_min": -34.936397552490234,
      "activations/layer11_attention_weight_max": 31.581724166870117,
      "activations/layer11_attention_weight_min": -31.178359985351562,
      "activations/layer12_attention_weight_max": 17.99136734008789,
      "activations/layer12_attention_weight_min": -27.121395111083984,
      "activations/layer13_attention_weight_max": 34.11752700805664,
      "activations/layer13_attention_weight_min": -29.088321685791016,
      "activations/layer14_attention_weight_max": 33.66565704345703,
      "activations/layer14_attention_weight_min": -31.714872360229492,
      "activations/layer15_attention_weight_max": 32.70952224731445,
      "activations/layer15_attention_weight_min": -30.65760612487793,
      "activations/layer16_attention_weight_max": 33.2136116027832,
      "activations/layer16_attention_weight_min": -30.404016494750977,
      "activations/layer17_attention_weight_max": 52.787715911865234,
      "activations/layer17_attention_weight_min": -43.98109436035156,
      "activations/layer18_attention_weight_max": 48.0167350769043,
      "activations/layer18_attention_weight_min": -40.742950439453125,
      "activations/layer19_attention_weight_max": 23.11004638671875,
      "activations/layer19_attention_weight_min": -24.626832962036133,
      "activations/layer1_attention_weight_max": 15.978371620178223,
      "activations/layer1_attention_weight_min": -12.423364639282227,
      "activations/layer20_attention_weight_max": 22.935632705688477,
      "activations/layer20_attention_weight_min": -23.393917083740234,
      "activations/layer21_attention_weight_max": 40.83109664916992,
      "activations/layer21_attention_weight_min": -26.209348678588867,
      "activations/layer22_attention_weight_max": 31.425275802612305,
      "activations/layer22_attention_weight_min": -29.465173721313477,
      "activations/layer23_attention_weight_max": 37.917057037353516,
      "activations/layer23_attention_weight_min": -27.016258239746094,
      "activations/layer2_attention_weight_max": 29.73749351501465,
      "activations/layer2_attention_weight_min": -27.88654327392578,
      "activations/layer3_attention_weight_max": 84.02349090576172,
      "activations/layer3_attention_weight_min": -87.64596557617188,
      "activations/layer4_attention_weight_max": 79.83228302001953,
      "activations/layer4_attention_weight_min": -87.87628936767578,
      "activations/layer5_attention_weight_max": 61.13759994506836,
      "activations/layer5_attention_weight_min": -78.60087585449219,
      "activations/layer6_attention_weight_max": 46.16734313964844,
      "activations/layer6_attention_weight_min": -47.33206558227539,
      "activations/layer7_attention_weight_max": 60.521854400634766,
      "activations/layer7_attention_weight_min": -58.49204635620117,
      "activations/layer8_attention_weight_max": 41.38982009887695,
      "activations/layer8_attention_weight_min": -44.9055061340332,
      "activations/layer9_attention_weight_max": 40.846614837646484,
      "activations/layer9_attention_weight_min": -42.419593811035156,
      "epoch": 16.38,
      "learning_rate": 4.482196969696969e-05,
      "loss": 2.7459,
      "step": 281900
    },
    {
      "activations/layer0_attention_weight_max": 15.654887199401855,
      "activations/layer0_attention_weight_min": -13.859905242919922,
      "activations/layer10_attention_weight_max": 37.48529052734375,
      "activations/layer10_attention_weight_min": -36.398475646972656,
      "activations/layer11_attention_weight_max": 35.11113739013672,
      "activations/layer11_attention_weight_min": -33.84904098510742,
      "activations/layer12_attention_weight_max": 18.179014205932617,
      "activations/layer12_attention_weight_min": -25.710067749023438,
      "activations/layer13_attention_weight_max": 31.98607063293457,
      "activations/layer13_attention_weight_min": -27.983285903930664,
      "activations/layer14_attention_weight_max": 33.81904602050781,
      "activations/layer14_attention_weight_min": -31.028478622436523,
      "activations/layer15_attention_weight_max": 30.34281349182129,
      "activations/layer15_attention_weight_min": -30.65513801574707,
      "activations/layer16_attention_weight_max": 30.88713836669922,
      "activations/layer16_attention_weight_min": -30.19108772277832,
      "activations/layer17_attention_weight_max": 47.63349151611328,
      "activations/layer17_attention_weight_min": -41.735782623291016,
      "activations/layer18_attention_weight_max": 44.855369567871094,
      "activations/layer18_attention_weight_min": -38.70661544799805,
      "activations/layer19_attention_weight_max": 22.3881778717041,
      "activations/layer19_attention_weight_min": -22.640775680541992,
      "activations/layer1_attention_weight_max": 15.362333297729492,
      "activations/layer1_attention_weight_min": -14.863890647888184,
      "activations/layer20_attention_weight_max": 24.43874740600586,
      "activations/layer20_attention_weight_min": -22.12552261352539,
      "activations/layer21_attention_weight_max": 37.495052337646484,
      "activations/layer21_attention_weight_min": -22.80983543395996,
      "activations/layer22_attention_weight_max": 25.3703556060791,
      "activations/layer22_attention_weight_min": -25.034740447998047,
      "activations/layer23_attention_weight_max": 33.73970413208008,
      "activations/layer23_attention_weight_min": -22.84854507446289,
      "activations/layer2_attention_weight_max": 31.07620620727539,
      "activations/layer2_attention_weight_min": -30.761037826538086,
      "activations/layer3_attention_weight_max": 88.9631576538086,
      "activations/layer3_attention_weight_min": -91.21922302246094,
      "activations/layer4_attention_weight_max": 87.23851776123047,
      "activations/layer4_attention_weight_min": -87.11396789550781,
      "activations/layer5_attention_weight_max": 62.459651947021484,
      "activations/layer5_attention_weight_min": -74.93307495117188,
      "activations/layer6_attention_weight_max": 52.72451400756836,
      "activations/layer6_attention_weight_min": -50.829524993896484,
      "activations/layer7_attention_weight_max": 62.905189514160156,
      "activations/layer7_attention_weight_min": -66.9764175415039,
      "activations/layer8_attention_weight_max": 48.866180419921875,
      "activations/layer8_attention_weight_min": -50.11142349243164,
      "activations/layer9_attention_weight_max": 47.04295349121094,
      "activations/layer9_attention_weight_min": -45.401832580566406,
      "epoch": 16.38,
      "learning_rate": 4.48030303030303e-05,
      "loss": 2.73,
      "step": 281950
    },
    {
      "activations/layer0_attention_weight_max": 16.516246795654297,
      "activations/layer0_attention_weight_min": -13.538325309753418,
      "activations/layer10_attention_weight_max": 38.78737258911133,
      "activations/layer10_attention_weight_min": -37.80857467651367,
      "activations/layer11_attention_weight_max": 34.19757080078125,
      "activations/layer11_attention_weight_min": -34.779747009277344,
      "activations/layer12_attention_weight_max": 19.64339828491211,
      "activations/layer12_attention_weight_min": -22.79743766784668,
      "activations/layer13_attention_weight_max": 37.69499588012695,
      "activations/layer13_attention_weight_min": -28.043102264404297,
      "activations/layer14_attention_weight_max": 35.78179931640625,
      "activations/layer14_attention_weight_min": -27.98552703857422,
      "activations/layer15_attention_weight_max": 34.972286224365234,
      "activations/layer15_attention_weight_min": -29.561769485473633,
      "activations/layer16_attention_weight_max": 34.493812561035156,
      "activations/layer16_attention_weight_min": -30.09447479248047,
      "activations/layer17_attention_weight_max": 52.766239166259766,
      "activations/layer17_attention_weight_min": -42.37678146362305,
      "activations/layer18_attention_weight_max": 47.80205154418945,
      "activations/layer18_attention_weight_min": -38.44905471801758,
      "activations/layer19_attention_weight_max": 23.904430389404297,
      "activations/layer19_attention_weight_min": -23.761510848999023,
      "activations/layer1_attention_weight_max": 15.909021377563477,
      "activations/layer1_attention_weight_min": -14.87587833404541,
      "activations/layer20_attention_weight_max": 25.486045837402344,
      "activations/layer20_attention_weight_min": -25.986454010009766,
      "activations/layer21_attention_weight_max": 34.77362060546875,
      "activations/layer21_attention_weight_min": -22.702606201171875,
      "activations/layer22_attention_weight_max": 27.863422393798828,
      "activations/layer22_attention_weight_min": -25.69474220275879,
      "activations/layer23_attention_weight_max": 38.34355163574219,
      "activations/layer23_attention_weight_min": -24.18439483642578,
      "activations/layer2_attention_weight_max": 31.112911224365234,
      "activations/layer2_attention_weight_min": -30.75259017944336,
      "activations/layer3_attention_weight_max": 91.7799301147461,
      "activations/layer3_attention_weight_min": -92.32233428955078,
      "activations/layer4_attention_weight_max": 90.9161148071289,
      "activations/layer4_attention_weight_min": -88.986572265625,
      "activations/layer5_attention_weight_max": 65.84827423095703,
      "activations/layer5_attention_weight_min": -70.01466369628906,
      "activations/layer6_attention_weight_max": 52.18124008178711,
      "activations/layer6_attention_weight_min": -52.45125198364258,
      "activations/layer7_attention_weight_max": 63.05915069580078,
      "activations/layer7_attention_weight_min": -64.20113372802734,
      "activations/layer8_attention_weight_max": 48.67060470581055,
      "activations/layer8_attention_weight_min": -51.20661163330078,
      "activations/layer9_attention_weight_max": 46.4730339050293,
      "activations/layer9_attention_weight_min": -48.81577682495117,
      "epoch": 16.39,
      "learning_rate": 4.478409090909091e-05,
      "loss": 2.7381,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4559,
      "eval_samples_per_second": 507.814,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4559,
      "eval_openwebtext_samples_per_second": 507.814,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9236,
      "eval_wikitext_samples_per_second": 237.059,
      "step": 282000
    },
    {
      "epoch": 16.39,
      "eval_lambada_loss": 2.54296875,
      "eval_lambada_ppl": 12.717369675355013,
      "eval_lambada_runtime": 9.5325,
      "eval_lambada_samples_per_second": 510.779,
      "step": 282000
    },
    {
      "activations/layer0_attention_weight_max": 14.955787658691406,
      "activations/layer0_attention_weight_min": -13.469858169555664,
      "activations/layer10_attention_weight_max": 33.39704513549805,
      "activations/layer10_attention_weight_min": -36.11349105834961,
      "activations/layer11_attention_weight_max": 32.973670959472656,
      "activations/layer11_attention_weight_min": -34.41591262817383,
      "activations/layer12_attention_weight_max": 18.308935165405273,
      "activations/layer12_attention_weight_min": -24.28802490234375,
      "activations/layer13_attention_weight_max": 34.41795349121094,
      "activations/layer13_attention_weight_min": -28.675731658935547,
      "activations/layer14_attention_weight_max": 34.92287063598633,
      "activations/layer14_attention_weight_min": -30.45948600769043,
      "activations/layer15_attention_weight_max": 35.19157409667969,
      "activations/layer15_attention_weight_min": -29.489654541015625,
      "activations/layer16_attention_weight_max": 32.092674255371094,
      "activations/layer16_attention_weight_min": -31.253772735595703,
      "activations/layer17_attention_weight_max": 51.01139831542969,
      "activations/layer17_attention_weight_min": -44.47206497192383,
      "activations/layer18_attention_weight_max": 44.724063873291016,
      "activations/layer18_attention_weight_min": -39.41876983642578,
      "activations/layer19_attention_weight_max": 24.623123168945312,
      "activations/layer19_attention_weight_min": -23.189327239990234,
      "activations/layer1_attention_weight_max": 15.955744743347168,
      "activations/layer1_attention_weight_min": -15.200934410095215,
      "activations/layer20_attention_weight_max": 23.325517654418945,
      "activations/layer20_attention_weight_min": -21.7252197265625,
      "activations/layer21_attention_weight_max": 44.80989456176758,
      "activations/layer21_attention_weight_min": -26.51279067993164,
      "activations/layer22_attention_weight_max": 30.555038452148438,
      "activations/layer22_attention_weight_min": -28.22551918029785,
      "activations/layer23_attention_weight_max": 36.776981353759766,
      "activations/layer23_attention_weight_min": -24.16271209716797,
      "activations/layer2_attention_weight_max": 30.685348510742188,
      "activations/layer2_attention_weight_min": -29.468223571777344,
      "activations/layer3_attention_weight_max": 84.28386688232422,
      "activations/layer3_attention_weight_min": -87.51666259765625,
      "activations/layer4_attention_weight_max": 82.82373809814453,
      "activations/layer4_attention_weight_min": -85.88610076904297,
      "activations/layer5_attention_weight_max": 63.34209442138672,
      "activations/layer5_attention_weight_min": -71.44105529785156,
      "activations/layer6_attention_weight_max": 48.139713287353516,
      "activations/layer6_attention_weight_min": -50.87422561645508,
      "activations/layer7_attention_weight_max": 61.43983840942383,
      "activations/layer7_attention_weight_min": -60.76372528076172,
      "activations/layer8_attention_weight_max": 44.13715744018555,
      "activations/layer8_attention_weight_min": -46.719295501708984,
      "activations/layer9_attention_weight_max": 42.8618278503418,
      "activations/layer9_attention_weight_min": -45.64674377441406,
      "epoch": 16.39,
      "learning_rate": 4.4765151515151516e-05,
      "loss": 2.7475,
      "step": 282050
    },
    {
      "activations/layer0_attention_weight_max": 15.326007843017578,
      "activations/layer0_attention_weight_min": -13.19587230682373,
      "activations/layer10_attention_weight_max": 35.609107971191406,
      "activations/layer10_attention_weight_min": -37.099464416503906,
      "activations/layer11_attention_weight_max": 34.45569610595703,
      "activations/layer11_attention_weight_min": -34.943016052246094,
      "activations/layer12_attention_weight_max": 19.103069305419922,
      "activations/layer12_attention_weight_min": -26.15610122680664,
      "activations/layer13_attention_weight_max": 34.279727935791016,
      "activations/layer13_attention_weight_min": -29.780223846435547,
      "activations/layer14_attention_weight_max": 37.02016067504883,
      "activations/layer14_attention_weight_min": -32.2591667175293,
      "activations/layer15_attention_weight_max": 34.213218688964844,
      "activations/layer15_attention_weight_min": -31.90212059020996,
      "activations/layer16_attention_weight_max": 34.56244659423828,
      "activations/layer16_attention_weight_min": -32.83833694458008,
      "activations/layer17_attention_weight_max": 52.80927658081055,
      "activations/layer17_attention_weight_min": -46.37044143676758,
      "activations/layer18_attention_weight_max": 47.316184997558594,
      "activations/layer18_attention_weight_min": -40.63005828857422,
      "activations/layer19_attention_weight_max": 25.611963272094727,
      "activations/layer19_attention_weight_min": -26.120756149291992,
      "activations/layer1_attention_weight_max": 18.09714698791504,
      "activations/layer1_attention_weight_min": -14.257119178771973,
      "activations/layer20_attention_weight_max": 22.74306297302246,
      "activations/layer20_attention_weight_min": -24.85268211364746,
      "activations/layer21_attention_weight_max": 37.01021194458008,
      "activations/layer21_attention_weight_min": -23.62811851501465,
      "activations/layer22_attention_weight_max": 31.120494842529297,
      "activations/layer22_attention_weight_min": -29.535560607910156,
      "activations/layer23_attention_weight_max": 38.335548400878906,
      "activations/layer23_attention_weight_min": -25.264732360839844,
      "activations/layer2_attention_weight_max": 30.136817932128906,
      "activations/layer2_attention_weight_min": -28.196800231933594,
      "activations/layer3_attention_weight_max": 86.83709716796875,
      "activations/layer3_attention_weight_min": -82.42573547363281,
      "activations/layer4_attention_weight_max": 86.23621368408203,
      "activations/layer4_attention_weight_min": -84.57244110107422,
      "activations/layer5_attention_weight_max": 64.51387023925781,
      "activations/layer5_attention_weight_min": -73.08772277832031,
      "activations/layer6_attention_weight_max": 50.03956604003906,
      "activations/layer6_attention_weight_min": -50.70912551879883,
      "activations/layer7_attention_weight_max": 61.3240852355957,
      "activations/layer7_attention_weight_min": -63.62866973876953,
      "activations/layer8_attention_weight_max": 44.88480758666992,
      "activations/layer8_attention_weight_min": -47.768211364746094,
      "activations/layer9_attention_weight_max": 45.66411209106445,
      "activations/layer9_attention_weight_min": -48.06008529663086,
      "epoch": 16.39,
      "learning_rate": 4.474621212121212e-05,
      "loss": 2.7367,
      "step": 282100
    },
    {
      "activations/layer0_attention_weight_max": 15.391186714172363,
      "activations/layer0_attention_weight_min": -13.879010200500488,
      "activations/layer10_attention_weight_max": 37.31853103637695,
      "activations/layer10_attention_weight_min": -36.73237991333008,
      "activations/layer11_attention_weight_max": 34.204864501953125,
      "activations/layer11_attention_weight_min": -35.5644416809082,
      "activations/layer12_attention_weight_max": 18.165607452392578,
      "activations/layer12_attention_weight_min": -23.64712905883789,
      "activations/layer13_attention_weight_max": 38.67177200317383,
      "activations/layer13_attention_weight_min": -32.79408264160156,
      "activations/layer14_attention_weight_max": 36.77370071411133,
      "activations/layer14_attention_weight_min": -29.884262084960938,
      "activations/layer15_attention_weight_max": 35.1290283203125,
      "activations/layer15_attention_weight_min": -30.320308685302734,
      "activations/layer16_attention_weight_max": 34.76718521118164,
      "activations/layer16_attention_weight_min": -31.757402420043945,
      "activations/layer17_attention_weight_max": 49.98389434814453,
      "activations/layer17_attention_weight_min": -45.4261360168457,
      "activations/layer18_attention_weight_max": 46.6819953918457,
      "activations/layer18_attention_weight_min": -40.03233337402344,
      "activations/layer19_attention_weight_max": 22.090383529663086,
      "activations/layer19_attention_weight_min": -23.7861270904541,
      "activations/layer1_attention_weight_max": 15.449316024780273,
      "activations/layer1_attention_weight_min": -14.013229370117188,
      "activations/layer20_attention_weight_max": 20.648191452026367,
      "activations/layer20_attention_weight_min": -21.68832015991211,
      "activations/layer21_attention_weight_max": 38.68439865112305,
      "activations/layer21_attention_weight_min": -22.503856658935547,
      "activations/layer22_attention_weight_max": 31.436738967895508,
      "activations/layer22_attention_weight_min": -25.51967430114746,
      "activations/layer23_attention_weight_max": 35.84368896484375,
      "activations/layer23_attention_weight_min": -24.772245407104492,
      "activations/layer2_attention_weight_max": 31.877365112304688,
      "activations/layer2_attention_weight_min": -29.458616256713867,
      "activations/layer3_attention_weight_max": 89.06396484375,
      "activations/layer3_attention_weight_min": -92.16205596923828,
      "activations/layer4_attention_weight_max": 91.70795440673828,
      "activations/layer4_attention_weight_min": -90.13400268554688,
      "activations/layer5_attention_weight_max": 67.22576904296875,
      "activations/layer5_attention_weight_min": -72.12916564941406,
      "activations/layer6_attention_weight_max": 51.7234992980957,
      "activations/layer6_attention_weight_min": -52.03050994873047,
      "activations/layer7_attention_weight_max": 67.3553237915039,
      "activations/layer7_attention_weight_min": -62.81278610229492,
      "activations/layer8_attention_weight_max": 47.0527458190918,
      "activations/layer8_attention_weight_min": -52.6942024230957,
      "activations/layer9_attention_weight_max": 45.371665954589844,
      "activations/layer9_attention_weight_min": -45.45954895019531,
      "epoch": 16.39,
      "learning_rate": 4.472727272727272e-05,
      "loss": 2.7435,
      "step": 282150
    },
    {
      "activations/layer0_attention_weight_max": 15.187153816223145,
      "activations/layer0_attention_weight_min": -14.08177375793457,
      "activations/layer10_attention_weight_max": 36.16651153564453,
      "activations/layer10_attention_weight_min": -36.92033767700195,
      "activations/layer11_attention_weight_max": 35.3060302734375,
      "activations/layer11_attention_weight_min": -35.96324920654297,
      "activations/layer12_attention_weight_max": 18.37307357788086,
      "activations/layer12_attention_weight_min": -32.80351257324219,
      "activations/layer13_attention_weight_max": 35.86650085449219,
      "activations/layer13_attention_weight_min": -29.857288360595703,
      "activations/layer14_attention_weight_max": 35.4503288269043,
      "activations/layer14_attention_weight_min": -29.920303344726562,
      "activations/layer15_attention_weight_max": 32.968788146972656,
      "activations/layer15_attention_weight_min": -30.629636764526367,
      "activations/layer16_attention_weight_max": 34.07102966308594,
      "activations/layer16_attention_weight_min": -31.70574188232422,
      "activations/layer17_attention_weight_max": 55.50273132324219,
      "activations/layer17_attention_weight_min": -46.54591751098633,
      "activations/layer18_attention_weight_max": 49.42461395263672,
      "activations/layer18_attention_weight_min": -41.10545349121094,
      "activations/layer19_attention_weight_max": 23.25982666015625,
      "activations/layer19_attention_weight_min": -24.60700798034668,
      "activations/layer1_attention_weight_max": 16.861896514892578,
      "activations/layer1_attention_weight_min": -14.610309600830078,
      "activations/layer20_attention_weight_max": 22.71851921081543,
      "activations/layer20_attention_weight_min": -22.552379608154297,
      "activations/layer21_attention_weight_max": 34.76279830932617,
      "activations/layer21_attention_weight_min": -22.43555450439453,
      "activations/layer22_attention_weight_max": 29.879161834716797,
      "activations/layer22_attention_weight_min": -26.68181610107422,
      "activations/layer23_attention_weight_max": 42.013790130615234,
      "activations/layer23_attention_weight_min": -27.344993591308594,
      "activations/layer2_attention_weight_max": 29.504894256591797,
      "activations/layer2_attention_weight_min": -27.570158004760742,
      "activations/layer3_attention_weight_max": 82.24027252197266,
      "activations/layer3_attention_weight_min": -82.07243347167969,
      "activations/layer4_attention_weight_max": 86.60688781738281,
      "activations/layer4_attention_weight_min": -84.07364654541016,
      "activations/layer5_attention_weight_max": 64.88063049316406,
      "activations/layer5_attention_weight_min": -64.35552215576172,
      "activations/layer6_attention_weight_max": 48.01721954345703,
      "activations/layer6_attention_weight_min": -51.0930061340332,
      "activations/layer7_attention_weight_max": 61.89815902709961,
      "activations/layer7_attention_weight_min": -61.931373596191406,
      "activations/layer8_attention_weight_max": 45.193565368652344,
      "activations/layer8_attention_weight_min": -44.9598274230957,
      "activations/layer9_attention_weight_max": 49.994449615478516,
      "activations/layer9_attention_weight_min": -45.9444694519043,
      "epoch": 16.4,
      "learning_rate": 4.4708333333333334e-05,
      "loss": 2.7437,
      "step": 282200
    },
    {
      "activations/layer0_attention_weight_max": 15.621670722961426,
      "activations/layer0_attention_weight_min": -13.833361625671387,
      "activations/layer10_attention_weight_max": 37.167083740234375,
      "activations/layer10_attention_weight_min": -36.223697662353516,
      "activations/layer11_attention_weight_max": 33.06525421142578,
      "activations/layer11_attention_weight_min": -34.10459899902344,
      "activations/layer12_attention_weight_max": 17.831314086914062,
      "activations/layer12_attention_weight_min": -23.823190689086914,
      "activations/layer13_attention_weight_max": 29.181203842163086,
      "activations/layer13_attention_weight_min": -28.049537658691406,
      "activations/layer14_attention_weight_max": 33.860870361328125,
      "activations/layer14_attention_weight_min": -32.50265121459961,
      "activations/layer15_attention_weight_max": 30.417125701904297,
      "activations/layer15_attention_weight_min": -30.089454650878906,
      "activations/layer16_attention_weight_max": 33.37174606323242,
      "activations/layer16_attention_weight_min": -34.37854766845703,
      "activations/layer17_attention_weight_max": 52.03996276855469,
      "activations/layer17_attention_weight_min": -48.01640701293945,
      "activations/layer18_attention_weight_max": 44.107452392578125,
      "activations/layer18_attention_weight_min": -39.81596374511719,
      "activations/layer19_attention_weight_max": 22.23766326904297,
      "activations/layer19_attention_weight_min": -23.316638946533203,
      "activations/layer1_attention_weight_max": 16.045230865478516,
      "activations/layer1_attention_weight_min": -13.640629768371582,
      "activations/layer20_attention_weight_max": 19.575651168823242,
      "activations/layer20_attention_weight_min": -23.037046432495117,
      "activations/layer21_attention_weight_max": 28.58052635192871,
      "activations/layer21_attention_weight_min": -20.02927017211914,
      "activations/layer22_attention_weight_max": 26.58625602722168,
      "activations/layer22_attention_weight_min": -25.864215850830078,
      "activations/layer23_attention_weight_max": 34.09575271606445,
      "activations/layer23_attention_weight_min": -24.29926300048828,
      "activations/layer2_attention_weight_max": 30.13692283630371,
      "activations/layer2_attention_weight_min": -29.288917541503906,
      "activations/layer3_attention_weight_max": 87.23524475097656,
      "activations/layer3_attention_weight_min": -88.8819808959961,
      "activations/layer4_attention_weight_max": 88.9760513305664,
      "activations/layer4_attention_weight_min": -89.75975799560547,
      "activations/layer5_attention_weight_max": 62.54004669189453,
      "activations/layer5_attention_weight_min": -80.23146057128906,
      "activations/layer6_attention_weight_max": 47.1400260925293,
      "activations/layer6_attention_weight_min": -50.754371643066406,
      "activations/layer7_attention_weight_max": 61.908695220947266,
      "activations/layer7_attention_weight_min": -64.88984680175781,
      "activations/layer8_attention_weight_max": 45.708621978759766,
      "activations/layer8_attention_weight_min": -49.47072982788086,
      "activations/layer9_attention_weight_max": 42.781028747558594,
      "activations/layer9_attention_weight_min": -48.08119583129883,
      "epoch": 16.4,
      "learning_rate": 4.4689393939393935e-05,
      "loss": 2.7393,
      "step": 282250
    },
    {
      "activations/layer0_attention_weight_max": 15.707769393920898,
      "activations/layer0_attention_weight_min": -13.596651077270508,
      "activations/layer10_attention_weight_max": 34.0406608581543,
      "activations/layer10_attention_weight_min": -32.9141731262207,
      "activations/layer11_attention_weight_max": 33.58876037597656,
      "activations/layer11_attention_weight_min": -33.26350402832031,
      "activations/layer12_attention_weight_max": 18.409364700317383,
      "activations/layer12_attention_weight_min": -28.307296752929688,
      "activations/layer13_attention_weight_max": 31.768587112426758,
      "activations/layer13_attention_weight_min": -27.48198127746582,
      "activations/layer14_attention_weight_max": 33.57344055175781,
      "activations/layer14_attention_weight_min": -31.922229766845703,
      "activations/layer15_attention_weight_max": 31.544239044189453,
      "activations/layer15_attention_weight_min": -30.713926315307617,
      "activations/layer16_attention_weight_max": 34.57931900024414,
      "activations/layer16_attention_weight_min": -32.33790969848633,
      "activations/layer17_attention_weight_max": 52.654815673828125,
      "activations/layer17_attention_weight_min": -46.16725540161133,
      "activations/layer18_attention_weight_max": 46.272090911865234,
      "activations/layer18_attention_weight_min": -41.69097137451172,
      "activations/layer19_attention_weight_max": 23.710765838623047,
      "activations/layer19_attention_weight_min": -25.266216278076172,
      "activations/layer1_attention_weight_max": 15.965543746948242,
      "activations/layer1_attention_weight_min": -14.62136459350586,
      "activations/layer20_attention_weight_max": 24.851755142211914,
      "activations/layer20_attention_weight_min": -23.29775619506836,
      "activations/layer21_attention_weight_max": 41.143741607666016,
      "activations/layer21_attention_weight_min": -26.11003303527832,
      "activations/layer22_attention_weight_max": 28.881818771362305,
      "activations/layer22_attention_weight_min": -27.14365577697754,
      "activations/layer23_attention_weight_max": 36.04092025756836,
      "activations/layer23_attention_weight_min": -26.2000789642334,
      "activations/layer2_attention_weight_max": 31.48202133178711,
      "activations/layer2_attention_weight_min": -28.874996185302734,
      "activations/layer3_attention_weight_max": 87.15493774414062,
      "activations/layer3_attention_weight_min": -86.49658966064453,
      "activations/layer4_attention_weight_max": 87.28861236572266,
      "activations/layer4_attention_weight_min": -84.10508728027344,
      "activations/layer5_attention_weight_max": 62.867042541503906,
      "activations/layer5_attention_weight_min": -70.76753234863281,
      "activations/layer6_attention_weight_max": 47.83657455444336,
      "activations/layer6_attention_weight_min": -48.454689025878906,
      "activations/layer7_attention_weight_max": 60.96888732910156,
      "activations/layer7_attention_weight_min": -59.406280517578125,
      "activations/layer8_attention_weight_max": 44.9459114074707,
      "activations/layer8_attention_weight_min": -46.55110168457031,
      "activations/layer9_attention_weight_max": 41.155860900878906,
      "activations/layer9_attention_weight_min": -46.506046295166016,
      "epoch": 16.4,
      "learning_rate": 4.4670454545454537e-05,
      "loss": 2.7418,
      "step": 282300
    },
    {
      "activations/layer0_attention_weight_max": 15.606324195861816,
      "activations/layer0_attention_weight_min": -13.81164836883545,
      "activations/layer10_attention_weight_max": 38.056663513183594,
      "activations/layer10_attention_weight_min": -35.99753189086914,
      "activations/layer11_attention_weight_max": 34.593475341796875,
      "activations/layer11_attention_weight_min": -33.465431213378906,
      "activations/layer12_attention_weight_max": 19.457063674926758,
      "activations/layer12_attention_weight_min": -23.82760238647461,
      "activations/layer13_attention_weight_max": 36.509246826171875,
      "activations/layer13_attention_weight_min": -26.764097213745117,
      "activations/layer14_attention_weight_max": 34.23075866699219,
      "activations/layer14_attention_weight_min": -29.7238826751709,
      "activations/layer15_attention_weight_max": 32.44126892089844,
      "activations/layer15_attention_weight_min": -30.00600814819336,
      "activations/layer16_attention_weight_max": 32.674827575683594,
      "activations/layer16_attention_weight_min": -30.791902542114258,
      "activations/layer17_attention_weight_max": 50.10152053833008,
      "activations/layer17_attention_weight_min": -42.97168731689453,
      "activations/layer18_attention_weight_max": 46.304256439208984,
      "activations/layer18_attention_weight_min": -38.25897216796875,
      "activations/layer19_attention_weight_max": 22.993518829345703,
      "activations/layer19_attention_weight_min": -23.154699325561523,
      "activations/layer1_attention_weight_max": 16.13690757751465,
      "activations/layer1_attention_weight_min": -15.04091739654541,
      "activations/layer20_attention_weight_max": 23.742462158203125,
      "activations/layer20_attention_weight_min": -23.079654693603516,
      "activations/layer21_attention_weight_max": 36.86696243286133,
      "activations/layer21_attention_weight_min": -24.299795150756836,
      "activations/layer22_attention_weight_max": 30.04148292541504,
      "activations/layer22_attention_weight_min": -27.06277084350586,
      "activations/layer23_attention_weight_max": 35.61450958251953,
      "activations/layer23_attention_weight_min": -22.65569305419922,
      "activations/layer2_attention_weight_max": 29.676212310791016,
      "activations/layer2_attention_weight_min": -27.564199447631836,
      "activations/layer3_attention_weight_max": 85.43938446044922,
      "activations/layer3_attention_weight_min": -81.71925354003906,
      "activations/layer4_attention_weight_max": 83.65074157714844,
      "activations/layer4_attention_weight_min": -81.38703918457031,
      "activations/layer5_attention_weight_max": 66.2229995727539,
      "activations/layer5_attention_weight_min": -70.68281555175781,
      "activations/layer6_attention_weight_max": 51.22224807739258,
      "activations/layer6_attention_weight_min": -49.76454544067383,
      "activations/layer7_attention_weight_max": 65.01622772216797,
      "activations/layer7_attention_weight_min": -59.67782974243164,
      "activations/layer8_attention_weight_max": 48.84587478637695,
      "activations/layer8_attention_weight_min": -45.037841796875,
      "activations/layer9_attention_weight_max": 46.07772445678711,
      "activations/layer9_attention_weight_min": -43.7572021484375,
      "epoch": 16.41,
      "learning_rate": 4.4651515151515145e-05,
      "loss": 2.7456,
      "step": 282350
    },
    {
      "activations/layer0_attention_weight_max": 16.685550689697266,
      "activations/layer0_attention_weight_min": -13.930717468261719,
      "activations/layer10_attention_weight_max": 35.10125732421875,
      "activations/layer10_attention_weight_min": -35.70220184326172,
      "activations/layer11_attention_weight_max": 31.981441497802734,
      "activations/layer11_attention_weight_min": -33.7989501953125,
      "activations/layer12_attention_weight_max": 18.329757690429688,
      "activations/layer12_attention_weight_min": -25.7066593170166,
      "activations/layer13_attention_weight_max": 35.06060028076172,
      "activations/layer13_attention_weight_min": -28.2965030670166,
      "activations/layer14_attention_weight_max": 34.180938720703125,
      "activations/layer14_attention_weight_min": -29.97593116760254,
      "activations/layer15_attention_weight_max": 29.917490005493164,
      "activations/layer15_attention_weight_min": -30.055217742919922,
      "activations/layer16_attention_weight_max": 34.799198150634766,
      "activations/layer16_attention_weight_min": -32.328529357910156,
      "activations/layer17_attention_weight_max": 51.430641174316406,
      "activations/layer17_attention_weight_min": -42.15085220336914,
      "activations/layer18_attention_weight_max": 45.36056900024414,
      "activations/layer18_attention_weight_min": -39.281951904296875,
      "activations/layer19_attention_weight_max": 23.222307205200195,
      "activations/layer19_attention_weight_min": -23.033321380615234,
      "activations/layer1_attention_weight_max": 15.84622573852539,
      "activations/layer1_attention_weight_min": -13.233382225036621,
      "activations/layer20_attention_weight_max": 22.43309783935547,
      "activations/layer20_attention_weight_min": -21.68617820739746,
      "activations/layer21_attention_weight_max": 31.70855712890625,
      "activations/layer21_attention_weight_min": -24.16847801208496,
      "activations/layer22_attention_weight_max": 27.27373504638672,
      "activations/layer22_attention_weight_min": -25.39200210571289,
      "activations/layer23_attention_weight_max": 38.46903610229492,
      "activations/layer23_attention_weight_min": -24.335493087768555,
      "activations/layer2_attention_weight_max": 30.444015502929688,
      "activations/layer2_attention_weight_min": -29.48909568786621,
      "activations/layer3_attention_weight_max": 84.28753662109375,
      "activations/layer3_attention_weight_min": -84.67768859863281,
      "activations/layer4_attention_weight_max": 85.6869125366211,
      "activations/layer4_attention_weight_min": -83.87703704833984,
      "activations/layer5_attention_weight_max": 65.3515853881836,
      "activations/layer5_attention_weight_min": -71.790771484375,
      "activations/layer6_attention_weight_max": 49.5994987487793,
      "activations/layer6_attention_weight_min": -48.807491302490234,
      "activations/layer7_attention_weight_max": 61.7941780090332,
      "activations/layer7_attention_weight_min": -60.205387115478516,
      "activations/layer8_attention_weight_max": 43.45511245727539,
      "activations/layer8_attention_weight_min": -48.2436637878418,
      "activations/layer9_attention_weight_max": 43.939857482910156,
      "activations/layer9_attention_weight_min": -44.8918342590332,
      "epoch": 16.41,
      "learning_rate": 4.463257575757575e-05,
      "loss": 2.7444,
      "step": 282400
    },
    {
      "activations/layer0_attention_weight_max": 15.777793884277344,
      "activations/layer0_attention_weight_min": -14.105440139770508,
      "activations/layer10_attention_weight_max": 33.1439094543457,
      "activations/layer10_attention_weight_min": -34.5107307434082,
      "activations/layer11_attention_weight_max": 32.08045196533203,
      "activations/layer11_attention_weight_min": -32.43681335449219,
      "activations/layer12_attention_weight_max": 19.586017608642578,
      "activations/layer12_attention_weight_min": -25.30476188659668,
      "activations/layer13_attention_weight_max": 32.12849044799805,
      "activations/layer13_attention_weight_min": -25.74295997619629,
      "activations/layer14_attention_weight_max": 34.990474700927734,
      "activations/layer14_attention_weight_min": -30.928922653198242,
      "activations/layer15_attention_weight_max": 31.111343383789062,
      "activations/layer15_attention_weight_min": -31.412887573242188,
      "activations/layer16_attention_weight_max": 33.86977005004883,
      "activations/layer16_attention_weight_min": -30.565616607666016,
      "activations/layer17_attention_weight_max": 48.86671447753906,
      "activations/layer17_attention_weight_min": -45.51058578491211,
      "activations/layer18_attention_weight_max": 45.76061248779297,
      "activations/layer18_attention_weight_min": -39.28899383544922,
      "activations/layer19_attention_weight_max": 23.53438949584961,
      "activations/layer19_attention_weight_min": -23.73735809326172,
      "activations/layer1_attention_weight_max": 15.929505348205566,
      "activations/layer1_attention_weight_min": -13.917000770568848,
      "activations/layer20_attention_weight_max": 21.002403259277344,
      "activations/layer20_attention_weight_min": -26.34817123413086,
      "activations/layer21_attention_weight_max": 34.025150299072266,
      "activations/layer21_attention_weight_min": -25.171018600463867,
      "activations/layer22_attention_weight_max": 27.57500648498535,
      "activations/layer22_attention_weight_min": -26.850048065185547,
      "activations/layer23_attention_weight_max": 38.55253601074219,
      "activations/layer23_attention_weight_min": -27.387245178222656,
      "activations/layer2_attention_weight_max": 31.369731903076172,
      "activations/layer2_attention_weight_min": -27.93575668334961,
      "activations/layer3_attention_weight_max": 82.0309066772461,
      "activations/layer3_attention_weight_min": -78.66192626953125,
      "activations/layer4_attention_weight_max": 84.62361907958984,
      "activations/layer4_attention_weight_min": -79.93207550048828,
      "activations/layer5_attention_weight_max": 64.4981918334961,
      "activations/layer5_attention_weight_min": -74.82847595214844,
      "activations/layer6_attention_weight_max": 45.93275451660156,
      "activations/layer6_attention_weight_min": -46.46131896972656,
      "activations/layer7_attention_weight_max": 63.045753479003906,
      "activations/layer7_attention_weight_min": -57.07231140136719,
      "activations/layer8_attention_weight_max": 43.23683547973633,
      "activations/layer8_attention_weight_min": -45.23844528198242,
      "activations/layer9_attention_weight_max": 42.475582122802734,
      "activations/layer9_attention_weight_min": -44.45673751831055,
      "epoch": 16.41,
      "learning_rate": 4.461363636363636e-05,
      "loss": 2.7605,
      "step": 282450
    },
    {
      "activations/layer0_attention_weight_max": 16.227096557617188,
      "activations/layer0_attention_weight_min": -13.987968444824219,
      "activations/layer10_attention_weight_max": 36.7235221862793,
      "activations/layer10_attention_weight_min": -35.323036193847656,
      "activations/layer11_attention_weight_max": 33.676395416259766,
      "activations/layer11_attention_weight_min": -33.23905563354492,
      "activations/layer12_attention_weight_max": 16.84044075012207,
      "activations/layer12_attention_weight_min": -22.959857940673828,
      "activations/layer13_attention_weight_max": 30.866884231567383,
      "activations/layer13_attention_weight_min": -27.031503677368164,
      "activations/layer14_attention_weight_max": 32.34602355957031,
      "activations/layer14_attention_weight_min": -30.561325073242188,
      "activations/layer15_attention_weight_max": 30.288272857666016,
      "activations/layer15_attention_weight_min": -29.711990356445312,
      "activations/layer16_attention_weight_max": 30.114660263061523,
      "activations/layer16_attention_weight_min": -29.7662353515625,
      "activations/layer17_attention_weight_max": 46.439002990722656,
      "activations/layer17_attention_weight_min": -43.342708587646484,
      "activations/layer18_attention_weight_max": 42.629791259765625,
      "activations/layer18_attention_weight_min": -37.761959075927734,
      "activations/layer19_attention_weight_max": 25.09307861328125,
      "activations/layer19_attention_weight_min": -21.99473762512207,
      "activations/layer1_attention_weight_max": 16.19796371459961,
      "activations/layer1_attention_weight_min": -15.504744529724121,
      "activations/layer20_attention_weight_max": 21.480913162231445,
      "activations/layer20_attention_weight_min": -25.894994735717773,
      "activations/layer21_attention_weight_max": 36.16713333129883,
      "activations/layer21_attention_weight_min": -24.135061264038086,
      "activations/layer22_attention_weight_max": 31.012523651123047,
      "activations/layer22_attention_weight_min": -25.198301315307617,
      "activations/layer23_attention_weight_max": 36.15374755859375,
      "activations/layer23_attention_weight_min": -23.393756866455078,
      "activations/layer2_attention_weight_max": 31.11702537536621,
      "activations/layer2_attention_weight_min": -29.119571685791016,
      "activations/layer3_attention_weight_max": 89.62581634521484,
      "activations/layer3_attention_weight_min": -87.97586059570312,
      "activations/layer4_attention_weight_max": 90.6949234008789,
      "activations/layer4_attention_weight_min": -85.08846282958984,
      "activations/layer5_attention_weight_max": 64.87023162841797,
      "activations/layer5_attention_weight_min": -71.27542114257812,
      "activations/layer6_attention_weight_max": 49.36307907104492,
      "activations/layer6_attention_weight_min": -51.95418167114258,
      "activations/layer7_attention_weight_max": 61.261146545410156,
      "activations/layer7_attention_weight_min": -61.71763229370117,
      "activations/layer8_attention_weight_max": 44.88270950317383,
      "activations/layer8_attention_weight_min": -48.34221267700195,
      "activations/layer9_attention_weight_max": 45.303287506103516,
      "activations/layer9_attention_weight_min": -44.738853454589844,
      "epoch": 16.41,
      "learning_rate": 4.459469696969696e-05,
      "loss": 2.7431,
      "step": 282500
    },
    {
      "activations/layer0_attention_weight_max": 16.50873374938965,
      "activations/layer0_attention_weight_min": -13.94485855102539,
      "activations/layer10_attention_weight_max": 30.99054527282715,
      "activations/layer10_attention_weight_min": -35.10550308227539,
      "activations/layer11_attention_weight_max": 29.81110382080078,
      "activations/layer11_attention_weight_min": -31.103879928588867,
      "activations/layer12_attention_weight_max": 20.188608169555664,
      "activations/layer12_attention_weight_min": -28.06134605407715,
      "activations/layer13_attention_weight_max": 31.47333526611328,
      "activations/layer13_attention_weight_min": -25.060224533081055,
      "activations/layer14_attention_weight_max": 34.64633560180664,
      "activations/layer14_attention_weight_min": -31.39591407775879,
      "activations/layer15_attention_weight_max": 31.09770393371582,
      "activations/layer15_attention_weight_min": -28.54140853881836,
      "activations/layer16_attention_weight_max": 30.882299423217773,
      "activations/layer16_attention_weight_min": -30.08196258544922,
      "activations/layer17_attention_weight_max": 47.57328414916992,
      "activations/layer17_attention_weight_min": -41.52771759033203,
      "activations/layer18_attention_weight_max": 42.79441452026367,
      "activations/layer18_attention_weight_min": -37.05183029174805,
      "activations/layer19_attention_weight_max": 22.558271408081055,
      "activations/layer19_attention_weight_min": -21.380996704101562,
      "activations/layer1_attention_weight_max": 16.5208683013916,
      "activations/layer1_attention_weight_min": -14.996298789978027,
      "activations/layer20_attention_weight_max": 21.24762725830078,
      "activations/layer20_attention_weight_min": -21.568384170532227,
      "activations/layer21_attention_weight_max": 30.17854881286621,
      "activations/layer21_attention_weight_min": -22.33559226989746,
      "activations/layer22_attention_weight_max": 30.21947479248047,
      "activations/layer22_attention_weight_min": -27.70932388305664,
      "activations/layer23_attention_weight_max": 35.7983283996582,
      "activations/layer23_attention_weight_min": -22.9079647064209,
      "activations/layer2_attention_weight_max": 30.57648468017578,
      "activations/layer2_attention_weight_min": -29.795963287353516,
      "activations/layer3_attention_weight_max": 80.32708740234375,
      "activations/layer3_attention_weight_min": -85.63260650634766,
      "activations/layer4_attention_weight_max": 86.45848083496094,
      "activations/layer4_attention_weight_min": -83.34898376464844,
      "activations/layer5_attention_weight_max": 61.48151779174805,
      "activations/layer5_attention_weight_min": -75.49578094482422,
      "activations/layer6_attention_weight_max": 46.45719528198242,
      "activations/layer6_attention_weight_min": -49.85340881347656,
      "activations/layer7_attention_weight_max": 62.031455993652344,
      "activations/layer7_attention_weight_min": -58.51839828491211,
      "activations/layer8_attention_weight_max": 41.81753921508789,
      "activations/layer8_attention_weight_min": -47.010215759277344,
      "activations/layer9_attention_weight_max": 43.54502868652344,
      "activations/layer9_attention_weight_min": -43.70894241333008,
      "epoch": 16.42,
      "learning_rate": 4.457575757575758e-05,
      "loss": 2.7526,
      "step": 282550
    },
    {
      "activations/layer0_attention_weight_max": 14.903637886047363,
      "activations/layer0_attention_weight_min": -13.975049018859863,
      "activations/layer10_attention_weight_max": 33.895233154296875,
      "activations/layer10_attention_weight_min": -33.70142364501953,
      "activations/layer11_attention_weight_max": 30.59296417236328,
      "activations/layer11_attention_weight_min": -33.702064514160156,
      "activations/layer12_attention_weight_max": 17.93718719482422,
      "activations/layer12_attention_weight_min": -26.203617095947266,
      "activations/layer13_attention_weight_max": 31.55510139465332,
      "activations/layer13_attention_weight_min": -27.966999053955078,
      "activations/layer14_attention_weight_max": 32.60041046142578,
      "activations/layer14_attention_weight_min": -28.886520385742188,
      "activations/layer15_attention_weight_max": 32.35793685913086,
      "activations/layer15_attention_weight_min": -29.702037811279297,
      "activations/layer16_attention_weight_max": 32.787986755371094,
      "activations/layer16_attention_weight_min": -31.25849151611328,
      "activations/layer17_attention_weight_max": 50.48885726928711,
      "activations/layer17_attention_weight_min": -43.769710540771484,
      "activations/layer18_attention_weight_max": 48.059261322021484,
      "activations/layer18_attention_weight_min": -40.548763275146484,
      "activations/layer19_attention_weight_max": 25.23277473449707,
      "activations/layer19_attention_weight_min": -24.70841407775879,
      "activations/layer1_attention_weight_max": 16.251710891723633,
      "activations/layer1_attention_weight_min": -14.447563171386719,
      "activations/layer20_attention_weight_max": 24.039213180541992,
      "activations/layer20_attention_weight_min": -23.37926483154297,
      "activations/layer21_attention_weight_max": 37.8676872253418,
      "activations/layer21_attention_weight_min": -23.578704833984375,
      "activations/layer22_attention_weight_max": 31.725461959838867,
      "activations/layer22_attention_weight_min": -26.608642578125,
      "activations/layer23_attention_weight_max": 37.646942138671875,
      "activations/layer23_attention_weight_min": -24.28093719482422,
      "activations/layer2_attention_weight_max": 30.27187728881836,
      "activations/layer2_attention_weight_min": -28.777729034423828,
      "activations/layer3_attention_weight_max": 83.65363311767578,
      "activations/layer3_attention_weight_min": -83.67435455322266,
      "activations/layer4_attention_weight_max": 85.900390625,
      "activations/layer4_attention_weight_min": -84.54460906982422,
      "activations/layer5_attention_weight_max": 64.13536071777344,
      "activations/layer5_attention_weight_min": -74.39401245117188,
      "activations/layer6_attention_weight_max": 47.80852508544922,
      "activations/layer6_attention_weight_min": -49.51148223876953,
      "activations/layer7_attention_weight_max": 62.60293197631836,
      "activations/layer7_attention_weight_min": -60.72494888305664,
      "activations/layer8_attention_weight_max": 44.17805862426758,
      "activations/layer8_attention_weight_min": -48.188716888427734,
      "activations/layer9_attention_weight_max": 40.081478118896484,
      "activations/layer9_attention_weight_min": -43.8087272644043,
      "epoch": 16.42,
      "learning_rate": 4.455681818181818e-05,
      "loss": 2.726,
      "step": 282600
    },
    {
      "activations/layer0_attention_weight_max": 15.33006477355957,
      "activations/layer0_attention_weight_min": -13.90491771697998,
      "activations/layer10_attention_weight_max": 35.268375396728516,
      "activations/layer10_attention_weight_min": -35.03940200805664,
      "activations/layer11_attention_weight_max": 31.001617431640625,
      "activations/layer11_attention_weight_min": -34.39494323730469,
      "activations/layer12_attention_weight_max": 18.37514305114746,
      "activations/layer12_attention_weight_min": -24.007593154907227,
      "activations/layer13_attention_weight_max": 31.62434959411621,
      "activations/layer13_attention_weight_min": -27.502710342407227,
      "activations/layer14_attention_weight_max": 32.198524475097656,
      "activations/layer14_attention_weight_min": -30.8696346282959,
      "activations/layer15_attention_weight_max": 33.813865661621094,
      "activations/layer15_attention_weight_min": -30.308603286743164,
      "activations/layer16_attention_weight_max": 30.606828689575195,
      "activations/layer16_attention_weight_min": -31.315290451049805,
      "activations/layer17_attention_weight_max": 45.116783142089844,
      "activations/layer17_attention_weight_min": -43.4013671875,
      "activations/layer18_attention_weight_max": 42.01693344116211,
      "activations/layer18_attention_weight_min": -38.682804107666016,
      "activations/layer19_attention_weight_max": 20.85468864440918,
      "activations/layer19_attention_weight_min": -21.61529541015625,
      "activations/layer1_attention_weight_max": 15.448075294494629,
      "activations/layer1_attention_weight_min": -15.700780868530273,
      "activations/layer20_attention_weight_max": 20.594526290893555,
      "activations/layer20_attention_weight_min": -22.051036834716797,
      "activations/layer21_attention_weight_max": 33.71370315551758,
      "activations/layer21_attention_weight_min": -20.81854248046875,
      "activations/layer22_attention_weight_max": 28.07114601135254,
      "activations/layer22_attention_weight_min": -24.748079299926758,
      "activations/layer23_attention_weight_max": 34.27439880371094,
      "activations/layer23_attention_weight_min": -24.182235717773438,
      "activations/layer2_attention_weight_max": 30.60810661315918,
      "activations/layer2_attention_weight_min": -30.156558990478516,
      "activations/layer3_attention_weight_max": 87.68592834472656,
      "activations/layer3_attention_weight_min": -86.33294677734375,
      "activations/layer4_attention_weight_max": 93.16857147216797,
      "activations/layer4_attention_weight_min": -89.59923553466797,
      "activations/layer5_attention_weight_max": 60.17831039428711,
      "activations/layer5_attention_weight_min": -72.13172912597656,
      "activations/layer6_attention_weight_max": 48.69206619262695,
      "activations/layer6_attention_weight_min": -50.030086517333984,
      "activations/layer7_attention_weight_max": 63.09808349609375,
      "activations/layer7_attention_weight_min": -61.64280700683594,
      "activations/layer8_attention_weight_max": 43.79619216918945,
      "activations/layer8_attention_weight_min": -49.12391662597656,
      "activations/layer9_attention_weight_max": 44.77224349975586,
      "activations/layer9_attention_weight_min": -44.14352035522461,
      "epoch": 16.42,
      "learning_rate": 4.453787878787878e-05,
      "loss": 2.7574,
      "step": 282650
    },
    {
      "activations/layer0_attention_weight_max": 15.467733383178711,
      "activations/layer0_attention_weight_min": -13.920083045959473,
      "activations/layer10_attention_weight_max": 35.865150451660156,
      "activations/layer10_attention_weight_min": -33.989646911621094,
      "activations/layer11_attention_weight_max": 32.19091033935547,
      "activations/layer11_attention_weight_min": -32.90911102294922,
      "activations/layer12_attention_weight_max": 18.498388290405273,
      "activations/layer12_attention_weight_min": -23.526596069335938,
      "activations/layer13_attention_weight_max": 33.30899429321289,
      "activations/layer13_attention_weight_min": -29.4210262298584,
      "activations/layer14_attention_weight_max": 34.34866714477539,
      "activations/layer14_attention_weight_min": -28.69929313659668,
      "activations/layer15_attention_weight_max": 31.676374435424805,
      "activations/layer15_attention_weight_min": -29.842588424682617,
      "activations/layer16_attention_weight_max": 33.68207931518555,
      "activations/layer16_attention_weight_min": -32.26424789428711,
      "activations/layer17_attention_weight_max": 51.874149322509766,
      "activations/layer17_attention_weight_min": -45.006893157958984,
      "activations/layer18_attention_weight_max": 44.50227737426758,
      "activations/layer18_attention_weight_min": -39.07563781738281,
      "activations/layer19_attention_weight_max": 22.280061721801758,
      "activations/layer19_attention_weight_min": -22.013334274291992,
      "activations/layer1_attention_weight_max": 16.778156280517578,
      "activations/layer1_attention_weight_min": -14.476998329162598,
      "activations/layer20_attention_weight_max": 21.996475219726562,
      "activations/layer20_attention_weight_min": -21.683576583862305,
      "activations/layer21_attention_weight_max": 34.710479736328125,
      "activations/layer21_attention_weight_min": -26.578336715698242,
      "activations/layer22_attention_weight_max": 27.983396530151367,
      "activations/layer22_attention_weight_min": -25.87915802001953,
      "activations/layer23_attention_weight_max": 36.37995910644531,
      "activations/layer23_attention_weight_min": -24.206350326538086,
      "activations/layer2_attention_weight_max": 30.043399810791016,
      "activations/layer2_attention_weight_min": -29.5675106048584,
      "activations/layer3_attention_weight_max": 84.21481323242188,
      "activations/layer3_attention_weight_min": -85.50827026367188,
      "activations/layer4_attention_weight_max": 86.91559600830078,
      "activations/layer4_attention_weight_min": -85.63094329833984,
      "activations/layer5_attention_weight_max": 65.32254791259766,
      "activations/layer5_attention_weight_min": -70.93209838867188,
      "activations/layer6_attention_weight_max": 50.01488494873047,
      "activations/layer6_attention_weight_min": -49.44514465332031,
      "activations/layer7_attention_weight_max": 64.6072006225586,
      "activations/layer7_attention_weight_min": -59.85273742675781,
      "activations/layer8_attention_weight_max": 45.616310119628906,
      "activations/layer8_attention_weight_min": -48.5174674987793,
      "activations/layer9_attention_weight_max": 43.48805236816406,
      "activations/layer9_attention_weight_min": -44.772491455078125,
      "epoch": 16.43,
      "learning_rate": 4.451893939393939e-05,
      "loss": 2.7262,
      "step": 282700
    },
    {
      "activations/layer0_attention_weight_max": 15.72728157043457,
      "activations/layer0_attention_weight_min": -14.003183364868164,
      "activations/layer10_attention_weight_max": 34.96251678466797,
      "activations/layer10_attention_weight_min": -35.00918960571289,
      "activations/layer11_attention_weight_max": 31.78121566772461,
      "activations/layer11_attention_weight_min": -32.751739501953125,
      "activations/layer12_attention_weight_max": 18.434438705444336,
      "activations/layer12_attention_weight_min": -24.235458374023438,
      "activations/layer13_attention_weight_max": 32.660091400146484,
      "activations/layer13_attention_weight_min": -28.090682983398438,
      "activations/layer14_attention_weight_max": 36.787967681884766,
      "activations/layer14_attention_weight_min": -29.739791870117188,
      "activations/layer15_attention_weight_max": 32.037384033203125,
      "activations/layer15_attention_weight_min": -29.597923278808594,
      "activations/layer16_attention_weight_max": 33.2547721862793,
      "activations/layer16_attention_weight_min": -33.197845458984375,
      "activations/layer17_attention_weight_max": 51.34348678588867,
      "activations/layer17_attention_weight_min": -43.94075393676758,
      "activations/layer18_attention_weight_max": 48.34726333618164,
      "activations/layer18_attention_weight_min": -39.98149108886719,
      "activations/layer19_attention_weight_max": 22.877260208129883,
      "activations/layer19_attention_weight_min": -22.95612144470215,
      "activations/layer1_attention_weight_max": 15.766447067260742,
      "activations/layer1_attention_weight_min": -14.399041175842285,
      "activations/layer20_attention_weight_max": 22.48464012145996,
      "activations/layer20_attention_weight_min": -21.60765266418457,
      "activations/layer21_attention_weight_max": 34.672630310058594,
      "activations/layer21_attention_weight_min": -23.185401916503906,
      "activations/layer22_attention_weight_max": 27.500350952148438,
      "activations/layer22_attention_weight_min": -26.919370651245117,
      "activations/layer23_attention_weight_max": 39.45556640625,
      "activations/layer23_attention_weight_min": -25.1893310546875,
      "activations/layer2_attention_weight_max": 28.750686645507812,
      "activations/layer2_attention_weight_min": -29.34314727783203,
      "activations/layer3_attention_weight_max": 82.18150329589844,
      "activations/layer3_attention_weight_min": -83.44535064697266,
      "activations/layer4_attention_weight_max": 86.16337585449219,
      "activations/layer4_attention_weight_min": -86.06621551513672,
      "activations/layer5_attention_weight_max": 64.28253173828125,
      "activations/layer5_attention_weight_min": -73.34774017333984,
      "activations/layer6_attention_weight_max": 50.27236557006836,
      "activations/layer6_attention_weight_min": -52.90056228637695,
      "activations/layer7_attention_weight_max": 61.82494354248047,
      "activations/layer7_attention_weight_min": -63.210227966308594,
      "activations/layer8_attention_weight_max": 45.97872543334961,
      "activations/layer8_attention_weight_min": -48.93152618408203,
      "activations/layer9_attention_weight_max": 44.580448150634766,
      "activations/layer9_attention_weight_min": -47.1274299621582,
      "epoch": 16.43,
      "learning_rate": 4.45e-05,
      "loss": 2.736,
      "step": 282750
    },
    {
      "activations/layer0_attention_weight_max": 15.97579574584961,
      "activations/layer0_attention_weight_min": -13.893647193908691,
      "activations/layer10_attention_weight_max": 36.8651123046875,
      "activations/layer10_attention_weight_min": -34.79523468017578,
      "activations/layer11_attention_weight_max": 32.28984451293945,
      "activations/layer11_attention_weight_min": -32.975799560546875,
      "activations/layer12_attention_weight_max": 19.884836196899414,
      "activations/layer12_attention_weight_min": -24.689640045166016,
      "activations/layer13_attention_weight_max": 32.61884689331055,
      "activations/layer13_attention_weight_min": -26.199241638183594,
      "activations/layer14_attention_weight_max": 34.99839401245117,
      "activations/layer14_attention_weight_min": -30.069747924804688,
      "activations/layer15_attention_weight_max": 32.10863494873047,
      "activations/layer15_attention_weight_min": -30.833616256713867,
      "activations/layer16_attention_weight_max": 33.56454849243164,
      "activations/layer16_attention_weight_min": -32.21024703979492,
      "activations/layer17_attention_weight_max": 50.90646743774414,
      "activations/layer17_attention_weight_min": -45.368186950683594,
      "activations/layer18_attention_weight_max": 45.01432418823242,
      "activations/layer18_attention_weight_min": -39.162261962890625,
      "activations/layer19_attention_weight_max": 22.674718856811523,
      "activations/layer19_attention_weight_min": -21.92133903503418,
      "activations/layer1_attention_weight_max": 15.998564720153809,
      "activations/layer1_attention_weight_min": -14.332647323608398,
      "activations/layer20_attention_weight_max": 20.929487228393555,
      "activations/layer20_attention_weight_min": -20.95648765563965,
      "activations/layer21_attention_weight_max": 34.8309211730957,
      "activations/layer21_attention_weight_min": -21.61913299560547,
      "activations/layer22_attention_weight_max": 28.253591537475586,
      "activations/layer22_attention_weight_min": -26.56154441833496,
      "activations/layer23_attention_weight_max": 36.51133728027344,
      "activations/layer23_attention_weight_min": -26.14825439453125,
      "activations/layer2_attention_weight_max": 29.083396911621094,
      "activations/layer2_attention_weight_min": -28.643125534057617,
      "activations/layer3_attention_weight_max": 88.88797760009766,
      "activations/layer3_attention_weight_min": -88.28271484375,
      "activations/layer4_attention_weight_max": 89.36528778076172,
      "activations/layer4_attention_weight_min": -87.63334655761719,
      "activations/layer5_attention_weight_max": 61.97178649902344,
      "activations/layer5_attention_weight_min": -70.59872436523438,
      "activations/layer6_attention_weight_max": 48.558353424072266,
      "activations/layer6_attention_weight_min": -51.80064010620117,
      "activations/layer7_attention_weight_max": 61.457122802734375,
      "activations/layer7_attention_weight_min": -65.71790313720703,
      "activations/layer8_attention_weight_max": 46.35249710083008,
      "activations/layer8_attention_weight_min": -48.71684265136719,
      "activations/layer9_attention_weight_max": 43.391883850097656,
      "activations/layer9_attention_weight_min": -47.541717529296875,
      "epoch": 16.43,
      "learning_rate": 4.4481060606060606e-05,
      "loss": 2.7484,
      "step": 282800
    },
    {
      "activations/layer0_attention_weight_max": 15.548715591430664,
      "activations/layer0_attention_weight_min": -13.99881649017334,
      "activations/layer10_attention_weight_max": 35.91525650024414,
      "activations/layer10_attention_weight_min": -36.017642974853516,
      "activations/layer11_attention_weight_max": 31.798810958862305,
      "activations/layer11_attention_weight_min": -32.785911560058594,
      "activations/layer12_attention_weight_max": 21.073251724243164,
      "activations/layer12_attention_weight_min": -28.260225296020508,
      "activations/layer13_attention_weight_max": 38.22751998901367,
      "activations/layer13_attention_weight_min": -28.834131240844727,
      "activations/layer14_attention_weight_max": 35.81797409057617,
      "activations/layer14_attention_weight_min": -30.309797286987305,
      "activations/layer15_attention_weight_max": 33.379276275634766,
      "activations/layer15_attention_weight_min": -30.20528221130371,
      "activations/layer16_attention_weight_max": 33.835105895996094,
      "activations/layer16_attention_weight_min": -29.87080955505371,
      "activations/layer17_attention_weight_max": 53.19158172607422,
      "activations/layer17_attention_weight_min": -42.1911735534668,
      "activations/layer18_attention_weight_max": 50.233299255371094,
      "activations/layer18_attention_weight_min": -35.86659622192383,
      "activations/layer19_attention_weight_max": 23.394180297851562,
      "activations/layer19_attention_weight_min": -22.822248458862305,
      "activations/layer1_attention_weight_max": 16.221271514892578,
      "activations/layer1_attention_weight_min": -13.835946083068848,
      "activations/layer20_attention_weight_max": 25.960838317871094,
      "activations/layer20_attention_weight_min": -21.359895706176758,
      "activations/layer21_attention_weight_max": 39.469024658203125,
      "activations/layer21_attention_weight_min": -21.874874114990234,
      "activations/layer22_attention_weight_max": 31.65769386291504,
      "activations/layer22_attention_weight_min": -24.050798416137695,
      "activations/layer23_attention_weight_max": 40.58333969116211,
      "activations/layer23_attention_weight_min": -25.146474838256836,
      "activations/layer2_attention_weight_max": 30.27399444580078,
      "activations/layer2_attention_weight_min": -30.600807189941406,
      "activations/layer3_attention_weight_max": 86.14996337890625,
      "activations/layer3_attention_weight_min": -87.24589538574219,
      "activations/layer4_attention_weight_max": 85.8630142211914,
      "activations/layer4_attention_weight_min": -86.68856048583984,
      "activations/layer5_attention_weight_max": 64.06681823730469,
      "activations/layer5_attention_weight_min": -72.18302154541016,
      "activations/layer6_attention_weight_max": 48.24020767211914,
      "activations/layer6_attention_weight_min": -49.20393753051758,
      "activations/layer7_attention_weight_max": 61.87136459350586,
      "activations/layer7_attention_weight_min": -61.12761306762695,
      "activations/layer8_attention_weight_max": 46.61775588989258,
      "activations/layer8_attention_weight_min": -46.97444152832031,
      "activations/layer9_attention_weight_max": 44.32565689086914,
      "activations/layer9_attention_weight_min": -43.86569595336914,
      "epoch": 16.44,
      "learning_rate": 4.4462499999999997e-05,
      "loss": 2.7448,
      "step": 282850
    },
    {
      "activations/layer0_attention_weight_max": 16.70663833618164,
      "activations/layer0_attention_weight_min": -13.76132869720459,
      "activations/layer10_attention_weight_max": 39.20512771606445,
      "activations/layer10_attention_weight_min": -40.05427932739258,
      "activations/layer11_attention_weight_max": 36.96739959716797,
      "activations/layer11_attention_weight_min": -35.56201171875,
      "activations/layer12_attention_weight_max": 25.365739822387695,
      "activations/layer12_attention_weight_min": -24.931018829345703,
      "activations/layer13_attention_weight_max": 39.378089904785156,
      "activations/layer13_attention_weight_min": -29.460241317749023,
      "activations/layer14_attention_weight_max": 44.47463607788086,
      "activations/layer14_attention_weight_min": -30.310916900634766,
      "activations/layer15_attention_weight_max": 39.024593353271484,
      "activations/layer15_attention_weight_min": -31.258508682250977,
      "activations/layer16_attention_weight_max": 35.927547454833984,
      "activations/layer16_attention_weight_min": -34.76185607910156,
      "activations/layer17_attention_weight_max": 57.61448287963867,
      "activations/layer17_attention_weight_min": -50.38181686401367,
      "activations/layer18_attention_weight_max": 51.08784484863281,
      "activations/layer18_attention_weight_min": -43.9570426940918,
      "activations/layer19_attention_weight_max": 27.749160766601562,
      "activations/layer19_attention_weight_min": -25.03586196899414,
      "activations/layer1_attention_weight_max": 15.84753704071045,
      "activations/layer1_attention_weight_min": -14.13821029663086,
      "activations/layer20_attention_weight_max": 28.587440490722656,
      "activations/layer20_attention_weight_min": -20.684505462646484,
      "activations/layer21_attention_weight_max": 43.01811981201172,
      "activations/layer21_attention_weight_min": -24.282190322875977,
      "activations/layer22_attention_weight_max": 30.021347045898438,
      "activations/layer22_attention_weight_min": -25.406070709228516,
      "activations/layer23_attention_weight_max": 39.6824951171875,
      "activations/layer23_attention_weight_min": -23.817123413085938,
      "activations/layer2_attention_weight_max": 33.33307647705078,
      "activations/layer2_attention_weight_min": -31.68938446044922,
      "activations/layer3_attention_weight_max": 94.41382598876953,
      "activations/layer3_attention_weight_min": -95.70806884765625,
      "activations/layer4_attention_weight_max": 94.01326751708984,
      "activations/layer4_attention_weight_min": -91.84374237060547,
      "activations/layer5_attention_weight_max": 65.48906707763672,
      "activations/layer5_attention_weight_min": -73.990478515625,
      "activations/layer6_attention_weight_max": 53.57754898071289,
      "activations/layer6_attention_weight_min": -53.07267761230469,
      "activations/layer7_attention_weight_max": 62.39696502685547,
      "activations/layer7_attention_weight_min": -63.37446212768555,
      "activations/layer8_attention_weight_max": 47.87847137451172,
      "activations/layer8_attention_weight_min": -51.32890319824219,
      "activations/layer9_attention_weight_max": 45.46625900268555,
      "activations/layer9_attention_weight_min": -48.112701416015625,
      "epoch": 16.44,
      "learning_rate": 4.44435606060606e-05,
      "loss": 2.7544,
      "step": 282900
    },
    {
      "activations/layer0_attention_weight_max": 16.355852127075195,
      "activations/layer0_attention_weight_min": -13.771421432495117,
      "activations/layer10_attention_weight_max": 35.071563720703125,
      "activations/layer10_attention_weight_min": -34.08654022216797,
      "activations/layer11_attention_weight_max": 31.78199577331543,
      "activations/layer11_attention_weight_min": -32.12683868408203,
      "activations/layer12_attention_weight_max": 21.63959503173828,
      "activations/layer12_attention_weight_min": -26.414186477661133,
      "activations/layer13_attention_weight_max": 33.801734924316406,
      "activations/layer13_attention_weight_min": -26.75139617919922,
      "activations/layer14_attention_weight_max": 31.035472869873047,
      "activations/layer14_attention_weight_min": -29.003734588623047,
      "activations/layer15_attention_weight_max": 28.29799461364746,
      "activations/layer15_attention_weight_min": -30.219009399414062,
      "activations/layer16_attention_weight_max": 31.24950408935547,
      "activations/layer16_attention_weight_min": -30.247920989990234,
      "activations/layer17_attention_weight_max": 48.288692474365234,
      "activations/layer17_attention_weight_min": -43.677040100097656,
      "activations/layer18_attention_weight_max": 46.656803131103516,
      "activations/layer18_attention_weight_min": -39.65098571777344,
      "activations/layer19_attention_weight_max": 23.92875862121582,
      "activations/layer19_attention_weight_min": -22.28816032409668,
      "activations/layer1_attention_weight_max": 15.294713020324707,
      "activations/layer1_attention_weight_min": -14.63071346282959,
      "activations/layer20_attention_weight_max": 23.94611167907715,
      "activations/layer20_attention_weight_min": -22.123580932617188,
      "activations/layer21_attention_weight_max": 32.100746154785156,
      "activations/layer21_attention_weight_min": -22.614900588989258,
      "activations/layer22_attention_weight_max": 28.12062644958496,
      "activations/layer22_attention_weight_min": -24.93496322631836,
      "activations/layer23_attention_weight_max": 34.30983352661133,
      "activations/layer23_attention_weight_min": -27.3433895111084,
      "activations/layer2_attention_weight_max": 30.40730857849121,
      "activations/layer2_attention_weight_min": -29.061269760131836,
      "activations/layer3_attention_weight_max": 86.29134368896484,
      "activations/layer3_attention_weight_min": -85.79283142089844,
      "activations/layer4_attention_weight_max": 87.60128021240234,
      "activations/layer4_attention_weight_min": -89.39253997802734,
      "activations/layer5_attention_weight_max": 63.98972702026367,
      "activations/layer5_attention_weight_min": -71.18151092529297,
      "activations/layer6_attention_weight_max": 47.87879943847656,
      "activations/layer6_attention_weight_min": -47.75998306274414,
      "activations/layer7_attention_weight_max": 61.08161926269531,
      "activations/layer7_attention_weight_min": -59.00554275512695,
      "activations/layer8_attention_weight_max": 46.91688919067383,
      "activations/layer8_attention_weight_min": -46.207401275634766,
      "activations/layer9_attention_weight_max": 45.00168228149414,
      "activations/layer9_attention_weight_min": -43.31732940673828,
      "epoch": 16.44,
      "learning_rate": 4.442462121212121e-05,
      "loss": 2.7358,
      "step": 282950
    },
    {
      "activations/layer0_attention_weight_max": 15.219483375549316,
      "activations/layer0_attention_weight_min": -13.646997451782227,
      "activations/layer10_attention_weight_max": 41.05887222290039,
      "activations/layer10_attention_weight_min": -41.91215515136719,
      "activations/layer11_attention_weight_max": 39.35170364379883,
      "activations/layer11_attention_weight_min": -35.78289794921875,
      "activations/layer12_attention_weight_max": 18.82107925415039,
      "activations/layer12_attention_weight_min": -26.202167510986328,
      "activations/layer13_attention_weight_max": 30.00741958618164,
      "activations/layer13_attention_weight_min": -29.36454963684082,
      "activations/layer14_attention_weight_max": 36.2855224609375,
      "activations/layer14_attention_weight_min": -31.1072940826416,
      "activations/layer15_attention_weight_max": 40.18069076538086,
      "activations/layer15_attention_weight_min": -29.929889678955078,
      "activations/layer16_attention_weight_max": 33.730098724365234,
      "activations/layer16_attention_weight_min": -32.042877197265625,
      "activations/layer17_attention_weight_max": 48.083984375,
      "activations/layer17_attention_weight_min": -46.0400505065918,
      "activations/layer18_attention_weight_max": 42.60312271118164,
      "activations/layer18_attention_weight_min": -39.73040008544922,
      "activations/layer19_attention_weight_max": 23.95125389099121,
      "activations/layer19_attention_weight_min": -23.461240768432617,
      "activations/layer1_attention_weight_max": 16.101144790649414,
      "activations/layer1_attention_weight_min": -12.847166061401367,
      "activations/layer20_attention_weight_max": 20.143705368041992,
      "activations/layer20_attention_weight_min": -21.977344512939453,
      "activations/layer21_attention_weight_max": 33.26535415649414,
      "activations/layer21_attention_weight_min": -23.27256202697754,
      "activations/layer22_attention_weight_max": 29.058979034423828,
      "activations/layer22_attention_weight_min": -27.35284996032715,
      "activations/layer23_attention_weight_max": 33.66744613647461,
      "activations/layer23_attention_weight_min": -25.87360191345215,
      "activations/layer2_attention_weight_max": 30.015308380126953,
      "activations/layer2_attention_weight_min": -30.40968894958496,
      "activations/layer3_attention_weight_max": 87.78675079345703,
      "activations/layer3_attention_weight_min": -85.58870697021484,
      "activations/layer4_attention_weight_max": 85.4189453125,
      "activations/layer4_attention_weight_min": -82.59965515136719,
      "activations/layer5_attention_weight_max": 60.69854736328125,
      "activations/layer5_attention_weight_min": -72.3669204711914,
      "activations/layer6_attention_weight_max": 48.541927337646484,
      "activations/layer6_attention_weight_min": -51.1826057434082,
      "activations/layer7_attention_weight_max": 73.10477447509766,
      "activations/layer7_attention_weight_min": -63.28396224975586,
      "activations/layer8_attention_weight_max": 54.16820526123047,
      "activations/layer8_attention_weight_min": -48.570068359375,
      "activations/layer9_attention_weight_max": 57.27627182006836,
      "activations/layer9_attention_weight_min": -45.490413665771484,
      "epoch": 16.44,
      "learning_rate": 4.4405681818181815e-05,
      "loss": 2.7458,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_loss": 2.703125,
      "eval_runtime": 8.4717,
      "eval_samples_per_second": 506.865,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_openwebtext_loss": 2.703125,
      "eval_openwebtext_ppl": 14.926303617194517,
      "eval_openwebtext_runtime": 8.4717,
      "eval_openwebtext_samples_per_second": 506.865,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_wikitext_loss": 2.93359375,
      "eval_wikitext_ppl": 18.79505399780919,
      "eval_wikitext_runtime": 1.9099,
      "eval_wikitext_samples_per_second": 238.751,
      "step": 283000
    },
    {
      "epoch": 16.44,
      "eval_lambada_loss": 2.517578125,
      "eval_lambada_ppl": 12.398532576458457,
      "eval_lambada_runtime": 9.5341,
      "eval_lambada_samples_per_second": 510.695,
      "step": 283000
    },
    {
      "activations/layer0_attention_weight_max": 16.45749282836914,
      "activations/layer0_attention_weight_min": -13.440088272094727,
      "activations/layer10_attention_weight_max": 33.84180450439453,
      "activations/layer10_attention_weight_min": -33.77490234375,
      "activations/layer11_attention_weight_max": 33.698997497558594,
      "activations/layer11_attention_weight_min": -31.994230270385742,
      "activations/layer12_attention_weight_max": 19.358562469482422,
      "activations/layer12_attention_weight_min": -25.27350425720215,
      "activations/layer13_attention_weight_max": 36.39822006225586,
      "activations/layer13_attention_weight_min": -29.805484771728516,
      "activations/layer14_attention_weight_max": 42.442359924316406,
      "activations/layer14_attention_weight_min": -31.457477569580078,
      "activations/layer15_attention_weight_max": 34.885963439941406,
      "activations/layer15_attention_weight_min": -29.8255672454834,
      "activations/layer16_attention_weight_max": 33.26058578491211,
      "activations/layer16_attention_weight_min": -31.489757537841797,
      "activations/layer17_attention_weight_max": 51.02308654785156,
      "activations/layer17_attention_weight_min": -45.08211135864258,
      "activations/layer18_attention_weight_max": 47.497257232666016,
      "activations/layer18_attention_weight_min": -41.116512298583984,
      "activations/layer19_attention_weight_max": 24.064924240112305,
      "activations/layer19_attention_weight_min": -24.427021026611328,
      "activations/layer1_attention_weight_max": 16.137008666992188,
      "activations/layer1_attention_weight_min": -14.04158878326416,
      "activations/layer20_attention_weight_max": 23.791555404663086,
      "activations/layer20_attention_weight_min": -24.7002010345459,
      "activations/layer21_attention_weight_max": 40.4673957824707,
      "activations/layer21_attention_weight_min": -23.110965728759766,
      "activations/layer22_attention_weight_max": 30.831111907958984,
      "activations/layer22_attention_weight_min": -26.79306411743164,
      "activations/layer23_attention_weight_max": 39.69926834106445,
      "activations/layer23_attention_weight_min": -24.336631774902344,
      "activations/layer2_attention_weight_max": 31.008047103881836,
      "activations/layer2_attention_weight_min": -31.354476928710938,
      "activations/layer3_attention_weight_max": 85.6114273071289,
      "activations/layer3_attention_weight_min": -93.62947082519531,
      "activations/layer4_attention_weight_max": 84.39749145507812,
      "activations/layer4_attention_weight_min": -85.73854064941406,
      "activations/layer5_attention_weight_max": 64.91635131835938,
      "activations/layer5_attention_weight_min": -73.33863067626953,
      "activations/layer6_attention_weight_max": 48.8234977722168,
      "activations/layer6_attention_weight_min": -46.940467834472656,
      "activations/layer7_attention_weight_max": 57.92829895019531,
      "activations/layer7_attention_weight_min": -59.21384048461914,
      "activations/layer8_attention_weight_max": 44.67036437988281,
      "activations/layer8_attention_weight_min": -44.931236267089844,
      "activations/layer9_attention_weight_max": 43.708106994628906,
      "activations/layer9_attention_weight_min": -43.45654296875,
      "epoch": 16.45,
      "learning_rate": 4.4386742424242416e-05,
      "loss": 2.7227,
      "step": 283050
    },
    {
      "activations/layer0_attention_weight_max": 16.221067428588867,
      "activations/layer0_attention_weight_min": -13.767666816711426,
      "activations/layer10_attention_weight_max": 36.089317321777344,
      "activations/layer10_attention_weight_min": -34.85124969482422,
      "activations/layer11_attention_weight_max": 33.33753204345703,
      "activations/layer11_attention_weight_min": -32.79994583129883,
      "activations/layer12_attention_weight_max": 20.233619689941406,
      "activations/layer12_attention_weight_min": -24.642688751220703,
      "activations/layer13_attention_weight_max": 37.25576400756836,
      "activations/layer13_attention_weight_min": -27.64174461364746,
      "activations/layer14_attention_weight_max": 35.96931457519531,
      "activations/layer14_attention_weight_min": -29.654205322265625,
      "activations/layer15_attention_weight_max": 33.98076629638672,
      "activations/layer15_attention_weight_min": -30.493207931518555,
      "activations/layer16_attention_weight_max": 32.58375930786133,
      "activations/layer16_attention_weight_min": -31.352930068969727,
      "activations/layer17_attention_weight_max": 50.79131317138672,
      "activations/layer17_attention_weight_min": -44.559547424316406,
      "activations/layer18_attention_weight_max": 46.40046310424805,
      "activations/layer18_attention_weight_min": -40.82416915893555,
      "activations/layer19_attention_weight_max": 24.434736251831055,
      "activations/layer19_attention_weight_min": -23.318748474121094,
      "activations/layer1_attention_weight_max": 15.991604804992676,
      "activations/layer1_attention_weight_min": -14.012202262878418,
      "activations/layer20_attention_weight_max": 23.242216110229492,
      "activations/layer20_attention_weight_min": -21.574052810668945,
      "activations/layer21_attention_weight_max": 39.50777053833008,
      "activations/layer21_attention_weight_min": -24.939146041870117,
      "activations/layer22_attention_weight_max": 32.37028884887695,
      "activations/layer22_attention_weight_min": -24.848373413085938,
      "activations/layer23_attention_weight_max": 40.258689880371094,
      "activations/layer23_attention_weight_min": -22.34950828552246,
      "activations/layer2_attention_weight_max": 32.281558990478516,
      "activations/layer2_attention_weight_min": -30.233562469482422,
      "activations/layer3_attention_weight_max": 90.16365814208984,
      "activations/layer3_attention_weight_min": -95.72273254394531,
      "activations/layer4_attention_weight_max": 86.43929290771484,
      "activations/layer4_attention_weight_min": -93.09954071044922,
      "activations/layer5_attention_weight_max": 61.38840866088867,
      "activations/layer5_attention_weight_min": -73.23784637451172,
      "activations/layer6_attention_weight_max": 49.52665328979492,
      "activations/layer6_attention_weight_min": -50.83816146850586,
      "activations/layer7_attention_weight_max": 60.6729850769043,
      "activations/layer7_attention_weight_min": -63.78685760498047,
      "activations/layer8_attention_weight_max": 45.84950256347656,
      "activations/layer8_attention_weight_min": -46.77469253540039,
      "activations/layer9_attention_weight_max": 43.01962661743164,
      "activations/layer9_attention_weight_min": -45.63576126098633,
      "epoch": 16.45,
      "learning_rate": 4.4367803030303024e-05,
      "loss": 2.7459,
      "step": 283100
    },
    {
      "activations/layer0_attention_weight_max": 15.833312034606934,
      "activations/layer0_attention_weight_min": -13.540706634521484,
      "activations/layer10_attention_weight_max": 35.441741943359375,
      "activations/layer10_attention_weight_min": -34.42369079589844,
      "activations/layer11_attention_weight_max": 32.21690368652344,
      "activations/layer11_attention_weight_min": -33.212738037109375,
      "activations/layer12_attention_weight_max": 18.71809959411621,
      "activations/layer12_attention_weight_min": -26.774019241333008,
      "activations/layer13_attention_weight_max": 34.24666976928711,
      "activations/layer13_attention_weight_min": -29.738082885742188,
      "activations/layer14_attention_weight_max": 33.71678924560547,
      "activations/layer14_attention_weight_min": -30.52281951904297,
      "activations/layer15_attention_weight_max": 31.84344482421875,
      "activations/layer15_attention_weight_min": -30.04319190979004,
      "activations/layer16_attention_weight_max": 34.00247573852539,
      "activations/layer16_attention_weight_min": -32.33441925048828,
      "activations/layer17_attention_weight_max": 49.72419357299805,
      "activations/layer17_attention_weight_min": -42.9599494934082,
      "activations/layer18_attention_weight_max": 46.43449783325195,
      "activations/layer18_attention_weight_min": -41.04957962036133,
      "activations/layer19_attention_weight_max": 28.571399688720703,
      "activations/layer19_attention_weight_min": -23.63031578063965,
      "activations/layer1_attention_weight_max": 15.812079429626465,
      "activations/layer1_attention_weight_min": -14.698400497436523,
      "activations/layer20_attention_weight_max": 24.109010696411133,
      "activations/layer20_attention_weight_min": -21.704387664794922,
      "activations/layer21_attention_weight_max": 39.610774993896484,
      "activations/layer21_attention_weight_min": -25.335901260375977,
      "activations/layer22_attention_weight_max": 32.01773452758789,
      "activations/layer22_attention_weight_min": -27.32801055908203,
      "activations/layer23_attention_weight_max": 36.786415100097656,
      "activations/layer23_attention_weight_min": -26.665929794311523,
      "activations/layer2_attention_weight_max": 30.575782775878906,
      "activations/layer2_attention_weight_min": -29.54047393798828,
      "activations/layer3_attention_weight_max": 86.86902618408203,
      "activations/layer3_attention_weight_min": -88.83454895019531,
      "activations/layer4_attention_weight_max": 89.27630615234375,
      "activations/layer4_attention_weight_min": -87.84239959716797,
      "activations/layer5_attention_weight_max": 62.307289123535156,
      "activations/layer5_attention_weight_min": -73.3128662109375,
      "activations/layer6_attention_weight_max": 49.94059753417969,
      "activations/layer6_attention_weight_min": -52.32240295410156,
      "activations/layer7_attention_weight_max": 66.0295639038086,
      "activations/layer7_attention_weight_min": -60.21480941772461,
      "activations/layer8_attention_weight_max": 45.75202560424805,
      "activations/layer8_attention_weight_min": -47.32217025756836,
      "activations/layer9_attention_weight_max": 46.62837219238281,
      "activations/layer9_attention_weight_min": -45.77449417114258,
      "epoch": 16.45,
      "learning_rate": 4.434886363636363e-05,
      "loss": 2.7258,
      "step": 283150
    },
    {
      "activations/layer0_attention_weight_max": 15.91620922088623,
      "activations/layer0_attention_weight_min": -13.374544143676758,
      "activations/layer10_attention_weight_max": 31.892351150512695,
      "activations/layer10_attention_weight_min": -31.211950302124023,
      "activations/layer11_attention_weight_max": 30.332015991210938,
      "activations/layer11_attention_weight_min": -32.166114807128906,
      "activations/layer12_attention_weight_max": 17.667728424072266,
      "activations/layer12_attention_weight_min": -27.092975616455078,
      "activations/layer13_attention_weight_max": 29.852914810180664,
      "activations/layer13_attention_weight_min": -24.726659774780273,
      "activations/layer14_attention_weight_max": 33.810699462890625,
      "activations/layer14_attention_weight_min": -30.364770889282227,
      "activations/layer15_attention_weight_max": 30.787954330444336,
      "activations/layer15_attention_weight_min": -28.716794967651367,
      "activations/layer16_attention_weight_max": 29.473217010498047,
      "activations/layer16_attention_weight_min": -29.55370330810547,
      "activations/layer17_attention_weight_max": 46.383872985839844,
      "activations/layer17_attention_weight_min": -41.91838073730469,
      "activations/layer18_attention_weight_max": 41.05909729003906,
      "activations/layer18_attention_weight_min": -38.082122802734375,
      "activations/layer19_attention_weight_max": 22.208847045898438,
      "activations/layer19_attention_weight_min": -24.42589569091797,
      "activations/layer1_attention_weight_max": 15.79256534576416,
      "activations/layer1_attention_weight_min": -14.063521385192871,
      "activations/layer20_attention_weight_max": 19.8643741607666,
      "activations/layer20_attention_weight_min": -21.94596290588379,
      "activations/layer21_attention_weight_max": 32.41037368774414,
      "activations/layer21_attention_weight_min": -23.790685653686523,
      "activations/layer22_attention_weight_max": 29.76332664489746,
      "activations/layer22_attention_weight_min": -27.140216827392578,
      "activations/layer23_attention_weight_max": 33.52782440185547,
      "activations/layer23_attention_weight_min": -24.051403045654297,
      "activations/layer2_attention_weight_max": 31.238515853881836,
      "activations/layer2_attention_weight_min": -31.285385131835938,
      "activations/layer3_attention_weight_max": 79.85960388183594,
      "activations/layer3_attention_weight_min": -83.6723403930664,
      "activations/layer4_attention_weight_max": 79.93453216552734,
      "activations/layer4_attention_weight_min": -81.34577178955078,
      "activations/layer5_attention_weight_max": 61.57636260986328,
      "activations/layer5_attention_weight_min": -72.04182434082031,
      "activations/layer6_attention_weight_max": 44.72958755493164,
      "activations/layer6_attention_weight_min": -47.88872146606445,
      "activations/layer7_attention_weight_max": 60.1551628112793,
      "activations/layer7_attention_weight_min": -59.5154914855957,
      "activations/layer8_attention_weight_max": 41.20637893676758,
      "activations/layer8_attention_weight_min": -44.94072723388672,
      "activations/layer9_attention_weight_max": 37.94855880737305,
      "activations/layer9_attention_weight_min": -42.01775360107422,
      "epoch": 16.46,
      "learning_rate": 4.432992424242424e-05,
      "loss": 2.748,
      "step": 283200
    },
    {
      "activations/layer0_attention_weight_max": 16.05669403076172,
      "activations/layer0_attention_weight_min": -13.13175106048584,
      "activations/layer10_attention_weight_max": 33.410560607910156,
      "activations/layer10_attention_weight_min": -35.39627456665039,
      "activations/layer11_attention_weight_max": 32.84284591674805,
      "activations/layer11_attention_weight_min": -33.13608932495117,
      "activations/layer12_attention_weight_max": 18.319293975830078,
      "activations/layer12_attention_weight_min": -28.98472023010254,
      "activations/layer13_attention_weight_max": 37.653289794921875,
      "activations/layer13_attention_weight_min": -30.087631225585938,
      "activations/layer14_attention_weight_max": 35.74433517456055,
      "activations/layer14_attention_weight_min": -29.669164657592773,
      "activations/layer15_attention_weight_max": 36.66364669799805,
      "activations/layer15_attention_weight_min": -28.929615020751953,
      "activations/layer16_attention_weight_max": 34.44245529174805,
      "activations/layer16_attention_weight_min": -32.661739349365234,
      "activations/layer17_attention_weight_max": 51.095947265625,
      "activations/layer17_attention_weight_min": -44.059349060058594,
      "activations/layer18_attention_weight_max": 46.031124114990234,
      "activations/layer18_attention_weight_min": -37.280548095703125,
      "activations/layer19_attention_weight_max": 22.513843536376953,
      "activations/layer19_attention_weight_min": -22.03705596923828,
      "activations/layer1_attention_weight_max": 14.944456100463867,
      "activations/layer1_attention_weight_min": -13.777338981628418,
      "activations/layer20_attention_weight_max": 23.789888381958008,
      "activations/layer20_attention_weight_min": -21.891355514526367,
      "activations/layer21_attention_weight_max": 32.99116897583008,
      "activations/layer21_attention_weight_min": -25.536376953125,
      "activations/layer22_attention_weight_max": 28.8979434967041,
      "activations/layer22_attention_weight_min": -26.891944885253906,
      "activations/layer23_attention_weight_max": 39.52152633666992,
      "activations/layer23_attention_weight_min": -25.72535514831543,
      "activations/layer2_attention_weight_max": 30.213504791259766,
      "activations/layer2_attention_weight_min": -28.951704025268555,
      "activations/layer3_attention_weight_max": 84.9017562866211,
      "activations/layer3_attention_weight_min": -83.62025451660156,
      "activations/layer4_attention_weight_max": 88.13450622558594,
      "activations/layer4_attention_weight_min": -86.46923065185547,
      "activations/layer5_attention_weight_max": 65.28993225097656,
      "activations/layer5_attention_weight_min": -73.8214340209961,
      "activations/layer6_attention_weight_max": 47.522701263427734,
      "activations/layer6_attention_weight_min": -49.18174362182617,
      "activations/layer7_attention_weight_max": 63.588714599609375,
      "activations/layer7_attention_weight_min": -61.22140121459961,
      "activations/layer8_attention_weight_max": 45.294864654541016,
      "activations/layer8_attention_weight_min": -45.268272399902344,
      "activations/layer9_attention_weight_max": 51.7335205078125,
      "activations/layer9_attention_weight_min": -46.02304458618164,
      "epoch": 16.46,
      "learning_rate": 4.431098484848484e-05,
      "loss": 2.7597,
      "step": 283250
    },
    {
      "activations/layer0_attention_weight_max": 16.380573272705078,
      "activations/layer0_attention_weight_min": -13.257845878601074,
      "activations/layer10_attention_weight_max": 33.732879638671875,
      "activations/layer10_attention_weight_min": -34.211761474609375,
      "activations/layer11_attention_weight_max": 31.243894577026367,
      "activations/layer11_attention_weight_min": -32.029808044433594,
      "activations/layer12_attention_weight_max": 19.402280807495117,
      "activations/layer12_attention_weight_min": -28.46721076965332,
      "activations/layer13_attention_weight_max": 34.07960891723633,
      "activations/layer13_attention_weight_min": -26.79032325744629,
      "activations/layer14_attention_weight_max": 37.417259216308594,
      "activations/layer14_attention_weight_min": -30.882518768310547,
      "activations/layer15_attention_weight_max": 36.81477737426758,
      "activations/layer15_attention_weight_min": -33.39496994018555,
      "activations/layer16_attention_weight_max": 33.56814956665039,
      "activations/layer16_attention_weight_min": -31.25821876525879,
      "activations/layer17_attention_weight_max": 52.35612869262695,
      "activations/layer17_attention_weight_min": -44.18260192871094,
      "activations/layer18_attention_weight_max": 52.356422424316406,
      "activations/layer18_attention_weight_min": -40.49407196044922,
      "activations/layer19_attention_weight_max": 25.142690658569336,
      "activations/layer19_attention_weight_min": -23.426651000976562,
      "activations/layer1_attention_weight_max": 16.473581314086914,
      "activations/layer1_attention_weight_min": -15.045061111450195,
      "activations/layer20_attention_weight_max": 25.987993240356445,
      "activations/layer20_attention_weight_min": -21.19198226928711,
      "activations/layer21_attention_weight_max": 37.869422912597656,
      "activations/layer21_attention_weight_min": -22.305158615112305,
      "activations/layer22_attention_weight_max": 29.491689682006836,
      "activations/layer22_attention_weight_min": -25.82988929748535,
      "activations/layer23_attention_weight_max": 38.61563491821289,
      "activations/layer23_attention_weight_min": -24.65850830078125,
      "activations/layer2_attention_weight_max": 31.335126876831055,
      "activations/layer2_attention_weight_min": -29.613468170166016,
      "activations/layer3_attention_weight_max": 84.74229431152344,
      "activations/layer3_attention_weight_min": -88.1865234375,
      "activations/layer4_attention_weight_max": 86.14741516113281,
      "activations/layer4_attention_weight_min": -84.9156723022461,
      "activations/layer5_attention_weight_max": 62.33360290527344,
      "activations/layer5_attention_weight_min": -72.41891479492188,
      "activations/layer6_attention_weight_max": 49.99519729614258,
      "activations/layer6_attention_weight_min": -50.08141326904297,
      "activations/layer7_attention_weight_max": 66.77964782714844,
      "activations/layer7_attention_weight_min": -62.85778045654297,
      "activations/layer8_attention_weight_max": 47.26478958129883,
      "activations/layer8_attention_weight_min": -50.21843719482422,
      "activations/layer9_attention_weight_max": 48.328887939453125,
      "activations/layer9_attention_weight_min": -44.009849548339844,
      "epoch": 16.46,
      "learning_rate": 4.4292045454545444e-05,
      "loss": 2.7459,
      "step": 283300
    },
    {
      "activations/layer0_attention_weight_max": 15.157193183898926,
      "activations/layer0_attention_weight_min": -13.653990745544434,
      "activations/layer10_attention_weight_max": 39.2425651550293,
      "activations/layer10_attention_weight_min": -34.817867279052734,
      "activations/layer11_attention_weight_max": 34.017539978027344,
      "activations/layer11_attention_weight_min": -34.799076080322266,
      "activations/layer12_attention_weight_max": 19.203603744506836,
      "activations/layer12_attention_weight_min": -23.938016891479492,
      "activations/layer13_attention_weight_max": 33.35926818847656,
      "activations/layer13_attention_weight_min": -28.71721839904785,
      "activations/layer14_attention_weight_max": 35.682044982910156,
      "activations/layer14_attention_weight_min": -31.69213104248047,
      "activations/layer15_attention_weight_max": 32.07500457763672,
      "activations/layer15_attention_weight_min": -29.23526382446289,
      "activations/layer16_attention_weight_max": 33.20557403564453,
      "activations/layer16_attention_weight_min": -31.961332321166992,
      "activations/layer17_attention_weight_max": 50.22877502441406,
      "activations/layer17_attention_weight_min": -45.18342208862305,
      "activations/layer18_attention_weight_max": 44.18745040893555,
      "activations/layer18_attention_weight_min": -39.631553649902344,
      "activations/layer19_attention_weight_max": 24.018827438354492,
      "activations/layer19_attention_weight_min": -22.980003356933594,
      "activations/layer1_attention_weight_max": 15.646571159362793,
      "activations/layer1_attention_weight_min": -14.369648933410645,
      "activations/layer20_attention_weight_max": 22.0221004486084,
      "activations/layer20_attention_weight_min": -24.566022872924805,
      "activations/layer21_attention_weight_max": 32.183536529541016,
      "activations/layer21_attention_weight_min": -22.27048683166504,
      "activations/layer22_attention_weight_max": 29.61419677734375,
      "activations/layer22_attention_weight_min": -27.75656509399414,
      "activations/layer23_attention_weight_max": 38.00331497192383,
      "activations/layer23_attention_weight_min": -27.83795928955078,
      "activations/layer2_attention_weight_max": 30.475982666015625,
      "activations/layer2_attention_weight_min": -28.8576717376709,
      "activations/layer3_attention_weight_max": 88.86592864990234,
      "activations/layer3_attention_weight_min": -87.8337631225586,
      "activations/layer4_attention_weight_max": 88.91761779785156,
      "activations/layer4_attention_weight_min": -86.61526489257812,
      "activations/layer5_attention_weight_max": 64.78193664550781,
      "activations/layer5_attention_weight_min": -70.88948059082031,
      "activations/layer6_attention_weight_max": 49.347755432128906,
      "activations/layer6_attention_weight_min": -49.28571701049805,
      "activations/layer7_attention_weight_max": 65.20507049560547,
      "activations/layer7_attention_weight_min": -63.48085403442383,
      "activations/layer8_attention_weight_max": 48.52424240112305,
      "activations/layer8_attention_weight_min": -48.51347732543945,
      "activations/layer9_attention_weight_max": 47.1955680847168,
      "activations/layer9_attention_weight_min": -45.77724075317383,
      "epoch": 16.46,
      "learning_rate": 4.427310606060606e-05,
      "loss": 2.733,
      "step": 283350
    },
    {
      "activations/layer0_attention_weight_max": 15.787586212158203,
      "activations/layer0_attention_weight_min": -13.271486282348633,
      "activations/layer10_attention_weight_max": 35.34273147583008,
      "activations/layer10_attention_weight_min": -35.891746520996094,
      "activations/layer11_attention_weight_max": 31.84579849243164,
      "activations/layer11_attention_weight_min": -32.94117736816406,
      "activations/layer12_attention_weight_max": 18.073270797729492,
      "activations/layer12_attention_weight_min": -24.59247398376465,
      "activations/layer13_attention_weight_max": 36.129547119140625,
      "activations/layer13_attention_weight_min": -31.06678009033203,
      "activations/layer14_attention_weight_max": 36.80254364013672,
      "activations/layer14_attention_weight_min": -34.18053436279297,
      "activations/layer15_attention_weight_max": 34.615699768066406,
      "activations/layer15_attention_weight_min": -33.25486755371094,
      "activations/layer16_attention_weight_max": 36.81107711791992,
      "activations/layer16_attention_weight_min": -34.77542495727539,
      "activations/layer17_attention_weight_max": 54.13591003417969,
      "activations/layer17_attention_weight_min": -49.72626495361328,
      "activations/layer18_attention_weight_max": 51.46379852294922,
      "activations/layer18_attention_weight_min": -43.941490173339844,
      "activations/layer19_attention_weight_max": 26.195219039916992,
      "activations/layer19_attention_weight_min": -25.450876235961914,
      "activations/layer1_attention_weight_max": 16.504220962524414,
      "activations/layer1_attention_weight_min": -14.088946342468262,
      "activations/layer20_attention_weight_max": 24.50243377685547,
      "activations/layer20_attention_weight_min": -21.587461471557617,
      "activations/layer21_attention_weight_max": 40.217811584472656,
      "activations/layer21_attention_weight_min": -23.894378662109375,
      "activations/layer22_attention_weight_max": 29.088958740234375,
      "activations/layer22_attention_weight_min": -26.80849266052246,
      "activations/layer23_attention_weight_max": 37.666053771972656,
      "activations/layer23_attention_weight_min": -24.982990264892578,
      "activations/layer2_attention_weight_max": 29.675588607788086,
      "activations/layer2_attention_weight_min": -28.17884063720703,
      "activations/layer3_attention_weight_max": 84.07364654541016,
      "activations/layer3_attention_weight_min": -82.83130645751953,
      "activations/layer4_attention_weight_max": 86.3765640258789,
      "activations/layer4_attention_weight_min": -85.6523666381836,
      "activations/layer5_attention_weight_max": 64.51947784423828,
      "activations/layer5_attention_weight_min": -71.93843078613281,
      "activations/layer6_attention_weight_max": 47.939247131347656,
      "activations/layer6_attention_weight_min": -49.1001091003418,
      "activations/layer7_attention_weight_max": 59.9520263671875,
      "activations/layer7_attention_weight_min": -62.80976867675781,
      "activations/layer8_attention_weight_max": 43.768836975097656,
      "activations/layer8_attention_weight_min": -48.22278594970703,
      "activations/layer9_attention_weight_max": 45.07685470581055,
      "activations/layer9_attention_weight_min": -44.81415939331055,
      "epoch": 16.47,
      "learning_rate": 4.425416666666666e-05,
      "loss": 2.7283,
      "step": 283400
    },
    {
      "activations/layer0_attention_weight_max": 15.80512523651123,
      "activations/layer0_attention_weight_min": -13.471156120300293,
      "activations/layer10_attention_weight_max": 35.66965866088867,
      "activations/layer10_attention_weight_min": -34.802581787109375,
      "activations/layer11_attention_weight_max": 30.198102951049805,
      "activations/layer11_attention_weight_min": -33.164512634277344,
      "activations/layer12_attention_weight_max": 18.60927391052246,
      "activations/layer12_attention_weight_min": -27.79015350341797,
      "activations/layer13_attention_weight_max": 33.29800033569336,
      "activations/layer13_attention_weight_min": -27.211830139160156,
      "activations/layer14_attention_weight_max": 35.936004638671875,
      "activations/layer14_attention_weight_min": -32.121212005615234,
      "activations/layer15_attention_weight_max": 34.52364730834961,
      "activations/layer15_attention_weight_min": -30.398141860961914,
      "activations/layer16_attention_weight_max": 34.61716079711914,
      "activations/layer16_attention_weight_min": -31.32381248474121,
      "activations/layer17_attention_weight_max": 50.93306350708008,
      "activations/layer17_attention_weight_min": -44.06953430175781,
      "activations/layer18_attention_weight_max": 44.26519012451172,
      "activations/layer18_attention_weight_min": -36.834537506103516,
      "activations/layer19_attention_weight_max": 26.624013900756836,
      "activations/layer19_attention_weight_min": -22.61876106262207,
      "activations/layer1_attention_weight_max": 15.472146034240723,
      "activations/layer1_attention_weight_min": -15.396539688110352,
      "activations/layer20_attention_weight_max": 26.035526275634766,
      "activations/layer20_attention_weight_min": -21.430524826049805,
      "activations/layer21_attention_weight_max": 35.79353713989258,
      "activations/layer21_attention_weight_min": -24.200998306274414,
      "activations/layer22_attention_weight_max": 29.658172607421875,
      "activations/layer22_attention_weight_min": -26.459089279174805,
      "activations/layer23_attention_weight_max": 36.952423095703125,
      "activations/layer23_attention_weight_min": -23.09815788269043,
      "activations/layer2_attention_weight_max": 31.30193328857422,
      "activations/layer2_attention_weight_min": -28.586532592773438,
      "activations/layer3_attention_weight_max": 85.7381591796875,
      "activations/layer3_attention_weight_min": -88.11279296875,
      "activations/layer4_attention_weight_max": 90.27497863769531,
      "activations/layer4_attention_weight_min": -87.3403549194336,
      "activations/layer5_attention_weight_max": 64.52881622314453,
      "activations/layer5_attention_weight_min": -76.02969360351562,
      "activations/layer6_attention_weight_max": 50.99136734008789,
      "activations/layer6_attention_weight_min": -49.965721130371094,
      "activations/layer7_attention_weight_max": 64.36874389648438,
      "activations/layer7_attention_weight_min": -60.52466583251953,
      "activations/layer8_attention_weight_max": 42.29690933227539,
      "activations/layer8_attention_weight_min": -44.743106842041016,
      "activations/layer9_attention_weight_max": 43.11944580078125,
      "activations/layer9_attention_weight_min": -43.08964538574219,
      "epoch": 16.47,
      "learning_rate": 4.423522727272727e-05,
      "loss": 2.7607,
      "step": 283450
    },
    {
      "activations/layer0_attention_weight_max": 16.817790985107422,
      "activations/layer0_attention_weight_min": -13.545766830444336,
      "activations/layer10_attention_weight_max": 36.09245681762695,
      "activations/layer10_attention_weight_min": -35.286476135253906,
      "activations/layer11_attention_weight_max": 32.018524169921875,
      "activations/layer11_attention_weight_min": -33.468231201171875,
      "activations/layer12_attention_weight_max": 18.972084045410156,
      "activations/layer12_attention_weight_min": -23.70006561279297,
      "activations/layer13_attention_weight_max": 32.133544921875,
      "activations/layer13_attention_weight_min": -28.6525936126709,
      "activations/layer14_attention_weight_max": 36.74135971069336,
      "activations/layer14_attention_weight_min": -33.05508041381836,
      "activations/layer15_attention_weight_max": 32.23166275024414,
      "activations/layer15_attention_weight_min": -32.73458480834961,
      "activations/layer16_attention_weight_max": 31.851268768310547,
      "activations/layer16_attention_weight_min": -30.37813377380371,
      "activations/layer17_attention_weight_max": 52.637882232666016,
      "activations/layer17_attention_weight_min": -44.5167121887207,
      "activations/layer18_attention_weight_max": 46.62123489379883,
      "activations/layer18_attention_weight_min": -38.307838439941406,
      "activations/layer19_attention_weight_max": 22.413373947143555,
      "activations/layer19_attention_weight_min": -21.51447105407715,
      "activations/layer1_attention_weight_max": 15.965314865112305,
      "activations/layer1_attention_weight_min": -12.431466102600098,
      "activations/layer20_attention_weight_max": 20.67499542236328,
      "activations/layer20_attention_weight_min": -21.207862854003906,
      "activations/layer21_attention_weight_max": 38.22265625,
      "activations/layer21_attention_weight_min": -24.544790267944336,
      "activations/layer22_attention_weight_max": 29.661388397216797,
      "activations/layer22_attention_weight_min": -25.27859115600586,
      "activations/layer23_attention_weight_max": 37.21220397949219,
      "activations/layer23_attention_weight_min": -22.640281677246094,
      "activations/layer2_attention_weight_max": 30.236713409423828,
      "activations/layer2_attention_weight_min": -28.362506866455078,
      "activations/layer3_attention_weight_max": 85.93607330322266,
      "activations/layer3_attention_weight_min": -84.41657257080078,
      "activations/layer4_attention_weight_max": 86.85940551757812,
      "activations/layer4_attention_weight_min": -84.67879486083984,
      "activations/layer5_attention_weight_max": 63.63080596923828,
      "activations/layer5_attention_weight_min": -73.53645324707031,
      "activations/layer6_attention_weight_max": 52.21837615966797,
      "activations/layer6_attention_weight_min": -51.58509826660156,
      "activations/layer7_attention_weight_max": 71.14363098144531,
      "activations/layer7_attention_weight_min": -63.952964782714844,
      "activations/layer8_attention_weight_max": 46.99797439575195,
      "activations/layer8_attention_weight_min": -49.664608001708984,
      "activations/layer9_attention_weight_max": 44.81150436401367,
      "activations/layer9_attention_weight_min": -47.528011322021484,
      "epoch": 16.47,
      "learning_rate": 4.421628787878788e-05,
      "loss": 2.7548,
      "step": 283500
    },
    {
      "activations/layer0_attention_weight_max": 16.043638229370117,
      "activations/layer0_attention_weight_min": -13.107260704040527,
      "activations/layer10_attention_weight_max": 32.610862731933594,
      "activations/layer10_attention_weight_min": -34.651222229003906,
      "activations/layer11_attention_weight_max": 30.591312408447266,
      "activations/layer11_attention_weight_min": -32.51462936401367,
      "activations/layer12_attention_weight_max": 19.38663101196289,
      "activations/layer12_attention_weight_min": -26.01878547668457,
      "activations/layer13_attention_weight_max": 37.33616638183594,
      "activations/layer13_attention_weight_min": -28.117406845092773,
      "activations/layer14_attention_weight_max": 38.48756790161133,
      "activations/layer14_attention_weight_min": -31.98078155517578,
      "activations/layer15_attention_weight_max": 35.44266891479492,
      "activations/layer15_attention_weight_min": -29.30903434753418,
      "activations/layer16_attention_weight_max": 37.31977844238281,
      "activations/layer16_attention_weight_min": -33.89177703857422,
      "activations/layer17_attention_weight_max": 57.9704475402832,
      "activations/layer17_attention_weight_min": -50.2566032409668,
      "activations/layer18_attention_weight_max": 54.0338249206543,
      "activations/layer18_attention_weight_min": -44.88767623901367,
      "activations/layer19_attention_weight_max": 28.2161808013916,
      "activations/layer19_attention_weight_min": -22.925235748291016,
      "activations/layer1_attention_weight_max": 16.102758407592773,
      "activations/layer1_attention_weight_min": -13.317387580871582,
      "activations/layer20_attention_weight_max": 27.564945220947266,
      "activations/layer20_attention_weight_min": -22.64203643798828,
      "activations/layer21_attention_weight_max": 42.72114944458008,
      "activations/layer21_attention_weight_min": -21.33375358581543,
      "activations/layer22_attention_weight_max": 34.068538665771484,
      "activations/layer22_attention_weight_min": -26.002113342285156,
      "activations/layer23_attention_weight_max": 39.72352981567383,
      "activations/layer23_attention_weight_min": -28.551237106323242,
      "activations/layer2_attention_weight_max": 28.699865341186523,
      "activations/layer2_attention_weight_min": -27.868244171142578,
      "activations/layer3_attention_weight_max": 84.25506591796875,
      "activations/layer3_attention_weight_min": -80.09073638916016,
      "activations/layer4_attention_weight_max": 84.06656646728516,
      "activations/layer4_attention_weight_min": -82.7017593383789,
      "activations/layer5_attention_weight_max": 62.042686462402344,
      "activations/layer5_attention_weight_min": -72.63493347167969,
      "activations/layer6_attention_weight_max": 47.129234313964844,
      "activations/layer6_attention_weight_min": -50.33894348144531,
      "activations/layer7_attention_weight_max": 60.94456100463867,
      "activations/layer7_attention_weight_min": -60.135772705078125,
      "activations/layer8_attention_weight_max": 44.538516998291016,
      "activations/layer8_attention_weight_min": -45.87654495239258,
      "activations/layer9_attention_weight_max": 48.03364944458008,
      "activations/layer9_attention_weight_min": -43.77530288696289,
      "epoch": 16.48,
      "learning_rate": 4.4197348484848485e-05,
      "loss": 2.7601,
      "step": 283550
    },
    {
      "activations/layer0_attention_weight_max": 15.834877967834473,
      "activations/layer0_attention_weight_min": -13.765944480895996,
      "activations/layer10_attention_weight_max": 38.91611099243164,
      "activations/layer10_attention_weight_min": -35.74721908569336,
      "activations/layer11_attention_weight_max": 36.13371276855469,
      "activations/layer11_attention_weight_min": -35.91412353515625,
      "activations/layer12_attention_weight_max": 20.51094627380371,
      "activations/layer12_attention_weight_min": -24.932483673095703,
      "activations/layer13_attention_weight_max": 38.41259765625,
      "activations/layer13_attention_weight_min": -27.74652671813965,
      "activations/layer14_attention_weight_max": 35.6851921081543,
      "activations/layer14_attention_weight_min": -29.921398162841797,
      "activations/layer15_attention_weight_max": 38.953636169433594,
      "activations/layer15_attention_weight_min": -31.573408126831055,
      "activations/layer16_attention_weight_max": 34.27444076538086,
      "activations/layer16_attention_weight_min": -31.574079513549805,
      "activations/layer17_attention_weight_max": 51.506900787353516,
      "activations/layer17_attention_weight_min": -45.9306755065918,
      "activations/layer18_attention_weight_max": 47.734615325927734,
      "activations/layer18_attention_weight_min": -40.340213775634766,
      "activations/layer19_attention_weight_max": 25.08161735534668,
      "activations/layer19_attention_weight_min": -23.49933624267578,
      "activations/layer1_attention_weight_max": 16.422502517700195,
      "activations/layer1_attention_weight_min": -14.782955169677734,
      "activations/layer20_attention_weight_max": 27.473735809326172,
      "activations/layer20_attention_weight_min": -23.592653274536133,
      "activations/layer21_attention_weight_max": 39.31877136230469,
      "activations/layer21_attention_weight_min": -21.88690185546875,
      "activations/layer22_attention_weight_max": 35.00875473022461,
      "activations/layer22_attention_weight_min": -25.174089431762695,
      "activations/layer23_attention_weight_max": 38.532527923583984,
      "activations/layer23_attention_weight_min": -23.54513931274414,
      "activations/layer2_attention_weight_max": 32.6697998046875,
      "activations/layer2_attention_weight_min": -29.24341583251953,
      "activations/layer3_attention_weight_max": 89.44007110595703,
      "activations/layer3_attention_weight_min": -89.66704559326172,
      "activations/layer4_attention_weight_max": 92.16554260253906,
      "activations/layer4_attention_weight_min": -87.22864532470703,
      "activations/layer5_attention_weight_max": 69.34609985351562,
      "activations/layer5_attention_weight_min": -72.51644897460938,
      "activations/layer6_attention_weight_max": 51.271244049072266,
      "activations/layer6_attention_weight_min": -50.40547180175781,
      "activations/layer7_attention_weight_max": 67.93084716796875,
      "activations/layer7_attention_weight_min": -64.18810272216797,
      "activations/layer8_attention_weight_max": 46.9738655090332,
      "activations/layer8_attention_weight_min": -50.336666107177734,
      "activations/layer9_attention_weight_max": 47.73525619506836,
      "activations/layer9_attention_weight_min": -46.672210693359375,
      "epoch": 16.48,
      "learning_rate": 4.4178787878787876e-05,
      "loss": 2.7448,
      "step": 283600
    },
    {
      "activations/layer0_attention_weight_max": 16.517152786254883,
      "activations/layer0_attention_weight_min": -13.637205123901367,
      "activations/layer10_attention_weight_max": 35.65696334838867,
      "activations/layer10_attention_weight_min": -34.043025970458984,
      "activations/layer11_attention_weight_max": 33.282466888427734,
      "activations/layer11_attention_weight_min": -32.90766143798828,
      "activations/layer12_attention_weight_max": 18.350509643554688,
      "activations/layer12_attention_weight_min": -31.997817993164062,
      "activations/layer13_attention_weight_max": 37.16465759277344,
      "activations/layer13_attention_weight_min": -28.31783103942871,
      "activations/layer14_attention_weight_max": 36.95561599731445,
      "activations/layer14_attention_weight_min": -29.382701873779297,
      "activations/layer15_attention_weight_max": 35.4876594543457,
      "activations/layer15_attention_weight_min": -29.32266616821289,
      "activations/layer16_attention_weight_max": 36.38157272338867,
      "activations/layer16_attention_weight_min": -30.708776473999023,
      "activations/layer17_attention_weight_max": 53.27220153808594,
      "activations/layer17_attention_weight_min": -44.76533889770508,
      "activations/layer18_attention_weight_max": 51.321475982666016,
      "activations/layer18_attention_weight_min": -40.4078369140625,
      "activations/layer19_attention_weight_max": 25.53659439086914,
      "activations/layer19_attention_weight_min": -22.67058563232422,
      "activations/layer1_attention_weight_max": 16.553699493408203,
      "activations/layer1_attention_weight_min": -14.157755851745605,
      "activations/layer20_attention_weight_max": 26.254375457763672,
      "activations/layer20_attention_weight_min": -22.56534194946289,
      "activations/layer21_attention_weight_max": 40.73011016845703,
      "activations/layer21_attention_weight_min": -21.161422729492188,
      "activations/layer22_attention_weight_max": 34.93281555175781,
      "activations/layer22_attention_weight_min": -26.647987365722656,
      "activations/layer23_attention_weight_max": 41.095157623291016,
      "activations/layer23_attention_weight_min": -25.397266387939453,
      "activations/layer2_attention_weight_max": 30.601661682128906,
      "activations/layer2_attention_weight_min": -27.224462509155273,
      "activations/layer3_attention_weight_max": 81.1429443359375,
      "activations/layer3_attention_weight_min": -75.10529327392578,
      "activations/layer4_attention_weight_max": 85.94573974609375,
      "activations/layer4_attention_weight_min": -80.22065734863281,
      "activations/layer5_attention_weight_max": 60.581817626953125,
      "activations/layer5_attention_weight_min": -71.98138427734375,
      "activations/layer6_attention_weight_max": 48.25305938720703,
      "activations/layer6_attention_weight_min": -46.147953033447266,
      "activations/layer7_attention_weight_max": 64.85859680175781,
      "activations/layer7_attention_weight_min": -62.858795166015625,
      "activations/layer8_attention_weight_max": 43.85734939575195,
      "activations/layer8_attention_weight_min": -44.85940170288086,
      "activations/layer9_attention_weight_max": 46.696327209472656,
      "activations/layer9_attention_weight_min": -45.464473724365234,
      "epoch": 16.48,
      "learning_rate": 4.415984848484848e-05,
      "loss": 2.7466,
      "step": 283650
    },
    {
      "activations/layer0_attention_weight_max": 15.148872375488281,
      "activations/layer0_attention_weight_min": -13.508500099182129,
      "activations/layer10_attention_weight_max": 38.358360290527344,
      "activations/layer10_attention_weight_min": -38.217864990234375,
      "activations/layer11_attention_weight_max": 34.72418212890625,
      "activations/layer11_attention_weight_min": -37.33559799194336,
      "activations/layer12_attention_weight_max": 19.47501564025879,
      "activations/layer12_attention_weight_min": -25.039321899414062,
      "activations/layer13_attention_weight_max": 39.625633239746094,
      "activations/layer13_attention_weight_min": -31.317245483398438,
      "activations/layer14_attention_weight_max": 37.346317291259766,
      "activations/layer14_attention_weight_min": -31.7652530670166,
      "activations/layer15_attention_weight_max": 34.14268493652344,
      "activations/layer15_attention_weight_min": -31.839445114135742,
      "activations/layer16_attention_weight_max": 38.03262710571289,
      "activations/layer16_attention_weight_min": -34.37965393066406,
      "activations/layer17_attention_weight_max": 59.31007385253906,
      "activations/layer17_attention_weight_min": -52.09159851074219,
      "activations/layer18_attention_weight_max": 54.163143157958984,
      "activations/layer18_attention_weight_min": -43.18004608154297,
      "activations/layer19_attention_weight_max": 27.13824462890625,
      "activations/layer19_attention_weight_min": -24.93627166748047,
      "activations/layer1_attention_weight_max": 16.35774803161621,
      "activations/layer1_attention_weight_min": -13.85994815826416,
      "activations/layer20_attention_weight_max": 27.713048934936523,
      "activations/layer20_attention_weight_min": -22.492136001586914,
      "activations/layer21_attention_weight_max": 44.114662170410156,
      "activations/layer21_attention_weight_min": -25.5930118560791,
      "activations/layer22_attention_weight_max": 33.56782913208008,
      "activations/layer22_attention_weight_min": -26.6745662689209,
      "activations/layer23_attention_weight_max": 40.82481002807617,
      "activations/layer23_attention_weight_min": -23.051549911499023,
      "activations/layer2_attention_weight_max": 31.144683837890625,
      "activations/layer2_attention_weight_min": -29.71356964111328,
      "activations/layer3_attention_weight_max": 83.38097381591797,
      "activations/layer3_attention_weight_min": -88.77806091308594,
      "activations/layer4_attention_weight_max": 89.06743621826172,
      "activations/layer4_attention_weight_min": -88.58562469482422,
      "activations/layer5_attention_weight_max": 64.52630615234375,
      "activations/layer5_attention_weight_min": -74.31608581542969,
      "activations/layer6_attention_weight_max": 51.23516082763672,
      "activations/layer6_attention_weight_min": -53.35196304321289,
      "activations/layer7_attention_weight_max": 64.71308135986328,
      "activations/layer7_attention_weight_min": -64.85074615478516,
      "activations/layer8_attention_weight_max": 48.877708435058594,
      "activations/layer8_attention_weight_min": -49.92733383178711,
      "activations/layer9_attention_weight_max": 49.578697204589844,
      "activations/layer9_attention_weight_min": -48.18857192993164,
      "epoch": 16.48,
      "learning_rate": 4.414090909090909e-05,
      "loss": 2.7451,
      "step": 283700
    },
    {
      "activations/layer0_attention_weight_max": 16.179916381835938,
      "activations/layer0_attention_weight_min": -13.197748184204102,
      "activations/layer10_attention_weight_max": 35.371337890625,
      "activations/layer10_attention_weight_min": -34.773704528808594,
      "activations/layer11_attention_weight_max": 33.4915885925293,
      "activations/layer11_attention_weight_min": -33.4737434387207,
      "activations/layer12_attention_weight_max": 18.767709732055664,
      "activations/layer12_attention_weight_min": -25.006200790405273,
      "activations/layer13_attention_weight_max": 32.55354309082031,
      "activations/layer13_attention_weight_min": -29.582191467285156,
      "activations/layer14_attention_weight_max": 34.47228240966797,
      "activations/layer14_attention_weight_min": -31.2191162109375,
      "activations/layer15_attention_weight_max": 31.3941707611084,
      "activations/layer15_attention_weight_min": -29.540191650390625,
      "activations/layer16_attention_weight_max": 32.9931755065918,
      "activations/layer16_attention_weight_min": -30.871126174926758,
      "activations/layer17_attention_weight_max": 50.36336898803711,
      "activations/layer17_attention_weight_min": -47.12390899658203,
      "activations/layer18_attention_weight_max": 44.13663101196289,
      "activations/layer18_attention_weight_min": -39.459964752197266,
      "activations/layer19_attention_weight_max": 21.575414657592773,
      "activations/layer19_attention_weight_min": -20.974218368530273,
      "activations/layer1_attention_weight_max": 16.22029685974121,
      "activations/layer1_attention_weight_min": -12.900922775268555,
      "activations/layer20_attention_weight_max": 21.01665687561035,
      "activations/layer20_attention_weight_min": -23.16974639892578,
      "activations/layer21_attention_weight_max": 30.204504013061523,
      "activations/layer21_attention_weight_min": -23.77335548400879,
      "activations/layer22_attention_weight_max": 27.701169967651367,
      "activations/layer22_attention_weight_min": -27.9277286529541,
      "activations/layer23_attention_weight_max": 35.1555061340332,
      "activations/layer23_attention_weight_min": -28.04268455505371,
      "activations/layer2_attention_weight_max": 28.56172752380371,
      "activations/layer2_attention_weight_min": -29.17646598815918,
      "activations/layer3_attention_weight_max": 84.34934997558594,
      "activations/layer3_attention_weight_min": -90.26828002929688,
      "activations/layer4_attention_weight_max": 84.17572021484375,
      "activations/layer4_attention_weight_min": -87.05902862548828,
      "activations/layer5_attention_weight_max": 63.015987396240234,
      "activations/layer5_attention_weight_min": -76.63788604736328,
      "activations/layer6_attention_weight_max": 47.27605056762695,
      "activations/layer6_attention_weight_min": -49.50775909423828,
      "activations/layer7_attention_weight_max": 62.37554168701172,
      "activations/layer7_attention_weight_min": -61.452850341796875,
      "activations/layer8_attention_weight_max": 44.666988372802734,
      "activations/layer8_attention_weight_min": -47.53505325317383,
      "activations/layer9_attention_weight_max": 41.86069107055664,
      "activations/layer9_attention_weight_min": -46.38134002685547,
      "epoch": 16.49,
      "learning_rate": 4.4121969696969694e-05,
      "loss": 2.7438,
      "step": 283750
    },
    {
      "activations/layer0_attention_weight_max": 16.153112411499023,
      "activations/layer0_attention_weight_min": -13.19135570526123,
      "activations/layer10_attention_weight_max": 36.41041946411133,
      "activations/layer10_attention_weight_min": -34.602027893066406,
      "activations/layer11_attention_weight_max": 31.53724479675293,
      "activations/layer11_attention_weight_min": -33.04093933105469,
      "activations/layer12_attention_weight_max": 17.287616729736328,
      "activations/layer12_attention_weight_min": -22.672809600830078,
      "activations/layer13_attention_weight_max": 32.31098556518555,
      "activations/layer13_attention_weight_min": -28.17167091369629,
      "activations/layer14_attention_weight_max": 32.3065299987793,
      "activations/layer14_attention_weight_min": -31.086181640625,
      "activations/layer15_attention_weight_max": 30.113332748413086,
      "activations/layer15_attention_weight_min": -31.560998916625977,
      "activations/layer16_attention_weight_max": 31.65799903869629,
      "activations/layer16_attention_weight_min": -29.779001235961914,
      "activations/layer17_attention_weight_max": 47.40447235107422,
      "activations/layer17_attention_weight_min": -41.861534118652344,
      "activations/layer18_attention_weight_max": 41.650211334228516,
      "activations/layer18_attention_weight_min": -36.826995849609375,
      "activations/layer19_attention_weight_max": 22.001869201660156,
      "activations/layer19_attention_weight_min": -22.745878219604492,
      "activations/layer1_attention_weight_max": 15.885909080505371,
      "activations/layer1_attention_weight_min": -14.669397354125977,
      "activations/layer20_attention_weight_max": 22.35723114013672,
      "activations/layer20_attention_weight_min": -22.510089874267578,
      "activations/layer21_attention_weight_max": 32.228946685791016,
      "activations/layer21_attention_weight_min": -23.275890350341797,
      "activations/layer22_attention_weight_max": 26.193851470947266,
      "activations/layer22_attention_weight_min": -26.799579620361328,
      "activations/layer23_attention_weight_max": 40.769657135009766,
      "activations/layer23_attention_weight_min": -26.277206420898438,
      "activations/layer2_attention_weight_max": 30.926578521728516,
      "activations/layer2_attention_weight_min": -30.24770736694336,
      "activations/layer3_attention_weight_max": 84.31848907470703,
      "activations/layer3_attention_weight_min": -87.30036926269531,
      "activations/layer4_attention_weight_max": 86.23655700683594,
      "activations/layer4_attention_weight_min": -84.17247772216797,
      "activations/layer5_attention_weight_max": 64.72767639160156,
      "activations/layer5_attention_weight_min": -75.70597839355469,
      "activations/layer6_attention_weight_max": 48.15851974487305,
      "activations/layer6_attention_weight_min": -50.58595275878906,
      "activations/layer7_attention_weight_max": 61.8114013671875,
      "activations/layer7_attention_weight_min": -62.178768157958984,
      "activations/layer8_attention_weight_max": 43.64554214477539,
      "activations/layer8_attention_weight_min": -48.46281051635742,
      "activations/layer9_attention_weight_max": 44.35751724243164,
      "activations/layer9_attention_weight_min": -43.84768295288086,
      "epoch": 16.49,
      "learning_rate": 4.4103030303030296e-05,
      "loss": 2.7335,
      "step": 283800
    },
    {
      "activations/layer0_attention_weight_max": 15.702112197875977,
      "activations/layer0_attention_weight_min": -13.181411743164062,
      "activations/layer10_attention_weight_max": 36.21016311645508,
      "activations/layer10_attention_weight_min": -36.171939849853516,
      "activations/layer11_attention_weight_max": 33.012474060058594,
      "activations/layer11_attention_weight_min": -34.5640869140625,
      "activations/layer12_attention_weight_max": 19.27817726135254,
      "activations/layer12_attention_weight_min": -23.61973762512207,
      "activations/layer13_attention_weight_max": 34.40452575683594,
      "activations/layer13_attention_weight_min": -30.850624084472656,
      "activations/layer14_attention_weight_max": 39.502254486083984,
      "activations/layer14_attention_weight_min": -34.63084030151367,
      "activations/layer15_attention_weight_max": 36.41157913208008,
      "activations/layer15_attention_weight_min": -34.10310363769531,
      "activations/layer16_attention_weight_max": 34.95962142944336,
      "activations/layer16_attention_weight_min": -32.19330596923828,
      "activations/layer17_attention_weight_max": 50.33917999267578,
      "activations/layer17_attention_weight_min": -45.63533401489258,
      "activations/layer18_attention_weight_max": 45.70698928833008,
      "activations/layer18_attention_weight_min": -40.46809387207031,
      "activations/layer19_attention_weight_max": 25.726119995117188,
      "activations/layer19_attention_weight_min": -22.632156372070312,
      "activations/layer1_attention_weight_max": 15.763809204101562,
      "activations/layer1_attention_weight_min": -13.898350715637207,
      "activations/layer20_attention_weight_max": 24.108474731445312,
      "activations/layer20_attention_weight_min": -23.218778610229492,
      "activations/layer21_attention_weight_max": 39.480350494384766,
      "activations/layer21_attention_weight_min": -24.456315994262695,
      "activations/layer22_attention_weight_max": 31.10159683227539,
      "activations/layer22_attention_weight_min": -28.52684211730957,
      "activations/layer23_attention_weight_max": 34.550071716308594,
      "activations/layer23_attention_weight_min": -24.338088989257812,
      "activations/layer2_attention_weight_max": 31.996496200561523,
      "activations/layer2_attention_weight_min": -30.089832305908203,
      "activations/layer3_attention_weight_max": 89.0003662109375,
      "activations/layer3_attention_weight_min": -91.27166748046875,
      "activations/layer4_attention_weight_max": 87.44489288330078,
      "activations/layer4_attention_weight_min": -87.8799057006836,
      "activations/layer5_attention_weight_max": 61.83324432373047,
      "activations/layer5_attention_weight_min": -72.25930786132812,
      "activations/layer6_attention_weight_max": 47.9157600402832,
      "activations/layer6_attention_weight_min": -50.30805969238281,
      "activations/layer7_attention_weight_max": 62.398345947265625,
      "activations/layer7_attention_weight_min": -64.93739318847656,
      "activations/layer8_attention_weight_max": 43.13347625732422,
      "activations/layer8_attention_weight_min": -49.392852783203125,
      "activations/layer9_attention_weight_max": 43.051544189453125,
      "activations/layer9_attention_weight_min": -50.643856048583984,
      "epoch": 16.49,
      "learning_rate": 4.4084090909090904e-05,
      "loss": 2.7486,
      "step": 283850
    },
    {
      "activations/layer0_attention_weight_max": 15.545182228088379,
      "activations/layer0_attention_weight_min": -13.556381225585938,
      "activations/layer10_attention_weight_max": 37.413795471191406,
      "activations/layer10_attention_weight_min": -36.0327262878418,
      "activations/layer11_attention_weight_max": 32.073585510253906,
      "activations/layer11_attention_weight_min": -35.53792190551758,
      "activations/layer12_attention_weight_max": 18.464323043823242,
      "activations/layer12_attention_weight_min": -24.76020622253418,
      "activations/layer13_attention_weight_max": 37.892662048339844,
      "activations/layer13_attention_weight_min": -31.287195205688477,
      "activations/layer14_attention_weight_max": 34.78750228881836,
      "activations/layer14_attention_weight_min": -29.54265022277832,
      "activations/layer15_attention_weight_max": 32.726966857910156,
      "activations/layer15_attention_weight_min": -28.56680679321289,
      "activations/layer16_attention_weight_max": 35.652488708496094,
      "activations/layer16_attention_weight_min": -33.15541458129883,
      "activations/layer17_attention_weight_max": 53.75182342529297,
      "activations/layer17_attention_weight_min": -46.3848991394043,
      "activations/layer18_attention_weight_max": 44.280460357666016,
      "activations/layer18_attention_weight_min": -39.32626724243164,
      "activations/layer19_attention_weight_max": 24.137601852416992,
      "activations/layer19_attention_weight_min": -22.103164672851562,
      "activations/layer1_attention_weight_max": 16.059974670410156,
      "activations/layer1_attention_weight_min": -15.094952583312988,
      "activations/layer20_attention_weight_max": 25.38485336303711,
      "activations/layer20_attention_weight_min": -20.65472412109375,
      "activations/layer21_attention_weight_max": 39.810821533203125,
      "activations/layer21_attention_weight_min": -22.9536075592041,
      "activations/layer22_attention_weight_max": 29.26611328125,
      "activations/layer22_attention_weight_min": -24.550643920898438,
      "activations/layer23_attention_weight_max": 40.45807647705078,
      "activations/layer23_attention_weight_min": -25.334470748901367,
      "activations/layer2_attention_weight_max": 30.52785301208496,
      "activations/layer2_attention_weight_min": -29.109806060791016,
      "activations/layer3_attention_weight_max": 85.42790222167969,
      "activations/layer3_attention_weight_min": -89.52328491210938,
      "activations/layer4_attention_weight_max": 87.25222778320312,
      "activations/layer4_attention_weight_min": -89.97230529785156,
      "activations/layer5_attention_weight_max": 64.98906707763672,
      "activations/layer5_attention_weight_min": -76.393798828125,
      "activations/layer6_attention_weight_max": 49.54237747192383,
      "activations/layer6_attention_weight_min": -53.07645034790039,
      "activations/layer7_attention_weight_max": 63.413150787353516,
      "activations/layer7_attention_weight_min": -65.38800811767578,
      "activations/layer8_attention_weight_max": 44.15340042114258,
      "activations/layer8_attention_weight_min": -48.9979248046875,
      "activations/layer9_attention_weight_max": 48.304054260253906,
      "activations/layer9_attention_weight_min": -46.15082550048828,
      "epoch": 16.5,
      "learning_rate": 4.406515151515151e-05,
      "loss": 2.7608,
      "step": 283900
    },
    {
      "activations/layer0_attention_weight_max": 16.101844787597656,
      "activations/layer0_attention_weight_min": -13.348335266113281,
      "activations/layer10_attention_weight_max": 37.44550323486328,
      "activations/layer10_attention_weight_min": -35.080909729003906,
      "activations/layer11_attention_weight_max": 34.404136657714844,
      "activations/layer11_attention_weight_min": -33.56695556640625,
      "activations/layer12_attention_weight_max": 18.30219078063965,
      "activations/layer12_attention_weight_min": -30.812654495239258,
      "activations/layer13_attention_weight_max": 37.05945587158203,
      "activations/layer13_attention_weight_min": -29.500080108642578,
      "activations/layer14_attention_weight_max": 34.27476501464844,
      "activations/layer14_attention_weight_min": -29.082300186157227,
      "activations/layer15_attention_weight_max": 33.85963439941406,
      "activations/layer15_attention_weight_min": -29.399354934692383,
      "activations/layer16_attention_weight_max": 31.162731170654297,
      "activations/layer16_attention_weight_min": -30.446996688842773,
      "activations/layer17_attention_weight_max": 49.8314094543457,
      "activations/layer17_attention_weight_min": -43.4046745300293,
      "activations/layer18_attention_weight_max": 45.941619873046875,
      "activations/layer18_attention_weight_min": -38.121822357177734,
      "activations/layer19_attention_weight_max": 22.688825607299805,
      "activations/layer19_attention_weight_min": -23.260311126708984,
      "activations/layer1_attention_weight_max": 16.77794075012207,
      "activations/layer1_attention_weight_min": -13.41033935546875,
      "activations/layer20_attention_weight_max": 21.359983444213867,
      "activations/layer20_attention_weight_min": -23.148290634155273,
      "activations/layer21_attention_weight_max": 33.76259994506836,
      "activations/layer21_attention_weight_min": -23.86724281311035,
      "activations/layer22_attention_weight_max": 27.459150314331055,
      "activations/layer22_attention_weight_min": -26.1723690032959,
      "activations/layer23_attention_weight_max": 39.0731086730957,
      "activations/layer23_attention_weight_min": -26.797077178955078,
      "activations/layer2_attention_weight_max": 30.848709106445312,
      "activations/layer2_attention_weight_min": -30.612926483154297,
      "activations/layer3_attention_weight_max": 87.85421752929688,
      "activations/layer3_attention_weight_min": -91.80252838134766,
      "activations/layer4_attention_weight_max": 91.82988739013672,
      "activations/layer4_attention_weight_min": -89.85411071777344,
      "activations/layer5_attention_weight_max": 67.24740600585938,
      "activations/layer5_attention_weight_min": -70.04693603515625,
      "activations/layer6_attention_weight_max": 49.65248489379883,
      "activations/layer6_attention_weight_min": -50.08090591430664,
      "activations/layer7_attention_weight_max": 65.41999816894531,
      "activations/layer7_attention_weight_min": -62.706024169921875,
      "activations/layer8_attention_weight_max": 48.47494125366211,
      "activations/layer8_attention_weight_min": -48.104244232177734,
      "activations/layer9_attention_weight_max": 45.02723693847656,
      "activations/layer9_attention_weight_min": -44.31382369995117,
      "epoch": 16.5,
      "learning_rate": 4.404621212121212e-05,
      "loss": 2.749,
      "step": 283950
    },
    {
      "activations/layer0_attention_weight_max": 16.664941787719727,
      "activations/layer0_attention_weight_min": -13.554106712341309,
      "activations/layer10_attention_weight_max": 35.456336975097656,
      "activations/layer10_attention_weight_min": -36.566680908203125,
      "activations/layer11_attention_weight_max": 32.8110466003418,
      "activations/layer11_attention_weight_min": -33.359031677246094,
      "activations/layer12_attention_weight_max": 18.565895080566406,
      "activations/layer12_attention_weight_min": -23.130508422851562,
      "activations/layer13_attention_weight_max": 31.506582260131836,
      "activations/layer13_attention_weight_min": -27.42526626586914,
      "activations/layer14_attention_weight_max": 33.56088638305664,
      "activations/layer14_attention_weight_min": -29.698240280151367,
      "activations/layer15_attention_weight_max": 33.32733154296875,
      "activations/layer15_attention_weight_min": -30.353981018066406,
      "activations/layer16_attention_weight_max": 34.10422134399414,
      "activations/layer16_attention_weight_min": -33.12978744506836,
      "activations/layer17_attention_weight_max": 49.39498519897461,
      "activations/layer17_attention_weight_min": -45.36256408691406,
      "activations/layer18_attention_weight_max": 46.4160041809082,
      "activations/layer18_attention_weight_min": -38.62899398803711,
      "activations/layer19_attention_weight_max": 23.399837493896484,
      "activations/layer19_attention_weight_min": -24.17266082763672,
      "activations/layer1_attention_weight_max": 16.39333724975586,
      "activations/layer1_attention_weight_min": -14.459220886230469,
      "activations/layer20_attention_weight_max": 23.639385223388672,
      "activations/layer20_attention_weight_min": -23.133956909179688,
      "activations/layer21_attention_weight_max": 36.823036193847656,
      "activations/layer21_attention_weight_min": -24.528162002563477,
      "activations/layer22_attention_weight_max": 29.82636260986328,
      "activations/layer22_attention_weight_min": -27.62381935119629,
      "activations/layer23_attention_weight_max": 37.533416748046875,
      "activations/layer23_attention_weight_min": -25.948902130126953,
      "activations/layer2_attention_weight_max": 32.743507385253906,
      "activations/layer2_attention_weight_min": -31.33833885192871,
      "activations/layer3_attention_weight_max": 87.07320404052734,
      "activations/layer3_attention_weight_min": -88.35823822021484,
      "activations/layer4_attention_weight_max": 90.72818756103516,
      "activations/layer4_attention_weight_min": -89.0337142944336,
      "activations/layer5_attention_weight_max": 71.81388854980469,
      "activations/layer5_attention_weight_min": -74.23368835449219,
      "activations/layer6_attention_weight_max": 50.592708587646484,
      "activations/layer6_attention_weight_min": -53.1739616394043,
      "activations/layer7_attention_weight_max": 61.69060516357422,
      "activations/layer7_attention_weight_min": -61.7350959777832,
      "activations/layer8_attention_weight_max": 47.378326416015625,
      "activations/layer8_attention_weight_min": -49.20170974731445,
      "activations/layer9_attention_weight_max": 41.741600036621094,
      "activations/layer9_attention_weight_min": -44.83218002319336,
      "epoch": 16.5,
      "learning_rate": 4.402727272727272e-05,
      "loss": 2.7416,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4413,
      "eval_samples_per_second": 508.69,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4413,
      "eval_openwebtext_samples_per_second": 508.69,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.9378,
      "eval_wikitext_samples_per_second": 235.319,
      "step": 284000
    },
    {
      "epoch": 16.5,
      "eval_lambada_loss": 2.470703125,
      "eval_lambada_ppl": 11.830762432692795,
      "eval_lambada_runtime": 9.523,
      "eval_lambada_samples_per_second": 511.289,
      "step": 284000
    },
    {
      "activations/layer0_attention_weight_max": 16.6263484954834,
      "activations/layer0_attention_weight_min": -13.815905570983887,
      "activations/layer10_attention_weight_max": 35.9521484375,
      "activations/layer10_attention_weight_min": -33.91621780395508,
      "activations/layer11_attention_weight_max": 31.570114135742188,
      "activations/layer11_attention_weight_min": -31.942798614501953,
      "activations/layer12_attention_weight_max": 18.408905029296875,
      "activations/layer12_attention_weight_min": -26.478736877441406,
      "activations/layer13_attention_weight_max": 35.167327880859375,
      "activations/layer13_attention_weight_min": -26.955341339111328,
      "activations/layer14_attention_weight_max": 35.45261001586914,
      "activations/layer14_attention_weight_min": -30.846372604370117,
      "activations/layer15_attention_weight_max": 30.35964012145996,
      "activations/layer15_attention_weight_min": -30.36267852783203,
      "activations/layer16_attention_weight_max": 32.95671081542969,
      "activations/layer16_attention_weight_min": -31.103755950927734,
      "activations/layer17_attention_weight_max": 53.76519012451172,
      "activations/layer17_attention_weight_min": -44.631465911865234,
      "activations/layer18_attention_weight_max": 45.60326385498047,
      "activations/layer18_attention_weight_min": -38.92402267456055,
      "activations/layer19_attention_weight_max": 23.960952758789062,
      "activations/layer19_attention_weight_min": -24.97696304321289,
      "activations/layer1_attention_weight_max": 16.458524703979492,
      "activations/layer1_attention_weight_min": -15.887667655944824,
      "activations/layer20_attention_weight_max": 23.381139755249023,
      "activations/layer20_attention_weight_min": -24.070472717285156,
      "activations/layer21_attention_weight_max": 40.68681335449219,
      "activations/layer21_attention_weight_min": -24.55206298828125,
      "activations/layer22_attention_weight_max": 30.7054500579834,
      "activations/layer22_attention_weight_min": -27.886322021484375,
      "activations/layer23_attention_weight_max": 40.968605041503906,
      "activations/layer23_attention_weight_min": -26.273054122924805,
      "activations/layer2_attention_weight_max": 29.25904655456543,
      "activations/layer2_attention_weight_min": -30.92525291442871,
      "activations/layer3_attention_weight_max": 83.74567413330078,
      "activations/layer3_attention_weight_min": -86.56072998046875,
      "activations/layer4_attention_weight_max": 83.30281829833984,
      "activations/layer4_attention_weight_min": -79.1749496459961,
      "activations/layer5_attention_weight_max": 61.478614807128906,
      "activations/layer5_attention_weight_min": -68.19688415527344,
      "activations/layer6_attention_weight_max": 49.06699752807617,
      "activations/layer6_attention_weight_min": -47.51115798950195,
      "activations/layer7_attention_weight_max": 62.776763916015625,
      "activations/layer7_attention_weight_min": -59.10194778442383,
      "activations/layer8_attention_weight_max": 49.97027587890625,
      "activations/layer8_attention_weight_min": -49.924415588378906,
      "activations/layer9_attention_weight_max": 46.16360092163086,
      "activations/layer9_attention_weight_min": -45.22040939331055,
      "epoch": 16.5,
      "learning_rate": 4.400833333333332e-05,
      "loss": 2.7519,
      "step": 284050
    },
    {
      "activations/layer0_attention_weight_max": 14.96886157989502,
      "activations/layer0_attention_weight_min": -14.05784797668457,
      "activations/layer10_attention_weight_max": 33.99647521972656,
      "activations/layer10_attention_weight_min": -33.62641525268555,
      "activations/layer11_attention_weight_max": 31.82196617126465,
      "activations/layer11_attention_weight_min": -34.960784912109375,
      "activations/layer12_attention_weight_max": 18.854263305664062,
      "activations/layer12_attention_weight_min": -29.12468147277832,
      "activations/layer13_attention_weight_max": 32.42858123779297,
      "activations/layer13_attention_weight_min": -26.951147079467773,
      "activations/layer14_attention_weight_max": 35.290748596191406,
      "activations/layer14_attention_weight_min": -30.40931510925293,
      "activations/layer15_attention_weight_max": 32.647315979003906,
      "activations/layer15_attention_weight_min": -30.883607864379883,
      "activations/layer16_attention_weight_max": 33.45030975341797,
      "activations/layer16_attention_weight_min": -31.96712303161621,
      "activations/layer17_attention_weight_max": 54.20252227783203,
      "activations/layer17_attention_weight_min": -45.03348922729492,
      "activations/layer18_attention_weight_max": 47.92655563354492,
      "activations/layer18_attention_weight_min": -39.259368896484375,
      "activations/layer19_attention_weight_max": 24.23882484436035,
      "activations/layer19_attention_weight_min": -25.223918914794922,
      "activations/layer1_attention_weight_max": 16.03104591369629,
      "activations/layer1_attention_weight_min": -15.161203384399414,
      "activations/layer20_attention_weight_max": 23.014114379882812,
      "activations/layer20_attention_weight_min": -22.736528396606445,
      "activations/layer21_attention_weight_max": 38.44392395019531,
      "activations/layer21_attention_weight_min": -22.869964599609375,
      "activations/layer22_attention_weight_max": 29.691011428833008,
      "activations/layer22_attention_weight_min": -25.53073501586914,
      "activations/layer23_attention_weight_max": 37.49299621582031,
      "activations/layer23_attention_weight_min": -25.15103530883789,
      "activations/layer2_attention_weight_max": 29.110553741455078,
      "activations/layer2_attention_weight_min": -28.554746627807617,
      "activations/layer3_attention_weight_max": 82.46918487548828,
      "activations/layer3_attention_weight_min": -83.576416015625,
      "activations/layer4_attention_weight_max": 82.04180145263672,
      "activations/layer4_attention_weight_min": -84.03121185302734,
      "activations/layer5_attention_weight_max": 64.80012512207031,
      "activations/layer5_attention_weight_min": -70.3189697265625,
      "activations/layer6_attention_weight_max": 48.526119232177734,
      "activations/layer6_attention_weight_min": -51.728885650634766,
      "activations/layer7_attention_weight_max": 62.78046798706055,
      "activations/layer7_attention_weight_min": -64.37737274169922,
      "activations/layer8_attention_weight_max": 46.24129104614258,
      "activations/layer8_attention_weight_min": -51.53364181518555,
      "activations/layer9_attention_weight_max": 43.26285171508789,
      "activations/layer9_attention_weight_min": -46.57278060913086,
      "epoch": 16.51,
      "learning_rate": 4.398939393939394e-05,
      "loss": 2.758,
      "step": 284100
    },
    {
      "activations/layer0_attention_weight_max": 16.34234046936035,
      "activations/layer0_attention_weight_min": -13.644962310791016,
      "activations/layer10_attention_weight_max": 44.02382278442383,
      "activations/layer10_attention_weight_min": -42.78548049926758,
      "activations/layer11_attention_weight_max": 42.34699630737305,
      "activations/layer11_attention_weight_min": -39.463294982910156,
      "activations/layer12_attention_weight_max": 17.863910675048828,
      "activations/layer12_attention_weight_min": -26.879322052001953,
      "activations/layer13_attention_weight_max": 39.45771026611328,
      "activations/layer13_attention_weight_min": -30.053651809692383,
      "activations/layer14_attention_weight_max": 33.58744812011719,
      "activations/layer14_attention_weight_min": -30.05131721496582,
      "activations/layer15_attention_weight_max": 33.89508819580078,
      "activations/layer15_attention_weight_min": -31.35628318786621,
      "activations/layer16_attention_weight_max": 31.230806350708008,
      "activations/layer16_attention_weight_min": -29.2542724609375,
      "activations/layer17_attention_weight_max": 48.1272087097168,
      "activations/layer17_attention_weight_min": -45.9677848815918,
      "activations/layer18_attention_weight_max": 44.29229736328125,
      "activations/layer18_attention_weight_min": -42.57539367675781,
      "activations/layer19_attention_weight_max": 20.809743881225586,
      "activations/layer19_attention_weight_min": -24.11838150024414,
      "activations/layer1_attention_weight_max": 16.41880989074707,
      "activations/layer1_attention_weight_min": -13.271625518798828,
      "activations/layer20_attention_weight_max": 21.669504165649414,
      "activations/layer20_attention_weight_min": -22.526905059814453,
      "activations/layer21_attention_weight_max": 34.3619499206543,
      "activations/layer21_attention_weight_min": -24.6816463470459,
      "activations/layer22_attention_weight_max": 28.176191329956055,
      "activations/layer22_attention_weight_min": -26.31422996520996,
      "activations/layer23_attention_weight_max": 37.32886505126953,
      "activations/layer23_attention_weight_min": -21.971240997314453,
      "activations/layer2_attention_weight_max": 30.927196502685547,
      "activations/layer2_attention_weight_min": -29.033843994140625,
      "activations/layer3_attention_weight_max": 86.30613708496094,
      "activations/layer3_attention_weight_min": -89.5367202758789,
      "activations/layer4_attention_weight_max": 92.10237884521484,
      "activations/layer4_attention_weight_min": -86.02854919433594,
      "activations/layer5_attention_weight_max": 73.00091552734375,
      "activations/layer5_attention_weight_min": -73.13866424560547,
      "activations/layer6_attention_weight_max": 48.34602737426758,
      "activations/layer6_attention_weight_min": -50.90769577026367,
      "activations/layer7_attention_weight_max": 74.7168197631836,
      "activations/layer7_attention_weight_min": -69.32179260253906,
      "activations/layer8_attention_weight_max": 54.10967254638672,
      "activations/layer8_attention_weight_min": -51.63334655761719,
      "activations/layer9_attention_weight_max": 62.39187240600586,
      "activations/layer9_attention_weight_min": -54.974639892578125,
      "epoch": 16.51,
      "learning_rate": 4.397045454545454e-05,
      "loss": 2.7432,
      "step": 284150
    },
    {
      "activations/layer0_attention_weight_max": 15.190657615661621,
      "activations/layer0_attention_weight_min": -14.209339141845703,
      "activations/layer10_attention_weight_max": 39.406105041503906,
      "activations/layer10_attention_weight_min": -38.46647262573242,
      "activations/layer11_attention_weight_max": 34.45767593383789,
      "activations/layer11_attention_weight_min": -33.122581481933594,
      "activations/layer12_attention_weight_max": 17.893796920776367,
      "activations/layer12_attention_weight_min": -25.052988052368164,
      "activations/layer13_attention_weight_max": 39.922760009765625,
      "activations/layer13_attention_weight_min": -26.68659782409668,
      "activations/layer14_attention_weight_max": 34.37953186035156,
      "activations/layer14_attention_weight_min": -29.84354019165039,
      "activations/layer15_attention_weight_max": 34.33782958984375,
      "activations/layer15_attention_weight_min": -28.67023468017578,
      "activations/layer16_attention_weight_max": 34.42495346069336,
      "activations/layer16_attention_weight_min": -32.77245330810547,
      "activations/layer17_attention_weight_max": 52.89729690551758,
      "activations/layer17_attention_weight_min": -46.9454460144043,
      "activations/layer18_attention_weight_max": 49.39794158935547,
      "activations/layer18_attention_weight_min": -40.302154541015625,
      "activations/layer19_attention_weight_max": 23.337263107299805,
      "activations/layer19_attention_weight_min": -22.373214721679688,
      "activations/layer1_attention_weight_max": 16.25717544555664,
      "activations/layer1_attention_weight_min": -13.13608455657959,
      "activations/layer20_attention_weight_max": 22.762338638305664,
      "activations/layer20_attention_weight_min": -21.139610290527344,
      "activations/layer21_attention_weight_max": 35.762813568115234,
      "activations/layer21_attention_weight_min": -23.175857543945312,
      "activations/layer22_attention_weight_max": 29.093767166137695,
      "activations/layer22_attention_weight_min": -24.63774871826172,
      "activations/layer23_attention_weight_max": 38.805213928222656,
      "activations/layer23_attention_weight_min": -24.486791610717773,
      "activations/layer2_attention_weight_max": 30.145252227783203,
      "activations/layer2_attention_weight_min": -29.505172729492188,
      "activations/layer3_attention_weight_max": 88.14199829101562,
      "activations/layer3_attention_weight_min": -84.70622253417969,
      "activations/layer4_attention_weight_max": 87.14253234863281,
      "activations/layer4_attention_weight_min": -86.360107421875,
      "activations/layer5_attention_weight_max": 62.90018844604492,
      "activations/layer5_attention_weight_min": -71.6556167602539,
      "activations/layer6_attention_weight_max": 49.62128829956055,
      "activations/layer6_attention_weight_min": -51.410911560058594,
      "activations/layer7_attention_weight_max": 65.15099334716797,
      "activations/layer7_attention_weight_min": -65.25284576416016,
      "activations/layer8_attention_weight_max": 45.710872650146484,
      "activations/layer8_attention_weight_min": -46.79008483886719,
      "activations/layer9_attention_weight_max": 46.987239837646484,
      "activations/layer9_attention_weight_min": -45.31928253173828,
      "epoch": 16.51,
      "learning_rate": 4.395151515151515e-05,
      "loss": 2.7447,
      "step": 284200
    },
    {
      "activations/layer0_attention_weight_max": 15.704743385314941,
      "activations/layer0_attention_weight_min": -13.90800952911377,
      "activations/layer10_attention_weight_max": 35.5528450012207,
      "activations/layer10_attention_weight_min": -33.41557312011719,
      "activations/layer11_attention_weight_max": 31.711891174316406,
      "activations/layer11_attention_weight_min": -32.805416107177734,
      "activations/layer12_attention_weight_max": 20.09324836730957,
      "activations/layer12_attention_weight_min": -21.352523803710938,
      "activations/layer13_attention_weight_max": 32.17887878417969,
      "activations/layer13_attention_weight_min": -29.183361053466797,
      "activations/layer14_attention_weight_max": 34.387481689453125,
      "activations/layer14_attention_weight_min": -29.704465866088867,
      "activations/layer15_attention_weight_max": 33.15201187133789,
      "activations/layer15_attention_weight_min": -30.697784423828125,
      "activations/layer16_attention_weight_max": 34.37335205078125,
      "activations/layer16_attention_weight_min": -33.015750885009766,
      "activations/layer17_attention_weight_max": 54.75606155395508,
      "activations/layer17_attention_weight_min": -49.55794143676758,
      "activations/layer18_attention_weight_max": 49.8231315612793,
      "activations/layer18_attention_weight_min": -44.904335021972656,
      "activations/layer19_attention_weight_max": 24.026735305786133,
      "activations/layer19_attention_weight_min": -22.876073837280273,
      "activations/layer1_attention_weight_max": 15.455127716064453,
      "activations/layer1_attention_weight_min": -14.586844444274902,
      "activations/layer20_attention_weight_max": 23.576017379760742,
      "activations/layer20_attention_weight_min": -22.734609603881836,
      "activations/layer21_attention_weight_max": 35.58039855957031,
      "activations/layer21_attention_weight_min": -22.94707679748535,
      "activations/layer22_attention_weight_max": 31.295747756958008,
      "activations/layer22_attention_weight_min": -24.7321834564209,
      "activations/layer23_attention_weight_max": 36.66578674316406,
      "activations/layer23_attention_weight_min": -23.653961181640625,
      "activations/layer2_attention_weight_max": 30.272550582885742,
      "activations/layer2_attention_weight_min": -28.587993621826172,
      "activations/layer3_attention_weight_max": 87.52628326416016,
      "activations/layer3_attention_weight_min": -84.96397399902344,
      "activations/layer4_attention_weight_max": 84.3115463256836,
      "activations/layer4_attention_weight_min": -83.51901245117188,
      "activations/layer5_attention_weight_max": 62.96575164794922,
      "activations/layer5_attention_weight_min": -74.88180541992188,
      "activations/layer6_attention_weight_max": 49.81446838378906,
      "activations/layer6_attention_weight_min": -50.0362663269043,
      "activations/layer7_attention_weight_max": 61.013465881347656,
      "activations/layer7_attention_weight_min": -61.46358871459961,
      "activations/layer8_attention_weight_max": 44.45298385620117,
      "activations/layer8_attention_weight_min": -47.35356140136719,
      "activations/layer9_attention_weight_max": 43.67252731323242,
      "activations/layer9_attention_weight_min": -45.67856979370117,
      "epoch": 16.52,
      "learning_rate": 4.393257575757575e-05,
      "loss": 2.7622,
      "step": 284250
    },
    {
      "activations/layer0_attention_weight_max": 15.597671508789062,
      "activations/layer0_attention_weight_min": -13.943745613098145,
      "activations/layer10_attention_weight_max": 38.21929168701172,
      "activations/layer10_attention_weight_min": -40.01701736450195,
      "activations/layer11_attention_weight_max": 34.650047302246094,
      "activations/layer11_attention_weight_min": -34.52273941040039,
      "activations/layer12_attention_weight_max": 19.53312873840332,
      "activations/layer12_attention_weight_min": -23.654457092285156,
      "activations/layer13_attention_weight_max": 39.078399658203125,
      "activations/layer13_attention_weight_min": -29.347505569458008,
      "activations/layer14_attention_weight_max": 42.43027114868164,
      "activations/layer14_attention_weight_min": -33.20893096923828,
      "activations/layer15_attention_weight_max": 36.05961990356445,
      "activations/layer15_attention_weight_min": -31.3682861328125,
      "activations/layer16_attention_weight_max": 37.58969497680664,
      "activations/layer16_attention_weight_min": -32.52968215942383,
      "activations/layer17_attention_weight_max": 58.35091018676758,
      "activations/layer17_attention_weight_min": -47.83998489379883,
      "activations/layer18_attention_weight_max": 49.50819396972656,
      "activations/layer18_attention_weight_min": -39.614715576171875,
      "activations/layer19_attention_weight_max": 24.5162353515625,
      "activations/layer19_attention_weight_min": -22.427207946777344,
      "activations/layer1_attention_weight_max": 16.04305648803711,
      "activations/layer1_attention_weight_min": -16.839527130126953,
      "activations/layer20_attention_weight_max": 24.64593505859375,
      "activations/layer20_attention_weight_min": -20.88245964050293,
      "activations/layer21_attention_weight_max": 39.50518798828125,
      "activations/layer21_attention_weight_min": -22.083942413330078,
      "activations/layer22_attention_weight_max": 29.963851928710938,
      "activations/layer22_attention_weight_min": -26.27937889099121,
      "activations/layer23_attention_weight_max": 39.28248596191406,
      "activations/layer23_attention_weight_min": -24.124103546142578,
      "activations/layer2_attention_weight_max": 30.39082908630371,
      "activations/layer2_attention_weight_min": -29.914533615112305,
      "activations/layer3_attention_weight_max": 86.38888549804688,
      "activations/layer3_attention_weight_min": -88.65201568603516,
      "activations/layer4_attention_weight_max": 88.71724700927734,
      "activations/layer4_attention_weight_min": -87.41423797607422,
      "activations/layer5_attention_weight_max": 61.939239501953125,
      "activations/layer5_attention_weight_min": -76.54318237304688,
      "activations/layer6_attention_weight_max": 48.897708892822266,
      "activations/layer6_attention_weight_min": -51.5243034362793,
      "activations/layer7_attention_weight_max": 62.345943450927734,
      "activations/layer7_attention_weight_min": -63.39849853515625,
      "activations/layer8_attention_weight_max": 49.315185546875,
      "activations/layer8_attention_weight_min": -49.32172775268555,
      "activations/layer9_attention_weight_max": 46.429378509521484,
      "activations/layer9_attention_weight_min": -47.16175842285156,
      "epoch": 16.52,
      "learning_rate": 4.3913636363636365e-05,
      "loss": 2.7533,
      "step": 284300
    },
    {
      "activations/layer0_attention_weight_max": 16.26805305480957,
      "activations/layer0_attention_weight_min": -14.29299545288086,
      "activations/layer10_attention_weight_max": 33.1271858215332,
      "activations/layer10_attention_weight_min": -32.821067810058594,
      "activations/layer11_attention_weight_max": 30.0651912689209,
      "activations/layer11_attention_weight_min": -30.868894577026367,
      "activations/layer12_attention_weight_max": 18.507720947265625,
      "activations/layer12_attention_weight_min": -26.16666603088379,
      "activations/layer13_attention_weight_max": 30.834535598754883,
      "activations/layer13_attention_weight_min": -26.695295333862305,
      "activations/layer14_attention_weight_max": 36.02442932128906,
      "activations/layer14_attention_weight_min": -30.17464256286621,
      "activations/layer15_attention_weight_max": 31.63402557373047,
      "activations/layer15_attention_weight_min": -29.436721801757812,
      "activations/layer16_attention_weight_max": 32.625728607177734,
      "activations/layer16_attention_weight_min": -32.04701232910156,
      "activations/layer17_attention_weight_max": 50.3428955078125,
      "activations/layer17_attention_weight_min": -44.82301330566406,
      "activations/layer18_attention_weight_max": 45.23571014404297,
      "activations/layer18_attention_weight_min": -39.63636016845703,
      "activations/layer19_attention_weight_max": 23.37702751159668,
      "activations/layer19_attention_weight_min": -22.13389778137207,
      "activations/layer1_attention_weight_max": 15.99213981628418,
      "activations/layer1_attention_weight_min": -13.941706657409668,
      "activations/layer20_attention_weight_max": 21.562786102294922,
      "activations/layer20_attention_weight_min": -22.87964630126953,
      "activations/layer21_attention_weight_max": 32.34602737426758,
      "activations/layer21_attention_weight_min": -23.62774658203125,
      "activations/layer22_attention_weight_max": 27.714372634887695,
      "activations/layer22_attention_weight_min": -27.128150939941406,
      "activations/layer23_attention_weight_max": 39.0419921875,
      "activations/layer23_attention_weight_min": -25.68037223815918,
      "activations/layer2_attention_weight_max": 29.442604064941406,
      "activations/layer2_attention_weight_min": -32.63213348388672,
      "activations/layer3_attention_weight_max": 82.09661865234375,
      "activations/layer3_attention_weight_min": -87.97124481201172,
      "activations/layer4_attention_weight_max": 82.87706756591797,
      "activations/layer4_attention_weight_min": -82.1111068725586,
      "activations/layer5_attention_weight_max": 60.436439514160156,
      "activations/layer5_attention_weight_min": -75.77467346191406,
      "activations/layer6_attention_weight_max": 49.755985260009766,
      "activations/layer6_attention_weight_min": -51.35867691040039,
      "activations/layer7_attention_weight_max": 65.46696472167969,
      "activations/layer7_attention_weight_min": -62.44817352294922,
      "activations/layer8_attention_weight_max": 43.95289993286133,
      "activations/layer8_attention_weight_min": -45.86027145385742,
      "activations/layer9_attention_weight_max": 42.522640228271484,
      "activations/layer9_attention_weight_min": -44.384178161621094,
      "epoch": 16.52,
      "learning_rate": 4.3894696969696966e-05,
      "loss": 2.7453,
      "step": 284350
    },
    {
      "activations/layer0_attention_weight_max": 15.551719665527344,
      "activations/layer0_attention_weight_min": -14.052054405212402,
      "activations/layer10_attention_weight_max": 32.93297576904297,
      "activations/layer10_attention_weight_min": -35.43092727661133,
      "activations/layer11_attention_weight_max": 30.428836822509766,
      "activations/layer11_attention_weight_min": -31.76968765258789,
      "activations/layer12_attention_weight_max": 18.13421630859375,
      "activations/layer12_attention_weight_min": -26.37051773071289,
      "activations/layer13_attention_weight_max": 33.709964752197266,
      "activations/layer13_attention_weight_min": -29.179372787475586,
      "activations/layer14_attention_weight_max": 36.52223587036133,
      "activations/layer14_attention_weight_min": -31.800973892211914,
      "activations/layer15_attention_weight_max": 32.14225769042969,
      "activations/layer15_attention_weight_min": -33.30681610107422,
      "activations/layer16_attention_weight_max": 32.72971725463867,
      "activations/layer16_attention_weight_min": -34.469261169433594,
      "activations/layer17_attention_weight_max": 50.22701644897461,
      "activations/layer17_attention_weight_min": -48.17961502075195,
      "activations/layer18_attention_weight_max": 42.991641998291016,
      "activations/layer18_attention_weight_min": -42.46987533569336,
      "activations/layer19_attention_weight_max": 22.716276168823242,
      "activations/layer19_attention_weight_min": -24.040578842163086,
      "activations/layer1_attention_weight_max": 16.681306838989258,
      "activations/layer1_attention_weight_min": -14.29077434539795,
      "activations/layer20_attention_weight_max": 21.878873825073242,
      "activations/layer20_attention_weight_min": -22.990598678588867,
      "activations/layer21_attention_weight_max": 35.0733757019043,
      "activations/layer21_attention_weight_min": -26.32384490966797,
      "activations/layer22_attention_weight_max": 30.27088165283203,
      "activations/layer22_attention_weight_min": -25.770450592041016,
      "activations/layer23_attention_weight_max": 37.15541458129883,
      "activations/layer23_attention_weight_min": -28.104604721069336,
      "activations/layer2_attention_weight_max": 29.859249114990234,
      "activations/layer2_attention_weight_min": -28.325515747070312,
      "activations/layer3_attention_weight_max": 84.27085876464844,
      "activations/layer3_attention_weight_min": -86.05889129638672,
      "activations/layer4_attention_weight_max": 85.65742492675781,
      "activations/layer4_attention_weight_min": -85.9454574584961,
      "activations/layer5_attention_weight_max": 61.74520492553711,
      "activations/layer5_attention_weight_min": -71.00086212158203,
      "activations/layer6_attention_weight_max": 47.74534225463867,
      "activations/layer6_attention_weight_min": -48.69778823852539,
      "activations/layer7_attention_weight_max": 59.98538589477539,
      "activations/layer7_attention_weight_min": -59.76126480102539,
      "activations/layer8_attention_weight_max": 41.93021774291992,
      "activations/layer8_attention_weight_min": -47.63606643676758,
      "activations/layer9_attention_weight_max": 40.87996292114258,
      "activations/layer9_attention_weight_min": -44.512550354003906,
      "epoch": 16.53,
      "learning_rate": 4.387575757575757e-05,
      "loss": 2.7496,
      "step": 284400
    },
    {
      "activations/layer0_attention_weight_max": 15.70700454711914,
      "activations/layer0_attention_weight_min": -13.981120109558105,
      "activations/layer10_attention_weight_max": 34.40699768066406,
      "activations/layer10_attention_weight_min": -34.024513244628906,
      "activations/layer11_attention_weight_max": 31.046072006225586,
      "activations/layer11_attention_weight_min": -32.69001770019531,
      "activations/layer12_attention_weight_max": 19.104867935180664,
      "activations/layer12_attention_weight_min": -24.986661911010742,
      "activations/layer13_attention_weight_max": 39.65406036376953,
      "activations/layer13_attention_weight_min": -27.429636001586914,
      "activations/layer14_attention_weight_max": 39.699378967285156,
      "activations/layer14_attention_weight_min": -31.664691925048828,
      "activations/layer15_attention_weight_max": 38.59634780883789,
      "activations/layer15_attention_weight_min": -31.857568740844727,
      "activations/layer16_attention_weight_max": 37.469791412353516,
      "activations/layer16_attention_weight_min": -33.54801940917969,
      "activations/layer17_attention_weight_max": 58.41715621948242,
      "activations/layer17_attention_weight_min": -46.547607421875,
      "activations/layer18_attention_weight_max": 51.487369537353516,
      "activations/layer18_attention_weight_min": -40.94694137573242,
      "activations/layer19_attention_weight_max": 26.557086944580078,
      "activations/layer19_attention_weight_min": -24.305662155151367,
      "activations/layer1_attention_weight_max": 16.486114501953125,
      "activations/layer1_attention_weight_min": -14.748985290527344,
      "activations/layer20_attention_weight_max": 24.26127815246582,
      "activations/layer20_attention_weight_min": -23.51775550842285,
      "activations/layer21_attention_weight_max": 44.73039245605469,
      "activations/layer21_attention_weight_min": -23.069908142089844,
      "activations/layer22_attention_weight_max": 29.92793846130371,
      "activations/layer22_attention_weight_min": -26.40864372253418,
      "activations/layer23_attention_weight_max": 38.05950927734375,
      "activations/layer23_attention_weight_min": -27.83009147644043,
      "activations/layer2_attention_weight_max": 30.14303970336914,
      "activations/layer2_attention_weight_min": -29.9188289642334,
      "activations/layer3_attention_weight_max": 81.6794662475586,
      "activations/layer3_attention_weight_min": -86.87456512451172,
      "activations/layer4_attention_weight_max": 82.25386047363281,
      "activations/layer4_attention_weight_min": -83.99327087402344,
      "activations/layer5_attention_weight_max": 61.5982666015625,
      "activations/layer5_attention_weight_min": -70.23466491699219,
      "activations/layer6_attention_weight_max": 48.8673210144043,
      "activations/layer6_attention_weight_min": -48.539546966552734,
      "activations/layer7_attention_weight_max": 63.731773376464844,
      "activations/layer7_attention_weight_min": -61.477237701416016,
      "activations/layer8_attention_weight_max": 41.65758514404297,
      "activations/layer8_attention_weight_min": -46.8794059753418,
      "activations/layer9_attention_weight_max": 42.84648132324219,
      "activations/layer9_attention_weight_min": -44.44337463378906,
      "epoch": 16.53,
      "learning_rate": 4.385681818181818e-05,
      "loss": 2.7491,
      "step": 284450
    },
    {
      "activations/layer0_attention_weight_max": 15.270171165466309,
      "activations/layer0_attention_weight_min": -13.826592445373535,
      "activations/layer10_attention_weight_max": 36.31538391113281,
      "activations/layer10_attention_weight_min": -35.632164001464844,
      "activations/layer11_attention_weight_max": 33.113731384277344,
      "activations/layer11_attention_weight_min": -35.7313346862793,
      "activations/layer12_attention_weight_max": 18.97405433654785,
      "activations/layer12_attention_weight_min": -23.435949325561523,
      "activations/layer13_attention_weight_max": 38.672821044921875,
      "activations/layer13_attention_weight_min": -29.943496704101562,
      "activations/layer14_attention_weight_max": 40.00825500488281,
      "activations/layer14_attention_weight_min": -33.51416015625,
      "activations/layer15_attention_weight_max": 38.216556549072266,
      "activations/layer15_attention_weight_min": -31.754396438598633,
      "activations/layer16_attention_weight_max": 34.02376937866211,
      "activations/layer16_attention_weight_min": -32.41744613647461,
      "activations/layer17_attention_weight_max": 52.45886993408203,
      "activations/layer17_attention_weight_min": -48.56989288330078,
      "activations/layer18_attention_weight_max": 49.95389175415039,
      "activations/layer18_attention_weight_min": -43.98744201660156,
      "activations/layer19_attention_weight_max": 24.269779205322266,
      "activations/layer19_attention_weight_min": -25.179115295410156,
      "activations/layer1_attention_weight_max": 15.366600036621094,
      "activations/layer1_attention_weight_min": -14.372511863708496,
      "activations/layer20_attention_weight_max": 28.983470916748047,
      "activations/layer20_attention_weight_min": -24.603464126586914,
      "activations/layer21_attention_weight_max": 41.41885757446289,
      "activations/layer21_attention_weight_min": -23.34294891357422,
      "activations/layer22_attention_weight_max": 34.17747116088867,
      "activations/layer22_attention_weight_min": -26.51695442199707,
      "activations/layer23_attention_weight_max": 41.764957427978516,
      "activations/layer23_attention_weight_min": -25.463428497314453,
      "activations/layer2_attention_weight_max": 31.47026824951172,
      "activations/layer2_attention_weight_min": -28.265281677246094,
      "activations/layer3_attention_weight_max": 82.65604400634766,
      "activations/layer3_attention_weight_min": -86.32697296142578,
      "activations/layer4_attention_weight_max": 88.32866668701172,
      "activations/layer4_attention_weight_min": -85.0659408569336,
      "activations/layer5_attention_weight_max": 64.72490692138672,
      "activations/layer5_attention_weight_min": -72.11015319824219,
      "activations/layer6_attention_weight_max": 49.55548858642578,
      "activations/layer6_attention_weight_min": -48.28401184082031,
      "activations/layer7_attention_weight_max": 63.396358489990234,
      "activations/layer7_attention_weight_min": -63.23065948486328,
      "activations/layer8_attention_weight_max": 47.57057189941406,
      "activations/layer8_attention_weight_min": -50.09495162963867,
      "activations/layer9_attention_weight_max": 49.13471984863281,
      "activations/layer9_attention_weight_min": -45.54777145385742,
      "epoch": 16.53,
      "learning_rate": 4.3837878787878784e-05,
      "loss": 2.7478,
      "step": 284500
    },
    {
      "activations/layer0_attention_weight_max": 15.727137565612793,
      "activations/layer0_attention_weight_min": -13.771183967590332,
      "activations/layer10_attention_weight_max": 32.88051223754883,
      "activations/layer10_attention_weight_min": -34.63114929199219,
      "activations/layer11_attention_weight_max": 30.12770652770996,
      "activations/layer11_attention_weight_min": -32.79020309448242,
      "activations/layer12_attention_weight_max": 18.245471954345703,
      "activations/layer12_attention_weight_min": -25.534826278686523,
      "activations/layer13_attention_weight_max": 31.357593536376953,
      "activations/layer13_attention_weight_min": -26.19356346130371,
      "activations/layer14_attention_weight_max": 34.145050048828125,
      "activations/layer14_attention_weight_min": -29.138896942138672,
      "activations/layer15_attention_weight_max": 31.709985733032227,
      "activations/layer15_attention_weight_min": -28.560306549072266,
      "activations/layer16_attention_weight_max": 33.170875549316406,
      "activations/layer16_attention_weight_min": -30.54091453552246,
      "activations/layer17_attention_weight_max": 49.999183654785156,
      "activations/layer17_attention_weight_min": -46.17397689819336,
      "activations/layer18_attention_weight_max": 45.39244842529297,
      "activations/layer18_attention_weight_min": -38.152931213378906,
      "activations/layer19_attention_weight_max": 23.028745651245117,
      "activations/layer19_attention_weight_min": -23.185087203979492,
      "activations/layer1_attention_weight_max": 17.047094345092773,
      "activations/layer1_attention_weight_min": -13.138336181640625,
      "activations/layer20_attention_weight_max": 22.60436248779297,
      "activations/layer20_attention_weight_min": -22.21491050720215,
      "activations/layer21_attention_weight_max": 32.77506637573242,
      "activations/layer21_attention_weight_min": -20.903892517089844,
      "activations/layer22_attention_weight_max": 28.26144790649414,
      "activations/layer22_attention_weight_min": -27.12380027770996,
      "activations/layer23_attention_weight_max": 35.336666107177734,
      "activations/layer23_attention_weight_min": -23.750471115112305,
      "activations/layer2_attention_weight_max": 29.892410278320312,
      "activations/layer2_attention_weight_min": -28.32489585876465,
      "activations/layer3_attention_weight_max": 82.40764617919922,
      "activations/layer3_attention_weight_min": -85.508544921875,
      "activations/layer4_attention_weight_max": 84.47262573242188,
      "activations/layer4_attention_weight_min": -83.7586898803711,
      "activations/layer5_attention_weight_max": 60.585533142089844,
      "activations/layer5_attention_weight_min": -71.38328552246094,
      "activations/layer6_attention_weight_max": 49.95195388793945,
      "activations/layer6_attention_weight_min": -49.76040267944336,
      "activations/layer7_attention_weight_max": 61.23884582519531,
      "activations/layer7_attention_weight_min": -61.67498779296875,
      "activations/layer8_attention_weight_max": 42.76502990722656,
      "activations/layer8_attention_weight_min": -47.68157958984375,
      "activations/layer9_attention_weight_max": 41.94657897949219,
      "activations/layer9_attention_weight_min": -43.9660530090332,
      "epoch": 16.53,
      "learning_rate": 4.381893939393939e-05,
      "loss": 2.7419,
      "step": 284550
    },
    {
      "activations/layer0_attention_weight_max": 15.642498970031738,
      "activations/layer0_attention_weight_min": -14.067299842834473,
      "activations/layer10_attention_weight_max": 36.028289794921875,
      "activations/layer10_attention_weight_min": -35.26112747192383,
      "activations/layer11_attention_weight_max": 30.788679122924805,
      "activations/layer11_attention_weight_min": -32.960601806640625,
      "activations/layer12_attention_weight_max": 18.429853439331055,
      "activations/layer12_attention_weight_min": -22.25834846496582,
      "activations/layer13_attention_weight_max": 32.731651306152344,
      "activations/layer13_attention_weight_min": -28.171810150146484,
      "activations/layer14_attention_weight_max": 36.07783508300781,
      "activations/layer14_attention_weight_min": -29.710153579711914,
      "activations/layer15_attention_weight_max": 33.47110366821289,
      "activations/layer15_attention_weight_min": -29.37554168701172,
      "activations/layer16_attention_weight_max": 34.0698127746582,
      "activations/layer16_attention_weight_min": -30.95983123779297,
      "activations/layer17_attention_weight_max": 51.76881790161133,
      "activations/layer17_attention_weight_min": -43.677947998046875,
      "activations/layer18_attention_weight_max": 49.993507385253906,
      "activations/layer18_attention_weight_min": -37.59451675415039,
      "activations/layer19_attention_weight_max": 22.753740310668945,
      "activations/layer19_attention_weight_min": -24.012102127075195,
      "activations/layer1_attention_weight_max": 16.143814086914062,
      "activations/layer1_attention_weight_min": -12.673696517944336,
      "activations/layer20_attention_weight_max": 23.114534378051758,
      "activations/layer20_attention_weight_min": -22.538341522216797,
      "activations/layer21_attention_weight_max": 35.51039505004883,
      "activations/layer21_attention_weight_min": -23.098196029663086,
      "activations/layer22_attention_weight_max": 31.431095123291016,
      "activations/layer22_attention_weight_min": -25.551538467407227,
      "activations/layer23_attention_weight_max": 37.61023712158203,
      "activations/layer23_attention_weight_min": -26.792789459228516,
      "activations/layer2_attention_weight_max": 29.62778091430664,
      "activations/layer2_attention_weight_min": -30.25001335144043,
      "activations/layer3_attention_weight_max": 82.3205795288086,
      "activations/layer3_attention_weight_min": -88.29898071289062,
      "activations/layer4_attention_weight_max": 85.02469635009766,
      "activations/layer4_attention_weight_min": -85.10660552978516,
      "activations/layer5_attention_weight_max": 62.4157829284668,
      "activations/layer5_attention_weight_min": -73.49195098876953,
      "activations/layer6_attention_weight_max": 50.44916534423828,
      "activations/layer6_attention_weight_min": -51.148338317871094,
      "activations/layer7_attention_weight_max": 63.58280563354492,
      "activations/layer7_attention_weight_min": -66.44817352294922,
      "activations/layer8_attention_weight_max": 42.66057205200195,
      "activations/layer8_attention_weight_min": -46.332252502441406,
      "activations/layer9_attention_weight_max": 43.55451202392578,
      "activations/layer9_attention_weight_min": -47.54998016357422,
      "epoch": 16.54,
      "learning_rate": 4.3799999999999994e-05,
      "loss": 2.751,
      "step": 284600
    },
    {
      "activations/layer0_attention_weight_max": 16.341575622558594,
      "activations/layer0_attention_weight_min": -13.720961570739746,
      "activations/layer10_attention_weight_max": 33.75548553466797,
      "activations/layer10_attention_weight_min": -34.711341857910156,
      "activations/layer11_attention_weight_max": 33.079742431640625,
      "activations/layer11_attention_weight_min": -33.92652893066406,
      "activations/layer12_attention_weight_max": 17.833845138549805,
      "activations/layer12_attention_weight_min": -25.391271591186523,
      "activations/layer13_attention_weight_max": 33.923248291015625,
      "activations/layer13_attention_weight_min": -26.859704971313477,
      "activations/layer14_attention_weight_max": 35.20711135864258,
      "activations/layer14_attention_weight_min": -30.887617111206055,
      "activations/layer15_attention_weight_max": 33.6260986328125,
      "activations/layer15_attention_weight_min": -30.65557289123535,
      "activations/layer16_attention_weight_max": 35.81650161743164,
      "activations/layer16_attention_weight_min": -33.582366943359375,
      "activations/layer17_attention_weight_max": 54.46160888671875,
      "activations/layer17_attention_weight_min": -50.83261489868164,
      "activations/layer18_attention_weight_max": 52.97651290893555,
      "activations/layer18_attention_weight_min": -43.23320770263672,
      "activations/layer19_attention_weight_max": 24.263765335083008,
      "activations/layer19_attention_weight_min": -22.641355514526367,
      "activations/layer1_attention_weight_max": 15.472354888916016,
      "activations/layer1_attention_weight_min": -14.311755180358887,
      "activations/layer20_attention_weight_max": 23.731184005737305,
      "activations/layer20_attention_weight_min": -21.76342010498047,
      "activations/layer21_attention_weight_max": 38.19390869140625,
      "activations/layer21_attention_weight_min": -23.175262451171875,
      "activations/layer22_attention_weight_max": 31.473726272583008,
      "activations/layer22_attention_weight_min": -25.29744529724121,
      "activations/layer23_attention_weight_max": 40.405303955078125,
      "activations/layer23_attention_weight_min": -25.063352584838867,
      "activations/layer2_attention_weight_max": 29.660152435302734,
      "activations/layer2_attention_weight_min": -28.462833404541016,
      "activations/layer3_attention_weight_max": 85.50416564941406,
      "activations/layer3_attention_weight_min": -84.94879913330078,
      "activations/layer4_attention_weight_max": 87.19613647460938,
      "activations/layer4_attention_weight_min": -85.7798843383789,
      "activations/layer5_attention_weight_max": 64.47805786132812,
      "activations/layer5_attention_weight_min": -79.04906463623047,
      "activations/layer6_attention_weight_max": 48.22940444946289,
      "activations/layer6_attention_weight_min": -49.86347961425781,
      "activations/layer7_attention_weight_max": 66.34978485107422,
      "activations/layer7_attention_weight_min": -60.48345184326172,
      "activations/layer8_attention_weight_max": 44.20717239379883,
      "activations/layer8_attention_weight_min": -51.04813003540039,
      "activations/layer9_attention_weight_max": 43.349308013916016,
      "activations/layer9_attention_weight_min": -45.063106536865234,
      "epoch": 16.54,
      "learning_rate": 4.37810606060606e-05,
      "loss": 2.7446,
      "step": 284650
    },
    {
      "activations/layer0_attention_weight_max": 15.666402816772461,
      "activations/layer0_attention_weight_min": -13.664804458618164,
      "activations/layer10_attention_weight_max": 35.71133804321289,
      "activations/layer10_attention_weight_min": -35.90514373779297,
      "activations/layer11_attention_weight_max": 33.346492767333984,
      "activations/layer11_attention_weight_min": -36.871299743652344,
      "activations/layer12_attention_weight_max": 19.045562744140625,
      "activations/layer12_attention_weight_min": -23.59368324279785,
      "activations/layer13_attention_weight_max": 39.97385787963867,
      "activations/layer13_attention_weight_min": -30.70164680480957,
      "activations/layer14_attention_weight_max": 35.93592834472656,
      "activations/layer14_attention_weight_min": -29.093164443969727,
      "activations/layer15_attention_weight_max": 33.147705078125,
      "activations/layer15_attention_weight_min": -29.644582748413086,
      "activations/layer16_attention_weight_max": 31.050979614257812,
      "activations/layer16_attention_weight_min": -28.97953224182129,
      "activations/layer17_attention_weight_max": 48.17040252685547,
      "activations/layer17_attention_weight_min": -40.5460090637207,
      "activations/layer18_attention_weight_max": 42.86066818237305,
      "activations/layer18_attention_weight_min": -35.94399642944336,
      "activations/layer19_attention_weight_max": 23.371076583862305,
      "activations/layer19_attention_weight_min": -21.871416091918945,
      "activations/layer1_attention_weight_max": 15.586272239685059,
      "activations/layer1_attention_weight_min": -14.01382064819336,
      "activations/layer20_attention_weight_max": 22.40559959411621,
      "activations/layer20_attention_weight_min": -22.932758331298828,
      "activations/layer21_attention_weight_max": 35.2952766418457,
      "activations/layer21_attention_weight_min": -23.694055557250977,
      "activations/layer22_attention_weight_max": 25.341434478759766,
      "activations/layer22_attention_weight_min": -24.03457260131836,
      "activations/layer23_attention_weight_max": 33.77301025390625,
      "activations/layer23_attention_weight_min": -26.27227783203125,
      "activations/layer2_attention_weight_max": 30.88027000427246,
      "activations/layer2_attention_weight_min": -29.325428009033203,
      "activations/layer3_attention_weight_max": 86.76447296142578,
      "activations/layer3_attention_weight_min": -88.1989974975586,
      "activations/layer4_attention_weight_max": 88.07414245605469,
      "activations/layer4_attention_weight_min": -86.7904281616211,
      "activations/layer5_attention_weight_max": 63.90163803100586,
      "activations/layer5_attention_weight_min": -77.37287902832031,
      "activations/layer6_attention_weight_max": 50.73963928222656,
      "activations/layer6_attention_weight_min": -53.13488006591797,
      "activations/layer7_attention_weight_max": 65.37263488769531,
      "activations/layer7_attention_weight_min": -66.80625915527344,
      "activations/layer8_attention_weight_max": 46.39741516113281,
      "activations/layer8_attention_weight_min": -50.473270416259766,
      "activations/layer9_attention_weight_max": 45.047149658203125,
      "activations/layer9_attention_weight_min": -47.15125274658203,
      "epoch": 16.54,
      "learning_rate": 4.376212121212121e-05,
      "loss": 2.744,
      "step": 284700
    },
    {
      "activations/layer0_attention_weight_max": 16.434959411621094,
      "activations/layer0_attention_weight_min": -13.639779090881348,
      "activations/layer10_attention_weight_max": 36.37355422973633,
      "activations/layer10_attention_weight_min": -36.2401123046875,
      "activations/layer11_attention_weight_max": 31.038448333740234,
      "activations/layer11_attention_weight_min": -34.49427795410156,
      "activations/layer12_attention_weight_max": 19.134252548217773,
      "activations/layer12_attention_weight_min": -28.884193420410156,
      "activations/layer13_attention_weight_max": 36.99664306640625,
      "activations/layer13_attention_weight_min": -28.437042236328125,
      "activations/layer14_attention_weight_max": 39.574432373046875,
      "activations/layer14_attention_weight_min": -31.86977767944336,
      "activations/layer15_attention_weight_max": 32.87028121948242,
      "activations/layer15_attention_weight_min": -32.203086853027344,
      "activations/layer16_attention_weight_max": 34.682044982910156,
      "activations/layer16_attention_weight_min": -34.66032028198242,
      "activations/layer17_attention_weight_max": 53.06508255004883,
      "activations/layer17_attention_weight_min": -47.37424087524414,
      "activations/layer18_attention_weight_max": 49.73820495605469,
      "activations/layer18_attention_weight_min": -43.34561538696289,
      "activations/layer19_attention_weight_max": 23.30008316040039,
      "activations/layer19_attention_weight_min": -22.13934326171875,
      "activations/layer1_attention_weight_max": 15.854972839355469,
      "activations/layer1_attention_weight_min": -16.301164627075195,
      "activations/layer20_attention_weight_max": 24.40643310546875,
      "activations/layer20_attention_weight_min": -22.787195205688477,
      "activations/layer21_attention_weight_max": 37.60675048828125,
      "activations/layer21_attention_weight_min": -22.667003631591797,
      "activations/layer22_attention_weight_max": 27.68498992919922,
      "activations/layer22_attention_weight_min": -25.24789810180664,
      "activations/layer23_attention_weight_max": 35.34336853027344,
      "activations/layer23_attention_weight_min": -22.60179901123047,
      "activations/layer2_attention_weight_max": 31.571081161499023,
      "activations/layer2_attention_weight_min": -29.31757164001465,
      "activations/layer3_attention_weight_max": 86.5553207397461,
      "activations/layer3_attention_weight_min": -86.48483276367188,
      "activations/layer4_attention_weight_max": 87.28877258300781,
      "activations/layer4_attention_weight_min": -86.84221649169922,
      "activations/layer5_attention_weight_max": 63.890419006347656,
      "activations/layer5_attention_weight_min": -70.80397033691406,
      "activations/layer6_attention_weight_max": 47.02179718017578,
      "activations/layer6_attention_weight_min": -48.464622497558594,
      "activations/layer7_attention_weight_max": 60.85588455200195,
      "activations/layer7_attention_weight_min": -65.44588470458984,
      "activations/layer8_attention_weight_max": 45.417877197265625,
      "activations/layer8_attention_weight_min": -50.51280212402344,
      "activations/layer9_attention_weight_max": 42.79669189453125,
      "activations/layer9_attention_weight_min": -46.06965255737305,
      "epoch": 16.55,
      "learning_rate": 4.374318181818181e-05,
      "loss": 2.7648,
      "step": 284750
    },
    {
      "activations/layer0_attention_weight_max": 15.908586502075195,
      "activations/layer0_attention_weight_min": -13.828213691711426,
      "activations/layer10_attention_weight_max": 37.73892593383789,
      "activations/layer10_attention_weight_min": -35.912288665771484,
      "activations/layer11_attention_weight_max": 34.488121032714844,
      "activations/layer11_attention_weight_min": -34.95387268066406,
      "activations/layer12_attention_weight_max": 20.91935157775879,
      "activations/layer12_attention_weight_min": -23.73370933532715,
      "activations/layer13_attention_weight_max": 34.485008239746094,
      "activations/layer13_attention_weight_min": -28.753835678100586,
      "activations/layer14_attention_weight_max": 38.70453643798828,
      "activations/layer14_attention_weight_min": -31.198732376098633,
      "activations/layer15_attention_weight_max": 37.238399505615234,
      "activations/layer15_attention_weight_min": -31.004352569580078,
      "activations/layer16_attention_weight_max": 34.97482681274414,
      "activations/layer16_attention_weight_min": -31.883785247802734,
      "activations/layer17_attention_weight_max": 51.02181625366211,
      "activations/layer17_attention_weight_min": -44.42441940307617,
      "activations/layer18_attention_weight_max": 50.10029602050781,
      "activations/layer18_attention_weight_min": -41.82990646362305,
      "activations/layer19_attention_weight_max": 25.738330841064453,
      "activations/layer19_attention_weight_min": -25.131174087524414,
      "activations/layer1_attention_weight_max": 16.838674545288086,
      "activations/layer1_attention_weight_min": -14.991097450256348,
      "activations/layer20_attention_weight_max": 23.329692840576172,
      "activations/layer20_attention_weight_min": -23.48192596435547,
      "activations/layer21_attention_weight_max": 39.8330192565918,
      "activations/layer21_attention_weight_min": -24.617895126342773,
      "activations/layer22_attention_weight_max": 30.16667366027832,
      "activations/layer22_attention_weight_min": -25.90875244140625,
      "activations/layer23_attention_weight_max": 36.46815490722656,
      "activations/layer23_attention_weight_min": -24.054777145385742,
      "activations/layer2_attention_weight_max": 31.70996856689453,
      "activations/layer2_attention_weight_min": -30.312946319580078,
      "activations/layer3_attention_weight_max": 87.3394546508789,
      "activations/layer3_attention_weight_min": -85.61908721923828,
      "activations/layer4_attention_weight_max": 90.82621002197266,
      "activations/layer4_attention_weight_min": -84.8180923461914,
      "activations/layer5_attention_weight_max": 62.168968200683594,
      "activations/layer5_attention_weight_min": -73.1903076171875,
      "activations/layer6_attention_weight_max": 50.011253356933594,
      "activations/layer6_attention_weight_min": -52.86445236206055,
      "activations/layer7_attention_weight_max": 63.020713806152344,
      "activations/layer7_attention_weight_min": -60.93968200683594,
      "activations/layer8_attention_weight_max": 45.67439651489258,
      "activations/layer8_attention_weight_min": -49.71333312988281,
      "activations/layer9_attention_weight_max": 42.14271545410156,
      "activations/layer9_attention_weight_min": -45.11591720581055,
      "epoch": 16.55,
      "learning_rate": 4.372424242424242e-05,
      "loss": 2.75,
      "step": 284800
    },
    {
      "activations/layer0_attention_weight_max": 15.750344276428223,
      "activations/layer0_attention_weight_min": -13.957262992858887,
      "activations/layer10_attention_weight_max": 37.5626220703125,
      "activations/layer10_attention_weight_min": -39.09464645385742,
      "activations/layer11_attention_weight_max": 33.508636474609375,
      "activations/layer11_attention_weight_min": -35.84062957763672,
      "activations/layer12_attention_weight_max": 18.120071411132812,
      "activations/layer12_attention_weight_min": -23.472307205200195,
      "activations/layer13_attention_weight_max": 34.3304443359375,
      "activations/layer13_attention_weight_min": -29.16288948059082,
      "activations/layer14_attention_weight_max": 33.463287353515625,
      "activations/layer14_attention_weight_min": -32.210289001464844,
      "activations/layer15_attention_weight_max": 31.695255279541016,
      "activations/layer15_attention_weight_min": -31.456527709960938,
      "activations/layer16_attention_weight_max": 37.07239532470703,
      "activations/layer16_attention_weight_min": -34.082244873046875,
      "activations/layer17_attention_weight_max": 56.45796203613281,
      "activations/layer17_attention_weight_min": -48.63714599609375,
      "activations/layer18_attention_weight_max": 49.558372497558594,
      "activations/layer18_attention_weight_min": -42.63636779785156,
      "activations/layer19_attention_weight_max": 23.716432571411133,
      "activations/layer19_attention_weight_min": -24.48946189880371,
      "activations/layer1_attention_weight_max": 16.48198127746582,
      "activations/layer1_attention_weight_min": -14.627396583557129,
      "activations/layer20_attention_weight_max": 24.74959945678711,
      "activations/layer20_attention_weight_min": -22.402587890625,
      "activations/layer21_attention_weight_max": 34.908416748046875,
      "activations/layer21_attention_weight_min": -21.664398193359375,
      "activations/layer22_attention_weight_max": 29.306367874145508,
      "activations/layer22_attention_weight_min": -26.60372543334961,
      "activations/layer23_attention_weight_max": 35.82862091064453,
      "activations/layer23_attention_weight_min": -24.444114685058594,
      "activations/layer2_attention_weight_max": 31.355091094970703,
      "activations/layer2_attention_weight_min": -29.708906173706055,
      "activations/layer3_attention_weight_max": 87.65106201171875,
      "activations/layer3_attention_weight_min": -93.26618957519531,
      "activations/layer4_attention_weight_max": 90.03929901123047,
      "activations/layer4_attention_weight_min": -93.56362915039062,
      "activations/layer5_attention_weight_max": 66.42284393310547,
      "activations/layer5_attention_weight_min": -81.0096664428711,
      "activations/layer6_attention_weight_max": 49.24345397949219,
      "activations/layer6_attention_weight_min": -52.57215881347656,
      "activations/layer7_attention_weight_max": 67.63748168945312,
      "activations/layer7_attention_weight_min": -67.01976776123047,
      "activations/layer8_attention_weight_max": 47.78596115112305,
      "activations/layer8_attention_weight_min": -51.27924346923828,
      "activations/layer9_attention_weight_max": 47.412349700927734,
      "activations/layer9_attention_weight_min": -48.3142204284668,
      "epoch": 16.55,
      "learning_rate": 4.370530303030303e-05,
      "loss": 2.7288,
      "step": 284850
    },
    {
      "activations/layer0_attention_weight_max": 15.127955436706543,
      "activations/layer0_attention_weight_min": -13.83917236328125,
      "activations/layer10_attention_weight_max": 33.92857360839844,
      "activations/layer10_attention_weight_min": -36.02494812011719,
      "activations/layer11_attention_weight_max": 32.49766540527344,
      "activations/layer11_attention_weight_min": -33.25205993652344,
      "activations/layer12_attention_weight_max": 20.343069076538086,
      "activations/layer12_attention_weight_min": -24.34641456604004,
      "activations/layer13_attention_weight_max": 32.85633850097656,
      "activations/layer13_attention_weight_min": -29.245325088500977,
      "activations/layer14_attention_weight_max": 35.01543426513672,
      "activations/layer14_attention_weight_min": -31.974334716796875,
      "activations/layer15_attention_weight_max": 33.69915771484375,
      "activations/layer15_attention_weight_min": -29.816097259521484,
      "activations/layer16_attention_weight_max": 33.10002136230469,
      "activations/layer16_attention_weight_min": -32.424312591552734,
      "activations/layer17_attention_weight_max": 50.55910873413086,
      "activations/layer17_attention_weight_min": -44.694488525390625,
      "activations/layer18_attention_weight_max": 46.5039176940918,
      "activations/layer18_attention_weight_min": -38.220645904541016,
      "activations/layer19_attention_weight_max": 23.212478637695312,
      "activations/layer19_attention_weight_min": -23.96009063720703,
      "activations/layer1_attention_weight_max": 16.691295623779297,
      "activations/layer1_attention_weight_min": -15.335015296936035,
      "activations/layer20_attention_weight_max": 22.421510696411133,
      "activations/layer20_attention_weight_min": -21.71282386779785,
      "activations/layer21_attention_weight_max": 35.0225715637207,
      "activations/layer21_attention_weight_min": -23.91259765625,
      "activations/layer22_attention_weight_max": 29.159862518310547,
      "activations/layer22_attention_weight_min": -26.47393226623535,
      "activations/layer23_attention_weight_max": 38.35987091064453,
      "activations/layer23_attention_weight_min": -26.31149673461914,
      "activations/layer2_attention_weight_max": 30.169326782226562,
      "activations/layer2_attention_weight_min": -29.685028076171875,
      "activations/layer3_attention_weight_max": 89.7833480834961,
      "activations/layer3_attention_weight_min": -92.06375122070312,
      "activations/layer4_attention_weight_max": 88.62886810302734,
      "activations/layer4_attention_weight_min": -91.33541870117188,
      "activations/layer5_attention_weight_max": 63.044090270996094,
      "activations/layer5_attention_weight_min": -75.1212158203125,
      "activations/layer6_attention_weight_max": 50.05646514892578,
      "activations/layer6_attention_weight_min": -51.338829040527344,
      "activations/layer7_attention_weight_max": 63.69771957397461,
      "activations/layer7_attention_weight_min": -65.50949096679688,
      "activations/layer8_attention_weight_max": 43.85708999633789,
      "activations/layer8_attention_weight_min": -49.980045318603516,
      "activations/layer9_attention_weight_max": 44.051673889160156,
      "activations/layer9_attention_weight_min": -46.401119232177734,
      "epoch": 16.55,
      "learning_rate": 4.368636363636363e-05,
      "loss": 2.7437,
      "step": 284900
    },
    {
      "activations/layer0_attention_weight_max": 16.10029411315918,
      "activations/layer0_attention_weight_min": -13.771614074707031,
      "activations/layer10_attention_weight_max": 31.4345703125,
      "activations/layer10_attention_weight_min": -33.95100784301758,
      "activations/layer11_attention_weight_max": 29.568716049194336,
      "activations/layer11_attention_weight_min": -33.05188751220703,
      "activations/layer12_attention_weight_max": 18.819307327270508,
      "activations/layer12_attention_weight_min": -27.017108917236328,
      "activations/layer13_attention_weight_max": 29.263484954833984,
      "activations/layer13_attention_weight_min": -26.099885940551758,
      "activations/layer14_attention_weight_max": 33.571754455566406,
      "activations/layer14_attention_weight_min": -32.017547607421875,
      "activations/layer15_attention_weight_max": 29.185556411743164,
      "activations/layer15_attention_weight_min": -32.43795394897461,
      "activations/layer16_attention_weight_max": 31.109390258789062,
      "activations/layer16_attention_weight_min": -30.22886848449707,
      "activations/layer17_attention_weight_max": 47.97038269042969,
      "activations/layer17_attention_weight_min": -42.65767288208008,
      "activations/layer18_attention_weight_max": 43.48262023925781,
      "activations/layer18_attention_weight_min": -37.682857513427734,
      "activations/layer19_attention_weight_max": 21.644954681396484,
      "activations/layer19_attention_weight_min": -21.245014190673828,
      "activations/layer1_attention_weight_max": 15.951834678649902,
      "activations/layer1_attention_weight_min": -13.817033767700195,
      "activations/layer20_attention_weight_max": 20.918962478637695,
      "activations/layer20_attention_weight_min": -23.15632438659668,
      "activations/layer21_attention_weight_max": 37.95724868774414,
      "activations/layer21_attention_weight_min": -21.753887176513672,
      "activations/layer22_attention_weight_max": 28.60999298095703,
      "activations/layer22_attention_weight_min": -24.29524040222168,
      "activations/layer23_attention_weight_max": 33.496185302734375,
      "activations/layer23_attention_weight_min": -25.404096603393555,
      "activations/layer2_attention_weight_max": 34.8839111328125,
      "activations/layer2_attention_weight_min": -31.10696029663086,
      "activations/layer3_attention_weight_max": 86.28032684326172,
      "activations/layer3_attention_weight_min": -84.27777099609375,
      "activations/layer4_attention_weight_max": 86.51219940185547,
      "activations/layer4_attention_weight_min": -83.65948486328125,
      "activations/layer5_attention_weight_max": 62.817840576171875,
      "activations/layer5_attention_weight_min": -71.85226440429688,
      "activations/layer6_attention_weight_max": 47.44768142700195,
      "activations/layer6_attention_weight_min": -49.130985260009766,
      "activations/layer7_attention_weight_max": 65.48153686523438,
      "activations/layer7_attention_weight_min": -62.07063293457031,
      "activations/layer8_attention_weight_max": 40.57373809814453,
      "activations/layer8_attention_weight_min": -44.8405647277832,
      "activations/layer9_attention_weight_max": 46.27790832519531,
      "activations/layer9_attention_weight_min": -42.577754974365234,
      "epoch": 16.56,
      "learning_rate": 4.366742424242424e-05,
      "loss": 2.7549,
      "step": 284950
    },
    {
      "activations/layer0_attention_weight_max": 16.24596405029297,
      "activations/layer0_attention_weight_min": -13.759053230285645,
      "activations/layer10_attention_weight_max": 38.38391876220703,
      "activations/layer10_attention_weight_min": -36.09651565551758,
      "activations/layer11_attention_weight_max": 34.31786346435547,
      "activations/layer11_attention_weight_min": -34.349483489990234,
      "activations/layer12_attention_weight_max": 20.676931381225586,
      "activations/layer12_attention_weight_min": -22.834959030151367,
      "activations/layer13_attention_weight_max": 44.43095397949219,
      "activations/layer13_attention_weight_min": -32.45209503173828,
      "activations/layer14_attention_weight_max": 35.531410217285156,
      "activations/layer14_attention_weight_min": -30.480146408081055,
      "activations/layer15_attention_weight_max": 33.433101654052734,
      "activations/layer15_attention_weight_min": -30.183008193969727,
      "activations/layer16_attention_weight_max": 34.82686996459961,
      "activations/layer16_attention_weight_min": -31.613052368164062,
      "activations/layer17_attention_weight_max": 53.47166061401367,
      "activations/layer17_attention_weight_min": -44.967445373535156,
      "activations/layer18_attention_weight_max": 48.89006423950195,
      "activations/layer18_attention_weight_min": -39.96100997924805,
      "activations/layer19_attention_weight_max": 24.913211822509766,
      "activations/layer19_attention_weight_min": -22.831817626953125,
      "activations/layer1_attention_weight_max": 15.893542289733887,
      "activations/layer1_attention_weight_min": -13.475829124450684,
      "activations/layer20_attention_weight_max": 22.73019790649414,
      "activations/layer20_attention_weight_min": -19.53392219543457,
      "activations/layer21_attention_weight_max": 35.46888732910156,
      "activations/layer21_attention_weight_min": -20.88642120361328,
      "activations/layer22_attention_weight_max": 32.78142547607422,
      "activations/layer22_attention_weight_min": -27.849647521972656,
      "activations/layer23_attention_weight_max": 36.565521240234375,
      "activations/layer23_attention_weight_min": -25.23775863647461,
      "activations/layer2_attention_weight_max": 30.532123565673828,
      "activations/layer2_attention_weight_min": -29.301748275756836,
      "activations/layer3_attention_weight_max": 87.50170135498047,
      "activations/layer3_attention_weight_min": -89.98294830322266,
      "activations/layer4_attention_weight_max": 89.4062728881836,
      "activations/layer4_attention_weight_min": -86.42064666748047,
      "activations/layer5_attention_weight_max": 66.8499526977539,
      "activations/layer5_attention_weight_min": -76.7683334350586,
      "activations/layer6_attention_weight_max": 49.835426330566406,
      "activations/layer6_attention_weight_min": -48.60531997680664,
      "activations/layer7_attention_weight_max": 64.89031219482422,
      "activations/layer7_attention_weight_min": -64.62397003173828,
      "activations/layer8_attention_weight_max": 43.97090530395508,
      "activations/layer8_attention_weight_min": -46.694557189941406,
      "activations/layer9_attention_weight_max": 44.246604919433594,
      "activations/layer9_attention_weight_min": -45.24488067626953,
      "epoch": 16.56,
      "learning_rate": 4.364848484848484e-05,
      "loss": 2.7269,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.3459,
      "eval_samples_per_second": 514.503,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.3459,
      "eval_openwebtext_samples_per_second": 514.503,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9702,
      "eval_wikitext_samples_per_second": 231.446,
      "step": 285000
    },
    {
      "epoch": 16.56,
      "eval_lambada_loss": 2.533203125,
      "eval_lambada_ppl": 12.593781054375055,
      "eval_lambada_runtime": 9.5221,
      "eval_lambada_samples_per_second": 511.338,
      "step": 285000
    },
    {
      "activations/layer0_attention_weight_max": 15.636444091796875,
      "activations/layer0_attention_weight_min": -13.934803009033203,
      "activations/layer10_attention_weight_max": 38.07060623168945,
      "activations/layer10_attention_weight_min": -35.22010040283203,
      "activations/layer11_attention_weight_max": 32.71108627319336,
      "activations/layer11_attention_weight_min": -34.33259201049805,
      "activations/layer12_attention_weight_max": 20.05307388305664,
      "activations/layer12_attention_weight_min": -29.16289520263672,
      "activations/layer13_attention_weight_max": 36.39027404785156,
      "activations/layer13_attention_weight_min": -28.492488861083984,
      "activations/layer14_attention_weight_max": 35.90238571166992,
      "activations/layer14_attention_weight_min": -31.001216888427734,
      "activations/layer15_attention_weight_max": 33.683719635009766,
      "activations/layer15_attention_weight_min": -33.53337478637695,
      "activations/layer16_attention_weight_max": 34.899845123291016,
      "activations/layer16_attention_weight_min": -33.49895095825195,
      "activations/layer17_attention_weight_max": 54.89575958251953,
      "activations/layer17_attention_weight_min": -47.51435089111328,
      "activations/layer18_attention_weight_max": 47.30038070678711,
      "activations/layer18_attention_weight_min": -39.092308044433594,
      "activations/layer19_attention_weight_max": 25.220054626464844,
      "activations/layer19_attention_weight_min": -22.211606979370117,
      "activations/layer1_attention_weight_max": 16.25491714477539,
      "activations/layer1_attention_weight_min": -13.360101699829102,
      "activations/layer20_attention_weight_max": 23.638456344604492,
      "activations/layer20_attention_weight_min": -20.772741317749023,
      "activations/layer21_attention_weight_max": 35.18611145019531,
      "activations/layer21_attention_weight_min": -19.110240936279297,
      "activations/layer22_attention_weight_max": 30.28846549987793,
      "activations/layer22_attention_weight_min": -27.389482498168945,
      "activations/layer23_attention_weight_max": 40.65036392211914,
      "activations/layer23_attention_weight_min": -22.031530380249023,
      "activations/layer2_attention_weight_max": 29.422880172729492,
      "activations/layer2_attention_weight_min": -30.854583740234375,
      "activations/layer3_attention_weight_max": 85.77015686035156,
      "activations/layer3_attention_weight_min": -90.86308288574219,
      "activations/layer4_attention_weight_max": 90.74614715576172,
      "activations/layer4_attention_weight_min": -88.9513168334961,
      "activations/layer5_attention_weight_max": 67.58174133300781,
      "activations/layer5_attention_weight_min": -76.39925384521484,
      "activations/layer6_attention_weight_max": 52.46604537963867,
      "activations/layer6_attention_weight_min": -49.86188888549805,
      "activations/layer7_attention_weight_max": 62.28529739379883,
      "activations/layer7_attention_weight_min": -61.044403076171875,
      "activations/layer8_attention_weight_max": 44.499298095703125,
      "activations/layer8_attention_weight_min": -46.2589111328125,
      "activations/layer9_attention_weight_max": 44.793514251708984,
      "activations/layer9_attention_weight_min": -44.746376037597656,
      "epoch": 16.56,
      "learning_rate": 4.3629545454545455e-05,
      "loss": 2.7492,
      "step": 285050
    },
    {
      "activations/layer0_attention_weight_max": 15.93699836730957,
      "activations/layer0_attention_weight_min": -13.615266799926758,
      "activations/layer10_attention_weight_max": 35.970977783203125,
      "activations/layer10_attention_weight_min": -35.838348388671875,
      "activations/layer11_attention_weight_max": 33.178043365478516,
      "activations/layer11_attention_weight_min": -34.150184631347656,
      "activations/layer12_attention_weight_max": 18.898170471191406,
      "activations/layer12_attention_weight_min": -28.625160217285156,
      "activations/layer13_attention_weight_max": 36.468994140625,
      "activations/layer13_attention_weight_min": -29.418004989624023,
      "activations/layer14_attention_weight_max": 37.9460334777832,
      "activations/layer14_attention_weight_min": -30.010793685913086,
      "activations/layer15_attention_weight_max": 35.98122024536133,
      "activations/layer15_attention_weight_min": -28.480741500854492,
      "activations/layer16_attention_weight_max": 34.66691207885742,
      "activations/layer16_attention_weight_min": -30.7984619140625,
      "activations/layer17_attention_weight_max": 53.27919387817383,
      "activations/layer17_attention_weight_min": -46.96840286254883,
      "activations/layer18_attention_weight_max": 50.7781867980957,
      "activations/layer18_attention_weight_min": -41.730777740478516,
      "activations/layer19_attention_weight_max": 26.228479385375977,
      "activations/layer19_attention_weight_min": -23.779829025268555,
      "activations/layer1_attention_weight_max": 15.019379615783691,
      "activations/layer1_attention_weight_min": -14.757187843322754,
      "activations/layer20_attention_weight_max": 25.437822341918945,
      "activations/layer20_attention_weight_min": -24.238191604614258,
      "activations/layer21_attention_weight_max": 38.05842971801758,
      "activations/layer21_attention_weight_min": -22.230880737304688,
      "activations/layer22_attention_weight_max": 32.518768310546875,
      "activations/layer22_attention_weight_min": -24.888547897338867,
      "activations/layer23_attention_weight_max": 40.36354064941406,
      "activations/layer23_attention_weight_min": -23.46027183532715,
      "activations/layer2_attention_weight_max": 30.86031150817871,
      "activations/layer2_attention_weight_min": -31.584003448486328,
      "activations/layer3_attention_weight_max": 89.0859603881836,
      "activations/layer3_attention_weight_min": -94.01085662841797,
      "activations/layer4_attention_weight_max": 91.92610931396484,
      "activations/layer4_attention_weight_min": -92.80549621582031,
      "activations/layer5_attention_weight_max": 64.62887573242188,
      "activations/layer5_attention_weight_min": -72.36614227294922,
      "activations/layer6_attention_weight_max": 50.963035583496094,
      "activations/layer6_attention_weight_min": -51.50773620605469,
      "activations/layer7_attention_weight_max": 63.466758728027344,
      "activations/layer7_attention_weight_min": -63.07950210571289,
      "activations/layer8_attention_weight_max": 45.389610290527344,
      "activations/layer8_attention_weight_min": -48.00758743286133,
      "activations/layer9_attention_weight_max": 46.685951232910156,
      "activations/layer9_attention_weight_min": -44.91516876220703,
      "epoch": 16.57,
      "learning_rate": 4.3610606060606056e-05,
      "loss": 2.7388,
      "step": 285100
    },
    {
      "activations/layer0_attention_weight_max": 15.547666549682617,
      "activations/layer0_attention_weight_min": -13.768035888671875,
      "activations/layer10_attention_weight_max": 37.75454330444336,
      "activations/layer10_attention_weight_min": -36.436065673828125,
      "activations/layer11_attention_weight_max": 34.161705017089844,
      "activations/layer11_attention_weight_min": -33.71366882324219,
      "activations/layer12_attention_weight_max": 21.71215057373047,
      "activations/layer12_attention_weight_min": -22.446245193481445,
      "activations/layer13_attention_weight_max": 55.937477111816406,
      "activations/layer13_attention_weight_min": -33.17234802246094,
      "activations/layer14_attention_weight_max": 65.2285385131836,
      "activations/layer14_attention_weight_min": -32.99802017211914,
      "activations/layer15_attention_weight_max": 55.691715240478516,
      "activations/layer15_attention_weight_min": -34.0319709777832,
      "activations/layer16_attention_weight_max": 43.41269302368164,
      "activations/layer16_attention_weight_min": -37.18397903442383,
      "activations/layer17_attention_weight_max": 69.5099105834961,
      "activations/layer17_attention_weight_min": -55.76750183105469,
      "activations/layer18_attention_weight_max": 66.52049255371094,
      "activations/layer18_attention_weight_min": -49.84040832519531,
      "activations/layer19_attention_weight_max": 35.841854095458984,
      "activations/layer19_attention_weight_min": -30.509414672851562,
      "activations/layer1_attention_weight_max": 15.994994163513184,
      "activations/layer1_attention_weight_min": -14.906317710876465,
      "activations/layer20_attention_weight_max": 35.821563720703125,
      "activations/layer20_attention_weight_min": -24.11334991455078,
      "activations/layer21_attention_weight_max": 56.71405792236328,
      "activations/layer21_attention_weight_min": -26.669010162353516,
      "activations/layer22_attention_weight_max": 40.28583908081055,
      "activations/layer22_attention_weight_min": -27.4764461517334,
      "activations/layer23_attention_weight_max": 45.39466857910156,
      "activations/layer23_attention_weight_min": -25.473777770996094,
      "activations/layer2_attention_weight_max": 31.7439022064209,
      "activations/layer2_attention_weight_min": -30.964553833007812,
      "activations/layer3_attention_weight_max": 89.90897369384766,
      "activations/layer3_attention_weight_min": -86.93212890625,
      "activations/layer4_attention_weight_max": 89.776123046875,
      "activations/layer4_attention_weight_min": -84.95152282714844,
      "activations/layer5_attention_weight_max": 62.44157028198242,
      "activations/layer5_attention_weight_min": -72.63611602783203,
      "activations/layer6_attention_weight_max": 47.19054412841797,
      "activations/layer6_attention_weight_min": -50.36460494995117,
      "activations/layer7_attention_weight_max": 69.8921890258789,
      "activations/layer7_attention_weight_min": -61.1185417175293,
      "activations/layer8_attention_weight_max": 47.96659469604492,
      "activations/layer8_attention_weight_min": -44.98921585083008,
      "activations/layer9_attention_weight_max": 49.29661178588867,
      "activations/layer9_attention_weight_min": -45.84834671020508,
      "epoch": 16.57,
      "learning_rate": 4.359166666666666e-05,
      "loss": 2.7464,
      "step": 285150
    },
    {
      "activations/layer0_attention_weight_max": 15.787054061889648,
      "activations/layer0_attention_weight_min": -13.945213317871094,
      "activations/layer10_attention_weight_max": 39.8436279296875,
      "activations/layer10_attention_weight_min": -38.678646087646484,
      "activations/layer11_attention_weight_max": 35.59690856933594,
      "activations/layer11_attention_weight_min": -37.924339294433594,
      "activations/layer12_attention_weight_max": 17.16059112548828,
      "activations/layer12_attention_weight_min": -24.73448371887207,
      "activations/layer13_attention_weight_max": 29.021081924438477,
      "activations/layer13_attention_weight_min": -29.019811630249023,
      "activations/layer14_attention_weight_max": 32.73609161376953,
      "activations/layer14_attention_weight_min": -28.508773803710938,
      "activations/layer15_attention_weight_max": 29.335163116455078,
      "activations/layer15_attention_weight_min": -29.60072898864746,
      "activations/layer16_attention_weight_max": 30.505142211914062,
      "activations/layer16_attention_weight_min": -28.168575286865234,
      "activations/layer17_attention_weight_max": 48.20473098754883,
      "activations/layer17_attention_weight_min": -43.22634506225586,
      "activations/layer18_attention_weight_max": 41.82846450805664,
      "activations/layer18_attention_weight_min": -38.395084381103516,
      "activations/layer19_attention_weight_max": 21.35140037536621,
      "activations/layer19_attention_weight_min": -22.332868576049805,
      "activations/layer1_attention_weight_max": 16.347694396972656,
      "activations/layer1_attention_weight_min": -15.671321868896484,
      "activations/layer20_attention_weight_max": 21.348342895507812,
      "activations/layer20_attention_weight_min": -22.59324836730957,
      "activations/layer21_attention_weight_max": 29.001033782958984,
      "activations/layer21_attention_weight_min": -21.169347763061523,
      "activations/layer22_attention_weight_max": 26.63927459716797,
      "activations/layer22_attention_weight_min": -26.166425704956055,
      "activations/layer23_attention_weight_max": 34.315513610839844,
      "activations/layer23_attention_weight_min": -22.862394332885742,
      "activations/layer2_attention_weight_max": 31.338665008544922,
      "activations/layer2_attention_weight_min": -31.070220947265625,
      "activations/layer3_attention_weight_max": 92.33110809326172,
      "activations/layer3_attention_weight_min": -95.03775024414062,
      "activations/layer4_attention_weight_max": 94.42623138427734,
      "activations/layer4_attention_weight_min": -90.21170806884766,
      "activations/layer5_attention_weight_max": 69.97552490234375,
      "activations/layer5_attention_weight_min": -70.74153137207031,
      "activations/layer6_attention_weight_max": 51.73556900024414,
      "activations/layer6_attention_weight_min": -53.03342819213867,
      "activations/layer7_attention_weight_max": 66.643310546875,
      "activations/layer7_attention_weight_min": -65.2212142944336,
      "activations/layer8_attention_weight_max": 49.39008712768555,
      "activations/layer8_attention_weight_min": -50.91244888305664,
      "activations/layer9_attention_weight_max": 47.850032806396484,
      "activations/layer9_attention_weight_min": -48.119102478027344,
      "epoch": 16.57,
      "learning_rate": 4.357272727272727e-05,
      "loss": 2.7391,
      "step": 285200
    },
    {
      "activations/layer0_attention_weight_max": 15.59244155883789,
      "activations/layer0_attention_weight_min": -14.205170631408691,
      "activations/layer10_attention_weight_max": 35.953086853027344,
      "activations/layer10_attention_weight_min": -34.95264434814453,
      "activations/layer11_attention_weight_max": 30.898250579833984,
      "activations/layer11_attention_weight_min": -34.53144836425781,
      "activations/layer12_attention_weight_max": 18.0229434967041,
      "activations/layer12_attention_weight_min": -24.879175186157227,
      "activations/layer13_attention_weight_max": 32.490447998046875,
      "activations/layer13_attention_weight_min": -25.893104553222656,
      "activations/layer14_attention_weight_max": 33.11184310913086,
      "activations/layer14_attention_weight_min": -30.464752197265625,
      "activations/layer15_attention_weight_max": 34.2950325012207,
      "activations/layer15_attention_weight_min": -29.799026489257812,
      "activations/layer16_attention_weight_max": 35.311553955078125,
      "activations/layer16_attention_weight_min": -35.441219329833984,
      "activations/layer17_attention_weight_max": 52.77357864379883,
      "activations/layer17_attention_weight_min": -50.274662017822266,
      "activations/layer18_attention_weight_max": 47.5800895690918,
      "activations/layer18_attention_weight_min": -43.83342742919922,
      "activations/layer19_attention_weight_max": 23.104345321655273,
      "activations/layer19_attention_weight_min": -24.48206329345703,
      "activations/layer1_attention_weight_max": 16.56946563720703,
      "activations/layer1_attention_weight_min": -16.900836944580078,
      "activations/layer20_attention_weight_max": 23.301454544067383,
      "activations/layer20_attention_weight_min": -22.617176055908203,
      "activations/layer21_attention_weight_max": 31.68402099609375,
      "activations/layer21_attention_weight_min": -24.221466064453125,
      "activations/layer22_attention_weight_max": 28.33321762084961,
      "activations/layer22_attention_weight_min": -26.61903190612793,
      "activations/layer23_attention_weight_max": 35.395694732666016,
      "activations/layer23_attention_weight_min": -24.118680953979492,
      "activations/layer2_attention_weight_max": 30.533714294433594,
      "activations/layer2_attention_weight_min": -30.848712921142578,
      "activations/layer3_attention_weight_max": 86.34061431884766,
      "activations/layer3_attention_weight_min": -87.24732208251953,
      "activations/layer4_attention_weight_max": 88.82819366455078,
      "activations/layer4_attention_weight_min": -85.93305969238281,
      "activations/layer5_attention_weight_max": 65.68794250488281,
      "activations/layer5_attention_weight_min": -70.89863586425781,
      "activations/layer6_attention_weight_max": 49.974369049072266,
      "activations/layer6_attention_weight_min": -50.32395553588867,
      "activations/layer7_attention_weight_max": 63.424659729003906,
      "activations/layer7_attention_weight_min": -62.89548873901367,
      "activations/layer8_attention_weight_max": 44.037879943847656,
      "activations/layer8_attention_weight_min": -51.805015563964844,
      "activations/layer9_attention_weight_max": 41.87222671508789,
      "activations/layer9_attention_weight_min": -46.323326110839844,
      "epoch": 16.57,
      "learning_rate": 4.3553787878787874e-05,
      "loss": 2.7417,
      "step": 285250
    },
    {
      "activations/layer0_attention_weight_max": 15.053010940551758,
      "activations/layer0_attention_weight_min": -13.603002548217773,
      "activations/layer10_attention_weight_max": 32.75229263305664,
      "activations/layer10_attention_weight_min": -35.21345901489258,
      "activations/layer11_attention_weight_max": 30.366018295288086,
      "activations/layer11_attention_weight_min": -33.961185455322266,
      "activations/layer12_attention_weight_max": 18.943403244018555,
      "activations/layer12_attention_weight_min": -31.316408157348633,
      "activations/layer13_attention_weight_max": 34.468719482421875,
      "activations/layer13_attention_weight_min": -27.30084991455078,
      "activations/layer14_attention_weight_max": 35.001827239990234,
      "activations/layer14_attention_weight_min": -30.701330184936523,
      "activations/layer15_attention_weight_max": 33.54026412963867,
      "activations/layer15_attention_weight_min": -30.69597625732422,
      "activations/layer16_attention_weight_max": 33.939369201660156,
      "activations/layer16_attention_weight_min": -31.84716796875,
      "activations/layer17_attention_weight_max": 49.58714294433594,
      "activations/layer17_attention_weight_min": -46.937686920166016,
      "activations/layer18_attention_weight_max": 44.950923919677734,
      "activations/layer18_attention_weight_min": -42.389408111572266,
      "activations/layer19_attention_weight_max": 21.919889450073242,
      "activations/layer19_attention_weight_min": -24.517648696899414,
      "activations/layer1_attention_weight_max": 15.792871475219727,
      "activations/layer1_attention_weight_min": -14.358195304870605,
      "activations/layer20_attention_weight_max": 22.5269832611084,
      "activations/layer20_attention_weight_min": -23.428213119506836,
      "activations/layer21_attention_weight_max": 36.932979583740234,
      "activations/layer21_attention_weight_min": -26.142675399780273,
      "activations/layer22_attention_weight_max": 29.709259033203125,
      "activations/layer22_attention_weight_min": -27.40141487121582,
      "activations/layer23_attention_weight_max": 36.629241943359375,
      "activations/layer23_attention_weight_min": -27.48685073852539,
      "activations/layer2_attention_weight_max": 32.75550079345703,
      "activations/layer2_attention_weight_min": -29.283992767333984,
      "activations/layer3_attention_weight_max": 90.29696655273438,
      "activations/layer3_attention_weight_min": -86.9717788696289,
      "activations/layer4_attention_weight_max": 88.08370208740234,
      "activations/layer4_attention_weight_min": -86.3752212524414,
      "activations/layer5_attention_weight_max": 61.48664093017578,
      "activations/layer5_attention_weight_min": -71.95613098144531,
      "activations/layer6_attention_weight_max": 47.51942443847656,
      "activations/layer6_attention_weight_min": -47.58290100097656,
      "activations/layer7_attention_weight_max": 59.357643127441406,
      "activations/layer7_attention_weight_min": -62.03858947753906,
      "activations/layer8_attention_weight_max": 42.613887786865234,
      "activations/layer8_attention_weight_min": -46.74268341064453,
      "activations/layer9_attention_weight_max": 40.11697006225586,
      "activations/layer9_attention_weight_min": -43.962181091308594,
      "epoch": 16.58,
      "learning_rate": 4.353484848484848e-05,
      "loss": 2.7536,
      "step": 285300
    },
    {
      "activations/layer0_attention_weight_max": 16.188121795654297,
      "activations/layer0_attention_weight_min": -13.742977142333984,
      "activations/layer10_attention_weight_max": 36.706825256347656,
      "activations/layer10_attention_weight_min": -35.67783737182617,
      "activations/layer11_attention_weight_max": 35.448028564453125,
      "activations/layer11_attention_weight_min": -34.7308349609375,
      "activations/layer12_attention_weight_max": 19.165271759033203,
      "activations/layer12_attention_weight_min": -21.97533416748047,
      "activations/layer13_attention_weight_max": 38.03563690185547,
      "activations/layer13_attention_weight_min": -26.752178192138672,
      "activations/layer14_attention_weight_max": 36.75770568847656,
      "activations/layer14_attention_weight_min": -29.231998443603516,
      "activations/layer15_attention_weight_max": 32.566627502441406,
      "activations/layer15_attention_weight_min": -30.520835876464844,
      "activations/layer16_attention_weight_max": 34.806392669677734,
      "activations/layer16_attention_weight_min": -31.83724021911621,
      "activations/layer17_attention_weight_max": 53.57146072387695,
      "activations/layer17_attention_weight_min": -44.990264892578125,
      "activations/layer18_attention_weight_max": 48.79850387573242,
      "activations/layer18_attention_weight_min": -39.8418083190918,
      "activations/layer19_attention_weight_max": 24.371063232421875,
      "activations/layer19_attention_weight_min": -24.755722045898438,
      "activations/layer1_attention_weight_max": 15.650747299194336,
      "activations/layer1_attention_weight_min": -13.876542091369629,
      "activations/layer20_attention_weight_max": 22.67149543762207,
      "activations/layer20_attention_weight_min": -22.020116806030273,
      "activations/layer21_attention_weight_max": 35.551185607910156,
      "activations/layer21_attention_weight_min": -21.7060604095459,
      "activations/layer22_attention_weight_max": 29.06180763244629,
      "activations/layer22_attention_weight_min": -25.828693389892578,
      "activations/layer23_attention_weight_max": 34.4447135925293,
      "activations/layer23_attention_weight_min": -24.906614303588867,
      "activations/layer2_attention_weight_max": 32.268741607666016,
      "activations/layer2_attention_weight_min": -32.34909439086914,
      "activations/layer3_attention_weight_max": 91.94590759277344,
      "activations/layer3_attention_weight_min": -91.66862487792969,
      "activations/layer4_attention_weight_max": 91.5550308227539,
      "activations/layer4_attention_weight_min": -90.93489837646484,
      "activations/layer5_attention_weight_max": 66.89329528808594,
      "activations/layer5_attention_weight_min": -77.9317398071289,
      "activations/layer6_attention_weight_max": 53.15468978881836,
      "activations/layer6_attention_weight_min": -50.7141227722168,
      "activations/layer7_attention_weight_max": 67.44217681884766,
      "activations/layer7_attention_weight_min": -61.78530502319336,
      "activations/layer8_attention_weight_max": 45.509971618652344,
      "activations/layer8_attention_weight_min": -45.919822692871094,
      "activations/layer9_attention_weight_max": 44.89527893066406,
      "activations/layer9_attention_weight_min": -44.612735748291016,
      "epoch": 16.58,
      "learning_rate": 4.3515909090909084e-05,
      "loss": 2.7407,
      "step": 285350
    },
    {
      "activations/layer0_attention_weight_max": 16.233810424804688,
      "activations/layer0_attention_weight_min": -13.676437377929688,
      "activations/layer10_attention_weight_max": 33.201900482177734,
      "activations/layer10_attention_weight_min": -33.659976959228516,
      "activations/layer11_attention_weight_max": 31.141563415527344,
      "activations/layer11_attention_weight_min": -32.627769470214844,
      "activations/layer12_attention_weight_max": 19.24454689025879,
      "activations/layer12_attention_weight_min": -30.023330688476562,
      "activations/layer13_attention_weight_max": 29.942352294921875,
      "activations/layer13_attention_weight_min": -27.076488494873047,
      "activations/layer14_attention_weight_max": 33.552024841308594,
      "activations/layer14_attention_weight_min": -29.260570526123047,
      "activations/layer15_attention_weight_max": 30.73304557800293,
      "activations/layer15_attention_weight_min": -29.385971069335938,
      "activations/layer16_attention_weight_max": 31.087739944458008,
      "activations/layer16_attention_weight_min": -30.441438674926758,
      "activations/layer17_attention_weight_max": 49.91310501098633,
      "activations/layer17_attention_weight_min": -43.273189544677734,
      "activations/layer18_attention_weight_max": 46.13333511352539,
      "activations/layer18_attention_weight_min": -38.47645568847656,
      "activations/layer19_attention_weight_max": 21.628828048706055,
      "activations/layer19_attention_weight_min": -22.158666610717773,
      "activations/layer1_attention_weight_max": 16.485233306884766,
      "activations/layer1_attention_weight_min": -13.445355415344238,
      "activations/layer20_attention_weight_max": 20.41989517211914,
      "activations/layer20_attention_weight_min": -23.808759689331055,
      "activations/layer21_attention_weight_max": 41.403076171875,
      "activations/layer21_attention_weight_min": -28.91299057006836,
      "activations/layer22_attention_weight_max": 27.465208053588867,
      "activations/layer22_attention_weight_min": -26.233966827392578,
      "activations/layer23_attention_weight_max": 34.41911315917969,
      "activations/layer23_attention_weight_min": -26.38019561767578,
      "activations/layer2_attention_weight_max": 33.29751205444336,
      "activations/layer2_attention_weight_min": -30.582855224609375,
      "activations/layer3_attention_weight_max": 91.71173095703125,
      "activations/layer3_attention_weight_min": -88.62906646728516,
      "activations/layer4_attention_weight_max": 89.6244125366211,
      "activations/layer4_attention_weight_min": -88.47740936279297,
      "activations/layer5_attention_weight_max": 64.65950012207031,
      "activations/layer5_attention_weight_min": -74.78094482421875,
      "activations/layer6_attention_weight_max": 48.87785720825195,
      "activations/layer6_attention_weight_min": -51.15384292602539,
      "activations/layer7_attention_weight_max": 63.774742126464844,
      "activations/layer7_attention_weight_min": -64.67803955078125,
      "activations/layer8_attention_weight_max": 42.301551818847656,
      "activations/layer8_attention_weight_min": -46.65860366821289,
      "activations/layer9_attention_weight_max": 42.22001266479492,
      "activations/layer9_attention_weight_min": -44.648033142089844,
      "epoch": 16.58,
      "learning_rate": 4.34969696969697e-05,
      "loss": 2.736,
      "step": 285400
    },
    {
      "activations/layer0_attention_weight_max": 16.625810623168945,
      "activations/layer0_attention_weight_min": -13.706656455993652,
      "activations/layer10_attention_weight_max": 33.5306282043457,
      "activations/layer10_attention_weight_min": -35.0732536315918,
      "activations/layer11_attention_weight_max": 31.307981491088867,
      "activations/layer11_attention_weight_min": -33.04332733154297,
      "activations/layer12_attention_weight_max": 18.8184814453125,
      "activations/layer12_attention_weight_min": -24.482208251953125,
      "activations/layer13_attention_weight_max": 33.22815704345703,
      "activations/layer13_attention_weight_min": -28.735782623291016,
      "activations/layer14_attention_weight_max": 33.30487823486328,
      "activations/layer14_attention_weight_min": -29.91061019897461,
      "activations/layer15_attention_weight_max": 30.314451217651367,
      "activations/layer15_attention_weight_min": -29.4411563873291,
      "activations/layer16_attention_weight_max": 30.17331886291504,
      "activations/layer16_attention_weight_min": -29.359086990356445,
      "activations/layer17_attention_weight_max": 45.82706832885742,
      "activations/layer17_attention_weight_min": -42.8524284362793,
      "activations/layer18_attention_weight_max": 41.41509246826172,
      "activations/layer18_attention_weight_min": -40.039398193359375,
      "activations/layer19_attention_weight_max": 22.564802169799805,
      "activations/layer19_attention_weight_min": -21.23140525817871,
      "activations/layer1_attention_weight_max": 15.570536613464355,
      "activations/layer1_attention_weight_min": -13.329140663146973,
      "activations/layer20_attention_weight_max": 21.207353591918945,
      "activations/layer20_attention_weight_min": -21.55479621887207,
      "activations/layer21_attention_weight_max": 30.892484664916992,
      "activations/layer21_attention_weight_min": -23.677871704101562,
      "activations/layer22_attention_weight_max": 28.033058166503906,
      "activations/layer22_attention_weight_min": -25.322500228881836,
      "activations/layer23_attention_weight_max": 38.34124755859375,
      "activations/layer23_attention_weight_min": -27.131614685058594,
      "activations/layer2_attention_weight_max": 30.628080368041992,
      "activations/layer2_attention_weight_min": -29.085416793823242,
      "activations/layer3_attention_weight_max": 86.32324981689453,
      "activations/layer3_attention_weight_min": -88.330322265625,
      "activations/layer4_attention_weight_max": 86.62395477294922,
      "activations/layer4_attention_weight_min": -84.43524932861328,
      "activations/layer5_attention_weight_max": 64.20980834960938,
      "activations/layer5_attention_weight_min": -76.6641616821289,
      "activations/layer6_attention_weight_max": 51.98196792602539,
      "activations/layer6_attention_weight_min": -50.233070373535156,
      "activations/layer7_attention_weight_max": 64.0218276977539,
      "activations/layer7_attention_weight_min": -64.65936279296875,
      "activations/layer8_attention_weight_max": 43.965110778808594,
      "activations/layer8_attention_weight_min": -47.1077880859375,
      "activations/layer9_attention_weight_max": 43.595909118652344,
      "activations/layer9_attention_weight_min": -44.7585563659668,
      "epoch": 16.59,
      "learning_rate": 4.34780303030303e-05,
      "loss": 2.7528,
      "step": 285450
    },
    {
      "activations/layer0_attention_weight_max": 15.474035263061523,
      "activations/layer0_attention_weight_min": -13.9100980758667,
      "activations/layer10_attention_weight_max": 34.396942138671875,
      "activations/layer10_attention_weight_min": -35.04150390625,
      "activations/layer11_attention_weight_max": 32.53742218017578,
      "activations/layer11_attention_weight_min": -34.08795166015625,
      "activations/layer12_attention_weight_max": 19.107297897338867,
      "activations/layer12_attention_weight_min": -26.098533630371094,
      "activations/layer13_attention_weight_max": 35.131046295166016,
      "activations/layer13_attention_weight_min": -27.81368637084961,
      "activations/layer14_attention_weight_max": 36.21576690673828,
      "activations/layer14_attention_weight_min": -29.159339904785156,
      "activations/layer15_attention_weight_max": 31.481185913085938,
      "activations/layer15_attention_weight_min": -28.141313552856445,
      "activations/layer16_attention_weight_max": 32.6864128112793,
      "activations/layer16_attention_weight_min": -32.031227111816406,
      "activations/layer17_attention_weight_max": 53.09849548339844,
      "activations/layer17_attention_weight_min": -45.778648376464844,
      "activations/layer18_attention_weight_max": 48.42353439331055,
      "activations/layer18_attention_weight_min": -41.07099914550781,
      "activations/layer19_attention_weight_max": 22.581336975097656,
      "activations/layer19_attention_weight_min": -24.126989364624023,
      "activations/layer1_attention_weight_max": 15.980504989624023,
      "activations/layer1_attention_weight_min": -13.398270606994629,
      "activations/layer20_attention_weight_max": 20.695005416870117,
      "activations/layer20_attention_weight_min": -21.959476470947266,
      "activations/layer21_attention_weight_max": 32.142879486083984,
      "activations/layer21_attention_weight_min": -26.179418563842773,
      "activations/layer22_attention_weight_max": 26.15803337097168,
      "activations/layer22_attention_weight_min": -25.0095157623291,
      "activations/layer23_attention_weight_max": 35.87395477294922,
      "activations/layer23_attention_weight_min": -28.024580001831055,
      "activations/layer2_attention_weight_max": 31.516422271728516,
      "activations/layer2_attention_weight_min": -30.90778350830078,
      "activations/layer3_attention_weight_max": 92.20980834960938,
      "activations/layer3_attention_weight_min": -90.6266860961914,
      "activations/layer4_attention_weight_max": 91.7163314819336,
      "activations/layer4_attention_weight_min": -85.81468963623047,
      "activations/layer5_attention_weight_max": 65.26426696777344,
      "activations/layer5_attention_weight_min": -69.6475601196289,
      "activations/layer6_attention_weight_max": 49.418418884277344,
      "activations/layer6_attention_weight_min": -50.54804611206055,
      "activations/layer7_attention_weight_max": 60.97002410888672,
      "activations/layer7_attention_weight_min": -59.71708679199219,
      "activations/layer8_attention_weight_max": 44.15451431274414,
      "activations/layer8_attention_weight_min": -48.77760314941406,
      "activations/layer9_attention_weight_max": 42.48466110229492,
      "activations/layer9_attention_weight_min": -42.5830192565918,
      "epoch": 16.59,
      "learning_rate": 4.34590909090909e-05,
      "loss": 2.745,
      "step": 285500
    },
    {
      "activations/layer0_attention_weight_max": 15.848821640014648,
      "activations/layer0_attention_weight_min": -13.815611839294434,
      "activations/layer10_attention_weight_max": 34.83530044555664,
      "activations/layer10_attention_weight_min": -34.342987060546875,
      "activations/layer11_attention_weight_max": 32.457244873046875,
      "activations/layer11_attention_weight_min": -34.06843185424805,
      "activations/layer12_attention_weight_max": 19.54471778869629,
      "activations/layer12_attention_weight_min": -29.04656410217285,
      "activations/layer13_attention_weight_max": 31.477935791015625,
      "activations/layer13_attention_weight_min": -26.871273040771484,
      "activations/layer14_attention_weight_max": 37.223419189453125,
      "activations/layer14_attention_weight_min": -29.010305404663086,
      "activations/layer15_attention_weight_max": 32.82460403442383,
      "activations/layer15_attention_weight_min": -29.438154220581055,
      "activations/layer16_attention_weight_max": 32.50483703613281,
      "activations/layer16_attention_weight_min": -29.608320236206055,
      "activations/layer17_attention_weight_max": 50.13011932373047,
      "activations/layer17_attention_weight_min": -44.85969924926758,
      "activations/layer18_attention_weight_max": 44.36789321899414,
      "activations/layer18_attention_weight_min": -37.12139129638672,
      "activations/layer19_attention_weight_max": 22.82071304321289,
      "activations/layer19_attention_weight_min": -22.09880828857422,
      "activations/layer1_attention_weight_max": 16.071746826171875,
      "activations/layer1_attention_weight_min": -14.634697914123535,
      "activations/layer20_attention_weight_max": 23.0048885345459,
      "activations/layer20_attention_weight_min": -23.371904373168945,
      "activations/layer21_attention_weight_max": 37.6875114440918,
      "activations/layer21_attention_weight_min": -21.822052001953125,
      "activations/layer22_attention_weight_max": 25.491352081298828,
      "activations/layer22_attention_weight_min": -24.87606430053711,
      "activations/layer23_attention_weight_max": 32.41085433959961,
      "activations/layer23_attention_weight_min": -21.409351348876953,
      "activations/layer2_attention_weight_max": 32.12435531616211,
      "activations/layer2_attention_weight_min": -30.328487396240234,
      "activations/layer3_attention_weight_max": 89.52944946289062,
      "activations/layer3_attention_weight_min": -88.03218078613281,
      "activations/layer4_attention_weight_max": 86.22411346435547,
      "activations/layer4_attention_weight_min": -85.98516082763672,
      "activations/layer5_attention_weight_max": 59.53428649902344,
      "activations/layer5_attention_weight_min": -68.6775894165039,
      "activations/layer6_attention_weight_max": 48.59235382080078,
      "activations/layer6_attention_weight_min": -48.110389709472656,
      "activations/layer7_attention_weight_max": 61.493587493896484,
      "activations/layer7_attention_weight_min": -61.515106201171875,
      "activations/layer8_attention_weight_max": 44.27398681640625,
      "activations/layer8_attention_weight_min": -50.18900680541992,
      "activations/layer9_attention_weight_max": 49.12430191040039,
      "activations/layer9_attention_weight_min": -43.67368698120117,
      "epoch": 16.59,
      "learning_rate": 4.344015151515151e-05,
      "loss": 2.7485,
      "step": 285550
    },
    {
      "activations/layer0_attention_weight_max": 16.20834732055664,
      "activations/layer0_attention_weight_min": -13.586928367614746,
      "activations/layer10_attention_weight_max": 38.026729583740234,
      "activations/layer10_attention_weight_min": -35.996620178222656,
      "activations/layer11_attention_weight_max": 33.65043640136719,
      "activations/layer11_attention_weight_min": -33.375457763671875,
      "activations/layer12_attention_weight_max": 20.440460205078125,
      "activations/layer12_attention_weight_min": -23.005714416503906,
      "activations/layer13_attention_weight_max": 37.18349838256836,
      "activations/layer13_attention_weight_min": -28.925491333007812,
      "activations/layer14_attention_weight_max": 35.287513732910156,
      "activations/layer14_attention_weight_min": -29.805837631225586,
      "activations/layer15_attention_weight_max": 34.04484558105469,
      "activations/layer15_attention_weight_min": -30.406606674194336,
      "activations/layer16_attention_weight_max": 34.31542205810547,
      "activations/layer16_attention_weight_min": -31.839704513549805,
      "activations/layer17_attention_weight_max": 54.4852180480957,
      "activations/layer17_attention_weight_min": -46.5220832824707,
      "activations/layer18_attention_weight_max": 49.21353530883789,
      "activations/layer18_attention_weight_min": -39.830753326416016,
      "activations/layer19_attention_weight_max": 24.293962478637695,
      "activations/layer19_attention_weight_min": -24.383705139160156,
      "activations/layer1_attention_weight_max": 15.936629295349121,
      "activations/layer1_attention_weight_min": -13.989435195922852,
      "activations/layer20_attention_weight_max": 25.43111228942871,
      "activations/layer20_attention_weight_min": -24.100378036499023,
      "activations/layer21_attention_weight_max": 38.52357864379883,
      "activations/layer21_attention_weight_min": -21.98723030090332,
      "activations/layer22_attention_weight_max": 29.002933502197266,
      "activations/layer22_attention_weight_min": -25.59196662902832,
      "activations/layer23_attention_weight_max": 38.52621841430664,
      "activations/layer23_attention_weight_min": -23.46266746520996,
      "activations/layer2_attention_weight_max": 31.536724090576172,
      "activations/layer2_attention_weight_min": -33.301353454589844,
      "activations/layer3_attention_weight_max": 88.60198211669922,
      "activations/layer3_attention_weight_min": -91.83773040771484,
      "activations/layer4_attention_weight_max": 91.45565795898438,
      "activations/layer4_attention_weight_min": -89.47209930419922,
      "activations/layer5_attention_weight_max": 67.22639465332031,
      "activations/layer5_attention_weight_min": -81.13395690917969,
      "activations/layer6_attention_weight_max": 51.26359558105469,
      "activations/layer6_attention_weight_min": -53.01070785522461,
      "activations/layer7_attention_weight_max": 68.35494232177734,
      "activations/layer7_attention_weight_min": -69.68070983886719,
      "activations/layer8_attention_weight_max": 49.99732971191406,
      "activations/layer8_attention_weight_min": -52.21437454223633,
      "activations/layer9_attention_weight_max": 46.70843505859375,
      "activations/layer9_attention_weight_min": -49.35710525512695,
      "epoch": 16.59,
      "learning_rate": 4.342121212121212e-05,
      "loss": 2.7467,
      "step": 285600
    },
    {
      "activations/layer0_attention_weight_max": 14.907124519348145,
      "activations/layer0_attention_weight_min": -14.155272483825684,
      "activations/layer10_attention_weight_max": 36.136070251464844,
      "activations/layer10_attention_weight_min": -35.68628692626953,
      "activations/layer11_attention_weight_max": 31.9288330078125,
      "activations/layer11_attention_weight_min": -35.41020965576172,
      "activations/layer12_attention_weight_max": 20.29673957824707,
      "activations/layer12_attention_weight_min": -27.356861114501953,
      "activations/layer13_attention_weight_max": 35.25279235839844,
      "activations/layer13_attention_weight_min": -28.608510971069336,
      "activations/layer14_attention_weight_max": 34.26652526855469,
      "activations/layer14_attention_weight_min": -30.246145248413086,
      "activations/layer15_attention_weight_max": 32.162689208984375,
      "activations/layer15_attention_weight_min": -29.979406356811523,
      "activations/layer16_attention_weight_max": 33.93814468383789,
      "activations/layer16_attention_weight_min": -31.23633575439453,
      "activations/layer17_attention_weight_max": 51.77595138549805,
      "activations/layer17_attention_weight_min": -45.642112731933594,
      "activations/layer18_attention_weight_max": 50.18914031982422,
      "activations/layer18_attention_weight_min": -39.769046783447266,
      "activations/layer19_attention_weight_max": 23.043668746948242,
      "activations/layer19_attention_weight_min": -22.979686737060547,
      "activations/layer1_attention_weight_max": 16.667621612548828,
      "activations/layer1_attention_weight_min": -13.930131912231445,
      "activations/layer20_attention_weight_max": 22.4512939453125,
      "activations/layer20_attention_weight_min": -21.57866668701172,
      "activations/layer21_attention_weight_max": 34.11807632446289,
      "activations/layer21_attention_weight_min": -23.4512882232666,
      "activations/layer22_attention_weight_max": 29.85358238220215,
      "activations/layer22_attention_weight_min": -25.797199249267578,
      "activations/layer23_attention_weight_max": 37.3316764831543,
      "activations/layer23_attention_weight_min": -24.285398483276367,
      "activations/layer2_attention_weight_max": 30.012914657592773,
      "activations/layer2_attention_weight_min": -29.66381072998047,
      "activations/layer3_attention_weight_max": 88.6283950805664,
      "activations/layer3_attention_weight_min": -86.69699096679688,
      "activations/layer4_attention_weight_max": 88.45995330810547,
      "activations/layer4_attention_weight_min": -88.9384994506836,
      "activations/layer5_attention_weight_max": 67.0565414428711,
      "activations/layer5_attention_weight_min": -70.4743881225586,
      "activations/layer6_attention_weight_max": 49.31529998779297,
      "activations/layer6_attention_weight_min": -51.07814407348633,
      "activations/layer7_attention_weight_max": 63.193016052246094,
      "activations/layer7_attention_weight_min": -63.25939178466797,
      "activations/layer8_attention_weight_max": 47.940223693847656,
      "activations/layer8_attention_weight_min": -50.861961364746094,
      "activations/layer9_attention_weight_max": 46.4001579284668,
      "activations/layer9_attention_weight_min": -44.52595138549805,
      "epoch": 16.6,
      "learning_rate": 4.340265151515151e-05,
      "loss": 2.7674,
      "step": 285650
    },
    {
      "activations/layer0_attention_weight_max": 15.653648376464844,
      "activations/layer0_attention_weight_min": -13.948847770690918,
      "activations/layer10_attention_weight_max": 36.9932861328125,
      "activations/layer10_attention_weight_min": -36.31001281738281,
      "activations/layer11_attention_weight_max": 31.642990112304688,
      "activations/layer11_attention_weight_min": -35.45166778564453,
      "activations/layer12_attention_weight_max": 18.358793258666992,
      "activations/layer12_attention_weight_min": -23.617284774780273,
      "activations/layer13_attention_weight_max": 32.55038833618164,
      "activations/layer13_attention_weight_min": -29.97023582458496,
      "activations/layer14_attention_weight_max": 33.73933792114258,
      "activations/layer14_attention_weight_min": -32.057472229003906,
      "activations/layer15_attention_weight_max": 29.775362014770508,
      "activations/layer15_attention_weight_min": -30.795833587646484,
      "activations/layer16_attention_weight_max": 34.30633544921875,
      "activations/layer16_attention_weight_min": -32.932701110839844,
      "activations/layer17_attention_weight_max": 47.83328628540039,
      "activations/layer17_attention_weight_min": -44.1686897277832,
      "activations/layer18_attention_weight_max": 44.861812591552734,
      "activations/layer18_attention_weight_min": -38.547664642333984,
      "activations/layer19_attention_weight_max": 23.365381240844727,
      "activations/layer19_attention_weight_min": -22.87013816833496,
      "activations/layer1_attention_weight_max": 16.56465721130371,
      "activations/layer1_attention_weight_min": -14.259197235107422,
      "activations/layer20_attention_weight_max": 23.614221572875977,
      "activations/layer20_attention_weight_min": -20.831880569458008,
      "activations/layer21_attention_weight_max": 35.14576721191406,
      "activations/layer21_attention_weight_min": -22.50604820251465,
      "activations/layer22_attention_weight_max": 28.440717697143555,
      "activations/layer22_attention_weight_min": -26.74734115600586,
      "activations/layer23_attention_weight_max": 34.919036865234375,
      "activations/layer23_attention_weight_min": -24.023555755615234,
      "activations/layer2_attention_weight_max": 31.41267967224121,
      "activations/layer2_attention_weight_min": -29.748065948486328,
      "activations/layer3_attention_weight_max": 87.78778839111328,
      "activations/layer3_attention_weight_min": -83.4871826171875,
      "activations/layer4_attention_weight_max": 87.7491683959961,
      "activations/layer4_attention_weight_min": -85.54320526123047,
      "activations/layer5_attention_weight_max": 59.97455978393555,
      "activations/layer5_attention_weight_min": -71.0772476196289,
      "activations/layer6_attention_weight_max": 48.194454193115234,
      "activations/layer6_attention_weight_min": -48.15721130371094,
      "activations/layer7_attention_weight_max": 68.51153564453125,
      "activations/layer7_attention_weight_min": -61.99827194213867,
      "activations/layer8_attention_weight_max": 44.37489700317383,
      "activations/layer8_attention_weight_min": -47.53682327270508,
      "activations/layer9_attention_weight_max": 42.848167419433594,
      "activations/layer9_attention_weight_min": -43.89890670776367,
      "epoch": 16.6,
      "learning_rate": 4.338371212121212e-05,
      "loss": 2.7478,
      "step": 285700
    },
    {
      "activations/layer0_attention_weight_max": 15.907044410705566,
      "activations/layer0_attention_weight_min": -14.248001098632812,
      "activations/layer10_attention_weight_max": 38.390140533447266,
      "activations/layer10_attention_weight_min": -38.38683319091797,
      "activations/layer11_attention_weight_max": 37.797325134277344,
      "activations/layer11_attention_weight_min": -35.80345153808594,
      "activations/layer12_attention_weight_max": 18.296173095703125,
      "activations/layer12_attention_weight_min": -26.98905372619629,
      "activations/layer13_attention_weight_max": 37.226802825927734,
      "activations/layer13_attention_weight_min": -29.004491806030273,
      "activations/layer14_attention_weight_max": 35.98725128173828,
      "activations/layer14_attention_weight_min": -29.44807243347168,
      "activations/layer15_attention_weight_max": 33.94351577758789,
      "activations/layer15_attention_weight_min": -29.654064178466797,
      "activations/layer16_attention_weight_max": 32.1570930480957,
      "activations/layer16_attention_weight_min": -29.05685806274414,
      "activations/layer17_attention_weight_max": 50.91249084472656,
      "activations/layer17_attention_weight_min": -44.405208587646484,
      "activations/layer18_attention_weight_max": 46.405731201171875,
      "activations/layer18_attention_weight_min": -37.139408111572266,
      "activations/layer19_attention_weight_max": 27.102773666381836,
      "activations/layer19_attention_weight_min": -20.060813903808594,
      "activations/layer1_attention_weight_max": 14.948277473449707,
      "activations/layer1_attention_weight_min": -13.547002792358398,
      "activations/layer20_attention_weight_max": 24.361562728881836,
      "activations/layer20_attention_weight_min": -21.816635131835938,
      "activations/layer21_attention_weight_max": 38.93547439575195,
      "activations/layer21_attention_weight_min": -22.2054386138916,
      "activations/layer22_attention_weight_max": 32.679378509521484,
      "activations/layer22_attention_weight_min": -27.104570388793945,
      "activations/layer23_attention_weight_max": 36.084537506103516,
      "activations/layer23_attention_weight_min": -26.531539916992188,
      "activations/layer2_attention_weight_max": 32.54234313964844,
      "activations/layer2_attention_weight_min": -30.6148738861084,
      "activations/layer3_attention_weight_max": 93.03826904296875,
      "activations/layer3_attention_weight_min": -90.1721420288086,
      "activations/layer4_attention_weight_max": 92.97756958007812,
      "activations/layer4_attention_weight_min": -89.44763946533203,
      "activations/layer5_attention_weight_max": 64.07632446289062,
      "activations/layer5_attention_weight_min": -76.12857818603516,
      "activations/layer6_attention_weight_max": 51.415138244628906,
      "activations/layer6_attention_weight_min": -50.51313400268555,
      "activations/layer7_attention_weight_max": 68.54485321044922,
      "activations/layer7_attention_weight_min": -60.85994338989258,
      "activations/layer8_attention_weight_max": 53.90913772583008,
      "activations/layer8_attention_weight_min": -47.54558563232422,
      "activations/layer9_attention_weight_max": 53.43886947631836,
      "activations/layer9_attention_weight_min": -46.50736618041992,
      "epoch": 16.6,
      "learning_rate": 4.336477272727272e-05,
      "loss": 2.768,
      "step": 285750
    },
    {
      "activations/layer0_attention_weight_max": 16.43184471130371,
      "activations/layer0_attention_weight_min": -13.631274223327637,
      "activations/layer10_attention_weight_max": 38.52070999145508,
      "activations/layer10_attention_weight_min": -36.69263458251953,
      "activations/layer11_attention_weight_max": 35.079071044921875,
      "activations/layer11_attention_weight_min": -34.264671325683594,
      "activations/layer12_attention_weight_max": 19.26280975341797,
      "activations/layer12_attention_weight_min": -26.03452491760254,
      "activations/layer13_attention_weight_max": 38.29924011230469,
      "activations/layer13_attention_weight_min": -30.131418228149414,
      "activations/layer14_attention_weight_max": 38.35594177246094,
      "activations/layer14_attention_weight_min": -33.243080139160156,
      "activations/layer15_attention_weight_max": 34.83417510986328,
      "activations/layer15_attention_weight_min": -34.90236282348633,
      "activations/layer16_attention_weight_max": 35.588218688964844,
      "activations/layer16_attention_weight_min": -34.63870620727539,
      "activations/layer17_attention_weight_max": 56.66171646118164,
      "activations/layer17_attention_weight_min": -52.21885299682617,
      "activations/layer18_attention_weight_max": 51.78050231933594,
      "activations/layer18_attention_weight_min": -45.92327117919922,
      "activations/layer19_attention_weight_max": 25.572364807128906,
      "activations/layer19_attention_weight_min": -23.622732162475586,
      "activations/layer1_attention_weight_max": 16.680343627929688,
      "activations/layer1_attention_weight_min": -14.237689971923828,
      "activations/layer20_attention_weight_max": 26.61239242553711,
      "activations/layer20_attention_weight_min": -22.8811092376709,
      "activations/layer21_attention_weight_max": 45.36130142211914,
      "activations/layer21_attention_weight_min": -23.749401092529297,
      "activations/layer22_attention_weight_max": 29.783681869506836,
      "activations/layer22_attention_weight_min": -26.095979690551758,
      "activations/layer23_attention_weight_max": 40.22419357299805,
      "activations/layer23_attention_weight_min": -25.083786010742188,
      "activations/layer2_attention_weight_max": 31.894241333007812,
      "activations/layer2_attention_weight_min": -29.221885681152344,
      "activations/layer3_attention_weight_max": 88.08338165283203,
      "activations/layer3_attention_weight_min": -87.25347137451172,
      "activations/layer4_attention_weight_max": 87.77764892578125,
      "activations/layer4_attention_weight_min": -86.61043548583984,
      "activations/layer5_attention_weight_max": 64.07461547851562,
      "activations/layer5_attention_weight_min": -70.72151184082031,
      "activations/layer6_attention_weight_max": 49.54536056518555,
      "activations/layer6_attention_weight_min": -49.60485076904297,
      "activations/layer7_attention_weight_max": 62.495323181152344,
      "activations/layer7_attention_weight_min": -67.47866821289062,
      "activations/layer8_attention_weight_max": 49.45479202270508,
      "activations/layer8_attention_weight_min": -49.280372619628906,
      "activations/layer9_attention_weight_max": 48.137237548828125,
      "activations/layer9_attention_weight_min": -45.99772262573242,
      "epoch": 16.61,
      "learning_rate": 4.3345833333333334e-05,
      "loss": 2.7565,
      "step": 285800
    },
    {
      "activations/layer0_attention_weight_max": 18.151498794555664,
      "activations/layer0_attention_weight_min": -13.851113319396973,
      "activations/layer10_attention_weight_max": 35.761993408203125,
      "activations/layer10_attention_weight_min": -37.14045333862305,
      "activations/layer11_attention_weight_max": 32.451454162597656,
      "activations/layer11_attention_weight_min": -33.89942932128906,
      "activations/layer12_attention_weight_max": 18.072708129882812,
      "activations/layer12_attention_weight_min": -23.959049224853516,
      "activations/layer13_attention_weight_max": 37.70994186401367,
      "activations/layer13_attention_weight_min": -29.03011131286621,
      "activations/layer14_attention_weight_max": 38.74641036987305,
      "activations/layer14_attention_weight_min": -31.44839096069336,
      "activations/layer15_attention_weight_max": 34.9673957824707,
      "activations/layer15_attention_weight_min": -30.715869903564453,
      "activations/layer16_attention_weight_max": 32.53318786621094,
      "activations/layer16_attention_weight_min": -31.946521759033203,
      "activations/layer17_attention_weight_max": 54.744075775146484,
      "activations/layer17_attention_weight_min": -45.20592498779297,
      "activations/layer18_attention_weight_max": 48.90943145751953,
      "activations/layer18_attention_weight_min": -41.37478256225586,
      "activations/layer19_attention_weight_max": 21.115636825561523,
      "activations/layer19_attention_weight_min": -22.282981872558594,
      "activations/layer1_attention_weight_max": 15.485158920288086,
      "activations/layer1_attention_weight_min": -14.804155349731445,
      "activations/layer20_attention_weight_max": 21.162181854248047,
      "activations/layer20_attention_weight_min": -25.09579086303711,
      "activations/layer21_attention_weight_max": 33.536590576171875,
      "activations/layer21_attention_weight_min": -23.578369140625,
      "activations/layer22_attention_weight_max": 26.012775421142578,
      "activations/layer22_attention_weight_min": -27.083759307861328,
      "activations/layer23_attention_weight_max": 35.60396957397461,
      "activations/layer23_attention_weight_min": -25.925716400146484,
      "activations/layer2_attention_weight_max": 31.93251609802246,
      "activations/layer2_attention_weight_min": -32.89556121826172,
      "activations/layer3_attention_weight_max": 90.08279418945312,
      "activations/layer3_attention_weight_min": -92.7739486694336,
      "activations/layer4_attention_weight_max": 87.80043029785156,
      "activations/layer4_attention_weight_min": -88.09241485595703,
      "activations/layer5_attention_weight_max": 63.23664855957031,
      "activations/layer5_attention_weight_min": -72.4379653930664,
      "activations/layer6_attention_weight_max": 49.818599700927734,
      "activations/layer6_attention_weight_min": -50.00613021850586,
      "activations/layer7_attention_weight_max": 63.65400314331055,
      "activations/layer7_attention_weight_min": -62.1622314453125,
      "activations/layer8_attention_weight_max": 47.4285774230957,
      "activations/layer8_attention_weight_min": -53.33888626098633,
      "activations/layer9_attention_weight_max": 45.9720458984375,
      "activations/layer9_attention_weight_min": -51.77889633178711,
      "epoch": 16.61,
      "learning_rate": 4.3326893939393936e-05,
      "loss": 2.7282,
      "step": 285850
    },
    {
      "activations/layer0_attention_weight_max": 14.820977210998535,
      "activations/layer0_attention_weight_min": -13.799043655395508,
      "activations/layer10_attention_weight_max": 32.473655700683594,
      "activations/layer10_attention_weight_min": -34.10710906982422,
      "activations/layer11_attention_weight_max": 30.415916442871094,
      "activations/layer11_attention_weight_min": -31.056659698486328,
      "activations/layer12_attention_weight_max": 19.994930267333984,
      "activations/layer12_attention_weight_min": -23.950519561767578,
      "activations/layer13_attention_weight_max": 33.15620803833008,
      "activations/layer13_attention_weight_min": -28.60152816772461,
      "activations/layer14_attention_weight_max": 36.37370681762695,
      "activations/layer14_attention_weight_min": -34.57545852661133,
      "activations/layer15_attention_weight_max": 34.6739501953125,
      "activations/layer15_attention_weight_min": -31.920944213867188,
      "activations/layer16_attention_weight_max": 34.02042007446289,
      "activations/layer16_attention_weight_min": -31.852352142333984,
      "activations/layer17_attention_weight_max": 51.784217834472656,
      "activations/layer17_attention_weight_min": -47.055484771728516,
      "activations/layer18_attention_weight_max": 51.35552215576172,
      "activations/layer18_attention_weight_min": -44.80929183959961,
      "activations/layer19_attention_weight_max": 27.4619083404541,
      "activations/layer19_attention_weight_min": -26.057443618774414,
      "activations/layer1_attention_weight_max": 16.63928985595703,
      "activations/layer1_attention_weight_min": -14.03036117553711,
      "activations/layer20_attention_weight_max": 28.185136795043945,
      "activations/layer20_attention_weight_min": -25.225706100463867,
      "activations/layer21_attention_weight_max": 42.24900817871094,
      "activations/layer21_attention_weight_min": -22.211156845092773,
      "activations/layer22_attention_weight_max": 29.212657928466797,
      "activations/layer22_attention_weight_min": -29.40355682373047,
      "activations/layer23_attention_weight_max": 37.87125778198242,
      "activations/layer23_attention_weight_min": -27.671682357788086,
      "activations/layer2_attention_weight_max": 29.8242244720459,
      "activations/layer2_attention_weight_min": -29.908065795898438,
      "activations/layer3_attention_weight_max": 83.87535095214844,
      "activations/layer3_attention_weight_min": -85.24401092529297,
      "activations/layer4_attention_weight_max": 87.08956146240234,
      "activations/layer4_attention_weight_min": -86.03936004638672,
      "activations/layer5_attention_weight_max": 64.16597747802734,
      "activations/layer5_attention_weight_min": -76.13594818115234,
      "activations/layer6_attention_weight_max": 50.18948745727539,
      "activations/layer6_attention_weight_min": -48.11081314086914,
      "activations/layer7_attention_weight_max": 62.25850296020508,
      "activations/layer7_attention_weight_min": -61.040706634521484,
      "activations/layer8_attention_weight_max": 43.105411529541016,
      "activations/layer8_attention_weight_min": -43.99468231201172,
      "activations/layer9_attention_weight_max": 40.39582824707031,
      "activations/layer9_attention_weight_min": -44.5238151550293,
      "epoch": 16.61,
      "learning_rate": 4.330795454545454e-05,
      "loss": 2.7519,
      "step": 285900
    },
    {
      "activations/layer0_attention_weight_max": 15.492253303527832,
      "activations/layer0_attention_weight_min": -13.670790672302246,
      "activations/layer10_attention_weight_max": 34.53778076171875,
      "activations/layer10_attention_weight_min": -35.67784118652344,
      "activations/layer11_attention_weight_max": 32.13897705078125,
      "activations/layer11_attention_weight_min": -34.16974639892578,
      "activations/layer12_attention_weight_max": 20.3085994720459,
      "activations/layer12_attention_weight_min": -25.788782119750977,
      "activations/layer13_attention_weight_max": 41.88447952270508,
      "activations/layer13_attention_weight_min": -30.355810165405273,
      "activations/layer14_attention_weight_max": 40.27666091918945,
      "activations/layer14_attention_weight_min": -30.184534072875977,
      "activations/layer15_attention_weight_max": 32.93793487548828,
      "activations/layer15_attention_weight_min": -30.44462013244629,
      "activations/layer16_attention_weight_max": 32.048465728759766,
      "activations/layer16_attention_weight_min": -33.06898880004883,
      "activations/layer17_attention_weight_max": 49.51100540161133,
      "activations/layer17_attention_weight_min": -46.4774055480957,
      "activations/layer18_attention_weight_max": 42.98783493041992,
      "activations/layer18_attention_weight_min": -40.210662841796875,
      "activations/layer19_attention_weight_max": 23.957374572753906,
      "activations/layer19_attention_weight_min": -23.05398178100586,
      "activations/layer1_attention_weight_max": 16.68407440185547,
      "activations/layer1_attention_weight_min": -15.530348777770996,
      "activations/layer20_attention_weight_max": 21.87577247619629,
      "activations/layer20_attention_weight_min": -20.90540885925293,
      "activations/layer21_attention_weight_max": 40.75351333618164,
      "activations/layer21_attention_weight_min": -24.512313842773438,
      "activations/layer22_attention_weight_max": 30.28672218322754,
      "activations/layer22_attention_weight_min": -27.18123435974121,
      "activations/layer23_attention_weight_max": 36.50178527832031,
      "activations/layer23_attention_weight_min": -25.038503646850586,
      "activations/layer2_attention_weight_max": 30.40545654296875,
      "activations/layer2_attention_weight_min": -30.440837860107422,
      "activations/layer3_attention_weight_max": 88.66026306152344,
      "activations/layer3_attention_weight_min": -89.65267181396484,
      "activations/layer4_attention_weight_max": 89.03539276123047,
      "activations/layer4_attention_weight_min": -88.73043060302734,
      "activations/layer5_attention_weight_max": 67.12257385253906,
      "activations/layer5_attention_weight_min": -71.5147476196289,
      "activations/layer6_attention_weight_max": 50.30986022949219,
      "activations/layer6_attention_weight_min": -51.33497619628906,
      "activations/layer7_attention_weight_max": 65.9127197265625,
      "activations/layer7_attention_weight_min": -62.132545471191406,
      "activations/layer8_attention_weight_max": 45.23825454711914,
      "activations/layer8_attention_weight_min": -49.788673400878906,
      "activations/layer9_attention_weight_max": 44.327308654785156,
      "activations/layer9_attention_weight_min": -47.59621810913086,
      "epoch": 16.62,
      "learning_rate": 4.3289015151515146e-05,
      "loss": 2.7501,
      "step": 285950
    },
    {
      "activations/layer0_attention_weight_max": 15.653456687927246,
      "activations/layer0_attention_weight_min": -13.59360408782959,
      "activations/layer10_attention_weight_max": 35.542030334472656,
      "activations/layer10_attention_weight_min": -35.28886795043945,
      "activations/layer11_attention_weight_max": 32.125125885009766,
      "activations/layer11_attention_weight_min": -32.504371643066406,
      "activations/layer12_attention_weight_max": 18.261703491210938,
      "activations/layer12_attention_weight_min": -24.801042556762695,
      "activations/layer13_attention_weight_max": 37.66727828979492,
      "activations/layer13_attention_weight_min": -28.909969329833984,
      "activations/layer14_attention_weight_max": 32.7594108581543,
      "activations/layer14_attention_weight_min": -30.708881378173828,
      "activations/layer15_attention_weight_max": 31.115474700927734,
      "activations/layer15_attention_weight_min": -29.1027774810791,
      "activations/layer16_attention_weight_max": 33.65865707397461,
      "activations/layer16_attention_weight_min": -31.846025466918945,
      "activations/layer17_attention_weight_max": 51.04389572143555,
      "activations/layer17_attention_weight_min": -45.57539749145508,
      "activations/layer18_attention_weight_max": 47.43265151977539,
      "activations/layer18_attention_weight_min": -40.12220001220703,
      "activations/layer19_attention_weight_max": 24.728313446044922,
      "activations/layer19_attention_weight_min": -23.18912124633789,
      "activations/layer1_attention_weight_max": 16.33050537109375,
      "activations/layer1_attention_weight_min": -13.27144718170166,
      "activations/layer20_attention_weight_max": 24.340539932250977,
      "activations/layer20_attention_weight_min": -21.072795867919922,
      "activations/layer21_attention_weight_max": 35.824676513671875,
      "activations/layer21_attention_weight_min": -19.847660064697266,
      "activations/layer22_attention_weight_max": 35.189453125,
      "activations/layer22_attention_weight_min": -26.595308303833008,
      "activations/layer23_attention_weight_max": 39.948089599609375,
      "activations/layer23_attention_weight_min": -23.303733825683594,
      "activations/layer2_attention_weight_max": 29.846181869506836,
      "activations/layer2_attention_weight_min": -31.183887481689453,
      "activations/layer3_attention_weight_max": 89.48710632324219,
      "activations/layer3_attention_weight_min": -92.62605285644531,
      "activations/layer4_attention_weight_max": 89.1775131225586,
      "activations/layer4_attention_weight_min": -87.8133544921875,
      "activations/layer5_attention_weight_max": 62.73529052734375,
      "activations/layer5_attention_weight_min": -75.11312866210938,
      "activations/layer6_attention_weight_max": 46.652244567871094,
      "activations/layer6_attention_weight_min": -48.30652618408203,
      "activations/layer7_attention_weight_max": 60.741676330566406,
      "activations/layer7_attention_weight_min": -61.93590545654297,
      "activations/layer8_attention_weight_max": 45.51600646972656,
      "activations/layer8_attention_weight_min": -46.59941482543945,
      "activations/layer9_attention_weight_max": 43.764190673828125,
      "activations/layer9_attention_weight_min": -44.604915618896484,
      "epoch": 16.62,
      "learning_rate": 4.3270075757575754e-05,
      "loss": 2.7403,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4669,
      "eval_samples_per_second": 507.148,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4669,
      "eval_openwebtext_samples_per_second": 507.148,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9876,
      "eval_wikitext_samples_per_second": 229.42,
      "step": 286000
    },
    {
      "epoch": 16.62,
      "eval_lambada_loss": 2.47265625,
      "eval_lambada_ppl": 11.853891970655884,
      "eval_lambada_runtime": 9.521,
      "eval_lambada_samples_per_second": 511.395,
      "step": 286000
    },
    {
      "activations/layer0_attention_weight_max": 16.18749237060547,
      "activations/layer0_attention_weight_min": -13.9976224899292,
      "activations/layer10_attention_weight_max": 33.16343688964844,
      "activations/layer10_attention_weight_min": -32.98876953125,
      "activations/layer11_attention_weight_max": 31.017562866210938,
      "activations/layer11_attention_weight_min": -31.530960083007812,
      "activations/layer12_attention_weight_max": 18.667282104492188,
      "activations/layer12_attention_weight_min": -29.215065002441406,
      "activations/layer13_attention_weight_max": 35.765201568603516,
      "activations/layer13_attention_weight_min": -27.054777145385742,
      "activations/layer14_attention_weight_max": 35.43501663208008,
      "activations/layer14_attention_weight_min": -29.981111526489258,
      "activations/layer15_attention_weight_max": 33.268310546875,
      "activations/layer15_attention_weight_min": -30.11334228515625,
      "activations/layer16_attention_weight_max": 34.84099578857422,
      "activations/layer16_attention_weight_min": -30.916576385498047,
      "activations/layer17_attention_weight_max": 50.41069412231445,
      "activations/layer17_attention_weight_min": -43.94417190551758,
      "activations/layer18_attention_weight_max": 45.56336975097656,
      "activations/layer18_attention_weight_min": -42.21092987060547,
      "activations/layer19_attention_weight_max": 23.47377586364746,
      "activations/layer19_attention_weight_min": -22.96469497680664,
      "activations/layer1_attention_weight_max": 15.591106414794922,
      "activations/layer1_attention_weight_min": -15.311447143554688,
      "activations/layer20_attention_weight_max": 23.233335494995117,
      "activations/layer20_attention_weight_min": -22.45018768310547,
      "activations/layer21_attention_weight_max": 35.25782012939453,
      "activations/layer21_attention_weight_min": -22.610448837280273,
      "activations/layer22_attention_weight_max": 28.9906063079834,
      "activations/layer22_attention_weight_min": -26.350549697875977,
      "activations/layer23_attention_weight_max": 39.548099517822266,
      "activations/layer23_attention_weight_min": -24.746566772460938,
      "activations/layer2_attention_weight_max": 31.329471588134766,
      "activations/layer2_attention_weight_min": -30.605201721191406,
      "activations/layer3_attention_weight_max": 89.50768280029297,
      "activations/layer3_attention_weight_min": -90.11924743652344,
      "activations/layer4_attention_weight_max": 88.82664489746094,
      "activations/layer4_attention_weight_min": -87.1007080078125,
      "activations/layer5_attention_weight_max": 62.35637664794922,
      "activations/layer5_attention_weight_min": -75.47831726074219,
      "activations/layer6_attention_weight_max": 49.571895599365234,
      "activations/layer6_attention_weight_min": -49.950504302978516,
      "activations/layer7_attention_weight_max": 65.61397552490234,
      "activations/layer7_attention_weight_min": -60.587337493896484,
      "activations/layer8_attention_weight_max": 44.408447265625,
      "activations/layer8_attention_weight_min": -47.85884475708008,
      "activations/layer9_attention_weight_max": 43.97752380371094,
      "activations/layer9_attention_weight_min": -45.537994384765625,
      "epoch": 16.62,
      "learning_rate": 4.325113636363636e-05,
      "loss": 2.7505,
      "step": 286050
    },
    {
      "activations/layer0_attention_weight_max": 15.322480201721191,
      "activations/layer0_attention_weight_min": -13.752291679382324,
      "activations/layer10_attention_weight_max": 34.062564849853516,
      "activations/layer10_attention_weight_min": -35.274627685546875,
      "activations/layer11_attention_weight_max": 33.98085021972656,
      "activations/layer11_attention_weight_min": -33.9878044128418,
      "activations/layer12_attention_weight_max": 18.706266403198242,
      "activations/layer12_attention_weight_min": -24.014636993408203,
      "activations/layer13_attention_weight_max": 40.65816879272461,
      "activations/layer13_attention_weight_min": -25.689952850341797,
      "activations/layer14_attention_weight_max": 34.72899627685547,
      "activations/layer14_attention_weight_min": -27.606889724731445,
      "activations/layer15_attention_weight_max": 31.93816375732422,
      "activations/layer15_attention_weight_min": -28.207103729248047,
      "activations/layer16_attention_weight_max": 31.35173225402832,
      "activations/layer16_attention_weight_min": -29.6260986328125,
      "activations/layer17_attention_weight_max": 50.50438690185547,
      "activations/layer17_attention_weight_min": -41.84286117553711,
      "activations/layer18_attention_weight_max": 46.749755859375,
      "activations/layer18_attention_weight_min": -37.11604690551758,
      "activations/layer19_attention_weight_max": 23.88509178161621,
      "activations/layer19_attention_weight_min": -22.74293327331543,
      "activations/layer1_attention_weight_max": 15.34417724609375,
      "activations/layer1_attention_weight_min": -14.069740295410156,
      "activations/layer20_attention_weight_max": 25.190828323364258,
      "activations/layer20_attention_weight_min": -20.181781768798828,
      "activations/layer21_attention_weight_max": 31.8877010345459,
      "activations/layer21_attention_weight_min": -20.886110305786133,
      "activations/layer22_attention_weight_max": 28.786815643310547,
      "activations/layer22_attention_weight_min": -26.674482345581055,
      "activations/layer23_attention_weight_max": 35.25510025024414,
      "activations/layer23_attention_weight_min": -25.664703369140625,
      "activations/layer2_attention_weight_max": 31.013830184936523,
      "activations/layer2_attention_weight_min": -30.50592041015625,
      "activations/layer3_attention_weight_max": 89.512939453125,
      "activations/layer3_attention_weight_min": -88.15240478515625,
      "activations/layer4_attention_weight_max": 90.8495101928711,
      "activations/layer4_attention_weight_min": -87.75920104980469,
      "activations/layer5_attention_weight_max": 64.95608520507812,
      "activations/layer5_attention_weight_min": -73.23939514160156,
      "activations/layer6_attention_weight_max": 48.76692199707031,
      "activations/layer6_attention_weight_min": -50.03089141845703,
      "activations/layer7_attention_weight_max": 66.55220031738281,
      "activations/layer7_attention_weight_min": -61.110313415527344,
      "activations/layer8_attention_weight_max": 47.16490173339844,
      "activations/layer8_attention_weight_min": -48.8668327331543,
      "activations/layer9_attention_weight_max": 43.30275344848633,
      "activations/layer9_attention_weight_min": -46.0169792175293,
      "epoch": 16.62,
      "learning_rate": 4.3232196969696964e-05,
      "loss": 2.7556,
      "step": 286100
    },
    {
      "activations/layer0_attention_weight_max": 17.11560821533203,
      "activations/layer0_attention_weight_min": -13.698447227478027,
      "activations/layer10_attention_weight_max": 35.24818801879883,
      "activations/layer10_attention_weight_min": -34.7415885925293,
      "activations/layer11_attention_weight_max": 33.11335754394531,
      "activations/layer11_attention_weight_min": -33.657630920410156,
      "activations/layer12_attention_weight_max": 18.817447662353516,
      "activations/layer12_attention_weight_min": -23.318395614624023,
      "activations/layer13_attention_weight_max": 30.6741886138916,
      "activations/layer13_attention_weight_min": -29.759241104125977,
      "activations/layer14_attention_weight_max": 34.267486572265625,
      "activations/layer14_attention_weight_min": -30.200504302978516,
      "activations/layer15_attention_weight_max": 34.150691986083984,
      "activations/layer15_attention_weight_min": -29.974973678588867,
      "activations/layer16_attention_weight_max": 33.75058364868164,
      "activations/layer16_attention_weight_min": -31.96787452697754,
      "activations/layer17_attention_weight_max": 49.31581497192383,
      "activations/layer17_attention_weight_min": -44.25333023071289,
      "activations/layer18_attention_weight_max": 47.98627471923828,
      "activations/layer18_attention_weight_min": -38.21058654785156,
      "activations/layer19_attention_weight_max": 22.30037498474121,
      "activations/layer19_attention_weight_min": -23.088642120361328,
      "activations/layer1_attention_weight_max": 16.382917404174805,
      "activations/layer1_attention_weight_min": -14.829937934875488,
      "activations/layer20_attention_weight_max": 21.012657165527344,
      "activations/layer20_attention_weight_min": -22.573972702026367,
      "activations/layer21_attention_weight_max": 31.327665328979492,
      "activations/layer21_attention_weight_min": -21.84234619140625,
      "activations/layer22_attention_weight_max": 27.402376174926758,
      "activations/layer22_attention_weight_min": -27.10176658630371,
      "activations/layer23_attention_weight_max": 34.97364044189453,
      "activations/layer23_attention_weight_min": -26.640655517578125,
      "activations/layer2_attention_weight_max": 31.84272003173828,
      "activations/layer2_attention_weight_min": -29.86498260498047,
      "activations/layer3_attention_weight_max": 92.31056213378906,
      "activations/layer3_attention_weight_min": -89.7502212524414,
      "activations/layer4_attention_weight_max": 92.65316009521484,
      "activations/layer4_attention_weight_min": -88.03206634521484,
      "activations/layer5_attention_weight_max": 61.363677978515625,
      "activations/layer5_attention_weight_min": -74.63101959228516,
      "activations/layer6_attention_weight_max": 51.05469512939453,
      "activations/layer6_attention_weight_min": -51.76432418823242,
      "activations/layer7_attention_weight_max": 62.816314697265625,
      "activations/layer7_attention_weight_min": -65.86788177490234,
      "activations/layer8_attention_weight_max": 44.367069244384766,
      "activations/layer8_attention_weight_min": -47.30739212036133,
      "activations/layer9_attention_weight_max": 41.816646575927734,
      "activations/layer9_attention_weight_min": -45.30781936645508,
      "epoch": 16.63,
      "learning_rate": 4.321325757575758e-05,
      "loss": 2.747,
      "step": 286150
    },
    {
      "activations/layer0_attention_weight_max": 15.769500732421875,
      "activations/layer0_attention_weight_min": -13.743349075317383,
      "activations/layer10_attention_weight_max": 43.04996871948242,
      "activations/layer10_attention_weight_min": -40.6063232421875,
      "activations/layer11_attention_weight_max": 41.6773567199707,
      "activations/layer11_attention_weight_min": -38.13749694824219,
      "activations/layer12_attention_weight_max": 20.2873477935791,
      "activations/layer12_attention_weight_min": -25.24667739868164,
      "activations/layer13_attention_weight_max": 39.08483123779297,
      "activations/layer13_attention_weight_min": -27.31007194519043,
      "activations/layer14_attention_weight_max": 44.22981262207031,
      "activations/layer14_attention_weight_min": -31.729618072509766,
      "activations/layer15_attention_weight_max": 46.37751388549805,
      "activations/layer15_attention_weight_min": -31.380815505981445,
      "activations/layer16_attention_weight_max": 35.68476104736328,
      "activations/layer16_attention_weight_min": -33.701316833496094,
      "activations/layer17_attention_weight_max": 57.47726058959961,
      "activations/layer17_attention_weight_min": -50.45545959472656,
      "activations/layer18_attention_weight_max": 52.623931884765625,
      "activations/layer18_attention_weight_min": -45.569007873535156,
      "activations/layer19_attention_weight_max": 23.963024139404297,
      "activations/layer19_attention_weight_min": -24.18696403503418,
      "activations/layer1_attention_weight_max": 16.999130249023438,
      "activations/layer1_attention_weight_min": -15.113770484924316,
      "activations/layer20_attention_weight_max": 24.617687225341797,
      "activations/layer20_attention_weight_min": -20.697050094604492,
      "activations/layer21_attention_weight_max": 36.29497528076172,
      "activations/layer21_attention_weight_min": -24.76766014099121,
      "activations/layer22_attention_weight_max": 30.863937377929688,
      "activations/layer22_attention_weight_min": -26.32292366027832,
      "activations/layer23_attention_weight_max": 36.93333435058594,
      "activations/layer23_attention_weight_min": -24.10260581970215,
      "activations/layer2_attention_weight_max": 31.84020233154297,
      "activations/layer2_attention_weight_min": -31.711288452148438,
      "activations/layer3_attention_weight_max": 92.45652770996094,
      "activations/layer3_attention_weight_min": -91.0967025756836,
      "activations/layer4_attention_weight_max": 98.46922302246094,
      "activations/layer4_attention_weight_min": -94.48262023925781,
      "activations/layer5_attention_weight_max": 71.7371826171875,
      "activations/layer5_attention_weight_min": -70.44950866699219,
      "activations/layer6_attention_weight_max": 53.640750885009766,
      "activations/layer6_attention_weight_min": -53.23586654663086,
      "activations/layer7_attention_weight_max": 77.2110595703125,
      "activations/layer7_attention_weight_min": -68.9739761352539,
      "activations/layer8_attention_weight_max": 55.60869216918945,
      "activations/layer8_attention_weight_min": -51.05510330200195,
      "activations/layer9_attention_weight_max": 59.195655822753906,
      "activations/layer9_attention_weight_min": -53.38398361206055,
      "epoch": 16.63,
      "learning_rate": 4.319431818181818e-05,
      "loss": 2.747,
      "step": 286200
    },
    {
      "activations/layer0_attention_weight_max": 16.51807975769043,
      "activations/layer0_attention_weight_min": -13.94212532043457,
      "activations/layer10_attention_weight_max": 36.52511978149414,
      "activations/layer10_attention_weight_min": -35.42777633666992,
      "activations/layer11_attention_weight_max": 33.59336853027344,
      "activations/layer11_attention_weight_min": -32.68495178222656,
      "activations/layer12_attention_weight_max": 18.79849624633789,
      "activations/layer12_attention_weight_min": -25.905746459960938,
      "activations/layer13_attention_weight_max": 54.092166900634766,
      "activations/layer13_attention_weight_min": -33.35232162475586,
      "activations/layer14_attention_weight_max": 40.239219665527344,
      "activations/layer14_attention_weight_min": -33.45103073120117,
      "activations/layer15_attention_weight_max": 38.655025482177734,
      "activations/layer15_attention_weight_min": -31.608287811279297,
      "activations/layer16_attention_weight_max": 35.35318374633789,
      "activations/layer16_attention_weight_min": -32.78125,
      "activations/layer17_attention_weight_max": 52.507347106933594,
      "activations/layer17_attention_weight_min": -46.11246871948242,
      "activations/layer18_attention_weight_max": 50.99641036987305,
      "activations/layer18_attention_weight_min": -40.33355712890625,
      "activations/layer19_attention_weight_max": 24.980257034301758,
      "activations/layer19_attention_weight_min": -21.802370071411133,
      "activations/layer1_attention_weight_max": 16.086748123168945,
      "activations/layer1_attention_weight_min": -14.006922721862793,
      "activations/layer20_attention_weight_max": 24.921403884887695,
      "activations/layer20_attention_weight_min": -20.77179718017578,
      "activations/layer21_attention_weight_max": 38.79948425292969,
      "activations/layer21_attention_weight_min": -21.4998779296875,
      "activations/layer22_attention_weight_max": 31.27506446838379,
      "activations/layer22_attention_weight_min": -24.507831573486328,
      "activations/layer23_attention_weight_max": 35.97165298461914,
      "activations/layer23_attention_weight_min": -22.300243377685547,
      "activations/layer2_attention_weight_max": 31.87635612487793,
      "activations/layer2_attention_weight_min": -31.13243865966797,
      "activations/layer3_attention_weight_max": 86.38272857666016,
      "activations/layer3_attention_weight_min": -87.31085968017578,
      "activations/layer4_attention_weight_max": 86.88976287841797,
      "activations/layer4_attention_weight_min": -87.51197814941406,
      "activations/layer5_attention_weight_max": 64.14185333251953,
      "activations/layer5_attention_weight_min": -72.62146759033203,
      "activations/layer6_attention_weight_max": 48.4214973449707,
      "activations/layer6_attention_weight_min": -49.028953552246094,
      "activations/layer7_attention_weight_max": 61.38555145263672,
      "activations/layer7_attention_weight_min": -61.9539909362793,
      "activations/layer8_attention_weight_max": 42.17705154418945,
      "activations/layer8_attention_weight_min": -47.12015151977539,
      "activations/layer9_attention_weight_max": 43.510498046875,
      "activations/layer9_attention_weight_min": -44.81086349487305,
      "epoch": 16.63,
      "learning_rate": 4.317537878787878e-05,
      "loss": 2.7441,
      "step": 286250
    },
    {
      "activations/layer0_attention_weight_max": 16.027822494506836,
      "activations/layer0_attention_weight_min": -14.195259094238281,
      "activations/layer10_attention_weight_max": 36.51341247558594,
      "activations/layer10_attention_weight_min": -36.628536224365234,
      "activations/layer11_attention_weight_max": 33.83833312988281,
      "activations/layer11_attention_weight_min": -35.37218475341797,
      "activations/layer12_attention_weight_max": 18.974681854248047,
      "activations/layer12_attention_weight_min": -24.99578285217285,
      "activations/layer13_attention_weight_max": 40.482505798339844,
      "activations/layer13_attention_weight_min": -28.33264923095703,
      "activations/layer14_attention_weight_max": 36.95393371582031,
      "activations/layer14_attention_weight_min": -31.28185272216797,
      "activations/layer15_attention_weight_max": 35.29997634887695,
      "activations/layer15_attention_weight_min": -30.758224487304688,
      "activations/layer16_attention_weight_max": 34.61381149291992,
      "activations/layer16_attention_weight_min": -33.69661331176758,
      "activations/layer17_attention_weight_max": 51.10765075683594,
      "activations/layer17_attention_weight_min": -46.555999755859375,
      "activations/layer18_attention_weight_max": 49.200618743896484,
      "activations/layer18_attention_weight_min": -41.49560546875,
      "activations/layer19_attention_weight_max": 26.6435489654541,
      "activations/layer19_attention_weight_min": -23.18216896057129,
      "activations/layer1_attention_weight_max": 15.656578063964844,
      "activations/layer1_attention_weight_min": -15.962961196899414,
      "activations/layer20_attention_weight_max": 25.919639587402344,
      "activations/layer20_attention_weight_min": -22.38570785522461,
      "activations/layer21_attention_weight_max": 36.832645416259766,
      "activations/layer21_attention_weight_min": -24.073760986328125,
      "activations/layer22_attention_weight_max": 35.65982437133789,
      "activations/layer22_attention_weight_min": -26.941234588623047,
      "activations/layer23_attention_weight_max": 40.922752380371094,
      "activations/layer23_attention_weight_min": -23.593612670898438,
      "activations/layer2_attention_weight_max": 31.82685089111328,
      "activations/layer2_attention_weight_min": -30.545530319213867,
      "activations/layer3_attention_weight_max": 91.28553771972656,
      "activations/layer3_attention_weight_min": -88.15308380126953,
      "activations/layer4_attention_weight_max": 89.83753967285156,
      "activations/layer4_attention_weight_min": -86.55016326904297,
      "activations/layer5_attention_weight_max": 65.96492004394531,
      "activations/layer5_attention_weight_min": -75.97052001953125,
      "activations/layer6_attention_weight_max": 50.860084533691406,
      "activations/layer6_attention_weight_min": -51.72356033325195,
      "activations/layer7_attention_weight_max": 64.3315658569336,
      "activations/layer7_attention_weight_min": -61.41714859008789,
      "activations/layer8_attention_weight_max": 43.47868728637695,
      "activations/layer8_attention_weight_min": -47.178009033203125,
      "activations/layer9_attention_weight_max": 43.5439338684082,
      "activations/layer9_attention_weight_min": -47.72224807739258,
      "epoch": 16.64,
      "learning_rate": 4.315643939393939e-05,
      "loss": 2.7513,
      "step": 286300
    },
    {
      "activations/layer0_attention_weight_max": 15.795493125915527,
      "activations/layer0_attention_weight_min": -14.289512634277344,
      "activations/layer10_attention_weight_max": 37.0888671875,
      "activations/layer10_attention_weight_min": -36.068511962890625,
      "activations/layer11_attention_weight_max": 34.02863311767578,
      "activations/layer11_attention_weight_min": -33.46638488769531,
      "activations/layer12_attention_weight_max": 18.11348533630371,
      "activations/layer12_attention_weight_min": -23.92223358154297,
      "activations/layer13_attention_weight_max": 32.92083740234375,
      "activations/layer13_attention_weight_min": -26.39206886291504,
      "activations/layer14_attention_weight_max": 35.0722770690918,
      "activations/layer14_attention_weight_min": -31.99620819091797,
      "activations/layer15_attention_weight_max": 33.94935989379883,
      "activations/layer15_attention_weight_min": -30.07568359375,
      "activations/layer16_attention_weight_max": 34.5026741027832,
      "activations/layer16_attention_weight_min": -31.800899505615234,
      "activations/layer17_attention_weight_max": 49.360294342041016,
      "activations/layer17_attention_weight_min": -45.13822937011719,
      "activations/layer18_attention_weight_max": 47.95222854614258,
      "activations/layer18_attention_weight_min": -37.57695007324219,
      "activations/layer19_attention_weight_max": 22.34699249267578,
      "activations/layer19_attention_weight_min": -23.80678367614746,
      "activations/layer1_attention_weight_max": 15.858760833740234,
      "activations/layer1_attention_weight_min": -14.751769065856934,
      "activations/layer20_attention_weight_max": 21.880603790283203,
      "activations/layer20_attention_weight_min": -21.681228637695312,
      "activations/layer21_attention_weight_max": 34.60657501220703,
      "activations/layer21_attention_weight_min": -25.893108367919922,
      "activations/layer22_attention_weight_max": 27.218158721923828,
      "activations/layer22_attention_weight_min": -25.749286651611328,
      "activations/layer23_attention_weight_max": 36.15420150756836,
      "activations/layer23_attention_weight_min": -23.25115203857422,
      "activations/layer2_attention_weight_max": 32.9490966796875,
      "activations/layer2_attention_weight_min": -32.104549407958984,
      "activations/layer3_attention_weight_max": 94.32041931152344,
      "activations/layer3_attention_weight_min": -97.79290771484375,
      "activations/layer4_attention_weight_max": 95.9016342163086,
      "activations/layer4_attention_weight_min": -92.74166107177734,
      "activations/layer5_attention_weight_max": 67.058837890625,
      "activations/layer5_attention_weight_min": -75.15684509277344,
      "activations/layer6_attention_weight_max": 53.5582389831543,
      "activations/layer6_attention_weight_min": -51.699378967285156,
      "activations/layer7_attention_weight_max": 64.48262786865234,
      "activations/layer7_attention_weight_min": -67.44257354736328,
      "activations/layer8_attention_weight_max": 47.86643981933594,
      "activations/layer8_attention_weight_min": -48.16255569458008,
      "activations/layer9_attention_weight_max": 45.320045471191406,
      "activations/layer9_attention_weight_min": -46.21108627319336,
      "epoch": 16.64,
      "learning_rate": 4.31375e-05,
      "loss": 2.7584,
      "step": 286350
    },
    {
      "activations/layer0_attention_weight_max": 16.869731903076172,
      "activations/layer0_attention_weight_min": -13.839770317077637,
      "activations/layer10_attention_weight_max": 34.564876556396484,
      "activations/layer10_attention_weight_min": -36.57930374145508,
      "activations/layer11_attention_weight_max": 31.797779083251953,
      "activations/layer11_attention_weight_min": -34.577789306640625,
      "activations/layer12_attention_weight_max": 20.25051498413086,
      "activations/layer12_attention_weight_min": -27.404233932495117,
      "activations/layer13_attention_weight_max": 34.47637176513672,
      "activations/layer13_attention_weight_min": -29.321731567382812,
      "activations/layer14_attention_weight_max": 35.3145751953125,
      "activations/layer14_attention_weight_min": -30.40974998474121,
      "activations/layer15_attention_weight_max": 32.21584701538086,
      "activations/layer15_attention_weight_min": -30.68634605407715,
      "activations/layer16_attention_weight_max": 33.20082473754883,
      "activations/layer16_attention_weight_min": -33.089942932128906,
      "activations/layer17_attention_weight_max": 49.67197036743164,
      "activations/layer17_attention_weight_min": -46.85685729980469,
      "activations/layer18_attention_weight_max": 45.26387405395508,
      "activations/layer18_attention_weight_min": -40.481285095214844,
      "activations/layer19_attention_weight_max": 22.61601448059082,
      "activations/layer19_attention_weight_min": -22.282058715820312,
      "activations/layer1_attention_weight_max": 16.396699905395508,
      "activations/layer1_attention_weight_min": -13.743053436279297,
      "activations/layer20_attention_weight_max": 22.19750213623047,
      "activations/layer20_attention_weight_min": -20.556053161621094,
      "activations/layer21_attention_weight_max": 37.785545349121094,
      "activations/layer21_attention_weight_min": -23.513774871826172,
      "activations/layer22_attention_weight_max": 26.369232177734375,
      "activations/layer22_attention_weight_min": -25.286184310913086,
      "activations/layer23_attention_weight_max": 36.330543518066406,
      "activations/layer23_attention_weight_min": -24.4935359954834,
      "activations/layer2_attention_weight_max": 30.97504425048828,
      "activations/layer2_attention_weight_min": -31.59530258178711,
      "activations/layer3_attention_weight_max": 87.5654525756836,
      "activations/layer3_attention_weight_min": -96.59239959716797,
      "activations/layer4_attention_weight_max": 88.0260238647461,
      "activations/layer4_attention_weight_min": -91.02843475341797,
      "activations/layer5_attention_weight_max": 65.0979232788086,
      "activations/layer5_attention_weight_min": -77.36395263671875,
      "activations/layer6_attention_weight_max": 50.14717102050781,
      "activations/layer6_attention_weight_min": -51.44062423706055,
      "activations/layer7_attention_weight_max": 64.78052520751953,
      "activations/layer7_attention_weight_min": -67.15564727783203,
      "activations/layer8_attention_weight_max": 46.084259033203125,
      "activations/layer8_attention_weight_min": -48.78624725341797,
      "activations/layer9_attention_weight_max": 44.99850845336914,
      "activations/layer9_attention_weight_min": -46.08808517456055,
      "epoch": 16.64,
      "learning_rate": 4.3118560606060606e-05,
      "loss": 2.7407,
      "step": 286400
    },
    {
      "activations/layer0_attention_weight_max": 17.38159942626953,
      "activations/layer0_attention_weight_min": -14.084927558898926,
      "activations/layer10_attention_weight_max": 36.76839065551758,
      "activations/layer10_attention_weight_min": -35.671573638916016,
      "activations/layer11_attention_weight_max": 35.54401397705078,
      "activations/layer11_attention_weight_min": -33.563087463378906,
      "activations/layer12_attention_weight_max": 18.959623336791992,
      "activations/layer12_attention_weight_min": -24.3958740234375,
      "activations/layer13_attention_weight_max": 38.650020599365234,
      "activations/layer13_attention_weight_min": -28.105249404907227,
      "activations/layer14_attention_weight_max": 35.5532341003418,
      "activations/layer14_attention_weight_min": -32.00102615356445,
      "activations/layer15_attention_weight_max": 31.822011947631836,
      "activations/layer15_attention_weight_min": -31.357885360717773,
      "activations/layer16_attention_weight_max": 32.838600158691406,
      "activations/layer16_attention_weight_min": -30.80070686340332,
      "activations/layer17_attention_weight_max": 50.990509033203125,
      "activations/layer17_attention_weight_min": -43.99831008911133,
      "activations/layer18_attention_weight_max": 46.92742156982422,
      "activations/layer18_attention_weight_min": -38.9576301574707,
      "activations/layer19_attention_weight_max": 23.169790267944336,
      "activations/layer19_attention_weight_min": -22.745201110839844,
      "activations/layer1_attention_weight_max": 15.15765380859375,
      "activations/layer1_attention_weight_min": -14.388286590576172,
      "activations/layer20_attention_weight_max": 22.793441772460938,
      "activations/layer20_attention_weight_min": -22.52467155456543,
      "activations/layer21_attention_weight_max": 35.964820861816406,
      "activations/layer21_attention_weight_min": -23.891977310180664,
      "activations/layer22_attention_weight_max": 28.67984962463379,
      "activations/layer22_attention_weight_min": -26.63247299194336,
      "activations/layer23_attention_weight_max": 38.42121124267578,
      "activations/layer23_attention_weight_min": -25.840015411376953,
      "activations/layer2_attention_weight_max": 31.00208282470703,
      "activations/layer2_attention_weight_min": -30.605350494384766,
      "activations/layer3_attention_weight_max": 91.5263671875,
      "activations/layer3_attention_weight_min": -90.60393524169922,
      "activations/layer4_attention_weight_max": 90.23458099365234,
      "activations/layer4_attention_weight_min": -91.65941619873047,
      "activations/layer5_attention_weight_max": 64.18913269042969,
      "activations/layer5_attention_weight_min": -78.82391357421875,
      "activations/layer6_attention_weight_max": 52.22695541381836,
      "activations/layer6_attention_weight_min": -49.60607147216797,
      "activations/layer7_attention_weight_max": 67.95429992675781,
      "activations/layer7_attention_weight_min": -64.16084289550781,
      "activations/layer8_attention_weight_max": 47.08156204223633,
      "activations/layer8_attention_weight_min": -49.55385208129883,
      "activations/layer9_attention_weight_max": 45.851261138916016,
      "activations/layer9_attention_weight_min": -47.227970123291016,
      "epoch": 16.64,
      "learning_rate": 4.310037878787879e-05,
      "loss": 2.7567,
      "step": 286450
    },
    {
      "activations/layer0_attention_weight_max": 16.736831665039062,
      "activations/layer0_attention_weight_min": -14.065872192382812,
      "activations/layer10_attention_weight_max": 35.93084716796875,
      "activations/layer10_attention_weight_min": -35.25563049316406,
      "activations/layer11_attention_weight_max": 33.734161376953125,
      "activations/layer11_attention_weight_min": -32.39626693725586,
      "activations/layer12_attention_weight_max": 19.24535369873047,
      "activations/layer12_attention_weight_min": -22.96677589416504,
      "activations/layer13_attention_weight_max": 35.418174743652344,
      "activations/layer13_attention_weight_min": -27.20916175842285,
      "activations/layer14_attention_weight_max": 35.60471725463867,
      "activations/layer14_attention_weight_min": -30.724777221679688,
      "activations/layer15_attention_weight_max": 32.509525299072266,
      "activations/layer15_attention_weight_min": -28.81117820739746,
      "activations/layer16_attention_weight_max": 34.26020050048828,
      "activations/layer16_attention_weight_min": -31.636857986450195,
      "activations/layer17_attention_weight_max": 50.46483612060547,
      "activations/layer17_attention_weight_min": -44.33181381225586,
      "activations/layer18_attention_weight_max": 45.15885543823242,
      "activations/layer18_attention_weight_min": -38.07827377319336,
      "activations/layer19_attention_weight_max": 25.073152542114258,
      "activations/layer19_attention_weight_min": -22.31744956970215,
      "activations/layer1_attention_weight_max": 15.978819847106934,
      "activations/layer1_attention_weight_min": -14.289310455322266,
      "activations/layer20_attention_weight_max": 23.945085525512695,
      "activations/layer20_attention_weight_min": -23.803852081298828,
      "activations/layer21_attention_weight_max": 40.053306579589844,
      "activations/layer21_attention_weight_min": -24.566579818725586,
      "activations/layer22_attention_weight_max": 30.490983963012695,
      "activations/layer22_attention_weight_min": -24.42525863647461,
      "activations/layer23_attention_weight_max": 38.397369384765625,
      "activations/layer23_attention_weight_min": -27.585386276245117,
      "activations/layer2_attention_weight_max": 29.685216903686523,
      "activations/layer2_attention_weight_min": -30.099475860595703,
      "activations/layer3_attention_weight_max": 85.2792739868164,
      "activations/layer3_attention_weight_min": -85.70415496826172,
      "activations/layer4_attention_weight_max": 89.56288146972656,
      "activations/layer4_attention_weight_min": -87.01799774169922,
      "activations/layer5_attention_weight_max": 63.15864562988281,
      "activations/layer5_attention_weight_min": -78.74146270751953,
      "activations/layer6_attention_weight_max": 50.24834060668945,
      "activations/layer6_attention_weight_min": -51.40224075317383,
      "activations/layer7_attention_weight_max": 63.60659408569336,
      "activations/layer7_attention_weight_min": -62.127384185791016,
      "activations/layer8_attention_weight_max": 48.19137191772461,
      "activations/layer8_attention_weight_min": -46.80908203125,
      "activations/layer9_attention_weight_max": 42.87765884399414,
      "activations/layer9_attention_weight_min": -42.39087677001953,
      "epoch": 16.65,
      "learning_rate": 4.308143939393939e-05,
      "loss": 2.7438,
      "step": 286500
    },
    {
      "activations/layer0_attention_weight_max": 15.299674034118652,
      "activations/layer0_attention_weight_min": -13.46739673614502,
      "activations/layer10_attention_weight_max": 35.70716857910156,
      "activations/layer10_attention_weight_min": -35.15182113647461,
      "activations/layer11_attention_weight_max": 31.42132568359375,
      "activations/layer11_attention_weight_min": -35.957576751708984,
      "activations/layer12_attention_weight_max": 18.691421508789062,
      "activations/layer12_attention_weight_min": -27.111412048339844,
      "activations/layer13_attention_weight_max": 36.21721649169922,
      "activations/layer13_attention_weight_min": -27.90065574645996,
      "activations/layer14_attention_weight_max": 39.8943977355957,
      "activations/layer14_attention_weight_min": -30.787097930908203,
      "activations/layer15_attention_weight_max": 37.1407470703125,
      "activations/layer15_attention_weight_min": -29.412311553955078,
      "activations/layer16_attention_weight_max": 33.8236198425293,
      "activations/layer16_attention_weight_min": -35.10662841796875,
      "activations/layer17_attention_weight_max": 52.03961181640625,
      "activations/layer17_attention_weight_min": -48.33302307128906,
      "activations/layer18_attention_weight_max": 45.91469955444336,
      "activations/layer18_attention_weight_min": -44.00270462036133,
      "activations/layer19_attention_weight_max": 24.72117042541504,
      "activations/layer19_attention_weight_min": -26.442520141601562,
      "activations/layer1_attention_weight_max": 14.92908763885498,
      "activations/layer1_attention_weight_min": -13.758285522460938,
      "activations/layer20_attention_weight_max": 27.088542938232422,
      "activations/layer20_attention_weight_min": -23.81018829345703,
      "activations/layer21_attention_weight_max": 43.98184585571289,
      "activations/layer21_attention_weight_min": -22.650827407836914,
      "activations/layer22_attention_weight_max": 34.22461700439453,
      "activations/layer22_attention_weight_min": -27.78575325012207,
      "activations/layer23_attention_weight_max": 38.38092041015625,
      "activations/layer23_attention_weight_min": -27.586767196655273,
      "activations/layer2_attention_weight_max": 30.965007781982422,
      "activations/layer2_attention_weight_min": -30.42510986328125,
      "activations/layer3_attention_weight_max": 84.48579406738281,
      "activations/layer3_attention_weight_min": -91.17858123779297,
      "activations/layer4_attention_weight_max": 88.6103286743164,
      "activations/layer4_attention_weight_min": -89.94184112548828,
      "activations/layer5_attention_weight_max": 61.059242248535156,
      "activations/layer5_attention_weight_min": -71.90841674804688,
      "activations/layer6_attention_weight_max": 49.229103088378906,
      "activations/layer6_attention_weight_min": -47.31174087524414,
      "activations/layer7_attention_weight_max": 62.372108459472656,
      "activations/layer7_attention_weight_min": -60.609375,
      "activations/layer8_attention_weight_max": 43.37351989746094,
      "activations/layer8_attention_weight_min": -47.38029479980469,
      "activations/layer9_attention_weight_max": 42.260650634765625,
      "activations/layer9_attention_weight_min": -43.29962158203125,
      "epoch": 16.65,
      "learning_rate": 4.3062499999999996e-05,
      "loss": 2.7497,
      "step": 286550
    },
    {
      "activations/layer0_attention_weight_max": 15.675148963928223,
      "activations/layer0_attention_weight_min": -13.961517333984375,
      "activations/layer10_attention_weight_max": 35.56831741333008,
      "activations/layer10_attention_weight_min": -34.828269958496094,
      "activations/layer11_attention_weight_max": 33.327510833740234,
      "activations/layer11_attention_weight_min": -32.222572326660156,
      "activations/layer12_attention_weight_max": 19.514564514160156,
      "activations/layer12_attention_weight_min": -28.57647132873535,
      "activations/layer13_attention_weight_max": 40.18858337402344,
      "activations/layer13_attention_weight_min": -31.02536964416504,
      "activations/layer14_attention_weight_max": 40.63432312011719,
      "activations/layer14_attention_weight_min": -34.03682327270508,
      "activations/layer15_attention_weight_max": 42.165122985839844,
      "activations/layer15_attention_weight_min": -31.65933609008789,
      "activations/layer16_attention_weight_max": 36.5894775390625,
      "activations/layer16_attention_weight_min": -31.842784881591797,
      "activations/layer17_attention_weight_max": 56.65141296386719,
      "activations/layer17_attention_weight_min": -48.28956985473633,
      "activations/layer18_attention_weight_max": 54.416717529296875,
      "activations/layer18_attention_weight_min": -42.12372589111328,
      "activations/layer19_attention_weight_max": 29.244461059570312,
      "activations/layer19_attention_weight_min": -24.58821678161621,
      "activations/layer1_attention_weight_max": 16.04726791381836,
      "activations/layer1_attention_weight_min": -15.658754348754883,
      "activations/layer20_attention_weight_max": 31.1031494140625,
      "activations/layer20_attention_weight_min": -24.162059783935547,
      "activations/layer21_attention_weight_max": 50.75079345703125,
      "activations/layer21_attention_weight_min": -26.92052459716797,
      "activations/layer22_attention_weight_max": 33.224422454833984,
      "activations/layer22_attention_weight_min": -30.4982967376709,
      "activations/layer23_attention_weight_max": 41.717952728271484,
      "activations/layer23_attention_weight_min": -24.461606979370117,
      "activations/layer2_attention_weight_max": 30.873443603515625,
      "activations/layer2_attention_weight_min": -30.41957664489746,
      "activations/layer3_attention_weight_max": 87.12911987304688,
      "activations/layer3_attention_weight_min": -88.57546997070312,
      "activations/layer4_attention_weight_max": 88.76577758789062,
      "activations/layer4_attention_weight_min": -86.07733917236328,
      "activations/layer5_attention_weight_max": 65.59803009033203,
      "activations/layer5_attention_weight_min": -75.44973754882812,
      "activations/layer6_attention_weight_max": 49.538841247558594,
      "activations/layer6_attention_weight_min": -50.78670120239258,
      "activations/layer7_attention_weight_max": 68.48065948486328,
      "activations/layer7_attention_weight_min": -65.72350311279297,
      "activations/layer8_attention_weight_max": 48.316184997558594,
      "activations/layer8_attention_weight_min": -49.76088333129883,
      "activations/layer9_attention_weight_max": 53.531280517578125,
      "activations/layer9_attention_weight_min": -44.38191604614258,
      "epoch": 16.65,
      "learning_rate": 4.3043560606060605e-05,
      "loss": 2.7411,
      "step": 286600
    },
    {
      "activations/layer0_attention_weight_max": 16.792640686035156,
      "activations/layer0_attention_weight_min": -13.731718063354492,
      "activations/layer10_attention_weight_max": 33.503692626953125,
      "activations/layer10_attention_weight_min": -34.39772415161133,
      "activations/layer11_attention_weight_max": 29.195186614990234,
      "activations/layer11_attention_weight_min": -32.37559509277344,
      "activations/layer12_attention_weight_max": 19.25986099243164,
      "activations/layer12_attention_weight_min": -27.043167114257812,
      "activations/layer13_attention_weight_max": 31.551088333129883,
      "activations/layer13_attention_weight_min": -25.72127914428711,
      "activations/layer14_attention_weight_max": 34.14994812011719,
      "activations/layer14_attention_weight_min": -30.36070442199707,
      "activations/layer15_attention_weight_max": 34.24909973144531,
      "activations/layer15_attention_weight_min": -28.428781509399414,
      "activations/layer16_attention_weight_max": 33.20942306518555,
      "activations/layer16_attention_weight_min": -29.62381362915039,
      "activations/layer17_attention_weight_max": 52.01611328125,
      "activations/layer17_attention_weight_min": -43.76777648925781,
      "activations/layer18_attention_weight_max": 48.24147415161133,
      "activations/layer18_attention_weight_min": -39.42268371582031,
      "activations/layer19_attention_weight_max": 23.256084442138672,
      "activations/layer19_attention_weight_min": -25.152599334716797,
      "activations/layer1_attention_weight_max": 17.30989646911621,
      "activations/layer1_attention_weight_min": -14.063551902770996,
      "activations/layer20_attention_weight_max": 22.396018981933594,
      "activations/layer20_attention_weight_min": -22.87699317932129,
      "activations/layer21_attention_weight_max": 38.11508560180664,
      "activations/layer21_attention_weight_min": -24.193344116210938,
      "activations/layer22_attention_weight_max": 30.831663131713867,
      "activations/layer22_attention_weight_min": -26.514692306518555,
      "activations/layer23_attention_weight_max": 38.73981475830078,
      "activations/layer23_attention_weight_min": -25.31587028503418,
      "activations/layer2_attention_weight_max": 31.632795333862305,
      "activations/layer2_attention_weight_min": -29.981786727905273,
      "activations/layer3_attention_weight_max": 86.07186889648438,
      "activations/layer3_attention_weight_min": -84.86808776855469,
      "activations/layer4_attention_weight_max": 87.5954818725586,
      "activations/layer4_attention_weight_min": -86.44332885742188,
      "activations/layer5_attention_weight_max": 61.79624557495117,
      "activations/layer5_attention_weight_min": -71.50325775146484,
      "activations/layer6_attention_weight_max": 48.48667526245117,
      "activations/layer6_attention_weight_min": -50.24692916870117,
      "activations/layer7_attention_weight_max": 62.55230712890625,
      "activations/layer7_attention_weight_min": -63.79287338256836,
      "activations/layer8_attention_weight_max": 44.482295989990234,
      "activations/layer8_attention_weight_min": -49.056243896484375,
      "activations/layer9_attention_weight_max": 41.559574127197266,
      "activations/layer9_attention_weight_min": -45.01448440551758,
      "epoch": 16.66,
      "learning_rate": 4.3024621212121206e-05,
      "loss": 2.7348,
      "step": 286650
    },
    {
      "activations/layer0_attention_weight_max": 16.824068069458008,
      "activations/layer0_attention_weight_min": -13.641204833984375,
      "activations/layer10_attention_weight_max": 38.37251663208008,
      "activations/layer10_attention_weight_min": -37.60687255859375,
      "activations/layer11_attention_weight_max": 34.684932708740234,
      "activations/layer11_attention_weight_min": -34.45277404785156,
      "activations/layer12_attention_weight_max": 19.12455940246582,
      "activations/layer12_attention_weight_min": -23.784534454345703,
      "activations/layer13_attention_weight_max": 34.876502990722656,
      "activations/layer13_attention_weight_min": -26.802148818969727,
      "activations/layer14_attention_weight_max": 35.072364807128906,
      "activations/layer14_attention_weight_min": -30.0925235748291,
      "activations/layer15_attention_weight_max": 33.71640396118164,
      "activations/layer15_attention_weight_min": -32.09284591674805,
      "activations/layer16_attention_weight_max": 34.526939392089844,
      "activations/layer16_attention_weight_min": -32.49403762817383,
      "activations/layer17_attention_weight_max": 54.55508804321289,
      "activations/layer17_attention_weight_min": -49.21319580078125,
      "activations/layer18_attention_weight_max": 46.85901641845703,
      "activations/layer18_attention_weight_min": -40.0759391784668,
      "activations/layer19_attention_weight_max": 28.104543685913086,
      "activations/layer19_attention_weight_min": -21.848295211791992,
      "activations/layer1_attention_weight_max": 16.922714233398438,
      "activations/layer1_attention_weight_min": -12.712618827819824,
      "activations/layer20_attention_weight_max": 25.242204666137695,
      "activations/layer20_attention_weight_min": -21.406448364257812,
      "activations/layer21_attention_weight_max": 36.6441535949707,
      "activations/layer21_attention_weight_min": -25.391216278076172,
      "activations/layer22_attention_weight_max": 29.0186767578125,
      "activations/layer22_attention_weight_min": -25.586097717285156,
      "activations/layer23_attention_weight_max": 37.064735412597656,
      "activations/layer23_attention_weight_min": -23.34712791442871,
      "activations/layer2_attention_weight_max": 32.627410888671875,
      "activations/layer2_attention_weight_min": -29.056171417236328,
      "activations/layer3_attention_weight_max": 88.1609115600586,
      "activations/layer3_attention_weight_min": -88.5122299194336,
      "activations/layer4_attention_weight_max": 93.18455505371094,
      "activations/layer4_attention_weight_min": -88.6722412109375,
      "activations/layer5_attention_weight_max": 67.41123962402344,
      "activations/layer5_attention_weight_min": -76.99620056152344,
      "activations/layer6_attention_weight_max": 49.52134323120117,
      "activations/layer6_attention_weight_min": -51.56876754760742,
      "activations/layer7_attention_weight_max": 70.88118743896484,
      "activations/layer7_attention_weight_min": -62.5169792175293,
      "activations/layer8_attention_weight_max": 47.584808349609375,
      "activations/layer8_attention_weight_min": -52.21819305419922,
      "activations/layer9_attention_weight_max": 44.264591217041016,
      "activations/layer9_attention_weight_min": -45.897945404052734,
      "epoch": 16.66,
      "learning_rate": 4.3005681818181814e-05,
      "loss": 2.7488,
      "step": 286700
    },
    {
      "activations/layer0_attention_weight_max": 17.625125885009766,
      "activations/layer0_attention_weight_min": -13.736263275146484,
      "activations/layer10_attention_weight_max": 34.9307861328125,
      "activations/layer10_attention_weight_min": -34.07801055908203,
      "activations/layer11_attention_weight_max": 31.825469970703125,
      "activations/layer11_attention_weight_min": -33.161956787109375,
      "activations/layer12_attention_weight_max": 21.777359008789062,
      "activations/layer12_attention_weight_min": -21.810016632080078,
      "activations/layer13_attention_weight_max": 37.505516052246094,
      "activations/layer13_attention_weight_min": -27.597049713134766,
      "activations/layer14_attention_weight_max": 35.70117950439453,
      "activations/layer14_attention_weight_min": -30.815732955932617,
      "activations/layer15_attention_weight_max": 34.37967300415039,
      "activations/layer15_attention_weight_min": -30.634105682373047,
      "activations/layer16_attention_weight_max": 36.63228988647461,
      "activations/layer16_attention_weight_min": -32.819358825683594,
      "activations/layer17_attention_weight_max": 55.18866729736328,
      "activations/layer17_attention_weight_min": -47.326629638671875,
      "activations/layer18_attention_weight_max": 56.37214660644531,
      "activations/layer18_attention_weight_min": -45.14362716674805,
      "activations/layer19_attention_weight_max": 27.51206398010254,
      "activations/layer19_attention_weight_min": -25.612808227539062,
      "activations/layer1_attention_weight_max": 16.391000747680664,
      "activations/layer1_attention_weight_min": -12.704110145568848,
      "activations/layer20_attention_weight_max": 24.757823944091797,
      "activations/layer20_attention_weight_min": -21.25136947631836,
      "activations/layer21_attention_weight_max": 42.554630279541016,
      "activations/layer21_attention_weight_min": -23.621707916259766,
      "activations/layer22_attention_weight_max": 28.953767776489258,
      "activations/layer22_attention_weight_min": -24.09521484375,
      "activations/layer23_attention_weight_max": 42.62189483642578,
      "activations/layer23_attention_weight_min": -25.106597900390625,
      "activations/layer2_attention_weight_max": 29.539459228515625,
      "activations/layer2_attention_weight_min": -28.510250091552734,
      "activations/layer3_attention_weight_max": 83.04476928710938,
      "activations/layer3_attention_weight_min": -84.09530639648438,
      "activations/layer4_attention_weight_max": 86.73052215576172,
      "activations/layer4_attention_weight_min": -83.65538787841797,
      "activations/layer5_attention_weight_max": 63.86399841308594,
      "activations/layer5_attention_weight_min": -71.04946899414062,
      "activations/layer6_attention_weight_max": 46.88016128540039,
      "activations/layer6_attention_weight_min": -49.05772399902344,
      "activations/layer7_attention_weight_max": 64.595947265625,
      "activations/layer7_attention_weight_min": -64.62197875976562,
      "activations/layer8_attention_weight_max": 42.196311950683594,
      "activations/layer8_attention_weight_min": -44.26171112060547,
      "activations/layer9_attention_weight_max": 44.77944564819336,
      "activations/layer9_attention_weight_min": -46.8169059753418,
      "epoch": 16.66,
      "learning_rate": 4.298674242424242e-05,
      "loss": 2.7363,
      "step": 286750
    },
    {
      "activations/layer0_attention_weight_max": 17.21575164794922,
      "activations/layer0_attention_weight_min": -13.755410194396973,
      "activations/layer10_attention_weight_max": 38.34907913208008,
      "activations/layer10_attention_weight_min": -38.094703674316406,
      "activations/layer11_attention_weight_max": 36.98419952392578,
      "activations/layer11_attention_weight_min": -36.01321792602539,
      "activations/layer12_attention_weight_max": 20.395036697387695,
      "activations/layer12_attention_weight_min": -24.26430892944336,
      "activations/layer13_attention_weight_max": 32.32572555541992,
      "activations/layer13_attention_weight_min": -26.603816986083984,
      "activations/layer14_attention_weight_max": 35.42313766479492,
      "activations/layer14_attention_weight_min": -28.921913146972656,
      "activations/layer15_attention_weight_max": 30.88957977294922,
      "activations/layer15_attention_weight_min": -30.43498992919922,
      "activations/layer16_attention_weight_max": 32.426090240478516,
      "activations/layer16_attention_weight_min": -30.525178909301758,
      "activations/layer17_attention_weight_max": 48.58280944824219,
      "activations/layer17_attention_weight_min": -42.4197998046875,
      "activations/layer18_attention_weight_max": 44.53911590576172,
      "activations/layer18_attention_weight_min": -36.61025619506836,
      "activations/layer19_attention_weight_max": 22.841552734375,
      "activations/layer19_attention_weight_min": -22.52739906311035,
      "activations/layer1_attention_weight_max": 15.675406455993652,
      "activations/layer1_attention_weight_min": -14.045071601867676,
      "activations/layer20_attention_weight_max": 23.089981079101562,
      "activations/layer20_attention_weight_min": -23.546831130981445,
      "activations/layer21_attention_weight_max": 31.95155906677246,
      "activations/layer21_attention_weight_min": -26.182687759399414,
      "activations/layer22_attention_weight_max": 32.4305305480957,
      "activations/layer22_attention_weight_min": -25.945581436157227,
      "activations/layer23_attention_weight_max": 35.51284408569336,
      "activations/layer23_attention_weight_min": -25.310691833496094,
      "activations/layer2_attention_weight_max": 31.542219161987305,
      "activations/layer2_attention_weight_min": -29.272375106811523,
      "activations/layer3_attention_weight_max": 86.8133316040039,
      "activations/layer3_attention_weight_min": -90.90702819824219,
      "activations/layer4_attention_weight_max": 89.9419174194336,
      "activations/layer4_attention_weight_min": -90.14752197265625,
      "activations/layer5_attention_weight_max": 66.36369323730469,
      "activations/layer5_attention_weight_min": -75.51885223388672,
      "activations/layer6_attention_weight_max": 50.79555892944336,
      "activations/layer6_attention_weight_min": -52.670005798339844,
      "activations/layer7_attention_weight_max": 67.73954772949219,
      "activations/layer7_attention_weight_min": -65.5821533203125,
      "activations/layer8_attention_weight_max": 47.67950439453125,
      "activations/layer8_attention_weight_min": -51.284423828125,
      "activations/layer9_attention_weight_max": 49.03681945800781,
      "activations/layer9_attention_weight_min": -49.154754638671875,
      "epoch": 16.66,
      "learning_rate": 4.2967803030303024e-05,
      "loss": 2.748,
      "step": 286800
    },
    {
      "activations/layer0_attention_weight_max": 16.828800201416016,
      "activations/layer0_attention_weight_min": -13.650126457214355,
      "activations/layer10_attention_weight_max": 42.07740783691406,
      "activations/layer10_attention_weight_min": -38.72944641113281,
      "activations/layer11_attention_weight_max": 35.38668441772461,
      "activations/layer11_attention_weight_min": -34.65298080444336,
      "activations/layer12_attention_weight_max": 19.219959259033203,
      "activations/layer12_attention_weight_min": -26.83255958557129,
      "activations/layer13_attention_weight_max": 35.886375427246094,
      "activations/layer13_attention_weight_min": -32.6981315612793,
      "activations/layer14_attention_weight_max": 34.75130844116211,
      "activations/layer14_attention_weight_min": -30.93063735961914,
      "activations/layer15_attention_weight_max": 32.403926849365234,
      "activations/layer15_attention_weight_min": -31.056156158447266,
      "activations/layer16_attention_weight_max": 34.18478012084961,
      "activations/layer16_attention_weight_min": -31.8818359375,
      "activations/layer17_attention_weight_max": 50.50330352783203,
      "activations/layer17_attention_weight_min": -47.09785842895508,
      "activations/layer18_attention_weight_max": 44.34304428100586,
      "activations/layer18_attention_weight_min": -41.16603469848633,
      "activations/layer19_attention_weight_max": 24.746070861816406,
      "activations/layer19_attention_weight_min": -21.833112716674805,
      "activations/layer1_attention_weight_max": 15.539891242980957,
      "activations/layer1_attention_weight_min": -13.337560653686523,
      "activations/layer20_attention_weight_max": 22.564786911010742,
      "activations/layer20_attention_weight_min": -20.722686767578125,
      "activations/layer21_attention_weight_max": 34.62514877319336,
      "activations/layer21_attention_weight_min": -23.365814208984375,
      "activations/layer22_attention_weight_max": 30.81191635131836,
      "activations/layer22_attention_weight_min": -25.15554428100586,
      "activations/layer23_attention_weight_max": 41.47037124633789,
      "activations/layer23_attention_weight_min": -23.601408004760742,
      "activations/layer2_attention_weight_max": 31.60004234313965,
      "activations/layer2_attention_weight_min": -28.625370025634766,
      "activations/layer3_attention_weight_max": 90.36524200439453,
      "activations/layer3_attention_weight_min": -89.37760162353516,
      "activations/layer4_attention_weight_max": 91.1236343383789,
      "activations/layer4_attention_weight_min": -90.70934295654297,
      "activations/layer5_attention_weight_max": 62.27520751953125,
      "activations/layer5_attention_weight_min": -70.13545227050781,
      "activations/layer6_attention_weight_max": 51.350215911865234,
      "activations/layer6_attention_weight_min": -50.025508880615234,
      "activations/layer7_attention_weight_max": 63.997920989990234,
      "activations/layer7_attention_weight_min": -63.82185363769531,
      "activations/layer8_attention_weight_max": 51.064151763916016,
      "activations/layer8_attention_weight_min": -51.75908279418945,
      "activations/layer9_attention_weight_max": 47.62397766113281,
      "activations/layer9_attention_weight_min": -47.37421798706055,
      "epoch": 16.67,
      "learning_rate": 4.294886363636363e-05,
      "loss": 2.7315,
      "step": 286850
    },
    {
      "activations/layer0_attention_weight_max": 16.02484703063965,
      "activations/layer0_attention_weight_min": -13.635554313659668,
      "activations/layer10_attention_weight_max": 35.5445556640625,
      "activations/layer10_attention_weight_min": -33.95220184326172,
      "activations/layer11_attention_weight_max": 33.773895263671875,
      "activations/layer11_attention_weight_min": -32.15167999267578,
      "activations/layer12_attention_weight_max": 18.690589904785156,
      "activations/layer12_attention_weight_min": -24.155059814453125,
      "activations/layer13_attention_weight_max": 33.376583099365234,
      "activations/layer13_attention_weight_min": -27.04946517944336,
      "activations/layer14_attention_weight_max": 34.143917083740234,
      "activations/layer14_attention_weight_min": -28.673398971557617,
      "activations/layer15_attention_weight_max": 33.111568450927734,
      "activations/layer15_attention_weight_min": -27.262218475341797,
      "activations/layer16_attention_weight_max": 30.097490310668945,
      "activations/layer16_attention_weight_min": -29.348459243774414,
      "activations/layer17_attention_weight_max": 47.704463958740234,
      "activations/layer17_attention_weight_min": -43.55666732788086,
      "activations/layer18_attention_weight_max": 44.39896011352539,
      "activations/layer18_attention_weight_min": -38.44770812988281,
      "activations/layer19_attention_weight_max": 23.174530029296875,
      "activations/layer19_attention_weight_min": -22.277347564697266,
      "activations/layer1_attention_weight_max": 16.013208389282227,
      "activations/layer1_attention_weight_min": -12.677945137023926,
      "activations/layer20_attention_weight_max": 22.366840362548828,
      "activations/layer20_attention_weight_min": -21.59008026123047,
      "activations/layer21_attention_weight_max": 33.17366027832031,
      "activations/layer21_attention_weight_min": -21.140928268432617,
      "activations/layer22_attention_weight_max": 30.919416427612305,
      "activations/layer22_attention_weight_min": -24.67462921142578,
      "activations/layer23_attention_weight_max": 39.3670539855957,
      "activations/layer23_attention_weight_min": -23.476573944091797,
      "activations/layer2_attention_weight_max": 31.45064926147461,
      "activations/layer2_attention_weight_min": -29.13611602783203,
      "activations/layer3_attention_weight_max": 88.69760131835938,
      "activations/layer3_attention_weight_min": -93.61885070800781,
      "activations/layer4_attention_weight_max": 89.97665405273438,
      "activations/layer4_attention_weight_min": -93.7431411743164,
      "activations/layer5_attention_weight_max": 64.6376724243164,
      "activations/layer5_attention_weight_min": -76.2125473022461,
      "activations/layer6_attention_weight_max": 50.994571685791016,
      "activations/layer6_attention_weight_min": -51.170692443847656,
      "activations/layer7_attention_weight_max": 61.326263427734375,
      "activations/layer7_attention_weight_min": -61.17212677001953,
      "activations/layer8_attention_weight_max": 44.6268310546875,
      "activations/layer8_attention_weight_min": -50.0574836730957,
      "activations/layer9_attention_weight_max": 42.4581184387207,
      "activations/layer9_attention_weight_min": -45.219356536865234,
      "epoch": 16.67,
      "learning_rate": 4.2929924242424234e-05,
      "loss": 2.7357,
      "step": 286900
    },
    {
      "activations/layer0_attention_weight_max": 15.571287155151367,
      "activations/layer0_attention_weight_min": -13.438986778259277,
      "activations/layer10_attention_weight_max": 36.53105545043945,
      "activations/layer10_attention_weight_min": -35.55570602416992,
      "activations/layer11_attention_weight_max": 32.496673583984375,
      "activations/layer11_attention_weight_min": -33.15874099731445,
      "activations/layer12_attention_weight_max": 18.352365493774414,
      "activations/layer12_attention_weight_min": -25.01296615600586,
      "activations/layer13_attention_weight_max": 38.74134826660156,
      "activations/layer13_attention_weight_min": -29.949752807617188,
      "activations/layer14_attention_weight_max": 38.76250076293945,
      "activations/layer14_attention_weight_min": -31.703304290771484,
      "activations/layer15_attention_weight_max": 34.366817474365234,
      "activations/layer15_attention_weight_min": -31.527671813964844,
      "activations/layer16_attention_weight_max": 33.630455017089844,
      "activations/layer16_attention_weight_min": -32.186279296875,
      "activations/layer17_attention_weight_max": 51.87636184692383,
      "activations/layer17_attention_weight_min": -44.91012191772461,
      "activations/layer18_attention_weight_max": 49.43733596801758,
      "activations/layer18_attention_weight_min": -39.86684036254883,
      "activations/layer19_attention_weight_max": 26.247894287109375,
      "activations/layer19_attention_weight_min": -22.434341430664062,
      "activations/layer1_attention_weight_max": 15.557295799255371,
      "activations/layer1_attention_weight_min": -14.125286102294922,
      "activations/layer20_attention_weight_max": 28.422216415405273,
      "activations/layer20_attention_weight_min": -24.2367000579834,
      "activations/layer21_attention_weight_max": 42.60946273803711,
      "activations/layer21_attention_weight_min": -22.326223373413086,
      "activations/layer22_attention_weight_max": 33.52132797241211,
      "activations/layer22_attention_weight_min": -25.304059982299805,
      "activations/layer23_attention_weight_max": 38.40098190307617,
      "activations/layer23_attention_weight_min": -23.947477340698242,
      "activations/layer2_attention_weight_max": 31.531352996826172,
      "activations/layer2_attention_weight_min": -30.121076583862305,
      "activations/layer3_attention_weight_max": 89.88751220703125,
      "activations/layer3_attention_weight_min": -91.78802490234375,
      "activations/layer4_attention_weight_max": 89.43695831298828,
      "activations/layer4_attention_weight_min": -89.47352600097656,
      "activations/layer5_attention_weight_max": 66.31649780273438,
      "activations/layer5_attention_weight_min": -75.62046813964844,
      "activations/layer6_attention_weight_max": 49.54716491699219,
      "activations/layer6_attention_weight_min": -51.46347427368164,
      "activations/layer7_attention_weight_max": 64.30329132080078,
      "activations/layer7_attention_weight_min": -60.598018646240234,
      "activations/layer8_attention_weight_max": 47.7325325012207,
      "activations/layer8_attention_weight_min": -52.459999084472656,
      "activations/layer9_attention_weight_max": 48.59458541870117,
      "activations/layer9_attention_weight_min": -45.907772064208984,
      "epoch": 16.67,
      "learning_rate": 4.291098484848485e-05,
      "loss": 2.7274,
      "step": 286950
    },
    {
      "activations/layer0_attention_weight_max": 15.609130859375,
      "activations/layer0_attention_weight_min": -13.818208694458008,
      "activations/layer10_attention_weight_max": 37.66414260864258,
      "activations/layer10_attention_weight_min": -36.237648010253906,
      "activations/layer11_attention_weight_max": 33.99565505981445,
      "activations/layer11_attention_weight_min": -36.12552261352539,
      "activations/layer12_attention_weight_max": 19.390226364135742,
      "activations/layer12_attention_weight_min": -23.969348907470703,
      "activations/layer13_attention_weight_max": 36.002235412597656,
      "activations/layer13_attention_weight_min": -27.907995223999023,
      "activations/layer14_attention_weight_max": 36.88603973388672,
      "activations/layer14_attention_weight_min": -30.23733901977539,
      "activations/layer15_attention_weight_max": 32.23807907104492,
      "activations/layer15_attention_weight_min": -29.560155868530273,
      "activations/layer16_attention_weight_max": 32.11254119873047,
      "activations/layer16_attention_weight_min": -30.35011863708496,
      "activations/layer17_attention_weight_max": 50.37759017944336,
      "activations/layer17_attention_weight_min": -44.12248611450195,
      "activations/layer18_attention_weight_max": 43.93000030517578,
      "activations/layer18_attention_weight_min": -40.649837493896484,
      "activations/layer19_attention_weight_max": 23.067962646484375,
      "activations/layer19_attention_weight_min": -21.587299346923828,
      "activations/layer1_attention_weight_max": 16.4918155670166,
      "activations/layer1_attention_weight_min": -13.752348899841309,
      "activations/layer20_attention_weight_max": 20.851552963256836,
      "activations/layer20_attention_weight_min": -23.042598724365234,
      "activations/layer21_attention_weight_max": 31.250591278076172,
      "activations/layer21_attention_weight_min": -22.720151901245117,
      "activations/layer22_attention_weight_max": 29.93092918395996,
      "activations/layer22_attention_weight_min": -25.79362678527832,
      "activations/layer23_attention_weight_max": 37.62834167480469,
      "activations/layer23_attention_weight_min": -26.041141510009766,
      "activations/layer2_attention_weight_max": 31.304744720458984,
      "activations/layer2_attention_weight_min": -30.224294662475586,
      "activations/layer3_attention_weight_max": 89.74464416503906,
      "activations/layer3_attention_weight_min": -90.50812530517578,
      "activations/layer4_attention_weight_max": 89.65913391113281,
      "activations/layer4_attention_weight_min": -90.59070587158203,
      "activations/layer5_attention_weight_max": 64.41435241699219,
      "activations/layer5_attention_weight_min": -76.1069564819336,
      "activations/layer6_attention_weight_max": 49.9087028503418,
      "activations/layer6_attention_weight_min": -48.936588287353516,
      "activations/layer7_attention_weight_max": 61.78483200073242,
      "activations/layer7_attention_weight_min": -63.79267501831055,
      "activations/layer8_attention_weight_max": 45.74092483520508,
      "activations/layer8_attention_weight_min": -46.59306716918945,
      "activations/layer9_attention_weight_max": 43.87422180175781,
      "activations/layer9_attention_weight_min": -45.18573760986328,
      "epoch": 16.68,
      "learning_rate": 4.289204545454545e-05,
      "loss": 2.748,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_loss": 2.701171875,
      "eval_runtime": 8.4442,
      "eval_samples_per_second": 508.516,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_openwebtext_loss": 2.701171875,
      "eval_openwebtext_ppl": 14.89717913158109,
      "eval_openwebtext_runtime": 8.4442,
      "eval_openwebtext_samples_per_second": 508.516,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9323,
      "eval_wikitext_samples_per_second": 235.988,
      "step": 287000
    },
    {
      "epoch": 16.68,
      "eval_lambada_loss": 2.45703125,
      "eval_lambada_ppl": 11.670114409757737,
      "eval_lambada_runtime": 9.5189,
      "eval_lambada_samples_per_second": 511.506,
      "step": 287000
    },
    {
      "activations/layer0_attention_weight_max": 16.60392951965332,
      "activations/layer0_attention_weight_min": -13.568960189819336,
      "activations/layer10_attention_weight_max": 37.73567199707031,
      "activations/layer10_attention_weight_min": -38.41781234741211,
      "activations/layer11_attention_weight_max": 34.46660232543945,
      "activations/layer11_attention_weight_min": -37.17443084716797,
      "activations/layer12_attention_weight_max": 20.56895637512207,
      "activations/layer12_attention_weight_min": -25.286333084106445,
      "activations/layer13_attention_weight_max": 42.02012252807617,
      "activations/layer13_attention_weight_min": -27.614826202392578,
      "activations/layer14_attention_weight_max": 35.608306884765625,
      "activations/layer14_attention_weight_min": -31.2958927154541,
      "activations/layer15_attention_weight_max": 40.015663146972656,
      "activations/layer15_attention_weight_min": -32.2625617980957,
      "activations/layer16_attention_weight_max": 35.80739974975586,
      "activations/layer16_attention_weight_min": -33.76222610473633,
      "activations/layer17_attention_weight_max": 51.36423110961914,
      "activations/layer17_attention_weight_min": -49.32393264770508,
      "activations/layer18_attention_weight_max": 47.58007049560547,
      "activations/layer18_attention_weight_min": -45.61009216308594,
      "activations/layer19_attention_weight_max": 22.466306686401367,
      "activations/layer19_attention_weight_min": -24.709735870361328,
      "activations/layer1_attention_weight_max": 17.440885543823242,
      "activations/layer1_attention_weight_min": -13.708531379699707,
      "activations/layer20_attention_weight_max": 21.279014587402344,
      "activations/layer20_attention_weight_min": -23.67190170288086,
      "activations/layer21_attention_weight_max": 35.142616271972656,
      "activations/layer21_attention_weight_min": -22.34708595275879,
      "activations/layer22_attention_weight_max": 27.17291831970215,
      "activations/layer22_attention_weight_min": -28.458473205566406,
      "activations/layer23_attention_weight_max": 37.92161560058594,
      "activations/layer23_attention_weight_min": -23.961973190307617,
      "activations/layer2_attention_weight_max": 32.32651138305664,
      "activations/layer2_attention_weight_min": -29.168415069580078,
      "activations/layer3_attention_weight_max": 85.59867095947266,
      "activations/layer3_attention_weight_min": -85.2603530883789,
      "activations/layer4_attention_weight_max": 90.25212860107422,
      "activations/layer4_attention_weight_min": -88.30684661865234,
      "activations/layer5_attention_weight_max": 63.682777404785156,
      "activations/layer5_attention_weight_min": -74.02290344238281,
      "activations/layer6_attention_weight_max": 50.615333557128906,
      "activations/layer6_attention_weight_min": -52.957027435302734,
      "activations/layer7_attention_weight_max": 67.42658996582031,
      "activations/layer7_attention_weight_min": -63.85859680175781,
      "activations/layer8_attention_weight_max": 48.788917541503906,
      "activations/layer8_attention_weight_min": -49.37215042114258,
      "activations/layer9_attention_weight_max": 45.8123779296875,
      "activations/layer9_attention_weight_min": -47.65471267700195,
      "epoch": 16.68,
      "learning_rate": 4.287310606060605e-05,
      "loss": 2.7426,
      "step": 287050
    },
    {
      "activations/layer0_attention_weight_max": 16.17473793029785,
      "activations/layer0_attention_weight_min": -13.420660018920898,
      "activations/layer10_attention_weight_max": 34.71416473388672,
      "activations/layer10_attention_weight_min": -35.63581085205078,
      "activations/layer11_attention_weight_max": 32.153175354003906,
      "activations/layer11_attention_weight_min": -33.8162841796875,
      "activations/layer12_attention_weight_max": 19.844989776611328,
      "activations/layer12_attention_weight_min": -23.714900970458984,
      "activations/layer13_attention_weight_max": 34.42549133300781,
      "activations/layer13_attention_weight_min": -29.558008193969727,
      "activations/layer14_attention_weight_max": 37.19426727294922,
      "activations/layer14_attention_weight_min": -31.825952529907227,
      "activations/layer15_attention_weight_max": 36.525146484375,
      "activations/layer15_attention_weight_min": -32.699310302734375,
      "activations/layer16_attention_weight_max": 36.08343505859375,
      "activations/layer16_attention_weight_min": -33.626495361328125,
      "activations/layer17_attention_weight_max": 54.5865364074707,
      "activations/layer17_attention_weight_min": -47.96677780151367,
      "activations/layer18_attention_weight_max": 51.18617248535156,
      "activations/layer18_attention_weight_min": -43.09331512451172,
      "activations/layer19_attention_weight_max": 25.10988998413086,
      "activations/layer19_attention_weight_min": -22.080703735351562,
      "activations/layer1_attention_weight_max": 16.80668067932129,
      "activations/layer1_attention_weight_min": -13.82692813873291,
      "activations/layer20_attention_weight_max": 24.215742111206055,
      "activations/layer20_attention_weight_min": -20.889892578125,
      "activations/layer21_attention_weight_max": 36.65266799926758,
      "activations/layer21_attention_weight_min": -21.110692977905273,
      "activations/layer22_attention_weight_max": 33.4857177734375,
      "activations/layer22_attention_weight_min": -26.772260665893555,
      "activations/layer23_attention_weight_max": 39.217628479003906,
      "activations/layer23_attention_weight_min": -25.21613121032715,
      "activations/layer2_attention_weight_max": 30.439716339111328,
      "activations/layer2_attention_weight_min": -29.878881454467773,
      "activations/layer3_attention_weight_max": 85.13642120361328,
      "activations/layer3_attention_weight_min": -87.1697998046875,
      "activations/layer4_attention_weight_max": 84.28339385986328,
      "activations/layer4_attention_weight_min": -85.64000701904297,
      "activations/layer5_attention_weight_max": 60.18418884277344,
      "activations/layer5_attention_weight_min": -76.66731262207031,
      "activations/layer6_attention_weight_max": 48.31540298461914,
      "activations/layer6_attention_weight_min": -47.041969299316406,
      "activations/layer7_attention_weight_max": 59.680999755859375,
      "activations/layer7_attention_weight_min": -64.21202850341797,
      "activations/layer8_attention_weight_max": 43.536991119384766,
      "activations/layer8_attention_weight_min": -48.384544372558594,
      "activations/layer9_attention_weight_max": 44.10999298095703,
      "activations/layer9_attention_weight_min": -44.80128479003906,
      "epoch": 16.68,
      "learning_rate": 4.285416666666666e-05,
      "loss": 2.7403,
      "step": 287100
    },
    {
      "activations/layer0_attention_weight_max": 16.549386978149414,
      "activations/layer0_attention_weight_min": -13.486672401428223,
      "activations/layer10_attention_weight_max": 35.38819122314453,
      "activations/layer10_attention_weight_min": -36.05977249145508,
      "activations/layer11_attention_weight_max": 32.427947998046875,
      "activations/layer11_attention_weight_min": -35.10504913330078,
      "activations/layer12_attention_weight_max": 19.235349655151367,
      "activations/layer12_attention_weight_min": -24.829345703125,
      "activations/layer13_attention_weight_max": 32.389686584472656,
      "activations/layer13_attention_weight_min": -30.27084732055664,
      "activations/layer14_attention_weight_max": 36.57067108154297,
      "activations/layer14_attention_weight_min": -30.979755401611328,
      "activations/layer15_attention_weight_max": 34.22610855102539,
      "activations/layer15_attention_weight_min": -32.99467468261719,
      "activations/layer16_attention_weight_max": 34.406612396240234,
      "activations/layer16_attention_weight_min": -34.981754302978516,
      "activations/layer17_attention_weight_max": 51.67793655395508,
      "activations/layer17_attention_weight_min": -50.33628463745117,
      "activations/layer18_attention_weight_max": 45.72133255004883,
      "activations/layer18_attention_weight_min": -40.684051513671875,
      "activations/layer19_attention_weight_max": 22.753238677978516,
      "activations/layer19_attention_weight_min": -25.024503707885742,
      "activations/layer1_attention_weight_max": 16.063232421875,
      "activations/layer1_attention_weight_min": -13.702751159667969,
      "activations/layer20_attention_weight_max": 21.14986801147461,
      "activations/layer20_attention_weight_min": -24.305875778198242,
      "activations/layer21_attention_weight_max": 37.32421112060547,
      "activations/layer21_attention_weight_min": -24.18402671813965,
      "activations/layer22_attention_weight_max": 28.60773277282715,
      "activations/layer22_attention_weight_min": -28.97141456604004,
      "activations/layer23_attention_weight_max": 34.9539794921875,
      "activations/layer23_attention_weight_min": -23.7625732421875,
      "activations/layer2_attention_weight_max": 30.061817169189453,
      "activations/layer2_attention_weight_min": -30.45061492919922,
      "activations/layer3_attention_weight_max": 82.7811279296875,
      "activations/layer3_attention_weight_min": -84.0267333984375,
      "activations/layer4_attention_weight_max": 85.40052032470703,
      "activations/layer4_attention_weight_min": -83.6775894165039,
      "activations/layer5_attention_weight_max": 60.25533676147461,
      "activations/layer5_attention_weight_min": -73.69380950927734,
      "activations/layer6_attention_weight_max": 47.50191879272461,
      "activations/layer6_attention_weight_min": -47.44822311401367,
      "activations/layer7_attention_weight_max": 61.803123474121094,
      "activations/layer7_attention_weight_min": -59.88140869140625,
      "activations/layer8_attention_weight_max": 44.368289947509766,
      "activations/layer8_attention_weight_min": -48.77913284301758,
      "activations/layer9_attention_weight_max": 43.51396942138672,
      "activations/layer9_attention_weight_min": -44.69708251953125,
      "epoch": 16.69,
      "learning_rate": 4.283522727272727e-05,
      "loss": 2.731,
      "step": 287150
    },
    {
      "activations/layer0_attention_weight_max": 16.74385643005371,
      "activations/layer0_attention_weight_min": -13.231942176818848,
      "activations/layer10_attention_weight_max": 36.68970489501953,
      "activations/layer10_attention_weight_min": -35.916603088378906,
      "activations/layer11_attention_weight_max": 31.776952743530273,
      "activations/layer11_attention_weight_min": -33.817623138427734,
      "activations/layer12_attention_weight_max": 18.8995304107666,
      "activations/layer12_attention_weight_min": -25.843399047851562,
      "activations/layer13_attention_weight_max": 31.901954650878906,
      "activations/layer13_attention_weight_min": -26.64234733581543,
      "activations/layer14_attention_weight_max": 38.28569030761719,
      "activations/layer14_attention_weight_min": -29.739656448364258,
      "activations/layer15_attention_weight_max": 30.246370315551758,
      "activations/layer15_attention_weight_min": -27.782081604003906,
      "activations/layer16_attention_weight_max": 31.208866119384766,
      "activations/layer16_attention_weight_min": -29.622283935546875,
      "activations/layer17_attention_weight_max": 51.553470611572266,
      "activations/layer17_attention_weight_min": -43.58271026611328,
      "activations/layer18_attention_weight_max": 46.356422424316406,
      "activations/layer18_attention_weight_min": -38.40291976928711,
      "activations/layer19_attention_weight_max": 22.81743049621582,
      "activations/layer19_attention_weight_min": -23.538604736328125,
      "activations/layer1_attention_weight_max": 16.038936614990234,
      "activations/layer1_attention_weight_min": -13.650341987609863,
      "activations/layer20_attention_weight_max": 21.451047897338867,
      "activations/layer20_attention_weight_min": -20.814741134643555,
      "activations/layer21_attention_weight_max": 36.96100616455078,
      "activations/layer21_attention_weight_min": -21.44628143310547,
      "activations/layer22_attention_weight_max": 28.97795867919922,
      "activations/layer22_attention_weight_min": -24.52559471130371,
      "activations/layer23_attention_weight_max": 42.111148834228516,
      "activations/layer23_attention_weight_min": -25.133586883544922,
      "activations/layer2_attention_weight_max": 32.36430740356445,
      "activations/layer2_attention_weight_min": -30.480276107788086,
      "activations/layer3_attention_weight_max": 85.78484344482422,
      "activations/layer3_attention_weight_min": -89.63192749023438,
      "activations/layer4_attention_weight_max": 86.55551147460938,
      "activations/layer4_attention_weight_min": -88.71852111816406,
      "activations/layer5_attention_weight_max": 63.7794189453125,
      "activations/layer5_attention_weight_min": -74.75910186767578,
      "activations/layer6_attention_weight_max": 48.238826751708984,
      "activations/layer6_attention_weight_min": -49.844581604003906,
      "activations/layer7_attention_weight_max": 66.8941879272461,
      "activations/layer7_attention_weight_min": -63.07868957519531,
      "activations/layer8_attention_weight_max": 45.694671630859375,
      "activations/layer8_attention_weight_min": -49.762657165527344,
      "activations/layer9_attention_weight_max": 48.725276947021484,
      "activations/layer9_attention_weight_min": -47.08113098144531,
      "epoch": 16.69,
      "learning_rate": 4.281628787878788e-05,
      "loss": 2.7487,
      "step": 287200
    },
    {
      "activations/layer0_attention_weight_max": 15.842700958251953,
      "activations/layer0_attention_weight_min": -13.637636184692383,
      "activations/layer10_attention_weight_max": 38.29608917236328,
      "activations/layer10_attention_weight_min": -35.69430160522461,
      "activations/layer11_attention_weight_max": 34.052894592285156,
      "activations/layer11_attention_weight_min": -34.623870849609375,
      "activations/layer12_attention_weight_max": 19.233144760131836,
      "activations/layer12_attention_weight_min": -34.49949645996094,
      "activations/layer13_attention_weight_max": 38.028038024902344,
      "activations/layer13_attention_weight_min": -32.619964599609375,
      "activations/layer14_attention_weight_max": 39.36690139770508,
      "activations/layer14_attention_weight_min": -32.5199089050293,
      "activations/layer15_attention_weight_max": 35.58945846557617,
      "activations/layer15_attention_weight_min": -31.363862991333008,
      "activations/layer16_attention_weight_max": 31.311853408813477,
      "activations/layer16_attention_weight_min": -31.325471878051758,
      "activations/layer17_attention_weight_max": 50.883113861083984,
      "activations/layer17_attention_weight_min": -44.16105270385742,
      "activations/layer18_attention_weight_max": 42.981101989746094,
      "activations/layer18_attention_weight_min": -41.44014358520508,
      "activations/layer19_attention_weight_max": 22.41266441345215,
      "activations/layer19_attention_weight_min": -22.563737869262695,
      "activations/layer1_attention_weight_max": 17.237884521484375,
      "activations/layer1_attention_weight_min": -15.289246559143066,
      "activations/layer20_attention_weight_max": 21.009782791137695,
      "activations/layer20_attention_weight_min": -23.1223201751709,
      "activations/layer21_attention_weight_max": 33.54630661010742,
      "activations/layer21_attention_weight_min": -22.358051300048828,
      "activations/layer22_attention_weight_max": 26.42961883544922,
      "activations/layer22_attention_weight_min": -28.514118194580078,
      "activations/layer23_attention_weight_max": 35.958927154541016,
      "activations/layer23_attention_weight_min": -25.642436981201172,
      "activations/layer2_attention_weight_max": 30.55434226989746,
      "activations/layer2_attention_weight_min": -30.959815979003906,
      "activations/layer3_attention_weight_max": 91.82258605957031,
      "activations/layer3_attention_weight_min": -93.00414276123047,
      "activations/layer4_attention_weight_max": 90.7464370727539,
      "activations/layer4_attention_weight_min": -87.97557830810547,
      "activations/layer5_attention_weight_max": 64.56272888183594,
      "activations/layer5_attention_weight_min": -74.24650573730469,
      "activations/layer6_attention_weight_max": 52.29701614379883,
      "activations/layer6_attention_weight_min": -51.25248718261719,
      "activations/layer7_attention_weight_max": 67.95035552978516,
      "activations/layer7_attention_weight_min": -66.20133209228516,
      "activations/layer8_attention_weight_max": 48.18708038330078,
      "activations/layer8_attention_weight_min": -50.982032775878906,
      "activations/layer9_attention_weight_max": 50.683807373046875,
      "activations/layer9_attention_weight_min": -46.191341400146484,
      "epoch": 16.69,
      "learning_rate": 4.279734848484848e-05,
      "loss": 2.7505,
      "step": 287250
    },
    {
      "activations/layer0_attention_weight_max": 15.287897109985352,
      "activations/layer0_attention_weight_min": -13.611607551574707,
      "activations/layer10_attention_weight_max": 36.405921936035156,
      "activations/layer10_attention_weight_min": -34.98221206665039,
      "activations/layer11_attention_weight_max": 32.60700988769531,
      "activations/layer11_attention_weight_min": -32.461158752441406,
      "activations/layer12_attention_weight_max": 20.04178237915039,
      "activations/layer12_attention_weight_min": -24.29119300842285,
      "activations/layer13_attention_weight_max": 38.38164520263672,
      "activations/layer13_attention_weight_min": -33.39096450805664,
      "activations/layer14_attention_weight_max": 39.523963928222656,
      "activations/layer14_attention_weight_min": -34.88156509399414,
      "activations/layer15_attention_weight_max": 36.52399444580078,
      "activations/layer15_attention_weight_min": -32.70295715332031,
      "activations/layer16_attention_weight_max": 35.48390579223633,
      "activations/layer16_attention_weight_min": -32.313968658447266,
      "activations/layer17_attention_weight_max": 57.47669982910156,
      "activations/layer17_attention_weight_min": -45.4415168762207,
      "activations/layer18_attention_weight_max": 52.6007080078125,
      "activations/layer18_attention_weight_min": -41.67994689941406,
      "activations/layer19_attention_weight_max": 25.04342269897461,
      "activations/layer19_attention_weight_min": -21.863800048828125,
      "activations/layer1_attention_weight_max": 15.792634010314941,
      "activations/layer1_attention_weight_min": -13.699005126953125,
      "activations/layer20_attention_weight_max": 24.187131881713867,
      "activations/layer20_attention_weight_min": -20.83298110961914,
      "activations/layer21_attention_weight_max": 37.42204666137695,
      "activations/layer21_attention_weight_min": -24.90981101989746,
      "activations/layer22_attention_weight_max": 32.00535202026367,
      "activations/layer22_attention_weight_min": -24.5125675201416,
      "activations/layer23_attention_weight_max": 41.26967239379883,
      "activations/layer23_attention_weight_min": -24.32906723022461,
      "activations/layer2_attention_weight_max": 31.559553146362305,
      "activations/layer2_attention_weight_min": -30.81210708618164,
      "activations/layer3_attention_weight_max": 85.62664031982422,
      "activations/layer3_attention_weight_min": -86.17755126953125,
      "activations/layer4_attention_weight_max": 86.39019012451172,
      "activations/layer4_attention_weight_min": -87.94166564941406,
      "activations/layer5_attention_weight_max": 62.778533935546875,
      "activations/layer5_attention_weight_min": -71.54120635986328,
      "activations/layer6_attention_weight_max": 49.68061447143555,
      "activations/layer6_attention_weight_min": -50.31839370727539,
      "activations/layer7_attention_weight_max": 62.90653991699219,
      "activations/layer7_attention_weight_min": -66.71490478515625,
      "activations/layer8_attention_weight_max": 48.3116340637207,
      "activations/layer8_attention_weight_min": -49.49064254760742,
      "activations/layer9_attention_weight_max": 44.67082595825195,
      "activations/layer9_attention_weight_min": -46.63216781616211,
      "epoch": 16.69,
      "learning_rate": 4.277840909090909e-05,
      "loss": 2.744,
      "step": 287300
    },
    {
      "activations/layer0_attention_weight_max": 15.4830961227417,
      "activations/layer0_attention_weight_min": -13.343905448913574,
      "activations/layer10_attention_weight_max": 37.457191467285156,
      "activations/layer10_attention_weight_min": -39.35923767089844,
      "activations/layer11_attention_weight_max": 35.529014587402344,
      "activations/layer11_attention_weight_min": -35.75929641723633,
      "activations/layer12_attention_weight_max": 18.105148315429688,
      "activations/layer12_attention_weight_min": -23.112585067749023,
      "activations/layer13_attention_weight_max": 35.88996124267578,
      "activations/layer13_attention_weight_min": -28.51441192626953,
      "activations/layer14_attention_weight_max": 35.98883056640625,
      "activations/layer14_attention_weight_min": -29.39015007019043,
      "activations/layer15_attention_weight_max": 33.2281379699707,
      "activations/layer15_attention_weight_min": -32.54465103149414,
      "activations/layer16_attention_weight_max": 33.4396858215332,
      "activations/layer16_attention_weight_min": -30.347522735595703,
      "activations/layer17_attention_weight_max": 50.56414031982422,
      "activations/layer17_attention_weight_min": -43.24960708618164,
      "activations/layer18_attention_weight_max": 44.740135192871094,
      "activations/layer18_attention_weight_min": -40.45083999633789,
      "activations/layer19_attention_weight_max": 25.67170524597168,
      "activations/layer19_attention_weight_min": -22.735870361328125,
      "activations/layer1_attention_weight_max": 15.631434440612793,
      "activations/layer1_attention_weight_min": -15.635713577270508,
      "activations/layer20_attention_weight_max": 21.848478317260742,
      "activations/layer20_attention_weight_min": -20.855295181274414,
      "activations/layer21_attention_weight_max": 39.005775451660156,
      "activations/layer21_attention_weight_min": -21.861711502075195,
      "activations/layer22_attention_weight_max": 30.876909255981445,
      "activations/layer22_attention_weight_min": -28.36097526550293,
      "activations/layer23_attention_weight_max": 37.63071060180664,
      "activations/layer23_attention_weight_min": -23.588117599487305,
      "activations/layer2_attention_weight_max": 32.35921096801758,
      "activations/layer2_attention_weight_min": -30.530460357666016,
      "activations/layer3_attention_weight_max": 91.96698760986328,
      "activations/layer3_attention_weight_min": -90.0177993774414,
      "activations/layer4_attention_weight_max": 87.3156509399414,
      "activations/layer4_attention_weight_min": -85.63507080078125,
      "activations/layer5_attention_weight_max": 68.7174301147461,
      "activations/layer5_attention_weight_min": -74.01953125,
      "activations/layer6_attention_weight_max": 49.93946075439453,
      "activations/layer6_attention_weight_min": -49.92027282714844,
      "activations/layer7_attention_weight_max": 66.81224822998047,
      "activations/layer7_attention_weight_min": -63.196895599365234,
      "activations/layer8_attention_weight_max": 48.62782669067383,
      "activations/layer8_attention_weight_min": -50.1798210144043,
      "activations/layer9_attention_weight_max": 46.491371154785156,
      "activations/layer9_attention_weight_min": -45.72981643676758,
      "epoch": 16.7,
      "learning_rate": 4.2759469696969695e-05,
      "loss": 2.754,
      "step": 287350
    },
    {
      "activations/layer0_attention_weight_max": 16.57362174987793,
      "activations/layer0_attention_weight_min": -13.763057708740234,
      "activations/layer10_attention_weight_max": 36.44123840332031,
      "activations/layer10_attention_weight_min": -37.60736846923828,
      "activations/layer11_attention_weight_max": 34.459354400634766,
      "activations/layer11_attention_weight_min": -37.206817626953125,
      "activations/layer12_attention_weight_max": 17.349102020263672,
      "activations/layer12_attention_weight_min": -24.463775634765625,
      "activations/layer13_attention_weight_max": 29.877662658691406,
      "activations/layer13_attention_weight_min": -26.80307960510254,
      "activations/layer14_attention_weight_max": 34.54347229003906,
      "activations/layer14_attention_weight_min": -29.45648765563965,
      "activations/layer15_attention_weight_max": 29.519573211669922,
      "activations/layer15_attention_weight_min": -29.712631225585938,
      "activations/layer16_attention_weight_max": 34.345882415771484,
      "activations/layer16_attention_weight_min": -32.16468811035156,
      "activations/layer17_attention_weight_max": 52.145774841308594,
      "activations/layer17_attention_weight_min": -45.349273681640625,
      "activations/layer18_attention_weight_max": 45.56830978393555,
      "activations/layer18_attention_weight_min": -40.734317779541016,
      "activations/layer19_attention_weight_max": 25.761106491088867,
      "activations/layer19_attention_weight_min": -23.357688903808594,
      "activations/layer1_attention_weight_max": 16.926498413085938,
      "activations/layer1_attention_weight_min": -14.816848754882812,
      "activations/layer20_attention_weight_max": 25.73186683654785,
      "activations/layer20_attention_weight_min": -21.321855545043945,
      "activations/layer21_attention_weight_max": 38.40131378173828,
      "activations/layer21_attention_weight_min": -25.627073287963867,
      "activations/layer22_attention_weight_max": 29.840852737426758,
      "activations/layer22_attention_weight_min": -25.299184799194336,
      "activations/layer23_attention_weight_max": 35.942413330078125,
      "activations/layer23_attention_weight_min": -23.904216766357422,
      "activations/layer2_attention_weight_max": 30.52139663696289,
      "activations/layer2_attention_weight_min": -30.427066802978516,
      "activations/layer3_attention_weight_max": 85.74036407470703,
      "activations/layer3_attention_weight_min": -86.9685287475586,
      "activations/layer4_attention_weight_max": 85.1991195678711,
      "activations/layer4_attention_weight_min": -82.08605194091797,
      "activations/layer5_attention_weight_max": 63.35224533081055,
      "activations/layer5_attention_weight_min": -74.13148498535156,
      "activations/layer6_attention_weight_max": 49.95869064331055,
      "activations/layer6_attention_weight_min": -51.08488082885742,
      "activations/layer7_attention_weight_max": 68.99691772460938,
      "activations/layer7_attention_weight_min": -73.38674926757812,
      "activations/layer8_attention_weight_max": 46.47642135620117,
      "activations/layer8_attention_weight_min": -50.78883361816406,
      "activations/layer9_attention_weight_max": 45.90227127075195,
      "activations/layer9_attention_weight_min": -45.32780075073242,
      "epoch": 16.7,
      "learning_rate": 4.2740530303030296e-05,
      "loss": 2.7452,
      "step": 287400
    },
    {
      "activations/layer0_attention_weight_max": 16.151487350463867,
      "activations/layer0_attention_weight_min": -13.52489948272705,
      "activations/layer10_attention_weight_max": 35.57390213012695,
      "activations/layer10_attention_weight_min": -37.411415100097656,
      "activations/layer11_attention_weight_max": 32.93199157714844,
      "activations/layer11_attention_weight_min": -33.64859390258789,
      "activations/layer12_attention_weight_max": 19.981586456298828,
      "activations/layer12_attention_weight_min": -28.659177780151367,
      "activations/layer13_attention_weight_max": 33.975860595703125,
      "activations/layer13_attention_weight_min": -28.872278213500977,
      "activations/layer14_attention_weight_max": 32.45774841308594,
      "activations/layer14_attention_weight_min": -30.771684646606445,
      "activations/layer15_attention_weight_max": 31.945465087890625,
      "activations/layer15_attention_weight_min": -31.423873901367188,
      "activations/layer16_attention_weight_max": 32.987247467041016,
      "activations/layer16_attention_weight_min": -32.69626235961914,
      "activations/layer17_attention_weight_max": 49.86102294921875,
      "activations/layer17_attention_weight_min": -46.28746795654297,
      "activations/layer18_attention_weight_max": 44.39277648925781,
      "activations/layer18_attention_weight_min": -38.03355026245117,
      "activations/layer19_attention_weight_max": 22.884963989257812,
      "activations/layer19_attention_weight_min": -23.515336990356445,
      "activations/layer1_attention_weight_max": 17.587650299072266,
      "activations/layer1_attention_weight_min": -14.32017993927002,
      "activations/layer20_attention_weight_max": 22.110986709594727,
      "activations/layer20_attention_weight_min": -23.232892990112305,
      "activations/layer21_attention_weight_max": 38.35400390625,
      "activations/layer21_attention_weight_min": -23.30413055419922,
      "activations/layer22_attention_weight_max": 29.928340911865234,
      "activations/layer22_attention_weight_min": -25.510618209838867,
      "activations/layer23_attention_weight_max": 41.29521942138672,
      "activations/layer23_attention_weight_min": -25.784526824951172,
      "activations/layer2_attention_weight_max": 32.00283432006836,
      "activations/layer2_attention_weight_min": -30.80809783935547,
      "activations/layer3_attention_weight_max": 83.42928314208984,
      "activations/layer3_attention_weight_min": -86.13568878173828,
      "activations/layer4_attention_weight_max": 87.01840209960938,
      "activations/layer4_attention_weight_min": -81.49761199951172,
      "activations/layer5_attention_weight_max": 63.11695098876953,
      "activations/layer5_attention_weight_min": -69.53935241699219,
      "activations/layer6_attention_weight_max": 50.65523910522461,
      "activations/layer6_attention_weight_min": -54.169559478759766,
      "activations/layer7_attention_weight_max": 66.05657958984375,
      "activations/layer7_attention_weight_min": -60.19205093383789,
      "activations/layer8_attention_weight_max": 45.59214782714844,
      "activations/layer8_attention_weight_min": -47.364288330078125,
      "activations/layer9_attention_weight_max": 45.242645263671875,
      "activations/layer9_attention_weight_min": -43.05826187133789,
      "epoch": 16.7,
      "learning_rate": 4.2721590909090905e-05,
      "loss": 2.7436,
      "step": 287450
    },
    {
      "activations/layer0_attention_weight_max": 15.53546142578125,
      "activations/layer0_attention_weight_min": -13.263301849365234,
      "activations/layer10_attention_weight_max": 39.21271514892578,
      "activations/layer10_attention_weight_min": -37.2325325012207,
      "activations/layer11_attention_weight_max": 36.17759323120117,
      "activations/layer11_attention_weight_min": -34.817665100097656,
      "activations/layer12_attention_weight_max": 21.016544342041016,
      "activations/layer12_attention_weight_min": -25.3388614654541,
      "activations/layer13_attention_weight_max": 49.32486343383789,
      "activations/layer13_attention_weight_min": -30.356536865234375,
      "activations/layer14_attention_weight_max": 48.09414291381836,
      "activations/layer14_attention_weight_min": -34.396480560302734,
      "activations/layer15_attention_weight_max": 51.884925842285156,
      "activations/layer15_attention_weight_min": -33.74933624267578,
      "activations/layer16_attention_weight_max": 41.902748107910156,
      "activations/layer16_attention_weight_min": -34.67921829223633,
      "activations/layer17_attention_weight_max": 64.33697509765625,
      "activations/layer17_attention_weight_min": -53.91780471801758,
      "activations/layer18_attention_weight_max": 63.8138313293457,
      "activations/layer18_attention_weight_min": -50.316219329833984,
      "activations/layer19_attention_weight_max": 30.83298110961914,
      "activations/layer19_attention_weight_min": -26.442087173461914,
      "activations/layer1_attention_weight_max": 15.45482063293457,
      "activations/layer1_attention_weight_min": -14.02959156036377,
      "activations/layer20_attention_weight_max": 29.870792388916016,
      "activations/layer20_attention_weight_min": -23.632795333862305,
      "activations/layer21_attention_weight_max": 60.31097412109375,
      "activations/layer21_attention_weight_min": -29.72539520263672,
      "activations/layer22_attention_weight_max": 38.44003677368164,
      "activations/layer22_attention_weight_min": -28.49773597717285,
      "activations/layer23_attention_weight_max": 54.99666976928711,
      "activations/layer23_attention_weight_min": -25.07410430908203,
      "activations/layer2_attention_weight_max": 34.21098327636719,
      "activations/layer2_attention_weight_min": -30.78841209411621,
      "activations/layer3_attention_weight_max": 92.4371566772461,
      "activations/layer3_attention_weight_min": -90.04296875,
      "activations/layer4_attention_weight_max": 91.07665252685547,
      "activations/layer4_attention_weight_min": -86.85466766357422,
      "activations/layer5_attention_weight_max": 64.84359741210938,
      "activations/layer5_attention_weight_min": -72.54043579101562,
      "activations/layer6_attention_weight_max": 53.325782775878906,
      "activations/layer6_attention_weight_min": -52.18613815307617,
      "activations/layer7_attention_weight_max": 69.08018493652344,
      "activations/layer7_attention_weight_min": -64.23544311523438,
      "activations/layer8_attention_weight_max": 51.135833740234375,
      "activations/layer8_attention_weight_min": -51.98507308959961,
      "activations/layer9_attention_weight_max": 46.75103759765625,
      "activations/layer9_attention_weight_min": -46.308589935302734,
      "epoch": 16.71,
      "learning_rate": 4.270265151515151e-05,
      "loss": 2.7408,
      "step": 287500
    },
    {
      "activations/layer0_attention_weight_max": 16.642902374267578,
      "activations/layer0_attention_weight_min": -13.070793151855469,
      "activations/layer10_attention_weight_max": 35.79916763305664,
      "activations/layer10_attention_weight_min": -35.129417419433594,
      "activations/layer11_attention_weight_max": 32.662109375,
      "activations/layer11_attention_weight_min": -33.217689514160156,
      "activations/layer12_attention_weight_max": 17.41777992248535,
      "activations/layer12_attention_weight_min": -23.11503791809082,
      "activations/layer13_attention_weight_max": 33.926734924316406,
      "activations/layer13_attention_weight_min": -27.43756675720215,
      "activations/layer14_attention_weight_max": 34.15018844604492,
      "activations/layer14_attention_weight_min": -30.6246337890625,
      "activations/layer15_attention_weight_max": 31.22957992553711,
      "activations/layer15_attention_weight_min": -29.809417724609375,
      "activations/layer16_attention_weight_max": 30.814062118530273,
      "activations/layer16_attention_weight_min": -30.490148544311523,
      "activations/layer17_attention_weight_max": 48.42252731323242,
      "activations/layer17_attention_weight_min": -43.038387298583984,
      "activations/layer18_attention_weight_max": 46.44210433959961,
      "activations/layer18_attention_weight_min": -39.4687385559082,
      "activations/layer19_attention_weight_max": 25.239164352416992,
      "activations/layer19_attention_weight_min": -22.148862838745117,
      "activations/layer1_attention_weight_max": 16.348661422729492,
      "activations/layer1_attention_weight_min": -15.796013832092285,
      "activations/layer20_attention_weight_max": 22.34532928466797,
      "activations/layer20_attention_weight_min": -20.820117950439453,
      "activations/layer21_attention_weight_max": 36.01895523071289,
      "activations/layer21_attention_weight_min": -26.63796615600586,
      "activations/layer22_attention_weight_max": 28.22245979309082,
      "activations/layer22_attention_weight_min": -24.707672119140625,
      "activations/layer23_attention_weight_max": 37.35453796386719,
      "activations/layer23_attention_weight_min": -23.531484603881836,
      "activations/layer2_attention_weight_max": 33.23674392700195,
      "activations/layer2_attention_weight_min": -33.276214599609375,
      "activations/layer3_attention_weight_max": 89.1758804321289,
      "activations/layer3_attention_weight_min": -98.24199676513672,
      "activations/layer4_attention_weight_max": 85.47348022460938,
      "activations/layer4_attention_weight_min": -90.27320861816406,
      "activations/layer5_attention_weight_max": 63.70391845703125,
      "activations/layer5_attention_weight_min": -72.6111831665039,
      "activations/layer6_attention_weight_max": 48.87767791748047,
      "activations/layer6_attention_weight_min": -49.645145416259766,
      "activations/layer7_attention_weight_max": 63.038124084472656,
      "activations/layer7_attention_weight_min": -62.73662185668945,
      "activations/layer8_attention_weight_max": 44.44017028808594,
      "activations/layer8_attention_weight_min": -49.19834899902344,
      "activations/layer9_attention_weight_max": 45.31709671020508,
      "activations/layer9_attention_weight_min": -46.03641891479492,
      "epoch": 16.71,
      "learning_rate": 4.268371212121212e-05,
      "loss": 2.739,
      "step": 287550
    },
    {
      "activations/layer0_attention_weight_max": 15.409371376037598,
      "activations/layer0_attention_weight_min": -13.186394691467285,
      "activations/layer10_attention_weight_max": 32.31878662109375,
      "activations/layer10_attention_weight_min": -33.60413360595703,
      "activations/layer11_attention_weight_max": 30.393905639648438,
      "activations/layer11_attention_weight_min": -32.17633819580078,
      "activations/layer12_attention_weight_max": 17.249744415283203,
      "activations/layer12_attention_weight_min": -25.383710861206055,
      "activations/layer13_attention_weight_max": 33.037654876708984,
      "activations/layer13_attention_weight_min": -31.6336727142334,
      "activations/layer14_attention_weight_max": 31.817867279052734,
      "activations/layer14_attention_weight_min": -28.698301315307617,
      "activations/layer15_attention_weight_max": 29.506526947021484,
      "activations/layer15_attention_weight_min": -29.314786911010742,
      "activations/layer16_attention_weight_max": 29.323781967163086,
      "activations/layer16_attention_weight_min": -29.908260345458984,
      "activations/layer17_attention_weight_max": 48.53921890258789,
      "activations/layer17_attention_weight_min": -45.95418930053711,
      "activations/layer18_attention_weight_max": 40.098819732666016,
      "activations/layer18_attention_weight_min": -38.81431198120117,
      "activations/layer19_attention_weight_max": 22.91087532043457,
      "activations/layer19_attention_weight_min": -24.281606674194336,
      "activations/layer1_attention_weight_max": 15.163670539855957,
      "activations/layer1_attention_weight_min": -14.01773452758789,
      "activations/layer20_attention_weight_max": 24.23665428161621,
      "activations/layer20_attention_weight_min": -21.781686782836914,
      "activations/layer21_attention_weight_max": 31.473190307617188,
      "activations/layer21_attention_weight_min": -23.171092987060547,
      "activations/layer22_attention_weight_max": 27.51957893371582,
      "activations/layer22_attention_weight_min": -27.02242088317871,
      "activations/layer23_attention_weight_max": 32.08784103393555,
      "activations/layer23_attention_weight_min": -27.205459594726562,
      "activations/layer2_attention_weight_max": 29.55976104736328,
      "activations/layer2_attention_weight_min": -29.98811149597168,
      "activations/layer3_attention_weight_max": 87.54481506347656,
      "activations/layer3_attention_weight_min": -90.98005676269531,
      "activations/layer4_attention_weight_max": 84.34644317626953,
      "activations/layer4_attention_weight_min": -82.72491455078125,
      "activations/layer5_attention_weight_max": 62.79296875,
      "activations/layer5_attention_weight_min": -71.07408905029297,
      "activations/layer6_attention_weight_max": 48.17097473144531,
      "activations/layer6_attention_weight_min": -46.754024505615234,
      "activations/layer7_attention_weight_max": 67.58338928222656,
      "activations/layer7_attention_weight_min": -60.14329528808594,
      "activations/layer8_attention_weight_max": 45.03310012817383,
      "activations/layer8_attention_weight_min": -46.397735595703125,
      "activations/layer9_attention_weight_max": 51.29021072387695,
      "activations/layer9_attention_weight_min": -45.78768539428711,
      "epoch": 16.71,
      "learning_rate": 4.266477272727272e-05,
      "loss": 2.7579,
      "step": 287600
    },
    {
      "activations/layer0_attention_weight_max": 15.812827110290527,
      "activations/layer0_attention_weight_min": -13.025951385498047,
      "activations/layer10_attention_weight_max": 32.05837631225586,
      "activations/layer10_attention_weight_min": -32.769283294677734,
      "activations/layer11_attention_weight_max": 31.407512664794922,
      "activations/layer11_attention_weight_min": -31.34071159362793,
      "activations/layer12_attention_weight_max": 18.503725051879883,
      "activations/layer12_attention_weight_min": -25.945981979370117,
      "activations/layer13_attention_weight_max": 37.4986457824707,
      "activations/layer13_attention_weight_min": -28.240955352783203,
      "activations/layer14_attention_weight_max": 34.181907653808594,
      "activations/layer14_attention_weight_min": -29.594650268554688,
      "activations/layer15_attention_weight_max": 30.70635414123535,
      "activations/layer15_attention_weight_min": -30.12563133239746,
      "activations/layer16_attention_weight_max": 31.013395309448242,
      "activations/layer16_attention_weight_min": -29.286041259765625,
      "activations/layer17_attention_weight_max": 50.74208450317383,
      "activations/layer17_attention_weight_min": -41.93977737426758,
      "activations/layer18_attention_weight_max": 43.09367370605469,
      "activations/layer18_attention_weight_min": -37.67268371582031,
      "activations/layer19_attention_weight_max": 22.805667877197266,
      "activations/layer19_attention_weight_min": -20.876373291015625,
      "activations/layer1_attention_weight_max": 15.849394798278809,
      "activations/layer1_attention_weight_min": -13.921385765075684,
      "activations/layer20_attention_weight_max": 21.74715805053711,
      "activations/layer20_attention_weight_min": -21.293413162231445,
      "activations/layer21_attention_weight_max": 34.45230484008789,
      "activations/layer21_attention_weight_min": -22.74283790588379,
      "activations/layer22_attention_weight_max": 28.10017967224121,
      "activations/layer22_attention_weight_min": -26.461166381835938,
      "activations/layer23_attention_weight_max": 36.151206970214844,
      "activations/layer23_attention_weight_min": -22.852096557617188,
      "activations/layer2_attention_weight_max": 30.82312774658203,
      "activations/layer2_attention_weight_min": -29.797809600830078,
      "activations/layer3_attention_weight_max": 88.31983947753906,
      "activations/layer3_attention_weight_min": -84.85678100585938,
      "activations/layer4_attention_weight_max": 88.79900360107422,
      "activations/layer4_attention_weight_min": -84.96015167236328,
      "activations/layer5_attention_weight_max": 63.039424896240234,
      "activations/layer5_attention_weight_min": -72.48562622070312,
      "activations/layer6_attention_weight_max": 47.14491271972656,
      "activations/layer6_attention_weight_min": -48.59517288208008,
      "activations/layer7_attention_weight_max": 63.48065948486328,
      "activations/layer7_attention_weight_min": -60.267433166503906,
      "activations/layer8_attention_weight_max": 44.11175537109375,
      "activations/layer8_attention_weight_min": -46.29694366455078,
      "activations/layer9_attention_weight_max": 41.29658889770508,
      "activations/layer9_attention_weight_min": -42.332035064697266,
      "epoch": 16.71,
      "learning_rate": 4.2645833333333324e-05,
      "loss": 2.7386,
      "step": 287650
    },
    {
      "activations/layer0_attention_weight_max": 15.61646556854248,
      "activations/layer0_attention_weight_min": -13.251941680908203,
      "activations/layer10_attention_weight_max": 37.08356475830078,
      "activations/layer10_attention_weight_min": -37.36158752441406,
      "activations/layer11_attention_weight_max": 33.58753204345703,
      "activations/layer11_attention_weight_min": -34.785179138183594,
      "activations/layer12_attention_weight_max": 20.068029403686523,
      "activations/layer12_attention_weight_min": -24.12063980102539,
      "activations/layer13_attention_weight_max": 32.08286666870117,
      "activations/layer13_attention_weight_min": -29.674022674560547,
      "activations/layer14_attention_weight_max": 36.50596237182617,
      "activations/layer14_attention_weight_min": -30.935165405273438,
      "activations/layer15_attention_weight_max": 32.61151123046875,
      "activations/layer15_attention_weight_min": -31.623220443725586,
      "activations/layer16_attention_weight_max": 35.653507232666016,
      "activations/layer16_attention_weight_min": -33.8818359375,
      "activations/layer17_attention_weight_max": 53.05579376220703,
      "activations/layer17_attention_weight_min": -50.636802673339844,
      "activations/layer18_attention_weight_max": 53.62215805053711,
      "activations/layer18_attention_weight_min": -46.17130661010742,
      "activations/layer19_attention_weight_max": 22.604318618774414,
      "activations/layer19_attention_weight_min": -25.079038619995117,
      "activations/layer1_attention_weight_max": 15.927285194396973,
      "activations/layer1_attention_weight_min": -15.299579620361328,
      "activations/layer20_attention_weight_max": 26.08989906311035,
      "activations/layer20_attention_weight_min": -22.5698184967041,
      "activations/layer21_attention_weight_max": 40.51456832885742,
      "activations/layer21_attention_weight_min": -23.094919204711914,
      "activations/layer22_attention_weight_max": 32.28894805908203,
      "activations/layer22_attention_weight_min": -28.323829650878906,
      "activations/layer23_attention_weight_max": 38.016258239746094,
      "activations/layer23_attention_weight_min": -24.342998504638672,
      "activations/layer2_attention_weight_max": 31.071292877197266,
      "activations/layer2_attention_weight_min": -29.595897674560547,
      "activations/layer3_attention_weight_max": 89.03985595703125,
      "activations/layer3_attention_weight_min": -87.23075103759766,
      "activations/layer4_attention_weight_max": 85.45982360839844,
      "activations/layer4_attention_weight_min": -85.53972625732422,
      "activations/layer5_attention_weight_max": 65.78817749023438,
      "activations/layer5_attention_weight_min": -70.07044982910156,
      "activations/layer6_attention_weight_max": 51.346858978271484,
      "activations/layer6_attention_weight_min": -52.279273986816406,
      "activations/layer7_attention_weight_max": 65.22557830810547,
      "activations/layer7_attention_weight_min": -62.83151626586914,
      "activations/layer8_attention_weight_max": 47.5745735168457,
      "activations/layer8_attention_weight_min": -51.14293670654297,
      "activations/layer9_attention_weight_max": 46.2177848815918,
      "activations/layer9_attention_weight_min": -47.60694122314453,
      "epoch": 16.72,
      "learning_rate": 4.262689393939394e-05,
      "loss": 2.747,
      "step": 287700
    },
    {
      "activations/layer0_attention_weight_max": 18.101917266845703,
      "activations/layer0_attention_weight_min": -13.484036445617676,
      "activations/layer10_attention_weight_max": 37.26961135864258,
      "activations/layer10_attention_weight_min": -37.66664123535156,
      "activations/layer11_attention_weight_max": 37.01703643798828,
      "activations/layer11_attention_weight_min": -36.24781036376953,
      "activations/layer12_attention_weight_max": 20.739877700805664,
      "activations/layer12_attention_weight_min": -28.224613189697266,
      "activations/layer13_attention_weight_max": 40.50592803955078,
      "activations/layer13_attention_weight_min": -31.116960525512695,
      "activations/layer14_attention_weight_max": 37.85204315185547,
      "activations/layer14_attention_weight_min": -31.884552001953125,
      "activations/layer15_attention_weight_max": 36.30839920043945,
      "activations/layer15_attention_weight_min": -30.01125144958496,
      "activations/layer16_attention_weight_max": 36.439266204833984,
      "activations/layer16_attention_weight_min": -33.16813659667969,
      "activations/layer17_attention_weight_max": 58.95981216430664,
      "activations/layer17_attention_weight_min": -51.177494049072266,
      "activations/layer18_attention_weight_max": 55.5067138671875,
      "activations/layer18_attention_weight_min": -45.20359420776367,
      "activations/layer19_attention_weight_max": 24.759857177734375,
      "activations/layer19_attention_weight_min": -22.573589324951172,
      "activations/layer1_attention_weight_max": 16.32076644897461,
      "activations/layer1_attention_weight_min": -15.703495025634766,
      "activations/layer20_attention_weight_max": 24.721050262451172,
      "activations/layer20_attention_weight_min": -21.730297088623047,
      "activations/layer21_attention_weight_max": 46.20143127441406,
      "activations/layer21_attention_weight_min": -22.44279670715332,
      "activations/layer22_attention_weight_max": 35.67040252685547,
      "activations/layer22_attention_weight_min": -26.549741744995117,
      "activations/layer23_attention_weight_max": 47.15782165527344,
      "activations/layer23_attention_weight_min": -22.893199920654297,
      "activations/layer2_attention_weight_max": 32.854103088378906,
      "activations/layer2_attention_weight_min": -30.810958862304688,
      "activations/layer3_attention_weight_max": 92.23400115966797,
      "activations/layer3_attention_weight_min": -89.56040954589844,
      "activations/layer4_attention_weight_max": 90.1980209350586,
      "activations/layer4_attention_weight_min": -87.07870483398438,
      "activations/layer5_attention_weight_max": 64.63928985595703,
      "activations/layer5_attention_weight_min": -72.03524780273438,
      "activations/layer6_attention_weight_max": 50.64995574951172,
      "activations/layer6_attention_weight_min": -50.49570083618164,
      "activations/layer7_attention_weight_max": 66.23974609375,
      "activations/layer7_attention_weight_min": -63.9476432800293,
      "activations/layer8_attention_weight_max": 47.492027282714844,
      "activations/layer8_attention_weight_min": -48.26461410522461,
      "activations/layer9_attention_weight_max": 49.28694152832031,
      "activations/layer9_attention_weight_min": -48.335296630859375,
      "epoch": 16.72,
      "learning_rate": 4.260795454545454e-05,
      "loss": 2.7397,
      "step": 287750
    },
    {
      "activations/layer0_attention_weight_max": 16.671241760253906,
      "activations/layer0_attention_weight_min": -13.388769149780273,
      "activations/layer10_attention_weight_max": 34.44167709350586,
      "activations/layer10_attention_weight_min": -34.9754753112793,
      "activations/layer11_attention_weight_max": 30.727874755859375,
      "activations/layer11_attention_weight_min": -33.73198318481445,
      "activations/layer12_attention_weight_max": 18.257261276245117,
      "activations/layer12_attention_weight_min": -28.77412223815918,
      "activations/layer13_attention_weight_max": 37.30792999267578,
      "activations/layer13_attention_weight_min": -30.143281936645508,
      "activations/layer14_attention_weight_max": 34.78447341918945,
      "activations/layer14_attention_weight_min": -33.77345275878906,
      "activations/layer15_attention_weight_max": 32.59134292602539,
      "activations/layer15_attention_weight_min": -31.877513885498047,
      "activations/layer16_attention_weight_max": 33.76412582397461,
      "activations/layer16_attention_weight_min": -32.17316436767578,
      "activations/layer17_attention_weight_max": 56.56138610839844,
      "activations/layer17_attention_weight_min": -45.457576751708984,
      "activations/layer18_attention_weight_max": 45.34516525268555,
      "activations/layer18_attention_weight_min": -38.82787322998047,
      "activations/layer19_attention_weight_max": 23.05964469909668,
      "activations/layer19_attention_weight_min": -21.603439331054688,
      "activations/layer1_attention_weight_max": 16.899009704589844,
      "activations/layer1_attention_weight_min": -13.637001037597656,
      "activations/layer20_attention_weight_max": 23.043474197387695,
      "activations/layer20_attention_weight_min": -23.469324111938477,
      "activations/layer21_attention_weight_max": 37.713897705078125,
      "activations/layer21_attention_weight_min": -21.63492774963379,
      "activations/layer22_attention_weight_max": 30.25214958190918,
      "activations/layer22_attention_weight_min": -26.768842697143555,
      "activations/layer23_attention_weight_max": 36.889801025390625,
      "activations/layer23_attention_weight_min": -28.266082763671875,
      "activations/layer2_attention_weight_max": 32.050907135009766,
      "activations/layer2_attention_weight_min": -29.31374740600586,
      "activations/layer3_attention_weight_max": 87.3066635131836,
      "activations/layer3_attention_weight_min": -86.11090850830078,
      "activations/layer4_attention_weight_max": 86.22093963623047,
      "activations/layer4_attention_weight_min": -90.43658447265625,
      "activations/layer5_attention_weight_max": 63.234649658203125,
      "activations/layer5_attention_weight_min": -72.97439575195312,
      "activations/layer6_attention_weight_max": 47.893089294433594,
      "activations/layer6_attention_weight_min": -48.85605239868164,
      "activations/layer7_attention_weight_max": 63.95149612426758,
      "activations/layer7_attention_weight_min": -60.65743637084961,
      "activations/layer8_attention_weight_max": 44.469905853271484,
      "activations/layer8_attention_weight_min": -48.55711364746094,
      "activations/layer9_attention_weight_max": 42.17174530029297,
      "activations/layer9_attention_weight_min": -46.9138298034668,
      "epoch": 16.72,
      "learning_rate": 4.258901515151515e-05,
      "loss": 2.7551,
      "step": 287800
    },
    {
      "activations/layer0_attention_weight_max": 15.782690048217773,
      "activations/layer0_attention_weight_min": -13.148435592651367,
      "activations/layer10_attention_weight_max": 34.565799713134766,
      "activations/layer10_attention_weight_min": -36.56764221191406,
      "activations/layer11_attention_weight_max": 31.65399742126465,
      "activations/layer11_attention_weight_min": -32.75612258911133,
      "activations/layer12_attention_weight_max": 19.22305679321289,
      "activations/layer12_attention_weight_min": -23.635210037231445,
      "activations/layer13_attention_weight_max": 39.1746826171875,
      "activations/layer13_attention_weight_min": -28.12131118774414,
      "activations/layer14_attention_weight_max": 37.40941619873047,
      "activations/layer14_attention_weight_min": -30.019662857055664,
      "activations/layer15_attention_weight_max": 37.62158203125,
      "activations/layer15_attention_weight_min": -29.702524185180664,
      "activations/layer16_attention_weight_max": 35.3701057434082,
      "activations/layer16_attention_weight_min": -31.544666290283203,
      "activations/layer17_attention_weight_max": 56.678436279296875,
      "activations/layer17_attention_weight_min": -45.70724868774414,
      "activations/layer18_attention_weight_max": 51.78683090209961,
      "activations/layer18_attention_weight_min": -43.96037673950195,
      "activations/layer19_attention_weight_max": 26.864105224609375,
      "activations/layer19_attention_weight_min": -24.541772842407227,
      "activations/layer1_attention_weight_max": 15.84878921508789,
      "activations/layer1_attention_weight_min": -13.357518196105957,
      "activations/layer20_attention_weight_max": 27.689611434936523,
      "activations/layer20_attention_weight_min": -21.636558532714844,
      "activations/layer21_attention_weight_max": 44.88115310668945,
      "activations/layer21_attention_weight_min": -25.175683975219727,
      "activations/layer22_attention_weight_max": 34.986331939697266,
      "activations/layer22_attention_weight_min": -26.47199249267578,
      "activations/layer23_attention_weight_max": 44.315574645996094,
      "activations/layer23_attention_weight_min": -22.926013946533203,
      "activations/layer2_attention_weight_max": 30.405563354492188,
      "activations/layer2_attention_weight_min": -29.905431747436523,
      "activations/layer3_attention_weight_max": 88.21366882324219,
      "activations/layer3_attention_weight_min": -88.49235534667969,
      "activations/layer4_attention_weight_max": 87.1794204711914,
      "activations/layer4_attention_weight_min": -86.87421417236328,
      "activations/layer5_attention_weight_max": 63.72486877441406,
      "activations/layer5_attention_weight_min": -73.00010681152344,
      "activations/layer6_attention_weight_max": 49.3843879699707,
      "activations/layer6_attention_weight_min": -51.545040130615234,
      "activations/layer7_attention_weight_max": 66.48451232910156,
      "activations/layer7_attention_weight_min": -65.00509643554688,
      "activations/layer8_attention_weight_max": 43.307132720947266,
      "activations/layer8_attention_weight_min": -49.385921478271484,
      "activations/layer9_attention_weight_max": 45.606868743896484,
      "activations/layer9_attention_weight_min": -46.10715866088867,
      "epoch": 16.73,
      "learning_rate": 4.257007575757575e-05,
      "loss": 2.7427,
      "step": 287850
    },
    {
      "activations/layer0_attention_weight_max": 16.09501838684082,
      "activations/layer0_attention_weight_min": -12.962748527526855,
      "activations/layer10_attention_weight_max": 34.021663665771484,
      "activations/layer10_attention_weight_min": -34.54409408569336,
      "activations/layer11_attention_weight_max": 32.885986328125,
      "activations/layer11_attention_weight_min": -32.19618225097656,
      "activations/layer12_attention_weight_max": 20.67326545715332,
      "activations/layer12_attention_weight_min": -27.304609298706055,
      "activations/layer13_attention_weight_max": 34.191871643066406,
      "activations/layer13_attention_weight_min": -26.70987892150879,
      "activations/layer14_attention_weight_max": 37.29353332519531,
      "activations/layer14_attention_weight_min": -29.461620330810547,
      "activations/layer15_attention_weight_max": 40.397705078125,
      "activations/layer15_attention_weight_min": -29.481901168823242,
      "activations/layer16_attention_weight_max": 32.95139694213867,
      "activations/layer16_attention_weight_min": -33.59079360961914,
      "activations/layer17_attention_weight_max": 49.22637939453125,
      "activations/layer17_attention_weight_min": -47.48767852783203,
      "activations/layer18_attention_weight_max": 44.9176139831543,
      "activations/layer18_attention_weight_min": -40.209354400634766,
      "activations/layer19_attention_weight_max": 22.827402114868164,
      "activations/layer19_attention_weight_min": -21.43897247314453,
      "activations/layer1_attention_weight_max": 15.77238941192627,
      "activations/layer1_attention_weight_min": -13.547147750854492,
      "activations/layer20_attention_weight_max": 26.01445770263672,
      "activations/layer20_attention_weight_min": -19.87639808654785,
      "activations/layer21_attention_weight_max": 37.467933654785156,
      "activations/layer21_attention_weight_min": -23.746685028076172,
      "activations/layer22_attention_weight_max": 31.21249008178711,
      "activations/layer22_attention_weight_min": -23.519359588623047,
      "activations/layer23_attention_weight_max": 33.341426849365234,
      "activations/layer23_attention_weight_min": -25.637187957763672,
      "activations/layer2_attention_weight_max": 31.822216033935547,
      "activations/layer2_attention_weight_min": -30.974767684936523,
      "activations/layer3_attention_weight_max": 89.50506591796875,
      "activations/layer3_attention_weight_min": -91.09210205078125,
      "activations/layer4_attention_weight_max": 90.61610412597656,
      "activations/layer4_attention_weight_min": -93.51826477050781,
      "activations/layer5_attention_weight_max": 67.8822021484375,
      "activations/layer5_attention_weight_min": -77.7415542602539,
      "activations/layer6_attention_weight_max": 50.06785583496094,
      "activations/layer6_attention_weight_min": -51.876731872558594,
      "activations/layer7_attention_weight_max": 69.66909790039062,
      "activations/layer7_attention_weight_min": -65.34191131591797,
      "activations/layer8_attention_weight_max": 47.17613983154297,
      "activations/layer8_attention_weight_min": -47.5611457824707,
      "activations/layer9_attention_weight_max": 57.466487884521484,
      "activations/layer9_attention_weight_min": -47.53256607055664,
      "epoch": 16.73,
      "learning_rate": 4.2551136363636365e-05,
      "loss": 2.7327,
      "step": 287900
    },
    {
      "activations/layer0_attention_weight_max": 15.239261627197266,
      "activations/layer0_attention_weight_min": -12.965224266052246,
      "activations/layer10_attention_weight_max": 36.9902229309082,
      "activations/layer10_attention_weight_min": -38.40789031982422,
      "activations/layer11_attention_weight_max": 33.63725280761719,
      "activations/layer11_attention_weight_min": -33.33750534057617,
      "activations/layer12_attention_weight_max": 19.834693908691406,
      "activations/layer12_attention_weight_min": -25.966754913330078,
      "activations/layer13_attention_weight_max": 35.61021041870117,
      "activations/layer13_attention_weight_min": -29.025142669677734,
      "activations/layer14_attention_weight_max": 38.59928512573242,
      "activations/layer14_attention_weight_min": -31.29667854309082,
      "activations/layer15_attention_weight_max": 35.57670593261719,
      "activations/layer15_attention_weight_min": -32.032955169677734,
      "activations/layer16_attention_weight_max": 35.613643646240234,
      "activations/layer16_attention_weight_min": -32.26028060913086,
      "activations/layer17_attention_weight_max": 53.8348274230957,
      "activations/layer17_attention_weight_min": -45.947181701660156,
      "activations/layer18_attention_weight_max": 49.841392517089844,
      "activations/layer18_attention_weight_min": -40.42509460449219,
      "activations/layer19_attention_weight_max": 24.759031295776367,
      "activations/layer19_attention_weight_min": -22.02310562133789,
      "activations/layer1_attention_weight_max": 16.553857803344727,
      "activations/layer1_attention_weight_min": -14.100870132446289,
      "activations/layer20_attention_weight_max": 25.28270149230957,
      "activations/layer20_attention_weight_min": -19.513273239135742,
      "activations/layer21_attention_weight_max": 39.30449295043945,
      "activations/layer21_attention_weight_min": -20.4169979095459,
      "activations/layer22_attention_weight_max": 33.1448860168457,
      "activations/layer22_attention_weight_min": -25.207958221435547,
      "activations/layer23_attention_weight_max": 40.669647216796875,
      "activations/layer23_attention_weight_min": -24.609699249267578,
      "activations/layer2_attention_weight_max": 32.43534469604492,
      "activations/layer2_attention_weight_min": -32.231658935546875,
      "activations/layer3_attention_weight_max": 92.3568344116211,
      "activations/layer3_attention_weight_min": -95.9368896484375,
      "activations/layer4_attention_weight_max": 92.62410736083984,
      "activations/layer4_attention_weight_min": -93.46764373779297,
      "activations/layer5_attention_weight_max": 68.14110565185547,
      "activations/layer5_attention_weight_min": -73.21455383300781,
      "activations/layer6_attention_weight_max": 51.220516204833984,
      "activations/layer6_attention_weight_min": -51.46177291870117,
      "activations/layer7_attention_weight_max": 64.80892944335938,
      "activations/layer7_attention_weight_min": -65.0759048461914,
      "activations/layer8_attention_weight_max": 49.02642822265625,
      "activations/layer8_attention_weight_min": -52.90156555175781,
      "activations/layer9_attention_weight_max": 47.34197998046875,
      "activations/layer9_attention_weight_min": -50.20878219604492,
      "epoch": 16.73,
      "learning_rate": 4.253219696969697e-05,
      "loss": 2.7581,
      "step": 287950
    },
    {
      "activations/layer0_attention_weight_max": 15.74439811706543,
      "activations/layer0_attention_weight_min": -13.439078330993652,
      "activations/layer10_attention_weight_max": 41.80237579345703,
      "activations/layer10_attention_weight_min": -40.06635284423828,
      "activations/layer11_attention_weight_max": 39.01075744628906,
      "activations/layer11_attention_weight_min": -37.03921127319336,
      "activations/layer12_attention_weight_max": 19.765518188476562,
      "activations/layer12_attention_weight_min": -26.000940322875977,
      "activations/layer13_attention_weight_max": 61.13290786743164,
      "activations/layer13_attention_weight_min": -30.29520606994629,
      "activations/layer14_attention_weight_max": 55.73888397216797,
      "activations/layer14_attention_weight_min": -29.97342300415039,
      "activations/layer15_attention_weight_max": 49.18477249145508,
      "activations/layer15_attention_weight_min": -29.78512191772461,
      "activations/layer16_attention_weight_max": 38.24953842163086,
      "activations/layer16_attention_weight_min": -33.95175552368164,
      "activations/layer17_attention_weight_max": 57.99386978149414,
      "activations/layer17_attention_weight_min": -48.779476165771484,
      "activations/layer18_attention_weight_max": 50.92619705200195,
      "activations/layer18_attention_weight_min": -42.035945892333984,
      "activations/layer19_attention_weight_max": 26.23722267150879,
      "activations/layer19_attention_weight_min": -24.04753875732422,
      "activations/layer1_attention_weight_max": 15.901968955993652,
      "activations/layer1_attention_weight_min": -14.38635540008545,
      "activations/layer20_attention_weight_max": 28.82988929748535,
      "activations/layer20_attention_weight_min": -21.34097671508789,
      "activations/layer21_attention_weight_max": 52.45806121826172,
      "activations/layer21_attention_weight_min": -25.409067153930664,
      "activations/layer22_attention_weight_max": 37.46756362915039,
      "activations/layer22_attention_weight_min": -25.55074691772461,
      "activations/layer23_attention_weight_max": 39.64348602294922,
      "activations/layer23_attention_weight_min": -23.271282196044922,
      "activations/layer2_attention_weight_max": 33.13653564453125,
      "activations/layer2_attention_weight_min": -32.72931671142578,
      "activations/layer3_attention_weight_max": 94.47721099853516,
      "activations/layer3_attention_weight_min": -97.21471405029297,
      "activations/layer4_attention_weight_max": 93.55899810791016,
      "activations/layer4_attention_weight_min": -91.3632583618164,
      "activations/layer5_attention_weight_max": 67.55970001220703,
      "activations/layer5_attention_weight_min": -72.91191101074219,
      "activations/layer6_attention_weight_max": 56.04034423828125,
      "activations/layer6_attention_weight_min": -54.16011428833008,
      "activations/layer7_attention_weight_max": 73.8404769897461,
      "activations/layer7_attention_weight_min": -66.07586669921875,
      "activations/layer8_attention_weight_max": 51.46367263793945,
      "activations/layer8_attention_weight_min": -52.87003707885742,
      "activations/layer9_attention_weight_max": 48.88373947143555,
      "activations/layer9_attention_weight_min": -48.72257614135742,
      "epoch": 16.73,
      "learning_rate": 4.251325757575757e-05,
      "loss": 2.7408,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4638,
      "eval_samples_per_second": 507.336,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4638,
      "eval_openwebtext_samples_per_second": 507.336,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_wikitext_loss": 2.935546875,
      "eval_wikitext_ppl": 18.83179895971962,
      "eval_wikitext_runtime": 1.966,
      "eval_wikitext_samples_per_second": 231.949,
      "step": 288000
    },
    {
      "epoch": 16.73,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.5366,
      "eval_lambada_samples_per_second": 510.558,
      "step": 288000
    },
    {
      "activations/layer0_attention_weight_max": 16.780065536499023,
      "activations/layer0_attention_weight_min": -13.472643852233887,
      "activations/layer10_attention_weight_max": 40.6441650390625,
      "activations/layer10_attention_weight_min": -40.85523986816406,
      "activations/layer11_attention_weight_max": 35.495094299316406,
      "activations/layer11_attention_weight_min": -36.306880950927734,
      "activations/layer12_attention_weight_max": 19.220117568969727,
      "activations/layer12_attention_weight_min": -23.02480697631836,
      "activations/layer13_attention_weight_max": 32.31768798828125,
      "activations/layer13_attention_weight_min": -26.0711612701416,
      "activations/layer14_attention_weight_max": 35.72951126098633,
      "activations/layer14_attention_weight_min": -30.625375747680664,
      "activations/layer15_attention_weight_max": 32.70817184448242,
      "activations/layer15_attention_weight_min": -31.802661895751953,
      "activations/layer16_attention_weight_max": 32.671443939208984,
      "activations/layer16_attention_weight_min": -30.527551651000977,
      "activations/layer17_attention_weight_max": 51.753692626953125,
      "activations/layer17_attention_weight_min": -44.58885955810547,
      "activations/layer18_attention_weight_max": 46.3433952331543,
      "activations/layer18_attention_weight_min": -37.723968505859375,
      "activations/layer19_attention_weight_max": 23.87716293334961,
      "activations/layer19_attention_weight_min": -22.31587791442871,
      "activations/layer1_attention_weight_max": 15.539566993713379,
      "activations/layer1_attention_weight_min": -13.620349884033203,
      "activations/layer20_attention_weight_max": 22.365215301513672,
      "activations/layer20_attention_weight_min": -20.434083938598633,
      "activations/layer21_attention_weight_max": 36.75153350830078,
      "activations/layer21_attention_weight_min": -24.312448501586914,
      "activations/layer22_attention_weight_max": 29.287338256835938,
      "activations/layer22_attention_weight_min": -24.580949783325195,
      "activations/layer23_attention_weight_max": 37.91456604003906,
      "activations/layer23_attention_weight_min": -23.378089904785156,
      "activations/layer2_attention_weight_max": 34.743431091308594,
      "activations/layer2_attention_weight_min": -32.12271499633789,
      "activations/layer3_attention_weight_max": 93.19042205810547,
      "activations/layer3_attention_weight_min": -93.60517120361328,
      "activations/layer4_attention_weight_max": 91.47498321533203,
      "activations/layer4_attention_weight_min": -90.03819274902344,
      "activations/layer5_attention_weight_max": 67.30387878417969,
      "activations/layer5_attention_weight_min": -73.85415649414062,
      "activations/layer6_attention_weight_max": 51.39623260498047,
      "activations/layer6_attention_weight_min": -55.88246536254883,
      "activations/layer7_attention_weight_max": 64.3311538696289,
      "activations/layer7_attention_weight_min": -72.79275512695312,
      "activations/layer8_attention_weight_max": 47.51912307739258,
      "activations/layer8_attention_weight_min": -48.2797737121582,
      "activations/layer9_attention_weight_max": 45.30564498901367,
      "activations/layer9_attention_weight_min": -47.09907150268555,
      "epoch": 16.74,
      "learning_rate": 4.2494318181818184e-05,
      "loss": 2.7444,
      "step": 288050
    },
    {
      "activations/layer0_attention_weight_max": 16.521493911743164,
      "activations/layer0_attention_weight_min": -13.472434043884277,
      "activations/layer10_attention_weight_max": 36.319210052490234,
      "activations/layer10_attention_weight_min": -35.04779052734375,
      "activations/layer11_attention_weight_max": 34.028968811035156,
      "activations/layer11_attention_weight_min": -33.82257080078125,
      "activations/layer12_attention_weight_max": 19.64885139465332,
      "activations/layer12_attention_weight_min": -39.249996185302734,
      "activations/layer13_attention_weight_max": 40.81112289428711,
      "activations/layer13_attention_weight_min": -31.81622886657715,
      "activations/layer14_attention_weight_max": 34.791351318359375,
      "activations/layer14_attention_weight_min": -30.82337760925293,
      "activations/layer15_attention_weight_max": 33.9407958984375,
      "activations/layer15_attention_weight_min": -29.953147888183594,
      "activations/layer16_attention_weight_max": 31.545665740966797,
      "activations/layer16_attention_weight_min": -30.41728973388672,
      "activations/layer17_attention_weight_max": 48.62076950073242,
      "activations/layer17_attention_weight_min": -43.388912200927734,
      "activations/layer18_attention_weight_max": 43.877166748046875,
      "activations/layer18_attention_weight_min": -38.605743408203125,
      "activations/layer19_attention_weight_max": 24.86278533935547,
      "activations/layer19_attention_weight_min": -24.774940490722656,
      "activations/layer1_attention_weight_max": 16.84540557861328,
      "activations/layer1_attention_weight_min": -14.213547706604004,
      "activations/layer20_attention_weight_max": 21.54796600341797,
      "activations/layer20_attention_weight_min": -23.573274612426758,
      "activations/layer21_attention_weight_max": 37.25156021118164,
      "activations/layer21_attention_weight_min": -24.568021774291992,
      "activations/layer22_attention_weight_max": 30.745529174804688,
      "activations/layer22_attention_weight_min": -26.500574111938477,
      "activations/layer23_attention_weight_max": 36.47608947753906,
      "activations/layer23_attention_weight_min": -26.73067283630371,
      "activations/layer2_attention_weight_max": 33.018653869628906,
      "activations/layer2_attention_weight_min": -33.27013397216797,
      "activations/layer3_attention_weight_max": 89.9627685546875,
      "activations/layer3_attention_weight_min": -91.8392105102539,
      "activations/layer4_attention_weight_max": 88.78058624267578,
      "activations/layer4_attention_weight_min": -89.4781723022461,
      "activations/layer5_attention_weight_max": 63.689300537109375,
      "activations/layer5_attention_weight_min": -69.54356384277344,
      "activations/layer6_attention_weight_max": 52.02031326293945,
      "activations/layer6_attention_weight_min": -51.4487419128418,
      "activations/layer7_attention_weight_max": 64.71427154541016,
      "activations/layer7_attention_weight_min": -61.696197509765625,
      "activations/layer8_attention_weight_max": 46.790191650390625,
      "activations/layer8_attention_weight_min": -45.44353103637695,
      "activations/layer9_attention_weight_max": 51.58697509765625,
      "activations/layer9_attention_weight_min": -44.71760559082031,
      "epoch": 16.74,
      "learning_rate": 4.2475378787878785e-05,
      "loss": 2.7433,
      "step": 288100
    },
    {
      "activations/layer0_attention_weight_max": 16.102970123291016,
      "activations/layer0_attention_weight_min": -13.118900299072266,
      "activations/layer10_attention_weight_max": 41.53369903564453,
      "activations/layer10_attention_weight_min": -39.578857421875,
      "activations/layer11_attention_weight_max": 37.097442626953125,
      "activations/layer11_attention_weight_min": -36.04459762573242,
      "activations/layer12_attention_weight_max": 19.296829223632812,
      "activations/layer12_attention_weight_min": -24.913305282592773,
      "activations/layer13_attention_weight_max": 49.706382751464844,
      "activations/layer13_attention_weight_min": -32.2791748046875,
      "activations/layer14_attention_weight_max": 55.49614715576172,
      "activations/layer14_attention_weight_min": -36.84370040893555,
      "activations/layer15_attention_weight_max": 47.77756118774414,
      "activations/layer15_attention_weight_min": -34.93026351928711,
      "activations/layer16_attention_weight_max": 40.74443054199219,
      "activations/layer16_attention_weight_min": -37.05716323852539,
      "activations/layer17_attention_weight_max": 62.74542236328125,
      "activations/layer17_attention_weight_min": -53.72057342529297,
      "activations/layer18_attention_weight_max": 56.817054748535156,
      "activations/layer18_attention_weight_min": -48.99529266357422,
      "activations/layer19_attention_weight_max": 29.112112045288086,
      "activations/layer19_attention_weight_min": -27.955047607421875,
      "activations/layer1_attention_weight_max": 16.148164749145508,
      "activations/layer1_attention_weight_min": -13.857614517211914,
      "activations/layer20_attention_weight_max": 31.861902236938477,
      "activations/layer20_attention_weight_min": -23.35649299621582,
      "activations/layer21_attention_weight_max": 55.106380462646484,
      "activations/layer21_attention_weight_min": -27.476816177368164,
      "activations/layer22_attention_weight_max": 37.89082717895508,
      "activations/layer22_attention_weight_min": -28.44927978515625,
      "activations/layer23_attention_weight_max": 43.599647521972656,
      "activations/layer23_attention_weight_min": -25.19054412841797,
      "activations/layer2_attention_weight_max": 35.18453598022461,
      "activations/layer2_attention_weight_min": -32.87751388549805,
      "activations/layer3_attention_weight_max": 94.61665344238281,
      "activations/layer3_attention_weight_min": -98.27762603759766,
      "activations/layer4_attention_weight_max": 92.94622802734375,
      "activations/layer4_attention_weight_min": -88.0699691772461,
      "activations/layer5_attention_weight_max": 64.63984680175781,
      "activations/layer5_attention_weight_min": -71.17507934570312,
      "activations/layer6_attention_weight_max": 49.59305953979492,
      "activations/layer6_attention_weight_min": -49.509952545166016,
      "activations/layer7_attention_weight_max": 75.90069580078125,
      "activations/layer7_attention_weight_min": -67.75006866455078,
      "activations/layer8_attention_weight_max": 53.20076370239258,
      "activations/layer8_attention_weight_min": -50.921199798583984,
      "activations/layer9_attention_weight_max": 55.996559143066406,
      "activations/layer9_attention_weight_min": -53.00647735595703,
      "epoch": 16.74,
      "learning_rate": 4.2456439393939386e-05,
      "loss": 2.7467,
      "step": 288150
    },
    {
      "activations/layer0_attention_weight_max": 15.396217346191406,
      "activations/layer0_attention_weight_min": -13.490287780761719,
      "activations/layer10_attention_weight_max": 35.037357330322266,
      "activations/layer10_attention_weight_min": -33.114688873291016,
      "activations/layer11_attention_weight_max": 32.91777038574219,
      "activations/layer11_attention_weight_min": -32.987796783447266,
      "activations/layer12_attention_weight_max": 17.92950439453125,
      "activations/layer12_attention_weight_min": -24.322099685668945,
      "activations/layer13_attention_weight_max": 33.5800666809082,
      "activations/layer13_attention_weight_min": -28.095664978027344,
      "activations/layer14_attention_weight_max": 35.02789306640625,
      "activations/layer14_attention_weight_min": -30.987333297729492,
      "activations/layer15_attention_weight_max": 32.054019927978516,
      "activations/layer15_attention_weight_min": -30.187503814697266,
      "activations/layer16_attention_weight_max": 31.071996688842773,
      "activations/layer16_attention_weight_min": -30.794878005981445,
      "activations/layer17_attention_weight_max": 45.151023864746094,
      "activations/layer17_attention_weight_min": -42.29035186767578,
      "activations/layer18_attention_weight_max": 43.436100006103516,
      "activations/layer18_attention_weight_min": -37.224342346191406,
      "activations/layer19_attention_weight_max": 22.53643226623535,
      "activations/layer19_attention_weight_min": -22.0466251373291,
      "activations/layer1_attention_weight_max": 16.39478302001953,
      "activations/layer1_attention_weight_min": -13.701543807983398,
      "activations/layer20_attention_weight_max": 20.179719924926758,
      "activations/layer20_attention_weight_min": -21.350984573364258,
      "activations/layer21_attention_weight_max": 32.806190490722656,
      "activations/layer21_attention_weight_min": -24.37799072265625,
      "activations/layer22_attention_weight_max": 28.708127975463867,
      "activations/layer22_attention_weight_min": -28.033613204956055,
      "activations/layer23_attention_weight_max": 34.6005973815918,
      "activations/layer23_attention_weight_min": -26.375263214111328,
      "activations/layer2_attention_weight_max": 31.314939498901367,
      "activations/layer2_attention_weight_min": -31.896099090576172,
      "activations/layer3_attention_weight_max": 83.57096862792969,
      "activations/layer3_attention_weight_min": -92.26222229003906,
      "activations/layer4_attention_weight_max": 86.637451171875,
      "activations/layer4_attention_weight_min": -83.3250732421875,
      "activations/layer5_attention_weight_max": 65.4090576171875,
      "activations/layer5_attention_weight_min": -73.38115692138672,
      "activations/layer6_attention_weight_max": 49.09293746948242,
      "activations/layer6_attention_weight_min": -47.71997833251953,
      "activations/layer7_attention_weight_max": 63.32625961303711,
      "activations/layer7_attention_weight_min": -58.584529876708984,
      "activations/layer8_attention_weight_max": 44.4783821105957,
      "activations/layer8_attention_weight_min": -46.073829650878906,
      "activations/layer9_attention_weight_max": 42.842315673828125,
      "activations/layer9_attention_weight_min": -43.15137481689453,
      "epoch": 16.75,
      "learning_rate": 4.2437499999999995e-05,
      "loss": 2.7504,
      "step": 288200
    },
    {
      "activations/layer0_attention_weight_max": 15.907949447631836,
      "activations/layer0_attention_weight_min": -13.275580406188965,
      "activations/layer10_attention_weight_max": 38.78546142578125,
      "activations/layer10_attention_weight_min": -39.00698471069336,
      "activations/layer11_attention_weight_max": 34.049095153808594,
      "activations/layer11_attention_weight_min": -35.720123291015625,
      "activations/layer12_attention_weight_max": 19.837575912475586,
      "activations/layer12_attention_weight_min": -26.232114791870117,
      "activations/layer13_attention_weight_max": 44.46120071411133,
      "activations/layer13_attention_weight_min": -32.35877990722656,
      "activations/layer14_attention_weight_max": 35.08057403564453,
      "activations/layer14_attention_weight_min": -30.137121200561523,
      "activations/layer15_attention_weight_max": 33.4268798828125,
      "activations/layer15_attention_weight_min": -29.988906860351562,
      "activations/layer16_attention_weight_max": 34.501705169677734,
      "activations/layer16_attention_weight_min": -31.053625106811523,
      "activations/layer17_attention_weight_max": 50.880435943603516,
      "activations/layer17_attention_weight_min": -48.50947952270508,
      "activations/layer18_attention_weight_max": 44.15253448486328,
      "activations/layer18_attention_weight_min": -46.761470794677734,
      "activations/layer19_attention_weight_max": 23.65304946899414,
      "activations/layer19_attention_weight_min": -25.83485984802246,
      "activations/layer1_attention_weight_max": 16.37580680847168,
      "activations/layer1_attention_weight_min": -15.112920761108398,
      "activations/layer20_attention_weight_max": 23.469928741455078,
      "activations/layer20_attention_weight_min": -22.30953025817871,
      "activations/layer21_attention_weight_max": 34.25157928466797,
      "activations/layer21_attention_weight_min": -24.28729248046875,
      "activations/layer22_attention_weight_max": 29.066131591796875,
      "activations/layer22_attention_weight_min": -27.140609741210938,
      "activations/layer23_attention_weight_max": 38.325687408447266,
      "activations/layer23_attention_weight_min": -25.762908935546875,
      "activations/layer2_attention_weight_max": 32.002769470214844,
      "activations/layer2_attention_weight_min": -31.22127342224121,
      "activations/layer3_attention_weight_max": 90.83915710449219,
      "activations/layer3_attention_weight_min": -89.49951934814453,
      "activations/layer4_attention_weight_max": 89.29244232177734,
      "activations/layer4_attention_weight_min": -86.52293395996094,
      "activations/layer5_attention_weight_max": 66.41761016845703,
      "activations/layer5_attention_weight_min": -74.14701080322266,
      "activations/layer6_attention_weight_max": 50.55592346191406,
      "activations/layer6_attention_weight_min": -50.0031852722168,
      "activations/layer7_attention_weight_max": 64.95114135742188,
      "activations/layer7_attention_weight_min": -64.43656921386719,
      "activations/layer8_attention_weight_max": 47.35779571533203,
      "activations/layer8_attention_weight_min": -53.480812072753906,
      "activations/layer9_attention_weight_max": 45.37336349487305,
      "activations/layer9_attention_weight_min": -48.13235855102539,
      "epoch": 16.75,
      "learning_rate": 4.24185606060606e-05,
      "loss": 2.7482,
      "step": 288250
    },
    {
      "activations/layer0_attention_weight_max": 17.101572036743164,
      "activations/layer0_attention_weight_min": -13.486757278442383,
      "activations/layer10_attention_weight_max": 35.200782775878906,
      "activations/layer10_attention_weight_min": -35.866477966308594,
      "activations/layer11_attention_weight_max": 32.62877655029297,
      "activations/layer11_attention_weight_min": -32.81883239746094,
      "activations/layer12_attention_weight_max": 18.906675338745117,
      "activations/layer12_attention_weight_min": -23.61094093322754,
      "activations/layer13_attention_weight_max": 31.445695877075195,
      "activations/layer13_attention_weight_min": -26.647159576416016,
      "activations/layer14_attention_weight_max": 34.449806213378906,
      "activations/layer14_attention_weight_min": -29.407926559448242,
      "activations/layer15_attention_weight_max": 32.87796401977539,
      "activations/layer15_attention_weight_min": -29.687162399291992,
      "activations/layer16_attention_weight_max": 31.254735946655273,
      "activations/layer16_attention_weight_min": -29.266889572143555,
      "activations/layer17_attention_weight_max": 47.180877685546875,
      "activations/layer17_attention_weight_min": -42.81075668334961,
      "activations/layer18_attention_weight_max": 43.04113006591797,
      "activations/layer18_attention_weight_min": -37.82634735107422,
      "activations/layer19_attention_weight_max": 24.360027313232422,
      "activations/layer19_attention_weight_min": -24.224864959716797,
      "activations/layer1_attention_weight_max": 15.215527534484863,
      "activations/layer1_attention_weight_min": -13.4935884475708,
      "activations/layer20_attention_weight_max": 22.1269588470459,
      "activations/layer20_attention_weight_min": -26.808547973632812,
      "activations/layer21_attention_weight_max": 31.48076057434082,
      "activations/layer21_attention_weight_min": -20.942323684692383,
      "activations/layer22_attention_weight_max": 28.63316535949707,
      "activations/layer22_attention_weight_min": -26.920759201049805,
      "activations/layer23_attention_weight_max": 37.90655517578125,
      "activations/layer23_attention_weight_min": -24.53879165649414,
      "activations/layer2_attention_weight_max": 31.877349853515625,
      "activations/layer2_attention_weight_min": -31.49695587158203,
      "activations/layer3_attention_weight_max": 89.79856872558594,
      "activations/layer3_attention_weight_min": -91.42456817626953,
      "activations/layer4_attention_weight_max": 88.75745391845703,
      "activations/layer4_attention_weight_min": -89.11978912353516,
      "activations/layer5_attention_weight_max": 65.71626281738281,
      "activations/layer5_attention_weight_min": -72.67772674560547,
      "activations/layer6_attention_weight_max": 49.045536041259766,
      "activations/layer6_attention_weight_min": -51.23464584350586,
      "activations/layer7_attention_weight_max": 63.63960266113281,
      "activations/layer7_attention_weight_min": -61.123321533203125,
      "activations/layer8_attention_weight_max": 46.30867004394531,
      "activations/layer8_attention_weight_min": -50.170631408691406,
      "activations/layer9_attention_weight_max": 49.66596603393555,
      "activations/layer9_attention_weight_min": -46.31621170043945,
      "epoch": 16.75,
      "learning_rate": 4.239962121212121e-05,
      "loss": 2.7378,
      "step": 288300
    },
    {
      "activations/layer0_attention_weight_max": 15.323089599609375,
      "activations/layer0_attention_weight_min": -13.08963394165039,
      "activations/layer10_attention_weight_max": 37.83940505981445,
      "activations/layer10_attention_weight_min": -37.32198715209961,
      "activations/layer11_attention_weight_max": 34.54682540893555,
      "activations/layer11_attention_weight_min": -35.971710205078125,
      "activations/layer12_attention_weight_max": 18.922210693359375,
      "activations/layer12_attention_weight_min": -26.817476272583008,
      "activations/layer13_attention_weight_max": 34.61175537109375,
      "activations/layer13_attention_weight_min": -29.08422088623047,
      "activations/layer14_attention_weight_max": 35.389122009277344,
      "activations/layer14_attention_weight_min": -31.73484230041504,
      "activations/layer15_attention_weight_max": 33.845672607421875,
      "activations/layer15_attention_weight_min": -30.280942916870117,
      "activations/layer16_attention_weight_max": 34.36668014526367,
      "activations/layer16_attention_weight_min": -32.55492401123047,
      "activations/layer17_attention_weight_max": 50.686100006103516,
      "activations/layer17_attention_weight_min": -45.05704116821289,
      "activations/layer18_attention_weight_max": 45.794769287109375,
      "activations/layer18_attention_weight_min": -39.326316833496094,
      "activations/layer19_attention_weight_max": 23.33635711669922,
      "activations/layer19_attention_weight_min": -22.981327056884766,
      "activations/layer1_attention_weight_max": 15.48106861114502,
      "activations/layer1_attention_weight_min": -14.292966842651367,
      "activations/layer20_attention_weight_max": 23.36760711669922,
      "activations/layer20_attention_weight_min": -22.412858963012695,
      "activations/layer21_attention_weight_max": 36.13994216918945,
      "activations/layer21_attention_weight_min": -20.615375518798828,
      "activations/layer22_attention_weight_max": 27.94245147705078,
      "activations/layer22_attention_weight_min": -27.779361724853516,
      "activations/layer23_attention_weight_max": 38.67898941040039,
      "activations/layer23_attention_weight_min": -24.39456558227539,
      "activations/layer2_attention_weight_max": 33.09953308105469,
      "activations/layer2_attention_weight_min": -30.693693161010742,
      "activations/layer3_attention_weight_max": 91.20633697509766,
      "activations/layer3_attention_weight_min": -88.23677062988281,
      "activations/layer4_attention_weight_max": 93.38545989990234,
      "activations/layer4_attention_weight_min": -90.23881530761719,
      "activations/layer5_attention_weight_max": 65.83349609375,
      "activations/layer5_attention_weight_min": -71.34152221679688,
      "activations/layer6_attention_weight_max": 52.591243743896484,
      "activations/layer6_attention_weight_min": -49.60035705566406,
      "activations/layer7_attention_weight_max": 63.1097526550293,
      "activations/layer7_attention_weight_min": -60.927345275878906,
      "activations/layer8_attention_weight_max": 48.40087890625,
      "activations/layer8_attention_weight_min": -51.73945617675781,
      "activations/layer9_attention_weight_max": 44.95579147338867,
      "activations/layer9_attention_weight_min": -45.07102584838867,
      "epoch": 16.75,
      "learning_rate": 4.238068181818181e-05,
      "loss": 2.736,
      "step": 288350
    },
    {
      "activations/layer0_attention_weight_max": 16.698118209838867,
      "activations/layer0_attention_weight_min": -13.526647567749023,
      "activations/layer10_attention_weight_max": 37.808937072753906,
      "activations/layer10_attention_weight_min": -35.67002868652344,
      "activations/layer11_attention_weight_max": 35.243919372558594,
      "activations/layer11_attention_weight_min": -33.185794830322266,
      "activations/layer12_attention_weight_max": 18.047046661376953,
      "activations/layer12_attention_weight_min": -29.549806594848633,
      "activations/layer13_attention_weight_max": 35.497745513916016,
      "activations/layer13_attention_weight_min": -32.16187286376953,
      "activations/layer14_attention_weight_max": 40.601619720458984,
      "activations/layer14_attention_weight_min": -31.11419105529785,
      "activations/layer15_attention_weight_max": 35.67636489868164,
      "activations/layer15_attention_weight_min": -29.480052947998047,
      "activations/layer16_attention_weight_max": 33.75040054321289,
      "activations/layer16_attention_weight_min": -33.209999084472656,
      "activations/layer17_attention_weight_max": 48.499149322509766,
      "activations/layer17_attention_weight_min": -45.2337532043457,
      "activations/layer18_attention_weight_max": 46.009708404541016,
      "activations/layer18_attention_weight_min": -39.84964370727539,
      "activations/layer19_attention_weight_max": 23.90045166015625,
      "activations/layer19_attention_weight_min": -21.624954223632812,
      "activations/layer1_attention_weight_max": 16.226465225219727,
      "activations/layer1_attention_weight_min": -14.824755668640137,
      "activations/layer20_attention_weight_max": 23.67973518371582,
      "activations/layer20_attention_weight_min": -22.050445556640625,
      "activations/layer21_attention_weight_max": 36.21998596191406,
      "activations/layer21_attention_weight_min": -23.29613494873047,
      "activations/layer22_attention_weight_max": 29.43673324584961,
      "activations/layer22_attention_weight_min": -27.004283905029297,
      "activations/layer23_attention_weight_max": 39.47687530517578,
      "activations/layer23_attention_weight_min": -25.78075408935547,
      "activations/layer2_attention_weight_max": 32.096256256103516,
      "activations/layer2_attention_weight_min": -31.217832565307617,
      "activations/layer3_attention_weight_max": 91.85748291015625,
      "activations/layer3_attention_weight_min": -94.79212951660156,
      "activations/layer4_attention_weight_max": 91.03253173828125,
      "activations/layer4_attention_weight_min": -87.34745025634766,
      "activations/layer5_attention_weight_max": 67.51710510253906,
      "activations/layer5_attention_weight_min": -69.93944549560547,
      "activations/layer6_attention_weight_max": 49.20273971557617,
      "activations/layer6_attention_weight_min": -50.69533157348633,
      "activations/layer7_attention_weight_max": 62.6017951965332,
      "activations/layer7_attention_weight_min": -63.21222686767578,
      "activations/layer8_attention_weight_max": 46.47856521606445,
      "activations/layer8_attention_weight_min": -48.50855255126953,
      "activations/layer9_attention_weight_max": 45.917598724365234,
      "activations/layer9_attention_weight_min": -46.469112396240234,
      "epoch": 16.76,
      "learning_rate": 4.2361742424242414e-05,
      "loss": 2.7582,
      "step": 288400
    },
    {
      "activations/layer0_attention_weight_max": 15.63692855834961,
      "activations/layer0_attention_weight_min": -13.593425750732422,
      "activations/layer10_attention_weight_max": 35.615631103515625,
      "activations/layer10_attention_weight_min": -34.232032775878906,
      "activations/layer11_attention_weight_max": 30.49518394470215,
      "activations/layer11_attention_weight_min": -31.340185165405273,
      "activations/layer12_attention_weight_max": 18.746076583862305,
      "activations/layer12_attention_weight_min": -26.900806427001953,
      "activations/layer13_attention_weight_max": 34.36517333984375,
      "activations/layer13_attention_weight_min": -25.18736457824707,
      "activations/layer14_attention_weight_max": 34.0821418762207,
      "activations/layer14_attention_weight_min": -30.719417572021484,
      "activations/layer15_attention_weight_max": 35.216617584228516,
      "activations/layer15_attention_weight_min": -29.427310943603516,
      "activations/layer16_attention_weight_max": 37.357666015625,
      "activations/layer16_attention_weight_min": -34.170684814453125,
      "activations/layer17_attention_weight_max": 53.97468948364258,
      "activations/layer17_attention_weight_min": -46.746620178222656,
      "activations/layer18_attention_weight_max": 51.30055618286133,
      "activations/layer18_attention_weight_min": -40.2460823059082,
      "activations/layer19_attention_weight_max": 24.30449867248535,
      "activations/layer19_attention_weight_min": -22.612680435180664,
      "activations/layer1_attention_weight_max": 15.25056266784668,
      "activations/layer1_attention_weight_min": -15.396781921386719,
      "activations/layer20_attention_weight_max": 22.50956153869629,
      "activations/layer20_attention_weight_min": -21.56328010559082,
      "activations/layer21_attention_weight_max": 34.63180160522461,
      "activations/layer21_attention_weight_min": -23.20550537109375,
      "activations/layer22_attention_weight_max": 33.03776931762695,
      "activations/layer22_attention_weight_min": -25.482086181640625,
      "activations/layer23_attention_weight_max": 37.71045684814453,
      "activations/layer23_attention_weight_min": -26.232358932495117,
      "activations/layer2_attention_weight_max": 30.478656768798828,
      "activations/layer2_attention_weight_min": -29.400190353393555,
      "activations/layer3_attention_weight_max": 87.2191390991211,
      "activations/layer3_attention_weight_min": -87.57878875732422,
      "activations/layer4_attention_weight_max": 84.79032897949219,
      "activations/layer4_attention_weight_min": -85.44741821289062,
      "activations/layer5_attention_weight_max": 63.338558197021484,
      "activations/layer5_attention_weight_min": -73.21408081054688,
      "activations/layer6_attention_weight_max": 48.651309967041016,
      "activations/layer6_attention_weight_min": -48.16731262207031,
      "activations/layer7_attention_weight_max": 62.71134567260742,
      "activations/layer7_attention_weight_min": -60.53008270263672,
      "activations/layer8_attention_weight_max": 45.39497375488281,
      "activations/layer8_attention_weight_min": -45.6853141784668,
      "activations/layer9_attention_weight_max": 40.73276901245117,
      "activations/layer9_attention_weight_min": -43.67586135864258,
      "epoch": 16.76,
      "learning_rate": 4.234280303030303e-05,
      "loss": 2.7471,
      "step": 288450
    },
    {
      "activations/layer0_attention_weight_max": 16.07573699951172,
      "activations/layer0_attention_weight_min": -13.729682922363281,
      "activations/layer10_attention_weight_max": 41.59663009643555,
      "activations/layer10_attention_weight_min": -40.121463775634766,
      "activations/layer11_attention_weight_max": 37.433677673339844,
      "activations/layer11_attention_weight_min": -35.038204193115234,
      "activations/layer12_attention_weight_max": 21.11152458190918,
      "activations/layer12_attention_weight_min": -26.183712005615234,
      "activations/layer13_attention_weight_max": 42.064666748046875,
      "activations/layer13_attention_weight_min": -30.85744857788086,
      "activations/layer14_attention_weight_max": 40.225460052490234,
      "activations/layer14_attention_weight_min": -32.9766845703125,
      "activations/layer15_attention_weight_max": 36.83932113647461,
      "activations/layer15_attention_weight_min": -31.611133575439453,
      "activations/layer16_attention_weight_max": 37.824058532714844,
      "activations/layer16_attention_weight_min": -35.68666458129883,
      "activations/layer17_attention_weight_max": 55.27830505371094,
      "activations/layer17_attention_weight_min": -50.097232818603516,
      "activations/layer18_attention_weight_max": 49.76917266845703,
      "activations/layer18_attention_weight_min": -44.776039123535156,
      "activations/layer19_attention_weight_max": 24.204748153686523,
      "activations/layer19_attention_weight_min": -24.70836639404297,
      "activations/layer1_attention_weight_max": 16.408533096313477,
      "activations/layer1_attention_weight_min": -14.929709434509277,
      "activations/layer20_attention_weight_max": 22.71534538269043,
      "activations/layer20_attention_weight_min": -21.831314086914062,
      "activations/layer21_attention_weight_max": 39.521297454833984,
      "activations/layer21_attention_weight_min": -26.2559814453125,
      "activations/layer22_attention_weight_max": 29.82021713256836,
      "activations/layer22_attention_weight_min": -30.46350860595703,
      "activations/layer23_attention_weight_max": 37.29482650756836,
      "activations/layer23_attention_weight_min": -26.001453399658203,
      "activations/layer2_attention_weight_max": 33.0271110534668,
      "activations/layer2_attention_weight_min": -31.903148651123047,
      "activations/layer3_attention_weight_max": 86.05341339111328,
      "activations/layer3_attention_weight_min": -89.3409652709961,
      "activations/layer4_attention_weight_max": 88.39876556396484,
      "activations/layer4_attention_weight_min": -86.8665542602539,
      "activations/layer5_attention_weight_max": 61.62980651855469,
      "activations/layer5_attention_weight_min": -71.1815185546875,
      "activations/layer6_attention_weight_max": 49.94673156738281,
      "activations/layer6_attention_weight_min": -49.63886260986328,
      "activations/layer7_attention_weight_max": 66.34671783447266,
      "activations/layer7_attention_weight_min": -66.87578582763672,
      "activations/layer8_attention_weight_max": 49.43074035644531,
      "activations/layer8_attention_weight_min": -50.79780960083008,
      "activations/layer9_attention_weight_max": 53.60329055786133,
      "activations/layer9_attention_weight_min": -49.03515625,
      "epoch": 16.76,
      "learning_rate": 4.232386363636363e-05,
      "loss": 2.7375,
      "step": 288500
    },
    {
      "activations/layer0_attention_weight_max": 15.944307327270508,
      "activations/layer0_attention_weight_min": -13.474464416503906,
      "activations/layer10_attention_weight_max": 32.860416412353516,
      "activations/layer10_attention_weight_min": -34.330902099609375,
      "activations/layer11_attention_weight_max": 32.145450592041016,
      "activations/layer11_attention_weight_min": -31.737707138061523,
      "activations/layer12_attention_weight_max": 19.40818214416504,
      "activations/layer12_attention_weight_min": -25.93743324279785,
      "activations/layer13_attention_weight_max": 37.8832893371582,
      "activations/layer13_attention_weight_min": -29.76300811767578,
      "activations/layer14_attention_weight_max": 37.53583526611328,
      "activations/layer14_attention_weight_min": -31.977312088012695,
      "activations/layer15_attention_weight_max": 34.38391876220703,
      "activations/layer15_attention_weight_min": -33.33999252319336,
      "activations/layer16_attention_weight_max": 34.20573043823242,
      "activations/layer16_attention_weight_min": -32.825469970703125,
      "activations/layer17_attention_weight_max": 53.39008331298828,
      "activations/layer17_attention_weight_min": -48.159671783447266,
      "activations/layer18_attention_weight_max": 50.233638763427734,
      "activations/layer18_attention_weight_min": -40.53069305419922,
      "activations/layer19_attention_weight_max": 24.37203598022461,
      "activations/layer19_attention_weight_min": -26.670608520507812,
      "activations/layer1_attention_weight_max": 16.244413375854492,
      "activations/layer1_attention_weight_min": -13.105745315551758,
      "activations/layer20_attention_weight_max": 23.537309646606445,
      "activations/layer20_attention_weight_min": -25.438152313232422,
      "activations/layer21_attention_weight_max": 46.4150276184082,
      "activations/layer21_attention_weight_min": -25.803359985351562,
      "activations/layer22_attention_weight_max": 31.485803604125977,
      "activations/layer22_attention_weight_min": -29.283607482910156,
      "activations/layer23_attention_weight_max": 40.190879821777344,
      "activations/layer23_attention_weight_min": -25.343658447265625,
      "activations/layer2_attention_weight_max": 31.564868927001953,
      "activations/layer2_attention_weight_min": -30.704404830932617,
      "activations/layer3_attention_weight_max": 90.03800964355469,
      "activations/layer3_attention_weight_min": -94.1263656616211,
      "activations/layer4_attention_weight_max": 85.98989868164062,
      "activations/layer4_attention_weight_min": -89.89832305908203,
      "activations/layer5_attention_weight_max": 62.89068603515625,
      "activations/layer5_attention_weight_min": -81.16644287109375,
      "activations/layer6_attention_weight_max": 49.002952575683594,
      "activations/layer6_attention_weight_min": -48.105342864990234,
      "activations/layer7_attention_weight_max": 59.07955551147461,
      "activations/layer7_attention_weight_min": -64.3126449584961,
      "activations/layer8_attention_weight_max": 43.848384857177734,
      "activations/layer8_attention_weight_min": -50.71539306640625,
      "activations/layer9_attention_weight_max": 43.19519805908203,
      "activations/layer9_attention_weight_min": -47.05241012573242,
      "epoch": 16.77,
      "learning_rate": 4.230492424242424e-05,
      "loss": 2.7533,
      "step": 288550
    },
    {
      "activations/layer0_attention_weight_max": 15.640849113464355,
      "activations/layer0_attention_weight_min": -13.661867141723633,
      "activations/layer10_attention_weight_max": 40.32685852050781,
      "activations/layer10_attention_weight_min": -37.30640411376953,
      "activations/layer11_attention_weight_max": 34.9727897644043,
      "activations/layer11_attention_weight_min": -35.31141662597656,
      "activations/layer12_attention_weight_max": 21.08257484436035,
      "activations/layer12_attention_weight_min": -26.55080795288086,
      "activations/layer13_attention_weight_max": 37.34177780151367,
      "activations/layer13_attention_weight_min": -28.72018814086914,
      "activations/layer14_attention_weight_max": 37.58115005493164,
      "activations/layer14_attention_weight_min": -29.545272827148438,
      "activations/layer15_attention_weight_max": 33.11088180541992,
      "activations/layer15_attention_weight_min": -29.557758331298828,
      "activations/layer16_attention_weight_max": 33.10024642944336,
      "activations/layer16_attention_weight_min": -31.839229583740234,
      "activations/layer17_attention_weight_max": 52.6868782043457,
      "activations/layer17_attention_weight_min": -42.850189208984375,
      "activations/layer18_attention_weight_max": 49.4143180847168,
      "activations/layer18_attention_weight_min": -38.37709045410156,
      "activations/layer19_attention_weight_max": 24.420625686645508,
      "activations/layer19_attention_weight_min": -21.948631286621094,
      "activations/layer1_attention_weight_max": 16.306447982788086,
      "activations/layer1_attention_weight_min": -14.392464637756348,
      "activations/layer20_attention_weight_max": 25.521223068237305,
      "activations/layer20_attention_weight_min": -19.3080997467041,
      "activations/layer21_attention_weight_max": 41.02043914794922,
      "activations/layer21_attention_weight_min": -19.189706802368164,
      "activations/layer22_attention_weight_max": 33.542057037353516,
      "activations/layer22_attention_weight_min": -24.461254119873047,
      "activations/layer23_attention_weight_max": 39.83214569091797,
      "activations/layer23_attention_weight_min": -24.611003875732422,
      "activations/layer2_attention_weight_max": 33.090362548828125,
      "activations/layer2_attention_weight_min": -32.491485595703125,
      "activations/layer3_attention_weight_max": 92.16842651367188,
      "activations/layer3_attention_weight_min": -93.41545867919922,
      "activations/layer4_attention_weight_max": 93.49764251708984,
      "activations/layer4_attention_weight_min": -90.37752532958984,
      "activations/layer5_attention_weight_max": 65.32647705078125,
      "activations/layer5_attention_weight_min": -74.1396713256836,
      "activations/layer6_attention_weight_max": 52.13621520996094,
      "activations/layer6_attention_weight_min": -50.95803451538086,
      "activations/layer7_attention_weight_max": 64.19640350341797,
      "activations/layer7_attention_weight_min": -64.90919494628906,
      "activations/layer8_attention_weight_max": 46.51216125488281,
      "activations/layer8_attention_weight_min": -48.89582061767578,
      "activations/layer9_attention_weight_max": 45.88839340209961,
      "activations/layer9_attention_weight_min": -46.5287971496582,
      "epoch": 16.77,
      "learning_rate": 4.228598484848484e-05,
      "loss": 2.733,
      "step": 288600
    },
    {
      "activations/layer0_attention_weight_max": 16.217370986938477,
      "activations/layer0_attention_weight_min": -13.458995819091797,
      "activations/layer10_attention_weight_max": 34.2454719543457,
      "activations/layer10_attention_weight_min": -34.219844818115234,
      "activations/layer11_attention_weight_max": 31.657875061035156,
      "activations/layer11_attention_weight_min": -31.938154220581055,
      "activations/layer12_attention_weight_max": 18.048215866088867,
      "activations/layer12_attention_weight_min": -25.842750549316406,
      "activations/layer13_attention_weight_max": 37.42789840698242,
      "activations/layer13_attention_weight_min": -29.100921630859375,
      "activations/layer14_attention_weight_max": 38.449501037597656,
      "activations/layer14_attention_weight_min": -31.951492309570312,
      "activations/layer15_attention_weight_max": 34.042762756347656,
      "activations/layer15_attention_weight_min": -31.79912567138672,
      "activations/layer16_attention_weight_max": 34.19577407836914,
      "activations/layer16_attention_weight_min": -33.122108459472656,
      "activations/layer17_attention_weight_max": 54.285621643066406,
      "activations/layer17_attention_weight_min": -46.270477294921875,
      "activations/layer18_attention_weight_max": 46.08291244506836,
      "activations/layer18_attention_weight_min": -41.33442687988281,
      "activations/layer19_attention_weight_max": 26.3116512298584,
      "activations/layer19_attention_weight_min": -23.52322006225586,
      "activations/layer1_attention_weight_max": 16.6026554107666,
      "activations/layer1_attention_weight_min": -13.36992073059082,
      "activations/layer20_attention_weight_max": 22.442045211791992,
      "activations/layer20_attention_weight_min": -22.294233322143555,
      "activations/layer21_attention_weight_max": 39.7942008972168,
      "activations/layer21_attention_weight_min": -24.316064834594727,
      "activations/layer22_attention_weight_max": 33.14028549194336,
      "activations/layer22_attention_weight_min": -25.710479736328125,
      "activations/layer23_attention_weight_max": 37.4631462097168,
      "activations/layer23_attention_weight_min": -26.336193084716797,
      "activations/layer2_attention_weight_max": 27.952980041503906,
      "activations/layer2_attention_weight_min": -27.845083236694336,
      "activations/layer3_attention_weight_max": 82.468994140625,
      "activations/layer3_attention_weight_min": -88.51154327392578,
      "activations/layer4_attention_weight_max": 83.39157104492188,
      "activations/layer4_attention_weight_min": -83.99529266357422,
      "activations/layer5_attention_weight_max": 63.91944122314453,
      "activations/layer5_attention_weight_min": -72.85334014892578,
      "activations/layer6_attention_weight_max": 47.6766242980957,
      "activations/layer6_attention_weight_min": -49.53091049194336,
      "activations/layer7_attention_weight_max": 65.21115112304688,
      "activations/layer7_attention_weight_min": -62.77885055541992,
      "activations/layer8_attention_weight_max": 43.612205505371094,
      "activations/layer8_attention_weight_min": -45.66147994995117,
      "activations/layer9_attention_weight_max": 43.51805877685547,
      "activations/layer9_attention_weight_min": -44.207149505615234,
      "epoch": 16.77,
      "learning_rate": 4.2267045454545456e-05,
      "loss": 2.7418,
      "step": 288650
    },
    {
      "activations/layer0_attention_weight_max": 16.384681701660156,
      "activations/layer0_attention_weight_min": -13.672630310058594,
      "activations/layer10_attention_weight_max": 40.740501403808594,
      "activations/layer10_attention_weight_min": -38.256195068359375,
      "activations/layer11_attention_weight_max": 35.361572265625,
      "activations/layer11_attention_weight_min": -36.4221305847168,
      "activations/layer12_attention_weight_max": 18.89642906188965,
      "activations/layer12_attention_weight_min": -26.200729370117188,
      "activations/layer13_attention_weight_max": 32.703514099121094,
      "activations/layer13_attention_weight_min": -30.738845825195312,
      "activations/layer14_attention_weight_max": 37.211143493652344,
      "activations/layer14_attention_weight_min": -30.861873626708984,
      "activations/layer15_attention_weight_max": 34.42265319824219,
      "activations/layer15_attention_weight_min": -31.144546508789062,
      "activations/layer16_attention_weight_max": 35.625057220458984,
      "activations/layer16_attention_weight_min": -31.87125587463379,
      "activations/layer17_attention_weight_max": 53.735191345214844,
      "activations/layer17_attention_weight_min": -47.36497116088867,
      "activations/layer18_attention_weight_max": 47.4295768737793,
      "activations/layer18_attention_weight_min": -39.27165222167969,
      "activations/layer19_attention_weight_max": 24.441205978393555,
      "activations/layer19_attention_weight_min": -23.580739974975586,
      "activations/layer1_attention_weight_max": 16.448440551757812,
      "activations/layer1_attention_weight_min": -13.741573333740234,
      "activations/layer20_attention_weight_max": 24.915014266967773,
      "activations/layer20_attention_weight_min": -21.6890926361084,
      "activations/layer21_attention_weight_max": 37.27041244506836,
      "activations/layer21_attention_weight_min": -24.709415435791016,
      "activations/layer22_attention_weight_max": 28.920400619506836,
      "activations/layer22_attention_weight_min": -28.15977668762207,
      "activations/layer23_attention_weight_max": 37.034523010253906,
      "activations/layer23_attention_weight_min": -26.504199981689453,
      "activations/layer2_attention_weight_max": 32.51974105834961,
      "activations/layer2_attention_weight_min": -29.730815887451172,
      "activations/layer3_attention_weight_max": 94.13016510009766,
      "activations/layer3_attention_weight_min": -92.63404083251953,
      "activations/layer4_attention_weight_max": 91.04134368896484,
      "activations/layer4_attention_weight_min": -91.41838836669922,
      "activations/layer5_attention_weight_max": 64.87654113769531,
      "activations/layer5_attention_weight_min": -77.43656158447266,
      "activations/layer6_attention_weight_max": 54.55896759033203,
      "activations/layer6_attention_weight_min": -54.481346130371094,
      "activations/layer7_attention_weight_max": 66.83685302734375,
      "activations/layer7_attention_weight_min": -64.54410552978516,
      "activations/layer8_attention_weight_max": 48.999935150146484,
      "activations/layer8_attention_weight_min": -52.72380065917969,
      "activations/layer9_attention_weight_max": 50.211368560791016,
      "activations/layer9_attention_weight_min": -48.8827018737793,
      "epoch": 16.78,
      "learning_rate": 4.224810606060606e-05,
      "loss": 2.74,
      "step": 288700
    },
    {
      "activations/layer0_attention_weight_max": 15.926302909851074,
      "activations/layer0_attention_weight_min": -13.34738826751709,
      "activations/layer10_attention_weight_max": 32.65950393676758,
      "activations/layer10_attention_weight_min": -32.84530258178711,
      "activations/layer11_attention_weight_max": 30.531021118164062,
      "activations/layer11_attention_weight_min": -31.328968048095703,
      "activations/layer12_attention_weight_max": 18.263784408569336,
      "activations/layer12_attention_weight_min": -27.279375076293945,
      "activations/layer13_attention_weight_max": 36.53484344482422,
      "activations/layer13_attention_weight_min": -29.443683624267578,
      "activations/layer14_attention_weight_max": 36.27116012573242,
      "activations/layer14_attention_weight_min": -32.369140625,
      "activations/layer15_attention_weight_max": 37.34098434448242,
      "activations/layer15_attention_weight_min": -32.22197723388672,
      "activations/layer16_attention_weight_max": 36.652198791503906,
      "activations/layer16_attention_weight_min": -33.411468505859375,
      "activations/layer17_attention_weight_max": 53.37502670288086,
      "activations/layer17_attention_weight_min": -50.78376770019531,
      "activations/layer18_attention_weight_max": 53.635318756103516,
      "activations/layer18_attention_weight_min": -47.78984832763672,
      "activations/layer19_attention_weight_max": 25.44043731689453,
      "activations/layer19_attention_weight_min": -24.363481521606445,
      "activations/layer1_attention_weight_max": 17.69937515258789,
      "activations/layer1_attention_weight_min": -12.846830368041992,
      "activations/layer20_attention_weight_max": 22.941509246826172,
      "activations/layer20_attention_weight_min": -23.342966079711914,
      "activations/layer21_attention_weight_max": 39.520965576171875,
      "activations/layer21_attention_weight_min": -21.17715835571289,
      "activations/layer22_attention_weight_max": 28.98334503173828,
      "activations/layer22_attention_weight_min": -26.452682495117188,
      "activations/layer23_attention_weight_max": 40.067909240722656,
      "activations/layer23_attention_weight_min": -23.476879119873047,
      "activations/layer2_attention_weight_max": 29.019302368164062,
      "activations/layer2_attention_weight_min": -25.91945457458496,
      "activations/layer3_attention_weight_max": 79.1941909790039,
      "activations/layer3_attention_weight_min": -81.50862121582031,
      "activations/layer4_attention_weight_max": 78.68917083740234,
      "activations/layer4_attention_weight_min": -81.66307830810547,
      "activations/layer5_attention_weight_max": 60.677608489990234,
      "activations/layer5_attention_weight_min": -69.26988220214844,
      "activations/layer6_attention_weight_max": 43.95391082763672,
      "activations/layer6_attention_weight_min": -47.008384704589844,
      "activations/layer7_attention_weight_max": 60.27903366088867,
      "activations/layer7_attention_weight_min": -62.936485290527344,
      "activations/layer8_attention_weight_max": 39.52663040161133,
      "activations/layer8_attention_weight_min": -45.43635177612305,
      "activations/layer9_attention_weight_max": 37.77111053466797,
      "activations/layer9_attention_weight_min": -46.586177825927734,
      "epoch": 16.78,
      "learning_rate": 4.222916666666666e-05,
      "loss": 2.7622,
      "step": 288750
    },
    {
      "activations/layer0_attention_weight_max": 15.750177383422852,
      "activations/layer0_attention_weight_min": -13.318955421447754,
      "activations/layer10_attention_weight_max": 35.436187744140625,
      "activations/layer10_attention_weight_min": -36.7743034362793,
      "activations/layer11_attention_weight_max": 29.7111873626709,
      "activations/layer11_attention_weight_min": -32.13810729980469,
      "activations/layer12_attention_weight_max": 19.31574058532715,
      "activations/layer12_attention_weight_min": -27.269926071166992,
      "activations/layer13_attention_weight_max": 36.433204650878906,
      "activations/layer13_attention_weight_min": -27.725296020507812,
      "activations/layer14_attention_weight_max": 37.0965461730957,
      "activations/layer14_attention_weight_min": -31.259984970092773,
      "activations/layer15_attention_weight_max": 35.00944137573242,
      "activations/layer15_attention_weight_min": -31.360883712768555,
      "activations/layer16_attention_weight_max": 35.69062805175781,
      "activations/layer16_attention_weight_min": -33.460533142089844,
      "activations/layer17_attention_weight_max": 53.638973236083984,
      "activations/layer17_attention_weight_min": -49.483154296875,
      "activations/layer18_attention_weight_max": 50.38596725463867,
      "activations/layer18_attention_weight_min": -44.8956184387207,
      "activations/layer19_attention_weight_max": 27.61089515686035,
      "activations/layer19_attention_weight_min": -25.165515899658203,
      "activations/layer1_attention_weight_max": 15.287728309631348,
      "activations/layer1_attention_weight_min": -13.406047821044922,
      "activations/layer20_attention_weight_max": 26.668811798095703,
      "activations/layer20_attention_weight_min": -22.469579696655273,
      "activations/layer21_attention_weight_max": 41.63405990600586,
      "activations/layer21_attention_weight_min": -22.965656280517578,
      "activations/layer22_attention_weight_max": 33.47331619262695,
      "activations/layer22_attention_weight_min": -26.651113510131836,
      "activations/layer23_attention_weight_max": 39.19645309448242,
      "activations/layer23_attention_weight_min": -23.893077850341797,
      "activations/layer2_attention_weight_max": 31.563621520996094,
      "activations/layer2_attention_weight_min": -29.579334259033203,
      "activations/layer3_attention_weight_max": 86.35884857177734,
      "activations/layer3_attention_weight_min": -86.58524322509766,
      "activations/layer4_attention_weight_max": 83.86774444580078,
      "activations/layer4_attention_weight_min": -83.8428726196289,
      "activations/layer5_attention_weight_max": 62.60439682006836,
      "activations/layer5_attention_weight_min": -70.00753021240234,
      "activations/layer6_attention_weight_max": 45.47560119628906,
      "activations/layer6_attention_weight_min": -47.95821762084961,
      "activations/layer7_attention_weight_max": 59.73482894897461,
      "activations/layer7_attention_weight_min": -57.813663482666016,
      "activations/layer8_attention_weight_max": 42.842891693115234,
      "activations/layer8_attention_weight_min": -44.553592681884766,
      "activations/layer9_attention_weight_max": 40.72300720214844,
      "activations/layer9_attention_weight_min": -42.93812561035156,
      "epoch": 16.78,
      "learning_rate": 4.221022727272727e-05,
      "loss": 2.7568,
      "step": 288800
    },
    {
      "activations/layer0_attention_weight_max": 16.397886276245117,
      "activations/layer0_attention_weight_min": -13.320080757141113,
      "activations/layer10_attention_weight_max": 36.439517974853516,
      "activations/layer10_attention_weight_min": -35.39122009277344,
      "activations/layer11_attention_weight_max": 32.28553771972656,
      "activations/layer11_attention_weight_min": -32.54388427734375,
      "activations/layer12_attention_weight_max": 20.525278091430664,
      "activations/layer12_attention_weight_min": -24.264108657836914,
      "activations/layer13_attention_weight_max": 35.43882369995117,
      "activations/layer13_attention_weight_min": -26.346220016479492,
      "activations/layer14_attention_weight_max": 36.44240188598633,
      "activations/layer14_attention_weight_min": -30.13288116455078,
      "activations/layer15_attention_weight_max": 31.1846866607666,
      "activations/layer15_attention_weight_min": -30.744014739990234,
      "activations/layer16_attention_weight_max": 33.676185607910156,
      "activations/layer16_attention_weight_min": -31.753223419189453,
      "activations/layer17_attention_weight_max": 49.2120361328125,
      "activations/layer17_attention_weight_min": -44.94391632080078,
      "activations/layer18_attention_weight_max": 44.51160430908203,
      "activations/layer18_attention_weight_min": -39.0153923034668,
      "activations/layer19_attention_weight_max": 22.957849502563477,
      "activations/layer19_attention_weight_min": -23.248043060302734,
      "activations/layer1_attention_weight_max": 16.274585723876953,
      "activations/layer1_attention_weight_min": -13.64881706237793,
      "activations/layer20_attention_weight_max": 23.83439826965332,
      "activations/layer20_attention_weight_min": -23.508991241455078,
      "activations/layer21_attention_weight_max": 36.60835647583008,
      "activations/layer21_attention_weight_min": -21.876068115234375,
      "activations/layer22_attention_weight_max": 29.30755615234375,
      "activations/layer22_attention_weight_min": -25.37896728515625,
      "activations/layer23_attention_weight_max": 34.47505187988281,
      "activations/layer23_attention_weight_min": -22.65005874633789,
      "activations/layer2_attention_weight_max": 30.446561813354492,
      "activations/layer2_attention_weight_min": -29.772418975830078,
      "activations/layer3_attention_weight_max": 87.59342956542969,
      "activations/layer3_attention_weight_min": -86.56868743896484,
      "activations/layer4_attention_weight_max": 87.9090576171875,
      "activations/layer4_attention_weight_min": -83.56452178955078,
      "activations/layer5_attention_weight_max": 62.07335662841797,
      "activations/layer5_attention_weight_min": -69.56912994384766,
      "activations/layer6_attention_weight_max": 48.409854888916016,
      "activations/layer6_attention_weight_min": -49.96907424926758,
      "activations/layer7_attention_weight_max": 61.779659271240234,
      "activations/layer7_attention_weight_min": -62.68667984008789,
      "activations/layer8_attention_weight_max": 44.913028717041016,
      "activations/layer8_attention_weight_min": -49.777000427246094,
      "activations/layer9_attention_weight_max": 44.22645568847656,
      "activations/layer9_attention_weight_min": -45.931922912597656,
      "epoch": 16.78,
      "learning_rate": 4.2191666666666664e-05,
      "loss": 2.7584,
      "step": 288850
    },
    {
      "activations/layer0_attention_weight_max": 16.414920806884766,
      "activations/layer0_attention_weight_min": -13.103458404541016,
      "activations/layer10_attention_weight_max": 40.06704330444336,
      "activations/layer10_attention_weight_min": -36.743099212646484,
      "activations/layer11_attention_weight_max": 34.427490234375,
      "activations/layer11_attention_weight_min": -34.345375061035156,
      "activations/layer12_attention_weight_max": 18.6638240814209,
      "activations/layer12_attention_weight_min": -22.71491813659668,
      "activations/layer13_attention_weight_max": 33.507137298583984,
      "activations/layer13_attention_weight_min": -26.745023727416992,
      "activations/layer14_attention_weight_max": 36.2796745300293,
      "activations/layer14_attention_weight_min": -31.57731819152832,
      "activations/layer15_attention_weight_max": 32.60179138183594,
      "activations/layer15_attention_weight_min": -30.622072219848633,
      "activations/layer16_attention_weight_max": 35.12451171875,
      "activations/layer16_attention_weight_min": -33.65057373046875,
      "activations/layer17_attention_weight_max": 51.6445198059082,
      "activations/layer17_attention_weight_min": -49.51503372192383,
      "activations/layer18_attention_weight_max": 47.42573165893555,
      "activations/layer18_attention_weight_min": -42.45903778076172,
      "activations/layer19_attention_weight_max": 27.67878532409668,
      "activations/layer19_attention_weight_min": -27.356679916381836,
      "activations/layer1_attention_weight_max": 16.445018768310547,
      "activations/layer1_attention_weight_min": -13.919144630432129,
      "activations/layer20_attention_weight_max": 25.244068145751953,
      "activations/layer20_attention_weight_min": -22.265649795532227,
      "activations/layer21_attention_weight_max": 37.61473846435547,
      "activations/layer21_attention_weight_min": -25.267601013183594,
      "activations/layer22_attention_weight_max": 29.415924072265625,
      "activations/layer22_attention_weight_min": -25.840295791625977,
      "activations/layer23_attention_weight_max": 35.63283157348633,
      "activations/layer23_attention_weight_min": -23.196243286132812,
      "activations/layer2_attention_weight_max": 31.955825805664062,
      "activations/layer2_attention_weight_min": -32.702144622802734,
      "activations/layer3_attention_weight_max": 86.28596496582031,
      "activations/layer3_attention_weight_min": -92.91817474365234,
      "activations/layer4_attention_weight_max": 87.09764862060547,
      "activations/layer4_attention_weight_min": -87.6095962524414,
      "activations/layer5_attention_weight_max": 64.09766387939453,
      "activations/layer5_attention_weight_min": -74.2857666015625,
      "activations/layer6_attention_weight_max": 50.81645965576172,
      "activations/layer6_attention_weight_min": -52.03827667236328,
      "activations/layer7_attention_weight_max": 63.787593841552734,
      "activations/layer7_attention_weight_min": -65.68505859375,
      "activations/layer8_attention_weight_max": 46.13188171386719,
      "activations/layer8_attention_weight_min": -47.76930618286133,
      "activations/layer9_attention_weight_max": 44.29249954223633,
      "activations/layer9_attention_weight_min": -48.65365982055664,
      "epoch": 16.79,
      "learning_rate": 4.2172727272727266e-05,
      "loss": 2.7527,
      "step": 288900
    },
    {
      "activations/layer0_attention_weight_max": 16.336429595947266,
      "activations/layer0_attention_weight_min": -13.253605842590332,
      "activations/layer10_attention_weight_max": 39.21906280517578,
      "activations/layer10_attention_weight_min": -39.6368408203125,
      "activations/layer11_attention_weight_max": 36.72713088989258,
      "activations/layer11_attention_weight_min": -38.743446350097656,
      "activations/layer12_attention_weight_max": 20.961307525634766,
      "activations/layer12_attention_weight_min": -24.710372924804688,
      "activations/layer13_attention_weight_max": 35.545570373535156,
      "activations/layer13_attention_weight_min": -32.23527145385742,
      "activations/layer14_attention_weight_max": 41.63926315307617,
      "activations/layer14_attention_weight_min": -35.442928314208984,
      "activations/layer15_attention_weight_max": 38.109676361083984,
      "activations/layer15_attention_weight_min": -33.72761154174805,
      "activations/layer16_attention_weight_max": 38.423526763916016,
      "activations/layer16_attention_weight_min": -36.67241287231445,
      "activations/layer17_attention_weight_max": 63.45602798461914,
      "activations/layer17_attention_weight_min": -52.51118469238281,
      "activations/layer18_attention_weight_max": 54.96898651123047,
      "activations/layer18_attention_weight_min": -46.55284881591797,
      "activations/layer19_attention_weight_max": 29.188390731811523,
      "activations/layer19_attention_weight_min": -24.664302825927734,
      "activations/layer1_attention_weight_max": 17.683429718017578,
      "activations/layer1_attention_weight_min": -14.559670448303223,
      "activations/layer20_attention_weight_max": 24.972911834716797,
      "activations/layer20_attention_weight_min": -23.490806579589844,
      "activations/layer21_attention_weight_max": 40.9930534362793,
      "activations/layer21_attention_weight_min": -24.48329734802246,
      "activations/layer22_attention_weight_max": 32.87089920043945,
      "activations/layer22_attention_weight_min": -26.086214065551758,
      "activations/layer23_attention_weight_max": 44.38792037963867,
      "activations/layer23_attention_weight_min": -25.489805221557617,
      "activations/layer2_attention_weight_max": 32.1262321472168,
      "activations/layer2_attention_weight_min": -31.775127410888672,
      "activations/layer3_attention_weight_max": 92.5789566040039,
      "activations/layer3_attention_weight_min": -93.91130065917969,
      "activations/layer4_attention_weight_max": 96.08373260498047,
      "activations/layer4_attention_weight_min": -94.8128890991211,
      "activations/layer5_attention_weight_max": 68.80999755859375,
      "activations/layer5_attention_weight_min": -78.74150085449219,
      "activations/layer6_attention_weight_max": 53.9492301940918,
      "activations/layer6_attention_weight_min": -54.223079681396484,
      "activations/layer7_attention_weight_max": 71.95092010498047,
      "activations/layer7_attention_weight_min": -71.32695007324219,
      "activations/layer8_attention_weight_max": 51.3636589050293,
      "activations/layer8_attention_weight_min": -56.61003112792969,
      "activations/layer9_attention_weight_max": 55.0916748046875,
      "activations/layer9_attention_weight_min": -51.58237838745117,
      "epoch": 16.79,
      "learning_rate": 4.2153787878787874e-05,
      "loss": 2.7465,
      "step": 288950
    },
    {
      "activations/layer0_attention_weight_max": 16.205549240112305,
      "activations/layer0_attention_weight_min": -13.412397384643555,
      "activations/layer10_attention_weight_max": 37.7985725402832,
      "activations/layer10_attention_weight_min": -38.11177444458008,
      "activations/layer11_attention_weight_max": 34.0312385559082,
      "activations/layer11_attention_weight_min": -35.83864212036133,
      "activations/layer12_attention_weight_max": 19.947011947631836,
      "activations/layer12_attention_weight_min": -26.59872817993164,
      "activations/layer13_attention_weight_max": 44.59632873535156,
      "activations/layer13_attention_weight_min": -29.885744094848633,
      "activations/layer14_attention_weight_max": 38.69879913330078,
      "activations/layer14_attention_weight_min": -32.29051971435547,
      "activations/layer15_attention_weight_max": 38.984336853027344,
      "activations/layer15_attention_weight_min": -30.744455337524414,
      "activations/layer16_attention_weight_max": 36.073299407958984,
      "activations/layer16_attention_weight_min": -31.328638076782227,
      "activations/layer17_attention_weight_max": 56.45826721191406,
      "activations/layer17_attention_weight_min": -45.073062896728516,
      "activations/layer18_attention_weight_max": 48.56792068481445,
      "activations/layer18_attention_weight_min": -42.18767166137695,
      "activations/layer19_attention_weight_max": 28.30730628967285,
      "activations/layer19_attention_weight_min": -21.439393997192383,
      "activations/layer1_attention_weight_max": 15.806960105895996,
      "activations/layer1_attention_weight_min": -14.541759490966797,
      "activations/layer20_attention_weight_max": 25.74746322631836,
      "activations/layer20_attention_weight_min": -21.24424934387207,
      "activations/layer21_attention_weight_max": 38.049015045166016,
      "activations/layer21_attention_weight_min": -21.9410343170166,
      "activations/layer22_attention_weight_max": 29.092288970947266,
      "activations/layer22_attention_weight_min": -26.2325439453125,
      "activations/layer23_attention_weight_max": 39.37267303466797,
      "activations/layer23_attention_weight_min": -24.926420211791992,
      "activations/layer2_attention_weight_max": 31.78188705444336,
      "activations/layer2_attention_weight_min": -30.80409812927246,
      "activations/layer3_attention_weight_max": 91.40494537353516,
      "activations/layer3_attention_weight_min": -92.8464584350586,
      "activations/layer4_attention_weight_max": 90.63491821289062,
      "activations/layer4_attention_weight_min": -94.1724853515625,
      "activations/layer5_attention_weight_max": 62.847713470458984,
      "activations/layer5_attention_weight_min": -77.93173217773438,
      "activations/layer6_attention_weight_max": 51.566673278808594,
      "activations/layer6_attention_weight_min": -54.96913528442383,
      "activations/layer7_attention_weight_max": 68.35809326171875,
      "activations/layer7_attention_weight_min": -64.21186828613281,
      "activations/layer8_attention_weight_max": 46.61388397216797,
      "activations/layer8_attention_weight_min": -50.852928161621094,
      "activations/layer9_attention_weight_max": 47.196285247802734,
      "activations/layer9_attention_weight_min": -46.459224700927734,
      "epoch": 16.79,
      "learning_rate": 4.213484848484848e-05,
      "loss": 2.7602,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4611,
      "eval_samples_per_second": 507.497,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4611,
      "eval_openwebtext_samples_per_second": 507.497,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9522,
      "eval_wikitext_samples_per_second": 233.587,
      "step": 289000
    },
    {
      "epoch": 16.79,
      "eval_lambada_loss": 2.49609375,
      "eval_lambada_ppl": 12.134998917818166,
      "eval_lambada_runtime": 9.5447,
      "eval_lambada_samples_per_second": 510.126,
      "step": 289000
    },
    {
      "activations/layer0_attention_weight_max": 15.396468162536621,
      "activations/layer0_attention_weight_min": -13.491640090942383,
      "activations/layer10_attention_weight_max": 34.004207611083984,
      "activations/layer10_attention_weight_min": -35.61078643798828,
      "activations/layer11_attention_weight_max": 33.23857116699219,
      "activations/layer11_attention_weight_min": -32.804771423339844,
      "activations/layer12_attention_weight_max": 17.40567398071289,
      "activations/layer12_attention_weight_min": -26.127567291259766,
      "activations/layer13_attention_weight_max": 29.99574851989746,
      "activations/layer13_attention_weight_min": -25.56684684753418,
      "activations/layer14_attention_weight_max": 32.171241760253906,
      "activations/layer14_attention_weight_min": -28.754379272460938,
      "activations/layer15_attention_weight_max": 29.646343231201172,
      "activations/layer15_attention_weight_min": -28.353595733642578,
      "activations/layer16_attention_weight_max": 31.087890625,
      "activations/layer16_attention_weight_min": -29.816879272460938,
      "activations/layer17_attention_weight_max": 48.989986419677734,
      "activations/layer17_attention_weight_min": -47.75133514404297,
      "activations/layer18_attention_weight_max": 42.3242301940918,
      "activations/layer18_attention_weight_min": -42.25886154174805,
      "activations/layer19_attention_weight_max": 21.158287048339844,
      "activations/layer19_attention_weight_min": -22.47572898864746,
      "activations/layer1_attention_weight_max": 17.37598991394043,
      "activations/layer1_attention_weight_min": -14.582406044006348,
      "activations/layer20_attention_weight_max": 21.7602596282959,
      "activations/layer20_attention_weight_min": -20.999347686767578,
      "activations/layer21_attention_weight_max": 32.11439895629883,
      "activations/layer21_attention_weight_min": -21.980419158935547,
      "activations/layer22_attention_weight_max": 27.068511962890625,
      "activations/layer22_attention_weight_min": -24.57415199279785,
      "activations/layer23_attention_weight_max": 32.13208770751953,
      "activations/layer23_attention_weight_min": -22.589969635009766,
      "activations/layer2_attention_weight_max": 32.36503601074219,
      "activations/layer2_attention_weight_min": -31.02058982849121,
      "activations/layer3_attention_weight_max": 84.61278533935547,
      "activations/layer3_attention_weight_min": -88.0324478149414,
      "activations/layer4_attention_weight_max": 85.59611511230469,
      "activations/layer4_attention_weight_min": -83.63716888427734,
      "activations/layer5_attention_weight_max": 59.78764343261719,
      "activations/layer5_attention_weight_min": -74.3909912109375,
      "activations/layer6_attention_weight_max": 47.84629821777344,
      "activations/layer6_attention_weight_min": -47.2846794128418,
      "activations/layer7_attention_weight_max": 66.51985168457031,
      "activations/layer7_attention_weight_min": -60.07950210571289,
      "activations/layer8_attention_weight_max": 45.41391372680664,
      "activations/layer8_attention_weight_min": -45.89778518676758,
      "activations/layer9_attention_weight_max": 49.59357833862305,
      "activations/layer9_attention_weight_min": -44.42906188964844,
      "epoch": 16.8,
      "learning_rate": 4.211590909090909e-05,
      "loss": 2.7579,
      "step": 289050
    },
    {
      "activations/layer0_attention_weight_max": 16.004968643188477,
      "activations/layer0_attention_weight_min": -13.841286659240723,
      "activations/layer10_attention_weight_max": 33.78519058227539,
      "activations/layer10_attention_weight_min": -32.640445709228516,
      "activations/layer11_attention_weight_max": 29.298065185546875,
      "activations/layer11_attention_weight_min": -32.79098129272461,
      "activations/layer12_attention_weight_max": 18.240877151489258,
      "activations/layer12_attention_weight_min": -24.548023223876953,
      "activations/layer13_attention_weight_max": 31.182212829589844,
      "activations/layer13_attention_weight_min": -28.397113800048828,
      "activations/layer14_attention_weight_max": 33.039634704589844,
      "activations/layer14_attention_weight_min": -30.59204864501953,
      "activations/layer15_attention_weight_max": 30.428508758544922,
      "activations/layer15_attention_weight_min": -31.011838912963867,
      "activations/layer16_attention_weight_max": 32.70541000366211,
      "activations/layer16_attention_weight_min": -31.394445419311523,
      "activations/layer17_attention_weight_max": 47.41305923461914,
      "activations/layer17_attention_weight_min": -43.85368347167969,
      "activations/layer18_attention_weight_max": 43.728431701660156,
      "activations/layer18_attention_weight_min": -39.43267059326172,
      "activations/layer19_attention_weight_max": 22.840118408203125,
      "activations/layer19_attention_weight_min": -21.98443603515625,
      "activations/layer1_attention_weight_max": 16.683666229248047,
      "activations/layer1_attention_weight_min": -13.5175142288208,
      "activations/layer20_attention_weight_max": 24.17274284362793,
      "activations/layer20_attention_weight_min": -21.560466766357422,
      "activations/layer21_attention_weight_max": 34.84423828125,
      "activations/layer21_attention_weight_min": -21.31820297241211,
      "activations/layer22_attention_weight_max": 28.712263107299805,
      "activations/layer22_attention_weight_min": -25.683292388916016,
      "activations/layer23_attention_weight_max": 35.48386001586914,
      "activations/layer23_attention_weight_min": -22.890033721923828,
      "activations/layer2_attention_weight_max": 31.05760955810547,
      "activations/layer2_attention_weight_min": -29.763967514038086,
      "activations/layer3_attention_weight_max": 87.61241912841797,
      "activations/layer3_attention_weight_min": -86.62797546386719,
      "activations/layer4_attention_weight_max": 83.9207534790039,
      "activations/layer4_attention_weight_min": -85.06534576416016,
      "activations/layer5_attention_weight_max": 60.395751953125,
      "activations/layer5_attention_weight_min": -73.22145080566406,
      "activations/layer6_attention_weight_max": 49.45075988769531,
      "activations/layer6_attention_weight_min": -49.11393356323242,
      "activations/layer7_attention_weight_max": 60.65499496459961,
      "activations/layer7_attention_weight_min": -59.08482360839844,
      "activations/layer8_attention_weight_max": 41.45722579956055,
      "activations/layer8_attention_weight_min": -47.77603530883789,
      "activations/layer9_attention_weight_max": 40.613258361816406,
      "activations/layer9_attention_weight_min": -44.3501091003418,
      "epoch": 16.8,
      "learning_rate": 4.209696969696969e-05,
      "loss": 2.7474,
      "step": 289100
    },
    {
      "activations/layer0_attention_weight_max": 15.652730941772461,
      "activations/layer0_attention_weight_min": -13.901018142700195,
      "activations/layer10_attention_weight_max": 35.8694953918457,
      "activations/layer10_attention_weight_min": -34.12968826293945,
      "activations/layer11_attention_weight_max": 33.33617401123047,
      "activations/layer11_attention_weight_min": -32.54849624633789,
      "activations/layer12_attention_weight_max": 18.34320640563965,
      "activations/layer12_attention_weight_min": -26.49367904663086,
      "activations/layer13_attention_weight_max": 31.634523391723633,
      "activations/layer13_attention_weight_min": -26.665035247802734,
      "activations/layer14_attention_weight_max": 30.962684631347656,
      "activations/layer14_attention_weight_min": -28.80171775817871,
      "activations/layer15_attention_weight_max": 29.631488800048828,
      "activations/layer15_attention_weight_min": -28.592540740966797,
      "activations/layer16_attention_weight_max": 30.17266845703125,
      "activations/layer16_attention_weight_min": -29.60685157775879,
      "activations/layer17_attention_weight_max": 47.979190826416016,
      "activations/layer17_attention_weight_min": -43.259037017822266,
      "activations/layer18_attention_weight_max": 41.98373794555664,
      "activations/layer18_attention_weight_min": -38.32933807373047,
      "activations/layer19_attention_weight_max": 19.893569946289062,
      "activations/layer19_attention_weight_min": -22.29878807067871,
      "activations/layer1_attention_weight_max": 16.98678207397461,
      "activations/layer1_attention_weight_min": -14.20805549621582,
      "activations/layer20_attention_weight_max": 19.22254180908203,
      "activations/layer20_attention_weight_min": -21.044816970825195,
      "activations/layer21_attention_weight_max": 30.82016944885254,
      "activations/layer21_attention_weight_min": -22.07628059387207,
      "activations/layer22_attention_weight_max": 25.71175193786621,
      "activations/layer22_attention_weight_min": -23.89969253540039,
      "activations/layer23_attention_weight_max": 33.872249603271484,
      "activations/layer23_attention_weight_min": -25.49385643005371,
      "activations/layer2_attention_weight_max": 34.5432243347168,
      "activations/layer2_attention_weight_min": -31.23866844177246,
      "activations/layer3_attention_weight_max": 91.70310974121094,
      "activations/layer3_attention_weight_min": -89.85395812988281,
      "activations/layer4_attention_weight_max": 88.39385986328125,
      "activations/layer4_attention_weight_min": -85.85286712646484,
      "activations/layer5_attention_weight_max": 62.58526611328125,
      "activations/layer5_attention_weight_min": -73.48383331298828,
      "activations/layer6_attention_weight_max": 49.60818862915039,
      "activations/layer6_attention_weight_min": -49.656063079833984,
      "activations/layer7_attention_weight_max": 62.701839447021484,
      "activations/layer7_attention_weight_min": -56.2962532043457,
      "activations/layer8_attention_weight_max": 42.70505905151367,
      "activations/layer8_attention_weight_min": -45.31051254272461,
      "activations/layer9_attention_weight_max": 46.68959426879883,
      "activations/layer9_attention_weight_min": -44.71830749511719,
      "epoch": 16.8,
      "learning_rate": 4.2078030303030294e-05,
      "loss": 2.7517,
      "step": 289150
    },
    {
      "activations/layer0_attention_weight_max": 16.350740432739258,
      "activations/layer0_attention_weight_min": -13.733275413513184,
      "activations/layer10_attention_weight_max": 36.12986755371094,
      "activations/layer10_attention_weight_min": -34.596954345703125,
      "activations/layer11_attention_weight_max": 32.36747741699219,
      "activations/layer11_attention_weight_min": -33.89447784423828,
      "activations/layer12_attention_weight_max": 17.10150146484375,
      "activations/layer12_attention_weight_min": -25.230331420898438,
      "activations/layer13_attention_weight_max": 32.552879333496094,
      "activations/layer13_attention_weight_min": -29.119447708129883,
      "activations/layer14_attention_weight_max": 32.274532318115234,
      "activations/layer14_attention_weight_min": -34.13524627685547,
      "activations/layer15_attention_weight_max": 32.1594123840332,
      "activations/layer15_attention_weight_min": -30.636199951171875,
      "activations/layer16_attention_weight_max": 31.441999435424805,
      "activations/layer16_attention_weight_min": -33.94061279296875,
      "activations/layer17_attention_weight_max": 47.05230712890625,
      "activations/layer17_attention_weight_min": -46.6946907043457,
      "activations/layer18_attention_weight_max": 44.168277740478516,
      "activations/layer18_attention_weight_min": -39.97621536254883,
      "activations/layer19_attention_weight_max": 25.687158584594727,
      "activations/layer19_attention_weight_min": -22.554357528686523,
      "activations/layer1_attention_weight_max": 16.327465057373047,
      "activations/layer1_attention_weight_min": -13.928412437438965,
      "activations/layer20_attention_weight_max": 21.11863899230957,
      "activations/layer20_attention_weight_min": -22.036117553710938,
      "activations/layer21_attention_weight_max": 35.352821350097656,
      "activations/layer21_attention_weight_min": -24.236297607421875,
      "activations/layer22_attention_weight_max": 28.26393699645996,
      "activations/layer22_attention_weight_min": -26.099138259887695,
      "activations/layer23_attention_weight_max": 37.48517608642578,
      "activations/layer23_attention_weight_min": -24.86068344116211,
      "activations/layer2_attention_weight_max": 31.587501525878906,
      "activations/layer2_attention_weight_min": -29.88625144958496,
      "activations/layer3_attention_weight_max": 90.12775421142578,
      "activations/layer3_attention_weight_min": -89.13456726074219,
      "activations/layer4_attention_weight_max": 89.68356323242188,
      "activations/layer4_attention_weight_min": -90.01829528808594,
      "activations/layer5_attention_weight_max": 63.16204071044922,
      "activations/layer5_attention_weight_min": -71.52396392822266,
      "activations/layer6_attention_weight_max": 50.25701141357422,
      "activations/layer6_attention_weight_min": -51.97127914428711,
      "activations/layer7_attention_weight_max": 65.9312973022461,
      "activations/layer7_attention_weight_min": -64.47642517089844,
      "activations/layer8_attention_weight_max": 46.6454963684082,
      "activations/layer8_attention_weight_min": -47.63374710083008,
      "activations/layer9_attention_weight_max": 44.10430908203125,
      "activations/layer9_attention_weight_min": -44.908016204833984,
      "epoch": 16.8,
      "learning_rate": 4.205909090909091e-05,
      "loss": 2.7416,
      "step": 289200
    },
    {
      "activations/layer0_attention_weight_max": 16.681535720825195,
      "activations/layer0_attention_weight_min": -13.77843952178955,
      "activations/layer10_attention_weight_max": 38.533447265625,
      "activations/layer10_attention_weight_min": -38.44178009033203,
      "activations/layer11_attention_weight_max": 34.65123748779297,
      "activations/layer11_attention_weight_min": -34.96453094482422,
      "activations/layer12_attention_weight_max": 19.562610626220703,
      "activations/layer12_attention_weight_min": -25.62095069885254,
      "activations/layer13_attention_weight_max": 44.25177764892578,
      "activations/layer13_attention_weight_min": -33.02642059326172,
      "activations/layer14_attention_weight_max": 37.512474060058594,
      "activations/layer14_attention_weight_min": -32.6139030456543,
      "activations/layer15_attention_weight_max": 34.744422912597656,
      "activations/layer15_attention_weight_min": -32.06931686401367,
      "activations/layer16_attention_weight_max": 34.75250244140625,
      "activations/layer16_attention_weight_min": -30.913883209228516,
      "activations/layer17_attention_weight_max": 54.666412353515625,
      "activations/layer17_attention_weight_min": -46.221736907958984,
      "activations/layer18_attention_weight_max": 51.31444549560547,
      "activations/layer18_attention_weight_min": -42.064083099365234,
      "activations/layer19_attention_weight_max": 26.7471923828125,
      "activations/layer19_attention_weight_min": -25.10230827331543,
      "activations/layer1_attention_weight_max": 17.786401748657227,
      "activations/layer1_attention_weight_min": -14.972570419311523,
      "activations/layer20_attention_weight_max": 26.71575164794922,
      "activations/layer20_attention_weight_min": -24.054845809936523,
      "activations/layer21_attention_weight_max": 37.45790100097656,
      "activations/layer21_attention_weight_min": -23.06011962890625,
      "activations/layer22_attention_weight_max": 30.912216186523438,
      "activations/layer22_attention_weight_min": -25.091367721557617,
      "activations/layer23_attention_weight_max": 39.80222702026367,
      "activations/layer23_attention_weight_min": -24.79729461669922,
      "activations/layer2_attention_weight_max": 32.75922393798828,
      "activations/layer2_attention_weight_min": -32.531578063964844,
      "activations/layer3_attention_weight_max": 88.32686614990234,
      "activations/layer3_attention_weight_min": -90.49759674072266,
      "activations/layer4_attention_weight_max": 90.45796203613281,
      "activations/layer4_attention_weight_min": -87.34770965576172,
      "activations/layer5_attention_weight_max": 65.31755065917969,
      "activations/layer5_attention_weight_min": -69.96397399902344,
      "activations/layer6_attention_weight_max": 53.52193069458008,
      "activations/layer6_attention_weight_min": -51.947635650634766,
      "activations/layer7_attention_weight_max": 68.86277770996094,
      "activations/layer7_attention_weight_min": -74.77706146240234,
      "activations/layer8_attention_weight_max": 52.63331604003906,
      "activations/layer8_attention_weight_min": -53.907535552978516,
      "activations/layer9_attention_weight_max": 48.3687858581543,
      "activations/layer9_attention_weight_min": -49.11442947387695,
      "epoch": 16.81,
      "learning_rate": 4.204015151515151e-05,
      "loss": 2.7593,
      "step": 289250
    },
    {
      "activations/layer0_attention_weight_max": 15.207175254821777,
      "activations/layer0_attention_weight_min": -13.912710189819336,
      "activations/layer10_attention_weight_max": 36.504661560058594,
      "activations/layer10_attention_weight_min": -34.3974723815918,
      "activations/layer11_attention_weight_max": 31.87601089477539,
      "activations/layer11_attention_weight_min": -33.62898635864258,
      "activations/layer12_attention_weight_max": 20.078420639038086,
      "activations/layer12_attention_weight_min": -23.935007095336914,
      "activations/layer13_attention_weight_max": 39.972801208496094,
      "activations/layer13_attention_weight_min": -29.052913665771484,
      "activations/layer14_attention_weight_max": 40.07821273803711,
      "activations/layer14_attention_weight_min": -30.96930694580078,
      "activations/layer15_attention_weight_max": 38.506683349609375,
      "activations/layer15_attention_weight_min": -30.29307746887207,
      "activations/layer16_attention_weight_max": 34.85208511352539,
      "activations/layer16_attention_weight_min": -31.762466430664062,
      "activations/layer17_attention_weight_max": 55.709861755371094,
      "activations/layer17_attention_weight_min": -44.02522277832031,
      "activations/layer18_attention_weight_max": 49.78665542602539,
      "activations/layer18_attention_weight_min": -38.785011291503906,
      "activations/layer19_attention_weight_max": 23.485692977905273,
      "activations/layer19_attention_weight_min": -23.35726547241211,
      "activations/layer1_attention_weight_max": 16.137147903442383,
      "activations/layer1_attention_weight_min": -15.482564926147461,
      "activations/layer20_attention_weight_max": 22.34734344482422,
      "activations/layer20_attention_weight_min": -22.614030838012695,
      "activations/layer21_attention_weight_max": 37.709110260009766,
      "activations/layer21_attention_weight_min": -22.394302368164062,
      "activations/layer22_attention_weight_max": 31.82883644104004,
      "activations/layer22_attention_weight_min": -28.00578498840332,
      "activations/layer23_attention_weight_max": 41.98884963989258,
      "activations/layer23_attention_weight_min": -22.76598358154297,
      "activations/layer2_attention_weight_max": 32.27964782714844,
      "activations/layer2_attention_weight_min": -31.508405685424805,
      "activations/layer3_attention_weight_max": 92.21736145019531,
      "activations/layer3_attention_weight_min": -94.28294372558594,
      "activations/layer4_attention_weight_max": 88.30671691894531,
      "activations/layer4_attention_weight_min": -88.8803939819336,
      "activations/layer5_attention_weight_max": 62.571712493896484,
      "activations/layer5_attention_weight_min": -70.21070098876953,
      "activations/layer6_attention_weight_max": 50.27217102050781,
      "activations/layer6_attention_weight_min": -50.640785217285156,
      "activations/layer7_attention_weight_max": 62.84091567993164,
      "activations/layer7_attention_weight_min": -61.15825653076172,
      "activations/layer8_attention_weight_max": 44.64496612548828,
      "activations/layer8_attention_weight_min": -47.2136116027832,
      "activations/layer9_attention_weight_max": 44.05856704711914,
      "activations/layer9_attention_weight_min": -45.75272750854492,
      "epoch": 16.81,
      "learning_rate": 4.202121212121212e-05,
      "loss": 2.7467,
      "step": 289300
    },
    {
      "activations/layer0_attention_weight_max": 15.79139518737793,
      "activations/layer0_attention_weight_min": -13.898067474365234,
      "activations/layer10_attention_weight_max": 35.167118072509766,
      "activations/layer10_attention_weight_min": -36.32868194580078,
      "activations/layer11_attention_weight_max": 32.70246505737305,
      "activations/layer11_attention_weight_min": -33.272125244140625,
      "activations/layer12_attention_weight_max": 18.258642196655273,
      "activations/layer12_attention_weight_min": -26.782283782958984,
      "activations/layer13_attention_weight_max": 32.83091735839844,
      "activations/layer13_attention_weight_min": -27.433368682861328,
      "activations/layer14_attention_weight_max": 35.234352111816406,
      "activations/layer14_attention_weight_min": -29.511503219604492,
      "activations/layer15_attention_weight_max": 32.43873977661133,
      "activations/layer15_attention_weight_min": -30.325382232666016,
      "activations/layer16_attention_weight_max": 34.09636688232422,
      "activations/layer16_attention_weight_min": -32.11789321899414,
      "activations/layer17_attention_weight_max": 51.08177185058594,
      "activations/layer17_attention_weight_min": -46.086421966552734,
      "activations/layer18_attention_weight_max": 43.43656921386719,
      "activations/layer18_attention_weight_min": -40.99878692626953,
      "activations/layer19_attention_weight_max": 21.922645568847656,
      "activations/layer19_attention_weight_min": -25.074726104736328,
      "activations/layer1_attention_weight_max": 16.071195602416992,
      "activations/layer1_attention_weight_min": -14.06689167022705,
      "activations/layer20_attention_weight_max": 21.426237106323242,
      "activations/layer20_attention_weight_min": -23.215103149414062,
      "activations/layer21_attention_weight_max": 34.925537109375,
      "activations/layer21_attention_weight_min": -24.63218116760254,
      "activations/layer22_attention_weight_max": 27.74380874633789,
      "activations/layer22_attention_weight_min": -28.273706436157227,
      "activations/layer23_attention_weight_max": 34.237945556640625,
      "activations/layer23_attention_weight_min": -24.504905700683594,
      "activations/layer2_attention_weight_max": 32.72190475463867,
      "activations/layer2_attention_weight_min": -31.069677352905273,
      "activations/layer3_attention_weight_max": 91.26087188720703,
      "activations/layer3_attention_weight_min": -91.31061553955078,
      "activations/layer4_attention_weight_max": 90.3810043334961,
      "activations/layer4_attention_weight_min": -90.3669662475586,
      "activations/layer5_attention_weight_max": 61.504390716552734,
      "activations/layer5_attention_weight_min": -70.74610137939453,
      "activations/layer6_attention_weight_max": 50.021812438964844,
      "activations/layer6_attention_weight_min": -51.25965118408203,
      "activations/layer7_attention_weight_max": 63.22615432739258,
      "activations/layer7_attention_weight_min": -61.708953857421875,
      "activations/layer8_attention_weight_max": 46.158241271972656,
      "activations/layer8_attention_weight_min": -52.05040740966797,
      "activations/layer9_attention_weight_max": 43.88111114501953,
      "activations/layer9_attention_weight_min": -44.30794906616211,
      "epoch": 16.81,
      "learning_rate": 4.200227272727272e-05,
      "loss": 2.7412,
      "step": 289350
    },
    {
      "activations/layer0_attention_weight_max": 15.726457595825195,
      "activations/layer0_attention_weight_min": -13.877372741699219,
      "activations/layer10_attention_weight_max": 36.95082473754883,
      "activations/layer10_attention_weight_min": -35.50899124145508,
      "activations/layer11_attention_weight_max": 33.93732833862305,
      "activations/layer11_attention_weight_min": -33.72055435180664,
      "activations/layer12_attention_weight_max": 21.592437744140625,
      "activations/layer12_attention_weight_min": -24.178741455078125,
      "activations/layer13_attention_weight_max": 39.83049011230469,
      "activations/layer13_attention_weight_min": -29.48419952392578,
      "activations/layer14_attention_weight_max": 39.52074432373047,
      "activations/layer14_attention_weight_min": -32.43855285644531,
      "activations/layer15_attention_weight_max": 37.049068450927734,
      "activations/layer15_attention_weight_min": -31.406696319580078,
      "activations/layer16_attention_weight_max": 38.31380081176758,
      "activations/layer16_attention_weight_min": -34.86361312866211,
      "activations/layer17_attention_weight_max": 55.5890998840332,
      "activations/layer17_attention_weight_min": -47.94196701049805,
      "activations/layer18_attention_weight_max": 53.18954849243164,
      "activations/layer18_attention_weight_min": -42.68307876586914,
      "activations/layer19_attention_weight_max": 26.189544677734375,
      "activations/layer19_attention_weight_min": -23.06361198425293,
      "activations/layer1_attention_weight_max": 16.21466827392578,
      "activations/layer1_attention_weight_min": -14.440105438232422,
      "activations/layer20_attention_weight_max": 26.538908004760742,
      "activations/layer20_attention_weight_min": -21.45574951171875,
      "activations/layer21_attention_weight_max": 42.94277572631836,
      "activations/layer21_attention_weight_min": -23.741832733154297,
      "activations/layer22_attention_weight_max": 33.51230239868164,
      "activations/layer22_attention_weight_min": -26.26005744934082,
      "activations/layer23_attention_weight_max": 38.23677062988281,
      "activations/layer23_attention_weight_min": -25.123563766479492,
      "activations/layer2_attention_weight_max": 31.115140914916992,
      "activations/layer2_attention_weight_min": -30.0955867767334,
      "activations/layer3_attention_weight_max": 89.53272247314453,
      "activations/layer3_attention_weight_min": -91.29731750488281,
      "activations/layer4_attention_weight_max": 89.39620971679688,
      "activations/layer4_attention_weight_min": -88.19098663330078,
      "activations/layer5_attention_weight_max": 63.964637756347656,
      "activations/layer5_attention_weight_min": -73.17811584472656,
      "activations/layer6_attention_weight_max": 50.65827560424805,
      "activations/layer6_attention_weight_min": -50.05746078491211,
      "activations/layer7_attention_weight_max": 64.04450225830078,
      "activations/layer7_attention_weight_min": -62.02953338623047,
      "activations/layer8_attention_weight_max": 47.1557731628418,
      "activations/layer8_attention_weight_min": -47.24354934692383,
      "activations/layer9_attention_weight_max": 43.53166580200195,
      "activations/layer9_attention_weight_min": -45.9920768737793,
      "epoch": 16.82,
      "learning_rate": 4.1983333333333335e-05,
      "loss": 2.758,
      "step": 289400
    },
    {
      "activations/layer0_attention_weight_max": 15.086106300354004,
      "activations/layer0_attention_weight_min": -13.720661163330078,
      "activations/layer10_attention_weight_max": 32.98592758178711,
      "activations/layer10_attention_weight_min": -33.39676284790039,
      "activations/layer11_attention_weight_max": 30.592178344726562,
      "activations/layer11_attention_weight_min": -35.99660873413086,
      "activations/layer12_attention_weight_max": 17.49056053161621,
      "activations/layer12_attention_weight_min": -25.08292007446289,
      "activations/layer13_attention_weight_max": 31.003847122192383,
      "activations/layer13_attention_weight_min": -24.37282943725586,
      "activations/layer14_attention_weight_max": 33.82307815551758,
      "activations/layer14_attention_weight_min": -28.656583786010742,
      "activations/layer15_attention_weight_max": 30.620092391967773,
      "activations/layer15_attention_weight_min": -28.720266342163086,
      "activations/layer16_attention_weight_max": 30.813215255737305,
      "activations/layer16_attention_weight_min": -27.980287551879883,
      "activations/layer17_attention_weight_max": 51.21369934082031,
      "activations/layer17_attention_weight_min": -42.09284591674805,
      "activations/layer18_attention_weight_max": 49.15502166748047,
      "activations/layer18_attention_weight_min": -37.424434661865234,
      "activations/layer19_attention_weight_max": 23.32074737548828,
      "activations/layer19_attention_weight_min": -23.029882431030273,
      "activations/layer1_attention_weight_max": 16.136926651000977,
      "activations/layer1_attention_weight_min": -14.837156295776367,
      "activations/layer20_attention_weight_max": 21.844154357910156,
      "activations/layer20_attention_weight_min": -22.3149471282959,
      "activations/layer21_attention_weight_max": 37.387908935546875,
      "activations/layer21_attention_weight_min": -24.800506591796875,
      "activations/layer22_attention_weight_max": 30.75878143310547,
      "activations/layer22_attention_weight_min": -26.7895565032959,
      "activations/layer23_attention_weight_max": 36.756736755371094,
      "activations/layer23_attention_weight_min": -22.971153259277344,
      "activations/layer2_attention_weight_max": 31.838470458984375,
      "activations/layer2_attention_weight_min": -31.019973754882812,
      "activations/layer3_attention_weight_max": 87.79732513427734,
      "activations/layer3_attention_weight_min": -90.25322723388672,
      "activations/layer4_attention_weight_max": 86.45209503173828,
      "activations/layer4_attention_weight_min": -87.42044830322266,
      "activations/layer5_attention_weight_max": 62.23133850097656,
      "activations/layer5_attention_weight_min": -74.1438980102539,
      "activations/layer6_attention_weight_max": 50.98762130737305,
      "activations/layer6_attention_weight_min": -52.332088470458984,
      "activations/layer7_attention_weight_max": 63.1917610168457,
      "activations/layer7_attention_weight_min": -60.670753479003906,
      "activations/layer8_attention_weight_max": 44.156436920166016,
      "activations/layer8_attention_weight_min": -47.527286529541016,
      "activations/layer9_attention_weight_max": 43.08535385131836,
      "activations/layer9_attention_weight_min": -43.97771453857422,
      "epoch": 16.82,
      "learning_rate": 4.196439393939394e-05,
      "loss": 2.7534,
      "step": 289450
    },
    {
      "activations/layer0_attention_weight_max": 15.951855659484863,
      "activations/layer0_attention_weight_min": -14.142132759094238,
      "activations/layer10_attention_weight_max": 35.06954574584961,
      "activations/layer10_attention_weight_min": -34.659332275390625,
      "activations/layer11_attention_weight_max": 30.62283706665039,
      "activations/layer11_attention_weight_min": -32.34250259399414,
      "activations/layer12_attention_weight_max": 18.918964385986328,
      "activations/layer12_attention_weight_min": -23.266191482543945,
      "activations/layer13_attention_weight_max": 33.949989318847656,
      "activations/layer13_attention_weight_min": -27.435579299926758,
      "activations/layer14_attention_weight_max": 33.325721740722656,
      "activations/layer14_attention_weight_min": -28.250139236450195,
      "activations/layer15_attention_weight_max": 30.541526794433594,
      "activations/layer15_attention_weight_min": -28.95937156677246,
      "activations/layer16_attention_weight_max": 31.013317108154297,
      "activations/layer16_attention_weight_min": -29.595834732055664,
      "activations/layer17_attention_weight_max": 50.912025451660156,
      "activations/layer17_attention_weight_min": -42.504295349121094,
      "activations/layer18_attention_weight_max": 44.89333724975586,
      "activations/layer18_attention_weight_min": -38.375755310058594,
      "activations/layer19_attention_weight_max": 23.642963409423828,
      "activations/layer19_attention_weight_min": -23.1840763092041,
      "activations/layer1_attention_weight_max": 15.953102111816406,
      "activations/layer1_attention_weight_min": -14.298392295837402,
      "activations/layer20_attention_weight_max": 21.7833194732666,
      "activations/layer20_attention_weight_min": -23.17767906188965,
      "activations/layer21_attention_weight_max": 31.906139373779297,
      "activations/layer21_attention_weight_min": -22.5317325592041,
      "activations/layer22_attention_weight_max": 29.459001541137695,
      "activations/layer22_attention_weight_min": -24.491968154907227,
      "activations/layer23_attention_weight_max": 41.37664031982422,
      "activations/layer23_attention_weight_min": -23.68817901611328,
      "activations/layer2_attention_weight_max": 30.623390197753906,
      "activations/layer2_attention_weight_min": -29.305822372436523,
      "activations/layer3_attention_weight_max": 89.51937103271484,
      "activations/layer3_attention_weight_min": -87.64146423339844,
      "activations/layer4_attention_weight_max": 87.11695098876953,
      "activations/layer4_attention_weight_min": -86.32101440429688,
      "activations/layer5_attention_weight_max": 66.68952941894531,
      "activations/layer5_attention_weight_min": -71.869140625,
      "activations/layer6_attention_weight_max": 48.625797271728516,
      "activations/layer6_attention_weight_min": -48.13237762451172,
      "activations/layer7_attention_weight_max": 60.031585693359375,
      "activations/layer7_attention_weight_min": -61.09275817871094,
      "activations/layer8_attention_weight_max": 42.3367805480957,
      "activations/layer8_attention_weight_min": -49.45357131958008,
      "activations/layer9_attention_weight_max": 42.937889099121094,
      "activations/layer9_attention_weight_min": -42.90000915527344,
      "epoch": 16.82,
      "learning_rate": 4.194545454545454e-05,
      "loss": 2.7392,
      "step": 289500
    },
    {
      "activations/layer0_attention_weight_max": 15.746283531188965,
      "activations/layer0_attention_weight_min": -13.881749153137207,
      "activations/layer10_attention_weight_max": 34.740631103515625,
      "activations/layer10_attention_weight_min": -34.656982421875,
      "activations/layer11_attention_weight_max": 32.0107421875,
      "activations/layer11_attention_weight_min": -32.06797790527344,
      "activations/layer12_attention_weight_max": 19.138561248779297,
      "activations/layer12_attention_weight_min": -24.70322036743164,
      "activations/layer13_attention_weight_max": 29.93218421936035,
      "activations/layer13_attention_weight_min": -28.783327102661133,
      "activations/layer14_attention_weight_max": 34.12455749511719,
      "activations/layer14_attention_weight_min": -33.6036262512207,
      "activations/layer15_attention_weight_max": 32.493080139160156,
      "activations/layer15_attention_weight_min": -33.189517974853516,
      "activations/layer16_attention_weight_max": 33.256134033203125,
      "activations/layer16_attention_weight_min": -32.66132736206055,
      "activations/layer17_attention_weight_max": 50.23808288574219,
      "activations/layer17_attention_weight_min": -47.974056243896484,
      "activations/layer18_attention_weight_max": 47.76667404174805,
      "activations/layer18_attention_weight_min": -43.96378707885742,
      "activations/layer19_attention_weight_max": 23.511445999145508,
      "activations/layer19_attention_weight_min": -26.856952667236328,
      "activations/layer1_attention_weight_max": 15.977288246154785,
      "activations/layer1_attention_weight_min": -15.065381050109863,
      "activations/layer20_attention_weight_max": 21.38410186767578,
      "activations/layer20_attention_weight_min": -23.923463821411133,
      "activations/layer21_attention_weight_max": 35.51258087158203,
      "activations/layer21_attention_weight_min": -25.07478904724121,
      "activations/layer22_attention_weight_max": 28.28912925720215,
      "activations/layer22_attention_weight_min": -31.229496002197266,
      "activations/layer23_attention_weight_max": 34.678916931152344,
      "activations/layer23_attention_weight_min": -26.571327209472656,
      "activations/layer2_attention_weight_max": 32.100833892822266,
      "activations/layer2_attention_weight_min": -31.202550888061523,
      "activations/layer3_attention_weight_max": 88.68009948730469,
      "activations/layer3_attention_weight_min": -88.46306610107422,
      "activations/layer4_attention_weight_max": 84.58490753173828,
      "activations/layer4_attention_weight_min": -85.8208236694336,
      "activations/layer5_attention_weight_max": 65.99305725097656,
      "activations/layer5_attention_weight_min": -73.71012115478516,
      "activations/layer6_attention_weight_max": 48.89323043823242,
      "activations/layer6_attention_weight_min": -50.883296966552734,
      "activations/layer7_attention_weight_max": 66.50241088867188,
      "activations/layer7_attention_weight_min": -65.96713256835938,
      "activations/layer8_attention_weight_max": 44.02936553955078,
      "activations/layer8_attention_weight_min": -47.78615188598633,
      "activations/layer9_attention_weight_max": 43.085811614990234,
      "activations/layer9_attention_weight_min": -43.532352447509766,
      "epoch": 16.82,
      "learning_rate": 4.1926515151515146e-05,
      "loss": 2.7508,
      "step": 289550
    },
    {
      "activations/layer0_attention_weight_max": 15.580150604248047,
      "activations/layer0_attention_weight_min": -13.52428913116455,
      "activations/layer10_attention_weight_max": 35.081546783447266,
      "activations/layer10_attention_weight_min": -34.20940017700195,
      "activations/layer11_attention_weight_max": 32.382354736328125,
      "activations/layer11_attention_weight_min": -33.43476867675781,
      "activations/layer12_attention_weight_max": 19.497451782226562,
      "activations/layer12_attention_weight_min": -24.74777603149414,
      "activations/layer13_attention_weight_max": 37.35893249511719,
      "activations/layer13_attention_weight_min": -27.771686553955078,
      "activations/layer14_attention_weight_max": 37.97697067260742,
      "activations/layer14_attention_weight_min": -29.369455337524414,
      "activations/layer15_attention_weight_max": 35.57926940917969,
      "activations/layer15_attention_weight_min": -32.09691619873047,
      "activations/layer16_attention_weight_max": 36.7601318359375,
      "activations/layer16_attention_weight_min": -34.426177978515625,
      "activations/layer17_attention_weight_max": 54.36220169067383,
      "activations/layer17_attention_weight_min": -47.44035720825195,
      "activations/layer18_attention_weight_max": 50.82231140136719,
      "activations/layer18_attention_weight_min": -42.23100662231445,
      "activations/layer19_attention_weight_max": 27.46697425842285,
      "activations/layer19_attention_weight_min": -24.671064376831055,
      "activations/layer1_attention_weight_max": 16.439905166625977,
      "activations/layer1_attention_weight_min": -16.36323356628418,
      "activations/layer20_attention_weight_max": 24.147489547729492,
      "activations/layer20_attention_weight_min": -21.1791934967041,
      "activations/layer21_attention_weight_max": 35.600379943847656,
      "activations/layer21_attention_weight_min": -22.807470321655273,
      "activations/layer22_attention_weight_max": 28.30274772644043,
      "activations/layer22_attention_weight_min": -26.21177101135254,
      "activations/layer23_attention_weight_max": 39.307952880859375,
      "activations/layer23_attention_weight_min": -24.580169677734375,
      "activations/layer2_attention_weight_max": 32.30653381347656,
      "activations/layer2_attention_weight_min": -32.359535217285156,
      "activations/layer3_attention_weight_max": 91.90912628173828,
      "activations/layer3_attention_weight_min": -92.54132080078125,
      "activations/layer4_attention_weight_max": 88.57858276367188,
      "activations/layer4_attention_weight_min": -86.34188079833984,
      "activations/layer5_attention_weight_max": 63.66368103027344,
      "activations/layer5_attention_weight_min": -73.1392593383789,
      "activations/layer6_attention_weight_max": 50.55485916137695,
      "activations/layer6_attention_weight_min": -51.59736633300781,
      "activations/layer7_attention_weight_max": 60.741058349609375,
      "activations/layer7_attention_weight_min": -62.11843490600586,
      "activations/layer8_attention_weight_max": 44.41563034057617,
      "activations/layer8_attention_weight_min": -48.76969528198242,
      "activations/layer9_attention_weight_max": 43.655582427978516,
      "activations/layer9_attention_weight_min": -45.47220993041992,
      "epoch": 16.83,
      "learning_rate": 4.1907575757575755e-05,
      "loss": 2.761,
      "step": 289600
    },
    {
      "activations/layer0_attention_weight_max": 15.698590278625488,
      "activations/layer0_attention_weight_min": -13.482446670532227,
      "activations/layer10_attention_weight_max": 38.18138122558594,
      "activations/layer10_attention_weight_min": -36.72492218017578,
      "activations/layer11_attention_weight_max": 34.89590072631836,
      "activations/layer11_attention_weight_min": -34.95925521850586,
      "activations/layer12_attention_weight_max": 18.72163963317871,
      "activations/layer12_attention_weight_min": -24.35637092590332,
      "activations/layer13_attention_weight_max": 40.03948211669922,
      "activations/layer13_attention_weight_min": -28.185951232910156,
      "activations/layer14_attention_weight_max": 38.134849548339844,
      "activations/layer14_attention_weight_min": -33.35399627685547,
      "activations/layer15_attention_weight_max": 35.023704528808594,
      "activations/layer15_attention_weight_min": -33.38142395019531,
      "activations/layer16_attention_weight_max": 34.73005676269531,
      "activations/layer16_attention_weight_min": -31.036584854125977,
      "activations/layer17_attention_weight_max": 59.32175827026367,
      "activations/layer17_attention_weight_min": -45.640106201171875,
      "activations/layer18_attention_weight_max": 51.3630485534668,
      "activations/layer18_attention_weight_min": -38.97652053833008,
      "activations/layer19_attention_weight_max": 24.76851463317871,
      "activations/layer19_attention_weight_min": -22.81870460510254,
      "activations/layer1_attention_weight_max": 16.598478317260742,
      "activations/layer1_attention_weight_min": -14.242474555969238,
      "activations/layer20_attention_weight_max": 22.858505249023438,
      "activations/layer20_attention_weight_min": -25.846269607543945,
      "activations/layer21_attention_weight_max": 32.9222297668457,
      "activations/layer21_attention_weight_min": -22.052358627319336,
      "activations/layer22_attention_weight_max": 27.1208438873291,
      "activations/layer22_attention_weight_min": -28.204654693603516,
      "activations/layer23_attention_weight_max": 37.138397216796875,
      "activations/layer23_attention_weight_min": -25.29971694946289,
      "activations/layer2_attention_weight_max": 30.680503845214844,
      "activations/layer2_attention_weight_min": -30.56996726989746,
      "activations/layer3_attention_weight_max": 88.12206268310547,
      "activations/layer3_attention_weight_min": -90.21998596191406,
      "activations/layer4_attention_weight_max": 87.92427825927734,
      "activations/layer4_attention_weight_min": -89.27754974365234,
      "activations/layer5_attention_weight_max": 64.50848388671875,
      "activations/layer5_attention_weight_min": -72.4954605102539,
      "activations/layer6_attention_weight_max": 51.921043395996094,
      "activations/layer6_attention_weight_min": -51.72624588012695,
      "activations/layer7_attention_weight_max": 66.98834228515625,
      "activations/layer7_attention_weight_min": -66.24378204345703,
      "activations/layer8_attention_weight_max": 52.35088348388672,
      "activations/layer8_attention_weight_min": -51.564247131347656,
      "activations/layer9_attention_weight_max": 46.4294319152832,
      "activations/layer9_attention_weight_min": -47.582130432128906,
      "epoch": 16.83,
      "learning_rate": 4.188863636363636e-05,
      "loss": 2.7472,
      "step": 289650
    },
    {
      "activations/layer0_attention_weight_max": 16.118684768676758,
      "activations/layer0_attention_weight_min": -13.370325088500977,
      "activations/layer10_attention_weight_max": 37.10348129272461,
      "activations/layer10_attention_weight_min": -34.12567901611328,
      "activations/layer11_attention_weight_max": 32.08175277709961,
      "activations/layer11_attention_weight_min": -31.732709884643555,
      "activations/layer12_attention_weight_max": 18.19074821472168,
      "activations/layer12_attention_weight_min": -28.528427124023438,
      "activations/layer13_attention_weight_max": 30.383106231689453,
      "activations/layer13_attention_weight_min": -27.208965301513672,
      "activations/layer14_attention_weight_max": 33.14337921142578,
      "activations/layer14_attention_weight_min": -30.278345108032227,
      "activations/layer15_attention_weight_max": 30.429088592529297,
      "activations/layer15_attention_weight_min": -29.724946975708008,
      "activations/layer16_attention_weight_max": 31.3656005859375,
      "activations/layer16_attention_weight_min": -30.578149795532227,
      "activations/layer17_attention_weight_max": 47.25376892089844,
      "activations/layer17_attention_weight_min": -43.91739273071289,
      "activations/layer18_attention_weight_max": 43.91761016845703,
      "activations/layer18_attention_weight_min": -37.34514617919922,
      "activations/layer19_attention_weight_max": 24.632200241088867,
      "activations/layer19_attention_weight_min": -24.291425704956055,
      "activations/layer1_attention_weight_max": 16.32919692993164,
      "activations/layer1_attention_weight_min": -14.327757835388184,
      "activations/layer20_attention_weight_max": 22.564714431762695,
      "activations/layer20_attention_weight_min": -21.801969528198242,
      "activations/layer21_attention_weight_max": 42.33537673950195,
      "activations/layer21_attention_weight_min": -24.40338134765625,
      "activations/layer22_attention_weight_max": 29.055728912353516,
      "activations/layer22_attention_weight_min": -24.760150909423828,
      "activations/layer23_attention_weight_max": 36.90604019165039,
      "activations/layer23_attention_weight_min": -26.130277633666992,
      "activations/layer2_attention_weight_max": 33.401954650878906,
      "activations/layer2_attention_weight_min": -29.58971405029297,
      "activations/layer3_attention_weight_max": 90.19390106201172,
      "activations/layer3_attention_weight_min": -85.36555480957031,
      "activations/layer4_attention_weight_max": 84.81901550292969,
      "activations/layer4_attention_weight_min": -86.75279998779297,
      "activations/layer5_attention_weight_max": 64.41360473632812,
      "activations/layer5_attention_weight_min": -75.48136901855469,
      "activations/layer6_attention_weight_max": 49.288211822509766,
      "activations/layer6_attention_weight_min": -51.55207443237305,
      "activations/layer7_attention_weight_max": 66.65399169921875,
      "activations/layer7_attention_weight_min": -58.8997917175293,
      "activations/layer8_attention_weight_max": 44.9356575012207,
      "activations/layer8_attention_weight_min": -45.453887939453125,
      "activations/layer9_attention_weight_max": 44.96628952026367,
      "activations/layer9_attention_weight_min": -45.416629791259766,
      "epoch": 16.83,
      "learning_rate": 4.1869696969696964e-05,
      "loss": 2.7557,
      "step": 289700
    },
    {
      "activations/layer0_attention_weight_max": 15.454933166503906,
      "activations/layer0_attention_weight_min": -13.772300720214844,
      "activations/layer10_attention_weight_max": 37.80048370361328,
      "activations/layer10_attention_weight_min": -37.1743049621582,
      "activations/layer11_attention_weight_max": 36.212615966796875,
      "activations/layer11_attention_weight_min": -36.03731918334961,
      "activations/layer12_attention_weight_max": 22.58246612548828,
      "activations/layer12_attention_weight_min": -24.037734985351562,
      "activations/layer13_attention_weight_max": 47.99495315551758,
      "activations/layer13_attention_weight_min": -29.0631046295166,
      "activations/layer14_attention_weight_max": 45.08405303955078,
      "activations/layer14_attention_weight_min": -32.29022979736328,
      "activations/layer15_attention_weight_max": 43.59124755859375,
      "activations/layer15_attention_weight_min": -31.250385284423828,
      "activations/layer16_attention_weight_max": 38.238258361816406,
      "activations/layer16_attention_weight_min": -31.968523025512695,
      "activations/layer17_attention_weight_max": 59.09563064575195,
      "activations/layer17_attention_weight_min": -50.12398910522461,
      "activations/layer18_attention_weight_max": 54.9835205078125,
      "activations/layer18_attention_weight_min": -45.03801727294922,
      "activations/layer19_attention_weight_max": 25.20109748840332,
      "activations/layer19_attention_weight_min": -25.771276473999023,
      "activations/layer1_attention_weight_max": 16.23818588256836,
      "activations/layer1_attention_weight_min": -15.417196273803711,
      "activations/layer20_attention_weight_max": 27.424203872680664,
      "activations/layer20_attention_weight_min": -22.371253967285156,
      "activations/layer21_attention_weight_max": 46.8011589050293,
      "activations/layer21_attention_weight_min": -23.143342971801758,
      "activations/layer22_attention_weight_max": 34.24652862548828,
      "activations/layer22_attention_weight_min": -26.9473876953125,
      "activations/layer23_attention_weight_max": 45.57275390625,
      "activations/layer23_attention_weight_min": -22.978424072265625,
      "activations/layer2_attention_weight_max": 32.95018768310547,
      "activations/layer2_attention_weight_min": -32.44356918334961,
      "activations/layer3_attention_weight_max": 91.58529663085938,
      "activations/layer3_attention_weight_min": -96.71741485595703,
      "activations/layer4_attention_weight_max": 92.24190521240234,
      "activations/layer4_attention_weight_min": -90.9055404663086,
      "activations/layer5_attention_weight_max": 69.07022094726562,
      "activations/layer5_attention_weight_min": -75.99604034423828,
      "activations/layer6_attention_weight_max": 48.44392776489258,
      "activations/layer6_attention_weight_min": -49.40877914428711,
      "activations/layer7_attention_weight_max": 67.1850814819336,
      "activations/layer7_attention_weight_min": -64.56798553466797,
      "activations/layer8_attention_weight_max": 46.61397171020508,
      "activations/layer8_attention_weight_min": -49.04436492919922,
      "activations/layer9_attention_weight_max": 50.01158905029297,
      "activations/layer9_attention_weight_min": -45.94849395751953,
      "epoch": 16.84,
      "learning_rate": 4.1850757575757566e-05,
      "loss": 2.7419,
      "step": 289750
    },
    {
      "activations/layer0_attention_weight_max": 15.747722625732422,
      "activations/layer0_attention_weight_min": -13.600213050842285,
      "activations/layer10_attention_weight_max": 37.28436279296875,
      "activations/layer10_attention_weight_min": -34.239173889160156,
      "activations/layer11_attention_weight_max": 30.253753662109375,
      "activations/layer11_attention_weight_min": -31.985794067382812,
      "activations/layer12_attention_weight_max": 17.763893127441406,
      "activations/layer12_attention_weight_min": -23.694040298461914,
      "activations/layer13_attention_weight_max": 35.091949462890625,
      "activations/layer13_attention_weight_min": -29.528369903564453,
      "activations/layer14_attention_weight_max": 34.564598083496094,
      "activations/layer14_attention_weight_min": -31.89033317565918,
      "activations/layer15_attention_weight_max": 33.058021545410156,
      "activations/layer15_attention_weight_min": -31.895071029663086,
      "activations/layer16_attention_weight_max": 32.476173400878906,
      "activations/layer16_attention_weight_min": -32.243099212646484,
      "activations/layer17_attention_weight_max": 52.983360290527344,
      "activations/layer17_attention_weight_min": -47.20029830932617,
      "activations/layer18_attention_weight_max": 50.62959289550781,
      "activations/layer18_attention_weight_min": -41.07074737548828,
      "activations/layer19_attention_weight_max": 23.84856414794922,
      "activations/layer19_attention_weight_min": -22.255172729492188,
      "activations/layer1_attention_weight_max": 16.013307571411133,
      "activations/layer1_attention_weight_min": -14.668986320495605,
      "activations/layer20_attention_weight_max": 20.801952362060547,
      "activations/layer20_attention_weight_min": -22.137672424316406,
      "activations/layer21_attention_weight_max": 33.83645248413086,
      "activations/layer21_attention_weight_min": -20.940622329711914,
      "activations/layer22_attention_weight_max": 31.692670822143555,
      "activations/layer22_attention_weight_min": -27.012563705444336,
      "activations/layer23_attention_weight_max": 39.63993835449219,
      "activations/layer23_attention_weight_min": -25.180654525756836,
      "activations/layer2_attention_weight_max": 30.797590255737305,
      "activations/layer2_attention_weight_min": -30.70653533935547,
      "activations/layer3_attention_weight_max": 89.20010375976562,
      "activations/layer3_attention_weight_min": -91.32410430908203,
      "activations/layer4_attention_weight_max": 93.47079467773438,
      "activations/layer4_attention_weight_min": -86.29370880126953,
      "activations/layer5_attention_weight_max": 67.0766830444336,
      "activations/layer5_attention_weight_min": -76.8831558227539,
      "activations/layer6_attention_weight_max": 50.204078674316406,
      "activations/layer6_attention_weight_min": -49.16579818725586,
      "activations/layer7_attention_weight_max": 63.570640563964844,
      "activations/layer7_attention_weight_min": -61.76812744140625,
      "activations/layer8_attention_weight_max": 43.71937561035156,
      "activations/layer8_attention_weight_min": -46.24544143676758,
      "activations/layer9_attention_weight_max": 39.66936111450195,
      "activations/layer9_attention_weight_min": -44.232112884521484,
      "epoch": 16.84,
      "learning_rate": 4.183181818181818e-05,
      "loss": 2.7479,
      "step": 289800
    },
    {
      "activations/layer0_attention_weight_max": 17.797231674194336,
      "activations/layer0_attention_weight_min": -13.718072891235352,
      "activations/layer10_attention_weight_max": 36.58071517944336,
      "activations/layer10_attention_weight_min": -36.31018829345703,
      "activations/layer11_attention_weight_max": 33.070457458496094,
      "activations/layer11_attention_weight_min": -36.00212097167969,
      "activations/layer12_attention_weight_max": 19.680055618286133,
      "activations/layer12_attention_weight_min": -24.2509765625,
      "activations/layer13_attention_weight_max": 32.69929122924805,
      "activations/layer13_attention_weight_min": -26.446544647216797,
      "activations/layer14_attention_weight_max": 33.12783432006836,
      "activations/layer14_attention_weight_min": -31.500404357910156,
      "activations/layer15_attention_weight_max": 31.03095054626465,
      "activations/layer15_attention_weight_min": -29.788236618041992,
      "activations/layer16_attention_weight_max": 33.64972686767578,
      "activations/layer16_attention_weight_min": -31.435640335083008,
      "activations/layer17_attention_weight_max": 49.0003776550293,
      "activations/layer17_attention_weight_min": -42.99077606201172,
      "activations/layer18_attention_weight_max": 42.7259635925293,
      "activations/layer18_attention_weight_min": -38.368953704833984,
      "activations/layer19_attention_weight_max": 23.035001754760742,
      "activations/layer19_attention_weight_min": -21.25791358947754,
      "activations/layer1_attention_weight_max": 15.923258781433105,
      "activations/layer1_attention_weight_min": -13.645194053649902,
      "activations/layer20_attention_weight_max": 22.00490379333496,
      "activations/layer20_attention_weight_min": -21.757802963256836,
      "activations/layer21_attention_weight_max": 35.201812744140625,
      "activations/layer21_attention_weight_min": -22.347557067871094,
      "activations/layer22_attention_weight_max": 30.055465698242188,
      "activations/layer22_attention_weight_min": -24.43211555480957,
      "activations/layer23_attention_weight_max": 34.07584762573242,
      "activations/layer23_attention_weight_min": -24.070987701416016,
      "activations/layer2_attention_weight_max": 31.92774772644043,
      "activations/layer2_attention_weight_min": -31.410968780517578,
      "activations/layer3_attention_weight_max": 87.59782409667969,
      "activations/layer3_attention_weight_min": -88.5495376586914,
      "activations/layer4_attention_weight_max": 86.9011001586914,
      "activations/layer4_attention_weight_min": -91.5342788696289,
      "activations/layer5_attention_weight_max": 65.56449890136719,
      "activations/layer5_attention_weight_min": -74.21773529052734,
      "activations/layer6_attention_weight_max": 49.81167221069336,
      "activations/layer6_attention_weight_min": -51.165122985839844,
      "activations/layer7_attention_weight_max": 65.46827697753906,
      "activations/layer7_attention_weight_min": -65.514404296875,
      "activations/layer8_attention_weight_max": 47.13114929199219,
      "activations/layer8_attention_weight_min": -49.46426773071289,
      "activations/layer9_attention_weight_max": 43.98303985595703,
      "activations/layer9_attention_weight_min": -45.35391616821289,
      "epoch": 16.84,
      "learning_rate": 4.181287878787878e-05,
      "loss": 2.7547,
      "step": 289850
    },
    {
      "activations/layer0_attention_weight_max": 15.582387924194336,
      "activations/layer0_attention_weight_min": -13.434124946594238,
      "activations/layer10_attention_weight_max": 40.55159378051758,
      "activations/layer10_attention_weight_min": -37.787933349609375,
      "activations/layer11_attention_weight_max": 37.976722717285156,
      "activations/layer11_attention_weight_min": -35.46980285644531,
      "activations/layer12_attention_weight_max": 19.25752067565918,
      "activations/layer12_attention_weight_min": -26.01932144165039,
      "activations/layer13_attention_weight_max": 35.483253479003906,
      "activations/layer13_attention_weight_min": -29.434412002563477,
      "activations/layer14_attention_weight_max": 42.71782302856445,
      "activations/layer14_attention_weight_min": -33.42271423339844,
      "activations/layer15_attention_weight_max": 38.588897705078125,
      "activations/layer15_attention_weight_min": -33.69057083129883,
      "activations/layer16_attention_weight_max": 40.472618103027344,
      "activations/layer16_attention_weight_min": -35.40151596069336,
      "activations/layer17_attention_weight_max": 62.17052459716797,
      "activations/layer17_attention_weight_min": -54.0230598449707,
      "activations/layer18_attention_weight_max": 62.01015090942383,
      "activations/layer18_attention_weight_min": -50.7650146484375,
      "activations/layer19_attention_weight_max": 32.184051513671875,
      "activations/layer19_attention_weight_min": -28.129709243774414,
      "activations/layer1_attention_weight_max": 17.57431411743164,
      "activations/layer1_attention_weight_min": -14.381159782409668,
      "activations/layer20_attention_weight_max": 35.04669952392578,
      "activations/layer20_attention_weight_min": -23.771581649780273,
      "activations/layer21_attention_weight_max": 58.720176696777344,
      "activations/layer21_attention_weight_min": -24.8463134765625,
      "activations/layer22_attention_weight_max": 37.51517105102539,
      "activations/layer22_attention_weight_min": -28.23162841796875,
      "activations/layer23_attention_weight_max": 45.422576904296875,
      "activations/layer23_attention_weight_min": -22.978347778320312,
      "activations/layer2_attention_weight_max": 32.97881317138672,
      "activations/layer2_attention_weight_min": -31.010831832885742,
      "activations/layer3_attention_weight_max": 92.11624145507812,
      "activations/layer3_attention_weight_min": -88.03759002685547,
      "activations/layer4_attention_weight_max": 87.94971466064453,
      "activations/layer4_attention_weight_min": -86.79659271240234,
      "activations/layer5_attention_weight_max": 60.50621032714844,
      "activations/layer5_attention_weight_min": -69.01283264160156,
      "activations/layer6_attention_weight_max": 50.80025863647461,
      "activations/layer6_attention_weight_min": -48.84599304199219,
      "activations/layer7_attention_weight_max": 67.27885437011719,
      "activations/layer7_attention_weight_min": -65.15956115722656,
      "activations/layer8_attention_weight_max": 51.57508850097656,
      "activations/layer8_attention_weight_min": -51.65062713623047,
      "activations/layer9_attention_weight_max": 51.962581634521484,
      "activations/layer9_attention_weight_min": -46.146278381347656,
      "epoch": 16.84,
      "learning_rate": 4.179393939393939e-05,
      "loss": 2.7505,
      "step": 289900
    },
    {
      "activations/layer0_attention_weight_max": 15.13811206817627,
      "activations/layer0_attention_weight_min": -13.338722229003906,
      "activations/layer10_attention_weight_max": 36.3193244934082,
      "activations/layer10_attention_weight_min": -38.318206787109375,
      "activations/layer11_attention_weight_max": 34.472564697265625,
      "activations/layer11_attention_weight_min": -35.53028106689453,
      "activations/layer12_attention_weight_max": 19.359235763549805,
      "activations/layer12_attention_weight_min": -23.43062973022461,
      "activations/layer13_attention_weight_max": 31.672115325927734,
      "activations/layer13_attention_weight_min": -29.145227432250977,
      "activations/layer14_attention_weight_max": 35.26612854003906,
      "activations/layer14_attention_weight_min": -31.011362075805664,
      "activations/layer15_attention_weight_max": 33.44899368286133,
      "activations/layer15_attention_weight_min": -30.723779678344727,
      "activations/layer16_attention_weight_max": 33.40727233886719,
      "activations/layer16_attention_weight_min": -32.943660736083984,
      "activations/layer17_attention_weight_max": 50.99485397338867,
      "activations/layer17_attention_weight_min": -49.004737854003906,
      "activations/layer18_attention_weight_max": 46.27365493774414,
      "activations/layer18_attention_weight_min": -43.964317321777344,
      "activations/layer19_attention_weight_max": 23.25408363342285,
      "activations/layer19_attention_weight_min": -22.872392654418945,
      "activations/layer1_attention_weight_max": 16.993898391723633,
      "activations/layer1_attention_weight_min": -14.90136432647705,
      "activations/layer20_attention_weight_max": 21.124835968017578,
      "activations/layer20_attention_weight_min": -21.73423957824707,
      "activations/layer21_attention_weight_max": 31.790945053100586,
      "activations/layer21_attention_weight_min": -21.312753677368164,
      "activations/layer22_attention_weight_max": 27.539060592651367,
      "activations/layer22_attention_weight_min": -25.712705612182617,
      "activations/layer23_attention_weight_max": 35.6453857421875,
      "activations/layer23_attention_weight_min": -23.64406967163086,
      "activations/layer2_attention_weight_max": 33.366416931152344,
      "activations/layer2_attention_weight_min": -31.24917984008789,
      "activations/layer3_attention_weight_max": 91.33323669433594,
      "activations/layer3_attention_weight_min": -90.34236907958984,
      "activations/layer4_attention_weight_max": 88.06591033935547,
      "activations/layer4_attention_weight_min": -90.43092346191406,
      "activations/layer5_attention_weight_max": 64.75244140625,
      "activations/layer5_attention_weight_min": -72.35478973388672,
      "activations/layer6_attention_weight_max": 48.044715881347656,
      "activations/layer6_attention_weight_min": -49.62876892089844,
      "activations/layer7_attention_weight_max": 64.01432037353516,
      "activations/layer7_attention_weight_min": -66.61510467529297,
      "activations/layer8_attention_weight_max": 45.19156265258789,
      "activations/layer8_attention_weight_min": -49.09231185913086,
      "activations/layer9_attention_weight_max": 46.308597564697266,
      "activations/layer9_attention_weight_min": -49.249874114990234,
      "epoch": 16.85,
      "learning_rate": 4.1775e-05,
      "loss": 2.7464,
      "step": 289950
    },
    {
      "activations/layer0_attention_weight_max": 15.719080924987793,
      "activations/layer0_attention_weight_min": -13.233413696289062,
      "activations/layer10_attention_weight_max": 36.481658935546875,
      "activations/layer10_attention_weight_min": -37.318870544433594,
      "activations/layer11_attention_weight_max": 34.40044403076172,
      "activations/layer11_attention_weight_min": -33.548240661621094,
      "activations/layer12_attention_weight_max": 18.542192459106445,
      "activations/layer12_attention_weight_min": -26.01670265197754,
      "activations/layer13_attention_weight_max": 35.98341751098633,
      "activations/layer13_attention_weight_min": -28.432273864746094,
      "activations/layer14_attention_weight_max": 37.14753341674805,
      "activations/layer14_attention_weight_min": -28.874568939208984,
      "activations/layer15_attention_weight_max": 30.919721603393555,
      "activations/layer15_attention_weight_min": -29.54130744934082,
      "activations/layer16_attention_weight_max": 29.86272621154785,
      "activations/layer16_attention_weight_min": -30.970043182373047,
      "activations/layer17_attention_weight_max": 48.48371505737305,
      "activations/layer17_attention_weight_min": -42.75547409057617,
      "activations/layer18_attention_weight_max": 41.6861686706543,
      "activations/layer18_attention_weight_min": -38.646026611328125,
      "activations/layer19_attention_weight_max": 22.456745147705078,
      "activations/layer19_attention_weight_min": -21.454790115356445,
      "activations/layer1_attention_weight_max": 16.523733139038086,
      "activations/layer1_attention_weight_min": -17.637592315673828,
      "activations/layer20_attention_weight_max": 20.115459442138672,
      "activations/layer20_attention_weight_min": -20.596357345581055,
      "activations/layer21_attention_weight_max": 28.138200759887695,
      "activations/layer21_attention_weight_min": -19.829036712646484,
      "activations/layer22_attention_weight_max": 27.125682830810547,
      "activations/layer22_attention_weight_min": -24.309297561645508,
      "activations/layer23_attention_weight_max": 33.85559844970703,
      "activations/layer23_attention_weight_min": -22.30824851989746,
      "activations/layer2_attention_weight_max": 31.17862319946289,
      "activations/layer2_attention_weight_min": -31.49840545654297,
      "activations/layer3_attention_weight_max": 82.76589965820312,
      "activations/layer3_attention_weight_min": -89.89864349365234,
      "activations/layer4_attention_weight_max": 83.13270568847656,
      "activations/layer4_attention_weight_min": -83.92308807373047,
      "activations/layer5_attention_weight_max": 61.27972412109375,
      "activations/layer5_attention_weight_min": -73.51673889160156,
      "activations/layer6_attention_weight_max": 47.06336212158203,
      "activations/layer6_attention_weight_min": -48.78603744506836,
      "activations/layer7_attention_weight_max": 63.73652267456055,
      "activations/layer7_attention_weight_min": -61.477413177490234,
      "activations/layer8_attention_weight_max": 48.44947814941406,
      "activations/layer8_attention_weight_min": -49.6520881652832,
      "activations/layer9_attention_weight_max": 46.076534271240234,
      "activations/layer9_attention_weight_min": -46.71284866333008,
      "epoch": 16.85,
      "learning_rate": 4.175606060606061e-05,
      "loss": 2.7475,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4189,
      "eval_samples_per_second": 510.044,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4189,
      "eval_openwebtext_samples_per_second": 510.044,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9343,
      "eval_wikitext_samples_per_second": 235.741,
      "step": 290000
    },
    {
      "epoch": 16.85,
      "eval_lambada_loss": 2.478515625,
      "eval_lambada_ppl": 11.923552252478798,
      "eval_lambada_runtime": 9.5053,
      "eval_lambada_samples_per_second": 512.238,
      "step": 290000
    },
    {
      "activations/layer0_attention_weight_max": 15.802762031555176,
      "activations/layer0_attention_weight_min": -13.442741394042969,
      "activations/layer10_attention_weight_max": 35.40678024291992,
      "activations/layer10_attention_weight_min": -35.516448974609375,
      "activations/layer11_attention_weight_max": 32.068824768066406,
      "activations/layer11_attention_weight_min": -33.79462814331055,
      "activations/layer12_attention_weight_max": 22.017745971679688,
      "activations/layer12_attention_weight_min": -25.884260177612305,
      "activations/layer13_attention_weight_max": 32.49494934082031,
      "activations/layer13_attention_weight_min": -27.42567253112793,
      "activations/layer14_attention_weight_max": 36.06864547729492,
      "activations/layer14_attention_weight_min": -31.335845947265625,
      "activations/layer15_attention_weight_max": 32.265140533447266,
      "activations/layer15_attention_weight_min": -31.700347900390625,
      "activations/layer16_attention_weight_max": 33.88254165649414,
      "activations/layer16_attention_weight_min": -32.08837890625,
      "activations/layer17_attention_weight_max": 51.85334014892578,
      "activations/layer17_attention_weight_min": -45.23751449584961,
      "activations/layer18_attention_weight_max": 47.42231369018555,
      "activations/layer18_attention_weight_min": -39.366676330566406,
      "activations/layer19_attention_weight_max": 23.752731323242188,
      "activations/layer19_attention_weight_min": -21.65048599243164,
      "activations/layer1_attention_weight_max": 16.26480484008789,
      "activations/layer1_attention_weight_min": -14.51391887664795,
      "activations/layer20_attention_weight_max": 22.65135383605957,
      "activations/layer20_attention_weight_min": -20.119110107421875,
      "activations/layer21_attention_weight_max": 31.295347213745117,
      "activations/layer21_attention_weight_min": -22.170671463012695,
      "activations/layer22_attention_weight_max": 31.69161033630371,
      "activations/layer22_attention_weight_min": -25.247453689575195,
      "activations/layer23_attention_weight_max": 36.46333312988281,
      "activations/layer23_attention_weight_min": -24.527090072631836,
      "activations/layer2_attention_weight_max": 32.40630340576172,
      "activations/layer2_attention_weight_min": -32.06186294555664,
      "activations/layer3_attention_weight_max": 91.8895492553711,
      "activations/layer3_attention_weight_min": -91.80087280273438,
      "activations/layer4_attention_weight_max": 91.03667449951172,
      "activations/layer4_attention_weight_min": -91.85527038574219,
      "activations/layer5_attention_weight_max": 66.06793975830078,
      "activations/layer5_attention_weight_min": -72.62239837646484,
      "activations/layer6_attention_weight_max": 50.65935516357422,
      "activations/layer6_attention_weight_min": -50.85197830200195,
      "activations/layer7_attention_weight_max": 67.28265380859375,
      "activations/layer7_attention_weight_min": -66.35169219970703,
      "activations/layer8_attention_weight_max": 45.21681594848633,
      "activations/layer8_attention_weight_min": -49.83064651489258,
      "activations/layer9_attention_weight_max": 42.43047332763672,
      "activations/layer9_attention_weight_min": -49.01738739013672,
      "epoch": 16.85,
      "learning_rate": 4.173712121212121e-05,
      "loss": 2.7404,
      "step": 290050
    },
    {
      "activations/layer0_attention_weight_max": 15.92931079864502,
      "activations/layer0_attention_weight_min": -13.63142204284668,
      "activations/layer10_attention_weight_max": 34.478736877441406,
      "activations/layer10_attention_weight_min": -34.665138244628906,
      "activations/layer11_attention_weight_max": 31.913848876953125,
      "activations/layer11_attention_weight_min": -35.18038558959961,
      "activations/layer12_attention_weight_max": 17.891464233398438,
      "activations/layer12_attention_weight_min": -26.160982131958008,
      "activations/layer13_attention_weight_max": 33.866519927978516,
      "activations/layer13_attention_weight_min": -28.08563804626465,
      "activations/layer14_attention_weight_max": 33.790679931640625,
      "activations/layer14_attention_weight_min": -31.921308517456055,
      "activations/layer15_attention_weight_max": 33.75828552246094,
      "activations/layer15_attention_weight_min": -29.688234329223633,
      "activations/layer16_attention_weight_max": 32.459537506103516,
      "activations/layer16_attention_weight_min": -30.49308204650879,
      "activations/layer17_attention_weight_max": 51.6867790222168,
      "activations/layer17_attention_weight_min": -44.422393798828125,
      "activations/layer18_attention_weight_max": 45.843963623046875,
      "activations/layer18_attention_weight_min": -38.243919372558594,
      "activations/layer19_attention_weight_max": 25.968505859375,
      "activations/layer19_attention_weight_min": -23.11077308654785,
      "activations/layer1_attention_weight_max": 15.356128692626953,
      "activations/layer1_attention_weight_min": -13.999246597290039,
      "activations/layer20_attention_weight_max": 24.787052154541016,
      "activations/layer20_attention_weight_min": -25.183536529541016,
      "activations/layer21_attention_weight_max": 35.88689041137695,
      "activations/layer21_attention_weight_min": -20.447805404663086,
      "activations/layer22_attention_weight_max": 31.539230346679688,
      "activations/layer22_attention_weight_min": -26.528244018554688,
      "activations/layer23_attention_weight_max": 39.22711181640625,
      "activations/layer23_attention_weight_min": -23.418140411376953,
      "activations/layer2_attention_weight_max": 31.8881893157959,
      "activations/layer2_attention_weight_min": -31.509428024291992,
      "activations/layer3_attention_weight_max": 92.522216796875,
      "activations/layer3_attention_weight_min": -92.00399017333984,
      "activations/layer4_attention_weight_max": 88.59700012207031,
      "activations/layer4_attention_weight_min": -88.95552062988281,
      "activations/layer5_attention_weight_max": 63.32019805908203,
      "activations/layer5_attention_weight_min": -74.99897766113281,
      "activations/layer6_attention_weight_max": 50.09511184692383,
      "activations/layer6_attention_weight_min": -51.710594177246094,
      "activations/layer7_attention_weight_max": 65.35934448242188,
      "activations/layer7_attention_weight_min": -62.12421798706055,
      "activations/layer8_attention_weight_max": 46.39872360229492,
      "activations/layer8_attention_weight_min": -47.42853927612305,
      "activations/layer9_attention_weight_max": 43.6926383972168,
      "activations/layer9_attention_weight_min": -44.866065979003906,
      "epoch": 16.86,
      "learning_rate": 4.171818181818181e-05,
      "loss": 2.735,
      "step": 290100
    },
    {
      "activations/layer0_attention_weight_max": 15.101025581359863,
      "activations/layer0_attention_weight_min": -13.341245651245117,
      "activations/layer10_attention_weight_max": 36.993896484375,
      "activations/layer10_attention_weight_min": -35.51886749267578,
      "activations/layer11_attention_weight_max": 36.13895797729492,
      "activations/layer11_attention_weight_min": -34.57425308227539,
      "activations/layer12_attention_weight_max": 18.71565818786621,
      "activations/layer12_attention_weight_min": -27.919740676879883,
      "activations/layer13_attention_weight_max": 34.14693069458008,
      "activations/layer13_attention_weight_min": -26.774805068969727,
      "activations/layer14_attention_weight_max": 34.32427978515625,
      "activations/layer14_attention_weight_min": -28.486820220947266,
      "activations/layer15_attention_weight_max": 32.33096694946289,
      "activations/layer15_attention_weight_min": -28.70688247680664,
      "activations/layer16_attention_weight_max": 31.930997848510742,
      "activations/layer16_attention_weight_min": -30.663753509521484,
      "activations/layer17_attention_weight_max": 47.417633056640625,
      "activations/layer17_attention_weight_min": -42.035884857177734,
      "activations/layer18_attention_weight_max": 45.20922088623047,
      "activations/layer18_attention_weight_min": -37.04721450805664,
      "activations/layer19_attention_weight_max": 22.42264747619629,
      "activations/layer19_attention_weight_min": -20.752723693847656,
      "activations/layer1_attention_weight_max": 16.261871337890625,
      "activations/layer1_attention_weight_min": -15.257311820983887,
      "activations/layer20_attention_weight_max": 20.73120880126953,
      "activations/layer20_attention_weight_min": -22.45264434814453,
      "activations/layer21_attention_weight_max": 31.6219482421875,
      "activations/layer21_attention_weight_min": -22.026878356933594,
      "activations/layer22_attention_weight_max": 27.602075576782227,
      "activations/layer22_attention_weight_min": -25.917301177978516,
      "activations/layer23_attention_weight_max": 34.70805358886719,
      "activations/layer23_attention_weight_min": -22.869924545288086,
      "activations/layer2_attention_weight_max": 30.53533935546875,
      "activations/layer2_attention_weight_min": -31.286109924316406,
      "activations/layer3_attention_weight_max": 89.04257202148438,
      "activations/layer3_attention_weight_min": -90.2826919555664,
      "activations/layer4_attention_weight_max": 89.8353271484375,
      "activations/layer4_attention_weight_min": -86.72129821777344,
      "activations/layer5_attention_weight_max": 65.240234375,
      "activations/layer5_attention_weight_min": -69.11907958984375,
      "activations/layer6_attention_weight_max": 49.22769546508789,
      "activations/layer6_attention_weight_min": -51.409034729003906,
      "activations/layer7_attention_weight_max": 63.17461395263672,
      "activations/layer7_attention_weight_min": -62.061920166015625,
      "activations/layer8_attention_weight_max": 45.43295669555664,
      "activations/layer8_attention_weight_min": -46.945735931396484,
      "activations/layer9_attention_weight_max": 42.139041900634766,
      "activations/layer9_attention_weight_min": -44.25601577758789,
      "epoch": 16.86,
      "learning_rate": 4.1699242424242425e-05,
      "loss": 2.7629,
      "step": 290150
    },
    {
      "activations/layer0_attention_weight_max": 16.138477325439453,
      "activations/layer0_attention_weight_min": -13.395305633544922,
      "activations/layer10_attention_weight_max": 35.54365539550781,
      "activations/layer10_attention_weight_min": -35.21699142456055,
      "activations/layer11_attention_weight_max": 33.60643768310547,
      "activations/layer11_attention_weight_min": -33.4323616027832,
      "activations/layer12_attention_weight_max": 18.3015079498291,
      "activations/layer12_attention_weight_min": -25.806127548217773,
      "activations/layer13_attention_weight_max": 31.365509033203125,
      "activations/layer13_attention_weight_min": -26.95732879638672,
      "activations/layer14_attention_weight_max": 34.128395080566406,
      "activations/layer14_attention_weight_min": -30.346426010131836,
      "activations/layer15_attention_weight_max": 31.694150924682617,
      "activations/layer15_attention_weight_min": -29.804676055908203,
      "activations/layer16_attention_weight_max": 32.602195739746094,
      "activations/layer16_attention_weight_min": -30.751602172851562,
      "activations/layer17_attention_weight_max": 48.455570220947266,
      "activations/layer17_attention_weight_min": -43.23456573486328,
      "activations/layer18_attention_weight_max": 41.64458465576172,
      "activations/layer18_attention_weight_min": -37.92158126831055,
      "activations/layer19_attention_weight_max": 22.233400344848633,
      "activations/layer19_attention_weight_min": -21.939056396484375,
      "activations/layer1_attention_weight_max": 15.820901870727539,
      "activations/layer1_attention_weight_min": -14.808867454528809,
      "activations/layer20_attention_weight_max": 20.142906188964844,
      "activations/layer20_attention_weight_min": -23.96104621887207,
      "activations/layer21_attention_weight_max": 30.823484420776367,
      "activations/layer21_attention_weight_min": -20.778053283691406,
      "activations/layer22_attention_weight_max": 29.915706634521484,
      "activations/layer22_attention_weight_min": -25.95128059387207,
      "activations/layer23_attention_weight_max": 34.36011505126953,
      "activations/layer23_attention_weight_min": -24.838275909423828,
      "activations/layer2_attention_weight_max": 34.57460021972656,
      "activations/layer2_attention_weight_min": -32.719295501708984,
      "activations/layer3_attention_weight_max": 92.0583267211914,
      "activations/layer3_attention_weight_min": -90.8594970703125,
      "activations/layer4_attention_weight_max": 90.43904876708984,
      "activations/layer4_attention_weight_min": -93.18431091308594,
      "activations/layer5_attention_weight_max": 66.15703582763672,
      "activations/layer5_attention_weight_min": -75.33123016357422,
      "activations/layer6_attention_weight_max": 49.26356506347656,
      "activations/layer6_attention_weight_min": -51.00881576538086,
      "activations/layer7_attention_weight_max": 67.2790756225586,
      "activations/layer7_attention_weight_min": -62.951194763183594,
      "activations/layer8_attention_weight_max": 46.26824951171875,
      "activations/layer8_attention_weight_min": -47.271873474121094,
      "activations/layer9_attention_weight_max": 45.860198974609375,
      "activations/layer9_attention_weight_min": -45.5056266784668,
      "epoch": 16.86,
      "learning_rate": 4.168030303030303e-05,
      "loss": 2.7409,
      "step": 290200
    },
    {
      "activations/layer0_attention_weight_max": 16.044143676757812,
      "activations/layer0_attention_weight_min": -14.37779426574707,
      "activations/layer10_attention_weight_max": 38.657230377197266,
      "activations/layer10_attention_weight_min": -37.27975082397461,
      "activations/layer11_attention_weight_max": 34.489139556884766,
      "activations/layer11_attention_weight_min": -35.503055572509766,
      "activations/layer12_attention_weight_max": 18.305679321289062,
      "activations/layer12_attention_weight_min": -26.16870880126953,
      "activations/layer13_attention_weight_max": 35.700286865234375,
      "activations/layer13_attention_weight_min": -29.21963119506836,
      "activations/layer14_attention_weight_max": 35.778953552246094,
      "activations/layer14_attention_weight_min": -29.668935775756836,
      "activations/layer15_attention_weight_max": 34.01332092285156,
      "activations/layer15_attention_weight_min": -29.742977142333984,
      "activations/layer16_attention_weight_max": 35.31855773925781,
      "activations/layer16_attention_weight_min": -31.376453399658203,
      "activations/layer17_attention_weight_max": 53.04039764404297,
      "activations/layer17_attention_weight_min": -44.58237075805664,
      "activations/layer18_attention_weight_max": 49.02878189086914,
      "activations/layer18_attention_weight_min": -37.01041793823242,
      "activations/layer19_attention_weight_max": 24.722492218017578,
      "activations/layer19_attention_weight_min": -21.25577163696289,
      "activations/layer1_attention_weight_max": 16.72806167602539,
      "activations/layer1_attention_weight_min": -14.88839340209961,
      "activations/layer20_attention_weight_max": 24.23423194885254,
      "activations/layer20_attention_weight_min": -21.78148651123047,
      "activations/layer21_attention_weight_max": 32.981040954589844,
      "activations/layer21_attention_weight_min": -20.83087158203125,
      "activations/layer22_attention_weight_max": 28.486061096191406,
      "activations/layer22_attention_weight_min": -24.3660831451416,
      "activations/layer23_attention_weight_max": 36.95215606689453,
      "activations/layer23_attention_weight_min": -24.488784790039062,
      "activations/layer2_attention_weight_max": 33.809059143066406,
      "activations/layer2_attention_weight_min": -31.84589385986328,
      "activations/layer3_attention_weight_max": 90.0075454711914,
      "activations/layer3_attention_weight_min": -92.0324478149414,
      "activations/layer4_attention_weight_max": 94.4298095703125,
      "activations/layer4_attention_weight_min": -92.62178039550781,
      "activations/layer5_attention_weight_max": 65.318115234375,
      "activations/layer5_attention_weight_min": -76.54633331298828,
      "activations/layer6_attention_weight_max": 52.620216369628906,
      "activations/layer6_attention_weight_min": -54.153011322021484,
      "activations/layer7_attention_weight_max": 64.08910369873047,
      "activations/layer7_attention_weight_min": -64.28634643554688,
      "activations/layer8_attention_weight_max": 50.616390228271484,
      "activations/layer8_attention_weight_min": -52.274959564208984,
      "activations/layer9_attention_weight_max": 46.914249420166016,
      "activations/layer9_attention_weight_min": -47.527076721191406,
      "epoch": 16.87,
      "learning_rate": 4.166136363636363e-05,
      "loss": 2.7624,
      "step": 290250
    },
    {
      "activations/layer0_attention_weight_max": 16.093542098999023,
      "activations/layer0_attention_weight_min": -13.92353630065918,
      "activations/layer10_attention_weight_max": 36.41184997558594,
      "activations/layer10_attention_weight_min": -37.212398529052734,
      "activations/layer11_attention_weight_max": 33.10329055786133,
      "activations/layer11_attention_weight_min": -33.23279571533203,
      "activations/layer12_attention_weight_max": 18.37676239013672,
      "activations/layer12_attention_weight_min": -26.984378814697266,
      "activations/layer13_attention_weight_max": 38.53019714355469,
      "activations/layer13_attention_weight_min": -29.44854736328125,
      "activations/layer14_attention_weight_max": 38.110992431640625,
      "activations/layer14_attention_weight_min": -31.317907333374023,
      "activations/layer15_attention_weight_max": 34.970130920410156,
      "activations/layer15_attention_weight_min": -31.021970748901367,
      "activations/layer16_attention_weight_max": 35.128849029541016,
      "activations/layer16_attention_weight_min": -32.611114501953125,
      "activations/layer17_attention_weight_max": 50.359649658203125,
      "activations/layer17_attention_weight_min": -46.14936447143555,
      "activations/layer18_attention_weight_max": 44.679466247558594,
      "activations/layer18_attention_weight_min": -39.90523910522461,
      "activations/layer19_attention_weight_max": 23.75660514831543,
      "activations/layer19_attention_weight_min": -23.296634674072266,
      "activations/layer1_attention_weight_max": 16.355777740478516,
      "activations/layer1_attention_weight_min": -14.164349555969238,
      "activations/layer20_attention_weight_max": 23.690258026123047,
      "activations/layer20_attention_weight_min": -22.246362686157227,
      "activations/layer21_attention_weight_max": 37.215850830078125,
      "activations/layer21_attention_weight_min": -22.479835510253906,
      "activations/layer22_attention_weight_max": 28.540355682373047,
      "activations/layer22_attention_weight_min": -26.4018611907959,
      "activations/layer23_attention_weight_max": 35.583988189697266,
      "activations/layer23_attention_weight_min": -25.276052474975586,
      "activations/layer2_attention_weight_max": 31.424365997314453,
      "activations/layer2_attention_weight_min": -30.6163330078125,
      "activations/layer3_attention_weight_max": 91.91355895996094,
      "activations/layer3_attention_weight_min": -90.01338958740234,
      "activations/layer4_attention_weight_max": 90.60303497314453,
      "activations/layer4_attention_weight_min": -87.9193344116211,
      "activations/layer5_attention_weight_max": 68.7013931274414,
      "activations/layer5_attention_weight_min": -74.39884948730469,
      "activations/layer6_attention_weight_max": 50.9119758605957,
      "activations/layer6_attention_weight_min": -51.427921295166016,
      "activations/layer7_attention_weight_max": 67.07394409179688,
      "activations/layer7_attention_weight_min": -63.43164825439453,
      "activations/layer8_attention_weight_max": 48.79912567138672,
      "activations/layer8_attention_weight_min": -50.41838836669922,
      "activations/layer9_attention_weight_max": 43.85594940185547,
      "activations/layer9_attention_weight_min": -45.66714096069336,
      "epoch": 16.87,
      "learning_rate": 4.1642424242424237e-05,
      "loss": 2.7514,
      "step": 290300
    },
    {
      "activations/layer0_attention_weight_max": 15.526769638061523,
      "activations/layer0_attention_weight_min": -13.83507251739502,
      "activations/layer10_attention_weight_max": 37.44514846801758,
      "activations/layer10_attention_weight_min": -37.30048370361328,
      "activations/layer11_attention_weight_max": 35.648712158203125,
      "activations/layer11_attention_weight_min": -35.50639343261719,
      "activations/layer12_attention_weight_max": 20.421934127807617,
      "activations/layer12_attention_weight_min": -24.37739372253418,
      "activations/layer13_attention_weight_max": 34.53806686401367,
      "activations/layer13_attention_weight_min": -27.774646759033203,
      "activations/layer14_attention_weight_max": 37.46558380126953,
      "activations/layer14_attention_weight_min": -30.995615005493164,
      "activations/layer15_attention_weight_max": 33.65708541870117,
      "activations/layer15_attention_weight_min": -31.062589645385742,
      "activations/layer16_attention_weight_max": 35.15201950073242,
      "activations/layer16_attention_weight_min": -35.22867965698242,
      "activations/layer17_attention_weight_max": 51.974544525146484,
      "activations/layer17_attention_weight_min": -47.60075759887695,
      "activations/layer18_attention_weight_max": 45.31537628173828,
      "activations/layer18_attention_weight_min": -40.896183013916016,
      "activations/layer19_attention_weight_max": 24.379207611083984,
      "activations/layer19_attention_weight_min": -22.26072883605957,
      "activations/layer1_attention_weight_max": 15.495769500732422,
      "activations/layer1_attention_weight_min": -14.373214721679688,
      "activations/layer20_attention_weight_max": 22.237041473388672,
      "activations/layer20_attention_weight_min": -21.35963249206543,
      "activations/layer21_attention_weight_max": 36.47025680541992,
      "activations/layer21_attention_weight_min": -25.069555282592773,
      "activations/layer22_attention_weight_max": 32.47354507446289,
      "activations/layer22_attention_weight_min": -25.46457290649414,
      "activations/layer23_attention_weight_max": 34.71484375,
      "activations/layer23_attention_weight_min": -23.10586166381836,
      "activations/layer2_attention_weight_max": 32.377235412597656,
      "activations/layer2_attention_weight_min": -31.66211700439453,
      "activations/layer3_attention_weight_max": 88.42638397216797,
      "activations/layer3_attention_weight_min": -91.07350158691406,
      "activations/layer4_attention_weight_max": 91.45999908447266,
      "activations/layer4_attention_weight_min": -92.5335922241211,
      "activations/layer5_attention_weight_max": 64.2900161743164,
      "activations/layer5_attention_weight_min": -76.42134094238281,
      "activations/layer6_attention_weight_max": 52.765235900878906,
      "activations/layer6_attention_weight_min": -52.82870101928711,
      "activations/layer7_attention_weight_max": 63.6881103515625,
      "activations/layer7_attention_weight_min": -64.84921264648438,
      "activations/layer8_attention_weight_max": 48.17030715942383,
      "activations/layer8_attention_weight_min": -52.59926223754883,
      "activations/layer9_attention_weight_max": 45.2984733581543,
      "activations/layer9_attention_weight_min": -47.38019943237305,
      "epoch": 16.87,
      "learning_rate": 4.1623484848484845e-05,
      "loss": 2.7472,
      "step": 290350
    },
    {
      "activations/layer0_attention_weight_max": 14.961766242980957,
      "activations/layer0_attention_weight_min": -13.800655364990234,
      "activations/layer10_attention_weight_max": 35.00908279418945,
      "activations/layer10_attention_weight_min": -34.1323356628418,
      "activations/layer11_attention_weight_max": 30.395719528198242,
      "activations/layer11_attention_weight_min": -33.351165771484375,
      "activations/layer12_attention_weight_max": 19.82340431213379,
      "activations/layer12_attention_weight_min": -22.487934112548828,
      "activations/layer13_attention_weight_max": 37.53874206542969,
      "activations/layer13_attention_weight_min": -30.39662742614746,
      "activations/layer14_attention_weight_max": 38.252437591552734,
      "activations/layer14_attention_weight_min": -31.925365447998047,
      "activations/layer15_attention_weight_max": 33.37514877319336,
      "activations/layer15_attention_weight_min": -33.280216217041016,
      "activations/layer16_attention_weight_max": 31.508121490478516,
      "activations/layer16_attention_weight_min": -31.069002151489258,
      "activations/layer17_attention_weight_max": 49.68741989135742,
      "activations/layer17_attention_weight_min": -43.72459411621094,
      "activations/layer18_attention_weight_max": 45.6482048034668,
      "activations/layer18_attention_weight_min": -36.86625671386719,
      "activations/layer19_attention_weight_max": 25.151063919067383,
      "activations/layer19_attention_weight_min": -21.486997604370117,
      "activations/layer1_attention_weight_max": 15.430379867553711,
      "activations/layer1_attention_weight_min": -13.420661926269531,
      "activations/layer20_attention_weight_max": 21.574413299560547,
      "activations/layer20_attention_weight_min": -22.089757919311523,
      "activations/layer21_attention_weight_max": 32.95747756958008,
      "activations/layer21_attention_weight_min": -20.89507484436035,
      "activations/layer22_attention_weight_max": 27.023746490478516,
      "activations/layer22_attention_weight_min": -24.76427459716797,
      "activations/layer23_attention_weight_max": 36.30141830444336,
      "activations/layer23_attention_weight_min": -22.508201599121094,
      "activations/layer2_attention_weight_max": 28.892826080322266,
      "activations/layer2_attention_weight_min": -28.93837547302246,
      "activations/layer3_attention_weight_max": 86.33454132080078,
      "activations/layer3_attention_weight_min": -87.8935775756836,
      "activations/layer4_attention_weight_max": 87.46780395507812,
      "activations/layer4_attention_weight_min": -83.74551391601562,
      "activations/layer5_attention_weight_max": 66.21263122558594,
      "activations/layer5_attention_weight_min": -76.95741271972656,
      "activations/layer6_attention_weight_max": 48.581016540527344,
      "activations/layer6_attention_weight_min": -48.792049407958984,
      "activations/layer7_attention_weight_max": 64.76996612548828,
      "activations/layer7_attention_weight_min": -67.35675811767578,
      "activations/layer8_attention_weight_max": 43.20539855957031,
      "activations/layer8_attention_weight_min": -45.8093147277832,
      "activations/layer9_attention_weight_max": 42.35182571411133,
      "activations/layer9_attention_weight_min": -44.447689056396484,
      "epoch": 16.87,
      "learning_rate": 4.160454545454545e-05,
      "loss": 2.7559,
      "step": 290400
    },
    {
      "activations/layer0_attention_weight_max": 16.519102096557617,
      "activations/layer0_attention_weight_min": -13.54745101928711,
      "activations/layer10_attention_weight_max": 49.58011245727539,
      "activations/layer10_attention_weight_min": -49.63825607299805,
      "activations/layer11_attention_weight_max": 44.66609191894531,
      "activations/layer11_attention_weight_min": -42.853546142578125,
      "activations/layer12_attention_weight_max": 19.409265518188477,
      "activations/layer12_attention_weight_min": -25.4705810546875,
      "activations/layer13_attention_weight_max": 53.554222106933594,
      "activations/layer13_attention_weight_min": -32.44700241088867,
      "activations/layer14_attention_weight_max": 54.60023498535156,
      "activations/layer14_attention_weight_min": -37.08203887939453,
      "activations/layer15_attention_weight_max": 58.310203552246094,
      "activations/layer15_attention_weight_min": -37.25526428222656,
      "activations/layer16_attention_weight_max": 41.605491638183594,
      "activations/layer16_attention_weight_min": -37.032779693603516,
      "activations/layer17_attention_weight_max": 60.13376998901367,
      "activations/layer17_attention_weight_min": -54.457820892333984,
      "activations/layer18_attention_weight_max": 54.4857063293457,
      "activations/layer18_attention_weight_min": -47.030269622802734,
      "activations/layer19_attention_weight_max": 23.910099029541016,
      "activations/layer19_attention_weight_min": -24.581012725830078,
      "activations/layer1_attention_weight_max": 16.439531326293945,
      "activations/layer1_attention_weight_min": -14.914055824279785,
      "activations/layer20_attention_weight_max": 22.375024795532227,
      "activations/layer20_attention_weight_min": -21.96967124938965,
      "activations/layer21_attention_weight_max": 33.90864181518555,
      "activations/layer21_attention_weight_min": -24.368595123291016,
      "activations/layer22_attention_weight_max": 32.26539611816406,
      "activations/layer22_attention_weight_min": -24.834890365600586,
      "activations/layer23_attention_weight_max": 38.410987854003906,
      "activations/layer23_attention_weight_min": -27.15981674194336,
      "activations/layer2_attention_weight_max": 32.118560791015625,
      "activations/layer2_attention_weight_min": -34.15069580078125,
      "activations/layer3_attention_weight_max": 91.46159362792969,
      "activations/layer3_attention_weight_min": -97.35627746582031,
      "activations/layer4_attention_weight_max": 93.22333526611328,
      "activations/layer4_attention_weight_min": -90.6700668334961,
      "activations/layer5_attention_weight_max": 67.41712188720703,
      "activations/layer5_attention_weight_min": -75.59349060058594,
      "activations/layer6_attention_weight_max": 58.23902893066406,
      "activations/layer6_attention_weight_min": -59.49391174316406,
      "activations/layer7_attention_weight_max": 90.7138900756836,
      "activations/layer7_attention_weight_min": -77.6554183959961,
      "activations/layer8_attention_weight_max": 61.44887161254883,
      "activations/layer8_attention_weight_min": -61.556884765625,
      "activations/layer9_attention_weight_max": 63.58386993408203,
      "activations/layer9_attention_weight_min": -55.413970947265625,
      "epoch": 16.88,
      "learning_rate": 4.1585606060606055e-05,
      "loss": 2.7489,
      "step": 290450
    },
    {
      "activations/layer0_attention_weight_max": 16.63170051574707,
      "activations/layer0_attention_weight_min": -13.46920394897461,
      "activations/layer10_attention_weight_max": 40.95387649536133,
      "activations/layer10_attention_weight_min": -37.03358459472656,
      "activations/layer11_attention_weight_max": 33.68085861206055,
      "activations/layer11_attention_weight_min": -33.573055267333984,
      "activations/layer12_attention_weight_max": 18.895919799804688,
      "activations/layer12_attention_weight_min": -25.734846115112305,
      "activations/layer13_attention_weight_max": 35.37905502319336,
      "activations/layer13_attention_weight_min": -27.04659652709961,
      "activations/layer14_attention_weight_max": 37.004066467285156,
      "activations/layer14_attention_weight_min": -30.918609619140625,
      "activations/layer15_attention_weight_max": 34.25891876220703,
      "activations/layer15_attention_weight_min": -30.803529739379883,
      "activations/layer16_attention_weight_max": 33.75011444091797,
      "activations/layer16_attention_weight_min": -32.214290618896484,
      "activations/layer17_attention_weight_max": 52.36043167114258,
      "activations/layer17_attention_weight_min": -46.431602478027344,
      "activations/layer18_attention_weight_max": 46.678226470947266,
      "activations/layer18_attention_weight_min": -39.78315734863281,
      "activations/layer19_attention_weight_max": 22.394742965698242,
      "activations/layer19_attention_weight_min": -23.708574295043945,
      "activations/layer1_attention_weight_max": 16.03119659423828,
      "activations/layer1_attention_weight_min": -13.699126243591309,
      "activations/layer20_attention_weight_max": 23.471229553222656,
      "activations/layer20_attention_weight_min": -21.1976318359375,
      "activations/layer21_attention_weight_max": 37.68168258666992,
      "activations/layer21_attention_weight_min": -23.039966583251953,
      "activations/layer22_attention_weight_max": 27.826841354370117,
      "activations/layer22_attention_weight_min": -25.402015686035156,
      "activations/layer23_attention_weight_max": 36.44379806518555,
      "activations/layer23_attention_weight_min": -25.494909286499023,
      "activations/layer2_attention_weight_max": 31.299657821655273,
      "activations/layer2_attention_weight_min": -28.98786163330078,
      "activations/layer3_attention_weight_max": 87.01299285888672,
      "activations/layer3_attention_weight_min": -88.21639251708984,
      "activations/layer4_attention_weight_max": 89.24243927001953,
      "activations/layer4_attention_weight_min": -90.05524444580078,
      "activations/layer5_attention_weight_max": 63.929046630859375,
      "activations/layer5_attention_weight_min": -71.57472229003906,
      "activations/layer6_attention_weight_max": 53.25453186035156,
      "activations/layer6_attention_weight_min": -53.20847702026367,
      "activations/layer7_attention_weight_max": 66.65271759033203,
      "activations/layer7_attention_weight_min": -65.65982818603516,
      "activations/layer8_attention_weight_max": 49.58598327636719,
      "activations/layer8_attention_weight_min": -51.84450149536133,
      "activations/layer9_attention_weight_max": 47.37382125854492,
      "activations/layer9_attention_weight_min": -46.760009765625,
      "epoch": 16.88,
      "learning_rate": 4.1566666666666656e-05,
      "loss": 2.7621,
      "step": 290500
    },
    {
      "activations/layer0_attention_weight_max": 16.540077209472656,
      "activations/layer0_attention_weight_min": -13.704160690307617,
      "activations/layer10_attention_weight_max": 37.689300537109375,
      "activations/layer10_attention_weight_min": -35.55660629272461,
      "activations/layer11_attention_weight_max": 35.45911407470703,
      "activations/layer11_attention_weight_min": -35.40952682495117,
      "activations/layer12_attention_weight_max": 18.34323501586914,
      "activations/layer12_attention_weight_min": -25.260149002075195,
      "activations/layer13_attention_weight_max": 35.058292388916016,
      "activations/layer13_attention_weight_min": -28.811134338378906,
      "activations/layer14_attention_weight_max": 38.898250579833984,
      "activations/layer14_attention_weight_min": -30.011247634887695,
      "activations/layer15_attention_weight_max": 35.54762268066406,
      "activations/layer15_attention_weight_min": -29.152420043945312,
      "activations/layer16_attention_weight_max": 35.988441467285156,
      "activations/layer16_attention_weight_min": -35.78128433227539,
      "activations/layer17_attention_weight_max": 55.59394073486328,
      "activations/layer17_attention_weight_min": -51.326148986816406,
      "activations/layer18_attention_weight_max": 48.81132125854492,
      "activations/layer18_attention_weight_min": -43.42782974243164,
      "activations/layer19_attention_weight_max": 24.67859649658203,
      "activations/layer19_attention_weight_min": -23.6998348236084,
      "activations/layer1_attention_weight_max": 16.22240447998047,
      "activations/layer1_attention_weight_min": -15.185912132263184,
      "activations/layer20_attention_weight_max": 23.242359161376953,
      "activations/layer20_attention_weight_min": -19.764068603515625,
      "activations/layer21_attention_weight_max": 33.47819900512695,
      "activations/layer21_attention_weight_min": -24.95149803161621,
      "activations/layer22_attention_weight_max": 28.362424850463867,
      "activations/layer22_attention_weight_min": -24.686603546142578,
      "activations/layer23_attention_weight_max": 35.75674819946289,
      "activations/layer23_attention_weight_min": -25.147422790527344,
      "activations/layer2_attention_weight_max": 33.232177734375,
      "activations/layer2_attention_weight_min": -31.78521728515625,
      "activations/layer3_attention_weight_max": 90.04582214355469,
      "activations/layer3_attention_weight_min": -89.11095428466797,
      "activations/layer4_attention_weight_max": 91.66655731201172,
      "activations/layer4_attention_weight_min": -89.90496063232422,
      "activations/layer5_attention_weight_max": 70.08183288574219,
      "activations/layer5_attention_weight_min": -74.52620697021484,
      "activations/layer6_attention_weight_max": 52.35379409790039,
      "activations/layer6_attention_weight_min": -56.479705810546875,
      "activations/layer7_attention_weight_max": 68.97802734375,
      "activations/layer7_attention_weight_min": -64.75748443603516,
      "activations/layer8_attention_weight_max": 46.22837448120117,
      "activations/layer8_attention_weight_min": -48.53022384643555,
      "activations/layer9_attention_weight_max": 43.33323669433594,
      "activations/layer9_attention_weight_min": -47.6385612487793,
      "epoch": 16.88,
      "learning_rate": 4.154772727272727e-05,
      "loss": 2.7528,
      "step": 290550
    },
    {
      "activations/layer0_attention_weight_max": 16.33397102355957,
      "activations/layer0_attention_weight_min": -13.712647438049316,
      "activations/layer10_attention_weight_max": 36.43314743041992,
      "activations/layer10_attention_weight_min": -35.84457778930664,
      "activations/layer11_attention_weight_max": 31.699560165405273,
      "activations/layer11_attention_weight_min": -34.41400146484375,
      "activations/layer12_attention_weight_max": 18.376201629638672,
      "activations/layer12_attention_weight_min": -27.6109561920166,
      "activations/layer13_attention_weight_max": 32.35982894897461,
      "activations/layer13_attention_weight_min": -29.61104965209961,
      "activations/layer14_attention_weight_max": 31.849592208862305,
      "activations/layer14_attention_weight_min": -30.357006072998047,
      "activations/layer15_attention_weight_max": 29.39019203186035,
      "activations/layer15_attention_weight_min": -28.754497528076172,
      "activations/layer16_attention_weight_max": 31.369949340820312,
      "activations/layer16_attention_weight_min": -31.10765266418457,
      "activations/layer17_attention_weight_max": 47.42601013183594,
      "activations/layer17_attention_weight_min": -43.07179641723633,
      "activations/layer18_attention_weight_max": 40.93962097167969,
      "activations/layer18_attention_weight_min": -37.662811279296875,
      "activations/layer19_attention_weight_max": 21.786922454833984,
      "activations/layer19_attention_weight_min": -22.362442016601562,
      "activations/layer1_attention_weight_max": 15.445758819580078,
      "activations/layer1_attention_weight_min": -13.919337272644043,
      "activations/layer20_attention_weight_max": 20.630826950073242,
      "activations/layer20_attention_weight_min": -23.516843795776367,
      "activations/layer21_attention_weight_max": 32.1898193359375,
      "activations/layer21_attention_weight_min": -21.209997177124023,
      "activations/layer22_attention_weight_max": 28.1392879486084,
      "activations/layer22_attention_weight_min": -29.788591384887695,
      "activations/layer23_attention_weight_max": 37.143531799316406,
      "activations/layer23_attention_weight_min": -23.421903610229492,
      "activations/layer2_attention_weight_max": 31.069623947143555,
      "activations/layer2_attention_weight_min": -31.51814079284668,
      "activations/layer3_attention_weight_max": 86.93487548828125,
      "activations/layer3_attention_weight_min": -91.95769500732422,
      "activations/layer4_attention_weight_max": 88.65764617919922,
      "activations/layer4_attention_weight_min": -91.88024139404297,
      "activations/layer5_attention_weight_max": 62.012611389160156,
      "activations/layer5_attention_weight_min": -74.2248306274414,
      "activations/layer6_attention_weight_max": 52.060813903808594,
      "activations/layer6_attention_weight_min": -51.303627014160156,
      "activations/layer7_attention_weight_max": 64.02242279052734,
      "activations/layer7_attention_weight_min": -63.51502990722656,
      "activations/layer8_attention_weight_max": 46.41141891479492,
      "activations/layer8_attention_weight_min": -47.385459899902344,
      "activations/layer9_attention_weight_max": 42.70283508300781,
      "activations/layer9_attention_weight_min": -44.17079162597656,
      "epoch": 16.89,
      "learning_rate": 4.152878787878787e-05,
      "loss": 2.7367,
      "step": 290600
    },
    {
      "activations/layer0_attention_weight_max": 16.465009689331055,
      "activations/layer0_attention_weight_min": -13.725044250488281,
      "activations/layer10_attention_weight_max": 35.51264572143555,
      "activations/layer10_attention_weight_min": -35.351539611816406,
      "activations/layer11_attention_weight_max": 32.721458435058594,
      "activations/layer11_attention_weight_min": -33.654354095458984,
      "activations/layer12_attention_weight_max": 17.181644439697266,
      "activations/layer12_attention_weight_min": -26.752578735351562,
      "activations/layer13_attention_weight_max": 33.85023880004883,
      "activations/layer13_attention_weight_min": -26.41587257385254,
      "activations/layer14_attention_weight_max": 37.09007263183594,
      "activations/layer14_attention_weight_min": -29.78714942932129,
      "activations/layer15_attention_weight_max": 35.51341247558594,
      "activations/layer15_attention_weight_min": -29.063251495361328,
      "activations/layer16_attention_weight_max": 31.865312576293945,
      "activations/layer16_attention_weight_min": -30.49072265625,
      "activations/layer17_attention_weight_max": 49.64891052246094,
      "activations/layer17_attention_weight_min": -43.767311096191406,
      "activations/layer18_attention_weight_max": 40.946537017822266,
      "activations/layer18_attention_weight_min": -40.99753952026367,
      "activations/layer19_attention_weight_max": 24.14223289489746,
      "activations/layer19_attention_weight_min": -23.09103012084961,
      "activations/layer1_attention_weight_max": 16.230981826782227,
      "activations/layer1_attention_weight_min": -14.463801383972168,
      "activations/layer20_attention_weight_max": 20.898176193237305,
      "activations/layer20_attention_weight_min": -19.971187591552734,
      "activations/layer21_attention_weight_max": 33.51981735229492,
      "activations/layer21_attention_weight_min": -22.325145721435547,
      "activations/layer22_attention_weight_max": 25.851898193359375,
      "activations/layer22_attention_weight_min": -24.0446834564209,
      "activations/layer23_attention_weight_max": 32.2098388671875,
      "activations/layer23_attention_weight_min": -23.520824432373047,
      "activations/layer2_attention_weight_max": 31.29290771484375,
      "activations/layer2_attention_weight_min": -29.721708297729492,
      "activations/layer3_attention_weight_max": 88.9984359741211,
      "activations/layer3_attention_weight_min": -88.96326446533203,
      "activations/layer4_attention_weight_max": 88.61630249023438,
      "activations/layer4_attention_weight_min": -82.85453033447266,
      "activations/layer5_attention_weight_max": 64.99049377441406,
      "activations/layer5_attention_weight_min": -75.644775390625,
      "activations/layer6_attention_weight_max": 47.58458709716797,
      "activations/layer6_attention_weight_min": -47.70991134643555,
      "activations/layer7_attention_weight_max": 68.15898895263672,
      "activations/layer7_attention_weight_min": -59.70136642456055,
      "activations/layer8_attention_weight_max": 46.712188720703125,
      "activations/layer8_attention_weight_min": -48.508750915527344,
      "activations/layer9_attention_weight_max": 48.91710662841797,
      "activations/layer9_attention_weight_min": -44.083003997802734,
      "epoch": 16.89,
      "learning_rate": 4.151022727272727e-05,
      "loss": 2.7365,
      "step": 290650
    },
    {
      "activations/layer0_attention_weight_max": 16.942533493041992,
      "activations/layer0_attention_weight_min": -13.396855354309082,
      "activations/layer10_attention_weight_max": 37.7727165222168,
      "activations/layer10_attention_weight_min": -37.20671081542969,
      "activations/layer11_attention_weight_max": 35.97235107421875,
      "activations/layer11_attention_weight_min": -33.495452880859375,
      "activations/layer12_attention_weight_max": 18.768701553344727,
      "activations/layer12_attention_weight_min": -26.305830001831055,
      "activations/layer13_attention_weight_max": 34.838134765625,
      "activations/layer13_attention_weight_min": -28.053085327148438,
      "activations/layer14_attention_weight_max": 35.95671844482422,
      "activations/layer14_attention_weight_min": -30.784381866455078,
      "activations/layer15_attention_weight_max": 34.02527618408203,
      "activations/layer15_attention_weight_min": -30.599393844604492,
      "activations/layer16_attention_weight_max": 35.4453239440918,
      "activations/layer16_attention_weight_min": -34.709476470947266,
      "activations/layer17_attention_weight_max": 57.61032485961914,
      "activations/layer17_attention_weight_min": -51.50277328491211,
      "activations/layer18_attention_weight_max": 53.95863723754883,
      "activations/layer18_attention_weight_min": -45.191707611083984,
      "activations/layer19_attention_weight_max": 24.374387741088867,
      "activations/layer19_attention_weight_min": -24.540428161621094,
      "activations/layer1_attention_weight_max": 16.90518569946289,
      "activations/layer1_attention_weight_min": -14.014235496520996,
      "activations/layer20_attention_weight_max": 23.00210952758789,
      "activations/layer20_attention_weight_min": -22.68924331665039,
      "activations/layer21_attention_weight_max": 38.44162368774414,
      "activations/layer21_attention_weight_min": -26.457256317138672,
      "activations/layer22_attention_weight_max": 31.684749603271484,
      "activations/layer22_attention_weight_min": -27.66326332092285,
      "activations/layer23_attention_weight_max": 35.268768310546875,
      "activations/layer23_attention_weight_min": -24.044326782226562,
      "activations/layer2_attention_weight_max": 35.1214599609375,
      "activations/layer2_attention_weight_min": -33.64870834350586,
      "activations/layer3_attention_weight_max": 91.51486206054688,
      "activations/layer3_attention_weight_min": -97.94339752197266,
      "activations/layer4_attention_weight_max": 91.3246841430664,
      "activations/layer4_attention_weight_min": -90.16802978515625,
      "activations/layer5_attention_weight_max": 68.07264709472656,
      "activations/layer5_attention_weight_min": -71.90340423583984,
      "activations/layer6_attention_weight_max": 51.007225036621094,
      "activations/layer6_attention_weight_min": -50.60647964477539,
      "activations/layer7_attention_weight_max": 64.06403350830078,
      "activations/layer7_attention_weight_min": -65.34211730957031,
      "activations/layer8_attention_weight_max": 46.67474365234375,
      "activations/layer8_attention_weight_min": -49.2198486328125,
      "activations/layer9_attention_weight_max": 46.485530853271484,
      "activations/layer9_attention_weight_min": -45.53569030761719,
      "epoch": 16.89,
      "learning_rate": 4.149128787878787e-05,
      "loss": 2.7439,
      "step": 290700
    },
    {
      "activations/layer0_attention_weight_max": 15.236335754394531,
      "activations/layer0_attention_weight_min": -14.134269714355469,
      "activations/layer10_attention_weight_max": 36.724735260009766,
      "activations/layer10_attention_weight_min": -37.65239334106445,
      "activations/layer11_attention_weight_max": 33.856239318847656,
      "activations/layer11_attention_weight_min": -35.03929138183594,
      "activations/layer12_attention_weight_max": 19.57823371887207,
      "activations/layer12_attention_weight_min": -28.006656646728516,
      "activations/layer13_attention_weight_max": 36.45221710205078,
      "activations/layer13_attention_weight_min": -28.363933563232422,
      "activations/layer14_attention_weight_max": 33.451568603515625,
      "activations/layer14_attention_weight_min": -31.089967727661133,
      "activations/layer15_attention_weight_max": 31.609697341918945,
      "activations/layer15_attention_weight_min": -29.636402130126953,
      "activations/layer16_attention_weight_max": 34.395233154296875,
      "activations/layer16_attention_weight_min": -29.92940902709961,
      "activations/layer17_attention_weight_max": 49.020992279052734,
      "activations/layer17_attention_weight_min": -42.80046081542969,
      "activations/layer18_attention_weight_max": 44.59199905395508,
      "activations/layer18_attention_weight_min": -38.60488510131836,
      "activations/layer19_attention_weight_max": 23.546850204467773,
      "activations/layer19_attention_weight_min": -21.445295333862305,
      "activations/layer1_attention_weight_max": 16.323152542114258,
      "activations/layer1_attention_weight_min": -13.925944328308105,
      "activations/layer20_attention_weight_max": 22.344083786010742,
      "activations/layer20_attention_weight_min": -22.535791397094727,
      "activations/layer21_attention_weight_max": 35.481807708740234,
      "activations/layer21_attention_weight_min": -21.58092498779297,
      "activations/layer22_attention_weight_max": 26.5096435546875,
      "activations/layer22_attention_weight_min": -25.57023811340332,
      "activations/layer23_attention_weight_max": 32.5716438293457,
      "activations/layer23_attention_weight_min": -23.19124984741211,
      "activations/layer2_attention_weight_max": 30.76710319519043,
      "activations/layer2_attention_weight_min": -28.932025909423828,
      "activations/layer3_attention_weight_max": 89.33502197265625,
      "activations/layer3_attention_weight_min": -92.95375061035156,
      "activations/layer4_attention_weight_max": 89.68453979492188,
      "activations/layer4_attention_weight_min": -89.72150421142578,
      "activations/layer5_attention_weight_max": 60.63373565673828,
      "activations/layer5_attention_weight_min": -70.52204895019531,
      "activations/layer6_attention_weight_max": 49.86193084716797,
      "activations/layer6_attention_weight_min": -51.66474533081055,
      "activations/layer7_attention_weight_max": 66.4377670288086,
      "activations/layer7_attention_weight_min": -61.13101577758789,
      "activations/layer8_attention_weight_max": 45.46442794799805,
      "activations/layer8_attention_weight_min": -49.501216888427734,
      "activations/layer9_attention_weight_max": 46.6043815612793,
      "activations/layer9_attention_weight_min": -47.06998825073242,
      "epoch": 16.89,
      "learning_rate": 4.147234848484848e-05,
      "loss": 2.746,
      "step": 290750
    },
    {
      "activations/layer0_attention_weight_max": 16.26201820373535,
      "activations/layer0_attention_weight_min": -13.897000312805176,
      "activations/layer10_attention_weight_max": 31.62315559387207,
      "activations/layer10_attention_weight_min": -32.349273681640625,
      "activations/layer11_attention_weight_max": 30.072675704956055,
      "activations/layer11_attention_weight_min": -32.072818756103516,
      "activations/layer12_attention_weight_max": 20.154293060302734,
      "activations/layer12_attention_weight_min": -29.076053619384766,
      "activations/layer13_attention_weight_max": 35.01063919067383,
      "activations/layer13_attention_weight_min": -26.90329360961914,
      "activations/layer14_attention_weight_max": 37.046775817871094,
      "activations/layer14_attention_weight_min": -29.422826766967773,
      "activations/layer15_attention_weight_max": 32.892303466796875,
      "activations/layer15_attention_weight_min": -30.933706283569336,
      "activations/layer16_attention_weight_max": 34.36808395385742,
      "activations/layer16_attention_weight_min": -33.21421813964844,
      "activations/layer17_attention_weight_max": 51.51454162597656,
      "activations/layer17_attention_weight_min": -45.76858901977539,
      "activations/layer18_attention_weight_max": 45.1300163269043,
      "activations/layer18_attention_weight_min": -35.89836883544922,
      "activations/layer19_attention_weight_max": 25.461172103881836,
      "activations/layer19_attention_weight_min": -21.211563110351562,
      "activations/layer1_attention_weight_max": 16.09756851196289,
      "activations/layer1_attention_weight_min": -13.829999923706055,
      "activations/layer20_attention_weight_max": 23.80617332458496,
      "activations/layer20_attention_weight_min": -23.119609832763672,
      "activations/layer21_attention_weight_max": 35.53925323486328,
      "activations/layer21_attention_weight_min": -21.46503257751465,
      "activations/layer22_attention_weight_max": 29.476844787597656,
      "activations/layer22_attention_weight_min": -26.017303466796875,
      "activations/layer23_attention_weight_max": 36.294288635253906,
      "activations/layer23_attention_weight_min": -26.43551254272461,
      "activations/layer2_attention_weight_max": 32.50065231323242,
      "activations/layer2_attention_weight_min": -31.042892456054688,
      "activations/layer3_attention_weight_max": 87.02918243408203,
      "activations/layer3_attention_weight_min": -83.60675811767578,
      "activations/layer4_attention_weight_max": 85.07245635986328,
      "activations/layer4_attention_weight_min": -80.46627044677734,
      "activations/layer5_attention_weight_max": 63.43653869628906,
      "activations/layer5_attention_weight_min": -77.95779418945312,
      "activations/layer6_attention_weight_max": 46.74772644042969,
      "activations/layer6_attention_weight_min": -45.60993576049805,
      "activations/layer7_attention_weight_max": 62.47367477416992,
      "activations/layer7_attention_weight_min": -57.69658279418945,
      "activations/layer8_attention_weight_max": 41.321678161621094,
      "activations/layer8_attention_weight_min": -45.878902435302734,
      "activations/layer9_attention_weight_max": 40.806358337402344,
      "activations/layer9_attention_weight_min": -40.172752380371094,
      "epoch": 16.9,
      "learning_rate": 4.145340909090909e-05,
      "loss": 2.7441,
      "step": 290800
    },
    {
      "activations/layer0_attention_weight_max": 16.010332107543945,
      "activations/layer0_attention_weight_min": -13.794062614440918,
      "activations/layer10_attention_weight_max": 36.41674041748047,
      "activations/layer10_attention_weight_min": -35.588836669921875,
      "activations/layer11_attention_weight_max": 33.60882568359375,
      "activations/layer11_attention_weight_min": -35.0045166015625,
      "activations/layer12_attention_weight_max": 19.181001663208008,
      "activations/layer12_attention_weight_min": -26.120927810668945,
      "activations/layer13_attention_weight_max": 34.6120719909668,
      "activations/layer13_attention_weight_min": -28.625051498413086,
      "activations/layer14_attention_weight_max": 37.34952926635742,
      "activations/layer14_attention_weight_min": -31.73777198791504,
      "activations/layer15_attention_weight_max": 33.5672607421875,
      "activations/layer15_attention_weight_min": -29.765335083007812,
      "activations/layer16_attention_weight_max": 34.755760192871094,
      "activations/layer16_attention_weight_min": -31.500452041625977,
      "activations/layer17_attention_weight_max": 50.24004364013672,
      "activations/layer17_attention_weight_min": -46.25336456298828,
      "activations/layer18_attention_weight_max": 48.61895751953125,
      "activations/layer18_attention_weight_min": -41.14316940307617,
      "activations/layer19_attention_weight_max": 25.1492919921875,
      "activations/layer19_attention_weight_min": -22.497900009155273,
      "activations/layer1_attention_weight_max": 16.32124137878418,
      "activations/layer1_attention_weight_min": -15.330225944519043,
      "activations/layer20_attention_weight_max": 23.217857360839844,
      "activations/layer20_attention_weight_min": -21.81587791442871,
      "activations/layer21_attention_weight_max": 39.99311447143555,
      "activations/layer21_attention_weight_min": -22.001018524169922,
      "activations/layer22_attention_weight_max": 28.2083740234375,
      "activations/layer22_attention_weight_min": -24.271570205688477,
      "activations/layer23_attention_weight_max": 38.30662536621094,
      "activations/layer23_attention_weight_min": -25.873493194580078,
      "activations/layer2_attention_weight_max": 32.208412170410156,
      "activations/layer2_attention_weight_min": -30.214893341064453,
      "activations/layer3_attention_weight_max": 88.64742279052734,
      "activations/layer3_attention_weight_min": -85.9197006225586,
      "activations/layer4_attention_weight_max": 89.37833404541016,
      "activations/layer4_attention_weight_min": -83.41686248779297,
      "activations/layer5_attention_weight_max": 66.51576232910156,
      "activations/layer5_attention_weight_min": -73.96713256835938,
      "activations/layer6_attention_weight_max": 54.31145095825195,
      "activations/layer6_attention_weight_min": -50.70075988769531,
      "activations/layer7_attention_weight_max": 67.14567565917969,
      "activations/layer7_attention_weight_min": -65.56462097167969,
      "activations/layer8_attention_weight_max": 48.67417526245117,
      "activations/layer8_attention_weight_min": -47.609474182128906,
      "activations/layer9_attention_weight_max": 46.69207763671875,
      "activations/layer9_attention_weight_min": -47.01158142089844,
      "epoch": 16.9,
      "learning_rate": 4.143446969696969e-05,
      "loss": 2.7562,
      "step": 290850
    },
    {
      "activations/layer0_attention_weight_max": 16.218469619750977,
      "activations/layer0_attention_weight_min": -13.803750991821289,
      "activations/layer10_attention_weight_max": 39.15850830078125,
      "activations/layer10_attention_weight_min": -36.11973571777344,
      "activations/layer11_attention_weight_max": 33.38170623779297,
      "activations/layer11_attention_weight_min": -33.08348846435547,
      "activations/layer12_attention_weight_max": 18.906198501586914,
      "activations/layer12_attention_weight_min": -24.546024322509766,
      "activations/layer13_attention_weight_max": 35.76430130004883,
      "activations/layer13_attention_weight_min": -29.55295753479004,
      "activations/layer14_attention_weight_max": 35.586395263671875,
      "activations/layer14_attention_weight_min": -30.72160530090332,
      "activations/layer15_attention_weight_max": 33.79349899291992,
      "activations/layer15_attention_weight_min": -29.486661911010742,
      "activations/layer16_attention_weight_max": 32.84000015258789,
      "activations/layer16_attention_weight_min": -30.65159797668457,
      "activations/layer17_attention_weight_max": 48.843414306640625,
      "activations/layer17_attention_weight_min": -44.3853645324707,
      "activations/layer18_attention_weight_max": 47.22805404663086,
      "activations/layer18_attention_weight_min": -39.290740966796875,
      "activations/layer19_attention_weight_max": 23.43337631225586,
      "activations/layer19_attention_weight_min": -21.603416442871094,
      "activations/layer1_attention_weight_max": 16.627681732177734,
      "activations/layer1_attention_weight_min": -14.759462356567383,
      "activations/layer20_attention_weight_max": 25.261253356933594,
      "activations/layer20_attention_weight_min": -20.100915908813477,
      "activations/layer21_attention_weight_max": 36.98249053955078,
      "activations/layer21_attention_weight_min": -25.16257095336914,
      "activations/layer22_attention_weight_max": 29.281370162963867,
      "activations/layer22_attention_weight_min": -25.627872467041016,
      "activations/layer23_attention_weight_max": 38.096229553222656,
      "activations/layer23_attention_weight_min": -24.699827194213867,
      "activations/layer2_attention_weight_max": 32.67292404174805,
      "activations/layer2_attention_weight_min": -30.86638641357422,
      "activations/layer3_attention_weight_max": 88.50691986083984,
      "activations/layer3_attention_weight_min": -93.36908721923828,
      "activations/layer4_attention_weight_max": 89.06282806396484,
      "activations/layer4_attention_weight_min": -87.86773681640625,
      "activations/layer5_attention_weight_max": 64.18602752685547,
      "activations/layer5_attention_weight_min": -72.85222625732422,
      "activations/layer6_attention_weight_max": 50.670310974121094,
      "activations/layer6_attention_weight_min": -53.37564468383789,
      "activations/layer7_attention_weight_max": 66.77095794677734,
      "activations/layer7_attention_weight_min": -65.1699447631836,
      "activations/layer8_attention_weight_max": 46.726036071777344,
      "activations/layer8_attention_weight_min": -51.7656364440918,
      "activations/layer9_attention_weight_max": 48.0057373046875,
      "activations/layer9_attention_weight_min": -46.617740631103516,
      "epoch": 16.9,
      "learning_rate": 4.1415530303030305e-05,
      "loss": 2.745,
      "step": 290900
    },
    {
      "activations/layer0_attention_weight_max": 15.4757080078125,
      "activations/layer0_attention_weight_min": -13.331758499145508,
      "activations/layer10_attention_weight_max": 38.613712310791016,
      "activations/layer10_attention_weight_min": -38.87409210205078,
      "activations/layer11_attention_weight_max": 38.55162811279297,
      "activations/layer11_attention_weight_min": -34.75645446777344,
      "activations/layer12_attention_weight_max": 19.003475189208984,
      "activations/layer12_attention_weight_min": -28.51156234741211,
      "activations/layer13_attention_weight_max": 38.16041564941406,
      "activations/layer13_attention_weight_min": -27.46061134338379,
      "activations/layer14_attention_weight_max": 34.31163787841797,
      "activations/layer14_attention_weight_min": -28.586910247802734,
      "activations/layer15_attention_weight_max": 31.768753051757812,
      "activations/layer15_attention_weight_min": -31.007810592651367,
      "activations/layer16_attention_weight_max": 34.39146423339844,
      "activations/layer16_attention_weight_min": -31.57499122619629,
      "activations/layer17_attention_weight_max": 53.51285934448242,
      "activations/layer17_attention_weight_min": -43.635009765625,
      "activations/layer18_attention_weight_max": 46.995361328125,
      "activations/layer18_attention_weight_min": -38.58759689331055,
      "activations/layer19_attention_weight_max": 22.97398567199707,
      "activations/layer19_attention_weight_min": -21.6789493560791,
      "activations/layer1_attention_weight_max": 16.177310943603516,
      "activations/layer1_attention_weight_min": -15.992246627807617,
      "activations/layer20_attention_weight_max": 23.783178329467773,
      "activations/layer20_attention_weight_min": -22.302791595458984,
      "activations/layer21_attention_weight_max": 36.42443084716797,
      "activations/layer21_attention_weight_min": -23.1765193939209,
      "activations/layer22_attention_weight_max": 28.07967758178711,
      "activations/layer22_attention_weight_min": -25.84314727783203,
      "activations/layer23_attention_weight_max": 37.6666374206543,
      "activations/layer23_attention_weight_min": -25.421875,
      "activations/layer2_attention_weight_max": 32.37864685058594,
      "activations/layer2_attention_weight_min": -33.25446319580078,
      "activations/layer3_attention_weight_max": 92.51727294921875,
      "activations/layer3_attention_weight_min": -93.56185913085938,
      "activations/layer4_attention_weight_max": 92.34661102294922,
      "activations/layer4_attention_weight_min": -87.80974578857422,
      "activations/layer5_attention_weight_max": 65.50303649902344,
      "activations/layer5_attention_weight_min": -71.53599548339844,
      "activations/layer6_attention_weight_max": 49.764686584472656,
      "activations/layer6_attention_weight_min": -50.77378463745117,
      "activations/layer7_attention_weight_max": 72.13555145263672,
      "activations/layer7_attention_weight_min": -66.84493255615234,
      "activations/layer8_attention_weight_max": 55.1429443359375,
      "activations/layer8_attention_weight_min": -50.13349533081055,
      "activations/layer9_attention_weight_max": 54.661739349365234,
      "activations/layer9_attention_weight_min": -45.74204635620117,
      "epoch": 16.91,
      "learning_rate": 4.1396590909090906e-05,
      "loss": 2.7485,
      "step": 290950
    },
    {
      "activations/layer0_attention_weight_max": 16.224878311157227,
      "activations/layer0_attention_weight_min": -13.866418838500977,
      "activations/layer10_attention_weight_max": 37.52455520629883,
      "activations/layer10_attention_weight_min": -37.47406768798828,
      "activations/layer11_attention_weight_max": 36.856632232666016,
      "activations/layer11_attention_weight_min": -34.699302673339844,
      "activations/layer12_attention_weight_max": 21.23573875427246,
      "activations/layer12_attention_weight_min": -25.14980697631836,
      "activations/layer13_attention_weight_max": 43.8892707824707,
      "activations/layer13_attention_weight_min": -28.734182357788086,
      "activations/layer14_attention_weight_max": 51.185447692871094,
      "activations/layer14_attention_weight_min": -32.207862854003906,
      "activations/layer15_attention_weight_max": 48.26997756958008,
      "activations/layer15_attention_weight_min": -32.04150390625,
      "activations/layer16_attention_weight_max": 35.0021858215332,
      "activations/layer16_attention_weight_min": -32.03525924682617,
      "activations/layer17_attention_weight_max": 57.94579315185547,
      "activations/layer17_attention_weight_min": -48.28190994262695,
      "activations/layer18_attention_weight_max": 50.232872009277344,
      "activations/layer18_attention_weight_min": -45.27029037475586,
      "activations/layer19_attention_weight_max": 27.922088623046875,
      "activations/layer19_attention_weight_min": -25.799673080444336,
      "activations/layer1_attention_weight_max": 16.453428268432617,
      "activations/layer1_attention_weight_min": -16.1197566986084,
      "activations/layer20_attention_weight_max": 25.445161819458008,
      "activations/layer20_attention_weight_min": -24.511241912841797,
      "activations/layer21_attention_weight_max": 45.61924362182617,
      "activations/layer21_attention_weight_min": -27.576005935668945,
      "activations/layer22_attention_weight_max": 32.920799255371094,
      "activations/layer22_attention_weight_min": -30.155006408691406,
      "activations/layer23_attention_weight_max": 37.19535827636719,
      "activations/layer23_attention_weight_min": -26.604976654052734,
      "activations/layer2_attention_weight_max": 31.685895919799805,
      "activations/layer2_attention_weight_min": -31.917160034179688,
      "activations/layer3_attention_weight_max": 86.27350616455078,
      "activations/layer3_attention_weight_min": -86.09434509277344,
      "activations/layer4_attention_weight_max": 85.55271911621094,
      "activations/layer4_attention_weight_min": -85.2538070678711,
      "activations/layer5_attention_weight_max": 65.64060974121094,
      "activations/layer5_attention_weight_min": -75.46240997314453,
      "activations/layer6_attention_weight_max": 47.64931869506836,
      "activations/layer6_attention_weight_min": -50.151973724365234,
      "activations/layer7_attention_weight_max": 68.57804870605469,
      "activations/layer7_attention_weight_min": -64.35494995117188,
      "activations/layer8_attention_weight_max": 46.8299560546875,
      "activations/layer8_attention_weight_min": -49.18391418457031,
      "activations/layer9_attention_weight_max": 51.79474639892578,
      "activations/layer9_attention_weight_min": -46.77670669555664,
      "epoch": 16.91,
      "learning_rate": 4.137765151515151e-05,
      "loss": 2.7503,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4548,
      "eval_samples_per_second": 507.875,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4548,
      "eval_openwebtext_samples_per_second": 507.875,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9514,
      "eval_wikitext_samples_per_second": 233.677,
      "step": 291000
    },
    {
      "epoch": 16.91,
      "eval_lambada_loss": 2.53515625,
      "eval_lambada_ppl": 12.618402319374093,
      "eval_lambada_runtime": 9.5266,
      "eval_lambada_samples_per_second": 511.096,
      "step": 291000
    },
    {
      "activations/layer0_attention_weight_max": 15.283295631408691,
      "activations/layer0_attention_weight_min": -13.581908226013184,
      "activations/layer10_attention_weight_max": 36.068931579589844,
      "activations/layer10_attention_weight_min": -39.26190185546875,
      "activations/layer11_attention_weight_max": 35.44584274291992,
      "activations/layer11_attention_weight_min": -36.85613250732422,
      "activations/layer12_attention_weight_max": 17.629806518554688,
      "activations/layer12_attention_weight_min": -27.991695404052734,
      "activations/layer13_attention_weight_max": 40.68288803100586,
      "activations/layer13_attention_weight_min": -28.05596160888672,
      "activations/layer14_attention_weight_max": 37.210662841796875,
      "activations/layer14_attention_weight_min": -30.712085723876953,
      "activations/layer15_attention_weight_max": 38.045162200927734,
      "activations/layer15_attention_weight_min": -31.307973861694336,
      "activations/layer16_attention_weight_max": 35.88749694824219,
      "activations/layer16_attention_weight_min": -31.444799423217773,
      "activations/layer17_attention_weight_max": 55.58708572387695,
      "activations/layer17_attention_weight_min": -47.34400939941406,
      "activations/layer18_attention_weight_max": 44.833805084228516,
      "activations/layer18_attention_weight_min": -40.145301818847656,
      "activations/layer19_attention_weight_max": 26.083242416381836,
      "activations/layer19_attention_weight_min": -22.552845001220703,
      "activations/layer1_attention_weight_max": 16.362865447998047,
      "activations/layer1_attention_weight_min": -15.358797073364258,
      "activations/layer20_attention_weight_max": 25.418766021728516,
      "activations/layer20_attention_weight_min": -21.179521560668945,
      "activations/layer21_attention_weight_max": 43.221927642822266,
      "activations/layer21_attention_weight_min": -22.121566772460938,
      "activations/layer22_attention_weight_max": 31.084003448486328,
      "activations/layer22_attention_weight_min": -23.73932456970215,
      "activations/layer23_attention_weight_max": 33.2814826965332,
      "activations/layer23_attention_weight_min": -24.8660888671875,
      "activations/layer2_attention_weight_max": 31.630878448486328,
      "activations/layer2_attention_weight_min": -31.04974365234375,
      "activations/layer3_attention_weight_max": 90.97637176513672,
      "activations/layer3_attention_weight_min": -92.17601776123047,
      "activations/layer4_attention_weight_max": 85.29127502441406,
      "activations/layer4_attention_weight_min": -84.79924774169922,
      "activations/layer5_attention_weight_max": 65.63005065917969,
      "activations/layer5_attention_weight_min": -70.45558166503906,
      "activations/layer6_attention_weight_max": 50.3967399597168,
      "activations/layer6_attention_weight_min": -50.06460189819336,
      "activations/layer7_attention_weight_max": 64.63986206054688,
      "activations/layer7_attention_weight_min": -62.813655853271484,
      "activations/layer8_attention_weight_max": 45.56447219848633,
      "activations/layer8_attention_weight_min": -50.301082611083984,
      "activations/layer9_attention_weight_max": 45.05066680908203,
      "activations/layer9_attention_weight_min": -47.431575775146484,
      "epoch": 16.91,
      "learning_rate": 4.1358712121212116e-05,
      "loss": 2.7493,
      "step": 291050
    },
    {
      "activations/layer0_attention_weight_max": 14.509520530700684,
      "activations/layer0_attention_weight_min": -13.57473087310791,
      "activations/layer10_attention_weight_max": 33.68439865112305,
      "activations/layer10_attention_weight_min": -34.47345733642578,
      "activations/layer11_attention_weight_max": 30.346187591552734,
      "activations/layer11_attention_weight_min": -31.40608024597168,
      "activations/layer12_attention_weight_max": 20.304397583007812,
      "activations/layer12_attention_weight_min": -25.24118423461914,
      "activations/layer13_attention_weight_max": 36.38336944580078,
      "activations/layer13_attention_weight_min": -26.999536514282227,
      "activations/layer14_attention_weight_max": 36.59415054321289,
      "activations/layer14_attention_weight_min": -32.0637321472168,
      "activations/layer15_attention_weight_max": 32.46575164794922,
      "activations/layer15_attention_weight_min": -31.71920394897461,
      "activations/layer16_attention_weight_max": 34.03144073486328,
      "activations/layer16_attention_weight_min": -33.06336975097656,
      "activations/layer17_attention_weight_max": 52.91408920288086,
      "activations/layer17_attention_weight_min": -47.273834228515625,
      "activations/layer18_attention_weight_max": 46.10407638549805,
      "activations/layer18_attention_weight_min": -42.79361343383789,
      "activations/layer19_attention_weight_max": 21.969257354736328,
      "activations/layer19_attention_weight_min": -23.96300506591797,
      "activations/layer1_attention_weight_max": 15.826424598693848,
      "activations/layer1_attention_weight_min": -16.90502166748047,
      "activations/layer20_attention_weight_max": 21.452884674072266,
      "activations/layer20_attention_weight_min": -22.170700073242188,
      "activations/layer21_attention_weight_max": 35.722198486328125,
      "activations/layer21_attention_weight_min": -21.690185546875,
      "activations/layer22_attention_weight_max": 29.371356964111328,
      "activations/layer22_attention_weight_min": -26.039899826049805,
      "activations/layer23_attention_weight_max": 34.58156967163086,
      "activations/layer23_attention_weight_min": -24.27448844909668,
      "activations/layer2_attention_weight_max": 30.085193634033203,
      "activations/layer2_attention_weight_min": -29.172719955444336,
      "activations/layer3_attention_weight_max": 83.79597473144531,
      "activations/layer3_attention_weight_min": -85.7877197265625,
      "activations/layer4_attention_weight_max": 82.60181427001953,
      "activations/layer4_attention_weight_min": -82.77436828613281,
      "activations/layer5_attention_weight_max": 63.55303955078125,
      "activations/layer5_attention_weight_min": -70.24054718017578,
      "activations/layer6_attention_weight_max": 48.15321731567383,
      "activations/layer6_attention_weight_min": -48.51362228393555,
      "activations/layer7_attention_weight_max": 61.45844650268555,
      "activations/layer7_attention_weight_min": -61.690513610839844,
      "activations/layer8_attention_weight_max": 46.21791458129883,
      "activations/layer8_attention_weight_min": -48.00547790527344,
      "activations/layer9_attention_weight_max": 48.37873077392578,
      "activations/layer9_attention_weight_min": -43.95976257324219,
      "epoch": 16.91,
      "learning_rate": 4.1339772727272724e-05,
      "loss": 2.7408,
      "step": 291100
    },
    {
      "activations/layer0_attention_weight_max": 15.365819931030273,
      "activations/layer0_attention_weight_min": -13.596399307250977,
      "activations/layer10_attention_weight_max": 36.695556640625,
      "activations/layer10_attention_weight_min": -38.56559753417969,
      "activations/layer11_attention_weight_max": 30.91638946533203,
      "activations/layer11_attention_weight_min": -33.81675338745117,
      "activations/layer12_attention_weight_max": 18.144763946533203,
      "activations/layer12_attention_weight_min": -25.08991050720215,
      "activations/layer13_attention_weight_max": 46.15574645996094,
      "activations/layer13_attention_weight_min": -32.33871078491211,
      "activations/layer14_attention_weight_max": 37.06867218017578,
      "activations/layer14_attention_weight_min": -29.697002410888672,
      "activations/layer15_attention_weight_max": 33.24960708618164,
      "activations/layer15_attention_weight_min": -30.112131118774414,
      "activations/layer16_attention_weight_max": 34.695045471191406,
      "activations/layer16_attention_weight_min": -31.416173934936523,
      "activations/layer17_attention_weight_max": 53.5626335144043,
      "activations/layer17_attention_weight_min": -44.01443862915039,
      "activations/layer18_attention_weight_max": 45.4803581237793,
      "activations/layer18_attention_weight_min": -40.75682830810547,
      "activations/layer19_attention_weight_max": 23.867353439331055,
      "activations/layer19_attention_weight_min": -25.506559371948242,
      "activations/layer1_attention_weight_max": 16.82798957824707,
      "activations/layer1_attention_weight_min": -15.903837203979492,
      "activations/layer20_attention_weight_max": 22.775569915771484,
      "activations/layer20_attention_weight_min": -22.97684669494629,
      "activations/layer21_attention_weight_max": 36.92947006225586,
      "activations/layer21_attention_weight_min": -27.997344970703125,
      "activations/layer22_attention_weight_max": 29.25444221496582,
      "activations/layer22_attention_weight_min": -25.331371307373047,
      "activations/layer23_attention_weight_max": 37.6451416015625,
      "activations/layer23_attention_weight_min": -27.984006881713867,
      "activations/layer2_attention_weight_max": 32.664119720458984,
      "activations/layer2_attention_weight_min": -31.544721603393555,
      "activations/layer3_attention_weight_max": 89.55509948730469,
      "activations/layer3_attention_weight_min": -92.06066131591797,
      "activations/layer4_attention_weight_max": 90.11904907226562,
      "activations/layer4_attention_weight_min": -89.59327697753906,
      "activations/layer5_attention_weight_max": 63.876441955566406,
      "activations/layer5_attention_weight_min": -72.6054458618164,
      "activations/layer6_attention_weight_max": 50.86418914794922,
      "activations/layer6_attention_weight_min": -52.879051208496094,
      "activations/layer7_attention_weight_max": 64.92486572265625,
      "activations/layer7_attention_weight_min": -62.58684539794922,
      "activations/layer8_attention_weight_max": 45.82699966430664,
      "activations/layer8_attention_weight_min": -49.932395935058594,
      "activations/layer9_attention_weight_max": 44.02470016479492,
      "activations/layer9_attention_weight_min": -46.71831512451172,
      "epoch": 16.92,
      "learning_rate": 4.132083333333333e-05,
      "loss": 2.7435,
      "step": 291150
    },
    {
      "activations/layer0_attention_weight_max": 15.639470100402832,
      "activations/layer0_attention_weight_min": -13.514558792114258,
      "activations/layer10_attention_weight_max": 36.45334243774414,
      "activations/layer10_attention_weight_min": -34.05579376220703,
      "activations/layer11_attention_weight_max": 32.94096374511719,
      "activations/layer11_attention_weight_min": -32.209991455078125,
      "activations/layer12_attention_weight_max": 21.25699806213379,
      "activations/layer12_attention_weight_min": -25.98418426513672,
      "activations/layer13_attention_weight_max": 38.00017166137695,
      "activations/layer13_attention_weight_min": -27.65226936340332,
      "activations/layer14_attention_weight_max": 37.38379669189453,
      "activations/layer14_attention_weight_min": -29.653146743774414,
      "activations/layer15_attention_weight_max": 33.64071273803711,
      "activations/layer15_attention_weight_min": -30.853513717651367,
      "activations/layer16_attention_weight_max": 33.48621368408203,
      "activations/layer16_attention_weight_min": -33.0869140625,
      "activations/layer17_attention_weight_max": 51.0698127746582,
      "activations/layer17_attention_weight_min": -50.023094177246094,
      "activations/layer18_attention_weight_max": 46.87322235107422,
      "activations/layer18_attention_weight_min": -41.05099868774414,
      "activations/layer19_attention_weight_max": 23.59955406188965,
      "activations/layer19_attention_weight_min": -22.145124435424805,
      "activations/layer1_attention_weight_max": 16.13774871826172,
      "activations/layer1_attention_weight_min": -15.637957572937012,
      "activations/layer20_attention_weight_max": 21.154569625854492,
      "activations/layer20_attention_weight_min": -21.01624298095703,
      "activations/layer21_attention_weight_max": 37.99953842163086,
      "activations/layer21_attention_weight_min": -22.190887451171875,
      "activations/layer22_attention_weight_max": 31.364282608032227,
      "activations/layer22_attention_weight_min": -25.944721221923828,
      "activations/layer23_attention_weight_max": 38.85710525512695,
      "activations/layer23_attention_weight_min": -23.15652847290039,
      "activations/layer2_attention_weight_max": 30.783390045166016,
      "activations/layer2_attention_weight_min": -31.315092086791992,
      "activations/layer3_attention_weight_max": 83.6553726196289,
      "activations/layer3_attention_weight_min": -85.51372528076172,
      "activations/layer4_attention_weight_max": 85.96721649169922,
      "activations/layer4_attention_weight_min": -82.91985321044922,
      "activations/layer5_attention_weight_max": 64.90019226074219,
      "activations/layer5_attention_weight_min": -71.01841735839844,
      "activations/layer6_attention_weight_max": 48.34865951538086,
      "activations/layer6_attention_weight_min": -52.21530532836914,
      "activations/layer7_attention_weight_max": 60.6172981262207,
      "activations/layer7_attention_weight_min": -65.50435638427734,
      "activations/layer8_attention_weight_max": 46.58290100097656,
      "activations/layer8_attention_weight_min": -49.98037338256836,
      "activations/layer9_attention_weight_max": 42.51731491088867,
      "activations/layer9_attention_weight_min": -45.958065032958984,
      "epoch": 16.92,
      "learning_rate": 4.1301893939393934e-05,
      "loss": 2.7399,
      "step": 291200
    },
    {
      "activations/layer0_attention_weight_max": 15.9073486328125,
      "activations/layer0_attention_weight_min": -13.50334358215332,
      "activations/layer10_attention_weight_max": 36.14973449707031,
      "activations/layer10_attention_weight_min": -37.141807556152344,
      "activations/layer11_attention_weight_max": 34.958717346191406,
      "activations/layer11_attention_weight_min": -32.99496841430664,
      "activations/layer12_attention_weight_max": 18.65397834777832,
      "activations/layer12_attention_weight_min": -24.52015495300293,
      "activations/layer13_attention_weight_max": 34.63447952270508,
      "activations/layer13_attention_weight_min": -27.94322395324707,
      "activations/layer14_attention_weight_max": 32.73310470581055,
      "activations/layer14_attention_weight_min": -31.268146514892578,
      "activations/layer15_attention_weight_max": 29.341575622558594,
      "activations/layer15_attention_weight_min": -30.26750946044922,
      "activations/layer16_attention_weight_max": 31.40641975402832,
      "activations/layer16_attention_weight_min": -32.53799819946289,
      "activations/layer17_attention_weight_max": 47.29985809326172,
      "activations/layer17_attention_weight_min": -42.67195129394531,
      "activations/layer18_attention_weight_max": 43.22323989868164,
      "activations/layer18_attention_weight_min": -39.94728469848633,
      "activations/layer19_attention_weight_max": 23.19641876220703,
      "activations/layer19_attention_weight_min": -24.220550537109375,
      "activations/layer1_attention_weight_max": 16.8243408203125,
      "activations/layer1_attention_weight_min": -14.191043853759766,
      "activations/layer20_attention_weight_max": 23.23685073852539,
      "activations/layer20_attention_weight_min": -26.345529556274414,
      "activations/layer21_attention_weight_max": 38.567237854003906,
      "activations/layer21_attention_weight_min": -21.22254180908203,
      "activations/layer22_attention_weight_max": 30.126916885375977,
      "activations/layer22_attention_weight_min": -31.587127685546875,
      "activations/layer23_attention_weight_max": 37.40022277832031,
      "activations/layer23_attention_weight_min": -26.08108901977539,
      "activations/layer2_attention_weight_max": 31.43208885192871,
      "activations/layer2_attention_weight_min": -30.602392196655273,
      "activations/layer3_attention_weight_max": 89.33363342285156,
      "activations/layer3_attention_weight_min": -88.61209106445312,
      "activations/layer4_attention_weight_max": 88.23567962646484,
      "activations/layer4_attention_weight_min": -88.3722152709961,
      "activations/layer5_attention_weight_max": 62.562679290771484,
      "activations/layer5_attention_weight_min": -71.01199340820312,
      "activations/layer6_attention_weight_max": 51.38745880126953,
      "activations/layer6_attention_weight_min": -50.42799377441406,
      "activations/layer7_attention_weight_max": 64.80921173095703,
      "activations/layer7_attention_weight_min": -63.42854690551758,
      "activations/layer8_attention_weight_max": 45.4312744140625,
      "activations/layer8_attention_weight_min": -49.72142791748047,
      "activations/layer9_attention_weight_max": 47.101497650146484,
      "activations/layer9_attention_weight_min": -48.277530670166016,
      "epoch": 16.92,
      "learning_rate": 4.1282954545454536e-05,
      "loss": 2.7629,
      "step": 291250
    },
    {
      "activations/layer0_attention_weight_max": 16.51626968383789,
      "activations/layer0_attention_weight_min": -13.517955780029297,
      "activations/layer10_attention_weight_max": 33.227447509765625,
      "activations/layer10_attention_weight_min": -34.57614517211914,
      "activations/layer11_attention_weight_max": 32.70363998413086,
      "activations/layer11_attention_weight_min": -35.39826202392578,
      "activations/layer12_attention_weight_max": 17.587886810302734,
      "activations/layer12_attention_weight_min": -22.72553825378418,
      "activations/layer13_attention_weight_max": 45.31542205810547,
      "activations/layer13_attention_weight_min": -34.1689338684082,
      "activations/layer14_attention_weight_max": 43.79602813720703,
      "activations/layer14_attention_weight_min": -30.474153518676758,
      "activations/layer15_attention_weight_max": 40.55195999145508,
      "activations/layer15_attention_weight_min": -30.7572021484375,
      "activations/layer16_attention_weight_max": 34.09925842285156,
      "activations/layer16_attention_weight_min": -32.2587890625,
      "activations/layer17_attention_weight_max": 52.351356506347656,
      "activations/layer17_attention_weight_min": -48.71276092529297,
      "activations/layer18_attention_weight_max": 49.35956954956055,
      "activations/layer18_attention_weight_min": -45.382591247558594,
      "activations/layer19_attention_weight_max": 25.456985473632812,
      "activations/layer19_attention_weight_min": -22.954484939575195,
      "activations/layer1_attention_weight_max": 16.590015411376953,
      "activations/layer1_attention_weight_min": -15.492003440856934,
      "activations/layer20_attention_weight_max": 26.552173614501953,
      "activations/layer20_attention_weight_min": -23.53961753845215,
      "activations/layer21_attention_weight_max": 43.37470626831055,
      "activations/layer21_attention_weight_min": -29.57906723022461,
      "activations/layer22_attention_weight_max": 34.585365295410156,
      "activations/layer22_attention_weight_min": -27.40874671936035,
      "activations/layer23_attention_weight_max": 39.98851013183594,
      "activations/layer23_attention_weight_min": -24.889698028564453,
      "activations/layer2_attention_weight_max": 31.683271408081055,
      "activations/layer2_attention_weight_min": -31.621152877807617,
      "activations/layer3_attention_weight_max": 87.62615203857422,
      "activations/layer3_attention_weight_min": -96.68700408935547,
      "activations/layer4_attention_weight_max": 87.9109878540039,
      "activations/layer4_attention_weight_min": -88.16035461425781,
      "activations/layer5_attention_weight_max": 64.79850006103516,
      "activations/layer5_attention_weight_min": -73.73454284667969,
      "activations/layer6_attention_weight_max": 49.94782257080078,
      "activations/layer6_attention_weight_min": -48.940242767333984,
      "activations/layer7_attention_weight_max": 67.79010009765625,
      "activations/layer7_attention_weight_min": -63.90290832519531,
      "activations/layer8_attention_weight_max": 45.31007766723633,
      "activations/layer8_attention_weight_min": -48.746273040771484,
      "activations/layer9_attention_weight_max": 43.356895446777344,
      "activations/layer9_attention_weight_min": -44.950984954833984,
      "epoch": 16.93,
      "learning_rate": 4.126401515151515e-05,
      "loss": 2.7469,
      "step": 291300
    },
    {
      "activations/layer0_attention_weight_max": 15.681177139282227,
      "activations/layer0_attention_weight_min": -13.355198860168457,
      "activations/layer10_attention_weight_max": 37.16884994506836,
      "activations/layer10_attention_weight_min": -35.89777755737305,
      "activations/layer11_attention_weight_max": 35.369407653808594,
      "activations/layer11_attention_weight_min": -33.327125549316406,
      "activations/layer12_attention_weight_max": 18.888010025024414,
      "activations/layer12_attention_weight_min": -26.575586318969727,
      "activations/layer13_attention_weight_max": 34.05143356323242,
      "activations/layer13_attention_weight_min": -28.576343536376953,
      "activations/layer14_attention_weight_max": 38.700706481933594,
      "activations/layer14_attention_weight_min": -31.738771438598633,
      "activations/layer15_attention_weight_max": 44.2512092590332,
      "activations/layer15_attention_weight_min": -29.791706085205078,
      "activations/layer16_attention_weight_max": 32.70945739746094,
      "activations/layer16_attention_weight_min": -31.721769332885742,
      "activations/layer17_attention_weight_max": 52.109596252441406,
      "activations/layer17_attention_weight_min": -43.850914001464844,
      "activations/layer18_attention_weight_max": 45.49755859375,
      "activations/layer18_attention_weight_min": -38.251068115234375,
      "activations/layer19_attention_weight_max": 22.313480377197266,
      "activations/layer19_attention_weight_min": -22.18476676940918,
      "activations/layer1_attention_weight_max": 16.230300903320312,
      "activations/layer1_attention_weight_min": -15.012726783752441,
      "activations/layer20_attention_weight_max": 20.878318786621094,
      "activations/layer20_attention_weight_min": -20.8985595703125,
      "activations/layer21_attention_weight_max": 34.1060905456543,
      "activations/layer21_attention_weight_min": -22.920499801635742,
      "activations/layer22_attention_weight_max": 28.209882736206055,
      "activations/layer22_attention_weight_min": -25.60512351989746,
      "activations/layer23_attention_weight_max": 34.497344970703125,
      "activations/layer23_attention_weight_min": -25.217994689941406,
      "activations/layer2_attention_weight_max": 29.502826690673828,
      "activations/layer2_attention_weight_min": -29.102458953857422,
      "activations/layer3_attention_weight_max": 85.47232055664062,
      "activations/layer3_attention_weight_min": -84.9661636352539,
      "activations/layer4_attention_weight_max": 86.63927459716797,
      "activations/layer4_attention_weight_min": -86.3396987915039,
      "activations/layer5_attention_weight_max": 65.22529602050781,
      "activations/layer5_attention_weight_min": -74.37782287597656,
      "activations/layer6_attention_weight_max": 48.91392517089844,
      "activations/layer6_attention_weight_min": -49.380531311035156,
      "activations/layer7_attention_weight_max": 67.25934600830078,
      "activations/layer7_attention_weight_min": -63.43956756591797,
      "activations/layer8_attention_weight_max": 46.663734436035156,
      "activations/layer8_attention_weight_min": -49.22164535522461,
      "activations/layer9_attention_weight_max": 48.64341735839844,
      "activations/layer9_attention_weight_min": -47.2991943359375,
      "epoch": 16.93,
      "learning_rate": 4.124507575757575e-05,
      "loss": 2.7443,
      "step": 291350
    },
    {
      "activations/layer0_attention_weight_max": 15.246857643127441,
      "activations/layer0_attention_weight_min": -13.1408052444458,
      "activations/layer10_attention_weight_max": 38.3335075378418,
      "activations/layer10_attention_weight_min": -38.09225845336914,
      "activations/layer11_attention_weight_max": 35.22462844848633,
      "activations/layer11_attention_weight_min": -35.137020111083984,
      "activations/layer12_attention_weight_max": 19.569875717163086,
      "activations/layer12_attention_weight_min": -25.900897979736328,
      "activations/layer13_attention_weight_max": 34.18189239501953,
      "activations/layer13_attention_weight_min": -28.78079605102539,
      "activations/layer14_attention_weight_max": 35.56538772583008,
      "activations/layer14_attention_weight_min": -33.82205581665039,
      "activations/layer15_attention_weight_max": 33.59036636352539,
      "activations/layer15_attention_weight_min": -33.33693313598633,
      "activations/layer16_attention_weight_max": 35.127418518066406,
      "activations/layer16_attention_weight_min": -34.2055549621582,
      "activations/layer17_attention_weight_max": 52.83987045288086,
      "activations/layer17_attention_weight_min": -49.18244934082031,
      "activations/layer18_attention_weight_max": 49.47486114501953,
      "activations/layer18_attention_weight_min": -45.27116775512695,
      "activations/layer19_attention_weight_max": 24.23768424987793,
      "activations/layer19_attention_weight_min": -25.05364990234375,
      "activations/layer1_attention_weight_max": 15.950512886047363,
      "activations/layer1_attention_weight_min": -14.883310317993164,
      "activations/layer20_attention_weight_max": 22.14387321472168,
      "activations/layer20_attention_weight_min": -21.983732223510742,
      "activations/layer21_attention_weight_max": 35.1253776550293,
      "activations/layer21_attention_weight_min": -26.27861785888672,
      "activations/layer22_attention_weight_max": 29.46851348876953,
      "activations/layer22_attention_weight_min": -26.59987449645996,
      "activations/layer23_attention_weight_max": 35.24018859863281,
      "activations/layer23_attention_weight_min": -27.329565048217773,
      "activations/layer2_attention_weight_max": 33.6973876953125,
      "activations/layer2_attention_weight_min": -30.87716293334961,
      "activations/layer3_attention_weight_max": 94.70829772949219,
      "activations/layer3_attention_weight_min": -91.59652709960938,
      "activations/layer4_attention_weight_max": 91.87358093261719,
      "activations/layer4_attention_weight_min": -86.40715789794922,
      "activations/layer5_attention_weight_max": 65.09413146972656,
      "activations/layer5_attention_weight_min": -73.40916442871094,
      "activations/layer6_attention_weight_max": 54.67408752441406,
      "activations/layer6_attention_weight_min": -54.104225158691406,
      "activations/layer7_attention_weight_max": 66.31854248046875,
      "activations/layer7_attention_weight_min": -68.17276000976562,
      "activations/layer8_attention_weight_max": 49.99319076538086,
      "activations/layer8_attention_weight_min": -53.64237976074219,
      "activations/layer9_attention_weight_max": 46.93722152709961,
      "activations/layer9_attention_weight_min": -46.7786750793457,
      "epoch": 16.93,
      "learning_rate": 4.122613636363636e-05,
      "loss": 2.747,
      "step": 291400
    },
    {
      "activations/layer0_attention_weight_max": 15.787032127380371,
      "activations/layer0_attention_weight_min": -13.419474601745605,
      "activations/layer10_attention_weight_max": 35.872154235839844,
      "activations/layer10_attention_weight_min": -37.242916107177734,
      "activations/layer11_attention_weight_max": 32.90553283691406,
      "activations/layer11_attention_weight_min": -35.03231430053711,
      "activations/layer12_attention_weight_max": 19.893552780151367,
      "activations/layer12_attention_weight_min": -28.312543869018555,
      "activations/layer13_attention_weight_max": 32.78109359741211,
      "activations/layer13_attention_weight_min": -26.23749351501465,
      "activations/layer14_attention_weight_max": 33.59663391113281,
      "activations/layer14_attention_weight_min": -29.442794799804688,
      "activations/layer15_attention_weight_max": 30.87420082092285,
      "activations/layer15_attention_weight_min": -29.19228172302246,
      "activations/layer16_attention_weight_max": 31.80076789855957,
      "activations/layer16_attention_weight_min": -30.088260650634766,
      "activations/layer17_attention_weight_max": 50.94389724731445,
      "activations/layer17_attention_weight_min": -42.007362365722656,
      "activations/layer18_attention_weight_max": 45.701416015625,
      "activations/layer18_attention_weight_min": -36.77499771118164,
      "activations/layer19_attention_weight_max": 22.911447525024414,
      "activations/layer19_attention_weight_min": -21.551982879638672,
      "activations/layer1_attention_weight_max": 15.895407676696777,
      "activations/layer1_attention_weight_min": -13.56594467163086,
      "activations/layer20_attention_weight_max": 23.151939392089844,
      "activations/layer20_attention_weight_min": -22.134925842285156,
      "activations/layer21_attention_weight_max": 33.12348175048828,
      "activations/layer21_attention_weight_min": -23.063596725463867,
      "activations/layer22_attention_weight_max": 28.747102737426758,
      "activations/layer22_attention_weight_min": -24.9007625579834,
      "activations/layer23_attention_weight_max": 37.236331939697266,
      "activations/layer23_attention_weight_min": -24.8421688079834,
      "activations/layer2_attention_weight_max": 31.485851287841797,
      "activations/layer2_attention_weight_min": -29.970252990722656,
      "activations/layer3_attention_weight_max": 85.42658996582031,
      "activations/layer3_attention_weight_min": -83.49807739257812,
      "activations/layer4_attention_weight_max": 85.96253204345703,
      "activations/layer4_attention_weight_min": -83.37767791748047,
      "activations/layer5_attention_weight_max": 65.26470947265625,
      "activations/layer5_attention_weight_min": -75.45197296142578,
      "activations/layer6_attention_weight_max": 48.795162200927734,
      "activations/layer6_attention_weight_min": -48.91586685180664,
      "activations/layer7_attention_weight_max": 66.48941802978516,
      "activations/layer7_attention_weight_min": -64.00250244140625,
      "activations/layer8_attention_weight_max": 44.14835739135742,
      "activations/layer8_attention_weight_min": -53.31050491333008,
      "activations/layer9_attention_weight_max": 44.329742431640625,
      "activations/layer9_attention_weight_min": -45.5474739074707,
      "epoch": 16.93,
      "learning_rate": 4.120719696969696e-05,
      "loss": 2.7508,
      "step": 291450
    },
    {
      "activations/layer0_attention_weight_max": 15.312235832214355,
      "activations/layer0_attention_weight_min": -13.343609809875488,
      "activations/layer10_attention_weight_max": 36.066505432128906,
      "activations/layer10_attention_weight_min": -35.97533416748047,
      "activations/layer11_attention_weight_max": 34.1825065612793,
      "activations/layer11_attention_weight_min": -33.728214263916016,
      "activations/layer12_attention_weight_max": 18.478328704833984,
      "activations/layer12_attention_weight_min": -25.75183868408203,
      "activations/layer13_attention_weight_max": 31.10531234741211,
      "activations/layer13_attention_weight_min": -26.688640594482422,
      "activations/layer14_attention_weight_max": 35.48848342895508,
      "activations/layer14_attention_weight_min": -29.787776947021484,
      "activations/layer15_attention_weight_max": 32.752906799316406,
      "activations/layer15_attention_weight_min": -30.960947036743164,
      "activations/layer16_attention_weight_max": 32.38330841064453,
      "activations/layer16_attention_weight_min": -32.084442138671875,
      "activations/layer17_attention_weight_max": 53.75062561035156,
      "activations/layer17_attention_weight_min": -46.159542083740234,
      "activations/layer18_attention_weight_max": 44.51036834716797,
      "activations/layer18_attention_weight_min": -39.207611083984375,
      "activations/layer19_attention_weight_max": 24.661884307861328,
      "activations/layer19_attention_weight_min": -23.39766502380371,
      "activations/layer1_attention_weight_max": 16.665483474731445,
      "activations/layer1_attention_weight_min": -13.144952774047852,
      "activations/layer20_attention_weight_max": 23.253114700317383,
      "activations/layer20_attention_weight_min": -21.46056365966797,
      "activations/layer21_attention_weight_max": 35.2562370300293,
      "activations/layer21_attention_weight_min": -21.58599281311035,
      "activations/layer22_attention_weight_max": 27.82923126220703,
      "activations/layer22_attention_weight_min": -25.913890838623047,
      "activations/layer23_attention_weight_max": 37.41950988769531,
      "activations/layer23_attention_weight_min": -25.416709899902344,
      "activations/layer2_attention_weight_max": 31.783191680908203,
      "activations/layer2_attention_weight_min": -30.35598373413086,
      "activations/layer3_attention_weight_max": 88.77641296386719,
      "activations/layer3_attention_weight_min": -89.69275665283203,
      "activations/layer4_attention_weight_max": 85.73531341552734,
      "activations/layer4_attention_weight_min": -85.13660430908203,
      "activations/layer5_attention_weight_max": 60.57264709472656,
      "activations/layer5_attention_weight_min": -75.13243865966797,
      "activations/layer6_attention_weight_max": 46.23938751220703,
      "activations/layer6_attention_weight_min": -50.036617279052734,
      "activations/layer7_attention_weight_max": 68.93792724609375,
      "activations/layer7_attention_weight_min": -59.74746322631836,
      "activations/layer8_attention_weight_max": 44.142032623291016,
      "activations/layer8_attention_weight_min": -48.588661193847656,
      "activations/layer9_attention_weight_max": 42.67386245727539,
      "activations/layer9_attention_weight_min": -45.26052474975586,
      "epoch": 16.94,
      "learning_rate": 4.118825757575758e-05,
      "loss": 2.7577,
      "step": 291500
    },
    {
      "activations/layer0_attention_weight_max": 16.420211791992188,
      "activations/layer0_attention_weight_min": -13.478175163269043,
      "activations/layer10_attention_weight_max": 40.1132926940918,
      "activations/layer10_attention_weight_min": -38.236724853515625,
      "activations/layer11_attention_weight_max": 33.663536071777344,
      "activations/layer11_attention_weight_min": -34.31798553466797,
      "activations/layer12_attention_weight_max": 20.54694938659668,
      "activations/layer12_attention_weight_min": -29.102754592895508,
      "activations/layer13_attention_weight_max": 36.43071746826172,
      "activations/layer13_attention_weight_min": -30.511402130126953,
      "activations/layer14_attention_weight_max": 39.99747848510742,
      "activations/layer14_attention_weight_min": -30.88755989074707,
      "activations/layer15_attention_weight_max": 32.549461364746094,
      "activations/layer15_attention_weight_min": -30.355649948120117,
      "activations/layer16_attention_weight_max": 32.29350662231445,
      "activations/layer16_attention_weight_min": -30.88553810119629,
      "activations/layer17_attention_weight_max": 51.3045654296875,
      "activations/layer17_attention_weight_min": -46.16863250732422,
      "activations/layer18_attention_weight_max": 48.1230583190918,
      "activations/layer18_attention_weight_min": -39.56905746459961,
      "activations/layer19_attention_weight_max": 26.71335220336914,
      "activations/layer19_attention_weight_min": -20.948978424072266,
      "activations/layer1_attention_weight_max": 16.082508087158203,
      "activations/layer1_attention_weight_min": -14.835502624511719,
      "activations/layer20_attention_weight_max": 23.25047492980957,
      "activations/layer20_attention_weight_min": -20.162254333496094,
      "activations/layer21_attention_weight_max": 37.264625549316406,
      "activations/layer21_attention_weight_min": -27.546361923217773,
      "activations/layer22_attention_weight_max": 27.86009979248047,
      "activations/layer22_attention_weight_min": -25.07933235168457,
      "activations/layer23_attention_weight_max": 33.73286056518555,
      "activations/layer23_attention_weight_min": -23.104507446289062,
      "activations/layer2_attention_weight_max": 31.76655387878418,
      "activations/layer2_attention_weight_min": -30.961082458496094,
      "activations/layer3_attention_weight_max": 91.28389739990234,
      "activations/layer3_attention_weight_min": -91.8261947631836,
      "activations/layer4_attention_weight_max": 90.24805450439453,
      "activations/layer4_attention_weight_min": -91.50068664550781,
      "activations/layer5_attention_weight_max": 63.344337463378906,
      "activations/layer5_attention_weight_min": -72.16242218017578,
      "activations/layer6_attention_weight_max": 48.96507263183594,
      "activations/layer6_attention_weight_min": -51.47500228881836,
      "activations/layer7_attention_weight_max": 63.333518981933594,
      "activations/layer7_attention_weight_min": -65.08114624023438,
      "activations/layer8_attention_weight_max": 48.697166442871094,
      "activations/layer8_attention_weight_min": -51.27347183227539,
      "activations/layer9_attention_weight_max": 43.935096740722656,
      "activations/layer9_attention_weight_min": -48.89059066772461,
      "epoch": 16.94,
      "learning_rate": 4.116931818181818e-05,
      "loss": 2.7496,
      "step": 291550
    },
    {
      "activations/layer0_attention_weight_max": 14.777850151062012,
      "activations/layer0_attention_weight_min": -13.273163795471191,
      "activations/layer10_attention_weight_max": 36.169769287109375,
      "activations/layer10_attention_weight_min": -36.11608123779297,
      "activations/layer11_attention_weight_max": 32.668758392333984,
      "activations/layer11_attention_weight_min": -34.87946319580078,
      "activations/layer12_attention_weight_max": 19.34022331237793,
      "activations/layer12_attention_weight_min": -24.499778747558594,
      "activations/layer13_attention_weight_max": 35.943050384521484,
      "activations/layer13_attention_weight_min": -27.30167579650879,
      "activations/layer14_attention_weight_max": 36.48740768432617,
      "activations/layer14_attention_weight_min": -29.61237335205078,
      "activations/layer15_attention_weight_max": 33.17848587036133,
      "activations/layer15_attention_weight_min": -31.600387573242188,
      "activations/layer16_attention_weight_max": 32.60407638549805,
      "activations/layer16_attention_weight_min": -30.517194747924805,
      "activations/layer17_attention_weight_max": 50.8390998840332,
      "activations/layer17_attention_weight_min": -46.520774841308594,
      "activations/layer18_attention_weight_max": 50.06582260131836,
      "activations/layer18_attention_weight_min": -39.13324737548828,
      "activations/layer19_attention_weight_max": 23.002424240112305,
      "activations/layer19_attention_weight_min": -22.317941665649414,
      "activations/layer1_attention_weight_max": 15.614779472351074,
      "activations/layer1_attention_weight_min": -14.896708488464355,
      "activations/layer20_attention_weight_max": 21.65872573852539,
      "activations/layer20_attention_weight_min": -22.743392944335938,
      "activations/layer21_attention_weight_max": 37.05946731567383,
      "activations/layer21_attention_weight_min": -22.798261642456055,
      "activations/layer22_attention_weight_max": 31.38015365600586,
      "activations/layer22_attention_weight_min": -26.8383846282959,
      "activations/layer23_attention_weight_max": 42.124473571777344,
      "activations/layer23_attention_weight_min": -24.239826202392578,
      "activations/layer2_attention_weight_max": 32.503753662109375,
      "activations/layer2_attention_weight_min": -30.461963653564453,
      "activations/layer3_attention_weight_max": 90.34281921386719,
      "activations/layer3_attention_weight_min": -88.92869567871094,
      "activations/layer4_attention_weight_max": 89.19851684570312,
      "activations/layer4_attention_weight_min": -93.31587219238281,
      "activations/layer5_attention_weight_max": 63.79014205932617,
      "activations/layer5_attention_weight_min": -76.19452667236328,
      "activations/layer6_attention_weight_max": 50.94398880004883,
      "activations/layer6_attention_weight_min": -51.44475555419922,
      "activations/layer7_attention_weight_max": 65.5390396118164,
      "activations/layer7_attention_weight_min": -63.13505172729492,
      "activations/layer8_attention_weight_max": 44.82304763793945,
      "activations/layer8_attention_weight_min": -51.1729736328125,
      "activations/layer9_attention_weight_max": 43.35888671875,
      "activations/layer9_attention_weight_min": -46.30647659301758,
      "epoch": 16.94,
      "learning_rate": 4.115037878787878e-05,
      "loss": 2.7346,
      "step": 291600
    },
    {
      "activations/layer0_attention_weight_max": 16.336713790893555,
      "activations/layer0_attention_weight_min": -13.49638843536377,
      "activations/layer10_attention_weight_max": 35.9898681640625,
      "activations/layer10_attention_weight_min": -36.452877044677734,
      "activations/layer11_attention_weight_max": 33.32075881958008,
      "activations/layer11_attention_weight_min": -34.10668182373047,
      "activations/layer12_attention_weight_max": 19.116455078125,
      "activations/layer12_attention_weight_min": -26.640344619750977,
      "activations/layer13_attention_weight_max": 36.72705078125,
      "activations/layer13_attention_weight_min": -31.219026565551758,
      "activations/layer14_attention_weight_max": 36.66539764404297,
      "activations/layer14_attention_weight_min": -31.83858871459961,
      "activations/layer15_attention_weight_max": 32.572383880615234,
      "activations/layer15_attention_weight_min": -29.454118728637695,
      "activations/layer16_attention_weight_max": 34.92935562133789,
      "activations/layer16_attention_weight_min": -32.16685485839844,
      "activations/layer17_attention_weight_max": 55.4050178527832,
      "activations/layer17_attention_weight_min": -46.528560638427734,
      "activations/layer18_attention_weight_max": 46.210567474365234,
      "activations/layer18_attention_weight_min": -40.4373893737793,
      "activations/layer19_attention_weight_max": 23.040319442749023,
      "activations/layer19_attention_weight_min": -23.48849105834961,
      "activations/layer1_attention_weight_max": 15.858019828796387,
      "activations/layer1_attention_weight_min": -14.90245532989502,
      "activations/layer20_attention_weight_max": 22.16259765625,
      "activations/layer20_attention_weight_min": -22.305700302124023,
      "activations/layer21_attention_weight_max": 34.389122009277344,
      "activations/layer21_attention_weight_min": -24.029361724853516,
      "activations/layer22_attention_weight_max": 30.78659439086914,
      "activations/layer22_attention_weight_min": -26.1109676361084,
      "activations/layer23_attention_weight_max": 43.078529357910156,
      "activations/layer23_attention_weight_min": -23.982824325561523,
      "activations/layer2_attention_weight_max": 32.38315200805664,
      "activations/layer2_attention_weight_min": -30.070354461669922,
      "activations/layer3_attention_weight_max": 88.97846984863281,
      "activations/layer3_attention_weight_min": -96.01847076416016,
      "activations/layer4_attention_weight_max": 87.15223693847656,
      "activations/layer4_attention_weight_min": -89.20867156982422,
      "activations/layer5_attention_weight_max": 64.36438751220703,
      "activations/layer5_attention_weight_min": -73.67760467529297,
      "activations/layer6_attention_weight_max": 53.89625549316406,
      "activations/layer6_attention_weight_min": -54.074745178222656,
      "activations/layer7_attention_weight_max": 62.70625686645508,
      "activations/layer7_attention_weight_min": -69.49246215820312,
      "activations/layer8_attention_weight_max": 46.24482727050781,
      "activations/layer8_attention_weight_min": -51.168800354003906,
      "activations/layer9_attention_weight_max": 44.461265563964844,
      "activations/layer9_attention_weight_min": -47.91320037841797,
      "epoch": 16.95,
      "learning_rate": 4.1131439393939395e-05,
      "loss": 2.7417,
      "step": 291650
    },
    {
      "activations/layer0_attention_weight_max": 15.384322166442871,
      "activations/layer0_attention_weight_min": -13.483062744140625,
      "activations/layer10_attention_weight_max": 41.69211196899414,
      "activations/layer10_attention_weight_min": -38.10169219970703,
      "activations/layer11_attention_weight_max": 36.94239807128906,
      "activations/layer11_attention_weight_min": -35.77177810668945,
      "activations/layer12_attention_weight_max": 18.952407836914062,
      "activations/layer12_attention_weight_min": -27.14692497253418,
      "activations/layer13_attention_weight_max": 34.12539291381836,
      "activations/layer13_attention_weight_min": -27.530197143554688,
      "activations/layer14_attention_weight_max": 34.29456329345703,
      "activations/layer14_attention_weight_min": -29.47115707397461,
      "activations/layer15_attention_weight_max": 31.96779441833496,
      "activations/layer15_attention_weight_min": -30.419231414794922,
      "activations/layer16_attention_weight_max": 33.600406646728516,
      "activations/layer16_attention_weight_min": -30.20575714111328,
      "activations/layer17_attention_weight_max": 50.49220657348633,
      "activations/layer17_attention_weight_min": -45.385520935058594,
      "activations/layer18_attention_weight_max": 45.49083709716797,
      "activations/layer18_attention_weight_min": -42.2236213684082,
      "activations/layer19_attention_weight_max": 23.029630661010742,
      "activations/layer19_attention_weight_min": -22.886751174926758,
      "activations/layer1_attention_weight_max": 15.67542839050293,
      "activations/layer1_attention_weight_min": -14.811577796936035,
      "activations/layer20_attention_weight_max": 22.53432273864746,
      "activations/layer20_attention_weight_min": -24.43020248413086,
      "activations/layer21_attention_weight_max": 33.0328369140625,
      "activations/layer21_attention_weight_min": -23.137704849243164,
      "activations/layer22_attention_weight_max": 28.22037124633789,
      "activations/layer22_attention_weight_min": -27.14815330505371,
      "activations/layer23_attention_weight_max": 38.9335823059082,
      "activations/layer23_attention_weight_min": -24.98811912536621,
      "activations/layer2_attention_weight_max": 33.23405838012695,
      "activations/layer2_attention_weight_min": -32.842994689941406,
      "activations/layer3_attention_weight_max": 92.35315704345703,
      "activations/layer3_attention_weight_min": -93.24633026123047,
      "activations/layer4_attention_weight_max": 88.77228546142578,
      "activations/layer4_attention_weight_min": -92.11620330810547,
      "activations/layer5_attention_weight_max": 62.34830856323242,
      "activations/layer5_attention_weight_min": -80.33218383789062,
      "activations/layer6_attention_weight_max": 50.490291595458984,
      "activations/layer6_attention_weight_min": -50.497379302978516,
      "activations/layer7_attention_weight_max": 67.022705078125,
      "activations/layer7_attention_weight_min": -67.30487823486328,
      "activations/layer8_attention_weight_max": 51.563438415527344,
      "activations/layer8_attention_weight_min": -52.022216796875,
      "activations/layer9_attention_weight_max": 50.894989013671875,
      "activations/layer9_attention_weight_min": -47.53889083862305,
      "epoch": 16.95,
      "learning_rate": 4.1112499999999996e-05,
      "loss": 2.7341,
      "step": 291700
    },
    {
      "activations/layer0_attention_weight_max": 16.026817321777344,
      "activations/layer0_attention_weight_min": -13.78249740600586,
      "activations/layer10_attention_weight_max": 34.308074951171875,
      "activations/layer10_attention_weight_min": -34.50513458251953,
      "activations/layer11_attention_weight_max": 33.30158233642578,
      "activations/layer11_attention_weight_min": -33.178714752197266,
      "activations/layer12_attention_weight_max": 19.467182159423828,
      "activations/layer12_attention_weight_min": -26.78255844116211,
      "activations/layer13_attention_weight_max": 32.460838317871094,
      "activations/layer13_attention_weight_min": -25.99326515197754,
      "activations/layer14_attention_weight_max": 35.22838592529297,
      "activations/layer14_attention_weight_min": -31.44584846496582,
      "activations/layer15_attention_weight_max": 32.78416061401367,
      "activations/layer15_attention_weight_min": -32.50551223754883,
      "activations/layer16_attention_weight_max": 34.09571838378906,
      "activations/layer16_attention_weight_min": -31.610427856445312,
      "activations/layer17_attention_weight_max": 51.28434371948242,
      "activations/layer17_attention_weight_min": -45.07454299926758,
      "activations/layer18_attention_weight_max": 45.056087493896484,
      "activations/layer18_attention_weight_min": -40.03157424926758,
      "activations/layer19_attention_weight_max": 23.325212478637695,
      "activations/layer19_attention_weight_min": -23.63638687133789,
      "activations/layer1_attention_weight_max": 16.23543930053711,
      "activations/layer1_attention_weight_min": -14.671624183654785,
      "activations/layer20_attention_weight_max": 23.740324020385742,
      "activations/layer20_attention_weight_min": -22.399375915527344,
      "activations/layer21_attention_weight_max": 37.4755859375,
      "activations/layer21_attention_weight_min": -22.261215209960938,
      "activations/layer22_attention_weight_max": 29.073640823364258,
      "activations/layer22_attention_weight_min": -26.16852569580078,
      "activations/layer23_attention_weight_max": 35.545265197753906,
      "activations/layer23_attention_weight_min": -25.171043395996094,
      "activations/layer2_attention_weight_max": 31.949344635009766,
      "activations/layer2_attention_weight_min": -30.964948654174805,
      "activations/layer3_attention_weight_max": 86.93466186523438,
      "activations/layer3_attention_weight_min": -87.7703857421875,
      "activations/layer4_attention_weight_max": 89.23011016845703,
      "activations/layer4_attention_weight_min": -88.68828582763672,
      "activations/layer5_attention_weight_max": 63.55120086669922,
      "activations/layer5_attention_weight_min": -71.64781951904297,
      "activations/layer6_attention_weight_max": 49.70866394042969,
      "activations/layer6_attention_weight_min": -50.669837951660156,
      "activations/layer7_attention_weight_max": 63.514427185058594,
      "activations/layer7_attention_weight_min": -60.31848907470703,
      "activations/layer8_attention_weight_max": 42.952022552490234,
      "activations/layer8_attention_weight_min": -46.057037353515625,
      "activations/layer9_attention_weight_max": 43.17989730834961,
      "activations/layer9_attention_weight_min": -43.79545211791992,
      "epoch": 16.95,
      "learning_rate": 4.1093560606060605e-05,
      "loss": 2.762,
      "step": 291750
    },
    {
      "activations/layer0_attention_weight_max": 15.01651668548584,
      "activations/layer0_attention_weight_min": -13.4273681640625,
      "activations/layer10_attention_weight_max": 37.004005432128906,
      "activations/layer10_attention_weight_min": -38.61033630371094,
      "activations/layer11_attention_weight_max": 36.51083755493164,
      "activations/layer11_attention_weight_min": -36.99443817138672,
      "activations/layer12_attention_weight_max": 18.33381462097168,
      "activations/layer12_attention_weight_min": -28.516401290893555,
      "activations/layer13_attention_weight_max": 33.00691223144531,
      "activations/layer13_attention_weight_min": -26.89329719543457,
      "activations/layer14_attention_weight_max": 35.36808395385742,
      "activations/layer14_attention_weight_min": -29.165802001953125,
      "activations/layer15_attention_weight_max": 33.298133850097656,
      "activations/layer15_attention_weight_min": -30.54269790649414,
      "activations/layer16_attention_weight_max": 32.96055221557617,
      "activations/layer16_attention_weight_min": -29.890121459960938,
      "activations/layer17_attention_weight_max": 57.87156295776367,
      "activations/layer17_attention_weight_min": -46.579593658447266,
      "activations/layer18_attention_weight_max": 49.94239044189453,
      "activations/layer18_attention_weight_min": -40.68815994262695,
      "activations/layer19_attention_weight_max": 23.97636604309082,
      "activations/layer19_attention_weight_min": -23.549423217773438,
      "activations/layer1_attention_weight_max": 16.168533325195312,
      "activations/layer1_attention_weight_min": -15.812250137329102,
      "activations/layer20_attention_weight_max": 23.858610153198242,
      "activations/layer20_attention_weight_min": -23.0454044342041,
      "activations/layer21_attention_weight_max": 40.31007385253906,
      "activations/layer21_attention_weight_min": -24.02811622619629,
      "activations/layer22_attention_weight_max": 29.40376091003418,
      "activations/layer22_attention_weight_min": -25.65491485595703,
      "activations/layer23_attention_weight_max": 35.04908752441406,
      "activations/layer23_attention_weight_min": -25.20674705505371,
      "activations/layer2_attention_weight_max": 37.51194381713867,
      "activations/layer2_attention_weight_min": -35.61204528808594,
      "activations/layer3_attention_weight_max": 95.8593978881836,
      "activations/layer3_attention_weight_min": -101.16877746582031,
      "activations/layer4_attention_weight_max": 93.93507385253906,
      "activations/layer4_attention_weight_min": -90.04725646972656,
      "activations/layer5_attention_weight_max": 67.01683044433594,
      "activations/layer5_attention_weight_min": -77.20355987548828,
      "activations/layer6_attention_weight_max": 51.43010711669922,
      "activations/layer6_attention_weight_min": -52.28873062133789,
      "activations/layer7_attention_weight_max": 64.43276977539062,
      "activations/layer7_attention_weight_min": -70.64191436767578,
      "activations/layer8_attention_weight_max": 51.788387298583984,
      "activations/layer8_attention_weight_min": -51.494869232177734,
      "activations/layer9_attention_weight_max": 48.33498001098633,
      "activations/layer9_attention_weight_min": -48.640506744384766,
      "epoch": 16.96,
      "learning_rate": 4.1074621212121206e-05,
      "loss": 2.7405,
      "step": 291800
    },
    {
      "activations/layer0_attention_weight_max": 15.63900089263916,
      "activations/layer0_attention_weight_min": -13.671709060668945,
      "activations/layer10_attention_weight_max": 39.91307830810547,
      "activations/layer10_attention_weight_min": -38.107601165771484,
      "activations/layer11_attention_weight_max": 37.28435134887695,
      "activations/layer11_attention_weight_min": -39.47340393066406,
      "activations/layer12_attention_weight_max": 19.111103057861328,
      "activations/layer12_attention_weight_min": -26.47562599182129,
      "activations/layer13_attention_weight_max": 33.396690368652344,
      "activations/layer13_attention_weight_min": -28.08576011657715,
      "activations/layer14_attention_weight_max": 34.72990036010742,
      "activations/layer14_attention_weight_min": -31.117891311645508,
      "activations/layer15_attention_weight_max": 33.39448928833008,
      "activations/layer15_attention_weight_min": -29.566009521484375,
      "activations/layer16_attention_weight_max": 34.71719741821289,
      "activations/layer16_attention_weight_min": -32.93183135986328,
      "activations/layer17_attention_weight_max": 51.70502853393555,
      "activations/layer17_attention_weight_min": -43.97297286987305,
      "activations/layer18_attention_weight_max": 46.88352584838867,
      "activations/layer18_attention_weight_min": -41.06202697753906,
      "activations/layer19_attention_weight_max": 23.58656120300293,
      "activations/layer19_attention_weight_min": -23.359697341918945,
      "activations/layer1_attention_weight_max": 16.307958602905273,
      "activations/layer1_attention_weight_min": -12.929853439331055,
      "activations/layer20_attention_weight_max": 21.786638259887695,
      "activations/layer20_attention_weight_min": -22.54966926574707,
      "activations/layer21_attention_weight_max": 35.04140090942383,
      "activations/layer21_attention_weight_min": -23.655059814453125,
      "activations/layer22_attention_weight_max": 29.293182373046875,
      "activations/layer22_attention_weight_min": -25.8320369720459,
      "activations/layer23_attention_weight_max": 38.463600158691406,
      "activations/layer23_attention_weight_min": -25.752304077148438,
      "activations/layer2_attention_weight_max": 30.34866714477539,
      "activations/layer2_attention_weight_min": -30.818700790405273,
      "activations/layer3_attention_weight_max": 89.22709655761719,
      "activations/layer3_attention_weight_min": -88.85004425048828,
      "activations/layer4_attention_weight_max": 91.93206024169922,
      "activations/layer4_attention_weight_min": -87.9984359741211,
      "activations/layer5_attention_weight_max": 64.09872436523438,
      "activations/layer5_attention_weight_min": -74.50514221191406,
      "activations/layer6_attention_weight_max": 52.68837356567383,
      "activations/layer6_attention_weight_min": -54.00838851928711,
      "activations/layer7_attention_weight_max": 65.93138122558594,
      "activations/layer7_attention_weight_min": -64.50366973876953,
      "activations/layer8_attention_weight_max": 49.22481155395508,
      "activations/layer8_attention_weight_min": -51.44877624511719,
      "activations/layer9_attention_weight_max": 49.07938766479492,
      "activations/layer9_attention_weight_min": -50.420352935791016,
      "epoch": 16.96,
      "learning_rate": 4.105568181818182e-05,
      "loss": 2.7495,
      "step": 291850
    },
    {
      "activations/layer0_attention_weight_max": 15.919891357421875,
      "activations/layer0_attention_weight_min": -13.7540922164917,
      "activations/layer10_attention_weight_max": 36.063926696777344,
      "activations/layer10_attention_weight_min": -36.3677864074707,
      "activations/layer11_attention_weight_max": 30.676036834716797,
      "activations/layer11_attention_weight_min": -34.98982238769531,
      "activations/layer12_attention_weight_max": 19.434640884399414,
      "activations/layer12_attention_weight_min": -23.4237117767334,
      "activations/layer13_attention_weight_max": 36.97792434692383,
      "activations/layer13_attention_weight_min": -29.36077117919922,
      "activations/layer14_attention_weight_max": 36.256370544433594,
      "activations/layer14_attention_weight_min": -30.142820358276367,
      "activations/layer15_attention_weight_max": 34.64762496948242,
      "activations/layer15_attention_weight_min": -29.685529708862305,
      "activations/layer16_attention_weight_max": 33.74662780761719,
      "activations/layer16_attention_weight_min": -32.65608596801758,
      "activations/layer17_attention_weight_max": 50.29065704345703,
      "activations/layer17_attention_weight_min": -46.38237762451172,
      "activations/layer18_attention_weight_max": 46.565589904785156,
      "activations/layer18_attention_weight_min": -42.846412658691406,
      "activations/layer19_attention_weight_max": 23.215383529663086,
      "activations/layer19_attention_weight_min": -23.162994384765625,
      "activations/layer1_attention_weight_max": 15.494942665100098,
      "activations/layer1_attention_weight_min": -13.618027687072754,
      "activations/layer20_attention_weight_max": 22.546260833740234,
      "activations/layer20_attention_weight_min": -22.39295768737793,
      "activations/layer21_attention_weight_max": 39.313377380371094,
      "activations/layer21_attention_weight_min": -25.971935272216797,
      "activations/layer22_attention_weight_max": 28.69446563720703,
      "activations/layer22_attention_weight_min": -27.641830444335938,
      "activations/layer23_attention_weight_max": 40.26702880859375,
      "activations/layer23_attention_weight_min": -24.907997131347656,
      "activations/layer2_attention_weight_max": 30.325542449951172,
      "activations/layer2_attention_weight_min": -29.62599754333496,
      "activations/layer3_attention_weight_max": 82.27314758300781,
      "activations/layer3_attention_weight_min": -85.18199157714844,
      "activations/layer4_attention_weight_max": 84.7650375366211,
      "activations/layer4_attention_weight_min": -85.8619384765625,
      "activations/layer5_attention_weight_max": 62.38695526123047,
      "activations/layer5_attention_weight_min": -70.28636932373047,
      "activations/layer6_attention_weight_max": 48.18097686767578,
      "activations/layer6_attention_weight_min": -52.53609085083008,
      "activations/layer7_attention_weight_max": 64.80952453613281,
      "activations/layer7_attention_weight_min": -63.689453125,
      "activations/layer8_attention_weight_max": 45.21550369262695,
      "activations/layer8_attention_weight_min": -50.80540084838867,
      "activations/layer9_attention_weight_max": 41.69221496582031,
      "activations/layer9_attention_weight_min": -45.90343475341797,
      "epoch": 16.96,
      "learning_rate": 4.103674242424242e-05,
      "loss": 2.7386,
      "step": 291900
    },
    {
      "activations/layer0_attention_weight_max": 17.333982467651367,
      "activations/layer0_attention_weight_min": -13.77833366394043,
      "activations/layer10_attention_weight_max": 38.91704177856445,
      "activations/layer10_attention_weight_min": -37.70648193359375,
      "activations/layer11_attention_weight_max": 34.994239807128906,
      "activations/layer11_attention_weight_min": -35.39753723144531,
      "activations/layer12_attention_weight_max": 17.74579620361328,
      "activations/layer12_attention_weight_min": -24.86650848388672,
      "activations/layer13_attention_weight_max": 30.75070571899414,
      "activations/layer13_attention_weight_min": -26.659915924072266,
      "activations/layer14_attention_weight_max": 31.991046905517578,
      "activations/layer14_attention_weight_min": -29.5528621673584,
      "activations/layer15_attention_weight_max": 29.85771942138672,
      "activations/layer15_attention_weight_min": -30.731626510620117,
      "activations/layer16_attention_weight_max": 31.488582611083984,
      "activations/layer16_attention_weight_min": -30.1099910736084,
      "activations/layer17_attention_weight_max": 49.85847854614258,
      "activations/layer17_attention_weight_min": -42.72523880004883,
      "activations/layer18_attention_weight_max": 44.4480094909668,
      "activations/layer18_attention_weight_min": -36.60504150390625,
      "activations/layer19_attention_weight_max": 21.01680564880371,
      "activations/layer19_attention_weight_min": -21.7626953125,
      "activations/layer1_attention_weight_max": 15.906835556030273,
      "activations/layer1_attention_weight_min": -15.43601131439209,
      "activations/layer20_attention_weight_max": 22.129732131958008,
      "activations/layer20_attention_weight_min": -21.666088104248047,
      "activations/layer21_attention_weight_max": 34.003692626953125,
      "activations/layer21_attention_weight_min": -20.307392120361328,
      "activations/layer22_attention_weight_max": 28.62989616394043,
      "activations/layer22_attention_weight_min": -26.283447265625,
      "activations/layer23_attention_weight_max": 36.48186111450195,
      "activations/layer23_attention_weight_min": -24.406452178955078,
      "activations/layer2_attention_weight_max": 32.814064025878906,
      "activations/layer2_attention_weight_min": -30.9782772064209,
      "activations/layer3_attention_weight_max": 91.8768310546875,
      "activations/layer3_attention_weight_min": -88.10465240478516,
      "activations/layer4_attention_weight_max": 93.05496978759766,
      "activations/layer4_attention_weight_min": -94.20768737792969,
      "activations/layer5_attention_weight_max": 69.44503021240234,
      "activations/layer5_attention_weight_min": -78.29727172851562,
      "activations/layer6_attention_weight_max": 50.692386627197266,
      "activations/layer6_attention_weight_min": -51.667728424072266,
      "activations/layer7_attention_weight_max": 66.17976379394531,
      "activations/layer7_attention_weight_min": -63.89168930053711,
      "activations/layer8_attention_weight_max": 46.906211853027344,
      "activations/layer8_attention_weight_min": -51.1964111328125,
      "activations/layer9_attention_weight_max": 45.82889938354492,
      "activations/layer9_attention_weight_min": -46.98237991333008,
      "epoch": 16.96,
      "learning_rate": 4.1017803030303024e-05,
      "loss": 2.7514,
      "step": 291950
    },
    {
      "activations/layer0_attention_weight_max": 15.72838306427002,
      "activations/layer0_attention_weight_min": -13.823404312133789,
      "activations/layer10_attention_weight_max": 35.074485778808594,
      "activations/layer10_attention_weight_min": -35.549293518066406,
      "activations/layer11_attention_weight_max": 31.252582550048828,
      "activations/layer11_attention_weight_min": -33.772735595703125,
      "activations/layer12_attention_weight_max": 17.98624610900879,
      "activations/layer12_attention_weight_min": -30.146774291992188,
      "activations/layer13_attention_weight_max": 38.89417266845703,
      "activations/layer13_attention_weight_min": -28.82111167907715,
      "activations/layer14_attention_weight_max": 43.4423713684082,
      "activations/layer14_attention_weight_min": -30.530670166015625,
      "activations/layer15_attention_weight_max": 44.97428894042969,
      "activations/layer15_attention_weight_min": -29.619644165039062,
      "activations/layer16_attention_weight_max": 32.42357635498047,
      "activations/layer16_attention_weight_min": -30.50482940673828,
      "activations/layer17_attention_weight_max": 50.855892181396484,
      "activations/layer17_attention_weight_min": -45.29213333129883,
      "activations/layer18_attention_weight_max": 44.92612075805664,
      "activations/layer18_attention_weight_min": -39.81195831298828,
      "activations/layer19_attention_weight_max": 22.428319931030273,
      "activations/layer19_attention_weight_min": -21.76797866821289,
      "activations/layer1_attention_weight_max": 15.134526252746582,
      "activations/layer1_attention_weight_min": -14.112771987915039,
      "activations/layer20_attention_weight_max": 20.893753051757812,
      "activations/layer20_attention_weight_min": -23.006370544433594,
      "activations/layer21_attention_weight_max": 35.48110580444336,
      "activations/layer21_attention_weight_min": -22.52875328063965,
      "activations/layer22_attention_weight_max": 25.96296501159668,
      "activations/layer22_attention_weight_min": -24.973651885986328,
      "activations/layer23_attention_weight_max": 32.374481201171875,
      "activations/layer23_attention_weight_min": -22.722553253173828,
      "activations/layer2_attention_weight_max": 30.282346725463867,
      "activations/layer2_attention_weight_min": -31.026683807373047,
      "activations/layer3_attention_weight_max": 86.00973510742188,
      "activations/layer3_attention_weight_min": -87.57115936279297,
      "activations/layer4_attention_weight_max": 90.04703521728516,
      "activations/layer4_attention_weight_min": -91.13031768798828,
      "activations/layer5_attention_weight_max": 64.1214828491211,
      "activations/layer5_attention_weight_min": -75.69287872314453,
      "activations/layer6_attention_weight_max": 49.743812561035156,
      "activations/layer6_attention_weight_min": -52.014339447021484,
      "activations/layer7_attention_weight_max": 64.90814971923828,
      "activations/layer7_attention_weight_min": -63.6955451965332,
      "activations/layer8_attention_weight_max": 46.112205505371094,
      "activations/layer8_attention_weight_min": -50.38243103027344,
      "activations/layer9_attention_weight_max": 43.377830505371094,
      "activations/layer9_attention_weight_min": -45.10896682739258,
      "epoch": 16.97,
      "learning_rate": 4.0999242424242415e-05,
      "loss": 2.7471,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_loss": 2.705078125,
      "eval_runtime": 8.4924,
      "eval_samples_per_second": 505.627,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_openwebtext_loss": 2.705078125,
      "eval_openwebtext_ppl": 14.95548504215564,
      "eval_openwebtext_runtime": 8.4924,
      "eval_openwebtext_samples_per_second": 505.627,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9373,
      "eval_wikitext_samples_per_second": 235.379,
      "step": 292000
    },
    {
      "epoch": 16.97,
      "eval_lambada_loss": 2.48828125,
      "eval_lambada_ppl": 12.040563607027826,
      "eval_lambada_runtime": 9.503,
      "eval_lambada_samples_per_second": 512.367,
      "step": 292000
    },
    {
      "activations/layer0_attention_weight_max": 14.944215774536133,
      "activations/layer0_attention_weight_min": -13.915081024169922,
      "activations/layer10_attention_weight_max": 39.42445755004883,
      "activations/layer10_attention_weight_min": -38.25370788574219,
      "activations/layer11_attention_weight_max": 36.14362716674805,
      "activations/layer11_attention_weight_min": -35.261497497558594,
      "activations/layer12_attention_weight_max": 18.110843658447266,
      "activations/layer12_attention_weight_min": -26.017000198364258,
      "activations/layer13_attention_weight_max": 35.33142852783203,
      "activations/layer13_attention_weight_min": -26.90371322631836,
      "activations/layer14_attention_weight_max": 36.753684997558594,
      "activations/layer14_attention_weight_min": -31.115467071533203,
      "activations/layer15_attention_weight_max": 32.21149444580078,
      "activations/layer15_attention_weight_min": -31.000391006469727,
      "activations/layer16_attention_weight_max": 33.41690444946289,
      "activations/layer16_attention_weight_min": -32.59169387817383,
      "activations/layer17_attention_weight_max": 52.598175048828125,
      "activations/layer17_attention_weight_min": -48.42852020263672,
      "activations/layer18_attention_weight_max": 48.424930572509766,
      "activations/layer18_attention_weight_min": -43.23880386352539,
      "activations/layer19_attention_weight_max": 24.701772689819336,
      "activations/layer19_attention_weight_min": -21.69684410095215,
      "activations/layer1_attention_weight_max": 15.675867080688477,
      "activations/layer1_attention_weight_min": -12.635283470153809,
      "activations/layer20_attention_weight_max": 25.775394439697266,
      "activations/layer20_attention_weight_min": -22.77789878845215,
      "activations/layer21_attention_weight_max": 40.73097229003906,
      "activations/layer21_attention_weight_min": -21.341100692749023,
      "activations/layer22_attention_weight_max": 29.11083221435547,
      "activations/layer22_attention_weight_min": -24.539270401000977,
      "activations/layer23_attention_weight_max": 41.06416702270508,
      "activations/layer23_attention_weight_min": -25.259244918823242,
      "activations/layer2_attention_weight_max": 29.951400756835938,
      "activations/layer2_attention_weight_min": -28.554832458496094,
      "activations/layer3_attention_weight_max": 80.23250579833984,
      "activations/layer3_attention_weight_min": -82.584716796875,
      "activations/layer4_attention_weight_max": 83.3059310913086,
      "activations/layer4_attention_weight_min": -84.62340545654297,
      "activations/layer5_attention_weight_max": 60.67310333251953,
      "activations/layer5_attention_weight_min": -73.0182113647461,
      "activations/layer6_attention_weight_max": 49.86996078491211,
      "activations/layer6_attention_weight_min": -50.90494918823242,
      "activations/layer7_attention_weight_max": 66.76844787597656,
      "activations/layer7_attention_weight_min": -63.02042007446289,
      "activations/layer8_attention_weight_max": 46.92646026611328,
      "activations/layer8_attention_weight_min": -49.072811126708984,
      "activations/layer9_attention_weight_max": 46.78304672241211,
      "activations/layer9_attention_weight_min": -48.352535247802734,
      "epoch": 16.97,
      "learning_rate": 4.098030303030303e-05,
      "loss": 2.7597,
      "step": 292050
    },
    {
      "activations/layer0_attention_weight_max": 15.904142379760742,
      "activations/layer0_attention_weight_min": -13.97716999053955,
      "activations/layer10_attention_weight_max": 37.713722229003906,
      "activations/layer10_attention_weight_min": -35.466033935546875,
      "activations/layer11_attention_weight_max": 35.119720458984375,
      "activations/layer11_attention_weight_min": -34.931575775146484,
      "activations/layer12_attention_weight_max": 19.531822204589844,
      "activations/layer12_attention_weight_min": -25.787887573242188,
      "activations/layer13_attention_weight_max": 38.74607849121094,
      "activations/layer13_attention_weight_min": -27.272388458251953,
      "activations/layer14_attention_weight_max": 35.89826202392578,
      "activations/layer14_attention_weight_min": -29.02443504333496,
      "activations/layer15_attention_weight_max": 32.336830139160156,
      "activations/layer15_attention_weight_min": -28.496906280517578,
      "activations/layer16_attention_weight_max": 32.999603271484375,
      "activations/layer16_attention_weight_min": -31.549083709716797,
      "activations/layer17_attention_weight_max": 49.96256637573242,
      "activations/layer17_attention_weight_min": -42.68668746948242,
      "activations/layer18_attention_weight_max": 46.84352493286133,
      "activations/layer18_attention_weight_min": -38.72779083251953,
      "activations/layer19_attention_weight_max": 24.007104873657227,
      "activations/layer19_attention_weight_min": -22.52107048034668,
      "activations/layer1_attention_weight_max": 15.718385696411133,
      "activations/layer1_attention_weight_min": -14.025941848754883,
      "activations/layer20_attention_weight_max": 22.901947021484375,
      "activations/layer20_attention_weight_min": -22.987930297851562,
      "activations/layer21_attention_weight_max": 39.940670013427734,
      "activations/layer21_attention_weight_min": -21.364797592163086,
      "activations/layer22_attention_weight_max": 29.20241355895996,
      "activations/layer22_attention_weight_min": -25.125019073486328,
      "activations/layer23_attention_weight_max": 39.136356353759766,
      "activations/layer23_attention_weight_min": -25.064268112182617,
      "activations/layer2_attention_weight_max": 31.92951011657715,
      "activations/layer2_attention_weight_min": -32.12239074707031,
      "activations/layer3_attention_weight_max": 86.66991424560547,
      "activations/layer3_attention_weight_min": -87.36363983154297,
      "activations/layer4_attention_weight_max": 87.6242446899414,
      "activations/layer4_attention_weight_min": -86.49484252929688,
      "activations/layer5_attention_weight_max": 63.50608825683594,
      "activations/layer5_attention_weight_min": -78.5562744140625,
      "activations/layer6_attention_weight_max": 49.62922286987305,
      "activations/layer6_attention_weight_min": -53.24058151245117,
      "activations/layer7_attention_weight_max": 66.36475372314453,
      "activations/layer7_attention_weight_min": -64.46878051757812,
      "activations/layer8_attention_weight_max": 46.53950881958008,
      "activations/layer8_attention_weight_min": -48.192874908447266,
      "activations/layer9_attention_weight_max": 45.00843048095703,
      "activations/layer9_attention_weight_min": -46.77811813354492,
      "epoch": 16.97,
      "learning_rate": 4.096136363636363e-05,
      "loss": 2.7355,
      "step": 292100
    },
    {
      "activations/layer0_attention_weight_max": 15.189413070678711,
      "activations/layer0_attention_weight_min": -13.878774642944336,
      "activations/layer10_attention_weight_max": 37.345340728759766,
      "activations/layer10_attention_weight_min": -36.0568962097168,
      "activations/layer11_attention_weight_max": 31.69869041442871,
      "activations/layer11_attention_weight_min": -35.600799560546875,
      "activations/layer12_attention_weight_max": 18.84810447692871,
      "activations/layer12_attention_weight_min": -26.59182357788086,
      "activations/layer13_attention_weight_max": 37.50060272216797,
      "activations/layer13_attention_weight_min": -27.470245361328125,
      "activations/layer14_attention_weight_max": 37.859222412109375,
      "activations/layer14_attention_weight_min": -30.53862762451172,
      "activations/layer15_attention_weight_max": 34.36051559448242,
      "activations/layer15_attention_weight_min": -31.39678382873535,
      "activations/layer16_attention_weight_max": 35.166603088378906,
      "activations/layer16_attention_weight_min": -32.03968048095703,
      "activations/layer17_attention_weight_max": 52.37394332885742,
      "activations/layer17_attention_weight_min": -44.620460510253906,
      "activations/layer18_attention_weight_max": 47.73532485961914,
      "activations/layer18_attention_weight_min": -39.295169830322266,
      "activations/layer19_attention_weight_max": 22.451526641845703,
      "activations/layer19_attention_weight_min": -23.462038040161133,
      "activations/layer1_attention_weight_max": 16.642290115356445,
      "activations/layer1_attention_weight_min": -15.387187957763672,
      "activations/layer20_attention_weight_max": 21.423358917236328,
      "activations/layer20_attention_weight_min": -22.198511123657227,
      "activations/layer21_attention_weight_max": 39.58506774902344,
      "activations/layer21_attention_weight_min": -23.274002075195312,
      "activations/layer22_attention_weight_max": 29.180479049682617,
      "activations/layer22_attention_weight_min": -25.15984535217285,
      "activations/layer23_attention_weight_max": 39.636085510253906,
      "activations/layer23_attention_weight_min": -24.329391479492188,
      "activations/layer2_attention_weight_max": 32.269432067871094,
      "activations/layer2_attention_weight_min": -31.307592391967773,
      "activations/layer3_attention_weight_max": 87.94900512695312,
      "activations/layer3_attention_weight_min": -90.13701629638672,
      "activations/layer4_attention_weight_max": 89.59537506103516,
      "activations/layer4_attention_weight_min": -85.28216552734375,
      "activations/layer5_attention_weight_max": 63.88355255126953,
      "activations/layer5_attention_weight_min": -75.55448913574219,
      "activations/layer6_attention_weight_max": 51.649295806884766,
      "activations/layer6_attention_weight_min": -51.48563766479492,
      "activations/layer7_attention_weight_max": 66.40055847167969,
      "activations/layer7_attention_weight_min": -66.76630401611328,
      "activations/layer8_attention_weight_max": 48.03418731689453,
      "activations/layer8_attention_weight_min": -51.7211799621582,
      "activations/layer9_attention_weight_max": 43.10895919799805,
      "activations/layer9_attention_weight_min": -45.6527099609375,
      "epoch": 16.98,
      "learning_rate": 4.094242424242424e-05,
      "loss": 2.7489,
      "step": 292150
    },
    {
      "activations/layer0_attention_weight_max": 15.673999786376953,
      "activations/layer0_attention_weight_min": -13.862916946411133,
      "activations/layer10_attention_weight_max": 38.3264045715332,
      "activations/layer10_attention_weight_min": -36.11380386352539,
      "activations/layer11_attention_weight_max": 35.45643615722656,
      "activations/layer11_attention_weight_min": -34.25785827636719,
      "activations/layer12_attention_weight_max": 19.458192825317383,
      "activations/layer12_attention_weight_min": -30.361528396606445,
      "activations/layer13_attention_weight_max": 35.132965087890625,
      "activations/layer13_attention_weight_min": -26.155893325805664,
      "activations/layer14_attention_weight_max": 36.561519622802734,
      "activations/layer14_attention_weight_min": -29.67962646484375,
      "activations/layer15_attention_weight_max": 34.54024887084961,
      "activations/layer15_attention_weight_min": -28.99343490600586,
      "activations/layer16_attention_weight_max": 34.90894317626953,
      "activations/layer16_attention_weight_min": -31.824588775634766,
      "activations/layer17_attention_weight_max": 53.5858039855957,
      "activations/layer17_attention_weight_min": -42.9590950012207,
      "activations/layer18_attention_weight_max": 49.0188102722168,
      "activations/layer18_attention_weight_min": -38.15008544921875,
      "activations/layer19_attention_weight_max": 22.131338119506836,
      "activations/layer19_attention_weight_min": -23.117008209228516,
      "activations/layer1_attention_weight_max": 16.069974899291992,
      "activations/layer1_attention_weight_min": -16.480051040649414,
      "activations/layer20_attention_weight_max": 21.96099090576172,
      "activations/layer20_attention_weight_min": -23.701383590698242,
      "activations/layer21_attention_weight_max": 39.70405960083008,
      "activations/layer21_attention_weight_min": -26.741844177246094,
      "activations/layer22_attention_weight_max": 30.91912078857422,
      "activations/layer22_attention_weight_min": -26.245100021362305,
      "activations/layer23_attention_weight_max": 36.461822509765625,
      "activations/layer23_attention_weight_min": -25.315746307373047,
      "activations/layer2_attention_weight_max": 33.633689880371094,
      "activations/layer2_attention_weight_min": -31.29192352294922,
      "activations/layer3_attention_weight_max": 89.30282592773438,
      "activations/layer3_attention_weight_min": -86.88908386230469,
      "activations/layer4_attention_weight_max": 86.90501403808594,
      "activations/layer4_attention_weight_min": -78.71199798583984,
      "activations/layer5_attention_weight_max": 64.94619750976562,
      "activations/layer5_attention_weight_min": -69.23823547363281,
      "activations/layer6_attention_weight_max": 50.05184555053711,
      "activations/layer6_attention_weight_min": -50.282936096191406,
      "activations/layer7_attention_weight_max": 65.109375,
      "activations/layer7_attention_weight_min": -61.49793243408203,
      "activations/layer8_attention_weight_max": 47.922855377197266,
      "activations/layer8_attention_weight_min": -47.48883056640625,
      "activations/layer9_attention_weight_max": 44.79104232788086,
      "activations/layer9_attention_weight_min": -45.58554458618164,
      "epoch": 16.98,
      "learning_rate": 4.092348484848484e-05,
      "loss": 2.7498,
      "step": 292200
    },
    {
      "activations/layer0_attention_weight_max": 16.344833374023438,
      "activations/layer0_attention_weight_min": -14.05550479888916,
      "activations/layer10_attention_weight_max": 37.93194580078125,
      "activations/layer10_attention_weight_min": -39.62800598144531,
      "activations/layer11_attention_weight_max": 34.489105224609375,
      "activations/layer11_attention_weight_min": -39.324424743652344,
      "activations/layer12_attention_weight_max": 18.635196685791016,
      "activations/layer12_attention_weight_min": -24.54996681213379,
      "activations/layer13_attention_weight_max": 34.79832458496094,
      "activations/layer13_attention_weight_min": -29.001537322998047,
      "activations/layer14_attention_weight_max": 34.91599655151367,
      "activations/layer14_attention_weight_min": -30.74944305419922,
      "activations/layer15_attention_weight_max": 34.65879440307617,
      "activations/layer15_attention_weight_min": -30.362306594848633,
      "activations/layer16_attention_weight_max": 33.31001663208008,
      "activations/layer16_attention_weight_min": -31.51845359802246,
      "activations/layer17_attention_weight_max": 50.247230529785156,
      "activations/layer17_attention_weight_min": -45.56132888793945,
      "activations/layer18_attention_weight_max": 45.747535705566406,
      "activations/layer18_attention_weight_min": -39.85472106933594,
      "activations/layer19_attention_weight_max": 24.74672508239746,
      "activations/layer19_attention_weight_min": -22.447120666503906,
      "activations/layer1_attention_weight_max": 16.305126190185547,
      "activations/layer1_attention_weight_min": -14.572484016418457,
      "activations/layer20_attention_weight_max": 22.492280960083008,
      "activations/layer20_attention_weight_min": -21.93218231201172,
      "activations/layer21_attention_weight_max": 32.1757698059082,
      "activations/layer21_attention_weight_min": -21.032140731811523,
      "activations/layer22_attention_weight_max": 28.60702896118164,
      "activations/layer22_attention_weight_min": -25.960386276245117,
      "activations/layer23_attention_weight_max": 37.22260665893555,
      "activations/layer23_attention_weight_min": -23.92205047607422,
      "activations/layer2_attention_weight_max": 30.267263412475586,
      "activations/layer2_attention_weight_min": -30.433897018432617,
      "activations/layer3_attention_weight_max": 84.54728698730469,
      "activations/layer3_attention_weight_min": -83.79796600341797,
      "activations/layer4_attention_weight_max": 87.19624328613281,
      "activations/layer4_attention_weight_min": -83.3647689819336,
      "activations/layer5_attention_weight_max": 62.096038818359375,
      "activations/layer5_attention_weight_min": -70.63818359375,
      "activations/layer6_attention_weight_max": 49.223636627197266,
      "activations/layer6_attention_weight_min": -48.94258117675781,
      "activations/layer7_attention_weight_max": 63.2754020690918,
      "activations/layer7_attention_weight_min": -62.78580093383789,
      "activations/layer8_attention_weight_max": 44.567481994628906,
      "activations/layer8_attention_weight_min": -46.670654296875,
      "activations/layer9_attention_weight_max": 45.32857894897461,
      "activations/layer9_attention_weight_min": -44.742130279541016,
      "epoch": 16.98,
      "learning_rate": 4.0904545454545456e-05,
      "loss": 2.7405,
      "step": 292250
    },
    {
      "activations/layer0_attention_weight_max": 15.066581726074219,
      "activations/layer0_attention_weight_min": -13.762267112731934,
      "activations/layer10_attention_weight_max": 35.92256546020508,
      "activations/layer10_attention_weight_min": -34.87682342529297,
      "activations/layer11_attention_weight_max": 33.63240051269531,
      "activations/layer11_attention_weight_min": -33.901275634765625,
      "activations/layer12_attention_weight_max": 19.701534271240234,
      "activations/layer12_attention_weight_min": -28.442615509033203,
      "activations/layer13_attention_weight_max": 33.14317321777344,
      "activations/layer13_attention_weight_min": -27.120296478271484,
      "activations/layer14_attention_weight_max": 34.63551712036133,
      "activations/layer14_attention_weight_min": -29.958667755126953,
      "activations/layer15_attention_weight_max": 33.902530670166016,
      "activations/layer15_attention_weight_min": -29.62880516052246,
      "activations/layer16_attention_weight_max": 33.69620132446289,
      "activations/layer16_attention_weight_min": -30.15669822692871,
      "activations/layer17_attention_weight_max": 51.35547637939453,
      "activations/layer17_attention_weight_min": -42.470035552978516,
      "activations/layer18_attention_weight_max": 49.17078399658203,
      "activations/layer18_attention_weight_min": -39.66738510131836,
      "activations/layer19_attention_weight_max": 23.532217025756836,
      "activations/layer19_attention_weight_min": -24.267332077026367,
      "activations/layer1_attention_weight_max": 15.997499465942383,
      "activations/layer1_attention_weight_min": -14.918824195861816,
      "activations/layer20_attention_weight_max": 20.99441909790039,
      "activations/layer20_attention_weight_min": -21.96862030029297,
      "activations/layer21_attention_weight_max": 33.15770721435547,
      "activations/layer21_attention_weight_min": -19.523635864257812,
      "activations/layer22_attention_weight_max": 30.758285522460938,
      "activations/layer22_attention_weight_min": -25.121383666992188,
      "activations/layer23_attention_weight_max": 37.11386489868164,
      "activations/layer23_attention_weight_min": -25.928237915039062,
      "activations/layer2_attention_weight_max": 30.064193725585938,
      "activations/layer2_attention_weight_min": -30.330337524414062,
      "activations/layer3_attention_weight_max": 85.60801696777344,
      "activations/layer3_attention_weight_min": -85.7972183227539,
      "activations/layer4_attention_weight_max": 85.84063720703125,
      "activations/layer4_attention_weight_min": -86.38265228271484,
      "activations/layer5_attention_weight_max": 64.991455078125,
      "activations/layer5_attention_weight_min": -73.0828628540039,
      "activations/layer6_attention_weight_max": 55.64344024658203,
      "activations/layer6_attention_weight_min": -56.84657669067383,
      "activations/layer7_attention_weight_max": 63.94255828857422,
      "activations/layer7_attention_weight_min": -69.33560180664062,
      "activations/layer8_attention_weight_max": 48.81757736206055,
      "activations/layer8_attention_weight_min": -53.23572540283203,
      "activations/layer9_attention_weight_max": 42.812007904052734,
      "activations/layer9_attention_weight_min": -46.68764114379883,
      "epoch": 16.98,
      "learning_rate": 4.088560606060606e-05,
      "loss": 2.7768,
      "step": 292300
    },
    {
      "activations/layer0_attention_weight_max": 14.574914932250977,
      "activations/layer0_attention_weight_min": -13.881945610046387,
      "activations/layer10_attention_weight_max": 34.5576171875,
      "activations/layer10_attention_weight_min": -34.99485778808594,
      "activations/layer11_attention_weight_max": 32.50647735595703,
      "activations/layer11_attention_weight_min": -34.436683654785156,
      "activations/layer12_attention_weight_max": 18.275054931640625,
      "activations/layer12_attention_weight_min": -25.292659759521484,
      "activations/layer13_attention_weight_max": 33.425533294677734,
      "activations/layer13_attention_weight_min": -28.649944305419922,
      "activations/layer14_attention_weight_max": 35.44416427612305,
      "activations/layer14_attention_weight_min": -30.49822998046875,
      "activations/layer15_attention_weight_max": 32.46361541748047,
      "activations/layer15_attention_weight_min": -30.21084976196289,
      "activations/layer16_attention_weight_max": 31.197341918945312,
      "activations/layer16_attention_weight_min": -30.4036865234375,
      "activations/layer17_attention_weight_max": 49.6544075012207,
      "activations/layer17_attention_weight_min": -45.03937911987305,
      "activations/layer18_attention_weight_max": 45.66087341308594,
      "activations/layer18_attention_weight_min": -39.53248596191406,
      "activations/layer19_attention_weight_max": 23.3621883392334,
      "activations/layer19_attention_weight_min": -23.941925048828125,
      "activations/layer1_attention_weight_max": 17.65538787841797,
      "activations/layer1_attention_weight_min": -15.48232364654541,
      "activations/layer20_attention_weight_max": 22.96706771850586,
      "activations/layer20_attention_weight_min": -21.042348861694336,
      "activations/layer21_attention_weight_max": 31.896047592163086,
      "activations/layer21_attention_weight_min": -20.108123779296875,
      "activations/layer22_attention_weight_max": 29.771686553955078,
      "activations/layer22_attention_weight_min": -25.906049728393555,
      "activations/layer23_attention_weight_max": 36.83662414550781,
      "activations/layer23_attention_weight_min": -24.719091415405273,
      "activations/layer2_attention_weight_max": 31.38568687438965,
      "activations/layer2_attention_weight_min": -32.229732513427734,
      "activations/layer3_attention_weight_max": 87.72480010986328,
      "activations/layer3_attention_weight_min": -85.80509948730469,
      "activations/layer4_attention_weight_max": 87.43627166748047,
      "activations/layer4_attention_weight_min": -88.21790313720703,
      "activations/layer5_attention_weight_max": 62.33159637451172,
      "activations/layer5_attention_weight_min": -73.24134063720703,
      "activations/layer6_attention_weight_max": 48.25244903564453,
      "activations/layer6_attention_weight_min": -49.97618865966797,
      "activations/layer7_attention_weight_max": 60.77113342285156,
      "activations/layer7_attention_weight_min": -64.0223617553711,
      "activations/layer8_attention_weight_max": 45.556129455566406,
      "activations/layer8_attention_weight_min": -49.56987762451172,
      "activations/layer9_attention_weight_max": 42.23224639892578,
      "activations/layer9_attention_weight_min": -45.45619583129883,
      "epoch": 16.99,
      "learning_rate": 4.086666666666666e-05,
      "loss": 2.7369,
      "step": 292350
    },
    {
      "activations/layer0_attention_weight_max": 15.128763198852539,
      "activations/layer0_attention_weight_min": -13.745270729064941,
      "activations/layer10_attention_weight_max": 34.84718704223633,
      "activations/layer10_attention_weight_min": -36.33197784423828,
      "activations/layer11_attention_weight_max": 32.059722900390625,
      "activations/layer11_attention_weight_min": -33.63758087158203,
      "activations/layer12_attention_weight_max": 18.772079467773438,
      "activations/layer12_attention_weight_min": -25.56342315673828,
      "activations/layer13_attention_weight_max": 34.26031494140625,
      "activations/layer13_attention_weight_min": -27.6955623626709,
      "activations/layer14_attention_weight_max": 34.33757400512695,
      "activations/layer14_attention_weight_min": -29.26075553894043,
      "activations/layer15_attention_weight_max": 32.320648193359375,
      "activations/layer15_attention_weight_min": -29.425809860229492,
      "activations/layer16_attention_weight_max": 33.04767608642578,
      "activations/layer16_attention_weight_min": -32.70193862915039,
      "activations/layer17_attention_weight_max": 51.0838508605957,
      "activations/layer17_attention_weight_min": -45.373504638671875,
      "activations/layer18_attention_weight_max": 49.85740661621094,
      "activations/layer18_attention_weight_min": -40.81821823120117,
      "activations/layer19_attention_weight_max": 24.11081886291504,
      "activations/layer19_attention_weight_min": -23.210037231445312,
      "activations/layer1_attention_weight_max": 18.45937156677246,
      "activations/layer1_attention_weight_min": -15.245495796203613,
      "activations/layer20_attention_weight_max": 23.28096580505371,
      "activations/layer20_attention_weight_min": -21.525842666625977,
      "activations/layer21_attention_weight_max": 37.47456741333008,
      "activations/layer21_attention_weight_min": -22.111188888549805,
      "activations/layer22_attention_weight_max": 31.236665725708008,
      "activations/layer22_attention_weight_min": -27.26549530029297,
      "activations/layer23_attention_weight_max": 35.627159118652344,
      "activations/layer23_attention_weight_min": -25.863609313964844,
      "activations/layer2_attention_weight_max": 30.069700241088867,
      "activations/layer2_attention_weight_min": -28.446880340576172,
      "activations/layer3_attention_weight_max": 85.76226806640625,
      "activations/layer3_attention_weight_min": -89.4173583984375,
      "activations/layer4_attention_weight_max": 84.86763763427734,
      "activations/layer4_attention_weight_min": -86.32997131347656,
      "activations/layer5_attention_weight_max": 67.59526062011719,
      "activations/layer5_attention_weight_min": -73.54696655273438,
      "activations/layer6_attention_weight_max": 51.012577056884766,
      "activations/layer6_attention_weight_min": -54.18202209472656,
      "activations/layer7_attention_weight_max": 66.96355438232422,
      "activations/layer7_attention_weight_min": -66.9316177368164,
      "activations/layer8_attention_weight_max": 49.47638702392578,
      "activations/layer8_attention_weight_min": -51.01749801635742,
      "activations/layer9_attention_weight_max": 45.33584213256836,
      "activations/layer9_attention_weight_min": -49.1237678527832,
      "epoch": 16.99,
      "learning_rate": 4.084772727272727e-05,
      "loss": 2.7377,
      "step": 292400
    },
    {
      "activations/layer0_attention_weight_max": 15.77405834197998,
      "activations/layer0_attention_weight_min": -13.874399185180664,
      "activations/layer10_attention_weight_max": 34.87570571899414,
      "activations/layer10_attention_weight_min": -34.48988342285156,
      "activations/layer11_attention_weight_max": 32.00918197631836,
      "activations/layer11_attention_weight_min": -32.436561584472656,
      "activations/layer12_attention_weight_max": 18.799467086791992,
      "activations/layer12_attention_weight_min": -24.18735694885254,
      "activations/layer13_attention_weight_max": 32.47774887084961,
      "activations/layer13_attention_weight_min": -26.981149673461914,
      "activations/layer14_attention_weight_max": 36.616539001464844,
      "activations/layer14_attention_weight_min": -30.69856071472168,
      "activations/layer15_attention_weight_max": 34.83369445800781,
      "activations/layer15_attention_weight_min": -31.19155502319336,
      "activations/layer16_attention_weight_max": 33.94186019897461,
      "activations/layer16_attention_weight_min": -32.061012268066406,
      "activations/layer17_attention_weight_max": 51.753684997558594,
      "activations/layer17_attention_weight_min": -44.262027740478516,
      "activations/layer18_attention_weight_max": 46.49388122558594,
      "activations/layer18_attention_weight_min": -37.89147186279297,
      "activations/layer19_attention_weight_max": 24.35069465637207,
      "activations/layer19_attention_weight_min": -23.982173919677734,
      "activations/layer1_attention_weight_max": 15.283795356750488,
      "activations/layer1_attention_weight_min": -13.82420539855957,
      "activations/layer20_attention_weight_max": 22.034343719482422,
      "activations/layer20_attention_weight_min": -23.3791446685791,
      "activations/layer21_attention_weight_max": 35.36035919189453,
      "activations/layer21_attention_weight_min": -22.357637405395508,
      "activations/layer22_attention_weight_max": 28.313873291015625,
      "activations/layer22_attention_weight_min": -28.2583065032959,
      "activations/layer23_attention_weight_max": 38.473411560058594,
      "activations/layer23_attention_weight_min": -24.77420425415039,
      "activations/layer2_attention_weight_max": 28.982345581054688,
      "activations/layer2_attention_weight_min": -26.818218231201172,
      "activations/layer3_attention_weight_max": 80.78018188476562,
      "activations/layer3_attention_weight_min": -78.47760009765625,
      "activations/layer4_attention_weight_max": 79.2024154663086,
      "activations/layer4_attention_weight_min": -79.68326568603516,
      "activations/layer5_attention_weight_max": 62.161705017089844,
      "activations/layer5_attention_weight_min": -69.77703857421875,
      "activations/layer6_attention_weight_max": 47.684322357177734,
      "activations/layer6_attention_weight_min": -48.00505447387695,
      "activations/layer7_attention_weight_max": 65.42314147949219,
      "activations/layer7_attention_weight_min": -62.060672760009766,
      "activations/layer8_attention_weight_max": 45.15972900390625,
      "activations/layer8_attention_weight_min": -50.84001541137695,
      "activations/layer9_attention_weight_max": 43.8819694519043,
      "activations/layer9_attention_weight_min": -45.929443359375,
      "epoch": 16.99,
      "learning_rate": 4.0828787878787876e-05,
      "loss": 2.7417,
      "step": 292450
    },
    {
      "activations/layer0_attention_weight_max": 15.276631355285645,
      "activations/layer0_attention_weight_min": -13.63461685180664,
      "activations/layer10_attention_weight_max": 36.35498046875,
      "activations/layer10_attention_weight_min": -36.44827651977539,
      "activations/layer11_attention_weight_max": 33.544273376464844,
      "activations/layer11_attention_weight_min": -33.95225524902344,
      "activations/layer12_attention_weight_max": 18.4473934173584,
      "activations/layer12_attention_weight_min": -27.89824104309082,
      "activations/layer13_attention_weight_max": 31.46095848083496,
      "activations/layer13_attention_weight_min": -29.31894874572754,
      "activations/layer14_attention_weight_max": 36.90032196044922,
      "activations/layer14_attention_weight_min": -30.47573471069336,
      "activations/layer15_attention_weight_max": 32.68191909790039,
      "activations/layer15_attention_weight_min": -29.3736515045166,
      "activations/layer16_attention_weight_max": 33.83968734741211,
      "activations/layer16_attention_weight_min": -30.6236572265625,
      "activations/layer17_attention_weight_max": 48.31386947631836,
      "activations/layer17_attention_weight_min": -44.44252014160156,
      "activations/layer18_attention_weight_max": 44.94853973388672,
      "activations/layer18_attention_weight_min": -38.58334732055664,
      "activations/layer19_attention_weight_max": 24.137985229492188,
      "activations/layer19_attention_weight_min": -22.03262710571289,
      "activations/layer1_attention_weight_max": 16.690547943115234,
      "activations/layer1_attention_weight_min": -14.539435386657715,
      "activations/layer20_attention_weight_max": 23.56473159790039,
      "activations/layer20_attention_weight_min": -22.313493728637695,
      "activations/layer21_attention_weight_max": 39.33024978637695,
      "activations/layer21_attention_weight_min": -24.655691146850586,
      "activations/layer22_attention_weight_max": 31.22992515563965,
      "activations/layer22_attention_weight_min": -25.00629234313965,
      "activations/layer23_attention_weight_max": 35.65557098388672,
      "activations/layer23_attention_weight_min": -24.47848892211914,
      "activations/layer2_attention_weight_max": 31.67140769958496,
      "activations/layer2_attention_weight_min": -33.02018356323242,
      "activations/layer3_attention_weight_max": 91.38258361816406,
      "activations/layer3_attention_weight_min": -94.61888885498047,
      "activations/layer4_attention_weight_max": 88.31094360351562,
      "activations/layer4_attention_weight_min": -86.41573333740234,
      "activations/layer5_attention_weight_max": 63.35590744018555,
      "activations/layer5_attention_weight_min": -68.40243530273438,
      "activations/layer6_attention_weight_max": 49.95432662963867,
      "activations/layer6_attention_weight_min": -50.463687896728516,
      "activations/layer7_attention_weight_max": 64.40933227539062,
      "activations/layer7_attention_weight_min": -64.84324645996094,
      "activations/layer8_attention_weight_max": 46.737586975097656,
      "activations/layer8_attention_weight_min": -48.548187255859375,
      "activations/layer9_attention_weight_max": 45.22713851928711,
      "activations/layer9_attention_weight_min": -45.50637435913086,
      "epoch": 17.0,
      "learning_rate": 4.0809848484848484e-05,
      "loss": 2.756,
      "step": 292500
    },
    {
      "activations/layer0_attention_weight_max": 15.597079277038574,
      "activations/layer0_attention_weight_min": -13.956779479980469,
      "activations/layer10_attention_weight_max": 35.86124038696289,
      "activations/layer10_attention_weight_min": -36.58017349243164,
      "activations/layer11_attention_weight_max": 33.92131805419922,
      "activations/layer11_attention_weight_min": -34.29931640625,
      "activations/layer12_attention_weight_max": 19.262969970703125,
      "activations/layer12_attention_weight_min": -28.93900489807129,
      "activations/layer13_attention_weight_max": 32.690650939941406,
      "activations/layer13_attention_weight_min": -27.56171417236328,
      "activations/layer14_attention_weight_max": 34.30490493774414,
      "activations/layer14_attention_weight_min": -32.07622528076172,
      "activations/layer15_attention_weight_max": 31.891111373901367,
      "activations/layer15_attention_weight_min": -32.044700622558594,
      "activations/layer16_attention_weight_max": 34.08671188354492,
      "activations/layer16_attention_weight_min": -31.399856567382812,
      "activations/layer17_attention_weight_max": 51.92376708984375,
      "activations/layer17_attention_weight_min": -46.4284553527832,
      "activations/layer18_attention_weight_max": 51.09667205810547,
      "activations/layer18_attention_weight_min": -40.1311149597168,
      "activations/layer19_attention_weight_max": 24.14375877380371,
      "activations/layer19_attention_weight_min": -22.694570541381836,
      "activations/layer1_attention_weight_max": 16.231891632080078,
      "activations/layer1_attention_weight_min": -16.723602294921875,
      "activations/layer20_attention_weight_max": 22.57564353942871,
      "activations/layer20_attention_weight_min": -21.976058959960938,
      "activations/layer21_attention_weight_max": 37.182945251464844,
      "activations/layer21_attention_weight_min": -23.11299705505371,
      "activations/layer22_attention_weight_max": 31.78702163696289,
      "activations/layer22_attention_weight_min": -27.262088775634766,
      "activations/layer23_attention_weight_max": 34.93946075439453,
      "activations/layer23_attention_weight_min": -25.570560455322266,
      "activations/layer2_attention_weight_max": 30.981863021850586,
      "activations/layer2_attention_weight_min": -29.358121871948242,
      "activations/layer3_attention_weight_max": 87.96055603027344,
      "activations/layer3_attention_weight_min": -86.51042175292969,
      "activations/layer4_attention_weight_max": 88.3507308959961,
      "activations/layer4_attention_weight_min": -82.40176391601562,
      "activations/layer5_attention_weight_max": 64.99928283691406,
      "activations/layer5_attention_weight_min": -75.7188491821289,
      "activations/layer6_attention_weight_max": 48.47343826293945,
      "activations/layer6_attention_weight_min": -49.14775466918945,
      "activations/layer7_attention_weight_max": 67.70563507080078,
      "activations/layer7_attention_weight_min": -65.64512634277344,
      "activations/layer8_attention_weight_max": 48.12057876586914,
      "activations/layer8_attention_weight_min": -49.16008758544922,
      "activations/layer9_attention_weight_max": 44.14873123168945,
      "activations/layer9_attention_weight_min": -46.93959045410156,
      "epoch": 17.0,
      "learning_rate": 4.0790909090909086e-05,
      "loss": 2.7433,
      "step": 292550
    },
    {
      "activations/layer0_attention_weight_max": 14.674674034118652,
      "activations/layer0_attention_weight_min": -13.689096450805664,
      "activations/layer10_attention_weight_max": 38.3002815246582,
      "activations/layer10_attention_weight_min": -39.51423645019531,
      "activations/layer11_attention_weight_max": 35.78647232055664,
      "activations/layer11_attention_weight_min": -35.43447494506836,
      "activations/layer12_attention_weight_max": 20.48207664489746,
      "activations/layer12_attention_weight_min": -25.184547424316406,
      "activations/layer13_attention_weight_max": 31.450153350830078,
      "activations/layer13_attention_weight_min": -26.66287612915039,
      "activations/layer14_attention_weight_max": 33.88664245605469,
      "activations/layer14_attention_weight_min": -29.16974639892578,
      "activations/layer15_attention_weight_max": 32.79252243041992,
      "activations/layer15_attention_weight_min": -29.171016693115234,
      "activations/layer16_attention_weight_max": 33.27338409423828,
      "activations/layer16_attention_weight_min": -30.820039749145508,
      "activations/layer17_attention_weight_max": 52.78831100463867,
      "activations/layer17_attention_weight_min": -41.43070983886719,
      "activations/layer18_attention_weight_max": 49.20456314086914,
      "activations/layer18_attention_weight_min": -38.53879928588867,
      "activations/layer19_attention_weight_max": 24.718345642089844,
      "activations/layer19_attention_weight_min": -23.300310134887695,
      "activations/layer1_attention_weight_max": 16.700761795043945,
      "activations/layer1_attention_weight_min": -14.446693420410156,
      "activations/layer20_attention_weight_max": 20.94717025756836,
      "activations/layer20_attention_weight_min": -23.03636932373047,
      "activations/layer21_attention_weight_max": 35.377498626708984,
      "activations/layer21_attention_weight_min": -20.280336380004883,
      "activations/layer22_attention_weight_max": 26.79693031311035,
      "activations/layer22_attention_weight_min": -24.77447509765625,
      "activations/layer23_attention_weight_max": 34.927616119384766,
      "activations/layer23_attention_weight_min": -23.849010467529297,
      "activations/layer2_attention_weight_max": 32.12935256958008,
      "activations/layer2_attention_weight_min": -29.697765350341797,
      "activations/layer3_attention_weight_max": 92.66278076171875,
      "activations/layer3_attention_weight_min": -88.10480499267578,
      "activations/layer4_attention_weight_max": 89.49722290039062,
      "activations/layer4_attention_weight_min": -84.86481475830078,
      "activations/layer5_attention_weight_max": 61.231834411621094,
      "activations/layer5_attention_weight_min": -72.98062133789062,
      "activations/layer6_attention_weight_max": 49.71982955932617,
      "activations/layer6_attention_weight_min": -52.405399322509766,
      "activations/layer7_attention_weight_max": 61.091064453125,
      "activations/layer7_attention_weight_min": -63.70880126953125,
      "activations/layer8_attention_weight_max": 46.18442916870117,
      "activations/layer8_attention_weight_min": -50.89485549926758,
      "activations/layer9_attention_weight_max": 44.44205093383789,
      "activations/layer9_attention_weight_min": -50.73200988769531,
      "epoch": 17.0,
      "learning_rate": 4.077159090909091e-05,
      "loss": 2.7899,
      "step": 292600
    },
    {
      "activations/layer0_attention_weight_max": 15.482994079589844,
      "activations/layer0_attention_weight_min": -14.024155616760254,
      "activations/layer10_attention_weight_max": 36.54140090942383,
      "activations/layer10_attention_weight_min": -36.726287841796875,
      "activations/layer11_attention_weight_max": 32.59563446044922,
      "activations/layer11_attention_weight_min": -32.04106521606445,
      "activations/layer12_attention_weight_max": 16.745107650756836,
      "activations/layer12_attention_weight_min": -26.24915885925293,
      "activations/layer13_attention_weight_max": 32.799617767333984,
      "activations/layer13_attention_weight_min": -30.04791259765625,
      "activations/layer14_attention_weight_max": 39.63496780395508,
      "activations/layer14_attention_weight_min": -28.46082305908203,
      "activations/layer15_attention_weight_max": 34.14433670043945,
      "activations/layer15_attention_weight_min": -28.764616012573242,
      "activations/layer16_attention_weight_max": 30.460079193115234,
      "activations/layer16_attention_weight_min": -31.019453048706055,
      "activations/layer17_attention_weight_max": 51.3869514465332,
      "activations/layer17_attention_weight_min": -47.27588653564453,
      "activations/layer18_attention_weight_max": 46.405216217041016,
      "activations/layer18_attention_weight_min": -41.783233642578125,
      "activations/layer19_attention_weight_max": 26.05373191833496,
      "activations/layer19_attention_weight_min": -24.656124114990234,
      "activations/layer1_attention_weight_max": 17.317279815673828,
      "activations/layer1_attention_weight_min": -14.109854698181152,
      "activations/layer20_attention_weight_max": 25.906782150268555,
      "activations/layer20_attention_weight_min": -23.326570510864258,
      "activations/layer21_attention_weight_max": 38.53591537475586,
      "activations/layer21_attention_weight_min": -23.009756088256836,
      "activations/layer22_attention_weight_max": 29.791074752807617,
      "activations/layer22_attention_weight_min": -28.374650955200195,
      "activations/layer23_attention_weight_max": 37.00872802734375,
      "activations/layer23_attention_weight_min": -24.87653350830078,
      "activations/layer2_attention_weight_max": 31.5570125579834,
      "activations/layer2_attention_weight_min": -30.234031677246094,
      "activations/layer3_attention_weight_max": 84.0590591430664,
      "activations/layer3_attention_weight_min": -86.75308990478516,
      "activations/layer4_attention_weight_max": 83.49495697021484,
      "activations/layer4_attention_weight_min": -84.25853729248047,
      "activations/layer5_attention_weight_max": 65.05703735351562,
      "activations/layer5_attention_weight_min": -69.5467529296875,
      "activations/layer6_attention_weight_max": 53.12007141113281,
      "activations/layer6_attention_weight_min": -50.445945739746094,
      "activations/layer7_attention_weight_max": 65.29927825927734,
      "activations/layer7_attention_weight_min": -64.34566497802734,
      "activations/layer8_attention_weight_max": 49.971736907958984,
      "activations/layer8_attention_weight_min": -50.3183479309082,
      "activations/layer9_attention_weight_max": 50.30354309082031,
      "activations/layer9_attention_weight_min": -44.57413864135742,
      "epoch": 17.0,
      "learning_rate": 4.075265151515151e-05,
      "loss": 2.7185,
      "step": 292650
    },
    {
      "activations/layer0_attention_weight_max": 14.895015716552734,
      "activations/layer0_attention_weight_min": -13.958724021911621,
      "activations/layer10_attention_weight_max": 33.24760437011719,
      "activations/layer10_attention_weight_min": -34.45637893676758,
      "activations/layer11_attention_weight_max": 31.742002487182617,
      "activations/layer11_attention_weight_min": -32.06391143798828,
      "activations/layer12_attention_weight_max": 19.10190200805664,
      "activations/layer12_attention_weight_min": -30.29254913330078,
      "activations/layer13_attention_weight_max": 34.19931411743164,
      "activations/layer13_attention_weight_min": -26.004112243652344,
      "activations/layer14_attention_weight_max": 35.3157844543457,
      "activations/layer14_attention_weight_min": -30.081697463989258,
      "activations/layer15_attention_weight_max": 33.18331527709961,
      "activations/layer15_attention_weight_min": -30.228500366210938,
      "activations/layer16_attention_weight_max": 32.754730224609375,
      "activations/layer16_attention_weight_min": -30.66753387451172,
      "activations/layer17_attention_weight_max": 48.58952713012695,
      "activations/layer17_attention_weight_min": -46.17170333862305,
      "activations/layer18_attention_weight_max": 47.1086540222168,
      "activations/layer18_attention_weight_min": -39.78168487548828,
      "activations/layer19_attention_weight_max": 24.542057037353516,
      "activations/layer19_attention_weight_min": -23.904085159301758,
      "activations/layer1_attention_weight_max": 18.04986572265625,
      "activations/layer1_attention_weight_min": -14.640509605407715,
      "activations/layer20_attention_weight_max": 22.932476043701172,
      "activations/layer20_attention_weight_min": -21.86546516418457,
      "activations/layer21_attention_weight_max": 38.605133056640625,
      "activations/layer21_attention_weight_min": -22.11899757385254,
      "activations/layer22_attention_weight_max": 31.067224502563477,
      "activations/layer22_attention_weight_min": -25.93707847595215,
      "activations/layer23_attention_weight_max": 36.79754638671875,
      "activations/layer23_attention_weight_min": -23.686107635498047,
      "activations/layer2_attention_weight_max": 30.225444793701172,
      "activations/layer2_attention_weight_min": -30.16817855834961,
      "activations/layer3_attention_weight_max": 88.66655731201172,
      "activations/layer3_attention_weight_min": -91.39285278320312,
      "activations/layer4_attention_weight_max": 85.81796264648438,
      "activations/layer4_attention_weight_min": -87.01371002197266,
      "activations/layer5_attention_weight_max": 62.97943878173828,
      "activations/layer5_attention_weight_min": -77.29953002929688,
      "activations/layer6_attention_weight_max": 49.928062438964844,
      "activations/layer6_attention_weight_min": -49.12003707885742,
      "activations/layer7_attention_weight_max": 63.21459197998047,
      "activations/layer7_attention_weight_min": -65.29621124267578,
      "activations/layer8_attention_weight_max": 45.163578033447266,
      "activations/layer8_attention_weight_min": -48.07878494262695,
      "activations/layer9_attention_weight_max": 41.644073486328125,
      "activations/layer9_attention_weight_min": -46.0297966003418,
      "epoch": 17.01,
      "learning_rate": 4.0733712121212114e-05,
      "loss": 2.7422,
      "step": 292700
    },
    {
      "activations/layer0_attention_weight_max": 15.32419204711914,
      "activations/layer0_attention_weight_min": -13.57440185546875,
      "activations/layer10_attention_weight_max": 34.116329193115234,
      "activations/layer10_attention_weight_min": -35.610870361328125,
      "activations/layer11_attention_weight_max": 31.749561309814453,
      "activations/layer11_attention_weight_min": -35.86284637451172,
      "activations/layer12_attention_weight_max": 19.174654006958008,
      "activations/layer12_attention_weight_min": -27.580957412719727,
      "activations/layer13_attention_weight_max": 31.214855194091797,
      "activations/layer13_attention_weight_min": -27.546281814575195,
      "activations/layer14_attention_weight_max": 33.09159851074219,
      "activations/layer14_attention_weight_min": -33.2602653503418,
      "activations/layer15_attention_weight_max": 35.11430358886719,
      "activations/layer15_attention_weight_min": -31.726612091064453,
      "activations/layer16_attention_weight_max": 32.986263275146484,
      "activations/layer16_attention_weight_min": -32.73038101196289,
      "activations/layer17_attention_weight_max": 51.47015380859375,
      "activations/layer17_attention_weight_min": -46.05659866333008,
      "activations/layer18_attention_weight_max": 47.137908935546875,
      "activations/layer18_attention_weight_min": -42.627708435058594,
      "activations/layer19_attention_weight_max": 23.792377471923828,
      "activations/layer19_attention_weight_min": -24.50551414489746,
      "activations/layer1_attention_weight_max": 15.456141471862793,
      "activations/layer1_attention_weight_min": -15.116665840148926,
      "activations/layer20_attention_weight_max": 22.716012954711914,
      "activations/layer20_attention_weight_min": -23.048559188842773,
      "activations/layer21_attention_weight_max": 37.75297164916992,
      "activations/layer21_attention_weight_min": -21.637409210205078,
      "activations/layer22_attention_weight_max": 30.06711769104004,
      "activations/layer22_attention_weight_min": -26.321401596069336,
      "activations/layer23_attention_weight_max": 39.78314208984375,
      "activations/layer23_attention_weight_min": -25.032194137573242,
      "activations/layer2_attention_weight_max": 30.215452194213867,
      "activations/layer2_attention_weight_min": -30.611263275146484,
      "activations/layer3_attention_weight_max": 86.10684204101562,
      "activations/layer3_attention_weight_min": -92.19054412841797,
      "activations/layer4_attention_weight_max": 85.06060791015625,
      "activations/layer4_attention_weight_min": -84.24054718017578,
      "activations/layer5_attention_weight_max": 65.52130126953125,
      "activations/layer5_attention_weight_min": -71.84364318847656,
      "activations/layer6_attention_weight_max": 50.97444152832031,
      "activations/layer6_attention_weight_min": -51.480186462402344,
      "activations/layer7_attention_weight_max": 71.99516296386719,
      "activations/layer7_attention_weight_min": -63.05837631225586,
      "activations/layer8_attention_weight_max": 44.03628921508789,
      "activations/layer8_attention_weight_min": -48.35683822631836,
      "activations/layer9_attention_weight_max": 44.309593200683594,
      "activations/layer9_attention_weight_min": -45.37854766845703,
      "epoch": 17.01,
      "learning_rate": 4.071477272727272e-05,
      "loss": 2.7485,
      "step": 292750
    },
    {
      "activations/layer0_attention_weight_max": 15.178316116333008,
      "activations/layer0_attention_weight_min": -13.742964744567871,
      "activations/layer10_attention_weight_max": 38.10848617553711,
      "activations/layer10_attention_weight_min": -38.44694137573242,
      "activations/layer11_attention_weight_max": 33.98656463623047,
      "activations/layer11_attention_weight_min": -35.182952880859375,
      "activations/layer12_attention_weight_max": 21.284229278564453,
      "activations/layer12_attention_weight_min": -23.508146286010742,
      "activations/layer13_attention_weight_max": 34.82182312011719,
      "activations/layer13_attention_weight_min": -27.903491973876953,
      "activations/layer14_attention_weight_max": 35.277671813964844,
      "activations/layer14_attention_weight_min": -31.85079574584961,
      "activations/layer15_attention_weight_max": 34.00957489013672,
      "activations/layer15_attention_weight_min": -30.56533432006836,
      "activations/layer16_attention_weight_max": 34.51772689819336,
      "activations/layer16_attention_weight_min": -32.20262908935547,
      "activations/layer17_attention_weight_max": 53.959228515625,
      "activations/layer17_attention_weight_min": -46.717472076416016,
      "activations/layer18_attention_weight_max": 51.68526077270508,
      "activations/layer18_attention_weight_min": -41.12952423095703,
      "activations/layer19_attention_weight_max": 26.21632194519043,
      "activations/layer19_attention_weight_min": -24.687515258789062,
      "activations/layer1_attention_weight_max": 15.216286659240723,
      "activations/layer1_attention_weight_min": -12.228303909301758,
      "activations/layer20_attention_weight_max": 26.30247688293457,
      "activations/layer20_attention_weight_min": -21.028533935546875,
      "activations/layer21_attention_weight_max": 42.73621368408203,
      "activations/layer21_attention_weight_min": -22.43468475341797,
      "activations/layer22_attention_weight_max": 33.22730255126953,
      "activations/layer22_attention_weight_min": -25.48996925354004,
      "activations/layer23_attention_weight_max": 38.746437072753906,
      "activations/layer23_attention_weight_min": -23.50636863708496,
      "activations/layer2_attention_weight_max": 30.288419723510742,
      "activations/layer2_attention_weight_min": -29.777530670166016,
      "activations/layer3_attention_weight_max": 85.95696258544922,
      "activations/layer3_attention_weight_min": -85.74952697753906,
      "activations/layer4_attention_weight_max": 90.0296401977539,
      "activations/layer4_attention_weight_min": -87.78645324707031,
      "activations/layer5_attention_weight_max": 61.98566818237305,
      "activations/layer5_attention_weight_min": -73.58934020996094,
      "activations/layer6_attention_weight_max": 51.404720306396484,
      "activations/layer6_attention_weight_min": -52.17115020751953,
      "activations/layer7_attention_weight_max": 66.68919372558594,
      "activations/layer7_attention_weight_min": -67.5452651977539,
      "activations/layer8_attention_weight_max": 44.28630447387695,
      "activations/layer8_attention_weight_min": -48.05656433105469,
      "activations/layer9_attention_weight_max": 47.863773345947266,
      "activations/layer9_attention_weight_min": -46.90926742553711,
      "epoch": 17.01,
      "learning_rate": 4.069583333333333e-05,
      "loss": 2.7311,
      "step": 292800
    },
    {
      "activations/layer0_attention_weight_max": 14.813860893249512,
      "activations/layer0_attention_weight_min": -14.37595272064209,
      "activations/layer10_attention_weight_max": 33.65373992919922,
      "activations/layer10_attention_weight_min": -33.216087341308594,
      "activations/layer11_attention_weight_max": 33.115394592285156,
      "activations/layer11_attention_weight_min": -32.459991455078125,
      "activations/layer12_attention_weight_max": 18.648059844970703,
      "activations/layer12_attention_weight_min": -23.405969619750977,
      "activations/layer13_attention_weight_max": 33.18581771850586,
      "activations/layer13_attention_weight_min": -31.79852867126465,
      "activations/layer14_attention_weight_max": 32.484683990478516,
      "activations/layer14_attention_weight_min": -28.45684814453125,
      "activations/layer15_attention_weight_max": 32.303768157958984,
      "activations/layer15_attention_weight_min": -29.00324249267578,
      "activations/layer16_attention_weight_max": 33.023292541503906,
      "activations/layer16_attention_weight_min": -32.653358459472656,
      "activations/layer17_attention_weight_max": 48.68637466430664,
      "activations/layer17_attention_weight_min": -43.040504455566406,
      "activations/layer18_attention_weight_max": 43.998783111572266,
      "activations/layer18_attention_weight_min": -39.68981170654297,
      "activations/layer19_attention_weight_max": 21.842037200927734,
      "activations/layer19_attention_weight_min": -20.012935638427734,
      "activations/layer1_attention_weight_max": 16.34527587890625,
      "activations/layer1_attention_weight_min": -14.006743431091309,
      "activations/layer20_attention_weight_max": 20.32594871520996,
      "activations/layer20_attention_weight_min": -22.882266998291016,
      "activations/layer21_attention_weight_max": 29.41697883605957,
      "activations/layer21_attention_weight_min": -24.3765869140625,
      "activations/layer22_attention_weight_max": 29.118770599365234,
      "activations/layer22_attention_weight_min": -23.310958862304688,
      "activations/layer23_attention_weight_max": 34.51332092285156,
      "activations/layer23_attention_weight_min": -23.862533569335938,
      "activations/layer2_attention_weight_max": 30.545568466186523,
      "activations/layer2_attention_weight_min": -30.478893280029297,
      "activations/layer3_attention_weight_max": 87.91947937011719,
      "activations/layer3_attention_weight_min": -89.70589447021484,
      "activations/layer4_attention_weight_max": 86.00093078613281,
      "activations/layer4_attention_weight_min": -84.50885009765625,
      "activations/layer5_attention_weight_max": 62.07635498046875,
      "activations/layer5_attention_weight_min": -73.74785614013672,
      "activations/layer6_attention_weight_max": 47.74109649658203,
      "activations/layer6_attention_weight_min": -47.752689361572266,
      "activations/layer7_attention_weight_max": 66.0955810546875,
      "activations/layer7_attention_weight_min": -63.36603546142578,
      "activations/layer8_attention_weight_max": 44.13499069213867,
      "activations/layer8_attention_weight_min": -46.70883560180664,
      "activations/layer9_attention_weight_max": 46.4487419128418,
      "activations/layer9_attention_weight_min": -47.57867431640625,
      "epoch": 17.02,
      "learning_rate": 4.067689393939394e-05,
      "loss": 2.7261,
      "step": 292850
    },
    {
      "activations/layer0_attention_weight_max": 15.522205352783203,
      "activations/layer0_attention_weight_min": -13.881438255310059,
      "activations/layer10_attention_weight_max": 37.04379653930664,
      "activations/layer10_attention_weight_min": -36.56446075439453,
      "activations/layer11_attention_weight_max": 33.07908630371094,
      "activations/layer11_attention_weight_min": -34.06257629394531,
      "activations/layer12_attention_weight_max": 19.854507446289062,
      "activations/layer12_attention_weight_min": -26.790157318115234,
      "activations/layer13_attention_weight_max": 32.727725982666016,
      "activations/layer13_attention_weight_min": -27.259023666381836,
      "activations/layer14_attention_weight_max": 36.05168533325195,
      "activations/layer14_attention_weight_min": -29.72372055053711,
      "activations/layer15_attention_weight_max": 32.49909591674805,
      "activations/layer15_attention_weight_min": -29.723939895629883,
      "activations/layer16_attention_weight_max": 35.19986343383789,
      "activations/layer16_attention_weight_min": -35.11388397216797,
      "activations/layer17_attention_weight_max": 50.5327033996582,
      "activations/layer17_attention_weight_min": -49.30500030517578,
      "activations/layer18_attention_weight_max": 47.52643966674805,
      "activations/layer18_attention_weight_min": -43.82695388793945,
      "activations/layer19_attention_weight_max": 22.741825103759766,
      "activations/layer19_attention_weight_min": -25.097366333007812,
      "activations/layer1_attention_weight_max": 16.369077682495117,
      "activations/layer1_attention_weight_min": -14.285861015319824,
      "activations/layer20_attention_weight_max": 20.904447555541992,
      "activations/layer20_attention_weight_min": -20.93735122680664,
      "activations/layer21_attention_weight_max": 36.69293975830078,
      "activations/layer21_attention_weight_min": -24.810134887695312,
      "activations/layer22_attention_weight_max": 30.72947120666504,
      "activations/layer22_attention_weight_min": -26.152265548706055,
      "activations/layer23_attention_weight_max": 37.1512336730957,
      "activations/layer23_attention_weight_min": -23.418838500976562,
      "activations/layer2_attention_weight_max": 33.80199432373047,
      "activations/layer2_attention_weight_min": -30.787738800048828,
      "activations/layer3_attention_weight_max": 90.60438537597656,
      "activations/layer3_attention_weight_min": -91.25758361816406,
      "activations/layer4_attention_weight_max": 89.67985534667969,
      "activations/layer4_attention_weight_min": -88.46771240234375,
      "activations/layer5_attention_weight_max": 60.00594711303711,
      "activations/layer5_attention_weight_min": -71.11834716796875,
      "activations/layer6_attention_weight_max": 48.11278533935547,
      "activations/layer6_attention_weight_min": -48.084659576416016,
      "activations/layer7_attention_weight_max": 62.61865234375,
      "activations/layer7_attention_weight_min": -62.97944641113281,
      "activations/layer8_attention_weight_max": 44.57929992675781,
      "activations/layer8_attention_weight_min": -46.577308654785156,
      "activations/layer9_attention_weight_max": 46.77970886230469,
      "activations/layer9_attention_weight_min": -47.732635498046875,
      "epoch": 17.02,
      "learning_rate": 4.065795454545454e-05,
      "loss": 2.7307,
      "step": 292900
    },
    {
      "activations/layer0_attention_weight_max": 16.293380737304688,
      "activations/layer0_attention_weight_min": -13.513296127319336,
      "activations/layer10_attention_weight_max": 37.48052978515625,
      "activations/layer10_attention_weight_min": -38.84902572631836,
      "activations/layer11_attention_weight_max": 32.44388198852539,
      "activations/layer11_attention_weight_min": -33.829742431640625,
      "activations/layer12_attention_weight_max": 19.041400909423828,
      "activations/layer12_attention_weight_min": -27.94655990600586,
      "activations/layer13_attention_weight_max": 31.902673721313477,
      "activations/layer13_attention_weight_min": -27.422733306884766,
      "activations/layer14_attention_weight_max": 32.531917572021484,
      "activations/layer14_attention_weight_min": -29.84343147277832,
      "activations/layer15_attention_weight_max": 31.85666847229004,
      "activations/layer15_attention_weight_min": -29.636417388916016,
      "activations/layer16_attention_weight_max": 32.482398986816406,
      "activations/layer16_attention_weight_min": -31.496061325073242,
      "activations/layer17_attention_weight_max": 45.65907669067383,
      "activations/layer17_attention_weight_min": -41.928138732910156,
      "activations/layer18_attention_weight_max": 46.21092987060547,
      "activations/layer18_attention_weight_min": -37.905982971191406,
      "activations/layer19_attention_weight_max": 23.933135986328125,
      "activations/layer19_attention_weight_min": -22.165279388427734,
      "activations/layer1_attention_weight_max": 16.620718002319336,
      "activations/layer1_attention_weight_min": -15.334315299987793,
      "activations/layer20_attention_weight_max": 22.01459312438965,
      "activations/layer20_attention_weight_min": -23.694141387939453,
      "activations/layer21_attention_weight_max": 35.16658401489258,
      "activations/layer21_attention_weight_min": -22.811006546020508,
      "activations/layer22_attention_weight_max": 28.85025405883789,
      "activations/layer22_attention_weight_min": -25.136892318725586,
      "activations/layer23_attention_weight_max": 34.991851806640625,
      "activations/layer23_attention_weight_min": -24.798080444335938,
      "activations/layer2_attention_weight_max": 32.95653533935547,
      "activations/layer2_attention_weight_min": -33.065696716308594,
      "activations/layer3_attention_weight_max": 94.97715759277344,
      "activations/layer3_attention_weight_min": -92.91126251220703,
      "activations/layer4_attention_weight_max": 94.64865112304688,
      "activations/layer4_attention_weight_min": -89.92374420166016,
      "activations/layer5_attention_weight_max": 69.46733093261719,
      "activations/layer5_attention_weight_min": -72.7121810913086,
      "activations/layer6_attention_weight_max": 53.10154724121094,
      "activations/layer6_attention_weight_min": -51.30630111694336,
      "activations/layer7_attention_weight_max": 65.5989990234375,
      "activations/layer7_attention_weight_min": -65.22119140625,
      "activations/layer8_attention_weight_max": 46.2317008972168,
      "activations/layer8_attention_weight_min": -49.012901306152344,
      "activations/layer9_attention_weight_max": 44.73637390136719,
      "activations/layer9_attention_weight_min": -43.89204025268555,
      "epoch": 17.02,
      "learning_rate": 4.063901515151514e-05,
      "loss": 2.7403,
      "step": 292950
    },
    {
      "activations/layer0_attention_weight_max": 17.27782440185547,
      "activations/layer0_attention_weight_min": -13.550104141235352,
      "activations/layer10_attention_weight_max": 36.02049255371094,
      "activations/layer10_attention_weight_min": -35.620826721191406,
      "activations/layer11_attention_weight_max": 31.245136260986328,
      "activations/layer11_attention_weight_min": -33.390533447265625,
      "activations/layer12_attention_weight_max": 19.036334991455078,
      "activations/layer12_attention_weight_min": -26.58736801147461,
      "activations/layer13_attention_weight_max": 32.733455657958984,
      "activations/layer13_attention_weight_min": -28.964473724365234,
      "activations/layer14_attention_weight_max": 37.656429290771484,
      "activations/layer14_attention_weight_min": -34.98141098022461,
      "activations/layer15_attention_weight_max": 32.40826416015625,
      "activations/layer15_attention_weight_min": -33.5494270324707,
      "activations/layer16_attention_weight_max": 34.151885986328125,
      "activations/layer16_attention_weight_min": -31.37311553955078,
      "activations/layer17_attention_weight_max": 52.49226760864258,
      "activations/layer17_attention_weight_min": -45.260494232177734,
      "activations/layer18_attention_weight_max": 47.26645278930664,
      "activations/layer18_attention_weight_min": -39.0077018737793,
      "activations/layer19_attention_weight_max": 22.715198516845703,
      "activations/layer19_attention_weight_min": -24.5987606048584,
      "activations/layer1_attention_weight_max": 16.613187789916992,
      "activations/layer1_attention_weight_min": -14.042099952697754,
      "activations/layer20_attention_weight_max": 20.374874114990234,
      "activations/layer20_attention_weight_min": -23.186328887939453,
      "activations/layer21_attention_weight_max": 35.62113571166992,
      "activations/layer21_attention_weight_min": -21.626806259155273,
      "activations/layer22_attention_weight_max": 29.26723289489746,
      "activations/layer22_attention_weight_min": -29.47159194946289,
      "activations/layer23_attention_weight_max": 34.757354736328125,
      "activations/layer23_attention_weight_min": -23.311481475830078,
      "activations/layer2_attention_weight_max": 30.123428344726562,
      "activations/layer2_attention_weight_min": -30.572097778320312,
      "activations/layer3_attention_weight_max": 85.6019287109375,
      "activations/layer3_attention_weight_min": -90.3592529296875,
      "activations/layer4_attention_weight_max": 88.43868255615234,
      "activations/layer4_attention_weight_min": -84.35592651367188,
      "activations/layer5_attention_weight_max": 62.549232482910156,
      "activations/layer5_attention_weight_min": -68.649658203125,
      "activations/layer6_attention_weight_max": 50.986446380615234,
      "activations/layer6_attention_weight_min": -52.54126739501953,
      "activations/layer7_attention_weight_max": 64.52705383300781,
      "activations/layer7_attention_weight_min": -66.30789184570312,
      "activations/layer8_attention_weight_max": 47.7744140625,
      "activations/layer8_attention_weight_min": -51.32506561279297,
      "activations/layer9_attention_weight_max": 44.52069854736328,
      "activations/layer9_attention_weight_min": -45.28947067260742,
      "epoch": 17.02,
      "learning_rate": 4.062007575757576e-05,
      "loss": 2.7372,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4805,
      "eval_samples_per_second": 506.338,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4805,
      "eval_openwebtext_samples_per_second": 506.338,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9771,
      "eval_wikitext_samples_per_second": 230.644,
      "step": 293000
    },
    {
      "epoch": 17.02,
      "eval_lambada_loss": 2.5390625,
      "eval_lambada_ppl": 12.667789349678872,
      "eval_lambada_runtime": 9.5842,
      "eval_lambada_samples_per_second": 508.024,
      "step": 293000
    },
    {
      "activations/layer0_attention_weight_max": 15.281632423400879,
      "activations/layer0_attention_weight_min": -13.72671890258789,
      "activations/layer10_attention_weight_max": 32.4979362487793,
      "activations/layer10_attention_weight_min": -34.53599548339844,
      "activations/layer11_attention_weight_max": 30.854305267333984,
      "activations/layer11_attention_weight_min": -33.348426818847656,
      "activations/layer12_attention_weight_max": 17.04269790649414,
      "activations/layer12_attention_weight_min": -24.878080368041992,
      "activations/layer13_attention_weight_max": 31.56703758239746,
      "activations/layer13_attention_weight_min": -27.21573257446289,
      "activations/layer14_attention_weight_max": 30.769289016723633,
      "activations/layer14_attention_weight_min": -29.6401309967041,
      "activations/layer15_attention_weight_max": 29.836833953857422,
      "activations/layer15_attention_weight_min": -31.234708786010742,
      "activations/layer16_attention_weight_max": 29.908876419067383,
      "activations/layer16_attention_weight_min": -32.121883392333984,
      "activations/layer17_attention_weight_max": 45.205047607421875,
      "activations/layer17_attention_weight_min": -44.028778076171875,
      "activations/layer18_attention_weight_max": 39.53468322753906,
      "activations/layer18_attention_weight_min": -39.850738525390625,
      "activations/layer19_attention_weight_max": 20.828645706176758,
      "activations/layer19_attention_weight_min": -22.30970573425293,
      "activations/layer1_attention_weight_max": 15.573576927185059,
      "activations/layer1_attention_weight_min": -14.256819725036621,
      "activations/layer20_attention_weight_max": 19.867422103881836,
      "activations/layer20_attention_weight_min": -21.66881561279297,
      "activations/layer21_attention_weight_max": 37.03604507446289,
      "activations/layer21_attention_weight_min": -24.22697639465332,
      "activations/layer22_attention_weight_max": 26.04294776916504,
      "activations/layer22_attention_weight_min": -25.84993553161621,
      "activations/layer23_attention_weight_max": 33.88587951660156,
      "activations/layer23_attention_weight_min": -25.000381469726562,
      "activations/layer2_attention_weight_max": 31.382488250732422,
      "activations/layer2_attention_weight_min": -29.395240783691406,
      "activations/layer3_attention_weight_max": 86.40414428710938,
      "activations/layer3_attention_weight_min": -86.47979736328125,
      "activations/layer4_attention_weight_max": 91.6117172241211,
      "activations/layer4_attention_weight_min": -88.10723114013672,
      "activations/layer5_attention_weight_max": 65.28265380859375,
      "activations/layer5_attention_weight_min": -75.57919311523438,
      "activations/layer6_attention_weight_max": 49.647525787353516,
      "activations/layer6_attention_weight_min": -49.5226936340332,
      "activations/layer7_attention_weight_max": 64.02493286132812,
      "activations/layer7_attention_weight_min": -64.07896423339844,
      "activations/layer8_attention_weight_max": 44.09247970581055,
      "activations/layer8_attention_weight_min": -46.10641860961914,
      "activations/layer9_attention_weight_max": 39.75893783569336,
      "activations/layer9_attention_weight_min": -45.73503112792969,
      "epoch": 17.03,
      "learning_rate": 4.060113636363636e-05,
      "loss": 2.7356,
      "step": 293050
    },
    {
      "activations/layer0_attention_weight_max": 15.711813926696777,
      "activations/layer0_attention_weight_min": -13.817191123962402,
      "activations/layer10_attention_weight_max": 36.123592376708984,
      "activations/layer10_attention_weight_min": -36.38910675048828,
      "activations/layer11_attention_weight_max": 34.32987976074219,
      "activations/layer11_attention_weight_min": -35.45225143432617,
      "activations/layer12_attention_weight_max": 18.127458572387695,
      "activations/layer12_attention_weight_min": -24.124032974243164,
      "activations/layer13_attention_weight_max": 35.344825744628906,
      "activations/layer13_attention_weight_min": -28.819034576416016,
      "activations/layer14_attention_weight_max": 35.424095153808594,
      "activations/layer14_attention_weight_min": -32.694602966308594,
      "activations/layer15_attention_weight_max": 31.418306350708008,
      "activations/layer15_attention_weight_min": -33.998512268066406,
      "activations/layer16_attention_weight_max": 32.12497329711914,
      "activations/layer16_attention_weight_min": -31.987442016601562,
      "activations/layer17_attention_weight_max": 51.71424102783203,
      "activations/layer17_attention_weight_min": -44.529457092285156,
      "activations/layer18_attention_weight_max": 44.91786193847656,
      "activations/layer18_attention_weight_min": -40.97544479370117,
      "activations/layer19_attention_weight_max": 22.817590713500977,
      "activations/layer19_attention_weight_min": -23.476842880249023,
      "activations/layer1_attention_weight_max": 15.829044342041016,
      "activations/layer1_attention_weight_min": -14.449982643127441,
      "activations/layer20_attention_weight_max": 21.376075744628906,
      "activations/layer20_attention_weight_min": -21.946704864501953,
      "activations/layer21_attention_weight_max": 35.31890106201172,
      "activations/layer21_attention_weight_min": -21.240840911865234,
      "activations/layer22_attention_weight_max": 29.92941665649414,
      "activations/layer22_attention_weight_min": -29.213987350463867,
      "activations/layer23_attention_weight_max": 37.558250427246094,
      "activations/layer23_attention_weight_min": -26.14466667175293,
      "activations/layer2_attention_weight_max": 30.865657806396484,
      "activations/layer2_attention_weight_min": -29.248497009277344,
      "activations/layer3_attention_weight_max": 87.450439453125,
      "activations/layer3_attention_weight_min": -91.75928497314453,
      "activations/layer4_attention_weight_max": 88.6832504272461,
      "activations/layer4_attention_weight_min": -91.0622787475586,
      "activations/layer5_attention_weight_max": 64.7280044555664,
      "activations/layer5_attention_weight_min": -73.84625244140625,
      "activations/layer6_attention_weight_max": 48.5797004699707,
      "activations/layer6_attention_weight_min": -49.50974655151367,
      "activations/layer7_attention_weight_max": 67.9735336303711,
      "activations/layer7_attention_weight_min": -65.40309143066406,
      "activations/layer8_attention_weight_max": 46.958614349365234,
      "activations/layer8_attention_weight_min": -48.75468826293945,
      "activations/layer9_attention_weight_max": 48.1330680847168,
      "activations/layer9_attention_weight_min": -46.91057205200195,
      "epoch": 17.03,
      "learning_rate": 4.058257575757575e-05,
      "loss": 2.7442,
      "step": 293100
    },
    {
      "activations/layer0_attention_weight_max": 15.004939079284668,
      "activations/layer0_attention_weight_min": -14.11170482635498,
      "activations/layer10_attention_weight_max": 41.2088508605957,
      "activations/layer10_attention_weight_min": -37.916290283203125,
      "activations/layer11_attention_weight_max": 33.55698776245117,
      "activations/layer11_attention_weight_min": -34.098419189453125,
      "activations/layer12_attention_weight_max": 17.057920455932617,
      "activations/layer12_attention_weight_min": -24.51383399963379,
      "activations/layer13_attention_weight_max": 33.04207992553711,
      "activations/layer13_attention_weight_min": -27.5257625579834,
      "activations/layer14_attention_weight_max": 33.961544036865234,
      "activations/layer14_attention_weight_min": -29.17469024658203,
      "activations/layer15_attention_weight_max": 32.57078170776367,
      "activations/layer15_attention_weight_min": -29.63834571838379,
      "activations/layer16_attention_weight_max": 32.104248046875,
      "activations/layer16_attention_weight_min": -31.498430252075195,
      "activations/layer17_attention_weight_max": 49.478904724121094,
      "activations/layer17_attention_weight_min": -43.26472854614258,
      "activations/layer18_attention_weight_max": 44.2064094543457,
      "activations/layer18_attention_weight_min": -37.40807342529297,
      "activations/layer19_attention_weight_max": 21.270429611206055,
      "activations/layer19_attention_weight_min": -22.16610336303711,
      "activations/layer1_attention_weight_max": 15.407068252563477,
      "activations/layer1_attention_weight_min": -14.7937593460083,
      "activations/layer20_attention_weight_max": 20.460634231567383,
      "activations/layer20_attention_weight_min": -21.89231300354004,
      "activations/layer21_attention_weight_max": 45.75130844116211,
      "activations/layer21_attention_weight_min": -24.80283546447754,
      "activations/layer22_attention_weight_max": 27.879426956176758,
      "activations/layer22_attention_weight_min": -26.21952247619629,
      "activations/layer23_attention_weight_max": 35.702110290527344,
      "activations/layer23_attention_weight_min": -26.33527946472168,
      "activations/layer2_attention_weight_max": 31.77779769897461,
      "activations/layer2_attention_weight_min": -30.677078247070312,
      "activations/layer3_attention_weight_max": 86.86827850341797,
      "activations/layer3_attention_weight_min": -89.41236114501953,
      "activations/layer4_attention_weight_max": 86.2619857788086,
      "activations/layer4_attention_weight_min": -86.17195892333984,
      "activations/layer5_attention_weight_max": 60.57986068725586,
      "activations/layer5_attention_weight_min": -74.67501831054688,
      "activations/layer6_attention_weight_max": 48.32801055908203,
      "activations/layer6_attention_weight_min": -49.77674102783203,
      "activations/layer7_attention_weight_max": 61.819358825683594,
      "activations/layer7_attention_weight_min": -62.69261932373047,
      "activations/layer8_attention_weight_max": 45.95213317871094,
      "activations/layer8_attention_weight_min": -49.221309661865234,
      "activations/layer9_attention_weight_max": 45.452171325683594,
      "activations/layer9_attention_weight_min": -47.88903045654297,
      "epoch": 17.03,
      "learning_rate": 4.056363636363636e-05,
      "loss": 2.7315,
      "step": 293150
    },
    {
      "activations/layer0_attention_weight_max": 16.507326126098633,
      "activations/layer0_attention_weight_min": -13.930055618286133,
      "activations/layer10_attention_weight_max": 38.725215911865234,
      "activations/layer10_attention_weight_min": -35.90914535522461,
      "activations/layer11_attention_weight_max": 35.32773971557617,
      "activations/layer11_attention_weight_min": -35.57708740234375,
      "activations/layer12_attention_weight_max": 18.867326736450195,
      "activations/layer12_attention_weight_min": -25.996397018432617,
      "activations/layer13_attention_weight_max": 34.20549392700195,
      "activations/layer13_attention_weight_min": -27.357152938842773,
      "activations/layer14_attention_weight_max": 36.40739440917969,
      "activations/layer14_attention_weight_min": -28.46977424621582,
      "activations/layer15_attention_weight_max": 33.007049560546875,
      "activations/layer15_attention_weight_min": -27.659637451171875,
      "activations/layer16_attention_weight_max": 33.91476058959961,
      "activations/layer16_attention_weight_min": -29.885663986206055,
      "activations/layer17_attention_weight_max": 49.759979248046875,
      "activations/layer17_attention_weight_min": -40.68513488769531,
      "activations/layer18_attention_weight_max": 47.17585372924805,
      "activations/layer18_attention_weight_min": -37.08068084716797,
      "activations/layer19_attention_weight_max": 23.146337509155273,
      "activations/layer19_attention_weight_min": -21.923084259033203,
      "activations/layer1_attention_weight_max": 15.849451065063477,
      "activations/layer1_attention_weight_min": -14.652247428894043,
      "activations/layer20_attention_weight_max": 21.88271141052246,
      "activations/layer20_attention_weight_min": -23.226118087768555,
      "activations/layer21_attention_weight_max": 34.425025939941406,
      "activations/layer21_attention_weight_min": -23.920513153076172,
      "activations/layer22_attention_weight_max": 27.40723419189453,
      "activations/layer22_attention_weight_min": -25.131671905517578,
      "activations/layer23_attention_weight_max": 36.84464645385742,
      "activations/layer23_attention_weight_min": -23.366270065307617,
      "activations/layer2_attention_weight_max": 32.133544921875,
      "activations/layer2_attention_weight_min": -27.77042007446289,
      "activations/layer3_attention_weight_max": 85.03012084960938,
      "activations/layer3_attention_weight_min": -80.56159973144531,
      "activations/layer4_attention_weight_max": 86.02893829345703,
      "activations/layer4_attention_weight_min": -84.04411315917969,
      "activations/layer5_attention_weight_max": 61.13182830810547,
      "activations/layer5_attention_weight_min": -73.20182800292969,
      "activations/layer6_attention_weight_max": 48.35633850097656,
      "activations/layer6_attention_weight_min": -48.99674987792969,
      "activations/layer7_attention_weight_max": 62.807212829589844,
      "activations/layer7_attention_weight_min": -62.27381896972656,
      "activations/layer8_attention_weight_max": 44.31188201904297,
      "activations/layer8_attention_weight_min": -49.6515998840332,
      "activations/layer9_attention_weight_max": 42.22700119018555,
      "activations/layer9_attention_weight_min": -44.34431457519531,
      "epoch": 17.04,
      "learning_rate": 4.0544696969696966e-05,
      "loss": 2.7321,
      "step": 293200
    },
    {
      "activations/layer0_attention_weight_max": 15.634775161743164,
      "activations/layer0_attention_weight_min": -13.885842323303223,
      "activations/layer10_attention_weight_max": 37.11726760864258,
      "activations/layer10_attention_weight_min": -38.28812026977539,
      "activations/layer11_attention_weight_max": 31.541053771972656,
      "activations/layer11_attention_weight_min": -34.01242446899414,
      "activations/layer12_attention_weight_max": 18.380531311035156,
      "activations/layer12_attention_weight_min": -25.970216751098633,
      "activations/layer13_attention_weight_max": 45.84370040893555,
      "activations/layer13_attention_weight_min": -31.467105865478516,
      "activations/layer14_attention_weight_max": 46.456703186035156,
      "activations/layer14_attention_weight_min": -34.378135681152344,
      "activations/layer15_attention_weight_max": 41.42289352416992,
      "activations/layer15_attention_weight_min": -33.385990142822266,
      "activations/layer16_attention_weight_max": 36.06564712524414,
      "activations/layer16_attention_weight_min": -34.87449264526367,
      "activations/layer17_attention_weight_max": 53.89965057373047,
      "activations/layer17_attention_weight_min": -49.77637481689453,
      "activations/layer18_attention_weight_max": 51.46078109741211,
      "activations/layer18_attention_weight_min": -41.60822677612305,
      "activations/layer19_attention_weight_max": 32.0058479309082,
      "activations/layer19_attention_weight_min": -23.609203338623047,
      "activations/layer1_attention_weight_max": 16.596208572387695,
      "activations/layer1_attention_weight_min": -13.621657371520996,
      "activations/layer20_attention_weight_max": 30.363561630249023,
      "activations/layer20_attention_weight_min": -23.27537727355957,
      "activations/layer21_attention_weight_max": 45.17789077758789,
      "activations/layer21_attention_weight_min": -24.640380859375,
      "activations/layer22_attention_weight_max": 33.16032791137695,
      "activations/layer22_attention_weight_min": -25.720680236816406,
      "activations/layer23_attention_weight_max": 41.975914001464844,
      "activations/layer23_attention_weight_min": -24.579580307006836,
      "activations/layer2_attention_weight_max": 31.537656784057617,
      "activations/layer2_attention_weight_min": -29.289051055908203,
      "activations/layer3_attention_weight_max": 88.69633483886719,
      "activations/layer3_attention_weight_min": -89.18397521972656,
      "activations/layer4_attention_weight_max": 88.63681030273438,
      "activations/layer4_attention_weight_min": -88.01415252685547,
      "activations/layer5_attention_weight_max": 61.45304870605469,
      "activations/layer5_attention_weight_min": -71.88601684570312,
      "activations/layer6_attention_weight_max": 50.19730758666992,
      "activations/layer6_attention_weight_min": -50.9240608215332,
      "activations/layer7_attention_weight_max": 65.07510375976562,
      "activations/layer7_attention_weight_min": -65.01258087158203,
      "activations/layer8_attention_weight_max": 46.255645751953125,
      "activations/layer8_attention_weight_min": -49.4927864074707,
      "activations/layer9_attention_weight_max": 45.99072265625,
      "activations/layer9_attention_weight_min": -47.16535949707031,
      "epoch": 17.04,
      "learning_rate": 4.0525757575757574e-05,
      "loss": 2.7357,
      "step": 293250
    },
    {
      "activations/layer0_attention_weight_max": 15.32642650604248,
      "activations/layer0_attention_weight_min": -13.507597923278809,
      "activations/layer10_attention_weight_max": 35.012779235839844,
      "activations/layer10_attention_weight_min": -34.490142822265625,
      "activations/layer11_attention_weight_max": 33.016632080078125,
      "activations/layer11_attention_weight_min": -33.44322967529297,
      "activations/layer12_attention_weight_max": 20.886159896850586,
      "activations/layer12_attention_weight_min": -27.414146423339844,
      "activations/layer13_attention_weight_max": 42.616912841796875,
      "activations/layer13_attention_weight_min": -30.20768165588379,
      "activations/layer14_attention_weight_max": 39.59992218017578,
      "activations/layer14_attention_weight_min": -30.994274139404297,
      "activations/layer15_attention_weight_max": 37.50960922241211,
      "activations/layer15_attention_weight_min": -30.9447078704834,
      "activations/layer16_attention_weight_max": 36.465858459472656,
      "activations/layer16_attention_weight_min": -33.62687301635742,
      "activations/layer17_attention_weight_max": 52.78873825073242,
      "activations/layer17_attention_weight_min": -46.384952545166016,
      "activations/layer18_attention_weight_max": 47.491416931152344,
      "activations/layer18_attention_weight_min": -40.384605407714844,
      "activations/layer19_attention_weight_max": 24.31886863708496,
      "activations/layer19_attention_weight_min": -22.90054702758789,
      "activations/layer1_attention_weight_max": 16.16640281677246,
      "activations/layer1_attention_weight_min": -16.471166610717773,
      "activations/layer20_attention_weight_max": 25.0214786529541,
      "activations/layer20_attention_weight_min": -21.2058162689209,
      "activations/layer21_attention_weight_max": 44.90790939331055,
      "activations/layer21_attention_weight_min": -22.905710220336914,
      "activations/layer22_attention_weight_max": 32.8597526550293,
      "activations/layer22_attention_weight_min": -25.32175064086914,
      "activations/layer23_attention_weight_max": 41.08063507080078,
      "activations/layer23_attention_weight_min": -25.727643966674805,
      "activations/layer2_attention_weight_max": 30.177473068237305,
      "activations/layer2_attention_weight_min": -29.46082305908203,
      "activations/layer3_attention_weight_max": 86.60533905029297,
      "activations/layer3_attention_weight_min": -86.40682983398438,
      "activations/layer4_attention_weight_max": 81.79381561279297,
      "activations/layer4_attention_weight_min": -81.50304412841797,
      "activations/layer5_attention_weight_max": 63.904293060302734,
      "activations/layer5_attention_weight_min": -72.51100158691406,
      "activations/layer6_attention_weight_max": 47.516868591308594,
      "activations/layer6_attention_weight_min": -48.483219146728516,
      "activations/layer7_attention_weight_max": 62.26570510864258,
      "activations/layer7_attention_weight_min": -63.34982681274414,
      "activations/layer8_attention_weight_max": 45.09588623046875,
      "activations/layer8_attention_weight_min": -48.931243896484375,
      "activations/layer9_attention_weight_max": 43.90306091308594,
      "activations/layer9_attention_weight_min": -44.565914154052734,
      "epoch": 17.04,
      "learning_rate": 4.0506818181818176e-05,
      "loss": 2.7442,
      "step": 293300
    },
    {
      "activations/layer0_attention_weight_max": 15.030182838439941,
      "activations/layer0_attention_weight_min": -13.968127250671387,
      "activations/layer10_attention_weight_max": 34.0126953125,
      "activations/layer10_attention_weight_min": -35.4647216796875,
      "activations/layer11_attention_weight_max": 33.589725494384766,
      "activations/layer11_attention_weight_min": -33.180973052978516,
      "activations/layer12_attention_weight_max": 19.033523559570312,
      "activations/layer12_attention_weight_min": -37.8524169921875,
      "activations/layer13_attention_weight_max": 29.713191986083984,
      "activations/layer13_attention_weight_min": -28.84432029724121,
      "activations/layer14_attention_weight_max": 31.68464469909668,
      "activations/layer14_attention_weight_min": -28.93663215637207,
      "activations/layer15_attention_weight_max": 29.84758186340332,
      "activations/layer15_attention_weight_min": -28.027511596679688,
      "activations/layer16_attention_weight_max": 31.30901336669922,
      "activations/layer16_attention_weight_min": -30.724010467529297,
      "activations/layer17_attention_weight_max": 48.09713363647461,
      "activations/layer17_attention_weight_min": -45.67216873168945,
      "activations/layer18_attention_weight_max": 41.54534149169922,
      "activations/layer18_attention_weight_min": -39.51228713989258,
      "activations/layer19_attention_weight_max": 21.716217041015625,
      "activations/layer19_attention_weight_min": -23.503501892089844,
      "activations/layer1_attention_weight_max": 16.565521240234375,
      "activations/layer1_attention_weight_min": -14.641512870788574,
      "activations/layer20_attention_weight_max": 22.264083862304688,
      "activations/layer20_attention_weight_min": -24.53691291809082,
      "activations/layer21_attention_weight_max": 35.891761779785156,
      "activations/layer21_attention_weight_min": -22.917570114135742,
      "activations/layer22_attention_weight_max": 26.05500030517578,
      "activations/layer22_attention_weight_min": -27.57245635986328,
      "activations/layer23_attention_weight_max": 35.900169372558594,
      "activations/layer23_attention_weight_min": -27.002227783203125,
      "activations/layer2_attention_weight_max": 31.1137752532959,
      "activations/layer2_attention_weight_min": -30.320068359375,
      "activations/layer3_attention_weight_max": 87.98973846435547,
      "activations/layer3_attention_weight_min": -88.09550476074219,
      "activations/layer4_attention_weight_max": 86.54593658447266,
      "activations/layer4_attention_weight_min": -85.0982894897461,
      "activations/layer5_attention_weight_max": 66.12290954589844,
      "activations/layer5_attention_weight_min": -77.14419555664062,
      "activations/layer6_attention_weight_max": 48.01079177856445,
      "activations/layer6_attention_weight_min": -50.4570426940918,
      "activations/layer7_attention_weight_max": 67.2511215209961,
      "activations/layer7_attention_weight_min": -66.09198760986328,
      "activations/layer8_attention_weight_max": 44.11061096191406,
      "activations/layer8_attention_weight_min": -50.52388381958008,
      "activations/layer9_attention_weight_max": 48.923038482666016,
      "activations/layer9_attention_weight_min": -47.918914794921875,
      "epoch": 17.05,
      "learning_rate": 4.048787878787878e-05,
      "loss": 2.7496,
      "step": 293350
    },
    {
      "activations/layer0_attention_weight_max": 15.429112434387207,
      "activations/layer0_attention_weight_min": -14.037860870361328,
      "activations/layer10_attention_weight_max": 37.06364822387695,
      "activations/layer10_attention_weight_min": -36.23489761352539,
      "activations/layer11_attention_weight_max": 34.141136169433594,
      "activations/layer11_attention_weight_min": -37.51626205444336,
      "activations/layer12_attention_weight_max": 19.566572189331055,
      "activations/layer12_attention_weight_min": -25.354721069335938,
      "activations/layer13_attention_weight_max": 34.09901428222656,
      "activations/layer13_attention_weight_min": -30.705318450927734,
      "activations/layer14_attention_weight_max": 35.31024932861328,
      "activations/layer14_attention_weight_min": -31.520435333251953,
      "activations/layer15_attention_weight_max": 33.077632904052734,
      "activations/layer15_attention_weight_min": -28.9472713470459,
      "activations/layer16_attention_weight_max": 33.46884536743164,
      "activations/layer16_attention_weight_min": -32.293636322021484,
      "activations/layer17_attention_weight_max": 49.597084045410156,
      "activations/layer17_attention_weight_min": -43.28365707397461,
      "activations/layer18_attention_weight_max": 44.72710418701172,
      "activations/layer18_attention_weight_min": -38.006629943847656,
      "activations/layer19_attention_weight_max": 23.82779884338379,
      "activations/layer19_attention_weight_min": -23.46392059326172,
      "activations/layer1_attention_weight_max": 15.72278881072998,
      "activations/layer1_attention_weight_min": -14.692615509033203,
      "activations/layer20_attention_weight_max": 25.27663230895996,
      "activations/layer20_attention_weight_min": -22.69236946105957,
      "activations/layer21_attention_weight_max": 35.508026123046875,
      "activations/layer21_attention_weight_min": -21.611602783203125,
      "activations/layer22_attention_weight_max": 29.16769790649414,
      "activations/layer22_attention_weight_min": -25.160812377929688,
      "activations/layer23_attention_weight_max": 38.81060791015625,
      "activations/layer23_attention_weight_min": -23.29922866821289,
      "activations/layer2_attention_weight_max": 30.712360382080078,
      "activations/layer2_attention_weight_min": -29.639041900634766,
      "activations/layer3_attention_weight_max": 89.4247055053711,
      "activations/layer3_attention_weight_min": -88.381103515625,
      "activations/layer4_attention_weight_max": 85.78742980957031,
      "activations/layer4_attention_weight_min": -87.22325134277344,
      "activations/layer5_attention_weight_max": 62.688621520996094,
      "activations/layer5_attention_weight_min": -74.65853881835938,
      "activations/layer6_attention_weight_max": 51.30341720581055,
      "activations/layer6_attention_weight_min": -50.80519104003906,
      "activations/layer7_attention_weight_max": 64.7306137084961,
      "activations/layer7_attention_weight_min": -69.05623626708984,
      "activations/layer8_attention_weight_max": 46.478614807128906,
      "activations/layer8_attention_weight_min": -49.32762908935547,
      "activations/layer9_attention_weight_max": 42.872291564941406,
      "activations/layer9_attention_weight_min": -46.21013259887695,
      "epoch": 17.05,
      "learning_rate": 4.046893939393939e-05,
      "loss": 2.7438,
      "step": 293400
    },
    {
      "activations/layer0_attention_weight_max": 15.799880981445312,
      "activations/layer0_attention_weight_min": -14.02975082397461,
      "activations/layer10_attention_weight_max": 33.17377853393555,
      "activations/layer10_attention_weight_min": -34.02540969848633,
      "activations/layer11_attention_weight_max": 32.74644088745117,
      "activations/layer11_attention_weight_min": -34.407901763916016,
      "activations/layer12_attention_weight_max": 18.938947677612305,
      "activations/layer12_attention_weight_min": -23.053245544433594,
      "activations/layer13_attention_weight_max": 31.6296443939209,
      "activations/layer13_attention_weight_min": -27.938323974609375,
      "activations/layer14_attention_weight_max": 35.49089050292969,
      "activations/layer14_attention_weight_min": -31.580289840698242,
      "activations/layer15_attention_weight_max": 31.88892364501953,
      "activations/layer15_attention_weight_min": -28.93225860595703,
      "activations/layer16_attention_weight_max": 33.755859375,
      "activations/layer16_attention_weight_min": -32.91879653930664,
      "activations/layer17_attention_weight_max": 51.098567962646484,
      "activations/layer17_attention_weight_min": -47.894439697265625,
      "activations/layer18_attention_weight_max": 47.837364196777344,
      "activations/layer18_attention_weight_min": -44.48551559448242,
      "activations/layer19_attention_weight_max": 24.981428146362305,
      "activations/layer19_attention_weight_min": -25.8193359375,
      "activations/layer1_attention_weight_max": 17.326013565063477,
      "activations/layer1_attention_weight_min": -15.948783874511719,
      "activations/layer20_attention_weight_max": 25.998485565185547,
      "activations/layer20_attention_weight_min": -23.16861915588379,
      "activations/layer21_attention_weight_max": 38.308631896972656,
      "activations/layer21_attention_weight_min": -21.973325729370117,
      "activations/layer22_attention_weight_max": 33.33774185180664,
      "activations/layer22_attention_weight_min": -26.47964859008789,
      "activations/layer23_attention_weight_max": 37.91721725463867,
      "activations/layer23_attention_weight_min": -24.02252960205078,
      "activations/layer2_attention_weight_max": 32.84645462036133,
      "activations/layer2_attention_weight_min": -32.13161849975586,
      "activations/layer3_attention_weight_max": 84.16028594970703,
      "activations/layer3_attention_weight_min": -88.86316680908203,
      "activations/layer4_attention_weight_max": 84.82256317138672,
      "activations/layer4_attention_weight_min": -83.7865982055664,
      "activations/layer5_attention_weight_max": 64.19987487792969,
      "activations/layer5_attention_weight_min": -71.57671356201172,
      "activations/layer6_attention_weight_max": 47.92420196533203,
      "activations/layer6_attention_weight_min": -49.74667739868164,
      "activations/layer7_attention_weight_max": 64.1548843383789,
      "activations/layer7_attention_weight_min": -61.85888671875,
      "activations/layer8_attention_weight_max": 43.59473419189453,
      "activations/layer8_attention_weight_min": -47.399044036865234,
      "activations/layer9_attention_weight_max": 40.38737487792969,
      "activations/layer9_attention_weight_min": -43.9268798828125,
      "epoch": 17.05,
      "learning_rate": 4.0449999999999994e-05,
      "loss": 2.7294,
      "step": 293450
    },
    {
      "activations/layer0_attention_weight_max": 15.304841995239258,
      "activations/layer0_attention_weight_min": -13.993697166442871,
      "activations/layer10_attention_weight_max": 36.39950180053711,
      "activations/layer10_attention_weight_min": -35.081016540527344,
      "activations/layer11_attention_weight_max": 34.36621856689453,
      "activations/layer11_attention_weight_min": -34.694580078125,
      "activations/layer12_attention_weight_max": 19.41303062438965,
      "activations/layer12_attention_weight_min": -24.17133903503418,
      "activations/layer13_attention_weight_max": 40.59941864013672,
      "activations/layer13_attention_weight_min": -30.257781982421875,
      "activations/layer14_attention_weight_max": 39.164608001708984,
      "activations/layer14_attention_weight_min": -32.40252685546875,
      "activations/layer15_attention_weight_max": 38.25624084472656,
      "activations/layer15_attention_weight_min": -31.2096004486084,
      "activations/layer16_attention_weight_max": 36.770145416259766,
      "activations/layer16_attention_weight_min": -32.564208984375,
      "activations/layer17_attention_weight_max": 57.15793228149414,
      "activations/layer17_attention_weight_min": -48.717926025390625,
      "activations/layer18_attention_weight_max": 49.09541702270508,
      "activations/layer18_attention_weight_min": -40.527435302734375,
      "activations/layer19_attention_weight_max": 26.272123336791992,
      "activations/layer19_attention_weight_min": -23.73480796813965,
      "activations/layer1_attention_weight_max": 17.014671325683594,
      "activations/layer1_attention_weight_min": -14.494759559631348,
      "activations/layer20_attention_weight_max": 26.201696395874023,
      "activations/layer20_attention_weight_min": -21.674896240234375,
      "activations/layer21_attention_weight_max": 50.311519622802734,
      "activations/layer21_attention_weight_min": -27.064773559570312,
      "activations/layer22_attention_weight_max": 30.112817764282227,
      "activations/layer22_attention_weight_min": -26.002317428588867,
      "activations/layer23_attention_weight_max": 40.54652786254883,
      "activations/layer23_attention_weight_min": -26.442241668701172,
      "activations/layer2_attention_weight_max": 32.72210693359375,
      "activations/layer2_attention_weight_min": -29.278003692626953,
      "activations/layer3_attention_weight_max": 86.31552124023438,
      "activations/layer3_attention_weight_min": -85.44096374511719,
      "activations/layer4_attention_weight_max": 85.45027923583984,
      "activations/layer4_attention_weight_min": -86.68660736083984,
      "activations/layer5_attention_weight_max": 65.58154296875,
      "activations/layer5_attention_weight_min": -70.2740249633789,
      "activations/layer6_attention_weight_max": 50.879154205322266,
      "activations/layer6_attention_weight_min": -51.08335876464844,
      "activations/layer7_attention_weight_max": 69.13072967529297,
      "activations/layer7_attention_weight_min": -64.63123321533203,
      "activations/layer8_attention_weight_max": 48.54225158691406,
      "activations/layer8_attention_weight_min": -50.84563064575195,
      "activations/layer9_attention_weight_max": 49.00150680541992,
      "activations/layer9_attention_weight_min": -47.329158782958984,
      "epoch": 17.05,
      "learning_rate": 4.04310606060606e-05,
      "loss": 2.748,
      "step": 293500
    },
    {
      "activations/layer0_attention_weight_max": 15.498409271240234,
      "activations/layer0_attention_weight_min": -13.803614616394043,
      "activations/layer10_attention_weight_max": 40.196380615234375,
      "activations/layer10_attention_weight_min": -39.694923400878906,
      "activations/layer11_attention_weight_max": 37.97258758544922,
      "activations/layer11_attention_weight_min": -36.29338836669922,
      "activations/layer12_attention_weight_max": 20.222135543823242,
      "activations/layer12_attention_weight_min": -24.997827529907227,
      "activations/layer13_attention_weight_max": 52.13943862915039,
      "activations/layer13_attention_weight_min": -36.34638977050781,
      "activations/layer14_attention_weight_max": 58.385005950927734,
      "activations/layer14_attention_weight_min": -36.41335678100586,
      "activations/layer15_attention_weight_max": 51.35881042480469,
      "activations/layer15_attention_weight_min": -36.462928771972656,
      "activations/layer16_attention_weight_max": 39.9330940246582,
      "activations/layer16_attention_weight_min": -38.457374572753906,
      "activations/layer17_attention_weight_max": 58.14198684692383,
      "activations/layer17_attention_weight_min": -53.12925720214844,
      "activations/layer18_attention_weight_max": 54.34691619873047,
      "activations/layer18_attention_weight_min": -49.51128387451172,
      "activations/layer19_attention_weight_max": 33.23093795776367,
      "activations/layer19_attention_weight_min": -31.196359634399414,
      "activations/layer1_attention_weight_max": 15.4773530960083,
      "activations/layer1_attention_weight_min": -14.035663604736328,
      "activations/layer20_attention_weight_max": 36.567317962646484,
      "activations/layer20_attention_weight_min": -28.129154205322266,
      "activations/layer21_attention_weight_max": 52.169456481933594,
      "activations/layer21_attention_weight_min": -31.355180740356445,
      "activations/layer22_attention_weight_max": 36.859710693359375,
      "activations/layer22_attention_weight_min": -31.173229217529297,
      "activations/layer23_attention_weight_max": 41.87162780761719,
      "activations/layer23_attention_weight_min": -29.52063751220703,
      "activations/layer2_attention_weight_max": 30.013479232788086,
      "activations/layer2_attention_weight_min": -30.16130828857422,
      "activations/layer3_attention_weight_max": 88.40900421142578,
      "activations/layer3_attention_weight_min": -92.47080993652344,
      "activations/layer4_attention_weight_max": 87.28346252441406,
      "activations/layer4_attention_weight_min": -87.58685302734375,
      "activations/layer5_attention_weight_max": 68.08472442626953,
      "activations/layer5_attention_weight_min": -72.43696594238281,
      "activations/layer6_attention_weight_max": 49.391624450683594,
      "activations/layer6_attention_weight_min": -50.580604553222656,
      "activations/layer7_attention_weight_max": 66.64739227294922,
      "activations/layer7_attention_weight_min": -68.1374740600586,
      "activations/layer8_attention_weight_max": 48.733116149902344,
      "activations/layer8_attention_weight_min": -52.236900329589844,
      "activations/layer9_attention_weight_max": 56.51055908203125,
      "activations/layer9_attention_weight_min": -48.55049514770508,
      "epoch": 17.06,
      "learning_rate": 4.041212121212121e-05,
      "loss": 2.7513,
      "step": 293550
    },
    {
      "activations/layer0_attention_weight_max": 14.8690824508667,
      "activations/layer0_attention_weight_min": -13.78591251373291,
      "activations/layer10_attention_weight_max": 38.670654296875,
      "activations/layer10_attention_weight_min": -37.188533782958984,
      "activations/layer11_attention_weight_max": 35.06775665283203,
      "activations/layer11_attention_weight_min": -36.6255989074707,
      "activations/layer12_attention_weight_max": 19.887662887573242,
      "activations/layer12_attention_weight_min": -23.521148681640625,
      "activations/layer13_attention_weight_max": 33.07326889038086,
      "activations/layer13_attention_weight_min": -28.070255279541016,
      "activations/layer14_attention_weight_max": 39.48412322998047,
      "activations/layer14_attention_weight_min": -32.180355072021484,
      "activations/layer15_attention_weight_max": 37.52660369873047,
      "activations/layer15_attention_weight_min": -30.033565521240234,
      "activations/layer16_attention_weight_max": 35.230133056640625,
      "activations/layer16_attention_weight_min": -31.981966018676758,
      "activations/layer17_attention_weight_max": 56.489349365234375,
      "activations/layer17_attention_weight_min": -45.418853759765625,
      "activations/layer18_attention_weight_max": 51.01408767700195,
      "activations/layer18_attention_weight_min": -41.6572380065918,
      "activations/layer19_attention_weight_max": 24.030607223510742,
      "activations/layer19_attention_weight_min": -24.1479434967041,
      "activations/layer1_attention_weight_max": 17.008201599121094,
      "activations/layer1_attention_weight_min": -15.300986289978027,
      "activations/layer20_attention_weight_max": 23.18575096130371,
      "activations/layer20_attention_weight_min": -21.25634002685547,
      "activations/layer21_attention_weight_max": 37.082244873046875,
      "activations/layer21_attention_weight_min": -23.3717041015625,
      "activations/layer22_attention_weight_max": 28.6033992767334,
      "activations/layer22_attention_weight_min": -25.18595314025879,
      "activations/layer23_attention_weight_max": 39.21660232543945,
      "activations/layer23_attention_weight_min": -26.59423065185547,
      "activations/layer2_attention_weight_max": 32.45729064941406,
      "activations/layer2_attention_weight_min": -29.845746994018555,
      "activations/layer3_attention_weight_max": 90.88652038574219,
      "activations/layer3_attention_weight_min": -92.76614379882812,
      "activations/layer4_attention_weight_max": 91.53040313720703,
      "activations/layer4_attention_weight_min": -86.7027816772461,
      "activations/layer5_attention_weight_max": 66.61917114257812,
      "activations/layer5_attention_weight_min": -73.93838500976562,
      "activations/layer6_attention_weight_max": 55.24501037597656,
      "activations/layer6_attention_weight_min": -52.221290588378906,
      "activations/layer7_attention_weight_max": 71.05160522460938,
      "activations/layer7_attention_weight_min": -69.76586151123047,
      "activations/layer8_attention_weight_max": 47.5596923828125,
      "activations/layer8_attention_weight_min": -50.58922576904297,
      "activations/layer9_attention_weight_max": 45.084022521972656,
      "activations/layer9_attention_weight_min": -46.23158264160156,
      "epoch": 17.06,
      "learning_rate": 4.039318181818182e-05,
      "loss": 2.7534,
      "step": 293600
    },
    {
      "activations/layer0_attention_weight_max": 16.060073852539062,
      "activations/layer0_attention_weight_min": -13.772682189941406,
      "activations/layer10_attention_weight_max": 36.76402282714844,
      "activations/layer10_attention_weight_min": -35.77771759033203,
      "activations/layer11_attention_weight_max": 34.64087677001953,
      "activations/layer11_attention_weight_min": -33.04735565185547,
      "activations/layer12_attention_weight_max": 17.404348373413086,
      "activations/layer12_attention_weight_min": -22.81135368347168,
      "activations/layer13_attention_weight_max": 34.231231689453125,
      "activations/layer13_attention_weight_min": -25.761695861816406,
      "activations/layer14_attention_weight_max": 35.59950256347656,
      "activations/layer14_attention_weight_min": -30.524248123168945,
      "activations/layer15_attention_weight_max": 31.89024543762207,
      "activations/layer15_attention_weight_min": -31.510520935058594,
      "activations/layer16_attention_weight_max": 31.13633918762207,
      "activations/layer16_attention_weight_min": -31.438623428344727,
      "activations/layer17_attention_weight_max": 50.988800048828125,
      "activations/layer17_attention_weight_min": -44.831607818603516,
      "activations/layer18_attention_weight_max": 48.20404815673828,
      "activations/layer18_attention_weight_min": -39.39275360107422,
      "activations/layer19_attention_weight_max": 23.421924591064453,
      "activations/layer19_attention_weight_min": -21.001462936401367,
      "activations/layer1_attention_weight_max": 16.482646942138672,
      "activations/layer1_attention_weight_min": -15.710923194885254,
      "activations/layer20_attention_weight_max": 21.75841522216797,
      "activations/layer20_attention_weight_min": -20.661273956298828,
      "activations/layer21_attention_weight_max": 33.295127868652344,
      "activations/layer21_attention_weight_min": -22.41799545288086,
      "activations/layer22_attention_weight_max": 29.310970306396484,
      "activations/layer22_attention_weight_min": -26.375608444213867,
      "activations/layer23_attention_weight_max": 38.08976745605469,
      "activations/layer23_attention_weight_min": -24.4777889251709,
      "activations/layer2_attention_weight_max": 30.407997131347656,
      "activations/layer2_attention_weight_min": -31.85433006286621,
      "activations/layer3_attention_weight_max": 92.48978424072266,
      "activations/layer3_attention_weight_min": -91.30120086669922,
      "activations/layer4_attention_weight_max": 90.44287109375,
      "activations/layer4_attention_weight_min": -89.35720825195312,
      "activations/layer5_attention_weight_max": 64.27996826171875,
      "activations/layer5_attention_weight_min": -74.56666564941406,
      "activations/layer6_attention_weight_max": 51.360984802246094,
      "activations/layer6_attention_weight_min": -54.95800018310547,
      "activations/layer7_attention_weight_max": 64.93598937988281,
      "activations/layer7_attention_weight_min": -66.64561462402344,
      "activations/layer8_attention_weight_max": 49.677913665771484,
      "activations/layer8_attention_weight_min": -51.153995513916016,
      "activations/layer9_attention_weight_max": 47.07416915893555,
      "activations/layer9_attention_weight_min": -47.2350959777832,
      "epoch": 17.06,
      "learning_rate": 4.037424242424242e-05,
      "loss": 2.7419,
      "step": 293650
    },
    {
      "activations/layer0_attention_weight_max": 14.989056587219238,
      "activations/layer0_attention_weight_min": -13.875005722045898,
      "activations/layer10_attention_weight_max": 38.8763427734375,
      "activations/layer10_attention_weight_min": -38.91987228393555,
      "activations/layer11_attention_weight_max": 36.81031036376953,
      "activations/layer11_attention_weight_min": -37.25160217285156,
      "activations/layer12_attention_weight_max": 23.48603057861328,
      "activations/layer12_attention_weight_min": -24.69479751586914,
      "activations/layer13_attention_weight_max": 38.2929801940918,
      "activations/layer13_attention_weight_min": -31.25921058654785,
      "activations/layer14_attention_weight_max": 36.34741973876953,
      "activations/layer14_attention_weight_min": -28.88511085510254,
      "activations/layer15_attention_weight_max": 33.19004440307617,
      "activations/layer15_attention_weight_min": -28.36086082458496,
      "activations/layer16_attention_weight_max": 34.22868728637695,
      "activations/layer16_attention_weight_min": -30.804521560668945,
      "activations/layer17_attention_weight_max": 53.45277786254883,
      "activations/layer17_attention_weight_min": -45.72500991821289,
      "activations/layer18_attention_weight_max": 49.941410064697266,
      "activations/layer18_attention_weight_min": -40.52400207519531,
      "activations/layer19_attention_weight_max": 24.361000061035156,
      "activations/layer19_attention_weight_min": -22.657106399536133,
      "activations/layer1_attention_weight_max": 15.001466751098633,
      "activations/layer1_attention_weight_min": -14.729912757873535,
      "activations/layer20_attention_weight_max": 25.523338317871094,
      "activations/layer20_attention_weight_min": -22.220914840698242,
      "activations/layer21_attention_weight_max": 44.58799743652344,
      "activations/layer21_attention_weight_min": -22.55590057373047,
      "activations/layer22_attention_weight_max": 33.28422164916992,
      "activations/layer22_attention_weight_min": -24.92876434326172,
      "activations/layer23_attention_weight_max": 39.524147033691406,
      "activations/layer23_attention_weight_min": -22.677539825439453,
      "activations/layer2_attention_weight_max": 29.76099967956543,
      "activations/layer2_attention_weight_min": -29.236934661865234,
      "activations/layer3_attention_weight_max": 86.8165283203125,
      "activations/layer3_attention_weight_min": -90.8717269897461,
      "activations/layer4_attention_weight_max": 87.70486450195312,
      "activations/layer4_attention_weight_min": -88.75159454345703,
      "activations/layer5_attention_weight_max": 61.13877487182617,
      "activations/layer5_attention_weight_min": -77.58711242675781,
      "activations/layer6_attention_weight_max": 48.09856414794922,
      "activations/layer6_attention_weight_min": -52.15550994873047,
      "activations/layer7_attention_weight_max": 66.18963623046875,
      "activations/layer7_attention_weight_min": -67.30429077148438,
      "activations/layer8_attention_weight_max": 47.32758331298828,
      "activations/layer8_attention_weight_min": -49.34666442871094,
      "activations/layer9_attention_weight_max": 47.93360137939453,
      "activations/layer9_attention_weight_min": -44.424072265625,
      "epoch": 17.07,
      "learning_rate": 4.035530303030302e-05,
      "loss": 2.7395,
      "step": 293700
    },
    {
      "activations/layer0_attention_weight_max": 15.30362319946289,
      "activations/layer0_attention_weight_min": -14.295475959777832,
      "activations/layer10_attention_weight_max": 36.29313278198242,
      "activations/layer10_attention_weight_min": -35.388954162597656,
      "activations/layer11_attention_weight_max": 36.01935577392578,
      "activations/layer11_attention_weight_min": -34.89604568481445,
      "activations/layer12_attention_weight_max": 19.793670654296875,
      "activations/layer12_attention_weight_min": -24.11188507080078,
      "activations/layer13_attention_weight_max": 36.26228332519531,
      "activations/layer13_attention_weight_min": -31.201303482055664,
      "activations/layer14_attention_weight_max": 45.627716064453125,
      "activations/layer14_attention_weight_min": -36.615325927734375,
      "activations/layer15_attention_weight_max": 43.86405944824219,
      "activations/layer15_attention_weight_min": -33.38849639892578,
      "activations/layer16_attention_weight_max": 37.56870651245117,
      "activations/layer16_attention_weight_min": -34.94313049316406,
      "activations/layer17_attention_weight_max": 57.4318733215332,
      "activations/layer17_attention_weight_min": -50.591575622558594,
      "activations/layer18_attention_weight_max": 51.024776458740234,
      "activations/layer18_attention_weight_min": -47.491119384765625,
      "activations/layer19_attention_weight_max": 23.62204360961914,
      "activations/layer19_attention_weight_min": -23.763408660888672,
      "activations/layer1_attention_weight_max": 16.78087043762207,
      "activations/layer1_attention_weight_min": -16.131505966186523,
      "activations/layer20_attention_weight_max": 23.587730407714844,
      "activations/layer20_attention_weight_min": -23.918737411499023,
      "activations/layer21_attention_weight_max": 41.89818572998047,
      "activations/layer21_attention_weight_min": -27.323808670043945,
      "activations/layer22_attention_weight_max": 31.75537872314453,
      "activations/layer22_attention_weight_min": -24.683279037475586,
      "activations/layer23_attention_weight_max": 37.04728698730469,
      "activations/layer23_attention_weight_min": -26.921283721923828,
      "activations/layer2_attention_weight_max": 31.41457176208496,
      "activations/layer2_attention_weight_min": -30.225711822509766,
      "activations/layer3_attention_weight_max": 86.36392211914062,
      "activations/layer3_attention_weight_min": -88.13861083984375,
      "activations/layer4_attention_weight_max": 87.71251678466797,
      "activations/layer4_attention_weight_min": -87.23898315429688,
      "activations/layer5_attention_weight_max": 64.67235565185547,
      "activations/layer5_attention_weight_min": -75.99251556396484,
      "activations/layer6_attention_weight_max": 50.058048248291016,
      "activations/layer6_attention_weight_min": -49.04579162597656,
      "activations/layer7_attention_weight_max": 66.93392181396484,
      "activations/layer7_attention_weight_min": -64.52843475341797,
      "activations/layer8_attention_weight_max": 47.050819396972656,
      "activations/layer8_attention_weight_min": -47.767913818359375,
      "activations/layer9_attention_weight_max": 55.75771713256836,
      "activations/layer9_attention_weight_min": -46.00379943847656,
      "epoch": 17.07,
      "learning_rate": 4.033636363636364e-05,
      "loss": 2.7357,
      "step": 293750
    },
    {
      "activations/layer0_attention_weight_max": 15.348187446594238,
      "activations/layer0_attention_weight_min": -13.707868576049805,
      "activations/layer10_attention_weight_max": 35.135311126708984,
      "activations/layer10_attention_weight_min": -35.485591888427734,
      "activations/layer11_attention_weight_max": 34.51971435546875,
      "activations/layer11_attention_weight_min": -33.985355377197266,
      "activations/layer12_attention_weight_max": 18.958703994750977,
      "activations/layer12_attention_weight_min": -33.67799758911133,
      "activations/layer13_attention_weight_max": 51.83804702758789,
      "activations/layer13_attention_weight_min": -30.92098617553711,
      "activations/layer14_attention_weight_max": 52.55366134643555,
      "activations/layer14_attention_weight_min": -30.098461151123047,
      "activations/layer15_attention_weight_max": 52.986446380615234,
      "activations/layer15_attention_weight_min": -29.064048767089844,
      "activations/layer16_attention_weight_max": 33.88351821899414,
      "activations/layer16_attention_weight_min": -31.395931243896484,
      "activations/layer17_attention_weight_max": 49.94706344604492,
      "activations/layer17_attention_weight_min": -44.56148910522461,
      "activations/layer18_attention_weight_max": 47.58551025390625,
      "activations/layer18_attention_weight_min": -38.42988967895508,
      "activations/layer19_attention_weight_max": 24.7091064453125,
      "activations/layer19_attention_weight_min": -22.091960906982422,
      "activations/layer1_attention_weight_max": 16.992748260498047,
      "activations/layer1_attention_weight_min": -14.946783065795898,
      "activations/layer20_attention_weight_max": 25.30733299255371,
      "activations/layer20_attention_weight_min": -20.676332473754883,
      "activations/layer21_attention_weight_max": 38.29345703125,
      "activations/layer21_attention_weight_min": -21.172304153442383,
      "activations/layer22_attention_weight_max": 28.48678970336914,
      "activations/layer22_attention_weight_min": -25.778148651123047,
      "activations/layer23_attention_weight_max": 37.27123260498047,
      "activations/layer23_attention_weight_min": -24.37828826904297,
      "activations/layer2_attention_weight_max": 32.21289825439453,
      "activations/layer2_attention_weight_min": -29.892841339111328,
      "activations/layer3_attention_weight_max": 92.05546569824219,
      "activations/layer3_attention_weight_min": -93.0513916015625,
      "activations/layer4_attention_weight_max": 88.42926788330078,
      "activations/layer4_attention_weight_min": -87.21296691894531,
      "activations/layer5_attention_weight_max": 64.03874206542969,
      "activations/layer5_attention_weight_min": -72.71257019042969,
      "activations/layer6_attention_weight_max": 51.39649963378906,
      "activations/layer6_attention_weight_min": -50.57783126831055,
      "activations/layer7_attention_weight_max": 67.66439819335938,
      "activations/layer7_attention_weight_min": -63.38417434692383,
      "activations/layer8_attention_weight_max": 47.3453254699707,
      "activations/layer8_attention_weight_min": -46.20128631591797,
      "activations/layer9_attention_weight_max": 51.192073822021484,
      "activations/layer9_attention_weight_min": -45.396854400634766,
      "epoch": 17.07,
      "learning_rate": 4.031742424242424e-05,
      "loss": 2.7382,
      "step": 293800
    },
    {
      "activations/layer0_attention_weight_max": 16.06931495666504,
      "activations/layer0_attention_weight_min": -13.95075511932373,
      "activations/layer10_attention_weight_max": 36.266456604003906,
      "activations/layer10_attention_weight_min": -36.335235595703125,
      "activations/layer11_attention_weight_max": 33.66316223144531,
      "activations/layer11_attention_weight_min": -35.998287200927734,
      "activations/layer12_attention_weight_max": 19.490880966186523,
      "activations/layer12_attention_weight_min": -22.617557525634766,
      "activations/layer13_attention_weight_max": 33.24919891357422,
      "activations/layer13_attention_weight_min": -27.78780174255371,
      "activations/layer14_attention_weight_max": 34.926998138427734,
      "activations/layer14_attention_weight_min": -30.375791549682617,
      "activations/layer15_attention_weight_max": 34.299072265625,
      "activations/layer15_attention_weight_min": -30.412479400634766,
      "activations/layer16_attention_weight_max": 33.146018981933594,
      "activations/layer16_attention_weight_min": -30.38677215576172,
      "activations/layer17_attention_weight_max": 53.5322265625,
      "activations/layer17_attention_weight_min": -43.95957946777344,
      "activations/layer18_attention_weight_max": 49.6114616394043,
      "activations/layer18_attention_weight_min": -41.048095703125,
      "activations/layer19_attention_weight_max": 25.281225204467773,
      "activations/layer19_attention_weight_min": -22.683658599853516,
      "activations/layer1_attention_weight_max": 16.396438598632812,
      "activations/layer1_attention_weight_min": -14.184407234191895,
      "activations/layer20_attention_weight_max": 23.880765914916992,
      "activations/layer20_attention_weight_min": -20.405176162719727,
      "activations/layer21_attention_weight_max": 36.22655487060547,
      "activations/layer21_attention_weight_min": -21.3714542388916,
      "activations/layer22_attention_weight_max": 31.083181381225586,
      "activations/layer22_attention_weight_min": -23.988388061523438,
      "activations/layer23_attention_weight_max": 39.39081954956055,
      "activations/layer23_attention_weight_min": -23.974721908569336,
      "activations/layer2_attention_weight_max": 31.31961441040039,
      "activations/layer2_attention_weight_min": -29.35360336303711,
      "activations/layer3_attention_weight_max": 90.48627471923828,
      "activations/layer3_attention_weight_min": -88.63216400146484,
      "activations/layer4_attention_weight_max": 88.21238708496094,
      "activations/layer4_attention_weight_min": -88.91845703125,
      "activations/layer5_attention_weight_max": 64.46822357177734,
      "activations/layer5_attention_weight_min": -74.19503784179688,
      "activations/layer6_attention_weight_max": 50.55390930175781,
      "activations/layer6_attention_weight_min": -51.007564544677734,
      "activations/layer7_attention_weight_max": 64.53497314453125,
      "activations/layer7_attention_weight_min": -65.81183624267578,
      "activations/layer8_attention_weight_max": 45.93846130371094,
      "activations/layer8_attention_weight_min": -48.88705062866211,
      "activations/layer9_attention_weight_max": 44.385337829589844,
      "activations/layer9_attention_weight_min": -45.569461822509766,
      "epoch": 17.07,
      "learning_rate": 4.0298484848484846e-05,
      "loss": 2.7383,
      "step": 293850
    },
    {
      "activations/layer0_attention_weight_max": 17.015989303588867,
      "activations/layer0_attention_weight_min": -14.075688362121582,
      "activations/layer10_attention_weight_max": 33.695648193359375,
      "activations/layer10_attention_weight_min": -35.02471923828125,
      "activations/layer11_attention_weight_max": 32.432861328125,
      "activations/layer11_attention_weight_min": -33.239376068115234,
      "activations/layer12_attention_weight_max": 17.51654624938965,
      "activations/layer12_attention_weight_min": -25.271881103515625,
      "activations/layer13_attention_weight_max": 34.99834060668945,
      "activations/layer13_attention_weight_min": -26.52581024169922,
      "activations/layer14_attention_weight_max": 30.938451766967773,
      "activations/layer14_attention_weight_min": -28.856496810913086,
      "activations/layer15_attention_weight_max": 28.527530670166016,
      "activations/layer15_attention_weight_min": -29.07967758178711,
      "activations/layer16_attention_weight_max": 28.49445343017578,
      "activations/layer16_attention_weight_min": -29.658432006835938,
      "activations/layer17_attention_weight_max": 45.63288497924805,
      "activations/layer17_attention_weight_min": -43.27729034423828,
      "activations/layer18_attention_weight_max": 42.08068084716797,
      "activations/layer18_attention_weight_min": -36.106239318847656,
      "activations/layer19_attention_weight_max": 23.197986602783203,
      "activations/layer19_attention_weight_min": -20.505842208862305,
      "activations/layer1_attention_weight_max": 16.884178161621094,
      "activations/layer1_attention_weight_min": -17.167118072509766,
      "activations/layer20_attention_weight_max": 20.601341247558594,
      "activations/layer20_attention_weight_min": -22.969955444335938,
      "activations/layer21_attention_weight_max": 30.362895965576172,
      "activations/layer21_attention_weight_min": -21.135103225708008,
      "activations/layer22_attention_weight_max": 29.555986404418945,
      "activations/layer22_attention_weight_min": -26.861860275268555,
      "activations/layer23_attention_weight_max": 33.14238357543945,
      "activations/layer23_attention_weight_min": -25.167654037475586,
      "activations/layer2_attention_weight_max": 30.732036590576172,
      "activations/layer2_attention_weight_min": -30.36530113220215,
      "activations/layer3_attention_weight_max": 85.06403350830078,
      "activations/layer3_attention_weight_min": -92.90975189208984,
      "activations/layer4_attention_weight_max": 86.20967864990234,
      "activations/layer4_attention_weight_min": -84.52733612060547,
      "activations/layer5_attention_weight_max": 65.33944702148438,
      "activations/layer5_attention_weight_min": -73.6054916381836,
      "activations/layer6_attention_weight_max": 51.345645904541016,
      "activations/layer6_attention_weight_min": -50.04720687866211,
      "activations/layer7_attention_weight_max": 63.43226623535156,
      "activations/layer7_attention_weight_min": -63.04731369018555,
      "activations/layer8_attention_weight_max": 45.169334411621094,
      "activations/layer8_attention_weight_min": -48.86152648925781,
      "activations/layer9_attention_weight_max": 43.08156967163086,
      "activations/layer9_attention_weight_min": -46.630706787109375,
      "epoch": 17.08,
      "learning_rate": 4.027954545454545e-05,
      "loss": 2.7498,
      "step": 293900
    },
    {
      "activations/layer0_attention_weight_max": 17.15728759765625,
      "activations/layer0_attention_weight_min": -13.713659286499023,
      "activations/layer10_attention_weight_max": 34.982696533203125,
      "activations/layer10_attention_weight_min": -36.176361083984375,
      "activations/layer11_attention_weight_max": 32.75019073486328,
      "activations/layer11_attention_weight_min": -34.00674057006836,
      "activations/layer12_attention_weight_max": 19.209653854370117,
      "activations/layer12_attention_weight_min": -28.2997989654541,
      "activations/layer13_attention_weight_max": 43.605201721191406,
      "activations/layer13_attention_weight_min": -30.041189193725586,
      "activations/layer14_attention_weight_max": 35.827545166015625,
      "activations/layer14_attention_weight_min": -28.55318260192871,
      "activations/layer15_attention_weight_max": 33.14788818359375,
      "activations/layer15_attention_weight_min": -30.254262924194336,
      "activations/layer16_attention_weight_max": 34.055999755859375,
      "activations/layer16_attention_weight_min": -30.054813385009766,
      "activations/layer17_attention_weight_max": 53.9140510559082,
      "activations/layer17_attention_weight_min": -46.493473052978516,
      "activations/layer18_attention_weight_max": 47.91847229003906,
      "activations/layer18_attention_weight_min": -38.25395202636719,
      "activations/layer19_attention_weight_max": 24.346845626831055,
      "activations/layer19_attention_weight_min": -22.79438018798828,
      "activations/layer1_attention_weight_max": 17.809757232666016,
      "activations/layer1_attention_weight_min": -14.34070873260498,
      "activations/layer20_attention_weight_max": 19.84020233154297,
      "activations/layer20_attention_weight_min": -20.96788787841797,
      "activations/layer21_attention_weight_max": 36.02560043334961,
      "activations/layer21_attention_weight_min": -23.2819766998291,
      "activations/layer22_attention_weight_max": 28.291650772094727,
      "activations/layer22_attention_weight_min": -24.92475700378418,
      "activations/layer23_attention_weight_max": 38.68391799926758,
      "activations/layer23_attention_weight_min": -23.679533004760742,
      "activations/layer2_attention_weight_max": 32.55751037597656,
      "activations/layer2_attention_weight_min": -31.22675323486328,
      "activations/layer3_attention_weight_max": 88.59980773925781,
      "activations/layer3_attention_weight_min": -89.86151885986328,
      "activations/layer4_attention_weight_max": 88.54761505126953,
      "activations/layer4_attention_weight_min": -88.4664306640625,
      "activations/layer5_attention_weight_max": 63.88819885253906,
      "activations/layer5_attention_weight_min": -75.93219757080078,
      "activations/layer6_attention_weight_max": 48.80607986450195,
      "activations/layer6_attention_weight_min": -51.02278518676758,
      "activations/layer7_attention_weight_max": 64.01998138427734,
      "activations/layer7_attention_weight_min": -63.96693801879883,
      "activations/layer8_attention_weight_max": 47.8853645324707,
      "activations/layer8_attention_weight_min": -48.55717468261719,
      "activations/layer9_attention_weight_max": 45.161006927490234,
      "activations/layer9_attention_weight_min": -45.72306442260742,
      "epoch": 17.08,
      "learning_rate": 4.026060606060606e-05,
      "loss": 2.7431,
      "step": 293950
    },
    {
      "activations/layer0_attention_weight_max": 15.407999992370605,
      "activations/layer0_attention_weight_min": -14.071266174316406,
      "activations/layer10_attention_weight_max": 37.051177978515625,
      "activations/layer10_attention_weight_min": -37.759368896484375,
      "activations/layer11_attention_weight_max": 36.12904739379883,
      "activations/layer11_attention_weight_min": -36.75977325439453,
      "activations/layer12_attention_weight_max": 20.335649490356445,
      "activations/layer12_attention_weight_min": -22.717435836791992,
      "activations/layer13_attention_weight_max": 36.933128356933594,
      "activations/layer13_attention_weight_min": -30.593477249145508,
      "activations/layer14_attention_weight_max": 37.58783721923828,
      "activations/layer14_attention_weight_min": -31.001270294189453,
      "activations/layer15_attention_weight_max": 34.04095458984375,
      "activations/layer15_attention_weight_min": -29.55341148376465,
      "activations/layer16_attention_weight_max": 35.75697708129883,
      "activations/layer16_attention_weight_min": -34.04166030883789,
      "activations/layer17_attention_weight_max": 56.74634552001953,
      "activations/layer17_attention_weight_min": -49.69062042236328,
      "activations/layer18_attention_weight_max": 50.232357025146484,
      "activations/layer18_attention_weight_min": -43.94819641113281,
      "activations/layer19_attention_weight_max": 25.593976974487305,
      "activations/layer19_attention_weight_min": -25.5809268951416,
      "activations/layer1_attention_weight_max": 16.134679794311523,
      "activations/layer1_attention_weight_min": -16.025253295898438,
      "activations/layer20_attention_weight_max": 23.619497299194336,
      "activations/layer20_attention_weight_min": -23.57761001586914,
      "activations/layer21_attention_weight_max": 37.505226135253906,
      "activations/layer21_attention_weight_min": -25.24650764465332,
      "activations/layer22_attention_weight_max": 31.36094856262207,
      "activations/layer22_attention_weight_min": -29.74277687072754,
      "activations/layer23_attention_weight_max": 37.64329528808594,
      "activations/layer23_attention_weight_min": -26.072128295898438,
      "activations/layer2_attention_weight_max": 31.309818267822266,
      "activations/layer2_attention_weight_min": -30.408872604370117,
      "activations/layer3_attention_weight_max": 88.8755874633789,
      "activations/layer3_attention_weight_min": -93.77010345458984,
      "activations/layer4_attention_weight_max": 89.57408905029297,
      "activations/layer4_attention_weight_min": -87.73104858398438,
      "activations/layer5_attention_weight_max": 65.71514892578125,
      "activations/layer5_attention_weight_min": -71.21554565429688,
      "activations/layer6_attention_weight_max": 53.16046905517578,
      "activations/layer6_attention_weight_min": -51.915321350097656,
      "activations/layer7_attention_weight_max": 69.16207885742188,
      "activations/layer7_attention_weight_min": -64.00433349609375,
      "activations/layer8_attention_weight_max": 50.98600769042969,
      "activations/layer8_attention_weight_min": -51.77333068847656,
      "activations/layer9_attention_weight_max": 50.775569915771484,
      "activations/layer9_attention_weight_min": -49.445411682128906,
      "epoch": 17.08,
      "learning_rate": 4.0241666666666665e-05,
      "loss": 2.738,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4113,
      "eval_samples_per_second": 510.503,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4113,
      "eval_openwebtext_samples_per_second": 510.503,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9577,
      "eval_wikitext_samples_per_second": 232.928,
      "step": 294000
    },
    {
      "epoch": 17.08,
      "eval_lambada_loss": 2.48046875,
      "eval_lambada_ppl": 11.946863197656471,
      "eval_lambada_runtime": 9.5225,
      "eval_lambada_samples_per_second": 511.316,
      "step": 294000
    },
    {
      "activations/layer0_attention_weight_max": 14.429303169250488,
      "activations/layer0_attention_weight_min": -13.860393524169922,
      "activations/layer10_attention_weight_max": 43.185272216796875,
      "activations/layer10_attention_weight_min": -43.66081237792969,
      "activations/layer11_attention_weight_max": 37.448184967041016,
      "activations/layer11_attention_weight_min": -38.72267532348633,
      "activations/layer12_attention_weight_max": 20.552724838256836,
      "activations/layer12_attention_weight_min": -25.889930725097656,
      "activations/layer13_attention_weight_max": 35.574737548828125,
      "activations/layer13_attention_weight_min": -26.350263595581055,
      "activations/layer14_attention_weight_max": 35.70024490356445,
      "activations/layer14_attention_weight_min": -32.24104690551758,
      "activations/layer15_attention_weight_max": 32.39425277709961,
      "activations/layer15_attention_weight_min": -33.18502426147461,
      "activations/layer16_attention_weight_max": 35.02119827270508,
      "activations/layer16_attention_weight_min": -34.83356475830078,
      "activations/layer17_attention_weight_max": 53.25175476074219,
      "activations/layer17_attention_weight_min": -47.2882080078125,
      "activations/layer18_attention_weight_max": 47.452762603759766,
      "activations/layer18_attention_weight_min": -42.4190673828125,
      "activations/layer19_attention_weight_max": 24.92508888244629,
      "activations/layer19_attention_weight_min": -23.54346466064453,
      "activations/layer1_attention_weight_max": 15.451603889465332,
      "activations/layer1_attention_weight_min": -15.418115615844727,
      "activations/layer20_attention_weight_max": 23.39444923400879,
      "activations/layer20_attention_weight_min": -21.6214656829834,
      "activations/layer21_attention_weight_max": 36.63400650024414,
      "activations/layer21_attention_weight_min": -20.429576873779297,
      "activations/layer22_attention_weight_max": 30.776845932006836,
      "activations/layer22_attention_weight_min": -26.62374496459961,
      "activations/layer23_attention_weight_max": 39.061363220214844,
      "activations/layer23_attention_weight_min": -23.610021591186523,
      "activations/layer2_attention_weight_max": 32.373939514160156,
      "activations/layer2_attention_weight_min": -32.727020263671875,
      "activations/layer3_attention_weight_max": 97.66650390625,
      "activations/layer3_attention_weight_min": -101.74554443359375,
      "activations/layer4_attention_weight_max": 94.90635681152344,
      "activations/layer4_attention_weight_min": -94.08570098876953,
      "activations/layer5_attention_weight_max": 67.7624282836914,
      "activations/layer5_attention_weight_min": -70.72354888916016,
      "activations/layer6_attention_weight_max": 58.27482223510742,
      "activations/layer6_attention_weight_min": -56.85808181762695,
      "activations/layer7_attention_weight_max": 75.12841796875,
      "activations/layer7_attention_weight_min": -75.82414245605469,
      "activations/layer8_attention_weight_max": 55.29386901855469,
      "activations/layer8_attention_weight_min": -57.63962936401367,
      "activations/layer9_attention_weight_max": 55.843353271484375,
      "activations/layer9_attention_weight_min": -52.692596435546875,
      "epoch": 17.09,
      "learning_rate": 4.0222727272727266e-05,
      "loss": 2.7337,
      "step": 294050
    },
    {
      "activations/layer0_attention_weight_max": 16.10047149658203,
      "activations/layer0_attention_weight_min": -13.567789077758789,
      "activations/layer10_attention_weight_max": 39.02307891845703,
      "activations/layer10_attention_weight_min": -38.20427322387695,
      "activations/layer11_attention_weight_max": 33.43590545654297,
      "activations/layer11_attention_weight_min": -33.964134216308594,
      "activations/layer12_attention_weight_max": 19.867055892944336,
      "activations/layer12_attention_weight_min": -26.006074905395508,
      "activations/layer13_attention_weight_max": 44.666831970214844,
      "activations/layer13_attention_weight_min": -29.856332778930664,
      "activations/layer14_attention_weight_max": 40.1783332824707,
      "activations/layer14_attention_weight_min": -30.9731502532959,
      "activations/layer15_attention_weight_max": 34.09067153930664,
      "activations/layer15_attention_weight_min": -32.41602325439453,
      "activations/layer16_attention_weight_max": 34.014869689941406,
      "activations/layer16_attention_weight_min": -32.1353645324707,
      "activations/layer17_attention_weight_max": 49.617431640625,
      "activations/layer17_attention_weight_min": -43.69804000854492,
      "activations/layer18_attention_weight_max": 48.06003189086914,
      "activations/layer18_attention_weight_min": -41.38425064086914,
      "activations/layer19_attention_weight_max": 24.431989669799805,
      "activations/layer19_attention_weight_min": -22.967355728149414,
      "activations/layer1_attention_weight_max": 15.86348819732666,
      "activations/layer1_attention_weight_min": -14.116462707519531,
      "activations/layer20_attention_weight_max": 23.38207244873047,
      "activations/layer20_attention_weight_min": -22.50731086730957,
      "activations/layer21_attention_weight_max": 36.194984436035156,
      "activations/layer21_attention_weight_min": -23.12969398498535,
      "activations/layer22_attention_weight_max": 28.64720916748047,
      "activations/layer22_attention_weight_min": -26.850589752197266,
      "activations/layer23_attention_weight_max": 37.69405746459961,
      "activations/layer23_attention_weight_min": -23.90707015991211,
      "activations/layer2_attention_weight_max": 29.11795425415039,
      "activations/layer2_attention_weight_min": -28.514812469482422,
      "activations/layer3_attention_weight_max": 88.46173095703125,
      "activations/layer3_attention_weight_min": -85.9524917602539,
      "activations/layer4_attention_weight_max": 87.66165161132812,
      "activations/layer4_attention_weight_min": -85.4314193725586,
      "activations/layer5_attention_weight_max": 65.74964904785156,
      "activations/layer5_attention_weight_min": -68.27313232421875,
      "activations/layer6_attention_weight_max": 48.01494216918945,
      "activations/layer6_attention_weight_min": -49.16884231567383,
      "activations/layer7_attention_weight_max": 63.27668380737305,
      "activations/layer7_attention_weight_min": -61.808467864990234,
      "activations/layer8_attention_weight_max": 45.93252944946289,
      "activations/layer8_attention_weight_min": -50.4969482421875,
      "activations/layer9_attention_weight_max": 45.953590393066406,
      "activations/layer9_attention_weight_min": -48.1134147644043,
      "epoch": 17.09,
      "learning_rate": 4.0203787878787874e-05,
      "loss": 2.7374,
      "step": 294100
    },
    {
      "activations/layer0_attention_weight_max": 15.84319019317627,
      "activations/layer0_attention_weight_min": -13.80468463897705,
      "activations/layer10_attention_weight_max": 36.06696701049805,
      "activations/layer10_attention_weight_min": -35.833839416503906,
      "activations/layer11_attention_weight_max": 31.670394897460938,
      "activations/layer11_attention_weight_min": -34.93607711791992,
      "activations/layer12_attention_weight_max": 19.518184661865234,
      "activations/layer12_attention_weight_min": -22.75933265686035,
      "activations/layer13_attention_weight_max": 37.75242614746094,
      "activations/layer13_attention_weight_min": -28.65663719177246,
      "activations/layer14_attention_weight_max": 39.66232681274414,
      "activations/layer14_attention_weight_min": -30.169227600097656,
      "activations/layer15_attention_weight_max": 36.24595642089844,
      "activations/layer15_attention_weight_min": -29.900575637817383,
      "activations/layer16_attention_weight_max": 37.454261779785156,
      "activations/layer16_attention_weight_min": -33.38297653198242,
      "activations/layer17_attention_weight_max": 55.35823440551758,
      "activations/layer17_attention_weight_min": -47.52078628540039,
      "activations/layer18_attention_weight_max": 52.49981689453125,
      "activations/layer18_attention_weight_min": -43.52578353881836,
      "activations/layer19_attention_weight_max": 29.268918991088867,
      "activations/layer19_attention_weight_min": -23.180143356323242,
      "activations/layer1_attention_weight_max": 15.667688369750977,
      "activations/layer1_attention_weight_min": -14.729754447937012,
      "activations/layer20_attention_weight_max": 27.757884979248047,
      "activations/layer20_attention_weight_min": -21.54306983947754,
      "activations/layer21_attention_weight_max": 43.49956512451172,
      "activations/layer21_attention_weight_min": -24.177196502685547,
      "activations/layer22_attention_weight_max": 34.77519989013672,
      "activations/layer22_attention_weight_min": -26.505584716796875,
      "activations/layer23_attention_weight_max": 48.116851806640625,
      "activations/layer23_attention_weight_min": -25.252601623535156,
      "activations/layer2_attention_weight_max": 31.253833770751953,
      "activations/layer2_attention_weight_min": -28.812541961669922,
      "activations/layer3_attention_weight_max": 89.2921371459961,
      "activations/layer3_attention_weight_min": -94.59815979003906,
      "activations/layer4_attention_weight_max": 88.5274429321289,
      "activations/layer4_attention_weight_min": -89.7679443359375,
      "activations/layer5_attention_weight_max": 62.23334884643555,
      "activations/layer5_attention_weight_min": -78.26129913330078,
      "activations/layer6_attention_weight_max": 49.89152526855469,
      "activations/layer6_attention_weight_min": -52.7824821472168,
      "activations/layer7_attention_weight_max": 68.1573486328125,
      "activations/layer7_attention_weight_min": -66.32568359375,
      "activations/layer8_attention_weight_max": 44.15245056152344,
      "activations/layer8_attention_weight_min": -47.93037796020508,
      "activations/layer9_attention_weight_max": 42.5526008605957,
      "activations/layer9_attention_weight_min": -49.73996353149414,
      "epoch": 17.09,
      "learning_rate": 4.018484848484848e-05,
      "loss": 2.746,
      "step": 294150
    },
    {
      "activations/layer0_attention_weight_max": 15.231088638305664,
      "activations/layer0_attention_weight_min": -14.041275978088379,
      "activations/layer10_attention_weight_max": 37.456546783447266,
      "activations/layer10_attention_weight_min": -37.199981689453125,
      "activations/layer11_attention_weight_max": 34.67013168334961,
      "activations/layer11_attention_weight_min": -33.806602478027344,
      "activations/layer12_attention_weight_max": 18.306543350219727,
      "activations/layer12_attention_weight_min": -24.244529724121094,
      "activations/layer13_attention_weight_max": 32.07001876831055,
      "activations/layer13_attention_weight_min": -26.45467758178711,
      "activations/layer14_attention_weight_max": 34.46592712402344,
      "activations/layer14_attention_weight_min": -31.09050941467285,
      "activations/layer15_attention_weight_max": 31.990163803100586,
      "activations/layer15_attention_weight_min": -30.719266891479492,
      "activations/layer16_attention_weight_max": 33.8265266418457,
      "activations/layer16_attention_weight_min": -32.75870132446289,
      "activations/layer17_attention_weight_max": 48.741355895996094,
      "activations/layer17_attention_weight_min": -46.49777603149414,
      "activations/layer18_attention_weight_max": 44.47484588623047,
      "activations/layer18_attention_weight_min": -39.074676513671875,
      "activations/layer19_attention_weight_max": 24.712875366210938,
      "activations/layer19_attention_weight_min": -23.938077926635742,
      "activations/layer1_attention_weight_max": 16.530874252319336,
      "activations/layer1_attention_weight_min": -14.655017852783203,
      "activations/layer20_attention_weight_max": 24.053895950317383,
      "activations/layer20_attention_weight_min": -21.15037727355957,
      "activations/layer21_attention_weight_max": 33.217708587646484,
      "activations/layer21_attention_weight_min": -21.558868408203125,
      "activations/layer22_attention_weight_max": 31.614946365356445,
      "activations/layer22_attention_weight_min": -23.769929885864258,
      "activations/layer23_attention_weight_max": 33.921661376953125,
      "activations/layer23_attention_weight_min": -24.188133239746094,
      "activations/layer2_attention_weight_max": 31.715656280517578,
      "activations/layer2_attention_weight_min": -29.925186157226562,
      "activations/layer3_attention_weight_max": 90.69339752197266,
      "activations/layer3_attention_weight_min": -88.04570770263672,
      "activations/layer4_attention_weight_max": 89.72859191894531,
      "activations/layer4_attention_weight_min": -89.10196685791016,
      "activations/layer5_attention_weight_max": 63.602088928222656,
      "activations/layer5_attention_weight_min": -70.09539031982422,
      "activations/layer6_attention_weight_max": 49.23149490356445,
      "activations/layer6_attention_weight_min": -53.70424270629883,
      "activations/layer7_attention_weight_max": 67.21243286132812,
      "activations/layer7_attention_weight_min": -68.63111114501953,
      "activations/layer8_attention_weight_max": 46.78349304199219,
      "activations/layer8_attention_weight_min": -50.999351501464844,
      "activations/layer9_attention_weight_max": 47.25675582885742,
      "activations/layer9_attention_weight_min": -47.93096160888672,
      "epoch": 17.09,
      "learning_rate": 4.016590909090909e-05,
      "loss": 2.736,
      "step": 294200
    },
    {
      "activations/layer0_attention_weight_max": 14.705958366394043,
      "activations/layer0_attention_weight_min": -12.655721664428711,
      "activations/layer10_attention_weight_max": 35.687034606933594,
      "activations/layer10_attention_weight_min": -35.17670822143555,
      "activations/layer11_attention_weight_max": 32.892662048339844,
      "activations/layer11_attention_weight_min": -32.696998596191406,
      "activations/layer12_attention_weight_max": 19.42254638671875,
      "activations/layer12_attention_weight_min": -25.38652992248535,
      "activations/layer13_attention_weight_max": 34.369903564453125,
      "activations/layer13_attention_weight_min": -27.54497528076172,
      "activations/layer14_attention_weight_max": 35.918663024902344,
      "activations/layer14_attention_weight_min": -31.3239803314209,
      "activations/layer15_attention_weight_max": 34.055511474609375,
      "activations/layer15_attention_weight_min": -31.559059143066406,
      "activations/layer16_attention_weight_max": 36.22311019897461,
      "activations/layer16_attention_weight_min": -31.867948532104492,
      "activations/layer17_attention_weight_max": 51.758609771728516,
      "activations/layer17_attention_weight_min": -45.04469680786133,
      "activations/layer18_attention_weight_max": 53.12810134887695,
      "activations/layer18_attention_weight_min": -40.074466705322266,
      "activations/layer19_attention_weight_max": 24.075284957885742,
      "activations/layer19_attention_weight_min": -22.564115524291992,
      "activations/layer1_attention_weight_max": 15.618687629699707,
      "activations/layer1_attention_weight_min": -13.24345874786377,
      "activations/layer20_attention_weight_max": 25.289478302001953,
      "activations/layer20_attention_weight_min": -22.847179412841797,
      "activations/layer21_attention_weight_max": 38.1120491027832,
      "activations/layer21_attention_weight_min": -23.982450485229492,
      "activations/layer22_attention_weight_max": 29.50692367553711,
      "activations/layer22_attention_weight_min": -24.652782440185547,
      "activations/layer23_attention_weight_max": 37.45851516723633,
      "activations/layer23_attention_weight_min": -23.490280151367188,
      "activations/layer2_attention_weight_max": 30.571165084838867,
      "activations/layer2_attention_weight_min": -29.523672103881836,
      "activations/layer3_attention_weight_max": 85.44390106201172,
      "activations/layer3_attention_weight_min": -90.00609588623047,
      "activations/layer4_attention_weight_max": 85.59310150146484,
      "activations/layer4_attention_weight_min": -88.1361312866211,
      "activations/layer5_attention_weight_max": 62.7421989440918,
      "activations/layer5_attention_weight_min": -78.11170959472656,
      "activations/layer6_attention_weight_max": 52.69972229003906,
      "activations/layer6_attention_weight_min": -49.74644088745117,
      "activations/layer7_attention_weight_max": 63.77644348144531,
      "activations/layer7_attention_weight_min": -67.41622161865234,
      "activations/layer8_attention_weight_max": 46.16985321044922,
      "activations/layer8_attention_weight_min": -51.10000991821289,
      "activations/layer9_attention_weight_max": 44.14470672607422,
      "activations/layer9_attention_weight_min": -47.54074478149414,
      "epoch": 17.1,
      "learning_rate": 4.014696969696969e-05,
      "loss": 2.74,
      "step": 294250
    },
    {
      "activations/layer0_attention_weight_max": 17.19527816772461,
      "activations/layer0_attention_weight_min": -12.315434455871582,
      "activations/layer10_attention_weight_max": 34.63507843017578,
      "activations/layer10_attention_weight_min": -33.26304626464844,
      "activations/layer11_attention_weight_max": 31.37763786315918,
      "activations/layer11_attention_weight_min": -32.02777862548828,
      "activations/layer12_attention_weight_max": 18.58399772644043,
      "activations/layer12_attention_weight_min": -25.88127326965332,
      "activations/layer13_attention_weight_max": 32.03755569458008,
      "activations/layer13_attention_weight_min": -27.82599449157715,
      "activations/layer14_attention_weight_max": 34.668968200683594,
      "activations/layer14_attention_weight_min": -29.868867874145508,
      "activations/layer15_attention_weight_max": 31.042137145996094,
      "activations/layer15_attention_weight_min": -30.08521270751953,
      "activations/layer16_attention_weight_max": 32.87104034423828,
      "activations/layer16_attention_weight_min": -30.097084045410156,
      "activations/layer17_attention_weight_max": 48.193023681640625,
      "activations/layer17_attention_weight_min": -41.757080078125,
      "activations/layer18_attention_weight_max": 45.68082809448242,
      "activations/layer18_attention_weight_min": -39.35124969482422,
      "activations/layer19_attention_weight_max": 23.68622589111328,
      "activations/layer19_attention_weight_min": -22.39198112487793,
      "activations/layer1_attention_weight_max": 15.660806655883789,
      "activations/layer1_attention_weight_min": -13.736539840698242,
      "activations/layer20_attention_weight_max": 23.522106170654297,
      "activations/layer20_attention_weight_min": -22.496475219726562,
      "activations/layer21_attention_weight_max": 38.59831619262695,
      "activations/layer21_attention_weight_min": -20.69966697692871,
      "activations/layer22_attention_weight_max": 31.45467185974121,
      "activations/layer22_attention_weight_min": -27.902971267700195,
      "activations/layer23_attention_weight_max": 41.265323638916016,
      "activations/layer23_attention_weight_min": -24.534137725830078,
      "activations/layer2_attention_weight_max": 31.036659240722656,
      "activations/layer2_attention_weight_min": -29.491910934448242,
      "activations/layer3_attention_weight_max": 89.00969696044922,
      "activations/layer3_attention_weight_min": -90.0711441040039,
      "activations/layer4_attention_weight_max": 89.85933685302734,
      "activations/layer4_attention_weight_min": -90.84359741210938,
      "activations/layer5_attention_weight_max": 65.61264038085938,
      "activations/layer5_attention_weight_min": -76.3688735961914,
      "activations/layer6_attention_weight_max": 50.035255432128906,
      "activations/layer6_attention_weight_min": -51.59646987915039,
      "activations/layer7_attention_weight_max": 64.05726623535156,
      "activations/layer7_attention_weight_min": -61.3730583190918,
      "activations/layer8_attention_weight_max": 44.38383865356445,
      "activations/layer8_attention_weight_min": -48.45985794067383,
      "activations/layer9_attention_weight_max": 44.36346435546875,
      "activations/layer9_attention_weight_min": -43.56145095825195,
      "epoch": 17.1,
      "learning_rate": 4.01280303030303e-05,
      "loss": 2.7289,
      "step": 294300
    },
    {
      "activations/layer0_attention_weight_max": 16.395030975341797,
      "activations/layer0_attention_weight_min": -12.452804565429688,
      "activations/layer10_attention_weight_max": 36.27943801879883,
      "activations/layer10_attention_weight_min": -35.59804153442383,
      "activations/layer11_attention_weight_max": 33.43500518798828,
      "activations/layer11_attention_weight_min": -35.177494049072266,
      "activations/layer12_attention_weight_max": 21.17532730102539,
      "activations/layer12_attention_weight_min": -22.210309982299805,
      "activations/layer13_attention_weight_max": 42.256614685058594,
      "activations/layer13_attention_weight_min": -27.273662567138672,
      "activations/layer14_attention_weight_max": 37.58357238769531,
      "activations/layer14_attention_weight_min": -29.347379684448242,
      "activations/layer15_attention_weight_max": 34.50746154785156,
      "activations/layer15_attention_weight_min": -30.955699920654297,
      "activations/layer16_attention_weight_max": 33.075260162353516,
      "activations/layer16_attention_weight_min": -31.55995750427246,
      "activations/layer17_attention_weight_max": 50.89036178588867,
      "activations/layer17_attention_weight_min": -44.86641311645508,
      "activations/layer18_attention_weight_max": 44.40460205078125,
      "activations/layer18_attention_weight_min": -41.78596878051758,
      "activations/layer19_attention_weight_max": 23.856473922729492,
      "activations/layer19_attention_weight_min": -22.980695724487305,
      "activations/layer1_attention_weight_max": 16.00867462158203,
      "activations/layer1_attention_weight_min": -13.688097953796387,
      "activations/layer20_attention_weight_max": 23.010221481323242,
      "activations/layer20_attention_weight_min": -21.084115982055664,
      "activations/layer21_attention_weight_max": 39.1623420715332,
      "activations/layer21_attention_weight_min": -21.36136245727539,
      "activations/layer22_attention_weight_max": 30.488679885864258,
      "activations/layer22_attention_weight_min": -25.6351261138916,
      "activations/layer23_attention_weight_max": 37.16362380981445,
      "activations/layer23_attention_weight_min": -23.515600204467773,
      "activations/layer2_attention_weight_max": 30.410308837890625,
      "activations/layer2_attention_weight_min": -28.45924186706543,
      "activations/layer3_attention_weight_max": 86.35260772705078,
      "activations/layer3_attention_weight_min": -86.18208312988281,
      "activations/layer4_attention_weight_max": 85.87251281738281,
      "activations/layer4_attention_weight_min": -89.63370513916016,
      "activations/layer5_attention_weight_max": 65.40335083007812,
      "activations/layer5_attention_weight_min": -72.78555297851562,
      "activations/layer6_attention_weight_max": 49.126102447509766,
      "activations/layer6_attention_weight_min": -49.35234069824219,
      "activations/layer7_attention_weight_max": 65.19581604003906,
      "activations/layer7_attention_weight_min": -64.54327392578125,
      "activations/layer8_attention_weight_max": 47.96119689941406,
      "activations/layer8_attention_weight_min": -49.95353698730469,
      "activations/layer9_attention_weight_max": 46.315059661865234,
      "activations/layer9_attention_weight_min": -46.31721496582031,
      "epoch": 17.1,
      "learning_rate": 4.010909090909091e-05,
      "loss": 2.7387,
      "step": 294350
    },
    {
      "activations/layer0_attention_weight_max": 16.17579460144043,
      "activations/layer0_attention_weight_min": -12.69899845123291,
      "activations/layer10_attention_weight_max": 36.14546585083008,
      "activations/layer10_attention_weight_min": -35.888580322265625,
      "activations/layer11_attention_weight_max": 32.134483337402344,
      "activations/layer11_attention_weight_min": -33.47637939453125,
      "activations/layer12_attention_weight_max": 16.659502029418945,
      "activations/layer12_attention_weight_min": -26.978960037231445,
      "activations/layer13_attention_weight_max": 31.520227432250977,
      "activations/layer13_attention_weight_min": -27.644920349121094,
      "activations/layer14_attention_weight_max": 32.13850021362305,
      "activations/layer14_attention_weight_min": -29.971830368041992,
      "activations/layer15_attention_weight_max": 29.95884895324707,
      "activations/layer15_attention_weight_min": -31.081886291503906,
      "activations/layer16_attention_weight_max": 32.3924560546875,
      "activations/layer16_attention_weight_min": -32.339847564697266,
      "activations/layer17_attention_weight_max": 50.13208770751953,
      "activations/layer17_attention_weight_min": -45.41411590576172,
      "activations/layer18_attention_weight_max": 44.834354400634766,
      "activations/layer18_attention_weight_min": -40.84734344482422,
      "activations/layer19_attention_weight_max": 23.81891632080078,
      "activations/layer19_attention_weight_min": -23.57758903503418,
      "activations/layer1_attention_weight_max": 15.895122528076172,
      "activations/layer1_attention_weight_min": -13.479293823242188,
      "activations/layer20_attention_weight_max": 21.936697006225586,
      "activations/layer20_attention_weight_min": -22.452678680419922,
      "activations/layer21_attention_weight_max": 33.1881103515625,
      "activations/layer21_attention_weight_min": -20.73771858215332,
      "activations/layer22_attention_weight_max": 27.321001052856445,
      "activations/layer22_attention_weight_min": -25.16484260559082,
      "activations/layer23_attention_weight_max": 32.82845687866211,
      "activations/layer23_attention_weight_min": -23.914520263671875,
      "activations/layer2_attention_weight_max": 31.01521873474121,
      "activations/layer2_attention_weight_min": -29.417049407958984,
      "activations/layer3_attention_weight_max": 87.73085021972656,
      "activations/layer3_attention_weight_min": -84.50017547607422,
      "activations/layer4_attention_weight_max": 86.31971740722656,
      "activations/layer4_attention_weight_min": -86.5588607788086,
      "activations/layer5_attention_weight_max": 61.930091857910156,
      "activations/layer5_attention_weight_min": -74.0836181640625,
      "activations/layer6_attention_weight_max": 50.72465515136719,
      "activations/layer6_attention_weight_min": -51.97541427612305,
      "activations/layer7_attention_weight_max": 64.29811096191406,
      "activations/layer7_attention_weight_min": -64.90513610839844,
      "activations/layer8_attention_weight_max": 47.37028121948242,
      "activations/layer8_attention_weight_min": -49.44234085083008,
      "activations/layer9_attention_weight_max": 44.003658294677734,
      "activations/layer9_attention_weight_min": -46.462158203125,
      "epoch": 17.11,
      "learning_rate": 4.00905303030303e-05,
      "loss": 2.7503,
      "step": 294400
    },
    {
      "activations/layer0_attention_weight_max": 15.046998023986816,
      "activations/layer0_attention_weight_min": -12.543659210205078,
      "activations/layer10_attention_weight_max": 41.94449996948242,
      "activations/layer10_attention_weight_min": -43.95609664916992,
      "activations/layer11_attention_weight_max": 38.5980110168457,
      "activations/layer11_attention_weight_min": -37.86114501953125,
      "activations/layer12_attention_weight_max": 19.444427490234375,
      "activations/layer12_attention_weight_min": -25.019346237182617,
      "activations/layer13_attention_weight_max": 36.5812873840332,
      "activations/layer13_attention_weight_min": -30.90882110595703,
      "activations/layer14_attention_weight_max": 36.67890930175781,
      "activations/layer14_attention_weight_min": -31.96381950378418,
      "activations/layer15_attention_weight_max": 36.82671356201172,
      "activations/layer15_attention_weight_min": -34.35609817504883,
      "activations/layer16_attention_weight_max": 37.21194839477539,
      "activations/layer16_attention_weight_min": -34.07771682739258,
      "activations/layer17_attention_weight_max": 52.69013595581055,
      "activations/layer17_attention_weight_min": -50.71962356567383,
      "activations/layer18_attention_weight_max": 47.43943405151367,
      "activations/layer18_attention_weight_min": -44.9078254699707,
      "activations/layer19_attention_weight_max": 27.507434844970703,
      "activations/layer19_attention_weight_min": -27.73724937438965,
      "activations/layer1_attention_weight_max": 16.118600845336914,
      "activations/layer1_attention_weight_min": -13.344639778137207,
      "activations/layer20_attention_weight_max": 25.526248931884766,
      "activations/layer20_attention_weight_min": -26.933225631713867,
      "activations/layer21_attention_weight_max": 40.163787841796875,
      "activations/layer21_attention_weight_min": -30.115604400634766,
      "activations/layer22_attention_weight_max": 31.62238311767578,
      "activations/layer22_attention_weight_min": -28.1370792388916,
      "activations/layer23_attention_weight_max": 39.98598098754883,
      "activations/layer23_attention_weight_min": -25.918926239013672,
      "activations/layer2_attention_weight_max": 31.121566772460938,
      "activations/layer2_attention_weight_min": -30.23586654663086,
      "activations/layer3_attention_weight_max": 86.33236694335938,
      "activations/layer3_attention_weight_min": -90.98892211914062,
      "activations/layer4_attention_weight_max": 87.54236602783203,
      "activations/layer4_attention_weight_min": -85.5562973022461,
      "activations/layer5_attention_weight_max": 64.96873474121094,
      "activations/layer5_attention_weight_min": -73.18087768554688,
      "activations/layer6_attention_weight_max": 49.254756927490234,
      "activations/layer6_attention_weight_min": -50.057838439941406,
      "activations/layer7_attention_weight_max": 68.64981079101562,
      "activations/layer7_attention_weight_min": -70.48538208007812,
      "activations/layer8_attention_weight_max": 53.13707733154297,
      "activations/layer8_attention_weight_min": -56.37166213989258,
      "activations/layer9_attention_weight_max": 56.34197998046875,
      "activations/layer9_attention_weight_min": -52.30295944213867,
      "epoch": 17.11,
      "learning_rate": 4.00715909090909e-05,
      "loss": 2.7307,
      "step": 294450
    },
    {
      "activations/layer0_attention_weight_max": 15.700836181640625,
      "activations/layer0_attention_weight_min": -12.644104957580566,
      "activations/layer10_attention_weight_max": 47.386802673339844,
      "activations/layer10_attention_weight_min": -45.66281509399414,
      "activations/layer11_attention_weight_max": 44.289756774902344,
      "activations/layer11_attention_weight_min": -40.45988082885742,
      "activations/layer12_attention_weight_max": 19.861257553100586,
      "activations/layer12_attention_weight_min": -27.653032302856445,
      "activations/layer13_attention_weight_max": 54.36820602416992,
      "activations/layer13_attention_weight_min": -36.6151237487793,
      "activations/layer14_attention_weight_max": 42.8594856262207,
      "activations/layer14_attention_weight_min": -33.56053924560547,
      "activations/layer15_attention_weight_max": 46.64339828491211,
      "activations/layer15_attention_weight_min": -35.36227798461914,
      "activations/layer16_attention_weight_max": 32.23981857299805,
      "activations/layer16_attention_weight_min": -29.675962448120117,
      "activations/layer17_attention_weight_max": 49.09381866455078,
      "activations/layer17_attention_weight_min": -45.33159637451172,
      "activations/layer18_attention_weight_max": 45.052528381347656,
      "activations/layer18_attention_weight_min": -41.475765228271484,
      "activations/layer19_attention_weight_max": 21.887914657592773,
      "activations/layer19_attention_weight_min": -24.9376220703125,
      "activations/layer1_attention_weight_max": 16.673736572265625,
      "activations/layer1_attention_weight_min": -14.415253639221191,
      "activations/layer20_attention_weight_max": 20.576719284057617,
      "activations/layer20_attention_weight_min": -22.67754364013672,
      "activations/layer21_attention_weight_max": 28.563552856445312,
      "activations/layer21_attention_weight_min": -25.855051040649414,
      "activations/layer22_attention_weight_max": 27.61539649963379,
      "activations/layer22_attention_weight_min": -26.62140655517578,
      "activations/layer23_attention_weight_max": 36.43512725830078,
      "activations/layer23_attention_weight_min": -25.29364776611328,
      "activations/layer2_attention_weight_max": 31.30752182006836,
      "activations/layer2_attention_weight_min": -31.02341079711914,
      "activations/layer3_attention_weight_max": 89.30706024169922,
      "activations/layer3_attention_weight_min": -85.03591918945312,
      "activations/layer4_attention_weight_max": 91.2947006225586,
      "activations/layer4_attention_weight_min": -88.6758041381836,
      "activations/layer5_attention_weight_max": 67.86953735351562,
      "activations/layer5_attention_weight_min": -75.99968719482422,
      "activations/layer6_attention_weight_max": 52.16172790527344,
      "activations/layer6_attention_weight_min": -50.107662200927734,
      "activations/layer7_attention_weight_max": 81.26765441894531,
      "activations/layer7_attention_weight_min": -70.40507507324219,
      "activations/layer8_attention_weight_max": 58.61452865600586,
      "activations/layer8_attention_weight_min": -55.10859680175781,
      "activations/layer9_attention_weight_max": 66.94520568847656,
      "activations/layer9_attention_weight_min": -55.14716720581055,
      "epoch": 17.11,
      "learning_rate": 4.0052651515151516e-05,
      "loss": 2.7388,
      "step": 294500
    },
    {
      "activations/layer0_attention_weight_max": 14.851962089538574,
      "activations/layer0_attention_weight_min": -12.70038890838623,
      "activations/layer10_attention_weight_max": 38.72633743286133,
      "activations/layer10_attention_weight_min": -40.8037109375,
      "activations/layer11_attention_weight_max": 35.12779998779297,
      "activations/layer11_attention_weight_min": -37.02876281738281,
      "activations/layer12_attention_weight_max": 18.725244522094727,
      "activations/layer12_attention_weight_min": -22.607547760009766,
      "activations/layer13_attention_weight_max": 35.67100524902344,
      "activations/layer13_attention_weight_min": -29.520601272583008,
      "activations/layer14_attention_weight_max": 33.637969970703125,
      "activations/layer14_attention_weight_min": -30.137989044189453,
      "activations/layer15_attention_weight_max": 32.26705551147461,
      "activations/layer15_attention_weight_min": -29.494091033935547,
      "activations/layer16_attention_weight_max": 31.743289947509766,
      "activations/layer16_attention_weight_min": -29.921802520751953,
      "activations/layer17_attention_weight_max": 49.756935119628906,
      "activations/layer17_attention_weight_min": -44.31911087036133,
      "activations/layer18_attention_weight_max": 44.9451904296875,
      "activations/layer18_attention_weight_min": -38.70841979980469,
      "activations/layer19_attention_weight_max": 21.405485153198242,
      "activations/layer19_attention_weight_min": -21.897315979003906,
      "activations/layer1_attention_weight_max": 16.23392105102539,
      "activations/layer1_attention_weight_min": -14.476903915405273,
      "activations/layer20_attention_weight_max": 21.318805694580078,
      "activations/layer20_attention_weight_min": -26.111351013183594,
      "activations/layer21_attention_weight_max": 32.790645599365234,
      "activations/layer21_attention_weight_min": -25.645042419433594,
      "activations/layer22_attention_weight_max": 27.65146255493164,
      "activations/layer22_attention_weight_min": -26.766265869140625,
      "activations/layer23_attention_weight_max": 34.93545150756836,
      "activations/layer23_attention_weight_min": -23.473682403564453,
      "activations/layer2_attention_weight_max": 30.740264892578125,
      "activations/layer2_attention_weight_min": -30.67257308959961,
      "activations/layer3_attention_weight_max": 88.57258605957031,
      "activations/layer3_attention_weight_min": -89.13579559326172,
      "activations/layer4_attention_weight_max": 94.34849548339844,
      "activations/layer4_attention_weight_min": -95.19854736328125,
      "activations/layer5_attention_weight_max": 62.73567199707031,
      "activations/layer5_attention_weight_min": -74.41582489013672,
      "activations/layer6_attention_weight_max": 53.979774475097656,
      "activations/layer6_attention_weight_min": -54.10149002075195,
      "activations/layer7_attention_weight_max": 65.33108520507812,
      "activations/layer7_attention_weight_min": -66.91413116455078,
      "activations/layer8_attention_weight_max": 47.77101516723633,
      "activations/layer8_attention_weight_min": -52.70903778076172,
      "activations/layer9_attention_weight_max": 45.984222412109375,
      "activations/layer9_attention_weight_min": -47.59511184692383,
      "epoch": 17.12,
      "learning_rate": 4.003371212121212e-05,
      "loss": 2.7391,
      "step": 294550
    },
    {
      "activations/layer0_attention_weight_max": 16.12858009338379,
      "activations/layer0_attention_weight_min": -12.397953987121582,
      "activations/layer10_attention_weight_max": 36.674068450927734,
      "activations/layer10_attention_weight_min": -37.49696731567383,
      "activations/layer11_attention_weight_max": 32.531341552734375,
      "activations/layer11_attention_weight_min": -35.03174591064453,
      "activations/layer12_attention_weight_max": 18.768999099731445,
      "activations/layer12_attention_weight_min": -25.22174835205078,
      "activations/layer13_attention_weight_max": 42.11912536621094,
      "activations/layer13_attention_weight_min": -27.2383975982666,
      "activations/layer14_attention_weight_max": 52.71648406982422,
      "activations/layer14_attention_weight_min": -30.89643669128418,
      "activations/layer15_attention_weight_max": 38.927207946777344,
      "activations/layer15_attention_weight_min": -30.347837448120117,
      "activations/layer16_attention_weight_max": 34.092742919921875,
      "activations/layer16_attention_weight_min": -33.02667236328125,
      "activations/layer17_attention_weight_max": 50.25053405761719,
      "activations/layer17_attention_weight_min": -45.02640914916992,
      "activations/layer18_attention_weight_max": 50.11758804321289,
      "activations/layer18_attention_weight_min": -40.961917877197266,
      "activations/layer19_attention_weight_max": 24.09235191345215,
      "activations/layer19_attention_weight_min": -25.215129852294922,
      "activations/layer1_attention_weight_max": 16.644908905029297,
      "activations/layer1_attention_weight_min": -13.428864479064941,
      "activations/layer20_attention_weight_max": 24.969820022583008,
      "activations/layer20_attention_weight_min": -21.38029670715332,
      "activations/layer21_attention_weight_max": 38.283050537109375,
      "activations/layer21_attention_weight_min": -24.7416934967041,
      "activations/layer22_attention_weight_max": 31.586877822875977,
      "activations/layer22_attention_weight_min": -25.99958038330078,
      "activations/layer23_attention_weight_max": 36.353763580322266,
      "activations/layer23_attention_weight_min": -25.671186447143555,
      "activations/layer2_attention_weight_max": 31.645572662353516,
      "activations/layer2_attention_weight_min": -30.462066650390625,
      "activations/layer3_attention_weight_max": 87.24626922607422,
      "activations/layer3_attention_weight_min": -86.0511245727539,
      "activations/layer4_attention_weight_max": 86.89484405517578,
      "activations/layer4_attention_weight_min": -82.1495590209961,
      "activations/layer5_attention_weight_max": 66.4462890625,
      "activations/layer5_attention_weight_min": -69.65419006347656,
      "activations/layer6_attention_weight_max": 49.72789764404297,
      "activations/layer6_attention_weight_min": -50.53153610229492,
      "activations/layer7_attention_weight_max": 63.54684829711914,
      "activations/layer7_attention_weight_min": -61.57939910888672,
      "activations/layer8_attention_weight_max": 44.680091857910156,
      "activations/layer8_attention_weight_min": -49.592037200927734,
      "activations/layer9_attention_weight_max": 42.999114990234375,
      "activations/layer9_attention_weight_min": -43.73712921142578,
      "epoch": 17.12,
      "learning_rate": 4.0014772727272726e-05,
      "loss": 2.7373,
      "step": 294600
    },
    {
      "activations/layer0_attention_weight_max": 15.399711608886719,
      "activations/layer0_attention_weight_min": -12.700759887695312,
      "activations/layer10_attention_weight_max": 38.51433181762695,
      "activations/layer10_attention_weight_min": -35.540496826171875,
      "activations/layer11_attention_weight_max": 40.117027282714844,
      "activations/layer11_attention_weight_min": -33.50895690917969,
      "activations/layer12_attention_weight_max": 20.17999839782715,
      "activations/layer12_attention_weight_min": -24.12578582763672,
      "activations/layer13_attention_weight_max": 37.1054801940918,
      "activations/layer13_attention_weight_min": -33.197017669677734,
      "activations/layer14_attention_weight_max": 39.94593811035156,
      "activations/layer14_attention_weight_min": -31.712318420410156,
      "activations/layer15_attention_weight_max": 39.421443939208984,
      "activations/layer15_attention_weight_min": -32.266902923583984,
      "activations/layer16_attention_weight_max": 31.444339752197266,
      "activations/layer16_attention_weight_min": -28.936416625976562,
      "activations/layer17_attention_weight_max": 53.33274459838867,
      "activations/layer17_attention_weight_min": -46.273006439208984,
      "activations/layer18_attention_weight_max": 46.354984283447266,
      "activations/layer18_attention_weight_min": -45.4365348815918,
      "activations/layer19_attention_weight_max": 24.760255813598633,
      "activations/layer19_attention_weight_min": -25.631460189819336,
      "activations/layer1_attention_weight_max": 15.889403343200684,
      "activations/layer1_attention_weight_min": -14.506988525390625,
      "activations/layer20_attention_weight_max": 24.174402236938477,
      "activations/layer20_attention_weight_min": -25.25604248046875,
      "activations/layer21_attention_weight_max": 38.12763595581055,
      "activations/layer21_attention_weight_min": -22.503087997436523,
      "activations/layer22_attention_weight_max": 31.77623176574707,
      "activations/layer22_attention_weight_min": -25.931312561035156,
      "activations/layer23_attention_weight_max": 33.26812744140625,
      "activations/layer23_attention_weight_min": -24.270689010620117,
      "activations/layer2_attention_weight_max": 32.64278793334961,
      "activations/layer2_attention_weight_min": -31.292015075683594,
      "activations/layer3_attention_weight_max": 89.4801025390625,
      "activations/layer3_attention_weight_min": -94.65347290039062,
      "activations/layer4_attention_weight_max": 89.41307830810547,
      "activations/layer4_attention_weight_min": -84.52767181396484,
      "activations/layer5_attention_weight_max": 68.03932189941406,
      "activations/layer5_attention_weight_min": -74.42110443115234,
      "activations/layer6_attention_weight_max": 49.73995590209961,
      "activations/layer6_attention_weight_min": -50.535926818847656,
      "activations/layer7_attention_weight_max": 72.1594467163086,
      "activations/layer7_attention_weight_min": -64.90213012695312,
      "activations/layer8_attention_weight_max": 52.47305679321289,
      "activations/layer8_attention_weight_min": -47.50695037841797,
      "activations/layer9_attention_weight_max": 54.17013168334961,
      "activations/layer9_attention_weight_min": -53.0395622253418,
      "epoch": 17.12,
      "learning_rate": 3.999583333333333e-05,
      "loss": 2.7362,
      "step": 294650
    },
    {
      "activations/layer0_attention_weight_max": 15.539799690246582,
      "activations/layer0_attention_weight_min": -12.809408187866211,
      "activations/layer10_attention_weight_max": 35.645843505859375,
      "activations/layer10_attention_weight_min": -36.23286056518555,
      "activations/layer11_attention_weight_max": 31.4832763671875,
      "activations/layer11_attention_weight_min": -33.862972259521484,
      "activations/layer12_attention_weight_max": 19.38178825378418,
      "activations/layer12_attention_weight_min": -25.131650924682617,
      "activations/layer13_attention_weight_max": 36.99009704589844,
      "activations/layer13_attention_weight_min": -26.367082595825195,
      "activations/layer14_attention_weight_max": 36.73445129394531,
      "activations/layer14_attention_weight_min": -29.073486328125,
      "activations/layer15_attention_weight_max": 32.69395446777344,
      "activations/layer15_attention_weight_min": -31.243318557739258,
      "activations/layer16_attention_weight_max": 32.20191955566406,
      "activations/layer16_attention_weight_min": -30.687158584594727,
      "activations/layer17_attention_weight_max": 48.16633987426758,
      "activations/layer17_attention_weight_min": -43.87651443481445,
      "activations/layer18_attention_weight_max": 43.40240478515625,
      "activations/layer18_attention_weight_min": -41.41295623779297,
      "activations/layer19_attention_weight_max": 23.55506706237793,
      "activations/layer19_attention_weight_min": -24.91744613647461,
      "activations/layer1_attention_weight_max": 16.287206649780273,
      "activations/layer1_attention_weight_min": -16.392967224121094,
      "activations/layer20_attention_weight_max": 23.92604637145996,
      "activations/layer20_attention_weight_min": -23.298006057739258,
      "activations/layer21_attention_weight_max": 38.925899505615234,
      "activations/layer21_attention_weight_min": -23.515789031982422,
      "activations/layer22_attention_weight_max": 27.805280685424805,
      "activations/layer22_attention_weight_min": -25.366548538208008,
      "activations/layer23_attention_weight_max": 38.63093566894531,
      "activations/layer23_attention_weight_min": -25.612045288085938,
      "activations/layer2_attention_weight_max": 30.547630310058594,
      "activations/layer2_attention_weight_min": -30.973613739013672,
      "activations/layer3_attention_weight_max": 83.07774353027344,
      "activations/layer3_attention_weight_min": -87.969970703125,
      "activations/layer4_attention_weight_max": 84.14277648925781,
      "activations/layer4_attention_weight_min": -86.59026336669922,
      "activations/layer5_attention_weight_max": 62.60464096069336,
      "activations/layer5_attention_weight_min": -71.95716857910156,
      "activations/layer6_attention_weight_max": 46.87416076660156,
      "activations/layer6_attention_weight_min": -50.896366119384766,
      "activations/layer7_attention_weight_max": 61.19780731201172,
      "activations/layer7_attention_weight_min": -61.843807220458984,
      "activations/layer8_attention_weight_max": 43.8868293762207,
      "activations/layer8_attention_weight_min": -50.20967102050781,
      "activations/layer9_attention_weight_max": 43.76420593261719,
      "activations/layer9_attention_weight_min": -47.56623840332031,
      "epoch": 17.12,
      "learning_rate": 3.997689393939394e-05,
      "loss": 2.7451,
      "step": 294700
    },
    {
      "activations/layer0_attention_weight_max": 15.628976821899414,
      "activations/layer0_attention_weight_min": -12.845885276794434,
      "activations/layer10_attention_weight_max": 37.376338958740234,
      "activations/layer10_attention_weight_min": -36.87506103515625,
      "activations/layer11_attention_weight_max": 33.84019088745117,
      "activations/layer11_attention_weight_min": -33.98664855957031,
      "activations/layer12_attention_weight_max": 18.7663631439209,
      "activations/layer12_attention_weight_min": -28.151081085205078,
      "activations/layer13_attention_weight_max": 29.861351013183594,
      "activations/layer13_attention_weight_min": -25.237756729125977,
      "activations/layer14_attention_weight_max": 34.46335983276367,
      "activations/layer14_attention_weight_min": -28.315465927124023,
      "activations/layer15_attention_weight_max": 31.991539001464844,
      "activations/layer15_attention_weight_min": -28.689165115356445,
      "activations/layer16_attention_weight_max": 29.648540496826172,
      "activations/layer16_attention_weight_min": -30.4410400390625,
      "activations/layer17_attention_weight_max": 45.49028396606445,
      "activations/layer17_attention_weight_min": -41.12465286254883,
      "activations/layer18_attention_weight_max": 40.29764938354492,
      "activations/layer18_attention_weight_min": -37.32719802856445,
      "activations/layer19_attention_weight_max": 23.31362533569336,
      "activations/layer19_attention_weight_min": -22.31698989868164,
      "activations/layer1_attention_weight_max": 15.352466583251953,
      "activations/layer1_attention_weight_min": -14.719539642333984,
      "activations/layer20_attention_weight_max": 20.41878318786621,
      "activations/layer20_attention_weight_min": -21.34439468383789,
      "activations/layer21_attention_weight_max": 30.714244842529297,
      "activations/layer21_attention_weight_min": -21.46309471130371,
      "activations/layer22_attention_weight_max": 26.12213706970215,
      "activations/layer22_attention_weight_min": -24.54328155517578,
      "activations/layer23_attention_weight_max": 32.90467834472656,
      "activations/layer23_attention_weight_min": -22.95076560974121,
      "activations/layer2_attention_weight_max": 33.598995208740234,
      "activations/layer2_attention_weight_min": -33.343238830566406,
      "activations/layer3_attention_weight_max": 87.53173065185547,
      "activations/layer3_attention_weight_min": -88.93144989013672,
      "activations/layer4_attention_weight_max": 87.71742248535156,
      "activations/layer4_attention_weight_min": -88.80168151855469,
      "activations/layer5_attention_weight_max": 64.27703094482422,
      "activations/layer5_attention_weight_min": -75.85163116455078,
      "activations/layer6_attention_weight_max": 51.20766830444336,
      "activations/layer6_attention_weight_min": -54.5627326965332,
      "activations/layer7_attention_weight_max": 66.18801879882812,
      "activations/layer7_attention_weight_min": -66.84962463378906,
      "activations/layer8_attention_weight_max": 48.681884765625,
      "activations/layer8_attention_weight_min": -53.58174133300781,
      "activations/layer9_attention_weight_max": 43.39565658569336,
      "activations/layer9_attention_weight_min": -46.96653366088867,
      "epoch": 17.13,
      "learning_rate": 3.9957954545454544e-05,
      "loss": 2.731,
      "step": 294750
    },
    {
      "activations/layer0_attention_weight_max": 14.647741317749023,
      "activations/layer0_attention_weight_min": -12.525575637817383,
      "activations/layer10_attention_weight_max": 36.4234733581543,
      "activations/layer10_attention_weight_min": -36.72282791137695,
      "activations/layer11_attention_weight_max": 34.7639045715332,
      "activations/layer11_attention_weight_min": -34.718544006347656,
      "activations/layer12_attention_weight_max": 20.819580078125,
      "activations/layer12_attention_weight_min": -23.698516845703125,
      "activations/layer13_attention_weight_max": 42.508575439453125,
      "activations/layer13_attention_weight_min": -33.46892547607422,
      "activations/layer14_attention_weight_max": 38.506919860839844,
      "activations/layer14_attention_weight_min": -33.35438537597656,
      "activations/layer15_attention_weight_max": 35.94508361816406,
      "activations/layer15_attention_weight_min": -32.629188537597656,
      "activations/layer16_attention_weight_max": 36.548309326171875,
      "activations/layer16_attention_weight_min": -33.864803314208984,
      "activations/layer17_attention_weight_max": 55.151973724365234,
      "activations/layer17_attention_weight_min": -48.33916091918945,
      "activations/layer18_attention_weight_max": 51.13801574707031,
      "activations/layer18_attention_weight_min": -45.60028839111328,
      "activations/layer19_attention_weight_max": 25.13681983947754,
      "activations/layer19_attention_weight_min": -25.55582618713379,
      "activations/layer1_attention_weight_max": 17.34715461730957,
      "activations/layer1_attention_weight_min": -16.57942008972168,
      "activations/layer20_attention_weight_max": 25.834638595581055,
      "activations/layer20_attention_weight_min": -26.334177017211914,
      "activations/layer21_attention_weight_max": 35.755069732666016,
      "activations/layer21_attention_weight_min": -27.506399154663086,
      "activations/layer22_attention_weight_max": 32.51313781738281,
      "activations/layer22_attention_weight_min": -27.604381561279297,
      "activations/layer23_attention_weight_max": 38.609825134277344,
      "activations/layer23_attention_weight_min": -26.079679489135742,
      "activations/layer2_attention_weight_max": 34.1053466796875,
      "activations/layer2_attention_weight_min": -31.460935592651367,
      "activations/layer3_attention_weight_max": 91.49160766601562,
      "activations/layer3_attention_weight_min": -87.09600830078125,
      "activations/layer4_attention_weight_max": 94.23210906982422,
      "activations/layer4_attention_weight_min": -90.91593933105469,
      "activations/layer5_attention_weight_max": 66.75775146484375,
      "activations/layer5_attention_weight_min": -74.16145324707031,
      "activations/layer6_attention_weight_max": 52.156455993652344,
      "activations/layer6_attention_weight_min": -50.87776184082031,
      "activations/layer7_attention_weight_max": 66.79708862304688,
      "activations/layer7_attention_weight_min": -68.35318756103516,
      "activations/layer8_attention_weight_max": 50.04410171508789,
      "activations/layer8_attention_weight_min": -52.307186126708984,
      "activations/layer9_attention_weight_max": 44.487003326416016,
      "activations/layer9_attention_weight_min": -47.132354736328125,
      "epoch": 17.13,
      "learning_rate": 3.9939015151515146e-05,
      "loss": 2.7322,
      "step": 294800
    },
    {
      "activations/layer0_attention_weight_max": 14.832574844360352,
      "activations/layer0_attention_weight_min": -12.603720664978027,
      "activations/layer10_attention_weight_max": 32.91023635864258,
      "activations/layer10_attention_weight_min": -35.8546142578125,
      "activations/layer11_attention_weight_max": 31.50326156616211,
      "activations/layer11_attention_weight_min": -32.678993225097656,
      "activations/layer12_attention_weight_max": 18.4146671295166,
      "activations/layer12_attention_weight_min": -23.75925636291504,
      "activations/layer13_attention_weight_max": 32.68514633178711,
      "activations/layer13_attention_weight_min": -29.63245391845703,
      "activations/layer14_attention_weight_max": 33.7366943359375,
      "activations/layer14_attention_weight_min": -32.126983642578125,
      "activations/layer15_attention_weight_max": 31.86284637451172,
      "activations/layer15_attention_weight_min": -30.90407943725586,
      "activations/layer16_attention_weight_max": 34.384552001953125,
      "activations/layer16_attention_weight_min": -32.3180046081543,
      "activations/layer17_attention_weight_max": 53.378662109375,
      "activations/layer17_attention_weight_min": -47.41460418701172,
      "activations/layer18_attention_weight_max": 49.208213806152344,
      "activations/layer18_attention_weight_min": -42.39133071899414,
      "activations/layer19_attention_weight_max": 25.031848907470703,
      "activations/layer19_attention_weight_min": -24.95145034790039,
      "activations/layer1_attention_weight_max": 15.832498550415039,
      "activations/layer1_attention_weight_min": -16.367368698120117,
      "activations/layer20_attention_weight_max": 24.389820098876953,
      "activations/layer20_attention_weight_min": -22.91358184814453,
      "activations/layer21_attention_weight_max": 39.960330963134766,
      "activations/layer21_attention_weight_min": -23.44261360168457,
      "activations/layer22_attention_weight_max": 29.703821182250977,
      "activations/layer22_attention_weight_min": -26.60064125061035,
      "activations/layer23_attention_weight_max": 38.39357376098633,
      "activations/layer23_attention_weight_min": -27.015060424804688,
      "activations/layer2_attention_weight_max": 32.83192443847656,
      "activations/layer2_attention_weight_min": -28.777843475341797,
      "activations/layer3_attention_weight_max": 85.37380981445312,
      "activations/layer3_attention_weight_min": -83.3047103881836,
      "activations/layer4_attention_weight_max": 83.62364959716797,
      "activations/layer4_attention_weight_min": -83.49637603759766,
      "activations/layer5_attention_weight_max": 62.1036376953125,
      "activations/layer5_attention_weight_min": -69.52384185791016,
      "activations/layer6_attention_weight_max": 46.63919448852539,
      "activations/layer6_attention_weight_min": -47.99440383911133,
      "activations/layer7_attention_weight_max": 63.08915328979492,
      "activations/layer7_attention_weight_min": -60.758243560791016,
      "activations/layer8_attention_weight_max": 40.17929458618164,
      "activations/layer8_attention_weight_min": -45.2560920715332,
      "activations/layer9_attention_weight_max": 42.28252029418945,
      "activations/layer9_attention_weight_min": -45.517765045166016,
      "epoch": 17.13,
      "learning_rate": 3.9920075757575754e-05,
      "loss": 2.7631,
      "step": 294850
    },
    {
      "activations/layer0_attention_weight_max": 16.041183471679688,
      "activations/layer0_attention_weight_min": -12.600288391113281,
      "activations/layer10_attention_weight_max": 39.1175422668457,
      "activations/layer10_attention_weight_min": -36.49870300292969,
      "activations/layer11_attention_weight_max": 36.335453033447266,
      "activations/layer11_attention_weight_min": -36.368534088134766,
      "activations/layer12_attention_weight_max": 21.2072811126709,
      "activations/layer12_attention_weight_min": -25.655216217041016,
      "activations/layer13_attention_weight_max": 37.53776168823242,
      "activations/layer13_attention_weight_min": -30.429893493652344,
      "activations/layer14_attention_weight_max": 36.99209976196289,
      "activations/layer14_attention_weight_min": -30.704072952270508,
      "activations/layer15_attention_weight_max": 34.37251663208008,
      "activations/layer15_attention_weight_min": -31.93604278564453,
      "activations/layer16_attention_weight_max": 36.219993591308594,
      "activations/layer16_attention_weight_min": -33.44903564453125,
      "activations/layer17_attention_weight_max": 55.58677673339844,
      "activations/layer17_attention_weight_min": -45.6767692565918,
      "activations/layer18_attention_weight_max": 51.35314178466797,
      "activations/layer18_attention_weight_min": -41.58039855957031,
      "activations/layer19_attention_weight_max": 26.47994041442871,
      "activations/layer19_attention_weight_min": -24.31730079650879,
      "activations/layer1_attention_weight_max": 16.110774993896484,
      "activations/layer1_attention_weight_min": -14.464200973510742,
      "activations/layer20_attention_weight_max": 24.595396041870117,
      "activations/layer20_attention_weight_min": -25.53023910522461,
      "activations/layer21_attention_weight_max": 41.37177276611328,
      "activations/layer21_attention_weight_min": -22.112720489501953,
      "activations/layer22_attention_weight_max": 32.38374328613281,
      "activations/layer22_attention_weight_min": -28.44748306274414,
      "activations/layer23_attention_weight_max": 39.811546325683594,
      "activations/layer23_attention_weight_min": -24.977039337158203,
      "activations/layer2_attention_weight_max": 32.396915435791016,
      "activations/layer2_attention_weight_min": -32.763160705566406,
      "activations/layer3_attention_weight_max": 86.97416687011719,
      "activations/layer3_attention_weight_min": -87.80331420898438,
      "activations/layer4_attention_weight_max": 92.14837646484375,
      "activations/layer4_attention_weight_min": -88.03670501708984,
      "activations/layer5_attention_weight_max": 63.27478790283203,
      "activations/layer5_attention_weight_min": -74.26441192626953,
      "activations/layer6_attention_weight_max": 54.36611557006836,
      "activations/layer6_attention_weight_min": -54.621009826660156,
      "activations/layer7_attention_weight_max": 68.6542739868164,
      "activations/layer7_attention_weight_min": -64.95795440673828,
      "activations/layer8_attention_weight_max": 48.14094924926758,
      "activations/layer8_attention_weight_min": -52.23276901245117,
      "activations/layer9_attention_weight_max": 45.611087799072266,
      "activations/layer9_attention_weight_min": -47.706787109375,
      "epoch": 17.14,
      "learning_rate": 3.990113636363636e-05,
      "loss": 2.7425,
      "step": 294900
    },
    {
      "activations/layer0_attention_weight_max": 15.470572471618652,
      "activations/layer0_attention_weight_min": -12.53276538848877,
      "activations/layer10_attention_weight_max": 34.66685104370117,
      "activations/layer10_attention_weight_min": -35.99374771118164,
      "activations/layer11_attention_weight_max": 34.22520065307617,
      "activations/layer11_attention_weight_min": -34.955177307128906,
      "activations/layer12_attention_weight_max": 19.294702529907227,
      "activations/layer12_attention_weight_min": -25.7829532623291,
      "activations/layer13_attention_weight_max": 33.34066390991211,
      "activations/layer13_attention_weight_min": -29.61081886291504,
      "activations/layer14_attention_weight_max": 34.54596710205078,
      "activations/layer14_attention_weight_min": -31.753276824951172,
      "activations/layer15_attention_weight_max": 31.764432907104492,
      "activations/layer15_attention_weight_min": -31.877330780029297,
      "activations/layer16_attention_weight_max": 33.32491683959961,
      "activations/layer16_attention_weight_min": -31.2581787109375,
      "activations/layer17_attention_weight_max": 52.46137237548828,
      "activations/layer17_attention_weight_min": -47.04976272583008,
      "activations/layer18_attention_weight_max": 46.83269500732422,
      "activations/layer18_attention_weight_min": -40.16672134399414,
      "activations/layer19_attention_weight_max": 22.405776977539062,
      "activations/layer19_attention_weight_min": -23.350439071655273,
      "activations/layer1_attention_weight_max": 17.074581146240234,
      "activations/layer1_attention_weight_min": -14.266615867614746,
      "activations/layer20_attention_weight_max": 23.420684814453125,
      "activations/layer20_attention_weight_min": -24.037548065185547,
      "activations/layer21_attention_weight_max": 34.563446044921875,
      "activations/layer21_attention_weight_min": -24.58678436279297,
      "activations/layer22_attention_weight_max": 29.24524688720703,
      "activations/layer22_attention_weight_min": -24.669525146484375,
      "activations/layer23_attention_weight_max": 40.43101501464844,
      "activations/layer23_attention_weight_min": -27.033048629760742,
      "activations/layer2_attention_weight_max": 30.755142211914062,
      "activations/layer2_attention_weight_min": -29.711135864257812,
      "activations/layer3_attention_weight_max": 86.5678482055664,
      "activations/layer3_attention_weight_min": -89.39000701904297,
      "activations/layer4_attention_weight_max": 87.83446502685547,
      "activations/layer4_attention_weight_min": -90.19232940673828,
      "activations/layer5_attention_weight_max": 63.67147445678711,
      "activations/layer5_attention_weight_min": -77.52626037597656,
      "activations/layer6_attention_weight_max": 49.15424728393555,
      "activations/layer6_attention_weight_min": -51.07832717895508,
      "activations/layer7_attention_weight_max": 65.70987701416016,
      "activations/layer7_attention_weight_min": -69.12350463867188,
      "activations/layer8_attention_weight_max": 43.7343864440918,
      "activations/layer8_attention_weight_min": -50.802268981933594,
      "activations/layer9_attention_weight_max": 43.31390380859375,
      "activations/layer9_attention_weight_min": -47.48991394042969,
      "epoch": 17.14,
      "learning_rate": 3.9882196969696964e-05,
      "loss": 2.7307,
      "step": 294950
    },
    {
      "activations/layer0_attention_weight_max": 15.868745803833008,
      "activations/layer0_attention_weight_min": -12.642905235290527,
      "activations/layer10_attention_weight_max": 35.92523956298828,
      "activations/layer10_attention_weight_min": -35.631500244140625,
      "activations/layer11_attention_weight_max": 32.742218017578125,
      "activations/layer11_attention_weight_min": -34.4102783203125,
      "activations/layer12_attention_weight_max": 18.759929656982422,
      "activations/layer12_attention_weight_min": -25.39793586730957,
      "activations/layer13_attention_weight_max": 33.332847595214844,
      "activations/layer13_attention_weight_min": -26.388492584228516,
      "activations/layer14_attention_weight_max": 34.68851089477539,
      "activations/layer14_attention_weight_min": -31.31279945373535,
      "activations/layer15_attention_weight_max": 31.451608657836914,
      "activations/layer15_attention_weight_min": -29.51874351501465,
      "activations/layer16_attention_weight_max": 32.348724365234375,
      "activations/layer16_attention_weight_min": -28.600685119628906,
      "activations/layer17_attention_weight_max": 49.88410186767578,
      "activations/layer17_attention_weight_min": -44.993221282958984,
      "activations/layer18_attention_weight_max": 46.617835998535156,
      "activations/layer18_attention_weight_min": -39.68804168701172,
      "activations/layer19_attention_weight_max": 23.469202041625977,
      "activations/layer19_attention_weight_min": -20.85236930847168,
      "activations/layer1_attention_weight_max": 16.479867935180664,
      "activations/layer1_attention_weight_min": -13.791305541992188,
      "activations/layer20_attention_weight_max": 23.088943481445312,
      "activations/layer20_attention_weight_min": -20.792070388793945,
      "activations/layer21_attention_weight_max": 31.103708267211914,
      "activations/layer21_attention_weight_min": -21.620899200439453,
      "activations/layer22_attention_weight_max": 27.988910675048828,
      "activations/layer22_attention_weight_min": -25.668701171875,
      "activations/layer23_attention_weight_max": 37.77554702758789,
      "activations/layer23_attention_weight_min": -23.301776885986328,
      "activations/layer2_attention_weight_max": 32.627044677734375,
      "activations/layer2_attention_weight_min": -30.129615783691406,
      "activations/layer3_attention_weight_max": 90.8986587524414,
      "activations/layer3_attention_weight_min": -88.44559478759766,
      "activations/layer4_attention_weight_max": 87.86502838134766,
      "activations/layer4_attention_weight_min": -86.37981414794922,
      "activations/layer5_attention_weight_max": 63.78620147705078,
      "activations/layer5_attention_weight_min": -72.79080200195312,
      "activations/layer6_attention_weight_max": 51.01345443725586,
      "activations/layer6_attention_weight_min": -54.83113479614258,
      "activations/layer7_attention_weight_max": 66.66730499267578,
      "activations/layer7_attention_weight_min": -66.46311950683594,
      "activations/layer8_attention_weight_max": 44.76554489135742,
      "activations/layer8_attention_weight_min": -49.13603210449219,
      "activations/layer9_attention_weight_max": 41.85770034790039,
      "activations/layer9_attention_weight_min": -45.25810623168945,
      "epoch": 17.14,
      "learning_rate": 3.986325757575757e-05,
      "loss": 2.7278,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4267,
      "eval_samples_per_second": 509.57,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4267,
      "eval_openwebtext_samples_per_second": 509.57,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9384,
      "eval_wikitext_samples_per_second": 235.241,
      "step": 295000
    },
    {
      "epoch": 17.14,
      "eval_lambada_loss": 2.486328125,
      "eval_lambada_ppl": 12.01706983184115,
      "eval_lambada_runtime": 9.5196,
      "eval_lambada_samples_per_second": 511.473,
      "step": 295000
    },
    {
      "activations/layer0_attention_weight_max": 16.942516326904297,
      "activations/layer0_attention_weight_min": -12.87663459777832,
      "activations/layer10_attention_weight_max": 36.957435607910156,
      "activations/layer10_attention_weight_min": -36.01042938232422,
      "activations/layer11_attention_weight_max": 32.369834899902344,
      "activations/layer11_attention_weight_min": -35.352134704589844,
      "activations/layer12_attention_weight_max": 19.04780387878418,
      "activations/layer12_attention_weight_min": -25.875953674316406,
      "activations/layer13_attention_weight_max": 30.716461181640625,
      "activations/layer13_attention_weight_min": -26.61185073852539,
      "activations/layer14_attention_weight_max": 35.02830505371094,
      "activations/layer14_attention_weight_min": -29.051610946655273,
      "activations/layer15_attention_weight_max": 32.51829147338867,
      "activations/layer15_attention_weight_min": -30.179393768310547,
      "activations/layer16_attention_weight_max": 33.758758544921875,
      "activations/layer16_attention_weight_min": -30.686140060424805,
      "activations/layer17_attention_weight_max": 53.10822296142578,
      "activations/layer17_attention_weight_min": -42.653892517089844,
      "activations/layer18_attention_weight_max": 44.61763000488281,
      "activations/layer18_attention_weight_min": -39.211517333984375,
      "activations/layer19_attention_weight_max": 22.85008430480957,
      "activations/layer19_attention_weight_min": -24.18446159362793,
      "activations/layer1_attention_weight_max": 16.866838455200195,
      "activations/layer1_attention_weight_min": -15.437654495239258,
      "activations/layer20_attention_weight_max": 21.868881225585938,
      "activations/layer20_attention_weight_min": -24.222944259643555,
      "activations/layer21_attention_weight_max": 33.38811111450195,
      "activations/layer21_attention_weight_min": -20.908761978149414,
      "activations/layer22_attention_weight_max": 26.5367488861084,
      "activations/layer22_attention_weight_min": -27.5551700592041,
      "activations/layer23_attention_weight_max": 35.366783142089844,
      "activations/layer23_attention_weight_min": -24.45431137084961,
      "activations/layer2_attention_weight_max": 32.58220672607422,
      "activations/layer2_attention_weight_min": -32.720703125,
      "activations/layer3_attention_weight_max": 91.14208221435547,
      "activations/layer3_attention_weight_min": -89.68560791015625,
      "activations/layer4_attention_weight_max": 88.69966888427734,
      "activations/layer4_attention_weight_min": -85.27616882324219,
      "activations/layer5_attention_weight_max": 63.0173454284668,
      "activations/layer5_attention_weight_min": -73.33732604980469,
      "activations/layer6_attention_weight_max": 50.692115783691406,
      "activations/layer6_attention_weight_min": -50.44676208496094,
      "activations/layer7_attention_weight_max": 70.85987854003906,
      "activations/layer7_attention_weight_min": -68.67171478271484,
      "activations/layer8_attention_weight_max": 47.8506965637207,
      "activations/layer8_attention_weight_min": -49.42118453979492,
      "activations/layer9_attention_weight_max": 45.732906341552734,
      "activations/layer9_attention_weight_min": -47.11448287963867,
      "epoch": 17.14,
      "learning_rate": 3.984431818181817e-05,
      "loss": 2.7218,
      "step": 295050
    },
    {
      "activations/layer0_attention_weight_max": 15.55216121673584,
      "activations/layer0_attention_weight_min": -12.481383323669434,
      "activations/layer10_attention_weight_max": 37.44044876098633,
      "activations/layer10_attention_weight_min": -38.034542083740234,
      "activations/layer11_attention_weight_max": 32.17624282836914,
      "activations/layer11_attention_weight_min": -34.78150177001953,
      "activations/layer12_attention_weight_max": 17.951704025268555,
      "activations/layer12_attention_weight_min": -24.165733337402344,
      "activations/layer13_attention_weight_max": 35.494136810302734,
      "activations/layer13_attention_weight_min": -28.957910537719727,
      "activations/layer14_attention_weight_max": 35.585811614990234,
      "activations/layer14_attention_weight_min": -29.70350456237793,
      "activations/layer15_attention_weight_max": 34.38093185424805,
      "activations/layer15_attention_weight_min": -30.613819122314453,
      "activations/layer16_attention_weight_max": 34.24029541015625,
      "activations/layer16_attention_weight_min": -30.673479080200195,
      "activations/layer17_attention_weight_max": 51.609596252441406,
      "activations/layer17_attention_weight_min": -42.391944885253906,
      "activations/layer18_attention_weight_max": 44.608558654785156,
      "activations/layer18_attention_weight_min": -37.38663101196289,
      "activations/layer19_attention_weight_max": 26.160099029541016,
      "activations/layer19_attention_weight_min": -22.890623092651367,
      "activations/layer1_attention_weight_max": 16.544986724853516,
      "activations/layer1_attention_weight_min": -14.325209617614746,
      "activations/layer20_attention_weight_max": 21.759294509887695,
      "activations/layer20_attention_weight_min": -21.861160278320312,
      "activations/layer21_attention_weight_max": 33.90064239501953,
      "activations/layer21_attention_weight_min": -21.1005859375,
      "activations/layer22_attention_weight_max": 27.81315040588379,
      "activations/layer22_attention_weight_min": -24.855012893676758,
      "activations/layer23_attention_weight_max": 34.77735137939453,
      "activations/layer23_attention_weight_min": -23.6146297454834,
      "activations/layer2_attention_weight_max": 32.41535949707031,
      "activations/layer2_attention_weight_min": -31.252918243408203,
      "activations/layer3_attention_weight_max": 89.26516723632812,
      "activations/layer3_attention_weight_min": -86.81903076171875,
      "activations/layer4_attention_weight_max": 86.94160461425781,
      "activations/layer4_attention_weight_min": -86.46874237060547,
      "activations/layer5_attention_weight_max": 63.46058654785156,
      "activations/layer5_attention_weight_min": -72.93229675292969,
      "activations/layer6_attention_weight_max": 49.31058120727539,
      "activations/layer6_attention_weight_min": -52.88380432128906,
      "activations/layer7_attention_weight_max": 63.693092346191406,
      "activations/layer7_attention_weight_min": -67.97663116455078,
      "activations/layer8_attention_weight_max": 48.497066497802734,
      "activations/layer8_attention_weight_min": -50.59233474731445,
      "activations/layer9_attention_weight_max": 47.502403259277344,
      "activations/layer9_attention_weight_min": -48.661746978759766,
      "epoch": 17.15,
      "learning_rate": 3.982537878787879e-05,
      "loss": 2.7404,
      "step": 295100
    },
    {
      "activations/layer0_attention_weight_max": 15.093587875366211,
      "activations/layer0_attention_weight_min": -12.598760604858398,
      "activations/layer10_attention_weight_max": 34.095279693603516,
      "activations/layer10_attention_weight_min": -35.33185577392578,
      "activations/layer11_attention_weight_max": 31.51934814453125,
      "activations/layer11_attention_weight_min": -33.63515853881836,
      "activations/layer12_attention_weight_max": 16.49567222595215,
      "activations/layer12_attention_weight_min": -31.65167808532715,
      "activations/layer13_attention_weight_max": 27.25895881652832,
      "activations/layer13_attention_weight_min": -26.0399112701416,
      "activations/layer14_attention_weight_max": 30.622421264648438,
      "activations/layer14_attention_weight_min": -30.54826545715332,
      "activations/layer15_attention_weight_max": 30.606775283813477,
      "activations/layer15_attention_weight_min": -28.588130950927734,
      "activations/layer16_attention_weight_max": 34.17353439331055,
      "activations/layer16_attention_weight_min": -31.263513565063477,
      "activations/layer17_attention_weight_max": 47.162811279296875,
      "activations/layer17_attention_weight_min": -45.26040267944336,
      "activations/layer18_attention_weight_max": 42.32884979248047,
      "activations/layer18_attention_weight_min": -40.60936737060547,
      "activations/layer19_attention_weight_max": 24.486282348632812,
      "activations/layer19_attention_weight_min": -21.23171043395996,
      "activations/layer1_attention_weight_max": 16.377756118774414,
      "activations/layer1_attention_weight_min": -14.539660453796387,
      "activations/layer20_attention_weight_max": 19.789310455322266,
      "activations/layer20_attention_weight_min": -20.204404830932617,
      "activations/layer21_attention_weight_max": 33.098289489746094,
      "activations/layer21_attention_weight_min": -21.251541137695312,
      "activations/layer22_attention_weight_max": 26.351030349731445,
      "activations/layer22_attention_weight_min": -24.784074783325195,
      "activations/layer23_attention_weight_max": 32.76495361328125,
      "activations/layer23_attention_weight_min": -22.05147933959961,
      "activations/layer2_attention_weight_max": 34.15439987182617,
      "activations/layer2_attention_weight_min": -31.97473907470703,
      "activations/layer3_attention_weight_max": 91.81818389892578,
      "activations/layer3_attention_weight_min": -93.33917999267578,
      "activations/layer4_attention_weight_max": 86.51329803466797,
      "activations/layer4_attention_weight_min": -86.41321563720703,
      "activations/layer5_attention_weight_max": 62.60358810424805,
      "activations/layer5_attention_weight_min": -70.47084045410156,
      "activations/layer6_attention_weight_max": 51.49782943725586,
      "activations/layer6_attention_weight_min": -48.989402770996094,
      "activations/layer7_attention_weight_max": 67.51293182373047,
      "activations/layer7_attention_weight_min": -60.80843734741211,
      "activations/layer8_attention_weight_max": 44.87084197998047,
      "activations/layer8_attention_weight_min": -48.37882995605469,
      "activations/layer9_attention_weight_max": 48.905914306640625,
      "activations/layer9_attention_weight_min": -47.735008239746094,
      "epoch": 17.15,
      "learning_rate": 3.980643939393939e-05,
      "loss": 2.7521,
      "step": 295150
    },
    {
      "activations/layer0_attention_weight_max": 15.530640602111816,
      "activations/layer0_attention_weight_min": -12.987598419189453,
      "activations/layer10_attention_weight_max": 37.48381042480469,
      "activations/layer10_attention_weight_min": -37.33476638793945,
      "activations/layer11_attention_weight_max": 32.938453674316406,
      "activations/layer11_attention_weight_min": -35.541297912597656,
      "activations/layer12_attention_weight_max": 20.22650146484375,
      "activations/layer12_attention_weight_min": -26.778451919555664,
      "activations/layer13_attention_weight_max": 33.55305099487305,
      "activations/layer13_attention_weight_min": -30.593984603881836,
      "activations/layer14_attention_weight_max": 34.62559127807617,
      "activations/layer14_attention_weight_min": -29.203296661376953,
      "activations/layer15_attention_weight_max": 32.747802734375,
      "activations/layer15_attention_weight_min": -30.454023361206055,
      "activations/layer16_attention_weight_max": 32.026973724365234,
      "activations/layer16_attention_weight_min": -32.87973403930664,
      "activations/layer17_attention_weight_max": 47.9070930480957,
      "activations/layer17_attention_weight_min": -42.955421447753906,
      "activations/layer18_attention_weight_max": 41.74777603149414,
      "activations/layer18_attention_weight_min": -38.21392822265625,
      "activations/layer19_attention_weight_max": 21.13730239868164,
      "activations/layer19_attention_weight_min": -21.88558578491211,
      "activations/layer1_attention_weight_max": 16.387222290039062,
      "activations/layer1_attention_weight_min": -15.60682487487793,
      "activations/layer20_attention_weight_max": 20.890085220336914,
      "activations/layer20_attention_weight_min": -21.555627822875977,
      "activations/layer21_attention_weight_max": 32.059417724609375,
      "activations/layer21_attention_weight_min": -22.864959716796875,
      "activations/layer22_attention_weight_max": 26.46246910095215,
      "activations/layer22_attention_weight_min": -25.11965560913086,
      "activations/layer23_attention_weight_max": 34.11217498779297,
      "activations/layer23_attention_weight_min": -24.861183166503906,
      "activations/layer2_attention_weight_max": 33.83557891845703,
      "activations/layer2_attention_weight_min": -32.1680908203125,
      "activations/layer3_attention_weight_max": 89.21745300292969,
      "activations/layer3_attention_weight_min": -96.73201751708984,
      "activations/layer4_attention_weight_max": 88.79684448242188,
      "activations/layer4_attention_weight_min": -87.43346405029297,
      "activations/layer5_attention_weight_max": 64.15367126464844,
      "activations/layer5_attention_weight_min": -70.4184799194336,
      "activations/layer6_attention_weight_max": 49.03406524658203,
      "activations/layer6_attention_weight_min": -53.2429313659668,
      "activations/layer7_attention_weight_max": 64.18695068359375,
      "activations/layer7_attention_weight_min": -71.53446960449219,
      "activations/layer8_attention_weight_max": 46.374629974365234,
      "activations/layer8_attention_weight_min": -51.611106872558594,
      "activations/layer9_attention_weight_max": 47.17683029174805,
      "activations/layer9_attention_weight_min": -50.15873336791992,
      "epoch": 17.15,
      "learning_rate": 3.978749999999999e-05,
      "loss": 2.7296,
      "step": 295200
    },
    {
      "activations/layer0_attention_weight_max": 16.196332931518555,
      "activations/layer0_attention_weight_min": -12.545307159423828,
      "activations/layer10_attention_weight_max": 33.06499481201172,
      "activations/layer10_attention_weight_min": -33.693843841552734,
      "activations/layer11_attention_weight_max": 31.612789154052734,
      "activations/layer11_attention_weight_min": -34.130149841308594,
      "activations/layer12_attention_weight_max": 19.312904357910156,
      "activations/layer12_attention_weight_min": -25.33724594116211,
      "activations/layer13_attention_weight_max": 38.13197326660156,
      "activations/layer13_attention_weight_min": -30.299373626708984,
      "activations/layer14_attention_weight_max": 40.709163665771484,
      "activations/layer14_attention_weight_min": -31.380390167236328,
      "activations/layer15_attention_weight_max": 40.8569450378418,
      "activations/layer15_attention_weight_min": -29.125774383544922,
      "activations/layer16_attention_weight_max": 32.974082946777344,
      "activations/layer16_attention_weight_min": -31.518810272216797,
      "activations/layer17_attention_weight_max": 52.31315612792969,
      "activations/layer17_attention_weight_min": -47.821556091308594,
      "activations/layer18_attention_weight_max": 47.71219253540039,
      "activations/layer18_attention_weight_min": -39.33585739135742,
      "activations/layer19_attention_weight_max": 25.196483612060547,
      "activations/layer19_attention_weight_min": -23.452919006347656,
      "activations/layer1_attention_weight_max": 16.504318237304688,
      "activations/layer1_attention_weight_min": -14.088563919067383,
      "activations/layer20_attention_weight_max": 22.30033302307129,
      "activations/layer20_attention_weight_min": -22.84062385559082,
      "activations/layer21_attention_weight_max": 46.65598678588867,
      "activations/layer21_attention_weight_min": -26.430644989013672,
      "activations/layer22_attention_weight_max": 29.80617904663086,
      "activations/layer22_attention_weight_min": -25.26104164123535,
      "activations/layer23_attention_weight_max": 36.84613037109375,
      "activations/layer23_attention_weight_min": -24.555999755859375,
      "activations/layer2_attention_weight_max": 31.435527801513672,
      "activations/layer2_attention_weight_min": -29.944259643554688,
      "activations/layer3_attention_weight_max": 85.76167297363281,
      "activations/layer3_attention_weight_min": -85.57624816894531,
      "activations/layer4_attention_weight_max": 85.79375457763672,
      "activations/layer4_attention_weight_min": -83.68126678466797,
      "activations/layer5_attention_weight_max": 61.20444869995117,
      "activations/layer5_attention_weight_min": -75.74630737304688,
      "activations/layer6_attention_weight_max": 47.966087341308594,
      "activations/layer6_attention_weight_min": -50.42214584350586,
      "activations/layer7_attention_weight_max": 62.0057258605957,
      "activations/layer7_attention_weight_min": -63.38521957397461,
      "activations/layer8_attention_weight_max": 43.56525421142578,
      "activations/layer8_attention_weight_min": -47.68503189086914,
      "activations/layer9_attention_weight_max": 40.37528991699219,
      "activations/layer9_attention_weight_min": -42.235408782958984,
      "epoch": 17.16,
      "learning_rate": 3.9768560606060606e-05,
      "loss": 2.7235,
      "step": 295250
    },
    {
      "activations/layer0_attention_weight_max": 15.308907508850098,
      "activations/layer0_attention_weight_min": -12.605937957763672,
      "activations/layer10_attention_weight_max": 33.92452621459961,
      "activations/layer10_attention_weight_min": -34.43341827392578,
      "activations/layer11_attention_weight_max": 29.655384063720703,
      "activations/layer11_attention_weight_min": -32.159278869628906,
      "activations/layer12_attention_weight_max": 19.564241409301758,
      "activations/layer12_attention_weight_min": -24.157384872436523,
      "activations/layer13_attention_weight_max": 33.61217498779297,
      "activations/layer13_attention_weight_min": -27.803936004638672,
      "activations/layer14_attention_weight_max": 36.901634216308594,
      "activations/layer14_attention_weight_min": -30.212642669677734,
      "activations/layer15_attention_weight_max": 35.178489685058594,
      "activations/layer15_attention_weight_min": -29.28986167907715,
      "activations/layer16_attention_weight_max": 35.16571807861328,
      "activations/layer16_attention_weight_min": -33.4562873840332,
      "activations/layer17_attention_weight_max": 55.64401626586914,
      "activations/layer17_attention_weight_min": -47.505645751953125,
      "activations/layer18_attention_weight_max": 52.01476287841797,
      "activations/layer18_attention_weight_min": -37.45490264892578,
      "activations/layer19_attention_weight_max": 22.148006439208984,
      "activations/layer19_attention_weight_min": -21.426694869995117,
      "activations/layer1_attention_weight_max": 15.913415908813477,
      "activations/layer1_attention_weight_min": -15.28262710571289,
      "activations/layer20_attention_weight_max": 21.49271011352539,
      "activations/layer20_attention_weight_min": -24.71891975402832,
      "activations/layer21_attention_weight_max": 35.020896911621094,
      "activations/layer21_attention_weight_min": -24.19606590270996,
      "activations/layer22_attention_weight_max": 29.09212875366211,
      "activations/layer22_attention_weight_min": -25.548969268798828,
      "activations/layer23_attention_weight_max": 37.36056900024414,
      "activations/layer23_attention_weight_min": -25.116268157958984,
      "activations/layer2_attention_weight_max": 31.31702995300293,
      "activations/layer2_attention_weight_min": -30.245838165283203,
      "activations/layer3_attention_weight_max": 89.23699188232422,
      "activations/layer3_attention_weight_min": -91.58504486083984,
      "activations/layer4_attention_weight_max": 85.83524322509766,
      "activations/layer4_attention_weight_min": -83.0284652709961,
      "activations/layer5_attention_weight_max": 63.01534652709961,
      "activations/layer5_attention_weight_min": -73.421630859375,
      "activations/layer6_attention_weight_max": 48.72690963745117,
      "activations/layer6_attention_weight_min": -49.463111877441406,
      "activations/layer7_attention_weight_max": 62.1128044128418,
      "activations/layer7_attention_weight_min": -62.48416519165039,
      "activations/layer8_attention_weight_max": 44.542354583740234,
      "activations/layer8_attention_weight_min": -46.163673400878906,
      "activations/layer9_attention_weight_max": 41.83991241455078,
      "activations/layer9_attention_weight_min": -43.1641845703125,
      "epoch": 17.16,
      "learning_rate": 3.974962121212121e-05,
      "loss": 2.7521,
      "step": 295300
    },
    {
      "activations/layer0_attention_weight_max": 15.622196197509766,
      "activations/layer0_attention_weight_min": -12.81728744506836,
      "activations/layer10_attention_weight_max": 33.44879150390625,
      "activations/layer10_attention_weight_min": -36.71864318847656,
      "activations/layer11_attention_weight_max": 31.12853240966797,
      "activations/layer11_attention_weight_min": -34.20398712158203,
      "activations/layer12_attention_weight_max": 18.03473472595215,
      "activations/layer12_attention_weight_min": -26.017105102539062,
      "activations/layer13_attention_weight_max": 28.677953720092773,
      "activations/layer13_attention_weight_min": -26.935216903686523,
      "activations/layer14_attention_weight_max": 33.20329284667969,
      "activations/layer14_attention_weight_min": -30.829877853393555,
      "activations/layer15_attention_weight_max": 32.421546936035156,
      "activations/layer15_attention_weight_min": -30.247177124023438,
      "activations/layer16_attention_weight_max": 35.17428970336914,
      "activations/layer16_attention_weight_min": -33.65311050415039,
      "activations/layer17_attention_weight_max": 50.00499725341797,
      "activations/layer17_attention_weight_min": -46.671573638916016,
      "activations/layer18_attention_weight_max": 47.553123474121094,
      "activations/layer18_attention_weight_min": -41.30460739135742,
      "activations/layer19_attention_weight_max": 23.28230857849121,
      "activations/layer19_attention_weight_min": -23.852689743041992,
      "activations/layer1_attention_weight_max": 16.430294036865234,
      "activations/layer1_attention_weight_min": -17.43889045715332,
      "activations/layer20_attention_weight_max": 21.998640060424805,
      "activations/layer20_attention_weight_min": -27.585039138793945,
      "activations/layer21_attention_weight_max": 36.43274688720703,
      "activations/layer21_attention_weight_min": -22.669986724853516,
      "activations/layer22_attention_weight_max": 30.406383514404297,
      "activations/layer22_attention_weight_min": -27.26045799255371,
      "activations/layer23_attention_weight_max": 36.21259689331055,
      "activations/layer23_attention_weight_min": -26.857425689697266,
      "activations/layer2_attention_weight_max": 32.94938659667969,
      "activations/layer2_attention_weight_min": -32.83807373046875,
      "activations/layer3_attention_weight_max": 89.34380340576172,
      "activations/layer3_attention_weight_min": -93.44792175292969,
      "activations/layer4_attention_weight_max": 86.89437103271484,
      "activations/layer4_attention_weight_min": -83.87342071533203,
      "activations/layer5_attention_weight_max": 62.31441879272461,
      "activations/layer5_attention_weight_min": -73.68684387207031,
      "activations/layer6_attention_weight_max": 50.668800354003906,
      "activations/layer6_attention_weight_min": -50.82221221923828,
      "activations/layer7_attention_weight_max": 64.12425231933594,
      "activations/layer7_attention_weight_min": -62.43382263183594,
      "activations/layer8_attention_weight_max": 44.04160690307617,
      "activations/layer8_attention_weight_min": -48.77847671508789,
      "activations/layer9_attention_weight_max": 42.54329299926758,
      "activations/layer9_attention_weight_min": -45.25294876098633,
      "epoch": 17.16,
      "learning_rate": 3.9730681818181816e-05,
      "loss": 2.7334,
      "step": 295350
    },
    {
      "activations/layer0_attention_weight_max": 14.932641983032227,
      "activations/layer0_attention_weight_min": -12.970840454101562,
      "activations/layer10_attention_weight_max": 37.271183013916016,
      "activations/layer10_attention_weight_min": -37.907073974609375,
      "activations/layer11_attention_weight_max": 34.56232452392578,
      "activations/layer11_attention_weight_min": -34.58517074584961,
      "activations/layer12_attention_weight_max": 19.280099868774414,
      "activations/layer12_attention_weight_min": -27.344846725463867,
      "activations/layer13_attention_weight_max": 29.71250343322754,
      "activations/layer13_attention_weight_min": -30.809579849243164,
      "activations/layer14_attention_weight_max": 35.88496398925781,
      "activations/layer14_attention_weight_min": -30.545366287231445,
      "activations/layer15_attention_weight_max": 32.813350677490234,
      "activations/layer15_attention_weight_min": -31.45024299621582,
      "activations/layer16_attention_weight_max": 33.63795852661133,
      "activations/layer16_attention_weight_min": -34.09870147705078,
      "activations/layer17_attention_weight_max": 52.041690826416016,
      "activations/layer17_attention_weight_min": -49.56206512451172,
      "activations/layer18_attention_weight_max": 45.87215042114258,
      "activations/layer18_attention_weight_min": -43.5345573425293,
      "activations/layer19_attention_weight_max": 24.235883712768555,
      "activations/layer19_attention_weight_min": -26.081836700439453,
      "activations/layer1_attention_weight_max": 17.299888610839844,
      "activations/layer1_attention_weight_min": -19.88121795654297,
      "activations/layer20_attention_weight_max": 23.600934982299805,
      "activations/layer20_attention_weight_min": -23.176651000976562,
      "activations/layer21_attention_weight_max": 41.50562286376953,
      "activations/layer21_attention_weight_min": -25.00577735900879,
      "activations/layer22_attention_weight_max": 35.600990295410156,
      "activations/layer22_attention_weight_min": -26.307878494262695,
      "activations/layer23_attention_weight_max": 37.88493347167969,
      "activations/layer23_attention_weight_min": -28.726810455322266,
      "activations/layer2_attention_weight_max": 31.325040817260742,
      "activations/layer2_attention_weight_min": -33.528663635253906,
      "activations/layer3_attention_weight_max": 88.60320281982422,
      "activations/layer3_attention_weight_min": -91.63123321533203,
      "activations/layer4_attention_weight_max": 84.96709442138672,
      "activations/layer4_attention_weight_min": -84.75959014892578,
      "activations/layer5_attention_weight_max": 62.2197265625,
      "activations/layer5_attention_weight_min": -75.74134826660156,
      "activations/layer6_attention_weight_max": 47.145992279052734,
      "activations/layer6_attention_weight_min": -48.54625701904297,
      "activations/layer7_attention_weight_max": 66.78758239746094,
      "activations/layer7_attention_weight_min": -62.87277603149414,
      "activations/layer8_attention_weight_max": 45.62137222290039,
      "activations/layer8_attention_weight_min": -46.1516227722168,
      "activations/layer9_attention_weight_max": 43.057987213134766,
      "activations/layer9_attention_weight_min": -43.457515716552734,
      "epoch": 17.16,
      "learning_rate": 3.971174242424242e-05,
      "loss": 2.7546,
      "step": 295400
    },
    {
      "activations/layer0_attention_weight_max": 16.01889419555664,
      "activations/layer0_attention_weight_min": -12.59502124786377,
      "activations/layer10_attention_weight_max": 35.3388671875,
      "activations/layer10_attention_weight_min": -34.28755569458008,
      "activations/layer11_attention_weight_max": 31.308055877685547,
      "activations/layer11_attention_weight_min": -33.49607849121094,
      "activations/layer12_attention_weight_max": 19.676971435546875,
      "activations/layer12_attention_weight_min": -25.461851119995117,
      "activations/layer13_attention_weight_max": 32.510414123535156,
      "activations/layer13_attention_weight_min": -29.50547981262207,
      "activations/layer14_attention_weight_max": 37.78139877319336,
      "activations/layer14_attention_weight_min": -30.67638397216797,
      "activations/layer15_attention_weight_max": 33.966182708740234,
      "activations/layer15_attention_weight_min": -31.634920120239258,
      "activations/layer16_attention_weight_max": 34.41386413574219,
      "activations/layer16_attention_weight_min": -32.97237014770508,
      "activations/layer17_attention_weight_max": 52.708438873291016,
      "activations/layer17_attention_weight_min": -45.45602798461914,
      "activations/layer18_attention_weight_max": 46.00649642944336,
      "activations/layer18_attention_weight_min": -39.0610237121582,
      "activations/layer19_attention_weight_max": 23.559724807739258,
      "activations/layer19_attention_weight_min": -22.491554260253906,
      "activations/layer1_attention_weight_max": 15.977104187011719,
      "activations/layer1_attention_weight_min": -16.582731246948242,
      "activations/layer20_attention_weight_max": 22.08684539794922,
      "activations/layer20_attention_weight_min": -23.686368942260742,
      "activations/layer21_attention_weight_max": 38.33449172973633,
      "activations/layer21_attention_weight_min": -24.377384185791016,
      "activations/layer22_attention_weight_max": 30.55691909790039,
      "activations/layer22_attention_weight_min": -26.345945358276367,
      "activations/layer23_attention_weight_max": 45.91521453857422,
      "activations/layer23_attention_weight_min": -24.068675994873047,
      "activations/layer2_attention_weight_max": 31.37946891784668,
      "activations/layer2_attention_weight_min": -30.941577911376953,
      "activations/layer3_attention_weight_max": 87.18745422363281,
      "activations/layer3_attention_weight_min": -88.87528228759766,
      "activations/layer4_attention_weight_max": 84.98235321044922,
      "activations/layer4_attention_weight_min": -83.85384368896484,
      "activations/layer5_attention_weight_max": 62.83631134033203,
      "activations/layer5_attention_weight_min": -70.73055267333984,
      "activations/layer6_attention_weight_max": 48.887149810791016,
      "activations/layer6_attention_weight_min": -51.68312454223633,
      "activations/layer7_attention_weight_max": 64.94933319091797,
      "activations/layer7_attention_weight_min": -61.868324279785156,
      "activations/layer8_attention_weight_max": 46.5693359375,
      "activations/layer8_attention_weight_min": -48.3631706237793,
      "activations/layer9_attention_weight_max": 43.68818283081055,
      "activations/layer9_attention_weight_min": -44.488990783691406,
      "epoch": 17.17,
      "learning_rate": 3.9693181818181815e-05,
      "loss": 2.7394,
      "step": 295450
    },
    {
      "activations/layer0_attention_weight_max": 16.457977294921875,
      "activations/layer0_attention_weight_min": -12.714340209960938,
      "activations/layer10_attention_weight_max": 32.60681915283203,
      "activations/layer10_attention_weight_min": -34.294551849365234,
      "activations/layer11_attention_weight_max": 29.210506439208984,
      "activations/layer11_attention_weight_min": -31.87853240966797,
      "activations/layer12_attention_weight_max": 19.31416130065918,
      "activations/layer12_attention_weight_min": -23.097612380981445,
      "activations/layer13_attention_weight_max": 32.975486755371094,
      "activations/layer13_attention_weight_min": -25.087284088134766,
      "activations/layer14_attention_weight_max": 34.581783294677734,
      "activations/layer14_attention_weight_min": -29.12723731994629,
      "activations/layer15_attention_weight_max": 30.175689697265625,
      "activations/layer15_attention_weight_min": -29.909101486206055,
      "activations/layer16_attention_weight_max": 33.29181671142578,
      "activations/layer16_attention_weight_min": -32.087650299072266,
      "activations/layer17_attention_weight_max": 51.2264289855957,
      "activations/layer17_attention_weight_min": -46.16745376586914,
      "activations/layer18_attention_weight_max": 49.31767272949219,
      "activations/layer18_attention_weight_min": -40.54090881347656,
      "activations/layer19_attention_weight_max": 23.26592445373535,
      "activations/layer19_attention_weight_min": -22.679946899414062,
      "activations/layer1_attention_weight_max": 16.026399612426758,
      "activations/layer1_attention_weight_min": -14.786383628845215,
      "activations/layer20_attention_weight_max": 21.328874588012695,
      "activations/layer20_attention_weight_min": -21.494426727294922,
      "activations/layer21_attention_weight_max": 35.88391876220703,
      "activations/layer21_attention_weight_min": -21.185009002685547,
      "activations/layer22_attention_weight_max": 29.57676887512207,
      "activations/layer22_attention_weight_min": -26.571043014526367,
      "activations/layer23_attention_weight_max": 36.260398864746094,
      "activations/layer23_attention_weight_min": -24.368728637695312,
      "activations/layer2_attention_weight_max": 32.23023986816406,
      "activations/layer2_attention_weight_min": -30.850830078125,
      "activations/layer3_attention_weight_max": 85.73668670654297,
      "activations/layer3_attention_weight_min": -85.79922485351562,
      "activations/layer4_attention_weight_max": 84.3768310546875,
      "activations/layer4_attention_weight_min": -84.19081115722656,
      "activations/layer5_attention_weight_max": 63.35099792480469,
      "activations/layer5_attention_weight_min": -76.91885375976562,
      "activations/layer6_attention_weight_max": 47.4229621887207,
      "activations/layer6_attention_weight_min": -48.30127716064453,
      "activations/layer7_attention_weight_max": 58.86265563964844,
      "activations/layer7_attention_weight_min": -63.71797561645508,
      "activations/layer8_attention_weight_max": 43.70051956176758,
      "activations/layer8_attention_weight_min": -46.4083137512207,
      "activations/layer9_attention_weight_max": 42.771121978759766,
      "activations/layer9_attention_weight_min": -41.71721649169922,
      "epoch": 17.17,
      "learning_rate": 3.9674242424242423e-05,
      "loss": 2.7381,
      "step": 295500
    },
    {
      "activations/layer0_attention_weight_max": 17.125507354736328,
      "activations/layer0_attention_weight_min": -12.71518325805664,
      "activations/layer10_attention_weight_max": 33.4808349609375,
      "activations/layer10_attention_weight_min": -33.97453689575195,
      "activations/layer11_attention_weight_max": 29.6744441986084,
      "activations/layer11_attention_weight_min": -32.25090789794922,
      "activations/layer12_attention_weight_max": 19.014108657836914,
      "activations/layer12_attention_weight_min": -24.48960304260254,
      "activations/layer13_attention_weight_max": 32.82234191894531,
      "activations/layer13_attention_weight_min": -27.847036361694336,
      "activations/layer14_attention_weight_max": 36.87465286254883,
      "activations/layer14_attention_weight_min": -32.39857864379883,
      "activations/layer15_attention_weight_max": 34.328369140625,
      "activations/layer15_attention_weight_min": -30.97893714904785,
      "activations/layer16_attention_weight_max": 34.331939697265625,
      "activations/layer16_attention_weight_min": -34.90715026855469,
      "activations/layer17_attention_weight_max": 52.61716842651367,
      "activations/layer17_attention_weight_min": -46.01240158081055,
      "activations/layer18_attention_weight_max": 49.31534957885742,
      "activations/layer18_attention_weight_min": -42.44916915893555,
      "activations/layer19_attention_weight_max": 22.25745964050293,
      "activations/layer19_attention_weight_min": -21.31178855895996,
      "activations/layer1_attention_weight_max": 15.125225067138672,
      "activations/layer1_attention_weight_min": -13.750798225402832,
      "activations/layer20_attention_weight_max": 22.596046447753906,
      "activations/layer20_attention_weight_min": -20.857208251953125,
      "activations/layer21_attention_weight_max": 35.65116500854492,
      "activations/layer21_attention_weight_min": -25.69034767150879,
      "activations/layer22_attention_weight_max": 31.25702476501465,
      "activations/layer22_attention_weight_min": -27.941064834594727,
      "activations/layer23_attention_weight_max": 37.072601318359375,
      "activations/layer23_attention_weight_min": -24.73819351196289,
      "activations/layer2_attention_weight_max": 33.062583923339844,
      "activations/layer2_attention_weight_min": -29.09889030456543,
      "activations/layer3_attention_weight_max": 86.89067077636719,
      "activations/layer3_attention_weight_min": -88.37590026855469,
      "activations/layer4_attention_weight_max": 88.0373764038086,
      "activations/layer4_attention_weight_min": -83.11546325683594,
      "activations/layer5_attention_weight_max": 63.31867980957031,
      "activations/layer5_attention_weight_min": -71.54032135009766,
      "activations/layer6_attention_weight_max": 47.20264434814453,
      "activations/layer6_attention_weight_min": -49.440635681152344,
      "activations/layer7_attention_weight_max": 67.12409973144531,
      "activations/layer7_attention_weight_min": -61.07823181152344,
      "activations/layer8_attention_weight_max": 43.56911087036133,
      "activations/layer8_attention_weight_min": -47.51129150390625,
      "activations/layer9_attention_weight_max": 43.56462478637695,
      "activations/layer9_attention_weight_min": -43.032508850097656,
      "epoch": 17.17,
      "learning_rate": 3.9655303030303025e-05,
      "loss": 2.7202,
      "step": 295550
    },
    {
      "activations/layer0_attention_weight_max": 15.683860778808594,
      "activations/layer0_attention_weight_min": -12.5217924118042,
      "activations/layer10_attention_weight_max": 37.05649185180664,
      "activations/layer10_attention_weight_min": -37.69234085083008,
      "activations/layer11_attention_weight_max": 34.697166442871094,
      "activations/layer11_attention_weight_min": -35.98632049560547,
      "activations/layer12_attention_weight_max": 18.478923797607422,
      "activations/layer12_attention_weight_min": -26.193342208862305,
      "activations/layer13_attention_weight_max": 33.65565490722656,
      "activations/layer13_attention_weight_min": -28.78594970703125,
      "activations/layer14_attention_weight_max": 35.435035705566406,
      "activations/layer14_attention_weight_min": -31.03261947631836,
      "activations/layer15_attention_weight_max": 36.553260803222656,
      "activations/layer15_attention_weight_min": -32.39533615112305,
      "activations/layer16_attention_weight_max": 35.023704528808594,
      "activations/layer16_attention_weight_min": -35.633121490478516,
      "activations/layer17_attention_weight_max": 50.86433792114258,
      "activations/layer17_attention_weight_min": -53.90902328491211,
      "activations/layer18_attention_weight_max": 46.4206657409668,
      "activations/layer18_attention_weight_min": -45.788185119628906,
      "activations/layer19_attention_weight_max": 24.566457748413086,
      "activations/layer19_attention_weight_min": -26.099332809448242,
      "activations/layer1_attention_weight_max": 16.135780334472656,
      "activations/layer1_attention_weight_min": -15.123106002807617,
      "activations/layer20_attention_weight_max": 23.467771530151367,
      "activations/layer20_attention_weight_min": -22.350666046142578,
      "activations/layer21_attention_weight_max": 35.56736755371094,
      "activations/layer21_attention_weight_min": -22.9855899810791,
      "activations/layer22_attention_weight_max": 26.387786865234375,
      "activations/layer22_attention_weight_min": -24.466537475585938,
      "activations/layer23_attention_weight_max": 32.186256408691406,
      "activations/layer23_attention_weight_min": -26.297130584716797,
      "activations/layer2_attention_weight_max": 33.01918411254883,
      "activations/layer2_attention_weight_min": -31.35750961303711,
      "activations/layer3_attention_weight_max": 89.94952392578125,
      "activations/layer3_attention_weight_min": -87.40071868896484,
      "activations/layer4_attention_weight_max": 87.3420639038086,
      "activations/layer4_attention_weight_min": -87.25676727294922,
      "activations/layer5_attention_weight_max": 67.32350158691406,
      "activations/layer5_attention_weight_min": -74.58961486816406,
      "activations/layer6_attention_weight_max": 49.92863464355469,
      "activations/layer6_attention_weight_min": -55.34320831298828,
      "activations/layer7_attention_weight_max": 64.52739715576172,
      "activations/layer7_attention_weight_min": -63.66793441772461,
      "activations/layer8_attention_weight_max": 47.76716613769531,
      "activations/layer8_attention_weight_min": -49.579925537109375,
      "activations/layer9_attention_weight_max": 46.902320861816406,
      "activations/layer9_attention_weight_min": -48.393672943115234,
      "epoch": 17.18,
      "learning_rate": 3.963636363636363e-05,
      "loss": 2.7305,
      "step": 295600
    },
    {
      "activations/layer0_attention_weight_max": 15.676844596862793,
      "activations/layer0_attention_weight_min": -12.835082054138184,
      "activations/layer10_attention_weight_max": 36.4071044921875,
      "activations/layer10_attention_weight_min": -34.867103576660156,
      "activations/layer11_attention_weight_max": 32.3471565246582,
      "activations/layer11_attention_weight_min": -33.410709381103516,
      "activations/layer12_attention_weight_max": 19.761337280273438,
      "activations/layer12_attention_weight_min": -24.986791610717773,
      "activations/layer13_attention_weight_max": 32.83010482788086,
      "activations/layer13_attention_weight_min": -25.67363166809082,
      "activations/layer14_attention_weight_max": 35.54690933227539,
      "activations/layer14_attention_weight_min": -30.054443359375,
      "activations/layer15_attention_weight_max": 30.806739807128906,
      "activations/layer15_attention_weight_min": -29.592636108398438,
      "activations/layer16_attention_weight_max": 32.91650390625,
      "activations/layer16_attention_weight_min": -30.48773956298828,
      "activations/layer17_attention_weight_max": 48.019222259521484,
      "activations/layer17_attention_weight_min": -43.136451721191406,
      "activations/layer18_attention_weight_max": 45.12702178955078,
      "activations/layer18_attention_weight_min": -38.95333480834961,
      "activations/layer19_attention_weight_max": 23.84090232849121,
      "activations/layer19_attention_weight_min": -22.029897689819336,
      "activations/layer1_attention_weight_max": 17.729074478149414,
      "activations/layer1_attention_weight_min": -15.041504859924316,
      "activations/layer20_attention_weight_max": 22.229825973510742,
      "activations/layer20_attention_weight_min": -21.874561309814453,
      "activations/layer21_attention_weight_max": 36.162261962890625,
      "activations/layer21_attention_weight_min": -22.276939392089844,
      "activations/layer22_attention_weight_max": 29.462907791137695,
      "activations/layer22_attention_weight_min": -25.950403213500977,
      "activations/layer23_attention_weight_max": 34.94251251220703,
      "activations/layer23_attention_weight_min": -23.851972579956055,
      "activations/layer2_attention_weight_max": 32.75154113769531,
      "activations/layer2_attention_weight_min": -29.3653564453125,
      "activations/layer3_attention_weight_max": 88.9902114868164,
      "activations/layer3_attention_weight_min": -85.6789321899414,
      "activations/layer4_attention_weight_max": 87.06566619873047,
      "activations/layer4_attention_weight_min": -84.29436492919922,
      "activations/layer5_attention_weight_max": 62.96783447265625,
      "activations/layer5_attention_weight_min": -71.5958480834961,
      "activations/layer6_attention_weight_max": 48.69422912597656,
      "activations/layer6_attention_weight_min": -49.35010528564453,
      "activations/layer7_attention_weight_max": 63.15491485595703,
      "activations/layer7_attention_weight_min": -63.856082916259766,
      "activations/layer8_attention_weight_max": 44.338600158691406,
      "activations/layer8_attention_weight_min": -47.621585845947266,
      "activations/layer9_attention_weight_max": 42.51935958862305,
      "activations/layer9_attention_weight_min": -44.74534606933594,
      "epoch": 17.18,
      "learning_rate": 3.961742424242424e-05,
      "loss": 2.7363,
      "step": 295650
    },
    {
      "activations/layer0_attention_weight_max": 15.905641555786133,
      "activations/layer0_attention_weight_min": -12.766701698303223,
      "activations/layer10_attention_weight_max": 38.548675537109375,
      "activations/layer10_attention_weight_min": -36.32528305053711,
      "activations/layer11_attention_weight_max": 34.46042251586914,
      "activations/layer11_attention_weight_min": -34.741477966308594,
      "activations/layer12_attention_weight_max": 18.179777145385742,
      "activations/layer12_attention_weight_min": -24.221574783325195,
      "activations/layer13_attention_weight_max": 35.004859924316406,
      "activations/layer13_attention_weight_min": -27.791250228881836,
      "activations/layer14_attention_weight_max": 33.582542419433594,
      "activations/layer14_attention_weight_min": -29.361949920654297,
      "activations/layer15_attention_weight_max": 32.760047912597656,
      "activations/layer15_attention_weight_min": -30.327001571655273,
      "activations/layer16_attention_weight_max": 34.34328842163086,
      "activations/layer16_attention_weight_min": -31.81614875793457,
      "activations/layer17_attention_weight_max": 51.34031295776367,
      "activations/layer17_attention_weight_min": -43.9456672668457,
      "activations/layer18_attention_weight_max": 51.0091552734375,
      "activations/layer18_attention_weight_min": -38.726409912109375,
      "activations/layer19_attention_weight_max": 23.662328720092773,
      "activations/layer19_attention_weight_min": -22.254552841186523,
      "activations/layer1_attention_weight_max": 16.561756134033203,
      "activations/layer1_attention_weight_min": -16.309526443481445,
      "activations/layer20_attention_weight_max": 22.722209930419922,
      "activations/layer20_attention_weight_min": -21.361602783203125,
      "activations/layer21_attention_weight_max": 36.42863464355469,
      "activations/layer21_attention_weight_min": -23.607797622680664,
      "activations/layer22_attention_weight_max": 29.59575080871582,
      "activations/layer22_attention_weight_min": -27.026533126831055,
      "activations/layer23_attention_weight_max": 41.84223937988281,
      "activations/layer23_attention_weight_min": -25.941022872924805,
      "activations/layer2_attention_weight_max": 30.997358322143555,
      "activations/layer2_attention_weight_min": -30.853862762451172,
      "activations/layer3_attention_weight_max": 88.22129821777344,
      "activations/layer3_attention_weight_min": -88.66283416748047,
      "activations/layer4_attention_weight_max": 88.61388397216797,
      "activations/layer4_attention_weight_min": -85.12711334228516,
      "activations/layer5_attention_weight_max": 61.93565368652344,
      "activations/layer5_attention_weight_min": -74.70826721191406,
      "activations/layer6_attention_weight_max": 48.802330017089844,
      "activations/layer6_attention_weight_min": -50.174861907958984,
      "activations/layer7_attention_weight_max": 62.754913330078125,
      "activations/layer7_attention_weight_min": -62.187347412109375,
      "activations/layer8_attention_weight_max": 44.60896682739258,
      "activations/layer8_attention_weight_min": -49.15669631958008,
      "activations/layer9_attention_weight_max": 43.49534225463867,
      "activations/layer9_attention_weight_min": -44.59574508666992,
      "epoch": 17.18,
      "learning_rate": 3.959848484848484e-05,
      "loss": 2.745,
      "step": 295700
    },
    {
      "activations/layer0_attention_weight_max": 15.024173736572266,
      "activations/layer0_attention_weight_min": -12.740965843200684,
      "activations/layer10_attention_weight_max": 33.66130447387695,
      "activations/layer10_attention_weight_min": -34.933502197265625,
      "activations/layer11_attention_weight_max": 33.34288787841797,
      "activations/layer11_attention_weight_min": -34.663516998291016,
      "activations/layer12_attention_weight_max": 18.639284133911133,
      "activations/layer12_attention_weight_min": -23.37822151184082,
      "activations/layer13_attention_weight_max": 32.25283432006836,
      "activations/layer13_attention_weight_min": -27.05129051208496,
      "activations/layer14_attention_weight_max": 34.805442810058594,
      "activations/layer14_attention_weight_min": -31.334064483642578,
      "activations/layer15_attention_weight_max": 32.30450439453125,
      "activations/layer15_attention_weight_min": -29.736608505249023,
      "activations/layer16_attention_weight_max": 34.162147521972656,
      "activations/layer16_attention_weight_min": -31.08294677734375,
      "activations/layer17_attention_weight_max": 55.60478591918945,
      "activations/layer17_attention_weight_min": -46.44247055053711,
      "activations/layer18_attention_weight_max": 48.38072967529297,
      "activations/layer18_attention_weight_min": -40.178409576416016,
      "activations/layer19_attention_weight_max": 23.259489059448242,
      "activations/layer19_attention_weight_min": -22.71708869934082,
      "activations/layer1_attention_weight_max": 16.09483528137207,
      "activations/layer1_attention_weight_min": -14.037562370300293,
      "activations/layer20_attention_weight_max": 20.534452438354492,
      "activations/layer20_attention_weight_min": -21.671964645385742,
      "activations/layer21_attention_weight_max": 33.1854133605957,
      "activations/layer21_attention_weight_min": -20.46026611328125,
      "activations/layer22_attention_weight_max": 28.371681213378906,
      "activations/layer22_attention_weight_min": -27.49858283996582,
      "activations/layer23_attention_weight_max": 37.964088439941406,
      "activations/layer23_attention_weight_min": -25.649044036865234,
      "activations/layer2_attention_weight_max": 32.547054290771484,
      "activations/layer2_attention_weight_min": -29.662254333496094,
      "activations/layer3_attention_weight_max": 91.27391815185547,
      "activations/layer3_attention_weight_min": -94.85369110107422,
      "activations/layer4_attention_weight_max": 85.69926452636719,
      "activations/layer4_attention_weight_min": -88.18151092529297,
      "activations/layer5_attention_weight_max": 66.94157409667969,
      "activations/layer5_attention_weight_min": -71.03939056396484,
      "activations/layer6_attention_weight_max": 49.762062072753906,
      "activations/layer6_attention_weight_min": -50.43556594848633,
      "activations/layer7_attention_weight_max": 65.93060302734375,
      "activations/layer7_attention_weight_min": -63.17457962036133,
      "activations/layer8_attention_weight_max": 44.9007453918457,
      "activations/layer8_attention_weight_min": -48.5136833190918,
      "activations/layer9_attention_weight_max": 42.152042388916016,
      "activations/layer9_attention_weight_min": -44.41997146606445,
      "epoch": 17.18,
      "learning_rate": 3.957954545454545e-05,
      "loss": 2.7287,
      "step": 295750
    },
    {
      "activations/layer0_attention_weight_max": 16.432811737060547,
      "activations/layer0_attention_weight_min": -13.52293586730957,
      "activations/layer10_attention_weight_max": 49.0469856262207,
      "activations/layer10_attention_weight_min": -46.18549728393555,
      "activations/layer11_attention_weight_max": 45.578269958496094,
      "activations/layer11_attention_weight_min": -42.41061019897461,
      "activations/layer12_attention_weight_max": 22.252073287963867,
      "activations/layer12_attention_weight_min": -29.59260368347168,
      "activations/layer13_attention_weight_max": 48.584373474121094,
      "activations/layer13_attention_weight_min": -31.320226669311523,
      "activations/layer14_attention_weight_max": 61.55625915527344,
      "activations/layer14_attention_weight_min": -36.290489196777344,
      "activations/layer15_attention_weight_max": 62.025474548339844,
      "activations/layer15_attention_weight_min": -33.42753982543945,
      "activations/layer16_attention_weight_max": 46.896488189697266,
      "activations/layer16_attention_weight_min": -40.69183349609375,
      "activations/layer17_attention_weight_max": 67.08047485351562,
      "activations/layer17_attention_weight_min": -59.54032516479492,
      "activations/layer18_attention_weight_max": 60.466365814208984,
      "activations/layer18_attention_weight_min": -52.68085479736328,
      "activations/layer19_attention_weight_max": 25.527084350585938,
      "activations/layer19_attention_weight_min": -25.601247787475586,
      "activations/layer1_attention_weight_max": 16.25714874267578,
      "activations/layer1_attention_weight_min": -14.263457298278809,
      "activations/layer20_attention_weight_max": 25.194822311401367,
      "activations/layer20_attention_weight_min": -22.313066482543945,
      "activations/layer21_attention_weight_max": 44.30973815917969,
      "activations/layer21_attention_weight_min": -26.003061294555664,
      "activations/layer22_attention_weight_max": 32.911285400390625,
      "activations/layer22_attention_weight_min": -25.50485610961914,
      "activations/layer23_attention_weight_max": 35.39226150512695,
      "activations/layer23_attention_weight_min": -22.87617301940918,
      "activations/layer2_attention_weight_max": 36.10177993774414,
      "activations/layer2_attention_weight_min": -32.40955352783203,
      "activations/layer3_attention_weight_max": 99.2175064086914,
      "activations/layer3_attention_weight_min": -95.17464447021484,
      "activations/layer4_attention_weight_max": 96.20803833007812,
      "activations/layer4_attention_weight_min": -90.69806671142578,
      "activations/layer5_attention_weight_max": 65.58964538574219,
      "activations/layer5_attention_weight_min": -69.06327819824219,
      "activations/layer6_attention_weight_max": 58.62294387817383,
      "activations/layer6_attention_weight_min": -55.878639221191406,
      "activations/layer7_attention_weight_max": 81.20213317871094,
      "activations/layer7_attention_weight_min": -75.0555191040039,
      "activations/layer8_attention_weight_max": 58.26129913330078,
      "activations/layer8_attention_weight_min": -57.29556655883789,
      "activations/layer9_attention_weight_max": 60.28398513793945,
      "activations/layer9_attention_weight_min": -53.39093017578125,
      "epoch": 17.19,
      "learning_rate": 3.956060606060605e-05,
      "loss": 2.7351,
      "step": 295800
    },
    {
      "activations/layer0_attention_weight_max": 15.722309112548828,
      "activations/layer0_attention_weight_min": -12.706184387207031,
      "activations/layer10_attention_weight_max": 37.1119499206543,
      "activations/layer10_attention_weight_min": -34.12687683105469,
      "activations/layer11_attention_weight_max": 33.50831604003906,
      "activations/layer11_attention_weight_min": -34.19601821899414,
      "activations/layer12_attention_weight_max": 20.094898223876953,
      "activations/layer12_attention_weight_min": -27.3055477142334,
      "activations/layer13_attention_weight_max": 43.50804901123047,
      "activations/layer13_attention_weight_min": -27.996265411376953,
      "activations/layer14_attention_weight_max": 41.4938850402832,
      "activations/layer14_attention_weight_min": -30.742807388305664,
      "activations/layer15_attention_weight_max": 44.458740234375,
      "activations/layer15_attention_weight_min": -30.473379135131836,
      "activations/layer16_attention_weight_max": 37.94902801513672,
      "activations/layer16_attention_weight_min": -31.8309326171875,
      "activations/layer17_attention_weight_max": 56.854736328125,
      "activations/layer17_attention_weight_min": -44.71552276611328,
      "activations/layer18_attention_weight_max": 56.85707473754883,
      "activations/layer18_attention_weight_min": -42.08885955810547,
      "activations/layer19_attention_weight_max": 25.296297073364258,
      "activations/layer19_attention_weight_min": -22.70216178894043,
      "activations/layer1_attention_weight_max": 15.959592819213867,
      "activations/layer1_attention_weight_min": -13.669818878173828,
      "activations/layer20_attention_weight_max": 24.409799575805664,
      "activations/layer20_attention_weight_min": -21.427366256713867,
      "activations/layer21_attention_weight_max": 50.6143913269043,
      "activations/layer21_attention_weight_min": -22.225082397460938,
      "activations/layer22_attention_weight_max": 29.578353881835938,
      "activations/layer22_attention_weight_min": -25.45323371887207,
      "activations/layer23_attention_weight_max": 37.89910888671875,
      "activations/layer23_attention_weight_min": -26.087417602539062,
      "activations/layer2_attention_weight_max": 31.817081451416016,
      "activations/layer2_attention_weight_min": -29.06157684326172,
      "activations/layer3_attention_weight_max": 85.49221801757812,
      "activations/layer3_attention_weight_min": -87.24736785888672,
      "activations/layer4_attention_weight_max": 86.17928314208984,
      "activations/layer4_attention_weight_min": -84.9070053100586,
      "activations/layer5_attention_weight_max": 65.06380462646484,
      "activations/layer5_attention_weight_min": -72.38743591308594,
      "activations/layer6_attention_weight_max": 47.337669372558594,
      "activations/layer6_attention_weight_min": -48.56303787231445,
      "activations/layer7_attention_weight_max": 65.9714126586914,
      "activations/layer7_attention_weight_min": -64.95747375488281,
      "activations/layer8_attention_weight_max": 45.67179489135742,
      "activations/layer8_attention_weight_min": -46.971866607666016,
      "activations/layer9_attention_weight_max": 48.20702362060547,
      "activations/layer9_attention_weight_min": -45.39900207519531,
      "epoch": 17.19,
      "learning_rate": 3.954166666666667e-05,
      "loss": 2.7457,
      "step": 295850
    },
    {
      "activations/layer0_attention_weight_max": 16.05527114868164,
      "activations/layer0_attention_weight_min": -12.631290435791016,
      "activations/layer10_attention_weight_max": 32.72624206542969,
      "activations/layer10_attention_weight_min": -33.659271240234375,
      "activations/layer11_attention_weight_max": 30.02454376220703,
      "activations/layer11_attention_weight_min": -31.108055114746094,
      "activations/layer12_attention_weight_max": 19.066537857055664,
      "activations/layer12_attention_weight_min": -25.04233169555664,
      "activations/layer13_attention_weight_max": 32.35565185546875,
      "activations/layer13_attention_weight_min": -25.97002601623535,
      "activations/layer14_attention_weight_max": 35.86888885498047,
      "activations/layer14_attention_weight_min": -30.889328002929688,
      "activations/layer15_attention_weight_max": 29.856592178344727,
      "activations/layer15_attention_weight_min": -28.866973876953125,
      "activations/layer16_attention_weight_max": 32.950923919677734,
      "activations/layer16_attention_weight_min": -32.008968353271484,
      "activations/layer17_attention_weight_max": 48.67995834350586,
      "activations/layer17_attention_weight_min": -46.161407470703125,
      "activations/layer18_attention_weight_max": 46.80182647705078,
      "activations/layer18_attention_weight_min": -40.651954650878906,
      "activations/layer19_attention_weight_max": 22.789365768432617,
      "activations/layer19_attention_weight_min": -22.488143920898438,
      "activations/layer1_attention_weight_max": 16.91135597229004,
      "activations/layer1_attention_weight_min": -15.53227424621582,
      "activations/layer20_attention_weight_max": 23.52704620361328,
      "activations/layer20_attention_weight_min": -23.11381721496582,
      "activations/layer21_attention_weight_max": 35.493408203125,
      "activations/layer21_attention_weight_min": -22.250886917114258,
      "activations/layer22_attention_weight_max": 28.031675338745117,
      "activations/layer22_attention_weight_min": -26.47812843322754,
      "activations/layer23_attention_weight_max": 36.872215270996094,
      "activations/layer23_attention_weight_min": -25.036117553710938,
      "activations/layer2_attention_weight_max": 29.996490478515625,
      "activations/layer2_attention_weight_min": -28.618144989013672,
      "activations/layer3_attention_weight_max": 82.01045989990234,
      "activations/layer3_attention_weight_min": -87.72297668457031,
      "activations/layer4_attention_weight_max": 84.10877990722656,
      "activations/layer4_attention_weight_min": -83.31581115722656,
      "activations/layer5_attention_weight_max": 62.86109924316406,
      "activations/layer5_attention_weight_min": -74.80389404296875,
      "activations/layer6_attention_weight_max": 50.83580780029297,
      "activations/layer6_attention_weight_min": -48.41101837158203,
      "activations/layer7_attention_weight_max": 60.85121536254883,
      "activations/layer7_attention_weight_min": -61.918067932128906,
      "activations/layer8_attention_weight_max": 42.410301208496094,
      "activations/layer8_attention_weight_min": -44.259422302246094,
      "activations/layer9_attention_weight_max": 41.51176452636719,
      "activations/layer9_attention_weight_min": -42.85003662109375,
      "epoch": 17.19,
      "learning_rate": 3.952272727272727e-05,
      "loss": 2.7396,
      "step": 295900
    },
    {
      "activations/layer0_attention_weight_max": 15.61623764038086,
      "activations/layer0_attention_weight_min": -12.717201232910156,
      "activations/layer10_attention_weight_max": 34.47654342651367,
      "activations/layer10_attention_weight_min": -34.17273712158203,
      "activations/layer11_attention_weight_max": 32.35475540161133,
      "activations/layer11_attention_weight_min": -32.22064971923828,
      "activations/layer12_attention_weight_max": 19.477508544921875,
      "activations/layer12_attention_weight_min": -23.661792755126953,
      "activations/layer13_attention_weight_max": 38.705955505371094,
      "activations/layer13_attention_weight_min": -31.748445510864258,
      "activations/layer14_attention_weight_max": 33.7153434753418,
      "activations/layer14_attention_weight_min": -29.646652221679688,
      "activations/layer15_attention_weight_max": 30.664274215698242,
      "activations/layer15_attention_weight_min": -28.083480834960938,
      "activations/layer16_attention_weight_max": 32.863868713378906,
      "activations/layer16_attention_weight_min": -30.59848976135254,
      "activations/layer17_attention_weight_max": 49.661861419677734,
      "activations/layer17_attention_weight_min": -41.52984619140625,
      "activations/layer18_attention_weight_max": 46.83369064331055,
      "activations/layer18_attention_weight_min": -35.87942886352539,
      "activations/layer19_attention_weight_max": 22.9066104888916,
      "activations/layer19_attention_weight_min": -22.826223373413086,
      "activations/layer1_attention_weight_max": 15.89391803741455,
      "activations/layer1_attention_weight_min": -14.787871360778809,
      "activations/layer20_attention_weight_max": 21.865697860717773,
      "activations/layer20_attention_weight_min": -22.079219818115234,
      "activations/layer21_attention_weight_max": 37.53899383544922,
      "activations/layer21_attention_weight_min": -23.91075325012207,
      "activations/layer22_attention_weight_max": 26.1274356842041,
      "activations/layer22_attention_weight_min": -25.793861389160156,
      "activations/layer23_attention_weight_max": 37.27953338623047,
      "activations/layer23_attention_weight_min": -26.486967086791992,
      "activations/layer2_attention_weight_max": 31.10170555114746,
      "activations/layer2_attention_weight_min": -30.475378036499023,
      "activations/layer3_attention_weight_max": 89.35586547851562,
      "activations/layer3_attention_weight_min": -89.51728057861328,
      "activations/layer4_attention_weight_max": 87.265625,
      "activations/layer4_attention_weight_min": -83.84838104248047,
      "activations/layer5_attention_weight_max": 65.42581176757812,
      "activations/layer5_attention_weight_min": -76.85059356689453,
      "activations/layer6_attention_weight_max": 50.88337707519531,
      "activations/layer6_attention_weight_min": -49.933387756347656,
      "activations/layer7_attention_weight_max": 62.37556457519531,
      "activations/layer7_attention_weight_min": -65.32572174072266,
      "activations/layer8_attention_weight_max": 43.475257873535156,
      "activations/layer8_attention_weight_min": -45.62736892700195,
      "activations/layer9_attention_weight_max": 40.0942268371582,
      "activations/layer9_attention_weight_min": -43.997554779052734,
      "epoch": 17.2,
      "learning_rate": 3.950378787878787e-05,
      "loss": 2.7235,
      "step": 295950
    },
    {
      "activations/layer0_attention_weight_max": 15.416801452636719,
      "activations/layer0_attention_weight_min": -12.75011157989502,
      "activations/layer10_attention_weight_max": 38.663307189941406,
      "activations/layer10_attention_weight_min": -37.36272048950195,
      "activations/layer11_attention_weight_max": 33.62190628051758,
      "activations/layer11_attention_weight_min": -35.779754638671875,
      "activations/layer12_attention_weight_max": 19.63612174987793,
      "activations/layer12_attention_weight_min": -24.255762100219727,
      "activations/layer13_attention_weight_max": 38.11932373046875,
      "activations/layer13_attention_weight_min": -28.545656204223633,
      "activations/layer14_attention_weight_max": 35.71717834472656,
      "activations/layer14_attention_weight_min": -29.989492416381836,
      "activations/layer15_attention_weight_max": 34.36941146850586,
      "activations/layer15_attention_weight_min": -30.8687744140625,
      "activations/layer16_attention_weight_max": 33.65876770019531,
      "activations/layer16_attention_weight_min": -31.38572883605957,
      "activations/layer17_attention_weight_max": 54.97705078125,
      "activations/layer17_attention_weight_min": -47.306190490722656,
      "activations/layer18_attention_weight_max": 46.28666305541992,
      "activations/layer18_attention_weight_min": -38.96575927734375,
      "activations/layer19_attention_weight_max": 22.606107711791992,
      "activations/layer19_attention_weight_min": -22.693063735961914,
      "activations/layer1_attention_weight_max": 15.911944389343262,
      "activations/layer1_attention_weight_min": -13.805792808532715,
      "activations/layer20_attention_weight_max": 22.584718704223633,
      "activations/layer20_attention_weight_min": -20.552207946777344,
      "activations/layer21_attention_weight_max": 33.4963264465332,
      "activations/layer21_attention_weight_min": -20.799640655517578,
      "activations/layer22_attention_weight_max": 26.07960319519043,
      "activations/layer22_attention_weight_min": -26.53191566467285,
      "activations/layer23_attention_weight_max": 39.6057014465332,
      "activations/layer23_attention_weight_min": -22.35995864868164,
      "activations/layer2_attention_weight_max": 32.13380813598633,
      "activations/layer2_attention_weight_min": -30.277294158935547,
      "activations/layer3_attention_weight_max": 91.79443359375,
      "activations/layer3_attention_weight_min": -91.11441040039062,
      "activations/layer4_attention_weight_max": 95.24828338623047,
      "activations/layer4_attention_weight_min": -87.50237274169922,
      "activations/layer5_attention_weight_max": 65.56578063964844,
      "activations/layer5_attention_weight_min": -73.66793823242188,
      "activations/layer6_attention_weight_max": 52.80598068237305,
      "activations/layer6_attention_weight_min": -51.018184661865234,
      "activations/layer7_attention_weight_max": 66.01105499267578,
      "activations/layer7_attention_weight_min": -65.51966094970703,
      "activations/layer8_attention_weight_max": 51.86796569824219,
      "activations/layer8_attention_weight_min": -48.48661804199219,
      "activations/layer9_attention_weight_max": 46.48957061767578,
      "activations/layer9_attention_weight_min": -48.00571823120117,
      "epoch": 17.2,
      "learning_rate": 3.948484848484848e-05,
      "loss": 2.7491,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_loss": 2.69921875,
      "eval_runtime": 8.4502,
      "eval_samples_per_second": 508.151,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_openwebtext_loss": 2.69921875,
      "eval_openwebtext_ppl": 14.868111474214228,
      "eval_openwebtext_runtime": 8.4502,
      "eval_openwebtext_samples_per_second": 508.151,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9284,
      "eval_wikitext_samples_per_second": 236.469,
      "step": 296000
    },
    {
      "epoch": 17.2,
      "eval_lambada_loss": 2.51953125,
      "eval_lambada_ppl": 12.422772124124576,
      "eval_lambada_runtime": 9.5099,
      "eval_lambada_samples_per_second": 511.994,
      "step": 296000
    },
    {
      "activations/layer0_attention_weight_max": 14.882973670959473,
      "activations/layer0_attention_weight_min": -12.629131317138672,
      "activations/layer10_attention_weight_max": 35.13146209716797,
      "activations/layer10_attention_weight_min": -36.417510986328125,
      "activations/layer11_attention_weight_max": 31.058895111083984,
      "activations/layer11_attention_weight_min": -32.17040252685547,
      "activations/layer12_attention_weight_max": 19.94553565979004,
      "activations/layer12_attention_weight_min": -27.912765502929688,
      "activations/layer13_attention_weight_max": 39.88214874267578,
      "activations/layer13_attention_weight_min": -29.035396575927734,
      "activations/layer14_attention_weight_max": 37.20087814331055,
      "activations/layer14_attention_weight_min": -30.43914222717285,
      "activations/layer15_attention_weight_max": 35.7302360534668,
      "activations/layer15_attention_weight_min": -29.849498748779297,
      "activations/layer16_attention_weight_max": 36.439605712890625,
      "activations/layer16_attention_weight_min": -33.981990814208984,
      "activations/layer17_attention_weight_max": 55.1111946105957,
      "activations/layer17_attention_weight_min": -46.34856414794922,
      "activations/layer18_attention_weight_max": 52.498294830322266,
      "activations/layer18_attention_weight_min": -39.81460189819336,
      "activations/layer19_attention_weight_max": 29.16440773010254,
      "activations/layer19_attention_weight_min": -23.308053970336914,
      "activations/layer1_attention_weight_max": 16.0861759185791,
      "activations/layer1_attention_weight_min": -14.487350463867188,
      "activations/layer20_attention_weight_max": 25.901443481445312,
      "activations/layer20_attention_weight_min": -22.627172470092773,
      "activations/layer21_attention_weight_max": 34.24516677856445,
      "activations/layer21_attention_weight_min": -24.265789031982422,
      "activations/layer22_attention_weight_max": 32.98597717285156,
      "activations/layer22_attention_weight_min": -28.1758975982666,
      "activations/layer23_attention_weight_max": 37.7396240234375,
      "activations/layer23_attention_weight_min": -24.949413299560547,
      "activations/layer2_attention_weight_max": 32.274810791015625,
      "activations/layer2_attention_weight_min": -29.76314353942871,
      "activations/layer3_attention_weight_max": 92.16659545898438,
      "activations/layer3_attention_weight_min": -92.48193359375,
      "activations/layer4_attention_weight_max": 87.6322250366211,
      "activations/layer4_attention_weight_min": -87.17797088623047,
      "activations/layer5_attention_weight_max": 65.20726013183594,
      "activations/layer5_attention_weight_min": -73.34725952148438,
      "activations/layer6_attention_weight_max": 47.67912673950195,
      "activations/layer6_attention_weight_min": -49.70827865600586,
      "activations/layer7_attention_weight_max": 62.642066955566406,
      "activations/layer7_attention_weight_min": -65.33135223388672,
      "activations/layer8_attention_weight_max": 45.11688232421875,
      "activations/layer8_attention_weight_min": -47.95729446411133,
      "activations/layer9_attention_weight_max": 44.0798454284668,
      "activations/layer9_attention_weight_min": -45.71863555908203,
      "epoch": 17.2,
      "learning_rate": 3.946590909090909e-05,
      "loss": 2.7393,
      "step": 296050
    },
    {
      "activations/layer0_attention_weight_max": 15.294958114624023,
      "activations/layer0_attention_weight_min": -13.593647956848145,
      "activations/layer10_attention_weight_max": 35.70924758911133,
      "activations/layer10_attention_weight_min": -35.07402038574219,
      "activations/layer11_attention_weight_max": 33.10223388671875,
      "activations/layer11_attention_weight_min": -32.8912467956543,
      "activations/layer12_attention_weight_max": 18.911203384399414,
      "activations/layer12_attention_weight_min": -25.222211837768555,
      "activations/layer13_attention_weight_max": 31.872528076171875,
      "activations/layer13_attention_weight_min": -26.833852767944336,
      "activations/layer14_attention_weight_max": 33.623741149902344,
      "activations/layer14_attention_weight_min": -32.283287048339844,
      "activations/layer15_attention_weight_max": 30.3945255279541,
      "activations/layer15_attention_weight_min": -30.884355545043945,
      "activations/layer16_attention_weight_max": 31.851146697998047,
      "activations/layer16_attention_weight_min": -31.149654388427734,
      "activations/layer17_attention_weight_max": 51.04745864868164,
      "activations/layer17_attention_weight_min": -47.50209426879883,
      "activations/layer18_attention_weight_max": 43.21992492675781,
      "activations/layer18_attention_weight_min": -41.90378189086914,
      "activations/layer19_attention_weight_max": 24.72398567199707,
      "activations/layer19_attention_weight_min": -21.997724533081055,
      "activations/layer1_attention_weight_max": 15.99317455291748,
      "activations/layer1_attention_weight_min": -13.873440742492676,
      "activations/layer20_attention_weight_max": 21.157913208007812,
      "activations/layer20_attention_weight_min": -21.071258544921875,
      "activations/layer21_attention_weight_max": 29.990713119506836,
      "activations/layer21_attention_weight_min": -21.1014461517334,
      "activations/layer22_attention_weight_max": 29.51671600341797,
      "activations/layer22_attention_weight_min": -24.83058738708496,
      "activations/layer23_attention_weight_max": 36.53101348876953,
      "activations/layer23_attention_weight_min": -22.12242889404297,
      "activations/layer2_attention_weight_max": 34.1420783996582,
      "activations/layer2_attention_weight_min": -32.318092346191406,
      "activations/layer3_attention_weight_max": 93.66960144042969,
      "activations/layer3_attention_weight_min": -99.8863754272461,
      "activations/layer4_attention_weight_max": 92.33602905273438,
      "activations/layer4_attention_weight_min": -88.05330657958984,
      "activations/layer5_attention_weight_max": 64.6227035522461,
      "activations/layer5_attention_weight_min": -71.5894775390625,
      "activations/layer6_attention_weight_max": 51.72853469848633,
      "activations/layer6_attention_weight_min": -52.61915588378906,
      "activations/layer7_attention_weight_max": 65.412841796875,
      "activations/layer7_attention_weight_min": -65.59709930419922,
      "activations/layer8_attention_weight_max": 43.44780349731445,
      "activations/layer8_attention_weight_min": -47.23069763183594,
      "activations/layer9_attention_weight_max": 41.81935119628906,
      "activations/layer9_attention_weight_min": -46.57838439941406,
      "epoch": 17.21,
      "learning_rate": 3.9446969696969696e-05,
      "loss": 2.7415,
      "step": 296100
    },
    {
      "activations/layer0_attention_weight_max": 15.720199584960938,
      "activations/layer0_attention_weight_min": -12.749200820922852,
      "activations/layer10_attention_weight_max": 32.84696578979492,
      "activations/layer10_attention_weight_min": -32.31285095214844,
      "activations/layer11_attention_weight_max": 30.298452377319336,
      "activations/layer11_attention_weight_min": -33.598411560058594,
      "activations/layer12_attention_weight_max": 17.56831169128418,
      "activations/layer12_attention_weight_min": -24.131311416625977,
      "activations/layer13_attention_weight_max": 30.188522338867188,
      "activations/layer13_attention_weight_min": -26.34023094177246,
      "activations/layer14_attention_weight_max": 32.96189880371094,
      "activations/layer14_attention_weight_min": -30.096248626708984,
      "activations/layer15_attention_weight_max": 32.838111877441406,
      "activations/layer15_attention_weight_min": -29.130748748779297,
      "activations/layer16_attention_weight_max": 31.294071197509766,
      "activations/layer16_attention_weight_min": -31.273258209228516,
      "activations/layer17_attention_weight_max": 48.665794372558594,
      "activations/layer17_attention_weight_min": -43.880313873291016,
      "activations/layer18_attention_weight_max": 42.86219024658203,
      "activations/layer18_attention_weight_min": -40.15637969970703,
      "activations/layer19_attention_weight_max": 22.040512084960938,
      "activations/layer19_attention_weight_min": -21.692663192749023,
      "activations/layer1_attention_weight_max": 15.825593948364258,
      "activations/layer1_attention_weight_min": -14.206750869750977,
      "activations/layer20_attention_weight_max": 21.511899948120117,
      "activations/layer20_attention_weight_min": -21.790090560913086,
      "activations/layer21_attention_weight_max": 33.427276611328125,
      "activations/layer21_attention_weight_min": -24.725027084350586,
      "activations/layer22_attention_weight_max": 26.58346939086914,
      "activations/layer22_attention_weight_min": -26.459930419921875,
      "activations/layer23_attention_weight_max": 33.673831939697266,
      "activations/layer23_attention_weight_min": -28.010831832885742,
      "activations/layer2_attention_weight_max": 32.97704315185547,
      "activations/layer2_attention_weight_min": -30.078472137451172,
      "activations/layer3_attention_weight_max": 88.97920989990234,
      "activations/layer3_attention_weight_min": -89.8628158569336,
      "activations/layer4_attention_weight_max": 84.3986587524414,
      "activations/layer4_attention_weight_min": -84.30326843261719,
      "activations/layer5_attention_weight_max": 61.95748519897461,
      "activations/layer5_attention_weight_min": -77.00086975097656,
      "activations/layer6_attention_weight_max": 48.75243377685547,
      "activations/layer6_attention_weight_min": -48.90414047241211,
      "activations/layer7_attention_weight_max": 63.96440505981445,
      "activations/layer7_attention_weight_min": -64.14346313476562,
      "activations/layer8_attention_weight_max": 42.59180450439453,
      "activations/layer8_attention_weight_min": -48.23147201538086,
      "activations/layer9_attention_weight_max": 42.45188903808594,
      "activations/layer9_attention_weight_min": -44.846012115478516,
      "epoch": 17.21,
      "learning_rate": 3.94280303030303e-05,
      "loss": 2.7398,
      "step": 296150
    },
    {
      "activations/layer0_attention_weight_max": 16.182329177856445,
      "activations/layer0_attention_weight_min": -13.685894012451172,
      "activations/layer10_attention_weight_max": 35.83391189575195,
      "activations/layer10_attention_weight_min": -36.464332580566406,
      "activations/layer11_attention_weight_max": 32.796714782714844,
      "activations/layer11_attention_weight_min": -32.849205017089844,
      "activations/layer12_attention_weight_max": 17.55255126953125,
      "activations/layer12_attention_weight_min": -28.65005111694336,
      "activations/layer13_attention_weight_max": 32.604557037353516,
      "activations/layer13_attention_weight_min": -28.734376907348633,
      "activations/layer14_attention_weight_max": 31.223304748535156,
      "activations/layer14_attention_weight_min": -30.763660430908203,
      "activations/layer15_attention_weight_max": 28.36394691467285,
      "activations/layer15_attention_weight_min": -30.706043243408203,
      "activations/layer16_attention_weight_max": 29.094152450561523,
      "activations/layer16_attention_weight_min": -30.40594482421875,
      "activations/layer17_attention_weight_max": 44.50844192504883,
      "activations/layer17_attention_weight_min": -41.87381362915039,
      "activations/layer18_attention_weight_max": 38.766929626464844,
      "activations/layer18_attention_weight_min": -35.771881103515625,
      "activations/layer19_attention_weight_max": 22.036603927612305,
      "activations/layer19_attention_weight_min": -20.369834899902344,
      "activations/layer1_attention_weight_max": 16.734329223632812,
      "activations/layer1_attention_weight_min": -14.932884216308594,
      "activations/layer20_attention_weight_max": 18.94688606262207,
      "activations/layer20_attention_weight_min": -20.2974910736084,
      "activations/layer21_attention_weight_max": 30.41811752319336,
      "activations/layer21_attention_weight_min": -21.891698837280273,
      "activations/layer22_attention_weight_max": 25.85840606689453,
      "activations/layer22_attention_weight_min": -24.31855583190918,
      "activations/layer23_attention_weight_max": 31.952970504760742,
      "activations/layer23_attention_weight_min": -26.90448760986328,
      "activations/layer2_attention_weight_max": 32.28569030761719,
      "activations/layer2_attention_weight_min": -30.975101470947266,
      "activations/layer3_attention_weight_max": 91.0280990600586,
      "activations/layer3_attention_weight_min": -89.40042877197266,
      "activations/layer4_attention_weight_max": 90.38013458251953,
      "activations/layer4_attention_weight_min": -84.89293670654297,
      "activations/layer5_attention_weight_max": 63.88273239135742,
      "activations/layer5_attention_weight_min": -71.26646423339844,
      "activations/layer6_attention_weight_max": 49.549720764160156,
      "activations/layer6_attention_weight_min": -49.343143463134766,
      "activations/layer7_attention_weight_max": 64.09645080566406,
      "activations/layer7_attention_weight_min": -63.786277770996094,
      "activations/layer8_attention_weight_max": 45.981693267822266,
      "activations/layer8_attention_weight_min": -48.973602294921875,
      "activations/layer9_attention_weight_max": 45.49662399291992,
      "activations/layer9_attention_weight_min": -48.43344497680664,
      "epoch": 17.21,
      "learning_rate": 3.940909090909091e-05,
      "loss": 2.7488,
      "step": 296200
    },
    {
      "activations/layer0_attention_weight_max": 15.352401733398438,
      "activations/layer0_attention_weight_min": -12.90834903717041,
      "activations/layer10_attention_weight_max": 32.49976348876953,
      "activations/layer10_attention_weight_min": -34.55031967163086,
      "activations/layer11_attention_weight_max": 30.15033531188965,
      "activations/layer11_attention_weight_min": -32.2882194519043,
      "activations/layer12_attention_weight_max": 17.248950958251953,
      "activations/layer12_attention_weight_min": -28.3646183013916,
      "activations/layer13_attention_weight_max": 33.77118682861328,
      "activations/layer13_attention_weight_min": -28.989376068115234,
      "activations/layer14_attention_weight_max": 31.459468841552734,
      "activations/layer14_attention_weight_min": -29.69645118713379,
      "activations/layer15_attention_weight_max": 30.965375900268555,
      "activations/layer15_attention_weight_min": -30.78737449645996,
      "activations/layer16_attention_weight_max": 32.48393630981445,
      "activations/layer16_attention_weight_min": -32.726409912109375,
      "activations/layer17_attention_weight_max": 48.314815521240234,
      "activations/layer17_attention_weight_min": -47.48765182495117,
      "activations/layer18_attention_weight_max": 40.28512191772461,
      "activations/layer18_attention_weight_min": -42.197139739990234,
      "activations/layer19_attention_weight_max": 20.939226150512695,
      "activations/layer19_attention_weight_min": -23.762577056884766,
      "activations/layer1_attention_weight_max": 16.23493194580078,
      "activations/layer1_attention_weight_min": -13.995739936828613,
      "activations/layer20_attention_weight_max": 21.690011978149414,
      "activations/layer20_attention_weight_min": -23.120065689086914,
      "activations/layer21_attention_weight_max": 32.67012023925781,
      "activations/layer21_attention_weight_min": -24.157268524169922,
      "activations/layer22_attention_weight_max": 28.58163833618164,
      "activations/layer22_attention_weight_min": -29.953861236572266,
      "activations/layer23_attention_weight_max": 34.770263671875,
      "activations/layer23_attention_weight_min": -25.818836212158203,
      "activations/layer2_attention_weight_max": 32.075225830078125,
      "activations/layer2_attention_weight_min": -30.2159423828125,
      "activations/layer3_attention_weight_max": 88.23661041259766,
      "activations/layer3_attention_weight_min": -88.43177032470703,
      "activations/layer4_attention_weight_max": 87.30815124511719,
      "activations/layer4_attention_weight_min": -83.59761047363281,
      "activations/layer5_attention_weight_max": 62.12495422363281,
      "activations/layer5_attention_weight_min": -72.4629898071289,
      "activations/layer6_attention_weight_max": 48.82162857055664,
      "activations/layer6_attention_weight_min": -49.37725067138672,
      "activations/layer7_attention_weight_max": 62.94746780395508,
      "activations/layer7_attention_weight_min": -65.28807067871094,
      "activations/layer8_attention_weight_max": 43.418243408203125,
      "activations/layer8_attention_weight_min": -45.0319709777832,
      "activations/layer9_attention_weight_max": 42.79304504394531,
      "activations/layer9_attention_weight_min": -45.64778137207031,
      "epoch": 17.21,
      "learning_rate": 3.9390151515151514e-05,
      "loss": 2.7453,
      "step": 296250
    },
    {
      "activations/layer0_attention_weight_max": 15.494709968566895,
      "activations/layer0_attention_weight_min": -12.827977180480957,
      "activations/layer10_attention_weight_max": 40.01416778564453,
      "activations/layer10_attention_weight_min": -36.25208282470703,
      "activations/layer11_attention_weight_max": 34.80523681640625,
      "activations/layer11_attention_weight_min": -33.651588439941406,
      "activations/layer12_attention_weight_max": 20.096097946166992,
      "activations/layer12_attention_weight_min": -27.081043243408203,
      "activations/layer13_attention_weight_max": 35.289703369140625,
      "activations/layer13_attention_weight_min": -27.00275993347168,
      "activations/layer14_attention_weight_max": 37.26383972167969,
      "activations/layer14_attention_weight_min": -29.885038375854492,
      "activations/layer15_attention_weight_max": 38.97498321533203,
      "activations/layer15_attention_weight_min": -29.04901123046875,
      "activations/layer16_attention_weight_max": 36.37428283691406,
      "activations/layer16_attention_weight_min": -33.466102600097656,
      "activations/layer17_attention_weight_max": 54.761070251464844,
      "activations/layer17_attention_weight_min": -47.93732833862305,
      "activations/layer18_attention_weight_max": 50.15282440185547,
      "activations/layer18_attention_weight_min": -42.60017013549805,
      "activations/layer19_attention_weight_max": 25.920801162719727,
      "activations/layer19_attention_weight_min": -26.501800537109375,
      "activations/layer1_attention_weight_max": 15.358835220336914,
      "activations/layer1_attention_weight_min": -15.275918960571289,
      "activations/layer20_attention_weight_max": 28.421480178833008,
      "activations/layer20_attention_weight_min": -21.47638702392578,
      "activations/layer21_attention_weight_max": 45.777652740478516,
      "activations/layer21_attention_weight_min": -23.68383026123047,
      "activations/layer22_attention_weight_max": 34.77255630493164,
      "activations/layer22_attention_weight_min": -26.797536849975586,
      "activations/layer23_attention_weight_max": 39.0235481262207,
      "activations/layer23_attention_weight_min": -23.85843276977539,
      "activations/layer2_attention_weight_max": 33.64943313598633,
      "activations/layer2_attention_weight_min": -32.19532012939453,
      "activations/layer3_attention_weight_max": 93.66486358642578,
      "activations/layer3_attention_weight_min": -92.67034149169922,
      "activations/layer4_attention_weight_max": 93.99671173095703,
      "activations/layer4_attention_weight_min": -90.62007904052734,
      "activations/layer5_attention_weight_max": 68.4233169555664,
      "activations/layer5_attention_weight_min": -80.5753402709961,
      "activations/layer6_attention_weight_max": 52.70934295654297,
      "activations/layer6_attention_weight_min": -54.755252838134766,
      "activations/layer7_attention_weight_max": 65.58120727539062,
      "activations/layer7_attention_weight_min": -71.12838745117188,
      "activations/layer8_attention_weight_max": 47.26384735107422,
      "activations/layer8_attention_weight_min": -51.80180358886719,
      "activations/layer9_attention_weight_max": 47.77682113647461,
      "activations/layer9_attention_weight_min": -46.000606536865234,
      "epoch": 17.22,
      "learning_rate": 3.9371212121212115e-05,
      "loss": 2.7407,
      "step": 296300
    },
    {
      "activations/layer0_attention_weight_max": 15.563265800476074,
      "activations/layer0_attention_weight_min": -12.891162872314453,
      "activations/layer10_attention_weight_max": 34.93358612060547,
      "activations/layer10_attention_weight_min": -34.41560745239258,
      "activations/layer11_attention_weight_max": 34.109867095947266,
      "activations/layer11_attention_weight_min": -32.677955627441406,
      "activations/layer12_attention_weight_max": 19.57586097717285,
      "activations/layer12_attention_weight_min": -28.24388885498047,
      "activations/layer13_attention_weight_max": 37.05776596069336,
      "activations/layer13_attention_weight_min": -29.62238311767578,
      "activations/layer14_attention_weight_max": 37.6822395324707,
      "activations/layer14_attention_weight_min": -28.59520149230957,
      "activations/layer15_attention_weight_max": 32.44411849975586,
      "activations/layer15_attention_weight_min": -28.12001609802246,
      "activations/layer16_attention_weight_max": 33.91710662841797,
      "activations/layer16_attention_weight_min": -31.83636474609375,
      "activations/layer17_attention_weight_max": 53.11003875732422,
      "activations/layer17_attention_weight_min": -44.67959976196289,
      "activations/layer18_attention_weight_max": 51.127559661865234,
      "activations/layer18_attention_weight_min": -41.13655471801758,
      "activations/layer19_attention_weight_max": 24.412763595581055,
      "activations/layer19_attention_weight_min": -22.674325942993164,
      "activations/layer1_attention_weight_max": 16.478967666625977,
      "activations/layer1_attention_weight_min": -14.384819984436035,
      "activations/layer20_attention_weight_max": 23.284568786621094,
      "activations/layer20_attention_weight_min": -22.626129150390625,
      "activations/layer21_attention_weight_max": 41.16082763671875,
      "activations/layer21_attention_weight_min": -21.42778778076172,
      "activations/layer22_attention_weight_max": 30.423128128051758,
      "activations/layer22_attention_weight_min": -28.14588165283203,
      "activations/layer23_attention_weight_max": 43.45721435546875,
      "activations/layer23_attention_weight_min": -23.506216049194336,
      "activations/layer2_attention_weight_max": 33.25414276123047,
      "activations/layer2_attention_weight_min": -31.751605987548828,
      "activations/layer3_attention_weight_max": 92.50340270996094,
      "activations/layer3_attention_weight_min": -94.73258209228516,
      "activations/layer4_attention_weight_max": 90.65755462646484,
      "activations/layer4_attention_weight_min": -88.92972564697266,
      "activations/layer5_attention_weight_max": 64.3971176147461,
      "activations/layer5_attention_weight_min": -71.1128921508789,
      "activations/layer6_attention_weight_max": 53.36219787597656,
      "activations/layer6_attention_weight_min": -51.39945602416992,
      "activations/layer7_attention_weight_max": 62.14070129394531,
      "activations/layer7_attention_weight_min": -63.51527404785156,
      "activations/layer8_attention_weight_max": 44.536094665527344,
      "activations/layer8_attention_weight_min": -48.17533493041992,
      "activations/layer9_attention_weight_max": 48.785179138183594,
      "activations/layer9_attention_weight_min": -44.36977767944336,
      "epoch": 17.22,
      "learning_rate": 3.9352272727272723e-05,
      "loss": 2.7389,
      "step": 296350
    },
    {
      "activations/layer0_attention_weight_max": 16.163667678833008,
      "activations/layer0_attention_weight_min": -13.018877983093262,
      "activations/layer10_attention_weight_max": 35.02146911621094,
      "activations/layer10_attention_weight_min": -34.314693450927734,
      "activations/layer11_attention_weight_max": 35.131103515625,
      "activations/layer11_attention_weight_min": -35.099464416503906,
      "activations/layer12_attention_weight_max": 20.834640502929688,
      "activations/layer12_attention_weight_min": -24.107114791870117,
      "activations/layer13_attention_weight_max": 31.461063385009766,
      "activations/layer13_attention_weight_min": -29.038860321044922,
      "activations/layer14_attention_weight_max": 34.00963592529297,
      "activations/layer14_attention_weight_min": -30.154743194580078,
      "activations/layer15_attention_weight_max": 33.33670425415039,
      "activations/layer15_attention_weight_min": -29.526004791259766,
      "activations/layer16_attention_weight_max": 34.03602981567383,
      "activations/layer16_attention_weight_min": -32.64017105102539,
      "activations/layer17_attention_weight_max": 51.176422119140625,
      "activations/layer17_attention_weight_min": -43.728363037109375,
      "activations/layer18_attention_weight_max": 48.7517204284668,
      "activations/layer18_attention_weight_min": -41.33885955810547,
      "activations/layer19_attention_weight_max": 25.961740493774414,
      "activations/layer19_attention_weight_min": -22.742116928100586,
      "activations/layer1_attention_weight_max": 16.986480712890625,
      "activations/layer1_attention_weight_min": -15.008379936218262,
      "activations/layer20_attention_weight_max": 23.2865047454834,
      "activations/layer20_attention_weight_min": -22.64767837524414,
      "activations/layer21_attention_weight_max": 38.28971862792969,
      "activations/layer21_attention_weight_min": -23.875267028808594,
      "activations/layer22_attention_weight_max": 30.568771362304688,
      "activations/layer22_attention_weight_min": -24.328895568847656,
      "activations/layer23_attention_weight_max": 40.44898223876953,
      "activations/layer23_attention_weight_min": -24.378021240234375,
      "activations/layer2_attention_weight_max": 31.98256492614746,
      "activations/layer2_attention_weight_min": -31.140411376953125,
      "activations/layer3_attention_weight_max": 90.9783706665039,
      "activations/layer3_attention_weight_min": -92.62468719482422,
      "activations/layer4_attention_weight_max": 87.25416564941406,
      "activations/layer4_attention_weight_min": -87.43645477294922,
      "activations/layer5_attention_weight_max": 63.6427116394043,
      "activations/layer5_attention_weight_min": -72.65888977050781,
      "activations/layer6_attention_weight_max": 50.45113754272461,
      "activations/layer6_attention_weight_min": -52.77907180786133,
      "activations/layer7_attention_weight_max": 64.38505554199219,
      "activations/layer7_attention_weight_min": -62.56156539916992,
      "activations/layer8_attention_weight_max": 47.55802536010742,
      "activations/layer8_attention_weight_min": -50.05805969238281,
      "activations/layer9_attention_weight_max": 42.85004806518555,
      "activations/layer9_attention_weight_min": -44.27253341674805,
      "epoch": 17.22,
      "learning_rate": 3.933333333333333e-05,
      "loss": 2.7558,
      "step": 296400
    },
    {
      "activations/layer0_attention_weight_max": 15.884359359741211,
      "activations/layer0_attention_weight_min": -12.885458946228027,
      "activations/layer10_attention_weight_max": 39.142433166503906,
      "activations/layer10_attention_weight_min": -37.78159713745117,
      "activations/layer11_attention_weight_max": 34.52892303466797,
      "activations/layer11_attention_weight_min": -33.86555480957031,
      "activations/layer12_attention_weight_max": 18.14633560180664,
      "activations/layer12_attention_weight_min": -27.386594772338867,
      "activations/layer13_attention_weight_max": 35.682064056396484,
      "activations/layer13_attention_weight_min": -27.67873764038086,
      "activations/layer14_attention_weight_max": 35.35205841064453,
      "activations/layer14_attention_weight_min": -29.76276969909668,
      "activations/layer15_attention_weight_max": 33.20413589477539,
      "activations/layer15_attention_weight_min": -30.827133178710938,
      "activations/layer16_attention_weight_max": 32.45779800415039,
      "activations/layer16_attention_weight_min": -31.776809692382812,
      "activations/layer17_attention_weight_max": 52.2953987121582,
      "activations/layer17_attention_weight_min": -47.669185638427734,
      "activations/layer18_attention_weight_max": 51.51358413696289,
      "activations/layer18_attention_weight_min": -39.67070770263672,
      "activations/layer19_attention_weight_max": 25.36055564880371,
      "activations/layer19_attention_weight_min": -23.573881149291992,
      "activations/layer1_attention_weight_max": 16.7186279296875,
      "activations/layer1_attention_weight_min": -15.724238395690918,
      "activations/layer20_attention_weight_max": 23.52433204650879,
      "activations/layer20_attention_weight_min": -21.347185134887695,
      "activations/layer21_attention_weight_max": 36.84528350830078,
      "activations/layer21_attention_weight_min": -24.184133529663086,
      "activations/layer22_attention_weight_max": 31.45675277709961,
      "activations/layer22_attention_weight_min": -25.71546173095703,
      "activations/layer23_attention_weight_max": 39.81396484375,
      "activations/layer23_attention_weight_min": -24.35999870300293,
      "activations/layer2_attention_weight_max": 33.26784896850586,
      "activations/layer2_attention_weight_min": -31.597448348999023,
      "activations/layer3_attention_weight_max": 99.25690460205078,
      "activations/layer3_attention_weight_min": -99.36357879638672,
      "activations/layer4_attention_weight_max": 93.02790832519531,
      "activations/layer4_attention_weight_min": -88.29912567138672,
      "activations/layer5_attention_weight_max": 66.84332275390625,
      "activations/layer5_attention_weight_min": -69.94296264648438,
      "activations/layer6_attention_weight_max": 51.04743194580078,
      "activations/layer6_attention_weight_min": -51.79912185668945,
      "activations/layer7_attention_weight_max": 63.16709899902344,
      "activations/layer7_attention_weight_min": -62.388423919677734,
      "activations/layer8_attention_weight_max": 46.869895935058594,
      "activations/layer8_attention_weight_min": -50.57591247558594,
      "activations/layer9_attention_weight_max": 45.84627914428711,
      "activations/layer9_attention_weight_min": -46.12437057495117,
      "epoch": 17.23,
      "learning_rate": 3.931439393939394e-05,
      "loss": 2.737,
      "step": 296450
    },
    {
      "activations/layer0_attention_weight_max": 15.375767707824707,
      "activations/layer0_attention_weight_min": -12.665387153625488,
      "activations/layer10_attention_weight_max": 33.96791458129883,
      "activations/layer10_attention_weight_min": -34.152984619140625,
      "activations/layer11_attention_weight_max": 30.11792755126953,
      "activations/layer11_attention_weight_min": -33.51155090332031,
      "activations/layer12_attention_weight_max": 18.991714477539062,
      "activations/layer12_attention_weight_min": -27.038557052612305,
      "activations/layer13_attention_weight_max": 34.09540557861328,
      "activations/layer13_attention_weight_min": -27.761445999145508,
      "activations/layer14_attention_weight_max": 32.53209686279297,
      "activations/layer14_attention_weight_min": -29.87554359436035,
      "activations/layer15_attention_weight_max": 29.92081069946289,
      "activations/layer15_attention_weight_min": -30.004722595214844,
      "activations/layer16_attention_weight_max": 31.054540634155273,
      "activations/layer16_attention_weight_min": -30.459985733032227,
      "activations/layer17_attention_weight_max": 48.89130783081055,
      "activations/layer17_attention_weight_min": -44.02943420410156,
      "activations/layer18_attention_weight_max": 44.9720573425293,
      "activations/layer18_attention_weight_min": -40.217464447021484,
      "activations/layer19_attention_weight_max": 23.548429489135742,
      "activations/layer19_attention_weight_min": -23.76313018798828,
      "activations/layer1_attention_weight_max": 15.753315925598145,
      "activations/layer1_attention_weight_min": -14.303411483764648,
      "activations/layer20_attention_weight_max": 21.97513198852539,
      "activations/layer20_attention_weight_min": -22.67999267578125,
      "activations/layer21_attention_weight_max": 29.064260482788086,
      "activations/layer21_attention_weight_min": -22.206933975219727,
      "activations/layer22_attention_weight_max": 28.020145416259766,
      "activations/layer22_attention_weight_min": -26.15111541748047,
      "activations/layer23_attention_weight_max": 34.325958251953125,
      "activations/layer23_attention_weight_min": -23.92377281188965,
      "activations/layer2_attention_weight_max": 32.78325653076172,
      "activations/layer2_attention_weight_min": -30.232337951660156,
      "activations/layer3_attention_weight_max": 90.00301361083984,
      "activations/layer3_attention_weight_min": -93.23941802978516,
      "activations/layer4_attention_weight_max": 87.26669311523438,
      "activations/layer4_attention_weight_min": -84.82832336425781,
      "activations/layer5_attention_weight_max": 66.39920043945312,
      "activations/layer5_attention_weight_min": -72.49496459960938,
      "activations/layer6_attention_weight_max": 49.1183967590332,
      "activations/layer6_attention_weight_min": -53.781654357910156,
      "activations/layer7_attention_weight_max": 65.57068634033203,
      "activations/layer7_attention_weight_min": -64.5227279663086,
      "activations/layer8_attention_weight_max": 45.69065475463867,
      "activations/layer8_attention_weight_min": -47.62349319458008,
      "activations/layer9_attention_weight_max": 40.841224670410156,
      "activations/layer9_attention_weight_min": -45.6465950012207,
      "epoch": 17.23,
      "learning_rate": 3.929545454545454e-05,
      "loss": 2.7559,
      "step": 296500
    },
    {
      "activations/layer0_attention_weight_max": 16.244993209838867,
      "activations/layer0_attention_weight_min": -12.828948974609375,
      "activations/layer10_attention_weight_max": 41.64541244506836,
      "activations/layer10_attention_weight_min": -40.7640266418457,
      "activations/layer11_attention_weight_max": 40.51026916503906,
      "activations/layer11_attention_weight_min": -39.11744689941406,
      "activations/layer12_attention_weight_max": 20.301971435546875,
      "activations/layer12_attention_weight_min": -29.503786087036133,
      "activations/layer13_attention_weight_max": 45.728885650634766,
      "activations/layer13_attention_weight_min": -33.173500061035156,
      "activations/layer14_attention_weight_max": 61.99943923950195,
      "activations/layer14_attention_weight_min": -35.37409210205078,
      "activations/layer15_attention_weight_max": 53.3475341796875,
      "activations/layer15_attention_weight_min": -37.80451583862305,
      "activations/layer16_attention_weight_max": 35.33500289916992,
      "activations/layer16_attention_weight_min": -32.68345642089844,
      "activations/layer17_attention_weight_max": 53.9046516418457,
      "activations/layer17_attention_weight_min": -47.501827239990234,
      "activations/layer18_attention_weight_max": 48.9647102355957,
      "activations/layer18_attention_weight_min": -44.8587532043457,
      "activations/layer19_attention_weight_max": 27.181211471557617,
      "activations/layer19_attention_weight_min": -24.95803451538086,
      "activations/layer1_attention_weight_max": 16.497068405151367,
      "activations/layer1_attention_weight_min": -15.98840618133545,
      "activations/layer20_attention_weight_max": 29.688209533691406,
      "activations/layer20_attention_weight_min": -24.93841552734375,
      "activations/layer21_attention_weight_max": 48.36575698852539,
      "activations/layer21_attention_weight_min": -22.276596069335938,
      "activations/layer22_attention_weight_max": 31.247528076171875,
      "activations/layer22_attention_weight_min": -25.708499908447266,
      "activations/layer23_attention_weight_max": 36.52143096923828,
      "activations/layer23_attention_weight_min": -24.665199279785156,
      "activations/layer2_attention_weight_max": 35.861778259277344,
      "activations/layer2_attention_weight_min": -32.21403503417969,
      "activations/layer3_attention_weight_max": 100.57727813720703,
      "activations/layer3_attention_weight_min": -95.54871368408203,
      "activations/layer4_attention_weight_max": 94.41077423095703,
      "activations/layer4_attention_weight_min": -92.35413360595703,
      "activations/layer5_attention_weight_max": 63.54226303100586,
      "activations/layer5_attention_weight_min": -77.87277221679688,
      "activations/layer6_attention_weight_max": 51.39436340332031,
      "activations/layer6_attention_weight_min": -52.74027633666992,
      "activations/layer7_attention_weight_max": 70.43917083740234,
      "activations/layer7_attention_weight_min": -68.26100158691406,
      "activations/layer8_attention_weight_max": 49.551841735839844,
      "activations/layer8_attention_weight_min": -48.88887405395508,
      "activations/layer9_attention_weight_max": 56.780921936035156,
      "activations/layer9_attention_weight_min": -46.66755294799805,
      "epoch": 17.23,
      "learning_rate": 3.927651515151514e-05,
      "loss": 2.7262,
      "step": 296550
    },
    {
      "activations/layer0_attention_weight_max": 16.101903915405273,
      "activations/layer0_attention_weight_min": -12.760135650634766,
      "activations/layer10_attention_weight_max": 38.780677795410156,
      "activations/layer10_attention_weight_min": -38.27436065673828,
      "activations/layer11_attention_weight_max": 35.071414947509766,
      "activations/layer11_attention_weight_min": -34.279632568359375,
      "activations/layer12_attention_weight_max": 19.73605728149414,
      "activations/layer12_attention_weight_min": -28.04427719116211,
      "activations/layer13_attention_weight_max": 33.37459945678711,
      "activations/layer13_attention_weight_min": -26.97590446472168,
      "activations/layer14_attention_weight_max": 42.85398864746094,
      "activations/layer14_attention_weight_min": -28.91779327392578,
      "activations/layer15_attention_weight_max": 37.433231353759766,
      "activations/layer15_attention_weight_min": -30.378877639770508,
      "activations/layer16_attention_weight_max": 39.486358642578125,
      "activations/layer16_attention_weight_min": -34.87772750854492,
      "activations/layer17_attention_weight_max": 62.0748291015625,
      "activations/layer17_attention_weight_min": -47.84657669067383,
      "activations/layer18_attention_weight_max": 53.662078857421875,
      "activations/layer18_attention_weight_min": -40.84333038330078,
      "activations/layer19_attention_weight_max": 25.598779678344727,
      "activations/layer19_attention_weight_min": -22.581199645996094,
      "activations/layer1_attention_weight_max": 15.916508674621582,
      "activations/layer1_attention_weight_min": -14.16512393951416,
      "activations/layer20_attention_weight_max": 22.94056510925293,
      "activations/layer20_attention_weight_min": -20.673954010009766,
      "activations/layer21_attention_weight_max": 42.212120056152344,
      "activations/layer21_attention_weight_min": -25.77901268005371,
      "activations/layer22_attention_weight_max": 29.60715675354004,
      "activations/layer22_attention_weight_min": -26.214141845703125,
      "activations/layer23_attention_weight_max": 36.392486572265625,
      "activations/layer23_attention_weight_min": -23.135971069335938,
      "activations/layer2_attention_weight_max": 33.063270568847656,
      "activations/layer2_attention_weight_min": -31.531375885009766,
      "activations/layer3_attention_weight_max": 91.09882354736328,
      "activations/layer3_attention_weight_min": -92.71372985839844,
      "activations/layer4_attention_weight_max": 89.53740692138672,
      "activations/layer4_attention_weight_min": -90.22093963623047,
      "activations/layer5_attention_weight_max": 63.30751037597656,
      "activations/layer5_attention_weight_min": -72.04130554199219,
      "activations/layer6_attention_weight_max": 51.72957992553711,
      "activations/layer6_attention_weight_min": -54.41340255737305,
      "activations/layer7_attention_weight_max": 66.78367614746094,
      "activations/layer7_attention_weight_min": -67.19036102294922,
      "activations/layer8_attention_weight_max": 45.302059173583984,
      "activations/layer8_attention_weight_min": -50.43659591674805,
      "activations/layer9_attention_weight_max": 43.69553756713867,
      "activations/layer9_attention_weight_min": -47.52436447143555,
      "epoch": 17.23,
      "learning_rate": 3.925757575757576e-05,
      "loss": 2.7372,
      "step": 296600
    },
    {
      "activations/layer0_attention_weight_max": 15.954671859741211,
      "activations/layer0_attention_weight_min": -13.129387855529785,
      "activations/layer10_attention_weight_max": 41.50992202758789,
      "activations/layer10_attention_weight_min": -38.888465881347656,
      "activations/layer11_attention_weight_max": 35.805294036865234,
      "activations/layer11_attention_weight_min": -37.4263916015625,
      "activations/layer12_attention_weight_max": 19.477954864501953,
      "activations/layer12_attention_weight_min": -24.634693145751953,
      "activations/layer13_attention_weight_max": 37.4666862487793,
      "activations/layer13_attention_weight_min": -29.81498146057129,
      "activations/layer14_attention_weight_max": 37.70149230957031,
      "activations/layer14_attention_weight_min": -30.611351013183594,
      "activations/layer15_attention_weight_max": 36.62142562866211,
      "activations/layer15_attention_weight_min": -31.455175399780273,
      "activations/layer16_attention_weight_max": 36.01136016845703,
      "activations/layer16_attention_weight_min": -33.746768951416016,
      "activations/layer17_attention_weight_max": 60.89488983154297,
      "activations/layer17_attention_weight_min": -50.61935043334961,
      "activations/layer18_attention_weight_max": 53.02049255371094,
      "activations/layer18_attention_weight_min": -42.60105895996094,
      "activations/layer19_attention_weight_max": 28.412961959838867,
      "activations/layer19_attention_weight_min": -24.40782356262207,
      "activations/layer1_attention_weight_max": 15.979303359985352,
      "activations/layer1_attention_weight_min": -14.844598770141602,
      "activations/layer20_attention_weight_max": 25.013334274291992,
      "activations/layer20_attention_weight_min": -21.98930549621582,
      "activations/layer21_attention_weight_max": 38.69063949584961,
      "activations/layer21_attention_weight_min": -23.42386817932129,
      "activations/layer22_attention_weight_max": 32.94877243041992,
      "activations/layer22_attention_weight_min": -26.177785873413086,
      "activations/layer23_attention_weight_max": 41.25408172607422,
      "activations/layer23_attention_weight_min": -22.945316314697266,
      "activations/layer2_attention_weight_max": 32.95310974121094,
      "activations/layer2_attention_weight_min": -31.121299743652344,
      "activations/layer3_attention_weight_max": 94.98249053955078,
      "activations/layer3_attention_weight_min": -94.87482452392578,
      "activations/layer4_attention_weight_max": 92.40621185302734,
      "activations/layer4_attention_weight_min": -90.05696868896484,
      "activations/layer5_attention_weight_max": 66.99400329589844,
      "activations/layer5_attention_weight_min": -77.26049041748047,
      "activations/layer6_attention_weight_max": 53.98894500732422,
      "activations/layer6_attention_weight_min": -54.273216247558594,
      "activations/layer7_attention_weight_max": 70.28015899658203,
      "activations/layer7_attention_weight_min": -69.1154556274414,
      "activations/layer8_attention_weight_max": 50.43376159667969,
      "activations/layer8_attention_weight_min": -54.90338897705078,
      "activations/layer9_attention_weight_max": 50.92388153076172,
      "activations/layer9_attention_weight_min": -47.406368255615234,
      "epoch": 17.24,
      "learning_rate": 3.923863636363636e-05,
      "loss": 2.743,
      "step": 296650
    },
    {
      "activations/layer0_attention_weight_max": 15.17236328125,
      "activations/layer0_attention_weight_min": -12.785994529724121,
      "activations/layer10_attention_weight_max": 35.04803466796875,
      "activations/layer10_attention_weight_min": -34.223209381103516,
      "activations/layer11_attention_weight_max": 32.280208587646484,
      "activations/layer11_attention_weight_min": -33.86500549316406,
      "activations/layer12_attention_weight_max": 18.226062774658203,
      "activations/layer12_attention_weight_min": -25.240985870361328,
      "activations/layer13_attention_weight_max": 30.89288902282715,
      "activations/layer13_attention_weight_min": -28.689302444458008,
      "activations/layer14_attention_weight_max": 33.74827194213867,
      "activations/layer14_attention_weight_min": -29.926620483398438,
      "activations/layer15_attention_weight_max": 31.933513641357422,
      "activations/layer15_attention_weight_min": -29.84711456298828,
      "activations/layer16_attention_weight_max": 32.618064880371094,
      "activations/layer16_attention_weight_min": -31.378450393676758,
      "activations/layer17_attention_weight_max": 48.398765563964844,
      "activations/layer17_attention_weight_min": -43.01942443847656,
      "activations/layer18_attention_weight_max": 44.55843734741211,
      "activations/layer18_attention_weight_min": -38.799312591552734,
      "activations/layer19_attention_weight_max": 21.785825729370117,
      "activations/layer19_attention_weight_min": -23.202327728271484,
      "activations/layer1_attention_weight_max": 15.467694282531738,
      "activations/layer1_attention_weight_min": -15.206931114196777,
      "activations/layer20_attention_weight_max": 23.60865592956543,
      "activations/layer20_attention_weight_min": -21.672466278076172,
      "activations/layer21_attention_weight_max": 32.9360237121582,
      "activations/layer21_attention_weight_min": -22.398218154907227,
      "activations/layer22_attention_weight_max": 27.636316299438477,
      "activations/layer22_attention_weight_min": -25.295122146606445,
      "activations/layer23_attention_weight_max": 35.99335479736328,
      "activations/layer23_attention_weight_min": -22.505542755126953,
      "activations/layer2_attention_weight_max": 31.700035095214844,
      "activations/layer2_attention_weight_min": -30.755199432373047,
      "activations/layer3_attention_weight_max": 90.53770446777344,
      "activations/layer3_attention_weight_min": -93.598388671875,
      "activations/layer4_attention_weight_max": 90.95684051513672,
      "activations/layer4_attention_weight_min": -87.96521759033203,
      "activations/layer5_attention_weight_max": 63.846187591552734,
      "activations/layer5_attention_weight_min": -74.77234649658203,
      "activations/layer6_attention_weight_max": 49.629295349121094,
      "activations/layer6_attention_weight_min": -49.754798889160156,
      "activations/layer7_attention_weight_max": 65.410400390625,
      "activations/layer7_attention_weight_min": -66.17511749267578,
      "activations/layer8_attention_weight_max": 44.4504280090332,
      "activations/layer8_attention_weight_min": -49.35368347167969,
      "activations/layer9_attention_weight_max": 41.877479553222656,
      "activations/layer9_attention_weight_min": -45.13066101074219,
      "epoch": 17.24,
      "learning_rate": 3.921969696969697e-05,
      "loss": 2.7373,
      "step": 296700
    },
    {
      "activations/layer0_attention_weight_max": 16.007278442382812,
      "activations/layer0_attention_weight_min": -12.638114929199219,
      "activations/layer10_attention_weight_max": 38.813690185546875,
      "activations/layer10_attention_weight_min": -35.97178649902344,
      "activations/layer11_attention_weight_max": 33.35770034790039,
      "activations/layer11_attention_weight_min": -34.02932357788086,
      "activations/layer12_attention_weight_max": 19.196025848388672,
      "activations/layer12_attention_weight_min": -23.844772338867188,
      "activations/layer13_attention_weight_max": 33.98326873779297,
      "activations/layer13_attention_weight_min": -29.180706024169922,
      "activations/layer14_attention_weight_max": 34.72861099243164,
      "activations/layer14_attention_weight_min": -32.2388916015625,
      "activations/layer15_attention_weight_max": 33.720035552978516,
      "activations/layer15_attention_weight_min": -28.66718864440918,
      "activations/layer16_attention_weight_max": 34.44408416748047,
      "activations/layer16_attention_weight_min": -32.38414001464844,
      "activations/layer17_attention_weight_max": 55.789039611816406,
      "activations/layer17_attention_weight_min": -50.16215896606445,
      "activations/layer18_attention_weight_max": 51.346920013427734,
      "activations/layer18_attention_weight_min": -44.20234298706055,
      "activations/layer19_attention_weight_max": 24.691221237182617,
      "activations/layer19_attention_weight_min": -22.72996711730957,
      "activations/layer1_attention_weight_max": 16.396303176879883,
      "activations/layer1_attention_weight_min": -14.80480670928955,
      "activations/layer20_attention_weight_max": 25.78571128845215,
      "activations/layer20_attention_weight_min": -21.765918731689453,
      "activations/layer21_attention_weight_max": 37.08842849731445,
      "activations/layer21_attention_weight_min": -21.58266830444336,
      "activations/layer22_attention_weight_max": 32.57339096069336,
      "activations/layer22_attention_weight_min": -27.110687255859375,
      "activations/layer23_attention_weight_max": 39.525360107421875,
      "activations/layer23_attention_weight_min": -26.745765686035156,
      "activations/layer2_attention_weight_max": 31.401874542236328,
      "activations/layer2_attention_weight_min": -32.01752853393555,
      "activations/layer3_attention_weight_max": 91.86174011230469,
      "activations/layer3_attention_weight_min": -95.32131958007812,
      "activations/layer4_attention_weight_max": 85.54263305664062,
      "activations/layer4_attention_weight_min": -88.42806243896484,
      "activations/layer5_attention_weight_max": 61.85340118408203,
      "activations/layer5_attention_weight_min": -74.39786529541016,
      "activations/layer6_attention_weight_max": 48.465389251708984,
      "activations/layer6_attention_weight_min": -51.29933166503906,
      "activations/layer7_attention_weight_max": 62.274131774902344,
      "activations/layer7_attention_weight_min": -62.3265266418457,
      "activations/layer8_attention_weight_max": 44.67803955078125,
      "activations/layer8_attention_weight_min": -47.166053771972656,
      "activations/layer9_attention_weight_max": 43.272342681884766,
      "activations/layer9_attention_weight_min": -44.0002555847168,
      "epoch": 17.24,
      "learning_rate": 3.920075757575757e-05,
      "loss": 2.7481,
      "step": 296750
    },
    {
      "activations/layer0_attention_weight_max": 14.976311683654785,
      "activations/layer0_attention_weight_min": -12.67182731628418,
      "activations/layer10_attention_weight_max": 33.573001861572266,
      "activations/layer10_attention_weight_min": -32.957210540771484,
      "activations/layer11_attention_weight_max": 31.951940536499023,
      "activations/layer11_attention_weight_min": -33.01747131347656,
      "activations/layer12_attention_weight_max": 17.65132713317871,
      "activations/layer12_attention_weight_min": -29.116724014282227,
      "activations/layer13_attention_weight_max": 41.7281608581543,
      "activations/layer13_attention_weight_min": -30.834115982055664,
      "activations/layer14_attention_weight_max": 36.99423599243164,
      "activations/layer14_attention_weight_min": -31.23764991760254,
      "activations/layer15_attention_weight_max": 33.628971099853516,
      "activations/layer15_attention_weight_min": -32.14891815185547,
      "activations/layer16_attention_weight_max": 33.00973129272461,
      "activations/layer16_attention_weight_min": -31.345918655395508,
      "activations/layer17_attention_weight_max": 50.25096893310547,
      "activations/layer17_attention_weight_min": -42.6295280456543,
      "activations/layer18_attention_weight_max": 45.55322265625,
      "activations/layer18_attention_weight_min": -37.976016998291016,
      "activations/layer19_attention_weight_max": 27.870370864868164,
      "activations/layer19_attention_weight_min": -20.39008140563965,
      "activations/layer1_attention_weight_max": 16.213388442993164,
      "activations/layer1_attention_weight_min": -14.207254409790039,
      "activations/layer20_attention_weight_max": 22.642175674438477,
      "activations/layer20_attention_weight_min": -20.168546676635742,
      "activations/layer21_attention_weight_max": 35.071510314941406,
      "activations/layer21_attention_weight_min": -19.787466049194336,
      "activations/layer22_attention_weight_max": 29.574474334716797,
      "activations/layer22_attention_weight_min": -23.32064437866211,
      "activations/layer23_attention_weight_max": 36.432926177978516,
      "activations/layer23_attention_weight_min": -20.686826705932617,
      "activations/layer2_attention_weight_max": 33.193504333496094,
      "activations/layer2_attention_weight_min": -30.706497192382812,
      "activations/layer3_attention_weight_max": 89.81324768066406,
      "activations/layer3_attention_weight_min": -90.31153106689453,
      "activations/layer4_attention_weight_max": 82.65769958496094,
      "activations/layer4_attention_weight_min": -83.70417785644531,
      "activations/layer5_attention_weight_max": 58.037017822265625,
      "activations/layer5_attention_weight_min": -72.55682373046875,
      "activations/layer6_attention_weight_max": 46.065956115722656,
      "activations/layer6_attention_weight_min": -45.88975524902344,
      "activations/layer7_attention_weight_max": 67.52702331542969,
      "activations/layer7_attention_weight_min": -62.3364372253418,
      "activations/layer8_attention_weight_max": 42.316707611083984,
      "activations/layer8_attention_weight_min": -42.731109619140625,
      "activations/layer9_attention_weight_max": 51.16556930541992,
      "activations/layer9_attention_weight_min": -41.387691497802734,
      "epoch": 17.25,
      "learning_rate": 3.918181818181818e-05,
      "loss": 2.748,
      "step": 296800
    },
    {
      "activations/layer0_attention_weight_max": 15.649637222290039,
      "activations/layer0_attention_weight_min": -13.132604598999023,
      "activations/layer10_attention_weight_max": 33.92738342285156,
      "activations/layer10_attention_weight_min": -34.891475677490234,
      "activations/layer11_attention_weight_max": 33.29008865356445,
      "activations/layer11_attention_weight_min": -34.86299133300781,
      "activations/layer12_attention_weight_max": 18.875028610229492,
      "activations/layer12_attention_weight_min": -24.594470977783203,
      "activations/layer13_attention_weight_max": 35.041648864746094,
      "activations/layer13_attention_weight_min": -27.972454071044922,
      "activations/layer14_attention_weight_max": 33.03106689453125,
      "activations/layer14_attention_weight_min": -31.28546142578125,
      "activations/layer15_attention_weight_max": 30.392642974853516,
      "activations/layer15_attention_weight_min": -30.0958251953125,
      "activations/layer16_attention_weight_max": 31.44948387145996,
      "activations/layer16_attention_weight_min": -31.050493240356445,
      "activations/layer17_attention_weight_max": 50.29153060913086,
      "activations/layer17_attention_weight_min": -45.497188568115234,
      "activations/layer18_attention_weight_max": 46.23863983154297,
      "activations/layer18_attention_weight_min": -38.71229553222656,
      "activations/layer19_attention_weight_max": 24.446821212768555,
      "activations/layer19_attention_weight_min": -21.96431541442871,
      "activations/layer1_attention_weight_max": 15.549155235290527,
      "activations/layer1_attention_weight_min": -14.035621643066406,
      "activations/layer20_attention_weight_max": 23.76081657409668,
      "activations/layer20_attention_weight_min": -22.5282039642334,
      "activations/layer21_attention_weight_max": 34.08656692504883,
      "activations/layer21_attention_weight_min": -23.23118019104004,
      "activations/layer22_attention_weight_max": 34.82139205932617,
      "activations/layer22_attention_weight_min": -26.827058792114258,
      "activations/layer23_attention_weight_max": 37.93885040283203,
      "activations/layer23_attention_weight_min": -27.024005889892578,
      "activations/layer2_attention_weight_max": 32.017879486083984,
      "activations/layer2_attention_weight_min": -31.298139572143555,
      "activations/layer3_attention_weight_max": 90.17608642578125,
      "activations/layer3_attention_weight_min": -92.10142517089844,
      "activations/layer4_attention_weight_max": 91.1434326171875,
      "activations/layer4_attention_weight_min": -87.1221694946289,
      "activations/layer5_attention_weight_max": 65.52787780761719,
      "activations/layer5_attention_weight_min": -73.09844970703125,
      "activations/layer6_attention_weight_max": 48.55949783325195,
      "activations/layer6_attention_weight_min": -52.67013931274414,
      "activations/layer7_attention_weight_max": 65.13301086425781,
      "activations/layer7_attention_weight_min": -64.05377960205078,
      "activations/layer8_attention_weight_max": 46.62721633911133,
      "activations/layer8_attention_weight_min": -48.19447326660156,
      "activations/layer9_attention_weight_max": 43.078086853027344,
      "activations/layer9_attention_weight_min": -45.024410247802734,
      "epoch": 17.25,
      "learning_rate": 3.9162878787878786e-05,
      "loss": 2.7341,
      "step": 296850
    },
    {
      "activations/layer0_attention_weight_max": 15.693652153015137,
      "activations/layer0_attention_weight_min": -13.001188278198242,
      "activations/layer10_attention_weight_max": 35.57121276855469,
      "activations/layer10_attention_weight_min": -34.62727355957031,
      "activations/layer11_attention_weight_max": 33.15106964111328,
      "activations/layer11_attention_weight_min": -34.208335876464844,
      "activations/layer12_attention_weight_max": 19.55853271484375,
      "activations/layer12_attention_weight_min": -26.52557945251465,
      "activations/layer13_attention_weight_max": 36.58142852783203,
      "activations/layer13_attention_weight_min": -28.271568298339844,
      "activations/layer14_attention_weight_max": 34.1989631652832,
      "activations/layer14_attention_weight_min": -30.987281799316406,
      "activations/layer15_attention_weight_max": 34.590030670166016,
      "activations/layer15_attention_weight_min": -30.752988815307617,
      "activations/layer16_attention_weight_max": 32.93788146972656,
      "activations/layer16_attention_weight_min": -32.92384338378906,
      "activations/layer17_attention_weight_max": 52.014522552490234,
      "activations/layer17_attention_weight_min": -45.64916229248047,
      "activations/layer18_attention_weight_max": 47.52341079711914,
      "activations/layer18_attention_weight_min": -39.38636016845703,
      "activations/layer19_attention_weight_max": 27.410736083984375,
      "activations/layer19_attention_weight_min": -21.613649368286133,
      "activations/layer1_attention_weight_max": 16.13303565979004,
      "activations/layer1_attention_weight_min": -15.44620132446289,
      "activations/layer20_attention_weight_max": 22.28154182434082,
      "activations/layer20_attention_weight_min": -22.064085006713867,
      "activations/layer21_attention_weight_max": 35.478431701660156,
      "activations/layer21_attention_weight_min": -21.318153381347656,
      "activations/layer22_attention_weight_max": 29.01385498046875,
      "activations/layer22_attention_weight_min": -26.1738338470459,
      "activations/layer23_attention_weight_max": 37.307430267333984,
      "activations/layer23_attention_weight_min": -25.083438873291016,
      "activations/layer2_attention_weight_max": 35.487579345703125,
      "activations/layer2_attention_weight_min": -31.079605102539062,
      "activations/layer3_attention_weight_max": 93.36162567138672,
      "activations/layer3_attention_weight_min": -95.75084686279297,
      "activations/layer4_attention_weight_max": 88.99542999267578,
      "activations/layer4_attention_weight_min": -88.93941497802734,
      "activations/layer5_attention_weight_max": 65.61214447021484,
      "activations/layer5_attention_weight_min": -74.54890441894531,
      "activations/layer6_attention_weight_max": 50.340599060058594,
      "activations/layer6_attention_weight_min": -52.21601486206055,
      "activations/layer7_attention_weight_max": 68.06256103515625,
      "activations/layer7_attention_weight_min": -65.82331848144531,
      "activations/layer8_attention_weight_max": 45.91044616699219,
      "activations/layer8_attention_weight_min": -48.79256057739258,
      "activations/layer9_attention_weight_max": 46.16007614135742,
      "activations/layer9_attention_weight_min": -43.48378372192383,
      "epoch": 17.25,
      "learning_rate": 3.914393939393939e-05,
      "loss": 2.754,
      "step": 296900
    },
    {
      "activations/layer0_attention_weight_max": 15.32603931427002,
      "activations/layer0_attention_weight_min": -13.192410469055176,
      "activations/layer10_attention_weight_max": 37.82692337036133,
      "activations/layer10_attention_weight_min": -36.418704986572266,
      "activations/layer11_attention_weight_max": 32.67973709106445,
      "activations/layer11_attention_weight_min": -34.88871765136719,
      "activations/layer12_attention_weight_max": 18.97184181213379,
      "activations/layer12_attention_weight_min": -25.663387298583984,
      "activations/layer13_attention_weight_max": 35.7015266418457,
      "activations/layer13_attention_weight_min": -30.124317169189453,
      "activations/layer14_attention_weight_max": 39.180355072021484,
      "activations/layer14_attention_weight_min": -30.304243087768555,
      "activations/layer15_attention_weight_max": 35.312313079833984,
      "activations/layer15_attention_weight_min": -33.13254165649414,
      "activations/layer16_attention_weight_max": 37.298736572265625,
      "activations/layer16_attention_weight_min": -33.10528564453125,
      "activations/layer17_attention_weight_max": 57.22031784057617,
      "activations/layer17_attention_weight_min": -47.432373046875,
      "activations/layer18_attention_weight_max": 52.56547164916992,
      "activations/layer18_attention_weight_min": -42.971927642822266,
      "activations/layer19_attention_weight_max": 25.43929672241211,
      "activations/layer19_attention_weight_min": -24.985990524291992,
      "activations/layer1_attention_weight_max": 16.770679473876953,
      "activations/layer1_attention_weight_min": -14.28509521484375,
      "activations/layer20_attention_weight_max": 25.01091194152832,
      "activations/layer20_attention_weight_min": -22.426395416259766,
      "activations/layer21_attention_weight_max": 46.088111877441406,
      "activations/layer21_attention_weight_min": -23.522319793701172,
      "activations/layer22_attention_weight_max": 32.196563720703125,
      "activations/layer22_attention_weight_min": -26.149513244628906,
      "activations/layer23_attention_weight_max": 43.11115646362305,
      "activations/layer23_attention_weight_min": -23.879302978515625,
      "activations/layer2_attention_weight_max": 32.563026428222656,
      "activations/layer2_attention_weight_min": -31.562328338623047,
      "activations/layer3_attention_weight_max": 91.67266082763672,
      "activations/layer3_attention_weight_min": -96.76744079589844,
      "activations/layer4_attention_weight_max": 90.09762573242188,
      "activations/layer4_attention_weight_min": -86.97786712646484,
      "activations/layer5_attention_weight_max": 65.29783630371094,
      "activations/layer5_attention_weight_min": -73.04835510253906,
      "activations/layer6_attention_weight_max": 51.12172317504883,
      "activations/layer6_attention_weight_min": -50.70998764038086,
      "activations/layer7_attention_weight_max": 66.70939636230469,
      "activations/layer7_attention_weight_min": -64.99309539794922,
      "activations/layer8_attention_weight_max": 46.443763732910156,
      "activations/layer8_attention_weight_min": -47.66823959350586,
      "activations/layer9_attention_weight_max": 45.94318771362305,
      "activations/layer9_attention_weight_min": -44.311588287353516,
      "epoch": 17.25,
      "learning_rate": 3.9124999999999996e-05,
      "loss": 2.7509,
      "step": 296950
    },
    {
      "activations/layer0_attention_weight_max": 15.53151798248291,
      "activations/layer0_attention_weight_min": -12.58202075958252,
      "activations/layer10_attention_weight_max": 34.19605255126953,
      "activations/layer10_attention_weight_min": -33.398006439208984,
      "activations/layer11_attention_weight_max": 33.0001106262207,
      "activations/layer11_attention_weight_min": -33.399051666259766,
      "activations/layer12_attention_weight_max": 18.083948135375977,
      "activations/layer12_attention_weight_min": -26.510164260864258,
      "activations/layer13_attention_weight_max": 33.73035430908203,
      "activations/layer13_attention_weight_min": -26.94669532775879,
      "activations/layer14_attention_weight_max": 32.49964141845703,
      "activations/layer14_attention_weight_min": -29.494359970092773,
      "activations/layer15_attention_weight_max": 29.184354782104492,
      "activations/layer15_attention_weight_min": -29.520212173461914,
      "activations/layer16_attention_weight_max": 31.56041145324707,
      "activations/layer16_attention_weight_min": -29.207799911499023,
      "activations/layer17_attention_weight_max": 49.7275390625,
      "activations/layer17_attention_weight_min": -42.6506233215332,
      "activations/layer18_attention_weight_max": 46.42137145996094,
      "activations/layer18_attention_weight_min": -37.04027557373047,
      "activations/layer19_attention_weight_max": 24.575822830200195,
      "activations/layer19_attention_weight_min": -22.46605682373047,
      "activations/layer1_attention_weight_max": 15.881704330444336,
      "activations/layer1_attention_weight_min": -15.171462059020996,
      "activations/layer20_attention_weight_max": 22.98283576965332,
      "activations/layer20_attention_weight_min": -21.499469757080078,
      "activations/layer21_attention_weight_max": 37.13325500488281,
      "activations/layer21_attention_weight_min": -22.278247833251953,
      "activations/layer22_attention_weight_max": 25.786659240722656,
      "activations/layer22_attention_weight_min": -24.38087272644043,
      "activations/layer23_attention_weight_max": 35.74927520751953,
      "activations/layer23_attention_weight_min": -25.324872970581055,
      "activations/layer2_attention_weight_max": 37.06428909301758,
      "activations/layer2_attention_weight_min": -32.37628173828125,
      "activations/layer3_attention_weight_max": 88.94258117675781,
      "activations/layer3_attention_weight_min": -87.94577026367188,
      "activations/layer4_attention_weight_max": 85.89696502685547,
      "activations/layer4_attention_weight_min": -83.38493347167969,
      "activations/layer5_attention_weight_max": 61.441864013671875,
      "activations/layer5_attention_weight_min": -68.61653137207031,
      "activations/layer6_attention_weight_max": 46.28818130493164,
      "activations/layer6_attention_weight_min": -48.0290412902832,
      "activations/layer7_attention_weight_max": 60.55006408691406,
      "activations/layer7_attention_weight_min": -67.5992660522461,
      "activations/layer8_attention_weight_max": 46.30525588989258,
      "activations/layer8_attention_weight_min": -45.40361785888672,
      "activations/layer9_attention_weight_max": 43.3379020690918,
      "activations/layer9_attention_weight_min": -46.77207565307617,
      "epoch": 17.26,
      "learning_rate": 3.9106060606060604e-05,
      "loss": 2.7452,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4517,
      "eval_samples_per_second": 508.066,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4517,
      "eval_openwebtext_samples_per_second": 508.066,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_wikitext_loss": 2.931640625,
      "eval_wikitext_ppl": 18.758380733362646,
      "eval_wikitext_runtime": 1.9303,
      "eval_wikitext_samples_per_second": 236.227,
      "step": 297000
    },
    {
      "epoch": 17.26,
      "eval_lambada_loss": 2.462890625,
      "eval_lambada_ppl": 11.738694709013188,
      "eval_lambada_runtime": 9.5393,
      "eval_lambada_samples_per_second": 510.416,
      "step": 297000
    },
    {
      "activations/layer0_attention_weight_max": 16.39187240600586,
      "activations/layer0_attention_weight_min": -13.419995307922363,
      "activations/layer10_attention_weight_max": 32.58693313598633,
      "activations/layer10_attention_weight_min": -34.366519927978516,
      "activations/layer11_attention_weight_max": 29.38927459716797,
      "activations/layer11_attention_weight_min": -32.11355209350586,
      "activations/layer12_attention_weight_max": 19.118162155151367,
      "activations/layer12_attention_weight_min": -25.070316314697266,
      "activations/layer13_attention_weight_max": 32.406768798828125,
      "activations/layer13_attention_weight_min": -25.216508865356445,
      "activations/layer14_attention_weight_max": 34.21135330200195,
      "activations/layer14_attention_weight_min": -32.323944091796875,
      "activations/layer15_attention_weight_max": 30.76194190979004,
      "activations/layer15_attention_weight_min": -29.17765998840332,
      "activations/layer16_attention_weight_max": 32.545013427734375,
      "activations/layer16_attention_weight_min": -30.881376266479492,
      "activations/layer17_attention_weight_max": 50.02983474731445,
      "activations/layer17_attention_weight_min": -41.92284393310547,
      "activations/layer18_attention_weight_max": 44.845157623291016,
      "activations/layer18_attention_weight_min": -39.17496109008789,
      "activations/layer19_attention_weight_max": 22.362707138061523,
      "activations/layer19_attention_weight_min": -21.6457576751709,
      "activations/layer1_attention_weight_max": 15.707784652709961,
      "activations/layer1_attention_weight_min": -14.069378852844238,
      "activations/layer20_attention_weight_max": 21.336565017700195,
      "activations/layer20_attention_weight_min": -23.777299880981445,
      "activations/layer21_attention_weight_max": 32.26310729980469,
      "activations/layer21_attention_weight_min": -24.817916870117188,
      "activations/layer22_attention_weight_max": 28.875701904296875,
      "activations/layer22_attention_weight_min": -28.31787109375,
      "activations/layer23_attention_weight_max": 34.04524612426758,
      "activations/layer23_attention_weight_min": -25.115825653076172,
      "activations/layer2_attention_weight_max": 32.27510070800781,
      "activations/layer2_attention_weight_min": -30.15899658203125,
      "activations/layer3_attention_weight_max": 88.72786712646484,
      "activations/layer3_attention_weight_min": -90.2596664428711,
      "activations/layer4_attention_weight_max": 83.03828430175781,
      "activations/layer4_attention_weight_min": -84.42822265625,
      "activations/layer5_attention_weight_max": 64.69481658935547,
      "activations/layer5_attention_weight_min": -70.40657806396484,
      "activations/layer6_attention_weight_max": 49.33897399902344,
      "activations/layer6_attention_weight_min": -49.39152145385742,
      "activations/layer7_attention_weight_max": 60.55012130737305,
      "activations/layer7_attention_weight_min": -61.83525085449219,
      "activations/layer8_attention_weight_max": 43.48999786376953,
      "activations/layer8_attention_weight_min": -46.12180709838867,
      "activations/layer9_attention_weight_max": 42.656402587890625,
      "activations/layer9_attention_weight_min": -43.04534149169922,
      "epoch": 17.26,
      "learning_rate": 3.9087121212121205e-05,
      "loss": 2.7309,
      "step": 297050
    },
    {
      "activations/layer0_attention_weight_max": 16.23990821838379,
      "activations/layer0_attention_weight_min": -12.689972877502441,
      "activations/layer10_attention_weight_max": 34.7538948059082,
      "activations/layer10_attention_weight_min": -37.16067123413086,
      "activations/layer11_attention_weight_max": 32.39621353149414,
      "activations/layer11_attention_weight_min": -36.6502799987793,
      "activations/layer12_attention_weight_max": 18.8858585357666,
      "activations/layer12_attention_weight_min": -25.626649856567383,
      "activations/layer13_attention_weight_max": 30.9352970123291,
      "activations/layer13_attention_weight_min": -27.45628547668457,
      "activations/layer14_attention_weight_max": 34.010738372802734,
      "activations/layer14_attention_weight_min": -29.841808319091797,
      "activations/layer15_attention_weight_max": 32.78382873535156,
      "activations/layer15_attention_weight_min": -28.987363815307617,
      "activations/layer16_attention_weight_max": 32.93858337402344,
      "activations/layer16_attention_weight_min": -31.429264068603516,
      "activations/layer17_attention_weight_max": 48.63423538208008,
      "activations/layer17_attention_weight_min": -44.38971710205078,
      "activations/layer18_attention_weight_max": 50.455047607421875,
      "activations/layer18_attention_weight_min": -38.1859245300293,
      "activations/layer19_attention_weight_max": 26.334264755249023,
      "activations/layer19_attention_weight_min": -26.0833797454834,
      "activations/layer1_attention_weight_max": 16.193233489990234,
      "activations/layer1_attention_weight_min": -13.767110824584961,
      "activations/layer20_attention_weight_max": 20.6232967376709,
      "activations/layer20_attention_weight_min": -25.804405212402344,
      "activations/layer21_attention_weight_max": 30.586868286132812,
      "activations/layer21_attention_weight_min": -20.420528411865234,
      "activations/layer22_attention_weight_max": 29.57779312133789,
      "activations/layer22_attention_weight_min": -24.751550674438477,
      "activations/layer23_attention_weight_max": 32.41530990600586,
      "activations/layer23_attention_weight_min": -25.176151275634766,
      "activations/layer2_attention_weight_max": 32.21161651611328,
      "activations/layer2_attention_weight_min": -29.912025451660156,
      "activations/layer3_attention_weight_max": 92.9703369140625,
      "activations/layer3_attention_weight_min": -94.84535217285156,
      "activations/layer4_attention_weight_max": 89.84744262695312,
      "activations/layer4_attention_weight_min": -84.40801239013672,
      "activations/layer5_attention_weight_max": 63.08241271972656,
      "activations/layer5_attention_weight_min": -71.32408905029297,
      "activations/layer6_attention_weight_max": 49.37342071533203,
      "activations/layer6_attention_weight_min": -49.48188018798828,
      "activations/layer7_attention_weight_max": 64.27002716064453,
      "activations/layer7_attention_weight_min": -62.440303802490234,
      "activations/layer8_attention_weight_max": 44.886775970458984,
      "activations/layer8_attention_weight_min": -49.93709182739258,
      "activations/layer9_attention_weight_max": 51.68655014038086,
      "activations/layer9_attention_weight_min": -44.20493698120117,
      "epoch": 17.26,
      "learning_rate": 3.9068181818181814e-05,
      "loss": 2.7379,
      "step": 297100
    },
    {
      "activations/layer0_attention_weight_max": 15.690523147583008,
      "activations/layer0_attention_weight_min": -12.871699333190918,
      "activations/layer10_attention_weight_max": 33.86702346801758,
      "activations/layer10_attention_weight_min": -35.898597717285156,
      "activations/layer11_attention_weight_max": 31.910255432128906,
      "activations/layer11_attention_weight_min": -34.294166564941406,
      "activations/layer12_attention_weight_max": 19.473310470581055,
      "activations/layer12_attention_weight_min": -25.064367294311523,
      "activations/layer13_attention_weight_max": 42.00579833984375,
      "activations/layer13_attention_weight_min": -33.45237731933594,
      "activations/layer14_attention_weight_max": 36.712650299072266,
      "activations/layer14_attention_weight_min": -28.473346710205078,
      "activations/layer15_attention_weight_max": 33.98550033569336,
      "activations/layer15_attention_weight_min": -29.589675903320312,
      "activations/layer16_attention_weight_max": 32.35246658325195,
      "activations/layer16_attention_weight_min": -30.556249618530273,
      "activations/layer17_attention_weight_max": 47.920616149902344,
      "activations/layer17_attention_weight_min": -42.855350494384766,
      "activations/layer18_attention_weight_max": 50.105316162109375,
      "activations/layer18_attention_weight_min": -42.4200553894043,
      "activations/layer19_attention_weight_max": 25.728872299194336,
      "activations/layer19_attention_weight_min": -23.57204818725586,
      "activations/layer1_attention_weight_max": 15.973153114318848,
      "activations/layer1_attention_weight_min": -13.540372848510742,
      "activations/layer20_attention_weight_max": 26.154895782470703,
      "activations/layer20_attention_weight_min": -21.41112518310547,
      "activations/layer21_attention_weight_max": 38.6943359375,
      "activations/layer21_attention_weight_min": -23.99704933166504,
      "activations/layer22_attention_weight_max": 31.675615310668945,
      "activations/layer22_attention_weight_min": -25.70893669128418,
      "activations/layer23_attention_weight_max": 40.18949890136719,
      "activations/layer23_attention_weight_min": -25.06757354736328,
      "activations/layer2_attention_weight_max": 32.228424072265625,
      "activations/layer2_attention_weight_min": -30.86273765563965,
      "activations/layer3_attention_weight_max": 94.90937805175781,
      "activations/layer3_attention_weight_min": -92.73536682128906,
      "activations/layer4_attention_weight_max": 88.47691345214844,
      "activations/layer4_attention_weight_min": -86.4737777709961,
      "activations/layer5_attention_weight_max": 65.68211364746094,
      "activations/layer5_attention_weight_min": -72.1381607055664,
      "activations/layer6_attention_weight_max": 49.05971908569336,
      "activations/layer6_attention_weight_min": -51.312278747558594,
      "activations/layer7_attention_weight_max": 62.35499954223633,
      "activations/layer7_attention_weight_min": -65.63806915283203,
      "activations/layer8_attention_weight_max": 45.624114990234375,
      "activations/layer8_attention_weight_min": -46.090152740478516,
      "activations/layer9_attention_weight_max": 41.709754943847656,
      "activations/layer9_attention_weight_min": -45.153926849365234,
      "epoch": 17.27,
      "learning_rate": 3.904924242424242e-05,
      "loss": 2.7466,
      "step": 297150
    },
    {
      "activations/layer0_attention_weight_max": 16.014541625976562,
      "activations/layer0_attention_weight_min": -13.120525360107422,
      "activations/layer10_attention_weight_max": 32.32701873779297,
      "activations/layer10_attention_weight_min": -34.646339416503906,
      "activations/layer11_attention_weight_max": 30.045461654663086,
      "activations/layer11_attention_weight_min": -32.10961151123047,
      "activations/layer12_attention_weight_max": 18.965808868408203,
      "activations/layer12_attention_weight_min": -22.24974822998047,
      "activations/layer13_attention_weight_max": 34.46441650390625,
      "activations/layer13_attention_weight_min": -29.259628295898438,
      "activations/layer14_attention_weight_max": 36.112945556640625,
      "activations/layer14_attention_weight_min": -30.603906631469727,
      "activations/layer15_attention_weight_max": 34.523014068603516,
      "activations/layer15_attention_weight_min": -32.316795349121094,
      "activations/layer16_attention_weight_max": 33.152992248535156,
      "activations/layer16_attention_weight_min": -32.73741912841797,
      "activations/layer17_attention_weight_max": 48.913978576660156,
      "activations/layer17_attention_weight_min": -46.08911895751953,
      "activations/layer18_attention_weight_max": 44.6768798828125,
      "activations/layer18_attention_weight_min": -38.011478424072266,
      "activations/layer19_attention_weight_max": 22.316150665283203,
      "activations/layer19_attention_weight_min": -22.448043823242188,
      "activations/layer1_attention_weight_max": 16.35372543334961,
      "activations/layer1_attention_weight_min": -15.216403007507324,
      "activations/layer20_attention_weight_max": 22.28367042541504,
      "activations/layer20_attention_weight_min": -20.442630767822266,
      "activations/layer21_attention_weight_max": 34.42872619628906,
      "activations/layer21_attention_weight_min": -20.95050621032715,
      "activations/layer22_attention_weight_max": 29.222211837768555,
      "activations/layer22_attention_weight_min": -26.68255043029785,
      "activations/layer23_attention_weight_max": 41.6656494140625,
      "activations/layer23_attention_weight_min": -25.44171714782715,
      "activations/layer2_attention_weight_max": 31.204238891601562,
      "activations/layer2_attention_weight_min": -32.26382064819336,
      "activations/layer3_attention_weight_max": 92.64982604980469,
      "activations/layer3_attention_weight_min": -94.81309509277344,
      "activations/layer4_attention_weight_max": 88.5556640625,
      "activations/layer4_attention_weight_min": -91.80132293701172,
      "activations/layer5_attention_weight_max": 64.00679016113281,
      "activations/layer5_attention_weight_min": -72.5592269897461,
      "activations/layer6_attention_weight_max": 48.153587341308594,
      "activations/layer6_attention_weight_min": -51.23153305053711,
      "activations/layer7_attention_weight_max": 61.51387023925781,
      "activations/layer7_attention_weight_min": -63.746437072753906,
      "activations/layer8_attention_weight_max": 41.50355911254883,
      "activations/layer8_attention_weight_min": -45.44626998901367,
      "activations/layer9_attention_weight_max": 39.33572769165039,
      "activations/layer9_attention_weight_min": -42.79562759399414,
      "epoch": 17.27,
      "learning_rate": 3.903030303030303e-05,
      "loss": 2.7336,
      "step": 297200
    },
    {
      "activations/layer0_attention_weight_max": 15.460983276367188,
      "activations/layer0_attention_weight_min": -13.253387451171875,
      "activations/layer10_attention_weight_max": 38.70649719238281,
      "activations/layer10_attention_weight_min": -37.37223434448242,
      "activations/layer11_attention_weight_max": 33.865509033203125,
      "activations/layer11_attention_weight_min": -36.283302307128906,
      "activations/layer12_attention_weight_max": 18.010046005249023,
      "activations/layer12_attention_weight_min": -28.055639266967773,
      "activations/layer13_attention_weight_max": 35.72401428222656,
      "activations/layer13_attention_weight_min": -32.78150939941406,
      "activations/layer14_attention_weight_max": 35.39458084106445,
      "activations/layer14_attention_weight_min": -30.532262802124023,
      "activations/layer15_attention_weight_max": 33.569461822509766,
      "activations/layer15_attention_weight_min": -30.4571590423584,
      "activations/layer16_attention_weight_max": 32.324153900146484,
      "activations/layer16_attention_weight_min": -31.159805297851562,
      "activations/layer17_attention_weight_max": 48.85777282714844,
      "activations/layer17_attention_weight_min": -42.904075622558594,
      "activations/layer18_attention_weight_max": 45.752662658691406,
      "activations/layer18_attention_weight_min": -38.487003326416016,
      "activations/layer19_attention_weight_max": 24.383983612060547,
      "activations/layer19_attention_weight_min": -23.578319549560547,
      "activations/layer1_attention_weight_max": 16.439016342163086,
      "activations/layer1_attention_weight_min": -16.23402214050293,
      "activations/layer20_attention_weight_max": 24.500951766967773,
      "activations/layer20_attention_weight_min": -21.757844924926758,
      "activations/layer21_attention_weight_max": 33.28189468383789,
      "activations/layer21_attention_weight_min": -21.35896110534668,
      "activations/layer22_attention_weight_max": 30.9327449798584,
      "activations/layer22_attention_weight_min": -26.2072696685791,
      "activations/layer23_attention_weight_max": 35.766624450683594,
      "activations/layer23_attention_weight_min": -25.564878463745117,
      "activations/layer2_attention_weight_max": 32.96745300292969,
      "activations/layer2_attention_weight_min": -31.526615142822266,
      "activations/layer3_attention_weight_max": 92.69749450683594,
      "activations/layer3_attention_weight_min": -92.72734069824219,
      "activations/layer4_attention_weight_max": 90.575927734375,
      "activations/layer4_attention_weight_min": -90.48147583007812,
      "activations/layer5_attention_weight_max": 68.12251281738281,
      "activations/layer5_attention_weight_min": -75.54170989990234,
      "activations/layer6_attention_weight_max": 55.38850402832031,
      "activations/layer6_attention_weight_min": -55.223854064941406,
      "activations/layer7_attention_weight_max": 69.72594451904297,
      "activations/layer7_attention_weight_min": -71.80289459228516,
      "activations/layer8_attention_weight_max": 49.35337829589844,
      "activations/layer8_attention_weight_min": -54.98219680786133,
      "activations/layer9_attention_weight_max": 48.600582122802734,
      "activations/layer9_attention_weight_min": -48.659996032714844,
      "epoch": 17.27,
      "learning_rate": 3.901136363636363e-05,
      "loss": 2.7401,
      "step": 297250
    },
    {
      "activations/layer0_attention_weight_max": 15.25252914428711,
      "activations/layer0_attention_weight_min": -12.991168975830078,
      "activations/layer10_attention_weight_max": 36.6669921875,
      "activations/layer10_attention_weight_min": -36.1800422668457,
      "activations/layer11_attention_weight_max": 33.37643051147461,
      "activations/layer11_attention_weight_min": -33.79611587524414,
      "activations/layer12_attention_weight_max": 17.986536026000977,
      "activations/layer12_attention_weight_min": -27.660179138183594,
      "activations/layer13_attention_weight_max": 34.72919845581055,
      "activations/layer13_attention_weight_min": -27.02567481994629,
      "activations/layer14_attention_weight_max": 34.20381164550781,
      "activations/layer14_attention_weight_min": -31.490995407104492,
      "activations/layer15_attention_weight_max": 31.931289672851562,
      "activations/layer15_attention_weight_min": -30.478988647460938,
      "activations/layer16_attention_weight_max": 33.028053283691406,
      "activations/layer16_attention_weight_min": -34.03243637084961,
      "activations/layer17_attention_weight_max": 49.30476379394531,
      "activations/layer17_attention_weight_min": -46.03818893432617,
      "activations/layer18_attention_weight_max": 46.253746032714844,
      "activations/layer18_attention_weight_min": -43.042503356933594,
      "activations/layer19_attention_weight_max": 25.000308990478516,
      "activations/layer19_attention_weight_min": -23.349966049194336,
      "activations/layer1_attention_weight_max": 16.117305755615234,
      "activations/layer1_attention_weight_min": -13.587103843688965,
      "activations/layer20_attention_weight_max": 23.66238784790039,
      "activations/layer20_attention_weight_min": -20.573068618774414,
      "activations/layer21_attention_weight_max": 33.7918701171875,
      "activations/layer21_attention_weight_min": -23.999156951904297,
      "activations/layer22_attention_weight_max": 30.64666748046875,
      "activations/layer22_attention_weight_min": -25.49406623840332,
      "activations/layer23_attention_weight_max": 36.9482307434082,
      "activations/layer23_attention_weight_min": -23.353851318359375,
      "activations/layer2_attention_weight_max": 31.965675354003906,
      "activations/layer2_attention_weight_min": -29.901599884033203,
      "activations/layer3_attention_weight_max": 89.12136840820312,
      "activations/layer3_attention_weight_min": -91.20868682861328,
      "activations/layer4_attention_weight_max": 88.05250549316406,
      "activations/layer4_attention_weight_min": -85.60108947753906,
      "activations/layer5_attention_weight_max": 69.38594055175781,
      "activations/layer5_attention_weight_min": -73.24589538574219,
      "activations/layer6_attention_weight_max": 49.22319412231445,
      "activations/layer6_attention_weight_min": -50.854373931884766,
      "activations/layer7_attention_weight_max": 62.89122772216797,
      "activations/layer7_attention_weight_min": -63.95439147949219,
      "activations/layer8_attention_weight_max": 45.28143310546875,
      "activations/layer8_attention_weight_min": -50.00756072998047,
      "activations/layer9_attention_weight_max": 43.26502990722656,
      "activations/layer9_attention_weight_min": -46.55723571777344,
      "epoch": 17.27,
      "learning_rate": 3.899242424242423e-05,
      "loss": 2.7403,
      "step": 297300
    },
    {
      "activations/layer0_attention_weight_max": 15.097014427185059,
      "activations/layer0_attention_weight_min": -12.77595043182373,
      "activations/layer10_attention_weight_max": 33.9672966003418,
      "activations/layer10_attention_weight_min": -32.592559814453125,
      "activations/layer11_attention_weight_max": 30.867610931396484,
      "activations/layer11_attention_weight_min": -33.007877349853516,
      "activations/layer12_attention_weight_max": 19.666305541992188,
      "activations/layer12_attention_weight_min": -23.697650909423828,
      "activations/layer13_attention_weight_max": 33.19868850708008,
      "activations/layer13_attention_weight_min": -26.289113998413086,
      "activations/layer14_attention_weight_max": 35.72709274291992,
      "activations/layer14_attention_weight_min": -28.5938777923584,
      "activations/layer15_attention_weight_max": 32.42740249633789,
      "activations/layer15_attention_weight_min": -29.626554489135742,
      "activations/layer16_attention_weight_max": 32.58359146118164,
      "activations/layer16_attention_weight_min": -31.042558670043945,
      "activations/layer17_attention_weight_max": 54.22017288208008,
      "activations/layer17_attention_weight_min": -45.811561584472656,
      "activations/layer18_attention_weight_max": 51.37837600708008,
      "activations/layer18_attention_weight_min": -40.95648956298828,
      "activations/layer19_attention_weight_max": 24.998903274536133,
      "activations/layer19_attention_weight_min": -23.710981369018555,
      "activations/layer1_attention_weight_max": 16.37039566040039,
      "activations/layer1_attention_weight_min": -13.221566200256348,
      "activations/layer20_attention_weight_max": 22.125585556030273,
      "activations/layer20_attention_weight_min": -23.43805503845215,
      "activations/layer21_attention_weight_max": 32.58879470825195,
      "activations/layer21_attention_weight_min": -23.38858413696289,
      "activations/layer22_attention_weight_max": 29.884166717529297,
      "activations/layer22_attention_weight_min": -25.247495651245117,
      "activations/layer23_attention_weight_max": 35.37376403808594,
      "activations/layer23_attention_weight_min": -24.799545288085938,
      "activations/layer2_attention_weight_max": 31.202939987182617,
      "activations/layer2_attention_weight_min": -31.66188621520996,
      "activations/layer3_attention_weight_max": 86.81019592285156,
      "activations/layer3_attention_weight_min": -94.8067626953125,
      "activations/layer4_attention_weight_max": 87.03353881835938,
      "activations/layer4_attention_weight_min": -84.7574234008789,
      "activations/layer5_attention_weight_max": 64.10746765136719,
      "activations/layer5_attention_weight_min": -76.7340087890625,
      "activations/layer6_attention_weight_max": 48.53363800048828,
      "activations/layer6_attention_weight_min": -48.297279357910156,
      "activations/layer7_attention_weight_max": 62.004573822021484,
      "activations/layer7_attention_weight_min": -63.53302001953125,
      "activations/layer8_attention_weight_max": 43.269569396972656,
      "activations/layer8_attention_weight_min": -46.7314338684082,
      "activations/layer9_attention_weight_max": 40.82949447631836,
      "activations/layer9_attention_weight_min": -47.73746109008789,
      "epoch": 17.28,
      "learning_rate": 3.897348484848485e-05,
      "loss": 2.7393,
      "step": 297350
    },
    {
      "activations/layer0_attention_weight_max": 16.088760375976562,
      "activations/layer0_attention_weight_min": -13.43304443359375,
      "activations/layer10_attention_weight_max": 36.98344039916992,
      "activations/layer10_attention_weight_min": -37.81435775756836,
      "activations/layer11_attention_weight_max": 34.05501937866211,
      "activations/layer11_attention_weight_min": -35.59397506713867,
      "activations/layer12_attention_weight_max": 17.9724178314209,
      "activations/layer12_attention_weight_min": -26.784082412719727,
      "activations/layer13_attention_weight_max": 44.51893997192383,
      "activations/layer13_attention_weight_min": -28.345874786376953,
      "activations/layer14_attention_weight_max": 52.259517669677734,
      "activations/layer14_attention_weight_min": -29.766027450561523,
      "activations/layer15_attention_weight_max": 45.45701599121094,
      "activations/layer15_attention_weight_min": -30.38197135925293,
      "activations/layer16_attention_weight_max": 34.07979965209961,
      "activations/layer16_attention_weight_min": -32.38094711303711,
      "activations/layer17_attention_weight_max": 51.113182067871094,
      "activations/layer17_attention_weight_min": -50.38786315917969,
      "activations/layer18_attention_weight_max": 49.46422576904297,
      "activations/layer18_attention_weight_min": -45.5437126159668,
      "activations/layer19_attention_weight_max": 24.256786346435547,
      "activations/layer19_attention_weight_min": -23.231298446655273,
      "activations/layer1_attention_weight_max": 16.171878814697266,
      "activations/layer1_attention_weight_min": -15.114557266235352,
      "activations/layer20_attention_weight_max": 24.221616744995117,
      "activations/layer20_attention_weight_min": -21.728933334350586,
      "activations/layer21_attention_weight_max": 37.55988311767578,
      "activations/layer21_attention_weight_min": -25.917369842529297,
      "activations/layer22_attention_weight_max": 29.63457489013672,
      "activations/layer22_attention_weight_min": -25.97130012512207,
      "activations/layer23_attention_weight_max": 36.570491790771484,
      "activations/layer23_attention_weight_min": -24.834922790527344,
      "activations/layer2_attention_weight_max": 34.4642333984375,
      "activations/layer2_attention_weight_min": -32.52384948730469,
      "activations/layer3_attention_weight_max": 95.45487213134766,
      "activations/layer3_attention_weight_min": -92.7327880859375,
      "activations/layer4_attention_weight_max": 87.7090072631836,
      "activations/layer4_attention_weight_min": -88.72715759277344,
      "activations/layer5_attention_weight_max": 62.09291076660156,
      "activations/layer5_attention_weight_min": -73.56925964355469,
      "activations/layer6_attention_weight_max": 49.432762145996094,
      "activations/layer6_attention_weight_min": -53.09389114379883,
      "activations/layer7_attention_weight_max": 62.98562240600586,
      "activations/layer7_attention_weight_min": -64.7952880859375,
      "activations/layer8_attention_weight_max": 46.911231994628906,
      "activations/layer8_attention_weight_min": -49.08204650878906,
      "activations/layer9_attention_weight_max": 46.820735931396484,
      "activations/layer9_attention_weight_min": -48.582393646240234,
      "epoch": 17.28,
      "learning_rate": 3.895454545454545e-05,
      "loss": 2.7277,
      "step": 297400
    },
    {
      "activations/layer0_attention_weight_max": 16.315797805786133,
      "activations/layer0_attention_weight_min": -12.800355911254883,
      "activations/layer10_attention_weight_max": 35.4893798828125,
      "activations/layer10_attention_weight_min": -36.02799987792969,
      "activations/layer11_attention_weight_max": 33.24047088623047,
      "activations/layer11_attention_weight_min": -35.22981643676758,
      "activations/layer12_attention_weight_max": 19.127315521240234,
      "activations/layer12_attention_weight_min": -26.960044860839844,
      "activations/layer13_attention_weight_max": 33.941566467285156,
      "activations/layer13_attention_weight_min": -26.3016414642334,
      "activations/layer14_attention_weight_max": 45.337955474853516,
      "activations/layer14_attention_weight_min": -29.964841842651367,
      "activations/layer15_attention_weight_max": 40.468936920166016,
      "activations/layer15_attention_weight_min": -30.610713958740234,
      "activations/layer16_attention_weight_max": 37.125526428222656,
      "activations/layer16_attention_weight_min": -34.96171188354492,
      "activations/layer17_attention_weight_max": 57.80753707885742,
      "activations/layer17_attention_weight_min": -54.50693893432617,
      "activations/layer18_attention_weight_max": 49.26451873779297,
      "activations/layer18_attention_weight_min": -43.25065231323242,
      "activations/layer19_attention_weight_max": 25.04631233215332,
      "activations/layer19_attention_weight_min": -23.524574279785156,
      "activations/layer1_attention_weight_max": 16.73759651184082,
      "activations/layer1_attention_weight_min": -15.456995010375977,
      "activations/layer20_attention_weight_max": 23.739009857177734,
      "activations/layer20_attention_weight_min": -21.362812042236328,
      "activations/layer21_attention_weight_max": 38.38330078125,
      "activations/layer21_attention_weight_min": -21.062646865844727,
      "activations/layer22_attention_weight_max": 32.71771240234375,
      "activations/layer22_attention_weight_min": -24.531160354614258,
      "activations/layer23_attention_weight_max": 41.38526916503906,
      "activations/layer23_attention_weight_min": -25.4088134765625,
      "activations/layer2_attention_weight_max": 33.408599853515625,
      "activations/layer2_attention_weight_min": -29.9970703125,
      "activations/layer3_attention_weight_max": 87.66559600830078,
      "activations/layer3_attention_weight_min": -88.09037780761719,
      "activations/layer4_attention_weight_max": 88.06499481201172,
      "activations/layer4_attention_weight_min": -84.69078063964844,
      "activations/layer5_attention_weight_max": 67.2895736694336,
      "activations/layer5_attention_weight_min": -69.68682861328125,
      "activations/layer6_attention_weight_max": 49.04164505004883,
      "activations/layer6_attention_weight_min": -50.0058708190918,
      "activations/layer7_attention_weight_max": 65.15625762939453,
      "activations/layer7_attention_weight_min": -65.9840087890625,
      "activations/layer8_attention_weight_max": 46.74224853515625,
      "activations/layer8_attention_weight_min": -48.94160842895508,
      "activations/layer9_attention_weight_max": 44.537559509277344,
      "activations/layer9_attention_weight_min": -46.92106628417969,
      "epoch": 17.28,
      "learning_rate": 3.893560606060606e-05,
      "loss": 2.745,
      "step": 297450
    },
    {
      "activations/layer0_attention_weight_max": 15.804675102233887,
      "activations/layer0_attention_weight_min": -12.642178535461426,
      "activations/layer10_attention_weight_max": 37.43326187133789,
      "activations/layer10_attention_weight_min": -38.65308380126953,
      "activations/layer11_attention_weight_max": 33.12255096435547,
      "activations/layer11_attention_weight_min": -34.76371765136719,
      "activations/layer12_attention_weight_max": 18.89677619934082,
      "activations/layer12_attention_weight_min": -29.238710403442383,
      "activations/layer13_attention_weight_max": 35.39160919189453,
      "activations/layer13_attention_weight_min": -28.696422576904297,
      "activations/layer14_attention_weight_max": 36.09683609008789,
      "activations/layer14_attention_weight_min": -31.663705825805664,
      "activations/layer15_attention_weight_max": 34.43930435180664,
      "activations/layer15_attention_weight_min": -30.5949649810791,
      "activations/layer16_attention_weight_max": 34.28624725341797,
      "activations/layer16_attention_weight_min": -32.00708770751953,
      "activations/layer17_attention_weight_max": 53.27943801879883,
      "activations/layer17_attention_weight_min": -48.18226623535156,
      "activations/layer18_attention_weight_max": 49.9425163269043,
      "activations/layer18_attention_weight_min": -42.0771598815918,
      "activations/layer19_attention_weight_max": 25.394651412963867,
      "activations/layer19_attention_weight_min": -24.07506561279297,
      "activations/layer1_attention_weight_max": 15.44854736328125,
      "activations/layer1_attention_weight_min": -14.958250999450684,
      "activations/layer20_attention_weight_max": 25.05760955810547,
      "activations/layer20_attention_weight_min": -21.47028923034668,
      "activations/layer21_attention_weight_max": 51.65024948120117,
      "activations/layer21_attention_weight_min": -25.25758934020996,
      "activations/layer22_attention_weight_max": 31.163002014160156,
      "activations/layer22_attention_weight_min": -25.485450744628906,
      "activations/layer23_attention_weight_max": 38.30799865722656,
      "activations/layer23_attention_weight_min": -23.897037506103516,
      "activations/layer2_attention_weight_max": 32.99828338623047,
      "activations/layer2_attention_weight_min": -30.846370697021484,
      "activations/layer3_attention_weight_max": 96.0996322631836,
      "activations/layer3_attention_weight_min": -94.7442855834961,
      "activations/layer4_attention_weight_max": 92.55154418945312,
      "activations/layer4_attention_weight_min": -89.74742126464844,
      "activations/layer5_attention_weight_max": 65.45144653320312,
      "activations/layer5_attention_weight_min": -78.33778381347656,
      "activations/layer6_attention_weight_max": 50.584590911865234,
      "activations/layer6_attention_weight_min": -53.95652770996094,
      "activations/layer7_attention_weight_max": 66.09591674804688,
      "activations/layer7_attention_weight_min": -65.37074279785156,
      "activations/layer8_attention_weight_max": 46.14106750488281,
      "activations/layer8_attention_weight_min": -48.067543029785156,
      "activations/layer9_attention_weight_max": 46.15965270996094,
      "activations/layer9_attention_weight_min": -46.03227996826172,
      "epoch": 17.29,
      "learning_rate": 3.891666666666666e-05,
      "loss": 2.7255,
      "step": 297500
    },
    {
      "activations/layer0_attention_weight_max": 15.962494850158691,
      "activations/layer0_attention_weight_min": -12.841544151306152,
      "activations/layer10_attention_weight_max": 35.944183349609375,
      "activations/layer10_attention_weight_min": -36.45893478393555,
      "activations/layer11_attention_weight_max": 32.51325607299805,
      "activations/layer11_attention_weight_min": -33.3228874206543,
      "activations/layer12_attention_weight_max": 17.536792755126953,
      "activations/layer12_attention_weight_min": -25.862855911254883,
      "activations/layer13_attention_weight_max": 34.834896087646484,
      "activations/layer13_attention_weight_min": -29.813312530517578,
      "activations/layer14_attention_weight_max": 33.93537902832031,
      "activations/layer14_attention_weight_min": -32.89409255981445,
      "activations/layer15_attention_weight_max": 33.52799987792969,
      "activations/layer15_attention_weight_min": -32.7897834777832,
      "activations/layer16_attention_weight_max": 34.702205657958984,
      "activations/layer16_attention_weight_min": -32.57528305053711,
      "activations/layer17_attention_weight_max": 53.723358154296875,
      "activations/layer17_attention_weight_min": -46.163230895996094,
      "activations/layer18_attention_weight_max": 51.73036193847656,
      "activations/layer18_attention_weight_min": -42.77715301513672,
      "activations/layer19_attention_weight_max": 25.77601432800293,
      "activations/layer19_attention_weight_min": -25.129377365112305,
      "activations/layer1_attention_weight_max": 16.132633209228516,
      "activations/layer1_attention_weight_min": -13.764678955078125,
      "activations/layer20_attention_weight_max": 25.533212661743164,
      "activations/layer20_attention_weight_min": -26.320711135864258,
      "activations/layer21_attention_weight_max": 37.95045852661133,
      "activations/layer21_attention_weight_min": -22.237295150756836,
      "activations/layer22_attention_weight_max": 32.68720626831055,
      "activations/layer22_attention_weight_min": -26.44378662109375,
      "activations/layer23_attention_weight_max": 37.39845275878906,
      "activations/layer23_attention_weight_min": -24.691425323486328,
      "activations/layer2_attention_weight_max": 31.84798240661621,
      "activations/layer2_attention_weight_min": -30.580068588256836,
      "activations/layer3_attention_weight_max": 88.2950439453125,
      "activations/layer3_attention_weight_min": -88.95182800292969,
      "activations/layer4_attention_weight_max": 87.11956787109375,
      "activations/layer4_attention_weight_min": -88.60735321044922,
      "activations/layer5_attention_weight_max": 63.172706604003906,
      "activations/layer5_attention_weight_min": -74.94125366210938,
      "activations/layer6_attention_weight_max": 49.47838592529297,
      "activations/layer6_attention_weight_min": -50.17338562011719,
      "activations/layer7_attention_weight_max": 63.16178894042969,
      "activations/layer7_attention_weight_min": -64.78955078125,
      "activations/layer8_attention_weight_max": 45.064361572265625,
      "activations/layer8_attention_weight_min": -53.73725891113281,
      "activations/layer9_attention_weight_max": 43.40126419067383,
      "activations/layer9_attention_weight_min": -47.1423225402832,
      "epoch": 17.29,
      "learning_rate": 3.8897727272727274e-05,
      "loss": 2.7401,
      "step": 297550
    },
    {
      "activations/layer0_attention_weight_max": 15.74171257019043,
      "activations/layer0_attention_weight_min": -13.26783561706543,
      "activations/layer10_attention_weight_max": 33.930362701416016,
      "activations/layer10_attention_weight_min": -35.11541748046875,
      "activations/layer11_attention_weight_max": 32.543052673339844,
      "activations/layer11_attention_weight_min": -32.3895263671875,
      "activations/layer12_attention_weight_max": 21.487171173095703,
      "activations/layer12_attention_weight_min": -28.739221572875977,
      "activations/layer13_attention_weight_max": 40.5671272277832,
      "activations/layer13_attention_weight_min": -31.811199188232422,
      "activations/layer14_attention_weight_max": 40.76473617553711,
      "activations/layer14_attention_weight_min": -29.997953414916992,
      "activations/layer15_attention_weight_max": 36.023704528808594,
      "activations/layer15_attention_weight_min": -29.961231231689453,
      "activations/layer16_attention_weight_max": 36.16896057128906,
      "activations/layer16_attention_weight_min": -34.69829559326172,
      "activations/layer17_attention_weight_max": 54.670108795166016,
      "activations/layer17_attention_weight_min": -47.811153411865234,
      "activations/layer18_attention_weight_max": 48.6267204284668,
      "activations/layer18_attention_weight_min": -41.79618453979492,
      "activations/layer19_attention_weight_max": 27.92994499206543,
      "activations/layer19_attention_weight_min": -27.184492111206055,
      "activations/layer1_attention_weight_max": 16.551532745361328,
      "activations/layer1_attention_weight_min": -13.838946342468262,
      "activations/layer20_attention_weight_max": 25.993446350097656,
      "activations/layer20_attention_weight_min": -24.3370418548584,
      "activations/layer21_attention_weight_max": 52.04913330078125,
      "activations/layer21_attention_weight_min": -23.793729782104492,
      "activations/layer22_attention_weight_max": 29.755449295043945,
      "activations/layer22_attention_weight_min": -25.812843322753906,
      "activations/layer23_attention_weight_max": 38.845863342285156,
      "activations/layer23_attention_weight_min": -24.094676971435547,
      "activations/layer2_attention_weight_max": 33.4128532409668,
      "activations/layer2_attention_weight_min": -31.9210262298584,
      "activations/layer3_attention_weight_max": 93.60783386230469,
      "activations/layer3_attention_weight_min": -94.0899658203125,
      "activations/layer4_attention_weight_max": 90.56363677978516,
      "activations/layer4_attention_weight_min": -90.85591888427734,
      "activations/layer5_attention_weight_max": 66.67416381835938,
      "activations/layer5_attention_weight_min": -74.80714416503906,
      "activations/layer6_attention_weight_max": 50.93937301635742,
      "activations/layer6_attention_weight_min": -50.6707649230957,
      "activations/layer7_attention_weight_max": 69.46036529541016,
      "activations/layer7_attention_weight_min": -63.643531799316406,
      "activations/layer8_attention_weight_max": 45.0611457824707,
      "activations/layer8_attention_weight_min": -46.55228042602539,
      "activations/layer9_attention_weight_max": 43.7613410949707,
      "activations/layer9_attention_weight_min": -46.297908782958984,
      "epoch": 17.29,
      "learning_rate": 3.8878787878787876e-05,
      "loss": 2.7345,
      "step": 297600
    },
    {
      "activations/layer0_attention_weight_max": 16.421934127807617,
      "activations/layer0_attention_weight_min": -12.638591766357422,
      "activations/layer10_attention_weight_max": 33.32239532470703,
      "activations/layer10_attention_weight_min": -35.360801696777344,
      "activations/layer11_attention_weight_max": 29.74304962158203,
      "activations/layer11_attention_weight_min": -31.373992919921875,
      "activations/layer12_attention_weight_max": 18.288990020751953,
      "activations/layer12_attention_weight_min": -24.707138061523438,
      "activations/layer13_attention_weight_max": 35.01451110839844,
      "activations/layer13_attention_weight_min": -28.56694984436035,
      "activations/layer14_attention_weight_max": 33.34162139892578,
      "activations/layer14_attention_weight_min": -29.77638816833496,
      "activations/layer15_attention_weight_max": 32.533443450927734,
      "activations/layer15_attention_weight_min": -29.17868995666504,
      "activations/layer16_attention_weight_max": 33.59620666503906,
      "activations/layer16_attention_weight_min": -31.896577835083008,
      "activations/layer17_attention_weight_max": 51.249908447265625,
      "activations/layer17_attention_weight_min": -46.977088928222656,
      "activations/layer18_attention_weight_max": 50.27873992919922,
      "activations/layer18_attention_weight_min": -42.20363235473633,
      "activations/layer19_attention_weight_max": 24.253053665161133,
      "activations/layer19_attention_weight_min": -24.48915672302246,
      "activations/layer1_attention_weight_max": 15.821891784667969,
      "activations/layer1_attention_weight_min": -15.319527626037598,
      "activations/layer20_attention_weight_max": 24.78905487060547,
      "activations/layer20_attention_weight_min": -23.43303108215332,
      "activations/layer21_attention_weight_max": 36.391387939453125,
      "activations/layer21_attention_weight_min": -23.95375633239746,
      "activations/layer22_attention_weight_max": 26.983863830566406,
      "activations/layer22_attention_weight_min": -25.917768478393555,
      "activations/layer23_attention_weight_max": 34.15764236450195,
      "activations/layer23_attention_weight_min": -26.16963005065918,
      "activations/layer2_attention_weight_max": 31.281843185424805,
      "activations/layer2_attention_weight_min": -30.398412704467773,
      "activations/layer3_attention_weight_max": 87.14850616455078,
      "activations/layer3_attention_weight_min": -85.54588317871094,
      "activations/layer4_attention_weight_max": 83.92469024658203,
      "activations/layer4_attention_weight_min": -83.48204803466797,
      "activations/layer5_attention_weight_max": 66.50487518310547,
      "activations/layer5_attention_weight_min": -72.72264862060547,
      "activations/layer6_attention_weight_max": 48.91062927246094,
      "activations/layer6_attention_weight_min": -48.3464469909668,
      "activations/layer7_attention_weight_max": 63.423057556152344,
      "activations/layer7_attention_weight_min": -60.827247619628906,
      "activations/layer8_attention_weight_max": 45.409664154052734,
      "activations/layer8_attention_weight_min": -46.86486053466797,
      "activations/layer9_attention_weight_max": 39.049007415771484,
      "activations/layer9_attention_weight_min": -42.80625915527344,
      "epoch": 17.3,
      "learning_rate": 3.885984848484848e-05,
      "loss": 2.734,
      "step": 297650
    },
    {
      "activations/layer0_attention_weight_max": 15.6776704788208,
      "activations/layer0_attention_weight_min": -13.056543350219727,
      "activations/layer10_attention_weight_max": 32.957576751708984,
      "activations/layer10_attention_weight_min": -34.277488708496094,
      "activations/layer11_attention_weight_max": 29.656049728393555,
      "activations/layer11_attention_weight_min": -32.982025146484375,
      "activations/layer12_attention_weight_max": 19.05202293395996,
      "activations/layer12_attention_weight_min": -24.843040466308594,
      "activations/layer13_attention_weight_max": 30.910003662109375,
      "activations/layer13_attention_weight_min": -27.53934669494629,
      "activations/layer14_attention_weight_max": 31.381723403930664,
      "activations/layer14_attention_weight_min": -31.82608985900879,
      "activations/layer15_attention_weight_max": 33.431190490722656,
      "activations/layer15_attention_weight_min": -31.595874786376953,
      "activations/layer16_attention_weight_max": 33.37919235229492,
      "activations/layer16_attention_weight_min": -30.518321990966797,
      "activations/layer17_attention_weight_max": 48.80216598510742,
      "activations/layer17_attention_weight_min": -45.7220573425293,
      "activations/layer18_attention_weight_max": 47.877403259277344,
      "activations/layer18_attention_weight_min": -42.1383056640625,
      "activations/layer19_attention_weight_max": 22.827455520629883,
      "activations/layer19_attention_weight_min": -22.784147262573242,
      "activations/layer1_attention_weight_max": 16.73243522644043,
      "activations/layer1_attention_weight_min": -13.411808967590332,
      "activations/layer20_attention_weight_max": 23.931095123291016,
      "activations/layer20_attention_weight_min": -24.326580047607422,
      "activations/layer21_attention_weight_max": 33.663761138916016,
      "activations/layer21_attention_weight_min": -22.96211814880371,
      "activations/layer22_attention_weight_max": 29.021583557128906,
      "activations/layer22_attention_weight_min": -28.231719970703125,
      "activations/layer23_attention_weight_max": 36.056671142578125,
      "activations/layer23_attention_weight_min": -27.608837127685547,
      "activations/layer2_attention_weight_max": 30.200286865234375,
      "activations/layer2_attention_weight_min": -30.597728729248047,
      "activations/layer3_attention_weight_max": 86.98077392578125,
      "activations/layer3_attention_weight_min": -82.61518096923828,
      "activations/layer4_attention_weight_max": 90.2718734741211,
      "activations/layer4_attention_weight_min": -85.3677978515625,
      "activations/layer5_attention_weight_max": 66.38270568847656,
      "activations/layer5_attention_weight_min": -72.25271606445312,
      "activations/layer6_attention_weight_max": 49.996402740478516,
      "activations/layer6_attention_weight_min": -51.8348503112793,
      "activations/layer7_attention_weight_max": 64.33629608154297,
      "activations/layer7_attention_weight_min": -65.36325073242188,
      "activations/layer8_attention_weight_max": 43.91262435913086,
      "activations/layer8_attention_weight_min": -44.25606155395508,
      "activations/layer9_attention_weight_max": 47.02207946777344,
      "activations/layer9_attention_weight_min": -42.94895553588867,
      "epoch": 17.3,
      "learning_rate": 3.8841287878787875e-05,
      "loss": 2.7332,
      "step": 297700
    },
    {
      "activations/layer0_attention_weight_max": 16.02083969116211,
      "activations/layer0_attention_weight_min": -13.522236824035645,
      "activations/layer10_attention_weight_max": 34.58245086669922,
      "activations/layer10_attention_weight_min": -35.99977111816406,
      "activations/layer11_attention_weight_max": 31.841428756713867,
      "activations/layer11_attention_weight_min": -33.04378128051758,
      "activations/layer12_attention_weight_max": 19.439786911010742,
      "activations/layer12_attention_weight_min": -25.028438568115234,
      "activations/layer13_attention_weight_max": 34.32998275756836,
      "activations/layer13_attention_weight_min": -27.220182418823242,
      "activations/layer14_attention_weight_max": 35.80509567260742,
      "activations/layer14_attention_weight_min": -31.06998062133789,
      "activations/layer15_attention_weight_max": 33.82585525512695,
      "activations/layer15_attention_weight_min": -31.331769943237305,
      "activations/layer16_attention_weight_max": 34.27128982543945,
      "activations/layer16_attention_weight_min": -32.33306884765625,
      "activations/layer17_attention_weight_max": 52.981529235839844,
      "activations/layer17_attention_weight_min": -47.044639587402344,
      "activations/layer18_attention_weight_max": 48.258243560791016,
      "activations/layer18_attention_weight_min": -40.38716506958008,
      "activations/layer19_attention_weight_max": 22.6928768157959,
      "activations/layer19_attention_weight_min": -23.07853889465332,
      "activations/layer1_attention_weight_max": 15.413655281066895,
      "activations/layer1_attention_weight_min": -13.269505500793457,
      "activations/layer20_attention_weight_max": 21.148454666137695,
      "activations/layer20_attention_weight_min": -22.63646125793457,
      "activations/layer21_attention_weight_max": 34.83867263793945,
      "activations/layer21_attention_weight_min": -29.146169662475586,
      "activations/layer22_attention_weight_max": 32.122642517089844,
      "activations/layer22_attention_weight_min": -27.34288215637207,
      "activations/layer23_attention_weight_max": 35.44400405883789,
      "activations/layer23_attention_weight_min": -31.742088317871094,
      "activations/layer2_attention_weight_max": 30.214282989501953,
      "activations/layer2_attention_weight_min": -28.901927947998047,
      "activations/layer3_attention_weight_max": 85.64600372314453,
      "activations/layer3_attention_weight_min": -86.76937103271484,
      "activations/layer4_attention_weight_max": 82.15691375732422,
      "activations/layer4_attention_weight_min": -84.22981262207031,
      "activations/layer5_attention_weight_max": 62.238037109375,
      "activations/layer5_attention_weight_min": -70.15666961669922,
      "activations/layer6_attention_weight_max": 47.095584869384766,
      "activations/layer6_attention_weight_min": -48.6604118347168,
      "activations/layer7_attention_weight_max": 62.933658599853516,
      "activations/layer7_attention_weight_min": -63.05685043334961,
      "activations/layer8_attention_weight_max": 44.69753646850586,
      "activations/layer8_attention_weight_min": -49.595760345458984,
      "activations/layer9_attention_weight_max": 46.63006591796875,
      "activations/layer9_attention_weight_min": -47.70096206665039,
      "epoch": 17.3,
      "learning_rate": 3.882234848484848e-05,
      "loss": 2.7462,
      "step": 297750
    },
    {
      "activations/layer0_attention_weight_max": 14.982144355773926,
      "activations/layer0_attention_weight_min": -12.942767143249512,
      "activations/layer10_attention_weight_max": 37.39175033569336,
      "activations/layer10_attention_weight_min": -37.09827423095703,
      "activations/layer11_attention_weight_max": 33.04920196533203,
      "activations/layer11_attention_weight_min": -34.39326095581055,
      "activations/layer12_attention_weight_max": 18.439579010009766,
      "activations/layer12_attention_weight_min": -24.075374603271484,
      "activations/layer13_attention_weight_max": 32.970123291015625,
      "activations/layer13_attention_weight_min": -30.20107650756836,
      "activations/layer14_attention_weight_max": 35.04286575317383,
      "activations/layer14_attention_weight_min": -30.623323440551758,
      "activations/layer15_attention_weight_max": 33.688819885253906,
      "activations/layer15_attention_weight_min": -31.61096954345703,
      "activations/layer16_attention_weight_max": 35.642921447753906,
      "activations/layer16_attention_weight_min": -34.635162353515625,
      "activations/layer17_attention_weight_max": 52.022010803222656,
      "activations/layer17_attention_weight_min": -47.245025634765625,
      "activations/layer18_attention_weight_max": 49.08368682861328,
      "activations/layer18_attention_weight_min": -43.44308090209961,
      "activations/layer19_attention_weight_max": 24.298927307128906,
      "activations/layer19_attention_weight_min": -22.96814727783203,
      "activations/layer1_attention_weight_max": 16.151283264160156,
      "activations/layer1_attention_weight_min": -13.716628074645996,
      "activations/layer20_attention_weight_max": 25.229276657104492,
      "activations/layer20_attention_weight_min": -23.039701461791992,
      "activations/layer21_attention_weight_max": 37.920494079589844,
      "activations/layer21_attention_weight_min": -21.88656234741211,
      "activations/layer22_attention_weight_max": 29.645418167114258,
      "activations/layer22_attention_weight_min": -27.051715850830078,
      "activations/layer23_attention_weight_max": 34.98125457763672,
      "activations/layer23_attention_weight_min": -27.569358825683594,
      "activations/layer2_attention_weight_max": 32.74513244628906,
      "activations/layer2_attention_weight_min": -32.16705322265625,
      "activations/layer3_attention_weight_max": 91.55747985839844,
      "activations/layer3_attention_weight_min": -91.52935028076172,
      "activations/layer4_attention_weight_max": 88.82854461669922,
      "activations/layer4_attention_weight_min": -90.7059555053711,
      "activations/layer5_attention_weight_max": 68.28047180175781,
      "activations/layer5_attention_weight_min": -76.73833465576172,
      "activations/layer6_attention_weight_max": 51.509578704833984,
      "activations/layer6_attention_weight_min": -52.38393783569336,
      "activations/layer7_attention_weight_max": 64.05457305908203,
      "activations/layer7_attention_weight_min": -63.48025131225586,
      "activations/layer8_attention_weight_max": 47.59636306762695,
      "activations/layer8_attention_weight_min": -50.99956512451172,
      "activations/layer9_attention_weight_max": 44.315181732177734,
      "activations/layer9_attention_weight_min": -47.11824417114258,
      "epoch": 17.3,
      "learning_rate": 3.8803409090909085e-05,
      "loss": 2.7348,
      "step": 297800
    },
    {
      "activations/layer0_attention_weight_max": 15.597860336303711,
      "activations/layer0_attention_weight_min": -12.659211158752441,
      "activations/layer10_attention_weight_max": 35.5692024230957,
      "activations/layer10_attention_weight_min": -36.558963775634766,
      "activations/layer11_attention_weight_max": 32.151695251464844,
      "activations/layer11_attention_weight_min": -36.16205596923828,
      "activations/layer12_attention_weight_max": 19.88040542602539,
      "activations/layer12_attention_weight_min": -30.289051055908203,
      "activations/layer13_attention_weight_max": 31.23162269592285,
      "activations/layer13_attention_weight_min": -31.08047866821289,
      "activations/layer14_attention_weight_max": 37.00339889526367,
      "activations/layer14_attention_weight_min": -32.89817810058594,
      "activations/layer15_attention_weight_max": 34.05927276611328,
      "activations/layer15_attention_weight_min": -31.210172653198242,
      "activations/layer16_attention_weight_max": 33.58185577392578,
      "activations/layer16_attention_weight_min": -32.050018310546875,
      "activations/layer17_attention_weight_max": 53.05788040161133,
      "activations/layer17_attention_weight_min": -44.62321090698242,
      "activations/layer18_attention_weight_max": 47.2570915222168,
      "activations/layer18_attention_weight_min": -41.91422653198242,
      "activations/layer19_attention_weight_max": 23.16864585876465,
      "activations/layer19_attention_weight_min": -23.987844467163086,
      "activations/layer1_attention_weight_max": 16.878719329833984,
      "activations/layer1_attention_weight_min": -14.32447624206543,
      "activations/layer20_attention_weight_max": 22.061113357543945,
      "activations/layer20_attention_weight_min": -23.783946990966797,
      "activations/layer21_attention_weight_max": 34.678504943847656,
      "activations/layer21_attention_weight_min": -24.375635147094727,
      "activations/layer22_attention_weight_max": 30.887495040893555,
      "activations/layer22_attention_weight_min": -25.25616455078125,
      "activations/layer23_attention_weight_max": 35.01964569091797,
      "activations/layer23_attention_weight_min": -23.113073348999023,
      "activations/layer2_attention_weight_max": 33.417457580566406,
      "activations/layer2_attention_weight_min": -30.910114288330078,
      "activations/layer3_attention_weight_max": 87.97432708740234,
      "activations/layer3_attention_weight_min": -90.05490112304688,
      "activations/layer4_attention_weight_max": 85.70413208007812,
      "activations/layer4_attention_weight_min": -86.54195404052734,
      "activations/layer5_attention_weight_max": 65.877197265625,
      "activations/layer5_attention_weight_min": -73.95237731933594,
      "activations/layer6_attention_weight_max": 49.72202682495117,
      "activations/layer6_attention_weight_min": -51.41898727416992,
      "activations/layer7_attention_weight_max": 67.24935150146484,
      "activations/layer7_attention_weight_min": -62.679622650146484,
      "activations/layer8_attention_weight_max": 45.72383499145508,
      "activations/layer8_attention_weight_min": -51.4444694519043,
      "activations/layer9_attention_weight_max": 47.69839096069336,
      "activations/layer9_attention_weight_min": -46.30733108520508,
      "epoch": 17.31,
      "learning_rate": 3.878446969696969e-05,
      "loss": 2.7455,
      "step": 297850
    },
    {
      "activations/layer0_attention_weight_max": 15.394357681274414,
      "activations/layer0_attention_weight_min": -12.754655838012695,
      "activations/layer10_attention_weight_max": 33.95988845825195,
      "activations/layer10_attention_weight_min": -33.58821105957031,
      "activations/layer11_attention_weight_max": 32.92401123046875,
      "activations/layer11_attention_weight_min": -31.37904167175293,
      "activations/layer12_attention_weight_max": 19.604114532470703,
      "activations/layer12_attention_weight_min": -27.47958755493164,
      "activations/layer13_attention_weight_max": 37.421260833740234,
      "activations/layer13_attention_weight_min": -28.534517288208008,
      "activations/layer14_attention_weight_max": 33.235443115234375,
      "activations/layer14_attention_weight_min": -28.27639389038086,
      "activations/layer15_attention_weight_max": 33.41062545776367,
      "activations/layer15_attention_weight_min": -28.14194679260254,
      "activations/layer16_attention_weight_max": 34.10719680786133,
      "activations/layer16_attention_weight_min": -31.54931640625,
      "activations/layer17_attention_weight_max": 51.40298080444336,
      "activations/layer17_attention_weight_min": -44.511077880859375,
      "activations/layer18_attention_weight_max": 48.9896240234375,
      "activations/layer18_attention_weight_min": -40.89291000366211,
      "activations/layer19_attention_weight_max": 24.66725730895996,
      "activations/layer19_attention_weight_min": -22.55203628540039,
      "activations/layer1_attention_weight_max": 14.988744735717773,
      "activations/layer1_attention_weight_min": -13.917807579040527,
      "activations/layer20_attention_weight_max": 21.822904586791992,
      "activations/layer20_attention_weight_min": -23.01852035522461,
      "activations/layer21_attention_weight_max": 35.796329498291016,
      "activations/layer21_attention_weight_min": -20.77788734436035,
      "activations/layer22_attention_weight_max": 27.434581756591797,
      "activations/layer22_attention_weight_min": -24.90900993347168,
      "activations/layer23_attention_weight_max": 32.94521713256836,
      "activations/layer23_attention_weight_min": -24.73093032836914,
      "activations/layer2_attention_weight_max": 35.05341339111328,
      "activations/layer2_attention_weight_min": -33.84004211425781,
      "activations/layer3_attention_weight_max": 89.09359741210938,
      "activations/layer3_attention_weight_min": -91.55595397949219,
      "activations/layer4_attention_weight_max": 84.08733367919922,
      "activations/layer4_attention_weight_min": -81.31224822998047,
      "activations/layer5_attention_weight_max": 60.09925079345703,
      "activations/layer5_attention_weight_min": -79.09376525878906,
      "activations/layer6_attention_weight_max": 48.00758361816406,
      "activations/layer6_attention_weight_min": -47.74833679199219,
      "activations/layer7_attention_weight_max": 61.13631820678711,
      "activations/layer7_attention_weight_min": -58.9776496887207,
      "activations/layer8_attention_weight_max": 42.8787956237793,
      "activations/layer8_attention_weight_min": -45.7275276184082,
      "activations/layer9_attention_weight_max": 45.531646728515625,
      "activations/layer9_attention_weight_min": -44.933441162109375,
      "epoch": 17.31,
      "learning_rate": 3.8765530303030295e-05,
      "loss": 2.7415,
      "step": 297900
    },
    {
      "activations/layer0_attention_weight_max": 15.834312438964844,
      "activations/layer0_attention_weight_min": -13.149842262268066,
      "activations/layer10_attention_weight_max": 35.87089920043945,
      "activations/layer10_attention_weight_min": -34.6093864440918,
      "activations/layer11_attention_weight_max": 34.29176330566406,
      "activations/layer11_attention_weight_min": -32.76420593261719,
      "activations/layer12_attention_weight_max": 17.09562110900879,
      "activations/layer12_attention_weight_min": -26.780378341674805,
      "activations/layer13_attention_weight_max": 40.51935577392578,
      "activations/layer13_attention_weight_min": -29.635629653930664,
      "activations/layer14_attention_weight_max": 41.95591735839844,
      "activations/layer14_attention_weight_min": -28.806684494018555,
      "activations/layer15_attention_weight_max": 40.521453857421875,
      "activations/layer15_attention_weight_min": -28.63466453552246,
      "activations/layer16_attention_weight_max": 30.104509353637695,
      "activations/layer16_attention_weight_min": -29.257524490356445,
      "activations/layer17_attention_weight_max": 47.14950942993164,
      "activations/layer17_attention_weight_min": -42.64978790283203,
      "activations/layer18_attention_weight_max": 40.73775863647461,
      "activations/layer18_attention_weight_min": -37.94196701049805,
      "activations/layer19_attention_weight_max": 20.43492317199707,
      "activations/layer19_attention_weight_min": -23.664758682250977,
      "activations/layer1_attention_weight_max": 15.707664489746094,
      "activations/layer1_attention_weight_min": -14.867964744567871,
      "activations/layer20_attention_weight_max": 20.33677101135254,
      "activations/layer20_attention_weight_min": -23.04181671142578,
      "activations/layer21_attention_weight_max": 33.80373001098633,
      "activations/layer21_attention_weight_min": -21.596290588378906,
      "activations/layer22_attention_weight_max": 28.64919090270996,
      "activations/layer22_attention_weight_min": -23.95217514038086,
      "activations/layer23_attention_weight_max": 33.49078369140625,
      "activations/layer23_attention_weight_min": -22.286800384521484,
      "activations/layer2_attention_weight_max": 32.7391357421875,
      "activations/layer2_attention_weight_min": -29.744054794311523,
      "activations/layer3_attention_weight_max": 88.00828552246094,
      "activations/layer3_attention_weight_min": -87.78423309326172,
      "activations/layer4_attention_weight_max": 86.17453002929688,
      "activations/layer4_attention_weight_min": -83.4457015991211,
      "activations/layer5_attention_weight_max": 62.22442626953125,
      "activations/layer5_attention_weight_min": -69.02579498291016,
      "activations/layer6_attention_weight_max": 48.709468841552734,
      "activations/layer6_attention_weight_min": -48.692405700683594,
      "activations/layer7_attention_weight_max": 64.49668884277344,
      "activations/layer7_attention_weight_min": -62.46296691894531,
      "activations/layer8_attention_weight_max": 45.85145568847656,
      "activations/layer8_attention_weight_min": -43.55781936645508,
      "activations/layer9_attention_weight_max": 49.43312454223633,
      "activations/layer9_attention_weight_min": -46.79959487915039,
      "epoch": 17.31,
      "learning_rate": 3.874659090909091e-05,
      "loss": 2.7319,
      "step": 297950
    },
    {
      "activations/layer0_attention_weight_max": 15.570225715637207,
      "activations/layer0_attention_weight_min": -13.265862464904785,
      "activations/layer10_attention_weight_max": 33.94540023803711,
      "activations/layer10_attention_weight_min": -34.5917854309082,
      "activations/layer11_attention_weight_max": 32.08441925048828,
      "activations/layer11_attention_weight_min": -33.236167907714844,
      "activations/layer12_attention_weight_max": 19.635635375976562,
      "activations/layer12_attention_weight_min": -29.16065788269043,
      "activations/layer13_attention_weight_max": 36.74254608154297,
      "activations/layer13_attention_weight_min": -26.29311752319336,
      "activations/layer14_attention_weight_max": 37.732757568359375,
      "activations/layer14_attention_weight_min": -30.221689224243164,
      "activations/layer15_attention_weight_max": 34.35395812988281,
      "activations/layer15_attention_weight_min": -31.115325927734375,
      "activations/layer16_attention_weight_max": 35.03569412231445,
      "activations/layer16_attention_weight_min": -32.03673553466797,
      "activations/layer17_attention_weight_max": 51.70595169067383,
      "activations/layer17_attention_weight_min": -46.80959701538086,
      "activations/layer18_attention_weight_max": 50.09345626831055,
      "activations/layer18_attention_weight_min": -40.61756896972656,
      "activations/layer19_attention_weight_max": 27.409835815429688,
      "activations/layer19_attention_weight_min": -25.940561294555664,
      "activations/layer1_attention_weight_max": 16.163373947143555,
      "activations/layer1_attention_weight_min": -13.756046295166016,
      "activations/layer20_attention_weight_max": 26.22734832763672,
      "activations/layer20_attention_weight_min": -26.33754539489746,
      "activations/layer21_attention_weight_max": 37.873451232910156,
      "activations/layer21_attention_weight_min": -23.16718292236328,
      "activations/layer22_attention_weight_max": 31.192502975463867,
      "activations/layer22_attention_weight_min": -26.068315505981445,
      "activations/layer23_attention_weight_max": 40.06859588623047,
      "activations/layer23_attention_weight_min": -22.609272003173828,
      "activations/layer2_attention_weight_max": 32.544219970703125,
      "activations/layer2_attention_weight_min": -30.214847564697266,
      "activations/layer3_attention_weight_max": 84.92799377441406,
      "activations/layer3_attention_weight_min": -86.53002166748047,
      "activations/layer4_attention_weight_max": 86.82272338867188,
      "activations/layer4_attention_weight_min": -88.07240295410156,
      "activations/layer5_attention_weight_max": 64.35798645019531,
      "activations/layer5_attention_weight_min": -70.10580444335938,
      "activations/layer6_attention_weight_max": 46.58995819091797,
      "activations/layer6_attention_weight_min": -49.79311752319336,
      "activations/layer7_attention_weight_max": 65.54023742675781,
      "activations/layer7_attention_weight_min": -67.92717742919922,
      "activations/layer8_attention_weight_max": 45.88338088989258,
      "activations/layer8_attention_weight_min": -48.22404861450195,
      "activations/layer9_attention_weight_max": 47.54154586791992,
      "activations/layer9_attention_weight_min": -46.06576919555664,
      "epoch": 17.32,
      "learning_rate": 3.872765151515151e-05,
      "loss": 2.7383,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4115,
      "eval_samples_per_second": 510.493,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4115,
      "eval_openwebtext_samples_per_second": 510.493,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_wikitext_loss": 2.884765625,
      "eval_wikitext_ppl": 17.89937193856263,
      "eval_wikitext_runtime": 1.924,
      "eval_wikitext_samples_per_second": 237.01,
      "step": 298000
    },
    {
      "epoch": 17.32,
      "eval_lambada_loss": 2.49609375,
      "eval_lambada_ppl": 12.134998917818166,
      "eval_lambada_runtime": 9.5166,
      "eval_lambada_samples_per_second": 511.632,
      "step": 298000
    },
    {
      "activations/layer0_attention_weight_max": 15.836786270141602,
      "activations/layer0_attention_weight_min": -13.326515197753906,
      "activations/layer10_attention_weight_max": 34.66511154174805,
      "activations/layer10_attention_weight_min": -37.063507080078125,
      "activations/layer11_attention_weight_max": 31.150619506835938,
      "activations/layer11_attention_weight_min": -33.448692321777344,
      "activations/layer12_attention_weight_max": 18.99009895324707,
      "activations/layer12_attention_weight_min": -25.68830680847168,
      "activations/layer13_attention_weight_max": 36.6650390625,
      "activations/layer13_attention_weight_min": -29.8066463470459,
      "activations/layer14_attention_weight_max": 37.3027229309082,
      "activations/layer14_attention_weight_min": -33.02559280395508,
      "activations/layer15_attention_weight_max": 36.163604736328125,
      "activations/layer15_attention_weight_min": -31.852882385253906,
      "activations/layer16_attention_weight_max": 37.355125427246094,
      "activations/layer16_attention_weight_min": -35.81309127807617,
      "activations/layer17_attention_weight_max": 54.506656646728516,
      "activations/layer17_attention_weight_min": -48.61733627319336,
      "activations/layer18_attention_weight_max": 50.104095458984375,
      "activations/layer18_attention_weight_min": -40.101234436035156,
      "activations/layer19_attention_weight_max": 26.390277862548828,
      "activations/layer19_attention_weight_min": -23.572925567626953,
      "activations/layer1_attention_weight_max": 17.504026412963867,
      "activations/layer1_attention_weight_min": -14.628353118896484,
      "activations/layer20_attention_weight_max": 25.02111053466797,
      "activations/layer20_attention_weight_min": -22.91777801513672,
      "activations/layer21_attention_weight_max": 37.43430709838867,
      "activations/layer21_attention_weight_min": -20.939964294433594,
      "activations/layer22_attention_weight_max": 29.478092193603516,
      "activations/layer22_attention_weight_min": -25.7386474609375,
      "activations/layer23_attention_weight_max": 36.40555953979492,
      "activations/layer23_attention_weight_min": -23.447065353393555,
      "activations/layer2_attention_weight_max": 33.721012115478516,
      "activations/layer2_attention_weight_min": -32.15996170043945,
      "activations/layer3_attention_weight_max": 89.31763458251953,
      "activations/layer3_attention_weight_min": -95.58415985107422,
      "activations/layer4_attention_weight_max": 84.38204193115234,
      "activations/layer4_attention_weight_min": -85.00725555419922,
      "activations/layer5_attention_weight_max": 65.38813781738281,
      "activations/layer5_attention_weight_min": -70.14330291748047,
      "activations/layer6_attention_weight_max": 49.94269943237305,
      "activations/layer6_attention_weight_min": -49.57072067260742,
      "activations/layer7_attention_weight_max": 61.988731384277344,
      "activations/layer7_attention_weight_min": -59.734989166259766,
      "activations/layer8_attention_weight_max": 43.15691375732422,
      "activations/layer8_attention_weight_min": -49.82600021362305,
      "activations/layer9_attention_weight_max": 43.57120895385742,
      "activations/layer9_attention_weight_min": -43.74528121948242,
      "epoch": 17.32,
      "learning_rate": 3.870871212121211e-05,
      "loss": 2.7525,
      "step": 298050
    },
    {
      "activations/layer0_attention_weight_max": 15.75445556640625,
      "activations/layer0_attention_weight_min": -12.784639358520508,
      "activations/layer10_attention_weight_max": 33.59422302246094,
      "activations/layer10_attention_weight_min": -34.24452209472656,
      "activations/layer11_attention_weight_max": 31.477888107299805,
      "activations/layer11_attention_weight_min": -33.501399993896484,
      "activations/layer12_attention_weight_max": 18.815488815307617,
      "activations/layer12_attention_weight_min": -31.4627628326416,
      "activations/layer13_attention_weight_max": 36.48234939575195,
      "activations/layer13_attention_weight_min": -28.53313446044922,
      "activations/layer14_attention_weight_max": 34.44697952270508,
      "activations/layer14_attention_weight_min": -31.793214797973633,
      "activations/layer15_attention_weight_max": 32.38944625854492,
      "activations/layer15_attention_weight_min": -31.1160831451416,
      "activations/layer16_attention_weight_max": 32.85023880004883,
      "activations/layer16_attention_weight_min": -33.86099624633789,
      "activations/layer17_attention_weight_max": 50.26813507080078,
      "activations/layer17_attention_weight_min": -48.1767463684082,
      "activations/layer18_attention_weight_max": 47.80215835571289,
      "activations/layer18_attention_weight_min": -41.54140853881836,
      "activations/layer19_attention_weight_max": 22.306228637695312,
      "activations/layer19_attention_weight_min": -24.607885360717773,
      "activations/layer1_attention_weight_max": 16.564733505249023,
      "activations/layer1_attention_weight_min": -15.019118309020996,
      "activations/layer20_attention_weight_max": 22.615007400512695,
      "activations/layer20_attention_weight_min": -23.23031234741211,
      "activations/layer21_attention_weight_max": 35.11848831176758,
      "activations/layer21_attention_weight_min": -23.495935440063477,
      "activations/layer22_attention_weight_max": 30.643497467041016,
      "activations/layer22_attention_weight_min": -25.938264846801758,
      "activations/layer23_attention_weight_max": 36.215267181396484,
      "activations/layer23_attention_weight_min": -25.134960174560547,
      "activations/layer2_attention_weight_max": 34.36542892456055,
      "activations/layer2_attention_weight_min": -32.033241271972656,
      "activations/layer3_attention_weight_max": 97.49858856201172,
      "activations/layer3_attention_weight_min": -94.45125579833984,
      "activations/layer4_attention_weight_max": 86.3249282836914,
      "activations/layer4_attention_weight_min": -82.21411895751953,
      "activations/layer5_attention_weight_max": 65.22538757324219,
      "activations/layer5_attention_weight_min": -77.93836975097656,
      "activations/layer6_attention_weight_max": 47.76818084716797,
      "activations/layer6_attention_weight_min": -49.63644027709961,
      "activations/layer7_attention_weight_max": 65.82938385009766,
      "activations/layer7_attention_weight_min": -63.1551513671875,
      "activations/layer8_attention_weight_max": 43.8006477355957,
      "activations/layer8_attention_weight_min": -45.13363265991211,
      "activations/layer9_attention_weight_max": 43.56863021850586,
      "activations/layer9_attention_weight_min": -47.76692199707031,
      "epoch": 17.32,
      "learning_rate": 3.868977272727273e-05,
      "loss": 2.7214,
      "step": 298100
    },
    {
      "activations/layer0_attention_weight_max": 15.557190895080566,
      "activations/layer0_attention_weight_min": -12.808316230773926,
      "activations/layer10_attention_weight_max": 37.65078353881836,
      "activations/layer10_attention_weight_min": -36.77125930786133,
      "activations/layer11_attention_weight_max": 33.30772399902344,
      "activations/layer11_attention_weight_min": -34.07520294189453,
      "activations/layer12_attention_weight_max": 19.957929611206055,
      "activations/layer12_attention_weight_min": -24.057861328125,
      "activations/layer13_attention_weight_max": 34.21148681640625,
      "activations/layer13_attention_weight_min": -28.31439971923828,
      "activations/layer14_attention_weight_max": 37.21390914916992,
      "activations/layer14_attention_weight_min": -30.737592697143555,
      "activations/layer15_attention_weight_max": 31.623132705688477,
      "activations/layer15_attention_weight_min": -30.804969787597656,
      "activations/layer16_attention_weight_max": 32.674373626708984,
      "activations/layer16_attention_weight_min": -31.529207229614258,
      "activations/layer17_attention_weight_max": 49.24333572387695,
      "activations/layer17_attention_weight_min": -44.11444854736328,
      "activations/layer18_attention_weight_max": 47.98713302612305,
      "activations/layer18_attention_weight_min": -39.21660232543945,
      "activations/layer19_attention_weight_max": 24.39400291442871,
      "activations/layer19_attention_weight_min": -24.20685386657715,
      "activations/layer1_attention_weight_max": 16.42906951904297,
      "activations/layer1_attention_weight_min": -13.978095054626465,
      "activations/layer20_attention_weight_max": 25.03965950012207,
      "activations/layer20_attention_weight_min": -24.078046798706055,
      "activations/layer21_attention_weight_max": 42.6610221862793,
      "activations/layer21_attention_weight_min": -27.28356170654297,
      "activations/layer22_attention_weight_max": 29.45093536376953,
      "activations/layer22_attention_weight_min": -27.1612491607666,
      "activations/layer23_attention_weight_max": 40.819618225097656,
      "activations/layer23_attention_weight_min": -26.17218589782715,
      "activations/layer2_attention_weight_max": 33.359893798828125,
      "activations/layer2_attention_weight_min": -30.900999069213867,
      "activations/layer3_attention_weight_max": 89.25064849853516,
      "activations/layer3_attention_weight_min": -90.92084503173828,
      "activations/layer4_attention_weight_max": 91.1054458618164,
      "activations/layer4_attention_weight_min": -87.37189483642578,
      "activations/layer5_attention_weight_max": 66.65441131591797,
      "activations/layer5_attention_weight_min": -71.85596466064453,
      "activations/layer6_attention_weight_max": 48.611610412597656,
      "activations/layer6_attention_weight_min": -49.655860900878906,
      "activations/layer7_attention_weight_max": 65.94713592529297,
      "activations/layer7_attention_weight_min": -64.67040252685547,
      "activations/layer8_attention_weight_max": 44.63969421386719,
      "activations/layer8_attention_weight_min": -46.256690979003906,
      "activations/layer9_attention_weight_max": 46.1965446472168,
      "activations/layer9_attention_weight_min": -44.34959030151367,
      "epoch": 17.32,
      "learning_rate": 3.867083333333333e-05,
      "loss": 2.734,
      "step": 298150
    },
    {
      "activations/layer0_attention_weight_max": 16.785036087036133,
      "activations/layer0_attention_weight_min": -13.016249656677246,
      "activations/layer10_attention_weight_max": 38.209651947021484,
      "activations/layer10_attention_weight_min": -36.55142593383789,
      "activations/layer11_attention_weight_max": 37.588626861572266,
      "activations/layer11_attention_weight_min": -34.25315475463867,
      "activations/layer12_attention_weight_max": 21.64645767211914,
      "activations/layer12_attention_weight_min": -23.0341854095459,
      "activations/layer13_attention_weight_max": 34.86521530151367,
      "activations/layer13_attention_weight_min": -29.74789047241211,
      "activations/layer14_attention_weight_max": 34.45471954345703,
      "activations/layer14_attention_weight_min": -30.122730255126953,
      "activations/layer15_attention_weight_max": 33.84247970581055,
      "activations/layer15_attention_weight_min": -30.355388641357422,
      "activations/layer16_attention_weight_max": 34.32771682739258,
      "activations/layer16_attention_weight_min": -30.249282836914062,
      "activations/layer17_attention_weight_max": 54.52839660644531,
      "activations/layer17_attention_weight_min": -42.624595642089844,
      "activations/layer18_attention_weight_max": 50.03042984008789,
      "activations/layer18_attention_weight_min": -40.28377914428711,
      "activations/layer19_attention_weight_max": 23.320743560791016,
      "activations/layer19_attention_weight_min": -22.27510643005371,
      "activations/layer1_attention_weight_max": 15.762322425842285,
      "activations/layer1_attention_weight_min": -14.048690795898438,
      "activations/layer20_attention_weight_max": 22.090543746948242,
      "activations/layer20_attention_weight_min": -22.825061798095703,
      "activations/layer21_attention_weight_max": 34.82664108276367,
      "activations/layer21_attention_weight_min": -23.91985511779785,
      "activations/layer22_attention_weight_max": 30.95502471923828,
      "activations/layer22_attention_weight_min": -25.72514533996582,
      "activations/layer23_attention_weight_max": 39.094337463378906,
      "activations/layer23_attention_weight_min": -27.025493621826172,
      "activations/layer2_attention_weight_max": 30.831584930419922,
      "activations/layer2_attention_weight_min": -28.935714721679688,
      "activations/layer3_attention_weight_max": 88.08775329589844,
      "activations/layer3_attention_weight_min": -90.2908706665039,
      "activations/layer4_attention_weight_max": 87.11809539794922,
      "activations/layer4_attention_weight_min": -87.25531768798828,
      "activations/layer5_attention_weight_max": 62.70756149291992,
      "activations/layer5_attention_weight_min": -71.82415771484375,
      "activations/layer6_attention_weight_max": 49.80150604248047,
      "activations/layer6_attention_weight_min": -49.1638069152832,
      "activations/layer7_attention_weight_max": 62.92652130126953,
      "activations/layer7_attention_weight_min": -61.43718338012695,
      "activations/layer8_attention_weight_max": 47.17892837524414,
      "activations/layer8_attention_weight_min": -45.77483367919922,
      "activations/layer9_attention_weight_max": 45.21699142456055,
      "activations/layer9_attention_weight_min": -44.44204330444336,
      "epoch": 17.33,
      "learning_rate": 3.865189393939394e-05,
      "loss": 2.7264,
      "step": 298200
    },
    {
      "activations/layer0_attention_weight_max": 15.654255867004395,
      "activations/layer0_attention_weight_min": -12.938677787780762,
      "activations/layer10_attention_weight_max": 35.40553665161133,
      "activations/layer10_attention_weight_min": -35.82261276245117,
      "activations/layer11_attention_weight_max": 34.06218338012695,
      "activations/layer11_attention_weight_min": -33.004852294921875,
      "activations/layer12_attention_weight_max": 22.51981544494629,
      "activations/layer12_attention_weight_min": -26.321937561035156,
      "activations/layer13_attention_weight_max": 34.305294036865234,
      "activations/layer13_attention_weight_min": -28.685739517211914,
      "activations/layer14_attention_weight_max": 36.78482437133789,
      "activations/layer14_attention_weight_min": -31.940879821777344,
      "activations/layer15_attention_weight_max": 33.29307556152344,
      "activations/layer15_attention_weight_min": -30.95419692993164,
      "activations/layer16_attention_weight_max": 34.878055572509766,
      "activations/layer16_attention_weight_min": -34.834041595458984,
      "activations/layer17_attention_weight_max": 52.27512741088867,
      "activations/layer17_attention_weight_min": -47.613033294677734,
      "activations/layer18_attention_weight_max": 47.71385955810547,
      "activations/layer18_attention_weight_min": -40.18548583984375,
      "activations/layer19_attention_weight_max": 25.626789093017578,
      "activations/layer19_attention_weight_min": -22.489505767822266,
      "activations/layer1_attention_weight_max": 15.251543998718262,
      "activations/layer1_attention_weight_min": -13.582923889160156,
      "activations/layer20_attention_weight_max": 21.672513961791992,
      "activations/layer20_attention_weight_min": -22.075273513793945,
      "activations/layer21_attention_weight_max": 43.254852294921875,
      "activations/layer21_attention_weight_min": -24.743736267089844,
      "activations/layer22_attention_weight_max": 32.67121887207031,
      "activations/layer22_attention_weight_min": -29.138099670410156,
      "activations/layer23_attention_weight_max": 39.900718688964844,
      "activations/layer23_attention_weight_min": -24.098722457885742,
      "activations/layer2_attention_weight_max": 31.85738182067871,
      "activations/layer2_attention_weight_min": -31.787755966186523,
      "activations/layer3_attention_weight_max": 92.53520965576172,
      "activations/layer3_attention_weight_min": -96.93558502197266,
      "activations/layer4_attention_weight_max": 87.51883697509766,
      "activations/layer4_attention_weight_min": -88.22177124023438,
      "activations/layer5_attention_weight_max": 61.80518341064453,
      "activations/layer5_attention_weight_min": -72.73118591308594,
      "activations/layer6_attention_weight_max": 50.89850616455078,
      "activations/layer6_attention_weight_min": -50.36315155029297,
      "activations/layer7_attention_weight_max": 62.51982498168945,
      "activations/layer7_attention_weight_min": -63.71457290649414,
      "activations/layer8_attention_weight_max": 44.99298095703125,
      "activations/layer8_attention_weight_min": -46.843631744384766,
      "activations/layer9_attention_weight_max": 42.0812873840332,
      "activations/layer9_attention_weight_min": -43.37824630737305,
      "epoch": 17.33,
      "learning_rate": 3.863295454545454e-05,
      "loss": 2.7481,
      "step": 298250
    },
    {
      "activations/layer0_attention_weight_max": 16.159820556640625,
      "activations/layer0_attention_weight_min": -12.971465110778809,
      "activations/layer10_attention_weight_max": 35.755374908447266,
      "activations/layer10_attention_weight_min": -34.659908294677734,
      "activations/layer11_attention_weight_max": 31.473386764526367,
      "activations/layer11_attention_weight_min": -33.38520050048828,
      "activations/layer12_attention_weight_max": 18.06553840637207,
      "activations/layer12_attention_weight_min": -23.031394958496094,
      "activations/layer13_attention_weight_max": 36.29387283325195,
      "activations/layer13_attention_weight_min": -27.3061466217041,
      "activations/layer14_attention_weight_max": 35.648406982421875,
      "activations/layer14_attention_weight_min": -28.443862915039062,
      "activations/layer15_attention_weight_max": 32.132789611816406,
      "activations/layer15_attention_weight_min": -29.175735473632812,
      "activations/layer16_attention_weight_max": 32.73774719238281,
      "activations/layer16_attention_weight_min": -31.05707550048828,
      "activations/layer17_attention_weight_max": 50.75132369995117,
      "activations/layer17_attention_weight_min": -44.20076370239258,
      "activations/layer18_attention_weight_max": 49.08123779296875,
      "activations/layer18_attention_weight_min": -38.72407150268555,
      "activations/layer19_attention_weight_max": 26.36452293395996,
      "activations/layer19_attention_weight_min": -23.068275451660156,
      "activations/layer1_attention_weight_max": 17.55447769165039,
      "activations/layer1_attention_weight_min": -14.080109596252441,
      "activations/layer20_attention_weight_max": 22.437313079833984,
      "activations/layer20_attention_weight_min": -22.63138771057129,
      "activations/layer21_attention_weight_max": 32.58734893798828,
      "activations/layer21_attention_weight_min": -21.07567024230957,
      "activations/layer22_attention_weight_max": 27.908899307250977,
      "activations/layer22_attention_weight_min": -26.759063720703125,
      "activations/layer23_attention_weight_max": 35.98509216308594,
      "activations/layer23_attention_weight_min": -25.204626083374023,
      "activations/layer2_attention_weight_max": 31.794052124023438,
      "activations/layer2_attention_weight_min": -30.996013641357422,
      "activations/layer3_attention_weight_max": 90.60155487060547,
      "activations/layer3_attention_weight_min": -88.74908447265625,
      "activations/layer4_attention_weight_max": 85.75175476074219,
      "activations/layer4_attention_weight_min": -86.29806518554688,
      "activations/layer5_attention_weight_max": 65.79013061523438,
      "activations/layer5_attention_weight_min": -74.7182388305664,
      "activations/layer6_attention_weight_max": 48.57795715332031,
      "activations/layer6_attention_weight_min": -49.78944778442383,
      "activations/layer7_attention_weight_max": 66.23078155517578,
      "activations/layer7_attention_weight_min": -64.01362609863281,
      "activations/layer8_attention_weight_max": 48.06816482543945,
      "activations/layer8_attention_weight_min": -48.98075866699219,
      "activations/layer9_attention_weight_max": 45.6011848449707,
      "activations/layer9_attention_weight_min": -45.56052780151367,
      "epoch": 17.33,
      "learning_rate": 3.8614015151515154e-05,
      "loss": 2.7518,
      "step": 298300
    },
    {
      "activations/layer0_attention_weight_max": 15.66617488861084,
      "activations/layer0_attention_weight_min": -13.11478328704834,
      "activations/layer10_attention_weight_max": 38.35877227783203,
      "activations/layer10_attention_weight_min": -36.93507766723633,
      "activations/layer11_attention_weight_max": 33.66324234008789,
      "activations/layer11_attention_weight_min": -35.42702865600586,
      "activations/layer12_attention_weight_max": 18.42069435119629,
      "activations/layer12_attention_weight_min": -23.49802589416504,
      "activations/layer13_attention_weight_max": 32.110618591308594,
      "activations/layer13_attention_weight_min": -28.01468276977539,
      "activations/layer14_attention_weight_max": 36.80585861206055,
      "activations/layer14_attention_weight_min": -32.577476501464844,
      "activations/layer15_attention_weight_max": 34.0313606262207,
      "activations/layer15_attention_weight_min": -31.37851905822754,
      "activations/layer16_attention_weight_max": 36.512481689453125,
      "activations/layer16_attention_weight_min": -36.045406341552734,
      "activations/layer17_attention_weight_max": 51.29977798461914,
      "activations/layer17_attention_weight_min": -48.920082092285156,
      "activations/layer18_attention_weight_max": 46.22697067260742,
      "activations/layer18_attention_weight_min": -41.11574935913086,
      "activations/layer19_attention_weight_max": 23.939247131347656,
      "activations/layer19_attention_weight_min": -23.442380905151367,
      "activations/layer1_attention_weight_max": 15.927709579467773,
      "activations/layer1_attention_weight_min": -14.02718448638916,
      "activations/layer20_attention_weight_max": 22.475114822387695,
      "activations/layer20_attention_weight_min": -23.997241973876953,
      "activations/layer21_attention_weight_max": 41.125152587890625,
      "activations/layer21_attention_weight_min": -23.862926483154297,
      "activations/layer22_attention_weight_max": 27.07793426513672,
      "activations/layer22_attention_weight_min": -26.06822395324707,
      "activations/layer23_attention_weight_max": 36.833045959472656,
      "activations/layer23_attention_weight_min": -24.220794677734375,
      "activations/layer2_attention_weight_max": 32.107177734375,
      "activations/layer2_attention_weight_min": -30.52414321899414,
      "activations/layer3_attention_weight_max": 91.15621185302734,
      "activations/layer3_attention_weight_min": -94.58573150634766,
      "activations/layer4_attention_weight_max": 88.17473602294922,
      "activations/layer4_attention_weight_min": -87.39996337890625,
      "activations/layer5_attention_weight_max": 61.69525146484375,
      "activations/layer5_attention_weight_min": -75.33660125732422,
      "activations/layer6_attention_weight_max": 48.97993850708008,
      "activations/layer6_attention_weight_min": -50.226905822753906,
      "activations/layer7_attention_weight_max": 66.3777084350586,
      "activations/layer7_attention_weight_min": -65.85845947265625,
      "activations/layer8_attention_weight_max": 48.4088134765625,
      "activations/layer8_attention_weight_min": -49.59547805786133,
      "activations/layer9_attention_weight_max": 49.310115814208984,
      "activations/layer9_attention_weight_min": -45.54405212402344,
      "epoch": 17.34,
      "learning_rate": 3.8595075757575755e-05,
      "loss": 2.7374,
      "step": 298350
    },
    {
      "activations/layer0_attention_weight_max": 16.176570892333984,
      "activations/layer0_attention_weight_min": -13.020794868469238,
      "activations/layer10_attention_weight_max": 33.054931640625,
      "activations/layer10_attention_weight_min": -32.4942626953125,
      "activations/layer11_attention_weight_max": 30.503692626953125,
      "activations/layer11_attention_weight_min": -32.84039306640625,
      "activations/layer12_attention_weight_max": 18.43434715270996,
      "activations/layer12_attention_weight_min": -23.10138702392578,
      "activations/layer13_attention_weight_max": 31.626169204711914,
      "activations/layer13_attention_weight_min": -27.97614288330078,
      "activations/layer14_attention_weight_max": 33.330177307128906,
      "activations/layer14_attention_weight_min": -30.26495361328125,
      "activations/layer15_attention_weight_max": 32.7453727722168,
      "activations/layer15_attention_weight_min": -29.969987869262695,
      "activations/layer16_attention_weight_max": 33.78071212768555,
      "activations/layer16_attention_weight_min": -33.130977630615234,
      "activations/layer17_attention_weight_max": 50.97980880737305,
      "activations/layer17_attention_weight_min": -44.291255950927734,
      "activations/layer18_attention_weight_max": 45.402198791503906,
      "activations/layer18_attention_weight_min": -38.627750396728516,
      "activations/layer19_attention_weight_max": 22.85523223876953,
      "activations/layer19_attention_weight_min": -24.792762756347656,
      "activations/layer1_attention_weight_max": 15.39169979095459,
      "activations/layer1_attention_weight_min": -13.491720199584961,
      "activations/layer20_attention_weight_max": 24.620027542114258,
      "activations/layer20_attention_weight_min": -21.86003875732422,
      "activations/layer21_attention_weight_max": 33.757118225097656,
      "activations/layer21_attention_weight_min": -21.094980239868164,
      "activations/layer22_attention_weight_max": 30.263521194458008,
      "activations/layer22_attention_weight_min": -25.802690505981445,
      "activations/layer23_attention_weight_max": 37.56010055541992,
      "activations/layer23_attention_weight_min": -24.734127044677734,
      "activations/layer2_attention_weight_max": 29.876800537109375,
      "activations/layer2_attention_weight_min": -29.522003173828125,
      "activations/layer3_attention_weight_max": 86.26067352294922,
      "activations/layer3_attention_weight_min": -88.22489929199219,
      "activations/layer4_attention_weight_max": 85.34964752197266,
      "activations/layer4_attention_weight_min": -83.85116577148438,
      "activations/layer5_attention_weight_max": 63.011512756347656,
      "activations/layer5_attention_weight_min": -74.91563415527344,
      "activations/layer6_attention_weight_max": 47.54180145263672,
      "activations/layer6_attention_weight_min": -48.59440994262695,
      "activations/layer7_attention_weight_max": 60.92689514160156,
      "activations/layer7_attention_weight_min": -62.081268310546875,
      "activations/layer8_attention_weight_max": 41.863948822021484,
      "activations/layer8_attention_weight_min": -44.29679870605469,
      "activations/layer9_attention_weight_max": 40.08420181274414,
      "activations/layer9_attention_weight_min": -42.887447357177734,
      "epoch": 17.34,
      "learning_rate": 3.857613636363636e-05,
      "loss": 2.7199,
      "step": 298400
    },
    {
      "activations/layer0_attention_weight_max": 16.224950790405273,
      "activations/layer0_attention_weight_min": -12.755563735961914,
      "activations/layer10_attention_weight_max": 34.70479965209961,
      "activations/layer10_attention_weight_min": -37.28247833251953,
      "activations/layer11_attention_weight_max": 34.55594253540039,
      "activations/layer11_attention_weight_min": -34.94813537597656,
      "activations/layer12_attention_weight_max": 17.64780616760254,
      "activations/layer12_attention_weight_min": -35.996089935302734,
      "activations/layer13_attention_weight_max": 41.42753982543945,
      "activations/layer13_attention_weight_min": -30.727313995361328,
      "activations/layer14_attention_weight_max": 34.8239860534668,
      "activations/layer14_attention_weight_min": -30.315902709960938,
      "activations/layer15_attention_weight_max": 34.18272399902344,
      "activations/layer15_attention_weight_min": -29.441234588623047,
      "activations/layer16_attention_weight_max": 34.84608840942383,
      "activations/layer16_attention_weight_min": -33.63955307006836,
      "activations/layer17_attention_weight_max": 51.612754821777344,
      "activations/layer17_attention_weight_min": -44.445743560791016,
      "activations/layer18_attention_weight_max": 46.75398635864258,
      "activations/layer18_attention_weight_min": -39.693389892578125,
      "activations/layer19_attention_weight_max": 22.537967681884766,
      "activations/layer19_attention_weight_min": -20.80018424987793,
      "activations/layer1_attention_weight_max": 17.123830795288086,
      "activations/layer1_attention_weight_min": -13.72632884979248,
      "activations/layer20_attention_weight_max": 21.90031623840332,
      "activations/layer20_attention_weight_min": -20.65634536743164,
      "activations/layer21_attention_weight_max": 40.66362380981445,
      "activations/layer21_attention_weight_min": -19.348796844482422,
      "activations/layer22_attention_weight_max": 28.76434326171875,
      "activations/layer22_attention_weight_min": -26.99408531188965,
      "activations/layer23_attention_weight_max": 35.84172821044922,
      "activations/layer23_attention_weight_min": -23.94414520263672,
      "activations/layer2_attention_weight_max": 30.70376205444336,
      "activations/layer2_attention_weight_min": -29.514949798583984,
      "activations/layer3_attention_weight_max": 90.5699462890625,
      "activations/layer3_attention_weight_min": -88.71428680419922,
      "activations/layer4_attention_weight_max": 88.47725677490234,
      "activations/layer4_attention_weight_min": -86.58678436279297,
      "activations/layer5_attention_weight_max": 63.98994064331055,
      "activations/layer5_attention_weight_min": -69.55697631835938,
      "activations/layer6_attention_weight_max": 49.91581726074219,
      "activations/layer6_attention_weight_min": -49.277320861816406,
      "activations/layer7_attention_weight_max": 71.19388580322266,
      "activations/layer7_attention_weight_min": -66.29049682617188,
      "activations/layer8_attention_weight_max": 47.15439987182617,
      "activations/layer8_attention_weight_min": -47.41703414916992,
      "activations/layer9_attention_weight_max": 54.785736083984375,
      "activations/layer9_attention_weight_min": -47.32120895385742,
      "epoch": 17.34,
      "learning_rate": 3.8557196969696965e-05,
      "loss": 2.7421,
      "step": 298450
    },
    {
      "activations/layer0_attention_weight_max": 15.425507545471191,
      "activations/layer0_attention_weight_min": -13.144487380981445,
      "activations/layer10_attention_weight_max": 33.036746978759766,
      "activations/layer10_attention_weight_min": -35.469974517822266,
      "activations/layer11_attention_weight_max": 31.831085205078125,
      "activations/layer11_attention_weight_min": -32.968963623046875,
      "activations/layer12_attention_weight_max": 18.86046600341797,
      "activations/layer12_attention_weight_min": -26.339984893798828,
      "activations/layer13_attention_weight_max": 34.59508514404297,
      "activations/layer13_attention_weight_min": -28.369359970092773,
      "activations/layer14_attention_weight_max": 32.870155334472656,
      "activations/layer14_attention_weight_min": -31.857316970825195,
      "activations/layer15_attention_weight_max": 30.77394676208496,
      "activations/layer15_attention_weight_min": -30.788009643554688,
      "activations/layer16_attention_weight_max": 33.44721221923828,
      "activations/layer16_attention_weight_min": -31.495153427124023,
      "activations/layer17_attention_weight_max": 52.552886962890625,
      "activations/layer17_attention_weight_min": -45.02583694458008,
      "activations/layer18_attention_weight_max": 47.55439376831055,
      "activations/layer18_attention_weight_min": -44.66257858276367,
      "activations/layer19_attention_weight_max": 23.429121017456055,
      "activations/layer19_attention_weight_min": -23.873933792114258,
      "activations/layer1_attention_weight_max": 17.46819305419922,
      "activations/layer1_attention_weight_min": -12.550948143005371,
      "activations/layer20_attention_weight_max": 21.535429000854492,
      "activations/layer20_attention_weight_min": -23.237531661987305,
      "activations/layer21_attention_weight_max": 39.97611618041992,
      "activations/layer21_attention_weight_min": -28.21805191040039,
      "activations/layer22_attention_weight_max": 28.847126007080078,
      "activations/layer22_attention_weight_min": -27.69676971435547,
      "activations/layer23_attention_weight_max": 34.607154846191406,
      "activations/layer23_attention_weight_min": -27.794513702392578,
      "activations/layer2_attention_weight_max": 26.740951538085938,
      "activations/layer2_attention_weight_min": -26.24201774597168,
      "activations/layer3_attention_weight_max": 78.68465423583984,
      "activations/layer3_attention_weight_min": -86.0928955078125,
      "activations/layer4_attention_weight_max": 79.77776336669922,
      "activations/layer4_attention_weight_min": -80.21656799316406,
      "activations/layer5_attention_weight_max": 64.2843246459961,
      "activations/layer5_attention_weight_min": -74.76852416992188,
      "activations/layer6_attention_weight_max": 45.68973159790039,
      "activations/layer6_attention_weight_min": -48.1652946472168,
      "activations/layer7_attention_weight_max": 64.15352630615234,
      "activations/layer7_attention_weight_min": -62.1707878112793,
      "activations/layer8_attention_weight_max": 44.71064376831055,
      "activations/layer8_attention_weight_min": -46.549346923828125,
      "activations/layer9_attention_weight_max": 42.134193420410156,
      "activations/layer9_attention_weight_min": -44.62107467651367,
      "epoch": 17.34,
      "learning_rate": 3.8538257575757573e-05,
      "loss": 2.7334,
      "step": 298500
    },
    {
      "activations/layer0_attention_weight_max": 15.834822654724121,
      "activations/layer0_attention_weight_min": -12.733550071716309,
      "activations/layer10_attention_weight_max": 35.73465347290039,
      "activations/layer10_attention_weight_min": -35.93623733520508,
      "activations/layer11_attention_weight_max": 31.680343627929688,
      "activations/layer11_attention_weight_min": -32.961326599121094,
      "activations/layer12_attention_weight_max": 17.905380249023438,
      "activations/layer12_attention_weight_min": -29.89722442626953,
      "activations/layer13_attention_weight_max": 34.993316650390625,
      "activations/layer13_attention_weight_min": -28.012928009033203,
      "activations/layer14_attention_weight_max": 37.53805923461914,
      "activations/layer14_attention_weight_min": -30.523700714111328,
      "activations/layer15_attention_weight_max": 36.23406982421875,
      "activations/layer15_attention_weight_min": -30.93968963623047,
      "activations/layer16_attention_weight_max": 34.97315979003906,
      "activations/layer16_attention_weight_min": -35.22283935546875,
      "activations/layer17_attention_weight_max": 53.10958480834961,
      "activations/layer17_attention_weight_min": -50.623451232910156,
      "activations/layer18_attention_weight_max": 46.52920150756836,
      "activations/layer18_attention_weight_min": -44.22459030151367,
      "activations/layer19_attention_weight_max": 22.36872673034668,
      "activations/layer19_attention_weight_min": -23.981229782104492,
      "activations/layer1_attention_weight_max": 16.46968650817871,
      "activations/layer1_attention_weight_min": -14.4251127243042,
      "activations/layer20_attention_weight_max": 22.390655517578125,
      "activations/layer20_attention_weight_min": -21.933238983154297,
      "activations/layer21_attention_weight_max": 34.937957763671875,
      "activations/layer21_attention_weight_min": -22.536170959472656,
      "activations/layer22_attention_weight_max": 25.633516311645508,
      "activations/layer22_attention_weight_min": -25.041446685791016,
      "activations/layer23_attention_weight_max": 34.73576736450195,
      "activations/layer23_attention_weight_min": -25.485855102539062,
      "activations/layer2_attention_weight_max": 32.88365173339844,
      "activations/layer2_attention_weight_min": -29.756004333496094,
      "activations/layer3_attention_weight_max": 90.24044799804688,
      "activations/layer3_attention_weight_min": -87.76592254638672,
      "activations/layer4_attention_weight_max": 88.08995819091797,
      "activations/layer4_attention_weight_min": -86.43004608154297,
      "activations/layer5_attention_weight_max": 63.505760192871094,
      "activations/layer5_attention_weight_min": -73.59359741210938,
      "activations/layer6_attention_weight_max": 49.14126205444336,
      "activations/layer6_attention_weight_min": -50.01530456542969,
      "activations/layer7_attention_weight_max": 60.46929931640625,
      "activations/layer7_attention_weight_min": -60.30764389038086,
      "activations/layer8_attention_weight_max": 44.170135498046875,
      "activations/layer8_attention_weight_min": -47.100074768066406,
      "activations/layer9_attention_weight_max": 45.164424896240234,
      "activations/layer9_attention_weight_min": -47.01894760131836,
      "epoch": 17.35,
      "learning_rate": 3.851931818181818e-05,
      "loss": 2.7377,
      "step": 298550
    },
    {
      "activations/layer0_attention_weight_max": 15.413994789123535,
      "activations/layer0_attention_weight_min": -14.239575386047363,
      "activations/layer10_attention_weight_max": 44.307220458984375,
      "activations/layer10_attention_weight_min": -40.73736572265625,
      "activations/layer11_attention_weight_max": 40.2368049621582,
      "activations/layer11_attention_weight_min": -35.54483413696289,
      "activations/layer12_attention_weight_max": 21.03935432434082,
      "activations/layer12_attention_weight_min": -33.370361328125,
      "activations/layer13_attention_weight_max": 52.98655700683594,
      "activations/layer13_attention_weight_min": -30.746139526367188,
      "activations/layer14_attention_weight_max": 51.342933654785156,
      "activations/layer14_attention_weight_min": -31.915145874023438,
      "activations/layer15_attention_weight_max": 49.08734893798828,
      "activations/layer15_attention_weight_min": -33.70631408691406,
      "activations/layer16_attention_weight_max": 37.74656295776367,
      "activations/layer16_attention_weight_min": -32.27409362792969,
      "activations/layer17_attention_weight_max": 55.61894989013672,
      "activations/layer17_attention_weight_min": -52.17890930175781,
      "activations/layer18_attention_weight_max": 52.0383186340332,
      "activations/layer18_attention_weight_min": -49.635719299316406,
      "activations/layer19_attention_weight_max": 28.29268455505371,
      "activations/layer19_attention_weight_min": -27.853992462158203,
      "activations/layer1_attention_weight_max": 16.078533172607422,
      "activations/layer1_attention_weight_min": -12.039827346801758,
      "activations/layer20_attention_weight_max": 28.305757522583008,
      "activations/layer20_attention_weight_min": -24.998872756958008,
      "activations/layer21_attention_weight_max": 44.10496139526367,
      "activations/layer21_attention_weight_min": -23.197113037109375,
      "activations/layer22_attention_weight_max": 32.555416107177734,
      "activations/layer22_attention_weight_min": -26.49908447265625,
      "activations/layer23_attention_weight_max": 33.64673614501953,
      "activations/layer23_attention_weight_min": -22.250507354736328,
      "activations/layer2_attention_weight_max": 34.5422477722168,
      "activations/layer2_attention_weight_min": -32.21169662475586,
      "activations/layer3_attention_weight_max": 91.1170883178711,
      "activations/layer3_attention_weight_min": -90.9970703125,
      "activations/layer4_attention_weight_max": 89.41464233398438,
      "activations/layer4_attention_weight_min": -87.55592346191406,
      "activations/layer5_attention_weight_max": 65.43367004394531,
      "activations/layer5_attention_weight_min": -71.80024719238281,
      "activations/layer6_attention_weight_max": 49.58546447753906,
      "activations/layer6_attention_weight_min": -51.49745178222656,
      "activations/layer7_attention_weight_max": 77.07254028320312,
      "activations/layer7_attention_weight_min": -68.1988754272461,
      "activations/layer8_attention_weight_max": 52.47517776489258,
      "activations/layer8_attention_weight_min": -51.048099517822266,
      "activations/layer9_attention_weight_max": 56.12569808959961,
      "activations/layer9_attention_weight_min": -47.59467697143555,
      "epoch": 17.35,
      "learning_rate": 3.850037878787878e-05,
      "loss": 2.7504,
      "step": 298600
    },
    {
      "activations/layer0_attention_weight_max": 16.63675880432129,
      "activations/layer0_attention_weight_min": -13.253297805786133,
      "activations/layer10_attention_weight_max": 33.077972412109375,
      "activations/layer10_attention_weight_min": -33.7413215637207,
      "activations/layer11_attention_weight_max": 31.15963363647461,
      "activations/layer11_attention_weight_min": -32.88991928100586,
      "activations/layer12_attention_weight_max": 17.533340454101562,
      "activations/layer12_attention_weight_min": -26.197772979736328,
      "activations/layer13_attention_weight_max": 29.52388572692871,
      "activations/layer13_attention_weight_min": -25.465110778808594,
      "activations/layer14_attention_weight_max": 32.10997772216797,
      "activations/layer14_attention_weight_min": -29.88237953186035,
      "activations/layer15_attention_weight_max": 31.003293991088867,
      "activations/layer15_attention_weight_min": -30.933862686157227,
      "activations/layer16_attention_weight_max": 31.15761375427246,
      "activations/layer16_attention_weight_min": -31.059585571289062,
      "activations/layer17_attention_weight_max": 50.679744720458984,
      "activations/layer17_attention_weight_min": -42.8705940246582,
      "activations/layer18_attention_weight_max": 42.88319778442383,
      "activations/layer18_attention_weight_min": -39.544681549072266,
      "activations/layer19_attention_weight_max": 20.489118576049805,
      "activations/layer19_attention_weight_min": -22.18385887145996,
      "activations/layer1_attention_weight_max": 16.088544845581055,
      "activations/layer1_attention_weight_min": -14.95612621307373,
      "activations/layer20_attention_weight_max": 19.35125160217285,
      "activations/layer20_attention_weight_min": -21.711082458496094,
      "activations/layer21_attention_weight_max": 28.530242919921875,
      "activations/layer21_attention_weight_min": -20.355045318603516,
      "activations/layer22_attention_weight_max": 27.35578155517578,
      "activations/layer22_attention_weight_min": -26.788545608520508,
      "activations/layer23_attention_weight_max": 32.15254211425781,
      "activations/layer23_attention_weight_min": -24.222217559814453,
      "activations/layer2_attention_weight_max": 31.1602840423584,
      "activations/layer2_attention_weight_min": -30.34351348876953,
      "activations/layer3_attention_weight_max": 88.11621856689453,
      "activations/layer3_attention_weight_min": -92.11740112304688,
      "activations/layer4_attention_weight_max": 84.52740478515625,
      "activations/layer4_attention_weight_min": -84.5129623413086,
      "activations/layer5_attention_weight_max": 65.48617553710938,
      "activations/layer5_attention_weight_min": -69.87490844726562,
      "activations/layer6_attention_weight_max": 49.56722640991211,
      "activations/layer6_attention_weight_min": -49.56641387939453,
      "activations/layer7_attention_weight_max": 60.8909912109375,
      "activations/layer7_attention_weight_min": -62.44890594482422,
      "activations/layer8_attention_weight_max": 43.948280334472656,
      "activations/layer8_attention_weight_min": -46.923484802246094,
      "activations/layer9_attention_weight_max": 44.92257308959961,
      "activations/layer9_attention_weight_min": -42.47410202026367,
      "epoch": 17.35,
      "learning_rate": 3.8481439393939385e-05,
      "loss": 2.7479,
      "step": 298650
    },
    {
      "activations/layer0_attention_weight_max": 16.66788673400879,
      "activations/layer0_attention_weight_min": -13.109477043151855,
      "activations/layer10_attention_weight_max": 35.68356704711914,
      "activations/layer10_attention_weight_min": -37.631561279296875,
      "activations/layer11_attention_weight_max": 34.216941833496094,
      "activations/layer11_attention_weight_min": -35.88336181640625,
      "activations/layer12_attention_weight_max": 18.131439208984375,
      "activations/layer12_attention_weight_min": -24.602081298828125,
      "activations/layer13_attention_weight_max": 38.473915100097656,
      "activations/layer13_attention_weight_min": -27.02590560913086,
      "activations/layer14_attention_weight_max": 37.77375411987305,
      "activations/layer14_attention_weight_min": -31.549163818359375,
      "activations/layer15_attention_weight_max": 34.48288345336914,
      "activations/layer15_attention_weight_min": -30.635717391967773,
      "activations/layer16_attention_weight_max": 34.634788513183594,
      "activations/layer16_attention_weight_min": -31.18415069580078,
      "activations/layer17_attention_weight_max": 50.01156997680664,
      "activations/layer17_attention_weight_min": -45.89579391479492,
      "activations/layer18_attention_weight_max": 46.577701568603516,
      "activations/layer18_attention_weight_min": -41.177574157714844,
      "activations/layer19_attention_weight_max": 23.573638916015625,
      "activations/layer19_attention_weight_min": -23.41769027709961,
      "activations/layer1_attention_weight_max": 16.855228424072266,
      "activations/layer1_attention_weight_min": -13.788779258728027,
      "activations/layer20_attention_weight_max": 21.518617630004883,
      "activations/layer20_attention_weight_min": -23.19200897216797,
      "activations/layer21_attention_weight_max": 36.431949615478516,
      "activations/layer21_attention_weight_min": -22.980392456054688,
      "activations/layer22_attention_weight_max": 31.655717849731445,
      "activations/layer22_attention_weight_min": -25.65928077697754,
      "activations/layer23_attention_weight_max": 42.489227294921875,
      "activations/layer23_attention_weight_min": -24.76394271850586,
      "activations/layer2_attention_weight_max": 33.086883544921875,
      "activations/layer2_attention_weight_min": -29.489192962646484,
      "activations/layer3_attention_weight_max": 89.89982604980469,
      "activations/layer3_attention_weight_min": -88.18728637695312,
      "activations/layer4_attention_weight_max": 88.57714080810547,
      "activations/layer4_attention_weight_min": -89.9591293334961,
      "activations/layer5_attention_weight_max": 67.51358795166016,
      "activations/layer5_attention_weight_min": -80.4748306274414,
      "activations/layer6_attention_weight_max": 50.70829772949219,
      "activations/layer6_attention_weight_min": -54.04147720336914,
      "activations/layer7_attention_weight_max": 67.88352966308594,
      "activations/layer7_attention_weight_min": -67.76173400878906,
      "activations/layer8_attention_weight_max": 47.22760009765625,
      "activations/layer8_attention_weight_min": -53.35574722290039,
      "activations/layer9_attention_weight_max": 43.99492645263672,
      "activations/layer9_attention_weight_min": -46.60774230957031,
      "epoch": 17.36,
      "learning_rate": 3.84625e-05,
      "loss": 2.7441,
      "step": 298700
    },
    {
      "activations/layer0_attention_weight_max": 15.15511703491211,
      "activations/layer0_attention_weight_min": -13.170440673828125,
      "activations/layer10_attention_weight_max": 36.1743049621582,
      "activations/layer10_attention_weight_min": -35.73641586303711,
      "activations/layer11_attention_weight_max": 32.43904495239258,
      "activations/layer11_attention_weight_min": -34.682796478271484,
      "activations/layer12_attention_weight_max": 19.31477928161621,
      "activations/layer12_attention_weight_min": -22.883602142333984,
      "activations/layer13_attention_weight_max": 34.182979583740234,
      "activations/layer13_attention_weight_min": -27.798778533935547,
      "activations/layer14_attention_weight_max": 34.5927734375,
      "activations/layer14_attention_weight_min": -29.766399383544922,
      "activations/layer15_attention_weight_max": 30.852270126342773,
      "activations/layer15_attention_weight_min": -29.2047061920166,
      "activations/layer16_attention_weight_max": 33.574642181396484,
      "activations/layer16_attention_weight_min": -31.47149658203125,
      "activations/layer17_attention_weight_max": 51.185855865478516,
      "activations/layer17_attention_weight_min": -45.185794830322266,
      "activations/layer18_attention_weight_max": 48.64902877807617,
      "activations/layer18_attention_weight_min": -42.9401741027832,
      "activations/layer19_attention_weight_max": 22.462514877319336,
      "activations/layer19_attention_weight_min": -23.374982833862305,
      "activations/layer1_attention_weight_max": 16.479656219482422,
      "activations/layer1_attention_weight_min": -14.219663619995117,
      "activations/layer20_attention_weight_max": 20.88096809387207,
      "activations/layer20_attention_weight_min": -21.700077056884766,
      "activations/layer21_attention_weight_max": 33.21464920043945,
      "activations/layer21_attention_weight_min": -22.261960983276367,
      "activations/layer22_attention_weight_max": 26.017189025878906,
      "activations/layer22_attention_weight_min": -27.403352737426758,
      "activations/layer23_attention_weight_max": 36.50700759887695,
      "activations/layer23_attention_weight_min": -24.382007598876953,
      "activations/layer2_attention_weight_max": 32.04075622558594,
      "activations/layer2_attention_weight_min": -30.68190574645996,
      "activations/layer3_attention_weight_max": 93.05896759033203,
      "activations/layer3_attention_weight_min": -96.85227966308594,
      "activations/layer4_attention_weight_max": 90.05064392089844,
      "activations/layer4_attention_weight_min": -92.56822967529297,
      "activations/layer5_attention_weight_max": 65.808837890625,
      "activations/layer5_attention_weight_min": -71.9942626953125,
      "activations/layer6_attention_weight_max": 52.23744583129883,
      "activations/layer6_attention_weight_min": -51.684120178222656,
      "activations/layer7_attention_weight_max": 66.16160583496094,
      "activations/layer7_attention_weight_min": -63.36220932006836,
      "activations/layer8_attention_weight_max": 47.3557014465332,
      "activations/layer8_attention_weight_min": -49.28746795654297,
      "activations/layer9_attention_weight_max": 45.46010971069336,
      "activations/layer9_attention_weight_min": -47.3138542175293,
      "epoch": 17.36,
      "learning_rate": 3.84435606060606e-05,
      "loss": 2.75,
      "step": 298750
    },
    {
      "activations/layer0_attention_weight_max": 15.723600387573242,
      "activations/layer0_attention_weight_min": -13.09420394897461,
      "activations/layer10_attention_weight_max": 34.9266242980957,
      "activations/layer10_attention_weight_min": -36.92308044433594,
      "activations/layer11_attention_weight_max": 31.292469024658203,
      "activations/layer11_attention_weight_min": -33.782981872558594,
      "activations/layer12_attention_weight_max": 19.487760543823242,
      "activations/layer12_attention_weight_min": -32.781761169433594,
      "activations/layer13_attention_weight_max": 33.37981414794922,
      "activations/layer13_attention_weight_min": -27.43421173095703,
      "activations/layer14_attention_weight_max": 39.56126403808594,
      "activations/layer14_attention_weight_min": -28.96748161315918,
      "activations/layer15_attention_weight_max": 36.69917678833008,
      "activations/layer15_attention_weight_min": -30.156055450439453,
      "activations/layer16_attention_weight_max": 36.53759765625,
      "activations/layer16_attention_weight_min": -32.76395034790039,
      "activations/layer17_attention_weight_max": 52.5138053894043,
      "activations/layer17_attention_weight_min": -45.37466812133789,
      "activations/layer18_attention_weight_max": 49.72173309326172,
      "activations/layer18_attention_weight_min": -39.44268035888672,
      "activations/layer19_attention_weight_max": 27.61172866821289,
      "activations/layer19_attention_weight_min": -25.9544620513916,
      "activations/layer1_attention_weight_max": 18.111833572387695,
      "activations/layer1_attention_weight_min": -14.594935417175293,
      "activations/layer20_attention_weight_max": 24.999540328979492,
      "activations/layer20_attention_weight_min": -24.797119140625,
      "activations/layer21_attention_weight_max": 41.09590530395508,
      "activations/layer21_attention_weight_min": -26.521987915039062,
      "activations/layer22_attention_weight_max": 30.32470703125,
      "activations/layer22_attention_weight_min": -27.41971206665039,
      "activations/layer23_attention_weight_max": 40.68247604370117,
      "activations/layer23_attention_weight_min": -28.04279899597168,
      "activations/layer2_attention_weight_max": 34.12339782714844,
      "activations/layer2_attention_weight_min": -30.941455841064453,
      "activations/layer3_attention_weight_max": 89.95967864990234,
      "activations/layer3_attention_weight_min": -86.72540283203125,
      "activations/layer4_attention_weight_max": 92.04911804199219,
      "activations/layer4_attention_weight_min": -84.2561264038086,
      "activations/layer5_attention_weight_max": 67.205810546875,
      "activations/layer5_attention_weight_min": -71.00872802734375,
      "activations/layer6_attention_weight_max": 51.20873260498047,
      "activations/layer6_attention_weight_min": -50.77730178833008,
      "activations/layer7_attention_weight_max": 66.4972152709961,
      "activations/layer7_attention_weight_min": -64.10032653808594,
      "activations/layer8_attention_weight_max": 45.609737396240234,
      "activations/layer8_attention_weight_min": -50.56087112426758,
      "activations/layer9_attention_weight_max": 43.93479537963867,
      "activations/layer9_attention_weight_min": -45.413578033447266,
      "epoch": 17.36,
      "learning_rate": 3.842462121212121e-05,
      "loss": 2.7465,
      "step": 298800
    },
    {
      "activations/layer0_attention_weight_max": 15.508854866027832,
      "activations/layer0_attention_weight_min": -13.255117416381836,
      "activations/layer10_attention_weight_max": 41.90480422973633,
      "activations/layer10_attention_weight_min": -38.31162643432617,
      "activations/layer11_attention_weight_max": 40.502052307128906,
      "activations/layer11_attention_weight_min": -36.10437774658203,
      "activations/layer12_attention_weight_max": 19.754545211791992,
      "activations/layer12_attention_weight_min": -25.197874069213867,
      "activations/layer13_attention_weight_max": 34.77395248413086,
      "activations/layer13_attention_weight_min": -31.463794708251953,
      "activations/layer14_attention_weight_max": 35.57783889770508,
      "activations/layer14_attention_weight_min": -31.24091911315918,
      "activations/layer15_attention_weight_max": 35.10044860839844,
      "activations/layer15_attention_weight_min": -31.3197021484375,
      "activations/layer16_attention_weight_max": 34.43893814086914,
      "activations/layer16_attention_weight_min": -33.30820083618164,
      "activations/layer17_attention_weight_max": 51.19140625,
      "activations/layer17_attention_weight_min": -45.49663543701172,
      "activations/layer18_attention_weight_max": 47.48951721191406,
      "activations/layer18_attention_weight_min": -41.46523666381836,
      "activations/layer19_attention_weight_max": 23.992534637451172,
      "activations/layer19_attention_weight_min": -23.463611602783203,
      "activations/layer1_attention_weight_max": 15.69766902923584,
      "activations/layer1_attention_weight_min": -13.963139533996582,
      "activations/layer20_attention_weight_max": 22.463092803955078,
      "activations/layer20_attention_weight_min": -22.3308162689209,
      "activations/layer21_attention_weight_max": 35.93217849731445,
      "activations/layer21_attention_weight_min": -25.044357299804688,
      "activations/layer22_attention_weight_max": 30.52879524230957,
      "activations/layer22_attention_weight_min": -29.312349319458008,
      "activations/layer23_attention_weight_max": 35.38896942138672,
      "activations/layer23_attention_weight_min": -26.463764190673828,
      "activations/layer2_attention_weight_max": 32.119110107421875,
      "activations/layer2_attention_weight_min": -30.637691497802734,
      "activations/layer3_attention_weight_max": 94.464111328125,
      "activations/layer3_attention_weight_min": -88.97969055175781,
      "activations/layer4_attention_weight_max": 91.38151550292969,
      "activations/layer4_attention_weight_min": -89.1513900756836,
      "activations/layer5_attention_weight_max": 67.3645248413086,
      "activations/layer5_attention_weight_min": -71.78892517089844,
      "activations/layer6_attention_weight_max": 55.18482208251953,
      "activations/layer6_attention_weight_min": -50.17841720581055,
      "activations/layer7_attention_weight_max": 68.26239776611328,
      "activations/layer7_attention_weight_min": -65.4320068359375,
      "activations/layer8_attention_weight_max": 52.35283279418945,
      "activations/layer8_attention_weight_min": -52.92597961425781,
      "activations/layer9_attention_weight_max": 48.054237365722656,
      "activations/layer9_attention_weight_min": -46.7984733581543,
      "epoch": 17.36,
      "learning_rate": 3.840568181818182e-05,
      "loss": 2.7433,
      "step": 298850
    },
    {
      "activations/layer0_attention_weight_max": 15.498637199401855,
      "activations/layer0_attention_weight_min": -13.385212898254395,
      "activations/layer10_attention_weight_max": 32.89345932006836,
      "activations/layer10_attention_weight_min": -32.48624038696289,
      "activations/layer11_attention_weight_max": 31.751625061035156,
      "activations/layer11_attention_weight_min": -33.62506103515625,
      "activations/layer12_attention_weight_max": 19.260772705078125,
      "activations/layer12_attention_weight_min": -24.29568099975586,
      "activations/layer13_attention_weight_max": 31.315086364746094,
      "activations/layer13_attention_weight_min": -27.034162521362305,
      "activations/layer14_attention_weight_max": 33.847869873046875,
      "activations/layer14_attention_weight_min": -32.56837844848633,
      "activations/layer15_attention_weight_max": 32.59446334838867,
      "activations/layer15_attention_weight_min": -30.894176483154297,
      "activations/layer16_attention_weight_max": 32.954315185546875,
      "activations/layer16_attention_weight_min": -30.5462646484375,
      "activations/layer17_attention_weight_max": 48.86637878417969,
      "activations/layer17_attention_weight_min": -43.23208236694336,
      "activations/layer18_attention_weight_max": 42.20646286010742,
      "activations/layer18_attention_weight_min": -39.38791275024414,
      "activations/layer19_attention_weight_max": 24.060184478759766,
      "activations/layer19_attention_weight_min": -21.181947708129883,
      "activations/layer1_attention_weight_max": 16.140810012817383,
      "activations/layer1_attention_weight_min": -16.032228469848633,
      "activations/layer20_attention_weight_max": 21.785961151123047,
      "activations/layer20_attention_weight_min": -21.48592758178711,
      "activations/layer21_attention_weight_max": 33.50630569458008,
      "activations/layer21_attention_weight_min": -19.635536193847656,
      "activations/layer22_attention_weight_max": 27.214658737182617,
      "activations/layer22_attention_weight_min": -27.3928279876709,
      "activations/layer23_attention_weight_max": 35.875667572021484,
      "activations/layer23_attention_weight_min": -23.385601043701172,
      "activations/layer2_attention_weight_max": 33.577537536621094,
      "activations/layer2_attention_weight_min": -33.84317398071289,
      "activations/layer3_attention_weight_max": 89.31920623779297,
      "activations/layer3_attention_weight_min": -90.18815612792969,
      "activations/layer4_attention_weight_max": 86.77730560302734,
      "activations/layer4_attention_weight_min": -85.47647094726562,
      "activations/layer5_attention_weight_max": 63.62010955810547,
      "activations/layer5_attention_weight_min": -74.87811279296875,
      "activations/layer6_attention_weight_max": 48.28611373901367,
      "activations/layer6_attention_weight_min": -48.935489654541016,
      "activations/layer7_attention_weight_max": 65.81278991699219,
      "activations/layer7_attention_weight_min": -63.41111755371094,
      "activations/layer8_attention_weight_max": 43.545902252197266,
      "activations/layer8_attention_weight_min": -46.80489730834961,
      "activations/layer9_attention_weight_max": 42.072486877441406,
      "activations/layer9_attention_weight_min": -42.40349578857422,
      "epoch": 17.37,
      "learning_rate": 3.838674242424242e-05,
      "loss": 2.7468,
      "step": 298900
    },
    {
      "activations/layer0_attention_weight_max": 16.160491943359375,
      "activations/layer0_attention_weight_min": -14.467195510864258,
      "activations/layer10_attention_weight_max": 35.571441650390625,
      "activations/layer10_attention_weight_min": -35.666908264160156,
      "activations/layer11_attention_weight_max": 33.621212005615234,
      "activations/layer11_attention_weight_min": -34.902671813964844,
      "activations/layer12_attention_weight_max": 18.026935577392578,
      "activations/layer12_attention_weight_min": -29.759479522705078,
      "activations/layer13_attention_weight_max": 28.82866859436035,
      "activations/layer13_attention_weight_min": -27.447452545166016,
      "activations/layer14_attention_weight_max": 33.235939025878906,
      "activations/layer14_attention_weight_min": -27.96647834777832,
      "activations/layer15_attention_weight_max": 29.029916763305664,
      "activations/layer15_attention_weight_min": -27.823476791381836,
      "activations/layer16_attention_weight_max": 30.544063568115234,
      "activations/layer16_attention_weight_min": -28.590316772460938,
      "activations/layer17_attention_weight_max": 46.03152847290039,
      "activations/layer17_attention_weight_min": -41.6336669921875,
      "activations/layer18_attention_weight_max": 41.713775634765625,
      "activations/layer18_attention_weight_min": -37.03628158569336,
      "activations/layer19_attention_weight_max": 22.657238006591797,
      "activations/layer19_attention_weight_min": -22.366506576538086,
      "activations/layer1_attention_weight_max": 18.463619232177734,
      "activations/layer1_attention_weight_min": -17.34514617919922,
      "activations/layer20_attention_weight_max": 20.693653106689453,
      "activations/layer20_attention_weight_min": -24.351755142211914,
      "activations/layer21_attention_weight_max": 30.061174392700195,
      "activations/layer21_attention_weight_min": -18.91277503967285,
      "activations/layer22_attention_weight_max": 27.283309936523438,
      "activations/layer22_attention_weight_min": -25.8730525970459,
      "activations/layer23_attention_weight_max": 33.702152252197266,
      "activations/layer23_attention_weight_min": -23.262439727783203,
      "activations/layer2_attention_weight_max": 33.9512939453125,
      "activations/layer2_attention_weight_min": -32.453433990478516,
      "activations/layer3_attention_weight_max": 95.77548217773438,
      "activations/layer3_attention_weight_min": -96.04247283935547,
      "activations/layer4_attention_weight_max": 90.2891845703125,
      "activations/layer4_attention_weight_min": -87.61894989013672,
      "activations/layer5_attention_weight_max": 63.925926208496094,
      "activations/layer5_attention_weight_min": -72.05097198486328,
      "activations/layer6_attention_weight_max": 49.31211853027344,
      "activations/layer6_attention_weight_min": -49.73984909057617,
      "activations/layer7_attention_weight_max": 66.65979766845703,
      "activations/layer7_attention_weight_min": -61.51382827758789,
      "activations/layer8_attention_weight_max": 42.95298385620117,
      "activations/layer8_attention_weight_min": -46.191524505615234,
      "activations/layer9_attention_weight_max": 46.942718505859375,
      "activations/layer9_attention_weight_min": -44.32345962524414,
      "epoch": 17.37,
      "learning_rate": 3.836780303030303e-05,
      "loss": 2.725,
      "step": 298950
    },
    {
      "activations/layer0_attention_weight_max": 16.24981689453125,
      "activations/layer0_attention_weight_min": -13.268050193786621,
      "activations/layer10_attention_weight_max": 36.12257385253906,
      "activations/layer10_attention_weight_min": -35.3072624206543,
      "activations/layer11_attention_weight_max": 33.810882568359375,
      "activations/layer11_attention_weight_min": -34.49860382080078,
      "activations/layer12_attention_weight_max": 18.510526657104492,
      "activations/layer12_attention_weight_min": -23.095218658447266,
      "activations/layer13_attention_weight_max": 35.08464813232422,
      "activations/layer13_attention_weight_min": -26.271879196166992,
      "activations/layer14_attention_weight_max": 36.986087799072266,
      "activations/layer14_attention_weight_min": -30.958845138549805,
      "activations/layer15_attention_weight_max": 34.97823715209961,
      "activations/layer15_attention_weight_min": -30.631359100341797,
      "activations/layer16_attention_weight_max": 37.33244323730469,
      "activations/layer16_attention_weight_min": -33.134376525878906,
      "activations/layer17_attention_weight_max": 58.71662139892578,
      "activations/layer17_attention_weight_min": -49.2569465637207,
      "activations/layer18_attention_weight_max": 54.74608612060547,
      "activations/layer18_attention_weight_min": -42.21220397949219,
      "activations/layer19_attention_weight_max": 26.101810455322266,
      "activations/layer19_attention_weight_min": -24.16974449157715,
      "activations/layer1_attention_weight_max": 15.550848960876465,
      "activations/layer1_attention_weight_min": -14.824090957641602,
      "activations/layer20_attention_weight_max": 24.47345542907715,
      "activations/layer20_attention_weight_min": -23.618452072143555,
      "activations/layer21_attention_weight_max": 37.96766662597656,
      "activations/layer21_attention_weight_min": -21.58673858642578,
      "activations/layer22_attention_weight_max": 33.889564514160156,
      "activations/layer22_attention_weight_min": -26.930145263671875,
      "activations/layer23_attention_weight_max": 44.458290100097656,
      "activations/layer23_attention_weight_min": -24.665191650390625,
      "activations/layer2_attention_weight_max": 33.882450103759766,
      "activations/layer2_attention_weight_min": -33.00250244140625,
      "activations/layer3_attention_weight_max": 91.85052490234375,
      "activations/layer3_attention_weight_min": -91.54277038574219,
      "activations/layer4_attention_weight_max": 88.62459564208984,
      "activations/layer4_attention_weight_min": -85.00847625732422,
      "activations/layer5_attention_weight_max": 66.829345703125,
      "activations/layer5_attention_weight_min": -71.60185241699219,
      "activations/layer6_attention_weight_max": 48.30961227416992,
      "activations/layer6_attention_weight_min": -49.568519592285156,
      "activations/layer7_attention_weight_max": 64.73523712158203,
      "activations/layer7_attention_weight_min": -64.716064453125,
      "activations/layer8_attention_weight_max": 46.94680404663086,
      "activations/layer8_attention_weight_min": -48.60468292236328,
      "activations/layer9_attention_weight_max": 45.07720947265625,
      "activations/layer9_attention_weight_min": -46.63672637939453,
      "epoch": 17.37,
      "learning_rate": 3.834886363636363e-05,
      "loss": 2.7337,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_loss": 2.697265625,
      "eval_runtime": 8.4816,
      "eval_samples_per_second": 506.271,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_openwebtext_loss": 2.697265625,
      "eval_openwebtext_ppl": 14.83910053420958,
      "eval_openwebtext_runtime": 8.4816,
      "eval_openwebtext_samples_per_second": 506.271,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9253,
      "eval_wikitext_samples_per_second": 236.843,
      "step": 299000
    },
    {
      "epoch": 17.37,
      "eval_lambada_loss": 2.494140625,
      "eval_lambada_ppl": 12.111320878668954,
      "eval_lambada_runtime": 9.5342,
      "eval_lambada_samples_per_second": 510.69,
      "step": 299000
    },
    {
      "activations/layer0_attention_weight_max": 15.654109954833984,
      "activations/layer0_attention_weight_min": -13.312657356262207,
      "activations/layer10_attention_weight_max": 33.343441009521484,
      "activations/layer10_attention_weight_min": -34.229454040527344,
      "activations/layer11_attention_weight_max": 31.758468627929688,
      "activations/layer11_attention_weight_min": -32.11621856689453,
      "activations/layer12_attention_weight_max": 19.365116119384766,
      "activations/layer12_attention_weight_min": -25.941221237182617,
      "activations/layer13_attention_weight_max": 29.437074661254883,
      "activations/layer13_attention_weight_min": -30.071481704711914,
      "activations/layer14_attention_weight_max": 33.201717376708984,
      "activations/layer14_attention_weight_min": -31.373538970947266,
      "activations/layer15_attention_weight_max": 31.761064529418945,
      "activations/layer15_attention_weight_min": -30.244964599609375,
      "activations/layer16_attention_weight_max": 33.83591842651367,
      "activations/layer16_attention_weight_min": -32.54345703125,
      "activations/layer17_attention_weight_max": 53.86126708984375,
      "activations/layer17_attention_weight_min": -47.94001770019531,
      "activations/layer18_attention_weight_max": 45.61370086669922,
      "activations/layer18_attention_weight_min": -40.061397552490234,
      "activations/layer19_attention_weight_max": 23.408649444580078,
      "activations/layer19_attention_weight_min": -25.182899475097656,
      "activations/layer1_attention_weight_max": 15.62490463256836,
      "activations/layer1_attention_weight_min": -13.759733200073242,
      "activations/layer20_attention_weight_max": 23.195201873779297,
      "activations/layer20_attention_weight_min": -24.094011306762695,
      "activations/layer21_attention_weight_max": 37.482215881347656,
      "activations/layer21_attention_weight_min": -21.69781494140625,
      "activations/layer22_attention_weight_max": 30.17780876159668,
      "activations/layer22_attention_weight_min": -26.329526901245117,
      "activations/layer23_attention_weight_max": 38.271270751953125,
      "activations/layer23_attention_weight_min": -25.977025985717773,
      "activations/layer2_attention_weight_max": 32.814796447753906,
      "activations/layer2_attention_weight_min": -32.18867874145508,
      "activations/layer3_attention_weight_max": 92.23333740234375,
      "activations/layer3_attention_weight_min": -89.34424591064453,
      "activations/layer4_attention_weight_max": 89.87384796142578,
      "activations/layer4_attention_weight_min": -86.60797882080078,
      "activations/layer5_attention_weight_max": 67.40563201904297,
      "activations/layer5_attention_weight_min": -72.89674377441406,
      "activations/layer6_attention_weight_max": 50.82095718383789,
      "activations/layer6_attention_weight_min": -52.40160369873047,
      "activations/layer7_attention_weight_max": 65.88178253173828,
      "activations/layer7_attention_weight_min": -63.369606018066406,
      "activations/layer8_attention_weight_max": 44.75105667114258,
      "activations/layer8_attention_weight_min": -46.8094482421875,
      "activations/layer9_attention_weight_max": 42.102012634277344,
      "activations/layer9_attention_weight_min": -46.025115966796875,
      "epoch": 17.38,
      "learning_rate": 3.8329924242424244e-05,
      "loss": 2.7363,
      "step": 299050
    },
    {
      "activations/layer0_attention_weight_max": 15.705272674560547,
      "activations/layer0_attention_weight_min": -12.936056137084961,
      "activations/layer10_attention_weight_max": 34.298980712890625,
      "activations/layer10_attention_weight_min": -34.48810577392578,
      "activations/layer11_attention_weight_max": 31.006912231445312,
      "activations/layer11_attention_weight_min": -33.26453399658203,
      "activations/layer12_attention_weight_max": 17.78046989440918,
      "activations/layer12_attention_weight_min": -24.11787223815918,
      "activations/layer13_attention_weight_max": 31.6591796875,
      "activations/layer13_attention_weight_min": -26.892372131347656,
      "activations/layer14_attention_weight_max": 35.90512466430664,
      "activations/layer14_attention_weight_min": -29.855289459228516,
      "activations/layer15_attention_weight_max": 31.160083770751953,
      "activations/layer15_attention_weight_min": -30.711894989013672,
      "activations/layer16_attention_weight_max": 32.50711441040039,
      "activations/layer16_attention_weight_min": -29.908464431762695,
      "activations/layer17_attention_weight_max": 49.46812438964844,
      "activations/layer17_attention_weight_min": -42.46714782714844,
      "activations/layer18_attention_weight_max": 42.976680755615234,
      "activations/layer18_attention_weight_min": -37.29197311401367,
      "activations/layer19_attention_weight_max": 23.249914169311523,
      "activations/layer19_attention_weight_min": -22.275123596191406,
      "activations/layer1_attention_weight_max": 16.262189865112305,
      "activations/layer1_attention_weight_min": -15.523819923400879,
      "activations/layer20_attention_weight_max": 21.316814422607422,
      "activations/layer20_attention_weight_min": -22.078407287597656,
      "activations/layer21_attention_weight_max": 33.791404724121094,
      "activations/layer21_attention_weight_min": -22.02208709716797,
      "activations/layer22_attention_weight_max": 26.487030029296875,
      "activations/layer22_attention_weight_min": -26.85464859008789,
      "activations/layer23_attention_weight_max": 33.80533218383789,
      "activations/layer23_attention_weight_min": -23.784347534179688,
      "activations/layer2_attention_weight_max": 34.46179962158203,
      "activations/layer2_attention_weight_min": -33.91109848022461,
      "activations/layer3_attention_weight_max": 91.12657165527344,
      "activations/layer3_attention_weight_min": -95.68476867675781,
      "activations/layer4_attention_weight_max": 89.27325439453125,
      "activations/layer4_attention_weight_min": -85.32952117919922,
      "activations/layer5_attention_weight_max": 68.14529418945312,
      "activations/layer5_attention_weight_min": -74.9751205444336,
      "activations/layer6_attention_weight_max": 52.70827102661133,
      "activations/layer6_attention_weight_min": -51.33641815185547,
      "activations/layer7_attention_weight_max": 67.40363311767578,
      "activations/layer7_attention_weight_min": -66.37841796875,
      "activations/layer8_attention_weight_max": 45.056365966796875,
      "activations/layer8_attention_weight_min": -50.548851013183594,
      "activations/layer9_attention_weight_max": 42.70975112915039,
      "activations/layer9_attention_weight_min": -44.160247802734375,
      "epoch": 17.38,
      "learning_rate": 3.8310984848484846e-05,
      "loss": 2.7417,
      "step": 299100
    },
    {
      "activations/layer0_attention_weight_max": 15.746499061584473,
      "activations/layer0_attention_weight_min": -13.380446434020996,
      "activations/layer10_attention_weight_max": 33.75522232055664,
      "activations/layer10_attention_weight_min": -35.524017333984375,
      "activations/layer11_attention_weight_max": 31.121673583984375,
      "activations/layer11_attention_weight_min": -33.04548645019531,
      "activations/layer12_attention_weight_max": 17.71980857849121,
      "activations/layer12_attention_weight_min": -23.36954116821289,
      "activations/layer13_attention_weight_max": 31.702913284301758,
      "activations/layer13_attention_weight_min": -27.214326858520508,
      "activations/layer14_attention_weight_max": 34.77798080444336,
      "activations/layer14_attention_weight_min": -29.88091468811035,
      "activations/layer15_attention_weight_max": 30.221765518188477,
      "activations/layer15_attention_weight_min": -29.73857879638672,
      "activations/layer16_attention_weight_max": 32.188724517822266,
      "activations/layer16_attention_weight_min": -31.719802856445312,
      "activations/layer17_attention_weight_max": 50.6583137512207,
      "activations/layer17_attention_weight_min": -43.66530227661133,
      "activations/layer18_attention_weight_max": 46.25777053833008,
      "activations/layer18_attention_weight_min": -39.1616096496582,
      "activations/layer19_attention_weight_max": 23.85491943359375,
      "activations/layer19_attention_weight_min": -23.253023147583008,
      "activations/layer1_attention_weight_max": 15.530192375183105,
      "activations/layer1_attention_weight_min": -14.050594329833984,
      "activations/layer20_attention_weight_max": 21.657169342041016,
      "activations/layer20_attention_weight_min": -24.82347297668457,
      "activations/layer21_attention_weight_max": 36.14570999145508,
      "activations/layer21_attention_weight_min": -24.8439884185791,
      "activations/layer22_attention_weight_max": 27.55186653137207,
      "activations/layer22_attention_weight_min": -25.28337860107422,
      "activations/layer23_attention_weight_max": 34.66014862060547,
      "activations/layer23_attention_weight_min": -24.619335174560547,
      "activations/layer2_attention_weight_max": 32.05912780761719,
      "activations/layer2_attention_weight_min": -31.981292724609375,
      "activations/layer3_attention_weight_max": 94.26737976074219,
      "activations/layer3_attention_weight_min": -94.61146545410156,
      "activations/layer4_attention_weight_max": 87.6097640991211,
      "activations/layer4_attention_weight_min": -90.22610473632812,
      "activations/layer5_attention_weight_max": 64.79830932617188,
      "activations/layer5_attention_weight_min": -78.77672576904297,
      "activations/layer6_attention_weight_max": 49.8119010925293,
      "activations/layer6_attention_weight_min": -49.941287994384766,
      "activations/layer7_attention_weight_max": 62.53508758544922,
      "activations/layer7_attention_weight_min": -66.47854614257812,
      "activations/layer8_attention_weight_max": 46.285579681396484,
      "activations/layer8_attention_weight_min": -48.29143142700195,
      "activations/layer9_attention_weight_max": 46.24387741088867,
      "activations/layer9_attention_weight_min": -43.96357345581055,
      "epoch": 17.38,
      "learning_rate": 3.829204545454545e-05,
      "loss": 2.7386,
      "step": 299150
    },
    {
      "activations/layer0_attention_weight_max": 15.344295501708984,
      "activations/layer0_attention_weight_min": -13.52810287475586,
      "activations/layer10_attention_weight_max": 34.7300910949707,
      "activations/layer10_attention_weight_min": -37.50617599487305,
      "activations/layer11_attention_weight_max": 32.782257080078125,
      "activations/layer11_attention_weight_min": -35.08533477783203,
      "activations/layer12_attention_weight_max": 18.942480087280273,
      "activations/layer12_attention_weight_min": -23.785964965820312,
      "activations/layer13_attention_weight_max": 37.80314254760742,
      "activations/layer13_attention_weight_min": -29.13490867614746,
      "activations/layer14_attention_weight_max": 43.095741271972656,
      "activations/layer14_attention_weight_min": -31.279996871948242,
      "activations/layer15_attention_weight_max": 40.2739143371582,
      "activations/layer15_attention_weight_min": -29.210187911987305,
      "activations/layer16_attention_weight_max": 34.1434326171875,
      "activations/layer16_attention_weight_min": -32.66425704956055,
      "activations/layer17_attention_weight_max": 52.03234100341797,
      "activations/layer17_attention_weight_min": -48.37673568725586,
      "activations/layer18_attention_weight_max": 52.258724212646484,
      "activations/layer18_attention_weight_min": -40.5870361328125,
      "activations/layer19_attention_weight_max": 25.043405532836914,
      "activations/layer19_attention_weight_min": -24.56294822692871,
      "activations/layer1_attention_weight_max": 15.556254386901855,
      "activations/layer1_attention_weight_min": -15.478067398071289,
      "activations/layer20_attention_weight_max": 26.934661865234375,
      "activations/layer20_attention_weight_min": -21.434268951416016,
      "activations/layer21_attention_weight_max": 33.44466781616211,
      "activations/layer21_attention_weight_min": -23.197906494140625,
      "activations/layer22_attention_weight_max": 29.286109924316406,
      "activations/layer22_attention_weight_min": -27.201631546020508,
      "activations/layer23_attention_weight_max": 38.3958625793457,
      "activations/layer23_attention_weight_min": -29.420928955078125,
      "activations/layer2_attention_weight_max": 33.8318977355957,
      "activations/layer2_attention_weight_min": -33.193603515625,
      "activations/layer3_attention_weight_max": 98.42351531982422,
      "activations/layer3_attention_weight_min": -99.17454528808594,
      "activations/layer4_attention_weight_max": 88.3839340209961,
      "activations/layer4_attention_weight_min": -88.7306137084961,
      "activations/layer5_attention_weight_max": 73.5003890991211,
      "activations/layer5_attention_weight_min": -73.73173522949219,
      "activations/layer6_attention_weight_max": 49.75389862060547,
      "activations/layer6_attention_weight_min": -52.170326232910156,
      "activations/layer7_attention_weight_max": 67.17837524414062,
      "activations/layer7_attention_weight_min": -64.4679946899414,
      "activations/layer8_attention_weight_max": 47.298519134521484,
      "activations/layer8_attention_weight_min": -49.24790954589844,
      "activations/layer9_attention_weight_max": 48.3518180847168,
      "activations/layer9_attention_weight_min": -44.53232192993164,
      "epoch": 17.39,
      "learning_rate": 3.8273106060606055e-05,
      "loss": 2.7407,
      "step": 299200
    },
    {
      "activations/layer0_attention_weight_max": 15.10399341583252,
      "activations/layer0_attention_weight_min": -13.175092697143555,
      "activations/layer10_attention_weight_max": 32.823158264160156,
      "activations/layer10_attention_weight_min": -35.04228591918945,
      "activations/layer11_attention_weight_max": 30.51049041748047,
      "activations/layer11_attention_weight_min": -33.34203338623047,
      "activations/layer12_attention_weight_max": 17.332111358642578,
      "activations/layer12_attention_weight_min": -25.183012008666992,
      "activations/layer13_attention_weight_max": 33.71461486816406,
      "activations/layer13_attention_weight_min": -27.0808048248291,
      "activations/layer14_attention_weight_max": 36.12764358520508,
      "activations/layer14_attention_weight_min": -30.876861572265625,
      "activations/layer15_attention_weight_max": 31.049360275268555,
      "activations/layer15_attention_weight_min": -29.47174835205078,
      "activations/layer16_attention_weight_max": 32.845035552978516,
      "activations/layer16_attention_weight_min": -31.932172775268555,
      "activations/layer17_attention_weight_max": 52.20599365234375,
      "activations/layer17_attention_weight_min": -41.923728942871094,
      "activations/layer18_attention_weight_max": 51.315975189208984,
      "activations/layer18_attention_weight_min": -41.169517517089844,
      "activations/layer19_attention_weight_max": 25.083614349365234,
      "activations/layer19_attention_weight_min": -24.06861114501953,
      "activations/layer1_attention_weight_max": 15.885889053344727,
      "activations/layer1_attention_weight_min": -13.870220184326172,
      "activations/layer20_attention_weight_max": 23.23164939880371,
      "activations/layer20_attention_weight_min": -21.226503372192383,
      "activations/layer21_attention_weight_max": 33.04871368408203,
      "activations/layer21_attention_weight_min": -22.16533851623535,
      "activations/layer22_attention_weight_max": 27.36068344116211,
      "activations/layer22_attention_weight_min": -26.29391098022461,
      "activations/layer23_attention_weight_max": 37.588260650634766,
      "activations/layer23_attention_weight_min": -23.304771423339844,
      "activations/layer2_attention_weight_max": 31.48354148864746,
      "activations/layer2_attention_weight_min": -30.98300552368164,
      "activations/layer3_attention_weight_max": 88.09574127197266,
      "activations/layer3_attention_weight_min": -92.98743438720703,
      "activations/layer4_attention_weight_max": 90.18052673339844,
      "activations/layer4_attention_weight_min": -88.89666748046875,
      "activations/layer5_attention_weight_max": 66.68182373046875,
      "activations/layer5_attention_weight_min": -75.85481262207031,
      "activations/layer6_attention_weight_max": 52.534244537353516,
      "activations/layer6_attention_weight_min": -52.70759201049805,
      "activations/layer7_attention_weight_max": 70.63232421875,
      "activations/layer7_attention_weight_min": -67.12723541259766,
      "activations/layer8_attention_weight_max": 44.050743103027344,
      "activations/layer8_attention_weight_min": -49.06172180175781,
      "activations/layer9_attention_weight_max": 39.79502487182617,
      "activations/layer9_attention_weight_min": -45.25672912597656,
      "epoch": 17.39,
      "learning_rate": 3.8254166666666664e-05,
      "loss": 2.7664,
      "step": 299250
    },
    {
      "activations/layer0_attention_weight_max": 15.337905883789062,
      "activations/layer0_attention_weight_min": -13.24696159362793,
      "activations/layer10_attention_weight_max": 34.54298782348633,
      "activations/layer10_attention_weight_min": -34.25638198852539,
      "activations/layer11_attention_weight_max": 31.349609375,
      "activations/layer11_attention_weight_min": -32.75495147705078,
      "activations/layer12_attention_weight_max": 19.67505645751953,
      "activations/layer12_attention_weight_min": -25.44672966003418,
      "activations/layer13_attention_weight_max": 35.07832336425781,
      "activations/layer13_attention_weight_min": -28.041662216186523,
      "activations/layer14_attention_weight_max": 36.65806198120117,
      "activations/layer14_attention_weight_min": -30.685073852539062,
      "activations/layer15_attention_weight_max": 32.66200256347656,
      "activations/layer15_attention_weight_min": -29.649261474609375,
      "activations/layer16_attention_weight_max": 35.349666595458984,
      "activations/layer16_attention_weight_min": -31.878170013427734,
      "activations/layer17_attention_weight_max": 52.945499420166016,
      "activations/layer17_attention_weight_min": -44.76865768432617,
      "activations/layer18_attention_weight_max": 49.876041412353516,
      "activations/layer18_attention_weight_min": -38.538604736328125,
      "activations/layer19_attention_weight_max": 24.179100036621094,
      "activations/layer19_attention_weight_min": -23.00398826599121,
      "activations/layer1_attention_weight_max": 15.337362289428711,
      "activations/layer1_attention_weight_min": -14.6404390335083,
      "activations/layer20_attention_weight_max": 25.552391052246094,
      "activations/layer20_attention_weight_min": -21.9255428314209,
      "activations/layer21_attention_weight_max": 36.48369216918945,
      "activations/layer21_attention_weight_min": -24.482946395874023,
      "activations/layer22_attention_weight_max": 31.78687858581543,
      "activations/layer22_attention_weight_min": -25.150672912597656,
      "activations/layer23_attention_weight_max": 42.299293518066406,
      "activations/layer23_attention_weight_min": -25.45216941833496,
      "activations/layer2_attention_weight_max": 32.61951446533203,
      "activations/layer2_attention_weight_min": -31.86966323852539,
      "activations/layer3_attention_weight_max": 89.08475494384766,
      "activations/layer3_attention_weight_min": -89.58882904052734,
      "activations/layer4_attention_weight_max": 87.84040832519531,
      "activations/layer4_attention_weight_min": -85.71720123291016,
      "activations/layer5_attention_weight_max": 64.54851531982422,
      "activations/layer5_attention_weight_min": -74.67713928222656,
      "activations/layer6_attention_weight_max": 49.41359329223633,
      "activations/layer6_attention_weight_min": -47.42854309082031,
      "activations/layer7_attention_weight_max": 63.675987243652344,
      "activations/layer7_attention_weight_min": -61.028602600097656,
      "activations/layer8_attention_weight_max": 43.6245231628418,
      "activations/layer8_attention_weight_min": -48.86834716796875,
      "activations/layer9_attention_weight_max": 40.998741149902344,
      "activations/layer9_attention_weight_min": -45.126426696777344,
      "epoch": 17.39,
      "learning_rate": 3.823522727272727e-05,
      "loss": 2.7226,
      "step": 299300
    },
    {
      "activations/layer0_attention_weight_max": 16.219707489013672,
      "activations/layer0_attention_weight_min": -13.58375072479248,
      "activations/layer10_attention_weight_max": 34.1942138671875,
      "activations/layer10_attention_weight_min": -33.752845764160156,
      "activations/layer11_attention_weight_max": 30.05164337158203,
      "activations/layer11_attention_weight_min": -32.856109619140625,
      "activations/layer12_attention_weight_max": 20.188831329345703,
      "activations/layer12_attention_weight_min": -24.87364387512207,
      "activations/layer13_attention_weight_max": 32.81456756591797,
      "activations/layer13_attention_weight_min": -28.33980369567871,
      "activations/layer14_attention_weight_max": 33.60865020751953,
      "activations/layer14_attention_weight_min": -29.53080940246582,
      "activations/layer15_attention_weight_max": 31.575496673583984,
      "activations/layer15_attention_weight_min": -29.963014602661133,
      "activations/layer16_attention_weight_max": 31.481779098510742,
      "activations/layer16_attention_weight_min": -31.031278610229492,
      "activations/layer17_attention_weight_max": 53.14863204956055,
      "activations/layer17_attention_weight_min": -43.72408676147461,
      "activations/layer18_attention_weight_max": 49.049808502197266,
      "activations/layer18_attention_weight_min": -39.38605880737305,
      "activations/layer19_attention_weight_max": 22.801816940307617,
      "activations/layer19_attention_weight_min": -24.174314498901367,
      "activations/layer1_attention_weight_max": 16.46499252319336,
      "activations/layer1_attention_weight_min": -14.17135238647461,
      "activations/layer20_attention_weight_max": 20.315366744995117,
      "activations/layer20_attention_weight_min": -23.560081481933594,
      "activations/layer21_attention_weight_max": 33.942466735839844,
      "activations/layer21_attention_weight_min": -25.167814254760742,
      "activations/layer22_attention_weight_max": 28.42571258544922,
      "activations/layer22_attention_weight_min": -25.720056533813477,
      "activations/layer23_attention_weight_max": 32.745872497558594,
      "activations/layer23_attention_weight_min": -22.114421844482422,
      "activations/layer2_attention_weight_max": 31.991806030273438,
      "activations/layer2_attention_weight_min": -31.01299285888672,
      "activations/layer3_attention_weight_max": 89.05878448486328,
      "activations/layer3_attention_weight_min": -91.22418212890625,
      "activations/layer4_attention_weight_max": 87.05528259277344,
      "activations/layer4_attention_weight_min": -83.20632934570312,
      "activations/layer5_attention_weight_max": 64.29946899414062,
      "activations/layer5_attention_weight_min": -70.44354248046875,
      "activations/layer6_attention_weight_max": 47.63191223144531,
      "activations/layer6_attention_weight_min": -48.71242141723633,
      "activations/layer7_attention_weight_max": 63.192813873291016,
      "activations/layer7_attention_weight_min": -58.78023910522461,
      "activations/layer8_attention_weight_max": 43.16960906982422,
      "activations/layer8_attention_weight_min": -44.75722122192383,
      "activations/layer9_attention_weight_max": 41.64897537231445,
      "activations/layer9_attention_weight_min": -44.10506820678711,
      "epoch": 17.39,
      "learning_rate": 3.8216287878787873e-05,
      "loss": 2.7402,
      "step": 299350
    },
    {
      "activations/layer0_attention_weight_max": 16.166345596313477,
      "activations/layer0_attention_weight_min": -13.001605033874512,
      "activations/layer10_attention_weight_max": 33.2020149230957,
      "activations/layer10_attention_weight_min": -33.078731536865234,
      "activations/layer11_attention_weight_max": 29.339702606201172,
      "activations/layer11_attention_weight_min": -33.18768310546875,
      "activations/layer12_attention_weight_max": 18.490131378173828,
      "activations/layer12_attention_weight_min": -22.768022537231445,
      "activations/layer13_attention_weight_max": 36.7874755859375,
      "activations/layer13_attention_weight_min": -27.224933624267578,
      "activations/layer14_attention_weight_max": 35.851287841796875,
      "activations/layer14_attention_weight_min": -29.537086486816406,
      "activations/layer15_attention_weight_max": 32.61165237426758,
      "activations/layer15_attention_weight_min": -29.752790451049805,
      "activations/layer16_attention_weight_max": 32.170143127441406,
      "activations/layer16_attention_weight_min": -32.302276611328125,
      "activations/layer17_attention_weight_max": 49.442867279052734,
      "activations/layer17_attention_weight_min": -44.483360290527344,
      "activations/layer18_attention_weight_max": 45.11156463623047,
      "activations/layer18_attention_weight_min": -38.75489044189453,
      "activations/layer19_attention_weight_max": 26.346744537353516,
      "activations/layer19_attention_weight_min": -23.44716453552246,
      "activations/layer1_attention_weight_max": 15.495926856994629,
      "activations/layer1_attention_weight_min": -14.534299850463867,
      "activations/layer20_attention_weight_max": 26.60662841796875,
      "activations/layer20_attention_weight_min": -22.624767303466797,
      "activations/layer21_attention_weight_max": 34.7922248840332,
      "activations/layer21_attention_weight_min": -23.54897117614746,
      "activations/layer22_attention_weight_max": 30.24015235900879,
      "activations/layer22_attention_weight_min": -23.82884407043457,
      "activations/layer23_attention_weight_max": 36.993988037109375,
      "activations/layer23_attention_weight_min": -24.41212272644043,
      "activations/layer2_attention_weight_max": 32.57963943481445,
      "activations/layer2_attention_weight_min": -31.045011520385742,
      "activations/layer3_attention_weight_max": 89.15644073486328,
      "activations/layer3_attention_weight_min": -85.89358520507812,
      "activations/layer4_attention_weight_max": 86.80925750732422,
      "activations/layer4_attention_weight_min": -83.84846496582031,
      "activations/layer5_attention_weight_max": 61.85691833496094,
      "activations/layer5_attention_weight_min": -72.48486328125,
      "activations/layer6_attention_weight_max": 47.57011032104492,
      "activations/layer6_attention_weight_min": -46.509010314941406,
      "activations/layer7_attention_weight_max": 58.93548583984375,
      "activations/layer7_attention_weight_min": -59.49220657348633,
      "activations/layer8_attention_weight_max": 41.689002990722656,
      "activations/layer8_attention_weight_min": -44.2930793762207,
      "activations/layer9_attention_weight_max": 40.89988327026367,
      "activations/layer9_attention_weight_min": -42.0559196472168,
      "epoch": 17.4,
      "learning_rate": 3.8197348484848475e-05,
      "loss": 2.7271,
      "step": 299400
    },
    {
      "activations/layer0_attention_weight_max": 15.609713554382324,
      "activations/layer0_attention_weight_min": -13.193343162536621,
      "activations/layer10_attention_weight_max": 37.72090148925781,
      "activations/layer10_attention_weight_min": -38.616943359375,
      "activations/layer11_attention_weight_max": 33.210636138916016,
      "activations/layer11_attention_weight_min": -35.44029998779297,
      "activations/layer12_attention_weight_max": 20.19202995300293,
      "activations/layer12_attention_weight_min": -25.435810089111328,
      "activations/layer13_attention_weight_max": 34.64868927001953,
      "activations/layer13_attention_weight_min": -28.66934585571289,
      "activations/layer14_attention_weight_max": 37.09287643432617,
      "activations/layer14_attention_weight_min": -32.30095291137695,
      "activations/layer15_attention_weight_max": 33.5208854675293,
      "activations/layer15_attention_weight_min": -32.07565689086914,
      "activations/layer16_attention_weight_max": 34.28157043457031,
      "activations/layer16_attention_weight_min": -31.680660247802734,
      "activations/layer17_attention_weight_max": 52.284576416015625,
      "activations/layer17_attention_weight_min": -43.97751998901367,
      "activations/layer18_attention_weight_max": 48.67868423461914,
      "activations/layer18_attention_weight_min": -40.21174621582031,
      "activations/layer19_attention_weight_max": 24.713733673095703,
      "activations/layer19_attention_weight_min": -23.799198150634766,
      "activations/layer1_attention_weight_max": 15.832859992980957,
      "activations/layer1_attention_weight_min": -13.921079635620117,
      "activations/layer20_attention_weight_max": 26.866188049316406,
      "activations/layer20_attention_weight_min": -22.021345138549805,
      "activations/layer21_attention_weight_max": 36.2557258605957,
      "activations/layer21_attention_weight_min": -22.26885414123535,
      "activations/layer22_attention_weight_max": 33.563995361328125,
      "activations/layer22_attention_weight_min": -25.58434295654297,
      "activations/layer23_attention_weight_max": 42.67759704589844,
      "activations/layer23_attention_weight_min": -23.975263595581055,
      "activations/layer2_attention_weight_max": 32.931678771972656,
      "activations/layer2_attention_weight_min": -30.7583065032959,
      "activations/layer3_attention_weight_max": 91.67266845703125,
      "activations/layer3_attention_weight_min": -98.06106567382812,
      "activations/layer4_attention_weight_max": 88.07683563232422,
      "activations/layer4_attention_weight_min": -88.08985900878906,
      "activations/layer5_attention_weight_max": 64.31900024414062,
      "activations/layer5_attention_weight_min": -73.58261108398438,
      "activations/layer6_attention_weight_max": 48.651920318603516,
      "activations/layer6_attention_weight_min": -50.62789535522461,
      "activations/layer7_attention_weight_max": 66.89730834960938,
      "activations/layer7_attention_weight_min": -65.48622131347656,
      "activations/layer8_attention_weight_max": 46.421390533447266,
      "activations/layer8_attention_weight_min": -51.156497955322266,
      "activations/layer9_attention_weight_max": 42.986385345458984,
      "activations/layer9_attention_weight_min": -46.706886291503906,
      "epoch": 17.4,
      "learning_rate": 3.817840909090909e-05,
      "loss": 2.7602,
      "step": 299450
    },
    {
      "activations/layer0_attention_weight_max": 16.321975708007812,
      "activations/layer0_attention_weight_min": -14.260525703430176,
      "activations/layer10_attention_weight_max": 36.41362762451172,
      "activations/layer10_attention_weight_min": -36.933197021484375,
      "activations/layer11_attention_weight_max": 32.97257614135742,
      "activations/layer11_attention_weight_min": -35.27253723144531,
      "activations/layer12_attention_weight_max": 18.692834854125977,
      "activations/layer12_attention_weight_min": -28.328285217285156,
      "activations/layer13_attention_weight_max": 30.65426254272461,
      "activations/layer13_attention_weight_min": -26.47913360595703,
      "activations/layer14_attention_weight_max": 33.17988967895508,
      "activations/layer14_attention_weight_min": -29.27191925048828,
      "activations/layer15_attention_weight_max": 33.353302001953125,
      "activations/layer15_attention_weight_min": -29.471654891967773,
      "activations/layer16_attention_weight_max": 34.34303665161133,
      "activations/layer16_attention_weight_min": -31.512224197387695,
      "activations/layer17_attention_weight_max": 54.9837760925293,
      "activations/layer17_attention_weight_min": -46.884056091308594,
      "activations/layer18_attention_weight_max": 46.647705078125,
      "activations/layer18_attention_weight_min": -42.263099670410156,
      "activations/layer19_attention_weight_max": 24.332719802856445,
      "activations/layer19_attention_weight_min": -23.80617332458496,
      "activations/layer1_attention_weight_max": 16.138233184814453,
      "activations/layer1_attention_weight_min": -14.358369827270508,
      "activations/layer20_attention_weight_max": 23.051061630249023,
      "activations/layer20_attention_weight_min": -22.162981033325195,
      "activations/layer21_attention_weight_max": 37.551368713378906,
      "activations/layer21_attention_weight_min": -21.601919174194336,
      "activations/layer22_attention_weight_max": 28.319129943847656,
      "activations/layer22_attention_weight_min": -27.693281173706055,
      "activations/layer23_attention_weight_max": 37.717384338378906,
      "activations/layer23_attention_weight_min": -25.77737045288086,
      "activations/layer2_attention_weight_max": 32.676795959472656,
      "activations/layer2_attention_weight_min": -33.32720184326172,
      "activations/layer3_attention_weight_max": 92.5284423828125,
      "activations/layer3_attention_weight_min": -101.33772277832031,
      "activations/layer4_attention_weight_max": 87.5844955444336,
      "activations/layer4_attention_weight_min": -86.62560272216797,
      "activations/layer5_attention_weight_max": 63.13072967529297,
      "activations/layer5_attention_weight_min": -75.449462890625,
      "activations/layer6_attention_weight_max": 50.33456802368164,
      "activations/layer6_attention_weight_min": -53.56557083129883,
      "activations/layer7_attention_weight_max": 62.15299987792969,
      "activations/layer7_attention_weight_min": -65.58028411865234,
      "activations/layer8_attention_weight_max": 45.1550407409668,
      "activations/layer8_attention_weight_min": -51.23729705810547,
      "activations/layer9_attention_weight_max": 40.88865280151367,
      "activations/layer9_attention_weight_min": -47.171627044677734,
      "epoch": 17.4,
      "learning_rate": 3.815946969696969e-05,
      "loss": 2.7191,
      "step": 299500
    },
    {
      "activations/layer0_attention_weight_max": 16.823768615722656,
      "activations/layer0_attention_weight_min": -13.17052936553955,
      "activations/layer10_attention_weight_max": 34.969905853271484,
      "activations/layer10_attention_weight_min": -34.890533447265625,
      "activations/layer11_attention_weight_max": 31.932167053222656,
      "activations/layer11_attention_weight_min": -31.70790672302246,
      "activations/layer12_attention_weight_max": 18.884769439697266,
      "activations/layer12_attention_weight_min": -26.318151473999023,
      "activations/layer13_attention_weight_max": 34.59333038330078,
      "activations/layer13_attention_weight_min": -28.256990432739258,
      "activations/layer14_attention_weight_max": 35.908470153808594,
      "activations/layer14_attention_weight_min": -30.10593605041504,
      "activations/layer15_attention_weight_max": 32.0463752746582,
      "activations/layer15_attention_weight_min": -31.030920028686523,
      "activations/layer16_attention_weight_max": 31.452224731445312,
      "activations/layer16_attention_weight_min": -29.382726669311523,
      "activations/layer17_attention_weight_max": 51.441871643066406,
      "activations/layer17_attention_weight_min": -40.86268615722656,
      "activations/layer18_attention_weight_max": 45.4090690612793,
      "activations/layer18_attention_weight_min": -37.824989318847656,
      "activations/layer19_attention_weight_max": 23.065595626831055,
      "activations/layer19_attention_weight_min": -22.364192962646484,
      "activations/layer1_attention_weight_max": 16.14790916442871,
      "activations/layer1_attention_weight_min": -16.18610382080078,
      "activations/layer20_attention_weight_max": 23.570598602294922,
      "activations/layer20_attention_weight_min": -22.861770629882812,
      "activations/layer21_attention_weight_max": 34.40921401977539,
      "activations/layer21_attention_weight_min": -21.796016693115234,
      "activations/layer22_attention_weight_max": 27.795061111450195,
      "activations/layer22_attention_weight_min": -25.0383358001709,
      "activations/layer23_attention_weight_max": 37.62117004394531,
      "activations/layer23_attention_weight_min": -23.031192779541016,
      "activations/layer2_attention_weight_max": 31.996261596679688,
      "activations/layer2_attention_weight_min": -31.480728149414062,
      "activations/layer3_attention_weight_max": 93.38468933105469,
      "activations/layer3_attention_weight_min": -89.66215515136719,
      "activations/layer4_attention_weight_max": 86.44480895996094,
      "activations/layer4_attention_weight_min": -83.62626647949219,
      "activations/layer5_attention_weight_max": 60.56995391845703,
      "activations/layer5_attention_weight_min": -68.84613037109375,
      "activations/layer6_attention_weight_max": 47.43922424316406,
      "activations/layer6_attention_weight_min": -48.744422912597656,
      "activations/layer7_attention_weight_max": 59.157386779785156,
      "activations/layer7_attention_weight_min": -61.567142486572266,
      "activations/layer8_attention_weight_max": 44.688880920410156,
      "activations/layer8_attention_weight_min": -49.93495559692383,
      "activations/layer9_attention_weight_max": 41.31528854370117,
      "activations/layer9_attention_weight_min": -45.934410095214844,
      "epoch": 17.41,
      "learning_rate": 3.81405303030303e-05,
      "loss": 2.7513,
      "step": 299550
    },
    {
      "activations/layer0_attention_weight_max": 15.748725891113281,
      "activations/layer0_attention_weight_min": -13.247546195983887,
      "activations/layer10_attention_weight_max": 31.025802612304688,
      "activations/layer10_attention_weight_min": -33.962337493896484,
      "activations/layer11_attention_weight_max": 28.34027099609375,
      "activations/layer11_attention_weight_min": -31.920652389526367,
      "activations/layer12_attention_weight_max": 18.438947677612305,
      "activations/layer12_attention_weight_min": -23.828937530517578,
      "activations/layer13_attention_weight_max": 30.678102493286133,
      "activations/layer13_attention_weight_min": -24.906028747558594,
      "activations/layer14_attention_weight_max": 36.03230667114258,
      "activations/layer14_attention_weight_min": -28.389266967773438,
      "activations/layer15_attention_weight_max": 31.894487380981445,
      "activations/layer15_attention_weight_min": -28.901514053344727,
      "activations/layer16_attention_weight_max": 36.20344924926758,
      "activations/layer16_attention_weight_min": -30.903169631958008,
      "activations/layer17_attention_weight_max": 52.54165267944336,
      "activations/layer17_attention_weight_min": -43.52116012573242,
      "activations/layer18_attention_weight_max": 46.22110366821289,
      "activations/layer18_attention_weight_min": -36.746158599853516,
      "activations/layer19_attention_weight_max": 23.01046371459961,
      "activations/layer19_attention_weight_min": -22.017919540405273,
      "activations/layer1_attention_weight_max": 17.20673942565918,
      "activations/layer1_attention_weight_min": -15.841381072998047,
      "activations/layer20_attention_weight_max": 21.208642959594727,
      "activations/layer20_attention_weight_min": -22.58708381652832,
      "activations/layer21_attention_weight_max": 35.84821319580078,
      "activations/layer21_attention_weight_min": -20.228113174438477,
      "activations/layer22_attention_weight_max": 29.03982925415039,
      "activations/layer22_attention_weight_min": -24.156797409057617,
      "activations/layer23_attention_weight_max": 36.19395446777344,
      "activations/layer23_attention_weight_min": -23.13820457458496,
      "activations/layer2_attention_weight_max": 31.282169342041016,
      "activations/layer2_attention_weight_min": -33.78704071044922,
      "activations/layer3_attention_weight_max": 89.80860900878906,
      "activations/layer3_attention_weight_min": -92.02682495117188,
      "activations/layer4_attention_weight_max": 84.53395080566406,
      "activations/layer4_attention_weight_min": -83.4835205078125,
      "activations/layer5_attention_weight_max": 62.967559814453125,
      "activations/layer5_attention_weight_min": -72.88268280029297,
      "activations/layer6_attention_weight_max": 45.34044647216797,
      "activations/layer6_attention_weight_min": -48.709964752197266,
      "activations/layer7_attention_weight_max": 62.145538330078125,
      "activations/layer7_attention_weight_min": -61.359012603759766,
      "activations/layer8_attention_weight_max": 41.53657913208008,
      "activations/layer8_attention_weight_min": -44.64668273925781,
      "activations/layer9_attention_weight_max": 40.06357955932617,
      "activations/layer9_attention_weight_min": -42.71953201293945,
      "epoch": 17.41,
      "learning_rate": 3.81215909090909e-05,
      "loss": 2.7545,
      "step": 299600
    },
    {
      "activations/layer0_attention_weight_max": 16.031089782714844,
      "activations/layer0_attention_weight_min": -13.084935188293457,
      "activations/layer10_attention_weight_max": 36.9879150390625,
      "activations/layer10_attention_weight_min": -34.989601135253906,
      "activations/layer11_attention_weight_max": 30.638341903686523,
      "activations/layer11_attention_weight_min": -32.50221252441406,
      "activations/layer12_attention_weight_max": 18.34491539001465,
      "activations/layer12_attention_weight_min": -25.8691463470459,
      "activations/layer13_attention_weight_max": 39.13005828857422,
      "activations/layer13_attention_weight_min": -29.452638626098633,
      "activations/layer14_attention_weight_max": 35.96333312988281,
      "activations/layer14_attention_weight_min": -29.060359954833984,
      "activations/layer15_attention_weight_max": 33.326133728027344,
      "activations/layer15_attention_weight_min": -29.746135711669922,
      "activations/layer16_attention_weight_max": 33.92457962036133,
      "activations/layer16_attention_weight_min": -31.084659576416016,
      "activations/layer17_attention_weight_max": 51.97005081176758,
      "activations/layer17_attention_weight_min": -45.45412826538086,
      "activations/layer18_attention_weight_max": 45.89486312866211,
      "activations/layer18_attention_weight_min": -39.12166213989258,
      "activations/layer19_attention_weight_max": 24.110280990600586,
      "activations/layer19_attention_weight_min": -23.02123260498047,
      "activations/layer1_attention_weight_max": 16.021106719970703,
      "activations/layer1_attention_weight_min": -15.266805648803711,
      "activations/layer20_attention_weight_max": 22.30464744567871,
      "activations/layer20_attention_weight_min": -21.307022094726562,
      "activations/layer21_attention_weight_max": 37.03605270385742,
      "activations/layer21_attention_weight_min": -21.057167053222656,
      "activations/layer22_attention_weight_max": 30.920827865600586,
      "activations/layer22_attention_weight_min": -26.412174224853516,
      "activations/layer23_attention_weight_max": 38.55989074707031,
      "activations/layer23_attention_weight_min": -22.656455993652344,
      "activations/layer2_attention_weight_max": 33.09907531738281,
      "activations/layer2_attention_weight_min": -32.81829833984375,
      "activations/layer3_attention_weight_max": 92.18370819091797,
      "activations/layer3_attention_weight_min": -92.63931274414062,
      "activations/layer4_attention_weight_max": 88.17623138427734,
      "activations/layer4_attention_weight_min": -88.89788055419922,
      "activations/layer5_attention_weight_max": 70.84164428710938,
      "activations/layer5_attention_weight_min": -73.8798599243164,
      "activations/layer6_attention_weight_max": 52.604042053222656,
      "activations/layer6_attention_weight_min": -52.025333404541016,
      "activations/layer7_attention_weight_max": 66.23900604248047,
      "activations/layer7_attention_weight_min": -65.5875473022461,
      "activations/layer8_attention_weight_max": 51.43384552001953,
      "activations/layer8_attention_weight_min": -50.724693298339844,
      "activations/layer9_attention_weight_max": 42.87223434448242,
      "activations/layer9_attention_weight_min": -46.60764694213867,
      "epoch": 17.41,
      "learning_rate": 3.8102651515151516e-05,
      "loss": 2.7391,
      "step": 299650
    },
    {
      "activations/layer0_attention_weight_max": 16.06622886657715,
      "activations/layer0_attention_weight_min": -12.90928840637207,
      "activations/layer10_attention_weight_max": 37.560089111328125,
      "activations/layer10_attention_weight_min": -36.89696502685547,
      "activations/layer11_attention_weight_max": 32.7857551574707,
      "activations/layer11_attention_weight_min": -35.05025863647461,
      "activations/layer12_attention_weight_max": 21.423934936523438,
      "activations/layer12_attention_weight_min": -24.150165557861328,
      "activations/layer13_attention_weight_max": 41.17042922973633,
      "activations/layer13_attention_weight_min": -29.4155216217041,
      "activations/layer14_attention_weight_max": 41.2701301574707,
      "activations/layer14_attention_weight_min": -30.585493087768555,
      "activations/layer15_attention_weight_max": 39.47187805175781,
      "activations/layer15_attention_weight_min": -30.117847442626953,
      "activations/layer16_attention_weight_max": 39.17781066894531,
      "activations/layer16_attention_weight_min": -34.983482360839844,
      "activations/layer17_attention_weight_max": 59.96829605102539,
      "activations/layer17_attention_weight_min": -47.67964553833008,
      "activations/layer18_attention_weight_max": 56.916439056396484,
      "activations/layer18_attention_weight_min": -40.716163635253906,
      "activations/layer19_attention_weight_max": 30.52166175842285,
      "activations/layer19_attention_weight_min": -25.224449157714844,
      "activations/layer1_attention_weight_max": 16.077749252319336,
      "activations/layer1_attention_weight_min": -14.971856117248535,
      "activations/layer20_attention_weight_max": 28.384193420410156,
      "activations/layer20_attention_weight_min": -21.26598358154297,
      "activations/layer21_attention_weight_max": 41.852455139160156,
      "activations/layer21_attention_weight_min": -20.885038375854492,
      "activations/layer22_attention_weight_max": 33.80855178833008,
      "activations/layer22_attention_weight_min": -26.258224487304688,
      "activations/layer23_attention_weight_max": 44.406517028808594,
      "activations/layer23_attention_weight_min": -23.811384201049805,
      "activations/layer2_attention_weight_max": 31.186025619506836,
      "activations/layer2_attention_weight_min": -30.96473503112793,
      "activations/layer3_attention_weight_max": 89.79417419433594,
      "activations/layer3_attention_weight_min": -90.60213470458984,
      "activations/layer4_attention_weight_max": 84.79991912841797,
      "activations/layer4_attention_weight_min": -83.60295867919922,
      "activations/layer5_attention_weight_max": 63.1837272644043,
      "activations/layer5_attention_weight_min": -72.22297668457031,
      "activations/layer6_attention_weight_max": 48.294864654541016,
      "activations/layer6_attention_weight_min": -49.276676177978516,
      "activations/layer7_attention_weight_max": 65.16158294677734,
      "activations/layer7_attention_weight_min": -60.54838943481445,
      "activations/layer8_attention_weight_max": 45.44807434082031,
      "activations/layer8_attention_weight_min": -48.4283332824707,
      "activations/layer9_attention_weight_max": 41.856727600097656,
      "activations/layer9_attention_weight_min": -47.28712844848633,
      "epoch": 17.41,
      "learning_rate": 3.808409090909091e-05,
      "loss": 2.7279,
      "step": 299700
    },
    {
      "activations/layer0_attention_weight_max": 16.199289321899414,
      "activations/layer0_attention_weight_min": -12.965444564819336,
      "activations/layer10_attention_weight_max": 34.872161865234375,
      "activations/layer10_attention_weight_min": -35.4278678894043,
      "activations/layer11_attention_weight_max": 32.69579315185547,
      "activations/layer11_attention_weight_min": -33.719139099121094,
      "activations/layer12_attention_weight_max": 17.953367233276367,
      "activations/layer12_attention_weight_min": -24.014081954956055,
      "activations/layer13_attention_weight_max": 31.475265502929688,
      "activations/layer13_attention_weight_min": -27.049480438232422,
      "activations/layer14_attention_weight_max": 32.94623565673828,
      "activations/layer14_attention_weight_min": -29.45178985595703,
      "activations/layer15_attention_weight_max": 30.124156951904297,
      "activations/layer15_attention_weight_min": -29.534204483032227,
      "activations/layer16_attention_weight_max": 33.18981170654297,
      "activations/layer16_attention_weight_min": -34.274845123291016,
      "activations/layer17_attention_weight_max": 47.234474182128906,
      "activations/layer17_attention_weight_min": -47.564361572265625,
      "activations/layer18_attention_weight_max": 42.98727035522461,
      "activations/layer18_attention_weight_min": -40.436893463134766,
      "activations/layer19_attention_weight_max": 22.527334213256836,
      "activations/layer19_attention_weight_min": -22.86564826965332,
      "activations/layer1_attention_weight_max": 15.561668395996094,
      "activations/layer1_attention_weight_min": -14.879148483276367,
      "activations/layer20_attention_weight_max": 21.379365921020508,
      "activations/layer20_attention_weight_min": -22.98368263244629,
      "activations/layer21_attention_weight_max": 35.0670280456543,
      "activations/layer21_attention_weight_min": -22.35763168334961,
      "activations/layer22_attention_weight_max": 26.928180694580078,
      "activations/layer22_attention_weight_min": -25.64978790283203,
      "activations/layer23_attention_weight_max": 33.243934631347656,
      "activations/layer23_attention_weight_min": -28.846708297729492,
      "activations/layer2_attention_weight_max": 33.32853698730469,
      "activations/layer2_attention_weight_min": -31.572040557861328,
      "activations/layer3_attention_weight_max": 94.78618621826172,
      "activations/layer3_attention_weight_min": -90.19873809814453,
      "activations/layer4_attention_weight_max": 91.28368377685547,
      "activations/layer4_attention_weight_min": -86.2757339477539,
      "activations/layer5_attention_weight_max": 67.94734191894531,
      "activations/layer5_attention_weight_min": -73.2484359741211,
      "activations/layer6_attention_weight_max": 49.241798400878906,
      "activations/layer6_attention_weight_min": -51.34974670410156,
      "activations/layer7_attention_weight_max": 66.7012710571289,
      "activations/layer7_attention_weight_min": -62.926517486572266,
      "activations/layer8_attention_weight_max": 45.67475891113281,
      "activations/layer8_attention_weight_min": -45.997013092041016,
      "activations/layer9_attention_weight_max": 43.23021697998047,
      "activations/layer9_attention_weight_min": -42.62160110473633,
      "epoch": 17.42,
      "learning_rate": 3.806515151515151e-05,
      "loss": 2.7287,
      "step": 299750
    },
    {
      "activations/layer0_attention_weight_max": 15.454949378967285,
      "activations/layer0_attention_weight_min": -13.05199909210205,
      "activations/layer10_attention_weight_max": 36.6295051574707,
      "activations/layer10_attention_weight_min": -35.161720275878906,
      "activations/layer11_attention_weight_max": 32.595733642578125,
      "activations/layer11_attention_weight_min": -34.68524169921875,
      "activations/layer12_attention_weight_max": 20.255306243896484,
      "activations/layer12_attention_weight_min": -27.601423263549805,
      "activations/layer13_attention_weight_max": 35.982810974121094,
      "activations/layer13_attention_weight_min": -30.266277313232422,
      "activations/layer14_attention_weight_max": 37.59263610839844,
      "activations/layer14_attention_weight_min": -33.19554901123047,
      "activations/layer15_attention_weight_max": 34.046871185302734,
      "activations/layer15_attention_weight_min": -32.55294418334961,
      "activations/layer16_attention_weight_max": 34.20969009399414,
      "activations/layer16_attention_weight_min": -34.489013671875,
      "activations/layer17_attention_weight_max": 54.7613525390625,
      "activations/layer17_attention_weight_min": -50.564910888671875,
      "activations/layer18_attention_weight_max": 48.788448333740234,
      "activations/layer18_attention_weight_min": -43.56332015991211,
      "activations/layer19_attention_weight_max": 23.098005294799805,
      "activations/layer19_attention_weight_min": -24.188657760620117,
      "activations/layer1_attention_weight_max": 16.686967849731445,
      "activations/layer1_attention_weight_min": -16.07171630859375,
      "activations/layer20_attention_weight_max": 22.478145599365234,
      "activations/layer20_attention_weight_min": -22.622270584106445,
      "activations/layer21_attention_weight_max": 37.527565002441406,
      "activations/layer21_attention_weight_min": -24.787887573242188,
      "activations/layer22_attention_weight_max": 30.21695327758789,
      "activations/layer22_attention_weight_min": -25.337291717529297,
      "activations/layer23_attention_weight_max": 37.18548583984375,
      "activations/layer23_attention_weight_min": -27.207012176513672,
      "activations/layer2_attention_weight_max": 32.58264923095703,
      "activations/layer2_attention_weight_min": -33.06601333618164,
      "activations/layer3_attention_weight_max": 91.0705795288086,
      "activations/layer3_attention_weight_min": -93.31842803955078,
      "activations/layer4_attention_weight_max": 88.02015686035156,
      "activations/layer4_attention_weight_min": -86.82110595703125,
      "activations/layer5_attention_weight_max": 65.15313720703125,
      "activations/layer5_attention_weight_min": -75.58830261230469,
      "activations/layer6_attention_weight_max": 50.418373107910156,
      "activations/layer6_attention_weight_min": -51.97969055175781,
      "activations/layer7_attention_weight_max": 65.96188354492188,
      "activations/layer7_attention_weight_min": -63.74406051635742,
      "activations/layer8_attention_weight_max": 46.61568069458008,
      "activations/layer8_attention_weight_min": -49.01518630981445,
      "activations/layer9_attention_weight_max": 46.0318717956543,
      "activations/layer9_attention_weight_min": -44.78767013549805,
      "epoch": 17.42,
      "learning_rate": 3.8046212121212124e-05,
      "loss": 2.7347,
      "step": 299800
    },
    {
      "activations/layer0_attention_weight_max": 16.082393646240234,
      "activations/layer0_attention_weight_min": -13.297085762023926,
      "activations/layer10_attention_weight_max": 33.56226348876953,
      "activations/layer10_attention_weight_min": -34.149539947509766,
      "activations/layer11_attention_weight_max": 30.005430221557617,
      "activations/layer11_attention_weight_min": -31.680051803588867,
      "activations/layer12_attention_weight_max": 17.18135643005371,
      "activations/layer12_attention_weight_min": -22.10154914855957,
      "activations/layer13_attention_weight_max": 32.299766540527344,
      "activations/layer13_attention_weight_min": -28.205570220947266,
      "activations/layer14_attention_weight_max": 38.900814056396484,
      "activations/layer14_attention_weight_min": -30.94183921813965,
      "activations/layer15_attention_weight_max": 32.970455169677734,
      "activations/layer15_attention_weight_min": -29.514318466186523,
      "activations/layer16_attention_weight_max": 36.83562088012695,
      "activations/layer16_attention_weight_min": -34.67228317260742,
      "activations/layer17_attention_weight_max": 55.69614791870117,
      "activations/layer17_attention_weight_min": -49.75416564941406,
      "activations/layer18_attention_weight_max": 48.79758834838867,
      "activations/layer18_attention_weight_min": -43.727149963378906,
      "activations/layer19_attention_weight_max": 26.935117721557617,
      "activations/layer19_attention_weight_min": -23.349632263183594,
      "activations/layer1_attention_weight_max": 16.3136043548584,
      "activations/layer1_attention_weight_min": -13.428213119506836,
      "activations/layer20_attention_weight_max": 23.903444290161133,
      "activations/layer20_attention_weight_min": -23.185155868530273,
      "activations/layer21_attention_weight_max": 38.510276794433594,
      "activations/layer21_attention_weight_min": -21.695749282836914,
      "activations/layer22_attention_weight_max": 33.25965881347656,
      "activations/layer22_attention_weight_min": -24.853351593017578,
      "activations/layer23_attention_weight_max": 34.68852233886719,
      "activations/layer23_attention_weight_min": -23.407695770263672,
      "activations/layer2_attention_weight_max": 33.8925666809082,
      "activations/layer2_attention_weight_min": -31.20547866821289,
      "activations/layer3_attention_weight_max": 100.45878601074219,
      "activations/layer3_attention_weight_min": -95.55836486816406,
      "activations/layer4_attention_weight_max": 86.51863861083984,
      "activations/layer4_attention_weight_min": -86.70784759521484,
      "activations/layer5_attention_weight_max": 63.913543701171875,
      "activations/layer5_attention_weight_min": -69.67496490478516,
      "activations/layer6_attention_weight_max": 47.41307830810547,
      "activations/layer6_attention_weight_min": -48.59973907470703,
      "activations/layer7_attention_weight_max": 60.53542709350586,
      "activations/layer7_attention_weight_min": -61.64356994628906,
      "activations/layer8_attention_weight_max": 45.34629440307617,
      "activations/layer8_attention_weight_min": -49.061397552490234,
      "activations/layer9_attention_weight_max": 41.238197326660156,
      "activations/layer9_attention_weight_min": -44.081031799316406,
      "epoch": 17.42,
      "learning_rate": 3.8027272727272725e-05,
      "loss": 2.7421,
      "step": 299850
    },
    {
      "activations/layer0_attention_weight_max": 16.041454315185547,
      "activations/layer0_attention_weight_min": -13.727132797241211,
      "activations/layer10_attention_weight_max": 38.19037628173828,
      "activations/layer10_attention_weight_min": -37.50634765625,
      "activations/layer11_attention_weight_max": 34.472557067871094,
      "activations/layer11_attention_weight_min": -34.21385192871094,
      "activations/layer12_attention_weight_max": 19.432950973510742,
      "activations/layer12_attention_weight_min": -27.054414749145508,
      "activations/layer13_attention_weight_max": 36.23052978515625,
      "activations/layer13_attention_weight_min": -27.506431579589844,
      "activations/layer14_attention_weight_max": 36.290611267089844,
      "activations/layer14_attention_weight_min": -30.876968383789062,
      "activations/layer15_attention_weight_max": 36.5656852722168,
      "activations/layer15_attention_weight_min": -33.165496826171875,
      "activations/layer16_attention_weight_max": 36.71318435668945,
      "activations/layer16_attention_weight_min": -34.686370849609375,
      "activations/layer17_attention_weight_max": 53.09764099121094,
      "activations/layer17_attention_weight_min": -49.14500045776367,
      "activations/layer18_attention_weight_max": 46.87565994262695,
      "activations/layer18_attention_weight_min": -44.3361701965332,
      "activations/layer19_attention_weight_max": 23.65122413635254,
      "activations/layer19_attention_weight_min": -23.426677703857422,
      "activations/layer1_attention_weight_max": 15.588272094726562,
      "activations/layer1_attention_weight_min": -14.889236450195312,
      "activations/layer20_attention_weight_max": 22.282188415527344,
      "activations/layer20_attention_weight_min": -23.699533462524414,
      "activations/layer21_attention_weight_max": 34.97237014770508,
      "activations/layer21_attention_weight_min": -22.777217864990234,
      "activations/layer22_attention_weight_max": 29.504955291748047,
      "activations/layer22_attention_weight_min": -26.67597770690918,
      "activations/layer23_attention_weight_max": 39.66521072387695,
      "activations/layer23_attention_weight_min": -24.58493423461914,
      "activations/layer2_attention_weight_max": 31.05402946472168,
      "activations/layer2_attention_weight_min": -30.698341369628906,
      "activations/layer3_attention_weight_max": 88.59563446044922,
      "activations/layer3_attention_weight_min": -90.04625701904297,
      "activations/layer4_attention_weight_max": 84.42855072021484,
      "activations/layer4_attention_weight_min": -86.0368423461914,
      "activations/layer5_attention_weight_max": 63.534217834472656,
      "activations/layer5_attention_weight_min": -73.33479309082031,
      "activations/layer6_attention_weight_max": 49.371944427490234,
      "activations/layer6_attention_weight_min": -50.32341003417969,
      "activations/layer7_attention_weight_max": 63.99028396606445,
      "activations/layer7_attention_weight_min": -65.64686584472656,
      "activations/layer8_attention_weight_max": 50.870113372802734,
      "activations/layer8_attention_weight_min": -50.009708404541016,
      "activations/layer9_attention_weight_max": 46.27814483642578,
      "activations/layer9_attention_weight_min": -50.25275802612305,
      "epoch": 17.43,
      "learning_rate": 3.8008333333333327e-05,
      "loss": 2.7427,
      "step": 299900
    },
    {
      "activations/layer0_attention_weight_max": 15.583096504211426,
      "activations/layer0_attention_weight_min": -13.094094276428223,
      "activations/layer10_attention_weight_max": 42.56365203857422,
      "activations/layer10_attention_weight_min": -39.577762603759766,
      "activations/layer11_attention_weight_max": 37.680641174316406,
      "activations/layer11_attention_weight_min": -38.992889404296875,
      "activations/layer12_attention_weight_max": 19.51032257080078,
      "activations/layer12_attention_weight_min": -37.32365036010742,
      "activations/layer13_attention_weight_max": 37.86156463623047,
      "activations/layer13_attention_weight_min": -31.240886688232422,
      "activations/layer14_attention_weight_max": 39.27451705932617,
      "activations/layer14_attention_weight_min": -34.33137512207031,
      "activations/layer15_attention_weight_max": 36.80342483520508,
      "activations/layer15_attention_weight_min": -33.12617492675781,
      "activations/layer16_attention_weight_max": 37.73579406738281,
      "activations/layer16_attention_weight_min": -34.97259521484375,
      "activations/layer17_attention_weight_max": 55.806434631347656,
      "activations/layer17_attention_weight_min": -47.8171272277832,
      "activations/layer18_attention_weight_max": 52.22631072998047,
      "activations/layer18_attention_weight_min": -41.953182220458984,
      "activations/layer19_attention_weight_max": 25.144920349121094,
      "activations/layer19_attention_weight_min": -24.260780334472656,
      "activations/layer1_attention_weight_max": 16.723508834838867,
      "activations/layer1_attention_weight_min": -14.079654693603516,
      "activations/layer20_attention_weight_max": 24.964750289916992,
      "activations/layer20_attention_weight_min": -21.39739227294922,
      "activations/layer21_attention_weight_max": 40.89962387084961,
      "activations/layer21_attention_weight_min": -22.819669723510742,
      "activations/layer22_attention_weight_max": 30.1917724609375,
      "activations/layer22_attention_weight_min": -27.56660270690918,
      "activations/layer23_attention_weight_max": 41.51339340209961,
      "activations/layer23_attention_weight_min": -23.089763641357422,
      "activations/layer2_attention_weight_max": 35.742427825927734,
      "activations/layer2_attention_weight_min": -32.12049865722656,
      "activations/layer3_attention_weight_max": 97.82428741455078,
      "activations/layer3_attention_weight_min": -96.35472869873047,
      "activations/layer4_attention_weight_max": 91.8442153930664,
      "activations/layer4_attention_weight_min": -93.2984848022461,
      "activations/layer5_attention_weight_max": 67.35246276855469,
      "activations/layer5_attention_weight_min": -74.05751037597656,
      "activations/layer6_attention_weight_max": 52.5799446105957,
      "activations/layer6_attention_weight_min": -57.11525344848633,
      "activations/layer7_attention_weight_max": 71.91282653808594,
      "activations/layer7_attention_weight_min": -68.90010070800781,
      "activations/layer8_attention_weight_max": 50.25885009765625,
      "activations/layer8_attention_weight_min": -52.45283889770508,
      "activations/layer9_attention_weight_max": 53.77971267700195,
      "activations/layer9_attention_weight_min": -51.96055221557617,
      "epoch": 17.43,
      "learning_rate": 3.7989393939393935e-05,
      "loss": 2.7286,
      "step": 299950
    },
    {
      "activations/layer0_attention_weight_max": 15.317194938659668,
      "activations/layer0_attention_weight_min": -13.20035457611084,
      "activations/layer10_attention_weight_max": 34.09511184692383,
      "activations/layer10_attention_weight_min": -35.98796844482422,
      "activations/layer11_attention_weight_max": 33.19713592529297,
      "activations/layer11_attention_weight_min": -34.84207534790039,
      "activations/layer12_attention_weight_max": 19.681737899780273,
      "activations/layer12_attention_weight_min": -27.947492599487305,
      "activations/layer13_attention_weight_max": 36.43862533569336,
      "activations/layer13_attention_weight_min": -28.92156219482422,
      "activations/layer14_attention_weight_max": 35.70816421508789,
      "activations/layer14_attention_weight_min": -31.49734115600586,
      "activations/layer15_attention_weight_max": 31.683853149414062,
      "activations/layer15_attention_weight_min": -28.974411010742188,
      "activations/layer16_attention_weight_max": 33.7380485534668,
      "activations/layer16_attention_weight_min": -34.165409088134766,
      "activations/layer17_attention_weight_max": 52.00156021118164,
      "activations/layer17_attention_weight_min": -44.83915710449219,
      "activations/layer18_attention_weight_max": 46.30780792236328,
      "activations/layer18_attention_weight_min": -39.66862106323242,
      "activations/layer19_attention_weight_max": 26.09993553161621,
      "activations/layer19_attention_weight_min": -22.308502197265625,
      "activations/layer1_attention_weight_max": 15.758807182312012,
      "activations/layer1_attention_weight_min": -14.3562593460083,
      "activations/layer20_attention_weight_max": 23.13857650756836,
      "activations/layer20_attention_weight_min": -21.90445327758789,
      "activations/layer21_attention_weight_max": 40.54388427734375,
      "activations/layer21_attention_weight_min": -23.767578125,
      "activations/layer22_attention_weight_max": 31.152891159057617,
      "activations/layer22_attention_weight_min": -26.130849838256836,
      "activations/layer23_attention_weight_max": 41.03744888305664,
      "activations/layer23_attention_weight_min": -24.662952423095703,
      "activations/layer2_attention_weight_max": 32.32373046875,
      "activations/layer2_attention_weight_min": -30.436948776245117,
      "activations/layer3_attention_weight_max": 91.69100952148438,
      "activations/layer3_attention_weight_min": -98.65814208984375,
      "activations/layer4_attention_weight_max": 88.03170013427734,
      "activations/layer4_attention_weight_min": -97.53643035888672,
      "activations/layer5_attention_weight_max": 64.95565795898438,
      "activations/layer5_attention_weight_min": -74.42903137207031,
      "activations/layer6_attention_weight_max": 50.60022735595703,
      "activations/layer6_attention_weight_min": -52.03413009643555,
      "activations/layer7_attention_weight_max": 61.818763732910156,
      "activations/layer7_attention_weight_min": -61.978851318359375,
      "activations/layer8_attention_weight_max": 46.337196350097656,
      "activations/layer8_attention_weight_min": -47.701271057128906,
      "activations/layer9_attention_weight_max": 44.91839599609375,
      "activations/layer9_attention_weight_min": -45.06824493408203,
      "epoch": 17.43,
      "learning_rate": 3.797045454545454e-05,
      "loss": 2.7408,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.4871,
      "eval_samples_per_second": 505.945,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.4871,
      "eval_openwebtext_samples_per_second": 505.945,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9318,
      "eval_wikitext_samples_per_second": 236.049,
      "step": 300000
    },
    {
      "epoch": 17.43,
      "eval_lambada_loss": 2.46875,
      "eval_lambada_ppl": 11.807678025521156,
      "eval_lambada_runtime": 9.5445,
      "eval_lambada_samples_per_second": 510.136,
      "step": 300000
    },
    {
      "activations/layer0_attention_weight_max": 15.351518630981445,
      "activations/layer0_attention_weight_min": -13.231919288635254,
      "activations/layer10_attention_weight_max": 37.230587005615234,
      "activations/layer10_attention_weight_min": -39.1384162902832,
      "activations/layer11_attention_weight_max": 37.081050872802734,
      "activations/layer11_attention_weight_min": -36.66141891479492,
      "activations/layer12_attention_weight_max": 19.371679306030273,
      "activations/layer12_attention_weight_min": -29.223350524902344,
      "activations/layer13_attention_weight_max": 35.152862548828125,
      "activations/layer13_attention_weight_min": -27.139060974121094,
      "activations/layer14_attention_weight_max": 38.60984420776367,
      "activations/layer14_attention_weight_min": -32.110774993896484,
      "activations/layer15_attention_weight_max": 37.66778564453125,
      "activations/layer15_attention_weight_min": -30.290924072265625,
      "activations/layer16_attention_weight_max": 38.67896270751953,
      "activations/layer16_attention_weight_min": -33.11530685424805,
      "activations/layer17_attention_weight_max": 61.61992263793945,
      "activations/layer17_attention_weight_min": -48.54671096801758,
      "activations/layer18_attention_weight_max": 55.06558609008789,
      "activations/layer18_attention_weight_min": -41.717472076416016,
      "activations/layer19_attention_weight_max": 27.326940536499023,
      "activations/layer19_attention_weight_min": -23.52260971069336,
      "activations/layer1_attention_weight_max": 16.05755615234375,
      "activations/layer1_attention_weight_min": -14.874387741088867,
      "activations/layer20_attention_weight_max": 27.018177032470703,
      "activations/layer20_attention_weight_min": -20.906091690063477,
      "activations/layer21_attention_weight_max": 54.072105407714844,
      "activations/layer21_attention_weight_min": -21.887649536132812,
      "activations/layer22_attention_weight_max": 33.8819694519043,
      "activations/layer22_attention_weight_min": -25.07975959777832,
      "activations/layer23_attention_weight_max": 41.67327117919922,
      "activations/layer23_attention_weight_min": -24.50240707397461,
      "activations/layer2_attention_weight_max": 32.282470703125,
      "activations/layer2_attention_weight_min": -31.16122055053711,
      "activations/layer3_attention_weight_max": 90.77146911621094,
      "activations/layer3_attention_weight_min": -91.10506439208984,
      "activations/layer4_attention_weight_max": 90.61201477050781,
      "activations/layer4_attention_weight_min": -86.68291473388672,
      "activations/layer5_attention_weight_max": 66.8292236328125,
      "activations/layer5_attention_weight_min": -76.8553466796875,
      "activations/layer6_attention_weight_max": 50.265689849853516,
      "activations/layer6_attention_weight_min": -50.93850326538086,
      "activations/layer7_attention_weight_max": 65.30488586425781,
      "activations/layer7_attention_weight_min": -68.05384826660156,
      "activations/layer8_attention_weight_max": 51.51395034790039,
      "activations/layer8_attention_weight_min": -50.0127067565918,
      "activations/layer9_attention_weight_max": 50.98623275756836,
      "activations/layer9_attention_weight_min": -47.141841888427734,
      "epoch": 17.43,
      "learning_rate": 3.795151515151515e-05,
      "loss": 2.7309,
      "step": 300050
    },
    {
      "activations/layer0_attention_weight_max": 15.23115062713623,
      "activations/layer0_attention_weight_min": -13.276987075805664,
      "activations/layer10_attention_weight_max": 35.84461212158203,
      "activations/layer10_attention_weight_min": -37.20042037963867,
      "activations/layer11_attention_weight_max": 32.931480407714844,
      "activations/layer11_attention_weight_min": -32.78468704223633,
      "activations/layer12_attention_weight_max": 17.9068660736084,
      "activations/layer12_attention_weight_min": -38.846900939941406,
      "activations/layer13_attention_weight_max": 35.53785705566406,
      "activations/layer13_attention_weight_min": -28.620445251464844,
      "activations/layer14_attention_weight_max": 32.11559295654297,
      "activations/layer14_attention_weight_min": -29.375293731689453,
      "activations/layer15_attention_weight_max": 30.834463119506836,
      "activations/layer15_attention_weight_min": -28.32071876525879,
      "activations/layer16_attention_weight_max": 32.26900100708008,
      "activations/layer16_attention_weight_min": -31.8148193359375,
      "activations/layer17_attention_weight_max": 49.137908935546875,
      "activations/layer17_attention_weight_min": -44.45732498168945,
      "activations/layer18_attention_weight_max": 43.49959182739258,
      "activations/layer18_attention_weight_min": -39.408443450927734,
      "activations/layer19_attention_weight_max": 20.757854461669922,
      "activations/layer19_attention_weight_min": -22.32771110534668,
      "activations/layer1_attention_weight_max": 15.93854808807373,
      "activations/layer1_attention_weight_min": -15.719930648803711,
      "activations/layer20_attention_weight_max": 20.706266403198242,
      "activations/layer20_attention_weight_min": -22.714536666870117,
      "activations/layer21_attention_weight_max": 34.95269775390625,
      "activations/layer21_attention_weight_min": -25.23268699645996,
      "activations/layer22_attention_weight_max": 24.663484573364258,
      "activations/layer22_attention_weight_min": -24.5759334564209,
      "activations/layer23_attention_weight_max": 31.797088623046875,
      "activations/layer23_attention_weight_min": -26.035802841186523,
      "activations/layer2_attention_weight_max": 32.156166076660156,
      "activations/layer2_attention_weight_min": -33.802494049072266,
      "activations/layer3_attention_weight_max": 89.90239715576172,
      "activations/layer3_attention_weight_min": -94.57609558105469,
      "activations/layer4_attention_weight_max": 85.96470642089844,
      "activations/layer4_attention_weight_min": -84.44224548339844,
      "activations/layer5_attention_weight_max": 64.46305847167969,
      "activations/layer5_attention_weight_min": -76.61077880859375,
      "activations/layer6_attention_weight_max": 47.51626968383789,
      "activations/layer6_attention_weight_min": -49.000267028808594,
      "activations/layer7_attention_weight_max": 71.36688995361328,
      "activations/layer7_attention_weight_min": -62.451385498046875,
      "activations/layer8_attention_weight_max": 46.49308776855469,
      "activations/layer8_attention_weight_min": -46.328948974609375,
      "activations/layer9_attention_weight_max": 53.0480842590332,
      "activations/layer9_attention_weight_min": -46.33881759643555,
      "epoch": 17.44,
      "learning_rate": 3.793257575757575e-05,
      "loss": 2.726,
      "step": 300100
    },
    {
      "activations/layer0_attention_weight_max": 15.528024673461914,
      "activations/layer0_attention_weight_min": -13.306157112121582,
      "activations/layer10_attention_weight_max": 35.27016067504883,
      "activations/layer10_attention_weight_min": -34.19293975830078,
      "activations/layer11_attention_weight_max": 31.687335968017578,
      "activations/layer11_attention_weight_min": -33.574737548828125,
      "activations/layer12_attention_weight_max": 18.608564376831055,
      "activations/layer12_attention_weight_min": -21.904624938964844,
      "activations/layer13_attention_weight_max": 33.59388732910156,
      "activations/layer13_attention_weight_min": -27.013383865356445,
      "activations/layer14_attention_weight_max": 35.58226013183594,
      "activations/layer14_attention_weight_min": -30.05364418029785,
      "activations/layer15_attention_weight_max": 33.87281036376953,
      "activations/layer15_attention_weight_min": -29.496366500854492,
      "activations/layer16_attention_weight_max": 31.836225509643555,
      "activations/layer16_attention_weight_min": -32.9174919128418,
      "activations/layer17_attention_weight_max": 50.665035247802734,
      "activations/layer17_attention_weight_min": -44.16912078857422,
      "activations/layer18_attention_weight_max": 45.002716064453125,
      "activations/layer18_attention_weight_min": -38.55461502075195,
      "activations/layer19_attention_weight_max": 27.211532592773438,
      "activations/layer19_attention_weight_min": -23.805641174316406,
      "activations/layer1_attention_weight_max": 16.426586151123047,
      "activations/layer1_attention_weight_min": -14.121087074279785,
      "activations/layer20_attention_weight_max": 25.368295669555664,
      "activations/layer20_attention_weight_min": -21.81857681274414,
      "activations/layer21_attention_weight_max": 39.162109375,
      "activations/layer21_attention_weight_min": -24.509763717651367,
      "activations/layer22_attention_weight_max": 30.410701751708984,
      "activations/layer22_attention_weight_min": -26.32056427001953,
      "activations/layer23_attention_weight_max": 36.53390121459961,
      "activations/layer23_attention_weight_min": -24.431325912475586,
      "activations/layer2_attention_weight_max": 31.857524871826172,
      "activations/layer2_attention_weight_min": -29.808250427246094,
      "activations/layer3_attention_weight_max": 91.04280090332031,
      "activations/layer3_attention_weight_min": -89.88926696777344,
      "activations/layer4_attention_weight_max": 89.03750610351562,
      "activations/layer4_attention_weight_min": -89.15019989013672,
      "activations/layer5_attention_weight_max": 64.31818389892578,
      "activations/layer5_attention_weight_min": -75.62117004394531,
      "activations/layer6_attention_weight_max": 50.34290313720703,
      "activations/layer6_attention_weight_min": -50.91525650024414,
      "activations/layer7_attention_weight_max": 63.9850959777832,
      "activations/layer7_attention_weight_min": -64.45447540283203,
      "activations/layer8_attention_weight_max": 44.94267654418945,
      "activations/layer8_attention_weight_min": -48.19426345825195,
      "activations/layer9_attention_weight_max": 43.702877044677734,
      "activations/layer9_attention_weight_min": -45.0038948059082,
      "epoch": 17.44,
      "learning_rate": 3.7913636363636354e-05,
      "loss": 2.7428,
      "step": 300150
    },
    {
      "activations/layer0_attention_weight_max": 15.537497520446777,
      "activations/layer0_attention_weight_min": -13.203193664550781,
      "activations/layer10_attention_weight_max": 35.09476852416992,
      "activations/layer10_attention_weight_min": -34.380733489990234,
      "activations/layer11_attention_weight_max": 33.97857666015625,
      "activations/layer11_attention_weight_min": -33.8780403137207,
      "activations/layer12_attention_weight_max": 22.892515182495117,
      "activations/layer12_attention_weight_min": -29.951122283935547,
      "activations/layer13_attention_weight_max": 34.739253997802734,
      "activations/layer13_attention_weight_min": -27.201730728149414,
      "activations/layer14_attention_weight_max": 39.80539321899414,
      "activations/layer14_attention_weight_min": -33.24474334716797,
      "activations/layer15_attention_weight_max": 35.1766357421875,
      "activations/layer15_attention_weight_min": -33.81671905517578,
      "activations/layer16_attention_weight_max": 38.429359436035156,
      "activations/layer16_attention_weight_min": -34.70879364013672,
      "activations/layer17_attention_weight_max": 57.20563507080078,
      "activations/layer17_attention_weight_min": -52.96818923950195,
      "activations/layer18_attention_weight_max": 56.106285095214844,
      "activations/layer18_attention_weight_min": -47.74085235595703,
      "activations/layer19_attention_weight_max": 25.8289794921875,
      "activations/layer19_attention_weight_min": -26.350637435913086,
      "activations/layer1_attention_weight_max": 15.506950378417969,
      "activations/layer1_attention_weight_min": -16.64226722717285,
      "activations/layer20_attention_weight_max": 23.322834014892578,
      "activations/layer20_attention_weight_min": -22.922658920288086,
      "activations/layer21_attention_weight_max": 38.36090087890625,
      "activations/layer21_attention_weight_min": -27.40286636352539,
      "activations/layer22_attention_weight_max": 31.043508529663086,
      "activations/layer22_attention_weight_min": -27.84124183654785,
      "activations/layer23_attention_weight_max": 40.828914642333984,
      "activations/layer23_attention_weight_min": -23.233440399169922,
      "activations/layer2_attention_weight_max": 33.572608947753906,
      "activations/layer2_attention_weight_min": -30.03895378112793,
      "activations/layer3_attention_weight_max": 87.63416290283203,
      "activations/layer3_attention_weight_min": -89.88594818115234,
      "activations/layer4_attention_weight_max": 88.7645492553711,
      "activations/layer4_attention_weight_min": -86.48242950439453,
      "activations/layer5_attention_weight_max": 64.22193908691406,
      "activations/layer5_attention_weight_min": -76.34759521484375,
      "activations/layer6_attention_weight_max": 51.99098587036133,
      "activations/layer6_attention_weight_min": -51.71892166137695,
      "activations/layer7_attention_weight_max": 67.28412628173828,
      "activations/layer7_attention_weight_min": -62.5529670715332,
      "activations/layer8_attention_weight_max": 44.46229934692383,
      "activations/layer8_attention_weight_min": -48.84838104248047,
      "activations/layer9_attention_weight_max": 45.09659194946289,
      "activations/layer9_attention_weight_min": -46.1510124206543,
      "epoch": 17.44,
      "learning_rate": 3.789469696969697e-05,
      "loss": 2.7404,
      "step": 300200
    },
    {
      "activations/layer0_attention_weight_max": 15.687950134277344,
      "activations/layer0_attention_weight_min": -13.441388130187988,
      "activations/layer10_attention_weight_max": 38.587677001953125,
      "activations/layer10_attention_weight_min": -40.60516357421875,
      "activations/layer11_attention_weight_max": 33.33161163330078,
      "activations/layer11_attention_weight_min": -34.710723876953125,
      "activations/layer12_attention_weight_max": 19.382457733154297,
      "activations/layer12_attention_weight_min": -23.409372329711914,
      "activations/layer13_attention_weight_max": 32.2780876159668,
      "activations/layer13_attention_weight_min": -29.143966674804688,
      "activations/layer14_attention_weight_max": 34.87501907348633,
      "activations/layer14_attention_weight_min": -29.17606544494629,
      "activations/layer15_attention_weight_max": 33.5496940612793,
      "activations/layer15_attention_weight_min": -29.472537994384766,
      "activations/layer16_attention_weight_max": 33.2762565612793,
      "activations/layer16_attention_weight_min": -31.641504287719727,
      "activations/layer17_attention_weight_max": 52.71291732788086,
      "activations/layer17_attention_weight_min": -45.4542121887207,
      "activations/layer18_attention_weight_max": 48.49190902709961,
      "activations/layer18_attention_weight_min": -40.07271957397461,
      "activations/layer19_attention_weight_max": 22.821386337280273,
      "activations/layer19_attention_weight_min": -24.354970932006836,
      "activations/layer1_attention_weight_max": 17.764009475708008,
      "activations/layer1_attention_weight_min": -15.851388931274414,
      "activations/layer20_attention_weight_max": 22.335657119750977,
      "activations/layer20_attention_weight_min": -23.108640670776367,
      "activations/layer21_attention_weight_max": 44.657806396484375,
      "activations/layer21_attention_weight_min": -27.59379768371582,
      "activations/layer22_attention_weight_max": 29.56995964050293,
      "activations/layer22_attention_weight_min": -25.317657470703125,
      "activations/layer23_attention_weight_max": 36.263153076171875,
      "activations/layer23_attention_weight_min": -27.03096580505371,
      "activations/layer2_attention_weight_max": 33.6919059753418,
      "activations/layer2_attention_weight_min": -31.195884704589844,
      "activations/layer3_attention_weight_max": 91.65149688720703,
      "activations/layer3_attention_weight_min": -90.9585189819336,
      "activations/layer4_attention_weight_max": 88.29371643066406,
      "activations/layer4_attention_weight_min": -85.66400909423828,
      "activations/layer5_attention_weight_max": 64.71717071533203,
      "activations/layer5_attention_weight_min": -73.95545959472656,
      "activations/layer6_attention_weight_max": 48.87142562866211,
      "activations/layer6_attention_weight_min": -48.461734771728516,
      "activations/layer7_attention_weight_max": 62.74515914916992,
      "activations/layer7_attention_weight_min": -63.234764099121094,
      "activations/layer8_attention_weight_max": 48.55931854248047,
      "activations/layer8_attention_weight_min": -49.07456970214844,
      "activations/layer9_attention_weight_max": 43.800228118896484,
      "activations/layer9_attention_weight_min": -45.63393783569336,
      "epoch": 17.45,
      "learning_rate": 3.787575757575757e-05,
      "loss": 2.7281,
      "step": 300250
    },
    {
      "activations/layer0_attention_weight_max": 15.309804916381836,
      "activations/layer0_attention_weight_min": -13.531952857971191,
      "activations/layer10_attention_weight_max": 33.62062454223633,
      "activations/layer10_attention_weight_min": -34.575843811035156,
      "activations/layer11_attention_weight_max": 31.207645416259766,
      "activations/layer11_attention_weight_min": -33.106101989746094,
      "activations/layer12_attention_weight_max": 22.283897399902344,
      "activations/layer12_attention_weight_min": -25.574705123901367,
      "activations/layer13_attention_weight_max": 32.268367767333984,
      "activations/layer13_attention_weight_min": -28.511646270751953,
      "activations/layer14_attention_weight_max": 37.51804733276367,
      "activations/layer14_attention_weight_min": -32.26836395263672,
      "activations/layer15_attention_weight_max": 33.17977523803711,
      "activations/layer15_attention_weight_min": -30.833986282348633,
      "activations/layer16_attention_weight_max": 36.316932678222656,
      "activations/layer16_attention_weight_min": -33.79022216796875,
      "activations/layer17_attention_weight_max": 53.14414596557617,
      "activations/layer17_attention_weight_min": -45.84196853637695,
      "activations/layer18_attention_weight_max": 49.25715255737305,
      "activations/layer18_attention_weight_min": -40.2222900390625,
      "activations/layer19_attention_weight_max": 27.098037719726562,
      "activations/layer19_attention_weight_min": -26.143003463745117,
      "activations/layer1_attention_weight_max": 16.286800384521484,
      "activations/layer1_attention_weight_min": -15.206093788146973,
      "activations/layer20_attention_weight_max": 26.23919677734375,
      "activations/layer20_attention_weight_min": -22.893400192260742,
      "activations/layer21_attention_weight_max": 34.78550338745117,
      "activations/layer21_attention_weight_min": -22.4693546295166,
      "activations/layer22_attention_weight_max": 29.380512237548828,
      "activations/layer22_attention_weight_min": -27.83736228942871,
      "activations/layer23_attention_weight_max": 37.10792541503906,
      "activations/layer23_attention_weight_min": -25.99898910522461,
      "activations/layer2_attention_weight_max": 32.80815124511719,
      "activations/layer2_attention_weight_min": -30.263309478759766,
      "activations/layer3_attention_weight_max": 87.52206420898438,
      "activations/layer3_attention_weight_min": -88.25455474853516,
      "activations/layer4_attention_weight_max": 87.1427993774414,
      "activations/layer4_attention_weight_min": -82.95606994628906,
      "activations/layer5_attention_weight_max": 67.70255279541016,
      "activations/layer5_attention_weight_min": -76.45028686523438,
      "activations/layer6_attention_weight_max": 48.73561096191406,
      "activations/layer6_attention_weight_min": -49.81853103637695,
      "activations/layer7_attention_weight_max": 67.48703002929688,
      "activations/layer7_attention_weight_min": -65.83133697509766,
      "activations/layer8_attention_weight_max": 43.78183364868164,
      "activations/layer8_attention_weight_min": -48.12849807739258,
      "activations/layer9_attention_weight_max": 41.65216064453125,
      "activations/layer9_attention_weight_min": -45.768096923828125,
      "epoch": 17.45,
      "learning_rate": 3.785681818181818e-05,
      "loss": 2.7419,
      "step": 300300
    },
    {
      "activations/layer0_attention_weight_max": 15.44435977935791,
      "activations/layer0_attention_weight_min": -13.451735496520996,
      "activations/layer10_attention_weight_max": 34.548519134521484,
      "activations/layer10_attention_weight_min": -35.34851837158203,
      "activations/layer11_attention_weight_max": 32.8982048034668,
      "activations/layer11_attention_weight_min": -34.01835250854492,
      "activations/layer12_attention_weight_max": 19.109994888305664,
      "activations/layer12_attention_weight_min": -26.028600692749023,
      "activations/layer13_attention_weight_max": 32.77387619018555,
      "activations/layer13_attention_weight_min": -28.109272003173828,
      "activations/layer14_attention_weight_max": 34.2001838684082,
      "activations/layer14_attention_weight_min": -30.47382926940918,
      "activations/layer15_attention_weight_max": 33.0350227355957,
      "activations/layer15_attention_weight_min": -30.465290069580078,
      "activations/layer16_attention_weight_max": 36.852413177490234,
      "activations/layer16_attention_weight_min": -33.52664566040039,
      "activations/layer17_attention_weight_max": 53.3544921875,
      "activations/layer17_attention_weight_min": -47.15869140625,
      "activations/layer18_attention_weight_max": 51.162376403808594,
      "activations/layer18_attention_weight_min": -41.41777801513672,
      "activations/layer19_attention_weight_max": 24.87079429626465,
      "activations/layer19_attention_weight_min": -26.11224937438965,
      "activations/layer1_attention_weight_max": 15.7019681930542,
      "activations/layer1_attention_weight_min": -14.546929359436035,
      "activations/layer20_attention_weight_max": 24.42255210876465,
      "activations/layer20_attention_weight_min": -22.586454391479492,
      "activations/layer21_attention_weight_max": 36.339515686035156,
      "activations/layer21_attention_weight_min": -23.28775978088379,
      "activations/layer22_attention_weight_max": 32.45675277709961,
      "activations/layer22_attention_weight_min": -29.56574058532715,
      "activations/layer23_attention_weight_max": 37.20566177368164,
      "activations/layer23_attention_weight_min": -25.132644653320312,
      "activations/layer2_attention_weight_max": 32.26847839355469,
      "activations/layer2_attention_weight_min": -29.29523468017578,
      "activations/layer3_attention_weight_max": 89.02984619140625,
      "activations/layer3_attention_weight_min": -89.94803619384766,
      "activations/layer4_attention_weight_max": 84.52281188964844,
      "activations/layer4_attention_weight_min": -87.7450942993164,
      "activations/layer5_attention_weight_max": 64.58058166503906,
      "activations/layer5_attention_weight_min": -80.02314758300781,
      "activations/layer6_attention_weight_max": 48.2260627746582,
      "activations/layer6_attention_weight_min": -49.93264389038086,
      "activations/layer7_attention_weight_max": 62.4249153137207,
      "activations/layer7_attention_weight_min": -62.84454345703125,
      "activations/layer8_attention_weight_max": 44.35562515258789,
      "activations/layer8_attention_weight_min": -49.68159866333008,
      "activations/layer9_attention_weight_max": 41.662899017333984,
      "activations/layer9_attention_weight_min": -45.0953483581543,
      "epoch": 17.45,
      "learning_rate": 3.783825757575757e-05,
      "loss": 2.7255,
      "step": 300350
    },
    {
      "activations/layer0_attention_weight_max": 15.015393257141113,
      "activations/layer0_attention_weight_min": -13.410466194152832,
      "activations/layer10_attention_weight_max": 34.22511672973633,
      "activations/layer10_attention_weight_min": -34.967796325683594,
      "activations/layer11_attention_weight_max": 33.5678825378418,
      "activations/layer11_attention_weight_min": -36.06937026977539,
      "activations/layer12_attention_weight_max": 20.016658782958984,
      "activations/layer12_attention_weight_min": -31.107872009277344,
      "activations/layer13_attention_weight_max": 34.32337188720703,
      "activations/layer13_attention_weight_min": -31.874374389648438,
      "activations/layer14_attention_weight_max": 37.91195297241211,
      "activations/layer14_attention_weight_min": -32.85641098022461,
      "activations/layer15_attention_weight_max": 35.48501205444336,
      "activations/layer15_attention_weight_min": -32.03908920288086,
      "activations/layer16_attention_weight_max": 36.554439544677734,
      "activations/layer16_attention_weight_min": -33.11871337890625,
      "activations/layer17_attention_weight_max": 55.257568359375,
      "activations/layer17_attention_weight_min": -47.679046630859375,
      "activations/layer18_attention_weight_max": 47.41470718383789,
      "activations/layer18_attention_weight_min": -43.6040153503418,
      "activations/layer19_attention_weight_max": 23.54462242126465,
      "activations/layer19_attention_weight_min": -25.174528121948242,
      "activations/layer1_attention_weight_max": 16.311046600341797,
      "activations/layer1_attention_weight_min": -14.536742210388184,
      "activations/layer20_attention_weight_max": 22.336519241333008,
      "activations/layer20_attention_weight_min": -22.886676788330078,
      "activations/layer21_attention_weight_max": 35.52084732055664,
      "activations/layer21_attention_weight_min": -27.035770416259766,
      "activations/layer22_attention_weight_max": 30.349674224853516,
      "activations/layer22_attention_weight_min": -26.348630905151367,
      "activations/layer23_attention_weight_max": 33.719520568847656,
      "activations/layer23_attention_weight_min": -28.955524444580078,
      "activations/layer2_attention_weight_max": 31.76595115661621,
      "activations/layer2_attention_weight_min": -31.531078338623047,
      "activations/layer3_attention_weight_max": 90.20088195800781,
      "activations/layer3_attention_weight_min": -96.72940826416016,
      "activations/layer4_attention_weight_max": 87.56194305419922,
      "activations/layer4_attention_weight_min": -86.2962646484375,
      "activations/layer5_attention_weight_max": 65.49444580078125,
      "activations/layer5_attention_weight_min": -76.36111450195312,
      "activations/layer6_attention_weight_max": 53.473209381103516,
      "activations/layer6_attention_weight_min": -49.70100021362305,
      "activations/layer7_attention_weight_max": 71.17899322509766,
      "activations/layer7_attention_weight_min": -64.95409393310547,
      "activations/layer8_attention_weight_max": 48.03339385986328,
      "activations/layer8_attention_weight_min": -48.3153076171875,
      "activations/layer9_attention_weight_max": 46.48176193237305,
      "activations/layer9_attention_weight_min": -43.8487663269043,
      "epoch": 17.45,
      "learning_rate": 3.781931818181818e-05,
      "loss": 2.7469,
      "step": 300400
    },
    {
      "activations/layer0_attention_weight_max": 15.504129409790039,
      "activations/layer0_attention_weight_min": -13.668476104736328,
      "activations/layer10_attention_weight_max": 38.83100128173828,
      "activations/layer10_attention_weight_min": -38.19685745239258,
      "activations/layer11_attention_weight_max": 34.364715576171875,
      "activations/layer11_attention_weight_min": -33.19615936279297,
      "activations/layer12_attention_weight_max": 19.818822860717773,
      "activations/layer12_attention_weight_min": -24.83086585998535,
      "activations/layer13_attention_weight_max": 34.88036346435547,
      "activations/layer13_attention_weight_min": -27.90606117248535,
      "activations/layer14_attention_weight_max": 35.57171630859375,
      "activations/layer14_attention_weight_min": -31.67238998413086,
      "activations/layer15_attention_weight_max": 32.228492736816406,
      "activations/layer15_attention_weight_min": -30.972763061523438,
      "activations/layer16_attention_weight_max": 34.262474060058594,
      "activations/layer16_attention_weight_min": -32.36189651489258,
      "activations/layer17_attention_weight_max": 50.189537048339844,
      "activations/layer17_attention_weight_min": -45.90370178222656,
      "activations/layer18_attention_weight_max": 47.43892288208008,
      "activations/layer18_attention_weight_min": -42.75786209106445,
      "activations/layer19_attention_weight_max": 26.27093505859375,
      "activations/layer19_attention_weight_min": -21.69322967529297,
      "activations/layer1_attention_weight_max": 16.18137550354004,
      "activations/layer1_attention_weight_min": -13.90705394744873,
      "activations/layer20_attention_weight_max": 23.3290958404541,
      "activations/layer20_attention_weight_min": -21.666519165039062,
      "activations/layer21_attention_weight_max": 35.17589569091797,
      "activations/layer21_attention_weight_min": -21.304523468017578,
      "activations/layer22_attention_weight_max": 27.275714874267578,
      "activations/layer22_attention_weight_min": -25.559642791748047,
      "activations/layer23_attention_weight_max": 34.341835021972656,
      "activations/layer23_attention_weight_min": -24.584190368652344,
      "activations/layer2_attention_weight_max": 31.03542137145996,
      "activations/layer2_attention_weight_min": -29.192855834960938,
      "activations/layer3_attention_weight_max": 89.74945068359375,
      "activations/layer3_attention_weight_min": -93.01940155029297,
      "activations/layer4_attention_weight_max": 87.92369842529297,
      "activations/layer4_attention_weight_min": -86.7203369140625,
      "activations/layer5_attention_weight_max": 62.227413177490234,
      "activations/layer5_attention_weight_min": -78.27677154541016,
      "activations/layer6_attention_weight_max": 49.402122497558594,
      "activations/layer6_attention_weight_min": -50.3056526184082,
      "activations/layer7_attention_weight_max": 65.23164367675781,
      "activations/layer7_attention_weight_min": -65.27703094482422,
      "activations/layer8_attention_weight_max": 47.3028678894043,
      "activations/layer8_attention_weight_min": -51.63785171508789,
      "activations/layer9_attention_weight_max": 46.847042083740234,
      "activations/layer9_attention_weight_min": -46.00125503540039,
      "epoch": 17.46,
      "learning_rate": 3.7800378787878787e-05,
      "loss": 2.7271,
      "step": 300450
    },
    {
      "activations/layer0_attention_weight_max": 15.511880874633789,
      "activations/layer0_attention_weight_min": -13.37880802154541,
      "activations/layer10_attention_weight_max": 40.4755859375,
      "activations/layer10_attention_weight_min": -38.81809997558594,
      "activations/layer11_attention_weight_max": 38.782005310058594,
      "activations/layer11_attention_weight_min": -34.79167175292969,
      "activations/layer12_attention_weight_max": 19.322263717651367,
      "activations/layer12_attention_weight_min": -24.592910766601562,
      "activations/layer13_attention_weight_max": 29.96091079711914,
      "activations/layer13_attention_weight_min": -30.634323120117188,
      "activations/layer14_attention_weight_max": 34.86267852783203,
      "activations/layer14_attention_weight_min": -32.57954406738281,
      "activations/layer15_attention_weight_max": 31.51887321472168,
      "activations/layer15_attention_weight_min": -33.55097579956055,
      "activations/layer16_attention_weight_max": 36.7637939453125,
      "activations/layer16_attention_weight_min": -34.14897155761719,
      "activations/layer17_attention_weight_max": 56.567073822021484,
      "activations/layer17_attention_weight_min": -52.781166076660156,
      "activations/layer18_attention_weight_max": 50.40469741821289,
      "activations/layer18_attention_weight_min": -44.86778259277344,
      "activations/layer19_attention_weight_max": 25.038991928100586,
      "activations/layer19_attention_weight_min": -25.243053436279297,
      "activations/layer1_attention_weight_max": 16.88487434387207,
      "activations/layer1_attention_weight_min": -13.689730644226074,
      "activations/layer20_attention_weight_max": 20.854551315307617,
      "activations/layer20_attention_weight_min": -25.33572006225586,
      "activations/layer21_attention_weight_max": 34.242645263671875,
      "activations/layer21_attention_weight_min": -22.83485984802246,
      "activations/layer22_attention_weight_max": 28.650606155395508,
      "activations/layer22_attention_weight_min": -27.42536163330078,
      "activations/layer23_attention_weight_max": 36.5499382019043,
      "activations/layer23_attention_weight_min": -23.64282989501953,
      "activations/layer2_attention_weight_max": 33.54861831665039,
      "activations/layer2_attention_weight_min": -31.049360275268555,
      "activations/layer3_attention_weight_max": 97.96221923828125,
      "activations/layer3_attention_weight_min": -93.3079833984375,
      "activations/layer4_attention_weight_max": 95.53527069091797,
      "activations/layer4_attention_weight_min": -86.76339721679688,
      "activations/layer5_attention_weight_max": 67.66961669921875,
      "activations/layer5_attention_weight_min": -74.09623718261719,
      "activations/layer6_attention_weight_max": 49.24406433105469,
      "activations/layer6_attention_weight_min": -49.566585540771484,
      "activations/layer7_attention_weight_max": 69.12720489501953,
      "activations/layer7_attention_weight_min": -62.77528762817383,
      "activations/layer8_attention_weight_max": 49.47745895385742,
      "activations/layer8_attention_weight_min": -46.743473052978516,
      "activations/layer9_attention_weight_max": 58.3364372253418,
      "activations/layer9_attention_weight_min": -46.909934997558594,
      "epoch": 17.46,
      "learning_rate": 3.778143939393939e-05,
      "loss": 2.7491,
      "step": 300500
    },
    {
      "activations/layer0_attention_weight_max": 16.26571273803711,
      "activations/layer0_attention_weight_min": -13.482402801513672,
      "activations/layer10_attention_weight_max": 36.98122787475586,
      "activations/layer10_attention_weight_min": -36.92251205444336,
      "activations/layer11_attention_weight_max": 32.47758483886719,
      "activations/layer11_attention_weight_min": -37.183555603027344,
      "activations/layer12_attention_weight_max": 21.470252990722656,
      "activations/layer12_attention_weight_min": -31.027341842651367,
      "activations/layer13_attention_weight_max": 35.39637756347656,
      "activations/layer13_attention_weight_min": -26.344364166259766,
      "activations/layer14_attention_weight_max": 36.74403762817383,
      "activations/layer14_attention_weight_min": -31.750642776489258,
      "activations/layer15_attention_weight_max": 32.22443389892578,
      "activations/layer15_attention_weight_min": -32.650848388671875,
      "activations/layer16_attention_weight_max": 33.398773193359375,
      "activations/layer16_attention_weight_min": -32.130165100097656,
      "activations/layer17_attention_weight_max": 50.06534957885742,
      "activations/layer17_attention_weight_min": -47.26551818847656,
      "activations/layer18_attention_weight_max": 45.88282012939453,
      "activations/layer18_attention_weight_min": -40.090877532958984,
      "activations/layer19_attention_weight_max": 26.462270736694336,
      "activations/layer19_attention_weight_min": -24.135663986206055,
      "activations/layer1_attention_weight_max": 15.988097190856934,
      "activations/layer1_attention_weight_min": -15.57736873626709,
      "activations/layer20_attention_weight_max": 23.7122745513916,
      "activations/layer20_attention_weight_min": -23.552288055419922,
      "activations/layer21_attention_weight_max": 34.1141357421875,
      "activations/layer21_attention_weight_min": -23.552562713623047,
      "activations/layer22_attention_weight_max": 30.626983642578125,
      "activations/layer22_attention_weight_min": -27.546955108642578,
      "activations/layer23_attention_weight_max": 36.47457504272461,
      "activations/layer23_attention_weight_min": -24.182697296142578,
      "activations/layer2_attention_weight_max": 31.77691650390625,
      "activations/layer2_attention_weight_min": -30.79669189453125,
      "activations/layer3_attention_weight_max": 95.23347473144531,
      "activations/layer3_attention_weight_min": -95.82835388183594,
      "activations/layer4_attention_weight_max": 88.41692352294922,
      "activations/layer4_attention_weight_min": -88.91332244873047,
      "activations/layer5_attention_weight_max": 63.91412353515625,
      "activations/layer5_attention_weight_min": -75.34841918945312,
      "activations/layer6_attention_weight_max": 50.39151382446289,
      "activations/layer6_attention_weight_min": -53.10976791381836,
      "activations/layer7_attention_weight_max": 67.14208984375,
      "activations/layer7_attention_weight_min": -64.25318145751953,
      "activations/layer8_attention_weight_max": 45.41954040527344,
      "activations/layer8_attention_weight_min": -48.84000778198242,
      "activations/layer9_attention_weight_max": 42.911190032958984,
      "activations/layer9_attention_weight_min": -46.45853805541992,
      "epoch": 17.46,
      "learning_rate": 3.776249999999999e-05,
      "loss": 2.7374,
      "step": 300550
    },
    {
      "activations/layer0_attention_weight_max": 15.195937156677246,
      "activations/layer0_attention_weight_min": -13.293139457702637,
      "activations/layer10_attention_weight_max": 35.88701629638672,
      "activations/layer10_attention_weight_min": -35.50933074951172,
      "activations/layer11_attention_weight_max": 32.45648956298828,
      "activations/layer11_attention_weight_min": -33.70227813720703,
      "activations/layer12_attention_weight_max": 19.195310592651367,
      "activations/layer12_attention_weight_min": -24.257455825805664,
      "activations/layer13_attention_weight_max": 38.031089782714844,
      "activations/layer13_attention_weight_min": -28.91423797607422,
      "activations/layer14_attention_weight_max": 36.101402282714844,
      "activations/layer14_attention_weight_min": -28.618471145629883,
      "activations/layer15_attention_weight_max": 32.587867736816406,
      "activations/layer15_attention_weight_min": -28.578977584838867,
      "activations/layer16_attention_weight_max": 33.685302734375,
      "activations/layer16_attention_weight_min": -30.324193954467773,
      "activations/layer17_attention_weight_max": 48.83705520629883,
      "activations/layer17_attention_weight_min": -42.773529052734375,
      "activations/layer18_attention_weight_max": 45.3189697265625,
      "activations/layer18_attention_weight_min": -36.855438232421875,
      "activations/layer19_attention_weight_max": 27.24039077758789,
      "activations/layer19_attention_weight_min": -22.66303825378418,
      "activations/layer1_attention_weight_max": 16.244365692138672,
      "activations/layer1_attention_weight_min": -19.03591537475586,
      "activations/layer20_attention_weight_max": 22.87183952331543,
      "activations/layer20_attention_weight_min": -27.214757919311523,
      "activations/layer21_attention_weight_max": 33.6143798828125,
      "activations/layer21_attention_weight_min": -19.884414672851562,
      "activations/layer22_attention_weight_max": 27.328136444091797,
      "activations/layer22_attention_weight_min": -25.841306686401367,
      "activations/layer23_attention_weight_max": 38.1406364440918,
      "activations/layer23_attention_weight_min": -24.2578182220459,
      "activations/layer2_attention_weight_max": 31.645450592041016,
      "activations/layer2_attention_weight_min": -29.92678451538086,
      "activations/layer3_attention_weight_max": 88.57048797607422,
      "activations/layer3_attention_weight_min": -89.07039642333984,
      "activations/layer4_attention_weight_max": 88.2803726196289,
      "activations/layer4_attention_weight_min": -89.21574401855469,
      "activations/layer5_attention_weight_max": 67.0328140258789,
      "activations/layer5_attention_weight_min": -73.04747009277344,
      "activations/layer6_attention_weight_max": 48.30879211425781,
      "activations/layer6_attention_weight_min": -47.62737274169922,
      "activations/layer7_attention_weight_max": 61.98369216918945,
      "activations/layer7_attention_weight_min": -62.234737396240234,
      "activations/layer8_attention_weight_max": 45.87604522705078,
      "activations/layer8_attention_weight_min": -47.53860855102539,
      "activations/layer9_attention_weight_max": 42.850563049316406,
      "activations/layer9_attention_weight_min": -44.643150329589844,
      "epoch": 17.47,
      "learning_rate": 3.7743560606060605e-05,
      "loss": 2.7427,
      "step": 300600
    },
    {
      "activations/layer0_attention_weight_max": 14.926944732666016,
      "activations/layer0_attention_weight_min": -13.744686126708984,
      "activations/layer10_attention_weight_max": 44.26276397705078,
      "activations/layer10_attention_weight_min": -41.30702590942383,
      "activations/layer11_attention_weight_max": 40.445404052734375,
      "activations/layer11_attention_weight_min": -37.359153747558594,
      "activations/layer12_attention_weight_max": 19.005990982055664,
      "activations/layer12_attention_weight_min": -25.76540756225586,
      "activations/layer13_attention_weight_max": 40.51169967651367,
      "activations/layer13_attention_weight_min": -29.36374855041504,
      "activations/layer14_attention_weight_max": 44.21691131591797,
      "activations/layer14_attention_weight_min": -30.50173568725586,
      "activations/layer15_attention_weight_max": 41.329524993896484,
      "activations/layer15_attention_weight_min": -29.749347686767578,
      "activations/layer16_attention_weight_max": 34.75039291381836,
      "activations/layer16_attention_weight_min": -31.55774688720703,
      "activations/layer17_attention_weight_max": 53.6884765625,
      "activations/layer17_attention_weight_min": -48.7281608581543,
      "activations/layer18_attention_weight_max": 50.021148681640625,
      "activations/layer18_attention_weight_min": -43.821678161621094,
      "activations/layer19_attention_weight_max": 24.101451873779297,
      "activations/layer19_attention_weight_min": -23.18807029724121,
      "activations/layer1_attention_weight_max": 17.250993728637695,
      "activations/layer1_attention_weight_min": -14.547091484069824,
      "activations/layer20_attention_weight_max": 24.835147857666016,
      "activations/layer20_attention_weight_min": -21.198301315307617,
      "activations/layer21_attention_weight_max": 45.007537841796875,
      "activations/layer21_attention_weight_min": -22.799869537353516,
      "activations/layer22_attention_weight_max": 33.012535095214844,
      "activations/layer22_attention_weight_min": -25.121492385864258,
      "activations/layer23_attention_weight_max": 34.03746032714844,
      "activations/layer23_attention_weight_min": -23.545303344726562,
      "activations/layer2_attention_weight_max": 33.64042282104492,
      "activations/layer2_attention_weight_min": -31.339496612548828,
      "activations/layer3_attention_weight_max": 91.48419189453125,
      "activations/layer3_attention_weight_min": -91.55902099609375,
      "activations/layer4_attention_weight_max": 88.97349548339844,
      "activations/layer4_attention_weight_min": -86.55452728271484,
      "activations/layer5_attention_weight_max": 63.04741668701172,
      "activations/layer5_attention_weight_min": -75.18569946289062,
      "activations/layer6_attention_weight_max": 51.57970428466797,
      "activations/layer6_attention_weight_min": -52.33450698852539,
      "activations/layer7_attention_weight_max": 75.35832214355469,
      "activations/layer7_attention_weight_min": -72.37334442138672,
      "activations/layer8_attention_weight_max": 53.33631896972656,
      "activations/layer8_attention_weight_min": -53.920074462890625,
      "activations/layer9_attention_weight_max": 56.10200881958008,
      "activations/layer9_attention_weight_min": -49.740108489990234,
      "epoch": 17.47,
      "learning_rate": 3.7724621212121206e-05,
      "loss": 2.7279,
      "step": 300650
    },
    {
      "activations/layer0_attention_weight_max": 15.433268547058105,
      "activations/layer0_attention_weight_min": -13.176629066467285,
      "activations/layer10_attention_weight_max": 36.18800735473633,
      "activations/layer10_attention_weight_min": -37.381290435791016,
      "activations/layer11_attention_weight_max": 34.521785736083984,
      "activations/layer11_attention_weight_min": -33.88141632080078,
      "activations/layer12_attention_weight_max": 18.782018661499023,
      "activations/layer12_attention_weight_min": -25.273109436035156,
      "activations/layer13_attention_weight_max": 30.55998420715332,
      "activations/layer13_attention_weight_min": -25.918134689331055,
      "activations/layer14_attention_weight_max": 35.04353332519531,
      "activations/layer14_attention_weight_min": -31.4147891998291,
      "activations/layer15_attention_weight_max": 32.4654426574707,
      "activations/layer15_attention_weight_min": -31.869415283203125,
      "activations/layer16_attention_weight_max": 32.37009048461914,
      "activations/layer16_attention_weight_min": -31.036510467529297,
      "activations/layer17_attention_weight_max": 49.8560905456543,
      "activations/layer17_attention_weight_min": -43.06971740722656,
      "activations/layer18_attention_weight_max": 48.618072509765625,
      "activations/layer18_attention_weight_min": -41.80617904663086,
      "activations/layer19_attention_weight_max": 26.486709594726562,
      "activations/layer19_attention_weight_min": -23.39055061340332,
      "activations/layer1_attention_weight_max": 16.27543067932129,
      "activations/layer1_attention_weight_min": -14.897098541259766,
      "activations/layer20_attention_weight_max": 24.365907669067383,
      "activations/layer20_attention_weight_min": -23.069656372070312,
      "activations/layer21_attention_weight_max": 38.51742172241211,
      "activations/layer21_attention_weight_min": -24.498228073120117,
      "activations/layer22_attention_weight_max": 30.18195152282715,
      "activations/layer22_attention_weight_min": -24.17038917541504,
      "activations/layer23_attention_weight_max": 40.518802642822266,
      "activations/layer23_attention_weight_min": -22.752483367919922,
      "activations/layer2_attention_weight_max": 34.1671257019043,
      "activations/layer2_attention_weight_min": -32.32212448120117,
      "activations/layer3_attention_weight_max": 93.80316162109375,
      "activations/layer3_attention_weight_min": -94.81221008300781,
      "activations/layer4_attention_weight_max": 89.1966781616211,
      "activations/layer4_attention_weight_min": -88.33528900146484,
      "activations/layer5_attention_weight_max": 62.42422103881836,
      "activations/layer5_attention_weight_min": -80.1559829711914,
      "activations/layer6_attention_weight_max": 50.6856575012207,
      "activations/layer6_attention_weight_min": -50.15303421020508,
      "activations/layer7_attention_weight_max": 65.83544921875,
      "activations/layer7_attention_weight_min": -68.61079406738281,
      "activations/layer8_attention_weight_max": 44.67565155029297,
      "activations/layer8_attention_weight_min": -47.980594635009766,
      "activations/layer9_attention_weight_max": 47.58008575439453,
      "activations/layer9_attention_weight_min": -48.1233024597168,
      "epoch": 17.47,
      "learning_rate": 3.7705681818181814e-05,
      "loss": 2.7488,
      "step": 300700
    },
    {
      "activations/layer0_attention_weight_max": 15.520098686218262,
      "activations/layer0_attention_weight_min": -13.071608543395996,
      "activations/layer10_attention_weight_max": 38.426124572753906,
      "activations/layer10_attention_weight_min": -37.321327209472656,
      "activations/layer11_attention_weight_max": 39.99663162231445,
      "activations/layer11_attention_weight_min": -35.8542594909668,
      "activations/layer12_attention_weight_max": 18.552648544311523,
      "activations/layer12_attention_weight_min": -25.338302612304688,
      "activations/layer13_attention_weight_max": 33.89766311645508,
      "activations/layer13_attention_weight_min": -26.74527359008789,
      "activations/layer14_attention_weight_max": 36.0085563659668,
      "activations/layer14_attention_weight_min": -29.739049911499023,
      "activations/layer15_attention_weight_max": 32.64569091796875,
      "activations/layer15_attention_weight_min": -29.735370635986328,
      "activations/layer16_attention_weight_max": 34.32369613647461,
      "activations/layer16_attention_weight_min": -31.679447174072266,
      "activations/layer17_attention_weight_max": 54.27402877807617,
      "activations/layer17_attention_weight_min": -46.83895492553711,
      "activations/layer18_attention_weight_max": 45.404541015625,
      "activations/layer18_attention_weight_min": -41.173133850097656,
      "activations/layer19_attention_weight_max": 23.377525329589844,
      "activations/layer19_attention_weight_min": -25.230405807495117,
      "activations/layer1_attention_weight_max": 16.94695281982422,
      "activations/layer1_attention_weight_min": -14.251631736755371,
      "activations/layer20_attention_weight_max": 21.29841423034668,
      "activations/layer20_attention_weight_min": -24.69806480407715,
      "activations/layer21_attention_weight_max": 39.19933319091797,
      "activations/layer21_attention_weight_min": -24.311904907226562,
      "activations/layer22_attention_weight_max": 29.66991424560547,
      "activations/layer22_attention_weight_min": -23.931381225585938,
      "activations/layer23_attention_weight_max": 39.732730865478516,
      "activations/layer23_attention_weight_min": -22.62593650817871,
      "activations/layer2_attention_weight_max": 31.101299285888672,
      "activations/layer2_attention_weight_min": -29.81756591796875,
      "activations/layer3_attention_weight_max": 91.21012878417969,
      "activations/layer3_attention_weight_min": -92.54251861572266,
      "activations/layer4_attention_weight_max": 89.76673126220703,
      "activations/layer4_attention_weight_min": -86.5316390991211,
      "activations/layer5_attention_weight_max": 70.98100280761719,
      "activations/layer5_attention_weight_min": -78.80130004882812,
      "activations/layer6_attention_weight_max": 49.158164978027344,
      "activations/layer6_attention_weight_min": -49.51093673706055,
      "activations/layer7_attention_weight_max": 76.70980072021484,
      "activations/layer7_attention_weight_min": -66.95616912841797,
      "activations/layer8_attention_weight_max": 52.30558776855469,
      "activations/layer8_attention_weight_min": -47.8706169128418,
      "activations/layer9_attention_weight_max": 60.481178283691406,
      "activations/layer9_attention_weight_min": -49.3309440612793,
      "epoch": 17.48,
      "learning_rate": 3.768674242424242e-05,
      "loss": 2.7346,
      "step": 300750
    },
    {
      "activations/layer0_attention_weight_max": 15.752001762390137,
      "activations/layer0_attention_weight_min": -13.21524429321289,
      "activations/layer10_attention_weight_max": 36.277374267578125,
      "activations/layer10_attention_weight_min": -34.44932556152344,
      "activations/layer11_attention_weight_max": 33.64165496826172,
      "activations/layer11_attention_weight_min": -32.92731857299805,
      "activations/layer12_attention_weight_max": 19.608768463134766,
      "activations/layer12_attention_weight_min": -25.338302612304688,
      "activations/layer13_attention_weight_max": 34.562191009521484,
      "activations/layer13_attention_weight_min": -28.95952796936035,
      "activations/layer14_attention_weight_max": 37.664005279541016,
      "activations/layer14_attention_weight_min": -31.729568481445312,
      "activations/layer15_attention_weight_max": 37.34754180908203,
      "activations/layer15_attention_weight_min": -32.48446273803711,
      "activations/layer16_attention_weight_max": 35.0866813659668,
      "activations/layer16_attention_weight_min": -34.29568862915039,
      "activations/layer17_attention_weight_max": 57.94093704223633,
      "activations/layer17_attention_weight_min": -45.864715576171875,
      "activations/layer18_attention_weight_max": 50.94512176513672,
      "activations/layer18_attention_weight_min": -42.20401382446289,
      "activations/layer19_attention_weight_max": 25.54766845703125,
      "activations/layer19_attention_weight_min": -24.841135025024414,
      "activations/layer1_attention_weight_max": 15.88005256652832,
      "activations/layer1_attention_weight_min": -14.562592506408691,
      "activations/layer20_attention_weight_max": 24.067604064941406,
      "activations/layer20_attention_weight_min": -24.228944778442383,
      "activations/layer21_attention_weight_max": 44.24332046508789,
      "activations/layer21_attention_weight_min": -24.413028717041016,
      "activations/layer22_attention_weight_max": 28.75794792175293,
      "activations/layer22_attention_weight_min": -25.98080825805664,
      "activations/layer23_attention_weight_max": 35.71998977661133,
      "activations/layer23_attention_weight_min": -23.254331588745117,
      "activations/layer2_attention_weight_max": 32.423553466796875,
      "activations/layer2_attention_weight_min": -29.655723571777344,
      "activations/layer3_attention_weight_max": 90.7795639038086,
      "activations/layer3_attention_weight_min": -87.58793640136719,
      "activations/layer4_attention_weight_max": 87.50712585449219,
      "activations/layer4_attention_weight_min": -82.24193572998047,
      "activations/layer5_attention_weight_max": 67.70365142822266,
      "activations/layer5_attention_weight_min": -78.04451751708984,
      "activations/layer6_attention_weight_max": 49.1396369934082,
      "activations/layer6_attention_weight_min": -48.980186462402344,
      "activations/layer7_attention_weight_max": 65.70370483398438,
      "activations/layer7_attention_weight_min": -61.91620635986328,
      "activations/layer8_attention_weight_max": 44.4277229309082,
      "activations/layer8_attention_weight_min": -46.554840087890625,
      "activations/layer9_attention_weight_max": 45.0836296081543,
      "activations/layer9_attention_weight_min": -43.879676818847656,
      "epoch": 17.48,
      "learning_rate": 3.766780303030303e-05,
      "loss": 2.7368,
      "step": 300800
    },
    {
      "activations/layer0_attention_weight_max": 15.701903343200684,
      "activations/layer0_attention_weight_min": -13.01395320892334,
      "activations/layer10_attention_weight_max": 35.845458984375,
      "activations/layer10_attention_weight_min": -36.65327835083008,
      "activations/layer11_attention_weight_max": 33.956275939941406,
      "activations/layer11_attention_weight_min": -36.117210388183594,
      "activations/layer12_attention_weight_max": 19.258752822875977,
      "activations/layer12_attention_weight_min": -25.61797523498535,
      "activations/layer13_attention_weight_max": 36.0584716796875,
      "activations/layer13_attention_weight_min": -28.756362915039062,
      "activations/layer14_attention_weight_max": 39.85186004638672,
      "activations/layer14_attention_weight_min": -31.718524932861328,
      "activations/layer15_attention_weight_max": 34.482303619384766,
      "activations/layer15_attention_weight_min": -31.11611557006836,
      "activations/layer16_attention_weight_max": 36.86679458618164,
      "activations/layer16_attention_weight_min": -33.284942626953125,
      "activations/layer17_attention_weight_max": 57.45391082763672,
      "activations/layer17_attention_weight_min": -46.06971740722656,
      "activations/layer18_attention_weight_max": 54.04536437988281,
      "activations/layer18_attention_weight_min": -40.65628433227539,
      "activations/layer19_attention_weight_max": 23.213909149169922,
      "activations/layer19_attention_weight_min": -20.642627716064453,
      "activations/layer1_attention_weight_max": 16.18667221069336,
      "activations/layer1_attention_weight_min": -15.15892505645752,
      "activations/layer20_attention_weight_max": 22.442522048950195,
      "activations/layer20_attention_weight_min": -20.17473030090332,
      "activations/layer21_attention_weight_max": 35.95225143432617,
      "activations/layer21_attention_weight_min": -22.364599227905273,
      "activations/layer22_attention_weight_max": 30.456811904907227,
      "activations/layer22_attention_weight_min": -24.620712280273438,
      "activations/layer23_attention_weight_max": 38.596981048583984,
      "activations/layer23_attention_weight_min": -22.477787017822266,
      "activations/layer2_attention_weight_max": 32.622493743896484,
      "activations/layer2_attention_weight_min": -30.239784240722656,
      "activations/layer3_attention_weight_max": 92.49810028076172,
      "activations/layer3_attention_weight_min": -92.48394775390625,
      "activations/layer4_attention_weight_max": 91.8586196899414,
      "activations/layer4_attention_weight_min": -92.9222640991211,
      "activations/layer5_attention_weight_max": 65.04248046875,
      "activations/layer5_attention_weight_min": -72.57764434814453,
      "activations/layer6_attention_weight_max": 48.634334564208984,
      "activations/layer6_attention_weight_min": -50.57939910888672,
      "activations/layer7_attention_weight_max": 66.99439239501953,
      "activations/layer7_attention_weight_min": -67.22659301757812,
      "activations/layer8_attention_weight_max": 46.876220703125,
      "activations/layer8_attention_weight_min": -49.508216857910156,
      "activations/layer9_attention_weight_max": 46.942195892333984,
      "activations/layer9_attention_weight_min": -46.06092071533203,
      "epoch": 17.48,
      "learning_rate": 3.764886363636363e-05,
      "loss": 2.7155,
      "step": 300850
    },
    {
      "activations/layer0_attention_weight_max": 15.015032768249512,
      "activations/layer0_attention_weight_min": -13.109485626220703,
      "activations/layer10_attention_weight_max": 32.597721099853516,
      "activations/layer10_attention_weight_min": -34.007789611816406,
      "activations/layer11_attention_weight_max": 29.354393005371094,
      "activations/layer11_attention_weight_min": -34.07185363769531,
      "activations/layer12_attention_weight_max": 18.506080627441406,
      "activations/layer12_attention_weight_min": -25.88401222229004,
      "activations/layer13_attention_weight_max": 33.53694152832031,
      "activations/layer13_attention_weight_min": -28.49860191345215,
      "activations/layer14_attention_weight_max": 35.75086212158203,
      "activations/layer14_attention_weight_min": -31.984697341918945,
      "activations/layer15_attention_weight_max": 33.370975494384766,
      "activations/layer15_attention_weight_min": -30.654422760009766,
      "activations/layer16_attention_weight_max": 36.59199523925781,
      "activations/layer16_attention_weight_min": -33.70489501953125,
      "activations/layer17_attention_weight_max": 51.49026107788086,
      "activations/layer17_attention_weight_min": -45.89921569824219,
      "activations/layer18_attention_weight_max": 45.78213882446289,
      "activations/layer18_attention_weight_min": -40.356178283691406,
      "activations/layer19_attention_weight_max": 22.740446090698242,
      "activations/layer19_attention_weight_min": -22.107030868530273,
      "activations/layer1_attention_weight_max": 15.897955894470215,
      "activations/layer1_attention_weight_min": -15.372264862060547,
      "activations/layer20_attention_weight_max": 21.20836639404297,
      "activations/layer20_attention_weight_min": -22.09056854248047,
      "activations/layer21_attention_weight_max": 32.935333251953125,
      "activations/layer21_attention_weight_min": -22.806974411010742,
      "activations/layer22_attention_weight_max": 30.179616928100586,
      "activations/layer22_attention_weight_min": -27.2862606048584,
      "activations/layer23_attention_weight_max": 35.65596389770508,
      "activations/layer23_attention_weight_min": -26.40859031677246,
      "activations/layer2_attention_weight_max": 33.681312561035156,
      "activations/layer2_attention_weight_min": -30.194684982299805,
      "activations/layer3_attention_weight_max": 89.59490203857422,
      "activations/layer3_attention_weight_min": -87.53311157226562,
      "activations/layer4_attention_weight_max": 87.18234252929688,
      "activations/layer4_attention_weight_min": -84.65826416015625,
      "activations/layer5_attention_weight_max": 63.9541130065918,
      "activations/layer5_attention_weight_min": -76.09183502197266,
      "activations/layer6_attention_weight_max": 47.337093353271484,
      "activations/layer6_attention_weight_min": -46.89624786376953,
      "activations/layer7_attention_weight_max": 62.7944450378418,
      "activations/layer7_attention_weight_min": -63.70842361450195,
      "activations/layer8_attention_weight_max": 45.183834075927734,
      "activations/layer8_attention_weight_min": -46.130165100097656,
      "activations/layer9_attention_weight_max": 41.94095230102539,
      "activations/layer9_attention_weight_min": -43.0430908203125,
      "epoch": 17.48,
      "learning_rate": 3.7629924242424234e-05,
      "loss": 2.7114,
      "step": 300900
    },
    {
      "activations/layer0_attention_weight_max": 16.469573974609375,
      "activations/layer0_attention_weight_min": -13.105375289916992,
      "activations/layer10_attention_weight_max": 34.81084060668945,
      "activations/layer10_attention_weight_min": -35.814945220947266,
      "activations/layer11_attention_weight_max": 31.689910888671875,
      "activations/layer11_attention_weight_min": -32.44125747680664,
      "activations/layer12_attention_weight_max": 17.94381332397461,
      "activations/layer12_attention_weight_min": -23.39861297607422,
      "activations/layer13_attention_weight_max": 31.092288970947266,
      "activations/layer13_attention_weight_min": -26.911836624145508,
      "activations/layer14_attention_weight_max": 33.171913146972656,
      "activations/layer14_attention_weight_min": -28.973556518554688,
      "activations/layer15_attention_weight_max": 31.137304306030273,
      "activations/layer15_attention_weight_min": -28.924821853637695,
      "activations/layer16_attention_weight_max": 30.988704681396484,
      "activations/layer16_attention_weight_min": -30.958797454833984,
      "activations/layer17_attention_weight_max": 50.68559265136719,
      "activations/layer17_attention_weight_min": -45.513206481933594,
      "activations/layer18_attention_weight_max": 42.58333969116211,
      "activations/layer18_attention_weight_min": -39.67775344848633,
      "activations/layer19_attention_weight_max": 23.3192195892334,
      "activations/layer19_attention_weight_min": -21.382993698120117,
      "activations/layer1_attention_weight_max": 15.538508415222168,
      "activations/layer1_attention_weight_min": -14.21377944946289,
      "activations/layer20_attention_weight_max": 21.755369186401367,
      "activations/layer20_attention_weight_min": -21.415525436401367,
      "activations/layer21_attention_weight_max": 35.16549301147461,
      "activations/layer21_attention_weight_min": -21.052043914794922,
      "activations/layer22_attention_weight_max": 28.884735107421875,
      "activations/layer22_attention_weight_min": -27.32294273376465,
      "activations/layer23_attention_weight_max": 34.23516845703125,
      "activations/layer23_attention_weight_min": -24.290342330932617,
      "activations/layer2_attention_weight_max": 31.173919677734375,
      "activations/layer2_attention_weight_min": -31.79664421081543,
      "activations/layer3_attention_weight_max": 87.87862396240234,
      "activations/layer3_attention_weight_min": -93.7730484008789,
      "activations/layer4_attention_weight_max": 87.38838958740234,
      "activations/layer4_attention_weight_min": -87.27701568603516,
      "activations/layer5_attention_weight_max": 66.38817596435547,
      "activations/layer5_attention_weight_min": -71.35858917236328,
      "activations/layer6_attention_weight_max": 48.87892532348633,
      "activations/layer6_attention_weight_min": -48.72368621826172,
      "activations/layer7_attention_weight_max": 61.67559814453125,
      "activations/layer7_attention_weight_min": -60.44123458862305,
      "activations/layer8_attention_weight_max": 46.83182907104492,
      "activations/layer8_attention_weight_min": -48.84128952026367,
      "activations/layer9_attention_weight_max": 41.97665023803711,
      "activations/layer9_attention_weight_min": -42.751399993896484,
      "epoch": 17.49,
      "learning_rate": 3.761098484848485e-05,
      "loss": 2.7302,
      "step": 300950
    },
    {
      "activations/layer0_attention_weight_max": 16.277576446533203,
      "activations/layer0_attention_weight_min": -13.014798164367676,
      "activations/layer10_attention_weight_max": 34.27741241455078,
      "activations/layer10_attention_weight_min": -34.3801155090332,
      "activations/layer11_attention_weight_max": 33.97008514404297,
      "activations/layer11_attention_weight_min": -33.73027801513672,
      "activations/layer12_attention_weight_max": 19.988309860229492,
      "activations/layer12_attention_weight_min": -31.057832717895508,
      "activations/layer13_attention_weight_max": 33.44929122924805,
      "activations/layer13_attention_weight_min": -26.05622100830078,
      "activations/layer14_attention_weight_max": 36.23798370361328,
      "activations/layer14_attention_weight_min": -29.385528564453125,
      "activations/layer15_attention_weight_max": 31.94151496887207,
      "activations/layer15_attention_weight_min": -29.08348274230957,
      "activations/layer16_attention_weight_max": 33.965824127197266,
      "activations/layer16_attention_weight_min": -31.426782608032227,
      "activations/layer17_attention_weight_max": 52.39725112915039,
      "activations/layer17_attention_weight_min": -44.54629135131836,
      "activations/layer18_attention_weight_max": 46.158599853515625,
      "activations/layer18_attention_weight_min": -41.24187469482422,
      "activations/layer19_attention_weight_max": 25.301027297973633,
      "activations/layer19_attention_weight_min": -22.036636352539062,
      "activations/layer1_attention_weight_max": 16.0210018157959,
      "activations/layer1_attention_weight_min": -15.871017456054688,
      "activations/layer20_attention_weight_max": 22.50559425354004,
      "activations/layer20_attention_weight_min": -21.231626510620117,
      "activations/layer21_attention_weight_max": 33.89514923095703,
      "activations/layer21_attention_weight_min": -20.74542236328125,
      "activations/layer22_attention_weight_max": 29.41981315612793,
      "activations/layer22_attention_weight_min": -24.01638412475586,
      "activations/layer23_attention_weight_max": 41.634803771972656,
      "activations/layer23_attention_weight_min": -24.5966796875,
      "activations/layer2_attention_weight_max": 32.26959228515625,
      "activations/layer2_attention_weight_min": -31.370269775390625,
      "activations/layer3_attention_weight_max": 94.03277587890625,
      "activations/layer3_attention_weight_min": -97.26260375976562,
      "activations/layer4_attention_weight_max": 89.47655487060547,
      "activations/layer4_attention_weight_min": -88.12299346923828,
      "activations/layer5_attention_weight_max": 67.27462005615234,
      "activations/layer5_attention_weight_min": -75.42476654052734,
      "activations/layer6_attention_weight_max": 49.943824768066406,
      "activations/layer6_attention_weight_min": -50.64552307128906,
      "activations/layer7_attention_weight_max": 66.101806640625,
      "activations/layer7_attention_weight_min": -63.500911712646484,
      "activations/layer8_attention_weight_max": 45.30927276611328,
      "activations/layer8_attention_weight_min": -46.67898941040039,
      "activations/layer9_attention_weight_max": 49.689735412597656,
      "activations/layer9_attention_weight_min": -44.99807357788086,
      "epoch": 17.49,
      "learning_rate": 3.759204545454545e-05,
      "loss": 2.7363,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4621,
      "eval_samples_per_second": 507.44,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4621,
      "eval_openwebtext_samples_per_second": 507.44,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9907,
      "eval_wikitext_samples_per_second": 229.071,
      "step": 301000
    },
    {
      "epoch": 17.49,
      "eval_lambada_loss": 2.48828125,
      "eval_lambada_ppl": 12.040563607027826,
      "eval_lambada_runtime": 9.5314,
      "eval_lambada_samples_per_second": 510.84,
      "step": 301000
    },
    {
      "activations/layer0_attention_weight_max": 15.906373023986816,
      "activations/layer0_attention_weight_min": -12.83609390258789,
      "activations/layer10_attention_weight_max": 37.33033752441406,
      "activations/layer10_attention_weight_min": -35.20702362060547,
      "activations/layer11_attention_weight_max": 32.80927276611328,
      "activations/layer11_attention_weight_min": -33.12192153930664,
      "activations/layer12_attention_weight_max": 20.568824768066406,
      "activations/layer12_attention_weight_min": -22.432601928710938,
      "activations/layer13_attention_weight_max": 36.44131851196289,
      "activations/layer13_attention_weight_min": -27.773746490478516,
      "activations/layer14_attention_weight_max": 39.33606719970703,
      "activations/layer14_attention_weight_min": -32.84251022338867,
      "activations/layer15_attention_weight_max": 40.2584342956543,
      "activations/layer15_attention_weight_min": -33.34771728515625,
      "activations/layer16_attention_weight_max": 37.21287155151367,
      "activations/layer16_attention_weight_min": -35.465553283691406,
      "activations/layer17_attention_weight_max": 54.785682678222656,
      "activations/layer17_attention_weight_min": -46.27266311645508,
      "activations/layer18_attention_weight_max": 49.199459075927734,
      "activations/layer18_attention_weight_min": -43.5206413269043,
      "activations/layer19_attention_weight_max": 24.3752498626709,
      "activations/layer19_attention_weight_min": -24.819494247436523,
      "activations/layer1_attention_weight_max": 15.614139556884766,
      "activations/layer1_attention_weight_min": -14.45704460144043,
      "activations/layer20_attention_weight_max": 26.20696258544922,
      "activations/layer20_attention_weight_min": -25.49445152282715,
      "activations/layer21_attention_weight_max": 46.50258255004883,
      "activations/layer21_attention_weight_min": -22.91819190979004,
      "activations/layer22_attention_weight_max": 32.498291015625,
      "activations/layer22_attention_weight_min": -26.07854461669922,
      "activations/layer23_attention_weight_max": 43.40235137939453,
      "activations/layer23_attention_weight_min": -23.181447982788086,
      "activations/layer2_attention_weight_max": 33.810997009277344,
      "activations/layer2_attention_weight_min": -31.903547286987305,
      "activations/layer3_attention_weight_max": 92.1898193359375,
      "activations/layer3_attention_weight_min": -90.85272979736328,
      "activations/layer4_attention_weight_max": 93.71995544433594,
      "activations/layer4_attention_weight_min": -90.50465393066406,
      "activations/layer5_attention_weight_max": 67.94889831542969,
      "activations/layer5_attention_weight_min": -76.15040588378906,
      "activations/layer6_attention_weight_max": 52.15732192993164,
      "activations/layer6_attention_weight_min": -50.727603912353516,
      "activations/layer7_attention_weight_max": 65.82422637939453,
      "activations/layer7_attention_weight_min": -67.54698944091797,
      "activations/layer8_attention_weight_max": 46.24919509887695,
      "activations/layer8_attention_weight_min": -47.11995315551758,
      "activations/layer9_attention_weight_max": 45.140445709228516,
      "activations/layer9_attention_weight_min": -44.222965240478516,
      "epoch": 17.49,
      "learning_rate": 3.757310606060606e-05,
      "loss": 2.7188,
      "step": 301050
    },
    {
      "activations/layer0_attention_weight_max": 14.818994522094727,
      "activations/layer0_attention_weight_min": -13.057376861572266,
      "activations/layer10_attention_weight_max": 34.57620620727539,
      "activations/layer10_attention_weight_min": -35.4094123840332,
      "activations/layer11_attention_weight_max": 34.85126495361328,
      "activations/layer11_attention_weight_min": -34.73604965209961,
      "activations/layer12_attention_weight_max": 18.98538589477539,
      "activations/layer12_attention_weight_min": -24.17967987060547,
      "activations/layer13_attention_weight_max": 33.17672348022461,
      "activations/layer13_attention_weight_min": -26.632160186767578,
      "activations/layer14_attention_weight_max": 35.58194351196289,
      "activations/layer14_attention_weight_min": -29.687570571899414,
      "activations/layer15_attention_weight_max": 32.49025344848633,
      "activations/layer15_attention_weight_min": -29.61756134033203,
      "activations/layer16_attention_weight_max": 34.99337387084961,
      "activations/layer16_attention_weight_min": -32.022762298583984,
      "activations/layer17_attention_weight_max": 49.629905700683594,
      "activations/layer17_attention_weight_min": -43.946922302246094,
      "activations/layer18_attention_weight_max": 45.76382827758789,
      "activations/layer18_attention_weight_min": -40.35081481933594,
      "activations/layer19_attention_weight_max": 23.531360626220703,
      "activations/layer19_attention_weight_min": -22.556116104125977,
      "activations/layer1_attention_weight_max": 17.21106719970703,
      "activations/layer1_attention_weight_min": -15.026982307434082,
      "activations/layer20_attention_weight_max": 21.09845542907715,
      "activations/layer20_attention_weight_min": -21.335832595825195,
      "activations/layer21_attention_weight_max": 34.41035842895508,
      "activations/layer21_attention_weight_min": -20.835235595703125,
      "activations/layer22_attention_weight_max": 29.613807678222656,
      "activations/layer22_attention_weight_min": -25.01954460144043,
      "activations/layer23_attention_weight_max": 33.80604553222656,
      "activations/layer23_attention_weight_min": -23.65427017211914,
      "activations/layer2_attention_weight_max": 29.970365524291992,
      "activations/layer2_attention_weight_min": -28.710651397705078,
      "activations/layer3_attention_weight_max": 88.88249206542969,
      "activations/layer3_attention_weight_min": -86.77515411376953,
      "activations/layer4_attention_weight_max": 86.81356048583984,
      "activations/layer4_attention_weight_min": -89.31218719482422,
      "activations/layer5_attention_weight_max": 66.22807312011719,
      "activations/layer5_attention_weight_min": -75.7441177368164,
      "activations/layer6_attention_weight_max": 48.34213638305664,
      "activations/layer6_attention_weight_min": -50.934505462646484,
      "activations/layer7_attention_weight_max": 68.17926025390625,
      "activations/layer7_attention_weight_min": -66.29646301269531,
      "activations/layer8_attention_weight_max": 45.5582275390625,
      "activations/layer8_attention_weight_min": -48.44906234741211,
      "activations/layer9_attention_weight_max": 45.500423431396484,
      "activations/layer9_attention_weight_min": -46.795562744140625,
      "epoch": 17.5,
      "learning_rate": 3.755416666666666e-05,
      "loss": 2.7293,
      "step": 301100
    },
    {
      "activations/layer0_attention_weight_max": 14.924296379089355,
      "activations/layer0_attention_weight_min": -13.278362274169922,
      "activations/layer10_attention_weight_max": 36.765403747558594,
      "activations/layer10_attention_weight_min": -37.45467758178711,
      "activations/layer11_attention_weight_max": 35.13288879394531,
      "activations/layer11_attention_weight_min": -35.95768356323242,
      "activations/layer12_attention_weight_max": 19.282636642456055,
      "activations/layer12_attention_weight_min": -28.406164169311523,
      "activations/layer13_attention_weight_max": 37.66683578491211,
      "activations/layer13_attention_weight_min": -27.388774871826172,
      "activations/layer14_attention_weight_max": 34.52138137817383,
      "activations/layer14_attention_weight_min": -31.842432022094727,
      "activations/layer15_attention_weight_max": 32.96611785888672,
      "activations/layer15_attention_weight_min": -30.884206771850586,
      "activations/layer16_attention_weight_max": 32.762298583984375,
      "activations/layer16_attention_weight_min": -32.68259048461914,
      "activations/layer17_attention_weight_max": 50.686912536621094,
      "activations/layer17_attention_weight_min": -43.2409553527832,
      "activations/layer18_attention_weight_max": 48.855194091796875,
      "activations/layer18_attention_weight_min": -38.11920166015625,
      "activations/layer19_attention_weight_max": 25.85774803161621,
      "activations/layer19_attention_weight_min": -23.101293563842773,
      "activations/layer1_attention_weight_max": 15.872989654541016,
      "activations/layer1_attention_weight_min": -13.831347465515137,
      "activations/layer20_attention_weight_max": 26.185861587524414,
      "activations/layer20_attention_weight_min": -22.02103042602539,
      "activations/layer21_attention_weight_max": 43.420616149902344,
      "activations/layer21_attention_weight_min": -23.17583465576172,
      "activations/layer22_attention_weight_max": 32.99368667602539,
      "activations/layer22_attention_weight_min": -27.19068145751953,
      "activations/layer23_attention_weight_max": 39.75904083251953,
      "activations/layer23_attention_weight_min": -23.32539176940918,
      "activations/layer2_attention_weight_max": 33.05720901489258,
      "activations/layer2_attention_weight_min": -30.557659149169922,
      "activations/layer3_attention_weight_max": 90.67649841308594,
      "activations/layer3_attention_weight_min": -91.40685272216797,
      "activations/layer4_attention_weight_max": 89.14701843261719,
      "activations/layer4_attention_weight_min": -90.77300262451172,
      "activations/layer5_attention_weight_max": 65.91915893554688,
      "activations/layer5_attention_weight_min": -77.45963287353516,
      "activations/layer6_attention_weight_max": 54.23485565185547,
      "activations/layer6_attention_weight_min": -55.42430114746094,
      "activations/layer7_attention_weight_max": 64.90746307373047,
      "activations/layer7_attention_weight_min": -70.60087585449219,
      "activations/layer8_attention_weight_max": 51.71119689941406,
      "activations/layer8_attention_weight_min": -55.762420654296875,
      "activations/layer9_attention_weight_max": 48.72612762451172,
      "activations/layer9_attention_weight_min": -51.85039138793945,
      "epoch": 17.5,
      "learning_rate": 3.7535227272727275e-05,
      "loss": 2.7339,
      "step": 301150
    },
    {
      "activations/layer0_attention_weight_max": 17.284400939941406,
      "activations/layer0_attention_weight_min": -12.960525512695312,
      "activations/layer10_attention_weight_max": 33.837982177734375,
      "activations/layer10_attention_weight_min": -35.34040451049805,
      "activations/layer11_attention_weight_max": 32.493431091308594,
      "activations/layer11_attention_weight_min": -35.212806701660156,
      "activations/layer12_attention_weight_max": 19.281280517578125,
      "activations/layer12_attention_weight_min": -24.019392013549805,
      "activations/layer13_attention_weight_max": 33.916568756103516,
      "activations/layer13_attention_weight_min": -26.938570022583008,
      "activations/layer14_attention_weight_max": 36.02540969848633,
      "activations/layer14_attention_weight_min": -30.41351890563965,
      "activations/layer15_attention_weight_max": 34.4420280456543,
      "activations/layer15_attention_weight_min": -31.823440551757812,
      "activations/layer16_attention_weight_max": 34.242244720458984,
      "activations/layer16_attention_weight_min": -33.1704216003418,
      "activations/layer17_attention_weight_max": 55.305179595947266,
      "activations/layer17_attention_weight_min": -47.18855667114258,
      "activations/layer18_attention_weight_max": 48.67527770996094,
      "activations/layer18_attention_weight_min": -40.05660629272461,
      "activations/layer19_attention_weight_max": 28.82935905456543,
      "activations/layer19_attention_weight_min": -23.710981369018555,
      "activations/layer1_attention_weight_max": 16.17914581298828,
      "activations/layer1_attention_weight_min": -13.872356414794922,
      "activations/layer20_attention_weight_max": 25.835552215576172,
      "activations/layer20_attention_weight_min": -21.49224853515625,
      "activations/layer21_attention_weight_max": 41.7620964050293,
      "activations/layer21_attention_weight_min": -23.016450881958008,
      "activations/layer22_attention_weight_max": 30.322296142578125,
      "activations/layer22_attention_weight_min": -28.457883834838867,
      "activations/layer23_attention_weight_max": 40.42242431640625,
      "activations/layer23_attention_weight_min": -25.542125701904297,
      "activations/layer2_attention_weight_max": 31.939159393310547,
      "activations/layer2_attention_weight_min": -29.186294555664062,
      "activations/layer3_attention_weight_max": 88.72367095947266,
      "activations/layer3_attention_weight_min": -90.23242950439453,
      "activations/layer4_attention_weight_max": 90.51795959472656,
      "activations/layer4_attention_weight_min": -90.0824203491211,
      "activations/layer5_attention_weight_max": 69.85516357421875,
      "activations/layer5_attention_weight_min": -75.6703872680664,
      "activations/layer6_attention_weight_max": 48.872432708740234,
      "activations/layer6_attention_weight_min": -51.98595428466797,
      "activations/layer7_attention_weight_max": 72.55587768554688,
      "activations/layer7_attention_weight_min": -63.927127838134766,
      "activations/layer8_attention_weight_max": 47.09444808959961,
      "activations/layer8_attention_weight_min": -48.036949157714844,
      "activations/layer9_attention_weight_max": 46.041648864746094,
      "activations/layer9_attention_weight_min": -44.633480072021484,
      "epoch": 17.5,
      "learning_rate": 3.751628787878788e-05,
      "loss": 2.7451,
      "step": 301200
    },
    {
      "activations/layer0_attention_weight_max": 15.421195983886719,
      "activations/layer0_attention_weight_min": -13.350837707519531,
      "activations/layer10_attention_weight_max": 34.96244812011719,
      "activations/layer10_attention_weight_min": -33.99445343017578,
      "activations/layer11_attention_weight_max": 30.926593780517578,
      "activations/layer11_attention_weight_min": -32.98311996459961,
      "activations/layer12_attention_weight_max": 18.11417579650879,
      "activations/layer12_attention_weight_min": -25.600624084472656,
      "activations/layer13_attention_weight_max": 32.62592315673828,
      "activations/layer13_attention_weight_min": -26.383968353271484,
      "activations/layer14_attention_weight_max": 31.797426223754883,
      "activations/layer14_attention_weight_min": -29.94990348815918,
      "activations/layer15_attention_weight_max": 32.7790641784668,
      "activations/layer15_attention_weight_min": -31.179956436157227,
      "activations/layer16_attention_weight_max": 31.47998809814453,
      "activations/layer16_attention_weight_min": -30.367015838623047,
      "activations/layer17_attention_weight_max": 47.30363845825195,
      "activations/layer17_attention_weight_min": -45.39957046508789,
      "activations/layer18_attention_weight_max": 45.54413986206055,
      "activations/layer18_attention_weight_min": -39.18647003173828,
      "activations/layer19_attention_weight_max": 24.510181427001953,
      "activations/layer19_attention_weight_min": -23.05597686767578,
      "activations/layer1_attention_weight_max": 16.0766544342041,
      "activations/layer1_attention_weight_min": -13.535128593444824,
      "activations/layer20_attention_weight_max": 21.20804786682129,
      "activations/layer20_attention_weight_min": -21.76730728149414,
      "activations/layer21_attention_weight_max": 35.946022033691406,
      "activations/layer21_attention_weight_min": -24.11528968811035,
      "activations/layer22_attention_weight_max": 28.764028549194336,
      "activations/layer22_attention_weight_min": -25.521780014038086,
      "activations/layer23_attention_weight_max": 35.96959686279297,
      "activations/layer23_attention_weight_min": -22.624370574951172,
      "activations/layer2_attention_weight_max": 31.40761375427246,
      "activations/layer2_attention_weight_min": -30.944499969482422,
      "activations/layer3_attention_weight_max": 88.31280517578125,
      "activations/layer3_attention_weight_min": -92.16158294677734,
      "activations/layer4_attention_weight_max": 86.56558227539062,
      "activations/layer4_attention_weight_min": -87.34685516357422,
      "activations/layer5_attention_weight_max": 63.527732849121094,
      "activations/layer5_attention_weight_min": -73.94053649902344,
      "activations/layer6_attention_weight_max": 50.55754089355469,
      "activations/layer6_attention_weight_min": -51.56996536254883,
      "activations/layer7_attention_weight_max": 58.72333526611328,
      "activations/layer7_attention_weight_min": -60.22064208984375,
      "activations/layer8_attention_weight_max": 43.71680450439453,
      "activations/layer8_attention_weight_min": -48.45366287231445,
      "activations/layer9_attention_weight_max": 43.930362701416016,
      "activations/layer9_attention_weight_min": -45.382720947265625,
      "epoch": 17.5,
      "learning_rate": 3.749734848484848e-05,
      "loss": 2.7521,
      "step": 301250
    },
    {
      "activations/layer0_attention_weight_max": 15.96467113494873,
      "activations/layer0_attention_weight_min": -12.815496444702148,
      "activations/layer10_attention_weight_max": 38.07231140136719,
      "activations/layer10_attention_weight_min": -37.3357048034668,
      "activations/layer11_attention_weight_max": 34.568275451660156,
      "activations/layer11_attention_weight_min": -36.80254364013672,
      "activations/layer12_attention_weight_max": 17.90319061279297,
      "activations/layer12_attention_weight_min": -24.032682418823242,
      "activations/layer13_attention_weight_max": 37.87089538574219,
      "activations/layer13_attention_weight_min": -32.00735855102539,
      "activations/layer14_attention_weight_max": 49.773067474365234,
      "activations/layer14_attention_weight_min": -30.788034439086914,
      "activations/layer15_attention_weight_max": 37.74382781982422,
      "activations/layer15_attention_weight_min": -30.91909408569336,
      "activations/layer16_attention_weight_max": 34.75535583496094,
      "activations/layer16_attention_weight_min": -33.31731414794922,
      "activations/layer17_attention_weight_max": 49.53443908691406,
      "activations/layer17_attention_weight_min": -48.0107421875,
      "activations/layer18_attention_weight_max": 46.72429656982422,
      "activations/layer18_attention_weight_min": -40.65532684326172,
      "activations/layer19_attention_weight_max": 24.21617317199707,
      "activations/layer19_attention_weight_min": -23.99231719970703,
      "activations/layer1_attention_weight_max": 16.328083038330078,
      "activations/layer1_attention_weight_min": -13.594583511352539,
      "activations/layer20_attention_weight_max": 25.395971298217773,
      "activations/layer20_attention_weight_min": -20.780162811279297,
      "activations/layer21_attention_weight_max": 40.985260009765625,
      "activations/layer21_attention_weight_min": -23.82918357849121,
      "activations/layer22_attention_weight_max": 33.27547073364258,
      "activations/layer22_attention_weight_min": -24.494794845581055,
      "activations/layer23_attention_weight_max": 33.067840576171875,
      "activations/layer23_attention_weight_min": -22.558094024658203,
      "activations/layer2_attention_weight_max": 32.575340270996094,
      "activations/layer2_attention_weight_min": -30.789304733276367,
      "activations/layer3_attention_weight_max": 97.41655731201172,
      "activations/layer3_attention_weight_min": -98.71961212158203,
      "activations/layer4_attention_weight_max": 92.1668472290039,
      "activations/layer4_attention_weight_min": -92.61571502685547,
      "activations/layer5_attention_weight_max": 67.41033172607422,
      "activations/layer5_attention_weight_min": -76.82829284667969,
      "activations/layer6_attention_weight_max": 53.667686462402344,
      "activations/layer6_attention_weight_min": -52.6893196105957,
      "activations/layer7_attention_weight_max": 62.530311584472656,
      "activations/layer7_attention_weight_min": -66.86060333251953,
      "activations/layer8_attention_weight_max": 47.62236404418945,
      "activations/layer8_attention_weight_min": -49.33040237426758,
      "activations/layer9_attention_weight_max": 46.32805633544922,
      "activations/layer9_attention_weight_min": -47.05251693725586,
      "epoch": 17.51,
      "learning_rate": 3.7478409090909086e-05,
      "loss": 2.7293,
      "step": 301300
    },
    {
      "activations/layer0_attention_weight_max": 16.013938903808594,
      "activations/layer0_attention_weight_min": -13.552338600158691,
      "activations/layer10_attention_weight_max": 37.233787536621094,
      "activations/layer10_attention_weight_min": -37.48960494995117,
      "activations/layer11_attention_weight_max": 34.392791748046875,
      "activations/layer11_attention_weight_min": -33.4538688659668,
      "activations/layer12_attention_weight_max": 19.113296508789062,
      "activations/layer12_attention_weight_min": -25.30795669555664,
      "activations/layer13_attention_weight_max": 34.945369720458984,
      "activations/layer13_attention_weight_min": -30.647602081298828,
      "activations/layer14_attention_weight_max": 36.99037170410156,
      "activations/layer14_attention_weight_min": -32.49058151245117,
      "activations/layer15_attention_weight_max": 33.678348541259766,
      "activations/layer15_attention_weight_min": -32.50505828857422,
      "activations/layer16_attention_weight_max": 35.45369338989258,
      "activations/layer16_attention_weight_min": -32.7428092956543,
      "activations/layer17_attention_weight_max": 51.38163375854492,
      "activations/layer17_attention_weight_min": -46.45811080932617,
      "activations/layer18_attention_weight_max": 47.404109954833984,
      "activations/layer18_attention_weight_min": -42.32144546508789,
      "activations/layer19_attention_weight_max": 25.135101318359375,
      "activations/layer19_attention_weight_min": -24.80000877380371,
      "activations/layer1_attention_weight_max": 16.328365325927734,
      "activations/layer1_attention_weight_min": -15.051724433898926,
      "activations/layer20_attention_weight_max": 21.856822967529297,
      "activations/layer20_attention_weight_min": -21.684412002563477,
      "activations/layer21_attention_weight_max": 36.5124397277832,
      "activations/layer21_attention_weight_min": -24.06492805480957,
      "activations/layer22_attention_weight_max": 28.609054565429688,
      "activations/layer22_attention_weight_min": -25.52870750427246,
      "activations/layer23_attention_weight_max": 34.27553939819336,
      "activations/layer23_attention_weight_min": -26.078895568847656,
      "activations/layer2_attention_weight_max": 31.154067993164062,
      "activations/layer2_attention_weight_min": -31.447111129760742,
      "activations/layer3_attention_weight_max": 93.03770446777344,
      "activations/layer3_attention_weight_min": -91.95233917236328,
      "activations/layer4_attention_weight_max": 89.46292114257812,
      "activations/layer4_attention_weight_min": -89.2718734741211,
      "activations/layer5_attention_weight_max": 67.5895004272461,
      "activations/layer5_attention_weight_min": -77.1571273803711,
      "activations/layer6_attention_weight_max": 51.109622955322266,
      "activations/layer6_attention_weight_min": -49.81694412231445,
      "activations/layer7_attention_weight_max": 65.7916259765625,
      "activations/layer7_attention_weight_min": -65.79074096679688,
      "activations/layer8_attention_weight_max": 48.94047927856445,
      "activations/layer8_attention_weight_min": -51.08032989501953,
      "activations/layer9_attention_weight_max": 47.45105743408203,
      "activations/layer9_attention_weight_min": -46.70116424560547,
      "epoch": 17.51,
      "learning_rate": 3.745984848484848e-05,
      "loss": 2.7334,
      "step": 301350
    },
    {
      "activations/layer0_attention_weight_max": 16.32522201538086,
      "activations/layer0_attention_weight_min": -13.234975814819336,
      "activations/layer10_attention_weight_max": 35.833946228027344,
      "activations/layer10_attention_weight_min": -35.68750762939453,
      "activations/layer11_attention_weight_max": 32.50423049926758,
      "activations/layer11_attention_weight_min": -34.227046966552734,
      "activations/layer12_attention_weight_max": 19.418609619140625,
      "activations/layer12_attention_weight_min": -25.0972843170166,
      "activations/layer13_attention_weight_max": 41.976707458496094,
      "activations/layer13_attention_weight_min": -29.40624237060547,
      "activations/layer14_attention_weight_max": 40.067623138427734,
      "activations/layer14_attention_weight_min": -31.5140438079834,
      "activations/layer15_attention_weight_max": 39.53689956665039,
      "activations/layer15_attention_weight_min": -32.23515701293945,
      "activations/layer16_attention_weight_max": 37.457313537597656,
      "activations/layer16_attention_weight_min": -34.6767463684082,
      "activations/layer17_attention_weight_max": 57.14373016357422,
      "activations/layer17_attention_weight_min": -48.15009307861328,
      "activations/layer18_attention_weight_max": 55.63344955444336,
      "activations/layer18_attention_weight_min": -42.91295623779297,
      "activations/layer19_attention_weight_max": 26.683917999267578,
      "activations/layer19_attention_weight_min": -24.305574417114258,
      "activations/layer1_attention_weight_max": 15.650774002075195,
      "activations/layer1_attention_weight_min": -16.519445419311523,
      "activations/layer20_attention_weight_max": 26.20038414001465,
      "activations/layer20_attention_weight_min": -22.064102172851562,
      "activations/layer21_attention_weight_max": 51.67724609375,
      "activations/layer21_attention_weight_min": -24.94369888305664,
      "activations/layer22_attention_weight_max": 34.12439727783203,
      "activations/layer22_attention_weight_min": -24.800378799438477,
      "activations/layer23_attention_weight_max": 41.922279357910156,
      "activations/layer23_attention_weight_min": -23.151670455932617,
      "activations/layer2_attention_weight_max": 31.53461456298828,
      "activations/layer2_attention_weight_min": -32.27771759033203,
      "activations/layer3_attention_weight_max": 84.78963470458984,
      "activations/layer3_attention_weight_min": -89.17394256591797,
      "activations/layer4_attention_weight_max": 85.32930755615234,
      "activations/layer4_attention_weight_min": -86.135498046875,
      "activations/layer5_attention_weight_max": 62.60395050048828,
      "activations/layer5_attention_weight_min": -70.18549346923828,
      "activations/layer6_attention_weight_max": 47.07146453857422,
      "activations/layer6_attention_weight_min": -50.529170989990234,
      "activations/layer7_attention_weight_max": 61.730247497558594,
      "activations/layer7_attention_weight_min": -62.91447830200195,
      "activations/layer8_attention_weight_max": 44.38245391845703,
      "activations/layer8_attention_weight_min": -46.906307220458984,
      "activations/layer9_attention_weight_max": 44.14350891113281,
      "activations/layer9_attention_weight_min": -44.76504135131836,
      "epoch": 17.51,
      "learning_rate": 3.7440909090909086e-05,
      "loss": 2.7518,
      "step": 301400
    },
    {
      "activations/layer0_attention_weight_max": 15.739017486572266,
      "activations/layer0_attention_weight_min": -13.39952564239502,
      "activations/layer10_attention_weight_max": 35.85943603515625,
      "activations/layer10_attention_weight_min": -35.108192443847656,
      "activations/layer11_attention_weight_max": 32.45844268798828,
      "activations/layer11_attention_weight_min": -33.95387268066406,
      "activations/layer12_attention_weight_max": 18.62571144104004,
      "activations/layer12_attention_weight_min": -25.893274307250977,
      "activations/layer13_attention_weight_max": 34.9445686340332,
      "activations/layer13_attention_weight_min": -28.810251235961914,
      "activations/layer14_attention_weight_max": 34.94991683959961,
      "activations/layer14_attention_weight_min": -30.671220779418945,
      "activations/layer15_attention_weight_max": 32.84181594848633,
      "activations/layer15_attention_weight_min": -31.464271545410156,
      "activations/layer16_attention_weight_max": 34.617252349853516,
      "activations/layer16_attention_weight_min": -30.644420623779297,
      "activations/layer17_attention_weight_max": 51.59339141845703,
      "activations/layer17_attention_weight_min": -43.994327545166016,
      "activations/layer18_attention_weight_max": 49.11334228515625,
      "activations/layer18_attention_weight_min": -39.34777069091797,
      "activations/layer19_attention_weight_max": 25.77105140686035,
      "activations/layer19_attention_weight_min": -23.860490798950195,
      "activations/layer1_attention_weight_max": 16.754623413085938,
      "activations/layer1_attention_weight_min": -15.871994018554688,
      "activations/layer20_attention_weight_max": 24.106931686401367,
      "activations/layer20_attention_weight_min": -24.683982849121094,
      "activations/layer21_attention_weight_max": 32.452423095703125,
      "activations/layer21_attention_weight_min": -21.0518856048584,
      "activations/layer22_attention_weight_max": 36.65451431274414,
      "activations/layer22_attention_weight_min": -26.573259353637695,
      "activations/layer23_attention_weight_max": 35.76875686645508,
      "activations/layer23_attention_weight_min": -24.3012752532959,
      "activations/layer2_attention_weight_max": 31.155479431152344,
      "activations/layer2_attention_weight_min": -30.05280113220215,
      "activations/layer3_attention_weight_max": 87.49553680419922,
      "activations/layer3_attention_weight_min": -91.57978820800781,
      "activations/layer4_attention_weight_max": 89.55941009521484,
      "activations/layer4_attention_weight_min": -89.26433563232422,
      "activations/layer5_attention_weight_max": 63.62762451171875,
      "activations/layer5_attention_weight_min": -80.58140563964844,
      "activations/layer6_attention_weight_max": 53.15361785888672,
      "activations/layer6_attention_weight_min": -53.945213317871094,
      "activations/layer7_attention_weight_max": 64.17627716064453,
      "activations/layer7_attention_weight_min": -66.00666809082031,
      "activations/layer8_attention_weight_max": 46.93275833129883,
      "activations/layer8_attention_weight_min": -47.91456985473633,
      "activations/layer9_attention_weight_max": 46.58464050292969,
      "activations/layer9_attention_weight_min": -44.627723693847656,
      "epoch": 17.52,
      "learning_rate": 3.7421969696969694e-05,
      "loss": 2.7548,
      "step": 301450
    },
    {
      "activations/layer0_attention_weight_max": 15.588029861450195,
      "activations/layer0_attention_weight_min": -13.372661590576172,
      "activations/layer10_attention_weight_max": 38.00716018676758,
      "activations/layer10_attention_weight_min": -35.88726806640625,
      "activations/layer11_attention_weight_max": 33.298988342285156,
      "activations/layer11_attention_weight_min": -34.778873443603516,
      "activations/layer12_attention_weight_max": 18.333621978759766,
      "activations/layer12_attention_weight_min": -22.45440101623535,
      "activations/layer13_attention_weight_max": 30.55309295654297,
      "activations/layer13_attention_weight_min": -28.933252334594727,
      "activations/layer14_attention_weight_max": 33.18186569213867,
      "activations/layer14_attention_weight_min": -30.049442291259766,
      "activations/layer15_attention_weight_max": 30.801015853881836,
      "activations/layer15_attention_weight_min": -28.998323440551758,
      "activations/layer16_attention_weight_max": 33.429161071777344,
      "activations/layer16_attention_weight_min": -31.95750617980957,
      "activations/layer17_attention_weight_max": 50.69253921508789,
      "activations/layer17_attention_weight_min": -44.4684944152832,
      "activations/layer18_attention_weight_max": 45.21596908569336,
      "activations/layer18_attention_weight_min": -38.74155807495117,
      "activations/layer19_attention_weight_max": 26.58760643005371,
      "activations/layer19_attention_weight_min": -25.342727661132812,
      "activations/layer1_attention_weight_max": 16.768115997314453,
      "activations/layer1_attention_weight_min": -14.46511459350586,
      "activations/layer20_attention_weight_max": 20.958457946777344,
      "activations/layer20_attention_weight_min": -21.797639846801758,
      "activations/layer21_attention_weight_max": 32.5801887512207,
      "activations/layer21_attention_weight_min": -24.568981170654297,
      "activations/layer22_attention_weight_max": 27.915634155273438,
      "activations/layer22_attention_weight_min": -25.57990264892578,
      "activations/layer23_attention_weight_max": 32.66771697998047,
      "activations/layer23_attention_weight_min": -23.519317626953125,
      "activations/layer2_attention_weight_max": 30.677473068237305,
      "activations/layer2_attention_weight_min": -28.990201950073242,
      "activations/layer3_attention_weight_max": 84.49647521972656,
      "activations/layer3_attention_weight_min": -85.51290130615234,
      "activations/layer4_attention_weight_max": 87.50814819335938,
      "activations/layer4_attention_weight_min": -87.9911117553711,
      "activations/layer5_attention_weight_max": 68.66047668457031,
      "activations/layer5_attention_weight_min": -76.77351379394531,
      "activations/layer6_attention_weight_max": 50.322776794433594,
      "activations/layer6_attention_weight_min": -51.62021255493164,
      "activations/layer7_attention_weight_max": 63.77022933959961,
      "activations/layer7_attention_weight_min": -66.89708709716797,
      "activations/layer8_attention_weight_max": 45.6292610168457,
      "activations/layer8_attention_weight_min": -48.548377990722656,
      "activations/layer9_attention_weight_max": 43.482059478759766,
      "activations/layer9_attention_weight_min": -46.67449188232422,
      "epoch": 17.52,
      "learning_rate": 3.740340909090909e-05,
      "loss": 2.7374,
      "step": 301500
    },
    {
      "activations/layer0_attention_weight_max": 15.730491638183594,
      "activations/layer0_attention_weight_min": -13.344869613647461,
      "activations/layer10_attention_weight_max": 33.509979248046875,
      "activations/layer10_attention_weight_min": -33.723819732666016,
      "activations/layer11_attention_weight_max": 30.899398803710938,
      "activations/layer11_attention_weight_min": -33.60166549682617,
      "activations/layer12_attention_weight_max": 18.79865074157715,
      "activations/layer12_attention_weight_min": -24.919328689575195,
      "activations/layer13_attention_weight_max": 29.69448471069336,
      "activations/layer13_attention_weight_min": -24.928760528564453,
      "activations/layer14_attention_weight_max": 31.3277530670166,
      "activations/layer14_attention_weight_min": -28.079425811767578,
      "activations/layer15_attention_weight_max": 29.267555236816406,
      "activations/layer15_attention_weight_min": -28.29077911376953,
      "activations/layer16_attention_weight_max": 29.64063835144043,
      "activations/layer16_attention_weight_min": -29.792375564575195,
      "activations/layer17_attention_weight_max": 45.30338668823242,
      "activations/layer17_attention_weight_min": -41.92031478881836,
      "activations/layer18_attention_weight_max": 40.30125427246094,
      "activations/layer18_attention_weight_min": -37.35381317138672,
      "activations/layer19_attention_weight_max": 20.30830192565918,
      "activations/layer19_attention_weight_min": -22.443477630615234,
      "activations/layer1_attention_weight_max": 16.916961669921875,
      "activations/layer1_attention_weight_min": -15.386832237243652,
      "activations/layer20_attention_weight_max": 18.520015716552734,
      "activations/layer20_attention_weight_min": -22.324119567871094,
      "activations/layer21_attention_weight_max": 28.91263771057129,
      "activations/layer21_attention_weight_min": -21.806564331054688,
      "activations/layer22_attention_weight_max": 26.111270904541016,
      "activations/layer22_attention_weight_min": -26.67329216003418,
      "activations/layer23_attention_weight_max": 31.604236602783203,
      "activations/layer23_attention_weight_min": -24.37574005126953,
      "activations/layer2_attention_weight_max": 30.683576583862305,
      "activations/layer2_attention_weight_min": -29.617748260498047,
      "activations/layer3_attention_weight_max": 85.85427856445312,
      "activations/layer3_attention_weight_min": -91.93566131591797,
      "activations/layer4_attention_weight_max": 85.98250579833984,
      "activations/layer4_attention_weight_min": -84.63224029541016,
      "activations/layer5_attention_weight_max": 63.095306396484375,
      "activations/layer5_attention_weight_min": -72.78498077392578,
      "activations/layer6_attention_weight_max": 50.68092727661133,
      "activations/layer6_attention_weight_min": -50.692752838134766,
      "activations/layer7_attention_weight_max": 63.672000885009766,
      "activations/layer7_attention_weight_min": -63.07008743286133,
      "activations/layer8_attention_weight_max": 46.59020233154297,
      "activations/layer8_attention_weight_min": -48.60662078857422,
      "activations/layer9_attention_weight_max": 41.10984420776367,
      "activations/layer9_attention_weight_min": -43.55137634277344,
      "epoch": 17.52,
      "learning_rate": 3.738446969696969e-05,
      "loss": 2.7281,
      "step": 301550
    },
    {
      "activations/layer0_attention_weight_max": 15.671954154968262,
      "activations/layer0_attention_weight_min": -13.002734184265137,
      "activations/layer10_attention_weight_max": 31.96483039855957,
      "activations/layer10_attention_weight_min": -34.6418571472168,
      "activations/layer11_attention_weight_max": 30.224864959716797,
      "activations/layer11_attention_weight_min": -33.68383026123047,
      "activations/layer12_attention_weight_max": 18.664031982421875,
      "activations/layer12_attention_weight_min": -25.539230346679688,
      "activations/layer13_attention_weight_max": 40.502105712890625,
      "activations/layer13_attention_weight_min": -29.431095123291016,
      "activations/layer14_attention_weight_max": 41.09675979614258,
      "activations/layer14_attention_weight_min": -31.72736930847168,
      "activations/layer15_attention_weight_max": 37.24582290649414,
      "activations/layer15_attention_weight_min": -32.35224151611328,
      "activations/layer16_attention_weight_max": 35.16555404663086,
      "activations/layer16_attention_weight_min": -31.515506744384766,
      "activations/layer17_attention_weight_max": 53.91812515258789,
      "activations/layer17_attention_weight_min": -47.45716094970703,
      "activations/layer18_attention_weight_max": 50.8746452331543,
      "activations/layer18_attention_weight_min": -43.27133560180664,
      "activations/layer19_attention_weight_max": 23.830307006835938,
      "activations/layer19_attention_weight_min": -23.324920654296875,
      "activations/layer1_attention_weight_max": 15.910029411315918,
      "activations/layer1_attention_weight_min": -15.130369186401367,
      "activations/layer20_attention_weight_max": 22.609020233154297,
      "activations/layer20_attention_weight_min": -21.60491180419922,
      "activations/layer21_attention_weight_max": 43.288002014160156,
      "activations/layer21_attention_weight_min": -23.968339920043945,
      "activations/layer22_attention_weight_max": 30.06186866760254,
      "activations/layer22_attention_weight_min": -25.09385108947754,
      "activations/layer23_attention_weight_max": 34.682865142822266,
      "activations/layer23_attention_weight_min": -24.47608757019043,
      "activations/layer2_attention_weight_max": 33.26403045654297,
      "activations/layer2_attention_weight_min": -29.94593620300293,
      "activations/layer3_attention_weight_max": 87.17317199707031,
      "activations/layer3_attention_weight_min": -86.96517944335938,
      "activations/layer4_attention_weight_max": 90.7276840209961,
      "activations/layer4_attention_weight_min": -82.78621673583984,
      "activations/layer5_attention_weight_max": 67.85765838623047,
      "activations/layer5_attention_weight_min": -69.94779205322266,
      "activations/layer6_attention_weight_max": 49.45723342895508,
      "activations/layer6_attention_weight_min": -47.315269470214844,
      "activations/layer7_attention_weight_max": 59.791316986083984,
      "activations/layer7_attention_weight_min": -60.25590896606445,
      "activations/layer8_attention_weight_max": 43.92630386352539,
      "activations/layer8_attention_weight_min": -47.0343132019043,
      "activations/layer9_attention_weight_max": 43.24100875854492,
      "activations/layer9_attention_weight_min": -44.83156967163086,
      "epoch": 17.52,
      "learning_rate": 3.73655303030303e-05,
      "loss": 2.7466,
      "step": 301600
    },
    {
      "activations/layer0_attention_weight_max": 15.60962200164795,
      "activations/layer0_attention_weight_min": -13.358687400817871,
      "activations/layer10_attention_weight_max": 31.838626861572266,
      "activations/layer10_attention_weight_min": -33.288917541503906,
      "activations/layer11_attention_weight_max": 29.882476806640625,
      "activations/layer11_attention_weight_min": -34.87518310546875,
      "activations/layer12_attention_weight_max": 19.42862319946289,
      "activations/layer12_attention_weight_min": -22.98459243774414,
      "activations/layer13_attention_weight_max": 33.27413558959961,
      "activations/layer13_attention_weight_min": -29.203983306884766,
      "activations/layer14_attention_weight_max": 37.389549255371094,
      "activations/layer14_attention_weight_min": -29.117786407470703,
      "activations/layer15_attention_weight_max": 33.79351043701172,
      "activations/layer15_attention_weight_min": -27.286823272705078,
      "activations/layer16_attention_weight_max": 34.65044403076172,
      "activations/layer16_attention_weight_min": -32.90536880493164,
      "activations/layer17_attention_weight_max": 51.92238998413086,
      "activations/layer17_attention_weight_min": -44.26581573486328,
      "activations/layer18_attention_weight_max": 48.33135986328125,
      "activations/layer18_attention_weight_min": -38.20694351196289,
      "activations/layer19_attention_weight_max": 23.656766891479492,
      "activations/layer19_attention_weight_min": -20.605648040771484,
      "activations/layer1_attention_weight_max": 15.707564353942871,
      "activations/layer1_attention_weight_min": -15.726750373840332,
      "activations/layer20_attention_weight_max": 22.561092376708984,
      "activations/layer20_attention_weight_min": -20.809085845947266,
      "activations/layer21_attention_weight_max": 37.77408218383789,
      "activations/layer21_attention_weight_min": -23.06171417236328,
      "activations/layer22_attention_weight_max": 28.86056137084961,
      "activations/layer22_attention_weight_min": -23.87858772277832,
      "activations/layer23_attention_weight_max": 38.316566467285156,
      "activations/layer23_attention_weight_min": -25.34889030456543,
      "activations/layer2_attention_weight_max": 31.808429718017578,
      "activations/layer2_attention_weight_min": -32.00798797607422,
      "activations/layer3_attention_weight_max": 87.14126586914062,
      "activations/layer3_attention_weight_min": -93.05667114257812,
      "activations/layer4_attention_weight_max": 86.16397094726562,
      "activations/layer4_attention_weight_min": -83.47441864013672,
      "activations/layer5_attention_weight_max": 66.6768569946289,
      "activations/layer5_attention_weight_min": -72.76445007324219,
      "activations/layer6_attention_weight_max": 47.184349060058594,
      "activations/layer6_attention_weight_min": -48.110618591308594,
      "activations/layer7_attention_weight_max": 60.75675582885742,
      "activations/layer7_attention_weight_min": -61.761573791503906,
      "activations/layer8_attention_weight_max": 43.753150939941406,
      "activations/layer8_attention_weight_min": -46.593868255615234,
      "activations/layer9_attention_weight_max": 39.172943115234375,
      "activations/layer9_attention_weight_min": -44.5930061340332,
      "epoch": 17.53,
      "learning_rate": 3.73465909090909e-05,
      "loss": 2.7517,
      "step": 301650
    },
    {
      "activations/layer0_attention_weight_max": 15.610681533813477,
      "activations/layer0_attention_weight_min": -14.477012634277344,
      "activations/layer10_attention_weight_max": 36.945030212402344,
      "activations/layer10_attention_weight_min": -35.643592834472656,
      "activations/layer11_attention_weight_max": 30.620182037353516,
      "activations/layer11_attention_weight_min": -34.328277587890625,
      "activations/layer12_attention_weight_max": 19.527233123779297,
      "activations/layer12_attention_weight_min": -26.3002986907959,
      "activations/layer13_attention_weight_max": 33.798492431640625,
      "activations/layer13_attention_weight_min": -25.95061492919922,
      "activations/layer14_attention_weight_max": 37.14684295654297,
      "activations/layer14_attention_weight_min": -31.352373123168945,
      "activations/layer15_attention_weight_max": 31.302845001220703,
      "activations/layer15_attention_weight_min": -29.363637924194336,
      "activations/layer16_attention_weight_max": 37.94733810424805,
      "activations/layer16_attention_weight_min": -35.644710540771484,
      "activations/layer17_attention_weight_max": 52.2192268371582,
      "activations/layer17_attention_weight_min": -51.16568374633789,
      "activations/layer18_attention_weight_max": 46.1546630859375,
      "activations/layer18_attention_weight_min": -44.62433624267578,
      "activations/layer19_attention_weight_max": 23.818744659423828,
      "activations/layer19_attention_weight_min": -25.048629760742188,
      "activations/layer1_attention_weight_max": 14.56873893737793,
      "activations/layer1_attention_weight_min": -15.494009971618652,
      "activations/layer20_attention_weight_max": 22.5350284576416,
      "activations/layer20_attention_weight_min": -24.659751892089844,
      "activations/layer21_attention_weight_max": 34.87142562866211,
      "activations/layer21_attention_weight_min": -23.447494506835938,
      "activations/layer22_attention_weight_max": 27.47437286376953,
      "activations/layer22_attention_weight_min": -28.733409881591797,
      "activations/layer23_attention_weight_max": 37.84955596923828,
      "activations/layer23_attention_weight_min": -28.45816421508789,
      "activations/layer2_attention_weight_max": 32.446285247802734,
      "activations/layer2_attention_weight_min": -29.220077514648438,
      "activations/layer3_attention_weight_max": 82.68590545654297,
      "activations/layer3_attention_weight_min": -89.68041229248047,
      "activations/layer4_attention_weight_max": 83.85865020751953,
      "activations/layer4_attention_weight_min": -88.14899444580078,
      "activations/layer5_attention_weight_max": 62.89936447143555,
      "activations/layer5_attention_weight_min": -73.63311004638672,
      "activations/layer6_attention_weight_max": 48.02418518066406,
      "activations/layer6_attention_weight_min": -49.546939849853516,
      "activations/layer7_attention_weight_max": 62.29608917236328,
      "activations/layer7_attention_weight_min": -62.8714599609375,
      "activations/layer8_attention_weight_max": 44.28947448730469,
      "activations/layer8_attention_weight_min": -49.0498161315918,
      "activations/layer9_attention_weight_max": 44.45935821533203,
      "activations/layer9_attention_weight_min": -46.1112174987793,
      "epoch": 17.53,
      "learning_rate": 3.732765151515151e-05,
      "loss": 2.7387,
      "step": 301700
    },
    {
      "activations/layer0_attention_weight_max": 16.065523147583008,
      "activations/layer0_attention_weight_min": -13.0337495803833,
      "activations/layer10_attention_weight_max": 33.03424072265625,
      "activations/layer10_attention_weight_min": -34.75967025756836,
      "activations/layer11_attention_weight_max": 31.505420684814453,
      "activations/layer11_attention_weight_min": -31.22900390625,
      "activations/layer12_attention_weight_max": 17.889963150024414,
      "activations/layer12_attention_weight_min": -22.87792205810547,
      "activations/layer13_attention_weight_max": 33.347877502441406,
      "activations/layer13_attention_weight_min": -25.266664505004883,
      "activations/layer14_attention_weight_max": 33.62533187866211,
      "activations/layer14_attention_weight_min": -29.978553771972656,
      "activations/layer15_attention_weight_max": 31.300296783447266,
      "activations/layer15_attention_weight_min": -29.74666404724121,
      "activations/layer16_attention_weight_max": 32.183860778808594,
      "activations/layer16_attention_weight_min": -31.02404022216797,
      "activations/layer17_attention_weight_max": 53.959327697753906,
      "activations/layer17_attention_weight_min": -46.55154037475586,
      "activations/layer18_attention_weight_max": 45.686824798583984,
      "activations/layer18_attention_weight_min": -40.249786376953125,
      "activations/layer19_attention_weight_max": 23.903791427612305,
      "activations/layer19_attention_weight_min": -22.90933609008789,
      "activations/layer1_attention_weight_max": 17.183273315429688,
      "activations/layer1_attention_weight_min": -13.899620056152344,
      "activations/layer20_attention_weight_max": 22.706918716430664,
      "activations/layer20_attention_weight_min": -21.715789794921875,
      "activations/layer21_attention_weight_max": 36.20966339111328,
      "activations/layer21_attention_weight_min": -22.7152099609375,
      "activations/layer22_attention_weight_max": 28.653331756591797,
      "activations/layer22_attention_weight_min": -27.494050979614258,
      "activations/layer23_attention_weight_max": 37.27684020996094,
      "activations/layer23_attention_weight_min": -22.63924789428711,
      "activations/layer2_attention_weight_max": 29.870092391967773,
      "activations/layer2_attention_weight_min": -30.086570739746094,
      "activations/layer3_attention_weight_max": 88.21505737304688,
      "activations/layer3_attention_weight_min": -87.80516815185547,
      "activations/layer4_attention_weight_max": 84.6062240600586,
      "activations/layer4_attention_weight_min": -85.86934661865234,
      "activations/layer5_attention_weight_max": 62.746742248535156,
      "activations/layer5_attention_weight_min": -73.05909729003906,
      "activations/layer6_attention_weight_max": 47.93999099731445,
      "activations/layer6_attention_weight_min": -48.16264343261719,
      "activations/layer7_attention_weight_max": 66.61124420166016,
      "activations/layer7_attention_weight_min": -62.23597717285156,
      "activations/layer8_attention_weight_max": 44.45840835571289,
      "activations/layer8_attention_weight_min": -49.659828186035156,
      "activations/layer9_attention_weight_max": 41.69748306274414,
      "activations/layer9_attention_weight_min": -44.09228515625,
      "epoch": 17.53,
      "learning_rate": 3.730871212121212e-05,
      "loss": 2.7338,
      "step": 301750
    },
    {
      "activations/layer0_attention_weight_max": 16.17485237121582,
      "activations/layer0_attention_weight_min": -13.640251159667969,
      "activations/layer10_attention_weight_max": 34.78424072265625,
      "activations/layer10_attention_weight_min": -34.95880889892578,
      "activations/layer11_attention_weight_max": 31.58625602722168,
      "activations/layer11_attention_weight_min": -33.27647399902344,
      "activations/layer12_attention_weight_max": 19.788089752197266,
      "activations/layer12_attention_weight_min": -25.340234756469727,
      "activations/layer13_attention_weight_max": 35.55410385131836,
      "activations/layer13_attention_weight_min": -28.693344116210938,
      "activations/layer14_attention_weight_max": 37.412933349609375,
      "activations/layer14_attention_weight_min": -31.467809677124023,
      "activations/layer15_attention_weight_max": 33.88686752319336,
      "activations/layer15_attention_weight_min": -32.424808502197266,
      "activations/layer16_attention_weight_max": 36.234954833984375,
      "activations/layer16_attention_weight_min": -34.05357360839844,
      "activations/layer17_attention_weight_max": 53.72846603393555,
      "activations/layer17_attention_weight_min": -46.1852912902832,
      "activations/layer18_attention_weight_max": 49.438560485839844,
      "activations/layer18_attention_weight_min": -40.51567840576172,
      "activations/layer19_attention_weight_max": 24.682905197143555,
      "activations/layer19_attention_weight_min": -22.96515464782715,
      "activations/layer1_attention_weight_max": 15.53044319152832,
      "activations/layer1_attention_weight_min": -13.716742515563965,
      "activations/layer20_attention_weight_max": 26.958683013916016,
      "activations/layer20_attention_weight_min": -20.75965118408203,
      "activations/layer21_attention_weight_max": 41.60232925415039,
      "activations/layer21_attention_weight_min": -23.414037704467773,
      "activations/layer22_attention_weight_max": 30.475332260131836,
      "activations/layer22_attention_weight_min": -24.53898811340332,
      "activations/layer23_attention_weight_max": 40.854087829589844,
      "activations/layer23_attention_weight_min": -23.616668701171875,
      "activations/layer2_attention_weight_max": 29.908676147460938,
      "activations/layer2_attention_weight_min": -28.113006591796875,
      "activations/layer3_attention_weight_max": 86.75314331054688,
      "activations/layer3_attention_weight_min": -87.63327026367188,
      "activations/layer4_attention_weight_max": 84.9134521484375,
      "activations/layer4_attention_weight_min": -86.51665496826172,
      "activations/layer5_attention_weight_max": 70.12155151367188,
      "activations/layer5_attention_weight_min": -73.46937561035156,
      "activations/layer6_attention_weight_max": 48.18550109863281,
      "activations/layer6_attention_weight_min": -50.49089813232422,
      "activations/layer7_attention_weight_max": 62.74668502807617,
      "activations/layer7_attention_weight_min": -66.2524642944336,
      "activations/layer8_attention_weight_max": 43.719905853271484,
      "activations/layer8_attention_weight_min": -48.31727981567383,
      "activations/layer9_attention_weight_max": 41.966617584228516,
      "activations/layer9_attention_weight_min": -45.99798583984375,
      "epoch": 17.54,
      "learning_rate": 3.728977272727272e-05,
      "loss": 2.7335,
      "step": 301800
    },
    {
      "activations/layer0_attention_weight_max": 15.802443504333496,
      "activations/layer0_attention_weight_min": -13.338226318359375,
      "activations/layer10_attention_weight_max": 43.30015182495117,
      "activations/layer10_attention_weight_min": -39.866905212402344,
      "activations/layer11_attention_weight_max": 42.636878967285156,
      "activations/layer11_attention_weight_min": -40.38782501220703,
      "activations/layer12_attention_weight_max": 17.207714080810547,
      "activations/layer12_attention_weight_min": -33.24434280395508,
      "activations/layer13_attention_weight_max": 37.87150192260742,
      "activations/layer13_attention_weight_min": -27.50331687927246,
      "activations/layer14_attention_weight_max": 36.489463806152344,
      "activations/layer14_attention_weight_min": -28.712411880493164,
      "activations/layer15_attention_weight_max": 36.07807159423828,
      "activations/layer15_attention_weight_min": -31.623228073120117,
      "activations/layer16_attention_weight_max": 35.34494400024414,
      "activations/layer16_attention_weight_min": -32.07207107543945,
      "activations/layer17_attention_weight_max": 52.103172302246094,
      "activations/layer17_attention_weight_min": -48.80887985229492,
      "activations/layer18_attention_weight_max": 49.117496490478516,
      "activations/layer18_attention_weight_min": -44.25617599487305,
      "activations/layer19_attention_weight_max": 25.457073211669922,
      "activations/layer19_attention_weight_min": -24.58595085144043,
      "activations/layer1_attention_weight_max": 15.800420761108398,
      "activations/layer1_attention_weight_min": -14.577688217163086,
      "activations/layer20_attention_weight_max": 24.833444595336914,
      "activations/layer20_attention_weight_min": -23.02579689025879,
      "activations/layer21_attention_weight_max": 41.21934509277344,
      "activations/layer21_attention_weight_min": -24.739028930664062,
      "activations/layer22_attention_weight_max": 28.48677635192871,
      "activations/layer22_attention_weight_min": -25.031940460205078,
      "activations/layer23_attention_weight_max": 34.49684524536133,
      "activations/layer23_attention_weight_min": -22.989707946777344,
      "activations/layer2_attention_weight_max": 33.34031295776367,
      "activations/layer2_attention_weight_min": -31.713237762451172,
      "activations/layer3_attention_weight_max": 95.10221862792969,
      "activations/layer3_attention_weight_min": -97.19864654541016,
      "activations/layer4_attention_weight_max": 89.75360870361328,
      "activations/layer4_attention_weight_min": -87.8105697631836,
      "activations/layer5_attention_weight_max": 69.24983978271484,
      "activations/layer5_attention_weight_min": -68.37074279785156,
      "activations/layer6_attention_weight_max": 51.55095672607422,
      "activations/layer6_attention_weight_min": -51.111392974853516,
      "activations/layer7_attention_weight_max": 77.18193054199219,
      "activations/layer7_attention_weight_min": -74.89360809326172,
      "activations/layer8_attention_weight_max": 54.54833984375,
      "activations/layer8_attention_weight_min": -52.09456253051758,
      "activations/layer9_attention_weight_max": 59.0606689453125,
      "activations/layer9_attention_weight_min": -50.21381759643555,
      "epoch": 17.54,
      "learning_rate": 3.727083333333333e-05,
      "loss": 2.719,
      "step": 301850
    },
    {
      "activations/layer0_attention_weight_max": 16.615293502807617,
      "activations/layer0_attention_weight_min": -13.414392471313477,
      "activations/layer10_attention_weight_max": 35.7170524597168,
      "activations/layer10_attention_weight_min": -35.250431060791016,
      "activations/layer11_attention_weight_max": 32.536102294921875,
      "activations/layer11_attention_weight_min": -32.77490234375,
      "activations/layer12_attention_weight_max": 18.514461517333984,
      "activations/layer12_attention_weight_min": -27.47275733947754,
      "activations/layer13_attention_weight_max": 40.6548957824707,
      "activations/layer13_attention_weight_min": -31.67298126220703,
      "activations/layer14_attention_weight_max": 38.076377868652344,
      "activations/layer14_attention_weight_min": -31.175371170043945,
      "activations/layer15_attention_weight_max": 38.3377571105957,
      "activations/layer15_attention_weight_min": -33.049922943115234,
      "activations/layer16_attention_weight_max": 33.094520568847656,
      "activations/layer16_attention_weight_min": -31.54784393310547,
      "activations/layer17_attention_weight_max": 49.54460525512695,
      "activations/layer17_attention_weight_min": -49.42176818847656,
      "activations/layer18_attention_weight_max": 43.562808990478516,
      "activations/layer18_attention_weight_min": -42.88302993774414,
      "activations/layer19_attention_weight_max": 23.242063522338867,
      "activations/layer19_attention_weight_min": -22.50104331970215,
      "activations/layer1_attention_weight_max": 16.19853973388672,
      "activations/layer1_attention_weight_min": -13.977818489074707,
      "activations/layer20_attention_weight_max": 20.853788375854492,
      "activations/layer20_attention_weight_min": -24.366256713867188,
      "activations/layer21_attention_weight_max": 46.29417037963867,
      "activations/layer21_attention_weight_min": -31.110179901123047,
      "activations/layer22_attention_weight_max": 28.878604888916016,
      "activations/layer22_attention_weight_min": -25.171649932861328,
      "activations/layer23_attention_weight_max": 37.21006774902344,
      "activations/layer23_attention_weight_min": -23.999732971191406,
      "activations/layer2_attention_weight_max": 31.54776382446289,
      "activations/layer2_attention_weight_min": -29.44244956970215,
      "activations/layer3_attention_weight_max": 90.72552490234375,
      "activations/layer3_attention_weight_min": -94.81843566894531,
      "activations/layer4_attention_weight_max": 87.83306121826172,
      "activations/layer4_attention_weight_min": -88.3128662109375,
      "activations/layer5_attention_weight_max": 63.687095642089844,
      "activations/layer5_attention_weight_min": -71.8389663696289,
      "activations/layer6_attention_weight_max": 49.28306579589844,
      "activations/layer6_attention_weight_min": -52.685298919677734,
      "activations/layer7_attention_weight_max": 64.04086303710938,
      "activations/layer7_attention_weight_min": -66.32661437988281,
      "activations/layer8_attention_weight_max": 48.948387145996094,
      "activations/layer8_attention_weight_min": -49.883941650390625,
      "activations/layer9_attention_weight_max": 45.72471237182617,
      "activations/layer9_attention_weight_min": -42.56000518798828,
      "epoch": 17.54,
      "learning_rate": 3.725189393939394e-05,
      "loss": 2.7388,
      "step": 301900
    },
    {
      "activations/layer0_attention_weight_max": 17.130937576293945,
      "activations/layer0_attention_weight_min": -13.30534839630127,
      "activations/layer10_attention_weight_max": 33.433170318603516,
      "activations/layer10_attention_weight_min": -33.30794143676758,
      "activations/layer11_attention_weight_max": 30.556865692138672,
      "activations/layer11_attention_weight_min": -33.91399002075195,
      "activations/layer12_attention_weight_max": 17.600040435791016,
      "activations/layer12_attention_weight_min": -28.782886505126953,
      "activations/layer13_attention_weight_max": 31.853221893310547,
      "activations/layer13_attention_weight_min": -28.46571922302246,
      "activations/layer14_attention_weight_max": 32.981224060058594,
      "activations/layer14_attention_weight_min": -29.939998626708984,
      "activations/layer15_attention_weight_max": 30.456104278564453,
      "activations/layer15_attention_weight_min": -29.84852409362793,
      "activations/layer16_attention_weight_max": 33.01285171508789,
      "activations/layer16_attention_weight_min": -31.344364166259766,
      "activations/layer17_attention_weight_max": 48.384910583496094,
      "activations/layer17_attention_weight_min": -42.974063873291016,
      "activations/layer18_attention_weight_max": 46.97551345825195,
      "activations/layer18_attention_weight_min": -39.063838958740234,
      "activations/layer19_attention_weight_max": 21.19501304626465,
      "activations/layer19_attention_weight_min": -23.033510208129883,
      "activations/layer1_attention_weight_max": 15.183116912841797,
      "activations/layer1_attention_weight_min": -13.903465270996094,
      "activations/layer20_attention_weight_max": 22.725400924682617,
      "activations/layer20_attention_weight_min": -21.84379005432129,
      "activations/layer21_attention_weight_max": 34.64751434326172,
      "activations/layer21_attention_weight_min": -22.913864135742188,
      "activations/layer22_attention_weight_max": 27.70187759399414,
      "activations/layer22_attention_weight_min": -26.56473159790039,
      "activations/layer23_attention_weight_max": 33.59477233886719,
      "activations/layer23_attention_weight_min": -26.7099666595459,
      "activations/layer2_attention_weight_max": 30.164051055908203,
      "activations/layer2_attention_weight_min": -31.246564865112305,
      "activations/layer3_attention_weight_max": 89.21227264404297,
      "activations/layer3_attention_weight_min": -97.18827819824219,
      "activations/layer4_attention_weight_max": 85.21617889404297,
      "activations/layer4_attention_weight_min": -84.69390106201172,
      "activations/layer5_attention_weight_max": 67.07716369628906,
      "activations/layer5_attention_weight_min": -76.69941711425781,
      "activations/layer6_attention_weight_max": 49.811622619628906,
      "activations/layer6_attention_weight_min": -48.80731964111328,
      "activations/layer7_attention_weight_max": 63.90696334838867,
      "activations/layer7_attention_weight_min": -61.98759078979492,
      "activations/layer8_attention_weight_max": 43.555728912353516,
      "activations/layer8_attention_weight_min": -45.27006912231445,
      "activations/layer9_attention_weight_max": 40.691471099853516,
      "activations/layer9_attention_weight_min": -43.16158676147461,
      "epoch": 17.55,
      "learning_rate": 3.7232954545454546e-05,
      "loss": 2.7347,
      "step": 301950
    },
    {
      "activations/layer0_attention_weight_max": 15.814005851745605,
      "activations/layer0_attention_weight_min": -13.313358306884766,
      "activations/layer10_attention_weight_max": 36.942787170410156,
      "activations/layer10_attention_weight_min": -37.57434844970703,
      "activations/layer11_attention_weight_max": 34.59037780761719,
      "activations/layer11_attention_weight_min": -34.854034423828125,
      "activations/layer12_attention_weight_max": 18.499361038208008,
      "activations/layer12_attention_weight_min": -23.342077255249023,
      "activations/layer13_attention_weight_max": 35.55027770996094,
      "activations/layer13_attention_weight_min": -26.62796401977539,
      "activations/layer14_attention_weight_max": 37.125457763671875,
      "activations/layer14_attention_weight_min": -30.576034545898438,
      "activations/layer15_attention_weight_max": 34.410850524902344,
      "activations/layer15_attention_weight_min": -32.06562423706055,
      "activations/layer16_attention_weight_max": 34.071598052978516,
      "activations/layer16_attention_weight_min": -32.37110137939453,
      "activations/layer17_attention_weight_max": 54.06951141357422,
      "activations/layer17_attention_weight_min": -50.2707633972168,
      "activations/layer18_attention_weight_max": 48.86540603637695,
      "activations/layer18_attention_weight_min": -45.23188018798828,
      "activations/layer19_attention_weight_max": 24.39937973022461,
      "activations/layer19_attention_weight_min": -25.831151962280273,
      "activations/layer1_attention_weight_max": 17.37590980529785,
      "activations/layer1_attention_weight_min": -16.287111282348633,
      "activations/layer20_attention_weight_max": 23.082761764526367,
      "activations/layer20_attention_weight_min": -26.83684730529785,
      "activations/layer21_attention_weight_max": 35.339534759521484,
      "activations/layer21_attention_weight_min": -22.85154151916504,
      "activations/layer22_attention_weight_max": 30.150245666503906,
      "activations/layer22_attention_weight_min": -31.32598114013672,
      "activations/layer23_attention_weight_max": 37.81233596801758,
      "activations/layer23_attention_weight_min": -26.328861236572266,
      "activations/layer2_attention_weight_max": 31.08271026611328,
      "activations/layer2_attention_weight_min": -30.7431697845459,
      "activations/layer3_attention_weight_max": 88.82456970214844,
      "activations/layer3_attention_weight_min": -94.20982360839844,
      "activations/layer4_attention_weight_max": 89.16524505615234,
      "activations/layer4_attention_weight_min": -89.5353012084961,
      "activations/layer5_attention_weight_max": 64.71321868896484,
      "activations/layer5_attention_weight_min": -77.98037719726562,
      "activations/layer6_attention_weight_max": 52.26417541503906,
      "activations/layer6_attention_weight_min": -54.30052947998047,
      "activations/layer7_attention_weight_max": 64.96275329589844,
      "activations/layer7_attention_weight_min": -68.89372253417969,
      "activations/layer8_attention_weight_max": 47.0809211730957,
      "activations/layer8_attention_weight_min": -54.839908599853516,
      "activations/layer9_attention_weight_max": 44.89536666870117,
      "activations/layer9_attention_weight_min": -47.967044830322266,
      "epoch": 17.55,
      "learning_rate": 3.721401515151515e-05,
      "loss": 2.7351,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4933,
      "eval_samples_per_second": 505.577,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4933,
      "eval_openwebtext_samples_per_second": 505.577,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9397,
      "eval_wikitext_samples_per_second": 235.087,
      "step": 302000
    },
    {
      "epoch": 17.55,
      "eval_lambada_loss": 2.521484375,
      "eval_lambada_ppl": 12.447059060920614,
      "eval_lambada_runtime": 9.5349,
      "eval_lambada_samples_per_second": 510.651,
      "step": 302000
    },
    {
      "activations/layer0_attention_weight_max": 15.556482315063477,
      "activations/layer0_attention_weight_min": -14.656131744384766,
      "activations/layer10_attention_weight_max": 34.649784088134766,
      "activations/layer10_attention_weight_min": -34.04423522949219,
      "activations/layer11_attention_weight_max": 30.969581604003906,
      "activations/layer11_attention_weight_min": -32.01531982421875,
      "activations/layer12_attention_weight_max": 19.49603843688965,
      "activations/layer12_attention_weight_min": -25.874788284301758,
      "activations/layer13_attention_weight_max": 35.85769271850586,
      "activations/layer13_attention_weight_min": -28.542232513427734,
      "activations/layer14_attention_weight_max": 37.13041687011719,
      "activations/layer14_attention_weight_min": -30.3427734375,
      "activations/layer15_attention_weight_max": 34.22697830200195,
      "activations/layer15_attention_weight_min": -30.9727840423584,
      "activations/layer16_attention_weight_max": 33.138641357421875,
      "activations/layer16_attention_weight_min": -32.264854431152344,
      "activations/layer17_attention_weight_max": 48.765201568603516,
      "activations/layer17_attention_weight_min": -43.12260437011719,
      "activations/layer18_attention_weight_max": 46.46597671508789,
      "activations/layer18_attention_weight_min": -39.31500244140625,
      "activations/layer19_attention_weight_max": 23.803003311157227,
      "activations/layer19_attention_weight_min": -22.902503967285156,
      "activations/layer1_attention_weight_max": 15.233271598815918,
      "activations/layer1_attention_weight_min": -14.939213752746582,
      "activations/layer20_attention_weight_max": 23.435001373291016,
      "activations/layer20_attention_weight_min": -22.540559768676758,
      "activations/layer21_attention_weight_max": 39.007957458496094,
      "activations/layer21_attention_weight_min": -26.395397186279297,
      "activations/layer22_attention_weight_max": 27.161184310913086,
      "activations/layer22_attention_weight_min": -25.458208084106445,
      "activations/layer23_attention_weight_max": 39.5325927734375,
      "activations/layer23_attention_weight_min": -23.69769287109375,
      "activations/layer2_attention_weight_max": 31.143505096435547,
      "activations/layer2_attention_weight_min": -30.375423431396484,
      "activations/layer3_attention_weight_max": 88.41168212890625,
      "activations/layer3_attention_weight_min": -89.3676986694336,
      "activations/layer4_attention_weight_max": 87.410888671875,
      "activations/layer4_attention_weight_min": -91.71739959716797,
      "activations/layer5_attention_weight_max": 66.44147491455078,
      "activations/layer5_attention_weight_min": -74.1848373413086,
      "activations/layer6_attention_weight_max": 50.2275276184082,
      "activations/layer6_attention_weight_min": -51.112083435058594,
      "activations/layer7_attention_weight_max": 66.10279846191406,
      "activations/layer7_attention_weight_min": -65.67474365234375,
      "activations/layer8_attention_weight_max": 45.021881103515625,
      "activations/layer8_attention_weight_min": -48.4986572265625,
      "activations/layer9_attention_weight_max": 44.599369049072266,
      "activations/layer9_attention_weight_min": -44.24064254760742,
      "epoch": 17.55,
      "learning_rate": 3.7195075757575755e-05,
      "loss": 2.7199,
      "step": 302050
    },
    {
      "activations/layer0_attention_weight_max": 15.922741889953613,
      "activations/layer0_attention_weight_min": -13.438413619995117,
      "activations/layer10_attention_weight_max": 38.24708557128906,
      "activations/layer10_attention_weight_min": -35.67094802856445,
      "activations/layer11_attention_weight_max": 33.32106018066406,
      "activations/layer11_attention_weight_min": -33.66697311401367,
      "activations/layer12_attention_weight_max": 19.606420516967773,
      "activations/layer12_attention_weight_min": -26.26287078857422,
      "activations/layer13_attention_weight_max": 37.31159973144531,
      "activations/layer13_attention_weight_min": -30.223430633544922,
      "activations/layer14_attention_weight_max": 39.01251983642578,
      "activations/layer14_attention_weight_min": -31.783355712890625,
      "activations/layer15_attention_weight_max": 34.575042724609375,
      "activations/layer15_attention_weight_min": -31.069772720336914,
      "activations/layer16_attention_weight_max": 33.48440170288086,
      "activations/layer16_attention_weight_min": -32.21544647216797,
      "activations/layer17_attention_weight_max": 49.73625564575195,
      "activations/layer17_attention_weight_min": -43.5370979309082,
      "activations/layer18_attention_weight_max": 46.45398712158203,
      "activations/layer18_attention_weight_min": -39.73923873901367,
      "activations/layer19_attention_weight_max": 23.36295509338379,
      "activations/layer19_attention_weight_min": -23.478391647338867,
      "activations/layer1_attention_weight_max": 15.908549308776855,
      "activations/layer1_attention_weight_min": -15.39331340789795,
      "activations/layer20_attention_weight_max": 24.5676326751709,
      "activations/layer20_attention_weight_min": -22.91315460205078,
      "activations/layer21_attention_weight_max": 40.29524612426758,
      "activations/layer21_attention_weight_min": -23.419565200805664,
      "activations/layer22_attention_weight_max": 32.0804443359375,
      "activations/layer22_attention_weight_min": -24.87230110168457,
      "activations/layer23_attention_weight_max": 41.618709564208984,
      "activations/layer23_attention_weight_min": -25.355480194091797,
      "activations/layer2_attention_weight_max": 30.344188690185547,
      "activations/layer2_attention_weight_min": -28.746910095214844,
      "activations/layer3_attention_weight_max": 90.36536407470703,
      "activations/layer3_attention_weight_min": -89.3922348022461,
      "activations/layer4_attention_weight_max": 86.99956512451172,
      "activations/layer4_attention_weight_min": -86.133056640625,
      "activations/layer5_attention_weight_max": 66.29253387451172,
      "activations/layer5_attention_weight_min": -75.48357391357422,
      "activations/layer6_attention_weight_max": 48.75393295288086,
      "activations/layer6_attention_weight_min": -50.2705192565918,
      "activations/layer7_attention_weight_max": 66.69754791259766,
      "activations/layer7_attention_weight_min": -64.16838836669922,
      "activations/layer8_attention_weight_max": 45.92025375366211,
      "activations/layer8_attention_weight_min": -48.39220428466797,
      "activations/layer9_attention_weight_max": 50.08843994140625,
      "activations/layer9_attention_weight_min": -47.161529541015625,
      "epoch": 17.55,
      "learning_rate": 3.717613636363636e-05,
      "loss": 2.7298,
      "step": 302100
    },
    {
      "activations/layer0_attention_weight_max": 16.334951400756836,
      "activations/layer0_attention_weight_min": -12.957884788513184,
      "activations/layer10_attention_weight_max": 33.164825439453125,
      "activations/layer10_attention_weight_min": -31.957347869873047,
      "activations/layer11_attention_weight_max": 31.493633270263672,
      "activations/layer11_attention_weight_min": -32.12185287475586,
      "activations/layer12_attention_weight_max": 16.3500919342041,
      "activations/layer12_attention_weight_min": -25.95628547668457,
      "activations/layer13_attention_weight_max": 30.149282455444336,
      "activations/layer13_attention_weight_min": -24.636606216430664,
      "activations/layer14_attention_weight_max": 31.84157943725586,
      "activations/layer14_attention_weight_min": -27.50886344909668,
      "activations/layer15_attention_weight_max": 29.865781784057617,
      "activations/layer15_attention_weight_min": -28.829448699951172,
      "activations/layer16_attention_weight_max": 29.332048416137695,
      "activations/layer16_attention_weight_min": -30.04184341430664,
      "activations/layer17_attention_weight_max": 49.72211456298828,
      "activations/layer17_attention_weight_min": -44.65510940551758,
      "activations/layer18_attention_weight_max": 43.42584228515625,
      "activations/layer18_attention_weight_min": -40.602603912353516,
      "activations/layer19_attention_weight_max": 23.17513084411621,
      "activations/layer19_attention_weight_min": -20.86355209350586,
      "activations/layer1_attention_weight_max": 15.836556434631348,
      "activations/layer1_attention_weight_min": -14.938488006591797,
      "activations/layer20_attention_weight_max": 20.563289642333984,
      "activations/layer20_attention_weight_min": -20.156370162963867,
      "activations/layer21_attention_weight_max": 32.132965087890625,
      "activations/layer21_attention_weight_min": -19.712196350097656,
      "activations/layer22_attention_weight_max": 28.855182647705078,
      "activations/layer22_attention_weight_min": -23.8614501953125,
      "activations/layer23_attention_weight_max": 34.49717712402344,
      "activations/layer23_attention_weight_min": -23.144550323486328,
      "activations/layer2_attention_weight_max": 30.213504791259766,
      "activations/layer2_attention_weight_min": -27.97775650024414,
      "activations/layer3_attention_weight_max": 85.47945404052734,
      "activations/layer3_attention_weight_min": -88.81509399414062,
      "activations/layer4_attention_weight_max": 84.85334777832031,
      "activations/layer4_attention_weight_min": -87.37816619873047,
      "activations/layer5_attention_weight_max": 67.57533264160156,
      "activations/layer5_attention_weight_min": -70.39591217041016,
      "activations/layer6_attention_weight_max": 48.39419174194336,
      "activations/layer6_attention_weight_min": -47.19118118286133,
      "activations/layer7_attention_weight_max": 63.740394592285156,
      "activations/layer7_attention_weight_min": -60.915279388427734,
      "activations/layer8_attention_weight_max": 43.380855560302734,
      "activations/layer8_attention_weight_min": -45.792083740234375,
      "activations/layer9_attention_weight_max": 41.70832824707031,
      "activations/layer9_attention_weight_min": -44.53600311279297,
      "epoch": 17.56,
      "learning_rate": 3.7157196969696965e-05,
      "loss": 2.7355,
      "step": 302150
    },
    {
      "activations/layer0_attention_weight_max": 16.252168655395508,
      "activations/layer0_attention_weight_min": -13.14412784576416,
      "activations/layer10_attention_weight_max": 34.280826568603516,
      "activations/layer10_attention_weight_min": -33.19790267944336,
      "activations/layer11_attention_weight_max": 31.314443588256836,
      "activations/layer11_attention_weight_min": -31.50356674194336,
      "activations/layer12_attention_weight_max": 18.202085494995117,
      "activations/layer12_attention_weight_min": -31.444194793701172,
      "activations/layer13_attention_weight_max": 30.256134033203125,
      "activations/layer13_attention_weight_min": -28.191741943359375,
      "activations/layer14_attention_weight_max": 32.0520133972168,
      "activations/layer14_attention_weight_min": -29.727359771728516,
      "activations/layer15_attention_weight_max": 31.22608184814453,
      "activations/layer15_attention_weight_min": -31.15169906616211,
      "activations/layer16_attention_weight_max": 31.834720611572266,
      "activations/layer16_attention_weight_min": -30.18073272705078,
      "activations/layer17_attention_weight_max": 45.69111633300781,
      "activations/layer17_attention_weight_min": -43.48814392089844,
      "activations/layer18_attention_weight_max": 43.28853225708008,
      "activations/layer18_attention_weight_min": -37.49456024169922,
      "activations/layer19_attention_weight_max": 23.0976619720459,
      "activations/layer19_attention_weight_min": -21.44562339782715,
      "activations/layer1_attention_weight_max": 16.16356658935547,
      "activations/layer1_attention_weight_min": -13.814583778381348,
      "activations/layer20_attention_weight_max": 21.08260154724121,
      "activations/layer20_attention_weight_min": -22.83253288269043,
      "activations/layer21_attention_weight_max": 36.34248352050781,
      "activations/layer21_attention_weight_min": -24.149497985839844,
      "activations/layer22_attention_weight_max": 25.61261558532715,
      "activations/layer22_attention_weight_min": -25.890087127685547,
      "activations/layer23_attention_weight_max": 37.91165542602539,
      "activations/layer23_attention_weight_min": -24.07394027709961,
      "activations/layer2_attention_weight_max": 31.64748191833496,
      "activations/layer2_attention_weight_min": -29.594078063964844,
      "activations/layer3_attention_weight_max": 89.82955169677734,
      "activations/layer3_attention_weight_min": -96.00421142578125,
      "activations/layer4_attention_weight_max": 87.03954315185547,
      "activations/layer4_attention_weight_min": -88.62950897216797,
      "activations/layer5_attention_weight_max": 64.59712219238281,
      "activations/layer5_attention_weight_min": -82.78840637207031,
      "activations/layer6_attention_weight_max": 48.14022445678711,
      "activations/layer6_attention_weight_min": -47.787471771240234,
      "activations/layer7_attention_weight_max": 66.31423950195312,
      "activations/layer7_attention_weight_min": -69.20564270019531,
      "activations/layer8_attention_weight_max": 43.780303955078125,
      "activations/layer8_attention_weight_min": -46.81995391845703,
      "activations/layer9_attention_weight_max": 41.20820236206055,
      "activations/layer9_attention_weight_min": -42.23893356323242,
      "epoch": 17.56,
      "learning_rate": 3.713825757575757e-05,
      "loss": 2.7321,
      "step": 302200
    },
    {
      "activations/layer0_attention_weight_max": 15.971318244934082,
      "activations/layer0_attention_weight_min": -13.277358055114746,
      "activations/layer10_attention_weight_max": 37.55155944824219,
      "activations/layer10_attention_weight_min": -36.753074645996094,
      "activations/layer11_attention_weight_max": 33.966732025146484,
      "activations/layer11_attention_weight_min": -34.29541778564453,
      "activations/layer12_attention_weight_max": 19.036090850830078,
      "activations/layer12_attention_weight_min": -27.63042640686035,
      "activations/layer13_attention_weight_max": 39.557220458984375,
      "activations/layer13_attention_weight_min": -29.968996047973633,
      "activations/layer14_attention_weight_max": 37.36933898925781,
      "activations/layer14_attention_weight_min": -31.012611389160156,
      "activations/layer15_attention_weight_max": 35.31237030029297,
      "activations/layer15_attention_weight_min": -29.612302780151367,
      "activations/layer16_attention_weight_max": 36.995174407958984,
      "activations/layer16_attention_weight_min": -34.20586395263672,
      "activations/layer17_attention_weight_max": 54.48855209350586,
      "activations/layer17_attention_weight_min": -46.992679595947266,
      "activations/layer18_attention_weight_max": 49.89571762084961,
      "activations/layer18_attention_weight_min": -43.533912658691406,
      "activations/layer19_attention_weight_max": 25.42765998840332,
      "activations/layer19_attention_weight_min": -23.72262191772461,
      "activations/layer1_attention_weight_max": 16.973045349121094,
      "activations/layer1_attention_weight_min": -14.61855697631836,
      "activations/layer20_attention_weight_max": 25.57768440246582,
      "activations/layer20_attention_weight_min": -22.632736206054688,
      "activations/layer21_attention_weight_max": 40.528656005859375,
      "activations/layer21_attention_weight_min": -25.946208953857422,
      "activations/layer22_attention_weight_max": 30.351226806640625,
      "activations/layer22_attention_weight_min": -27.082775115966797,
      "activations/layer23_attention_weight_max": 46.398406982421875,
      "activations/layer23_attention_weight_min": -25.777435302734375,
      "activations/layer2_attention_weight_max": 32.117454528808594,
      "activations/layer2_attention_weight_min": -30.143766403198242,
      "activations/layer3_attention_weight_max": 96.79795837402344,
      "activations/layer3_attention_weight_min": -93.38720703125,
      "activations/layer4_attention_weight_max": 95.45024871826172,
      "activations/layer4_attention_weight_min": -91.57543182373047,
      "activations/layer5_attention_weight_max": 68.79013061523438,
      "activations/layer5_attention_weight_min": -75.339111328125,
      "activations/layer6_attention_weight_max": 51.15815353393555,
      "activations/layer6_attention_weight_min": -50.51203155517578,
      "activations/layer7_attention_weight_max": 66.04551696777344,
      "activations/layer7_attention_weight_min": -66.40023040771484,
      "activations/layer8_attention_weight_max": 48.76909255981445,
      "activations/layer8_attention_weight_min": -49.6669807434082,
      "activations/layer9_attention_weight_max": 49.247005462646484,
      "activations/layer9_attention_weight_min": -46.52737808227539,
      "epoch": 17.56,
      "learning_rate": 3.711931818181818e-05,
      "loss": 2.7173,
      "step": 302250
    },
    {
      "activations/layer0_attention_weight_max": 16.357772827148438,
      "activations/layer0_attention_weight_min": -13.117305755615234,
      "activations/layer10_attention_weight_max": 39.40672302246094,
      "activations/layer10_attention_weight_min": -36.23070526123047,
      "activations/layer11_attention_weight_max": 34.00907897949219,
      "activations/layer11_attention_weight_min": -35.63690185546875,
      "activations/layer12_attention_weight_max": 23.16272735595703,
      "activations/layer12_attention_weight_min": -24.87131690979004,
      "activations/layer13_attention_weight_max": 34.82508087158203,
      "activations/layer13_attention_weight_min": -28.353618621826172,
      "activations/layer14_attention_weight_max": 36.880374908447266,
      "activations/layer14_attention_weight_min": -30.44627571105957,
      "activations/layer15_attention_weight_max": 35.26611328125,
      "activations/layer15_attention_weight_min": -30.3676700592041,
      "activations/layer16_attention_weight_max": 34.873111724853516,
      "activations/layer16_attention_weight_min": -33.229705810546875,
      "activations/layer17_attention_weight_max": 53.03068542480469,
      "activations/layer17_attention_weight_min": -46.3245735168457,
      "activations/layer18_attention_weight_max": 51.06242752075195,
      "activations/layer18_attention_weight_min": -39.88217544555664,
      "activations/layer19_attention_weight_max": 25.07035255432129,
      "activations/layer19_attention_weight_min": -23.18165397644043,
      "activations/layer1_attention_weight_max": 15.735251426696777,
      "activations/layer1_attention_weight_min": -14.228983879089355,
      "activations/layer20_attention_weight_max": 23.21510887145996,
      "activations/layer20_attention_weight_min": -19.19580841064453,
      "activations/layer21_attention_weight_max": 39.32729721069336,
      "activations/layer21_attention_weight_min": -24.443138122558594,
      "activations/layer22_attention_weight_max": 35.45819091796875,
      "activations/layer22_attention_weight_min": -26.953807830810547,
      "activations/layer23_attention_weight_max": 41.14730453491211,
      "activations/layer23_attention_weight_min": -24.08847427368164,
      "activations/layer2_attention_weight_max": 30.215681076049805,
      "activations/layer2_attention_weight_min": -29.3548641204834,
      "activations/layer3_attention_weight_max": 88.42518615722656,
      "activations/layer3_attention_weight_min": -89.30340576171875,
      "activations/layer4_attention_weight_max": 91.53412628173828,
      "activations/layer4_attention_weight_min": -89.76213836669922,
      "activations/layer5_attention_weight_max": 67.93802642822266,
      "activations/layer5_attention_weight_min": -73.14454650878906,
      "activations/layer6_attention_weight_max": 51.50298309326172,
      "activations/layer6_attention_weight_min": -51.76973342895508,
      "activations/layer7_attention_weight_max": 68.23944091796875,
      "activations/layer7_attention_weight_min": -63.36255645751953,
      "activations/layer8_attention_weight_max": 50.27449417114258,
      "activations/layer8_attention_weight_min": -51.731056213378906,
      "activations/layer9_attention_weight_max": 49.20182418823242,
      "activations/layer9_attention_weight_min": -49.05556106567383,
      "epoch": 17.57,
      "learning_rate": 3.710037878787878e-05,
      "loss": 2.7254,
      "step": 302300
    },
    {
      "activations/layer0_attention_weight_max": 15.416744232177734,
      "activations/layer0_attention_weight_min": -13.211050987243652,
      "activations/layer10_attention_weight_max": 37.14555358886719,
      "activations/layer10_attention_weight_min": -38.845970153808594,
      "activations/layer11_attention_weight_max": 32.756500244140625,
      "activations/layer11_attention_weight_min": -34.499656677246094,
      "activations/layer12_attention_weight_max": 18.227235794067383,
      "activations/layer12_attention_weight_min": -28.557880401611328,
      "activations/layer13_attention_weight_max": 30.82705307006836,
      "activations/layer13_attention_weight_min": -28.454570770263672,
      "activations/layer14_attention_weight_max": 34.37370681762695,
      "activations/layer14_attention_weight_min": -29.898622512817383,
      "activations/layer15_attention_weight_max": 32.71104431152344,
      "activations/layer15_attention_weight_min": -30.349285125732422,
      "activations/layer16_attention_weight_max": 35.040306091308594,
      "activations/layer16_attention_weight_min": -31.753353118896484,
      "activations/layer17_attention_weight_max": 56.87274932861328,
      "activations/layer17_attention_weight_min": -48.87372970581055,
      "activations/layer18_attention_weight_max": 51.29670715332031,
      "activations/layer18_attention_weight_min": -40.527687072753906,
      "activations/layer19_attention_weight_max": 24.28632164001465,
      "activations/layer19_attention_weight_min": -22.603635787963867,
      "activations/layer1_attention_weight_max": 16.271650314331055,
      "activations/layer1_attention_weight_min": -13.86499309539795,
      "activations/layer20_attention_weight_max": 21.496540069580078,
      "activations/layer20_attention_weight_min": -21.508808135986328,
      "activations/layer21_attention_weight_max": 35.15032196044922,
      "activations/layer21_attention_weight_min": -21.46489906311035,
      "activations/layer22_attention_weight_max": 27.09783935546875,
      "activations/layer22_attention_weight_min": -23.3822021484375,
      "activations/layer23_attention_weight_max": 36.48276138305664,
      "activations/layer23_attention_weight_min": -27.05073356628418,
      "activations/layer2_attention_weight_max": 30.99005889892578,
      "activations/layer2_attention_weight_min": -31.736997604370117,
      "activations/layer3_attention_weight_max": 87.49955749511719,
      "activations/layer3_attention_weight_min": -92.30804443359375,
      "activations/layer4_attention_weight_max": 89.50608825683594,
      "activations/layer4_attention_weight_min": -91.44249725341797,
      "activations/layer5_attention_weight_max": 65.29608154296875,
      "activations/layer5_attention_weight_min": -73.9664077758789,
      "activations/layer6_attention_weight_max": 50.23720932006836,
      "activations/layer6_attention_weight_min": -53.0257453918457,
      "activations/layer7_attention_weight_max": 66.68718719482422,
      "activations/layer7_attention_weight_min": -70.48007202148438,
      "activations/layer8_attention_weight_max": 47.79105758666992,
      "activations/layer8_attention_weight_min": -51.88420104980469,
      "activations/layer9_attention_weight_max": 44.2774543762207,
      "activations/layer9_attention_weight_min": -48.2439079284668,
      "epoch": 17.57,
      "learning_rate": 3.708143939393939e-05,
      "loss": 2.7409,
      "step": 302350
    },
    {
      "activations/layer0_attention_weight_max": 16.204849243164062,
      "activations/layer0_attention_weight_min": -13.88986873626709,
      "activations/layer10_attention_weight_max": 35.50305938720703,
      "activations/layer10_attention_weight_min": -33.37459182739258,
      "activations/layer11_attention_weight_max": 33.784690856933594,
      "activations/layer11_attention_weight_min": -34.47980499267578,
      "activations/layer12_attention_weight_max": 19.62824249267578,
      "activations/layer12_attention_weight_min": -24.35349464416504,
      "activations/layer13_attention_weight_max": 35.06951141357422,
      "activations/layer13_attention_weight_min": -27.830846786499023,
      "activations/layer14_attention_weight_max": 33.70082092285156,
      "activations/layer14_attention_weight_min": -32.04216003417969,
      "activations/layer15_attention_weight_max": 31.6373348236084,
      "activations/layer15_attention_weight_min": -31.941362380981445,
      "activations/layer16_attention_weight_max": 34.040382385253906,
      "activations/layer16_attention_weight_min": -32.54759216308594,
      "activations/layer17_attention_weight_max": 51.35437774658203,
      "activations/layer17_attention_weight_min": -46.755062103271484,
      "activations/layer18_attention_weight_max": 44.30778884887695,
      "activations/layer18_attention_weight_min": -41.94310760498047,
      "activations/layer19_attention_weight_max": 24.24690818786621,
      "activations/layer19_attention_weight_min": -25.048192977905273,
      "activations/layer1_attention_weight_max": 17.057453155517578,
      "activations/layer1_attention_weight_min": -14.16576862335205,
      "activations/layer20_attention_weight_max": 23.743892669677734,
      "activations/layer20_attention_weight_min": -23.382190704345703,
      "activations/layer21_attention_weight_max": 36.5225715637207,
      "activations/layer21_attention_weight_min": -24.9111385345459,
      "activations/layer22_attention_weight_max": 30.4660701751709,
      "activations/layer22_attention_weight_min": -28.76375389099121,
      "activations/layer23_attention_weight_max": 36.8832893371582,
      "activations/layer23_attention_weight_min": -25.0587158203125,
      "activations/layer2_attention_weight_max": 29.539073944091797,
      "activations/layer2_attention_weight_min": -28.190580368041992,
      "activations/layer3_attention_weight_max": 82.78852081298828,
      "activations/layer3_attention_weight_min": -88.97513580322266,
      "activations/layer4_attention_weight_max": 86.95946502685547,
      "activations/layer4_attention_weight_min": -80.94418334960938,
      "activations/layer5_attention_weight_max": 62.462501525878906,
      "activations/layer5_attention_weight_min": -71.66313934326172,
      "activations/layer6_attention_weight_max": 48.68205642700195,
      "activations/layer6_attention_weight_min": -48.08159255981445,
      "activations/layer7_attention_weight_max": 64.31256103515625,
      "activations/layer7_attention_weight_min": -62.118473052978516,
      "activations/layer8_attention_weight_max": 44.215232849121094,
      "activations/layer8_attention_weight_min": -45.50608825683594,
      "activations/layer9_attention_weight_max": 41.083457946777344,
      "activations/layer9_attention_weight_min": -45.35528564453125,
      "epoch": 17.57,
      "learning_rate": 3.706249999999999e-05,
      "loss": 2.743,
      "step": 302400
    },
    {
      "activations/layer0_attention_weight_max": 15.627511024475098,
      "activations/layer0_attention_weight_min": -13.16293716430664,
      "activations/layer10_attention_weight_max": 37.40571975708008,
      "activations/layer10_attention_weight_min": -39.94331359863281,
      "activations/layer11_attention_weight_max": 33.5838623046875,
      "activations/layer11_attention_weight_min": -35.93486022949219,
      "activations/layer12_attention_weight_max": 20.044666290283203,
      "activations/layer12_attention_weight_min": -26.910587310791016,
      "activations/layer13_attention_weight_max": 35.40802001953125,
      "activations/layer13_attention_weight_min": -30.102378845214844,
      "activations/layer14_attention_weight_max": 34.3869743347168,
      "activations/layer14_attention_weight_min": -29.567800521850586,
      "activations/layer15_attention_weight_max": 30.975833892822266,
      "activations/layer15_attention_weight_min": -31.043258666992188,
      "activations/layer16_attention_weight_max": 31.886449813842773,
      "activations/layer16_attention_weight_min": -32.14226150512695,
      "activations/layer17_attention_weight_max": 50.29490280151367,
      "activations/layer17_attention_weight_min": -45.148109436035156,
      "activations/layer18_attention_weight_max": 45.80358123779297,
      "activations/layer18_attention_weight_min": -40.24291229248047,
      "activations/layer19_attention_weight_max": 23.18514060974121,
      "activations/layer19_attention_weight_min": -23.161205291748047,
      "activations/layer1_attention_weight_max": 16.042604446411133,
      "activations/layer1_attention_weight_min": -15.526895523071289,
      "activations/layer20_attention_weight_max": 21.98541831970215,
      "activations/layer20_attention_weight_min": -22.253559112548828,
      "activations/layer21_attention_weight_max": 41.402469635009766,
      "activations/layer21_attention_weight_min": -24.841333389282227,
      "activations/layer22_attention_weight_max": 29.840715408325195,
      "activations/layer22_attention_weight_min": -26.751123428344727,
      "activations/layer23_attention_weight_max": 37.75739288330078,
      "activations/layer23_attention_weight_min": -27.711044311523438,
      "activations/layer2_attention_weight_max": 33.132083892822266,
      "activations/layer2_attention_weight_min": -30.298622131347656,
      "activations/layer3_attention_weight_max": 92.05481719970703,
      "activations/layer3_attention_weight_min": -89.9654769897461,
      "activations/layer4_attention_weight_max": 90.45773315429688,
      "activations/layer4_attention_weight_min": -87.28783416748047,
      "activations/layer5_attention_weight_max": 67.88436889648438,
      "activations/layer5_attention_weight_min": -71.40901184082031,
      "activations/layer6_attention_weight_max": 52.10709762573242,
      "activations/layer6_attention_weight_min": -52.022865295410156,
      "activations/layer7_attention_weight_max": 62.561546325683594,
      "activations/layer7_attention_weight_min": -64.12744903564453,
      "activations/layer8_attention_weight_max": 46.61315155029297,
      "activations/layer8_attention_weight_min": -52.849830627441406,
      "activations/layer9_attention_weight_max": 45.70818328857422,
      "activations/layer9_attention_weight_min": -49.639766693115234,
      "epoch": 17.57,
      "learning_rate": 3.70435606060606e-05,
      "loss": 2.7175,
      "step": 302450
    },
    {
      "activations/layer0_attention_weight_max": 16.041812896728516,
      "activations/layer0_attention_weight_min": -13.33980655670166,
      "activations/layer10_attention_weight_max": 35.135986328125,
      "activations/layer10_attention_weight_min": -36.50722122192383,
      "activations/layer11_attention_weight_max": 31.38250732421875,
      "activations/layer11_attention_weight_min": -32.947845458984375,
      "activations/layer12_attention_weight_max": 18.962045669555664,
      "activations/layer12_attention_weight_min": -24.628625869750977,
      "activations/layer13_attention_weight_max": 40.653717041015625,
      "activations/layer13_attention_weight_min": -29.98604393005371,
      "activations/layer14_attention_weight_max": 38.487571716308594,
      "activations/layer14_attention_weight_min": -31.433536529541016,
      "activations/layer15_attention_weight_max": 35.77202606201172,
      "activations/layer15_attention_weight_min": -30.944534301757812,
      "activations/layer16_attention_weight_max": 35.58760452270508,
      "activations/layer16_attention_weight_min": -33.43917465209961,
      "activations/layer17_attention_weight_max": 57.50361633300781,
      "activations/layer17_attention_weight_min": -45.561920166015625,
      "activations/layer18_attention_weight_max": 48.777427673339844,
      "activations/layer18_attention_weight_min": -38.5060920715332,
      "activations/layer19_attention_weight_max": 22.749635696411133,
      "activations/layer19_attention_weight_min": -22.35259437561035,
      "activations/layer1_attention_weight_max": 15.804917335510254,
      "activations/layer1_attention_weight_min": -14.415111541748047,
      "activations/layer20_attention_weight_max": 23.81084632873535,
      "activations/layer20_attention_weight_min": -20.649370193481445,
      "activations/layer21_attention_weight_max": 46.548583984375,
      "activations/layer21_attention_weight_min": -24.36642837524414,
      "activations/layer22_attention_weight_max": 30.178268432617188,
      "activations/layer22_attention_weight_min": -23.64560317993164,
      "activations/layer23_attention_weight_max": 35.99783706665039,
      "activations/layer23_attention_weight_min": -23.689132690429688,
      "activations/layer2_attention_weight_max": 33.54682159423828,
      "activations/layer2_attention_weight_min": -31.597877502441406,
      "activations/layer3_attention_weight_max": 93.34986877441406,
      "activations/layer3_attention_weight_min": -93.57048797607422,
      "activations/layer4_attention_weight_max": 90.46844482421875,
      "activations/layer4_attention_weight_min": -88.08098602294922,
      "activations/layer5_attention_weight_max": 64.43608093261719,
      "activations/layer5_attention_weight_min": -70.37583923339844,
      "activations/layer6_attention_weight_max": 49.3353157043457,
      "activations/layer6_attention_weight_min": -51.1558952331543,
      "activations/layer7_attention_weight_max": 64.90255737304688,
      "activations/layer7_attention_weight_min": -62.45867919921875,
      "activations/layer8_attention_weight_max": 47.877933502197266,
      "activations/layer8_attention_weight_min": -48.5595703125,
      "activations/layer9_attention_weight_max": 45.4102668762207,
      "activations/layer9_attention_weight_min": -46.42095184326172,
      "epoch": 17.58,
      "learning_rate": 3.702462121212121e-05,
      "loss": 2.7496,
      "step": 302500
    },
    {
      "activations/layer0_attention_weight_max": 15.56923770904541,
      "activations/layer0_attention_weight_min": -12.966596603393555,
      "activations/layer10_attention_weight_max": 36.69930648803711,
      "activations/layer10_attention_weight_min": -36.161930084228516,
      "activations/layer11_attention_weight_max": 33.88062286376953,
      "activations/layer11_attention_weight_min": -34.79791259765625,
      "activations/layer12_attention_weight_max": 19.20350456237793,
      "activations/layer12_attention_weight_min": -27.34916877746582,
      "activations/layer13_attention_weight_max": 35.69157409667969,
      "activations/layer13_attention_weight_min": -29.78209686279297,
      "activations/layer14_attention_weight_max": 36.893192291259766,
      "activations/layer14_attention_weight_min": -31.143966674804688,
      "activations/layer15_attention_weight_max": 35.642608642578125,
      "activations/layer15_attention_weight_min": -31.907264709472656,
      "activations/layer16_attention_weight_max": 34.18656539916992,
      "activations/layer16_attention_weight_min": -32.99889373779297,
      "activations/layer17_attention_weight_max": 50.45341873168945,
      "activations/layer17_attention_weight_min": -47.45146560668945,
      "activations/layer18_attention_weight_max": 46.9652214050293,
      "activations/layer18_attention_weight_min": -41.182586669921875,
      "activations/layer19_attention_weight_max": 22.977806091308594,
      "activations/layer19_attention_weight_min": -24.08331871032715,
      "activations/layer1_attention_weight_max": 15.205737113952637,
      "activations/layer1_attention_weight_min": -13.91579532623291,
      "activations/layer20_attention_weight_max": 21.21234130859375,
      "activations/layer20_attention_weight_min": -22.00328254699707,
      "activations/layer21_attention_weight_max": 40.689571380615234,
      "activations/layer21_attention_weight_min": -23.073232650756836,
      "activations/layer22_attention_weight_max": 26.181325912475586,
      "activations/layer22_attention_weight_min": -28.950681686401367,
      "activations/layer23_attention_weight_max": 35.204063415527344,
      "activations/layer23_attention_weight_min": -26.681678771972656,
      "activations/layer2_attention_weight_max": 29.33267593383789,
      "activations/layer2_attention_weight_min": -28.81899070739746,
      "activations/layer3_attention_weight_max": 82.19497680664062,
      "activations/layer3_attention_weight_min": -86.0921859741211,
      "activations/layer4_attention_weight_max": 81.55162811279297,
      "activations/layer4_attention_weight_min": -87.60807037353516,
      "activations/layer5_attention_weight_max": 63.72090148925781,
      "activations/layer5_attention_weight_min": -73.52273559570312,
      "activations/layer6_attention_weight_max": 45.31254959106445,
      "activations/layer6_attention_weight_min": -49.08857345581055,
      "activations/layer7_attention_weight_max": 61.897586822509766,
      "activations/layer7_attention_weight_min": -62.65473175048828,
      "activations/layer8_attention_weight_max": 45.92079544067383,
      "activations/layer8_attention_weight_min": -48.44961166381836,
      "activations/layer9_attention_weight_max": 43.892372131347656,
      "activations/layer9_attention_weight_min": -46.42549514770508,
      "epoch": 17.58,
      "learning_rate": 3.700568181818182e-05,
      "loss": 2.7349,
      "step": 302550
    },
    {
      "activations/layer0_attention_weight_max": 15.920531272888184,
      "activations/layer0_attention_weight_min": -13.152266502380371,
      "activations/layer10_attention_weight_max": 34.63056182861328,
      "activations/layer10_attention_weight_min": -33.791778564453125,
      "activations/layer11_attention_weight_max": 30.91963768005371,
      "activations/layer11_attention_weight_min": -31.390241622924805,
      "activations/layer12_attention_weight_max": 19.45256805419922,
      "activations/layer12_attention_weight_min": -25.50139045715332,
      "activations/layer13_attention_weight_max": 40.4256591796875,
      "activations/layer13_attention_weight_min": -29.14720344543457,
      "activations/layer14_attention_weight_max": 37.05939865112305,
      "activations/layer14_attention_weight_min": -29.589767456054688,
      "activations/layer15_attention_weight_max": 35.90109634399414,
      "activations/layer15_attention_weight_min": -28.87801170349121,
      "activations/layer16_attention_weight_max": 35.92231369018555,
      "activations/layer16_attention_weight_min": -32.280052185058594,
      "activations/layer17_attention_weight_max": 55.72468948364258,
      "activations/layer17_attention_weight_min": -46.674034118652344,
      "activations/layer18_attention_weight_max": 52.33674240112305,
      "activations/layer18_attention_weight_min": -40.71666717529297,
      "activations/layer19_attention_weight_max": 24.85451316833496,
      "activations/layer19_attention_weight_min": -22.469547271728516,
      "activations/layer1_attention_weight_max": 15.564923286437988,
      "activations/layer1_attention_weight_min": -11.76571273803711,
      "activations/layer20_attention_weight_max": 26.208433151245117,
      "activations/layer20_attention_weight_min": -20.031665802001953,
      "activations/layer21_attention_weight_max": 47.07230758666992,
      "activations/layer21_attention_weight_min": -21.454439163208008,
      "activations/layer22_attention_weight_max": 32.83399963378906,
      "activations/layer22_attention_weight_min": -25.900197982788086,
      "activations/layer23_attention_weight_max": 40.11566925048828,
      "activations/layer23_attention_weight_min": -24.85420799255371,
      "activations/layer2_attention_weight_max": 29.61116600036621,
      "activations/layer2_attention_weight_min": -28.94174575805664,
      "activations/layer3_attention_weight_max": 81.78284454345703,
      "activations/layer3_attention_weight_min": -83.10151672363281,
      "activations/layer4_attention_weight_max": 83.15153503417969,
      "activations/layer4_attention_weight_min": -82.19375610351562,
      "activations/layer5_attention_weight_max": 60.76282501220703,
      "activations/layer5_attention_weight_min": -71.39024353027344,
      "activations/layer6_attention_weight_max": 46.99055480957031,
      "activations/layer6_attention_weight_min": -46.24708557128906,
      "activations/layer7_attention_weight_max": 63.298072814941406,
      "activations/layer7_attention_weight_min": -59.87278747558594,
      "activations/layer8_attention_weight_max": 44.15123748779297,
      "activations/layer8_attention_weight_min": -45.393917083740234,
      "activations/layer9_attention_weight_max": 41.94612503051758,
      "activations/layer9_attention_weight_min": -44.639156341552734,
      "epoch": 17.58,
      "learning_rate": 3.6986742424242426e-05,
      "loss": 2.7668,
      "step": 302600
    },
    {
      "activations/layer0_attention_weight_max": 15.54323673248291,
      "activations/layer0_attention_weight_min": -13.921991348266602,
      "activations/layer10_attention_weight_max": 36.018550872802734,
      "activations/layer10_attention_weight_min": -36.77841567993164,
      "activations/layer11_attention_weight_max": 34.64406204223633,
      "activations/layer11_attention_weight_min": -36.3381233215332,
      "activations/layer12_attention_weight_max": 19.580093383789062,
      "activations/layer12_attention_weight_min": -23.467273712158203,
      "activations/layer13_attention_weight_max": 36.26121139526367,
      "activations/layer13_attention_weight_min": -30.20490074157715,
      "activations/layer14_attention_weight_max": 36.94461441040039,
      "activations/layer14_attention_weight_min": -34.562171936035156,
      "activations/layer15_attention_weight_max": 34.90367889404297,
      "activations/layer15_attention_weight_min": -33.385986328125,
      "activations/layer16_attention_weight_max": 37.100589752197266,
      "activations/layer16_attention_weight_min": -35.12922668457031,
      "activations/layer17_attention_weight_max": 56.812225341796875,
      "activations/layer17_attention_weight_min": -52.33265686035156,
      "activations/layer18_attention_weight_max": 51.46894454956055,
      "activations/layer18_attention_weight_min": -46.3176383972168,
      "activations/layer19_attention_weight_max": 23.509605407714844,
      "activations/layer19_attention_weight_min": -24.755834579467773,
      "activations/layer1_attention_weight_max": 17.396133422851562,
      "activations/layer1_attention_weight_min": -14.8812837600708,
      "activations/layer20_attention_weight_max": 23.75901222229004,
      "activations/layer20_attention_weight_min": -21.676250457763672,
      "activations/layer21_attention_weight_max": 33.703861236572266,
      "activations/layer21_attention_weight_min": -21.960243225097656,
      "activations/layer22_attention_weight_max": 32.45038604736328,
      "activations/layer22_attention_weight_min": -26.574445724487305,
      "activations/layer23_attention_weight_max": 38.84052276611328,
      "activations/layer23_attention_weight_min": -22.60765838623047,
      "activations/layer2_attention_weight_max": 32.37262725830078,
      "activations/layer2_attention_weight_min": -30.592344284057617,
      "activations/layer3_attention_weight_max": 88.4098892211914,
      "activations/layer3_attention_weight_min": -91.20594024658203,
      "activations/layer4_attention_weight_max": 89.6561050415039,
      "activations/layer4_attention_weight_min": -89.1374740600586,
      "activations/layer5_attention_weight_max": 67.84869384765625,
      "activations/layer5_attention_weight_min": -74.36617279052734,
      "activations/layer6_attention_weight_max": 51.616390228271484,
      "activations/layer6_attention_weight_min": -50.37207794189453,
      "activations/layer7_attention_weight_max": 70.05133819580078,
      "activations/layer7_attention_weight_min": -64.08587646484375,
      "activations/layer8_attention_weight_max": 47.633056640625,
      "activations/layer8_attention_weight_min": -51.405269622802734,
      "activations/layer9_attention_weight_max": 44.2696418762207,
      "activations/layer9_attention_weight_min": -45.91788101196289,
      "epoch": 17.59,
      "learning_rate": 3.696780303030303e-05,
      "loss": 2.748,
      "step": 302650
    },
    {
      "activations/layer0_attention_weight_max": 15.726729393005371,
      "activations/layer0_attention_weight_min": -12.926128387451172,
      "activations/layer10_attention_weight_max": 34.999366760253906,
      "activations/layer10_attention_weight_min": -34.50609588623047,
      "activations/layer11_attention_weight_max": 32.998531341552734,
      "activations/layer11_attention_weight_min": -33.137264251708984,
      "activations/layer12_attention_weight_max": 17.8447322845459,
      "activations/layer12_attention_weight_min": -31.371126174926758,
      "activations/layer13_attention_weight_max": 32.09346389770508,
      "activations/layer13_attention_weight_min": -26.58147621154785,
      "activations/layer14_attention_weight_max": 32.40995788574219,
      "activations/layer14_attention_weight_min": -29.474637985229492,
      "activations/layer15_attention_weight_max": 28.379791259765625,
      "activations/layer15_attention_weight_min": -28.6279354095459,
      "activations/layer16_attention_weight_max": 30.647382736206055,
      "activations/layer16_attention_weight_min": -31.145740509033203,
      "activations/layer17_attention_weight_max": 45.39385223388672,
      "activations/layer17_attention_weight_min": -42.93973922729492,
      "activations/layer18_attention_weight_max": 44.458396911621094,
      "activations/layer18_attention_weight_min": -35.76543426513672,
      "activations/layer19_attention_weight_max": 22.4095458984375,
      "activations/layer19_attention_weight_min": -20.860340118408203,
      "activations/layer1_attention_weight_max": 16.17876625061035,
      "activations/layer1_attention_weight_min": -13.873798370361328,
      "activations/layer20_attention_weight_max": 21.254106521606445,
      "activations/layer20_attention_weight_min": -19.496013641357422,
      "activations/layer21_attention_weight_max": 33.33644104003906,
      "activations/layer21_attention_weight_min": -19.951919555664062,
      "activations/layer22_attention_weight_max": 28.1921329498291,
      "activations/layer22_attention_weight_min": -25.028594970703125,
      "activations/layer23_attention_weight_max": 35.081077575683594,
      "activations/layer23_attention_weight_min": -23.890241622924805,
      "activations/layer2_attention_weight_max": 30.80068588256836,
      "activations/layer2_attention_weight_min": -29.10439109802246,
      "activations/layer3_attention_weight_max": 94.745361328125,
      "activations/layer3_attention_weight_min": -90.56676483154297,
      "activations/layer4_attention_weight_max": 95.48900604248047,
      "activations/layer4_attention_weight_min": -89.6147689819336,
      "activations/layer5_attention_weight_max": 65.29348754882812,
      "activations/layer5_attention_weight_min": -74.54844665527344,
      "activations/layer6_attention_weight_max": 50.901268005371094,
      "activations/layer6_attention_weight_min": -50.20622253417969,
      "activations/layer7_attention_weight_max": 61.79193878173828,
      "activations/layer7_attention_weight_min": -63.2933464050293,
      "activations/layer8_attention_weight_max": 45.48726272583008,
      "activations/layer8_attention_weight_min": -48.07857131958008,
      "activations/layer9_attention_weight_max": 45.07217025756836,
      "activations/layer9_attention_weight_min": -46.10358810424805,
      "epoch": 17.59,
      "learning_rate": 3.6948863636363636e-05,
      "loss": 2.7465,
      "step": 302700
    },
    {
      "activations/layer0_attention_weight_max": 15.478669166564941,
      "activations/layer0_attention_weight_min": -13.341536521911621,
      "activations/layer10_attention_weight_max": 32.95219421386719,
      "activations/layer10_attention_weight_min": -33.18678665161133,
      "activations/layer11_attention_weight_max": 32.21700668334961,
      "activations/layer11_attention_weight_min": -32.8758659362793,
      "activations/layer12_attention_weight_max": 18.90105628967285,
      "activations/layer12_attention_weight_min": -29.926061630249023,
      "activations/layer13_attention_weight_max": 33.23613357543945,
      "activations/layer13_attention_weight_min": -25.64634132385254,
      "activations/layer14_attention_weight_max": 35.61077117919922,
      "activations/layer14_attention_weight_min": -30.49991798400879,
      "activations/layer15_attention_weight_max": 34.821292877197266,
      "activations/layer15_attention_weight_min": -32.00690460205078,
      "activations/layer16_attention_weight_max": 35.40969467163086,
      "activations/layer16_attention_weight_min": -32.05762481689453,
      "activations/layer17_attention_weight_max": 52.93604278564453,
      "activations/layer17_attention_weight_min": -46.80511474609375,
      "activations/layer18_attention_weight_max": 48.905296325683594,
      "activations/layer18_attention_weight_min": -40.13804626464844,
      "activations/layer19_attention_weight_max": 27.677270889282227,
      "activations/layer19_attention_weight_min": -22.96961212158203,
      "activations/layer1_attention_weight_max": 15.537976264953613,
      "activations/layer1_attention_weight_min": -12.869756698608398,
      "activations/layer20_attention_weight_max": 27.354108810424805,
      "activations/layer20_attention_weight_min": -21.466514587402344,
      "activations/layer21_attention_weight_max": 35.577735900878906,
      "activations/layer21_attention_weight_min": -23.73143196105957,
      "activations/layer22_attention_weight_max": 31.22095489501953,
      "activations/layer22_attention_weight_min": -28.42705726623535,
      "activations/layer23_attention_weight_max": 37.302799224853516,
      "activations/layer23_attention_weight_min": -24.256637573242188,
      "activations/layer2_attention_weight_max": 26.7047119140625,
      "activations/layer2_attention_weight_min": -27.341899871826172,
      "activations/layer3_attention_weight_max": 78.60326385498047,
      "activations/layer3_attention_weight_min": -81.41499328613281,
      "activations/layer4_attention_weight_max": 79.04395294189453,
      "activations/layer4_attention_weight_min": -79.47272491455078,
      "activations/layer5_attention_weight_max": 59.932167053222656,
      "activations/layer5_attention_weight_min": -73.93467712402344,
      "activations/layer6_attention_weight_max": 45.41133499145508,
      "activations/layer6_attention_weight_min": -47.94961166381836,
      "activations/layer7_attention_weight_max": 60.2012939453125,
      "activations/layer7_attention_weight_min": -61.0513801574707,
      "activations/layer8_attention_weight_max": 42.64627456665039,
      "activations/layer8_attention_weight_min": -45.092716217041016,
      "activations/layer9_attention_weight_max": 39.8824577331543,
      "activations/layer9_attention_weight_min": -42.049560546875,
      "epoch": 17.59,
      "learning_rate": 3.692992424242424e-05,
      "loss": 2.7428,
      "step": 302750
    },
    {
      "activations/layer0_attention_weight_max": 16.46053123474121,
      "activations/layer0_attention_weight_min": -15.5872802734375,
      "activations/layer10_attention_weight_max": 37.60609817504883,
      "activations/layer10_attention_weight_min": -34.98090362548828,
      "activations/layer11_attention_weight_max": 33.842384338378906,
      "activations/layer11_attention_weight_min": -34.15346908569336,
      "activations/layer12_attention_weight_max": 19.527130126953125,
      "activations/layer12_attention_weight_min": -24.99598503112793,
      "activations/layer13_attention_weight_max": 38.93438720703125,
      "activations/layer13_attention_weight_min": -31.252817153930664,
      "activations/layer14_attention_weight_max": 38.87763977050781,
      "activations/layer14_attention_weight_min": -32.797393798828125,
      "activations/layer15_attention_weight_max": 34.943790435791016,
      "activations/layer15_attention_weight_min": -31.37374496459961,
      "activations/layer16_attention_weight_max": 34.78347396850586,
      "activations/layer16_attention_weight_min": -32.63759994506836,
      "activations/layer17_attention_weight_max": 50.41822052001953,
      "activations/layer17_attention_weight_min": -47.279258728027344,
      "activations/layer18_attention_weight_max": 45.868812561035156,
      "activations/layer18_attention_weight_min": -41.82956314086914,
      "activations/layer19_attention_weight_max": 22.56904411315918,
      "activations/layer19_attention_weight_min": -24.018571853637695,
      "activations/layer1_attention_weight_max": 15.48606014251709,
      "activations/layer1_attention_weight_min": -14.506692886352539,
      "activations/layer20_attention_weight_max": 22.263851165771484,
      "activations/layer20_attention_weight_min": -23.479900360107422,
      "activations/layer21_attention_weight_max": 32.97880935668945,
      "activations/layer21_attention_weight_min": -23.216827392578125,
      "activations/layer22_attention_weight_max": 28.13068199157715,
      "activations/layer22_attention_weight_min": -27.011642456054688,
      "activations/layer23_attention_weight_max": 35.560691833496094,
      "activations/layer23_attention_weight_min": -26.407821655273438,
      "activations/layer2_attention_weight_max": 31.13800621032715,
      "activations/layer2_attention_weight_min": -30.881610870361328,
      "activations/layer3_attention_weight_max": 90.41246795654297,
      "activations/layer3_attention_weight_min": -92.97317504882812,
      "activations/layer4_attention_weight_max": 94.81912231445312,
      "activations/layer4_attention_weight_min": -88.5025863647461,
      "activations/layer5_attention_weight_max": 66.11783599853516,
      "activations/layer5_attention_weight_min": -74.73660278320312,
      "activations/layer6_attention_weight_max": 49.61111831665039,
      "activations/layer6_attention_weight_min": -49.765342712402344,
      "activations/layer7_attention_weight_max": 63.75054168701172,
      "activations/layer7_attention_weight_min": -64.61164093017578,
      "activations/layer8_attention_weight_max": 46.27299880981445,
      "activations/layer8_attention_weight_min": -49.56843948364258,
      "activations/layer9_attention_weight_max": 44.46686935424805,
      "activations/layer9_attention_weight_min": -45.93743133544922,
      "epoch": 17.59,
      "learning_rate": 3.6910984848484845e-05,
      "loss": 2.7526,
      "step": 302800
    },
    {
      "activations/layer0_attention_weight_max": 16.186004638671875,
      "activations/layer0_attention_weight_min": -15.10955810546875,
      "activations/layer10_attention_weight_max": 35.594390869140625,
      "activations/layer10_attention_weight_min": -35.068363189697266,
      "activations/layer11_attention_weight_max": 32.75076675415039,
      "activations/layer11_attention_weight_min": -34.58573913574219,
      "activations/layer12_attention_weight_max": 22.024391174316406,
      "activations/layer12_attention_weight_min": -25.516454696655273,
      "activations/layer13_attention_weight_max": 33.4912109375,
      "activations/layer13_attention_weight_min": -28.75473976135254,
      "activations/layer14_attention_weight_max": 36.91319274902344,
      "activations/layer14_attention_weight_min": -29.595623016357422,
      "activations/layer15_attention_weight_max": 32.711299896240234,
      "activations/layer15_attention_weight_min": -28.51980209350586,
      "activations/layer16_attention_weight_max": 33.473690032958984,
      "activations/layer16_attention_weight_min": -30.52549171447754,
      "activations/layer17_attention_weight_max": 50.13986587524414,
      "activations/layer17_attention_weight_min": -43.47470474243164,
      "activations/layer18_attention_weight_max": 46.84278106689453,
      "activations/layer18_attention_weight_min": -40.14577102661133,
      "activations/layer19_attention_weight_max": 22.903207778930664,
      "activations/layer19_attention_weight_min": -22.952146530151367,
      "activations/layer1_attention_weight_max": 16.15250587463379,
      "activations/layer1_attention_weight_min": -14.24181079864502,
      "activations/layer20_attention_weight_max": 21.962926864624023,
      "activations/layer20_attention_weight_min": -23.12840461730957,
      "activations/layer21_attention_weight_max": 35.889404296875,
      "activations/layer21_attention_weight_min": -21.73955726623535,
      "activations/layer22_attention_weight_max": 29.151161193847656,
      "activations/layer22_attention_weight_min": -25.856626510620117,
      "activations/layer23_attention_weight_max": 38.62187957763672,
      "activations/layer23_attention_weight_min": -24.089527130126953,
      "activations/layer2_attention_weight_max": 31.135513305664062,
      "activations/layer2_attention_weight_min": -29.084613800048828,
      "activations/layer3_attention_weight_max": 90.14668273925781,
      "activations/layer3_attention_weight_min": -92.17573547363281,
      "activations/layer4_attention_weight_max": 89.99754333496094,
      "activations/layer4_attention_weight_min": -86.74639129638672,
      "activations/layer5_attention_weight_max": 61.698150634765625,
      "activations/layer5_attention_weight_min": -72.86012268066406,
      "activations/layer6_attention_weight_max": 50.19483947753906,
      "activations/layer6_attention_weight_min": -50.7733039855957,
      "activations/layer7_attention_weight_max": 63.536956787109375,
      "activations/layer7_attention_weight_min": -66.61105346679688,
      "activations/layer8_attention_weight_max": 44.85939407348633,
      "activations/layer8_attention_weight_min": -48.82097625732422,
      "activations/layer9_attention_weight_max": 43.952144622802734,
      "activations/layer9_attention_weight_min": -44.86614227294922,
      "epoch": 17.6,
      "learning_rate": 3.6892045454545454e-05,
      "loss": 2.7431,
      "step": 302850
    },
    {
      "activations/layer0_attention_weight_max": 16.30472183227539,
      "activations/layer0_attention_weight_min": -13.028648376464844,
      "activations/layer10_attention_weight_max": 36.88723373413086,
      "activations/layer10_attention_weight_min": -37.586856842041016,
      "activations/layer11_attention_weight_max": 33.222198486328125,
      "activations/layer11_attention_weight_min": -34.75625991821289,
      "activations/layer12_attention_weight_max": 19.527589797973633,
      "activations/layer12_attention_weight_min": -30.496192932128906,
      "activations/layer13_attention_weight_max": 35.89284896850586,
      "activations/layer13_attention_weight_min": -26.601985931396484,
      "activations/layer14_attention_weight_max": 36.884822845458984,
      "activations/layer14_attention_weight_min": -29.216630935668945,
      "activations/layer15_attention_weight_max": 33.48479461669922,
      "activations/layer15_attention_weight_min": -29.915393829345703,
      "activations/layer16_attention_weight_max": 35.597984313964844,
      "activations/layer16_attention_weight_min": -33.02397918701172,
      "activations/layer17_attention_weight_max": 54.54812240600586,
      "activations/layer17_attention_weight_min": -47.185970306396484,
      "activations/layer18_attention_weight_max": 50.528053283691406,
      "activations/layer18_attention_weight_min": -41.883941650390625,
      "activations/layer19_attention_weight_max": 22.193084716796875,
      "activations/layer19_attention_weight_min": -22.662742614746094,
      "activations/layer1_attention_weight_max": 16.225257873535156,
      "activations/layer1_attention_weight_min": -14.106255531311035,
      "activations/layer20_attention_weight_max": 20.837993621826172,
      "activations/layer20_attention_weight_min": -22.72674560546875,
      "activations/layer21_attention_weight_max": 33.387481689453125,
      "activations/layer21_attention_weight_min": -18.56148338317871,
      "activations/layer22_attention_weight_max": 26.863475799560547,
      "activations/layer22_attention_weight_min": -26.16586685180664,
      "activations/layer23_attention_weight_max": 35.89373779296875,
      "activations/layer23_attention_weight_min": -24.60421371459961,
      "activations/layer2_attention_weight_max": 32.809547424316406,
      "activations/layer2_attention_weight_min": -30.420522689819336,
      "activations/layer3_attention_weight_max": 86.09398651123047,
      "activations/layer3_attention_weight_min": -91.07518768310547,
      "activations/layer4_attention_weight_max": 87.7502670288086,
      "activations/layer4_attention_weight_min": -86.69673919677734,
      "activations/layer5_attention_weight_max": 65.03280639648438,
      "activations/layer5_attention_weight_min": -74.05117797851562,
      "activations/layer6_attention_weight_max": 47.0981330871582,
      "activations/layer6_attention_weight_min": -48.9827995300293,
      "activations/layer7_attention_weight_max": 71.37741088867188,
      "activations/layer7_attention_weight_min": -62.904052734375,
      "activations/layer8_attention_weight_max": 48.79942321777344,
      "activations/layer8_attention_weight_min": -49.691314697265625,
      "activations/layer9_attention_weight_max": 51.247676849365234,
      "activations/layer9_attention_weight_min": -47.03156280517578,
      "epoch": 17.6,
      "learning_rate": 3.6873106060606055e-05,
      "loss": 2.7446,
      "step": 302900
    },
    {
      "activations/layer0_attention_weight_max": 15.682053565979004,
      "activations/layer0_attention_weight_min": -13.250018119812012,
      "activations/layer10_attention_weight_max": 40.1585578918457,
      "activations/layer10_attention_weight_min": -39.02455139160156,
      "activations/layer11_attention_weight_max": 37.86516571044922,
      "activations/layer11_attention_weight_min": -38.33788299560547,
      "activations/layer12_attention_weight_max": 18.532901763916016,
      "activations/layer12_attention_weight_min": -25.610761642456055,
      "activations/layer13_attention_weight_max": 47.039371490478516,
      "activations/layer13_attention_weight_min": -30.708763122558594,
      "activations/layer14_attention_weight_max": 45.418792724609375,
      "activations/layer14_attention_weight_min": -33.120880126953125,
      "activations/layer15_attention_weight_max": 45.32960510253906,
      "activations/layer15_attention_weight_min": -35.44929122924805,
      "activations/layer16_attention_weight_max": 39.3880615234375,
      "activations/layer16_attention_weight_min": -35.207088470458984,
      "activations/layer17_attention_weight_max": 60.91742706298828,
      "activations/layer17_attention_weight_min": -53.07658767700195,
      "activations/layer18_attention_weight_max": 56.424407958984375,
      "activations/layer18_attention_weight_min": -47.812740325927734,
      "activations/layer19_attention_weight_max": 25.90032386779785,
      "activations/layer19_attention_weight_min": -25.24083709716797,
      "activations/layer1_attention_weight_max": 16.051362991333008,
      "activations/layer1_attention_weight_min": -15.204325675964355,
      "activations/layer20_attention_weight_max": 26.781503677368164,
      "activations/layer20_attention_weight_min": -23.084800720214844,
      "activations/layer21_attention_weight_max": 49.06275177001953,
      "activations/layer21_attention_weight_min": -25.68092918395996,
      "activations/layer22_attention_weight_max": 34.266845703125,
      "activations/layer22_attention_weight_min": -28.447265625,
      "activations/layer23_attention_weight_max": 39.60365295410156,
      "activations/layer23_attention_weight_min": -22.507991790771484,
      "activations/layer2_attention_weight_max": 34.377925872802734,
      "activations/layer2_attention_weight_min": -31.77826690673828,
      "activations/layer3_attention_weight_max": 96.45201110839844,
      "activations/layer3_attention_weight_min": -98.07756805419922,
      "activations/layer4_attention_weight_max": 93.65038299560547,
      "activations/layer4_attention_weight_min": -91.28624725341797,
      "activations/layer5_attention_weight_max": 62.07994842529297,
      "activations/layer5_attention_weight_min": -72.76887512207031,
      "activations/layer6_attention_weight_max": 51.34898376464844,
      "activations/layer6_attention_weight_min": -50.504642486572266,
      "activations/layer7_attention_weight_max": 69.63746643066406,
      "activations/layer7_attention_weight_min": -70.36908721923828,
      "activations/layer8_attention_weight_max": 48.416717529296875,
      "activations/layer8_attention_weight_min": -49.62370681762695,
      "activations/layer9_attention_weight_max": 50.232357025146484,
      "activations/layer9_attention_weight_min": -50.47274398803711,
      "epoch": 17.6,
      "learning_rate": 3.6854166666666663e-05,
      "loss": 2.7388,
      "step": 302950
    },
    {
      "activations/layer0_attention_weight_max": 15.844636917114258,
      "activations/layer0_attention_weight_min": -13.361659049987793,
      "activations/layer10_attention_weight_max": 36.84629821777344,
      "activations/layer10_attention_weight_min": -37.43275833129883,
      "activations/layer11_attention_weight_max": 32.04175567626953,
      "activations/layer11_attention_weight_min": -35.59061050415039,
      "activations/layer12_attention_weight_max": 19.378467559814453,
      "activations/layer12_attention_weight_min": -22.845237731933594,
      "activations/layer13_attention_weight_max": 33.24134826660156,
      "activations/layer13_attention_weight_min": -28.422290802001953,
      "activations/layer14_attention_weight_max": 34.790016174316406,
      "activations/layer14_attention_weight_min": -31.44758415222168,
      "activations/layer15_attention_weight_max": 31.2231388092041,
      "activations/layer15_attention_weight_min": -29.75711441040039,
      "activations/layer16_attention_weight_max": 34.1514778137207,
      "activations/layer16_attention_weight_min": -31.795324325561523,
      "activations/layer17_attention_weight_max": 51.728206634521484,
      "activations/layer17_attention_weight_min": -44.138328552246094,
      "activations/layer18_attention_weight_max": 47.09326171875,
      "activations/layer18_attention_weight_min": -41.36112594604492,
      "activations/layer19_attention_weight_max": 21.91860008239746,
      "activations/layer19_attention_weight_min": -22.503217697143555,
      "activations/layer1_attention_weight_max": 15.990039825439453,
      "activations/layer1_attention_weight_min": -13.706154823303223,
      "activations/layer20_attention_weight_max": 22.714820861816406,
      "activations/layer20_attention_weight_min": -22.094362258911133,
      "activations/layer21_attention_weight_max": 35.60238265991211,
      "activations/layer21_attention_weight_min": -21.90262222290039,
      "activations/layer22_attention_weight_max": 31.25472068786621,
      "activations/layer22_attention_weight_min": -24.790687561035156,
      "activations/layer23_attention_weight_max": 37.01408386230469,
      "activations/layer23_attention_weight_min": -23.018421173095703,
      "activations/layer2_attention_weight_max": 29.945148468017578,
      "activations/layer2_attention_weight_min": -29.47182273864746,
      "activations/layer3_attention_weight_max": 87.25041198730469,
      "activations/layer3_attention_weight_min": -89.8142318725586,
      "activations/layer4_attention_weight_max": 89.4870834350586,
      "activations/layer4_attention_weight_min": -89.89399719238281,
      "activations/layer5_attention_weight_max": 65.18514251708984,
      "activations/layer5_attention_weight_min": -74.9687271118164,
      "activations/layer6_attention_weight_max": 51.10765075683594,
      "activations/layer6_attention_weight_min": -52.95570755004883,
      "activations/layer7_attention_weight_max": 64.75732421875,
      "activations/layer7_attention_weight_min": -65.7513427734375,
      "activations/layer8_attention_weight_max": 48.25495147705078,
      "activations/layer8_attention_weight_min": -55.04208755493164,
      "activations/layer9_attention_weight_max": 44.6131477355957,
      "activations/layer9_attention_weight_min": -47.014522552490234,
      "epoch": 17.61,
      "learning_rate": 3.683522727272727e-05,
      "loss": 2.7454,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4712,
      "eval_samples_per_second": 506.892,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4712,
      "eval_openwebtext_samples_per_second": 506.892,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9607,
      "eval_wikitext_samples_per_second": 232.573,
      "step": 303000
    },
    {
      "epoch": 17.61,
      "eval_lambada_loss": 2.501953125,
      "eval_lambada_ppl": 12.206311145618837,
      "eval_lambada_runtime": 9.521,
      "eval_lambada_samples_per_second": 511.395,
      "step": 303000
    },
    {
      "activations/layer0_attention_weight_max": 15.426358222961426,
      "activations/layer0_attention_weight_min": -13.447876930236816,
      "activations/layer10_attention_weight_max": 33.40801239013672,
      "activations/layer10_attention_weight_min": -33.99147033691406,
      "activations/layer11_attention_weight_max": 31.48123550415039,
      "activations/layer11_attention_weight_min": -33.55308151245117,
      "activations/layer12_attention_weight_max": 20.436918258666992,
      "activations/layer12_attention_weight_min": -24.250694274902344,
      "activations/layer13_attention_weight_max": 40.236778259277344,
      "activations/layer13_attention_weight_min": -28.57511329650879,
      "activations/layer14_attention_weight_max": 36.363216400146484,
      "activations/layer14_attention_weight_min": -29.953214645385742,
      "activations/layer15_attention_weight_max": 33.53607177734375,
      "activations/layer15_attention_weight_min": -29.093856811523438,
      "activations/layer16_attention_weight_max": 32.85364532470703,
      "activations/layer16_attention_weight_min": -33.080665588378906,
      "activations/layer17_attention_weight_max": 51.1273307800293,
      "activations/layer17_attention_weight_min": -44.3091926574707,
      "activations/layer18_attention_weight_max": 47.54441452026367,
      "activations/layer18_attention_weight_min": -39.13127517700195,
      "activations/layer19_attention_weight_max": 24.923309326171875,
      "activations/layer19_attention_weight_min": -22.443397521972656,
      "activations/layer1_attention_weight_max": 18.30223274230957,
      "activations/layer1_attention_weight_min": -14.805920600891113,
      "activations/layer20_attention_weight_max": 21.97138786315918,
      "activations/layer20_attention_weight_min": -20.974245071411133,
      "activations/layer21_attention_weight_max": 35.31941604614258,
      "activations/layer21_attention_weight_min": -23.21761703491211,
      "activations/layer22_attention_weight_max": 27.85310173034668,
      "activations/layer22_attention_weight_min": -26.28557777404785,
      "activations/layer23_attention_weight_max": 35.978675842285156,
      "activations/layer23_attention_weight_min": -24.851409912109375,
      "activations/layer2_attention_weight_max": 30.475723266601562,
      "activations/layer2_attention_weight_min": -28.44513511657715,
      "activations/layer3_attention_weight_max": 84.16426849365234,
      "activations/layer3_attention_weight_min": -87.49929809570312,
      "activations/layer4_attention_weight_max": 83.15996551513672,
      "activations/layer4_attention_weight_min": -84.01205444335938,
      "activations/layer5_attention_weight_max": 63.155792236328125,
      "activations/layer5_attention_weight_min": -75.26270294189453,
      "activations/layer6_attention_weight_max": 46.58121871948242,
      "activations/layer6_attention_weight_min": -50.149208068847656,
      "activations/layer7_attention_weight_max": 65.0778579711914,
      "activations/layer7_attention_weight_min": -62.05709457397461,
      "activations/layer8_attention_weight_max": 43.842769622802734,
      "activations/layer8_attention_weight_min": -49.61711120605469,
      "activations/layer9_attention_weight_max": 41.682464599609375,
      "activations/layer9_attention_weight_min": -44.974632263183594,
      "epoch": 17.61,
      "learning_rate": 3.681628787878787e-05,
      "loss": 2.7527,
      "step": 303050
    },
    {
      "activations/layer0_attention_weight_max": 15.89625358581543,
      "activations/layer0_attention_weight_min": -13.079253196716309,
      "activations/layer10_attention_weight_max": 35.078521728515625,
      "activations/layer10_attention_weight_min": -34.26713943481445,
      "activations/layer11_attention_weight_max": 30.864368438720703,
      "activations/layer11_attention_weight_min": -32.812129974365234,
      "activations/layer12_attention_weight_max": 19.16144371032715,
      "activations/layer12_attention_weight_min": -22.574234008789062,
      "activations/layer13_attention_weight_max": 34.74364471435547,
      "activations/layer13_attention_weight_min": -28.478113174438477,
      "activations/layer14_attention_weight_max": 38.41472625732422,
      "activations/layer14_attention_weight_min": -29.368181228637695,
      "activations/layer15_attention_weight_max": 35.496986389160156,
      "activations/layer15_attention_weight_min": -28.77269744873047,
      "activations/layer16_attention_weight_max": 36.32704544067383,
      "activations/layer16_attention_weight_min": -32.012489318847656,
      "activations/layer17_attention_weight_max": 53.16831970214844,
      "activations/layer17_attention_weight_min": -43.17107009887695,
      "activations/layer18_attention_weight_max": 47.37785720825195,
      "activations/layer18_attention_weight_min": -39.861610412597656,
      "activations/layer19_attention_weight_max": 22.985612869262695,
      "activations/layer19_attention_weight_min": -22.39754295349121,
      "activations/layer1_attention_weight_max": 16.524822235107422,
      "activations/layer1_attention_weight_min": -13.244808197021484,
      "activations/layer20_attention_weight_max": 22.794301986694336,
      "activations/layer20_attention_weight_min": -20.682767868041992,
      "activations/layer21_attention_weight_max": 38.02406692504883,
      "activations/layer21_attention_weight_min": -23.745153427124023,
      "activations/layer22_attention_weight_max": 30.079742431640625,
      "activations/layer22_attention_weight_min": -26.73664093017578,
      "activations/layer23_attention_weight_max": 39.5330696105957,
      "activations/layer23_attention_weight_min": -25.72955322265625,
      "activations/layer2_attention_weight_max": 31.154193878173828,
      "activations/layer2_attention_weight_min": -30.484996795654297,
      "activations/layer3_attention_weight_max": 89.4898452758789,
      "activations/layer3_attention_weight_min": -91.64190673828125,
      "activations/layer4_attention_weight_max": 90.1653060913086,
      "activations/layer4_attention_weight_min": -89.19829559326172,
      "activations/layer5_attention_weight_max": 65.16259765625,
      "activations/layer5_attention_weight_min": -77.6460189819336,
      "activations/layer6_attention_weight_max": 48.32020568847656,
      "activations/layer6_attention_weight_min": -50.11073684692383,
      "activations/layer7_attention_weight_max": 64.69884490966797,
      "activations/layer7_attention_weight_min": -63.5206413269043,
      "activations/layer8_attention_weight_max": 46.094085693359375,
      "activations/layer8_attention_weight_min": -53.38336181640625,
      "activations/layer9_attention_weight_max": 42.17327117919922,
      "activations/layer9_attention_weight_min": -44.67918014526367,
      "epoch": 17.61,
      "learning_rate": 3.679734848484848e-05,
      "loss": 2.7391,
      "step": 303100
    },
    {
      "activations/layer0_attention_weight_max": 15.412851333618164,
      "activations/layer0_attention_weight_min": -13.454379081726074,
      "activations/layer10_attention_weight_max": 35.4539680480957,
      "activations/layer10_attention_weight_min": -33.617462158203125,
      "activations/layer11_attention_weight_max": 32.23957824707031,
      "activations/layer11_attention_weight_min": -32.89189147949219,
      "activations/layer12_attention_weight_max": 19.312463760375977,
      "activations/layer12_attention_weight_min": -24.02415657043457,
      "activations/layer13_attention_weight_max": 36.86195755004883,
      "activations/layer13_attention_weight_min": -27.000688552856445,
      "activations/layer14_attention_weight_max": 37.461551666259766,
      "activations/layer14_attention_weight_min": -29.839557647705078,
      "activations/layer15_attention_weight_max": 33.924903869628906,
      "activations/layer15_attention_weight_min": -30.471044540405273,
      "activations/layer16_attention_weight_max": 35.75691604614258,
      "activations/layer16_attention_weight_min": -32.23857498168945,
      "activations/layer17_attention_weight_max": 51.604679107666016,
      "activations/layer17_attention_weight_min": -45.4337043762207,
      "activations/layer18_attention_weight_max": 48.88376998901367,
      "activations/layer18_attention_weight_min": -39.37882995605469,
      "activations/layer19_attention_weight_max": 24.46324348449707,
      "activations/layer19_attention_weight_min": -23.738189697265625,
      "activations/layer1_attention_weight_max": 15.429902076721191,
      "activations/layer1_attention_weight_min": -12.87348461151123,
      "activations/layer20_attention_weight_max": 22.16146469116211,
      "activations/layer20_attention_weight_min": -21.78781509399414,
      "activations/layer21_attention_weight_max": 35.1308479309082,
      "activations/layer21_attention_weight_min": -21.703378677368164,
      "activations/layer22_attention_weight_max": 31.478981018066406,
      "activations/layer22_attention_weight_min": -25.53529167175293,
      "activations/layer23_attention_weight_max": 46.038970947265625,
      "activations/layer23_attention_weight_min": -23.26496696472168,
      "activations/layer2_attention_weight_max": 29.837032318115234,
      "activations/layer2_attention_weight_min": -28.532398223876953,
      "activations/layer3_attention_weight_max": 87.50627136230469,
      "activations/layer3_attention_weight_min": -87.44770050048828,
      "activations/layer4_attention_weight_max": 86.49207305908203,
      "activations/layer4_attention_weight_min": -85.43512725830078,
      "activations/layer5_attention_weight_max": 65.20592498779297,
      "activations/layer5_attention_weight_min": -74.01512145996094,
      "activations/layer6_attention_weight_max": 48.03744125366211,
      "activations/layer6_attention_weight_min": -50.35930633544922,
      "activations/layer7_attention_weight_max": 64.77608489990234,
      "activations/layer7_attention_weight_min": -63.38910675048828,
      "activations/layer8_attention_weight_max": 45.88075637817383,
      "activations/layer8_attention_weight_min": -49.23572540283203,
      "activations/layer9_attention_weight_max": 44.140174865722656,
      "activations/layer9_attention_weight_min": -47.85190963745117,
      "epoch": 17.61,
      "learning_rate": 3.677840909090908e-05,
      "loss": 2.7305,
      "step": 303150
    },
    {
      "activations/layer0_attention_weight_max": 15.299442291259766,
      "activations/layer0_attention_weight_min": -13.550585746765137,
      "activations/layer10_attention_weight_max": 36.1768798828125,
      "activations/layer10_attention_weight_min": -35.93156051635742,
      "activations/layer11_attention_weight_max": 32.53367614746094,
      "activations/layer11_attention_weight_min": -34.7414665222168,
      "activations/layer12_attention_weight_max": 19.309326171875,
      "activations/layer12_attention_weight_min": -26.9937686920166,
      "activations/layer13_attention_weight_max": 43.17277526855469,
      "activations/layer13_attention_weight_min": -29.136795043945312,
      "activations/layer14_attention_weight_max": 37.35673904418945,
      "activations/layer14_attention_weight_min": -30.27766990661621,
      "activations/layer15_attention_weight_max": 33.62774658203125,
      "activations/layer15_attention_weight_min": -30.04451560974121,
      "activations/layer16_attention_weight_max": 34.773311614990234,
      "activations/layer16_attention_weight_min": -31.74930763244629,
      "activations/layer17_attention_weight_max": 56.12943649291992,
      "activations/layer17_attention_weight_min": -46.61807632446289,
      "activations/layer18_attention_weight_max": 49.299095153808594,
      "activations/layer18_attention_weight_min": -41.17021560668945,
      "activations/layer19_attention_weight_max": 25.662841796875,
      "activations/layer19_attention_weight_min": -24.29448699951172,
      "activations/layer1_attention_weight_max": 17.146324157714844,
      "activations/layer1_attention_weight_min": -13.64463996887207,
      "activations/layer20_attention_weight_max": 24.696372985839844,
      "activations/layer20_attention_weight_min": -21.244138717651367,
      "activations/layer21_attention_weight_max": 37.56861114501953,
      "activations/layer21_attention_weight_min": -25.294572830200195,
      "activations/layer22_attention_weight_max": 29.17479705810547,
      "activations/layer22_attention_weight_min": -26.85655975341797,
      "activations/layer23_attention_weight_max": 38.397308349609375,
      "activations/layer23_attention_weight_min": -24.570363998413086,
      "activations/layer2_attention_weight_max": 30.946088790893555,
      "activations/layer2_attention_weight_min": -29.397205352783203,
      "activations/layer3_attention_weight_max": 90.4475326538086,
      "activations/layer3_attention_weight_min": -89.7070541381836,
      "activations/layer4_attention_weight_max": 87.1812515258789,
      "activations/layer4_attention_weight_min": -86.55511474609375,
      "activations/layer5_attention_weight_max": 63.688453674316406,
      "activations/layer5_attention_weight_min": -71.42852783203125,
      "activations/layer6_attention_weight_max": 48.104148864746094,
      "activations/layer6_attention_weight_min": -49.78763961791992,
      "activations/layer7_attention_weight_max": 68.60699462890625,
      "activations/layer7_attention_weight_min": -64.6814956665039,
      "activations/layer8_attention_weight_max": 46.47659683227539,
      "activations/layer8_attention_weight_min": -49.33162307739258,
      "activations/layer9_attention_weight_max": 43.8625602722168,
      "activations/layer9_attention_weight_min": -47.46281051635742,
      "epoch": 17.62,
      "learning_rate": 3.675946969696969e-05,
      "loss": 2.7449,
      "step": 303200
    },
    {
      "activations/layer0_attention_weight_max": 15.344381332397461,
      "activations/layer0_attention_weight_min": -13.435338020324707,
      "activations/layer10_attention_weight_max": 35.615718841552734,
      "activations/layer10_attention_weight_min": -33.9101448059082,
      "activations/layer11_attention_weight_max": 33.791168212890625,
      "activations/layer11_attention_weight_min": -33.77519226074219,
      "activations/layer12_attention_weight_max": 19.4815673828125,
      "activations/layer12_attention_weight_min": -28.192655563354492,
      "activations/layer13_attention_weight_max": 36.57088851928711,
      "activations/layer13_attention_weight_min": -29.091415405273438,
      "activations/layer14_attention_weight_max": 36.461097717285156,
      "activations/layer14_attention_weight_min": -31.349761962890625,
      "activations/layer15_attention_weight_max": 34.52890396118164,
      "activations/layer15_attention_weight_min": -30.881603240966797,
      "activations/layer16_attention_weight_max": 34.997283935546875,
      "activations/layer16_attention_weight_min": -33.52656555175781,
      "activations/layer17_attention_weight_max": 48.71969223022461,
      "activations/layer17_attention_weight_min": -44.35923385620117,
      "activations/layer18_attention_weight_max": 49.12550735473633,
      "activations/layer18_attention_weight_min": -41.68418502807617,
      "activations/layer19_attention_weight_max": 23.539045333862305,
      "activations/layer19_attention_weight_min": -25.000139236450195,
      "activations/layer1_attention_weight_max": 16.074792861938477,
      "activations/layer1_attention_weight_min": -13.119329452514648,
      "activations/layer20_attention_weight_max": 22.667316436767578,
      "activations/layer20_attention_weight_min": -21.449684143066406,
      "activations/layer21_attention_weight_max": 37.017608642578125,
      "activations/layer21_attention_weight_min": -23.0385684967041,
      "activations/layer22_attention_weight_max": 31.49435806274414,
      "activations/layer22_attention_weight_min": -25.693031311035156,
      "activations/layer23_attention_weight_max": 37.01796340942383,
      "activations/layer23_attention_weight_min": -24.4814453125,
      "activations/layer2_attention_weight_max": 32.57596206665039,
      "activations/layer2_attention_weight_min": -31.42775535583496,
      "activations/layer3_attention_weight_max": 89.81259155273438,
      "activations/layer3_attention_weight_min": -88.8472900390625,
      "activations/layer4_attention_weight_max": 86.17814636230469,
      "activations/layer4_attention_weight_min": -87.74569702148438,
      "activations/layer5_attention_weight_max": 65.32307434082031,
      "activations/layer5_attention_weight_min": -76.80997467041016,
      "activations/layer6_attention_weight_max": 49.80947494506836,
      "activations/layer6_attention_weight_min": -50.02846145629883,
      "activations/layer7_attention_weight_max": 69.51228332519531,
      "activations/layer7_attention_weight_min": -62.307586669921875,
      "activations/layer8_attention_weight_max": 47.07423400878906,
      "activations/layer8_attention_weight_min": -49.29521179199219,
      "activations/layer9_attention_weight_max": 51.2945671081543,
      "activations/layer9_attention_weight_min": -46.368064880371094,
      "epoch": 17.62,
      "learning_rate": 3.67405303030303e-05,
      "loss": 2.7354,
      "step": 303250
    },
    {
      "activations/layer0_attention_weight_max": 15.353768348693848,
      "activations/layer0_attention_weight_min": -13.46329116821289,
      "activations/layer10_attention_weight_max": 35.46547317504883,
      "activations/layer10_attention_weight_min": -38.220367431640625,
      "activations/layer11_attention_weight_max": 31.597270965576172,
      "activations/layer11_attention_weight_min": -35.395992279052734,
      "activations/layer12_attention_weight_max": 19.39900779724121,
      "activations/layer12_attention_weight_min": -27.6859188079834,
      "activations/layer13_attention_weight_max": 37.795997619628906,
      "activations/layer13_attention_weight_min": -30.44348907470703,
      "activations/layer14_attention_weight_max": 38.89949417114258,
      "activations/layer14_attention_weight_min": -32.44584274291992,
      "activations/layer15_attention_weight_max": 35.30316925048828,
      "activations/layer15_attention_weight_min": -29.61568832397461,
      "activations/layer16_attention_weight_max": 37.52424240112305,
      "activations/layer16_attention_weight_min": -33.83246994018555,
      "activations/layer17_attention_weight_max": 55.138641357421875,
      "activations/layer17_attention_weight_min": -46.28068542480469,
      "activations/layer18_attention_weight_max": 51.20263671875,
      "activations/layer18_attention_weight_min": -39.2855224609375,
      "activations/layer19_attention_weight_max": 24.74031639099121,
      "activations/layer19_attention_weight_min": -22.685598373413086,
      "activations/layer1_attention_weight_max": 15.757516860961914,
      "activations/layer1_attention_weight_min": -13.149352073669434,
      "activations/layer20_attention_weight_max": 27.649307250976562,
      "activations/layer20_attention_weight_min": -22.571653366088867,
      "activations/layer21_attention_weight_max": 38.801639556884766,
      "activations/layer21_attention_weight_min": -21.324840545654297,
      "activations/layer22_attention_weight_max": 32.49664306640625,
      "activations/layer22_attention_weight_min": -24.323986053466797,
      "activations/layer23_attention_weight_max": 44.618751525878906,
      "activations/layer23_attention_weight_min": -25.298904418945312,
      "activations/layer2_attention_weight_max": 30.96723175048828,
      "activations/layer2_attention_weight_min": -29.80499267578125,
      "activations/layer3_attention_weight_max": 87.1009521484375,
      "activations/layer3_attention_weight_min": -91.6439208984375,
      "activations/layer4_attention_weight_max": 86.87578582763672,
      "activations/layer4_attention_weight_min": -85.82572937011719,
      "activations/layer5_attention_weight_max": 67.05644226074219,
      "activations/layer5_attention_weight_min": -77.9935073852539,
      "activations/layer6_attention_weight_max": 47.847896575927734,
      "activations/layer6_attention_weight_min": -49.376651763916016,
      "activations/layer7_attention_weight_max": 62.83351516723633,
      "activations/layer7_attention_weight_min": -62.30959701538086,
      "activations/layer8_attention_weight_max": 47.44685363769531,
      "activations/layer8_attention_weight_min": -49.298316955566406,
      "activations/layer9_attention_weight_max": 44.24673080444336,
      "activations/layer9_attention_weight_min": -45.64934539794922,
      "epoch": 17.62,
      "learning_rate": 3.672159090909091e-05,
      "loss": 2.7442,
      "step": 303300
    },
    {
      "activations/layer0_attention_weight_max": 15.69067668914795,
      "activations/layer0_attention_weight_min": -13.408491134643555,
      "activations/layer10_attention_weight_max": 35.77356719970703,
      "activations/layer10_attention_weight_min": -35.27555465698242,
      "activations/layer11_attention_weight_max": 32.69170379638672,
      "activations/layer11_attention_weight_min": -34.760379791259766,
      "activations/layer12_attention_weight_max": 18.89302635192871,
      "activations/layer12_attention_weight_min": -25.795730590820312,
      "activations/layer13_attention_weight_max": 36.46190643310547,
      "activations/layer13_attention_weight_min": -30.25740623474121,
      "activations/layer14_attention_weight_max": 39.05829620361328,
      "activations/layer14_attention_weight_min": -31.22330093383789,
      "activations/layer15_attention_weight_max": 37.739498138427734,
      "activations/layer15_attention_weight_min": -30.709728240966797,
      "activations/layer16_attention_weight_max": 36.869380950927734,
      "activations/layer16_attention_weight_min": -33.836429595947266,
      "activations/layer17_attention_weight_max": 56.70454788208008,
      "activations/layer17_attention_weight_min": -47.129703521728516,
      "activations/layer18_attention_weight_max": 51.60266876220703,
      "activations/layer18_attention_weight_min": -41.648475646972656,
      "activations/layer19_attention_weight_max": 27.075361251831055,
      "activations/layer19_attention_weight_min": -25.499549865722656,
      "activations/layer1_attention_weight_max": 15.63001537322998,
      "activations/layer1_attention_weight_min": -14.418771743774414,
      "activations/layer20_attention_weight_max": 27.569318771362305,
      "activations/layer20_attention_weight_min": -23.574533462524414,
      "activations/layer21_attention_weight_max": 44.10497283935547,
      "activations/layer21_attention_weight_min": -25.793060302734375,
      "activations/layer22_attention_weight_max": 33.258262634277344,
      "activations/layer22_attention_weight_min": -29.041013717651367,
      "activations/layer23_attention_weight_max": 38.86241912841797,
      "activations/layer23_attention_weight_min": -27.94288444519043,
      "activations/layer2_attention_weight_max": 31.953296661376953,
      "activations/layer2_attention_weight_min": -28.473125457763672,
      "activations/layer3_attention_weight_max": 88.85254669189453,
      "activations/layer3_attention_weight_min": -84.68492889404297,
      "activations/layer4_attention_weight_max": 81.97325134277344,
      "activations/layer4_attention_weight_min": -83.1278305053711,
      "activations/layer5_attention_weight_max": 71.50920867919922,
      "activations/layer5_attention_weight_min": -70.9268569946289,
      "activations/layer6_attention_weight_max": 46.11423873901367,
      "activations/layer6_attention_weight_min": -48.41223907470703,
      "activations/layer7_attention_weight_max": 61.91196060180664,
      "activations/layer7_attention_weight_min": -64.62894439697266,
      "activations/layer8_attention_weight_max": 42.06361770629883,
      "activations/layer8_attention_weight_min": -43.68210983276367,
      "activations/layer9_attention_weight_max": 43.02223587036133,
      "activations/layer9_attention_weight_min": -45.133235931396484,
      "epoch": 17.63,
      "learning_rate": 3.670265151515151e-05,
      "loss": 2.748,
      "step": 303350
    },
    {
      "activations/layer0_attention_weight_max": 15.52091121673584,
      "activations/layer0_attention_weight_min": -13.44739818572998,
      "activations/layer10_attention_weight_max": 34.417022705078125,
      "activations/layer10_attention_weight_min": -33.662933349609375,
      "activations/layer11_attention_weight_max": 31.051803588867188,
      "activations/layer11_attention_weight_min": -33.0148811340332,
      "activations/layer12_attention_weight_max": 17.758848190307617,
      "activations/layer12_attention_weight_min": -26.876319885253906,
      "activations/layer13_attention_weight_max": 31.539234161376953,
      "activations/layer13_attention_weight_min": -29.56695556640625,
      "activations/layer14_attention_weight_max": 35.21335220336914,
      "activations/layer14_attention_weight_min": -29.794858932495117,
      "activations/layer15_attention_weight_max": 32.381614685058594,
      "activations/layer15_attention_weight_min": -29.850624084472656,
      "activations/layer16_attention_weight_max": 33.66466522216797,
      "activations/layer16_attention_weight_min": -33.53165054321289,
      "activations/layer17_attention_weight_max": 52.446434020996094,
      "activations/layer17_attention_weight_min": -49.04560470581055,
      "activations/layer18_attention_weight_max": 45.20916748046875,
      "activations/layer18_attention_weight_min": -43.325111389160156,
      "activations/layer19_attention_weight_max": 24.07248878479004,
      "activations/layer19_attention_weight_min": -23.058019638061523,
      "activations/layer1_attention_weight_max": 15.454225540161133,
      "activations/layer1_attention_weight_min": -14.41143798828125,
      "activations/layer20_attention_weight_max": 21.908018112182617,
      "activations/layer20_attention_weight_min": -21.397319793701172,
      "activations/layer21_attention_weight_max": 35.360870361328125,
      "activations/layer21_attention_weight_min": -22.16633415222168,
      "activations/layer22_attention_weight_max": 29.25360870361328,
      "activations/layer22_attention_weight_min": -26.42793846130371,
      "activations/layer23_attention_weight_max": 36.600223541259766,
      "activations/layer23_attention_weight_min": -25.275672912597656,
      "activations/layer2_attention_weight_max": 31.737503051757812,
      "activations/layer2_attention_weight_min": -30.47614288330078,
      "activations/layer3_attention_weight_max": 89.43826293945312,
      "activations/layer3_attention_weight_min": -90.09870910644531,
      "activations/layer4_attention_weight_max": 85.85643005371094,
      "activations/layer4_attention_weight_min": -83.01565551757812,
      "activations/layer5_attention_weight_max": 62.690269470214844,
      "activations/layer5_attention_weight_min": -71.62669372558594,
      "activations/layer6_attention_weight_max": 46.89087677001953,
      "activations/layer6_attention_weight_min": -49.72096633911133,
      "activations/layer7_attention_weight_max": 63.46369552612305,
      "activations/layer7_attention_weight_min": -61.275760650634766,
      "activations/layer8_attention_weight_max": 43.09174346923828,
      "activations/layer8_attention_weight_min": -45.8140754699707,
      "activations/layer9_attention_weight_max": 41.04731750488281,
      "activations/layer9_attention_weight_min": -44.99266052246094,
      "epoch": 17.63,
      "learning_rate": 3.668371212121212e-05,
      "loss": 2.7251,
      "step": 303400
    },
    {
      "activations/layer0_attention_weight_max": 16.902379989624023,
      "activations/layer0_attention_weight_min": -14.455937385559082,
      "activations/layer10_attention_weight_max": 36.339046478271484,
      "activations/layer10_attention_weight_min": -35.825740814208984,
      "activations/layer11_attention_weight_max": 33.970062255859375,
      "activations/layer11_attention_weight_min": -32.88152313232422,
      "activations/layer12_attention_weight_max": 18.782201766967773,
      "activations/layer12_attention_weight_min": -23.101303100585938,
      "activations/layer13_attention_weight_max": 32.85810089111328,
      "activations/layer13_attention_weight_min": -29.00038719177246,
      "activations/layer14_attention_weight_max": 34.55385208129883,
      "activations/layer14_attention_weight_min": -31.251895904541016,
      "activations/layer15_attention_weight_max": 32.82869338989258,
      "activations/layer15_attention_weight_min": -29.831493377685547,
      "activations/layer16_attention_weight_max": 32.666439056396484,
      "activations/layer16_attention_weight_min": -32.637847900390625,
      "activations/layer17_attention_weight_max": 48.06691360473633,
      "activations/layer17_attention_weight_min": -45.72019958496094,
      "activations/layer18_attention_weight_max": 44.46929931640625,
      "activations/layer18_attention_weight_min": -40.63554000854492,
      "activations/layer19_attention_weight_max": 23.283781051635742,
      "activations/layer19_attention_weight_min": -22.23004150390625,
      "activations/layer1_attention_weight_max": 16.616073608398438,
      "activations/layer1_attention_weight_min": -13.66480827331543,
      "activations/layer20_attention_weight_max": 22.537450790405273,
      "activations/layer20_attention_weight_min": -21.733867645263672,
      "activations/layer21_attention_weight_max": 32.03350830078125,
      "activations/layer21_attention_weight_min": -25.30771827697754,
      "activations/layer22_attention_weight_max": 26.81666374206543,
      "activations/layer22_attention_weight_min": -25.49881362915039,
      "activations/layer23_attention_weight_max": 35.32880783081055,
      "activations/layer23_attention_weight_min": -25.1312255859375,
      "activations/layer2_attention_weight_max": 30.33686065673828,
      "activations/layer2_attention_weight_min": -30.02081871032715,
      "activations/layer3_attention_weight_max": 84.2453384399414,
      "activations/layer3_attention_weight_min": -84.89733123779297,
      "activations/layer4_attention_weight_max": 85.0823745727539,
      "activations/layer4_attention_weight_min": -86.99908447265625,
      "activations/layer5_attention_weight_max": 64.86915588378906,
      "activations/layer5_attention_weight_min": -73.89312744140625,
      "activations/layer6_attention_weight_max": 49.20924758911133,
      "activations/layer6_attention_weight_min": -49.37892532348633,
      "activations/layer7_attention_weight_max": 64.44427490234375,
      "activations/layer7_attention_weight_min": -63.68680191040039,
      "activations/layer8_attention_weight_max": 42.860233306884766,
      "activations/layer8_attention_weight_min": -47.38555145263672,
      "activations/layer9_attention_weight_max": 42.08364486694336,
      "activations/layer9_attention_weight_min": -45.45671463012695,
      "epoch": 17.63,
      "learning_rate": 3.6664772727272726e-05,
      "loss": 2.722,
      "step": 303450
    },
    {
      "activations/layer0_attention_weight_max": 15.62593936920166,
      "activations/layer0_attention_weight_min": -13.003183364868164,
      "activations/layer10_attention_weight_max": 34.17889404296875,
      "activations/layer10_attention_weight_min": -34.80037307739258,
      "activations/layer11_attention_weight_max": 31.256683349609375,
      "activations/layer11_attention_weight_min": -34.158607482910156,
      "activations/layer12_attention_weight_max": 19.707901000976562,
      "activations/layer12_attention_weight_min": -25.20171546936035,
      "activations/layer13_attention_weight_max": 41.48426818847656,
      "activations/layer13_attention_weight_min": -35.60014724731445,
      "activations/layer14_attention_weight_max": 35.08578109741211,
      "activations/layer14_attention_weight_min": -30.887374877929688,
      "activations/layer15_attention_weight_max": 32.132266998291016,
      "activations/layer15_attention_weight_min": -28.23674964904785,
      "activations/layer16_attention_weight_max": 33.397247314453125,
      "activations/layer16_attention_weight_min": -31.157018661499023,
      "activations/layer17_attention_weight_max": 50.42634963989258,
      "activations/layer17_attention_weight_min": -44.09468078613281,
      "activations/layer18_attention_weight_max": 46.02627944946289,
      "activations/layer18_attention_weight_min": -39.674217224121094,
      "activations/layer19_attention_weight_max": 21.078636169433594,
      "activations/layer19_attention_weight_min": -21.856510162353516,
      "activations/layer1_attention_weight_max": 15.310766220092773,
      "activations/layer1_attention_weight_min": -13.909217834472656,
      "activations/layer20_attention_weight_max": 20.25494956970215,
      "activations/layer20_attention_weight_min": -21.462587356567383,
      "activations/layer21_attention_weight_max": 29.21160888671875,
      "activations/layer21_attention_weight_min": -24.04741096496582,
      "activations/layer22_attention_weight_max": 25.813262939453125,
      "activations/layer22_attention_weight_min": -24.601032257080078,
      "activations/layer23_attention_weight_max": 32.632850646972656,
      "activations/layer23_attention_weight_min": -24.02981948852539,
      "activations/layer2_attention_weight_max": 30.97932243347168,
      "activations/layer2_attention_weight_min": -30.335941314697266,
      "activations/layer3_attention_weight_max": 85.14669036865234,
      "activations/layer3_attention_weight_min": -84.78872680664062,
      "activations/layer4_attention_weight_max": 84.50516510009766,
      "activations/layer4_attention_weight_min": -85.45316314697266,
      "activations/layer5_attention_weight_max": 59.92487716674805,
      "activations/layer5_attention_weight_min": -71.07563018798828,
      "activations/layer6_attention_weight_max": 45.77128219604492,
      "activations/layer6_attention_weight_min": -49.876304626464844,
      "activations/layer7_attention_weight_max": 62.766136169433594,
      "activations/layer7_attention_weight_min": -62.26298141479492,
      "activations/layer8_attention_weight_max": 43.73064041137695,
      "activations/layer8_attention_weight_min": -49.59288787841797,
      "activations/layer9_attention_weight_max": 42.20967102050781,
      "activations/layer9_attention_weight_min": -44.9379768371582,
      "epoch": 17.64,
      "learning_rate": 3.664583333333333e-05,
      "loss": 2.757,
      "step": 303500
    },
    {
      "activations/layer0_attention_weight_max": 15.372133255004883,
      "activations/layer0_attention_weight_min": -13.182733535766602,
      "activations/layer10_attention_weight_max": 36.143157958984375,
      "activations/layer10_attention_weight_min": -36.234031677246094,
      "activations/layer11_attention_weight_max": 32.87897491455078,
      "activations/layer11_attention_weight_min": -33.809146881103516,
      "activations/layer12_attention_weight_max": 18.467893600463867,
      "activations/layer12_attention_weight_min": -26.697376251220703,
      "activations/layer13_attention_weight_max": 39.11334991455078,
      "activations/layer13_attention_weight_min": -28.087459564208984,
      "activations/layer14_attention_weight_max": 35.94312286376953,
      "activations/layer14_attention_weight_min": -30.59479331970215,
      "activations/layer15_attention_weight_max": 33.32798767089844,
      "activations/layer15_attention_weight_min": -29.96449089050293,
      "activations/layer16_attention_weight_max": 34.65852737426758,
      "activations/layer16_attention_weight_min": -34.565547943115234,
      "activations/layer17_attention_weight_max": 53.0960578918457,
      "activations/layer17_attention_weight_min": -50.16850662231445,
      "activations/layer18_attention_weight_max": 48.32262420654297,
      "activations/layer18_attention_weight_min": -40.869140625,
      "activations/layer19_attention_weight_max": 25.18775749206543,
      "activations/layer19_attention_weight_min": -23.538494110107422,
      "activations/layer1_attention_weight_max": 16.11861801147461,
      "activations/layer1_attention_weight_min": -14.646015167236328,
      "activations/layer20_attention_weight_max": 24.87175178527832,
      "activations/layer20_attention_weight_min": -20.287864685058594,
      "activations/layer21_attention_weight_max": 45.950355529785156,
      "activations/layer21_attention_weight_min": -25.638198852539062,
      "activations/layer22_attention_weight_max": 31.75578498840332,
      "activations/layer22_attention_weight_min": -26.195377349853516,
      "activations/layer23_attention_weight_max": 40.754642486572266,
      "activations/layer23_attention_weight_min": -28.544681549072266,
      "activations/layer2_attention_weight_max": 31.76508140563965,
      "activations/layer2_attention_weight_min": -33.50988006591797,
      "activations/layer3_attention_weight_max": 91.0976333618164,
      "activations/layer3_attention_weight_min": -94.25050354003906,
      "activations/layer4_attention_weight_max": 87.73414611816406,
      "activations/layer4_attention_weight_min": -87.20738220214844,
      "activations/layer5_attention_weight_max": 66.13131713867188,
      "activations/layer5_attention_weight_min": -72.18803405761719,
      "activations/layer6_attention_weight_max": 51.13759231567383,
      "activations/layer6_attention_weight_min": -54.52366638183594,
      "activations/layer7_attention_weight_max": 64.98341369628906,
      "activations/layer7_attention_weight_min": -62.859676361083984,
      "activations/layer8_attention_weight_max": 47.996334075927734,
      "activations/layer8_attention_weight_min": -50.47597122192383,
      "activations/layer9_attention_weight_max": 43.867183685302734,
      "activations/layer9_attention_weight_min": -45.29038619995117,
      "epoch": 17.64,
      "learning_rate": 3.6626893939393936e-05,
      "loss": 2.7339,
      "step": 303550
    },
    {
      "activations/layer0_attention_weight_max": 15.03220272064209,
      "activations/layer0_attention_weight_min": -13.15234661102295,
      "activations/layer10_attention_weight_max": 33.42274475097656,
      "activations/layer10_attention_weight_min": -34.34502410888672,
      "activations/layer11_attention_weight_max": 32.81321716308594,
      "activations/layer11_attention_weight_min": -34.045833587646484,
      "activations/layer12_attention_weight_max": 18.72720718383789,
      "activations/layer12_attention_weight_min": -22.676071166992188,
      "activations/layer13_attention_weight_max": 32.295570373535156,
      "activations/layer13_attention_weight_min": -28.556941986083984,
      "activations/layer14_attention_weight_max": 33.80827713012695,
      "activations/layer14_attention_weight_min": -30.695392608642578,
      "activations/layer15_attention_weight_max": 32.77825927734375,
      "activations/layer15_attention_weight_min": -28.908063888549805,
      "activations/layer16_attention_weight_max": 34.58424377441406,
      "activations/layer16_attention_weight_min": -32.09939193725586,
      "activations/layer17_attention_weight_max": 50.45565414428711,
      "activations/layer17_attention_weight_min": -45.145084381103516,
      "activations/layer18_attention_weight_max": 44.63605499267578,
      "activations/layer18_attention_weight_min": -39.6002311706543,
      "activations/layer19_attention_weight_max": 24.114423751831055,
      "activations/layer19_attention_weight_min": -22.367300033569336,
      "activations/layer1_attention_weight_max": 15.786721229553223,
      "activations/layer1_attention_weight_min": -15.123440742492676,
      "activations/layer20_attention_weight_max": 21.979589462280273,
      "activations/layer20_attention_weight_min": -21.992816925048828,
      "activations/layer21_attention_weight_max": 37.82649612426758,
      "activations/layer21_attention_weight_min": -23.68243980407715,
      "activations/layer22_attention_weight_max": 27.67422866821289,
      "activations/layer22_attention_weight_min": -24.916032791137695,
      "activations/layer23_attention_weight_max": 37.05893325805664,
      "activations/layer23_attention_weight_min": -25.18204116821289,
      "activations/layer2_attention_weight_max": 32.55015563964844,
      "activations/layer2_attention_weight_min": -29.45806121826172,
      "activations/layer3_attention_weight_max": 94.74872589111328,
      "activations/layer3_attention_weight_min": -90.31611633300781,
      "activations/layer4_attention_weight_max": 95.57305145263672,
      "activations/layer4_attention_weight_min": -89.06038665771484,
      "activations/layer5_attention_weight_max": 67.50945281982422,
      "activations/layer5_attention_weight_min": -74.40137481689453,
      "activations/layer6_attention_weight_max": 52.91352462768555,
      "activations/layer6_attention_weight_min": -50.223594665527344,
      "activations/layer7_attention_weight_max": 64.71992492675781,
      "activations/layer7_attention_weight_min": -65.05357360839844,
      "activations/layer8_attention_weight_max": 47.03643798828125,
      "activations/layer8_attention_weight_min": -49.039459228515625,
      "activations/layer9_attention_weight_max": 42.37974166870117,
      "activations/layer9_attention_weight_min": -47.41194534301758,
      "epoch": 17.64,
      "learning_rate": 3.6607954545454544e-05,
      "loss": 2.7391,
      "step": 303600
    },
    {
      "activations/layer0_attention_weight_max": 15.494693756103516,
      "activations/layer0_attention_weight_min": -13.031116485595703,
      "activations/layer10_attention_weight_max": 37.21080780029297,
      "activations/layer10_attention_weight_min": -35.77996063232422,
      "activations/layer11_attention_weight_max": 30.95501708984375,
      "activations/layer11_attention_weight_min": -33.923011779785156,
      "activations/layer12_attention_weight_max": 19.156497955322266,
      "activations/layer12_attention_weight_min": -26.454605102539062,
      "activations/layer13_attention_weight_max": 37.32166290283203,
      "activations/layer13_attention_weight_min": -30.50444793701172,
      "activations/layer14_attention_weight_max": 35.02016067504883,
      "activations/layer14_attention_weight_min": -32.36287307739258,
      "activations/layer15_attention_weight_max": 31.249996185302734,
      "activations/layer15_attention_weight_min": -30.20366668701172,
      "activations/layer16_attention_weight_max": 32.11613464355469,
      "activations/layer16_attention_weight_min": -31.901994705200195,
      "activations/layer17_attention_weight_max": 50.50432205200195,
      "activations/layer17_attention_weight_min": -45.785667419433594,
      "activations/layer18_attention_weight_max": 45.04938888549805,
      "activations/layer18_attention_weight_min": -39.55003356933594,
      "activations/layer19_attention_weight_max": 22.675870895385742,
      "activations/layer19_attention_weight_min": -20.93556785583496,
      "activations/layer1_attention_weight_max": 16.45562744140625,
      "activations/layer1_attention_weight_min": -14.327085494995117,
      "activations/layer20_attention_weight_max": 21.72612953186035,
      "activations/layer20_attention_weight_min": -26.617835998535156,
      "activations/layer21_attention_weight_max": 34.619232177734375,
      "activations/layer21_attention_weight_min": -23.399255752563477,
      "activations/layer22_attention_weight_max": 28.735132217407227,
      "activations/layer22_attention_weight_min": -25.68427276611328,
      "activations/layer23_attention_weight_max": 36.553192138671875,
      "activations/layer23_attention_weight_min": -24.268054962158203,
      "activations/layer2_attention_weight_max": 30.182903289794922,
      "activations/layer2_attention_weight_min": -30.365415573120117,
      "activations/layer3_attention_weight_max": 89.44158172607422,
      "activations/layer3_attention_weight_min": -89.34983825683594,
      "activations/layer4_attention_weight_max": 88.1118392944336,
      "activations/layer4_attention_weight_min": -86.87840270996094,
      "activations/layer5_attention_weight_max": 63.30449676513672,
      "activations/layer5_attention_weight_min": -75.39510345458984,
      "activations/layer6_attention_weight_max": 48.93411636352539,
      "activations/layer6_attention_weight_min": -51.764137268066406,
      "activations/layer7_attention_weight_max": 63.884246826171875,
      "activations/layer7_attention_weight_min": -63.242183685302734,
      "activations/layer8_attention_weight_max": 44.70802688598633,
      "activations/layer8_attention_weight_min": -48.159461975097656,
      "activations/layer9_attention_weight_max": 41.46572494506836,
      "activations/layer9_attention_weight_min": -44.44443893432617,
      "epoch": 17.64,
      "learning_rate": 3.658901515151515e-05,
      "loss": 2.7328,
      "step": 303650
    },
    {
      "activations/layer0_attention_weight_max": 15.473176956176758,
      "activations/layer0_attention_weight_min": -12.983159065246582,
      "activations/layer10_attention_weight_max": 35.79140090942383,
      "activations/layer10_attention_weight_min": -35.85102844238281,
      "activations/layer11_attention_weight_max": 33.701873779296875,
      "activations/layer11_attention_weight_min": -35.316436767578125,
      "activations/layer12_attention_weight_max": 19.102882385253906,
      "activations/layer12_attention_weight_min": -25.99945831298828,
      "activations/layer13_attention_weight_max": 31.550487518310547,
      "activations/layer13_attention_weight_min": -28.7260799407959,
      "activations/layer14_attention_weight_max": 34.10213851928711,
      "activations/layer14_attention_weight_min": -29.180484771728516,
      "activations/layer15_attention_weight_max": 31.43223762512207,
      "activations/layer15_attention_weight_min": -30.66484260559082,
      "activations/layer16_attention_weight_max": 32.67056655883789,
      "activations/layer16_attention_weight_min": -31.089635848999023,
      "activations/layer17_attention_weight_max": 51.0346565246582,
      "activations/layer17_attention_weight_min": -42.1602668762207,
      "activations/layer18_attention_weight_max": 46.420658111572266,
      "activations/layer18_attention_weight_min": -39.626625061035156,
      "activations/layer19_attention_weight_max": 23.600553512573242,
      "activations/layer19_attention_weight_min": -24.498289108276367,
      "activations/layer1_attention_weight_max": 15.795655250549316,
      "activations/layer1_attention_weight_min": -14.25329303741455,
      "activations/layer20_attention_weight_max": 23.8282413482666,
      "activations/layer20_attention_weight_min": -22.241262435913086,
      "activations/layer21_attention_weight_max": 33.8984375,
      "activations/layer21_attention_weight_min": -24.248693466186523,
      "activations/layer22_attention_weight_max": 29.60640525817871,
      "activations/layer22_attention_weight_min": -25.251232147216797,
      "activations/layer23_attention_weight_max": 36.323333740234375,
      "activations/layer23_attention_weight_min": -26.46035385131836,
      "activations/layer2_attention_weight_max": 31.43963623046875,
      "activations/layer2_attention_weight_min": -29.983823776245117,
      "activations/layer3_attention_weight_max": 85.4459457397461,
      "activations/layer3_attention_weight_min": -92.93299102783203,
      "activations/layer4_attention_weight_max": 84.81211853027344,
      "activations/layer4_attention_weight_min": -84.9006118774414,
      "activations/layer5_attention_weight_max": 65.55902862548828,
      "activations/layer5_attention_weight_min": -73.969970703125,
      "activations/layer6_attention_weight_max": 47.017642974853516,
      "activations/layer6_attention_weight_min": -50.10991287231445,
      "activations/layer7_attention_weight_max": 64.3547592163086,
      "activations/layer7_attention_weight_min": -63.631622314453125,
      "activations/layer8_attention_weight_max": 43.842803955078125,
      "activations/layer8_attention_weight_min": -48.6351318359375,
      "activations/layer9_attention_weight_max": 40.87043380737305,
      "activations/layer9_attention_weight_min": -43.58065414428711,
      "epoch": 17.65,
      "learning_rate": 3.6570075757575754e-05,
      "loss": 2.7581,
      "step": 303700
    },
    {
      "activations/layer0_attention_weight_max": 15.653043746948242,
      "activations/layer0_attention_weight_min": -13.185042381286621,
      "activations/layer10_attention_weight_max": 36.823646545410156,
      "activations/layer10_attention_weight_min": -35.85908889770508,
      "activations/layer11_attention_weight_max": 33.390071868896484,
      "activations/layer11_attention_weight_min": -33.99350357055664,
      "activations/layer12_attention_weight_max": 19.673141479492188,
      "activations/layer12_attention_weight_min": -25.80389976501465,
      "activations/layer13_attention_weight_max": 36.83583450317383,
      "activations/layer13_attention_weight_min": -27.680917739868164,
      "activations/layer14_attention_weight_max": 38.84785461425781,
      "activations/layer14_attention_weight_min": -33.8302001953125,
      "activations/layer15_attention_weight_max": 37.43307876586914,
      "activations/layer15_attention_weight_min": -30.599336624145508,
      "activations/layer16_attention_weight_max": 39.13777542114258,
      "activations/layer16_attention_weight_min": -35.839481353759766,
      "activations/layer17_attention_weight_max": 58.00657272338867,
      "activations/layer17_attention_weight_min": -50.73021697998047,
      "activations/layer18_attention_weight_max": 54.0893669128418,
      "activations/layer18_attention_weight_min": -42.86473846435547,
      "activations/layer19_attention_weight_max": 27.522546768188477,
      "activations/layer19_attention_weight_min": -24.683679580688477,
      "activations/layer1_attention_weight_max": 15.474644660949707,
      "activations/layer1_attention_weight_min": -14.265767097473145,
      "activations/layer20_attention_weight_max": 25.167551040649414,
      "activations/layer20_attention_weight_min": -23.56309700012207,
      "activations/layer21_attention_weight_max": 40.652549743652344,
      "activations/layer21_attention_weight_min": -22.959369659423828,
      "activations/layer22_attention_weight_max": 30.740856170654297,
      "activations/layer22_attention_weight_min": -25.256393432617188,
      "activations/layer23_attention_weight_max": 35.67271423339844,
      "activations/layer23_attention_weight_min": -23.799179077148438,
      "activations/layer2_attention_weight_max": 32.65113830566406,
      "activations/layer2_attention_weight_min": -32.56432342529297,
      "activations/layer3_attention_weight_max": 92.33061981201172,
      "activations/layer3_attention_weight_min": -97.66838073730469,
      "activations/layer4_attention_weight_max": 91.98934173583984,
      "activations/layer4_attention_weight_min": -85.7283706665039,
      "activations/layer5_attention_weight_max": 66.7229995727539,
      "activations/layer5_attention_weight_min": -72.8604736328125,
      "activations/layer6_attention_weight_max": 48.3431510925293,
      "activations/layer6_attention_weight_min": -50.329322814941406,
      "activations/layer7_attention_weight_max": 64.77970886230469,
      "activations/layer7_attention_weight_min": -64.3468246459961,
      "activations/layer8_attention_weight_max": 43.80086135864258,
      "activations/layer8_attention_weight_min": -49.71864700317383,
      "activations/layer9_attention_weight_max": 44.0888557434082,
      "activations/layer9_attention_weight_min": -44.604793548583984,
      "epoch": 17.65,
      "learning_rate": 3.655151515151515e-05,
      "loss": 2.7468,
      "step": 303750
    },
    {
      "activations/layer0_attention_weight_max": 15.52931022644043,
      "activations/layer0_attention_weight_min": -12.83298397064209,
      "activations/layer10_attention_weight_max": 37.42066192626953,
      "activations/layer10_attention_weight_min": -35.975337982177734,
      "activations/layer11_attention_weight_max": 32.18787384033203,
      "activations/layer11_attention_weight_min": -33.76420211791992,
      "activations/layer12_attention_weight_max": 19.084352493286133,
      "activations/layer12_attention_weight_min": -24.643083572387695,
      "activations/layer13_attention_weight_max": 31.79448890686035,
      "activations/layer13_attention_weight_min": -26.82498550415039,
      "activations/layer14_attention_weight_max": 35.07428741455078,
      "activations/layer14_attention_weight_min": -30.120624542236328,
      "activations/layer15_attention_weight_max": 32.16445541381836,
      "activations/layer15_attention_weight_min": -30.259275436401367,
      "activations/layer16_attention_weight_max": 31.714309692382812,
      "activations/layer16_attention_weight_min": -31.488866806030273,
      "activations/layer17_attention_weight_max": 48.46098709106445,
      "activations/layer17_attention_weight_min": -42.898746490478516,
      "activations/layer18_attention_weight_max": 44.72061538696289,
      "activations/layer18_attention_weight_min": -39.694149017333984,
      "activations/layer19_attention_weight_max": 21.830480575561523,
      "activations/layer19_attention_weight_min": -23.82148551940918,
      "activations/layer1_attention_weight_max": 15.471086502075195,
      "activations/layer1_attention_weight_min": -14.013679504394531,
      "activations/layer20_attention_weight_max": 20.358171463012695,
      "activations/layer20_attention_weight_min": -21.585153579711914,
      "activations/layer21_attention_weight_max": 32.84455108642578,
      "activations/layer21_attention_weight_min": -23.616947174072266,
      "activations/layer22_attention_weight_max": 28.455354690551758,
      "activations/layer22_attention_weight_min": -26.162025451660156,
      "activations/layer23_attention_weight_max": 36.59276580810547,
      "activations/layer23_attention_weight_min": -25.420303344726562,
      "activations/layer2_attention_weight_max": 31.278249740600586,
      "activations/layer2_attention_weight_min": -28.871749877929688,
      "activations/layer3_attention_weight_max": 86.1051254272461,
      "activations/layer3_attention_weight_min": -89.58736419677734,
      "activations/layer4_attention_weight_max": 85.1375961303711,
      "activations/layer4_attention_weight_min": -86.36394500732422,
      "activations/layer5_attention_weight_max": 61.81980514526367,
      "activations/layer5_attention_weight_min": -77.3409423828125,
      "activations/layer6_attention_weight_max": 50.142601013183594,
      "activations/layer6_attention_weight_min": -49.892513275146484,
      "activations/layer7_attention_weight_max": 64.9796371459961,
      "activations/layer7_attention_weight_min": -62.60641860961914,
      "activations/layer8_attention_weight_max": 45.09574890136719,
      "activations/layer8_attention_weight_min": -49.85500717163086,
      "activations/layer9_attention_weight_max": 44.98310852050781,
      "activations/layer9_attention_weight_min": -45.86991500854492,
      "epoch": 17.65,
      "learning_rate": 3.653257575757575e-05,
      "loss": 2.7316,
      "step": 303800
    },
    {
      "activations/layer0_attention_weight_max": 14.736295700073242,
      "activations/layer0_attention_weight_min": -13.085973739624023,
      "activations/layer10_attention_weight_max": 34.8272590637207,
      "activations/layer10_attention_weight_min": -34.39491271972656,
      "activations/layer11_attention_weight_max": 32.69786834716797,
      "activations/layer11_attention_weight_min": -33.38209915161133,
      "activations/layer12_attention_weight_max": 20.133014678955078,
      "activations/layer12_attention_weight_min": -25.77593231201172,
      "activations/layer13_attention_weight_max": 32.59513473510742,
      "activations/layer13_attention_weight_min": -27.790637969970703,
      "activations/layer14_attention_weight_max": 35.440147399902344,
      "activations/layer14_attention_weight_min": -32.10289764404297,
      "activations/layer15_attention_weight_max": 33.86992645263672,
      "activations/layer15_attention_weight_min": -32.372684478759766,
      "activations/layer16_attention_weight_max": 35.1329231262207,
      "activations/layer16_attention_weight_min": -32.35056686401367,
      "activations/layer17_attention_weight_max": 49.63991928100586,
      "activations/layer17_attention_weight_min": -45.38642120361328,
      "activations/layer18_attention_weight_max": 46.00160217285156,
      "activations/layer18_attention_weight_min": -37.40772247314453,
      "activations/layer19_attention_weight_max": 26.385557174682617,
      "activations/layer19_attention_weight_min": -23.56456184387207,
      "activations/layer1_attention_weight_max": 16.319740295410156,
      "activations/layer1_attention_weight_min": -13.733111381530762,
      "activations/layer20_attention_weight_max": 22.813461303710938,
      "activations/layer20_attention_weight_min": -22.256338119506836,
      "activations/layer21_attention_weight_max": 35.05994415283203,
      "activations/layer21_attention_weight_min": -25.942337036132812,
      "activations/layer22_attention_weight_max": 29.51309585571289,
      "activations/layer22_attention_weight_min": -28.191932678222656,
      "activations/layer23_attention_weight_max": 37.70151138305664,
      "activations/layer23_attention_weight_min": -24.545881271362305,
      "activations/layer2_attention_weight_max": 30.428749084472656,
      "activations/layer2_attention_weight_min": -31.761558532714844,
      "activations/layer3_attention_weight_max": 87.02903747558594,
      "activations/layer3_attention_weight_min": -96.84827423095703,
      "activations/layer4_attention_weight_max": 83.21168518066406,
      "activations/layer4_attention_weight_min": -84.93080139160156,
      "activations/layer5_attention_weight_max": 61.64293670654297,
      "activations/layer5_attention_weight_min": -72.40258026123047,
      "activations/layer6_attention_weight_max": 45.748817443847656,
      "activations/layer6_attention_weight_min": -47.10274887084961,
      "activations/layer7_attention_weight_max": 63.75621032714844,
      "activations/layer7_attention_weight_min": -60.80799865722656,
      "activations/layer8_attention_weight_max": 43.90251541137695,
      "activations/layer8_attention_weight_min": -47.52761459350586,
      "activations/layer9_attention_weight_max": 44.275882720947266,
      "activations/layer9_attention_weight_min": -43.72123336791992,
      "epoch": 17.66,
      "learning_rate": 3.651363636363636e-05,
      "loss": 2.7332,
      "step": 303850
    },
    {
      "activations/layer0_attention_weight_max": 15.759617805480957,
      "activations/layer0_attention_weight_min": -13.765554428100586,
      "activations/layer10_attention_weight_max": 33.22110366821289,
      "activations/layer10_attention_weight_min": -33.45919418334961,
      "activations/layer11_attention_weight_max": 30.89417839050293,
      "activations/layer11_attention_weight_min": -32.931297302246094,
      "activations/layer12_attention_weight_max": 18.83341407775879,
      "activations/layer12_attention_weight_min": -26.11972999572754,
      "activations/layer13_attention_weight_max": 33.649192810058594,
      "activations/layer13_attention_weight_min": -26.728361129760742,
      "activations/layer14_attention_weight_max": 34.55107879638672,
      "activations/layer14_attention_weight_min": -29.476959228515625,
      "activations/layer15_attention_weight_max": 33.755889892578125,
      "activations/layer15_attention_weight_min": -30.99770736694336,
      "activations/layer16_attention_weight_max": 32.46253204345703,
      "activations/layer16_attention_weight_min": -31.500043869018555,
      "activations/layer17_attention_weight_max": 51.1008186340332,
      "activations/layer17_attention_weight_min": -45.75401306152344,
      "activations/layer18_attention_weight_max": 44.971004486083984,
      "activations/layer18_attention_weight_min": -40.18773651123047,
      "activations/layer19_attention_weight_max": 25.746835708618164,
      "activations/layer19_attention_weight_min": -23.61690330505371,
      "activations/layer1_attention_weight_max": 16.357881546020508,
      "activations/layer1_attention_weight_min": -15.070289611816406,
      "activations/layer20_attention_weight_max": 26.635847091674805,
      "activations/layer20_attention_weight_min": -26.38900375366211,
      "activations/layer21_attention_weight_max": 38.24329376220703,
      "activations/layer21_attention_weight_min": -29.2734432220459,
      "activations/layer22_attention_weight_max": 27.539188385009766,
      "activations/layer22_attention_weight_min": -26.304443359375,
      "activations/layer23_attention_weight_max": 38.53873062133789,
      "activations/layer23_attention_weight_min": -26.292251586914062,
      "activations/layer2_attention_weight_max": 30.553630828857422,
      "activations/layer2_attention_weight_min": -31.777509689331055,
      "activations/layer3_attention_weight_max": 85.78862762451172,
      "activations/layer3_attention_weight_min": -92.65615844726562,
      "activations/layer4_attention_weight_max": 84.8797378540039,
      "activations/layer4_attention_weight_min": -87.26375579833984,
      "activations/layer5_attention_weight_max": 60.38772201538086,
      "activations/layer5_attention_weight_min": -72.88585662841797,
      "activations/layer6_attention_weight_max": 47.397544860839844,
      "activations/layer6_attention_weight_min": -49.35025405883789,
      "activations/layer7_attention_weight_max": 62.587982177734375,
      "activations/layer7_attention_weight_min": -61.956573486328125,
      "activations/layer8_attention_weight_max": 43.45035934448242,
      "activations/layer8_attention_weight_min": -47.740257263183594,
      "activations/layer9_attention_weight_max": 41.680233001708984,
      "activations/layer9_attention_weight_min": -43.69179153442383,
      "epoch": 17.66,
      "learning_rate": 3.649469696969696e-05,
      "loss": 2.7392,
      "step": 303900
    },
    {
      "activations/layer0_attention_weight_max": 16.080167770385742,
      "activations/layer0_attention_weight_min": -13.106426239013672,
      "activations/layer10_attention_weight_max": 33.66786575317383,
      "activations/layer10_attention_weight_min": -34.4796028137207,
      "activations/layer11_attention_weight_max": 31.56989097595215,
      "activations/layer11_attention_weight_min": -34.007572174072266,
      "activations/layer12_attention_weight_max": 21.116397857666016,
      "activations/layer12_attention_weight_min": -23.6430606842041,
      "activations/layer13_attention_weight_max": 36.713134765625,
      "activations/layer13_attention_weight_min": -28.16779136657715,
      "activations/layer14_attention_weight_max": 36.1497688293457,
      "activations/layer14_attention_weight_min": -31.19622230529785,
      "activations/layer15_attention_weight_max": 34.406898498535156,
      "activations/layer15_attention_weight_min": -32.345184326171875,
      "activations/layer16_attention_weight_max": 34.35776901245117,
      "activations/layer16_attention_weight_min": -31.910375595092773,
      "activations/layer17_attention_weight_max": 55.49803924560547,
      "activations/layer17_attention_weight_min": -47.3071403503418,
      "activations/layer18_attention_weight_max": 51.603023529052734,
      "activations/layer18_attention_weight_min": -42.89279556274414,
      "activations/layer19_attention_weight_max": 25.88864517211914,
      "activations/layer19_attention_weight_min": -24.562108993530273,
      "activations/layer1_attention_weight_max": 16.780241012573242,
      "activations/layer1_attention_weight_min": -14.804990768432617,
      "activations/layer20_attention_weight_max": 25.131114959716797,
      "activations/layer20_attention_weight_min": -23.966466903686523,
      "activations/layer21_attention_weight_max": 41.0009880065918,
      "activations/layer21_attention_weight_min": -27.309307098388672,
      "activations/layer22_attention_weight_max": 34.380401611328125,
      "activations/layer22_attention_weight_min": -27.8287410736084,
      "activations/layer23_attention_weight_max": 40.926658630371094,
      "activations/layer23_attention_weight_min": -26.318159103393555,
      "activations/layer2_attention_weight_max": 31.615875244140625,
      "activations/layer2_attention_weight_min": -28.88888168334961,
      "activations/layer3_attention_weight_max": 85.12714385986328,
      "activations/layer3_attention_weight_min": -87.3187255859375,
      "activations/layer4_attention_weight_max": 87.41132354736328,
      "activations/layer4_attention_weight_min": -88.50414276123047,
      "activations/layer5_attention_weight_max": 65.27165985107422,
      "activations/layer5_attention_weight_min": -71.96027374267578,
      "activations/layer6_attention_weight_max": 48.33361053466797,
      "activations/layer6_attention_weight_min": -48.870235443115234,
      "activations/layer7_attention_weight_max": 60.974334716796875,
      "activations/layer7_attention_weight_min": -61.724727630615234,
      "activations/layer8_attention_weight_max": 45.38209533691406,
      "activations/layer8_attention_weight_min": -51.74579620361328,
      "activations/layer9_attention_weight_max": 42.21964645385742,
      "activations/layer9_attention_weight_min": -44.75075912475586,
      "epoch": 17.66,
      "learning_rate": 3.647575757575757e-05,
      "loss": 2.7608,
      "step": 303950
    },
    {
      "activations/layer0_attention_weight_max": 15.938916206359863,
      "activations/layer0_attention_weight_min": -12.662100791931152,
      "activations/layer10_attention_weight_max": 36.26816177368164,
      "activations/layer10_attention_weight_min": -34.867645263671875,
      "activations/layer11_attention_weight_max": 31.214378356933594,
      "activations/layer11_attention_weight_min": -33.43646240234375,
      "activations/layer12_attention_weight_max": 18.13222312927246,
      "activations/layer12_attention_weight_min": -25.83401107788086,
      "activations/layer13_attention_weight_max": 36.83934783935547,
      "activations/layer13_attention_weight_min": -29.15532875061035,
      "activations/layer14_attention_weight_max": 34.5598258972168,
      "activations/layer14_attention_weight_min": -29.955669403076172,
      "activations/layer15_attention_weight_max": 32.263328552246094,
      "activations/layer15_attention_weight_min": -30.771730422973633,
      "activations/layer16_attention_weight_max": 32.664031982421875,
      "activations/layer16_attention_weight_min": -29.989391326904297,
      "activations/layer17_attention_weight_max": 53.7967643737793,
      "activations/layer17_attention_weight_min": -44.10903549194336,
      "activations/layer18_attention_weight_max": 47.44552230834961,
      "activations/layer18_attention_weight_min": -38.620052337646484,
      "activations/layer19_attention_weight_max": 26.335805892944336,
      "activations/layer19_attention_weight_min": -20.880277633666992,
      "activations/layer1_attention_weight_max": 15.19865894317627,
      "activations/layer1_attention_weight_min": -13.113136291503906,
      "activations/layer20_attention_weight_max": 22.81513023376465,
      "activations/layer20_attention_weight_min": -20.292036056518555,
      "activations/layer21_attention_weight_max": 35.361846923828125,
      "activations/layer21_attention_weight_min": -20.22450828552246,
      "activations/layer22_attention_weight_max": 29.238014221191406,
      "activations/layer22_attention_weight_min": -26.121917724609375,
      "activations/layer23_attention_weight_max": 38.75580596923828,
      "activations/layer23_attention_weight_min": -23.236425399780273,
      "activations/layer2_attention_weight_max": 29.575223922729492,
      "activations/layer2_attention_weight_min": -30.12490463256836,
      "activations/layer3_attention_weight_max": 89.65619659423828,
      "activations/layer3_attention_weight_min": -94.46314239501953,
      "activations/layer4_attention_weight_max": 86.00972747802734,
      "activations/layer4_attention_weight_min": -86.94540405273438,
      "activations/layer5_attention_weight_max": 62.16302490234375,
      "activations/layer5_attention_weight_min": -73.76058959960938,
      "activations/layer6_attention_weight_max": 46.48780059814453,
      "activations/layer6_attention_weight_min": -49.23215866088867,
      "activations/layer7_attention_weight_max": 62.7951545715332,
      "activations/layer7_attention_weight_min": -60.67572021484375,
      "activations/layer8_attention_weight_max": 44.07972717285156,
      "activations/layer8_attention_weight_min": -49.49186325073242,
      "activations/layer9_attention_weight_max": 42.175472259521484,
      "activations/layer9_attention_weight_min": -44.696632385253906,
      "epoch": 17.66,
      "learning_rate": 3.645681818181818e-05,
      "loss": 2.7428,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_loss": 2.6953125,
      "eval_runtime": 8.4752,
      "eval_samples_per_second": 506.652,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_openwebtext_loss": 2.6953125,
      "eval_openwebtext_ppl": 14.81014620089916,
      "eval_openwebtext_runtime": 8.4752,
      "eval_openwebtext_samples_per_second": 506.652,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_wikitext_loss": 2.9296875,
      "eval_wikitext_ppl": 18.721779026482544,
      "eval_wikitext_runtime": 1.9467,
      "eval_wikitext_samples_per_second": 234.242,
      "step": 304000
    },
    {
      "epoch": 17.66,
      "eval_lambada_loss": 2.53515625,
      "eval_lambada_ppl": 12.618402319374093,
      "eval_lambada_runtime": 9.5129,
      "eval_lambada_samples_per_second": 511.831,
      "step": 304000
    },
    {
      "activations/layer0_attention_weight_max": 16.93528175354004,
      "activations/layer0_attention_weight_min": -13.439940452575684,
      "activations/layer10_attention_weight_max": 38.270355224609375,
      "activations/layer10_attention_weight_min": -37.95402526855469,
      "activations/layer11_attention_weight_max": 33.59809112548828,
      "activations/layer11_attention_weight_min": -37.49101638793945,
      "activations/layer12_attention_weight_max": 20.157058715820312,
      "activations/layer12_attention_weight_min": -25.376325607299805,
      "activations/layer13_attention_weight_max": 36.354347229003906,
      "activations/layer13_attention_weight_min": -26.184642791748047,
      "activations/layer14_attention_weight_max": 36.3674201965332,
      "activations/layer14_attention_weight_min": -32.6581916809082,
      "activations/layer15_attention_weight_max": 40.06018829345703,
      "activations/layer15_attention_weight_min": -31.745220184326172,
      "activations/layer16_attention_weight_max": 33.34787368774414,
      "activations/layer16_attention_weight_min": -32.45365524291992,
      "activations/layer17_attention_weight_max": 50.529903411865234,
      "activations/layer17_attention_weight_min": -43.701717376708984,
      "activations/layer18_attention_weight_max": 48.23762130737305,
      "activations/layer18_attention_weight_min": -40.91252899169922,
      "activations/layer19_attention_weight_max": 24.574262619018555,
      "activations/layer19_attention_weight_min": -22.510509490966797,
      "activations/layer1_attention_weight_max": 16.234394073486328,
      "activations/layer1_attention_weight_min": -14.729028701782227,
      "activations/layer20_attention_weight_max": 23.543874740600586,
      "activations/layer20_attention_weight_min": -22.962749481201172,
      "activations/layer21_attention_weight_max": 36.24110794067383,
      "activations/layer21_attention_weight_min": -21.616636276245117,
      "activations/layer22_attention_weight_max": 29.242416381835938,
      "activations/layer22_attention_weight_min": -26.242042541503906,
      "activations/layer23_attention_weight_max": 41.17623519897461,
      "activations/layer23_attention_weight_min": -22.398300170898438,
      "activations/layer2_attention_weight_max": 34.42639923095703,
      "activations/layer2_attention_weight_min": -30.909626007080078,
      "activations/layer3_attention_weight_max": 90.03977966308594,
      "activations/layer3_attention_weight_min": -91.55064392089844,
      "activations/layer4_attention_weight_max": 89.74266815185547,
      "activations/layer4_attention_weight_min": -88.93788146972656,
      "activations/layer5_attention_weight_max": 64.59630584716797,
      "activations/layer5_attention_weight_min": -74.0648193359375,
      "activations/layer6_attention_weight_max": 48.114505767822266,
      "activations/layer6_attention_weight_min": -51.99738693237305,
      "activations/layer7_attention_weight_max": 65.57903289794922,
      "activations/layer7_attention_weight_min": -68.08503723144531,
      "activations/layer8_attention_weight_max": 48.13571548461914,
      "activations/layer8_attention_weight_min": -53.114131927490234,
      "activations/layer9_attention_weight_max": 48.475154876708984,
      "activations/layer9_attention_weight_min": -46.48781204223633,
      "epoch": 17.67,
      "learning_rate": 3.643787878787879e-05,
      "loss": 2.7289,
      "step": 304050
    },
    {
      "activations/layer0_attention_weight_max": 15.446123123168945,
      "activations/layer0_attention_weight_min": -12.961963653564453,
      "activations/layer10_attention_weight_max": 37.71900177001953,
      "activations/layer10_attention_weight_min": -37.56424331665039,
      "activations/layer11_attention_weight_max": 33.07823944091797,
      "activations/layer11_attention_weight_min": -35.61754608154297,
      "activations/layer12_attention_weight_max": 19.797096252441406,
      "activations/layer12_attention_weight_min": -27.006916046142578,
      "activations/layer13_attention_weight_max": 33.964813232421875,
      "activations/layer13_attention_weight_min": -29.014236450195312,
      "activations/layer14_attention_weight_max": 36.499961853027344,
      "activations/layer14_attention_weight_min": -34.31670379638672,
      "activations/layer15_attention_weight_max": 32.46167755126953,
      "activations/layer15_attention_weight_min": -33.5560302734375,
      "activations/layer16_attention_weight_max": 30.64613914489746,
      "activations/layer16_attention_weight_min": -30.381179809570312,
      "activations/layer17_attention_weight_max": 49.074363708496094,
      "activations/layer17_attention_weight_min": -43.01528549194336,
      "activations/layer18_attention_weight_max": 46.506507873535156,
      "activations/layer18_attention_weight_min": -38.612064361572266,
      "activations/layer19_attention_weight_max": 23.520845413208008,
      "activations/layer19_attention_weight_min": -22.065547943115234,
      "activations/layer1_attention_weight_max": 16.317794799804688,
      "activations/layer1_attention_weight_min": -14.50362777709961,
      "activations/layer20_attention_weight_max": 23.676408767700195,
      "activations/layer20_attention_weight_min": -20.594200134277344,
      "activations/layer21_attention_weight_max": 33.742835998535156,
      "activations/layer21_attention_weight_min": -23.31065559387207,
      "activations/layer22_attention_weight_max": 28.626150131225586,
      "activations/layer22_attention_weight_min": -25.954662322998047,
      "activations/layer23_attention_weight_max": 36.19747543334961,
      "activations/layer23_attention_weight_min": -24.348730087280273,
      "activations/layer2_attention_weight_max": 32.4454460144043,
      "activations/layer2_attention_weight_min": -31.37152099609375,
      "activations/layer3_attention_weight_max": 97.38484954833984,
      "activations/layer3_attention_weight_min": -97.43087768554688,
      "activations/layer4_attention_weight_max": 93.05779266357422,
      "activations/layer4_attention_weight_min": -91.84759521484375,
      "activations/layer5_attention_weight_max": 65.02375793457031,
      "activations/layer5_attention_weight_min": -71.4140853881836,
      "activations/layer6_attention_weight_max": 50.21693420410156,
      "activations/layer6_attention_weight_min": -53.154964447021484,
      "activations/layer7_attention_weight_max": 63.991214752197266,
      "activations/layer7_attention_weight_min": -63.75047302246094,
      "activations/layer8_attention_weight_max": 49.006893157958984,
      "activations/layer8_attention_weight_min": -51.59160614013672,
      "activations/layer9_attention_weight_max": 45.87268829345703,
      "activations/layer9_attention_weight_min": -48.40858840942383,
      "epoch": 17.67,
      "learning_rate": 3.641893939393939e-05,
      "loss": 2.7373,
      "step": 304100
    },
    {
      "activations/layer0_attention_weight_max": 15.880302429199219,
      "activations/layer0_attention_weight_min": -12.891962051391602,
      "activations/layer10_attention_weight_max": 36.87387466430664,
      "activations/layer10_attention_weight_min": -37.466922760009766,
      "activations/layer11_attention_weight_max": 34.4278564453125,
      "activations/layer11_attention_weight_min": -37.548213958740234,
      "activations/layer12_attention_weight_max": 20.099292755126953,
      "activations/layer12_attention_weight_min": -25.429813385009766,
      "activations/layer13_attention_weight_max": 35.110687255859375,
      "activations/layer13_attention_weight_min": -28.831514358520508,
      "activations/layer14_attention_weight_max": 40.32179641723633,
      "activations/layer14_attention_weight_min": -33.11044692993164,
      "activations/layer15_attention_weight_max": 35.908939361572266,
      "activations/layer15_attention_weight_min": -32.573116302490234,
      "activations/layer16_attention_weight_max": 37.545074462890625,
      "activations/layer16_attention_weight_min": -32.713623046875,
      "activations/layer17_attention_weight_max": 59.10227584838867,
      "activations/layer17_attention_weight_min": -48.3414306640625,
      "activations/layer18_attention_weight_max": 51.87973403930664,
      "activations/layer18_attention_weight_min": -41.7065315246582,
      "activations/layer19_attention_weight_max": 26.117910385131836,
      "activations/layer19_attention_weight_min": -23.521358489990234,
      "activations/layer1_attention_weight_max": 16.477542877197266,
      "activations/layer1_attention_weight_min": -14.25265884399414,
      "activations/layer20_attention_weight_max": 23.829233169555664,
      "activations/layer20_attention_weight_min": -21.84149932861328,
      "activations/layer21_attention_weight_max": 40.75309753417969,
      "activations/layer21_attention_weight_min": -23.63797378540039,
      "activations/layer22_attention_weight_max": 31.68503761291504,
      "activations/layer22_attention_weight_min": -25.95871353149414,
      "activations/layer23_attention_weight_max": 37.67484664916992,
      "activations/layer23_attention_weight_min": -24.68006134033203,
      "activations/layer2_attention_weight_max": 28.240554809570312,
      "activations/layer2_attention_weight_min": -28.025121688842773,
      "activations/layer3_attention_weight_max": 81.82146453857422,
      "activations/layer3_attention_weight_min": -83.78440856933594,
      "activations/layer4_attention_weight_max": 83.85420989990234,
      "activations/layer4_attention_weight_min": -86.44171905517578,
      "activations/layer5_attention_weight_max": 59.499000549316406,
      "activations/layer5_attention_weight_min": -74.53950500488281,
      "activations/layer6_attention_weight_max": 51.043067932128906,
      "activations/layer6_attention_weight_min": -56.23502731323242,
      "activations/layer7_attention_weight_max": 61.916595458984375,
      "activations/layer7_attention_weight_min": -64.28328704833984,
      "activations/layer8_attention_weight_max": 48.97127151489258,
      "activations/layer8_attention_weight_min": -52.40879440307617,
      "activations/layer9_attention_weight_max": 46.22077178955078,
      "activations/layer9_attention_weight_min": -47.866172790527344,
      "epoch": 17.67,
      "learning_rate": 3.64e-05,
      "loss": 2.7621,
      "step": 304150
    },
    {
      "activations/layer0_attention_weight_max": 15.633484840393066,
      "activations/layer0_attention_weight_min": -13.337858200073242,
      "activations/layer10_attention_weight_max": 39.655738830566406,
      "activations/layer10_attention_weight_min": -36.61890411376953,
      "activations/layer11_attention_weight_max": 34.81904602050781,
      "activations/layer11_attention_weight_min": -34.584747314453125,
      "activations/layer12_attention_weight_max": 20.05494499206543,
      "activations/layer12_attention_weight_min": -23.76957893371582,
      "activations/layer13_attention_weight_max": 35.01106262207031,
      "activations/layer13_attention_weight_min": -28.89493751525879,
      "activations/layer14_attention_weight_max": 36.22395706176758,
      "activations/layer14_attention_weight_min": -31.566099166870117,
      "activations/layer15_attention_weight_max": 34.013790130615234,
      "activations/layer15_attention_weight_min": -31.1942138671875,
      "activations/layer16_attention_weight_max": 34.04097366333008,
      "activations/layer16_attention_weight_min": -32.2336540222168,
      "activations/layer17_attention_weight_max": 53.089935302734375,
      "activations/layer17_attention_weight_min": -44.876434326171875,
      "activations/layer18_attention_weight_max": 48.26785659790039,
      "activations/layer18_attention_weight_min": -40.837650299072266,
      "activations/layer19_attention_weight_max": 25.449203491210938,
      "activations/layer19_attention_weight_min": -21.878210067749023,
      "activations/layer1_attention_weight_max": 15.695414543151855,
      "activations/layer1_attention_weight_min": -13.022749900817871,
      "activations/layer20_attention_weight_max": 23.381168365478516,
      "activations/layer20_attention_weight_min": -21.573246002197266,
      "activations/layer21_attention_weight_max": 37.832427978515625,
      "activations/layer21_attention_weight_min": -25.076932907104492,
      "activations/layer22_attention_weight_max": 30.242578506469727,
      "activations/layer22_attention_weight_min": -26.82959747314453,
      "activations/layer23_attention_weight_max": 35.0859375,
      "activations/layer23_attention_weight_min": -24.686681747436523,
      "activations/layer2_attention_weight_max": 30.421947479248047,
      "activations/layer2_attention_weight_min": -29.138690948486328,
      "activations/layer3_attention_weight_max": 91.0195083618164,
      "activations/layer3_attention_weight_min": -92.82392120361328,
      "activations/layer4_attention_weight_max": 91.4324722290039,
      "activations/layer4_attention_weight_min": -86.45235443115234,
      "activations/layer5_attention_weight_max": 66.33533477783203,
      "activations/layer5_attention_weight_min": -77.59068298339844,
      "activations/layer6_attention_weight_max": 53.68217468261719,
      "activations/layer6_attention_weight_min": -51.17816925048828,
      "activations/layer7_attention_weight_max": 64.0798110961914,
      "activations/layer7_attention_weight_min": -64.86272430419922,
      "activations/layer8_attention_weight_max": 47.75239562988281,
      "activations/layer8_attention_weight_min": -51.25068283081055,
      "activations/layer9_attention_weight_max": 45.90607833862305,
      "activations/layer9_attention_weight_min": -46.291358947753906,
      "epoch": 17.68,
      "learning_rate": 3.63810606060606e-05,
      "loss": 2.7478,
      "step": 304200
    },
    {
      "activations/layer0_attention_weight_max": 16.935083389282227,
      "activations/layer0_attention_weight_min": -13.118041038513184,
      "activations/layer10_attention_weight_max": 34.92531204223633,
      "activations/layer10_attention_weight_min": -34.7411994934082,
      "activations/layer11_attention_weight_max": 33.743682861328125,
      "activations/layer11_attention_weight_min": -35.71738052368164,
      "activations/layer12_attention_weight_max": 18.722196578979492,
      "activations/layer12_attention_weight_min": -23.98193359375,
      "activations/layer13_attention_weight_max": 33.490394592285156,
      "activations/layer13_attention_weight_min": -29.795589447021484,
      "activations/layer14_attention_weight_max": 34.856285095214844,
      "activations/layer14_attention_weight_min": -30.531021118164062,
      "activations/layer15_attention_weight_max": 31.092437744140625,
      "activations/layer15_attention_weight_min": -30.15349006652832,
      "activations/layer16_attention_weight_max": 35.09093475341797,
      "activations/layer16_attention_weight_min": -33.59424591064453,
      "activations/layer17_attention_weight_max": 51.44401550292969,
      "activations/layer17_attention_weight_min": -46.8990592956543,
      "activations/layer18_attention_weight_max": 46.45460510253906,
      "activations/layer18_attention_weight_min": -40.26500701904297,
      "activations/layer19_attention_weight_max": 23.76738739013672,
      "activations/layer19_attention_weight_min": -23.21979331970215,
      "activations/layer1_attention_weight_max": 15.50688648223877,
      "activations/layer1_attention_weight_min": -13.741541862487793,
      "activations/layer20_attention_weight_max": 22.53942108154297,
      "activations/layer20_attention_weight_min": -22.041784286499023,
      "activations/layer21_attention_weight_max": 34.537166595458984,
      "activations/layer21_attention_weight_min": -21.703319549560547,
      "activations/layer22_attention_weight_max": 31.02303695678711,
      "activations/layer22_attention_weight_min": -27.846832275390625,
      "activations/layer23_attention_weight_max": 34.164146423339844,
      "activations/layer23_attention_weight_min": -25.5050048828125,
      "activations/layer2_attention_weight_max": 30.49791717529297,
      "activations/layer2_attention_weight_min": -29.69487762451172,
      "activations/layer3_attention_weight_max": 92.12261962890625,
      "activations/layer3_attention_weight_min": -90.90122985839844,
      "activations/layer4_attention_weight_max": 88.17791748046875,
      "activations/layer4_attention_weight_min": -86.5656509399414,
      "activations/layer5_attention_weight_max": 62.62137985229492,
      "activations/layer5_attention_weight_min": -72.21637725830078,
      "activations/layer6_attention_weight_max": 47.970115661621094,
      "activations/layer6_attention_weight_min": -48.952884674072266,
      "activations/layer7_attention_weight_max": 61.24311828613281,
      "activations/layer7_attention_weight_min": -66.19580841064453,
      "activations/layer8_attention_weight_max": 45.119476318359375,
      "activations/layer8_attention_weight_min": -48.157684326171875,
      "activations/layer9_attention_weight_max": 44.35858917236328,
      "activations/layer9_attention_weight_min": -46.06013870239258,
      "epoch": 17.68,
      "learning_rate": 3.636212121212121e-05,
      "loss": 2.7435,
      "step": 304250
    },
    {
      "activations/layer0_attention_weight_max": 15.619458198547363,
      "activations/layer0_attention_weight_min": -13.412188529968262,
      "activations/layer10_attention_weight_max": 34.04045486450195,
      "activations/layer10_attention_weight_min": -34.120548248291016,
      "activations/layer11_attention_weight_max": 30.629070281982422,
      "activations/layer11_attention_weight_min": -32.40047836303711,
      "activations/layer12_attention_weight_max": 18.57744789123535,
      "activations/layer12_attention_weight_min": -27.97216033935547,
      "activations/layer13_attention_weight_max": 31.946800231933594,
      "activations/layer13_attention_weight_min": -26.64011001586914,
      "activations/layer14_attention_weight_max": 33.14702224731445,
      "activations/layer14_attention_weight_min": -29.00559425354004,
      "activations/layer15_attention_weight_max": 32.7630729675293,
      "activations/layer15_attention_weight_min": -28.397563934326172,
      "activations/layer16_attention_weight_max": 33.478515625,
      "activations/layer16_attention_weight_min": -31.942764282226562,
      "activations/layer17_attention_weight_max": 48.96782302856445,
      "activations/layer17_attention_weight_min": -42.98938751220703,
      "activations/layer18_attention_weight_max": 43.83507537841797,
      "activations/layer18_attention_weight_min": -37.473907470703125,
      "activations/layer19_attention_weight_max": 25.871875762939453,
      "activations/layer19_attention_weight_min": -22.882184982299805,
      "activations/layer1_attention_weight_max": 15.985820770263672,
      "activations/layer1_attention_weight_min": -12.112749099731445,
      "activations/layer20_attention_weight_max": 21.27824592590332,
      "activations/layer20_attention_weight_min": -20.571287155151367,
      "activations/layer21_attention_weight_max": 36.96626663208008,
      "activations/layer21_attention_weight_min": -23.402074813842773,
      "activations/layer22_attention_weight_max": 29.38985252380371,
      "activations/layer22_attention_weight_min": -26.359758377075195,
      "activations/layer23_attention_weight_max": 36.72864532470703,
      "activations/layer23_attention_weight_min": -23.51423454284668,
      "activations/layer2_attention_weight_max": 29.414175033569336,
      "activations/layer2_attention_weight_min": -29.497982025146484,
      "activations/layer3_attention_weight_max": 84.88294219970703,
      "activations/layer3_attention_weight_min": -89.41190338134766,
      "activations/layer4_attention_weight_max": 83.27957153320312,
      "activations/layer4_attention_weight_min": -86.03314208984375,
      "activations/layer5_attention_weight_max": 61.01934814453125,
      "activations/layer5_attention_weight_min": -78.51768493652344,
      "activations/layer6_attention_weight_max": 48.250022888183594,
      "activations/layer6_attention_weight_min": -49.721824645996094,
      "activations/layer7_attention_weight_max": 58.96940231323242,
      "activations/layer7_attention_weight_min": -62.26929473876953,
      "activations/layer8_attention_weight_max": 43.99979782104492,
      "activations/layer8_attention_weight_min": -48.892879486083984,
      "activations/layer9_attention_weight_max": 40.500518798828125,
      "activations/layer9_attention_weight_min": -43.99298858642578,
      "epoch": 17.68,
      "learning_rate": 3.63435606060606e-05,
      "loss": 2.73,
      "step": 304300
    },
    {
      "activations/layer0_attention_weight_max": 15.189986228942871,
      "activations/layer0_attention_weight_min": -13.21240234375,
      "activations/layer10_attention_weight_max": 40.52519226074219,
      "activations/layer10_attention_weight_min": -39.96785354614258,
      "activations/layer11_attention_weight_max": 38.48846435546875,
      "activations/layer11_attention_weight_min": -35.783905029296875,
      "activations/layer12_attention_weight_max": 19.753665924072266,
      "activations/layer12_attention_weight_min": -25.92192840576172,
      "activations/layer13_attention_weight_max": 41.01237869262695,
      "activations/layer13_attention_weight_min": -32.14830017089844,
      "activations/layer14_attention_weight_max": 35.79706573486328,
      "activations/layer14_attention_weight_min": -30.342683792114258,
      "activations/layer15_attention_weight_max": 34.54384994506836,
      "activations/layer15_attention_weight_min": -30.301170349121094,
      "activations/layer16_attention_weight_max": 35.24061584472656,
      "activations/layer16_attention_weight_min": -32.44192123413086,
      "activations/layer17_attention_weight_max": 50.6041145324707,
      "activations/layer17_attention_weight_min": -47.472023010253906,
      "activations/layer18_attention_weight_max": 46.23039245605469,
      "activations/layer18_attention_weight_min": -43.680789947509766,
      "activations/layer19_attention_weight_max": 25.646854400634766,
      "activations/layer19_attention_weight_min": -24.211652755737305,
      "activations/layer1_attention_weight_max": 16.048982620239258,
      "activations/layer1_attention_weight_min": -12.965339660644531,
      "activations/layer20_attention_weight_max": 23.744604110717773,
      "activations/layer20_attention_weight_min": -23.849023818969727,
      "activations/layer21_attention_weight_max": 36.383026123046875,
      "activations/layer21_attention_weight_min": -24.6640682220459,
      "activations/layer22_attention_weight_max": 31.027437210083008,
      "activations/layer22_attention_weight_min": -26.993900299072266,
      "activations/layer23_attention_weight_max": 39.38523864746094,
      "activations/layer23_attention_weight_min": -26.767040252685547,
      "activations/layer2_attention_weight_max": 31.676776885986328,
      "activations/layer2_attention_weight_min": -30.176992416381836,
      "activations/layer3_attention_weight_max": 95.22563934326172,
      "activations/layer3_attention_weight_min": -97.10858917236328,
      "activations/layer4_attention_weight_max": 89.24898529052734,
      "activations/layer4_attention_weight_min": -93.70742797851562,
      "activations/layer5_attention_weight_max": 65.6808853149414,
      "activations/layer5_attention_weight_min": -76.71455383300781,
      "activations/layer6_attention_weight_max": 50.522335052490234,
      "activations/layer6_attention_weight_min": -52.67573928833008,
      "activations/layer7_attention_weight_max": 69.95756530761719,
      "activations/layer7_attention_weight_min": -65.84799194335938,
      "activations/layer8_attention_weight_max": 54.61819839477539,
      "activations/layer8_attention_weight_min": -51.241004943847656,
      "activations/layer9_attention_weight_max": 51.72822952270508,
      "activations/layer9_attention_weight_min": -50.37497329711914,
      "epoch": 17.68,
      "learning_rate": 3.6324621212121206e-05,
      "loss": 2.7239,
      "step": 304350
    },
    {
      "activations/layer0_attention_weight_max": 15.431526184082031,
      "activations/layer0_attention_weight_min": -13.39256763458252,
      "activations/layer10_attention_weight_max": 36.60169219970703,
      "activations/layer10_attention_weight_min": -34.510704040527344,
      "activations/layer11_attention_weight_max": 31.225460052490234,
      "activations/layer11_attention_weight_min": -32.93353271484375,
      "activations/layer12_attention_weight_max": 19.73723030090332,
      "activations/layer12_attention_weight_min": -27.586793899536133,
      "activations/layer13_attention_weight_max": 35.27476501464844,
      "activations/layer13_attention_weight_min": -29.93082618713379,
      "activations/layer14_attention_weight_max": 34.616119384765625,
      "activations/layer14_attention_weight_min": -30.01056480407715,
      "activations/layer15_attention_weight_max": 32.73220443725586,
      "activations/layer15_attention_weight_min": -29.914409637451172,
      "activations/layer16_attention_weight_max": 32.63047790527344,
      "activations/layer16_attention_weight_min": -31.360368728637695,
      "activations/layer17_attention_weight_max": 50.04249954223633,
      "activations/layer17_attention_weight_min": -44.57069778442383,
      "activations/layer18_attention_weight_max": 47.343963623046875,
      "activations/layer18_attention_weight_min": -40.114681243896484,
      "activations/layer19_attention_weight_max": 24.322526931762695,
      "activations/layer19_attention_weight_min": -22.775028228759766,
      "activations/layer1_attention_weight_max": 16.476863861083984,
      "activations/layer1_attention_weight_min": -14.414325714111328,
      "activations/layer20_attention_weight_max": 22.02528953552246,
      "activations/layer20_attention_weight_min": -22.61937141418457,
      "activations/layer21_attention_weight_max": 40.23189163208008,
      "activations/layer21_attention_weight_min": -23.910905838012695,
      "activations/layer22_attention_weight_max": 29.914730072021484,
      "activations/layer22_attention_weight_min": -26.444551467895508,
      "activations/layer23_attention_weight_max": 33.550968170166016,
      "activations/layer23_attention_weight_min": -22.730566024780273,
      "activations/layer2_attention_weight_max": 30.47188949584961,
      "activations/layer2_attention_weight_min": -31.25801658630371,
      "activations/layer3_attention_weight_max": 92.34306335449219,
      "activations/layer3_attention_weight_min": -94.57842254638672,
      "activations/layer4_attention_weight_max": 87.52229309082031,
      "activations/layer4_attention_weight_min": -85.46846771240234,
      "activations/layer5_attention_weight_max": 66.88839721679688,
      "activations/layer5_attention_weight_min": -72.42417907714844,
      "activations/layer6_attention_weight_max": 48.267940521240234,
      "activations/layer6_attention_weight_min": -50.96086883544922,
      "activations/layer7_attention_weight_max": 68.05511474609375,
      "activations/layer7_attention_weight_min": -62.904842376708984,
      "activations/layer8_attention_weight_max": 44.78309631347656,
      "activations/layer8_attention_weight_min": -47.57007598876953,
      "activations/layer9_attention_weight_max": 47.163448333740234,
      "activations/layer9_attention_weight_min": -46.47107696533203,
      "epoch": 17.69,
      "learning_rate": 3.6305681818181814e-05,
      "loss": 2.7187,
      "step": 304400
    },
    {
      "activations/layer0_attention_weight_max": 16.070709228515625,
      "activations/layer0_attention_weight_min": -13.142988204956055,
      "activations/layer10_attention_weight_max": 38.037086486816406,
      "activations/layer10_attention_weight_min": -36.90006637573242,
      "activations/layer11_attention_weight_max": 33.12249755859375,
      "activations/layer11_attention_weight_min": -33.97550582885742,
      "activations/layer12_attention_weight_max": 19.379758834838867,
      "activations/layer12_attention_weight_min": -25.80057144165039,
      "activations/layer13_attention_weight_max": 34.73086929321289,
      "activations/layer13_attention_weight_min": -28.5906982421875,
      "activations/layer14_attention_weight_max": 37.02771759033203,
      "activations/layer14_attention_weight_min": -28.774507522583008,
      "activations/layer15_attention_weight_max": 34.19732666015625,
      "activations/layer15_attention_weight_min": -29.840242385864258,
      "activations/layer16_attention_weight_max": 36.652740478515625,
      "activations/layer16_attention_weight_min": -31.635419845581055,
      "activations/layer17_attention_weight_max": 52.88508987426758,
      "activations/layer17_attention_weight_min": -45.6588020324707,
      "activations/layer18_attention_weight_max": 52.912227630615234,
      "activations/layer18_attention_weight_min": -38.5924186706543,
      "activations/layer19_attention_weight_max": 25.055288314819336,
      "activations/layer19_attention_weight_min": -21.859506607055664,
      "activations/layer1_attention_weight_max": 17.66929054260254,
      "activations/layer1_attention_weight_min": -13.254215240478516,
      "activations/layer20_attention_weight_max": 24.093656539916992,
      "activations/layer20_attention_weight_min": -21.29898452758789,
      "activations/layer21_attention_weight_max": 41.382537841796875,
      "activations/layer21_attention_weight_min": -22.18867301940918,
      "activations/layer22_attention_weight_max": 31.476037979125977,
      "activations/layer22_attention_weight_min": -24.420217514038086,
      "activations/layer23_attention_weight_max": 39.32626724243164,
      "activations/layer23_attention_weight_min": -24.85407829284668,
      "activations/layer2_attention_weight_max": 29.351383209228516,
      "activations/layer2_attention_weight_min": -29.574604034423828,
      "activations/layer3_attention_weight_max": 89.9396743774414,
      "activations/layer3_attention_weight_min": -94.2311019897461,
      "activations/layer4_attention_weight_max": 89.49647521972656,
      "activations/layer4_attention_weight_min": -85.60802459716797,
      "activations/layer5_attention_weight_max": 67.15350341796875,
      "activations/layer5_attention_weight_min": -78.66267395019531,
      "activations/layer6_attention_weight_max": 51.830299377441406,
      "activations/layer6_attention_weight_min": -50.13872146606445,
      "activations/layer7_attention_weight_max": 69.20958709716797,
      "activations/layer7_attention_weight_min": -62.75667190551758,
      "activations/layer8_attention_weight_max": 48.684444427490234,
      "activations/layer8_attention_weight_min": -48.33729553222656,
      "activations/layer9_attention_weight_max": 52.00958251953125,
      "activations/layer9_attention_weight_min": -46.230220794677734,
      "epoch": 17.69,
      "learning_rate": 3.628674242424242e-05,
      "loss": 2.7357,
      "step": 304450
    },
    {
      "activations/layer0_attention_weight_max": 16.3050594329834,
      "activations/layer0_attention_weight_min": -13.122662544250488,
      "activations/layer10_attention_weight_max": 32.54052734375,
      "activations/layer10_attention_weight_min": -33.31215286254883,
      "activations/layer11_attention_weight_max": 30.693225860595703,
      "activations/layer11_attention_weight_min": -32.75313186645508,
      "activations/layer12_attention_weight_max": 18.357240676879883,
      "activations/layer12_attention_weight_min": -25.361902236938477,
      "activations/layer13_attention_weight_max": 32.17212677001953,
      "activations/layer13_attention_weight_min": -25.029232025146484,
      "activations/layer14_attention_weight_max": 33.25249481201172,
      "activations/layer14_attention_weight_min": -28.312732696533203,
      "activations/layer15_attention_weight_max": 31.066370010375977,
      "activations/layer15_attention_weight_min": -30.277462005615234,
      "activations/layer16_attention_weight_max": 31.493579864501953,
      "activations/layer16_attention_weight_min": -32.59380340576172,
      "activations/layer17_attention_weight_max": 45.31218719482422,
      "activations/layer17_attention_weight_min": -43.02524185180664,
      "activations/layer18_attention_weight_max": 43.826351165771484,
      "activations/layer18_attention_weight_min": -37.567745208740234,
      "activations/layer19_attention_weight_max": 25.137176513671875,
      "activations/layer19_attention_weight_min": -23.003751754760742,
      "activations/layer1_attention_weight_max": 15.425442695617676,
      "activations/layer1_attention_weight_min": -12.902629852294922,
      "activations/layer20_attention_weight_max": 24.10956573486328,
      "activations/layer20_attention_weight_min": -22.190710067749023,
      "activations/layer21_attention_weight_max": 32.60523986816406,
      "activations/layer21_attention_weight_min": -22.034452438354492,
      "activations/layer22_attention_weight_max": 27.426334381103516,
      "activations/layer22_attention_weight_min": -25.993432998657227,
      "activations/layer23_attention_weight_max": 36.06902313232422,
      "activations/layer23_attention_weight_min": -24.789714813232422,
      "activations/layer2_attention_weight_max": 30.103130340576172,
      "activations/layer2_attention_weight_min": -28.678754806518555,
      "activations/layer3_attention_weight_max": 86.51728820800781,
      "activations/layer3_attention_weight_min": -87.93174743652344,
      "activations/layer4_attention_weight_max": 84.7602767944336,
      "activations/layer4_attention_weight_min": -85.64837646484375,
      "activations/layer5_attention_weight_max": 61.60457992553711,
      "activations/layer5_attention_weight_min": -72.93955993652344,
      "activations/layer6_attention_weight_max": 47.81359100341797,
      "activations/layer6_attention_weight_min": -48.07236862182617,
      "activations/layer7_attention_weight_max": 64.22391510009766,
      "activations/layer7_attention_weight_min": -58.24159622192383,
      "activations/layer8_attention_weight_max": 43.77718734741211,
      "activations/layer8_attention_weight_min": -45.36086654663086,
      "activations/layer9_attention_weight_max": 44.84554672241211,
      "activations/layer9_attention_weight_min": -42.483985900878906,
      "epoch": 17.69,
      "learning_rate": 3.626780303030303e-05,
      "loss": 2.7224,
      "step": 304500
    },
    {
      "activations/layer0_attention_weight_max": 15.427163124084473,
      "activations/layer0_attention_weight_min": -13.13471508026123,
      "activations/layer10_attention_weight_max": 34.73869323730469,
      "activations/layer10_attention_weight_min": -34.33667755126953,
      "activations/layer11_attention_weight_max": 30.847227096557617,
      "activations/layer11_attention_weight_min": -34.228736877441406,
      "activations/layer12_attention_weight_max": 21.052412033081055,
      "activations/layer12_attention_weight_min": -30.07795524597168,
      "activations/layer13_attention_weight_max": 36.1239013671875,
      "activations/layer13_attention_weight_min": -28.851219177246094,
      "activations/layer14_attention_weight_max": 38.11247634887695,
      "activations/layer14_attention_weight_min": -31.53496551513672,
      "activations/layer15_attention_weight_max": 35.90116500854492,
      "activations/layer15_attention_weight_min": -31.46539878845215,
      "activations/layer16_attention_weight_max": 35.110504150390625,
      "activations/layer16_attention_weight_min": -32.39167785644531,
      "activations/layer17_attention_weight_max": 53.00075149536133,
      "activations/layer17_attention_weight_min": -47.4405403137207,
      "activations/layer18_attention_weight_max": 50.293331146240234,
      "activations/layer18_attention_weight_min": -41.06502151489258,
      "activations/layer19_attention_weight_max": 27.052953720092773,
      "activations/layer19_attention_weight_min": -24.49684715270996,
      "activations/layer1_attention_weight_max": 16.699039459228516,
      "activations/layer1_attention_weight_min": -15.109877586364746,
      "activations/layer20_attention_weight_max": 25.570280075073242,
      "activations/layer20_attention_weight_min": -24.785409927368164,
      "activations/layer21_attention_weight_max": 42.798824310302734,
      "activations/layer21_attention_weight_min": -23.8510684967041,
      "activations/layer22_attention_weight_max": 35.75012969970703,
      "activations/layer22_attention_weight_min": -27.629976272583008,
      "activations/layer23_attention_weight_max": 38.812740325927734,
      "activations/layer23_attention_weight_min": -27.202848434448242,
      "activations/layer2_attention_weight_max": 31.883827209472656,
      "activations/layer2_attention_weight_min": -28.80727195739746,
      "activations/layer3_attention_weight_max": 90.57176971435547,
      "activations/layer3_attention_weight_min": -90.25934600830078,
      "activations/layer4_attention_weight_max": 88.10294342041016,
      "activations/layer4_attention_weight_min": -87.69242858886719,
      "activations/layer5_attention_weight_max": 65.6853256225586,
      "activations/layer5_attention_weight_min": -71.71708679199219,
      "activations/layer6_attention_weight_max": 49.71635055541992,
      "activations/layer6_attention_weight_min": -51.32924270629883,
      "activations/layer7_attention_weight_max": 65.44803619384766,
      "activations/layer7_attention_weight_min": -63.02851104736328,
      "activations/layer8_attention_weight_max": 44.76158905029297,
      "activations/layer8_attention_weight_min": -48.433712005615234,
      "activations/layer9_attention_weight_max": 42.88010787963867,
      "activations/layer9_attention_weight_min": -43.645301818847656,
      "epoch": 17.7,
      "learning_rate": 3.624886363636363e-05,
      "loss": 2.7421,
      "step": 304550
    },
    {
      "activations/layer0_attention_weight_max": 15.106939315795898,
      "activations/layer0_attention_weight_min": -13.232778549194336,
      "activations/layer10_attention_weight_max": 35.833778381347656,
      "activations/layer10_attention_weight_min": -36.04414367675781,
      "activations/layer11_attention_weight_max": 32.80061340332031,
      "activations/layer11_attention_weight_min": -35.341373443603516,
      "activations/layer12_attention_weight_max": 18.943300247192383,
      "activations/layer12_attention_weight_min": -25.84025001525879,
      "activations/layer13_attention_weight_max": 32.06026840209961,
      "activations/layer13_attention_weight_min": -30.34832191467285,
      "activations/layer14_attention_weight_max": 35.14390182495117,
      "activations/layer14_attention_weight_min": -29.01204490661621,
      "activations/layer15_attention_weight_max": 32.60286331176758,
      "activations/layer15_attention_weight_min": -30.78363609313965,
      "activations/layer16_attention_weight_max": 32.16434097290039,
      "activations/layer16_attention_weight_min": -30.192127227783203,
      "activations/layer17_attention_weight_max": 47.808837890625,
      "activations/layer17_attention_weight_min": -42.14695358276367,
      "activations/layer18_attention_weight_max": 44.888729095458984,
      "activations/layer18_attention_weight_min": -38.21793746948242,
      "activations/layer19_attention_weight_max": 22.003406524658203,
      "activations/layer19_attention_weight_min": -22.902549743652344,
      "activations/layer1_attention_weight_max": 15.840301513671875,
      "activations/layer1_attention_weight_min": -12.743836402893066,
      "activations/layer20_attention_weight_max": 21.887170791625977,
      "activations/layer20_attention_weight_min": -22.777976989746094,
      "activations/layer21_attention_weight_max": 39.17594909667969,
      "activations/layer21_attention_weight_min": -25.111309051513672,
      "activations/layer22_attention_weight_max": 27.488780975341797,
      "activations/layer22_attention_weight_min": -24.835371017456055,
      "activations/layer23_attention_weight_max": 35.226593017578125,
      "activations/layer23_attention_weight_min": -24.69001579284668,
      "activations/layer2_attention_weight_max": 32.38589096069336,
      "activations/layer2_attention_weight_min": -30.319889068603516,
      "activations/layer3_attention_weight_max": 92.80844116210938,
      "activations/layer3_attention_weight_min": -89.75276184082031,
      "activations/layer4_attention_weight_max": 86.03782653808594,
      "activations/layer4_attention_weight_min": -89.04449462890625,
      "activations/layer5_attention_weight_max": 62.07538604736328,
      "activations/layer5_attention_weight_min": -74.49050903320312,
      "activations/layer6_attention_weight_max": 50.551639556884766,
      "activations/layer6_attention_weight_min": -50.412662506103516,
      "activations/layer7_attention_weight_max": 65.37120819091797,
      "activations/layer7_attention_weight_min": -63.83729934692383,
      "activations/layer8_attention_weight_max": 45.69396209716797,
      "activations/layer8_attention_weight_min": -49.30325698852539,
      "activations/layer9_attention_weight_max": 44.33048629760742,
      "activations/layer9_attention_weight_min": -44.73197555541992,
      "epoch": 17.7,
      "learning_rate": 3.622992424242424e-05,
      "loss": 2.7327,
      "step": 304600
    },
    {
      "activations/layer0_attention_weight_max": 16.222990036010742,
      "activations/layer0_attention_weight_min": -13.17470645904541,
      "activations/layer10_attention_weight_max": 33.51042175292969,
      "activations/layer10_attention_weight_min": -34.20148468017578,
      "activations/layer11_attention_weight_max": 34.37666702270508,
      "activations/layer11_attention_weight_min": -33.497501373291016,
      "activations/layer12_attention_weight_max": 16.440107345581055,
      "activations/layer12_attention_weight_min": -30.228620529174805,
      "activations/layer13_attention_weight_max": 37.205528259277344,
      "activations/layer13_attention_weight_min": -28.59090805053711,
      "activations/layer14_attention_weight_max": 42.96361541748047,
      "activations/layer14_attention_weight_min": -29.29512596130371,
      "activations/layer15_attention_weight_max": 42.99256134033203,
      "activations/layer15_attention_weight_min": -31.762027740478516,
      "activations/layer16_attention_weight_max": 31.311609268188477,
      "activations/layer16_attention_weight_min": -29.557628631591797,
      "activations/layer17_attention_weight_max": 50.43648910522461,
      "activations/layer17_attention_weight_min": -45.286041259765625,
      "activations/layer18_attention_weight_max": 43.64106750488281,
      "activations/layer18_attention_weight_min": -44.100914001464844,
      "activations/layer19_attention_weight_max": 22.49458885192871,
      "activations/layer19_attention_weight_min": -24.534603118896484,
      "activations/layer1_attention_weight_max": 18.046741485595703,
      "activations/layer1_attention_weight_min": -14.660626411437988,
      "activations/layer20_attention_weight_max": 23.675474166870117,
      "activations/layer20_attention_weight_min": -24.6849422454834,
      "activations/layer21_attention_weight_max": 39.65555953979492,
      "activations/layer21_attention_weight_min": -23.47288703918457,
      "activations/layer22_attention_weight_max": 28.84823226928711,
      "activations/layer22_attention_weight_min": -27.35868263244629,
      "activations/layer23_attention_weight_max": 36.461944580078125,
      "activations/layer23_attention_weight_min": -23.748779296875,
      "activations/layer2_attention_weight_max": 32.91358947753906,
      "activations/layer2_attention_weight_min": -30.280662536621094,
      "activations/layer3_attention_weight_max": 89.35618591308594,
      "activations/layer3_attention_weight_min": -89.48094177246094,
      "activations/layer4_attention_weight_max": 88.34457397460938,
      "activations/layer4_attention_weight_min": -87.00102233886719,
      "activations/layer5_attention_weight_max": 66.13081359863281,
      "activations/layer5_attention_weight_min": -72.36478424072266,
      "activations/layer6_attention_weight_max": 49.85794448852539,
      "activations/layer6_attention_weight_min": -50.20145797729492,
      "activations/layer7_attention_weight_max": 67.29381561279297,
      "activations/layer7_attention_weight_min": -65.83280181884766,
      "activations/layer8_attention_weight_max": 47.05060577392578,
      "activations/layer8_attention_weight_min": -46.49489974975586,
      "activations/layer9_attention_weight_max": 57.139801025390625,
      "activations/layer9_attention_weight_min": -43.821475982666016,
      "epoch": 17.7,
      "learning_rate": 3.621098484848484e-05,
      "loss": 2.749,
      "step": 304650
    },
    {
      "activations/layer0_attention_weight_max": 15.952557563781738,
      "activations/layer0_attention_weight_min": -13.90597152709961,
      "activations/layer10_attention_weight_max": 34.72357940673828,
      "activations/layer10_attention_weight_min": -34.788299560546875,
      "activations/layer11_attention_weight_max": 30.888288497924805,
      "activations/layer11_attention_weight_min": -33.33864974975586,
      "activations/layer12_attention_weight_max": 18.256479263305664,
      "activations/layer12_attention_weight_min": -27.688674926757812,
      "activations/layer13_attention_weight_max": 33.88450241088867,
      "activations/layer13_attention_weight_min": -25.864622116088867,
      "activations/layer14_attention_weight_max": 34.31304168701172,
      "activations/layer14_attention_weight_min": -29.18161392211914,
      "activations/layer15_attention_weight_max": 33.53616714477539,
      "activations/layer15_attention_weight_min": -29.859485626220703,
      "activations/layer16_attention_weight_max": 33.071292877197266,
      "activations/layer16_attention_weight_min": -31.179386138916016,
      "activations/layer17_attention_weight_max": 49.93180847167969,
      "activations/layer17_attention_weight_min": -46.073421478271484,
      "activations/layer18_attention_weight_max": 44.840782165527344,
      "activations/layer18_attention_weight_min": -39.256080627441406,
      "activations/layer19_attention_weight_max": 23.34065818786621,
      "activations/layer19_attention_weight_min": -21.431734085083008,
      "activations/layer1_attention_weight_max": 16.00848960876465,
      "activations/layer1_attention_weight_min": -14.783084869384766,
      "activations/layer20_attention_weight_max": 20.282716751098633,
      "activations/layer20_attention_weight_min": -22.010326385498047,
      "activations/layer21_attention_weight_max": 31.480783462524414,
      "activations/layer21_attention_weight_min": -20.976314544677734,
      "activations/layer22_attention_weight_max": 27.274993896484375,
      "activations/layer22_attention_weight_min": -24.827301025390625,
      "activations/layer23_attention_weight_max": 34.791568756103516,
      "activations/layer23_attention_weight_min": -22.69977378845215,
      "activations/layer2_attention_weight_max": 31.443328857421875,
      "activations/layer2_attention_weight_min": -29.693037033081055,
      "activations/layer3_attention_weight_max": 88.90546417236328,
      "activations/layer3_attention_weight_min": -86.94602966308594,
      "activations/layer4_attention_weight_max": 87.90126037597656,
      "activations/layer4_attention_weight_min": -85.00304412841797,
      "activations/layer5_attention_weight_max": 63.06235885620117,
      "activations/layer5_attention_weight_min": -75.57130432128906,
      "activations/layer6_attention_weight_max": 47.29723358154297,
      "activations/layer6_attention_weight_min": -50.43864822387695,
      "activations/layer7_attention_weight_max": 63.73835372924805,
      "activations/layer7_attention_weight_min": -62.119598388671875,
      "activations/layer8_attention_weight_max": 46.29672622680664,
      "activations/layer8_attention_weight_min": -46.65744400024414,
      "activations/layer9_attention_weight_max": 40.1910400390625,
      "activations/layer9_attention_weight_min": -47.172428131103516,
      "epoch": 17.7,
      "learning_rate": 3.619204545454545e-05,
      "loss": 2.7273,
      "step": 304700
    },
    {
      "activations/layer0_attention_weight_max": 16.15494728088379,
      "activations/layer0_attention_weight_min": -13.407035827636719,
      "activations/layer10_attention_weight_max": 35.05337905883789,
      "activations/layer10_attention_weight_min": -36.25864791870117,
      "activations/layer11_attention_weight_max": 33.890480041503906,
      "activations/layer11_attention_weight_min": -33.04317855834961,
      "activations/layer12_attention_weight_max": 19.083782196044922,
      "activations/layer12_attention_weight_min": -24.253068923950195,
      "activations/layer13_attention_weight_max": 34.76094055175781,
      "activations/layer13_attention_weight_min": -28.66624641418457,
      "activations/layer14_attention_weight_max": 34.3523063659668,
      "activations/layer14_attention_weight_min": -30.152803421020508,
      "activations/layer15_attention_weight_max": 33.304054260253906,
      "activations/layer15_attention_weight_min": -31.494457244873047,
      "activations/layer16_attention_weight_max": 33.77849197387695,
      "activations/layer16_attention_weight_min": -30.654041290283203,
      "activations/layer17_attention_weight_max": 53.099708557128906,
      "activations/layer17_attention_weight_min": -42.36094665527344,
      "activations/layer18_attention_weight_max": 46.83026123046875,
      "activations/layer18_attention_weight_min": -38.61845779418945,
      "activations/layer19_attention_weight_max": 22.98403549194336,
      "activations/layer19_attention_weight_min": -24.402708053588867,
      "activations/layer1_attention_weight_max": 16.219966888427734,
      "activations/layer1_attention_weight_min": -14.647344589233398,
      "activations/layer20_attention_weight_max": 24.28689956665039,
      "activations/layer20_attention_weight_min": -23.084362030029297,
      "activations/layer21_attention_weight_max": 35.278533935546875,
      "activations/layer21_attention_weight_min": -23.458377838134766,
      "activations/layer22_attention_weight_max": 29.74515724182129,
      "activations/layer22_attention_weight_min": -26.25418472290039,
      "activations/layer23_attention_weight_max": 36.21371078491211,
      "activations/layer23_attention_weight_min": -24.193227767944336,
      "activations/layer2_attention_weight_max": 31.856922149658203,
      "activations/layer2_attention_weight_min": -30.101932525634766,
      "activations/layer3_attention_weight_max": 85.89276123046875,
      "activations/layer3_attention_weight_min": -88.88642883300781,
      "activations/layer4_attention_weight_max": 87.5895767211914,
      "activations/layer4_attention_weight_min": -83.47352600097656,
      "activations/layer5_attention_weight_max": 66.29447937011719,
      "activations/layer5_attention_weight_min": -75.43220520019531,
      "activations/layer6_attention_weight_max": 47.62965393066406,
      "activations/layer6_attention_weight_min": -49.35118103027344,
      "activations/layer7_attention_weight_max": 67.10087585449219,
      "activations/layer7_attention_weight_min": -60.885337829589844,
      "activations/layer8_attention_weight_max": 45.8878288269043,
      "activations/layer8_attention_weight_min": -47.00353240966797,
      "activations/layer9_attention_weight_max": 44.19776916503906,
      "activations/layer9_attention_weight_min": -45.08659744262695,
      "epoch": 17.71,
      "learning_rate": 3.617310606060606e-05,
      "loss": 2.7261,
      "step": 304750
    },
    {
      "activations/layer0_attention_weight_max": 16.2047176361084,
      "activations/layer0_attention_weight_min": -13.126394271850586,
      "activations/layer10_attention_weight_max": 41.118812561035156,
      "activations/layer10_attention_weight_min": -40.454471588134766,
      "activations/layer11_attention_weight_max": 34.71549987792969,
      "activations/layer11_attention_weight_min": -35.19366455078125,
      "activations/layer12_attention_weight_max": 19.149036407470703,
      "activations/layer12_attention_weight_min": -25.49516487121582,
      "activations/layer13_attention_weight_max": 37.43209457397461,
      "activations/layer13_attention_weight_min": -27.096397399902344,
      "activations/layer14_attention_weight_max": 33.805419921875,
      "activations/layer14_attention_weight_min": -30.899629592895508,
      "activations/layer15_attention_weight_max": 31.66583251953125,
      "activations/layer15_attention_weight_min": -30.67690658569336,
      "activations/layer16_attention_weight_max": 34.39427185058594,
      "activations/layer16_attention_weight_min": -34.29358673095703,
      "activations/layer17_attention_weight_max": 53.05570602416992,
      "activations/layer17_attention_weight_min": -46.8904914855957,
      "activations/layer18_attention_weight_max": 49.403995513916016,
      "activations/layer18_attention_weight_min": -43.28596878051758,
      "activations/layer19_attention_weight_max": 26.67166519165039,
      "activations/layer19_attention_weight_min": -26.917652130126953,
      "activations/layer1_attention_weight_max": 15.459794998168945,
      "activations/layer1_attention_weight_min": -13.910820007324219,
      "activations/layer20_attention_weight_max": 23.960691452026367,
      "activations/layer20_attention_weight_min": -21.346519470214844,
      "activations/layer21_attention_weight_max": 37.61921691894531,
      "activations/layer21_attention_weight_min": -24.483110427856445,
      "activations/layer22_attention_weight_max": 29.326066970825195,
      "activations/layer22_attention_weight_min": -25.78589630126953,
      "activations/layer23_attention_weight_max": 36.94586181640625,
      "activations/layer23_attention_weight_min": -23.825407028198242,
      "activations/layer2_attention_weight_max": 30.20473289489746,
      "activations/layer2_attention_weight_min": -29.444318771362305,
      "activations/layer3_attention_weight_max": 90.39703369140625,
      "activations/layer3_attention_weight_min": -94.14472198486328,
      "activations/layer4_attention_weight_max": 88.91053771972656,
      "activations/layer4_attention_weight_min": -85.34557342529297,
      "activations/layer5_attention_weight_max": 65.34452056884766,
      "activations/layer5_attention_weight_min": -76.8116455078125,
      "activations/layer6_attention_weight_max": 51.98100662231445,
      "activations/layer6_attention_weight_min": -52.568729400634766,
      "activations/layer7_attention_weight_max": 66.88378143310547,
      "activations/layer7_attention_weight_min": -68.38507843017578,
      "activations/layer8_attention_weight_max": 53.555259704589844,
      "activations/layer8_attention_weight_min": -54.442893981933594,
      "activations/layer9_attention_weight_max": 54.738460540771484,
      "activations/layer9_attention_weight_min": -51.062007904052734,
      "epoch": 17.71,
      "learning_rate": 3.615416666666667e-05,
      "loss": 2.7511,
      "step": 304800
    },
    {
      "activations/layer0_attention_weight_max": 14.803571701049805,
      "activations/layer0_attention_weight_min": -12.95572566986084,
      "activations/layer10_attention_weight_max": 35.7999153137207,
      "activations/layer10_attention_weight_min": -35.22076416015625,
      "activations/layer11_attention_weight_max": 34.88536834716797,
      "activations/layer11_attention_weight_min": -34.14801788330078,
      "activations/layer12_attention_weight_max": 17.940786361694336,
      "activations/layer12_attention_weight_min": -25.782651901245117,
      "activations/layer13_attention_weight_max": 43.26453399658203,
      "activations/layer13_attention_weight_min": -25.693126678466797,
      "activations/layer14_attention_weight_max": 36.0324821472168,
      "activations/layer14_attention_weight_min": -28.990785598754883,
      "activations/layer15_attention_weight_max": 33.89735412597656,
      "activations/layer15_attention_weight_min": -29.713258743286133,
      "activations/layer16_attention_weight_max": 34.05081558227539,
      "activations/layer16_attention_weight_min": -30.677331924438477,
      "activations/layer17_attention_weight_max": 51.1905403137207,
      "activations/layer17_attention_weight_min": -42.68522262573242,
      "activations/layer18_attention_weight_max": 45.79216003417969,
      "activations/layer18_attention_weight_min": -37.263919830322266,
      "activations/layer19_attention_weight_max": 24.35942268371582,
      "activations/layer19_attention_weight_min": -22.192031860351562,
      "activations/layer1_attention_weight_max": 15.314751625061035,
      "activations/layer1_attention_weight_min": -14.588362693786621,
      "activations/layer20_attention_weight_max": 22.42107391357422,
      "activations/layer20_attention_weight_min": -21.854900360107422,
      "activations/layer21_attention_weight_max": 36.83837127685547,
      "activations/layer21_attention_weight_min": -20.477651596069336,
      "activations/layer22_attention_weight_max": 29.1743221282959,
      "activations/layer22_attention_weight_min": -24.104787826538086,
      "activations/layer23_attention_weight_max": 35.81575012207031,
      "activations/layer23_attention_weight_min": -24.54186248779297,
      "activations/layer2_attention_weight_max": 33.8418083190918,
      "activations/layer2_attention_weight_min": -30.49028778076172,
      "activations/layer3_attention_weight_max": 101.0643081665039,
      "activations/layer3_attention_weight_min": -97.59368133544922,
      "activations/layer4_attention_weight_max": 93.24889373779297,
      "activations/layer4_attention_weight_min": -88.89073181152344,
      "activations/layer5_attention_weight_max": 69.38441467285156,
      "activations/layer5_attention_weight_min": -74.37716674804688,
      "activations/layer6_attention_weight_max": 47.43803787231445,
      "activations/layer6_attention_weight_min": -49.651634216308594,
      "activations/layer7_attention_weight_max": 68.396484375,
      "activations/layer7_attention_weight_min": -64.53693389892578,
      "activations/layer8_attention_weight_max": 48.428627014160156,
      "activations/layer8_attention_weight_min": -47.53939437866211,
      "activations/layer9_attention_weight_max": 54.561805725097656,
      "activations/layer9_attention_weight_min": -45.59702682495117,
      "epoch": 17.71,
      "learning_rate": 3.613522727272727e-05,
      "loss": 2.7443,
      "step": 304850
    },
    {
      "activations/layer0_attention_weight_max": 16.14087677001953,
      "activations/layer0_attention_weight_min": -13.269856452941895,
      "activations/layer10_attention_weight_max": 37.611446380615234,
      "activations/layer10_attention_weight_min": -37.71350860595703,
      "activations/layer11_attention_weight_max": 34.17353057861328,
      "activations/layer11_attention_weight_min": -32.77443313598633,
      "activations/layer12_attention_weight_max": 19.104734420776367,
      "activations/layer12_attention_weight_min": -25.443952560424805,
      "activations/layer13_attention_weight_max": 48.17866134643555,
      "activations/layer13_attention_weight_min": -33.052734375,
      "activations/layer14_attention_weight_max": 52.04093933105469,
      "activations/layer14_attention_weight_min": -30.43364143371582,
      "activations/layer15_attention_weight_max": 47.06961441040039,
      "activations/layer15_attention_weight_min": -30.789810180664062,
      "activations/layer16_attention_weight_max": 36.01943588256836,
      "activations/layer16_attention_weight_min": -34.33627700805664,
      "activations/layer17_attention_weight_max": 54.607749938964844,
      "activations/layer17_attention_weight_min": -51.1512336730957,
      "activations/layer18_attention_weight_max": 50.20356369018555,
      "activations/layer18_attention_weight_min": -47.36130142211914,
      "activations/layer19_attention_weight_max": 30.030012130737305,
      "activations/layer19_attention_weight_min": -24.22965431213379,
      "activations/layer1_attention_weight_max": 16.48910140991211,
      "activations/layer1_attention_weight_min": -15.083539962768555,
      "activations/layer20_attention_weight_max": 30.114927291870117,
      "activations/layer20_attention_weight_min": -20.258731842041016,
      "activations/layer21_attention_weight_max": 56.47307586669922,
      "activations/layer21_attention_weight_min": -24.58941650390625,
      "activations/layer22_attention_weight_max": 29.924341201782227,
      "activations/layer22_attention_weight_min": -24.654918670654297,
      "activations/layer23_attention_weight_max": 36.810890197753906,
      "activations/layer23_attention_weight_min": -25.800153732299805,
      "activations/layer2_attention_weight_max": 32.58148956298828,
      "activations/layer2_attention_weight_min": -31.924650192260742,
      "activations/layer3_attention_weight_max": 95.55543518066406,
      "activations/layer3_attention_weight_min": -93.98424530029297,
      "activations/layer4_attention_weight_max": 89.1975326538086,
      "activations/layer4_attention_weight_min": -88.54376220703125,
      "activations/layer5_attention_weight_max": 64.47640228271484,
      "activations/layer5_attention_weight_min": -77.04954528808594,
      "activations/layer6_attention_weight_max": 51.672245025634766,
      "activations/layer6_attention_weight_min": -51.60089874267578,
      "activations/layer7_attention_weight_max": 64.44361877441406,
      "activations/layer7_attention_weight_min": -63.72089385986328,
      "activations/layer8_attention_weight_max": 48.10101318359375,
      "activations/layer8_attention_weight_min": -51.29297637939453,
      "activations/layer9_attention_weight_max": 51.5999755859375,
      "activations/layer9_attention_weight_min": -47.72148513793945,
      "epoch": 17.72,
      "learning_rate": 3.6116287878787877e-05,
      "loss": 2.7472,
      "step": 304900
    },
    {
      "activations/layer0_attention_weight_max": 15.597073554992676,
      "activations/layer0_attention_weight_min": -13.139006614685059,
      "activations/layer10_attention_weight_max": 36.811378479003906,
      "activations/layer10_attention_weight_min": -35.242767333984375,
      "activations/layer11_attention_weight_max": 33.489479064941406,
      "activations/layer11_attention_weight_min": -33.60793685913086,
      "activations/layer12_attention_weight_max": 18.213468551635742,
      "activations/layer12_attention_weight_min": -27.32024383544922,
      "activations/layer13_attention_weight_max": 40.83051300048828,
      "activations/layer13_attention_weight_min": -30.63341522216797,
      "activations/layer14_attention_weight_max": 34.3946533203125,
      "activations/layer14_attention_weight_min": -30.7009334564209,
      "activations/layer15_attention_weight_max": 32.16737747192383,
      "activations/layer15_attention_weight_min": -31.138723373413086,
      "activations/layer16_attention_weight_max": 34.709449768066406,
      "activations/layer16_attention_weight_min": -34.76033020019531,
      "activations/layer17_attention_weight_max": 50.21550369262695,
      "activations/layer17_attention_weight_min": -44.94889831542969,
      "activations/layer18_attention_weight_max": 49.074703216552734,
      "activations/layer18_attention_weight_min": -38.931488037109375,
      "activations/layer19_attention_weight_max": 23.401517868041992,
      "activations/layer19_attention_weight_min": -24.632125854492188,
      "activations/layer1_attention_weight_max": 18.352453231811523,
      "activations/layer1_attention_weight_min": -16.468360900878906,
      "activations/layer20_attention_weight_max": 22.655916213989258,
      "activations/layer20_attention_weight_min": -24.359935760498047,
      "activations/layer21_attention_weight_max": 35.58149337768555,
      "activations/layer21_attention_weight_min": -26.38334083557129,
      "activations/layer22_attention_weight_max": 28.843172073364258,
      "activations/layer22_attention_weight_min": -26.302732467651367,
      "activations/layer23_attention_weight_max": 35.28684616088867,
      "activations/layer23_attention_weight_min": -25.012441635131836,
      "activations/layer2_attention_weight_max": 32.71991729736328,
      "activations/layer2_attention_weight_min": -30.429243087768555,
      "activations/layer3_attention_weight_max": 96.30453491210938,
      "activations/layer3_attention_weight_min": -94.69432067871094,
      "activations/layer4_attention_weight_max": 98.38134002685547,
      "activations/layer4_attention_weight_min": -89.67854309082031,
      "activations/layer5_attention_weight_max": 66.34944152832031,
      "activations/layer5_attention_weight_min": -77.04180145263672,
      "activations/layer6_attention_weight_max": 53.02077865600586,
      "activations/layer6_attention_weight_min": -55.25965881347656,
      "activations/layer7_attention_weight_max": 65.90655517578125,
      "activations/layer7_attention_weight_min": -65.9632568359375,
      "activations/layer8_attention_weight_max": 47.30387878417969,
      "activations/layer8_attention_weight_min": -48.19123077392578,
      "activations/layer9_attention_weight_max": 42.56731414794922,
      "activations/layer9_attention_weight_min": -46.178245544433594,
      "epoch": 17.72,
      "learning_rate": 3.609734848484848e-05,
      "loss": 2.7438,
      "step": 304950
    },
    {
      "activations/layer0_attention_weight_max": 16.31669044494629,
      "activations/layer0_attention_weight_min": -13.205060005187988,
      "activations/layer10_attention_weight_max": 33.95901870727539,
      "activations/layer10_attention_weight_min": -34.31245040893555,
      "activations/layer11_attention_weight_max": 31.801876068115234,
      "activations/layer11_attention_weight_min": -36.248443603515625,
      "activations/layer12_attention_weight_max": 17.797391891479492,
      "activations/layer12_attention_weight_min": -24.69997787475586,
      "activations/layer13_attention_weight_max": 34.849002838134766,
      "activations/layer13_attention_weight_min": -26.64210319519043,
      "activations/layer14_attention_weight_max": 33.23112869262695,
      "activations/layer14_attention_weight_min": -29.978300094604492,
      "activations/layer15_attention_weight_max": 30.76460075378418,
      "activations/layer15_attention_weight_min": -29.902469635009766,
      "activations/layer16_attention_weight_max": 32.41108322143555,
      "activations/layer16_attention_weight_min": -31.873645782470703,
      "activations/layer17_attention_weight_max": 50.74834060668945,
      "activations/layer17_attention_weight_min": -45.40592956542969,
      "activations/layer18_attention_weight_max": 46.64512634277344,
      "activations/layer18_attention_weight_min": -39.435455322265625,
      "activations/layer19_attention_weight_max": 21.658620834350586,
      "activations/layer19_attention_weight_min": -24.101640701293945,
      "activations/layer1_attention_weight_max": 16.036043167114258,
      "activations/layer1_attention_weight_min": -13.36224365234375,
      "activations/layer20_attention_weight_max": 21.48183822631836,
      "activations/layer20_attention_weight_min": -20.37453842163086,
      "activations/layer21_attention_weight_max": 33.40269088745117,
      "activations/layer21_attention_weight_min": -22.623836517333984,
      "activations/layer22_attention_weight_max": 27.87209129333496,
      "activations/layer22_attention_weight_min": -26.537370681762695,
      "activations/layer23_attention_weight_max": 32.607666015625,
      "activations/layer23_attention_weight_min": -26.157989501953125,
      "activations/layer2_attention_weight_max": 29.181655883789062,
      "activations/layer2_attention_weight_min": -29.481908798217773,
      "activations/layer3_attention_weight_max": 86.17500305175781,
      "activations/layer3_attention_weight_min": -87.37449645996094,
      "activations/layer4_attention_weight_max": 80.03778839111328,
      "activations/layer4_attention_weight_min": -80.66043090820312,
      "activations/layer5_attention_weight_max": 62.51117706298828,
      "activations/layer5_attention_weight_min": -72.54890441894531,
      "activations/layer6_attention_weight_max": 45.45289993286133,
      "activations/layer6_attention_weight_min": -47.00859069824219,
      "activations/layer7_attention_weight_max": 64.20622253417969,
      "activations/layer7_attention_weight_min": -59.55821990966797,
      "activations/layer8_attention_weight_max": 41.66249465942383,
      "activations/layer8_attention_weight_min": -46.49665451049805,
      "activations/layer9_attention_weight_max": 43.39571762084961,
      "activations/layer9_attention_weight_min": -43.104026794433594,
      "epoch": 17.72,
      "learning_rate": 3.6078409090909086e-05,
      "loss": 2.7487,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4784,
      "eval_samples_per_second": 506.462,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4784,
      "eval_openwebtext_samples_per_second": 506.462,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_wikitext_loss": 2.927734375,
      "eval_wikitext_ppl": 18.6852487375444,
      "eval_wikitext_runtime": 1.9454,
      "eval_wikitext_samples_per_second": 234.401,
      "step": 305000
    },
    {
      "epoch": 17.72,
      "eval_lambada_loss": 2.517578125,
      "eval_lambada_ppl": 12.398532576458457,
      "eval_lambada_runtime": 9.5254,
      "eval_lambada_samples_per_second": 511.161,
      "step": 305000
    },
    {
      "activations/layer0_attention_weight_max": 15.942780494689941,
      "activations/layer0_attention_weight_min": -13.873407363891602,
      "activations/layer10_attention_weight_max": 37.688682556152344,
      "activations/layer10_attention_weight_min": -36.54307174682617,
      "activations/layer11_attention_weight_max": 32.8448600769043,
      "activations/layer11_attention_weight_min": -33.67552947998047,
      "activations/layer12_attention_weight_max": 18.749954223632812,
      "activations/layer12_attention_weight_min": -26.184078216552734,
      "activations/layer13_attention_weight_max": 31.442047119140625,
      "activations/layer13_attention_weight_min": -30.184589385986328,
      "activations/layer14_attention_weight_max": 37.23893737792969,
      "activations/layer14_attention_weight_min": -32.42720413208008,
      "activations/layer15_attention_weight_max": 35.80100631713867,
      "activations/layer15_attention_weight_min": -31.53989028930664,
      "activations/layer16_attention_weight_max": 32.803253173828125,
      "activations/layer16_attention_weight_min": -30.426591873168945,
      "activations/layer17_attention_weight_max": 50.18253707885742,
      "activations/layer17_attention_weight_min": -42.90303039550781,
      "activations/layer18_attention_weight_max": 42.93737030029297,
      "activations/layer18_attention_weight_min": -39.73404312133789,
      "activations/layer19_attention_weight_max": 22.131153106689453,
      "activations/layer19_attention_weight_min": -20.141931533813477,
      "activations/layer1_attention_weight_max": 16.90428924560547,
      "activations/layer1_attention_weight_min": -13.693809509277344,
      "activations/layer20_attention_weight_max": 20.11883544921875,
      "activations/layer20_attention_weight_min": -22.308414459228516,
      "activations/layer21_attention_weight_max": 31.84506607055664,
      "activations/layer21_attention_weight_min": -23.919946670532227,
      "activations/layer22_attention_weight_max": 29.45781135559082,
      "activations/layer22_attention_weight_min": -25.07796859741211,
      "activations/layer23_attention_weight_max": 38.90941619873047,
      "activations/layer23_attention_weight_min": -25.25104522705078,
      "activations/layer2_attention_weight_max": 31.509239196777344,
      "activations/layer2_attention_weight_min": -29.686256408691406,
      "activations/layer3_attention_weight_max": 88.70343780517578,
      "activations/layer3_attention_weight_min": -89.52766418457031,
      "activations/layer4_attention_weight_max": 87.81944274902344,
      "activations/layer4_attention_weight_min": -88.91192626953125,
      "activations/layer5_attention_weight_max": 63.963478088378906,
      "activations/layer5_attention_weight_min": -73.11947631835938,
      "activations/layer6_attention_weight_max": 49.2990608215332,
      "activations/layer6_attention_weight_min": -54.45314025878906,
      "activations/layer7_attention_weight_max": 65.82512664794922,
      "activations/layer7_attention_weight_min": -66.72268676757812,
      "activations/layer8_attention_weight_max": 45.438446044921875,
      "activations/layer8_attention_weight_min": -53.00331497192383,
      "activations/layer9_attention_weight_max": 45.76961898803711,
      "activations/layer9_attention_weight_min": -45.75736618041992,
      "epoch": 17.73,
      "learning_rate": 3.6059469696969695e-05,
      "loss": 2.7439,
      "step": 305050
    },
    {
      "activations/layer0_attention_weight_max": 16.424230575561523,
      "activations/layer0_attention_weight_min": -12.90373706817627,
      "activations/layer10_attention_weight_max": 35.02033996582031,
      "activations/layer10_attention_weight_min": -35.45880126953125,
      "activations/layer11_attention_weight_max": 33.647151947021484,
      "activations/layer11_attention_weight_min": -34.73130798339844,
      "activations/layer12_attention_weight_max": 19.368331909179688,
      "activations/layer12_attention_weight_min": -25.335479736328125,
      "activations/layer13_attention_weight_max": 35.30927658081055,
      "activations/layer13_attention_weight_min": -28.618249893188477,
      "activations/layer14_attention_weight_max": 36.60072708129883,
      "activations/layer14_attention_weight_min": -31.04060173034668,
      "activations/layer15_attention_weight_max": 33.804141998291016,
      "activations/layer15_attention_weight_min": -33.06766891479492,
      "activations/layer16_attention_weight_max": 34.17551803588867,
      "activations/layer16_attention_weight_min": -33.78424072265625,
      "activations/layer17_attention_weight_max": 53.726375579833984,
      "activations/layer17_attention_weight_min": -49.09056091308594,
      "activations/layer18_attention_weight_max": 50.531036376953125,
      "activations/layer18_attention_weight_min": -46.00904846191406,
      "activations/layer19_attention_weight_max": 25.951658248901367,
      "activations/layer19_attention_weight_min": -25.97945213317871,
      "activations/layer1_attention_weight_max": 15.6061372756958,
      "activations/layer1_attention_weight_min": -14.7960844039917,
      "activations/layer20_attention_weight_max": 25.12394905090332,
      "activations/layer20_attention_weight_min": -22.31755828857422,
      "activations/layer21_attention_weight_max": 38.36772918701172,
      "activations/layer21_attention_weight_min": -23.18256187438965,
      "activations/layer22_attention_weight_max": 30.602413177490234,
      "activations/layer22_attention_weight_min": -26.54033660888672,
      "activations/layer23_attention_weight_max": 33.870643615722656,
      "activations/layer23_attention_weight_min": -25.27975845336914,
      "activations/layer2_attention_weight_max": 32.05378723144531,
      "activations/layer2_attention_weight_min": -28.339900970458984,
      "activations/layer3_attention_weight_max": 87.32986450195312,
      "activations/layer3_attention_weight_min": -89.90032958984375,
      "activations/layer4_attention_weight_max": 87.86609649658203,
      "activations/layer4_attention_weight_min": -87.99797058105469,
      "activations/layer5_attention_weight_max": 64.65574645996094,
      "activations/layer5_attention_weight_min": -73.909423828125,
      "activations/layer6_attention_weight_max": 51.20497131347656,
      "activations/layer6_attention_weight_min": -49.895835876464844,
      "activations/layer7_attention_weight_max": 66.63219451904297,
      "activations/layer7_attention_weight_min": -65.87972259521484,
      "activations/layer8_attention_weight_max": 44.894439697265625,
      "activations/layer8_attention_weight_min": -50.31059646606445,
      "activations/layer9_attention_weight_max": 41.59487533569336,
      "activations/layer9_attention_weight_min": -45.197052001953125,
      "epoch": 17.73,
      "learning_rate": 3.60405303030303e-05,
      "loss": 2.7188,
      "step": 305100
    },
    {
      "activations/layer0_attention_weight_max": 15.601426124572754,
      "activations/layer0_attention_weight_min": -13.094696998596191,
      "activations/layer10_attention_weight_max": 36.504180908203125,
      "activations/layer10_attention_weight_min": -33.7723388671875,
      "activations/layer11_attention_weight_max": 31.84178924560547,
      "activations/layer11_attention_weight_min": -31.5987606048584,
      "activations/layer12_attention_weight_max": 19.69051170349121,
      "activations/layer12_attention_weight_min": -24.10080909729004,
      "activations/layer13_attention_weight_max": 32.31794738769531,
      "activations/layer13_attention_weight_min": -25.474178314208984,
      "activations/layer14_attention_weight_max": 35.95436096191406,
      "activations/layer14_attention_weight_min": -31.07306671142578,
      "activations/layer15_attention_weight_max": 33.8162727355957,
      "activations/layer15_attention_weight_min": -32.21101379394531,
      "activations/layer16_attention_weight_max": 35.48439025878906,
      "activations/layer16_attention_weight_min": -32.16349792480469,
      "activations/layer17_attention_weight_max": 53.87645721435547,
      "activations/layer17_attention_weight_min": -47.14524459838867,
      "activations/layer18_attention_weight_max": 49.14881896972656,
      "activations/layer18_attention_weight_min": -39.94518280029297,
      "activations/layer19_attention_weight_max": 28.075040817260742,
      "activations/layer19_attention_weight_min": -23.316341400146484,
      "activations/layer1_attention_weight_max": 16.204723358154297,
      "activations/layer1_attention_weight_min": -15.269763946533203,
      "activations/layer20_attention_weight_max": 25.694684982299805,
      "activations/layer20_attention_weight_min": -21.07779312133789,
      "activations/layer21_attention_weight_max": 37.807125091552734,
      "activations/layer21_attention_weight_min": -22.581762313842773,
      "activations/layer22_attention_weight_max": 31.8482666015625,
      "activations/layer22_attention_weight_min": -26.09518051147461,
      "activations/layer23_attention_weight_max": 38.679420471191406,
      "activations/layer23_attention_weight_min": -27.617002487182617,
      "activations/layer2_attention_weight_max": 30.93415641784668,
      "activations/layer2_attention_weight_min": -29.81574249267578,
      "activations/layer3_attention_weight_max": 89.57450103759766,
      "activations/layer3_attention_weight_min": -93.355712890625,
      "activations/layer4_attention_weight_max": 87.81746673583984,
      "activations/layer4_attention_weight_min": -84.52877807617188,
      "activations/layer5_attention_weight_max": 66.90457916259766,
      "activations/layer5_attention_weight_min": -78.24665832519531,
      "activations/layer6_attention_weight_max": 48.36083221435547,
      "activations/layer6_attention_weight_min": -50.5840950012207,
      "activations/layer7_attention_weight_max": 62.08811950683594,
      "activations/layer7_attention_weight_min": -63.94170379638672,
      "activations/layer8_attention_weight_max": 43.82001495361328,
      "activations/layer8_attention_weight_min": -47.273956298828125,
      "activations/layer9_attention_weight_max": 40.93589401245117,
      "activations/layer9_attention_weight_min": -43.8110466003418,
      "epoch": 17.73,
      "learning_rate": 3.6021590909090904e-05,
      "loss": 2.7275,
      "step": 305150
    },
    {
      "activations/layer0_attention_weight_max": 16.269973754882812,
      "activations/layer0_attention_weight_min": -13.12274169921875,
      "activations/layer10_attention_weight_max": 40.20410919189453,
      "activations/layer10_attention_weight_min": -37.28919219970703,
      "activations/layer11_attention_weight_max": 34.78266143798828,
      "activations/layer11_attention_weight_min": -33.318153381347656,
      "activations/layer12_attention_weight_max": 18.337615966796875,
      "activations/layer12_attention_weight_min": -24.760135650634766,
      "activations/layer13_attention_weight_max": 38.20774459838867,
      "activations/layer13_attention_weight_min": -28.928508758544922,
      "activations/layer14_attention_weight_max": 35.94578552246094,
      "activations/layer14_attention_weight_min": -28.475614547729492,
      "activations/layer15_attention_weight_max": 30.603057861328125,
      "activations/layer15_attention_weight_min": -27.624784469604492,
      "activations/layer16_attention_weight_max": 30.72716522216797,
      "activations/layer16_attention_weight_min": -29.297889709472656,
      "activations/layer17_attention_weight_max": 46.77774429321289,
      "activations/layer17_attention_weight_min": -41.503501892089844,
      "activations/layer18_attention_weight_max": 42.88036346435547,
      "activations/layer18_attention_weight_min": -36.736854553222656,
      "activations/layer19_attention_weight_max": 25.684040069580078,
      "activations/layer19_attention_weight_min": -20.59237289428711,
      "activations/layer1_attention_weight_max": 16.399057388305664,
      "activations/layer1_attention_weight_min": -14.016019821166992,
      "activations/layer20_attention_weight_max": 20.471956253051758,
      "activations/layer20_attention_weight_min": -20.24580955505371,
      "activations/layer21_attention_weight_max": 37.324188232421875,
      "activations/layer21_attention_weight_min": -21.777833938598633,
      "activations/layer22_attention_weight_max": 29.309993743896484,
      "activations/layer22_attention_weight_min": -24.100421905517578,
      "activations/layer23_attention_weight_max": 35.20273208618164,
      "activations/layer23_attention_weight_min": -24.794525146484375,
      "activations/layer2_attention_weight_max": 31.52655029296875,
      "activations/layer2_attention_weight_min": -28.978057861328125,
      "activations/layer3_attention_weight_max": 92.19884490966797,
      "activations/layer3_attention_weight_min": -92.23905181884766,
      "activations/layer4_attention_weight_max": 93.68949890136719,
      "activations/layer4_attention_weight_min": -91.09732055664062,
      "activations/layer5_attention_weight_max": 61.32432556152344,
      "activations/layer5_attention_weight_min": -68.42327880859375,
      "activations/layer6_attention_weight_max": 50.74290466308594,
      "activations/layer6_attention_weight_min": -50.1567268371582,
      "activations/layer7_attention_weight_max": 61.52424621582031,
      "activations/layer7_attention_weight_min": -65.31800079345703,
      "activations/layer8_attention_weight_max": 47.89463806152344,
      "activations/layer8_attention_weight_min": -51.0164794921875,
      "activations/layer9_attention_weight_max": 45.31351852416992,
      "activations/layer9_attention_weight_min": -47.198219299316406,
      "epoch": 17.73,
      "learning_rate": 3.600265151515151e-05,
      "loss": 2.7427,
      "step": 305200
    },
    {
      "activations/layer0_attention_weight_max": 15.268656730651855,
      "activations/layer0_attention_weight_min": -12.925742149353027,
      "activations/layer10_attention_weight_max": 38.03670120239258,
      "activations/layer10_attention_weight_min": -36.448211669921875,
      "activations/layer11_attention_weight_max": 34.450584411621094,
      "activations/layer11_attention_weight_min": -33.246124267578125,
      "activations/layer12_attention_weight_max": 20.076459884643555,
      "activations/layer12_attention_weight_min": -27.96048355102539,
      "activations/layer13_attention_weight_max": 40.29786682128906,
      "activations/layer13_attention_weight_min": -30.034992218017578,
      "activations/layer14_attention_weight_max": 38.178871154785156,
      "activations/layer14_attention_weight_min": -32.23369216918945,
      "activations/layer15_attention_weight_max": 35.7177734375,
      "activations/layer15_attention_weight_min": -31.106481552124023,
      "activations/layer16_attention_weight_max": 35.44339370727539,
      "activations/layer16_attention_weight_min": -31.869962692260742,
      "activations/layer17_attention_weight_max": 57.62788772583008,
      "activations/layer17_attention_weight_min": -46.29680633544922,
      "activations/layer18_attention_weight_max": 51.99710464477539,
      "activations/layer18_attention_weight_min": -38.98296356201172,
      "activations/layer19_attention_weight_max": 26.237258911132812,
      "activations/layer19_attention_weight_min": -23.0787353515625,
      "activations/layer1_attention_weight_max": 15.982562065124512,
      "activations/layer1_attention_weight_min": -13.815571784973145,
      "activations/layer20_attention_weight_max": 27.37053680419922,
      "activations/layer20_attention_weight_min": -23.41359519958496,
      "activations/layer21_attention_weight_max": 38.588226318359375,
      "activations/layer21_attention_weight_min": -24.359447479248047,
      "activations/layer22_attention_weight_max": 31.59849739074707,
      "activations/layer22_attention_weight_min": -27.61461639404297,
      "activations/layer23_attention_weight_max": 39.108482360839844,
      "activations/layer23_attention_weight_min": -24.604251861572266,
      "activations/layer2_attention_weight_max": 31.89464569091797,
      "activations/layer2_attention_weight_min": -31.585285186767578,
      "activations/layer3_attention_weight_max": 90.58063507080078,
      "activations/layer3_attention_weight_min": -96.86997985839844,
      "activations/layer4_attention_weight_max": 87.96294403076172,
      "activations/layer4_attention_weight_min": -92.81834411621094,
      "activations/layer5_attention_weight_max": 65.04792785644531,
      "activations/layer5_attention_weight_min": -72.99846649169922,
      "activations/layer6_attention_weight_max": 50.85569381713867,
      "activations/layer6_attention_weight_min": -52.48887252807617,
      "activations/layer7_attention_weight_max": 66.1559066772461,
      "activations/layer7_attention_weight_min": -64.33972930908203,
      "activations/layer8_attention_weight_max": 48.28020477294922,
      "activations/layer8_attention_weight_min": -52.39197540283203,
      "activations/layer9_attention_weight_max": 45.81711959838867,
      "activations/layer9_attention_weight_min": -44.5623664855957,
      "epoch": 17.74,
      "learning_rate": 3.5983712121212114e-05,
      "loss": 2.7464,
      "step": 305250
    },
    {
      "activations/layer0_attention_weight_max": 15.592482566833496,
      "activations/layer0_attention_weight_min": -12.789382934570312,
      "activations/layer10_attention_weight_max": 35.78348159790039,
      "activations/layer10_attention_weight_min": -35.39860916137695,
      "activations/layer11_attention_weight_max": 32.85712432861328,
      "activations/layer11_attention_weight_min": -34.15528869628906,
      "activations/layer12_attention_weight_max": 18.377912521362305,
      "activations/layer12_attention_weight_min": -24.835994720458984,
      "activations/layer13_attention_weight_max": 32.16200637817383,
      "activations/layer13_attention_weight_min": -28.9486141204834,
      "activations/layer14_attention_weight_max": 34.28278350830078,
      "activations/layer14_attention_weight_min": -29.33796501159668,
      "activations/layer15_attention_weight_max": 33.12689208984375,
      "activations/layer15_attention_weight_min": -30.127248764038086,
      "activations/layer16_attention_weight_max": 34.41481018066406,
      "activations/layer16_attention_weight_min": -31.478321075439453,
      "activations/layer17_attention_weight_max": 52.873008728027344,
      "activations/layer17_attention_weight_min": -45.15790557861328,
      "activations/layer18_attention_weight_max": 47.588191986083984,
      "activations/layer18_attention_weight_min": -39.06726837158203,
      "activations/layer19_attention_weight_max": 24.191776275634766,
      "activations/layer19_attention_weight_min": -21.874004364013672,
      "activations/layer1_attention_weight_max": 15.211555480957031,
      "activations/layer1_attention_weight_min": -13.976887702941895,
      "activations/layer20_attention_weight_max": 22.315025329589844,
      "activations/layer20_attention_weight_min": -21.1501407623291,
      "activations/layer21_attention_weight_max": 32.68027877807617,
      "activations/layer21_attention_weight_min": -21.662960052490234,
      "activations/layer22_attention_weight_max": 28.23482894897461,
      "activations/layer22_attention_weight_min": -26.143905639648438,
      "activations/layer23_attention_weight_max": 33.05959701538086,
      "activations/layer23_attention_weight_min": -24.21200180053711,
      "activations/layer2_attention_weight_max": 30.467267990112305,
      "activations/layer2_attention_weight_min": -31.67929458618164,
      "activations/layer3_attention_weight_max": 86.84878540039062,
      "activations/layer3_attention_weight_min": -91.34236145019531,
      "activations/layer4_attention_weight_max": 83.32811737060547,
      "activations/layer4_attention_weight_min": -84.92390441894531,
      "activations/layer5_attention_weight_max": 62.179527282714844,
      "activations/layer5_attention_weight_min": -75.53114318847656,
      "activations/layer6_attention_weight_max": 49.130802154541016,
      "activations/layer6_attention_weight_min": -51.314937591552734,
      "activations/layer7_attention_weight_max": 65.82356262207031,
      "activations/layer7_attention_weight_min": -62.425559997558594,
      "activations/layer8_attention_weight_max": 45.46251678466797,
      "activations/layer8_attention_weight_min": -50.501155853271484,
      "activations/layer9_attention_weight_max": 46.23845672607422,
      "activations/layer9_attention_weight_min": -46.536041259765625,
      "epoch": 17.74,
      "learning_rate": 3.596477272727272e-05,
      "loss": 2.7479,
      "step": 305300
    },
    {
      "activations/layer0_attention_weight_max": 15.129850387573242,
      "activations/layer0_attention_weight_min": -12.736503601074219,
      "activations/layer10_attention_weight_max": 36.61143493652344,
      "activations/layer10_attention_weight_min": -35.16384506225586,
      "activations/layer11_attention_weight_max": 33.398406982421875,
      "activations/layer11_attention_weight_min": -35.385658264160156,
      "activations/layer12_attention_weight_max": 18.76325035095215,
      "activations/layer12_attention_weight_min": -23.874542236328125,
      "activations/layer13_attention_weight_max": 33.67742919921875,
      "activations/layer13_attention_weight_min": -26.368391036987305,
      "activations/layer14_attention_weight_max": 35.60883331298828,
      "activations/layer14_attention_weight_min": -31.273591995239258,
      "activations/layer15_attention_weight_max": 32.923377990722656,
      "activations/layer15_attention_weight_min": -30.851213455200195,
      "activations/layer16_attention_weight_max": 34.6728630065918,
      "activations/layer16_attention_weight_min": -32.49179458618164,
      "activations/layer17_attention_weight_max": 52.315799713134766,
      "activations/layer17_attention_weight_min": -45.49908447265625,
      "activations/layer18_attention_weight_max": 46.62739944458008,
      "activations/layer18_attention_weight_min": -40.647159576416016,
      "activations/layer19_attention_weight_max": 23.588781356811523,
      "activations/layer19_attention_weight_min": -22.103553771972656,
      "activations/layer1_attention_weight_max": 15.98928165435791,
      "activations/layer1_attention_weight_min": -13.559392929077148,
      "activations/layer20_attention_weight_max": 22.55060577392578,
      "activations/layer20_attention_weight_min": -23.056621551513672,
      "activations/layer21_attention_weight_max": 33.78456497192383,
      "activations/layer21_attention_weight_min": -20.12052345275879,
      "activations/layer22_attention_weight_max": 27.72816276550293,
      "activations/layer22_attention_weight_min": -24.916318893432617,
      "activations/layer23_attention_weight_max": 38.07600402832031,
      "activations/layer23_attention_weight_min": -23.719947814941406,
      "activations/layer2_attention_weight_max": 33.086143493652344,
      "activations/layer2_attention_weight_min": -29.481306076049805,
      "activations/layer3_attention_weight_max": 89.57746124267578,
      "activations/layer3_attention_weight_min": -91.22870635986328,
      "activations/layer4_attention_weight_max": 87.28223419189453,
      "activations/layer4_attention_weight_min": -87.43289947509766,
      "activations/layer5_attention_weight_max": 68.98139953613281,
      "activations/layer5_attention_weight_min": -74.50494384765625,
      "activations/layer6_attention_weight_max": 49.46038055419922,
      "activations/layer6_attention_weight_min": -49.89531326293945,
      "activations/layer7_attention_weight_max": 61.47722625732422,
      "activations/layer7_attention_weight_min": -62.02068328857422,
      "activations/layer8_attention_weight_max": 46.257083892822266,
      "activations/layer8_attention_weight_min": -47.56679153442383,
      "activations/layer9_attention_weight_max": 43.207481384277344,
      "activations/layer9_attention_weight_min": -48.04165267944336,
      "epoch": 17.74,
      "learning_rate": 3.594583333333333e-05,
      "loss": 2.7403,
      "step": 305350
    },
    {
      "activations/layer0_attention_weight_max": 16.262428283691406,
      "activations/layer0_attention_weight_min": -13.112044334411621,
      "activations/layer10_attention_weight_max": 36.013885498046875,
      "activations/layer10_attention_weight_min": -35.3351936340332,
      "activations/layer11_attention_weight_max": 31.91254234313965,
      "activations/layer11_attention_weight_min": -32.50168991088867,
      "activations/layer12_attention_weight_max": 18.034265518188477,
      "activations/layer12_attention_weight_min": -24.85551643371582,
      "activations/layer13_attention_weight_max": 34.76709747314453,
      "activations/layer13_attention_weight_min": -26.436155319213867,
      "activations/layer14_attention_weight_max": 33.951412200927734,
      "activations/layer14_attention_weight_min": -28.31138038635254,
      "activations/layer15_attention_weight_max": 31.051307678222656,
      "activations/layer15_attention_weight_min": -28.739234924316406,
      "activations/layer16_attention_weight_max": 31.19732666015625,
      "activations/layer16_attention_weight_min": -29.623615264892578,
      "activations/layer17_attention_weight_max": 50.568058013916016,
      "activations/layer17_attention_weight_min": -43.81865310668945,
      "activations/layer18_attention_weight_max": 42.30948257446289,
      "activations/layer18_attention_weight_min": -38.83034896850586,
      "activations/layer19_attention_weight_max": 22.590612411499023,
      "activations/layer19_attention_weight_min": -21.88361167907715,
      "activations/layer1_attention_weight_max": 17.15135383605957,
      "activations/layer1_attention_weight_min": -14.078985214233398,
      "activations/layer20_attention_weight_max": 20.45208168029785,
      "activations/layer20_attention_weight_min": -20.551939010620117,
      "activations/layer21_attention_weight_max": 30.895957946777344,
      "activations/layer21_attention_weight_min": -24.161962509155273,
      "activations/layer22_attention_weight_max": 27.884708404541016,
      "activations/layer22_attention_weight_min": -24.438997268676758,
      "activations/layer23_attention_weight_max": 37.19763946533203,
      "activations/layer23_attention_weight_min": -25.22066879272461,
      "activations/layer2_attention_weight_max": 30.349578857421875,
      "activations/layer2_attention_weight_min": -27.792179107666016,
      "activations/layer3_attention_weight_max": 90.4284896850586,
      "activations/layer3_attention_weight_min": -87.75230407714844,
      "activations/layer4_attention_weight_max": 92.34781646728516,
      "activations/layer4_attention_weight_min": -89.20040130615234,
      "activations/layer5_attention_weight_max": 68.5678482055664,
      "activations/layer5_attention_weight_min": -76.79345703125,
      "activations/layer6_attention_weight_max": 50.35639953613281,
      "activations/layer6_attention_weight_min": -52.062496185302734,
      "activations/layer7_attention_weight_max": 66.35464477539062,
      "activations/layer7_attention_weight_min": -67.40210723876953,
      "activations/layer8_attention_weight_max": 46.05690002441406,
      "activations/layer8_attention_weight_min": -50.529178619384766,
      "activations/layer9_attention_weight_max": 45.2121696472168,
      "activations/layer9_attention_weight_min": -47.9648323059082,
      "epoch": 17.75,
      "learning_rate": 3.592689393939394e-05,
      "loss": 2.7262,
      "step": 305400
    },
    {
      "activations/layer0_attention_weight_max": 16.248186111450195,
      "activations/layer0_attention_weight_min": -13.763317108154297,
      "activations/layer10_attention_weight_max": 35.5247802734375,
      "activations/layer10_attention_weight_min": -35.66383743286133,
      "activations/layer11_attention_weight_max": 30.841901779174805,
      "activations/layer11_attention_weight_min": -35.90958023071289,
      "activations/layer12_attention_weight_max": 21.404422760009766,
      "activations/layer12_attention_weight_min": -24.4171199798584,
      "activations/layer13_attention_weight_max": 32.82084655761719,
      "activations/layer13_attention_weight_min": -27.89975357055664,
      "activations/layer14_attention_weight_max": 35.4742546081543,
      "activations/layer14_attention_weight_min": -31.70842742919922,
      "activations/layer15_attention_weight_max": 33.751678466796875,
      "activations/layer15_attention_weight_min": -30.805898666381836,
      "activations/layer16_attention_weight_max": 35.47864532470703,
      "activations/layer16_attention_weight_min": -33.196773529052734,
      "activations/layer17_attention_weight_max": 51.82316970825195,
      "activations/layer17_attention_weight_min": -45.16520309448242,
      "activations/layer18_attention_weight_max": 46.6197509765625,
      "activations/layer18_attention_weight_min": -37.75135803222656,
      "activations/layer19_attention_weight_max": 22.623977661132812,
      "activations/layer19_attention_weight_min": -22.19913673400879,
      "activations/layer1_attention_weight_max": 16.012325286865234,
      "activations/layer1_attention_weight_min": -14.2633638381958,
      "activations/layer20_attention_weight_max": 21.13212013244629,
      "activations/layer20_attention_weight_min": -22.0433406829834,
      "activations/layer21_attention_weight_max": 34.77606201171875,
      "activations/layer21_attention_weight_min": -21.774471282958984,
      "activations/layer22_attention_weight_max": 29.17310905456543,
      "activations/layer22_attention_weight_min": -25.5987548828125,
      "activations/layer23_attention_weight_max": 35.20834732055664,
      "activations/layer23_attention_weight_min": -24.50066375732422,
      "activations/layer2_attention_weight_max": 30.319602966308594,
      "activations/layer2_attention_weight_min": -29.620895385742188,
      "activations/layer3_attention_weight_max": 87.68182373046875,
      "activations/layer3_attention_weight_min": -92.36278533935547,
      "activations/layer4_attention_weight_max": 85.32543182373047,
      "activations/layer4_attention_weight_min": -85.85408782958984,
      "activations/layer5_attention_weight_max": 64.15415954589844,
      "activations/layer5_attention_weight_min": -78.8100814819336,
      "activations/layer6_attention_weight_max": 48.105812072753906,
      "activations/layer6_attention_weight_min": -51.06144332885742,
      "activations/layer7_attention_weight_max": 64.3668441772461,
      "activations/layer7_attention_weight_min": -64.1641616821289,
      "activations/layer8_attention_weight_max": 44.04460525512695,
      "activations/layer8_attention_weight_min": -49.20001220703125,
      "activations/layer9_attention_weight_max": 43.113094329833984,
      "activations/layer9_attention_weight_min": -45.37866973876953,
      "epoch": 17.75,
      "learning_rate": 3.590795454545455e-05,
      "loss": 2.7397,
      "step": 305450
    },
    {
      "activations/layer0_attention_weight_max": 15.22982406616211,
      "activations/layer0_attention_weight_min": -12.9816312789917,
      "activations/layer10_attention_weight_max": 37.61009979248047,
      "activations/layer10_attention_weight_min": -35.7374153137207,
      "activations/layer11_attention_weight_max": 33.195621490478516,
      "activations/layer11_attention_weight_min": -35.39995574951172,
      "activations/layer12_attention_weight_max": 17.699825286865234,
      "activations/layer12_attention_weight_min": -22.477771759033203,
      "activations/layer13_attention_weight_max": 35.24793243408203,
      "activations/layer13_attention_weight_min": -28.33013916015625,
      "activations/layer14_attention_weight_max": 33.46827697753906,
      "activations/layer14_attention_weight_min": -31.647581100463867,
      "activations/layer15_attention_weight_max": 30.895524978637695,
      "activations/layer15_attention_weight_min": -29.51712417602539,
      "activations/layer16_attention_weight_max": 30.38200569152832,
      "activations/layer16_attention_weight_min": -30.0174503326416,
      "activations/layer17_attention_weight_max": 48.43289566040039,
      "activations/layer17_attention_weight_min": -46.890159606933594,
      "activations/layer18_attention_weight_max": 44.51384735107422,
      "activations/layer18_attention_weight_min": -39.94799041748047,
      "activations/layer19_attention_weight_max": 22.798809051513672,
      "activations/layer19_attention_weight_min": -22.723487854003906,
      "activations/layer1_attention_weight_max": 16.774593353271484,
      "activations/layer1_attention_weight_min": -14.039525032043457,
      "activations/layer20_attention_weight_max": 22.592039108276367,
      "activations/layer20_attention_weight_min": -21.502544403076172,
      "activations/layer21_attention_weight_max": 33.6273078918457,
      "activations/layer21_attention_weight_min": -21.698232650756836,
      "activations/layer22_attention_weight_max": 25.204532623291016,
      "activations/layer22_attention_weight_min": -24.07261085510254,
      "activations/layer23_attention_weight_max": 36.43021011352539,
      "activations/layer23_attention_weight_min": -25.682065963745117,
      "activations/layer2_attention_weight_max": 30.4965877532959,
      "activations/layer2_attention_weight_min": -28.323514938354492,
      "activations/layer3_attention_weight_max": 87.7253646850586,
      "activations/layer3_attention_weight_min": -88.42288970947266,
      "activations/layer4_attention_weight_max": 84.4377670288086,
      "activations/layer4_attention_weight_min": -88.00709533691406,
      "activations/layer5_attention_weight_max": 65.16815185546875,
      "activations/layer5_attention_weight_min": -69.52098846435547,
      "activations/layer6_attention_weight_max": 48.081260681152344,
      "activations/layer6_attention_weight_min": -48.89491271972656,
      "activations/layer7_attention_weight_max": 61.47719955444336,
      "activations/layer7_attention_weight_min": -60.07560729980469,
      "activations/layer8_attention_weight_max": 44.657012939453125,
      "activations/layer8_attention_weight_min": -45.75547790527344,
      "activations/layer9_attention_weight_max": 41.52227020263672,
      "activations/layer9_attention_weight_min": -44.928550720214844,
      "epoch": 17.75,
      "learning_rate": 3.588901515151515e-05,
      "loss": 2.7395,
      "step": 305500
    },
    {
      "activations/layer0_attention_weight_max": 16.959470748901367,
      "activations/layer0_attention_weight_min": -13.220043182373047,
      "activations/layer10_attention_weight_max": 32.199867248535156,
      "activations/layer10_attention_weight_min": -34.321678161621094,
      "activations/layer11_attention_weight_max": 31.320293426513672,
      "activations/layer11_attention_weight_min": -34.4274787902832,
      "activations/layer12_attention_weight_max": 17.481319427490234,
      "activations/layer12_attention_weight_min": -27.477516174316406,
      "activations/layer13_attention_weight_max": 31.536632537841797,
      "activations/layer13_attention_weight_min": -26.51012420654297,
      "activations/layer14_attention_weight_max": 32.333251953125,
      "activations/layer14_attention_weight_min": -27.479398727416992,
      "activations/layer15_attention_weight_max": 32.469749450683594,
      "activations/layer15_attention_weight_min": -29.699262619018555,
      "activations/layer16_attention_weight_max": 30.575489044189453,
      "activations/layer16_attention_weight_min": -30.004077911376953,
      "activations/layer17_attention_weight_max": 48.68963623046875,
      "activations/layer17_attention_weight_min": -40.92685317993164,
      "activations/layer18_attention_weight_max": 41.20418167114258,
      "activations/layer18_attention_weight_min": -36.58769989013672,
      "activations/layer19_attention_weight_max": 23.43234634399414,
      "activations/layer19_attention_weight_min": -22.44703483581543,
      "activations/layer1_attention_weight_max": 15.898055076599121,
      "activations/layer1_attention_weight_min": -14.23040771484375,
      "activations/layer20_attention_weight_max": 24.55447006225586,
      "activations/layer20_attention_weight_min": -22.26854133605957,
      "activations/layer21_attention_weight_max": 32.38603973388672,
      "activations/layer21_attention_weight_min": -23.578655242919922,
      "activations/layer22_attention_weight_max": 27.318927764892578,
      "activations/layer22_attention_weight_min": -25.236820220947266,
      "activations/layer23_attention_weight_max": 37.60195541381836,
      "activations/layer23_attention_weight_min": -23.36128807067871,
      "activations/layer2_attention_weight_max": 29.927867889404297,
      "activations/layer2_attention_weight_min": -28.08039093017578,
      "activations/layer3_attention_weight_max": 86.22164154052734,
      "activations/layer3_attention_weight_min": -87.80674743652344,
      "activations/layer4_attention_weight_max": 83.4135971069336,
      "activations/layer4_attention_weight_min": -82.94774627685547,
      "activations/layer5_attention_weight_max": 59.94474411010742,
      "activations/layer5_attention_weight_min": -72.07772827148438,
      "activations/layer6_attention_weight_max": 48.15327835083008,
      "activations/layer6_attention_weight_min": -49.689186096191406,
      "activations/layer7_attention_weight_max": 66.31520080566406,
      "activations/layer7_attention_weight_min": -59.59005355834961,
      "activations/layer8_attention_weight_max": 45.63279724121094,
      "activations/layer8_attention_weight_min": -47.9813346862793,
      "activations/layer9_attention_weight_max": 49.339359283447266,
      "activations/layer9_attention_weight_min": -43.6434211730957,
      "epoch": 17.75,
      "learning_rate": 3.587007575757576e-05,
      "loss": 2.7538,
      "step": 305550
    },
    {
      "activations/layer0_attention_weight_max": 15.455159187316895,
      "activations/layer0_attention_weight_min": -13.372526168823242,
      "activations/layer10_attention_weight_max": 33.94752883911133,
      "activations/layer10_attention_weight_min": -34.42085647583008,
      "activations/layer11_attention_weight_max": 31.15615463256836,
      "activations/layer11_attention_weight_min": -34.468536376953125,
      "activations/layer12_attention_weight_max": 19.12661361694336,
      "activations/layer12_attention_weight_min": -27.549291610717773,
      "activations/layer13_attention_weight_max": 32.17564010620117,
      "activations/layer13_attention_weight_min": -29.69400978088379,
      "activations/layer14_attention_weight_max": 34.09917449951172,
      "activations/layer14_attention_weight_min": -29.908218383789062,
      "activations/layer15_attention_weight_max": 33.029273986816406,
      "activations/layer15_attention_weight_min": -32.37271499633789,
      "activations/layer16_attention_weight_max": 33.38575744628906,
      "activations/layer16_attention_weight_min": -30.917972564697266,
      "activations/layer17_attention_weight_max": 47.5684700012207,
      "activations/layer17_attention_weight_min": -44.55577087402344,
      "activations/layer18_attention_weight_max": 45.325191497802734,
      "activations/layer18_attention_weight_min": -39.12322235107422,
      "activations/layer19_attention_weight_max": 25.127819061279297,
      "activations/layer19_attention_weight_min": -21.51150894165039,
      "activations/layer1_attention_weight_max": 17.116914749145508,
      "activations/layer1_attention_weight_min": -13.4578218460083,
      "activations/layer20_attention_weight_max": 24.712804794311523,
      "activations/layer20_attention_weight_min": -21.92589569091797,
      "activations/layer21_attention_weight_max": 40.877349853515625,
      "activations/layer21_attention_weight_min": -21.988431930541992,
      "activations/layer22_attention_weight_max": 30.986068725585938,
      "activations/layer22_attention_weight_min": -26.600507736206055,
      "activations/layer23_attention_weight_max": 37.42974853515625,
      "activations/layer23_attention_weight_min": -25.01317596435547,
      "activations/layer2_attention_weight_max": 30.32461166381836,
      "activations/layer2_attention_weight_min": -29.64975357055664,
      "activations/layer3_attention_weight_max": 88.69023132324219,
      "activations/layer3_attention_weight_min": -92.46629333496094,
      "activations/layer4_attention_weight_max": 88.381591796875,
      "activations/layer4_attention_weight_min": -86.55738830566406,
      "activations/layer5_attention_weight_max": 64.01438903808594,
      "activations/layer5_attention_weight_min": -77.6377944946289,
      "activations/layer6_attention_weight_max": 48.151180267333984,
      "activations/layer6_attention_weight_min": -48.77680969238281,
      "activations/layer7_attention_weight_max": 68.31436157226562,
      "activations/layer7_attention_weight_min": -61.59351348876953,
      "activations/layer8_attention_weight_max": 44.43719482421875,
      "activations/layer8_attention_weight_min": -47.9465217590332,
      "activations/layer9_attention_weight_max": 47.457393646240234,
      "activations/layer9_attention_weight_min": -43.34679412841797,
      "epoch": 17.76,
      "learning_rate": 3.585113636363636e-05,
      "loss": 2.7377,
      "step": 305600
    },
    {
      "activations/layer0_attention_weight_max": 15.46350383758545,
      "activations/layer0_attention_weight_min": -13.205550193786621,
      "activations/layer10_attention_weight_max": 36.690853118896484,
      "activations/layer10_attention_weight_min": -35.37796401977539,
      "activations/layer11_attention_weight_max": 35.71464538574219,
      "activations/layer11_attention_weight_min": -36.17418670654297,
      "activations/layer12_attention_weight_max": 19.525028228759766,
      "activations/layer12_attention_weight_min": -24.800643920898438,
      "activations/layer13_attention_weight_max": 31.70439910888672,
      "activations/layer13_attention_weight_min": -27.092973709106445,
      "activations/layer14_attention_weight_max": 36.01838302612305,
      "activations/layer14_attention_weight_min": -31.084074020385742,
      "activations/layer15_attention_weight_max": 32.68700408935547,
      "activations/layer15_attention_weight_min": -30.390653610229492,
      "activations/layer16_attention_weight_max": 35.73660659790039,
      "activations/layer16_attention_weight_min": -34.049232482910156,
      "activations/layer17_attention_weight_max": 52.863365173339844,
      "activations/layer17_attention_weight_min": -48.38690185546875,
      "activations/layer18_attention_weight_max": 47.484771728515625,
      "activations/layer18_attention_weight_min": -40.06781768798828,
      "activations/layer19_attention_weight_max": 22.809253692626953,
      "activations/layer19_attention_weight_min": -23.348247528076172,
      "activations/layer1_attention_weight_max": 16.227142333984375,
      "activations/layer1_attention_weight_min": -13.954142570495605,
      "activations/layer20_attention_weight_max": 22.3126220703125,
      "activations/layer20_attention_weight_min": -24.55758285522461,
      "activations/layer21_attention_weight_max": 36.470558166503906,
      "activations/layer21_attention_weight_min": -21.461816787719727,
      "activations/layer22_attention_weight_max": 29.664758682250977,
      "activations/layer22_attention_weight_min": -25.520612716674805,
      "activations/layer23_attention_weight_max": 37.47882080078125,
      "activations/layer23_attention_weight_min": -26.405067443847656,
      "activations/layer2_attention_weight_max": 31.743000030517578,
      "activations/layer2_attention_weight_min": -28.887107849121094,
      "activations/layer3_attention_weight_max": 89.01605224609375,
      "activations/layer3_attention_weight_min": -89.0487289428711,
      "activations/layer4_attention_weight_max": 88.78083038330078,
      "activations/layer4_attention_weight_min": -86.07869720458984,
      "activations/layer5_attention_weight_max": 65.24668884277344,
      "activations/layer5_attention_weight_min": -74.44752502441406,
      "activations/layer6_attention_weight_max": 52.190189361572266,
      "activations/layer6_attention_weight_min": -51.641868591308594,
      "activations/layer7_attention_weight_max": 64.90352630615234,
      "activations/layer7_attention_weight_min": -60.73841857910156,
      "activations/layer8_attention_weight_max": 45.95139694213867,
      "activations/layer8_attention_weight_min": -48.6579475402832,
      "activations/layer9_attention_weight_max": 45.26035690307617,
      "activations/layer9_attention_weight_min": -46.15633773803711,
      "epoch": 17.76,
      "learning_rate": 3.583219696969697e-05,
      "loss": 2.7447,
      "step": 305650
    },
    {
      "activations/layer0_attention_weight_max": 15.634126663208008,
      "activations/layer0_attention_weight_min": -13.272387504577637,
      "activations/layer10_attention_weight_max": 33.09724044799805,
      "activations/layer10_attention_weight_min": -33.67289352416992,
      "activations/layer11_attention_weight_max": 31.717857360839844,
      "activations/layer11_attention_weight_min": -33.49419021606445,
      "activations/layer12_attention_weight_max": 20.084177017211914,
      "activations/layer12_attention_weight_min": -27.9057674407959,
      "activations/layer13_attention_weight_max": 38.962547302246094,
      "activations/layer13_attention_weight_min": -29.673070907592773,
      "activations/layer14_attention_weight_max": 38.07585906982422,
      "activations/layer14_attention_weight_min": -30.934309005737305,
      "activations/layer15_attention_weight_max": 38.848533630371094,
      "activations/layer15_attention_weight_min": -32.540611267089844,
      "activations/layer16_attention_weight_max": 38.52033615112305,
      "activations/layer16_attention_weight_min": -36.26609420776367,
      "activations/layer17_attention_weight_max": 55.66111373901367,
      "activations/layer17_attention_weight_min": -48.5117301940918,
      "activations/layer18_attention_weight_max": 51.991065979003906,
      "activations/layer18_attention_weight_min": -43.08293914794922,
      "activations/layer19_attention_weight_max": 28.905078887939453,
      "activations/layer19_attention_weight_min": -23.921545028686523,
      "activations/layer1_attention_weight_max": 15.952703475952148,
      "activations/layer1_attention_weight_min": -14.853723526000977,
      "activations/layer20_attention_weight_max": 26.19265365600586,
      "activations/layer20_attention_weight_min": -21.556455612182617,
      "activations/layer21_attention_weight_max": 38.038719177246094,
      "activations/layer21_attention_weight_min": -21.254276275634766,
      "activations/layer22_attention_weight_max": 33.04505157470703,
      "activations/layer22_attention_weight_min": -25.861183166503906,
      "activations/layer23_attention_weight_max": 41.08013153076172,
      "activations/layer23_attention_weight_min": -22.669811248779297,
      "activations/layer2_attention_weight_max": 30.487768173217773,
      "activations/layer2_attention_weight_min": -30.570415496826172,
      "activations/layer3_attention_weight_max": 86.08468627929688,
      "activations/layer3_attention_weight_min": -89.7506103515625,
      "activations/layer4_attention_weight_max": 86.09357452392578,
      "activations/layer4_attention_weight_min": -85.1659164428711,
      "activations/layer5_attention_weight_max": 64.65000915527344,
      "activations/layer5_attention_weight_min": -73.49586486816406,
      "activations/layer6_attention_weight_max": 48.67677688598633,
      "activations/layer6_attention_weight_min": -50.91319274902344,
      "activations/layer7_attention_weight_max": 65.18610382080078,
      "activations/layer7_attention_weight_min": -61.5692024230957,
      "activations/layer8_attention_weight_max": 47.123504638671875,
      "activations/layer8_attention_weight_min": -46.87941360473633,
      "activations/layer9_attention_weight_max": 43.46086883544922,
      "activations/layer9_attention_weight_min": -46.260475158691406,
      "epoch": 17.76,
      "learning_rate": 3.581325757575757e-05,
      "loss": 2.7438,
      "step": 305700
    },
    {
      "activations/layer0_attention_weight_max": 15.194561004638672,
      "activations/layer0_attention_weight_min": -13.824448585510254,
      "activations/layer10_attention_weight_max": 32.94690704345703,
      "activations/layer10_attention_weight_min": -33.75454330444336,
      "activations/layer11_attention_weight_max": 32.42936325073242,
      "activations/layer11_attention_weight_min": -32.7103157043457,
      "activations/layer12_attention_weight_max": 17.939102172851562,
      "activations/layer12_attention_weight_min": -26.14228057861328,
      "activations/layer13_attention_weight_max": 44.021480560302734,
      "activations/layer13_attention_weight_min": -25.94579315185547,
      "activations/layer14_attention_weight_max": 52.4718017578125,
      "activations/layer14_attention_weight_min": -31.49117660522461,
      "activations/layer15_attention_weight_max": 47.588706970214844,
      "activations/layer15_attention_weight_min": -30.041820526123047,
      "activations/layer16_attention_weight_max": 32.79975891113281,
      "activations/layer16_attention_weight_min": -32.26699447631836,
      "activations/layer17_attention_weight_max": 51.33072280883789,
      "activations/layer17_attention_weight_min": -50.46009063720703,
      "activations/layer18_attention_weight_max": 46.420772552490234,
      "activations/layer18_attention_weight_min": -47.10382080078125,
      "activations/layer19_attention_weight_max": 28.09641456604004,
      "activations/layer19_attention_weight_min": -25.10328483581543,
      "activations/layer1_attention_weight_max": 16.273283004760742,
      "activations/layer1_attention_weight_min": -14.376829147338867,
      "activations/layer20_attention_weight_max": 27.2481689453125,
      "activations/layer20_attention_weight_min": -22.85799789428711,
      "activations/layer21_attention_weight_max": 39.10579299926758,
      "activations/layer21_attention_weight_min": -25.894792556762695,
      "activations/layer22_attention_weight_max": 30.657651901245117,
      "activations/layer22_attention_weight_min": -26.52608299255371,
      "activations/layer23_attention_weight_max": 40.102455139160156,
      "activations/layer23_attention_weight_min": -23.65298843383789,
      "activations/layer2_attention_weight_max": 32.060462951660156,
      "activations/layer2_attention_weight_min": -28.39111328125,
      "activations/layer3_attention_weight_max": 90.83177947998047,
      "activations/layer3_attention_weight_min": -90.78536987304688,
      "activations/layer4_attention_weight_max": 88.66378021240234,
      "activations/layer4_attention_weight_min": -83.91783905029297,
      "activations/layer5_attention_weight_max": 64.17146301269531,
      "activations/layer5_attention_weight_min": -69.2874755859375,
      "activations/layer6_attention_weight_max": 48.83138656616211,
      "activations/layer6_attention_weight_min": -47.57831573486328,
      "activations/layer7_attention_weight_max": 64.44683074951172,
      "activations/layer7_attention_weight_min": -62.62617111206055,
      "activations/layer8_attention_weight_max": 44.72365188598633,
      "activations/layer8_attention_weight_min": -44.89299011230469,
      "activations/layer9_attention_weight_max": 47.17236328125,
      "activations/layer9_attention_weight_min": -43.78920364379883,
      "epoch": 17.77,
      "learning_rate": 3.5794318181818176e-05,
      "loss": 2.7513,
      "step": 305750
    },
    {
      "activations/layer0_attention_weight_max": 15.009757041931152,
      "activations/layer0_attention_weight_min": -13.849557876586914,
      "activations/layer10_attention_weight_max": 35.90681076049805,
      "activations/layer10_attention_weight_min": -34.623626708984375,
      "activations/layer11_attention_weight_max": 32.82539367675781,
      "activations/layer11_attention_weight_min": -34.016902923583984,
      "activations/layer12_attention_weight_max": 17.56351089477539,
      "activations/layer12_attention_weight_min": -24.06269073486328,
      "activations/layer13_attention_weight_max": 32.793434143066406,
      "activations/layer13_attention_weight_min": -28.203020095825195,
      "activations/layer14_attention_weight_max": 34.29520797729492,
      "activations/layer14_attention_weight_min": -28.80548667907715,
      "activations/layer15_attention_weight_max": 33.71893310546875,
      "activations/layer15_attention_weight_min": -29.003437042236328,
      "activations/layer16_attention_weight_max": 31.734830856323242,
      "activations/layer16_attention_weight_min": -30.945301055908203,
      "activations/layer17_attention_weight_max": 49.00018310546875,
      "activations/layer17_attention_weight_min": -43.68059539794922,
      "activations/layer18_attention_weight_max": 44.90435028076172,
      "activations/layer18_attention_weight_min": -40.45566940307617,
      "activations/layer19_attention_weight_max": 24.93280792236328,
      "activations/layer19_attention_weight_min": -22.224979400634766,
      "activations/layer1_attention_weight_max": 15.690176963806152,
      "activations/layer1_attention_weight_min": -14.292037963867188,
      "activations/layer20_attention_weight_max": 22.257463455200195,
      "activations/layer20_attention_weight_min": -20.476011276245117,
      "activations/layer21_attention_weight_max": 34.54586410522461,
      "activations/layer21_attention_weight_min": -23.424896240234375,
      "activations/layer22_attention_weight_max": 31.157941818237305,
      "activations/layer22_attention_weight_min": -25.770811080932617,
      "activations/layer23_attention_weight_max": 37.365848541259766,
      "activations/layer23_attention_weight_min": -24.271102905273438,
      "activations/layer2_attention_weight_max": 31.786998748779297,
      "activations/layer2_attention_weight_min": -28.17798614501953,
      "activations/layer3_attention_weight_max": 86.87755584716797,
      "activations/layer3_attention_weight_min": -86.28215789794922,
      "activations/layer4_attention_weight_max": 88.2138900756836,
      "activations/layer4_attention_weight_min": -87.15497589111328,
      "activations/layer5_attention_weight_max": 60.60549545288086,
      "activations/layer5_attention_weight_min": -74.63361358642578,
      "activations/layer6_attention_weight_max": 49.71726989746094,
      "activations/layer6_attention_weight_min": -47.972801208496094,
      "activations/layer7_attention_weight_max": 63.15846633911133,
      "activations/layer7_attention_weight_min": -61.13210678100586,
      "activations/layer8_attention_weight_max": 46.39641571044922,
      "activations/layer8_attention_weight_min": -48.5332145690918,
      "activations/layer9_attention_weight_max": 43.84123229980469,
      "activations/layer9_attention_weight_min": -42.6692008972168,
      "epoch": 17.77,
      "learning_rate": 3.5775378787878785e-05,
      "loss": 2.7298,
      "step": 305800
    },
    {
      "activations/layer0_attention_weight_max": 16.9255428314209,
      "activations/layer0_attention_weight_min": -13.653508186340332,
      "activations/layer10_attention_weight_max": 37.615718841552734,
      "activations/layer10_attention_weight_min": -36.62842559814453,
      "activations/layer11_attention_weight_max": 34.19866943359375,
      "activations/layer11_attention_weight_min": -34.73082733154297,
      "activations/layer12_attention_weight_max": 20.410369873046875,
      "activations/layer12_attention_weight_min": -24.315305709838867,
      "activations/layer13_attention_weight_max": 40.40681838989258,
      "activations/layer13_attention_weight_min": -26.97004508972168,
      "activations/layer14_attention_weight_max": 37.5328254699707,
      "activations/layer14_attention_weight_min": -30.45001983642578,
      "activations/layer15_attention_weight_max": 35.76386260986328,
      "activations/layer15_attention_weight_min": -31.477338790893555,
      "activations/layer16_attention_weight_max": 36.50151824951172,
      "activations/layer16_attention_weight_min": -31.59290313720703,
      "activations/layer17_attention_weight_max": 53.709014892578125,
      "activations/layer17_attention_weight_min": -44.52267837524414,
      "activations/layer18_attention_weight_max": 49.03118133544922,
      "activations/layer18_attention_weight_min": -40.106327056884766,
      "activations/layer19_attention_weight_max": 25.223060607910156,
      "activations/layer19_attention_weight_min": -22.657791137695312,
      "activations/layer1_attention_weight_max": 17.37807273864746,
      "activations/layer1_attention_weight_min": -16.08553123474121,
      "activations/layer20_attention_weight_max": 27.5744571685791,
      "activations/layer20_attention_weight_min": -22.324687957763672,
      "activations/layer21_attention_weight_max": 45.33662796020508,
      "activations/layer21_attention_weight_min": -23.90586280822754,
      "activations/layer22_attention_weight_max": 31.03189468383789,
      "activations/layer22_attention_weight_min": -25.66519546508789,
      "activations/layer23_attention_weight_max": 38.86646270751953,
      "activations/layer23_attention_weight_min": -23.742061614990234,
      "activations/layer2_attention_weight_max": 29.73987579345703,
      "activations/layer2_attention_weight_min": -28.417720794677734,
      "activations/layer3_attention_weight_max": 88.71808624267578,
      "activations/layer3_attention_weight_min": -90.68023681640625,
      "activations/layer4_attention_weight_max": 88.41844177246094,
      "activations/layer4_attention_weight_min": -89.02845764160156,
      "activations/layer5_attention_weight_max": 62.99070358276367,
      "activations/layer5_attention_weight_min": -74.60196685791016,
      "activations/layer6_attention_weight_max": 50.467918395996094,
      "activations/layer6_attention_weight_min": -50.477027893066406,
      "activations/layer7_attention_weight_max": 59.53828811645508,
      "activations/layer7_attention_weight_min": -65.39727783203125,
      "activations/layer8_attention_weight_max": 46.733367919921875,
      "activations/layer8_attention_weight_min": -49.51420211791992,
      "activations/layer9_attention_weight_max": 43.66728973388672,
      "activations/layer9_attention_weight_min": -48.033042907714844,
      "epoch": 17.77,
      "learning_rate": 3.575643939393939e-05,
      "loss": 2.7416,
      "step": 305850
    },
    {
      "activations/layer0_attention_weight_max": 15.377703666687012,
      "activations/layer0_attention_weight_min": -13.896843910217285,
      "activations/layer10_attention_weight_max": 35.534542083740234,
      "activations/layer10_attention_weight_min": -34.136131286621094,
      "activations/layer11_attention_weight_max": 32.34414291381836,
      "activations/layer11_attention_weight_min": -31.992835998535156,
      "activations/layer12_attention_weight_max": 18.440109252929688,
      "activations/layer12_attention_weight_min": -27.8209285736084,
      "activations/layer13_attention_weight_max": 37.92190170288086,
      "activations/layer13_attention_weight_min": -26.61042594909668,
      "activations/layer14_attention_weight_max": 34.51539993286133,
      "activations/layer14_attention_weight_min": -28.1098690032959,
      "activations/layer15_attention_weight_max": 31.126405715942383,
      "activations/layer15_attention_weight_min": -28.685028076171875,
      "activations/layer16_attention_weight_max": 31.435298919677734,
      "activations/layer16_attention_weight_min": -29.258480072021484,
      "activations/layer17_attention_weight_max": 50.128318786621094,
      "activations/layer17_attention_weight_min": -42.65253829956055,
      "activations/layer18_attention_weight_max": 44.41157150268555,
      "activations/layer18_attention_weight_min": -39.24906539916992,
      "activations/layer19_attention_weight_max": 23.535276412963867,
      "activations/layer19_attention_weight_min": -23.51654624938965,
      "activations/layer1_attention_weight_max": 17.46060562133789,
      "activations/layer1_attention_weight_min": -15.170708656311035,
      "activations/layer20_attention_weight_max": 24.297704696655273,
      "activations/layer20_attention_weight_min": -20.27579116821289,
      "activations/layer21_attention_weight_max": 36.7921142578125,
      "activations/layer21_attention_weight_min": -25.185081481933594,
      "activations/layer22_attention_weight_max": 27.723283767700195,
      "activations/layer22_attention_weight_min": -25.879257202148438,
      "activations/layer23_attention_weight_max": 37.939483642578125,
      "activations/layer23_attention_weight_min": -24.388181686401367,
      "activations/layer2_attention_weight_max": 30.223297119140625,
      "activations/layer2_attention_weight_min": -28.433639526367188,
      "activations/layer3_attention_weight_max": 89.40557861328125,
      "activations/layer3_attention_weight_min": -91.57686614990234,
      "activations/layer4_attention_weight_max": 89.88220977783203,
      "activations/layer4_attention_weight_min": -86.22364044189453,
      "activations/layer5_attention_weight_max": 60.67039108276367,
      "activations/layer5_attention_weight_min": -68.38277435302734,
      "activations/layer6_attention_weight_max": 47.52861785888672,
      "activations/layer6_attention_weight_min": -48.55315017700195,
      "activations/layer7_attention_weight_max": 61.21617889404297,
      "activations/layer7_attention_weight_min": -63.685272216796875,
      "activations/layer8_attention_weight_max": 44.05729293823242,
      "activations/layer8_attention_weight_min": -46.97361373901367,
      "activations/layer9_attention_weight_max": 43.20830154418945,
      "activations/layer9_attention_weight_min": -44.18661117553711,
      "epoch": 17.77,
      "learning_rate": 3.5737499999999995e-05,
      "loss": 2.7465,
      "step": 305900
    },
    {
      "activations/layer0_attention_weight_max": 15.402616500854492,
      "activations/layer0_attention_weight_min": -13.91378116607666,
      "activations/layer10_attention_weight_max": 36.494380950927734,
      "activations/layer10_attention_weight_min": -36.24183654785156,
      "activations/layer11_attention_weight_max": 32.735633850097656,
      "activations/layer11_attention_weight_min": -34.181800842285156,
      "activations/layer12_attention_weight_max": 18.902841567993164,
      "activations/layer12_attention_weight_min": -26.015689849853516,
      "activations/layer13_attention_weight_max": 33.878021240234375,
      "activations/layer13_attention_weight_min": -28.129474639892578,
      "activations/layer14_attention_weight_max": 35.22069549560547,
      "activations/layer14_attention_weight_min": -30.62250328063965,
      "activations/layer15_attention_weight_max": 33.591217041015625,
      "activations/layer15_attention_weight_min": -30.667312622070312,
      "activations/layer16_attention_weight_max": 33.35927963256836,
      "activations/layer16_attention_weight_min": -30.600345611572266,
      "activations/layer17_attention_weight_max": 51.03160095214844,
      "activations/layer17_attention_weight_min": -42.92558670043945,
      "activations/layer18_attention_weight_max": 46.16373825073242,
      "activations/layer18_attention_weight_min": -39.999176025390625,
      "activations/layer19_attention_weight_max": 24.409910202026367,
      "activations/layer19_attention_weight_min": -22.97328758239746,
      "activations/layer1_attention_weight_max": 15.88090991973877,
      "activations/layer1_attention_weight_min": -14.04440689086914,
      "activations/layer20_attention_weight_max": 24.74779510498047,
      "activations/layer20_attention_weight_min": -21.148862838745117,
      "activations/layer21_attention_weight_max": 36.777061462402344,
      "activations/layer21_attention_weight_min": -21.966815948486328,
      "activations/layer22_attention_weight_max": 31.231740951538086,
      "activations/layer22_attention_weight_min": -27.511613845825195,
      "activations/layer23_attention_weight_max": 36.773704528808594,
      "activations/layer23_attention_weight_min": -24.091825485229492,
      "activations/layer2_attention_weight_max": 32.16287612915039,
      "activations/layer2_attention_weight_min": -29.711217880249023,
      "activations/layer3_attention_weight_max": 92.77557373046875,
      "activations/layer3_attention_weight_min": -94.04525756835938,
      "activations/layer4_attention_weight_max": 88.23928833007812,
      "activations/layer4_attention_weight_min": -85.98833465576172,
      "activations/layer5_attention_weight_max": 60.67011642456055,
      "activations/layer5_attention_weight_min": -78.38127136230469,
      "activations/layer6_attention_weight_max": 49.28954315185547,
      "activations/layer6_attention_weight_min": -50.22840118408203,
      "activations/layer7_attention_weight_max": 64.34561920166016,
      "activations/layer7_attention_weight_min": -60.01188278198242,
      "activations/layer8_attention_weight_max": 44.96981430053711,
      "activations/layer8_attention_weight_min": -49.144920349121094,
      "activations/layer9_attention_weight_max": 42.85569381713867,
      "activations/layer9_attention_weight_min": -45.30890655517578,
      "epoch": 17.78,
      "learning_rate": 3.57185606060606e-05,
      "loss": 2.7388,
      "step": 305950
    },
    {
      "activations/layer0_attention_weight_max": 15.312288284301758,
      "activations/layer0_attention_weight_min": -13.52408218383789,
      "activations/layer10_attention_weight_max": 35.24502944946289,
      "activations/layer10_attention_weight_min": -34.11927032470703,
      "activations/layer11_attention_weight_max": 31.84223175048828,
      "activations/layer11_attention_weight_min": -34.045494079589844,
      "activations/layer12_attention_weight_max": 19.296171188354492,
      "activations/layer12_attention_weight_min": -27.422727584838867,
      "activations/layer13_attention_weight_max": 30.756515502929688,
      "activations/layer13_attention_weight_min": -26.24428367614746,
      "activations/layer14_attention_weight_max": 33.241146087646484,
      "activations/layer14_attention_weight_min": -29.52924919128418,
      "activations/layer15_attention_weight_max": 32.081443786621094,
      "activations/layer15_attention_weight_min": -30.02144432067871,
      "activations/layer16_attention_weight_max": 32.310508728027344,
      "activations/layer16_attention_weight_min": -32.23981475830078,
      "activations/layer17_attention_weight_max": 54.061805725097656,
      "activations/layer17_attention_weight_min": -46.506614685058594,
      "activations/layer18_attention_weight_max": 47.23941421508789,
      "activations/layer18_attention_weight_min": -43.88019943237305,
      "activations/layer19_attention_weight_max": 26.399728775024414,
      "activations/layer19_attention_weight_min": -24.484426498413086,
      "activations/layer1_attention_weight_max": 15.706033706665039,
      "activations/layer1_attention_weight_min": -14.726192474365234,
      "activations/layer20_attention_weight_max": 24.21491050720215,
      "activations/layer20_attention_weight_min": -24.547550201416016,
      "activations/layer21_attention_weight_max": 29.757707595825195,
      "activations/layer21_attention_weight_min": -20.173276901245117,
      "activations/layer22_attention_weight_max": 32.54117202758789,
      "activations/layer22_attention_weight_min": -27.16109275817871,
      "activations/layer23_attention_weight_max": 36.21754837036133,
      "activations/layer23_attention_weight_min": -25.495487213134766,
      "activations/layer2_attention_weight_max": 33.52455520629883,
      "activations/layer2_attention_weight_min": -29.923917770385742,
      "activations/layer3_attention_weight_max": 93.60916900634766,
      "activations/layer3_attention_weight_min": -97.47611236572266,
      "activations/layer4_attention_weight_max": 87.80857849121094,
      "activations/layer4_attention_weight_min": -88.51900482177734,
      "activations/layer5_attention_weight_max": 63.62977600097656,
      "activations/layer5_attention_weight_min": -73.58628845214844,
      "activations/layer6_attention_weight_max": 50.860469818115234,
      "activations/layer6_attention_weight_min": -51.0142707824707,
      "activations/layer7_attention_weight_max": 63.03820037841797,
      "activations/layer7_attention_weight_min": -63.500709533691406,
      "activations/layer8_attention_weight_max": 47.90386962890625,
      "activations/layer8_attention_weight_min": -52.13774490356445,
      "activations/layer9_attention_weight_max": 43.64606857299805,
      "activations/layer9_attention_weight_min": -46.977699279785156,
      "epoch": 17.78,
      "learning_rate": 3.5699621212121204e-05,
      "loss": 2.7408,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.454,
      "eval_samples_per_second": 507.926,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.454,
      "eval_openwebtext_samples_per_second": 507.926,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9479,
      "eval_wikitext_samples_per_second": 234.098,
      "step": 306000
    },
    {
      "epoch": 17.78,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.5369,
      "eval_lambada_samples_per_second": 510.543,
      "step": 306000
    },
    {
      "activations/layer0_attention_weight_max": 16.044706344604492,
      "activations/layer0_attention_weight_min": -13.871487617492676,
      "activations/layer10_attention_weight_max": 35.88167953491211,
      "activations/layer10_attention_weight_min": -37.03450393676758,
      "activations/layer11_attention_weight_max": 33.31695556640625,
      "activations/layer11_attention_weight_min": -33.8524284362793,
      "activations/layer12_attention_weight_max": 19.913400650024414,
      "activations/layer12_attention_weight_min": -25.286396026611328,
      "activations/layer13_attention_weight_max": 37.39122772216797,
      "activations/layer13_attention_weight_min": -30.422000885009766,
      "activations/layer14_attention_weight_max": 35.71589279174805,
      "activations/layer14_attention_weight_min": -29.661104202270508,
      "activations/layer15_attention_weight_max": 33.25777053833008,
      "activations/layer15_attention_weight_min": -29.177047729492188,
      "activations/layer16_attention_weight_max": 36.041908264160156,
      "activations/layer16_attention_weight_min": -32.2696533203125,
      "activations/layer17_attention_weight_max": 56.16139602661133,
      "activations/layer17_attention_weight_min": -47.20699691772461,
      "activations/layer18_attention_weight_max": 50.29623031616211,
      "activations/layer18_attention_weight_min": -40.928279876708984,
      "activations/layer19_attention_weight_max": 24.350141525268555,
      "activations/layer19_attention_weight_min": -23.010961532592773,
      "activations/layer1_attention_weight_max": 15.810638427734375,
      "activations/layer1_attention_weight_min": -13.036288261413574,
      "activations/layer20_attention_weight_max": 25.952564239501953,
      "activations/layer20_attention_weight_min": -20.983182907104492,
      "activations/layer21_attention_weight_max": 34.2369270324707,
      "activations/layer21_attention_weight_min": -23.868492126464844,
      "activations/layer22_attention_weight_max": 31.523170471191406,
      "activations/layer22_attention_weight_min": -25.921037673950195,
      "activations/layer23_attention_weight_max": 39.55963134765625,
      "activations/layer23_attention_weight_min": -25.994163513183594,
      "activations/layer2_attention_weight_max": 30.948558807373047,
      "activations/layer2_attention_weight_min": -29.813167572021484,
      "activations/layer3_attention_weight_max": 91.79175567626953,
      "activations/layer3_attention_weight_min": -91.08769989013672,
      "activations/layer4_attention_weight_max": 87.78176879882812,
      "activations/layer4_attention_weight_min": -90.2057113647461,
      "activations/layer5_attention_weight_max": 64.05076599121094,
      "activations/layer5_attention_weight_min": -73.17340850830078,
      "activations/layer6_attention_weight_max": 49.62425994873047,
      "activations/layer6_attention_weight_min": -53.36435317993164,
      "activations/layer7_attention_weight_max": 60.22879409790039,
      "activations/layer7_attention_weight_min": -63.991329193115234,
      "activations/layer8_attention_weight_max": 46.200538635253906,
      "activations/layer8_attention_weight_min": -52.141815185546875,
      "activations/layer9_attention_weight_max": 43.84702682495117,
      "activations/layer9_attention_weight_min": -46.97726058959961,
      "epoch": 17.78,
      "learning_rate": 3.568068181818181e-05,
      "loss": 2.739,
      "step": 306050
    },
    {
      "activations/layer0_attention_weight_max": 14.884306907653809,
      "activations/layer0_attention_weight_min": -13.565752983093262,
      "activations/layer10_attention_weight_max": 33.9427490234375,
      "activations/layer10_attention_weight_min": -36.34670639038086,
      "activations/layer11_attention_weight_max": 32.21626281738281,
      "activations/layer11_attention_weight_min": -34.19846725463867,
      "activations/layer12_attention_weight_max": 19.588973999023438,
      "activations/layer12_attention_weight_min": -27.388568878173828,
      "activations/layer13_attention_weight_max": 32.28437042236328,
      "activations/layer13_attention_weight_min": -26.93511199951172,
      "activations/layer14_attention_weight_max": 36.435882568359375,
      "activations/layer14_attention_weight_min": -30.01789093017578,
      "activations/layer15_attention_weight_max": 30.688949584960938,
      "activations/layer15_attention_weight_min": -29.908428192138672,
      "activations/layer16_attention_weight_max": 32.780513763427734,
      "activations/layer16_attention_weight_min": -30.779634475708008,
      "activations/layer17_attention_weight_max": 49.6226921081543,
      "activations/layer17_attention_weight_min": -44.30925750732422,
      "activations/layer18_attention_weight_max": 46.41697692871094,
      "activations/layer18_attention_weight_min": -38.254417419433594,
      "activations/layer19_attention_weight_max": 25.259897232055664,
      "activations/layer19_attention_weight_min": -22.130971908569336,
      "activations/layer1_attention_weight_max": 15.891100883483887,
      "activations/layer1_attention_weight_min": -14.0015230178833,
      "activations/layer20_attention_weight_max": 22.832378387451172,
      "activations/layer20_attention_weight_min": -20.25289535522461,
      "activations/layer21_attention_weight_max": 35.400733947753906,
      "activations/layer21_attention_weight_min": -21.241138458251953,
      "activations/layer22_attention_weight_max": 32.08021545410156,
      "activations/layer22_attention_weight_min": -24.1958065032959,
      "activations/layer23_attention_weight_max": 33.3677978515625,
      "activations/layer23_attention_weight_min": -24.570234298706055,
      "activations/layer2_attention_weight_max": 30.740604400634766,
      "activations/layer2_attention_weight_min": -29.298812866210938,
      "activations/layer3_attention_weight_max": 87.1763916015625,
      "activations/layer3_attention_weight_min": -91.49727630615234,
      "activations/layer4_attention_weight_max": 87.59480285644531,
      "activations/layer4_attention_weight_min": -89.84434509277344,
      "activations/layer5_attention_weight_max": 64.14299774169922,
      "activations/layer5_attention_weight_min": -73.82333374023438,
      "activations/layer6_attention_weight_max": 48.534080505371094,
      "activations/layer6_attention_weight_min": -50.743690490722656,
      "activations/layer7_attention_weight_max": 63.810028076171875,
      "activations/layer7_attention_weight_min": -62.20011901855469,
      "activations/layer8_attention_weight_max": 47.86042022705078,
      "activations/layer8_attention_weight_min": -49.76796340942383,
      "activations/layer9_attention_weight_max": 42.64435958862305,
      "activations/layer9_attention_weight_min": -46.94203186035156,
      "epoch": 17.79,
      "learning_rate": 3.566174242424242e-05,
      "loss": 2.7337,
      "step": 306100
    },
    {
      "activations/layer0_attention_weight_max": 16.086584091186523,
      "activations/layer0_attention_weight_min": -13.533278465270996,
      "activations/layer10_attention_weight_max": 35.78120803833008,
      "activations/layer10_attention_weight_min": -36.824527740478516,
      "activations/layer11_attention_weight_max": 34.70960235595703,
      "activations/layer11_attention_weight_min": -35.506595611572266,
      "activations/layer12_attention_weight_max": 19.97712516784668,
      "activations/layer12_attention_weight_min": -25.057106018066406,
      "activations/layer13_attention_weight_max": 39.79200744628906,
      "activations/layer13_attention_weight_min": -28.667333602905273,
      "activations/layer14_attention_weight_max": 35.9131965637207,
      "activations/layer14_attention_weight_min": -32.053226470947266,
      "activations/layer15_attention_weight_max": 36.696712493896484,
      "activations/layer15_attention_weight_min": -33.22373580932617,
      "activations/layer16_attention_weight_max": 37.40800857543945,
      "activations/layer16_attention_weight_min": -37.381004333496094,
      "activations/layer17_attention_weight_max": 52.550655364990234,
      "activations/layer17_attention_weight_min": -47.6535530090332,
      "activations/layer18_attention_weight_max": 45.708553314208984,
      "activations/layer18_attention_weight_min": -43.4129753112793,
      "activations/layer19_attention_weight_max": 23.438005447387695,
      "activations/layer19_attention_weight_min": -24.446613311767578,
      "activations/layer1_attention_weight_max": 16.02427864074707,
      "activations/layer1_attention_weight_min": -13.978784561157227,
      "activations/layer20_attention_weight_max": 23.341793060302734,
      "activations/layer20_attention_weight_min": -21.05154800415039,
      "activations/layer21_attention_weight_max": 37.3054084777832,
      "activations/layer21_attention_weight_min": -22.2243709564209,
      "activations/layer22_attention_weight_max": 29.17725372314453,
      "activations/layer22_attention_weight_min": -26.749494552612305,
      "activations/layer23_attention_weight_max": 35.578369140625,
      "activations/layer23_attention_weight_min": -22.307113647460938,
      "activations/layer2_attention_weight_max": 32.1875,
      "activations/layer2_attention_weight_min": -29.461849212646484,
      "activations/layer3_attention_weight_max": 92.49447631835938,
      "activations/layer3_attention_weight_min": -91.826416015625,
      "activations/layer4_attention_weight_max": 93.67512512207031,
      "activations/layer4_attention_weight_min": -89.77460479736328,
      "activations/layer5_attention_weight_max": 64.79612731933594,
      "activations/layer5_attention_weight_min": -73.22186279296875,
      "activations/layer6_attention_weight_max": 51.645328521728516,
      "activations/layer6_attention_weight_min": -50.557437896728516,
      "activations/layer7_attention_weight_max": 65.31319427490234,
      "activations/layer7_attention_weight_min": -62.95613479614258,
      "activations/layer8_attention_weight_max": 50.25034713745117,
      "activations/layer8_attention_weight_min": -48.55857849121094,
      "activations/layer9_attention_weight_max": 50.8718376159668,
      "activations/layer9_attention_weight_min": -46.646209716796875,
      "epoch": 17.79,
      "learning_rate": 3.564280303030303e-05,
      "loss": 2.7394,
      "step": 306150
    },
    {
      "activations/layer0_attention_weight_max": 15.364093780517578,
      "activations/layer0_attention_weight_min": -13.542094230651855,
      "activations/layer10_attention_weight_max": 34.41706085205078,
      "activations/layer10_attention_weight_min": -34.57147979736328,
      "activations/layer11_attention_weight_max": 32.47388458251953,
      "activations/layer11_attention_weight_min": -33.25746154785156,
      "activations/layer12_attention_weight_max": 18.546192169189453,
      "activations/layer12_attention_weight_min": -26.850624084472656,
      "activations/layer13_attention_weight_max": 35.86466979980469,
      "activations/layer13_attention_weight_min": -29.71300506591797,
      "activations/layer14_attention_weight_max": 35.83999252319336,
      "activations/layer14_attention_weight_min": -31.044979095458984,
      "activations/layer15_attention_weight_max": 34.20159149169922,
      "activations/layer15_attention_weight_min": -31.626829147338867,
      "activations/layer16_attention_weight_max": 34.28536605834961,
      "activations/layer16_attention_weight_min": -31.32212257385254,
      "activations/layer17_attention_weight_max": 53.927059173583984,
      "activations/layer17_attention_weight_min": -45.94200897216797,
      "activations/layer18_attention_weight_max": 48.269229888916016,
      "activations/layer18_attention_weight_min": -38.2708740234375,
      "activations/layer19_attention_weight_max": 22.470165252685547,
      "activations/layer19_attention_weight_min": -22.05238914489746,
      "activations/layer1_attention_weight_max": 16.363338470458984,
      "activations/layer1_attention_weight_min": -13.704095840454102,
      "activations/layer20_attention_weight_max": 24.195096969604492,
      "activations/layer20_attention_weight_min": -22.021709442138672,
      "activations/layer21_attention_weight_max": 45.977298736572266,
      "activations/layer21_attention_weight_min": -24.332805633544922,
      "activations/layer22_attention_weight_max": 29.25255584716797,
      "activations/layer22_attention_weight_min": -25.450965881347656,
      "activations/layer23_attention_weight_max": 36.063411712646484,
      "activations/layer23_attention_weight_min": -25.44202423095703,
      "activations/layer2_attention_weight_max": 30.742938995361328,
      "activations/layer2_attention_weight_min": -30.053556442260742,
      "activations/layer3_attention_weight_max": 84.32029724121094,
      "activations/layer3_attention_weight_min": -84.9910888671875,
      "activations/layer4_attention_weight_max": 84.3490982055664,
      "activations/layer4_attention_weight_min": -79.53046417236328,
      "activations/layer5_attention_weight_max": 60.58314514160156,
      "activations/layer5_attention_weight_min": -69.2507095336914,
      "activations/layer6_attention_weight_max": 47.52956771850586,
      "activations/layer6_attention_weight_min": -50.352577209472656,
      "activations/layer7_attention_weight_max": 63.37909698486328,
      "activations/layer7_attention_weight_min": -61.99378967285156,
      "activations/layer8_attention_weight_max": 46.60660171508789,
      "activations/layer8_attention_weight_min": -49.47708511352539,
      "activations/layer9_attention_weight_max": 42.22077560424805,
      "activations/layer9_attention_weight_min": -46.55717849731445,
      "epoch": 17.79,
      "learning_rate": 3.562386363636364e-05,
      "loss": 2.7419,
      "step": 306200
    },
    {
      "activations/layer0_attention_weight_max": 15.69233512878418,
      "activations/layer0_attention_weight_min": -13.511614799499512,
      "activations/layer10_attention_weight_max": 34.471797943115234,
      "activations/layer10_attention_weight_min": -33.41993713378906,
      "activations/layer11_attention_weight_max": 30.047332763671875,
      "activations/layer11_attention_weight_min": -32.8895263671875,
      "activations/layer12_attention_weight_max": 18.249332427978516,
      "activations/layer12_attention_weight_min": -24.587867736816406,
      "activations/layer13_attention_weight_max": 29.867359161376953,
      "activations/layer13_attention_weight_min": -27.560874938964844,
      "activations/layer14_attention_weight_max": 35.068519592285156,
      "activations/layer14_attention_weight_min": -29.97367286682129,
      "activations/layer15_attention_weight_max": 32.10148620605469,
      "activations/layer15_attention_weight_min": -30.490482330322266,
      "activations/layer16_attention_weight_max": 32.02341079711914,
      "activations/layer16_attention_weight_min": -30.273759841918945,
      "activations/layer17_attention_weight_max": 53.05541229248047,
      "activations/layer17_attention_weight_min": -42.078163146972656,
      "activations/layer18_attention_weight_max": 46.96309280395508,
      "activations/layer18_attention_weight_min": -36.5540885925293,
      "activations/layer19_attention_weight_max": 20.59871482849121,
      "activations/layer19_attention_weight_min": -20.442092895507812,
      "activations/layer1_attention_weight_max": 16.804128646850586,
      "activations/layer1_attention_weight_min": -14.262328147888184,
      "activations/layer20_attention_weight_max": 21.542871475219727,
      "activations/layer20_attention_weight_min": -20.051097869873047,
      "activations/layer21_attention_weight_max": 29.10544204711914,
      "activations/layer21_attention_weight_min": -20.84671401977539,
      "activations/layer22_attention_weight_max": 26.306848526000977,
      "activations/layer22_attention_weight_min": -23.98895835876465,
      "activations/layer23_attention_weight_max": 34.560115814208984,
      "activations/layer23_attention_weight_min": -23.26831817626953,
      "activations/layer2_attention_weight_max": 29.820941925048828,
      "activations/layer2_attention_weight_min": -27.55415916442871,
      "activations/layer3_attention_weight_max": 88.45988464355469,
      "activations/layer3_attention_weight_min": -85.67094421386719,
      "activations/layer4_attention_weight_max": 83.40584564208984,
      "activations/layer4_attention_weight_min": -85.0048599243164,
      "activations/layer5_attention_weight_max": 63.442626953125,
      "activations/layer5_attention_weight_min": -70.2896728515625,
      "activations/layer6_attention_weight_max": 46.51128005981445,
      "activations/layer6_attention_weight_min": -47.61904525756836,
      "activations/layer7_attention_weight_max": 63.7657585144043,
      "activations/layer7_attention_weight_min": -57.92889404296875,
      "activations/layer8_attention_weight_max": 41.993560791015625,
      "activations/layer8_attention_weight_min": -45.301151275634766,
      "activations/layer9_attention_weight_max": 41.05302429199219,
      "activations/layer9_attention_weight_min": -42.54370880126953,
      "epoch": 17.79,
      "learning_rate": 3.560492424242424e-05,
      "loss": 2.7611,
      "step": 306250
    },
    {
      "activations/layer0_attention_weight_max": 15.856145858764648,
      "activations/layer0_attention_weight_min": -13.490917205810547,
      "activations/layer10_attention_weight_max": 31.385406494140625,
      "activations/layer10_attention_weight_min": -35.65724563598633,
      "activations/layer11_attention_weight_max": 29.06366729736328,
      "activations/layer11_attention_weight_min": -34.58604049682617,
      "activations/layer12_attention_weight_max": 20.136672973632812,
      "activations/layer12_attention_weight_min": -27.69447135925293,
      "activations/layer13_attention_weight_max": 34.49022674560547,
      "activations/layer13_attention_weight_min": -27.00718116760254,
      "activations/layer14_attention_weight_max": 37.000308990478516,
      "activations/layer14_attention_weight_min": -30.55326271057129,
      "activations/layer15_attention_weight_max": 35.50917053222656,
      "activations/layer15_attention_weight_min": -31.996967315673828,
      "activations/layer16_attention_weight_max": 37.601600646972656,
      "activations/layer16_attention_weight_min": -35.376285552978516,
      "activations/layer17_attention_weight_max": 57.24571990966797,
      "activations/layer17_attention_weight_min": -51.154415130615234,
      "activations/layer18_attention_weight_max": 54.14682388305664,
      "activations/layer18_attention_weight_min": -44.77412033081055,
      "activations/layer19_attention_weight_max": 27.939542770385742,
      "activations/layer19_attention_weight_min": -26.014541625976562,
      "activations/layer1_attention_weight_max": 16.306123733520508,
      "activations/layer1_attention_weight_min": -15.397385597229004,
      "activations/layer20_attention_weight_max": 28.556705474853516,
      "activations/layer20_attention_weight_min": -22.40418243408203,
      "activations/layer21_attention_weight_max": 42.59812545776367,
      "activations/layer21_attention_weight_min": -25.4578914642334,
      "activations/layer22_attention_weight_max": 32.0329475402832,
      "activations/layer22_attention_weight_min": -28.22921371459961,
      "activations/layer23_attention_weight_max": 40.84886169433594,
      "activations/layer23_attention_weight_min": -23.74652671813965,
      "activations/layer2_attention_weight_max": 27.08455467224121,
      "activations/layer2_attention_weight_min": -26.183963775634766,
      "activations/layer3_attention_weight_max": 79.30610656738281,
      "activations/layer3_attention_weight_min": -88.1673812866211,
      "activations/layer4_attention_weight_max": 78.57830047607422,
      "activations/layer4_attention_weight_min": -83.02859497070312,
      "activations/layer5_attention_weight_max": 56.761287689208984,
      "activations/layer5_attention_weight_min": -68.99597930908203,
      "activations/layer6_attention_weight_max": 43.80606460571289,
      "activations/layer6_attention_weight_min": -46.08510971069336,
      "activations/layer7_attention_weight_max": 56.82936477661133,
      "activations/layer7_attention_weight_min": -57.00947952270508,
      "activations/layer8_attention_weight_max": 39.24409866333008,
      "activations/layer8_attention_weight_min": -45.2723503112793,
      "activations/layer9_attention_weight_max": 40.84487533569336,
      "activations/layer9_attention_weight_min": -42.18843078613281,
      "epoch": 17.8,
      "learning_rate": 3.558598484848485e-05,
      "loss": 2.73,
      "step": 306300
    },
    {
      "activations/layer0_attention_weight_max": 15.590668678283691,
      "activations/layer0_attention_weight_min": -13.19378662109375,
      "activations/layer10_attention_weight_max": 35.05889892578125,
      "activations/layer10_attention_weight_min": -35.508514404296875,
      "activations/layer11_attention_weight_max": 32.13834762573242,
      "activations/layer11_attention_weight_min": -34.39571762084961,
      "activations/layer12_attention_weight_max": 20.599411010742188,
      "activations/layer12_attention_weight_min": -24.002531051635742,
      "activations/layer13_attention_weight_max": 31.707908630371094,
      "activations/layer13_attention_weight_min": -25.420412063598633,
      "activations/layer14_attention_weight_max": 35.89024353027344,
      "activations/layer14_attention_weight_min": -30.49107551574707,
      "activations/layer15_attention_weight_max": 34.431087493896484,
      "activations/layer15_attention_weight_min": -30.077287673950195,
      "activations/layer16_attention_weight_max": 33.498130798339844,
      "activations/layer16_attention_weight_min": -31.495771408081055,
      "activations/layer17_attention_weight_max": 51.50515365600586,
      "activations/layer17_attention_weight_min": -44.07319259643555,
      "activations/layer18_attention_weight_max": 46.02189636230469,
      "activations/layer18_attention_weight_min": -39.91482925415039,
      "activations/layer19_attention_weight_max": 23.057559967041016,
      "activations/layer19_attention_weight_min": -22.03248405456543,
      "activations/layer1_attention_weight_max": 16.343509674072266,
      "activations/layer1_attention_weight_min": -12.580079078674316,
      "activations/layer20_attention_weight_max": 19.91393280029297,
      "activations/layer20_attention_weight_min": -22.941347122192383,
      "activations/layer21_attention_weight_max": 35.23054885864258,
      "activations/layer21_attention_weight_min": -22.420135498046875,
      "activations/layer22_attention_weight_max": 29.736541748046875,
      "activations/layer22_attention_weight_min": -24.712196350097656,
      "activations/layer23_attention_weight_max": 39.7607421875,
      "activations/layer23_attention_weight_min": -23.428829193115234,
      "activations/layer2_attention_weight_max": 29.595672607421875,
      "activations/layer2_attention_weight_min": -28.875099182128906,
      "activations/layer3_attention_weight_max": 85.46697235107422,
      "activations/layer3_attention_weight_min": -91.56069946289062,
      "activations/layer4_attention_weight_max": 85.90587615966797,
      "activations/layer4_attention_weight_min": -83.70684814453125,
      "activations/layer5_attention_weight_max": 60.48216247558594,
      "activations/layer5_attention_weight_min": -73.08869171142578,
      "activations/layer6_attention_weight_max": 50.908775329589844,
      "activations/layer6_attention_weight_min": -49.327144622802734,
      "activations/layer7_attention_weight_max": 60.74556350708008,
      "activations/layer7_attention_weight_min": -61.270206451416016,
      "activations/layer8_attention_weight_max": 47.49770736694336,
      "activations/layer8_attention_weight_min": -49.917442321777344,
      "activations/layer9_attention_weight_max": 41.6262321472168,
      "activations/layer9_attention_weight_min": -47.201324462890625,
      "epoch": 17.8,
      "learning_rate": 3.556704545454545e-05,
      "loss": 2.7539,
      "step": 306350
    },
    {
      "activations/layer0_attention_weight_max": 16.120397567749023,
      "activations/layer0_attention_weight_min": -13.152090072631836,
      "activations/layer10_attention_weight_max": 34.631229400634766,
      "activations/layer10_attention_weight_min": -35.38124465942383,
      "activations/layer11_attention_weight_max": 32.71892547607422,
      "activations/layer11_attention_weight_min": -36.46552276611328,
      "activations/layer12_attention_weight_max": 18.412818908691406,
      "activations/layer12_attention_weight_min": -25.64350700378418,
      "activations/layer13_attention_weight_max": 33.88048553466797,
      "activations/layer13_attention_weight_min": -25.93503189086914,
      "activations/layer14_attention_weight_max": 34.62908935546875,
      "activations/layer14_attention_weight_min": -29.012035369873047,
      "activations/layer15_attention_weight_max": 30.98352813720703,
      "activations/layer15_attention_weight_min": -30.085954666137695,
      "activations/layer16_attention_weight_max": 30.388837814331055,
      "activations/layer16_attention_weight_min": -29.118234634399414,
      "activations/layer17_attention_weight_max": 47.412471771240234,
      "activations/layer17_attention_weight_min": -42.771949768066406,
      "activations/layer18_attention_weight_max": 43.08667755126953,
      "activations/layer18_attention_weight_min": -39.0300407409668,
      "activations/layer19_attention_weight_max": 21.931427001953125,
      "activations/layer19_attention_weight_min": -23.86920166015625,
      "activations/layer1_attention_weight_max": 16.26531982421875,
      "activations/layer1_attention_weight_min": -13.163416862487793,
      "activations/layer20_attention_weight_max": 20.518360137939453,
      "activations/layer20_attention_weight_min": -23.79472541809082,
      "activations/layer21_attention_weight_max": 30.55257225036621,
      "activations/layer21_attention_weight_min": -20.78028106689453,
      "activations/layer22_attention_weight_max": 25.36735725402832,
      "activations/layer22_attention_weight_min": -25.1799373626709,
      "activations/layer23_attention_weight_max": 34.16123962402344,
      "activations/layer23_attention_weight_min": -25.105796813964844,
      "activations/layer2_attention_weight_max": 31.512386322021484,
      "activations/layer2_attention_weight_min": -31.060184478759766,
      "activations/layer3_attention_weight_max": 90.78562927246094,
      "activations/layer3_attention_weight_min": -100.15880584716797,
      "activations/layer4_attention_weight_max": 91.98368072509766,
      "activations/layer4_attention_weight_min": -88.986328125,
      "activations/layer5_attention_weight_max": 63.68926239013672,
      "activations/layer5_attention_weight_min": -74.36454010009766,
      "activations/layer6_attention_weight_max": 51.22932815551758,
      "activations/layer6_attention_weight_min": -51.38473892211914,
      "activations/layer7_attention_weight_max": 64.64076232910156,
      "activations/layer7_attention_weight_min": -63.62621307373047,
      "activations/layer8_attention_weight_max": 46.030418395996094,
      "activations/layer8_attention_weight_min": -51.09897232055664,
      "activations/layer9_attention_weight_max": 42.590885162353516,
      "activations/layer9_attention_weight_min": -47.2496337890625,
      "epoch": 17.8,
      "learning_rate": 3.5548863636363636e-05,
      "loss": 2.7417,
      "step": 306400
    },
    {
      "activations/layer0_attention_weight_max": 15.350855827331543,
      "activations/layer0_attention_weight_min": -14.221522331237793,
      "activations/layer10_attention_weight_max": 36.74995040893555,
      "activations/layer10_attention_weight_min": -36.764530181884766,
      "activations/layer11_attention_weight_max": 34.12607955932617,
      "activations/layer11_attention_weight_min": -34.97144317626953,
      "activations/layer12_attention_weight_max": 19.185251235961914,
      "activations/layer12_attention_weight_min": -23.586349487304688,
      "activations/layer13_attention_weight_max": 35.26447296142578,
      "activations/layer13_attention_weight_min": -27.80213165283203,
      "activations/layer14_attention_weight_max": 39.5760383605957,
      "activations/layer14_attention_weight_min": -30.891826629638672,
      "activations/layer15_attention_weight_max": 35.89413070678711,
      "activations/layer15_attention_weight_min": -30.81789779663086,
      "activations/layer16_attention_weight_max": 36.7216796875,
      "activations/layer16_attention_weight_min": -33.95382308959961,
      "activations/layer17_attention_weight_max": 56.556636810302734,
      "activations/layer17_attention_weight_min": -48.866668701171875,
      "activations/layer18_attention_weight_max": 53.49351119995117,
      "activations/layer18_attention_weight_min": -45.16628646850586,
      "activations/layer19_attention_weight_max": 27.102014541625977,
      "activations/layer19_attention_weight_min": -24.770109176635742,
      "activations/layer1_attention_weight_max": 15.413654327392578,
      "activations/layer1_attention_weight_min": -13.632220268249512,
      "activations/layer20_attention_weight_max": 26.176883697509766,
      "activations/layer20_attention_weight_min": -21.437742233276367,
      "activations/layer21_attention_weight_max": 42.076210021972656,
      "activations/layer21_attention_weight_min": -23.85173797607422,
      "activations/layer22_attention_weight_max": 34.71546936035156,
      "activations/layer22_attention_weight_min": -27.34445571899414,
      "activations/layer23_attention_weight_max": 37.477779388427734,
      "activations/layer23_attention_weight_min": -25.587120056152344,
      "activations/layer2_attention_weight_max": 30.882389068603516,
      "activations/layer2_attention_weight_min": -28.204708099365234,
      "activations/layer3_attention_weight_max": 85.68901824951172,
      "activations/layer3_attention_weight_min": -89.19226837158203,
      "activations/layer4_attention_weight_max": 85.86971282958984,
      "activations/layer4_attention_weight_min": -85.50020599365234,
      "activations/layer5_attention_weight_max": 64.80622863769531,
      "activations/layer5_attention_weight_min": -78.97386932373047,
      "activations/layer6_attention_weight_max": 49.707618713378906,
      "activations/layer6_attention_weight_min": -51.46363067626953,
      "activations/layer7_attention_weight_max": 64.88874816894531,
      "activations/layer7_attention_weight_min": -62.75925827026367,
      "activations/layer8_attention_weight_max": 45.91327667236328,
      "activations/layer8_attention_weight_min": -51.47087097167969,
      "activations/layer9_attention_weight_max": 45.282630920410156,
      "activations/layer9_attention_weight_min": -46.71931076049805,
      "epoch": 17.81,
      "learning_rate": 3.552992424242424e-05,
      "loss": 2.7434,
      "step": 306450
    },
    {
      "activations/layer0_attention_weight_max": 15.278677940368652,
      "activations/layer0_attention_weight_min": -13.129142761230469,
      "activations/layer10_attention_weight_max": 38.10490798950195,
      "activations/layer10_attention_weight_min": -37.10063171386719,
      "activations/layer11_attention_weight_max": 35.349220275878906,
      "activations/layer11_attention_weight_min": -35.91006851196289,
      "activations/layer12_attention_weight_max": 18.949792861938477,
      "activations/layer12_attention_weight_min": -24.3004207611084,
      "activations/layer13_attention_weight_max": 34.68235778808594,
      "activations/layer13_attention_weight_min": -27.263370513916016,
      "activations/layer14_attention_weight_max": 36.51880645751953,
      "activations/layer14_attention_weight_min": -29.631105422973633,
      "activations/layer15_attention_weight_max": 34.93821716308594,
      "activations/layer15_attention_weight_min": -29.189119338989258,
      "activations/layer16_attention_weight_max": 34.1925163269043,
      "activations/layer16_attention_weight_min": -32.004398345947266,
      "activations/layer17_attention_weight_max": 54.029441833496094,
      "activations/layer17_attention_weight_min": -45.54090881347656,
      "activations/layer18_attention_weight_max": 49.656150817871094,
      "activations/layer18_attention_weight_min": -37.90019607543945,
      "activations/layer19_attention_weight_max": 27.929712295532227,
      "activations/layer19_attention_weight_min": -20.674091339111328,
      "activations/layer1_attention_weight_max": 16.54525375366211,
      "activations/layer1_attention_weight_min": -13.584162712097168,
      "activations/layer20_attention_weight_max": 25.154802322387695,
      "activations/layer20_attention_weight_min": -20.61153221130371,
      "activations/layer21_attention_weight_max": 37.546939849853516,
      "activations/layer21_attention_weight_min": -20.612802505493164,
      "activations/layer22_attention_weight_max": 33.14411544799805,
      "activations/layer22_attention_weight_min": -25.792951583862305,
      "activations/layer23_attention_weight_max": 39.736881256103516,
      "activations/layer23_attention_weight_min": -22.768760681152344,
      "activations/layer2_attention_weight_max": 31.56647491455078,
      "activations/layer2_attention_weight_min": -28.594770431518555,
      "activations/layer3_attention_weight_max": 88.8921127319336,
      "activations/layer3_attention_weight_min": -91.86420440673828,
      "activations/layer4_attention_weight_max": 85.89363098144531,
      "activations/layer4_attention_weight_min": -87.43058013916016,
      "activations/layer5_attention_weight_max": 65.2137680053711,
      "activations/layer5_attention_weight_min": -78.38556671142578,
      "activations/layer6_attention_weight_max": 51.59038543701172,
      "activations/layer6_attention_weight_min": -51.63059997558594,
      "activations/layer7_attention_weight_max": 64.3828353881836,
      "activations/layer7_attention_weight_min": -62.53254699707031,
      "activations/layer8_attention_weight_max": 47.376834869384766,
      "activations/layer8_attention_weight_min": -48.00300979614258,
      "activations/layer9_attention_weight_max": 45.121273040771484,
      "activations/layer9_attention_weight_min": -46.321895599365234,
      "epoch": 17.81,
      "learning_rate": 3.5510984848484845e-05,
      "loss": 2.7327,
      "step": 306500
    },
    {
      "activations/layer0_attention_weight_max": 15.536062240600586,
      "activations/layer0_attention_weight_min": -13.097607612609863,
      "activations/layer10_attention_weight_max": 34.27960968017578,
      "activations/layer10_attention_weight_min": -35.80224609375,
      "activations/layer11_attention_weight_max": 31.48333740234375,
      "activations/layer11_attention_weight_min": -37.640281677246094,
      "activations/layer12_attention_weight_max": 18.281246185302734,
      "activations/layer12_attention_weight_min": -27.674884796142578,
      "activations/layer13_attention_weight_max": 31.39565658569336,
      "activations/layer13_attention_weight_min": -28.54833984375,
      "activations/layer14_attention_weight_max": 33.830257415771484,
      "activations/layer14_attention_weight_min": -29.913528442382812,
      "activations/layer15_attention_weight_max": 33.39826965332031,
      "activations/layer15_attention_weight_min": -28.47632598876953,
      "activations/layer16_attention_weight_max": 33.71464538574219,
      "activations/layer16_attention_weight_min": -32.17226791381836,
      "activations/layer17_attention_weight_max": 49.00638198852539,
      "activations/layer17_attention_weight_min": -42.284603118896484,
      "activations/layer18_attention_weight_max": 45.869544982910156,
      "activations/layer18_attention_weight_min": -39.105560302734375,
      "activations/layer19_attention_weight_max": 24.269683837890625,
      "activations/layer19_attention_weight_min": -22.15753746032715,
      "activations/layer1_attention_weight_max": 15.745927810668945,
      "activations/layer1_attention_weight_min": -14.03778076171875,
      "activations/layer20_attention_weight_max": 23.759401321411133,
      "activations/layer20_attention_weight_min": -20.5008544921875,
      "activations/layer21_attention_weight_max": 40.27996826171875,
      "activations/layer21_attention_weight_min": -23.002470016479492,
      "activations/layer22_attention_weight_max": 30.80371856689453,
      "activations/layer22_attention_weight_min": -26.78955841064453,
      "activations/layer23_attention_weight_max": 36.93448257446289,
      "activations/layer23_attention_weight_min": -22.876087188720703,
      "activations/layer2_attention_weight_max": 31.4802303314209,
      "activations/layer2_attention_weight_min": -29.014699935913086,
      "activations/layer3_attention_weight_max": 86.66226196289062,
      "activations/layer3_attention_weight_min": -88.21150970458984,
      "activations/layer4_attention_weight_max": 82.6243896484375,
      "activations/layer4_attention_weight_min": -85.19686126708984,
      "activations/layer5_attention_weight_max": 64.28617858886719,
      "activations/layer5_attention_weight_min": -74.55009460449219,
      "activations/layer6_attention_weight_max": 50.37978744506836,
      "activations/layer6_attention_weight_min": -49.00657272338867,
      "activations/layer7_attention_weight_max": 61.312416076660156,
      "activations/layer7_attention_weight_min": -61.1026611328125,
      "activations/layer8_attention_weight_max": 47.59324264526367,
      "activations/layer8_attention_weight_min": -48.20741653442383,
      "activations/layer9_attention_weight_max": 42.655521392822266,
      "activations/layer9_attention_weight_min": -47.82102966308594,
      "epoch": 17.81,
      "learning_rate": 3.5492045454545454e-05,
      "loss": 2.7422,
      "step": 306550
    },
    {
      "activations/layer0_attention_weight_max": 16.69484519958496,
      "activations/layer0_attention_weight_min": -13.276144981384277,
      "activations/layer10_attention_weight_max": 37.87791061401367,
      "activations/layer10_attention_weight_min": -35.5793342590332,
      "activations/layer11_attention_weight_max": 32.76689147949219,
      "activations/layer11_attention_weight_min": -32.38144302368164,
      "activations/layer12_attention_weight_max": 18.092933654785156,
      "activations/layer12_attention_weight_min": -25.892038345336914,
      "activations/layer13_attention_weight_max": 32.788734436035156,
      "activations/layer13_attention_weight_min": -27.66897201538086,
      "activations/layer14_attention_weight_max": 36.20680618286133,
      "activations/layer14_attention_weight_min": -30.377681732177734,
      "activations/layer15_attention_weight_max": 32.28331756591797,
      "activations/layer15_attention_weight_min": -29.51704216003418,
      "activations/layer16_attention_weight_max": 32.98879623413086,
      "activations/layer16_attention_weight_min": -32.08955764770508,
      "activations/layer17_attention_weight_max": 51.71867370605469,
      "activations/layer17_attention_weight_min": -45.45692443847656,
      "activations/layer18_attention_weight_max": 43.59281539916992,
      "activations/layer18_attention_weight_min": -38.96034622192383,
      "activations/layer19_attention_weight_max": 26.67807960510254,
      "activations/layer19_attention_weight_min": -21.77230453491211,
      "activations/layer1_attention_weight_max": 16.177555084228516,
      "activations/layer1_attention_weight_min": -13.956303596496582,
      "activations/layer20_attention_weight_max": 23.5476131439209,
      "activations/layer20_attention_weight_min": -20.75348663330078,
      "activations/layer21_attention_weight_max": 31.594619750976562,
      "activations/layer21_attention_weight_min": -21.276870727539062,
      "activations/layer22_attention_weight_max": 30.887325286865234,
      "activations/layer22_attention_weight_min": -26.31684112548828,
      "activations/layer23_attention_weight_max": 38.335968017578125,
      "activations/layer23_attention_weight_min": -25.0772762298584,
      "activations/layer2_attention_weight_max": 29.048709869384766,
      "activations/layer2_attention_weight_min": -27.884489059448242,
      "activations/layer3_attention_weight_max": 88.97708892822266,
      "activations/layer3_attention_weight_min": -87.26213073730469,
      "activations/layer4_attention_weight_max": 89.16319274902344,
      "activations/layer4_attention_weight_min": -88.07774353027344,
      "activations/layer5_attention_weight_max": 67.30525970458984,
      "activations/layer5_attention_weight_min": -78.1577377319336,
      "activations/layer6_attention_weight_max": 48.49875259399414,
      "activations/layer6_attention_weight_min": -50.05019760131836,
      "activations/layer7_attention_weight_max": 62.60989761352539,
      "activations/layer7_attention_weight_min": -63.564449310302734,
      "activations/layer8_attention_weight_max": 43.778076171875,
      "activations/layer8_attention_weight_min": -46.630924224853516,
      "activations/layer9_attention_weight_max": 42.55607223510742,
      "activations/layer9_attention_weight_min": -48.116947174072266,
      "epoch": 17.82,
      "learning_rate": 3.547310606060606e-05,
      "loss": 2.7479,
      "step": 306600
    },
    {
      "activations/layer0_attention_weight_max": 14.746423721313477,
      "activations/layer0_attention_weight_min": -14.035405158996582,
      "activations/layer10_attention_weight_max": 41.39630126953125,
      "activations/layer10_attention_weight_min": -37.42810821533203,
      "activations/layer11_attention_weight_max": 35.484031677246094,
      "activations/layer11_attention_weight_min": -34.722076416015625,
      "activations/layer12_attention_weight_max": 18.25126075744629,
      "activations/layer12_attention_weight_min": -24.64265251159668,
      "activations/layer13_attention_weight_max": 35.111541748046875,
      "activations/layer13_attention_weight_min": -28.959434509277344,
      "activations/layer14_attention_weight_max": 36.126644134521484,
      "activations/layer14_attention_weight_min": -30.812870025634766,
      "activations/layer15_attention_weight_max": 34.161651611328125,
      "activations/layer15_attention_weight_min": -28.75745391845703,
      "activations/layer16_attention_weight_max": 34.65957260131836,
      "activations/layer16_attention_weight_min": -31.584712982177734,
      "activations/layer17_attention_weight_max": 51.067264556884766,
      "activations/layer17_attention_weight_min": -45.74558639526367,
      "activations/layer18_attention_weight_max": 48.24586486816406,
      "activations/layer18_attention_weight_min": -40.77779769897461,
      "activations/layer19_attention_weight_max": 23.97710418701172,
      "activations/layer19_attention_weight_min": -25.718412399291992,
      "activations/layer1_attention_weight_max": 16.034320831298828,
      "activations/layer1_attention_weight_min": -13.8474702835083,
      "activations/layer20_attention_weight_max": 27.200910568237305,
      "activations/layer20_attention_weight_min": -21.723356246948242,
      "activations/layer21_attention_weight_max": 43.77745056152344,
      "activations/layer21_attention_weight_min": -22.557411193847656,
      "activations/layer22_attention_weight_max": 27.906234741210938,
      "activations/layer22_attention_weight_min": -25.867483139038086,
      "activations/layer23_attention_weight_max": 35.753448486328125,
      "activations/layer23_attention_weight_min": -23.868396759033203,
      "activations/layer2_attention_weight_max": 34.44112014770508,
      "activations/layer2_attention_weight_min": -31.81454849243164,
      "activations/layer3_attention_weight_max": 96.702880859375,
      "activations/layer3_attention_weight_min": -95.25244903564453,
      "activations/layer4_attention_weight_max": 92.93601989746094,
      "activations/layer4_attention_weight_min": -85.00818634033203,
      "activations/layer5_attention_weight_max": 64.54281616210938,
      "activations/layer5_attention_weight_min": -74.0977554321289,
      "activations/layer6_attention_weight_max": 53.157493591308594,
      "activations/layer6_attention_weight_min": -52.31908416748047,
      "activations/layer7_attention_weight_max": 64.69342803955078,
      "activations/layer7_attention_weight_min": -64.2478256225586,
      "activations/layer8_attention_weight_max": 50.2707405090332,
      "activations/layer8_attention_weight_min": -52.83380889892578,
      "activations/layer9_attention_weight_max": 47.07780075073242,
      "activations/layer9_attention_weight_min": -47.393165588378906,
      "epoch": 17.82,
      "learning_rate": 3.545416666666666e-05,
      "loss": 2.7503,
      "step": 306650
    },
    {
      "activations/layer0_attention_weight_max": 16.526287078857422,
      "activations/layer0_attention_weight_min": -13.647987365722656,
      "activations/layer10_attention_weight_max": 33.78415298461914,
      "activations/layer10_attention_weight_min": -34.165687561035156,
      "activations/layer11_attention_weight_max": 32.2353630065918,
      "activations/layer11_attention_weight_min": -33.59845733642578,
      "activations/layer12_attention_weight_max": 20.27349853515625,
      "activations/layer12_attention_weight_min": -23.25921630859375,
      "activations/layer13_attention_weight_max": 37.75325393676758,
      "activations/layer13_attention_weight_min": -29.907489776611328,
      "activations/layer14_attention_weight_max": 35.8691520690918,
      "activations/layer14_attention_weight_min": -33.472896575927734,
      "activations/layer15_attention_weight_max": 36.31543731689453,
      "activations/layer15_attention_weight_min": -31.8464298248291,
      "activations/layer16_attention_weight_max": 39.235862731933594,
      "activations/layer16_attention_weight_min": -38.229488372802734,
      "activations/layer17_attention_weight_max": 56.05845642089844,
      "activations/layer17_attention_weight_min": -52.9864616394043,
      "activations/layer18_attention_weight_max": 53.70725631713867,
      "activations/layer18_attention_weight_min": -44.00321960449219,
      "activations/layer19_attention_weight_max": 25.50713348388672,
      "activations/layer19_attention_weight_min": -23.6123104095459,
      "activations/layer1_attention_weight_max": 16.224266052246094,
      "activations/layer1_attention_weight_min": -14.550541877746582,
      "activations/layer20_attention_weight_max": 25.255624771118164,
      "activations/layer20_attention_weight_min": -23.71391487121582,
      "activations/layer21_attention_weight_max": 40.91858673095703,
      "activations/layer21_attention_weight_min": -25.707035064697266,
      "activations/layer22_attention_weight_max": 32.71799087524414,
      "activations/layer22_attention_weight_min": -25.61432647705078,
      "activations/layer23_attention_weight_max": 37.40350341796875,
      "activations/layer23_attention_weight_min": -23.719505310058594,
      "activations/layer2_attention_weight_max": 31.004169464111328,
      "activations/layer2_attention_weight_min": -30.790037155151367,
      "activations/layer3_attention_weight_max": 86.99761199951172,
      "activations/layer3_attention_weight_min": -92.20388793945312,
      "activations/layer4_attention_weight_max": 91.84125518798828,
      "activations/layer4_attention_weight_min": -86.03533935546875,
      "activations/layer5_attention_weight_max": 65.78870391845703,
      "activations/layer5_attention_weight_min": -70.95504760742188,
      "activations/layer6_attention_weight_max": 49.18022537231445,
      "activations/layer6_attention_weight_min": -49.70930862426758,
      "activations/layer7_attention_weight_max": 61.600669860839844,
      "activations/layer7_attention_weight_min": -59.501792907714844,
      "activations/layer8_attention_weight_max": 44.215274810791016,
      "activations/layer8_attention_weight_min": -48.285945892333984,
      "activations/layer9_attention_weight_max": 42.82558059692383,
      "activations/layer9_attention_weight_min": -44.027679443359375,
      "epoch": 17.82,
      "learning_rate": 3.543522727272727e-05,
      "loss": 2.73,
      "step": 306700
    },
    {
      "activations/layer0_attention_weight_max": 15.317146301269531,
      "activations/layer0_attention_weight_min": -13.697003364562988,
      "activations/layer10_attention_weight_max": 36.46736526489258,
      "activations/layer10_attention_weight_min": -38.92405700683594,
      "activations/layer11_attention_weight_max": 32.66345977783203,
      "activations/layer11_attention_weight_min": -37.524288177490234,
      "activations/layer12_attention_weight_max": 19.376636505126953,
      "activations/layer12_attention_weight_min": -26.181495666503906,
      "activations/layer13_attention_weight_max": 34.31874465942383,
      "activations/layer13_attention_weight_min": -28.28144073486328,
      "activations/layer14_attention_weight_max": 37.426876068115234,
      "activations/layer14_attention_weight_min": -31.084802627563477,
      "activations/layer15_attention_weight_max": 37.3880729675293,
      "activations/layer15_attention_weight_min": -32.072425842285156,
      "activations/layer16_attention_weight_max": 36.79173278808594,
      "activations/layer16_attention_weight_min": -33.10387420654297,
      "activations/layer17_attention_weight_max": 53.761138916015625,
      "activations/layer17_attention_weight_min": -45.964473724365234,
      "activations/layer18_attention_weight_max": 49.79126739501953,
      "activations/layer18_attention_weight_min": -39.211124420166016,
      "activations/layer19_attention_weight_max": 25.925251007080078,
      "activations/layer19_attention_weight_min": -21.39692497253418,
      "activations/layer1_attention_weight_max": 16.477127075195312,
      "activations/layer1_attention_weight_min": -13.8480863571167,
      "activations/layer20_attention_weight_max": 21.964073181152344,
      "activations/layer20_attention_weight_min": -20.50688362121582,
      "activations/layer21_attention_weight_max": 42.356781005859375,
      "activations/layer21_attention_weight_min": -22.778751373291016,
      "activations/layer22_attention_weight_max": 30.55884552001953,
      "activations/layer22_attention_weight_min": -23.08439064025879,
      "activations/layer23_attention_weight_max": 37.67144775390625,
      "activations/layer23_attention_weight_min": -27.41415786743164,
      "activations/layer2_attention_weight_max": 32.13703155517578,
      "activations/layer2_attention_weight_min": -30.31513023376465,
      "activations/layer3_attention_weight_max": 89.71711730957031,
      "activations/layer3_attention_weight_min": -92.02852630615234,
      "activations/layer4_attention_weight_max": 87.4758529663086,
      "activations/layer4_attention_weight_min": -90.3628158569336,
      "activations/layer5_attention_weight_max": 63.889869689941406,
      "activations/layer5_attention_weight_min": -75.93497467041016,
      "activations/layer6_attention_weight_max": 49.95003890991211,
      "activations/layer6_attention_weight_min": -53.31410217285156,
      "activations/layer7_attention_weight_max": 62.143348693847656,
      "activations/layer7_attention_weight_min": -63.66241455078125,
      "activations/layer8_attention_weight_max": 46.54008865356445,
      "activations/layer8_attention_weight_min": -50.30717849731445,
      "activations/layer9_attention_weight_max": 42.70446014404297,
      "activations/layer9_attention_weight_min": -47.4864616394043,
      "epoch": 17.82,
      "learning_rate": 3.541628787878787e-05,
      "loss": 2.7524,
      "step": 306750
    },
    {
      "activations/layer0_attention_weight_max": 16.12151336669922,
      "activations/layer0_attention_weight_min": -13.17365550994873,
      "activations/layer10_attention_weight_max": 35.410064697265625,
      "activations/layer10_attention_weight_min": -36.17561721801758,
      "activations/layer11_attention_weight_max": 32.84071731567383,
      "activations/layer11_attention_weight_min": -34.86152648925781,
      "activations/layer12_attention_weight_max": 19.77589988708496,
      "activations/layer12_attention_weight_min": -30.00737190246582,
      "activations/layer13_attention_weight_max": 35.23907470703125,
      "activations/layer13_attention_weight_min": -26.9846134185791,
      "activations/layer14_attention_weight_max": 36.84334945678711,
      "activations/layer14_attention_weight_min": -30.121185302734375,
      "activations/layer15_attention_weight_max": 38.421810150146484,
      "activations/layer15_attention_weight_min": -30.644039154052734,
      "activations/layer16_attention_weight_max": 35.53885269165039,
      "activations/layer16_attention_weight_min": -32.46485900878906,
      "activations/layer17_attention_weight_max": 54.081398010253906,
      "activations/layer17_attention_weight_min": -45.31103515625,
      "activations/layer18_attention_weight_max": 52.23512268066406,
      "activations/layer18_attention_weight_min": -42.31676483154297,
      "activations/layer19_attention_weight_max": 27.004962921142578,
      "activations/layer19_attention_weight_min": -23.8652286529541,
      "activations/layer1_attention_weight_max": 16.461862564086914,
      "activations/layer1_attention_weight_min": -14.583697319030762,
      "activations/layer20_attention_weight_max": 26.7080135345459,
      "activations/layer20_attention_weight_min": -22.71808624267578,
      "activations/layer21_attention_weight_max": 44.41596603393555,
      "activations/layer21_attention_weight_min": -23.118392944335938,
      "activations/layer22_attention_weight_max": 32.48031997680664,
      "activations/layer22_attention_weight_min": -26.608858108520508,
      "activations/layer23_attention_weight_max": 41.51239776611328,
      "activations/layer23_attention_weight_min": -23.4356689453125,
      "activations/layer2_attention_weight_max": 31.10714340209961,
      "activations/layer2_attention_weight_min": -29.20572280883789,
      "activations/layer3_attention_weight_max": 89.24656677246094,
      "activations/layer3_attention_weight_min": -90.6808853149414,
      "activations/layer4_attention_weight_max": 89.0723648071289,
      "activations/layer4_attention_weight_min": -90.49337005615234,
      "activations/layer5_attention_weight_max": 64.44119262695312,
      "activations/layer5_attention_weight_min": -79.44223022460938,
      "activations/layer6_attention_weight_max": 50.6053352355957,
      "activations/layer6_attention_weight_min": -53.79862594604492,
      "activations/layer7_attention_weight_max": 68.0013656616211,
      "activations/layer7_attention_weight_min": -62.57706069946289,
      "activations/layer8_attention_weight_max": 45.3974494934082,
      "activations/layer8_attention_weight_min": -48.606937408447266,
      "activations/layer9_attention_weight_max": 43.18461227416992,
      "activations/layer9_attention_weight_min": -46.1412467956543,
      "epoch": 17.83,
      "learning_rate": 3.539734848484848e-05,
      "loss": 2.7344,
      "step": 306800
    },
    {
      "activations/layer0_attention_weight_max": 15.523846626281738,
      "activations/layer0_attention_weight_min": -13.316805839538574,
      "activations/layer10_attention_weight_max": 32.4470100402832,
      "activations/layer10_attention_weight_min": -34.786659240722656,
      "activations/layer11_attention_weight_max": 30.433639526367188,
      "activations/layer11_attention_weight_min": -33.27535629272461,
      "activations/layer12_attention_weight_max": 17.72156524658203,
      "activations/layer12_attention_weight_min": -26.345861434936523,
      "activations/layer13_attention_weight_max": 31.79082679748535,
      "activations/layer13_attention_weight_min": -27.555208206176758,
      "activations/layer14_attention_weight_max": 34.081687927246094,
      "activations/layer14_attention_weight_min": -28.84921646118164,
      "activations/layer15_attention_weight_max": 31.017995834350586,
      "activations/layer15_attention_weight_min": -28.605627059936523,
      "activations/layer16_attention_weight_max": 31.726734161376953,
      "activations/layer16_attention_weight_min": -31.01326560974121,
      "activations/layer17_attention_weight_max": 46.42360305786133,
      "activations/layer17_attention_weight_min": -41.76386642456055,
      "activations/layer18_attention_weight_max": 43.39086151123047,
      "activations/layer18_attention_weight_min": -37.539337158203125,
      "activations/layer19_attention_weight_max": 22.40960693359375,
      "activations/layer19_attention_weight_min": -22.139339447021484,
      "activations/layer1_attention_weight_max": 15.976323127746582,
      "activations/layer1_attention_weight_min": -12.771344184875488,
      "activations/layer20_attention_weight_max": 21.126182556152344,
      "activations/layer20_attention_weight_min": -20.955753326416016,
      "activations/layer21_attention_weight_max": 39.08818435668945,
      "activations/layer21_attention_weight_min": -23.76390266418457,
      "activations/layer22_attention_weight_max": 31.303958892822266,
      "activations/layer22_attention_weight_min": -25.62598991394043,
      "activations/layer23_attention_weight_max": 41.177085876464844,
      "activations/layer23_attention_weight_min": -24.644826889038086,
      "activations/layer2_attention_weight_max": 29.069164276123047,
      "activations/layer2_attention_weight_min": -28.19717788696289,
      "activations/layer3_attention_weight_max": 83.29888153076172,
      "activations/layer3_attention_weight_min": -85.88253784179688,
      "activations/layer4_attention_weight_max": 83.41282653808594,
      "activations/layer4_attention_weight_min": -79.59300994873047,
      "activations/layer5_attention_weight_max": 62.745662689208984,
      "activations/layer5_attention_weight_min": -75.3992919921875,
      "activations/layer6_attention_weight_max": 47.35356140136719,
      "activations/layer6_attention_weight_min": -48.78049087524414,
      "activations/layer7_attention_weight_max": 61.599830627441406,
      "activations/layer7_attention_weight_min": -58.188934326171875,
      "activations/layer8_attention_weight_max": 42.92827224731445,
      "activations/layer8_attention_weight_min": -45.424312591552734,
      "activations/layer9_attention_weight_max": 40.671443939208984,
      "activations/layer9_attention_weight_min": -42.2808952331543,
      "epoch": 17.83,
      "learning_rate": 3.537840909090909e-05,
      "loss": 2.7208,
      "step": 306850
    },
    {
      "activations/layer0_attention_weight_max": 15.924530029296875,
      "activations/layer0_attention_weight_min": -13.488668441772461,
      "activations/layer10_attention_weight_max": 35.4394645690918,
      "activations/layer10_attention_weight_min": -34.39802932739258,
      "activations/layer11_attention_weight_max": 31.85341453552246,
      "activations/layer11_attention_weight_min": -34.35932159423828,
      "activations/layer12_attention_weight_max": 18.19671630859375,
      "activations/layer12_attention_weight_min": -29.541263580322266,
      "activations/layer13_attention_weight_max": 35.544578552246094,
      "activations/layer13_attention_weight_min": -28.713029861450195,
      "activations/layer14_attention_weight_max": 33.23297119140625,
      "activations/layer14_attention_weight_min": -29.60427474975586,
      "activations/layer15_attention_weight_max": 30.763763427734375,
      "activations/layer15_attention_weight_min": -29.8492431640625,
      "activations/layer16_attention_weight_max": 31.819156646728516,
      "activations/layer16_attention_weight_min": -31.789318084716797,
      "activations/layer17_attention_weight_max": 48.198707580566406,
      "activations/layer17_attention_weight_min": -45.3570442199707,
      "activations/layer18_attention_weight_max": 42.20718765258789,
      "activations/layer18_attention_weight_min": -39.35165786743164,
      "activations/layer19_attention_weight_max": 22.414682388305664,
      "activations/layer19_attention_weight_min": -23.730043411254883,
      "activations/layer1_attention_weight_max": 15.812206268310547,
      "activations/layer1_attention_weight_min": -14.297845840454102,
      "activations/layer20_attention_weight_max": 20.79660415649414,
      "activations/layer20_attention_weight_min": -22.045028686523438,
      "activations/layer21_attention_weight_max": 34.92584991455078,
      "activations/layer21_attention_weight_min": -22.479188919067383,
      "activations/layer22_attention_weight_max": 25.876794815063477,
      "activations/layer22_attention_weight_min": -25.843721389770508,
      "activations/layer23_attention_weight_max": 35.20806121826172,
      "activations/layer23_attention_weight_min": -23.7846736907959,
      "activations/layer2_attention_weight_max": 31.485532760620117,
      "activations/layer2_attention_weight_min": -30.190101623535156,
      "activations/layer3_attention_weight_max": 91.43983459472656,
      "activations/layer3_attention_weight_min": -90.84457397460938,
      "activations/layer4_attention_weight_max": 84.89646911621094,
      "activations/layer4_attention_weight_min": -87.31450653076172,
      "activations/layer5_attention_weight_max": 66.59068298339844,
      "activations/layer5_attention_weight_min": -76.79986572265625,
      "activations/layer6_attention_weight_max": 49.16020965576172,
      "activations/layer6_attention_weight_min": -48.97456359863281,
      "activations/layer7_attention_weight_max": 62.72824478149414,
      "activations/layer7_attention_weight_min": -63.61833190917969,
      "activations/layer8_attention_weight_max": 47.253173828125,
      "activations/layer8_attention_weight_min": -49.260128021240234,
      "activations/layer9_attention_weight_max": 43.69205093383789,
      "activations/layer9_attention_weight_min": -44.07548141479492,
      "epoch": 17.83,
      "learning_rate": 3.535946969696969e-05,
      "loss": 2.7179,
      "step": 306900
    },
    {
      "activations/layer0_attention_weight_max": 15.62521743774414,
      "activations/layer0_attention_weight_min": -13.710763931274414,
      "activations/layer10_attention_weight_max": 37.23410415649414,
      "activations/layer10_attention_weight_min": -35.27386474609375,
      "activations/layer11_attention_weight_max": 33.476890563964844,
      "activations/layer11_attention_weight_min": -33.6622428894043,
      "activations/layer12_attention_weight_max": 18.86515998840332,
      "activations/layer12_attention_weight_min": -29.15696907043457,
      "activations/layer13_attention_weight_max": 35.643760681152344,
      "activations/layer13_attention_weight_min": -28.80927276611328,
      "activations/layer14_attention_weight_max": 37.505672454833984,
      "activations/layer14_attention_weight_min": -29.774085998535156,
      "activations/layer15_attention_weight_max": 32.08134078979492,
      "activations/layer15_attention_weight_min": -31.400341033935547,
      "activations/layer16_attention_weight_max": 35.476478576660156,
      "activations/layer16_attention_weight_min": -31.926090240478516,
      "activations/layer17_attention_weight_max": 51.807682037353516,
      "activations/layer17_attention_weight_min": -48.29378128051758,
      "activations/layer18_attention_weight_max": 44.87733840942383,
      "activations/layer18_attention_weight_min": -40.95571517944336,
      "activations/layer19_attention_weight_max": 24.309621810913086,
      "activations/layer19_attention_weight_min": -23.983074188232422,
      "activations/layer1_attention_weight_max": 16.95022201538086,
      "activations/layer1_attention_weight_min": -13.815000534057617,
      "activations/layer20_attention_weight_max": 23.630823135375977,
      "activations/layer20_attention_weight_min": -20.98138999938965,
      "activations/layer21_attention_weight_max": 43.95852279663086,
      "activations/layer21_attention_weight_min": -22.569795608520508,
      "activations/layer22_attention_weight_max": 30.682025909423828,
      "activations/layer22_attention_weight_min": -26.11809539794922,
      "activations/layer23_attention_weight_max": 35.22528839111328,
      "activations/layer23_attention_weight_min": -27.532909393310547,
      "activations/layer2_attention_weight_max": 32.7794303894043,
      "activations/layer2_attention_weight_min": -30.987205505371094,
      "activations/layer3_attention_weight_max": 94.71214294433594,
      "activations/layer3_attention_weight_min": -94.65480041503906,
      "activations/layer4_attention_weight_max": 92.32365417480469,
      "activations/layer4_attention_weight_min": -92.22471618652344,
      "activations/layer5_attention_weight_max": 65.49920654296875,
      "activations/layer5_attention_weight_min": -73.92772674560547,
      "activations/layer6_attention_weight_max": 50.61448669433594,
      "activations/layer6_attention_weight_min": -52.315895080566406,
      "activations/layer7_attention_weight_max": 67.8945083618164,
      "activations/layer7_attention_weight_min": -62.961753845214844,
      "activations/layer8_attention_weight_max": 44.95195388793945,
      "activations/layer8_attention_weight_min": -48.378196716308594,
      "activations/layer9_attention_weight_max": 44.76605224609375,
      "activations/layer9_attention_weight_min": -43.43320846557617,
      "epoch": 17.84,
      "learning_rate": 3.53405303030303e-05,
      "loss": 2.736,
      "step": 306950
    },
    {
      "activations/layer0_attention_weight_max": 16.290800094604492,
      "activations/layer0_attention_weight_min": -13.245006561279297,
      "activations/layer10_attention_weight_max": 33.65495681762695,
      "activations/layer10_attention_weight_min": -32.615394592285156,
      "activations/layer11_attention_weight_max": 30.42298126220703,
      "activations/layer11_attention_weight_min": -32.58229064941406,
      "activations/layer12_attention_weight_max": 19.841888427734375,
      "activations/layer12_attention_weight_min": -27.948387145996094,
      "activations/layer13_attention_weight_max": 33.457550048828125,
      "activations/layer13_attention_weight_min": -27.055253982543945,
      "activations/layer14_attention_weight_max": 36.32643127441406,
      "activations/layer14_attention_weight_min": -33.9604606628418,
      "activations/layer15_attention_weight_max": 35.780052185058594,
      "activations/layer15_attention_weight_min": -31.62405776977539,
      "activations/layer16_attention_weight_max": 35.60313034057617,
      "activations/layer16_attention_weight_min": -31.52228546142578,
      "activations/layer17_attention_weight_max": 56.06074142456055,
      "activations/layer17_attention_weight_min": -46.8427734375,
      "activations/layer18_attention_weight_max": 54.58988952636719,
      "activations/layer18_attention_weight_min": -41.28945541381836,
      "activations/layer19_attention_weight_max": 25.104761123657227,
      "activations/layer19_attention_weight_min": -24.047327041625977,
      "activations/layer1_attention_weight_max": 17.016653060913086,
      "activations/layer1_attention_weight_min": -13.819167137145996,
      "activations/layer20_attention_weight_max": 24.441877365112305,
      "activations/layer20_attention_weight_min": -21.18539047241211,
      "activations/layer21_attention_weight_max": 39.47974395751953,
      "activations/layer21_attention_weight_min": -21.64053726196289,
      "activations/layer22_attention_weight_max": 33.94351577758789,
      "activations/layer22_attention_weight_min": -26.25664520263672,
      "activations/layer23_attention_weight_max": 39.028236389160156,
      "activations/layer23_attention_weight_min": -24.86797332763672,
      "activations/layer2_attention_weight_max": 31.578998565673828,
      "activations/layer2_attention_weight_min": -29.706253051757812,
      "activations/layer3_attention_weight_max": 86.0643539428711,
      "activations/layer3_attention_weight_min": -86.89302825927734,
      "activations/layer4_attention_weight_max": 83.4317398071289,
      "activations/layer4_attention_weight_min": -88.4977798461914,
      "activations/layer5_attention_weight_max": 60.66450500488281,
      "activations/layer5_attention_weight_min": -70.51266479492188,
      "activations/layer6_attention_weight_max": 46.29927444458008,
      "activations/layer6_attention_weight_min": -47.352210998535156,
      "activations/layer7_attention_weight_max": 60.50056076049805,
      "activations/layer7_attention_weight_min": -57.03374099731445,
      "activations/layer8_attention_weight_max": 42.75087356567383,
      "activations/layer8_attention_weight_min": -44.481136322021484,
      "activations/layer9_attention_weight_max": 42.02139663696289,
      "activations/layer9_attention_weight_min": -44.69554138183594,
      "epoch": 17.84,
      "learning_rate": 3.532159090909091e-05,
      "loss": 2.7425,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4615,
      "eval_samples_per_second": 507.473,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4615,
      "eval_openwebtext_samples_per_second": 507.473,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9425,
      "eval_wikitext_samples_per_second": 234.748,
      "step": 307000
    },
    {
      "epoch": 17.84,
      "eval_lambada_loss": 2.4609375,
      "eval_lambada_ppl": 11.715789946123383,
      "eval_lambada_runtime": 9.5487,
      "eval_lambada_samples_per_second": 509.913,
      "step": 307000
    },
    {
      "activations/layer0_attention_weight_max": 16.050880432128906,
      "activations/layer0_attention_weight_min": -13.448088645935059,
      "activations/layer10_attention_weight_max": 32.922401428222656,
      "activations/layer10_attention_weight_min": -34.49243927001953,
      "activations/layer11_attention_weight_max": 30.15131378173828,
      "activations/layer11_attention_weight_min": -33.23584747314453,
      "activations/layer12_attention_weight_max": 18.20918846130371,
      "activations/layer12_attention_weight_min": -30.091577529907227,
      "activations/layer13_attention_weight_max": 32.177528381347656,
      "activations/layer13_attention_weight_min": -27.275239944458008,
      "activations/layer14_attention_weight_max": 32.42915344238281,
      "activations/layer14_attention_weight_min": -28.687742233276367,
      "activations/layer15_attention_weight_max": 31.066184997558594,
      "activations/layer15_attention_weight_min": -29.494693756103516,
      "activations/layer16_attention_weight_max": 30.629566192626953,
      "activations/layer16_attention_weight_min": -30.87849998474121,
      "activations/layer17_attention_weight_max": 50.947471618652344,
      "activations/layer17_attention_weight_min": -45.9830207824707,
      "activations/layer18_attention_weight_max": 46.89689254760742,
      "activations/layer18_attention_weight_min": -43.42631530761719,
      "activations/layer19_attention_weight_max": 21.57512092590332,
      "activations/layer19_attention_weight_min": -22.71502685546875,
      "activations/layer1_attention_weight_max": 15.718676567077637,
      "activations/layer1_attention_weight_min": -14.902887344360352,
      "activations/layer20_attention_weight_max": 23.004037857055664,
      "activations/layer20_attention_weight_min": -20.88979721069336,
      "activations/layer21_attention_weight_max": 33.394901275634766,
      "activations/layer21_attention_weight_min": -21.409730911254883,
      "activations/layer22_attention_weight_max": 29.495792388916016,
      "activations/layer22_attention_weight_min": -24.373271942138672,
      "activations/layer23_attention_weight_max": 33.72937774658203,
      "activations/layer23_attention_weight_min": -24.45677947998047,
      "activations/layer2_attention_weight_max": 30.11469268798828,
      "activations/layer2_attention_weight_min": -29.610265731811523,
      "activations/layer3_attention_weight_max": 88.18994903564453,
      "activations/layer3_attention_weight_min": -89.04679870605469,
      "activations/layer4_attention_weight_max": 85.45775604248047,
      "activations/layer4_attention_weight_min": -85.67577362060547,
      "activations/layer5_attention_weight_max": 63.430416107177734,
      "activations/layer5_attention_weight_min": -77.4383544921875,
      "activations/layer6_attention_weight_max": 48.66154861450195,
      "activations/layer6_attention_weight_min": -49.134212493896484,
      "activations/layer7_attention_weight_max": 64.3768081665039,
      "activations/layer7_attention_weight_min": -59.70107650756836,
      "activations/layer8_attention_weight_max": 43.927188873291016,
      "activations/layer8_attention_weight_min": -46.04534149169922,
      "activations/layer9_attention_weight_max": 44.79133605957031,
      "activations/layer9_attention_weight_min": -43.40311050415039,
      "epoch": 17.84,
      "learning_rate": 3.530265151515151e-05,
      "loss": 2.7503,
      "step": 307050
    },
    {
      "activations/layer0_attention_weight_max": 15.744002342224121,
      "activations/layer0_attention_weight_min": -13.662832260131836,
      "activations/layer10_attention_weight_max": 36.006221771240234,
      "activations/layer10_attention_weight_min": -34.88014221191406,
      "activations/layer11_attention_weight_max": 32.38386917114258,
      "activations/layer11_attention_weight_min": -33.873863220214844,
      "activations/layer12_attention_weight_max": 19.39363670349121,
      "activations/layer12_attention_weight_min": -24.372953414916992,
      "activations/layer13_attention_weight_max": 34.172950744628906,
      "activations/layer13_attention_weight_min": -27.079519271850586,
      "activations/layer14_attention_weight_max": 38.311561584472656,
      "activations/layer14_attention_weight_min": -31.29448127746582,
      "activations/layer15_attention_weight_max": 33.973758697509766,
      "activations/layer15_attention_weight_min": -30.350048065185547,
      "activations/layer16_attention_weight_max": 35.82768630981445,
      "activations/layer16_attention_weight_min": -34.44296646118164,
      "activations/layer17_attention_weight_max": 53.91434860229492,
      "activations/layer17_attention_weight_min": -47.67348861694336,
      "activations/layer18_attention_weight_max": 49.91859436035156,
      "activations/layer18_attention_weight_min": -39.46384048461914,
      "activations/layer19_attention_weight_max": 23.339107513427734,
      "activations/layer19_attention_weight_min": -24.53305435180664,
      "activations/layer1_attention_weight_max": 15.737324714660645,
      "activations/layer1_attention_weight_min": -12.41831111907959,
      "activations/layer20_attention_weight_max": 21.75927734375,
      "activations/layer20_attention_weight_min": -23.235795974731445,
      "activations/layer21_attention_weight_max": 38.182167053222656,
      "activations/layer21_attention_weight_min": -22.423906326293945,
      "activations/layer22_attention_weight_max": 27.85108757019043,
      "activations/layer22_attention_weight_min": -28.124502182006836,
      "activations/layer23_attention_weight_max": 34.29417419433594,
      "activations/layer23_attention_weight_min": -24.88177490234375,
      "activations/layer2_attention_weight_max": 32.54344940185547,
      "activations/layer2_attention_weight_min": -28.19371795654297,
      "activations/layer3_attention_weight_max": 95.37358856201172,
      "activations/layer3_attention_weight_min": -90.20867919921875,
      "activations/layer4_attention_weight_max": 90.9129638671875,
      "activations/layer4_attention_weight_min": -89.21306610107422,
      "activations/layer5_attention_weight_max": 65.20640563964844,
      "activations/layer5_attention_weight_min": -75.2356185913086,
      "activations/layer6_attention_weight_max": 51.47099304199219,
      "activations/layer6_attention_weight_min": -48.341026306152344,
      "activations/layer7_attention_weight_max": 64.27815246582031,
      "activations/layer7_attention_weight_min": -60.38277816772461,
      "activations/layer8_attention_weight_max": 47.390106201171875,
      "activations/layer8_attention_weight_min": -50.404659271240234,
      "activations/layer9_attention_weight_max": 42.004058837890625,
      "activations/layer9_attention_weight_min": -43.653785705566406,
      "epoch": 17.84,
      "learning_rate": 3.528371212121212e-05,
      "loss": 2.7528,
      "step": 307100
    },
    {
      "activations/layer0_attention_weight_max": 15.563713073730469,
      "activations/layer0_attention_weight_min": -13.193331718444824,
      "activations/layer10_attention_weight_max": 39.072723388671875,
      "activations/layer10_attention_weight_min": -36.81819534301758,
      "activations/layer11_attention_weight_max": 33.99004364013672,
      "activations/layer11_attention_weight_min": -33.642601013183594,
      "activations/layer12_attention_weight_max": 18.539112091064453,
      "activations/layer12_attention_weight_min": -25.860984802246094,
      "activations/layer13_attention_weight_max": 36.730567932128906,
      "activations/layer13_attention_weight_min": -29.743623733520508,
      "activations/layer14_attention_weight_max": 38.4544792175293,
      "activations/layer14_attention_weight_min": -30.557353973388672,
      "activations/layer15_attention_weight_max": 33.20426559448242,
      "activations/layer15_attention_weight_min": -30.669580459594727,
      "activations/layer16_attention_weight_max": 33.684749603271484,
      "activations/layer16_attention_weight_min": -30.071853637695312,
      "activations/layer17_attention_weight_max": 50.31455993652344,
      "activations/layer17_attention_weight_min": -43.03748321533203,
      "activations/layer18_attention_weight_max": 45.42400360107422,
      "activations/layer18_attention_weight_min": -38.590179443359375,
      "activations/layer19_attention_weight_max": 23.577327728271484,
      "activations/layer19_attention_weight_min": -22.618148803710938,
      "activations/layer1_attention_weight_max": 15.749479293823242,
      "activations/layer1_attention_weight_min": -13.650846481323242,
      "activations/layer20_attention_weight_max": 21.977476119995117,
      "activations/layer20_attention_weight_min": -21.131492614746094,
      "activations/layer21_attention_weight_max": 32.606536865234375,
      "activations/layer21_attention_weight_min": -21.977909088134766,
      "activations/layer22_attention_weight_max": 28.263427734375,
      "activations/layer22_attention_weight_min": -24.30226707458496,
      "activations/layer23_attention_weight_max": 37.82462692260742,
      "activations/layer23_attention_weight_min": -23.18112564086914,
      "activations/layer2_attention_weight_max": 29.423986434936523,
      "activations/layer2_attention_weight_min": -28.643112182617188,
      "activations/layer3_attention_weight_max": 88.69277954101562,
      "activations/layer3_attention_weight_min": -98.42411041259766,
      "activations/layer4_attention_weight_max": 87.15281677246094,
      "activations/layer4_attention_weight_min": -87.1468276977539,
      "activations/layer5_attention_weight_max": 64.9388427734375,
      "activations/layer5_attention_weight_min": -72.34955596923828,
      "activations/layer6_attention_weight_max": 50.264034271240234,
      "activations/layer6_attention_weight_min": -52.85737991333008,
      "activations/layer7_attention_weight_max": 62.4846305847168,
      "activations/layer7_attention_weight_min": -67.28230285644531,
      "activations/layer8_attention_weight_max": 47.41933822631836,
      "activations/layer8_attention_weight_min": -52.931087493896484,
      "activations/layer9_attention_weight_max": 45.14254379272461,
      "activations/layer9_attention_weight_min": -48.8439826965332,
      "epoch": 17.85,
      "learning_rate": 3.5264772727272726e-05,
      "loss": 2.7317,
      "step": 307150
    },
    {
      "activations/layer0_attention_weight_max": 15.130125999450684,
      "activations/layer0_attention_weight_min": -13.63198184967041,
      "activations/layer10_attention_weight_max": 36.95241928100586,
      "activations/layer10_attention_weight_min": -36.12948226928711,
      "activations/layer11_attention_weight_max": 36.46711349487305,
      "activations/layer11_attention_weight_min": -33.87800216674805,
      "activations/layer12_attention_weight_max": 19.29903793334961,
      "activations/layer12_attention_weight_min": -35.4109001159668,
      "activations/layer13_attention_weight_max": 36.64388656616211,
      "activations/layer13_attention_weight_min": -29.552074432373047,
      "activations/layer14_attention_weight_max": 36.18082046508789,
      "activations/layer14_attention_weight_min": -30.856584548950195,
      "activations/layer15_attention_weight_max": 35.62171173095703,
      "activations/layer15_attention_weight_min": -30.07417106628418,
      "activations/layer16_attention_weight_max": 34.16389465332031,
      "activations/layer16_attention_weight_min": -31.24176025390625,
      "activations/layer17_attention_weight_max": 53.277061462402344,
      "activations/layer17_attention_weight_min": -45.830360412597656,
      "activations/layer18_attention_weight_max": 49.81503677368164,
      "activations/layer18_attention_weight_min": -38.95124053955078,
      "activations/layer19_attention_weight_max": 23.949188232421875,
      "activations/layer19_attention_weight_min": -22.250959396362305,
      "activations/layer1_attention_weight_max": 15.664671897888184,
      "activations/layer1_attention_weight_min": -13.830580711364746,
      "activations/layer20_attention_weight_max": 24.27446174621582,
      "activations/layer20_attention_weight_min": -22.289457321166992,
      "activations/layer21_attention_weight_max": 35.36618423461914,
      "activations/layer21_attention_weight_min": -21.58485984802246,
      "activations/layer22_attention_weight_max": 30.977508544921875,
      "activations/layer22_attention_weight_min": -25.443933486938477,
      "activations/layer23_attention_weight_max": 43.3216667175293,
      "activations/layer23_attention_weight_min": -23.449050903320312,
      "activations/layer2_attention_weight_max": 31.47705078125,
      "activations/layer2_attention_weight_min": -29.794401168823242,
      "activations/layer3_attention_weight_max": 90.2021713256836,
      "activations/layer3_attention_weight_min": -87.80023956298828,
      "activations/layer4_attention_weight_max": 88.81021118164062,
      "activations/layer4_attention_weight_min": -85.80712127685547,
      "activations/layer5_attention_weight_max": 64.67648315429688,
      "activations/layer5_attention_weight_min": -71.64244079589844,
      "activations/layer6_attention_weight_max": 49.96906661987305,
      "activations/layer6_attention_weight_min": -50.49436569213867,
      "activations/layer7_attention_weight_max": 67.3035659790039,
      "activations/layer7_attention_weight_min": -63.2934684753418,
      "activations/layer8_attention_weight_max": 46.22126388549805,
      "activations/layer8_attention_weight_min": -49.55205154418945,
      "activations/layer9_attention_weight_max": 50.570350646972656,
      "activations/layer9_attention_weight_min": -46.50331497192383,
      "epoch": 17.85,
      "learning_rate": 3.524583333333333e-05,
      "loss": 2.7338,
      "step": 307200
    },
    {
      "activations/layer0_attention_weight_max": 15.322598457336426,
      "activations/layer0_attention_weight_min": -13.537153244018555,
      "activations/layer10_attention_weight_max": 34.73381423950195,
      "activations/layer10_attention_weight_min": -35.57135772705078,
      "activations/layer11_attention_weight_max": 32.81755065917969,
      "activations/layer11_attention_weight_min": -34.393211364746094,
      "activations/layer12_attention_weight_max": 21.102893829345703,
      "activations/layer12_attention_weight_min": -26.077186584472656,
      "activations/layer13_attention_weight_max": 36.997901916503906,
      "activations/layer13_attention_weight_min": -27.32065200805664,
      "activations/layer14_attention_weight_max": 35.18828582763672,
      "activations/layer14_attention_weight_min": -31.914989471435547,
      "activations/layer15_attention_weight_max": 31.69084358215332,
      "activations/layer15_attention_weight_min": -32.31952667236328,
      "activations/layer16_attention_weight_max": 34.26482009887695,
      "activations/layer16_attention_weight_min": -32.05165481567383,
      "activations/layer17_attention_weight_max": 49.6663703918457,
      "activations/layer17_attention_weight_min": -46.509273529052734,
      "activations/layer18_attention_weight_max": 46.16231155395508,
      "activations/layer18_attention_weight_min": -38.84191131591797,
      "activations/layer19_attention_weight_max": 24.65626335144043,
      "activations/layer19_attention_weight_min": -24.244354248046875,
      "activations/layer1_attention_weight_max": 15.658899307250977,
      "activations/layer1_attention_weight_min": -15.300422668457031,
      "activations/layer20_attention_weight_max": 22.407196044921875,
      "activations/layer20_attention_weight_min": -22.880558013916016,
      "activations/layer21_attention_weight_max": 38.434608459472656,
      "activations/layer21_attention_weight_min": -22.558774948120117,
      "activations/layer22_attention_weight_max": 29.330535888671875,
      "activations/layer22_attention_weight_min": -25.506818771362305,
      "activations/layer23_attention_weight_max": 35.878509521484375,
      "activations/layer23_attention_weight_min": -22.52998161315918,
      "activations/layer2_attention_weight_max": 35.16991424560547,
      "activations/layer2_attention_weight_min": -33.68158721923828,
      "activations/layer3_attention_weight_max": 88.05965423583984,
      "activations/layer3_attention_weight_min": -89.30982971191406,
      "activations/layer4_attention_weight_max": 87.8521499633789,
      "activations/layer4_attention_weight_min": -85.32501983642578,
      "activations/layer5_attention_weight_max": 64.73629760742188,
      "activations/layer5_attention_weight_min": -72.45213317871094,
      "activations/layer6_attention_weight_max": 48.37468719482422,
      "activations/layer6_attention_weight_min": -51.10440444946289,
      "activations/layer7_attention_weight_max": 62.27909851074219,
      "activations/layer7_attention_weight_min": -60.349403381347656,
      "activations/layer8_attention_weight_max": 45.2825813293457,
      "activations/layer8_attention_weight_min": -47.77267837524414,
      "activations/layer9_attention_weight_max": 45.75771713256836,
      "activations/layer9_attention_weight_min": -44.72038269042969,
      "epoch": 17.85,
      "learning_rate": 3.5226893939393935e-05,
      "loss": 2.7543,
      "step": 307250
    },
    {
      "activations/layer0_attention_weight_max": 15.479007720947266,
      "activations/layer0_attention_weight_min": -13.082229614257812,
      "activations/layer10_attention_weight_max": 35.25442886352539,
      "activations/layer10_attention_weight_min": -34.622947692871094,
      "activations/layer11_attention_weight_max": 32.259552001953125,
      "activations/layer11_attention_weight_min": -32.244537353515625,
      "activations/layer12_attention_weight_max": 22.9656982421875,
      "activations/layer12_attention_weight_min": -26.387758255004883,
      "activations/layer13_attention_weight_max": 34.95323944091797,
      "activations/layer13_attention_weight_min": -28.836322784423828,
      "activations/layer14_attention_weight_max": 34.67522048950195,
      "activations/layer14_attention_weight_min": -28.848915100097656,
      "activations/layer15_attention_weight_max": 33.89588928222656,
      "activations/layer15_attention_weight_min": -30.163951873779297,
      "activations/layer16_attention_weight_max": 35.27851867675781,
      "activations/layer16_attention_weight_min": -31.75592803955078,
      "activations/layer17_attention_weight_max": 57.02952194213867,
      "activations/layer17_attention_weight_min": -46.08116912841797,
      "activations/layer18_attention_weight_max": 48.03832244873047,
      "activations/layer18_attention_weight_min": -39.86127471923828,
      "activations/layer19_attention_weight_max": 25.514785766601562,
      "activations/layer19_attention_weight_min": -23.208019256591797,
      "activations/layer1_attention_weight_max": 15.36752986907959,
      "activations/layer1_attention_weight_min": -13.202425956726074,
      "activations/layer20_attention_weight_max": 22.614276885986328,
      "activations/layer20_attention_weight_min": -23.61213493347168,
      "activations/layer21_attention_weight_max": 33.920989990234375,
      "activations/layer21_attention_weight_min": -23.9033145904541,
      "activations/layer22_attention_weight_max": 31.208892822265625,
      "activations/layer22_attention_weight_min": -28.299579620361328,
      "activations/layer23_attention_weight_max": 36.00504684448242,
      "activations/layer23_attention_weight_min": -25.84042739868164,
      "activations/layer2_attention_weight_max": 29.710548400878906,
      "activations/layer2_attention_weight_min": -28.375045776367188,
      "activations/layer3_attention_weight_max": 78.87255859375,
      "activations/layer3_attention_weight_min": -81.96817016601562,
      "activations/layer4_attention_weight_max": 80.53881072998047,
      "activations/layer4_attention_weight_min": -82.99549865722656,
      "activations/layer5_attention_weight_max": 60.810546875,
      "activations/layer5_attention_weight_min": -76.102783203125,
      "activations/layer6_attention_weight_max": 46.39019012451172,
      "activations/layer6_attention_weight_min": -48.22941970825195,
      "activations/layer7_attention_weight_max": 60.55331039428711,
      "activations/layer7_attention_weight_min": -60.95543670654297,
      "activations/layer8_attention_weight_max": 44.69267654418945,
      "activations/layer8_attention_weight_min": -45.17863845825195,
      "activations/layer9_attention_weight_max": 42.97053909301758,
      "activations/layer9_attention_weight_min": -47.6313591003418,
      "epoch": 17.86,
      "learning_rate": 3.5207954545454544e-05,
      "loss": 2.747,
      "step": 307300
    },
    {
      "activations/layer0_attention_weight_max": 15.686888694763184,
      "activations/layer0_attention_weight_min": -13.706340789794922,
      "activations/layer10_attention_weight_max": 34.32388687133789,
      "activations/layer10_attention_weight_min": -34.69496536254883,
      "activations/layer11_attention_weight_max": 31.643901824951172,
      "activations/layer11_attention_weight_min": -33.422401428222656,
      "activations/layer12_attention_weight_max": 19.539342880249023,
      "activations/layer12_attention_weight_min": -25.47376823425293,
      "activations/layer13_attention_weight_max": 32.4571418762207,
      "activations/layer13_attention_weight_min": -28.74267578125,
      "activations/layer14_attention_weight_max": 33.722110748291016,
      "activations/layer14_attention_weight_min": -30.335926055908203,
      "activations/layer15_attention_weight_max": 32.27862548828125,
      "activations/layer15_attention_weight_min": -33.64651107788086,
      "activations/layer16_attention_weight_max": 36.01077651977539,
      "activations/layer16_attention_weight_min": -34.325721740722656,
      "activations/layer17_attention_weight_max": 53.61806869506836,
      "activations/layer17_attention_weight_min": -48.326107025146484,
      "activations/layer18_attention_weight_max": 47.67768478393555,
      "activations/layer18_attention_weight_min": -44.628055572509766,
      "activations/layer19_attention_weight_max": 24.890079498291016,
      "activations/layer19_attention_weight_min": -27.256412506103516,
      "activations/layer1_attention_weight_max": 16.106840133666992,
      "activations/layer1_attention_weight_min": -13.343278884887695,
      "activations/layer20_attention_weight_max": 22.891448974609375,
      "activations/layer20_attention_weight_min": -27.944812774658203,
      "activations/layer21_attention_weight_max": 36.51523208618164,
      "activations/layer21_attention_weight_min": -24.967954635620117,
      "activations/layer22_attention_weight_max": 29.68262481689453,
      "activations/layer22_attention_weight_min": -27.72480010986328,
      "activations/layer23_attention_weight_max": 39.08956527709961,
      "activations/layer23_attention_weight_min": -25.444395065307617,
      "activations/layer2_attention_weight_max": 30.498729705810547,
      "activations/layer2_attention_weight_min": -29.524559020996094,
      "activations/layer3_attention_weight_max": 97.73446655273438,
      "activations/layer3_attention_weight_min": -93.79621124267578,
      "activations/layer4_attention_weight_max": 92.8837890625,
      "activations/layer4_attention_weight_min": -88.04039764404297,
      "activations/layer5_attention_weight_max": 65.83236694335938,
      "activations/layer5_attention_weight_min": -75.87107849121094,
      "activations/layer6_attention_weight_max": 51.31137466430664,
      "activations/layer6_attention_weight_min": -52.081119537353516,
      "activations/layer7_attention_weight_max": 62.87486267089844,
      "activations/layer7_attention_weight_min": -61.425559997558594,
      "activations/layer8_attention_weight_max": 45.727508544921875,
      "activations/layer8_attention_weight_min": -50.99422073364258,
      "activations/layer9_attention_weight_max": 44.551631927490234,
      "activations/layer9_attention_weight_min": -46.14225387573242,
      "epoch": 17.86,
      "learning_rate": 3.518901515151515e-05,
      "loss": 2.7448,
      "step": 307350
    },
    {
      "activations/layer0_attention_weight_max": 15.459760665893555,
      "activations/layer0_attention_weight_min": -14.165508270263672,
      "activations/layer10_attention_weight_max": 36.276214599609375,
      "activations/layer10_attention_weight_min": -35.44343948364258,
      "activations/layer11_attention_weight_max": 34.651153564453125,
      "activations/layer11_attention_weight_min": -33.90325164794922,
      "activations/layer12_attention_weight_max": 18.98719024658203,
      "activations/layer12_attention_weight_min": -27.522428512573242,
      "activations/layer13_attention_weight_max": 34.88690948486328,
      "activations/layer13_attention_weight_min": -29.537715911865234,
      "activations/layer14_attention_weight_max": 35.218788146972656,
      "activations/layer14_attention_weight_min": -30.688095092773438,
      "activations/layer15_attention_weight_max": 31.728904724121094,
      "activations/layer15_attention_weight_min": -30.850208282470703,
      "activations/layer16_attention_weight_max": 35.68198776245117,
      "activations/layer16_attention_weight_min": -34.00040054321289,
      "activations/layer17_attention_weight_max": 52.93379211425781,
      "activations/layer17_attention_weight_min": -46.65337371826172,
      "activations/layer18_attention_weight_max": 49.6185188293457,
      "activations/layer18_attention_weight_min": -43.37517547607422,
      "activations/layer19_attention_weight_max": 26.004728317260742,
      "activations/layer19_attention_weight_min": -25.18376922607422,
      "activations/layer1_attention_weight_max": 16.299640655517578,
      "activations/layer1_attention_weight_min": -15.501545906066895,
      "activations/layer20_attention_weight_max": 23.280895233154297,
      "activations/layer20_attention_weight_min": -23.456083297729492,
      "activations/layer21_attention_weight_max": 36.928741455078125,
      "activations/layer21_attention_weight_min": -25.5092716217041,
      "activations/layer22_attention_weight_max": 31.603330612182617,
      "activations/layer22_attention_weight_min": -26.87981414794922,
      "activations/layer23_attention_weight_max": 38.433265686035156,
      "activations/layer23_attention_weight_min": -26.46759605407715,
      "activations/layer2_attention_weight_max": 29.842594146728516,
      "activations/layer2_attention_weight_min": -26.77338409423828,
      "activations/layer3_attention_weight_max": 88.91224670410156,
      "activations/layer3_attention_weight_min": -80.56480407714844,
      "activations/layer4_attention_weight_max": 89.6533432006836,
      "activations/layer4_attention_weight_min": -81.7612075805664,
      "activations/layer5_attention_weight_max": 63.3089599609375,
      "activations/layer5_attention_weight_min": -68.31288146972656,
      "activations/layer6_attention_weight_max": 48.88650894165039,
      "activations/layer6_attention_weight_min": -47.48087692260742,
      "activations/layer7_attention_weight_max": 60.4052848815918,
      "activations/layer7_attention_weight_min": -59.086822509765625,
      "activations/layer8_attention_weight_max": 46.81684112548828,
      "activations/layer8_attention_weight_min": -47.82902908325195,
      "activations/layer9_attention_weight_max": 44.70492172241211,
      "activations/layer9_attention_weight_min": -44.038631439208984,
      "epoch": 17.86,
      "learning_rate": 3.5170075757575754e-05,
      "loss": 2.7437,
      "step": 307400
    },
    {
      "activations/layer0_attention_weight_max": 16.590087890625,
      "activations/layer0_attention_weight_min": -13.548434257507324,
      "activations/layer10_attention_weight_max": 33.25465393066406,
      "activations/layer10_attention_weight_min": -34.787593841552734,
      "activations/layer11_attention_weight_max": 31.703840255737305,
      "activations/layer11_attention_weight_min": -33.215858459472656,
      "activations/layer12_attention_weight_max": 18.25335693359375,
      "activations/layer12_attention_weight_min": -25.627355575561523,
      "activations/layer13_attention_weight_max": 32.67475128173828,
      "activations/layer13_attention_weight_min": -27.537456512451172,
      "activations/layer14_attention_weight_max": 35.05299377441406,
      "activations/layer14_attention_weight_min": -31.30011749267578,
      "activations/layer15_attention_weight_max": 32.608062744140625,
      "activations/layer15_attention_weight_min": -29.63538360595703,
      "activations/layer16_attention_weight_max": 34.7572021484375,
      "activations/layer16_attention_weight_min": -32.68733215332031,
      "activations/layer17_attention_weight_max": 52.343807220458984,
      "activations/layer17_attention_weight_min": -44.45574188232422,
      "activations/layer18_attention_weight_max": 48.09623336791992,
      "activations/layer18_attention_weight_min": -40.90237808227539,
      "activations/layer19_attention_weight_max": 25.16996955871582,
      "activations/layer19_attention_weight_min": -22.05681610107422,
      "activations/layer1_attention_weight_max": 16.567522048950195,
      "activations/layer1_attention_weight_min": -13.700236320495605,
      "activations/layer20_attention_weight_max": 22.326622009277344,
      "activations/layer20_attention_weight_min": -21.659196853637695,
      "activations/layer21_attention_weight_max": 33.64662551879883,
      "activations/layer21_attention_weight_min": -21.549692153930664,
      "activations/layer22_attention_weight_max": 28.09507179260254,
      "activations/layer22_attention_weight_min": -24.67677879333496,
      "activations/layer23_attention_weight_max": 35.44916915893555,
      "activations/layer23_attention_weight_min": -25.109683990478516,
      "activations/layer2_attention_weight_max": 28.001943588256836,
      "activations/layer2_attention_weight_min": -28.358190536499023,
      "activations/layer3_attention_weight_max": 84.20577239990234,
      "activations/layer3_attention_weight_min": -84.7476577758789,
      "activations/layer4_attention_weight_max": 82.04032135009766,
      "activations/layer4_attention_weight_min": -85.57476043701172,
      "activations/layer5_attention_weight_max": 60.00464630126953,
      "activations/layer5_attention_weight_min": -74.44921875,
      "activations/layer6_attention_weight_max": 45.841285705566406,
      "activations/layer6_attention_weight_min": -50.17220687866211,
      "activations/layer7_attention_weight_max": 62.10860061645508,
      "activations/layer7_attention_weight_min": -60.839717864990234,
      "activations/layer8_attention_weight_max": 43.78032684326172,
      "activations/layer8_attention_weight_min": -48.21088409423828,
      "activations/layer9_attention_weight_max": 39.53384780883789,
      "activations/layer9_attention_weight_min": -42.73382568359375,
      "epoch": 17.86,
      "learning_rate": 3.515113636363636e-05,
      "loss": 2.7421,
      "step": 307450
    },
    {
      "activations/layer0_attention_weight_max": 15.544207572937012,
      "activations/layer0_attention_weight_min": -13.871611595153809,
      "activations/layer10_attention_weight_max": 35.65083694458008,
      "activations/layer10_attention_weight_min": -34.406341552734375,
      "activations/layer11_attention_weight_max": 33.83429718017578,
      "activations/layer11_attention_weight_min": -35.01652145385742,
      "activations/layer12_attention_weight_max": 18.97504234313965,
      "activations/layer12_attention_weight_min": -26.38759994506836,
      "activations/layer13_attention_weight_max": 28.530914306640625,
      "activations/layer13_attention_weight_min": -27.608964920043945,
      "activations/layer14_attention_weight_max": 36.11178207397461,
      "activations/layer14_attention_weight_min": -31.996519088745117,
      "activations/layer15_attention_weight_max": 33.05623245239258,
      "activations/layer15_attention_weight_min": -31.05141830444336,
      "activations/layer16_attention_weight_max": 35.23928451538086,
      "activations/layer16_attention_weight_min": -33.59436798095703,
      "activations/layer17_attention_weight_max": 53.83188247680664,
      "activations/layer17_attention_weight_min": -45.522735595703125,
      "activations/layer18_attention_weight_max": 48.78812789916992,
      "activations/layer18_attention_weight_min": -42.51847457885742,
      "activations/layer19_attention_weight_max": 23.858444213867188,
      "activations/layer19_attention_weight_min": -23.69984245300293,
      "activations/layer1_attention_weight_max": 15.067500114440918,
      "activations/layer1_attention_weight_min": -13.612483978271484,
      "activations/layer20_attention_weight_max": 20.366079330444336,
      "activations/layer20_attention_weight_min": -23.704862594604492,
      "activations/layer21_attention_weight_max": 31.60284996032715,
      "activations/layer21_attention_weight_min": -23.215953826904297,
      "activations/layer22_attention_weight_max": 28.09766960144043,
      "activations/layer22_attention_weight_min": -26.180280685424805,
      "activations/layer23_attention_weight_max": 35.36558532714844,
      "activations/layer23_attention_weight_min": -26.345035552978516,
      "activations/layer2_attention_weight_max": 30.149524688720703,
      "activations/layer2_attention_weight_min": -29.539342880249023,
      "activations/layer3_attention_weight_max": 88.54904174804688,
      "activations/layer3_attention_weight_min": -85.81964874267578,
      "activations/layer4_attention_weight_max": 82.57850646972656,
      "activations/layer4_attention_weight_min": -79.01290130615234,
      "activations/layer5_attention_weight_max": 61.697792053222656,
      "activations/layer5_attention_weight_min": -73.45187377929688,
      "activations/layer6_attention_weight_max": 48.764808654785156,
      "activations/layer6_attention_weight_min": -47.80587387084961,
      "activations/layer7_attention_weight_max": 64.26893615722656,
      "activations/layer7_attention_weight_min": -63.28938674926758,
      "activations/layer8_attention_weight_max": 48.070621490478516,
      "activations/layer8_attention_weight_min": -47.8172492980957,
      "activations/layer9_attention_weight_max": 45.01884078979492,
      "activations/layer9_attention_weight_min": -44.34315872192383,
      "epoch": 17.87,
      "learning_rate": 3.513219696969696e-05,
      "loss": 2.7297,
      "step": 307500
    },
    {
      "activations/layer0_attention_weight_max": 15.667692184448242,
      "activations/layer0_attention_weight_min": -13.921547889709473,
      "activations/layer10_attention_weight_max": 36.21432113647461,
      "activations/layer10_attention_weight_min": -36.0811882019043,
      "activations/layer11_attention_weight_max": 32.907249450683594,
      "activations/layer11_attention_weight_min": -35.22907257080078,
      "activations/layer12_attention_weight_max": 19.54349136352539,
      "activations/layer12_attention_weight_min": -23.453964233398438,
      "activations/layer13_attention_weight_max": 32.42759704589844,
      "activations/layer13_attention_weight_min": -27.35674476623535,
      "activations/layer14_attention_weight_max": 37.176239013671875,
      "activations/layer14_attention_weight_min": -29.870655059814453,
      "activations/layer15_attention_weight_max": 33.64038848876953,
      "activations/layer15_attention_weight_min": -31.701156616210938,
      "activations/layer16_attention_weight_max": 33.82785415649414,
      "activations/layer16_attention_weight_min": -30.707168579101562,
      "activations/layer17_attention_weight_max": 56.18239212036133,
      "activations/layer17_attention_weight_min": -45.270084381103516,
      "activations/layer18_attention_weight_max": 46.00875473022461,
      "activations/layer18_attention_weight_min": -37.58742904663086,
      "activations/layer19_attention_weight_max": 25.44083023071289,
      "activations/layer19_attention_weight_min": -23.862539291381836,
      "activations/layer1_attention_weight_max": 15.755733489990234,
      "activations/layer1_attention_weight_min": -13.751751899719238,
      "activations/layer20_attention_weight_max": 22.504220962524414,
      "activations/layer20_attention_weight_min": -23.290517807006836,
      "activations/layer21_attention_weight_max": 38.67565155029297,
      "activations/layer21_attention_weight_min": -23.44034194946289,
      "activations/layer22_attention_weight_max": 27.85749053955078,
      "activations/layer22_attention_weight_min": -27.32585906982422,
      "activations/layer23_attention_weight_max": 38.73394775390625,
      "activations/layer23_attention_weight_min": -24.608848571777344,
      "activations/layer2_attention_weight_max": 29.665748596191406,
      "activations/layer2_attention_weight_min": -28.304479598999023,
      "activations/layer3_attention_weight_max": 81.83545684814453,
      "activations/layer3_attention_weight_min": -85.4090347290039,
      "activations/layer4_attention_weight_max": 83.59244537353516,
      "activations/layer4_attention_weight_min": -81.69468688964844,
      "activations/layer5_attention_weight_max": 60.33149719238281,
      "activations/layer5_attention_weight_min": -70.8673324584961,
      "activations/layer6_attention_weight_max": 47.30276870727539,
      "activations/layer6_attention_weight_min": -48.270320892333984,
      "activations/layer7_attention_weight_max": 60.7491455078125,
      "activations/layer7_attention_weight_min": -59.799560546875,
      "activations/layer8_attention_weight_max": 43.23282241821289,
      "activations/layer8_attention_weight_min": -52.12233352661133,
      "activations/layer9_attention_weight_max": 44.460357666015625,
      "activations/layer9_attention_weight_min": -47.48516845703125,
      "epoch": 17.87,
      "learning_rate": 3.511325757575757e-05,
      "loss": 2.7636,
      "step": 307550
    },
    {
      "activations/layer0_attention_weight_max": 16.04411506652832,
      "activations/layer0_attention_weight_min": -13.754975318908691,
      "activations/layer10_attention_weight_max": 33.17279052734375,
      "activations/layer10_attention_weight_min": -33.58387756347656,
      "activations/layer11_attention_weight_max": 31.564786911010742,
      "activations/layer11_attention_weight_min": -32.441715240478516,
      "activations/layer12_attention_weight_max": 20.86723518371582,
      "activations/layer12_attention_weight_min": -25.581199645996094,
      "activations/layer13_attention_weight_max": 32.6822509765625,
      "activations/layer13_attention_weight_min": -29.10585594177246,
      "activations/layer14_attention_weight_max": 34.724605560302734,
      "activations/layer14_attention_weight_min": -31.384275436401367,
      "activations/layer15_attention_weight_max": 32.67143249511719,
      "activations/layer15_attention_weight_min": -30.761465072631836,
      "activations/layer16_attention_weight_max": 33.73585510253906,
      "activations/layer16_attention_weight_min": -31.210966110229492,
      "activations/layer17_attention_weight_max": 50.56562805175781,
      "activations/layer17_attention_weight_min": -45.10548782348633,
      "activations/layer18_attention_weight_max": 48.36697769165039,
      "activations/layer18_attention_weight_min": -38.059669494628906,
      "activations/layer19_attention_weight_max": 24.05482292175293,
      "activations/layer19_attention_weight_min": -21.999473571777344,
      "activations/layer1_attention_weight_max": 15.887395858764648,
      "activations/layer1_attention_weight_min": -15.406144142150879,
      "activations/layer20_attention_weight_max": 22.414207458496094,
      "activations/layer20_attention_weight_min": -24.202421188354492,
      "activations/layer21_attention_weight_max": 33.92380905151367,
      "activations/layer21_attention_weight_min": -22.7042236328125,
      "activations/layer22_attention_weight_max": 28.41630744934082,
      "activations/layer22_attention_weight_min": -28.342418670654297,
      "activations/layer23_attention_weight_max": 35.23771286010742,
      "activations/layer23_attention_weight_min": -26.755292892456055,
      "activations/layer2_attention_weight_max": 30.42683219909668,
      "activations/layer2_attention_weight_min": -29.39611053466797,
      "activations/layer3_attention_weight_max": 89.51754760742188,
      "activations/layer3_attention_weight_min": -88.63021087646484,
      "activations/layer4_attention_weight_max": 87.85501098632812,
      "activations/layer4_attention_weight_min": -85.62872314453125,
      "activations/layer5_attention_weight_max": 63.928558349609375,
      "activations/layer5_attention_weight_min": -72.5306167602539,
      "activations/layer6_attention_weight_max": 50.07088088989258,
      "activations/layer6_attention_weight_min": -48.69606399536133,
      "activations/layer7_attention_weight_max": 61.84976577758789,
      "activations/layer7_attention_weight_min": -64.78096008300781,
      "activations/layer8_attention_weight_max": 44.1695442199707,
      "activations/layer8_attention_weight_min": -48.44954299926758,
      "activations/layer9_attention_weight_max": 42.492652893066406,
      "activations/layer9_attention_weight_min": -43.74851989746094,
      "epoch": 17.87,
      "learning_rate": 3.509431818181818e-05,
      "loss": 2.7473,
      "step": 307600
    },
    {
      "activations/layer0_attention_weight_max": 15.378300666809082,
      "activations/layer0_attention_weight_min": -13.532848358154297,
      "activations/layer10_attention_weight_max": 38.611663818359375,
      "activations/layer10_attention_weight_min": -37.16794204711914,
      "activations/layer11_attention_weight_max": 33.925743103027344,
      "activations/layer11_attention_weight_min": -35.3432731628418,
      "activations/layer12_attention_weight_max": 18.854721069335938,
      "activations/layer12_attention_weight_min": -25.34353256225586,
      "activations/layer13_attention_weight_max": 33.95384216308594,
      "activations/layer13_attention_weight_min": -26.90525245666504,
      "activations/layer14_attention_weight_max": 34.164100646972656,
      "activations/layer14_attention_weight_min": -31.46650505065918,
      "activations/layer15_attention_weight_max": 32.581642150878906,
      "activations/layer15_attention_weight_min": -32.622344970703125,
      "activations/layer16_attention_weight_max": 34.243350982666016,
      "activations/layer16_attention_weight_min": -33.748451232910156,
      "activations/layer17_attention_weight_max": 51.99831008911133,
      "activations/layer17_attention_weight_min": -48.00838088989258,
      "activations/layer18_attention_weight_max": 48.39680099487305,
      "activations/layer18_attention_weight_min": -39.187347412109375,
      "activations/layer19_attention_weight_max": 26.09207534790039,
      "activations/layer19_attention_weight_min": -23.04853630065918,
      "activations/layer1_attention_weight_max": 16.99250602722168,
      "activations/layer1_attention_weight_min": -14.626657485961914,
      "activations/layer20_attention_weight_max": 23.110754013061523,
      "activations/layer20_attention_weight_min": -23.527257919311523,
      "activations/layer21_attention_weight_max": 38.37739944458008,
      "activations/layer21_attention_weight_min": -23.589567184448242,
      "activations/layer22_attention_weight_max": 31.009199142456055,
      "activations/layer22_attention_weight_min": -27.236875534057617,
      "activations/layer23_attention_weight_max": 39.75834655761719,
      "activations/layer23_attention_weight_min": -25.6007080078125,
      "activations/layer2_attention_weight_max": 31.516830444335938,
      "activations/layer2_attention_weight_min": -28.572059631347656,
      "activations/layer3_attention_weight_max": 86.46492767333984,
      "activations/layer3_attention_weight_min": -86.57582092285156,
      "activations/layer4_attention_weight_max": 88.33792114257812,
      "activations/layer4_attention_weight_min": -83.81462860107422,
      "activations/layer5_attention_weight_max": 64.28559875488281,
      "activations/layer5_attention_weight_min": -76.07859802246094,
      "activations/layer6_attention_weight_max": 50.749000549316406,
      "activations/layer6_attention_weight_min": -49.819522857666016,
      "activations/layer7_attention_weight_max": 64.97233581542969,
      "activations/layer7_attention_weight_min": -63.81904983520508,
      "activations/layer8_attention_weight_max": 47.84614562988281,
      "activations/layer8_attention_weight_min": -48.8458137512207,
      "activations/layer9_attention_weight_max": 47.304683685302734,
      "activations/layer9_attention_weight_min": -45.74968338012695,
      "epoch": 17.88,
      "learning_rate": 3.507575757575757e-05,
      "loss": 2.7363,
      "step": 307650
    },
    {
      "activations/layer0_attention_weight_max": 14.963374137878418,
      "activations/layer0_attention_weight_min": -14.084269523620605,
      "activations/layer10_attention_weight_max": 40.4213752746582,
      "activations/layer10_attention_weight_min": -37.78392028808594,
      "activations/layer11_attention_weight_max": 37.29942321777344,
      "activations/layer11_attention_weight_min": -36.7981071472168,
      "activations/layer12_attention_weight_max": 22.615310668945312,
      "activations/layer12_attention_weight_min": -23.033533096313477,
      "activations/layer13_attention_weight_max": 46.11274337768555,
      "activations/layer13_attention_weight_min": -29.917619705200195,
      "activations/layer14_attention_weight_max": 41.3395881652832,
      "activations/layer14_attention_weight_min": -32.953399658203125,
      "activations/layer15_attention_weight_max": 38.9130973815918,
      "activations/layer15_attention_weight_min": -32.86125564575195,
      "activations/layer16_attention_weight_max": 42.78148651123047,
      "activations/layer16_attention_weight_min": -37.46251678466797,
      "activations/layer17_attention_weight_max": 61.17766571044922,
      "activations/layer17_attention_weight_min": -48.05207061767578,
      "activations/layer18_attention_weight_max": 58.81748580932617,
      "activations/layer18_attention_weight_min": -45.39229965209961,
      "activations/layer19_attention_weight_max": 32.321468353271484,
      "activations/layer19_attention_weight_min": -25.34554100036621,
      "activations/layer1_attention_weight_max": 15.416863441467285,
      "activations/layer1_attention_weight_min": -13.499649047851562,
      "activations/layer20_attention_weight_max": 28.307022094726562,
      "activations/layer20_attention_weight_min": -21.615720748901367,
      "activations/layer21_attention_weight_max": 41.6241569519043,
      "activations/layer21_attention_weight_min": -23.507877349853516,
      "activations/layer22_attention_weight_max": 30.13136100769043,
      "activations/layer22_attention_weight_min": -25.192508697509766,
      "activations/layer23_attention_weight_max": 37.67122268676758,
      "activations/layer23_attention_weight_min": -25.15570068359375,
      "activations/layer2_attention_weight_max": 29.69851303100586,
      "activations/layer2_attention_weight_min": -30.958702087402344,
      "activations/layer3_attention_weight_max": 82.9727783203125,
      "activations/layer3_attention_weight_min": -91.70285034179688,
      "activations/layer4_attention_weight_max": 86.0587158203125,
      "activations/layer4_attention_weight_min": -86.12325286865234,
      "activations/layer5_attention_weight_max": 64.2960205078125,
      "activations/layer5_attention_weight_min": -69.59637451171875,
      "activations/layer6_attention_weight_max": 47.23953628540039,
      "activations/layer6_attention_weight_min": -50.14380645751953,
      "activations/layer7_attention_weight_max": 63.38825225830078,
      "activations/layer7_attention_weight_min": -59.70695877075195,
      "activations/layer8_attention_weight_max": 49.3245849609375,
      "activations/layer8_attention_weight_min": -51.75132369995117,
      "activations/layer9_attention_weight_max": 46.530982971191406,
      "activations/layer9_attention_weight_min": -47.46305465698242,
      "epoch": 17.88,
      "learning_rate": 3.505681818181818e-05,
      "loss": 2.7433,
      "step": 307700
    },
    {
      "activations/layer0_attention_weight_max": 15.237588882446289,
      "activations/layer0_attention_weight_min": -13.638774871826172,
      "activations/layer10_attention_weight_max": 35.4562873840332,
      "activations/layer10_attention_weight_min": -36.42613220214844,
      "activations/layer11_attention_weight_max": 31.052635192871094,
      "activations/layer11_attention_weight_min": -34.05155944824219,
      "activations/layer12_attention_weight_max": 20.330196380615234,
      "activations/layer12_attention_weight_min": -26.28636360168457,
      "activations/layer13_attention_weight_max": 32.22453308105469,
      "activations/layer13_attention_weight_min": -31.827472686767578,
      "activations/layer14_attention_weight_max": 35.250831604003906,
      "activations/layer14_attention_weight_min": -31.277448654174805,
      "activations/layer15_attention_weight_max": 32.8847541809082,
      "activations/layer15_attention_weight_min": -30.723997116088867,
      "activations/layer16_attention_weight_max": 37.974334716796875,
      "activations/layer16_attention_weight_min": -35.69889831542969,
      "activations/layer17_attention_weight_max": 56.90993118286133,
      "activations/layer17_attention_weight_min": -49.18238830566406,
      "activations/layer18_attention_weight_max": 54.32781219482422,
      "activations/layer18_attention_weight_min": -46.80404281616211,
      "activations/layer19_attention_weight_max": 25.32452392578125,
      "activations/layer19_attention_weight_min": -25.159578323364258,
      "activations/layer1_attention_weight_max": 16.048643112182617,
      "activations/layer1_attention_weight_min": -12.991689682006836,
      "activations/layer20_attention_weight_max": 21.864591598510742,
      "activations/layer20_attention_weight_min": -22.511436462402344,
      "activations/layer21_attention_weight_max": 37.48551559448242,
      "activations/layer21_attention_weight_min": -24.567813873291016,
      "activations/layer22_attention_weight_max": 31.690765380859375,
      "activations/layer22_attention_weight_min": -30.106733322143555,
      "activations/layer23_attention_weight_max": 34.501564025878906,
      "activations/layer23_attention_weight_min": -27.901145935058594,
      "activations/layer2_attention_weight_max": 28.154760360717773,
      "activations/layer2_attention_weight_min": -26.982755661010742,
      "activations/layer3_attention_weight_max": 77.09355163574219,
      "activations/layer3_attention_weight_min": -81.6361083984375,
      "activations/layer4_attention_weight_max": 82.49207305908203,
      "activations/layer4_attention_weight_min": -84.57443237304688,
      "activations/layer5_attention_weight_max": 62.719398498535156,
      "activations/layer5_attention_weight_min": -71.39299011230469,
      "activations/layer6_attention_weight_max": 47.50164031982422,
      "activations/layer6_attention_weight_min": -50.07435607910156,
      "activations/layer7_attention_weight_max": 65.44171905517578,
      "activations/layer7_attention_weight_min": -63.56806564331055,
      "activations/layer8_attention_weight_max": 46.175662994384766,
      "activations/layer8_attention_weight_min": -49.00126266479492,
      "activations/layer9_attention_weight_max": 48.17478561401367,
      "activations/layer9_attention_weight_min": -46.50596237182617,
      "epoch": 17.88,
      "learning_rate": 3.503787878787879e-05,
      "loss": 2.7465,
      "step": 307750
    },
    {
      "activations/layer0_attention_weight_max": 17.054584503173828,
      "activations/layer0_attention_weight_min": -13.532143592834473,
      "activations/layer10_attention_weight_max": 37.953487396240234,
      "activations/layer10_attention_weight_min": -36.1076545715332,
      "activations/layer11_attention_weight_max": 32.50373077392578,
      "activations/layer11_attention_weight_min": -35.029266357421875,
      "activations/layer12_attention_weight_max": 18.412710189819336,
      "activations/layer12_attention_weight_min": -22.26894760131836,
      "activations/layer13_attention_weight_max": 33.879913330078125,
      "activations/layer13_attention_weight_min": -28.10177230834961,
      "activations/layer14_attention_weight_max": 33.73527145385742,
      "activations/layer14_attention_weight_min": -29.777986526489258,
      "activations/layer15_attention_weight_max": 33.704036712646484,
      "activations/layer15_attention_weight_min": -29.819473266601562,
      "activations/layer16_attention_weight_max": 33.368099212646484,
      "activations/layer16_attention_weight_min": -30.90044403076172,
      "activations/layer17_attention_weight_max": 49.8788948059082,
      "activations/layer17_attention_weight_min": -43.35003662109375,
      "activations/layer18_attention_weight_max": 45.50164794921875,
      "activations/layer18_attention_weight_min": -39.65179443359375,
      "activations/layer19_attention_weight_max": 24.06339454650879,
      "activations/layer19_attention_weight_min": -23.25699234008789,
      "activations/layer1_attention_weight_max": 15.546555519104004,
      "activations/layer1_attention_weight_min": -13.734238624572754,
      "activations/layer20_attention_weight_max": 24.423322677612305,
      "activations/layer20_attention_weight_min": -22.128257751464844,
      "activations/layer21_attention_weight_max": 36.041011810302734,
      "activations/layer21_attention_weight_min": -22.316146850585938,
      "activations/layer22_attention_weight_max": 31.038578033447266,
      "activations/layer22_attention_weight_min": -24.288555145263672,
      "activations/layer23_attention_weight_max": 39.45765686035156,
      "activations/layer23_attention_weight_min": -24.9031982421875,
      "activations/layer2_attention_weight_max": 30.15864372253418,
      "activations/layer2_attention_weight_min": -30.011884689331055,
      "activations/layer3_attention_weight_max": 85.72001647949219,
      "activations/layer3_attention_weight_min": -88.27210998535156,
      "activations/layer4_attention_weight_max": 88.37511444091797,
      "activations/layer4_attention_weight_min": -88.42990112304688,
      "activations/layer5_attention_weight_max": 62.117733001708984,
      "activations/layer5_attention_weight_min": -76.24925994873047,
      "activations/layer6_attention_weight_max": 50.605796813964844,
      "activations/layer6_attention_weight_min": -51.38114547729492,
      "activations/layer7_attention_weight_max": 64.94141387939453,
      "activations/layer7_attention_weight_min": -63.85794448852539,
      "activations/layer8_attention_weight_max": 51.12126922607422,
      "activations/layer8_attention_weight_min": -55.30527877807617,
      "activations/layer9_attention_weight_max": 47.27424240112305,
      "activations/layer9_attention_weight_min": -48.284149169921875,
      "epoch": 17.88,
      "learning_rate": 3.501893939393939e-05,
      "loss": 2.7425,
      "step": 307800
    },
    {
      "activations/layer0_attention_weight_max": 15.253055572509766,
      "activations/layer0_attention_weight_min": -13.171792984008789,
      "activations/layer10_attention_weight_max": 35.468841552734375,
      "activations/layer10_attention_weight_min": -33.72006607055664,
      "activations/layer11_attention_weight_max": 33.43565368652344,
      "activations/layer11_attention_weight_min": -33.69080352783203,
      "activations/layer12_attention_weight_max": 19.26473045349121,
      "activations/layer12_attention_weight_min": -28.13633918762207,
      "activations/layer13_attention_weight_max": 36.761260986328125,
      "activations/layer13_attention_weight_min": -29.439437866210938,
      "activations/layer14_attention_weight_max": 33.567169189453125,
      "activations/layer14_attention_weight_min": -29.855703353881836,
      "activations/layer15_attention_weight_max": 31.62228775024414,
      "activations/layer15_attention_weight_min": -31.650320053100586,
      "activations/layer16_attention_weight_max": 31.50177001953125,
      "activations/layer16_attention_weight_min": -28.84145736694336,
      "activations/layer17_attention_weight_max": 49.610782623291016,
      "activations/layer17_attention_weight_min": -42.390357971191406,
      "activations/layer18_attention_weight_max": 43.75602722167969,
      "activations/layer18_attention_weight_min": -37.98588562011719,
      "activations/layer19_attention_weight_max": 22.457422256469727,
      "activations/layer19_attention_weight_min": -22.183691024780273,
      "activations/layer1_attention_weight_max": 15.700042724609375,
      "activations/layer1_attention_weight_min": -13.663971900939941,
      "activations/layer20_attention_weight_max": 21.52118492126465,
      "activations/layer20_attention_weight_min": -21.261083602905273,
      "activations/layer21_attention_weight_max": 38.333683013916016,
      "activations/layer21_attention_weight_min": -20.868831634521484,
      "activations/layer22_attention_weight_max": 28.62748146057129,
      "activations/layer22_attention_weight_min": -25.4947509765625,
      "activations/layer23_attention_weight_max": 37.639461517333984,
      "activations/layer23_attention_weight_min": -24.077457427978516,
      "activations/layer2_attention_weight_max": 30.615299224853516,
      "activations/layer2_attention_weight_min": -29.388690948486328,
      "activations/layer3_attention_weight_max": 83.94987487792969,
      "activations/layer3_attention_weight_min": -86.40567779541016,
      "activations/layer4_attention_weight_max": 81.43651580810547,
      "activations/layer4_attention_weight_min": -85.29816436767578,
      "activations/layer5_attention_weight_max": 60.83732223510742,
      "activations/layer5_attention_weight_min": -71.75094604492188,
      "activations/layer6_attention_weight_max": 47.49149703979492,
      "activations/layer6_attention_weight_min": -47.32766342163086,
      "activations/layer7_attention_weight_max": 62.20018005371094,
      "activations/layer7_attention_weight_min": -61.8525276184082,
      "activations/layer8_attention_weight_max": 44.11253356933594,
      "activations/layer8_attention_weight_min": -48.86538314819336,
      "activations/layer9_attention_weight_max": 43.73141098022461,
      "activations/layer9_attention_weight_min": -43.73080825805664,
      "epoch": 17.89,
      "learning_rate": 3.5e-05,
      "loss": 2.7333,
      "step": 307850
    },
    {
      "activations/layer0_attention_weight_max": 14.408225059509277,
      "activations/layer0_attention_weight_min": -14.239151000976562,
      "activations/layer10_attention_weight_max": 36.016693115234375,
      "activations/layer10_attention_weight_min": -36.01787185668945,
      "activations/layer11_attention_weight_max": 32.24892807006836,
      "activations/layer11_attention_weight_min": -34.865333557128906,
      "activations/layer12_attention_weight_max": 18.12217140197754,
      "activations/layer12_attention_weight_min": -26.07806968688965,
      "activations/layer13_attention_weight_max": 31.100048065185547,
      "activations/layer13_attention_weight_min": -27.705150604248047,
      "activations/layer14_attention_weight_max": 33.06481170654297,
      "activations/layer14_attention_weight_min": -30.148910522460938,
      "activations/layer15_attention_weight_max": 33.36201095581055,
      "activations/layer15_attention_weight_min": -29.43103790283203,
      "activations/layer16_attention_weight_max": 33.076866149902344,
      "activations/layer16_attention_weight_min": -31.81290626525879,
      "activations/layer17_attention_weight_max": 48.67757797241211,
      "activations/layer17_attention_weight_min": -44.47487258911133,
      "activations/layer18_attention_weight_max": 47.80236053466797,
      "activations/layer18_attention_weight_min": -40.03151321411133,
      "activations/layer19_attention_weight_max": 23.549985885620117,
      "activations/layer19_attention_weight_min": -23.23128890991211,
      "activations/layer1_attention_weight_max": 15.126675605773926,
      "activations/layer1_attention_weight_min": -14.06098747253418,
      "activations/layer20_attention_weight_max": 23.628095626831055,
      "activations/layer20_attention_weight_min": -22.384532928466797,
      "activations/layer21_attention_weight_max": 37.1462287902832,
      "activations/layer21_attention_weight_min": -25.294645309448242,
      "activations/layer22_attention_weight_max": 29.696313858032227,
      "activations/layer22_attention_weight_min": -26.51639175415039,
      "activations/layer23_attention_weight_max": 39.334896087646484,
      "activations/layer23_attention_weight_min": -27.527402877807617,
      "activations/layer2_attention_weight_max": 30.750770568847656,
      "activations/layer2_attention_weight_min": -30.083782196044922,
      "activations/layer3_attention_weight_max": 84.80606079101562,
      "activations/layer3_attention_weight_min": -88.7570571899414,
      "activations/layer4_attention_weight_max": 85.70541381835938,
      "activations/layer4_attention_weight_min": -87.64508819580078,
      "activations/layer5_attention_weight_max": 60.50328826904297,
      "activations/layer5_attention_weight_min": -71.67236328125,
      "activations/layer6_attention_weight_max": 48.66025924682617,
      "activations/layer6_attention_weight_min": -49.28398132324219,
      "activations/layer7_attention_weight_max": 62.31028366088867,
      "activations/layer7_attention_weight_min": -62.952735900878906,
      "activations/layer8_attention_weight_max": 44.192283630371094,
      "activations/layer8_attention_weight_min": -47.973270416259766,
      "activations/layer9_attention_weight_max": 43.049922943115234,
      "activations/layer9_attention_weight_min": -46.79304504394531,
      "epoch": 17.89,
      "learning_rate": 3.49810606060606e-05,
      "loss": 2.7439,
      "step": 307900
    },
    {
      "activations/layer0_attention_weight_max": 15.028833389282227,
      "activations/layer0_attention_weight_min": -13.356361389160156,
      "activations/layer10_attention_weight_max": 44.9827995300293,
      "activations/layer10_attention_weight_min": -43.86853790283203,
      "activations/layer11_attention_weight_max": 43.22760009765625,
      "activations/layer11_attention_weight_min": -42.13728332519531,
      "activations/layer12_attention_weight_max": 19.824724197387695,
      "activations/layer12_attention_weight_min": -26.607418060302734,
      "activations/layer13_attention_weight_max": 36.60895919799805,
      "activations/layer13_attention_weight_min": -28.80584716796875,
      "activations/layer14_attention_weight_max": 38.59656524658203,
      "activations/layer14_attention_weight_min": -34.16008758544922,
      "activations/layer15_attention_weight_max": 40.253173828125,
      "activations/layer15_attention_weight_min": -32.759464263916016,
      "activations/layer16_attention_weight_max": 40.409358978271484,
      "activations/layer16_attention_weight_min": -37.4376220703125,
      "activations/layer17_attention_weight_max": 62.48963165283203,
      "activations/layer17_attention_weight_min": -57.25017547607422,
      "activations/layer18_attention_weight_max": 55.12866973876953,
      "activations/layer18_attention_weight_min": -50.17826843261719,
      "activations/layer19_attention_weight_max": 24.715734481811523,
      "activations/layer19_attention_weight_min": -25.036245346069336,
      "activations/layer1_attention_weight_max": 16.186738967895508,
      "activations/layer1_attention_weight_min": -15.022110939025879,
      "activations/layer20_attention_weight_max": 25.379087448120117,
      "activations/layer20_attention_weight_min": -21.871349334716797,
      "activations/layer21_attention_weight_max": 40.8692626953125,
      "activations/layer21_attention_weight_min": -21.579139709472656,
      "activations/layer22_attention_weight_max": 33.114139556884766,
      "activations/layer22_attention_weight_min": -25.82306671142578,
      "activations/layer23_attention_weight_max": 34.5736083984375,
      "activations/layer23_attention_weight_min": -24.97557830810547,
      "activations/layer2_attention_weight_max": 32.51869583129883,
      "activations/layer2_attention_weight_min": -31.750219345092773,
      "activations/layer3_attention_weight_max": 97.18013763427734,
      "activations/layer3_attention_weight_min": -98.02676391601562,
      "activations/layer4_attention_weight_max": 94.60303497314453,
      "activations/layer4_attention_weight_min": -91.90298461914062,
      "activations/layer5_attention_weight_max": 72.79032897949219,
      "activations/layer5_attention_weight_min": -72.87757873535156,
      "activations/layer6_attention_weight_max": 50.811439514160156,
      "activations/layer6_attention_weight_min": -51.884159088134766,
      "activations/layer7_attention_weight_max": 79.10318756103516,
      "activations/layer7_attention_weight_min": -69.71867370605469,
      "activations/layer8_attention_weight_max": 57.66330337524414,
      "activations/layer8_attention_weight_min": -55.54820251464844,
      "activations/layer9_attention_weight_max": 63.87166213989258,
      "activations/layer9_attention_weight_min": -53.33034896850586,
      "epoch": 17.89,
      "learning_rate": 3.496212121212121e-05,
      "loss": 2.7447,
      "step": 307950
    },
    {
      "activations/layer0_attention_weight_max": 14.861994743347168,
      "activations/layer0_attention_weight_min": -13.371292114257812,
      "activations/layer10_attention_weight_max": 38.27021026611328,
      "activations/layer10_attention_weight_min": -38.189979553222656,
      "activations/layer11_attention_weight_max": 31.587507247924805,
      "activations/layer11_attention_weight_min": -33.28467559814453,
      "activations/layer12_attention_weight_max": 17.75324058532715,
      "activations/layer12_attention_weight_min": -29.41010093688965,
      "activations/layer13_attention_weight_max": 43.9996223449707,
      "activations/layer13_attention_weight_min": -30.667041778564453,
      "activations/layer14_attention_weight_max": 34.91932678222656,
      "activations/layer14_attention_weight_min": -29.18783950805664,
      "activations/layer15_attention_weight_max": 31.650066375732422,
      "activations/layer15_attention_weight_min": -30.45499038696289,
      "activations/layer16_attention_weight_max": 30.76616096496582,
      "activations/layer16_attention_weight_min": -29.675111770629883,
      "activations/layer17_attention_weight_max": 49.26869583129883,
      "activations/layer17_attention_weight_min": -44.29008865356445,
      "activations/layer18_attention_weight_max": 44.79115295410156,
      "activations/layer18_attention_weight_min": -40.71257019042969,
      "activations/layer19_attention_weight_max": 23.688013076782227,
      "activations/layer19_attention_weight_min": -21.63330078125,
      "activations/layer1_attention_weight_max": 15.65722370147705,
      "activations/layer1_attention_weight_min": -13.405661582946777,
      "activations/layer20_attention_weight_max": 22.94931983947754,
      "activations/layer20_attention_weight_min": -23.991283416748047,
      "activations/layer21_attention_weight_max": 34.46229934692383,
      "activations/layer21_attention_weight_min": -22.456222534179688,
      "activations/layer22_attention_weight_max": 32.014686584472656,
      "activations/layer22_attention_weight_min": -25.509687423706055,
      "activations/layer23_attention_weight_max": 36.781620025634766,
      "activations/layer23_attention_weight_min": -24.53247833251953,
      "activations/layer2_attention_weight_max": 31.03809928894043,
      "activations/layer2_attention_weight_min": -30.08859634399414,
      "activations/layer3_attention_weight_max": 86.69792938232422,
      "activations/layer3_attention_weight_min": -86.42243957519531,
      "activations/layer4_attention_weight_max": 83.8680648803711,
      "activations/layer4_attention_weight_min": -84.36707305908203,
      "activations/layer5_attention_weight_max": 62.60804748535156,
      "activations/layer5_attention_weight_min": -72.81809997558594,
      "activations/layer6_attention_weight_max": 48.18462371826172,
      "activations/layer6_attention_weight_min": -49.79588317871094,
      "activations/layer7_attention_weight_max": 60.18674850463867,
      "activations/layer7_attention_weight_min": -59.455596923828125,
      "activations/layer8_attention_weight_max": 46.421504974365234,
      "activations/layer8_attention_weight_min": -52.89585876464844,
      "activations/layer9_attention_weight_max": 43.53178024291992,
      "activations/layer9_attention_weight_min": -46.33964157104492,
      "epoch": 17.9,
      "learning_rate": 3.4943181818181815e-05,
      "loss": 2.7439,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4495,
      "eval_samples_per_second": 508.198,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4495,
      "eval_openwebtext_samples_per_second": 508.198,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9718,
      "eval_wikitext_samples_per_second": 231.267,
      "step": 308000
    },
    {
      "epoch": 17.9,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5535,
      "eval_lambada_samples_per_second": 509.655,
      "step": 308000
    },
    {
      "activations/layer0_attention_weight_max": 15.497808456420898,
      "activations/layer0_attention_weight_min": -13.452178001403809,
      "activations/layer10_attention_weight_max": 35.16377258300781,
      "activations/layer10_attention_weight_min": -36.120758056640625,
      "activations/layer11_attention_weight_max": 31.926692962646484,
      "activations/layer11_attention_weight_min": -34.09189224243164,
      "activations/layer12_attention_weight_max": 18.438562393188477,
      "activations/layer12_attention_weight_min": -26.028337478637695,
      "activations/layer13_attention_weight_max": 33.2213249206543,
      "activations/layer13_attention_weight_min": -31.83504867553711,
      "activations/layer14_attention_weight_max": 34.92729187011719,
      "activations/layer14_attention_weight_min": -30.453529357910156,
      "activations/layer15_attention_weight_max": 34.6263313293457,
      "activations/layer15_attention_weight_min": -32.12071228027344,
      "activations/layer16_attention_weight_max": 35.20058822631836,
      "activations/layer16_attention_weight_min": -30.827587127685547,
      "activations/layer17_attention_weight_max": 50.5653076171875,
      "activations/layer17_attention_weight_min": -44.92313003540039,
      "activations/layer18_attention_weight_max": 48.961830139160156,
      "activations/layer18_attention_weight_min": -38.90795135498047,
      "activations/layer19_attention_weight_max": 24.98891830444336,
      "activations/layer19_attention_weight_min": -22.532238006591797,
      "activations/layer1_attention_weight_max": 15.611801147460938,
      "activations/layer1_attention_weight_min": -13.449319839477539,
      "activations/layer20_attention_weight_max": 23.343917846679688,
      "activations/layer20_attention_weight_min": -21.68699836730957,
      "activations/layer21_attention_weight_max": 34.45139694213867,
      "activations/layer21_attention_weight_min": -22.883832931518555,
      "activations/layer22_attention_weight_max": 29.452831268310547,
      "activations/layer22_attention_weight_min": -26.41189193725586,
      "activations/layer23_attention_weight_max": 32.388084411621094,
      "activations/layer23_attention_weight_min": -23.718978881835938,
      "activations/layer2_attention_weight_max": 31.315876007080078,
      "activations/layer2_attention_weight_min": -31.612951278686523,
      "activations/layer3_attention_weight_max": 91.07501983642578,
      "activations/layer3_attention_weight_min": -95.48194122314453,
      "activations/layer4_attention_weight_max": 83.87444305419922,
      "activations/layer4_attention_weight_min": -87.2583999633789,
      "activations/layer5_attention_weight_max": 61.69993591308594,
      "activations/layer5_attention_weight_min": -76.6115951538086,
      "activations/layer6_attention_weight_max": 48.77971267700195,
      "activations/layer6_attention_weight_min": -53.44837188720703,
      "activations/layer7_attention_weight_max": 63.090389251708984,
      "activations/layer7_attention_weight_min": -64.55757904052734,
      "activations/layer8_attention_weight_max": 46.648170471191406,
      "activations/layer8_attention_weight_min": -48.932212829589844,
      "activations/layer9_attention_weight_max": 43.12129592895508,
      "activations/layer9_attention_weight_min": -44.33301544189453,
      "epoch": 17.9,
      "learning_rate": 3.492424242424242e-05,
      "loss": 2.758,
      "step": 308050
    },
    {
      "activations/layer0_attention_weight_max": 15.478326797485352,
      "activations/layer0_attention_weight_min": -13.550028800964355,
      "activations/layer10_attention_weight_max": 34.20915603637695,
      "activations/layer10_attention_weight_min": -34.70623779296875,
      "activations/layer11_attention_weight_max": 29.662330627441406,
      "activations/layer11_attention_weight_min": -34.773067474365234,
      "activations/layer12_attention_weight_max": 17.731304168701172,
      "activations/layer12_attention_weight_min": -24.37133026123047,
      "activations/layer13_attention_weight_max": 30.00517463684082,
      "activations/layer13_attention_weight_min": -27.200313568115234,
      "activations/layer14_attention_weight_max": 32.70357131958008,
      "activations/layer14_attention_weight_min": -30.897287368774414,
      "activations/layer15_attention_weight_max": 33.16575241088867,
      "activations/layer15_attention_weight_min": -30.655500411987305,
      "activations/layer16_attention_weight_max": 30.63704490661621,
      "activations/layer16_attention_weight_min": -31.378934860229492,
      "activations/layer17_attention_weight_max": 47.93705749511719,
      "activations/layer17_attention_weight_min": -45.19513702392578,
      "activations/layer18_attention_weight_max": 45.37503433227539,
      "activations/layer18_attention_weight_min": -39.51641082763672,
      "activations/layer19_attention_weight_max": 26.20307731628418,
      "activations/layer19_attention_weight_min": -22.368284225463867,
      "activations/layer1_attention_weight_max": 16.132261276245117,
      "activations/layer1_attention_weight_min": -13.41938591003418,
      "activations/layer20_attention_weight_max": 22.6144962310791,
      "activations/layer20_attention_weight_min": -21.2364559173584,
      "activations/layer21_attention_weight_max": 32.057044982910156,
      "activations/layer21_attention_weight_min": -22.917760848999023,
      "activations/layer22_attention_weight_max": 27.86075782775879,
      "activations/layer22_attention_weight_min": -25.409835815429688,
      "activations/layer23_attention_weight_max": 37.899410247802734,
      "activations/layer23_attention_weight_min": -25.66364860534668,
      "activations/layer2_attention_weight_max": 31.628293991088867,
      "activations/layer2_attention_weight_min": -30.44959259033203,
      "activations/layer3_attention_weight_max": 88.11686706542969,
      "activations/layer3_attention_weight_min": -86.2254409790039,
      "activations/layer4_attention_weight_max": 84.32451629638672,
      "activations/layer4_attention_weight_min": -86.52469635009766,
      "activations/layer5_attention_weight_max": 62.062129974365234,
      "activations/layer5_attention_weight_min": -73.22763061523438,
      "activations/layer6_attention_weight_max": 48.39287185668945,
      "activations/layer6_attention_weight_min": -51.47462463378906,
      "activations/layer7_attention_weight_max": 61.17815017700195,
      "activations/layer7_attention_weight_min": -63.11835479736328,
      "activations/layer8_attention_weight_max": 43.40255355834961,
      "activations/layer8_attention_weight_min": -49.048362731933594,
      "activations/layer9_attention_weight_max": 40.516239166259766,
      "activations/layer9_attention_weight_min": -43.997310638427734,
      "epoch": 17.9,
      "learning_rate": 3.490530303030303e-05,
      "loss": 2.7433,
      "step": 308100
    },
    {
      "activations/layer0_attention_weight_max": 15.207919120788574,
      "activations/layer0_attention_weight_min": -13.54848575592041,
      "activations/layer10_attention_weight_max": 35.702056884765625,
      "activations/layer10_attention_weight_min": -34.656898498535156,
      "activations/layer11_attention_weight_max": 32.39904022216797,
      "activations/layer11_attention_weight_min": -33.75853729248047,
      "activations/layer12_attention_weight_max": 18.388608932495117,
      "activations/layer12_attention_weight_min": -25.621814727783203,
      "activations/layer13_attention_weight_max": 37.14740753173828,
      "activations/layer13_attention_weight_min": -27.346799850463867,
      "activations/layer14_attention_weight_max": 34.7887077331543,
      "activations/layer14_attention_weight_min": -28.807117462158203,
      "activations/layer15_attention_weight_max": 32.13265609741211,
      "activations/layer15_attention_weight_min": -30.053913116455078,
      "activations/layer16_attention_weight_max": 30.96067237854004,
      "activations/layer16_attention_weight_min": -30.236894607543945,
      "activations/layer17_attention_weight_max": 49.630977630615234,
      "activations/layer17_attention_weight_min": -42.594993591308594,
      "activations/layer18_attention_weight_max": 44.69630813598633,
      "activations/layer18_attention_weight_min": -38.1986198425293,
      "activations/layer19_attention_weight_max": 23.518043518066406,
      "activations/layer19_attention_weight_min": -20.659177780151367,
      "activations/layer1_attention_weight_max": 15.455809593200684,
      "activations/layer1_attention_weight_min": -15.118062973022461,
      "activations/layer20_attention_weight_max": 21.85836410522461,
      "activations/layer20_attention_weight_min": -21.67808723449707,
      "activations/layer21_attention_weight_max": 35.7125129699707,
      "activations/layer21_attention_weight_min": -23.915624618530273,
      "activations/layer22_attention_weight_max": 27.599695205688477,
      "activations/layer22_attention_weight_min": -26.01667594909668,
      "activations/layer23_attention_weight_max": 33.67353057861328,
      "activations/layer23_attention_weight_min": -24.22871208190918,
      "activations/layer2_attention_weight_max": 31.489288330078125,
      "activations/layer2_attention_weight_min": -29.439611434936523,
      "activations/layer3_attention_weight_max": 87.95301055908203,
      "activations/layer3_attention_weight_min": -94.4801254272461,
      "activations/layer4_attention_weight_max": 87.45539093017578,
      "activations/layer4_attention_weight_min": -88.0263900756836,
      "activations/layer5_attention_weight_max": 64.23793029785156,
      "activations/layer5_attention_weight_min": -72.2003173828125,
      "activations/layer6_attention_weight_max": 51.98158645629883,
      "activations/layer6_attention_weight_min": -49.604251861572266,
      "activations/layer7_attention_weight_max": 67.8169937133789,
      "activations/layer7_attention_weight_min": -64.07331848144531,
      "activations/layer8_attention_weight_max": 46.910892486572266,
      "activations/layer8_attention_weight_min": -49.85866165161133,
      "activations/layer9_attention_weight_max": 44.04875946044922,
      "activations/layer9_attention_weight_min": -46.216426849365234,
      "epoch": 17.91,
      "learning_rate": 3.488636363636363e-05,
      "loss": 2.7228,
      "step": 308150
    },
    {
      "activations/layer0_attention_weight_max": 15.217438697814941,
      "activations/layer0_attention_weight_min": -13.583791732788086,
      "activations/layer10_attention_weight_max": 43.09834289550781,
      "activations/layer10_attention_weight_min": -41.83379364013672,
      "activations/layer11_attention_weight_max": 41.777244567871094,
      "activations/layer11_attention_weight_min": -36.957801818847656,
      "activations/layer12_attention_weight_max": 18.72027015686035,
      "activations/layer12_attention_weight_min": -28.592687606811523,
      "activations/layer13_attention_weight_max": 39.799652099609375,
      "activations/layer13_attention_weight_min": -28.711023330688477,
      "activations/layer14_attention_weight_max": 36.46733474731445,
      "activations/layer14_attention_weight_min": -30.610498428344727,
      "activations/layer15_attention_weight_max": 34.466590881347656,
      "activations/layer15_attention_weight_min": -31.758146286010742,
      "activations/layer16_attention_weight_max": 34.22646713256836,
      "activations/layer16_attention_weight_min": -32.00318908691406,
      "activations/layer17_attention_weight_max": 52.48371124267578,
      "activations/layer17_attention_weight_min": -45.813629150390625,
      "activations/layer18_attention_weight_max": 46.37222671508789,
      "activations/layer18_attention_weight_min": -40.159690856933594,
      "activations/layer19_attention_weight_max": 22.754215240478516,
      "activations/layer19_attention_weight_min": -24.48441505432129,
      "activations/layer1_attention_weight_max": 16.098102569580078,
      "activations/layer1_attention_weight_min": -14.825559616088867,
      "activations/layer20_attention_weight_max": 21.87139320373535,
      "activations/layer20_attention_weight_min": -21.643953323364258,
      "activations/layer21_attention_weight_max": 35.727569580078125,
      "activations/layer21_attention_weight_min": -22.70941925048828,
      "activations/layer22_attention_weight_max": 28.304956436157227,
      "activations/layer22_attention_weight_min": -25.244354248046875,
      "activations/layer23_attention_weight_max": 33.603172302246094,
      "activations/layer23_attention_weight_min": -24.15394401550293,
      "activations/layer2_attention_weight_max": 32.64256286621094,
      "activations/layer2_attention_weight_min": -33.87598419189453,
      "activations/layer3_attention_weight_max": 89.59117126464844,
      "activations/layer3_attention_weight_min": -93.40291595458984,
      "activations/layer4_attention_weight_max": 91.13086700439453,
      "activations/layer4_attention_weight_min": -87.47525787353516,
      "activations/layer5_attention_weight_max": 67.1507797241211,
      "activations/layer5_attention_weight_min": -74.07948303222656,
      "activations/layer6_attention_weight_max": 48.34870147705078,
      "activations/layer6_attention_weight_min": -47.14106750488281,
      "activations/layer7_attention_weight_max": 71.36389923095703,
      "activations/layer7_attention_weight_min": -63.30081558227539,
      "activations/layer8_attention_weight_max": 50.65647888183594,
      "activations/layer8_attention_weight_min": -49.25899887084961,
      "activations/layer9_attention_weight_max": 56.45634841918945,
      "activations/layer9_attention_weight_min": -47.46657943725586,
      "epoch": 17.91,
      "learning_rate": 3.486742424242424e-05,
      "loss": 2.7416,
      "step": 308200
    },
    {
      "activations/layer0_attention_weight_max": 15.787402153015137,
      "activations/layer0_attention_weight_min": -13.533123016357422,
      "activations/layer10_attention_weight_max": 35.917388916015625,
      "activations/layer10_attention_weight_min": -33.69952392578125,
      "activations/layer11_attention_weight_max": 31.48529624938965,
      "activations/layer11_attention_weight_min": -32.83642578125,
      "activations/layer12_attention_weight_max": 19.47627830505371,
      "activations/layer12_attention_weight_min": -23.810165405273438,
      "activations/layer13_attention_weight_max": 39.79356384277344,
      "activations/layer13_attention_weight_min": -28.04647445678711,
      "activations/layer14_attention_weight_max": 35.475460052490234,
      "activations/layer14_attention_weight_min": -29.730741500854492,
      "activations/layer15_attention_weight_max": 33.252471923828125,
      "activations/layer15_attention_weight_min": -29.63196563720703,
      "activations/layer16_attention_weight_max": 32.79532241821289,
      "activations/layer16_attention_weight_min": -30.798816680908203,
      "activations/layer17_attention_weight_max": 48.24915313720703,
      "activations/layer17_attention_weight_min": -41.76008605957031,
      "activations/layer18_attention_weight_max": 47.72759246826172,
      "activations/layer18_attention_weight_min": -37.42815399169922,
      "activations/layer19_attention_weight_max": 24.215499877929688,
      "activations/layer19_attention_weight_min": -21.78365707397461,
      "activations/layer1_attention_weight_max": 16.5294246673584,
      "activations/layer1_attention_weight_min": -12.30233383178711,
      "activations/layer20_attention_weight_max": 23.253402709960938,
      "activations/layer20_attention_weight_min": -21.602689743041992,
      "activations/layer21_attention_weight_max": 36.680519104003906,
      "activations/layer21_attention_weight_min": -21.658653259277344,
      "activations/layer22_attention_weight_max": 30.747671127319336,
      "activations/layer22_attention_weight_min": -25.82814598083496,
      "activations/layer23_attention_weight_max": 38.18882751464844,
      "activations/layer23_attention_weight_min": -23.846012115478516,
      "activations/layer2_attention_weight_max": 28.489639282226562,
      "activations/layer2_attention_weight_min": -28.09250259399414,
      "activations/layer3_attention_weight_max": 81.82395935058594,
      "activations/layer3_attention_weight_min": -81.95531463623047,
      "activations/layer4_attention_weight_max": 82.0333023071289,
      "activations/layer4_attention_weight_min": -80.84698486328125,
      "activations/layer5_attention_weight_max": 59.315887451171875,
      "activations/layer5_attention_weight_min": -68.22505950927734,
      "activations/layer6_attention_weight_max": 47.11131286621094,
      "activations/layer6_attention_weight_min": -47.47142028808594,
      "activations/layer7_attention_weight_max": 59.41629409790039,
      "activations/layer7_attention_weight_min": -62.01095199584961,
      "activations/layer8_attention_weight_max": 47.88113784790039,
      "activations/layer8_attention_weight_min": -48.42637634277344,
      "activations/layer9_attention_weight_max": 45.387779235839844,
      "activations/layer9_attention_weight_min": -46.31178283691406,
      "epoch": 17.91,
      "learning_rate": 3.484848484848484e-05,
      "loss": 2.7304,
      "step": 308250
    },
    {
      "activations/layer0_attention_weight_max": 15.231112480163574,
      "activations/layer0_attention_weight_min": -13.427522659301758,
      "activations/layer10_attention_weight_max": 36.121559143066406,
      "activations/layer10_attention_weight_min": -36.58424758911133,
      "activations/layer11_attention_weight_max": 33.75324249267578,
      "activations/layer11_attention_weight_min": -34.71697235107422,
      "activations/layer12_attention_weight_max": 21.72690200805664,
      "activations/layer12_attention_weight_min": -24.90753746032715,
      "activations/layer13_attention_weight_max": 40.51577377319336,
      "activations/layer13_attention_weight_min": -30.452465057373047,
      "activations/layer14_attention_weight_max": 35.67845153808594,
      "activations/layer14_attention_weight_min": -31.323484420776367,
      "activations/layer15_attention_weight_max": 34.957916259765625,
      "activations/layer15_attention_weight_min": -32.31111145019531,
      "activations/layer16_attention_weight_max": 35.80046844482422,
      "activations/layer16_attention_weight_min": -31.444412231445312,
      "activations/layer17_attention_weight_max": 53.46817398071289,
      "activations/layer17_attention_weight_min": -46.823787689208984,
      "activations/layer18_attention_weight_max": 51.46611785888672,
      "activations/layer18_attention_weight_min": -41.55419158935547,
      "activations/layer19_attention_weight_max": 25.022478103637695,
      "activations/layer19_attention_weight_min": -23.942506790161133,
      "activations/layer1_attention_weight_max": 15.250371932983398,
      "activations/layer1_attention_weight_min": -13.976600646972656,
      "activations/layer20_attention_weight_max": 22.45064926147461,
      "activations/layer20_attention_weight_min": -21.79677391052246,
      "activations/layer21_attention_weight_max": 37.420875549316406,
      "activations/layer21_attention_weight_min": -22.227500915527344,
      "activations/layer22_attention_weight_max": 29.882104873657227,
      "activations/layer22_attention_weight_min": -26.086017608642578,
      "activations/layer23_attention_weight_max": 40.79117965698242,
      "activations/layer23_attention_weight_min": -24.665390014648438,
      "activations/layer2_attention_weight_max": 31.772680282592773,
      "activations/layer2_attention_weight_min": -30.641006469726562,
      "activations/layer3_attention_weight_max": 86.72834777832031,
      "activations/layer3_attention_weight_min": -86.16175842285156,
      "activations/layer4_attention_weight_max": 84.21676635742188,
      "activations/layer4_attention_weight_min": -82.84046173095703,
      "activations/layer5_attention_weight_max": 64.12200927734375,
      "activations/layer5_attention_weight_min": -72.35692596435547,
      "activations/layer6_attention_weight_max": 51.783260345458984,
      "activations/layer6_attention_weight_min": -50.15890121459961,
      "activations/layer7_attention_weight_max": 68.69900512695312,
      "activations/layer7_attention_weight_min": -62.71131134033203,
      "activations/layer8_attention_weight_max": 47.22007369995117,
      "activations/layer8_attention_weight_min": -51.68287658691406,
      "activations/layer9_attention_weight_max": 46.64888381958008,
      "activations/layer9_attention_weight_min": -45.32052230834961,
      "epoch": 17.91,
      "learning_rate": 3.482954545454545e-05,
      "loss": 2.7417,
      "step": 308300
    },
    {
      "activations/layer0_attention_weight_max": 15.931181907653809,
      "activations/layer0_attention_weight_min": -13.35982894897461,
      "activations/layer10_attention_weight_max": 35.86893844604492,
      "activations/layer10_attention_weight_min": -34.556087493896484,
      "activations/layer11_attention_weight_max": 33.730323791503906,
      "activations/layer11_attention_weight_min": -36.10905456542969,
      "activations/layer12_attention_weight_max": 20.555479049682617,
      "activations/layer12_attention_weight_min": -24.923845291137695,
      "activations/layer13_attention_weight_max": 35.736175537109375,
      "activations/layer13_attention_weight_min": -28.369958877563477,
      "activations/layer14_attention_weight_max": 36.96731185913086,
      "activations/layer14_attention_weight_min": -30.54207992553711,
      "activations/layer15_attention_weight_max": 35.38636779785156,
      "activations/layer15_attention_weight_min": -30.67357635498047,
      "activations/layer16_attention_weight_max": 36.56803894042969,
      "activations/layer16_attention_weight_min": -32.45888137817383,
      "activations/layer17_attention_weight_max": 53.32973098754883,
      "activations/layer17_attention_weight_min": -44.10468673706055,
      "activations/layer18_attention_weight_max": 47.2080192565918,
      "activations/layer18_attention_weight_min": -38.70389938354492,
      "activations/layer19_attention_weight_max": 24.307109832763672,
      "activations/layer19_attention_weight_min": -23.27858543395996,
      "activations/layer1_attention_weight_max": 15.572428703308105,
      "activations/layer1_attention_weight_min": -13.624096870422363,
      "activations/layer20_attention_weight_max": 21.061279296875,
      "activations/layer20_attention_weight_min": -20.65788459777832,
      "activations/layer21_attention_weight_max": 38.62052917480469,
      "activations/layer21_attention_weight_min": -21.993087768554688,
      "activations/layer22_attention_weight_max": 28.792606353759766,
      "activations/layer22_attention_weight_min": -25.209232330322266,
      "activations/layer23_attention_weight_max": 35.58255386352539,
      "activations/layer23_attention_weight_min": -25.146873474121094,
      "activations/layer2_attention_weight_max": 30.229080200195312,
      "activations/layer2_attention_weight_min": -30.36536407470703,
      "activations/layer3_attention_weight_max": 80.25695037841797,
      "activations/layer3_attention_weight_min": -82.24665069580078,
      "activations/layer4_attention_weight_max": 83.39749145507812,
      "activations/layer4_attention_weight_min": -83.27510833740234,
      "activations/layer5_attention_weight_max": 62.87835693359375,
      "activations/layer5_attention_weight_min": -71.78829193115234,
      "activations/layer6_attention_weight_max": 46.93177032470703,
      "activations/layer6_attention_weight_min": -47.7733039855957,
      "activations/layer7_attention_weight_max": 64.34204864501953,
      "activations/layer7_attention_weight_min": -64.64869689941406,
      "activations/layer8_attention_weight_max": 44.7225456237793,
      "activations/layer8_attention_weight_min": -50.00296401977539,
      "activations/layer9_attention_weight_max": 42.04165267944336,
      "activations/layer9_attention_weight_min": -45.10939025878906,
      "epoch": 17.92,
      "learning_rate": 3.481060606060606e-05,
      "loss": 2.7472,
      "step": 308350
    },
    {
      "activations/layer0_attention_weight_max": 15.515294075012207,
      "activations/layer0_attention_weight_min": -13.056951522827148,
      "activations/layer10_attention_weight_max": 38.1109504699707,
      "activations/layer10_attention_weight_min": -36.016597747802734,
      "activations/layer11_attention_weight_max": 34.50299072265625,
      "activations/layer11_attention_weight_min": -35.325069427490234,
      "activations/layer12_attention_weight_max": 18.524822235107422,
      "activations/layer12_attention_weight_min": -27.489988327026367,
      "activations/layer13_attention_weight_max": 34.822601318359375,
      "activations/layer13_attention_weight_min": -28.726516723632812,
      "activations/layer14_attention_weight_max": 34.679100036621094,
      "activations/layer14_attention_weight_min": -28.735151290893555,
      "activations/layer15_attention_weight_max": 31.737979888916016,
      "activations/layer15_attention_weight_min": -29.9455509185791,
      "activations/layer16_attention_weight_max": 32.67829513549805,
      "activations/layer16_attention_weight_min": -30.827482223510742,
      "activations/layer17_attention_weight_max": 48.212284088134766,
      "activations/layer17_attention_weight_min": -43.44927978515625,
      "activations/layer18_attention_weight_max": 44.18940353393555,
      "activations/layer18_attention_weight_min": -36.91743469238281,
      "activations/layer19_attention_weight_max": 22.600444793701172,
      "activations/layer19_attention_weight_min": -22.897987365722656,
      "activations/layer1_attention_weight_max": 16.245880126953125,
      "activations/layer1_attention_weight_min": -12.674503326416016,
      "activations/layer20_attention_weight_max": 20.25939178466797,
      "activations/layer20_attention_weight_min": -23.023305892944336,
      "activations/layer21_attention_weight_max": 34.58308792114258,
      "activations/layer21_attention_weight_min": -19.399585723876953,
      "activations/layer22_attention_weight_max": 28.21026039123535,
      "activations/layer22_attention_weight_min": -23.765541076660156,
      "activations/layer23_attention_weight_max": 35.11172866821289,
      "activations/layer23_attention_weight_min": -21.93336296081543,
      "activations/layer2_attention_weight_max": 28.95159149169922,
      "activations/layer2_attention_weight_min": -27.960681915283203,
      "activations/layer3_attention_weight_max": 92.07303619384766,
      "activations/layer3_attention_weight_min": -89.46797943115234,
      "activations/layer4_attention_weight_max": 85.85391998291016,
      "activations/layer4_attention_weight_min": -84.12260437011719,
      "activations/layer5_attention_weight_max": 64.00263977050781,
      "activations/layer5_attention_weight_min": -73.19960021972656,
      "activations/layer6_attention_weight_max": 49.5043830871582,
      "activations/layer6_attention_weight_min": -51.98750305175781,
      "activations/layer7_attention_weight_max": 62.25954055786133,
      "activations/layer7_attention_weight_min": -63.08727264404297,
      "activations/layer8_attention_weight_max": 48.3248405456543,
      "activations/layer8_attention_weight_min": -52.37794494628906,
      "activations/layer9_attention_weight_max": 46.92591857910156,
      "activations/layer9_attention_weight_min": -46.931610107421875,
      "epoch": 17.92,
      "learning_rate": 3.479166666666667e-05,
      "loss": 2.7286,
      "step": 308400
    },
    {
      "activations/layer0_attention_weight_max": 15.910974502563477,
      "activations/layer0_attention_weight_min": -13.142019271850586,
      "activations/layer10_attention_weight_max": 35.941925048828125,
      "activations/layer10_attention_weight_min": -34.798004150390625,
      "activations/layer11_attention_weight_max": 32.949058532714844,
      "activations/layer11_attention_weight_min": -33.13502502441406,
      "activations/layer12_attention_weight_max": 18.871078491210938,
      "activations/layer12_attention_weight_min": -25.618165969848633,
      "activations/layer13_attention_weight_max": 35.60689926147461,
      "activations/layer13_attention_weight_min": -28.120996475219727,
      "activations/layer14_attention_weight_max": 35.40298080444336,
      "activations/layer14_attention_weight_min": -30.72626304626465,
      "activations/layer15_attention_weight_max": 34.58949661254883,
      "activations/layer15_attention_weight_min": -30.28044319152832,
      "activations/layer16_attention_weight_max": 37.611942291259766,
      "activations/layer16_attention_weight_min": -33.33758544921875,
      "activations/layer17_attention_weight_max": 55.936092376708984,
      "activations/layer17_attention_weight_min": -48.06960678100586,
      "activations/layer18_attention_weight_max": 51.990901947021484,
      "activations/layer18_attention_weight_min": -42.01543426513672,
      "activations/layer19_attention_weight_max": 25.722736358642578,
      "activations/layer19_attention_weight_min": -24.279760360717773,
      "activations/layer1_attention_weight_max": 16.387426376342773,
      "activations/layer1_attention_weight_min": -12.591673851013184,
      "activations/layer20_attention_weight_max": 23.141563415527344,
      "activations/layer20_attention_weight_min": -20.739025115966797,
      "activations/layer21_attention_weight_max": 45.3961067199707,
      "activations/layer21_attention_weight_min": -21.93177032470703,
      "activations/layer22_attention_weight_max": 35.491188049316406,
      "activations/layer22_attention_weight_min": -25.321104049682617,
      "activations/layer23_attention_weight_max": 35.12322235107422,
      "activations/layer23_attention_weight_min": -25.150266647338867,
      "activations/layer2_attention_weight_max": 29.210247039794922,
      "activations/layer2_attention_weight_min": -28.795310974121094,
      "activations/layer3_attention_weight_max": 78.79910278320312,
      "activations/layer3_attention_weight_min": -78.44915008544922,
      "activations/layer4_attention_weight_max": 78.25579833984375,
      "activations/layer4_attention_weight_min": -76.85540008544922,
      "activations/layer5_attention_weight_max": 59.020660400390625,
      "activations/layer5_attention_weight_min": -72.72041320800781,
      "activations/layer6_attention_weight_max": 48.657718658447266,
      "activations/layer6_attention_weight_min": -46.85710525512695,
      "activations/layer7_attention_weight_max": 59.45059585571289,
      "activations/layer7_attention_weight_min": -61.59918975830078,
      "activations/layer8_attention_weight_max": 43.85336685180664,
      "activations/layer8_attention_weight_min": -49.69825744628906,
      "activations/layer9_attention_weight_max": 44.64933776855469,
      "activations/layer9_attention_weight_min": -45.52418899536133,
      "epoch": 17.92,
      "learning_rate": 3.477272727272727e-05,
      "loss": 2.7343,
      "step": 308450
    },
    {
      "activations/layer0_attention_weight_max": 17.446712493896484,
      "activations/layer0_attention_weight_min": -13.438652038574219,
      "activations/layer10_attention_weight_max": 36.80320739746094,
      "activations/layer10_attention_weight_min": -35.755653381347656,
      "activations/layer11_attention_weight_max": 31.40530776977539,
      "activations/layer11_attention_weight_min": -35.02693176269531,
      "activations/layer12_attention_weight_max": 18.8737735748291,
      "activations/layer12_attention_weight_min": -29.85161018371582,
      "activations/layer13_attention_weight_max": 31.51016616821289,
      "activations/layer13_attention_weight_min": -27.700754165649414,
      "activations/layer14_attention_weight_max": 33.03551483154297,
      "activations/layer14_attention_weight_min": -30.317047119140625,
      "activations/layer15_attention_weight_max": 31.444761276245117,
      "activations/layer15_attention_weight_min": -31.376157760620117,
      "activations/layer16_attention_weight_max": 32.64637756347656,
      "activations/layer16_attention_weight_min": -30.89739227294922,
      "activations/layer17_attention_weight_max": 49.73513412475586,
      "activations/layer17_attention_weight_min": -48.59169387817383,
      "activations/layer18_attention_weight_max": 43.476646423339844,
      "activations/layer18_attention_weight_min": -41.119686126708984,
      "activations/layer19_attention_weight_max": 22.4627742767334,
      "activations/layer19_attention_weight_min": -22.405630111694336,
      "activations/layer1_attention_weight_max": 16.612844467163086,
      "activations/layer1_attention_weight_min": -12.622736930847168,
      "activations/layer20_attention_weight_max": 23.001859664916992,
      "activations/layer20_attention_weight_min": -21.64183235168457,
      "activations/layer21_attention_weight_max": 34.0832405090332,
      "activations/layer21_attention_weight_min": -22.42608642578125,
      "activations/layer22_attention_weight_max": 27.495319366455078,
      "activations/layer22_attention_weight_min": -27.81125259399414,
      "activations/layer23_attention_weight_max": 36.15243911743164,
      "activations/layer23_attention_weight_min": -26.738086700439453,
      "activations/layer2_attention_weight_max": 30.39967155456543,
      "activations/layer2_attention_weight_min": -28.631019592285156,
      "activations/layer3_attention_weight_max": 95.49353790283203,
      "activations/layer3_attention_weight_min": -90.62593841552734,
      "activations/layer4_attention_weight_max": 88.82438659667969,
      "activations/layer4_attention_weight_min": -84.9402084350586,
      "activations/layer5_attention_weight_max": 66.90939331054688,
      "activations/layer5_attention_weight_min": -69.8753662109375,
      "activations/layer6_attention_weight_max": 50.450218200683594,
      "activations/layer6_attention_weight_min": -49.87074661254883,
      "activations/layer7_attention_weight_max": 64.36876678466797,
      "activations/layer7_attention_weight_min": -62.73435592651367,
      "activations/layer8_attention_weight_max": 43.68696212768555,
      "activations/layer8_attention_weight_min": -49.87145233154297,
      "activations/layer9_attention_weight_max": 42.41111373901367,
      "activations/layer9_attention_weight_min": -46.574928283691406,
      "epoch": 17.93,
      "learning_rate": 3.475378787878788e-05,
      "loss": 2.7372,
      "step": 308500
    },
    {
      "activations/layer0_attention_weight_max": 15.871143341064453,
      "activations/layer0_attention_weight_min": -13.468779563903809,
      "activations/layer10_attention_weight_max": 34.558013916015625,
      "activations/layer10_attention_weight_min": -35.72808837890625,
      "activations/layer11_attention_weight_max": 31.65636444091797,
      "activations/layer11_attention_weight_min": -33.024497985839844,
      "activations/layer12_attention_weight_max": 18.604978561401367,
      "activations/layer12_attention_weight_min": -28.507553100585938,
      "activations/layer13_attention_weight_max": 36.931556701660156,
      "activations/layer13_attention_weight_min": -28.96613121032715,
      "activations/layer14_attention_weight_max": 35.96923828125,
      "activations/layer14_attention_weight_min": -33.40522384643555,
      "activations/layer15_attention_weight_max": 32.934654235839844,
      "activations/layer15_attention_weight_min": -31.432737350463867,
      "activations/layer16_attention_weight_max": 34.42461013793945,
      "activations/layer16_attention_weight_min": -33.31833267211914,
      "activations/layer17_attention_weight_max": 51.795799255371094,
      "activations/layer17_attention_weight_min": -50.874759674072266,
      "activations/layer18_attention_weight_max": 49.25470733642578,
      "activations/layer18_attention_weight_min": -43.595638275146484,
      "activations/layer19_attention_weight_max": 26.452960968017578,
      "activations/layer19_attention_weight_min": -30.019208908081055,
      "activations/layer1_attention_weight_max": 15.814445495605469,
      "activations/layer1_attention_weight_min": -12.400382041931152,
      "activations/layer20_attention_weight_max": 24.153074264526367,
      "activations/layer20_attention_weight_min": -25.46340560913086,
      "activations/layer21_attention_weight_max": 36.9009895324707,
      "activations/layer21_attention_weight_min": -26.935415267944336,
      "activations/layer22_attention_weight_max": 28.58953094482422,
      "activations/layer22_attention_weight_min": -29.26830291748047,
      "activations/layer23_attention_weight_max": 35.46052551269531,
      "activations/layer23_attention_weight_min": -26.01449203491211,
      "activations/layer2_attention_weight_max": 31.541812896728516,
      "activations/layer2_attention_weight_min": -28.28411102294922,
      "activations/layer3_attention_weight_max": 82.35437774658203,
      "activations/layer3_attention_weight_min": -80.3906478881836,
      "activations/layer4_attention_weight_max": 81.4599380493164,
      "activations/layer4_attention_weight_min": -78.31230163574219,
      "activations/layer5_attention_weight_max": 62.367401123046875,
      "activations/layer5_attention_weight_min": -70.20438385009766,
      "activations/layer6_attention_weight_max": 47.348751068115234,
      "activations/layer6_attention_weight_min": -50.9914665222168,
      "activations/layer7_attention_weight_max": 61.42966842651367,
      "activations/layer7_attention_weight_min": -59.388633728027344,
      "activations/layer8_attention_weight_max": 45.03873062133789,
      "activations/layer8_attention_weight_min": -49.89226150512695,
      "activations/layer9_attention_weight_max": 42.29338455200195,
      "activations/layer9_attention_weight_min": -44.90792465209961,
      "epoch": 17.93,
      "learning_rate": 3.473484848484848e-05,
      "loss": 2.7342,
      "step": 308550
    },
    {
      "activations/layer0_attention_weight_max": 15.601507186889648,
      "activations/layer0_attention_weight_min": -13.334977149963379,
      "activations/layer10_attention_weight_max": 38.10090637207031,
      "activations/layer10_attention_weight_min": -37.196720123291016,
      "activations/layer11_attention_weight_max": 34.03070831298828,
      "activations/layer11_attention_weight_min": -35.675777435302734,
      "activations/layer12_attention_weight_max": 20.855243682861328,
      "activations/layer12_attention_weight_min": -30.21586799621582,
      "activations/layer13_attention_weight_max": 33.28553771972656,
      "activations/layer13_attention_weight_min": -29.212078094482422,
      "activations/layer14_attention_weight_max": 33.98481750488281,
      "activations/layer14_attention_weight_min": -30.890697479248047,
      "activations/layer15_attention_weight_max": 33.45742416381836,
      "activations/layer15_attention_weight_min": -28.599220275878906,
      "activations/layer16_attention_weight_max": 34.408836364746094,
      "activations/layer16_attention_weight_min": -32.38673782348633,
      "activations/layer17_attention_weight_max": 52.323638916015625,
      "activations/layer17_attention_weight_min": -45.6442756652832,
      "activations/layer18_attention_weight_max": 48.365997314453125,
      "activations/layer18_attention_weight_min": -39.32524108886719,
      "activations/layer19_attention_weight_max": 23.414291381835938,
      "activations/layer19_attention_weight_min": -22.915937423706055,
      "activations/layer1_attention_weight_max": 16.48924446105957,
      "activations/layer1_attention_weight_min": -12.60575008392334,
      "activations/layer20_attention_weight_max": 24.124805450439453,
      "activations/layer20_attention_weight_min": -20.564085006713867,
      "activations/layer21_attention_weight_max": 43.64152145385742,
      "activations/layer21_attention_weight_min": -26.45134162902832,
      "activations/layer22_attention_weight_max": 30.447364807128906,
      "activations/layer22_attention_weight_min": -26.09708023071289,
      "activations/layer23_attention_weight_max": 38.625404357910156,
      "activations/layer23_attention_weight_min": -23.314443588256836,
      "activations/layer2_attention_weight_max": 30.827964782714844,
      "activations/layer2_attention_weight_min": -28.065147399902344,
      "activations/layer3_attention_weight_max": 84.5866928100586,
      "activations/layer3_attention_weight_min": -82.2479019165039,
      "activations/layer4_attention_weight_max": 82.67794036865234,
      "activations/layer4_attention_weight_min": -83.15985107421875,
      "activations/layer5_attention_weight_max": 62.51728439331055,
      "activations/layer5_attention_weight_min": -70.37186431884766,
      "activations/layer6_attention_weight_max": 47.033817291259766,
      "activations/layer6_attention_weight_min": -48.81950759887695,
      "activations/layer7_attention_weight_max": 60.35903549194336,
      "activations/layer7_attention_weight_min": -61.84326934814453,
      "activations/layer8_attention_weight_max": 43.28382110595703,
      "activations/layer8_attention_weight_min": -45.771488189697266,
      "activations/layer9_attention_weight_max": 46.52479934692383,
      "activations/layer9_attention_weight_min": -46.060218811035156,
      "epoch": 17.93,
      "learning_rate": 3.471590909090909e-05,
      "loss": 2.7493,
      "step": 308600
    },
    {
      "activations/layer0_attention_weight_max": 16.084083557128906,
      "activations/layer0_attention_weight_min": -13.808880805969238,
      "activations/layer10_attention_weight_max": 38.803016662597656,
      "activations/layer10_attention_weight_min": -35.99873733520508,
      "activations/layer11_attention_weight_max": 32.49811553955078,
      "activations/layer11_attention_weight_min": -34.05194091796875,
      "activations/layer12_attention_weight_max": 20.344072341918945,
      "activations/layer12_attention_weight_min": -30.18486785888672,
      "activations/layer13_attention_weight_max": 34.88529586791992,
      "activations/layer13_attention_weight_min": -30.083471298217773,
      "activations/layer14_attention_weight_max": 34.147560119628906,
      "activations/layer14_attention_weight_min": -29.315582275390625,
      "activations/layer15_attention_weight_max": 31.737268447875977,
      "activations/layer15_attention_weight_min": -29.853195190429688,
      "activations/layer16_attention_weight_max": 32.41624450683594,
      "activations/layer16_attention_weight_min": -31.128246307373047,
      "activations/layer17_attention_weight_max": 51.782283782958984,
      "activations/layer17_attention_weight_min": -44.90205383300781,
      "activations/layer18_attention_weight_max": 44.17393493652344,
      "activations/layer18_attention_weight_min": -40.983455657958984,
      "activations/layer19_attention_weight_max": 24.577709197998047,
      "activations/layer19_attention_weight_min": -22.406389236450195,
      "activations/layer1_attention_weight_max": 16.403743743896484,
      "activations/layer1_attention_weight_min": -13.218306541442871,
      "activations/layer20_attention_weight_max": 22.904460906982422,
      "activations/layer20_attention_weight_min": -24.39580726623535,
      "activations/layer21_attention_weight_max": 36.7332763671875,
      "activations/layer21_attention_weight_min": -21.85613441467285,
      "activations/layer22_attention_weight_max": 31.014490127563477,
      "activations/layer22_attention_weight_min": -25.39354705810547,
      "activations/layer23_attention_weight_max": 36.958221435546875,
      "activations/layer23_attention_weight_min": -22.75660514831543,
      "activations/layer2_attention_weight_max": 29.903043746948242,
      "activations/layer2_attention_weight_min": -27.941381454467773,
      "activations/layer3_attention_weight_max": 84.48494720458984,
      "activations/layer3_attention_weight_min": -83.53450775146484,
      "activations/layer4_attention_weight_max": 86.95120239257812,
      "activations/layer4_attention_weight_min": -85.48303985595703,
      "activations/layer5_attention_weight_max": 60.498992919921875,
      "activations/layer5_attention_weight_min": -72.52903747558594,
      "activations/layer6_attention_weight_max": 49.61748504638672,
      "activations/layer6_attention_weight_min": -50.34378433227539,
      "activations/layer7_attention_weight_max": 62.51158905029297,
      "activations/layer7_attention_weight_min": -62.25560760498047,
      "activations/layer8_attention_weight_max": 46.74299621582031,
      "activations/layer8_attention_weight_min": -51.598270416259766,
      "activations/layer9_attention_weight_max": 46.09764099121094,
      "activations/layer9_attention_weight_min": -47.27341842651367,
      "epoch": 17.93,
      "learning_rate": 3.4696969696969695e-05,
      "loss": 2.7424,
      "step": 308650
    },
    {
      "activations/layer0_attention_weight_max": 15.828611373901367,
      "activations/layer0_attention_weight_min": -13.538677215576172,
      "activations/layer10_attention_weight_max": 35.387939453125,
      "activations/layer10_attention_weight_min": -34.48499298095703,
      "activations/layer11_attention_weight_max": 31.954910278320312,
      "activations/layer11_attention_weight_min": -33.000762939453125,
      "activations/layer12_attention_weight_max": 20.611562728881836,
      "activations/layer12_attention_weight_min": -24.267484664916992,
      "activations/layer13_attention_weight_max": 31.791080474853516,
      "activations/layer13_attention_weight_min": -26.4886417388916,
      "activations/layer14_attention_weight_max": 34.81035614013672,
      "activations/layer14_attention_weight_min": -32.26279830932617,
      "activations/layer15_attention_weight_max": 33.60429382324219,
      "activations/layer15_attention_weight_min": -30.57883071899414,
      "activations/layer16_attention_weight_max": 31.545547485351562,
      "activations/layer16_attention_weight_min": -31.401119232177734,
      "activations/layer17_attention_weight_max": 50.255516052246094,
      "activations/layer17_attention_weight_min": -44.88422393798828,
      "activations/layer18_attention_weight_max": 46.791385650634766,
      "activations/layer18_attention_weight_min": -41.19778823852539,
      "activations/layer19_attention_weight_max": 24.192180633544922,
      "activations/layer19_attention_weight_min": -22.969228744506836,
      "activations/layer1_attention_weight_max": 15.884793281555176,
      "activations/layer1_attention_weight_min": -12.191267967224121,
      "activations/layer20_attention_weight_max": 23.268531799316406,
      "activations/layer20_attention_weight_min": -21.496339797973633,
      "activations/layer21_attention_weight_max": 36.70005416870117,
      "activations/layer21_attention_weight_min": -23.656700134277344,
      "activations/layer22_attention_weight_max": 32.08866882324219,
      "activations/layer22_attention_weight_min": -28.229232788085938,
      "activations/layer23_attention_weight_max": 35.46559143066406,
      "activations/layer23_attention_weight_min": -22.66438865661621,
      "activations/layer2_attention_weight_max": 30.10797119140625,
      "activations/layer2_attention_weight_min": -27.998022079467773,
      "activations/layer3_attention_weight_max": 82.6622314453125,
      "activations/layer3_attention_weight_min": -85.55947875976562,
      "activations/layer4_attention_weight_max": 81.57032775878906,
      "activations/layer4_attention_weight_min": -81.05795288085938,
      "activations/layer5_attention_weight_max": 58.49100875854492,
      "activations/layer5_attention_weight_min": -67.0416030883789,
      "activations/layer6_attention_weight_max": 45.7567253112793,
      "activations/layer6_attention_weight_min": -47.2584342956543,
      "activations/layer7_attention_weight_max": 58.56059265136719,
      "activations/layer7_attention_weight_min": -59.14729309082031,
      "activations/layer8_attention_weight_max": 43.37396240234375,
      "activations/layer8_attention_weight_min": -45.06775665283203,
      "activations/layer9_attention_weight_max": 41.579978942871094,
      "activations/layer9_attention_weight_min": -43.81818771362305,
      "epoch": 17.94,
      "learning_rate": 3.4678030303030304e-05,
      "loss": 2.7274,
      "step": 308700
    },
    {
      "activations/layer0_attention_weight_max": 16.13822364807129,
      "activations/layer0_attention_weight_min": -13.523720741271973,
      "activations/layer10_attention_weight_max": 34.66925811767578,
      "activations/layer10_attention_weight_min": -36.091773986816406,
      "activations/layer11_attention_weight_max": 32.30167007446289,
      "activations/layer11_attention_weight_min": -33.642669677734375,
      "activations/layer12_attention_weight_max": 18.1796932220459,
      "activations/layer12_attention_weight_min": -26.57044792175293,
      "activations/layer13_attention_weight_max": 35.51067352294922,
      "activations/layer13_attention_weight_min": -26.921411514282227,
      "activations/layer14_attention_weight_max": 34.58600997924805,
      "activations/layer14_attention_weight_min": -31.275787353515625,
      "activations/layer15_attention_weight_max": 34.737037658691406,
      "activations/layer15_attention_weight_min": -32.098018646240234,
      "activations/layer16_attention_weight_max": 34.28896713256836,
      "activations/layer16_attention_weight_min": -31.37847900390625,
      "activations/layer17_attention_weight_max": 48.8105354309082,
      "activations/layer17_attention_weight_min": -44.411136627197266,
      "activations/layer18_attention_weight_max": 44.47994613647461,
      "activations/layer18_attention_weight_min": -38.5523796081543,
      "activations/layer19_attention_weight_max": 25.22930335998535,
      "activations/layer19_attention_weight_min": -23.268451690673828,
      "activations/layer1_attention_weight_max": 15.58987808227539,
      "activations/layer1_attention_weight_min": -13.255820274353027,
      "activations/layer20_attention_weight_max": 21.728559494018555,
      "activations/layer20_attention_weight_min": -21.24771499633789,
      "activations/layer21_attention_weight_max": 34.2677116394043,
      "activations/layer21_attention_weight_min": -21.151473999023438,
      "activations/layer22_attention_weight_max": 31.700700759887695,
      "activations/layer22_attention_weight_min": -26.47881317138672,
      "activations/layer23_attention_weight_max": 38.96632385253906,
      "activations/layer23_attention_weight_min": -28.695037841796875,
      "activations/layer2_attention_weight_max": 30.52846336364746,
      "activations/layer2_attention_weight_min": -29.248985290527344,
      "activations/layer3_attention_weight_max": 87.98526000976562,
      "activations/layer3_attention_weight_min": -92.63986206054688,
      "activations/layer4_attention_weight_max": 86.02486419677734,
      "activations/layer4_attention_weight_min": -84.76986694335938,
      "activations/layer5_attention_weight_max": 64.53257751464844,
      "activations/layer5_attention_weight_min": -74.70074462890625,
      "activations/layer6_attention_weight_max": 48.402164459228516,
      "activations/layer6_attention_weight_min": -51.05601119995117,
      "activations/layer7_attention_weight_max": 61.833763122558594,
      "activations/layer7_attention_weight_min": -61.197059631347656,
      "activations/layer8_attention_weight_max": 48.67646026611328,
      "activations/layer8_attention_weight_min": -49.394439697265625,
      "activations/layer9_attention_weight_max": 43.199867248535156,
      "activations/layer9_attention_weight_min": -44.978389739990234,
      "epoch": 17.94,
      "learning_rate": 3.4659090909090905e-05,
      "loss": 2.7479,
      "step": 308750
    },
    {
      "activations/layer0_attention_weight_max": 16.378490447998047,
      "activations/layer0_attention_weight_min": -13.669477462768555,
      "activations/layer10_attention_weight_max": 35.277259826660156,
      "activations/layer10_attention_weight_min": -33.92621612548828,
      "activations/layer11_attention_weight_max": 31.051753997802734,
      "activations/layer11_attention_weight_min": -33.530860900878906,
      "activations/layer12_attention_weight_max": 18.531755447387695,
      "activations/layer12_attention_weight_min": -27.103593826293945,
      "activations/layer13_attention_weight_max": 33.70464324951172,
      "activations/layer13_attention_weight_min": -30.646568298339844,
      "activations/layer14_attention_weight_max": 38.377586364746094,
      "activations/layer14_attention_weight_min": -34.0925407409668,
      "activations/layer15_attention_weight_max": 33.38595962524414,
      "activations/layer15_attention_weight_min": -33.33867263793945,
      "activations/layer16_attention_weight_max": 36.410743713378906,
      "activations/layer16_attention_weight_min": -34.31134033203125,
      "activations/layer17_attention_weight_max": 52.82107925415039,
      "activations/layer17_attention_weight_min": -46.82481384277344,
      "activations/layer18_attention_weight_max": 50.24337387084961,
      "activations/layer18_attention_weight_min": -41.30180740356445,
      "activations/layer19_attention_weight_max": 27.213647842407227,
      "activations/layer19_attention_weight_min": -23.78461456298828,
      "activations/layer1_attention_weight_max": 15.648941993713379,
      "activations/layer1_attention_weight_min": -12.905126571655273,
      "activations/layer20_attention_weight_max": 27.86153221130371,
      "activations/layer20_attention_weight_min": -25.189298629760742,
      "activations/layer21_attention_weight_max": 48.154563903808594,
      "activations/layer21_attention_weight_min": -27.8814754486084,
      "activations/layer22_attention_weight_max": 35.81007766723633,
      "activations/layer22_attention_weight_min": -30.597890853881836,
      "activations/layer23_attention_weight_max": 38.99601745605469,
      "activations/layer23_attention_weight_min": -24.38637924194336,
      "activations/layer2_attention_weight_max": 29.884212493896484,
      "activations/layer2_attention_weight_min": -29.667776107788086,
      "activations/layer3_attention_weight_max": 88.21572875976562,
      "activations/layer3_attention_weight_min": -89.05415344238281,
      "activations/layer4_attention_weight_max": 85.37406158447266,
      "activations/layer4_attention_weight_min": -83.85294342041016,
      "activations/layer5_attention_weight_max": 59.10951232910156,
      "activations/layer5_attention_weight_min": -71.55137634277344,
      "activations/layer6_attention_weight_max": 49.702144622802734,
      "activations/layer6_attention_weight_min": -48.16749954223633,
      "activations/layer7_attention_weight_max": 58.74697494506836,
      "activations/layer7_attention_weight_min": -58.4781379699707,
      "activations/layer8_attention_weight_max": 44.911930084228516,
      "activations/layer8_attention_weight_min": -48.04517364501953,
      "activations/layer9_attention_weight_max": 41.5513916015625,
      "activations/layer9_attention_weight_min": -42.54915237426758,
      "epoch": 17.94,
      "learning_rate": 3.4640151515151513e-05,
      "loss": 2.7416,
      "step": 308800
    },
    {
      "activations/layer0_attention_weight_max": 15.349701881408691,
      "activations/layer0_attention_weight_min": -13.687210083007812,
      "activations/layer10_attention_weight_max": 35.457618713378906,
      "activations/layer10_attention_weight_min": -35.972877502441406,
      "activations/layer11_attention_weight_max": 33.341651916503906,
      "activations/layer11_attention_weight_min": -33.95172882080078,
      "activations/layer12_attention_weight_max": 18.496231079101562,
      "activations/layer12_attention_weight_min": -22.705265045166016,
      "activations/layer13_attention_weight_max": 37.00094223022461,
      "activations/layer13_attention_weight_min": -28.056238174438477,
      "activations/layer14_attention_weight_max": 35.06255340576172,
      "activations/layer14_attention_weight_min": -29.718263626098633,
      "activations/layer15_attention_weight_max": 32.749977111816406,
      "activations/layer15_attention_weight_min": -29.649295806884766,
      "activations/layer16_attention_weight_max": 32.196685791015625,
      "activations/layer16_attention_weight_min": -29.68277931213379,
      "activations/layer17_attention_weight_max": 48.9367790222168,
      "activations/layer17_attention_weight_min": -46.02769470214844,
      "activations/layer18_attention_weight_max": 50.26618576049805,
      "activations/layer18_attention_weight_min": -41.62290954589844,
      "activations/layer19_attention_weight_max": 26.115568161010742,
      "activations/layer19_attention_weight_min": -23.718002319335938,
      "activations/layer1_attention_weight_max": 15.423537254333496,
      "activations/layer1_attention_weight_min": -12.958051681518555,
      "activations/layer20_attention_weight_max": 24.38442039489746,
      "activations/layer20_attention_weight_min": -23.702678680419922,
      "activations/layer21_attention_weight_max": 45.47243118286133,
      "activations/layer21_attention_weight_min": -21.76175880432129,
      "activations/layer22_attention_weight_max": 29.30742073059082,
      "activations/layer22_attention_weight_min": -24.89666175842285,
      "activations/layer23_attention_weight_max": 38.664573669433594,
      "activations/layer23_attention_weight_min": -24.310749053955078,
      "activations/layer2_attention_weight_max": 29.528165817260742,
      "activations/layer2_attention_weight_min": -27.809856414794922,
      "activations/layer3_attention_weight_max": 85.90576934814453,
      "activations/layer3_attention_weight_min": -84.99754333496094,
      "activations/layer4_attention_weight_max": 87.8562240600586,
      "activations/layer4_attention_weight_min": -87.63526153564453,
      "activations/layer5_attention_weight_max": 63.72300338745117,
      "activations/layer5_attention_weight_min": -71.63124084472656,
      "activations/layer6_attention_weight_max": 50.77106857299805,
      "activations/layer6_attention_weight_min": -51.81827926635742,
      "activations/layer7_attention_weight_max": 62.80659103393555,
      "activations/layer7_attention_weight_min": -63.305763244628906,
      "activations/layer8_attention_weight_max": 46.381839752197266,
      "activations/layer8_attention_weight_min": -49.53739547729492,
      "activations/layer9_attention_weight_max": 44.15507125854492,
      "activations/layer9_attention_weight_min": -45.55722427368164,
      "epoch": 17.95,
      "learning_rate": 3.4621212121212115e-05,
      "loss": 2.7356,
      "step": 308850
    },
    {
      "activations/layer0_attention_weight_max": 16.18276596069336,
      "activations/layer0_attention_weight_min": -13.622272491455078,
      "activations/layer10_attention_weight_max": 33.76886749267578,
      "activations/layer10_attention_weight_min": -34.69594192504883,
      "activations/layer11_attention_weight_max": 32.08718490600586,
      "activations/layer11_attention_weight_min": -32.796722412109375,
      "activations/layer12_attention_weight_max": 19.25058937072754,
      "activations/layer12_attention_weight_min": -26.98897933959961,
      "activations/layer13_attention_weight_max": 29.864282608032227,
      "activations/layer13_attention_weight_min": -26.328760147094727,
      "activations/layer14_attention_weight_max": 32.4373893737793,
      "activations/layer14_attention_weight_min": -29.580110549926758,
      "activations/layer15_attention_weight_max": 33.270694732666016,
      "activations/layer15_attention_weight_min": -30.740835189819336,
      "activations/layer16_attention_weight_max": 33.77163314819336,
      "activations/layer16_attention_weight_min": -30.85747528076172,
      "activations/layer17_attention_weight_max": 53.424896240234375,
      "activations/layer17_attention_weight_min": -46.878902435302734,
      "activations/layer18_attention_weight_max": 50.224578857421875,
      "activations/layer18_attention_weight_min": -40.8974723815918,
      "activations/layer19_attention_weight_max": 26.204076766967773,
      "activations/layer19_attention_weight_min": -22.911664962768555,
      "activations/layer1_attention_weight_max": 15.679357528686523,
      "activations/layer1_attention_weight_min": -14.04952335357666,
      "activations/layer20_attention_weight_max": 24.01700782775879,
      "activations/layer20_attention_weight_min": -20.477949142456055,
      "activations/layer21_attention_weight_max": 33.391422271728516,
      "activations/layer21_attention_weight_min": -22.670806884765625,
      "activations/layer22_attention_weight_max": 29.688716888427734,
      "activations/layer22_attention_weight_min": -23.525489807128906,
      "activations/layer23_attention_weight_max": 39.32158660888672,
      "activations/layer23_attention_weight_min": -22.07842254638672,
      "activations/layer2_attention_weight_max": 31.02309799194336,
      "activations/layer2_attention_weight_min": -29.68769645690918,
      "activations/layer3_attention_weight_max": 85.82992553710938,
      "activations/layer3_attention_weight_min": -89.02151489257812,
      "activations/layer4_attention_weight_max": 85.33447265625,
      "activations/layer4_attention_weight_min": -88.12047576904297,
      "activations/layer5_attention_weight_max": 62.477272033691406,
      "activations/layer5_attention_weight_min": -77.55746459960938,
      "activations/layer6_attention_weight_max": 46.607234954833984,
      "activations/layer6_attention_weight_min": -48.68232727050781,
      "activations/layer7_attention_weight_max": 67.0108871459961,
      "activations/layer7_attention_weight_min": -63.844871520996094,
      "activations/layer8_attention_weight_max": 46.57033157348633,
      "activations/layer8_attention_weight_min": -48.8317756652832,
      "activations/layer9_attention_weight_max": 49.242469787597656,
      "activations/layer9_attention_weight_min": -43.77820587158203,
      "epoch": 17.95,
      "learning_rate": 3.460227272727272e-05,
      "loss": 2.7441,
      "step": 308900
    },
    {
      "activations/layer0_attention_weight_max": 17.256378173828125,
      "activations/layer0_attention_weight_min": -13.886717796325684,
      "activations/layer10_attention_weight_max": 35.06641387939453,
      "activations/layer10_attention_weight_min": -34.84410095214844,
      "activations/layer11_attention_weight_max": 33.897518157958984,
      "activations/layer11_attention_weight_min": -31.951330184936523,
      "activations/layer12_attention_weight_max": 18.590845108032227,
      "activations/layer12_attention_weight_min": -25.853145599365234,
      "activations/layer13_attention_weight_max": 35.0395622253418,
      "activations/layer13_attention_weight_min": -27.944351196289062,
      "activations/layer14_attention_weight_max": 46.278724670410156,
      "activations/layer14_attention_weight_min": -31.804105758666992,
      "activations/layer15_attention_weight_max": 40.27579116821289,
      "activations/layer15_attention_weight_min": -30.381418228149414,
      "activations/layer16_attention_weight_max": 34.426212310791016,
      "activations/layer16_attention_weight_min": -34.669185638427734,
      "activations/layer17_attention_weight_max": 50.55183792114258,
      "activations/layer17_attention_weight_min": -47.74814224243164,
      "activations/layer18_attention_weight_max": 48.0491828918457,
      "activations/layer18_attention_weight_min": -40.97066879272461,
      "activations/layer19_attention_weight_max": 27.53700065612793,
      "activations/layer19_attention_weight_min": -23.948556900024414,
      "activations/layer1_attention_weight_max": 16.912721633911133,
      "activations/layer1_attention_weight_min": -14.199512481689453,
      "activations/layer20_attention_weight_max": 29.496965408325195,
      "activations/layer20_attention_weight_min": -21.917633056640625,
      "activations/layer21_attention_weight_max": 43.16407012939453,
      "activations/layer21_attention_weight_min": -25.496166229248047,
      "activations/layer22_attention_weight_max": 33.59619903564453,
      "activations/layer22_attention_weight_min": -27.269500732421875,
      "activations/layer23_attention_weight_max": 41.14198684692383,
      "activations/layer23_attention_weight_min": -23.549480438232422,
      "activations/layer2_attention_weight_max": 32.54597854614258,
      "activations/layer2_attention_weight_min": -30.311756134033203,
      "activations/layer3_attention_weight_max": 92.08785247802734,
      "activations/layer3_attention_weight_min": -92.29138946533203,
      "activations/layer4_attention_weight_max": 84.52093505859375,
      "activations/layer4_attention_weight_min": -86.7419662475586,
      "activations/layer5_attention_weight_max": 60.21430206298828,
      "activations/layer5_attention_weight_min": -71.73005676269531,
      "activations/layer6_attention_weight_max": 48.325382232666016,
      "activations/layer6_attention_weight_min": -48.35261535644531,
      "activations/layer7_attention_weight_max": 62.2995491027832,
      "activations/layer7_attention_weight_min": -64.49075317382812,
      "activations/layer8_attention_weight_max": 44.04384231567383,
      "activations/layer8_attention_weight_min": -46.87600326538086,
      "activations/layer9_attention_weight_max": 45.510169982910156,
      "activations/layer9_attention_weight_min": -43.542877197265625,
      "epoch": 17.95,
      "learning_rate": 3.458333333333333e-05,
      "loss": 2.7445,
      "step": 308950
    },
    {
      "activations/layer0_attention_weight_max": 15.143562316894531,
      "activations/layer0_attention_weight_min": -13.505744934082031,
      "activations/layer10_attention_weight_max": 36.05574417114258,
      "activations/layer10_attention_weight_min": -35.84892272949219,
      "activations/layer11_attention_weight_max": 35.23289489746094,
      "activations/layer11_attention_weight_min": -35.016204833984375,
      "activations/layer12_attention_weight_max": 18.357383728027344,
      "activations/layer12_attention_weight_min": -25.930986404418945,
      "activations/layer13_attention_weight_max": 34.782501220703125,
      "activations/layer13_attention_weight_min": -28.86405372619629,
      "activations/layer14_attention_weight_max": 35.530311584472656,
      "activations/layer14_attention_weight_min": -31.05155372619629,
      "activations/layer15_attention_weight_max": 34.242698669433594,
      "activations/layer15_attention_weight_min": -31.70150375366211,
      "activations/layer16_attention_weight_max": 35.76523971557617,
      "activations/layer16_attention_weight_min": -33.94536590576172,
      "activations/layer17_attention_weight_max": 52.60049057006836,
      "activations/layer17_attention_weight_min": -46.6766242980957,
      "activations/layer18_attention_weight_max": 48.23735046386719,
      "activations/layer18_attention_weight_min": -41.032203674316406,
      "activations/layer19_attention_weight_max": 24.09685707092285,
      "activations/layer19_attention_weight_min": -22.293487548828125,
      "activations/layer1_attention_weight_max": 16.729202270507812,
      "activations/layer1_attention_weight_min": -12.317984580993652,
      "activations/layer20_attention_weight_max": 21.68216323852539,
      "activations/layer20_attention_weight_min": -20.064464569091797,
      "activations/layer21_attention_weight_max": 36.55037307739258,
      "activations/layer21_attention_weight_min": -24.637588500976562,
      "activations/layer22_attention_weight_max": 28.238496780395508,
      "activations/layer22_attention_weight_min": -24.47609519958496,
      "activations/layer23_attention_weight_max": 37.383949279785156,
      "activations/layer23_attention_weight_min": -26.4051570892334,
      "activations/layer2_attention_weight_max": 30.926353454589844,
      "activations/layer2_attention_weight_min": -29.099164962768555,
      "activations/layer3_attention_weight_max": 89.71344757080078,
      "activations/layer3_attention_weight_min": -89.38912963867188,
      "activations/layer4_attention_weight_max": 88.29429626464844,
      "activations/layer4_attention_weight_min": -88.1045150756836,
      "activations/layer5_attention_weight_max": 66.12841796875,
      "activations/layer5_attention_weight_min": -71.10591125488281,
      "activations/layer6_attention_weight_max": 49.791324615478516,
      "activations/layer6_attention_weight_min": -50.82390594482422,
      "activations/layer7_attention_weight_max": 65.57691955566406,
      "activations/layer7_attention_weight_min": -61.66096878051758,
      "activations/layer8_attention_weight_max": 46.774688720703125,
      "activations/layer8_attention_weight_min": -50.91059494018555,
      "activations/layer9_attention_weight_max": 44.270076751708984,
      "activations/layer9_attention_weight_min": -45.24382781982422,
      "epoch": 17.95,
      "learning_rate": 3.456439393939393e-05,
      "loss": 2.7254,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4555,
      "eval_samples_per_second": 507.834,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4555,
      "eval_openwebtext_samples_per_second": 507.834,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9896,
      "eval_wikitext_samples_per_second": 229.189,
      "step": 309000
    },
    {
      "epoch": 17.95,
      "eval_lambada_loss": 2.455078125,
      "eval_lambada_ppl": 11.647343462043494,
      "eval_lambada_runtime": 9.5221,
      "eval_lambada_samples_per_second": 511.337,
      "step": 309000
    },
    {
      "activations/layer0_attention_weight_max": 15.73073959350586,
      "activations/layer0_attention_weight_min": -13.73917293548584,
      "activations/layer10_attention_weight_max": 36.643707275390625,
      "activations/layer10_attention_weight_min": -37.885372161865234,
      "activations/layer11_attention_weight_max": 37.51280212402344,
      "activations/layer11_attention_weight_min": -36.73244094848633,
      "activations/layer12_attention_weight_max": 18.019987106323242,
      "activations/layer12_attention_weight_min": -31.14201545715332,
      "activations/layer13_attention_weight_max": 33.627803802490234,
      "activations/layer13_attention_weight_min": -25.3115291595459,
      "activations/layer14_attention_weight_max": 34.11033248901367,
      "activations/layer14_attention_weight_min": -27.60784912109375,
      "activations/layer15_attention_weight_max": 31.69892692565918,
      "activations/layer15_attention_weight_min": -28.768239974975586,
      "activations/layer16_attention_weight_max": 29.369539260864258,
      "activations/layer16_attention_weight_min": -28.28981590270996,
      "activations/layer17_attention_weight_max": 44.98350143432617,
      "activations/layer17_attention_weight_min": -43.199676513671875,
      "activations/layer18_attention_weight_max": 38.35332489013672,
      "activations/layer18_attention_weight_min": -41.13080978393555,
      "activations/layer19_attention_weight_max": 20.979846954345703,
      "activations/layer19_attention_weight_min": -21.023075103759766,
      "activations/layer1_attention_weight_max": 16.77499771118164,
      "activations/layer1_attention_weight_min": -13.839960098266602,
      "activations/layer20_attention_weight_max": 18.8117733001709,
      "activations/layer20_attention_weight_min": -22.666141510009766,
      "activations/layer21_attention_weight_max": 29.296579360961914,
      "activations/layer21_attention_weight_min": -20.169422149658203,
      "activations/layer22_attention_weight_max": 27.709627151489258,
      "activations/layer22_attention_weight_min": -25.3116397857666,
      "activations/layer23_attention_weight_max": 32.81745910644531,
      "activations/layer23_attention_weight_min": -23.829010009765625,
      "activations/layer2_attention_weight_max": 35.2608642578125,
      "activations/layer2_attention_weight_min": -31.291786193847656,
      "activations/layer3_attention_weight_max": 93.65397644042969,
      "activations/layer3_attention_weight_min": -91.51588439941406,
      "activations/layer4_attention_weight_max": 93.09207916259766,
      "activations/layer4_attention_weight_min": -88.39653015136719,
      "activations/layer5_attention_weight_max": 70.01707458496094,
      "activations/layer5_attention_weight_min": -71.75128936767578,
      "activations/layer6_attention_weight_max": 49.04951095581055,
      "activations/layer6_attention_weight_min": -49.65890121459961,
      "activations/layer7_attention_weight_max": 77.35816955566406,
      "activations/layer7_attention_weight_min": -64.04926300048828,
      "activations/layer8_attention_weight_max": 51.86710739135742,
      "activations/layer8_attention_weight_min": -48.10376739501953,
      "activations/layer9_attention_weight_max": 59.316890716552734,
      "activations/layer9_attention_weight_min": -47.70199966430664,
      "epoch": 17.96,
      "learning_rate": 3.454545454545454e-05,
      "loss": 2.747,
      "step": 309050
    },
    {
      "activations/layer0_attention_weight_max": 15.160126686096191,
      "activations/layer0_attention_weight_min": -13.465286254882812,
      "activations/layer10_attention_weight_max": 35.7266731262207,
      "activations/layer10_attention_weight_min": -35.880340576171875,
      "activations/layer11_attention_weight_max": 32.919979095458984,
      "activations/layer11_attention_weight_min": -35.34675979614258,
      "activations/layer12_attention_weight_max": 17.41156768798828,
      "activations/layer12_attention_weight_min": -24.330774307250977,
      "activations/layer13_attention_weight_max": 34.69623565673828,
      "activations/layer13_attention_weight_min": -27.73746109008789,
      "activations/layer14_attention_weight_max": 34.50012969970703,
      "activations/layer14_attention_weight_min": -32.238319396972656,
      "activations/layer15_attention_weight_max": 32.54985046386719,
      "activations/layer15_attention_weight_min": -31.287254333496094,
      "activations/layer16_attention_weight_max": 32.240821838378906,
      "activations/layer16_attention_weight_min": -32.017173767089844,
      "activations/layer17_attention_weight_max": 50.708290100097656,
      "activations/layer17_attention_weight_min": -45.28598403930664,
      "activations/layer18_attention_weight_max": 45.00002670288086,
      "activations/layer18_attention_weight_min": -42.68601608276367,
      "activations/layer19_attention_weight_max": 22.96699333190918,
      "activations/layer19_attention_weight_min": -25.03976058959961,
      "activations/layer1_attention_weight_max": 15.628768920898438,
      "activations/layer1_attention_weight_min": -13.920267105102539,
      "activations/layer20_attention_weight_max": 22.544090270996094,
      "activations/layer20_attention_weight_min": -21.39055633544922,
      "activations/layer21_attention_weight_max": 33.1655158996582,
      "activations/layer21_attention_weight_min": -21.56247329711914,
      "activations/layer22_attention_weight_max": 27.64794158935547,
      "activations/layer22_attention_weight_min": -25.245861053466797,
      "activations/layer23_attention_weight_max": 37.38925552368164,
      "activations/layer23_attention_weight_min": -26.381813049316406,
      "activations/layer2_attention_weight_max": 31.00275993347168,
      "activations/layer2_attention_weight_min": -30.038272857666016,
      "activations/layer3_attention_weight_max": 87.70441436767578,
      "activations/layer3_attention_weight_min": -90.74445343017578,
      "activations/layer4_attention_weight_max": 84.61345672607422,
      "activations/layer4_attention_weight_min": -87.38703155517578,
      "activations/layer5_attention_weight_max": 61.6154899597168,
      "activations/layer5_attention_weight_min": -72.11349487304688,
      "activations/layer6_attention_weight_max": 49.187801361083984,
      "activations/layer6_attention_weight_min": -49.89399337768555,
      "activations/layer7_attention_weight_max": 63.316219329833984,
      "activations/layer7_attention_weight_min": -63.49106216430664,
      "activations/layer8_attention_weight_max": 44.69826126098633,
      "activations/layer8_attention_weight_min": -49.457244873046875,
      "activations/layer9_attention_weight_max": 43.67454528808594,
      "activations/layer9_attention_weight_min": -46.56834030151367,
      "epoch": 17.96,
      "learning_rate": 3.452651515151515e-05,
      "loss": 2.7333,
      "step": 309100
    },
    {
      "activations/layer0_attention_weight_max": 15.353059768676758,
      "activations/layer0_attention_weight_min": -13.464487075805664,
      "activations/layer10_attention_weight_max": 35.516502380371094,
      "activations/layer10_attention_weight_min": -34.28807830810547,
      "activations/layer11_attention_weight_max": 32.72132110595703,
      "activations/layer11_attention_weight_min": -32.91007995605469,
      "activations/layer12_attention_weight_max": 18.24484634399414,
      "activations/layer12_attention_weight_min": -25.618379592895508,
      "activations/layer13_attention_weight_max": 33.62115478515625,
      "activations/layer13_attention_weight_min": -30.064712524414062,
      "activations/layer14_attention_weight_max": 33.91408920288086,
      "activations/layer14_attention_weight_min": -29.62993812561035,
      "activations/layer15_attention_weight_max": 31.754297256469727,
      "activations/layer15_attention_weight_min": -29.777963638305664,
      "activations/layer16_attention_weight_max": 32.1224250793457,
      "activations/layer16_attention_weight_min": -32.15158462524414,
      "activations/layer17_attention_weight_max": 51.46980667114258,
      "activations/layer17_attention_weight_min": -43.64855194091797,
      "activations/layer18_attention_weight_max": 49.421085357666016,
      "activations/layer18_attention_weight_min": -41.02908706665039,
      "activations/layer19_attention_weight_max": 23.81174087524414,
      "activations/layer19_attention_weight_min": -23.948667526245117,
      "activations/layer1_attention_weight_max": 15.723380088806152,
      "activations/layer1_attention_weight_min": -14.550957679748535,
      "activations/layer20_attention_weight_max": 23.526887893676758,
      "activations/layer20_attention_weight_min": -22.10489845275879,
      "activations/layer21_attention_weight_max": 33.13689041137695,
      "activations/layer21_attention_weight_min": -22.267946243286133,
      "activations/layer22_attention_weight_max": 29.842004776000977,
      "activations/layer22_attention_weight_min": -26.243968963623047,
      "activations/layer23_attention_weight_max": 35.99701690673828,
      "activations/layer23_attention_weight_min": -23.645214080810547,
      "activations/layer2_attention_weight_max": 29.629682540893555,
      "activations/layer2_attention_weight_min": -28.524229049682617,
      "activations/layer3_attention_weight_max": 82.66314697265625,
      "activations/layer3_attention_weight_min": -81.93541717529297,
      "activations/layer4_attention_weight_max": 82.72942352294922,
      "activations/layer4_attention_weight_min": -82.86707305908203,
      "activations/layer5_attention_weight_max": 64.59503936767578,
      "activations/layer5_attention_weight_min": -71.25157165527344,
      "activations/layer6_attention_weight_max": 44.97335433959961,
      "activations/layer6_attention_weight_min": -46.89035415649414,
      "activations/layer7_attention_weight_max": 61.31209945678711,
      "activations/layer7_attention_weight_min": -57.812599182128906,
      "activations/layer8_attention_weight_max": 42.7456169128418,
      "activations/layer8_attention_weight_min": -45.87644958496094,
      "activations/layer9_attention_weight_max": 41.41355895996094,
      "activations/layer9_attention_weight_min": -46.43572998046875,
      "epoch": 17.96,
      "learning_rate": 3.450757575757576e-05,
      "loss": 2.7456,
      "step": 309150
    },
    {
      "activations/layer0_attention_weight_max": 15.32076358795166,
      "activations/layer0_attention_weight_min": -13.3969144821167,
      "activations/layer10_attention_weight_max": 33.44594192504883,
      "activations/layer10_attention_weight_min": -33.85955047607422,
      "activations/layer11_attention_weight_max": 34.29707336425781,
      "activations/layer11_attention_weight_min": -34.021968841552734,
      "activations/layer12_attention_weight_max": 18.80140495300293,
      "activations/layer12_attention_weight_min": -27.372163772583008,
      "activations/layer13_attention_weight_max": 37.13397216796875,
      "activations/layer13_attention_weight_min": -27.84397315979004,
      "activations/layer14_attention_weight_max": 34.78461837768555,
      "activations/layer14_attention_weight_min": -29.802188873291016,
      "activations/layer15_attention_weight_max": 32.30683135986328,
      "activations/layer15_attention_weight_min": -29.27814292907715,
      "activations/layer16_attention_weight_max": 32.44479751586914,
      "activations/layer16_attention_weight_min": -30.737573623657227,
      "activations/layer17_attention_weight_max": 49.10399627685547,
      "activations/layer17_attention_weight_min": -43.3035774230957,
      "activations/layer18_attention_weight_max": 46.47573471069336,
      "activations/layer18_attention_weight_min": -40.16572570800781,
      "activations/layer19_attention_weight_max": 25.61263084411621,
      "activations/layer19_attention_weight_min": -24.832609176635742,
      "activations/layer1_attention_weight_max": 18.108814239501953,
      "activations/layer1_attention_weight_min": -13.62501335144043,
      "activations/layer20_attention_weight_max": 25.28090476989746,
      "activations/layer20_attention_weight_min": -23.63462257385254,
      "activations/layer21_attention_weight_max": 37.70368576049805,
      "activations/layer21_attention_weight_min": -21.446924209594727,
      "activations/layer22_attention_weight_max": 30.860363006591797,
      "activations/layer22_attention_weight_min": -25.84629249572754,
      "activations/layer23_attention_weight_max": 41.97991943359375,
      "activations/layer23_attention_weight_min": -22.358402252197266,
      "activations/layer2_attention_weight_max": 31.345718383789062,
      "activations/layer2_attention_weight_min": -29.41668701171875,
      "activations/layer3_attention_weight_max": 85.66271209716797,
      "activations/layer3_attention_weight_min": -86.11971282958984,
      "activations/layer4_attention_weight_max": 84.80492401123047,
      "activations/layer4_attention_weight_min": -86.70628356933594,
      "activations/layer5_attention_weight_max": 60.56928253173828,
      "activations/layer5_attention_weight_min": -73.23538970947266,
      "activations/layer6_attention_weight_max": 48.7979621887207,
      "activations/layer6_attention_weight_min": -49.21023941040039,
      "activations/layer7_attention_weight_max": 63.29476547241211,
      "activations/layer7_attention_weight_min": -60.156715393066406,
      "activations/layer8_attention_weight_max": 44.72303009033203,
      "activations/layer8_attention_weight_min": -48.90093231201172,
      "activations/layer9_attention_weight_max": 43.80918502807617,
      "activations/layer9_attention_weight_min": -43.99186325073242,
      "epoch": 17.97,
      "learning_rate": 3.448863636363636e-05,
      "loss": 2.7533,
      "step": 309200
    },
    {
      "activations/layer0_attention_weight_max": 16.45363426208496,
      "activations/layer0_attention_weight_min": -13.777737617492676,
      "activations/layer10_attention_weight_max": 35.52306365966797,
      "activations/layer10_attention_weight_min": -35.66483688354492,
      "activations/layer11_attention_weight_max": 32.83270263671875,
      "activations/layer11_attention_weight_min": -34.37037658691406,
      "activations/layer12_attention_weight_max": 18.606855392456055,
      "activations/layer12_attention_weight_min": -24.16510581970215,
      "activations/layer13_attention_weight_max": 35.005157470703125,
      "activations/layer13_attention_weight_min": -27.978790283203125,
      "activations/layer14_attention_weight_max": 35.50417709350586,
      "activations/layer14_attention_weight_min": -29.91510581970215,
      "activations/layer15_attention_weight_max": 32.544097900390625,
      "activations/layer15_attention_weight_min": -28.680788040161133,
      "activations/layer16_attention_weight_max": 31.89647674560547,
      "activations/layer16_attention_weight_min": -31.19334602355957,
      "activations/layer17_attention_weight_max": 49.78590774536133,
      "activations/layer17_attention_weight_min": -46.02313232421875,
      "activations/layer18_attention_weight_max": 43.730567932128906,
      "activations/layer18_attention_weight_min": -39.49921798706055,
      "activations/layer19_attention_weight_max": 23.2418155670166,
      "activations/layer19_attention_weight_min": -22.16065216064453,
      "activations/layer1_attention_weight_max": 15.774934768676758,
      "activations/layer1_attention_weight_min": -13.05411148071289,
      "activations/layer20_attention_weight_max": 23.00554084777832,
      "activations/layer20_attention_weight_min": -21.12345314025879,
      "activations/layer21_attention_weight_max": 36.24517822265625,
      "activations/layer21_attention_weight_min": -24.125835418701172,
      "activations/layer22_attention_weight_max": 27.716094970703125,
      "activations/layer22_attention_weight_min": -25.25202751159668,
      "activations/layer23_attention_weight_max": 34.13221740722656,
      "activations/layer23_attention_weight_min": -25.57740020751953,
      "activations/layer2_attention_weight_max": 29.701736450195312,
      "activations/layer2_attention_weight_min": -28.019643783569336,
      "activations/layer3_attention_weight_max": 79.54590606689453,
      "activations/layer3_attention_weight_min": -88.3007583618164,
      "activations/layer4_attention_weight_max": 81.4685287475586,
      "activations/layer4_attention_weight_min": -82.34687805175781,
      "activations/layer5_attention_weight_max": 63.6959228515625,
      "activations/layer5_attention_weight_min": -72.350341796875,
      "activations/layer6_attention_weight_max": 48.80970001220703,
      "activations/layer6_attention_weight_min": -50.04496383666992,
      "activations/layer7_attention_weight_max": 59.446876525878906,
      "activations/layer7_attention_weight_min": -65.6090316772461,
      "activations/layer8_attention_weight_max": 43.65253829956055,
      "activations/layer8_attention_weight_min": -46.45998001098633,
      "activations/layer9_attention_weight_max": 43.54555892944336,
      "activations/layer9_attention_weight_min": -44.01722717285156,
      "epoch": 17.97,
      "learning_rate": 3.446969696969697e-05,
      "loss": 2.7482,
      "step": 309250
    },
    {
      "activations/layer0_attention_weight_max": 15.516498565673828,
      "activations/layer0_attention_weight_min": -13.578441619873047,
      "activations/layer10_attention_weight_max": 34.22175979614258,
      "activations/layer10_attention_weight_min": -34.8173713684082,
      "activations/layer11_attention_weight_max": 32.38463592529297,
      "activations/layer11_attention_weight_min": -34.23111343383789,
      "activations/layer12_attention_weight_max": 18.58999252319336,
      "activations/layer12_attention_weight_min": -27.343650817871094,
      "activations/layer13_attention_weight_max": 33.4970703125,
      "activations/layer13_attention_weight_min": -27.575260162353516,
      "activations/layer14_attention_weight_max": 36.570518493652344,
      "activations/layer14_attention_weight_min": -29.24932861328125,
      "activations/layer15_attention_weight_max": 32.618770599365234,
      "activations/layer15_attention_weight_min": -29.297496795654297,
      "activations/layer16_attention_weight_max": 33.15477752685547,
      "activations/layer16_attention_weight_min": -30.771509170532227,
      "activations/layer17_attention_weight_max": 51.802066802978516,
      "activations/layer17_attention_weight_min": -46.93243408203125,
      "activations/layer18_attention_weight_max": 48.12922286987305,
      "activations/layer18_attention_weight_min": -42.29057312011719,
      "activations/layer19_attention_weight_max": 23.5960636138916,
      "activations/layer19_attention_weight_min": -24.104646682739258,
      "activations/layer1_attention_weight_max": 16.931238174438477,
      "activations/layer1_attention_weight_min": -14.5864896774292,
      "activations/layer20_attention_weight_max": 23.87906837463379,
      "activations/layer20_attention_weight_min": -24.13471221923828,
      "activations/layer21_attention_weight_max": 41.47561264038086,
      "activations/layer21_attention_weight_min": -22.86670684814453,
      "activations/layer22_attention_weight_max": 29.523998260498047,
      "activations/layer22_attention_weight_min": -25.489025115966797,
      "activations/layer23_attention_weight_max": 38.823150634765625,
      "activations/layer23_attention_weight_min": -24.071521759033203,
      "activations/layer2_attention_weight_max": 31.743324279785156,
      "activations/layer2_attention_weight_min": -32.95807647705078,
      "activations/layer3_attention_weight_max": 84.66297149658203,
      "activations/layer3_attention_weight_min": -84.7265396118164,
      "activations/layer4_attention_weight_max": 84.2553482055664,
      "activations/layer4_attention_weight_min": -85.27654266357422,
      "activations/layer5_attention_weight_max": 58.815269470214844,
      "activations/layer5_attention_weight_min": -78.8410873413086,
      "activations/layer6_attention_weight_max": 47.53372573852539,
      "activations/layer6_attention_weight_min": -49.38566970825195,
      "activations/layer7_attention_weight_max": 62.97037124633789,
      "activations/layer7_attention_weight_min": -69.52627563476562,
      "activations/layer8_attention_weight_max": 45.556358337402344,
      "activations/layer8_attention_weight_min": -48.03432846069336,
      "activations/layer9_attention_weight_max": 47.91670608520508,
      "activations/layer9_attention_weight_min": -45.147396087646484,
      "epoch": 17.97,
      "learning_rate": 3.445075757575757e-05,
      "loss": 2.725,
      "step": 309300
    },
    {
      "activations/layer0_attention_weight_max": 15.501680374145508,
      "activations/layer0_attention_weight_min": -13.306488990783691,
      "activations/layer10_attention_weight_max": 36.82626724243164,
      "activations/layer10_attention_weight_min": -35.78950881958008,
      "activations/layer11_attention_weight_max": 32.31975555419922,
      "activations/layer11_attention_weight_min": -35.903236389160156,
      "activations/layer12_attention_weight_max": 18.257665634155273,
      "activations/layer12_attention_weight_min": -24.06339454650879,
      "activations/layer13_attention_weight_max": 34.66726303100586,
      "activations/layer13_attention_weight_min": -27.532516479492188,
      "activations/layer14_attention_weight_max": 34.270599365234375,
      "activations/layer14_attention_weight_min": -28.6453857421875,
      "activations/layer15_attention_weight_max": 32.21223449707031,
      "activations/layer15_attention_weight_min": -29.689697265625,
      "activations/layer16_attention_weight_max": 32.50851058959961,
      "activations/layer16_attention_weight_min": -33.75126266479492,
      "activations/layer17_attention_weight_max": 51.02335739135742,
      "activations/layer17_attention_weight_min": -47.719688415527344,
      "activations/layer18_attention_weight_max": 46.682674407958984,
      "activations/layer18_attention_weight_min": -39.05764389038086,
      "activations/layer19_attention_weight_max": 27.003376007080078,
      "activations/layer19_attention_weight_min": -22.642227172851562,
      "activations/layer1_attention_weight_max": 15.403614044189453,
      "activations/layer1_attention_weight_min": -13.18340015411377,
      "activations/layer20_attention_weight_max": 24.18357276916504,
      "activations/layer20_attention_weight_min": -20.302595138549805,
      "activations/layer21_attention_weight_max": 37.64163589477539,
      "activations/layer21_attention_weight_min": -23.61590003967285,
      "activations/layer22_attention_weight_max": 29.764558792114258,
      "activations/layer22_attention_weight_min": -26.34471321105957,
      "activations/layer23_attention_weight_max": 37.35702896118164,
      "activations/layer23_attention_weight_min": -25.381675720214844,
      "activations/layer2_attention_weight_max": 31.860342025756836,
      "activations/layer2_attention_weight_min": -29.215986251831055,
      "activations/layer3_attention_weight_max": 87.89048767089844,
      "activations/layer3_attention_weight_min": -85.96935272216797,
      "activations/layer4_attention_weight_max": 86.21625518798828,
      "activations/layer4_attention_weight_min": -85.41609191894531,
      "activations/layer5_attention_weight_max": 64.1747817993164,
      "activations/layer5_attention_weight_min": -73.01704406738281,
      "activations/layer6_attention_weight_max": 51.847415924072266,
      "activations/layer6_attention_weight_min": -50.27470397949219,
      "activations/layer7_attention_weight_max": 65.07709503173828,
      "activations/layer7_attention_weight_min": -61.73883819580078,
      "activations/layer8_attention_weight_max": 45.53315353393555,
      "activations/layer8_attention_weight_min": -49.458683013916016,
      "activations/layer9_attention_weight_max": 41.8187255859375,
      "activations/layer9_attention_weight_min": -44.82566833496094,
      "epoch": 17.97,
      "learning_rate": 3.443181818181818e-05,
      "loss": 2.7287,
      "step": 309350
    },
    {
      "activations/layer0_attention_weight_max": 15.757525444030762,
      "activations/layer0_attention_weight_min": -13.092888832092285,
      "activations/layer10_attention_weight_max": 38.32863235473633,
      "activations/layer10_attention_weight_min": -35.94906234741211,
      "activations/layer11_attention_weight_max": 34.88859558105469,
      "activations/layer11_attention_weight_min": -36.085594177246094,
      "activations/layer12_attention_weight_max": 19.36956787109375,
      "activations/layer12_attention_weight_min": -24.11229133605957,
      "activations/layer13_attention_weight_max": 33.19636154174805,
      "activations/layer13_attention_weight_min": -31.12470245361328,
      "activations/layer14_attention_weight_max": 35.46068572998047,
      "activations/layer14_attention_weight_min": -31.172929763793945,
      "activations/layer15_attention_weight_max": 31.02067756652832,
      "activations/layer15_attention_weight_min": -30.590970993041992,
      "activations/layer16_attention_weight_max": 33.208152770996094,
      "activations/layer16_attention_weight_min": -31.51239013671875,
      "activations/layer17_attention_weight_max": 52.85224533081055,
      "activations/layer17_attention_weight_min": -44.6474609375,
      "activations/layer18_attention_weight_max": 46.30183029174805,
      "activations/layer18_attention_weight_min": -39.48286056518555,
      "activations/layer19_attention_weight_max": 23.01791000366211,
      "activations/layer19_attention_weight_min": -23.36295509338379,
      "activations/layer1_attention_weight_max": 16.566762924194336,
      "activations/layer1_attention_weight_min": -14.192890167236328,
      "activations/layer20_attention_weight_max": 20.989952087402344,
      "activations/layer20_attention_weight_min": -24.404502868652344,
      "activations/layer21_attention_weight_max": 33.11612319946289,
      "activations/layer21_attention_weight_min": -22.715505599975586,
      "activations/layer22_attention_weight_max": 30.382600784301758,
      "activations/layer22_attention_weight_min": -24.480045318603516,
      "activations/layer23_attention_weight_max": 38.78549575805664,
      "activations/layer23_attention_weight_min": -24.20608139038086,
      "activations/layer2_attention_weight_max": 30.275959014892578,
      "activations/layer2_attention_weight_min": -28.914573669433594,
      "activations/layer3_attention_weight_max": 85.68933868408203,
      "activations/layer3_attention_weight_min": -85.42406463623047,
      "activations/layer4_attention_weight_max": 83.20975494384766,
      "activations/layer4_attention_weight_min": -82.67357635498047,
      "activations/layer5_attention_weight_max": 62.33009338378906,
      "activations/layer5_attention_weight_min": -70.78130340576172,
      "activations/layer6_attention_weight_max": 49.95713424682617,
      "activations/layer6_attention_weight_min": -51.336997985839844,
      "activations/layer7_attention_weight_max": 64.54344940185547,
      "activations/layer7_attention_weight_min": -62.3171501159668,
      "activations/layer8_attention_weight_max": 46.11097717285156,
      "activations/layer8_attention_weight_min": -45.995723724365234,
      "activations/layer9_attention_weight_max": 46.17836380004883,
      "activations/layer9_attention_weight_min": -44.53530502319336,
      "epoch": 17.98,
      "learning_rate": 3.4412878787878786e-05,
      "loss": 2.7568,
      "step": 309400
    },
    {
      "activations/layer0_attention_weight_max": 15.892950057983398,
      "activations/layer0_attention_weight_min": -13.172422409057617,
      "activations/layer10_attention_weight_max": 35.87942886352539,
      "activations/layer10_attention_weight_min": -36.88825225830078,
      "activations/layer11_attention_weight_max": 32.1573600769043,
      "activations/layer11_attention_weight_min": -34.348854064941406,
      "activations/layer12_attention_weight_max": 18.38951301574707,
      "activations/layer12_attention_weight_min": -26.63437843322754,
      "activations/layer13_attention_weight_max": 34.771400451660156,
      "activations/layer13_attention_weight_min": -28.35480499267578,
      "activations/layer14_attention_weight_max": 39.69660568237305,
      "activations/layer14_attention_weight_min": -30.116016387939453,
      "activations/layer15_attention_weight_max": 36.428138732910156,
      "activations/layer15_attention_weight_min": -29.358129501342773,
      "activations/layer16_attention_weight_max": 35.273460388183594,
      "activations/layer16_attention_weight_min": -32.20058059692383,
      "activations/layer17_attention_weight_max": 51.49333572387695,
      "activations/layer17_attention_weight_min": -46.95947265625,
      "activations/layer18_attention_weight_max": 50.97393035888672,
      "activations/layer18_attention_weight_min": -45.05964279174805,
      "activations/layer19_attention_weight_max": 27.728689193725586,
      "activations/layer19_attention_weight_min": -26.107229232788086,
      "activations/layer1_attention_weight_max": 17.701274871826172,
      "activations/layer1_attention_weight_min": -14.338997840881348,
      "activations/layer20_attention_weight_max": 28.595951080322266,
      "activations/layer20_attention_weight_min": -22.156497955322266,
      "activations/layer21_attention_weight_max": 48.15013122558594,
      "activations/layer21_attention_weight_min": -26.90886688232422,
      "activations/layer22_attention_weight_max": 32.309356689453125,
      "activations/layer22_attention_weight_min": -26.775903701782227,
      "activations/layer23_attention_weight_max": 40.37328338623047,
      "activations/layer23_attention_weight_min": -25.853750228881836,
      "activations/layer2_attention_weight_max": 30.596593856811523,
      "activations/layer2_attention_weight_min": -29.092382431030273,
      "activations/layer3_attention_weight_max": 86.22127532958984,
      "activations/layer3_attention_weight_min": -86.43914031982422,
      "activations/layer4_attention_weight_max": 87.82933044433594,
      "activations/layer4_attention_weight_min": -83.58763885498047,
      "activations/layer5_attention_weight_max": 61.38096618652344,
      "activations/layer5_attention_weight_min": -71.71595001220703,
      "activations/layer6_attention_weight_max": 48.215965270996094,
      "activations/layer6_attention_weight_min": -50.1750373840332,
      "activations/layer7_attention_weight_max": 65.7742691040039,
      "activations/layer7_attention_weight_min": -62.04100799560547,
      "activations/layer8_attention_weight_max": 46.77116775512695,
      "activations/layer8_attention_weight_min": -47.569461822509766,
      "activations/layer9_attention_weight_max": 45.37395477294922,
      "activations/layer9_attention_weight_min": -45.12397766113281,
      "epoch": 17.98,
      "learning_rate": 3.4393939393939394e-05,
      "loss": 2.7435,
      "step": 309450
    },
    {
      "activations/layer0_attention_weight_max": 15.4815092086792,
      "activations/layer0_attention_weight_min": -13.286724090576172,
      "activations/layer10_attention_weight_max": 38.697357177734375,
      "activations/layer10_attention_weight_min": -37.95048904418945,
      "activations/layer11_attention_weight_max": 34.85076141357422,
      "activations/layer11_attention_weight_min": -37.41487503051758,
      "activations/layer12_attention_weight_max": 18.913848876953125,
      "activations/layer12_attention_weight_min": -24.654809951782227,
      "activations/layer13_attention_weight_max": 34.28384780883789,
      "activations/layer13_attention_weight_min": -29.426883697509766,
      "activations/layer14_attention_weight_max": 33.6976318359375,
      "activations/layer14_attention_weight_min": -31.699920654296875,
      "activations/layer15_attention_weight_max": 32.44220733642578,
      "activations/layer15_attention_weight_min": -31.742570877075195,
      "activations/layer16_attention_weight_max": 35.279075622558594,
      "activations/layer16_attention_weight_min": -33.09148406982422,
      "activations/layer17_attention_weight_max": 52.81555938720703,
      "activations/layer17_attention_weight_min": -47.22385025024414,
      "activations/layer18_attention_weight_max": 51.17254638671875,
      "activations/layer18_attention_weight_min": -42.32805252075195,
      "activations/layer19_attention_weight_max": 26.266828536987305,
      "activations/layer19_attention_weight_min": -24.448122024536133,
      "activations/layer1_attention_weight_max": 15.602043151855469,
      "activations/layer1_attention_weight_min": -12.580025672912598,
      "activations/layer20_attention_weight_max": 26.044498443603516,
      "activations/layer20_attention_weight_min": -23.012126922607422,
      "activations/layer21_attention_weight_max": 39.24171447753906,
      "activations/layer21_attention_weight_min": -20.99159812927246,
      "activations/layer22_attention_weight_max": 32.456016540527344,
      "activations/layer22_attention_weight_min": -27.966035842895508,
      "activations/layer23_attention_weight_max": 40.39149475097656,
      "activations/layer23_attention_weight_min": -24.318113327026367,
      "activations/layer2_attention_weight_max": 30.769405364990234,
      "activations/layer2_attention_weight_min": -29.43244171142578,
      "activations/layer3_attention_weight_max": 85.33129119873047,
      "activations/layer3_attention_weight_min": -89.27794647216797,
      "activations/layer4_attention_weight_max": 87.19596862792969,
      "activations/layer4_attention_weight_min": -89.21794891357422,
      "activations/layer5_attention_weight_max": 67.96728515625,
      "activations/layer5_attention_weight_min": -73.28549194335938,
      "activations/layer6_attention_weight_max": 53.92884826660156,
      "activations/layer6_attention_weight_min": -55.28213882446289,
      "activations/layer7_attention_weight_max": 62.8391227722168,
      "activations/layer7_attention_weight_min": -66.3250961303711,
      "activations/layer8_attention_weight_max": 49.46345901489258,
      "activations/layer8_attention_weight_min": -49.52463912963867,
      "activations/layer9_attention_weight_max": 47.92224884033203,
      "activations/layer9_attention_weight_min": -47.38966751098633,
      "epoch": 17.98,
      "learning_rate": 3.4374999999999995e-05,
      "loss": 2.7461,
      "step": 309500
    },
    {
      "activations/layer0_attention_weight_max": 15.380069732666016,
      "activations/layer0_attention_weight_min": -13.648624420166016,
      "activations/layer10_attention_weight_max": 37.41902542114258,
      "activations/layer10_attention_weight_min": -33.589290618896484,
      "activations/layer11_attention_weight_max": 30.95853042602539,
      "activations/layer11_attention_weight_min": -32.698265075683594,
      "activations/layer12_attention_weight_max": 19.456844329833984,
      "activations/layer12_attention_weight_min": -26.184457778930664,
      "activations/layer13_attention_weight_max": 38.55121612548828,
      "activations/layer13_attention_weight_min": -26.91469955444336,
      "activations/layer14_attention_weight_max": 36.00651168823242,
      "activations/layer14_attention_weight_min": -31.233675003051758,
      "activations/layer15_attention_weight_max": 36.08244705200195,
      "activations/layer15_attention_weight_min": -29.213733673095703,
      "activations/layer16_attention_weight_max": 34.50558853149414,
      "activations/layer16_attention_weight_min": -32.77363967895508,
      "activations/layer17_attention_weight_max": 53.00442123413086,
      "activations/layer17_attention_weight_min": -45.455631256103516,
      "activations/layer18_attention_weight_max": 49.65903854370117,
      "activations/layer18_attention_weight_min": -40.010833740234375,
      "activations/layer19_attention_weight_max": 25.146535873413086,
      "activations/layer19_attention_weight_min": -21.15477180480957,
      "activations/layer1_attention_weight_max": 15.628946304321289,
      "activations/layer1_attention_weight_min": -13.178738594055176,
      "activations/layer20_attention_weight_max": 24.401512145996094,
      "activations/layer20_attention_weight_min": -20.790996551513672,
      "activations/layer21_attention_weight_max": 35.90718460083008,
      "activations/layer21_attention_weight_min": -21.9316463470459,
      "activations/layer22_attention_weight_max": 29.987621307373047,
      "activations/layer22_attention_weight_min": -25.385040283203125,
      "activations/layer23_attention_weight_max": 38.626766204833984,
      "activations/layer23_attention_weight_min": -22.52345085144043,
      "activations/layer2_attention_weight_max": 28.51907730102539,
      "activations/layer2_attention_weight_min": -26.606609344482422,
      "activations/layer3_attention_weight_max": 80.11231231689453,
      "activations/layer3_attention_weight_min": -82.64784240722656,
      "activations/layer4_attention_weight_max": 83.59784698486328,
      "activations/layer4_attention_weight_min": -80.60741424560547,
      "activations/layer5_attention_weight_max": 58.183753967285156,
      "activations/layer5_attention_weight_min": -68.47130584716797,
      "activations/layer6_attention_weight_max": 47.89252853393555,
      "activations/layer6_attention_weight_min": -49.81867218017578,
      "activations/layer7_attention_weight_max": 60.954856872558594,
      "activations/layer7_attention_weight_min": -60.183345794677734,
      "activations/layer8_attention_weight_max": 44.90123748779297,
      "activations/layer8_attention_weight_min": -46.049583435058594,
      "activations/layer9_attention_weight_max": 43.06937789916992,
      "activations/layer9_attention_weight_min": -43.993858337402344,
      "epoch": 17.99,
      "learning_rate": 3.435643939393939e-05,
      "loss": 2.7572,
      "step": 309550
    },
    {
      "activations/layer0_attention_weight_max": 15.52662467956543,
      "activations/layer0_attention_weight_min": -13.484330177307129,
      "activations/layer10_attention_weight_max": 35.60590362548828,
      "activations/layer10_attention_weight_min": -34.3653564453125,
      "activations/layer11_attention_weight_max": 33.91569137573242,
      "activations/layer11_attention_weight_min": -34.45355224609375,
      "activations/layer12_attention_weight_max": 19.3760929107666,
      "activations/layer12_attention_weight_min": -27.91794204711914,
      "activations/layer13_attention_weight_max": 35.06272888183594,
      "activations/layer13_attention_weight_min": -28.108356475830078,
      "activations/layer14_attention_weight_max": 36.88815689086914,
      "activations/layer14_attention_weight_min": -30.49571418762207,
      "activations/layer15_attention_weight_max": 33.4063720703125,
      "activations/layer15_attention_weight_min": -29.73215103149414,
      "activations/layer16_attention_weight_max": 33.285858154296875,
      "activations/layer16_attention_weight_min": -31.035802841186523,
      "activations/layer17_attention_weight_max": 51.22494125366211,
      "activations/layer17_attention_weight_min": -45.123775482177734,
      "activations/layer18_attention_weight_max": 43.86262130737305,
      "activations/layer18_attention_weight_min": -38.924861907958984,
      "activations/layer19_attention_weight_max": 22.416784286499023,
      "activations/layer19_attention_weight_min": -22.53505516052246,
      "activations/layer1_attention_weight_max": 15.104022026062012,
      "activations/layer1_attention_weight_min": -14.502348899841309,
      "activations/layer20_attention_weight_max": 22.719480514526367,
      "activations/layer20_attention_weight_min": -21.83351707458496,
      "activations/layer21_attention_weight_max": 40.292724609375,
      "activations/layer21_attention_weight_min": -23.202476501464844,
      "activations/layer22_attention_weight_max": 27.917226791381836,
      "activations/layer22_attention_weight_min": -25.77313995361328,
      "activations/layer23_attention_weight_max": 36.351375579833984,
      "activations/layer23_attention_weight_min": -27.252277374267578,
      "activations/layer2_attention_weight_max": 31.22236442565918,
      "activations/layer2_attention_weight_min": -29.623809814453125,
      "activations/layer3_attention_weight_max": 84.4985580444336,
      "activations/layer3_attention_weight_min": -84.31584930419922,
      "activations/layer4_attention_weight_max": 86.11162567138672,
      "activations/layer4_attention_weight_min": -82.91191101074219,
      "activations/layer5_attention_weight_max": 60.21520233154297,
      "activations/layer5_attention_weight_min": -68.26884460449219,
      "activations/layer6_attention_weight_max": 47.95433807373047,
      "activations/layer6_attention_weight_min": -49.002479553222656,
      "activations/layer7_attention_weight_max": 60.62577819824219,
      "activations/layer7_attention_weight_min": -64.59237670898438,
      "activations/layer8_attention_weight_max": 43.70718765258789,
      "activations/layer8_attention_weight_min": -47.546722412109375,
      "activations/layer9_attention_weight_max": 42.86045455932617,
      "activations/layer9_attention_weight_min": -44.14358901977539,
      "epoch": 17.99,
      "learning_rate": 3.4337499999999994e-05,
      "loss": 2.734,
      "step": 309600
    },
    {
      "activations/layer0_attention_weight_max": 16.329906463623047,
      "activations/layer0_attention_weight_min": -13.365002632141113,
      "activations/layer10_attention_weight_max": 35.837318420410156,
      "activations/layer10_attention_weight_min": -35.343605041503906,
      "activations/layer11_attention_weight_max": 32.909454345703125,
      "activations/layer11_attention_weight_min": -32.863197326660156,
      "activations/layer12_attention_weight_max": 17.646957397460938,
      "activations/layer12_attention_weight_min": -27.359146118164062,
      "activations/layer13_attention_weight_max": 30.510887145996094,
      "activations/layer13_attention_weight_min": -28.819744110107422,
      "activations/layer14_attention_weight_max": 34.914642333984375,
      "activations/layer14_attention_weight_min": -31.697298049926758,
      "activations/layer15_attention_weight_max": 31.311613082885742,
      "activations/layer15_attention_weight_min": -31.216358184814453,
      "activations/layer16_attention_weight_max": 33.254398345947266,
      "activations/layer16_attention_weight_min": -31.50518798828125,
      "activations/layer17_attention_weight_max": 49.810935974121094,
      "activations/layer17_attention_weight_min": -45.52229690551758,
      "activations/layer18_attention_weight_max": 45.44101333618164,
      "activations/layer18_attention_weight_min": -41.01545333862305,
      "activations/layer19_attention_weight_max": 25.616247177124023,
      "activations/layer19_attention_weight_min": -24.6817569732666,
      "activations/layer1_attention_weight_max": 16.87864112854004,
      "activations/layer1_attention_weight_min": -13.24143123626709,
      "activations/layer20_attention_weight_max": 25.338951110839844,
      "activations/layer20_attention_weight_min": -23.77845001220703,
      "activations/layer21_attention_weight_max": 40.1429328918457,
      "activations/layer21_attention_weight_min": -23.334070205688477,
      "activations/layer22_attention_weight_max": 34.68085479736328,
      "activations/layer22_attention_weight_min": -26.773523330688477,
      "activations/layer23_attention_weight_max": 36.655296325683594,
      "activations/layer23_attention_weight_min": -23.39020347595215,
      "activations/layer2_attention_weight_max": 29.231712341308594,
      "activations/layer2_attention_weight_min": -26.558685302734375,
      "activations/layer3_attention_weight_max": 84.76480102539062,
      "activations/layer3_attention_weight_min": -83.79153442382812,
      "activations/layer4_attention_weight_max": 84.42864227294922,
      "activations/layer4_attention_weight_min": -88.45674133300781,
      "activations/layer5_attention_weight_max": 62.70924377441406,
      "activations/layer5_attention_weight_min": -70.73336029052734,
      "activations/layer6_attention_weight_max": 47.413658142089844,
      "activations/layer6_attention_weight_min": -48.78156661987305,
      "activations/layer7_attention_weight_max": 58.25566864013672,
      "activations/layer7_attention_weight_min": -60.42768478393555,
      "activations/layer8_attention_weight_max": 44.117000579833984,
      "activations/layer8_attention_weight_min": -45.1938591003418,
      "activations/layer9_attention_weight_max": 42.986995697021484,
      "activations/layer9_attention_weight_min": -43.350830078125,
      "epoch": 17.99,
      "learning_rate": 3.43185606060606e-05,
      "loss": 2.7342,
      "step": 309650
    },
    {
      "activations/layer0_attention_weight_max": 15.389900207519531,
      "activations/layer0_attention_weight_min": -13.361048698425293,
      "activations/layer10_attention_weight_max": 38.43295669555664,
      "activations/layer10_attention_weight_min": -37.18751907348633,
      "activations/layer11_attention_weight_max": 35.694908142089844,
      "activations/layer11_attention_weight_min": -34.61314010620117,
      "activations/layer12_attention_weight_max": 20.75336265563965,
      "activations/layer12_attention_weight_min": -25.614572525024414,
      "activations/layer13_attention_weight_max": 35.905826568603516,
      "activations/layer13_attention_weight_min": -29.61613655090332,
      "activations/layer14_attention_weight_max": 38.024497985839844,
      "activations/layer14_attention_weight_min": -30.1663761138916,
      "activations/layer15_attention_weight_max": 34.506038665771484,
      "activations/layer15_attention_weight_min": -31.497167587280273,
      "activations/layer16_attention_weight_max": 38.36874771118164,
      "activations/layer16_attention_weight_min": -33.05974578857422,
      "activations/layer17_attention_weight_max": 53.8255615234375,
      "activations/layer17_attention_weight_min": -50.31632614135742,
      "activations/layer18_attention_weight_max": 51.62238693237305,
      "activations/layer18_attention_weight_min": -41.00458908081055,
      "activations/layer19_attention_weight_max": 23.73724937438965,
      "activations/layer19_attention_weight_min": -24.728857040405273,
      "activations/layer1_attention_weight_max": 16.18267250061035,
      "activations/layer1_attention_weight_min": -14.327659606933594,
      "activations/layer20_attention_weight_max": 23.670255661010742,
      "activations/layer20_attention_weight_min": -24.794462203979492,
      "activations/layer21_attention_weight_max": 37.03657913208008,
      "activations/layer21_attention_weight_min": -23.469039916992188,
      "activations/layer22_attention_weight_max": 34.568153381347656,
      "activations/layer22_attention_weight_min": -27.25661277770996,
      "activations/layer23_attention_weight_max": 38.836524963378906,
      "activations/layer23_attention_weight_min": -25.52016258239746,
      "activations/layer2_attention_weight_max": 28.893573760986328,
      "activations/layer2_attention_weight_min": -26.41344451904297,
      "activations/layer3_attention_weight_max": 84.83567810058594,
      "activations/layer3_attention_weight_min": -82.81364440917969,
      "activations/layer4_attention_weight_max": 83.63446807861328,
      "activations/layer4_attention_weight_min": -82.61532592773438,
      "activations/layer5_attention_weight_max": 62.22084045410156,
      "activations/layer5_attention_weight_min": -70.7815170288086,
      "activations/layer6_attention_weight_max": 47.8892936706543,
      "activations/layer6_attention_weight_min": -47.954105377197266,
      "activations/layer7_attention_weight_max": 60.26216125488281,
      "activations/layer7_attention_weight_min": -63.073570251464844,
      "activations/layer8_attention_weight_max": 47.82124328613281,
      "activations/layer8_attention_weight_min": -49.41084671020508,
      "activations/layer9_attention_weight_max": 45.248836517333984,
      "activations/layer9_attention_weight_min": -46.64195251464844,
      "epoch": 18.0,
      "learning_rate": 3.4299621212121204e-05,
      "loss": 2.7568,
      "step": 309700
    },
    {
      "activations/layer0_attention_weight_max": 15.926042556762695,
      "activations/layer0_attention_weight_min": -13.459932327270508,
      "activations/layer10_attention_weight_max": 34.47780990600586,
      "activations/layer10_attention_weight_min": -37.15146255493164,
      "activations/layer11_attention_weight_max": 32.175086975097656,
      "activations/layer11_attention_weight_min": -32.722042083740234,
      "activations/layer12_attention_weight_max": 19.030710220336914,
      "activations/layer12_attention_weight_min": -27.395328521728516,
      "activations/layer13_attention_weight_max": 32.508941650390625,
      "activations/layer13_attention_weight_min": -26.139339447021484,
      "activations/layer14_attention_weight_max": 34.439022064208984,
      "activations/layer14_attention_weight_min": -28.977689743041992,
      "activations/layer15_attention_weight_max": 30.38814353942871,
      "activations/layer15_attention_weight_min": -29.454097747802734,
      "activations/layer16_attention_weight_max": 32.318092346191406,
      "activations/layer16_attention_weight_min": -31.0347900390625,
      "activations/layer17_attention_weight_max": 47.219566345214844,
      "activations/layer17_attention_weight_min": -42.65023422241211,
      "activations/layer18_attention_weight_max": 42.29750442504883,
      "activations/layer18_attention_weight_min": -37.58483123779297,
      "activations/layer19_attention_weight_max": 21.756017684936523,
      "activations/layer19_attention_weight_min": -22.151880264282227,
      "activations/layer1_attention_weight_max": 15.10749626159668,
      "activations/layer1_attention_weight_min": -12.485468864440918,
      "activations/layer20_attention_weight_max": 19.47988510131836,
      "activations/layer20_attention_weight_min": -21.00305938720703,
      "activations/layer21_attention_weight_max": 34.52991485595703,
      "activations/layer21_attention_weight_min": -21.444948196411133,
      "activations/layer22_attention_weight_max": 26.387605667114258,
      "activations/layer22_attention_weight_min": -24.03000831604004,
      "activations/layer23_attention_weight_max": 33.65906524658203,
      "activations/layer23_attention_weight_min": -25.267927169799805,
      "activations/layer2_attention_weight_max": 29.502197265625,
      "activations/layer2_attention_weight_min": -28.17415428161621,
      "activations/layer3_attention_weight_max": 84.2241439819336,
      "activations/layer3_attention_weight_min": -87.47056579589844,
      "activations/layer4_attention_weight_max": 80.08165740966797,
      "activations/layer4_attention_weight_min": -83.35619354248047,
      "activations/layer5_attention_weight_max": 58.96271896362305,
      "activations/layer5_attention_weight_min": -69.33503723144531,
      "activations/layer6_attention_weight_max": 48.612632751464844,
      "activations/layer6_attention_weight_min": -49.03575134277344,
      "activations/layer7_attention_weight_max": 63.06889724731445,
      "activations/layer7_attention_weight_min": -59.99003219604492,
      "activations/layer8_attention_weight_max": 44.52788543701172,
      "activations/layer8_attention_weight_min": -51.343204498291016,
      "activations/layer9_attention_weight_max": 44.24616622924805,
      "activations/layer9_attention_weight_min": -47.19330978393555,
      "epoch": 18.0,
      "learning_rate": 3.428068181818181e-05,
      "loss": 2.7462,
      "step": 309750
    },
    {
      "activations/layer0_attention_weight_max": 15.393919944763184,
      "activations/layer0_attention_weight_min": -13.825943946838379,
      "activations/layer10_attention_weight_max": 32.81349563598633,
      "activations/layer10_attention_weight_min": -33.68167495727539,
      "activations/layer11_attention_weight_max": 31.75544548034668,
      "activations/layer11_attention_weight_min": -34.15286636352539,
      "activations/layer12_attention_weight_max": 19.136947631835938,
      "activations/layer12_attention_weight_min": -26.936328887939453,
      "activations/layer13_attention_weight_max": 41.38533020019531,
      "activations/layer13_attention_weight_min": -26.206113815307617,
      "activations/layer14_attention_weight_max": 39.05351638793945,
      "activations/layer14_attention_weight_min": -30.99913787841797,
      "activations/layer15_attention_weight_max": 33.95104217529297,
      "activations/layer15_attention_weight_min": -30.402742385864258,
      "activations/layer16_attention_weight_max": 37.628929138183594,
      "activations/layer16_attention_weight_min": -33.50021743774414,
      "activations/layer17_attention_weight_max": 57.59355926513672,
      "activations/layer17_attention_weight_min": -50.2337532043457,
      "activations/layer18_attention_weight_max": 51.78419494628906,
      "activations/layer18_attention_weight_min": -41.31279754638672,
      "activations/layer19_attention_weight_max": 25.824951171875,
      "activations/layer19_attention_weight_min": -24.372636795043945,
      "activations/layer1_attention_weight_max": 15.326072692871094,
      "activations/layer1_attention_weight_min": -13.300605773925781,
      "activations/layer20_attention_weight_max": 26.152559280395508,
      "activations/layer20_attention_weight_min": -22.32109832763672,
      "activations/layer21_attention_weight_max": 45.49757385253906,
      "activations/layer21_attention_weight_min": -22.59073829650879,
      "activations/layer22_attention_weight_max": 35.7981071472168,
      "activations/layer22_attention_weight_min": -24.77094841003418,
      "activations/layer23_attention_weight_max": 43.046241760253906,
      "activations/layer23_attention_weight_min": -23.51629638671875,
      "activations/layer2_attention_weight_max": 29.085927963256836,
      "activations/layer2_attention_weight_min": -25.874670028686523,
      "activations/layer3_attention_weight_max": 78.41397857666016,
      "activations/layer3_attention_weight_min": -78.59636688232422,
      "activations/layer4_attention_weight_max": 82.47610473632812,
      "activations/layer4_attention_weight_min": -84.21461486816406,
      "activations/layer5_attention_weight_max": 59.967830657958984,
      "activations/layer5_attention_weight_min": -73.94807434082031,
      "activations/layer6_attention_weight_max": 47.60603713989258,
      "activations/layer6_attention_weight_min": -47.81691360473633,
      "activations/layer7_attention_weight_max": 62.08901596069336,
      "activations/layer7_attention_weight_min": -60.55265808105469,
      "activations/layer8_attention_weight_max": 44.175533294677734,
      "activations/layer8_attention_weight_min": -47.50942611694336,
      "activations/layer9_attention_weight_max": 42.73683166503906,
      "activations/layer9_attention_weight_min": -43.7060661315918,
      "epoch": 18.0,
      "learning_rate": 3.426174242424242e-05,
      "loss": 2.7765,
      "step": 309800
    },
    {
      "activations/layer0_attention_weight_max": 14.961234092712402,
      "activations/layer0_attention_weight_min": -13.43266487121582,
      "activations/layer10_attention_weight_max": 38.381813049316406,
      "activations/layer10_attention_weight_min": -38.2665901184082,
      "activations/layer11_attention_weight_max": 36.30261993408203,
      "activations/layer11_attention_weight_min": -37.312278747558594,
      "activations/layer12_attention_weight_max": 18.65495491027832,
      "activations/layer12_attention_weight_min": -23.651880264282227,
      "activations/layer13_attention_weight_max": 31.54385757446289,
      "activations/layer13_attention_weight_min": -27.001811981201172,
      "activations/layer14_attention_weight_max": 33.3446159362793,
      "activations/layer14_attention_weight_min": -30.704835891723633,
      "activations/layer15_attention_weight_max": 30.140043258666992,
      "activations/layer15_attention_weight_min": -30.365400314331055,
      "activations/layer16_attention_weight_max": 34.57010269165039,
      "activations/layer16_attention_weight_min": -31.406917572021484,
      "activations/layer17_attention_weight_max": 52.05507278442383,
      "activations/layer17_attention_weight_min": -45.318382263183594,
      "activations/layer18_attention_weight_max": 45.05405044555664,
      "activations/layer18_attention_weight_min": -36.766143798828125,
      "activations/layer19_attention_weight_max": 26.106115341186523,
      "activations/layer19_attention_weight_min": -21.300119400024414,
      "activations/layer1_attention_weight_max": 16.875988006591797,
      "activations/layer1_attention_weight_min": -14.10141658782959,
      "activations/layer20_attention_weight_max": 21.180702209472656,
      "activations/layer20_attention_weight_min": -20.979591369628906,
      "activations/layer21_attention_weight_max": 38.6839485168457,
      "activations/layer21_attention_weight_min": -23.6318359375,
      "activations/layer22_attention_weight_max": 30.23529052734375,
      "activations/layer22_attention_weight_min": -25.43427848815918,
      "activations/layer23_attention_weight_max": 37.780975341796875,
      "activations/layer23_attention_weight_min": -23.732406616210938,
      "activations/layer2_attention_weight_max": 30.749858856201172,
      "activations/layer2_attention_weight_min": -27.576963424682617,
      "activations/layer3_attention_weight_max": 84.13298034667969,
      "activations/layer3_attention_weight_min": -85.87800598144531,
      "activations/layer4_attention_weight_max": 87.75605773925781,
      "activations/layer4_attention_weight_min": -89.20231628417969,
      "activations/layer5_attention_weight_max": 65.7906723022461,
      "activations/layer5_attention_weight_min": -81.40584564208984,
      "activations/layer6_attention_weight_max": 51.91188430786133,
      "activations/layer6_attention_weight_min": -59.760650634765625,
      "activations/layer7_attention_weight_max": 66.00039672851562,
      "activations/layer7_attention_weight_min": -71.54781341552734,
      "activations/layer8_attention_weight_max": 48.678428649902344,
      "activations/layer8_attention_weight_min": -53.92753982543945,
      "activations/layer9_attention_weight_max": 47.05563735961914,
      "activations/layer9_attention_weight_min": -49.54496383666992,
      "epoch": 18.0,
      "learning_rate": 3.424280303030303e-05,
      "loss": 2.7238,
      "step": 309850
    },
    {
      "activations/layer0_attention_weight_max": 15.042167663574219,
      "activations/layer0_attention_weight_min": -13.195369720458984,
      "activations/layer10_attention_weight_max": 35.28812789916992,
      "activations/layer10_attention_weight_min": -36.769317626953125,
      "activations/layer11_attention_weight_max": 32.339080810546875,
      "activations/layer11_attention_weight_min": -32.692543029785156,
      "activations/layer12_attention_weight_max": 18.80702781677246,
      "activations/layer12_attention_weight_min": -26.298189163208008,
      "activations/layer13_attention_weight_max": 52.50466537475586,
      "activations/layer13_attention_weight_min": -33.92225646972656,
      "activations/layer14_attention_weight_max": 36.99919509887695,
      "activations/layer14_attention_weight_min": -30.130823135375977,
      "activations/layer15_attention_weight_max": 33.14104461669922,
      "activations/layer15_attention_weight_min": -28.844703674316406,
      "activations/layer16_attention_weight_max": 32.360931396484375,
      "activations/layer16_attention_weight_min": -32.03329086303711,
      "activations/layer17_attention_weight_max": 50.65264892578125,
      "activations/layer17_attention_weight_min": -44.401737213134766,
      "activations/layer18_attention_weight_max": 43.86063003540039,
      "activations/layer18_attention_weight_min": -36.83784866333008,
      "activations/layer19_attention_weight_max": 25.137876510620117,
      "activations/layer19_attention_weight_min": -20.69734764099121,
      "activations/layer1_attention_weight_max": 15.892853736877441,
      "activations/layer1_attention_weight_min": -14.019048690795898,
      "activations/layer20_attention_weight_max": 20.935983657836914,
      "activations/layer20_attention_weight_min": -22.11760711669922,
      "activations/layer21_attention_weight_max": 32.91569137573242,
      "activations/layer21_attention_weight_min": -20.75716209411621,
      "activations/layer22_attention_weight_max": 28.670015335083008,
      "activations/layer22_attention_weight_min": -24.052934646606445,
      "activations/layer23_attention_weight_max": 36.16209030151367,
      "activations/layer23_attention_weight_min": -22.77985191345215,
      "activations/layer2_attention_weight_max": 29.891408920288086,
      "activations/layer2_attention_weight_min": -29.109798431396484,
      "activations/layer3_attention_weight_max": 82.65142059326172,
      "activations/layer3_attention_weight_min": -84.32244110107422,
      "activations/layer4_attention_weight_max": 85.29215240478516,
      "activations/layer4_attention_weight_min": -86.37177276611328,
      "activations/layer5_attention_weight_max": 61.35984420776367,
      "activations/layer5_attention_weight_min": -74.44017028808594,
      "activations/layer6_attention_weight_max": 49.68564987182617,
      "activations/layer6_attention_weight_min": -47.28737258911133,
      "activations/layer7_attention_weight_max": 64.29257202148438,
      "activations/layer7_attention_weight_min": -60.0803337097168,
      "activations/layer8_attention_weight_max": 44.27020263671875,
      "activations/layer8_attention_weight_min": -45.820377349853516,
      "activations/layer9_attention_weight_max": 45.585365295410156,
      "activations/layer9_attention_weight_min": -45.73337173461914,
      "epoch": 18.01,
      "learning_rate": 3.422386363636364e-05,
      "loss": 2.7076,
      "step": 309900
    },
    {
      "activations/layer0_attention_weight_max": 15.938760757446289,
      "activations/layer0_attention_weight_min": -13.471674919128418,
      "activations/layer10_attention_weight_max": 36.315494537353516,
      "activations/layer10_attention_weight_min": -36.137298583984375,
      "activations/layer11_attention_weight_max": 32.31085968017578,
      "activations/layer11_attention_weight_min": -33.78069305419922,
      "activations/layer12_attention_weight_max": 19.508041381835938,
      "activations/layer12_attention_weight_min": -24.606531143188477,
      "activations/layer13_attention_weight_max": 33.7684326171875,
      "activations/layer13_attention_weight_min": -28.860387802124023,
      "activations/layer14_attention_weight_max": 35.6612548828125,
      "activations/layer14_attention_weight_min": -30.797407150268555,
      "activations/layer15_attention_weight_max": 31.382843017578125,
      "activations/layer15_attention_weight_min": -29.86590576171875,
      "activations/layer16_attention_weight_max": 33.024742126464844,
      "activations/layer16_attention_weight_min": -30.024316787719727,
      "activations/layer17_attention_weight_max": 48.37820816040039,
      "activations/layer17_attention_weight_min": -43.770320892333984,
      "activations/layer18_attention_weight_max": 43.53108215332031,
      "activations/layer18_attention_weight_min": -40.81721496582031,
      "activations/layer19_attention_weight_max": 20.685523986816406,
      "activations/layer19_attention_weight_min": -22.512392044067383,
      "activations/layer1_attention_weight_max": 17.305986404418945,
      "activations/layer1_attention_weight_min": -13.623541831970215,
      "activations/layer20_attention_weight_max": 20.482641220092773,
      "activations/layer20_attention_weight_min": -23.017702102661133,
      "activations/layer21_attention_weight_max": 34.605377197265625,
      "activations/layer21_attention_weight_min": -25.235280990600586,
      "activations/layer22_attention_weight_max": 28.231910705566406,
      "activations/layer22_attention_weight_min": -28.01350975036621,
      "activations/layer23_attention_weight_max": 38.86133575439453,
      "activations/layer23_attention_weight_min": -24.74939727783203,
      "activations/layer2_attention_weight_max": 28.237625122070312,
      "activations/layer2_attention_weight_min": -28.401113510131836,
      "activations/layer3_attention_weight_max": 80.0792236328125,
      "activations/layer3_attention_weight_min": -87.32247161865234,
      "activations/layer4_attention_weight_max": 80.67225646972656,
      "activations/layer4_attention_weight_min": -85.08696746826172,
      "activations/layer5_attention_weight_max": 58.14048385620117,
      "activations/layer5_attention_weight_min": -74.23866271972656,
      "activations/layer6_attention_weight_max": 48.836421966552734,
      "activations/layer6_attention_weight_min": -50.20138931274414,
      "activations/layer7_attention_weight_max": 64.43657684326172,
      "activations/layer7_attention_weight_min": -61.21617126464844,
      "activations/layer8_attention_weight_max": 45.265220642089844,
      "activations/layer8_attention_weight_min": -47.75253677368164,
      "activations/layer9_attention_weight_max": 41.88533401489258,
      "activations/layer9_attention_weight_min": -47.212459564208984,
      "epoch": 18.01,
      "learning_rate": 3.420492424242424e-05,
      "loss": 2.7149,
      "step": 309950
    },
    {
      "activations/layer0_attention_weight_max": 16.582443237304688,
      "activations/layer0_attention_weight_min": -13.24312973022461,
      "activations/layer10_attention_weight_max": 38.90531539916992,
      "activations/layer10_attention_weight_min": -40.08790969848633,
      "activations/layer11_attention_weight_max": 36.948360443115234,
      "activations/layer11_attention_weight_min": -38.42094802856445,
      "activations/layer12_attention_weight_max": 20.038570404052734,
      "activations/layer12_attention_weight_min": -25.288206100463867,
      "activations/layer13_attention_weight_max": 33.99494934082031,
      "activations/layer13_attention_weight_min": -26.28030014038086,
      "activations/layer14_attention_weight_max": 39.75587844848633,
      "activations/layer14_attention_weight_min": -29.81824493408203,
      "activations/layer15_attention_weight_max": 36.355533599853516,
      "activations/layer15_attention_weight_min": -30.751609802246094,
      "activations/layer16_attention_weight_max": 37.19900894165039,
      "activations/layer16_attention_weight_min": -34.01823043823242,
      "activations/layer17_attention_weight_max": 57.74229049682617,
      "activations/layer17_attention_weight_min": -48.14677047729492,
      "activations/layer18_attention_weight_max": 48.75284957885742,
      "activations/layer18_attention_weight_min": -41.03606033325195,
      "activations/layer19_attention_weight_max": 23.484025955200195,
      "activations/layer19_attention_weight_min": -23.12646484375,
      "activations/layer1_attention_weight_max": 15.844486236572266,
      "activations/layer1_attention_weight_min": -13.955408096313477,
      "activations/layer20_attention_weight_max": 23.153860092163086,
      "activations/layer20_attention_weight_min": -20.643692016601562,
      "activations/layer21_attention_weight_max": 37.5083122253418,
      "activations/layer21_attention_weight_min": -23.38720703125,
      "activations/layer22_attention_weight_max": 28.26189422607422,
      "activations/layer22_attention_weight_min": -24.80940055847168,
      "activations/layer23_attention_weight_max": 36.47834014892578,
      "activations/layer23_attention_weight_min": -21.58039665222168,
      "activations/layer2_attention_weight_max": 29.875648498535156,
      "activations/layer2_attention_weight_min": -28.355024337768555,
      "activations/layer3_attention_weight_max": 83.69745635986328,
      "activations/layer3_attention_weight_min": -86.53565216064453,
      "activations/layer4_attention_weight_max": 85.70391845703125,
      "activations/layer4_attention_weight_min": -86.89391326904297,
      "activations/layer5_attention_weight_max": 62.59751510620117,
      "activations/layer5_attention_weight_min": -73.36388397216797,
      "activations/layer6_attention_weight_max": 48.0767936706543,
      "activations/layer6_attention_weight_min": -49.44351577758789,
      "activations/layer7_attention_weight_max": 66.75917053222656,
      "activations/layer7_attention_weight_min": -62.46346664428711,
      "activations/layer8_attention_weight_max": 48.6623649597168,
      "activations/layer8_attention_weight_min": -49.85155487060547,
      "activations/layer9_attention_weight_max": 46.194576263427734,
      "activations/layer9_attention_weight_min": -45.69926834106445,
      "epoch": 18.01,
      "learning_rate": 3.418598484848485e-05,
      "loss": 2.7207,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4934,
      "eval_samples_per_second": 505.571,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4934,
      "eval_openwebtext_samples_per_second": 505.571,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 2.0156,
      "eval_wikitext_samples_per_second": 226.234,
      "step": 310000
    },
    {
      "epoch": 18.01,
      "eval_lambada_loss": 2.46484375,
      "eval_lambada_ppl": 11.761644251483835,
      "eval_lambada_runtime": 9.5763,
      "eval_lambada_samples_per_second": 508.444,
      "step": 310000
    },
    {
      "activations/layer0_attention_weight_max": 15.560260772705078,
      "activations/layer0_attention_weight_min": -13.476180076599121,
      "activations/layer10_attention_weight_max": 40.673744201660156,
      "activations/layer10_attention_weight_min": -37.56000900268555,
      "activations/layer11_attention_weight_max": 38.14218521118164,
      "activations/layer11_attention_weight_min": -36.07683181762695,
      "activations/layer12_attention_weight_max": 19.11465072631836,
      "activations/layer12_attention_weight_min": -29.093421936035156,
      "activations/layer13_attention_weight_max": 32.084381103515625,
      "activations/layer13_attention_weight_min": -26.463542938232422,
      "activations/layer14_attention_weight_max": 34.805458068847656,
      "activations/layer14_attention_weight_min": -29.97585105895996,
      "activations/layer15_attention_weight_max": 33.64060974121094,
      "activations/layer15_attention_weight_min": -30.71762466430664,
      "activations/layer16_attention_weight_max": 36.18436813354492,
      "activations/layer16_attention_weight_min": -32.18411636352539,
      "activations/layer17_attention_weight_max": 56.4604377746582,
      "activations/layer17_attention_weight_min": -46.950809478759766,
      "activations/layer18_attention_weight_max": 50.524658203125,
      "activations/layer18_attention_weight_min": -43.45397186279297,
      "activations/layer19_attention_weight_max": 27.623794555664062,
      "activations/layer19_attention_weight_min": -27.053421020507812,
      "activations/layer1_attention_weight_max": 16.276578903198242,
      "activations/layer1_attention_weight_min": -14.234089851379395,
      "activations/layer20_attention_weight_max": 26.22477912902832,
      "activations/layer20_attention_weight_min": -23.232561111450195,
      "activations/layer21_attention_weight_max": 40.54736328125,
      "activations/layer21_attention_weight_min": -24.545757293701172,
      "activations/layer22_attention_weight_max": 31.014686584472656,
      "activations/layer22_attention_weight_min": -28.504928588867188,
      "activations/layer23_attention_weight_max": 40.41671371459961,
      "activations/layer23_attention_weight_min": -24.801137924194336,
      "activations/layer2_attention_weight_max": 32.20936584472656,
      "activations/layer2_attention_weight_min": -30.041606903076172,
      "activations/layer3_attention_weight_max": 97.06986999511719,
      "activations/layer3_attention_weight_min": -93.25343322753906,
      "activations/layer4_attention_weight_max": 93.93226623535156,
      "activations/layer4_attention_weight_min": -88.9676742553711,
      "activations/layer5_attention_weight_max": 64.66365051269531,
      "activations/layer5_attention_weight_min": -69.455810546875,
      "activations/layer6_attention_weight_max": 49.23493576049805,
      "activations/layer6_attention_weight_min": -50.5744743347168,
      "activations/layer7_attention_weight_max": 72.06461334228516,
      "activations/layer7_attention_weight_min": -69.43562316894531,
      "activations/layer8_attention_weight_max": 52.6851692199707,
      "activations/layer8_attention_weight_min": -50.10946273803711,
      "activations/layer9_attention_weight_max": 53.9266471862793,
      "activations/layer9_attention_weight_min": -49.645912170410156,
      "epoch": 18.02,
      "learning_rate": 3.416704545454545e-05,
      "loss": 2.7335,
      "step": 310050
    },
    {
      "activations/layer0_attention_weight_max": 15.670479774475098,
      "activations/layer0_attention_weight_min": -13.67846965789795,
      "activations/layer10_attention_weight_max": 36.07377243041992,
      "activations/layer10_attention_weight_min": -34.32798767089844,
      "activations/layer11_attention_weight_max": 33.49813461303711,
      "activations/layer11_attention_weight_min": -34.28376770019531,
      "activations/layer12_attention_weight_max": 20.789457321166992,
      "activations/layer12_attention_weight_min": -23.43047523498535,
      "activations/layer13_attention_weight_max": 36.034385681152344,
      "activations/layer13_attention_weight_min": -29.734203338623047,
      "activations/layer14_attention_weight_max": 37.62991714477539,
      "activations/layer14_attention_weight_min": -35.67534637451172,
      "activations/layer15_attention_weight_max": 37.454463958740234,
      "activations/layer15_attention_weight_min": -32.642601013183594,
      "activations/layer16_attention_weight_max": 39.789398193359375,
      "activations/layer16_attention_weight_min": -36.691627502441406,
      "activations/layer17_attention_weight_max": 57.03916549682617,
      "activations/layer17_attention_weight_min": -49.64712905883789,
      "activations/layer18_attention_weight_max": 51.48568344116211,
      "activations/layer18_attention_weight_min": -43.71959686279297,
      "activations/layer19_attention_weight_max": 25.867084503173828,
      "activations/layer19_attention_weight_min": -24.26921272277832,
      "activations/layer1_attention_weight_max": 16.3297119140625,
      "activations/layer1_attention_weight_min": -13.323023796081543,
      "activations/layer20_attention_weight_max": 22.869037628173828,
      "activations/layer20_attention_weight_min": -21.981212615966797,
      "activations/layer21_attention_weight_max": 38.14180374145508,
      "activations/layer21_attention_weight_min": -23.65399169921875,
      "activations/layer22_attention_weight_max": 29.54574966430664,
      "activations/layer22_attention_weight_min": -25.88918685913086,
      "activations/layer23_attention_weight_max": 33.91405487060547,
      "activations/layer23_attention_weight_min": -25.165220260620117,
      "activations/layer2_attention_weight_max": 30.95510482788086,
      "activations/layer2_attention_weight_min": -30.056869506835938,
      "activations/layer3_attention_weight_max": 82.8260726928711,
      "activations/layer3_attention_weight_min": -84.906494140625,
      "activations/layer4_attention_weight_max": 84.18070220947266,
      "activations/layer4_attention_weight_min": -83.15754699707031,
      "activations/layer5_attention_weight_max": 62.59808349609375,
      "activations/layer5_attention_weight_min": -75.61538696289062,
      "activations/layer6_attention_weight_max": 49.72456741333008,
      "activations/layer6_attention_weight_min": -50.535682678222656,
      "activations/layer7_attention_weight_max": 61.6187744140625,
      "activations/layer7_attention_weight_min": -62.14915084838867,
      "activations/layer8_attention_weight_max": 46.129852294921875,
      "activations/layer8_attention_weight_min": -48.87898635864258,
      "activations/layer9_attention_weight_max": 44.3640022277832,
      "activations/layer9_attention_weight_min": -47.5537223815918,
      "epoch": 18.02,
      "learning_rate": 3.414810606060606e-05,
      "loss": 2.73,
      "step": 310100
    },
    {
      "activations/layer0_attention_weight_max": 15.65621566772461,
      "activations/layer0_attention_weight_min": -13.58960247039795,
      "activations/layer10_attention_weight_max": 35.19671630859375,
      "activations/layer10_attention_weight_min": -38.02637481689453,
      "activations/layer11_attention_weight_max": 31.664396286010742,
      "activations/layer11_attention_weight_min": -35.29548263549805,
      "activations/layer12_attention_weight_max": 17.101964950561523,
      "activations/layer12_attention_weight_min": -27.461471557617188,
      "activations/layer13_attention_weight_max": 36.916229248046875,
      "activations/layer13_attention_weight_min": -27.15660858154297,
      "activations/layer14_attention_weight_max": 44.13638687133789,
      "activations/layer14_attention_weight_min": -32.47639846801758,
      "activations/layer15_attention_weight_max": 43.58298110961914,
      "activations/layer15_attention_weight_min": -32.17525863647461,
      "activations/layer16_attention_weight_max": 35.027252197265625,
      "activations/layer16_attention_weight_min": -31.733915328979492,
      "activations/layer17_attention_weight_max": 53.47380828857422,
      "activations/layer17_attention_weight_min": -50.45637893676758,
      "activations/layer18_attention_weight_max": 50.39414596557617,
      "activations/layer18_attention_weight_min": -47.213226318359375,
      "activations/layer19_attention_weight_max": 25.836755752563477,
      "activations/layer19_attention_weight_min": -24.13960838317871,
      "activations/layer1_attention_weight_max": 15.641365051269531,
      "activations/layer1_attention_weight_min": -13.385844230651855,
      "activations/layer20_attention_weight_max": 23.27887535095215,
      "activations/layer20_attention_weight_min": -22.569866180419922,
      "activations/layer21_attention_weight_max": 38.70122528076172,
      "activations/layer21_attention_weight_min": -26.379329681396484,
      "activations/layer22_attention_weight_max": 29.26255989074707,
      "activations/layer22_attention_weight_min": -25.349782943725586,
      "activations/layer23_attention_weight_max": 36.070640563964844,
      "activations/layer23_attention_weight_min": -25.221481323242188,
      "activations/layer2_attention_weight_max": 31.772838592529297,
      "activations/layer2_attention_weight_min": -31.33206558227539,
      "activations/layer3_attention_weight_max": 91.08721923828125,
      "activations/layer3_attention_weight_min": -90.779541015625,
      "activations/layer4_attention_weight_max": 86.82356262207031,
      "activations/layer4_attention_weight_min": -87.87580871582031,
      "activations/layer5_attention_weight_max": 62.94020080566406,
      "activations/layer5_attention_weight_min": -70.13135528564453,
      "activations/layer6_attention_weight_max": 47.90814971923828,
      "activations/layer6_attention_weight_min": -50.950782775878906,
      "activations/layer7_attention_weight_max": 69.10662841796875,
      "activations/layer7_attention_weight_min": -68.46698760986328,
      "activations/layer8_attention_weight_max": 47.22915267944336,
      "activations/layer8_attention_weight_min": -49.189353942871094,
      "activations/layer9_attention_weight_max": 52.47840118408203,
      "activations/layer9_attention_weight_min": -48.389095306396484,
      "epoch": 18.02,
      "learning_rate": 3.4129166666666665e-05,
      "loss": 2.7318,
      "step": 310150
    },
    {
      "activations/layer0_attention_weight_max": 15.528324127197266,
      "activations/layer0_attention_weight_min": -13.684514999389648,
      "activations/layer10_attention_weight_max": 32.99638748168945,
      "activations/layer10_attention_weight_min": -34.60233688354492,
      "activations/layer11_attention_weight_max": 31.335386276245117,
      "activations/layer11_attention_weight_min": -32.94767379760742,
      "activations/layer12_attention_weight_max": 17.671829223632812,
      "activations/layer12_attention_weight_min": -26.40128517150879,
      "activations/layer13_attention_weight_max": 29.22650718688965,
      "activations/layer13_attention_weight_min": -27.47478485107422,
      "activations/layer14_attention_weight_max": 33.15080642700195,
      "activations/layer14_attention_weight_min": -29.132522583007812,
      "activations/layer15_attention_weight_max": 31.37872314453125,
      "activations/layer15_attention_weight_min": -30.297704696655273,
      "activations/layer16_attention_weight_max": 31.33205223083496,
      "activations/layer16_attention_weight_min": -31.256975173950195,
      "activations/layer17_attention_weight_max": 51.25309753417969,
      "activations/layer17_attention_weight_min": -47.05386734008789,
      "activations/layer18_attention_weight_max": 42.46489334106445,
      "activations/layer18_attention_weight_min": -39.59868240356445,
      "activations/layer19_attention_weight_max": 23.38556480407715,
      "activations/layer19_attention_weight_min": -21.258493423461914,
      "activations/layer1_attention_weight_max": 15.456059455871582,
      "activations/layer1_attention_weight_min": -13.128203392028809,
      "activations/layer20_attention_weight_max": 21.2247257232666,
      "activations/layer20_attention_weight_min": -23.735759735107422,
      "activations/layer21_attention_weight_max": 35.300270080566406,
      "activations/layer21_attention_weight_min": -22.267324447631836,
      "activations/layer22_attention_weight_max": 27.513044357299805,
      "activations/layer22_attention_weight_min": -25.368711471557617,
      "activations/layer23_attention_weight_max": 32.75929260253906,
      "activations/layer23_attention_weight_min": -28.530208587646484,
      "activations/layer2_attention_weight_max": 31.397207260131836,
      "activations/layer2_attention_weight_min": -28.54971694946289,
      "activations/layer3_attention_weight_max": 82.66244506835938,
      "activations/layer3_attention_weight_min": -84.90290069580078,
      "activations/layer4_attention_weight_max": 84.43123626708984,
      "activations/layer4_attention_weight_min": -86.30683135986328,
      "activations/layer5_attention_weight_max": 60.899227142333984,
      "activations/layer5_attention_weight_min": -69.99246215820312,
      "activations/layer6_attention_weight_max": 48.486934661865234,
      "activations/layer6_attention_weight_min": -50.30170822143555,
      "activations/layer7_attention_weight_max": 60.014244079589844,
      "activations/layer7_attention_weight_min": -63.99606704711914,
      "activations/layer8_attention_weight_max": 44.190826416015625,
      "activations/layer8_attention_weight_min": -48.74589538574219,
      "activations/layer9_attention_weight_max": 42.49078369140625,
      "activations/layer9_attention_weight_min": -43.163761138916016,
      "epoch": 18.02,
      "learning_rate": 3.411022727272727e-05,
      "loss": 2.7232,
      "step": 310200
    },
    {
      "activations/layer0_attention_weight_max": 15.283531188964844,
      "activations/layer0_attention_weight_min": -13.387374877929688,
      "activations/layer10_attention_weight_max": 38.19150924682617,
      "activations/layer10_attention_weight_min": -37.414920806884766,
      "activations/layer11_attention_weight_max": 32.410362243652344,
      "activations/layer11_attention_weight_min": -34.11030578613281,
      "activations/layer12_attention_weight_max": 18.695068359375,
      "activations/layer12_attention_weight_min": -25.00111198425293,
      "activations/layer13_attention_weight_max": 34.89314270019531,
      "activations/layer13_attention_weight_min": -25.666227340698242,
      "activations/layer14_attention_weight_max": 33.331138610839844,
      "activations/layer14_attention_weight_min": -28.86421775817871,
      "activations/layer15_attention_weight_max": 32.429344177246094,
      "activations/layer15_attention_weight_min": -28.917741775512695,
      "activations/layer16_attention_weight_max": 31.98323631286621,
      "activations/layer16_attention_weight_min": -29.76534080505371,
      "activations/layer17_attention_weight_max": 48.98881912231445,
      "activations/layer17_attention_weight_min": -44.66473388671875,
      "activations/layer18_attention_weight_max": 44.49308395385742,
      "activations/layer18_attention_weight_min": -39.5830078125,
      "activations/layer19_attention_weight_max": 21.854679107666016,
      "activations/layer19_attention_weight_min": -22.2476749420166,
      "activations/layer1_attention_weight_max": 15.308616638183594,
      "activations/layer1_attention_weight_min": -12.394210815429688,
      "activations/layer20_attention_weight_max": 21.52519989013672,
      "activations/layer20_attention_weight_min": -19.569250106811523,
      "activations/layer21_attention_weight_max": 34.019554138183594,
      "activations/layer21_attention_weight_min": -20.8859806060791,
      "activations/layer22_attention_weight_max": 27.580169677734375,
      "activations/layer22_attention_weight_min": -23.79230499267578,
      "activations/layer23_attention_weight_max": 34.77656173706055,
      "activations/layer23_attention_weight_min": -22.53473663330078,
      "activations/layer2_attention_weight_max": 30.103801727294922,
      "activations/layer2_attention_weight_min": -27.719463348388672,
      "activations/layer3_attention_weight_max": 84.0009765625,
      "activations/layer3_attention_weight_min": -81.02318572998047,
      "activations/layer4_attention_weight_max": 81.92931365966797,
      "activations/layer4_attention_weight_min": -87.0599594116211,
      "activations/layer5_attention_weight_max": 62.69676208496094,
      "activations/layer5_attention_weight_min": -76.23015594482422,
      "activations/layer6_attention_weight_max": 49.737247467041016,
      "activations/layer6_attention_weight_min": -51.04627227783203,
      "activations/layer7_attention_weight_max": 60.563175201416016,
      "activations/layer7_attention_weight_min": -63.50031661987305,
      "activations/layer8_attention_weight_max": 46.187232971191406,
      "activations/layer8_attention_weight_min": -48.85317611694336,
      "activations/layer9_attention_weight_max": 44.9434700012207,
      "activations/layer9_attention_weight_min": -46.78575897216797,
      "epoch": 18.03,
      "learning_rate": 3.4091287878787875e-05,
      "loss": 2.7372,
      "step": 310250
    },
    {
      "activations/layer0_attention_weight_max": 15.86001205444336,
      "activations/layer0_attention_weight_min": -13.682713508605957,
      "activations/layer10_attention_weight_max": 35.84891891479492,
      "activations/layer10_attention_weight_min": -36.004756927490234,
      "activations/layer11_attention_weight_max": 32.842899322509766,
      "activations/layer11_attention_weight_min": -34.45491027832031,
      "activations/layer12_attention_weight_max": 20.228939056396484,
      "activations/layer12_attention_weight_min": -24.335779190063477,
      "activations/layer13_attention_weight_max": 38.94358825683594,
      "activations/layer13_attention_weight_min": -27.640094757080078,
      "activations/layer14_attention_weight_max": 34.69065856933594,
      "activations/layer14_attention_weight_min": -31.904605865478516,
      "activations/layer15_attention_weight_max": 35.2647705078125,
      "activations/layer15_attention_weight_min": -30.39690399169922,
      "activations/layer16_attention_weight_max": 34.403560638427734,
      "activations/layer16_attention_weight_min": -31.332874298095703,
      "activations/layer17_attention_weight_max": 50.35634994506836,
      "activations/layer17_attention_weight_min": -43.996910095214844,
      "activations/layer18_attention_weight_max": 46.8575553894043,
      "activations/layer18_attention_weight_min": -38.92755889892578,
      "activations/layer19_attention_weight_max": 23.276206970214844,
      "activations/layer19_attention_weight_min": -22.79248809814453,
      "activations/layer1_attention_weight_max": 15.564753532409668,
      "activations/layer1_attention_weight_min": -14.636260032653809,
      "activations/layer20_attention_weight_max": 21.85531997680664,
      "activations/layer20_attention_weight_min": -22.585765838623047,
      "activations/layer21_attention_weight_max": 34.747581481933594,
      "activations/layer21_attention_weight_min": -23.28652000427246,
      "activations/layer22_attention_weight_max": 25.974987030029297,
      "activations/layer22_attention_weight_min": -24.475223541259766,
      "activations/layer23_attention_weight_max": 37.90016174316406,
      "activations/layer23_attention_weight_min": -27.44333267211914,
      "activations/layer2_attention_weight_max": 30.744182586669922,
      "activations/layer2_attention_weight_min": -29.095314025878906,
      "activations/layer3_attention_weight_max": 85.4350814819336,
      "activations/layer3_attention_weight_min": -88.696044921875,
      "activations/layer4_attention_weight_max": 86.17342376708984,
      "activations/layer4_attention_weight_min": -90.53449249267578,
      "activations/layer5_attention_weight_max": 61.799591064453125,
      "activations/layer5_attention_weight_min": -72.87850189208984,
      "activations/layer6_attention_weight_max": 51.62266159057617,
      "activations/layer6_attention_weight_min": -50.33073425292969,
      "activations/layer7_attention_weight_max": 60.50654602050781,
      "activations/layer7_attention_weight_min": -63.81361770629883,
      "activations/layer8_attention_weight_max": 45.31757736206055,
      "activations/layer8_attention_weight_min": -49.79776382446289,
      "activations/layer9_attention_weight_max": 42.92925262451172,
      "activations/layer9_attention_weight_min": -47.620460510253906,
      "epoch": 18.03,
      "learning_rate": 3.407234848484848e-05,
      "loss": 2.7327,
      "step": 310300
    },
    {
      "activations/layer0_attention_weight_max": 15.066205978393555,
      "activations/layer0_attention_weight_min": -13.534255027770996,
      "activations/layer10_attention_weight_max": 37.89892578125,
      "activations/layer10_attention_weight_min": -37.07813262939453,
      "activations/layer11_attention_weight_max": 34.59461975097656,
      "activations/layer11_attention_weight_min": -35.559814453125,
      "activations/layer12_attention_weight_max": 20.856534957885742,
      "activations/layer12_attention_weight_min": -24.984615325927734,
      "activations/layer13_attention_weight_max": 36.82283020019531,
      "activations/layer13_attention_weight_min": -27.456279754638672,
      "activations/layer14_attention_weight_max": 37.94975280761719,
      "activations/layer14_attention_weight_min": -30.761693954467773,
      "activations/layer15_attention_weight_max": 35.64360046386719,
      "activations/layer15_attention_weight_min": -30.57127571105957,
      "activations/layer16_attention_weight_max": 36.79551696777344,
      "activations/layer16_attention_weight_min": -32.73660659790039,
      "activations/layer17_attention_weight_max": 55.85962677001953,
      "activations/layer17_attention_weight_min": -45.34775161743164,
      "activations/layer18_attention_weight_max": 50.00758361816406,
      "activations/layer18_attention_weight_min": -40.11296463012695,
      "activations/layer19_attention_weight_max": 24.300748825073242,
      "activations/layer19_attention_weight_min": -22.353546142578125,
      "activations/layer1_attention_weight_max": 17.082937240600586,
      "activations/layer1_attention_weight_min": -13.908716201782227,
      "activations/layer20_attention_weight_max": 23.95071029663086,
      "activations/layer20_attention_weight_min": -22.302404403686523,
      "activations/layer21_attention_weight_max": 32.054405212402344,
      "activations/layer21_attention_weight_min": -22.628143310546875,
      "activations/layer22_attention_weight_max": 31.94069480895996,
      "activations/layer22_attention_weight_min": -25.54274559020996,
      "activations/layer23_attention_weight_max": 35.93686294555664,
      "activations/layer23_attention_weight_min": -25.020856857299805,
      "activations/layer2_attention_weight_max": 28.536466598510742,
      "activations/layer2_attention_weight_min": -28.15370750427246,
      "activations/layer3_attention_weight_max": 81.13677978515625,
      "activations/layer3_attention_weight_min": -83.79955291748047,
      "activations/layer4_attention_weight_max": 82.52848815917969,
      "activations/layer4_attention_weight_min": -86.13418579101562,
      "activations/layer5_attention_weight_max": 67.38301086425781,
      "activations/layer5_attention_weight_min": -77.21253967285156,
      "activations/layer6_attention_weight_max": 50.577938079833984,
      "activations/layer6_attention_weight_min": -52.20594024658203,
      "activations/layer7_attention_weight_max": 64.3928451538086,
      "activations/layer7_attention_weight_min": -62.49404525756836,
      "activations/layer8_attention_weight_max": 48.7635383605957,
      "activations/layer8_attention_weight_min": -49.959632873535156,
      "activations/layer9_attention_weight_max": 48.243804931640625,
      "activations/layer9_attention_weight_min": -48.05498504638672,
      "epoch": 18.03,
      "learning_rate": 3.4053409090909085e-05,
      "loss": 2.7259,
      "step": 310350
    },
    {
      "activations/layer0_attention_weight_max": 16.60613441467285,
      "activations/layer0_attention_weight_min": -13.519818305969238,
      "activations/layer10_attention_weight_max": 36.46826934814453,
      "activations/layer10_attention_weight_min": -35.33820343017578,
      "activations/layer11_attention_weight_max": 33.68282699584961,
      "activations/layer11_attention_weight_min": -33.0684814453125,
      "activations/layer12_attention_weight_max": 19.191791534423828,
      "activations/layer12_attention_weight_min": -26.63866424560547,
      "activations/layer13_attention_weight_max": 34.95045852661133,
      "activations/layer13_attention_weight_min": -27.93874168395996,
      "activations/layer14_attention_weight_max": 36.79259490966797,
      "activations/layer14_attention_weight_min": -30.98326301574707,
      "activations/layer15_attention_weight_max": 36.60610580444336,
      "activations/layer15_attention_weight_min": -31.447616577148438,
      "activations/layer16_attention_weight_max": 34.45494842529297,
      "activations/layer16_attention_weight_min": -31.450122833251953,
      "activations/layer17_attention_weight_max": 53.8182258605957,
      "activations/layer17_attention_weight_min": -44.2593994140625,
      "activations/layer18_attention_weight_max": 47.1495475769043,
      "activations/layer18_attention_weight_min": -40.01973342895508,
      "activations/layer19_attention_weight_max": 24.307710647583008,
      "activations/layer19_attention_weight_min": -21.565292358398438,
      "activations/layer1_attention_weight_max": 15.969005584716797,
      "activations/layer1_attention_weight_min": -14.401601791381836,
      "activations/layer20_attention_weight_max": 20.487972259521484,
      "activations/layer20_attention_weight_min": -22.780166625976562,
      "activations/layer21_attention_weight_max": 36.58003234863281,
      "activations/layer21_attention_weight_min": -22.81560707092285,
      "activations/layer22_attention_weight_max": 30.718076705932617,
      "activations/layer22_attention_weight_min": -25.99684715270996,
      "activations/layer23_attention_weight_max": 34.18089294433594,
      "activations/layer23_attention_weight_min": -24.92531967163086,
      "activations/layer2_attention_weight_max": 31.274372100830078,
      "activations/layer2_attention_weight_min": -28.368865966796875,
      "activations/layer3_attention_weight_max": 83.56420135498047,
      "activations/layer3_attention_weight_min": -81.9688491821289,
      "activations/layer4_attention_weight_max": 84.84307861328125,
      "activations/layer4_attention_weight_min": -81.65196228027344,
      "activations/layer5_attention_weight_max": 63.71635437011719,
      "activations/layer5_attention_weight_min": -76.9513168334961,
      "activations/layer6_attention_weight_max": 50.24711990356445,
      "activations/layer6_attention_weight_min": -53.04579162597656,
      "activations/layer7_attention_weight_max": 63.26973342895508,
      "activations/layer7_attention_weight_min": -66.78539276123047,
      "activations/layer8_attention_weight_max": 46.87313461303711,
      "activations/layer8_attention_weight_min": -50.73403549194336,
      "activations/layer9_attention_weight_max": 46.40626907348633,
      "activations/layer9_attention_weight_min": -47.07607650756836,
      "epoch": 18.04,
      "learning_rate": 3.403446969696969e-05,
      "loss": 2.7209,
      "step": 310400
    },
    {
      "activations/layer0_attention_weight_max": 14.990888595581055,
      "activations/layer0_attention_weight_min": -13.86575698852539,
      "activations/layer10_attention_weight_max": 38.3196907043457,
      "activations/layer10_attention_weight_min": -35.994075775146484,
      "activations/layer11_attention_weight_max": 34.401344299316406,
      "activations/layer11_attention_weight_min": -34.41200637817383,
      "activations/layer12_attention_weight_max": 19.063350677490234,
      "activations/layer12_attention_weight_min": -26.757591247558594,
      "activations/layer13_attention_weight_max": 37.6458854675293,
      "activations/layer13_attention_weight_min": -26.538951873779297,
      "activations/layer14_attention_weight_max": 35.341705322265625,
      "activations/layer14_attention_weight_min": -28.65182113647461,
      "activations/layer15_attention_weight_max": 33.17026138305664,
      "activations/layer15_attention_weight_min": -30.42841339111328,
      "activations/layer16_attention_weight_max": 34.990474700927734,
      "activations/layer16_attention_weight_min": -32.44293212890625,
      "activations/layer17_attention_weight_max": 51.39143371582031,
      "activations/layer17_attention_weight_min": -44.439876556396484,
      "activations/layer18_attention_weight_max": 48.857566833496094,
      "activations/layer18_attention_weight_min": -40.28299331665039,
      "activations/layer19_attention_weight_max": 24.68505859375,
      "activations/layer19_attention_weight_min": -22.21596336364746,
      "activations/layer1_attention_weight_max": 16.57609748840332,
      "activations/layer1_attention_weight_min": -15.369763374328613,
      "activations/layer20_attention_weight_max": 22.685054779052734,
      "activations/layer20_attention_weight_min": -20.71685028076172,
      "activations/layer21_attention_weight_max": 35.20083999633789,
      "activations/layer21_attention_weight_min": -23.6577091217041,
      "activations/layer22_attention_weight_max": 30.11380386352539,
      "activations/layer22_attention_weight_min": -24.85126304626465,
      "activations/layer23_attention_weight_max": 38.173118591308594,
      "activations/layer23_attention_weight_min": -25.523788452148438,
      "activations/layer2_attention_weight_max": 32.60613250732422,
      "activations/layer2_attention_weight_min": -32.146461486816406,
      "activations/layer3_attention_weight_max": 87.51882934570312,
      "activations/layer3_attention_weight_min": -90.14860534667969,
      "activations/layer4_attention_weight_max": 89.90116119384766,
      "activations/layer4_attention_weight_min": -85.58705139160156,
      "activations/layer5_attention_weight_max": 66.9461441040039,
      "activations/layer5_attention_weight_min": -77.14913940429688,
      "activations/layer6_attention_weight_max": 54.173484802246094,
      "activations/layer6_attention_weight_min": -52.461524963378906,
      "activations/layer7_attention_weight_max": 66.22510528564453,
      "activations/layer7_attention_weight_min": -63.07451629638672,
      "activations/layer8_attention_weight_max": 47.29018020629883,
      "activations/layer8_attention_weight_min": -51.44145202636719,
      "activations/layer9_attention_weight_max": 46.07842254638672,
      "activations/layer9_attention_weight_min": -48.14220428466797,
      "epoch": 18.04,
      "learning_rate": 3.40155303030303e-05,
      "loss": 2.7277,
      "step": 310450
    },
    {
      "activations/layer0_attention_weight_max": 15.336328506469727,
      "activations/layer0_attention_weight_min": -13.825150489807129,
      "activations/layer10_attention_weight_max": 35.844085693359375,
      "activations/layer10_attention_weight_min": -36.8046875,
      "activations/layer11_attention_weight_max": 34.37527847290039,
      "activations/layer11_attention_weight_min": -33.223880767822266,
      "activations/layer12_attention_weight_max": 18.89203453063965,
      "activations/layer12_attention_weight_min": -27.14370346069336,
      "activations/layer13_attention_weight_max": 38.43549728393555,
      "activations/layer13_attention_weight_min": -27.439502716064453,
      "activations/layer14_attention_weight_max": 37.99474334716797,
      "activations/layer14_attention_weight_min": -29.29766845703125,
      "activations/layer15_attention_weight_max": 37.040306091308594,
      "activations/layer15_attention_weight_min": -29.245864868164062,
      "activations/layer16_attention_weight_max": 34.200008392333984,
      "activations/layer16_attention_weight_min": -30.06829071044922,
      "activations/layer17_attention_weight_max": 52.18608474731445,
      "activations/layer17_attention_weight_min": -46.84653854370117,
      "activations/layer18_attention_weight_max": 47.92063522338867,
      "activations/layer18_attention_weight_min": -44.47562026977539,
      "activations/layer19_attention_weight_max": 26.412363052368164,
      "activations/layer19_attention_weight_min": -23.699216842651367,
      "activations/layer1_attention_weight_max": 15.458853721618652,
      "activations/layer1_attention_weight_min": -14.401456832885742,
      "activations/layer20_attention_weight_max": 25.10664939880371,
      "activations/layer20_attention_weight_min": -24.303747177124023,
      "activations/layer21_attention_weight_max": 44.97602462768555,
      "activations/layer21_attention_weight_min": -28.945207595825195,
      "activations/layer22_attention_weight_max": 28.168251037597656,
      "activations/layer22_attention_weight_min": -26.08519172668457,
      "activations/layer23_attention_weight_max": 37.483154296875,
      "activations/layer23_attention_weight_min": -27.829708099365234,
      "activations/layer2_attention_weight_max": 32.79378890991211,
      "activations/layer2_attention_weight_min": -30.959028244018555,
      "activations/layer3_attention_weight_max": 87.05289459228516,
      "activations/layer3_attention_weight_min": -91.04270935058594,
      "activations/layer4_attention_weight_max": 88.05040740966797,
      "activations/layer4_attention_weight_min": -87.53951263427734,
      "activations/layer5_attention_weight_max": 61.38542175292969,
      "activations/layer5_attention_weight_min": -72.90586853027344,
      "activations/layer6_attention_weight_max": 50.9394645690918,
      "activations/layer6_attention_weight_min": -49.733909606933594,
      "activations/layer7_attention_weight_max": 65.67648315429688,
      "activations/layer7_attention_weight_min": -63.12210464477539,
      "activations/layer8_attention_weight_max": 47.4048957824707,
      "activations/layer8_attention_weight_min": -48.21427917480469,
      "activations/layer9_attention_weight_max": 49.846412658691406,
      "activations/layer9_attention_weight_min": -44.83832931518555,
      "epoch": 18.04,
      "learning_rate": 3.399659090909091e-05,
      "loss": 2.731,
      "step": 310500
    },
    {
      "activations/layer0_attention_weight_max": 15.468838691711426,
      "activations/layer0_attention_weight_min": -13.50816535949707,
      "activations/layer10_attention_weight_max": 35.64344024658203,
      "activations/layer10_attention_weight_min": -35.59358596801758,
      "activations/layer11_attention_weight_max": 31.7542724609375,
      "activations/layer11_attention_weight_min": -34.948482513427734,
      "activations/layer12_attention_weight_max": 20.134445190429688,
      "activations/layer12_attention_weight_min": -25.138303756713867,
      "activations/layer13_attention_weight_max": 37.32054901123047,
      "activations/layer13_attention_weight_min": -29.788162231445312,
      "activations/layer14_attention_weight_max": 39.15018081665039,
      "activations/layer14_attention_weight_min": -33.85552978515625,
      "activations/layer15_attention_weight_max": 35.240943908691406,
      "activations/layer15_attention_weight_min": -29.628076553344727,
      "activations/layer16_attention_weight_max": 35.8549919128418,
      "activations/layer16_attention_weight_min": -32.17584228515625,
      "activations/layer17_attention_weight_max": 56.11256790161133,
      "activations/layer17_attention_weight_min": -46.64033889770508,
      "activations/layer18_attention_weight_max": 51.979339599609375,
      "activations/layer18_attention_weight_min": -40.91359329223633,
      "activations/layer19_attention_weight_max": 27.986074447631836,
      "activations/layer19_attention_weight_min": -24.88582420349121,
      "activations/layer1_attention_weight_max": 15.912982940673828,
      "activations/layer1_attention_weight_min": -13.889062881469727,
      "activations/layer20_attention_weight_max": 30.718217849731445,
      "activations/layer20_attention_weight_min": -21.404048919677734,
      "activations/layer21_attention_weight_max": 41.50788116455078,
      "activations/layer21_attention_weight_min": -21.93004608154297,
      "activations/layer22_attention_weight_max": 31.86368751525879,
      "activations/layer22_attention_weight_min": -27.3935546875,
      "activations/layer23_attention_weight_max": 37.9891242980957,
      "activations/layer23_attention_weight_min": -23.819843292236328,
      "activations/layer2_attention_weight_max": 30.115943908691406,
      "activations/layer2_attention_weight_min": -29.470930099487305,
      "activations/layer3_attention_weight_max": 85.52046966552734,
      "activations/layer3_attention_weight_min": -88.84662628173828,
      "activations/layer4_attention_weight_max": 86.8381576538086,
      "activations/layer4_attention_weight_min": -87.23194122314453,
      "activations/layer5_attention_weight_max": 61.88854217529297,
      "activations/layer5_attention_weight_min": -74.25816345214844,
      "activations/layer6_attention_weight_max": 50.7503776550293,
      "activations/layer6_attention_weight_min": -50.947208404541016,
      "activations/layer7_attention_weight_max": 64.06983947753906,
      "activations/layer7_attention_weight_min": -62.56249237060547,
      "activations/layer8_attention_weight_max": 47.79911422729492,
      "activations/layer8_attention_weight_min": -49.956661224365234,
      "activations/layer9_attention_weight_max": 43.4784049987793,
      "activations/layer9_attention_weight_min": -48.32296371459961,
      "epoch": 18.04,
      "learning_rate": 3.397765151515151e-05,
      "loss": 2.7156,
      "step": 310550
    },
    {
      "activations/layer0_attention_weight_max": 15.32215690612793,
      "activations/layer0_attention_weight_min": -13.498583793640137,
      "activations/layer10_attention_weight_max": 34.40522003173828,
      "activations/layer10_attention_weight_min": -36.749359130859375,
      "activations/layer11_attention_weight_max": 32.97666549682617,
      "activations/layer11_attention_weight_min": -33.78638458251953,
      "activations/layer12_attention_weight_max": 18.876264572143555,
      "activations/layer12_attention_weight_min": -26.356096267700195,
      "activations/layer13_attention_weight_max": 33.51969528198242,
      "activations/layer13_attention_weight_min": -28.12969398498535,
      "activations/layer14_attention_weight_max": 34.424476623535156,
      "activations/layer14_attention_weight_min": -28.798343658447266,
      "activations/layer15_attention_weight_max": 32.279232025146484,
      "activations/layer15_attention_weight_min": -29.91692352294922,
      "activations/layer16_attention_weight_max": 32.13360595703125,
      "activations/layer16_attention_weight_min": -30.785184860229492,
      "activations/layer17_attention_weight_max": 49.41629409790039,
      "activations/layer17_attention_weight_min": -42.754356384277344,
      "activations/layer18_attention_weight_max": 42.71673583984375,
      "activations/layer18_attention_weight_min": -38.76449203491211,
      "activations/layer19_attention_weight_max": 21.888038635253906,
      "activations/layer19_attention_weight_min": -22.183645248413086,
      "activations/layer1_attention_weight_max": 18.531787872314453,
      "activations/layer1_attention_weight_min": -14.937501907348633,
      "activations/layer20_attention_weight_max": 20.348705291748047,
      "activations/layer20_attention_weight_min": -23.293720245361328,
      "activations/layer21_attention_weight_max": 33.41594696044922,
      "activations/layer21_attention_weight_min": -23.169910430908203,
      "activations/layer22_attention_weight_max": 27.14473533630371,
      "activations/layer22_attention_weight_min": -26.79851531982422,
      "activations/layer23_attention_weight_max": 33.767704010009766,
      "activations/layer23_attention_weight_min": -23.817716598510742,
      "activations/layer2_attention_weight_max": 33.702728271484375,
      "activations/layer2_attention_weight_min": -32.106353759765625,
      "activations/layer3_attention_weight_max": 95.77408599853516,
      "activations/layer3_attention_weight_min": -93.22950744628906,
      "activations/layer4_attention_weight_max": 89.9995346069336,
      "activations/layer4_attention_weight_min": -87.75592041015625,
      "activations/layer5_attention_weight_max": 61.46888732910156,
      "activations/layer5_attention_weight_min": -70.97966003417969,
      "activations/layer6_attention_weight_max": 50.41061782836914,
      "activations/layer6_attention_weight_min": -50.06338119506836,
      "activations/layer7_attention_weight_max": 67.51112365722656,
      "activations/layer7_attention_weight_min": -61.44723129272461,
      "activations/layer8_attention_weight_max": 47.24831008911133,
      "activations/layer8_attention_weight_min": -50.17311096191406,
      "activations/layer9_attention_weight_max": 44.268409729003906,
      "activations/layer9_attention_weight_min": -46.169437408447266,
      "epoch": 18.05,
      "learning_rate": 3.395909090909091e-05,
      "loss": 2.7319,
      "step": 310600
    },
    {
      "activations/layer0_attention_weight_max": 14.741986274719238,
      "activations/layer0_attention_weight_min": -13.500530242919922,
      "activations/layer10_attention_weight_max": 36.20756912231445,
      "activations/layer10_attention_weight_min": -35.38876724243164,
      "activations/layer11_attention_weight_max": 33.02021789550781,
      "activations/layer11_attention_weight_min": -34.179927825927734,
      "activations/layer12_attention_weight_max": 18.18997573852539,
      "activations/layer12_attention_weight_min": -23.751800537109375,
      "activations/layer13_attention_weight_max": 32.06198501586914,
      "activations/layer13_attention_weight_min": -33.93907928466797,
      "activations/layer14_attention_weight_max": 36.24198913574219,
      "activations/layer14_attention_weight_min": -32.42518615722656,
      "activations/layer15_attention_weight_max": 33.28227996826172,
      "activations/layer15_attention_weight_min": -32.36390686035156,
      "activations/layer16_attention_weight_max": 32.835147857666016,
      "activations/layer16_attention_weight_min": -32.34734344482422,
      "activations/layer17_attention_weight_max": 51.26201248168945,
      "activations/layer17_attention_weight_min": -48.03837203979492,
      "activations/layer18_attention_weight_max": 46.044456481933594,
      "activations/layer18_attention_weight_min": -39.581886291503906,
      "activations/layer19_attention_weight_max": 25.01951789855957,
      "activations/layer19_attention_weight_min": -23.684614181518555,
      "activations/layer1_attention_weight_max": 16.921438217163086,
      "activations/layer1_attention_weight_min": -14.104741096496582,
      "activations/layer20_attention_weight_max": 23.384998321533203,
      "activations/layer20_attention_weight_min": -21.838199615478516,
      "activations/layer21_attention_weight_max": 34.7945442199707,
      "activations/layer21_attention_weight_min": -23.859947204589844,
      "activations/layer22_attention_weight_max": 28.426212310791016,
      "activations/layer22_attention_weight_min": -27.135042190551758,
      "activations/layer23_attention_weight_max": 34.80433654785156,
      "activations/layer23_attention_weight_min": -24.027257919311523,
      "activations/layer2_attention_weight_max": 32.7480354309082,
      "activations/layer2_attention_weight_min": -29.541732788085938,
      "activations/layer3_attention_weight_max": 92.04104614257812,
      "activations/layer3_attention_weight_min": -92.30043029785156,
      "activations/layer4_attention_weight_max": 89.36801147460938,
      "activations/layer4_attention_weight_min": -88.88722229003906,
      "activations/layer5_attention_weight_max": 61.81398010253906,
      "activations/layer5_attention_weight_min": -74.0512466430664,
      "activations/layer6_attention_weight_max": 48.68264389038086,
      "activations/layer6_attention_weight_min": -53.88548278808594,
      "activations/layer7_attention_weight_max": 62.477195739746094,
      "activations/layer7_attention_weight_min": -60.91921615600586,
      "activations/layer8_attention_weight_max": 46.0969123840332,
      "activations/layer8_attention_weight_min": -48.25978088378906,
      "activations/layer9_attention_weight_max": 41.95914840698242,
      "activations/layer9_attention_weight_min": -43.694068908691406,
      "epoch": 18.05,
      "learning_rate": 3.394015151515151e-05,
      "loss": 2.7509,
      "step": 310650
    },
    {
      "activations/layer0_attention_weight_max": 15.603912353515625,
      "activations/layer0_attention_weight_min": -13.31808090209961,
      "activations/layer10_attention_weight_max": 35.97254180908203,
      "activations/layer10_attention_weight_min": -36.867977142333984,
      "activations/layer11_attention_weight_max": 32.86724853515625,
      "activations/layer11_attention_weight_min": -34.263092041015625,
      "activations/layer12_attention_weight_max": 19.002784729003906,
      "activations/layer12_attention_weight_min": -31.311466217041016,
      "activations/layer13_attention_weight_max": 38.181175231933594,
      "activations/layer13_attention_weight_min": -27.635421752929688,
      "activations/layer14_attention_weight_max": 34.7394905090332,
      "activations/layer14_attention_weight_min": -29.470096588134766,
      "activations/layer15_attention_weight_max": 31.451631546020508,
      "activations/layer15_attention_weight_min": -31.298398971557617,
      "activations/layer16_attention_weight_max": 32.43729782104492,
      "activations/layer16_attention_weight_min": -31.565135955810547,
      "activations/layer17_attention_weight_max": 49.22926712036133,
      "activations/layer17_attention_weight_min": -43.70522689819336,
      "activations/layer18_attention_weight_max": 46.47236251831055,
      "activations/layer18_attention_weight_min": -40.86989212036133,
      "activations/layer19_attention_weight_max": 23.766836166381836,
      "activations/layer19_attention_weight_min": -25.015560150146484,
      "activations/layer1_attention_weight_max": 15.735591888427734,
      "activations/layer1_attention_weight_min": -14.205788612365723,
      "activations/layer20_attention_weight_max": 22.688255310058594,
      "activations/layer20_attention_weight_min": -23.62869644165039,
      "activations/layer21_attention_weight_max": 37.575775146484375,
      "activations/layer21_attention_weight_min": -25.93701171875,
      "activations/layer22_attention_weight_max": 29.913572311401367,
      "activations/layer22_attention_weight_min": -26.11551284790039,
      "activations/layer23_attention_weight_max": 33.914730072021484,
      "activations/layer23_attention_weight_min": -25.738327026367188,
      "activations/layer2_attention_weight_max": 31.387279510498047,
      "activations/layer2_attention_weight_min": -29.886747360229492,
      "activations/layer3_attention_weight_max": 86.47662353515625,
      "activations/layer3_attention_weight_min": -85.6017837524414,
      "activations/layer4_attention_weight_max": 85.64775848388672,
      "activations/layer4_attention_weight_min": -83.25598907470703,
      "activations/layer5_attention_weight_max": 61.751094818115234,
      "activations/layer5_attention_weight_min": -74.43760681152344,
      "activations/layer6_attention_weight_max": 51.57515335083008,
      "activations/layer6_attention_weight_min": -50.96257781982422,
      "activations/layer7_attention_weight_max": 65.86683654785156,
      "activations/layer7_attention_weight_min": -62.32758331298828,
      "activations/layer8_attention_weight_max": 43.74980545043945,
      "activations/layer8_attention_weight_min": -47.130760192871094,
      "activations/layer9_attention_weight_max": 41.7177848815918,
      "activations/layer9_attention_weight_min": -46.05887985229492,
      "epoch": 18.05,
      "learning_rate": 3.392121212121212e-05,
      "loss": 2.742,
      "step": 310700
    },
    {
      "activations/layer0_attention_weight_max": 15.044949531555176,
      "activations/layer0_attention_weight_min": -13.08373737335205,
      "activations/layer10_attention_weight_max": 34.82849884033203,
      "activations/layer10_attention_weight_min": -34.47458267211914,
      "activations/layer11_attention_weight_max": 31.791378021240234,
      "activations/layer11_attention_weight_min": -33.61408233642578,
      "activations/layer12_attention_weight_max": 18.150989532470703,
      "activations/layer12_attention_weight_min": -25.806907653808594,
      "activations/layer13_attention_weight_max": 31.527877807617188,
      "activations/layer13_attention_weight_min": -27.2391300201416,
      "activations/layer14_attention_weight_max": 32.24900436401367,
      "activations/layer14_attention_weight_min": -30.37863540649414,
      "activations/layer15_attention_weight_max": 30.52956771850586,
      "activations/layer15_attention_weight_min": -29.006135940551758,
      "activations/layer16_attention_weight_max": 32.67182540893555,
      "activations/layer16_attention_weight_min": -31.298681259155273,
      "activations/layer17_attention_weight_max": 48.63529586791992,
      "activations/layer17_attention_weight_min": -49.82756042480469,
      "activations/layer18_attention_weight_max": 46.91999053955078,
      "activations/layer18_attention_weight_min": -40.599266052246094,
      "activations/layer19_attention_weight_max": 25.03265380859375,
      "activations/layer19_attention_weight_min": -25.190519332885742,
      "activations/layer1_attention_weight_max": 16.136449813842773,
      "activations/layer1_attention_weight_min": -13.899856567382812,
      "activations/layer20_attention_weight_max": 22.535917282104492,
      "activations/layer20_attention_weight_min": -21.37157440185547,
      "activations/layer21_attention_weight_max": 39.56340408325195,
      "activations/layer21_attention_weight_min": -20.808290481567383,
      "activations/layer22_attention_weight_max": 29.48297882080078,
      "activations/layer22_attention_weight_min": -25.768049240112305,
      "activations/layer23_attention_weight_max": 35.888206481933594,
      "activations/layer23_attention_weight_min": -24.393478393554688,
      "activations/layer2_attention_weight_max": 30.40362548828125,
      "activations/layer2_attention_weight_min": -28.0035400390625,
      "activations/layer3_attention_weight_max": 84.73387908935547,
      "activations/layer3_attention_weight_min": -82.2671127319336,
      "activations/layer4_attention_weight_max": 88.1895523071289,
      "activations/layer4_attention_weight_min": -85.468994140625,
      "activations/layer5_attention_weight_max": 63.13910675048828,
      "activations/layer5_attention_weight_min": -76.01331329345703,
      "activations/layer6_attention_weight_max": 51.31270217895508,
      "activations/layer6_attention_weight_min": -50.82514190673828,
      "activations/layer7_attention_weight_max": 66.2088394165039,
      "activations/layer7_attention_weight_min": -65.63584899902344,
      "activations/layer8_attention_weight_max": 48.15642166137695,
      "activations/layer8_attention_weight_min": -52.4870491027832,
      "activations/layer9_attention_weight_max": 42.81719207763672,
      "activations/layer9_attention_weight_min": -44.36227035522461,
      "epoch": 18.06,
      "learning_rate": 3.390227272727272e-05,
      "loss": 2.7116,
      "step": 310750
    },
    {
      "activations/layer0_attention_weight_max": 15.467353820800781,
      "activations/layer0_attention_weight_min": -14.455859184265137,
      "activations/layer10_attention_weight_max": 40.73269271850586,
      "activations/layer10_attention_weight_min": -38.04853820800781,
      "activations/layer11_attention_weight_max": 38.1839599609375,
      "activations/layer11_attention_weight_min": -36.92729187011719,
      "activations/layer12_attention_weight_max": 19.633359909057617,
      "activations/layer12_attention_weight_min": -25.313800811767578,
      "activations/layer13_attention_weight_max": 47.13569259643555,
      "activations/layer13_attention_weight_min": -27.676612854003906,
      "activations/layer14_attention_weight_max": 46.72439193725586,
      "activations/layer14_attention_weight_min": -31.11267852783203,
      "activations/layer15_attention_weight_max": 45.542930603027344,
      "activations/layer15_attention_weight_min": -31.79316520690918,
      "activations/layer16_attention_weight_max": 33.9206428527832,
      "activations/layer16_attention_weight_min": -34.35922622680664,
      "activations/layer17_attention_weight_max": 49.82368850708008,
      "activations/layer17_attention_weight_min": -47.30453109741211,
      "activations/layer18_attention_weight_max": 46.66011047363281,
      "activations/layer18_attention_weight_min": -43.93612289428711,
      "activations/layer19_attention_weight_max": 24.354312896728516,
      "activations/layer19_attention_weight_min": -26.181161880493164,
      "activations/layer1_attention_weight_max": 16.547027587890625,
      "activations/layer1_attention_weight_min": -13.28184986114502,
      "activations/layer20_attention_weight_max": 23.42047119140625,
      "activations/layer20_attention_weight_min": -24.727048873901367,
      "activations/layer21_attention_weight_max": 36.447021484375,
      "activations/layer21_attention_weight_min": -27.467973709106445,
      "activations/layer22_attention_weight_max": 32.25969314575195,
      "activations/layer22_attention_weight_min": -29.958513259887695,
      "activations/layer23_attention_weight_max": 38.33739471435547,
      "activations/layer23_attention_weight_min": -25.814733505249023,
      "activations/layer2_attention_weight_max": 30.53244400024414,
      "activations/layer2_attention_weight_min": -28.882802963256836,
      "activations/layer3_attention_weight_max": 92.59236907958984,
      "activations/layer3_attention_weight_min": -93.99899291992188,
      "activations/layer4_attention_weight_max": 92.08395385742188,
      "activations/layer4_attention_weight_min": -91.16390991210938,
      "activations/layer5_attention_weight_max": 65.16007995605469,
      "activations/layer5_attention_weight_min": -73.2228775024414,
      "activations/layer6_attention_weight_max": 49.564552307128906,
      "activations/layer6_attention_weight_min": -49.55385208129883,
      "activations/layer7_attention_weight_max": 70.01851654052734,
      "activations/layer7_attention_weight_min": -65.2554702758789,
      "activations/layer8_attention_weight_max": 50.48197937011719,
      "activations/layer8_attention_weight_min": -49.485652923583984,
      "activations/layer9_attention_weight_max": 54.25580596923828,
      "activations/layer9_attention_weight_min": -46.63905715942383,
      "epoch": 18.06,
      "learning_rate": 3.388333333333333e-05,
      "loss": 2.7197,
      "step": 310800
    },
    {
      "activations/layer0_attention_weight_max": 16.43348503112793,
      "activations/layer0_attention_weight_min": -12.9851655960083,
      "activations/layer10_attention_weight_max": 32.99300003051758,
      "activations/layer10_attention_weight_min": -33.133548736572266,
      "activations/layer11_attention_weight_max": 29.496173858642578,
      "activations/layer11_attention_weight_min": -33.129878997802734,
      "activations/layer12_attention_weight_max": 17.864948272705078,
      "activations/layer12_attention_weight_min": -23.22132110595703,
      "activations/layer13_attention_weight_max": 32.912906646728516,
      "activations/layer13_attention_weight_min": -28.69487190246582,
      "activations/layer14_attention_weight_max": 32.43943405151367,
      "activations/layer14_attention_weight_min": -30.50748634338379,
      "activations/layer15_attention_weight_max": 32.61817932128906,
      "activations/layer15_attention_weight_min": -28.345075607299805,
      "activations/layer16_attention_weight_max": 32.52406311035156,
      "activations/layer16_attention_weight_min": -31.256242752075195,
      "activations/layer17_attention_weight_max": 47.22361373901367,
      "activations/layer17_attention_weight_min": -42.25275421142578,
      "activations/layer18_attention_weight_max": 43.63432693481445,
      "activations/layer18_attention_weight_min": -37.81753158569336,
      "activations/layer19_attention_weight_max": 20.646860122680664,
      "activations/layer19_attention_weight_min": -21.462522506713867,
      "activations/layer1_attention_weight_max": 15.790465354919434,
      "activations/layer1_attention_weight_min": -13.598402976989746,
      "activations/layer20_attention_weight_max": 21.7393741607666,
      "activations/layer20_attention_weight_min": -21.027280807495117,
      "activations/layer21_attention_weight_max": 31.157278060913086,
      "activations/layer21_attention_weight_min": -21.607866287231445,
      "activations/layer22_attention_weight_max": 25.564077377319336,
      "activations/layer22_attention_weight_min": -23.883352279663086,
      "activations/layer23_attention_weight_max": 33.4715576171875,
      "activations/layer23_attention_weight_min": -25.689970016479492,
      "activations/layer2_attention_weight_max": 27.835561752319336,
      "activations/layer2_attention_weight_min": -27.209026336669922,
      "activations/layer3_attention_weight_max": 78.98025512695312,
      "activations/layer3_attention_weight_min": -83.43180084228516,
      "activations/layer4_attention_weight_max": 79.09919738769531,
      "activations/layer4_attention_weight_min": -79.20650482177734,
      "activations/layer5_attention_weight_max": 60.40772247314453,
      "activations/layer5_attention_weight_min": -68.16325378417969,
      "activations/layer6_attention_weight_max": 47.046390533447266,
      "activations/layer6_attention_weight_min": -46.45405197143555,
      "activations/layer7_attention_weight_max": 59.24775695800781,
      "activations/layer7_attention_weight_min": -58.13960647583008,
      "activations/layer8_attention_weight_max": 45.20899200439453,
      "activations/layer8_attention_weight_min": -48.00794982910156,
      "activations/layer9_attention_weight_max": 40.47638702392578,
      "activations/layer9_attention_weight_min": -43.61787033081055,
      "epoch": 18.06,
      "learning_rate": 3.3864393939393936e-05,
      "loss": 2.7571,
      "step": 310850
    },
    {
      "activations/layer0_attention_weight_max": 15.61577033996582,
      "activations/layer0_attention_weight_min": -13.227720260620117,
      "activations/layer10_attention_weight_max": 35.330230712890625,
      "activations/layer10_attention_weight_min": -36.725555419921875,
      "activations/layer11_attention_weight_max": 33.82280349731445,
      "activations/layer11_attention_weight_min": -35.41257095336914,
      "activations/layer12_attention_weight_max": 19.138273239135742,
      "activations/layer12_attention_weight_min": -26.899171829223633,
      "activations/layer13_attention_weight_max": 35.57600784301758,
      "activations/layer13_attention_weight_min": -28.650123596191406,
      "activations/layer14_attention_weight_max": 34.11289596557617,
      "activations/layer14_attention_weight_min": -30.073713302612305,
      "activations/layer15_attention_weight_max": 30.979042053222656,
      "activations/layer15_attention_weight_min": -30.74505615234375,
      "activations/layer16_attention_weight_max": 34.597930908203125,
      "activations/layer16_attention_weight_min": -31.643314361572266,
      "activations/layer17_attention_weight_max": 51.21683120727539,
      "activations/layer17_attention_weight_min": -43.298423767089844,
      "activations/layer18_attention_weight_max": 50.53590393066406,
      "activations/layer18_attention_weight_min": -38.385074615478516,
      "activations/layer19_attention_weight_max": 22.01103401184082,
      "activations/layer19_attention_weight_min": -23.310209274291992,
      "activations/layer1_attention_weight_max": 16.360342025756836,
      "activations/layer1_attention_weight_min": -13.332690238952637,
      "activations/layer20_attention_weight_max": 22.87540054321289,
      "activations/layer20_attention_weight_min": -22.776744842529297,
      "activations/layer21_attention_weight_max": 37.81300735473633,
      "activations/layer21_attention_weight_min": -22.26414680480957,
      "activations/layer22_attention_weight_max": 30.572040557861328,
      "activations/layer22_attention_weight_min": -24.65687370300293,
      "activations/layer23_attention_weight_max": 38.37331008911133,
      "activations/layer23_attention_weight_min": -24.433645248413086,
      "activations/layer2_attention_weight_max": 31.17703628540039,
      "activations/layer2_attention_weight_min": -29.635425567626953,
      "activations/layer3_attention_weight_max": 88.67266845703125,
      "activations/layer3_attention_weight_min": -85.64786529541016,
      "activations/layer4_attention_weight_max": 84.13108825683594,
      "activations/layer4_attention_weight_min": -84.10607147216797,
      "activations/layer5_attention_weight_max": 60.33721160888672,
      "activations/layer5_attention_weight_min": -73.21653747558594,
      "activations/layer6_attention_weight_max": 47.82320785522461,
      "activations/layer6_attention_weight_min": -50.372703552246094,
      "activations/layer7_attention_weight_max": 60.90461730957031,
      "activations/layer7_attention_weight_min": -63.06608581542969,
      "activations/layer8_attention_weight_max": 46.53981018066406,
      "activations/layer8_attention_weight_min": -49.59358596801758,
      "activations/layer9_attention_weight_max": 45.08347702026367,
      "activations/layer9_attention_weight_min": -45.09843444824219,
      "epoch": 18.07,
      "learning_rate": 3.3845454545454545e-05,
      "loss": 2.7485,
      "step": 310900
    },
    {
      "activations/layer0_attention_weight_max": 15.313725471496582,
      "activations/layer0_attention_weight_min": -13.404221534729004,
      "activations/layer10_attention_weight_max": 36.892086029052734,
      "activations/layer10_attention_weight_min": -37.92730712890625,
      "activations/layer11_attention_weight_max": 33.510196685791016,
      "activations/layer11_attention_weight_min": -36.80472946166992,
      "activations/layer12_attention_weight_max": 18.159706115722656,
      "activations/layer12_attention_weight_min": -23.14686393737793,
      "activations/layer13_attention_weight_max": 31.05192756652832,
      "activations/layer13_attention_weight_min": -28.756410598754883,
      "activations/layer14_attention_weight_max": 32.32944107055664,
      "activations/layer14_attention_weight_min": -29.326086044311523,
      "activations/layer15_attention_weight_max": 32.00670623779297,
      "activations/layer15_attention_weight_min": -29.288753509521484,
      "activations/layer16_attention_weight_max": 30.656925201416016,
      "activations/layer16_attention_weight_min": -30.754175186157227,
      "activations/layer17_attention_weight_max": 49.711341857910156,
      "activations/layer17_attention_weight_min": -44.635868072509766,
      "activations/layer18_attention_weight_max": 46.20978546142578,
      "activations/layer18_attention_weight_min": -37.563087463378906,
      "activations/layer19_attention_weight_max": 21.48349952697754,
      "activations/layer19_attention_weight_min": -21.312429428100586,
      "activations/layer1_attention_weight_max": 15.97288703918457,
      "activations/layer1_attention_weight_min": -15.232275009155273,
      "activations/layer20_attention_weight_max": 20.524185180664062,
      "activations/layer20_attention_weight_min": -22.336626052856445,
      "activations/layer21_attention_weight_max": 33.355098724365234,
      "activations/layer21_attention_weight_min": -23.274150848388672,
      "activations/layer22_attention_weight_max": 27.201976776123047,
      "activations/layer22_attention_weight_min": -24.596141815185547,
      "activations/layer23_attention_weight_max": 37.89057922363281,
      "activations/layer23_attention_weight_min": -26.626354217529297,
      "activations/layer2_attention_weight_max": 33.96548843383789,
      "activations/layer2_attention_weight_min": -32.47038650512695,
      "activations/layer3_attention_weight_max": 96.4554443359375,
      "activations/layer3_attention_weight_min": -97.56439971923828,
      "activations/layer4_attention_weight_max": 94.70458221435547,
      "activations/layer4_attention_weight_min": -93.42134857177734,
      "activations/layer5_attention_weight_max": 67.55074310302734,
      "activations/layer5_attention_weight_min": -80.34851837158203,
      "activations/layer6_attention_weight_max": 51.6356315612793,
      "activations/layer6_attention_weight_min": -52.90021514892578,
      "activations/layer7_attention_weight_max": 69.33740997314453,
      "activations/layer7_attention_weight_min": -70.5882339477539,
      "activations/layer8_attention_weight_max": 49.236412048339844,
      "activations/layer8_attention_weight_min": -50.4843864440918,
      "activations/layer9_attention_weight_max": 46.464988708496094,
      "activations/layer9_attention_weight_min": -47.74940872192383,
      "epoch": 18.07,
      "learning_rate": 3.382651515151515e-05,
      "loss": 2.739,
      "step": 310950
    },
    {
      "activations/layer0_attention_weight_max": 15.20251178741455,
      "activations/layer0_attention_weight_min": -14.409019470214844,
      "activations/layer10_attention_weight_max": 35.07948684692383,
      "activations/layer10_attention_weight_min": -35.850929260253906,
      "activations/layer11_attention_weight_max": 33.34892272949219,
      "activations/layer11_attention_weight_min": -35.00941467285156,
      "activations/layer12_attention_weight_max": 18.919126510620117,
      "activations/layer12_attention_weight_min": -35.09455108642578,
      "activations/layer13_attention_weight_max": 31.370698928833008,
      "activations/layer13_attention_weight_min": -26.695138931274414,
      "activations/layer14_attention_weight_max": 32.67768096923828,
      "activations/layer14_attention_weight_min": -28.420007705688477,
      "activations/layer15_attention_weight_max": 32.87722396850586,
      "activations/layer15_attention_weight_min": -29.1158504486084,
      "activations/layer16_attention_weight_max": 34.80501937866211,
      "activations/layer16_attention_weight_min": -32.03907775878906,
      "activations/layer17_attention_weight_max": 53.1910285949707,
      "activations/layer17_attention_weight_min": -46.22398376464844,
      "activations/layer18_attention_weight_max": 48.01421356201172,
      "activations/layer18_attention_weight_min": -38.4436149597168,
      "activations/layer19_attention_weight_max": 26.36406898498535,
      "activations/layer19_attention_weight_min": -23.100317001342773,
      "activations/layer1_attention_weight_max": 15.699270248413086,
      "activations/layer1_attention_weight_min": -14.43124008178711,
      "activations/layer20_attention_weight_max": 23.117931365966797,
      "activations/layer20_attention_weight_min": -19.946836471557617,
      "activations/layer21_attention_weight_max": 38.16255187988281,
      "activations/layer21_attention_weight_min": -19.254314422607422,
      "activations/layer22_attention_weight_max": 34.155025482177734,
      "activations/layer22_attention_weight_min": -23.86602020263672,
      "activations/layer23_attention_weight_max": 34.776790618896484,
      "activations/layer23_attention_weight_min": -21.50935173034668,
      "activations/layer2_attention_weight_max": 28.602697372436523,
      "activations/layer2_attention_weight_min": -28.44696617126465,
      "activations/layer3_attention_weight_max": 82.24351501464844,
      "activations/layer3_attention_weight_min": -81.7220230102539,
      "activations/layer4_attention_weight_max": 82.89799499511719,
      "activations/layer4_attention_weight_min": -80.73880767822266,
      "activations/layer5_attention_weight_max": 63.83502197265625,
      "activations/layer5_attention_weight_min": -72.86170959472656,
      "activations/layer6_attention_weight_max": 49.04848861694336,
      "activations/layer6_attention_weight_min": -53.30878829956055,
      "activations/layer7_attention_weight_max": 60.929290771484375,
      "activations/layer7_attention_weight_min": -61.47936248779297,
      "activations/layer8_attention_weight_max": 46.05644226074219,
      "activations/layer8_attention_weight_min": -51.046875,
      "activations/layer9_attention_weight_max": 48.313385009765625,
      "activations/layer9_attention_weight_min": -48.68313980102539,
      "epoch": 18.07,
      "learning_rate": 3.3807575757575754e-05,
      "loss": 2.7438,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4418,
      "eval_samples_per_second": 508.661,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4418,
      "eval_openwebtext_samples_per_second": 508.661,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9883,
      "eval_wikitext_samples_per_second": 229.339,
      "step": 311000
    },
    {
      "epoch": 18.07,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.6417,
      "eval_lambada_samples_per_second": 504.993,
      "step": 311000
    },
    {
      "activations/layer0_attention_weight_max": 15.3073091506958,
      "activations/layer0_attention_weight_min": -14.016768455505371,
      "activations/layer10_attention_weight_max": 35.97841262817383,
      "activations/layer10_attention_weight_min": -35.93645095825195,
      "activations/layer11_attention_weight_max": 32.00834655761719,
      "activations/layer11_attention_weight_min": -33.38804244995117,
      "activations/layer12_attention_weight_max": 19.48223876953125,
      "activations/layer12_attention_weight_min": -25.28752326965332,
      "activations/layer13_attention_weight_max": 31.12286949157715,
      "activations/layer13_attention_weight_min": -26.596723556518555,
      "activations/layer14_attention_weight_max": 34.62689208984375,
      "activations/layer14_attention_weight_min": -28.50077247619629,
      "activations/layer15_attention_weight_max": 32.95172119140625,
      "activations/layer15_attention_weight_min": -28.74199104309082,
      "activations/layer16_attention_weight_max": 33.07130432128906,
      "activations/layer16_attention_weight_min": -32.21448516845703,
      "activations/layer17_attention_weight_max": 54.950801849365234,
      "activations/layer17_attention_weight_min": -45.85417938232422,
      "activations/layer18_attention_weight_max": 48.38083267211914,
      "activations/layer18_attention_weight_min": -39.321929931640625,
      "activations/layer19_attention_weight_max": 23.5433406829834,
      "activations/layer19_attention_weight_min": -23.420734405517578,
      "activations/layer1_attention_weight_max": 15.07900333404541,
      "activations/layer1_attention_weight_min": -14.17864990234375,
      "activations/layer20_attention_weight_max": 23.803796768188477,
      "activations/layer20_attention_weight_min": -22.54599952697754,
      "activations/layer21_attention_weight_max": 38.16322708129883,
      "activations/layer21_attention_weight_min": -24.466594696044922,
      "activations/layer22_attention_weight_max": 31.76694679260254,
      "activations/layer22_attention_weight_min": -25.625629425048828,
      "activations/layer23_attention_weight_max": 38.616798400878906,
      "activations/layer23_attention_weight_min": -24.049015045166016,
      "activations/layer2_attention_weight_max": 30.279613494873047,
      "activations/layer2_attention_weight_min": -27.088829040527344,
      "activations/layer3_attention_weight_max": 83.4874267578125,
      "activations/layer3_attention_weight_min": -83.06005859375,
      "activations/layer4_attention_weight_max": 82.14041900634766,
      "activations/layer4_attention_weight_min": -84.76532745361328,
      "activations/layer5_attention_weight_max": 64.93257141113281,
      "activations/layer5_attention_weight_min": -72.61770629882812,
      "activations/layer6_attention_weight_max": 49.4107551574707,
      "activations/layer6_attention_weight_min": -52.20746994018555,
      "activations/layer7_attention_weight_max": 62.07734298706055,
      "activations/layer7_attention_weight_min": -62.006370544433594,
      "activations/layer8_attention_weight_max": 44.530433654785156,
      "activations/layer8_attention_weight_min": -49.73723220825195,
      "activations/layer9_attention_weight_max": 42.13625717163086,
      "activations/layer9_attention_weight_min": -46.49022674560547,
      "epoch": 18.07,
      "learning_rate": 3.378863636363636e-05,
      "loss": 2.7245,
      "step": 311050
    },
    {
      "activations/layer0_attention_weight_max": 14.741418838500977,
      "activations/layer0_attention_weight_min": -13.095430374145508,
      "activations/layer10_attention_weight_max": 33.94532012939453,
      "activations/layer10_attention_weight_min": -35.4166145324707,
      "activations/layer11_attention_weight_max": 32.015071868896484,
      "activations/layer11_attention_weight_min": -32.878990173339844,
      "activations/layer12_attention_weight_max": 18.48362922668457,
      "activations/layer12_attention_weight_min": -25.98827362060547,
      "activations/layer13_attention_weight_max": 32.60185241699219,
      "activations/layer13_attention_weight_min": -28.334619522094727,
      "activations/layer14_attention_weight_max": 35.258296966552734,
      "activations/layer14_attention_weight_min": -29.98823356628418,
      "activations/layer15_attention_weight_max": 34.70149230957031,
      "activations/layer15_attention_weight_min": -29.691781997680664,
      "activations/layer16_attention_weight_max": 33.91720199584961,
      "activations/layer16_attention_weight_min": -30.30049705505371,
      "activations/layer17_attention_weight_max": 49.77383041381836,
      "activations/layer17_attention_weight_min": -44.04940414428711,
      "activations/layer18_attention_weight_max": 47.980228424072266,
      "activations/layer18_attention_weight_min": -39.939456939697266,
      "activations/layer19_attention_weight_max": 22.760480880737305,
      "activations/layer19_attention_weight_min": -20.939329147338867,
      "activations/layer1_attention_weight_max": 15.822182655334473,
      "activations/layer1_attention_weight_min": -16.80579948425293,
      "activations/layer20_attention_weight_max": 20.440109252929688,
      "activations/layer20_attention_weight_min": -22.218677520751953,
      "activations/layer21_attention_weight_max": 32.80111312866211,
      "activations/layer21_attention_weight_min": -22.968053817749023,
      "activations/layer22_attention_weight_max": 27.447248458862305,
      "activations/layer22_attention_weight_min": -25.4502010345459,
      "activations/layer23_attention_weight_max": 35.220237731933594,
      "activations/layer23_attention_weight_min": -25.690502166748047,
      "activations/layer2_attention_weight_max": 29.99647331237793,
      "activations/layer2_attention_weight_min": -29.161544799804688,
      "activations/layer3_attention_weight_max": 80.40707397460938,
      "activations/layer3_attention_weight_min": -88.09127807617188,
      "activations/layer4_attention_weight_max": 78.76273345947266,
      "activations/layer4_attention_weight_min": -81.11665344238281,
      "activations/layer5_attention_weight_max": 60.947288513183594,
      "activations/layer5_attention_weight_min": -71.76939392089844,
      "activations/layer6_attention_weight_max": 48.21574783325195,
      "activations/layer6_attention_weight_min": -49.73491668701172,
      "activations/layer7_attention_weight_max": 62.493892669677734,
      "activations/layer7_attention_weight_min": -61.297969818115234,
      "activations/layer8_attention_weight_max": 43.335384368896484,
      "activations/layer8_attention_weight_min": -47.28097915649414,
      "activations/layer9_attention_weight_max": 44.06837463378906,
      "activations/layer9_attention_weight_min": -44.38420486450195,
      "epoch": 18.08,
      "learning_rate": 3.3769696969696964e-05,
      "loss": 2.738,
      "step": 311100
    },
    {
      "activations/layer0_attention_weight_max": 14.78802490234375,
      "activations/layer0_attention_weight_min": -13.201807975769043,
      "activations/layer10_attention_weight_max": 35.42510986328125,
      "activations/layer10_attention_weight_min": -36.09514236450195,
      "activations/layer11_attention_weight_max": 33.06645965576172,
      "activations/layer11_attention_weight_min": -34.58692169189453,
      "activations/layer12_attention_weight_max": 17.632801055908203,
      "activations/layer12_attention_weight_min": -25.67909049987793,
      "activations/layer13_attention_weight_max": 33.41655349731445,
      "activations/layer13_attention_weight_min": -30.011795043945312,
      "activations/layer14_attention_weight_max": 35.498165130615234,
      "activations/layer14_attention_weight_min": -29.6667423248291,
      "activations/layer15_attention_weight_max": 34.35661697387695,
      "activations/layer15_attention_weight_min": -30.196205139160156,
      "activations/layer16_attention_weight_max": 34.422760009765625,
      "activations/layer16_attention_weight_min": -32.302940368652344,
      "activations/layer17_attention_weight_max": 53.40131378173828,
      "activations/layer17_attention_weight_min": -44.76092529296875,
      "activations/layer18_attention_weight_max": 47.84005355834961,
      "activations/layer18_attention_weight_min": -37.60273361206055,
      "activations/layer19_attention_weight_max": 24.3980655670166,
      "activations/layer19_attention_weight_min": -21.450220108032227,
      "activations/layer1_attention_weight_max": 15.913277626037598,
      "activations/layer1_attention_weight_min": -13.63614273071289,
      "activations/layer20_attention_weight_max": 23.43695068359375,
      "activations/layer20_attention_weight_min": -24.485898971557617,
      "activations/layer21_attention_weight_max": 34.19630813598633,
      "activations/layer21_attention_weight_min": -21.374032974243164,
      "activations/layer22_attention_weight_max": 29.50544548034668,
      "activations/layer22_attention_weight_min": -25.26197624206543,
      "activations/layer23_attention_weight_max": 37.2620849609375,
      "activations/layer23_attention_weight_min": -25.69447898864746,
      "activations/layer2_attention_weight_max": 31.624713897705078,
      "activations/layer2_attention_weight_min": -30.290138244628906,
      "activations/layer3_attention_weight_max": 86.91303253173828,
      "activations/layer3_attention_weight_min": -91.30375671386719,
      "activations/layer4_attention_weight_max": 86.01368713378906,
      "activations/layer4_attention_weight_min": -84.7564697265625,
      "activations/layer5_attention_weight_max": 61.7568359375,
      "activations/layer5_attention_weight_min": -78.95211791992188,
      "activations/layer6_attention_weight_max": 50.559303283691406,
      "activations/layer6_attention_weight_min": -53.73908615112305,
      "activations/layer7_attention_weight_max": 62.41897964477539,
      "activations/layer7_attention_weight_min": -66.94834899902344,
      "activations/layer8_attention_weight_max": 45.6068000793457,
      "activations/layer8_attention_weight_min": -53.243412017822266,
      "activations/layer9_attention_weight_max": 44.233970642089844,
      "activations/layer9_attention_weight_min": -49.03794479370117,
      "epoch": 18.08,
      "learning_rate": 3.375075757575757e-05,
      "loss": 2.752,
      "step": 311150
    },
    {
      "activations/layer0_attention_weight_max": 15.082879066467285,
      "activations/layer0_attention_weight_min": -13.985726356506348,
      "activations/layer10_attention_weight_max": 32.967411041259766,
      "activations/layer10_attention_weight_min": -36.29993438720703,
      "activations/layer11_attention_weight_max": 31.689083099365234,
      "activations/layer11_attention_weight_min": -33.5740852355957,
      "activations/layer12_attention_weight_max": 18.523250579833984,
      "activations/layer12_attention_weight_min": -23.492246627807617,
      "activations/layer13_attention_weight_max": 33.53489685058594,
      "activations/layer13_attention_weight_min": -27.705612182617188,
      "activations/layer14_attention_weight_max": 32.84595489501953,
      "activations/layer14_attention_weight_min": -31.767070770263672,
      "activations/layer15_attention_weight_max": 33.46421813964844,
      "activations/layer15_attention_weight_min": -31.083845138549805,
      "activations/layer16_attention_weight_max": 34.48506546020508,
      "activations/layer16_attention_weight_min": -32.58086013793945,
      "activations/layer17_attention_weight_max": 49.84148025512695,
      "activations/layer17_attention_weight_min": -45.89031982421875,
      "activations/layer18_attention_weight_max": 46.94731903076172,
      "activations/layer18_attention_weight_min": -39.375282287597656,
      "activations/layer19_attention_weight_max": 25.520078659057617,
      "activations/layer19_attention_weight_min": -23.23344612121582,
      "activations/layer1_attention_weight_max": 16.95881462097168,
      "activations/layer1_attention_weight_min": -14.945920944213867,
      "activations/layer20_attention_weight_max": 24.176830291748047,
      "activations/layer20_attention_weight_min": -21.815189361572266,
      "activations/layer21_attention_weight_max": 36.34671401977539,
      "activations/layer21_attention_weight_min": -20.021438598632812,
      "activations/layer22_attention_weight_max": 31.7304744720459,
      "activations/layer22_attention_weight_min": -27.53536033630371,
      "activations/layer23_attention_weight_max": 38.8850212097168,
      "activations/layer23_attention_weight_min": -23.779953002929688,
      "activations/layer2_attention_weight_max": 30.12435531616211,
      "activations/layer2_attention_weight_min": -27.92593765258789,
      "activations/layer3_attention_weight_max": 86.21060943603516,
      "activations/layer3_attention_weight_min": -87.28837585449219,
      "activations/layer4_attention_weight_max": 91.0952377319336,
      "activations/layer4_attention_weight_min": -86.5118637084961,
      "activations/layer5_attention_weight_max": 64.51820373535156,
      "activations/layer5_attention_weight_min": -76.46251678466797,
      "activations/layer6_attention_weight_max": 49.406368255615234,
      "activations/layer6_attention_weight_min": -51.202144622802734,
      "activations/layer7_attention_weight_max": 61.88611602783203,
      "activations/layer7_attention_weight_min": -65.05735778808594,
      "activations/layer8_attention_weight_max": 44.691986083984375,
      "activations/layer8_attention_weight_min": -50.33479690551758,
      "activations/layer9_attention_weight_max": 42.706207275390625,
      "activations/layer9_attention_weight_min": -44.69211959838867,
      "epoch": 18.08,
      "learning_rate": 3.373181818181818e-05,
      "loss": 2.7319,
      "step": 311200
    },
    {
      "activations/layer0_attention_weight_max": 14.744354248046875,
      "activations/layer0_attention_weight_min": -13.487955093383789,
      "activations/layer10_attention_weight_max": 36.58698272705078,
      "activations/layer10_attention_weight_min": -36.48008346557617,
      "activations/layer11_attention_weight_max": 31.814197540283203,
      "activations/layer11_attention_weight_min": -34.37692642211914,
      "activations/layer12_attention_weight_max": 17.905790328979492,
      "activations/layer12_attention_weight_min": -24.31340980529785,
      "activations/layer13_attention_weight_max": 33.886619567871094,
      "activations/layer13_attention_weight_min": -30.01454734802246,
      "activations/layer14_attention_weight_max": 35.776363372802734,
      "activations/layer14_attention_weight_min": -30.210718154907227,
      "activations/layer15_attention_weight_max": 33.39996337890625,
      "activations/layer15_attention_weight_min": -29.174354553222656,
      "activations/layer16_attention_weight_max": 32.8674430847168,
      "activations/layer16_attention_weight_min": -31.063318252563477,
      "activations/layer17_attention_weight_max": 51.659820556640625,
      "activations/layer17_attention_weight_min": -44.5490608215332,
      "activations/layer18_attention_weight_max": 44.872840881347656,
      "activations/layer18_attention_weight_min": -39.00819396972656,
      "activations/layer19_attention_weight_max": 22.590774536132812,
      "activations/layer19_attention_weight_min": -22.543203353881836,
      "activations/layer1_attention_weight_max": 17.63609504699707,
      "activations/layer1_attention_weight_min": -15.449175834655762,
      "activations/layer20_attention_weight_max": 22.16074562072754,
      "activations/layer20_attention_weight_min": -22.676481246948242,
      "activations/layer21_attention_weight_max": 35.630035400390625,
      "activations/layer21_attention_weight_min": -27.26998519897461,
      "activations/layer22_attention_weight_max": 31.71379852294922,
      "activations/layer22_attention_weight_min": -26.454940795898438,
      "activations/layer23_attention_weight_max": 36.57351303100586,
      "activations/layer23_attention_weight_min": -23.909786224365234,
      "activations/layer2_attention_weight_max": 30.18121910095215,
      "activations/layer2_attention_weight_min": -28.48335075378418,
      "activations/layer3_attention_weight_max": 84.52052307128906,
      "activations/layer3_attention_weight_min": -83.80311584472656,
      "activations/layer4_attention_weight_max": 84.01527404785156,
      "activations/layer4_attention_weight_min": -82.61263275146484,
      "activations/layer5_attention_weight_max": 62.4210205078125,
      "activations/layer5_attention_weight_min": -74.48605346679688,
      "activations/layer6_attention_weight_max": 49.969669342041016,
      "activations/layer6_attention_weight_min": -50.57583999633789,
      "activations/layer7_attention_weight_max": 61.930908203125,
      "activations/layer7_attention_weight_min": -62.414276123046875,
      "activations/layer8_attention_weight_max": 46.343353271484375,
      "activations/layer8_attention_weight_min": -49.043766021728516,
      "activations/layer9_attention_weight_max": 44.087982177734375,
      "activations/layer9_attention_weight_min": -45.77302932739258,
      "epoch": 18.09,
      "learning_rate": 3.371287878787879e-05,
      "loss": 2.7385,
      "step": 311250
    },
    {
      "activations/layer0_attention_weight_max": 15.185506820678711,
      "activations/layer0_attention_weight_min": -12.837310791015625,
      "activations/layer10_attention_weight_max": 36.20703887939453,
      "activations/layer10_attention_weight_min": -36.11893081665039,
      "activations/layer11_attention_weight_max": 32.64634704589844,
      "activations/layer11_attention_weight_min": -34.012962341308594,
      "activations/layer12_attention_weight_max": 19.010995864868164,
      "activations/layer12_attention_weight_min": -22.933109283447266,
      "activations/layer13_attention_weight_max": 37.33998489379883,
      "activations/layer13_attention_weight_min": -29.594789505004883,
      "activations/layer14_attention_weight_max": 39.00136184692383,
      "activations/layer14_attention_weight_min": -30.214570999145508,
      "activations/layer15_attention_weight_max": 33.978553771972656,
      "activations/layer15_attention_weight_min": -28.900407791137695,
      "activations/layer16_attention_weight_max": 34.809871673583984,
      "activations/layer16_attention_weight_min": -30.283390045166016,
      "activations/layer17_attention_weight_max": 51.55681228637695,
      "activations/layer17_attention_weight_min": -42.551780700683594,
      "activations/layer18_attention_weight_max": 51.02428436279297,
      "activations/layer18_attention_weight_min": -40.11418914794922,
      "activations/layer19_attention_weight_max": 25.615354537963867,
      "activations/layer19_attention_weight_min": -25.386388778686523,
      "activations/layer1_attention_weight_max": 15.532331466674805,
      "activations/layer1_attention_weight_min": -14.232683181762695,
      "activations/layer20_attention_weight_max": 27.304845809936523,
      "activations/layer20_attention_weight_min": -22.63874626159668,
      "activations/layer21_attention_weight_max": 37.89663314819336,
      "activations/layer21_attention_weight_min": -21.919084548950195,
      "activations/layer22_attention_weight_max": 30.504549026489258,
      "activations/layer22_attention_weight_min": -26.51664924621582,
      "activations/layer23_attention_weight_max": 39.938629150390625,
      "activations/layer23_attention_weight_min": -26.895191192626953,
      "activations/layer2_attention_weight_max": 30.603092193603516,
      "activations/layer2_attention_weight_min": -28.959640502929688,
      "activations/layer3_attention_weight_max": 85.83934783935547,
      "activations/layer3_attention_weight_min": -86.07423400878906,
      "activations/layer4_attention_weight_max": 87.53457641601562,
      "activations/layer4_attention_weight_min": -86.40706634521484,
      "activations/layer5_attention_weight_max": 61.415977478027344,
      "activations/layer5_attention_weight_min": -70.91839599609375,
      "activations/layer6_attention_weight_max": 49.60997009277344,
      "activations/layer6_attention_weight_min": -51.92079162597656,
      "activations/layer7_attention_weight_max": 63.118228912353516,
      "activations/layer7_attention_weight_min": -64.62228393554688,
      "activations/layer8_attention_weight_max": 45.5349235534668,
      "activations/layer8_attention_weight_min": -47.52989196777344,
      "activations/layer9_attention_weight_max": 42.8568000793457,
      "activations/layer9_attention_weight_min": -44.45426559448242,
      "epoch": 18.09,
      "learning_rate": 3.369393939393939e-05,
      "loss": 2.7487,
      "step": 311300
    },
    {
      "activations/layer0_attention_weight_max": 14.781060218811035,
      "activations/layer0_attention_weight_min": -13.490802764892578,
      "activations/layer10_attention_weight_max": 39.74369430541992,
      "activations/layer10_attention_weight_min": -41.005348205566406,
      "activations/layer11_attention_weight_max": 36.21101379394531,
      "activations/layer11_attention_weight_min": -37.308406829833984,
      "activations/layer12_attention_weight_max": 23.562089920043945,
      "activations/layer12_attention_weight_min": -25.87352180480957,
      "activations/layer13_attention_weight_max": 35.9511833190918,
      "activations/layer13_attention_weight_min": -30.21543312072754,
      "activations/layer14_attention_weight_max": 36.31612777709961,
      "activations/layer14_attention_weight_min": -31.878952026367188,
      "activations/layer15_attention_weight_max": 33.25788116455078,
      "activations/layer15_attention_weight_min": -33.39095687866211,
      "activations/layer16_attention_weight_max": 34.98355484008789,
      "activations/layer16_attention_weight_min": -32.230655670166016,
      "activations/layer17_attention_weight_max": 51.966064453125,
      "activations/layer17_attention_weight_min": -45.867698669433594,
      "activations/layer18_attention_weight_max": 46.31083679199219,
      "activations/layer18_attention_weight_min": -42.24274826049805,
      "activations/layer19_attention_weight_max": 25.304676055908203,
      "activations/layer19_attention_weight_min": -24.38427734375,
      "activations/layer1_attention_weight_max": 15.346939086914062,
      "activations/layer1_attention_weight_min": -15.117498397827148,
      "activations/layer20_attention_weight_max": 23.048095703125,
      "activations/layer20_attention_weight_min": -24.335206985473633,
      "activations/layer21_attention_weight_max": 39.36518096923828,
      "activations/layer21_attention_weight_min": -28.868087768554688,
      "activations/layer22_attention_weight_max": 33.42462921142578,
      "activations/layer22_attention_weight_min": -27.888277053833008,
      "activations/layer23_attention_weight_max": 35.801055908203125,
      "activations/layer23_attention_weight_min": -27.00746726989746,
      "activations/layer2_attention_weight_max": 31.45332145690918,
      "activations/layer2_attention_weight_min": -30.894577026367188,
      "activations/layer3_attention_weight_max": 89.48277282714844,
      "activations/layer3_attention_weight_min": -90.48777770996094,
      "activations/layer4_attention_weight_max": 88.93672943115234,
      "activations/layer4_attention_weight_min": -88.71819305419922,
      "activations/layer5_attention_weight_max": 63.73786163330078,
      "activations/layer5_attention_weight_min": -75.68701934814453,
      "activations/layer6_attention_weight_max": 50.83890914916992,
      "activations/layer6_attention_weight_min": -54.53665542602539,
      "activations/layer7_attention_weight_max": 63.88298797607422,
      "activations/layer7_attention_weight_min": -63.7661018371582,
      "activations/layer8_attention_weight_max": 48.84703063964844,
      "activations/layer8_attention_weight_min": -51.89824295043945,
      "activations/layer9_attention_weight_max": 45.78731918334961,
      "activations/layer9_attention_weight_min": -46.97468185424805,
      "epoch": 18.09,
      "learning_rate": 3.3675e-05,
      "loss": 2.7383,
      "step": 311350
    },
    {
      "activations/layer0_attention_weight_max": 15.998165130615234,
      "activations/layer0_attention_weight_min": -12.786800384521484,
      "activations/layer10_attention_weight_max": 35.783851623535156,
      "activations/layer10_attention_weight_min": -36.18181228637695,
      "activations/layer11_attention_weight_max": 31.390804290771484,
      "activations/layer11_attention_weight_min": -38.39924240112305,
      "activations/layer12_attention_weight_max": 17.99942398071289,
      "activations/layer12_attention_weight_min": -25.37834358215332,
      "activations/layer13_attention_weight_max": 29.689531326293945,
      "activations/layer13_attention_weight_min": -27.47671127319336,
      "activations/layer14_attention_weight_max": 34.062686920166016,
      "activations/layer14_attention_weight_min": -30.493879318237305,
      "activations/layer15_attention_weight_max": 31.954856872558594,
      "activations/layer15_attention_weight_min": -30.084686279296875,
      "activations/layer16_attention_weight_max": 33.597991943359375,
      "activations/layer16_attention_weight_min": -32.566383361816406,
      "activations/layer17_attention_weight_max": 52.06301498413086,
      "activations/layer17_attention_weight_min": -45.947601318359375,
      "activations/layer18_attention_weight_max": 43.16868209838867,
      "activations/layer18_attention_weight_min": -42.864410400390625,
      "activations/layer19_attention_weight_max": 23.307249069213867,
      "activations/layer19_attention_weight_min": -24.89448356628418,
      "activations/layer1_attention_weight_max": 16.282508850097656,
      "activations/layer1_attention_weight_min": -14.903914451599121,
      "activations/layer20_attention_weight_max": 21.506023406982422,
      "activations/layer20_attention_weight_min": -26.966962814331055,
      "activations/layer21_attention_weight_max": 36.59358596801758,
      "activations/layer21_attention_weight_min": -22.7054386138916,
      "activations/layer22_attention_weight_max": 27.638832092285156,
      "activations/layer22_attention_weight_min": -25.660316467285156,
      "activations/layer23_attention_weight_max": 34.634857177734375,
      "activations/layer23_attention_weight_min": -23.384498596191406,
      "activations/layer2_attention_weight_max": 30.673751831054688,
      "activations/layer2_attention_weight_min": -27.548309326171875,
      "activations/layer3_attention_weight_max": 80.1903305053711,
      "activations/layer3_attention_weight_min": -82.54960632324219,
      "activations/layer4_attention_weight_max": 81.12327575683594,
      "activations/layer4_attention_weight_min": -80.83733367919922,
      "activations/layer5_attention_weight_max": 59.812957763671875,
      "activations/layer5_attention_weight_min": -70.73966217041016,
      "activations/layer6_attention_weight_max": 47.95817947387695,
      "activations/layer6_attention_weight_min": -49.669315338134766,
      "activations/layer7_attention_weight_max": 59.084930419921875,
      "activations/layer7_attention_weight_min": -61.163063049316406,
      "activations/layer8_attention_weight_max": 44.092445373535156,
      "activations/layer8_attention_weight_min": -47.09177780151367,
      "activations/layer9_attention_weight_max": 41.01028060913086,
      "activations/layer9_attention_weight_min": -45.28578567504883,
      "epoch": 18.09,
      "learning_rate": 3.36560606060606e-05,
      "loss": 2.7277,
      "step": 311400
    },
    {
      "activations/layer0_attention_weight_max": 16.35512924194336,
      "activations/layer0_attention_weight_min": -12.739352226257324,
      "activations/layer10_attention_weight_max": 35.51313018798828,
      "activations/layer10_attention_weight_min": -36.29963684082031,
      "activations/layer11_attention_weight_max": 32.58564376831055,
      "activations/layer11_attention_weight_min": -34.56230545043945,
      "activations/layer12_attention_weight_max": 17.414077758789062,
      "activations/layer12_attention_weight_min": -23.166183471679688,
      "activations/layer13_attention_weight_max": 37.62287902832031,
      "activations/layer13_attention_weight_min": -30.752553939819336,
      "activations/layer14_attention_weight_max": 36.0169677734375,
      "activations/layer14_attention_weight_min": -31.856250762939453,
      "activations/layer15_attention_weight_max": 32.74861526489258,
      "activations/layer15_attention_weight_min": -30.807504653930664,
      "activations/layer16_attention_weight_max": 34.76841735839844,
      "activations/layer16_attention_weight_min": -32.721275329589844,
      "activations/layer17_attention_weight_max": 55.52272033691406,
      "activations/layer17_attention_weight_min": -47.46025848388672,
      "activations/layer18_attention_weight_max": 51.16108322143555,
      "activations/layer18_attention_weight_min": -39.928375244140625,
      "activations/layer19_attention_weight_max": 25.171606063842773,
      "activations/layer19_attention_weight_min": -22.119129180908203,
      "activations/layer1_attention_weight_max": 15.384925842285156,
      "activations/layer1_attention_weight_min": -14.391965866088867,
      "activations/layer20_attention_weight_max": 25.3719425201416,
      "activations/layer20_attention_weight_min": -21.44091033935547,
      "activations/layer21_attention_weight_max": 39.91973114013672,
      "activations/layer21_attention_weight_min": -22.918643951416016,
      "activations/layer22_attention_weight_max": 32.2729606628418,
      "activations/layer22_attention_weight_min": -25.429765701293945,
      "activations/layer23_attention_weight_max": 39.16242218017578,
      "activations/layer23_attention_weight_min": -24.383155822753906,
      "activations/layer2_attention_weight_max": 31.427913665771484,
      "activations/layer2_attention_weight_min": -29.130802154541016,
      "activations/layer3_attention_weight_max": 83.41694641113281,
      "activations/layer3_attention_weight_min": -85.28711700439453,
      "activations/layer4_attention_weight_max": 81.05663299560547,
      "activations/layer4_attention_weight_min": -83.16636657714844,
      "activations/layer5_attention_weight_max": 59.73325729370117,
      "activations/layer5_attention_weight_min": -69.79815673828125,
      "activations/layer6_attention_weight_max": 47.64078903198242,
      "activations/layer6_attention_weight_min": -50.41407775878906,
      "activations/layer7_attention_weight_max": 62.15876007080078,
      "activations/layer7_attention_weight_min": -66.28153991699219,
      "activations/layer8_attention_weight_max": 46.31312561035156,
      "activations/layer8_attention_weight_min": -48.49025344848633,
      "activations/layer9_attention_weight_max": 45.92476272583008,
      "activations/layer9_attention_weight_min": -46.01875686645508,
      "epoch": 18.1,
      "learning_rate": 3.363712121212121e-05,
      "loss": 2.7302,
      "step": 311450
    },
    {
      "activations/layer0_attention_weight_max": 15.222780227661133,
      "activations/layer0_attention_weight_min": -13.49541187286377,
      "activations/layer10_attention_weight_max": 38.46121597290039,
      "activations/layer10_attention_weight_min": -36.063968658447266,
      "activations/layer11_attention_weight_max": 35.546226501464844,
      "activations/layer11_attention_weight_min": -33.49583053588867,
      "activations/layer12_attention_weight_max": 19.866666793823242,
      "activations/layer12_attention_weight_min": -26.92520523071289,
      "activations/layer13_attention_weight_max": 41.468894958496094,
      "activations/layer13_attention_weight_min": -29.48174476623535,
      "activations/layer14_attention_weight_max": 51.02390670776367,
      "activations/layer14_attention_weight_min": -30.756101608276367,
      "activations/layer15_attention_weight_max": 43.91165542602539,
      "activations/layer15_attention_weight_min": -30.72499656677246,
      "activations/layer16_attention_weight_max": 35.46482849121094,
      "activations/layer16_attention_weight_min": -33.31143569946289,
      "activations/layer17_attention_weight_max": 53.33529281616211,
      "activations/layer17_attention_weight_min": -50.06269836425781,
      "activations/layer18_attention_weight_max": 52.082130432128906,
      "activations/layer18_attention_weight_min": -47.862571716308594,
      "activations/layer19_attention_weight_max": 28.225820541381836,
      "activations/layer19_attention_weight_min": -25.927804946899414,
      "activations/layer1_attention_weight_max": 16.81462287902832,
      "activations/layer1_attention_weight_min": -14.279449462890625,
      "activations/layer20_attention_weight_max": 25.63429832458496,
      "activations/layer20_attention_weight_min": -21.900718688964844,
      "activations/layer21_attention_weight_max": 37.461280822753906,
      "activations/layer21_attention_weight_min": -24.536149978637695,
      "activations/layer22_attention_weight_max": 29.84049415588379,
      "activations/layer22_attention_weight_min": -25.103147506713867,
      "activations/layer23_attention_weight_max": 36.6236457824707,
      "activations/layer23_attention_weight_min": -24.0144100189209,
      "activations/layer2_attention_weight_max": 29.364408493041992,
      "activations/layer2_attention_weight_min": -29.240413665771484,
      "activations/layer3_attention_weight_max": 87.95792388916016,
      "activations/layer3_attention_weight_min": -89.18190002441406,
      "activations/layer4_attention_weight_max": 83.40248107910156,
      "activations/layer4_attention_weight_min": -83.02074432373047,
      "activations/layer5_attention_weight_max": 65.22346496582031,
      "activations/layer5_attention_weight_min": -74.26056671142578,
      "activations/layer6_attention_weight_max": 48.13590621948242,
      "activations/layer6_attention_weight_min": -48.905845642089844,
      "activations/layer7_attention_weight_max": 68.93769836425781,
      "activations/layer7_attention_weight_min": -63.92479705810547,
      "activations/layer8_attention_weight_max": 46.782447814941406,
      "activations/layer8_attention_weight_min": -46.97723388671875,
      "activations/layer9_attention_weight_max": 51.452919006347656,
      "activations/layer9_attention_weight_min": -46.200984954833984,
      "epoch": 18.1,
      "learning_rate": 3.361818181818182e-05,
      "loss": 2.7367,
      "step": 311500
    },
    {
      "activations/layer0_attention_weight_max": 15.935129165649414,
      "activations/layer0_attention_weight_min": -12.92350959777832,
      "activations/layer10_attention_weight_max": 34.9890022277832,
      "activations/layer10_attention_weight_min": -35.35441589355469,
      "activations/layer11_attention_weight_max": 31.61964988708496,
      "activations/layer11_attention_weight_min": -33.915496826171875,
      "activations/layer12_attention_weight_max": 17.74690055847168,
      "activations/layer12_attention_weight_min": -23.877552032470703,
      "activations/layer13_attention_weight_max": 33.048484802246094,
      "activations/layer13_attention_weight_min": -28.49492645263672,
      "activations/layer14_attention_weight_max": 36.2647705078125,
      "activations/layer14_attention_weight_min": -30.158050537109375,
      "activations/layer15_attention_weight_max": 31.67799949645996,
      "activations/layer15_attention_weight_min": -30.755046844482422,
      "activations/layer16_attention_weight_max": 32.69404602050781,
      "activations/layer16_attention_weight_min": -29.809574127197266,
      "activations/layer17_attention_weight_max": 49.18425369262695,
      "activations/layer17_attention_weight_min": -44.79829025268555,
      "activations/layer18_attention_weight_max": 45.313987731933594,
      "activations/layer18_attention_weight_min": -40.01064682006836,
      "activations/layer19_attention_weight_max": 26.299591064453125,
      "activations/layer19_attention_weight_min": -23.3478946685791,
      "activations/layer1_attention_weight_max": 16.62961196899414,
      "activations/layer1_attention_weight_min": -16.069231033325195,
      "activations/layer20_attention_weight_max": 24.419343948364258,
      "activations/layer20_attention_weight_min": -24.470590591430664,
      "activations/layer21_attention_weight_max": 35.008827209472656,
      "activations/layer21_attention_weight_min": -23.288314819335938,
      "activations/layer22_attention_weight_max": 28.82477378845215,
      "activations/layer22_attention_weight_min": -26.099498748779297,
      "activations/layer23_attention_weight_max": 35.58364486694336,
      "activations/layer23_attention_weight_min": -24.839855194091797,
      "activations/layer2_attention_weight_max": 32.39521408081055,
      "activations/layer2_attention_weight_min": -30.595672607421875,
      "activations/layer3_attention_weight_max": 83.72660064697266,
      "activations/layer3_attention_weight_min": -88.16996002197266,
      "activations/layer4_attention_weight_max": 84.62374877929688,
      "activations/layer4_attention_weight_min": -84.21788024902344,
      "activations/layer5_attention_weight_max": 65.30632019042969,
      "activations/layer5_attention_weight_min": -70.48880004882812,
      "activations/layer6_attention_weight_max": 49.07616424560547,
      "activations/layer6_attention_weight_min": -52.5307731628418,
      "activations/layer7_attention_weight_max": 69.9584732055664,
      "activations/layer7_attention_weight_min": -62.63543701171875,
      "activations/layer8_attention_weight_max": 45.91904067993164,
      "activations/layer8_attention_weight_min": -48.585716247558594,
      "activations/layer9_attention_weight_max": 41.68366622924805,
      "activations/layer9_attention_weight_min": -45.19204330444336,
      "epoch": 18.1,
      "learning_rate": 3.3599242424242425e-05,
      "loss": 2.722,
      "step": 311550
    },
    {
      "activations/layer0_attention_weight_max": 14.904329299926758,
      "activations/layer0_attention_weight_min": -12.782124519348145,
      "activations/layer10_attention_weight_max": 35.5804443359375,
      "activations/layer10_attention_weight_min": -34.863189697265625,
      "activations/layer11_attention_weight_max": 32.440834045410156,
      "activations/layer11_attention_weight_min": -33.62120819091797,
      "activations/layer12_attention_weight_max": 18.17951774597168,
      "activations/layer12_attention_weight_min": -24.968976974487305,
      "activations/layer13_attention_weight_max": 38.67020797729492,
      "activations/layer13_attention_weight_min": -28.62603187561035,
      "activations/layer14_attention_weight_max": 37.05166244506836,
      "activations/layer14_attention_weight_min": -30.24942970275879,
      "activations/layer15_attention_weight_max": 33.7139892578125,
      "activations/layer15_attention_weight_min": -30.165679931640625,
      "activations/layer16_attention_weight_max": 34.98135757446289,
      "activations/layer16_attention_weight_min": -32.2608528137207,
      "activations/layer17_attention_weight_max": 58.10317611694336,
      "activations/layer17_attention_weight_min": -45.78794479370117,
      "activations/layer18_attention_weight_max": 47.69167709350586,
      "activations/layer18_attention_weight_min": -39.5816535949707,
      "activations/layer19_attention_weight_max": 23.255338668823242,
      "activations/layer19_attention_weight_min": -21.589885711669922,
      "activations/layer1_attention_weight_max": 15.33555793762207,
      "activations/layer1_attention_weight_min": -15.032167434692383,
      "activations/layer20_attention_weight_max": 23.816390991210938,
      "activations/layer20_attention_weight_min": -22.89653778076172,
      "activations/layer21_attention_weight_max": 38.811370849609375,
      "activations/layer21_attention_weight_min": -24.02766990661621,
      "activations/layer22_attention_weight_max": 29.51603889465332,
      "activations/layer22_attention_weight_min": -24.952728271484375,
      "activations/layer23_attention_weight_max": 43.454063415527344,
      "activations/layer23_attention_weight_min": -24.772703170776367,
      "activations/layer2_attention_weight_max": 29.746763229370117,
      "activations/layer2_attention_weight_min": -28.70624542236328,
      "activations/layer3_attention_weight_max": 84.37457275390625,
      "activations/layer3_attention_weight_min": -85.09993743896484,
      "activations/layer4_attention_weight_max": 84.64158630371094,
      "activations/layer4_attention_weight_min": -84.25316619873047,
      "activations/layer5_attention_weight_max": 65.38821411132812,
      "activations/layer5_attention_weight_min": -76.44866180419922,
      "activations/layer6_attention_weight_max": 50.83660125732422,
      "activations/layer6_attention_weight_min": -50.95075988769531,
      "activations/layer7_attention_weight_max": 63.79194259643555,
      "activations/layer7_attention_weight_min": -64.31434631347656,
      "activations/layer8_attention_weight_max": 48.27736282348633,
      "activations/layer8_attention_weight_min": -51.351078033447266,
      "activations/layer9_attention_weight_max": 45.70650863647461,
      "activations/layer9_attention_weight_min": -45.72800827026367,
      "epoch": 18.11,
      "learning_rate": 3.3580303030303026e-05,
      "loss": 2.7536,
      "step": 311600
    },
    {
      "activations/layer0_attention_weight_max": 15.33719539642334,
      "activations/layer0_attention_weight_min": -14.345846176147461,
      "activations/layer10_attention_weight_max": 36.0991325378418,
      "activations/layer10_attention_weight_min": -35.750030517578125,
      "activations/layer11_attention_weight_max": 33.3192253112793,
      "activations/layer11_attention_weight_min": -34.473114013671875,
      "activations/layer12_attention_weight_max": 22.526426315307617,
      "activations/layer12_attention_weight_min": -24.422433853149414,
      "activations/layer13_attention_weight_max": 37.032012939453125,
      "activations/layer13_attention_weight_min": -27.139787673950195,
      "activations/layer14_attention_weight_max": 43.3614501953125,
      "activations/layer14_attention_weight_min": -31.411754608154297,
      "activations/layer15_attention_weight_max": 40.46364974975586,
      "activations/layer15_attention_weight_min": -31.548301696777344,
      "activations/layer16_attention_weight_max": 41.15681076049805,
      "activations/layer16_attention_weight_min": -36.2929801940918,
      "activations/layer17_attention_weight_max": 62.30096435546875,
      "activations/layer17_attention_weight_min": -51.35850143432617,
      "activations/layer18_attention_weight_max": 54.54885482788086,
      "activations/layer18_attention_weight_min": -42.54894256591797,
      "activations/layer19_attention_weight_max": 27.58197021484375,
      "activations/layer19_attention_weight_min": -25.5975341796875,
      "activations/layer1_attention_weight_max": 15.664752960205078,
      "activations/layer1_attention_weight_min": -14.487419128417969,
      "activations/layer20_attention_weight_max": 27.025920867919922,
      "activations/layer20_attention_weight_min": -21.69554901123047,
      "activations/layer21_attention_weight_max": 40.88918685913086,
      "activations/layer21_attention_weight_min": -22.74812889099121,
      "activations/layer22_attention_weight_max": 35.5908317565918,
      "activations/layer22_attention_weight_min": -29.10923194885254,
      "activations/layer23_attention_weight_max": 43.172821044921875,
      "activations/layer23_attention_weight_min": -27.190044403076172,
      "activations/layer2_attention_weight_max": 31.857418060302734,
      "activations/layer2_attention_weight_min": -29.14883804321289,
      "activations/layer3_attention_weight_max": 82.03837585449219,
      "activations/layer3_attention_weight_min": -84.25730895996094,
      "activations/layer4_attention_weight_max": 81.58265686035156,
      "activations/layer4_attention_weight_min": -82.71174621582031,
      "activations/layer5_attention_weight_max": 62.76571273803711,
      "activations/layer5_attention_weight_min": -73.95172119140625,
      "activations/layer6_attention_weight_max": 48.46262741088867,
      "activations/layer6_attention_weight_min": -49.2318000793457,
      "activations/layer7_attention_weight_max": 61.70233917236328,
      "activations/layer7_attention_weight_min": -62.21974182128906,
      "activations/layer8_attention_weight_max": 47.99284744262695,
      "activations/layer8_attention_weight_min": -51.912086486816406,
      "activations/layer9_attention_weight_max": 43.30198287963867,
      "activations/layer9_attention_weight_min": -45.234771728515625,
      "epoch": 18.11,
      "learning_rate": 3.3561363636363635e-05,
      "loss": 2.7375,
      "step": 311650
    },
    {
      "activations/layer0_attention_weight_max": 14.574764251708984,
      "activations/layer0_attention_weight_min": -12.919844627380371,
      "activations/layer10_attention_weight_max": 33.632240295410156,
      "activations/layer10_attention_weight_min": -35.14352798461914,
      "activations/layer11_attention_weight_max": 32.98721694946289,
      "activations/layer11_attention_weight_min": -35.22137451171875,
      "activations/layer12_attention_weight_max": 18.40604591369629,
      "activations/layer12_attention_weight_min": -25.143539428710938,
      "activations/layer13_attention_weight_max": 30.385250091552734,
      "activations/layer13_attention_weight_min": -27.383419036865234,
      "activations/layer14_attention_weight_max": 33.32404708862305,
      "activations/layer14_attention_weight_min": -29.195894241333008,
      "activations/layer15_attention_weight_max": 30.56279754638672,
      "activations/layer15_attention_weight_min": -29.886762619018555,
      "activations/layer16_attention_weight_max": 35.02505111694336,
      "activations/layer16_attention_weight_min": -32.49878692626953,
      "activations/layer17_attention_weight_max": 53.221309661865234,
      "activations/layer17_attention_weight_min": -49.04441833496094,
      "activations/layer18_attention_weight_max": 46.83774185180664,
      "activations/layer18_attention_weight_min": -44.46324920654297,
      "activations/layer19_attention_weight_max": 22.83932113647461,
      "activations/layer19_attention_weight_min": -24.573637008666992,
      "activations/layer1_attention_weight_max": 16.165119171142578,
      "activations/layer1_attention_weight_min": -14.190296173095703,
      "activations/layer20_attention_weight_max": 22.101655960083008,
      "activations/layer20_attention_weight_min": -23.558605194091797,
      "activations/layer21_attention_weight_max": 33.323978424072266,
      "activations/layer21_attention_weight_min": -24.0517520904541,
      "activations/layer22_attention_weight_max": 29.013547897338867,
      "activations/layer22_attention_weight_min": -25.47900390625,
      "activations/layer23_attention_weight_max": 34.18634796142578,
      "activations/layer23_attention_weight_min": -25.815649032592773,
      "activations/layer2_attention_weight_max": 31.045650482177734,
      "activations/layer2_attention_weight_min": -30.371017456054688,
      "activations/layer3_attention_weight_max": 83.33877563476562,
      "activations/layer3_attention_weight_min": -87.12503051757812,
      "activations/layer4_attention_weight_max": 81.51311492919922,
      "activations/layer4_attention_weight_min": -82.38692474365234,
      "activations/layer5_attention_weight_max": 62.09565734863281,
      "activations/layer5_attention_weight_min": -72.07138061523438,
      "activations/layer6_attention_weight_max": 47.739967346191406,
      "activations/layer6_attention_weight_min": -47.02553176879883,
      "activations/layer7_attention_weight_max": 61.82587814331055,
      "activations/layer7_attention_weight_min": -59.67823791503906,
      "activations/layer8_attention_weight_max": 43.320465087890625,
      "activations/layer8_attention_weight_min": -48.2208366394043,
      "activations/layer9_attention_weight_max": 43.42161560058594,
      "activations/layer9_attention_weight_min": -44.0302848815918,
      "epoch": 18.11,
      "learning_rate": 3.354242424242424e-05,
      "loss": 2.7225,
      "step": 311700
    },
    {
      "activations/layer0_attention_weight_max": 16.230152130126953,
      "activations/layer0_attention_weight_min": -12.849994659423828,
      "activations/layer10_attention_weight_max": 37.81299591064453,
      "activations/layer10_attention_weight_min": -35.26179122924805,
      "activations/layer11_attention_weight_max": 34.58692169189453,
      "activations/layer11_attention_weight_min": -33.16241455078125,
      "activations/layer12_attention_weight_max": 19.332740783691406,
      "activations/layer12_attention_weight_min": -27.01456069946289,
      "activations/layer13_attention_weight_max": 36.12444305419922,
      "activations/layer13_attention_weight_min": -28.37880516052246,
      "activations/layer14_attention_weight_max": 36.088645935058594,
      "activations/layer14_attention_weight_min": -30.140871047973633,
      "activations/layer15_attention_weight_max": 34.682010650634766,
      "activations/layer15_attention_weight_min": -29.984975814819336,
      "activations/layer16_attention_weight_max": 32.31559753417969,
      "activations/layer16_attention_weight_min": -30.765459060668945,
      "activations/layer17_attention_weight_max": 49.05007553100586,
      "activations/layer17_attention_weight_min": -43.99196243286133,
      "activations/layer18_attention_weight_max": 48.0985107421875,
      "activations/layer18_attention_weight_min": -38.276145935058594,
      "activations/layer19_attention_weight_max": 24.850296020507812,
      "activations/layer19_attention_weight_min": -23.57706069946289,
      "activations/layer1_attention_weight_max": 15.915033340454102,
      "activations/layer1_attention_weight_min": -13.2103910446167,
      "activations/layer20_attention_weight_max": 24.58910369873047,
      "activations/layer20_attention_weight_min": -22.37285041809082,
      "activations/layer21_attention_weight_max": 36.860477447509766,
      "activations/layer21_attention_weight_min": -21.9471492767334,
      "activations/layer22_attention_weight_max": 29.24551010131836,
      "activations/layer22_attention_weight_min": -27.48987579345703,
      "activations/layer23_attention_weight_max": 36.52278137207031,
      "activations/layer23_attention_weight_min": -25.75587272644043,
      "activations/layer2_attention_weight_max": 30.539731979370117,
      "activations/layer2_attention_weight_min": -27.554113388061523,
      "activations/layer3_attention_weight_max": 79.44242858886719,
      "activations/layer3_attention_weight_min": -86.32303619384766,
      "activations/layer4_attention_weight_max": 80.48796844482422,
      "activations/layer4_attention_weight_min": -82.9209976196289,
      "activations/layer5_attention_weight_max": 64.8565673828125,
      "activations/layer5_attention_weight_min": -74.99092102050781,
      "activations/layer6_attention_weight_max": 47.78321838378906,
      "activations/layer6_attention_weight_min": -50.899497985839844,
      "activations/layer7_attention_weight_max": 62.34518051147461,
      "activations/layer7_attention_weight_min": -64.70101165771484,
      "activations/layer8_attention_weight_max": 47.67381286621094,
      "activations/layer8_attention_weight_min": -48.81866455078125,
      "activations/layer9_attention_weight_max": 45.368953704833984,
      "activations/layer9_attention_weight_min": -46.604488372802734,
      "epoch": 18.11,
      "learning_rate": 3.3523484848484844e-05,
      "loss": 2.7391,
      "step": 311750
    },
    {
      "activations/layer0_attention_weight_max": 17.699295043945312,
      "activations/layer0_attention_weight_min": -14.506046295166016,
      "activations/layer10_attention_weight_max": 37.06038284301758,
      "activations/layer10_attention_weight_min": -35.77598190307617,
      "activations/layer11_attention_weight_max": 32.28556442260742,
      "activations/layer11_attention_weight_min": -33.699623107910156,
      "activations/layer12_attention_weight_max": 18.4472713470459,
      "activations/layer12_attention_weight_min": -26.250211715698242,
      "activations/layer13_attention_weight_max": 32.6854362487793,
      "activations/layer13_attention_weight_min": -27.528718948364258,
      "activations/layer14_attention_weight_max": 33.40919876098633,
      "activations/layer14_attention_weight_min": -29.94233512878418,
      "activations/layer15_attention_weight_max": 31.450963973999023,
      "activations/layer15_attention_weight_min": -30.840686798095703,
      "activations/layer16_attention_weight_max": 31.042720794677734,
      "activations/layer16_attention_weight_min": -31.061763763427734,
      "activations/layer17_attention_weight_max": 51.31144714355469,
      "activations/layer17_attention_weight_min": -43.53914260864258,
      "activations/layer18_attention_weight_max": 44.72710418701172,
      "activations/layer18_attention_weight_min": -39.099822998046875,
      "activations/layer19_attention_weight_max": 23.106969833374023,
      "activations/layer19_attention_weight_min": -20.961944580078125,
      "activations/layer1_attention_weight_max": 16.52126121520996,
      "activations/layer1_attention_weight_min": -13.778263092041016,
      "activations/layer20_attention_weight_max": 22.265335083007812,
      "activations/layer20_attention_weight_min": -23.257946014404297,
      "activations/layer21_attention_weight_max": 33.9044303894043,
      "activations/layer21_attention_weight_min": -24.00248908996582,
      "activations/layer22_attention_weight_max": 30.57181739807129,
      "activations/layer22_attention_weight_min": -25.128803253173828,
      "activations/layer23_attention_weight_max": 35.48628616333008,
      "activations/layer23_attention_weight_min": -25.590187072753906,
      "activations/layer2_attention_weight_max": 30.240673065185547,
      "activations/layer2_attention_weight_min": -27.373435974121094,
      "activations/layer3_attention_weight_max": 83.4411392211914,
      "activations/layer3_attention_weight_min": -80.33377838134766,
      "activations/layer4_attention_weight_max": 82.1622314453125,
      "activations/layer4_attention_weight_min": -81.4967269897461,
      "activations/layer5_attention_weight_max": 61.916526794433594,
      "activations/layer5_attention_weight_min": -73.18245697021484,
      "activations/layer6_attention_weight_max": 46.61992263793945,
      "activations/layer6_attention_weight_min": -48.72285842895508,
      "activations/layer7_attention_weight_max": 65.8150405883789,
      "activations/layer7_attention_weight_min": -62.681636810302734,
      "activations/layer8_attention_weight_max": 45.046504974365234,
      "activations/layer8_attention_weight_min": -47.92505645751953,
      "activations/layer9_attention_weight_max": 44.49753189086914,
      "activations/layer9_attention_weight_min": -43.03641891479492,
      "epoch": 18.12,
      "learning_rate": 3.350454545454545e-05,
      "loss": 2.7398,
      "step": 311800
    },
    {
      "activations/layer0_attention_weight_max": 14.997221946716309,
      "activations/layer0_attention_weight_min": -12.816051483154297,
      "activations/layer10_attention_weight_max": 36.37173080444336,
      "activations/layer10_attention_weight_min": -37.443031311035156,
      "activations/layer11_attention_weight_max": 33.636192321777344,
      "activations/layer11_attention_weight_min": -33.4876708984375,
      "activations/layer12_attention_weight_max": 19.330677032470703,
      "activations/layer12_attention_weight_min": -24.04714584350586,
      "activations/layer13_attention_weight_max": 41.683589935302734,
      "activations/layer13_attention_weight_min": -29.059228897094727,
      "activations/layer14_attention_weight_max": 38.094879150390625,
      "activations/layer14_attention_weight_min": -31.753965377807617,
      "activations/layer15_attention_weight_max": 36.348487854003906,
      "activations/layer15_attention_weight_min": -30.49925422668457,
      "activations/layer16_attention_weight_max": 36.427978515625,
      "activations/layer16_attention_weight_min": -34.49850845336914,
      "activations/layer17_attention_weight_max": 57.33238983154297,
      "activations/layer17_attention_weight_min": -47.94792556762695,
      "activations/layer18_attention_weight_max": 48.33689880371094,
      "activations/layer18_attention_weight_min": -39.720726013183594,
      "activations/layer19_attention_weight_max": 24.27541160583496,
      "activations/layer19_attention_weight_min": -22.438749313354492,
      "activations/layer1_attention_weight_max": 16.28384017944336,
      "activations/layer1_attention_weight_min": -13.573545455932617,
      "activations/layer20_attention_weight_max": 21.46420669555664,
      "activations/layer20_attention_weight_min": -21.352062225341797,
      "activations/layer21_attention_weight_max": 40.67420196533203,
      "activations/layer21_attention_weight_min": -20.315427780151367,
      "activations/layer22_attention_weight_max": 29.97481346130371,
      "activations/layer22_attention_weight_min": -25.513704299926758,
      "activations/layer23_attention_weight_max": 38.11772918701172,
      "activations/layer23_attention_weight_min": -23.119970321655273,
      "activations/layer2_attention_weight_max": 31.124387741088867,
      "activations/layer2_attention_weight_min": -30.061918258666992,
      "activations/layer3_attention_weight_max": 87.072021484375,
      "activations/layer3_attention_weight_min": -88.45610046386719,
      "activations/layer4_attention_weight_max": 84.32980346679688,
      "activations/layer4_attention_weight_min": -85.97174072265625,
      "activations/layer5_attention_weight_max": 64.35610961914062,
      "activations/layer5_attention_weight_min": -75.19148254394531,
      "activations/layer6_attention_weight_max": 49.902034759521484,
      "activations/layer6_attention_weight_min": -51.486637115478516,
      "activations/layer7_attention_weight_max": 66.19718170166016,
      "activations/layer7_attention_weight_min": -66.17799377441406,
      "activations/layer8_attention_weight_max": 46.41024398803711,
      "activations/layer8_attention_weight_min": -47.7377815246582,
      "activations/layer9_attention_weight_max": 48.39740753173828,
      "activations/layer9_attention_weight_min": -45.59123992919922,
      "epoch": 18.12,
      "learning_rate": 3.3485606060606054e-05,
      "loss": 2.7375,
      "step": 311850
    },
    {
      "activations/layer0_attention_weight_max": 16.60411262512207,
      "activations/layer0_attention_weight_min": -12.586929321289062,
      "activations/layer10_attention_weight_max": 33.05572509765625,
      "activations/layer10_attention_weight_min": -35.11713409423828,
      "activations/layer11_attention_weight_max": 31.697553634643555,
      "activations/layer11_attention_weight_min": -34.24898147583008,
      "activations/layer12_attention_weight_max": 18.475339889526367,
      "activations/layer12_attention_weight_min": -25.38548469543457,
      "activations/layer13_attention_weight_max": 32.58850860595703,
      "activations/layer13_attention_weight_min": -27.433876037597656,
      "activations/layer14_attention_weight_max": 34.28329086303711,
      "activations/layer14_attention_weight_min": -29.846351623535156,
      "activations/layer15_attention_weight_max": 31.514659881591797,
      "activations/layer15_attention_weight_min": -30.668800354003906,
      "activations/layer16_attention_weight_max": 31.39060401916504,
      "activations/layer16_attention_weight_min": -29.59518051147461,
      "activations/layer17_attention_weight_max": 47.90029525756836,
      "activations/layer17_attention_weight_min": -43.95409393310547,
      "activations/layer18_attention_weight_max": 44.509220123291016,
      "activations/layer18_attention_weight_min": -40.35560607910156,
      "activations/layer19_attention_weight_max": 24.596349716186523,
      "activations/layer19_attention_weight_min": -22.851606369018555,
      "activations/layer1_attention_weight_max": 15.96731185913086,
      "activations/layer1_attention_weight_min": -14.90211009979248,
      "activations/layer20_attention_weight_max": 23.54124641418457,
      "activations/layer20_attention_weight_min": -20.723478317260742,
      "activations/layer21_attention_weight_max": 36.15861892700195,
      "activations/layer21_attention_weight_min": -22.838552474975586,
      "activations/layer22_attention_weight_max": 30.045686721801758,
      "activations/layer22_attention_weight_min": -25.782989501953125,
      "activations/layer23_attention_weight_max": 40.397769927978516,
      "activations/layer23_attention_weight_min": -22.914764404296875,
      "activations/layer2_attention_weight_max": 29.676593780517578,
      "activations/layer2_attention_weight_min": -28.023723602294922,
      "activations/layer3_attention_weight_max": 82.20381927490234,
      "activations/layer3_attention_weight_min": -83.88856506347656,
      "activations/layer4_attention_weight_max": 82.79528045654297,
      "activations/layer4_attention_weight_min": -82.79772186279297,
      "activations/layer5_attention_weight_max": 61.25989532470703,
      "activations/layer5_attention_weight_min": -77.23909759521484,
      "activations/layer6_attention_weight_max": 47.74039077758789,
      "activations/layer6_attention_weight_min": -49.1973991394043,
      "activations/layer7_attention_weight_max": 61.07088088989258,
      "activations/layer7_attention_weight_min": -63.57000732421875,
      "activations/layer8_attention_weight_max": 43.802513122558594,
      "activations/layer8_attention_weight_min": -46.8792610168457,
      "activations/layer9_attention_weight_max": 42.22062301635742,
      "activations/layer9_attention_weight_min": -45.85262680053711,
      "epoch": 18.12,
      "learning_rate": 3.346666666666666e-05,
      "loss": 2.7353,
      "step": 311900
    },
    {
      "activations/layer0_attention_weight_max": 15.003313064575195,
      "activations/layer0_attention_weight_min": -12.923221588134766,
      "activations/layer10_attention_weight_max": 41.06926345825195,
      "activations/layer10_attention_weight_min": -38.94327163696289,
      "activations/layer11_attention_weight_max": 39.68317413330078,
      "activations/layer11_attention_weight_min": -34.66484832763672,
      "activations/layer12_attention_weight_max": 17.705297470092773,
      "activations/layer12_attention_weight_min": -28.11325454711914,
      "activations/layer13_attention_weight_max": 39.29487228393555,
      "activations/layer13_attention_weight_min": -27.840471267700195,
      "activations/layer14_attention_weight_max": 46.25052261352539,
      "activations/layer14_attention_weight_min": -29.528839111328125,
      "activations/layer15_attention_weight_max": 38.51305389404297,
      "activations/layer15_attention_weight_min": -30.42057991027832,
      "activations/layer16_attention_weight_max": 35.53756332397461,
      "activations/layer16_attention_weight_min": -32.56380844116211,
      "activations/layer17_attention_weight_max": 51.456207275390625,
      "activations/layer17_attention_weight_min": -46.32286071777344,
      "activations/layer18_attention_weight_max": 51.36893081665039,
      "activations/layer18_attention_weight_min": -43.149505615234375,
      "activations/layer19_attention_weight_max": 23.806631088256836,
      "activations/layer19_attention_weight_min": -22.298681259155273,
      "activations/layer1_attention_weight_max": 16.234411239624023,
      "activations/layer1_attention_weight_min": -15.402135848999023,
      "activations/layer20_attention_weight_max": 23.22174835205078,
      "activations/layer20_attention_weight_min": -21.50223159790039,
      "activations/layer21_attention_weight_max": 34.59352493286133,
      "activations/layer21_attention_weight_min": -22.396957397460938,
      "activations/layer22_attention_weight_max": 31.26905632019043,
      "activations/layer22_attention_weight_min": -25.66075325012207,
      "activations/layer23_attention_weight_max": 34.733421325683594,
      "activations/layer23_attention_weight_min": -23.702045440673828,
      "activations/layer2_attention_weight_max": 30.241788864135742,
      "activations/layer2_attention_weight_min": -29.568870544433594,
      "activations/layer3_attention_weight_max": 84.0449447631836,
      "activations/layer3_attention_weight_min": -88.08135986328125,
      "activations/layer4_attention_weight_max": 85.16570281982422,
      "activations/layer4_attention_weight_min": -85.03260040283203,
      "activations/layer5_attention_weight_max": 63.766502380371094,
      "activations/layer5_attention_weight_min": -67.3168716430664,
      "activations/layer6_attention_weight_max": 49.071537017822266,
      "activations/layer6_attention_weight_min": -50.36173629760742,
      "activations/layer7_attention_weight_max": 72.00090789794922,
      "activations/layer7_attention_weight_min": -62.42203140258789,
      "activations/layer8_attention_weight_max": 48.97346496582031,
      "activations/layer8_attention_weight_min": -49.88764953613281,
      "activations/layer9_attention_weight_max": 54.58536911010742,
      "activations/layer9_attention_weight_min": -47.44384002685547,
      "epoch": 18.13,
      "learning_rate": 3.344772727272727e-05,
      "loss": 2.7263,
      "step": 311950
    },
    {
      "activations/layer0_attention_weight_max": 16.666501998901367,
      "activations/layer0_attention_weight_min": -13.179984092712402,
      "activations/layer10_attention_weight_max": 36.097076416015625,
      "activations/layer10_attention_weight_min": -35.4443359375,
      "activations/layer11_attention_weight_max": 32.11590576171875,
      "activations/layer11_attention_weight_min": -34.44822311401367,
      "activations/layer12_attention_weight_max": 18.3260555267334,
      "activations/layer12_attention_weight_min": -24.001462936401367,
      "activations/layer13_attention_weight_max": 35.692344665527344,
      "activations/layer13_attention_weight_min": -27.677387237548828,
      "activations/layer14_attention_weight_max": 34.94192886352539,
      "activations/layer14_attention_weight_min": -32.09001922607422,
      "activations/layer15_attention_weight_max": 34.81172180175781,
      "activations/layer15_attention_weight_min": -30.612485885620117,
      "activations/layer16_attention_weight_max": 34.76313400268555,
      "activations/layer16_attention_weight_min": -32.724395751953125,
      "activations/layer17_attention_weight_max": 52.7261962890625,
      "activations/layer17_attention_weight_min": -44.43695831298828,
      "activations/layer18_attention_weight_max": 48.84878921508789,
      "activations/layer18_attention_weight_min": -40.30692672729492,
      "activations/layer19_attention_weight_max": 24.559188842773438,
      "activations/layer19_attention_weight_min": -24.510313034057617,
      "activations/layer1_attention_weight_max": 15.67797565460205,
      "activations/layer1_attention_weight_min": -14.062705039978027,
      "activations/layer20_attention_weight_max": 23.81572151184082,
      "activations/layer20_attention_weight_min": -22.318449020385742,
      "activations/layer21_attention_weight_max": 37.923133850097656,
      "activations/layer21_attention_weight_min": -22.441402435302734,
      "activations/layer22_attention_weight_max": 30.342384338378906,
      "activations/layer22_attention_weight_min": -25.668895721435547,
      "activations/layer23_attention_weight_max": 38.186241149902344,
      "activations/layer23_attention_weight_min": -24.48989486694336,
      "activations/layer2_attention_weight_max": 30.942914962768555,
      "activations/layer2_attention_weight_min": -29.222225189208984,
      "activations/layer3_attention_weight_max": 89.64021301269531,
      "activations/layer3_attention_weight_min": -85.0418472290039,
      "activations/layer4_attention_weight_max": 87.2354507446289,
      "activations/layer4_attention_weight_min": -85.92615509033203,
      "activations/layer5_attention_weight_max": 62.02210998535156,
      "activations/layer5_attention_weight_min": -76.22724914550781,
      "activations/layer6_attention_weight_max": 48.94466781616211,
      "activations/layer6_attention_weight_min": -51.31727600097656,
      "activations/layer7_attention_weight_max": 66.24118041992188,
      "activations/layer7_attention_weight_min": -64.35000610351562,
      "activations/layer8_attention_weight_max": 47.40774917602539,
      "activations/layer8_attention_weight_min": -49.482852935791016,
      "activations/layer9_attention_weight_max": 45.12640380859375,
      "activations/layer9_attention_weight_min": -49.44100570678711,
      "epoch": 18.13,
      "learning_rate": 3.342878787878788e-05,
      "loss": 2.7319,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4769,
      "eval_samples_per_second": 506.55,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4769,
      "eval_openwebtext_samples_per_second": 506.55,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9908,
      "eval_wikitext_samples_per_second": 229.051,
      "step": 312000
    },
    {
      "epoch": 18.13,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.5479,
      "eval_lambada_samples_per_second": 509.955,
      "step": 312000
    },
    {
      "activations/layer0_attention_weight_max": 15.06645393371582,
      "activations/layer0_attention_weight_min": -13.184083938598633,
      "activations/layer10_attention_weight_max": 36.45170593261719,
      "activations/layer10_attention_weight_min": -36.169132232666016,
      "activations/layer11_attention_weight_max": 32.97248458862305,
      "activations/layer11_attention_weight_min": -35.069541931152344,
      "activations/layer12_attention_weight_max": 17.81648063659668,
      "activations/layer12_attention_weight_min": -23.621862411499023,
      "activations/layer13_attention_weight_max": 32.92612838745117,
      "activations/layer13_attention_weight_min": -29.980411529541016,
      "activations/layer14_attention_weight_max": 32.65666580200195,
      "activations/layer14_attention_weight_min": -29.278043746948242,
      "activations/layer15_attention_weight_max": 30.4323787689209,
      "activations/layer15_attention_weight_min": -31.176647186279297,
      "activations/layer16_attention_weight_max": 31.232730865478516,
      "activations/layer16_attention_weight_min": -31.752017974853516,
      "activations/layer17_attention_weight_max": 48.54640579223633,
      "activations/layer17_attention_weight_min": -45.47574234008789,
      "activations/layer18_attention_weight_max": 43.08330535888672,
      "activations/layer18_attention_weight_min": -43.09877395629883,
      "activations/layer19_attention_weight_max": 22.361629486083984,
      "activations/layer19_attention_weight_min": -22.08760643005371,
      "activations/layer1_attention_weight_max": 15.975255012512207,
      "activations/layer1_attention_weight_min": -13.157997131347656,
      "activations/layer20_attention_weight_max": 21.300701141357422,
      "activations/layer20_attention_weight_min": -22.934656143188477,
      "activations/layer21_attention_weight_max": 31.08913803100586,
      "activations/layer21_attention_weight_min": -23.780071258544922,
      "activations/layer22_attention_weight_max": 29.45856475830078,
      "activations/layer22_attention_weight_min": -29.304088592529297,
      "activations/layer23_attention_weight_max": 33.853599548339844,
      "activations/layer23_attention_weight_min": -23.821147918701172,
      "activations/layer2_attention_weight_max": 29.441213607788086,
      "activations/layer2_attention_weight_min": -29.38265037536621,
      "activations/layer3_attention_weight_max": 85.05054473876953,
      "activations/layer3_attention_weight_min": -85.84742736816406,
      "activations/layer4_attention_weight_max": 87.3974838256836,
      "activations/layer4_attention_weight_min": -85.2937240600586,
      "activations/layer5_attention_weight_max": 63.05521774291992,
      "activations/layer5_attention_weight_min": -71.37115478515625,
      "activations/layer6_attention_weight_max": 49.25737762451172,
      "activations/layer6_attention_weight_min": -51.69717788696289,
      "activations/layer7_attention_weight_max": 64.77197265625,
      "activations/layer7_attention_weight_min": -64.09473419189453,
      "activations/layer8_attention_weight_max": 48.54850769042969,
      "activations/layer8_attention_weight_min": -53.93444061279297,
      "activations/layer9_attention_weight_max": 44.970706939697266,
      "activations/layer9_attention_weight_min": -48.421085357666016,
      "epoch": 18.13,
      "learning_rate": 3.340984848484848e-05,
      "loss": 2.7228,
      "step": 312050
    },
    {
      "activations/layer0_attention_weight_max": 14.875228881835938,
      "activations/layer0_attention_weight_min": -13.20559024810791,
      "activations/layer10_attention_weight_max": 34.6448974609375,
      "activations/layer10_attention_weight_min": -34.75312423706055,
      "activations/layer11_attention_weight_max": 31.149364471435547,
      "activations/layer11_attention_weight_min": -33.24815368652344,
      "activations/layer12_attention_weight_max": 20.380361557006836,
      "activations/layer12_attention_weight_min": -25.94194221496582,
      "activations/layer13_attention_weight_max": 32.51775360107422,
      "activations/layer13_attention_weight_min": -27.991586685180664,
      "activations/layer14_attention_weight_max": 34.628639221191406,
      "activations/layer14_attention_weight_min": -32.121177673339844,
      "activations/layer15_attention_weight_max": 35.401126861572266,
      "activations/layer15_attention_weight_min": -31.0200138092041,
      "activations/layer16_attention_weight_max": 33.989166259765625,
      "activations/layer16_attention_weight_min": -31.480016708374023,
      "activations/layer17_attention_weight_max": 51.624874114990234,
      "activations/layer17_attention_weight_min": -47.27894973754883,
      "activations/layer18_attention_weight_max": 48.352325439453125,
      "activations/layer18_attention_weight_min": -40.52271270751953,
      "activations/layer19_attention_weight_max": 23.17804527282715,
      "activations/layer19_attention_weight_min": -24.366243362426758,
      "activations/layer1_attention_weight_max": 16.11837387084961,
      "activations/layer1_attention_weight_min": -13.16122055053711,
      "activations/layer20_attention_weight_max": 23.425168991088867,
      "activations/layer20_attention_weight_min": -22.263957977294922,
      "activations/layer21_attention_weight_max": 37.59705352783203,
      "activations/layer21_attention_weight_min": -22.220829010009766,
      "activations/layer22_attention_weight_max": 30.940990447998047,
      "activations/layer22_attention_weight_min": -27.06266975402832,
      "activations/layer23_attention_weight_max": 35.900917053222656,
      "activations/layer23_attention_weight_min": -25.904403686523438,
      "activations/layer2_attention_weight_max": 30.701984405517578,
      "activations/layer2_attention_weight_min": -29.157230377197266,
      "activations/layer3_attention_weight_max": 83.71578216552734,
      "activations/layer3_attention_weight_min": -81.73563385009766,
      "activations/layer4_attention_weight_max": 83.64107513427734,
      "activations/layer4_attention_weight_min": -83.14969635009766,
      "activations/layer5_attention_weight_max": 62.88825988769531,
      "activations/layer5_attention_weight_min": -69.54649353027344,
      "activations/layer6_attention_weight_max": 51.533687591552734,
      "activations/layer6_attention_weight_min": -50.45396041870117,
      "activations/layer7_attention_weight_max": 68.65103149414062,
      "activations/layer7_attention_weight_min": -61.88422393798828,
      "activations/layer8_attention_weight_max": 48.99765396118164,
      "activations/layer8_attention_weight_min": -50.6022834777832,
      "activations/layer9_attention_weight_max": 46.703792572021484,
      "activations/layer9_attention_weight_min": -44.807315826416016,
      "epoch": 18.13,
      "learning_rate": 3.339090909090909e-05,
      "loss": 2.7392,
      "step": 312100
    },
    {
      "activations/layer0_attention_weight_max": 15.257043838500977,
      "activations/layer0_attention_weight_min": -13.041481018066406,
      "activations/layer10_attention_weight_max": 35.21233367919922,
      "activations/layer10_attention_weight_min": -34.60160827636719,
      "activations/layer11_attention_weight_max": 32.100013732910156,
      "activations/layer11_attention_weight_min": -33.57088088989258,
      "activations/layer12_attention_weight_max": 19.436996459960938,
      "activations/layer12_attention_weight_min": -25.071374893188477,
      "activations/layer13_attention_weight_max": 36.413265228271484,
      "activations/layer13_attention_weight_min": -28.94122886657715,
      "activations/layer14_attention_weight_max": 39.5734748840332,
      "activations/layer14_attention_weight_min": -30.638687133789062,
      "activations/layer15_attention_weight_max": 34.644901275634766,
      "activations/layer15_attention_weight_min": -30.212793350219727,
      "activations/layer16_attention_weight_max": 35.515846252441406,
      "activations/layer16_attention_weight_min": -32.759132385253906,
      "activations/layer17_attention_weight_max": 52.916603088378906,
      "activations/layer17_attention_weight_min": -46.506141662597656,
      "activations/layer18_attention_weight_max": 48.46175765991211,
      "activations/layer18_attention_weight_min": -43.997711181640625,
      "activations/layer19_attention_weight_max": 23.2176456451416,
      "activations/layer19_attention_weight_min": -24.70679473876953,
      "activations/layer1_attention_weight_max": 16.60733413696289,
      "activations/layer1_attention_weight_min": -14.228279113769531,
      "activations/layer20_attention_weight_max": 21.183231353759766,
      "activations/layer20_attention_weight_min": -22.522647857666016,
      "activations/layer21_attention_weight_max": 36.187564849853516,
      "activations/layer21_attention_weight_min": -23.2424259185791,
      "activations/layer22_attention_weight_max": 26.963226318359375,
      "activations/layer22_attention_weight_min": -24.990882873535156,
      "activations/layer23_attention_weight_max": 35.472869873046875,
      "activations/layer23_attention_weight_min": -23.084693908691406,
      "activations/layer2_attention_weight_max": 30.175609588623047,
      "activations/layer2_attention_weight_min": -28.164470672607422,
      "activations/layer3_attention_weight_max": 85.4520492553711,
      "activations/layer3_attention_weight_min": -84.08262634277344,
      "activations/layer4_attention_weight_max": 85.64043426513672,
      "activations/layer4_attention_weight_min": -86.92647552490234,
      "activations/layer5_attention_weight_max": 60.90961456298828,
      "activations/layer5_attention_weight_min": -70.8744125366211,
      "activations/layer6_attention_weight_max": 49.98557662963867,
      "activations/layer6_attention_weight_min": -51.460426330566406,
      "activations/layer7_attention_weight_max": 62.782867431640625,
      "activations/layer7_attention_weight_min": -63.76305389404297,
      "activations/layer8_attention_weight_max": 47.87764358520508,
      "activations/layer8_attention_weight_min": -52.04218292236328,
      "activations/layer9_attention_weight_max": 43.44317626953125,
      "activations/layer9_attention_weight_min": -46.04679489135742,
      "epoch": 18.14,
      "learning_rate": 3.337196969696969e-05,
      "loss": 2.7355,
      "step": 312150
    },
    {
      "activations/layer0_attention_weight_max": 15.7606201171875,
      "activations/layer0_attention_weight_min": -13.11437702178955,
      "activations/layer10_attention_weight_max": 34.89414978027344,
      "activations/layer10_attention_weight_min": -35.46110916137695,
      "activations/layer11_attention_weight_max": 33.975807189941406,
      "activations/layer11_attention_weight_min": -33.0277099609375,
      "activations/layer12_attention_weight_max": 18.179580688476562,
      "activations/layer12_attention_weight_min": -24.27267837524414,
      "activations/layer13_attention_weight_max": 38.97749710083008,
      "activations/layer13_attention_weight_min": -27.603750228881836,
      "activations/layer14_attention_weight_max": 32.3984489440918,
      "activations/layer14_attention_weight_min": -30.912002563476562,
      "activations/layer15_attention_weight_max": 30.289813995361328,
      "activations/layer15_attention_weight_min": -29.974210739135742,
      "activations/layer16_attention_weight_max": 33.0902214050293,
      "activations/layer16_attention_weight_min": -33.31782913208008,
      "activations/layer17_attention_weight_max": 46.45187759399414,
      "activations/layer17_attention_weight_min": -45.53168869018555,
      "activations/layer18_attention_weight_max": 44.32786560058594,
      "activations/layer18_attention_weight_min": -39.274166107177734,
      "activations/layer19_attention_weight_max": 24.74791145324707,
      "activations/layer19_attention_weight_min": -22.264244079589844,
      "activations/layer1_attention_weight_max": 17.291946411132812,
      "activations/layer1_attention_weight_min": -14.329399108886719,
      "activations/layer20_attention_weight_max": 21.214494705200195,
      "activations/layer20_attention_weight_min": -23.671630859375,
      "activations/layer21_attention_weight_max": 32.31098937988281,
      "activations/layer21_attention_weight_min": -21.615297317504883,
      "activations/layer22_attention_weight_max": 26.18891716003418,
      "activations/layer22_attention_weight_min": -23.688995361328125,
      "activations/layer23_attention_weight_max": 30.813148498535156,
      "activations/layer23_attention_weight_min": -22.37232208251953,
      "activations/layer2_attention_weight_max": 30.055830001831055,
      "activations/layer2_attention_weight_min": -28.43097496032715,
      "activations/layer3_attention_weight_max": 82.58848571777344,
      "activations/layer3_attention_weight_min": -85.25323486328125,
      "activations/layer4_attention_weight_max": 82.01876831054688,
      "activations/layer4_attention_weight_min": -83.58170318603516,
      "activations/layer5_attention_weight_max": 61.68330383300781,
      "activations/layer5_attention_weight_min": -78.42919158935547,
      "activations/layer6_attention_weight_max": 47.66992950439453,
      "activations/layer6_attention_weight_min": -49.23048782348633,
      "activations/layer7_attention_weight_max": 64.03211975097656,
      "activations/layer7_attention_weight_min": -61.22814178466797,
      "activations/layer8_attention_weight_max": 43.469627380371094,
      "activations/layer8_attention_weight_min": -47.09913635253906,
      "activations/layer9_attention_weight_max": 44.050106048583984,
      "activations/layer9_attention_weight_min": -45.627197265625,
      "epoch": 18.14,
      "learning_rate": 3.33530303030303e-05,
      "loss": 2.7377,
      "step": 312200
    },
    {
      "activations/layer0_attention_weight_max": 15.267898559570312,
      "activations/layer0_attention_weight_min": -13.144822120666504,
      "activations/layer10_attention_weight_max": 36.71541213989258,
      "activations/layer10_attention_weight_min": -37.12324905395508,
      "activations/layer11_attention_weight_max": 34.770164489746094,
      "activations/layer11_attention_weight_min": -36.58900833129883,
      "activations/layer12_attention_weight_max": 20.21358871459961,
      "activations/layer12_attention_weight_min": -25.623512268066406,
      "activations/layer13_attention_weight_max": 35.22003173828125,
      "activations/layer13_attention_weight_min": -27.150102615356445,
      "activations/layer14_attention_weight_max": 36.93192672729492,
      "activations/layer14_attention_weight_min": -28.96475601196289,
      "activations/layer15_attention_weight_max": 33.732574462890625,
      "activations/layer15_attention_weight_min": -28.670021057128906,
      "activations/layer16_attention_weight_max": 33.767578125,
      "activations/layer16_attention_weight_min": -31.981538772583008,
      "activations/layer17_attention_weight_max": 53.89381790161133,
      "activations/layer17_attention_weight_min": -44.02000045776367,
      "activations/layer18_attention_weight_max": 46.06088638305664,
      "activations/layer18_attention_weight_min": -37.484039306640625,
      "activations/layer19_attention_weight_max": 23.94928550720215,
      "activations/layer19_attention_weight_min": -23.55103302001953,
      "activations/layer1_attention_weight_max": 16.55320167541504,
      "activations/layer1_attention_weight_min": -13.888371467590332,
      "activations/layer20_attention_weight_max": 22.86702537536621,
      "activations/layer20_attention_weight_min": -24.03147315979004,
      "activations/layer21_attention_weight_max": 36.798011779785156,
      "activations/layer21_attention_weight_min": -21.465538024902344,
      "activations/layer22_attention_weight_max": 28.91570281982422,
      "activations/layer22_attention_weight_min": -26.508752822875977,
      "activations/layer23_attention_weight_max": 38.97515869140625,
      "activations/layer23_attention_weight_min": -23.65427017211914,
      "activations/layer2_attention_weight_max": 30.818021774291992,
      "activations/layer2_attention_weight_min": -29.69789695739746,
      "activations/layer3_attention_weight_max": 90.06390380859375,
      "activations/layer3_attention_weight_min": -89.48805236816406,
      "activations/layer4_attention_weight_max": 88.45744323730469,
      "activations/layer4_attention_weight_min": -86.15042877197266,
      "activations/layer5_attention_weight_max": 64.8761215209961,
      "activations/layer5_attention_weight_min": -72.2274169921875,
      "activations/layer6_attention_weight_max": 53.15031433105469,
      "activations/layer6_attention_weight_min": -50.60670852661133,
      "activations/layer7_attention_weight_max": 64.16510772705078,
      "activations/layer7_attention_weight_min": -64.6445083618164,
      "activations/layer8_attention_weight_max": 46.571556091308594,
      "activations/layer8_attention_weight_min": -49.66534423828125,
      "activations/layer9_attention_weight_max": 44.33027648925781,
      "activations/layer9_attention_weight_min": -45.999908447265625,
      "epoch": 18.14,
      "learning_rate": 3.333409090909091e-05,
      "loss": 2.7445,
      "step": 312250
    },
    {
      "activations/layer0_attention_weight_max": 14.823165893554688,
      "activations/layer0_attention_weight_min": -13.045761108398438,
      "activations/layer10_attention_weight_max": 34.871185302734375,
      "activations/layer10_attention_weight_min": -36.04138946533203,
      "activations/layer11_attention_weight_max": 33.38465881347656,
      "activations/layer11_attention_weight_min": -32.64915466308594,
      "activations/layer12_attention_weight_max": 19.138153076171875,
      "activations/layer12_attention_weight_min": -28.972604751586914,
      "activations/layer13_attention_weight_max": 35.02553176879883,
      "activations/layer13_attention_weight_min": -29.449657440185547,
      "activations/layer14_attention_weight_max": 38.320796966552734,
      "activations/layer14_attention_weight_min": -29.371477127075195,
      "activations/layer15_attention_weight_max": 34.11220932006836,
      "activations/layer15_attention_weight_min": -29.367969512939453,
      "activations/layer16_attention_weight_max": 33.74977493286133,
      "activations/layer16_attention_weight_min": -29.2255916595459,
      "activations/layer17_attention_weight_max": 47.960914611816406,
      "activations/layer17_attention_weight_min": -42.628318786621094,
      "activations/layer18_attention_weight_max": 44.11917495727539,
      "activations/layer18_attention_weight_min": -37.470638275146484,
      "activations/layer19_attention_weight_max": 23.290849685668945,
      "activations/layer19_attention_weight_min": -20.516845703125,
      "activations/layer1_attention_weight_max": 15.465306282043457,
      "activations/layer1_attention_weight_min": -13.46812629699707,
      "activations/layer20_attention_weight_max": 18.90330696105957,
      "activations/layer20_attention_weight_min": -20.90508460998535,
      "activations/layer21_attention_weight_max": 32.092594146728516,
      "activations/layer21_attention_weight_min": -21.802671432495117,
      "activations/layer22_attention_weight_max": 29.675094604492188,
      "activations/layer22_attention_weight_min": -24.56916046142578,
      "activations/layer23_attention_weight_max": 36.78371047973633,
      "activations/layer23_attention_weight_min": -25.476301193237305,
      "activations/layer2_attention_weight_max": 30.023136138916016,
      "activations/layer2_attention_weight_min": -28.17107391357422,
      "activations/layer3_attention_weight_max": 83.47000885009766,
      "activations/layer3_attention_weight_min": -82.07747650146484,
      "activations/layer4_attention_weight_max": 79.74502563476562,
      "activations/layer4_attention_weight_min": -79.29134368896484,
      "activations/layer5_attention_weight_max": 60.71946716308594,
      "activations/layer5_attention_weight_min": -67.69541931152344,
      "activations/layer6_attention_weight_max": 47.30927658081055,
      "activations/layer6_attention_weight_min": -49.51643753051758,
      "activations/layer7_attention_weight_max": 59.045982360839844,
      "activations/layer7_attention_weight_min": -59.531253814697266,
      "activations/layer8_attention_weight_max": 45.9072151184082,
      "activations/layer8_attention_weight_min": -47.697471618652344,
      "activations/layer9_attention_weight_max": 40.306419372558594,
      "activations/layer9_attention_weight_min": -44.69233322143555,
      "epoch": 18.15,
      "learning_rate": 3.3315151515151515e-05,
      "loss": 2.7391,
      "step": 312300
    },
    {
      "activations/layer0_attention_weight_max": 14.759492874145508,
      "activations/layer0_attention_weight_min": -13.081298828125,
      "activations/layer10_attention_weight_max": 37.8038215637207,
      "activations/layer10_attention_weight_min": -36.885162353515625,
      "activations/layer11_attention_weight_max": 32.27537155151367,
      "activations/layer11_attention_weight_min": -32.573631286621094,
      "activations/layer12_attention_weight_max": 20.532272338867188,
      "activations/layer12_attention_weight_min": -24.994421005249023,
      "activations/layer13_attention_weight_max": 40.16217803955078,
      "activations/layer13_attention_weight_min": -28.476585388183594,
      "activations/layer14_attention_weight_max": 37.73229217529297,
      "activations/layer14_attention_weight_min": -33.201072692871094,
      "activations/layer15_attention_weight_max": 35.630592346191406,
      "activations/layer15_attention_weight_min": -32.625980377197266,
      "activations/layer16_attention_weight_max": 37.29881286621094,
      "activations/layer16_attention_weight_min": -33.300865173339844,
      "activations/layer17_attention_weight_max": 57.77592849731445,
      "activations/layer17_attention_weight_min": -48.096126556396484,
      "activations/layer18_attention_weight_max": 50.042301177978516,
      "activations/layer18_attention_weight_min": -40.58432388305664,
      "activations/layer19_attention_weight_max": 24.14311981201172,
      "activations/layer19_attention_weight_min": -22.24962043762207,
      "activations/layer1_attention_weight_max": 16.347139358520508,
      "activations/layer1_attention_weight_min": -15.811163902282715,
      "activations/layer20_attention_weight_max": 22.680105209350586,
      "activations/layer20_attention_weight_min": -23.163820266723633,
      "activations/layer21_attention_weight_max": 37.24736022949219,
      "activations/layer21_attention_weight_min": -23.455509185791016,
      "activations/layer22_attention_weight_max": 29.264936447143555,
      "activations/layer22_attention_weight_min": -24.768802642822266,
      "activations/layer23_attention_weight_max": 36.27025604248047,
      "activations/layer23_attention_weight_min": -22.923185348510742,
      "activations/layer2_attention_weight_max": 30.552570343017578,
      "activations/layer2_attention_weight_min": -29.615758895874023,
      "activations/layer3_attention_weight_max": 82.80511474609375,
      "activations/layer3_attention_weight_min": -82.82178497314453,
      "activations/layer4_attention_weight_max": 81.83074951171875,
      "activations/layer4_attention_weight_min": -81.74311828613281,
      "activations/layer5_attention_weight_max": 62.96064758300781,
      "activations/layer5_attention_weight_min": -77.77182006835938,
      "activations/layer6_attention_weight_max": 48.580726623535156,
      "activations/layer6_attention_weight_min": -48.812015533447266,
      "activations/layer7_attention_weight_max": 65.7021484375,
      "activations/layer7_attention_weight_min": -63.979278564453125,
      "activations/layer8_attention_weight_max": 49.39728546142578,
      "activations/layer8_attention_weight_min": -49.461483001708984,
      "activations/layer9_attention_weight_max": 43.579193115234375,
      "activations/layer9_attention_weight_min": -43.604888916015625,
      "epoch": 18.15,
      "learning_rate": 3.3296212121212117e-05,
      "loss": 2.7296,
      "step": 312350
    },
    {
      "activations/layer0_attention_weight_max": 15.943233489990234,
      "activations/layer0_attention_weight_min": -13.165392875671387,
      "activations/layer10_attention_weight_max": 35.78279495239258,
      "activations/layer10_attention_weight_min": -36.34593200683594,
      "activations/layer11_attention_weight_max": 34.56614303588867,
      "activations/layer11_attention_weight_min": -33.59382629394531,
      "activations/layer12_attention_weight_max": 19.430156707763672,
      "activations/layer12_attention_weight_min": -25.40508460998535,
      "activations/layer13_attention_weight_max": 36.477840423583984,
      "activations/layer13_attention_weight_min": -29.855119705200195,
      "activations/layer14_attention_weight_max": 38.084163665771484,
      "activations/layer14_attention_weight_min": -31.11990737915039,
      "activations/layer15_attention_weight_max": 37.70686721801758,
      "activations/layer15_attention_weight_min": -32.22663879394531,
      "activations/layer16_attention_weight_max": 36.288944244384766,
      "activations/layer16_attention_weight_min": -32.67462158203125,
      "activations/layer17_attention_weight_max": 54.39366912841797,
      "activations/layer17_attention_weight_min": -48.190696716308594,
      "activations/layer18_attention_weight_max": 54.29453659057617,
      "activations/layer18_attention_weight_min": -41.88165283203125,
      "activations/layer19_attention_weight_max": 33.53462600708008,
      "activations/layer19_attention_weight_min": -26.521520614624023,
      "activations/layer1_attention_weight_max": 15.097606658935547,
      "activations/layer1_attention_weight_min": -15.219315528869629,
      "activations/layer20_attention_weight_max": 33.477325439453125,
      "activations/layer20_attention_weight_min": -22.29723358154297,
      "activations/layer21_attention_weight_max": 42.129791259765625,
      "activations/layer21_attention_weight_min": -24.679166793823242,
      "activations/layer22_attention_weight_max": 35.989498138427734,
      "activations/layer22_attention_weight_min": -27.34173583984375,
      "activations/layer23_attention_weight_max": 40.42063522338867,
      "activations/layer23_attention_weight_min": -26.475202560424805,
      "activations/layer2_attention_weight_max": 32.258941650390625,
      "activations/layer2_attention_weight_min": -29.147319793701172,
      "activations/layer3_attention_weight_max": 85.73992156982422,
      "activations/layer3_attention_weight_min": -84.3407211303711,
      "activations/layer4_attention_weight_max": 87.9076919555664,
      "activations/layer4_attention_weight_min": -83.64836883544922,
      "activations/layer5_attention_weight_max": 64.59874725341797,
      "activations/layer5_attention_weight_min": -73.69698333740234,
      "activations/layer6_attention_weight_max": 52.30469512939453,
      "activations/layer6_attention_weight_min": -51.0760612487793,
      "activations/layer7_attention_weight_max": 68.14767456054688,
      "activations/layer7_attention_weight_min": -63.3479118347168,
      "activations/layer8_attention_weight_max": 45.83855438232422,
      "activations/layer8_attention_weight_min": -49.10881423950195,
      "activations/layer9_attention_weight_max": 42.8818359375,
      "activations/layer9_attention_weight_min": -44.0976676940918,
      "epoch": 18.15,
      "learning_rate": 3.3277272727272725e-05,
      "loss": 2.7191,
      "step": 312400
    },
    {
      "activations/layer0_attention_weight_max": 15.333330154418945,
      "activations/layer0_attention_weight_min": -13.125710487365723,
      "activations/layer10_attention_weight_max": 38.356529235839844,
      "activations/layer10_attention_weight_min": -36.0862922668457,
      "activations/layer11_attention_weight_max": 34.8612060546875,
      "activations/layer11_attention_weight_min": -35.03496551513672,
      "activations/layer12_attention_weight_max": 17.928436279296875,
      "activations/layer12_attention_weight_min": -31.682878494262695,
      "activations/layer13_attention_weight_max": 30.727699279785156,
      "activations/layer13_attention_weight_min": -29.6989688873291,
      "activations/layer14_attention_weight_max": 34.37094497680664,
      "activations/layer14_attention_weight_min": -30.21002197265625,
      "activations/layer15_attention_weight_max": 33.75336456298828,
      "activations/layer15_attention_weight_min": -30.171754837036133,
      "activations/layer16_attention_weight_max": 32.47217559814453,
      "activations/layer16_attention_weight_min": -32.239322662353516,
      "activations/layer17_attention_weight_max": 51.648841857910156,
      "activations/layer17_attention_weight_min": -45.235408782958984,
      "activations/layer18_attention_weight_max": 46.494102478027344,
      "activations/layer18_attention_weight_min": -40.06377029418945,
      "activations/layer19_attention_weight_max": 22.91123390197754,
      "activations/layer19_attention_weight_min": -21.829967498779297,
      "activations/layer1_attention_weight_max": 17.15825843811035,
      "activations/layer1_attention_weight_min": -15.012805938720703,
      "activations/layer20_attention_weight_max": 20.485759735107422,
      "activations/layer20_attention_weight_min": -21.340290069580078,
      "activations/layer21_attention_weight_max": 34.119781494140625,
      "activations/layer21_attention_weight_min": -24.695737838745117,
      "activations/layer22_attention_weight_max": 29.668672561645508,
      "activations/layer22_attention_weight_min": -24.863903045654297,
      "activations/layer23_attention_weight_max": 35.685298919677734,
      "activations/layer23_attention_weight_min": -23.842483520507812,
      "activations/layer2_attention_weight_max": 32.641441345214844,
      "activations/layer2_attention_weight_min": -31.819808959960938,
      "activations/layer3_attention_weight_max": 86.39459228515625,
      "activations/layer3_attention_weight_min": -89.9326400756836,
      "activations/layer4_attention_weight_max": 83.71721649169922,
      "activations/layer4_attention_weight_min": -85.84380340576172,
      "activations/layer5_attention_weight_max": 60.727699279785156,
      "activations/layer5_attention_weight_min": -74.78706359863281,
      "activations/layer6_attention_weight_max": 49.22211837768555,
      "activations/layer6_attention_weight_min": -51.51243209838867,
      "activations/layer7_attention_weight_max": 67.33192443847656,
      "activations/layer7_attention_weight_min": -61.34592056274414,
      "activations/layer8_attention_weight_max": 48.100494384765625,
      "activations/layer8_attention_weight_min": -47.514442443847656,
      "activations/layer9_attention_weight_max": 54.19294357299805,
      "activations/layer9_attention_weight_min": -43.947509765625,
      "epoch": 18.16,
      "learning_rate": 3.3258333333333326e-05,
      "loss": 2.7431,
      "step": 312450
    },
    {
      "activations/layer0_attention_weight_max": 15.576522827148438,
      "activations/layer0_attention_weight_min": -12.973310470581055,
      "activations/layer10_attention_weight_max": 36.834815979003906,
      "activations/layer10_attention_weight_min": -35.72777557373047,
      "activations/layer11_attention_weight_max": 34.115760803222656,
      "activations/layer11_attention_weight_min": -35.90940856933594,
      "activations/layer12_attention_weight_max": 18.74928092956543,
      "activations/layer12_attention_weight_min": -24.677637100219727,
      "activations/layer13_attention_weight_max": 37.11727523803711,
      "activations/layer13_attention_weight_min": -29.374155044555664,
      "activations/layer14_attention_weight_max": 37.13987731933594,
      "activations/layer14_attention_weight_min": -32.20417022705078,
      "activations/layer15_attention_weight_max": 32.860931396484375,
      "activations/layer15_attention_weight_min": -29.04990005493164,
      "activations/layer16_attention_weight_max": 33.14081573486328,
      "activations/layer16_attention_weight_min": -31.147733688354492,
      "activations/layer17_attention_weight_max": 52.79047775268555,
      "activations/layer17_attention_weight_min": -43.66278839111328,
      "activations/layer18_attention_weight_max": 48.46938705444336,
      "activations/layer18_attention_weight_min": -38.64130783081055,
      "activations/layer19_attention_weight_max": 26.258146286010742,
      "activations/layer19_attention_weight_min": -22.047531127929688,
      "activations/layer1_attention_weight_max": 14.527297019958496,
      "activations/layer1_attention_weight_min": -13.463224411010742,
      "activations/layer20_attention_weight_max": 23.19483757019043,
      "activations/layer20_attention_weight_min": -22.554590225219727,
      "activations/layer21_attention_weight_max": 36.40225601196289,
      "activations/layer21_attention_weight_min": -24.163785934448242,
      "activations/layer22_attention_weight_max": 28.707090377807617,
      "activations/layer22_attention_weight_min": -25.767223358154297,
      "activations/layer23_attention_weight_max": 34.669986724853516,
      "activations/layer23_attention_weight_min": -23.252220153808594,
      "activations/layer2_attention_weight_max": 31.288206100463867,
      "activations/layer2_attention_weight_min": -30.892234802246094,
      "activations/layer3_attention_weight_max": 93.10919189453125,
      "activations/layer3_attention_weight_min": -91.3253173828125,
      "activations/layer4_attention_weight_max": 86.65798950195312,
      "activations/layer4_attention_weight_min": -86.632080078125,
      "activations/layer5_attention_weight_max": 62.06346130371094,
      "activations/layer5_attention_weight_min": -75.658447265625,
      "activations/layer6_attention_weight_max": 52.822811126708984,
      "activations/layer6_attention_weight_min": -50.712806701660156,
      "activations/layer7_attention_weight_max": 64.98328399658203,
      "activations/layer7_attention_weight_min": -63.903690338134766,
      "activations/layer8_attention_weight_max": 48.010826110839844,
      "activations/layer8_attention_weight_min": -51.42610168457031,
      "activations/layer9_attention_weight_max": 43.80259704589844,
      "activations/layer9_attention_weight_min": -45.43504333496094,
      "epoch": 18.16,
      "learning_rate": 3.3239393939393935e-05,
      "loss": 2.7409,
      "step": 312500
    },
    {
      "activations/layer0_attention_weight_max": 14.501484870910645,
      "activations/layer0_attention_weight_min": -13.003170013427734,
      "activations/layer10_attention_weight_max": 37.57354736328125,
      "activations/layer10_attention_weight_min": -35.37054443359375,
      "activations/layer11_attention_weight_max": 36.0496826171875,
      "activations/layer11_attention_weight_min": -35.7596549987793,
      "activations/layer12_attention_weight_max": 18.918386459350586,
      "activations/layer12_attention_weight_min": -22.392290115356445,
      "activations/layer13_attention_weight_max": 36.28232955932617,
      "activations/layer13_attention_weight_min": -27.902603149414062,
      "activations/layer14_attention_weight_max": 35.30138397216797,
      "activations/layer14_attention_weight_min": -29.41032600402832,
      "activations/layer15_attention_weight_max": 32.714935302734375,
      "activations/layer15_attention_weight_min": -31.855430603027344,
      "activations/layer16_attention_weight_max": 32.13034439086914,
      "activations/layer16_attention_weight_min": -30.69013214111328,
      "activations/layer17_attention_weight_max": 48.982139587402344,
      "activations/layer17_attention_weight_min": -47.281734466552734,
      "activations/layer18_attention_weight_max": 47.957244873046875,
      "activations/layer18_attention_weight_min": -42.410099029541016,
      "activations/layer19_attention_weight_max": 25.986074447631836,
      "activations/layer19_attention_weight_min": -22.627208709716797,
      "activations/layer1_attention_weight_max": 15.512840270996094,
      "activations/layer1_attention_weight_min": -14.381321907043457,
      "activations/layer20_attention_weight_max": 23.494895935058594,
      "activations/layer20_attention_weight_min": -21.931785583496094,
      "activations/layer21_attention_weight_max": 37.51123046875,
      "activations/layer21_attention_weight_min": -21.363656997680664,
      "activations/layer22_attention_weight_max": 33.83653259277344,
      "activations/layer22_attention_weight_min": -25.111047744750977,
      "activations/layer23_attention_weight_max": 38.94786071777344,
      "activations/layer23_attention_weight_min": -24.22259521484375,
      "activations/layer2_attention_weight_max": 29.965621948242188,
      "activations/layer2_attention_weight_min": -28.92462921142578,
      "activations/layer3_attention_weight_max": 86.133056640625,
      "activations/layer3_attention_weight_min": -87.60960388183594,
      "activations/layer4_attention_weight_max": 86.09190368652344,
      "activations/layer4_attention_weight_min": -88.1202392578125,
      "activations/layer5_attention_weight_max": 69.2667465209961,
      "activations/layer5_attention_weight_min": -78.94757080078125,
      "activations/layer6_attention_weight_max": 49.21155548095703,
      "activations/layer6_attention_weight_min": -53.82292175292969,
      "activations/layer7_attention_weight_max": 63.91253662109375,
      "activations/layer7_attention_weight_min": -64.36871337890625,
      "activations/layer8_attention_weight_max": 44.839054107666016,
      "activations/layer8_attention_weight_min": -48.264739990234375,
      "activations/layer9_attention_weight_max": 43.71426010131836,
      "activations/layer9_attention_weight_min": -45.56708526611328,
      "epoch": 18.16,
      "learning_rate": 3.322045454545454e-05,
      "loss": 2.7193,
      "step": 312550
    },
    {
      "activations/layer0_attention_weight_max": 14.509346008300781,
      "activations/layer0_attention_weight_min": -13.14831829071045,
      "activations/layer10_attention_weight_max": 37.77882766723633,
      "activations/layer10_attention_weight_min": -35.86581039428711,
      "activations/layer11_attention_weight_max": 38.08833312988281,
      "activations/layer11_attention_weight_min": -34.72975540161133,
      "activations/layer12_attention_weight_max": 18.830411911010742,
      "activations/layer12_attention_weight_min": -25.371994018554688,
      "activations/layer13_attention_weight_max": 33.093170166015625,
      "activations/layer13_attention_weight_min": -27.710866928100586,
      "activations/layer14_attention_weight_max": 34.291656494140625,
      "activations/layer14_attention_weight_min": -30.53821563720703,
      "activations/layer15_attention_weight_max": 33.42338943481445,
      "activations/layer15_attention_weight_min": -28.26749610900879,
      "activations/layer16_attention_weight_max": 35.70932388305664,
      "activations/layer16_attention_weight_min": -33.42990493774414,
      "activations/layer17_attention_weight_max": 52.647499084472656,
      "activations/layer17_attention_weight_min": -47.994075775146484,
      "activations/layer18_attention_weight_max": 48.66584396362305,
      "activations/layer18_attention_weight_min": -40.6750602722168,
      "activations/layer19_attention_weight_max": 23.409486770629883,
      "activations/layer19_attention_weight_min": -24.17129898071289,
      "activations/layer1_attention_weight_max": 15.908561706542969,
      "activations/layer1_attention_weight_min": -14.1150541305542,
      "activations/layer20_attention_weight_max": 22.63334846496582,
      "activations/layer20_attention_weight_min": -19.888805389404297,
      "activations/layer21_attention_weight_max": 30.07797622680664,
      "activations/layer21_attention_weight_min": -20.6409854888916,
      "activations/layer22_attention_weight_max": 30.95478057861328,
      "activations/layer22_attention_weight_min": -27.310272216796875,
      "activations/layer23_attention_weight_max": 38.3199577331543,
      "activations/layer23_attention_weight_min": -22.64309310913086,
      "activations/layer2_attention_weight_max": 32.40309524536133,
      "activations/layer2_attention_weight_min": -30.9169979095459,
      "activations/layer3_attention_weight_max": 93.20657348632812,
      "activations/layer3_attention_weight_min": -91.78164672851562,
      "activations/layer4_attention_weight_max": 93.03260803222656,
      "activations/layer4_attention_weight_min": -86.8954086303711,
      "activations/layer5_attention_weight_max": 72.49317169189453,
      "activations/layer5_attention_weight_min": -74.16398620605469,
      "activations/layer6_attention_weight_max": 49.40812301635742,
      "activations/layer6_attention_weight_min": -50.28074264526367,
      "activations/layer7_attention_weight_max": 78.09333038330078,
      "activations/layer7_attention_weight_min": -67.35388946533203,
      "activations/layer8_attention_weight_max": 50.87789535522461,
      "activations/layer8_attention_weight_min": -49.191829681396484,
      "activations/layer9_attention_weight_max": 54.8785400390625,
      "activations/layer9_attention_weight_min": -50.85367965698242,
      "epoch": 18.16,
      "learning_rate": 3.320151515151515e-05,
      "loss": 2.7151,
      "step": 312600
    },
    {
      "activations/layer0_attention_weight_max": 16.178916931152344,
      "activations/layer0_attention_weight_min": -12.730674743652344,
      "activations/layer10_attention_weight_max": 38.18075942993164,
      "activations/layer10_attention_weight_min": -36.05286407470703,
      "activations/layer11_attention_weight_max": 33.60193634033203,
      "activations/layer11_attention_weight_min": -33.454872131347656,
      "activations/layer12_attention_weight_max": 19.38971710205078,
      "activations/layer12_attention_weight_min": -27.786178588867188,
      "activations/layer13_attention_weight_max": 33.658531188964844,
      "activations/layer13_attention_weight_min": -28.6904354095459,
      "activations/layer14_attention_weight_max": 34.982906341552734,
      "activations/layer14_attention_weight_min": -30.307682037353516,
      "activations/layer15_attention_weight_max": 32.1846923828125,
      "activations/layer15_attention_weight_min": -30.07432746887207,
      "activations/layer16_attention_weight_max": 32.85788345336914,
      "activations/layer16_attention_weight_min": -30.910327911376953,
      "activations/layer17_attention_weight_max": 50.92634582519531,
      "activations/layer17_attention_weight_min": -43.9190788269043,
      "activations/layer18_attention_weight_max": 46.216121673583984,
      "activations/layer18_attention_weight_min": -39.981201171875,
      "activations/layer19_attention_weight_max": 25.87946128845215,
      "activations/layer19_attention_weight_min": -25.147485733032227,
      "activations/layer1_attention_weight_max": 15.249113082885742,
      "activations/layer1_attention_weight_min": -14.800589561462402,
      "activations/layer20_attention_weight_max": 22.046464920043945,
      "activations/layer20_attention_weight_min": -21.623857498168945,
      "activations/layer21_attention_weight_max": 36.24571228027344,
      "activations/layer21_attention_weight_min": -23.327260971069336,
      "activations/layer22_attention_weight_max": 32.091033935546875,
      "activations/layer22_attention_weight_min": -26.38129997253418,
      "activations/layer23_attention_weight_max": 36.05048370361328,
      "activations/layer23_attention_weight_min": -22.04628562927246,
      "activations/layer2_attention_weight_max": 32.00136184692383,
      "activations/layer2_attention_weight_min": -30.772201538085938,
      "activations/layer3_attention_weight_max": 86.65090942382812,
      "activations/layer3_attention_weight_min": -90.12700653076172,
      "activations/layer4_attention_weight_max": 86.86087799072266,
      "activations/layer4_attention_weight_min": -86.21723175048828,
      "activations/layer5_attention_weight_max": 61.85346603393555,
      "activations/layer5_attention_weight_min": -70.2210922241211,
      "activations/layer6_attention_weight_max": 46.978824615478516,
      "activations/layer6_attention_weight_min": -48.711490631103516,
      "activations/layer7_attention_weight_max": 61.316650390625,
      "activations/layer7_attention_weight_min": -67.09271240234375,
      "activations/layer8_attention_weight_max": 46.62105178833008,
      "activations/layer8_attention_weight_min": -50.78816223144531,
      "activations/layer9_attention_weight_max": 44.910579681396484,
      "activations/layer9_attention_weight_min": -47.6118278503418,
      "epoch": 18.17,
      "learning_rate": 3.318257575757576e-05,
      "loss": 2.7376,
      "step": 312650
    },
    {
      "activations/layer0_attention_weight_max": 14.762710571289062,
      "activations/layer0_attention_weight_min": -12.79899787902832,
      "activations/layer10_attention_weight_max": 34.75210189819336,
      "activations/layer10_attention_weight_min": -34.21643829345703,
      "activations/layer11_attention_weight_max": 32.11835479736328,
      "activations/layer11_attention_weight_min": -32.522804260253906,
      "activations/layer12_attention_weight_max": 19.7071475982666,
      "activations/layer12_attention_weight_min": -25.600284576416016,
      "activations/layer13_attention_weight_max": 34.44915008544922,
      "activations/layer13_attention_weight_min": -28.505685806274414,
      "activations/layer14_attention_weight_max": 35.12201690673828,
      "activations/layer14_attention_weight_min": -32.46492004394531,
      "activations/layer15_attention_weight_max": 34.058990478515625,
      "activations/layer15_attention_weight_min": -30.69167709350586,
      "activations/layer16_attention_weight_max": 33.48003387451172,
      "activations/layer16_attention_weight_min": -32.34890365600586,
      "activations/layer17_attention_weight_max": 51.628807067871094,
      "activations/layer17_attention_weight_min": -45.45258712768555,
      "activations/layer18_attention_weight_max": 47.63340759277344,
      "activations/layer18_attention_weight_min": -42.0974006652832,
      "activations/layer19_attention_weight_max": 24.486248016357422,
      "activations/layer19_attention_weight_min": -22.623231887817383,
      "activations/layer1_attention_weight_max": 16.320266723632812,
      "activations/layer1_attention_weight_min": -14.183289527893066,
      "activations/layer20_attention_weight_max": 23.389299392700195,
      "activations/layer20_attention_weight_min": -22.779699325561523,
      "activations/layer21_attention_weight_max": 36.24994659423828,
      "activations/layer21_attention_weight_min": -22.320280075073242,
      "activations/layer22_attention_weight_max": 28.889638900756836,
      "activations/layer22_attention_weight_min": -28.30548667907715,
      "activations/layer23_attention_weight_max": 35.98841857910156,
      "activations/layer23_attention_weight_min": -26.18155288696289,
      "activations/layer2_attention_weight_max": 31.189456939697266,
      "activations/layer2_attention_weight_min": -30.45636749267578,
      "activations/layer3_attention_weight_max": 86.07545471191406,
      "activations/layer3_attention_weight_min": -86.81452178955078,
      "activations/layer4_attention_weight_max": 84.63214874267578,
      "activations/layer4_attention_weight_min": -88.24139404296875,
      "activations/layer5_attention_weight_max": 60.771263122558594,
      "activations/layer5_attention_weight_min": -72.69279479980469,
      "activations/layer6_attention_weight_max": 49.57333755493164,
      "activations/layer6_attention_weight_min": -50.68379211425781,
      "activations/layer7_attention_weight_max": 63.060951232910156,
      "activations/layer7_attention_weight_min": -60.12400817871094,
      "activations/layer8_attention_weight_max": 45.95160675048828,
      "activations/layer8_attention_weight_min": -48.5526237487793,
      "activations/layer9_attention_weight_max": 45.32546615600586,
      "activations/layer9_attention_weight_min": -46.92899703979492,
      "epoch": 18.17,
      "learning_rate": 3.316363636363636e-05,
      "loss": 2.7336,
      "step": 312700
    },
    {
      "activations/layer0_attention_weight_max": 14.756590843200684,
      "activations/layer0_attention_weight_min": -12.819899559020996,
      "activations/layer10_attention_weight_max": 33.6158447265625,
      "activations/layer10_attention_weight_min": -34.40867614746094,
      "activations/layer11_attention_weight_max": 30.545507431030273,
      "activations/layer11_attention_weight_min": -34.050537109375,
      "activations/layer12_attention_weight_max": 19.448251724243164,
      "activations/layer12_attention_weight_min": -27.072269439697266,
      "activations/layer13_attention_weight_max": 31.550750732421875,
      "activations/layer13_attention_weight_min": -28.770708084106445,
      "activations/layer14_attention_weight_max": 33.04949951171875,
      "activations/layer14_attention_weight_min": -29.248666763305664,
      "activations/layer15_attention_weight_max": 31.303464889526367,
      "activations/layer15_attention_weight_min": -27.98869514465332,
      "activations/layer16_attention_weight_max": 33.758907318115234,
      "activations/layer16_attention_weight_min": -32.265899658203125,
      "activations/layer17_attention_weight_max": 46.6015739440918,
      "activations/layer17_attention_weight_min": -43.657264709472656,
      "activations/layer18_attention_weight_max": 41.285404205322266,
      "activations/layer18_attention_weight_min": -38.08622360229492,
      "activations/layer19_attention_weight_max": 23.441259384155273,
      "activations/layer19_attention_weight_min": -24.337961196899414,
      "activations/layer1_attention_weight_max": 14.919349670410156,
      "activations/layer1_attention_weight_min": -13.619196891784668,
      "activations/layer20_attention_weight_max": 21.27004623413086,
      "activations/layer20_attention_weight_min": -23.51512908935547,
      "activations/layer21_attention_weight_max": 34.62635040283203,
      "activations/layer21_attention_weight_min": -23.662139892578125,
      "activations/layer22_attention_weight_max": 30.627595901489258,
      "activations/layer22_attention_weight_min": -26.298175811767578,
      "activations/layer23_attention_weight_max": 34.946144104003906,
      "activations/layer23_attention_weight_min": -27.58302879333496,
      "activations/layer2_attention_weight_max": 29.70447540283203,
      "activations/layer2_attention_weight_min": -29.089237213134766,
      "activations/layer3_attention_weight_max": 85.27042388916016,
      "activations/layer3_attention_weight_min": -84.39096069335938,
      "activations/layer4_attention_weight_max": 83.39328002929688,
      "activations/layer4_attention_weight_min": -82.18990325927734,
      "activations/layer5_attention_weight_max": 63.765541076660156,
      "activations/layer5_attention_weight_min": -73.30603790283203,
      "activations/layer6_attention_weight_max": 46.906436920166016,
      "activations/layer6_attention_weight_min": -48.27591323852539,
      "activations/layer7_attention_weight_max": 62.75375747680664,
      "activations/layer7_attention_weight_min": -60.83956527709961,
      "activations/layer8_attention_weight_max": 42.763816833496094,
      "activations/layer8_attention_weight_min": -48.58717727661133,
      "activations/layer9_attention_weight_max": 41.266544342041016,
      "activations/layer9_attention_weight_min": -44.17826843261719,
      "epoch": 18.17,
      "learning_rate": 3.314469696969697e-05,
      "loss": 2.7308,
      "step": 312750
    },
    {
      "activations/layer0_attention_weight_max": 15.4278564453125,
      "activations/layer0_attention_weight_min": -12.975968360900879,
      "activations/layer10_attention_weight_max": 35.590370178222656,
      "activations/layer10_attention_weight_min": -35.848182678222656,
      "activations/layer11_attention_weight_max": 33.525123596191406,
      "activations/layer11_attention_weight_min": -35.19915008544922,
      "activations/layer12_attention_weight_max": 19.01483726501465,
      "activations/layer12_attention_weight_min": -26.431825637817383,
      "activations/layer13_attention_weight_max": 39.91691970825195,
      "activations/layer13_attention_weight_min": -29.1193904876709,
      "activations/layer14_attention_weight_max": 37.517120361328125,
      "activations/layer14_attention_weight_min": -30.8431396484375,
      "activations/layer15_attention_weight_max": 39.78218460083008,
      "activations/layer15_attention_weight_min": -30.75926399230957,
      "activations/layer16_attention_weight_max": 38.003971099853516,
      "activations/layer16_attention_weight_min": -31.89800262451172,
      "activations/layer17_attention_weight_max": 55.35551834106445,
      "activations/layer17_attention_weight_min": -46.99341583251953,
      "activations/layer18_attention_weight_max": 49.38192367553711,
      "activations/layer18_attention_weight_min": -41.86347198486328,
      "activations/layer19_attention_weight_max": 22.113162994384766,
      "activations/layer19_attention_weight_min": -21.989633560180664,
      "activations/layer1_attention_weight_max": 16.325790405273438,
      "activations/layer1_attention_weight_min": -13.598811149597168,
      "activations/layer20_attention_weight_max": 23.33604621887207,
      "activations/layer20_attention_weight_min": -21.21080207824707,
      "activations/layer21_attention_weight_max": 40.65462112426758,
      "activations/layer21_attention_weight_min": -22.273103713989258,
      "activations/layer22_attention_weight_max": 32.87331008911133,
      "activations/layer22_attention_weight_min": -24.44060516357422,
      "activations/layer23_attention_weight_max": 35.093257904052734,
      "activations/layer23_attention_weight_min": -25.168071746826172,
      "activations/layer2_attention_weight_max": 35.532691955566406,
      "activations/layer2_attention_weight_min": -32.8991813659668,
      "activations/layer3_attention_weight_max": 91.60970306396484,
      "activations/layer3_attention_weight_min": -89.8802261352539,
      "activations/layer4_attention_weight_max": 92.447021484375,
      "activations/layer4_attention_weight_min": -92.02693176269531,
      "activations/layer5_attention_weight_max": 64.26376342773438,
      "activations/layer5_attention_weight_min": -77.8992919921875,
      "activations/layer6_attention_weight_max": 53.014366149902344,
      "activations/layer6_attention_weight_min": -53.97884750366211,
      "activations/layer7_attention_weight_max": 69.5120849609375,
      "activations/layer7_attention_weight_min": -68.30420684814453,
      "activations/layer8_attention_weight_max": 48.70896530151367,
      "activations/layer8_attention_weight_min": -49.72087478637695,
      "activations/layer9_attention_weight_max": 46.589229583740234,
      "activations/layer9_attention_weight_min": -48.55666732788086,
      "epoch": 18.18,
      "learning_rate": 3.312613636363636e-05,
      "loss": 2.7174,
      "step": 312800
    },
    {
      "activations/layer0_attention_weight_max": 14.57873821258545,
      "activations/layer0_attention_weight_min": -13.262361526489258,
      "activations/layer10_attention_weight_max": 39.149085998535156,
      "activations/layer10_attention_weight_min": -42.608211517333984,
      "activations/layer11_attention_weight_max": 37.508155822753906,
      "activations/layer11_attention_weight_min": -37.5541877746582,
      "activations/layer12_attention_weight_max": 19.93269157409668,
      "activations/layer12_attention_weight_min": -33.13732147216797,
      "activations/layer13_attention_weight_max": 33.46500778198242,
      "activations/layer13_attention_weight_min": -31.342195510864258,
      "activations/layer14_attention_weight_max": 36.245216369628906,
      "activations/layer14_attention_weight_min": -31.8367977142334,
      "activations/layer15_attention_weight_max": 34.11735534667969,
      "activations/layer15_attention_weight_min": -32.9018669128418,
      "activations/layer16_attention_weight_max": 34.86337661743164,
      "activations/layer16_attention_weight_min": -33.215641021728516,
      "activations/layer17_attention_weight_max": 51.718421936035156,
      "activations/layer17_attention_weight_min": -45.38199234008789,
      "activations/layer18_attention_weight_max": 48.022621154785156,
      "activations/layer18_attention_weight_min": -41.36883544921875,
      "activations/layer19_attention_weight_max": 23.985807418823242,
      "activations/layer19_attention_weight_min": -26.16792869567871,
      "activations/layer1_attention_weight_max": 16.129032135009766,
      "activations/layer1_attention_weight_min": -15.123620986938477,
      "activations/layer20_attention_weight_max": 22.700952529907227,
      "activations/layer20_attention_weight_min": -21.83700942993164,
      "activations/layer21_attention_weight_max": 36.727542877197266,
      "activations/layer21_attention_weight_min": -23.76793670654297,
      "activations/layer22_attention_weight_max": 30.42283821105957,
      "activations/layer22_attention_weight_min": -27.44167709350586,
      "activations/layer23_attention_weight_max": 36.38127517700195,
      "activations/layer23_attention_weight_min": -25.066936492919922,
      "activations/layer2_attention_weight_max": 32.081138610839844,
      "activations/layer2_attention_weight_min": -32.30363082885742,
      "activations/layer3_attention_weight_max": 89.84172058105469,
      "activations/layer3_attention_weight_min": -87.90364837646484,
      "activations/layer4_attention_weight_max": 91.99039459228516,
      "activations/layer4_attention_weight_min": -91.64099884033203,
      "activations/layer5_attention_weight_max": 67.3492202758789,
      "activations/layer5_attention_weight_min": -75.37248229980469,
      "activations/layer6_attention_weight_max": 52.778507232666016,
      "activations/layer6_attention_weight_min": -54.4179801940918,
      "activations/layer7_attention_weight_max": 66.87565612792969,
      "activations/layer7_attention_weight_min": -71.03050231933594,
      "activations/layer8_attention_weight_max": 52.25243377685547,
      "activations/layer8_attention_weight_min": -56.62092208862305,
      "activations/layer9_attention_weight_max": 48.68490219116211,
      "activations/layer9_attention_weight_min": -52.69558334350586,
      "epoch": 18.18,
      "learning_rate": 3.310719696969697e-05,
      "loss": 2.7204,
      "step": 312850
    },
    {
      "activations/layer0_attention_weight_max": 15.329275131225586,
      "activations/layer0_attention_weight_min": -12.908029556274414,
      "activations/layer10_attention_weight_max": 37.360225677490234,
      "activations/layer10_attention_weight_min": -38.63600540161133,
      "activations/layer11_attention_weight_max": 33.499839782714844,
      "activations/layer11_attention_weight_min": -36.95287322998047,
      "activations/layer12_attention_weight_max": 20.2464656829834,
      "activations/layer12_attention_weight_min": -22.714168548583984,
      "activations/layer13_attention_weight_max": 38.00141143798828,
      "activations/layer13_attention_weight_min": -31.659170150756836,
      "activations/layer14_attention_weight_max": 37.867549896240234,
      "activations/layer14_attention_weight_min": -32.70465087890625,
      "activations/layer15_attention_weight_max": 35.83152389526367,
      "activations/layer15_attention_weight_min": -32.641754150390625,
      "activations/layer16_attention_weight_max": 36.478240966796875,
      "activations/layer16_attention_weight_min": -33.182830810546875,
      "activations/layer17_attention_weight_max": 54.980140686035156,
      "activations/layer17_attention_weight_min": -48.543453216552734,
      "activations/layer18_attention_weight_max": 50.92307662963867,
      "activations/layer18_attention_weight_min": -42.81987380981445,
      "activations/layer19_attention_weight_max": 29.33197021484375,
      "activations/layer19_attention_weight_min": -24.646358489990234,
      "activations/layer1_attention_weight_max": 15.798463821411133,
      "activations/layer1_attention_weight_min": -14.319256782531738,
      "activations/layer20_attention_weight_max": 27.91805648803711,
      "activations/layer20_attention_weight_min": -24.35222625732422,
      "activations/layer21_attention_weight_max": 40.104576110839844,
      "activations/layer21_attention_weight_min": -26.544816970825195,
      "activations/layer22_attention_weight_max": 32.24614334106445,
      "activations/layer22_attention_weight_min": -26.448070526123047,
      "activations/layer23_attention_weight_max": 41.795570373535156,
      "activations/layer23_attention_weight_min": -25.55120849609375,
      "activations/layer2_attention_weight_max": 31.66115379333496,
      "activations/layer2_attention_weight_min": -31.53668212890625,
      "activations/layer3_attention_weight_max": 87.57799530029297,
      "activations/layer3_attention_weight_min": -89.87208557128906,
      "activations/layer4_attention_weight_max": 87.43284606933594,
      "activations/layer4_attention_weight_min": -86.20729064941406,
      "activations/layer5_attention_weight_max": 65.8114242553711,
      "activations/layer5_attention_weight_min": -76.24685668945312,
      "activations/layer6_attention_weight_max": 51.64872360229492,
      "activations/layer6_attention_weight_min": -51.33787536621094,
      "activations/layer7_attention_weight_max": 69.41791534423828,
      "activations/layer7_attention_weight_min": -65.7895736694336,
      "activations/layer8_attention_weight_max": 48.68436813354492,
      "activations/layer8_attention_weight_min": -52.55573654174805,
      "activations/layer9_attention_weight_max": 45.14194869995117,
      "activations/layer9_attention_weight_min": -47.689571380615234,
      "epoch": 18.18,
      "learning_rate": 3.308825757575757e-05,
      "loss": 2.7335,
      "step": 312900
    },
    {
      "activations/layer0_attention_weight_max": 14.910836219787598,
      "activations/layer0_attention_weight_min": -13.204822540283203,
      "activations/layer10_attention_weight_max": 35.530906677246094,
      "activations/layer10_attention_weight_min": -38.11870574951172,
      "activations/layer11_attention_weight_max": 33.65991973876953,
      "activations/layer11_attention_weight_min": -35.46773147583008,
      "activations/layer12_attention_weight_max": 20.46356201171875,
      "activations/layer12_attention_weight_min": -24.312280654907227,
      "activations/layer13_attention_weight_max": 33.53103256225586,
      "activations/layer13_attention_weight_min": -27.70212745666504,
      "activations/layer14_attention_weight_max": 36.55004119873047,
      "activations/layer14_attention_weight_min": -30.542016983032227,
      "activations/layer15_attention_weight_max": 32.76171112060547,
      "activations/layer15_attention_weight_min": -31.43015480041504,
      "activations/layer16_attention_weight_max": 33.823036193847656,
      "activations/layer16_attention_weight_min": -31.38990020751953,
      "activations/layer17_attention_weight_max": 49.50416946411133,
      "activations/layer17_attention_weight_min": -44.807861328125,
      "activations/layer18_attention_weight_max": 42.19129943847656,
      "activations/layer18_attention_weight_min": -39.508296966552734,
      "activations/layer19_attention_weight_max": 20.461759567260742,
      "activations/layer19_attention_weight_min": -21.151391983032227,
      "activations/layer1_attention_weight_max": 15.792654991149902,
      "activations/layer1_attention_weight_min": -14.78109073638916,
      "activations/layer20_attention_weight_max": 18.86045265197754,
      "activations/layer20_attention_weight_min": -22.445159912109375,
      "activations/layer21_attention_weight_max": 33.13969421386719,
      "activations/layer21_attention_weight_min": -21.21857261657715,
      "activations/layer22_attention_weight_max": 25.42222023010254,
      "activations/layer22_attention_weight_min": -25.17862892150879,
      "activations/layer23_attention_weight_max": 35.017303466796875,
      "activations/layer23_attention_weight_min": -24.430715560913086,
      "activations/layer2_attention_weight_max": 32.43379592895508,
      "activations/layer2_attention_weight_min": -32.494747161865234,
      "activations/layer3_attention_weight_max": 87.19503784179688,
      "activations/layer3_attention_weight_min": -87.64591979980469,
      "activations/layer4_attention_weight_max": 86.96446990966797,
      "activations/layer4_attention_weight_min": -86.52706146240234,
      "activations/layer5_attention_weight_max": 64.32640075683594,
      "activations/layer5_attention_weight_min": -74.54055786132812,
      "activations/layer6_attention_weight_max": 50.70293426513672,
      "activations/layer6_attention_weight_min": -52.94703674316406,
      "activations/layer7_attention_weight_max": 66.69666290283203,
      "activations/layer7_attention_weight_min": -62.804996490478516,
      "activations/layer8_attention_weight_max": 47.31719207763672,
      "activations/layer8_attention_weight_min": -51.445068359375,
      "activations/layer9_attention_weight_max": 44.02922439575195,
      "activations/layer9_attention_weight_min": -47.541481018066406,
      "epoch": 18.18,
      "learning_rate": 3.306931818181818e-05,
      "loss": 2.7284,
      "step": 312950
    },
    {
      "activations/layer0_attention_weight_max": 14.85008430480957,
      "activations/layer0_attention_weight_min": -13.029170989990234,
      "activations/layer10_attention_weight_max": 36.11048889160156,
      "activations/layer10_attention_weight_min": -36.665199279785156,
      "activations/layer11_attention_weight_max": 35.052978515625,
      "activations/layer11_attention_weight_min": -35.81064224243164,
      "activations/layer12_attention_weight_max": 21.200050354003906,
      "activations/layer12_attention_weight_min": -25.919757843017578,
      "activations/layer13_attention_weight_max": 37.34897994995117,
      "activations/layer13_attention_weight_min": -29.266910552978516,
      "activations/layer14_attention_weight_max": 38.473819732666016,
      "activations/layer14_attention_weight_min": -31.898353576660156,
      "activations/layer15_attention_weight_max": 35.13913345336914,
      "activations/layer15_attention_weight_min": -32.50727844238281,
      "activations/layer16_attention_weight_max": 35.00124740600586,
      "activations/layer16_attention_weight_min": -31.509998321533203,
      "activations/layer17_attention_weight_max": 54.84947967529297,
      "activations/layer17_attention_weight_min": -44.32859802246094,
      "activations/layer18_attention_weight_max": 54.271934509277344,
      "activations/layer18_attention_weight_min": -40.9935417175293,
      "activations/layer19_attention_weight_max": 28.426712036132812,
      "activations/layer19_attention_weight_min": -25.51796531677246,
      "activations/layer1_attention_weight_max": 15.710411071777344,
      "activations/layer1_attention_weight_min": -14.034307479858398,
      "activations/layer20_attention_weight_max": 24.256389617919922,
      "activations/layer20_attention_weight_min": -23.273237228393555,
      "activations/layer21_attention_weight_max": 35.1447639465332,
      "activations/layer21_attention_weight_min": -23.267553329467773,
      "activations/layer22_attention_weight_max": 30.452247619628906,
      "activations/layer22_attention_weight_min": -27.138025283813477,
      "activations/layer23_attention_weight_max": 41.56604766845703,
      "activations/layer23_attention_weight_min": -25.313674926757812,
      "activations/layer2_attention_weight_max": 33.35298156738281,
      "activations/layer2_attention_weight_min": -32.356040954589844,
      "activations/layer3_attention_weight_max": 90.36360931396484,
      "activations/layer3_attention_weight_min": -90.51033782958984,
      "activations/layer4_attention_weight_max": 87.9472427368164,
      "activations/layer4_attention_weight_min": -88.18280792236328,
      "activations/layer5_attention_weight_max": 63.625770568847656,
      "activations/layer5_attention_weight_min": -79.4009017944336,
      "activations/layer6_attention_weight_max": 52.518367767333984,
      "activations/layer6_attention_weight_min": -52.84895324707031,
      "activations/layer7_attention_weight_max": 65.22193908691406,
      "activations/layer7_attention_weight_min": -64.92871856689453,
      "activations/layer8_attention_weight_max": 48.49714660644531,
      "activations/layer8_attention_weight_min": -53.69273376464844,
      "activations/layer9_attention_weight_max": 45.34096908569336,
      "activations/layer9_attention_weight_min": -48.274723052978516,
      "epoch": 18.19,
      "learning_rate": 3.3050378787878786e-05,
      "loss": 2.7357,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.5103,
      "eval_samples_per_second": 504.563,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.5103,
      "eval_openwebtext_samples_per_second": 504.563,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9662,
      "eval_wikitext_samples_per_second": 231.923,
      "step": 313000
    },
    {
      "epoch": 18.19,
      "eval_lambada_loss": 2.484375,
      "eval_lambada_ppl": 11.993621898152476,
      "eval_lambada_runtime": 9.5939,
      "eval_lambada_samples_per_second": 507.509,
      "step": 313000
    },
    {
      "activations/layer0_attention_weight_max": 15.229166030883789,
      "activations/layer0_attention_weight_min": -13.049772262573242,
      "activations/layer10_attention_weight_max": 33.64223861694336,
      "activations/layer10_attention_weight_min": -34.94408416748047,
      "activations/layer11_attention_weight_max": 31.093000411987305,
      "activations/layer11_attention_weight_min": -32.905670166015625,
      "activations/layer12_attention_weight_max": 18.804269790649414,
      "activations/layer12_attention_weight_min": -27.61248207092285,
      "activations/layer13_attention_weight_max": 30.835643768310547,
      "activations/layer13_attention_weight_min": -28.113677978515625,
      "activations/layer14_attention_weight_max": 32.901084899902344,
      "activations/layer14_attention_weight_min": -31.63302230834961,
      "activations/layer15_attention_weight_max": 30.68031883239746,
      "activations/layer15_attention_weight_min": -30.90970230102539,
      "activations/layer16_attention_weight_max": 31.85044288635254,
      "activations/layer16_attention_weight_min": -30.369518280029297,
      "activations/layer17_attention_weight_max": 51.72948455810547,
      "activations/layer17_attention_weight_min": -44.31415557861328,
      "activations/layer18_attention_weight_max": 43.358272552490234,
      "activations/layer18_attention_weight_min": -39.91970443725586,
      "activations/layer19_attention_weight_max": 22.402393341064453,
      "activations/layer19_attention_weight_min": -24.700881958007812,
      "activations/layer1_attention_weight_max": 15.558548927307129,
      "activations/layer1_attention_weight_min": -12.854362487792969,
      "activations/layer20_attention_weight_max": 21.6246395111084,
      "activations/layer20_attention_weight_min": -22.48276710510254,
      "activations/layer21_attention_weight_max": 43.191097259521484,
      "activations/layer21_attention_weight_min": -31.50763511657715,
      "activations/layer22_attention_weight_max": 26.726913452148438,
      "activations/layer22_attention_weight_min": -26.023456573486328,
      "activations/layer23_attention_weight_max": 34.07133483886719,
      "activations/layer23_attention_weight_min": -24.014785766601562,
      "activations/layer2_attention_weight_max": 30.442485809326172,
      "activations/layer2_attention_weight_min": -29.96965789794922,
      "activations/layer3_attention_weight_max": 84.95921325683594,
      "activations/layer3_attention_weight_min": -86.8702621459961,
      "activations/layer4_attention_weight_max": 78.18653869628906,
      "activations/layer4_attention_weight_min": -76.51249694824219,
      "activations/layer5_attention_weight_max": 60.378273010253906,
      "activations/layer5_attention_weight_min": -66.20343780517578,
      "activations/layer6_attention_weight_max": 45.890159606933594,
      "activations/layer6_attention_weight_min": -49.53757095336914,
      "activations/layer7_attention_weight_max": 59.81947708129883,
      "activations/layer7_attention_weight_min": -59.035587310791016,
      "activations/layer8_attention_weight_max": 43.4713020324707,
      "activations/layer8_attention_weight_min": -46.8608512878418,
      "activations/layer9_attention_weight_max": 45.969322204589844,
      "activations/layer9_attention_weight_min": -43.517337799072266,
      "epoch": 18.19,
      "learning_rate": 3.3031439393939395e-05,
      "loss": 2.7351,
      "step": 313050
    },
    {
      "activations/layer0_attention_weight_max": 16.56831169128418,
      "activations/layer0_attention_weight_min": -13.125853538513184,
      "activations/layer10_attention_weight_max": 36.3930778503418,
      "activations/layer10_attention_weight_min": -35.09429931640625,
      "activations/layer11_attention_weight_max": 34.59909439086914,
      "activations/layer11_attention_weight_min": -36.078269958496094,
      "activations/layer12_attention_weight_max": 20.845291137695312,
      "activations/layer12_attention_weight_min": -26.159446716308594,
      "activations/layer13_attention_weight_max": 36.54583740234375,
      "activations/layer13_attention_weight_min": -28.0919246673584,
      "activations/layer14_attention_weight_max": 36.448455810546875,
      "activations/layer14_attention_weight_min": -31.268835067749023,
      "activations/layer15_attention_weight_max": 37.026302337646484,
      "activations/layer15_attention_weight_min": -33.120758056640625,
      "activations/layer16_attention_weight_max": 36.95089340209961,
      "activations/layer16_attention_weight_min": -34.23814392089844,
      "activations/layer17_attention_weight_max": 54.91810607910156,
      "activations/layer17_attention_weight_min": -49.06692123413086,
      "activations/layer18_attention_weight_max": 47.199275970458984,
      "activations/layer18_attention_weight_min": -41.75590515136719,
      "activations/layer19_attention_weight_max": 24.12407875061035,
      "activations/layer19_attention_weight_min": -24.680356979370117,
      "activations/layer1_attention_weight_max": 16.22517204284668,
      "activations/layer1_attention_weight_min": -13.043251037597656,
      "activations/layer20_attention_weight_max": 25.278791427612305,
      "activations/layer20_attention_weight_min": -21.125463485717773,
      "activations/layer21_attention_weight_max": 42.45857620239258,
      "activations/layer21_attention_weight_min": -21.705686569213867,
      "activations/layer22_attention_weight_max": 32.03801727294922,
      "activations/layer22_attention_weight_min": -25.385509490966797,
      "activations/layer23_attention_weight_max": 39.11895751953125,
      "activations/layer23_attention_weight_min": -25.632938385009766,
      "activations/layer2_attention_weight_max": 31.973331451416016,
      "activations/layer2_attention_weight_min": -29.93730926513672,
      "activations/layer3_attention_weight_max": 82.25098419189453,
      "activations/layer3_attention_weight_min": -85.54605865478516,
      "activations/layer4_attention_weight_max": 81.44561004638672,
      "activations/layer4_attention_weight_min": -82.18413543701172,
      "activations/layer5_attention_weight_max": 60.069637298583984,
      "activations/layer5_attention_weight_min": -68.2262191772461,
      "activations/layer6_attention_weight_max": 47.8679313659668,
      "activations/layer6_attention_weight_min": -50.635250091552734,
      "activations/layer7_attention_weight_max": 67.4505615234375,
      "activations/layer7_attention_weight_min": -64.64197540283203,
      "activations/layer8_attention_weight_max": 51.490623474121094,
      "activations/layer8_attention_weight_min": -53.61610794067383,
      "activations/layer9_attention_weight_max": 51.919673919677734,
      "activations/layer9_attention_weight_min": -49.0933952331543,
      "epoch": 18.19,
      "learning_rate": 3.3012499999999996e-05,
      "loss": 2.7337,
      "step": 313100
    },
    {
      "activations/layer0_attention_weight_max": 15.173195838928223,
      "activations/layer0_attention_weight_min": -13.021574020385742,
      "activations/layer10_attention_weight_max": 32.29221725463867,
      "activations/layer10_attention_weight_min": -33.0954704284668,
      "activations/layer11_attention_weight_max": 30.028789520263672,
      "activations/layer11_attention_weight_min": -32.135894775390625,
      "activations/layer12_attention_weight_max": 17.231550216674805,
      "activations/layer12_attention_weight_min": -31.410221099853516,
      "activations/layer13_attention_weight_max": 35.00267028808594,
      "activations/layer13_attention_weight_min": -27.233434677124023,
      "activations/layer14_attention_weight_max": 33.30949020385742,
      "activations/layer14_attention_weight_min": -29.199787139892578,
      "activations/layer15_attention_weight_max": 32.54315948486328,
      "activations/layer15_attention_weight_min": -31.06296730041504,
      "activations/layer16_attention_weight_max": 31.021678924560547,
      "activations/layer16_attention_weight_min": -29.209264755249023,
      "activations/layer17_attention_weight_max": 50.8982048034668,
      "activations/layer17_attention_weight_min": -41.286659240722656,
      "activations/layer18_attention_weight_max": 46.23094177246094,
      "activations/layer18_attention_weight_min": -36.14984893798828,
      "activations/layer19_attention_weight_max": 26.09086036682129,
      "activations/layer19_attention_weight_min": -21.50874900817871,
      "activations/layer1_attention_weight_max": 15.607569694519043,
      "activations/layer1_attention_weight_min": -12.98048210144043,
      "activations/layer20_attention_weight_max": 22.606006622314453,
      "activations/layer20_attention_weight_min": -20.82072639465332,
      "activations/layer21_attention_weight_max": 35.25197219848633,
      "activations/layer21_attention_weight_min": -23.835308074951172,
      "activations/layer22_attention_weight_max": 28.848888397216797,
      "activations/layer22_attention_weight_min": -24.48505401611328,
      "activations/layer23_attention_weight_max": 36.070560455322266,
      "activations/layer23_attention_weight_min": -24.852039337158203,
      "activations/layer2_attention_weight_max": 30.2880859375,
      "activations/layer2_attention_weight_min": -27.548093795776367,
      "activations/layer3_attention_weight_max": 82.98299407958984,
      "activations/layer3_attention_weight_min": -80.17810821533203,
      "activations/layer4_attention_weight_max": 81.7955093383789,
      "activations/layer4_attention_weight_min": -77.0787582397461,
      "activations/layer5_attention_weight_max": 62.054500579833984,
      "activations/layer5_attention_weight_min": -74.55952453613281,
      "activations/layer6_attention_weight_max": 46.49753952026367,
      "activations/layer6_attention_weight_min": -47.67976379394531,
      "activations/layer7_attention_weight_max": 65.5193099975586,
      "activations/layer7_attention_weight_min": -63.896522521972656,
      "activations/layer8_attention_weight_max": 41.953399658203125,
      "activations/layer8_attention_weight_min": -46.420799255371094,
      "activations/layer9_attention_weight_max": 40.74321746826172,
      "activations/layer9_attention_weight_min": -42.66892623901367,
      "epoch": 18.2,
      "learning_rate": 3.2993560606060604e-05,
      "loss": 2.7418,
      "step": 313150
    },
    {
      "activations/layer0_attention_weight_max": 15.357952117919922,
      "activations/layer0_attention_weight_min": -12.97876262664795,
      "activations/layer10_attention_weight_max": 32.681976318359375,
      "activations/layer10_attention_weight_min": -35.5163688659668,
      "activations/layer11_attention_weight_max": 32.13812255859375,
      "activations/layer11_attention_weight_min": -32.30221176147461,
      "activations/layer12_attention_weight_max": 17.237518310546875,
      "activations/layer12_attention_weight_min": -26.00661849975586,
      "activations/layer13_attention_weight_max": 32.199256896972656,
      "activations/layer13_attention_weight_min": -26.148645401000977,
      "activations/layer14_attention_weight_max": 33.190494537353516,
      "activations/layer14_attention_weight_min": -28.45710563659668,
      "activations/layer15_attention_weight_max": 31.392642974853516,
      "activations/layer15_attention_weight_min": -29.783828735351562,
      "activations/layer16_attention_weight_max": 30.765079498291016,
      "activations/layer16_attention_weight_min": -28.626129150390625,
      "activations/layer17_attention_weight_max": 46.5079231262207,
      "activations/layer17_attention_weight_min": -40.42189407348633,
      "activations/layer18_attention_weight_max": 42.02695846557617,
      "activations/layer18_attention_weight_min": -35.64297866821289,
      "activations/layer19_attention_weight_max": 22.715688705444336,
      "activations/layer19_attention_weight_min": -23.137746810913086,
      "activations/layer1_attention_weight_max": 16.511676788330078,
      "activations/layer1_attention_weight_min": -13.278828620910645,
      "activations/layer20_attention_weight_max": 22.912817001342773,
      "activations/layer20_attention_weight_min": -21.283071517944336,
      "activations/layer21_attention_weight_max": 36.59489440917969,
      "activations/layer21_attention_weight_min": -22.79767417907715,
      "activations/layer22_attention_weight_max": 29.342817306518555,
      "activations/layer22_attention_weight_min": -25.69411849975586,
      "activations/layer23_attention_weight_max": 35.836578369140625,
      "activations/layer23_attention_weight_min": -24.147809982299805,
      "activations/layer2_attention_weight_max": 31.12440299987793,
      "activations/layer2_attention_weight_min": -31.803306579589844,
      "activations/layer3_attention_weight_max": 85.78330993652344,
      "activations/layer3_attention_weight_min": -91.25745391845703,
      "activations/layer4_attention_weight_max": 83.05764770507812,
      "activations/layer4_attention_weight_min": -87.9704360961914,
      "activations/layer5_attention_weight_max": 61.1446647644043,
      "activations/layer5_attention_weight_min": -73.68528747558594,
      "activations/layer6_attention_weight_max": 46.676490783691406,
      "activations/layer6_attention_weight_min": -49.733665466308594,
      "activations/layer7_attention_weight_max": 60.40585708618164,
      "activations/layer7_attention_weight_min": -61.68760681152344,
      "activations/layer8_attention_weight_max": 45.58460235595703,
      "activations/layer8_attention_weight_min": -48.19094467163086,
      "activations/layer9_attention_weight_max": 42.90839767456055,
      "activations/layer9_attention_weight_min": -43.17155075073242,
      "epoch": 18.2,
      "learning_rate": 3.2974621212121206e-05,
      "loss": 2.7358,
      "step": 313200
    },
    {
      "activations/layer0_attention_weight_max": 15.040352821350098,
      "activations/layer0_attention_weight_min": -12.923391342163086,
      "activations/layer10_attention_weight_max": 38.29607009887695,
      "activations/layer10_attention_weight_min": -37.117897033691406,
      "activations/layer11_attention_weight_max": 35.014068603515625,
      "activations/layer11_attention_weight_min": -33.464752197265625,
      "activations/layer12_attention_weight_max": 18.539274215698242,
      "activations/layer12_attention_weight_min": -23.990331649780273,
      "activations/layer13_attention_weight_max": 33.22413635253906,
      "activations/layer13_attention_weight_min": -28.12728500366211,
      "activations/layer14_attention_weight_max": 32.537899017333984,
      "activations/layer14_attention_weight_min": -30.971355438232422,
      "activations/layer15_attention_weight_max": 29.98328971862793,
      "activations/layer15_attention_weight_min": -30.736156463623047,
      "activations/layer16_attention_weight_max": 31.216365814208984,
      "activations/layer16_attention_weight_min": -31.336864471435547,
      "activations/layer17_attention_weight_max": 49.86201858520508,
      "activations/layer17_attention_weight_min": -44.74007797241211,
      "activations/layer18_attention_weight_max": 48.079811096191406,
      "activations/layer18_attention_weight_min": -39.028690338134766,
      "activations/layer19_attention_weight_max": 22.550874710083008,
      "activations/layer19_attention_weight_min": -22.924510955810547,
      "activations/layer1_attention_weight_max": 15.19250202178955,
      "activations/layer1_attention_weight_min": -13.808015823364258,
      "activations/layer20_attention_weight_max": 21.26900863647461,
      "activations/layer20_attention_weight_min": -23.0885066986084,
      "activations/layer21_attention_weight_max": 29.35495376586914,
      "activations/layer21_attention_weight_min": -20.940881729125977,
      "activations/layer22_attention_weight_max": 27.58928871154785,
      "activations/layer22_attention_weight_min": -25.87047004699707,
      "activations/layer23_attention_weight_max": 33.10520553588867,
      "activations/layer23_attention_weight_min": -24.975723266601562,
      "activations/layer2_attention_weight_max": 32.446693420410156,
      "activations/layer2_attention_weight_min": -29.24384117126465,
      "activations/layer3_attention_weight_max": 86.60333251953125,
      "activations/layer3_attention_weight_min": -87.08778381347656,
      "activations/layer4_attention_weight_max": 88.33092498779297,
      "activations/layer4_attention_weight_min": -85.41532897949219,
      "activations/layer5_attention_weight_max": 66.00271606445312,
      "activations/layer5_attention_weight_min": -73.95856475830078,
      "activations/layer6_attention_weight_max": 50.275394439697266,
      "activations/layer6_attention_weight_min": -51.52117156982422,
      "activations/layer7_attention_weight_max": 64.5916976928711,
      "activations/layer7_attention_weight_min": -62.107261657714844,
      "activations/layer8_attention_weight_max": 48.37102508544922,
      "activations/layer8_attention_weight_min": -51.682472229003906,
      "activations/layer9_attention_weight_max": 45.92375564575195,
      "activations/layer9_attention_weight_min": -45.9801025390625,
      "epoch": 18.2,
      "learning_rate": 3.2955681818181814e-05,
      "loss": 2.7442,
      "step": 313250
    },
    {
      "activations/layer0_attention_weight_max": 15.798303604125977,
      "activations/layer0_attention_weight_min": -12.980973243713379,
      "activations/layer10_attention_weight_max": 34.96575927734375,
      "activations/layer10_attention_weight_min": -32.333473205566406,
      "activations/layer11_attention_weight_max": 31.892515182495117,
      "activations/layer11_attention_weight_min": -31.854145050048828,
      "activations/layer12_attention_weight_max": 18.720434188842773,
      "activations/layer12_attention_weight_min": -26.15049934387207,
      "activations/layer13_attention_weight_max": 33.15216064453125,
      "activations/layer13_attention_weight_min": -27.945396423339844,
      "activations/layer14_attention_weight_max": 36.0851936340332,
      "activations/layer14_attention_weight_min": -28.5580997467041,
      "activations/layer15_attention_weight_max": 31.80579376220703,
      "activations/layer15_attention_weight_min": -28.1696720123291,
      "activations/layer16_attention_weight_max": 31.906450271606445,
      "activations/layer16_attention_weight_min": -30.024948120117188,
      "activations/layer17_attention_weight_max": 48.29932403564453,
      "activations/layer17_attention_weight_min": -42.220802307128906,
      "activations/layer18_attention_weight_max": 43.011600494384766,
      "activations/layer18_attention_weight_min": -36.95842361450195,
      "activations/layer19_attention_weight_max": 23.177532196044922,
      "activations/layer19_attention_weight_min": -20.785846710205078,
      "activations/layer1_attention_weight_max": 17.431154251098633,
      "activations/layer1_attention_weight_min": -14.577301979064941,
      "activations/layer20_attention_weight_max": 20.46774673461914,
      "activations/layer20_attention_weight_min": -19.942428588867188,
      "activations/layer21_attention_weight_max": 32.54050064086914,
      "activations/layer21_attention_weight_min": -22.845108032226562,
      "activations/layer22_attention_weight_max": 29.215770721435547,
      "activations/layer22_attention_weight_min": -25.76203155517578,
      "activations/layer23_attention_weight_max": 37.09283447265625,
      "activations/layer23_attention_weight_min": -25.155027389526367,
      "activations/layer2_attention_weight_max": 30.53535270690918,
      "activations/layer2_attention_weight_min": -29.037212371826172,
      "activations/layer3_attention_weight_max": 85.05577087402344,
      "activations/layer3_attention_weight_min": -83.95870971679688,
      "activations/layer4_attention_weight_max": 82.8152084350586,
      "activations/layer4_attention_weight_min": -81.86478424072266,
      "activations/layer5_attention_weight_max": 58.27944564819336,
      "activations/layer5_attention_weight_min": -71.82125091552734,
      "activations/layer6_attention_weight_max": 47.82069778442383,
      "activations/layer6_attention_weight_min": -48.40848159790039,
      "activations/layer7_attention_weight_max": 58.55421447753906,
      "activations/layer7_attention_weight_min": -62.331058502197266,
      "activations/layer8_attention_weight_max": 43.246238708496094,
      "activations/layer8_attention_weight_min": -47.5693473815918,
      "activations/layer9_attention_weight_max": 40.47124481201172,
      "activations/layer9_attention_weight_min": -45.94932174682617,
      "epoch": 18.2,
      "learning_rate": 3.293674242424242e-05,
      "loss": 2.7404,
      "step": 313300
    },
    {
      "activations/layer0_attention_weight_max": 14.867143630981445,
      "activations/layer0_attention_weight_min": -12.877765655517578,
      "activations/layer10_attention_weight_max": 34.6241569519043,
      "activations/layer10_attention_weight_min": -33.39219665527344,
      "activations/layer11_attention_weight_max": 33.269447326660156,
      "activations/layer11_attention_weight_min": -33.07426452636719,
      "activations/layer12_attention_weight_max": 18.202136993408203,
      "activations/layer12_attention_weight_min": -27.739566802978516,
      "activations/layer13_attention_weight_max": 36.504608154296875,
      "activations/layer13_attention_weight_min": -27.430721282958984,
      "activations/layer14_attention_weight_max": 34.18275833129883,
      "activations/layer14_attention_weight_min": -29.480085372924805,
      "activations/layer15_attention_weight_max": 30.762399673461914,
      "activations/layer15_attention_weight_min": -29.610660552978516,
      "activations/layer16_attention_weight_max": 32.5584716796875,
      "activations/layer16_attention_weight_min": -31.82726287841797,
      "activations/layer17_attention_weight_max": 46.98697280883789,
      "activations/layer17_attention_weight_min": -42.0632438659668,
      "activations/layer18_attention_weight_max": 44.70259094238281,
      "activations/layer18_attention_weight_min": -38.837913513183594,
      "activations/layer19_attention_weight_max": 22.775917053222656,
      "activations/layer19_attention_weight_min": -23.791690826416016,
      "activations/layer1_attention_weight_max": 16.264142990112305,
      "activations/layer1_attention_weight_min": -14.413946151733398,
      "activations/layer20_attention_weight_max": 21.677152633666992,
      "activations/layer20_attention_weight_min": -22.077207565307617,
      "activations/layer21_attention_weight_max": 41.23453903198242,
      "activations/layer21_attention_weight_min": -24.00802993774414,
      "activations/layer22_attention_weight_max": 27.07364273071289,
      "activations/layer22_attention_weight_min": -25.26166534423828,
      "activations/layer23_attention_weight_max": 33.52415466308594,
      "activations/layer23_attention_weight_min": -22.767009735107422,
      "activations/layer2_attention_weight_max": 30.621244430541992,
      "activations/layer2_attention_weight_min": -30.021528244018555,
      "activations/layer3_attention_weight_max": 79.67972564697266,
      "activations/layer3_attention_weight_min": -86.03112030029297,
      "activations/layer4_attention_weight_max": 82.07952880859375,
      "activations/layer4_attention_weight_min": -81.18157196044922,
      "activations/layer5_attention_weight_max": 61.14778137207031,
      "activations/layer5_attention_weight_min": -69.2032470703125,
      "activations/layer6_attention_weight_max": 49.01597595214844,
      "activations/layer6_attention_weight_min": -46.50470733642578,
      "activations/layer7_attention_weight_max": 60.22337341308594,
      "activations/layer7_attention_weight_min": -59.098731994628906,
      "activations/layer8_attention_weight_max": 43.729156494140625,
      "activations/layer8_attention_weight_min": -44.37806701660156,
      "activations/layer9_attention_weight_max": 42.46076583862305,
      "activations/layer9_attention_weight_min": -44.616249084472656,
      "epoch": 18.21,
      "learning_rate": 3.291780303030303e-05,
      "loss": 2.725,
      "step": 313350
    },
    {
      "activations/layer0_attention_weight_max": 14.457447052001953,
      "activations/layer0_attention_weight_min": -13.133953094482422,
      "activations/layer10_attention_weight_max": 37.035728454589844,
      "activations/layer10_attention_weight_min": -37.818458557128906,
      "activations/layer11_attention_weight_max": 34.086692810058594,
      "activations/layer11_attention_weight_min": -34.29368591308594,
      "activations/layer12_attention_weight_max": 22.076950073242188,
      "activations/layer12_attention_weight_min": -23.880247116088867,
      "activations/layer13_attention_weight_max": 36.01026153564453,
      "activations/layer13_attention_weight_min": -29.74742889404297,
      "activations/layer14_attention_weight_max": 34.24055099487305,
      "activations/layer14_attention_weight_min": -27.629745483398438,
      "activations/layer15_attention_weight_max": 32.195899963378906,
      "activations/layer15_attention_weight_min": -28.9296817779541,
      "activations/layer16_attention_weight_max": 34.491458892822266,
      "activations/layer16_attention_weight_min": -29.674291610717773,
      "activations/layer17_attention_weight_max": 50.16383743286133,
      "activations/layer17_attention_weight_min": -42.458457946777344,
      "activations/layer18_attention_weight_max": 47.21562957763672,
      "activations/layer18_attention_weight_min": -36.674537658691406,
      "activations/layer19_attention_weight_max": 27.683897018432617,
      "activations/layer19_attention_weight_min": -23.185688018798828,
      "activations/layer1_attention_weight_max": 15.778407096862793,
      "activations/layer1_attention_weight_min": -13.532635688781738,
      "activations/layer20_attention_weight_max": 25.4200439453125,
      "activations/layer20_attention_weight_min": -21.21610450744629,
      "activations/layer21_attention_weight_max": 39.563907623291016,
      "activations/layer21_attention_weight_min": -24.66794776916504,
      "activations/layer22_attention_weight_max": 28.04741668701172,
      "activations/layer22_attention_weight_min": -25.76985740661621,
      "activations/layer23_attention_weight_max": 36.02205276489258,
      "activations/layer23_attention_weight_min": -24.451210021972656,
      "activations/layer2_attention_weight_max": 31.43549919128418,
      "activations/layer2_attention_weight_min": -30.514881134033203,
      "activations/layer3_attention_weight_max": 88.84502410888672,
      "activations/layer3_attention_weight_min": -93.66059112548828,
      "activations/layer4_attention_weight_max": 86.1966323852539,
      "activations/layer4_attention_weight_min": -87.45015716552734,
      "activations/layer5_attention_weight_max": 63.06142807006836,
      "activations/layer5_attention_weight_min": -70.62126159667969,
      "activations/layer6_attention_weight_max": 51.04111862182617,
      "activations/layer6_attention_weight_min": -48.68653869628906,
      "activations/layer7_attention_weight_max": 62.901153564453125,
      "activations/layer7_attention_weight_min": -61.38672637939453,
      "activations/layer8_attention_weight_max": 46.236297607421875,
      "activations/layer8_attention_weight_min": -47.49806213378906,
      "activations/layer9_attention_weight_max": 43.32310104370117,
      "activations/layer9_attention_weight_min": -46.613731384277344,
      "epoch": 18.21,
      "learning_rate": 3.289886363636363e-05,
      "loss": 2.7166,
      "step": 313400
    },
    {
      "activations/layer0_attention_weight_max": 17.064912796020508,
      "activations/layer0_attention_weight_min": -13.03817081451416,
      "activations/layer10_attention_weight_max": 35.67314529418945,
      "activations/layer10_attention_weight_min": -36.73118591308594,
      "activations/layer11_attention_weight_max": 32.448036193847656,
      "activations/layer11_attention_weight_min": -32.81242752075195,
      "activations/layer12_attention_weight_max": 19.847888946533203,
      "activations/layer12_attention_weight_min": -25.99439811706543,
      "activations/layer13_attention_weight_max": 33.00742721557617,
      "activations/layer13_attention_weight_min": -27.522680282592773,
      "activations/layer14_attention_weight_max": 34.24359130859375,
      "activations/layer14_attention_weight_min": -29.080907821655273,
      "activations/layer15_attention_weight_max": 30.461002349853516,
      "activations/layer15_attention_weight_min": -28.944303512573242,
      "activations/layer16_attention_weight_max": 34.97146987915039,
      "activations/layer16_attention_weight_min": -31.82054901123047,
      "activations/layer17_attention_weight_max": 51.73006820678711,
      "activations/layer17_attention_weight_min": -45.39604568481445,
      "activations/layer18_attention_weight_max": 47.08698272705078,
      "activations/layer18_attention_weight_min": -38.61139678955078,
      "activations/layer19_attention_weight_max": 22.85776710510254,
      "activations/layer19_attention_weight_min": -23.721059799194336,
      "activations/layer1_attention_weight_max": 15.21586799621582,
      "activations/layer1_attention_weight_min": -13.449756622314453,
      "activations/layer20_attention_weight_max": 21.16608238220215,
      "activations/layer20_attention_weight_min": -22.21549415588379,
      "activations/layer21_attention_weight_max": 37.521018981933594,
      "activations/layer21_attention_weight_min": -23.09446907043457,
      "activations/layer22_attention_weight_max": 27.14480209350586,
      "activations/layer22_attention_weight_min": -25.723270416259766,
      "activations/layer23_attention_weight_max": 36.25757598876953,
      "activations/layer23_attention_weight_min": -23.097166061401367,
      "activations/layer2_attention_weight_max": 31.836963653564453,
      "activations/layer2_attention_weight_min": -28.764480590820312,
      "activations/layer3_attention_weight_max": 84.38235473632812,
      "activations/layer3_attention_weight_min": -84.15708923339844,
      "activations/layer4_attention_weight_max": 82.7142562866211,
      "activations/layer4_attention_weight_min": -83.5191421508789,
      "activations/layer5_attention_weight_max": 67.75213623046875,
      "activations/layer5_attention_weight_min": -69.64942932128906,
      "activations/layer6_attention_weight_max": 53.74191665649414,
      "activations/layer6_attention_weight_min": -48.74711227416992,
      "activations/layer7_attention_weight_max": 65.07518005371094,
      "activations/layer7_attention_weight_min": -61.79958724975586,
      "activations/layer8_attention_weight_max": 46.83870315551758,
      "activations/layer8_attention_weight_min": -49.111572265625,
      "activations/layer9_attention_weight_max": 42.72770309448242,
      "activations/layer9_attention_weight_min": -43.99759292602539,
      "epoch": 18.21,
      "learning_rate": 3.287992424242424e-05,
      "loss": 2.7341,
      "step": 313450
    },
    {
      "activations/layer0_attention_weight_max": 15.211179733276367,
      "activations/layer0_attention_weight_min": -12.627243995666504,
      "activations/layer10_attention_weight_max": 37.45698928833008,
      "activations/layer10_attention_weight_min": -38.22648620605469,
      "activations/layer11_attention_weight_max": 32.982810974121094,
      "activations/layer11_attention_weight_min": -36.38951110839844,
      "activations/layer12_attention_weight_max": 18.927526473999023,
      "activations/layer12_attention_weight_min": -23.854162216186523,
      "activations/layer13_attention_weight_max": 45.22046661376953,
      "activations/layer13_attention_weight_min": -33.168338775634766,
      "activations/layer14_attention_weight_max": 36.024410247802734,
      "activations/layer14_attention_weight_min": -32.71891403198242,
      "activations/layer15_attention_weight_max": 34.529441833496094,
      "activations/layer15_attention_weight_min": -33.5746955871582,
      "activations/layer16_attention_weight_max": 35.90034866333008,
      "activations/layer16_attention_weight_min": -34.51435470581055,
      "activations/layer17_attention_weight_max": 53.28334045410156,
      "activations/layer17_attention_weight_min": -47.7909049987793,
      "activations/layer18_attention_weight_max": 46.13401794433594,
      "activations/layer18_attention_weight_min": -42.65147018432617,
      "activations/layer19_attention_weight_max": 25.24342918395996,
      "activations/layer19_attention_weight_min": -22.880516052246094,
      "activations/layer1_attention_weight_max": 16.258304595947266,
      "activations/layer1_attention_weight_min": -13.265560150146484,
      "activations/layer20_attention_weight_max": 21.464599609375,
      "activations/layer20_attention_weight_min": -22.708635330200195,
      "activations/layer21_attention_weight_max": 33.87051773071289,
      "activations/layer21_attention_weight_min": -23.19533348083496,
      "activations/layer22_attention_weight_max": 30.76352310180664,
      "activations/layer22_attention_weight_min": -26.5494327545166,
      "activations/layer23_attention_weight_max": 37.156734466552734,
      "activations/layer23_attention_weight_min": -26.266998291015625,
      "activations/layer2_attention_weight_max": 30.528356552124023,
      "activations/layer2_attention_weight_min": -30.33099365234375,
      "activations/layer3_attention_weight_max": 84.64225006103516,
      "activations/layer3_attention_weight_min": -85.16731262207031,
      "activations/layer4_attention_weight_max": 87.89685821533203,
      "activations/layer4_attention_weight_min": -85.5025634765625,
      "activations/layer5_attention_weight_max": 65.86222839355469,
      "activations/layer5_attention_weight_min": -76.37625122070312,
      "activations/layer6_attention_weight_max": 53.04723358154297,
      "activations/layer6_attention_weight_min": -53.99985122680664,
      "activations/layer7_attention_weight_max": 65.74335479736328,
      "activations/layer7_attention_weight_min": -71.85216522216797,
      "activations/layer8_attention_weight_max": 48.524681091308594,
      "activations/layer8_attention_weight_min": -51.93039321899414,
      "activations/layer9_attention_weight_max": 48.46091842651367,
      "activations/layer9_attention_weight_min": -50.44869613647461,
      "epoch": 18.22,
      "learning_rate": 3.286098484848485e-05,
      "loss": 2.7423,
      "step": 313500
    },
    {
      "activations/layer0_attention_weight_max": 15.07611083984375,
      "activations/layer0_attention_weight_min": -13.688064575195312,
      "activations/layer10_attention_weight_max": 35.706214904785156,
      "activations/layer10_attention_weight_min": -36.2920036315918,
      "activations/layer11_attention_weight_max": 34.75828552246094,
      "activations/layer11_attention_weight_min": -34.33836364746094,
      "activations/layer12_attention_weight_max": 19.064706802368164,
      "activations/layer12_attention_weight_min": -26.092761993408203,
      "activations/layer13_attention_weight_max": 35.63644027709961,
      "activations/layer13_attention_weight_min": -28.684534072875977,
      "activations/layer14_attention_weight_max": 36.049095153808594,
      "activations/layer14_attention_weight_min": -32.681610107421875,
      "activations/layer15_attention_weight_max": 33.38698959350586,
      "activations/layer15_attention_weight_min": -30.661102294921875,
      "activations/layer16_attention_weight_max": 34.966758728027344,
      "activations/layer16_attention_weight_min": -34.075172424316406,
      "activations/layer17_attention_weight_max": 54.564674377441406,
      "activations/layer17_attention_weight_min": -47.90150833129883,
      "activations/layer18_attention_weight_max": 50.53897476196289,
      "activations/layer18_attention_weight_min": -45.24907684326172,
      "activations/layer19_attention_weight_max": 25.484481811523438,
      "activations/layer19_attention_weight_min": -23.325319290161133,
      "activations/layer1_attention_weight_max": 15.667210578918457,
      "activations/layer1_attention_weight_min": -13.895575523376465,
      "activations/layer20_attention_weight_max": 23.63188934326172,
      "activations/layer20_attention_weight_min": -23.550065994262695,
      "activations/layer21_attention_weight_max": 38.26404571533203,
      "activations/layer21_attention_weight_min": -23.867008209228516,
      "activations/layer22_attention_weight_max": 31.56081771850586,
      "activations/layer22_attention_weight_min": -28.789947509765625,
      "activations/layer23_attention_weight_max": 34.199195861816406,
      "activations/layer23_attention_weight_min": -25.286962509155273,
      "activations/layer2_attention_weight_max": 31.385486602783203,
      "activations/layer2_attention_weight_min": -29.53383445739746,
      "activations/layer3_attention_weight_max": 85.30992889404297,
      "activations/layer3_attention_weight_min": -84.49835968017578,
      "activations/layer4_attention_weight_max": 83.69998168945312,
      "activations/layer4_attention_weight_min": -82.30181884765625,
      "activations/layer5_attention_weight_max": 63.33695983886719,
      "activations/layer5_attention_weight_min": -76.80250549316406,
      "activations/layer6_attention_weight_max": 49.792789459228516,
      "activations/layer6_attention_weight_min": -50.55546951293945,
      "activations/layer7_attention_weight_max": 68.23085021972656,
      "activations/layer7_attention_weight_min": -67.9830551147461,
      "activations/layer8_attention_weight_max": 48.146785736083984,
      "activations/layer8_attention_weight_min": -48.24414825439453,
      "activations/layer9_attention_weight_max": 48.132625579833984,
      "activations/layer9_attention_weight_min": -46.43015670776367,
      "epoch": 18.22,
      "learning_rate": 3.284204545454545e-05,
      "loss": 2.7274,
      "step": 313550
    },
    {
      "activations/layer0_attention_weight_max": 14.658661842346191,
      "activations/layer0_attention_weight_min": -12.83249282836914,
      "activations/layer10_attention_weight_max": 34.38077926635742,
      "activations/layer10_attention_weight_min": -35.98027420043945,
      "activations/layer11_attention_weight_max": 33.324951171875,
      "activations/layer11_attention_weight_min": -36.824188232421875,
      "activations/layer12_attention_weight_max": 18.945398330688477,
      "activations/layer12_attention_weight_min": -22.228958129882812,
      "activations/layer13_attention_weight_max": 34.50953674316406,
      "activations/layer13_attention_weight_min": -27.98320198059082,
      "activations/layer14_attention_weight_max": 35.705894470214844,
      "activations/layer14_attention_weight_min": -31.679563522338867,
      "activations/layer15_attention_weight_max": 33.873111724853516,
      "activations/layer15_attention_weight_min": -31.264511108398438,
      "activations/layer16_attention_weight_max": 36.39134216308594,
      "activations/layer16_attention_weight_min": -34.11442565917969,
      "activations/layer17_attention_weight_max": 52.91388702392578,
      "activations/layer17_attention_weight_min": -46.31203079223633,
      "activations/layer18_attention_weight_max": 48.390235900878906,
      "activations/layer18_attention_weight_min": -45.08742904663086,
      "activations/layer19_attention_weight_max": 24.69507598876953,
      "activations/layer19_attention_weight_min": -22.149063110351562,
      "activations/layer1_attention_weight_max": 15.219075202941895,
      "activations/layer1_attention_weight_min": -13.02347469329834,
      "activations/layer20_attention_weight_max": 23.322946548461914,
      "activations/layer20_attention_weight_min": -22.453523635864258,
      "activations/layer21_attention_weight_max": 36.08122634887695,
      "activations/layer21_attention_weight_min": -21.33851432800293,
      "activations/layer22_attention_weight_max": 29.18639373779297,
      "activations/layer22_attention_weight_min": -24.914527893066406,
      "activations/layer23_attention_weight_max": 40.56672668457031,
      "activations/layer23_attention_weight_min": -24.656360626220703,
      "activations/layer2_attention_weight_max": 30.389684677124023,
      "activations/layer2_attention_weight_min": -31.06661033630371,
      "activations/layer3_attention_weight_max": 88.53987884521484,
      "activations/layer3_attention_weight_min": -89.86933898925781,
      "activations/layer4_attention_weight_max": 90.24398040771484,
      "activations/layer4_attention_weight_min": -88.12081909179688,
      "activations/layer5_attention_weight_max": 64.57588195800781,
      "activations/layer5_attention_weight_min": -73.13362121582031,
      "activations/layer6_attention_weight_max": 51.314369201660156,
      "activations/layer6_attention_weight_min": -52.81032943725586,
      "activations/layer7_attention_weight_max": 65.87322235107422,
      "activations/layer7_attention_weight_min": -63.90560531616211,
      "activations/layer8_attention_weight_max": 45.61278533935547,
      "activations/layer8_attention_weight_min": -48.21123123168945,
      "activations/layer9_attention_weight_max": 42.236568450927734,
      "activations/layer9_attention_weight_min": -47.36478805541992,
      "epoch": 18.22,
      "learning_rate": 3.282310606060606e-05,
      "loss": 2.7434,
      "step": 313600
    },
    {
      "activations/layer0_attention_weight_max": 14.801000595092773,
      "activations/layer0_attention_weight_min": -13.155014038085938,
      "activations/layer10_attention_weight_max": 35.18484878540039,
      "activations/layer10_attention_weight_min": -35.00986099243164,
      "activations/layer11_attention_weight_max": 31.944379806518555,
      "activations/layer11_attention_weight_min": -34.21749496459961,
      "activations/layer12_attention_weight_max": 19.11557388305664,
      "activations/layer12_attention_weight_min": -25.249555587768555,
      "activations/layer13_attention_weight_max": 33.483543395996094,
      "activations/layer13_attention_weight_min": -26.00380516052246,
      "activations/layer14_attention_weight_max": 33.83024597167969,
      "activations/layer14_attention_weight_min": -29.216066360473633,
      "activations/layer15_attention_weight_max": 34.312400817871094,
      "activations/layer15_attention_weight_min": -28.94312286376953,
      "activations/layer16_attention_weight_max": 34.8078498840332,
      "activations/layer16_attention_weight_min": -32.288578033447266,
      "activations/layer17_attention_weight_max": 52.636966705322266,
      "activations/layer17_attention_weight_min": -46.72854232788086,
      "activations/layer18_attention_weight_max": 46.31770324707031,
      "activations/layer18_attention_weight_min": -41.52412033081055,
      "activations/layer19_attention_weight_max": 23.71131706237793,
      "activations/layer19_attention_weight_min": -22.757108688354492,
      "activations/layer1_attention_weight_max": 16.493833541870117,
      "activations/layer1_attention_weight_min": -13.399125099182129,
      "activations/layer20_attention_weight_max": 23.52613639831543,
      "activations/layer20_attention_weight_min": -22.07902717590332,
      "activations/layer21_attention_weight_max": 43.29182434082031,
      "activations/layer21_attention_weight_min": -25.652599334716797,
      "activations/layer22_attention_weight_max": 28.872224807739258,
      "activations/layer22_attention_weight_min": -24.888282775878906,
      "activations/layer23_attention_weight_max": 36.45091247558594,
      "activations/layer23_attention_weight_min": -25.685623168945312,
      "activations/layer2_attention_weight_max": 30.791065216064453,
      "activations/layer2_attention_weight_min": -29.46005630493164,
      "activations/layer3_attention_weight_max": 86.0031967163086,
      "activations/layer3_attention_weight_min": -84.66580963134766,
      "activations/layer4_attention_weight_max": 84.45232391357422,
      "activations/layer4_attention_weight_min": -81.04844665527344,
      "activations/layer5_attention_weight_max": 64.26011657714844,
      "activations/layer5_attention_weight_min": -71.07943725585938,
      "activations/layer6_attention_weight_max": 50.7673454284668,
      "activations/layer6_attention_weight_min": -50.04792022705078,
      "activations/layer7_attention_weight_max": 63.28714370727539,
      "activations/layer7_attention_weight_min": -60.66902160644531,
      "activations/layer8_attention_weight_max": 47.0793571472168,
      "activations/layer8_attention_weight_min": -49.30449295043945,
      "activations/layer9_attention_weight_max": 43.5228385925293,
      "activations/layer9_attention_weight_min": -43.7461051940918,
      "epoch": 18.22,
      "learning_rate": 3.280416666666667e-05,
      "loss": 2.7292,
      "step": 313650
    },
    {
      "activations/layer0_attention_weight_max": 14.8086519241333,
      "activations/layer0_attention_weight_min": -12.944411277770996,
      "activations/layer10_attention_weight_max": 34.80956268310547,
      "activations/layer10_attention_weight_min": -35.64677429199219,
      "activations/layer11_attention_weight_max": 32.244056701660156,
      "activations/layer11_attention_weight_min": -33.3925666809082,
      "activations/layer12_attention_weight_max": 20.90352439880371,
      "activations/layer12_attention_weight_min": -26.126605987548828,
      "activations/layer13_attention_weight_max": 35.537315368652344,
      "activations/layer13_attention_weight_min": -27.936344146728516,
      "activations/layer14_attention_weight_max": 35.657466888427734,
      "activations/layer14_attention_weight_min": -28.524295806884766,
      "activations/layer15_attention_weight_max": 34.02719497680664,
      "activations/layer15_attention_weight_min": -28.55354118347168,
      "activations/layer16_attention_weight_max": 31.978694915771484,
      "activations/layer16_attention_weight_min": -30.289777755737305,
      "activations/layer17_attention_weight_max": 50.33749008178711,
      "activations/layer17_attention_weight_min": -41.7841682434082,
      "activations/layer18_attention_weight_max": 46.4032096862793,
      "activations/layer18_attention_weight_min": -37.32606506347656,
      "activations/layer19_attention_weight_max": 24.491785049438477,
      "activations/layer19_attention_weight_min": -22.079505920410156,
      "activations/layer1_attention_weight_max": 15.27876091003418,
      "activations/layer1_attention_weight_min": -14.433411598205566,
      "activations/layer20_attention_weight_max": 25.6455078125,
      "activations/layer20_attention_weight_min": -20.21389389038086,
      "activations/layer21_attention_weight_max": 38.332855224609375,
      "activations/layer21_attention_weight_min": -21.8000545501709,
      "activations/layer22_attention_weight_max": 32.611053466796875,
      "activations/layer22_attention_weight_min": -24.47182846069336,
      "activations/layer23_attention_weight_max": 34.36479949951172,
      "activations/layer23_attention_weight_min": -23.542402267456055,
      "activations/layer2_attention_weight_max": 32.56083297729492,
      "activations/layer2_attention_weight_min": -31.987409591674805,
      "activations/layer3_attention_weight_max": 93.14970397949219,
      "activations/layer3_attention_weight_min": -92.33069610595703,
      "activations/layer4_attention_weight_max": 86.93828582763672,
      "activations/layer4_attention_weight_min": -89.14043426513672,
      "activations/layer5_attention_weight_max": 62.6700439453125,
      "activations/layer5_attention_weight_min": -72.2872543334961,
      "activations/layer6_attention_weight_max": 51.088680267333984,
      "activations/layer6_attention_weight_min": -50.47600555419922,
      "activations/layer7_attention_weight_max": 60.44325637817383,
      "activations/layer7_attention_weight_min": -63.949954986572266,
      "activations/layer8_attention_weight_max": 44.39825439453125,
      "activations/layer8_attention_weight_min": -48.528045654296875,
      "activations/layer9_attention_weight_max": 44.0911865234375,
      "activations/layer9_attention_weight_min": -47.70888137817383,
      "epoch": 18.23,
      "learning_rate": 3.278522727272727e-05,
      "loss": 2.7407,
      "step": 313700
    },
    {
      "activations/layer0_attention_weight_max": 15.974896430969238,
      "activations/layer0_attention_weight_min": -13.230843544006348,
      "activations/layer10_attention_weight_max": 40.91303253173828,
      "activations/layer10_attention_weight_min": -37.79465103149414,
      "activations/layer11_attention_weight_max": 39.02593994140625,
      "activations/layer11_attention_weight_min": -36.035091400146484,
      "activations/layer12_attention_weight_max": 20.065357208251953,
      "activations/layer12_attention_weight_min": -28.7679386138916,
      "activations/layer13_attention_weight_max": 49.3223762512207,
      "activations/layer13_attention_weight_min": -30.107027053833008,
      "activations/layer14_attention_weight_max": 53.740352630615234,
      "activations/layer14_attention_weight_min": -30.881568908691406,
      "activations/layer15_attention_weight_max": 42.484004974365234,
      "activations/layer15_attention_weight_min": -29.0539608001709,
      "activations/layer16_attention_weight_max": 37.400882720947266,
      "activations/layer16_attention_weight_min": -32.56985855102539,
      "activations/layer17_attention_weight_max": 57.26498794555664,
      "activations/layer17_attention_weight_min": -46.29630661010742,
      "activations/layer18_attention_weight_max": 52.2257194519043,
      "activations/layer18_attention_weight_min": -41.17717742919922,
      "activations/layer19_attention_weight_max": 25.543354034423828,
      "activations/layer19_attention_weight_min": -23.849708557128906,
      "activations/layer1_attention_weight_max": 16.092191696166992,
      "activations/layer1_attention_weight_min": -14.743623733520508,
      "activations/layer20_attention_weight_max": 25.321691513061523,
      "activations/layer20_attention_weight_min": -21.026304244995117,
      "activations/layer21_attention_weight_max": 45.33599853515625,
      "activations/layer21_attention_weight_min": -23.40511131286621,
      "activations/layer22_attention_weight_max": 33.79452896118164,
      "activations/layer22_attention_weight_min": -26.00066375732422,
      "activations/layer23_attention_weight_max": 39.93148422241211,
      "activations/layer23_attention_weight_min": -25.009136199951172,
      "activations/layer2_attention_weight_max": 32.0543212890625,
      "activations/layer2_attention_weight_min": -30.991376876831055,
      "activations/layer3_attention_weight_max": 87.43689727783203,
      "activations/layer3_attention_weight_min": -92.90145111083984,
      "activations/layer4_attention_weight_max": 83.97858428955078,
      "activations/layer4_attention_weight_min": -86.07826232910156,
      "activations/layer5_attention_weight_max": 67.06526184082031,
      "activations/layer5_attention_weight_min": -70.42904663085938,
      "activations/layer6_attention_weight_max": 51.92930221557617,
      "activations/layer6_attention_weight_min": -52.12163543701172,
      "activations/layer7_attention_weight_max": 63.8662109375,
      "activations/layer7_attention_weight_min": -63.6456298828125,
      "activations/layer8_attention_weight_max": 47.0839958190918,
      "activations/layer8_attention_weight_min": -50.14442825317383,
      "activations/layer9_attention_weight_max": 53.6146354675293,
      "activations/layer9_attention_weight_min": -44.75611114501953,
      "epoch": 18.23,
      "learning_rate": 3.2766287878787876e-05,
      "loss": 2.7467,
      "step": 313750
    },
    {
      "activations/layer0_attention_weight_max": 14.965124130249023,
      "activations/layer0_attention_weight_min": -12.925251960754395,
      "activations/layer10_attention_weight_max": 37.97412872314453,
      "activations/layer10_attention_weight_min": -36.11676788330078,
      "activations/layer11_attention_weight_max": 34.730377197265625,
      "activations/layer11_attention_weight_min": -34.31244659423828,
      "activations/layer12_attention_weight_max": 19.413349151611328,
      "activations/layer12_attention_weight_min": -26.15158462524414,
      "activations/layer13_attention_weight_max": 31.369728088378906,
      "activations/layer13_attention_weight_min": -24.66170310974121,
      "activations/layer14_attention_weight_max": 35.23923110961914,
      "activations/layer14_attention_weight_min": -28.410945892333984,
      "activations/layer15_attention_weight_max": 34.66952133178711,
      "activations/layer15_attention_weight_min": -30.66073226928711,
      "activations/layer16_attention_weight_max": 39.37914276123047,
      "activations/layer16_attention_weight_min": -35.22858428955078,
      "activations/layer17_attention_weight_max": 56.792232513427734,
      "activations/layer17_attention_weight_min": -47.301883697509766,
      "activations/layer18_attention_weight_max": 52.916236877441406,
      "activations/layer18_attention_weight_min": -41.57809829711914,
      "activations/layer19_attention_weight_max": 23.49326515197754,
      "activations/layer19_attention_weight_min": -20.940876007080078,
      "activations/layer1_attention_weight_max": 15.604066848754883,
      "activations/layer1_attention_weight_min": -13.1779146194458,
      "activations/layer20_attention_weight_max": 21.70766258239746,
      "activations/layer20_attention_weight_min": -20.41085433959961,
      "activations/layer21_attention_weight_max": 30.685380935668945,
      "activations/layer21_attention_weight_min": -20.464197158813477,
      "activations/layer22_attention_weight_max": 32.03385925292969,
      "activations/layer22_attention_weight_min": -25.59930992126465,
      "activations/layer23_attention_weight_max": 33.667327880859375,
      "activations/layer23_attention_weight_min": -23.055959701538086,
      "activations/layer2_attention_weight_max": 31.262203216552734,
      "activations/layer2_attention_weight_min": -29.617294311523438,
      "activations/layer3_attention_weight_max": 86.36414337158203,
      "activations/layer3_attention_weight_min": -85.38121032714844,
      "activations/layer4_attention_weight_max": 85.42697143554688,
      "activations/layer4_attention_weight_min": -85.67823791503906,
      "activations/layer5_attention_weight_max": 58.85988998413086,
      "activations/layer5_attention_weight_min": -72.9368667602539,
      "activations/layer6_attention_weight_max": 48.42825698852539,
      "activations/layer6_attention_weight_min": -50.18868637084961,
      "activations/layer7_attention_weight_max": 64.44116973876953,
      "activations/layer7_attention_weight_min": -64.82794189453125,
      "activations/layer8_attention_weight_max": 47.63047409057617,
      "activations/layer8_attention_weight_min": -50.04463577270508,
      "activations/layer9_attention_weight_max": 47.50666046142578,
      "activations/layer9_attention_weight_min": -44.89641189575195,
      "epoch": 18.23,
      "learning_rate": 3.2747348484848485e-05,
      "loss": 2.7241,
      "step": 313800
    },
    {
      "activations/layer0_attention_weight_max": 15.50273609161377,
      "activations/layer0_attention_weight_min": -13.056047439575195,
      "activations/layer10_attention_weight_max": 35.528358459472656,
      "activations/layer10_attention_weight_min": -34.17213821411133,
      "activations/layer11_attention_weight_max": 32.43060302734375,
      "activations/layer11_attention_weight_min": -32.7288818359375,
      "activations/layer12_attention_weight_max": 20.484079360961914,
      "activations/layer12_attention_weight_min": -26.608518600463867,
      "activations/layer13_attention_weight_max": 34.08994674682617,
      "activations/layer13_attention_weight_min": -28.642671585083008,
      "activations/layer14_attention_weight_max": 35.22575759887695,
      "activations/layer14_attention_weight_min": -29.03631591796875,
      "activations/layer15_attention_weight_max": 33.18815994262695,
      "activations/layer15_attention_weight_min": -30.166118621826172,
      "activations/layer16_attention_weight_max": 32.083045959472656,
      "activations/layer16_attention_weight_min": -31.616796493530273,
      "activations/layer17_attention_weight_max": 50.086795806884766,
      "activations/layer17_attention_weight_min": -43.78693771362305,
      "activations/layer18_attention_weight_max": 43.6937141418457,
      "activations/layer18_attention_weight_min": -42.329002380371094,
      "activations/layer19_attention_weight_max": 23.3331298828125,
      "activations/layer19_attention_weight_min": -23.576297760009766,
      "activations/layer1_attention_weight_max": 16.06943702697754,
      "activations/layer1_attention_weight_min": -14.847485542297363,
      "activations/layer20_attention_weight_max": 20.701021194458008,
      "activations/layer20_attention_weight_min": -22.545665740966797,
      "activations/layer21_attention_weight_max": 36.957454681396484,
      "activations/layer21_attention_weight_min": -24.15757942199707,
      "activations/layer22_attention_weight_max": 29.16991424560547,
      "activations/layer22_attention_weight_min": -26.47981071472168,
      "activations/layer23_attention_weight_max": 36.919097900390625,
      "activations/layer23_attention_weight_min": -24.921911239624023,
      "activations/layer2_attention_weight_max": 30.819936752319336,
      "activations/layer2_attention_weight_min": -29.361263275146484,
      "activations/layer3_attention_weight_max": 87.51105499267578,
      "activations/layer3_attention_weight_min": -84.6489486694336,
      "activations/layer4_attention_weight_max": 82.14884185791016,
      "activations/layer4_attention_weight_min": -81.74375915527344,
      "activations/layer5_attention_weight_max": 63.16315460205078,
      "activations/layer5_attention_weight_min": -78.09297180175781,
      "activations/layer6_attention_weight_max": 48.82461929321289,
      "activations/layer6_attention_weight_min": -48.621097564697266,
      "activations/layer7_attention_weight_max": 63.82461929321289,
      "activations/layer7_attention_weight_min": -61.684600830078125,
      "activations/layer8_attention_weight_max": 46.40474319458008,
      "activations/layer8_attention_weight_min": -47.39479446411133,
      "activations/layer9_attention_weight_max": 41.759098052978516,
      "activations/layer9_attention_weight_min": -44.22886657714844,
      "epoch": 18.24,
      "learning_rate": 3.2728409090909086e-05,
      "loss": 2.7238,
      "step": 313850
    },
    {
      "activations/layer0_attention_weight_max": 16.068559646606445,
      "activations/layer0_attention_weight_min": -13.004981994628906,
      "activations/layer10_attention_weight_max": 38.499568939208984,
      "activations/layer10_attention_weight_min": -37.78368377685547,
      "activations/layer11_attention_weight_max": 32.87002944946289,
      "activations/layer11_attention_weight_min": -34.679237365722656,
      "activations/layer12_attention_weight_max": 19.42193603515625,
      "activations/layer12_attention_weight_min": -28.57833480834961,
      "activations/layer13_attention_weight_max": 32.5225830078125,
      "activations/layer13_attention_weight_min": -28.319307327270508,
      "activations/layer14_attention_weight_max": 38.453853607177734,
      "activations/layer14_attention_weight_min": -32.27593231201172,
      "activations/layer15_attention_weight_max": 35.6548957824707,
      "activations/layer15_attention_weight_min": -30.27105140686035,
      "activations/layer16_attention_weight_max": 36.20247268676758,
      "activations/layer16_attention_weight_min": -31.90888214111328,
      "activations/layer17_attention_weight_max": 57.60969924926758,
      "activations/layer17_attention_weight_min": -48.3911018371582,
      "activations/layer18_attention_weight_max": 52.3146858215332,
      "activations/layer18_attention_weight_min": -42.88215255737305,
      "activations/layer19_attention_weight_max": 24.47677230834961,
      "activations/layer19_attention_weight_min": -21.634687423706055,
      "activations/layer1_attention_weight_max": 15.86196517944336,
      "activations/layer1_attention_weight_min": -14.407123565673828,
      "activations/layer20_attention_weight_max": 22.467945098876953,
      "activations/layer20_attention_weight_min": -20.062070846557617,
      "activations/layer21_attention_weight_max": 36.774070739746094,
      "activations/layer21_attention_weight_min": -21.813554763793945,
      "activations/layer22_attention_weight_max": 30.50413703918457,
      "activations/layer22_attention_weight_min": -26.711097717285156,
      "activations/layer23_attention_weight_max": 40.35930633544922,
      "activations/layer23_attention_weight_min": -24.293926239013672,
      "activations/layer2_attention_weight_max": 31.793949127197266,
      "activations/layer2_attention_weight_min": -30.724407196044922,
      "activations/layer3_attention_weight_max": 93.95053100585938,
      "activations/layer3_attention_weight_min": -93.2833251953125,
      "activations/layer4_attention_weight_max": 85.99553680419922,
      "activations/layer4_attention_weight_min": -89.3344497680664,
      "activations/layer5_attention_weight_max": 62.993873596191406,
      "activations/layer5_attention_weight_min": -76.3817138671875,
      "activations/layer6_attention_weight_max": 52.38336944580078,
      "activations/layer6_attention_weight_min": -53.15522384643555,
      "activations/layer7_attention_weight_max": 66.2574234008789,
      "activations/layer7_attention_weight_min": -66.00163269042969,
      "activations/layer8_attention_weight_max": 48.03275680541992,
      "activations/layer8_attention_weight_min": -49.73821258544922,
      "activations/layer9_attention_weight_max": 46.37478256225586,
      "activations/layer9_attention_weight_min": -47.55720901489258,
      "epoch": 18.24,
      "learning_rate": 3.2709848484848484e-05,
      "loss": 2.7464,
      "step": 313900
    },
    {
      "activations/layer0_attention_weight_max": 14.517647743225098,
      "activations/layer0_attention_weight_min": -13.180279731750488,
      "activations/layer10_attention_weight_max": 40.818817138671875,
      "activations/layer10_attention_weight_min": -41.66750717163086,
      "activations/layer11_attention_weight_max": 37.20484161376953,
      "activations/layer11_attention_weight_min": -37.87840270996094,
      "activations/layer12_attention_weight_max": 18.822141647338867,
      "activations/layer12_attention_weight_min": -27.893312454223633,
      "activations/layer13_attention_weight_max": 34.032527923583984,
      "activations/layer13_attention_weight_min": -28.591890335083008,
      "activations/layer14_attention_weight_max": 36.15586853027344,
      "activations/layer14_attention_weight_min": -32.91368103027344,
      "activations/layer15_attention_weight_max": 32.858428955078125,
      "activations/layer15_attention_weight_min": -30.205181121826172,
      "activations/layer16_attention_weight_max": 37.92255401611328,
      "activations/layer16_attention_weight_min": -35.23430252075195,
      "activations/layer17_attention_weight_max": 53.00391387939453,
      "activations/layer17_attention_weight_min": -49.61171340942383,
      "activations/layer18_attention_weight_max": 46.846221923828125,
      "activations/layer18_attention_weight_min": -43.34931564331055,
      "activations/layer19_attention_weight_max": 23.716798782348633,
      "activations/layer19_attention_weight_min": -24.340229034423828,
      "activations/layer1_attention_weight_max": 16.106964111328125,
      "activations/layer1_attention_weight_min": -14.580936431884766,
      "activations/layer20_attention_weight_max": 20.815214157104492,
      "activations/layer20_attention_weight_min": -21.72315216064453,
      "activations/layer21_attention_weight_max": 37.90814971923828,
      "activations/layer21_attention_weight_min": -23.067066192626953,
      "activations/layer22_attention_weight_max": 27.437376022338867,
      "activations/layer22_attention_weight_min": -27.414113998413086,
      "activations/layer23_attention_weight_max": 34.64230728149414,
      "activations/layer23_attention_weight_min": -24.057647705078125,
      "activations/layer2_attention_weight_max": 31.363279342651367,
      "activations/layer2_attention_weight_min": -29.000385284423828,
      "activations/layer3_attention_weight_max": 88.56761932373047,
      "activations/layer3_attention_weight_min": -91.06498718261719,
      "activations/layer4_attention_weight_max": 92.7589111328125,
      "activations/layer4_attention_weight_min": -87.3563232421875,
      "activations/layer5_attention_weight_max": 63.790283203125,
      "activations/layer5_attention_weight_min": -78.61082458496094,
      "activations/layer6_attention_weight_max": 49.42519760131836,
      "activations/layer6_attention_weight_min": -51.476356506347656,
      "activations/layer7_attention_weight_max": 64.94599151611328,
      "activations/layer7_attention_weight_min": -67.65502166748047,
      "activations/layer8_attention_weight_max": 52.3325080871582,
      "activations/layer8_attention_weight_min": -54.26453399658203,
      "activations/layer9_attention_weight_max": 51.35640335083008,
      "activations/layer9_attention_weight_min": -51.32593536376953,
      "epoch": 18.24,
      "learning_rate": 3.2690909090909085e-05,
      "loss": 2.7396,
      "step": 313950
    },
    {
      "activations/layer0_attention_weight_max": 15.886658668518066,
      "activations/layer0_attention_weight_min": -14.628652572631836,
      "activations/layer10_attention_weight_max": 39.50550842285156,
      "activations/layer10_attention_weight_min": -36.42755889892578,
      "activations/layer11_attention_weight_max": 34.573768615722656,
      "activations/layer11_attention_weight_min": -34.17018508911133,
      "activations/layer12_attention_weight_max": 21.070592880249023,
      "activations/layer12_attention_weight_min": -28.10390853881836,
      "activations/layer13_attention_weight_max": 37.51470184326172,
      "activations/layer13_attention_weight_min": -26.660409927368164,
      "activations/layer14_attention_weight_max": 35.745540618896484,
      "activations/layer14_attention_weight_min": -29.946868896484375,
      "activations/layer15_attention_weight_max": 34.6444091796875,
      "activations/layer15_attention_weight_min": -29.949668884277344,
      "activations/layer16_attention_weight_max": 34.06631851196289,
      "activations/layer16_attention_weight_min": -30.75969123840332,
      "activations/layer17_attention_weight_max": 51.01927185058594,
      "activations/layer17_attention_weight_min": -43.744869232177734,
      "activations/layer18_attention_weight_max": 46.971797943115234,
      "activations/layer18_attention_weight_min": -36.978424072265625,
      "activations/layer19_attention_weight_max": 24.84394645690918,
      "activations/layer19_attention_weight_min": -21.67917251586914,
      "activations/layer1_attention_weight_max": 15.040711402893066,
      "activations/layer1_attention_weight_min": -15.412774085998535,
      "activations/layer20_attention_weight_max": 22.851642608642578,
      "activations/layer20_attention_weight_min": -20.067646026611328,
      "activations/layer21_attention_weight_max": 38.05754089355469,
      "activations/layer21_attention_weight_min": -21.459415435791016,
      "activations/layer22_attention_weight_max": 29.114112854003906,
      "activations/layer22_attention_weight_min": -24.327281951904297,
      "activations/layer23_attention_weight_max": 38.40122985839844,
      "activations/layer23_attention_weight_min": -23.133140563964844,
      "activations/layer2_attention_weight_max": 32.024967193603516,
      "activations/layer2_attention_weight_min": -31.63903045654297,
      "activations/layer3_attention_weight_max": 93.76066589355469,
      "activations/layer3_attention_weight_min": -91.99636840820312,
      "activations/layer4_attention_weight_max": 89.79125213623047,
      "activations/layer4_attention_weight_min": -84.82293701171875,
      "activations/layer5_attention_weight_max": 63.768768310546875,
      "activations/layer5_attention_weight_min": -74.41514587402344,
      "activations/layer6_attention_weight_max": 51.11027145385742,
      "activations/layer6_attention_weight_min": -52.44359588623047,
      "activations/layer7_attention_weight_max": 64.22319030761719,
      "activations/layer7_attention_weight_min": -61.80068588256836,
      "activations/layer8_attention_weight_max": 48.807369232177734,
      "activations/layer8_attention_weight_min": -48.74959182739258,
      "activations/layer9_attention_weight_max": 47.02997970581055,
      "activations/layer9_attention_weight_min": -46.30959701538086,
      "epoch": 18.25,
      "learning_rate": 3.2671969696969694e-05,
      "loss": 2.7415,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.486,
      "eval_samples_per_second": 506.01,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.486,
      "eval_openwebtext_samples_per_second": 506.01,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.9718,
      "eval_wikitext_samples_per_second": 231.258,
      "step": 314000
    },
    {
      "epoch": 18.25,
      "eval_lambada_loss": 2.453125,
      "eval_lambada_ppl": 11.624616945432633,
      "eval_lambada_runtime": 9.5775,
      "eval_lambada_samples_per_second": 508.381,
      "step": 314000
    },
    {
      "activations/layer0_attention_weight_max": 16.9760799407959,
      "activations/layer0_attention_weight_min": -13.523930549621582,
      "activations/layer10_attention_weight_max": 37.32603073120117,
      "activations/layer10_attention_weight_min": -37.14627456665039,
      "activations/layer11_attention_weight_max": 34.50001525878906,
      "activations/layer11_attention_weight_min": -34.90290832519531,
      "activations/layer12_attention_weight_max": 18.94173240661621,
      "activations/layer12_attention_weight_min": -25.96428871154785,
      "activations/layer13_attention_weight_max": 32.81315994262695,
      "activations/layer13_attention_weight_min": -30.191789627075195,
      "activations/layer14_attention_weight_max": 36.01066970825195,
      "activations/layer14_attention_weight_min": -30.088571548461914,
      "activations/layer15_attention_weight_max": 34.1085090637207,
      "activations/layer15_attention_weight_min": -31.28481101989746,
      "activations/layer16_attention_weight_max": 32.659603118896484,
      "activations/layer16_attention_weight_min": -31.93489646911621,
      "activations/layer17_attention_weight_max": 52.228763580322266,
      "activations/layer17_attention_weight_min": -47.30510330200195,
      "activations/layer18_attention_weight_max": 48.78630828857422,
      "activations/layer18_attention_weight_min": -39.163089752197266,
      "activations/layer19_attention_weight_max": 25.98520278930664,
      "activations/layer19_attention_weight_min": -22.443212509155273,
      "activations/layer1_attention_weight_max": 16.39783477783203,
      "activations/layer1_attention_weight_min": -15.15487289428711,
      "activations/layer20_attention_weight_max": 22.83746910095215,
      "activations/layer20_attention_weight_min": -20.93264389038086,
      "activations/layer21_attention_weight_max": 33.76100540161133,
      "activations/layer21_attention_weight_min": -20.139184951782227,
      "activations/layer22_attention_weight_max": 31.47728729248047,
      "activations/layer22_attention_weight_min": -25.846784591674805,
      "activations/layer23_attention_weight_max": 38.73360061645508,
      "activations/layer23_attention_weight_min": -27.150991439819336,
      "activations/layer2_attention_weight_max": 30.803930282592773,
      "activations/layer2_attention_weight_min": -29.678966522216797,
      "activations/layer3_attention_weight_max": 87.39654541015625,
      "activations/layer3_attention_weight_min": -85.75446319580078,
      "activations/layer4_attention_weight_max": 87.07705688476562,
      "activations/layer4_attention_weight_min": -87.65251159667969,
      "activations/layer5_attention_weight_max": 63.61225128173828,
      "activations/layer5_attention_weight_min": -70.97066497802734,
      "activations/layer6_attention_weight_max": 50.27184295654297,
      "activations/layer6_attention_weight_min": -51.03784942626953,
      "activations/layer7_attention_weight_max": 69.64309692382812,
      "activations/layer7_attention_weight_min": -62.3539924621582,
      "activations/layer8_attention_weight_max": 47.74897003173828,
      "activations/layer8_attention_weight_min": -49.405094146728516,
      "activations/layer9_attention_weight_max": 46.92304611206055,
      "activations/layer9_attention_weight_min": -46.29059600830078,
      "epoch": 18.25,
      "learning_rate": 3.26530303030303e-05,
      "loss": 2.7271,
      "step": 314050
    },
    {
      "activations/layer0_attention_weight_max": 15.387672424316406,
      "activations/layer0_attention_weight_min": -13.33034896850586,
      "activations/layer10_attention_weight_max": 33.25327682495117,
      "activations/layer10_attention_weight_min": -34.202205657958984,
      "activations/layer11_attention_weight_max": 31.132404327392578,
      "activations/layer11_attention_weight_min": -35.85407257080078,
      "activations/layer12_attention_weight_max": 19.850399017333984,
      "activations/layer12_attention_weight_min": -26.85736846923828,
      "activations/layer13_attention_weight_max": 30.711349487304688,
      "activations/layer13_attention_weight_min": -28.50088119506836,
      "activations/layer14_attention_weight_max": 34.442047119140625,
      "activations/layer14_attention_weight_min": -31.814836502075195,
      "activations/layer15_attention_weight_max": 31.112411499023438,
      "activations/layer15_attention_weight_min": -29.13198471069336,
      "activations/layer16_attention_weight_max": 33.237945556640625,
      "activations/layer16_attention_weight_min": -30.73985481262207,
      "activations/layer17_attention_weight_max": 51.664878845214844,
      "activations/layer17_attention_weight_min": -42.562400817871094,
      "activations/layer18_attention_weight_max": 46.654029846191406,
      "activations/layer18_attention_weight_min": -38.213172912597656,
      "activations/layer19_attention_weight_max": 27.242658615112305,
      "activations/layer19_attention_weight_min": -22.45362663269043,
      "activations/layer1_attention_weight_max": 15.635769844055176,
      "activations/layer1_attention_weight_min": -13.398736000061035,
      "activations/layer20_attention_weight_max": 20.640125274658203,
      "activations/layer20_attention_weight_min": -22.417564392089844,
      "activations/layer21_attention_weight_max": 34.938011169433594,
      "activations/layer21_attention_weight_min": -25.88002586364746,
      "activations/layer22_attention_weight_max": 29.21529197692871,
      "activations/layer22_attention_weight_min": -26.530054092407227,
      "activations/layer23_attention_weight_max": 36.542057037353516,
      "activations/layer23_attention_weight_min": -25.797910690307617,
      "activations/layer2_attention_weight_max": 32.149330139160156,
      "activations/layer2_attention_weight_min": -30.019184112548828,
      "activations/layer3_attention_weight_max": 90.99271392822266,
      "activations/layer3_attention_weight_min": -87.5105972290039,
      "activations/layer4_attention_weight_max": 83.58859252929688,
      "activations/layer4_attention_weight_min": -81.99844360351562,
      "activations/layer5_attention_weight_max": 60.631099700927734,
      "activations/layer5_attention_weight_min": -71.764404296875,
      "activations/layer6_attention_weight_max": 46.72283172607422,
      "activations/layer6_attention_weight_min": -48.232975006103516,
      "activations/layer7_attention_weight_max": 60.995323181152344,
      "activations/layer7_attention_weight_min": -58.4895133972168,
      "activations/layer8_attention_weight_max": 43.84295654296875,
      "activations/layer8_attention_weight_min": -47.522666931152344,
      "activations/layer9_attention_weight_max": 41.680580139160156,
      "activations/layer9_attention_weight_min": -44.5582275390625,
      "epoch": 18.25,
      "learning_rate": 3.263409090909091e-05,
      "loss": 2.7204,
      "step": 314100
    },
    {
      "activations/layer0_attention_weight_max": 16.136754989624023,
      "activations/layer0_attention_weight_min": -13.592790603637695,
      "activations/layer10_attention_weight_max": 37.01832962036133,
      "activations/layer10_attention_weight_min": -36.170936584472656,
      "activations/layer11_attention_weight_max": 34.417945861816406,
      "activations/layer11_attention_weight_min": -35.48237991333008,
      "activations/layer12_attention_weight_max": 19.898406982421875,
      "activations/layer12_attention_weight_min": -25.858871459960938,
      "activations/layer13_attention_weight_max": 33.048858642578125,
      "activations/layer13_attention_weight_min": -27.600353240966797,
      "activations/layer14_attention_weight_max": 35.67308807373047,
      "activations/layer14_attention_weight_min": -30.915117263793945,
      "activations/layer15_attention_weight_max": 34.13982009887695,
      "activations/layer15_attention_weight_min": -32.07188034057617,
      "activations/layer16_attention_weight_max": 36.39765548706055,
      "activations/layer16_attention_weight_min": -32.67715835571289,
      "activations/layer17_attention_weight_max": 55.515377044677734,
      "activations/layer17_attention_weight_min": -46.87499237060547,
      "activations/layer18_attention_weight_max": 50.250755310058594,
      "activations/layer18_attention_weight_min": -41.34556579589844,
      "activations/layer19_attention_weight_max": 29.399988174438477,
      "activations/layer19_attention_weight_min": -23.59741973876953,
      "activations/layer1_attention_weight_max": 15.058906555175781,
      "activations/layer1_attention_weight_min": -14.574766159057617,
      "activations/layer20_attention_weight_max": 25.445161819458008,
      "activations/layer20_attention_weight_min": -20.961423873901367,
      "activations/layer21_attention_weight_max": 39.149871826171875,
      "activations/layer21_attention_weight_min": -21.031511306762695,
      "activations/layer22_attention_weight_max": 31.352767944335938,
      "activations/layer22_attention_weight_min": -25.98956298828125,
      "activations/layer23_attention_weight_max": 38.025482177734375,
      "activations/layer23_attention_weight_min": -23.74176597595215,
      "activations/layer2_attention_weight_max": 33.15416717529297,
      "activations/layer2_attention_weight_min": -29.71601104736328,
      "activations/layer3_attention_weight_max": 92.14838409423828,
      "activations/layer3_attention_weight_min": -90.60713195800781,
      "activations/layer4_attention_weight_max": 89.9002685546875,
      "activations/layer4_attention_weight_min": -84.68976593017578,
      "activations/layer5_attention_weight_max": 63.70122528076172,
      "activations/layer5_attention_weight_min": -77.2420883178711,
      "activations/layer6_attention_weight_max": 52.88164520263672,
      "activations/layer6_attention_weight_min": -52.16489791870117,
      "activations/layer7_attention_weight_max": 67.7370376586914,
      "activations/layer7_attention_weight_min": -65.34237670898438,
      "activations/layer8_attention_weight_max": 46.44174575805664,
      "activations/layer8_attention_weight_min": -53.62422561645508,
      "activations/layer9_attention_weight_max": 45.21943283081055,
      "activations/layer9_attention_weight_min": -46.9975700378418,
      "epoch": 18.25,
      "learning_rate": 3.261515151515151e-05,
      "loss": 2.7379,
      "step": 314150
    },
    {
      "activations/layer0_attention_weight_max": 14.826774597167969,
      "activations/layer0_attention_weight_min": -13.392287254333496,
      "activations/layer10_attention_weight_max": 34.008968353271484,
      "activations/layer10_attention_weight_min": -34.50560760498047,
      "activations/layer11_attention_weight_max": 31.26814842224121,
      "activations/layer11_attention_weight_min": -32.380210876464844,
      "activations/layer12_attention_weight_max": 18.501419067382812,
      "activations/layer12_attention_weight_min": -27.8664493560791,
      "activations/layer13_attention_weight_max": 39.6137580871582,
      "activations/layer13_attention_weight_min": -29.301782608032227,
      "activations/layer14_attention_weight_max": 38.26755905151367,
      "activations/layer14_attention_weight_min": -31.51702308654785,
      "activations/layer15_attention_weight_max": 35.96290969848633,
      "activations/layer15_attention_weight_min": -31.27164077758789,
      "activations/layer16_attention_weight_max": 38.3892707824707,
      "activations/layer16_attention_weight_min": -34.15078353881836,
      "activations/layer17_attention_weight_max": 54.694149017333984,
      "activations/layer17_attention_weight_min": -48.12393569946289,
      "activations/layer18_attention_weight_max": 54.13159942626953,
      "activations/layer18_attention_weight_min": -43.707576751708984,
      "activations/layer19_attention_weight_max": 30.58030128479004,
      "activations/layer19_attention_weight_min": -25.024263381958008,
      "activations/layer1_attention_weight_max": 15.691455841064453,
      "activations/layer1_attention_weight_min": -14.901337623596191,
      "activations/layer20_attention_weight_max": 29.271831512451172,
      "activations/layer20_attention_weight_min": -23.521854400634766,
      "activations/layer21_attention_weight_max": 46.260250091552734,
      "activations/layer21_attention_weight_min": -26.147245407104492,
      "activations/layer22_attention_weight_max": 33.355159759521484,
      "activations/layer22_attention_weight_min": -27.880403518676758,
      "activations/layer23_attention_weight_max": 40.98163604736328,
      "activations/layer23_attention_weight_min": -24.838308334350586,
      "activations/layer2_attention_weight_max": 31.473434448242188,
      "activations/layer2_attention_weight_min": -28.591548919677734,
      "activations/layer3_attention_weight_max": 84.82041931152344,
      "activations/layer3_attention_weight_min": -86.5772476196289,
      "activations/layer4_attention_weight_max": 84.32670593261719,
      "activations/layer4_attention_weight_min": -85.24288177490234,
      "activations/layer5_attention_weight_max": 63.67457580566406,
      "activations/layer5_attention_weight_min": -77.52884674072266,
      "activations/layer6_attention_weight_max": 49.70340347290039,
      "activations/layer6_attention_weight_min": -50.59400939941406,
      "activations/layer7_attention_weight_max": 65.80022430419922,
      "activations/layer7_attention_weight_min": -62.39023971557617,
      "activations/layer8_attention_weight_max": 46.291648864746094,
      "activations/layer8_attention_weight_min": -48.110870361328125,
      "activations/layer9_attention_weight_max": 44.52241134643555,
      "activations/layer9_attention_weight_min": -44.99458312988281,
      "epoch": 18.26,
      "learning_rate": 3.259621212121212e-05,
      "loss": 2.7413,
      "step": 314200
    },
    {
      "activations/layer0_attention_weight_max": 15.74789047241211,
      "activations/layer0_attention_weight_min": -13.402812004089355,
      "activations/layer10_attention_weight_max": 35.94417953491211,
      "activations/layer10_attention_weight_min": -36.166656494140625,
      "activations/layer11_attention_weight_max": 33.88874053955078,
      "activations/layer11_attention_weight_min": -33.46355056762695,
      "activations/layer12_attention_weight_max": 17.979124069213867,
      "activations/layer12_attention_weight_min": -24.338504791259766,
      "activations/layer13_attention_weight_max": 34.18598556518555,
      "activations/layer13_attention_weight_min": -25.990589141845703,
      "activations/layer14_attention_weight_max": 34.96710205078125,
      "activations/layer14_attention_weight_min": -30.700014114379883,
      "activations/layer15_attention_weight_max": 34.722740173339844,
      "activations/layer15_attention_weight_min": -30.92475700378418,
      "activations/layer16_attention_weight_max": 31.88173484802246,
      "activations/layer16_attention_weight_min": -30.50543975830078,
      "activations/layer17_attention_weight_max": 49.35456848144531,
      "activations/layer17_attention_weight_min": -43.518070220947266,
      "activations/layer18_attention_weight_max": 47.74313735961914,
      "activations/layer18_attention_weight_min": -40.4668083190918,
      "activations/layer19_attention_weight_max": 24.94074058532715,
      "activations/layer19_attention_weight_min": -22.31537437438965,
      "activations/layer1_attention_weight_max": 15.538483619689941,
      "activations/layer1_attention_weight_min": -14.311708450317383,
      "activations/layer20_attention_weight_max": 23.701732635498047,
      "activations/layer20_attention_weight_min": -20.55320167541504,
      "activations/layer21_attention_weight_max": 33.61448669433594,
      "activations/layer21_attention_weight_min": -23.577354431152344,
      "activations/layer22_attention_weight_max": 28.498125076293945,
      "activations/layer22_attention_weight_min": -24.43720054626465,
      "activations/layer23_attention_weight_max": 34.75819396972656,
      "activations/layer23_attention_weight_min": -24.1814022064209,
      "activations/layer2_attention_weight_max": 31.911518096923828,
      "activations/layer2_attention_weight_min": -29.6235408782959,
      "activations/layer3_attention_weight_max": 85.00579071044922,
      "activations/layer3_attention_weight_min": -86.89637756347656,
      "activations/layer4_attention_weight_max": 85.96916198730469,
      "activations/layer4_attention_weight_min": -86.4551773071289,
      "activations/layer5_attention_weight_max": 63.31012725830078,
      "activations/layer5_attention_weight_min": -74.38007354736328,
      "activations/layer6_attention_weight_max": 49.45210647583008,
      "activations/layer6_attention_weight_min": -48.336544036865234,
      "activations/layer7_attention_weight_max": 61.7614631652832,
      "activations/layer7_attention_weight_min": -62.220008850097656,
      "activations/layer8_attention_weight_max": 44.33030700683594,
      "activations/layer8_attention_weight_min": -47.5984001159668,
      "activations/layer9_attention_weight_max": 41.22774887084961,
      "activations/layer9_attention_weight_min": -42.91415023803711,
      "epoch": 18.26,
      "learning_rate": 3.257727272727272e-05,
      "loss": 2.7256,
      "step": 314250
    },
    {
      "activations/layer0_attention_weight_max": 15.327778816223145,
      "activations/layer0_attention_weight_min": -15.123507499694824,
      "activations/layer10_attention_weight_max": 34.75684356689453,
      "activations/layer10_attention_weight_min": -35.671146392822266,
      "activations/layer11_attention_weight_max": 32.46091079711914,
      "activations/layer11_attention_weight_min": -33.45915222167969,
      "activations/layer12_attention_weight_max": 22.109710693359375,
      "activations/layer12_attention_weight_min": -23.11224365234375,
      "activations/layer13_attention_weight_max": 37.51115417480469,
      "activations/layer13_attention_weight_min": -27.770662307739258,
      "activations/layer14_attention_weight_max": 37.21141052246094,
      "activations/layer14_attention_weight_min": -29.923471450805664,
      "activations/layer15_attention_weight_max": 35.50870895385742,
      "activations/layer15_attention_weight_min": -32.600990295410156,
      "activations/layer16_attention_weight_max": 34.16822052001953,
      "activations/layer16_attention_weight_min": -32.95669937133789,
      "activations/layer17_attention_weight_max": 54.28215026855469,
      "activations/layer17_attention_weight_min": -46.53086853027344,
      "activations/layer18_attention_weight_max": 49.683353424072266,
      "activations/layer18_attention_weight_min": -42.73685073852539,
      "activations/layer19_attention_weight_max": 26.824350357055664,
      "activations/layer19_attention_weight_min": -23.7712345123291,
      "activations/layer1_attention_weight_max": 16.576631546020508,
      "activations/layer1_attention_weight_min": -13.974557876586914,
      "activations/layer20_attention_weight_max": 26.955982208251953,
      "activations/layer20_attention_weight_min": -23.429113388061523,
      "activations/layer21_attention_weight_max": 43.08959197998047,
      "activations/layer21_attention_weight_min": -23.70665740966797,
      "activations/layer22_attention_weight_max": 30.697412490844727,
      "activations/layer22_attention_weight_min": -28.101581573486328,
      "activations/layer23_attention_weight_max": 41.18598937988281,
      "activations/layer23_attention_weight_min": -25.920639038085938,
      "activations/layer2_attention_weight_max": 30.157516479492188,
      "activations/layer2_attention_weight_min": -29.290569305419922,
      "activations/layer3_attention_weight_max": 83.37813568115234,
      "activations/layer3_attention_weight_min": -86.80378723144531,
      "activations/layer4_attention_weight_max": 81.55696105957031,
      "activations/layer4_attention_weight_min": -81.95410919189453,
      "activations/layer5_attention_weight_max": 62.04702377319336,
      "activations/layer5_attention_weight_min": -69.43185424804688,
      "activations/layer6_attention_weight_max": 49.71518325805664,
      "activations/layer6_attention_weight_min": -53.884681701660156,
      "activations/layer7_attention_weight_max": 62.29219055175781,
      "activations/layer7_attention_weight_min": -60.03106689453125,
      "activations/layer8_attention_weight_max": 44.646549224853516,
      "activations/layer8_attention_weight_min": -47.58638381958008,
      "activations/layer9_attention_weight_max": 42.68351364135742,
      "activations/layer9_attention_weight_min": -44.62799835205078,
      "epoch": 18.26,
      "learning_rate": 3.255833333333333e-05,
      "loss": 2.7305,
      "step": 314300
    },
    {
      "activations/layer0_attention_weight_max": 14.889992713928223,
      "activations/layer0_attention_weight_min": -13.550455093383789,
      "activations/layer10_attention_weight_max": 40.05741882324219,
      "activations/layer10_attention_weight_min": -38.277862548828125,
      "activations/layer11_attention_weight_max": 34.431785583496094,
      "activations/layer11_attention_weight_min": -35.70211410522461,
      "activations/layer12_attention_weight_max": 20.088151931762695,
      "activations/layer12_attention_weight_min": -25.46100425720215,
      "activations/layer13_attention_weight_max": 34.51163101196289,
      "activations/layer13_attention_weight_min": -29.147680282592773,
      "activations/layer14_attention_weight_max": 37.17218780517578,
      "activations/layer14_attention_weight_min": -32.70603561401367,
      "activations/layer15_attention_weight_max": 33.63233184814453,
      "activations/layer15_attention_weight_min": -32.165504455566406,
      "activations/layer16_attention_weight_max": 33.947120666503906,
      "activations/layer16_attention_weight_min": -32.9037971496582,
      "activations/layer17_attention_weight_max": 52.27143096923828,
      "activations/layer17_attention_weight_min": -48.75137710571289,
      "activations/layer18_attention_weight_max": 48.63934326171875,
      "activations/layer18_attention_weight_min": -43.011470794677734,
      "activations/layer19_attention_weight_max": 26.744977951049805,
      "activations/layer19_attention_weight_min": -23.736692428588867,
      "activations/layer1_attention_weight_max": 15.655928611755371,
      "activations/layer1_attention_weight_min": -13.99646282196045,
      "activations/layer20_attention_weight_max": 23.313127517700195,
      "activations/layer20_attention_weight_min": -21.270536422729492,
      "activations/layer21_attention_weight_max": 40.596290588378906,
      "activations/layer21_attention_weight_min": -20.43505859375,
      "activations/layer22_attention_weight_max": 32.103633880615234,
      "activations/layer22_attention_weight_min": -26.47197723388672,
      "activations/layer23_attention_weight_max": 38.98884582519531,
      "activations/layer23_attention_weight_min": -23.2495174407959,
      "activations/layer2_attention_weight_max": 31.114696502685547,
      "activations/layer2_attention_weight_min": -28.769412994384766,
      "activations/layer3_attention_weight_max": 89.02590942382812,
      "activations/layer3_attention_weight_min": -89.24366760253906,
      "activations/layer4_attention_weight_max": 88.22408294677734,
      "activations/layer4_attention_weight_min": -92.1190185546875,
      "activations/layer5_attention_weight_max": 64.68892669677734,
      "activations/layer5_attention_weight_min": -73.35516357421875,
      "activations/layer6_attention_weight_max": 50.41385269165039,
      "activations/layer6_attention_weight_min": -53.49628448486328,
      "activations/layer7_attention_weight_max": 69.32145690917969,
      "activations/layer7_attention_weight_min": -66.36628723144531,
      "activations/layer8_attention_weight_max": 48.79793930053711,
      "activations/layer8_attention_weight_min": -50.215919494628906,
      "activations/layer9_attention_weight_max": 51.05964279174805,
      "activations/layer9_attention_weight_min": -48.399478912353516,
      "epoch": 18.27,
      "learning_rate": 3.253977272727272e-05,
      "loss": 2.7469,
      "step": 314350
    },
    {
      "activations/layer0_attention_weight_max": 15.415684700012207,
      "activations/layer0_attention_weight_min": -13.200425148010254,
      "activations/layer10_attention_weight_max": 36.37571716308594,
      "activations/layer10_attention_weight_min": -36.944854736328125,
      "activations/layer11_attention_weight_max": 32.909828186035156,
      "activations/layer11_attention_weight_min": -33.98881149291992,
      "activations/layer12_attention_weight_max": 21.135820388793945,
      "activations/layer12_attention_weight_min": -25.49588394165039,
      "activations/layer13_attention_weight_max": 36.24763107299805,
      "activations/layer13_attention_weight_min": -27.604721069335938,
      "activations/layer14_attention_weight_max": 34.94724655151367,
      "activations/layer14_attention_weight_min": -29.667232513427734,
      "activations/layer15_attention_weight_max": 32.57011032104492,
      "activations/layer15_attention_weight_min": -29.160911560058594,
      "activations/layer16_attention_weight_max": 33.329872131347656,
      "activations/layer16_attention_weight_min": -33.996177673339844,
      "activations/layer17_attention_weight_max": 51.58454513549805,
      "activations/layer17_attention_weight_min": -47.32372283935547,
      "activations/layer18_attention_weight_max": 46.94126510620117,
      "activations/layer18_attention_weight_min": -41.66175842285156,
      "activations/layer19_attention_weight_max": 28.39154052734375,
      "activations/layer19_attention_weight_min": -24.96282958984375,
      "activations/layer1_attention_weight_max": 17.019052505493164,
      "activations/layer1_attention_weight_min": -14.01491928100586,
      "activations/layer20_attention_weight_max": 25.764314651489258,
      "activations/layer20_attention_weight_min": -21.4980411529541,
      "activations/layer21_attention_weight_max": 42.62763595581055,
      "activations/layer21_attention_weight_min": -20.494199752807617,
      "activations/layer22_attention_weight_max": 30.35402488708496,
      "activations/layer22_attention_weight_min": -26.459327697753906,
      "activations/layer23_attention_weight_max": 37.212867736816406,
      "activations/layer23_attention_weight_min": -22.864246368408203,
      "activations/layer2_attention_weight_max": 31.456829071044922,
      "activations/layer2_attention_weight_min": -29.055564880371094,
      "activations/layer3_attention_weight_max": 84.50652313232422,
      "activations/layer3_attention_weight_min": -82.51586151123047,
      "activations/layer4_attention_weight_max": 82.34627532958984,
      "activations/layer4_attention_weight_min": -79.05927276611328,
      "activations/layer5_attention_weight_max": 61.70023727416992,
      "activations/layer5_attention_weight_min": -71.6135025024414,
      "activations/layer6_attention_weight_max": 46.71086502075195,
      "activations/layer6_attention_weight_min": -47.48239517211914,
      "activations/layer7_attention_weight_max": 63.40509033203125,
      "activations/layer7_attention_weight_min": -62.46455764770508,
      "activations/layer8_attention_weight_max": 47.3314323425293,
      "activations/layer8_attention_weight_min": -47.3669548034668,
      "activations/layer9_attention_weight_max": 50.474971771240234,
      "activations/layer9_attention_weight_min": -44.91019821166992,
      "epoch": 18.27,
      "learning_rate": 3.252083333333333e-05,
      "loss": 2.7418,
      "step": 314400
    },
    {
      "activations/layer0_attention_weight_max": 14.632835388183594,
      "activations/layer0_attention_weight_min": -13.61400032043457,
      "activations/layer10_attention_weight_max": 34.162261962890625,
      "activations/layer10_attention_weight_min": -33.659950256347656,
      "activations/layer11_attention_weight_max": 33.492923736572266,
      "activations/layer11_attention_weight_min": -32.61626434326172,
      "activations/layer12_attention_weight_max": 18.680578231811523,
      "activations/layer12_attention_weight_min": -25.722248077392578,
      "activations/layer13_attention_weight_max": 32.81828689575195,
      "activations/layer13_attention_weight_min": -26.31487274169922,
      "activations/layer14_attention_weight_max": 35.07355880737305,
      "activations/layer14_attention_weight_min": -31.332279205322266,
      "activations/layer15_attention_weight_max": 32.59938049316406,
      "activations/layer15_attention_weight_min": -32.77788162231445,
      "activations/layer16_attention_weight_max": 34.15175247192383,
      "activations/layer16_attention_weight_min": -33.11550521850586,
      "activations/layer17_attention_weight_max": 51.668216705322266,
      "activations/layer17_attention_weight_min": -47.90888214111328,
      "activations/layer18_attention_weight_max": 48.317848205566406,
      "activations/layer18_attention_weight_min": -40.02342987060547,
      "activations/layer19_attention_weight_max": 22.822978973388672,
      "activations/layer19_attention_weight_min": -21.83631706237793,
      "activations/layer1_attention_weight_max": 16.833526611328125,
      "activations/layer1_attention_weight_min": -13.364699363708496,
      "activations/layer20_attention_weight_max": 22.640012741088867,
      "activations/layer20_attention_weight_min": -21.225284576416016,
      "activations/layer21_attention_weight_max": 36.7222900390625,
      "activations/layer21_attention_weight_min": -20.259868621826172,
      "activations/layer22_attention_weight_max": 31.938461303710938,
      "activations/layer22_attention_weight_min": -25.113149642944336,
      "activations/layer23_attention_weight_max": 39.67608642578125,
      "activations/layer23_attention_weight_min": -23.527175903320312,
      "activations/layer2_attention_weight_max": 30.911834716796875,
      "activations/layer2_attention_weight_min": -28.128551483154297,
      "activations/layer3_attention_weight_max": 82.48001861572266,
      "activations/layer3_attention_weight_min": -83.52397155761719,
      "activations/layer4_attention_weight_max": 83.00237274169922,
      "activations/layer4_attention_weight_min": -84.8731460571289,
      "activations/layer5_attention_weight_max": 61.677268981933594,
      "activations/layer5_attention_weight_min": -78.27491760253906,
      "activations/layer6_attention_weight_max": 47.749942779541016,
      "activations/layer6_attention_weight_min": -49.7573127746582,
      "activations/layer7_attention_weight_max": 65.1525650024414,
      "activations/layer7_attention_weight_min": -59.86076736450195,
      "activations/layer8_attention_weight_max": 43.41450881958008,
      "activations/layer8_attention_weight_min": -46.63411331176758,
      "activations/layer9_attention_weight_max": 50.1512336730957,
      "activations/layer9_attention_weight_min": -44.732723236083984,
      "epoch": 18.27,
      "learning_rate": 3.250189393939394e-05,
      "loss": 2.7363,
      "step": 314450
    },
    {
      "activations/layer0_attention_weight_max": 15.7182035446167,
      "activations/layer0_attention_weight_min": -13.497811317443848,
      "activations/layer10_attention_weight_max": 34.01358413696289,
      "activations/layer10_attention_weight_min": -35.422393798828125,
      "activations/layer11_attention_weight_max": 31.458349227905273,
      "activations/layer11_attention_weight_min": -33.630619049072266,
      "activations/layer12_attention_weight_max": 21.37321662902832,
      "activations/layer12_attention_weight_min": -28.480716705322266,
      "activations/layer13_attention_weight_max": 35.241455078125,
      "activations/layer13_attention_weight_min": -27.298046112060547,
      "activations/layer14_attention_weight_max": 37.67710876464844,
      "activations/layer14_attention_weight_min": -29.92288589477539,
      "activations/layer15_attention_weight_max": 34.32575988769531,
      "activations/layer15_attention_weight_min": -30.38214874267578,
      "activations/layer16_attention_weight_max": 36.363285064697266,
      "activations/layer16_attention_weight_min": -32.176517486572266,
      "activations/layer17_attention_weight_max": 53.933868408203125,
      "activations/layer17_attention_weight_min": -44.646766662597656,
      "activations/layer18_attention_weight_max": 49.57894515991211,
      "activations/layer18_attention_weight_min": -40.363765716552734,
      "activations/layer19_attention_weight_max": 25.66543960571289,
      "activations/layer19_attention_weight_min": -23.758886337280273,
      "activations/layer1_attention_weight_max": 16.48902130126953,
      "activations/layer1_attention_weight_min": -14.274097442626953,
      "activations/layer20_attention_weight_max": 26.2777099609375,
      "activations/layer20_attention_weight_min": -20.571975708007812,
      "activations/layer21_attention_weight_max": 43.403045654296875,
      "activations/layer21_attention_weight_min": -25.433801651000977,
      "activations/layer22_attention_weight_max": 28.84172248840332,
      "activations/layer22_attention_weight_min": -26.3438720703125,
      "activations/layer23_attention_weight_max": 40.49407196044922,
      "activations/layer23_attention_weight_min": -24.716238021850586,
      "activations/layer2_attention_weight_max": 31.403533935546875,
      "activations/layer2_attention_weight_min": -29.03236961364746,
      "activations/layer3_attention_weight_max": 82.83876037597656,
      "activations/layer3_attention_weight_min": -83.71637725830078,
      "activations/layer4_attention_weight_max": 79.3402099609375,
      "activations/layer4_attention_weight_min": -82.62834167480469,
      "activations/layer5_attention_weight_max": 58.408103942871094,
      "activations/layer5_attention_weight_min": -72.3941650390625,
      "activations/layer6_attention_weight_max": 45.81385040283203,
      "activations/layer6_attention_weight_min": -47.99335861206055,
      "activations/layer7_attention_weight_max": 62.036983489990234,
      "activations/layer7_attention_weight_min": -68.01493072509766,
      "activations/layer8_attention_weight_max": 42.015342712402344,
      "activations/layer8_attention_weight_min": -46.050384521484375,
      "activations/layer9_attention_weight_max": 43.890472412109375,
      "activations/layer9_attention_weight_min": -45.3246955871582,
      "epoch": 18.27,
      "learning_rate": 3.2482954545454545e-05,
      "loss": 2.723,
      "step": 314500
    },
    {
      "activations/layer0_attention_weight_max": 15.37745189666748,
      "activations/layer0_attention_weight_min": -13.452821731567383,
      "activations/layer10_attention_weight_max": 41.15544509887695,
      "activations/layer10_attention_weight_min": -39.48420715332031,
      "activations/layer11_attention_weight_max": 39.587562561035156,
      "activations/layer11_attention_weight_min": -38.50053405761719,
      "activations/layer12_attention_weight_max": 20.151611328125,
      "activations/layer12_attention_weight_min": -27.809412002563477,
      "activations/layer13_attention_weight_max": 40.72046661376953,
      "activations/layer13_attention_weight_min": -30.050752639770508,
      "activations/layer14_attention_weight_max": 41.21760177612305,
      "activations/layer14_attention_weight_min": -31.615598678588867,
      "activations/layer15_attention_weight_max": 39.1425666809082,
      "activations/layer15_attention_weight_min": -32.162235260009766,
      "activations/layer16_attention_weight_max": 37.13300323486328,
      "activations/layer16_attention_weight_min": -30.883678436279297,
      "activations/layer17_attention_weight_max": 52.57818603515625,
      "activations/layer17_attention_weight_min": -44.78371047973633,
      "activations/layer18_attention_weight_max": 52.498592376708984,
      "activations/layer18_attention_weight_min": -44.27901077270508,
      "activations/layer19_attention_weight_max": 27.14373779296875,
      "activations/layer19_attention_weight_min": -27.319116592407227,
      "activations/layer1_attention_weight_max": 15.734450340270996,
      "activations/layer1_attention_weight_min": -14.401215553283691,
      "activations/layer20_attention_weight_max": 26.899919509887695,
      "activations/layer20_attention_weight_min": -22.587329864501953,
      "activations/layer21_attention_weight_max": 40.760719299316406,
      "activations/layer21_attention_weight_min": -23.615478515625,
      "activations/layer22_attention_weight_max": 32.54831314086914,
      "activations/layer22_attention_weight_min": -27.45489501953125,
      "activations/layer23_attention_weight_max": 44.31847381591797,
      "activations/layer23_attention_weight_min": -25.101104736328125,
      "activations/layer2_attention_weight_max": 31.164236068725586,
      "activations/layer2_attention_weight_min": -29.22005271911621,
      "activations/layer3_attention_weight_max": 88.23078918457031,
      "activations/layer3_attention_weight_min": -86.9564208984375,
      "activations/layer4_attention_weight_max": 88.30618286132812,
      "activations/layer4_attention_weight_min": -88.817626953125,
      "activations/layer5_attention_weight_max": 62.97671127319336,
      "activations/layer5_attention_weight_min": -72.29682922363281,
      "activations/layer6_attention_weight_max": 52.34647750854492,
      "activations/layer6_attention_weight_min": -50.7866325378418,
      "activations/layer7_attention_weight_max": 72.91168212890625,
      "activations/layer7_attention_weight_min": -70.84992980957031,
      "activations/layer8_attention_weight_max": 50.88529586791992,
      "activations/layer8_attention_weight_min": -51.58423614501953,
      "activations/layer9_attention_weight_max": 55.06824493408203,
      "activations/layer9_attention_weight_min": -51.47758865356445,
      "epoch": 18.28,
      "learning_rate": 3.2464015151515154e-05,
      "loss": 2.7346,
      "step": 314550
    },
    {
      "activations/layer0_attention_weight_max": 15.61076831817627,
      "activations/layer0_attention_weight_min": -13.827706336975098,
      "activations/layer10_attention_weight_max": 38.836158752441406,
      "activations/layer10_attention_weight_min": -37.112510681152344,
      "activations/layer11_attention_weight_max": 33.835853576660156,
      "activations/layer11_attention_weight_min": -35.142242431640625,
      "activations/layer12_attention_weight_max": 18.995040893554688,
      "activations/layer12_attention_weight_min": -23.84185791015625,
      "activations/layer13_attention_weight_max": 34.64642333984375,
      "activations/layer13_attention_weight_min": -29.90479278564453,
      "activations/layer14_attention_weight_max": 33.54658126831055,
      "activations/layer14_attention_weight_min": -32.22063064575195,
      "activations/layer15_attention_weight_max": 33.218475341796875,
      "activations/layer15_attention_weight_min": -31.031187057495117,
      "activations/layer16_attention_weight_max": 33.0097770690918,
      "activations/layer16_attention_weight_min": -31.34684944152832,
      "activations/layer17_attention_weight_max": 50.42070770263672,
      "activations/layer17_attention_weight_min": -44.209041595458984,
      "activations/layer18_attention_weight_max": 46.57432556152344,
      "activations/layer18_attention_weight_min": -39.273033142089844,
      "activations/layer19_attention_weight_max": 22.407087326049805,
      "activations/layer19_attention_weight_min": -22.192514419555664,
      "activations/layer1_attention_weight_max": 15.463510513305664,
      "activations/layer1_attention_weight_min": -13.470524787902832,
      "activations/layer20_attention_weight_max": 21.657684326171875,
      "activations/layer20_attention_weight_min": -23.87384605407715,
      "activations/layer21_attention_weight_max": 38.28891372680664,
      "activations/layer21_attention_weight_min": -24.074186325073242,
      "activations/layer22_attention_weight_max": 29.067243576049805,
      "activations/layer22_attention_weight_min": -25.57254409790039,
      "activations/layer23_attention_weight_max": 39.33855438232422,
      "activations/layer23_attention_weight_min": -24.672460556030273,
      "activations/layer2_attention_weight_max": 30.99159812927246,
      "activations/layer2_attention_weight_min": -28.7945556640625,
      "activations/layer3_attention_weight_max": 89.06332397460938,
      "activations/layer3_attention_weight_min": -85.49591827392578,
      "activations/layer4_attention_weight_max": 86.41551971435547,
      "activations/layer4_attention_weight_min": -84.90355682373047,
      "activations/layer5_attention_weight_max": 61.441139221191406,
      "activations/layer5_attention_weight_min": -69.04676818847656,
      "activations/layer6_attention_weight_max": 51.0407600402832,
      "activations/layer6_attention_weight_min": -52.1035041809082,
      "activations/layer7_attention_weight_max": 63.20887756347656,
      "activations/layer7_attention_weight_min": -62.235538482666016,
      "activations/layer8_attention_weight_max": 47.75408935546875,
      "activations/layer8_attention_weight_min": -50.51015853881836,
      "activations/layer9_attention_weight_max": 43.5927734375,
      "activations/layer9_attention_weight_min": -47.35288619995117,
      "epoch": 18.28,
      "learning_rate": 3.2445075757575755e-05,
      "loss": 2.7226,
      "step": 314600
    },
    {
      "activations/layer0_attention_weight_max": 14.867823600769043,
      "activations/layer0_attention_weight_min": -13.421148300170898,
      "activations/layer10_attention_weight_max": 35.75327682495117,
      "activations/layer10_attention_weight_min": -34.838172912597656,
      "activations/layer11_attention_weight_max": 31.947385787963867,
      "activations/layer11_attention_weight_min": -34.47097396850586,
      "activations/layer12_attention_weight_max": 19.2458438873291,
      "activations/layer12_attention_weight_min": -23.133325576782227,
      "activations/layer13_attention_weight_max": 34.035858154296875,
      "activations/layer13_attention_weight_min": -27.87186622619629,
      "activations/layer14_attention_weight_max": 35.4938850402832,
      "activations/layer14_attention_weight_min": -30.95873260498047,
      "activations/layer15_attention_weight_max": 32.750370025634766,
      "activations/layer15_attention_weight_min": -31.509326934814453,
      "activations/layer16_attention_weight_max": 32.96122360229492,
      "activations/layer16_attention_weight_min": -31.77989959716797,
      "activations/layer17_attention_weight_max": 50.8723030090332,
      "activations/layer17_attention_weight_min": -47.26298141479492,
      "activations/layer18_attention_weight_max": 44.585853576660156,
      "activations/layer18_attention_weight_min": -39.542823791503906,
      "activations/layer19_attention_weight_max": 24.14883041381836,
      "activations/layer19_attention_weight_min": -22.424222946166992,
      "activations/layer1_attention_weight_max": 16.61094856262207,
      "activations/layer1_attention_weight_min": -14.458928108215332,
      "activations/layer20_attention_weight_max": 21.686546325683594,
      "activations/layer20_attention_weight_min": -21.518600463867188,
      "activations/layer21_attention_weight_max": 32.490882873535156,
      "activations/layer21_attention_weight_min": -24.27837562561035,
      "activations/layer22_attention_weight_max": 25.98625946044922,
      "activations/layer22_attention_weight_min": -25.702880859375,
      "activations/layer23_attention_weight_max": 34.31392288208008,
      "activations/layer23_attention_weight_min": -26.03781509399414,
      "activations/layer2_attention_weight_max": 32.581275939941406,
      "activations/layer2_attention_weight_min": -30.36385726928711,
      "activations/layer3_attention_weight_max": 85.14028930664062,
      "activations/layer3_attention_weight_min": -87.61238861083984,
      "activations/layer4_attention_weight_max": 82.22412872314453,
      "activations/layer4_attention_weight_min": -83.28983306884766,
      "activations/layer5_attention_weight_max": 60.171566009521484,
      "activations/layer5_attention_weight_min": -69.69969940185547,
      "activations/layer6_attention_weight_max": 48.62102127075195,
      "activations/layer6_attention_weight_min": -51.26240539550781,
      "activations/layer7_attention_weight_max": 61.26527786254883,
      "activations/layer7_attention_weight_min": -62.69567108154297,
      "activations/layer8_attention_weight_max": 46.70449447631836,
      "activations/layer8_attention_weight_min": -49.76314163208008,
      "activations/layer9_attention_weight_max": 42.62398910522461,
      "activations/layer9_attention_weight_min": -43.66951370239258,
      "epoch": 18.28,
      "learning_rate": 3.242613636363636e-05,
      "loss": 2.7242,
      "step": 314650
    },
    {
      "activations/layer0_attention_weight_max": 15.326361656188965,
      "activations/layer0_attention_weight_min": -13.469637870788574,
      "activations/layer10_attention_weight_max": 38.07793045043945,
      "activations/layer10_attention_weight_min": -36.16151428222656,
      "activations/layer11_attention_weight_max": 34.62810516357422,
      "activations/layer11_attention_weight_min": -36.694091796875,
      "activations/layer12_attention_weight_max": 18.78744888305664,
      "activations/layer12_attention_weight_min": -22.791845321655273,
      "activations/layer13_attention_weight_max": 37.311134338378906,
      "activations/layer13_attention_weight_min": -30.91969871520996,
      "activations/layer14_attention_weight_max": 36.591819763183594,
      "activations/layer14_attention_weight_min": -32.989498138427734,
      "activations/layer15_attention_weight_max": 32.535560607910156,
      "activations/layer15_attention_weight_min": -31.276660919189453,
      "activations/layer16_attention_weight_max": 33.0107307434082,
      "activations/layer16_attention_weight_min": -31.641881942749023,
      "activations/layer17_attention_weight_max": 53.43560028076172,
      "activations/layer17_attention_weight_min": -43.561466217041016,
      "activations/layer18_attention_weight_max": 43.6319694519043,
      "activations/layer18_attention_weight_min": -38.01850509643555,
      "activations/layer19_attention_weight_max": 23.21821403503418,
      "activations/layer19_attention_weight_min": -21.39138412475586,
      "activations/layer1_attention_weight_max": 15.493167877197266,
      "activations/layer1_attention_weight_min": -13.8491849899292,
      "activations/layer20_attention_weight_max": 23.11895179748535,
      "activations/layer20_attention_weight_min": -21.373716354370117,
      "activations/layer21_attention_weight_max": 38.038387298583984,
      "activations/layer21_attention_weight_min": -21.47394371032715,
      "activations/layer22_attention_weight_max": 27.89488410949707,
      "activations/layer22_attention_weight_min": -25.9755916595459,
      "activations/layer23_attention_weight_max": 37.18185043334961,
      "activations/layer23_attention_weight_min": -26.185361862182617,
      "activations/layer2_attention_weight_max": 30.46356964111328,
      "activations/layer2_attention_weight_min": -29.66371726989746,
      "activations/layer3_attention_weight_max": 83.9979476928711,
      "activations/layer3_attention_weight_min": -88.94873046875,
      "activations/layer4_attention_weight_max": 83.63677215576172,
      "activations/layer4_attention_weight_min": -84.54167938232422,
      "activations/layer5_attention_weight_max": 61.445655822753906,
      "activations/layer5_attention_weight_min": -73.34542083740234,
      "activations/layer6_attention_weight_max": 50.24497604370117,
      "activations/layer6_attention_weight_min": -48.84410858154297,
      "activations/layer7_attention_weight_max": 65.4301986694336,
      "activations/layer7_attention_weight_min": -64.22433471679688,
      "activations/layer8_attention_weight_max": 45.553653717041016,
      "activations/layer8_attention_weight_min": -47.49782180786133,
      "activations/layer9_attention_weight_max": 44.68583297729492,
      "activations/layer9_attention_weight_min": -46.02628707885742,
      "epoch": 18.29,
      "learning_rate": 3.2407196969696965e-05,
      "loss": 2.7468,
      "step": 314700
    },
    {
      "activations/layer0_attention_weight_max": 15.623414993286133,
      "activations/layer0_attention_weight_min": -13.499089241027832,
      "activations/layer10_attention_weight_max": 38.914024353027344,
      "activations/layer10_attention_weight_min": -37.380126953125,
      "activations/layer11_attention_weight_max": 36.206478118896484,
      "activations/layer11_attention_weight_min": -35.43260192871094,
      "activations/layer12_attention_weight_max": 19.31626319885254,
      "activations/layer12_attention_weight_min": -25.891542434692383,
      "activations/layer13_attention_weight_max": 39.634002685546875,
      "activations/layer13_attention_weight_min": -27.26283836364746,
      "activations/layer14_attention_weight_max": 35.98810958862305,
      "activations/layer14_attention_weight_min": -31.12224006652832,
      "activations/layer15_attention_weight_max": 36.041500091552734,
      "activations/layer15_attention_weight_min": -31.788829803466797,
      "activations/layer16_attention_weight_max": 37.68743896484375,
      "activations/layer16_attention_weight_min": -34.67195129394531,
      "activations/layer17_attention_weight_max": 54.96760940551758,
      "activations/layer17_attention_weight_min": -46.72607421875,
      "activations/layer18_attention_weight_max": 53.36273193359375,
      "activations/layer18_attention_weight_min": -43.17361831665039,
      "activations/layer19_attention_weight_max": 26.514734268188477,
      "activations/layer19_attention_weight_min": -25.206972122192383,
      "activations/layer1_attention_weight_max": 15.802586555480957,
      "activations/layer1_attention_weight_min": -14.048338890075684,
      "activations/layer20_attention_weight_max": 26.324983596801758,
      "activations/layer20_attention_weight_min": -23.615123748779297,
      "activations/layer21_attention_weight_max": 44.73622512817383,
      "activations/layer21_attention_weight_min": -20.66598892211914,
      "activations/layer22_attention_weight_max": 32.17317199707031,
      "activations/layer22_attention_weight_min": -28.21745491027832,
      "activations/layer23_attention_weight_max": 44.08641815185547,
      "activations/layer23_attention_weight_min": -26.242345809936523,
      "activations/layer2_attention_weight_max": 33.30106735229492,
      "activations/layer2_attention_weight_min": -30.968961715698242,
      "activations/layer3_attention_weight_max": 87.32052612304688,
      "activations/layer3_attention_weight_min": -86.8078384399414,
      "activations/layer4_attention_weight_max": 90.30237579345703,
      "activations/layer4_attention_weight_min": -85.15509796142578,
      "activations/layer5_attention_weight_max": 63.34504318237305,
      "activations/layer5_attention_weight_min": -74.51020050048828,
      "activations/layer6_attention_weight_max": 52.05506134033203,
      "activations/layer6_attention_weight_min": -51.77571105957031,
      "activations/layer7_attention_weight_max": 71.96292877197266,
      "activations/layer7_attention_weight_min": -65.17031860351562,
      "activations/layer8_attention_weight_max": 48.61210250854492,
      "activations/layer8_attention_weight_min": -52.07119369506836,
      "activations/layer9_attention_weight_max": 48.66654968261719,
      "activations/layer9_attention_weight_min": -44.733951568603516,
      "epoch": 18.29,
      "learning_rate": 3.238825757575757e-05,
      "loss": 2.7357,
      "step": 314750
    },
    {
      "activations/layer0_attention_weight_max": 15.474449157714844,
      "activations/layer0_attention_weight_min": -13.940197944641113,
      "activations/layer10_attention_weight_max": 39.951290130615234,
      "activations/layer10_attention_weight_min": -36.91987991333008,
      "activations/layer11_attention_weight_max": 34.026084899902344,
      "activations/layer11_attention_weight_min": -35.106285095214844,
      "activations/layer12_attention_weight_max": 17.304075241088867,
      "activations/layer12_attention_weight_min": -25.36886215209961,
      "activations/layer13_attention_weight_max": 33.40890121459961,
      "activations/layer13_attention_weight_min": -26.035131454467773,
      "activations/layer14_attention_weight_max": 31.125225067138672,
      "activations/layer14_attention_weight_min": -28.883665084838867,
      "activations/layer15_attention_weight_max": 28.041364669799805,
      "activations/layer15_attention_weight_min": -28.434871673583984,
      "activations/layer16_attention_weight_max": 30.24729347229004,
      "activations/layer16_attention_weight_min": -31.762027740478516,
      "activations/layer17_attention_weight_max": 45.42000198364258,
      "activations/layer17_attention_weight_min": -42.77094650268555,
      "activations/layer18_attention_weight_max": 42.18745040893555,
      "activations/layer18_attention_weight_min": -39.75176239013672,
      "activations/layer19_attention_weight_max": 22.858089447021484,
      "activations/layer19_attention_weight_min": -21.640180587768555,
      "activations/layer1_attention_weight_max": 16.391769409179688,
      "activations/layer1_attention_weight_min": -17.11568260192871,
      "activations/layer20_attention_weight_max": 22.966341018676758,
      "activations/layer20_attention_weight_min": -25.406469345092773,
      "activations/layer21_attention_weight_max": 35.52392578125,
      "activations/layer21_attention_weight_min": -23.6809139251709,
      "activations/layer22_attention_weight_max": 28.82369613647461,
      "activations/layer22_attention_weight_min": -25.046777725219727,
      "activations/layer23_attention_weight_max": 32.5429801940918,
      "activations/layer23_attention_weight_min": -23.84417724609375,
      "activations/layer2_attention_weight_max": 35.17722702026367,
      "activations/layer2_attention_weight_min": -30.872329711914062,
      "activations/layer3_attention_weight_max": 91.7681655883789,
      "activations/layer3_attention_weight_min": -87.90070343017578,
      "activations/layer4_attention_weight_max": 94.3681869506836,
      "activations/layer4_attention_weight_min": -90.25286102294922,
      "activations/layer5_attention_weight_max": 68.41101837158203,
      "activations/layer5_attention_weight_min": -76.08909606933594,
      "activations/layer6_attention_weight_max": 52.97913360595703,
      "activations/layer6_attention_weight_min": -53.10060501098633,
      "activations/layer7_attention_weight_max": 65.9830322265625,
      "activations/layer7_attention_weight_min": -68.46986389160156,
      "activations/layer8_attention_weight_max": 52.0239143371582,
      "activations/layer8_attention_weight_min": -51.10386657714844,
      "activations/layer9_attention_weight_max": 46.897212982177734,
      "activations/layer9_attention_weight_min": -44.726295471191406,
      "epoch": 18.29,
      "learning_rate": 3.236931818181818e-05,
      "loss": 2.719,
      "step": 314800
    },
    {
      "activations/layer0_attention_weight_max": 15.832064628601074,
      "activations/layer0_attention_weight_min": -13.673144340515137,
      "activations/layer10_attention_weight_max": 32.31690216064453,
      "activations/layer10_attention_weight_min": -33.91851806640625,
      "activations/layer11_attention_weight_max": 30.546520233154297,
      "activations/layer11_attention_weight_min": -35.40851974487305,
      "activations/layer12_attention_weight_max": 18.991901397705078,
      "activations/layer12_attention_weight_min": -24.2109375,
      "activations/layer13_attention_weight_max": 35.48847198486328,
      "activations/layer13_attention_weight_min": -27.20082664489746,
      "activations/layer14_attention_weight_max": 36.51808547973633,
      "activations/layer14_attention_weight_min": -30.3559513092041,
      "activations/layer15_attention_weight_max": 32.100852966308594,
      "activations/layer15_attention_weight_min": -30.3862247467041,
      "activations/layer16_attention_weight_max": 33.61370086669922,
      "activations/layer16_attention_weight_min": -31.95792579650879,
      "activations/layer17_attention_weight_max": 49.943634033203125,
      "activations/layer17_attention_weight_min": -44.63444519042969,
      "activations/layer18_attention_weight_max": 48.216983795166016,
      "activations/layer18_attention_weight_min": -41.17545700073242,
      "activations/layer19_attention_weight_max": 23.8044490814209,
      "activations/layer19_attention_weight_min": -24.535602569580078,
      "activations/layer1_attention_weight_max": 15.597403526306152,
      "activations/layer1_attention_weight_min": -13.727652549743652,
      "activations/layer20_attention_weight_max": 23.25914192199707,
      "activations/layer20_attention_weight_min": -25.72682762145996,
      "activations/layer21_attention_weight_max": 38.6412467956543,
      "activations/layer21_attention_weight_min": -24.845834732055664,
      "activations/layer22_attention_weight_max": 29.542804718017578,
      "activations/layer22_attention_weight_min": -26.60200309753418,
      "activations/layer23_attention_weight_max": 40.38532638549805,
      "activations/layer23_attention_weight_min": -23.417980194091797,
      "activations/layer2_attention_weight_max": 30.114110946655273,
      "activations/layer2_attention_weight_min": -30.791610717773438,
      "activations/layer3_attention_weight_max": 83.11112976074219,
      "activations/layer3_attention_weight_min": -86.5462646484375,
      "activations/layer4_attention_weight_max": 78.32572937011719,
      "activations/layer4_attention_weight_min": -80.37667083740234,
      "activations/layer5_attention_weight_max": 57.80804443359375,
      "activations/layer5_attention_weight_min": -72.05281066894531,
      "activations/layer6_attention_weight_max": 45.1356315612793,
      "activations/layer6_attention_weight_min": -50.34209442138672,
      "activations/layer7_attention_weight_max": 58.223182678222656,
      "activations/layer7_attention_weight_min": -60.74656677246094,
      "activations/layer8_attention_weight_max": 44.4626579284668,
      "activations/layer8_attention_weight_min": -46.66746139526367,
      "activations/layer9_attention_weight_max": 41.273040771484375,
      "activations/layer9_attention_weight_min": -42.43128204345703,
      "epoch": 18.29,
      "learning_rate": 3.235037878787879e-05,
      "loss": 2.722,
      "step": 314850
    },
    {
      "activations/layer0_attention_weight_max": 15.38308048248291,
      "activations/layer0_attention_weight_min": -13.45933723449707,
      "activations/layer10_attention_weight_max": 35.62023162841797,
      "activations/layer10_attention_weight_min": -35.07049560546875,
      "activations/layer11_attention_weight_max": 31.4312801361084,
      "activations/layer11_attention_weight_min": -33.7509651184082,
      "activations/layer12_attention_weight_max": 19.645893096923828,
      "activations/layer12_attention_weight_min": -23.765222549438477,
      "activations/layer13_attention_weight_max": 31.73814582824707,
      "activations/layer13_attention_weight_min": -27.499507904052734,
      "activations/layer14_attention_weight_max": 35.79938507080078,
      "activations/layer14_attention_weight_min": -28.530271530151367,
      "activations/layer15_attention_weight_max": 30.9013614654541,
      "activations/layer15_attention_weight_min": -28.786293029785156,
      "activations/layer16_attention_weight_max": 31.93756866455078,
      "activations/layer16_attention_weight_min": -30.86545753479004,
      "activations/layer17_attention_weight_max": 49.318153381347656,
      "activations/layer17_attention_weight_min": -43.13383102416992,
      "activations/layer18_attention_weight_max": 44.61856460571289,
      "activations/layer18_attention_weight_min": -38.54336166381836,
      "activations/layer19_attention_weight_max": 22.548229217529297,
      "activations/layer19_attention_weight_min": -23.087844848632812,
      "activations/layer1_attention_weight_max": 15.29393482208252,
      "activations/layer1_attention_weight_min": -14.132905960083008,
      "activations/layer20_attention_weight_max": 22.39775848388672,
      "activations/layer20_attention_weight_min": -23.58397102355957,
      "activations/layer21_attention_weight_max": 34.43223571777344,
      "activations/layer21_attention_weight_min": -25.047149658203125,
      "activations/layer22_attention_weight_max": 27.9818058013916,
      "activations/layer22_attention_weight_min": -24.910024642944336,
      "activations/layer23_attention_weight_max": 32.91191101074219,
      "activations/layer23_attention_weight_min": -23.31646728515625,
      "activations/layer2_attention_weight_max": 33.41632080078125,
      "activations/layer2_attention_weight_min": -30.282283782958984,
      "activations/layer3_attention_weight_max": 87.18130493164062,
      "activations/layer3_attention_weight_min": -86.10822296142578,
      "activations/layer4_attention_weight_max": 85.4331283569336,
      "activations/layer4_attention_weight_min": -84.60037994384766,
      "activations/layer5_attention_weight_max": 61.85724639892578,
      "activations/layer5_attention_weight_min": -75.92567443847656,
      "activations/layer6_attention_weight_max": 49.568458557128906,
      "activations/layer6_attention_weight_min": -52.8546257019043,
      "activations/layer7_attention_weight_max": 66.94364929199219,
      "activations/layer7_attention_weight_min": -63.82382583618164,
      "activations/layer8_attention_weight_max": 46.506690979003906,
      "activations/layer8_attention_weight_min": -48.37141799926758,
      "activations/layer9_attention_weight_max": 42.63113021850586,
      "activations/layer9_attention_weight_min": -44.42185592651367,
      "epoch": 18.3,
      "learning_rate": 3.233143939393939e-05,
      "loss": 2.7456,
      "step": 314900
    },
    {
      "activations/layer0_attention_weight_max": 15.594755172729492,
      "activations/layer0_attention_weight_min": -13.543648719787598,
      "activations/layer10_attention_weight_max": 39.28408432006836,
      "activations/layer10_attention_weight_min": -37.60296630859375,
      "activations/layer11_attention_weight_max": 36.70136260986328,
      "activations/layer11_attention_weight_min": -35.617149353027344,
      "activations/layer12_attention_weight_max": 18.890634536743164,
      "activations/layer12_attention_weight_min": -25.733049392700195,
      "activations/layer13_attention_weight_max": 36.708885192871094,
      "activations/layer13_attention_weight_min": -28.983400344848633,
      "activations/layer14_attention_weight_max": 37.869693756103516,
      "activations/layer14_attention_weight_min": -31.69463539123535,
      "activations/layer15_attention_weight_max": 34.78044509887695,
      "activations/layer15_attention_weight_min": -30.649656295776367,
      "activations/layer16_attention_weight_max": 35.05494689941406,
      "activations/layer16_attention_weight_min": -32.07242965698242,
      "activations/layer17_attention_weight_max": 52.17867660522461,
      "activations/layer17_attention_weight_min": -45.71304702758789,
      "activations/layer18_attention_weight_max": 55.1968879699707,
      "activations/layer18_attention_weight_min": -40.58760070800781,
      "activations/layer19_attention_weight_max": 26.238418579101562,
      "activations/layer19_attention_weight_min": -25.08395004272461,
      "activations/layer1_attention_weight_max": 16.601974487304688,
      "activations/layer1_attention_weight_min": -17.390098571777344,
      "activations/layer20_attention_weight_max": 27.859968185424805,
      "activations/layer20_attention_weight_min": -22.620149612426758,
      "activations/layer21_attention_weight_max": 46.8138427734375,
      "activations/layer21_attention_weight_min": -25.087350845336914,
      "activations/layer22_attention_weight_max": 31.500526428222656,
      "activations/layer22_attention_weight_min": -25.620975494384766,
      "activations/layer23_attention_weight_max": 41.34126281738281,
      "activations/layer23_attention_weight_min": -24.660629272460938,
      "activations/layer2_attention_weight_max": 33.762535095214844,
      "activations/layer2_attention_weight_min": -33.01137924194336,
      "activations/layer3_attention_weight_max": 92.1967544555664,
      "activations/layer3_attention_weight_min": -94.52677154541016,
      "activations/layer4_attention_weight_max": 87.84563446044922,
      "activations/layer4_attention_weight_min": -88.0077896118164,
      "activations/layer5_attention_weight_max": 64.65788269042969,
      "activations/layer5_attention_weight_min": -74.36724853515625,
      "activations/layer6_attention_weight_max": 51.689369201660156,
      "activations/layer6_attention_weight_min": -51.692996978759766,
      "activations/layer7_attention_weight_max": 66.32926177978516,
      "activations/layer7_attention_weight_min": -63.639854431152344,
      "activations/layer8_attention_weight_max": 50.28032302856445,
      "activations/layer8_attention_weight_min": -52.85307693481445,
      "activations/layer9_attention_weight_max": 55.20545196533203,
      "activations/layer9_attention_weight_min": -48.80564880371094,
      "epoch": 18.3,
      "learning_rate": 3.23125e-05,
      "loss": 2.7641,
      "step": 314950
    },
    {
      "activations/layer0_attention_weight_max": 15.052535057067871,
      "activations/layer0_attention_weight_min": -14.276127815246582,
      "activations/layer10_attention_weight_max": 35.6949348449707,
      "activations/layer10_attention_weight_min": -34.93375015258789,
      "activations/layer11_attention_weight_max": 33.828372955322266,
      "activations/layer11_attention_weight_min": -36.45811462402344,
      "activations/layer12_attention_weight_max": 20.1925106048584,
      "activations/layer12_attention_weight_min": -24.816129684448242,
      "activations/layer13_attention_weight_max": 36.6513671875,
      "activations/layer13_attention_weight_min": -29.45564842224121,
      "activations/layer14_attention_weight_max": 37.2498664855957,
      "activations/layer14_attention_weight_min": -32.41392517089844,
      "activations/layer15_attention_weight_max": 34.83457565307617,
      "activations/layer15_attention_weight_min": -29.181121826171875,
      "activations/layer16_attention_weight_max": 34.304195404052734,
      "activations/layer16_attention_weight_min": -33.0856819152832,
      "activations/layer17_attention_weight_max": 51.4172477722168,
      "activations/layer17_attention_weight_min": -45.681236267089844,
      "activations/layer18_attention_weight_max": 48.84947967529297,
      "activations/layer18_attention_weight_min": -41.9572868347168,
      "activations/layer19_attention_weight_max": 25.342924118041992,
      "activations/layer19_attention_weight_min": -23.26832389831543,
      "activations/layer1_attention_weight_max": 16.314762115478516,
      "activations/layer1_attention_weight_min": -14.210101127624512,
      "activations/layer20_attention_weight_max": 25.27566146850586,
      "activations/layer20_attention_weight_min": -21.70116424560547,
      "activations/layer21_attention_weight_max": 38.778907775878906,
      "activations/layer21_attention_weight_min": -27.884098052978516,
      "activations/layer22_attention_weight_max": 30.91634750366211,
      "activations/layer22_attention_weight_min": -25.749576568603516,
      "activations/layer23_attention_weight_max": 40.71910858154297,
      "activations/layer23_attention_weight_min": -25.74555778503418,
      "activations/layer2_attention_weight_max": 31.149099349975586,
      "activations/layer2_attention_weight_min": -31.522872924804688,
      "activations/layer3_attention_weight_max": 88.3985595703125,
      "activations/layer3_attention_weight_min": -89.51988220214844,
      "activations/layer4_attention_weight_max": 88.8038330078125,
      "activations/layer4_attention_weight_min": -86.47293090820312,
      "activations/layer5_attention_weight_max": 64.9638900756836,
      "activations/layer5_attention_weight_min": -71.89298248291016,
      "activations/layer6_attention_weight_max": 50.649574279785156,
      "activations/layer6_attention_weight_min": -53.71317672729492,
      "activations/layer7_attention_weight_max": 67.43215942382812,
      "activations/layer7_attention_weight_min": -65.73067474365234,
      "activations/layer8_attention_weight_max": 44.894989013671875,
      "activations/layer8_attention_weight_min": -49.13990020751953,
      "activations/layer9_attention_weight_max": 43.89957809448242,
      "activations/layer9_attention_weight_min": -46.82181930541992,
      "epoch": 18.3,
      "learning_rate": 3.22935606060606e-05,
      "loss": 2.7318,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4436,
      "eval_samples_per_second": 508.552,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4436,
      "eval_openwebtext_samples_per_second": 508.552,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 1.9741,
      "eval_wikitext_samples_per_second": 230.995,
      "step": 315000
    },
    {
      "epoch": 18.3,
      "eval_lambada_loss": 2.50390625,
      "eval_lambada_ppl": 12.230174893930755,
      "eval_lambada_runtime": 9.55,
      "eval_lambada_samples_per_second": 509.844,
      "step": 315000
    },
    {
      "activations/layer0_attention_weight_max": 15.043033599853516,
      "activations/layer0_attention_weight_min": -13.537505149841309,
      "activations/layer10_attention_weight_max": 37.43068313598633,
      "activations/layer10_attention_weight_min": -37.240108489990234,
      "activations/layer11_attention_weight_max": 33.07064437866211,
      "activations/layer11_attention_weight_min": -34.81794357299805,
      "activations/layer12_attention_weight_max": 17.519027709960938,
      "activations/layer12_attention_weight_min": -26.128843307495117,
      "activations/layer13_attention_weight_max": 39.779598236083984,
      "activations/layer13_attention_weight_min": -29.487377166748047,
      "activations/layer14_attention_weight_max": 36.9406623840332,
      "activations/layer14_attention_weight_min": -29.080944061279297,
      "activations/layer15_attention_weight_max": 35.00168228149414,
      "activations/layer15_attention_weight_min": -29.464706420898438,
      "activations/layer16_attention_weight_max": 30.863094329833984,
      "activations/layer16_attention_weight_min": -28.60939598083496,
      "activations/layer17_attention_weight_max": 51.31898880004883,
      "activations/layer17_attention_weight_min": -45.277870178222656,
      "activations/layer18_attention_weight_max": 42.051082611083984,
      "activations/layer18_attention_weight_min": -42.23827362060547,
      "activations/layer19_attention_weight_max": 22.656599044799805,
      "activations/layer19_attention_weight_min": -23.996828079223633,
      "activations/layer1_attention_weight_max": 15.367440223693848,
      "activations/layer1_attention_weight_min": -14.499937057495117,
      "activations/layer20_attention_weight_max": 20.741479873657227,
      "activations/layer20_attention_weight_min": -22.260452270507812,
      "activations/layer21_attention_weight_max": 36.284080505371094,
      "activations/layer21_attention_weight_min": -23.600488662719727,
      "activations/layer22_attention_weight_max": 28.212623596191406,
      "activations/layer22_attention_weight_min": -25.1043701171875,
      "activations/layer23_attention_weight_max": 37.495689392089844,
      "activations/layer23_attention_weight_min": -27.558815002441406,
      "activations/layer2_attention_weight_max": 33.461429595947266,
      "activations/layer2_attention_weight_min": -33.2568244934082,
      "activations/layer3_attention_weight_max": 82.60218048095703,
      "activations/layer3_attention_weight_min": -88.03933715820312,
      "activations/layer4_attention_weight_max": 76.4547119140625,
      "activations/layer4_attention_weight_min": -77.55767822265625,
      "activations/layer5_attention_weight_max": 60.21662521362305,
      "activations/layer5_attention_weight_min": -75.72441101074219,
      "activations/layer6_attention_weight_max": 44.11479949951172,
      "activations/layer6_attention_weight_min": -46.077701568603516,
      "activations/layer7_attention_weight_max": 60.29580307006836,
      "activations/layer7_attention_weight_min": -61.274051666259766,
      "activations/layer8_attention_weight_max": 46.02900314331055,
      "activations/layer8_attention_weight_min": -47.19983673095703,
      "activations/layer9_attention_weight_max": 47.3521842956543,
      "activations/layer9_attention_weight_min": -46.63711929321289,
      "epoch": 18.31,
      "learning_rate": 3.227462121212121e-05,
      "loss": 2.7223,
      "step": 315050
    },
    {
      "activations/layer0_attention_weight_max": 15.23156452178955,
      "activations/layer0_attention_weight_min": -13.668182373046875,
      "activations/layer10_attention_weight_max": 36.140235900878906,
      "activations/layer10_attention_weight_min": -36.04511260986328,
      "activations/layer11_attention_weight_max": 32.80083084106445,
      "activations/layer11_attention_weight_min": -35.84727478027344,
      "activations/layer12_attention_weight_max": 19.43155288696289,
      "activations/layer12_attention_weight_min": -24.34596061706543,
      "activations/layer13_attention_weight_max": 35.471229553222656,
      "activations/layer13_attention_weight_min": -29.396648406982422,
      "activations/layer14_attention_weight_max": 36.588157653808594,
      "activations/layer14_attention_weight_min": -30.9974365234375,
      "activations/layer15_attention_weight_max": 32.660682678222656,
      "activations/layer15_attention_weight_min": -29.688215255737305,
      "activations/layer16_attention_weight_max": 33.349124908447266,
      "activations/layer16_attention_weight_min": -30.766080856323242,
      "activations/layer17_attention_weight_max": 52.03805923461914,
      "activations/layer17_attention_weight_min": -44.722599029541016,
      "activations/layer18_attention_weight_max": 53.00979995727539,
      "activations/layer18_attention_weight_min": -40.54906463623047,
      "activations/layer19_attention_weight_max": 26.298587799072266,
      "activations/layer19_attention_weight_min": -23.057279586791992,
      "activations/layer1_attention_weight_max": 16.041921615600586,
      "activations/layer1_attention_weight_min": -13.838557243347168,
      "activations/layer20_attention_weight_max": 25.75157928466797,
      "activations/layer20_attention_weight_min": -24.023977279663086,
      "activations/layer21_attention_weight_max": 42.2504997253418,
      "activations/layer21_attention_weight_min": -24.10129165649414,
      "activations/layer22_attention_weight_max": 31.648298263549805,
      "activations/layer22_attention_weight_min": -26.938955307006836,
      "activations/layer23_attention_weight_max": 34.195709228515625,
      "activations/layer23_attention_weight_min": -25.571561813354492,
      "activations/layer2_attention_weight_max": 31.771255493164062,
      "activations/layer2_attention_weight_min": -29.353803634643555,
      "activations/layer3_attention_weight_max": 86.92318725585938,
      "activations/layer3_attention_weight_min": -85.48280334472656,
      "activations/layer4_attention_weight_max": 83.55901336669922,
      "activations/layer4_attention_weight_min": -88.62020111083984,
      "activations/layer5_attention_weight_max": 70.21524810791016,
      "activations/layer5_attention_weight_min": -74.52342987060547,
      "activations/layer6_attention_weight_max": 50.24951934814453,
      "activations/layer6_attention_weight_min": -51.73165512084961,
      "activations/layer7_attention_weight_max": 67.34778594970703,
      "activations/layer7_attention_weight_min": -66.6637954711914,
      "activations/layer8_attention_weight_max": 46.234230041503906,
      "activations/layer8_attention_weight_min": -50.80867385864258,
      "activations/layer9_attention_weight_max": 48.66199493408203,
      "activations/layer9_attention_weight_min": -46.48649597167969,
      "epoch": 18.31,
      "learning_rate": 3.225568181818181e-05,
      "loss": 2.7333,
      "step": 315100
    },
    {
      "activations/layer0_attention_weight_max": 15.063982009887695,
      "activations/layer0_attention_weight_min": -13.643492698669434,
      "activations/layer10_attention_weight_max": 36.59064865112305,
      "activations/layer10_attention_weight_min": -36.470924377441406,
      "activations/layer11_attention_weight_max": 34.776424407958984,
      "activations/layer11_attention_weight_min": -36.1013298034668,
      "activations/layer12_attention_weight_max": 18.997098922729492,
      "activations/layer12_attention_weight_min": -24.20058822631836,
      "activations/layer13_attention_weight_max": 34.49076461791992,
      "activations/layer13_attention_weight_min": -31.71832847595215,
      "activations/layer14_attention_weight_max": 32.93328094482422,
      "activations/layer14_attention_weight_min": -31.879680633544922,
      "activations/layer15_attention_weight_max": 32.064178466796875,
      "activations/layer15_attention_weight_min": -29.763582229614258,
      "activations/layer16_attention_weight_max": 30.82916259765625,
      "activations/layer16_attention_weight_min": -31.03367042541504,
      "activations/layer17_attention_weight_max": 47.03727340698242,
      "activations/layer17_attention_weight_min": -45.722984313964844,
      "activations/layer18_attention_weight_max": 44.15715789794922,
      "activations/layer18_attention_weight_min": -41.78081512451172,
      "activations/layer19_attention_weight_max": 24.28553581237793,
      "activations/layer19_attention_weight_min": -21.92922019958496,
      "activations/layer1_attention_weight_max": 15.973797798156738,
      "activations/layer1_attention_weight_min": -13.23918628692627,
      "activations/layer20_attention_weight_max": 24.148399353027344,
      "activations/layer20_attention_weight_min": -20.405858993530273,
      "activations/layer21_attention_weight_max": 33.51298522949219,
      "activations/layer21_attention_weight_min": -22.97295379638672,
      "activations/layer22_attention_weight_max": 26.525774002075195,
      "activations/layer22_attention_weight_min": -25.40146255493164,
      "activations/layer23_attention_weight_max": 35.626869201660156,
      "activations/layer23_attention_weight_min": -24.39571762084961,
      "activations/layer2_attention_weight_max": 35.53978729248047,
      "activations/layer2_attention_weight_min": -34.13936996459961,
      "activations/layer3_attention_weight_max": 89.5345230102539,
      "activations/layer3_attention_weight_min": -93.85152435302734,
      "activations/layer4_attention_weight_max": 83.80717468261719,
      "activations/layer4_attention_weight_min": -83.81549835205078,
      "activations/layer5_attention_weight_max": 60.57231903076172,
      "activations/layer5_attention_weight_min": -74.69342041015625,
      "activations/layer6_attention_weight_max": 50.47636413574219,
      "activations/layer6_attention_weight_min": -52.39356994628906,
      "activations/layer7_attention_weight_max": 65.18743896484375,
      "activations/layer7_attention_weight_min": -68.11270141601562,
      "activations/layer8_attention_weight_max": 50.29414367675781,
      "activations/layer8_attention_weight_min": -50.08305740356445,
      "activations/layer9_attention_weight_max": 48.34395980834961,
      "activations/layer9_attention_weight_min": -48.02018356323242,
      "epoch": 18.31,
      "learning_rate": 3.223674242424242e-05,
      "loss": 2.7327,
      "step": 315150
    },
    {
      "activations/layer0_attention_weight_max": 15.387923240661621,
      "activations/layer0_attention_weight_min": -13.35639762878418,
      "activations/layer10_attention_weight_max": 36.739131927490234,
      "activations/layer10_attention_weight_min": -35.910057067871094,
      "activations/layer11_attention_weight_max": 34.06360626220703,
      "activations/layer11_attention_weight_min": -35.53809356689453,
      "activations/layer12_attention_weight_max": 21.400012969970703,
      "activations/layer12_attention_weight_min": -26.05401039123535,
      "activations/layer13_attention_weight_max": 31.16611671447754,
      "activations/layer13_attention_weight_min": -27.46358871459961,
      "activations/layer14_attention_weight_max": 34.241371154785156,
      "activations/layer14_attention_weight_min": -29.10039710998535,
      "activations/layer15_attention_weight_max": 34.091094970703125,
      "activations/layer15_attention_weight_min": -28.4648494720459,
      "activations/layer16_attention_weight_max": 32.62313461303711,
      "activations/layer16_attention_weight_min": -30.123794555664062,
      "activations/layer17_attention_weight_max": 50.57470703125,
      "activations/layer17_attention_weight_min": -44.61174774169922,
      "activations/layer18_attention_weight_max": 47.63573455810547,
      "activations/layer18_attention_weight_min": -40.323570251464844,
      "activations/layer19_attention_weight_max": 22.532508850097656,
      "activations/layer19_attention_weight_min": -22.543275833129883,
      "activations/layer1_attention_weight_max": 15.777032852172852,
      "activations/layer1_attention_weight_min": -13.189167022705078,
      "activations/layer20_attention_weight_max": 21.99945640563965,
      "activations/layer20_attention_weight_min": -21.52414321899414,
      "activations/layer21_attention_weight_max": 33.349281311035156,
      "activations/layer21_attention_weight_min": -18.920650482177734,
      "activations/layer22_attention_weight_max": 29.301010131835938,
      "activations/layer22_attention_weight_min": -24.8818359375,
      "activations/layer23_attention_weight_max": 33.38542175292969,
      "activations/layer23_attention_weight_min": -23.388242721557617,
      "activations/layer2_attention_weight_max": 31.575544357299805,
      "activations/layer2_attention_weight_min": -27.78447723388672,
      "activations/layer3_attention_weight_max": 84.16351318359375,
      "activations/layer3_attention_weight_min": -81.6323013305664,
      "activations/layer4_attention_weight_max": 83.72783660888672,
      "activations/layer4_attention_weight_min": -83.55814361572266,
      "activations/layer5_attention_weight_max": 60.91134262084961,
      "activations/layer5_attention_weight_min": -72.12448120117188,
      "activations/layer6_attention_weight_max": 46.7408332824707,
      "activations/layer6_attention_weight_min": -49.47761535644531,
      "activations/layer7_attention_weight_max": 60.64244079589844,
      "activations/layer7_attention_weight_min": -65.32467651367188,
      "activations/layer8_attention_weight_max": 46.6506462097168,
      "activations/layer8_attention_weight_min": -50.14532470703125,
      "activations/layer9_attention_weight_max": 43.926902770996094,
      "activations/layer9_attention_weight_min": -45.888057708740234,
      "epoch": 18.31,
      "learning_rate": 3.221780303030303e-05,
      "loss": 2.7196,
      "step": 315200
    },
    {
      "activations/layer0_attention_weight_max": 15.354264259338379,
      "activations/layer0_attention_weight_min": -13.73050308227539,
      "activations/layer10_attention_weight_max": 37.571922302246094,
      "activations/layer10_attention_weight_min": -35.73759078979492,
      "activations/layer11_attention_weight_max": 35.533138275146484,
      "activations/layer11_attention_weight_min": -35.6985969543457,
      "activations/layer12_attention_weight_max": 19.485111236572266,
      "activations/layer12_attention_weight_min": -26.726587295532227,
      "activations/layer13_attention_weight_max": 36.444496154785156,
      "activations/layer13_attention_weight_min": -30.3721981048584,
      "activations/layer14_attention_weight_max": 37.681373596191406,
      "activations/layer14_attention_weight_min": -32.91807174682617,
      "activations/layer15_attention_weight_max": 31.46677017211914,
      "activations/layer15_attention_weight_min": -34.031288146972656,
      "activations/layer16_attention_weight_max": 33.157169342041016,
      "activations/layer16_attention_weight_min": -33.86470031738281,
      "activations/layer17_attention_weight_max": 52.9399299621582,
      "activations/layer17_attention_weight_min": -47.392635345458984,
      "activations/layer18_attention_weight_max": 46.06122589111328,
      "activations/layer18_attention_weight_min": -41.75498580932617,
      "activations/layer19_attention_weight_max": 22.242042541503906,
      "activations/layer19_attention_weight_min": -22.332162857055664,
      "activations/layer1_attention_weight_max": 16.73242950439453,
      "activations/layer1_attention_weight_min": -13.047091484069824,
      "activations/layer20_attention_weight_max": 22.471837997436523,
      "activations/layer20_attention_weight_min": -23.599037170410156,
      "activations/layer21_attention_weight_max": 34.92571258544922,
      "activations/layer21_attention_weight_min": -24.71059226989746,
      "activations/layer22_attention_weight_max": 28.6962947845459,
      "activations/layer22_attention_weight_min": -24.39316749572754,
      "activations/layer23_attention_weight_max": 36.54019546508789,
      "activations/layer23_attention_weight_min": -24.209922790527344,
      "activations/layer2_attention_weight_max": 31.720678329467773,
      "activations/layer2_attention_weight_min": -29.446205139160156,
      "activations/layer3_attention_weight_max": 88.41656494140625,
      "activations/layer3_attention_weight_min": -84.2677001953125,
      "activations/layer4_attention_weight_max": 84.3288345336914,
      "activations/layer4_attention_weight_min": -82.78994750976562,
      "activations/layer5_attention_weight_max": 61.610633850097656,
      "activations/layer5_attention_weight_min": -76.515869140625,
      "activations/layer6_attention_weight_max": 52.48781204223633,
      "activations/layer6_attention_weight_min": -50.746219635009766,
      "activations/layer7_attention_weight_max": 63.91531753540039,
      "activations/layer7_attention_weight_min": -67.3830337524414,
      "activations/layer8_attention_weight_max": 48.427791595458984,
      "activations/layer8_attention_weight_min": -48.8124885559082,
      "activations/layer9_attention_weight_max": 44.65169143676758,
      "activations/layer9_attention_weight_min": -44.98635482788086,
      "epoch": 18.32,
      "learning_rate": 3.2198863636363635e-05,
      "loss": 2.7505,
      "step": 315250
    },
    {
      "activations/layer0_attention_weight_max": 14.595077514648438,
      "activations/layer0_attention_weight_min": -13.394233703613281,
      "activations/layer10_attention_weight_max": 34.55177688598633,
      "activations/layer10_attention_weight_min": -34.89461135864258,
      "activations/layer11_attention_weight_max": 34.634246826171875,
      "activations/layer11_attention_weight_min": -33.01019287109375,
      "activations/layer12_attention_weight_max": 19.202383041381836,
      "activations/layer12_attention_weight_min": -23.963163375854492,
      "activations/layer13_attention_weight_max": 35.32048797607422,
      "activations/layer13_attention_weight_min": -27.592809677124023,
      "activations/layer14_attention_weight_max": 34.85944366455078,
      "activations/layer14_attention_weight_min": -29.4614315032959,
      "activations/layer15_attention_weight_max": 33.69302749633789,
      "activations/layer15_attention_weight_min": -29.939374923706055,
      "activations/layer16_attention_weight_max": 32.952152252197266,
      "activations/layer16_attention_weight_min": -30.752286911010742,
      "activations/layer17_attention_weight_max": 54.72563171386719,
      "activations/layer17_attention_weight_min": -46.05767822265625,
      "activations/layer18_attention_weight_max": 50.50145721435547,
      "activations/layer18_attention_weight_min": -42.992191314697266,
      "activations/layer19_attention_weight_max": 24.94594955444336,
      "activations/layer19_attention_weight_min": -25.779239654541016,
      "activations/layer1_attention_weight_max": 16.9916934967041,
      "activations/layer1_attention_weight_min": -14.962526321411133,
      "activations/layer20_attention_weight_max": 25.638303756713867,
      "activations/layer20_attention_weight_min": -25.557666778564453,
      "activations/layer21_attention_weight_max": 39.44473648071289,
      "activations/layer21_attention_weight_min": -26.426727294921875,
      "activations/layer22_attention_weight_max": 29.334897994995117,
      "activations/layer22_attention_weight_min": -26.306320190429688,
      "activations/layer23_attention_weight_max": 39.494041442871094,
      "activations/layer23_attention_weight_min": -23.95966339111328,
      "activations/layer2_attention_weight_max": 33.126426696777344,
      "activations/layer2_attention_weight_min": -31.633827209472656,
      "activations/layer3_attention_weight_max": 85.71163177490234,
      "activations/layer3_attention_weight_min": -82.01898956298828,
      "activations/layer4_attention_weight_max": 81.45735931396484,
      "activations/layer4_attention_weight_min": -79.59400939941406,
      "activations/layer5_attention_weight_max": 60.099578857421875,
      "activations/layer5_attention_weight_min": -70.69518280029297,
      "activations/layer6_attention_weight_max": 48.39419174194336,
      "activations/layer6_attention_weight_min": -50.342525482177734,
      "activations/layer7_attention_weight_max": 61.33320617675781,
      "activations/layer7_attention_weight_min": -64.72637176513672,
      "activations/layer8_attention_weight_max": 43.87269592285156,
      "activations/layer8_attention_weight_min": -46.974735260009766,
      "activations/layer9_attention_weight_max": 43.4863395690918,
      "activations/layer9_attention_weight_min": -45.712032318115234,
      "epoch": 18.32,
      "learning_rate": 3.217992424242424e-05,
      "loss": 2.7288,
      "step": 315300
    },
    {
      "activations/layer0_attention_weight_max": 15.371421813964844,
      "activations/layer0_attention_weight_min": -13.733380317687988,
      "activations/layer10_attention_weight_max": 38.62591552734375,
      "activations/layer10_attention_weight_min": -36.990596771240234,
      "activations/layer11_attention_weight_max": 33.634647369384766,
      "activations/layer11_attention_weight_min": -35.960182189941406,
      "activations/layer12_attention_weight_max": 19.465177536010742,
      "activations/layer12_attention_weight_min": -24.228179931640625,
      "activations/layer13_attention_weight_max": 34.78295135498047,
      "activations/layer13_attention_weight_min": -29.678903579711914,
      "activations/layer14_attention_weight_max": 33.949554443359375,
      "activations/layer14_attention_weight_min": -30.195524215698242,
      "activations/layer15_attention_weight_max": 33.79731369018555,
      "activations/layer15_attention_weight_min": -29.14096450805664,
      "activations/layer16_attention_weight_max": 34.04460144042969,
      "activations/layer16_attention_weight_min": -30.22203254699707,
      "activations/layer17_attention_weight_max": 51.19401168823242,
      "activations/layer17_attention_weight_min": -44.949302673339844,
      "activations/layer18_attention_weight_max": 51.45620346069336,
      "activations/layer18_attention_weight_min": -40.555397033691406,
      "activations/layer19_attention_weight_max": 26.263458251953125,
      "activations/layer19_attention_weight_min": -24.352527618408203,
      "activations/layer1_attention_weight_max": 14.960308074951172,
      "activations/layer1_attention_weight_min": -14.449297904968262,
      "activations/layer20_attention_weight_max": 28.199241638183594,
      "activations/layer20_attention_weight_min": -25.541446685791016,
      "activations/layer21_attention_weight_max": 43.91814422607422,
      "activations/layer21_attention_weight_min": -24.09378433227539,
      "activations/layer22_attention_weight_max": 28.138383865356445,
      "activations/layer22_attention_weight_min": -24.72071647644043,
      "activations/layer23_attention_weight_max": 40.86086654663086,
      "activations/layer23_attention_weight_min": -22.745771408081055,
      "activations/layer2_attention_weight_max": 31.167701721191406,
      "activations/layer2_attention_weight_min": -29.246261596679688,
      "activations/layer3_attention_weight_max": 85.32611083984375,
      "activations/layer3_attention_weight_min": -87.67108917236328,
      "activations/layer4_attention_weight_max": 83.6946792602539,
      "activations/layer4_attention_weight_min": -83.70931243896484,
      "activations/layer5_attention_weight_max": 63.50934600830078,
      "activations/layer5_attention_weight_min": -77.77647399902344,
      "activations/layer6_attention_weight_max": 49.23408889770508,
      "activations/layer6_attention_weight_min": -51.35026931762695,
      "activations/layer7_attention_weight_max": 63.285789489746094,
      "activations/layer7_attention_weight_min": -61.138763427734375,
      "activations/layer8_attention_weight_max": 46.148563385009766,
      "activations/layer8_attention_weight_min": -48.89473342895508,
      "activations/layer9_attention_weight_max": 43.427040100097656,
      "activations/layer9_attention_weight_min": -45.643489837646484,
      "epoch": 18.32,
      "learning_rate": 3.2160984848484845e-05,
      "loss": 2.7213,
      "step": 315350
    },
    {
      "activations/layer0_attention_weight_max": 14.927278518676758,
      "activations/layer0_attention_weight_min": -13.603569030761719,
      "activations/layer10_attention_weight_max": 35.830650329589844,
      "activations/layer10_attention_weight_min": -36.32345199584961,
      "activations/layer11_attention_weight_max": 33.77804946899414,
      "activations/layer11_attention_weight_min": -36.456947326660156,
      "activations/layer12_attention_weight_max": 18.519691467285156,
      "activations/layer12_attention_weight_min": -25.52518081665039,
      "activations/layer13_attention_weight_max": 36.9724235534668,
      "activations/layer13_attention_weight_min": -28.36829376220703,
      "activations/layer14_attention_weight_max": 34.89905548095703,
      "activations/layer14_attention_weight_min": -31.3306884765625,
      "activations/layer15_attention_weight_max": 34.45212173461914,
      "activations/layer15_attention_weight_min": -32.47711944580078,
      "activations/layer16_attention_weight_max": 35.87683868408203,
      "activations/layer16_attention_weight_min": -33.51980972290039,
      "activations/layer17_attention_weight_max": 56.95643997192383,
      "activations/layer17_attention_weight_min": -50.45389175415039,
      "activations/layer18_attention_weight_max": 47.89042282104492,
      "activations/layer18_attention_weight_min": -40.91646957397461,
      "activations/layer19_attention_weight_max": 23.95366668701172,
      "activations/layer19_attention_weight_min": -22.953365325927734,
      "activations/layer1_attention_weight_max": 16.61416244506836,
      "activations/layer1_attention_weight_min": -13.977843284606934,
      "activations/layer20_attention_weight_max": 22.709928512573242,
      "activations/layer20_attention_weight_min": -23.001562118530273,
      "activations/layer21_attention_weight_max": 36.13349151611328,
      "activations/layer21_attention_weight_min": -21.959674835205078,
      "activations/layer22_attention_weight_max": 29.73291015625,
      "activations/layer22_attention_weight_min": -26.247270584106445,
      "activations/layer23_attention_weight_max": 38.83219909667969,
      "activations/layer23_attention_weight_min": -25.753860473632812,
      "activations/layer2_attention_weight_max": 32.38274383544922,
      "activations/layer2_attention_weight_min": -29.274200439453125,
      "activations/layer3_attention_weight_max": 84.20958709716797,
      "activations/layer3_attention_weight_min": -83.92867279052734,
      "activations/layer4_attention_weight_max": 81.43626403808594,
      "activations/layer4_attention_weight_min": -83.17658996582031,
      "activations/layer5_attention_weight_max": 65.89047241210938,
      "activations/layer5_attention_weight_min": -72.83295440673828,
      "activations/layer6_attention_weight_max": 50.0100212097168,
      "activations/layer6_attention_weight_min": -49.200050354003906,
      "activations/layer7_attention_weight_max": 63.820560455322266,
      "activations/layer7_attention_weight_min": -62.7780876159668,
      "activations/layer8_attention_weight_max": 45.43288040161133,
      "activations/layer8_attention_weight_min": -48.73225402832031,
      "activations/layer9_attention_weight_max": 43.39326095581055,
      "activations/layer9_attention_weight_min": -47.010276794433594,
      "epoch": 18.33,
      "learning_rate": 3.2142045454545453e-05,
      "loss": 2.7408,
      "step": 315400
    },
    {
      "activations/layer0_attention_weight_max": 15.546087265014648,
      "activations/layer0_attention_weight_min": -13.508121490478516,
      "activations/layer10_attention_weight_max": 34.61418533325195,
      "activations/layer10_attention_weight_min": -35.386287689208984,
      "activations/layer11_attention_weight_max": 33.14094543457031,
      "activations/layer11_attention_weight_min": -35.013362884521484,
      "activations/layer12_attention_weight_max": 20.13307762145996,
      "activations/layer12_attention_weight_min": -26.02399444580078,
      "activations/layer13_attention_weight_max": 36.772212982177734,
      "activations/layer13_attention_weight_min": -27.063005447387695,
      "activations/layer14_attention_weight_max": 38.295196533203125,
      "activations/layer14_attention_weight_min": -31.641157150268555,
      "activations/layer15_attention_weight_max": 36.26386642456055,
      "activations/layer15_attention_weight_min": -32.869293212890625,
      "activations/layer16_attention_weight_max": 37.16724395751953,
      "activations/layer16_attention_weight_min": -33.74654006958008,
      "activations/layer17_attention_weight_max": 53.223182678222656,
      "activations/layer17_attention_weight_min": -46.6215934753418,
      "activations/layer18_attention_weight_max": 50.41904067993164,
      "activations/layer18_attention_weight_min": -39.481658935546875,
      "activations/layer19_attention_weight_max": 27.174407958984375,
      "activations/layer19_attention_weight_min": -22.6158504486084,
      "activations/layer1_attention_weight_max": 16.09174156188965,
      "activations/layer1_attention_weight_min": -14.0313081741333,
      "activations/layer20_attention_weight_max": 25.4304256439209,
      "activations/layer20_attention_weight_min": -22.21331214904785,
      "activations/layer21_attention_weight_max": 37.87211990356445,
      "activations/layer21_attention_weight_min": -23.168365478515625,
      "activations/layer22_attention_weight_max": 28.90431785583496,
      "activations/layer22_attention_weight_min": -24.52971839904785,
      "activations/layer23_attention_weight_max": 38.35485076904297,
      "activations/layer23_attention_weight_min": -24.942668914794922,
      "activations/layer2_attention_weight_max": 30.584571838378906,
      "activations/layer2_attention_weight_min": -28.86787223815918,
      "activations/layer3_attention_weight_max": 86.0130844116211,
      "activations/layer3_attention_weight_min": -85.85101318359375,
      "activations/layer4_attention_weight_max": 85.02014923095703,
      "activations/layer4_attention_weight_min": -84.83490753173828,
      "activations/layer5_attention_weight_max": 63.26302719116211,
      "activations/layer5_attention_weight_min": -71.73419189453125,
      "activations/layer6_attention_weight_max": 50.270957946777344,
      "activations/layer6_attention_weight_min": -49.95060348510742,
      "activations/layer7_attention_weight_max": 63.5716552734375,
      "activations/layer7_attention_weight_min": -63.30958557128906,
      "activations/layer8_attention_weight_max": 45.90188217163086,
      "activations/layer8_attention_weight_min": -48.975032806396484,
      "activations/layer9_attention_weight_max": 43.49723434448242,
      "activations/layer9_attention_weight_min": -46.46693420410156,
      "epoch": 18.33,
      "learning_rate": 3.2123106060606055e-05,
      "loss": 2.7393,
      "step": 315450
    },
    {
      "activations/layer0_attention_weight_max": 15.25252628326416,
      "activations/layer0_attention_weight_min": -13.948357582092285,
      "activations/layer10_attention_weight_max": 34.62358856201172,
      "activations/layer10_attention_weight_min": -35.15413284301758,
      "activations/layer11_attention_weight_max": 30.970977783203125,
      "activations/layer11_attention_weight_min": -33.23823165893555,
      "activations/layer12_attention_weight_max": 17.585432052612305,
      "activations/layer12_attention_weight_min": -23.155372619628906,
      "activations/layer13_attention_weight_max": 32.24728012084961,
      "activations/layer13_attention_weight_min": -26.907957077026367,
      "activations/layer14_attention_weight_max": 34.99393081665039,
      "activations/layer14_attention_weight_min": -29.8009033203125,
      "activations/layer15_attention_weight_max": 31.761821746826172,
      "activations/layer15_attention_weight_min": -30.175769805908203,
      "activations/layer16_attention_weight_max": 31.882278442382812,
      "activations/layer16_attention_weight_min": -30.392240524291992,
      "activations/layer17_attention_weight_max": 50.81248474121094,
      "activations/layer17_attention_weight_min": -42.96638870239258,
      "activations/layer18_attention_weight_max": 46.64105224609375,
      "activations/layer18_attention_weight_min": -38.52061080932617,
      "activations/layer19_attention_weight_max": 25.057270050048828,
      "activations/layer19_attention_weight_min": -23.166433334350586,
      "activations/layer1_attention_weight_max": 15.98903751373291,
      "activations/layer1_attention_weight_min": -14.099846839904785,
      "activations/layer20_attention_weight_max": 23.47711944580078,
      "activations/layer20_attention_weight_min": -20.817777633666992,
      "activations/layer21_attention_weight_max": 35.35915756225586,
      "activations/layer21_attention_weight_min": -21.672292709350586,
      "activations/layer22_attention_weight_max": 28.644550323486328,
      "activations/layer22_attention_weight_min": -23.794567108154297,
      "activations/layer23_attention_weight_max": 38.612892150878906,
      "activations/layer23_attention_weight_min": -24.41364860534668,
      "activations/layer2_attention_weight_max": 31.219114303588867,
      "activations/layer2_attention_weight_min": -30.073476791381836,
      "activations/layer3_attention_weight_max": 86.95362854003906,
      "activations/layer3_attention_weight_min": -82.32302856445312,
      "activations/layer4_attention_weight_max": 80.27705383300781,
      "activations/layer4_attention_weight_min": -83.18062591552734,
      "activations/layer5_attention_weight_max": 58.590240478515625,
      "activations/layer5_attention_weight_min": -75.30152893066406,
      "activations/layer6_attention_weight_max": 48.83948516845703,
      "activations/layer6_attention_weight_min": -50.128665924072266,
      "activations/layer7_attention_weight_max": 64.11641693115234,
      "activations/layer7_attention_weight_min": -63.30679702758789,
      "activations/layer8_attention_weight_max": 44.24906921386719,
      "activations/layer8_attention_weight_min": -48.46821212768555,
      "activations/layer9_attention_weight_max": 46.38386535644531,
      "activations/layer9_attention_weight_min": -44.805809020996094,
      "epoch": 18.33,
      "learning_rate": 3.210416666666666e-05,
      "loss": 2.7347,
      "step": 315500
    },
    {
      "activations/layer0_attention_weight_max": 14.674742698669434,
      "activations/layer0_attention_weight_min": -13.433128356933594,
      "activations/layer10_attention_weight_max": 37.610496520996094,
      "activations/layer10_attention_weight_min": -36.870521545410156,
      "activations/layer11_attention_weight_max": 34.2825927734375,
      "activations/layer11_attention_weight_min": -35.21525573730469,
      "activations/layer12_attention_weight_max": 19.673437118530273,
      "activations/layer12_attention_weight_min": -23.040658950805664,
      "activations/layer13_attention_weight_max": 36.01491165161133,
      "activations/layer13_attention_weight_min": -29.186616897583008,
      "activations/layer14_attention_weight_max": 34.86248016357422,
      "activations/layer14_attention_weight_min": -29.110076904296875,
      "activations/layer15_attention_weight_max": 38.08572769165039,
      "activations/layer15_attention_weight_min": -29.339317321777344,
      "activations/layer16_attention_weight_max": 34.220497131347656,
      "activations/layer16_attention_weight_min": -30.57944679260254,
      "activations/layer17_attention_weight_max": 47.1691780090332,
      "activations/layer17_attention_weight_min": -44.6263313293457,
      "activations/layer18_attention_weight_max": 47.258750915527344,
      "activations/layer18_attention_weight_min": -39.731441497802734,
      "activations/layer19_attention_weight_max": 24.433486938476562,
      "activations/layer19_attention_weight_min": -23.031957626342773,
      "activations/layer1_attention_weight_max": 15.349501609802246,
      "activations/layer1_attention_weight_min": -16.190658569335938,
      "activations/layer20_attention_weight_max": 21.519672393798828,
      "activations/layer20_attention_weight_min": -21.409727096557617,
      "activations/layer21_attention_weight_max": 36.966522216796875,
      "activations/layer21_attention_weight_min": -23.874221801757812,
      "activations/layer22_attention_weight_max": 31.381242752075195,
      "activations/layer22_attention_weight_min": -26.30909538269043,
      "activations/layer23_attention_weight_max": 36.10314178466797,
      "activations/layer23_attention_weight_min": -25.575164794921875,
      "activations/layer2_attention_weight_max": 30.572952270507812,
      "activations/layer2_attention_weight_min": -30.290990829467773,
      "activations/layer3_attention_weight_max": 87.33419036865234,
      "activations/layer3_attention_weight_min": -90.98271942138672,
      "activations/layer4_attention_weight_max": 83.45542907714844,
      "activations/layer4_attention_weight_min": -83.03717041015625,
      "activations/layer5_attention_weight_max": 60.56665802001953,
      "activations/layer5_attention_weight_min": -76.30155944824219,
      "activations/layer6_attention_weight_max": 47.181358337402344,
      "activations/layer6_attention_weight_min": -50.050411224365234,
      "activations/layer7_attention_weight_max": 62.97404479980469,
      "activations/layer7_attention_weight_min": -65.56385040283203,
      "activations/layer8_attention_weight_max": 46.33518600463867,
      "activations/layer8_attention_weight_min": -49.16707229614258,
      "activations/layer9_attention_weight_max": 52.40357208251953,
      "activations/layer9_attention_weight_min": -47.9292106628418,
      "epoch": 18.34,
      "learning_rate": 3.208522727272727e-05,
      "loss": 2.7269,
      "step": 315550
    },
    {
      "activations/layer0_attention_weight_max": 14.921343803405762,
      "activations/layer0_attention_weight_min": -13.057836532592773,
      "activations/layer10_attention_weight_max": 35.75623321533203,
      "activations/layer10_attention_weight_min": -37.7490234375,
      "activations/layer11_attention_weight_max": 33.132713317871094,
      "activations/layer11_attention_weight_min": -33.56786346435547,
      "activations/layer12_attention_weight_max": 18.314023971557617,
      "activations/layer12_attention_weight_min": -25.22709083557129,
      "activations/layer13_attention_weight_max": 32.99832534790039,
      "activations/layer13_attention_weight_min": -26.13555335998535,
      "activations/layer14_attention_weight_max": 34.99098587036133,
      "activations/layer14_attention_weight_min": -29.8691463470459,
      "activations/layer15_attention_weight_max": 32.149375915527344,
      "activations/layer15_attention_weight_min": -28.99290657043457,
      "activations/layer16_attention_weight_max": 34.98300552368164,
      "activations/layer16_attention_weight_min": -32.942291259765625,
      "activations/layer17_attention_weight_max": 51.30430221557617,
      "activations/layer17_attention_weight_min": -45.687034606933594,
      "activations/layer18_attention_weight_max": 45.53114318847656,
      "activations/layer18_attention_weight_min": -37.062889099121094,
      "activations/layer19_attention_weight_max": 23.437925338745117,
      "activations/layer19_attention_weight_min": -21.659988403320312,
      "activations/layer1_attention_weight_max": 15.9898681640625,
      "activations/layer1_attention_weight_min": -14.815321922302246,
      "activations/layer20_attention_weight_max": 23.960617065429688,
      "activations/layer20_attention_weight_min": -23.240001678466797,
      "activations/layer21_attention_weight_max": 36.2708740234375,
      "activations/layer21_attention_weight_min": -24.274581909179688,
      "activations/layer22_attention_weight_max": 29.837989807128906,
      "activations/layer22_attention_weight_min": -25.8570613861084,
      "activations/layer23_attention_weight_max": 37.58513259887695,
      "activations/layer23_attention_weight_min": -25.283859252929688,
      "activations/layer2_attention_weight_max": 30.99919891357422,
      "activations/layer2_attention_weight_min": -29.97378921508789,
      "activations/layer3_attention_weight_max": 83.52261352539062,
      "activations/layer3_attention_weight_min": -87.56908416748047,
      "activations/layer4_attention_weight_max": 84.70018768310547,
      "activations/layer4_attention_weight_min": -85.39258575439453,
      "activations/layer5_attention_weight_max": 62.78391647338867,
      "activations/layer5_attention_weight_min": -72.63468933105469,
      "activations/layer6_attention_weight_max": 51.22049331665039,
      "activations/layer6_attention_weight_min": -53.36806869506836,
      "activations/layer7_attention_weight_max": 66.2060775756836,
      "activations/layer7_attention_weight_min": -64.7569808959961,
      "activations/layer8_attention_weight_max": 48.12565994262695,
      "activations/layer8_attention_weight_min": -53.472442626953125,
      "activations/layer9_attention_weight_max": 42.91474533081055,
      "activations/layer9_attention_weight_min": -46.748043060302734,
      "epoch": 18.34,
      "learning_rate": 3.206628787878788e-05,
      "loss": 2.7562,
      "step": 315600
    },
    {
      "activations/layer0_attention_weight_max": 15.397161483764648,
      "activations/layer0_attention_weight_min": -13.256407737731934,
      "activations/layer10_attention_weight_max": 34.075077056884766,
      "activations/layer10_attention_weight_min": -37.08045959472656,
      "activations/layer11_attention_weight_max": 34.41636276245117,
      "activations/layer11_attention_weight_min": -35.22796630859375,
      "activations/layer12_attention_weight_max": 17.64945411682129,
      "activations/layer12_attention_weight_min": -24.68307876586914,
      "activations/layer13_attention_weight_max": 33.36460494995117,
      "activations/layer13_attention_weight_min": -27.93875503540039,
      "activations/layer14_attention_weight_max": 34.11463928222656,
      "activations/layer14_attention_weight_min": -30.165307998657227,
      "activations/layer15_attention_weight_max": 32.21923828125,
      "activations/layer15_attention_weight_min": -31.16106414794922,
      "activations/layer16_attention_weight_max": 32.93421936035156,
      "activations/layer16_attention_weight_min": -32.24342346191406,
      "activations/layer17_attention_weight_max": 50.43495559692383,
      "activations/layer17_attention_weight_min": -43.59101867675781,
      "activations/layer18_attention_weight_max": 48.54861068725586,
      "activations/layer18_attention_weight_min": -40.66426086425781,
      "activations/layer19_attention_weight_max": 22.872556686401367,
      "activations/layer19_attention_weight_min": -25.421213150024414,
      "activations/layer1_attention_weight_max": 16.52725601196289,
      "activations/layer1_attention_weight_min": -14.43286418914795,
      "activations/layer20_attention_weight_max": 20.80221939086914,
      "activations/layer20_attention_weight_min": -23.653154373168945,
      "activations/layer21_attention_weight_max": 30.744243621826172,
      "activations/layer21_attention_weight_min": -28.27665901184082,
      "activations/layer22_attention_weight_max": 28.569997787475586,
      "activations/layer22_attention_weight_min": -27.857698440551758,
      "activations/layer23_attention_weight_max": 34.79966735839844,
      "activations/layer23_attention_weight_min": -32.7310791015625,
      "activations/layer2_attention_weight_max": 33.05531311035156,
      "activations/layer2_attention_weight_min": -28.73046112060547,
      "activations/layer3_attention_weight_max": 88.48941802978516,
      "activations/layer3_attention_weight_min": -88.270751953125,
      "activations/layer4_attention_weight_max": 88.10896301269531,
      "activations/layer4_attention_weight_min": -87.59789276123047,
      "activations/layer5_attention_weight_max": 62.4261474609375,
      "activations/layer5_attention_weight_min": -76.32994079589844,
      "activations/layer6_attention_weight_max": 52.65409851074219,
      "activations/layer6_attention_weight_min": -52.20803451538086,
      "activations/layer7_attention_weight_max": 68.04032897949219,
      "activations/layer7_attention_weight_min": -66.68247985839844,
      "activations/layer8_attention_weight_max": 46.34077072143555,
      "activations/layer8_attention_weight_min": -49.12893295288086,
      "activations/layer9_attention_weight_max": 45.344661712646484,
      "activations/layer9_attention_weight_min": -47.27401351928711,
      "epoch": 18.34,
      "learning_rate": 3.204734848484848e-05,
      "loss": 2.7394,
      "step": 315650
    },
    {
      "activations/layer0_attention_weight_max": 15.265677452087402,
      "activations/layer0_attention_weight_min": -13.597291946411133,
      "activations/layer10_attention_weight_max": 35.864784240722656,
      "activations/layer10_attention_weight_min": -36.02360916137695,
      "activations/layer11_attention_weight_max": 32.43589401245117,
      "activations/layer11_attention_weight_min": -32.89409255981445,
      "activations/layer12_attention_weight_max": 17.754201889038086,
      "activations/layer12_attention_weight_min": -24.426298141479492,
      "activations/layer13_attention_weight_max": 29.52581787109375,
      "activations/layer13_attention_weight_min": -26.78432273864746,
      "activations/layer14_attention_weight_max": 32.460060119628906,
      "activations/layer14_attention_weight_min": -29.23360824584961,
      "activations/layer15_attention_weight_max": 31.24085235595703,
      "activations/layer15_attention_weight_min": -30.00843048095703,
      "activations/layer16_attention_weight_max": 32.768218994140625,
      "activations/layer16_attention_weight_min": -30.358936309814453,
      "activations/layer17_attention_weight_max": 48.90287780761719,
      "activations/layer17_attention_weight_min": -44.292015075683594,
      "activations/layer18_attention_weight_max": 47.53244400024414,
      "activations/layer18_attention_weight_min": -39.768646240234375,
      "activations/layer19_attention_weight_max": 22.26270866394043,
      "activations/layer19_attention_weight_min": -24.330678939819336,
      "activations/layer1_attention_weight_max": 21.101001739501953,
      "activations/layer1_attention_weight_min": -14.062429428100586,
      "activations/layer20_attention_weight_max": 20.75535011291504,
      "activations/layer20_attention_weight_min": -25.327669143676758,
      "activations/layer21_attention_weight_max": 34.127708435058594,
      "activations/layer21_attention_weight_min": -25.18402671813965,
      "activations/layer22_attention_weight_max": 26.283920288085938,
      "activations/layer22_attention_weight_min": -27.397247314453125,
      "activations/layer23_attention_weight_max": 34.77076721191406,
      "activations/layer23_attention_weight_min": -26.691722869873047,
      "activations/layer2_attention_weight_max": 31.26543426513672,
      "activations/layer2_attention_weight_min": -30.27945327758789,
      "activations/layer3_attention_weight_max": 82.30734252929688,
      "activations/layer3_attention_weight_min": -81.11804962158203,
      "activations/layer4_attention_weight_max": 83.94683074951172,
      "activations/layer4_attention_weight_min": -80.98925018310547,
      "activations/layer5_attention_weight_max": 61.392425537109375,
      "activations/layer5_attention_weight_min": -74.78211975097656,
      "activations/layer6_attention_weight_max": 50.55376434326172,
      "activations/layer6_attention_weight_min": -53.506446838378906,
      "activations/layer7_attention_weight_max": 68.43913269042969,
      "activations/layer7_attention_weight_min": -66.60658264160156,
      "activations/layer8_attention_weight_max": 46.85068130493164,
      "activations/layer8_attention_weight_min": -48.80140686035156,
      "activations/layer9_attention_weight_max": 46.28101348876953,
      "activations/layer9_attention_weight_min": -48.6788444519043,
      "epoch": 18.34,
      "learning_rate": 3.202840909090909e-05,
      "loss": 2.7391,
      "step": 315700
    },
    {
      "activations/layer0_attention_weight_max": 15.319256782531738,
      "activations/layer0_attention_weight_min": -13.917976379394531,
      "activations/layer10_attention_weight_max": 37.64154815673828,
      "activations/layer10_attention_weight_min": -36.06610107421875,
      "activations/layer11_attention_weight_max": 35.55047607421875,
      "activations/layer11_attention_weight_min": -33.67146682739258,
      "activations/layer12_attention_weight_max": 19.569416046142578,
      "activations/layer12_attention_weight_min": -28.1414852142334,
      "activations/layer13_attention_weight_max": 36.77430725097656,
      "activations/layer13_attention_weight_min": -27.249547958374023,
      "activations/layer14_attention_weight_max": 35.68348693847656,
      "activations/layer14_attention_weight_min": -31.463478088378906,
      "activations/layer15_attention_weight_max": 35.2292366027832,
      "activations/layer15_attention_weight_min": -29.57838249206543,
      "activations/layer16_attention_weight_max": 35.57817077636719,
      "activations/layer16_attention_weight_min": -34.174842834472656,
      "activations/layer17_attention_weight_max": 53.94598388671875,
      "activations/layer17_attention_weight_min": -47.68709945678711,
      "activations/layer18_attention_weight_max": 50.77103042602539,
      "activations/layer18_attention_weight_min": -43.83708572387695,
      "activations/layer19_attention_weight_max": 25.121320724487305,
      "activations/layer19_attention_weight_min": -23.193756103515625,
      "activations/layer1_attention_weight_max": 15.699917793273926,
      "activations/layer1_attention_weight_min": -14.54798698425293,
      "activations/layer20_attention_weight_max": 24.548717498779297,
      "activations/layer20_attention_weight_min": -21.222787857055664,
      "activations/layer21_attention_weight_max": 38.81747817993164,
      "activations/layer21_attention_weight_min": -22.681926727294922,
      "activations/layer22_attention_weight_max": 31.453353881835938,
      "activations/layer22_attention_weight_min": -26.603166580200195,
      "activations/layer23_attention_weight_max": 40.5704460144043,
      "activations/layer23_attention_weight_min": -23.22423553466797,
      "activations/layer2_attention_weight_max": 31.524656295776367,
      "activations/layer2_attention_weight_min": -30.457561492919922,
      "activations/layer3_attention_weight_max": 76.43035125732422,
      "activations/layer3_attention_weight_min": -83.84636688232422,
      "activations/layer4_attention_weight_max": 82.1170425415039,
      "activations/layer4_attention_weight_min": -80.58717346191406,
      "activations/layer5_attention_weight_max": 57.1893310546875,
      "activations/layer5_attention_weight_min": -71.64522552490234,
      "activations/layer6_attention_weight_max": 49.70274353027344,
      "activations/layer6_attention_weight_min": -48.99361801147461,
      "activations/layer7_attention_weight_max": 62.19270324707031,
      "activations/layer7_attention_weight_min": -63.7647590637207,
      "activations/layer8_attention_weight_max": 46.37161636352539,
      "activations/layer8_attention_weight_min": -49.55034255981445,
      "activations/layer9_attention_weight_max": 43.97861862182617,
      "activations/layer9_attention_weight_min": -46.37284469604492,
      "epoch": 18.35,
      "learning_rate": 3.200946969696969e-05,
      "loss": 2.748,
      "step": 315750
    },
    {
      "activations/layer0_attention_weight_max": 15.450430870056152,
      "activations/layer0_attention_weight_min": -13.817174911499023,
      "activations/layer10_attention_weight_max": 36.962646484375,
      "activations/layer10_attention_weight_min": -35.214935302734375,
      "activations/layer11_attention_weight_max": 34.16865921020508,
      "activations/layer11_attention_weight_min": -35.449337005615234,
      "activations/layer12_attention_weight_max": 18.757083892822266,
      "activations/layer12_attention_weight_min": -26.743513107299805,
      "activations/layer13_attention_weight_max": 32.56570053100586,
      "activations/layer13_attention_weight_min": -27.222566604614258,
      "activations/layer14_attention_weight_max": 32.898929595947266,
      "activations/layer14_attention_weight_min": -31.335079193115234,
      "activations/layer15_attention_weight_max": 32.32567596435547,
      "activations/layer15_attention_weight_min": -29.011688232421875,
      "activations/layer16_attention_weight_max": 33.88155746459961,
      "activations/layer16_attention_weight_min": -31.65320587158203,
      "activations/layer17_attention_weight_max": 47.459110260009766,
      "activations/layer17_attention_weight_min": -42.9697151184082,
      "activations/layer18_attention_weight_max": 47.596858978271484,
      "activations/layer18_attention_weight_min": -36.60258102416992,
      "activations/layer19_attention_weight_max": 23.056310653686523,
      "activations/layer19_attention_weight_min": -22.27606201171875,
      "activations/layer1_attention_weight_max": 16.34490394592285,
      "activations/layer1_attention_weight_min": -14.322433471679688,
      "activations/layer20_attention_weight_max": 23.1368408203125,
      "activations/layer20_attention_weight_min": -21.635326385498047,
      "activations/layer21_attention_weight_max": 33.48710632324219,
      "activations/layer21_attention_weight_min": -20.912778854370117,
      "activations/layer22_attention_weight_max": 31.55634117126465,
      "activations/layer22_attention_weight_min": -25.927412033081055,
      "activations/layer23_attention_weight_max": 37.03388214111328,
      "activations/layer23_attention_weight_min": -22.615232467651367,
      "activations/layer2_attention_weight_max": 31.67725372314453,
      "activations/layer2_attention_weight_min": -28.972347259521484,
      "activations/layer3_attention_weight_max": 82.96104431152344,
      "activations/layer3_attention_weight_min": -81.21084594726562,
      "activations/layer4_attention_weight_max": 79.74392700195312,
      "activations/layer4_attention_weight_min": -77.59107971191406,
      "activations/layer5_attention_weight_max": 60.79045867919922,
      "activations/layer5_attention_weight_min": -71.17616271972656,
      "activations/layer6_attention_weight_max": 47.518863677978516,
      "activations/layer6_attention_weight_min": -48.29905319213867,
      "activations/layer7_attention_weight_max": 62.072853088378906,
      "activations/layer7_attention_weight_min": -63.0936393737793,
      "activations/layer8_attention_weight_max": 46.17824172973633,
      "activations/layer8_attention_weight_min": -49.65834045410156,
      "activations/layer9_attention_weight_max": 43.34385299682617,
      "activations/layer9_attention_weight_min": -45.26679611206055,
      "epoch": 18.35,
      "learning_rate": 3.19905303030303e-05,
      "loss": 2.7489,
      "step": 315800
    },
    {
      "activations/layer0_attention_weight_max": 15.528430938720703,
      "activations/layer0_attention_weight_min": -13.586649894714355,
      "activations/layer10_attention_weight_max": 34.96421813964844,
      "activations/layer10_attention_weight_min": -34.69363021850586,
      "activations/layer11_attention_weight_max": 31.75101089477539,
      "activations/layer11_attention_weight_min": -33.06024169921875,
      "activations/layer12_attention_weight_max": 19.375961303710938,
      "activations/layer12_attention_weight_min": -28.994834899902344,
      "activations/layer13_attention_weight_max": 34.6224250793457,
      "activations/layer13_attention_weight_min": -28.29001808166504,
      "activations/layer14_attention_weight_max": 33.70085144042969,
      "activations/layer14_attention_weight_min": -29.531879425048828,
      "activations/layer15_attention_weight_max": 32.562469482421875,
      "activations/layer15_attention_weight_min": -29.911258697509766,
      "activations/layer16_attention_weight_max": 32.880531311035156,
      "activations/layer16_attention_weight_min": -32.048011779785156,
      "activations/layer17_attention_weight_max": 48.67568588256836,
      "activations/layer17_attention_weight_min": -46.52429962158203,
      "activations/layer18_attention_weight_max": 45.84748077392578,
      "activations/layer18_attention_weight_min": -43.06879806518555,
      "activations/layer19_attention_weight_max": 24.354267120361328,
      "activations/layer19_attention_weight_min": -23.320558547973633,
      "activations/layer1_attention_weight_max": 16.384674072265625,
      "activations/layer1_attention_weight_min": -13.723498344421387,
      "activations/layer20_attention_weight_max": 23.216215133666992,
      "activations/layer20_attention_weight_min": -22.09735679626465,
      "activations/layer21_attention_weight_max": 34.48763656616211,
      "activations/layer21_attention_weight_min": -22.333465576171875,
      "activations/layer22_attention_weight_max": 27.923309326171875,
      "activations/layer22_attention_weight_min": -25.829689025878906,
      "activations/layer23_attention_weight_max": 35.344608306884766,
      "activations/layer23_attention_weight_min": -27.780288696289062,
      "activations/layer2_attention_weight_max": 31.8621883392334,
      "activations/layer2_attention_weight_min": -31.259397506713867,
      "activations/layer3_attention_weight_max": 82.2657699584961,
      "activations/layer3_attention_weight_min": -87.4682388305664,
      "activations/layer4_attention_weight_max": 80.31414031982422,
      "activations/layer4_attention_weight_min": -83.40023040771484,
      "activations/layer5_attention_weight_max": 62.23277282714844,
      "activations/layer5_attention_weight_min": -73.88690948486328,
      "activations/layer6_attention_weight_max": 45.738059997558594,
      "activations/layer6_attention_weight_min": -50.00931167602539,
      "activations/layer7_attention_weight_max": 67.00345611572266,
      "activations/layer7_attention_weight_min": -66.94954681396484,
      "activations/layer8_attention_weight_max": 45.00419235229492,
      "activations/layer8_attention_weight_min": -47.58948516845703,
      "activations/layer9_attention_weight_max": 41.696250915527344,
      "activations/layer9_attention_weight_min": -44.39560317993164,
      "epoch": 18.35,
      "learning_rate": 3.197159090909091e-05,
      "loss": 2.7339,
      "step": 315850
    },
    {
      "activations/layer0_attention_weight_max": 15.09097957611084,
      "activations/layer0_attention_weight_min": -13.289825439453125,
      "activations/layer10_attention_weight_max": 37.73210525512695,
      "activations/layer10_attention_weight_min": -35.80821990966797,
      "activations/layer11_attention_weight_max": 32.075775146484375,
      "activations/layer11_attention_weight_min": -33.60854721069336,
      "activations/layer12_attention_weight_max": 18.94206428527832,
      "activations/layer12_attention_weight_min": -25.43081283569336,
      "activations/layer13_attention_weight_max": 31.463075637817383,
      "activations/layer13_attention_weight_min": -28.699344635009766,
      "activations/layer14_attention_weight_max": 33.48448181152344,
      "activations/layer14_attention_weight_min": -31.28250503540039,
      "activations/layer15_attention_weight_max": 31.99663543701172,
      "activations/layer15_attention_weight_min": -30.49534797668457,
      "activations/layer16_attention_weight_max": 34.12187576293945,
      "activations/layer16_attention_weight_min": -33.828304290771484,
      "activations/layer17_attention_weight_max": 49.920345306396484,
      "activations/layer17_attention_weight_min": -46.42744445800781,
      "activations/layer18_attention_weight_max": 46.8194694519043,
      "activations/layer18_attention_weight_min": -38.9359130859375,
      "activations/layer19_attention_weight_max": 24.976110458374023,
      "activations/layer19_attention_weight_min": -24.220340728759766,
      "activations/layer1_attention_weight_max": 15.702373504638672,
      "activations/layer1_attention_weight_min": -14.031310081481934,
      "activations/layer20_attention_weight_max": 23.587913513183594,
      "activations/layer20_attention_weight_min": -23.621097564697266,
      "activations/layer21_attention_weight_max": 38.996376037597656,
      "activations/layer21_attention_weight_min": -23.06591796875,
      "activations/layer22_attention_weight_max": 27.297916412353516,
      "activations/layer22_attention_weight_min": -25.87679100036621,
      "activations/layer23_attention_weight_max": 33.179996490478516,
      "activations/layer23_attention_weight_min": -25.027027130126953,
      "activations/layer2_attention_weight_max": 32.82792663574219,
      "activations/layer2_attention_weight_min": -31.58711051940918,
      "activations/layer3_attention_weight_max": 84.8947525024414,
      "activations/layer3_attention_weight_min": -88.61319732666016,
      "activations/layer4_attention_weight_max": 88.04386138916016,
      "activations/layer4_attention_weight_min": -85.87796020507812,
      "activations/layer5_attention_weight_max": 64.63159942626953,
      "activations/layer5_attention_weight_min": -71.88204956054688,
      "activations/layer6_attention_weight_max": 52.17619705200195,
      "activations/layer6_attention_weight_min": -53.006221771240234,
      "activations/layer7_attention_weight_max": 67.20999145507812,
      "activations/layer7_attention_weight_min": -65.97474670410156,
      "activations/layer8_attention_weight_max": 47.5260009765625,
      "activations/layer8_attention_weight_min": -50.673622131347656,
      "activations/layer9_attention_weight_max": 44.099090576171875,
      "activations/layer9_attention_weight_min": -45.80340576171875,
      "epoch": 18.36,
      "learning_rate": 3.1952651515151516e-05,
      "loss": 2.7404,
      "step": 315900
    },
    {
      "activations/layer0_attention_weight_max": 16.437591552734375,
      "activations/layer0_attention_weight_min": -13.540079116821289,
      "activations/layer10_attention_weight_max": 36.374271392822266,
      "activations/layer10_attention_weight_min": -37.05892562866211,
      "activations/layer11_attention_weight_max": 31.894397735595703,
      "activations/layer11_attention_weight_min": -35.37803649902344,
      "activations/layer12_attention_weight_max": 16.79669761657715,
      "activations/layer12_attention_weight_min": -25.87444305419922,
      "activations/layer13_attention_weight_max": 31.628665924072266,
      "activations/layer13_attention_weight_min": -29.02925682067871,
      "activations/layer14_attention_weight_max": 30.612154006958008,
      "activations/layer14_attention_weight_min": -28.185850143432617,
      "activations/layer15_attention_weight_max": 28.05776596069336,
      "activations/layer15_attention_weight_min": -28.321651458740234,
      "activations/layer16_attention_weight_max": 31.1060791015625,
      "activations/layer16_attention_weight_min": -29.797842025756836,
      "activations/layer17_attention_weight_max": 46.12163162231445,
      "activations/layer17_attention_weight_min": -42.747013092041016,
      "activations/layer18_attention_weight_max": 41.694847106933594,
      "activations/layer18_attention_weight_min": -42.33969497680664,
      "activations/layer19_attention_weight_max": 24.46259307861328,
      "activations/layer19_attention_weight_min": -25.93112564086914,
      "activations/layer1_attention_weight_max": 16.51720428466797,
      "activations/layer1_attention_weight_min": -14.841804504394531,
      "activations/layer20_attention_weight_max": 24.09322738647461,
      "activations/layer20_attention_weight_min": -21.7603816986084,
      "activations/layer21_attention_weight_max": 38.32439422607422,
      "activations/layer21_attention_weight_min": -27.640716552734375,
      "activations/layer22_attention_weight_max": 26.944429397583008,
      "activations/layer22_attention_weight_min": -24.682790756225586,
      "activations/layer23_attention_weight_max": 35.22960662841797,
      "activations/layer23_attention_weight_min": -25.286075592041016,
      "activations/layer2_attention_weight_max": 32.0622673034668,
      "activations/layer2_attention_weight_min": -31.11457061767578,
      "activations/layer3_attention_weight_max": 84.31351470947266,
      "activations/layer3_attention_weight_min": -85.7510757446289,
      "activations/layer4_attention_weight_max": 81.59261322021484,
      "activations/layer4_attention_weight_min": -82.79784393310547,
      "activations/layer5_attention_weight_max": 67.77275085449219,
      "activations/layer5_attention_weight_min": -73.03492736816406,
      "activations/layer6_attention_weight_max": 48.25971984863281,
      "activations/layer6_attention_weight_min": -49.4959831237793,
      "activations/layer7_attention_weight_max": 63.0390510559082,
      "activations/layer7_attention_weight_min": -63.175559997558594,
      "activations/layer8_attention_weight_max": 44.73283004760742,
      "activations/layer8_attention_weight_min": -50.24646759033203,
      "activations/layer9_attention_weight_max": 42.70186996459961,
      "activations/layer9_attention_weight_min": -46.302982330322266,
      "epoch": 18.36,
      "learning_rate": 3.193371212121212e-05,
      "loss": 2.7149,
      "step": 315950
    },
    {
      "activations/layer0_attention_weight_max": 17.090606689453125,
      "activations/layer0_attention_weight_min": -13.656251907348633,
      "activations/layer10_attention_weight_max": 41.71604537963867,
      "activations/layer10_attention_weight_min": -39.53730010986328,
      "activations/layer11_attention_weight_max": 41.040611267089844,
      "activations/layer11_attention_weight_min": -37.60136032104492,
      "activations/layer12_attention_weight_max": 18.584569931030273,
      "activations/layer12_attention_weight_min": -23.826568603515625,
      "activations/layer13_attention_weight_max": 47.739891052246094,
      "activations/layer13_attention_weight_min": -30.034420013427734,
      "activations/layer14_attention_weight_max": 45.29320526123047,
      "activations/layer14_attention_weight_min": -30.164417266845703,
      "activations/layer15_attention_weight_max": 41.1716423034668,
      "activations/layer15_attention_weight_min": -31.20720672607422,
      "activations/layer16_attention_weight_max": 32.57032012939453,
      "activations/layer16_attention_weight_min": -32.09203338623047,
      "activations/layer17_attention_weight_max": 53.00077438354492,
      "activations/layer17_attention_weight_min": -46.83850860595703,
      "activations/layer18_attention_weight_max": 46.88612365722656,
      "activations/layer18_attention_weight_min": -40.04389572143555,
      "activations/layer19_attention_weight_max": 22.69301414489746,
      "activations/layer19_attention_weight_min": -22.173505783081055,
      "activations/layer1_attention_weight_max": 16.045516967773438,
      "activations/layer1_attention_weight_min": -14.617898941040039,
      "activations/layer20_attention_weight_max": 22.69713592529297,
      "activations/layer20_attention_weight_min": -22.987321853637695,
      "activations/layer21_attention_weight_max": 36.91681671142578,
      "activations/layer21_attention_weight_min": -23.650466918945312,
      "activations/layer22_attention_weight_max": 27.89221954345703,
      "activations/layer22_attention_weight_min": -27.66473960876465,
      "activations/layer23_attention_weight_max": 36.64698791503906,
      "activations/layer23_attention_weight_min": -26.315311431884766,
      "activations/layer2_attention_weight_max": 34.59196090698242,
      "activations/layer2_attention_weight_min": -31.241397857666016,
      "activations/layer3_attention_weight_max": 86.82247161865234,
      "activations/layer3_attention_weight_min": -88.3253173828125,
      "activations/layer4_attention_weight_max": 88.2176742553711,
      "activations/layer4_attention_weight_min": -88.49449920654297,
      "activations/layer5_attention_weight_max": 60.92852020263672,
      "activations/layer5_attention_weight_min": -68.48249053955078,
      "activations/layer6_attention_weight_max": 50.025691986083984,
      "activations/layer6_attention_weight_min": -49.60789108276367,
      "activations/layer7_attention_weight_max": 72.13333892822266,
      "activations/layer7_attention_weight_min": -64.67769622802734,
      "activations/layer8_attention_weight_max": 51.990787506103516,
      "activations/layer8_attention_weight_min": -52.50584030151367,
      "activations/layer9_attention_weight_max": 56.49532699584961,
      "activations/layer9_attention_weight_min": -48.84160614013672,
      "epoch": 18.36,
      "learning_rate": 3.1914772727272726e-05,
      "loss": 2.7202,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.4648,
      "eval_samples_per_second": 507.28,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.4648,
      "eval_openwebtext_samples_per_second": 507.28,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9742,
      "eval_wikitext_samples_per_second": 230.979,
      "step": 316000
    },
    {
      "epoch": 18.36,
      "eval_lambada_loss": 2.43359375,
      "eval_lambada_ppl": 11.399776500625777,
      "eval_lambada_runtime": 9.5561,
      "eval_lambada_samples_per_second": 509.517,
      "step": 316000
    },
    {
      "activations/layer0_attention_weight_max": 15.222363471984863,
      "activations/layer0_attention_weight_min": -13.28358268737793,
      "activations/layer10_attention_weight_max": 34.61127471923828,
      "activations/layer10_attention_weight_min": -35.805938720703125,
      "activations/layer11_attention_weight_max": 31.114139556884766,
      "activations/layer11_attention_weight_min": -33.04637145996094,
      "activations/layer12_attention_weight_max": 17.7415714263916,
      "activations/layer12_attention_weight_min": -24.81353187561035,
      "activations/layer13_attention_weight_max": 41.11579895019531,
      "activations/layer13_attention_weight_min": -25.887624740600586,
      "activations/layer14_attention_weight_max": 42.61997985839844,
      "activations/layer14_attention_weight_min": -30.504005432128906,
      "activations/layer15_attention_weight_max": 39.48070526123047,
      "activations/layer15_attention_weight_min": -29.312345504760742,
      "activations/layer16_attention_weight_max": 39.40488052368164,
      "activations/layer16_attention_weight_min": -33.27485275268555,
      "activations/layer17_attention_weight_max": 60.29867172241211,
      "activations/layer17_attention_weight_min": -48.75996780395508,
      "activations/layer18_attention_weight_max": 60.02198028564453,
      "activations/layer18_attention_weight_min": -45.21402359008789,
      "activations/layer19_attention_weight_max": 29.489791870117188,
      "activations/layer19_attention_weight_min": -24.58247184753418,
      "activations/layer1_attention_weight_max": 16.970083236694336,
      "activations/layer1_attention_weight_min": -14.218257904052734,
      "activations/layer20_attention_weight_max": 30.74407196044922,
      "activations/layer20_attention_weight_min": -20.463603973388672,
      "activations/layer21_attention_weight_max": 57.02766036987305,
      "activations/layer21_attention_weight_min": -23.740293502807617,
      "activations/layer22_attention_weight_max": 36.591880798339844,
      "activations/layer22_attention_weight_min": -25.151702880859375,
      "activations/layer23_attention_weight_max": 47.85123825073242,
      "activations/layer23_attention_weight_min": -24.490028381347656,
      "activations/layer2_attention_weight_max": 31.83837890625,
      "activations/layer2_attention_weight_min": -30.99189567565918,
      "activations/layer3_attention_weight_max": 85.05880737304688,
      "activations/layer3_attention_weight_min": -89.30473327636719,
      "activations/layer4_attention_weight_max": 79.98118591308594,
      "activations/layer4_attention_weight_min": -83.20539093017578,
      "activations/layer5_attention_weight_max": 61.33854675292969,
      "activations/layer5_attention_weight_min": -72.36746978759766,
      "activations/layer6_attention_weight_max": 48.31224822998047,
      "activations/layer6_attention_weight_min": -47.52634811401367,
      "activations/layer7_attention_weight_max": 62.13963317871094,
      "activations/layer7_attention_weight_min": -59.68553161621094,
      "activations/layer8_attention_weight_max": 43.98763656616211,
      "activations/layer8_attention_weight_min": -47.25950622558594,
      "activations/layer9_attention_weight_max": 44.01455307006836,
      "activations/layer9_attention_weight_min": -45.048675537109375,
      "epoch": 18.36,
      "learning_rate": 3.189583333333333e-05,
      "loss": 2.7364,
      "step": 316050
    },
    {
      "activations/layer0_attention_weight_max": 14.87964153289795,
      "activations/layer0_attention_weight_min": -13.242050170898438,
      "activations/layer10_attention_weight_max": 37.44192123413086,
      "activations/layer10_attention_weight_min": -38.7306022644043,
      "activations/layer11_attention_weight_max": 33.02616882324219,
      "activations/layer11_attention_weight_min": -33.277652740478516,
      "activations/layer12_attention_weight_max": 18.971601486206055,
      "activations/layer12_attention_weight_min": -24.473731994628906,
      "activations/layer13_attention_weight_max": 34.781837463378906,
      "activations/layer13_attention_weight_min": -30.348045349121094,
      "activations/layer14_attention_weight_max": 35.04201126098633,
      "activations/layer14_attention_weight_min": -30.46844482421875,
      "activations/layer15_attention_weight_max": 33.271053314208984,
      "activations/layer15_attention_weight_min": -30.50156593322754,
      "activations/layer16_attention_weight_max": 35.97632598876953,
      "activations/layer16_attention_weight_min": -33.88518524169922,
      "activations/layer17_attention_weight_max": 50.480472564697266,
      "activations/layer17_attention_weight_min": -46.02214431762695,
      "activations/layer18_attention_weight_max": 47.20856475830078,
      "activations/layer18_attention_weight_min": -38.7332763671875,
      "activations/layer19_attention_weight_max": 25.062986373901367,
      "activations/layer19_attention_weight_min": -22.428205490112305,
      "activations/layer1_attention_weight_max": 15.872632026672363,
      "activations/layer1_attention_weight_min": -13.551236152648926,
      "activations/layer20_attention_weight_max": 24.45363426208496,
      "activations/layer20_attention_weight_min": -23.229278564453125,
      "activations/layer21_attention_weight_max": 39.634315490722656,
      "activations/layer21_attention_weight_min": -26.93413543701172,
      "activations/layer22_attention_weight_max": 28.163572311401367,
      "activations/layer22_attention_weight_min": -26.67304039001465,
      "activations/layer23_attention_weight_max": 38.830482482910156,
      "activations/layer23_attention_weight_min": -25.140296936035156,
      "activations/layer2_attention_weight_max": 31.90935516357422,
      "activations/layer2_attention_weight_min": -30.676097869873047,
      "activations/layer3_attention_weight_max": 85.6590576171875,
      "activations/layer3_attention_weight_min": -87.66456604003906,
      "activations/layer4_attention_weight_max": 83.28295135498047,
      "activations/layer4_attention_weight_min": -83.43840789794922,
      "activations/layer5_attention_weight_max": 61.242034912109375,
      "activations/layer5_attention_weight_min": -71.3314437866211,
      "activations/layer6_attention_weight_max": 50.85475158691406,
      "activations/layer6_attention_weight_min": -51.46591567993164,
      "activations/layer7_attention_weight_max": 67.2251968383789,
      "activations/layer7_attention_weight_min": -68.45861053466797,
      "activations/layer8_attention_weight_max": 48.4241828918457,
      "activations/layer8_attention_weight_min": -52.80537796020508,
      "activations/layer9_attention_weight_max": 45.957340240478516,
      "activations/layer9_attention_weight_min": -47.51568603515625,
      "epoch": 18.37,
      "learning_rate": 3.1876893939393935e-05,
      "loss": 2.7243,
      "step": 316100
    },
    {
      "activations/layer0_attention_weight_max": 15.469996452331543,
      "activations/layer0_attention_weight_min": -13.645143508911133,
      "activations/layer10_attention_weight_max": 34.72229766845703,
      "activations/layer10_attention_weight_min": -34.75700759887695,
      "activations/layer11_attention_weight_max": 31.670148849487305,
      "activations/layer11_attention_weight_min": -32.22772216796875,
      "activations/layer12_attention_weight_max": 18.66693878173828,
      "activations/layer12_attention_weight_min": -25.7824764251709,
      "activations/layer13_attention_weight_max": 35.28465270996094,
      "activations/layer13_attention_weight_min": -28.50444984436035,
      "activations/layer14_attention_weight_max": 37.21811294555664,
      "activations/layer14_attention_weight_min": -30.630578994750977,
      "activations/layer15_attention_weight_max": 33.65541076660156,
      "activations/layer15_attention_weight_min": -30.390939712524414,
      "activations/layer16_attention_weight_max": 34.54380416870117,
      "activations/layer16_attention_weight_min": -32.40953826904297,
      "activations/layer17_attention_weight_max": 55.00973129272461,
      "activations/layer17_attention_weight_min": -44.145172119140625,
      "activations/layer18_attention_weight_max": 50.38749313354492,
      "activations/layer18_attention_weight_min": -41.1284294128418,
      "activations/layer19_attention_weight_max": 27.82196044921875,
      "activations/layer19_attention_weight_min": -23.379209518432617,
      "activations/layer1_attention_weight_max": 17.201417922973633,
      "activations/layer1_attention_weight_min": -14.368330955505371,
      "activations/layer20_attention_weight_max": 27.02391815185547,
      "activations/layer20_attention_weight_min": -21.877052307128906,
      "activations/layer21_attention_weight_max": 39.79492950439453,
      "activations/layer21_attention_weight_min": -20.637451171875,
      "activations/layer22_attention_weight_max": 36.26029586791992,
      "activations/layer22_attention_weight_min": -25.057838439941406,
      "activations/layer23_attention_weight_max": 47.237876892089844,
      "activations/layer23_attention_weight_min": -24.251646041870117,
      "activations/layer2_attention_weight_max": 30.155834197998047,
      "activations/layer2_attention_weight_min": -29.02737045288086,
      "activations/layer3_attention_weight_max": 80.75099182128906,
      "activations/layer3_attention_weight_min": -81.16815185546875,
      "activations/layer4_attention_weight_max": 79.2840805053711,
      "activations/layer4_attention_weight_min": -80.67684173583984,
      "activations/layer5_attention_weight_max": 57.50218200683594,
      "activations/layer5_attention_weight_min": -77.37161254882812,
      "activations/layer6_attention_weight_max": 46.022518157958984,
      "activations/layer6_attention_weight_min": -48.29271697998047,
      "activations/layer7_attention_weight_max": 59.09355545043945,
      "activations/layer7_attention_weight_min": -63.14678192138672,
      "activations/layer8_attention_weight_max": 41.90393829345703,
      "activations/layer8_attention_weight_min": -46.7435188293457,
      "activations/layer9_attention_weight_max": 41.986515045166016,
      "activations/layer9_attention_weight_min": -43.56386184692383,
      "epoch": 18.37,
      "learning_rate": 3.1857954545454544e-05,
      "loss": 2.7369,
      "step": 316150
    },
    {
      "activations/layer0_attention_weight_max": 15.509361267089844,
      "activations/layer0_attention_weight_min": -13.848779678344727,
      "activations/layer10_attention_weight_max": 40.55257797241211,
      "activations/layer10_attention_weight_min": -38.15312957763672,
      "activations/layer11_attention_weight_max": 39.05264663696289,
      "activations/layer11_attention_weight_min": -36.510650634765625,
      "activations/layer12_attention_weight_max": 19.922094345092773,
      "activations/layer12_attention_weight_min": -25.625566482543945,
      "activations/layer13_attention_weight_max": 33.82712936401367,
      "activations/layer13_attention_weight_min": -29.689048767089844,
      "activations/layer14_attention_weight_max": 37.41105651855469,
      "activations/layer14_attention_weight_min": -30.498058319091797,
      "activations/layer15_attention_weight_max": 33.97416305541992,
      "activations/layer15_attention_weight_min": -30.898052215576172,
      "activations/layer16_attention_weight_max": 34.01889419555664,
      "activations/layer16_attention_weight_min": -32.06357955932617,
      "activations/layer17_attention_weight_max": 52.69108200073242,
      "activations/layer17_attention_weight_min": -45.88162612915039,
      "activations/layer18_attention_weight_max": 47.53745651245117,
      "activations/layer18_attention_weight_min": -43.45383071899414,
      "activations/layer19_attention_weight_max": 25.734289169311523,
      "activations/layer19_attention_weight_min": -24.206836700439453,
      "activations/layer1_attention_weight_max": 16.184032440185547,
      "activations/layer1_attention_weight_min": -13.522056579589844,
      "activations/layer20_attention_weight_max": 25.911808013916016,
      "activations/layer20_attention_weight_min": -23.484920501708984,
      "activations/layer21_attention_weight_max": 37.918663024902344,
      "activations/layer21_attention_weight_min": -23.400108337402344,
      "activations/layer22_attention_weight_max": 29.043582916259766,
      "activations/layer22_attention_weight_min": -27.595518112182617,
      "activations/layer23_attention_weight_max": 35.35511016845703,
      "activations/layer23_attention_weight_min": -24.18788719177246,
      "activations/layer2_attention_weight_max": 33.75347900390625,
      "activations/layer2_attention_weight_min": -32.21550369262695,
      "activations/layer3_attention_weight_max": 86.07520294189453,
      "activations/layer3_attention_weight_min": -89.9171371459961,
      "activations/layer4_attention_weight_max": 86.537353515625,
      "activations/layer4_attention_weight_min": -86.83080291748047,
      "activations/layer5_attention_weight_max": 60.75587463378906,
      "activations/layer5_attention_weight_min": -69.61691284179688,
      "activations/layer6_attention_weight_max": 53.50420379638672,
      "activations/layer6_attention_weight_min": -55.00590133666992,
      "activations/layer7_attention_weight_max": 73.27669525146484,
      "activations/layer7_attention_weight_min": -65.1808853149414,
      "activations/layer8_attention_weight_max": 53.850643157958984,
      "activations/layer8_attention_weight_min": -55.781105041503906,
      "activations/layer9_attention_weight_max": 51.8353271484375,
      "activations/layer9_attention_weight_min": -47.28010177612305,
      "epoch": 18.37,
      "learning_rate": 3.183901515151515e-05,
      "loss": 2.7336,
      "step": 316200
    },
    {
      "activations/layer0_attention_weight_max": 15.731897354125977,
      "activations/layer0_attention_weight_min": -13.840715408325195,
      "activations/layer10_attention_weight_max": 37.03124237060547,
      "activations/layer10_attention_weight_min": -37.303123474121094,
      "activations/layer11_attention_weight_max": 34.61988067626953,
      "activations/layer11_attention_weight_min": -36.070098876953125,
      "activations/layer12_attention_weight_max": 21.298063278198242,
      "activations/layer12_attention_weight_min": -21.99867057800293,
      "activations/layer13_attention_weight_max": 35.68983840942383,
      "activations/layer13_attention_weight_min": -28.493364334106445,
      "activations/layer14_attention_weight_max": 35.540977478027344,
      "activations/layer14_attention_weight_min": -30.375762939453125,
      "activations/layer15_attention_weight_max": 32.03006362915039,
      "activations/layer15_attention_weight_min": -29.788341522216797,
      "activations/layer16_attention_weight_max": 32.294395446777344,
      "activations/layer16_attention_weight_min": -30.659133911132812,
      "activations/layer17_attention_weight_max": 54.71796417236328,
      "activations/layer17_attention_weight_min": -45.29374694824219,
      "activations/layer18_attention_weight_max": 44.484493255615234,
      "activations/layer18_attention_weight_min": -38.72435760498047,
      "activations/layer19_attention_weight_max": 23.905136108398438,
      "activations/layer19_attention_weight_min": -21.559711456298828,
      "activations/layer1_attention_weight_max": 15.337811470031738,
      "activations/layer1_attention_weight_min": -14.212725639343262,
      "activations/layer20_attention_weight_max": 22.398107528686523,
      "activations/layer20_attention_weight_min": -20.547792434692383,
      "activations/layer21_attention_weight_max": 32.0589714050293,
      "activations/layer21_attention_weight_min": -22.028196334838867,
      "activations/layer22_attention_weight_max": 26.5490665435791,
      "activations/layer22_attention_weight_min": -25.66502571105957,
      "activations/layer23_attention_weight_max": 35.529937744140625,
      "activations/layer23_attention_weight_min": -23.764814376831055,
      "activations/layer2_attention_weight_max": 29.300785064697266,
      "activations/layer2_attention_weight_min": -27.854557037353516,
      "activations/layer3_attention_weight_max": 83.04325103759766,
      "activations/layer3_attention_weight_min": -83.74032592773438,
      "activations/layer4_attention_weight_max": 81.02155303955078,
      "activations/layer4_attention_weight_min": -84.13243865966797,
      "activations/layer5_attention_weight_max": 60.223106384277344,
      "activations/layer5_attention_weight_min": -73.89825439453125,
      "activations/layer6_attention_weight_max": 48.3365592956543,
      "activations/layer6_attention_weight_min": -50.65317153930664,
      "activations/layer7_attention_weight_max": 61.72611618041992,
      "activations/layer7_attention_weight_min": -60.47544860839844,
      "activations/layer8_attention_weight_max": 48.05720520019531,
      "activations/layer8_attention_weight_min": -49.54344177246094,
      "activations/layer9_attention_weight_max": 46.16829299926758,
      "activations/layer9_attention_weight_min": -46.83475112915039,
      "epoch": 18.38,
      "learning_rate": 3.1820075757575753e-05,
      "loss": 2.7343,
      "step": 316250
    },
    {
      "activations/layer0_attention_weight_max": 15.066532135009766,
      "activations/layer0_attention_weight_min": -13.793943405151367,
      "activations/layer10_attention_weight_max": 35.66123962402344,
      "activations/layer10_attention_weight_min": -37.52294921875,
      "activations/layer11_attention_weight_max": 33.50117111206055,
      "activations/layer11_attention_weight_min": -34.9688835144043,
      "activations/layer12_attention_weight_max": 17.997875213623047,
      "activations/layer12_attention_weight_min": -26.40410041809082,
      "activations/layer13_attention_weight_max": 34.05598068237305,
      "activations/layer13_attention_weight_min": -28.56136703491211,
      "activations/layer14_attention_weight_max": 38.95219421386719,
      "activations/layer14_attention_weight_min": -29.993701934814453,
      "activations/layer15_attention_weight_max": 35.81056213378906,
      "activations/layer15_attention_weight_min": -30.811805725097656,
      "activations/layer16_attention_weight_max": 36.2071418762207,
      "activations/layer16_attention_weight_min": -33.49072265625,
      "activations/layer17_attention_weight_max": 55.65504837036133,
      "activations/layer17_attention_weight_min": -46.295867919921875,
      "activations/layer18_attention_weight_max": 49.82136917114258,
      "activations/layer18_attention_weight_min": -41.439884185791016,
      "activations/layer19_attention_weight_max": 27.568683624267578,
      "activations/layer19_attention_weight_min": -24.18401336669922,
      "activations/layer1_attention_weight_max": 16.710819244384766,
      "activations/layer1_attention_weight_min": -13.402865409851074,
      "activations/layer20_attention_weight_max": 24.949661254882812,
      "activations/layer20_attention_weight_min": -21.685871124267578,
      "activations/layer21_attention_weight_max": 38.8052978515625,
      "activations/layer21_attention_weight_min": -24.308923721313477,
      "activations/layer22_attention_weight_max": 32.04482650756836,
      "activations/layer22_attention_weight_min": -25.580034255981445,
      "activations/layer23_attention_weight_max": 37.39778137207031,
      "activations/layer23_attention_weight_min": -25.232913970947266,
      "activations/layer2_attention_weight_max": 31.115367889404297,
      "activations/layer2_attention_weight_min": -31.959104537963867,
      "activations/layer3_attention_weight_max": 89.77827453613281,
      "activations/layer3_attention_weight_min": -89.71369171142578,
      "activations/layer4_attention_weight_max": 82.39153289794922,
      "activations/layer4_attention_weight_min": -84.69039154052734,
      "activations/layer5_attention_weight_max": 61.239097595214844,
      "activations/layer5_attention_weight_min": -67.9859848022461,
      "activations/layer6_attention_weight_max": 48.519508361816406,
      "activations/layer6_attention_weight_min": -52.711402893066406,
      "activations/layer7_attention_weight_max": 64.8454360961914,
      "activations/layer7_attention_weight_min": -62.85133743286133,
      "activations/layer8_attention_weight_max": 46.52022171020508,
      "activations/layer8_attention_weight_min": -49.70866775512695,
      "activations/layer9_attention_weight_max": 45.84371566772461,
      "activations/layer9_attention_weight_min": -48.4626579284668,
      "epoch": 18.38,
      "learning_rate": 3.180113636363636e-05,
      "loss": 2.7431,
      "step": 316300
    },
    {
      "activations/layer0_attention_weight_max": 16.357955932617188,
      "activations/layer0_attention_weight_min": -13.689239501953125,
      "activations/layer10_attention_weight_max": 33.865234375,
      "activations/layer10_attention_weight_min": -33.282745361328125,
      "activations/layer11_attention_weight_max": 31.39590072631836,
      "activations/layer11_attention_weight_min": -32.00940704345703,
      "activations/layer12_attention_weight_max": 17.212108612060547,
      "activations/layer12_attention_weight_min": -23.055830001831055,
      "activations/layer13_attention_weight_max": 32.11590576171875,
      "activations/layer13_attention_weight_min": -26.381145477294922,
      "activations/layer14_attention_weight_max": 30.45876693725586,
      "activations/layer14_attention_weight_min": -27.076936721801758,
      "activations/layer15_attention_weight_max": 29.386154174804688,
      "activations/layer15_attention_weight_min": -28.77928352355957,
      "activations/layer16_attention_weight_max": 28.840721130371094,
      "activations/layer16_attention_weight_min": -30.18603515625,
      "activations/layer17_attention_weight_max": 44.17911911010742,
      "activations/layer17_attention_weight_min": -42.892276763916016,
      "activations/layer18_attention_weight_max": 42.547149658203125,
      "activations/layer18_attention_weight_min": -38.956546783447266,
      "activations/layer19_attention_weight_max": 23.129289627075195,
      "activations/layer19_attention_weight_min": -23.565916061401367,
      "activations/layer1_attention_weight_max": 15.192310333251953,
      "activations/layer1_attention_weight_min": -13.435481071472168,
      "activations/layer20_attention_weight_max": 21.48152732849121,
      "activations/layer20_attention_weight_min": -23.1878719329834,
      "activations/layer21_attention_weight_max": 32.82612991333008,
      "activations/layer21_attention_weight_min": -21.48678207397461,
      "activations/layer22_attention_weight_max": 26.878131866455078,
      "activations/layer22_attention_weight_min": -26.008487701416016,
      "activations/layer23_attention_weight_max": 40.532562255859375,
      "activations/layer23_attention_weight_min": -25.12324333190918,
      "activations/layer2_attention_weight_max": 29.702619552612305,
      "activations/layer2_attention_weight_min": -29.994810104370117,
      "activations/layer3_attention_weight_max": 77.37571716308594,
      "activations/layer3_attention_weight_min": -79.47562408447266,
      "activations/layer4_attention_weight_max": 79.84080505371094,
      "activations/layer4_attention_weight_min": -82.32038116455078,
      "activations/layer5_attention_weight_max": 60.602027893066406,
      "activations/layer5_attention_weight_min": -70.99968719482422,
      "activations/layer6_attention_weight_max": 47.740108489990234,
      "activations/layer6_attention_weight_min": -50.75456619262695,
      "activations/layer7_attention_weight_max": 62.67102813720703,
      "activations/layer7_attention_weight_min": -62.692256927490234,
      "activations/layer8_attention_weight_max": 43.438690185546875,
      "activations/layer8_attention_weight_min": -47.1545295715332,
      "activations/layer9_attention_weight_max": 42.28677749633789,
      "activations/layer9_attention_weight_min": -44.14413070678711,
      "epoch": 18.38,
      "learning_rate": 3.178257575757576e-05,
      "loss": 2.7274,
      "step": 316350
    },
    {
      "activations/layer0_attention_weight_max": 15.285271644592285,
      "activations/layer0_attention_weight_min": -13.44890308380127,
      "activations/layer10_attention_weight_max": 35.06154251098633,
      "activations/layer10_attention_weight_min": -36.111671447753906,
      "activations/layer11_attention_weight_max": 32.72169494628906,
      "activations/layer11_attention_weight_min": -35.30351257324219,
      "activations/layer12_attention_weight_max": 20.72479820251465,
      "activations/layer12_attention_weight_min": -26.375267028808594,
      "activations/layer13_attention_weight_max": 37.29644012451172,
      "activations/layer13_attention_weight_min": -27.368074417114258,
      "activations/layer14_attention_weight_max": 35.796424865722656,
      "activations/layer14_attention_weight_min": -31.8210506439209,
      "activations/layer15_attention_weight_max": 33.1892204284668,
      "activations/layer15_attention_weight_min": -32.40522766113281,
      "activations/layer16_attention_weight_max": 36.30766677856445,
      "activations/layer16_attention_weight_min": -36.54611587524414,
      "activations/layer17_attention_weight_max": 53.0267448425293,
      "activations/layer17_attention_weight_min": -51.12213134765625,
      "activations/layer18_attention_weight_max": 46.37340545654297,
      "activations/layer18_attention_weight_min": -41.896907806396484,
      "activations/layer19_attention_weight_max": 24.651540756225586,
      "activations/layer19_attention_weight_min": -23.504579544067383,
      "activations/layer1_attention_weight_max": 16.70688247680664,
      "activations/layer1_attention_weight_min": -13.395895004272461,
      "activations/layer20_attention_weight_max": 23.521726608276367,
      "activations/layer20_attention_weight_min": -21.795146942138672,
      "activations/layer21_attention_weight_max": 38.00232696533203,
      "activations/layer21_attention_weight_min": -24.135202407836914,
      "activations/layer22_attention_weight_max": 29.73822784423828,
      "activations/layer22_attention_weight_min": -26.492422103881836,
      "activations/layer23_attention_weight_max": 36.270118713378906,
      "activations/layer23_attention_weight_min": -24.707752227783203,
      "activations/layer2_attention_weight_max": 29.71413803100586,
      "activations/layer2_attention_weight_min": -28.4281005859375,
      "activations/layer3_attention_weight_max": 84.9085693359375,
      "activations/layer3_attention_weight_min": -83.26091766357422,
      "activations/layer4_attention_weight_max": 79.28194427490234,
      "activations/layer4_attention_weight_min": -78.56071472167969,
      "activations/layer5_attention_weight_max": 59.12493133544922,
      "activations/layer5_attention_weight_min": -69.92384338378906,
      "activations/layer6_attention_weight_max": 48.28810501098633,
      "activations/layer6_attention_weight_min": -49.57554244995117,
      "activations/layer7_attention_weight_max": 65.51451873779297,
      "activations/layer7_attention_weight_min": -65.07186126708984,
      "activations/layer8_attention_weight_max": 44.955101013183594,
      "activations/layer8_attention_weight_min": -49.9069938659668,
      "activations/layer9_attention_weight_max": 45.8973503112793,
      "activations/layer9_attention_weight_min": -44.89376449584961,
      "epoch": 18.38,
      "learning_rate": 3.176363636363636e-05,
      "loss": 2.7513,
      "step": 316400
    },
    {
      "activations/layer0_attention_weight_max": 15.436469078063965,
      "activations/layer0_attention_weight_min": -13.821958541870117,
      "activations/layer10_attention_weight_max": 34.00727081298828,
      "activations/layer10_attention_weight_min": -36.71150588989258,
      "activations/layer11_attention_weight_max": 32.68540954589844,
      "activations/layer11_attention_weight_min": -35.26974868774414,
      "activations/layer12_attention_weight_max": 19.04094696044922,
      "activations/layer12_attention_weight_min": -23.839567184448242,
      "activations/layer13_attention_weight_max": 35.31919860839844,
      "activations/layer13_attention_weight_min": -27.02170753479004,
      "activations/layer14_attention_weight_max": 33.159873962402344,
      "activations/layer14_attention_weight_min": -29.62908363342285,
      "activations/layer15_attention_weight_max": 30.395734786987305,
      "activations/layer15_attention_weight_min": -30.56494903564453,
      "activations/layer16_attention_weight_max": 32.938140869140625,
      "activations/layer16_attention_weight_min": -32.04926681518555,
      "activations/layer17_attention_weight_max": 49.560420989990234,
      "activations/layer17_attention_weight_min": -45.42476272583008,
      "activations/layer18_attention_weight_max": 45.76468276977539,
      "activations/layer18_attention_weight_min": -39.44954299926758,
      "activations/layer19_attention_weight_max": 23.163522720336914,
      "activations/layer19_attention_weight_min": -22.796533584594727,
      "activations/layer1_attention_weight_max": 17.980344772338867,
      "activations/layer1_attention_weight_min": -14.011683464050293,
      "activations/layer20_attention_weight_max": 22.657472610473633,
      "activations/layer20_attention_weight_min": -21.263580322265625,
      "activations/layer21_attention_weight_max": 31.87815284729004,
      "activations/layer21_attention_weight_min": -20.61635398864746,
      "activations/layer22_attention_weight_max": 30.035377502441406,
      "activations/layer22_attention_weight_min": -25.583106994628906,
      "activations/layer23_attention_weight_max": 39.97574234008789,
      "activations/layer23_attention_weight_min": -25.50830841064453,
      "activations/layer2_attention_weight_max": 30.72161865234375,
      "activations/layer2_attention_weight_min": -29.89187240600586,
      "activations/layer3_attention_weight_max": 79.45661926269531,
      "activations/layer3_attention_weight_min": -87.18144989013672,
      "activations/layer4_attention_weight_max": 80.33832550048828,
      "activations/layer4_attention_weight_min": -83.45293426513672,
      "activations/layer5_attention_weight_max": 57.55528259277344,
      "activations/layer5_attention_weight_min": -75.06884765625,
      "activations/layer6_attention_weight_max": 46.95558547973633,
      "activations/layer6_attention_weight_min": -48.519229888916016,
      "activations/layer7_attention_weight_max": 59.79529571533203,
      "activations/layer7_attention_weight_min": -60.26100540161133,
      "activations/layer8_attention_weight_max": 46.40028762817383,
      "activations/layer8_attention_weight_min": -46.85039520263672,
      "activations/layer9_attention_weight_max": 41.89731979370117,
      "activations/layer9_attention_weight_min": -45.111122131347656,
      "epoch": 18.39,
      "learning_rate": 3.174469696969697e-05,
      "loss": 2.7448,
      "step": 316450
    },
    {
      "activations/layer0_attention_weight_max": 15.613924980163574,
      "activations/layer0_attention_weight_min": -13.27254867553711,
      "activations/layer10_attention_weight_max": 34.13941192626953,
      "activations/layer10_attention_weight_min": -35.09162139892578,
      "activations/layer11_attention_weight_max": 35.221736907958984,
      "activations/layer11_attention_weight_min": -34.91936492919922,
      "activations/layer12_attention_weight_max": 19.787153244018555,
      "activations/layer12_attention_weight_min": -26.931150436401367,
      "activations/layer13_attention_weight_max": 35.941741943359375,
      "activations/layer13_attention_weight_min": -27.597543716430664,
      "activations/layer14_attention_weight_max": 35.02290344238281,
      "activations/layer14_attention_weight_min": -29.139022827148438,
      "activations/layer15_attention_weight_max": 33.185726165771484,
      "activations/layer15_attention_weight_min": -29.418787002563477,
      "activations/layer16_attention_weight_max": 33.92762756347656,
      "activations/layer16_attention_weight_min": -30.98503303527832,
      "activations/layer17_attention_weight_max": 49.86940383911133,
      "activations/layer17_attention_weight_min": -44.71432113647461,
      "activations/layer18_attention_weight_max": 41.48453903198242,
      "activations/layer18_attention_weight_min": -39.474853515625,
      "activations/layer19_attention_weight_max": 21.29918098449707,
      "activations/layer19_attention_weight_min": -23.468374252319336,
      "activations/layer1_attention_weight_max": 16.660072326660156,
      "activations/layer1_attention_weight_min": -13.4108304977417,
      "activations/layer20_attention_weight_max": 20.298234939575195,
      "activations/layer20_attention_weight_min": -23.345897674560547,
      "activations/layer21_attention_weight_max": 34.07211685180664,
      "activations/layer21_attention_weight_min": -21.907651901245117,
      "activations/layer22_attention_weight_max": 27.142969131469727,
      "activations/layer22_attention_weight_min": -25.15676498413086,
      "activations/layer23_attention_weight_max": 35.04441833496094,
      "activations/layer23_attention_weight_min": -23.16473960876465,
      "activations/layer2_attention_weight_max": 33.08014678955078,
      "activations/layer2_attention_weight_min": -30.989765167236328,
      "activations/layer3_attention_weight_max": 93.38288879394531,
      "activations/layer3_attention_weight_min": -99.52073669433594,
      "activations/layer4_attention_weight_max": 89.42463684082031,
      "activations/layer4_attention_weight_min": -84.96656799316406,
      "activations/layer5_attention_weight_max": 60.12763977050781,
      "activations/layer5_attention_weight_min": -69.14970397949219,
      "activations/layer6_attention_weight_max": 47.87702178955078,
      "activations/layer6_attention_weight_min": -51.042503356933594,
      "activations/layer7_attention_weight_max": 64.9136734008789,
      "activations/layer7_attention_weight_min": -64.92229461669922,
      "activations/layer8_attention_weight_max": 46.414798736572266,
      "activations/layer8_attention_weight_min": -48.839012145996094,
      "activations/layer9_attention_weight_max": 50.99644088745117,
      "activations/layer9_attention_weight_min": -46.37839126586914,
      "epoch": 18.39,
      "learning_rate": 3.172575757575757e-05,
      "loss": 2.7272,
      "step": 316500
    },
    {
      "activations/layer0_attention_weight_max": 15.406627655029297,
      "activations/layer0_attention_weight_min": -13.46707820892334,
      "activations/layer10_attention_weight_max": 41.36354446411133,
      "activations/layer10_attention_weight_min": -37.44324493408203,
      "activations/layer11_attention_weight_max": 39.079593658447266,
      "activations/layer11_attention_weight_min": -36.70917510986328,
      "activations/layer12_attention_weight_max": 19.25751304626465,
      "activations/layer12_attention_weight_min": -29.321474075317383,
      "activations/layer13_attention_weight_max": 35.81134033203125,
      "activations/layer13_attention_weight_min": -30.05929183959961,
      "activations/layer14_attention_weight_max": 35.62746810913086,
      "activations/layer14_attention_weight_min": -30.325050354003906,
      "activations/layer15_attention_weight_max": 36.06549072265625,
      "activations/layer15_attention_weight_min": -30.14078712463379,
      "activations/layer16_attention_weight_max": 34.19694900512695,
      "activations/layer16_attention_weight_min": -33.368011474609375,
      "activations/layer17_attention_weight_max": 51.70301055908203,
      "activations/layer17_attention_weight_min": -46.846534729003906,
      "activations/layer18_attention_weight_max": 46.37863540649414,
      "activations/layer18_attention_weight_min": -39.74159240722656,
      "activations/layer19_attention_weight_max": 25.00843048095703,
      "activations/layer19_attention_weight_min": -23.77254867553711,
      "activations/layer1_attention_weight_max": 16.376630783081055,
      "activations/layer1_attention_weight_min": -12.62865161895752,
      "activations/layer20_attention_weight_max": 23.26097869873047,
      "activations/layer20_attention_weight_min": -22.04703712463379,
      "activations/layer21_attention_weight_max": 37.28427505493164,
      "activations/layer21_attention_weight_min": -25.01879119873047,
      "activations/layer22_attention_weight_max": 29.106735229492188,
      "activations/layer22_attention_weight_min": -27.163991928100586,
      "activations/layer23_attention_weight_max": 43.196990966796875,
      "activations/layer23_attention_weight_min": -26.828001022338867,
      "activations/layer2_attention_weight_max": 28.75275993347168,
      "activations/layer2_attention_weight_min": -30.686298370361328,
      "activations/layer3_attention_weight_max": 86.03807830810547,
      "activations/layer3_attention_weight_min": -90.69388580322266,
      "activations/layer4_attention_weight_max": 83.63990783691406,
      "activations/layer4_attention_weight_min": -89.73067474365234,
      "activations/layer5_attention_weight_max": 63.94367980957031,
      "activations/layer5_attention_weight_min": -75.63697052001953,
      "activations/layer6_attention_weight_max": 51.821414947509766,
      "activations/layer6_attention_weight_min": -52.83877944946289,
      "activations/layer7_attention_weight_max": 67.3875732421875,
      "activations/layer7_attention_weight_min": -65.1819076538086,
      "activations/layer8_attention_weight_max": 49.76134490966797,
      "activations/layer8_attention_weight_min": -50.42628479003906,
      "activations/layer9_attention_weight_max": 47.924312591552734,
      "activations/layer9_attention_weight_min": -46.6804313659668,
      "epoch": 18.39,
      "learning_rate": 3.170681818181818e-05,
      "loss": 2.7408,
      "step": 316550
    },
    {
      "activations/layer0_attention_weight_max": 15.832573890686035,
      "activations/layer0_attention_weight_min": -14.445008277893066,
      "activations/layer10_attention_weight_max": 35.9119758605957,
      "activations/layer10_attention_weight_min": -37.53114318847656,
      "activations/layer11_attention_weight_max": 34.21225357055664,
      "activations/layer11_attention_weight_min": -34.78733825683594,
      "activations/layer12_attention_weight_max": 18.183706283569336,
      "activations/layer12_attention_weight_min": -28.518089294433594,
      "activations/layer13_attention_weight_max": 48.84169387817383,
      "activations/layer13_attention_weight_min": -28.083711624145508,
      "activations/layer14_attention_weight_max": 33.851715087890625,
      "activations/layer14_attention_weight_min": -31.389934539794922,
      "activations/layer15_attention_weight_max": 31.593839645385742,
      "activations/layer15_attention_weight_min": -30.033613204956055,
      "activations/layer16_attention_weight_max": 32.916648864746094,
      "activations/layer16_attention_weight_min": -31.43022918701172,
      "activations/layer17_attention_weight_max": 54.554649353027344,
      "activations/layer17_attention_weight_min": -48.39744567871094,
      "activations/layer18_attention_weight_max": 49.03557205200195,
      "activations/layer18_attention_weight_min": -42.30736541748047,
      "activations/layer19_attention_weight_max": 22.972864151000977,
      "activations/layer19_attention_weight_min": -23.524396896362305,
      "activations/layer1_attention_weight_max": 16.865432739257812,
      "activations/layer1_attention_weight_min": -14.895133972167969,
      "activations/layer20_attention_weight_max": 21.72535514831543,
      "activations/layer20_attention_weight_min": -21.941465377807617,
      "activations/layer21_attention_weight_max": 41.926658630371094,
      "activations/layer21_attention_weight_min": -20.95831871032715,
      "activations/layer22_attention_weight_max": 29.428571701049805,
      "activations/layer22_attention_weight_min": -26.02806854248047,
      "activations/layer23_attention_weight_max": 36.477210998535156,
      "activations/layer23_attention_weight_min": -25.718883514404297,
      "activations/layer2_attention_weight_max": 30.23302459716797,
      "activations/layer2_attention_weight_min": -28.654354095458984,
      "activations/layer3_attention_weight_max": 82.03357696533203,
      "activations/layer3_attention_weight_min": -83.1524429321289,
      "activations/layer4_attention_weight_max": 80.35662078857422,
      "activations/layer4_attention_weight_min": -81.67837524414062,
      "activations/layer5_attention_weight_max": 60.69475555419922,
      "activations/layer5_attention_weight_min": -70.1970443725586,
      "activations/layer6_attention_weight_max": 48.42026901245117,
      "activations/layer6_attention_weight_min": -49.880191802978516,
      "activations/layer7_attention_weight_max": 66.78438568115234,
      "activations/layer7_attention_weight_min": -63.93422317504883,
      "activations/layer8_attention_weight_max": 46.71083450317383,
      "activations/layer8_attention_weight_min": -52.51318359375,
      "activations/layer9_attention_weight_max": 44.7548942565918,
      "activations/layer9_attention_weight_min": -45.33234786987305,
      "epoch": 18.4,
      "learning_rate": 3.168787878787879e-05,
      "loss": 2.7215,
      "step": 316600
    },
    {
      "activations/layer0_attention_weight_max": 15.611771583557129,
      "activations/layer0_attention_weight_min": -13.575328826904297,
      "activations/layer10_attention_weight_max": 35.547218322753906,
      "activations/layer10_attention_weight_min": -36.014183044433594,
      "activations/layer11_attention_weight_max": 31.19260025024414,
      "activations/layer11_attention_weight_min": -34.56298065185547,
      "activations/layer12_attention_weight_max": 19.95822525024414,
      "activations/layer12_attention_weight_min": -23.38380241394043,
      "activations/layer13_attention_weight_max": 40.0435791015625,
      "activations/layer13_attention_weight_min": -26.463655471801758,
      "activations/layer14_attention_weight_max": 37.86836242675781,
      "activations/layer14_attention_weight_min": -30.13252830505371,
      "activations/layer15_attention_weight_max": 35.520233154296875,
      "activations/layer15_attention_weight_min": -29.584497451782227,
      "activations/layer16_attention_weight_max": 36.20326614379883,
      "activations/layer16_attention_weight_min": -31.402074813842773,
      "activations/layer17_attention_weight_max": 51.88203048706055,
      "activations/layer17_attention_weight_min": -45.274497985839844,
      "activations/layer18_attention_weight_max": 49.49536895751953,
      "activations/layer18_attention_weight_min": -43.75799560546875,
      "activations/layer19_attention_weight_max": 27.239309310913086,
      "activations/layer19_attention_weight_min": -25.357677459716797,
      "activations/layer1_attention_weight_max": 15.920221328735352,
      "activations/layer1_attention_weight_min": -14.140458106994629,
      "activations/layer20_attention_weight_max": 24.38414764404297,
      "activations/layer20_attention_weight_min": -22.757484436035156,
      "activations/layer21_attention_weight_max": 37.82148742675781,
      "activations/layer21_attention_weight_min": -23.004302978515625,
      "activations/layer22_attention_weight_max": 31.47980308532715,
      "activations/layer22_attention_weight_min": -26.72495460510254,
      "activations/layer23_attention_weight_max": 36.177040100097656,
      "activations/layer23_attention_weight_min": -23.958053588867188,
      "activations/layer2_attention_weight_max": 30.18221092224121,
      "activations/layer2_attention_weight_min": -29.051719665527344,
      "activations/layer3_attention_weight_max": 84.22118377685547,
      "activations/layer3_attention_weight_min": -83.9598617553711,
      "activations/layer4_attention_weight_max": 83.83863067626953,
      "activations/layer4_attention_weight_min": -83.1074447631836,
      "activations/layer5_attention_weight_max": 60.425411224365234,
      "activations/layer5_attention_weight_min": -69.02655792236328,
      "activations/layer6_attention_weight_max": 50.00190353393555,
      "activations/layer6_attention_weight_min": -51.357810974121094,
      "activations/layer7_attention_weight_max": 61.48326873779297,
      "activations/layer7_attention_weight_min": -61.61652755737305,
      "activations/layer8_attention_weight_max": 46.85292053222656,
      "activations/layer8_attention_weight_min": -53.76145935058594,
      "activations/layer9_attention_weight_max": 43.6604118347168,
      "activations/layer9_attention_weight_min": -46.2327995300293,
      "epoch": 18.4,
      "learning_rate": 3.1668939393939395e-05,
      "loss": 2.7396,
      "step": 316650
    },
    {
      "activations/layer0_attention_weight_max": 15.605860710144043,
      "activations/layer0_attention_weight_min": -13.639114379882812,
      "activations/layer10_attention_weight_max": 37.263572692871094,
      "activations/layer10_attention_weight_min": -38.495887756347656,
      "activations/layer11_attention_weight_max": 34.223060607910156,
      "activations/layer11_attention_weight_min": -38.432796478271484,
      "activations/layer12_attention_weight_max": 19.799129486083984,
      "activations/layer12_attention_weight_min": -26.448545455932617,
      "activations/layer13_attention_weight_max": 33.83371353149414,
      "activations/layer13_attention_weight_min": -33.045711517333984,
      "activations/layer14_attention_weight_max": 37.63023376464844,
      "activations/layer14_attention_weight_min": -32.80063247680664,
      "activations/layer15_attention_weight_max": 34.79033660888672,
      "activations/layer15_attention_weight_min": -31.124263763427734,
      "activations/layer16_attention_weight_max": 36.08324432373047,
      "activations/layer16_attention_weight_min": -32.62753677368164,
      "activations/layer17_attention_weight_max": 55.411293029785156,
      "activations/layer17_attention_weight_min": -48.848907470703125,
      "activations/layer18_attention_weight_max": 48.53386688232422,
      "activations/layer18_attention_weight_min": -41.19805145263672,
      "activations/layer19_attention_weight_max": 24.512569427490234,
      "activations/layer19_attention_weight_min": -23.609214782714844,
      "activations/layer1_attention_weight_max": 15.538301467895508,
      "activations/layer1_attention_weight_min": -12.61978530883789,
      "activations/layer20_attention_weight_max": 22.87557601928711,
      "activations/layer20_attention_weight_min": -23.821550369262695,
      "activations/layer21_attention_weight_max": 38.79462814331055,
      "activations/layer21_attention_weight_min": -23.682973861694336,
      "activations/layer22_attention_weight_max": 29.543380737304688,
      "activations/layer22_attention_weight_min": -27.024126052856445,
      "activations/layer23_attention_weight_max": 39.770172119140625,
      "activations/layer23_attention_weight_min": -25.913272857666016,
      "activations/layer2_attention_weight_max": 31.61077880859375,
      "activations/layer2_attention_weight_min": -29.82162094116211,
      "activations/layer3_attention_weight_max": 82.7912826538086,
      "activations/layer3_attention_weight_min": -85.0337905883789,
      "activations/layer4_attention_weight_max": 85.4161376953125,
      "activations/layer4_attention_weight_min": -85.18126678466797,
      "activations/layer5_attention_weight_max": 59.76512908935547,
      "activations/layer5_attention_weight_min": -74.7170181274414,
      "activations/layer6_attention_weight_max": 48.71171951293945,
      "activations/layer6_attention_weight_min": -50.87269592285156,
      "activations/layer7_attention_weight_max": 62.828758239746094,
      "activations/layer7_attention_weight_min": -63.45161056518555,
      "activations/layer8_attention_weight_max": 50.887325286865234,
      "activations/layer8_attention_weight_min": -50.47123718261719,
      "activations/layer9_attention_weight_max": 47.16123962402344,
      "activations/layer9_attention_weight_min": -45.89795684814453,
      "epoch": 18.4,
      "learning_rate": 3.165e-05,
      "loss": 2.7238,
      "step": 316700
    },
    {
      "activations/layer0_attention_weight_max": 15.125324249267578,
      "activations/layer0_attention_weight_min": -13.216649055480957,
      "activations/layer10_attention_weight_max": 32.906158447265625,
      "activations/layer10_attention_weight_min": -33.241790771484375,
      "activations/layer11_attention_weight_max": 30.221874237060547,
      "activations/layer11_attention_weight_min": -31.835981369018555,
      "activations/layer12_attention_weight_max": 18.34587860107422,
      "activations/layer12_attention_weight_min": -24.289325714111328,
      "activations/layer13_attention_weight_max": 30.054353713989258,
      "activations/layer13_attention_weight_min": -27.039852142333984,
      "activations/layer14_attention_weight_max": 33.54924392700195,
      "activations/layer14_attention_weight_min": -27.878053665161133,
      "activations/layer15_attention_weight_max": 31.592395782470703,
      "activations/layer15_attention_weight_min": -28.640789031982422,
      "activations/layer16_attention_weight_max": 32.29882049560547,
      "activations/layer16_attention_weight_min": -29.764005661010742,
      "activations/layer17_attention_weight_max": 50.562870025634766,
      "activations/layer17_attention_weight_min": -41.1862678527832,
      "activations/layer18_attention_weight_max": 47.120826721191406,
      "activations/layer18_attention_weight_min": -35.97974395751953,
      "activations/layer19_attention_weight_max": 21.607425689697266,
      "activations/layer19_attention_weight_min": -21.21841049194336,
      "activations/layer1_attention_weight_max": 16.589767456054688,
      "activations/layer1_attention_weight_min": -13.395915985107422,
      "activations/layer20_attention_weight_max": 19.723651885986328,
      "activations/layer20_attention_weight_min": -20.810121536254883,
      "activations/layer21_attention_weight_max": 35.530364990234375,
      "activations/layer21_attention_weight_min": -20.09873390197754,
      "activations/layer22_attention_weight_max": 29.30222511291504,
      "activations/layer22_attention_weight_min": -23.065528869628906,
      "activations/layer23_attention_weight_max": 35.08600616455078,
      "activations/layer23_attention_weight_min": -25.428571701049805,
      "activations/layer2_attention_weight_max": 31.068958282470703,
      "activations/layer2_attention_weight_min": -28.406736373901367,
      "activations/layer3_attention_weight_max": 80.64925384521484,
      "activations/layer3_attention_weight_min": -81.51591491699219,
      "activations/layer4_attention_weight_max": 78.3991928100586,
      "activations/layer4_attention_weight_min": -83.05918884277344,
      "activations/layer5_attention_weight_max": 60.38098907470703,
      "activations/layer5_attention_weight_min": -73.31742858886719,
      "activations/layer6_attention_weight_max": 45.69112014770508,
      "activations/layer6_attention_weight_min": -48.551597595214844,
      "activations/layer7_attention_weight_max": 62.9640007019043,
      "activations/layer7_attention_weight_min": -58.644737243652344,
      "activations/layer8_attention_weight_max": 41.93327713012695,
      "activations/layer8_attention_weight_min": -48.95167922973633,
      "activations/layer9_attention_weight_max": 44.60169982910156,
      "activations/layer9_attention_weight_min": -42.228233337402344,
      "epoch": 18.4,
      "learning_rate": 3.1631060606060605e-05,
      "loss": 2.7341,
      "step": 316750
    },
    {
      "activations/layer0_attention_weight_max": 15.686421394348145,
      "activations/layer0_attention_weight_min": -13.500802040100098,
      "activations/layer10_attention_weight_max": 35.71820068359375,
      "activations/layer10_attention_weight_min": -37.234317779541016,
      "activations/layer11_attention_weight_max": 33.55170440673828,
      "activations/layer11_attention_weight_min": -34.435096740722656,
      "activations/layer12_attention_weight_max": 17.611547470092773,
      "activations/layer12_attention_weight_min": -24.273115158081055,
      "activations/layer13_attention_weight_max": 29.933582305908203,
      "activations/layer13_attention_weight_min": -26.19563102722168,
      "activations/layer14_attention_weight_max": 33.646766662597656,
      "activations/layer14_attention_weight_min": -29.27161407470703,
      "activations/layer15_attention_weight_max": 32.1099739074707,
      "activations/layer15_attention_weight_min": -29.029661178588867,
      "activations/layer16_attention_weight_max": 34.24054718017578,
      "activations/layer16_attention_weight_min": -31.853527069091797,
      "activations/layer17_attention_weight_max": 53.55793762207031,
      "activations/layer17_attention_weight_min": -48.02965545654297,
      "activations/layer18_attention_weight_max": 48.74596405029297,
      "activations/layer18_attention_weight_min": -42.311580657958984,
      "activations/layer19_attention_weight_max": 22.755136489868164,
      "activations/layer19_attention_weight_min": -23.038171768188477,
      "activations/layer1_attention_weight_max": 16.43501853942871,
      "activations/layer1_attention_weight_min": -14.398964881896973,
      "activations/layer20_attention_weight_max": 21.966203689575195,
      "activations/layer20_attention_weight_min": -21.40532112121582,
      "activations/layer21_attention_weight_max": 32.040184020996094,
      "activations/layer21_attention_weight_min": -22.8463077545166,
      "activations/layer22_attention_weight_max": 34.99106979370117,
      "activations/layer22_attention_weight_min": -24.91534996032715,
      "activations/layer23_attention_weight_max": 36.178001403808594,
      "activations/layer23_attention_weight_min": -24.186126708984375,
      "activations/layer2_attention_weight_max": 30.271957397460938,
      "activations/layer2_attention_weight_min": -29.668376922607422,
      "activations/layer3_attention_weight_max": 82.5154037475586,
      "activations/layer3_attention_weight_min": -83.89036560058594,
      "activations/layer4_attention_weight_max": 83.53815460205078,
      "activations/layer4_attention_weight_min": -81.1853256225586,
      "activations/layer5_attention_weight_max": 60.225765228271484,
      "activations/layer5_attention_weight_min": -70.83616638183594,
      "activations/layer6_attention_weight_max": 48.80179214477539,
      "activations/layer6_attention_weight_min": -51.18910598754883,
      "activations/layer7_attention_weight_max": 61.41107940673828,
      "activations/layer7_attention_weight_min": -64.63831329345703,
      "activations/layer8_attention_weight_max": 44.71278762817383,
      "activations/layer8_attention_weight_min": -50.94860076904297,
      "activations/layer9_attention_weight_max": 49.30015182495117,
      "activations/layer9_attention_weight_min": -47.03091049194336,
      "epoch": 18.41,
      "learning_rate": 3.1612121212121207e-05,
      "loss": 2.747,
      "step": 316800
    },
    {
      "activations/layer0_attention_weight_max": 15.307934761047363,
      "activations/layer0_attention_weight_min": -13.343793869018555,
      "activations/layer10_attention_weight_max": 35.0274658203125,
      "activations/layer10_attention_weight_min": -35.7192268371582,
      "activations/layer11_attention_weight_max": 31.302410125732422,
      "activations/layer11_attention_weight_min": -33.162620544433594,
      "activations/layer12_attention_weight_max": 19.12851905822754,
      "activations/layer12_attention_weight_min": -23.25387191772461,
      "activations/layer13_attention_weight_max": 37.82258605957031,
      "activations/layer13_attention_weight_min": -28.810583114624023,
      "activations/layer14_attention_weight_max": 37.85489273071289,
      "activations/layer14_attention_weight_min": -29.851959228515625,
      "activations/layer15_attention_weight_max": 36.947505950927734,
      "activations/layer15_attention_weight_min": -30.267715454101562,
      "activations/layer16_attention_weight_max": 36.90875244140625,
      "activations/layer16_attention_weight_min": -31.42108917236328,
      "activations/layer17_attention_weight_max": 51.993682861328125,
      "activations/layer17_attention_weight_min": -42.28337097167969,
      "activations/layer18_attention_weight_max": 48.19707489013672,
      "activations/layer18_attention_weight_min": -37.82093048095703,
      "activations/layer19_attention_weight_max": 25.13825798034668,
      "activations/layer19_attention_weight_min": -22.219192504882812,
      "activations/layer1_attention_weight_max": 15.687470436096191,
      "activations/layer1_attention_weight_min": -14.297759056091309,
      "activations/layer20_attention_weight_max": 24.959068298339844,
      "activations/layer20_attention_weight_min": -22.164457321166992,
      "activations/layer21_attention_weight_max": 41.12514877319336,
      "activations/layer21_attention_weight_min": -24.338973999023438,
      "activations/layer22_attention_weight_max": 28.9124698638916,
      "activations/layer22_attention_weight_min": -23.3981876373291,
      "activations/layer23_attention_weight_max": 38.882572174072266,
      "activations/layer23_attention_weight_min": -24.34789276123047,
      "activations/layer2_attention_weight_max": 31.881954193115234,
      "activations/layer2_attention_weight_min": -28.2613525390625,
      "activations/layer3_attention_weight_max": 85.6971206665039,
      "activations/layer3_attention_weight_min": -82.86043548583984,
      "activations/layer4_attention_weight_max": 84.63841247558594,
      "activations/layer4_attention_weight_min": -81.1399917602539,
      "activations/layer5_attention_weight_max": 62.10308837890625,
      "activations/layer5_attention_weight_min": -71.43490600585938,
      "activations/layer6_attention_weight_max": 51.44460678100586,
      "activations/layer6_attention_weight_min": -51.331363677978516,
      "activations/layer7_attention_weight_max": 64.34867858886719,
      "activations/layer7_attention_weight_min": -62.83478927612305,
      "activations/layer8_attention_weight_max": 46.629669189453125,
      "activations/layer8_attention_weight_min": -47.975128173828125,
      "activations/layer9_attention_weight_max": 43.539249420166016,
      "activations/layer9_attention_weight_min": -45.22333908081055,
      "epoch": 18.41,
      "learning_rate": 3.1593181818181815e-05,
      "loss": 2.744,
      "step": 316850
    },
    {
      "activations/layer0_attention_weight_max": 15.193419456481934,
      "activations/layer0_attention_weight_min": -13.22852897644043,
      "activations/layer10_attention_weight_max": 32.442352294921875,
      "activations/layer10_attention_weight_min": -34.3758544921875,
      "activations/layer11_attention_weight_max": 30.380741119384766,
      "activations/layer11_attention_weight_min": -32.478187561035156,
      "activations/layer12_attention_weight_max": 18.678464889526367,
      "activations/layer12_attention_weight_min": -23.206756591796875,
      "activations/layer13_attention_weight_max": 36.44184112548828,
      "activations/layer13_attention_weight_min": -28.588289260864258,
      "activations/layer14_attention_weight_max": 35.693458557128906,
      "activations/layer14_attention_weight_min": -30.537214279174805,
      "activations/layer15_attention_weight_max": 33.40483856201172,
      "activations/layer15_attention_weight_min": -30.981414794921875,
      "activations/layer16_attention_weight_max": 36.75072479248047,
      "activations/layer16_attention_weight_min": -33.750396728515625,
      "activations/layer17_attention_weight_max": 55.05876541137695,
      "activations/layer17_attention_weight_min": -48.61503982543945,
      "activations/layer18_attention_weight_max": 49.55387496948242,
      "activations/layer18_attention_weight_min": -41.00942611694336,
      "activations/layer19_attention_weight_max": 24.141454696655273,
      "activations/layer19_attention_weight_min": -25.294448852539062,
      "activations/layer1_attention_weight_max": 15.277364730834961,
      "activations/layer1_attention_weight_min": -13.445086479187012,
      "activations/layer20_attention_weight_max": 23.957067489624023,
      "activations/layer20_attention_weight_min": -24.98958969116211,
      "activations/layer21_attention_weight_max": 35.010337829589844,
      "activations/layer21_attention_weight_min": -22.1027889251709,
      "activations/layer22_attention_weight_max": 33.2966423034668,
      "activations/layer22_attention_weight_min": -28.474138259887695,
      "activations/layer23_attention_weight_max": 38.32654571533203,
      "activations/layer23_attention_weight_min": -24.926023483276367,
      "activations/layer2_attention_weight_max": 33.15256881713867,
      "activations/layer2_attention_weight_min": -30.203758239746094,
      "activations/layer3_attention_weight_max": 87.6395034790039,
      "activations/layer3_attention_weight_min": -86.3375473022461,
      "activations/layer4_attention_weight_max": 85.5183334350586,
      "activations/layer4_attention_weight_min": -85.0337905883789,
      "activations/layer5_attention_weight_max": 60.5291862487793,
      "activations/layer5_attention_weight_min": -67.19633483886719,
      "activations/layer6_attention_weight_max": 48.22587585449219,
      "activations/layer6_attention_weight_min": -48.20679473876953,
      "activations/layer7_attention_weight_max": 61.756629943847656,
      "activations/layer7_attention_weight_min": -62.96134567260742,
      "activations/layer8_attention_weight_max": 43.932125091552734,
      "activations/layer8_attention_weight_min": -47.33080291748047,
      "activations/layer9_attention_weight_max": 41.87705612182617,
      "activations/layer9_attention_weight_min": -44.27412796020508,
      "epoch": 18.41,
      "learning_rate": 3.157424242424242e-05,
      "loss": 2.723,
      "step": 316900
    },
    {
      "activations/layer0_attention_weight_max": 16.244720458984375,
      "activations/layer0_attention_weight_min": -13.721096992492676,
      "activations/layer10_attention_weight_max": 41.69398880004883,
      "activations/layer10_attention_weight_min": -38.16715621948242,
      "activations/layer11_attention_weight_max": 33.12116622924805,
      "activations/layer11_attention_weight_min": -36.41929244995117,
      "activations/layer12_attention_weight_max": 20.736974716186523,
      "activations/layer12_attention_weight_min": -23.78648567199707,
      "activations/layer13_attention_weight_max": 35.62261199951172,
      "activations/layer13_attention_weight_min": -27.782318115234375,
      "activations/layer14_attention_weight_max": 37.135738372802734,
      "activations/layer14_attention_weight_min": -29.675888061523438,
      "activations/layer15_attention_weight_max": 34.203758239746094,
      "activations/layer15_attention_weight_min": -29.23786163330078,
      "activations/layer16_attention_weight_max": 37.4845085144043,
      "activations/layer16_attention_weight_min": -34.633827209472656,
      "activations/layer17_attention_weight_max": 53.678462982177734,
      "activations/layer17_attention_weight_min": -46.49314498901367,
      "activations/layer18_attention_weight_max": 50.07677459716797,
      "activations/layer18_attention_weight_min": -41.90019607543945,
      "activations/layer19_attention_weight_max": 24.484153747558594,
      "activations/layer19_attention_weight_min": -24.392065048217773,
      "activations/layer1_attention_weight_max": 15.608591079711914,
      "activations/layer1_attention_weight_min": -13.88813304901123,
      "activations/layer20_attention_weight_max": 24.752290725708008,
      "activations/layer20_attention_weight_min": -23.283905029296875,
      "activations/layer21_attention_weight_max": 40.457862854003906,
      "activations/layer21_attention_weight_min": -22.540071487426758,
      "activations/layer22_attention_weight_max": 37.207645416259766,
      "activations/layer22_attention_weight_min": -25.51177215576172,
      "activations/layer23_attention_weight_max": 39.412681579589844,
      "activations/layer23_attention_weight_min": -23.470130920410156,
      "activations/layer2_attention_weight_max": 29.942440032958984,
      "activations/layer2_attention_weight_min": -29.155263900756836,
      "activations/layer3_attention_weight_max": 85.94120025634766,
      "activations/layer3_attention_weight_min": -86.73131561279297,
      "activations/layer4_attention_weight_max": 84.05195617675781,
      "activations/layer4_attention_weight_min": -83.85870361328125,
      "activations/layer5_attention_weight_max": 64.11418151855469,
      "activations/layer5_attention_weight_min": -74.86293029785156,
      "activations/layer6_attention_weight_max": 49.910335540771484,
      "activations/layer6_attention_weight_min": -53.90543746948242,
      "activations/layer7_attention_weight_max": 63.78619384765625,
      "activations/layer7_attention_weight_min": -65.02391052246094,
      "activations/layer8_attention_weight_max": 50.71946334838867,
      "activations/layer8_attention_weight_min": -54.31206130981445,
      "activations/layer9_attention_weight_max": 46.53854751586914,
      "activations/layer9_attention_weight_min": -51.12825393676758,
      "epoch": 18.42,
      "learning_rate": 3.1555303030303025e-05,
      "loss": 2.7367,
      "step": 316950
    },
    {
      "activations/layer0_attention_weight_max": 15.2047758102417,
      "activations/layer0_attention_weight_min": -13.596123695373535,
      "activations/layer10_attention_weight_max": 45.817928314208984,
      "activations/layer10_attention_weight_min": -43.39500427246094,
      "activations/layer11_attention_weight_max": 45.097633361816406,
      "activations/layer11_attention_weight_min": -39.59690475463867,
      "activations/layer12_attention_weight_max": 17.9770450592041,
      "activations/layer12_attention_weight_min": -24.24517822265625,
      "activations/layer13_attention_weight_max": 39.785606384277344,
      "activations/layer13_attention_weight_min": -26.617740631103516,
      "activations/layer14_attention_weight_max": 38.196800231933594,
      "activations/layer14_attention_weight_min": -28.27659034729004,
      "activations/layer15_attention_weight_max": 37.941162109375,
      "activations/layer15_attention_weight_min": -28.818567276000977,
      "activations/layer16_attention_weight_max": 33.30086135864258,
      "activations/layer16_attention_weight_min": -31.89959716796875,
      "activations/layer17_attention_weight_max": 50.2308349609375,
      "activations/layer17_attention_weight_min": -45.05082702636719,
      "activations/layer18_attention_weight_max": 46.20833206176758,
      "activations/layer18_attention_weight_min": -42.3660774230957,
      "activations/layer19_attention_weight_max": 24.34330940246582,
      "activations/layer19_attention_weight_min": -24.350805282592773,
      "activations/layer1_attention_weight_max": 15.764604568481445,
      "activations/layer1_attention_weight_min": -13.336816787719727,
      "activations/layer20_attention_weight_max": 23.117355346679688,
      "activations/layer20_attention_weight_min": -21.7469482421875,
      "activations/layer21_attention_weight_max": 33.49072265625,
      "activations/layer21_attention_weight_min": -21.34177589416504,
      "activations/layer22_attention_weight_max": 31.19132423400879,
      "activations/layer22_attention_weight_min": -23.713945388793945,
      "activations/layer23_attention_weight_max": 35.07989501953125,
      "activations/layer23_attention_weight_min": -26.903255462646484,
      "activations/layer2_attention_weight_max": 33.51234817504883,
      "activations/layer2_attention_weight_min": -32.75901794433594,
      "activations/layer3_attention_weight_max": 90.60308837890625,
      "activations/layer3_attention_weight_min": -90.77132415771484,
      "activations/layer4_attention_weight_max": 96.89470672607422,
      "activations/layer4_attention_weight_min": -87.74129486083984,
      "activations/layer5_attention_weight_max": 74.59188842773438,
      "activations/layer5_attention_weight_min": -80.94529724121094,
      "activations/layer6_attention_weight_max": 58.43358612060547,
      "activations/layer6_attention_weight_min": -56.289772033691406,
      "activations/layer7_attention_weight_max": 84.67172241210938,
      "activations/layer7_attention_weight_min": -77.42498016357422,
      "activations/layer8_attention_weight_max": 61.51780319213867,
      "activations/layer8_attention_weight_min": -58.717041015625,
      "activations/layer9_attention_weight_max": 63.17780303955078,
      "activations/layer9_attention_weight_min": -55.24856185913086,
      "epoch": 18.42,
      "learning_rate": 3.153636363636363e-05,
      "loss": 2.7395,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4574,
      "eval_samples_per_second": 507.723,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4574,
      "eval_openwebtext_samples_per_second": 507.723,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9968,
      "eval_wikitext_samples_per_second": 228.365,
      "step": 317000
    },
    {
      "epoch": 18.42,
      "eval_lambada_loss": 2.453125,
      "eval_lambada_ppl": 11.624616945432633,
      "eval_lambada_runtime": 9.5525,
      "eval_lambada_samples_per_second": 509.708,
      "step": 317000
    },
    {
      "activations/layer0_attention_weight_max": 14.873849868774414,
      "activations/layer0_attention_weight_min": -13.946333885192871,
      "activations/layer10_attention_weight_max": 37.03914260864258,
      "activations/layer10_attention_weight_min": -36.453453063964844,
      "activations/layer11_attention_weight_max": 33.57973861694336,
      "activations/layer11_attention_weight_min": -35.56390380859375,
      "activations/layer12_attention_weight_max": 19.319433212280273,
      "activations/layer12_attention_weight_min": -23.498390197753906,
      "activations/layer13_attention_weight_max": 34.467926025390625,
      "activations/layer13_attention_weight_min": -30.108318328857422,
      "activations/layer14_attention_weight_max": 36.50048828125,
      "activations/layer14_attention_weight_min": -31.792516708374023,
      "activations/layer15_attention_weight_max": 33.503665924072266,
      "activations/layer15_attention_weight_min": -30.589805603027344,
      "activations/layer16_attention_weight_max": 35.29709243774414,
      "activations/layer16_attention_weight_min": -32.725467681884766,
      "activations/layer17_attention_weight_max": 49.362884521484375,
      "activations/layer17_attention_weight_min": -44.97015380859375,
      "activations/layer18_attention_weight_max": 45.52963638305664,
      "activations/layer18_attention_weight_min": -40.23674774169922,
      "activations/layer19_attention_weight_max": 23.9852294921875,
      "activations/layer19_attention_weight_min": -24.08574867248535,
      "activations/layer1_attention_weight_max": 16.22556495666504,
      "activations/layer1_attention_weight_min": -14.251561164855957,
      "activations/layer20_attention_weight_max": 22.916650772094727,
      "activations/layer20_attention_weight_min": -22.012834548950195,
      "activations/layer21_attention_weight_max": 38.506656646728516,
      "activations/layer21_attention_weight_min": -21.69736099243164,
      "activations/layer22_attention_weight_max": 28.453285217285156,
      "activations/layer22_attention_weight_min": -25.503482818603516,
      "activations/layer23_attention_weight_max": 36.059486389160156,
      "activations/layer23_attention_weight_min": -22.96546745300293,
      "activations/layer2_attention_weight_max": 32.96397018432617,
      "activations/layer2_attention_weight_min": -28.923017501831055,
      "activations/layer3_attention_weight_max": 87.28912353515625,
      "activations/layer3_attention_weight_min": -83.95340728759766,
      "activations/layer4_attention_weight_max": 84.50521087646484,
      "activations/layer4_attention_weight_min": -81.18272399902344,
      "activations/layer5_attention_weight_max": 66.81893920898438,
      "activations/layer5_attention_weight_min": -76.63581848144531,
      "activations/layer6_attention_weight_max": 51.18729782104492,
      "activations/layer6_attention_weight_min": -51.61321258544922,
      "activations/layer7_attention_weight_max": 68.84452819824219,
      "activations/layer7_attention_weight_min": -68.19544219970703,
      "activations/layer8_attention_weight_max": 49.120277404785156,
      "activations/layer8_attention_weight_min": -54.29210662841797,
      "activations/layer9_attention_weight_max": 43.63431930541992,
      "activations/layer9_attention_weight_min": -48.609161376953125,
      "epoch": 18.42,
      "learning_rate": 3.151742424242424e-05,
      "loss": 2.7246,
      "step": 317050
    },
    {
      "activations/layer0_attention_weight_max": 15.165998458862305,
      "activations/layer0_attention_weight_min": -13.669134140014648,
      "activations/layer10_attention_weight_max": 40.287025451660156,
      "activations/layer10_attention_weight_min": -36.74397659301758,
      "activations/layer11_attention_weight_max": 34.56486511230469,
      "activations/layer11_attention_weight_min": -35.53382873535156,
      "activations/layer12_attention_weight_max": 18.259305953979492,
      "activations/layer12_attention_weight_min": -22.59465217590332,
      "activations/layer13_attention_weight_max": 33.57566833496094,
      "activations/layer13_attention_weight_min": -28.48227310180664,
      "activations/layer14_attention_weight_max": 33.287376403808594,
      "activations/layer14_attention_weight_min": -29.7115535736084,
      "activations/layer15_attention_weight_max": 30.658721923828125,
      "activations/layer15_attention_weight_min": -29.057390213012695,
      "activations/layer16_attention_weight_max": 32.65753173828125,
      "activations/layer16_attention_weight_min": -31.01483154296875,
      "activations/layer17_attention_weight_max": 50.93864440917969,
      "activations/layer17_attention_weight_min": -46.46501541137695,
      "activations/layer18_attention_weight_max": 42.46487808227539,
      "activations/layer18_attention_weight_min": -41.009742736816406,
      "activations/layer19_attention_weight_max": 24.46613883972168,
      "activations/layer19_attention_weight_min": -23.631925582885742,
      "activations/layer1_attention_weight_max": 15.699468612670898,
      "activations/layer1_attention_weight_min": -13.50912094116211,
      "activations/layer20_attention_weight_max": 24.47587776184082,
      "activations/layer20_attention_weight_min": -22.502094268798828,
      "activations/layer21_attention_weight_max": 41.45151901245117,
      "activations/layer21_attention_weight_min": -28.13857650756836,
      "activations/layer22_attention_weight_max": 26.388351440429688,
      "activations/layer22_attention_weight_min": -24.599655151367188,
      "activations/layer23_attention_weight_max": 33.60293960571289,
      "activations/layer23_attention_weight_min": -23.870342254638672,
      "activations/layer2_attention_weight_max": 30.102519989013672,
      "activations/layer2_attention_weight_min": -29.71871566772461,
      "activations/layer3_attention_weight_max": 84.39875793457031,
      "activations/layer3_attention_weight_min": -85.04998016357422,
      "activations/layer4_attention_weight_max": 81.5335922241211,
      "activations/layer4_attention_weight_min": -82.37158203125,
      "activations/layer5_attention_weight_max": 63.14183044433594,
      "activations/layer5_attention_weight_min": -73.05551147460938,
      "activations/layer6_attention_weight_max": 49.25900650024414,
      "activations/layer6_attention_weight_min": -49.911808013916016,
      "activations/layer7_attention_weight_max": 65.30150604248047,
      "activations/layer7_attention_weight_min": -63.77032470703125,
      "activations/layer8_attention_weight_max": 46.79448318481445,
      "activations/layer8_attention_weight_min": -48.338226318359375,
      "activations/layer9_attention_weight_max": 43.84879684448242,
      "activations/layer9_attention_weight_min": -44.855125427246094,
      "epoch": 18.43,
      "learning_rate": 3.149848484848484e-05,
      "loss": 2.7226,
      "step": 317100
    },
    {
      "activations/layer0_attention_weight_max": 14.973416328430176,
      "activations/layer0_attention_weight_min": -14.288045883178711,
      "activations/layer10_attention_weight_max": 34.49132537841797,
      "activations/layer10_attention_weight_min": -35.253150939941406,
      "activations/layer11_attention_weight_max": 32.855289459228516,
      "activations/layer11_attention_weight_min": -34.1051025390625,
      "activations/layer12_attention_weight_max": 18.659460067749023,
      "activations/layer12_attention_weight_min": -25.056411743164062,
      "activations/layer13_attention_weight_max": 35.12311935424805,
      "activations/layer13_attention_weight_min": -27.808334350585938,
      "activations/layer14_attention_weight_max": 35.382530212402344,
      "activations/layer14_attention_weight_min": -30.4480037689209,
      "activations/layer15_attention_weight_max": 33.246639251708984,
      "activations/layer15_attention_weight_min": -30.98531150817871,
      "activations/layer16_attention_weight_max": 33.43073654174805,
      "activations/layer16_attention_weight_min": -31.753026962280273,
      "activations/layer17_attention_weight_max": 52.579463958740234,
      "activations/layer17_attention_weight_min": -47.828285217285156,
      "activations/layer18_attention_weight_max": 45.701988220214844,
      "activations/layer18_attention_weight_min": -40.67125701904297,
      "activations/layer19_attention_weight_max": 24.88233757019043,
      "activations/layer19_attention_weight_min": -25.01148796081543,
      "activations/layer1_attention_weight_max": 15.393913269042969,
      "activations/layer1_attention_weight_min": -13.879217147827148,
      "activations/layer20_attention_weight_max": 22.47905158996582,
      "activations/layer20_attention_weight_min": -22.641069412231445,
      "activations/layer21_attention_weight_max": 38.10938262939453,
      "activations/layer21_attention_weight_min": -25.04326629638672,
      "activations/layer22_attention_weight_max": 25.918537139892578,
      "activations/layer22_attention_weight_min": -25.38031005859375,
      "activations/layer23_attention_weight_max": 36.88328552246094,
      "activations/layer23_attention_weight_min": -24.86214828491211,
      "activations/layer2_attention_weight_max": 31.796772003173828,
      "activations/layer2_attention_weight_min": -28.727514266967773,
      "activations/layer3_attention_weight_max": 84.81233215332031,
      "activations/layer3_attention_weight_min": -81.68949127197266,
      "activations/layer4_attention_weight_max": 83.78555297851562,
      "activations/layer4_attention_weight_min": -83.29402160644531,
      "activations/layer5_attention_weight_max": 63.51322555541992,
      "activations/layer5_attention_weight_min": -75.53749084472656,
      "activations/layer6_attention_weight_max": 49.457454681396484,
      "activations/layer6_attention_weight_min": -51.50962448120117,
      "activations/layer7_attention_weight_max": 64.38185119628906,
      "activations/layer7_attention_weight_min": -63.49573516845703,
      "activations/layer8_attention_weight_max": 48.00577163696289,
      "activations/layer8_attention_weight_min": -49.61215591430664,
      "activations/layer9_attention_weight_max": 44.05463790893555,
      "activations/layer9_attention_weight_min": -48.09994125366211,
      "epoch": 18.43,
      "learning_rate": 3.147954545454545e-05,
      "loss": 2.7347,
      "step": 317150
    },
    {
      "activations/layer0_attention_weight_max": 15.50821590423584,
      "activations/layer0_attention_weight_min": -14.01637077331543,
      "activations/layer10_attention_weight_max": 34.13898468017578,
      "activations/layer10_attention_weight_min": -32.861854553222656,
      "activations/layer11_attention_weight_max": 29.866758346557617,
      "activations/layer11_attention_weight_min": -30.98027801513672,
      "activations/layer12_attention_weight_max": 17.53977394104004,
      "activations/layer12_attention_weight_min": -24.252315521240234,
      "activations/layer13_attention_weight_max": 31.949546813964844,
      "activations/layer13_attention_weight_min": -26.566181182861328,
      "activations/layer14_attention_weight_max": 32.947593688964844,
      "activations/layer14_attention_weight_min": -28.612232208251953,
      "activations/layer15_attention_weight_max": 30.060935974121094,
      "activations/layer15_attention_weight_min": -28.812026977539062,
      "activations/layer16_attention_weight_max": 35.72964859008789,
      "activations/layer16_attention_weight_min": -30.861005783081055,
      "activations/layer17_attention_weight_max": 51.81620407104492,
      "activations/layer17_attention_weight_min": -45.29228210449219,
      "activations/layer18_attention_weight_max": 43.2420768737793,
      "activations/layer18_attention_weight_min": -39.98115158081055,
      "activations/layer19_attention_weight_max": 22.737829208374023,
      "activations/layer19_attention_weight_min": -23.74251365661621,
      "activations/layer1_attention_weight_max": 15.84094524383545,
      "activations/layer1_attention_weight_min": -13.44351577758789,
      "activations/layer20_attention_weight_max": 21.136049270629883,
      "activations/layer20_attention_weight_min": -22.35834503173828,
      "activations/layer21_attention_weight_max": 33.58717727661133,
      "activations/layer21_attention_weight_min": -19.465736389160156,
      "activations/layer22_attention_weight_max": 31.538545608520508,
      "activations/layer22_attention_weight_min": -25.781539916992188,
      "activations/layer23_attention_weight_max": 33.29975128173828,
      "activations/layer23_attention_weight_min": -25.43830108642578,
      "activations/layer2_attention_weight_max": 29.869766235351562,
      "activations/layer2_attention_weight_min": -27.60988426208496,
      "activations/layer3_attention_weight_max": 82.70604705810547,
      "activations/layer3_attention_weight_min": -80.86070251464844,
      "activations/layer4_attention_weight_max": 78.1414566040039,
      "activations/layer4_attention_weight_min": -77.58384704589844,
      "activations/layer5_attention_weight_max": 58.771209716796875,
      "activations/layer5_attention_weight_min": -72.72415161132812,
      "activations/layer6_attention_weight_max": 48.666542053222656,
      "activations/layer6_attention_weight_min": -48.83452606201172,
      "activations/layer7_attention_weight_max": 62.685813903808594,
      "activations/layer7_attention_weight_min": -62.40963363647461,
      "activations/layer8_attention_weight_max": 43.324222564697266,
      "activations/layer8_attention_weight_min": -46.56868362426758,
      "activations/layer9_attention_weight_max": 39.789215087890625,
      "activations/layer9_attention_weight_min": -43.506591796875,
      "epoch": 18.43,
      "learning_rate": 3.146060606060606e-05,
      "loss": 2.749,
      "step": 317200
    },
    {
      "activations/layer0_attention_weight_max": 16.15816879272461,
      "activations/layer0_attention_weight_min": -13.572992324829102,
      "activations/layer10_attention_weight_max": 38.96478271484375,
      "activations/layer10_attention_weight_min": -39.724220275878906,
      "activations/layer11_attention_weight_max": 35.01563262939453,
      "activations/layer11_attention_weight_min": -36.55785369873047,
      "activations/layer12_attention_weight_max": 21.48478889465332,
      "activations/layer12_attention_weight_min": -26.82821273803711,
      "activations/layer13_attention_weight_max": 34.895076751708984,
      "activations/layer13_attention_weight_min": -28.33962631225586,
      "activations/layer14_attention_weight_max": 38.07609939575195,
      "activations/layer14_attention_weight_min": -30.986499786376953,
      "activations/layer15_attention_weight_max": 34.796573638916016,
      "activations/layer15_attention_weight_min": -30.045860290527344,
      "activations/layer16_attention_weight_max": 36.25072479248047,
      "activations/layer16_attention_weight_min": -32.19622802734375,
      "activations/layer17_attention_weight_max": 55.99650955200195,
      "activations/layer17_attention_weight_min": -44.65260314941406,
      "activations/layer18_attention_weight_max": 51.62350082397461,
      "activations/layer18_attention_weight_min": -40.42970657348633,
      "activations/layer19_attention_weight_max": 27.624713897705078,
      "activations/layer19_attention_weight_min": -25.07811737060547,
      "activations/layer1_attention_weight_max": 16.875377655029297,
      "activations/layer1_attention_weight_min": -12.90638256072998,
      "activations/layer20_attention_weight_max": 27.9541015625,
      "activations/layer20_attention_weight_min": -21.390520095825195,
      "activations/layer21_attention_weight_max": 39.012245178222656,
      "activations/layer21_attention_weight_min": -24.17496681213379,
      "activations/layer22_attention_weight_max": 30.137006759643555,
      "activations/layer22_attention_weight_min": -25.442800521850586,
      "activations/layer23_attention_weight_max": 37.8905029296875,
      "activations/layer23_attention_weight_min": -23.36904525756836,
      "activations/layer2_attention_weight_max": 30.991628646850586,
      "activations/layer2_attention_weight_min": -29.029754638671875,
      "activations/layer3_attention_weight_max": 83.90289306640625,
      "activations/layer3_attention_weight_min": -85.20712280273438,
      "activations/layer4_attention_weight_max": 81.39849090576172,
      "activations/layer4_attention_weight_min": -83.94939422607422,
      "activations/layer5_attention_weight_max": 58.754371643066406,
      "activations/layer5_attention_weight_min": -69.83849334716797,
      "activations/layer6_attention_weight_max": 47.41065979003906,
      "activations/layer6_attention_weight_min": -50.663455963134766,
      "activations/layer7_attention_weight_max": 60.4920768737793,
      "activations/layer7_attention_weight_min": -63.076499938964844,
      "activations/layer8_attention_weight_max": 48.211212158203125,
      "activations/layer8_attention_weight_min": -51.49274444580078,
      "activations/layer9_attention_weight_max": 45.45151901245117,
      "activations/layer9_attention_weight_min": -47.49347686767578,
      "epoch": 18.43,
      "learning_rate": 3.144166666666666e-05,
      "loss": 2.7411,
      "step": 317250
    },
    {
      "activations/layer0_attention_weight_max": 15.749127388000488,
      "activations/layer0_attention_weight_min": -13.817852020263672,
      "activations/layer10_attention_weight_max": 32.73198699951172,
      "activations/layer10_attention_weight_min": -34.04755401611328,
      "activations/layer11_attention_weight_max": 30.04113006591797,
      "activations/layer11_attention_weight_min": -33.94493103027344,
      "activations/layer12_attention_weight_max": 18.660259246826172,
      "activations/layer12_attention_weight_min": -24.87131118774414,
      "activations/layer13_attention_weight_max": 33.57064437866211,
      "activations/layer13_attention_weight_min": -29.445377349853516,
      "activations/layer14_attention_weight_max": 32.929805755615234,
      "activations/layer14_attention_weight_min": -32.23298645019531,
      "activations/layer15_attention_weight_max": 30.705341339111328,
      "activations/layer15_attention_weight_min": -31.473155975341797,
      "activations/layer16_attention_weight_max": 32.83177185058594,
      "activations/layer16_attention_weight_min": -31.025789260864258,
      "activations/layer17_attention_weight_max": 50.02269744873047,
      "activations/layer17_attention_weight_min": -44.794918060302734,
      "activations/layer18_attention_weight_max": 46.23406219482422,
      "activations/layer18_attention_weight_min": -39.08304977416992,
      "activations/layer19_attention_weight_max": 21.716039657592773,
      "activations/layer19_attention_weight_min": -22.18674659729004,
      "activations/layer1_attention_weight_max": 15.203080177307129,
      "activations/layer1_attention_weight_min": -14.734748840332031,
      "activations/layer20_attention_weight_max": 20.36236000061035,
      "activations/layer20_attention_weight_min": -24.038646697998047,
      "activations/layer21_attention_weight_max": 32.50332260131836,
      "activations/layer21_attention_weight_min": -22.928115844726562,
      "activations/layer22_attention_weight_max": 28.20081329345703,
      "activations/layer22_attention_weight_min": -26.223169326782227,
      "activations/layer23_attention_weight_max": 33.50632095336914,
      "activations/layer23_attention_weight_min": -24.347740173339844,
      "activations/layer2_attention_weight_max": 29.767637252807617,
      "activations/layer2_attention_weight_min": -26.55165672302246,
      "activations/layer3_attention_weight_max": 84.0089111328125,
      "activations/layer3_attention_weight_min": -81.65670013427734,
      "activations/layer4_attention_weight_max": 78.603271484375,
      "activations/layer4_attention_weight_min": -78.76470184326172,
      "activations/layer5_attention_weight_max": 65.22098541259766,
      "activations/layer5_attention_weight_min": -68.42523956298828,
      "activations/layer6_attention_weight_max": 49.575096130371094,
      "activations/layer6_attention_weight_min": -49.588523864746094,
      "activations/layer7_attention_weight_max": 62.905433654785156,
      "activations/layer7_attention_weight_min": -62.22475051879883,
      "activations/layer8_attention_weight_max": 47.25861740112305,
      "activations/layer8_attention_weight_min": -46.30165100097656,
      "activations/layer9_attention_weight_max": 42.31301498413086,
      "activations/layer9_attention_weight_min": -42.6922492980957,
      "epoch": 18.44,
      "learning_rate": 3.142272727272727e-05,
      "loss": 2.7473,
      "step": 317300
    },
    {
      "activations/layer0_attention_weight_max": 15.57986831665039,
      "activations/layer0_attention_weight_min": -13.649917602539062,
      "activations/layer10_attention_weight_max": 40.221458435058594,
      "activations/layer10_attention_weight_min": -36.884857177734375,
      "activations/layer11_attention_weight_max": 39.00892639160156,
      "activations/layer11_attention_weight_min": -35.970977783203125,
      "activations/layer12_attention_weight_max": 20.11490821838379,
      "activations/layer12_attention_weight_min": -25.95522689819336,
      "activations/layer13_attention_weight_max": 37.86065673828125,
      "activations/layer13_attention_weight_min": -31.257335662841797,
      "activations/layer14_attention_weight_max": 44.19923782348633,
      "activations/layer14_attention_weight_min": -31.672470092773438,
      "activations/layer15_attention_weight_max": 45.09981155395508,
      "activations/layer15_attention_weight_min": -31.39889907836914,
      "activations/layer16_attention_weight_max": 34.9853515625,
      "activations/layer16_attention_weight_min": -33.0627555847168,
      "activations/layer17_attention_weight_max": 50.74946975708008,
      "activations/layer17_attention_weight_min": -45.656227111816406,
      "activations/layer18_attention_weight_max": 45.947261810302734,
      "activations/layer18_attention_weight_min": -42.285518646240234,
      "activations/layer19_attention_weight_max": 23.062734603881836,
      "activations/layer19_attention_weight_min": -23.277816772460938,
      "activations/layer1_attention_weight_max": 15.922411918640137,
      "activations/layer1_attention_weight_min": -13.499730110168457,
      "activations/layer20_attention_weight_max": 22.851640701293945,
      "activations/layer20_attention_weight_min": -23.132444381713867,
      "activations/layer21_attention_weight_max": 42.68914794921875,
      "activations/layer21_attention_weight_min": -24.862804412841797,
      "activations/layer22_attention_weight_max": 30.191987991333008,
      "activations/layer22_attention_weight_min": -27.27350425720215,
      "activations/layer23_attention_weight_max": 37.721797943115234,
      "activations/layer23_attention_weight_min": -24.54962921142578,
      "activations/layer2_attention_weight_max": 31.251766204833984,
      "activations/layer2_attention_weight_min": -29.19172477722168,
      "activations/layer3_attention_weight_max": 80.40069580078125,
      "activations/layer3_attention_weight_min": -86.03382873535156,
      "activations/layer4_attention_weight_max": 78.6939697265625,
      "activations/layer4_attention_weight_min": -79.46575927734375,
      "activations/layer5_attention_weight_max": 61.87445831298828,
      "activations/layer5_attention_weight_min": -72.84957885742188,
      "activations/layer6_attention_weight_max": 47.98000717163086,
      "activations/layer6_attention_weight_min": -49.58268737792969,
      "activations/layer7_attention_weight_max": 61.753074645996094,
      "activations/layer7_attention_weight_min": -62.98130416870117,
      "activations/layer8_attention_weight_max": 48.06682586669922,
      "activations/layer8_attention_weight_min": -49.50188064575195,
      "activations/layer9_attention_weight_max": 50.69786834716797,
      "activations/layer9_attention_weight_min": -47.07675552368164,
      "epoch": 18.44,
      "learning_rate": 3.140378787878788e-05,
      "loss": 2.7229,
      "step": 317350
    },
    {
      "activations/layer0_attention_weight_max": 15.207498550415039,
      "activations/layer0_attention_weight_min": -13.65120792388916,
      "activations/layer10_attention_weight_max": 48.76239776611328,
      "activations/layer10_attention_weight_min": -45.58395004272461,
      "activations/layer11_attention_weight_max": 45.543312072753906,
      "activations/layer11_attention_weight_min": -41.76484680175781,
      "activations/layer12_attention_weight_max": 19.165618896484375,
      "activations/layer12_attention_weight_min": -35.61703109741211,
      "activations/layer13_attention_weight_max": 48.45941925048828,
      "activations/layer13_attention_weight_min": -32.521644592285156,
      "activations/layer14_attention_weight_max": 39.58684539794922,
      "activations/layer14_attention_weight_min": -32.18534851074219,
      "activations/layer15_attention_weight_max": 38.8023796081543,
      "activations/layer15_attention_weight_min": -31.985864639282227,
      "activations/layer16_attention_weight_max": 36.23476791381836,
      "activations/layer16_attention_weight_min": -31.70281219482422,
      "activations/layer17_attention_weight_max": 55.66829299926758,
      "activations/layer17_attention_weight_min": -47.54853057861328,
      "activations/layer18_attention_weight_max": 52.92267608642578,
      "activations/layer18_attention_weight_min": -42.46754455566406,
      "activations/layer19_attention_weight_max": 25.333463668823242,
      "activations/layer19_attention_weight_min": -24.49501609802246,
      "activations/layer1_attention_weight_max": 16.073183059692383,
      "activations/layer1_attention_weight_min": -13.530611991882324,
      "activations/layer20_attention_weight_max": 27.326297760009766,
      "activations/layer20_attention_weight_min": -23.135761260986328,
      "activations/layer21_attention_weight_max": 44.34458541870117,
      "activations/layer21_attention_weight_min": -22.549419403076172,
      "activations/layer22_attention_weight_max": 31.122251510620117,
      "activations/layer22_attention_weight_min": -26.397891998291016,
      "activations/layer23_attention_weight_max": 38.21166229248047,
      "activations/layer23_attention_weight_min": -25.36610221862793,
      "activations/layer2_attention_weight_max": 32.323482513427734,
      "activations/layer2_attention_weight_min": -31.101421356201172,
      "activations/layer3_attention_weight_max": 91.77916717529297,
      "activations/layer3_attention_weight_min": -89.07583618164062,
      "activations/layer4_attention_weight_max": 89.63880157470703,
      "activations/layer4_attention_weight_min": -89.88858795166016,
      "activations/layer5_attention_weight_max": 65.71196746826172,
      "activations/layer5_attention_weight_min": -72.87914276123047,
      "activations/layer6_attention_weight_max": 54.89509582519531,
      "activations/layer6_attention_weight_min": -53.99130630493164,
      "activations/layer7_attention_weight_max": 79.50678253173828,
      "activations/layer7_attention_weight_min": -70.50919342041016,
      "activations/layer8_attention_weight_max": 62.42589569091797,
      "activations/layer8_attention_weight_min": -57.56878662109375,
      "activations/layer9_attention_weight_max": 65.57543182373047,
      "activations/layer9_attention_weight_min": -51.6361083984375,
      "epoch": 18.44,
      "learning_rate": 3.1384848484848485e-05,
      "loss": 2.7295,
      "step": 317400
    },
    {
      "activations/layer0_attention_weight_max": 15.67532730102539,
      "activations/layer0_attention_weight_min": -13.69293212890625,
      "activations/layer10_attention_weight_max": 35.51605987548828,
      "activations/layer10_attention_weight_min": -38.236846923828125,
      "activations/layer11_attention_weight_max": 32.44386291503906,
      "activations/layer11_attention_weight_min": -35.75372314453125,
      "activations/layer12_attention_weight_max": 19.104990005493164,
      "activations/layer12_attention_weight_min": -25.864778518676758,
      "activations/layer13_attention_weight_max": 38.0949592590332,
      "activations/layer13_attention_weight_min": -26.42508888244629,
      "activations/layer14_attention_weight_max": 34.26190948486328,
      "activations/layer14_attention_weight_min": -29.095605850219727,
      "activations/layer15_attention_weight_max": 33.197357177734375,
      "activations/layer15_attention_weight_min": -29.677230834960938,
      "activations/layer16_attention_weight_max": 34.41965103149414,
      "activations/layer16_attention_weight_min": -32.31591033935547,
      "activations/layer17_attention_weight_max": 54.02736282348633,
      "activations/layer17_attention_weight_min": -47.427555084228516,
      "activations/layer18_attention_weight_max": 47.946170806884766,
      "activations/layer18_attention_weight_min": -43.68046188354492,
      "activations/layer19_attention_weight_max": 23.307392120361328,
      "activations/layer19_attention_weight_min": -24.758209228515625,
      "activations/layer1_attention_weight_max": 15.236611366271973,
      "activations/layer1_attention_weight_min": -13.837047576904297,
      "activations/layer20_attention_weight_max": 22.494829177856445,
      "activations/layer20_attention_weight_min": -23.107288360595703,
      "activations/layer21_attention_weight_max": 36.461952209472656,
      "activations/layer21_attention_weight_min": -22.813207626342773,
      "activations/layer22_attention_weight_max": 30.65850830078125,
      "activations/layer22_attention_weight_min": -26.135652542114258,
      "activations/layer23_attention_weight_max": 38.13385009765625,
      "activations/layer23_attention_weight_min": -26.77895736694336,
      "activations/layer2_attention_weight_max": 29.739261627197266,
      "activations/layer2_attention_weight_min": -29.862346649169922,
      "activations/layer3_attention_weight_max": 82.55924224853516,
      "activations/layer3_attention_weight_min": -81.75785064697266,
      "activations/layer4_attention_weight_max": 80.45632934570312,
      "activations/layer4_attention_weight_min": -81.1615219116211,
      "activations/layer5_attention_weight_max": 61.00849151611328,
      "activations/layer5_attention_weight_min": -73.10401916503906,
      "activations/layer6_attention_weight_max": 48.92213439941406,
      "activations/layer6_attention_weight_min": -49.33992385864258,
      "activations/layer7_attention_weight_max": 62.020294189453125,
      "activations/layer7_attention_weight_min": -62.795143127441406,
      "activations/layer8_attention_weight_max": 47.95538330078125,
      "activations/layer8_attention_weight_min": -49.80022048950195,
      "activations/layer9_attention_weight_max": 41.9375114440918,
      "activations/layer9_attention_weight_min": -47.08980941772461,
      "epoch": 18.45,
      "learning_rate": 3.136590909090909e-05,
      "loss": 2.7323,
      "step": 317450
    },
    {
      "activations/layer0_attention_weight_max": 15.003602981567383,
      "activations/layer0_attention_weight_min": -13.412667274475098,
      "activations/layer10_attention_weight_max": 36.442413330078125,
      "activations/layer10_attention_weight_min": -35.61118698120117,
      "activations/layer11_attention_weight_max": 31.94576072692871,
      "activations/layer11_attention_weight_min": -33.225181579589844,
      "activations/layer12_attention_weight_max": 21.395477294921875,
      "activations/layer12_attention_weight_min": -26.409029006958008,
      "activations/layer13_attention_weight_max": 35.43373489379883,
      "activations/layer13_attention_weight_min": -27.292903900146484,
      "activations/layer14_attention_weight_max": 39.429649353027344,
      "activations/layer14_attention_weight_min": -34.76275634765625,
      "activations/layer15_attention_weight_max": 37.38582229614258,
      "activations/layer15_attention_weight_min": -33.86097717285156,
      "activations/layer16_attention_weight_max": 39.7985725402832,
      "activations/layer16_attention_weight_min": -36.0041389465332,
      "activations/layer17_attention_weight_max": 59.64091873168945,
      "activations/layer17_attention_weight_min": -50.36865234375,
      "activations/layer18_attention_weight_max": 52.158145904541016,
      "activations/layer18_attention_weight_min": -43.674747467041016,
      "activations/layer19_attention_weight_max": 27.34836769104004,
      "activations/layer19_attention_weight_min": -25.939746856689453,
      "activations/layer1_attention_weight_max": 15.8737211227417,
      "activations/layer1_attention_weight_min": -13.784066200256348,
      "activations/layer20_attention_weight_max": 27.93447494506836,
      "activations/layer20_attention_weight_min": -23.055713653564453,
      "activations/layer21_attention_weight_max": 40.81365966796875,
      "activations/layer21_attention_weight_min": -21.675567626953125,
      "activations/layer22_attention_weight_max": 28.90936851501465,
      "activations/layer22_attention_weight_min": -29.098407745361328,
      "activations/layer23_attention_weight_max": 40.64826965332031,
      "activations/layer23_attention_weight_min": -23.71149253845215,
      "activations/layer2_attention_weight_max": 31.509593963623047,
      "activations/layer2_attention_weight_min": -29.602920532226562,
      "activations/layer3_attention_weight_max": 85.90373992919922,
      "activations/layer3_attention_weight_min": -86.66746520996094,
      "activations/layer4_attention_weight_max": 82.36339569091797,
      "activations/layer4_attention_weight_min": -81.88742065429688,
      "activations/layer5_attention_weight_max": 60.92751693725586,
      "activations/layer5_attention_weight_min": -69.71488952636719,
      "activations/layer6_attention_weight_max": 50.39350891113281,
      "activations/layer6_attention_weight_min": -51.218196868896484,
      "activations/layer7_attention_weight_max": 60.84380340576172,
      "activations/layer7_attention_weight_min": -60.16128921508789,
      "activations/layer8_attention_weight_max": 47.519752502441406,
      "activations/layer8_attention_weight_min": -53.07682418823242,
      "activations/layer9_attention_weight_max": 42.986961364746094,
      "activations/layer9_attention_weight_min": -43.72025680541992,
      "epoch": 18.45,
      "learning_rate": 3.1347348484848485e-05,
      "loss": 2.7281,
      "step": 317500
    },
    {
      "activations/layer0_attention_weight_max": 15.841682434082031,
      "activations/layer0_attention_weight_min": -13.66820240020752,
      "activations/layer10_attention_weight_max": 34.753631591796875,
      "activations/layer10_attention_weight_min": -34.41237258911133,
      "activations/layer11_attention_weight_max": 32.300724029541016,
      "activations/layer11_attention_weight_min": -33.78559112548828,
      "activations/layer12_attention_weight_max": 17.27877426147461,
      "activations/layer12_attention_weight_min": -26.95702362060547,
      "activations/layer13_attention_weight_max": 31.705766677856445,
      "activations/layer13_attention_weight_min": -29.1429386138916,
      "activations/layer14_attention_weight_max": 34.38462829589844,
      "activations/layer14_attention_weight_min": -31.233232498168945,
      "activations/layer15_attention_weight_max": 35.807613372802734,
      "activations/layer15_attention_weight_min": -30.73904800415039,
      "activations/layer16_attention_weight_max": 35.648399353027344,
      "activations/layer16_attention_weight_min": -35.136104583740234,
      "activations/layer17_attention_weight_max": 53.112884521484375,
      "activations/layer17_attention_weight_min": -44.94575881958008,
      "activations/layer18_attention_weight_max": 47.65483856201172,
      "activations/layer18_attention_weight_min": -37.95735549926758,
      "activations/layer19_attention_weight_max": 23.208301544189453,
      "activations/layer19_attention_weight_min": -21.421308517456055,
      "activations/layer1_attention_weight_max": 16.76455307006836,
      "activations/layer1_attention_weight_min": -14.231258392333984,
      "activations/layer20_attention_weight_max": 23.25384521484375,
      "activations/layer20_attention_weight_min": -21.97972297668457,
      "activations/layer21_attention_weight_max": 39.18077087402344,
      "activations/layer21_attention_weight_min": -26.576080322265625,
      "activations/layer22_attention_weight_max": 30.506534576416016,
      "activations/layer22_attention_weight_min": -24.845863342285156,
      "activations/layer23_attention_weight_max": 34.77875900268555,
      "activations/layer23_attention_weight_min": -24.606647491455078,
      "activations/layer2_attention_weight_max": 30.72796630859375,
      "activations/layer2_attention_weight_min": -30.788639068603516,
      "activations/layer3_attention_weight_max": 85.78546142578125,
      "activations/layer3_attention_weight_min": -85.94535827636719,
      "activations/layer4_attention_weight_max": 80.104736328125,
      "activations/layer4_attention_weight_min": -81.90799713134766,
      "activations/layer5_attention_weight_max": 60.03908920288086,
      "activations/layer5_attention_weight_min": -75.3757095336914,
      "activations/layer6_attention_weight_max": 50.76069641113281,
      "activations/layer6_attention_weight_min": -51.87096405029297,
      "activations/layer7_attention_weight_max": 63.739498138427734,
      "activations/layer7_attention_weight_min": -63.98160171508789,
      "activations/layer8_attention_weight_max": 47.112060546875,
      "activations/layer8_attention_weight_min": -48.82637405395508,
      "activations/layer9_attention_weight_max": 44.367374420166016,
      "activations/layer9_attention_weight_min": -44.51841354370117,
      "epoch": 18.45,
      "learning_rate": 3.1328409090909086e-05,
      "loss": 2.728,
      "step": 317550
    },
    {
      "activations/layer0_attention_weight_max": 16.105899810791016,
      "activations/layer0_attention_weight_min": -13.620942115783691,
      "activations/layer10_attention_weight_max": 32.70402526855469,
      "activations/layer10_attention_weight_min": -33.4859619140625,
      "activations/layer11_attention_weight_max": 31.12421417236328,
      "activations/layer11_attention_weight_min": -32.17487335205078,
      "activations/layer12_attention_weight_max": 18.126535415649414,
      "activations/layer12_attention_weight_min": -28.349674224853516,
      "activations/layer13_attention_weight_max": 34.226280212402344,
      "activations/layer13_attention_weight_min": -29.854907989501953,
      "activations/layer14_attention_weight_max": 33.53203582763672,
      "activations/layer14_attention_weight_min": -29.689239501953125,
      "activations/layer15_attention_weight_max": 32.81501007080078,
      "activations/layer15_attention_weight_min": -31.7522029876709,
      "activations/layer16_attention_weight_max": 31.30253791809082,
      "activations/layer16_attention_weight_min": -29.74789047241211,
      "activations/layer17_attention_weight_max": 51.152957916259766,
      "activations/layer17_attention_weight_min": -43.714996337890625,
      "activations/layer18_attention_weight_max": 47.23895263671875,
      "activations/layer18_attention_weight_min": -38.438926696777344,
      "activations/layer19_attention_weight_max": 26.82317543029785,
      "activations/layer19_attention_weight_min": -22.178281784057617,
      "activations/layer1_attention_weight_max": 15.758116722106934,
      "activations/layer1_attention_weight_min": -12.870948791503906,
      "activations/layer20_attention_weight_max": 24.095722198486328,
      "activations/layer20_attention_weight_min": -23.663297653198242,
      "activations/layer21_attention_weight_max": 35.599586486816406,
      "activations/layer21_attention_weight_min": -22.61237144470215,
      "activations/layer22_attention_weight_max": 29.718292236328125,
      "activations/layer22_attention_weight_min": -25.517250061035156,
      "activations/layer23_attention_weight_max": 40.05315399169922,
      "activations/layer23_attention_weight_min": -23.657634735107422,
      "activations/layer2_attention_weight_max": 29.028121948242188,
      "activations/layer2_attention_weight_min": -26.361003875732422,
      "activations/layer3_attention_weight_max": 74.6715316772461,
      "activations/layer3_attention_weight_min": -78.66405487060547,
      "activations/layer4_attention_weight_max": 74.14083099365234,
      "activations/layer4_attention_weight_min": -74.78292083740234,
      "activations/layer5_attention_weight_max": 58.36006164550781,
      "activations/layer5_attention_weight_min": -70.21235656738281,
      "activations/layer6_attention_weight_max": 44.71738815307617,
      "activations/layer6_attention_weight_min": -46.54896545410156,
      "activations/layer7_attention_weight_max": 57.591129302978516,
      "activations/layer7_attention_weight_min": -59.3709716796875,
      "activations/layer8_attention_weight_max": 44.04092025756836,
      "activations/layer8_attention_weight_min": -47.7560920715332,
      "activations/layer9_attention_weight_max": 42.48090744018555,
      "activations/layer9_attention_weight_min": -42.365997314453125,
      "epoch": 18.45,
      "learning_rate": 3.1309469696969694e-05,
      "loss": 2.7523,
      "step": 317600
    },
    {
      "activations/layer0_attention_weight_max": 16.60648536682129,
      "activations/layer0_attention_weight_min": -15.316634178161621,
      "activations/layer10_attention_weight_max": 34.26079177856445,
      "activations/layer10_attention_weight_min": -36.064849853515625,
      "activations/layer11_attention_weight_max": 32.046051025390625,
      "activations/layer11_attention_weight_min": -33.3788948059082,
      "activations/layer12_attention_weight_max": 18.815542221069336,
      "activations/layer12_attention_weight_min": -25.55722427368164,
      "activations/layer13_attention_weight_max": 35.361820220947266,
      "activations/layer13_attention_weight_min": -28.01712989807129,
      "activations/layer14_attention_weight_max": 40.5359001159668,
      "activations/layer14_attention_weight_min": -32.27211380004883,
      "activations/layer15_attention_weight_max": 34.59150695800781,
      "activations/layer15_attention_weight_min": -31.60407066345215,
      "activations/layer16_attention_weight_max": 41.0897102355957,
      "activations/layer16_attention_weight_min": -37.02847671508789,
      "activations/layer17_attention_weight_max": 57.57754135131836,
      "activations/layer17_attention_weight_min": -49.42625045776367,
      "activations/layer18_attention_weight_max": 53.25764846801758,
      "activations/layer18_attention_weight_min": -44.0622673034668,
      "activations/layer19_attention_weight_max": 24.68927001953125,
      "activations/layer19_attention_weight_min": -26.09710121154785,
      "activations/layer1_attention_weight_max": 16.714412689208984,
      "activations/layer1_attention_weight_min": -13.351058959960938,
      "activations/layer20_attention_weight_max": 25.593116760253906,
      "activations/layer20_attention_weight_min": -20.95283317565918,
      "activations/layer21_attention_weight_max": 48.220333099365234,
      "activations/layer21_attention_weight_min": -25.560012817382812,
      "activations/layer22_attention_weight_max": 32.31037139892578,
      "activations/layer22_attention_weight_min": -24.836254119873047,
      "activations/layer23_attention_weight_max": 35.84445571899414,
      "activations/layer23_attention_weight_min": -23.257856369018555,
      "activations/layer2_attention_weight_max": 29.828697204589844,
      "activations/layer2_attention_weight_min": -25.383880615234375,
      "activations/layer3_attention_weight_max": 75.88656616210938,
      "activations/layer3_attention_weight_min": -72.13269805908203,
      "activations/layer4_attention_weight_max": 74.34464263916016,
      "activations/layer4_attention_weight_min": -71.01860046386719,
      "activations/layer5_attention_weight_max": 59.10118103027344,
      "activations/layer5_attention_weight_min": -63.82868957519531,
      "activations/layer6_attention_weight_max": 49.18721389770508,
      "activations/layer6_attention_weight_min": -48.22146987915039,
      "activations/layer7_attention_weight_max": 64.24823760986328,
      "activations/layer7_attention_weight_min": -61.46799087524414,
      "activations/layer8_attention_weight_max": 43.623565673828125,
      "activations/layer8_attention_weight_min": -49.65713882446289,
      "activations/layer9_attention_weight_max": 42.87381362915039,
      "activations/layer9_attention_weight_min": -47.2320442199707,
      "epoch": 18.46,
      "learning_rate": 3.1290909090909085e-05,
      "loss": 2.7447,
      "step": 317650
    },
    {
      "activations/layer0_attention_weight_max": 16.144519805908203,
      "activations/layer0_attention_weight_min": -13.791460037231445,
      "activations/layer10_attention_weight_max": 36.457332611083984,
      "activations/layer10_attention_weight_min": -36.141536712646484,
      "activations/layer11_attention_weight_max": 31.98126983642578,
      "activations/layer11_attention_weight_min": -33.233219146728516,
      "activations/layer12_attention_weight_max": 18.816247940063477,
      "activations/layer12_attention_weight_min": -23.199153900146484,
      "activations/layer13_attention_weight_max": 36.300540924072266,
      "activations/layer13_attention_weight_min": -27.924537658691406,
      "activations/layer14_attention_weight_max": 37.07938766479492,
      "activations/layer14_attention_weight_min": -29.983745574951172,
      "activations/layer15_attention_weight_max": 35.63301467895508,
      "activations/layer15_attention_weight_min": -30.549087524414062,
      "activations/layer16_attention_weight_max": 34.813560485839844,
      "activations/layer16_attention_weight_min": -32.146148681640625,
      "activations/layer17_attention_weight_max": 53.901222229003906,
      "activations/layer17_attention_weight_min": -44.8832893371582,
      "activations/layer18_attention_weight_max": 47.86732482910156,
      "activations/layer18_attention_weight_min": -39.01534652709961,
      "activations/layer19_attention_weight_max": 24.34876823425293,
      "activations/layer19_attention_weight_min": -23.325468063354492,
      "activations/layer1_attention_weight_max": 16.401737213134766,
      "activations/layer1_attention_weight_min": -13.423547744750977,
      "activations/layer20_attention_weight_max": 23.221105575561523,
      "activations/layer20_attention_weight_min": -20.779808044433594,
      "activations/layer21_attention_weight_max": 36.8343505859375,
      "activations/layer21_attention_weight_min": -23.236480712890625,
      "activations/layer22_attention_weight_max": 31.89362144470215,
      "activations/layer22_attention_weight_min": -26.824968338012695,
      "activations/layer23_attention_weight_max": 39.01128387451172,
      "activations/layer23_attention_weight_min": -23.542354583740234,
      "activations/layer2_attention_weight_max": 30.649066925048828,
      "activations/layer2_attention_weight_min": -30.905250549316406,
      "activations/layer3_attention_weight_max": 80.779541015625,
      "activations/layer3_attention_weight_min": -84.65428161621094,
      "activations/layer4_attention_weight_max": 79.1609115600586,
      "activations/layer4_attention_weight_min": -81.71150207519531,
      "activations/layer5_attention_weight_max": 59.66981506347656,
      "activations/layer5_attention_weight_min": -73.49029541015625,
      "activations/layer6_attention_weight_max": 48.947757720947266,
      "activations/layer6_attention_weight_min": -49.78463363647461,
      "activations/layer7_attention_weight_max": 63.23072814941406,
      "activations/layer7_attention_weight_min": -62.18141174316406,
      "activations/layer8_attention_weight_max": 43.21751022338867,
      "activations/layer8_attention_weight_min": -46.196739196777344,
      "activations/layer9_attention_weight_max": 43.732486724853516,
      "activations/layer9_attention_weight_min": -43.80491256713867,
      "epoch": 18.46,
      "learning_rate": 3.1271969696969693e-05,
      "loss": 2.7411,
      "step": 317700
    },
    {
      "activations/layer0_attention_weight_max": 14.851271629333496,
      "activations/layer0_attention_weight_min": -13.432167053222656,
      "activations/layer10_attention_weight_max": 35.98546600341797,
      "activations/layer10_attention_weight_min": -35.94575881958008,
      "activations/layer11_attention_weight_max": 32.966583251953125,
      "activations/layer11_attention_weight_min": -33.088157653808594,
      "activations/layer12_attention_weight_max": 20.27970314025879,
      "activations/layer12_attention_weight_min": -27.42698097229004,
      "activations/layer13_attention_weight_max": 39.489356994628906,
      "activations/layer13_attention_weight_min": -30.53153419494629,
      "activations/layer14_attention_weight_max": 37.69013214111328,
      "activations/layer14_attention_weight_min": -32.03379821777344,
      "activations/layer15_attention_weight_max": 36.98217010498047,
      "activations/layer15_attention_weight_min": -32.50437545776367,
      "activations/layer16_attention_weight_max": 35.66783142089844,
      "activations/layer16_attention_weight_min": -34.51859664916992,
      "activations/layer17_attention_weight_max": 51.23514938354492,
      "activations/layer17_attention_weight_min": -45.12803268432617,
      "activations/layer18_attention_weight_max": 47.31828308105469,
      "activations/layer18_attention_weight_min": -39.3027229309082,
      "activations/layer19_attention_weight_max": 23.60899543762207,
      "activations/layer19_attention_weight_min": -23.026826858520508,
      "activations/layer1_attention_weight_max": 15.664597511291504,
      "activations/layer1_attention_weight_min": -13.28549575805664,
      "activations/layer20_attention_weight_max": 22.598051071166992,
      "activations/layer20_attention_weight_min": -22.426746368408203,
      "activations/layer21_attention_weight_max": 37.91493606567383,
      "activations/layer21_attention_weight_min": -23.72101402282715,
      "activations/layer22_attention_weight_max": 29.651134490966797,
      "activations/layer22_attention_weight_min": -25.485376358032227,
      "activations/layer23_attention_weight_max": 35.2538948059082,
      "activations/layer23_attention_weight_min": -25.864835739135742,
      "activations/layer2_attention_weight_max": 31.297481536865234,
      "activations/layer2_attention_weight_min": -28.75104522705078,
      "activations/layer3_attention_weight_max": 83.10916900634766,
      "activations/layer3_attention_weight_min": -81.12152099609375,
      "activations/layer4_attention_weight_max": 78.0575180053711,
      "activations/layer4_attention_weight_min": -81.72655487060547,
      "activations/layer5_attention_weight_max": 60.381893157958984,
      "activations/layer5_attention_weight_min": -69.18696594238281,
      "activations/layer6_attention_weight_max": 50.52028274536133,
      "activations/layer6_attention_weight_min": -48.10206604003906,
      "activations/layer7_attention_weight_max": 62.46529006958008,
      "activations/layer7_attention_weight_min": -61.345733642578125,
      "activations/layer8_attention_weight_max": 46.06856155395508,
      "activations/layer8_attention_weight_min": -49.72092819213867,
      "activations/layer9_attention_weight_max": 45.697994232177734,
      "activations/layer9_attention_weight_min": -45.43422317504883,
      "epoch": 18.46,
      "learning_rate": 3.12530303030303e-05,
      "loss": 2.7276,
      "step": 317750
    },
    {
      "activations/layer0_attention_weight_max": 16.22987174987793,
      "activations/layer0_attention_weight_min": -14.05652904510498,
      "activations/layer10_attention_weight_max": 34.61937713623047,
      "activations/layer10_attention_weight_min": -35.40521240234375,
      "activations/layer11_attention_weight_max": 32.38063430786133,
      "activations/layer11_attention_weight_min": -33.401092529296875,
      "activations/layer12_attention_weight_max": 19.29133415222168,
      "activations/layer12_attention_weight_min": -26.456409454345703,
      "activations/layer13_attention_weight_max": 32.759159088134766,
      "activations/layer13_attention_weight_min": -28.28972053527832,
      "activations/layer14_attention_weight_max": 34.98862838745117,
      "activations/layer14_attention_weight_min": -29.58380126953125,
      "activations/layer15_attention_weight_max": 32.237422943115234,
      "activations/layer15_attention_weight_min": -29.39078712463379,
      "activations/layer16_attention_weight_max": 32.139869689941406,
      "activations/layer16_attention_weight_min": -31.2457275390625,
      "activations/layer17_attention_weight_max": 51.75939178466797,
      "activations/layer17_attention_weight_min": -45.02762222290039,
      "activations/layer18_attention_weight_max": 44.267784118652344,
      "activations/layer18_attention_weight_min": -40.26171875,
      "activations/layer19_attention_weight_max": 22.07404899597168,
      "activations/layer19_attention_weight_min": -22.59310531616211,
      "activations/layer1_attention_weight_max": 16.339479446411133,
      "activations/layer1_attention_weight_min": -13.749873161315918,
      "activations/layer20_attention_weight_max": 19.739521026611328,
      "activations/layer20_attention_weight_min": -20.911815643310547,
      "activations/layer21_attention_weight_max": 33.486663818359375,
      "activations/layer21_attention_weight_min": -22.6284122467041,
      "activations/layer22_attention_weight_max": 30.113311767578125,
      "activations/layer22_attention_weight_min": -25.65329933166504,
      "activations/layer23_attention_weight_max": 36.99528503417969,
      "activations/layer23_attention_weight_min": -23.770179748535156,
      "activations/layer2_attention_weight_max": 29.574844360351562,
      "activations/layer2_attention_weight_min": -30.78203582763672,
      "activations/layer3_attention_weight_max": 80.30841064453125,
      "activations/layer3_attention_weight_min": -78.33177947998047,
      "activations/layer4_attention_weight_max": 77.16169738769531,
      "activations/layer4_attention_weight_min": -77.28144073486328,
      "activations/layer5_attention_weight_max": 60.41606903076172,
      "activations/layer5_attention_weight_min": -67.64997863769531,
      "activations/layer6_attention_weight_max": 47.45408248901367,
      "activations/layer6_attention_weight_min": -46.7260627746582,
      "activations/layer7_attention_weight_max": 60.75858688354492,
      "activations/layer7_attention_weight_min": -59.822601318359375,
      "activations/layer8_attention_weight_max": 43.30534362792969,
      "activations/layer8_attention_weight_min": -49.6090087890625,
      "activations/layer9_attention_weight_max": 44.15754318237305,
      "activations/layer9_attention_weight_min": -46.40653610229492,
      "epoch": 18.47,
      "learning_rate": 3.123409090909091e-05,
      "loss": 2.728,
      "step": 317800
    },
    {
      "activations/layer0_attention_weight_max": 15.904142379760742,
      "activations/layer0_attention_weight_min": -13.19179916381836,
      "activations/layer10_attention_weight_max": 40.99687957763672,
      "activations/layer10_attention_weight_min": -36.4488410949707,
      "activations/layer11_attention_weight_max": 34.80937957763672,
      "activations/layer11_attention_weight_min": -35.94319152832031,
      "activations/layer12_attention_weight_max": 22.08421516418457,
      "activations/layer12_attention_weight_min": -21.256250381469727,
      "activations/layer13_attention_weight_max": 37.0715446472168,
      "activations/layer13_attention_weight_min": -30.483224868774414,
      "activations/layer14_attention_weight_max": 36.57255172729492,
      "activations/layer14_attention_weight_min": -29.78767204284668,
      "activations/layer15_attention_weight_max": 32.667598724365234,
      "activations/layer15_attention_weight_min": -29.379058837890625,
      "activations/layer16_attention_weight_max": 33.513668060302734,
      "activations/layer16_attention_weight_min": -30.177120208740234,
      "activations/layer17_attention_weight_max": 51.51759719848633,
      "activations/layer17_attention_weight_min": -44.75188446044922,
      "activations/layer18_attention_weight_max": 48.6725959777832,
      "activations/layer18_attention_weight_min": -37.491302490234375,
      "activations/layer19_attention_weight_max": 25.19573211669922,
      "activations/layer19_attention_weight_min": -21.362964630126953,
      "activations/layer1_attention_weight_max": 16.052257537841797,
      "activations/layer1_attention_weight_min": -14.121294021606445,
      "activations/layer20_attention_weight_max": 22.48600196838379,
      "activations/layer20_attention_weight_min": -21.784652709960938,
      "activations/layer21_attention_weight_max": 35.2362174987793,
      "activations/layer21_attention_weight_min": -20.880273818969727,
      "activations/layer22_attention_weight_max": 29.434900283813477,
      "activations/layer22_attention_weight_min": -25.165111541748047,
      "activations/layer23_attention_weight_max": 36.68681335449219,
      "activations/layer23_attention_weight_min": -23.238065719604492,
      "activations/layer2_attention_weight_max": 32.96016311645508,
      "activations/layer2_attention_weight_min": -33.36377716064453,
      "activations/layer3_attention_weight_max": 91.0963134765625,
      "activations/layer3_attention_weight_min": -86.8150863647461,
      "activations/layer4_attention_weight_max": 88.49510955810547,
      "activations/layer4_attention_weight_min": -86.08738708496094,
      "activations/layer5_attention_weight_max": 64.5257797241211,
      "activations/layer5_attention_weight_min": -67.99980163574219,
      "activations/layer6_attention_weight_max": 52.2165412902832,
      "activations/layer6_attention_weight_min": -53.849796295166016,
      "activations/layer7_attention_weight_max": 66.19935607910156,
      "activations/layer7_attention_weight_min": -62.59412384033203,
      "activations/layer8_attention_weight_max": 47.55586242675781,
      "activations/layer8_attention_weight_min": -50.4171257019043,
      "activations/layer9_attention_weight_max": 45.28108215332031,
      "activations/layer9_attention_weight_min": -44.4840202331543,
      "epoch": 18.47,
      "learning_rate": 3.12155303030303e-05,
      "loss": 2.7349,
      "step": 317850
    },
    {
      "activations/layer0_attention_weight_max": 15.969266891479492,
      "activations/layer0_attention_weight_min": -13.98740005493164,
      "activations/layer10_attention_weight_max": 37.315879821777344,
      "activations/layer10_attention_weight_min": -37.239097595214844,
      "activations/layer11_attention_weight_max": 34.81355285644531,
      "activations/layer11_attention_weight_min": -37.44351577758789,
      "activations/layer12_attention_weight_max": 18.860610961914062,
      "activations/layer12_attention_weight_min": -24.930763244628906,
      "activations/layer13_attention_weight_max": 34.04114532470703,
      "activations/layer13_attention_weight_min": -30.125112533569336,
      "activations/layer14_attention_weight_max": 36.042335510253906,
      "activations/layer14_attention_weight_min": -29.48281478881836,
      "activations/layer15_attention_weight_max": 32.4022331237793,
      "activations/layer15_attention_weight_min": -30.48373794555664,
      "activations/layer16_attention_weight_max": 34.74385452270508,
      "activations/layer16_attention_weight_min": -31.202165603637695,
      "activations/layer17_attention_weight_max": 51.08913803100586,
      "activations/layer17_attention_weight_min": -43.680728912353516,
      "activations/layer18_attention_weight_max": 49.26105880737305,
      "activations/layer18_attention_weight_min": -39.254791259765625,
      "activations/layer19_attention_weight_max": 23.381750106811523,
      "activations/layer19_attention_weight_min": -21.417524337768555,
      "activations/layer1_attention_weight_max": 15.130410194396973,
      "activations/layer1_attention_weight_min": -13.905380249023438,
      "activations/layer20_attention_weight_max": 22.70296859741211,
      "activations/layer20_attention_weight_min": -21.57809066772461,
      "activations/layer21_attention_weight_max": 36.627933502197266,
      "activations/layer21_attention_weight_min": -22.697668075561523,
      "activations/layer22_attention_weight_max": 28.69359588623047,
      "activations/layer22_attention_weight_min": -25.4841365814209,
      "activations/layer23_attention_weight_max": 36.804683685302734,
      "activations/layer23_attention_weight_min": -25.577001571655273,
      "activations/layer2_attention_weight_max": 30.387149810791016,
      "activations/layer2_attention_weight_min": -30.49342155456543,
      "activations/layer3_attention_weight_max": 87.01258087158203,
      "activations/layer3_attention_weight_min": -85.37420654296875,
      "activations/layer4_attention_weight_max": 85.87602233886719,
      "activations/layer4_attention_weight_min": -85.23519134521484,
      "activations/layer5_attention_weight_max": 63.73945617675781,
      "activations/layer5_attention_weight_min": -73.52386474609375,
      "activations/layer6_attention_weight_max": 51.16716384887695,
      "activations/layer6_attention_weight_min": -52.68227005004883,
      "activations/layer7_attention_weight_max": 66.97875213623047,
      "activations/layer7_attention_weight_min": -64.31888580322266,
      "activations/layer8_attention_weight_max": 48.038028717041016,
      "activations/layer8_attention_weight_min": -54.42254638671875,
      "activations/layer9_attention_weight_max": 47.05427932739258,
      "activations/layer9_attention_weight_min": -49.01448440551758,
      "epoch": 18.47,
      "learning_rate": 3.119659090909091e-05,
      "loss": 2.7409,
      "step": 317900
    },
    {
      "activations/layer0_attention_weight_max": 16.350265502929688,
      "activations/layer0_attention_weight_min": -13.339542388916016,
      "activations/layer10_attention_weight_max": 40.33406066894531,
      "activations/layer10_attention_weight_min": -38.92653274536133,
      "activations/layer11_attention_weight_max": 38.59307861328125,
      "activations/layer11_attention_weight_min": -35.899017333984375,
      "activations/layer12_attention_weight_max": 19.658695220947266,
      "activations/layer12_attention_weight_min": -24.998130798339844,
      "activations/layer13_attention_weight_max": 32.854671478271484,
      "activations/layer13_attention_weight_min": -28.18811798095703,
      "activations/layer14_attention_weight_max": 36.01041030883789,
      "activations/layer14_attention_weight_min": -29.45090675354004,
      "activations/layer15_attention_weight_max": 33.10600280761719,
      "activations/layer15_attention_weight_min": -31.76361083984375,
      "activations/layer16_attention_weight_max": 34.97734832763672,
      "activations/layer16_attention_weight_min": -31.038782119750977,
      "activations/layer17_attention_weight_max": 52.993553161621094,
      "activations/layer17_attention_weight_min": -45.777252197265625,
      "activations/layer18_attention_weight_max": 49.83723831176758,
      "activations/layer18_attention_weight_min": -39.91404342651367,
      "activations/layer19_attention_weight_max": 24.628740310668945,
      "activations/layer19_attention_weight_min": -22.514371871948242,
      "activations/layer1_attention_weight_max": 16.16782569885254,
      "activations/layer1_attention_weight_min": -13.809691429138184,
      "activations/layer20_attention_weight_max": 22.045469284057617,
      "activations/layer20_attention_weight_min": -22.53643035888672,
      "activations/layer21_attention_weight_max": 33.60036087036133,
      "activations/layer21_attention_weight_min": -21.70890998840332,
      "activations/layer22_attention_weight_max": 31.152294158935547,
      "activations/layer22_attention_weight_min": -26.979496002197266,
      "activations/layer23_attention_weight_max": 37.846923828125,
      "activations/layer23_attention_weight_min": -25.472034454345703,
      "activations/layer2_attention_weight_max": 30.09844970703125,
      "activations/layer2_attention_weight_min": -31.210960388183594,
      "activations/layer3_attention_weight_max": 83.26643371582031,
      "activations/layer3_attention_weight_min": -89.53721618652344,
      "activations/layer4_attention_weight_max": 80.34101104736328,
      "activations/layer4_attention_weight_min": -86.43621063232422,
      "activations/layer5_attention_weight_max": 61.82415008544922,
      "activations/layer5_attention_weight_min": -77.17596435546875,
      "activations/layer6_attention_weight_max": 51.94047546386719,
      "activations/layer6_attention_weight_min": -52.90200424194336,
      "activations/layer7_attention_weight_max": 66.83387756347656,
      "activations/layer7_attention_weight_min": -67.64567565917969,
      "activations/layer8_attention_weight_max": 49.039878845214844,
      "activations/layer8_attention_weight_min": -51.30658721923828,
      "activations/layer9_attention_weight_max": 47.44045639038086,
      "activations/layer9_attention_weight_min": -52.84381103515625,
      "epoch": 18.47,
      "learning_rate": 3.117765151515151e-05,
      "loss": 2.7308,
      "step": 317950
    },
    {
      "activations/layer0_attention_weight_max": 15.393875122070312,
      "activations/layer0_attention_weight_min": -13.244047164916992,
      "activations/layer10_attention_weight_max": 36.677223205566406,
      "activations/layer10_attention_weight_min": -34.39057159423828,
      "activations/layer11_attention_weight_max": 35.701454162597656,
      "activations/layer11_attention_weight_min": -33.789878845214844,
      "activations/layer12_attention_weight_max": 19.038488388061523,
      "activations/layer12_attention_weight_min": -26.546058654785156,
      "activations/layer13_attention_weight_max": 44.29509735107422,
      "activations/layer13_attention_weight_min": -27.36897850036621,
      "activations/layer14_attention_weight_max": 43.32796096801758,
      "activations/layer14_attention_weight_min": -29.908227920532227,
      "activations/layer15_attention_weight_max": 45.65114974975586,
      "activations/layer15_attention_weight_min": -30.112260818481445,
      "activations/layer16_attention_weight_max": 37.310455322265625,
      "activations/layer16_attention_weight_min": -32.623626708984375,
      "activations/layer17_attention_weight_max": 54.549049377441406,
      "activations/layer17_attention_weight_min": -46.81755065917969,
      "activations/layer18_attention_weight_max": 50.851593017578125,
      "activations/layer18_attention_weight_min": -43.70537185668945,
      "activations/layer19_attention_weight_max": 27.701862335205078,
      "activations/layer19_attention_weight_min": -25.329477310180664,
      "activations/layer1_attention_weight_max": 14.774711608886719,
      "activations/layer1_attention_weight_min": -11.059901237487793,
      "activations/layer20_attention_weight_max": 28.134281158447266,
      "activations/layer20_attention_weight_min": -22.356367111206055,
      "activations/layer21_attention_weight_max": 43.04736328125,
      "activations/layer21_attention_weight_min": -23.883853912353516,
      "activations/layer22_attention_weight_max": 31.654769897460938,
      "activations/layer22_attention_weight_min": -22.606592178344727,
      "activations/layer23_attention_weight_max": 35.579559326171875,
      "activations/layer23_attention_weight_min": -21.728199005126953,
      "activations/layer2_attention_weight_max": 30.464927673339844,
      "activations/layer2_attention_weight_min": -28.774272918701172,
      "activations/layer3_attention_weight_max": 87.14742279052734,
      "activations/layer3_attention_weight_min": -81.03466796875,
      "activations/layer4_attention_weight_max": 80.83195495605469,
      "activations/layer4_attention_weight_min": -77.03153991699219,
      "activations/layer5_attention_weight_max": 63.81428527832031,
      "activations/layer5_attention_weight_min": -76.88809204101562,
      "activations/layer6_attention_weight_max": 47.89981460571289,
      "activations/layer6_attention_weight_min": -47.92401885986328,
      "activations/layer7_attention_weight_max": 63.75714111328125,
      "activations/layer7_attention_weight_min": -61.25516891479492,
      "activations/layer8_attention_weight_max": 45.927955627441406,
      "activations/layer8_attention_weight_min": -45.90105056762695,
      "activations/layer9_attention_weight_max": 45.64316940307617,
      "activations/layer9_attention_weight_min": -46.488040924072266,
      "epoch": 18.48,
      "learning_rate": 3.115871212121212e-05,
      "loss": 2.7251,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_loss": 2.69140625,
      "eval_runtime": 8.5094,
      "eval_samples_per_second": 504.616,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_openwebtext_loss": 2.69140625,
      "eval_openwebtext_ppl": 14.75240691276831,
      "eval_openwebtext_runtime": 8.5094,
      "eval_openwebtext_samples_per_second": 504.616,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.287,
      "step": 318000
    },
    {
      "epoch": 18.48,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5794,
      "eval_lambada_samples_per_second": 508.28,
      "step": 318000
    },
    {
      "activations/layer0_attention_weight_max": 16.000499725341797,
      "activations/layer0_attention_weight_min": -12.999094009399414,
      "activations/layer10_attention_weight_max": 36.254478454589844,
      "activations/layer10_attention_weight_min": -35.45610046386719,
      "activations/layer11_attention_weight_max": 34.529945373535156,
      "activations/layer11_attention_weight_min": -34.403953552246094,
      "activations/layer12_attention_weight_max": 19.76409339904785,
      "activations/layer12_attention_weight_min": -24.100513458251953,
      "activations/layer13_attention_weight_max": 36.16200637817383,
      "activations/layer13_attention_weight_min": -28.19529914855957,
      "activations/layer14_attention_weight_max": 36.82560729980469,
      "activations/layer14_attention_weight_min": -30.9442195892334,
      "activations/layer15_attention_weight_max": 31.840097427368164,
      "activations/layer15_attention_weight_min": -31.690462112426758,
      "activations/layer16_attention_weight_max": 32.433349609375,
      "activations/layer16_attention_weight_min": -31.168672561645508,
      "activations/layer17_attention_weight_max": 50.53616714477539,
      "activations/layer17_attention_weight_min": -44.91138458251953,
      "activations/layer18_attention_weight_max": 45.21902847290039,
      "activations/layer18_attention_weight_min": -39.05064010620117,
      "activations/layer19_attention_weight_max": 24.707901000976562,
      "activations/layer19_attention_weight_min": -22.531171798706055,
      "activations/layer1_attention_weight_max": 16.01715660095215,
      "activations/layer1_attention_weight_min": -12.923468589782715,
      "activations/layer20_attention_weight_max": 24.37102699279785,
      "activations/layer20_attention_weight_min": -22.25218963623047,
      "activations/layer21_attention_weight_max": 38.21181106567383,
      "activations/layer21_attention_weight_min": -23.634389877319336,
      "activations/layer22_attention_weight_max": 29.579214096069336,
      "activations/layer22_attention_weight_min": -26.041006088256836,
      "activations/layer23_attention_weight_max": 35.98210144042969,
      "activations/layer23_attention_weight_min": -23.538650512695312,
      "activations/layer2_attention_weight_max": 29.27387237548828,
      "activations/layer2_attention_weight_min": -28.875829696655273,
      "activations/layer3_attention_weight_max": 82.27468872070312,
      "activations/layer3_attention_weight_min": -85.57918548583984,
      "activations/layer4_attention_weight_max": 85.68328857421875,
      "activations/layer4_attention_weight_min": -82.81971740722656,
      "activations/layer5_attention_weight_max": 61.12034225463867,
      "activations/layer5_attention_weight_min": -76.90335083007812,
      "activations/layer6_attention_weight_max": 50.79695129394531,
      "activations/layer6_attention_weight_min": -51.43439865112305,
      "activations/layer7_attention_weight_max": 66.04998779296875,
      "activations/layer7_attention_weight_min": -63.712615966796875,
      "activations/layer8_attention_weight_max": 50.03935623168945,
      "activations/layer8_attention_weight_min": -49.6657600402832,
      "activations/layer9_attention_weight_max": 46.296844482421875,
      "activations/layer9_attention_weight_min": -44.56013107299805,
      "epoch": 18.48,
      "learning_rate": 3.113977272727272e-05,
      "loss": 2.7143,
      "step": 318050
    },
    {
      "activations/layer0_attention_weight_max": 15.778682708740234,
      "activations/layer0_attention_weight_min": -13.325566291809082,
      "activations/layer10_attention_weight_max": 36.55805587768555,
      "activations/layer10_attention_weight_min": -37.182159423828125,
      "activations/layer11_attention_weight_max": 32.73615264892578,
      "activations/layer11_attention_weight_min": -34.55443572998047,
      "activations/layer12_attention_weight_max": 18.319244384765625,
      "activations/layer12_attention_weight_min": -25.527109146118164,
      "activations/layer13_attention_weight_max": 30.663734436035156,
      "activations/layer13_attention_weight_min": -31.26388931274414,
      "activations/layer14_attention_weight_max": 33.19763946533203,
      "activations/layer14_attention_weight_min": -30.044837951660156,
      "activations/layer15_attention_weight_max": 31.204044342041016,
      "activations/layer15_attention_weight_min": -30.825916290283203,
      "activations/layer16_attention_weight_max": 32.0809211730957,
      "activations/layer16_attention_weight_min": -32.43047332763672,
      "activations/layer17_attention_weight_max": 49.07996368408203,
      "activations/layer17_attention_weight_min": -44.111289978027344,
      "activations/layer18_attention_weight_max": 43.60516357421875,
      "activations/layer18_attention_weight_min": -42.311180114746094,
      "activations/layer19_attention_weight_max": 22.971683502197266,
      "activations/layer19_attention_weight_min": -23.08746910095215,
      "activations/layer1_attention_weight_max": 15.747125625610352,
      "activations/layer1_attention_weight_min": -15.191511154174805,
      "activations/layer20_attention_weight_max": 24.131986618041992,
      "activations/layer20_attention_weight_min": -24.151573181152344,
      "activations/layer21_attention_weight_max": 33.08927536010742,
      "activations/layer21_attention_weight_min": -23.365013122558594,
      "activations/layer22_attention_weight_max": 26.454858779907227,
      "activations/layer22_attention_weight_min": -27.97243881225586,
      "activations/layer23_attention_weight_max": 30.46630859375,
      "activations/layer23_attention_weight_min": -24.06473159790039,
      "activations/layer2_attention_weight_max": 32.620506286621094,
      "activations/layer2_attention_weight_min": -30.559192657470703,
      "activations/layer3_attention_weight_max": 90.2917251586914,
      "activations/layer3_attention_weight_min": -89.96330261230469,
      "activations/layer4_attention_weight_max": 86.52227020263672,
      "activations/layer4_attention_weight_min": -87.07677459716797,
      "activations/layer5_attention_weight_max": 65.50909423828125,
      "activations/layer5_attention_weight_min": -76.26353454589844,
      "activations/layer6_attention_weight_max": 52.17401885986328,
      "activations/layer6_attention_weight_min": -52.97551727294922,
      "activations/layer7_attention_weight_max": 67.37100219726562,
      "activations/layer7_attention_weight_min": -68.3468246459961,
      "activations/layer8_attention_weight_max": 46.61684036254883,
      "activations/layer8_attention_weight_min": -51.1120491027832,
      "activations/layer9_attention_weight_max": 43.032928466796875,
      "activations/layer9_attention_weight_min": -46.39434051513672,
      "epoch": 18.48,
      "learning_rate": 3.112083333333333e-05,
      "loss": 2.7291,
      "step": 318100
    },
    {
      "activations/layer0_attention_weight_max": 15.933090209960938,
      "activations/layer0_attention_weight_min": -13.358766555786133,
      "activations/layer10_attention_weight_max": 35.242855072021484,
      "activations/layer10_attention_weight_min": -33.342262268066406,
      "activations/layer11_attention_weight_max": 32.008827209472656,
      "activations/layer11_attention_weight_min": -33.466773986816406,
      "activations/layer12_attention_weight_max": 18.621807098388672,
      "activations/layer12_attention_weight_min": -24.79047203063965,
      "activations/layer13_attention_weight_max": 38.51105880737305,
      "activations/layer13_attention_weight_min": -26.708879470825195,
      "activations/layer14_attention_weight_max": 32.64081954956055,
      "activations/layer14_attention_weight_min": -28.44989013671875,
      "activations/layer15_attention_weight_max": 30.693735122680664,
      "activations/layer15_attention_weight_min": -28.4965877532959,
      "activations/layer16_attention_weight_max": 32.28131103515625,
      "activations/layer16_attention_weight_min": -31.836889266967773,
      "activations/layer17_attention_weight_max": 49.516334533691406,
      "activations/layer17_attention_weight_min": -44.629581451416016,
      "activations/layer18_attention_weight_max": 44.29868698120117,
      "activations/layer18_attention_weight_min": -40.16913986206055,
      "activations/layer19_attention_weight_max": 23.737825393676758,
      "activations/layer19_attention_weight_min": -24.269506454467773,
      "activations/layer1_attention_weight_max": 15.561331748962402,
      "activations/layer1_attention_weight_min": -12.792384147644043,
      "activations/layer20_attention_weight_max": 23.905176162719727,
      "activations/layer20_attention_weight_min": -22.832815170288086,
      "activations/layer21_attention_weight_max": 34.15693283081055,
      "activations/layer21_attention_weight_min": -23.2077693939209,
      "activations/layer22_attention_weight_max": 30.85230827331543,
      "activations/layer22_attention_weight_min": -26.201135635375977,
      "activations/layer23_attention_weight_max": 34.45741271972656,
      "activations/layer23_attention_weight_min": -24.610830307006836,
      "activations/layer2_attention_weight_max": 28.23782730102539,
      "activations/layer2_attention_weight_min": -26.863704681396484,
      "activations/layer3_attention_weight_max": 78.75251007080078,
      "activations/layer3_attention_weight_min": -79.22769165039062,
      "activations/layer4_attention_weight_max": 79.92802429199219,
      "activations/layer4_attention_weight_min": -77.86971282958984,
      "activations/layer5_attention_weight_max": 65.95125579833984,
      "activations/layer5_attention_weight_min": -68.63700866699219,
      "activations/layer6_attention_weight_max": 49.55150604248047,
      "activations/layer6_attention_weight_min": -48.013118743896484,
      "activations/layer7_attention_weight_max": 63.90070343017578,
      "activations/layer7_attention_weight_min": -62.76759719848633,
      "activations/layer8_attention_weight_max": 45.04045104980469,
      "activations/layer8_attention_weight_min": -45.35688781738281,
      "activations/layer9_attention_weight_max": 45.31684494018555,
      "activations/layer9_attention_weight_min": -44.32076644897461,
      "epoch": 18.49,
      "learning_rate": 3.110189393939394e-05,
      "loss": 2.7293,
      "step": 318150
    },
    {
      "activations/layer0_attention_weight_max": 15.46915054321289,
      "activations/layer0_attention_weight_min": -13.265432357788086,
      "activations/layer10_attention_weight_max": 36.639373779296875,
      "activations/layer10_attention_weight_min": -36.6065788269043,
      "activations/layer11_attention_weight_max": 35.40187454223633,
      "activations/layer11_attention_weight_min": -37.53977966308594,
      "activations/layer12_attention_weight_max": 17.947805404663086,
      "activations/layer12_attention_weight_min": -23.766193389892578,
      "activations/layer13_attention_weight_max": 33.7099723815918,
      "activations/layer13_attention_weight_min": -28.46713638305664,
      "activations/layer14_attention_weight_max": 35.8607292175293,
      "activations/layer14_attention_weight_min": -29.251008987426758,
      "activations/layer15_attention_weight_max": 31.562782287597656,
      "activations/layer15_attention_weight_min": -29.743494033813477,
      "activations/layer16_attention_weight_max": 32.46561050415039,
      "activations/layer16_attention_weight_min": -31.47276496887207,
      "activations/layer17_attention_weight_max": 50.05476760864258,
      "activations/layer17_attention_weight_min": -44.66035079956055,
      "activations/layer18_attention_weight_max": 46.86112976074219,
      "activations/layer18_attention_weight_min": -39.86980056762695,
      "activations/layer19_attention_weight_max": 25.62540626525879,
      "activations/layer19_attention_weight_min": -23.63534927368164,
      "activations/layer1_attention_weight_max": 16.612707138061523,
      "activations/layer1_attention_weight_min": -14.984451293945312,
      "activations/layer20_attention_weight_max": 23.346038818359375,
      "activations/layer20_attention_weight_min": -24.83852195739746,
      "activations/layer21_attention_weight_max": 35.9721794128418,
      "activations/layer21_attention_weight_min": -22.502689361572266,
      "activations/layer22_attention_weight_max": 31.079200744628906,
      "activations/layer22_attention_weight_min": -25.2138729095459,
      "activations/layer23_attention_weight_max": 40.37577819824219,
      "activations/layer23_attention_weight_min": -25.449295043945312,
      "activations/layer2_attention_weight_max": 32.31620407104492,
      "activations/layer2_attention_weight_min": -30.011096954345703,
      "activations/layer3_attention_weight_max": 88.34459686279297,
      "activations/layer3_attention_weight_min": -85.63311767578125,
      "activations/layer4_attention_weight_max": 83.50004577636719,
      "activations/layer4_attention_weight_min": -83.29863739013672,
      "activations/layer5_attention_weight_max": 61.92118453979492,
      "activations/layer5_attention_weight_min": -74.03164672851562,
      "activations/layer6_attention_weight_max": 50.34556579589844,
      "activations/layer6_attention_weight_min": -51.47563552856445,
      "activations/layer7_attention_weight_max": 64.45506286621094,
      "activations/layer7_attention_weight_min": -63.437801361083984,
      "activations/layer8_attention_weight_max": 48.65676498413086,
      "activations/layer8_attention_weight_min": -52.170135498046875,
      "activations/layer9_attention_weight_max": 44.52849578857422,
      "activations/layer9_attention_weight_min": -48.232757568359375,
      "epoch": 18.49,
      "learning_rate": 3.1082954545454545e-05,
      "loss": 2.7346,
      "step": 318200
    },
    {
      "activations/layer0_attention_weight_max": 15.856633186340332,
      "activations/layer0_attention_weight_min": -13.873071670532227,
      "activations/layer10_attention_weight_max": 36.20479965209961,
      "activations/layer10_attention_weight_min": -36.43058395385742,
      "activations/layer11_attention_weight_max": 33.90040588378906,
      "activations/layer11_attention_weight_min": -36.33448028564453,
      "activations/layer12_attention_weight_max": 17.585309982299805,
      "activations/layer12_attention_weight_min": -26.633777618408203,
      "activations/layer13_attention_weight_max": 32.58218002319336,
      "activations/layer13_attention_weight_min": -32.931907653808594,
      "activations/layer14_attention_weight_max": 34.98139190673828,
      "activations/layer14_attention_weight_min": -30.07500648498535,
      "activations/layer15_attention_weight_max": 31.853893280029297,
      "activations/layer15_attention_weight_min": -31.106924057006836,
      "activations/layer16_attention_weight_max": 32.768184661865234,
      "activations/layer16_attention_weight_min": -33.86054611206055,
      "activations/layer17_attention_weight_max": 51.15623092651367,
      "activations/layer17_attention_weight_min": -50.00660705566406,
      "activations/layer18_attention_weight_max": 44.35702133178711,
      "activations/layer18_attention_weight_min": -41.49528884887695,
      "activations/layer19_attention_weight_max": 25.489511489868164,
      "activations/layer19_attention_weight_min": -22.77643394470215,
      "activations/layer1_attention_weight_max": 16.388046264648438,
      "activations/layer1_attention_weight_min": -14.630480766296387,
      "activations/layer20_attention_weight_max": 22.887495040893555,
      "activations/layer20_attention_weight_min": -22.648290634155273,
      "activations/layer21_attention_weight_max": 33.21271896362305,
      "activations/layer21_attention_weight_min": -21.262964248657227,
      "activations/layer22_attention_weight_max": 27.528467178344727,
      "activations/layer22_attention_weight_min": -26.483335494995117,
      "activations/layer23_attention_weight_max": 36.062889099121094,
      "activations/layer23_attention_weight_min": -22.949359893798828,
      "activations/layer2_attention_weight_max": 32.12525177001953,
      "activations/layer2_attention_weight_min": -30.838321685791016,
      "activations/layer3_attention_weight_max": 85.31965637207031,
      "activations/layer3_attention_weight_min": -85.05670166015625,
      "activations/layer4_attention_weight_max": 86.2115249633789,
      "activations/layer4_attention_weight_min": -89.05166625976562,
      "activations/layer5_attention_weight_max": 63.07408905029297,
      "activations/layer5_attention_weight_min": -79.01842498779297,
      "activations/layer6_attention_weight_max": 49.955543518066406,
      "activations/layer6_attention_weight_min": -51.13848114013672,
      "activations/layer7_attention_weight_max": 62.4792594909668,
      "activations/layer7_attention_weight_min": -62.88644790649414,
      "activations/layer8_attention_weight_max": 47.49385452270508,
      "activations/layer8_attention_weight_min": -50.34016418457031,
      "activations/layer9_attention_weight_max": 47.71287155151367,
      "activations/layer9_attention_weight_min": -47.99166488647461,
      "epoch": 18.49,
      "learning_rate": 3.1064015151515153e-05,
      "loss": 2.7359,
      "step": 318250
    },
    {
      "activations/layer0_attention_weight_max": 15.46200942993164,
      "activations/layer0_attention_weight_min": -13.346552848815918,
      "activations/layer10_attention_weight_max": 36.57253646850586,
      "activations/layer10_attention_weight_min": -35.628421783447266,
      "activations/layer11_attention_weight_max": 30.846912384033203,
      "activations/layer11_attention_weight_min": -32.6103515625,
      "activations/layer12_attention_weight_max": 19.032169342041016,
      "activations/layer12_attention_weight_min": -24.06136703491211,
      "activations/layer13_attention_weight_max": 33.52543258666992,
      "activations/layer13_attention_weight_min": -30.85561752319336,
      "activations/layer14_attention_weight_max": 36.71784973144531,
      "activations/layer14_attention_weight_min": -28.7713565826416,
      "activations/layer15_attention_weight_max": 30.835844039916992,
      "activations/layer15_attention_weight_min": -29.541290283203125,
      "activations/layer16_attention_weight_max": 31.821611404418945,
      "activations/layer16_attention_weight_min": -31.78001594543457,
      "activations/layer17_attention_weight_max": 50.99811553955078,
      "activations/layer17_attention_weight_min": -42.63270950317383,
      "activations/layer18_attention_weight_max": 45.41513442993164,
      "activations/layer18_attention_weight_min": -39.850677490234375,
      "activations/layer19_attention_weight_max": 22.84394073486328,
      "activations/layer19_attention_weight_min": -22.798324584960938,
      "activations/layer1_attention_weight_max": 16.20555305480957,
      "activations/layer1_attention_weight_min": -12.64619255065918,
      "activations/layer20_attention_weight_max": 20.80052375793457,
      "activations/layer20_attention_weight_min": -25.191699981689453,
      "activations/layer21_attention_weight_max": 31.238262176513672,
      "activations/layer21_attention_weight_min": -23.699352264404297,
      "activations/layer22_attention_weight_max": 27.71358299255371,
      "activations/layer22_attention_weight_min": -23.3375186920166,
      "activations/layer23_attention_weight_max": 36.32781982421875,
      "activations/layer23_attention_weight_min": -22.087583541870117,
      "activations/layer2_attention_weight_max": 30.76651382446289,
      "activations/layer2_attention_weight_min": -30.951313018798828,
      "activations/layer3_attention_weight_max": 86.30488586425781,
      "activations/layer3_attention_weight_min": -88.766845703125,
      "activations/layer4_attention_weight_max": 83.31504821777344,
      "activations/layer4_attention_weight_min": -81.54080200195312,
      "activations/layer5_attention_weight_max": 64.06903076171875,
      "activations/layer5_attention_weight_min": -81.9412841796875,
      "activations/layer6_attention_weight_max": 51.54817581176758,
      "activations/layer6_attention_weight_min": -50.74819564819336,
      "activations/layer7_attention_weight_max": 66.99337005615234,
      "activations/layer7_attention_weight_min": -63.60981750488281,
      "activations/layer8_attention_weight_max": 45.494873046875,
      "activations/layer8_attention_weight_min": -49.89177322387695,
      "activations/layer9_attention_weight_max": 45.225181579589844,
      "activations/layer9_attention_weight_min": -44.8559684753418,
      "epoch": 18.5,
      "learning_rate": 3.1045075757575755e-05,
      "loss": 2.7349,
      "step": 318300
    },
    {
      "activations/layer0_attention_weight_max": 15.821020126342773,
      "activations/layer0_attention_weight_min": -13.152535438537598,
      "activations/layer10_attention_weight_max": 34.19102478027344,
      "activations/layer10_attention_weight_min": -36.556365966796875,
      "activations/layer11_attention_weight_max": 32.76320266723633,
      "activations/layer11_attention_weight_min": -35.60774230957031,
      "activations/layer12_attention_weight_max": 19.181917190551758,
      "activations/layer12_attention_weight_min": -25.67966651916504,
      "activations/layer13_attention_weight_max": 34.565155029296875,
      "activations/layer13_attention_weight_min": -26.698659896850586,
      "activations/layer14_attention_weight_max": 34.487754821777344,
      "activations/layer14_attention_weight_min": -28.95524024963379,
      "activations/layer15_attention_weight_max": 33.344032287597656,
      "activations/layer15_attention_weight_min": -28.846017837524414,
      "activations/layer16_attention_weight_max": 33.95308303833008,
      "activations/layer16_attention_weight_min": -31.195064544677734,
      "activations/layer17_attention_weight_max": 48.960227966308594,
      "activations/layer17_attention_weight_min": -43.186798095703125,
      "activations/layer18_attention_weight_max": 48.16746520996094,
      "activations/layer18_attention_weight_min": -38.345977783203125,
      "activations/layer19_attention_weight_max": 26.19681167602539,
      "activations/layer19_attention_weight_min": -23.856983184814453,
      "activations/layer1_attention_weight_max": 16.05891227722168,
      "activations/layer1_attention_weight_min": -13.771293640136719,
      "activations/layer20_attention_weight_max": 25.260452270507812,
      "activations/layer20_attention_weight_min": -22.149089813232422,
      "activations/layer21_attention_weight_max": 36.91215896606445,
      "activations/layer21_attention_weight_min": -22.020002365112305,
      "activations/layer22_attention_weight_max": 31.711084365844727,
      "activations/layer22_attention_weight_min": -28.199750900268555,
      "activations/layer23_attention_weight_max": 44.79258346557617,
      "activations/layer23_attention_weight_min": -27.623172760009766,
      "activations/layer2_attention_weight_max": 30.779874801635742,
      "activations/layer2_attention_weight_min": -29.85224151611328,
      "activations/layer3_attention_weight_max": 83.99443817138672,
      "activations/layer3_attention_weight_min": -83.88504028320312,
      "activations/layer4_attention_weight_max": 79.82567596435547,
      "activations/layer4_attention_weight_min": -78.76254272460938,
      "activations/layer5_attention_weight_max": 61.07640075683594,
      "activations/layer5_attention_weight_min": -76.87403869628906,
      "activations/layer6_attention_weight_max": 48.729061126708984,
      "activations/layer6_attention_weight_min": -48.64836502075195,
      "activations/layer7_attention_weight_max": 64.05211639404297,
      "activations/layer7_attention_weight_min": -62.15776824951172,
      "activations/layer8_attention_weight_max": 47.53749465942383,
      "activations/layer8_attention_weight_min": -51.559139251708984,
      "activations/layer9_attention_weight_max": 46.704856872558594,
      "activations/layer9_attention_weight_min": -44.56794357299805,
      "epoch": 18.5,
      "learning_rate": 3.102613636363636e-05,
      "loss": 2.7351,
      "step": 318350
    },
    {
      "activations/layer0_attention_weight_max": 14.782214164733887,
      "activations/layer0_attention_weight_min": -13.215988159179688,
      "activations/layer10_attention_weight_max": 37.10691452026367,
      "activations/layer10_attention_weight_min": -36.09542465209961,
      "activations/layer11_attention_weight_max": 33.60878372192383,
      "activations/layer11_attention_weight_min": -33.5214958190918,
      "activations/layer12_attention_weight_max": 18.278793334960938,
      "activations/layer12_attention_weight_min": -24.91927719116211,
      "activations/layer13_attention_weight_max": 30.435094833374023,
      "activations/layer13_attention_weight_min": -25.9220027923584,
      "activations/layer14_attention_weight_max": 36.65937805175781,
      "activations/layer14_attention_weight_min": -28.033456802368164,
      "activations/layer15_attention_weight_max": 34.733741760253906,
      "activations/layer15_attention_weight_min": -30.481231689453125,
      "activations/layer16_attention_weight_max": 34.00837707519531,
      "activations/layer16_attention_weight_min": -31.302722930908203,
      "activations/layer17_attention_weight_max": 50.832027435302734,
      "activations/layer17_attention_weight_min": -43.83805847167969,
      "activations/layer18_attention_weight_max": 44.69862365722656,
      "activations/layer18_attention_weight_min": -40.14036178588867,
      "activations/layer19_attention_weight_max": 25.451539993286133,
      "activations/layer19_attention_weight_min": -22.917905807495117,
      "activations/layer1_attention_weight_max": 15.895458221435547,
      "activations/layer1_attention_weight_min": -14.06209945678711,
      "activations/layer20_attention_weight_max": 23.433534622192383,
      "activations/layer20_attention_weight_min": -23.215667724609375,
      "activations/layer21_attention_weight_max": 46.92302322387695,
      "activations/layer21_attention_weight_min": -27.04655647277832,
      "activations/layer22_attention_weight_max": 31.56726837158203,
      "activations/layer22_attention_weight_min": -26.84315299987793,
      "activations/layer23_attention_weight_max": 36.39091491699219,
      "activations/layer23_attention_weight_min": -24.709983825683594,
      "activations/layer2_attention_weight_max": 34.9910888671875,
      "activations/layer2_attention_weight_min": -31.20534324645996,
      "activations/layer3_attention_weight_max": 87.29129791259766,
      "activations/layer3_attention_weight_min": -87.43990325927734,
      "activations/layer4_attention_weight_max": 84.9225082397461,
      "activations/layer4_attention_weight_min": -83.22979736328125,
      "activations/layer5_attention_weight_max": 66.58741760253906,
      "activations/layer5_attention_weight_min": -71.13851165771484,
      "activations/layer6_attention_weight_max": 50.267066955566406,
      "activations/layer6_attention_weight_min": -51.36610412597656,
      "activations/layer7_attention_weight_max": 69.65424346923828,
      "activations/layer7_attention_weight_min": -68.33216857910156,
      "activations/layer8_attention_weight_max": 49.889434814453125,
      "activations/layer8_attention_weight_min": -50.067630767822266,
      "activations/layer9_attention_weight_max": 47.79275131225586,
      "activations/layer9_attention_weight_min": -44.8923454284668,
      "epoch": 18.5,
      "learning_rate": 3.1007196969696965e-05,
      "loss": 2.754,
      "step": 318400
    },
    {
      "activations/layer0_attention_weight_max": 16.419841766357422,
      "activations/layer0_attention_weight_min": -13.948657035827637,
      "activations/layer10_attention_weight_max": 36.64277267456055,
      "activations/layer10_attention_weight_min": -36.61306381225586,
      "activations/layer11_attention_weight_max": 36.117706298828125,
      "activations/layer11_attention_weight_min": -36.22791290283203,
      "activations/layer12_attention_weight_max": 20.55486488342285,
      "activations/layer12_attention_weight_min": -26.348691940307617,
      "activations/layer13_attention_weight_max": 33.309932708740234,
      "activations/layer13_attention_weight_min": -27.794601440429688,
      "activations/layer14_attention_weight_max": 36.02546691894531,
      "activations/layer14_attention_weight_min": -29.628231048583984,
      "activations/layer15_attention_weight_max": 34.192684173583984,
      "activations/layer15_attention_weight_min": -31.367551803588867,
      "activations/layer16_attention_weight_max": 36.0855712890625,
      "activations/layer16_attention_weight_min": -33.310630798339844,
      "activations/layer17_attention_weight_max": 51.80946731567383,
      "activations/layer17_attention_weight_min": -46.35629653930664,
      "activations/layer18_attention_weight_max": 47.88767623901367,
      "activations/layer18_attention_weight_min": -40.37534713745117,
      "activations/layer19_attention_weight_max": 22.651412963867188,
      "activations/layer19_attention_weight_min": -22.592124938964844,
      "activations/layer1_attention_weight_max": 15.88659381866455,
      "activations/layer1_attention_weight_min": -12.857162475585938,
      "activations/layer20_attention_weight_max": 24.27880096435547,
      "activations/layer20_attention_weight_min": -20.799766540527344,
      "activations/layer21_attention_weight_max": 37.86176300048828,
      "activations/layer21_attention_weight_min": -22.024152755737305,
      "activations/layer22_attention_weight_max": 31.642141342163086,
      "activations/layer22_attention_weight_min": -27.055343627929688,
      "activations/layer23_attention_weight_max": 39.7288703918457,
      "activations/layer23_attention_weight_min": -24.585065841674805,
      "activations/layer2_attention_weight_max": 30.61536979675293,
      "activations/layer2_attention_weight_min": -28.782882690429688,
      "activations/layer3_attention_weight_max": 86.0224609375,
      "activations/layer3_attention_weight_min": -83.96491241455078,
      "activations/layer4_attention_weight_max": 87.15086364746094,
      "activations/layer4_attention_weight_min": -83.5309066772461,
      "activations/layer5_attention_weight_max": 61.890769958496094,
      "activations/layer5_attention_weight_min": -72.96397399902344,
      "activations/layer6_attention_weight_max": 51.468570709228516,
      "activations/layer6_attention_weight_min": -51.17587661743164,
      "activations/layer7_attention_weight_max": 67.63270568847656,
      "activations/layer7_attention_weight_min": -63.96021270751953,
      "activations/layer8_attention_weight_max": 50.140235900878906,
      "activations/layer8_attention_weight_min": -51.329158782958984,
      "activations/layer9_attention_weight_max": 49.35026168823242,
      "activations/layer9_attention_weight_min": -47.15903091430664,
      "epoch": 18.5,
      "learning_rate": 3.098825757575757e-05,
      "loss": 2.7261,
      "step": 318450
    },
    {
      "activations/layer0_attention_weight_max": 15.905607223510742,
      "activations/layer0_attention_weight_min": -13.86880111694336,
      "activations/layer10_attention_weight_max": 34.612728118896484,
      "activations/layer10_attention_weight_min": -35.261383056640625,
      "activations/layer11_attention_weight_max": 30.414175033569336,
      "activations/layer11_attention_weight_min": -34.10274124145508,
      "activations/layer12_attention_weight_max": 19.40042495727539,
      "activations/layer12_attention_weight_min": -23.359455108642578,
      "activations/layer13_attention_weight_max": 34.613582611083984,
      "activations/layer13_attention_weight_min": -30.599130630493164,
      "activations/layer14_attention_weight_max": 37.919681549072266,
      "activations/layer14_attention_weight_min": -34.025840759277344,
      "activations/layer15_attention_weight_max": 38.230804443359375,
      "activations/layer15_attention_weight_min": -34.11551284790039,
      "activations/layer16_attention_weight_max": 37.968505859375,
      "activations/layer16_attention_weight_min": -34.999114990234375,
      "activations/layer17_attention_weight_max": 53.32461929321289,
      "activations/layer17_attention_weight_min": -49.326576232910156,
      "activations/layer18_attention_weight_max": 47.33591079711914,
      "activations/layer18_attention_weight_min": -45.21213150024414,
      "activations/layer19_attention_weight_max": 25.24773597717285,
      "activations/layer19_attention_weight_min": -24.976978302001953,
      "activations/layer1_attention_weight_max": 15.670705795288086,
      "activations/layer1_attention_weight_min": -13.65546703338623,
      "activations/layer20_attention_weight_max": 25.219669342041016,
      "activations/layer20_attention_weight_min": -23.484153747558594,
      "activations/layer21_attention_weight_max": 37.434539794921875,
      "activations/layer21_attention_weight_min": -24.334793090820312,
      "activations/layer22_attention_weight_max": 30.095151901245117,
      "activations/layer22_attention_weight_min": -26.366947174072266,
      "activations/layer23_attention_weight_max": 36.8355827331543,
      "activations/layer23_attention_weight_min": -26.623279571533203,
      "activations/layer2_attention_weight_max": 29.309383392333984,
      "activations/layer2_attention_weight_min": -28.614933013916016,
      "activations/layer3_attention_weight_max": 80.99850463867188,
      "activations/layer3_attention_weight_min": -82.18016815185547,
      "activations/layer4_attention_weight_max": 80.81079864501953,
      "activations/layer4_attention_weight_min": -77.9582748413086,
      "activations/layer5_attention_weight_max": 57.997711181640625,
      "activations/layer5_attention_weight_min": -70.91070556640625,
      "activations/layer6_attention_weight_max": 47.91098403930664,
      "activations/layer6_attention_weight_min": -50.79248809814453,
      "activations/layer7_attention_weight_max": 64.48577117919922,
      "activations/layer7_attention_weight_min": -61.229312896728516,
      "activations/layer8_attention_weight_max": 44.023380279541016,
      "activations/layer8_attention_weight_min": -49.23609161376953,
      "activations/layer9_attention_weight_max": 42.476295471191406,
      "activations/layer9_attention_weight_min": -45.79994201660156,
      "epoch": 18.51,
      "learning_rate": 3.096931818181818e-05,
      "loss": 2.7362,
      "step": 318500
    },
    {
      "activations/layer0_attention_weight_max": 16.34238624572754,
      "activations/layer0_attention_weight_min": -13.643263816833496,
      "activations/layer10_attention_weight_max": 39.03280258178711,
      "activations/layer10_attention_weight_min": -37.56028747558594,
      "activations/layer11_attention_weight_max": 36.064666748046875,
      "activations/layer11_attention_weight_min": -35.81537628173828,
      "activations/layer12_attention_weight_max": 18.914621353149414,
      "activations/layer12_attention_weight_min": -25.791404724121094,
      "activations/layer13_attention_weight_max": 33.99808883666992,
      "activations/layer13_attention_weight_min": -28.009010314941406,
      "activations/layer14_attention_weight_max": 36.981170654296875,
      "activations/layer14_attention_weight_min": -30.14195442199707,
      "activations/layer15_attention_weight_max": 37.87522506713867,
      "activations/layer15_attention_weight_min": -30.416767120361328,
      "activations/layer16_attention_weight_max": 35.76799392700195,
      "activations/layer16_attention_weight_min": -32.56836700439453,
      "activations/layer17_attention_weight_max": 52.20994567871094,
      "activations/layer17_attention_weight_min": -46.68732452392578,
      "activations/layer18_attention_weight_max": 48.040740966796875,
      "activations/layer18_attention_weight_min": -40.47331619262695,
      "activations/layer19_attention_weight_max": 25.5709171295166,
      "activations/layer19_attention_weight_min": -21.78059959411621,
      "activations/layer1_attention_weight_max": 15.883559226989746,
      "activations/layer1_attention_weight_min": -13.400919914245605,
      "activations/layer20_attention_weight_max": 24.685392379760742,
      "activations/layer20_attention_weight_min": -21.844877243041992,
      "activations/layer21_attention_weight_max": 35.961143493652344,
      "activations/layer21_attention_weight_min": -26.488359451293945,
      "activations/layer22_attention_weight_max": 32.487274169921875,
      "activations/layer22_attention_weight_min": -24.947513580322266,
      "activations/layer23_attention_weight_max": 37.722320556640625,
      "activations/layer23_attention_weight_min": -24.65160369873047,
      "activations/layer2_attention_weight_max": 31.179527282714844,
      "activations/layer2_attention_weight_min": -29.32594108581543,
      "activations/layer3_attention_weight_max": 87.3335952758789,
      "activations/layer3_attention_weight_min": -86.78585052490234,
      "activations/layer4_attention_weight_max": 85.45796966552734,
      "activations/layer4_attention_weight_min": -88.56079864501953,
      "activations/layer5_attention_weight_max": 63.78721237182617,
      "activations/layer5_attention_weight_min": -74.4434814453125,
      "activations/layer6_attention_weight_max": 50.044193267822266,
      "activations/layer6_attention_weight_min": -51.52355194091797,
      "activations/layer7_attention_weight_max": 65.30532836914062,
      "activations/layer7_attention_weight_min": -65.2701416015625,
      "activations/layer8_attention_weight_max": 53.87228012084961,
      "activations/layer8_attention_weight_min": -57.21459197998047,
      "activations/layer9_attention_weight_max": 49.071250915527344,
      "activations/layer9_attention_weight_min": -51.402862548828125,
      "epoch": 18.51,
      "learning_rate": 3.095037878787879e-05,
      "loss": 2.7349,
      "step": 318550
    },
    {
      "activations/layer0_attention_weight_max": 16.04160499572754,
      "activations/layer0_attention_weight_min": -13.782567024230957,
      "activations/layer10_attention_weight_max": 36.14287185668945,
      "activations/layer10_attention_weight_min": -34.66680908203125,
      "activations/layer11_attention_weight_max": 32.903167724609375,
      "activations/layer11_attention_weight_min": -33.951393127441406,
      "activations/layer12_attention_weight_max": 17.77279281616211,
      "activations/layer12_attention_weight_min": -27.60094451904297,
      "activations/layer13_attention_weight_max": 32.95852279663086,
      "activations/layer13_attention_weight_min": -27.953643798828125,
      "activations/layer14_attention_weight_max": 35.73097229003906,
      "activations/layer14_attention_weight_min": -29.189695358276367,
      "activations/layer15_attention_weight_max": 34.0126838684082,
      "activations/layer15_attention_weight_min": -28.554407119750977,
      "activations/layer16_attention_weight_max": 33.82862854003906,
      "activations/layer16_attention_weight_min": -30.8477783203125,
      "activations/layer17_attention_weight_max": 51.79764175415039,
      "activations/layer17_attention_weight_min": -44.84480667114258,
      "activations/layer18_attention_weight_max": 49.45429611206055,
      "activations/layer18_attention_weight_min": -41.55613708496094,
      "activations/layer19_attention_weight_max": 27.04984474182129,
      "activations/layer19_attention_weight_min": -22.29695701599121,
      "activations/layer1_attention_weight_max": 15.81524658203125,
      "activations/layer1_attention_weight_min": -12.991373062133789,
      "activations/layer20_attention_weight_max": 22.513273239135742,
      "activations/layer20_attention_weight_min": -18.92848777770996,
      "activations/layer21_attention_weight_max": 40.495357513427734,
      "activations/layer21_attention_weight_min": -23.89438247680664,
      "activations/layer22_attention_weight_max": 29.93193244934082,
      "activations/layer22_attention_weight_min": -25.134126663208008,
      "activations/layer23_attention_weight_max": 36.57912826538086,
      "activations/layer23_attention_weight_min": -25.962209701538086,
      "activations/layer2_attention_weight_max": 32.54279327392578,
      "activations/layer2_attention_weight_min": -29.553905487060547,
      "activations/layer3_attention_weight_max": 89.2811279296875,
      "activations/layer3_attention_weight_min": -88.71945190429688,
      "activations/layer4_attention_weight_max": 84.94922637939453,
      "activations/layer4_attention_weight_min": -82.17379760742188,
      "activations/layer5_attention_weight_max": 64.04771423339844,
      "activations/layer5_attention_weight_min": -74.1895751953125,
      "activations/layer6_attention_weight_max": 51.49660110473633,
      "activations/layer6_attention_weight_min": -51.026084899902344,
      "activations/layer7_attention_weight_max": 62.502864837646484,
      "activations/layer7_attention_weight_min": -66.1070556640625,
      "activations/layer8_attention_weight_max": 50.538116455078125,
      "activations/layer8_attention_weight_min": -51.24373245239258,
      "activations/layer9_attention_weight_max": 45.774139404296875,
      "activations/layer9_attention_weight_min": -44.71194839477539,
      "epoch": 18.51,
      "learning_rate": 3.093143939393939e-05,
      "loss": 2.7279,
      "step": 318600
    },
    {
      "activations/layer0_attention_weight_max": 15.460680961608887,
      "activations/layer0_attention_weight_min": -13.827767372131348,
      "activations/layer10_attention_weight_max": 36.358314514160156,
      "activations/layer10_attention_weight_min": -39.04264831542969,
      "activations/layer11_attention_weight_max": 34.957557678222656,
      "activations/layer11_attention_weight_min": -35.73574447631836,
      "activations/layer12_attention_weight_max": 19.18516731262207,
      "activations/layer12_attention_weight_min": -23.03162384033203,
      "activations/layer13_attention_weight_max": 38.989349365234375,
      "activations/layer13_attention_weight_min": -29.695077896118164,
      "activations/layer14_attention_weight_max": 35.91621398925781,
      "activations/layer14_attention_weight_min": -31.376829147338867,
      "activations/layer15_attention_weight_max": 34.03990173339844,
      "activations/layer15_attention_weight_min": -31.91191291809082,
      "activations/layer16_attention_weight_max": 34.952842712402344,
      "activations/layer16_attention_weight_min": -32.59793472290039,
      "activations/layer17_attention_weight_max": 50.77043533325195,
      "activations/layer17_attention_weight_min": -50.80293655395508,
      "activations/layer18_attention_weight_max": 46.73469924926758,
      "activations/layer18_attention_weight_min": -46.16185760498047,
      "activations/layer19_attention_weight_max": 25.24593162536621,
      "activations/layer19_attention_weight_min": -24.728878021240234,
      "activations/layer1_attention_weight_max": 16.294998168945312,
      "activations/layer1_attention_weight_min": -13.852639198303223,
      "activations/layer20_attention_weight_max": 22.83657455444336,
      "activations/layer20_attention_weight_min": -24.95040512084961,
      "activations/layer21_attention_weight_max": 41.87113571166992,
      "activations/layer21_attention_weight_min": -29.31767463684082,
      "activations/layer22_attention_weight_max": 30.357173919677734,
      "activations/layer22_attention_weight_min": -26.40536117553711,
      "activations/layer23_attention_weight_max": 38.025962829589844,
      "activations/layer23_attention_weight_min": -25.193937301635742,
      "activations/layer2_attention_weight_max": 34.15254592895508,
      "activations/layer2_attention_weight_min": -32.12189865112305,
      "activations/layer3_attention_weight_max": 88.99877166748047,
      "activations/layer3_attention_weight_min": -90.04515838623047,
      "activations/layer4_attention_weight_max": 84.9053955078125,
      "activations/layer4_attention_weight_min": -84.29985809326172,
      "activations/layer5_attention_weight_max": 64.21723937988281,
      "activations/layer5_attention_weight_min": -70.1851806640625,
      "activations/layer6_attention_weight_max": 48.93461608886719,
      "activations/layer6_attention_weight_min": -51.47331237792969,
      "activations/layer7_attention_weight_max": 63.36446762084961,
      "activations/layer7_attention_weight_min": -61.81851577758789,
      "activations/layer8_attention_weight_max": 45.55699157714844,
      "activations/layer8_attention_weight_min": -49.60236740112305,
      "activations/layer9_attention_weight_max": 44.609779357910156,
      "activations/layer9_attention_weight_min": -46.078399658203125,
      "epoch": 18.52,
      "learning_rate": 3.09125e-05,
      "loss": 2.7386,
      "step": 318650
    },
    {
      "activations/layer0_attention_weight_max": 15.633066177368164,
      "activations/layer0_attention_weight_min": -13.620705604553223,
      "activations/layer10_attention_weight_max": 42.89362716674805,
      "activations/layer10_attention_weight_min": -38.65860366821289,
      "activations/layer11_attention_weight_max": 40.04637908935547,
      "activations/layer11_attention_weight_min": -37.12635040283203,
      "activations/layer12_attention_weight_max": 19.4542236328125,
      "activations/layer12_attention_weight_min": -28.812618255615234,
      "activations/layer13_attention_weight_max": 31.15581703186035,
      "activations/layer13_attention_weight_min": -28.47505760192871,
      "activations/layer14_attention_weight_max": 34.94495391845703,
      "activations/layer14_attention_weight_min": -29.54265022277832,
      "activations/layer15_attention_weight_max": 32.702091217041016,
      "activations/layer15_attention_weight_min": -29.756982803344727,
      "activations/layer16_attention_weight_max": 33.330238342285156,
      "activations/layer16_attention_weight_min": -31.876020431518555,
      "activations/layer17_attention_weight_max": 50.46928787231445,
      "activations/layer17_attention_weight_min": -43.95328140258789,
      "activations/layer18_attention_weight_max": 44.90477752685547,
      "activations/layer18_attention_weight_min": -39.439632415771484,
      "activations/layer19_attention_weight_max": 21.184505462646484,
      "activations/layer19_attention_weight_min": -23.039649963378906,
      "activations/layer1_attention_weight_max": 16.295841217041016,
      "activations/layer1_attention_weight_min": -13.019139289855957,
      "activations/layer20_attention_weight_max": 19.717493057250977,
      "activations/layer20_attention_weight_min": -22.318029403686523,
      "activations/layer21_attention_weight_max": 32.855377197265625,
      "activations/layer21_attention_weight_min": -23.419187545776367,
      "activations/layer22_attention_weight_max": 26.62005615234375,
      "activations/layer22_attention_weight_min": -25.32706642150879,
      "activations/layer23_attention_weight_max": 35.21869659423828,
      "activations/layer23_attention_weight_min": -24.047414779663086,
      "activations/layer2_attention_weight_max": 30.334091186523438,
      "activations/layer2_attention_weight_min": -28.333227157592773,
      "activations/layer3_attention_weight_max": 88.12893676757812,
      "activations/layer3_attention_weight_min": -86.19529724121094,
      "activations/layer4_attention_weight_max": 84.70861053466797,
      "activations/layer4_attention_weight_min": -81.50798797607422,
      "activations/layer5_attention_weight_max": 63.796104431152344,
      "activations/layer5_attention_weight_min": -67.27753448486328,
      "activations/layer6_attention_weight_max": 52.00997543334961,
      "activations/layer6_attention_weight_min": -52.751163482666016,
      "activations/layer7_attention_weight_max": 66.30695343017578,
      "activations/layer7_attention_weight_min": -63.018001556396484,
      "activations/layer8_attention_weight_max": 51.0334587097168,
      "activations/layer8_attention_weight_min": -51.875770568847656,
      "activations/layer9_attention_weight_max": 46.92085647583008,
      "activations/layer9_attention_weight_min": -48.50525665283203,
      "epoch": 18.52,
      "learning_rate": 3.08935606060606e-05,
      "loss": 2.7443,
      "step": 318700
    },
    {
      "activations/layer0_attention_weight_max": 16.293935775756836,
      "activations/layer0_attention_weight_min": -13.804405212402344,
      "activations/layer10_attention_weight_max": 36.15523147583008,
      "activations/layer10_attention_weight_min": -36.34062194824219,
      "activations/layer11_attention_weight_max": 38.764305114746094,
      "activations/layer11_attention_weight_min": -34.49870300292969,
      "activations/layer12_attention_weight_max": 18.006671905517578,
      "activations/layer12_attention_weight_min": -22.932828903198242,
      "activations/layer13_attention_weight_max": 35.154293060302734,
      "activations/layer13_attention_weight_min": -27.977439880371094,
      "activations/layer14_attention_weight_max": 41.14021301269531,
      "activations/layer14_attention_weight_min": -30.153888702392578,
      "activations/layer15_attention_weight_max": 43.16740036010742,
      "activations/layer15_attention_weight_min": -32.03774642944336,
      "activations/layer16_attention_weight_max": 35.813575744628906,
      "activations/layer16_attention_weight_min": -31.08323097229004,
      "activations/layer17_attention_weight_max": 53.366661071777344,
      "activations/layer17_attention_weight_min": -46.10493469238281,
      "activations/layer18_attention_weight_max": 52.4124870300293,
      "activations/layer18_attention_weight_min": -39.743892669677734,
      "activations/layer19_attention_weight_max": 28.116958618164062,
      "activations/layer19_attention_weight_min": -25.698963165283203,
      "activations/layer1_attention_weight_max": 16.573383331298828,
      "activations/layer1_attention_weight_min": -15.48158073425293,
      "activations/layer20_attention_weight_max": 27.786731719970703,
      "activations/layer20_attention_weight_min": -22.859140396118164,
      "activations/layer21_attention_weight_max": 45.60823059082031,
      "activations/layer21_attention_weight_min": -25.61305809020996,
      "activations/layer22_attention_weight_max": 31.247928619384766,
      "activations/layer22_attention_weight_min": -24.57276153564453,
      "activations/layer23_attention_weight_max": 39.19085693359375,
      "activations/layer23_attention_weight_min": -24.219297409057617,
      "activations/layer2_attention_weight_max": 32.63014221191406,
      "activations/layer2_attention_weight_min": -31.72681427001953,
      "activations/layer3_attention_weight_max": 87.00784301757812,
      "activations/layer3_attention_weight_min": -87.85560607910156,
      "activations/layer4_attention_weight_max": 87.69639587402344,
      "activations/layer4_attention_weight_min": -86.03740692138672,
      "activations/layer5_attention_weight_max": 61.59381866455078,
      "activations/layer5_attention_weight_min": -74.58167266845703,
      "activations/layer6_attention_weight_max": 52.71541213989258,
      "activations/layer6_attention_weight_min": -50.08883285522461,
      "activations/layer7_attention_weight_max": 67.2366714477539,
      "activations/layer7_attention_weight_min": -61.595054626464844,
      "activations/layer8_attention_weight_max": 45.22815704345703,
      "activations/layer8_attention_weight_min": -49.05915451049805,
      "activations/layer9_attention_weight_max": 51.19059753417969,
      "activations/layer9_attention_weight_min": -44.987037658691406,
      "epoch": 18.52,
      "learning_rate": 3.087462121212121e-05,
      "loss": 2.7343,
      "step": 318750
    },
    {
      "activations/layer0_attention_weight_max": 15.282793998718262,
      "activations/layer0_attention_weight_min": -14.493045806884766,
      "activations/layer10_attention_weight_max": 32.69397735595703,
      "activations/layer10_attention_weight_min": -35.219276428222656,
      "activations/layer11_attention_weight_max": 31.661544799804688,
      "activations/layer11_attention_weight_min": -33.66704177856445,
      "activations/layer12_attention_weight_max": 17.01939582824707,
      "activations/layer12_attention_weight_min": -28.306140899658203,
      "activations/layer13_attention_weight_max": 30.980045318603516,
      "activations/layer13_attention_weight_min": -27.443450927734375,
      "activations/layer14_attention_weight_max": 32.203739166259766,
      "activations/layer14_attention_weight_min": -29.300676345825195,
      "activations/layer15_attention_weight_max": 30.18877410888672,
      "activations/layer15_attention_weight_min": -29.448911666870117,
      "activations/layer16_attention_weight_max": 31.255569458007812,
      "activations/layer16_attention_weight_min": -29.24746322631836,
      "activations/layer17_attention_weight_max": 48.82499313354492,
      "activations/layer17_attention_weight_min": -43.92359924316406,
      "activations/layer18_attention_weight_max": 43.056129455566406,
      "activations/layer18_attention_weight_min": -39.07588195800781,
      "activations/layer19_attention_weight_max": 22.591064453125,
      "activations/layer19_attention_weight_min": -20.382373809814453,
      "activations/layer1_attention_weight_max": 16.455615997314453,
      "activations/layer1_attention_weight_min": -13.69251537322998,
      "activations/layer20_attention_weight_max": 19.97288703918457,
      "activations/layer20_attention_weight_min": -21.850505828857422,
      "activations/layer21_attention_weight_max": 28.225982666015625,
      "activations/layer21_attention_weight_min": -20.6035099029541,
      "activations/layer22_attention_weight_max": 28.66158103942871,
      "activations/layer22_attention_weight_min": -24.769147872924805,
      "activations/layer23_attention_weight_max": 36.565269470214844,
      "activations/layer23_attention_weight_min": -23.864530563354492,
      "activations/layer2_attention_weight_max": 31.042659759521484,
      "activations/layer2_attention_weight_min": -29.32637596130371,
      "activations/layer3_attention_weight_max": 82.39269256591797,
      "activations/layer3_attention_weight_min": -86.48767852783203,
      "activations/layer4_attention_weight_max": 78.98939514160156,
      "activations/layer4_attention_weight_min": -83.10188293457031,
      "activations/layer5_attention_weight_max": 57.602622985839844,
      "activations/layer5_attention_weight_min": -70.984619140625,
      "activations/layer6_attention_weight_max": 47.16718292236328,
      "activations/layer6_attention_weight_min": -48.88460159301758,
      "activations/layer7_attention_weight_max": 62.85738754272461,
      "activations/layer7_attention_weight_min": -61.034446716308594,
      "activations/layer8_attention_weight_max": 44.299617767333984,
      "activations/layer8_attention_weight_min": -48.59694290161133,
      "activations/layer9_attention_weight_max": 44.692535400390625,
      "activations/layer9_attention_weight_min": -44.46470642089844,
      "epoch": 18.52,
      "learning_rate": 3.085568181818182e-05,
      "loss": 2.737,
      "step": 318800
    },
    {
      "activations/layer0_attention_weight_max": 15.770467758178711,
      "activations/layer0_attention_weight_min": -13.953278541564941,
      "activations/layer10_attention_weight_max": 35.20275115966797,
      "activations/layer10_attention_weight_min": -33.29922103881836,
      "activations/layer11_attention_weight_max": 32.30586624145508,
      "activations/layer11_attention_weight_min": -33.754173278808594,
      "activations/layer12_attention_weight_max": 19.975955963134766,
      "activations/layer12_attention_weight_min": -27.018152236938477,
      "activations/layer13_attention_weight_max": 35.22611618041992,
      "activations/layer13_attention_weight_min": -25.710569381713867,
      "activations/layer14_attention_weight_max": 33.4764518737793,
      "activations/layer14_attention_weight_min": -28.564987182617188,
      "activations/layer15_attention_weight_max": 32.046241760253906,
      "activations/layer15_attention_weight_min": -28.538530349731445,
      "activations/layer16_attention_weight_max": 34.42558288574219,
      "activations/layer16_attention_weight_min": -31.190303802490234,
      "activations/layer17_attention_weight_max": 55.96171951293945,
      "activations/layer17_attention_weight_min": -45.9867057800293,
      "activations/layer18_attention_weight_max": 50.71186828613281,
      "activations/layer18_attention_weight_min": -42.78887939453125,
      "activations/layer19_attention_weight_max": 24.806581497192383,
      "activations/layer19_attention_weight_min": -23.52924346923828,
      "activations/layer1_attention_weight_max": 17.030841827392578,
      "activations/layer1_attention_weight_min": -14.627939224243164,
      "activations/layer20_attention_weight_max": 25.047935485839844,
      "activations/layer20_attention_weight_min": -22.19542694091797,
      "activations/layer21_attention_weight_max": 33.77547836303711,
      "activations/layer21_attention_weight_min": -20.080596923828125,
      "activations/layer22_attention_weight_max": 30.33849334716797,
      "activations/layer22_attention_weight_min": -25.850162506103516,
      "activations/layer23_attention_weight_max": 34.98699188232422,
      "activations/layer23_attention_weight_min": -24.142822265625,
      "activations/layer2_attention_weight_max": 30.835472106933594,
      "activations/layer2_attention_weight_min": -30.18096923828125,
      "activations/layer3_attention_weight_max": 84.07349395751953,
      "activations/layer3_attention_weight_min": -85.59740447998047,
      "activations/layer4_attention_weight_max": 82.7180404663086,
      "activations/layer4_attention_weight_min": -82.89591979980469,
      "activations/layer5_attention_weight_max": 63.36418914794922,
      "activations/layer5_attention_weight_min": -72.11354064941406,
      "activations/layer6_attention_weight_max": 50.83699035644531,
      "activations/layer6_attention_weight_min": -48.02346420288086,
      "activations/layer7_attention_weight_max": 62.06372833251953,
      "activations/layer7_attention_weight_min": -58.8143310546875,
      "activations/layer8_attention_weight_max": 43.13142395019531,
      "activations/layer8_attention_weight_min": -47.153072357177734,
      "activations/layer9_attention_weight_max": 42.925621032714844,
      "activations/layer9_attention_weight_min": -43.38103485107422,
      "epoch": 18.53,
      "learning_rate": 3.0836742424242426e-05,
      "loss": 2.7424,
      "step": 318850
    },
    {
      "activations/layer0_attention_weight_max": 15.864591598510742,
      "activations/layer0_attention_weight_min": -13.50244140625,
      "activations/layer10_attention_weight_max": 36.27215576171875,
      "activations/layer10_attention_weight_min": -35.065731048583984,
      "activations/layer11_attention_weight_max": 33.700584411621094,
      "activations/layer11_attention_weight_min": -32.53572082519531,
      "activations/layer12_attention_weight_max": 18.281476974487305,
      "activations/layer12_attention_weight_min": -25.5427188873291,
      "activations/layer13_attention_weight_max": 37.55401611328125,
      "activations/layer13_attention_weight_min": -28.18778419494629,
      "activations/layer14_attention_weight_max": 35.804405212402344,
      "activations/layer14_attention_weight_min": -29.401952743530273,
      "activations/layer15_attention_weight_max": 34.050533294677734,
      "activations/layer15_attention_weight_min": -32.34543228149414,
      "activations/layer16_attention_weight_max": 33.2876091003418,
      "activations/layer16_attention_weight_min": -31.58424949645996,
      "activations/layer17_attention_weight_max": 48.47682189941406,
      "activations/layer17_attention_weight_min": -42.70414733886719,
      "activations/layer18_attention_weight_max": 46.01850509643555,
      "activations/layer18_attention_weight_min": -40.51753616333008,
      "activations/layer19_attention_weight_max": 25.17411231994629,
      "activations/layer19_attention_weight_min": -25.460399627685547,
      "activations/layer1_attention_weight_max": 16.263504028320312,
      "activations/layer1_attention_weight_min": -13.75269603729248,
      "activations/layer20_attention_weight_max": 23.732858657836914,
      "activations/layer20_attention_weight_min": -21.83578109741211,
      "activations/layer21_attention_weight_max": 37.84678268432617,
      "activations/layer21_attention_weight_min": -22.946701049804688,
      "activations/layer22_attention_weight_max": 32.42140197753906,
      "activations/layer22_attention_weight_min": -26.439619064331055,
      "activations/layer23_attention_weight_max": 37.28348159790039,
      "activations/layer23_attention_weight_min": -23.31433868408203,
      "activations/layer2_attention_weight_max": 32.310577392578125,
      "activations/layer2_attention_weight_min": -30.024127960205078,
      "activations/layer3_attention_weight_max": 88.0209732055664,
      "activations/layer3_attention_weight_min": -88.14983367919922,
      "activations/layer4_attention_weight_max": 86.08076477050781,
      "activations/layer4_attention_weight_min": -83.64714813232422,
      "activations/layer5_attention_weight_max": 62.17498779296875,
      "activations/layer5_attention_weight_min": -72.9439926147461,
      "activations/layer6_attention_weight_max": 51.30050277709961,
      "activations/layer6_attention_weight_min": -50.24868392944336,
      "activations/layer7_attention_weight_max": 63.39445495605469,
      "activations/layer7_attention_weight_min": -63.13872146606445,
      "activations/layer8_attention_weight_max": 46.58308792114258,
      "activations/layer8_attention_weight_min": -48.26333236694336,
      "activations/layer9_attention_weight_max": 43.48366928100586,
      "activations/layer9_attention_weight_min": -46.56538009643555,
      "epoch": 18.53,
      "learning_rate": 3.081780303030303e-05,
      "loss": 2.7353,
      "step": 318900
    },
    {
      "activations/layer0_attention_weight_max": 15.24888801574707,
      "activations/layer0_attention_weight_min": -13.555606842041016,
      "activations/layer10_attention_weight_max": 34.50566482543945,
      "activations/layer10_attention_weight_min": -35.25007629394531,
      "activations/layer11_attention_weight_max": 31.456844329833984,
      "activations/layer11_attention_weight_min": -32.57383728027344,
      "activations/layer12_attention_weight_max": 19.75366973876953,
      "activations/layer12_attention_weight_min": -24.319616317749023,
      "activations/layer13_attention_weight_max": 31.202266693115234,
      "activations/layer13_attention_weight_min": -26.19866180419922,
      "activations/layer14_attention_weight_max": 34.41526794433594,
      "activations/layer14_attention_weight_min": -29.44202995300293,
      "activations/layer15_attention_weight_max": 30.371768951416016,
      "activations/layer15_attention_weight_min": -27.86009979248047,
      "activations/layer16_attention_weight_max": 30.916051864624023,
      "activations/layer16_attention_weight_min": -30.62956428527832,
      "activations/layer17_attention_weight_max": 48.900108337402344,
      "activations/layer17_attention_weight_min": -42.22554397583008,
      "activations/layer18_attention_weight_max": 45.43135452270508,
      "activations/layer18_attention_weight_min": -36.10682678222656,
      "activations/layer19_attention_weight_max": 22.2783145904541,
      "activations/layer19_attention_weight_min": -22.329858779907227,
      "activations/layer1_attention_weight_max": 15.403738021850586,
      "activations/layer1_attention_weight_min": -13.693059921264648,
      "activations/layer20_attention_weight_max": 20.359216690063477,
      "activations/layer20_attention_weight_min": -20.43790054321289,
      "activations/layer21_attention_weight_max": 31.693552017211914,
      "activations/layer21_attention_weight_min": -22.32219886779785,
      "activations/layer22_attention_weight_max": 26.762266159057617,
      "activations/layer22_attention_weight_min": -22.805601119995117,
      "activations/layer23_attention_weight_max": 34.41621017456055,
      "activations/layer23_attention_weight_min": -27.599891662597656,
      "activations/layer2_attention_weight_max": 30.82392120361328,
      "activations/layer2_attention_weight_min": -31.47433853149414,
      "activations/layer3_attention_weight_max": 86.58521270751953,
      "activations/layer3_attention_weight_min": -89.72592163085938,
      "activations/layer4_attention_weight_max": 84.69722747802734,
      "activations/layer4_attention_weight_min": -84.13214111328125,
      "activations/layer5_attention_weight_max": 63.67533874511719,
      "activations/layer5_attention_weight_min": -78.07719421386719,
      "activations/layer6_attention_weight_max": 50.36177062988281,
      "activations/layer6_attention_weight_min": -49.7525749206543,
      "activations/layer7_attention_weight_max": 65.92752075195312,
      "activations/layer7_attention_weight_min": -65.88665771484375,
      "activations/layer8_attention_weight_max": 45.73881912231445,
      "activations/layer8_attention_weight_min": -47.80915069580078,
      "activations/layer9_attention_weight_max": 45.50936508178711,
      "activations/layer9_attention_weight_min": -49.232398986816406,
      "epoch": 18.53,
      "learning_rate": 3.0798863636363635e-05,
      "loss": 2.7439,
      "step": 318950
    },
    {
      "activations/layer0_attention_weight_max": 15.6251802444458,
      "activations/layer0_attention_weight_min": -13.544604301452637,
      "activations/layer10_attention_weight_max": 35.07870101928711,
      "activations/layer10_attention_weight_min": -36.439002990722656,
      "activations/layer11_attention_weight_max": 32.28709411621094,
      "activations/layer11_attention_weight_min": -35.12570571899414,
      "activations/layer12_attention_weight_max": 18.621339797973633,
      "activations/layer12_attention_weight_min": -29.274044036865234,
      "activations/layer13_attention_weight_max": 34.27743911743164,
      "activations/layer13_attention_weight_min": -26.52454948425293,
      "activations/layer14_attention_weight_max": 33.63032913208008,
      "activations/layer14_attention_weight_min": -30.375465393066406,
      "activations/layer15_attention_weight_max": 32.03451156616211,
      "activations/layer15_attention_weight_min": -29.932756423950195,
      "activations/layer16_attention_weight_max": 32.18013381958008,
      "activations/layer16_attention_weight_min": -30.969514846801758,
      "activations/layer17_attention_weight_max": 51.16347122192383,
      "activations/layer17_attention_weight_min": -47.06764602661133,
      "activations/layer18_attention_weight_max": 50.34109878540039,
      "activations/layer18_attention_weight_min": -42.22452926635742,
      "activations/layer19_attention_weight_max": 23.911331176757812,
      "activations/layer19_attention_weight_min": -24.1361141204834,
      "activations/layer1_attention_weight_max": 16.371416091918945,
      "activations/layer1_attention_weight_min": -14.643625259399414,
      "activations/layer20_attention_weight_max": 20.979721069335938,
      "activations/layer20_attention_weight_min": -22.79918670654297,
      "activations/layer21_attention_weight_max": 35.02869415283203,
      "activations/layer21_attention_weight_min": -21.726125717163086,
      "activations/layer22_attention_weight_max": 33.143775939941406,
      "activations/layer22_attention_weight_min": -25.07161521911621,
      "activations/layer23_attention_weight_max": 37.41908264160156,
      "activations/layer23_attention_weight_min": -25.083911895751953,
      "activations/layer2_attention_weight_max": 29.688648223876953,
      "activations/layer2_attention_weight_min": -28.623992919921875,
      "activations/layer3_attention_weight_max": 83.83260345458984,
      "activations/layer3_attention_weight_min": -84.11647033691406,
      "activations/layer4_attention_weight_max": 81.90535736083984,
      "activations/layer4_attention_weight_min": -81.39328002929688,
      "activations/layer5_attention_weight_max": 64.07762908935547,
      "activations/layer5_attention_weight_min": -71.15083312988281,
      "activations/layer6_attention_weight_max": 48.24031448364258,
      "activations/layer6_attention_weight_min": -49.775882720947266,
      "activations/layer7_attention_weight_max": 65.87480163574219,
      "activations/layer7_attention_weight_min": -60.66391372680664,
      "activations/layer8_attention_weight_max": 45.47456741333008,
      "activations/layer8_attention_weight_min": -47.34623718261719,
      "activations/layer9_attention_weight_max": 46.54554748535156,
      "activations/layer9_attention_weight_min": -44.75883483886719,
      "epoch": 18.54,
      "learning_rate": 3.077992424242424e-05,
      "loss": 2.715,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4773,
      "eval_samples_per_second": 506.53,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4773,
      "eval_openwebtext_samples_per_second": 506.53,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9551,
      "eval_wikitext_samples_per_second": 233.236,
      "step": 319000
    },
    {
      "epoch": 18.54,
      "eval_lambada_loss": 2.439453125,
      "eval_lambada_ppl": 11.46676813896006,
      "eval_lambada_runtime": 9.6017,
      "eval_lambada_samples_per_second": 507.099,
      "step": 319000
    },
    {
      "activations/layer0_attention_weight_max": 14.68523120880127,
      "activations/layer0_attention_weight_min": -13.57007884979248,
      "activations/layer10_attention_weight_max": 39.923004150390625,
      "activations/layer10_attention_weight_min": -36.7066535949707,
      "activations/layer11_attention_weight_max": 38.339576721191406,
      "activations/layer11_attention_weight_min": -34.938720703125,
      "activations/layer12_attention_weight_max": 18.506948471069336,
      "activations/layer12_attention_weight_min": -23.52476692199707,
      "activations/layer13_attention_weight_max": 37.24969482421875,
      "activations/layer13_attention_weight_min": -29.402673721313477,
      "activations/layer14_attention_weight_max": 34.073883056640625,
      "activations/layer14_attention_weight_min": -30.943967819213867,
      "activations/layer15_attention_weight_max": 31.220523834228516,
      "activations/layer15_attention_weight_min": -31.00315284729004,
      "activations/layer16_attention_weight_max": 35.19994354248047,
      "activations/layer16_attention_weight_min": -31.91571044921875,
      "activations/layer17_attention_weight_max": 51.693477630615234,
      "activations/layer17_attention_weight_min": -44.06309509277344,
      "activations/layer18_attention_weight_max": 46.410499572753906,
      "activations/layer18_attention_weight_min": -39.067604064941406,
      "activations/layer19_attention_weight_max": 22.378122329711914,
      "activations/layer19_attention_weight_min": -23.499427795410156,
      "activations/layer1_attention_weight_max": 17.494407653808594,
      "activations/layer1_attention_weight_min": -14.45865535736084,
      "activations/layer20_attention_weight_max": 22.81167984008789,
      "activations/layer20_attention_weight_min": -22.073715209960938,
      "activations/layer21_attention_weight_max": 40.28799819946289,
      "activations/layer21_attention_weight_min": -25.208215713500977,
      "activations/layer22_attention_weight_max": 29.216279983520508,
      "activations/layer22_attention_weight_min": -28.64214324951172,
      "activations/layer23_attention_weight_max": 41.310943603515625,
      "activations/layer23_attention_weight_min": -25.301130294799805,
      "activations/layer2_attention_weight_max": 36.466407775878906,
      "activations/layer2_attention_weight_min": -30.762313842773438,
      "activations/layer3_attention_weight_max": 92.95394134521484,
      "activations/layer3_attention_weight_min": -89.0853271484375,
      "activations/layer4_attention_weight_max": 87.21571350097656,
      "activations/layer4_attention_weight_min": -82.71927642822266,
      "activations/layer5_attention_weight_max": 62.318641662597656,
      "activations/layer5_attention_weight_min": -66.63626098632812,
      "activations/layer6_attention_weight_max": 50.78578186035156,
      "activations/layer6_attention_weight_min": -48.711063385009766,
      "activations/layer7_attention_weight_max": 66.74945831298828,
      "activations/layer7_attention_weight_min": -67.33828735351562,
      "activations/layer8_attention_weight_max": 47.864234924316406,
      "activations/layer8_attention_weight_min": -49.57333755493164,
      "activations/layer9_attention_weight_max": 52.97785186767578,
      "activations/layer9_attention_weight_min": -48.27723693847656,
      "epoch": 18.54,
      "learning_rate": 3.0760984848484845e-05,
      "loss": 2.7301,
      "step": 319050
    },
    {
      "activations/layer0_attention_weight_max": 15.290093421936035,
      "activations/layer0_attention_weight_min": -13.671767234802246,
      "activations/layer10_attention_weight_max": 36.83472442626953,
      "activations/layer10_attention_weight_min": -35.492591857910156,
      "activations/layer11_attention_weight_max": 32.443153381347656,
      "activations/layer11_attention_weight_min": -34.03993606567383,
      "activations/layer12_attention_weight_max": 17.909841537475586,
      "activations/layer12_attention_weight_min": -23.480554580688477,
      "activations/layer13_attention_weight_max": 33.39884948730469,
      "activations/layer13_attention_weight_min": -30.424596786499023,
      "activations/layer14_attention_weight_max": 33.6287841796875,
      "activations/layer14_attention_weight_min": -31.45519256591797,
      "activations/layer15_attention_weight_max": 31.779743194580078,
      "activations/layer15_attention_weight_min": -31.80312156677246,
      "activations/layer16_attention_weight_max": 32.918479919433594,
      "activations/layer16_attention_weight_min": -29.7491397857666,
      "activations/layer17_attention_weight_max": 50.6757698059082,
      "activations/layer17_attention_weight_min": -41.900508880615234,
      "activations/layer18_attention_weight_max": 47.102787017822266,
      "activations/layer18_attention_weight_min": -38.646087646484375,
      "activations/layer19_attention_weight_max": 22.861074447631836,
      "activations/layer19_attention_weight_min": -24.768360137939453,
      "activations/layer1_attention_weight_max": 15.55785083770752,
      "activations/layer1_attention_weight_min": -13.533470153808594,
      "activations/layer20_attention_weight_max": 22.439281463623047,
      "activations/layer20_attention_weight_min": -21.57504653930664,
      "activations/layer21_attention_weight_max": 32.8564338684082,
      "activations/layer21_attention_weight_min": -22.02147674560547,
      "activations/layer22_attention_weight_max": 28.56694221496582,
      "activations/layer22_attention_weight_min": -25.82290267944336,
      "activations/layer23_attention_weight_max": 39.31871032714844,
      "activations/layer23_attention_weight_min": -23.910581588745117,
      "activations/layer2_attention_weight_max": 32.070838928222656,
      "activations/layer2_attention_weight_min": -31.204906463623047,
      "activations/layer3_attention_weight_max": 85.20409393310547,
      "activations/layer3_attention_weight_min": -82.30409240722656,
      "activations/layer4_attention_weight_max": 85.21173858642578,
      "activations/layer4_attention_weight_min": -83.02143859863281,
      "activations/layer5_attention_weight_max": 63.426544189453125,
      "activations/layer5_attention_weight_min": -75.54031372070312,
      "activations/layer6_attention_weight_max": 48.20671463012695,
      "activations/layer6_attention_weight_min": -52.84819793701172,
      "activations/layer7_attention_weight_max": 69.09486389160156,
      "activations/layer7_attention_weight_min": -64.24466705322266,
      "activations/layer8_attention_weight_max": 42.86360549926758,
      "activations/layer8_attention_weight_min": -46.74549865722656,
      "activations/layer9_attention_weight_max": 41.582733154296875,
      "activations/layer9_attention_weight_min": -45.094268798828125,
      "epoch": 18.54,
      "learning_rate": 3.074204545454545e-05,
      "loss": 2.7361,
      "step": 319100
    },
    {
      "activations/layer0_attention_weight_max": 15.30494499206543,
      "activations/layer0_attention_weight_min": -13.668542861938477,
      "activations/layer10_attention_weight_max": 33.5985221862793,
      "activations/layer10_attention_weight_min": -33.95219802856445,
      "activations/layer11_attention_weight_max": 31.435928344726562,
      "activations/layer11_attention_weight_min": -33.5858154296875,
      "activations/layer12_attention_weight_max": 20.759597778320312,
      "activations/layer12_attention_weight_min": -23.877010345458984,
      "activations/layer13_attention_weight_max": 32.05393600463867,
      "activations/layer13_attention_weight_min": -28.326745986938477,
      "activations/layer14_attention_weight_max": 34.73701477050781,
      "activations/layer14_attention_weight_min": -29.954572677612305,
      "activations/layer15_attention_weight_max": 34.227237701416016,
      "activations/layer15_attention_weight_min": -31.17153549194336,
      "activations/layer16_attention_weight_max": 34.69804000854492,
      "activations/layer16_attention_weight_min": -31.782901763916016,
      "activations/layer17_attention_weight_max": 50.67350769042969,
      "activations/layer17_attention_weight_min": -46.24153518676758,
      "activations/layer18_attention_weight_max": 47.126686096191406,
      "activations/layer18_attention_weight_min": -42.65056610107422,
      "activations/layer19_attention_weight_max": 25.69556999206543,
      "activations/layer19_attention_weight_min": -24.783037185668945,
      "activations/layer1_attention_weight_max": 15.21575927734375,
      "activations/layer1_attention_weight_min": -13.942770957946777,
      "activations/layer20_attention_weight_max": 23.88011360168457,
      "activations/layer20_attention_weight_min": -22.37975311279297,
      "activations/layer21_attention_weight_max": 36.23021697998047,
      "activations/layer21_attention_weight_min": -24.66765785217285,
      "activations/layer22_attention_weight_max": 32.443031311035156,
      "activations/layer22_attention_weight_min": -28.38409423828125,
      "activations/layer23_attention_weight_max": 39.49935531616211,
      "activations/layer23_attention_weight_min": -24.68148422241211,
      "activations/layer2_attention_weight_max": 31.486494064331055,
      "activations/layer2_attention_weight_min": -30.16814422607422,
      "activations/layer3_attention_weight_max": 84.0257339477539,
      "activations/layer3_attention_weight_min": -83.6601791381836,
      "activations/layer4_attention_weight_max": 83.84883117675781,
      "activations/layer4_attention_weight_min": -82.27726745605469,
      "activations/layer5_attention_weight_max": 59.71411895751953,
      "activations/layer5_attention_weight_min": -69.88640594482422,
      "activations/layer6_attention_weight_max": 48.97378921508789,
      "activations/layer6_attention_weight_min": -48.20120620727539,
      "activations/layer7_attention_weight_max": 64.88589477539062,
      "activations/layer7_attention_weight_min": -60.38703155517578,
      "activations/layer8_attention_weight_max": 43.417789459228516,
      "activations/layer8_attention_weight_min": -49.675209045410156,
      "activations/layer9_attention_weight_max": 40.77713394165039,
      "activations/layer9_attention_weight_min": -45.557762145996094,
      "epoch": 18.54,
      "learning_rate": 3.0723106060606055e-05,
      "loss": 2.7329,
      "step": 319150
    },
    {
      "activations/layer0_attention_weight_max": 16.10342788696289,
      "activations/layer0_attention_weight_min": -13.620603561401367,
      "activations/layer10_attention_weight_max": 35.48359298706055,
      "activations/layer10_attention_weight_min": -38.71101760864258,
      "activations/layer11_attention_weight_max": 33.51526641845703,
      "activations/layer11_attention_weight_min": -36.12010192871094,
      "activations/layer12_attention_weight_max": 18.64514923095703,
      "activations/layer12_attention_weight_min": -25.319416046142578,
      "activations/layer13_attention_weight_max": 33.22737503051758,
      "activations/layer13_attention_weight_min": -27.69107437133789,
      "activations/layer14_attention_weight_max": 34.4006233215332,
      "activations/layer14_attention_weight_min": -31.650188446044922,
      "activations/layer15_attention_weight_max": 32.04132080078125,
      "activations/layer15_attention_weight_min": -33.304595947265625,
      "activations/layer16_attention_weight_max": 34.546810150146484,
      "activations/layer16_attention_weight_min": -33.22853469848633,
      "activations/layer17_attention_weight_max": 52.29071807861328,
      "activations/layer17_attention_weight_min": -47.64291000366211,
      "activations/layer18_attention_weight_max": 43.67512893676758,
      "activations/layer18_attention_weight_min": -42.80760192871094,
      "activations/layer19_attention_weight_max": 23.061864852905273,
      "activations/layer19_attention_weight_min": -22.960128784179688,
      "activations/layer1_attention_weight_max": 16.60968017578125,
      "activations/layer1_attention_weight_min": -15.524984359741211,
      "activations/layer20_attention_weight_max": 22.055633544921875,
      "activations/layer20_attention_weight_min": -21.712852478027344,
      "activations/layer21_attention_weight_max": 41.527217864990234,
      "activations/layer21_attention_weight_min": -25.999065399169922,
      "activations/layer22_attention_weight_max": 30.473033905029297,
      "activations/layer22_attention_weight_min": -25.812110900878906,
      "activations/layer23_attention_weight_max": 37.93608474731445,
      "activations/layer23_attention_weight_min": -25.651458740234375,
      "activations/layer2_attention_weight_max": 31.389324188232422,
      "activations/layer2_attention_weight_min": -29.824827194213867,
      "activations/layer3_attention_weight_max": 84.55516052246094,
      "activations/layer3_attention_weight_min": -86.5753402709961,
      "activations/layer4_attention_weight_max": 85.39232635498047,
      "activations/layer4_attention_weight_min": -82.78947448730469,
      "activations/layer5_attention_weight_max": 66.73701477050781,
      "activations/layer5_attention_weight_min": -74.12005615234375,
      "activations/layer6_attention_weight_max": 51.0139045715332,
      "activations/layer6_attention_weight_min": -54.478580474853516,
      "activations/layer7_attention_weight_max": 66.74848175048828,
      "activations/layer7_attention_weight_min": -64.92051696777344,
      "activations/layer8_attention_weight_max": 47.80030822753906,
      "activations/layer8_attention_weight_min": -49.35243606567383,
      "activations/layer9_attention_weight_max": 44.36159896850586,
      "activations/layer9_attention_weight_min": -48.776729583740234,
      "epoch": 18.55,
      "learning_rate": 3.070416666666666e-05,
      "loss": 2.7267,
      "step": 319200
    },
    {
      "activations/layer0_attention_weight_max": 16.610803604125977,
      "activations/layer0_attention_weight_min": -13.61921501159668,
      "activations/layer10_attention_weight_max": 38.51459503173828,
      "activations/layer10_attention_weight_min": -37.093475341796875,
      "activations/layer11_attention_weight_max": 34.95664596557617,
      "activations/layer11_attention_weight_min": -35.093955993652344,
      "activations/layer12_attention_weight_max": 18.720035552978516,
      "activations/layer12_attention_weight_min": -22.75760841369629,
      "activations/layer13_attention_weight_max": 40.46804428100586,
      "activations/layer13_attention_weight_min": -29.17966079711914,
      "activations/layer14_attention_weight_max": 36.37443923950195,
      "activations/layer14_attention_weight_min": -33.76570510864258,
      "activations/layer15_attention_weight_max": 38.4189567565918,
      "activations/layer15_attention_weight_min": -30.88791847229004,
      "activations/layer16_attention_weight_max": 34.39224624633789,
      "activations/layer16_attention_weight_min": -32.8962516784668,
      "activations/layer17_attention_weight_max": 53.337642669677734,
      "activations/layer17_attention_weight_min": -45.71637725830078,
      "activations/layer18_attention_weight_max": 50.039215087890625,
      "activations/layer18_attention_weight_min": -40.84992599487305,
      "activations/layer19_attention_weight_max": 25.200315475463867,
      "activations/layer19_attention_weight_min": -24.657543182373047,
      "activations/layer1_attention_weight_max": 16.379486083984375,
      "activations/layer1_attention_weight_min": -15.376591682434082,
      "activations/layer20_attention_weight_max": 22.824840545654297,
      "activations/layer20_attention_weight_min": -22.152795791625977,
      "activations/layer21_attention_weight_max": 38.939903259277344,
      "activations/layer21_attention_weight_min": -25.252050399780273,
      "activations/layer22_attention_weight_max": 32.267940521240234,
      "activations/layer22_attention_weight_min": -27.00296401977539,
      "activations/layer23_attention_weight_max": 43.29349899291992,
      "activations/layer23_attention_weight_min": -26.263809204101562,
      "activations/layer2_attention_weight_max": 30.564184188842773,
      "activations/layer2_attention_weight_min": -29.672758102416992,
      "activations/layer3_attention_weight_max": 86.20024871826172,
      "activations/layer3_attention_weight_min": -86.16685485839844,
      "activations/layer4_attention_weight_max": 84.06321716308594,
      "activations/layer4_attention_weight_min": -83.31880187988281,
      "activations/layer5_attention_weight_max": 64.70504760742188,
      "activations/layer5_attention_weight_min": -69.92625427246094,
      "activations/layer6_attention_weight_max": 52.391658782958984,
      "activations/layer6_attention_weight_min": -52.582664489746094,
      "activations/layer7_attention_weight_max": 66.22727966308594,
      "activations/layer7_attention_weight_min": -63.588375091552734,
      "activations/layer8_attention_weight_max": 48.69813537597656,
      "activations/layer8_attention_weight_min": -51.14799118041992,
      "activations/layer9_attention_weight_max": 47.17554473876953,
      "activations/layer9_attention_weight_min": -48.1737174987793,
      "epoch": 18.55,
      "learning_rate": 3.068522727272727e-05,
      "loss": 2.7293,
      "step": 319250
    },
    {
      "activations/layer0_attention_weight_max": 15.668779373168945,
      "activations/layer0_attention_weight_min": -13.381879806518555,
      "activations/layer10_attention_weight_max": 36.51276779174805,
      "activations/layer10_attention_weight_min": -35.19682693481445,
      "activations/layer11_attention_weight_max": 34.40227508544922,
      "activations/layer11_attention_weight_min": -34.033180236816406,
      "activations/layer12_attention_weight_max": 20.51907730102539,
      "activations/layer12_attention_weight_min": -23.558143615722656,
      "activations/layer13_attention_weight_max": 41.3809700012207,
      "activations/layer13_attention_weight_min": -28.042634963989258,
      "activations/layer14_attention_weight_max": 38.714500427246094,
      "activations/layer14_attention_weight_min": -30.752059936523438,
      "activations/layer15_attention_weight_max": 38.008262634277344,
      "activations/layer15_attention_weight_min": -30.770252227783203,
      "activations/layer16_attention_weight_max": 38.66651916503906,
      "activations/layer16_attention_weight_min": -34.914527893066406,
      "activations/layer17_attention_weight_max": 59.31407928466797,
      "activations/layer17_attention_weight_min": -49.69580841064453,
      "activations/layer18_attention_weight_max": 54.61377716064453,
      "activations/layer18_attention_weight_min": -45.23899841308594,
      "activations/layer19_attention_weight_max": 27.1518497467041,
      "activations/layer19_attention_weight_min": -24.345365524291992,
      "activations/layer1_attention_weight_max": 16.456993103027344,
      "activations/layer1_attention_weight_min": -14.95676040649414,
      "activations/layer20_attention_weight_max": 26.72100830078125,
      "activations/layer20_attention_weight_min": -22.598554611206055,
      "activations/layer21_attention_weight_max": 42.329811096191406,
      "activations/layer21_attention_weight_min": -25.410558700561523,
      "activations/layer22_attention_weight_max": 31.87826919555664,
      "activations/layer22_attention_weight_min": -25.137897491455078,
      "activations/layer23_attention_weight_max": 39.55141830444336,
      "activations/layer23_attention_weight_min": -24.02619743347168,
      "activations/layer2_attention_weight_max": 31.784122467041016,
      "activations/layer2_attention_weight_min": -30.935680389404297,
      "activations/layer3_attention_weight_max": 84.88792419433594,
      "activations/layer3_attention_weight_min": -89.96280670166016,
      "activations/layer4_attention_weight_max": 84.5873031616211,
      "activations/layer4_attention_weight_min": -81.9193344116211,
      "activations/layer5_attention_weight_max": 64.06946563720703,
      "activations/layer5_attention_weight_min": -74.6994400024414,
      "activations/layer6_attention_weight_max": 51.9698600769043,
      "activations/layer6_attention_weight_min": -51.22215270996094,
      "activations/layer7_attention_weight_max": 64.43095397949219,
      "activations/layer7_attention_weight_min": -63.27705001831055,
      "activations/layer8_attention_weight_max": 47.43654251098633,
      "activations/layer8_attention_weight_min": -50.82841873168945,
      "activations/layer9_attention_weight_max": 42.1742057800293,
      "activations/layer9_attention_weight_min": -45.26871109008789,
      "epoch": 18.55,
      "learning_rate": 3.066628787878788e-05,
      "loss": 2.7306,
      "step": 319300
    },
    {
      "activations/layer0_attention_weight_max": 14.69778823852539,
      "activations/layer0_attention_weight_min": -13.414177894592285,
      "activations/layer10_attention_weight_max": 38.00265884399414,
      "activations/layer10_attention_weight_min": -35.16261291503906,
      "activations/layer11_attention_weight_max": 34.46352005004883,
      "activations/layer11_attention_weight_min": -34.189720153808594,
      "activations/layer12_attention_weight_max": 19.864986419677734,
      "activations/layer12_attention_weight_min": -26.921146392822266,
      "activations/layer13_attention_weight_max": 34.13706970214844,
      "activations/layer13_attention_weight_min": -26.108844757080078,
      "activations/layer14_attention_weight_max": 34.52101516723633,
      "activations/layer14_attention_weight_min": -30.372007369995117,
      "activations/layer15_attention_weight_max": 31.035337448120117,
      "activations/layer15_attention_weight_min": -30.99464225769043,
      "activations/layer16_attention_weight_max": 35.468624114990234,
      "activations/layer16_attention_weight_min": -31.758272171020508,
      "activations/layer17_attention_weight_max": 54.825233459472656,
      "activations/layer17_attention_weight_min": -45.43770217895508,
      "activations/layer18_attention_weight_max": 51.27497863769531,
      "activations/layer18_attention_weight_min": -40.93037414550781,
      "activations/layer19_attention_weight_max": 24.2110538482666,
      "activations/layer19_attention_weight_min": -22.14369010925293,
      "activations/layer1_attention_weight_max": 16.06958770751953,
      "activations/layer1_attention_weight_min": -13.761174201965332,
      "activations/layer20_attention_weight_max": 21.330217361450195,
      "activations/layer20_attention_weight_min": -20.07522964477539,
      "activations/layer21_attention_weight_max": 40.36661911010742,
      "activations/layer21_attention_weight_min": -20.118270874023438,
      "activations/layer22_attention_weight_max": 28.869287490844727,
      "activations/layer22_attention_weight_min": -24.79733657836914,
      "activations/layer23_attention_weight_max": 35.477081298828125,
      "activations/layer23_attention_weight_min": -21.03436279296875,
      "activations/layer2_attention_weight_max": 31.31308364868164,
      "activations/layer2_attention_weight_min": -31.343137741088867,
      "activations/layer3_attention_weight_max": 89.01419067382812,
      "activations/layer3_attention_weight_min": -90.6998519897461,
      "activations/layer4_attention_weight_max": 82.49199676513672,
      "activations/layer4_attention_weight_min": -81.10823059082031,
      "activations/layer5_attention_weight_max": 58.323421478271484,
      "activations/layer5_attention_weight_min": -71.8641128540039,
      "activations/layer6_attention_weight_max": 49.373252868652344,
      "activations/layer6_attention_weight_min": -51.26698684692383,
      "activations/layer7_attention_weight_max": 61.830936431884766,
      "activations/layer7_attention_weight_min": -60.02127456665039,
      "activations/layer8_attention_weight_max": 45.76308059692383,
      "activations/layer8_attention_weight_min": -45.77330017089844,
      "activations/layer9_attention_weight_max": 44.7067985534668,
      "activations/layer9_attention_weight_min": -43.03213119506836,
      "epoch": 18.56,
      "learning_rate": 3.064734848484848e-05,
      "loss": 2.7351,
      "step": 319350
    },
    {
      "activations/layer0_attention_weight_max": 15.746638298034668,
      "activations/layer0_attention_weight_min": -13.626463890075684,
      "activations/layer10_attention_weight_max": 37.50366973876953,
      "activations/layer10_attention_weight_min": -39.51416778564453,
      "activations/layer11_attention_weight_max": 34.830101013183594,
      "activations/layer11_attention_weight_min": -35.30012893676758,
      "activations/layer12_attention_weight_max": 19.8580265045166,
      "activations/layer12_attention_weight_min": -27.057783126831055,
      "activations/layer13_attention_weight_max": 36.63517379760742,
      "activations/layer13_attention_weight_min": -28.94190216064453,
      "activations/layer14_attention_weight_max": 36.816627502441406,
      "activations/layer14_attention_weight_min": -32.46851348876953,
      "activations/layer15_attention_weight_max": 34.50559616088867,
      "activations/layer15_attention_weight_min": -32.28549575805664,
      "activations/layer16_attention_weight_max": 37.86830520629883,
      "activations/layer16_attention_weight_min": -32.24597930908203,
      "activations/layer17_attention_weight_max": 57.71078109741211,
      "activations/layer17_attention_weight_min": -46.77547836303711,
      "activations/layer18_attention_weight_max": 56.810020446777344,
      "activations/layer18_attention_weight_min": -41.76791763305664,
      "activations/layer19_attention_weight_max": 28.55506706237793,
      "activations/layer19_attention_weight_min": -23.86760711669922,
      "activations/layer1_attention_weight_max": 15.506308555603027,
      "activations/layer1_attention_weight_min": -14.161460876464844,
      "activations/layer20_attention_weight_max": 29.405595779418945,
      "activations/layer20_attention_weight_min": -22.110353469848633,
      "activations/layer21_attention_weight_max": 40.430419921875,
      "activations/layer21_attention_weight_min": -20.845125198364258,
      "activations/layer22_attention_weight_max": 32.2612190246582,
      "activations/layer22_attention_weight_min": -26.66225814819336,
      "activations/layer23_attention_weight_max": 42.67725372314453,
      "activations/layer23_attention_weight_min": -23.841461181640625,
      "activations/layer2_attention_weight_max": 32.235198974609375,
      "activations/layer2_attention_weight_min": -30.817848205566406,
      "activations/layer3_attention_weight_max": 85.52972412109375,
      "activations/layer3_attention_weight_min": -84.59069061279297,
      "activations/layer4_attention_weight_max": 85.68247985839844,
      "activations/layer4_attention_weight_min": -85.56744384765625,
      "activations/layer5_attention_weight_max": 62.56962203979492,
      "activations/layer5_attention_weight_min": -67.57807922363281,
      "activations/layer6_attention_weight_max": 49.942386627197266,
      "activations/layer6_attention_weight_min": -50.99779510498047,
      "activations/layer7_attention_weight_max": 64.1766128540039,
      "activations/layer7_attention_weight_min": -63.57938766479492,
      "activations/layer8_attention_weight_max": 47.44355392456055,
      "activations/layer8_attention_weight_min": -50.3665657043457,
      "activations/layer9_attention_weight_max": 44.21947479248047,
      "activations/layer9_attention_weight_min": -45.304656982421875,
      "epoch": 18.56,
      "learning_rate": 3.062840909090909e-05,
      "loss": 2.7382,
      "step": 319400
    },
    {
      "activations/layer0_attention_weight_max": 14.770523071289062,
      "activations/layer0_attention_weight_min": -13.610963821411133,
      "activations/layer10_attention_weight_max": 35.673580169677734,
      "activations/layer10_attention_weight_min": -35.220985412597656,
      "activations/layer11_attention_weight_max": 33.33737564086914,
      "activations/layer11_attention_weight_min": -34.642967224121094,
      "activations/layer12_attention_weight_max": 20.272844314575195,
      "activations/layer12_attention_weight_min": -23.715328216552734,
      "activations/layer13_attention_weight_max": 34.13542938232422,
      "activations/layer13_attention_weight_min": -28.74910545349121,
      "activations/layer14_attention_weight_max": 37.15517044067383,
      "activations/layer14_attention_weight_min": -30.482248306274414,
      "activations/layer15_attention_weight_max": 33.167598724365234,
      "activations/layer15_attention_weight_min": -28.620044708251953,
      "activations/layer16_attention_weight_max": 33.26915740966797,
      "activations/layer16_attention_weight_min": -34.248878479003906,
      "activations/layer17_attention_weight_max": 51.69227981567383,
      "activations/layer17_attention_weight_min": -44.57049560546875,
      "activations/layer18_attention_weight_max": 45.1932258605957,
      "activations/layer18_attention_weight_min": -40.632240295410156,
      "activations/layer19_attention_weight_max": 24.256376266479492,
      "activations/layer19_attention_weight_min": -22.60361099243164,
      "activations/layer1_attention_weight_max": 15.17192554473877,
      "activations/layer1_attention_weight_min": -13.898765563964844,
      "activations/layer20_attention_weight_max": 23.043760299682617,
      "activations/layer20_attention_weight_min": -21.12852668762207,
      "activations/layer21_attention_weight_max": 33.19763946533203,
      "activations/layer21_attention_weight_min": -22.30792236328125,
      "activations/layer22_attention_weight_max": 30.921873092651367,
      "activations/layer22_attention_weight_min": -25.41705322265625,
      "activations/layer23_attention_weight_max": 39.44609069824219,
      "activations/layer23_attention_weight_min": -25.164310455322266,
      "activations/layer2_attention_weight_max": 31.408687591552734,
      "activations/layer2_attention_weight_min": -29.578413009643555,
      "activations/layer3_attention_weight_max": 85.80213928222656,
      "activations/layer3_attention_weight_min": -86.86485290527344,
      "activations/layer4_attention_weight_max": 82.66692352294922,
      "activations/layer4_attention_weight_min": -83.13121795654297,
      "activations/layer5_attention_weight_max": 59.55769348144531,
      "activations/layer5_attention_weight_min": -70.5949478149414,
      "activations/layer6_attention_weight_max": 49.474735260009766,
      "activations/layer6_attention_weight_min": -51.827205657958984,
      "activations/layer7_attention_weight_max": 63.13819122314453,
      "activations/layer7_attention_weight_min": -65.1739501953125,
      "activations/layer8_attention_weight_max": 46.11455535888672,
      "activations/layer8_attention_weight_min": -50.04468536376953,
      "activations/layer9_attention_weight_max": 45.55418395996094,
      "activations/layer9_attention_weight_min": -46.0758171081543,
      "epoch": 18.56,
      "learning_rate": 3.060946969696969e-05,
      "loss": 2.7352,
      "step": 319450
    },
    {
      "activations/layer0_attention_weight_max": 16.477571487426758,
      "activations/layer0_attention_weight_min": -13.669632911682129,
      "activations/layer10_attention_weight_max": 39.206275939941406,
      "activations/layer10_attention_weight_min": -36.216705322265625,
      "activations/layer11_attention_weight_max": 32.18327331542969,
      "activations/layer11_attention_weight_min": -34.689727783203125,
      "activations/layer12_attention_weight_max": 20.940322875976562,
      "activations/layer12_attention_weight_min": -25.75164222717285,
      "activations/layer13_attention_weight_max": 40.05616760253906,
      "activations/layer13_attention_weight_min": -29.07994270324707,
      "activations/layer14_attention_weight_max": 39.140357971191406,
      "activations/layer14_attention_weight_min": -30.802770614624023,
      "activations/layer15_attention_weight_max": 36.44243621826172,
      "activations/layer15_attention_weight_min": -29.94293975830078,
      "activations/layer16_attention_weight_max": 39.001564025878906,
      "activations/layer16_attention_weight_min": -34.56511688232422,
      "activations/layer17_attention_weight_max": 60.90850830078125,
      "activations/layer17_attention_weight_min": -52.0683479309082,
      "activations/layer18_attention_weight_max": 57.42335891723633,
      "activations/layer18_attention_weight_min": -43.692527770996094,
      "activations/layer19_attention_weight_max": 25.441434860229492,
      "activations/layer19_attention_weight_min": -24.15493392944336,
      "activations/layer1_attention_weight_max": 15.846208572387695,
      "activations/layer1_attention_weight_min": -13.901318550109863,
      "activations/layer20_attention_weight_max": 26.226327896118164,
      "activations/layer20_attention_weight_min": -20.870147705078125,
      "activations/layer21_attention_weight_max": 41.8997917175293,
      "activations/layer21_attention_weight_min": -22.7833194732666,
      "activations/layer22_attention_weight_max": 36.60471725463867,
      "activations/layer22_attention_weight_min": -26.324365615844727,
      "activations/layer23_attention_weight_max": 46.57911682128906,
      "activations/layer23_attention_weight_min": -24.637325286865234,
      "activations/layer2_attention_weight_max": 32.100364685058594,
      "activations/layer2_attention_weight_min": -29.38784408569336,
      "activations/layer3_attention_weight_max": 83.37411499023438,
      "activations/layer3_attention_weight_min": -90.289306640625,
      "activations/layer4_attention_weight_max": 81.29429626464844,
      "activations/layer4_attention_weight_min": -81.85822296142578,
      "activations/layer5_attention_weight_max": 60.58330535888672,
      "activations/layer5_attention_weight_min": -75.62081909179688,
      "activations/layer6_attention_weight_max": 49.842674255371094,
      "activations/layer6_attention_weight_min": -53.90534591674805,
      "activations/layer7_attention_weight_max": 66.22240447998047,
      "activations/layer7_attention_weight_min": -63.826568603515625,
      "activations/layer8_attention_weight_max": 45.15660095214844,
      "activations/layer8_attention_weight_min": -47.66514587402344,
      "activations/layer9_attention_weight_max": 43.15776443481445,
      "activations/layer9_attention_weight_min": -46.40322494506836,
      "epoch": 18.56,
      "learning_rate": 3.05905303030303e-05,
      "loss": 2.7247,
      "step": 319500
    },
    {
      "activations/layer0_attention_weight_max": 15.361846923828125,
      "activations/layer0_attention_weight_min": -13.88311767578125,
      "activations/layer10_attention_weight_max": 36.64945983886719,
      "activations/layer10_attention_weight_min": -37.832820892333984,
      "activations/layer11_attention_weight_max": 32.81062698364258,
      "activations/layer11_attention_weight_min": -34.73474884033203,
      "activations/layer12_attention_weight_max": 20.178089141845703,
      "activations/layer12_attention_weight_min": -22.13266372680664,
      "activations/layer13_attention_weight_max": 37.05101013183594,
      "activations/layer13_attention_weight_min": -32.62323760986328,
      "activations/layer14_attention_weight_max": 38.54307556152344,
      "activations/layer14_attention_weight_min": -31.43600845336914,
      "activations/layer15_attention_weight_max": 36.43012237548828,
      "activations/layer15_attention_weight_min": -31.39220428466797,
      "activations/layer16_attention_weight_max": 35.69069290161133,
      "activations/layer16_attention_weight_min": -34.305301666259766,
      "activations/layer17_attention_weight_max": 56.37110137939453,
      "activations/layer17_attention_weight_min": -47.83195877075195,
      "activations/layer18_attention_weight_max": 55.5253791809082,
      "activations/layer18_attention_weight_min": -42.85884094238281,
      "activations/layer19_attention_weight_max": 28.32972526550293,
      "activations/layer19_attention_weight_min": -22.744062423706055,
      "activations/layer1_attention_weight_max": 16.322879791259766,
      "activations/layer1_attention_weight_min": -13.946917533874512,
      "activations/layer20_attention_weight_max": 26.6359920501709,
      "activations/layer20_attention_weight_min": -20.14527130126953,
      "activations/layer21_attention_weight_max": 38.839271545410156,
      "activations/layer21_attention_weight_min": -22.2439022064209,
      "activations/layer22_attention_weight_max": 34.95773696899414,
      "activations/layer22_attention_weight_min": -25.71636962890625,
      "activations/layer23_attention_weight_max": 40.05530548095703,
      "activations/layer23_attention_weight_min": -22.252958297729492,
      "activations/layer2_attention_weight_max": 32.24756622314453,
      "activations/layer2_attention_weight_min": -29.260908126831055,
      "activations/layer3_attention_weight_max": 87.26460266113281,
      "activations/layer3_attention_weight_min": -84.27813720703125,
      "activations/layer4_attention_weight_max": 86.21508026123047,
      "activations/layer4_attention_weight_min": -86.80245208740234,
      "activations/layer5_attention_weight_max": 66.0522232055664,
      "activations/layer5_attention_weight_min": -74.7654800415039,
      "activations/layer6_attention_weight_max": 51.161685943603516,
      "activations/layer6_attention_weight_min": -51.586307525634766,
      "activations/layer7_attention_weight_max": 65.99854278564453,
      "activations/layer7_attention_weight_min": -65.86482238769531,
      "activations/layer8_attention_weight_max": 49.0345344543457,
      "activations/layer8_attention_weight_min": -50.5723991394043,
      "activations/layer9_attention_weight_max": 45.030826568603516,
      "activations/layer9_attention_weight_min": -49.16496276855469,
      "epoch": 18.57,
      "learning_rate": 3.057159090909091e-05,
      "loss": 2.733,
      "step": 319550
    },
    {
      "activations/layer0_attention_weight_max": 15.091517448425293,
      "activations/layer0_attention_weight_min": -13.844057083129883,
      "activations/layer10_attention_weight_max": 36.58534622192383,
      "activations/layer10_attention_weight_min": -35.43534851074219,
      "activations/layer11_attention_weight_max": 34.84727096557617,
      "activations/layer11_attention_weight_min": -34.25428009033203,
      "activations/layer12_attention_weight_max": 19.39341163635254,
      "activations/layer12_attention_weight_min": -29.730756759643555,
      "activations/layer13_attention_weight_max": 43.04703903198242,
      "activations/layer13_attention_weight_min": -28.46338653564453,
      "activations/layer14_attention_weight_max": 34.31675338745117,
      "activations/layer14_attention_weight_min": -28.072673797607422,
      "activations/layer15_attention_weight_max": 31.60904312133789,
      "activations/layer15_attention_weight_min": -29.96001625061035,
      "activations/layer16_attention_weight_max": 33.600284576416016,
      "activations/layer16_attention_weight_min": -32.01105880737305,
      "activations/layer17_attention_weight_max": 52.59029769897461,
      "activations/layer17_attention_weight_min": -44.95531463623047,
      "activations/layer18_attention_weight_max": 46.77251052856445,
      "activations/layer18_attention_weight_min": -41.08176040649414,
      "activations/layer19_attention_weight_max": 25.92524528503418,
      "activations/layer19_attention_weight_min": -23.54695701599121,
      "activations/layer1_attention_weight_max": 15.267082214355469,
      "activations/layer1_attention_weight_min": -13.665958404541016,
      "activations/layer20_attention_weight_max": 21.73352813720703,
      "activations/layer20_attention_weight_min": -22.738998413085938,
      "activations/layer21_attention_weight_max": 39.07483673095703,
      "activations/layer21_attention_weight_min": -22.464170455932617,
      "activations/layer22_attention_weight_max": 29.26637840270996,
      "activations/layer22_attention_weight_min": -25.30971908569336,
      "activations/layer23_attention_weight_max": 34.411643981933594,
      "activations/layer23_attention_weight_min": -25.367727279663086,
      "activations/layer2_attention_weight_max": 33.7912483215332,
      "activations/layer2_attention_weight_min": -32.16088104248047,
      "activations/layer3_attention_weight_max": 91.04808807373047,
      "activations/layer3_attention_weight_min": -89.44139862060547,
      "activations/layer4_attention_weight_max": 86.0577163696289,
      "activations/layer4_attention_weight_min": -82.74766540527344,
      "activations/layer5_attention_weight_max": 67.3993148803711,
      "activations/layer5_attention_weight_min": -72.22802734375,
      "activations/layer6_attention_weight_max": 51.593833923339844,
      "activations/layer6_attention_weight_min": -50.678367614746094,
      "activations/layer7_attention_weight_max": 63.311485290527344,
      "activations/layer7_attention_weight_min": -70.4402084350586,
      "activations/layer8_attention_weight_max": 48.85392761230469,
      "activations/layer8_attention_weight_min": -47.298316955566406,
      "activations/layer9_attention_weight_max": 42.8418083190918,
      "activations/layer9_attention_weight_min": -45.55144119262695,
      "epoch": 18.57,
      "learning_rate": 3.0552651515151516e-05,
      "loss": 2.7227,
      "step": 319600
    },
    {
      "activations/layer0_attention_weight_max": 15.775480270385742,
      "activations/layer0_attention_weight_min": -13.607404708862305,
      "activations/layer10_attention_weight_max": 34.195037841796875,
      "activations/layer10_attention_weight_min": -34.268680572509766,
      "activations/layer11_attention_weight_max": 31.938865661621094,
      "activations/layer11_attention_weight_min": -32.536373138427734,
      "activations/layer12_attention_weight_max": 18.98752784729004,
      "activations/layer12_attention_weight_min": -24.793838500976562,
      "activations/layer13_attention_weight_max": 38.808815002441406,
      "activations/layer13_attention_weight_min": -26.89016342163086,
      "activations/layer14_attention_weight_max": 35.729915618896484,
      "activations/layer14_attention_weight_min": -30.73098373413086,
      "activations/layer15_attention_weight_max": 34.45348358154297,
      "activations/layer15_attention_weight_min": -29.700641632080078,
      "activations/layer16_attention_weight_max": 35.03565979003906,
      "activations/layer16_attention_weight_min": -32.46011734008789,
      "activations/layer17_attention_weight_max": 51.434017181396484,
      "activations/layer17_attention_weight_min": -43.43669891357422,
      "activations/layer18_attention_weight_max": 49.33388900756836,
      "activations/layer18_attention_weight_min": -39.27058792114258,
      "activations/layer19_attention_weight_max": 23.719240188598633,
      "activations/layer19_attention_weight_min": -21.977718353271484,
      "activations/layer1_attention_weight_max": 15.381246566772461,
      "activations/layer1_attention_weight_min": -13.942118644714355,
      "activations/layer20_attention_weight_max": 24.110448837280273,
      "activations/layer20_attention_weight_min": -20.76555633544922,
      "activations/layer21_attention_weight_max": 37.851783752441406,
      "activations/layer21_attention_weight_min": -22.99022102355957,
      "activations/layer22_attention_weight_max": 30.760183334350586,
      "activations/layer22_attention_weight_min": -25.34556007385254,
      "activations/layer23_attention_weight_max": 41.50890350341797,
      "activations/layer23_attention_weight_min": -25.721576690673828,
      "activations/layer2_attention_weight_max": 32.254852294921875,
      "activations/layer2_attention_weight_min": -30.033233642578125,
      "activations/layer3_attention_weight_max": 86.05528259277344,
      "activations/layer3_attention_weight_min": -85.68769836425781,
      "activations/layer4_attention_weight_max": 84.26333618164062,
      "activations/layer4_attention_weight_min": -86.0751953125,
      "activations/layer5_attention_weight_max": 63.500667572021484,
      "activations/layer5_attention_weight_min": -70.18113708496094,
      "activations/layer6_attention_weight_max": 48.78956985473633,
      "activations/layer6_attention_weight_min": -47.467689514160156,
      "activations/layer7_attention_weight_max": 66.09553527832031,
      "activations/layer7_attention_weight_min": -60.85671615600586,
      "activations/layer8_attention_weight_max": 44.974544525146484,
      "activations/layer8_attention_weight_min": -46.87373733520508,
      "activations/layer9_attention_weight_max": 44.31894302368164,
      "activations/layer9_attention_weight_min": -45.6432991027832,
      "epoch": 18.57,
      "learning_rate": 3.053371212121212e-05,
      "loss": 2.7454,
      "step": 319650
    },
    {
      "activations/layer0_attention_weight_max": 15.299327850341797,
      "activations/layer0_attention_weight_min": -13.826055526733398,
      "activations/layer10_attention_weight_max": 37.618751525878906,
      "activations/layer10_attention_weight_min": -35.586971282958984,
      "activations/layer11_attention_weight_max": 33.48341369628906,
      "activations/layer11_attention_weight_min": -34.34111022949219,
      "activations/layer12_attention_weight_max": 18.846149444580078,
      "activations/layer12_attention_weight_min": -24.275175094604492,
      "activations/layer13_attention_weight_max": 34.92075729370117,
      "activations/layer13_attention_weight_min": -26.726795196533203,
      "activations/layer14_attention_weight_max": 35.11576843261719,
      "activations/layer14_attention_weight_min": -29.727638244628906,
      "activations/layer15_attention_weight_max": 33.435733795166016,
      "activations/layer15_attention_weight_min": -28.970325469970703,
      "activations/layer16_attention_weight_max": 34.213134765625,
      "activations/layer16_attention_weight_min": -30.718936920166016,
      "activations/layer17_attention_weight_max": 52.51328659057617,
      "activations/layer17_attention_weight_min": -42.34688186645508,
      "activations/layer18_attention_weight_max": 47.6566276550293,
      "activations/layer18_attention_weight_min": -38.1301383972168,
      "activations/layer19_attention_weight_max": 25.094879150390625,
      "activations/layer19_attention_weight_min": -21.97657585144043,
      "activations/layer1_attention_weight_max": 15.828916549682617,
      "activations/layer1_attention_weight_min": -13.47630786895752,
      "activations/layer20_attention_weight_max": 23.919734954833984,
      "activations/layer20_attention_weight_min": -23.204132080078125,
      "activations/layer21_attention_weight_max": 35.61623001098633,
      "activations/layer21_attention_weight_min": -22.276790618896484,
      "activations/layer22_attention_weight_max": 29.701366424560547,
      "activations/layer22_attention_weight_min": -26.120025634765625,
      "activations/layer23_attention_weight_max": 35.713905334472656,
      "activations/layer23_attention_weight_min": -25.55738639831543,
      "activations/layer2_attention_weight_max": 32.09813690185547,
      "activations/layer2_attention_weight_min": -30.726909637451172,
      "activations/layer3_attention_weight_max": 86.36763763427734,
      "activations/layer3_attention_weight_min": -92.02722930908203,
      "activations/layer4_attention_weight_max": 84.42692565917969,
      "activations/layer4_attention_weight_min": -84.43560028076172,
      "activations/layer5_attention_weight_max": 61.861083984375,
      "activations/layer5_attention_weight_min": -72.34967803955078,
      "activations/layer6_attention_weight_max": 48.67461395263672,
      "activations/layer6_attention_weight_min": -49.0590934753418,
      "activations/layer7_attention_weight_max": 66.4482192993164,
      "activations/layer7_attention_weight_min": -62.571651458740234,
      "activations/layer8_attention_weight_max": 46.47917938232422,
      "activations/layer8_attention_weight_min": -47.413944244384766,
      "activations/layer9_attention_weight_max": 44.043785095214844,
      "activations/layer9_attention_weight_min": -44.09885787963867,
      "epoch": 18.58,
      "learning_rate": 3.0514772727272725e-05,
      "loss": 2.7347,
      "step": 319700
    },
    {
      "activations/layer0_attention_weight_max": 15.493813514709473,
      "activations/layer0_attention_weight_min": -13.731568336486816,
      "activations/layer10_attention_weight_max": 37.69828414916992,
      "activations/layer10_attention_weight_min": -36.98150634765625,
      "activations/layer11_attention_weight_max": 33.432594299316406,
      "activations/layer11_attention_weight_min": -35.80955505371094,
      "activations/layer12_attention_weight_max": 19.57659149169922,
      "activations/layer12_attention_weight_min": -23.742324829101562,
      "activations/layer13_attention_weight_max": 34.929847717285156,
      "activations/layer13_attention_weight_min": -30.688405990600586,
      "activations/layer14_attention_weight_max": 41.642215728759766,
      "activations/layer14_attention_weight_min": -31.70136833190918,
      "activations/layer15_attention_weight_max": 41.78835678100586,
      "activations/layer15_attention_weight_min": -31.160999298095703,
      "activations/layer16_attention_weight_max": 35.29563903808594,
      "activations/layer16_attention_weight_min": -34.82705307006836,
      "activations/layer17_attention_weight_max": 51.9282341003418,
      "activations/layer17_attention_weight_min": -51.1955680847168,
      "activations/layer18_attention_weight_max": 51.401161193847656,
      "activations/layer18_attention_weight_min": -47.58906936645508,
      "activations/layer19_attention_weight_max": 27.955713272094727,
      "activations/layer19_attention_weight_min": -26.614089965820312,
      "activations/layer1_attention_weight_max": 16.770769119262695,
      "activations/layer1_attention_weight_min": -14.163290023803711,
      "activations/layer20_attention_weight_max": 26.909116744995117,
      "activations/layer20_attention_weight_min": -22.672494888305664,
      "activations/layer21_attention_weight_max": 39.62335968017578,
      "activations/layer21_attention_weight_min": -25.08622169494629,
      "activations/layer22_attention_weight_max": 26.996185302734375,
      "activations/layer22_attention_weight_min": -27.563013076782227,
      "activations/layer23_attention_weight_max": 35.27500915527344,
      "activations/layer23_attention_weight_min": -26.54037094116211,
      "activations/layer2_attention_weight_max": 31.67084503173828,
      "activations/layer2_attention_weight_min": -31.508832931518555,
      "activations/layer3_attention_weight_max": 86.5995864868164,
      "activations/layer3_attention_weight_min": -92.22028350830078,
      "activations/layer4_attention_weight_max": 85.24282836914062,
      "activations/layer4_attention_weight_min": -84.76302337646484,
      "activations/layer5_attention_weight_max": 63.46173095703125,
      "activations/layer5_attention_weight_min": -72.52509307861328,
      "activations/layer6_attention_weight_max": 51.02664566040039,
      "activations/layer6_attention_weight_min": -51.54757308959961,
      "activations/layer7_attention_weight_max": 66.9953384399414,
      "activations/layer7_attention_weight_min": -66.62017059326172,
      "activations/layer8_attention_weight_max": 46.77959442138672,
      "activations/layer8_attention_weight_min": -50.51937484741211,
      "activations/layer9_attention_weight_max": 46.52531051635742,
      "activations/layer9_attention_weight_min": -46.92965316772461,
      "epoch": 18.58,
      "learning_rate": 3.049583333333333e-05,
      "loss": 2.7261,
      "step": 319750
    },
    {
      "activations/layer0_attention_weight_max": 16.06645393371582,
      "activations/layer0_attention_weight_min": -13.655890464782715,
      "activations/layer10_attention_weight_max": 37.11949920654297,
      "activations/layer10_attention_weight_min": -35.706539154052734,
      "activations/layer11_attention_weight_max": 34.13074493408203,
      "activations/layer11_attention_weight_min": -34.700584411621094,
      "activations/layer12_attention_weight_max": 18.26963233947754,
      "activations/layer12_attention_weight_min": -26.756132125854492,
      "activations/layer13_attention_weight_max": 36.0795783996582,
      "activations/layer13_attention_weight_min": -26.516347885131836,
      "activations/layer14_attention_weight_max": 32.52280044555664,
      "activations/layer14_attention_weight_min": -29.478412628173828,
      "activations/layer15_attention_weight_max": 31.232927322387695,
      "activations/layer15_attention_weight_min": -28.650405883789062,
      "activations/layer16_attention_weight_max": 29.962093353271484,
      "activations/layer16_attention_weight_min": -30.30143165588379,
      "activations/layer17_attention_weight_max": 48.92763900756836,
      "activations/layer17_attention_weight_min": -42.51373291015625,
      "activations/layer18_attention_weight_max": 45.60057067871094,
      "activations/layer18_attention_weight_min": -36.462459564208984,
      "activations/layer19_attention_weight_max": 24.44117546081543,
      "activations/layer19_attention_weight_min": -23.6042537689209,
      "activations/layer1_attention_weight_max": 16.5471248626709,
      "activations/layer1_attention_weight_min": -14.850665092468262,
      "activations/layer20_attention_weight_max": 24.950105667114258,
      "activations/layer20_attention_weight_min": -21.148056030273438,
      "activations/layer21_attention_weight_max": 32.84864807128906,
      "activations/layer21_attention_weight_min": -19.392343521118164,
      "activations/layer22_attention_weight_max": 28.66831398010254,
      "activations/layer22_attention_weight_min": -24.03483009338379,
      "activations/layer23_attention_weight_max": 34.218544006347656,
      "activations/layer23_attention_weight_min": -23.09252166748047,
      "activations/layer2_attention_weight_max": 32.906768798828125,
      "activations/layer2_attention_weight_min": -33.661624908447266,
      "activations/layer3_attention_weight_max": 89.09159851074219,
      "activations/layer3_attention_weight_min": -87.82170104980469,
      "activations/layer4_attention_weight_max": 84.36695861816406,
      "activations/layer4_attention_weight_min": -84.93999481201172,
      "activations/layer5_attention_weight_max": 63.11259078979492,
      "activations/layer5_attention_weight_min": -75.08193969726562,
      "activations/layer6_attention_weight_max": 47.95707702636719,
      "activations/layer6_attention_weight_min": -50.47419357299805,
      "activations/layer7_attention_weight_max": 65.32769775390625,
      "activations/layer7_attention_weight_min": -63.89576721191406,
      "activations/layer8_attention_weight_max": 44.447242736816406,
      "activations/layer8_attention_weight_min": -49.59144973754883,
      "activations/layer9_attention_weight_max": 43.55061340332031,
      "activations/layer9_attention_weight_min": -46.04638671875,
      "epoch": 18.58,
      "learning_rate": 3.047689393939394e-05,
      "loss": 2.7338,
      "step": 319800
    },
    {
      "activations/layer0_attention_weight_max": 14.713397026062012,
      "activations/layer0_attention_weight_min": -14.074954986572266,
      "activations/layer10_attention_weight_max": 36.6474609375,
      "activations/layer10_attention_weight_min": -36.044189453125,
      "activations/layer11_attention_weight_max": 34.18645477294922,
      "activations/layer11_attention_weight_min": -33.934139251708984,
      "activations/layer12_attention_weight_max": 18.836477279663086,
      "activations/layer12_attention_weight_min": -23.06206703186035,
      "activations/layer13_attention_weight_max": 31.331432342529297,
      "activations/layer13_attention_weight_min": -29.744117736816406,
      "activations/layer14_attention_weight_max": 34.9195556640625,
      "activations/layer14_attention_weight_min": -31.785640716552734,
      "activations/layer15_attention_weight_max": 35.38832473754883,
      "activations/layer15_attention_weight_min": -30.64227867126465,
      "activations/layer16_attention_weight_max": 33.221710205078125,
      "activations/layer16_attention_weight_min": -32.67759704589844,
      "activations/layer17_attention_weight_max": 50.74348449707031,
      "activations/layer17_attention_weight_min": -42.816097259521484,
      "activations/layer18_attention_weight_max": 48.69146728515625,
      "activations/layer18_attention_weight_min": -38.54918670654297,
      "activations/layer19_attention_weight_max": 24.29874610900879,
      "activations/layer19_attention_weight_min": -24.873952865600586,
      "activations/layer1_attention_weight_max": 18.447599411010742,
      "activations/layer1_attention_weight_min": -13.091628074645996,
      "activations/layer20_attention_weight_max": 23.852588653564453,
      "activations/layer20_attention_weight_min": -23.90435218811035,
      "activations/layer21_attention_weight_max": 35.585548400878906,
      "activations/layer21_attention_weight_min": -22.843229293823242,
      "activations/layer22_attention_weight_max": 28.55214500427246,
      "activations/layer22_attention_weight_min": -26.02029037475586,
      "activations/layer23_attention_weight_max": 37.6999397277832,
      "activations/layer23_attention_weight_min": -25.2991943359375,
      "activations/layer2_attention_weight_max": 32.13896560668945,
      "activations/layer2_attention_weight_min": -30.463605880737305,
      "activations/layer3_attention_weight_max": 88.5105972290039,
      "activations/layer3_attention_weight_min": -92.45236206054688,
      "activations/layer4_attention_weight_max": 83.47411346435547,
      "activations/layer4_attention_weight_min": -84.00765991210938,
      "activations/layer5_attention_weight_max": 60.23237228393555,
      "activations/layer5_attention_weight_min": -72.18009185791016,
      "activations/layer6_attention_weight_max": 50.647117614746094,
      "activations/layer6_attention_weight_min": -53.05583572387695,
      "activations/layer7_attention_weight_max": 68.86650085449219,
      "activations/layer7_attention_weight_min": -62.63883972167969,
      "activations/layer8_attention_weight_max": 46.3454704284668,
      "activations/layer8_attention_weight_min": -48.15980911254883,
      "activations/layer9_attention_weight_max": 44.049407958984375,
      "activations/layer9_attention_weight_min": -45.39252471923828,
      "epoch": 18.59,
      "learning_rate": 3.045795454545454e-05,
      "loss": 2.7222,
      "step": 319850
    },
    {
      "activations/layer0_attention_weight_max": 15.66811466217041,
      "activations/layer0_attention_weight_min": -14.133586883544922,
      "activations/layer10_attention_weight_max": 39.636207580566406,
      "activations/layer10_attention_weight_min": -39.09871292114258,
      "activations/layer11_attention_weight_max": 35.51048278808594,
      "activations/layer11_attention_weight_min": -38.34663009643555,
      "activations/layer12_attention_weight_max": 19.939905166625977,
      "activations/layer12_attention_weight_min": -24.560028076171875,
      "activations/layer13_attention_weight_max": 34.1759147644043,
      "activations/layer13_attention_weight_min": -31.311508178710938,
      "activations/layer14_attention_weight_max": 34.91162109375,
      "activations/layer14_attention_weight_min": -29.89830207824707,
      "activations/layer15_attention_weight_max": 33.267860412597656,
      "activations/layer15_attention_weight_min": -29.445289611816406,
      "activations/layer16_attention_weight_max": 33.452938079833984,
      "activations/layer16_attention_weight_min": -30.104442596435547,
      "activations/layer17_attention_weight_max": 52.39541244506836,
      "activations/layer17_attention_weight_min": -43.6901741027832,
      "activations/layer18_attention_weight_max": 48.90253829956055,
      "activations/layer18_attention_weight_min": -40.09288787841797,
      "activations/layer19_attention_weight_max": 26.766708374023438,
      "activations/layer19_attention_weight_min": -23.528593063354492,
      "activations/layer1_attention_weight_max": 15.2774019241333,
      "activations/layer1_attention_weight_min": -14.027769088745117,
      "activations/layer20_attention_weight_max": 26.116987228393555,
      "activations/layer20_attention_weight_min": -21.73759651184082,
      "activations/layer21_attention_weight_max": 40.865352630615234,
      "activations/layer21_attention_weight_min": -20.79613494873047,
      "activations/layer22_attention_weight_max": 34.462730407714844,
      "activations/layer22_attention_weight_min": -25.888837814331055,
      "activations/layer23_attention_weight_max": 36.87557601928711,
      "activations/layer23_attention_weight_min": -23.685379028320312,
      "activations/layer2_attention_weight_max": 35.126068115234375,
      "activations/layer2_attention_weight_min": -30.810626983642578,
      "activations/layer3_attention_weight_max": 91.46357727050781,
      "activations/layer3_attention_weight_min": -89.58496856689453,
      "activations/layer4_attention_weight_max": 90.5701675415039,
      "activations/layer4_attention_weight_min": -91.37118530273438,
      "activations/layer5_attention_weight_max": 68.94868469238281,
      "activations/layer5_attention_weight_min": -80.12646484375,
      "activations/layer6_attention_weight_max": 54.101863861083984,
      "activations/layer6_attention_weight_min": -53.90181350708008,
      "activations/layer7_attention_weight_max": 70.3360595703125,
      "activations/layer7_attention_weight_min": -64.01097869873047,
      "activations/layer8_attention_weight_max": 49.82838821411133,
      "activations/layer8_attention_weight_min": -56.50289535522461,
      "activations/layer9_attention_weight_max": 48.336639404296875,
      "activations/layer9_attention_weight_min": -51.05373764038086,
      "epoch": 18.59,
      "learning_rate": 3.043901515151515e-05,
      "loss": 2.7389,
      "step": 319900
    },
    {
      "activations/layer0_attention_weight_max": 15.047690391540527,
      "activations/layer0_attention_weight_min": -14.223319053649902,
      "activations/layer10_attention_weight_max": 36.79438400268555,
      "activations/layer10_attention_weight_min": -36.7503547668457,
      "activations/layer11_attention_weight_max": 33.862281799316406,
      "activations/layer11_attention_weight_min": -34.71935272216797,
      "activations/layer12_attention_weight_max": 20.251346588134766,
      "activations/layer12_attention_weight_min": -22.205480575561523,
      "activations/layer13_attention_weight_max": 35.25532531738281,
      "activations/layer13_attention_weight_min": -28.174043655395508,
      "activations/layer14_attention_weight_max": 38.34913635253906,
      "activations/layer14_attention_weight_min": -30.894075393676758,
      "activations/layer15_attention_weight_max": 34.53904724121094,
      "activations/layer15_attention_weight_min": -29.77333641052246,
      "activations/layer16_attention_weight_max": 34.93352127075195,
      "activations/layer16_attention_weight_min": -32.23889923095703,
      "activations/layer17_attention_weight_max": 50.71360778808594,
      "activations/layer17_attention_weight_min": -45.03141784667969,
      "activations/layer18_attention_weight_max": 45.85285568237305,
      "activations/layer18_attention_weight_min": -41.02973556518555,
      "activations/layer19_attention_weight_max": 22.32051658630371,
      "activations/layer19_attention_weight_min": -24.11216163635254,
      "activations/layer1_attention_weight_max": 15.455767631530762,
      "activations/layer1_attention_weight_min": -14.161215782165527,
      "activations/layer20_attention_weight_max": 20.627099990844727,
      "activations/layer20_attention_weight_min": -22.652864456176758,
      "activations/layer21_attention_weight_max": 36.370506286621094,
      "activations/layer21_attention_weight_min": -22.351884841918945,
      "activations/layer22_attention_weight_max": 30.725746154785156,
      "activations/layer22_attention_weight_min": -26.60796356201172,
      "activations/layer23_attention_weight_max": 34.195499420166016,
      "activations/layer23_attention_weight_min": -25.23080062866211,
      "activations/layer2_attention_weight_max": 30.286161422729492,
      "activations/layer2_attention_weight_min": -30.118488311767578,
      "activations/layer3_attention_weight_max": 85.43949890136719,
      "activations/layer3_attention_weight_min": -90.28758239746094,
      "activations/layer4_attention_weight_max": 84.48616790771484,
      "activations/layer4_attention_weight_min": -85.24810791015625,
      "activations/layer5_attention_weight_max": 60.74223327636719,
      "activations/layer5_attention_weight_min": -72.20330810546875,
      "activations/layer6_attention_weight_max": 49.70608139038086,
      "activations/layer6_attention_weight_min": -50.06486129760742,
      "activations/layer7_attention_weight_max": 61.9995002746582,
      "activations/layer7_attention_weight_min": -63.556068420410156,
      "activations/layer8_attention_weight_max": 46.17306900024414,
      "activations/layer8_attention_weight_min": -47.35603713989258,
      "activations/layer9_attention_weight_max": 45.59311294555664,
      "activations/layer9_attention_weight_min": -43.760066986083984,
      "epoch": 18.59,
      "learning_rate": 3.0420075757575757e-05,
      "loss": 2.743,
      "step": 319950
    },
    {
      "activations/layer0_attention_weight_max": 15.913649559020996,
      "activations/layer0_attention_weight_min": -13.875850677490234,
      "activations/layer10_attention_weight_max": 35.676361083984375,
      "activations/layer10_attention_weight_min": -36.225982666015625,
      "activations/layer11_attention_weight_max": 32.99813461303711,
      "activations/layer11_attention_weight_min": -35.62608337402344,
      "activations/layer12_attention_weight_max": 17.77927017211914,
      "activations/layer12_attention_weight_min": -24.025222778320312,
      "activations/layer13_attention_weight_max": 31.63585090637207,
      "activations/layer13_attention_weight_min": -29.381906509399414,
      "activations/layer14_attention_weight_max": 34.34096908569336,
      "activations/layer14_attention_weight_min": -29.572509765625,
      "activations/layer15_attention_weight_max": 29.72689437866211,
      "activations/layer15_attention_weight_min": -29.24302101135254,
      "activations/layer16_attention_weight_max": 30.455595016479492,
      "activations/layer16_attention_weight_min": -30.191865921020508,
      "activations/layer17_attention_weight_max": 49.27391052246094,
      "activations/layer17_attention_weight_min": -41.37627029418945,
      "activations/layer18_attention_weight_max": 44.1555290222168,
      "activations/layer18_attention_weight_min": -39.32582473754883,
      "activations/layer19_attention_weight_max": 22.71783447265625,
      "activations/layer19_attention_weight_min": -22.435638427734375,
      "activations/layer1_attention_weight_max": 15.531242370605469,
      "activations/layer1_attention_weight_min": -14.85185432434082,
      "activations/layer20_attention_weight_max": 20.609281539916992,
      "activations/layer20_attention_weight_min": -22.213045120239258,
      "activations/layer21_attention_weight_max": 31.00292205810547,
      "activations/layer21_attention_weight_min": -20.714950561523438,
      "activations/layer22_attention_weight_max": 27.175052642822266,
      "activations/layer22_attention_weight_min": -26.18597412109375,
      "activations/layer23_attention_weight_max": 35.41667175292969,
      "activations/layer23_attention_weight_min": -25.32611083984375,
      "activations/layer2_attention_weight_max": 31.903989791870117,
      "activations/layer2_attention_weight_min": -30.505298614501953,
      "activations/layer3_attention_weight_max": 85.22547149658203,
      "activations/layer3_attention_weight_min": -88.88088989257812,
      "activations/layer4_attention_weight_max": 81.57500457763672,
      "activations/layer4_attention_weight_min": -84.18180847167969,
      "activations/layer5_attention_weight_max": 62.83700180053711,
      "activations/layer5_attention_weight_min": -72.80389404296875,
      "activations/layer6_attention_weight_max": 51.64561462402344,
      "activations/layer6_attention_weight_min": -53.68208694458008,
      "activations/layer7_attention_weight_max": 64.34114837646484,
      "activations/layer7_attention_weight_min": -64.78414154052734,
      "activations/layer8_attention_weight_max": 45.375797271728516,
      "activations/layer8_attention_weight_min": -52.54316329956055,
      "activations/layer9_attention_weight_max": 44.233211517333984,
      "activations/layer9_attention_weight_min": -46.73567199707031,
      "epoch": 18.59,
      "learning_rate": 3.040113636363636e-05,
      "loss": 2.7136,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4488,
      "eval_samples_per_second": 508.238,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4488,
      "eval_openwebtext_samples_per_second": 508.238,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9617,
      "eval_wikitext_samples_per_second": 232.45,
      "step": 320000
    },
    {
      "epoch": 18.59,
      "eval_lambada_loss": 2.4375,
      "eval_lambada_ppl": 11.444393964331121,
      "eval_lambada_runtime": 9.5549,
      "eval_lambada_samples_per_second": 509.581,
      "step": 320000
    },
    {
      "activations/layer0_attention_weight_max": 14.939065933227539,
      "activations/layer0_attention_weight_min": -14.192155838012695,
      "activations/layer10_attention_weight_max": 39.48712921142578,
      "activations/layer10_attention_weight_min": -39.88678741455078,
      "activations/layer11_attention_weight_max": 38.19098663330078,
      "activations/layer11_attention_weight_min": -38.13080978393555,
      "activations/layer12_attention_weight_max": 20.482601165771484,
      "activations/layer12_attention_weight_min": -26.10594367980957,
      "activations/layer13_attention_weight_max": 50.576045989990234,
      "activations/layer13_attention_weight_min": -30.8992977142334,
      "activations/layer14_attention_weight_max": 48.47507095336914,
      "activations/layer14_attention_weight_min": -33.14299011230469,
      "activations/layer15_attention_weight_max": 46.31605529785156,
      "activations/layer15_attention_weight_min": -31.591323852539062,
      "activations/layer16_attention_weight_max": 40.252498626708984,
      "activations/layer16_attention_weight_min": -37.318336486816406,
      "activations/layer17_attention_weight_max": 64.38301849365234,
      "activations/layer17_attention_weight_min": -52.33320617675781,
      "activations/layer18_attention_weight_max": 63.513763427734375,
      "activations/layer18_attention_weight_min": -46.46718215942383,
      "activations/layer19_attention_weight_max": 29.14885902404785,
      "activations/layer19_attention_weight_min": -24.69894790649414,
      "activations/layer1_attention_weight_max": 15.97452163696289,
      "activations/layer1_attention_weight_min": -15.220815658569336,
      "activations/layer20_attention_weight_max": 30.53068733215332,
      "activations/layer20_attention_weight_min": -21.47051239013672,
      "activations/layer21_attention_weight_max": 50.86375427246094,
      "activations/layer21_attention_weight_min": -23.21525001525879,
      "activations/layer22_attention_weight_max": 35.64884948730469,
      "activations/layer22_attention_weight_min": -26.513778686523438,
      "activations/layer23_attention_weight_max": 44.214134216308594,
      "activations/layer23_attention_weight_min": -23.999710083007812,
      "activations/layer2_attention_weight_max": 32.2166862487793,
      "activations/layer2_attention_weight_min": -31.148008346557617,
      "activations/layer3_attention_weight_max": 88.17845153808594,
      "activations/layer3_attention_weight_min": -90.55840301513672,
      "activations/layer4_attention_weight_max": 87.52831268310547,
      "activations/layer4_attention_weight_min": -90.28148651123047,
      "activations/layer5_attention_weight_max": 66.97731018066406,
      "activations/layer5_attention_weight_min": -74.5507583618164,
      "activations/layer6_attention_weight_max": 51.3823356628418,
      "activations/layer6_attention_weight_min": -54.64052200317383,
      "activations/layer7_attention_weight_max": 70.10636901855469,
      "activations/layer7_attention_weight_min": -67.52810668945312,
      "activations/layer8_attention_weight_max": 50.898048400878906,
      "activations/layer8_attention_weight_min": -55.372562408447266,
      "activations/layer9_attention_weight_max": 47.72663497924805,
      "activations/layer9_attention_weight_min": -50.189964294433594,
      "epoch": 18.6,
      "learning_rate": 3.038219696969697e-05,
      "loss": 2.7291,
      "step": 320050
    },
    {
      "activations/layer0_attention_weight_max": 15.523398399353027,
      "activations/layer0_attention_weight_min": -14.230118751525879,
      "activations/layer10_attention_weight_max": 36.32930374145508,
      "activations/layer10_attention_weight_min": -35.52655792236328,
      "activations/layer11_attention_weight_max": 35.79580307006836,
      "activations/layer11_attention_weight_min": -35.25984191894531,
      "activations/layer12_attention_weight_max": 19.41042709350586,
      "activations/layer12_attention_weight_min": -22.297704696655273,
      "activations/layer13_attention_weight_max": 52.24680709838867,
      "activations/layer13_attention_weight_min": -26.29473304748535,
      "activations/layer14_attention_weight_max": 47.475250244140625,
      "activations/layer14_attention_weight_min": -30.23748779296875,
      "activations/layer15_attention_weight_max": 44.97575378417969,
      "activations/layer15_attention_weight_min": -30.231212615966797,
      "activations/layer16_attention_weight_max": 34.87528991699219,
      "activations/layer16_attention_weight_min": -32.937198638916016,
      "activations/layer17_attention_weight_max": 55.67660140991211,
      "activations/layer17_attention_weight_min": -48.48078918457031,
      "activations/layer18_attention_weight_max": 46.47882843017578,
      "activations/layer18_attention_weight_min": -41.74335479736328,
      "activations/layer19_attention_weight_max": 24.831418991088867,
      "activations/layer19_attention_weight_min": -21.842073440551758,
      "activations/layer1_attention_weight_max": 16.212688446044922,
      "activations/layer1_attention_weight_min": -13.537081718444824,
      "activations/layer20_attention_weight_max": 22.510032653808594,
      "activations/layer20_attention_weight_min": -23.615629196166992,
      "activations/layer21_attention_weight_max": 33.521751403808594,
      "activations/layer21_attention_weight_min": -24.313493728637695,
      "activations/layer22_attention_weight_max": 27.573368072509766,
      "activations/layer22_attention_weight_min": -25.845767974853516,
      "activations/layer23_attention_weight_max": 33.60773849487305,
      "activations/layer23_attention_weight_min": -25.71413803100586,
      "activations/layer2_attention_weight_max": 32.12017059326172,
      "activations/layer2_attention_weight_min": -29.87039566040039,
      "activations/layer3_attention_weight_max": 90.25228881835938,
      "activations/layer3_attention_weight_min": -86.58711242675781,
      "activations/layer4_attention_weight_max": 86.58431243896484,
      "activations/layer4_attention_weight_min": -83.85265350341797,
      "activations/layer5_attention_weight_max": 64.50212860107422,
      "activations/layer5_attention_weight_min": -71.15708923339844,
      "activations/layer6_attention_weight_max": 50.15165710449219,
      "activations/layer6_attention_weight_min": -53.25786209106445,
      "activations/layer7_attention_weight_max": 69.05293273925781,
      "activations/layer7_attention_weight_min": -65.25099182128906,
      "activations/layer8_attention_weight_max": 47.44895935058594,
      "activations/layer8_attention_weight_min": -48.4749755859375,
      "activations/layer9_attention_weight_max": 50.15395736694336,
      "activations/layer9_attention_weight_min": -46.38462448120117,
      "epoch": 18.6,
      "learning_rate": 3.036325757575757e-05,
      "loss": 2.7392,
      "step": 320100
    },
    {
      "activations/layer0_attention_weight_max": 15.375676155090332,
      "activations/layer0_attention_weight_min": -13.874534606933594,
      "activations/layer10_attention_weight_max": 34.7818489074707,
      "activations/layer10_attention_weight_min": -35.8100700378418,
      "activations/layer11_attention_weight_max": 32.25914001464844,
      "activations/layer11_attention_weight_min": -34.69244384765625,
      "activations/layer12_attention_weight_max": 19.71989631652832,
      "activations/layer12_attention_weight_min": -25.687559127807617,
      "activations/layer13_attention_weight_max": 32.60227584838867,
      "activations/layer13_attention_weight_min": -28.092973709106445,
      "activations/layer14_attention_weight_max": 37.002296447753906,
      "activations/layer14_attention_weight_min": -30.7878475189209,
      "activations/layer15_attention_weight_max": 34.520198822021484,
      "activations/layer15_attention_weight_min": -29.96401023864746,
      "activations/layer16_attention_weight_max": 33.67498016357422,
      "activations/layer16_attention_weight_min": -32.446510314941406,
      "activations/layer17_attention_weight_max": 52.79011154174805,
      "activations/layer17_attention_weight_min": -45.57233428955078,
      "activations/layer18_attention_weight_max": 48.03606033325195,
      "activations/layer18_attention_weight_min": -39.965999603271484,
      "activations/layer19_attention_weight_max": 24.759212493896484,
      "activations/layer19_attention_weight_min": -23.67763328552246,
      "activations/layer1_attention_weight_max": 16.59259605407715,
      "activations/layer1_attention_weight_min": -13.884859085083008,
      "activations/layer20_attention_weight_max": 26.82017707824707,
      "activations/layer20_attention_weight_min": -19.745882034301758,
      "activations/layer21_attention_weight_max": 37.50585174560547,
      "activations/layer21_attention_weight_min": -22.011205673217773,
      "activations/layer22_attention_weight_max": 31.70208740234375,
      "activations/layer22_attention_weight_min": -24.928546905517578,
      "activations/layer23_attention_weight_max": 37.34127426147461,
      "activations/layer23_attention_weight_min": -24.00221061706543,
      "activations/layer2_attention_weight_max": 31.71265411376953,
      "activations/layer2_attention_weight_min": -30.44237518310547,
      "activations/layer3_attention_weight_max": 90.44549560546875,
      "activations/layer3_attention_weight_min": -92.75359344482422,
      "activations/layer4_attention_weight_max": 83.77642822265625,
      "activations/layer4_attention_weight_min": -84.88518524169922,
      "activations/layer5_attention_weight_max": 62.75004577636719,
      "activations/layer5_attention_weight_min": -74.7177505493164,
      "activations/layer6_attention_weight_max": 51.2191276550293,
      "activations/layer6_attention_weight_min": -50.86937713623047,
      "activations/layer7_attention_weight_max": 66.63233184814453,
      "activations/layer7_attention_weight_min": -66.75469207763672,
      "activations/layer8_attention_weight_max": 47.15269088745117,
      "activations/layer8_attention_weight_min": -48.73611068725586,
      "activations/layer9_attention_weight_max": 43.368160247802734,
      "activations/layer9_attention_weight_min": -45.44993209838867,
      "epoch": 18.6,
      "learning_rate": 3.034431818181818e-05,
      "loss": 2.7271,
      "step": 320150
    },
    {
      "activations/layer0_attention_weight_max": 15.163195610046387,
      "activations/layer0_attention_weight_min": -13.92546272277832,
      "activations/layer10_attention_weight_max": 36.30667495727539,
      "activations/layer10_attention_weight_min": -34.92557907104492,
      "activations/layer11_attention_weight_max": 32.46738052368164,
      "activations/layer11_attention_weight_min": -34.48982238769531,
      "activations/layer12_attention_weight_max": 18.66428565979004,
      "activations/layer12_attention_weight_min": -23.511322021484375,
      "activations/layer13_attention_weight_max": 34.152584075927734,
      "activations/layer13_attention_weight_min": -26.124814987182617,
      "activations/layer14_attention_weight_max": 36.11745834350586,
      "activations/layer14_attention_weight_min": -27.781225204467773,
      "activations/layer15_attention_weight_max": 33.52266311645508,
      "activations/layer15_attention_weight_min": -29.697126388549805,
      "activations/layer16_attention_weight_max": 32.556209564208984,
      "activations/layer16_attention_weight_min": -29.16094398498535,
      "activations/layer17_attention_weight_max": 47.45615005493164,
      "activations/layer17_attention_weight_min": -41.751747131347656,
      "activations/layer18_attention_weight_max": 43.735450744628906,
      "activations/layer18_attention_weight_min": -37.95777893066406,
      "activations/layer19_attention_weight_max": 23.153474807739258,
      "activations/layer19_attention_weight_min": -20.9948787689209,
      "activations/layer1_attention_weight_max": 15.411396026611328,
      "activations/layer1_attention_weight_min": -13.986477851867676,
      "activations/layer20_attention_weight_max": 20.10729217529297,
      "activations/layer20_attention_weight_min": -23.79276466369629,
      "activations/layer21_attention_weight_max": 38.33045959472656,
      "activations/layer21_attention_weight_min": -21.164077758789062,
      "activations/layer22_attention_weight_max": 26.409564971923828,
      "activations/layer22_attention_weight_min": -23.260406494140625,
      "activations/layer23_attention_weight_max": 34.431724548339844,
      "activations/layer23_attention_weight_min": -21.951871871948242,
      "activations/layer2_attention_weight_max": 30.612829208374023,
      "activations/layer2_attention_weight_min": -30.707387924194336,
      "activations/layer3_attention_weight_max": 85.86042022705078,
      "activations/layer3_attention_weight_min": -93.1511459350586,
      "activations/layer4_attention_weight_max": 82.00047302246094,
      "activations/layer4_attention_weight_min": -87.94554901123047,
      "activations/layer5_attention_weight_max": 64.52581024169922,
      "activations/layer5_attention_weight_min": -74.3082504272461,
      "activations/layer6_attention_weight_max": 48.735904693603516,
      "activations/layer6_attention_weight_min": -49.22412872314453,
      "activations/layer7_attention_weight_max": 63.985050201416016,
      "activations/layer7_attention_weight_min": -65.78829956054688,
      "activations/layer8_attention_weight_max": 45.27454376220703,
      "activations/layer8_attention_weight_min": -48.17724609375,
      "activations/layer9_attention_weight_max": 43.45579528808594,
      "activations/layer9_attention_weight_min": -44.542457580566406,
      "epoch": 18.61,
      "learning_rate": 3.0325378787878784e-05,
      "loss": 2.7084,
      "step": 320200
    },
    {
      "activations/layer0_attention_weight_max": 15.16806697845459,
      "activations/layer0_attention_weight_min": -13.997864723205566,
      "activations/layer10_attention_weight_max": 37.997833251953125,
      "activations/layer10_attention_weight_min": -35.8611946105957,
      "activations/layer11_attention_weight_max": 33.7813720703125,
      "activations/layer11_attention_weight_min": -34.40191650390625,
      "activations/layer12_attention_weight_max": 21.987018585205078,
      "activations/layer12_attention_weight_min": -23.014759063720703,
      "activations/layer13_attention_weight_max": 36.35493850708008,
      "activations/layer13_attention_weight_min": -28.17388343811035,
      "activations/layer14_attention_weight_max": 35.643150329589844,
      "activations/layer14_attention_weight_min": -29.1715145111084,
      "activations/layer15_attention_weight_max": 33.520572662353516,
      "activations/layer15_attention_weight_min": -29.753984451293945,
      "activations/layer16_attention_weight_max": 32.59870147705078,
      "activations/layer16_attention_weight_min": -30.317264556884766,
      "activations/layer17_attention_weight_max": 51.88593673706055,
      "activations/layer17_attention_weight_min": -48.43111801147461,
      "activations/layer18_attention_weight_max": 46.84920120239258,
      "activations/layer18_attention_weight_min": -42.0014533996582,
      "activations/layer19_attention_weight_max": 23.431631088256836,
      "activations/layer19_attention_weight_min": -22.293094635009766,
      "activations/layer1_attention_weight_max": 16.027616500854492,
      "activations/layer1_attention_weight_min": -13.343046188354492,
      "activations/layer20_attention_weight_max": 23.169279098510742,
      "activations/layer20_attention_weight_min": -21.090038299560547,
      "activations/layer21_attention_weight_max": 35.86335372924805,
      "activations/layer21_attention_weight_min": -24.35113525390625,
      "activations/layer22_attention_weight_max": 27.968978881835938,
      "activations/layer22_attention_weight_min": -25.879995346069336,
      "activations/layer23_attention_weight_max": 35.8919792175293,
      "activations/layer23_attention_weight_min": -23.978168487548828,
      "activations/layer2_attention_weight_max": 31.559965133666992,
      "activations/layer2_attention_weight_min": -29.086597442626953,
      "activations/layer3_attention_weight_max": 88.50692749023438,
      "activations/layer3_attention_weight_min": -86.45722961425781,
      "activations/layer4_attention_weight_max": 86.34764099121094,
      "activations/layer4_attention_weight_min": -85.14128875732422,
      "activations/layer5_attention_weight_max": 65.24992370605469,
      "activations/layer5_attention_weight_min": -78.7577133178711,
      "activations/layer6_attention_weight_max": 50.4348030090332,
      "activations/layer6_attention_weight_min": -50.98644256591797,
      "activations/layer7_attention_weight_max": 67.341064453125,
      "activations/layer7_attention_weight_min": -64.57276916503906,
      "activations/layer8_attention_weight_max": 49.112796783447266,
      "activations/layer8_attention_weight_min": -53.967987060546875,
      "activations/layer9_attention_weight_max": 45.787776947021484,
      "activations/layer9_attention_weight_min": -46.93840408325195,
      "epoch": 18.61,
      "learning_rate": 3.0306439393939393e-05,
      "loss": 2.7186,
      "step": 320250
    },
    {
      "activations/layer0_attention_weight_max": 14.761734962463379,
      "activations/layer0_attention_weight_min": -13.714215278625488,
      "activations/layer10_attention_weight_max": 35.54085159301758,
      "activations/layer10_attention_weight_min": -33.670631408691406,
      "activations/layer11_attention_weight_max": 32.165260314941406,
      "activations/layer11_attention_weight_min": -33.19957733154297,
      "activations/layer12_attention_weight_max": 19.31028175354004,
      "activations/layer12_attention_weight_min": -24.763668060302734,
      "activations/layer13_attention_weight_max": 33.19294357299805,
      "activations/layer13_attention_weight_min": -26.485368728637695,
      "activations/layer14_attention_weight_max": 36.11205291748047,
      "activations/layer14_attention_weight_min": -31.938676834106445,
      "activations/layer15_attention_weight_max": 33.39554977416992,
      "activations/layer15_attention_weight_min": -31.403581619262695,
      "activations/layer16_attention_weight_max": 32.736106872558594,
      "activations/layer16_attention_weight_min": -32.03300476074219,
      "activations/layer17_attention_weight_max": 49.6201057434082,
      "activations/layer17_attention_weight_min": -43.774620056152344,
      "activations/layer18_attention_weight_max": 45.94340515136719,
      "activations/layer18_attention_weight_min": -38.39734649658203,
      "activations/layer19_attention_weight_max": 23.2917537689209,
      "activations/layer19_attention_weight_min": -21.22139549255371,
      "activations/layer1_attention_weight_max": 15.542449951171875,
      "activations/layer1_attention_weight_min": -14.403369903564453,
      "activations/layer20_attention_weight_max": 23.123966217041016,
      "activations/layer20_attention_weight_min": -19.516599655151367,
      "activations/layer21_attention_weight_max": 30.60574722290039,
      "activations/layer21_attention_weight_min": -20.446914672851562,
      "activations/layer22_attention_weight_max": 28.068729400634766,
      "activations/layer22_attention_weight_min": -25.318315505981445,
      "activations/layer23_attention_weight_max": 36.819480895996094,
      "activations/layer23_attention_weight_min": -23.259464263916016,
      "activations/layer2_attention_weight_max": 31.431575775146484,
      "activations/layer2_attention_weight_min": -29.130964279174805,
      "activations/layer3_attention_weight_max": 83.5771484375,
      "activations/layer3_attention_weight_min": -80.83423614501953,
      "activations/layer4_attention_weight_max": 79.9708480834961,
      "activations/layer4_attention_weight_min": -79.87239837646484,
      "activations/layer5_attention_weight_max": 60.30864715576172,
      "activations/layer5_attention_weight_min": -70.47819519042969,
      "activations/layer6_attention_weight_max": 48.92372131347656,
      "activations/layer6_attention_weight_min": -48.23122024536133,
      "activations/layer7_attention_weight_max": 63.77839660644531,
      "activations/layer7_attention_weight_min": -62.4316520690918,
      "activations/layer8_attention_weight_max": 46.24635696411133,
      "activations/layer8_attention_weight_min": -48.43556594848633,
      "activations/layer9_attention_weight_max": 44.30330276489258,
      "activations/layer9_attention_weight_min": -43.15880584716797,
      "epoch": 18.61,
      "learning_rate": 3.0287499999999998e-05,
      "loss": 2.7296,
      "step": 320300
    },
    {
      "activations/layer0_attention_weight_max": 15.734155654907227,
      "activations/layer0_attention_weight_min": -13.87527847290039,
      "activations/layer10_attention_weight_max": 33.506649017333984,
      "activations/layer10_attention_weight_min": -34.270294189453125,
      "activations/layer11_attention_weight_max": 31.01434898376465,
      "activations/layer11_attention_weight_min": -33.513450622558594,
      "activations/layer12_attention_weight_max": 18.415647506713867,
      "activations/layer12_attention_weight_min": -25.075708389282227,
      "activations/layer13_attention_weight_max": 32.598045349121094,
      "activations/layer13_attention_weight_min": -25.046812057495117,
      "activations/layer14_attention_weight_max": 33.470298767089844,
      "activations/layer14_attention_weight_min": -29.77755355834961,
      "activations/layer15_attention_weight_max": 30.417795181274414,
      "activations/layer15_attention_weight_min": -30.421432495117188,
      "activations/layer16_attention_weight_max": 32.221553802490234,
      "activations/layer16_attention_weight_min": -32.303401947021484,
      "activations/layer17_attention_weight_max": 49.101348876953125,
      "activations/layer17_attention_weight_min": -45.107120513916016,
      "activations/layer18_attention_weight_max": 43.73223114013672,
      "activations/layer18_attention_weight_min": -39.0857048034668,
      "activations/layer19_attention_weight_max": 22.113218307495117,
      "activations/layer19_attention_weight_min": -22.444007873535156,
      "activations/layer1_attention_weight_max": 16.4877872467041,
      "activations/layer1_attention_weight_min": -14.090619087219238,
      "activations/layer20_attention_weight_max": 21.61802864074707,
      "activations/layer20_attention_weight_min": -21.322233200073242,
      "activations/layer21_attention_weight_max": 29.31902313232422,
      "activations/layer21_attention_weight_min": -20.192304611206055,
      "activations/layer22_attention_weight_max": 29.85693359375,
      "activations/layer22_attention_weight_min": -24.932472229003906,
      "activations/layer23_attention_weight_max": 33.815940856933594,
      "activations/layer23_attention_weight_min": -25.390819549560547,
      "activations/layer2_attention_weight_max": 30.71583366394043,
      "activations/layer2_attention_weight_min": -29.906156539916992,
      "activations/layer3_attention_weight_max": 86.56312561035156,
      "activations/layer3_attention_weight_min": -86.40121459960938,
      "activations/layer4_attention_weight_max": 82.93670654296875,
      "activations/layer4_attention_weight_min": -84.95877838134766,
      "activations/layer5_attention_weight_max": 59.76463317871094,
      "activations/layer5_attention_weight_min": -74.9380111694336,
      "activations/layer6_attention_weight_max": 48.1287727355957,
      "activations/layer6_attention_weight_min": -48.299888610839844,
      "activations/layer7_attention_weight_max": 62.8436393737793,
      "activations/layer7_attention_weight_min": -64.93706512451172,
      "activations/layer8_attention_weight_max": 43.78889083862305,
      "activations/layer8_attention_weight_min": -47.75980758666992,
      "activations/layer9_attention_weight_max": 40.42524337768555,
      "activations/layer9_attention_weight_min": -44.79484558105469,
      "epoch": 18.61,
      "learning_rate": 3.0268560606060606e-05,
      "loss": 2.7331,
      "step": 320350
    },
    {
      "activations/layer0_attention_weight_max": 15.392809867858887,
      "activations/layer0_attention_weight_min": -13.839134216308594,
      "activations/layer10_attention_weight_max": 36.371639251708984,
      "activations/layer10_attention_weight_min": -36.350528717041016,
      "activations/layer11_attention_weight_max": 33.48394775390625,
      "activations/layer11_attention_weight_min": -34.830631256103516,
      "activations/layer12_attention_weight_max": 19.527202606201172,
      "activations/layer12_attention_weight_min": -24.873376846313477,
      "activations/layer13_attention_weight_max": 36.703033447265625,
      "activations/layer13_attention_weight_min": -28.987438201904297,
      "activations/layer14_attention_weight_max": 36.67538070678711,
      "activations/layer14_attention_weight_min": -30.688796997070312,
      "activations/layer15_attention_weight_max": 33.29237747192383,
      "activations/layer15_attention_weight_min": -28.9608211517334,
      "activations/layer16_attention_weight_max": 35.24873352050781,
      "activations/layer16_attention_weight_min": -32.49011993408203,
      "activations/layer17_attention_weight_max": 55.0938720703125,
      "activations/layer17_attention_weight_min": -46.20553207397461,
      "activations/layer18_attention_weight_max": 46.63958740234375,
      "activations/layer18_attention_weight_min": -41.338191986083984,
      "activations/layer19_attention_weight_max": 23.446149826049805,
      "activations/layer19_attention_weight_min": -23.11134910583496,
      "activations/layer1_attention_weight_max": 15.044356346130371,
      "activations/layer1_attention_weight_min": -13.343710899353027,
      "activations/layer20_attention_weight_max": 23.776649475097656,
      "activations/layer20_attention_weight_min": -21.943330764770508,
      "activations/layer21_attention_weight_max": 38.39759826660156,
      "activations/layer21_attention_weight_min": -21.4473819732666,
      "activations/layer22_attention_weight_max": 30.792619705200195,
      "activations/layer22_attention_weight_min": -27.55413818359375,
      "activations/layer23_attention_weight_max": 37.986732482910156,
      "activations/layer23_attention_weight_min": -24.542104721069336,
      "activations/layer2_attention_weight_max": 31.472259521484375,
      "activations/layer2_attention_weight_min": -28.874465942382812,
      "activations/layer3_attention_weight_max": 90.86457061767578,
      "activations/layer3_attention_weight_min": -86.2315444946289,
      "activations/layer4_attention_weight_max": 85.34722137451172,
      "activations/layer4_attention_weight_min": -79.70812225341797,
      "activations/layer5_attention_weight_max": 66.55973815917969,
      "activations/layer5_attention_weight_min": -72.6346435546875,
      "activations/layer6_attention_weight_max": 54.91050720214844,
      "activations/layer6_attention_weight_min": -50.806983947753906,
      "activations/layer7_attention_weight_max": 64.70962524414062,
      "activations/layer7_attention_weight_min": -62.129207611083984,
      "activations/layer8_attention_weight_max": 46.28590393066406,
      "activations/layer8_attention_weight_min": -48.9433479309082,
      "activations/layer9_attention_weight_max": 43.24645233154297,
      "activations/layer9_attention_weight_min": -44.77079772949219,
      "epoch": 18.62,
      "learning_rate": 3.0249999999999997e-05,
      "loss": 2.7251,
      "step": 320400
    },
    {
      "activations/layer0_attention_weight_max": 15.81506061553955,
      "activations/layer0_attention_weight_min": -13.624552726745605,
      "activations/layer10_attention_weight_max": 34.44203186035156,
      "activations/layer10_attention_weight_min": -37.57844543457031,
      "activations/layer11_attention_weight_max": 34.66712188720703,
      "activations/layer11_attention_weight_min": -37.333030700683594,
      "activations/layer12_attention_weight_max": 18.75594139099121,
      "activations/layer12_attention_weight_min": -26.236974716186523,
      "activations/layer13_attention_weight_max": 35.10441207885742,
      "activations/layer13_attention_weight_min": -29.567285537719727,
      "activations/layer14_attention_weight_max": 33.27740478515625,
      "activations/layer14_attention_weight_min": -29.292034149169922,
      "activations/layer15_attention_weight_max": 31.922754287719727,
      "activations/layer15_attention_weight_min": -30.744152069091797,
      "activations/layer16_attention_weight_max": 32.46370315551758,
      "activations/layer16_attention_weight_min": -32.409786224365234,
      "activations/layer17_attention_weight_max": 49.348724365234375,
      "activations/layer17_attention_weight_min": -44.27791213989258,
      "activations/layer18_attention_weight_max": 46.21229934692383,
      "activations/layer18_attention_weight_min": -37.19492721557617,
      "activations/layer19_attention_weight_max": 22.511951446533203,
      "activations/layer19_attention_weight_min": -22.9370174407959,
      "activations/layer1_attention_weight_max": 15.532641410827637,
      "activations/layer1_attention_weight_min": -13.756736755371094,
      "activations/layer20_attention_weight_max": 21.792579650878906,
      "activations/layer20_attention_weight_min": -22.0120906829834,
      "activations/layer21_attention_weight_max": 35.42448425292969,
      "activations/layer21_attention_weight_min": -21.934757232666016,
      "activations/layer22_attention_weight_max": 26.086017608642578,
      "activations/layer22_attention_weight_min": -25.698612213134766,
      "activations/layer23_attention_weight_max": 36.08821105957031,
      "activations/layer23_attention_weight_min": -24.040409088134766,
      "activations/layer2_attention_weight_max": 33.70069122314453,
      "activations/layer2_attention_weight_min": -29.992141723632812,
      "activations/layer3_attention_weight_max": 91.16545867919922,
      "activations/layer3_attention_weight_min": -91.36537170410156,
      "activations/layer4_attention_weight_max": 86.75655364990234,
      "activations/layer4_attention_weight_min": -85.24688720703125,
      "activations/layer5_attention_weight_max": 65.07515716552734,
      "activations/layer5_attention_weight_min": -76.60314178466797,
      "activations/layer6_attention_weight_max": 50.859371185302734,
      "activations/layer6_attention_weight_min": -56.53227996826172,
      "activations/layer7_attention_weight_max": 67.14215850830078,
      "activations/layer7_attention_weight_min": -63.43912887573242,
      "activations/layer8_attention_weight_max": 45.26338577270508,
      "activations/layer8_attention_weight_min": -47.652217864990234,
      "activations/layer9_attention_weight_max": 42.96235656738281,
      "activations/layer9_attention_weight_min": -45.69120788574219,
      "epoch": 18.62,
      "learning_rate": 3.0231060606060605e-05,
      "loss": 2.7304,
      "step": 320450
    },
    {
      "activations/layer0_attention_weight_max": 15.691993713378906,
      "activations/layer0_attention_weight_min": -14.109091758728027,
      "activations/layer10_attention_weight_max": 40.19424057006836,
      "activations/layer10_attention_weight_min": -37.757240295410156,
      "activations/layer11_attention_weight_max": 37.64460754394531,
      "activations/layer11_attention_weight_min": -34.576446533203125,
      "activations/layer12_attention_weight_max": 19.041759490966797,
      "activations/layer12_attention_weight_min": -25.7290096282959,
      "activations/layer13_attention_weight_max": 40.87227249145508,
      "activations/layer13_attention_weight_min": -31.325164794921875,
      "activations/layer14_attention_weight_max": 37.46390151977539,
      "activations/layer14_attention_weight_min": -32.159263610839844,
      "activations/layer15_attention_weight_max": 37.19643783569336,
      "activations/layer15_attention_weight_min": -31.81140899658203,
      "activations/layer16_attention_weight_max": 36.157691955566406,
      "activations/layer16_attention_weight_min": -33.73893737792969,
      "activations/layer17_attention_weight_max": 57.00575637817383,
      "activations/layer17_attention_weight_min": -47.4505500793457,
      "activations/layer18_attention_weight_max": 53.41819381713867,
      "activations/layer18_attention_weight_min": -41.81788635253906,
      "activations/layer19_attention_weight_max": 27.87288475036621,
      "activations/layer19_attention_weight_min": -21.65888023376465,
      "activations/layer1_attention_weight_max": 16.726455688476562,
      "activations/layer1_attention_weight_min": -14.352147102355957,
      "activations/layer20_attention_weight_max": 23.98755645751953,
      "activations/layer20_attention_weight_min": -23.738924026489258,
      "activations/layer21_attention_weight_max": 39.40325927734375,
      "activations/layer21_attention_weight_min": -23.40265464782715,
      "activations/layer22_attention_weight_max": 31.83670997619629,
      "activations/layer22_attention_weight_min": -26.616039276123047,
      "activations/layer23_attention_weight_max": 39.279624938964844,
      "activations/layer23_attention_weight_min": -22.961353302001953,
      "activations/layer2_attention_weight_max": 33.02183151245117,
      "activations/layer2_attention_weight_min": -30.442564010620117,
      "activations/layer3_attention_weight_max": 90.56649017333984,
      "activations/layer3_attention_weight_min": -88.25492858886719,
      "activations/layer4_attention_weight_max": 84.11053466796875,
      "activations/layer4_attention_weight_min": -83.55534362792969,
      "activations/layer5_attention_weight_max": 62.386348724365234,
      "activations/layer5_attention_weight_min": -70.71125030517578,
      "activations/layer6_attention_weight_max": 51.91194534301758,
      "activations/layer6_attention_weight_min": -53.46696853637695,
      "activations/layer7_attention_weight_max": 66.47307586669922,
      "activations/layer7_attention_weight_min": -66.24832153320312,
      "activations/layer8_attention_weight_max": 50.17530822753906,
      "activations/layer8_attention_weight_min": -54.08308410644531,
      "activations/layer9_attention_weight_max": 51.16286849975586,
      "activations/layer9_attention_weight_min": -48.411041259765625,
      "epoch": 18.62,
      "learning_rate": 3.021212121212121e-05,
      "loss": 2.7474,
      "step": 320500
    },
    {
      "activations/layer0_attention_weight_max": 15.556760787963867,
      "activations/layer0_attention_weight_min": -13.82271671295166,
      "activations/layer10_attention_weight_max": 37.9290771484375,
      "activations/layer10_attention_weight_min": -37.600982666015625,
      "activations/layer11_attention_weight_max": 32.96232986450195,
      "activations/layer11_attention_weight_min": -36.48945617675781,
      "activations/layer12_attention_weight_max": 19.080888748168945,
      "activations/layer12_attention_weight_min": -23.953094482421875,
      "activations/layer13_attention_weight_max": 36.19113540649414,
      "activations/layer13_attention_weight_min": -26.449909210205078,
      "activations/layer14_attention_weight_max": 38.7500114440918,
      "activations/layer14_attention_weight_min": -31.99282455444336,
      "activations/layer15_attention_weight_max": 34.69541549682617,
      "activations/layer15_attention_weight_min": -32.23362350463867,
      "activations/layer16_attention_weight_max": 38.65280532836914,
      "activations/layer16_attention_weight_min": -37.950870513916016,
      "activations/layer17_attention_weight_max": 56.553497314453125,
      "activations/layer17_attention_weight_min": -53.933998107910156,
      "activations/layer18_attention_weight_max": 51.593685150146484,
      "activations/layer18_attention_weight_min": -49.41963577270508,
      "activations/layer19_attention_weight_max": 26.63711929321289,
      "activations/layer19_attention_weight_min": -28.353759765625,
      "activations/layer1_attention_weight_max": 16.00303840637207,
      "activations/layer1_attention_weight_min": -14.256856918334961,
      "activations/layer20_attention_weight_max": 25.123811721801758,
      "activations/layer20_attention_weight_min": -25.16242027282715,
      "activations/layer21_attention_weight_max": 38.65431213378906,
      "activations/layer21_attention_weight_min": -26.831905364990234,
      "activations/layer22_attention_weight_max": 29.33321189880371,
      "activations/layer22_attention_weight_min": -27.9222412109375,
      "activations/layer23_attention_weight_max": 35.255802154541016,
      "activations/layer23_attention_weight_min": -29.155433654785156,
      "activations/layer2_attention_weight_max": 31.957523345947266,
      "activations/layer2_attention_weight_min": -29.61309814453125,
      "activations/layer3_attention_weight_max": 84.1744384765625,
      "activations/layer3_attention_weight_min": -86.31869506835938,
      "activations/layer4_attention_weight_max": 80.36201477050781,
      "activations/layer4_attention_weight_min": -84.90776824951172,
      "activations/layer5_attention_weight_max": 59.34190368652344,
      "activations/layer5_attention_weight_min": -70.23627471923828,
      "activations/layer6_attention_weight_max": 49.7661018371582,
      "activations/layer6_attention_weight_min": -51.022911071777344,
      "activations/layer7_attention_weight_max": 66.43267059326172,
      "activations/layer7_attention_weight_min": -66.0789566040039,
      "activations/layer8_attention_weight_max": 46.04117965698242,
      "activations/layer8_attention_weight_min": -50.18302917480469,
      "activations/layer9_attention_weight_max": 43.547542572021484,
      "activations/layer9_attention_weight_min": -47.33906173706055,
      "epoch": 18.63,
      "learning_rate": 3.0193181818181815e-05,
      "loss": 2.7378,
      "step": 320550
    },
    {
      "activations/layer0_attention_weight_max": 14.69092845916748,
      "activations/layer0_attention_weight_min": -13.720959663391113,
      "activations/layer10_attention_weight_max": 39.09980010986328,
      "activations/layer10_attention_weight_min": -37.08465576171875,
      "activations/layer11_attention_weight_max": 33.810089111328125,
      "activations/layer11_attention_weight_min": -34.8154411315918,
      "activations/layer12_attention_weight_max": 17.953540802001953,
      "activations/layer12_attention_weight_min": -26.568830490112305,
      "activations/layer13_attention_weight_max": 32.98760986328125,
      "activations/layer13_attention_weight_min": -31.730003356933594,
      "activations/layer14_attention_weight_max": 34.75434875488281,
      "activations/layer14_attention_weight_min": -31.5914306640625,
      "activations/layer15_attention_weight_max": 33.12467575073242,
      "activations/layer15_attention_weight_min": -31.623559951782227,
      "activations/layer16_attention_weight_max": 35.16594314575195,
      "activations/layer16_attention_weight_min": -34.54551315307617,
      "activations/layer17_attention_weight_max": 53.086402893066406,
      "activations/layer17_attention_weight_min": -48.69275665283203,
      "activations/layer18_attention_weight_max": 48.2693977355957,
      "activations/layer18_attention_weight_min": -45.38996505737305,
      "activations/layer19_attention_weight_max": 24.95250129699707,
      "activations/layer19_attention_weight_min": -23.663204193115234,
      "activations/layer1_attention_weight_max": 15.283089637756348,
      "activations/layer1_attention_weight_min": -12.15346908569336,
      "activations/layer20_attention_weight_max": 22.014312744140625,
      "activations/layer20_attention_weight_min": -22.03659439086914,
      "activations/layer21_attention_weight_max": 34.58510208129883,
      "activations/layer21_attention_weight_min": -24.7761287689209,
      "activations/layer22_attention_weight_max": 28.702863693237305,
      "activations/layer22_attention_weight_min": -26.230894088745117,
      "activations/layer23_attention_weight_max": 34.35639572143555,
      "activations/layer23_attention_weight_min": -24.52560043334961,
      "activations/layer2_attention_weight_max": 31.598806381225586,
      "activations/layer2_attention_weight_min": -31.361507415771484,
      "activations/layer3_attention_weight_max": 81.21890258789062,
      "activations/layer3_attention_weight_min": -81.37609100341797,
      "activations/layer4_attention_weight_max": 81.71892547607422,
      "activations/layer4_attention_weight_min": -76.89646911621094,
      "activations/layer5_attention_weight_max": 60.31780242919922,
      "activations/layer5_attention_weight_min": -68.15670776367188,
      "activations/layer6_attention_weight_max": 47.57393264770508,
      "activations/layer6_attention_weight_min": -46.974918365478516,
      "activations/layer7_attention_weight_max": 68.34843444824219,
      "activations/layer7_attention_weight_min": -60.077781677246094,
      "activations/layer8_attention_weight_max": 46.1954231262207,
      "activations/layer8_attention_weight_min": -47.46919631958008,
      "activations/layer9_attention_weight_max": 43.12382507324219,
      "activations/layer9_attention_weight_min": -44.49235153198242,
      "epoch": 18.63,
      "learning_rate": 3.017424242424242e-05,
      "loss": 2.7333,
      "step": 320600
    },
    {
      "activations/layer0_attention_weight_max": 15.619200706481934,
      "activations/layer0_attention_weight_min": -13.691198348999023,
      "activations/layer10_attention_weight_max": 35.54762649536133,
      "activations/layer10_attention_weight_min": -37.0860595703125,
      "activations/layer11_attention_weight_max": 31.786296844482422,
      "activations/layer11_attention_weight_min": -34.67884826660156,
      "activations/layer12_attention_weight_max": 18.687063217163086,
      "activations/layer12_attention_weight_min": -29.524229049682617,
      "activations/layer13_attention_weight_max": 31.29926109313965,
      "activations/layer13_attention_weight_min": -28.607240676879883,
      "activations/layer14_attention_weight_max": 34.655452728271484,
      "activations/layer14_attention_weight_min": -30.656042098999023,
      "activations/layer15_attention_weight_max": 33.63515090942383,
      "activations/layer15_attention_weight_min": -32.27668762207031,
      "activations/layer16_attention_weight_max": 32.93222427368164,
      "activations/layer16_attention_weight_min": -32.54653549194336,
      "activations/layer17_attention_weight_max": 50.30775833129883,
      "activations/layer17_attention_weight_min": -47.67421340942383,
      "activations/layer18_attention_weight_max": 47.75273513793945,
      "activations/layer18_attention_weight_min": -42.574676513671875,
      "activations/layer19_attention_weight_max": 24.721967697143555,
      "activations/layer19_attention_weight_min": -24.294034957885742,
      "activations/layer1_attention_weight_max": 16.92269515991211,
      "activations/layer1_attention_weight_min": -14.535420417785645,
      "activations/layer20_attention_weight_max": 22.636455535888672,
      "activations/layer20_attention_weight_min": -24.021739959716797,
      "activations/layer21_attention_weight_max": 32.60053253173828,
      "activations/layer21_attention_weight_min": -25.288955688476562,
      "activations/layer22_attention_weight_max": 32.40980911254883,
      "activations/layer22_attention_weight_min": -25.46234893798828,
      "activations/layer23_attention_weight_max": 36.474884033203125,
      "activations/layer23_attention_weight_min": -23.533924102783203,
      "activations/layer2_attention_weight_max": 33.7447509765625,
      "activations/layer2_attention_weight_min": -33.33245086669922,
      "activations/layer3_attention_weight_max": 84.47512817382812,
      "activations/layer3_attention_weight_min": -80.62431335449219,
      "activations/layer4_attention_weight_max": 82.26602172851562,
      "activations/layer4_attention_weight_min": -79.26878356933594,
      "activations/layer5_attention_weight_max": 60.06357955932617,
      "activations/layer5_attention_weight_min": -73.24748229980469,
      "activations/layer6_attention_weight_max": 48.8353157043457,
      "activations/layer6_attention_weight_min": -50.727787017822266,
      "activations/layer7_attention_weight_max": 59.77217102050781,
      "activations/layer7_attention_weight_min": -61.272369384765625,
      "activations/layer8_attention_weight_max": 45.000343322753906,
      "activations/layer8_attention_weight_min": -48.68474197387695,
      "activations/layer9_attention_weight_max": 42.968902587890625,
      "activations/layer9_attention_weight_min": -44.43410873413086,
      "epoch": 18.63,
      "learning_rate": 3.0155303030303028e-05,
      "loss": 2.7382,
      "step": 320650
    },
    {
      "activations/layer0_attention_weight_max": 14.874935150146484,
      "activations/layer0_attention_weight_min": -13.731144905090332,
      "activations/layer10_attention_weight_max": 37.82819366455078,
      "activations/layer10_attention_weight_min": -38.4345703125,
      "activations/layer11_attention_weight_max": 33.26216506958008,
      "activations/layer11_attention_weight_min": -34.49498748779297,
      "activations/layer12_attention_weight_max": 18.562332153320312,
      "activations/layer12_attention_weight_min": -20.524404525756836,
      "activations/layer13_attention_weight_max": 36.98330307006836,
      "activations/layer13_attention_weight_min": -26.391414642333984,
      "activations/layer14_attention_weight_max": 36.18080139160156,
      "activations/layer14_attention_weight_min": -27.80545997619629,
      "activations/layer15_attention_weight_max": 33.740760803222656,
      "activations/layer15_attention_weight_min": -28.48596954345703,
      "activations/layer16_attention_weight_max": 34.61664962768555,
      "activations/layer16_attention_weight_min": -30.840417861938477,
      "activations/layer17_attention_weight_max": 51.64145278930664,
      "activations/layer17_attention_weight_min": -42.153446197509766,
      "activations/layer18_attention_weight_max": 51.058013916015625,
      "activations/layer18_attention_weight_min": -38.21168518066406,
      "activations/layer19_attention_weight_max": 27.242334365844727,
      "activations/layer19_attention_weight_min": -23.4610652923584,
      "activations/layer1_attention_weight_max": 15.702685356140137,
      "activations/layer1_attention_weight_min": -13.605759620666504,
      "activations/layer20_attention_weight_max": 24.687747955322266,
      "activations/layer20_attention_weight_min": -22.501659393310547,
      "activations/layer21_attention_weight_max": 35.13190841674805,
      "activations/layer21_attention_weight_min": -22.349828720092773,
      "activations/layer22_attention_weight_max": 28.279022216796875,
      "activations/layer22_attention_weight_min": -25.298046112060547,
      "activations/layer23_attention_weight_max": 38.0472412109375,
      "activations/layer23_attention_weight_min": -23.221525192260742,
      "activations/layer2_attention_weight_max": 31.18128204345703,
      "activations/layer2_attention_weight_min": -31.348936080932617,
      "activations/layer3_attention_weight_max": 86.0132827758789,
      "activations/layer3_attention_weight_min": -88.25493621826172,
      "activations/layer4_attention_weight_max": 83.94731903076172,
      "activations/layer4_attention_weight_min": -83.68132019042969,
      "activations/layer5_attention_weight_max": 65.34803771972656,
      "activations/layer5_attention_weight_min": -70.31748962402344,
      "activations/layer6_attention_weight_max": 48.30901336669922,
      "activations/layer6_attention_weight_min": -51.98562240600586,
      "activations/layer7_attention_weight_max": 70.02456665039062,
      "activations/layer7_attention_weight_min": -63.836238861083984,
      "activations/layer8_attention_weight_max": 46.8316535949707,
      "activations/layer8_attention_weight_min": -51.92499542236328,
      "activations/layer9_attention_weight_max": 45.06080627441406,
      "activations/layer9_attention_weight_min": -46.3709831237793,
      "epoch": 18.63,
      "learning_rate": 3.0136363636363633e-05,
      "loss": 2.7424,
      "step": 320700
    },
    {
      "activations/layer0_attention_weight_max": 16.526729583740234,
      "activations/layer0_attention_weight_min": -14.028996467590332,
      "activations/layer10_attention_weight_max": 35.938838958740234,
      "activations/layer10_attention_weight_min": -37.0196533203125,
      "activations/layer11_attention_weight_max": 32.5599365234375,
      "activations/layer11_attention_weight_min": -35.51839828491211,
      "activations/layer12_attention_weight_max": 18.840349197387695,
      "activations/layer12_attention_weight_min": -27.006559371948242,
      "activations/layer13_attention_weight_max": 33.30784606933594,
      "activations/layer13_attention_weight_min": -27.727500915527344,
      "activations/layer14_attention_weight_max": 36.877071380615234,
      "activations/layer14_attention_weight_min": -30.598804473876953,
      "activations/layer15_attention_weight_max": 34.267024993896484,
      "activations/layer15_attention_weight_min": -31.2213134765625,
      "activations/layer16_attention_weight_max": 33.75571823120117,
      "activations/layer16_attention_weight_min": -31.950712203979492,
      "activations/layer17_attention_weight_max": 56.3665657043457,
      "activations/layer17_attention_weight_min": -45.99699401855469,
      "activations/layer18_attention_weight_max": 51.03696060180664,
      "activations/layer18_attention_weight_min": -41.036678314208984,
      "activations/layer19_attention_weight_max": 26.246475219726562,
      "activations/layer19_attention_weight_min": -23.77520179748535,
      "activations/layer1_attention_weight_max": 16.399301528930664,
      "activations/layer1_attention_weight_min": -13.04807186126709,
      "activations/layer20_attention_weight_max": 24.847339630126953,
      "activations/layer20_attention_weight_min": -22.044437408447266,
      "activations/layer21_attention_weight_max": 37.128265380859375,
      "activations/layer21_attention_weight_min": -22.9112548828125,
      "activations/layer22_attention_weight_max": 30.0714168548584,
      "activations/layer22_attention_weight_min": -24.900768280029297,
      "activations/layer23_attention_weight_max": 37.03712844848633,
      "activations/layer23_attention_weight_min": -23.23421859741211,
      "activations/layer2_attention_weight_max": 30.487823486328125,
      "activations/layer2_attention_weight_min": -28.541608810424805,
      "activations/layer3_attention_weight_max": 86.52912902832031,
      "activations/layer3_attention_weight_min": -91.56053161621094,
      "activations/layer4_attention_weight_max": 78.23197174072266,
      "activations/layer4_attention_weight_min": -80.04898834228516,
      "activations/layer5_attention_weight_max": 62.071468353271484,
      "activations/layer5_attention_weight_min": -74.04814147949219,
      "activations/layer6_attention_weight_max": 48.73759460449219,
      "activations/layer6_attention_weight_min": -51.73392105102539,
      "activations/layer7_attention_weight_max": 65.57193756103516,
      "activations/layer7_attention_weight_min": -64.67185974121094,
      "activations/layer8_attention_weight_max": 49.88959503173828,
      "activations/layer8_attention_weight_min": -55.06988525390625,
      "activations/layer9_attention_weight_max": 43.25147247314453,
      "activations/layer9_attention_weight_min": -47.401302337646484,
      "epoch": 18.64,
      "learning_rate": 3.011742424242424e-05,
      "loss": 2.7181,
      "step": 320750
    },
    {
      "activations/layer0_attention_weight_max": 15.446234703063965,
      "activations/layer0_attention_weight_min": -13.65774917602539,
      "activations/layer10_attention_weight_max": 36.93062210083008,
      "activations/layer10_attention_weight_min": -35.14556884765625,
      "activations/layer11_attention_weight_max": 34.65985870361328,
      "activations/layer11_attention_weight_min": -36.853214263916016,
      "activations/layer12_attention_weight_max": 19.317523956298828,
      "activations/layer12_attention_weight_min": -22.1400146484375,
      "activations/layer13_attention_weight_max": 36.58510208129883,
      "activations/layer13_attention_weight_min": -27.69517707824707,
      "activations/layer14_attention_weight_max": 37.31110763549805,
      "activations/layer14_attention_weight_min": -29.266382217407227,
      "activations/layer15_attention_weight_max": 35.18376541137695,
      "activations/layer15_attention_weight_min": -29.484804153442383,
      "activations/layer16_attention_weight_max": 35.861549377441406,
      "activations/layer16_attention_weight_min": -33.911319732666016,
      "activations/layer17_attention_weight_max": 55.617095947265625,
      "activations/layer17_attention_weight_min": -51.8723258972168,
      "activations/layer18_attention_weight_max": 51.597877502441406,
      "activations/layer18_attention_weight_min": -47.42836380004883,
      "activations/layer19_attention_weight_max": 26.621042251586914,
      "activations/layer19_attention_weight_min": -25.99139976501465,
      "activations/layer1_attention_weight_max": 16.438173294067383,
      "activations/layer1_attention_weight_min": -13.209604263305664,
      "activations/layer20_attention_weight_max": 26.027929306030273,
      "activations/layer20_attention_weight_min": -22.123525619506836,
      "activations/layer21_attention_weight_max": 39.23003005981445,
      "activations/layer21_attention_weight_min": -24.249603271484375,
      "activations/layer22_attention_weight_max": 31.444379806518555,
      "activations/layer22_attention_weight_min": -25.655927658081055,
      "activations/layer23_attention_weight_max": 39.54832458496094,
      "activations/layer23_attention_weight_min": -23.850181579589844,
      "activations/layer2_attention_weight_max": 28.850543975830078,
      "activations/layer2_attention_weight_min": -28.360944747924805,
      "activations/layer3_attention_weight_max": 81.88634490966797,
      "activations/layer3_attention_weight_min": -82.81610107421875,
      "activations/layer4_attention_weight_max": 84.1862564086914,
      "activations/layer4_attention_weight_min": -79.13167572021484,
      "activations/layer5_attention_weight_max": 64.52975463867188,
      "activations/layer5_attention_weight_min": -73.09744262695312,
      "activations/layer6_attention_weight_max": 48.336448669433594,
      "activations/layer6_attention_weight_min": -48.37316131591797,
      "activations/layer7_attention_weight_max": 67.55956268310547,
      "activations/layer7_attention_weight_min": -61.58803176879883,
      "activations/layer8_attention_weight_max": 46.23501968383789,
      "activations/layer8_attention_weight_min": -51.369415283203125,
      "activations/layer9_attention_weight_max": 45.44938278198242,
      "activations/layer9_attention_weight_min": -46.11558151245117,
      "epoch": 18.64,
      "learning_rate": 3.0098484848484843e-05,
      "loss": 2.7375,
      "step": 320800
    },
    {
      "activations/layer0_attention_weight_max": 15.661982536315918,
      "activations/layer0_attention_weight_min": -13.914942741394043,
      "activations/layer10_attention_weight_max": 38.331024169921875,
      "activations/layer10_attention_weight_min": -37.22722244262695,
      "activations/layer11_attention_weight_max": 33.83677291870117,
      "activations/layer11_attention_weight_min": -33.92557907104492,
      "activations/layer12_attention_weight_max": 19.32258415222168,
      "activations/layer12_attention_weight_min": -26.252344131469727,
      "activations/layer13_attention_weight_max": 34.873863220214844,
      "activations/layer13_attention_weight_min": -29.260087966918945,
      "activations/layer14_attention_weight_max": 35.63239288330078,
      "activations/layer14_attention_weight_min": -29.523380279541016,
      "activations/layer15_attention_weight_max": 31.243614196777344,
      "activations/layer15_attention_weight_min": -29.997081756591797,
      "activations/layer16_attention_weight_max": 33.193172454833984,
      "activations/layer16_attention_weight_min": -31.95013427734375,
      "activations/layer17_attention_weight_max": 53.15059280395508,
      "activations/layer17_attention_weight_min": -46.42024230957031,
      "activations/layer18_attention_weight_max": 46.36912536621094,
      "activations/layer18_attention_weight_min": -41.88672637939453,
      "activations/layer19_attention_weight_max": 24.60218620300293,
      "activations/layer19_attention_weight_min": -22.481271743774414,
      "activations/layer1_attention_weight_max": 18.13336181640625,
      "activations/layer1_attention_weight_min": -16.3289737701416,
      "activations/layer20_attention_weight_max": 25.029233932495117,
      "activations/layer20_attention_weight_min": -22.112632751464844,
      "activations/layer21_attention_weight_max": 37.968685150146484,
      "activations/layer21_attention_weight_min": -21.99860191345215,
      "activations/layer22_attention_weight_max": 28.32141876220703,
      "activations/layer22_attention_weight_min": -25.906770706176758,
      "activations/layer23_attention_weight_max": 37.59739685058594,
      "activations/layer23_attention_weight_min": -25.506376266479492,
      "activations/layer2_attention_weight_max": 32.304351806640625,
      "activations/layer2_attention_weight_min": -28.534404754638672,
      "activations/layer3_attention_weight_max": 86.71615600585938,
      "activations/layer3_attention_weight_min": -90.1396484375,
      "activations/layer4_attention_weight_max": 84.93540954589844,
      "activations/layer4_attention_weight_min": -83.4746322631836,
      "activations/layer5_attention_weight_max": 62.19245910644531,
      "activations/layer5_attention_weight_min": -69.651611328125,
      "activations/layer6_attention_weight_max": 51.52644348144531,
      "activations/layer6_attention_weight_min": -49.45327377319336,
      "activations/layer7_attention_weight_max": 63.788963317871094,
      "activations/layer7_attention_weight_min": -61.403255462646484,
      "activations/layer8_attention_weight_max": 48.95635223388672,
      "activations/layer8_attention_weight_min": -49.87870407104492,
      "activations/layer9_attention_weight_max": 44.48635482788086,
      "activations/layer9_attention_weight_min": -46.305450439453125,
      "epoch": 18.64,
      "learning_rate": 3.007954545454545e-05,
      "loss": 2.7199,
      "step": 320850
    },
    {
      "activations/layer0_attention_weight_max": 15.323373794555664,
      "activations/layer0_attention_weight_min": -14.039068222045898,
      "activations/layer10_attention_weight_max": 36.4541130065918,
      "activations/layer10_attention_weight_min": -35.33610534667969,
      "activations/layer11_attention_weight_max": 32.98394012451172,
      "activations/layer11_attention_weight_min": -33.96820068359375,
      "activations/layer12_attention_weight_max": 18.9472599029541,
      "activations/layer12_attention_weight_min": -29.489606857299805,
      "activations/layer13_attention_weight_max": 35.48941421508789,
      "activations/layer13_attention_weight_min": -25.92536735534668,
      "activations/layer14_attention_weight_max": 35.5106086730957,
      "activations/layer14_attention_weight_min": -29.96162986755371,
      "activations/layer15_attention_weight_max": 33.27133560180664,
      "activations/layer15_attention_weight_min": -31.22071647644043,
      "activations/layer16_attention_weight_max": 35.0409049987793,
      "activations/layer16_attention_weight_min": -31.888259887695312,
      "activations/layer17_attention_weight_max": 53.93916702270508,
      "activations/layer17_attention_weight_min": -43.71745681762695,
      "activations/layer18_attention_weight_max": 51.34048843383789,
      "activations/layer18_attention_weight_min": -39.69281768798828,
      "activations/layer19_attention_weight_max": 25.446767807006836,
      "activations/layer19_attention_weight_min": -24.942365646362305,
      "activations/layer1_attention_weight_max": 16.005712509155273,
      "activations/layer1_attention_weight_min": -14.207864761352539,
      "activations/layer20_attention_weight_max": 22.525930404663086,
      "activations/layer20_attention_weight_min": -21.829307556152344,
      "activations/layer21_attention_weight_max": 32.69245910644531,
      "activations/layer21_attention_weight_min": -22.142526626586914,
      "activations/layer22_attention_weight_max": 29.957565307617188,
      "activations/layer22_attention_weight_min": -24.72534942626953,
      "activations/layer23_attention_weight_max": 35.967979431152344,
      "activations/layer23_attention_weight_min": -25.003036499023438,
      "activations/layer2_attention_weight_max": 30.36656951904297,
      "activations/layer2_attention_weight_min": -29.18515396118164,
      "activations/layer3_attention_weight_max": 83.99711608886719,
      "activations/layer3_attention_weight_min": -84.38780975341797,
      "activations/layer4_attention_weight_max": 84.2606430053711,
      "activations/layer4_attention_weight_min": -80.53413391113281,
      "activations/layer5_attention_weight_max": 66.89903259277344,
      "activations/layer5_attention_weight_min": -73.94221496582031,
      "activations/layer6_attention_weight_max": 50.80605697631836,
      "activations/layer6_attention_weight_min": -49.65695571899414,
      "activations/layer7_attention_weight_max": 69.52214813232422,
      "activations/layer7_attention_weight_min": -66.20838165283203,
      "activations/layer8_attention_weight_max": 48.70909118652344,
      "activations/layer8_attention_weight_min": -51.1353759765625,
      "activations/layer9_attention_weight_max": 45.14334487915039,
      "activations/layer9_attention_weight_min": -44.941253662109375,
      "epoch": 18.65,
      "learning_rate": 3.006060606060606e-05,
      "loss": 2.7363,
      "step": 320900
    },
    {
      "activations/layer0_attention_weight_max": 15.036811828613281,
      "activations/layer0_attention_weight_min": -13.802399635314941,
      "activations/layer10_attention_weight_max": 37.53961944580078,
      "activations/layer10_attention_weight_min": -36.17767333984375,
      "activations/layer11_attention_weight_max": 33.878482818603516,
      "activations/layer11_attention_weight_min": -33.91123580932617,
      "activations/layer12_attention_weight_max": 17.74462890625,
      "activations/layer12_attention_weight_min": -25.094146728515625,
      "activations/layer13_attention_weight_max": 38.90691375732422,
      "activations/layer13_attention_weight_min": -35.90951156616211,
      "activations/layer14_attention_weight_max": 35.61110305786133,
      "activations/layer14_attention_weight_min": -34.99335479736328,
      "activations/layer15_attention_weight_max": 32.61296463012695,
      "activations/layer15_attention_weight_min": -30.041833877563477,
      "activations/layer16_attention_weight_max": 33.83515548706055,
      "activations/layer16_attention_weight_min": -32.12483215332031,
      "activations/layer17_attention_weight_max": 48.84968948364258,
      "activations/layer17_attention_weight_min": -44.0495491027832,
      "activations/layer18_attention_weight_max": 45.421417236328125,
      "activations/layer18_attention_weight_min": -39.06316375732422,
      "activations/layer19_attention_weight_max": 26.94232177734375,
      "activations/layer19_attention_weight_min": -24.526607513427734,
      "activations/layer1_attention_weight_max": 15.994850158691406,
      "activations/layer1_attention_weight_min": -16.349300384521484,
      "activations/layer20_attention_weight_max": 24.125383377075195,
      "activations/layer20_attention_weight_min": -24.056392669677734,
      "activations/layer21_attention_weight_max": 38.567325592041016,
      "activations/layer21_attention_weight_min": -23.624242782592773,
      "activations/layer22_attention_weight_max": 30.29473876953125,
      "activations/layer22_attention_weight_min": -28.09632682800293,
      "activations/layer23_attention_weight_max": 38.13129425048828,
      "activations/layer23_attention_weight_min": -23.752681732177734,
      "activations/layer2_attention_weight_max": 33.76677703857422,
      "activations/layer2_attention_weight_min": -31.05609130859375,
      "activations/layer3_attention_weight_max": 84.69520568847656,
      "activations/layer3_attention_weight_min": -82.41703033447266,
      "activations/layer4_attention_weight_max": 84.17261505126953,
      "activations/layer4_attention_weight_min": -79.52080535888672,
      "activations/layer5_attention_weight_max": 62.279537200927734,
      "activations/layer5_attention_weight_min": -69.32743072509766,
      "activations/layer6_attention_weight_max": 50.784385681152344,
      "activations/layer6_attention_weight_min": -50.79412078857422,
      "activations/layer7_attention_weight_max": 69.35384368896484,
      "activations/layer7_attention_weight_min": -64.58954620361328,
      "activations/layer8_attention_weight_max": 47.79088592529297,
      "activations/layer8_attention_weight_min": -50.622432708740234,
      "activations/layer9_attention_weight_max": 45.59016799926758,
      "activations/layer9_attention_weight_min": -44.906795501708984,
      "epoch": 18.65,
      "learning_rate": 3.0041666666666664e-05,
      "loss": 2.735,
      "step": 320950
    },
    {
      "activations/layer0_attention_weight_max": 15.166881561279297,
      "activations/layer0_attention_weight_min": -13.626012802124023,
      "activations/layer10_attention_weight_max": 39.113040924072266,
      "activations/layer10_attention_weight_min": -37.91465377807617,
      "activations/layer11_attention_weight_max": 35.357154846191406,
      "activations/layer11_attention_weight_min": -34.680450439453125,
      "activations/layer12_attention_weight_max": 19.031452178955078,
      "activations/layer12_attention_weight_min": -23.40656280517578,
      "activations/layer13_attention_weight_max": 34.89759826660156,
      "activations/layer13_attention_weight_min": -28.412979125976562,
      "activations/layer14_attention_weight_max": 34.36970520019531,
      "activations/layer14_attention_weight_min": -30.08634376525879,
      "activations/layer15_attention_weight_max": 32.366214752197266,
      "activations/layer15_attention_weight_min": -31.6552791595459,
      "activations/layer16_attention_weight_max": 33.05632400512695,
      "activations/layer16_attention_weight_min": -30.94346809387207,
      "activations/layer17_attention_weight_max": 48.8988037109375,
      "activations/layer17_attention_weight_min": -44.07646179199219,
      "activations/layer18_attention_weight_max": 47.55258560180664,
      "activations/layer18_attention_weight_min": -41.74074172973633,
      "activations/layer19_attention_weight_max": 24.745574951171875,
      "activations/layer19_attention_weight_min": -23.584692001342773,
      "activations/layer1_attention_weight_max": 16.168996810913086,
      "activations/layer1_attention_weight_min": -14.511316299438477,
      "activations/layer20_attention_weight_max": 22.681676864624023,
      "activations/layer20_attention_weight_min": -22.45575714111328,
      "activations/layer21_attention_weight_max": 33.50408172607422,
      "activations/layer21_attention_weight_min": -22.540681838989258,
      "activations/layer22_attention_weight_max": 28.499603271484375,
      "activations/layer22_attention_weight_min": -25.382932662963867,
      "activations/layer23_attention_weight_max": 37.94727325439453,
      "activations/layer23_attention_weight_min": -22.81629753112793,
      "activations/layer2_attention_weight_max": 32.98039245605469,
      "activations/layer2_attention_weight_min": -32.0888671875,
      "activations/layer3_attention_weight_max": 93.02486419677734,
      "activations/layer3_attention_weight_min": -87.71322631835938,
      "activations/layer4_attention_weight_max": 91.10782623291016,
      "activations/layer4_attention_weight_min": -85.8006591796875,
      "activations/layer5_attention_weight_max": 61.90796661376953,
      "activations/layer5_attention_weight_min": -71.77720642089844,
      "activations/layer6_attention_weight_max": 52.89289093017578,
      "activations/layer6_attention_weight_min": -53.399635314941406,
      "activations/layer7_attention_weight_max": 65.90532684326172,
      "activations/layer7_attention_weight_min": -62.422908782958984,
      "activations/layer8_attention_weight_max": 46.8447151184082,
      "activations/layer8_attention_weight_min": -48.567378997802734,
      "activations/layer9_attention_weight_max": 43.8116455078125,
      "activations/layer9_attention_weight_min": -44.36116027832031,
      "epoch": 18.65,
      "learning_rate": 3.0022727272727272e-05,
      "loss": 2.7306,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4625,
      "eval_samples_per_second": 507.414,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4625,
      "eval_openwebtext_samples_per_second": 507.414,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 1.9861,
      "eval_wikitext_samples_per_second": 229.59,
      "step": 321000
    },
    {
      "epoch": 18.65,
      "eval_lambada_loss": 2.443359375,
      "eval_lambada_ppl": 11.511647800524438,
      "eval_lambada_runtime": 9.5963,
      "eval_lambada_samples_per_second": 507.381,
      "step": 321000
    },
    {
      "activations/layer0_attention_weight_max": 14.869175910949707,
      "activations/layer0_attention_weight_min": -13.90064811706543,
      "activations/layer10_attention_weight_max": 37.47877502441406,
      "activations/layer10_attention_weight_min": -37.22077941894531,
      "activations/layer11_attention_weight_max": 34.94860076904297,
      "activations/layer11_attention_weight_min": -34.341217041015625,
      "activations/layer12_attention_weight_max": 21.261564254760742,
      "activations/layer12_attention_weight_min": -24.79644203186035,
      "activations/layer13_attention_weight_max": 37.42558670043945,
      "activations/layer13_attention_weight_min": -29.81017303466797,
      "activations/layer14_attention_weight_max": 38.59461212158203,
      "activations/layer14_attention_weight_min": -31.89533042907715,
      "activations/layer15_attention_weight_max": 35.251800537109375,
      "activations/layer15_attention_weight_min": -33.78078842163086,
      "activations/layer16_attention_weight_max": 37.409481048583984,
      "activations/layer16_attention_weight_min": -33.94919967651367,
      "activations/layer17_attention_weight_max": 54.84501647949219,
      "activations/layer17_attention_weight_min": -47.2723503112793,
      "activations/layer18_attention_weight_max": 51.9853401184082,
      "activations/layer18_attention_weight_min": -42.17562484741211,
      "activations/layer19_attention_weight_max": 27.914426803588867,
      "activations/layer19_attention_weight_min": -25.0091495513916,
      "activations/layer1_attention_weight_max": 15.514362335205078,
      "activations/layer1_attention_weight_min": -14.657747268676758,
      "activations/layer20_attention_weight_max": 24.902172088623047,
      "activations/layer20_attention_weight_min": -24.241151809692383,
      "activations/layer21_attention_weight_max": 39.52934646606445,
      "activations/layer21_attention_weight_min": -24.3973388671875,
      "activations/layer22_attention_weight_max": 31.24091911315918,
      "activations/layer22_attention_weight_min": -26.467269897460938,
      "activations/layer23_attention_weight_max": 42.85873031616211,
      "activations/layer23_attention_weight_min": -25.538433074951172,
      "activations/layer2_attention_weight_max": 32.9140510559082,
      "activations/layer2_attention_weight_min": -32.4300537109375,
      "activations/layer3_attention_weight_max": 85.15963745117188,
      "activations/layer3_attention_weight_min": -85.36297607421875,
      "activations/layer4_attention_weight_max": 87.07011413574219,
      "activations/layer4_attention_weight_min": -83.23189544677734,
      "activations/layer5_attention_weight_max": 63.16777038574219,
      "activations/layer5_attention_weight_min": -73.08140563964844,
      "activations/layer6_attention_weight_max": 51.53330993652344,
      "activations/layer6_attention_weight_min": -51.50960922241211,
      "activations/layer7_attention_weight_max": 63.99211502075195,
      "activations/layer7_attention_weight_min": -65.30824279785156,
      "activations/layer8_attention_weight_max": 49.310516357421875,
      "activations/layer8_attention_weight_min": -51.36544418334961,
      "activations/layer9_attention_weight_max": 46.65273666381836,
      "activations/layer9_attention_weight_min": -47.67081832885742,
      "epoch": 18.65,
      "learning_rate": 3.0003787878787877e-05,
      "loss": 2.721,
      "step": 321050
    },
    {
      "activations/layer0_attention_weight_max": 15.783164024353027,
      "activations/layer0_attention_weight_min": -14.258406639099121,
      "activations/layer10_attention_weight_max": 36.537628173828125,
      "activations/layer10_attention_weight_min": -36.74960708618164,
      "activations/layer11_attention_weight_max": 33.78338623046875,
      "activations/layer11_attention_weight_min": -35.91681671142578,
      "activations/layer12_attention_weight_max": 20.157041549682617,
      "activations/layer12_attention_weight_min": -27.336732864379883,
      "activations/layer13_attention_weight_max": 37.90705871582031,
      "activations/layer13_attention_weight_min": -24.976102828979492,
      "activations/layer14_attention_weight_max": 37.59906768798828,
      "activations/layer14_attention_weight_min": -29.05158042907715,
      "activations/layer15_attention_weight_max": 38.00017166137695,
      "activations/layer15_attention_weight_min": -28.8930721282959,
      "activations/layer16_attention_weight_max": 34.32719039916992,
      "activations/layer16_attention_weight_min": -31.52537727355957,
      "activations/layer17_attention_weight_max": 51.65974426269531,
      "activations/layer17_attention_weight_min": -41.95951843261719,
      "activations/layer18_attention_weight_max": 49.44675827026367,
      "activations/layer18_attention_weight_min": -38.85330581665039,
      "activations/layer19_attention_weight_max": 26.390859603881836,
      "activations/layer19_attention_weight_min": -24.16803550720215,
      "activations/layer1_attention_weight_max": 16.162561416625977,
      "activations/layer1_attention_weight_min": -13.684844017028809,
      "activations/layer20_attention_weight_max": 24.329776763916016,
      "activations/layer20_attention_weight_min": -20.326372146606445,
      "activations/layer21_attention_weight_max": 41.6345100402832,
      "activations/layer21_attention_weight_min": -22.977691650390625,
      "activations/layer22_attention_weight_max": 32.85697555541992,
      "activations/layer22_attention_weight_min": -25.263553619384766,
      "activations/layer23_attention_weight_max": 37.303924560546875,
      "activations/layer23_attention_weight_min": -23.045421600341797,
      "activations/layer2_attention_weight_max": 30.02783203125,
      "activations/layer2_attention_weight_min": -28.591588973999023,
      "activations/layer3_attention_weight_max": 75.41960906982422,
      "activations/layer3_attention_weight_min": -82.03824615478516,
      "activations/layer4_attention_weight_max": 77.09205627441406,
      "activations/layer4_attention_weight_min": -75.08155059814453,
      "activations/layer5_attention_weight_max": 61.10506820678711,
      "activations/layer5_attention_weight_min": -70.65971374511719,
      "activations/layer6_attention_weight_max": 50.87171173095703,
      "activations/layer6_attention_weight_min": -54.21220779418945,
      "activations/layer7_attention_weight_max": 65.3202896118164,
      "activations/layer7_attention_weight_min": -64.45657348632812,
      "activations/layer8_attention_weight_max": 45.9464111328125,
      "activations/layer8_attention_weight_min": -52.64046096801758,
      "activations/layer9_attention_weight_max": 42.926605224609375,
      "activations/layer9_attention_weight_min": -46.94345474243164,
      "epoch": 18.66,
      "learning_rate": 2.9984848484848485e-05,
      "loss": 2.7438,
      "step": 321100
    },
    {
      "activations/layer0_attention_weight_max": 14.851806640625,
      "activations/layer0_attention_weight_min": -13.602941513061523,
      "activations/layer10_attention_weight_max": 34.86429977416992,
      "activations/layer10_attention_weight_min": -36.79838180541992,
      "activations/layer11_attention_weight_max": 32.37592315673828,
      "activations/layer11_attention_weight_min": -35.211814880371094,
      "activations/layer12_attention_weight_max": 18.680706024169922,
      "activations/layer12_attention_weight_min": -23.35281753540039,
      "activations/layer13_attention_weight_max": 35.129432678222656,
      "activations/layer13_attention_weight_min": -24.596086502075195,
      "activations/layer14_attention_weight_max": 35.718231201171875,
      "activations/layer14_attention_weight_min": -29.031978607177734,
      "activations/layer15_attention_weight_max": 32.937259674072266,
      "activations/layer15_attention_weight_min": -29.004825592041016,
      "activations/layer16_attention_weight_max": 34.618282318115234,
      "activations/layer16_attention_weight_min": -32.077476501464844,
      "activations/layer17_attention_weight_max": 53.17954635620117,
      "activations/layer17_attention_weight_min": -47.10248565673828,
      "activations/layer18_attention_weight_max": 47.375091552734375,
      "activations/layer18_attention_weight_min": -40.310997009277344,
      "activations/layer19_attention_weight_max": 25.150697708129883,
      "activations/layer19_attention_weight_min": -24.47435760498047,
      "activations/layer1_attention_weight_max": 15.32505989074707,
      "activations/layer1_attention_weight_min": -14.263822555541992,
      "activations/layer20_attention_weight_max": 22.418691635131836,
      "activations/layer20_attention_weight_min": -25.081148147583008,
      "activations/layer21_attention_weight_max": 37.280296325683594,
      "activations/layer21_attention_weight_min": -24.06304168701172,
      "activations/layer22_attention_weight_max": 27.11786651611328,
      "activations/layer22_attention_weight_min": -26.79571533203125,
      "activations/layer23_attention_weight_max": 36.0632438659668,
      "activations/layer23_attention_weight_min": -26.167936325073242,
      "activations/layer2_attention_weight_max": 31.14727783203125,
      "activations/layer2_attention_weight_min": -29.999908447265625,
      "activations/layer3_attention_weight_max": 85.95997619628906,
      "activations/layer3_attention_weight_min": -87.60723876953125,
      "activations/layer4_attention_weight_max": 84.2636947631836,
      "activations/layer4_attention_weight_min": -83.36412048339844,
      "activations/layer5_attention_weight_max": 61.900848388671875,
      "activations/layer5_attention_weight_min": -77.23240661621094,
      "activations/layer6_attention_weight_max": 49.71687316894531,
      "activations/layer6_attention_weight_min": -51.82096862792969,
      "activations/layer7_attention_weight_max": 69.45516967773438,
      "activations/layer7_attention_weight_min": -64.91008758544922,
      "activations/layer8_attention_weight_max": 46.828758239746094,
      "activations/layer8_attention_weight_min": -48.784141540527344,
      "activations/layer9_attention_weight_max": 43.50703048706055,
      "activations/layer9_attention_weight_min": -45.71370315551758,
      "epoch": 18.66,
      "learning_rate": 2.9965909090909087e-05,
      "loss": 2.738,
      "step": 321150
    },
    {
      "activations/layer0_attention_weight_max": 15.552905082702637,
      "activations/layer0_attention_weight_min": -13.953680992126465,
      "activations/layer10_attention_weight_max": 40.11403274536133,
      "activations/layer10_attention_weight_min": -39.37640380859375,
      "activations/layer11_attention_weight_max": 35.089473724365234,
      "activations/layer11_attention_weight_min": -35.82477951049805,
      "activations/layer12_attention_weight_max": 19.844131469726562,
      "activations/layer12_attention_weight_min": -25.22518539428711,
      "activations/layer13_attention_weight_max": 32.140625,
      "activations/layer13_attention_weight_min": -29.365243911743164,
      "activations/layer14_attention_weight_max": 34.294769287109375,
      "activations/layer14_attention_weight_min": -29.827783584594727,
      "activations/layer15_attention_weight_max": 32.25599670410156,
      "activations/layer15_attention_weight_min": -30.70099639892578,
      "activations/layer16_attention_weight_max": 35.853450775146484,
      "activations/layer16_attention_weight_min": -33.88227081298828,
      "activations/layer17_attention_weight_max": 51.368797302246094,
      "activations/layer17_attention_weight_min": -46.219669342041016,
      "activations/layer18_attention_weight_max": 46.55142593383789,
      "activations/layer18_attention_weight_min": -42.830650329589844,
      "activations/layer19_attention_weight_max": 22.76259994506836,
      "activations/layer19_attention_weight_min": -24.13077735900879,
      "activations/layer1_attention_weight_max": 16.151111602783203,
      "activations/layer1_attention_weight_min": -13.82439136505127,
      "activations/layer20_attention_weight_max": 22.328012466430664,
      "activations/layer20_attention_weight_min": -24.6113224029541,
      "activations/layer21_attention_weight_max": 29.553043365478516,
      "activations/layer21_attention_weight_min": -21.00458335876465,
      "activations/layer22_attention_weight_max": 28.293691635131836,
      "activations/layer22_attention_weight_min": -26.32606315612793,
      "activations/layer23_attention_weight_max": 35.16459274291992,
      "activations/layer23_attention_weight_min": -23.497182846069336,
      "activations/layer2_attention_weight_max": 33.91743087768555,
      "activations/layer2_attention_weight_min": -31.05379295349121,
      "activations/layer3_attention_weight_max": 83.543701171875,
      "activations/layer3_attention_weight_min": -89.17361450195312,
      "activations/layer4_attention_weight_max": 83.71290588378906,
      "activations/layer4_attention_weight_min": -81.45003509521484,
      "activations/layer5_attention_weight_max": 61.96238327026367,
      "activations/layer5_attention_weight_min": -72.0723876953125,
      "activations/layer6_attention_weight_max": 51.12322235107422,
      "activations/layer6_attention_weight_min": -49.92786407470703,
      "activations/layer7_attention_weight_max": 67.28813171386719,
      "activations/layer7_attention_weight_min": -63.764564514160156,
      "activations/layer8_attention_weight_max": 47.86376953125,
      "activations/layer8_attention_weight_min": -50.87381362915039,
      "activations/layer9_attention_weight_max": 53.315059661865234,
      "activations/layer9_attention_weight_min": -46.959564208984375,
      "epoch": 18.66,
      "learning_rate": 2.9946969696969695e-05,
      "loss": 2.7308,
      "step": 321200
    },
    {
      "activations/layer0_attention_weight_max": 15.510154724121094,
      "activations/layer0_attention_weight_min": -13.39570140838623,
      "activations/layer10_attention_weight_max": 35.03938674926758,
      "activations/layer10_attention_weight_min": -36.12874984741211,
      "activations/layer11_attention_weight_max": 34.16022491455078,
      "activations/layer11_attention_weight_min": -36.08665084838867,
      "activations/layer12_attention_weight_max": 18.582977294921875,
      "activations/layer12_attention_weight_min": -21.71240234375,
      "activations/layer13_attention_weight_max": 31.54327392578125,
      "activations/layer13_attention_weight_min": -28.214632034301758,
      "activations/layer14_attention_weight_max": 33.54827880859375,
      "activations/layer14_attention_weight_min": -33.0063362121582,
      "activations/layer15_attention_weight_max": 33.560428619384766,
      "activations/layer15_attention_weight_min": -29.41788101196289,
      "activations/layer16_attention_weight_max": 35.589107513427734,
      "activations/layer16_attention_weight_min": -32.07821273803711,
      "activations/layer17_attention_weight_max": 49.532630920410156,
      "activations/layer17_attention_weight_min": -43.3776741027832,
      "activations/layer18_attention_weight_max": 44.941314697265625,
      "activations/layer18_attention_weight_min": -37.838096618652344,
      "activations/layer19_attention_weight_max": 23.614316940307617,
      "activations/layer19_attention_weight_min": -22.72377586364746,
      "activations/layer1_attention_weight_max": 15.907435417175293,
      "activations/layer1_attention_weight_min": -12.903664588928223,
      "activations/layer20_attention_weight_max": 22.520038604736328,
      "activations/layer20_attention_weight_min": -22.408979415893555,
      "activations/layer21_attention_weight_max": 33.846160888671875,
      "activations/layer21_attention_weight_min": -25.45232391357422,
      "activations/layer22_attention_weight_max": 28.18939971923828,
      "activations/layer22_attention_weight_min": -25.605167388916016,
      "activations/layer23_attention_weight_max": 35.962398529052734,
      "activations/layer23_attention_weight_min": -25.979307174682617,
      "activations/layer2_attention_weight_max": 31.994367599487305,
      "activations/layer2_attention_weight_min": -29.685394287109375,
      "activations/layer3_attention_weight_max": 84.43183135986328,
      "activations/layer3_attention_weight_min": -83.62935638427734,
      "activations/layer4_attention_weight_max": 81.71858215332031,
      "activations/layer4_attention_weight_min": -76.10979461669922,
      "activations/layer5_attention_weight_max": 63.19415283203125,
      "activations/layer5_attention_weight_min": -68.73123168945312,
      "activations/layer6_attention_weight_max": 48.797325134277344,
      "activations/layer6_attention_weight_min": -48.77865219116211,
      "activations/layer7_attention_weight_max": 65.10189819335938,
      "activations/layer7_attention_weight_min": -65.43443298339844,
      "activations/layer8_attention_weight_max": 46.07005310058594,
      "activations/layer8_attention_weight_min": -48.68703079223633,
      "activations/layer9_attention_weight_max": 42.4415283203125,
      "activations/layer9_attention_weight_min": -45.611385345458984,
      "epoch": 18.67,
      "learning_rate": 2.99280303030303e-05,
      "loss": 2.7378,
      "step": 321250
    },
    {
      "activations/layer0_attention_weight_max": 15.679580688476562,
      "activations/layer0_attention_weight_min": -13.81216049194336,
      "activations/layer10_attention_weight_max": 33.807716369628906,
      "activations/layer10_attention_weight_min": -35.056636810302734,
      "activations/layer11_attention_weight_max": 32.6575927734375,
      "activations/layer11_attention_weight_min": -32.792152404785156,
      "activations/layer12_attention_weight_max": 17.78087615966797,
      "activations/layer12_attention_weight_min": -25.384994506835938,
      "activations/layer13_attention_weight_max": 36.53218078613281,
      "activations/layer13_attention_weight_min": -26.994901657104492,
      "activations/layer14_attention_weight_max": 34.69781494140625,
      "activations/layer14_attention_weight_min": -29.688888549804688,
      "activations/layer15_attention_weight_max": 32.366920471191406,
      "activations/layer15_attention_weight_min": -30.248287200927734,
      "activations/layer16_attention_weight_max": 33.538665771484375,
      "activations/layer16_attention_weight_min": -33.813621520996094,
      "activations/layer17_attention_weight_max": 50.4510383605957,
      "activations/layer17_attention_weight_min": -45.48488998413086,
      "activations/layer18_attention_weight_max": 45.5368766784668,
      "activations/layer18_attention_weight_min": -37.900146484375,
      "activations/layer19_attention_weight_max": 23.4794979095459,
      "activations/layer19_attention_weight_min": -23.259321212768555,
      "activations/layer1_attention_weight_max": 15.655123710632324,
      "activations/layer1_attention_weight_min": -13.428144454956055,
      "activations/layer20_attention_weight_max": 21.64429473876953,
      "activations/layer20_attention_weight_min": -25.26705551147461,
      "activations/layer21_attention_weight_max": 35.218719482421875,
      "activations/layer21_attention_weight_min": -23.34636688232422,
      "activations/layer22_attention_weight_max": 29.422069549560547,
      "activations/layer22_attention_weight_min": -25.556245803833008,
      "activations/layer23_attention_weight_max": 36.425559997558594,
      "activations/layer23_attention_weight_min": -25.121845245361328,
      "activations/layer2_attention_weight_max": 30.40944480895996,
      "activations/layer2_attention_weight_min": -28.360916137695312,
      "activations/layer3_attention_weight_max": 81.76594543457031,
      "activations/layer3_attention_weight_min": -87.81075286865234,
      "activations/layer4_attention_weight_max": 80.70629119873047,
      "activations/layer4_attention_weight_min": -81.70858764648438,
      "activations/layer5_attention_weight_max": 62.663204193115234,
      "activations/layer5_attention_weight_min": -73.9102783203125,
      "activations/layer6_attention_weight_max": 49.3249626159668,
      "activations/layer6_attention_weight_min": -50.52156066894531,
      "activations/layer7_attention_weight_max": 64.87665557861328,
      "activations/layer7_attention_weight_min": -65.80206298828125,
      "activations/layer8_attention_weight_max": 44.15346145629883,
      "activations/layer8_attention_weight_min": -48.561405181884766,
      "activations/layer9_attention_weight_max": 41.803977966308594,
      "activations/layer9_attention_weight_min": -43.21433639526367,
      "epoch": 18.67,
      "learning_rate": 2.9909090909090908e-05,
      "loss": 2.7506,
      "step": 321300
    },
    {
      "activations/layer0_attention_weight_max": 15.937644958496094,
      "activations/layer0_attention_weight_min": -14.106526374816895,
      "activations/layer10_attention_weight_max": 38.75075149536133,
      "activations/layer10_attention_weight_min": -39.02535629272461,
      "activations/layer11_attention_weight_max": 36.26018524169922,
      "activations/layer11_attention_weight_min": -37.61011505126953,
      "activations/layer12_attention_weight_max": 19.591520309448242,
      "activations/layer12_attention_weight_min": -29.523164749145508,
      "activations/layer13_attention_weight_max": 39.03738021850586,
      "activations/layer13_attention_weight_min": -30.325881958007812,
      "activations/layer14_attention_weight_max": 41.75154113769531,
      "activations/layer14_attention_weight_min": -30.34964370727539,
      "activations/layer15_attention_weight_max": 40.909942626953125,
      "activations/layer15_attention_weight_min": -30.878273010253906,
      "activations/layer16_attention_weight_max": 35.737388610839844,
      "activations/layer16_attention_weight_min": -33.590431213378906,
      "activations/layer17_attention_weight_max": 57.47574234008789,
      "activations/layer17_attention_weight_min": -48.65439224243164,
      "activations/layer18_attention_weight_max": 53.27817153930664,
      "activations/layer18_attention_weight_min": -42.804630279541016,
      "activations/layer19_attention_weight_max": 27.609375,
      "activations/layer19_attention_weight_min": -22.33465576171875,
      "activations/layer1_attention_weight_max": 16.22395896911621,
      "activations/layer1_attention_weight_min": -14.510004997253418,
      "activations/layer20_attention_weight_max": 25.5688419342041,
      "activations/layer20_attention_weight_min": -20.736797332763672,
      "activations/layer21_attention_weight_max": 45.53841781616211,
      "activations/layer21_attention_weight_min": -27.472593307495117,
      "activations/layer22_attention_weight_max": 34.70286560058594,
      "activations/layer22_attention_weight_min": -24.99609375,
      "activations/layer23_attention_weight_max": 40.83558654785156,
      "activations/layer23_attention_weight_min": -22.30214500427246,
      "activations/layer2_attention_weight_max": 31.308242797851562,
      "activations/layer2_attention_weight_min": -30.057348251342773,
      "activations/layer3_attention_weight_max": 85.59610748291016,
      "activations/layer3_attention_weight_min": -88.02555847167969,
      "activations/layer4_attention_weight_max": 84.25035858154297,
      "activations/layer4_attention_weight_min": -83.76873016357422,
      "activations/layer5_attention_weight_max": 65.37862396240234,
      "activations/layer5_attention_weight_min": -70.66242980957031,
      "activations/layer6_attention_weight_max": 51.1129035949707,
      "activations/layer6_attention_weight_min": -52.280860900878906,
      "activations/layer7_attention_weight_max": 67.16050720214844,
      "activations/layer7_attention_weight_min": -65.49031829833984,
      "activations/layer8_attention_weight_max": 49.22560119628906,
      "activations/layer8_attention_weight_min": -52.298702239990234,
      "activations/layer9_attention_weight_max": 47.01899719238281,
      "activations/layer9_attention_weight_min": -49.38761901855469,
      "epoch": 18.67,
      "learning_rate": 2.989015151515151e-05,
      "loss": 2.7356,
      "step": 321350
    },
    {
      "activations/layer0_attention_weight_max": 17.192781448364258,
      "activations/layer0_attention_weight_min": -13.443028450012207,
      "activations/layer10_attention_weight_max": 33.46627426147461,
      "activations/layer10_attention_weight_min": -35.4835205078125,
      "activations/layer11_attention_weight_max": 33.561851501464844,
      "activations/layer11_attention_weight_min": -34.0332145690918,
      "activations/layer12_attention_weight_max": 18.57080078125,
      "activations/layer12_attention_weight_min": -25.43602752685547,
      "activations/layer13_attention_weight_max": 33.575469970703125,
      "activations/layer13_attention_weight_min": -26.620079040527344,
      "activations/layer14_attention_weight_max": 33.177860260009766,
      "activations/layer14_attention_weight_min": -32.53574752807617,
      "activations/layer15_attention_weight_max": 30.832679748535156,
      "activations/layer15_attention_weight_min": -31.698379516601562,
      "activations/layer16_attention_weight_max": 34.4930534362793,
      "activations/layer16_attention_weight_min": -35.35164260864258,
      "activations/layer17_attention_weight_max": 49.684810638427734,
      "activations/layer17_attention_weight_min": -49.6507682800293,
      "activations/layer18_attention_weight_max": 43.556251525878906,
      "activations/layer18_attention_weight_min": -41.7510871887207,
      "activations/layer19_attention_weight_max": 24.604833602905273,
      "activations/layer19_attention_weight_min": -22.95752716064453,
      "activations/layer1_attention_weight_max": 15.194267272949219,
      "activations/layer1_attention_weight_min": -15.473559379577637,
      "activations/layer20_attention_weight_max": 23.128808975219727,
      "activations/layer20_attention_weight_min": -22.418821334838867,
      "activations/layer21_attention_weight_max": 33.677677154541016,
      "activations/layer21_attention_weight_min": -21.80755043029785,
      "activations/layer22_attention_weight_max": 29.706134796142578,
      "activations/layer22_attention_weight_min": -26.508413314819336,
      "activations/layer23_attention_weight_max": 35.35124588012695,
      "activations/layer23_attention_weight_min": -25.413467407226562,
      "activations/layer2_attention_weight_max": 30.20937728881836,
      "activations/layer2_attention_weight_min": -29.85504913330078,
      "activations/layer3_attention_weight_max": 82.72813415527344,
      "activations/layer3_attention_weight_min": -81.77571105957031,
      "activations/layer4_attention_weight_max": 79.49852752685547,
      "activations/layer4_attention_weight_min": -79.83940887451172,
      "activations/layer5_attention_weight_max": 58.289161682128906,
      "activations/layer5_attention_weight_min": -73.98222351074219,
      "activations/layer6_attention_weight_max": 46.76811218261719,
      "activations/layer6_attention_weight_min": -50.01343536376953,
      "activations/layer7_attention_weight_max": 62.72443389892578,
      "activations/layer7_attention_weight_min": -62.204833984375,
      "activations/layer8_attention_weight_max": 44.1085090637207,
      "activations/layer8_attention_weight_min": -48.42736053466797,
      "activations/layer9_attention_weight_max": 43.24147415161133,
      "activations/layer9_attention_weight_min": -44.0819206237793,
      "epoch": 18.68,
      "learning_rate": 2.9871212121212118e-05,
      "loss": 2.7335,
      "step": 321400
    },
    {
      "activations/layer0_attention_weight_max": 16.043928146362305,
      "activations/layer0_attention_weight_min": -13.634225845336914,
      "activations/layer10_attention_weight_max": 37.58695983886719,
      "activations/layer10_attention_weight_min": -37.24058532714844,
      "activations/layer11_attention_weight_max": 33.21098709106445,
      "activations/layer11_attention_weight_min": -35.55317687988281,
      "activations/layer12_attention_weight_max": 17.786731719970703,
      "activations/layer12_attention_weight_min": -26.55828094482422,
      "activations/layer13_attention_weight_max": 36.082984924316406,
      "activations/layer13_attention_weight_min": -27.758056640625,
      "activations/layer14_attention_weight_max": 45.92183303833008,
      "activations/layer14_attention_weight_min": -31.184001922607422,
      "activations/layer15_attention_weight_max": 39.539371490478516,
      "activations/layer15_attention_weight_min": -31.481586456298828,
      "activations/layer16_attention_weight_max": 34.30813980102539,
      "activations/layer16_attention_weight_min": -33.03620147705078,
      "activations/layer17_attention_weight_max": 52.84349822998047,
      "activations/layer17_attention_weight_min": -49.94801712036133,
      "activations/layer18_attention_weight_max": 49.69263458251953,
      "activations/layer18_attention_weight_min": -43.55989456176758,
      "activations/layer19_attention_weight_max": 25.716064453125,
      "activations/layer19_attention_weight_min": -25.14522933959961,
      "activations/layer1_attention_weight_max": 15.485235214233398,
      "activations/layer1_attention_weight_min": -13.367435455322266,
      "activations/layer20_attention_weight_max": 27.454673767089844,
      "activations/layer20_attention_weight_min": -23.6831111907959,
      "activations/layer21_attention_weight_max": 45.557861328125,
      "activations/layer21_attention_weight_min": -26.17693519592285,
      "activations/layer22_attention_weight_max": 27.42177963256836,
      "activations/layer22_attention_weight_min": -28.27809715270996,
      "activations/layer23_attention_weight_max": 36.17183303833008,
      "activations/layer23_attention_weight_min": -23.843374252319336,
      "activations/layer2_attention_weight_max": 28.96923828125,
      "activations/layer2_attention_weight_min": -28.785564422607422,
      "activations/layer3_attention_weight_max": 80.6166000366211,
      "activations/layer3_attention_weight_min": -86.52362823486328,
      "activations/layer4_attention_weight_max": 80.32514190673828,
      "activations/layer4_attention_weight_min": -82.73869323730469,
      "activations/layer5_attention_weight_max": 61.30624008178711,
      "activations/layer5_attention_weight_min": -72.84326171875,
      "activations/layer6_attention_weight_max": 47.423057556152344,
      "activations/layer6_attention_weight_min": -51.82485580444336,
      "activations/layer7_attention_weight_max": 64.30130004882812,
      "activations/layer7_attention_weight_min": -65.905029296875,
      "activations/layer8_attention_weight_max": 47.31368637084961,
      "activations/layer8_attention_weight_min": -51.778594970703125,
      "activations/layer9_attention_weight_max": 45.33399963378906,
      "activations/layer9_attention_weight_min": -46.094600677490234,
      "epoch": 18.68,
      "learning_rate": 2.9852272727272723e-05,
      "loss": 2.7347,
      "step": 321450
    },
    {
      "activations/layer0_attention_weight_max": 15.595973014831543,
      "activations/layer0_attention_weight_min": -13.644882202148438,
      "activations/layer10_attention_weight_max": 36.82832717895508,
      "activations/layer10_attention_weight_min": -37.15474319458008,
      "activations/layer11_attention_weight_max": 33.750240325927734,
      "activations/layer11_attention_weight_min": -34.723785400390625,
      "activations/layer12_attention_weight_max": 20.294950485229492,
      "activations/layer12_attention_weight_min": -26.283388137817383,
      "activations/layer13_attention_weight_max": 39.11993408203125,
      "activations/layer13_attention_weight_min": -27.27532386779785,
      "activations/layer14_attention_weight_max": 38.20143127441406,
      "activations/layer14_attention_weight_min": -30.50234603881836,
      "activations/layer15_attention_weight_max": 36.77830505371094,
      "activations/layer15_attention_weight_min": -29.84577751159668,
      "activations/layer16_attention_weight_max": 38.72910690307617,
      "activations/layer16_attention_weight_min": -34.18393325805664,
      "activations/layer17_attention_weight_max": 55.73321533203125,
      "activations/layer17_attention_weight_min": -45.0991325378418,
      "activations/layer18_attention_weight_max": 48.351341247558594,
      "activations/layer18_attention_weight_min": -41.78934097290039,
      "activations/layer19_attention_weight_max": 24.965539932250977,
      "activations/layer19_attention_weight_min": -25.328947067260742,
      "activations/layer1_attention_weight_max": 16.403087615966797,
      "activations/layer1_attention_weight_min": -14.770829200744629,
      "activations/layer20_attention_weight_max": 23.201202392578125,
      "activations/layer20_attention_weight_min": -24.74764633178711,
      "activations/layer21_attention_weight_max": 41.63816833496094,
      "activations/layer21_attention_weight_min": -21.90653419494629,
      "activations/layer22_attention_weight_max": 31.3111629486084,
      "activations/layer22_attention_weight_min": -28.52632713317871,
      "activations/layer23_attention_weight_max": 42.50346374511719,
      "activations/layer23_attention_weight_min": -28.533750534057617,
      "activations/layer2_attention_weight_max": 32.391136169433594,
      "activations/layer2_attention_weight_min": -32.68602752685547,
      "activations/layer3_attention_weight_max": 85.76477813720703,
      "activations/layer3_attention_weight_min": -87.6613540649414,
      "activations/layer4_attention_weight_max": 84.52916717529297,
      "activations/layer4_attention_weight_min": -86.41413116455078,
      "activations/layer5_attention_weight_max": 60.892818450927734,
      "activations/layer5_attention_weight_min": -71.70270538330078,
      "activations/layer6_attention_weight_max": 52.117897033691406,
      "activations/layer6_attention_weight_min": -54.29763412475586,
      "activations/layer7_attention_weight_max": 66.60916900634766,
      "activations/layer7_attention_weight_min": -69.2246322631836,
      "activations/layer8_attention_weight_max": 49.27985763549805,
      "activations/layer8_attention_weight_min": -53.447208404541016,
      "activations/layer9_attention_weight_max": 49.15421676635742,
      "activations/layer9_attention_weight_min": -48.6807861328125,
      "epoch": 18.68,
      "learning_rate": 2.983333333333333e-05,
      "loss": 2.737,
      "step": 321500
    },
    {
      "activations/layer0_attention_weight_max": 15.85435962677002,
      "activations/layer0_attention_weight_min": -13.791631698608398,
      "activations/layer10_attention_weight_max": 35.58605194091797,
      "activations/layer10_attention_weight_min": -34.95875930786133,
      "activations/layer11_attention_weight_max": 33.908592224121094,
      "activations/layer11_attention_weight_min": -34.00632858276367,
      "activations/layer12_attention_weight_max": 19.084720611572266,
      "activations/layer12_attention_weight_min": -23.320716857910156,
      "activations/layer13_attention_weight_max": 34.97325134277344,
      "activations/layer13_attention_weight_min": -29.080547332763672,
      "activations/layer14_attention_weight_max": 35.23566436767578,
      "activations/layer14_attention_weight_min": -30.837539672851562,
      "activations/layer15_attention_weight_max": 30.960599899291992,
      "activations/layer15_attention_weight_min": -30.500431060791016,
      "activations/layer16_attention_weight_max": 31.98015594482422,
      "activations/layer16_attention_weight_min": -30.620059967041016,
      "activations/layer17_attention_weight_max": 53.8994026184082,
      "activations/layer17_attention_weight_min": -44.716949462890625,
      "activations/layer18_attention_weight_max": 46.53054428100586,
      "activations/layer18_attention_weight_min": -39.262107849121094,
      "activations/layer19_attention_weight_max": 25.2062931060791,
      "activations/layer19_attention_weight_min": -25.923709869384766,
      "activations/layer1_attention_weight_max": 15.34364128112793,
      "activations/layer1_attention_weight_min": -15.379901885986328,
      "activations/layer20_attention_weight_max": 23.02360725402832,
      "activations/layer20_attention_weight_min": -23.429481506347656,
      "activations/layer21_attention_weight_max": 34.52749252319336,
      "activations/layer21_attention_weight_min": -23.84666633605957,
      "activations/layer22_attention_weight_max": 27.18154525756836,
      "activations/layer22_attention_weight_min": -25.88578987121582,
      "activations/layer23_attention_weight_max": 33.62195587158203,
      "activations/layer23_attention_weight_min": -24.64101219177246,
      "activations/layer2_attention_weight_max": 32.18043518066406,
      "activations/layer2_attention_weight_min": -31.145078659057617,
      "activations/layer3_attention_weight_max": 85.04708099365234,
      "activations/layer3_attention_weight_min": -86.10152435302734,
      "activations/layer4_attention_weight_max": 83.70455169677734,
      "activations/layer4_attention_weight_min": -81.71622467041016,
      "activations/layer5_attention_weight_max": 60.70835876464844,
      "activations/layer5_attention_weight_min": -70.49769592285156,
      "activations/layer6_attention_weight_max": 49.44132614135742,
      "activations/layer6_attention_weight_min": -50.5643196105957,
      "activations/layer7_attention_weight_max": 63.77769088745117,
      "activations/layer7_attention_weight_min": -65.4472427368164,
      "activations/layer8_attention_weight_max": 47.040321350097656,
      "activations/layer8_attention_weight_min": -48.84627151489258,
      "activations/layer9_attention_weight_max": 44.55521011352539,
      "activations/layer9_attention_weight_min": -49.17887496948242,
      "epoch": 18.68,
      "learning_rate": 2.9814393939393936e-05,
      "loss": 2.7408,
      "step": 321550
    },
    {
      "activations/layer0_attention_weight_max": 15.12362289428711,
      "activations/layer0_attention_weight_min": -13.801058769226074,
      "activations/layer10_attention_weight_max": 31.97228240966797,
      "activations/layer10_attention_weight_min": -36.100440979003906,
      "activations/layer11_attention_weight_max": 31.201698303222656,
      "activations/layer11_attention_weight_min": -32.808204650878906,
      "activations/layer12_attention_weight_max": 20.34018325805664,
      "activations/layer12_attention_weight_min": -31.5699520111084,
      "activations/layer13_attention_weight_max": 36.379032135009766,
      "activations/layer13_attention_weight_min": -26.90435218811035,
      "activations/layer14_attention_weight_max": 39.25890350341797,
      "activations/layer14_attention_weight_min": -32.29055404663086,
      "activations/layer15_attention_weight_max": 35.737701416015625,
      "activations/layer15_attention_weight_min": -31.717775344848633,
      "activations/layer16_attention_weight_max": 36.67002868652344,
      "activations/layer16_attention_weight_min": -33.60529708862305,
      "activations/layer17_attention_weight_max": 56.28691864013672,
      "activations/layer17_attention_weight_min": -44.62513732910156,
      "activations/layer18_attention_weight_max": 49.43326950073242,
      "activations/layer18_attention_weight_min": -39.13807678222656,
      "activations/layer19_attention_weight_max": 25.18408966064453,
      "activations/layer19_attention_weight_min": -22.153165817260742,
      "activations/layer1_attention_weight_max": 16.89012336730957,
      "activations/layer1_attention_weight_min": -14.571772575378418,
      "activations/layer20_attention_weight_max": 23.54754638671875,
      "activations/layer20_attention_weight_min": -23.251476287841797,
      "activations/layer21_attention_weight_max": 39.42460632324219,
      "activations/layer21_attention_weight_min": -23.888378143310547,
      "activations/layer22_attention_weight_max": 32.07600402832031,
      "activations/layer22_attention_weight_min": -25.289602279663086,
      "activations/layer23_attention_weight_max": 36.751190185546875,
      "activations/layer23_attention_weight_min": -23.366897583007812,
      "activations/layer2_attention_weight_max": 30.492122650146484,
      "activations/layer2_attention_weight_min": -29.890338897705078,
      "activations/layer3_attention_weight_max": 79.2430648803711,
      "activations/layer3_attention_weight_min": -82.91618347167969,
      "activations/layer4_attention_weight_max": 80.89251708984375,
      "activations/layer4_attention_weight_min": -77.8077621459961,
      "activations/layer5_attention_weight_max": 63.59894943237305,
      "activations/layer5_attention_weight_min": -69.28521728515625,
      "activations/layer6_attention_weight_max": 45.78068542480469,
      "activations/layer6_attention_weight_min": -49.32241439819336,
      "activations/layer7_attention_weight_max": 68.89348602294922,
      "activations/layer7_attention_weight_min": -62.052207946777344,
      "activations/layer8_attention_weight_max": 44.33099365234375,
      "activations/layer8_attention_weight_min": -48.34372329711914,
      "activations/layer9_attention_weight_max": 49.49357604980469,
      "activations/layer9_attention_weight_min": -43.15938949584961,
      "epoch": 18.69,
      "learning_rate": 2.9795454545454544e-05,
      "loss": 2.7418,
      "step": 321600
    },
    {
      "activations/layer0_attention_weight_max": 16.551166534423828,
      "activations/layer0_attention_weight_min": -13.660310745239258,
      "activations/layer10_attention_weight_max": 34.3832893371582,
      "activations/layer10_attention_weight_min": -34.06087875366211,
      "activations/layer11_attention_weight_max": 30.70953941345215,
      "activations/layer11_attention_weight_min": -32.43037414550781,
      "activations/layer12_attention_weight_max": 18.39784812927246,
      "activations/layer12_attention_weight_min": -21.95934295654297,
      "activations/layer13_attention_weight_max": 31.269893646240234,
      "activations/layer13_attention_weight_min": -26.95359230041504,
      "activations/layer14_attention_weight_max": 34.516197204589844,
      "activations/layer14_attention_weight_min": -30.677595138549805,
      "activations/layer15_attention_weight_max": 33.49770736694336,
      "activations/layer15_attention_weight_min": -31.36788558959961,
      "activations/layer16_attention_weight_max": 33.9605598449707,
      "activations/layer16_attention_weight_min": -31.315759658813477,
      "activations/layer17_attention_weight_max": 51.180973052978516,
      "activations/layer17_attention_weight_min": -48.724342346191406,
      "activations/layer18_attention_weight_max": 45.04398727416992,
      "activations/layer18_attention_weight_min": -40.0802116394043,
      "activations/layer19_attention_weight_max": 22.307737350463867,
      "activations/layer19_attention_weight_min": -21.93561553955078,
      "activations/layer1_attention_weight_max": 15.37172794342041,
      "activations/layer1_attention_weight_min": -13.822206497192383,
      "activations/layer20_attention_weight_max": 23.217138290405273,
      "activations/layer20_attention_weight_min": -24.608171463012695,
      "activations/layer21_attention_weight_max": 37.05644226074219,
      "activations/layer21_attention_weight_min": -22.676265716552734,
      "activations/layer22_attention_weight_max": 28.84103012084961,
      "activations/layer22_attention_weight_min": -24.41118621826172,
      "activations/layer23_attention_weight_max": 35.51958084106445,
      "activations/layer23_attention_weight_min": -23.751667022705078,
      "activations/layer2_attention_weight_max": 31.444347381591797,
      "activations/layer2_attention_weight_min": -29.422462463378906,
      "activations/layer3_attention_weight_max": 85.09521484375,
      "activations/layer3_attention_weight_min": -86.08972930908203,
      "activations/layer4_attention_weight_max": 79.80931091308594,
      "activations/layer4_attention_weight_min": -79.29235076904297,
      "activations/layer5_attention_weight_max": 59.15721130371094,
      "activations/layer5_attention_weight_min": -74.32237243652344,
      "activations/layer6_attention_weight_max": 46.59269714355469,
      "activations/layer6_attention_weight_min": -52.08180618286133,
      "activations/layer7_attention_weight_max": 63.41981887817383,
      "activations/layer7_attention_weight_min": -63.22958755493164,
      "activations/layer8_attention_weight_max": 42.83376693725586,
      "activations/layer8_attention_weight_min": -47.129425048828125,
      "activations/layer9_attention_weight_max": 41.080928802490234,
      "activations/layer9_attention_weight_min": -46.04203414916992,
      "epoch": 18.69,
      "learning_rate": 2.9776515151515146e-05,
      "loss": 2.755,
      "step": 321650
    },
    {
      "activations/layer0_attention_weight_max": 15.759690284729004,
      "activations/layer0_attention_weight_min": -13.680008888244629,
      "activations/layer10_attention_weight_max": 34.186256408691406,
      "activations/layer10_attention_weight_min": -34.338619232177734,
      "activations/layer11_attention_weight_max": 32.32038879394531,
      "activations/layer11_attention_weight_min": -32.71749496459961,
      "activations/layer12_attention_weight_max": 18.442554473876953,
      "activations/layer12_attention_weight_min": -22.904457092285156,
      "activations/layer13_attention_weight_max": 38.542205810546875,
      "activations/layer13_attention_weight_min": -27.964309692382812,
      "activations/layer14_attention_weight_max": 34.92278289794922,
      "activations/layer14_attention_weight_min": -31.466190338134766,
      "activations/layer15_attention_weight_max": 33.20958709716797,
      "activations/layer15_attention_weight_min": -30.171178817749023,
      "activations/layer16_attention_weight_max": 33.319522857666016,
      "activations/layer16_attention_weight_min": -31.906145095825195,
      "activations/layer17_attention_weight_max": 48.7054328918457,
      "activations/layer17_attention_weight_min": -44.75559616088867,
      "activations/layer18_attention_weight_max": 47.430686950683594,
      "activations/layer18_attention_weight_min": -42.70907974243164,
      "activations/layer19_attention_weight_max": 25.101015090942383,
      "activations/layer19_attention_weight_min": -23.90440559387207,
      "activations/layer1_attention_weight_max": 16.726425170898438,
      "activations/layer1_attention_weight_min": -13.980761528015137,
      "activations/layer20_attention_weight_max": 22.96544075012207,
      "activations/layer20_attention_weight_min": -25.1966552734375,
      "activations/layer21_attention_weight_max": 34.40447998046875,
      "activations/layer21_attention_weight_min": -27.60162925720215,
      "activations/layer22_attention_weight_max": 33.88521957397461,
      "activations/layer22_attention_weight_min": -27.77159309387207,
      "activations/layer23_attention_weight_max": 39.107627868652344,
      "activations/layer23_attention_weight_min": -27.327777862548828,
      "activations/layer2_attention_weight_max": 30.51386070251465,
      "activations/layer2_attention_weight_min": -28.210060119628906,
      "activations/layer3_attention_weight_max": 83.33145904541016,
      "activations/layer3_attention_weight_min": -84.83722686767578,
      "activations/layer4_attention_weight_max": 78.72124481201172,
      "activations/layer4_attention_weight_min": -81.63462829589844,
      "activations/layer5_attention_weight_max": 59.371307373046875,
      "activations/layer5_attention_weight_min": -70.86107635498047,
      "activations/layer6_attention_weight_max": 49.23060607910156,
      "activations/layer6_attention_weight_min": -50.57960510253906,
      "activations/layer7_attention_weight_max": 63.930198669433594,
      "activations/layer7_attention_weight_min": -62.52704620361328,
      "activations/layer8_attention_weight_max": 49.14236831665039,
      "activations/layer8_attention_weight_min": -52.21055221557617,
      "activations/layer9_attention_weight_max": 43.70710754394531,
      "activations/layer9_attention_weight_min": -45.33967971801758,
      "epoch": 18.69,
      "learning_rate": 2.9757575757575754e-05,
      "loss": 2.7397,
      "step": 321700
    },
    {
      "activations/layer0_attention_weight_max": 14.7971830368042,
      "activations/layer0_attention_weight_min": -13.787796020507812,
      "activations/layer10_attention_weight_max": 36.75118637084961,
      "activations/layer10_attention_weight_min": -35.75596237182617,
      "activations/layer11_attention_weight_max": 33.17987823486328,
      "activations/layer11_attention_weight_min": -34.787071228027344,
      "activations/layer12_attention_weight_max": 21.473953247070312,
      "activations/layer12_attention_weight_min": -30.252071380615234,
      "activations/layer13_attention_weight_max": 55.84600067138672,
      "activations/layer13_attention_weight_min": -34.04045486450195,
      "activations/layer14_attention_weight_max": 41.93281173706055,
      "activations/layer14_attention_weight_min": -29.197168350219727,
      "activations/layer15_attention_weight_max": 36.56350326538086,
      "activations/layer15_attention_weight_min": -28.979759216308594,
      "activations/layer16_attention_weight_max": 34.88274002075195,
      "activations/layer16_attention_weight_min": -30.878435134887695,
      "activations/layer17_attention_weight_max": 56.01477813720703,
      "activations/layer17_attention_weight_min": -44.95778274536133,
      "activations/layer18_attention_weight_max": 48.96708297729492,
      "activations/layer18_attention_weight_min": -39.14049530029297,
      "activations/layer19_attention_weight_max": 25.514270782470703,
      "activations/layer19_attention_weight_min": -23.531200408935547,
      "activations/layer1_attention_weight_max": 15.738344192504883,
      "activations/layer1_attention_weight_min": -13.960294723510742,
      "activations/layer20_attention_weight_max": 22.568557739257812,
      "activations/layer20_attention_weight_min": -26.4466552734375,
      "activations/layer21_attention_weight_max": 39.45428466796875,
      "activations/layer21_attention_weight_min": -28.278728485107422,
      "activations/layer22_attention_weight_max": 28.67509651184082,
      "activations/layer22_attention_weight_min": -26.882213592529297,
      "activations/layer23_attention_weight_max": 37.708091735839844,
      "activations/layer23_attention_weight_min": -24.81982421875,
      "activations/layer2_attention_weight_max": 31.301284790039062,
      "activations/layer2_attention_weight_min": -28.4515438079834,
      "activations/layer3_attention_weight_max": 84.21450805664062,
      "activations/layer3_attention_weight_min": -87.9808578491211,
      "activations/layer4_attention_weight_max": 84.06856536865234,
      "activations/layer4_attention_weight_min": -81.50756072998047,
      "activations/layer5_attention_weight_max": 59.47309875488281,
      "activations/layer5_attention_weight_min": -70.67345428466797,
      "activations/layer6_attention_weight_max": 51.40715789794922,
      "activations/layer6_attention_weight_min": -52.59440994262695,
      "activations/layer7_attention_weight_max": 69.03241729736328,
      "activations/layer7_attention_weight_min": -66.76707458496094,
      "activations/layer8_attention_weight_max": 44.17456817626953,
      "activations/layer8_attention_weight_min": -50.384002685546875,
      "activations/layer9_attention_weight_max": 44.146575927734375,
      "activations/layer9_attention_weight_min": -47.94367218017578,
      "epoch": 18.7,
      "learning_rate": 2.9738636363636362e-05,
      "loss": 2.7326,
      "step": 321750
    },
    {
      "activations/layer0_attention_weight_max": 15.175688743591309,
      "activations/layer0_attention_weight_min": -13.800385475158691,
      "activations/layer10_attention_weight_max": 36.44169998168945,
      "activations/layer10_attention_weight_min": -35.99546432495117,
      "activations/layer11_attention_weight_max": 33.787105560302734,
      "activations/layer11_attention_weight_min": -35.87538528442383,
      "activations/layer12_attention_weight_max": 23.816797256469727,
      "activations/layer12_attention_weight_min": -24.630659103393555,
      "activations/layer13_attention_weight_max": 32.894256591796875,
      "activations/layer13_attention_weight_min": -27.48541259765625,
      "activations/layer14_attention_weight_max": 34.15393829345703,
      "activations/layer14_attention_weight_min": -31.563417434692383,
      "activations/layer15_attention_weight_max": 32.96433639526367,
      "activations/layer15_attention_weight_min": -30.130496978759766,
      "activations/layer16_attention_weight_max": 35.019020080566406,
      "activations/layer16_attention_weight_min": -33.547061920166016,
      "activations/layer17_attention_weight_max": 57.08121109008789,
      "activations/layer17_attention_weight_min": -46.95625686645508,
      "activations/layer18_attention_weight_max": 50.85112762451172,
      "activations/layer18_attention_weight_min": -42.977359771728516,
      "activations/layer19_attention_weight_max": 24.961402893066406,
      "activations/layer19_attention_weight_min": -25.468542098999023,
      "activations/layer1_attention_weight_max": 15.791479110717773,
      "activations/layer1_attention_weight_min": -14.361809730529785,
      "activations/layer20_attention_weight_max": 24.431039810180664,
      "activations/layer20_attention_weight_min": -20.742399215698242,
      "activations/layer21_attention_weight_max": 36.49403381347656,
      "activations/layer21_attention_weight_min": -23.745481491088867,
      "activations/layer22_attention_weight_max": 27.443586349487305,
      "activations/layer22_attention_weight_min": -26.90823745727539,
      "activations/layer23_attention_weight_max": 32.55400466918945,
      "activations/layer23_attention_weight_min": -27.633275985717773,
      "activations/layer2_attention_weight_max": 31.450000762939453,
      "activations/layer2_attention_weight_min": -30.474964141845703,
      "activations/layer3_attention_weight_max": 86.73627471923828,
      "activations/layer3_attention_weight_min": -86.74991607666016,
      "activations/layer4_attention_weight_max": 81.50438690185547,
      "activations/layer4_attention_weight_min": -81.41592407226562,
      "activations/layer5_attention_weight_max": 63.98188781738281,
      "activations/layer5_attention_weight_min": -72.95622253417969,
      "activations/layer6_attention_weight_max": 47.94815444946289,
      "activations/layer6_attention_weight_min": -48.72260284423828,
      "activations/layer7_attention_weight_max": 61.50289535522461,
      "activations/layer7_attention_weight_min": -63.103424072265625,
      "activations/layer8_attention_weight_max": 45.7933349609375,
      "activations/layer8_attention_weight_min": -48.14051055908203,
      "activations/layer9_attention_weight_max": 41.761253356933594,
      "activations/layer9_attention_weight_min": -46.9494514465332,
      "epoch": 18.7,
      "learning_rate": 2.9719696969696967e-05,
      "loss": 2.7238,
      "step": 321800
    },
    {
      "activations/layer0_attention_weight_max": 16.24109649658203,
      "activations/layer0_attention_weight_min": -13.789200782775879,
      "activations/layer10_attention_weight_max": 33.11961364746094,
      "activations/layer10_attention_weight_min": -34.14512252807617,
      "activations/layer11_attention_weight_max": 30.874736785888672,
      "activations/layer11_attention_weight_min": -32.26571273803711,
      "activations/layer12_attention_weight_max": 18.22532081604004,
      "activations/layer12_attention_weight_min": -26.90127944946289,
      "activations/layer13_attention_weight_max": 29.8386173248291,
      "activations/layer13_attention_weight_min": -26.130884170532227,
      "activations/layer14_attention_weight_max": 33.8605842590332,
      "activations/layer14_attention_weight_min": -29.48214340209961,
      "activations/layer15_attention_weight_max": 33.183345794677734,
      "activations/layer15_attention_weight_min": -29.688886642456055,
      "activations/layer16_attention_weight_max": 33.389373779296875,
      "activations/layer16_attention_weight_min": -31.481611251831055,
      "activations/layer17_attention_weight_max": 49.583984375,
      "activations/layer17_attention_weight_min": -45.69842529296875,
      "activations/layer18_attention_weight_max": 41.68346405029297,
      "activations/layer18_attention_weight_min": -38.20735168457031,
      "activations/layer19_attention_weight_max": 22.20441436767578,
      "activations/layer19_attention_weight_min": -23.159135818481445,
      "activations/layer1_attention_weight_max": 16.774423599243164,
      "activations/layer1_attention_weight_min": -14.672892570495605,
      "activations/layer20_attention_weight_max": 22.370216369628906,
      "activations/layer20_attention_weight_min": -22.95650291442871,
      "activations/layer21_attention_weight_max": 36.35131072998047,
      "activations/layer21_attention_weight_min": -23.652202606201172,
      "activations/layer22_attention_weight_max": 27.894786834716797,
      "activations/layer22_attention_weight_min": -26.64484977722168,
      "activations/layer23_attention_weight_max": 34.16830825805664,
      "activations/layer23_attention_weight_min": -23.98068618774414,
      "activations/layer2_attention_weight_max": 32.94271469116211,
      "activations/layer2_attention_weight_min": -31.684598922729492,
      "activations/layer3_attention_weight_max": 88.66859436035156,
      "activations/layer3_attention_weight_min": -82.99807739257812,
      "activations/layer4_attention_weight_max": 80.74678802490234,
      "activations/layer4_attention_weight_min": -77.80636596679688,
      "activations/layer5_attention_weight_max": 57.47159957885742,
      "activations/layer5_attention_weight_min": -68.89965057373047,
      "activations/layer6_attention_weight_max": 51.250160217285156,
      "activations/layer6_attention_weight_min": -47.56654739379883,
      "activations/layer7_attention_weight_max": 62.81135940551758,
      "activations/layer7_attention_weight_min": -62.91699981689453,
      "activations/layer8_attention_weight_max": 43.60881423950195,
      "activations/layer8_attention_weight_min": -45.187992095947266,
      "activations/layer9_attention_weight_max": 41.53697204589844,
      "activations/layer9_attention_weight_min": -46.06618881225586,
      "epoch": 18.7,
      "learning_rate": 2.9700757575757576e-05,
      "loss": 2.7348,
      "step": 321850
    },
    {
      "activations/layer0_attention_weight_max": 14.983809471130371,
      "activations/layer0_attention_weight_min": -14.054357528686523,
      "activations/layer10_attention_weight_max": 36.324745178222656,
      "activations/layer10_attention_weight_min": -37.00381851196289,
      "activations/layer11_attention_weight_max": 31.27800750732422,
      "activations/layer11_attention_weight_min": -32.81596755981445,
      "activations/layer12_attention_weight_max": 19.341188430786133,
      "activations/layer12_attention_weight_min": -27.364933013916016,
      "activations/layer13_attention_weight_max": 31.4140625,
      "activations/layer13_attention_weight_min": -27.184003829956055,
      "activations/layer14_attention_weight_max": 34.40464401245117,
      "activations/layer14_attention_weight_min": -29.918376922607422,
      "activations/layer15_attention_weight_max": 32.33042526245117,
      "activations/layer15_attention_weight_min": -30.88112449645996,
      "activations/layer16_attention_weight_max": 33.634273529052734,
      "activations/layer16_attention_weight_min": -31.352054595947266,
      "activations/layer17_attention_weight_max": 51.63302993774414,
      "activations/layer17_attention_weight_min": -44.0071907043457,
      "activations/layer18_attention_weight_max": 45.099571228027344,
      "activations/layer18_attention_weight_min": -39.98543930053711,
      "activations/layer19_attention_weight_max": 24.4588680267334,
      "activations/layer19_attention_weight_min": -22.959341049194336,
      "activations/layer1_attention_weight_max": 15.767236709594727,
      "activations/layer1_attention_weight_min": -14.391279220581055,
      "activations/layer20_attention_weight_max": 24.456823348999023,
      "activations/layer20_attention_weight_min": -24.348474502563477,
      "activations/layer21_attention_weight_max": 40.364994049072266,
      "activations/layer21_attention_weight_min": -25.019023895263672,
      "activations/layer22_attention_weight_max": 30.94353485107422,
      "activations/layer22_attention_weight_min": -25.703725814819336,
      "activations/layer23_attention_weight_max": 37.742401123046875,
      "activations/layer23_attention_weight_min": -23.890666961669922,
      "activations/layer2_attention_weight_max": 31.841590881347656,
      "activations/layer2_attention_weight_min": -32.01926040649414,
      "activations/layer3_attention_weight_max": 82.6114730834961,
      "activations/layer3_attention_weight_min": -84.39965057373047,
      "activations/layer4_attention_weight_max": 79.72210693359375,
      "activations/layer4_attention_weight_min": -77.99870300292969,
      "activations/layer5_attention_weight_max": 60.34328842163086,
      "activations/layer5_attention_weight_min": -71.6875,
      "activations/layer6_attention_weight_max": 47.07359313964844,
      "activations/layer6_attention_weight_min": -48.37028884887695,
      "activations/layer7_attention_weight_max": 63.10456848144531,
      "activations/layer7_attention_weight_min": -59.520172119140625,
      "activations/layer8_attention_weight_max": 46.13590621948242,
      "activations/layer8_attention_weight_min": -47.62939453125,
      "activations/layer9_attention_weight_max": 44.543785095214844,
      "activations/layer9_attention_weight_min": -43.722286224365234,
      "epoch": 18.7,
      "learning_rate": 2.968181818181818e-05,
      "loss": 2.7285,
      "step": 321900
    },
    {
      "activations/layer0_attention_weight_max": 15.711694717407227,
      "activations/layer0_attention_weight_min": -13.781083106994629,
      "activations/layer10_attention_weight_max": 34.9160041809082,
      "activations/layer10_attention_weight_min": -33.71662521362305,
      "activations/layer11_attention_weight_max": 32.09058380126953,
      "activations/layer11_attention_weight_min": -33.965736389160156,
      "activations/layer12_attention_weight_max": 19.577444076538086,
      "activations/layer12_attention_weight_min": -22.967487335205078,
      "activations/layer13_attention_weight_max": 34.0843391418457,
      "activations/layer13_attention_weight_min": -26.262544631958008,
      "activations/layer14_attention_weight_max": 34.406005859375,
      "activations/layer14_attention_weight_min": -29.576534271240234,
      "activations/layer15_attention_weight_max": 31.510805130004883,
      "activations/layer15_attention_weight_min": -28.83106803894043,
      "activations/layer16_attention_weight_max": 32.210758209228516,
      "activations/layer16_attention_weight_min": -30.333654403686523,
      "activations/layer17_attention_weight_max": 54.05791091918945,
      "activations/layer17_attention_weight_min": -46.688880920410156,
      "activations/layer18_attention_weight_max": 44.69987487792969,
      "activations/layer18_attention_weight_min": -39.5045166015625,
      "activations/layer19_attention_weight_max": 24.426990509033203,
      "activations/layer19_attention_weight_min": -23.848215103149414,
      "activations/layer1_attention_weight_max": 17.254474639892578,
      "activations/layer1_attention_weight_min": -16.503618240356445,
      "activations/layer20_attention_weight_max": 24.82170295715332,
      "activations/layer20_attention_weight_min": -22.98272132873535,
      "activations/layer21_attention_weight_max": 35.13664627075195,
      "activations/layer21_attention_weight_min": -22.381479263305664,
      "activations/layer22_attention_weight_max": 27.3017635345459,
      "activations/layer22_attention_weight_min": -26.478864669799805,
      "activations/layer23_attention_weight_max": 37.28107452392578,
      "activations/layer23_attention_weight_min": -23.815792083740234,
      "activations/layer2_attention_weight_max": 29.00250816345215,
      "activations/layer2_attention_weight_min": -28.406646728515625,
      "activations/layer3_attention_weight_max": 78.33846282958984,
      "activations/layer3_attention_weight_min": -79.98670196533203,
      "activations/layer4_attention_weight_max": 77.74214172363281,
      "activations/layer4_attention_weight_min": -76.89263153076172,
      "activations/layer5_attention_weight_max": 57.74030685424805,
      "activations/layer5_attention_weight_min": -73.9063949584961,
      "activations/layer6_attention_weight_max": 44.829471588134766,
      "activations/layer6_attention_weight_min": -48.15024948120117,
      "activations/layer7_attention_weight_max": 63.99961853027344,
      "activations/layer7_attention_weight_min": -62.82189178466797,
      "activations/layer8_attention_weight_max": 44.150726318359375,
      "activations/layer8_attention_weight_min": -47.16141891479492,
      "activations/layer9_attention_weight_max": 40.85299301147461,
      "activations/layer9_attention_weight_min": -47.31794738769531,
      "epoch": 18.71,
      "learning_rate": 2.9662878787878785e-05,
      "loss": 2.7308,
      "step": 321950
    },
    {
      "activations/layer0_attention_weight_max": 15.250490188598633,
      "activations/layer0_attention_weight_min": -13.787968635559082,
      "activations/layer10_attention_weight_max": 35.00675582885742,
      "activations/layer10_attention_weight_min": -35.28961944580078,
      "activations/layer11_attention_weight_max": 32.50326919555664,
      "activations/layer11_attention_weight_min": -33.61656188964844,
      "activations/layer12_attention_weight_max": 19.101036071777344,
      "activations/layer12_attention_weight_min": -25.064990997314453,
      "activations/layer13_attention_weight_max": 32.74366760253906,
      "activations/layer13_attention_weight_min": -27.191368103027344,
      "activations/layer14_attention_weight_max": 32.896175384521484,
      "activations/layer14_attention_weight_min": -29.103961944580078,
      "activations/layer15_attention_weight_max": 30.421890258789062,
      "activations/layer15_attention_weight_min": -30.942678451538086,
      "activations/layer16_attention_weight_max": 33.59842300415039,
      "activations/layer16_attention_weight_min": -32.495628356933594,
      "activations/layer17_attention_weight_max": 46.54103469848633,
      "activations/layer17_attention_weight_min": -45.31831741333008,
      "activations/layer18_attention_weight_max": 43.88743209838867,
      "activations/layer18_attention_weight_min": -42.883750915527344,
      "activations/layer19_attention_weight_max": 23.23388671875,
      "activations/layer19_attention_weight_min": -25.5837345123291,
      "activations/layer1_attention_weight_max": 15.515921592712402,
      "activations/layer1_attention_weight_min": -15.118597984313965,
      "activations/layer20_attention_weight_max": 20.12338638305664,
      "activations/layer20_attention_weight_min": -24.717466354370117,
      "activations/layer21_attention_weight_max": 32.081764221191406,
      "activations/layer21_attention_weight_min": -22.87876319885254,
      "activations/layer22_attention_weight_max": 28.87847328186035,
      "activations/layer22_attention_weight_min": -27.83456802368164,
      "activations/layer23_attention_weight_max": 33.816261291503906,
      "activations/layer23_attention_weight_min": -24.170377731323242,
      "activations/layer2_attention_weight_max": 35.096641540527344,
      "activations/layer2_attention_weight_min": -31.357421875,
      "activations/layer3_attention_weight_max": 86.57022094726562,
      "activations/layer3_attention_weight_min": -73.56355285644531,
      "activations/layer4_attention_weight_max": 79.26158142089844,
      "activations/layer4_attention_weight_min": -74.00005340576172,
      "activations/layer5_attention_weight_max": 59.36730194091797,
      "activations/layer5_attention_weight_min": -69.2530288696289,
      "activations/layer6_attention_weight_max": 48.11024856567383,
      "activations/layer6_attention_weight_min": -46.24336624145508,
      "activations/layer7_attention_weight_max": 65.47669982910156,
      "activations/layer7_attention_weight_min": -61.06899642944336,
      "activations/layer8_attention_weight_max": 44.669010162353516,
      "activations/layer8_attention_weight_min": -49.39744567871094,
      "activations/layer9_attention_weight_max": 41.6573600769043,
      "activations/layer9_attention_weight_min": -44.58320999145508,
      "epoch": 18.71,
      "learning_rate": 2.964393939393939e-05,
      "loss": 2.7408,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_loss": 2.693359375,
      "eval_runtime": 8.4761,
      "eval_samples_per_second": 506.601,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_openwebtext_loss": 2.693359375,
      "eval_openwebtext_ppl": 14.781248363830914,
      "eval_openwebtext_runtime": 8.4761,
      "eval_openwebtext_samples_per_second": 506.601,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9788,
      "eval_wikitext_samples_per_second": 230.446,
      "step": 322000
    },
    {
      "epoch": 18.71,
      "eval_lambada_loss": 2.484375,
      "eval_lambada_ppl": 11.993621898152476,
      "eval_lambada_runtime": 9.565,
      "eval_lambada_samples_per_second": 509.045,
      "step": 322000
    },
    {
      "activations/layer0_attention_weight_max": 14.75610065460205,
      "activations/layer0_attention_weight_min": -14.46342658996582,
      "activations/layer10_attention_weight_max": 37.3305778503418,
      "activations/layer10_attention_weight_min": -34.30078125,
      "activations/layer11_attention_weight_max": 34.656410217285156,
      "activations/layer11_attention_weight_min": -33.84889221191406,
      "activations/layer12_attention_weight_max": 20.184467315673828,
      "activations/layer12_attention_weight_min": -22.778427124023438,
      "activations/layer13_attention_weight_max": 39.10435485839844,
      "activations/layer13_attention_weight_min": -30.26427459716797,
      "activations/layer14_attention_weight_max": 37.22697830200195,
      "activations/layer14_attention_weight_min": -30.778820037841797,
      "activations/layer15_attention_weight_max": 34.17296600341797,
      "activations/layer15_attention_weight_min": -30.878326416015625,
      "activations/layer16_attention_weight_max": 36.54630661010742,
      "activations/layer16_attention_weight_min": -32.96555709838867,
      "activations/layer17_attention_weight_max": 52.861270904541016,
      "activations/layer17_attention_weight_min": -45.108211517333984,
      "activations/layer18_attention_weight_max": 47.23601150512695,
      "activations/layer18_attention_weight_min": -41.292518615722656,
      "activations/layer19_attention_weight_max": 24.318927764892578,
      "activations/layer19_attention_weight_min": -22.66412925720215,
      "activations/layer1_attention_weight_max": 17.25063705444336,
      "activations/layer1_attention_weight_min": -14.606369972229004,
      "activations/layer20_attention_weight_max": 21.4097957611084,
      "activations/layer20_attention_weight_min": -24.210472106933594,
      "activations/layer21_attention_weight_max": 34.57270050048828,
      "activations/layer21_attention_weight_min": -22.649608612060547,
      "activations/layer22_attention_weight_max": 26.79827880859375,
      "activations/layer22_attention_weight_min": -25.57782554626465,
      "activations/layer23_attention_weight_max": 35.746238708496094,
      "activations/layer23_attention_weight_min": -23.550785064697266,
      "activations/layer2_attention_weight_max": 31.43590545654297,
      "activations/layer2_attention_weight_min": -28.65308952331543,
      "activations/layer3_attention_weight_max": 81.0787124633789,
      "activations/layer3_attention_weight_min": -82.30207824707031,
      "activations/layer4_attention_weight_max": 83.02985382080078,
      "activations/layer4_attention_weight_min": -79.52986145019531,
      "activations/layer5_attention_weight_max": 60.6490478515625,
      "activations/layer5_attention_weight_min": -69.50373077392578,
      "activations/layer6_attention_weight_max": 49.72589874267578,
      "activations/layer6_attention_weight_min": -48.80646514892578,
      "activations/layer7_attention_weight_max": 67.84864044189453,
      "activations/layer7_attention_weight_min": -63.43112564086914,
      "activations/layer8_attention_weight_max": 47.09407043457031,
      "activations/layer8_attention_weight_min": -50.05018615722656,
      "activations/layer9_attention_weight_max": 44.069950103759766,
      "activations/layer9_attention_weight_min": -44.03612518310547,
      "epoch": 18.71,
      "learning_rate": 2.9625e-05,
      "loss": 2.7306,
      "step": 322050
    },
    {
      "activations/layer0_attention_weight_max": 16.23712921142578,
      "activations/layer0_attention_weight_min": -13.901741981506348,
      "activations/layer10_attention_weight_max": 37.83856964111328,
      "activations/layer10_attention_weight_min": -38.44182205200195,
      "activations/layer11_attention_weight_max": 34.25756072998047,
      "activations/layer11_attention_weight_min": -36.88383102416992,
      "activations/layer12_attention_weight_max": 18.706523895263672,
      "activations/layer12_attention_weight_min": -29.384782791137695,
      "activations/layer13_attention_weight_max": 35.6474494934082,
      "activations/layer13_attention_weight_min": -30.35122299194336,
      "activations/layer14_attention_weight_max": 36.081138610839844,
      "activations/layer14_attention_weight_min": -34.14955139160156,
      "activations/layer15_attention_weight_max": 34.841182708740234,
      "activations/layer15_attention_weight_min": -32.596927642822266,
      "activations/layer16_attention_weight_max": 38.79307556152344,
      "activations/layer16_attention_weight_min": -38.17879867553711,
      "activations/layer17_attention_weight_max": 57.51698303222656,
      "activations/layer17_attention_weight_min": -50.950035095214844,
      "activations/layer18_attention_weight_max": 58.3339958190918,
      "activations/layer18_attention_weight_min": -49.90919494628906,
      "activations/layer19_attention_weight_max": 30.719955444335938,
      "activations/layer19_attention_weight_min": -27.565872192382812,
      "activations/layer1_attention_weight_max": 15.923208236694336,
      "activations/layer1_attention_weight_min": -15.640835762023926,
      "activations/layer20_attention_weight_max": 30.945621490478516,
      "activations/layer20_attention_weight_min": -21.826683044433594,
      "activations/layer21_attention_weight_max": 40.40227127075195,
      "activations/layer21_attention_weight_min": -25.237173080444336,
      "activations/layer22_attention_weight_max": 31.7674560546875,
      "activations/layer22_attention_weight_min": -28.8616943359375,
      "activations/layer23_attention_weight_max": 43.56963348388672,
      "activations/layer23_attention_weight_min": -24.81678009033203,
      "activations/layer2_attention_weight_max": 32.28868865966797,
      "activations/layer2_attention_weight_min": -31.046926498413086,
      "activations/layer3_attention_weight_max": 89.99018859863281,
      "activations/layer3_attention_weight_min": -88.88658142089844,
      "activations/layer4_attention_weight_max": 82.68771362304688,
      "activations/layer4_attention_weight_min": -84.14789581298828,
      "activations/layer5_attention_weight_max": 62.16352081298828,
      "activations/layer5_attention_weight_min": -72.16162109375,
      "activations/layer6_attention_weight_max": 50.31718444824219,
      "activations/layer6_attention_weight_min": -50.881553649902344,
      "activations/layer7_attention_weight_max": 63.17092514038086,
      "activations/layer7_attention_weight_min": -70.20206451416016,
      "activations/layer8_attention_weight_max": 48.90797805786133,
      "activations/layer8_attention_weight_min": -49.942039489746094,
      "activations/layer9_attention_weight_max": 46.71175765991211,
      "activations/layer9_attention_weight_min": -46.97776412963867,
      "epoch": 18.72,
      "learning_rate": 2.9606060606060603e-05,
      "loss": 2.729,
      "step": 322100
    },
    {
      "activations/layer0_attention_weight_max": 15.689031600952148,
      "activations/layer0_attention_weight_min": -13.837512969970703,
      "activations/layer10_attention_weight_max": 35.128787994384766,
      "activations/layer10_attention_weight_min": -35.852108001708984,
      "activations/layer11_attention_weight_max": 33.92742919921875,
      "activations/layer11_attention_weight_min": -36.06229019165039,
      "activations/layer12_attention_weight_max": 20.037736892700195,
      "activations/layer12_attention_weight_min": -29.538286209106445,
      "activations/layer13_attention_weight_max": 35.217193603515625,
      "activations/layer13_attention_weight_min": -28.585474014282227,
      "activations/layer14_attention_weight_max": 37.97566604614258,
      "activations/layer14_attention_weight_min": -33.260231018066406,
      "activations/layer15_attention_weight_max": 37.12252426147461,
      "activations/layer15_attention_weight_min": -32.91977310180664,
      "activations/layer16_attention_weight_max": 37.25959777832031,
      "activations/layer16_attention_weight_min": -35.79777908325195,
      "activations/layer17_attention_weight_max": 56.25165939331055,
      "activations/layer17_attention_weight_min": -49.939308166503906,
      "activations/layer18_attention_weight_max": 52.32449722290039,
      "activations/layer18_attention_weight_min": -41.047977447509766,
      "activations/layer19_attention_weight_max": 30.415700912475586,
      "activations/layer19_attention_weight_min": -22.14529800415039,
      "activations/layer1_attention_weight_max": 15.957103729248047,
      "activations/layer1_attention_weight_min": -13.239368438720703,
      "activations/layer20_attention_weight_max": 24.34752655029297,
      "activations/layer20_attention_weight_min": -21.611684799194336,
      "activations/layer21_attention_weight_max": 34.19635009765625,
      "activations/layer21_attention_weight_min": -23.19207763671875,
      "activations/layer22_attention_weight_max": 29.40083885192871,
      "activations/layer22_attention_weight_min": -24.253816604614258,
      "activations/layer23_attention_weight_max": 37.35679626464844,
      "activations/layer23_attention_weight_min": -21.587093353271484,
      "activations/layer2_attention_weight_max": 31.40765953063965,
      "activations/layer2_attention_weight_min": -28.09613037109375,
      "activations/layer3_attention_weight_max": 84.76734924316406,
      "activations/layer3_attention_weight_min": -85.72157287597656,
      "activations/layer4_attention_weight_max": 83.4308090209961,
      "activations/layer4_attention_weight_min": -83.25154113769531,
      "activations/layer5_attention_weight_max": 60.786773681640625,
      "activations/layer5_attention_weight_min": -71.68238830566406,
      "activations/layer6_attention_weight_max": 48.71486282348633,
      "activations/layer6_attention_weight_min": -51.95288848876953,
      "activations/layer7_attention_weight_max": 63.86783981323242,
      "activations/layer7_attention_weight_min": -64.8939208984375,
      "activations/layer8_attention_weight_max": 47.05995178222656,
      "activations/layer8_attention_weight_min": -51.00519561767578,
      "activations/layer9_attention_weight_max": 47.218509674072266,
      "activations/layer9_attention_weight_min": -44.943355560302734,
      "epoch": 18.72,
      "learning_rate": 2.958712121212121e-05,
      "loss": 2.7287,
      "step": 322150
    },
    {
      "activations/layer0_attention_weight_max": 15.30727481842041,
      "activations/layer0_attention_weight_min": -14.004561424255371,
      "activations/layer10_attention_weight_max": 41.097869873046875,
      "activations/layer10_attention_weight_min": -38.10503005981445,
      "activations/layer11_attention_weight_max": 35.04766082763672,
      "activations/layer11_attention_weight_min": -33.57362365722656,
      "activations/layer12_attention_weight_max": 18.927244186401367,
      "activations/layer12_attention_weight_min": -24.08133316040039,
      "activations/layer13_attention_weight_max": 37.2645378112793,
      "activations/layer13_attention_weight_min": -29.195077896118164,
      "activations/layer14_attention_weight_max": 36.169044494628906,
      "activations/layer14_attention_weight_min": -29.27155876159668,
      "activations/layer15_attention_weight_max": 38.853858947753906,
      "activations/layer15_attention_weight_min": -29.258621215820312,
      "activations/layer16_attention_weight_max": 37.46994400024414,
      "activations/layer16_attention_weight_min": -31.084415435791016,
      "activations/layer17_attention_weight_max": 49.782127380371094,
      "activations/layer17_attention_weight_min": -43.110713958740234,
      "activations/layer18_attention_weight_max": 44.19581985473633,
      "activations/layer18_attention_weight_min": -37.19207000732422,
      "activations/layer19_attention_weight_max": 23.66031837463379,
      "activations/layer19_attention_weight_min": -21.706546783447266,
      "activations/layer1_attention_weight_max": 16.785249710083008,
      "activations/layer1_attention_weight_min": -13.395293235778809,
      "activations/layer20_attention_weight_max": 25.062528610229492,
      "activations/layer20_attention_weight_min": -21.351308822631836,
      "activations/layer21_attention_weight_max": 31.40373992919922,
      "activations/layer21_attention_weight_min": -19.339401245117188,
      "activations/layer22_attention_weight_max": 33.51156234741211,
      "activations/layer22_attention_weight_min": -25.84843635559082,
      "activations/layer23_attention_weight_max": 38.33092498779297,
      "activations/layer23_attention_weight_min": -23.62921905517578,
      "activations/layer2_attention_weight_max": 30.713626861572266,
      "activations/layer2_attention_weight_min": -28.781002044677734,
      "activations/layer3_attention_weight_max": 84.90512084960938,
      "activations/layer3_attention_weight_min": -81.92959594726562,
      "activations/layer4_attention_weight_max": 81.90880584716797,
      "activations/layer4_attention_weight_min": -79.9559555053711,
      "activations/layer5_attention_weight_max": 62.15560531616211,
      "activations/layer5_attention_weight_min": -75.67073059082031,
      "activations/layer6_attention_weight_max": 48.120521545410156,
      "activations/layer6_attention_weight_min": -48.111351013183594,
      "activations/layer7_attention_weight_max": 63.82561492919922,
      "activations/layer7_attention_weight_min": -62.410888671875,
      "activations/layer8_attention_weight_max": 47.47785186767578,
      "activations/layer8_attention_weight_min": -50.934532165527344,
      "activations/layer9_attention_weight_max": 45.730751037597656,
      "activations/layer9_attention_weight_min": -45.62386703491211,
      "epoch": 18.72,
      "learning_rate": 2.9568181818181813e-05,
      "loss": 2.7432,
      "step": 322200
    },
    {
      "activations/layer0_attention_weight_max": 15.3468599319458,
      "activations/layer0_attention_weight_min": -14.056421279907227,
      "activations/layer10_attention_weight_max": 33.28029251098633,
      "activations/layer10_attention_weight_min": -32.009979248046875,
      "activations/layer11_attention_weight_max": 31.484350204467773,
      "activations/layer11_attention_weight_min": -32.16118240356445,
      "activations/layer12_attention_weight_max": 18.296796798706055,
      "activations/layer12_attention_weight_min": -31.395519256591797,
      "activations/layer13_attention_weight_max": 32.72053146362305,
      "activations/layer13_attention_weight_min": -27.953100204467773,
      "activations/layer14_attention_weight_max": 33.84760284423828,
      "activations/layer14_attention_weight_min": -30.37396240234375,
      "activations/layer15_attention_weight_max": 31.09816551208496,
      "activations/layer15_attention_weight_min": -31.484474182128906,
      "activations/layer16_attention_weight_max": 32.2220573425293,
      "activations/layer16_attention_weight_min": -31.358118057250977,
      "activations/layer17_attention_weight_max": 46.553367614746094,
      "activations/layer17_attention_weight_min": -43.43962860107422,
      "activations/layer18_attention_weight_max": 43.411224365234375,
      "activations/layer18_attention_weight_min": -39.642608642578125,
      "activations/layer19_attention_weight_max": 22.558395385742188,
      "activations/layer19_attention_weight_min": -22.844446182250977,
      "activations/layer1_attention_weight_max": 16.553253173828125,
      "activations/layer1_attention_weight_min": -13.634100914001465,
      "activations/layer20_attention_weight_max": 21.912879943847656,
      "activations/layer20_attention_weight_min": -23.1220703125,
      "activations/layer21_attention_weight_max": 37.603736877441406,
      "activations/layer21_attention_weight_min": -25.995361328125,
      "activations/layer22_attention_weight_max": 27.7664852142334,
      "activations/layer22_attention_weight_min": -25.537841796875,
      "activations/layer23_attention_weight_max": 32.727989196777344,
      "activations/layer23_attention_weight_min": -24.098325729370117,
      "activations/layer2_attention_weight_max": 31.67097282409668,
      "activations/layer2_attention_weight_min": -29.459468841552734,
      "activations/layer3_attention_weight_max": 82.88783264160156,
      "activations/layer3_attention_weight_min": -84.6649398803711,
      "activations/layer4_attention_weight_max": 81.28618621826172,
      "activations/layer4_attention_weight_min": -81.46501922607422,
      "activations/layer5_attention_weight_max": 65.88946533203125,
      "activations/layer5_attention_weight_min": -69.32669067382812,
      "activations/layer6_attention_weight_max": 48.48503112792969,
      "activations/layer6_attention_weight_min": -48.64778137207031,
      "activations/layer7_attention_weight_max": 65.89045715332031,
      "activations/layer7_attention_weight_min": -62.477394104003906,
      "activations/layer8_attention_weight_max": 42.97397994995117,
      "activations/layer8_attention_weight_min": -44.71171951293945,
      "activations/layer9_attention_weight_max": 44.38433074951172,
      "activations/layer9_attention_weight_min": -46.100372314453125,
      "epoch": 18.72,
      "learning_rate": 2.954924242424242e-05,
      "loss": 2.7513,
      "step": 322250
    },
    {
      "activations/layer0_attention_weight_max": 15.068944931030273,
      "activations/layer0_attention_weight_min": -13.890109062194824,
      "activations/layer10_attention_weight_max": 39.03907775878906,
      "activations/layer10_attention_weight_min": -39.895137786865234,
      "activations/layer11_attention_weight_max": 35.45109939575195,
      "activations/layer11_attention_weight_min": -39.692604064941406,
      "activations/layer12_attention_weight_max": 18.789180755615234,
      "activations/layer12_attention_weight_min": -26.34804916381836,
      "activations/layer13_attention_weight_max": 36.15618133544922,
      "activations/layer13_attention_weight_min": -30.370258331298828,
      "activations/layer14_attention_weight_max": 36.40846633911133,
      "activations/layer14_attention_weight_min": -31.952861785888672,
      "activations/layer15_attention_weight_max": 33.68505859375,
      "activations/layer15_attention_weight_min": -30.03569793701172,
      "activations/layer16_attention_weight_max": 35.38055419921875,
      "activations/layer16_attention_weight_min": -33.60993576049805,
      "activations/layer17_attention_weight_max": 52.96379470825195,
      "activations/layer17_attention_weight_min": -44.36896896362305,
      "activations/layer18_attention_weight_max": 46.46149826049805,
      "activations/layer18_attention_weight_min": -43.66720962524414,
      "activations/layer19_attention_weight_max": 25.596797943115234,
      "activations/layer19_attention_weight_min": -23.347898483276367,
      "activations/layer1_attention_weight_max": 16.08463478088379,
      "activations/layer1_attention_weight_min": -12.841849327087402,
      "activations/layer20_attention_weight_max": 23.362951278686523,
      "activations/layer20_attention_weight_min": -22.475173950195312,
      "activations/layer21_attention_weight_max": 37.82570266723633,
      "activations/layer21_attention_weight_min": -23.651700973510742,
      "activations/layer22_attention_weight_max": 28.817646026611328,
      "activations/layer22_attention_weight_min": -25.84647560119629,
      "activations/layer23_attention_weight_max": 33.97173309326172,
      "activations/layer23_attention_weight_min": -25.55929183959961,
      "activations/layer2_attention_weight_max": 32.821022033691406,
      "activations/layer2_attention_weight_min": -31.308021545410156,
      "activations/layer3_attention_weight_max": 90.78556823730469,
      "activations/layer3_attention_weight_min": -93.45098876953125,
      "activations/layer4_attention_weight_max": 85.46027374267578,
      "activations/layer4_attention_weight_min": -91.45764923095703,
      "activations/layer5_attention_weight_max": 65.8088150024414,
      "activations/layer5_attention_weight_min": -73.3379135131836,
      "activations/layer6_attention_weight_max": 54.45764923095703,
      "activations/layer6_attention_weight_min": -52.164649963378906,
      "activations/layer7_attention_weight_max": 72.28041076660156,
      "activations/layer7_attention_weight_min": -63.83979034423828,
      "activations/layer8_attention_weight_max": 49.5528678894043,
      "activations/layer8_attention_weight_min": -50.94074630737305,
      "activations/layer9_attention_weight_max": 45.31438064575195,
      "activations/layer9_attention_weight_min": -47.64048385620117,
      "epoch": 18.73,
      "learning_rate": 2.9530303030303026e-05,
      "loss": 2.7254,
      "step": 322300
    },
    {
      "activations/layer0_attention_weight_max": 16.089841842651367,
      "activations/layer0_attention_weight_min": -13.977734565734863,
      "activations/layer10_attention_weight_max": 40.298095703125,
      "activations/layer10_attention_weight_min": -40.8042106628418,
      "activations/layer11_attention_weight_max": 41.69203186035156,
      "activations/layer11_attention_weight_min": -36.52409362792969,
      "activations/layer12_attention_weight_max": 18.11555290222168,
      "activations/layer12_attention_weight_min": -27.26170539855957,
      "activations/layer13_attention_weight_max": 35.81649398803711,
      "activations/layer13_attention_weight_min": -25.70328712463379,
      "activations/layer14_attention_weight_max": 35.05866241455078,
      "activations/layer14_attention_weight_min": -30.096553802490234,
      "activations/layer15_attention_weight_max": 35.517147064208984,
      "activations/layer15_attention_weight_min": -28.593076705932617,
      "activations/layer16_attention_weight_max": 33.70494079589844,
      "activations/layer16_attention_weight_min": -33.96706771850586,
      "activations/layer17_attention_weight_max": 48.83626174926758,
      "activations/layer17_attention_weight_min": -44.69327163696289,
      "activations/layer18_attention_weight_max": 46.15009689331055,
      "activations/layer18_attention_weight_min": -38.68019104003906,
      "activations/layer19_attention_weight_max": 23.07103729248047,
      "activations/layer19_attention_weight_min": -23.564373016357422,
      "activations/layer1_attention_weight_max": 15.972166061401367,
      "activations/layer1_attention_weight_min": -13.938883781433105,
      "activations/layer20_attention_weight_max": 23.980289459228516,
      "activations/layer20_attention_weight_min": -21.23236656188965,
      "activations/layer21_attention_weight_max": 33.97636795043945,
      "activations/layer21_attention_weight_min": -21.732776641845703,
      "activations/layer22_attention_weight_max": 34.13078689575195,
      "activations/layer22_attention_weight_min": -23.497007369995117,
      "activations/layer23_attention_weight_max": 37.763458251953125,
      "activations/layer23_attention_weight_min": -22.409391403198242,
      "activations/layer2_attention_weight_max": 33.257999420166016,
      "activations/layer2_attention_weight_min": -31.159767150878906,
      "activations/layer3_attention_weight_max": 86.25337982177734,
      "activations/layer3_attention_weight_min": -89.39894104003906,
      "activations/layer4_attention_weight_max": 85.5384750366211,
      "activations/layer4_attention_weight_min": -87.94512176513672,
      "activations/layer5_attention_weight_max": 59.294097900390625,
      "activations/layer5_attention_weight_min": -70.55670928955078,
      "activations/layer6_attention_weight_max": 49.792110443115234,
      "activations/layer6_attention_weight_min": -50.71477127075195,
      "activations/layer7_attention_weight_max": 71.0121078491211,
      "activations/layer7_attention_weight_min": -71.45331573486328,
      "activations/layer8_attention_weight_max": 56.03790283203125,
      "activations/layer8_attention_weight_min": -50.79157638549805,
      "activations/layer9_attention_weight_max": 55.118865966796875,
      "activations/layer9_attention_weight_min": -50.034358978271484,
      "epoch": 18.73,
      "learning_rate": 2.9511363636363634e-05,
      "loss": 2.7204,
      "step": 322350
    },
    {
      "activations/layer0_attention_weight_max": 15.301857948303223,
      "activations/layer0_attention_weight_min": -13.787981986999512,
      "activations/layer10_attention_weight_max": 36.10432434082031,
      "activations/layer10_attention_weight_min": -36.248451232910156,
      "activations/layer11_attention_weight_max": 32.58319091796875,
      "activations/layer11_attention_weight_min": -36.45027160644531,
      "activations/layer12_attention_weight_max": 19.824655532836914,
      "activations/layer12_attention_weight_min": -25.09676742553711,
      "activations/layer13_attention_weight_max": 33.08096694946289,
      "activations/layer13_attention_weight_min": -27.600595474243164,
      "activations/layer14_attention_weight_max": 34.897972106933594,
      "activations/layer14_attention_weight_min": -31.759275436401367,
      "activations/layer15_attention_weight_max": 31.431421279907227,
      "activations/layer15_attention_weight_min": -30.77341079711914,
      "activations/layer16_attention_weight_max": 34.94286346435547,
      "activations/layer16_attention_weight_min": -33.08140182495117,
      "activations/layer17_attention_weight_max": 51.90120315551758,
      "activations/layer17_attention_weight_min": -46.63631820678711,
      "activations/layer18_attention_weight_max": 48.459373474121094,
      "activations/layer18_attention_weight_min": -40.71648406982422,
      "activations/layer19_attention_weight_max": 25.212810516357422,
      "activations/layer19_attention_weight_min": -24.808631896972656,
      "activations/layer1_attention_weight_max": 15.585041999816895,
      "activations/layer1_attention_weight_min": -14.915261268615723,
      "activations/layer20_attention_weight_max": 24.450088500976562,
      "activations/layer20_attention_weight_min": -21.4298152923584,
      "activations/layer21_attention_weight_max": 36.99073791503906,
      "activations/layer21_attention_weight_min": -20.912368774414062,
      "activations/layer22_attention_weight_max": 26.22440528869629,
      "activations/layer22_attention_weight_min": -25.559961318969727,
      "activations/layer23_attention_weight_max": 37.14631271362305,
      "activations/layer23_attention_weight_min": -22.665538787841797,
      "activations/layer2_attention_weight_max": 29.268802642822266,
      "activations/layer2_attention_weight_min": -29.67438316345215,
      "activations/layer3_attention_weight_max": 79.38351440429688,
      "activations/layer3_attention_weight_min": -83.12451934814453,
      "activations/layer4_attention_weight_max": 81.85232543945312,
      "activations/layer4_attention_weight_min": -81.71475219726562,
      "activations/layer5_attention_weight_max": 61.06549835205078,
      "activations/layer5_attention_weight_min": -76.73374938964844,
      "activations/layer6_attention_weight_max": 48.017154693603516,
      "activations/layer6_attention_weight_min": -49.20121765136719,
      "activations/layer7_attention_weight_max": 64.17598724365234,
      "activations/layer7_attention_weight_min": -61.684513092041016,
      "activations/layer8_attention_weight_max": 44.32495880126953,
      "activations/layer8_attention_weight_min": -46.71062088012695,
      "activations/layer9_attention_weight_max": 41.453773498535156,
      "activations/layer9_attention_weight_min": -42.97415542602539,
      "epoch": 18.73,
      "learning_rate": 2.949242424242424e-05,
      "loss": 2.743,
      "step": 322400
    },
    {
      "activations/layer0_attention_weight_max": 17.655227661132812,
      "activations/layer0_attention_weight_min": -14.249566078186035,
      "activations/layer10_attention_weight_max": 34.359615325927734,
      "activations/layer10_attention_weight_min": -35.12872314453125,
      "activations/layer11_attention_weight_max": 31.369701385498047,
      "activations/layer11_attention_weight_min": -33.510318756103516,
      "activations/layer12_attention_weight_max": 18.464126586914062,
      "activations/layer12_attention_weight_min": -23.637659072875977,
      "activations/layer13_attention_weight_max": 32.30197525024414,
      "activations/layer13_attention_weight_min": -25.588838577270508,
      "activations/layer14_attention_weight_max": 34.63260269165039,
      "activations/layer14_attention_weight_min": -29.86268424987793,
      "activations/layer15_attention_weight_max": 32.978824615478516,
      "activations/layer15_attention_weight_min": -29.55438995361328,
      "activations/layer16_attention_weight_max": 32.22030258178711,
      "activations/layer16_attention_weight_min": -30.107969284057617,
      "activations/layer17_attention_weight_max": 49.222808837890625,
      "activations/layer17_attention_weight_min": -45.66890335083008,
      "activations/layer18_attention_weight_max": 47.31064987182617,
      "activations/layer18_attention_weight_min": -41.487220764160156,
      "activations/layer19_attention_weight_max": 25.62629508972168,
      "activations/layer19_attention_weight_min": -25.43156623840332,
      "activations/layer1_attention_weight_max": 17.060691833496094,
      "activations/layer1_attention_weight_min": -15.575779914855957,
      "activations/layer20_attention_weight_max": 22.92181396484375,
      "activations/layer20_attention_weight_min": -22.79262924194336,
      "activations/layer21_attention_weight_max": 37.23573303222656,
      "activations/layer21_attention_weight_min": -20.601055145263672,
      "activations/layer22_attention_weight_max": 27.68128204345703,
      "activations/layer22_attention_weight_min": -25.55496597290039,
      "activations/layer23_attention_weight_max": 36.34272384643555,
      "activations/layer23_attention_weight_min": -23.689346313476562,
      "activations/layer2_attention_weight_max": 32.62254333496094,
      "activations/layer2_attention_weight_min": -30.85004234313965,
      "activations/layer3_attention_weight_max": 82.84696197509766,
      "activations/layer3_attention_weight_min": -83.41299438476562,
      "activations/layer4_attention_weight_max": 81.7628173828125,
      "activations/layer4_attention_weight_min": -84.2524185180664,
      "activations/layer5_attention_weight_max": 62.635040283203125,
      "activations/layer5_attention_weight_min": -73.02749633789062,
      "activations/layer6_attention_weight_max": 48.51069641113281,
      "activations/layer6_attention_weight_min": -50.92306900024414,
      "activations/layer7_attention_weight_max": 66.57735443115234,
      "activations/layer7_attention_weight_min": -62.00680923461914,
      "activations/layer8_attention_weight_max": 45.49554443359375,
      "activations/layer8_attention_weight_min": -50.506038665771484,
      "activations/layer9_attention_weight_max": 41.727909088134766,
      "activations/layer9_attention_weight_min": -45.099998474121094,
      "epoch": 18.74,
      "learning_rate": 2.9473484848484848e-05,
      "loss": 2.7256,
      "step": 322450
    },
    {
      "activations/layer0_attention_weight_max": 15.806536674499512,
      "activations/layer0_attention_weight_min": -14.163818359375,
      "activations/layer10_attention_weight_max": 35.56534957885742,
      "activations/layer10_attention_weight_min": -34.660194396972656,
      "activations/layer11_attention_weight_max": 33.59514617919922,
      "activations/layer11_attention_weight_min": -35.23914337158203,
      "activations/layer12_attention_weight_max": 19.111236572265625,
      "activations/layer12_attention_weight_min": -27.671533584594727,
      "activations/layer13_attention_weight_max": 31.628082275390625,
      "activations/layer13_attention_weight_min": -27.702810287475586,
      "activations/layer14_attention_weight_max": 34.51678466796875,
      "activations/layer14_attention_weight_min": -30.672325134277344,
      "activations/layer15_attention_weight_max": 31.411895751953125,
      "activations/layer15_attention_weight_min": -29.21790885925293,
      "activations/layer16_attention_weight_max": 33.90420913696289,
      "activations/layer16_attention_weight_min": -31.817838668823242,
      "activations/layer17_attention_weight_max": 52.111053466796875,
      "activations/layer17_attention_weight_min": -43.50988006591797,
      "activations/layer18_attention_weight_max": 46.304534912109375,
      "activations/layer18_attention_weight_min": -41.17630386352539,
      "activations/layer19_attention_weight_max": 24.06615447998047,
      "activations/layer19_attention_weight_min": -21.80928611755371,
      "activations/layer1_attention_weight_max": 15.834101676940918,
      "activations/layer1_attention_weight_min": -13.908025741577148,
      "activations/layer20_attention_weight_max": 22.095239639282227,
      "activations/layer20_attention_weight_min": -20.71257209777832,
      "activations/layer21_attention_weight_max": 34.72222900390625,
      "activations/layer21_attention_weight_min": -23.086206436157227,
      "activations/layer22_attention_weight_max": 29.728404998779297,
      "activations/layer22_attention_weight_min": -25.45829963684082,
      "activations/layer23_attention_weight_max": 36.242069244384766,
      "activations/layer23_attention_weight_min": -23.704296112060547,
      "activations/layer2_attention_weight_max": 31.368244171142578,
      "activations/layer2_attention_weight_min": -29.452878952026367,
      "activations/layer3_attention_weight_max": 85.3641357421875,
      "activations/layer3_attention_weight_min": -86.4713134765625,
      "activations/layer4_attention_weight_max": 83.37115478515625,
      "activations/layer4_attention_weight_min": -81.56402587890625,
      "activations/layer5_attention_weight_max": 60.19196701049805,
      "activations/layer5_attention_weight_min": -70.96580505371094,
      "activations/layer6_attention_weight_max": 48.96043395996094,
      "activations/layer6_attention_weight_min": -52.35100555419922,
      "activations/layer7_attention_weight_max": 64.31887817382812,
      "activations/layer7_attention_weight_min": -65.92028045654297,
      "activations/layer8_attention_weight_max": 46.966896057128906,
      "activations/layer8_attention_weight_min": -48.9799690246582,
      "activations/layer9_attention_weight_max": 43.2686653137207,
      "activations/layer9_attention_weight_min": -44.92325973510742,
      "epoch": 18.74,
      "learning_rate": 2.945454545454545e-05,
      "loss": 2.7395,
      "step": 322500
    },
    {
      "activations/layer0_attention_weight_max": 15.08419132232666,
      "activations/layer0_attention_weight_min": -14.218523979187012,
      "activations/layer10_attention_weight_max": 35.12284469604492,
      "activations/layer10_attention_weight_min": -36.9067268371582,
      "activations/layer11_attention_weight_max": 32.01163864135742,
      "activations/layer11_attention_weight_min": -32.98768615722656,
      "activations/layer12_attention_weight_max": 20.24652862548828,
      "activations/layer12_attention_weight_min": -24.44286346435547,
      "activations/layer13_attention_weight_max": 35.81020736694336,
      "activations/layer13_attention_weight_min": -27.894479751586914,
      "activations/layer14_attention_weight_max": 37.19557189941406,
      "activations/layer14_attention_weight_min": -29.856664657592773,
      "activations/layer15_attention_weight_max": 33.81684494018555,
      "activations/layer15_attention_weight_min": -30.312976837158203,
      "activations/layer16_attention_weight_max": 36.424129486083984,
      "activations/layer16_attention_weight_min": -32.38203430175781,
      "activations/layer17_attention_weight_max": 53.17825698852539,
      "activations/layer17_attention_weight_min": -45.59431838989258,
      "activations/layer18_attention_weight_max": 54.07030487060547,
      "activations/layer18_attention_weight_min": -42.801448822021484,
      "activations/layer19_attention_weight_max": 26.445035934448242,
      "activations/layer19_attention_weight_min": -24.67378044128418,
      "activations/layer1_attention_weight_max": 15.553342819213867,
      "activations/layer1_attention_weight_min": -13.672096252441406,
      "activations/layer20_attention_weight_max": 22.27840805053711,
      "activations/layer20_attention_weight_min": -22.816429138183594,
      "activations/layer21_attention_weight_max": 39.440216064453125,
      "activations/layer21_attention_weight_min": -21.285837173461914,
      "activations/layer22_attention_weight_max": 28.555479049682617,
      "activations/layer22_attention_weight_min": -26.801639556884766,
      "activations/layer23_attention_weight_max": 38.408905029296875,
      "activations/layer23_attention_weight_min": -27.167095184326172,
      "activations/layer2_attention_weight_max": 31.841022491455078,
      "activations/layer2_attention_weight_min": -29.609128952026367,
      "activations/layer3_attention_weight_max": 83.33262634277344,
      "activations/layer3_attention_weight_min": -86.60636138916016,
      "activations/layer4_attention_weight_max": 83.71539306640625,
      "activations/layer4_attention_weight_min": -81.5559310913086,
      "activations/layer5_attention_weight_max": 63.233821868896484,
      "activations/layer5_attention_weight_min": -72.25885772705078,
      "activations/layer6_attention_weight_max": 49.531558990478516,
      "activations/layer6_attention_weight_min": -49.74143981933594,
      "activations/layer7_attention_weight_max": 63.30154037475586,
      "activations/layer7_attention_weight_min": -66.57073211669922,
      "activations/layer8_attention_weight_max": 49.170169830322266,
      "activations/layer8_attention_weight_min": -48.23705291748047,
      "activations/layer9_attention_weight_max": 43.37123107910156,
      "activations/layer9_attention_weight_min": -45.40559005737305,
      "epoch": 18.74,
      "learning_rate": 2.9435606060606057e-05,
      "loss": 2.733,
      "step": 322550
    },
    {
      "activations/layer0_attention_weight_max": 15.70226001739502,
      "activations/layer0_attention_weight_min": -14.079033851623535,
      "activations/layer10_attention_weight_max": 38.14068603515625,
      "activations/layer10_attention_weight_min": -38.53308868408203,
      "activations/layer11_attention_weight_max": 36.875953674316406,
      "activations/layer11_attention_weight_min": -35.56700897216797,
      "activations/layer12_attention_weight_max": 18.474191665649414,
      "activations/layer12_attention_weight_min": -25.564672470092773,
      "activations/layer13_attention_weight_max": 34.18203353881836,
      "activations/layer13_attention_weight_min": -27.388832092285156,
      "activations/layer14_attention_weight_max": 40.9954719543457,
      "activations/layer14_attention_weight_min": -31.656042098999023,
      "activations/layer15_attention_weight_max": 37.71958541870117,
      "activations/layer15_attention_weight_min": -32.945682525634766,
      "activations/layer16_attention_weight_max": 38.89812469482422,
      "activations/layer16_attention_weight_min": -33.965335845947266,
      "activations/layer17_attention_weight_max": 56.91291046142578,
      "activations/layer17_attention_weight_min": -45.71278381347656,
      "activations/layer18_attention_weight_max": 54.76525115966797,
      "activations/layer18_attention_weight_min": -40.970760345458984,
      "activations/layer19_attention_weight_max": 31.078908920288086,
      "activations/layer19_attention_weight_min": -26.027795791625977,
      "activations/layer1_attention_weight_max": 15.686850547790527,
      "activations/layer1_attention_weight_min": -14.982354164123535,
      "activations/layer20_attention_weight_max": 28.74215316772461,
      "activations/layer20_attention_weight_min": -22.888437271118164,
      "activations/layer21_attention_weight_max": 42.048301696777344,
      "activations/layer21_attention_weight_min": -25.598941802978516,
      "activations/layer22_attention_weight_max": 31.721927642822266,
      "activations/layer22_attention_weight_min": -25.928884506225586,
      "activations/layer23_attention_weight_max": 39.483673095703125,
      "activations/layer23_attention_weight_min": -24.093379974365234,
      "activations/layer2_attention_weight_max": 35.15447235107422,
      "activations/layer2_attention_weight_min": -32.4849739074707,
      "activations/layer3_attention_weight_max": 88.49413299560547,
      "activations/layer3_attention_weight_min": -90.45170593261719,
      "activations/layer4_attention_weight_max": 83.57199096679688,
      "activations/layer4_attention_weight_min": -82.97904205322266,
      "activations/layer5_attention_weight_max": 61.54298400878906,
      "activations/layer5_attention_weight_min": -72.56449890136719,
      "activations/layer6_attention_weight_max": 55.5399055480957,
      "activations/layer6_attention_weight_min": -51.678382873535156,
      "activations/layer7_attention_weight_max": 71.50254821777344,
      "activations/layer7_attention_weight_min": -70.12901306152344,
      "activations/layer8_attention_weight_max": 51.39735794067383,
      "activations/layer8_attention_weight_min": -53.65842056274414,
      "activations/layer9_attention_weight_max": 53.26514434814453,
      "activations/layer9_attention_weight_min": -47.798431396484375,
      "epoch": 18.74,
      "learning_rate": 2.9416666666666666e-05,
      "loss": 2.7388,
      "step": 322600
    },
    {
      "activations/layer0_attention_weight_max": 16.67999839782715,
      "activations/layer0_attention_weight_min": -14.074503898620605,
      "activations/layer10_attention_weight_max": 39.62796401977539,
      "activations/layer10_attention_weight_min": -37.6671028137207,
      "activations/layer11_attention_weight_max": 34.37895965576172,
      "activations/layer11_attention_weight_min": -34.94893264770508,
      "activations/layer12_attention_weight_max": 19.99127197265625,
      "activations/layer12_attention_weight_min": -24.20069122314453,
      "activations/layer13_attention_weight_max": 42.6730842590332,
      "activations/layer13_attention_weight_min": -29.048660278320312,
      "activations/layer14_attention_weight_max": 42.605323791503906,
      "activations/layer14_attention_weight_min": -30.74728012084961,
      "activations/layer15_attention_weight_max": 38.0108528137207,
      "activations/layer15_attention_weight_min": -32.121334075927734,
      "activations/layer16_attention_weight_max": 36.775604248046875,
      "activations/layer16_attention_weight_min": -35.5749397277832,
      "activations/layer17_attention_weight_max": 55.05208206176758,
      "activations/layer17_attention_weight_min": -50.292598724365234,
      "activations/layer18_attention_weight_max": 49.17343521118164,
      "activations/layer18_attention_weight_min": -44.8433952331543,
      "activations/layer19_attention_weight_max": 24.67473793029785,
      "activations/layer19_attention_weight_min": -23.77950668334961,
      "activations/layer1_attention_weight_max": 15.519148826599121,
      "activations/layer1_attention_weight_min": -13.44605541229248,
      "activations/layer20_attention_weight_max": 22.929838180541992,
      "activations/layer20_attention_weight_min": -20.917694091796875,
      "activations/layer21_attention_weight_max": 36.18878936767578,
      "activations/layer21_attention_weight_min": -22.03701400756836,
      "activations/layer22_attention_weight_max": 29.13114356994629,
      "activations/layer22_attention_weight_min": -26.057580947875977,
      "activations/layer23_attention_weight_max": 36.421775817871094,
      "activations/layer23_attention_weight_min": -25.08501434326172,
      "activations/layer2_attention_weight_max": 32.649879455566406,
      "activations/layer2_attention_weight_min": -31.22902488708496,
      "activations/layer3_attention_weight_max": 85.09709167480469,
      "activations/layer3_attention_weight_min": -88.61177825927734,
      "activations/layer4_attention_weight_max": 81.42996978759766,
      "activations/layer4_attention_weight_min": -81.25605773925781,
      "activations/layer5_attention_weight_max": 61.60930633544922,
      "activations/layer5_attention_weight_min": -71.82093048095703,
      "activations/layer6_attention_weight_max": 51.327980041503906,
      "activations/layer6_attention_weight_min": -50.14638900756836,
      "activations/layer7_attention_weight_max": 68.86605834960938,
      "activations/layer7_attention_weight_min": -61.57163619995117,
      "activations/layer8_attention_weight_max": 47.83298110961914,
      "activations/layer8_attention_weight_min": -51.47444534301758,
      "activations/layer9_attention_weight_max": 45.14625930786133,
      "activations/layer9_attention_weight_min": -47.729793548583984,
      "epoch": 18.75,
      "learning_rate": 2.939772727272727e-05,
      "loss": 2.7257,
      "step": 322650
    },
    {
      "activations/layer0_attention_weight_max": 16.80281639099121,
      "activations/layer0_attention_weight_min": -13.958380699157715,
      "activations/layer10_attention_weight_max": 35.110816955566406,
      "activations/layer10_attention_weight_min": -34.787296295166016,
      "activations/layer11_attention_weight_max": 33.598480224609375,
      "activations/layer11_attention_weight_min": -34.69179916381836,
      "activations/layer12_attention_weight_max": 18.14628028869629,
      "activations/layer12_attention_weight_min": -23.30902671813965,
      "activations/layer13_attention_weight_max": 32.99805450439453,
      "activations/layer13_attention_weight_min": -27.288074493408203,
      "activations/layer14_attention_weight_max": 35.1817741394043,
      "activations/layer14_attention_weight_min": -29.16431427001953,
      "activations/layer15_attention_weight_max": 32.26897430419922,
      "activations/layer15_attention_weight_min": -29.75940704345703,
      "activations/layer16_attention_weight_max": 33.50052261352539,
      "activations/layer16_attention_weight_min": -31.42767333984375,
      "activations/layer17_attention_weight_max": 52.89833450317383,
      "activations/layer17_attention_weight_min": -46.19857406616211,
      "activations/layer18_attention_weight_max": 46.449459075927734,
      "activations/layer18_attention_weight_min": -38.41680145263672,
      "activations/layer19_attention_weight_max": 24.49907875061035,
      "activations/layer19_attention_weight_min": -22.164154052734375,
      "activations/layer1_attention_weight_max": 15.806584358215332,
      "activations/layer1_attention_weight_min": -13.95140552520752,
      "activations/layer20_attention_weight_max": 22.399782180786133,
      "activations/layer20_attention_weight_min": -22.846454620361328,
      "activations/layer21_attention_weight_max": 35.74875259399414,
      "activations/layer21_attention_weight_min": -22.183725357055664,
      "activations/layer22_attention_weight_max": 26.389083862304688,
      "activations/layer22_attention_weight_min": -24.4642333984375,
      "activations/layer23_attention_weight_max": 34.68406295776367,
      "activations/layer23_attention_weight_min": -22.71056365966797,
      "activations/layer2_attention_weight_max": 32.56895065307617,
      "activations/layer2_attention_weight_min": -31.172962188720703,
      "activations/layer3_attention_weight_max": 84.64508819580078,
      "activations/layer3_attention_weight_min": -86.74441528320312,
      "activations/layer4_attention_weight_max": 84.51390075683594,
      "activations/layer4_attention_weight_min": -82.49732208251953,
      "activations/layer5_attention_weight_max": 62.4681510925293,
      "activations/layer5_attention_weight_min": -76.43197631835938,
      "activations/layer6_attention_weight_max": 49.156455993652344,
      "activations/layer6_attention_weight_min": -50.81241989135742,
      "activations/layer7_attention_weight_max": 65.1738052368164,
      "activations/layer7_attention_weight_min": -61.99425506591797,
      "activations/layer8_attention_weight_max": 46.65972900390625,
      "activations/layer8_attention_weight_min": -46.40272903442383,
      "activations/layer9_attention_weight_max": 41.73501205444336,
      "activations/layer9_attention_weight_min": -44.121002197265625,
      "epoch": 18.75,
      "learning_rate": 2.937878787878788e-05,
      "loss": 2.7339,
      "step": 322700
    },
    {
      "activations/layer0_attention_weight_max": 15.935282707214355,
      "activations/layer0_attention_weight_min": -14.098306655883789,
      "activations/layer10_attention_weight_max": 34.20394515991211,
      "activations/layer10_attention_weight_min": -35.38838577270508,
      "activations/layer11_attention_weight_max": 30.370450973510742,
      "activations/layer11_attention_weight_min": -33.4504280090332,
      "activations/layer12_attention_weight_max": 18.126596450805664,
      "activations/layer12_attention_weight_min": -27.013751983642578,
      "activations/layer13_attention_weight_max": 32.252052307128906,
      "activations/layer13_attention_weight_min": -26.25516128540039,
      "activations/layer14_attention_weight_max": 32.49177169799805,
      "activations/layer14_attention_weight_min": -30.479890823364258,
      "activations/layer15_attention_weight_max": 31.104921340942383,
      "activations/layer15_attention_weight_min": -32.20351791381836,
      "activations/layer16_attention_weight_max": 32.11436080932617,
      "activations/layer16_attention_weight_min": -30.87171173095703,
      "activations/layer17_attention_weight_max": 49.45721435546875,
      "activations/layer17_attention_weight_min": -43.424720764160156,
      "activations/layer18_attention_weight_max": 43.99012756347656,
      "activations/layer18_attention_weight_min": -37.88206481933594,
      "activations/layer19_attention_weight_max": 21.394742965698242,
      "activations/layer19_attention_weight_min": -21.64576530456543,
      "activations/layer1_attention_weight_max": 16.443737030029297,
      "activations/layer1_attention_weight_min": -14.604537010192871,
      "activations/layer20_attention_weight_max": 21.182640075683594,
      "activations/layer20_attention_weight_min": -21.654537200927734,
      "activations/layer21_attention_weight_max": 30.63993263244629,
      "activations/layer21_attention_weight_min": -22.56524658203125,
      "activations/layer22_attention_weight_max": 28.04138946533203,
      "activations/layer22_attention_weight_min": -24.85556983947754,
      "activations/layer23_attention_weight_max": 35.27381134033203,
      "activations/layer23_attention_weight_min": -27.152408599853516,
      "activations/layer2_attention_weight_max": 29.32850456237793,
      "activations/layer2_attention_weight_min": -29.336570739746094,
      "activations/layer3_attention_weight_max": 81.4674301147461,
      "activations/layer3_attention_weight_min": -81.95858764648438,
      "activations/layer4_attention_weight_max": 77.06748962402344,
      "activations/layer4_attention_weight_min": -76.33023071289062,
      "activations/layer5_attention_weight_max": 57.92052459716797,
      "activations/layer5_attention_weight_min": -70.99403381347656,
      "activations/layer6_attention_weight_max": 47.179771423339844,
      "activations/layer6_attention_weight_min": -49.36989212036133,
      "activations/layer7_attention_weight_max": 62.69544219970703,
      "activations/layer7_attention_weight_min": -59.63697814941406,
      "activations/layer8_attention_weight_max": 42.270809173583984,
      "activations/layer8_attention_weight_min": -46.290279388427734,
      "activations/layer9_attention_weight_max": 42.6266975402832,
      "activations/layer9_attention_weight_min": -43.541160583496094,
      "epoch": 18.75,
      "learning_rate": 2.936022727272727e-05,
      "loss": 2.7328,
      "step": 322750
    },
    {
      "activations/layer0_attention_weight_max": 16.279964447021484,
      "activations/layer0_attention_weight_min": -14.178633689880371,
      "activations/layer10_attention_weight_max": 35.776546478271484,
      "activations/layer10_attention_weight_min": -37.3422966003418,
      "activations/layer11_attention_weight_max": 36.31439208984375,
      "activations/layer11_attention_weight_min": -37.09537887573242,
      "activations/layer12_attention_weight_max": 18.877540588378906,
      "activations/layer12_attention_weight_min": -23.191226959228516,
      "activations/layer13_attention_weight_max": 42.13155746459961,
      "activations/layer13_attention_weight_min": -28.055246353149414,
      "activations/layer14_attention_weight_max": 35.4509162902832,
      "activations/layer14_attention_weight_min": -30.53107452392578,
      "activations/layer15_attention_weight_max": 37.898868560791016,
      "activations/layer15_attention_weight_min": -31.74212646484375,
      "activations/layer16_attention_weight_max": 34.22114944458008,
      "activations/layer16_attention_weight_min": -34.79923629760742,
      "activations/layer17_attention_weight_max": 52.40431594848633,
      "activations/layer17_attention_weight_min": -47.8314323425293,
      "activations/layer18_attention_weight_max": 46.99935531616211,
      "activations/layer18_attention_weight_min": -43.112064361572266,
      "activations/layer19_attention_weight_max": 24.848339080810547,
      "activations/layer19_attention_weight_min": -27.79683494567871,
      "activations/layer1_attention_weight_max": 16.525529861450195,
      "activations/layer1_attention_weight_min": -14.393651962280273,
      "activations/layer20_attention_weight_max": 23.91126251220703,
      "activations/layer20_attention_weight_min": -24.16905975341797,
      "activations/layer21_attention_weight_max": 42.94801712036133,
      "activations/layer21_attention_weight_min": -28.868295669555664,
      "activations/layer22_attention_weight_max": 29.084505081176758,
      "activations/layer22_attention_weight_min": -29.21932029724121,
      "activations/layer23_attention_weight_max": 33.86457824707031,
      "activations/layer23_attention_weight_min": -24.23613739013672,
      "activations/layer2_attention_weight_max": 32.449501037597656,
      "activations/layer2_attention_weight_min": -31.865163803100586,
      "activations/layer3_attention_weight_max": 85.39881134033203,
      "activations/layer3_attention_weight_min": -87.57351684570312,
      "activations/layer4_attention_weight_max": 83.71278381347656,
      "activations/layer4_attention_weight_min": -83.06585693359375,
      "activations/layer5_attention_weight_max": 63.081214904785156,
      "activations/layer5_attention_weight_min": -67.4574203491211,
      "activations/layer6_attention_weight_max": 50.763126373291016,
      "activations/layer6_attention_weight_min": -52.66291427612305,
      "activations/layer7_attention_weight_max": 65.66539764404297,
      "activations/layer7_attention_weight_min": -67.04231262207031,
      "activations/layer8_attention_weight_max": 49.13103103637695,
      "activations/layer8_attention_weight_min": -50.42783737182617,
      "activations/layer9_attention_weight_max": 45.88753128051758,
      "activations/layer9_attention_weight_min": -50.15027618408203,
      "epoch": 18.76,
      "learning_rate": 2.9341287878787878e-05,
      "loss": 2.7468,
      "step": 322800
    },
    {
      "activations/layer0_attention_weight_max": 15.7523193359375,
      "activations/layer0_attention_weight_min": -14.113633155822754,
      "activations/layer10_attention_weight_max": 38.80760192871094,
      "activations/layer10_attention_weight_min": -37.0689697265625,
      "activations/layer11_attention_weight_max": 33.865699768066406,
      "activations/layer11_attention_weight_min": -33.68662643432617,
      "activations/layer12_attention_weight_max": 19.578510284423828,
      "activations/layer12_attention_weight_min": -23.753467559814453,
      "activations/layer13_attention_weight_max": 36.94489669799805,
      "activations/layer13_attention_weight_min": -30.45659828186035,
      "activations/layer14_attention_weight_max": 37.40537643432617,
      "activations/layer14_attention_weight_min": -31.00006103515625,
      "activations/layer15_attention_weight_max": 35.12832260131836,
      "activations/layer15_attention_weight_min": -32.03459167480469,
      "activations/layer16_attention_weight_max": 36.00404739379883,
      "activations/layer16_attention_weight_min": -34.38807678222656,
      "activations/layer17_attention_weight_max": 53.34959030151367,
      "activations/layer17_attention_weight_min": -47.0609016418457,
      "activations/layer18_attention_weight_max": 49.14631271362305,
      "activations/layer18_attention_weight_min": -40.58965301513672,
      "activations/layer19_attention_weight_max": 24.01360321044922,
      "activations/layer19_attention_weight_min": -23.024030685424805,
      "activations/layer1_attention_weight_max": 15.492919921875,
      "activations/layer1_attention_weight_min": -12.346040725708008,
      "activations/layer20_attention_weight_max": 24.570823669433594,
      "activations/layer20_attention_weight_min": -20.8444766998291,
      "activations/layer21_attention_weight_max": 30.411846160888672,
      "activations/layer21_attention_weight_min": -22.1249942779541,
      "activations/layer22_attention_weight_max": 29.150142669677734,
      "activations/layer22_attention_weight_min": -24.76607894897461,
      "activations/layer23_attention_weight_max": 35.45851135253906,
      "activations/layer23_attention_weight_min": -27.01953125,
      "activations/layer2_attention_weight_max": 32.222251892089844,
      "activations/layer2_attention_weight_min": -29.622386932373047,
      "activations/layer3_attention_weight_max": 85.24794006347656,
      "activations/layer3_attention_weight_min": -84.06696319580078,
      "activations/layer4_attention_weight_max": 88.18122100830078,
      "activations/layer4_attention_weight_min": -83.38872528076172,
      "activations/layer5_attention_weight_max": 61.93296813964844,
      "activations/layer5_attention_weight_min": -71.91258239746094,
      "activations/layer6_attention_weight_max": 51.01791763305664,
      "activations/layer6_attention_weight_min": -51.31057357788086,
      "activations/layer7_attention_weight_max": 64.16497039794922,
      "activations/layer7_attention_weight_min": -63.22831344604492,
      "activations/layer8_attention_weight_max": 50.0079231262207,
      "activations/layer8_attention_weight_min": -54.132713317871094,
      "activations/layer9_attention_weight_max": 46.811458587646484,
      "activations/layer9_attention_weight_min": -47.6097526550293,
      "epoch": 18.76,
      "learning_rate": 2.9322348484848483e-05,
      "loss": 2.7249,
      "step": 322850
    },
    {
      "activations/layer0_attention_weight_max": 15.325115203857422,
      "activations/layer0_attention_weight_min": -14.332695960998535,
      "activations/layer10_attention_weight_max": 34.72857666015625,
      "activations/layer10_attention_weight_min": -35.45735168457031,
      "activations/layer11_attention_weight_max": 31.658279418945312,
      "activations/layer11_attention_weight_min": -36.10213851928711,
      "activations/layer12_attention_weight_max": 19.829391479492188,
      "activations/layer12_attention_weight_min": -29.287403106689453,
      "activations/layer13_attention_weight_max": 35.954986572265625,
      "activations/layer13_attention_weight_min": -28.792659759521484,
      "activations/layer14_attention_weight_max": 34.479888916015625,
      "activations/layer14_attention_weight_min": -28.199237823486328,
      "activations/layer15_attention_weight_max": 32.93843078613281,
      "activations/layer15_attention_weight_min": -29.287334442138672,
      "activations/layer16_attention_weight_max": 34.013763427734375,
      "activations/layer16_attention_weight_min": -30.57509422302246,
      "activations/layer17_attention_weight_max": 49.12800979614258,
      "activations/layer17_attention_weight_min": -43.84942626953125,
      "activations/layer18_attention_weight_max": 46.15851974487305,
      "activations/layer18_attention_weight_min": -39.55883026123047,
      "activations/layer19_attention_weight_max": 21.790868759155273,
      "activations/layer19_attention_weight_min": -22.596281051635742,
      "activations/layer1_attention_weight_max": 16.392486572265625,
      "activations/layer1_attention_weight_min": -13.360806465148926,
      "activations/layer20_attention_weight_max": 21.14951515197754,
      "activations/layer20_attention_weight_min": -21.805665969848633,
      "activations/layer21_attention_weight_max": 31.989702224731445,
      "activations/layer21_attention_weight_min": -22.08441734313965,
      "activations/layer22_attention_weight_max": 29.458723068237305,
      "activations/layer22_attention_weight_min": -25.44347381591797,
      "activations/layer23_attention_weight_max": 34.15720748901367,
      "activations/layer23_attention_weight_min": -25.201765060424805,
      "activations/layer2_attention_weight_max": 31.889921188354492,
      "activations/layer2_attention_weight_min": -30.046241760253906,
      "activations/layer3_attention_weight_max": 82.0708236694336,
      "activations/layer3_attention_weight_min": -84.5329360961914,
      "activations/layer4_attention_weight_max": 84.63651275634766,
      "activations/layer4_attention_weight_min": -82.3212661743164,
      "activations/layer5_attention_weight_max": 65.94087219238281,
      "activations/layer5_attention_weight_min": -67.24073791503906,
      "activations/layer6_attention_weight_max": 47.65719223022461,
      "activations/layer6_attention_weight_min": -48.461612701416016,
      "activations/layer7_attention_weight_max": 63.74168395996094,
      "activations/layer7_attention_weight_min": -62.556358337402344,
      "activations/layer8_attention_weight_max": 43.7862434387207,
      "activations/layer8_attention_weight_min": -49.246559143066406,
      "activations/layer9_attention_weight_max": 43.297889709472656,
      "activations/layer9_attention_weight_min": -43.99559020996094,
      "epoch": 18.76,
      "learning_rate": 2.930340909090909e-05,
      "loss": 2.7412,
      "step": 322900
    },
    {
      "activations/layer0_attention_weight_max": 15.632708549499512,
      "activations/layer0_attention_weight_min": -14.419761657714844,
      "activations/layer10_attention_weight_max": 38.01142120361328,
      "activations/layer10_attention_weight_min": -36.84832763671875,
      "activations/layer11_attention_weight_max": 32.92763137817383,
      "activations/layer11_attention_weight_min": -34.455387115478516,
      "activations/layer12_attention_weight_max": 18.160404205322266,
      "activations/layer12_attention_weight_min": -26.599836349487305,
      "activations/layer13_attention_weight_max": 33.75028610229492,
      "activations/layer13_attention_weight_min": -27.783061981201172,
      "activations/layer14_attention_weight_max": 36.51641845703125,
      "activations/layer14_attention_weight_min": -28.76938819885254,
      "activations/layer15_attention_weight_max": 34.30984115600586,
      "activations/layer15_attention_weight_min": -28.335283279418945,
      "activations/layer16_attention_weight_max": 34.885498046875,
      "activations/layer16_attention_weight_min": -32.37395477294922,
      "activations/layer17_attention_weight_max": 51.47380065917969,
      "activations/layer17_attention_weight_min": -44.92012023925781,
      "activations/layer18_attention_weight_max": 46.37478256225586,
      "activations/layer18_attention_weight_min": -37.563262939453125,
      "activations/layer19_attention_weight_max": 23.238426208496094,
      "activations/layer19_attention_weight_min": -21.704687118530273,
      "activations/layer1_attention_weight_max": 16.202289581298828,
      "activations/layer1_attention_weight_min": -13.407125473022461,
      "activations/layer20_attention_weight_max": 22.02927589416504,
      "activations/layer20_attention_weight_min": -20.946210861206055,
      "activations/layer21_attention_weight_max": 31.77762794494629,
      "activations/layer21_attention_weight_min": -22.5580997467041,
      "activations/layer22_attention_weight_max": 29.614458084106445,
      "activations/layer22_attention_weight_min": -24.38558006286621,
      "activations/layer23_attention_weight_max": 35.846683502197266,
      "activations/layer23_attention_weight_min": -24.341398239135742,
      "activations/layer2_attention_weight_max": 30.361499786376953,
      "activations/layer2_attention_weight_min": -29.641149520874023,
      "activations/layer3_attention_weight_max": 83.83383178710938,
      "activations/layer3_attention_weight_min": -87.69729614257812,
      "activations/layer4_attention_weight_max": 83.89653778076172,
      "activations/layer4_attention_weight_min": -85.78209686279297,
      "activations/layer5_attention_weight_max": 62.55685043334961,
      "activations/layer5_attention_weight_min": -74.17326354980469,
      "activations/layer6_attention_weight_max": 51.683570861816406,
      "activations/layer6_attention_weight_min": -52.10719299316406,
      "activations/layer7_attention_weight_max": 65.41453552246094,
      "activations/layer7_attention_weight_min": -63.82283401489258,
      "activations/layer8_attention_weight_max": 46.365596771240234,
      "activations/layer8_attention_weight_min": -51.1156120300293,
      "activations/layer9_attention_weight_max": 44.993263244628906,
      "activations/layer9_attention_weight_min": -47.29572677612305,
      "epoch": 18.77,
      "learning_rate": 2.9284469696969693e-05,
      "loss": 2.7413,
      "step": 322950
    },
    {
      "activations/layer0_attention_weight_max": 15.38228702545166,
      "activations/layer0_attention_weight_min": -14.144293785095215,
      "activations/layer10_attention_weight_max": 35.0255126953125,
      "activations/layer10_attention_weight_min": -38.096534729003906,
      "activations/layer11_attention_weight_max": 34.1534423828125,
      "activations/layer11_attention_weight_min": -35.45275115966797,
      "activations/layer12_attention_weight_max": 17.509538650512695,
      "activations/layer12_attention_weight_min": -23.560779571533203,
      "activations/layer13_attention_weight_max": 29.94732093811035,
      "activations/layer13_attention_weight_min": -27.719823837280273,
      "activations/layer14_attention_weight_max": 31.86974334716797,
      "activations/layer14_attention_weight_min": -29.820219039916992,
      "activations/layer15_attention_weight_max": 29.990869522094727,
      "activations/layer15_attention_weight_min": -30.968061447143555,
      "activations/layer16_attention_weight_max": 31.01365089416504,
      "activations/layer16_attention_weight_min": -32.1160888671875,
      "activations/layer17_attention_weight_max": 47.634708404541016,
      "activations/layer17_attention_weight_min": -43.63429260253906,
      "activations/layer18_attention_weight_max": 42.94894790649414,
      "activations/layer18_attention_weight_min": -38.46802520751953,
      "activations/layer19_attention_weight_max": 22.437944412231445,
      "activations/layer19_attention_weight_min": -20.78059959411621,
      "activations/layer1_attention_weight_max": 15.574458122253418,
      "activations/layer1_attention_weight_min": -12.743319511413574,
      "activations/layer20_attention_weight_max": 21.561988830566406,
      "activations/layer20_attention_weight_min": -22.764074325561523,
      "activations/layer21_attention_weight_max": 34.05009841918945,
      "activations/layer21_attention_weight_min": -26.480709075927734,
      "activations/layer22_attention_weight_max": 31.757049560546875,
      "activations/layer22_attention_weight_min": -29.90877914428711,
      "activations/layer23_attention_weight_max": 35.13679885864258,
      "activations/layer23_attention_weight_min": -24.526473999023438,
      "activations/layer2_attention_weight_max": 30.019384384155273,
      "activations/layer2_attention_weight_min": -29.534961700439453,
      "activations/layer3_attention_weight_max": 84.01275634765625,
      "activations/layer3_attention_weight_min": -85.63711547851562,
      "activations/layer4_attention_weight_max": 82.43355560302734,
      "activations/layer4_attention_weight_min": -82.11888122558594,
      "activations/layer5_attention_weight_max": 63.55268096923828,
      "activations/layer5_attention_weight_min": -73.75891876220703,
      "activations/layer6_attention_weight_max": 49.49818420410156,
      "activations/layer6_attention_weight_min": -50.576881408691406,
      "activations/layer7_attention_weight_max": 67.91287994384766,
      "activations/layer7_attention_weight_min": -63.76203536987305,
      "activations/layer8_attention_weight_max": 44.65531921386719,
      "activations/layer8_attention_weight_min": -47.5500602722168,
      "activations/layer9_attention_weight_max": 43.4776496887207,
      "activations/layer9_attention_weight_min": -46.0822868347168,
      "epoch": 18.77,
      "learning_rate": 2.92655303030303e-05,
      "loss": 2.7589,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_loss": 2.689453125,
      "eval_runtime": 8.4693,
      "eval_samples_per_second": 507.006,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_openwebtext_loss": 2.689453125,
      "eval_openwebtext_ppl": 14.723621737689907,
      "eval_openwebtext_runtime": 8.4693,
      "eval_openwebtext_samples_per_second": 507.006,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9666,
      "eval_wikitext_samples_per_second": 231.876,
      "step": 323000
    },
    {
      "epoch": 18.77,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.5295,
      "eval_lambada_samples_per_second": 510.941,
      "step": 323000
    },
    {
      "activations/layer0_attention_weight_max": 15.703436851501465,
      "activations/layer0_attention_weight_min": -14.149491310119629,
      "activations/layer10_attention_weight_max": 36.322120666503906,
      "activations/layer10_attention_weight_min": -36.37295150756836,
      "activations/layer11_attention_weight_max": 35.80184555053711,
      "activations/layer11_attention_weight_min": -38.22592544555664,
      "activations/layer12_attention_weight_max": 20.0799617767334,
      "activations/layer12_attention_weight_min": -22.44742774963379,
      "activations/layer13_attention_weight_max": 34.926170349121094,
      "activations/layer13_attention_weight_min": -27.217348098754883,
      "activations/layer14_attention_weight_max": 36.060462951660156,
      "activations/layer14_attention_weight_min": -30.054275512695312,
      "activations/layer15_attention_weight_max": 34.068668365478516,
      "activations/layer15_attention_weight_min": -31.954015731811523,
      "activations/layer16_attention_weight_max": 35.0743522644043,
      "activations/layer16_attention_weight_min": -32.91362380981445,
      "activations/layer17_attention_weight_max": 53.8923225402832,
      "activations/layer17_attention_weight_min": -46.0955696105957,
      "activations/layer18_attention_weight_max": 50.71944046020508,
      "activations/layer18_attention_weight_min": -43.55475616455078,
      "activations/layer19_attention_weight_max": 25.2232723236084,
      "activations/layer19_attention_weight_min": -24.93352508544922,
      "activations/layer1_attention_weight_max": 15.3631591796875,
      "activations/layer1_attention_weight_min": -14.099494934082031,
      "activations/layer20_attention_weight_max": 25.563133239746094,
      "activations/layer20_attention_weight_min": -22.861330032348633,
      "activations/layer21_attention_weight_max": 37.92445755004883,
      "activations/layer21_attention_weight_min": -25.681840896606445,
      "activations/layer22_attention_weight_max": 30.50322151184082,
      "activations/layer22_attention_weight_min": -25.01866912841797,
      "activations/layer23_attention_weight_max": 41.05284881591797,
      "activations/layer23_attention_weight_min": -25.54145050048828,
      "activations/layer2_attention_weight_max": 29.501296997070312,
      "activations/layer2_attention_weight_min": -29.750728607177734,
      "activations/layer3_attention_weight_max": 84.35762023925781,
      "activations/layer3_attention_weight_min": -85.3265380859375,
      "activations/layer4_attention_weight_max": 86.1777114868164,
      "activations/layer4_attention_weight_min": -83.77653503417969,
      "activations/layer5_attention_weight_max": 62.566078186035156,
      "activations/layer5_attention_weight_min": -75.2951431274414,
      "activations/layer6_attention_weight_max": 50.61444854736328,
      "activations/layer6_attention_weight_min": -51.322635650634766,
      "activations/layer7_attention_weight_max": 63.02589416503906,
      "activations/layer7_attention_weight_min": -65.0513687133789,
      "activations/layer8_attention_weight_max": 48.28358840942383,
      "activations/layer8_attention_weight_min": -55.77876281738281,
      "activations/layer9_attention_weight_max": 45.74938201904297,
      "activations/layer9_attention_weight_min": -47.48875045776367,
      "epoch": 18.77,
      "learning_rate": 2.9246590909090906e-05,
      "loss": 2.7422,
      "step": 323050
    },
    {
      "activations/layer0_attention_weight_max": 15.740896224975586,
      "activations/layer0_attention_weight_min": -13.853347778320312,
      "activations/layer10_attention_weight_max": 36.6487922668457,
      "activations/layer10_attention_weight_min": -37.25827407836914,
      "activations/layer11_attention_weight_max": 33.240257263183594,
      "activations/layer11_attention_weight_min": -33.296226501464844,
      "activations/layer12_attention_weight_max": 18.651220321655273,
      "activations/layer12_attention_weight_min": -28.413928985595703,
      "activations/layer13_attention_weight_max": 37.707054138183594,
      "activations/layer13_attention_weight_min": -28.183940887451172,
      "activations/layer14_attention_weight_max": 35.44274139404297,
      "activations/layer14_attention_weight_min": -31.571460723876953,
      "activations/layer15_attention_weight_max": 35.66993713378906,
      "activations/layer15_attention_weight_min": -31.317197799682617,
      "activations/layer16_attention_weight_max": 34.73796844482422,
      "activations/layer16_attention_weight_min": -34.250675201416016,
      "activations/layer17_attention_weight_max": 56.32353591918945,
      "activations/layer17_attention_weight_min": -52.74112319946289,
      "activations/layer18_attention_weight_max": 48.27626419067383,
      "activations/layer18_attention_weight_min": -43.32028579711914,
      "activations/layer19_attention_weight_max": 23.0340518951416,
      "activations/layer19_attention_weight_min": -23.94282341003418,
      "activations/layer1_attention_weight_max": 15.569363594055176,
      "activations/layer1_attention_weight_min": -13.364398956298828,
      "activations/layer20_attention_weight_max": 22.33793830871582,
      "activations/layer20_attention_weight_min": -23.446102142333984,
      "activations/layer21_attention_weight_max": 33.7723274230957,
      "activations/layer21_attention_weight_min": -23.311065673828125,
      "activations/layer22_attention_weight_max": 29.92136001586914,
      "activations/layer22_attention_weight_min": -26.28718376159668,
      "activations/layer23_attention_weight_max": 33.559085845947266,
      "activations/layer23_attention_weight_min": -24.307491302490234,
      "activations/layer2_attention_weight_max": 33.97761535644531,
      "activations/layer2_attention_weight_min": -31.090484619140625,
      "activations/layer3_attention_weight_max": 87.60338592529297,
      "activations/layer3_attention_weight_min": -84.34385681152344,
      "activations/layer4_attention_weight_max": 85.68241882324219,
      "activations/layer4_attention_weight_min": -79.32774353027344,
      "activations/layer5_attention_weight_max": 61.61138153076172,
      "activations/layer5_attention_weight_min": -74.1016616821289,
      "activations/layer6_attention_weight_max": 53.066688537597656,
      "activations/layer6_attention_weight_min": -49.114707946777344,
      "activations/layer7_attention_weight_max": 66.04051208496094,
      "activations/layer7_attention_weight_min": -63.33710479736328,
      "activations/layer8_attention_weight_max": 46.58790969848633,
      "activations/layer8_attention_weight_min": -48.00070571899414,
      "activations/layer9_attention_weight_max": 44.79194259643555,
      "activations/layer9_attention_weight_min": -46.6878662109375,
      "epoch": 18.77,
      "learning_rate": 2.9228030303030303e-05,
      "loss": 2.7265,
      "step": 323100
    },
    {
      "activations/layer0_attention_weight_max": 16.458209991455078,
      "activations/layer0_attention_weight_min": -14.250730514526367,
      "activations/layer10_attention_weight_max": 34.39493179321289,
      "activations/layer10_attention_weight_min": -35.442230224609375,
      "activations/layer11_attention_weight_max": 35.2625846862793,
      "activations/layer11_attention_weight_min": -35.56199645996094,
      "activations/layer12_attention_weight_max": 20.24826431274414,
      "activations/layer12_attention_weight_min": -26.838350296020508,
      "activations/layer13_attention_weight_max": 34.574310302734375,
      "activations/layer13_attention_weight_min": -28.752723693847656,
      "activations/layer14_attention_weight_max": 35.03331756591797,
      "activations/layer14_attention_weight_min": -30.216028213500977,
      "activations/layer15_attention_weight_max": 34.40083694458008,
      "activations/layer15_attention_weight_min": -31.24043846130371,
      "activations/layer16_attention_weight_max": 35.46824645996094,
      "activations/layer16_attention_weight_min": -31.960693359375,
      "activations/layer17_attention_weight_max": 55.04378890991211,
      "activations/layer17_attention_weight_min": -47.14884567260742,
      "activations/layer18_attention_weight_max": 51.63681411743164,
      "activations/layer18_attention_weight_min": -42.05701446533203,
      "activations/layer19_attention_weight_max": 25.094566345214844,
      "activations/layer19_attention_weight_min": -21.90005874633789,
      "activations/layer1_attention_weight_max": 15.8975830078125,
      "activations/layer1_attention_weight_min": -14.941929817199707,
      "activations/layer20_attention_weight_max": 21.808422088623047,
      "activations/layer20_attention_weight_min": -22.263620376586914,
      "activations/layer21_attention_weight_max": 32.60759735107422,
      "activations/layer21_attention_weight_min": -21.071914672851562,
      "activations/layer22_attention_weight_max": 27.50263214111328,
      "activations/layer22_attention_weight_min": -27.28372573852539,
      "activations/layer23_attention_weight_max": 34.287109375,
      "activations/layer23_attention_weight_min": -23.43001365661621,
      "activations/layer2_attention_weight_max": 30.19147300720215,
      "activations/layer2_attention_weight_min": -28.78744888305664,
      "activations/layer3_attention_weight_max": 85.5477523803711,
      "activations/layer3_attention_weight_min": -86.50900268554688,
      "activations/layer4_attention_weight_max": 84.2923812866211,
      "activations/layer4_attention_weight_min": -83.66813659667969,
      "activations/layer5_attention_weight_max": 61.747657775878906,
      "activations/layer5_attention_weight_min": -73.1312255859375,
      "activations/layer6_attention_weight_max": 50.30168151855469,
      "activations/layer6_attention_weight_min": -53.46620559692383,
      "activations/layer7_attention_weight_max": 62.13233184814453,
      "activations/layer7_attention_weight_min": -63.56915283203125,
      "activations/layer8_attention_weight_max": 47.6170539855957,
      "activations/layer8_attention_weight_min": -50.535369873046875,
      "activations/layer9_attention_weight_max": 43.95378112792969,
      "activations/layer9_attention_weight_min": -46.59471893310547,
      "epoch": 18.78,
      "learning_rate": 2.9209090909090905e-05,
      "loss": 2.7174,
      "step": 323150
    },
    {
      "activations/layer0_attention_weight_max": 15.660959243774414,
      "activations/layer0_attention_weight_min": -14.386016845703125,
      "activations/layer10_attention_weight_max": 35.532470703125,
      "activations/layer10_attention_weight_min": -35.695064544677734,
      "activations/layer11_attention_weight_max": 32.16594314575195,
      "activations/layer11_attention_weight_min": -34.19520950317383,
      "activations/layer12_attention_weight_max": 19.190631866455078,
      "activations/layer12_attention_weight_min": -29.947359085083008,
      "activations/layer13_attention_weight_max": 31.970306396484375,
      "activations/layer13_attention_weight_min": -26.29944610595703,
      "activations/layer14_attention_weight_max": 38.112396240234375,
      "activations/layer14_attention_weight_min": -30.53220558166504,
      "activations/layer15_attention_weight_max": 34.951595306396484,
      "activations/layer15_attention_weight_min": -29.82849884033203,
      "activations/layer16_attention_weight_max": 35.79814529418945,
      "activations/layer16_attention_weight_min": -32.65348434448242,
      "activations/layer17_attention_weight_max": 51.15291976928711,
      "activations/layer17_attention_weight_min": -45.560096740722656,
      "activations/layer18_attention_weight_max": 52.19148254394531,
      "activations/layer18_attention_weight_min": -42.051368713378906,
      "activations/layer19_attention_weight_max": 25.91574478149414,
      "activations/layer19_attention_weight_min": -24.04576873779297,
      "activations/layer1_attention_weight_max": 15.429117202758789,
      "activations/layer1_attention_weight_min": -13.628580093383789,
      "activations/layer20_attention_weight_max": 26.508527755737305,
      "activations/layer20_attention_weight_min": -21.33622169494629,
      "activations/layer21_attention_weight_max": 37.20127487182617,
      "activations/layer21_attention_weight_min": -23.211164474487305,
      "activations/layer22_attention_weight_max": 28.115917205810547,
      "activations/layer22_attention_weight_min": -25.22759437561035,
      "activations/layer23_attention_weight_max": 38.09961700439453,
      "activations/layer23_attention_weight_min": -24.95380401611328,
      "activations/layer2_attention_weight_max": 31.745498657226562,
      "activations/layer2_attention_weight_min": -30.539348602294922,
      "activations/layer3_attention_weight_max": 86.72157287597656,
      "activations/layer3_attention_weight_min": -84.62787628173828,
      "activations/layer4_attention_weight_max": 87.2228012084961,
      "activations/layer4_attention_weight_min": -85.44828033447266,
      "activations/layer5_attention_weight_max": 64.80073547363281,
      "activations/layer5_attention_weight_min": -75.20636749267578,
      "activations/layer6_attention_weight_max": 52.0179557800293,
      "activations/layer6_attention_weight_min": -51.620887756347656,
      "activations/layer7_attention_weight_max": 67.28299713134766,
      "activations/layer7_attention_weight_min": -62.60065460205078,
      "activations/layer8_attention_weight_max": 47.909454345703125,
      "activations/layer8_attention_weight_min": -54.43392562866211,
      "activations/layer9_attention_weight_max": 44.87655258178711,
      "activations/layer9_attention_weight_min": -46.22966384887695,
      "epoch": 18.78,
      "learning_rate": 2.9190151515151513e-05,
      "loss": 2.7385,
      "step": 323200
    },
    {
      "activations/layer0_attention_weight_max": 15.815189361572266,
      "activations/layer0_attention_weight_min": -13.99026870727539,
      "activations/layer10_attention_weight_max": 32.33185577392578,
      "activations/layer10_attention_weight_min": -34.138671875,
      "activations/layer11_attention_weight_max": 31.495868682861328,
      "activations/layer11_attention_weight_min": -34.31447219848633,
      "activations/layer12_attention_weight_max": 17.972789764404297,
      "activations/layer12_attention_weight_min": -23.672683715820312,
      "activations/layer13_attention_weight_max": 28.86328125,
      "activations/layer13_attention_weight_min": -27.208362579345703,
      "activations/layer14_attention_weight_max": 32.60563659667969,
      "activations/layer14_attention_weight_min": -30.36454963684082,
      "activations/layer15_attention_weight_max": 31.237186431884766,
      "activations/layer15_attention_weight_min": -30.516481399536133,
      "activations/layer16_attention_weight_max": 32.72767639160156,
      "activations/layer16_attention_weight_min": -30.78465461730957,
      "activations/layer17_attention_weight_max": 48.57976150512695,
      "activations/layer17_attention_weight_min": -46.50059127807617,
      "activations/layer18_attention_weight_max": 48.471778869628906,
      "activations/layer18_attention_weight_min": -43.56796646118164,
      "activations/layer19_attention_weight_max": 24.700037002563477,
      "activations/layer19_attention_weight_min": -24.22540855407715,
      "activations/layer1_attention_weight_max": 15.74476146697998,
      "activations/layer1_attention_weight_min": -12.997220039367676,
      "activations/layer20_attention_weight_max": 23.10223960876465,
      "activations/layer20_attention_weight_min": -24.750619888305664,
      "activations/layer21_attention_weight_max": 36.468414306640625,
      "activations/layer21_attention_weight_min": -24.42117691040039,
      "activations/layer22_attention_weight_max": 26.87987518310547,
      "activations/layer22_attention_weight_min": -27.955780029296875,
      "activations/layer23_attention_weight_max": 33.23429870605469,
      "activations/layer23_attention_weight_min": -23.146873474121094,
      "activations/layer2_attention_weight_max": 30.671722412109375,
      "activations/layer2_attention_weight_min": -29.452892303466797,
      "activations/layer3_attention_weight_max": 88.40991973876953,
      "activations/layer3_attention_weight_min": -84.52656555175781,
      "activations/layer4_attention_weight_max": 82.51860809326172,
      "activations/layer4_attention_weight_min": -78.89218139648438,
      "activations/layer5_attention_weight_max": 62.0044059753418,
      "activations/layer5_attention_weight_min": -70.02879333496094,
      "activations/layer6_attention_weight_max": 47.87910461425781,
      "activations/layer6_attention_weight_min": -48.60582733154297,
      "activations/layer7_attention_weight_max": 66.55183410644531,
      "activations/layer7_attention_weight_min": -62.37308883666992,
      "activations/layer8_attention_weight_max": 43.58479690551758,
      "activations/layer8_attention_weight_min": -45.809814453125,
      "activations/layer9_attention_weight_max": 42.913543701171875,
      "activations/layer9_attention_weight_min": -43.94915771484375,
      "epoch": 18.78,
      "learning_rate": 2.9171212121212118e-05,
      "loss": 2.7209,
      "step": 323250
    },
    {
      "activations/layer0_attention_weight_max": 16.065031051635742,
      "activations/layer0_attention_weight_min": -13.783486366271973,
      "activations/layer10_attention_weight_max": 34.38793182373047,
      "activations/layer10_attention_weight_min": -34.62131881713867,
      "activations/layer11_attention_weight_max": 32.86662292480469,
      "activations/layer11_attention_weight_min": -33.64436340332031,
      "activations/layer12_attention_weight_max": 18.86577796936035,
      "activations/layer12_attention_weight_min": -28.13698959350586,
      "activations/layer13_attention_weight_max": 33.73683166503906,
      "activations/layer13_attention_weight_min": -28.268930435180664,
      "activations/layer14_attention_weight_max": 35.40674591064453,
      "activations/layer14_attention_weight_min": -30.914466857910156,
      "activations/layer15_attention_weight_max": 33.21706771850586,
      "activations/layer15_attention_weight_min": -29.639917373657227,
      "activations/layer16_attention_weight_max": 33.75082778930664,
      "activations/layer16_attention_weight_min": -31.107166290283203,
      "activations/layer17_attention_weight_max": 52.69728088378906,
      "activations/layer17_attention_weight_min": -45.2166748046875,
      "activations/layer18_attention_weight_max": 50.08029556274414,
      "activations/layer18_attention_weight_min": -40.787593841552734,
      "activations/layer19_attention_weight_max": 24.35634422302246,
      "activations/layer19_attention_weight_min": -21.613569259643555,
      "activations/layer1_attention_weight_max": 16.108537673950195,
      "activations/layer1_attention_weight_min": -13.929194450378418,
      "activations/layer20_attention_weight_max": 22.229658126831055,
      "activations/layer20_attention_weight_min": -21.423078536987305,
      "activations/layer21_attention_weight_max": 34.77257537841797,
      "activations/layer21_attention_weight_min": -21.813961029052734,
      "activations/layer22_attention_weight_max": 32.42110061645508,
      "activations/layer22_attention_weight_min": -24.810436248779297,
      "activations/layer23_attention_weight_max": 37.153316497802734,
      "activations/layer23_attention_weight_min": -26.276103973388672,
      "activations/layer2_attention_weight_max": 31.50466537475586,
      "activations/layer2_attention_weight_min": -29.449119567871094,
      "activations/layer3_attention_weight_max": 78.97937774658203,
      "activations/layer3_attention_weight_min": -81.55968475341797,
      "activations/layer4_attention_weight_max": 80.05956268310547,
      "activations/layer4_attention_weight_min": -80.94974517822266,
      "activations/layer5_attention_weight_max": 60.44768524169922,
      "activations/layer5_attention_weight_min": -71.04257202148438,
      "activations/layer6_attention_weight_max": 47.94257354736328,
      "activations/layer6_attention_weight_min": -49.719078063964844,
      "activations/layer7_attention_weight_max": 63.667259216308594,
      "activations/layer7_attention_weight_min": -62.14064407348633,
      "activations/layer8_attention_weight_max": 42.30363464355469,
      "activations/layer8_attention_weight_min": -47.7568473815918,
      "activations/layer9_attention_weight_max": 43.35404968261719,
      "activations/layer9_attention_weight_min": -43.15725326538086,
      "epoch": 18.79,
      "learning_rate": 2.9152272727272726e-05,
      "loss": 2.7179,
      "step": 323300
    },
    {
      "activations/layer0_attention_weight_max": 16.028133392333984,
      "activations/layer0_attention_weight_min": -13.908065795898438,
      "activations/layer10_attention_weight_max": 37.31940460205078,
      "activations/layer10_attention_weight_min": -37.45746612548828,
      "activations/layer11_attention_weight_max": 33.55504608154297,
      "activations/layer11_attention_weight_min": -35.559417724609375,
      "activations/layer12_attention_weight_max": 21.45267677307129,
      "activations/layer12_attention_weight_min": -23.8403377532959,
      "activations/layer13_attention_weight_max": 35.637062072753906,
      "activations/layer13_attention_weight_min": -29.91170310974121,
      "activations/layer14_attention_weight_max": 37.795902252197266,
      "activations/layer14_attention_weight_min": -31.62569808959961,
      "activations/layer15_attention_weight_max": 35.74806594848633,
      "activations/layer15_attention_weight_min": -32.059329986572266,
      "activations/layer16_attention_weight_max": 38.05039596557617,
      "activations/layer16_attention_weight_min": -34.6989860534668,
      "activations/layer17_attention_weight_max": 56.72053146362305,
      "activations/layer17_attention_weight_min": -48.1859016418457,
      "activations/layer18_attention_weight_max": 53.1370964050293,
      "activations/layer18_attention_weight_min": -42.44718933105469,
      "activations/layer19_attention_weight_max": 26.426395416259766,
      "activations/layer19_attention_weight_min": -21.978647232055664,
      "activations/layer1_attention_weight_max": 15.077341079711914,
      "activations/layer1_attention_weight_min": -13.428627967834473,
      "activations/layer20_attention_weight_max": 25.046875,
      "activations/layer20_attention_weight_min": -21.540138244628906,
      "activations/layer21_attention_weight_max": 33.64655303955078,
      "activations/layer21_attention_weight_min": -19.20187759399414,
      "activations/layer22_attention_weight_max": 29.238204956054688,
      "activations/layer22_attention_weight_min": -24.08648109436035,
      "activations/layer23_attention_weight_max": 36.22085952758789,
      "activations/layer23_attention_weight_min": -25.168697357177734,
      "activations/layer2_attention_weight_max": 32.47294616699219,
      "activations/layer2_attention_weight_min": -29.90439224243164,
      "activations/layer3_attention_weight_max": 85.35334014892578,
      "activations/layer3_attention_weight_min": -81.87410736083984,
      "activations/layer4_attention_weight_max": 80.10662078857422,
      "activations/layer4_attention_weight_min": -81.90065002441406,
      "activations/layer5_attention_weight_max": 62.68809509277344,
      "activations/layer5_attention_weight_min": -72.23516082763672,
      "activations/layer6_attention_weight_max": 48.9803581237793,
      "activations/layer6_attention_weight_min": -48.50548553466797,
      "activations/layer7_attention_weight_max": 61.89931869506836,
      "activations/layer7_attention_weight_min": -62.93101501464844,
      "activations/layer8_attention_weight_max": 47.896663665771484,
      "activations/layer8_attention_weight_min": -48.42987060546875,
      "activations/layer9_attention_weight_max": 43.06010055541992,
      "activations/layer9_attention_weight_min": -43.14996337890625,
      "epoch": 18.79,
      "learning_rate": 2.9133333333333328e-05,
      "loss": 2.7441,
      "step": 323350
    },
    {
      "activations/layer0_attention_weight_max": 16.404207229614258,
      "activations/layer0_attention_weight_min": -14.13534164428711,
      "activations/layer10_attention_weight_max": 35.54790496826172,
      "activations/layer10_attention_weight_min": -34.15312576293945,
      "activations/layer11_attention_weight_max": 32.3928108215332,
      "activations/layer11_attention_weight_min": -34.426361083984375,
      "activations/layer12_attention_weight_max": 19.397541046142578,
      "activations/layer12_attention_weight_min": -25.735862731933594,
      "activations/layer13_attention_weight_max": 37.50043487548828,
      "activations/layer13_attention_weight_min": -28.23064422607422,
      "activations/layer14_attention_weight_max": 38.36850357055664,
      "activations/layer14_attention_weight_min": -30.001855850219727,
      "activations/layer15_attention_weight_max": 36.248252868652344,
      "activations/layer15_attention_weight_min": -31.958763122558594,
      "activations/layer16_attention_weight_max": 35.27920913696289,
      "activations/layer16_attention_weight_min": -33.848140716552734,
      "activations/layer17_attention_weight_max": 50.06135940551758,
      "activations/layer17_attention_weight_min": -48.18009567260742,
      "activations/layer18_attention_weight_max": 51.48564529418945,
      "activations/layer18_attention_weight_min": -42.35478973388672,
      "activations/layer19_attention_weight_max": 27.666671752929688,
      "activations/layer19_attention_weight_min": -23.86505699157715,
      "activations/layer1_attention_weight_max": 15.262109756469727,
      "activations/layer1_attention_weight_min": -14.029488563537598,
      "activations/layer20_attention_weight_max": 28.60247802734375,
      "activations/layer20_attention_weight_min": -21.04295539855957,
      "activations/layer21_attention_weight_max": 42.193790435791016,
      "activations/layer21_attention_weight_min": -24.601640701293945,
      "activations/layer22_attention_weight_max": 30.738569259643555,
      "activations/layer22_attention_weight_min": -25.016510009765625,
      "activations/layer23_attention_weight_max": 40.28401184082031,
      "activations/layer23_attention_weight_min": -25.35478973388672,
      "activations/layer2_attention_weight_max": 31.245243072509766,
      "activations/layer2_attention_weight_min": -30.285263061523438,
      "activations/layer3_attention_weight_max": 85.20234680175781,
      "activations/layer3_attention_weight_min": -86.32411193847656,
      "activations/layer4_attention_weight_max": 85.09606170654297,
      "activations/layer4_attention_weight_min": -86.1851806640625,
      "activations/layer5_attention_weight_max": 66.70720672607422,
      "activations/layer5_attention_weight_min": -76.73565673828125,
      "activations/layer6_attention_weight_max": 50.080116271972656,
      "activations/layer6_attention_weight_min": -49.204444885253906,
      "activations/layer7_attention_weight_max": 71.54913330078125,
      "activations/layer7_attention_weight_min": -64.29981994628906,
      "activations/layer8_attention_weight_max": 47.8112907409668,
      "activations/layer8_attention_weight_min": -48.607967376708984,
      "activations/layer9_attention_weight_max": 48.67470932006836,
      "activations/layer9_attention_weight_min": -46.66316223144531,
      "epoch": 18.79,
      "learning_rate": 2.9114393939393936e-05,
      "loss": 2.7525,
      "step": 323400
    },
    {
      "activations/layer0_attention_weight_max": 15.554874420166016,
      "activations/layer0_attention_weight_min": -13.896600723266602,
      "activations/layer10_attention_weight_max": 34.921875,
      "activations/layer10_attention_weight_min": -34.634315490722656,
      "activations/layer11_attention_weight_max": 33.52968978881836,
      "activations/layer11_attention_weight_min": -34.25794982910156,
      "activations/layer12_attention_weight_max": 20.72661781311035,
      "activations/layer12_attention_weight_min": -23.28878402709961,
      "activations/layer13_attention_weight_max": 40.450477600097656,
      "activations/layer13_attention_weight_min": -28.704627990722656,
      "activations/layer14_attention_weight_max": 37.90824508666992,
      "activations/layer14_attention_weight_min": -31.385639190673828,
      "activations/layer15_attention_weight_max": 33.40229797363281,
      "activations/layer15_attention_weight_min": -30.79037857055664,
      "activations/layer16_attention_weight_max": 34.90176773071289,
      "activations/layer16_attention_weight_min": -32.80702590942383,
      "activations/layer17_attention_weight_max": 50.81173324584961,
      "activations/layer17_attention_weight_min": -46.039527893066406,
      "activations/layer18_attention_weight_max": 48.33799362182617,
      "activations/layer18_attention_weight_min": -40.49656295776367,
      "activations/layer19_attention_weight_max": 26.48611831665039,
      "activations/layer19_attention_weight_min": -23.19924545288086,
      "activations/layer1_attention_weight_max": 16.106002807617188,
      "activations/layer1_attention_weight_min": -13.4629487991333,
      "activations/layer20_attention_weight_max": 23.955276489257812,
      "activations/layer20_attention_weight_min": -23.114538192749023,
      "activations/layer21_attention_weight_max": 38.192626953125,
      "activations/layer21_attention_weight_min": -22.92940330505371,
      "activations/layer22_attention_weight_max": 30.96187400817871,
      "activations/layer22_attention_weight_min": -26.184799194335938,
      "activations/layer23_attention_weight_max": 38.86235046386719,
      "activations/layer23_attention_weight_min": -23.976364135742188,
      "activations/layer2_attention_weight_max": 29.290908813476562,
      "activations/layer2_attention_weight_min": -28.562429428100586,
      "activations/layer3_attention_weight_max": 80.16915893554688,
      "activations/layer3_attention_weight_min": -82.56097412109375,
      "activations/layer4_attention_weight_max": 76.19034576416016,
      "activations/layer4_attention_weight_min": -79.36420440673828,
      "activations/layer5_attention_weight_max": 59.43207550048828,
      "activations/layer5_attention_weight_min": -70.14129638671875,
      "activations/layer6_attention_weight_max": 46.92238998413086,
      "activations/layer6_attention_weight_min": -47.551212310791016,
      "activations/layer7_attention_weight_max": 64.16844177246094,
      "activations/layer7_attention_weight_min": -61.484161376953125,
      "activations/layer8_attention_weight_max": 46.11372756958008,
      "activations/layer8_attention_weight_min": -48.72269821166992,
      "activations/layer9_attention_weight_max": 45.944969177246094,
      "activations/layer9_attention_weight_min": -46.024776458740234,
      "epoch": 18.79,
      "learning_rate": 2.909545454545454e-05,
      "loss": 2.7396,
      "step": 323450
    },
    {
      "activations/layer0_attention_weight_max": 15.482209205627441,
      "activations/layer0_attention_weight_min": -14.084000587463379,
      "activations/layer10_attention_weight_max": 35.5446662902832,
      "activations/layer10_attention_weight_min": -37.191650390625,
      "activations/layer11_attention_weight_max": 31.78828239440918,
      "activations/layer11_attention_weight_min": -36.14904022216797,
      "activations/layer12_attention_weight_max": 18.673585891723633,
      "activations/layer12_attention_weight_min": -25.04475212097168,
      "activations/layer13_attention_weight_max": 35.31047058105469,
      "activations/layer13_attention_weight_min": -29.940011978149414,
      "activations/layer14_attention_weight_max": 38.1244010925293,
      "activations/layer14_attention_weight_min": -31.165729522705078,
      "activations/layer15_attention_weight_max": 33.04844665527344,
      "activations/layer15_attention_weight_min": -30.190963745117188,
      "activations/layer16_attention_weight_max": 34.09654235839844,
      "activations/layer16_attention_weight_min": -31.725984573364258,
      "activations/layer17_attention_weight_max": 49.9406623840332,
      "activations/layer17_attention_weight_min": -44.795997619628906,
      "activations/layer18_attention_weight_max": 49.319705963134766,
      "activations/layer18_attention_weight_min": -43.058380126953125,
      "activations/layer19_attention_weight_max": 23.047353744506836,
      "activations/layer19_attention_weight_min": -23.565649032592773,
      "activations/layer1_attention_weight_max": 15.978918075561523,
      "activations/layer1_attention_weight_min": -14.236557960510254,
      "activations/layer20_attention_weight_max": 22.486103057861328,
      "activations/layer20_attention_weight_min": -24.885486602783203,
      "activations/layer21_attention_weight_max": 35.89407730102539,
      "activations/layer21_attention_weight_min": -23.15056800842285,
      "activations/layer22_attention_weight_max": 27.57085609436035,
      "activations/layer22_attention_weight_min": -25.110254287719727,
      "activations/layer23_attention_weight_max": 33.65898513793945,
      "activations/layer23_attention_weight_min": -23.637454986572266,
      "activations/layer2_attention_weight_max": 29.952787399291992,
      "activations/layer2_attention_weight_min": -31.320974349975586,
      "activations/layer3_attention_weight_max": 80.18701934814453,
      "activations/layer3_attention_weight_min": -87.45160675048828,
      "activations/layer4_attention_weight_max": 77.76384735107422,
      "activations/layer4_attention_weight_min": -80.81427764892578,
      "activations/layer5_attention_weight_max": 59.77931213378906,
      "activations/layer5_attention_weight_min": -71.06119537353516,
      "activations/layer6_attention_weight_max": 49.70720672607422,
      "activations/layer6_attention_weight_min": -48.55644226074219,
      "activations/layer7_attention_weight_max": 71.68949890136719,
      "activations/layer7_attention_weight_min": -65.11165618896484,
      "activations/layer8_attention_weight_max": 45.637969970703125,
      "activations/layer8_attention_weight_min": -48.13713455200195,
      "activations/layer9_attention_weight_max": 45.92959213256836,
      "activations/layer9_attention_weight_min": -43.886226654052734,
      "epoch": 18.8,
      "learning_rate": 2.907651515151515e-05,
      "loss": 2.7339,
      "step": 323500
    },
    {
      "activations/layer0_attention_weight_max": 16.052772521972656,
      "activations/layer0_attention_weight_min": -13.921730041503906,
      "activations/layer10_attention_weight_max": 33.44242858886719,
      "activations/layer10_attention_weight_min": -35.50737762451172,
      "activations/layer11_attention_weight_max": 31.693782806396484,
      "activations/layer11_attention_weight_min": -35.08417510986328,
      "activations/layer12_attention_weight_max": 19.630008697509766,
      "activations/layer12_attention_weight_min": -27.834367752075195,
      "activations/layer13_attention_weight_max": 33.72264862060547,
      "activations/layer13_attention_weight_min": -27.709522247314453,
      "activations/layer14_attention_weight_max": 35.58552169799805,
      "activations/layer14_attention_weight_min": -30.016162872314453,
      "activations/layer15_attention_weight_max": 33.688907623291016,
      "activations/layer15_attention_weight_min": -29.78079605102539,
      "activations/layer16_attention_weight_max": 34.35704040527344,
      "activations/layer16_attention_weight_min": -32.70790481567383,
      "activations/layer17_attention_weight_max": 55.32699966430664,
      "activations/layer17_attention_weight_min": -45.084716796875,
      "activations/layer18_attention_weight_max": 47.9267692565918,
      "activations/layer18_attention_weight_min": -38.68669509887695,
      "activations/layer19_attention_weight_max": 22.990276336669922,
      "activations/layer19_attention_weight_min": -23.487300872802734,
      "activations/layer1_attention_weight_max": 15.874351501464844,
      "activations/layer1_attention_weight_min": -14.054095268249512,
      "activations/layer20_attention_weight_max": 21.063453674316406,
      "activations/layer20_attention_weight_min": -23.545265197753906,
      "activations/layer21_attention_weight_max": 31.359697341918945,
      "activations/layer21_attention_weight_min": -22.771005630493164,
      "activations/layer22_attention_weight_max": 30.8565673828125,
      "activations/layer22_attention_weight_min": -25.691816329956055,
      "activations/layer23_attention_weight_max": 32.602176666259766,
      "activations/layer23_attention_weight_min": -23.552562713623047,
      "activations/layer2_attention_weight_max": 30.523733139038086,
      "activations/layer2_attention_weight_min": -30.45502471923828,
      "activations/layer3_attention_weight_max": 82.7751235961914,
      "activations/layer3_attention_weight_min": -88.99103546142578,
      "activations/layer4_attention_weight_max": 80.79454803466797,
      "activations/layer4_attention_weight_min": -79.94306945800781,
      "activations/layer5_attention_weight_max": 61.41686248779297,
      "activations/layer5_attention_weight_min": -71.72540283203125,
      "activations/layer6_attention_weight_max": 51.78984832763672,
      "activations/layer6_attention_weight_min": -49.66474914550781,
      "activations/layer7_attention_weight_max": 64.58992767333984,
      "activations/layer7_attention_weight_min": -63.25041580200195,
      "activations/layer8_attention_weight_max": 45.697269439697266,
      "activations/layer8_attention_weight_min": -47.50507354736328,
      "activations/layer9_attention_weight_max": 42.04096221923828,
      "activations/layer9_attention_weight_min": -45.18424606323242,
      "epoch": 18.8,
      "learning_rate": 2.9057575757575754e-05,
      "loss": 2.7369,
      "step": 323550
    },
    {
      "activations/layer0_attention_weight_max": 15.388968467712402,
      "activations/layer0_attention_weight_min": -13.920669555664062,
      "activations/layer10_attention_weight_max": 35.90578079223633,
      "activations/layer10_attention_weight_min": -36.20195388793945,
      "activations/layer11_attention_weight_max": 31.561737060546875,
      "activations/layer11_attention_weight_min": -32.537437438964844,
      "activations/layer12_attention_weight_max": 22.436033248901367,
      "activations/layer12_attention_weight_min": -23.03775405883789,
      "activations/layer13_attention_weight_max": 36.53181076049805,
      "activations/layer13_attention_weight_min": -28.699886322021484,
      "activations/layer14_attention_weight_max": 37.22888946533203,
      "activations/layer14_attention_weight_min": -30.830657958984375,
      "activations/layer15_attention_weight_max": 37.779090881347656,
      "activations/layer15_attention_weight_min": -30.968475341796875,
      "activations/layer16_attention_weight_max": 37.323150634765625,
      "activations/layer16_attention_weight_min": -32.942710876464844,
      "activations/layer17_attention_weight_max": 54.12370300292969,
      "activations/layer17_attention_weight_min": -46.544921875,
      "activations/layer18_attention_weight_max": 49.7170524597168,
      "activations/layer18_attention_weight_min": -39.975284576416016,
      "activations/layer19_attention_weight_max": 25.65663719177246,
      "activations/layer19_attention_weight_min": -23.436155319213867,
      "activations/layer1_attention_weight_max": 15.687578201293945,
      "activations/layer1_attention_weight_min": -14.522455215454102,
      "activations/layer20_attention_weight_max": 24.4659366607666,
      "activations/layer20_attention_weight_min": -22.991317749023438,
      "activations/layer21_attention_weight_max": 36.023406982421875,
      "activations/layer21_attention_weight_min": -21.87923812866211,
      "activations/layer22_attention_weight_max": 30.3021297454834,
      "activations/layer22_attention_weight_min": -25.762638092041016,
      "activations/layer23_attention_weight_max": 39.043060302734375,
      "activations/layer23_attention_weight_min": -25.40209197998047,
      "activations/layer2_attention_weight_max": 30.616798400878906,
      "activations/layer2_attention_weight_min": -28.755828857421875,
      "activations/layer3_attention_weight_max": 77.5299301147461,
      "activations/layer3_attention_weight_min": -82.55352783203125,
      "activations/layer4_attention_weight_max": 76.11519622802734,
      "activations/layer4_attention_weight_min": -76.94868469238281,
      "activations/layer5_attention_weight_max": 60.671875,
      "activations/layer5_attention_weight_min": -66.41096496582031,
      "activations/layer6_attention_weight_max": 48.80437469482422,
      "activations/layer6_attention_weight_min": -49.0088996887207,
      "activations/layer7_attention_weight_max": 61.93310546875,
      "activations/layer7_attention_weight_min": -64.5776138305664,
      "activations/layer8_attention_weight_max": 45.580810546875,
      "activations/layer8_attention_weight_min": -47.48466873168945,
      "activations/layer9_attention_weight_max": 42.98772430419922,
      "activations/layer9_attention_weight_min": -46.30156326293945,
      "epoch": 18.8,
      "learning_rate": 2.9038636363636362e-05,
      "loss": 2.7426,
      "step": 323600
    },
    {
      "activations/layer0_attention_weight_max": 15.204427719116211,
      "activations/layer0_attention_weight_min": -14.161962509155273,
      "activations/layer10_attention_weight_max": 37.62263870239258,
      "activations/layer10_attention_weight_min": -35.69705581665039,
      "activations/layer11_attention_weight_max": 33.36963653564453,
      "activations/layer11_attention_weight_min": -35.483699798583984,
      "activations/layer12_attention_weight_max": 20.09151840209961,
      "activations/layer12_attention_weight_min": -24.179916381835938,
      "activations/layer13_attention_weight_max": 40.56708526611328,
      "activations/layer13_attention_weight_min": -29.853792190551758,
      "activations/layer14_attention_weight_max": 40.00209426879883,
      "activations/layer14_attention_weight_min": -31.009431838989258,
      "activations/layer15_attention_weight_max": 36.113853454589844,
      "activations/layer15_attention_weight_min": -31.0524845123291,
      "activations/layer16_attention_weight_max": 36.23712921142578,
      "activations/layer16_attention_weight_min": -31.792102813720703,
      "activations/layer17_attention_weight_max": 56.383575439453125,
      "activations/layer17_attention_weight_min": -45.66398239135742,
      "activations/layer18_attention_weight_max": 53.07319259643555,
      "activations/layer18_attention_weight_min": -41.11003875732422,
      "activations/layer19_attention_weight_max": 25.873050689697266,
      "activations/layer19_attention_weight_min": -24.88509178161621,
      "activations/layer1_attention_weight_max": 16.078723907470703,
      "activations/layer1_attention_weight_min": -14.197256088256836,
      "activations/layer20_attention_weight_max": 25.606584548950195,
      "activations/layer20_attention_weight_min": -22.542280197143555,
      "activations/layer21_attention_weight_max": 41.397247314453125,
      "activations/layer21_attention_weight_min": -26.1363525390625,
      "activations/layer22_attention_weight_max": 30.262346267700195,
      "activations/layer22_attention_weight_min": -27.770723342895508,
      "activations/layer23_attention_weight_max": 39.84054946899414,
      "activations/layer23_attention_weight_min": -26.070816040039062,
      "activations/layer2_attention_weight_max": 29.672203063964844,
      "activations/layer2_attention_weight_min": -28.466049194335938,
      "activations/layer3_attention_weight_max": 82.74773406982422,
      "activations/layer3_attention_weight_min": -86.3238754272461,
      "activations/layer4_attention_weight_max": 83.87184143066406,
      "activations/layer4_attention_weight_min": -79.65682983398438,
      "activations/layer5_attention_weight_max": 61.89883041381836,
      "activations/layer5_attention_weight_min": -67.0766830444336,
      "activations/layer6_attention_weight_max": 49.01344299316406,
      "activations/layer6_attention_weight_min": -48.45106887817383,
      "activations/layer7_attention_weight_max": 66.65505981445312,
      "activations/layer7_attention_weight_min": -61.7848014831543,
      "activations/layer8_attention_weight_max": 45.87495422363281,
      "activations/layer8_attention_weight_min": -48.54342269897461,
      "activations/layer9_attention_weight_max": 43.722625732421875,
      "activations/layer9_attention_weight_min": -43.73867416381836,
      "epoch": 18.81,
      "learning_rate": 2.901969696969697e-05,
      "loss": 2.7273,
      "step": 323650
    },
    {
      "activations/layer0_attention_weight_max": 15.563475608825684,
      "activations/layer0_attention_weight_min": -13.716782569885254,
      "activations/layer10_attention_weight_max": 36.46782302856445,
      "activations/layer10_attention_weight_min": -36.43603515625,
      "activations/layer11_attention_weight_max": 33.79163360595703,
      "activations/layer11_attention_weight_min": -34.61030197143555,
      "activations/layer12_attention_weight_max": 19.321916580200195,
      "activations/layer12_attention_weight_min": -24.374897003173828,
      "activations/layer13_attention_weight_max": 38.70547103881836,
      "activations/layer13_attention_weight_min": -29.553749084472656,
      "activations/layer14_attention_weight_max": 37.710594177246094,
      "activations/layer14_attention_weight_min": -30.72934913635254,
      "activations/layer15_attention_weight_max": 40.38724136352539,
      "activations/layer15_attention_weight_min": -31.20261573791504,
      "activations/layer16_attention_weight_max": 38.702484130859375,
      "activations/layer16_attention_weight_min": -34.003822326660156,
      "activations/layer17_attention_weight_max": 59.74429702758789,
      "activations/layer17_attention_weight_min": -50.192771911621094,
      "activations/layer18_attention_weight_max": 57.0474967956543,
      "activations/layer18_attention_weight_min": -44.48714065551758,
      "activations/layer19_attention_weight_max": 28.017744064331055,
      "activations/layer19_attention_weight_min": -24.150720596313477,
      "activations/layer1_attention_weight_max": 15.839877128601074,
      "activations/layer1_attention_weight_min": -13.80901050567627,
      "activations/layer20_attention_weight_max": 26.9422550201416,
      "activations/layer20_attention_weight_min": -22.46173667907715,
      "activations/layer21_attention_weight_max": 45.54151916503906,
      "activations/layer21_attention_weight_min": -23.345884323120117,
      "activations/layer22_attention_weight_max": 35.202632904052734,
      "activations/layer22_attention_weight_min": -27.137311935424805,
      "activations/layer23_attention_weight_max": 47.4632568359375,
      "activations/layer23_attention_weight_min": -24.101110458374023,
      "activations/layer2_attention_weight_max": 33.305965423583984,
      "activations/layer2_attention_weight_min": -28.77626609802246,
      "activations/layer3_attention_weight_max": 90.0140151977539,
      "activations/layer3_attention_weight_min": -84.53723907470703,
      "activations/layer4_attention_weight_max": 86.22303009033203,
      "activations/layer4_attention_weight_min": -82.84496307373047,
      "activations/layer5_attention_weight_max": 62.7830810546875,
      "activations/layer5_attention_weight_min": -69.99547576904297,
      "activations/layer6_attention_weight_max": 50.23450469970703,
      "activations/layer6_attention_weight_min": -49.11185836791992,
      "activations/layer7_attention_weight_max": 64.83460235595703,
      "activations/layer7_attention_weight_min": -65.00684356689453,
      "activations/layer8_attention_weight_max": 46.34857177734375,
      "activations/layer8_attention_weight_min": -47.92926788330078,
      "activations/layer9_attention_weight_max": 46.63934326171875,
      "activations/layer9_attention_weight_min": -51.84345245361328,
      "epoch": 18.81,
      "learning_rate": 2.9000757575757572e-05,
      "loss": 2.7316,
      "step": 323700
    },
    {
      "activations/layer0_attention_weight_max": 15.221720695495605,
      "activations/layer0_attention_weight_min": -13.397820472717285,
      "activations/layer10_attention_weight_max": 36.797210693359375,
      "activations/layer10_attention_weight_min": -36.859275817871094,
      "activations/layer11_attention_weight_max": 33.40652847290039,
      "activations/layer11_attention_weight_min": -35.08256530761719,
      "activations/layer12_attention_weight_max": 19.628694534301758,
      "activations/layer12_attention_weight_min": -23.90128517150879,
      "activations/layer13_attention_weight_max": 38.254539489746094,
      "activations/layer13_attention_weight_min": -30.770118713378906,
      "activations/layer14_attention_weight_max": 41.05610656738281,
      "activations/layer14_attention_weight_min": -32.42988204956055,
      "activations/layer15_attention_weight_max": 37.5009765625,
      "activations/layer15_attention_weight_min": -33.22572326660156,
      "activations/layer16_attention_weight_max": 37.63140869140625,
      "activations/layer16_attention_weight_min": -33.62571716308594,
      "activations/layer17_attention_weight_max": 55.5761833190918,
      "activations/layer17_attention_weight_min": -47.41376876831055,
      "activations/layer18_attention_weight_max": 52.939029693603516,
      "activations/layer18_attention_weight_min": -44.425148010253906,
      "activations/layer19_attention_weight_max": 26.079614639282227,
      "activations/layer19_attention_weight_min": -25.130842208862305,
      "activations/layer1_attention_weight_max": 16.53167724609375,
      "activations/layer1_attention_weight_min": -14.231374740600586,
      "activations/layer20_attention_weight_max": 27.43524169921875,
      "activations/layer20_attention_weight_min": -20.808231353759766,
      "activations/layer21_attention_weight_max": 40.8520622253418,
      "activations/layer21_attention_weight_min": -22.67305564880371,
      "activations/layer22_attention_weight_max": 30.680654525756836,
      "activations/layer22_attention_weight_min": -26.950164794921875,
      "activations/layer23_attention_weight_max": 38.49215316772461,
      "activations/layer23_attention_weight_min": -23.593242645263672,
      "activations/layer2_attention_weight_max": 30.936412811279297,
      "activations/layer2_attention_weight_min": -29.82769775390625,
      "activations/layer3_attention_weight_max": 85.21598052978516,
      "activations/layer3_attention_weight_min": -85.9520263671875,
      "activations/layer4_attention_weight_max": 81.2927474975586,
      "activations/layer4_attention_weight_min": -81.15428924560547,
      "activations/layer5_attention_weight_max": 63.31517791748047,
      "activations/layer5_attention_weight_min": -72.96395874023438,
      "activations/layer6_attention_weight_max": 52.485755920410156,
      "activations/layer6_attention_weight_min": -49.71092987060547,
      "activations/layer7_attention_weight_max": 65.10855102539062,
      "activations/layer7_attention_weight_min": -63.18307113647461,
      "activations/layer8_attention_weight_max": 48.5052375793457,
      "activations/layer8_attention_weight_min": -47.8472785949707,
      "activations/layer9_attention_weight_max": 44.79848098754883,
      "activations/layer9_attention_weight_min": -44.444061279296875,
      "epoch": 18.81,
      "learning_rate": 2.898181818181818e-05,
      "loss": 2.7315,
      "step": 323750
    },
    {
      "activations/layer0_attention_weight_max": 15.575775146484375,
      "activations/layer0_attention_weight_min": -13.74936580657959,
      "activations/layer10_attention_weight_max": 38.22179412841797,
      "activations/layer10_attention_weight_min": -38.59157943725586,
      "activations/layer11_attention_weight_max": 33.53434371948242,
      "activations/layer11_attention_weight_min": -34.93529510498047,
      "activations/layer12_attention_weight_max": 19.665359497070312,
      "activations/layer12_attention_weight_min": -24.203760147094727,
      "activations/layer13_attention_weight_max": 41.096580505371094,
      "activations/layer13_attention_weight_min": -29.867834091186523,
      "activations/layer14_attention_weight_max": 44.24313735961914,
      "activations/layer14_attention_weight_min": -31.770387649536133,
      "activations/layer15_attention_weight_max": 44.925254821777344,
      "activations/layer15_attention_weight_min": -31.747610092163086,
      "activations/layer16_attention_weight_max": 36.37233352661133,
      "activations/layer16_attention_weight_min": -34.36984634399414,
      "activations/layer17_attention_weight_max": 51.3542366027832,
      "activations/layer17_attention_weight_min": -47.82936477661133,
      "activations/layer18_attention_weight_max": 48.523563385009766,
      "activations/layer18_attention_weight_min": -39.2742919921875,
      "activations/layer19_attention_weight_max": 27.092945098876953,
      "activations/layer19_attention_weight_min": -24.0091609954834,
      "activations/layer1_attention_weight_max": 16.02064323425293,
      "activations/layer1_attention_weight_min": -14.172372817993164,
      "activations/layer20_attention_weight_max": 26.966468811035156,
      "activations/layer20_attention_weight_min": -19.78436279296875,
      "activations/layer21_attention_weight_max": 40.541656494140625,
      "activations/layer21_attention_weight_min": -19.729114532470703,
      "activations/layer22_attention_weight_max": 28.68849754333496,
      "activations/layer22_attention_weight_min": -24.93876075744629,
      "activations/layer23_attention_weight_max": 38.682830810546875,
      "activations/layer23_attention_weight_min": -23.282590866088867,
      "activations/layer2_attention_weight_max": 27.965789794921875,
      "activations/layer2_attention_weight_min": -29.081069946289062,
      "activations/layer3_attention_weight_max": 79.2618408203125,
      "activations/layer3_attention_weight_min": -83.75599670410156,
      "activations/layer4_attention_weight_max": 75.64329528808594,
      "activations/layer4_attention_weight_min": -76.21495819091797,
      "activations/layer5_attention_weight_max": 61.45998764038086,
      "activations/layer5_attention_weight_min": -72.45220947265625,
      "activations/layer6_attention_weight_max": 45.24714660644531,
      "activations/layer6_attention_weight_min": -49.530513763427734,
      "activations/layer7_attention_weight_max": 63.535179138183594,
      "activations/layer7_attention_weight_min": -64.11653900146484,
      "activations/layer8_attention_weight_max": 48.75166702270508,
      "activations/layer8_attention_weight_min": -52.25577926635742,
      "activations/layer9_attention_weight_max": 45.12553405761719,
      "activations/layer9_attention_weight_min": -47.010562896728516,
      "epoch": 18.81,
      "learning_rate": 2.8962878787878785e-05,
      "loss": 2.7191,
      "step": 323800
    },
    {
      "activations/layer0_attention_weight_max": 15.339095115661621,
      "activations/layer0_attention_weight_min": -14.087414741516113,
      "activations/layer10_attention_weight_max": 35.52687072753906,
      "activations/layer10_attention_weight_min": -34.32075881958008,
      "activations/layer11_attention_weight_max": 33.477630615234375,
      "activations/layer11_attention_weight_min": -34.99589538574219,
      "activations/layer12_attention_weight_max": 18.834379196166992,
      "activations/layer12_attention_weight_min": -23.097448348999023,
      "activations/layer13_attention_weight_max": 36.42677688598633,
      "activations/layer13_attention_weight_min": -27.358257293701172,
      "activations/layer14_attention_weight_max": 32.421226501464844,
      "activations/layer14_attention_weight_min": -29.285581588745117,
      "activations/layer15_attention_weight_max": 31.5894718170166,
      "activations/layer15_attention_weight_min": -28.912778854370117,
      "activations/layer16_attention_weight_max": 31.87885284423828,
      "activations/layer16_attention_weight_min": -31.341516494750977,
      "activations/layer17_attention_weight_max": 47.079158782958984,
      "activations/layer17_attention_weight_min": -43.74592208862305,
      "activations/layer18_attention_weight_max": 44.15107345581055,
      "activations/layer18_attention_weight_min": -40.94208908081055,
      "activations/layer19_attention_weight_max": 22.12873649597168,
      "activations/layer19_attention_weight_min": -23.123823165893555,
      "activations/layer1_attention_weight_max": 16.50028419494629,
      "activations/layer1_attention_weight_min": -13.494427680969238,
      "activations/layer20_attention_weight_max": 19.477272033691406,
      "activations/layer20_attention_weight_min": -25.0751953125,
      "activations/layer21_attention_weight_max": 32.83753967285156,
      "activations/layer21_attention_weight_min": -23.43912124633789,
      "activations/layer22_attention_weight_max": 25.226966857910156,
      "activations/layer22_attention_weight_min": -26.659442901611328,
      "activations/layer23_attention_weight_max": 34.76872253417969,
      "activations/layer23_attention_weight_min": -24.268871307373047,
      "activations/layer2_attention_weight_max": 31.99758529663086,
      "activations/layer2_attention_weight_min": -29.045133590698242,
      "activations/layer3_attention_weight_max": 86.12600708007812,
      "activations/layer3_attention_weight_min": -81.97216033935547,
      "activations/layer4_attention_weight_max": 80.95746612548828,
      "activations/layer4_attention_weight_min": -76.1825180053711,
      "activations/layer5_attention_weight_max": 59.6727294921875,
      "activations/layer5_attention_weight_min": -71.97743225097656,
      "activations/layer6_attention_weight_max": 47.43248748779297,
      "activations/layer6_attention_weight_min": -47.41550827026367,
      "activations/layer7_attention_weight_max": 60.59870147705078,
      "activations/layer7_attention_weight_min": -58.51531982421875,
      "activations/layer8_attention_weight_max": 43.57300567626953,
      "activations/layer8_attention_weight_min": -45.915565490722656,
      "activations/layer9_attention_weight_max": 42.449005126953125,
      "activations/layer9_attention_weight_min": -42.58464813232422,
      "epoch": 18.82,
      "learning_rate": 2.8943939393939393e-05,
      "loss": 2.7185,
      "step": 323850
    },
    {
      "activations/layer0_attention_weight_max": 15.078055381774902,
      "activations/layer0_attention_weight_min": -13.765772819519043,
      "activations/layer10_attention_weight_max": 36.0217170715332,
      "activations/layer10_attention_weight_min": -35.05120849609375,
      "activations/layer11_attention_weight_max": 32.03952407836914,
      "activations/layer11_attention_weight_min": -33.874481201171875,
      "activations/layer12_attention_weight_max": 18.651845932006836,
      "activations/layer12_attention_weight_min": -26.620454788208008,
      "activations/layer13_attention_weight_max": 31.228923797607422,
      "activations/layer13_attention_weight_min": -28.697927474975586,
      "activations/layer14_attention_weight_max": 35.6389045715332,
      "activations/layer14_attention_weight_min": -32.90644454956055,
      "activations/layer15_attention_weight_max": 31.962907791137695,
      "activations/layer15_attention_weight_min": -31.7816219329834,
      "activations/layer16_attention_weight_max": 33.962886810302734,
      "activations/layer16_attention_weight_min": -32.45116424560547,
      "activations/layer17_attention_weight_max": 50.57203674316406,
      "activations/layer17_attention_weight_min": -44.00334167480469,
      "activations/layer18_attention_weight_max": 47.112693786621094,
      "activations/layer18_attention_weight_min": -38.28972625732422,
      "activations/layer19_attention_weight_max": 24.452983856201172,
      "activations/layer19_attention_weight_min": -23.429306030273438,
      "activations/layer1_attention_weight_max": 15.687623023986816,
      "activations/layer1_attention_weight_min": -13.999799728393555,
      "activations/layer20_attention_weight_max": 22.6060791015625,
      "activations/layer20_attention_weight_min": -25.684329986572266,
      "activations/layer21_attention_weight_max": 34.87144088745117,
      "activations/layer21_attention_weight_min": -21.559961318969727,
      "activations/layer22_attention_weight_max": 25.886486053466797,
      "activations/layer22_attention_weight_min": -26.83114242553711,
      "activations/layer23_attention_weight_max": 36.76145935058594,
      "activations/layer23_attention_weight_min": -28.090377807617188,
      "activations/layer2_attention_weight_max": 31.233198165893555,
      "activations/layer2_attention_weight_min": -31.78592300415039,
      "activations/layer3_attention_weight_max": 83.0906982421875,
      "activations/layer3_attention_weight_min": -84.76000213623047,
      "activations/layer4_attention_weight_max": 80.4182357788086,
      "activations/layer4_attention_weight_min": -83.84368133544922,
      "activations/layer5_attention_weight_max": 59.139923095703125,
      "activations/layer5_attention_weight_min": -68.77928161621094,
      "activations/layer6_attention_weight_max": 47.22370529174805,
      "activations/layer6_attention_weight_min": -48.240455627441406,
      "activations/layer7_attention_weight_max": 67.09939575195312,
      "activations/layer7_attention_weight_min": -60.94569396972656,
      "activations/layer8_attention_weight_max": 45.662353515625,
      "activations/layer8_attention_weight_min": -48.35037612915039,
      "activations/layer9_attention_weight_max": 43.22904586791992,
      "activations/layer9_attention_weight_min": -45.930240631103516,
      "epoch": 18.82,
      "learning_rate": 2.8925e-05,
      "loss": 2.7201,
      "step": 323900
    },
    {
      "activations/layer0_attention_weight_max": 15.777534484863281,
      "activations/layer0_attention_weight_min": -14.397199630737305,
      "activations/layer10_attention_weight_max": 35.897056579589844,
      "activations/layer10_attention_weight_min": -36.877220153808594,
      "activations/layer11_attention_weight_max": 33.773014068603516,
      "activations/layer11_attention_weight_min": -35.36193084716797,
      "activations/layer12_attention_weight_max": 19.191892623901367,
      "activations/layer12_attention_weight_min": -25.939523696899414,
      "activations/layer13_attention_weight_max": 35.88942337036133,
      "activations/layer13_attention_weight_min": -28.29638671875,
      "activations/layer14_attention_weight_max": 34.940391540527344,
      "activations/layer14_attention_weight_min": -30.583023071289062,
      "activations/layer15_attention_weight_max": 33.94215774536133,
      "activations/layer15_attention_weight_min": -29.241497039794922,
      "activations/layer16_attention_weight_max": 34.83363723754883,
      "activations/layer16_attention_weight_min": -32.721893310546875,
      "activations/layer17_attention_weight_max": 51.132659912109375,
      "activations/layer17_attention_weight_min": -43.981475830078125,
      "activations/layer18_attention_weight_max": 47.22482681274414,
      "activations/layer18_attention_weight_min": -39.26813888549805,
      "activations/layer19_attention_weight_max": 23.308889389038086,
      "activations/layer19_attention_weight_min": -22.36505699157715,
      "activations/layer1_attention_weight_max": 16.104747772216797,
      "activations/layer1_attention_weight_min": -13.399665832519531,
      "activations/layer20_attention_weight_max": 22.409513473510742,
      "activations/layer20_attention_weight_min": -23.06099510192871,
      "activations/layer21_attention_weight_max": 39.63585662841797,
      "activations/layer21_attention_weight_min": -24.73091697692871,
      "activations/layer22_attention_weight_max": 30.564294815063477,
      "activations/layer22_attention_weight_min": -25.014265060424805,
      "activations/layer23_attention_weight_max": 39.720550537109375,
      "activations/layer23_attention_weight_min": -24.591777801513672,
      "activations/layer2_attention_weight_max": 30.699298858642578,
      "activations/layer2_attention_weight_min": -30.00044059753418,
      "activations/layer3_attention_weight_max": 81.76596069335938,
      "activations/layer3_attention_weight_min": -84.1969985961914,
      "activations/layer4_attention_weight_max": 80.31535339355469,
      "activations/layer4_attention_weight_min": -79.04369354248047,
      "activations/layer5_attention_weight_max": 63.35146713256836,
      "activations/layer5_attention_weight_min": -72.97537231445312,
      "activations/layer6_attention_weight_max": 49.678199768066406,
      "activations/layer6_attention_weight_min": -48.45627212524414,
      "activations/layer7_attention_weight_max": 67.03144073486328,
      "activations/layer7_attention_weight_min": -65.53787231445312,
      "activations/layer8_attention_weight_max": 46.04852294921875,
      "activations/layer8_attention_weight_min": -51.330806732177734,
      "activations/layer9_attention_weight_max": 46.354042053222656,
      "activations/layer9_attention_weight_min": -46.17284393310547,
      "epoch": 18.82,
      "learning_rate": 2.8906060606060603e-05,
      "loss": 2.7499,
      "step": 323950
    },
    {
      "activations/layer0_attention_weight_max": 15.0215425491333,
      "activations/layer0_attention_weight_min": -13.739302635192871,
      "activations/layer10_attention_weight_max": 38.82513427734375,
      "activations/layer10_attention_weight_min": -34.527565002441406,
      "activations/layer11_attention_weight_max": 34.19633483886719,
      "activations/layer11_attention_weight_min": -35.290184020996094,
      "activations/layer12_attention_weight_max": 19.433961868286133,
      "activations/layer12_attention_weight_min": -24.933162689208984,
      "activations/layer13_attention_weight_max": 36.08724594116211,
      "activations/layer13_attention_weight_min": -29.255748748779297,
      "activations/layer14_attention_weight_max": 40.19889831542969,
      "activations/layer14_attention_weight_min": -31.87726593017578,
      "activations/layer15_attention_weight_max": 37.965309143066406,
      "activations/layer15_attention_weight_min": -30.723045349121094,
      "activations/layer16_attention_weight_max": 38.803802490234375,
      "activations/layer16_attention_weight_min": -34.00936508178711,
      "activations/layer17_attention_weight_max": 56.09375,
      "activations/layer17_attention_weight_min": -46.04069137573242,
      "activations/layer18_attention_weight_max": 56.33766555786133,
      "activations/layer18_attention_weight_min": -41.96371078491211,
      "activations/layer19_attention_weight_max": 31.96238136291504,
      "activations/layer19_attention_weight_min": -26.624561309814453,
      "activations/layer1_attention_weight_max": 16.169384002685547,
      "activations/layer1_attention_weight_min": -13.976425170898438,
      "activations/layer20_attention_weight_max": 32.53364944458008,
      "activations/layer20_attention_weight_min": -23.884550094604492,
      "activations/layer21_attention_weight_max": 47.35992431640625,
      "activations/layer21_attention_weight_min": -23.793237686157227,
      "activations/layer22_attention_weight_max": 37.228172302246094,
      "activations/layer22_attention_weight_min": -26.19276237487793,
      "activations/layer23_attention_weight_max": 51.7684440612793,
      "activations/layer23_attention_weight_min": -24.250720977783203,
      "activations/layer2_attention_weight_max": 32.728912353515625,
      "activations/layer2_attention_weight_min": -30.882099151611328,
      "activations/layer3_attention_weight_max": 84.84669494628906,
      "activations/layer3_attention_weight_min": -86.35823059082031,
      "activations/layer4_attention_weight_max": 81.4547119140625,
      "activations/layer4_attention_weight_min": -79.18144989013672,
      "activations/layer5_attention_weight_max": 63.266441345214844,
      "activations/layer5_attention_weight_min": -74.75139617919922,
      "activations/layer6_attention_weight_max": 49.01866912841797,
      "activations/layer6_attention_weight_min": -48.5455207824707,
      "activations/layer7_attention_weight_max": 63.4211311340332,
      "activations/layer7_attention_weight_min": -63.113746643066406,
      "activations/layer8_attention_weight_max": 45.53290939331055,
      "activations/layer8_attention_weight_min": -47.45798873901367,
      "activations/layer9_attention_weight_max": 44.31477355957031,
      "activations/layer9_attention_weight_min": -43.846797943115234,
      "epoch": 18.83,
      "learning_rate": 2.8887121212121208e-05,
      "loss": 2.7248,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_loss": 2.6875,
      "eval_runtime": 8.4453,
      "eval_samples_per_second": 508.447,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.4453,
      "eval_openwebtext_samples_per_second": 508.447,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 1.9578,
      "eval_wikitext_samples_per_second": 232.913,
      "step": 324000
    },
    {
      "epoch": 18.83,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5468,
      "eval_lambada_samples_per_second": 510.011,
      "step": 324000
    },
    {
      "activations/layer0_attention_weight_max": 15.45717716217041,
      "activations/layer0_attention_weight_min": -13.806052207946777,
      "activations/layer10_attention_weight_max": 36.83509063720703,
      "activations/layer10_attention_weight_min": -38.567718505859375,
      "activations/layer11_attention_weight_max": 32.81755828857422,
      "activations/layer11_attention_weight_min": -36.735321044921875,
      "activations/layer12_attention_weight_max": 19.226892471313477,
      "activations/layer12_attention_weight_min": -26.2547664642334,
      "activations/layer13_attention_weight_max": 34.26982116699219,
      "activations/layer13_attention_weight_min": -31.77303123474121,
      "activations/layer14_attention_weight_max": 38.54576110839844,
      "activations/layer14_attention_weight_min": -33.858985900878906,
      "activations/layer15_attention_weight_max": 34.2446403503418,
      "activations/layer15_attention_weight_min": -34.29408264160156,
      "activations/layer16_attention_weight_max": 35.6473503112793,
      "activations/layer16_attention_weight_min": -33.468345642089844,
      "activations/layer17_attention_weight_max": 53.229312896728516,
      "activations/layer17_attention_weight_min": -47.455326080322266,
      "activations/layer18_attention_weight_max": 48.29149627685547,
      "activations/layer18_attention_weight_min": -43.89728546142578,
      "activations/layer19_attention_weight_max": 27.33953857421875,
      "activations/layer19_attention_weight_min": -25.496747970581055,
      "activations/layer1_attention_weight_max": 15.20568561553955,
      "activations/layer1_attention_weight_min": -13.54654598236084,
      "activations/layer20_attention_weight_max": 29.717693328857422,
      "activations/layer20_attention_weight_min": -24.96239471435547,
      "activations/layer21_attention_weight_max": 44.00782775878906,
      "activations/layer21_attention_weight_min": -26.725616455078125,
      "activations/layer22_attention_weight_max": 30.17955207824707,
      "activations/layer22_attention_weight_min": -26.810462951660156,
      "activations/layer23_attention_weight_max": 41.55344772338867,
      "activations/layer23_attention_weight_min": -24.410755157470703,
      "activations/layer2_attention_weight_max": 31.837522506713867,
      "activations/layer2_attention_weight_min": -30.369033813476562,
      "activations/layer3_attention_weight_max": 84.5300064086914,
      "activations/layer3_attention_weight_min": -88.966796875,
      "activations/layer4_attention_weight_max": 82.53396606445312,
      "activations/layer4_attention_weight_min": -81.2113265991211,
      "activations/layer5_attention_weight_max": 63.27111053466797,
      "activations/layer5_attention_weight_min": -68.81407165527344,
      "activations/layer6_attention_weight_max": 49.73540115356445,
      "activations/layer6_attention_weight_min": -50.7011604309082,
      "activations/layer7_attention_weight_max": 60.98541259765625,
      "activations/layer7_attention_weight_min": -64.4576187133789,
      "activations/layer8_attention_weight_max": 45.52157211303711,
      "activations/layer8_attention_weight_min": -49.98082733154297,
      "activations/layer9_attention_weight_max": 42.67927169799805,
      "activations/layer9_attention_weight_min": -48.04842758178711,
      "epoch": 18.83,
      "learning_rate": 2.8868181818181816e-05,
      "loss": 2.7154,
      "step": 324050
    },
    {
      "activations/layer0_attention_weight_max": 16.115623474121094,
      "activations/layer0_attention_weight_min": -13.767667770385742,
      "activations/layer10_attention_weight_max": 36.26984405517578,
      "activations/layer10_attention_weight_min": -34.546417236328125,
      "activations/layer11_attention_weight_max": 33.3183479309082,
      "activations/layer11_attention_weight_min": -32.93217849731445,
      "activations/layer12_attention_weight_max": 17.967975616455078,
      "activations/layer12_attention_weight_min": -24.83163070678711,
      "activations/layer13_attention_weight_max": 41.636940002441406,
      "activations/layer13_attention_weight_min": -34.05289077758789,
      "activations/layer14_attention_weight_max": 34.20082473754883,
      "activations/layer14_attention_weight_min": -33.85301208496094,
      "activations/layer15_attention_weight_max": 31.5644474029541,
      "activations/layer15_attention_weight_min": -31.474464416503906,
      "activations/layer16_attention_weight_max": 36.05222702026367,
      "activations/layer16_attention_weight_min": -34.63835906982422,
      "activations/layer17_attention_weight_max": 50.504940032958984,
      "activations/layer17_attention_weight_min": -47.24351501464844,
      "activations/layer18_attention_weight_max": 45.14289474487305,
      "activations/layer18_attention_weight_min": -41.076202392578125,
      "activations/layer19_attention_weight_max": 23.50330924987793,
      "activations/layer19_attention_weight_min": -24.648710250854492,
      "activations/layer1_attention_weight_max": 15.55402660369873,
      "activations/layer1_attention_weight_min": -14.527164459228516,
      "activations/layer20_attention_weight_max": 24.79186248779297,
      "activations/layer20_attention_weight_min": -23.53875732421875,
      "activations/layer21_attention_weight_max": 33.92461395263672,
      "activations/layer21_attention_weight_min": -24.99497413635254,
      "activations/layer22_attention_weight_max": 28.63084602355957,
      "activations/layer22_attention_weight_min": -25.85009765625,
      "activations/layer23_attention_weight_max": 33.73750305175781,
      "activations/layer23_attention_weight_min": -24.456403732299805,
      "activations/layer2_attention_weight_max": 31.138755798339844,
      "activations/layer2_attention_weight_min": -29.95917510986328,
      "activations/layer3_attention_weight_max": 85.74639129638672,
      "activations/layer3_attention_weight_min": -83.39519500732422,
      "activations/layer4_attention_weight_max": 80.7647705078125,
      "activations/layer4_attention_weight_min": -82.18331146240234,
      "activations/layer5_attention_weight_max": 61.40717697143555,
      "activations/layer5_attention_weight_min": -76.4070053100586,
      "activations/layer6_attention_weight_max": 46.71516418457031,
      "activations/layer6_attention_weight_min": -49.48875045776367,
      "activations/layer7_attention_weight_max": 62.942134857177734,
      "activations/layer7_attention_weight_min": -61.99678039550781,
      "activations/layer8_attention_weight_max": 47.72472381591797,
      "activations/layer8_attention_weight_min": -49.845638275146484,
      "activations/layer9_attention_weight_max": 45.456298828125,
      "activations/layer9_attention_weight_min": -43.21885681152344,
      "epoch": 18.83,
      "learning_rate": 2.884924242424242e-05,
      "loss": 2.7376,
      "step": 324100
    },
    {
      "activations/layer0_attention_weight_max": 15.489412307739258,
      "activations/layer0_attention_weight_min": -13.70470905303955,
      "activations/layer10_attention_weight_max": 34.30559539794922,
      "activations/layer10_attention_weight_min": -34.20069122314453,
      "activations/layer11_attention_weight_max": 33.15279769897461,
      "activations/layer11_attention_weight_min": -32.47773742675781,
      "activations/layer12_attention_weight_max": 17.99287223815918,
      "activations/layer12_attention_weight_min": -27.339027404785156,
      "activations/layer13_attention_weight_max": 36.23512268066406,
      "activations/layer13_attention_weight_min": -26.574817657470703,
      "activations/layer14_attention_weight_max": 36.930084228515625,
      "activations/layer14_attention_weight_min": -28.271488189697266,
      "activations/layer15_attention_weight_max": 33.116085052490234,
      "activations/layer15_attention_weight_min": -28.68887710571289,
      "activations/layer16_attention_weight_max": 33.569435119628906,
      "activations/layer16_attention_weight_min": -30.15234375,
      "activations/layer17_attention_weight_max": 49.5048828125,
      "activations/layer17_attention_weight_min": -43.90291213989258,
      "activations/layer18_attention_weight_max": 44.21784591674805,
      "activations/layer18_attention_weight_min": -38.60496520996094,
      "activations/layer19_attention_weight_max": 26.06305503845215,
      "activations/layer19_attention_weight_min": -25.04907989501953,
      "activations/layer1_attention_weight_max": 15.099967956542969,
      "activations/layer1_attention_weight_min": -13.832536697387695,
      "activations/layer20_attention_weight_max": 23.33544921875,
      "activations/layer20_attention_weight_min": -21.26633071899414,
      "activations/layer21_attention_weight_max": 35.967140197753906,
      "activations/layer21_attention_weight_min": -22.1322078704834,
      "activations/layer22_attention_weight_max": 28.686975479125977,
      "activations/layer22_attention_weight_min": -23.736970901489258,
      "activations/layer23_attention_weight_max": 31.274799346923828,
      "activations/layer23_attention_weight_min": -22.03211784362793,
      "activations/layer2_attention_weight_max": 31.458683013916016,
      "activations/layer2_attention_weight_min": -29.865325927734375,
      "activations/layer3_attention_weight_max": 83.08763885498047,
      "activations/layer3_attention_weight_min": -83.9590072631836,
      "activations/layer4_attention_weight_max": 80.0367431640625,
      "activations/layer4_attention_weight_min": -81.10790252685547,
      "activations/layer5_attention_weight_max": 60.04602813720703,
      "activations/layer5_attention_weight_min": -69.30662536621094,
      "activations/layer6_attention_weight_max": 46.095054626464844,
      "activations/layer6_attention_weight_min": -47.77430725097656,
      "activations/layer7_attention_weight_max": 63.9563102722168,
      "activations/layer7_attention_weight_min": -59.518863677978516,
      "activations/layer8_attention_weight_max": 46.663516998291016,
      "activations/layer8_attention_weight_min": -49.60126495361328,
      "activations/layer9_attention_weight_max": 46.0507926940918,
      "activations/layer9_attention_weight_min": -46.0428352355957,
      "epoch": 18.83,
      "learning_rate": 2.883030303030303e-05,
      "loss": 2.725,
      "step": 324150
    },
    {
      "activations/layer0_attention_weight_max": 15.532191276550293,
      "activations/layer0_attention_weight_min": -13.814332962036133,
      "activations/layer10_attention_weight_max": 34.230979919433594,
      "activations/layer10_attention_weight_min": -35.09082794189453,
      "activations/layer11_attention_weight_max": 33.75170135498047,
      "activations/layer11_attention_weight_min": -33.338294982910156,
      "activations/layer12_attention_weight_max": 19.80435562133789,
      "activations/layer12_attention_weight_min": -25.217609405517578,
      "activations/layer13_attention_weight_max": 41.785797119140625,
      "activations/layer13_attention_weight_min": -27.168888092041016,
      "activations/layer14_attention_weight_max": 42.87944793701172,
      "activations/layer14_attention_weight_min": -28.998981475830078,
      "activations/layer15_attention_weight_max": 39.72636032104492,
      "activations/layer15_attention_weight_min": -29.3188419342041,
      "activations/layer16_attention_weight_max": 35.526309967041016,
      "activations/layer16_attention_weight_min": -32.4789924621582,
      "activations/layer17_attention_weight_max": 54.50468826293945,
      "activations/layer17_attention_weight_min": -43.985408782958984,
      "activations/layer18_attention_weight_max": 54.92210006713867,
      "activations/layer18_attention_weight_min": -39.79608154296875,
      "activations/layer19_attention_weight_max": 26.319143295288086,
      "activations/layer19_attention_weight_min": -22.4047794342041,
      "activations/layer1_attention_weight_max": 16.462688446044922,
      "activations/layer1_attention_weight_min": -14.074976921081543,
      "activations/layer20_attention_weight_max": 26.535898208618164,
      "activations/layer20_attention_weight_min": -22.542142868041992,
      "activations/layer21_attention_weight_max": 37.8183479309082,
      "activations/layer21_attention_weight_min": -23.374595642089844,
      "activations/layer22_attention_weight_max": 27.81488037109375,
      "activations/layer22_attention_weight_min": -25.15285301208496,
      "activations/layer23_attention_weight_max": 33.04302215576172,
      "activations/layer23_attention_weight_min": -23.416675567626953,
      "activations/layer2_attention_weight_max": 31.74802017211914,
      "activations/layer2_attention_weight_min": -30.102130889892578,
      "activations/layer3_attention_weight_max": 82.1021728515625,
      "activations/layer3_attention_weight_min": -82.50692749023438,
      "activations/layer4_attention_weight_max": 83.72916412353516,
      "activations/layer4_attention_weight_min": -81.22199249267578,
      "activations/layer5_attention_weight_max": 63.996437072753906,
      "activations/layer5_attention_weight_min": -68.19514465332031,
      "activations/layer6_attention_weight_max": 51.181373596191406,
      "activations/layer6_attention_weight_min": -50.4925422668457,
      "activations/layer7_attention_weight_max": 71.51168060302734,
      "activations/layer7_attention_weight_min": -64.37332916259766,
      "activations/layer8_attention_weight_max": 47.69355773925781,
      "activations/layer8_attention_weight_min": -51.502559661865234,
      "activations/layer9_attention_weight_max": 51.652626037597656,
      "activations/layer9_attention_weight_min": -45.42802810668945,
      "epoch": 18.84,
      "learning_rate": 2.881136363636363e-05,
      "loss": 2.7328,
      "step": 324200
    },
    {
      "activations/layer0_attention_weight_max": 15.380782127380371,
      "activations/layer0_attention_weight_min": -13.939661026000977,
      "activations/layer10_attention_weight_max": 37.385406494140625,
      "activations/layer10_attention_weight_min": -37.67497634887695,
      "activations/layer11_attention_weight_max": 31.945838928222656,
      "activations/layer11_attention_weight_min": -31.895610809326172,
      "activations/layer12_attention_weight_max": 20.24740219116211,
      "activations/layer12_attention_weight_min": -25.564964294433594,
      "activations/layer13_attention_weight_max": 36.366214752197266,
      "activations/layer13_attention_weight_min": -28.00208282470703,
      "activations/layer14_attention_weight_max": 36.18001937866211,
      "activations/layer14_attention_weight_min": -29.785877227783203,
      "activations/layer15_attention_weight_max": 33.693485260009766,
      "activations/layer15_attention_weight_min": -30.789249420166016,
      "activations/layer16_attention_weight_max": 35.04124069213867,
      "activations/layer16_attention_weight_min": -33.82921600341797,
      "activations/layer17_attention_weight_max": 52.94334411621094,
      "activations/layer17_attention_weight_min": -49.11065673828125,
      "activations/layer18_attention_weight_max": 46.60960388183594,
      "activations/layer18_attention_weight_min": -42.345951080322266,
      "activations/layer19_attention_weight_max": 23.74156379699707,
      "activations/layer19_attention_weight_min": -24.546865463256836,
      "activations/layer1_attention_weight_max": 15.163126945495605,
      "activations/layer1_attention_weight_min": -13.117751121520996,
      "activations/layer20_attention_weight_max": 22.55467414855957,
      "activations/layer20_attention_weight_min": -24.984617233276367,
      "activations/layer21_attention_weight_max": 35.86792755126953,
      "activations/layer21_attention_weight_min": -20.002988815307617,
      "activations/layer22_attention_weight_max": 30.612674713134766,
      "activations/layer22_attention_weight_min": -26.00973892211914,
      "activations/layer23_attention_weight_max": 40.37300491333008,
      "activations/layer23_attention_weight_min": -27.895408630371094,
      "activations/layer2_attention_weight_max": 29.74177360534668,
      "activations/layer2_attention_weight_min": -27.682373046875,
      "activations/layer3_attention_weight_max": 78.19827270507812,
      "activations/layer3_attention_weight_min": -76.40542602539062,
      "activations/layer4_attention_weight_max": 81.09691619873047,
      "activations/layer4_attention_weight_min": -77.17704772949219,
      "activations/layer5_attention_weight_max": 59.757835388183594,
      "activations/layer5_attention_weight_min": -68.02278137207031,
      "activations/layer6_attention_weight_max": 47.999088287353516,
      "activations/layer6_attention_weight_min": -48.642845153808594,
      "activations/layer7_attention_weight_max": 63.27217102050781,
      "activations/layer7_attention_weight_min": -61.95526885986328,
      "activations/layer8_attention_weight_max": 44.95102310180664,
      "activations/layer8_attention_weight_min": -50.446720123291016,
      "activations/layer9_attention_weight_max": 43.27260971069336,
      "activations/layer9_attention_weight_min": -45.6784782409668,
      "epoch": 18.84,
      "learning_rate": 2.879242424242424e-05,
      "loss": 2.7095,
      "step": 324250
    },
    {
      "activations/layer0_attention_weight_max": 16.016584396362305,
      "activations/layer0_attention_weight_min": -13.51288890838623,
      "activations/layer10_attention_weight_max": 37.021759033203125,
      "activations/layer10_attention_weight_min": -36.65016555786133,
      "activations/layer11_attention_weight_max": 35.69498825073242,
      "activations/layer11_attention_weight_min": -34.78340148925781,
      "activations/layer12_attention_weight_max": 20.577884674072266,
      "activations/layer12_attention_weight_min": -25.785131454467773,
      "activations/layer13_attention_weight_max": 35.0076904296875,
      "activations/layer13_attention_weight_min": -31.42041015625,
      "activations/layer14_attention_weight_max": 37.06382369995117,
      "activations/layer14_attention_weight_min": -33.30924606323242,
      "activations/layer15_attention_weight_max": 34.53693389892578,
      "activations/layer15_attention_weight_min": -32.15583038330078,
      "activations/layer16_attention_weight_max": 37.01297378540039,
      "activations/layer16_attention_weight_min": -33.86386489868164,
      "activations/layer17_attention_weight_max": 52.178558349609375,
      "activations/layer17_attention_weight_min": -48.153564453125,
      "activations/layer18_attention_weight_max": 51.93202590942383,
      "activations/layer18_attention_weight_min": -42.68357467651367,
      "activations/layer19_attention_weight_max": 30.251684188842773,
      "activations/layer19_attention_weight_min": -27.759159088134766,
      "activations/layer1_attention_weight_max": 16.181133270263672,
      "activations/layer1_attention_weight_min": -12.91504192352295,
      "activations/layer20_attention_weight_max": 28.484041213989258,
      "activations/layer20_attention_weight_min": -23.605112075805664,
      "activations/layer21_attention_weight_max": 40.753761291503906,
      "activations/layer21_attention_weight_min": -23.805437088012695,
      "activations/layer22_attention_weight_max": 33.333736419677734,
      "activations/layer22_attention_weight_min": -28.355716705322266,
      "activations/layer23_attention_weight_max": 41.194602966308594,
      "activations/layer23_attention_weight_min": -24.512813568115234,
      "activations/layer2_attention_weight_max": 30.549901962280273,
      "activations/layer2_attention_weight_min": -28.110462188720703,
      "activations/layer3_attention_weight_max": 83.78974151611328,
      "activations/layer3_attention_weight_min": -84.45772552490234,
      "activations/layer4_attention_weight_max": 81.25337982177734,
      "activations/layer4_attention_weight_min": -80.68505096435547,
      "activations/layer5_attention_weight_max": 62.070716857910156,
      "activations/layer5_attention_weight_min": -71.29627990722656,
      "activations/layer6_attention_weight_max": 49.10679244995117,
      "activations/layer6_attention_weight_min": -48.15959548950195,
      "activations/layer7_attention_weight_max": 65.46990966796875,
      "activations/layer7_attention_weight_min": -64.70700073242188,
      "activations/layer8_attention_weight_max": 47.98541259765625,
      "activations/layer8_attention_weight_min": -52.024688720703125,
      "activations/layer9_attention_weight_max": 44.682003021240234,
      "activations/layer9_attention_weight_min": -49.088829040527344,
      "epoch": 18.84,
      "learning_rate": 2.8773863636363634e-05,
      "loss": 2.7378,
      "step": 324300
    },
    {
      "activations/layer0_attention_weight_max": 15.164410591125488,
      "activations/layer0_attention_weight_min": -13.791862487792969,
      "activations/layer10_attention_weight_max": 39.68410110473633,
      "activations/layer10_attention_weight_min": -38.584991455078125,
      "activations/layer11_attention_weight_max": 32.8888053894043,
      "activations/layer11_attention_weight_min": -33.86497497558594,
      "activations/layer12_attention_weight_max": 19.093494415283203,
      "activations/layer12_attention_weight_min": -24.37097930908203,
      "activations/layer13_attention_weight_max": 33.1889762878418,
      "activations/layer13_attention_weight_min": -27.15430450439453,
      "activations/layer14_attention_weight_max": 33.73694610595703,
      "activations/layer14_attention_weight_min": -29.51052474975586,
      "activations/layer15_attention_weight_max": 30.992908477783203,
      "activations/layer15_attention_weight_min": -29.995914459228516,
      "activations/layer16_attention_weight_max": 33.25539779663086,
      "activations/layer16_attention_weight_min": -31.444515228271484,
      "activations/layer17_attention_weight_max": 53.71553421020508,
      "activations/layer17_attention_weight_min": -44.45012664794922,
      "activations/layer18_attention_weight_max": 44.98615646362305,
      "activations/layer18_attention_weight_min": -37.02123260498047,
      "activations/layer19_attention_weight_max": 24.82065773010254,
      "activations/layer19_attention_weight_min": -21.328237533569336,
      "activations/layer1_attention_weight_max": 16.152942657470703,
      "activations/layer1_attention_weight_min": -14.621234893798828,
      "activations/layer20_attention_weight_max": 22.702573776245117,
      "activations/layer20_attention_weight_min": -21.80489158630371,
      "activations/layer21_attention_weight_max": 38.25860595703125,
      "activations/layer21_attention_weight_min": -22.202537536621094,
      "activations/layer22_attention_weight_max": 27.98558807373047,
      "activations/layer22_attention_weight_min": -24.852794647216797,
      "activations/layer23_attention_weight_max": 35.7245979309082,
      "activations/layer23_attention_weight_min": -23.957176208496094,
      "activations/layer2_attention_weight_max": 32.47570037841797,
      "activations/layer2_attention_weight_min": -31.083267211914062,
      "activations/layer3_attention_weight_max": 83.75435638427734,
      "activations/layer3_attention_weight_min": -86.01652526855469,
      "activations/layer4_attention_weight_max": 85.61636352539062,
      "activations/layer4_attention_weight_min": -80.74407958984375,
      "activations/layer5_attention_weight_max": 60.863861083984375,
      "activations/layer5_attention_weight_min": -71.89456939697266,
      "activations/layer6_attention_weight_max": 49.847694396972656,
      "activations/layer6_attention_weight_min": -50.04487991333008,
      "activations/layer7_attention_weight_max": 62.2994499206543,
      "activations/layer7_attention_weight_min": -63.50956726074219,
      "activations/layer8_attention_weight_max": 49.970272064208984,
      "activations/layer8_attention_weight_min": -53.18559265136719,
      "activations/layer9_attention_weight_max": 45.105491638183594,
      "activations/layer9_attention_weight_min": -46.548892974853516,
      "epoch": 18.85,
      "learning_rate": 2.8754924242424242e-05,
      "loss": 2.7361,
      "step": 324350
    },
    {
      "activations/layer0_attention_weight_max": 16.751237869262695,
      "activations/layer0_attention_weight_min": -13.77080249786377,
      "activations/layer10_attention_weight_max": 33.9498405456543,
      "activations/layer10_attention_weight_min": -34.145416259765625,
      "activations/layer11_attention_weight_max": 31.321170806884766,
      "activations/layer11_attention_weight_min": -34.697906494140625,
      "activations/layer12_attention_weight_max": 20.55331039428711,
      "activations/layer12_attention_weight_min": -24.363712310791016,
      "activations/layer13_attention_weight_max": 34.422359466552734,
      "activations/layer13_attention_weight_min": -26.46377182006836,
      "activations/layer14_attention_weight_max": 35.52436065673828,
      "activations/layer14_attention_weight_min": -31.283267974853516,
      "activations/layer15_attention_weight_max": 33.11603927612305,
      "activations/layer15_attention_weight_min": -30.758792877197266,
      "activations/layer16_attention_weight_max": 35.08064270019531,
      "activations/layer16_attention_weight_min": -33.84221649169922,
      "activations/layer17_attention_weight_max": 51.148963928222656,
      "activations/layer17_attention_weight_min": -46.323909759521484,
      "activations/layer18_attention_weight_max": 47.5068244934082,
      "activations/layer18_attention_weight_min": -41.559593200683594,
      "activations/layer19_attention_weight_max": 23.160770416259766,
      "activations/layer19_attention_weight_min": -23.886423110961914,
      "activations/layer1_attention_weight_max": 16.183273315429688,
      "activations/layer1_attention_weight_min": -14.592775344848633,
      "activations/layer20_attention_weight_max": 22.18560028076172,
      "activations/layer20_attention_weight_min": -21.643230438232422,
      "activations/layer21_attention_weight_max": 37.684608459472656,
      "activations/layer21_attention_weight_min": -22.492416381835938,
      "activations/layer22_attention_weight_max": 27.143850326538086,
      "activations/layer22_attention_weight_min": -25.809616088867188,
      "activations/layer23_attention_weight_max": 35.390220642089844,
      "activations/layer23_attention_weight_min": -22.760494232177734,
      "activations/layer2_attention_weight_max": 30.581104278564453,
      "activations/layer2_attention_weight_min": -30.470800399780273,
      "activations/layer3_attention_weight_max": 85.30119323730469,
      "activations/layer3_attention_weight_min": -82.25326538085938,
      "activations/layer4_attention_weight_max": 81.92334747314453,
      "activations/layer4_attention_weight_min": -78.82860565185547,
      "activations/layer5_attention_weight_max": 61.1464958190918,
      "activations/layer5_attention_weight_min": -73.48400115966797,
      "activations/layer6_attention_weight_max": 46.15080261230469,
      "activations/layer6_attention_weight_min": -48.05862808227539,
      "activations/layer7_attention_weight_max": 63.382877349853516,
      "activations/layer7_attention_weight_min": -60.30622863769531,
      "activations/layer8_attention_weight_max": 44.582008361816406,
      "activations/layer8_attention_weight_min": -45.53273391723633,
      "activations/layer9_attention_weight_max": 42.44332504272461,
      "activations/layer9_attention_weight_min": -45.39681625366211,
      "epoch": 18.85,
      "learning_rate": 2.8735984848484843e-05,
      "loss": 2.7349,
      "step": 324400
    },
    {
      "activations/layer0_attention_weight_max": 15.324149131774902,
      "activations/layer0_attention_weight_min": -13.661565780639648,
      "activations/layer10_attention_weight_max": 39.79982376098633,
      "activations/layer10_attention_weight_min": -39.85873031616211,
      "activations/layer11_attention_weight_max": 38.555816650390625,
      "activations/layer11_attention_weight_min": -39.17852020263672,
      "activations/layer12_attention_weight_max": 21.783004760742188,
      "activations/layer12_attention_weight_min": -28.008302688598633,
      "activations/layer13_attention_weight_max": 38.201080322265625,
      "activations/layer13_attention_weight_min": -32.0774040222168,
      "activations/layer14_attention_weight_max": 42.435646057128906,
      "activations/layer14_attention_weight_min": -32.4101448059082,
      "activations/layer15_attention_weight_max": 39.407432556152344,
      "activations/layer15_attention_weight_min": -32.39863586425781,
      "activations/layer16_attention_weight_max": 35.90819549560547,
      "activations/layer16_attention_weight_min": -35.17167663574219,
      "activations/layer17_attention_weight_max": 50.7735481262207,
      "activations/layer17_attention_weight_min": -49.143592834472656,
      "activations/layer18_attention_weight_max": 50.8116340637207,
      "activations/layer18_attention_weight_min": -43.32597732543945,
      "activations/layer19_attention_weight_max": 24.187211990356445,
      "activations/layer19_attention_weight_min": -25.114452362060547,
      "activations/layer1_attention_weight_max": 16.600683212280273,
      "activations/layer1_attention_weight_min": -13.690567970275879,
      "activations/layer20_attention_weight_max": 24.78338050842285,
      "activations/layer20_attention_weight_min": -24.96324348449707,
      "activations/layer21_attention_weight_max": 41.69777297973633,
      "activations/layer21_attention_weight_min": -23.2158203125,
      "activations/layer22_attention_weight_max": 31.42778778076172,
      "activations/layer22_attention_weight_min": -25.78041648864746,
      "activations/layer23_attention_weight_max": 36.84029006958008,
      "activations/layer23_attention_weight_min": -24.424869537353516,
      "activations/layer2_attention_weight_max": 31.846263885498047,
      "activations/layer2_attention_weight_min": -29.94667625427246,
      "activations/layer3_attention_weight_max": 85.31710815429688,
      "activations/layer3_attention_weight_min": -87.1389389038086,
      "activations/layer4_attention_weight_max": 85.76210021972656,
      "activations/layer4_attention_weight_min": -86.77671813964844,
      "activations/layer5_attention_weight_max": 62.84074020385742,
      "activations/layer5_attention_weight_min": -76.23974609375,
      "activations/layer6_attention_weight_max": 50.715145111083984,
      "activations/layer6_attention_weight_min": -52.31654739379883,
      "activations/layer7_attention_weight_max": 70.47013854980469,
      "activations/layer7_attention_weight_min": -66.15352630615234,
      "activations/layer8_attention_weight_max": 50.022682189941406,
      "activations/layer8_attention_weight_min": -50.21760559082031,
      "activations/layer9_attention_weight_max": 59.2874755859375,
      "activations/layer9_attention_weight_min": -51.25747299194336,
      "epoch": 18.85,
      "learning_rate": 2.871704545454545e-05,
      "loss": 2.708,
      "step": 324450
    },
    {
      "activations/layer0_attention_weight_max": 15.664125442504883,
      "activations/layer0_attention_weight_min": -13.965639114379883,
      "activations/layer10_attention_weight_max": 36.79743194580078,
      "activations/layer10_attention_weight_min": -35.031436920166016,
      "activations/layer11_attention_weight_max": 32.858062744140625,
      "activations/layer11_attention_weight_min": -33.261871337890625,
      "activations/layer12_attention_weight_max": 19.063974380493164,
      "activations/layer12_attention_weight_min": -25.51130485534668,
      "activations/layer13_attention_weight_max": 36.1100959777832,
      "activations/layer13_attention_weight_min": -28.799198150634766,
      "activations/layer14_attention_weight_max": 41.10368347167969,
      "activations/layer14_attention_weight_min": -31.02741813659668,
      "activations/layer15_attention_weight_max": 36.46616744995117,
      "activations/layer15_attention_weight_min": -30.65253257751465,
      "activations/layer16_attention_weight_max": 33.32626724243164,
      "activations/layer16_attention_weight_min": -33.56949234008789,
      "activations/layer17_attention_weight_max": 51.73625564575195,
      "activations/layer17_attention_weight_min": -46.615108489990234,
      "activations/layer18_attention_weight_max": 47.36978530883789,
      "activations/layer18_attention_weight_min": -39.2743034362793,
      "activations/layer19_attention_weight_max": 24.75115394592285,
      "activations/layer19_attention_weight_min": -21.562841415405273,
      "activations/layer1_attention_weight_max": 15.853123664855957,
      "activations/layer1_attention_weight_min": -14.746180534362793,
      "activations/layer20_attention_weight_max": 25.672029495239258,
      "activations/layer20_attention_weight_min": -22.070463180541992,
      "activations/layer21_attention_weight_max": 39.68281936645508,
      "activations/layer21_attention_weight_min": -22.08101463317871,
      "activations/layer22_attention_weight_max": 28.565793991088867,
      "activations/layer22_attention_weight_min": -24.70100212097168,
      "activations/layer23_attention_weight_max": 38.70380783081055,
      "activations/layer23_attention_weight_min": -23.617488861083984,
      "activations/layer2_attention_weight_max": 31.705066680908203,
      "activations/layer2_attention_weight_min": -29.473663330078125,
      "activations/layer3_attention_weight_max": 84.39324188232422,
      "activations/layer3_attention_weight_min": -84.01422882080078,
      "activations/layer4_attention_weight_max": 81.5797348022461,
      "activations/layer4_attention_weight_min": -82.02677917480469,
      "activations/layer5_attention_weight_max": 61.34441375732422,
      "activations/layer5_attention_weight_min": -72.5342788696289,
      "activations/layer6_attention_weight_max": 50.778419494628906,
      "activations/layer6_attention_weight_min": -50.02534866333008,
      "activations/layer7_attention_weight_max": 63.71429443359375,
      "activations/layer7_attention_weight_min": -63.98432159423828,
      "activations/layer8_attention_weight_max": 43.910526275634766,
      "activations/layer8_attention_weight_min": -49.67588806152344,
      "activations/layer9_attention_weight_max": 42.74575424194336,
      "activations/layer9_attention_weight_min": -44.51054000854492,
      "epoch": 18.86,
      "learning_rate": 2.8698106060606056e-05,
      "loss": 2.7422,
      "step": 324500
    },
    {
      "activations/layer0_attention_weight_max": 14.580612182617188,
      "activations/layer0_attention_weight_min": -13.968064308166504,
      "activations/layer10_attention_weight_max": 32.15718078613281,
      "activations/layer10_attention_weight_min": -33.219337463378906,
      "activations/layer11_attention_weight_max": 31.21210289001465,
      "activations/layer11_attention_weight_min": -32.78252029418945,
      "activations/layer12_attention_weight_max": 19.074798583984375,
      "activations/layer12_attention_weight_min": -23.443416595458984,
      "activations/layer13_attention_weight_max": 33.068172454833984,
      "activations/layer13_attention_weight_min": -28.193588256835938,
      "activations/layer14_attention_weight_max": 34.58816909790039,
      "activations/layer14_attention_weight_min": -30.199060440063477,
      "activations/layer15_attention_weight_max": 31.908435821533203,
      "activations/layer15_attention_weight_min": -30.263866424560547,
      "activations/layer16_attention_weight_max": 33.394954681396484,
      "activations/layer16_attention_weight_min": -32.766632080078125,
      "activations/layer17_attention_weight_max": 48.52754211425781,
      "activations/layer17_attention_weight_min": -45.6769905090332,
      "activations/layer18_attention_weight_max": 48.439849853515625,
      "activations/layer18_attention_weight_min": -38.79796600341797,
      "activations/layer19_attention_weight_max": 25.62742042541504,
      "activations/layer19_attention_weight_min": -23.470426559448242,
      "activations/layer1_attention_weight_max": 15.781198501586914,
      "activations/layer1_attention_weight_min": -12.995918273925781,
      "activations/layer20_attention_weight_max": 21.53799819946289,
      "activations/layer20_attention_weight_min": -20.95737075805664,
      "activations/layer21_attention_weight_max": 43.6124382019043,
      "activations/layer21_attention_weight_min": -23.761425018310547,
      "activations/layer22_attention_weight_max": 28.347322463989258,
      "activations/layer22_attention_weight_min": -26.33766746520996,
      "activations/layer23_attention_weight_max": 33.53276062011719,
      "activations/layer23_attention_weight_min": -25.129161834716797,
      "activations/layer2_attention_weight_max": 29.95210075378418,
      "activations/layer2_attention_weight_min": -27.887557983398438,
      "activations/layer3_attention_weight_max": 72.4350814819336,
      "activations/layer3_attention_weight_min": -73.53148651123047,
      "activations/layer4_attention_weight_max": 73.75098419189453,
      "activations/layer4_attention_weight_min": -73.06725311279297,
      "activations/layer5_attention_weight_max": 57.805755615234375,
      "activations/layer5_attention_weight_min": -70.00579833984375,
      "activations/layer6_attention_weight_max": 46.77394485473633,
      "activations/layer6_attention_weight_min": -45.486873626708984,
      "activations/layer7_attention_weight_max": 61.342689514160156,
      "activations/layer7_attention_weight_min": -59.930904388427734,
      "activations/layer8_attention_weight_max": 43.68208694458008,
      "activations/layer8_attention_weight_min": -46.37761306762695,
      "activations/layer9_attention_weight_max": 38.951107025146484,
      "activations/layer9_attention_weight_min": -44.76690673828125,
      "epoch": 18.86,
      "learning_rate": 2.8679166666666665e-05,
      "loss": 2.7322,
      "step": 324550
    },
    {
      "activations/layer0_attention_weight_max": 16.158342361450195,
      "activations/layer0_attention_weight_min": -13.822328567504883,
      "activations/layer10_attention_weight_max": 36.107208251953125,
      "activations/layer10_attention_weight_min": -38.613304138183594,
      "activations/layer11_attention_weight_max": 32.674224853515625,
      "activations/layer11_attention_weight_min": -35.14884567260742,
      "activations/layer12_attention_weight_max": 19.56972885131836,
      "activations/layer12_attention_weight_min": -24.953428268432617,
      "activations/layer13_attention_weight_max": 37.11625289916992,
      "activations/layer13_attention_weight_min": -30.56468391418457,
      "activations/layer14_attention_weight_max": 35.78205108642578,
      "activations/layer14_attention_weight_min": -31.16932487487793,
      "activations/layer15_attention_weight_max": 36.2619514465332,
      "activations/layer15_attention_weight_min": -33.50514602661133,
      "activations/layer16_attention_weight_max": 37.104591369628906,
      "activations/layer16_attention_weight_min": -34.82180404663086,
      "activations/layer17_attention_weight_max": 57.0396614074707,
      "activations/layer17_attention_weight_min": -51.49997329711914,
      "activations/layer18_attention_weight_max": 52.568931579589844,
      "activations/layer18_attention_weight_min": -45.0862922668457,
      "activations/layer19_attention_weight_max": 27.001361846923828,
      "activations/layer19_attention_weight_min": -26.333410263061523,
      "activations/layer1_attention_weight_max": 14.939739227294922,
      "activations/layer1_attention_weight_min": -13.092190742492676,
      "activations/layer20_attention_weight_max": 26.214963912963867,
      "activations/layer20_attention_weight_min": -25.006362915039062,
      "activations/layer21_attention_weight_max": 39.98543930053711,
      "activations/layer21_attention_weight_min": -25.59423065185547,
      "activations/layer22_attention_weight_max": 31.673160552978516,
      "activations/layer22_attention_weight_min": -26.609405517578125,
      "activations/layer23_attention_weight_max": 36.345062255859375,
      "activations/layer23_attention_weight_min": -24.257726669311523,
      "activations/layer2_attention_weight_max": 31.99574089050293,
      "activations/layer2_attention_weight_min": -31.631559371948242,
      "activations/layer3_attention_weight_max": 90.46171569824219,
      "activations/layer3_attention_weight_min": -85.12124633789062,
      "activations/layer4_attention_weight_max": 83.80989074707031,
      "activations/layer4_attention_weight_min": -80.55389404296875,
      "activations/layer5_attention_weight_max": 60.3101921081543,
      "activations/layer5_attention_weight_min": -71.94729614257812,
      "activations/layer6_attention_weight_max": 49.21829605102539,
      "activations/layer6_attention_weight_min": -52.59084701538086,
      "activations/layer7_attention_weight_max": 65.01578521728516,
      "activations/layer7_attention_weight_min": -61.66686248779297,
      "activations/layer8_attention_weight_max": 47.00238037109375,
      "activations/layer8_attention_weight_min": -49.21155548095703,
      "activations/layer9_attention_weight_max": 47.97450256347656,
      "activations/layer9_attention_weight_min": -46.64846420288086,
      "epoch": 18.86,
      "learning_rate": 2.8660227272727273e-05,
      "loss": 2.7397,
      "step": 324600
    },
    {
      "activations/layer0_attention_weight_max": 16.587026596069336,
      "activations/layer0_attention_weight_min": -13.956035614013672,
      "activations/layer10_attention_weight_max": 33.06237030029297,
      "activations/layer10_attention_weight_min": -34.6209602355957,
      "activations/layer11_attention_weight_max": 30.478397369384766,
      "activations/layer11_attention_weight_min": -34.15778350830078,
      "activations/layer12_attention_weight_max": 19.421268463134766,
      "activations/layer12_attention_weight_min": -26.74576759338379,
      "activations/layer13_attention_weight_max": 39.79026412963867,
      "activations/layer13_attention_weight_min": -28.641393661499023,
      "activations/layer14_attention_weight_max": 36.27229690551758,
      "activations/layer14_attention_weight_min": -31.745731353759766,
      "activations/layer15_attention_weight_max": 35.817378997802734,
      "activations/layer15_attention_weight_min": -31.759593963623047,
      "activations/layer16_attention_weight_max": 35.35226821899414,
      "activations/layer16_attention_weight_min": -33.6064567565918,
      "activations/layer17_attention_weight_max": 53.97970962524414,
      "activations/layer17_attention_weight_min": -47.39759063720703,
      "activations/layer18_attention_weight_max": 52.147823333740234,
      "activations/layer18_attention_weight_min": -42.846954345703125,
      "activations/layer19_attention_weight_max": 23.541027069091797,
      "activations/layer19_attention_weight_min": -23.77662467956543,
      "activations/layer1_attention_weight_max": 15.525782585144043,
      "activations/layer1_attention_weight_min": -14.335466384887695,
      "activations/layer20_attention_weight_max": 22.538707733154297,
      "activations/layer20_attention_weight_min": -22.391874313354492,
      "activations/layer21_attention_weight_max": 36.75062561035156,
      "activations/layer21_attention_weight_min": -22.938486099243164,
      "activations/layer22_attention_weight_max": 30.211641311645508,
      "activations/layer22_attention_weight_min": -24.91342544555664,
      "activations/layer23_attention_weight_max": 39.09000015258789,
      "activations/layer23_attention_weight_min": -23.776710510253906,
      "activations/layer2_attention_weight_max": 30.185226440429688,
      "activations/layer2_attention_weight_min": -29.329687118530273,
      "activations/layer3_attention_weight_max": 84.29417419433594,
      "activations/layer3_attention_weight_min": -79.06600952148438,
      "activations/layer4_attention_weight_max": 80.71675109863281,
      "activations/layer4_attention_weight_min": -79.40225219726562,
      "activations/layer5_attention_weight_max": 61.60472869873047,
      "activations/layer5_attention_weight_min": -75.11720275878906,
      "activations/layer6_attention_weight_max": 46.40437316894531,
      "activations/layer6_attention_weight_min": -47.904930114746094,
      "activations/layer7_attention_weight_max": 63.887447357177734,
      "activations/layer7_attention_weight_min": -62.30405044555664,
      "activations/layer8_attention_weight_max": 43.432716369628906,
      "activations/layer8_attention_weight_min": -50.31301498413086,
      "activations/layer9_attention_weight_max": 44.807064056396484,
      "activations/layer9_attention_weight_min": -45.12345504760742,
      "epoch": 18.86,
      "learning_rate": 2.8641287878787874e-05,
      "loss": 2.7362,
      "step": 324650
    },
    {
      "activations/layer0_attention_weight_max": 15.53947639465332,
      "activations/layer0_attention_weight_min": -14.179328918457031,
      "activations/layer10_attention_weight_max": 39.81818771362305,
      "activations/layer10_attention_weight_min": -36.419921875,
      "activations/layer11_attention_weight_max": 37.16395568847656,
      "activations/layer11_attention_weight_min": -33.76788330078125,
      "activations/layer12_attention_weight_max": 20.901525497436523,
      "activations/layer12_attention_weight_min": -28.057559967041016,
      "activations/layer13_attention_weight_max": 35.17534637451172,
      "activations/layer13_attention_weight_min": -29.690603256225586,
      "activations/layer14_attention_weight_max": 38.35243225097656,
      "activations/layer14_attention_weight_min": -31.08692169189453,
      "activations/layer15_attention_weight_max": 34.42460250854492,
      "activations/layer15_attention_weight_min": -31.262889862060547,
      "activations/layer16_attention_weight_max": 34.21428680419922,
      "activations/layer16_attention_weight_min": -32.24374771118164,
      "activations/layer17_attention_weight_max": 52.52638244628906,
      "activations/layer17_attention_weight_min": -45.917503356933594,
      "activations/layer18_attention_weight_max": 46.9600715637207,
      "activations/layer18_attention_weight_min": -41.174076080322266,
      "activations/layer19_attention_weight_max": 23.407072067260742,
      "activations/layer19_attention_weight_min": -23.79243278503418,
      "activations/layer1_attention_weight_max": 16.52733039855957,
      "activations/layer1_attention_weight_min": -13.896276473999023,
      "activations/layer20_attention_weight_max": 22.817182540893555,
      "activations/layer20_attention_weight_min": -22.335359573364258,
      "activations/layer21_attention_weight_max": 36.661888122558594,
      "activations/layer21_attention_weight_min": -22.321908950805664,
      "activations/layer22_attention_weight_max": 27.53053092956543,
      "activations/layer22_attention_weight_min": -24.949440002441406,
      "activations/layer23_attention_weight_max": 37.08092498779297,
      "activations/layer23_attention_weight_min": -25.47945785522461,
      "activations/layer2_attention_weight_max": 32.758296966552734,
      "activations/layer2_attention_weight_min": -31.023794174194336,
      "activations/layer3_attention_weight_max": 90.3111572265625,
      "activations/layer3_attention_weight_min": -85.57206726074219,
      "activations/layer4_attention_weight_max": 81.05290985107422,
      "activations/layer4_attention_weight_min": -80.94258880615234,
      "activations/layer5_attention_weight_max": 59.5789909362793,
      "activations/layer5_attention_weight_min": -64.54560852050781,
      "activations/layer6_attention_weight_max": 46.7880744934082,
      "activations/layer6_attention_weight_min": -48.467185974121094,
      "activations/layer7_attention_weight_max": 69.30037689208984,
      "activations/layer7_attention_weight_min": -69.60718536376953,
      "activations/layer8_attention_weight_max": 51.95888137817383,
      "activations/layer8_attention_weight_min": -52.4420051574707,
      "activations/layer9_attention_weight_max": 50.9893913269043,
      "activations/layer9_attention_weight_min": -47.861595153808594,
      "epoch": 18.87,
      "learning_rate": 2.8622348484848483e-05,
      "loss": 2.7278,
      "step": 324700
    },
    {
      "activations/layer0_attention_weight_max": 15.430685997009277,
      "activations/layer0_attention_weight_min": -13.72227668762207,
      "activations/layer10_attention_weight_max": 34.396270751953125,
      "activations/layer10_attention_weight_min": -34.46706008911133,
      "activations/layer11_attention_weight_max": 31.80135726928711,
      "activations/layer11_attention_weight_min": -33.35717010498047,
      "activations/layer12_attention_weight_max": 18.70257568359375,
      "activations/layer12_attention_weight_min": -26.03704071044922,
      "activations/layer13_attention_weight_max": 31.131641387939453,
      "activations/layer13_attention_weight_min": -28.49225425720215,
      "activations/layer14_attention_weight_max": 35.517669677734375,
      "activations/layer14_attention_weight_min": -29.741504669189453,
      "activations/layer15_attention_weight_max": 32.40226745605469,
      "activations/layer15_attention_weight_min": -29.099626541137695,
      "activations/layer16_attention_weight_max": 34.387508392333984,
      "activations/layer16_attention_weight_min": -31.128536224365234,
      "activations/layer17_attention_weight_max": 51.75355911254883,
      "activations/layer17_attention_weight_min": -46.11540222167969,
      "activations/layer18_attention_weight_max": 42.69534683227539,
      "activations/layer18_attention_weight_min": -39.11274337768555,
      "activations/layer19_attention_weight_max": 22.84653663635254,
      "activations/layer19_attention_weight_min": -22.060461044311523,
      "activations/layer1_attention_weight_max": 15.083425521850586,
      "activations/layer1_attention_weight_min": -14.069161415100098,
      "activations/layer20_attention_weight_max": 21.838247299194336,
      "activations/layer20_attention_weight_min": -21.60044288635254,
      "activations/layer21_attention_weight_max": 32.10475158691406,
      "activations/layer21_attention_weight_min": -21.32933235168457,
      "activations/layer22_attention_weight_max": 26.71184539794922,
      "activations/layer22_attention_weight_min": -25.221757888793945,
      "activations/layer23_attention_weight_max": 33.99620819091797,
      "activations/layer23_attention_weight_min": -23.649694442749023,
      "activations/layer2_attention_weight_max": 33.683380126953125,
      "activations/layer2_attention_weight_min": -29.412649154663086,
      "activations/layer3_attention_weight_max": 87.3923568725586,
      "activations/layer3_attention_weight_min": -89.76241302490234,
      "activations/layer4_attention_weight_max": 82.88671112060547,
      "activations/layer4_attention_weight_min": -81.47853088378906,
      "activations/layer5_attention_weight_max": 60.88554000854492,
      "activations/layer5_attention_weight_min": -70.86709594726562,
      "activations/layer6_attention_weight_max": 47.39735794067383,
      "activations/layer6_attention_weight_min": -47.795310974121094,
      "activations/layer7_attention_weight_max": 63.934356689453125,
      "activations/layer7_attention_weight_min": -58.967472076416016,
      "activations/layer8_attention_weight_max": 44.367000579833984,
      "activations/layer8_attention_weight_min": -48.73261642456055,
      "activations/layer9_attention_weight_max": 41.981781005859375,
      "activations/layer9_attention_weight_min": -44.69731521606445,
      "epoch": 18.87,
      "learning_rate": 2.8603409090909088e-05,
      "loss": 2.7233,
      "step": 324750
    },
    {
      "activations/layer0_attention_weight_max": 15.759486198425293,
      "activations/layer0_attention_weight_min": -13.753068923950195,
      "activations/layer10_attention_weight_max": 42.223243713378906,
      "activations/layer10_attention_weight_min": -39.7601203918457,
      "activations/layer11_attention_weight_max": 42.98809814453125,
      "activations/layer11_attention_weight_min": -39.00210952758789,
      "activations/layer12_attention_weight_max": 21.992774963378906,
      "activations/layer12_attention_weight_min": -23.88213539123535,
      "activations/layer13_attention_weight_max": 46.142574310302734,
      "activations/layer13_attention_weight_min": -33.344947814941406,
      "activations/layer14_attention_weight_max": 47.789390563964844,
      "activations/layer14_attention_weight_min": -34.518272399902344,
      "activations/layer15_attention_weight_max": 48.804134368896484,
      "activations/layer15_attention_weight_min": -33.925071716308594,
      "activations/layer16_attention_weight_max": 39.466827392578125,
      "activations/layer16_attention_weight_min": -34.715179443359375,
      "activations/layer17_attention_weight_max": 60.35732650756836,
      "activations/layer17_attention_weight_min": -54.5276985168457,
      "activations/layer18_attention_weight_max": 59.51795196533203,
      "activations/layer18_attention_weight_min": -50.54954528808594,
      "activations/layer19_attention_weight_max": 29.041467666625977,
      "activations/layer19_attention_weight_min": -27.37283706665039,
      "activations/layer1_attention_weight_max": 16.031723022460938,
      "activations/layer1_attention_weight_min": -14.017692565917969,
      "activations/layer20_attention_weight_max": 26.079668045043945,
      "activations/layer20_attention_weight_min": -25.068559646606445,
      "activations/layer21_attention_weight_max": 46.1085090637207,
      "activations/layer21_attention_weight_min": -25.95340919494629,
      "activations/layer22_attention_weight_max": 32.509220123291016,
      "activations/layer22_attention_weight_min": -26.719167709350586,
      "activations/layer23_attention_weight_max": 36.98283386230469,
      "activations/layer23_attention_weight_min": -24.998096466064453,
      "activations/layer2_attention_weight_max": 37.0391960144043,
      "activations/layer2_attention_weight_min": -34.40462875366211,
      "activations/layer3_attention_weight_max": 102.56288146972656,
      "activations/layer3_attention_weight_min": -94.02242279052734,
      "activations/layer4_attention_weight_max": 101.7054672241211,
      "activations/layer4_attention_weight_min": -93.7559814453125,
      "activations/layer5_attention_weight_max": 67.35675048828125,
      "activations/layer5_attention_weight_min": -68.35880279541016,
      "activations/layer6_attention_weight_max": 56.95918273925781,
      "activations/layer6_attention_weight_min": -54.88483428955078,
      "activations/layer7_attention_weight_max": 83.03475952148438,
      "activations/layer7_attention_weight_min": -66.57968139648438,
      "activations/layer8_attention_weight_max": 53.98823928833008,
      "activations/layer8_attention_weight_min": -51.51466369628906,
      "activations/layer9_attention_weight_max": 59.28462600708008,
      "activations/layer9_attention_weight_min": -47.24282455444336,
      "epoch": 18.87,
      "learning_rate": 2.8584469696969696e-05,
      "loss": 2.7327,
      "step": 324800
    },
    {
      "activations/layer0_attention_weight_max": 16.432188034057617,
      "activations/layer0_attention_weight_min": -13.829936981201172,
      "activations/layer10_attention_weight_max": 37.95878601074219,
      "activations/layer10_attention_weight_min": -35.414398193359375,
      "activations/layer11_attention_weight_max": 36.685150146484375,
      "activations/layer11_attention_weight_min": -37.11038589477539,
      "activations/layer12_attention_weight_max": 18.53875732421875,
      "activations/layer12_attention_weight_min": -27.37873077392578,
      "activations/layer13_attention_weight_max": 29.862422943115234,
      "activations/layer13_attention_weight_min": -28.081588745117188,
      "activations/layer14_attention_weight_max": 34.990936279296875,
      "activations/layer14_attention_weight_min": -29.25406265258789,
      "activations/layer15_attention_weight_max": 33.236751556396484,
      "activations/layer15_attention_weight_min": -30.57741928100586,
      "activations/layer16_attention_weight_max": 31.999523162841797,
      "activations/layer16_attention_weight_min": -29.86064338684082,
      "activations/layer17_attention_weight_max": 52.64665603637695,
      "activations/layer17_attention_weight_min": -45.30487060546875,
      "activations/layer18_attention_weight_max": 43.85399627685547,
      "activations/layer18_attention_weight_min": -40.53447723388672,
      "activations/layer19_attention_weight_max": 22.097881317138672,
      "activations/layer19_attention_weight_min": -23.298330307006836,
      "activations/layer1_attention_weight_max": 16.76639175415039,
      "activations/layer1_attention_weight_min": -15.467364311218262,
      "activations/layer20_attention_weight_max": 21.374589920043945,
      "activations/layer20_attention_weight_min": -21.185680389404297,
      "activations/layer21_attention_weight_max": 33.00404357910156,
      "activations/layer21_attention_weight_min": -21.51226806640625,
      "activations/layer22_attention_weight_max": 28.166961669921875,
      "activations/layer22_attention_weight_min": -25.555927276611328,
      "activations/layer23_attention_weight_max": 36.57666015625,
      "activations/layer23_attention_weight_min": -26.767070770263672,
      "activations/layer2_attention_weight_max": 30.003408432006836,
      "activations/layer2_attention_weight_min": -29.207626342773438,
      "activations/layer3_attention_weight_max": 86.20922088623047,
      "activations/layer3_attention_weight_min": -83.9157485961914,
      "activations/layer4_attention_weight_max": 88.73905181884766,
      "activations/layer4_attention_weight_min": -86.69347381591797,
      "activations/layer5_attention_weight_max": 60.849971771240234,
      "activations/layer5_attention_weight_min": -72.65760803222656,
      "activations/layer6_attention_weight_max": 49.91048812866211,
      "activations/layer6_attention_weight_min": -50.21269226074219,
      "activations/layer7_attention_weight_max": 66.18376159667969,
      "activations/layer7_attention_weight_min": -65.55980682373047,
      "activations/layer8_attention_weight_max": 49.814857482910156,
      "activations/layer8_attention_weight_min": -48.813358306884766,
      "activations/layer9_attention_weight_max": 50.5021858215332,
      "activations/layer9_attention_weight_min": -47.0891227722168,
      "epoch": 18.88,
      "learning_rate": 2.85655303030303e-05,
      "loss": 2.7398,
      "step": 324850
    },
    {
      "activations/layer0_attention_weight_max": 15.913728713989258,
      "activations/layer0_attention_weight_min": -13.971209526062012,
      "activations/layer10_attention_weight_max": 36.54560089111328,
      "activations/layer10_attention_weight_min": -35.7501335144043,
      "activations/layer11_attention_weight_max": 34.56670379638672,
      "activations/layer11_attention_weight_min": -36.59204864501953,
      "activations/layer12_attention_weight_max": 19.50994300842285,
      "activations/layer12_attention_weight_min": -25.249853134155273,
      "activations/layer13_attention_weight_max": 37.79523849487305,
      "activations/layer13_attention_weight_min": -27.01640510559082,
      "activations/layer14_attention_weight_max": 41.0349235534668,
      "activations/layer14_attention_weight_min": -30.152664184570312,
      "activations/layer15_attention_weight_max": 36.078643798828125,
      "activations/layer15_attention_weight_min": -30.067777633666992,
      "activations/layer16_attention_weight_max": 36.227928161621094,
      "activations/layer16_attention_weight_min": -34.6639404296875,
      "activations/layer17_attention_weight_max": 52.60296630859375,
      "activations/layer17_attention_weight_min": -45.50957489013672,
      "activations/layer18_attention_weight_max": 47.51496887207031,
      "activations/layer18_attention_weight_min": -39.94517517089844,
      "activations/layer19_attention_weight_max": 25.03151512145996,
      "activations/layer19_attention_weight_min": -22.300966262817383,
      "activations/layer1_attention_weight_max": 15.359061241149902,
      "activations/layer1_attention_weight_min": -13.989031791687012,
      "activations/layer20_attention_weight_max": 21.857730865478516,
      "activations/layer20_attention_weight_min": -20.617633819580078,
      "activations/layer21_attention_weight_max": 34.80583572387695,
      "activations/layer21_attention_weight_min": -26.059118270874023,
      "activations/layer22_attention_weight_max": 29.995019912719727,
      "activations/layer22_attention_weight_min": -26.09007453918457,
      "activations/layer23_attention_weight_max": 35.22998046875,
      "activations/layer23_attention_weight_min": -25.5679988861084,
      "activations/layer2_attention_weight_max": 31.691890716552734,
      "activations/layer2_attention_weight_min": -30.40230369567871,
      "activations/layer3_attention_weight_max": 84.45249938964844,
      "activations/layer3_attention_weight_min": -84.42265319824219,
      "activations/layer4_attention_weight_max": 81.11873626708984,
      "activations/layer4_attention_weight_min": -81.85820770263672,
      "activations/layer5_attention_weight_max": 62.51191329956055,
      "activations/layer5_attention_weight_min": -74.68304443359375,
      "activations/layer6_attention_weight_max": 55.234745025634766,
      "activations/layer6_attention_weight_min": -52.16902542114258,
      "activations/layer7_attention_weight_max": 66.66641998291016,
      "activations/layer7_attention_weight_min": -65.62315368652344,
      "activations/layer8_attention_weight_max": 46.45224380493164,
      "activations/layer8_attention_weight_min": -49.91770553588867,
      "activations/layer9_attention_weight_max": 44.921302795410156,
      "activations/layer9_attention_weight_min": -47.901649475097656,
      "epoch": 18.88,
      "learning_rate": 2.854659090909091e-05,
      "loss": 2.7093,
      "step": 324900
    },
    {
      "activations/layer0_attention_weight_max": 15.621994018554688,
      "activations/layer0_attention_weight_min": -14.008179664611816,
      "activations/layer10_attention_weight_max": 36.35223388671875,
      "activations/layer10_attention_weight_min": -38.8263053894043,
      "activations/layer11_attention_weight_max": 35.044281005859375,
      "activations/layer11_attention_weight_min": -38.4176025390625,
      "activations/layer12_attention_weight_max": 19.163557052612305,
      "activations/layer12_attention_weight_min": -25.09829330444336,
      "activations/layer13_attention_weight_max": 38.66688537597656,
      "activations/layer13_attention_weight_min": -29.0360107421875,
      "activations/layer14_attention_weight_max": 37.12487030029297,
      "activations/layer14_attention_weight_min": -31.134214401245117,
      "activations/layer15_attention_weight_max": 35.48221969604492,
      "activations/layer15_attention_weight_min": -32.03998947143555,
      "activations/layer16_attention_weight_max": 35.896305084228516,
      "activations/layer16_attention_weight_min": -33.52479934692383,
      "activations/layer17_attention_weight_max": 56.19705581665039,
      "activations/layer17_attention_weight_min": -47.89863586425781,
      "activations/layer18_attention_weight_max": 48.699005126953125,
      "activations/layer18_attention_weight_min": -42.62700653076172,
      "activations/layer19_attention_weight_max": 26.191205978393555,
      "activations/layer19_attention_weight_min": -25.72378921508789,
      "activations/layer1_attention_weight_max": 16.172412872314453,
      "activations/layer1_attention_weight_min": -13.571258544921875,
      "activations/layer20_attention_weight_max": 22.435941696166992,
      "activations/layer20_attention_weight_min": -22.08949089050293,
      "activations/layer21_attention_weight_max": 35.45793914794922,
      "activations/layer21_attention_weight_min": -21.386428833007812,
      "activations/layer22_attention_weight_max": 32.54613494873047,
      "activations/layer22_attention_weight_min": -28.59151840209961,
      "activations/layer23_attention_weight_max": 39.95702362060547,
      "activations/layer23_attention_weight_min": -25.199445724487305,
      "activations/layer2_attention_weight_max": 31.292539596557617,
      "activations/layer2_attention_weight_min": -31.148937225341797,
      "activations/layer3_attention_weight_max": 87.3046646118164,
      "activations/layer3_attention_weight_min": -86.62105560302734,
      "activations/layer4_attention_weight_max": 84.03956604003906,
      "activations/layer4_attention_weight_min": -84.00475311279297,
      "activations/layer5_attention_weight_max": 61.07908630371094,
      "activations/layer5_attention_weight_min": -69.69815063476562,
      "activations/layer6_attention_weight_max": 51.977046966552734,
      "activations/layer6_attention_weight_min": -52.47708511352539,
      "activations/layer7_attention_weight_max": 67.41060638427734,
      "activations/layer7_attention_weight_min": -68.36302947998047,
      "activations/layer8_attention_weight_max": 45.12967300415039,
      "activations/layer8_attention_weight_min": -51.840152740478516,
      "activations/layer9_attention_weight_max": 45.50751876831055,
      "activations/layer9_attention_weight_min": -46.941341400146484,
      "epoch": 18.88,
      "learning_rate": 2.852765151515151e-05,
      "loss": 2.7403,
      "step": 324950
    },
    {
      "activations/layer0_attention_weight_max": 15.664721488952637,
      "activations/layer0_attention_weight_min": -13.748068809509277,
      "activations/layer10_attention_weight_max": 31.897850036621094,
      "activations/layer10_attention_weight_min": -32.18747329711914,
      "activations/layer11_attention_weight_max": 29.942005157470703,
      "activations/layer11_attention_weight_min": -31.481346130371094,
      "activations/layer12_attention_weight_max": 18.62220573425293,
      "activations/layer12_attention_weight_min": -27.727121353149414,
      "activations/layer13_attention_weight_max": 32.663421630859375,
      "activations/layer13_attention_weight_min": -26.5932559967041,
      "activations/layer14_attention_weight_max": 32.45833206176758,
      "activations/layer14_attention_weight_min": -29.35608673095703,
      "activations/layer15_attention_weight_max": 32.230655670166016,
      "activations/layer15_attention_weight_min": -28.64622688293457,
      "activations/layer16_attention_weight_max": 31.75075340270996,
      "activations/layer16_attention_weight_min": -30.835987091064453,
      "activations/layer17_attention_weight_max": 47.27431106567383,
      "activations/layer17_attention_weight_min": -42.5849494934082,
      "activations/layer18_attention_weight_max": 43.74240493774414,
      "activations/layer18_attention_weight_min": -39.21086502075195,
      "activations/layer19_attention_weight_max": 24.241695404052734,
      "activations/layer19_attention_weight_min": -21.73946189880371,
      "activations/layer1_attention_weight_max": 15.834598541259766,
      "activations/layer1_attention_weight_min": -14.373473167419434,
      "activations/layer20_attention_weight_max": 23.781137466430664,
      "activations/layer20_attention_weight_min": -20.099042892456055,
      "activations/layer21_attention_weight_max": 35.35688018798828,
      "activations/layer21_attention_weight_min": -21.43727684020996,
      "activations/layer22_attention_weight_max": 28.886323928833008,
      "activations/layer22_attention_weight_min": -23.794498443603516,
      "activations/layer23_attention_weight_max": 36.92158508300781,
      "activations/layer23_attention_weight_min": -21.965801239013672,
      "activations/layer2_attention_weight_max": 30.564603805541992,
      "activations/layer2_attention_weight_min": -29.488910675048828,
      "activations/layer3_attention_weight_max": 81.556640625,
      "activations/layer3_attention_weight_min": -82.72164916992188,
      "activations/layer4_attention_weight_max": 80.81134033203125,
      "activations/layer4_attention_weight_min": -80.1265640258789,
      "activations/layer5_attention_weight_max": 58.06986999511719,
      "activations/layer5_attention_weight_min": -71.47015380859375,
      "activations/layer6_attention_weight_max": 47.9379997253418,
      "activations/layer6_attention_weight_min": -48.1328239440918,
      "activations/layer7_attention_weight_max": 60.667625427246094,
      "activations/layer7_attention_weight_min": -60.2514762878418,
      "activations/layer8_attention_weight_max": 47.40251922607422,
      "activations/layer8_attention_weight_min": -48.80585479736328,
      "activations/layer9_attention_weight_max": 41.07657241821289,
      "activations/layer9_attention_weight_min": -43.57149887084961,
      "epoch": 18.88,
      "learning_rate": 2.850871212121212e-05,
      "loss": 2.735,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_loss": 2.6875,
      "eval_runtime": 8.4971,
      "eval_samples_per_second": 505.346,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.4971,
      "eval_openwebtext_samples_per_second": 505.346,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.9592,
      "eval_wikitext_samples_per_second": 232.753,
      "step": 325000
    },
    {
      "epoch": 18.88,
      "eval_lambada_loss": 2.46875,
      "eval_lambada_ppl": 11.807678025521156,
      "eval_lambada_runtime": 9.5538,
      "eval_lambada_samples_per_second": 509.641,
      "step": 325000
    },
    {
      "activations/layer0_attention_weight_max": 15.780587196350098,
      "activations/layer0_attention_weight_min": -13.505521774291992,
      "activations/layer10_attention_weight_max": 34.700992584228516,
      "activations/layer10_attention_weight_min": -37.06550216674805,
      "activations/layer11_attention_weight_max": 32.951568603515625,
      "activations/layer11_attention_weight_min": -33.315635681152344,
      "activations/layer12_attention_weight_max": 18.245046615600586,
      "activations/layer12_attention_weight_min": -25.978166580200195,
      "activations/layer13_attention_weight_max": 31.97107696533203,
      "activations/layer13_attention_weight_min": -27.024166107177734,
      "activations/layer14_attention_weight_max": 32.694087982177734,
      "activations/layer14_attention_weight_min": -29.853281021118164,
      "activations/layer15_attention_weight_max": 30.591445922851562,
      "activations/layer15_attention_weight_min": -31.373733520507812,
      "activations/layer16_attention_weight_max": 32.9707145690918,
      "activations/layer16_attention_weight_min": -32.0767707824707,
      "activations/layer17_attention_weight_max": 48.51918411254883,
      "activations/layer17_attention_weight_min": -43.84735870361328,
      "activations/layer18_attention_weight_max": 44.06163024902344,
      "activations/layer18_attention_weight_min": -39.39578628540039,
      "activations/layer19_attention_weight_max": 23.530668258666992,
      "activations/layer19_attention_weight_min": -25.040424346923828,
      "activations/layer1_attention_weight_max": 15.417136192321777,
      "activations/layer1_attention_weight_min": -12.45561408996582,
      "activations/layer20_attention_weight_max": 23.110271453857422,
      "activations/layer20_attention_weight_min": -22.676799774169922,
      "activations/layer21_attention_weight_max": 44.4802360534668,
      "activations/layer21_attention_weight_min": -24.511211395263672,
      "activations/layer22_attention_weight_max": 28.784408569335938,
      "activations/layer22_attention_weight_min": -25.755414962768555,
      "activations/layer23_attention_weight_max": 38.9260368347168,
      "activations/layer23_attention_weight_min": -27.28394889831543,
      "activations/layer2_attention_weight_max": 33.501731872558594,
      "activations/layer2_attention_weight_min": -30.28498649597168,
      "activations/layer3_attention_weight_max": 86.32874298095703,
      "activations/layer3_attention_weight_min": -85.51673126220703,
      "activations/layer4_attention_weight_max": 84.246337890625,
      "activations/layer4_attention_weight_min": -81.98478698730469,
      "activations/layer5_attention_weight_max": 61.09226989746094,
      "activations/layer5_attention_weight_min": -69.43402099609375,
      "activations/layer6_attention_weight_max": 49.3050651550293,
      "activations/layer6_attention_weight_min": -50.30779266357422,
      "activations/layer7_attention_weight_max": 61.961997985839844,
      "activations/layer7_attention_weight_min": -66.77759552001953,
      "activations/layer8_attention_weight_max": 45.415672302246094,
      "activations/layer8_attention_weight_min": -50.745994567871094,
      "activations/layer9_attention_weight_max": 43.82767868041992,
      "activations/layer9_attention_weight_min": -49.400840759277344,
      "epoch": 18.89,
      "learning_rate": 2.8489772727272724e-05,
      "loss": 2.7237,
      "step": 325050
    },
    {
      "activations/layer0_attention_weight_max": 15.48460865020752,
      "activations/layer0_attention_weight_min": -13.121658325195312,
      "activations/layer10_attention_weight_max": 39.298187255859375,
      "activations/layer10_attention_weight_min": -37.821285247802734,
      "activations/layer11_attention_weight_max": 38.80317687988281,
      "activations/layer11_attention_weight_min": -36.65524673461914,
      "activations/layer12_attention_weight_max": 19.867761611938477,
      "activations/layer12_attention_weight_min": -24.10153579711914,
      "activations/layer13_attention_weight_max": 35.30607604980469,
      "activations/layer13_attention_weight_min": -27.59157943725586,
      "activations/layer14_attention_weight_max": 38.403533935546875,
      "activations/layer14_attention_weight_min": -31.491792678833008,
      "activations/layer15_attention_weight_max": 38.216129302978516,
      "activations/layer15_attention_weight_min": -30.574859619140625,
      "activations/layer16_attention_weight_max": 37.384063720703125,
      "activations/layer16_attention_weight_min": -33.85997772216797,
      "activations/layer17_attention_weight_max": 54.08251190185547,
      "activations/layer17_attention_weight_min": -45.16118621826172,
      "activations/layer18_attention_weight_max": 51.13433074951172,
      "activations/layer18_attention_weight_min": -42.92607116699219,
      "activations/layer19_attention_weight_max": 25.013168334960938,
      "activations/layer19_attention_weight_min": -22.910383224487305,
      "activations/layer1_attention_weight_max": 16.618619918823242,
      "activations/layer1_attention_weight_min": -13.916812896728516,
      "activations/layer20_attention_weight_max": 25.484092712402344,
      "activations/layer20_attention_weight_min": -21.686365127563477,
      "activations/layer21_attention_weight_max": 37.67524337768555,
      "activations/layer21_attention_weight_min": -20.928619384765625,
      "activations/layer22_attention_weight_max": 31.37941551208496,
      "activations/layer22_attention_weight_min": -26.537569046020508,
      "activations/layer23_attention_weight_max": 40.609764099121094,
      "activations/layer23_attention_weight_min": -25.052244186401367,
      "activations/layer2_attention_weight_max": 36.07146453857422,
      "activations/layer2_attention_weight_min": -31.843997955322266,
      "activations/layer3_attention_weight_max": 90.91609954833984,
      "activations/layer3_attention_weight_min": -81.68521118164062,
      "activations/layer4_attention_weight_max": 90.38291931152344,
      "activations/layer4_attention_weight_min": -80.17927551269531,
      "activations/layer5_attention_weight_max": 58.58726119995117,
      "activations/layer5_attention_weight_min": -73.03218078613281,
      "activations/layer6_attention_weight_max": 50.4471549987793,
      "activations/layer6_attention_weight_min": -49.08363723754883,
      "activations/layer7_attention_weight_max": 68.69091796875,
      "activations/layer7_attention_weight_min": -64.99296569824219,
      "activations/layer8_attention_weight_max": 52.753196716308594,
      "activations/layer8_attention_weight_min": -50.27974319458008,
      "activations/layer9_attention_weight_max": 54.84382247924805,
      "activations/layer9_attention_weight_min": -46.52077102661133,
      "epoch": 18.89,
      "learning_rate": 2.8470833333333332e-05,
      "loss": 2.7217,
      "step": 325100
    },
    {
      "activations/layer0_attention_weight_max": 16.61136245727539,
      "activations/layer0_attention_weight_min": -13.60935115814209,
      "activations/layer10_attention_weight_max": 38.48942184448242,
      "activations/layer10_attention_weight_min": -38.171329498291016,
      "activations/layer11_attention_weight_max": 36.61212921142578,
      "activations/layer11_attention_weight_min": -37.18981170654297,
      "activations/layer12_attention_weight_max": 18.536514282226562,
      "activations/layer12_attention_weight_min": -22.897092819213867,
      "activations/layer13_attention_weight_max": 38.368980407714844,
      "activations/layer13_attention_weight_min": -31.62790298461914,
      "activations/layer14_attention_weight_max": 35.122161865234375,
      "activations/layer14_attention_weight_min": -30.24025535583496,
      "activations/layer15_attention_weight_max": 33.2137565612793,
      "activations/layer15_attention_weight_min": -31.071184158325195,
      "activations/layer16_attention_weight_max": 32.845703125,
      "activations/layer16_attention_weight_min": -30.64328956604004,
      "activations/layer17_attention_weight_max": 49.08150100708008,
      "activations/layer17_attention_weight_min": -43.46080780029297,
      "activations/layer18_attention_weight_max": 45.11563491821289,
      "activations/layer18_attention_weight_min": -40.7360954284668,
      "activations/layer19_attention_weight_max": 23.71541976928711,
      "activations/layer19_attention_weight_min": -21.211822509765625,
      "activations/layer1_attention_weight_max": 15.618502616882324,
      "activations/layer1_attention_weight_min": -15.841081619262695,
      "activations/layer20_attention_weight_max": 22.14019012451172,
      "activations/layer20_attention_weight_min": -21.896699905395508,
      "activations/layer21_attention_weight_max": 31.75693702697754,
      "activations/layer21_attention_weight_min": -22.335968017578125,
      "activations/layer22_attention_weight_max": 26.76497459411621,
      "activations/layer22_attention_weight_min": -26.076841354370117,
      "activations/layer23_attention_weight_max": 35.27265930175781,
      "activations/layer23_attention_weight_min": -24.042953491210938,
      "activations/layer2_attention_weight_max": 32.64134979248047,
      "activations/layer2_attention_weight_min": -31.512197494506836,
      "activations/layer3_attention_weight_max": 89.44158935546875,
      "activations/layer3_attention_weight_min": -88.34662628173828,
      "activations/layer4_attention_weight_max": 87.19580078125,
      "activations/layer4_attention_weight_min": -86.45402526855469,
      "activations/layer5_attention_weight_max": 62.46697998046875,
      "activations/layer5_attention_weight_min": -69.85780334472656,
      "activations/layer6_attention_weight_max": 55.713687896728516,
      "activations/layer6_attention_weight_min": -55.09233093261719,
      "activations/layer7_attention_weight_max": 66.96366882324219,
      "activations/layer7_attention_weight_min": -65.41187286376953,
      "activations/layer8_attention_weight_max": 51.858238220214844,
      "activations/layer8_attention_weight_min": -53.13050842285156,
      "activations/layer9_attention_weight_max": 48.61506271362305,
      "activations/layer9_attention_weight_min": -49.7336540222168,
      "epoch": 18.89,
      "learning_rate": 2.8452272727272723e-05,
      "loss": 2.7492,
      "step": 325150
    },
    {
      "activations/layer0_attention_weight_max": 15.030890464782715,
      "activations/layer0_attention_weight_min": -13.439196586608887,
      "activations/layer10_attention_weight_max": 34.35868835449219,
      "activations/layer10_attention_weight_min": -35.18598175048828,
      "activations/layer11_attention_weight_max": 31.92000389099121,
      "activations/layer11_attention_weight_min": -33.82413864135742,
      "activations/layer12_attention_weight_max": 18.93373680114746,
      "activations/layer12_attention_weight_min": -28.73624038696289,
      "activations/layer13_attention_weight_max": 32.049678802490234,
      "activations/layer13_attention_weight_min": -26.13916778564453,
      "activations/layer14_attention_weight_max": 38.61085891723633,
      "activations/layer14_attention_weight_min": -30.253084182739258,
      "activations/layer15_attention_weight_max": 35.43928527832031,
      "activations/layer15_attention_weight_min": -30.08620834350586,
      "activations/layer16_attention_weight_max": 35.012123107910156,
      "activations/layer16_attention_weight_min": -32.23835754394531,
      "activations/layer17_attention_weight_max": 51.350830078125,
      "activations/layer17_attention_weight_min": -44.13959884643555,
      "activations/layer18_attention_weight_max": 50.91387939453125,
      "activations/layer18_attention_weight_min": -40.9622802734375,
      "activations/layer19_attention_weight_max": 28.432941436767578,
      "activations/layer19_attention_weight_min": -25.52669334411621,
      "activations/layer1_attention_weight_max": 15.94943904876709,
      "activations/layer1_attention_weight_min": -13.174111366271973,
      "activations/layer20_attention_weight_max": 27.867807388305664,
      "activations/layer20_attention_weight_min": -24.061662673950195,
      "activations/layer21_attention_weight_max": 46.128421783447266,
      "activations/layer21_attention_weight_min": -25.077024459838867,
      "activations/layer22_attention_weight_max": 33.132049560546875,
      "activations/layer22_attention_weight_min": -25.4649658203125,
      "activations/layer23_attention_weight_max": 39.80228042602539,
      "activations/layer23_attention_weight_min": -24.584623336791992,
      "activations/layer2_attention_weight_max": 28.98177719116211,
      "activations/layer2_attention_weight_min": -27.349090576171875,
      "activations/layer3_attention_weight_max": 83.34730529785156,
      "activations/layer3_attention_weight_min": -83.36006927490234,
      "activations/layer4_attention_weight_max": 81.02764129638672,
      "activations/layer4_attention_weight_min": -77.7545394897461,
      "activations/layer5_attention_weight_max": 63.01913070678711,
      "activations/layer5_attention_weight_min": -69.44231414794922,
      "activations/layer6_attention_weight_max": 48.04951858520508,
      "activations/layer6_attention_weight_min": -48.19558334350586,
      "activations/layer7_attention_weight_max": 61.292354583740234,
      "activations/layer7_attention_weight_min": -63.725608825683594,
      "activations/layer8_attention_weight_max": 45.43644332885742,
      "activations/layer8_attention_weight_min": -47.58845520019531,
      "activations/layer9_attention_weight_max": 44.10969161987305,
      "activations/layer9_attention_weight_min": -43.03377151489258,
      "epoch": 18.9,
      "learning_rate": 2.843333333333333e-05,
      "loss": 2.742,
      "step": 325200
    },
    {
      "activations/layer0_attention_weight_max": 14.792160034179688,
      "activations/layer0_attention_weight_min": -13.657073974609375,
      "activations/layer10_attention_weight_max": 42.164554595947266,
      "activations/layer10_attention_weight_min": -42.2431526184082,
      "activations/layer11_attention_weight_max": 39.914642333984375,
      "activations/layer11_attention_weight_min": -38.405006408691406,
      "activations/layer12_attention_weight_max": 20.51399040222168,
      "activations/layer12_attention_weight_min": -27.28998565673828,
      "activations/layer13_attention_weight_max": 43.74922180175781,
      "activations/layer13_attention_weight_min": -31.34640121459961,
      "activations/layer14_attention_weight_max": 40.824161529541016,
      "activations/layer14_attention_weight_min": -31.650495529174805,
      "activations/layer15_attention_weight_max": 44.79806900024414,
      "activations/layer15_attention_weight_min": -29.96735954284668,
      "activations/layer16_attention_weight_max": 35.69007110595703,
      "activations/layer16_attention_weight_min": -32.99355697631836,
      "activations/layer17_attention_weight_max": 55.51755142211914,
      "activations/layer17_attention_weight_min": -48.00532913208008,
      "activations/layer18_attention_weight_max": 49.53740310668945,
      "activations/layer18_attention_weight_min": -46.477725982666016,
      "activations/layer19_attention_weight_max": 24.373380661010742,
      "activations/layer19_attention_weight_min": -24.62168312072754,
      "activations/layer1_attention_weight_max": 15.740059852600098,
      "activations/layer1_attention_weight_min": -14.036721229553223,
      "activations/layer20_attention_weight_max": 22.30961799621582,
      "activations/layer20_attention_weight_min": -25.088966369628906,
      "activations/layer21_attention_weight_max": 31.49509048461914,
      "activations/layer21_attention_weight_min": -22.526716232299805,
      "activations/layer22_attention_weight_max": 28.23353385925293,
      "activations/layer22_attention_weight_min": -25.96390724182129,
      "activations/layer23_attention_weight_max": 36.99090576171875,
      "activations/layer23_attention_weight_min": -23.244131088256836,
      "activations/layer2_attention_weight_max": 31.765104293823242,
      "activations/layer2_attention_weight_min": -29.48885154724121,
      "activations/layer3_attention_weight_max": 84.0047607421875,
      "activations/layer3_attention_weight_min": -81.56768798828125,
      "activations/layer4_attention_weight_max": 85.52922058105469,
      "activations/layer4_attention_weight_min": -84.19388580322266,
      "activations/layer5_attention_weight_max": 61.78668975830078,
      "activations/layer5_attention_weight_min": -72.86227416992188,
      "activations/layer6_attention_weight_max": 53.41558074951172,
      "activations/layer6_attention_weight_min": -53.43775939941406,
      "activations/layer7_attention_weight_max": 73.06415557861328,
      "activations/layer7_attention_weight_min": -66.10890197753906,
      "activations/layer8_attention_weight_max": 52.15968704223633,
      "activations/layer8_attention_weight_min": -55.92365646362305,
      "activations/layer9_attention_weight_max": 57.84521484375,
      "activations/layer9_attention_weight_min": -53.29661178588867,
      "epoch": 18.9,
      "learning_rate": 2.8414393939393936e-05,
      "loss": 2.7342,
      "step": 325250
    },
    {
      "activations/layer0_attention_weight_max": 15.7476806640625,
      "activations/layer0_attention_weight_min": -13.659626007080078,
      "activations/layer10_attention_weight_max": 36.282562255859375,
      "activations/layer10_attention_weight_min": -35.68712615966797,
      "activations/layer11_attention_weight_max": 32.568763732910156,
      "activations/layer11_attention_weight_min": -33.99725341796875,
      "activations/layer12_attention_weight_max": 19.223722457885742,
      "activations/layer12_attention_weight_min": -26.787424087524414,
      "activations/layer13_attention_weight_max": 32.742271423339844,
      "activations/layer13_attention_weight_min": -28.240970611572266,
      "activations/layer14_attention_weight_max": 34.559059143066406,
      "activations/layer14_attention_weight_min": -31.520793914794922,
      "activations/layer15_attention_weight_max": 32.3652229309082,
      "activations/layer15_attention_weight_min": -31.533334732055664,
      "activations/layer16_attention_weight_max": 35.39778518676758,
      "activations/layer16_attention_weight_min": -33.72099304199219,
      "activations/layer17_attention_weight_max": 52.200416564941406,
      "activations/layer17_attention_weight_min": -45.860191345214844,
      "activations/layer18_attention_weight_max": 49.25615692138672,
      "activations/layer18_attention_weight_min": -38.8856086730957,
      "activations/layer19_attention_weight_max": 26.518054962158203,
      "activations/layer19_attention_weight_min": -23.3533935546875,
      "activations/layer1_attention_weight_max": 16.473106384277344,
      "activations/layer1_attention_weight_min": -14.174114227294922,
      "activations/layer20_attention_weight_max": 24.175739288330078,
      "activations/layer20_attention_weight_min": -21.575777053833008,
      "activations/layer21_attention_weight_max": 40.39551544189453,
      "activations/layer21_attention_weight_min": -20.682403564453125,
      "activations/layer22_attention_weight_max": 31.924898147583008,
      "activations/layer22_attention_weight_min": -25.40918731689453,
      "activations/layer23_attention_weight_max": 45.26688766479492,
      "activations/layer23_attention_weight_min": -23.499393463134766,
      "activations/layer2_attention_weight_max": 31.325254440307617,
      "activations/layer2_attention_weight_min": -29.363353729248047,
      "activations/layer3_attention_weight_max": 82.50843048095703,
      "activations/layer3_attention_weight_min": -81.91193389892578,
      "activations/layer4_attention_weight_max": 79.98380279541016,
      "activations/layer4_attention_weight_min": -76.24070739746094,
      "activations/layer5_attention_weight_max": 65.01539611816406,
      "activations/layer5_attention_weight_min": -70.59550476074219,
      "activations/layer6_attention_weight_max": 49.10260772705078,
      "activations/layer6_attention_weight_min": -51.54367446899414,
      "activations/layer7_attention_weight_max": 66.61459350585938,
      "activations/layer7_attention_weight_min": -61.7643928527832,
      "activations/layer8_attention_weight_max": 47.7685432434082,
      "activations/layer8_attention_weight_min": -51.54357147216797,
      "activations/layer9_attention_weight_max": 44.501949310302734,
      "activations/layer9_attention_weight_min": -46.45819854736328,
      "epoch": 18.9,
      "learning_rate": 2.8395454545454544e-05,
      "loss": 2.7498,
      "step": 325300
    },
    {
      "activations/layer0_attention_weight_max": 16.211742401123047,
      "activations/layer0_attention_weight_min": -13.98544692993164,
      "activations/layer10_attention_weight_max": 36.2960319519043,
      "activations/layer10_attention_weight_min": -36.58913040161133,
      "activations/layer11_attention_weight_max": 34.546836853027344,
      "activations/layer11_attention_weight_min": -35.09493637084961,
      "activations/layer12_attention_weight_max": 18.714555740356445,
      "activations/layer12_attention_weight_min": -24.20159339904785,
      "activations/layer13_attention_weight_max": 36.34965133666992,
      "activations/layer13_attention_weight_min": -26.089441299438477,
      "activations/layer14_attention_weight_max": 35.3622932434082,
      "activations/layer14_attention_weight_min": -29.38074493408203,
      "activations/layer15_attention_weight_max": 32.28176498413086,
      "activations/layer15_attention_weight_min": -30.196979522705078,
      "activations/layer16_attention_weight_max": 34.88700485229492,
      "activations/layer16_attention_weight_min": -32.9835319519043,
      "activations/layer17_attention_weight_max": 53.92585754394531,
      "activations/layer17_attention_weight_min": -46.284324645996094,
      "activations/layer18_attention_weight_max": 49.32414627075195,
      "activations/layer18_attention_weight_min": -41.10619354248047,
      "activations/layer19_attention_weight_max": 26.577001571655273,
      "activations/layer19_attention_weight_min": -22.53013801574707,
      "activations/layer1_attention_weight_max": 15.56965160369873,
      "activations/layer1_attention_weight_min": -14.08034610748291,
      "activations/layer20_attention_weight_max": 23.66069221496582,
      "activations/layer20_attention_weight_min": -22.869163513183594,
      "activations/layer21_attention_weight_max": 33.66358947753906,
      "activations/layer21_attention_weight_min": -22.916006088256836,
      "activations/layer22_attention_weight_max": 30.781240463256836,
      "activations/layer22_attention_weight_min": -25.653114318847656,
      "activations/layer23_attention_weight_max": 40.38788604736328,
      "activations/layer23_attention_weight_min": -28.37360382080078,
      "activations/layer2_attention_weight_max": 31.953861236572266,
      "activations/layer2_attention_weight_min": -30.63906478881836,
      "activations/layer3_attention_weight_max": 86.58809661865234,
      "activations/layer3_attention_weight_min": -88.0361328125,
      "activations/layer4_attention_weight_max": 82.2628173828125,
      "activations/layer4_attention_weight_min": -82.44135284423828,
      "activations/layer5_attention_weight_max": 63.476280212402344,
      "activations/layer5_attention_weight_min": -75.44929504394531,
      "activations/layer6_attention_weight_max": 50.999568939208984,
      "activations/layer6_attention_weight_min": -50.107826232910156,
      "activations/layer7_attention_weight_max": 63.524356842041016,
      "activations/layer7_attention_weight_min": -64.61336517333984,
      "activations/layer8_attention_weight_max": 47.846797943115234,
      "activations/layer8_attention_weight_min": -49.84868240356445,
      "activations/layer9_attention_weight_max": 41.12495803833008,
      "activations/layer9_attention_weight_min": -44.74528121948242,
      "epoch": 18.9,
      "learning_rate": 2.8376515151515146e-05,
      "loss": 2.7283,
      "step": 325350
    },
    {
      "activations/layer0_attention_weight_max": 14.784917831420898,
      "activations/layer0_attention_weight_min": -13.833159446716309,
      "activations/layer10_attention_weight_max": 36.713958740234375,
      "activations/layer10_attention_weight_min": -37.32746505737305,
      "activations/layer11_attention_weight_max": 34.104732513427734,
      "activations/layer11_attention_weight_min": -36.578758239746094,
      "activations/layer12_attention_weight_max": 19.576946258544922,
      "activations/layer12_attention_weight_min": -23.2520751953125,
      "activations/layer13_attention_weight_max": 34.77879333496094,
      "activations/layer13_attention_weight_min": -27.726518630981445,
      "activations/layer14_attention_weight_max": 37.59238815307617,
      "activations/layer14_attention_weight_min": -31.645158767700195,
      "activations/layer15_attention_weight_max": 36.339237213134766,
      "activations/layer15_attention_weight_min": -31.828521728515625,
      "activations/layer16_attention_weight_max": 37.1976432800293,
      "activations/layer16_attention_weight_min": -35.12397003173828,
      "activations/layer17_attention_weight_max": 59.3876953125,
      "activations/layer17_attention_weight_min": -49.334716796875,
      "activations/layer18_attention_weight_max": 51.5540657043457,
      "activations/layer18_attention_weight_min": -41.769126892089844,
      "activations/layer19_attention_weight_max": 24.86079216003418,
      "activations/layer19_attention_weight_min": -23.129573822021484,
      "activations/layer1_attention_weight_max": 15.753291130065918,
      "activations/layer1_attention_weight_min": -13.909738540649414,
      "activations/layer20_attention_weight_max": 24.002025604248047,
      "activations/layer20_attention_weight_min": -21.1102237701416,
      "activations/layer21_attention_weight_max": 33.92414093017578,
      "activations/layer21_attention_weight_min": -21.900054931640625,
      "activations/layer22_attention_weight_max": 30.009132385253906,
      "activations/layer22_attention_weight_min": -26.062728881835938,
      "activations/layer23_attention_weight_max": 36.05730438232422,
      "activations/layer23_attention_weight_min": -23.183387756347656,
      "activations/layer2_attention_weight_max": 30.623943328857422,
      "activations/layer2_attention_weight_min": -29.821456909179688,
      "activations/layer3_attention_weight_max": 86.30023193359375,
      "activations/layer3_attention_weight_min": -84.67202758789062,
      "activations/layer4_attention_weight_max": 83.9088363647461,
      "activations/layer4_attention_weight_min": -82.28195190429688,
      "activations/layer5_attention_weight_max": 61.077293395996094,
      "activations/layer5_attention_weight_min": -73.89900970458984,
      "activations/layer6_attention_weight_max": 50.896175384521484,
      "activations/layer6_attention_weight_min": -50.38735580444336,
      "activations/layer7_attention_weight_max": 65.28813171386719,
      "activations/layer7_attention_weight_min": -65.18389892578125,
      "activations/layer8_attention_weight_max": 45.78595733642578,
      "activations/layer8_attention_weight_min": -50.83943557739258,
      "activations/layer9_attention_weight_max": 44.81989669799805,
      "activations/layer9_attention_weight_min": -48.45378112792969,
      "epoch": 18.91,
      "learning_rate": 2.8357575757575754e-05,
      "loss": 2.7293,
      "step": 325400
    },
    {
      "activations/layer0_attention_weight_max": 16.168956756591797,
      "activations/layer0_attention_weight_min": -13.759307861328125,
      "activations/layer10_attention_weight_max": 33.828182220458984,
      "activations/layer10_attention_weight_min": -32.77154541015625,
      "activations/layer11_attention_weight_max": 29.292949676513672,
      "activations/layer11_attention_weight_min": -31.428464889526367,
      "activations/layer12_attention_weight_max": 19.49983024597168,
      "activations/layer12_attention_weight_min": -30.626022338867188,
      "activations/layer13_attention_weight_max": 33.92853546142578,
      "activations/layer13_attention_weight_min": -27.256162643432617,
      "activations/layer14_attention_weight_max": 35.67458724975586,
      "activations/layer14_attention_weight_min": -31.382381439208984,
      "activations/layer15_attention_weight_max": 31.755558013916016,
      "activations/layer15_attention_weight_min": -30.768253326416016,
      "activations/layer16_attention_weight_max": 36.61073684692383,
      "activations/layer16_attention_weight_min": -34.51467514038086,
      "activations/layer17_attention_weight_max": 55.55332565307617,
      "activations/layer17_attention_weight_min": -51.80593490600586,
      "activations/layer18_attention_weight_max": 47.29418182373047,
      "activations/layer18_attention_weight_min": -45.575836181640625,
      "activations/layer19_attention_weight_max": 23.503341674804688,
      "activations/layer19_attention_weight_min": -25.42265510559082,
      "activations/layer1_attention_weight_max": 16.099727630615234,
      "activations/layer1_attention_weight_min": -13.584593772888184,
      "activations/layer20_attention_weight_max": 23.554248809814453,
      "activations/layer20_attention_weight_min": -24.269573211669922,
      "activations/layer21_attention_weight_max": 41.954254150390625,
      "activations/layer21_attention_weight_min": -24.53012466430664,
      "activations/layer22_attention_weight_max": 29.666032791137695,
      "activations/layer22_attention_weight_min": -27.28817367553711,
      "activations/layer23_attention_weight_max": 36.729469299316406,
      "activations/layer23_attention_weight_min": -24.16863250732422,
      "activations/layer2_attention_weight_max": 30.198394775390625,
      "activations/layer2_attention_weight_min": -29.621440887451172,
      "activations/layer3_attention_weight_max": 77.61825561523438,
      "activations/layer3_attention_weight_min": -75.26497650146484,
      "activations/layer4_attention_weight_max": 74.8486099243164,
      "activations/layer4_attention_weight_min": -75.95124816894531,
      "activations/layer5_attention_weight_max": 55.879371643066406,
      "activations/layer5_attention_weight_min": -67.93358612060547,
      "activations/layer6_attention_weight_max": 47.158138275146484,
      "activations/layer6_attention_weight_min": -46.77260208129883,
      "activations/layer7_attention_weight_max": 64.23853302001953,
      "activations/layer7_attention_weight_min": -59.96177673339844,
      "activations/layer8_attention_weight_max": 45.15188980102539,
      "activations/layer8_attention_weight_min": -47.82829666137695,
      "activations/layer9_attention_weight_max": 42.67022705078125,
      "activations/layer9_attention_weight_min": -43.12159729003906,
      "epoch": 18.91,
      "learning_rate": 2.833863636363636e-05,
      "loss": 2.7452,
      "step": 325450
    },
    {
      "activations/layer0_attention_weight_max": 15.253912925720215,
      "activations/layer0_attention_weight_min": -13.888340950012207,
      "activations/layer10_attention_weight_max": 35.944725036621094,
      "activations/layer10_attention_weight_min": -36.55117416381836,
      "activations/layer11_attention_weight_max": 32.58275604248047,
      "activations/layer11_attention_weight_min": -34.22210693359375,
      "activations/layer12_attention_weight_max": 21.020767211914062,
      "activations/layer12_attention_weight_min": -28.150089263916016,
      "activations/layer13_attention_weight_max": 33.298095703125,
      "activations/layer13_attention_weight_min": -25.453189849853516,
      "activations/layer14_attention_weight_max": 36.771080017089844,
      "activations/layer14_attention_weight_min": -29.37107276916504,
      "activations/layer15_attention_weight_max": 36.9948616027832,
      "activations/layer15_attention_weight_min": -31.522245407104492,
      "activations/layer16_attention_weight_max": 37.53118133544922,
      "activations/layer16_attention_weight_min": -33.84598922729492,
      "activations/layer17_attention_weight_max": 53.944881439208984,
      "activations/layer17_attention_weight_min": -47.48957443237305,
      "activations/layer18_attention_weight_max": 52.55448913574219,
      "activations/layer18_attention_weight_min": -43.10966491699219,
      "activations/layer19_attention_weight_max": 25.63089942932129,
      "activations/layer19_attention_weight_min": -24.533567428588867,
      "activations/layer1_attention_weight_max": 15.520106315612793,
      "activations/layer1_attention_weight_min": -13.8201265335083,
      "activations/layer20_attention_weight_max": 23.712505340576172,
      "activations/layer20_attention_weight_min": -21.079753875732422,
      "activations/layer21_attention_weight_max": 37.3485221862793,
      "activations/layer21_attention_weight_min": -23.468029022216797,
      "activations/layer22_attention_weight_max": 29.549057006835938,
      "activations/layer22_attention_weight_min": -26.45033073425293,
      "activations/layer23_attention_weight_max": 39.2630615234375,
      "activations/layer23_attention_weight_min": -26.55052947998047,
      "activations/layer2_attention_weight_max": 34.72789764404297,
      "activations/layer2_attention_weight_min": -30.695159912109375,
      "activations/layer3_attention_weight_max": 81.57360076904297,
      "activations/layer3_attention_weight_min": -80.22784423828125,
      "activations/layer4_attention_weight_max": 80.20015716552734,
      "activations/layer4_attention_weight_min": -79.56871795654297,
      "activations/layer5_attention_weight_max": 61.578819274902344,
      "activations/layer5_attention_weight_min": -70.51844024658203,
      "activations/layer6_attention_weight_max": 47.45357894897461,
      "activations/layer6_attention_weight_min": -48.16774368286133,
      "activations/layer7_attention_weight_max": 68.80636596679688,
      "activations/layer7_attention_weight_min": -64.47322082519531,
      "activations/layer8_attention_weight_max": 46.21504592895508,
      "activations/layer8_attention_weight_min": -49.52860641479492,
      "activations/layer9_attention_weight_max": 43.811546325683594,
      "activations/layer9_attention_weight_min": -46.03519058227539,
      "epoch": 18.91,
      "learning_rate": 2.8319696969696967e-05,
      "loss": 2.7149,
      "step": 325500
    },
    {
      "activations/layer0_attention_weight_max": 15.449211120605469,
      "activations/layer0_attention_weight_min": -13.438770294189453,
      "activations/layer10_attention_weight_max": 37.91574478149414,
      "activations/layer10_attention_weight_min": -35.87226867675781,
      "activations/layer11_attention_weight_max": 36.349605560302734,
      "activations/layer11_attention_weight_min": -32.727394104003906,
      "activations/layer12_attention_weight_max": 18.249692916870117,
      "activations/layer12_attention_weight_min": -34.68573760986328,
      "activations/layer13_attention_weight_max": 33.558555603027344,
      "activations/layer13_attention_weight_min": -26.386730194091797,
      "activations/layer14_attention_weight_max": 36.29981231689453,
      "activations/layer14_attention_weight_min": -31.15436553955078,
      "activations/layer15_attention_weight_max": 33.98717498779297,
      "activations/layer15_attention_weight_min": -29.30343246459961,
      "activations/layer16_attention_weight_max": 35.44042205810547,
      "activations/layer16_attention_weight_min": -31.780790328979492,
      "activations/layer17_attention_weight_max": 49.9867057800293,
      "activations/layer17_attention_weight_min": -44.203060150146484,
      "activations/layer18_attention_weight_max": 46.111454010009766,
      "activations/layer18_attention_weight_min": -38.822181701660156,
      "activations/layer19_attention_weight_max": 22.674640655517578,
      "activations/layer19_attention_weight_min": -21.148534774780273,
      "activations/layer1_attention_weight_max": 15.815342903137207,
      "activations/layer1_attention_weight_min": -13.681676864624023,
      "activations/layer20_attention_weight_max": 20.394195556640625,
      "activations/layer20_attention_weight_min": -24.551145553588867,
      "activations/layer21_attention_weight_max": 32.833343505859375,
      "activations/layer21_attention_weight_min": -22.09373664855957,
      "activations/layer22_attention_weight_max": 26.978290557861328,
      "activations/layer22_attention_weight_min": -25.8460636138916,
      "activations/layer23_attention_weight_max": 32.26279067993164,
      "activations/layer23_attention_weight_min": -24.096555709838867,
      "activations/layer2_attention_weight_max": 32.36695098876953,
      "activations/layer2_attention_weight_min": -29.56855010986328,
      "activations/layer3_attention_weight_max": 80.703369140625,
      "activations/layer3_attention_weight_min": -79.58164978027344,
      "activations/layer4_attention_weight_max": 79.87488555908203,
      "activations/layer4_attention_weight_min": -78.71539306640625,
      "activations/layer5_attention_weight_max": 61.04533004760742,
      "activations/layer5_attention_weight_min": -74.0116195678711,
      "activations/layer6_attention_weight_max": 47.44103240966797,
      "activations/layer6_attention_weight_min": -48.863529205322266,
      "activations/layer7_attention_weight_max": 70.9327163696289,
      "activations/layer7_attention_weight_min": -67.95626831054688,
      "activations/layer8_attention_weight_max": 47.61394500732422,
      "activations/layer8_attention_weight_min": -45.9966926574707,
      "activations/layer9_attention_weight_max": 52.66072463989258,
      "activations/layer9_attention_weight_min": -49.31550979614258,
      "epoch": 18.92,
      "learning_rate": 2.8300757575757575e-05,
      "loss": 2.7354,
      "step": 325550
    },
    {
      "activations/layer0_attention_weight_max": 15.097992897033691,
      "activations/layer0_attention_weight_min": -13.564313888549805,
      "activations/layer10_attention_weight_max": 35.28571701049805,
      "activations/layer10_attention_weight_min": -36.86941146850586,
      "activations/layer11_attention_weight_max": 32.17986297607422,
      "activations/layer11_attention_weight_min": -34.80354690551758,
      "activations/layer12_attention_weight_max": 19.10079002380371,
      "activations/layer12_attention_weight_min": -27.298765182495117,
      "activations/layer13_attention_weight_max": 34.736820220947266,
      "activations/layer13_attention_weight_min": -28.183076858520508,
      "activations/layer14_attention_weight_max": 37.42237091064453,
      "activations/layer14_attention_weight_min": -30.477571487426758,
      "activations/layer15_attention_weight_max": 42.537200927734375,
      "activations/layer15_attention_weight_min": -30.170656204223633,
      "activations/layer16_attention_weight_max": 38.66055679321289,
      "activations/layer16_attention_weight_min": -34.604209899902344,
      "activations/layer17_attention_weight_max": 55.60594940185547,
      "activations/layer17_attention_weight_min": -47.09410095214844,
      "activations/layer18_attention_weight_max": 49.19439697265625,
      "activations/layer18_attention_weight_min": -42.80503845214844,
      "activations/layer19_attention_weight_max": 24.759153366088867,
      "activations/layer19_attention_weight_min": -24.917356491088867,
      "activations/layer1_attention_weight_max": 17.04730796813965,
      "activations/layer1_attention_weight_min": -15.372222900390625,
      "activations/layer20_attention_weight_max": 24.79962158203125,
      "activations/layer20_attention_weight_min": -22.999065399169922,
      "activations/layer21_attention_weight_max": 42.203697204589844,
      "activations/layer21_attention_weight_min": -21.313756942749023,
      "activations/layer22_attention_weight_max": 30.95126724243164,
      "activations/layer22_attention_weight_min": -26.71685791015625,
      "activations/layer23_attention_weight_max": 38.10679626464844,
      "activations/layer23_attention_weight_min": -25.095077514648438,
      "activations/layer2_attention_weight_max": 33.43080139160156,
      "activations/layer2_attention_weight_min": -31.539897918701172,
      "activations/layer3_attention_weight_max": 88.22631072998047,
      "activations/layer3_attention_weight_min": -87.12226867675781,
      "activations/layer4_attention_weight_max": 86.35330200195312,
      "activations/layer4_attention_weight_min": -85.61680603027344,
      "activations/layer5_attention_weight_max": 64.74748229980469,
      "activations/layer5_attention_weight_min": -72.67857360839844,
      "activations/layer6_attention_weight_max": 56.25294876098633,
      "activations/layer6_attention_weight_min": -54.10475158691406,
      "activations/layer7_attention_weight_max": 70.50727081298828,
      "activations/layer7_attention_weight_min": -66.38335418701172,
      "activations/layer8_attention_weight_max": 53.556339263916016,
      "activations/layer8_attention_weight_min": -51.644866943359375,
      "activations/layer9_attention_weight_max": 46.77694320678711,
      "activations/layer9_attention_weight_min": -49.07406234741211,
      "epoch": 18.92,
      "learning_rate": 2.828181818181818e-05,
      "loss": 2.7345,
      "step": 325600
    },
    {
      "activations/layer0_attention_weight_max": 15.103910446166992,
      "activations/layer0_attention_weight_min": -13.7886381149292,
      "activations/layer10_attention_weight_max": 38.80167007446289,
      "activations/layer10_attention_weight_min": -36.52348327636719,
      "activations/layer11_attention_weight_max": 34.30371856689453,
      "activations/layer11_attention_weight_min": -35.56768798828125,
      "activations/layer12_attention_weight_max": 19.508182525634766,
      "activations/layer12_attention_weight_min": -25.515308380126953,
      "activations/layer13_attention_weight_max": 42.08291244506836,
      "activations/layer13_attention_weight_min": -31.2905216217041,
      "activations/layer14_attention_weight_max": 40.660404205322266,
      "activations/layer14_attention_weight_min": -33.87340545654297,
      "activations/layer15_attention_weight_max": 36.917015075683594,
      "activations/layer15_attention_weight_min": -32.29498291015625,
      "activations/layer16_attention_weight_max": 37.28042984008789,
      "activations/layer16_attention_weight_min": -32.93330764770508,
      "activations/layer17_attention_weight_max": 57.54763412475586,
      "activations/layer17_attention_weight_min": -49.13336181640625,
      "activations/layer18_attention_weight_max": 50.73516082763672,
      "activations/layer18_attention_weight_min": -41.1777458190918,
      "activations/layer19_attention_weight_max": 27.651615142822266,
      "activations/layer19_attention_weight_min": -24.307815551757812,
      "activations/layer1_attention_weight_max": 15.816908836364746,
      "activations/layer1_attention_weight_min": -14.203230857849121,
      "activations/layer20_attention_weight_max": 27.686416625976562,
      "activations/layer20_attention_weight_min": -22.470487594604492,
      "activations/layer21_attention_weight_max": 47.02467727661133,
      "activations/layer21_attention_weight_min": -24.446624755859375,
      "activations/layer22_attention_weight_max": 30.047651290893555,
      "activations/layer22_attention_weight_min": -25.89974594116211,
      "activations/layer23_attention_weight_max": 40.43010711669922,
      "activations/layer23_attention_weight_min": -25.85220718383789,
      "activations/layer2_attention_weight_max": 32.25261306762695,
      "activations/layer2_attention_weight_min": -29.99114990234375,
      "activations/layer3_attention_weight_max": 81.87468719482422,
      "activations/layer3_attention_weight_min": -80.12698364257812,
      "activations/layer4_attention_weight_max": 78.67172241210938,
      "activations/layer4_attention_weight_min": -80.62916564941406,
      "activations/layer5_attention_weight_max": 59.629547119140625,
      "activations/layer5_attention_weight_min": -68.9495849609375,
      "activations/layer6_attention_weight_max": 48.58614730834961,
      "activations/layer6_attention_weight_min": -47.251468658447266,
      "activations/layer7_attention_weight_max": 65.02925109863281,
      "activations/layer7_attention_weight_min": -65.7725601196289,
      "activations/layer8_attention_weight_max": 47.26948928833008,
      "activations/layer8_attention_weight_min": -51.3713264465332,
      "activations/layer9_attention_weight_max": 45.57929611206055,
      "activations/layer9_attention_weight_min": -49.17135238647461,
      "epoch": 18.92,
      "learning_rate": 2.826287878787879e-05,
      "loss": 2.7075,
      "step": 325650
    },
    {
      "activations/layer0_attention_weight_max": 15.425196647644043,
      "activations/layer0_attention_weight_min": -13.520273208618164,
      "activations/layer10_attention_weight_max": 37.88362121582031,
      "activations/layer10_attention_weight_min": -37.45997619628906,
      "activations/layer11_attention_weight_max": 37.37030792236328,
      "activations/layer11_attention_weight_min": -36.902957916259766,
      "activations/layer12_attention_weight_max": 18.530969619750977,
      "activations/layer12_attention_weight_min": -28.237009048461914,
      "activations/layer13_attention_weight_max": 36.49980545043945,
      "activations/layer13_attention_weight_min": -30.896501541137695,
      "activations/layer14_attention_weight_max": 33.705440521240234,
      "activations/layer14_attention_weight_min": -30.557254791259766,
      "activations/layer15_attention_weight_max": 33.57582092285156,
      "activations/layer15_attention_weight_min": -31.08757781982422,
      "activations/layer16_attention_weight_max": 33.275177001953125,
      "activations/layer16_attention_weight_min": -31.702388763427734,
      "activations/layer17_attention_weight_max": 48.24284744262695,
      "activations/layer17_attention_weight_min": -46.28180694580078,
      "activations/layer18_attention_weight_max": 46.72748947143555,
      "activations/layer18_attention_weight_min": -41.24982833862305,
      "activations/layer19_attention_weight_max": 21.515478134155273,
      "activations/layer19_attention_weight_min": -23.84894371032715,
      "activations/layer1_attention_weight_max": 15.678467750549316,
      "activations/layer1_attention_weight_min": -13.15567398071289,
      "activations/layer20_attention_weight_max": 22.61270523071289,
      "activations/layer20_attention_weight_min": -20.55207633972168,
      "activations/layer21_attention_weight_max": 37.803794860839844,
      "activations/layer21_attention_weight_min": -24.63134765625,
      "activations/layer22_attention_weight_max": 26.991046905517578,
      "activations/layer22_attention_weight_min": -23.841672897338867,
      "activations/layer23_attention_weight_max": 36.65038299560547,
      "activations/layer23_attention_weight_min": -22.059886932373047,
      "activations/layer2_attention_weight_max": 33.49909210205078,
      "activations/layer2_attention_weight_min": -31.674467086791992,
      "activations/layer3_attention_weight_max": 86.81729888916016,
      "activations/layer3_attention_weight_min": -85.10464477539062,
      "activations/layer4_attention_weight_max": 80.51262664794922,
      "activations/layer4_attention_weight_min": -81.59185791015625,
      "activations/layer5_attention_weight_max": 63.93382263183594,
      "activations/layer5_attention_weight_min": -73.94854736328125,
      "activations/layer6_attention_weight_max": 50.6997184753418,
      "activations/layer6_attention_weight_min": -49.77851867675781,
      "activations/layer7_attention_weight_max": 67.24321746826172,
      "activations/layer7_attention_weight_min": -65.53980255126953,
      "activations/layer8_attention_weight_max": 47.316104888916016,
      "activations/layer8_attention_weight_min": -50.111412048339844,
      "activations/layer9_attention_weight_max": 53.18928146362305,
      "activations/layer9_attention_weight_min": -49.600860595703125,
      "epoch": 18.92,
      "learning_rate": 2.824393939393939e-05,
      "loss": 2.7125,
      "step": 325700
    },
    {
      "activations/layer0_attention_weight_max": 14.99034595489502,
      "activations/layer0_attention_weight_min": -13.931938171386719,
      "activations/layer10_attention_weight_max": 35.79170608520508,
      "activations/layer10_attention_weight_min": -35.344886779785156,
      "activations/layer11_attention_weight_max": 32.34850311279297,
      "activations/layer11_attention_weight_min": -33.28940200805664,
      "activations/layer12_attention_weight_max": 19.250812530517578,
      "activations/layer12_attention_weight_min": -25.222023010253906,
      "activations/layer13_attention_weight_max": 32.780555725097656,
      "activations/layer13_attention_weight_min": -27.263309478759766,
      "activations/layer14_attention_weight_max": 35.56291198730469,
      "activations/layer14_attention_weight_min": -31.45316505432129,
      "activations/layer15_attention_weight_max": 32.039005279541016,
      "activations/layer15_attention_weight_min": -32.65011978149414,
      "activations/layer16_attention_weight_max": 32.923587799072266,
      "activations/layer16_attention_weight_min": -33.87692642211914,
      "activations/layer17_attention_weight_max": 52.30982208251953,
      "activations/layer17_attention_weight_min": -47.69340515136719,
      "activations/layer18_attention_weight_max": 45.42589569091797,
      "activations/layer18_attention_weight_min": -47.67218780517578,
      "activations/layer19_attention_weight_max": 25.230825424194336,
      "activations/layer19_attention_weight_min": -28.818906784057617,
      "activations/layer1_attention_weight_max": 16.422380447387695,
      "activations/layer1_attention_weight_min": -14.32425308227539,
      "activations/layer20_attention_weight_max": 25.130496978759766,
      "activations/layer20_attention_weight_min": -24.98858070373535,
      "activations/layer21_attention_weight_max": 36.4708251953125,
      "activations/layer21_attention_weight_min": -24.168371200561523,
      "activations/layer22_attention_weight_max": 27.22611427307129,
      "activations/layer22_attention_weight_min": -27.40511131286621,
      "activations/layer23_attention_weight_max": 34.78600311279297,
      "activations/layer23_attention_weight_min": -26.4742431640625,
      "activations/layer2_attention_weight_max": 32.28146743774414,
      "activations/layer2_attention_weight_min": -29.244871139526367,
      "activations/layer3_attention_weight_max": 82.33987426757812,
      "activations/layer3_attention_weight_min": -81.11639404296875,
      "activations/layer4_attention_weight_max": 77.20501708984375,
      "activations/layer4_attention_weight_min": -77.93150329589844,
      "activations/layer5_attention_weight_max": 60.87559127807617,
      "activations/layer5_attention_weight_min": -70.09310913085938,
      "activations/layer6_attention_weight_max": 48.58425521850586,
      "activations/layer6_attention_weight_min": -48.846473693847656,
      "activations/layer7_attention_weight_max": 62.78383255004883,
      "activations/layer7_attention_weight_min": -59.444583892822266,
      "activations/layer8_attention_weight_max": 44.14202880859375,
      "activations/layer8_attention_weight_min": -46.755184173583984,
      "activations/layer9_attention_weight_max": 42.510128021240234,
      "activations/layer9_attention_weight_min": -44.3087272644043,
      "epoch": 18.93,
      "learning_rate": 2.8224999999999998e-05,
      "loss": 2.7457,
      "step": 325750
    },
    {
      "activations/layer0_attention_weight_max": 15.93757438659668,
      "activations/layer0_attention_weight_min": -13.74417495727539,
      "activations/layer10_attention_weight_max": 37.06428146362305,
      "activations/layer10_attention_weight_min": -37.40830993652344,
      "activations/layer11_attention_weight_max": 35.91840744018555,
      "activations/layer11_attention_weight_min": -36.3296012878418,
      "activations/layer12_attention_weight_max": 18.494754791259766,
      "activations/layer12_attention_weight_min": -22.62358283996582,
      "activations/layer13_attention_weight_max": 31.906312942504883,
      "activations/layer13_attention_weight_min": -26.99407196044922,
      "activations/layer14_attention_weight_max": 33.46121597290039,
      "activations/layer14_attention_weight_min": -29.944992065429688,
      "activations/layer15_attention_weight_max": 31.199542999267578,
      "activations/layer15_attention_weight_min": -31.646257400512695,
      "activations/layer16_attention_weight_max": 33.94303512573242,
      "activations/layer16_attention_weight_min": -31.01570701599121,
      "activations/layer17_attention_weight_max": 48.992252349853516,
      "activations/layer17_attention_weight_min": -43.10658645629883,
      "activations/layer18_attention_weight_max": 44.53194046020508,
      "activations/layer18_attention_weight_min": -36.91282653808594,
      "activations/layer19_attention_weight_max": 23.310989379882812,
      "activations/layer19_attention_weight_min": -21.45510482788086,
      "activations/layer1_attention_weight_max": 15.950078010559082,
      "activations/layer1_attention_weight_min": -14.210437774658203,
      "activations/layer20_attention_weight_max": 21.550559997558594,
      "activations/layer20_attention_weight_min": -22.945981979370117,
      "activations/layer21_attention_weight_max": 30.79788589477539,
      "activations/layer21_attention_weight_min": -19.973299026489258,
      "activations/layer22_attention_weight_max": 27.575735092163086,
      "activations/layer22_attention_weight_min": -25.326337814331055,
      "activations/layer23_attention_weight_max": 35.26819610595703,
      "activations/layer23_attention_weight_min": -21.209148406982422,
      "activations/layer2_attention_weight_max": 32.156551361083984,
      "activations/layer2_attention_weight_min": -31.032766342163086,
      "activations/layer3_attention_weight_max": 85.62892150878906,
      "activations/layer3_attention_weight_min": -83.77608489990234,
      "activations/layer4_attention_weight_max": 83.89701080322266,
      "activations/layer4_attention_weight_min": -81.30818939208984,
      "activations/layer5_attention_weight_max": 62.80134582519531,
      "activations/layer5_attention_weight_min": -73.28689575195312,
      "activations/layer6_attention_weight_max": 47.40159606933594,
      "activations/layer6_attention_weight_min": -49.609046936035156,
      "activations/layer7_attention_weight_max": 66.88519287109375,
      "activations/layer7_attention_weight_min": -63.453861236572266,
      "activations/layer8_attention_weight_max": 46.735984802246094,
      "activations/layer8_attention_weight_min": -50.1723518371582,
      "activations/layer9_attention_weight_max": 43.17177963256836,
      "activations/layer9_attention_weight_min": -45.453704833984375,
      "epoch": 18.93,
      "learning_rate": 2.8206060606060603e-05,
      "loss": 2.7311,
      "step": 325800
    },
    {
      "activations/layer0_attention_weight_max": 14.95776653289795,
      "activations/layer0_attention_weight_min": -13.433259963989258,
      "activations/layer10_attention_weight_max": 37.28236770629883,
      "activations/layer10_attention_weight_min": -37.7327766418457,
      "activations/layer11_attention_weight_max": 35.305580139160156,
      "activations/layer11_attention_weight_min": -37.81645965576172,
      "activations/layer12_attention_weight_max": 19.519195556640625,
      "activations/layer12_attention_weight_min": -25.60567855834961,
      "activations/layer13_attention_weight_max": 34.96296310424805,
      "activations/layer13_attention_weight_min": -27.092662811279297,
      "activations/layer14_attention_weight_max": 34.98017883300781,
      "activations/layer14_attention_weight_min": -29.086008071899414,
      "activations/layer15_attention_weight_max": 31.58714485168457,
      "activations/layer15_attention_weight_min": -29.364620208740234,
      "activations/layer16_attention_weight_max": 34.757015228271484,
      "activations/layer16_attention_weight_min": -31.12830352783203,
      "activations/layer17_attention_weight_max": 53.765621185302734,
      "activations/layer17_attention_weight_min": -44.48043441772461,
      "activations/layer18_attention_weight_max": 47.0931510925293,
      "activations/layer18_attention_weight_min": -40.016639709472656,
      "activations/layer19_attention_weight_max": 25.353761672973633,
      "activations/layer19_attention_weight_min": -22.301843643188477,
      "activations/layer1_attention_weight_max": 17.607471466064453,
      "activations/layer1_attention_weight_min": -13.788881301879883,
      "activations/layer20_attention_weight_max": 23.894607543945312,
      "activations/layer20_attention_weight_min": -20.487993240356445,
      "activations/layer21_attention_weight_max": 32.055747985839844,
      "activations/layer21_attention_weight_min": -20.826873779296875,
      "activations/layer22_attention_weight_max": 28.2692928314209,
      "activations/layer22_attention_weight_min": -25.80049705505371,
      "activations/layer23_attention_weight_max": 36.953712463378906,
      "activations/layer23_attention_weight_min": -24.24410629272461,
      "activations/layer2_attention_weight_max": 31.682422637939453,
      "activations/layer2_attention_weight_min": -30.561965942382812,
      "activations/layer3_attention_weight_max": 85.66751098632812,
      "activations/layer3_attention_weight_min": -87.07134246826172,
      "activations/layer4_attention_weight_max": 83.13401794433594,
      "activations/layer4_attention_weight_min": -83.50743103027344,
      "activations/layer5_attention_weight_max": 61.40681457519531,
      "activations/layer5_attention_weight_min": -67.91825103759766,
      "activations/layer6_attention_weight_max": 50.0820426940918,
      "activations/layer6_attention_weight_min": -52.136409759521484,
      "activations/layer7_attention_weight_max": 64.26881408691406,
      "activations/layer7_attention_weight_min": -62.43076705932617,
      "activations/layer8_attention_weight_max": 51.52578353881836,
      "activations/layer8_attention_weight_min": -51.26814651489258,
      "activations/layer9_attention_weight_max": 47.1339225769043,
      "activations/layer9_attention_weight_min": -47.02059555053711,
      "epoch": 18.93,
      "learning_rate": 2.818712121212121e-05,
      "loss": 2.7303,
      "step": 325850
    },
    {
      "activations/layer0_attention_weight_max": 16.607324600219727,
      "activations/layer0_attention_weight_min": -13.707886695861816,
      "activations/layer10_attention_weight_max": 35.32225036621094,
      "activations/layer10_attention_weight_min": -35.42085647583008,
      "activations/layer11_attention_weight_max": 32.70827102661133,
      "activations/layer11_attention_weight_min": -33.212860107421875,
      "activations/layer12_attention_weight_max": 21.120071411132812,
      "activations/layer12_attention_weight_min": -27.706687927246094,
      "activations/layer13_attention_weight_max": 41.08344650268555,
      "activations/layer13_attention_weight_min": -27.044523239135742,
      "activations/layer14_attention_weight_max": 34.37163543701172,
      "activations/layer14_attention_weight_min": -29.781436920166016,
      "activations/layer15_attention_weight_max": 36.78240203857422,
      "activations/layer15_attention_weight_min": -29.951398849487305,
      "activations/layer16_attention_weight_max": 34.2167854309082,
      "activations/layer16_attention_weight_min": -32.5335807800293,
      "activations/layer17_attention_weight_max": 52.95096969604492,
      "activations/layer17_attention_weight_min": -44.44765090942383,
      "activations/layer18_attention_weight_max": 50.22136688232422,
      "activations/layer18_attention_weight_min": -40.1171760559082,
      "activations/layer19_attention_weight_max": 26.021421432495117,
      "activations/layer19_attention_weight_min": -22.96613311767578,
      "activations/layer1_attention_weight_max": 16.6751651763916,
      "activations/layer1_attention_weight_min": -15.734821319580078,
      "activations/layer20_attention_weight_max": 25.001934051513672,
      "activations/layer20_attention_weight_min": -21.250553131103516,
      "activations/layer21_attention_weight_max": 39.977813720703125,
      "activations/layer21_attention_weight_min": -20.562725067138672,
      "activations/layer22_attention_weight_max": 29.79295539855957,
      "activations/layer22_attention_weight_min": -25.026046752929688,
      "activations/layer23_attention_weight_max": 39.759029388427734,
      "activations/layer23_attention_weight_min": -24.063575744628906,
      "activations/layer2_attention_weight_max": 31.702606201171875,
      "activations/layer2_attention_weight_min": -30.24950408935547,
      "activations/layer3_attention_weight_max": 87.05782318115234,
      "activations/layer3_attention_weight_min": -84.36786651611328,
      "activations/layer4_attention_weight_max": 84.2669906616211,
      "activations/layer4_attention_weight_min": -82.38689422607422,
      "activations/layer5_attention_weight_max": 63.408653259277344,
      "activations/layer5_attention_weight_min": -72.74620056152344,
      "activations/layer6_attention_weight_max": 49.421913146972656,
      "activations/layer6_attention_weight_min": -51.92786407470703,
      "activations/layer7_attention_weight_max": 65.42149353027344,
      "activations/layer7_attention_weight_min": -63.647281646728516,
      "activations/layer8_attention_weight_max": 47.925445556640625,
      "activations/layer8_attention_weight_min": -50.35416793823242,
      "activations/layer9_attention_weight_max": 44.567222595214844,
      "activations/layer9_attention_weight_min": -46.559288024902344,
      "epoch": 18.94,
      "learning_rate": 2.8168181818181816e-05,
      "loss": 2.7131,
      "step": 325900
    },
    {
      "activations/layer0_attention_weight_max": 16.150001525878906,
      "activations/layer0_attention_weight_min": -13.471380233764648,
      "activations/layer10_attention_weight_max": 34.76535415649414,
      "activations/layer10_attention_weight_min": -37.092140197753906,
      "activations/layer11_attention_weight_max": 32.31629943847656,
      "activations/layer11_attention_weight_min": -33.23590850830078,
      "activations/layer12_attention_weight_max": 19.23921775817871,
      "activations/layer12_attention_weight_min": -27.414539337158203,
      "activations/layer13_attention_weight_max": 36.58057403564453,
      "activations/layer13_attention_weight_min": -32.60798645019531,
      "activations/layer14_attention_weight_max": 34.704803466796875,
      "activations/layer14_attention_weight_min": -32.4991340637207,
      "activations/layer15_attention_weight_max": 35.90863800048828,
      "activations/layer15_attention_weight_min": -33.4952507019043,
      "activations/layer16_attention_weight_max": 35.45713424682617,
      "activations/layer16_attention_weight_min": -35.7342529296875,
      "activations/layer17_attention_weight_max": 53.906532287597656,
      "activations/layer17_attention_weight_min": -49.19432830810547,
      "activations/layer18_attention_weight_max": 49.298980712890625,
      "activations/layer18_attention_weight_min": -45.232940673828125,
      "activations/layer19_attention_weight_max": 26.068649291992188,
      "activations/layer19_attention_weight_min": -26.36746597290039,
      "activations/layer1_attention_weight_max": 16.1019287109375,
      "activations/layer1_attention_weight_min": -14.948999404907227,
      "activations/layer20_attention_weight_max": 25.050640106201172,
      "activations/layer20_attention_weight_min": -22.951480865478516,
      "activations/layer21_attention_weight_max": 39.383026123046875,
      "activations/layer21_attention_weight_min": -27.375993728637695,
      "activations/layer22_attention_weight_max": 29.631216049194336,
      "activations/layer22_attention_weight_min": -28.433759689331055,
      "activations/layer23_attention_weight_max": 37.704498291015625,
      "activations/layer23_attention_weight_min": -24.938854217529297,
      "activations/layer2_attention_weight_max": 33.62021255493164,
      "activations/layer2_attention_weight_min": -32.44085693359375,
      "activations/layer3_attention_weight_max": 90.26995849609375,
      "activations/layer3_attention_weight_min": -88.95732116699219,
      "activations/layer4_attention_weight_max": 85.46244812011719,
      "activations/layer4_attention_weight_min": -84.46197509765625,
      "activations/layer5_attention_weight_max": 61.5998649597168,
      "activations/layer5_attention_weight_min": -70.14738464355469,
      "activations/layer6_attention_weight_max": 56.4930419921875,
      "activations/layer6_attention_weight_min": -56.54037857055664,
      "activations/layer7_attention_weight_max": 62.73075485229492,
      "activations/layer7_attention_weight_min": -67.70794677734375,
      "activations/layer8_attention_weight_max": 46.766239166259766,
      "activations/layer8_attention_weight_min": -49.47653579711914,
      "activations/layer9_attention_weight_max": 44.56488800048828,
      "activations/layer9_attention_weight_min": -45.33140182495117,
      "epoch": 18.94,
      "learning_rate": 2.814924242424242e-05,
      "loss": 2.732,
      "step": 325950
    },
    {
      "activations/layer0_attention_weight_max": 15.522614479064941,
      "activations/layer0_attention_weight_min": -13.445351600646973,
      "activations/layer10_attention_weight_max": 34.45429229736328,
      "activations/layer10_attention_weight_min": -35.408935546875,
      "activations/layer11_attention_weight_max": 32.9710693359375,
      "activations/layer11_attention_weight_min": -34.74522399902344,
      "activations/layer12_attention_weight_max": 19.196430206298828,
      "activations/layer12_attention_weight_min": -26.43621253967285,
      "activations/layer13_attention_weight_max": 37.254451751708984,
      "activations/layer13_attention_weight_min": -27.708675384521484,
      "activations/layer14_attention_weight_max": 35.93577575683594,
      "activations/layer14_attention_weight_min": -29.835243225097656,
      "activations/layer15_attention_weight_max": 32.528079986572266,
      "activations/layer15_attention_weight_min": -29.951305389404297,
      "activations/layer16_attention_weight_max": 34.78132247924805,
      "activations/layer16_attention_weight_min": -32.85956954956055,
      "activations/layer17_attention_weight_max": 50.86342239379883,
      "activations/layer17_attention_weight_min": -45.740142822265625,
      "activations/layer18_attention_weight_max": 48.10726547241211,
      "activations/layer18_attention_weight_min": -40.26936340332031,
      "activations/layer19_attention_weight_max": 23.68894386291504,
      "activations/layer19_attention_weight_min": -22.56233024597168,
      "activations/layer1_attention_weight_max": 15.289515495300293,
      "activations/layer1_attention_weight_min": -14.986963272094727,
      "activations/layer20_attention_weight_max": 23.333267211914062,
      "activations/layer20_attention_weight_min": -23.82036590576172,
      "activations/layer21_attention_weight_max": 39.132774353027344,
      "activations/layer21_attention_weight_min": -23.467931747436523,
      "activations/layer22_attention_weight_max": 27.74404525756836,
      "activations/layer22_attention_weight_min": -26.782073974609375,
      "activations/layer23_attention_weight_max": 36.96100616455078,
      "activations/layer23_attention_weight_min": -25.441036224365234,
      "activations/layer2_attention_weight_max": 32.43175506591797,
      "activations/layer2_attention_weight_min": -30.981155395507812,
      "activations/layer3_attention_weight_max": 88.47480773925781,
      "activations/layer3_attention_weight_min": -86.50689697265625,
      "activations/layer4_attention_weight_max": 86.4582748413086,
      "activations/layer4_attention_weight_min": -82.94532012939453,
      "activations/layer5_attention_weight_max": 61.8099479675293,
      "activations/layer5_attention_weight_min": -69.36502075195312,
      "activations/layer6_attention_weight_max": 50.274696350097656,
      "activations/layer6_attention_weight_min": -51.268733978271484,
      "activations/layer7_attention_weight_max": 66.61471557617188,
      "activations/layer7_attention_weight_min": -66.65574645996094,
      "activations/layer8_attention_weight_max": 49.55379104614258,
      "activations/layer8_attention_weight_min": -48.026771545410156,
      "activations/layer9_attention_weight_max": 42.39437484741211,
      "activations/layer9_attention_weight_min": -43.94355392456055,
      "epoch": 18.94,
      "learning_rate": 2.8130303030303026e-05,
      "loss": 2.7335,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_loss": 2.6875,
      "eval_runtime": 8.4558,
      "eval_samples_per_second": 507.819,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.4558,
      "eval_openwebtext_samples_per_second": 507.819,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 1.968,
      "eval_wikitext_samples_per_second": 231.704,
      "step": 326000
    },
    {
      "epoch": 18.94,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.6048,
      "eval_lambada_samples_per_second": 506.935,
      "step": 326000
    },
    {
      "activations/layer0_attention_weight_max": 15.475719451904297,
      "activations/layer0_attention_weight_min": -13.407140731811523,
      "activations/layer10_attention_weight_max": 38.7216796875,
      "activations/layer10_attention_weight_min": -38.77238845825195,
      "activations/layer11_attention_weight_max": 33.19712829589844,
      "activations/layer11_attention_weight_min": -34.665950775146484,
      "activations/layer12_attention_weight_max": 20.501657485961914,
      "activations/layer12_attention_weight_min": -25.13629722595215,
      "activations/layer13_attention_weight_max": 30.874671936035156,
      "activations/layer13_attention_weight_min": -30.64057159423828,
      "activations/layer14_attention_weight_max": 35.923606872558594,
      "activations/layer14_attention_weight_min": -31.999805450439453,
      "activations/layer15_attention_weight_max": 32.46112823486328,
      "activations/layer15_attention_weight_min": -31.334800720214844,
      "activations/layer16_attention_weight_max": 35.52609634399414,
      "activations/layer16_attention_weight_min": -33.79584503173828,
      "activations/layer17_attention_weight_max": 52.40224838256836,
      "activations/layer17_attention_weight_min": -46.52647018432617,
      "activations/layer18_attention_weight_max": 46.35771560668945,
      "activations/layer18_attention_weight_min": -42.60709762573242,
      "activations/layer19_attention_weight_max": 23.001285552978516,
      "activations/layer19_attention_weight_min": -22.8770809173584,
      "activations/layer1_attention_weight_max": 15.423847198486328,
      "activations/layer1_attention_weight_min": -13.956316947937012,
      "activations/layer20_attention_weight_max": 21.85460662841797,
      "activations/layer20_attention_weight_min": -23.6312255859375,
      "activations/layer21_attention_weight_max": 38.008514404296875,
      "activations/layer21_attention_weight_min": -26.318126678466797,
      "activations/layer22_attention_weight_max": 29.018728256225586,
      "activations/layer22_attention_weight_min": -26.733579635620117,
      "activations/layer23_attention_weight_max": 34.72692108154297,
      "activations/layer23_attention_weight_min": -25.646160125732422,
      "activations/layer2_attention_weight_max": 33.91505432128906,
      "activations/layer2_attention_weight_min": -31.535551071166992,
      "activations/layer3_attention_weight_max": 88.79901885986328,
      "activations/layer3_attention_weight_min": -89.34052276611328,
      "activations/layer4_attention_weight_max": 86.44378662109375,
      "activations/layer4_attention_weight_min": -83.32765197753906,
      "activations/layer5_attention_weight_max": 59.86835479736328,
      "activations/layer5_attention_weight_min": -69.30732727050781,
      "activations/layer6_attention_weight_max": 50.019493103027344,
      "activations/layer6_attention_weight_min": -54.52700424194336,
      "activations/layer7_attention_weight_max": 65.04069519042969,
      "activations/layer7_attention_weight_min": -65.03819274902344,
      "activations/layer8_attention_weight_max": 48.249576568603516,
      "activations/layer8_attention_weight_min": -49.946815490722656,
      "activations/layer9_attention_weight_max": 44.99582290649414,
      "activations/layer9_attention_weight_min": -47.18441390991211,
      "epoch": 18.95,
      "learning_rate": 2.8111363636363634e-05,
      "loss": 2.7366,
      "step": 326050
    },
    {
      "activations/layer0_attention_weight_max": 15.672261238098145,
      "activations/layer0_attention_weight_min": -13.604483604431152,
      "activations/layer10_attention_weight_max": 37.70975112915039,
      "activations/layer10_attention_weight_min": -37.45072555541992,
      "activations/layer11_attention_weight_max": 35.01307678222656,
      "activations/layer11_attention_weight_min": -35.77351760864258,
      "activations/layer12_attention_weight_max": 20.48798179626465,
      "activations/layer12_attention_weight_min": -27.16584014892578,
      "activations/layer13_attention_weight_max": 36.435935974121094,
      "activations/layer13_attention_weight_min": -27.116260528564453,
      "activations/layer14_attention_weight_max": 38.410648345947266,
      "activations/layer14_attention_weight_min": -31.61090660095215,
      "activations/layer15_attention_weight_max": 36.46754455566406,
      "activations/layer15_attention_weight_min": -30.763174057006836,
      "activations/layer16_attention_weight_max": 36.960628509521484,
      "activations/layer16_attention_weight_min": -33.56393814086914,
      "activations/layer17_attention_weight_max": 52.36393356323242,
      "activations/layer17_attention_weight_min": -45.16923522949219,
      "activations/layer18_attention_weight_max": 49.80181121826172,
      "activations/layer18_attention_weight_min": -41.47760772705078,
      "activations/layer19_attention_weight_max": 24.548749923706055,
      "activations/layer19_attention_weight_min": -21.201351165771484,
      "activations/layer1_attention_weight_max": 16.727298736572266,
      "activations/layer1_attention_weight_min": -13.635109901428223,
      "activations/layer20_attention_weight_max": 22.41216468811035,
      "activations/layer20_attention_weight_min": -21.088911056518555,
      "activations/layer21_attention_weight_max": 35.923072814941406,
      "activations/layer21_attention_weight_min": -23.404464721679688,
      "activations/layer22_attention_weight_max": 28.470043182373047,
      "activations/layer22_attention_weight_min": -24.353534698486328,
      "activations/layer23_attention_weight_max": 38.47477722167969,
      "activations/layer23_attention_weight_min": -23.536039352416992,
      "activations/layer2_attention_weight_max": 32.51496505737305,
      "activations/layer2_attention_weight_min": -31.020122528076172,
      "activations/layer3_attention_weight_max": 91.09912109375,
      "activations/layer3_attention_weight_min": -88.58454132080078,
      "activations/layer4_attention_weight_max": 84.01725006103516,
      "activations/layer4_attention_weight_min": -84.20985412597656,
      "activations/layer5_attention_weight_max": 61.79098129272461,
      "activations/layer5_attention_weight_min": -73.32821655273438,
      "activations/layer6_attention_weight_max": 52.050289154052734,
      "activations/layer6_attention_weight_min": -51.998924255371094,
      "activations/layer7_attention_weight_max": 68.55690002441406,
      "activations/layer7_attention_weight_min": -63.15848159790039,
      "activations/layer8_attention_weight_max": 46.307029724121094,
      "activations/layer8_attention_weight_min": -51.578773498535156,
      "activations/layer9_attention_weight_max": 45.73173904418945,
      "activations/layer9_attention_weight_min": -45.30060958862305,
      "epoch": 18.95,
      "learning_rate": 2.809242424242424e-05,
      "loss": 2.733,
      "step": 326100
    },
    {
      "activations/layer0_attention_weight_max": 15.805326461791992,
      "activations/layer0_attention_weight_min": -13.452592849731445,
      "activations/layer10_attention_weight_max": 33.525390625,
      "activations/layer10_attention_weight_min": -34.304603576660156,
      "activations/layer11_attention_weight_max": 31.71590232849121,
      "activations/layer11_attention_weight_min": -34.23445129394531,
      "activations/layer12_attention_weight_max": 18.388654708862305,
      "activations/layer12_attention_weight_min": -25.419063568115234,
      "activations/layer13_attention_weight_max": 38.470428466796875,
      "activations/layer13_attention_weight_min": -31.44717025756836,
      "activations/layer14_attention_weight_max": 35.65312957763672,
      "activations/layer14_attention_weight_min": -31.39682388305664,
      "activations/layer15_attention_weight_max": 32.66648483276367,
      "activations/layer15_attention_weight_min": -30.199926376342773,
      "activations/layer16_attention_weight_max": 33.5724983215332,
      "activations/layer16_attention_weight_min": -31.783002853393555,
      "activations/layer17_attention_weight_max": 54.95284652709961,
      "activations/layer17_attention_weight_min": -45.393009185791016,
      "activations/layer18_attention_weight_max": 47.79706573486328,
      "activations/layer18_attention_weight_min": -40.45730972290039,
      "activations/layer19_attention_weight_max": 25.0244140625,
      "activations/layer19_attention_weight_min": -22.748310089111328,
      "activations/layer1_attention_weight_max": 14.936971664428711,
      "activations/layer1_attention_weight_min": -14.113109588623047,
      "activations/layer20_attention_weight_max": 21.324270248413086,
      "activations/layer20_attention_weight_min": -20.33775520324707,
      "activations/layer21_attention_weight_max": 39.27646255493164,
      "activations/layer21_attention_weight_min": -27.860477447509766,
      "activations/layer22_attention_weight_max": 28.840065002441406,
      "activations/layer22_attention_weight_min": -25.965499877929688,
      "activations/layer23_attention_weight_max": 38.14603805541992,
      "activations/layer23_attention_weight_min": -25.978092193603516,
      "activations/layer2_attention_weight_max": 33.804283142089844,
      "activations/layer2_attention_weight_min": -31.298032760620117,
      "activations/layer3_attention_weight_max": 88.85078430175781,
      "activations/layer3_attention_weight_min": -91.68280029296875,
      "activations/layer4_attention_weight_max": 83.43795776367188,
      "activations/layer4_attention_weight_min": -79.93817138671875,
      "activations/layer5_attention_weight_max": 60.22796630859375,
      "activations/layer5_attention_weight_min": -69.85773468017578,
      "activations/layer6_attention_weight_max": 48.45761489868164,
      "activations/layer6_attention_weight_min": -51.51529312133789,
      "activations/layer7_attention_weight_max": 62.82680130004883,
      "activations/layer7_attention_weight_min": -63.79188537597656,
      "activations/layer8_attention_weight_max": 44.33781814575195,
      "activations/layer8_attention_weight_min": -46.641075134277344,
      "activations/layer9_attention_weight_max": 41.47080612182617,
      "activations/layer9_attention_weight_min": -44.76237106323242,
      "epoch": 18.95,
      "learning_rate": 2.8073484848484847e-05,
      "loss": 2.7314,
      "step": 326150
    },
    {
      "activations/layer0_attention_weight_max": 14.947250366210938,
      "activations/layer0_attention_weight_min": -13.426041603088379,
      "activations/layer10_attention_weight_max": 35.866703033447266,
      "activations/layer10_attention_weight_min": -35.47400665283203,
      "activations/layer11_attention_weight_max": 32.230308532714844,
      "activations/layer11_attention_weight_min": -33.43348693847656,
      "activations/layer12_attention_weight_max": 20.351795196533203,
      "activations/layer12_attention_weight_min": -28.194477081298828,
      "activations/layer13_attention_weight_max": 32.94378662109375,
      "activations/layer13_attention_weight_min": -26.83287811279297,
      "activations/layer14_attention_weight_max": 34.44034957885742,
      "activations/layer14_attention_weight_min": -28.154813766479492,
      "activations/layer15_attention_weight_max": 31.714784622192383,
      "activations/layer15_attention_weight_min": -29.83840560913086,
      "activations/layer16_attention_weight_max": 32.970909118652344,
      "activations/layer16_attention_weight_min": -33.09189224243164,
      "activations/layer17_attention_weight_max": 49.44907760620117,
      "activations/layer17_attention_weight_min": -44.70326614379883,
      "activations/layer18_attention_weight_max": 45.276424407958984,
      "activations/layer18_attention_weight_min": -39.44525146484375,
      "activations/layer19_attention_weight_max": 24.170217514038086,
      "activations/layer19_attention_weight_min": -23.85091209411621,
      "activations/layer1_attention_weight_max": 16.484973907470703,
      "activations/layer1_attention_weight_min": -13.750869750976562,
      "activations/layer20_attention_weight_max": 21.72165298461914,
      "activations/layer20_attention_weight_min": -24.367168426513672,
      "activations/layer21_attention_weight_max": 35.83935546875,
      "activations/layer21_attention_weight_min": -24.854516983032227,
      "activations/layer22_attention_weight_max": 28.03797721862793,
      "activations/layer22_attention_weight_min": -25.679054260253906,
      "activations/layer23_attention_weight_max": 35.885475158691406,
      "activations/layer23_attention_weight_min": -24.79681396484375,
      "activations/layer2_attention_weight_max": 31.907756805419922,
      "activations/layer2_attention_weight_min": -30.497848510742188,
      "activations/layer3_attention_weight_max": 81.84506225585938,
      "activations/layer3_attention_weight_min": -85.89149475097656,
      "activations/layer4_attention_weight_max": 80.4475326538086,
      "activations/layer4_attention_weight_min": -80.72759246826172,
      "activations/layer5_attention_weight_max": 61.59421920776367,
      "activations/layer5_attention_weight_min": -69.05853271484375,
      "activations/layer6_attention_weight_max": 47.937984466552734,
      "activations/layer6_attention_weight_min": -53.17363357543945,
      "activations/layer7_attention_weight_max": 65.85585021972656,
      "activations/layer7_attention_weight_min": -63.09859085083008,
      "activations/layer8_attention_weight_max": 47.86233901977539,
      "activations/layer8_attention_weight_min": -51.244720458984375,
      "activations/layer9_attention_weight_max": 43.06684494018555,
      "activations/layer9_attention_weight_min": -45.255043029785156,
      "epoch": 18.95,
      "learning_rate": 2.805454545454545e-05,
      "loss": 2.7456,
      "step": 326200
    },
    {
      "activations/layer0_attention_weight_max": 16.671293258666992,
      "activations/layer0_attention_weight_min": -13.300260543823242,
      "activations/layer10_attention_weight_max": 35.250404357910156,
      "activations/layer10_attention_weight_min": -35.23714065551758,
      "activations/layer11_attention_weight_max": 32.62907409667969,
      "activations/layer11_attention_weight_min": -34.71743392944336,
      "activations/layer12_attention_weight_max": 18.16645050048828,
      "activations/layer12_attention_weight_min": -26.136690139770508,
      "activations/layer13_attention_weight_max": 31.565004348754883,
      "activations/layer13_attention_weight_min": -27.043405532836914,
      "activations/layer14_attention_weight_max": 35.78807830810547,
      "activations/layer14_attention_weight_min": -29.300491333007812,
      "activations/layer15_attention_weight_max": 33.42635726928711,
      "activations/layer15_attention_weight_min": -29.138065338134766,
      "activations/layer16_attention_weight_max": 32.11742401123047,
      "activations/layer16_attention_weight_min": -30.28542137145996,
      "activations/layer17_attention_weight_max": 47.70774459838867,
      "activations/layer17_attention_weight_min": -42.41122055053711,
      "activations/layer18_attention_weight_max": 47.194984436035156,
      "activations/layer18_attention_weight_min": -41.204776763916016,
      "activations/layer19_attention_weight_max": 24.644638061523438,
      "activations/layer19_attention_weight_min": -23.984302520751953,
      "activations/layer1_attention_weight_max": 15.680193901062012,
      "activations/layer1_attention_weight_min": -13.576736450195312,
      "activations/layer20_attention_weight_max": 22.90049934387207,
      "activations/layer20_attention_weight_min": -25.302471160888672,
      "activations/layer21_attention_weight_max": 38.0163459777832,
      "activations/layer21_attention_weight_min": -22.095827102661133,
      "activations/layer22_attention_weight_max": 29.21265411376953,
      "activations/layer22_attention_weight_min": -28.264394760131836,
      "activations/layer23_attention_weight_max": 41.444217681884766,
      "activations/layer23_attention_weight_min": -27.285749435424805,
      "activations/layer2_attention_weight_max": 33.67686080932617,
      "activations/layer2_attention_weight_min": -29.992155075073242,
      "activations/layer3_attention_weight_max": 92.04838562011719,
      "activations/layer3_attention_weight_min": -82.79191589355469,
      "activations/layer4_attention_weight_max": 83.88607788085938,
      "activations/layer4_attention_weight_min": -81.6623306274414,
      "activations/layer5_attention_weight_max": 65.7590103149414,
      "activations/layer5_attention_weight_min": -70.69629669189453,
      "activations/layer6_attention_weight_max": 50.177616119384766,
      "activations/layer6_attention_weight_min": -50.29106521606445,
      "activations/layer7_attention_weight_max": 67.90260314941406,
      "activations/layer7_attention_weight_min": -64.81951904296875,
      "activations/layer8_attention_weight_max": 47.081790924072266,
      "activations/layer8_attention_weight_min": -49.77330017089844,
      "activations/layer9_attention_weight_max": 42.646175384521484,
      "activations/layer9_attention_weight_min": -44.485416412353516,
      "epoch": 18.96,
      "learning_rate": 2.8035606060606057e-05,
      "loss": 2.7238,
      "step": 326250
    },
    {
      "activations/layer0_attention_weight_max": 16.09608268737793,
      "activations/layer0_attention_weight_min": -13.54842758178711,
      "activations/layer10_attention_weight_max": 35.400390625,
      "activations/layer10_attention_weight_min": -35.72467803955078,
      "activations/layer11_attention_weight_max": 34.082515716552734,
      "activations/layer11_attention_weight_min": -36.6619873046875,
      "activations/layer12_attention_weight_max": 18.45114517211914,
      "activations/layer12_attention_weight_min": -23.194873809814453,
      "activations/layer13_attention_weight_max": 31.651575088500977,
      "activations/layer13_attention_weight_min": -27.989824295043945,
      "activations/layer14_attention_weight_max": 32.268775939941406,
      "activations/layer14_attention_weight_min": -31.27623748779297,
      "activations/layer15_attention_weight_max": 34.56694030761719,
      "activations/layer15_attention_weight_min": -31.977081298828125,
      "activations/layer16_attention_weight_max": 33.16386413574219,
      "activations/layer16_attention_weight_min": -33.819339752197266,
      "activations/layer17_attention_weight_max": 49.29127502441406,
      "activations/layer17_attention_weight_min": -49.028472900390625,
      "activations/layer18_attention_weight_max": 45.96235275268555,
      "activations/layer18_attention_weight_min": -44.16261672973633,
      "activations/layer19_attention_weight_max": 23.212278366088867,
      "activations/layer19_attention_weight_min": -24.053390502929688,
      "activations/layer1_attention_weight_max": 15.085177421569824,
      "activations/layer1_attention_weight_min": -13.302873611450195,
      "activations/layer20_attention_weight_max": 19.52806854248047,
      "activations/layer20_attention_weight_min": -22.759035110473633,
      "activations/layer21_attention_weight_max": 35.5291862487793,
      "activations/layer21_attention_weight_min": -22.677095413208008,
      "activations/layer22_attention_weight_max": 27.748214721679688,
      "activations/layer22_attention_weight_min": -26.97738265991211,
      "activations/layer23_attention_weight_max": 33.04657745361328,
      "activations/layer23_attention_weight_min": -24.305112838745117,
      "activations/layer2_attention_weight_max": 31.989334106445312,
      "activations/layer2_attention_weight_min": -30.89992904663086,
      "activations/layer3_attention_weight_max": 87.73882293701172,
      "activations/layer3_attention_weight_min": -88.27680206298828,
      "activations/layer4_attention_weight_max": 83.45146942138672,
      "activations/layer4_attention_weight_min": -82.50878143310547,
      "activations/layer5_attention_weight_max": 63.37010192871094,
      "activations/layer5_attention_weight_min": -74.10136413574219,
      "activations/layer6_attention_weight_max": 51.64241027832031,
      "activations/layer6_attention_weight_min": -50.18021774291992,
      "activations/layer7_attention_weight_max": 61.80669403076172,
      "activations/layer7_attention_weight_min": -65.79299926757812,
      "activations/layer8_attention_weight_max": 45.06077575683594,
      "activations/layer8_attention_weight_min": -49.41513442993164,
      "activations/layer9_attention_weight_max": 42.42807388305664,
      "activations/layer9_attention_weight_min": -46.70762634277344,
      "epoch": 18.96,
      "learning_rate": 2.8016666666666662e-05,
      "loss": 2.7214,
      "step": 326300
    },
    {
      "activations/layer0_attention_weight_max": 16.024433135986328,
      "activations/layer0_attention_weight_min": -13.271926879882812,
      "activations/layer10_attention_weight_max": 40.786537170410156,
      "activations/layer10_attention_weight_min": -40.6630744934082,
      "activations/layer11_attention_weight_max": 39.64137268066406,
      "activations/layer11_attention_weight_min": -37.64524459838867,
      "activations/layer12_attention_weight_max": 20.378055572509766,
      "activations/layer12_attention_weight_min": -24.625226974487305,
      "activations/layer13_attention_weight_max": 37.57402801513672,
      "activations/layer13_attention_weight_min": -30.601797103881836,
      "activations/layer14_attention_weight_max": 35.95027542114258,
      "activations/layer14_attention_weight_min": -30.61300277709961,
      "activations/layer15_attention_weight_max": 33.17815017700195,
      "activations/layer15_attention_weight_min": -31.404457092285156,
      "activations/layer16_attention_weight_max": 34.848304748535156,
      "activations/layer16_attention_weight_min": -32.464996337890625,
      "activations/layer17_attention_weight_max": 52.98180389404297,
      "activations/layer17_attention_weight_min": -47.02864074707031,
      "activations/layer18_attention_weight_max": 46.97968673706055,
      "activations/layer18_attention_weight_min": -39.08687210083008,
      "activations/layer19_attention_weight_max": 25.1170711517334,
      "activations/layer19_attention_weight_min": -22.334562301635742,
      "activations/layer1_attention_weight_max": 16.051774978637695,
      "activations/layer1_attention_weight_min": -13.85135555267334,
      "activations/layer20_attention_weight_max": 23.988941192626953,
      "activations/layer20_attention_weight_min": -20.54570198059082,
      "activations/layer21_attention_weight_max": 33.50315856933594,
      "activations/layer21_attention_weight_min": -20.747661590576172,
      "activations/layer22_attention_weight_max": 26.94999885559082,
      "activations/layer22_attention_weight_min": -23.974756240844727,
      "activations/layer23_attention_weight_max": 34.0960693359375,
      "activations/layer23_attention_weight_min": -24.996929168701172,
      "activations/layer2_attention_weight_max": 34.114776611328125,
      "activations/layer2_attention_weight_min": -33.784141540527344,
      "activations/layer3_attention_weight_max": 92.41272735595703,
      "activations/layer3_attention_weight_min": -91.72463989257812,
      "activations/layer4_attention_weight_max": 86.93807220458984,
      "activations/layer4_attention_weight_min": -88.04558563232422,
      "activations/layer5_attention_weight_max": 63.07353973388672,
      "activations/layer5_attention_weight_min": -67.9250717163086,
      "activations/layer6_attention_weight_max": 47.05854034423828,
      "activations/layer6_attention_weight_min": -49.63372039794922,
      "activations/layer7_attention_weight_max": 64.83802032470703,
      "activations/layer7_attention_weight_min": -63.38486862182617,
      "activations/layer8_attention_weight_max": 48.91182327270508,
      "activations/layer8_attention_weight_min": -47.55208969116211,
      "activations/layer9_attention_weight_max": 48.33913803100586,
      "activations/layer9_attention_weight_min": -46.36323928833008,
      "epoch": 18.96,
      "learning_rate": 2.799772727272727e-05,
      "loss": 2.7165,
      "step": 326350
    },
    {
      "activations/layer0_attention_weight_max": 15.259117126464844,
      "activations/layer0_attention_weight_min": -13.435530662536621,
      "activations/layer10_attention_weight_max": 37.80086135864258,
      "activations/layer10_attention_weight_min": -36.14499282836914,
      "activations/layer11_attention_weight_max": 32.96883010864258,
      "activations/layer11_attention_weight_min": -33.09304428100586,
      "activations/layer12_attention_weight_max": 18.4128360748291,
      "activations/layer12_attention_weight_min": -33.04084396362305,
      "activations/layer13_attention_weight_max": 34.3017463684082,
      "activations/layer13_attention_weight_min": -30.70522117614746,
      "activations/layer14_attention_weight_max": 35.74852752685547,
      "activations/layer14_attention_weight_min": -31.019651412963867,
      "activations/layer15_attention_weight_max": 33.569515228271484,
      "activations/layer15_attention_weight_min": -31.517742156982422,
      "activations/layer16_attention_weight_max": 36.02726364135742,
      "activations/layer16_attention_weight_min": -32.820716857910156,
      "activations/layer17_attention_weight_max": 51.218135833740234,
      "activations/layer17_attention_weight_min": -47.89501953125,
      "activations/layer18_attention_weight_max": 45.50055694580078,
      "activations/layer18_attention_weight_min": -41.15989685058594,
      "activations/layer19_attention_weight_max": 23.63654899597168,
      "activations/layer19_attention_weight_min": -22.086957931518555,
      "activations/layer1_attention_weight_max": 15.672613143920898,
      "activations/layer1_attention_weight_min": -13.753313064575195,
      "activations/layer20_attention_weight_max": 23.098817825317383,
      "activations/layer20_attention_weight_min": -21.843517303466797,
      "activations/layer21_attention_weight_max": 33.298667907714844,
      "activations/layer21_attention_weight_min": -24.824649810791016,
      "activations/layer22_attention_weight_max": 29.522674560546875,
      "activations/layer22_attention_weight_min": -26.95502471923828,
      "activations/layer23_attention_weight_max": 34.75935363769531,
      "activations/layer23_attention_weight_min": -24.566993713378906,
      "activations/layer2_attention_weight_max": 31.949819564819336,
      "activations/layer2_attention_weight_min": -30.585124969482422,
      "activations/layer3_attention_weight_max": 88.23880004882812,
      "activations/layer3_attention_weight_min": -86.9347915649414,
      "activations/layer4_attention_weight_max": 83.6628646850586,
      "activations/layer4_attention_weight_min": -87.83771514892578,
      "activations/layer5_attention_weight_max": 62.40541076660156,
      "activations/layer5_attention_weight_min": -69.68496704101562,
      "activations/layer6_attention_weight_max": 51.21430969238281,
      "activations/layer6_attention_weight_min": -51.63748550415039,
      "activations/layer7_attention_weight_max": 66.08387756347656,
      "activations/layer7_attention_weight_min": -67.0767822265625,
      "activations/layer8_attention_weight_max": 43.091522216796875,
      "activations/layer8_attention_weight_min": -46.74846267700195,
      "activations/layer9_attention_weight_max": 41.25590515136719,
      "activations/layer9_attention_weight_min": -44.323394775390625,
      "epoch": 18.97,
      "learning_rate": 2.797878787878788e-05,
      "loss": 2.7416,
      "step": 326400
    },
    {
      "activations/layer0_attention_weight_max": 15.84597396850586,
      "activations/layer0_attention_weight_min": -13.596762657165527,
      "activations/layer10_attention_weight_max": 34.719215393066406,
      "activations/layer10_attention_weight_min": -36.61039733886719,
      "activations/layer11_attention_weight_max": 32.753753662109375,
      "activations/layer11_attention_weight_min": -35.44911575317383,
      "activations/layer12_attention_weight_max": 18.54275131225586,
      "activations/layer12_attention_weight_min": -24.719303131103516,
      "activations/layer13_attention_weight_max": 30.474924087524414,
      "activations/layer13_attention_weight_min": -25.883602142333984,
      "activations/layer14_attention_weight_max": 34.84379196166992,
      "activations/layer14_attention_weight_min": -28.73489761352539,
      "activations/layer15_attention_weight_max": 32.02394104003906,
      "activations/layer15_attention_weight_min": -29.43804168701172,
      "activations/layer16_attention_weight_max": 32.587188720703125,
      "activations/layer16_attention_weight_min": -31.766712188720703,
      "activations/layer17_attention_weight_max": 49.92447280883789,
      "activations/layer17_attention_weight_min": -45.53737258911133,
      "activations/layer18_attention_weight_max": 44.85068130493164,
      "activations/layer18_attention_weight_min": -40.09928512573242,
      "activations/layer19_attention_weight_max": 23.823509216308594,
      "activations/layer19_attention_weight_min": -24.127994537353516,
      "activations/layer1_attention_weight_max": 15.248723030090332,
      "activations/layer1_attention_weight_min": -13.395268440246582,
      "activations/layer20_attention_weight_max": 23.003887176513672,
      "activations/layer20_attention_weight_min": -23.35465431213379,
      "activations/layer21_attention_weight_max": 33.26495361328125,
      "activations/layer21_attention_weight_min": -24.245132446289062,
      "activations/layer22_attention_weight_max": 27.53095245361328,
      "activations/layer22_attention_weight_min": -28.02852439880371,
      "activations/layer23_attention_weight_max": 36.942230224609375,
      "activations/layer23_attention_weight_min": -24.279014587402344,
      "activations/layer2_attention_weight_max": 30.988082885742188,
      "activations/layer2_attention_weight_min": -30.609783172607422,
      "activations/layer3_attention_weight_max": 84.23550415039062,
      "activations/layer3_attention_weight_min": -83.49931335449219,
      "activations/layer4_attention_weight_max": 82.08843994140625,
      "activations/layer4_attention_weight_min": -83.29147338867188,
      "activations/layer5_attention_weight_max": 64.1978759765625,
      "activations/layer5_attention_weight_min": -70.3336410522461,
      "activations/layer6_attention_weight_max": 48.0485954284668,
      "activations/layer6_attention_weight_min": -51.414119720458984,
      "activations/layer7_attention_weight_max": 63.413021087646484,
      "activations/layer7_attention_weight_min": -64.30854797363281,
      "activations/layer8_attention_weight_max": 44.562686920166016,
      "activations/layer8_attention_weight_min": -49.70914840698242,
      "activations/layer9_attention_weight_max": 42.20643615722656,
      "activations/layer9_attention_weight_min": -45.45677185058594,
      "epoch": 18.97,
      "learning_rate": 2.7959848484848484e-05,
      "loss": 2.7231,
      "step": 326450
    },
    {
      "activations/layer0_attention_weight_max": 15.611077308654785,
      "activations/layer0_attention_weight_min": -13.3301420211792,
      "activations/layer10_attention_weight_max": 34.54191970825195,
      "activations/layer10_attention_weight_min": -34.33551025390625,
      "activations/layer11_attention_weight_max": 34.12757873535156,
      "activations/layer11_attention_weight_min": -35.02911376953125,
      "activations/layer12_attention_weight_max": 20.81063461303711,
      "activations/layer12_attention_weight_min": -25.797225952148438,
      "activations/layer13_attention_weight_max": 32.383644104003906,
      "activations/layer13_attention_weight_min": -27.824804306030273,
      "activations/layer14_attention_weight_max": 34.81049728393555,
      "activations/layer14_attention_weight_min": -30.494041442871094,
      "activations/layer15_attention_weight_max": 33.11134338378906,
      "activations/layer15_attention_weight_min": -30.302532196044922,
      "activations/layer16_attention_weight_max": 32.91402053833008,
      "activations/layer16_attention_weight_min": -31.758716583251953,
      "activations/layer17_attention_weight_max": 51.77389907836914,
      "activations/layer17_attention_weight_min": -47.0517463684082,
      "activations/layer18_attention_weight_max": 45.844085693359375,
      "activations/layer18_attention_weight_min": -44.237876892089844,
      "activations/layer19_attention_weight_max": 24.20237922668457,
      "activations/layer19_attention_weight_min": -23.62156867980957,
      "activations/layer1_attention_weight_max": 15.223118782043457,
      "activations/layer1_attention_weight_min": -13.512794494628906,
      "activations/layer20_attention_weight_max": 24.32468032836914,
      "activations/layer20_attention_weight_min": -22.647075653076172,
      "activations/layer21_attention_weight_max": 33.05554962158203,
      "activations/layer21_attention_weight_min": -25.865060806274414,
      "activations/layer22_attention_weight_max": 29.39045524597168,
      "activations/layer22_attention_weight_min": -25.42932891845703,
      "activations/layer23_attention_weight_max": 35.295738220214844,
      "activations/layer23_attention_weight_min": -24.14547348022461,
      "activations/layer2_attention_weight_max": 31.554027557373047,
      "activations/layer2_attention_weight_min": -31.990245819091797,
      "activations/layer3_attention_weight_max": 85.08392333984375,
      "activations/layer3_attention_weight_min": -92.58491516113281,
      "activations/layer4_attention_weight_max": 82.87434387207031,
      "activations/layer4_attention_weight_min": -82.21882629394531,
      "activations/layer5_attention_weight_max": 60.359920501708984,
      "activations/layer5_attention_weight_min": -69.52547454833984,
      "activations/layer6_attention_weight_max": 47.85820388793945,
      "activations/layer6_attention_weight_min": -51.15557098388672,
      "activations/layer7_attention_weight_max": 64.2626953125,
      "activations/layer7_attention_weight_min": -65.28224182128906,
      "activations/layer8_attention_weight_max": 43.565399169921875,
      "activations/layer8_attention_weight_min": -45.70771789550781,
      "activations/layer9_attention_weight_max": 41.550132751464844,
      "activations/layer9_attention_weight_min": -44.406063079833984,
      "epoch": 18.97,
      "learning_rate": 2.7940909090909092e-05,
      "loss": 2.735,
      "step": 326500
    },
    {
      "activations/layer0_attention_weight_max": 14.890560150146484,
      "activations/layer0_attention_weight_min": -13.593351364135742,
      "activations/layer10_attention_weight_max": 36.67829132080078,
      "activations/layer10_attention_weight_min": -36.022926330566406,
      "activations/layer11_attention_weight_max": 32.935585021972656,
      "activations/layer11_attention_weight_min": -34.093910217285156,
      "activations/layer12_attention_weight_max": 18.890491485595703,
      "activations/layer12_attention_weight_min": -24.403160095214844,
      "activations/layer13_attention_weight_max": 36.074363708496094,
      "activations/layer13_attention_weight_min": -27.544897079467773,
      "activations/layer14_attention_weight_max": 35.49306106567383,
      "activations/layer14_attention_weight_min": -31.524789810180664,
      "activations/layer15_attention_weight_max": 34.36545944213867,
      "activations/layer15_attention_weight_min": -31.727397918701172,
      "activations/layer16_attention_weight_max": 33.94581985473633,
      "activations/layer16_attention_weight_min": -31.45574188232422,
      "activations/layer17_attention_weight_max": 51.2944221496582,
      "activations/layer17_attention_weight_min": -44.22946548461914,
      "activations/layer18_attention_weight_max": 47.591758728027344,
      "activations/layer18_attention_weight_min": -40.669273376464844,
      "activations/layer19_attention_weight_max": 24.67351531982422,
      "activations/layer19_attention_weight_min": -24.070199966430664,
      "activations/layer1_attention_weight_max": 15.912992477416992,
      "activations/layer1_attention_weight_min": -14.104608535766602,
      "activations/layer20_attention_weight_max": 22.754621505737305,
      "activations/layer20_attention_weight_min": -23.675762176513672,
      "activations/layer21_attention_weight_max": 34.89270782470703,
      "activations/layer21_attention_weight_min": -26.64706802368164,
      "activations/layer22_attention_weight_max": 28.825057983398438,
      "activations/layer22_attention_weight_min": -25.74111557006836,
      "activations/layer23_attention_weight_max": 35.79511260986328,
      "activations/layer23_attention_weight_min": -24.707843780517578,
      "activations/layer2_attention_weight_max": 33.053489685058594,
      "activations/layer2_attention_weight_min": -30.366270065307617,
      "activations/layer3_attention_weight_max": 85.44615173339844,
      "activations/layer3_attention_weight_min": -82.3482666015625,
      "activations/layer4_attention_weight_max": 81.50005340576172,
      "activations/layer4_attention_weight_min": -80.33626556396484,
      "activations/layer5_attention_weight_max": 62.05549240112305,
      "activations/layer5_attention_weight_min": -69.58695220947266,
      "activations/layer6_attention_weight_max": 51.5391731262207,
      "activations/layer6_attention_weight_min": -51.07918930053711,
      "activations/layer7_attention_weight_max": 65.06819915771484,
      "activations/layer7_attention_weight_min": -62.03762435913086,
      "activations/layer8_attention_weight_max": 48.154964447021484,
      "activations/layer8_attention_weight_min": -49.43938064575195,
      "activations/layer9_attention_weight_max": 45.50141525268555,
      "activations/layer9_attention_weight_min": -44.96317672729492,
      "epoch": 18.97,
      "learning_rate": 2.7921969696969693e-05,
      "loss": 2.7467,
      "step": 326550
    },
    {
      "activations/layer0_attention_weight_max": 15.70617389678955,
      "activations/layer0_attention_weight_min": -14.049893379211426,
      "activations/layer10_attention_weight_max": 39.66244888305664,
      "activations/layer10_attention_weight_min": -38.77021026611328,
      "activations/layer11_attention_weight_max": 37.71168518066406,
      "activations/layer11_attention_weight_min": -35.43727493286133,
      "activations/layer12_attention_weight_max": 19.44354248046875,
      "activations/layer12_attention_weight_min": -28.070083618164062,
      "activations/layer13_attention_weight_max": 33.95016860961914,
      "activations/layer13_attention_weight_min": -25.27321434020996,
      "activations/layer14_attention_weight_max": 36.33405685424805,
      "activations/layer14_attention_weight_min": -31.69349479675293,
      "activations/layer15_attention_weight_max": 33.22263717651367,
      "activations/layer15_attention_weight_min": -29.414264678955078,
      "activations/layer16_attention_weight_max": 35.076637268066406,
      "activations/layer16_attention_weight_min": -35.14384078979492,
      "activations/layer17_attention_weight_max": 55.30881881713867,
      "activations/layer17_attention_weight_min": -48.5701789855957,
      "activations/layer18_attention_weight_max": 47.84455108642578,
      "activations/layer18_attention_weight_min": -41.25038146972656,
      "activations/layer19_attention_weight_max": 23.249807357788086,
      "activations/layer19_attention_weight_min": -22.753456115722656,
      "activations/layer1_attention_weight_max": 14.662660598754883,
      "activations/layer1_attention_weight_min": -14.97397518157959,
      "activations/layer20_attention_weight_max": 21.036975860595703,
      "activations/layer20_attention_weight_min": -23.160818099975586,
      "activations/layer21_attention_weight_max": 34.194122314453125,
      "activations/layer21_attention_weight_min": -21.84800148010254,
      "activations/layer22_attention_weight_max": 27.41939353942871,
      "activations/layer22_attention_weight_min": -27.04724884033203,
      "activations/layer23_attention_weight_max": 38.077842712402344,
      "activations/layer23_attention_weight_min": -25.506805419921875,
      "activations/layer2_attention_weight_max": 35.022056579589844,
      "activations/layer2_attention_weight_min": -30.765174865722656,
      "activations/layer3_attention_weight_max": 91.19837188720703,
      "activations/layer3_attention_weight_min": -92.71878814697266,
      "activations/layer4_attention_weight_max": 88.14962005615234,
      "activations/layer4_attention_weight_min": -86.14266967773438,
      "activations/layer5_attention_weight_max": 67.59056091308594,
      "activations/layer5_attention_weight_min": -73.31603240966797,
      "activations/layer6_attention_weight_max": 54.11875915527344,
      "activations/layer6_attention_weight_min": -55.340179443359375,
      "activations/layer7_attention_weight_max": 70.81583404541016,
      "activations/layer7_attention_weight_min": -67.15773010253906,
      "activations/layer8_attention_weight_max": 51.9471549987793,
      "activations/layer8_attention_weight_min": -55.9605598449707,
      "activations/layer9_attention_weight_max": 46.90215301513672,
      "activations/layer9_attention_weight_min": -48.95206832885742,
      "epoch": 18.98,
      "learning_rate": 2.79030303030303e-05,
      "loss": 2.7575,
      "step": 326600
    },
    {
      "activations/layer0_attention_weight_max": 15.202676773071289,
      "activations/layer0_attention_weight_min": -13.455020904541016,
      "activations/layer10_attention_weight_max": 38.627689361572266,
      "activations/layer10_attention_weight_min": -37.34783172607422,
      "activations/layer11_attention_weight_max": 33.39453125,
      "activations/layer11_attention_weight_min": -33.68319320678711,
      "activations/layer12_attention_weight_max": 18.032930374145508,
      "activations/layer12_attention_weight_min": -25.866413116455078,
      "activations/layer13_attention_weight_max": 32.91732406616211,
      "activations/layer13_attention_weight_min": -28.592296600341797,
      "activations/layer14_attention_weight_max": 33.748695373535156,
      "activations/layer14_attention_weight_min": -30.6653995513916,
      "activations/layer15_attention_weight_max": 32.21553421020508,
      "activations/layer15_attention_weight_min": -30.74479103088379,
      "activations/layer16_attention_weight_max": 31.76712989807129,
      "activations/layer16_attention_weight_min": -32.550357818603516,
      "activations/layer17_attention_weight_max": 48.01019287109375,
      "activations/layer17_attention_weight_min": -45.98956298828125,
      "activations/layer18_attention_weight_max": 47.84059143066406,
      "activations/layer18_attention_weight_min": -40.114994049072266,
      "activations/layer19_attention_weight_max": 24.626346588134766,
      "activations/layer19_attention_weight_min": -22.651060104370117,
      "activations/layer1_attention_weight_max": 15.893113136291504,
      "activations/layer1_attention_weight_min": -13.733561515808105,
      "activations/layer20_attention_weight_max": 21.73894691467285,
      "activations/layer20_attention_weight_min": -21.599266052246094,
      "activations/layer21_attention_weight_max": 32.75254821777344,
      "activations/layer21_attention_weight_min": -22.47193145751953,
      "activations/layer22_attention_weight_max": 29.361774444580078,
      "activations/layer22_attention_weight_min": -26.28245735168457,
      "activations/layer23_attention_weight_max": 40.76849365234375,
      "activations/layer23_attention_weight_min": -24.25885581970215,
      "activations/layer2_attention_weight_max": 32.26866912841797,
      "activations/layer2_attention_weight_min": -30.901626586914062,
      "activations/layer3_attention_weight_max": 91.94844055175781,
      "activations/layer3_attention_weight_min": -86.08651733398438,
      "activations/layer4_attention_weight_max": 82.81425476074219,
      "activations/layer4_attention_weight_min": -80.9365463256836,
      "activations/layer5_attention_weight_max": 58.727867126464844,
      "activations/layer5_attention_weight_min": -75.04737091064453,
      "activations/layer6_attention_weight_max": 49.04915237426758,
      "activations/layer6_attention_weight_min": -48.543678283691406,
      "activations/layer7_attention_weight_max": 70.4942398071289,
      "activations/layer7_attention_weight_min": -62.84026336669922,
      "activations/layer8_attention_weight_max": 46.92787170410156,
      "activations/layer8_attention_weight_min": -48.47909927368164,
      "activations/layer9_attention_weight_max": 46.762847900390625,
      "activations/layer9_attention_weight_min": -47.095027923583984,
      "epoch": 18.98,
      "learning_rate": 2.7884090909090906e-05,
      "loss": 2.7172,
      "step": 326650
    },
    {
      "activations/layer0_attention_weight_max": 14.656517028808594,
      "activations/layer0_attention_weight_min": -13.34304428100586,
      "activations/layer10_attention_weight_max": 38.27260208129883,
      "activations/layer10_attention_weight_min": -38.36140823364258,
      "activations/layer11_attention_weight_max": 36.486061096191406,
      "activations/layer11_attention_weight_min": -38.99085998535156,
      "activations/layer12_attention_weight_max": 23.718725204467773,
      "activations/layer12_attention_weight_min": -24.546777725219727,
      "activations/layer13_attention_weight_max": 33.02397155761719,
      "activations/layer13_attention_weight_min": -25.415966033935547,
      "activations/layer14_attention_weight_max": 35.44295883178711,
      "activations/layer14_attention_weight_min": -29.751529693603516,
      "activations/layer15_attention_weight_max": 31.798965454101562,
      "activations/layer15_attention_weight_min": -30.141313552856445,
      "activations/layer16_attention_weight_max": 31.21286964416504,
      "activations/layer16_attention_weight_min": -31.474929809570312,
      "activations/layer17_attention_weight_max": 49.08773422241211,
      "activations/layer17_attention_weight_min": -42.59022903442383,
      "activations/layer18_attention_weight_max": 43.87760543823242,
      "activations/layer18_attention_weight_min": -38.2978630065918,
      "activations/layer19_attention_weight_max": 22.238767623901367,
      "activations/layer19_attention_weight_min": -21.7668514251709,
      "activations/layer1_attention_weight_max": 16.187313079833984,
      "activations/layer1_attention_weight_min": -13.465699195861816,
      "activations/layer20_attention_weight_max": 21.47096824645996,
      "activations/layer20_attention_weight_min": -20.986846923828125,
      "activations/layer21_attention_weight_max": 36.08028030395508,
      "activations/layer21_attention_weight_min": -22.291812896728516,
      "activations/layer22_attention_weight_max": 28.70220947265625,
      "activations/layer22_attention_weight_min": -23.932649612426758,
      "activations/layer23_attention_weight_max": 34.93912887573242,
      "activations/layer23_attention_weight_min": -23.13258171081543,
      "activations/layer2_attention_weight_max": 31.614097595214844,
      "activations/layer2_attention_weight_min": -29.379188537597656,
      "activations/layer3_attention_weight_max": 82.93220520019531,
      "activations/layer3_attention_weight_min": -83.20406341552734,
      "activations/layer4_attention_weight_max": 82.3755111694336,
      "activations/layer4_attention_weight_min": -80.11009216308594,
      "activations/layer5_attention_weight_max": 62.737510681152344,
      "activations/layer5_attention_weight_min": -71.57530212402344,
      "activations/layer6_attention_weight_max": 52.5904655456543,
      "activations/layer6_attention_weight_min": -48.71733856201172,
      "activations/layer7_attention_weight_max": 70.60844421386719,
      "activations/layer7_attention_weight_min": -69.04015350341797,
      "activations/layer8_attention_weight_max": 49.86308670043945,
      "activations/layer8_attention_weight_min": -51.08501434326172,
      "activations/layer9_attention_weight_max": 46.34234619140625,
      "activations/layer9_attention_weight_min": -45.54865264892578,
      "epoch": 18.98,
      "learning_rate": 2.7865151515151515e-05,
      "loss": 2.7364,
      "step": 326700
    },
    {
      "activations/layer0_attention_weight_max": 15.20417594909668,
      "activations/layer0_attention_weight_min": -13.427364349365234,
      "activations/layer10_attention_weight_max": 35.4246711730957,
      "activations/layer10_attention_weight_min": -35.06557846069336,
      "activations/layer11_attention_weight_max": 32.188499450683594,
      "activations/layer11_attention_weight_min": -34.198246002197266,
      "activations/layer12_attention_weight_max": 18.90838050842285,
      "activations/layer12_attention_weight_min": -22.69126319885254,
      "activations/layer13_attention_weight_max": 36.313392639160156,
      "activations/layer13_attention_weight_min": -30.06388282775879,
      "activations/layer14_attention_weight_max": 36.91924285888672,
      "activations/layer14_attention_weight_min": -34.59761428833008,
      "activations/layer15_attention_weight_max": 36.461517333984375,
      "activations/layer15_attention_weight_min": -33.802040100097656,
      "activations/layer16_attention_weight_max": 36.746795654296875,
      "activations/layer16_attention_weight_min": -36.50465774536133,
      "activations/layer17_attention_weight_max": 53.22581100463867,
      "activations/layer17_attention_weight_min": -51.404380798339844,
      "activations/layer18_attention_weight_max": 48.13288497924805,
      "activations/layer18_attention_weight_min": -42.88660430908203,
      "activations/layer19_attention_weight_max": 26.06034278869629,
      "activations/layer19_attention_weight_min": -21.808963775634766,
      "activations/layer1_attention_weight_max": 16.200164794921875,
      "activations/layer1_attention_weight_min": -13.19331169128418,
      "activations/layer20_attention_weight_max": 22.900341033935547,
      "activations/layer20_attention_weight_min": -24.09288787841797,
      "activations/layer21_attention_weight_max": 37.103302001953125,
      "activations/layer21_attention_weight_min": -21.82428550720215,
      "activations/layer22_attention_weight_max": 31.668006896972656,
      "activations/layer22_attention_weight_min": -25.15423583984375,
      "activations/layer23_attention_weight_max": 37.44151306152344,
      "activations/layer23_attention_weight_min": -22.986865997314453,
      "activations/layer2_attention_weight_max": 31.039113998413086,
      "activations/layer2_attention_weight_min": -29.86447525024414,
      "activations/layer3_attention_weight_max": 81.81307220458984,
      "activations/layer3_attention_weight_min": -81.32450103759766,
      "activations/layer4_attention_weight_max": 80.9726791381836,
      "activations/layer4_attention_weight_min": -80.72403717041016,
      "activations/layer5_attention_weight_max": 66.08564758300781,
      "activations/layer5_attention_weight_min": -73.23512268066406,
      "activations/layer6_attention_weight_max": 46.80329895019531,
      "activations/layer6_attention_weight_min": -49.24614334106445,
      "activations/layer7_attention_weight_max": 65.31163024902344,
      "activations/layer7_attention_weight_min": -61.85050582885742,
      "activations/layer8_attention_weight_max": 46.04097366333008,
      "activations/layer8_attention_weight_min": -48.45438003540039,
      "activations/layer9_attention_weight_max": 44.33894729614258,
      "activations/layer9_attention_weight_min": -44.51789474487305,
      "epoch": 18.99,
      "learning_rate": 2.7846212121212116e-05,
      "loss": 2.729,
      "step": 326750
    },
    {
      "activations/layer0_attention_weight_max": 15.638794898986816,
      "activations/layer0_attention_weight_min": -13.85995864868164,
      "activations/layer10_attention_weight_max": 39.17609786987305,
      "activations/layer10_attention_weight_min": -38.503841400146484,
      "activations/layer11_attention_weight_max": 36.00566864013672,
      "activations/layer11_attention_weight_min": -35.443721771240234,
      "activations/layer12_attention_weight_max": 20.04846954345703,
      "activations/layer12_attention_weight_min": -25.825681686401367,
      "activations/layer13_attention_weight_max": 37.89209747314453,
      "activations/layer13_attention_weight_min": -28.183635711669922,
      "activations/layer14_attention_weight_max": 37.89777374267578,
      "activations/layer14_attention_weight_min": -30.628955841064453,
      "activations/layer15_attention_weight_max": 35.79148483276367,
      "activations/layer15_attention_weight_min": -30.825849533081055,
      "activations/layer16_attention_weight_max": 37.05167770385742,
      "activations/layer16_attention_weight_min": -33.813995361328125,
      "activations/layer17_attention_weight_max": 56.495391845703125,
      "activations/layer17_attention_weight_min": -48.90435791015625,
      "activations/layer18_attention_weight_max": 51.10248947143555,
      "activations/layer18_attention_weight_min": -44.927852630615234,
      "activations/layer19_attention_weight_max": 26.399805068969727,
      "activations/layer19_attention_weight_min": -22.307275772094727,
      "activations/layer1_attention_weight_max": 15.742302894592285,
      "activations/layer1_attention_weight_min": -14.387591361999512,
      "activations/layer20_attention_weight_max": 24.93617820739746,
      "activations/layer20_attention_weight_min": -20.090782165527344,
      "activations/layer21_attention_weight_max": 35.164215087890625,
      "activations/layer21_attention_weight_min": -22.483739852905273,
      "activations/layer22_attention_weight_max": 29.24561309814453,
      "activations/layer22_attention_weight_min": -25.84572982788086,
      "activations/layer23_attention_weight_max": 37.77119064331055,
      "activations/layer23_attention_weight_min": -23.43989372253418,
      "activations/layer2_attention_weight_max": 33.19595718383789,
      "activations/layer2_attention_weight_min": -30.823333740234375,
      "activations/layer3_attention_weight_max": 92.16060638427734,
      "activations/layer3_attention_weight_min": -84.36152648925781,
      "activations/layer4_attention_weight_max": 87.64948272705078,
      "activations/layer4_attention_weight_min": -84.33512878417969,
      "activations/layer5_attention_weight_max": 67.89071655273438,
      "activations/layer5_attention_weight_min": -72.89399719238281,
      "activations/layer6_attention_weight_max": 55.130035400390625,
      "activations/layer6_attention_weight_min": -51.72414016723633,
      "activations/layer7_attention_weight_max": 67.83467864990234,
      "activations/layer7_attention_weight_min": -69.6969985961914,
      "activations/layer8_attention_weight_max": 49.391273498535156,
      "activations/layer8_attention_weight_min": -52.0230598449707,
      "activations/layer9_attention_weight_max": 46.1856803894043,
      "activations/layer9_attention_weight_min": -48.16632080078125,
      "epoch": 18.99,
      "learning_rate": 2.7827272727272725e-05,
      "loss": 2.728,
      "step": 326800
    },
    {
      "activations/layer0_attention_weight_max": 15.92227840423584,
      "activations/layer0_attention_weight_min": -13.550332069396973,
      "activations/layer10_attention_weight_max": 39.854225158691406,
      "activations/layer10_attention_weight_min": -37.25374221801758,
      "activations/layer11_attention_weight_max": 39.462406158447266,
      "activations/layer11_attention_weight_min": -35.47068405151367,
      "activations/layer12_attention_weight_max": 18.95058250427246,
      "activations/layer12_attention_weight_min": -26.001569747924805,
      "activations/layer13_attention_weight_max": 40.077552795410156,
      "activations/layer13_attention_weight_min": -27.936243057250977,
      "activations/layer14_attention_weight_max": 39.50401306152344,
      "activations/layer14_attention_weight_min": -30.5998477935791,
      "activations/layer15_attention_weight_max": 44.33440017700195,
      "activations/layer15_attention_weight_min": -29.656299591064453,
      "activations/layer16_attention_weight_max": 32.29372024536133,
      "activations/layer16_attention_weight_min": -31.384721755981445,
      "activations/layer17_attention_weight_max": 51.11624526977539,
      "activations/layer17_attention_weight_min": -46.30357360839844,
      "activations/layer18_attention_weight_max": 46.34931945800781,
      "activations/layer18_attention_weight_min": -40.9263801574707,
      "activations/layer19_attention_weight_max": 24.782455444335938,
      "activations/layer19_attention_weight_min": -22.007169723510742,
      "activations/layer1_attention_weight_max": 17.067386627197266,
      "activations/layer1_attention_weight_min": -13.024693489074707,
      "activations/layer20_attention_weight_max": 21.195295333862305,
      "activations/layer20_attention_weight_min": -20.269296646118164,
      "activations/layer21_attention_weight_max": 36.3000602722168,
      "activations/layer21_attention_weight_min": -24.474346160888672,
      "activations/layer22_attention_weight_max": 30.045793533325195,
      "activations/layer22_attention_weight_min": -28.694313049316406,
      "activations/layer23_attention_weight_max": 35.508995056152344,
      "activations/layer23_attention_weight_min": -27.59357452392578,
      "activations/layer2_attention_weight_max": 32.66436767578125,
      "activations/layer2_attention_weight_min": -32.490692138671875,
      "activations/layer3_attention_weight_max": 91.1496810913086,
      "activations/layer3_attention_weight_min": -92.58075714111328,
      "activations/layer4_attention_weight_max": 86.49176025390625,
      "activations/layer4_attention_weight_min": -85.30662536621094,
      "activations/layer5_attention_weight_max": 61.32540512084961,
      "activations/layer5_attention_weight_min": -68.52398681640625,
      "activations/layer6_attention_weight_max": 50.309688568115234,
      "activations/layer6_attention_weight_min": -50.77888488769531,
      "activations/layer7_attention_weight_max": 70.13212585449219,
      "activations/layer7_attention_weight_min": -63.88587188720703,
      "activations/layer8_attention_weight_max": 48.58831024169922,
      "activations/layer8_attention_weight_min": -49.66187286376953,
      "activations/layer9_attention_weight_max": 57.3874397277832,
      "activations/layer9_attention_weight_min": -46.23649215698242,
      "epoch": 18.99,
      "learning_rate": 2.780833333333333e-05,
      "loss": 2.7351,
      "step": 326850
    },
    {
      "activations/layer0_attention_weight_max": 14.902655601501465,
      "activations/layer0_attention_weight_min": -13.17607593536377,
      "activations/layer10_attention_weight_max": 34.8815803527832,
      "activations/layer10_attention_weight_min": -35.767173767089844,
      "activations/layer11_attention_weight_max": 33.08843231201172,
      "activations/layer11_attention_weight_min": -32.53035354614258,
      "activations/layer12_attention_weight_max": 19.584054946899414,
      "activations/layer12_attention_weight_min": -28.65645980834961,
      "activations/layer13_attention_weight_max": 37.49778747558594,
      "activations/layer13_attention_weight_min": -28.58768653869629,
      "activations/layer14_attention_weight_max": 33.99603271484375,
      "activations/layer14_attention_weight_min": -29.413110733032227,
      "activations/layer15_attention_weight_max": 31.353689193725586,
      "activations/layer15_attention_weight_min": -28.642215728759766,
      "activations/layer16_attention_weight_max": 33.59008026123047,
      "activations/layer16_attention_weight_min": -31.283544540405273,
      "activations/layer17_attention_weight_max": 48.933937072753906,
      "activations/layer17_attention_weight_min": -43.44099807739258,
      "activations/layer18_attention_weight_max": 47.90644454956055,
      "activations/layer18_attention_weight_min": -41.064720153808594,
      "activations/layer19_attention_weight_max": 26.82828140258789,
      "activations/layer19_attention_weight_min": -23.83774757385254,
      "activations/layer1_attention_weight_max": 15.828149795532227,
      "activations/layer1_attention_weight_min": -13.66582202911377,
      "activations/layer20_attention_weight_max": 24.967029571533203,
      "activations/layer20_attention_weight_min": -20.816343307495117,
      "activations/layer21_attention_weight_max": 39.43102264404297,
      "activations/layer21_attention_weight_min": -25.693370819091797,
      "activations/layer22_attention_weight_max": 29.532901763916016,
      "activations/layer22_attention_weight_min": -25.747543334960938,
      "activations/layer23_attention_weight_max": 37.97937774658203,
      "activations/layer23_attention_weight_min": -26.554832458496094,
      "activations/layer2_attention_weight_max": 32.40922546386719,
      "activations/layer2_attention_weight_min": -31.438411712646484,
      "activations/layer3_attention_weight_max": 83.95183563232422,
      "activations/layer3_attention_weight_min": -86.91177368164062,
      "activations/layer4_attention_weight_max": 87.26766204833984,
      "activations/layer4_attention_weight_min": -82.01708984375,
      "activations/layer5_attention_weight_max": 63.33024597167969,
      "activations/layer5_attention_weight_min": -67.65157318115234,
      "activations/layer6_attention_weight_max": 48.69984817504883,
      "activations/layer6_attention_weight_min": -50.55704879760742,
      "activations/layer7_attention_weight_max": 64.85724639892578,
      "activations/layer7_attention_weight_min": -61.80266189575195,
      "activations/layer8_attention_weight_max": 44.54034423828125,
      "activations/layer8_attention_weight_min": -47.743988037109375,
      "activations/layer9_attention_weight_max": 42.05418014526367,
      "activations/layer9_attention_weight_min": -45.281646728515625,
      "epoch": 18.99,
      "learning_rate": 2.7789393939393938e-05,
      "loss": 2.727,
      "step": 326900
    },
    {
      "activations/layer0_attention_weight_max": 15.021446228027344,
      "activations/layer0_attention_weight_min": -13.555194854736328,
      "activations/layer10_attention_weight_max": 32.36097717285156,
      "activations/layer10_attention_weight_min": -33.70336151123047,
      "activations/layer11_attention_weight_max": 33.45004653930664,
      "activations/layer11_attention_weight_min": -33.853599548339844,
      "activations/layer12_attention_weight_max": 19.060657501220703,
      "activations/layer12_attention_weight_min": -29.673154830932617,
      "activations/layer13_attention_weight_max": 33.35223388671875,
      "activations/layer13_attention_weight_min": -27.370256423950195,
      "activations/layer14_attention_weight_max": 34.11245346069336,
      "activations/layer14_attention_weight_min": -29.9694766998291,
      "activations/layer15_attention_weight_max": 34.324371337890625,
      "activations/layer15_attention_weight_min": -31.760114669799805,
      "activations/layer16_attention_weight_max": 33.393646240234375,
      "activations/layer16_attention_weight_min": -31.150724411010742,
      "activations/layer17_attention_weight_max": 50.3016242980957,
      "activations/layer17_attention_weight_min": -44.66400909423828,
      "activations/layer18_attention_weight_max": 45.24325942993164,
      "activations/layer18_attention_weight_min": -38.400428771972656,
      "activations/layer19_attention_weight_max": 25.073755264282227,
      "activations/layer19_attention_weight_min": -22.05267906188965,
      "activations/layer1_attention_weight_max": 15.749338150024414,
      "activations/layer1_attention_weight_min": -13.718716621398926,
      "activations/layer20_attention_weight_max": 23.039987564086914,
      "activations/layer20_attention_weight_min": -23.517894744873047,
      "activations/layer21_attention_weight_max": 37.833492279052734,
      "activations/layer21_attention_weight_min": -24.93589210510254,
      "activations/layer22_attention_weight_max": 27.20888328552246,
      "activations/layer22_attention_weight_min": -25.270484924316406,
      "activations/layer23_attention_weight_max": 36.890411376953125,
      "activations/layer23_attention_weight_min": -24.57815933227539,
      "activations/layer2_attention_weight_max": 33.831661224365234,
      "activations/layer2_attention_weight_min": -31.67803955078125,
      "activations/layer3_attention_weight_max": 89.9059829711914,
      "activations/layer3_attention_weight_min": -92.37651062011719,
      "activations/layer4_attention_weight_max": 83.36922454833984,
      "activations/layer4_attention_weight_min": -82.02485656738281,
      "activations/layer5_attention_weight_max": 61.70550537109375,
      "activations/layer5_attention_weight_min": -72.82289123535156,
      "activations/layer6_attention_weight_max": 50.14348602294922,
      "activations/layer6_attention_weight_min": -48.22458267211914,
      "activations/layer7_attention_weight_max": 65.2684555053711,
      "activations/layer7_attention_weight_min": -65.25572204589844,
      "activations/layer8_attention_weight_max": 44.354698181152344,
      "activations/layer8_attention_weight_min": -46.511844635009766,
      "activations/layer9_attention_weight_max": 43.96057891845703,
      "activations/layer9_attention_weight_min": -47.357749938964844,
      "epoch": 19.0,
      "learning_rate": 2.7770454545454543e-05,
      "loss": 2.7426,
      "step": 326950
    },
    {
      "activations/layer0_attention_weight_max": 15.219223976135254,
      "activations/layer0_attention_weight_min": -13.534849166870117,
      "activations/layer10_attention_weight_max": 33.918277740478516,
      "activations/layer10_attention_weight_min": -33.77863693237305,
      "activations/layer11_attention_weight_max": 32.26109313964844,
      "activations/layer11_attention_weight_min": -34.90751266479492,
      "activations/layer12_attention_weight_max": 17.938520431518555,
      "activations/layer12_attention_weight_min": -25.38616943359375,
      "activations/layer13_attention_weight_max": 35.31162643432617,
      "activations/layer13_attention_weight_min": -28.402849197387695,
      "activations/layer14_attention_weight_max": 35.647457122802734,
      "activations/layer14_attention_weight_min": -30.385324478149414,
      "activations/layer15_attention_weight_max": 35.86018371582031,
      "activations/layer15_attention_weight_min": -31.403507232666016,
      "activations/layer16_attention_weight_max": 33.605735778808594,
      "activations/layer16_attention_weight_min": -34.18675994873047,
      "activations/layer17_attention_weight_max": 50.56218338012695,
      "activations/layer17_attention_weight_min": -50.26185607910156,
      "activations/layer18_attention_weight_max": 50.84846115112305,
      "activations/layer18_attention_weight_min": -43.33961868286133,
      "activations/layer19_attention_weight_max": 22.431306838989258,
      "activations/layer19_attention_weight_min": -23.03212547302246,
      "activations/layer1_attention_weight_max": 15.208542823791504,
      "activations/layer1_attention_weight_min": -13.569314956665039,
      "activations/layer20_attention_weight_max": 23.005847930908203,
      "activations/layer20_attention_weight_min": -22.305034637451172,
      "activations/layer21_attention_weight_max": 40.2166633605957,
      "activations/layer21_attention_weight_min": -23.1442813873291,
      "activations/layer22_attention_weight_max": 30.18513298034668,
      "activations/layer22_attention_weight_min": -25.21701431274414,
      "activations/layer23_attention_weight_max": 37.39522933959961,
      "activations/layer23_attention_weight_min": -23.296199798583984,
      "activations/layer2_attention_weight_max": 31.580493927001953,
      "activations/layer2_attention_weight_min": -30.15959930419922,
      "activations/layer3_attention_weight_max": 87.11995697021484,
      "activations/layer3_attention_weight_min": -84.42764282226562,
      "activations/layer4_attention_weight_max": 82.23112487792969,
      "activations/layer4_attention_weight_min": -81.91004943847656,
      "activations/layer5_attention_weight_max": 66.47314453125,
      "activations/layer5_attention_weight_min": -70.18270874023438,
      "activations/layer6_attention_weight_max": 49.17146682739258,
      "activations/layer6_attention_weight_min": -49.78352737426758,
      "activations/layer7_attention_weight_max": 68.3679428100586,
      "activations/layer7_attention_weight_min": -63.53363037109375,
      "activations/layer8_attention_weight_max": 47.32117462158203,
      "activations/layer8_attention_weight_min": -48.27863311767578,
      "activations/layer9_attention_weight_max": 45.280879974365234,
      "activations/layer9_attention_weight_min": -43.96149444580078,
      "epoch": 19.0,
      "learning_rate": 2.775113636363636e-05,
      "loss": 2.7731,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.5749,
      "eval_samples_per_second": 500.762,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.5749,
      "eval_openwebtext_samples_per_second": 500.762,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_wikitext_loss": 2.921875,
      "eval_wikitext_ppl": 18.576084986220376,
      "eval_wikitext_runtime": 2.0071,
      "eval_wikitext_samples_per_second": 227.191,
      "step": 327000
    },
    {
      "epoch": 19.0,
      "eval_lambada_loss": 2.482421875,
      "eval_lambada_ppl": 11.970219716515004,
      "eval_lambada_runtime": 9.6309,
      "eval_lambada_samples_per_second": 505.56,
      "step": 327000
    },
    {
      "activations/layer0_attention_weight_max": 15.678491592407227,
      "activations/layer0_attention_weight_min": -13.771026611328125,
      "activations/layer10_attention_weight_max": 33.79698944091797,
      "activations/layer10_attention_weight_min": -34.4210319519043,
      "activations/layer11_attention_weight_max": 31.97987937927246,
      "activations/layer11_attention_weight_min": -33.50871658325195,
      "activations/layer12_attention_weight_max": 19.190868377685547,
      "activations/layer12_attention_weight_min": -22.46619987487793,
      "activations/layer13_attention_weight_max": 29.29892921447754,
      "activations/layer13_attention_weight_min": -27.35465431213379,
      "activations/layer14_attention_weight_max": 33.697486877441406,
      "activations/layer14_attention_weight_min": -32.8476448059082,
      "activations/layer15_attention_weight_max": 31.942014694213867,
      "activations/layer15_attention_weight_min": -31.430587768554688,
      "activations/layer16_attention_weight_max": 33.913055419921875,
      "activations/layer16_attention_weight_min": -34.57781219482422,
      "activations/layer17_attention_weight_max": 52.21821975708008,
      "activations/layer17_attention_weight_min": -48.63402557373047,
      "activations/layer18_attention_weight_max": 50.00786209106445,
      "activations/layer18_attention_weight_min": -47.88959884643555,
      "activations/layer19_attention_weight_max": 27.041738510131836,
      "activations/layer19_attention_weight_min": -29.39170265197754,
      "activations/layer1_attention_weight_max": 15.298309326171875,
      "activations/layer1_attention_weight_min": -14.306310653686523,
      "activations/layer20_attention_weight_max": 26.340551376342773,
      "activations/layer20_attention_weight_min": -23.032785415649414,
      "activations/layer21_attention_weight_max": 33.91679000854492,
      "activations/layer21_attention_weight_min": -22.55388832092285,
      "activations/layer22_attention_weight_max": 31.386308670043945,
      "activations/layer22_attention_weight_min": -26.76167869567871,
      "activations/layer23_attention_weight_max": 32.22714614868164,
      "activations/layer23_attention_weight_min": -26.137121200561523,
      "activations/layer2_attention_weight_max": 32.97644805908203,
      "activations/layer2_attention_weight_min": -30.109115600585938,
      "activations/layer3_attention_weight_max": 93.64093780517578,
      "activations/layer3_attention_weight_min": -90.16155242919922,
      "activations/layer4_attention_weight_max": 86.18965148925781,
      "activations/layer4_attention_weight_min": -83.69612884521484,
      "activations/layer5_attention_weight_max": 65.23005676269531,
      "activations/layer5_attention_weight_min": -70.01539611816406,
      "activations/layer6_attention_weight_max": 50.21329116821289,
      "activations/layer6_attention_weight_min": -50.692649841308594,
      "activations/layer7_attention_weight_max": 72.67987823486328,
      "activations/layer7_attention_weight_min": -65.51134490966797,
      "activations/layer8_attention_weight_max": 44.675758361816406,
      "activations/layer8_attention_weight_min": -48.51036071777344,
      "activations/layer9_attention_weight_max": 41.041385650634766,
      "activations/layer9_attention_weight_min": -45.59767150878906,
      "epoch": 19.0,
      "learning_rate": 2.7732196969696966e-05,
      "loss": 2.712,
      "step": 327050
    },
    {
      "activations/layer0_attention_weight_max": 16.53000259399414,
      "activations/layer0_attention_weight_min": -13.949568748474121,
      "activations/layer10_attention_weight_max": 35.56895446777344,
      "activations/layer10_attention_weight_min": -36.41109085083008,
      "activations/layer11_attention_weight_max": 32.619606018066406,
      "activations/layer11_attention_weight_min": -35.27809143066406,
      "activations/layer12_attention_weight_max": 20.122114181518555,
      "activations/layer12_attention_weight_min": -24.611913681030273,
      "activations/layer13_attention_weight_max": 46.88800811767578,
      "activations/layer13_attention_weight_min": -30.43709373474121,
      "activations/layer14_attention_weight_max": 41.154056549072266,
      "activations/layer14_attention_weight_min": -34.42147445678711,
      "activations/layer15_attention_weight_max": 40.6949577331543,
      "activations/layer15_attention_weight_min": -32.443817138671875,
      "activations/layer16_attention_weight_max": 39.30550003051758,
      "activations/layer16_attention_weight_min": -37.95335006713867,
      "activations/layer17_attention_weight_max": 59.19346237182617,
      "activations/layer17_attention_weight_min": -51.18368911743164,
      "activations/layer18_attention_weight_max": 49.09186553955078,
      "activations/layer18_attention_weight_min": -45.44112014770508,
      "activations/layer19_attention_weight_max": 23.234960556030273,
      "activations/layer19_attention_weight_min": -22.365514755249023,
      "activations/layer1_attention_weight_max": 14.857120513916016,
      "activations/layer1_attention_weight_min": -14.121074676513672,
      "activations/layer20_attention_weight_max": 23.72739601135254,
      "activations/layer20_attention_weight_min": -21.47956657409668,
      "activations/layer21_attention_weight_max": 48.82744216918945,
      "activations/layer21_attention_weight_min": -21.972854614257812,
      "activations/layer22_attention_weight_max": 32.28886413574219,
      "activations/layer22_attention_weight_min": -27.553173065185547,
      "activations/layer23_attention_weight_max": 38.32522964477539,
      "activations/layer23_attention_weight_min": -23.983318328857422,
      "activations/layer2_attention_weight_max": 31.878944396972656,
      "activations/layer2_attention_weight_min": -29.540775299072266,
      "activations/layer3_attention_weight_max": 85.9542236328125,
      "activations/layer3_attention_weight_min": -85.62142181396484,
      "activations/layer4_attention_weight_max": 88.16353607177734,
      "activations/layer4_attention_weight_min": -84.97258758544922,
      "activations/layer5_attention_weight_max": 63.149932861328125,
      "activations/layer5_attention_weight_min": -71.13201904296875,
      "activations/layer6_attention_weight_max": 48.97855758666992,
      "activations/layer6_attention_weight_min": -50.65620803833008,
      "activations/layer7_attention_weight_max": 64.38680267333984,
      "activations/layer7_attention_weight_min": -63.712127685546875,
      "activations/layer8_attention_weight_max": 44.6078987121582,
      "activations/layer8_attention_weight_min": -50.16945266723633,
      "activations/layer9_attention_weight_max": 42.14315414428711,
      "activations/layer9_attention_weight_min": -44.127750396728516,
      "epoch": 19.01,
      "learning_rate": 2.7713257575757575e-05,
      "loss": 2.7311,
      "step": 327100
    },
    {
      "activations/layer0_attention_weight_max": 14.991621971130371,
      "activations/layer0_attention_weight_min": -13.704227447509766,
      "activations/layer10_attention_weight_max": 39.96407699584961,
      "activations/layer10_attention_weight_min": -38.875518798828125,
      "activations/layer11_attention_weight_max": 35.679527282714844,
      "activations/layer11_attention_weight_min": -33.455810546875,
      "activations/layer12_attention_weight_max": 20.096389770507812,
      "activations/layer12_attention_weight_min": -27.228025436401367,
      "activations/layer13_attention_weight_max": 36.980979919433594,
      "activations/layer13_attention_weight_min": -28.05937385559082,
      "activations/layer14_attention_weight_max": 40.01921463012695,
      "activations/layer14_attention_weight_min": -31.964763641357422,
      "activations/layer15_attention_weight_max": 35.38181686401367,
      "activations/layer15_attention_weight_min": -29.726673126220703,
      "activations/layer16_attention_weight_max": 34.12240219116211,
      "activations/layer16_attention_weight_min": -33.53346252441406,
      "activations/layer17_attention_weight_max": 52.04319763183594,
      "activations/layer17_attention_weight_min": -46.94722366333008,
      "activations/layer18_attention_weight_max": 49.01411819458008,
      "activations/layer18_attention_weight_min": -41.114723205566406,
      "activations/layer19_attention_weight_max": 23.367374420166016,
      "activations/layer19_attention_weight_min": -21.002849578857422,
      "activations/layer1_attention_weight_max": 16.890758514404297,
      "activations/layer1_attention_weight_min": -13.568614959716797,
      "activations/layer20_attention_weight_max": 23.276174545288086,
      "activations/layer20_attention_weight_min": -21.93356704711914,
      "activations/layer21_attention_weight_max": 38.9559211730957,
      "activations/layer21_attention_weight_min": -23.150209426879883,
      "activations/layer22_attention_weight_max": 31.567699432373047,
      "activations/layer22_attention_weight_min": -24.375225067138672,
      "activations/layer23_attention_weight_max": 39.231693267822266,
      "activations/layer23_attention_weight_min": -25.803081512451172,
      "activations/layer2_attention_weight_max": 31.07024383544922,
      "activations/layer2_attention_weight_min": -30.441389083862305,
      "activations/layer3_attention_weight_max": 91.3919906616211,
      "activations/layer3_attention_weight_min": -88.23699951171875,
      "activations/layer4_attention_weight_max": 85.39861297607422,
      "activations/layer4_attention_weight_min": -78.7419662475586,
      "activations/layer5_attention_weight_max": 61.6295280456543,
      "activations/layer5_attention_weight_min": -69.03396606445312,
      "activations/layer6_attention_weight_max": 50.78719711303711,
      "activations/layer6_attention_weight_min": -51.61845779418945,
      "activations/layer7_attention_weight_max": 65.07687377929688,
      "activations/layer7_attention_weight_min": -61.95662307739258,
      "activations/layer8_attention_weight_max": 47.007530212402344,
      "activations/layer8_attention_weight_min": -51.3408203125,
      "activations/layer9_attention_weight_max": 45.245750427246094,
      "activations/layer9_attention_weight_min": -46.4281120300293,
      "epoch": 19.01,
      "learning_rate": 2.7694318181818183e-05,
      "loss": 2.7135,
      "step": 327150
    },
    {
      "activations/layer0_attention_weight_max": 15.134578704833984,
      "activations/layer0_attention_weight_min": -13.575448036193848,
      "activations/layer10_attention_weight_max": 36.1743049621582,
      "activations/layer10_attention_weight_min": -35.414363861083984,
      "activations/layer11_attention_weight_max": 31.728736877441406,
      "activations/layer11_attention_weight_min": -33.12725830078125,
      "activations/layer12_attention_weight_max": 16.987810134887695,
      "activations/layer12_attention_weight_min": -25.392715454101562,
      "activations/layer13_attention_weight_max": 33.589073181152344,
      "activations/layer13_attention_weight_min": -27.68435287475586,
      "activations/layer14_attention_weight_max": 33.518978118896484,
      "activations/layer14_attention_weight_min": -28.905847549438477,
      "activations/layer15_attention_weight_max": 32.84938049316406,
      "activations/layer15_attention_weight_min": -29.4031925201416,
      "activations/layer16_attention_weight_max": 33.70710754394531,
      "activations/layer16_attention_weight_min": -31.365856170654297,
      "activations/layer17_attention_weight_max": 54.608726501464844,
      "activations/layer17_attention_weight_min": -43.00300598144531,
      "activations/layer18_attention_weight_max": 43.00642013549805,
      "activations/layer18_attention_weight_min": -38.54975509643555,
      "activations/layer19_attention_weight_max": 22.938274383544922,
      "activations/layer19_attention_weight_min": -21.20758628845215,
      "activations/layer1_attention_weight_max": 15.515213966369629,
      "activations/layer1_attention_weight_min": -13.873041152954102,
      "activations/layer20_attention_weight_max": 22.68757438659668,
      "activations/layer20_attention_weight_min": -19.909629821777344,
      "activations/layer21_attention_weight_max": 37.08007049560547,
      "activations/layer21_attention_weight_min": -21.82376480102539,
      "activations/layer22_attention_weight_max": 27.11198616027832,
      "activations/layer22_attention_weight_min": -26.14588165283203,
      "activations/layer23_attention_weight_max": 34.27585983276367,
      "activations/layer23_attention_weight_min": -23.14944076538086,
      "activations/layer2_attention_weight_max": 32.035396575927734,
      "activations/layer2_attention_weight_min": -30.793292999267578,
      "activations/layer3_attention_weight_max": 87.6021728515625,
      "activations/layer3_attention_weight_min": -86.18766021728516,
      "activations/layer4_attention_weight_max": 83.07424926757812,
      "activations/layer4_attention_weight_min": -85.90670013427734,
      "activations/layer5_attention_weight_max": 59.52048873901367,
      "activations/layer5_attention_weight_min": -72.96383666992188,
      "activations/layer6_attention_weight_max": 48.228965759277344,
      "activations/layer6_attention_weight_min": -50.59284210205078,
      "activations/layer7_attention_weight_max": 64.48332977294922,
      "activations/layer7_attention_weight_min": -65.16262817382812,
      "activations/layer8_attention_weight_max": 45.83217239379883,
      "activations/layer8_attention_weight_min": -49.67852783203125,
      "activations/layer9_attention_weight_max": 40.89218521118164,
      "activations/layer9_attention_weight_min": -45.765235900878906,
      "epoch": 19.01,
      "learning_rate": 2.7675378787878784e-05,
      "loss": 2.7198,
      "step": 327200
    },
    {
      "activations/layer0_attention_weight_max": 16.220455169677734,
      "activations/layer0_attention_weight_min": -13.779001235961914,
      "activations/layer10_attention_weight_max": 38.851654052734375,
      "activations/layer10_attention_weight_min": -38.34353256225586,
      "activations/layer11_attention_weight_max": 35.623130798339844,
      "activations/layer11_attention_weight_min": -38.69243240356445,
      "activations/layer12_attention_weight_max": 20.39272689819336,
      "activations/layer12_attention_weight_min": -23.526798248291016,
      "activations/layer13_attention_weight_max": 44.434165954589844,
      "activations/layer13_attention_weight_min": -26.706127166748047,
      "activations/layer14_attention_weight_max": 39.05142593383789,
      "activations/layer14_attention_weight_min": -31.204601287841797,
      "activations/layer15_attention_weight_max": 35.47474670410156,
      "activations/layer15_attention_weight_min": -31.9940128326416,
      "activations/layer16_attention_weight_max": 34.638885498046875,
      "activations/layer16_attention_weight_min": -32.82020950317383,
      "activations/layer17_attention_weight_max": 57.7802619934082,
      "activations/layer17_attention_weight_min": -46.52203369140625,
      "activations/layer18_attention_weight_max": 47.851627349853516,
      "activations/layer18_attention_weight_min": -40.16057586669922,
      "activations/layer19_attention_weight_max": 26.228179931640625,
      "activations/layer19_attention_weight_min": -24.390546798706055,
      "activations/layer1_attention_weight_max": 15.884783744812012,
      "activations/layer1_attention_weight_min": -14.19363021850586,
      "activations/layer20_attention_weight_max": 23.684804916381836,
      "activations/layer20_attention_weight_min": -19.467065811157227,
      "activations/layer21_attention_weight_max": 32.62065505981445,
      "activations/layer21_attention_weight_min": -18.658878326416016,
      "activations/layer22_attention_weight_max": 30.02753448486328,
      "activations/layer22_attention_weight_min": -24.370277404785156,
      "activations/layer23_attention_weight_max": 41.550048828125,
      "activations/layer23_attention_weight_min": -22.008039474487305,
      "activations/layer2_attention_weight_max": 32.65953826904297,
      "activations/layer2_attention_weight_min": -31.818716049194336,
      "activations/layer3_attention_weight_max": 85.76849365234375,
      "activations/layer3_attention_weight_min": -89.1146469116211,
      "activations/layer4_attention_weight_max": 87.31261444091797,
      "activations/layer4_attention_weight_min": -86.88211822509766,
      "activations/layer5_attention_weight_max": 65.10499572753906,
      "activations/layer5_attention_weight_min": -73.4775390625,
      "activations/layer6_attention_weight_max": 51.345401763916016,
      "activations/layer6_attention_weight_min": -51.84275817871094,
      "activations/layer7_attention_weight_max": 71.95703125,
      "activations/layer7_attention_weight_min": -65.81542205810547,
      "activations/layer8_attention_weight_max": 48.68004608154297,
      "activations/layer8_attention_weight_min": -51.5712890625,
      "activations/layer9_attention_weight_max": 53.312068939208984,
      "activations/layer9_attention_weight_min": -49.22091293334961,
      "epoch": 19.02,
      "learning_rate": 2.7656439393939393e-05,
      "loss": 2.7195,
      "step": 327250
    },
    {
      "activations/layer0_attention_weight_max": 15.240680694580078,
      "activations/layer0_attention_weight_min": -13.6668701171875,
      "activations/layer10_attention_weight_max": 34.502132415771484,
      "activations/layer10_attention_weight_min": -38.21004104614258,
      "activations/layer11_attention_weight_max": 31.371257781982422,
      "activations/layer11_attention_weight_min": -33.971923828125,
      "activations/layer12_attention_weight_max": 17.75613784790039,
      "activations/layer12_attention_weight_min": -24.86614227294922,
      "activations/layer13_attention_weight_max": 38.69389343261719,
      "activations/layer13_attention_weight_min": -27.781526565551758,
      "activations/layer14_attention_weight_max": 33.34229278564453,
      "activations/layer14_attention_weight_min": -29.246204376220703,
      "activations/layer15_attention_weight_max": 31.876949310302734,
      "activations/layer15_attention_weight_min": -28.67134666442871,
      "activations/layer16_attention_weight_max": 33.86574935913086,
      "activations/layer16_attention_weight_min": -31.275907516479492,
      "activations/layer17_attention_weight_max": 48.145355224609375,
      "activations/layer17_attention_weight_min": -42.59235763549805,
      "activations/layer18_attention_weight_max": 45.95204544067383,
      "activations/layer18_attention_weight_min": -37.777366638183594,
      "activations/layer19_attention_weight_max": 22.69606590270996,
      "activations/layer19_attention_weight_min": -21.712629318237305,
      "activations/layer1_attention_weight_max": 14.769636154174805,
      "activations/layer1_attention_weight_min": -13.615232467651367,
      "activations/layer20_attention_weight_max": 21.019798278808594,
      "activations/layer20_attention_weight_min": -22.82932472229004,
      "activations/layer21_attention_weight_max": 33.56776809692383,
      "activations/layer21_attention_weight_min": -25.470458984375,
      "activations/layer22_attention_weight_max": 27.761987686157227,
      "activations/layer22_attention_weight_min": -26.62413215637207,
      "activations/layer23_attention_weight_max": 37.19478225708008,
      "activations/layer23_attention_weight_min": -27.870861053466797,
      "activations/layer2_attention_weight_max": 31.62500762939453,
      "activations/layer2_attention_weight_min": -30.173185348510742,
      "activations/layer3_attention_weight_max": 83.61300659179688,
      "activations/layer3_attention_weight_min": -86.45254516601562,
      "activations/layer4_attention_weight_max": 80.86910247802734,
      "activations/layer4_attention_weight_min": -85.19002532958984,
      "activations/layer5_attention_weight_max": 60.43702697753906,
      "activations/layer5_attention_weight_min": -71.70379638671875,
      "activations/layer6_attention_weight_max": 48.37128448486328,
      "activations/layer6_attention_weight_min": -51.11153793334961,
      "activations/layer7_attention_weight_max": 67.42843627929688,
      "activations/layer7_attention_weight_min": -68.1907958984375,
      "activations/layer8_attention_weight_max": 45.94136428833008,
      "activations/layer8_attention_weight_min": -47.78487014770508,
      "activations/layer9_attention_weight_max": 44.89494705200195,
      "activations/layer9_attention_weight_min": -48.36811065673828,
      "epoch": 19.02,
      "learning_rate": 2.7637499999999998e-05,
      "loss": 2.7265,
      "step": 327300
    },
    {
      "activations/layer0_attention_weight_max": 15.455082893371582,
      "activations/layer0_attention_weight_min": -13.626380920410156,
      "activations/layer10_attention_weight_max": 39.2828254699707,
      "activations/layer10_attention_weight_min": -37.9761848449707,
      "activations/layer11_attention_weight_max": 36.214229583740234,
      "activations/layer11_attention_weight_min": -38.741355895996094,
      "activations/layer12_attention_weight_max": 19.16225242614746,
      "activations/layer12_attention_weight_min": -26.654624938964844,
      "activations/layer13_attention_weight_max": 31.265731811523438,
      "activations/layer13_attention_weight_min": -27.517595291137695,
      "activations/layer14_attention_weight_max": 34.19465637207031,
      "activations/layer14_attention_weight_min": -30.473665237426758,
      "activations/layer15_attention_weight_max": 31.942317962646484,
      "activations/layer15_attention_weight_min": -30.89681625366211,
      "activations/layer16_attention_weight_max": 33.32780456542969,
      "activations/layer16_attention_weight_min": -32.63792419433594,
      "activations/layer17_attention_weight_max": 52.77699661254883,
      "activations/layer17_attention_weight_min": -48.13218307495117,
      "activations/layer18_attention_weight_max": 46.35763931274414,
      "activations/layer18_attention_weight_min": -43.15805435180664,
      "activations/layer19_attention_weight_max": 23.59254264831543,
      "activations/layer19_attention_weight_min": -24.851951599121094,
      "activations/layer1_attention_weight_max": 15.329985618591309,
      "activations/layer1_attention_weight_min": -13.775569915771484,
      "activations/layer20_attention_weight_max": 23.348268508911133,
      "activations/layer20_attention_weight_min": -22.187707901000977,
      "activations/layer21_attention_weight_max": 32.08305358886719,
      "activations/layer21_attention_weight_min": -29.31346893310547,
      "activations/layer22_attention_weight_max": 27.007667541503906,
      "activations/layer22_attention_weight_min": -25.55002212524414,
      "activations/layer23_attention_weight_max": 33.7811164855957,
      "activations/layer23_attention_weight_min": -25.502513885498047,
      "activations/layer2_attention_weight_max": 31.118282318115234,
      "activations/layer2_attention_weight_min": -30.040130615234375,
      "activations/layer3_attention_weight_max": 84.85491943359375,
      "activations/layer3_attention_weight_min": -85.72357940673828,
      "activations/layer4_attention_weight_max": 80.65966796875,
      "activations/layer4_attention_weight_min": -81.50077056884766,
      "activations/layer5_attention_weight_max": 65.34480285644531,
      "activations/layer5_attention_weight_min": -72.13665771484375,
      "activations/layer6_attention_weight_max": 50.70722579956055,
      "activations/layer6_attention_weight_min": -51.56258010864258,
      "activations/layer7_attention_weight_max": 69.05008697509766,
      "activations/layer7_attention_weight_min": -62.7887077331543,
      "activations/layer8_attention_weight_max": 48.140384674072266,
      "activations/layer8_attention_weight_min": -49.71708679199219,
      "activations/layer9_attention_weight_max": 47.710243225097656,
      "activations/layer9_attention_weight_min": -48.47007369995117,
      "epoch": 19.02,
      "learning_rate": 2.7618939393939392e-05,
      "loss": 2.7291,
      "step": 327350
    },
    {
      "activations/layer0_attention_weight_max": 15.543176651000977,
      "activations/layer0_attention_weight_min": -13.409745216369629,
      "activations/layer10_attention_weight_max": 38.44325256347656,
      "activations/layer10_attention_weight_min": -38.59026336669922,
      "activations/layer11_attention_weight_max": 36.14350128173828,
      "activations/layer11_attention_weight_min": -37.68373107910156,
      "activations/layer12_attention_weight_max": 23.08611488342285,
      "activations/layer12_attention_weight_min": -27.02655792236328,
      "activations/layer13_attention_weight_max": 37.55868148803711,
      "activations/layer13_attention_weight_min": -30.84957504272461,
      "activations/layer14_attention_weight_max": 38.59992980957031,
      "activations/layer14_attention_weight_min": -33.27703094482422,
      "activations/layer15_attention_weight_max": 34.809776306152344,
      "activations/layer15_attention_weight_min": -32.27561950683594,
      "activations/layer16_attention_weight_max": 38.0192756652832,
      "activations/layer16_attention_weight_min": -34.72540283203125,
      "activations/layer17_attention_weight_max": 59.66691970825195,
      "activations/layer17_attention_weight_min": -49.541202545166016,
      "activations/layer18_attention_weight_max": 57.31935501098633,
      "activations/layer18_attention_weight_min": -46.480167388916016,
      "activations/layer19_attention_weight_max": 26.6494140625,
      "activations/layer19_attention_weight_min": -25.726964950561523,
      "activations/layer1_attention_weight_max": 15.02772331237793,
      "activations/layer1_attention_weight_min": -12.962604522705078,
      "activations/layer20_attention_weight_max": 27.124820709228516,
      "activations/layer20_attention_weight_min": -22.41448974609375,
      "activations/layer21_attention_weight_max": 43.0081787109375,
      "activations/layer21_attention_weight_min": -21.979568481445312,
      "activations/layer22_attention_weight_max": 33.13338088989258,
      "activations/layer22_attention_weight_min": -27.15392303466797,
      "activations/layer23_attention_weight_max": 38.535484313964844,
      "activations/layer23_attention_weight_min": -24.67262077331543,
      "activations/layer2_attention_weight_max": 32.80219268798828,
      "activations/layer2_attention_weight_min": -30.51805877685547,
      "activations/layer3_attention_weight_max": 89.97129821777344,
      "activations/layer3_attention_weight_min": -87.44054412841797,
      "activations/layer4_attention_weight_max": 85.85195922851562,
      "activations/layer4_attention_weight_min": -83.18331146240234,
      "activations/layer5_attention_weight_max": 63.91033935546875,
      "activations/layer5_attention_weight_min": -70.53282165527344,
      "activations/layer6_attention_weight_max": 54.870792388916016,
      "activations/layer6_attention_weight_min": -52.44076156616211,
      "activations/layer7_attention_weight_max": 70.70037078857422,
      "activations/layer7_attention_weight_min": -68.97832489013672,
      "activations/layer8_attention_weight_max": 53.009971618652344,
      "activations/layer8_attention_weight_min": -52.936500549316406,
      "activations/layer9_attention_weight_max": 48.17690658569336,
      "activations/layer9_attention_weight_min": -49.67375946044922,
      "epoch": 19.02,
      "learning_rate": 2.7599999999999997e-05,
      "loss": 2.7389,
      "step": 327400
    },
    {
      "activations/layer0_attention_weight_max": 16.532865524291992,
      "activations/layer0_attention_weight_min": -13.593789100646973,
      "activations/layer10_attention_weight_max": 35.62915802001953,
      "activations/layer10_attention_weight_min": -34.166561126708984,
      "activations/layer11_attention_weight_max": 31.34785270690918,
      "activations/layer11_attention_weight_min": -33.01782989501953,
      "activations/layer12_attention_weight_max": 17.180309295654297,
      "activations/layer12_attention_weight_min": -25.39417266845703,
      "activations/layer13_attention_weight_max": 29.779048919677734,
      "activations/layer13_attention_weight_min": -26.257545471191406,
      "activations/layer14_attention_weight_max": 31.74427032470703,
      "activations/layer14_attention_weight_min": -29.026418685913086,
      "activations/layer15_attention_weight_max": 30.281482696533203,
      "activations/layer15_attention_weight_min": -28.530866622924805,
      "activations/layer16_attention_weight_max": 33.32320785522461,
      "activations/layer16_attention_weight_min": -31.319351196289062,
      "activations/layer17_attention_weight_max": 47.94096374511719,
      "activations/layer17_attention_weight_min": -42.036033630371094,
      "activations/layer18_attention_weight_max": 47.98491287231445,
      "activations/layer18_attention_weight_min": -36.983985900878906,
      "activations/layer19_attention_weight_max": 23.246177673339844,
      "activations/layer19_attention_weight_min": -22.380258560180664,
      "activations/layer1_attention_weight_max": 16.241891860961914,
      "activations/layer1_attention_weight_min": -14.352510452270508,
      "activations/layer20_attention_weight_max": 20.29517936706543,
      "activations/layer20_attention_weight_min": -20.864927291870117,
      "activations/layer21_attention_weight_max": 33.0063591003418,
      "activations/layer21_attention_weight_min": -20.713815689086914,
      "activations/layer22_attention_weight_max": 25.581241607666016,
      "activations/layer22_attention_weight_min": -23.42479705810547,
      "activations/layer23_attention_weight_max": 33.322853088378906,
      "activations/layer23_attention_weight_min": -24.149883270263672,
      "activations/layer2_attention_weight_max": 31.92870330810547,
      "activations/layer2_attention_weight_min": -29.560422897338867,
      "activations/layer3_attention_weight_max": 85.78497314453125,
      "activations/layer3_attention_weight_min": -82.13539123535156,
      "activations/layer4_attention_weight_max": 84.910888671875,
      "activations/layer4_attention_weight_min": -80.98453521728516,
      "activations/layer5_attention_weight_max": 64.03014373779297,
      "activations/layer5_attention_weight_min": -69.27204895019531,
      "activations/layer6_attention_weight_max": 49.081886291503906,
      "activations/layer6_attention_weight_min": -48.30026626586914,
      "activations/layer7_attention_weight_max": 64.78034973144531,
      "activations/layer7_attention_weight_min": -60.40436553955078,
      "activations/layer8_attention_weight_max": 43.4325065612793,
      "activations/layer8_attention_weight_min": -46.7706184387207,
      "activations/layer9_attention_weight_max": 42.44740295410156,
      "activations/layer9_attention_weight_min": -45.46549606323242,
      "epoch": 19.03,
      "learning_rate": 2.7581060606060605e-05,
      "loss": 2.7237,
      "step": 327450
    },
    {
      "activations/layer0_attention_weight_max": 14.761938095092773,
      "activations/layer0_attention_weight_min": -13.630027770996094,
      "activations/layer10_attention_weight_max": 33.504493713378906,
      "activations/layer10_attention_weight_min": -35.05113220214844,
      "activations/layer11_attention_weight_max": 31.64206314086914,
      "activations/layer11_attention_weight_min": -33.757789611816406,
      "activations/layer12_attention_weight_max": 20.460193634033203,
      "activations/layer12_attention_weight_min": -25.758100509643555,
      "activations/layer13_attention_weight_max": 32.12507629394531,
      "activations/layer13_attention_weight_min": -30.269466400146484,
      "activations/layer14_attention_weight_max": 36.1248893737793,
      "activations/layer14_attention_weight_min": -30.380544662475586,
      "activations/layer15_attention_weight_max": 34.28167724609375,
      "activations/layer15_attention_weight_min": -30.19477653503418,
      "activations/layer16_attention_weight_max": 35.40585708618164,
      "activations/layer16_attention_weight_min": -34.8347282409668,
      "activations/layer17_attention_weight_max": 54.74689483642578,
      "activations/layer17_attention_weight_min": -44.57396697998047,
      "activations/layer18_attention_weight_max": 51.37407684326172,
      "activations/layer18_attention_weight_min": -41.3282356262207,
      "activations/layer19_attention_weight_max": 29.543848037719727,
      "activations/layer19_attention_weight_min": -26.8201904296875,
      "activations/layer1_attention_weight_max": 15.759247779846191,
      "activations/layer1_attention_weight_min": -13.554667472839355,
      "activations/layer20_attention_weight_max": 26.276309967041016,
      "activations/layer20_attention_weight_min": -24.584705352783203,
      "activations/layer21_attention_weight_max": 39.65635681152344,
      "activations/layer21_attention_weight_min": -26.05229949951172,
      "activations/layer22_attention_weight_max": 34.49707794189453,
      "activations/layer22_attention_weight_min": -28.073577880859375,
      "activations/layer23_attention_weight_max": 38.640323638916016,
      "activations/layer23_attention_weight_min": -26.50514793395996,
      "activations/layer2_attention_weight_max": 32.24085998535156,
      "activations/layer2_attention_weight_min": -30.87770652770996,
      "activations/layer3_attention_weight_max": 85.98111724853516,
      "activations/layer3_attention_weight_min": -83.49214935302734,
      "activations/layer4_attention_weight_max": 85.05721282958984,
      "activations/layer4_attention_weight_min": -82.13955688476562,
      "activations/layer5_attention_weight_max": 64.26203155517578,
      "activations/layer5_attention_weight_min": -74.32620239257812,
      "activations/layer6_attention_weight_max": 48.75087356567383,
      "activations/layer6_attention_weight_min": -50.65751647949219,
      "activations/layer7_attention_weight_max": 64.48795318603516,
      "activations/layer7_attention_weight_min": -63.21344757080078,
      "activations/layer8_attention_weight_max": 44.7558708190918,
      "activations/layer8_attention_weight_min": -46.831268310546875,
      "activations/layer9_attention_weight_max": 43.107093811035156,
      "activations/layer9_attention_weight_min": -47.91946029663086,
      "epoch": 19.03,
      "learning_rate": 2.756212121212121e-05,
      "loss": 2.7086,
      "step": 327500
    },
    {
      "activations/layer0_attention_weight_max": 15.73015308380127,
      "activations/layer0_attention_weight_min": -13.748563766479492,
      "activations/layer10_attention_weight_max": 33.22478103637695,
      "activations/layer10_attention_weight_min": -34.48176193237305,
      "activations/layer11_attention_weight_max": 33.58634948730469,
      "activations/layer11_attention_weight_min": -32.818634033203125,
      "activations/layer12_attention_weight_max": 19.913976669311523,
      "activations/layer12_attention_weight_min": -27.72945785522461,
      "activations/layer13_attention_weight_max": 34.071449279785156,
      "activations/layer13_attention_weight_min": -28.83293914794922,
      "activations/layer14_attention_weight_max": 32.888668060302734,
      "activations/layer14_attention_weight_min": -30.766712188720703,
      "activations/layer15_attention_weight_max": 33.079185485839844,
      "activations/layer15_attention_weight_min": -30.313093185424805,
      "activations/layer16_attention_weight_max": 34.70905303955078,
      "activations/layer16_attention_weight_min": -32.31177520751953,
      "activations/layer17_attention_weight_max": 49.61623001098633,
      "activations/layer17_attention_weight_min": -43.88606643676758,
      "activations/layer18_attention_weight_max": 48.79232406616211,
      "activations/layer18_attention_weight_min": -41.39725112915039,
      "activations/layer19_attention_weight_max": 25.668432235717773,
      "activations/layer19_attention_weight_min": -24.466421127319336,
      "activations/layer1_attention_weight_max": 15.32432746887207,
      "activations/layer1_attention_weight_min": -14.017912864685059,
      "activations/layer20_attention_weight_max": 23.364355087280273,
      "activations/layer20_attention_weight_min": -23.118467330932617,
      "activations/layer21_attention_weight_max": 36.037940979003906,
      "activations/layer21_attention_weight_min": -25.534610748291016,
      "activations/layer22_attention_weight_max": 29.8173828125,
      "activations/layer22_attention_weight_min": -28.443222045898438,
      "activations/layer23_attention_weight_max": 39.92183303833008,
      "activations/layer23_attention_weight_min": -23.211332321166992,
      "activations/layer2_attention_weight_max": 32.280784606933594,
      "activations/layer2_attention_weight_min": -28.976356506347656,
      "activations/layer3_attention_weight_max": 85.01683807373047,
      "activations/layer3_attention_weight_min": -83.46395111083984,
      "activations/layer4_attention_weight_max": 81.5147933959961,
      "activations/layer4_attention_weight_min": -80.97434997558594,
      "activations/layer5_attention_weight_max": 61.27873992919922,
      "activations/layer5_attention_weight_min": -69.68885040283203,
      "activations/layer6_attention_weight_max": 47.43178939819336,
      "activations/layer6_attention_weight_min": -50.002464294433594,
      "activations/layer7_attention_weight_max": 64.32456970214844,
      "activations/layer7_attention_weight_min": -63.694583892822266,
      "activations/layer8_attention_weight_max": 46.17220687866211,
      "activations/layer8_attention_weight_min": -49.07932662963867,
      "activations/layer9_attention_weight_max": 41.973270416259766,
      "activations/layer9_attention_weight_min": -45.48760223388672,
      "epoch": 19.03,
      "learning_rate": 2.7543181818181818e-05,
      "loss": 2.7219,
      "step": 327550
    },
    {
      "activations/layer0_attention_weight_max": 15.140360832214355,
      "activations/layer0_attention_weight_min": -13.587759017944336,
      "activations/layer10_attention_weight_max": 36.479549407958984,
      "activations/layer10_attention_weight_min": -40.809879302978516,
      "activations/layer11_attention_weight_max": 33.75694274902344,
      "activations/layer11_attention_weight_min": -34.77552795410156,
      "activations/layer12_attention_weight_max": 18.275577545166016,
      "activations/layer12_attention_weight_min": -36.1184196472168,
      "activations/layer13_attention_weight_max": 31.9023380279541,
      "activations/layer13_attention_weight_min": -28.863872528076172,
      "activations/layer14_attention_weight_max": 35.88809585571289,
      "activations/layer14_attention_weight_min": -31.746944427490234,
      "activations/layer15_attention_weight_max": 32.99691390991211,
      "activations/layer15_attention_weight_min": -32.17457962036133,
      "activations/layer16_attention_weight_max": 33.46516036987305,
      "activations/layer16_attention_weight_min": -32.727848052978516,
      "activations/layer17_attention_weight_max": 52.59127426147461,
      "activations/layer17_attention_weight_min": -45.581111907958984,
      "activations/layer18_attention_weight_max": 48.67649841308594,
      "activations/layer18_attention_weight_min": -41.10831832885742,
      "activations/layer19_attention_weight_max": 24.90230941772461,
      "activations/layer19_attention_weight_min": -22.147541046142578,
      "activations/layer1_attention_weight_max": 15.410052299499512,
      "activations/layer1_attention_weight_min": -13.767520904541016,
      "activations/layer20_attention_weight_max": 24.73039436340332,
      "activations/layer20_attention_weight_min": -21.178560256958008,
      "activations/layer21_attention_weight_max": 36.701717376708984,
      "activations/layer21_attention_weight_min": -20.717395782470703,
      "activations/layer22_attention_weight_max": 30.9644775390625,
      "activations/layer22_attention_weight_min": -26.987926483154297,
      "activations/layer23_attention_weight_max": 44.73341369628906,
      "activations/layer23_attention_weight_min": -24.300283432006836,
      "activations/layer2_attention_weight_max": 31.889301300048828,
      "activations/layer2_attention_weight_min": -30.14710807800293,
      "activations/layer3_attention_weight_max": 84.39852142333984,
      "activations/layer3_attention_weight_min": -84.06413269042969,
      "activations/layer4_attention_weight_max": 83.69689178466797,
      "activations/layer4_attention_weight_min": -81.5832290649414,
      "activations/layer5_attention_weight_max": 65.09517669677734,
      "activations/layer5_attention_weight_min": -68.31008911132812,
      "activations/layer6_attention_weight_max": 51.60523986816406,
      "activations/layer6_attention_weight_min": -49.48672866821289,
      "activations/layer7_attention_weight_max": 68.37939453125,
      "activations/layer7_attention_weight_min": -63.346153259277344,
      "activations/layer8_attention_weight_max": 46.60990524291992,
      "activations/layer8_attention_weight_min": -49.91243362426758,
      "activations/layer9_attention_weight_max": 50.45566940307617,
      "activations/layer9_attention_weight_min": -49.6731071472168,
      "epoch": 19.04,
      "learning_rate": 2.752424242424242e-05,
      "loss": 2.7305,
      "step": 327600
    },
    {
      "activations/layer0_attention_weight_max": 15.083931922912598,
      "activations/layer0_attention_weight_min": -13.804922103881836,
      "activations/layer10_attention_weight_max": 37.099365234375,
      "activations/layer10_attention_weight_min": -36.81502151489258,
      "activations/layer11_attention_weight_max": 31.84449005126953,
      "activations/layer11_attention_weight_min": -33.96217346191406,
      "activations/layer12_attention_weight_max": 19.724653244018555,
      "activations/layer12_attention_weight_min": -23.215057373046875,
      "activations/layer13_attention_weight_max": 37.964630126953125,
      "activations/layer13_attention_weight_min": -30.79140281677246,
      "activations/layer14_attention_weight_max": 36.370994567871094,
      "activations/layer14_attention_weight_min": -30.581789016723633,
      "activations/layer15_attention_weight_max": 31.90239715576172,
      "activations/layer15_attention_weight_min": -30.913782119750977,
      "activations/layer16_attention_weight_max": 34.2046012878418,
      "activations/layer16_attention_weight_min": -32.433048248291016,
      "activations/layer17_attention_weight_max": 51.122535705566406,
      "activations/layer17_attention_weight_min": -45.61250686645508,
      "activations/layer18_attention_weight_max": 46.58467102050781,
      "activations/layer18_attention_weight_min": -39.298553466796875,
      "activations/layer19_attention_weight_max": 23.560575485229492,
      "activations/layer19_attention_weight_min": -21.26118278503418,
      "activations/layer1_attention_weight_max": 15.981745719909668,
      "activations/layer1_attention_weight_min": -12.748018264770508,
      "activations/layer20_attention_weight_max": 22.019554138183594,
      "activations/layer20_attention_weight_min": -21.574649810791016,
      "activations/layer21_attention_weight_max": 32.530094146728516,
      "activations/layer21_attention_weight_min": -21.0572452545166,
      "activations/layer22_attention_weight_max": 26.332096099853516,
      "activations/layer22_attention_weight_min": -26.262985229492188,
      "activations/layer23_attention_weight_max": 36.07090759277344,
      "activations/layer23_attention_weight_min": -24.825469970703125,
      "activations/layer2_attention_weight_max": 29.603588104248047,
      "activations/layer2_attention_weight_min": -27.561676025390625,
      "activations/layer3_attention_weight_max": 81.52449798583984,
      "activations/layer3_attention_weight_min": -80.68021392822266,
      "activations/layer4_attention_weight_max": 79.38976287841797,
      "activations/layer4_attention_weight_min": -79.02384185791016,
      "activations/layer5_attention_weight_max": 63.56946563720703,
      "activations/layer5_attention_weight_min": -71.30623626708984,
      "activations/layer6_attention_weight_max": 48.80765151977539,
      "activations/layer6_attention_weight_min": -49.31965255737305,
      "activations/layer7_attention_weight_max": 63.610103607177734,
      "activations/layer7_attention_weight_min": -62.07035446166992,
      "activations/layer8_attention_weight_max": 44.98412322998047,
      "activations/layer8_attention_weight_min": -49.514766693115234,
      "activations/layer9_attention_weight_max": 44.777889251708984,
      "activations/layer9_attention_weight_min": -45.11398696899414,
      "epoch": 19.04,
      "learning_rate": 2.7505303030303028e-05,
      "loss": 2.7333,
      "step": 327650
    },
    {
      "activations/layer0_attention_weight_max": 15.932432174682617,
      "activations/layer0_attention_weight_min": -13.802451133728027,
      "activations/layer10_attention_weight_max": 36.73527908325195,
      "activations/layer10_attention_weight_min": -34.19147872924805,
      "activations/layer11_attention_weight_max": 34.66365051269531,
      "activations/layer11_attention_weight_min": -32.920509338378906,
      "activations/layer12_attention_weight_max": 17.661579132080078,
      "activations/layer12_attention_weight_min": -38.353336334228516,
      "activations/layer13_attention_weight_max": 38.675331115722656,
      "activations/layer13_attention_weight_min": -29.9072208404541,
      "activations/layer14_attention_weight_max": 35.20635986328125,
      "activations/layer14_attention_weight_min": -30.45985221862793,
      "activations/layer15_attention_weight_max": 31.878023147583008,
      "activations/layer15_attention_weight_min": -28.758569717407227,
      "activations/layer16_attention_weight_max": 31.24248504638672,
      "activations/layer16_attention_weight_min": -29.47882652282715,
      "activations/layer17_attention_weight_max": 46.15278244018555,
      "activations/layer17_attention_weight_min": -43.99490737915039,
      "activations/layer18_attention_weight_max": 45.15579605102539,
      "activations/layer18_attention_weight_min": -38.380733489990234,
      "activations/layer19_attention_weight_max": 24.786108016967773,
      "activations/layer19_attention_weight_min": -23.390016555786133,
      "activations/layer1_attention_weight_max": 16.199796676635742,
      "activations/layer1_attention_weight_min": -14.073670387268066,
      "activations/layer20_attention_weight_max": 23.83543586730957,
      "activations/layer20_attention_weight_min": -24.019609451293945,
      "activations/layer21_attention_weight_max": 36.36809158325195,
      "activations/layer21_attention_weight_min": -20.887060165405273,
      "activations/layer22_attention_weight_max": 29.25615119934082,
      "activations/layer22_attention_weight_min": -26.01172637939453,
      "activations/layer23_attention_weight_max": 38.88645935058594,
      "activations/layer23_attention_weight_min": -25.824356079101562,
      "activations/layer2_attention_weight_max": 31.206815719604492,
      "activations/layer2_attention_weight_min": -29.22789764404297,
      "activations/layer3_attention_weight_max": 85.5672607421875,
      "activations/layer3_attention_weight_min": -82.15750885009766,
      "activations/layer4_attention_weight_max": 81.57823181152344,
      "activations/layer4_attention_weight_min": -80.92613983154297,
      "activations/layer5_attention_weight_max": 63.03823471069336,
      "activations/layer5_attention_weight_min": -77.05770874023438,
      "activations/layer6_attention_weight_max": 48.75876235961914,
      "activations/layer6_attention_weight_min": -47.6901741027832,
      "activations/layer7_attention_weight_max": 66.90877532958984,
      "activations/layer7_attention_weight_min": -64.44471740722656,
      "activations/layer8_attention_weight_max": 45.777984619140625,
      "activations/layer8_attention_weight_min": -45.05305862426758,
      "activations/layer9_attention_weight_max": 45.35422134399414,
      "activations/layer9_attention_weight_min": -43.83049011230469,
      "epoch": 19.04,
      "learning_rate": 2.7486742424242422e-05,
      "loss": 2.719,
      "step": 327700
    },
    {
      "activations/layer0_attention_weight_max": 15.397636413574219,
      "activations/layer0_attention_weight_min": -13.580034255981445,
      "activations/layer10_attention_weight_max": 39.002052307128906,
      "activations/layer10_attention_weight_min": -36.630287170410156,
      "activations/layer11_attention_weight_max": 34.99144744873047,
      "activations/layer11_attention_weight_min": -34.71034622192383,
      "activations/layer12_attention_weight_max": 19.09765625,
      "activations/layer12_attention_weight_min": -26.299070358276367,
      "activations/layer13_attention_weight_max": 35.61738586425781,
      "activations/layer13_attention_weight_min": -27.75475311279297,
      "activations/layer14_attention_weight_max": 41.54046630859375,
      "activations/layer14_attention_weight_min": -30.914852142333984,
      "activations/layer15_attention_weight_max": 36.38960266113281,
      "activations/layer15_attention_weight_min": -28.91954803466797,
      "activations/layer16_attention_weight_max": 38.74506759643555,
      "activations/layer16_attention_weight_min": -32.054840087890625,
      "activations/layer17_attention_weight_max": 57.79043960571289,
      "activations/layer17_attention_weight_min": -49.595001220703125,
      "activations/layer18_attention_weight_max": 51.195838928222656,
      "activations/layer18_attention_weight_min": -44.26398849487305,
      "activations/layer19_attention_weight_max": 26.922870635986328,
      "activations/layer19_attention_weight_min": -27.10770606994629,
      "activations/layer1_attention_weight_max": 17.277868270874023,
      "activations/layer1_attention_weight_min": -13.027048110961914,
      "activations/layer20_attention_weight_max": 26.81622886657715,
      "activations/layer20_attention_weight_min": -21.385234832763672,
      "activations/layer21_attention_weight_max": 40.01189041137695,
      "activations/layer21_attention_weight_min": -22.35955238342285,
      "activations/layer22_attention_weight_max": 32.25455093383789,
      "activations/layer22_attention_weight_min": -24.87984848022461,
      "activations/layer23_attention_weight_max": 42.7190055847168,
      "activations/layer23_attention_weight_min": -23.867908477783203,
      "activations/layer2_attention_weight_max": 31.159732818603516,
      "activations/layer2_attention_weight_min": -30.388736724853516,
      "activations/layer3_attention_weight_max": 80.25884246826172,
      "activations/layer3_attention_weight_min": -80.68316650390625,
      "activations/layer4_attention_weight_max": 79.29829406738281,
      "activations/layer4_attention_weight_min": -80.6839828491211,
      "activations/layer5_attention_weight_max": 61.31977081298828,
      "activations/layer5_attention_weight_min": -67.54835510253906,
      "activations/layer6_attention_weight_max": 46.015445709228516,
      "activations/layer6_attention_weight_min": -46.90723419189453,
      "activations/layer7_attention_weight_max": 67.44432067871094,
      "activations/layer7_attention_weight_min": -65.46064758300781,
      "activations/layer8_attention_weight_max": 49.51216506958008,
      "activations/layer8_attention_weight_min": -50.27690505981445,
      "activations/layer9_attention_weight_max": 52.6866569519043,
      "activations/layer9_attention_weight_min": -46.925350189208984,
      "epoch": 19.04,
      "learning_rate": 2.746780303030303e-05,
      "loss": 2.729,
      "step": 327750
    },
    {
      "activations/layer0_attention_weight_max": 16.093032836914062,
      "activations/layer0_attention_weight_min": -13.669622421264648,
      "activations/layer10_attention_weight_max": 37.52671813964844,
      "activations/layer10_attention_weight_min": -39.66896438598633,
      "activations/layer11_attention_weight_max": 35.29903793334961,
      "activations/layer11_attention_weight_min": -38.811607360839844,
      "activations/layer12_attention_weight_max": 19.89842414855957,
      "activations/layer12_attention_weight_min": -25.512590408325195,
      "activations/layer13_attention_weight_max": 46.291873931884766,
      "activations/layer13_attention_weight_min": -30.38058090209961,
      "activations/layer14_attention_weight_max": 38.45048522949219,
      "activations/layer14_attention_weight_min": -31.883586883544922,
      "activations/layer15_attention_weight_max": 36.62955856323242,
      "activations/layer15_attention_weight_min": -32.028995513916016,
      "activations/layer16_attention_weight_max": 37.490257263183594,
      "activations/layer16_attention_weight_min": -34.13895034790039,
      "activations/layer17_attention_weight_max": 52.72939682006836,
      "activations/layer17_attention_weight_min": -48.218162536621094,
      "activations/layer18_attention_weight_max": 49.42378616333008,
      "activations/layer18_attention_weight_min": -42.219970703125,
      "activations/layer19_attention_weight_max": 24.354246139526367,
      "activations/layer19_attention_weight_min": -23.847633361816406,
      "activations/layer1_attention_weight_max": 17.696552276611328,
      "activations/layer1_attention_weight_min": -12.856918334960938,
      "activations/layer20_attention_weight_max": 21.79380226135254,
      "activations/layer20_attention_weight_min": -23.497282028198242,
      "activations/layer21_attention_weight_max": 43.885765075683594,
      "activations/layer21_attention_weight_min": -20.786596298217773,
      "activations/layer22_attention_weight_max": 31.179393768310547,
      "activations/layer22_attention_weight_min": -25.11992835998535,
      "activations/layer23_attention_weight_max": 42.000606536865234,
      "activations/layer23_attention_weight_min": -24.323291778564453,
      "activations/layer2_attention_weight_max": 30.920743942260742,
      "activations/layer2_attention_weight_min": -31.121986389160156,
      "activations/layer3_attention_weight_max": 85.81230926513672,
      "activations/layer3_attention_weight_min": -85.45635986328125,
      "activations/layer4_attention_weight_max": 80.60304260253906,
      "activations/layer4_attention_weight_min": -83.92980194091797,
      "activations/layer5_attention_weight_max": 66.63904571533203,
      "activations/layer5_attention_weight_min": -75.73362731933594,
      "activations/layer6_attention_weight_max": 49.446510314941406,
      "activations/layer6_attention_weight_min": -52.39025115966797,
      "activations/layer7_attention_weight_max": 67.9678726196289,
      "activations/layer7_attention_weight_min": -65.20023345947266,
      "activations/layer8_attention_weight_max": 50.327110290527344,
      "activations/layer8_attention_weight_min": -49.27527618408203,
      "activations/layer9_attention_weight_max": 54.02393341064453,
      "activations/layer9_attention_weight_min": -47.09136962890625,
      "epoch": 19.05,
      "learning_rate": 2.7448863636363632e-05,
      "loss": 2.7362,
      "step": 327800
    },
    {
      "activations/layer0_attention_weight_max": 15.595088005065918,
      "activations/layer0_attention_weight_min": -14.07109260559082,
      "activations/layer10_attention_weight_max": 35.08191680908203,
      "activations/layer10_attention_weight_min": -32.722938537597656,
      "activations/layer11_attention_weight_max": 32.20616149902344,
      "activations/layer11_attention_weight_min": -32.90765380859375,
      "activations/layer12_attention_weight_max": 18.28628158569336,
      "activations/layer12_attention_weight_min": -27.312318801879883,
      "activations/layer13_attention_weight_max": 37.143917083740234,
      "activations/layer13_attention_weight_min": -26.74904441833496,
      "activations/layer14_attention_weight_max": 35.784297943115234,
      "activations/layer14_attention_weight_min": -28.03288459777832,
      "activations/layer15_attention_weight_max": 34.86390686035156,
      "activations/layer15_attention_weight_min": -29.83005714416504,
      "activations/layer16_attention_weight_max": 33.56105041503906,
      "activations/layer16_attention_weight_min": -32.27928924560547,
      "activations/layer17_attention_weight_max": 49.43023681640625,
      "activations/layer17_attention_weight_min": -43.166404724121094,
      "activations/layer18_attention_weight_max": 47.96732711791992,
      "activations/layer18_attention_weight_min": -41.39826583862305,
      "activations/layer19_attention_weight_max": 26.70281410217285,
      "activations/layer19_attention_weight_min": -23.391807556152344,
      "activations/layer1_attention_weight_max": 15.951876640319824,
      "activations/layer1_attention_weight_min": -13.658780097961426,
      "activations/layer20_attention_weight_max": 24.852083206176758,
      "activations/layer20_attention_weight_min": -20.74367332458496,
      "activations/layer21_attention_weight_max": 36.496463775634766,
      "activations/layer21_attention_weight_min": -26.018945693969727,
      "activations/layer22_attention_weight_max": 29.257612228393555,
      "activations/layer22_attention_weight_min": -26.293996810913086,
      "activations/layer23_attention_weight_max": 35.40374755859375,
      "activations/layer23_attention_weight_min": -22.851600646972656,
      "activations/layer2_attention_weight_max": 31.870052337646484,
      "activations/layer2_attention_weight_min": -30.653671264648438,
      "activations/layer3_attention_weight_max": 85.0671615600586,
      "activations/layer3_attention_weight_min": -90.4869384765625,
      "activations/layer4_attention_weight_max": 79.13671875,
      "activations/layer4_attention_weight_min": -85.48832702636719,
      "activations/layer5_attention_weight_max": 58.65957260131836,
      "activations/layer5_attention_weight_min": -73.48445129394531,
      "activations/layer6_attention_weight_max": 49.861331939697266,
      "activations/layer6_attention_weight_min": -49.7735710144043,
      "activations/layer7_attention_weight_max": 64.07335662841797,
      "activations/layer7_attention_weight_min": -65.74907684326172,
      "activations/layer8_attention_weight_max": 46.97339630126953,
      "activations/layer8_attention_weight_min": -50.87528991699219,
      "activations/layer9_attention_weight_max": 48.38071823120117,
      "activations/layer9_attention_weight_min": -47.26592254638672,
      "epoch": 19.05,
      "learning_rate": 2.742992424242424e-05,
      "loss": 2.7218,
      "step": 327850
    },
    {
      "activations/layer0_attention_weight_max": 15.968941688537598,
      "activations/layer0_attention_weight_min": -13.595746040344238,
      "activations/layer10_attention_weight_max": 37.277565002441406,
      "activations/layer10_attention_weight_min": -34.69968795776367,
      "activations/layer11_attention_weight_max": 33.73534393310547,
      "activations/layer11_attention_weight_min": -32.711463928222656,
      "activations/layer12_attention_weight_max": 18.86163902282715,
      "activations/layer12_attention_weight_min": -23.68548011779785,
      "activations/layer13_attention_weight_max": 35.89735412597656,
      "activations/layer13_attention_weight_min": -29.877918243408203,
      "activations/layer14_attention_weight_max": 38.542945861816406,
      "activations/layer14_attention_weight_min": -30.706684112548828,
      "activations/layer15_attention_weight_max": 35.170528411865234,
      "activations/layer15_attention_weight_min": -30.5407657623291,
      "activations/layer16_attention_weight_max": 38.62398910522461,
      "activations/layer16_attention_weight_min": -33.70441436767578,
      "activations/layer17_attention_weight_max": 56.65076446533203,
      "activations/layer17_attention_weight_min": -48.923980712890625,
      "activations/layer18_attention_weight_max": 51.4316520690918,
      "activations/layer18_attention_weight_min": -43.05171203613281,
      "activations/layer19_attention_weight_max": 27.42266273498535,
      "activations/layer19_attention_weight_min": -24.486833572387695,
      "activations/layer1_attention_weight_max": 15.208881378173828,
      "activations/layer1_attention_weight_min": -14.129020690917969,
      "activations/layer20_attention_weight_max": 22.931194305419922,
      "activations/layer20_attention_weight_min": -22.88609504699707,
      "activations/layer21_attention_weight_max": 39.4183349609375,
      "activations/layer21_attention_weight_min": -23.975830078125,
      "activations/layer22_attention_weight_max": 30.792478561401367,
      "activations/layer22_attention_weight_min": -26.75929069519043,
      "activations/layer23_attention_weight_max": 37.41468048095703,
      "activations/layer23_attention_weight_min": -26.524669647216797,
      "activations/layer2_attention_weight_max": 32.303794860839844,
      "activations/layer2_attention_weight_min": -32.56513214111328,
      "activations/layer3_attention_weight_max": 80.6951904296875,
      "activations/layer3_attention_weight_min": -87.57170867919922,
      "activations/layer4_attention_weight_max": 78.48307037353516,
      "activations/layer4_attention_weight_min": -80.66231536865234,
      "activations/layer5_attention_weight_max": 59.78890609741211,
      "activations/layer5_attention_weight_min": -71.3853759765625,
      "activations/layer6_attention_weight_max": 50.04273986816406,
      "activations/layer6_attention_weight_min": -48.40304183959961,
      "activations/layer7_attention_weight_max": 62.48525619506836,
      "activations/layer7_attention_weight_min": -61.037906646728516,
      "activations/layer8_attention_weight_max": 49.164058685302734,
      "activations/layer8_attention_weight_min": -52.83745193481445,
      "activations/layer9_attention_weight_max": 43.2481803894043,
      "activations/layer9_attention_weight_min": -46.41621017456055,
      "epoch": 19.05,
      "learning_rate": 2.7410984848484845e-05,
      "loss": 2.7157,
      "step": 327900
    },
    {
      "activations/layer0_attention_weight_max": 15.954524993896484,
      "activations/layer0_attention_weight_min": -13.821477890014648,
      "activations/layer10_attention_weight_max": 35.63328552246094,
      "activations/layer10_attention_weight_min": -36.60371017456055,
      "activations/layer11_attention_weight_max": 34.60935592651367,
      "activations/layer11_attention_weight_min": -35.470252990722656,
      "activations/layer12_attention_weight_max": 19.29793357849121,
      "activations/layer12_attention_weight_min": -27.216001510620117,
      "activations/layer13_attention_weight_max": 30.288965225219727,
      "activations/layer13_attention_weight_min": -28.574329376220703,
      "activations/layer14_attention_weight_max": 33.4755859375,
      "activations/layer14_attention_weight_min": -29.036922454833984,
      "activations/layer15_attention_weight_max": 32.51958465576172,
      "activations/layer15_attention_weight_min": -29.505985260009766,
      "activations/layer16_attention_weight_max": 35.560028076171875,
      "activations/layer16_attention_weight_min": -32.711360931396484,
      "activations/layer17_attention_weight_max": 52.99329376220703,
      "activations/layer17_attention_weight_min": -46.22504806518555,
      "activations/layer18_attention_weight_max": 45.4332160949707,
      "activations/layer18_attention_weight_min": -37.933135986328125,
      "activations/layer19_attention_weight_max": 26.739551544189453,
      "activations/layer19_attention_weight_min": -22.271509170532227,
      "activations/layer1_attention_weight_max": 16.2309627532959,
      "activations/layer1_attention_weight_min": -14.878546714782715,
      "activations/layer20_attention_weight_max": 23.838083267211914,
      "activations/layer20_attention_weight_min": -22.212444305419922,
      "activations/layer21_attention_weight_max": 34.10103225708008,
      "activations/layer21_attention_weight_min": -21.385828018188477,
      "activations/layer22_attention_weight_max": 28.46172332763672,
      "activations/layer22_attention_weight_min": -25.6962890625,
      "activations/layer23_attention_weight_max": 33.71308135986328,
      "activations/layer23_attention_weight_min": -26.076475143432617,
      "activations/layer2_attention_weight_max": 30.7157039642334,
      "activations/layer2_attention_weight_min": -30.20941162109375,
      "activations/layer3_attention_weight_max": 84.43339538574219,
      "activations/layer3_attention_weight_min": -81.32062530517578,
      "activations/layer4_attention_weight_max": 81.490234375,
      "activations/layer4_attention_weight_min": -82.60499572753906,
      "activations/layer5_attention_weight_max": 61.4723014831543,
      "activations/layer5_attention_weight_min": -72.37356567382812,
      "activations/layer6_attention_weight_max": 49.34345245361328,
      "activations/layer6_attention_weight_min": -48.65291213989258,
      "activations/layer7_attention_weight_max": 64.03899383544922,
      "activations/layer7_attention_weight_min": -64.04969787597656,
      "activations/layer8_attention_weight_max": 45.50905227661133,
      "activations/layer8_attention_weight_min": -48.1262092590332,
      "activations/layer9_attention_weight_max": 43.09418487548828,
      "activations/layer9_attention_weight_min": -45.25899887084961,
      "epoch": 19.06,
      "learning_rate": 2.7392045454545453e-05,
      "loss": 2.7339,
      "step": 327950
    },
    {
      "activations/layer0_attention_weight_max": 15.320294380187988,
      "activations/layer0_attention_weight_min": -13.86493968963623,
      "activations/layer10_attention_weight_max": 38.39506149291992,
      "activations/layer10_attention_weight_min": -36.95038986206055,
      "activations/layer11_attention_weight_max": 32.74443817138672,
      "activations/layer11_attention_weight_min": -34.24504089355469,
      "activations/layer12_attention_weight_max": 18.192852020263672,
      "activations/layer12_attention_weight_min": -25.944101333618164,
      "activations/layer13_attention_weight_max": 48.21841049194336,
      "activations/layer13_attention_weight_min": -27.461259841918945,
      "activations/layer14_attention_weight_max": 38.47465896606445,
      "activations/layer14_attention_weight_min": -28.910167694091797,
      "activations/layer15_attention_weight_max": 44.654170989990234,
      "activations/layer15_attention_weight_min": -28.759441375732422,
      "activations/layer16_attention_weight_max": 38.453067779541016,
      "activations/layer16_attention_weight_min": -34.504886627197266,
      "activations/layer17_attention_weight_max": 60.70076370239258,
      "activations/layer17_attention_weight_min": -51.560001373291016,
      "activations/layer18_attention_weight_max": 57.481388092041016,
      "activations/layer18_attention_weight_min": -46.33916473388672,
      "activations/layer19_attention_weight_max": 31.19692039489746,
      "activations/layer19_attention_weight_min": -26.874486923217773,
      "activations/layer1_attention_weight_max": 15.724630355834961,
      "activations/layer1_attention_weight_min": -14.680886268615723,
      "activations/layer20_attention_weight_max": 28.918943405151367,
      "activations/layer20_attention_weight_min": -24.058870315551758,
      "activations/layer21_attention_weight_max": 49.988006591796875,
      "activations/layer21_attention_weight_min": -25.248027801513672,
      "activations/layer22_attention_weight_max": 36.586727142333984,
      "activations/layer22_attention_weight_min": -26.72545623779297,
      "activations/layer23_attention_weight_max": 48.246543884277344,
      "activations/layer23_attention_weight_min": -24.075199127197266,
      "activations/layer2_attention_weight_max": 31.9307861328125,
      "activations/layer2_attention_weight_min": -30.86286735534668,
      "activations/layer3_attention_weight_max": 86.80538177490234,
      "activations/layer3_attention_weight_min": -86.82848358154297,
      "activations/layer4_attention_weight_max": 82.40298461914062,
      "activations/layer4_attention_weight_min": -80.56787872314453,
      "activations/layer5_attention_weight_max": 62.18705749511719,
      "activations/layer5_attention_weight_min": -69.51144409179688,
      "activations/layer6_attention_weight_max": 49.748165130615234,
      "activations/layer6_attention_weight_min": -50.62465286254883,
      "activations/layer7_attention_weight_max": 66.7791519165039,
      "activations/layer7_attention_weight_min": -62.72134017944336,
      "activations/layer8_attention_weight_max": 48.8333854675293,
      "activations/layer8_attention_weight_min": -50.63945770263672,
      "activations/layer9_attention_weight_max": 47.648841857910156,
      "activations/layer9_attention_weight_min": -46.05316162109375,
      "epoch": 19.06,
      "learning_rate": 2.7373106060606058e-05,
      "loss": 2.7238,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_loss": 2.6875,
      "eval_runtime": 8.483,
      "eval_samples_per_second": 506.189,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_openwebtext_loss": 2.6875,
      "eval_openwebtext_ppl": 14.694892728788941,
      "eval_openwebtext_runtime": 8.483,
      "eval_openwebtext_samples_per_second": 506.189,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.0018,
      "eval_wikitext_samples_per_second": 227.794,
      "step": 328000
    },
    {
      "epoch": 19.06,
      "eval_lambada_loss": 2.44921875,
      "eval_lambada_ppl": 11.57929685891098,
      "eval_lambada_runtime": 9.5819,
      "eval_lambada_samples_per_second": 508.146,
      "step": 328000
    },
    {
      "activations/layer0_attention_weight_max": 15.791021347045898,
      "activations/layer0_attention_weight_min": -13.723859786987305,
      "activations/layer10_attention_weight_max": 36.96977615356445,
      "activations/layer10_attention_weight_min": -36.28840255737305,
      "activations/layer11_attention_weight_max": 33.75515365600586,
      "activations/layer11_attention_weight_min": -35.72869873046875,
      "activations/layer12_attention_weight_max": 20.13579750061035,
      "activations/layer12_attention_weight_min": -26.861371994018555,
      "activations/layer13_attention_weight_max": 37.55226135253906,
      "activations/layer13_attention_weight_min": -29.72101593017578,
      "activations/layer14_attention_weight_max": 36.9387321472168,
      "activations/layer14_attention_weight_min": -31.300811767578125,
      "activations/layer15_attention_weight_max": 35.08321762084961,
      "activations/layer15_attention_weight_min": -29.780874252319336,
      "activations/layer16_attention_weight_max": 37.54486083984375,
      "activations/layer16_attention_weight_min": -33.84341812133789,
      "activations/layer17_attention_weight_max": 52.75238800048828,
      "activations/layer17_attention_weight_min": -48.014461517333984,
      "activations/layer18_attention_weight_max": 50.09105682373047,
      "activations/layer18_attention_weight_min": -39.06155014038086,
      "activations/layer19_attention_weight_max": 25.659515380859375,
      "activations/layer19_attention_weight_min": -22.279396057128906,
      "activations/layer1_attention_weight_max": 17.192440032958984,
      "activations/layer1_attention_weight_min": -15.668427467346191,
      "activations/layer20_attention_weight_max": 22.073740005493164,
      "activations/layer20_attention_weight_min": -20.61008071899414,
      "activations/layer21_attention_weight_max": 38.742759704589844,
      "activations/layer21_attention_weight_min": -23.414072036743164,
      "activations/layer22_attention_weight_max": 28.24842071533203,
      "activations/layer22_attention_weight_min": -24.770126342773438,
      "activations/layer23_attention_weight_max": 37.689361572265625,
      "activations/layer23_attention_weight_min": -23.125896453857422,
      "activations/layer2_attention_weight_max": 33.31541442871094,
      "activations/layer2_attention_weight_min": -31.222103118896484,
      "activations/layer3_attention_weight_max": 88.19012451171875,
      "activations/layer3_attention_weight_min": -87.46440887451172,
      "activations/layer4_attention_weight_max": 79.3794174194336,
      "activations/layer4_attention_weight_min": -78.73552703857422,
      "activations/layer5_attention_weight_max": 58.67237091064453,
      "activations/layer5_attention_weight_min": -72.68102264404297,
      "activations/layer6_attention_weight_max": 48.24761199951172,
      "activations/layer6_attention_weight_min": -49.5607795715332,
      "activations/layer7_attention_weight_max": 61.403438568115234,
      "activations/layer7_attention_weight_min": -66.15526580810547,
      "activations/layer8_attention_weight_max": 45.191131591796875,
      "activations/layer8_attention_weight_min": -50.42644119262695,
      "activations/layer9_attention_weight_max": 44.791202545166016,
      "activations/layer9_attention_weight_min": -45.789913177490234,
      "epoch": 19.06,
      "learning_rate": 2.7354166666666663e-05,
      "loss": 2.7354,
      "step": 328050
    },
    {
      "activations/layer0_attention_weight_max": 15.691837310791016,
      "activations/layer0_attention_weight_min": -13.807418823242188,
      "activations/layer10_attention_weight_max": 35.83527755737305,
      "activations/layer10_attention_weight_min": -39.58884811401367,
      "activations/layer11_attention_weight_max": 34.481712341308594,
      "activations/layer11_attention_weight_min": -39.24052429199219,
      "activations/layer12_attention_weight_max": 19.160810470581055,
      "activations/layer12_attention_weight_min": -25.621461868286133,
      "activations/layer13_attention_weight_max": 33.061187744140625,
      "activations/layer13_attention_weight_min": -28.4906005859375,
      "activations/layer14_attention_weight_max": 33.276512145996094,
      "activations/layer14_attention_weight_min": -30.32037925720215,
      "activations/layer15_attention_weight_max": 31.791349411010742,
      "activations/layer15_attention_weight_min": -32.22823715209961,
      "activations/layer16_attention_weight_max": 35.66464614868164,
      "activations/layer16_attention_weight_min": -33.37987518310547,
      "activations/layer17_attention_weight_max": 52.15711212158203,
      "activations/layer17_attention_weight_min": -44.79994201660156,
      "activations/layer18_attention_weight_max": 50.26034164428711,
      "activations/layer18_attention_weight_min": -40.994022369384766,
      "activations/layer19_attention_weight_max": 24.07908058166504,
      "activations/layer19_attention_weight_min": -25.641742706298828,
      "activations/layer1_attention_weight_max": 16.26935386657715,
      "activations/layer1_attention_weight_min": -13.937202453613281,
      "activations/layer20_attention_weight_max": 25.051780700683594,
      "activations/layer20_attention_weight_min": -23.76251220703125,
      "activations/layer21_attention_weight_max": 33.88747024536133,
      "activations/layer21_attention_weight_min": -26.216997146606445,
      "activations/layer22_attention_weight_max": 30.38603973388672,
      "activations/layer22_attention_weight_min": -26.7592716217041,
      "activations/layer23_attention_weight_max": 36.94697570800781,
      "activations/layer23_attention_weight_min": -25.59531021118164,
      "activations/layer2_attention_weight_max": 34.05348205566406,
      "activations/layer2_attention_weight_min": -33.04975128173828,
      "activations/layer3_attention_weight_max": 92.22734069824219,
      "activations/layer3_attention_weight_min": -95.53984069824219,
      "activations/layer4_attention_weight_max": 85.9784164428711,
      "activations/layer4_attention_weight_min": -87.13228607177734,
      "activations/layer5_attention_weight_max": 62.42394256591797,
      "activations/layer5_attention_weight_min": -75.76435852050781,
      "activations/layer6_attention_weight_max": 53.03988265991211,
      "activations/layer6_attention_weight_min": -56.76140594482422,
      "activations/layer7_attention_weight_max": 68.89598083496094,
      "activations/layer7_attention_weight_min": -71.88139343261719,
      "activations/layer8_attention_weight_max": 51.331478118896484,
      "activations/layer8_attention_weight_min": -54.872398376464844,
      "activations/layer9_attention_weight_max": 52.04102325439453,
      "activations/layer9_attention_weight_min": -49.546356201171875,
      "epoch": 19.06,
      "learning_rate": 2.7335227272727268e-05,
      "loss": 2.7344,
      "step": 328100
    },
    {
      "activations/layer0_attention_weight_max": 14.700529098510742,
      "activations/layer0_attention_weight_min": -13.812047004699707,
      "activations/layer10_attention_weight_max": 35.502708435058594,
      "activations/layer10_attention_weight_min": -38.122352600097656,
      "activations/layer11_attention_weight_max": 31.87506675720215,
      "activations/layer11_attention_weight_min": -36.494384765625,
      "activations/layer12_attention_weight_max": 17.791156768798828,
      "activations/layer12_attention_weight_min": -24.46245574951172,
      "activations/layer13_attention_weight_max": 31.04236602783203,
      "activations/layer13_attention_weight_min": -34.10881042480469,
      "activations/layer14_attention_weight_max": 32.578514099121094,
      "activations/layer14_attention_weight_min": -30.93455696105957,
      "activations/layer15_attention_weight_max": 31.91716194152832,
      "activations/layer15_attention_weight_min": -31.926122665405273,
      "activations/layer16_attention_weight_max": 32.24235534667969,
      "activations/layer16_attention_weight_min": -33.42574691772461,
      "activations/layer17_attention_weight_max": 45.904335021972656,
      "activations/layer17_attention_weight_min": -46.0988883972168,
      "activations/layer18_attention_weight_max": 44.971370697021484,
      "activations/layer18_attention_weight_min": -39.03729248046875,
      "activations/layer19_attention_weight_max": 21.355024337768555,
      "activations/layer19_attention_weight_min": -22.50434684753418,
      "activations/layer1_attention_weight_max": 15.471803665161133,
      "activations/layer1_attention_weight_min": -14.243638038635254,
      "activations/layer20_attention_weight_max": 20.769641876220703,
      "activations/layer20_attention_weight_min": -22.963754653930664,
      "activations/layer21_attention_weight_max": 33.75645065307617,
      "activations/layer21_attention_weight_min": -24.079729080200195,
      "activations/layer22_attention_weight_max": 27.71194076538086,
      "activations/layer22_attention_weight_min": -27.511642456054688,
      "activations/layer23_attention_weight_max": 33.132667541503906,
      "activations/layer23_attention_weight_min": -27.84596061706543,
      "activations/layer2_attention_weight_max": 31.959043502807617,
      "activations/layer2_attention_weight_min": -30.280363082885742,
      "activations/layer3_attention_weight_max": 84.81278228759766,
      "activations/layer3_attention_weight_min": -79.66117095947266,
      "activations/layer4_attention_weight_max": 83.59033966064453,
      "activations/layer4_attention_weight_min": -81.2217025756836,
      "activations/layer5_attention_weight_max": 63.17479705810547,
      "activations/layer5_attention_weight_min": -75.78315734863281,
      "activations/layer6_attention_weight_max": 49.184932708740234,
      "activations/layer6_attention_weight_min": -50.61232376098633,
      "activations/layer7_attention_weight_max": 66.66578674316406,
      "activations/layer7_attention_weight_min": -64.49011993408203,
      "activations/layer8_attention_weight_max": 45.94852066040039,
      "activations/layer8_attention_weight_min": -51.05411148071289,
      "activations/layer9_attention_weight_max": 42.445152282714844,
      "activations/layer9_attention_weight_min": -45.17724609375,
      "epoch": 19.07,
      "learning_rate": 2.7316287878787876e-05,
      "loss": 2.7383,
      "step": 328150
    },
    {
      "activations/layer0_attention_weight_max": 15.611021995544434,
      "activations/layer0_attention_weight_min": -13.724356651306152,
      "activations/layer10_attention_weight_max": 37.50395202636719,
      "activations/layer10_attention_weight_min": -36.481964111328125,
      "activations/layer11_attention_weight_max": 34.31182098388672,
      "activations/layer11_attention_weight_min": -34.30353546142578,
      "activations/layer12_attention_weight_max": 19.483734130859375,
      "activations/layer12_attention_weight_min": -24.022171020507812,
      "activations/layer13_attention_weight_max": 30.840402603149414,
      "activations/layer13_attention_weight_min": -29.266529083251953,
      "activations/layer14_attention_weight_max": 34.092918395996094,
      "activations/layer14_attention_weight_min": -29.73329734802246,
      "activations/layer15_attention_weight_max": 32.177181243896484,
      "activations/layer15_attention_weight_min": -29.915136337280273,
      "activations/layer16_attention_weight_max": 33.708717346191406,
      "activations/layer16_attention_weight_min": -32.62075424194336,
      "activations/layer17_attention_weight_max": 53.401161193847656,
      "activations/layer17_attention_weight_min": -45.88066482543945,
      "activations/layer18_attention_weight_max": 50.26475524902344,
      "activations/layer18_attention_weight_min": -38.693389892578125,
      "activations/layer19_attention_weight_max": 23.577421188354492,
      "activations/layer19_attention_weight_min": -22.499080657958984,
      "activations/layer1_attention_weight_max": 15.434428215026855,
      "activations/layer1_attention_weight_min": -14.628478050231934,
      "activations/layer20_attention_weight_max": 21.384197235107422,
      "activations/layer20_attention_weight_min": -21.162490844726562,
      "activations/layer21_attention_weight_max": 39.76018524169922,
      "activations/layer21_attention_weight_min": -24.284255981445312,
      "activations/layer22_attention_weight_max": 34.13298416137695,
      "activations/layer22_attention_weight_min": -24.25173568725586,
      "activations/layer23_attention_weight_max": 35.677734375,
      "activations/layer23_attention_weight_min": -23.94158935546875,
      "activations/layer2_attention_weight_max": 30.275569915771484,
      "activations/layer2_attention_weight_min": -30.223800659179688,
      "activations/layer3_attention_weight_max": 83.23833465576172,
      "activations/layer3_attention_weight_min": -80.10276794433594,
      "activations/layer4_attention_weight_max": 80.67815399169922,
      "activations/layer4_attention_weight_min": -82.6868667602539,
      "activations/layer5_attention_weight_max": 61.77474594116211,
      "activations/layer5_attention_weight_min": -72.2263412475586,
      "activations/layer6_attention_weight_max": 49.13774490356445,
      "activations/layer6_attention_weight_min": -52.14878463745117,
      "activations/layer7_attention_weight_max": 60.945343017578125,
      "activations/layer7_attention_weight_min": -65.0653305053711,
      "activations/layer8_attention_weight_max": 48.02813720703125,
      "activations/layer8_attention_weight_min": -51.73868942260742,
      "activations/layer9_attention_weight_max": 43.43465805053711,
      "activations/layer9_attention_weight_min": -46.79093551635742,
      "epoch": 19.07,
      "learning_rate": 2.7297348484848484e-05,
      "loss": 2.7267,
      "step": 328200
    },
    {
      "activations/layer0_attention_weight_max": 15.291576385498047,
      "activations/layer0_attention_weight_min": -13.670238494873047,
      "activations/layer10_attention_weight_max": 33.9117546081543,
      "activations/layer10_attention_weight_min": -35.09425735473633,
      "activations/layer11_attention_weight_max": 31.44313621520996,
      "activations/layer11_attention_weight_min": -32.73219299316406,
      "activations/layer12_attention_weight_max": 19.295278549194336,
      "activations/layer12_attention_weight_min": -24.451066970825195,
      "activations/layer13_attention_weight_max": 30.779190063476562,
      "activations/layer13_attention_weight_min": -26.56145668029785,
      "activations/layer14_attention_weight_max": 32.86962890625,
      "activations/layer14_attention_weight_min": -30.853233337402344,
      "activations/layer15_attention_weight_max": 30.3472957611084,
      "activations/layer15_attention_weight_min": -32.839290618896484,
      "activations/layer16_attention_weight_max": 33.1400146484375,
      "activations/layer16_attention_weight_min": -33.613651275634766,
      "activations/layer17_attention_weight_max": 51.2273063659668,
      "activations/layer17_attention_weight_min": -46.13053894042969,
      "activations/layer18_attention_weight_max": 44.92457580566406,
      "activations/layer18_attention_weight_min": -38.80620193481445,
      "activations/layer19_attention_weight_max": 23.806730270385742,
      "activations/layer19_attention_weight_min": -23.549299240112305,
      "activations/layer1_attention_weight_max": 16.372407913208008,
      "activations/layer1_attention_weight_min": -15.506796836853027,
      "activations/layer20_attention_weight_max": 21.063404083251953,
      "activations/layer20_attention_weight_min": -25.115922927856445,
      "activations/layer21_attention_weight_max": 31.513851165771484,
      "activations/layer21_attention_weight_min": -22.76749610900879,
      "activations/layer22_attention_weight_max": 26.09345054626465,
      "activations/layer22_attention_weight_min": -28.322399139404297,
      "activations/layer23_attention_weight_max": 34.38759994506836,
      "activations/layer23_attention_weight_min": -24.802593231201172,
      "activations/layer2_attention_weight_max": 31.05493927001953,
      "activations/layer2_attention_weight_min": -28.47261619567871,
      "activations/layer3_attention_weight_max": 81.0025863647461,
      "activations/layer3_attention_weight_min": -79.47887420654297,
      "activations/layer4_attention_weight_max": 78.40740203857422,
      "activations/layer4_attention_weight_min": -80.06725311279297,
      "activations/layer5_attention_weight_max": 60.926082611083984,
      "activations/layer5_attention_weight_min": -75.25993347167969,
      "activations/layer6_attention_weight_max": 47.16508102416992,
      "activations/layer6_attention_weight_min": -49.345176696777344,
      "activations/layer7_attention_weight_max": 67.45221710205078,
      "activations/layer7_attention_weight_min": -61.74357986450195,
      "activations/layer8_attention_weight_max": 44.11435317993164,
      "activations/layer8_attention_weight_min": -46.5112190246582,
      "activations/layer9_attention_weight_max": 39.720458984375,
      "activations/layer9_attention_weight_min": -42.98603439331055,
      "epoch": 19.07,
      "learning_rate": 2.727840909090909e-05,
      "loss": 2.7257,
      "step": 328250
    },
    {
      "activations/layer0_attention_weight_max": 15.137495994567871,
      "activations/layer0_attention_weight_min": -13.747651100158691,
      "activations/layer10_attention_weight_max": 37.37246322631836,
      "activations/layer10_attention_weight_min": -37.27922439575195,
      "activations/layer11_attention_weight_max": 35.78079605102539,
      "activations/layer11_attention_weight_min": -36.112220764160156,
      "activations/layer12_attention_weight_max": 18.920452117919922,
      "activations/layer12_attention_weight_min": -24.922666549682617,
      "activations/layer13_attention_weight_max": 44.30286407470703,
      "activations/layer13_attention_weight_min": -29.226232528686523,
      "activations/layer14_attention_weight_max": 38.49869155883789,
      "activations/layer14_attention_weight_min": -30.87518310546875,
      "activations/layer15_attention_weight_max": 38.294315338134766,
      "activations/layer15_attention_weight_min": -32.384525299072266,
      "activations/layer16_attention_weight_max": 37.04379653930664,
      "activations/layer16_attention_weight_min": -32.77965545654297,
      "activations/layer17_attention_weight_max": 54.533416748046875,
      "activations/layer17_attention_weight_min": -45.6541633605957,
      "activations/layer18_attention_weight_max": 52.016414642333984,
      "activations/layer18_attention_weight_min": -41.35764694213867,
      "activations/layer19_attention_weight_max": 26.54850196838379,
      "activations/layer19_attention_weight_min": -23.277193069458008,
      "activations/layer1_attention_weight_max": 15.672926902770996,
      "activations/layer1_attention_weight_min": -14.429346084594727,
      "activations/layer20_attention_weight_max": 27.96184730529785,
      "activations/layer20_attention_weight_min": -24.07331085205078,
      "activations/layer21_attention_weight_max": 42.80342483520508,
      "activations/layer21_attention_weight_min": -26.17691993713379,
      "activations/layer22_attention_weight_max": 33.96559524536133,
      "activations/layer22_attention_weight_min": -26.11808967590332,
      "activations/layer23_attention_weight_max": 42.02836608886719,
      "activations/layer23_attention_weight_min": -23.544902801513672,
      "activations/layer2_attention_weight_max": 33.06760787963867,
      "activations/layer2_attention_weight_min": -30.414960861206055,
      "activations/layer3_attention_weight_max": 87.95833587646484,
      "activations/layer3_attention_weight_min": -83.91596221923828,
      "activations/layer4_attention_weight_max": 84.29631805419922,
      "activations/layer4_attention_weight_min": -81.27162170410156,
      "activations/layer5_attention_weight_max": 62.921791076660156,
      "activations/layer5_attention_weight_min": -71.58706665039062,
      "activations/layer6_attention_weight_max": 49.83045196533203,
      "activations/layer6_attention_weight_min": -49.86121368408203,
      "activations/layer7_attention_weight_max": 64.7796630859375,
      "activations/layer7_attention_weight_min": -63.43503189086914,
      "activations/layer8_attention_weight_max": 47.27727127075195,
      "activations/layer8_attention_weight_min": -48.34297180175781,
      "activations/layer9_attention_weight_max": 45.50228500366211,
      "activations/layer9_attention_weight_min": -44.98717498779297,
      "epoch": 19.08,
      "learning_rate": 2.7259469696969698e-05,
      "loss": 2.7059,
      "step": 328300
    },
    {
      "activations/layer0_attention_weight_max": 15.035874366760254,
      "activations/layer0_attention_weight_min": -13.666706085205078,
      "activations/layer10_attention_weight_max": 35.2479133605957,
      "activations/layer10_attention_weight_min": -35.608558654785156,
      "activations/layer11_attention_weight_max": 34.30756759643555,
      "activations/layer11_attention_weight_min": -34.480472564697266,
      "activations/layer12_attention_weight_max": 19.115659713745117,
      "activations/layer12_attention_weight_min": -29.510103225708008,
      "activations/layer13_attention_weight_max": 28.897323608398438,
      "activations/layer13_attention_weight_min": -26.301298141479492,
      "activations/layer14_attention_weight_max": 33.79359436035156,
      "activations/layer14_attention_weight_min": -29.262685775756836,
      "activations/layer15_attention_weight_max": 29.7321834564209,
      "activations/layer15_attention_weight_min": -30.343441009521484,
      "activations/layer16_attention_weight_max": 32.695655822753906,
      "activations/layer16_attention_weight_min": -31.371021270751953,
      "activations/layer17_attention_weight_max": 46.1951789855957,
      "activations/layer17_attention_weight_min": -44.68392562866211,
      "activations/layer18_attention_weight_max": 42.7401123046875,
      "activations/layer18_attention_weight_min": -39.177703857421875,
      "activations/layer19_attention_weight_max": 23.156312942504883,
      "activations/layer19_attention_weight_min": -22.856544494628906,
      "activations/layer1_attention_weight_max": 15.714217185974121,
      "activations/layer1_attention_weight_min": -15.719193458557129,
      "activations/layer20_attention_weight_max": 22.679147720336914,
      "activations/layer20_attention_weight_min": -23.84205436706543,
      "activations/layer21_attention_weight_max": 31.528234481811523,
      "activations/layer21_attention_weight_min": -23.675968170166016,
      "activations/layer22_attention_weight_max": 28.267833709716797,
      "activations/layer22_attention_weight_min": -27.11414909362793,
      "activations/layer23_attention_weight_max": 32.80188751220703,
      "activations/layer23_attention_weight_min": -24.514049530029297,
      "activations/layer2_attention_weight_max": 32.03506851196289,
      "activations/layer2_attention_weight_min": -31.872844696044922,
      "activations/layer3_attention_weight_max": 87.23926544189453,
      "activations/layer3_attention_weight_min": -86.01521301269531,
      "activations/layer4_attention_weight_max": 84.60230255126953,
      "activations/layer4_attention_weight_min": -84.63280487060547,
      "activations/layer5_attention_weight_max": 63.887123107910156,
      "activations/layer5_attention_weight_min": -69.8171157836914,
      "activations/layer6_attention_weight_max": 52.21367645263672,
      "activations/layer6_attention_weight_min": -50.28726577758789,
      "activations/layer7_attention_weight_max": 67.5645751953125,
      "activations/layer7_attention_weight_min": -65.0375747680664,
      "activations/layer8_attention_weight_max": 47.14374542236328,
      "activations/layer8_attention_weight_min": -52.555938720703125,
      "activations/layer9_attention_weight_max": 42.511165618896484,
      "activations/layer9_attention_weight_min": -46.8075065612793,
      "epoch": 19.08,
      "learning_rate": 2.72405303030303e-05,
      "loss": 2.7297,
      "step": 328350
    },
    {
      "activations/layer0_attention_weight_max": 15.494399070739746,
      "activations/layer0_attention_weight_min": -13.581270217895508,
      "activations/layer10_attention_weight_max": 33.76497268676758,
      "activations/layer10_attention_weight_min": -34.0302848815918,
      "activations/layer11_attention_weight_max": 30.821086883544922,
      "activations/layer11_attention_weight_min": -35.71202850341797,
      "activations/layer12_attention_weight_max": 18.41916847229004,
      "activations/layer12_attention_weight_min": -24.967445373535156,
      "activations/layer13_attention_weight_max": 38.318511962890625,
      "activations/layer13_attention_weight_min": -28.190603256225586,
      "activations/layer14_attention_weight_max": 33.74691390991211,
      "activations/layer14_attention_weight_min": -32.635093688964844,
      "activations/layer15_attention_weight_max": 32.09147644042969,
      "activations/layer15_attention_weight_min": -33.582847595214844,
      "activations/layer16_attention_weight_max": 34.023658752441406,
      "activations/layer16_attention_weight_min": -33.23757553100586,
      "activations/layer17_attention_weight_max": 53.1827507019043,
      "activations/layer17_attention_weight_min": -46.103694915771484,
      "activations/layer18_attention_weight_max": 46.49165725708008,
      "activations/layer18_attention_weight_min": -41.62027359008789,
      "activations/layer19_attention_weight_max": 24.514501571655273,
      "activations/layer19_attention_weight_min": -22.591169357299805,
      "activations/layer1_attention_weight_max": 17.095294952392578,
      "activations/layer1_attention_weight_min": -13.226710319519043,
      "activations/layer20_attention_weight_max": 20.94076156616211,
      "activations/layer20_attention_weight_min": -21.309608459472656,
      "activations/layer21_attention_weight_max": 32.73284912109375,
      "activations/layer21_attention_weight_min": -20.278512954711914,
      "activations/layer22_attention_weight_max": 28.70069122314453,
      "activations/layer22_attention_weight_min": -24.91908836364746,
      "activations/layer23_attention_weight_max": 37.08116149902344,
      "activations/layer23_attention_weight_min": -23.459203720092773,
      "activations/layer2_attention_weight_max": 34.371360778808594,
      "activations/layer2_attention_weight_min": -29.35391616821289,
      "activations/layer3_attention_weight_max": 81.32571411132812,
      "activations/layer3_attention_weight_min": -84.71293640136719,
      "activations/layer4_attention_weight_max": 77.80607604980469,
      "activations/layer4_attention_weight_min": -78.06242370605469,
      "activations/layer5_attention_weight_max": 59.474761962890625,
      "activations/layer5_attention_weight_min": -70.88687896728516,
      "activations/layer6_attention_weight_max": 46.637046813964844,
      "activations/layer6_attention_weight_min": -48.03068161010742,
      "activations/layer7_attention_weight_max": 62.9337043762207,
      "activations/layer7_attention_weight_min": -62.829559326171875,
      "activations/layer8_attention_weight_max": 44.55607986450195,
      "activations/layer8_attention_weight_min": -49.6330451965332,
      "activations/layer9_attention_weight_max": 42.51719284057617,
      "activations/layer9_attention_weight_min": -44.21409606933594,
      "epoch": 19.08,
      "learning_rate": 2.7221590909090907e-05,
      "loss": 2.72,
      "step": 328400
    },
    {
      "activations/layer0_attention_weight_max": 15.788665771484375,
      "activations/layer0_attention_weight_min": -13.309737205505371,
      "activations/layer10_attention_weight_max": 35.343868255615234,
      "activations/layer10_attention_weight_min": -36.431640625,
      "activations/layer11_attention_weight_max": 33.26019287109375,
      "activations/layer11_attention_weight_min": -34.47028350830078,
      "activations/layer12_attention_weight_max": 19.596134185791016,
      "activations/layer12_attention_weight_min": -31.151594161987305,
      "activations/layer13_attention_weight_max": 39.50298309326172,
      "activations/layer13_attention_weight_min": -30.0827693939209,
      "activations/layer14_attention_weight_max": 38.192325592041016,
      "activations/layer14_attention_weight_min": -33.112022399902344,
      "activations/layer15_attention_weight_max": 35.86610412597656,
      "activations/layer15_attention_weight_min": -32.176788330078125,
      "activations/layer16_attention_weight_max": 36.020111083984375,
      "activations/layer16_attention_weight_min": -34.13221740722656,
      "activations/layer17_attention_weight_max": 53.43541717529297,
      "activations/layer17_attention_weight_min": -45.548828125,
      "activations/layer18_attention_weight_max": 49.18111038208008,
      "activations/layer18_attention_weight_min": -40.31814956665039,
      "activations/layer19_attention_weight_max": 26.389806747436523,
      "activations/layer19_attention_weight_min": -23.393274307250977,
      "activations/layer1_attention_weight_max": 15.650232315063477,
      "activations/layer1_attention_weight_min": -14.115653991699219,
      "activations/layer20_attention_weight_max": 26.333080291748047,
      "activations/layer20_attention_weight_min": -22.979887008666992,
      "activations/layer21_attention_weight_max": 34.288330078125,
      "activations/layer21_attention_weight_min": -25.934118270874023,
      "activations/layer22_attention_weight_max": 30.278100967407227,
      "activations/layer22_attention_weight_min": -25.50265121459961,
      "activations/layer23_attention_weight_max": 42.22827911376953,
      "activations/layer23_attention_weight_min": -24.276220321655273,
      "activations/layer2_attention_weight_max": 32.52578353881836,
      "activations/layer2_attention_weight_min": -29.87591552734375,
      "activations/layer3_attention_weight_max": 82.92833709716797,
      "activations/layer3_attention_weight_min": -81.45902252197266,
      "activations/layer4_attention_weight_max": 81.20315551757812,
      "activations/layer4_attention_weight_min": -80.24076080322266,
      "activations/layer5_attention_weight_max": 59.88567352294922,
      "activations/layer5_attention_weight_min": -69.64028930664062,
      "activations/layer6_attention_weight_max": 49.17418670654297,
      "activations/layer6_attention_weight_min": -50.587181091308594,
      "activations/layer7_attention_weight_max": 64.15478515625,
      "activations/layer7_attention_weight_min": -64.62264251708984,
      "activations/layer8_attention_weight_max": 46.083866119384766,
      "activations/layer8_attention_weight_min": -48.667083740234375,
      "activations/layer9_attention_weight_max": 43.006832122802734,
      "activations/layer9_attention_weight_min": -45.82003402709961,
      "epoch": 19.08,
      "learning_rate": 2.7202651515151512e-05,
      "loss": 2.7301,
      "step": 328450
    },
    {
      "activations/layer0_attention_weight_max": 15.271196365356445,
      "activations/layer0_attention_weight_min": -13.387229919433594,
      "activations/layer10_attention_weight_max": 37.34119415283203,
      "activations/layer10_attention_weight_min": -36.532684326171875,
      "activations/layer11_attention_weight_max": 34.493804931640625,
      "activations/layer11_attention_weight_min": -35.00436019897461,
      "activations/layer12_attention_weight_max": 19.904891967773438,
      "activations/layer12_attention_weight_min": -26.126726150512695,
      "activations/layer13_attention_weight_max": 38.25911331176758,
      "activations/layer13_attention_weight_min": -26.889371871948242,
      "activations/layer14_attention_weight_max": 36.87893295288086,
      "activations/layer14_attention_weight_min": -30.505369186401367,
      "activations/layer15_attention_weight_max": 34.93058395385742,
      "activations/layer15_attention_weight_min": -29.737829208374023,
      "activations/layer16_attention_weight_max": 33.91808319091797,
      "activations/layer16_attention_weight_min": -31.865327835083008,
      "activations/layer17_attention_weight_max": 53.52088165283203,
      "activations/layer17_attention_weight_min": -44.92618942260742,
      "activations/layer18_attention_weight_max": 48.6046142578125,
      "activations/layer18_attention_weight_min": -38.66921615600586,
      "activations/layer19_attention_weight_max": 24.940265655517578,
      "activations/layer19_attention_weight_min": -21.970487594604492,
      "activations/layer1_attention_weight_max": 16.24484634399414,
      "activations/layer1_attention_weight_min": -14.233596801757812,
      "activations/layer20_attention_weight_max": 24.957658767700195,
      "activations/layer20_attention_weight_min": -20.084569931030273,
      "activations/layer21_attention_weight_max": 38.88999557495117,
      "activations/layer21_attention_weight_min": -22.772512435913086,
      "activations/layer22_attention_weight_max": 28.87358283996582,
      "activations/layer22_attention_weight_min": -26.944129943847656,
      "activations/layer23_attention_weight_max": 34.89902114868164,
      "activations/layer23_attention_weight_min": -25.179845809936523,
      "activations/layer2_attention_weight_max": 33.86674880981445,
      "activations/layer2_attention_weight_min": -31.665802001953125,
      "activations/layer3_attention_weight_max": 87.8971176147461,
      "activations/layer3_attention_weight_min": -86.69924926757812,
      "activations/layer4_attention_weight_max": 84.99925231933594,
      "activations/layer4_attention_weight_min": -83.73545837402344,
      "activations/layer5_attention_weight_max": 63.07032775878906,
      "activations/layer5_attention_weight_min": -72.29481506347656,
      "activations/layer6_attention_weight_max": 51.608131408691406,
      "activations/layer6_attention_weight_min": -51.095706939697266,
      "activations/layer7_attention_weight_max": 69.00406646728516,
      "activations/layer7_attention_weight_min": -63.25518035888672,
      "activations/layer8_attention_weight_max": 45.84723663330078,
      "activations/layer8_attention_weight_min": -47.99109649658203,
      "activations/layer9_attention_weight_max": 43.84330368041992,
      "activations/layer9_attention_weight_min": -44.262725830078125,
      "epoch": 19.09,
      "learning_rate": 2.718371212121212e-05,
      "loss": 2.7366,
      "step": 328500
    },
    {
      "activations/layer0_attention_weight_max": 15.427306175231934,
      "activations/layer0_attention_weight_min": -14.679703712463379,
      "activations/layer10_attention_weight_max": 35.69673538208008,
      "activations/layer10_attention_weight_min": -35.69635009765625,
      "activations/layer11_attention_weight_max": 32.637107849121094,
      "activations/layer11_attention_weight_min": -35.600990295410156,
      "activations/layer12_attention_weight_max": 20.200334548950195,
      "activations/layer12_attention_weight_min": -28.74616050720215,
      "activations/layer13_attention_weight_max": 37.369598388671875,
      "activations/layer13_attention_weight_min": -29.360530853271484,
      "activations/layer14_attention_weight_max": 37.11811447143555,
      "activations/layer14_attention_weight_min": -31.03997039794922,
      "activations/layer15_attention_weight_max": 33.66266632080078,
      "activations/layer15_attention_weight_min": -31.789674758911133,
      "activations/layer16_attention_weight_max": 36.767608642578125,
      "activations/layer16_attention_weight_min": -33.23904037475586,
      "activations/layer17_attention_weight_max": 52.45743179321289,
      "activations/layer17_attention_weight_min": -44.61699295043945,
      "activations/layer18_attention_weight_max": 50.13221740722656,
      "activations/layer18_attention_weight_min": -43.2866325378418,
      "activations/layer19_attention_weight_max": 23.622722625732422,
      "activations/layer19_attention_weight_min": -22.267492294311523,
      "activations/layer1_attention_weight_max": 15.735260963439941,
      "activations/layer1_attention_weight_min": -13.255807876586914,
      "activations/layer20_attention_weight_max": 22.630924224853516,
      "activations/layer20_attention_weight_min": -22.68944549560547,
      "activations/layer21_attention_weight_max": 36.70933151245117,
      "activations/layer21_attention_weight_min": -23.84311294555664,
      "activations/layer22_attention_weight_max": 28.58272361755371,
      "activations/layer22_attention_weight_min": -26.05434226989746,
      "activations/layer23_attention_weight_max": 33.97523880004883,
      "activations/layer23_attention_weight_min": -24.829792022705078,
      "activations/layer2_attention_weight_max": 28.936796188354492,
      "activations/layer2_attention_weight_min": -28.216014862060547,
      "activations/layer3_attention_weight_max": 76.81985473632812,
      "activations/layer3_attention_weight_min": -83.29277038574219,
      "activations/layer4_attention_weight_max": 77.55713653564453,
      "activations/layer4_attention_weight_min": -75.80056762695312,
      "activations/layer5_attention_weight_max": 59.5504264831543,
      "activations/layer5_attention_weight_min": -68.64811706542969,
      "activations/layer6_attention_weight_max": 46.68330764770508,
      "activations/layer6_attention_weight_min": -49.9305305480957,
      "activations/layer7_attention_weight_max": 62.6075325012207,
      "activations/layer7_attention_weight_min": -65.64666748046875,
      "activations/layer8_attention_weight_max": 45.22126007080078,
      "activations/layer8_attention_weight_min": -50.56950759887695,
      "activations/layer9_attention_weight_max": 41.90272903442383,
      "activations/layer9_attention_weight_min": -44.63978576660156,
      "epoch": 19.09,
      "learning_rate": 2.7164772727272725e-05,
      "loss": 2.7219,
      "step": 328550
    },
    {
      "activations/layer0_attention_weight_max": 14.933247566223145,
      "activations/layer0_attention_weight_min": -13.726263999938965,
      "activations/layer10_attention_weight_max": 35.68498992919922,
      "activations/layer10_attention_weight_min": -34.790870666503906,
      "activations/layer11_attention_weight_max": 33.109867095947266,
      "activations/layer11_attention_weight_min": -34.50780487060547,
      "activations/layer12_attention_weight_max": 18.69785499572754,
      "activations/layer12_attention_weight_min": -23.825971603393555,
      "activations/layer13_attention_weight_max": 31.79474639892578,
      "activations/layer13_attention_weight_min": -25.630504608154297,
      "activations/layer14_attention_weight_max": 36.1141242980957,
      "activations/layer14_attention_weight_min": -30.44098663330078,
      "activations/layer15_attention_weight_max": 33.652530670166016,
      "activations/layer15_attention_weight_min": -32.280555725097656,
      "activations/layer16_attention_weight_max": 36.12049865722656,
      "activations/layer16_attention_weight_min": -32.94572067260742,
      "activations/layer17_attention_weight_max": 53.39371109008789,
      "activations/layer17_attention_weight_min": -45.4215202331543,
      "activations/layer18_attention_weight_max": 48.0247917175293,
      "activations/layer18_attention_weight_min": -43.1437873840332,
      "activations/layer19_attention_weight_max": 25.137998580932617,
      "activations/layer19_attention_weight_min": -22.395950317382812,
      "activations/layer1_attention_weight_max": 16.381114959716797,
      "activations/layer1_attention_weight_min": -14.148468971252441,
      "activations/layer20_attention_weight_max": 23.234081268310547,
      "activations/layer20_attention_weight_min": -21.36448860168457,
      "activations/layer21_attention_weight_max": 38.1367073059082,
      "activations/layer21_attention_weight_min": -22.2352237701416,
      "activations/layer22_attention_weight_max": 33.00788116455078,
      "activations/layer22_attention_weight_min": -25.024391174316406,
      "activations/layer23_attention_weight_max": 41.414939880371094,
      "activations/layer23_attention_weight_min": -25.57607078552246,
      "activations/layer2_attention_weight_max": 29.921772003173828,
      "activations/layer2_attention_weight_min": -27.663536071777344,
      "activations/layer3_attention_weight_max": 80.06150817871094,
      "activations/layer3_attention_weight_min": -80.62862396240234,
      "activations/layer4_attention_weight_max": 82.13817596435547,
      "activations/layer4_attention_weight_min": -77.13066864013672,
      "activations/layer5_attention_weight_max": 62.1883544921875,
      "activations/layer5_attention_weight_min": -70.09530639648438,
      "activations/layer6_attention_weight_max": 47.913917541503906,
      "activations/layer6_attention_weight_min": -50.553165435791016,
      "activations/layer7_attention_weight_max": 66.97837829589844,
      "activations/layer7_attention_weight_min": -61.14827346801758,
      "activations/layer8_attention_weight_max": 43.21779251098633,
      "activations/layer8_attention_weight_min": -47.80588912963867,
      "activations/layer9_attention_weight_max": 41.68596267700195,
      "activations/layer9_attention_weight_min": -43.5733528137207,
      "epoch": 19.09,
      "learning_rate": 2.7145833333333334e-05,
      "loss": 2.7291,
      "step": 328600
    },
    {
      "activations/layer0_attention_weight_max": 15.895439147949219,
      "activations/layer0_attention_weight_min": -13.658470153808594,
      "activations/layer10_attention_weight_max": 31.289596557617188,
      "activations/layer10_attention_weight_min": -33.18758010864258,
      "activations/layer11_attention_weight_max": 28.94082260131836,
      "activations/layer11_attention_weight_min": -30.802898406982422,
      "activations/layer12_attention_weight_max": 17.344120025634766,
      "activations/layer12_attention_weight_min": -25.879798889160156,
      "activations/layer13_attention_weight_max": 27.539522171020508,
      "activations/layer13_attention_weight_min": -25.68598175048828,
      "activations/layer14_attention_weight_max": 30.127103805541992,
      "activations/layer14_attention_weight_min": -30.40069580078125,
      "activations/layer15_attention_weight_max": 28.495025634765625,
      "activations/layer15_attention_weight_min": -31.725618362426758,
      "activations/layer16_attention_weight_max": 30.269424438476562,
      "activations/layer16_attention_weight_min": -32.51704788208008,
      "activations/layer17_attention_weight_max": 45.45668411254883,
      "activations/layer17_attention_weight_min": -43.88734817504883,
      "activations/layer18_attention_weight_max": 40.32724380493164,
      "activations/layer18_attention_weight_min": -39.44817352294922,
      "activations/layer19_attention_weight_max": 22.634929656982422,
      "activations/layer19_attention_weight_min": -24.948225021362305,
      "activations/layer1_attention_weight_max": 16.10948371887207,
      "activations/layer1_attention_weight_min": -12.844836235046387,
      "activations/layer20_attention_weight_max": 23.85326385498047,
      "activations/layer20_attention_weight_min": -23.4659366607666,
      "activations/layer21_attention_weight_max": 32.854644775390625,
      "activations/layer21_attention_weight_min": -22.980188369750977,
      "activations/layer22_attention_weight_max": 24.803144454956055,
      "activations/layer22_attention_weight_min": -25.147018432617188,
      "activations/layer23_attention_weight_max": 30.703693389892578,
      "activations/layer23_attention_weight_min": -25.75416374206543,
      "activations/layer2_attention_weight_max": 31.86389923095703,
      "activations/layer2_attention_weight_min": -29.87710952758789,
      "activations/layer3_attention_weight_max": 81.53899383544922,
      "activations/layer3_attention_weight_min": -82.70903015136719,
      "activations/layer4_attention_weight_max": 77.79495239257812,
      "activations/layer4_attention_weight_min": -77.40265655517578,
      "activations/layer5_attention_weight_max": 59.660362243652344,
      "activations/layer5_attention_weight_min": -67.60917663574219,
      "activations/layer6_attention_weight_max": 45.005367279052734,
      "activations/layer6_attention_weight_min": -45.799720764160156,
      "activations/layer7_attention_weight_max": 62.76288604736328,
      "activations/layer7_attention_weight_min": -59.22969436645508,
      "activations/layer8_attention_weight_max": 42.73526382446289,
      "activations/layer8_attention_weight_min": -47.24434280395508,
      "activations/layer9_attention_weight_max": 40.672061920166016,
      "activations/layer9_attention_weight_min": -44.523616790771484,
      "epoch": 19.1,
      "learning_rate": 2.7126893939393935e-05,
      "loss": 2.7218,
      "step": 328650
    },
    {
      "activations/layer0_attention_weight_max": 15.385586738586426,
      "activations/layer0_attention_weight_min": -13.462374687194824,
      "activations/layer10_attention_weight_max": 34.39719772338867,
      "activations/layer10_attention_weight_min": -34.61634063720703,
      "activations/layer11_attention_weight_max": 31.711652755737305,
      "activations/layer11_attention_weight_min": -34.56355285644531,
      "activations/layer12_attention_weight_max": 18.56739616394043,
      "activations/layer12_attention_weight_min": -23.6397705078125,
      "activations/layer13_attention_weight_max": 31.316686630249023,
      "activations/layer13_attention_weight_min": -28.047679901123047,
      "activations/layer14_attention_weight_max": 32.543148040771484,
      "activations/layer14_attention_weight_min": -30.74417495727539,
      "activations/layer15_attention_weight_max": 30.859596252441406,
      "activations/layer15_attention_weight_min": -30.434797286987305,
      "activations/layer16_attention_weight_max": 33.21797180175781,
      "activations/layer16_attention_weight_min": -33.39578628540039,
      "activations/layer17_attention_weight_max": 50.33644104003906,
      "activations/layer17_attention_weight_min": -47.222999572753906,
      "activations/layer18_attention_weight_max": 44.38786697387695,
      "activations/layer18_attention_weight_min": -42.602352142333984,
      "activations/layer19_attention_weight_max": 22.095922470092773,
      "activations/layer19_attention_weight_min": -24.31026268005371,
      "activations/layer1_attention_weight_max": 15.590981483459473,
      "activations/layer1_attention_weight_min": -15.270732879638672,
      "activations/layer20_attention_weight_max": 22.9476318359375,
      "activations/layer20_attention_weight_min": -26.838518142700195,
      "activations/layer21_attention_weight_max": 41.7913818359375,
      "activations/layer21_attention_weight_min": -22.26067352294922,
      "activations/layer22_attention_weight_max": 32.54279327392578,
      "activations/layer22_attention_weight_min": -30.83745574951172,
      "activations/layer23_attention_weight_max": 37.40410232543945,
      "activations/layer23_attention_weight_min": -25.004329681396484,
      "activations/layer2_attention_weight_max": 34.65154266357422,
      "activations/layer2_attention_weight_min": -32.8033561706543,
      "activations/layer3_attention_weight_max": 83.96954345703125,
      "activations/layer3_attention_weight_min": -89.27835083007812,
      "activations/layer4_attention_weight_max": 82.25505828857422,
      "activations/layer4_attention_weight_min": -84.22649383544922,
      "activations/layer5_attention_weight_max": 66.24490356445312,
      "activations/layer5_attention_weight_min": -72.15724182128906,
      "activations/layer6_attention_weight_max": 51.09284210205078,
      "activations/layer6_attention_weight_min": -53.764488220214844,
      "activations/layer7_attention_weight_max": 66.90595245361328,
      "activations/layer7_attention_weight_min": -65.95533752441406,
      "activations/layer8_attention_weight_max": 46.24386978149414,
      "activations/layer8_attention_weight_min": -51.23054122924805,
      "activations/layer9_attention_weight_max": 43.51997375488281,
      "activations/layer9_attention_weight_min": -46.26871871948242,
      "epoch": 19.1,
      "learning_rate": 2.7107954545454543e-05,
      "loss": 2.7359,
      "step": 328700
    },
    {
      "activations/layer0_attention_weight_max": 15.775248527526855,
      "activations/layer0_attention_weight_min": -13.481069564819336,
      "activations/layer10_attention_weight_max": 36.486148834228516,
      "activations/layer10_attention_weight_min": -36.16828155517578,
      "activations/layer11_attention_weight_max": 33.67298126220703,
      "activations/layer11_attention_weight_min": -36.15408706665039,
      "activations/layer12_attention_weight_max": 20.261402130126953,
      "activations/layer12_attention_weight_min": -23.079696655273438,
      "activations/layer13_attention_weight_max": 39.6413459777832,
      "activations/layer13_attention_weight_min": -28.46904182434082,
      "activations/layer14_attention_weight_max": 39.09859848022461,
      "activations/layer14_attention_weight_min": -32.162750244140625,
      "activations/layer15_attention_weight_max": 34.4053955078125,
      "activations/layer15_attention_weight_min": -32.95719528198242,
      "activations/layer16_attention_weight_max": 36.26744842529297,
      "activations/layer16_attention_weight_min": -34.30887222290039,
      "activations/layer17_attention_weight_max": 55.05167770385742,
      "activations/layer17_attention_weight_min": -48.6994514465332,
      "activations/layer18_attention_weight_max": 48.661643981933594,
      "activations/layer18_attention_weight_min": -43.129066467285156,
      "activations/layer19_attention_weight_max": 23.75745391845703,
      "activations/layer19_attention_weight_min": -23.43419075012207,
      "activations/layer1_attention_weight_max": 16.276138305664062,
      "activations/layer1_attention_weight_min": -13.307677268981934,
      "activations/layer20_attention_weight_max": 21.633251190185547,
      "activations/layer20_attention_weight_min": -21.157512664794922,
      "activations/layer21_attention_weight_max": 41.382259368896484,
      "activations/layer21_attention_weight_min": -25.82255744934082,
      "activations/layer22_attention_weight_max": 31.51953125,
      "activations/layer22_attention_weight_min": -26.96981430053711,
      "activations/layer23_attention_weight_max": 38.83495330810547,
      "activations/layer23_attention_weight_min": -23.566959381103516,
      "activations/layer2_attention_weight_max": 32.627830505371094,
      "activations/layer2_attention_weight_min": -30.13814926147461,
      "activations/layer3_attention_weight_max": 84.61799621582031,
      "activations/layer3_attention_weight_min": -83.40058898925781,
      "activations/layer4_attention_weight_max": 82.52819061279297,
      "activations/layer4_attention_weight_min": -79.3525619506836,
      "activations/layer5_attention_weight_max": 59.522239685058594,
      "activations/layer5_attention_weight_min": -69.35397338867188,
      "activations/layer6_attention_weight_max": 48.559452056884766,
      "activations/layer6_attention_weight_min": -47.67393493652344,
      "activations/layer7_attention_weight_max": 64.79934692382812,
      "activations/layer7_attention_weight_min": -62.89776611328125,
      "activations/layer8_attention_weight_max": 48.69478225708008,
      "activations/layer8_attention_weight_min": -51.89270782470703,
      "activations/layer9_attention_weight_max": 48.65693283081055,
      "activations/layer9_attention_weight_min": -46.79438400268555,
      "epoch": 19.1,
      "learning_rate": 2.7089015151515148e-05,
      "loss": 2.7269,
      "step": 328750
    },
    {
      "activations/layer0_attention_weight_max": 15.404507637023926,
      "activations/layer0_attention_weight_min": -13.815455436706543,
      "activations/layer10_attention_weight_max": 39.434173583984375,
      "activations/layer10_attention_weight_min": -39.3574333190918,
      "activations/layer11_attention_weight_max": 36.594200134277344,
      "activations/layer11_attention_weight_min": -37.25164794921875,
      "activations/layer12_attention_weight_max": 21.032562255859375,
      "activations/layer12_attention_weight_min": -25.288537979125977,
      "activations/layer13_attention_weight_max": 56.142852783203125,
      "activations/layer13_attention_weight_min": -31.4083309173584,
      "activations/layer14_attention_weight_max": 42.70620346069336,
      "activations/layer14_attention_weight_min": -35.456207275390625,
      "activations/layer15_attention_weight_max": 45.75114822387695,
      "activations/layer15_attention_weight_min": -34.11100769042969,
      "activations/layer16_attention_weight_max": 38.32390594482422,
      "activations/layer16_attention_weight_min": -36.331756591796875,
      "activations/layer17_attention_weight_max": 59.95287322998047,
      "activations/layer17_attention_weight_min": -53.0545539855957,
      "activations/layer18_attention_weight_max": 53.88656234741211,
      "activations/layer18_attention_weight_min": -50.09262466430664,
      "activations/layer19_attention_weight_max": 25.880386352539062,
      "activations/layer19_attention_weight_min": -25.850269317626953,
      "activations/layer1_attention_weight_max": 16.121015548706055,
      "activations/layer1_attention_weight_min": -13.176142692565918,
      "activations/layer20_attention_weight_max": 27.1888370513916,
      "activations/layer20_attention_weight_min": -25.3430233001709,
      "activations/layer21_attention_weight_max": 44.3159294128418,
      "activations/layer21_attention_weight_min": -22.51564598083496,
      "activations/layer22_attention_weight_max": 30.537033081054688,
      "activations/layer22_attention_weight_min": -26.2985782623291,
      "activations/layer23_attention_weight_max": 35.171897888183594,
      "activations/layer23_attention_weight_min": -23.207759857177734,
      "activations/layer2_attention_weight_max": 31.4332275390625,
      "activations/layer2_attention_weight_min": -30.3885555267334,
      "activations/layer3_attention_weight_max": 84.62894439697266,
      "activations/layer3_attention_weight_min": -87.7848129272461,
      "activations/layer4_attention_weight_max": 79.57091522216797,
      "activations/layer4_attention_weight_min": -83.82427215576172,
      "activations/layer5_attention_weight_max": 56.88727951049805,
      "activations/layer5_attention_weight_min": -70.12600708007812,
      "activations/layer6_attention_weight_max": 46.743228912353516,
      "activations/layer6_attention_weight_min": -50.02872085571289,
      "activations/layer7_attention_weight_max": 64.88589477539062,
      "activations/layer7_attention_weight_min": -68.99141693115234,
      "activations/layer8_attention_weight_max": 49.27241897583008,
      "activations/layer8_attention_weight_min": -53.12480926513672,
      "activations/layer9_attention_weight_max": 51.90266036987305,
      "activations/layer9_attention_weight_min": -53.37690353393555,
      "epoch": 19.11,
      "learning_rate": 2.7070075757575757e-05,
      "loss": 2.7253,
      "step": 328800
    },
    {
      "activations/layer0_attention_weight_max": 15.238618850708008,
      "activations/layer0_attention_weight_min": -13.619241714477539,
      "activations/layer10_attention_weight_max": 35.5109748840332,
      "activations/layer10_attention_weight_min": -36.9635009765625,
      "activations/layer11_attention_weight_max": 31.935102462768555,
      "activations/layer11_attention_weight_min": -35.23401641845703,
      "activations/layer12_attention_weight_max": 19.05952262878418,
      "activations/layer12_attention_weight_min": -30.889741897583008,
      "activations/layer13_attention_weight_max": 31.70928192138672,
      "activations/layer13_attention_weight_min": -27.553722381591797,
      "activations/layer14_attention_weight_max": 34.40385437011719,
      "activations/layer14_attention_weight_min": -31.906152725219727,
      "activations/layer15_attention_weight_max": 32.19557571411133,
      "activations/layer15_attention_weight_min": -31.223630905151367,
      "activations/layer16_attention_weight_max": 33.5807991027832,
      "activations/layer16_attention_weight_min": -32.83326721191406,
      "activations/layer17_attention_weight_max": 52.91582489013672,
      "activations/layer17_attention_weight_min": -46.538719177246094,
      "activations/layer18_attention_weight_max": 47.89033889770508,
      "activations/layer18_attention_weight_min": -40.97802734375,
      "activations/layer19_attention_weight_max": 23.72478675842285,
      "activations/layer19_attention_weight_min": -22.80904769897461,
      "activations/layer1_attention_weight_max": 15.698369026184082,
      "activations/layer1_attention_weight_min": -14.590167999267578,
      "activations/layer20_attention_weight_max": 21.453105926513672,
      "activations/layer20_attention_weight_min": -20.91881561279297,
      "activations/layer21_attention_weight_max": 36.85160827636719,
      "activations/layer21_attention_weight_min": -22.672658920288086,
      "activations/layer22_attention_weight_max": 31.32745933532715,
      "activations/layer22_attention_weight_min": -25.530569076538086,
      "activations/layer23_attention_weight_max": 38.32910919189453,
      "activations/layer23_attention_weight_min": -24.979366302490234,
      "activations/layer2_attention_weight_max": 32.387229919433594,
      "activations/layer2_attention_weight_min": -31.659818649291992,
      "activations/layer3_attention_weight_max": 85.65033721923828,
      "activations/layer3_attention_weight_min": -85.86795806884766,
      "activations/layer4_attention_weight_max": 84.11949920654297,
      "activations/layer4_attention_weight_min": -83.05757904052734,
      "activations/layer5_attention_weight_max": 64.56214904785156,
      "activations/layer5_attention_weight_min": -71.8950424194336,
      "activations/layer6_attention_weight_max": 51.403682708740234,
      "activations/layer6_attention_weight_min": -53.43977355957031,
      "activations/layer7_attention_weight_max": 64.41583251953125,
      "activations/layer7_attention_weight_min": -64.74276733398438,
      "activations/layer8_attention_weight_max": 48.61737060546875,
      "activations/layer8_attention_weight_min": -50.73991012573242,
      "activations/layer9_attention_weight_max": 43.72030258178711,
      "activations/layer9_attention_weight_min": -47.73447799682617,
      "epoch": 19.11,
      "learning_rate": 2.7051515151515147e-05,
      "loss": 2.721,
      "step": 328850
    },
    {
      "activations/layer0_attention_weight_max": 15.565710067749023,
      "activations/layer0_attention_weight_min": -14.1239595413208,
      "activations/layer10_attention_weight_max": 36.918792724609375,
      "activations/layer10_attention_weight_min": -38.27804946899414,
      "activations/layer11_attention_weight_max": 38.48960876464844,
      "activations/layer11_attention_weight_min": -37.887447357177734,
      "activations/layer12_attention_weight_max": 18.37204360961914,
      "activations/layer12_attention_weight_min": -30.021543502807617,
      "activations/layer13_attention_weight_max": 32.780609130859375,
      "activations/layer13_attention_weight_min": -28.372257232666016,
      "activations/layer14_attention_weight_max": 35.53411865234375,
      "activations/layer14_attention_weight_min": -30.237335205078125,
      "activations/layer15_attention_weight_max": 31.571687698364258,
      "activations/layer15_attention_weight_min": -30.415205001831055,
      "activations/layer16_attention_weight_max": 33.6649055480957,
      "activations/layer16_attention_weight_min": -31.750944137573242,
      "activations/layer17_attention_weight_max": 53.60900115966797,
      "activations/layer17_attention_weight_min": -46.70484161376953,
      "activations/layer18_attention_weight_max": 49.96361541748047,
      "activations/layer18_attention_weight_min": -42.486576080322266,
      "activations/layer19_attention_weight_max": 25.018035888671875,
      "activations/layer19_attention_weight_min": -24.443044662475586,
      "activations/layer1_attention_weight_max": 15.634916305541992,
      "activations/layer1_attention_weight_min": -13.760029792785645,
      "activations/layer20_attention_weight_max": 23.890825271606445,
      "activations/layer20_attention_weight_min": -22.027246475219727,
      "activations/layer21_attention_weight_max": 37.077476501464844,
      "activations/layer21_attention_weight_min": -22.22927474975586,
      "activations/layer22_attention_weight_max": 33.24799346923828,
      "activations/layer22_attention_weight_min": -27.47174835205078,
      "activations/layer23_attention_weight_max": 35.30128479003906,
      "activations/layer23_attention_weight_min": -23.131175994873047,
      "activations/layer2_attention_weight_max": 29.939815521240234,
      "activations/layer2_attention_weight_min": -29.907642364501953,
      "activations/layer3_attention_weight_max": 89.15144348144531,
      "activations/layer3_attention_weight_min": -87.66511535644531,
      "activations/layer4_attention_weight_max": 88.60222625732422,
      "activations/layer4_attention_weight_min": -86.96993255615234,
      "activations/layer5_attention_weight_max": 67.61650085449219,
      "activations/layer5_attention_weight_min": -73.39550018310547,
      "activations/layer6_attention_weight_max": 51.66248321533203,
      "activations/layer6_attention_weight_min": -50.87198257446289,
      "activations/layer7_attention_weight_max": 73.14093780517578,
      "activations/layer7_attention_weight_min": -66.11703491210938,
      "activations/layer8_attention_weight_max": 50.63331985473633,
      "activations/layer8_attention_weight_min": -53.08624267578125,
      "activations/layer9_attention_weight_max": 58.842140197753906,
      "activations/layer9_attention_weight_min": -53.5990104675293,
      "epoch": 19.11,
      "learning_rate": 2.7032954545454545e-05,
      "loss": 2.7098,
      "step": 328900
    },
    {
      "activations/layer0_attention_weight_max": 15.141006469726562,
      "activations/layer0_attention_weight_min": -13.342513084411621,
      "activations/layer10_attention_weight_max": 36.36135482788086,
      "activations/layer10_attention_weight_min": -37.07893371582031,
      "activations/layer11_attention_weight_max": 34.751461029052734,
      "activations/layer11_attention_weight_min": -35.08631134033203,
      "activations/layer12_attention_weight_max": 20.90939712524414,
      "activations/layer12_attention_weight_min": -25.92780876159668,
      "activations/layer13_attention_weight_max": 35.45573043823242,
      "activations/layer13_attention_weight_min": -26.748821258544922,
      "activations/layer14_attention_weight_max": 36.862770080566406,
      "activations/layer14_attention_weight_min": -30.098608016967773,
      "activations/layer15_attention_weight_max": 37.54536056518555,
      "activations/layer15_attention_weight_min": -30.568204879760742,
      "activations/layer16_attention_weight_max": 36.91111755371094,
      "activations/layer16_attention_weight_min": -34.315181732177734,
      "activations/layer17_attention_weight_max": 53.98807144165039,
      "activations/layer17_attention_weight_min": -46.25230407714844,
      "activations/layer18_attention_weight_max": 50.47761535644531,
      "activations/layer18_attention_weight_min": -41.22935485839844,
      "activations/layer19_attention_weight_max": 30.869489669799805,
      "activations/layer19_attention_weight_min": -24.758546829223633,
      "activations/layer1_attention_weight_max": 15.806774139404297,
      "activations/layer1_attention_weight_min": -13.500038146972656,
      "activations/layer20_attention_weight_max": 28.472026824951172,
      "activations/layer20_attention_weight_min": -23.2990665435791,
      "activations/layer21_attention_weight_max": 37.38035583496094,
      "activations/layer21_attention_weight_min": -24.675806045532227,
      "activations/layer22_attention_weight_max": 31.86690330505371,
      "activations/layer22_attention_weight_min": -24.608552932739258,
      "activations/layer23_attention_weight_max": 46.06251525878906,
      "activations/layer23_attention_weight_min": -24.421323776245117,
      "activations/layer2_attention_weight_max": 28.93326187133789,
      "activations/layer2_attention_weight_min": -29.842565536499023,
      "activations/layer3_attention_weight_max": 81.32528686523438,
      "activations/layer3_attention_weight_min": -82.65589141845703,
      "activations/layer4_attention_weight_max": 80.56560516357422,
      "activations/layer4_attention_weight_min": -81.71697998046875,
      "activations/layer5_attention_weight_max": 64.74764251708984,
      "activations/layer5_attention_weight_min": -73.19265747070312,
      "activations/layer6_attention_weight_max": 50.801944732666016,
      "activations/layer6_attention_weight_min": -53.056297302246094,
      "activations/layer7_attention_weight_max": 67.20852661132812,
      "activations/layer7_attention_weight_min": -66.09496307373047,
      "activations/layer8_attention_weight_max": 49.41576385498047,
      "activations/layer8_attention_weight_min": -52.0003776550293,
      "activations/layer9_attention_weight_max": 43.53315734863281,
      "activations/layer9_attention_weight_min": -46.81452941894531,
      "epoch": 19.11,
      "learning_rate": 2.7014015151515146e-05,
      "loss": 2.7337,
      "step": 328950
    },
    {
      "activations/layer0_attention_weight_max": 15.974270820617676,
      "activations/layer0_attention_weight_min": -13.281999588012695,
      "activations/layer10_attention_weight_max": 38.27158737182617,
      "activations/layer10_attention_weight_min": -39.00151824951172,
      "activations/layer11_attention_weight_max": 36.107032775878906,
      "activations/layer11_attention_weight_min": -38.64325714111328,
      "activations/layer12_attention_weight_max": 19.344266891479492,
      "activations/layer12_attention_weight_min": -25.464818954467773,
      "activations/layer13_attention_weight_max": 32.35643005371094,
      "activations/layer13_attention_weight_min": -27.87207794189453,
      "activations/layer14_attention_weight_max": 33.997337341308594,
      "activations/layer14_attention_weight_min": -30.63231658935547,
      "activations/layer15_attention_weight_max": 33.43349075317383,
      "activations/layer15_attention_weight_min": -30.76853370666504,
      "activations/layer16_attention_weight_max": 33.09665298461914,
      "activations/layer16_attention_weight_min": -31.88340950012207,
      "activations/layer17_attention_weight_max": 51.63088607788086,
      "activations/layer17_attention_weight_min": -50.27688980102539,
      "activations/layer18_attention_weight_max": 45.51018524169922,
      "activations/layer18_attention_weight_min": -42.95098114013672,
      "activations/layer19_attention_weight_max": 22.718229293823242,
      "activations/layer19_attention_weight_min": -22.36167335510254,
      "activations/layer1_attention_weight_max": 16.855749130249023,
      "activations/layer1_attention_weight_min": -12.931472778320312,
      "activations/layer20_attention_weight_max": 24.48476219177246,
      "activations/layer20_attention_weight_min": -21.960166931152344,
      "activations/layer21_attention_weight_max": 36.39023208618164,
      "activations/layer21_attention_weight_min": -22.16864776611328,
      "activations/layer22_attention_weight_max": 28.434709548950195,
      "activations/layer22_attention_weight_min": -26.399139404296875,
      "activations/layer23_attention_weight_max": 37.118186950683594,
      "activations/layer23_attention_weight_min": -24.189838409423828,
      "activations/layer2_attention_weight_max": 33.28678894042969,
      "activations/layer2_attention_weight_min": -29.828842163085938,
      "activations/layer3_attention_weight_max": 90.75718688964844,
      "activations/layer3_attention_weight_min": -85.77830505371094,
      "activations/layer4_attention_weight_max": 85.29417419433594,
      "activations/layer4_attention_weight_min": -85.46390533447266,
      "activations/layer5_attention_weight_max": 68.07654571533203,
      "activations/layer5_attention_weight_min": -71.81519317626953,
      "activations/layer6_attention_weight_max": 53.8046989440918,
      "activations/layer6_attention_weight_min": -52.726253509521484,
      "activations/layer7_attention_weight_max": 69.05567932128906,
      "activations/layer7_attention_weight_min": -65.76085662841797,
      "activations/layer8_attention_weight_max": 47.563697814941406,
      "activations/layer8_attention_weight_min": -52.525306701660156,
      "activations/layer9_attention_weight_max": 47.124603271484375,
      "activations/layer9_attention_weight_min": -46.47175598144531,
      "epoch": 19.12,
      "learning_rate": 2.6995075757575755e-05,
      "loss": 2.7311,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.5079,
      "eval_samples_per_second": 504.705,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.5079,
      "eval_openwebtext_samples_per_second": 504.705,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9856,
      "eval_wikitext_samples_per_second": 229.656,
      "step": 329000
    },
    {
      "epoch": 19.12,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5559,
      "eval_lambada_samples_per_second": 509.527,
      "step": 329000
    },
    {
      "activations/layer0_attention_weight_max": 15.210685729980469,
      "activations/layer0_attention_weight_min": -13.688755989074707,
      "activations/layer10_attention_weight_max": 37.107200622558594,
      "activations/layer10_attention_weight_min": -38.61200714111328,
      "activations/layer11_attention_weight_max": 34.654598236083984,
      "activations/layer11_attention_weight_min": -36.01118469238281,
      "activations/layer12_attention_weight_max": 19.248849868774414,
      "activations/layer12_attention_weight_min": -23.16693878173828,
      "activations/layer13_attention_weight_max": 36.62388610839844,
      "activations/layer13_attention_weight_min": -33.97776794433594,
      "activations/layer14_attention_weight_max": 35.398597717285156,
      "activations/layer14_attention_weight_min": -30.85836410522461,
      "activations/layer15_attention_weight_max": 35.41728210449219,
      "activations/layer15_attention_weight_min": -30.151704788208008,
      "activations/layer16_attention_weight_max": 34.6267204284668,
      "activations/layer16_attention_weight_min": -34.88689422607422,
      "activations/layer17_attention_weight_max": 52.33876037597656,
      "activations/layer17_attention_weight_min": -46.24199676513672,
      "activations/layer18_attention_weight_max": 47.414337158203125,
      "activations/layer18_attention_weight_min": -40.00033187866211,
      "activations/layer19_attention_weight_max": 24.914655685424805,
      "activations/layer19_attention_weight_min": -25.05384063720703,
      "activations/layer1_attention_weight_max": 17.95164680480957,
      "activations/layer1_attention_weight_min": -13.642674446105957,
      "activations/layer20_attention_weight_max": 24.0367488861084,
      "activations/layer20_attention_weight_min": -21.252561569213867,
      "activations/layer21_attention_weight_max": 34.33498001098633,
      "activations/layer21_attention_weight_min": -25.96548843383789,
      "activations/layer22_attention_weight_max": 30.716739654541016,
      "activations/layer22_attention_weight_min": -24.568214416503906,
      "activations/layer23_attention_weight_max": 40.51074981689453,
      "activations/layer23_attention_weight_min": -24.94306182861328,
      "activations/layer2_attention_weight_max": 33.58094024658203,
      "activations/layer2_attention_weight_min": -30.41909408569336,
      "activations/layer3_attention_weight_max": 87.49818420410156,
      "activations/layer3_attention_weight_min": -85.01050567626953,
      "activations/layer4_attention_weight_max": 87.62899017333984,
      "activations/layer4_attention_weight_min": -86.2990951538086,
      "activations/layer5_attention_weight_max": 67.43802642822266,
      "activations/layer5_attention_weight_min": -75.23149108886719,
      "activations/layer6_attention_weight_max": 52.424232482910156,
      "activations/layer6_attention_weight_min": -55.3832893371582,
      "activations/layer7_attention_weight_max": 67.71958923339844,
      "activations/layer7_attention_weight_min": -69.41556549072266,
      "activations/layer8_attention_weight_max": 48.59782791137695,
      "activations/layer8_attention_weight_min": -52.90354919433594,
      "activations/layer9_attention_weight_max": 48.52695846557617,
      "activations/layer9_attention_weight_min": -51.07744598388672,
      "epoch": 19.12,
      "learning_rate": 2.697613636363636e-05,
      "loss": 2.7133,
      "step": 329050
    },
    {
      "activations/layer0_attention_weight_max": 15.55864429473877,
      "activations/layer0_attention_weight_min": -13.908177375793457,
      "activations/layer10_attention_weight_max": 33.803367614746094,
      "activations/layer10_attention_weight_min": -35.418331146240234,
      "activations/layer11_attention_weight_max": 34.043609619140625,
      "activations/layer11_attention_weight_min": -33.037506103515625,
      "activations/layer12_attention_weight_max": 19.30525779724121,
      "activations/layer12_attention_weight_min": -24.486976623535156,
      "activations/layer13_attention_weight_max": 29.98557472229004,
      "activations/layer13_attention_weight_min": -27.5614070892334,
      "activations/layer14_attention_weight_max": 36.33611297607422,
      "activations/layer14_attention_weight_min": -31.58746910095215,
      "activations/layer15_attention_weight_max": 34.50857925415039,
      "activations/layer15_attention_weight_min": -32.24908447265625,
      "activations/layer16_attention_weight_max": 36.34851837158203,
      "activations/layer16_attention_weight_min": -33.51031494140625,
      "activations/layer17_attention_weight_max": 53.32450866699219,
      "activations/layer17_attention_weight_min": -45.02223587036133,
      "activations/layer18_attention_weight_max": 48.403751373291016,
      "activations/layer18_attention_weight_min": -42.44902420043945,
      "activations/layer19_attention_weight_max": 21.820125579833984,
      "activations/layer19_attention_weight_min": -23.0325984954834,
      "activations/layer1_attention_weight_max": 16.19198226928711,
      "activations/layer1_attention_weight_min": -13.701025009155273,
      "activations/layer20_attention_weight_max": 19.81881332397461,
      "activations/layer20_attention_weight_min": -23.022769927978516,
      "activations/layer21_attention_weight_max": 41.490665435791016,
      "activations/layer21_attention_weight_min": -23.90768814086914,
      "activations/layer22_attention_weight_max": 27.675676345825195,
      "activations/layer22_attention_weight_min": -25.36968231201172,
      "activations/layer23_attention_weight_max": 33.44849395751953,
      "activations/layer23_attention_weight_min": -24.801273345947266,
      "activations/layer2_attention_weight_max": 32.90730285644531,
      "activations/layer2_attention_weight_min": -32.423736572265625,
      "activations/layer3_attention_weight_max": 86.56787109375,
      "activations/layer3_attention_weight_min": -88.28742218017578,
      "activations/layer4_attention_weight_max": 83.05943298339844,
      "activations/layer4_attention_weight_min": -80.45558166503906,
      "activations/layer5_attention_weight_max": 62.46295928955078,
      "activations/layer5_attention_weight_min": -74.9187240600586,
      "activations/layer6_attention_weight_max": 49.96455383300781,
      "activations/layer6_attention_weight_min": -49.04506301879883,
      "activations/layer7_attention_weight_max": 68.3498764038086,
      "activations/layer7_attention_weight_min": -63.667388916015625,
      "activations/layer8_attention_weight_max": 43.01836013793945,
      "activations/layer8_attention_weight_min": -48.58960723876953,
      "activations/layer9_attention_weight_max": 40.687068939208984,
      "activations/layer9_attention_weight_min": -45.992523193359375,
      "epoch": 19.12,
      "learning_rate": 2.6957196969696968e-05,
      "loss": 2.6976,
      "step": 329100
    },
    {
      "activations/layer0_attention_weight_max": 15.417689323425293,
      "activations/layer0_attention_weight_min": -13.314611434936523,
      "activations/layer10_attention_weight_max": 36.767520904541016,
      "activations/layer10_attention_weight_min": -36.984291076660156,
      "activations/layer11_attention_weight_max": 32.737850189208984,
      "activations/layer11_attention_weight_min": -35.297908782958984,
      "activations/layer12_attention_weight_max": 19.436321258544922,
      "activations/layer12_attention_weight_min": -25.311723709106445,
      "activations/layer13_attention_weight_max": 31.904991149902344,
      "activations/layer13_attention_weight_min": -27.03354835510254,
      "activations/layer14_attention_weight_max": 37.37511444091797,
      "activations/layer14_attention_weight_min": -30.840282440185547,
      "activations/layer15_attention_weight_max": 33.662811279296875,
      "activations/layer15_attention_weight_min": -33.8147087097168,
      "activations/layer16_attention_weight_max": 34.724300384521484,
      "activations/layer16_attention_weight_min": -32.660037994384766,
      "activations/layer17_attention_weight_max": 53.17823028564453,
      "activations/layer17_attention_weight_min": -46.27819061279297,
      "activations/layer18_attention_weight_max": 46.95486831665039,
      "activations/layer18_attention_weight_min": -42.43663787841797,
      "activations/layer19_attention_weight_max": 23.527334213256836,
      "activations/layer19_attention_weight_min": -23.84073829650879,
      "activations/layer1_attention_weight_max": 16.535799026489258,
      "activations/layer1_attention_weight_min": -13.41651439666748,
      "activations/layer20_attention_weight_max": 22.06853675842285,
      "activations/layer20_attention_weight_min": -24.79100227355957,
      "activations/layer21_attention_weight_max": 35.02349090576172,
      "activations/layer21_attention_weight_min": -29.932384490966797,
      "activations/layer22_attention_weight_max": 29.4718074798584,
      "activations/layer22_attention_weight_min": -27.2447452545166,
      "activations/layer23_attention_weight_max": 34.051578521728516,
      "activations/layer23_attention_weight_min": -23.64521026611328,
      "activations/layer2_attention_weight_max": 31.73499870300293,
      "activations/layer2_attention_weight_min": -31.104114532470703,
      "activations/layer3_attention_weight_max": 88.02479553222656,
      "activations/layer3_attention_weight_min": -86.73889923095703,
      "activations/layer4_attention_weight_max": 89.55906677246094,
      "activations/layer4_attention_weight_min": -85.65090942382812,
      "activations/layer5_attention_weight_max": 64.1631851196289,
      "activations/layer5_attention_weight_min": -74.94108581542969,
      "activations/layer6_attention_weight_max": 50.94851303100586,
      "activations/layer6_attention_weight_min": -54.9865608215332,
      "activations/layer7_attention_weight_max": 64.78401947021484,
      "activations/layer7_attention_weight_min": -63.47828674316406,
      "activations/layer8_attention_weight_max": 48.404388427734375,
      "activations/layer8_attention_weight_min": -48.08000564575195,
      "activations/layer9_attention_weight_max": 47.5577507019043,
      "activations/layer9_attention_weight_min": -44.60686492919922,
      "epoch": 19.13,
      "learning_rate": 2.6938257575757573e-05,
      "loss": 2.7105,
      "step": 329150
    },
    {
      "activations/layer0_attention_weight_max": 15.952948570251465,
      "activations/layer0_attention_weight_min": -13.658515930175781,
      "activations/layer10_attention_weight_max": 35.02079772949219,
      "activations/layer10_attention_weight_min": -34.60075759887695,
      "activations/layer11_attention_weight_max": 31.195951461791992,
      "activations/layer11_attention_weight_min": -33.858612060546875,
      "activations/layer12_attention_weight_max": 19.365257263183594,
      "activations/layer12_attention_weight_min": -27.54194450378418,
      "activations/layer13_attention_weight_max": 39.363853454589844,
      "activations/layer13_attention_weight_min": -29.004825592041016,
      "activations/layer14_attention_weight_max": 36.52118682861328,
      "activations/layer14_attention_weight_min": -30.44449806213379,
      "activations/layer15_attention_weight_max": 33.98190689086914,
      "activations/layer15_attention_weight_min": -30.096960067749023,
      "activations/layer16_attention_weight_max": 34.66992950439453,
      "activations/layer16_attention_weight_min": -33.587425231933594,
      "activations/layer17_attention_weight_max": 51.60287094116211,
      "activations/layer17_attention_weight_min": -46.41026306152344,
      "activations/layer18_attention_weight_max": 47.6351432800293,
      "activations/layer18_attention_weight_min": -41.49638748168945,
      "activations/layer19_attention_weight_max": 22.521162033081055,
      "activations/layer19_attention_weight_min": -24.50617218017578,
      "activations/layer1_attention_weight_max": 16.266565322875977,
      "activations/layer1_attention_weight_min": -13.880263328552246,
      "activations/layer20_attention_weight_max": 22.720319747924805,
      "activations/layer20_attention_weight_min": -24.096744537353516,
      "activations/layer21_attention_weight_max": 33.00490951538086,
      "activations/layer21_attention_weight_min": -23.294336318969727,
      "activations/layer22_attention_weight_max": 29.848583221435547,
      "activations/layer22_attention_weight_min": -27.748342514038086,
      "activations/layer23_attention_weight_max": 34.175498962402344,
      "activations/layer23_attention_weight_min": -26.299163818359375,
      "activations/layer2_attention_weight_max": 33.07615280151367,
      "activations/layer2_attention_weight_min": -31.597068786621094,
      "activations/layer3_attention_weight_max": 87.1127700805664,
      "activations/layer3_attention_weight_min": -86.7923812866211,
      "activations/layer4_attention_weight_max": 83.78001403808594,
      "activations/layer4_attention_weight_min": -81.4144058227539,
      "activations/layer5_attention_weight_max": 63.99473190307617,
      "activations/layer5_attention_weight_min": -74.46520233154297,
      "activations/layer6_attention_weight_max": 50.394508361816406,
      "activations/layer6_attention_weight_min": -54.349266052246094,
      "activations/layer7_attention_weight_max": 68.44778442382812,
      "activations/layer7_attention_weight_min": -65.6838150024414,
      "activations/layer8_attention_weight_max": 46.385894775390625,
      "activations/layer8_attention_weight_min": -52.3155403137207,
      "activations/layer9_attention_weight_max": 43.93076705932617,
      "activations/layer9_attention_weight_min": -45.222957611083984,
      "epoch": 19.13,
      "learning_rate": 2.691931818181818e-05,
      "loss": 2.7283,
      "step": 329200
    },
    {
      "activations/layer0_attention_weight_max": 15.24781608581543,
      "activations/layer0_attention_weight_min": -13.607603073120117,
      "activations/layer10_attention_weight_max": 35.657493591308594,
      "activations/layer10_attention_weight_min": -35.617130279541016,
      "activations/layer11_attention_weight_max": 31.465261459350586,
      "activations/layer11_attention_weight_min": -34.42213821411133,
      "activations/layer12_attention_weight_max": 19.987899780273438,
      "activations/layer12_attention_weight_min": -27.75216293334961,
      "activations/layer13_attention_weight_max": 35.099388122558594,
      "activations/layer13_attention_weight_min": -27.64072608947754,
      "activations/layer14_attention_weight_max": 34.260860443115234,
      "activations/layer14_attention_weight_min": -28.741432189941406,
      "activations/layer15_attention_weight_max": 31.052165985107422,
      "activations/layer15_attention_weight_min": -28.537206649780273,
      "activations/layer16_attention_weight_max": 34.69351577758789,
      "activations/layer16_attention_weight_min": -32.614837646484375,
      "activations/layer17_attention_weight_max": 50.52271270751953,
      "activations/layer17_attention_weight_min": -42.53821563720703,
      "activations/layer18_attention_weight_max": 47.176937103271484,
      "activations/layer18_attention_weight_min": -36.130889892578125,
      "activations/layer19_attention_weight_max": 23.678993225097656,
      "activations/layer19_attention_weight_min": -22.523792266845703,
      "activations/layer1_attention_weight_max": 15.93541145324707,
      "activations/layer1_attention_weight_min": -14.265737533569336,
      "activations/layer20_attention_weight_max": 22.670860290527344,
      "activations/layer20_attention_weight_min": -23.227787017822266,
      "activations/layer21_attention_weight_max": 35.820068359375,
      "activations/layer21_attention_weight_min": -23.59950828552246,
      "activations/layer22_attention_weight_max": 27.926185607910156,
      "activations/layer22_attention_weight_min": -25.441574096679688,
      "activations/layer23_attention_weight_max": 36.239967346191406,
      "activations/layer23_attention_weight_min": -23.898405075073242,
      "activations/layer2_attention_weight_max": 30.871976852416992,
      "activations/layer2_attention_weight_min": -30.634075164794922,
      "activations/layer3_attention_weight_max": 83.489013671875,
      "activations/layer3_attention_weight_min": -83.68189239501953,
      "activations/layer4_attention_weight_max": 83.27352142333984,
      "activations/layer4_attention_weight_min": -85.61190032958984,
      "activations/layer5_attention_weight_max": 57.37298583984375,
      "activations/layer5_attention_weight_min": -71.6649169921875,
      "activations/layer6_attention_weight_max": 48.25076675415039,
      "activations/layer6_attention_weight_min": -50.538414001464844,
      "activations/layer7_attention_weight_max": 61.752532958984375,
      "activations/layer7_attention_weight_min": -63.4143180847168,
      "activations/layer8_attention_weight_max": 44.054443359375,
      "activations/layer8_attention_weight_min": -49.8526611328125,
      "activations/layer9_attention_weight_max": 42.43296813964844,
      "activations/layer9_attention_weight_min": -45.311058044433594,
      "epoch": 19.13,
      "learning_rate": 2.6900378787878786e-05,
      "loss": 2.717,
      "step": 329250
    },
    {
      "activations/layer0_attention_weight_max": 15.662227630615234,
      "activations/layer0_attention_weight_min": -15.227201461791992,
      "activations/layer10_attention_weight_max": 34.24323654174805,
      "activations/layer10_attention_weight_min": -34.62098693847656,
      "activations/layer11_attention_weight_max": 31.506446838378906,
      "activations/layer11_attention_weight_min": -34.9940185546875,
      "activations/layer12_attention_weight_max": 19.93735122680664,
      "activations/layer12_attention_weight_min": -28.46558380126953,
      "activations/layer13_attention_weight_max": 34.6229248046875,
      "activations/layer13_attention_weight_min": -27.847820281982422,
      "activations/layer14_attention_weight_max": 37.30525207519531,
      "activations/layer14_attention_weight_min": -31.692506790161133,
      "activations/layer15_attention_weight_max": 35.183433532714844,
      "activations/layer15_attention_weight_min": -30.544824600219727,
      "activations/layer16_attention_weight_max": 38.10164260864258,
      "activations/layer16_attention_weight_min": -35.71893310546875,
      "activations/layer17_attention_weight_max": 53.55052947998047,
      "activations/layer17_attention_weight_min": -50.711631774902344,
      "activations/layer18_attention_weight_max": 51.11051559448242,
      "activations/layer18_attention_weight_min": -43.38669204711914,
      "activations/layer19_attention_weight_max": 24.700740814208984,
      "activations/layer19_attention_weight_min": -24.031965255737305,
      "activations/layer1_attention_weight_max": 15.41774845123291,
      "activations/layer1_attention_weight_min": -14.178956031799316,
      "activations/layer20_attention_weight_max": 23.119537353515625,
      "activations/layer20_attention_weight_min": -25.133188247680664,
      "activations/layer21_attention_weight_max": 39.20264434814453,
      "activations/layer21_attention_weight_min": -22.630075454711914,
      "activations/layer22_attention_weight_max": 28.549802780151367,
      "activations/layer22_attention_weight_min": -25.781349182128906,
      "activations/layer23_attention_weight_max": 38.826576232910156,
      "activations/layer23_attention_weight_min": -23.47181510925293,
      "activations/layer2_attention_weight_max": 31.919950485229492,
      "activations/layer2_attention_weight_min": -29.496122360229492,
      "activations/layer3_attention_weight_max": 82.06683349609375,
      "activations/layer3_attention_weight_min": -80.90396118164062,
      "activations/layer4_attention_weight_max": 81.57266235351562,
      "activations/layer4_attention_weight_min": -80.79615020751953,
      "activations/layer5_attention_weight_max": 64.19230651855469,
      "activations/layer5_attention_weight_min": -66.14666748046875,
      "activations/layer6_attention_weight_max": 47.49393081665039,
      "activations/layer6_attention_weight_min": -47.543094635009766,
      "activations/layer7_attention_weight_max": 63.03302001953125,
      "activations/layer7_attention_weight_min": -62.64499282836914,
      "activations/layer8_attention_weight_max": 45.0967903137207,
      "activations/layer8_attention_weight_min": -48.98725509643555,
      "activations/layer9_attention_weight_max": 43.723323822021484,
      "activations/layer9_attention_weight_min": -44.964054107666016,
      "epoch": 19.13,
      "learning_rate": 2.688143939393939e-05,
      "loss": 2.7322,
      "step": 329300
    },
    {
      "activations/layer0_attention_weight_max": 15.032803535461426,
      "activations/layer0_attention_weight_min": -13.222222328186035,
      "activations/layer10_attention_weight_max": 34.659664154052734,
      "activations/layer10_attention_weight_min": -35.40114212036133,
      "activations/layer11_attention_weight_max": 31.09927749633789,
      "activations/layer11_attention_weight_min": -33.699851989746094,
      "activations/layer12_attention_weight_max": 19.48526382446289,
      "activations/layer12_attention_weight_min": -26.015262603759766,
      "activations/layer13_attention_weight_max": 35.27300262451172,
      "activations/layer13_attention_weight_min": -27.561508178710938,
      "activations/layer14_attention_weight_max": 35.62553405761719,
      "activations/layer14_attention_weight_min": -30.745275497436523,
      "activations/layer15_attention_weight_max": 34.00785827636719,
      "activations/layer15_attention_weight_min": -30.231487274169922,
      "activations/layer16_attention_weight_max": 34.785179138183594,
      "activations/layer16_attention_weight_min": -31.49180793762207,
      "activations/layer17_attention_weight_max": 51.69375991821289,
      "activations/layer17_attention_weight_min": -43.76607131958008,
      "activations/layer18_attention_weight_max": 45.90268325805664,
      "activations/layer18_attention_weight_min": -35.137882232666016,
      "activations/layer19_attention_weight_max": 23.494497299194336,
      "activations/layer19_attention_weight_min": -22.181604385375977,
      "activations/layer1_attention_weight_max": 15.14478874206543,
      "activations/layer1_attention_weight_min": -13.207978248596191,
      "activations/layer20_attention_weight_max": 20.477481842041016,
      "activations/layer20_attention_weight_min": -20.994943618774414,
      "activations/layer21_attention_weight_max": 34.831764221191406,
      "activations/layer21_attention_weight_min": -22.107805252075195,
      "activations/layer22_attention_weight_max": 29.264625549316406,
      "activations/layer22_attention_weight_min": -24.551124572753906,
      "activations/layer23_attention_weight_max": 37.64155578613281,
      "activations/layer23_attention_weight_min": -22.87096405029297,
      "activations/layer2_attention_weight_max": 31.1474609375,
      "activations/layer2_attention_weight_min": -29.684993743896484,
      "activations/layer3_attention_weight_max": 86.33077239990234,
      "activations/layer3_attention_weight_min": -80.9763412475586,
      "activations/layer4_attention_weight_max": 81.31634521484375,
      "activations/layer4_attention_weight_min": -78.51993560791016,
      "activations/layer5_attention_weight_max": 62.95293426513672,
      "activations/layer5_attention_weight_min": -69.96839904785156,
      "activations/layer6_attention_weight_max": 48.43519592285156,
      "activations/layer6_attention_weight_min": -51.522579193115234,
      "activations/layer7_attention_weight_max": 63.32367706298828,
      "activations/layer7_attention_weight_min": -63.69233322143555,
      "activations/layer8_attention_weight_max": 46.08197784423828,
      "activations/layer8_attention_weight_min": -50.71875762939453,
      "activations/layer9_attention_weight_max": 42.22779846191406,
      "activations/layer9_attention_weight_min": -46.748626708984375,
      "epoch": 19.14,
      "learning_rate": 2.68625e-05,
      "loss": 2.7181,
      "step": 329350
    },
    {
      "activations/layer0_attention_weight_max": 14.958990097045898,
      "activations/layer0_attention_weight_min": -13.48856258392334,
      "activations/layer10_attention_weight_max": 35.74158477783203,
      "activations/layer10_attention_weight_min": -37.55295181274414,
      "activations/layer11_attention_weight_max": 31.86510467529297,
      "activations/layer11_attention_weight_min": -35.408050537109375,
      "activations/layer12_attention_weight_max": 19.10700798034668,
      "activations/layer12_attention_weight_min": -24.4776611328125,
      "activations/layer13_attention_weight_max": 32.22428894042969,
      "activations/layer13_attention_weight_min": -27.601696014404297,
      "activations/layer14_attention_weight_max": 34.48902893066406,
      "activations/layer14_attention_weight_min": -31.185338973999023,
      "activations/layer15_attention_weight_max": 32.556339263916016,
      "activations/layer15_attention_weight_min": -31.103702545166016,
      "activations/layer16_attention_weight_max": 35.87755584716797,
      "activations/layer16_attention_weight_min": -34.61585998535156,
      "activations/layer17_attention_weight_max": 53.59465408325195,
      "activations/layer17_attention_weight_min": -45.989803314208984,
      "activations/layer18_attention_weight_max": 49.01401901245117,
      "activations/layer18_attention_weight_min": -42.062896728515625,
      "activations/layer19_attention_weight_max": 24.09492301940918,
      "activations/layer19_attention_weight_min": -23.88983917236328,
      "activations/layer1_attention_weight_max": 15.774454116821289,
      "activations/layer1_attention_weight_min": -14.048624038696289,
      "activations/layer20_attention_weight_max": 23.977741241455078,
      "activations/layer20_attention_weight_min": -22.21906089782715,
      "activations/layer21_attention_weight_max": 35.522735595703125,
      "activations/layer21_attention_weight_min": -20.98700714111328,
      "activations/layer22_attention_weight_max": 26.61191749572754,
      "activations/layer22_attention_weight_min": -26.31756591796875,
      "activations/layer23_attention_weight_max": 34.00490951538086,
      "activations/layer23_attention_weight_min": -26.36156463623047,
      "activations/layer2_attention_weight_max": 31.274930953979492,
      "activations/layer2_attention_weight_min": -29.367048263549805,
      "activations/layer3_attention_weight_max": 80.97098541259766,
      "activations/layer3_attention_weight_min": -81.9610366821289,
      "activations/layer4_attention_weight_max": 79.0988540649414,
      "activations/layer4_attention_weight_min": -78.9883041381836,
      "activations/layer5_attention_weight_max": 60.02043151855469,
      "activations/layer5_attention_weight_min": -75.01385498046875,
      "activations/layer6_attention_weight_max": 48.401954650878906,
      "activations/layer6_attention_weight_min": -49.71445083618164,
      "activations/layer7_attention_weight_max": 65.3650131225586,
      "activations/layer7_attention_weight_min": -60.508365631103516,
      "activations/layer8_attention_weight_max": 47.357303619384766,
      "activations/layer8_attention_weight_min": -51.43280029296875,
      "activations/layer9_attention_weight_max": 43.409053802490234,
      "activations/layer9_attention_weight_min": -46.81437301635742,
      "epoch": 19.14,
      "learning_rate": 2.6843560606060604e-05,
      "loss": 2.7323,
      "step": 329400
    },
    {
      "activations/layer0_attention_weight_max": 15.375473976135254,
      "activations/layer0_attention_weight_min": -13.201473236083984,
      "activations/layer10_attention_weight_max": 41.357906341552734,
      "activations/layer10_attention_weight_min": -41.03251266479492,
      "activations/layer11_attention_weight_max": 36.39932632446289,
      "activations/layer11_attention_weight_min": -38.69289779663086,
      "activations/layer12_attention_weight_max": 21.458389282226562,
      "activations/layer12_attention_weight_min": -22.784717559814453,
      "activations/layer13_attention_weight_max": 39.208919525146484,
      "activations/layer13_attention_weight_min": -28.155094146728516,
      "activations/layer14_attention_weight_max": 37.55314254760742,
      "activations/layer14_attention_weight_min": -30.533653259277344,
      "activations/layer15_attention_weight_max": 35.895450592041016,
      "activations/layer15_attention_weight_min": -31.56393814086914,
      "activations/layer16_attention_weight_max": 36.07307434082031,
      "activations/layer16_attention_weight_min": -34.15408706665039,
      "activations/layer17_attention_weight_max": 56.31746292114258,
      "activations/layer17_attention_weight_min": -47.80086898803711,
      "activations/layer18_attention_weight_max": 50.08141326904297,
      "activations/layer18_attention_weight_min": -43.27815628051758,
      "activations/layer19_attention_weight_max": 23.738969802856445,
      "activations/layer19_attention_weight_min": -23.940954208374023,
      "activations/layer1_attention_weight_max": 15.385063171386719,
      "activations/layer1_attention_weight_min": -15.03825569152832,
      "activations/layer20_attention_weight_max": 22.10161590576172,
      "activations/layer20_attention_weight_min": -24.048309326171875,
      "activations/layer21_attention_weight_max": 32.7672233581543,
      "activations/layer21_attention_weight_min": -23.555484771728516,
      "activations/layer22_attention_weight_max": 26.991943359375,
      "activations/layer22_attention_weight_min": -27.994916915893555,
      "activations/layer23_attention_weight_max": 36.86298370361328,
      "activations/layer23_attention_weight_min": -23.067054748535156,
      "activations/layer2_attention_weight_max": 30.551206588745117,
      "activations/layer2_attention_weight_min": -31.333839416503906,
      "activations/layer3_attention_weight_max": 86.92771911621094,
      "activations/layer3_attention_weight_min": -86.72764587402344,
      "activations/layer4_attention_weight_max": 85.56222534179688,
      "activations/layer4_attention_weight_min": -86.658203125,
      "activations/layer5_attention_weight_max": 63.733638763427734,
      "activations/layer5_attention_weight_min": -72.72064208984375,
      "activations/layer6_attention_weight_max": 53.38505935668945,
      "activations/layer6_attention_weight_min": -52.85599136352539,
      "activations/layer7_attention_weight_max": 66.32990264892578,
      "activations/layer7_attention_weight_min": -68.78351593017578,
      "activations/layer8_attention_weight_max": 51.73649215698242,
      "activations/layer8_attention_weight_min": -55.72248077392578,
      "activations/layer9_attention_weight_max": 48.545555114746094,
      "activations/layer9_attention_weight_min": -51.901519775390625,
      "epoch": 19.14,
      "learning_rate": 2.6824621212121212e-05,
      "loss": 2.7186,
      "step": 329450
    },
    {
      "activations/layer0_attention_weight_max": 14.656106948852539,
      "activations/layer0_attention_weight_min": -13.420235633850098,
      "activations/layer10_attention_weight_max": 37.71522903442383,
      "activations/layer10_attention_weight_min": -37.02061080932617,
      "activations/layer11_attention_weight_max": 35.24338912963867,
      "activations/layer11_attention_weight_min": -32.720947265625,
      "activations/layer12_attention_weight_max": 19.379926681518555,
      "activations/layer12_attention_weight_min": -25.248411178588867,
      "activations/layer13_attention_weight_max": 32.61399459838867,
      "activations/layer13_attention_weight_min": -28.356008529663086,
      "activations/layer14_attention_weight_max": 36.024436950683594,
      "activations/layer14_attention_weight_min": -30.124971389770508,
      "activations/layer15_attention_weight_max": 34.235862731933594,
      "activations/layer15_attention_weight_min": -30.229183197021484,
      "activations/layer16_attention_weight_max": 36.18021011352539,
      "activations/layer16_attention_weight_min": -30.85700225830078,
      "activations/layer17_attention_weight_max": 53.39555740356445,
      "activations/layer17_attention_weight_min": -44.44126892089844,
      "activations/layer18_attention_weight_max": 53.084625244140625,
      "activations/layer18_attention_weight_min": -40.63090133666992,
      "activations/layer19_attention_weight_max": 28.275943756103516,
      "activations/layer19_attention_weight_min": -25.42890739440918,
      "activations/layer1_attention_weight_max": 15.584554672241211,
      "activations/layer1_attention_weight_min": -12.674696922302246,
      "activations/layer20_attention_weight_max": 27.641189575195312,
      "activations/layer20_attention_weight_min": -23.384180068969727,
      "activations/layer21_attention_weight_max": 46.203670501708984,
      "activations/layer21_attention_weight_min": -23.99674415588379,
      "activations/layer22_attention_weight_max": 32.82782745361328,
      "activations/layer22_attention_weight_min": -26.49838638305664,
      "activations/layer23_attention_weight_max": 39.11087417602539,
      "activations/layer23_attention_weight_min": -24.53949546813965,
      "activations/layer2_attention_weight_max": 31.435604095458984,
      "activations/layer2_attention_weight_min": -30.6822566986084,
      "activations/layer3_attention_weight_max": 84.70576477050781,
      "activations/layer3_attention_weight_min": -81.7144546508789,
      "activations/layer4_attention_weight_max": 82.1780014038086,
      "activations/layer4_attention_weight_min": -83.72725677490234,
      "activations/layer5_attention_weight_max": 60.37793731689453,
      "activations/layer5_attention_weight_min": -67.27779388427734,
      "activations/layer6_attention_weight_max": 48.09817123413086,
      "activations/layer6_attention_weight_min": -49.02088928222656,
      "activations/layer7_attention_weight_max": 70.2181625366211,
      "activations/layer7_attention_weight_min": -64.24095916748047,
      "activations/layer8_attention_weight_max": 49.65100860595703,
      "activations/layer8_attention_weight_min": -48.09553527832031,
      "activations/layer9_attention_weight_max": 55.488311767578125,
      "activations/layer9_attention_weight_min": -44.94619369506836,
      "epoch": 19.15,
      "learning_rate": 2.6805681818181814e-05,
      "loss": 2.6952,
      "step": 329500
    },
    {
      "activations/layer0_attention_weight_max": 17.49469757080078,
      "activations/layer0_attention_weight_min": -13.608736991882324,
      "activations/layer10_attention_weight_max": 38.38931655883789,
      "activations/layer10_attention_weight_min": -36.5521240234375,
      "activations/layer11_attention_weight_max": 32.80990219116211,
      "activations/layer11_attention_weight_min": -34.826507568359375,
      "activations/layer12_attention_weight_max": 20.29928207397461,
      "activations/layer12_attention_weight_min": -23.640445709228516,
      "activations/layer13_attention_weight_max": 33.31256866455078,
      "activations/layer13_attention_weight_min": -25.766923904418945,
      "activations/layer14_attention_weight_max": 36.87297439575195,
      "activations/layer14_attention_weight_min": -29.12952423095703,
      "activations/layer15_attention_weight_max": 32.70662307739258,
      "activations/layer15_attention_weight_min": -30.392255783081055,
      "activations/layer16_attention_weight_max": 34.014129638671875,
      "activations/layer16_attention_weight_min": -32.03451919555664,
      "activations/layer17_attention_weight_max": 54.70744323730469,
      "activations/layer17_attention_weight_min": -47.2137565612793,
      "activations/layer18_attention_weight_max": 57.46683120727539,
      "activations/layer18_attention_weight_min": -45.88684844970703,
      "activations/layer19_attention_weight_max": 28.21994972229004,
      "activations/layer19_attention_weight_min": -26.214340209960938,
      "activations/layer1_attention_weight_max": 15.359859466552734,
      "activations/layer1_attention_weight_min": -13.114794731140137,
      "activations/layer20_attention_weight_max": 27.383216857910156,
      "activations/layer20_attention_weight_min": -22.282268524169922,
      "activations/layer21_attention_weight_max": 41.95097351074219,
      "activations/layer21_attention_weight_min": -24.033672332763672,
      "activations/layer22_attention_weight_max": 33.61604309082031,
      "activations/layer22_attention_weight_min": -26.804107666015625,
      "activations/layer23_attention_weight_max": 41.15654754638672,
      "activations/layer23_attention_weight_min": -23.915939331054688,
      "activations/layer2_attention_weight_max": 31.653745651245117,
      "activations/layer2_attention_weight_min": -30.055038452148438,
      "activations/layer3_attention_weight_max": 82.72491455078125,
      "activations/layer3_attention_weight_min": -87.2257080078125,
      "activations/layer4_attention_weight_max": 81.42848205566406,
      "activations/layer4_attention_weight_min": -82.7495346069336,
      "activations/layer5_attention_weight_max": 61.33523178100586,
      "activations/layer5_attention_weight_min": -77.49208068847656,
      "activations/layer6_attention_weight_max": 50.74702835083008,
      "activations/layer6_attention_weight_min": -49.402870178222656,
      "activations/layer7_attention_weight_max": 63.084617614746094,
      "activations/layer7_attention_weight_min": -65.62962341308594,
      "activations/layer8_attention_weight_max": 44.94289016723633,
      "activations/layer8_attention_weight_min": -51.54793930053711,
      "activations/layer9_attention_weight_max": 43.14280319213867,
      "activations/layer9_attention_weight_min": -48.558570861816406,
      "epoch": 19.15,
      "learning_rate": 2.6786742424242422e-05,
      "loss": 2.7224,
      "step": 329550
    },
    {
      "activations/layer0_attention_weight_max": 15.176584243774414,
      "activations/layer0_attention_weight_min": -13.829695701599121,
      "activations/layer10_attention_weight_max": 35.123592376708984,
      "activations/layer10_attention_weight_min": -36.254066467285156,
      "activations/layer11_attention_weight_max": 32.91291427612305,
      "activations/layer11_attention_weight_min": -36.24213409423828,
      "activations/layer12_attention_weight_max": 18.463459014892578,
      "activations/layer12_attention_weight_min": -23.499637603759766,
      "activations/layer13_attention_weight_max": 33.27926254272461,
      "activations/layer13_attention_weight_min": -27.809383392333984,
      "activations/layer14_attention_weight_max": 35.33473587036133,
      "activations/layer14_attention_weight_min": -29.238231658935547,
      "activations/layer15_attention_weight_max": 32.51695251464844,
      "activations/layer15_attention_weight_min": -29.400163650512695,
      "activations/layer16_attention_weight_max": 32.615325927734375,
      "activations/layer16_attention_weight_min": -30.114574432373047,
      "activations/layer17_attention_weight_max": 53.90752029418945,
      "activations/layer17_attention_weight_min": -43.03978729248047,
      "activations/layer18_attention_weight_max": 51.24523162841797,
      "activations/layer18_attention_weight_min": -37.43962097167969,
      "activations/layer19_attention_weight_max": 22.364343643188477,
      "activations/layer19_attention_weight_min": -22.233154296875,
      "activations/layer1_attention_weight_max": 15.16429615020752,
      "activations/layer1_attention_weight_min": -13.109945297241211,
      "activations/layer20_attention_weight_max": 22.317724227905273,
      "activations/layer20_attention_weight_min": -21.610139846801758,
      "activations/layer21_attention_weight_max": 33.053382873535156,
      "activations/layer21_attention_weight_min": -23.422922134399414,
      "activations/layer22_attention_weight_max": 26.48649787902832,
      "activations/layer22_attention_weight_min": -25.2059326171875,
      "activations/layer23_attention_weight_max": 35.608558654785156,
      "activations/layer23_attention_weight_min": -23.86249542236328,
      "activations/layer2_attention_weight_max": 30.996631622314453,
      "activations/layer2_attention_weight_min": -31.040449142456055,
      "activations/layer3_attention_weight_max": 88.33914184570312,
      "activations/layer3_attention_weight_min": -89.79779815673828,
      "activations/layer4_attention_weight_max": 83.2644271850586,
      "activations/layer4_attention_weight_min": -82.43355560302734,
      "activations/layer5_attention_weight_max": 63.5700798034668,
      "activations/layer5_attention_weight_min": -72.01472473144531,
      "activations/layer6_attention_weight_max": 47.94479751586914,
      "activations/layer6_attention_weight_min": -50.91062927246094,
      "activations/layer7_attention_weight_max": 66.55679321289062,
      "activations/layer7_attention_weight_min": -64.10491943359375,
      "activations/layer8_attention_weight_max": 46.7235221862793,
      "activations/layer8_attention_weight_min": -50.17209243774414,
      "activations/layer9_attention_weight_max": 41.94474792480469,
      "activations/layer9_attention_weight_min": -44.32793426513672,
      "epoch": 19.15,
      "learning_rate": 2.6767803030303027e-05,
      "loss": 2.7235,
      "step": 329600
    },
    {
      "activations/layer0_attention_weight_max": 16.477127075195312,
      "activations/layer0_attention_weight_min": -13.350930213928223,
      "activations/layer10_attention_weight_max": 35.2144775390625,
      "activations/layer10_attention_weight_min": -36.162818908691406,
      "activations/layer11_attention_weight_max": 34.807525634765625,
      "activations/layer11_attention_weight_min": -35.17127990722656,
      "activations/layer12_attention_weight_max": 19.83599853515625,
      "activations/layer12_attention_weight_min": -28.267642974853516,
      "activations/layer13_attention_weight_max": 31.65091896057129,
      "activations/layer13_attention_weight_min": -27.653247833251953,
      "activations/layer14_attention_weight_max": 35.425472259521484,
      "activations/layer14_attention_weight_min": -32.1104850769043,
      "activations/layer15_attention_weight_max": 35.44448471069336,
      "activations/layer15_attention_weight_min": -32.837764739990234,
      "activations/layer16_attention_weight_max": 33.06650924682617,
      "activations/layer16_attention_weight_min": -34.06694793701172,
      "activations/layer17_attention_weight_max": 50.34483337402344,
      "activations/layer17_attention_weight_min": -50.119544982910156,
      "activations/layer18_attention_weight_max": 44.9323844909668,
      "activations/layer18_attention_weight_min": -40.07662582397461,
      "activations/layer19_attention_weight_max": 21.940763473510742,
      "activations/layer19_attention_weight_min": -22.663537979125977,
      "activations/layer1_attention_weight_max": 15.275480270385742,
      "activations/layer1_attention_weight_min": -13.303922653198242,
      "activations/layer20_attention_weight_max": 22.312522888183594,
      "activations/layer20_attention_weight_min": -22.750825881958008,
      "activations/layer21_attention_weight_max": 35.87823486328125,
      "activations/layer21_attention_weight_min": -23.84295082092285,
      "activations/layer22_attention_weight_max": 26.40011978149414,
      "activations/layer22_attention_weight_min": -27.52263832092285,
      "activations/layer23_attention_weight_max": 33.56517028808594,
      "activations/layer23_attention_weight_min": -23.824430465698242,
      "activations/layer2_attention_weight_max": 31.460432052612305,
      "activations/layer2_attention_weight_min": -30.536001205444336,
      "activations/layer3_attention_weight_max": 86.7430648803711,
      "activations/layer3_attention_weight_min": -85.90571594238281,
      "activations/layer4_attention_weight_max": 85.62354278564453,
      "activations/layer4_attention_weight_min": -85.47298431396484,
      "activations/layer5_attention_weight_max": 60.81385803222656,
      "activations/layer5_attention_weight_min": -72.2911376953125,
      "activations/layer6_attention_weight_max": 50.22261047363281,
      "activations/layer6_attention_weight_min": -52.68000411987305,
      "activations/layer7_attention_weight_max": 70.00550842285156,
      "activations/layer7_attention_weight_min": -66.56649017333984,
      "activations/layer8_attention_weight_max": 45.93577575683594,
      "activations/layer8_attention_weight_min": -51.82696533203125,
      "activations/layer9_attention_weight_max": 54.722198486328125,
      "activations/layer9_attention_weight_min": -49.531280517578125,
      "epoch": 19.15,
      "learning_rate": 2.6748863636363635e-05,
      "loss": 2.7222,
      "step": 329650
    },
    {
      "activations/layer0_attention_weight_max": 15.668707847595215,
      "activations/layer0_attention_weight_min": -13.701355934143066,
      "activations/layer10_attention_weight_max": 35.326778411865234,
      "activations/layer10_attention_weight_min": -35.19548416137695,
      "activations/layer11_attention_weight_max": 32.54685974121094,
      "activations/layer11_attention_weight_min": -32.71919250488281,
      "activations/layer12_attention_weight_max": 17.057598114013672,
      "activations/layer12_attention_weight_min": -24.318191528320312,
      "activations/layer13_attention_weight_max": 32.99067687988281,
      "activations/layer13_attention_weight_min": -27.00276756286621,
      "activations/layer14_attention_weight_max": 32.53767395019531,
      "activations/layer14_attention_weight_min": -28.973690032958984,
      "activations/layer15_attention_weight_max": 34.518733978271484,
      "activations/layer15_attention_weight_min": -28.98029327392578,
      "activations/layer16_attention_weight_max": 28.665987014770508,
      "activations/layer16_attention_weight_min": -28.835914611816406,
      "activations/layer17_attention_weight_max": 42.95052719116211,
      "activations/layer17_attention_weight_min": -42.15739822387695,
      "activations/layer18_attention_weight_max": 40.58580017089844,
      "activations/layer18_attention_weight_min": -38.47323226928711,
      "activations/layer19_attention_weight_max": 20.66945457458496,
      "activations/layer19_attention_weight_min": -21.33786392211914,
      "activations/layer1_attention_weight_max": 15.675846099853516,
      "activations/layer1_attention_weight_min": -13.658499717712402,
      "activations/layer20_attention_weight_max": 19.50829315185547,
      "activations/layer20_attention_weight_min": -21.136672973632812,
      "activations/layer21_attention_weight_max": 32.70319366455078,
      "activations/layer21_attention_weight_min": -25.799591064453125,
      "activations/layer22_attention_weight_max": 30.024953842163086,
      "activations/layer22_attention_weight_min": -24.49859046936035,
      "activations/layer23_attention_weight_max": 32.9564323425293,
      "activations/layer23_attention_weight_min": -23.822141647338867,
      "activations/layer2_attention_weight_max": 30.674434661865234,
      "activations/layer2_attention_weight_min": -28.55168914794922,
      "activations/layer3_attention_weight_max": 85.63390350341797,
      "activations/layer3_attention_weight_min": -81.1275634765625,
      "activations/layer4_attention_weight_max": 80.37062072753906,
      "activations/layer4_attention_weight_min": -79.1314926147461,
      "activations/layer5_attention_weight_max": 60.14562225341797,
      "activations/layer5_attention_weight_min": -73.48513793945312,
      "activations/layer6_attention_weight_max": 49.43876647949219,
      "activations/layer6_attention_weight_min": -51.424991607666016,
      "activations/layer7_attention_weight_max": 61.6617317199707,
      "activations/layer7_attention_weight_min": -61.31052017211914,
      "activations/layer8_attention_weight_max": 44.468196868896484,
      "activations/layer8_attention_weight_min": -48.20281982421875,
      "activations/layer9_attention_weight_max": 46.773475646972656,
      "activations/layer9_attention_weight_min": -45.04476547241211,
      "epoch": 19.16,
      "learning_rate": 2.672992424242424e-05,
      "loss": 2.7279,
      "step": 329700
    },
    {
      "activations/layer0_attention_weight_max": 15.388741493225098,
      "activations/layer0_attention_weight_min": -13.601861000061035,
      "activations/layer10_attention_weight_max": 33.21876907348633,
      "activations/layer10_attention_weight_min": -34.40680694580078,
      "activations/layer11_attention_weight_max": 33.4648323059082,
      "activations/layer11_attention_weight_min": -34.67452621459961,
      "activations/layer12_attention_weight_max": 19.046579360961914,
      "activations/layer12_attention_weight_min": -25.17857551574707,
      "activations/layer13_attention_weight_max": 34.13565444946289,
      "activations/layer13_attention_weight_min": -30.347393035888672,
      "activations/layer14_attention_weight_max": 34.4218864440918,
      "activations/layer14_attention_weight_min": -32.97980880737305,
      "activations/layer15_attention_weight_max": 33.40088653564453,
      "activations/layer15_attention_weight_min": -34.558406829833984,
      "activations/layer16_attention_weight_max": 37.85987091064453,
      "activations/layer16_attention_weight_min": -36.972415924072266,
      "activations/layer17_attention_weight_max": 61.52767562866211,
      "activations/layer17_attention_weight_min": -55.956398010253906,
      "activations/layer18_attention_weight_max": 50.2262077331543,
      "activations/layer18_attention_weight_min": -45.5318489074707,
      "activations/layer19_attention_weight_max": 23.732830047607422,
      "activations/layer19_attention_weight_min": -24.067180633544922,
      "activations/layer1_attention_weight_max": 15.84201717376709,
      "activations/layer1_attention_weight_min": -13.508707046508789,
      "activations/layer20_attention_weight_max": 23.295995712280273,
      "activations/layer20_attention_weight_min": -23.854780197143555,
      "activations/layer21_attention_weight_max": 33.87337112426758,
      "activations/layer21_attention_weight_min": -22.709884643554688,
      "activations/layer22_attention_weight_max": 28.819509506225586,
      "activations/layer22_attention_weight_min": -25.08364486694336,
      "activations/layer23_attention_weight_max": 34.08310317993164,
      "activations/layer23_attention_weight_min": -23.46530532836914,
      "activations/layer2_attention_weight_max": 30.513275146484375,
      "activations/layer2_attention_weight_min": -29.31002426147461,
      "activations/layer3_attention_weight_max": 81.96305084228516,
      "activations/layer3_attention_weight_min": -79.8595199584961,
      "activations/layer4_attention_weight_max": 79.44190216064453,
      "activations/layer4_attention_weight_min": -79.05736541748047,
      "activations/layer5_attention_weight_max": 60.185943603515625,
      "activations/layer5_attention_weight_min": -68.99203491210938,
      "activations/layer6_attention_weight_max": 45.71187210083008,
      "activations/layer6_attention_weight_min": -48.215965270996094,
      "activations/layer7_attention_weight_max": 63.000083923339844,
      "activations/layer7_attention_weight_min": -63.30967712402344,
      "activations/layer8_attention_weight_max": 43.817718505859375,
      "activations/layer8_attention_weight_min": -50.10100555419922,
      "activations/layer9_attention_weight_max": 45.262237548828125,
      "activations/layer9_attention_weight_min": -46.99870681762695,
      "epoch": 19.16,
      "learning_rate": 2.6710984848484848e-05,
      "loss": 2.7106,
      "step": 329750
    },
    {
      "activations/layer0_attention_weight_max": 15.155657768249512,
      "activations/layer0_attention_weight_min": -13.63685131072998,
      "activations/layer10_attention_weight_max": 36.08366012573242,
      "activations/layer10_attention_weight_min": -36.09567642211914,
      "activations/layer11_attention_weight_max": 33.635345458984375,
      "activations/layer11_attention_weight_min": -33.30763244628906,
      "activations/layer12_attention_weight_max": 19.63949966430664,
      "activations/layer12_attention_weight_min": -26.307411193847656,
      "activations/layer13_attention_weight_max": 45.742923736572266,
      "activations/layer13_attention_weight_min": -28.47587013244629,
      "activations/layer14_attention_weight_max": 44.177276611328125,
      "activations/layer14_attention_weight_min": -30.408803939819336,
      "activations/layer15_attention_weight_max": 49.74376678466797,
      "activations/layer15_attention_weight_min": -32.486576080322266,
      "activations/layer16_attention_weight_max": 40.47554397583008,
      "activations/layer16_attention_weight_min": -37.251705169677734,
      "activations/layer17_attention_weight_max": 59.9462776184082,
      "activations/layer17_attention_weight_min": -56.21253967285156,
      "activations/layer18_attention_weight_max": 50.98258590698242,
      "activations/layer18_attention_weight_min": -45.54812240600586,
      "activations/layer19_attention_weight_max": 28.357547760009766,
      "activations/layer19_attention_weight_min": -24.45900535583496,
      "activations/layer1_attention_weight_max": 17.223377227783203,
      "activations/layer1_attention_weight_min": -12.820584297180176,
      "activations/layer20_attention_weight_max": 23.395261764526367,
      "activations/layer20_attention_weight_min": -23.3131046295166,
      "activations/layer21_attention_weight_max": 37.44319534301758,
      "activations/layer21_attention_weight_min": -22.6844482421875,
      "activations/layer22_attention_weight_max": 31.90268898010254,
      "activations/layer22_attention_weight_min": -28.68520736694336,
      "activations/layer23_attention_weight_max": 40.90287780761719,
      "activations/layer23_attention_weight_min": -24.269256591796875,
      "activations/layer2_attention_weight_max": 32.417503356933594,
      "activations/layer2_attention_weight_min": -30.97736167907715,
      "activations/layer3_attention_weight_max": 88.59410095214844,
      "activations/layer3_attention_weight_min": -86.42822265625,
      "activations/layer4_attention_weight_max": 82.6761245727539,
      "activations/layer4_attention_weight_min": -80.73246002197266,
      "activations/layer5_attention_weight_max": 62.551143646240234,
      "activations/layer5_attention_weight_min": -72.84069061279297,
      "activations/layer6_attention_weight_max": 48.086788177490234,
      "activations/layer6_attention_weight_min": -49.563804626464844,
      "activations/layer7_attention_weight_max": 67.4141616821289,
      "activations/layer7_attention_weight_min": -65.0911636352539,
      "activations/layer8_attention_weight_max": 46.63530349731445,
      "activations/layer8_attention_weight_min": -48.43819046020508,
      "activations/layer9_attention_weight_max": 51.928924560546875,
      "activations/layer9_attention_weight_min": -44.05331039428711,
      "epoch": 19.16,
      "learning_rate": 2.669204545454545e-05,
      "loss": 2.7272,
      "step": 329800
    },
    {
      "activations/layer0_attention_weight_max": 16.591236114501953,
      "activations/layer0_attention_weight_min": -14.230058670043945,
      "activations/layer10_attention_weight_max": 36.93682098388672,
      "activations/layer10_attention_weight_min": -36.45642852783203,
      "activations/layer11_attention_weight_max": 34.47047424316406,
      "activations/layer11_attention_weight_min": -34.72552490234375,
      "activations/layer12_attention_weight_max": 19.703426361083984,
      "activations/layer12_attention_weight_min": -26.22024917602539,
      "activations/layer13_attention_weight_max": 36.075740814208984,
      "activations/layer13_attention_weight_min": -28.270822525024414,
      "activations/layer14_attention_weight_max": 37.98806381225586,
      "activations/layer14_attention_weight_min": -30.391904830932617,
      "activations/layer15_attention_weight_max": 33.99367141723633,
      "activations/layer15_attention_weight_min": -30.4024600982666,
      "activations/layer16_attention_weight_max": 37.145137786865234,
      "activations/layer16_attention_weight_min": -32.576507568359375,
      "activations/layer17_attention_weight_max": 52.866737365722656,
      "activations/layer17_attention_weight_min": -44.6723518371582,
      "activations/layer18_attention_weight_max": 50.05670166015625,
      "activations/layer18_attention_weight_min": -38.767486572265625,
      "activations/layer19_attention_weight_max": 25.319578170776367,
      "activations/layer19_attention_weight_min": -23.706218719482422,
      "activations/layer1_attention_weight_max": 16.097190856933594,
      "activations/layer1_attention_weight_min": -12.963648796081543,
      "activations/layer20_attention_weight_max": 26.454261779785156,
      "activations/layer20_attention_weight_min": -20.67338752746582,
      "activations/layer21_attention_weight_max": 38.22829055786133,
      "activations/layer21_attention_weight_min": -22.729013442993164,
      "activations/layer22_attention_weight_max": 28.74451446533203,
      "activations/layer22_attention_weight_min": -24.035831451416016,
      "activations/layer23_attention_weight_max": 39.1468391418457,
      "activations/layer23_attention_weight_min": -23.180099487304688,
      "activations/layer2_attention_weight_max": 31.09780502319336,
      "activations/layer2_attention_weight_min": -29.3349666595459,
      "activations/layer3_attention_weight_max": 84.8685302734375,
      "activations/layer3_attention_weight_min": -86.5589370727539,
      "activations/layer4_attention_weight_max": 85.13990020751953,
      "activations/layer4_attention_weight_min": -87.7948226928711,
      "activations/layer5_attention_weight_max": 62.232940673828125,
      "activations/layer5_attention_weight_min": -72.7681884765625,
      "activations/layer6_attention_weight_max": 49.691219329833984,
      "activations/layer6_attention_weight_min": -51.35729217529297,
      "activations/layer7_attention_weight_max": 65.72998046875,
      "activations/layer7_attention_weight_min": -67.31689453125,
      "activations/layer8_attention_weight_max": 48.88794708251953,
      "activations/layer8_attention_weight_min": -53.129241943359375,
      "activations/layer9_attention_weight_max": 43.216270446777344,
      "activations/layer9_attention_weight_min": -48.94350814819336,
      "epoch": 19.17,
      "learning_rate": 2.6673106060606058e-05,
      "loss": 2.7184,
      "step": 329850
    },
    {
      "activations/layer0_attention_weight_max": 15.99079704284668,
      "activations/layer0_attention_weight_min": -13.711251258850098,
      "activations/layer10_attention_weight_max": 34.957950592041016,
      "activations/layer10_attention_weight_min": -34.15960693359375,
      "activations/layer11_attention_weight_max": 33.35532760620117,
      "activations/layer11_attention_weight_min": -34.87085723876953,
      "activations/layer12_attention_weight_max": 19.964200973510742,
      "activations/layer12_attention_weight_min": -24.28661346435547,
      "activations/layer13_attention_weight_max": 34.36467361450195,
      "activations/layer13_attention_weight_min": -25.97623062133789,
      "activations/layer14_attention_weight_max": 35.61093521118164,
      "activations/layer14_attention_weight_min": -30.20111083984375,
      "activations/layer15_attention_weight_max": 33.03265380859375,
      "activations/layer15_attention_weight_min": -30.76382064819336,
      "activations/layer16_attention_weight_max": 35.92357635498047,
      "activations/layer16_attention_weight_min": -32.05950927734375,
      "activations/layer17_attention_weight_max": 54.622745513916016,
      "activations/layer17_attention_weight_min": -45.91265106201172,
      "activations/layer18_attention_weight_max": 49.34809494018555,
      "activations/layer18_attention_weight_min": -40.567108154296875,
      "activations/layer19_attention_weight_max": 25.359825134277344,
      "activations/layer19_attention_weight_min": -24.842344284057617,
      "activations/layer1_attention_weight_max": 15.400343894958496,
      "activations/layer1_attention_weight_min": -14.311169624328613,
      "activations/layer20_attention_weight_max": 24.11272621154785,
      "activations/layer20_attention_weight_min": -22.603239059448242,
      "activations/layer21_attention_weight_max": 34.52116775512695,
      "activations/layer21_attention_weight_min": -21.612133026123047,
      "activations/layer22_attention_weight_max": 26.747039794921875,
      "activations/layer22_attention_weight_min": -28.04494285583496,
      "activations/layer23_attention_weight_max": 35.34000015258789,
      "activations/layer23_attention_weight_min": -23.850353240966797,
      "activations/layer2_attention_weight_max": 31.079681396484375,
      "activations/layer2_attention_weight_min": -30.07855796813965,
      "activations/layer3_attention_weight_max": 84.65751647949219,
      "activations/layer3_attention_weight_min": -82.25994110107422,
      "activations/layer4_attention_weight_max": 80.23448944091797,
      "activations/layer4_attention_weight_min": -79.01936340332031,
      "activations/layer5_attention_weight_max": 58.85758590698242,
      "activations/layer5_attention_weight_min": -70.92268371582031,
      "activations/layer6_attention_weight_max": 47.271949768066406,
      "activations/layer6_attention_weight_min": -49.60573196411133,
      "activations/layer7_attention_weight_max": 63.65220642089844,
      "activations/layer7_attention_weight_min": -62.99277877807617,
      "activations/layer8_attention_weight_max": 45.39672088623047,
      "activations/layer8_attention_weight_min": -48.941123962402344,
      "activations/layer9_attention_weight_max": 42.562564849853516,
      "activations/layer9_attention_weight_min": -43.819793701171875,
      "epoch": 19.17,
      "learning_rate": 2.6654545454545452e-05,
      "loss": 2.7277,
      "step": 329900
    },
    {
      "activations/layer0_attention_weight_max": 15.24144172668457,
      "activations/layer0_attention_weight_min": -13.579570770263672,
      "activations/layer10_attention_weight_max": 37.291351318359375,
      "activations/layer10_attention_weight_min": -37.1859130859375,
      "activations/layer11_attention_weight_max": 33.69682312011719,
      "activations/layer11_attention_weight_min": -34.62107467651367,
      "activations/layer12_attention_weight_max": 18.208166122436523,
      "activations/layer12_attention_weight_min": -25.083702087402344,
      "activations/layer13_attention_weight_max": 33.54517364501953,
      "activations/layer13_attention_weight_min": -27.543224334716797,
      "activations/layer14_attention_weight_max": 35.71147155761719,
      "activations/layer14_attention_weight_min": -31.095130920410156,
      "activations/layer15_attention_weight_max": 32.79485321044922,
      "activations/layer15_attention_weight_min": -31.87971305847168,
      "activations/layer16_attention_weight_max": 33.689186096191406,
      "activations/layer16_attention_weight_min": -32.99867248535156,
      "activations/layer17_attention_weight_max": 53.1037483215332,
      "activations/layer17_attention_weight_min": -47.536399841308594,
      "activations/layer18_attention_weight_max": 43.88752746582031,
      "activations/layer18_attention_weight_min": -38.93035125732422,
      "activations/layer19_attention_weight_max": 23.263547897338867,
      "activations/layer19_attention_weight_min": -22.700340270996094,
      "activations/layer1_attention_weight_max": 15.83886432647705,
      "activations/layer1_attention_weight_min": -13.330535888671875,
      "activations/layer20_attention_weight_max": 20.557559967041016,
      "activations/layer20_attention_weight_min": -22.249481201171875,
      "activations/layer21_attention_weight_max": 34.14826583862305,
      "activations/layer21_attention_weight_min": -24.24766731262207,
      "activations/layer22_attention_weight_max": 25.88056182861328,
      "activations/layer22_attention_weight_min": -25.180471420288086,
      "activations/layer23_attention_weight_max": 36.07861328125,
      "activations/layer23_attention_weight_min": -23.806119918823242,
      "activations/layer2_attention_weight_max": 31.109237670898438,
      "activations/layer2_attention_weight_min": -30.170753479003906,
      "activations/layer3_attention_weight_max": 84.9970474243164,
      "activations/layer3_attention_weight_min": -82.47440338134766,
      "activations/layer4_attention_weight_max": 85.27399444580078,
      "activations/layer4_attention_weight_min": -81.15625,
      "activations/layer5_attention_weight_max": 65.63861083984375,
      "activations/layer5_attention_weight_min": -74.03323364257812,
      "activations/layer6_attention_weight_max": 50.714168548583984,
      "activations/layer6_attention_weight_min": -50.548458099365234,
      "activations/layer7_attention_weight_max": 66.42858123779297,
      "activations/layer7_attention_weight_min": -61.869476318359375,
      "activations/layer8_attention_weight_max": 49.83397674560547,
      "activations/layer8_attention_weight_min": -49.358028411865234,
      "activations/layer9_attention_weight_max": 45.8198356628418,
      "activations/layer9_attention_weight_min": -44.92207717895508,
      "epoch": 19.17,
      "learning_rate": 2.6635606060606057e-05,
      "loss": 2.7297,
      "step": 329950
    },
    {
      "activations/layer0_attention_weight_max": 16.016843795776367,
      "activations/layer0_attention_weight_min": -13.620030403137207,
      "activations/layer10_attention_weight_max": 36.66566848754883,
      "activations/layer10_attention_weight_min": -37.605587005615234,
      "activations/layer11_attention_weight_max": 34.95265197753906,
      "activations/layer11_attention_weight_min": -35.953121185302734,
      "activations/layer12_attention_weight_max": 21.20966148376465,
      "activations/layer12_attention_weight_min": -22.956111907958984,
      "activations/layer13_attention_weight_max": 32.94737243652344,
      "activations/layer13_attention_weight_min": -27.866134643554688,
      "activations/layer14_attention_weight_max": 36.380889892578125,
      "activations/layer14_attention_weight_min": -30.527231216430664,
      "activations/layer15_attention_weight_max": 34.063575744628906,
      "activations/layer15_attention_weight_min": -31.00725555419922,
      "activations/layer16_attention_weight_max": 36.514060974121094,
      "activations/layer16_attention_weight_min": -33.21831130981445,
      "activations/layer17_attention_weight_max": 54.52104568481445,
      "activations/layer17_attention_weight_min": -45.49415588378906,
      "activations/layer18_attention_weight_max": 51.03213882446289,
      "activations/layer18_attention_weight_min": -39.701786041259766,
      "activations/layer19_attention_weight_max": 25.280529022216797,
      "activations/layer19_attention_weight_min": -22.806455612182617,
      "activations/layer1_attention_weight_max": 15.567736625671387,
      "activations/layer1_attention_weight_min": -14.569432258605957,
      "activations/layer20_attention_weight_max": 25.449861526489258,
      "activations/layer20_attention_weight_min": -23.967247009277344,
      "activations/layer21_attention_weight_max": 42.35738754272461,
      "activations/layer21_attention_weight_min": -27.13664436340332,
      "activations/layer22_attention_weight_max": 32.48073959350586,
      "activations/layer22_attention_weight_min": -26.222145080566406,
      "activations/layer23_attention_weight_max": 42.43297576904297,
      "activations/layer23_attention_weight_min": -23.307613372802734,
      "activations/layer2_attention_weight_max": 29.290876388549805,
      "activations/layer2_attention_weight_min": -28.00611686706543,
      "activations/layer3_attention_weight_max": 85.36872863769531,
      "activations/layer3_attention_weight_min": -82.56153106689453,
      "activations/layer4_attention_weight_max": 82.2103500366211,
      "activations/layer4_attention_weight_min": -81.66468811035156,
      "activations/layer5_attention_weight_max": 62.3567008972168,
      "activations/layer5_attention_weight_min": -70.56416320800781,
      "activations/layer6_attention_weight_max": 48.145992279052734,
      "activations/layer6_attention_weight_min": -51.51945877075195,
      "activations/layer7_attention_weight_max": 64.6049575805664,
      "activations/layer7_attention_weight_min": -64.78649139404297,
      "activations/layer8_attention_weight_max": 47.8924674987793,
      "activations/layer8_attention_weight_min": -51.163936614990234,
      "activations/layer9_attention_weight_max": 45.559993743896484,
      "activations/layer9_attention_weight_min": -47.383399963378906,
      "epoch": 19.17,
      "learning_rate": 2.6616666666666662e-05,
      "loss": 2.7191,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.7331,
      "eval_samples_per_second": 491.694,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.7331,
      "eval_openwebtext_samples_per_second": 491.694,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 2.0005,
      "eval_wikitext_samples_per_second": 227.941,
      "step": 330000
    },
    {
      "epoch": 19.17,
      "eval_lambada_loss": 2.412109375,
      "eval_lambada_ppl": 11.157471631854794,
      "eval_lambada_runtime": 9.6483,
      "eval_lambada_samples_per_second": 504.647,
      "step": 330000
    },
    {
      "activations/layer0_attention_weight_max": 14.812355995178223,
      "activations/layer0_attention_weight_min": -13.403985977172852,
      "activations/layer10_attention_weight_max": 35.29861831665039,
      "activations/layer10_attention_weight_min": -35.5814208984375,
      "activations/layer11_attention_weight_max": 33.639923095703125,
      "activations/layer11_attention_weight_min": -35.25666046142578,
      "activations/layer12_attention_weight_max": 20.89352798461914,
      "activations/layer12_attention_weight_min": -31.509849548339844,
      "activations/layer13_attention_weight_max": 33.09950256347656,
      "activations/layer13_attention_weight_min": -26.48752784729004,
      "activations/layer14_attention_weight_max": 35.24089050292969,
      "activations/layer14_attention_weight_min": -29.87158203125,
      "activations/layer15_attention_weight_max": 33.669681549072266,
      "activations/layer15_attention_weight_min": -31.19200897216797,
      "activations/layer16_attention_weight_max": 35.13691711425781,
      "activations/layer16_attention_weight_min": -31.73493194580078,
      "activations/layer17_attention_weight_max": 51.123504638671875,
      "activations/layer17_attention_weight_min": -46.05595779418945,
      "activations/layer18_attention_weight_max": 50.16254806518555,
      "activations/layer18_attention_weight_min": -42.37628936767578,
      "activations/layer19_attention_weight_max": 23.586210250854492,
      "activations/layer19_attention_weight_min": -23.62447738647461,
      "activations/layer1_attention_weight_max": 16.067712783813477,
      "activations/layer1_attention_weight_min": -13.359749794006348,
      "activations/layer20_attention_weight_max": 22.744487762451172,
      "activations/layer20_attention_weight_min": -24.07775115966797,
      "activations/layer21_attention_weight_max": 34.7641716003418,
      "activations/layer21_attention_weight_min": -20.654844284057617,
      "activations/layer22_attention_weight_max": 34.05142593383789,
      "activations/layer22_attention_weight_min": -25.655277252197266,
      "activations/layer23_attention_weight_max": 36.89457702636719,
      "activations/layer23_attention_weight_min": -24.508872985839844,
      "activations/layer2_attention_weight_max": 32.334537506103516,
      "activations/layer2_attention_weight_min": -31.223770141601562,
      "activations/layer3_attention_weight_max": 87.28309631347656,
      "activations/layer3_attention_weight_min": -82.3775405883789,
      "activations/layer4_attention_weight_max": 81.49239349365234,
      "activations/layer4_attention_weight_min": -82.78241729736328,
      "activations/layer5_attention_weight_max": 62.509490966796875,
      "activations/layer5_attention_weight_min": -73.17894744873047,
      "activations/layer6_attention_weight_max": 52.1692008972168,
      "activations/layer6_attention_weight_min": -54.55223846435547,
      "activations/layer7_attention_weight_max": 65.05824279785156,
      "activations/layer7_attention_weight_min": -65.97489166259766,
      "activations/layer8_attention_weight_max": 47.88050079345703,
      "activations/layer8_attention_weight_min": -50.47849655151367,
      "activations/layer9_attention_weight_max": 44.4708366394043,
      "activations/layer9_attention_weight_min": -44.85237121582031,
      "epoch": 19.18,
      "learning_rate": 2.659772727272727e-05,
      "loss": 2.7034,
      "step": 330050
    },
    {
      "activations/layer0_attention_weight_max": 14.689321517944336,
      "activations/layer0_attention_weight_min": -13.74268627166748,
      "activations/layer10_attention_weight_max": 37.42661666870117,
      "activations/layer10_attention_weight_min": -35.8076057434082,
      "activations/layer11_attention_weight_max": 34.566131591796875,
      "activations/layer11_attention_weight_min": -34.56332015991211,
      "activations/layer12_attention_weight_max": 19.642019271850586,
      "activations/layer12_attention_weight_min": -25.246782302856445,
      "activations/layer13_attention_weight_max": 37.5703239440918,
      "activations/layer13_attention_weight_min": -26.8016414642334,
      "activations/layer14_attention_weight_max": 37.624324798583984,
      "activations/layer14_attention_weight_min": -30.89939308166504,
      "activations/layer15_attention_weight_max": 31.99365997314453,
      "activations/layer15_attention_weight_min": -30.0441837310791,
      "activations/layer16_attention_weight_max": 34.88363265991211,
      "activations/layer16_attention_weight_min": -33.38223648071289,
      "activations/layer17_attention_weight_max": 50.83734893798828,
      "activations/layer17_attention_weight_min": -46.78794479370117,
      "activations/layer18_attention_weight_max": 47.535762786865234,
      "activations/layer18_attention_weight_min": -40.58315658569336,
      "activations/layer19_attention_weight_max": 25.371749877929688,
      "activations/layer19_attention_weight_min": -24.268550872802734,
      "activations/layer1_attention_weight_max": 16.438241958618164,
      "activations/layer1_attention_weight_min": -13.304424285888672,
      "activations/layer20_attention_weight_max": 24.396549224853516,
      "activations/layer20_attention_weight_min": -20.93876838684082,
      "activations/layer21_attention_weight_max": 43.2423095703125,
      "activations/layer21_attention_weight_min": -23.109384536743164,
      "activations/layer22_attention_weight_max": 30.19150733947754,
      "activations/layer22_attention_weight_min": -24.038999557495117,
      "activations/layer23_attention_weight_max": 40.31184005737305,
      "activations/layer23_attention_weight_min": -21.304744720458984,
      "activations/layer2_attention_weight_max": 31.956592559814453,
      "activations/layer2_attention_weight_min": -31.488065719604492,
      "activations/layer3_attention_weight_max": 86.36287689208984,
      "activations/layer3_attention_weight_min": -88.38294982910156,
      "activations/layer4_attention_weight_max": 78.86266326904297,
      "activations/layer4_attention_weight_min": -81.91512298583984,
      "activations/layer5_attention_weight_max": 59.94642639160156,
      "activations/layer5_attention_weight_min": -70.06288146972656,
      "activations/layer6_attention_weight_max": 51.54054641723633,
      "activations/layer6_attention_weight_min": -49.388954162597656,
      "activations/layer7_attention_weight_max": 68.55786895751953,
      "activations/layer7_attention_weight_min": -67.62786102294922,
      "activations/layer8_attention_weight_max": 47.17673110961914,
      "activations/layer8_attention_weight_min": -48.27719497680664,
      "activations/layer9_attention_weight_max": 44.91114807128906,
      "activations/layer9_attention_weight_min": -45.408382415771484,
      "epoch": 19.18,
      "learning_rate": 2.6578787878787875e-05,
      "loss": 2.7106,
      "step": 330100
    },
    {
      "activations/layer0_attention_weight_max": 15.642909049987793,
      "activations/layer0_attention_weight_min": -13.38003921508789,
      "activations/layer10_attention_weight_max": 32.72986602783203,
      "activations/layer10_attention_weight_min": -32.481903076171875,
      "activations/layer11_attention_weight_max": 30.3164119720459,
      "activations/layer11_attention_weight_min": -32.904640197753906,
      "activations/layer12_attention_weight_max": 17.639114379882812,
      "activations/layer12_attention_weight_min": -27.297765731811523,
      "activations/layer13_attention_weight_max": 33.26021957397461,
      "activations/layer13_attention_weight_min": -27.072484970092773,
      "activations/layer14_attention_weight_max": 33.262813568115234,
      "activations/layer14_attention_weight_min": -30.545028686523438,
      "activations/layer15_attention_weight_max": 30.63251495361328,
      "activations/layer15_attention_weight_min": -30.161535263061523,
      "activations/layer16_attention_weight_max": 33.04666519165039,
      "activations/layer16_attention_weight_min": -32.34058380126953,
      "activations/layer17_attention_weight_max": 48.86866760253906,
      "activations/layer17_attention_weight_min": -43.994510650634766,
      "activations/layer18_attention_weight_max": 44.51886749267578,
      "activations/layer18_attention_weight_min": -38.032047271728516,
      "activations/layer19_attention_weight_max": 21.43476676940918,
      "activations/layer19_attention_weight_min": -22.78934097290039,
      "activations/layer1_attention_weight_max": 16.313020706176758,
      "activations/layer1_attention_weight_min": -14.561559677124023,
      "activations/layer20_attention_weight_max": 21.341075897216797,
      "activations/layer20_attention_weight_min": -20.31003761291504,
      "activations/layer21_attention_weight_max": 34.915496826171875,
      "activations/layer21_attention_weight_min": -22.69110107421875,
      "activations/layer22_attention_weight_max": 27.642772674560547,
      "activations/layer22_attention_weight_min": -25.832992553710938,
      "activations/layer23_attention_weight_max": 39.309940338134766,
      "activations/layer23_attention_weight_min": -23.65088653564453,
      "activations/layer2_attention_weight_max": 30.357681274414062,
      "activations/layer2_attention_weight_min": -29.00384521484375,
      "activations/layer3_attention_weight_max": 84.90606689453125,
      "activations/layer3_attention_weight_min": -80.7349853515625,
      "activations/layer4_attention_weight_max": 81.59796905517578,
      "activations/layer4_attention_weight_min": -78.3087158203125,
      "activations/layer5_attention_weight_max": 64.19387817382812,
      "activations/layer5_attention_weight_min": -67.04496765136719,
      "activations/layer6_attention_weight_max": 50.24166488647461,
      "activations/layer6_attention_weight_min": -47.17378234863281,
      "activations/layer7_attention_weight_max": 68.44645690917969,
      "activations/layer7_attention_weight_min": -60.14749526977539,
      "activations/layer8_attention_weight_max": 42.730350494384766,
      "activations/layer8_attention_weight_min": -44.89617156982422,
      "activations/layer9_attention_weight_max": 38.63777160644531,
      "activations/layer9_attention_weight_min": -42.40043258666992,
      "epoch": 19.18,
      "learning_rate": 2.6559848484848483e-05,
      "loss": 2.72,
      "step": 330150
    },
    {
      "activations/layer0_attention_weight_max": 15.06935977935791,
      "activations/layer0_attention_weight_min": -13.588595390319824,
      "activations/layer10_attention_weight_max": 36.585243225097656,
      "activations/layer10_attention_weight_min": -35.82244873046875,
      "activations/layer11_attention_weight_max": 35.82514190673828,
      "activations/layer11_attention_weight_min": -35.01116180419922,
      "activations/layer12_attention_weight_max": 18.429105758666992,
      "activations/layer12_attention_weight_min": -25.550241470336914,
      "activations/layer13_attention_weight_max": 33.20128631591797,
      "activations/layer13_attention_weight_min": -27.29673194885254,
      "activations/layer14_attention_weight_max": 32.21723175048828,
      "activations/layer14_attention_weight_min": -30.211191177368164,
      "activations/layer15_attention_weight_max": 31.090667724609375,
      "activations/layer15_attention_weight_min": -31.88470458984375,
      "activations/layer16_attention_weight_max": 32.621299743652344,
      "activations/layer16_attention_weight_min": -32.0505485534668,
      "activations/layer17_attention_weight_max": 51.89363479614258,
      "activations/layer17_attention_weight_min": -43.66047286987305,
      "activations/layer18_attention_weight_max": 44.533626556396484,
      "activations/layer18_attention_weight_min": -37.92930603027344,
      "activations/layer19_attention_weight_max": 22.19649887084961,
      "activations/layer19_attention_weight_min": -22.764381408691406,
      "activations/layer1_attention_weight_max": 16.310466766357422,
      "activations/layer1_attention_weight_min": -13.278675079345703,
      "activations/layer20_attention_weight_max": 21.922710418701172,
      "activations/layer20_attention_weight_min": -21.251510620117188,
      "activations/layer21_attention_weight_max": 35.21601867675781,
      "activations/layer21_attention_weight_min": -21.88090705871582,
      "activations/layer22_attention_weight_max": 28.133474349975586,
      "activations/layer22_attention_weight_min": -25.329296112060547,
      "activations/layer23_attention_weight_max": 34.66929626464844,
      "activations/layer23_attention_weight_min": -24.601110458374023,
      "activations/layer2_attention_weight_max": 29.785245895385742,
      "activations/layer2_attention_weight_min": -30.135412216186523,
      "activations/layer3_attention_weight_max": 84.31288146972656,
      "activations/layer3_attention_weight_min": -86.17346954345703,
      "activations/layer4_attention_weight_max": 80.55220794677734,
      "activations/layer4_attention_weight_min": -79.21653747558594,
      "activations/layer5_attention_weight_max": 57.002952575683594,
      "activations/layer5_attention_weight_min": -63.712745666503906,
      "activations/layer6_attention_weight_max": 43.75434494018555,
      "activations/layer6_attention_weight_min": -49.05720520019531,
      "activations/layer7_attention_weight_max": 60.05649948120117,
      "activations/layer7_attention_weight_min": -61.822853088378906,
      "activations/layer8_attention_weight_max": 44.466529846191406,
      "activations/layer8_attention_weight_min": -47.94188690185547,
      "activations/layer9_attention_weight_max": 44.09626770019531,
      "activations/layer9_attention_weight_min": -43.4833984375,
      "epoch": 19.19,
      "learning_rate": 2.654090909090909e-05,
      "loss": 2.718,
      "step": 330200
    },
    {
      "activations/layer0_attention_weight_max": 15.211753845214844,
      "activations/layer0_attention_weight_min": -13.703210830688477,
      "activations/layer10_attention_weight_max": 37.67814636230469,
      "activations/layer10_attention_weight_min": -38.179534912109375,
      "activations/layer11_attention_weight_max": 36.36433792114258,
      "activations/layer11_attention_weight_min": -35.92784118652344,
      "activations/layer12_attention_weight_max": 20.790145874023438,
      "activations/layer12_attention_weight_min": -25.133785247802734,
      "activations/layer13_attention_weight_max": 41.95432662963867,
      "activations/layer13_attention_weight_min": -31.47394561767578,
      "activations/layer14_attention_weight_max": 37.92860794067383,
      "activations/layer14_attention_weight_min": -31.806074142456055,
      "activations/layer15_attention_weight_max": 37.79396057128906,
      "activations/layer15_attention_weight_min": -32.16735076904297,
      "activations/layer16_attention_weight_max": 40.76899337768555,
      "activations/layer16_attention_weight_min": -35.685916900634766,
      "activations/layer17_attention_weight_max": 59.75507354736328,
      "activations/layer17_attention_weight_min": -49.416717529296875,
      "activations/layer18_attention_weight_max": 57.65983581542969,
      "activations/layer18_attention_weight_min": -42.35568618774414,
      "activations/layer19_attention_weight_max": 29.730972290039062,
      "activations/layer19_attention_weight_min": -24.70994758605957,
      "activations/layer1_attention_weight_max": 15.245320320129395,
      "activations/layer1_attention_weight_min": -13.333645820617676,
      "activations/layer20_attention_weight_max": 29.166332244873047,
      "activations/layer20_attention_weight_min": -22.997758865356445,
      "activations/layer21_attention_weight_max": 47.27030563354492,
      "activations/layer21_attention_weight_min": -23.728179931640625,
      "activations/layer22_attention_weight_max": 36.2872314453125,
      "activations/layer22_attention_weight_min": -28.727252960205078,
      "activations/layer23_attention_weight_max": 48.63072967529297,
      "activations/layer23_attention_weight_min": -23.946533203125,
      "activations/layer2_attention_weight_max": 29.308279037475586,
      "activations/layer2_attention_weight_min": -29.33302879333496,
      "activations/layer3_attention_weight_max": 79.93877410888672,
      "activations/layer3_attention_weight_min": -83.57598114013672,
      "activations/layer4_attention_weight_max": 81.08358001708984,
      "activations/layer4_attention_weight_min": -80.3130111694336,
      "activations/layer5_attention_weight_max": 60.702972412109375,
      "activations/layer5_attention_weight_min": -71.87094116210938,
      "activations/layer6_attention_weight_max": 51.37906265258789,
      "activations/layer6_attention_weight_min": -52.574031829833984,
      "activations/layer7_attention_weight_max": 70.01226043701172,
      "activations/layer7_attention_weight_min": -64.53795623779297,
      "activations/layer8_attention_weight_max": 51.17323684692383,
      "activations/layer8_attention_weight_min": -50.9409294128418,
      "activations/layer9_attention_weight_max": 47.520015716552734,
      "activations/layer9_attention_weight_min": -47.15582275390625,
      "epoch": 19.19,
      "learning_rate": 2.6521969696969693e-05,
      "loss": 2.7269,
      "step": 330250
    },
    {
      "activations/layer0_attention_weight_max": 15.829500198364258,
      "activations/layer0_attention_weight_min": -13.147830963134766,
      "activations/layer10_attention_weight_max": 36.9833869934082,
      "activations/layer10_attention_weight_min": -36.53955841064453,
      "activations/layer11_attention_weight_max": 32.493404388427734,
      "activations/layer11_attention_weight_min": -33.49943923950195,
      "activations/layer12_attention_weight_max": 18.57625961303711,
      "activations/layer12_attention_weight_min": -25.777135848999023,
      "activations/layer13_attention_weight_max": 32.414791107177734,
      "activations/layer13_attention_weight_min": -25.259092330932617,
      "activations/layer14_attention_weight_max": 35.57119369506836,
      "activations/layer14_attention_weight_min": -30.846080780029297,
      "activations/layer15_attention_weight_max": 34.497161865234375,
      "activations/layer15_attention_weight_min": -31.38792610168457,
      "activations/layer16_attention_weight_max": 35.725711822509766,
      "activations/layer16_attention_weight_min": -35.1164436340332,
      "activations/layer17_attention_weight_max": 56.58070755004883,
      "activations/layer17_attention_weight_min": -46.8599853515625,
      "activations/layer18_attention_weight_max": 51.00481414794922,
      "activations/layer18_attention_weight_min": -42.48044204711914,
      "activations/layer19_attention_weight_max": 26.300201416015625,
      "activations/layer19_attention_weight_min": -25.34686279296875,
      "activations/layer1_attention_weight_max": 14.971363067626953,
      "activations/layer1_attention_weight_min": -14.528401374816895,
      "activations/layer20_attention_weight_max": 22.8415584564209,
      "activations/layer20_attention_weight_min": -21.838726043701172,
      "activations/layer21_attention_weight_max": 37.12514114379883,
      "activations/layer21_attention_weight_min": -24.237071990966797,
      "activations/layer22_attention_weight_max": 27.166915893554688,
      "activations/layer22_attention_weight_min": -25.832265853881836,
      "activations/layer23_attention_weight_max": 35.319576263427734,
      "activations/layer23_attention_weight_min": -26.324779510498047,
      "activations/layer2_attention_weight_max": 29.74384307861328,
      "activations/layer2_attention_weight_min": -28.594823837280273,
      "activations/layer3_attention_weight_max": 86.53422546386719,
      "activations/layer3_attention_weight_min": -82.73180389404297,
      "activations/layer4_attention_weight_max": 83.35572052001953,
      "activations/layer4_attention_weight_min": -82.8279037475586,
      "activations/layer5_attention_weight_max": 63.231834411621094,
      "activations/layer5_attention_weight_min": -72.91947174072266,
      "activations/layer6_attention_weight_max": 48.99964141845703,
      "activations/layer6_attention_weight_min": -52.41434097290039,
      "activations/layer7_attention_weight_max": 59.957332611083984,
      "activations/layer7_attention_weight_min": -63.38212585449219,
      "activations/layer8_attention_weight_max": 45.702110290527344,
      "activations/layer8_attention_weight_min": -50.133277893066406,
      "activations/layer9_attention_weight_max": 41.491573333740234,
      "activations/layer9_attention_weight_min": -44.484432220458984,
      "epoch": 19.19,
      "learning_rate": 2.65030303030303e-05,
      "loss": 2.7186,
      "step": 330300
    },
    {
      "activations/layer0_attention_weight_max": 16.30256462097168,
      "activations/layer0_attention_weight_min": -13.17760181427002,
      "activations/layer10_attention_weight_max": 36.04829788208008,
      "activations/layer10_attention_weight_min": -36.909725189208984,
      "activations/layer11_attention_weight_max": 33.15424728393555,
      "activations/layer11_attention_weight_min": -35.234291076660156,
      "activations/layer12_attention_weight_max": 19.76921272277832,
      "activations/layer12_attention_weight_min": -27.270069122314453,
      "activations/layer13_attention_weight_max": 36.67026901245117,
      "activations/layer13_attention_weight_min": -28.29390525817871,
      "activations/layer14_attention_weight_max": 37.84502029418945,
      "activations/layer14_attention_weight_min": -30.433263778686523,
      "activations/layer15_attention_weight_max": 37.940635681152344,
      "activations/layer15_attention_weight_min": -29.25604248046875,
      "activations/layer16_attention_weight_max": 34.69927978515625,
      "activations/layer16_attention_weight_min": -32.11466979980469,
      "activations/layer17_attention_weight_max": 56.810115814208984,
      "activations/layer17_attention_weight_min": -45.02036666870117,
      "activations/layer18_attention_weight_max": 49.397377014160156,
      "activations/layer18_attention_weight_min": -41.09779357910156,
      "activations/layer19_attention_weight_max": 24.974645614624023,
      "activations/layer19_attention_weight_min": -23.489669799804688,
      "activations/layer1_attention_weight_max": 15.56863021850586,
      "activations/layer1_attention_weight_min": -13.152870178222656,
      "activations/layer20_attention_weight_max": 26.437755584716797,
      "activations/layer20_attention_weight_min": -20.810020446777344,
      "activations/layer21_attention_weight_max": 44.85105514526367,
      "activations/layer21_attention_weight_min": -23.021930694580078,
      "activations/layer22_attention_weight_max": 32.64714050292969,
      "activations/layer22_attention_weight_min": -24.66716766357422,
      "activations/layer23_attention_weight_max": 42.86802291870117,
      "activations/layer23_attention_weight_min": -23.574079513549805,
      "activations/layer2_attention_weight_max": 29.948009490966797,
      "activations/layer2_attention_weight_min": -28.687780380249023,
      "activations/layer3_attention_weight_max": 83.81727600097656,
      "activations/layer3_attention_weight_min": -83.90481567382812,
      "activations/layer4_attention_weight_max": 74.23731231689453,
      "activations/layer4_attention_weight_min": -76.74777221679688,
      "activations/layer5_attention_weight_max": 62.415931701660156,
      "activations/layer5_attention_weight_min": -71.2103042602539,
      "activations/layer6_attention_weight_max": 46.49891662597656,
      "activations/layer6_attention_weight_min": -50.884300231933594,
      "activations/layer7_attention_weight_max": 61.827049255371094,
      "activations/layer7_attention_weight_min": -62.384517669677734,
      "activations/layer8_attention_weight_max": 45.447444915771484,
      "activations/layer8_attention_weight_min": -51.4771842956543,
      "activations/layer9_attention_weight_max": 43.173946380615234,
      "activations/layer9_attention_weight_min": -45.60175704956055,
      "epoch": 19.2,
      "learning_rate": 2.6484090909090906e-05,
      "loss": 2.6995,
      "step": 330350
    },
    {
      "activations/layer0_attention_weight_max": 15.958687782287598,
      "activations/layer0_attention_weight_min": -13.654565811157227,
      "activations/layer10_attention_weight_max": 36.1071891784668,
      "activations/layer10_attention_weight_min": -36.397361755371094,
      "activations/layer11_attention_weight_max": 33.62038803100586,
      "activations/layer11_attention_weight_min": -34.866390228271484,
      "activations/layer12_attention_weight_max": 21.0467586517334,
      "activations/layer12_attention_weight_min": -27.79879379272461,
      "activations/layer13_attention_weight_max": 36.71479415893555,
      "activations/layer13_attention_weight_min": -30.74510955810547,
      "activations/layer14_attention_weight_max": 39.095245361328125,
      "activations/layer14_attention_weight_min": -32.44645690917969,
      "activations/layer15_attention_weight_max": 34.73491287231445,
      "activations/layer15_attention_weight_min": -31.311525344848633,
      "activations/layer16_attention_weight_max": 35.22231674194336,
      "activations/layer16_attention_weight_min": -32.065364837646484,
      "activations/layer17_attention_weight_max": 54.18497848510742,
      "activations/layer17_attention_weight_min": -45.040382385253906,
      "activations/layer18_attention_weight_max": 47.84536361694336,
      "activations/layer18_attention_weight_min": -39.758888244628906,
      "activations/layer19_attention_weight_max": 25.9259090423584,
      "activations/layer19_attention_weight_min": -22.0921573638916,
      "activations/layer1_attention_weight_max": 15.793391227722168,
      "activations/layer1_attention_weight_min": -13.051702499389648,
      "activations/layer20_attention_weight_max": 23.667831420898438,
      "activations/layer20_attention_weight_min": -23.265989303588867,
      "activations/layer21_attention_weight_max": 38.181549072265625,
      "activations/layer21_attention_weight_min": -24.126262664794922,
      "activations/layer22_attention_weight_max": 27.579404830932617,
      "activations/layer22_attention_weight_min": -26.952993392944336,
      "activations/layer23_attention_weight_max": 33.81428146362305,
      "activations/layer23_attention_weight_min": -23.96797752380371,
      "activations/layer2_attention_weight_max": 29.94357681274414,
      "activations/layer2_attention_weight_min": -30.229473114013672,
      "activations/layer3_attention_weight_max": 87.53948974609375,
      "activations/layer3_attention_weight_min": -89.51167297363281,
      "activations/layer4_attention_weight_max": 82.96863555908203,
      "activations/layer4_attention_weight_min": -82.71858978271484,
      "activations/layer5_attention_weight_max": 62.55426788330078,
      "activations/layer5_attention_weight_min": -72.52845001220703,
      "activations/layer6_attention_weight_max": 48.045448303222656,
      "activations/layer6_attention_weight_min": -49.53059387207031,
      "activations/layer7_attention_weight_max": 65.79751586914062,
      "activations/layer7_attention_weight_min": -66.39567565917969,
      "activations/layer8_attention_weight_max": 47.61320114135742,
      "activations/layer8_attention_weight_min": -51.03668212890625,
      "activations/layer9_attention_weight_max": 45.28083801269531,
      "activations/layer9_attention_weight_min": -45.47969436645508,
      "epoch": 19.2,
      "learning_rate": 2.6465151515151515e-05,
      "loss": 2.7297,
      "step": 330400
    },
    {
      "activations/layer0_attention_weight_max": 16.13775062561035,
      "activations/layer0_attention_weight_min": -13.505361557006836,
      "activations/layer10_attention_weight_max": 43.21550369262695,
      "activations/layer10_attention_weight_min": -41.792808532714844,
      "activations/layer11_attention_weight_max": 42.92487716674805,
      "activations/layer11_attention_weight_min": -39.617401123046875,
      "activations/layer12_attention_weight_max": 20.634830474853516,
      "activations/layer12_attention_weight_min": -27.26609992980957,
      "activations/layer13_attention_weight_max": 45.93265151977539,
      "activations/layer13_attention_weight_min": -31.330120086669922,
      "activations/layer14_attention_weight_max": 49.872413635253906,
      "activations/layer14_attention_weight_min": -32.40866470336914,
      "activations/layer15_attention_weight_max": 50.80580139160156,
      "activations/layer15_attention_weight_min": -31.277359008789062,
      "activations/layer16_attention_weight_max": 42.92726135253906,
      "activations/layer16_attention_weight_min": -37.78092575073242,
      "activations/layer17_attention_weight_max": 60.45220947265625,
      "activations/layer17_attention_weight_min": -49.65481185913086,
      "activations/layer18_attention_weight_max": 55.55348587036133,
      "activations/layer18_attention_weight_min": -47.94404220581055,
      "activations/layer19_attention_weight_max": 32.608524322509766,
      "activations/layer19_attention_weight_min": -28.43086814880371,
      "activations/layer1_attention_weight_max": 16.193416595458984,
      "activations/layer1_attention_weight_min": -13.037464141845703,
      "activations/layer20_attention_weight_max": 34.3141975402832,
      "activations/layer20_attention_weight_min": -23.332979202270508,
      "activations/layer21_attention_weight_max": 51.71446228027344,
      "activations/layer21_attention_weight_min": -22.90250015258789,
      "activations/layer22_attention_weight_max": 39.08747482299805,
      "activations/layer22_attention_weight_min": -26.404451370239258,
      "activations/layer23_attention_weight_max": 42.75696563720703,
      "activations/layer23_attention_weight_min": -25.078514099121094,
      "activations/layer2_attention_weight_max": 31.265682220458984,
      "activations/layer2_attention_weight_min": -31.169675827026367,
      "activations/layer3_attention_weight_max": 85.15628051757812,
      "activations/layer3_attention_weight_min": -84.07979583740234,
      "activations/layer4_attention_weight_max": 80.46361541748047,
      "activations/layer4_attention_weight_min": -79.0210189819336,
      "activations/layer5_attention_weight_max": 62.40827178955078,
      "activations/layer5_attention_weight_min": -73.585205078125,
      "activations/layer6_attention_weight_max": 49.14398956298828,
      "activations/layer6_attention_weight_min": -51.265201568603516,
      "activations/layer7_attention_weight_max": 79.2763671875,
      "activations/layer7_attention_weight_min": -71.95794677734375,
      "activations/layer8_attention_weight_max": 54.081546783447266,
      "activations/layer8_attention_weight_min": -54.82164001464844,
      "activations/layer9_attention_weight_max": 55.9505729675293,
      "activations/layer9_attention_weight_min": -46.7154426574707,
      "epoch": 19.2,
      "learning_rate": 2.644621212121212e-05,
      "loss": 2.7205,
      "step": 330450
    },
    {
      "activations/layer0_attention_weight_max": 15.465134620666504,
      "activations/layer0_attention_weight_min": -13.973000526428223,
      "activations/layer10_attention_weight_max": 37.756385803222656,
      "activations/layer10_attention_weight_min": -36.335777282714844,
      "activations/layer11_attention_weight_max": 34.3033561706543,
      "activations/layer11_attention_weight_min": -34.79071807861328,
      "activations/layer12_attention_weight_max": 19.812604904174805,
      "activations/layer12_attention_weight_min": -26.375778198242188,
      "activations/layer13_attention_weight_max": 32.23120880126953,
      "activations/layer13_attention_weight_min": -26.37386703491211,
      "activations/layer14_attention_weight_max": 35.04856872558594,
      "activations/layer14_attention_weight_min": -29.725507736206055,
      "activations/layer15_attention_weight_max": 33.42948532104492,
      "activations/layer15_attention_weight_min": -30.198726654052734,
      "activations/layer16_attention_weight_max": 33.83161163330078,
      "activations/layer16_attention_weight_min": -32.218040466308594,
      "activations/layer17_attention_weight_max": 54.770965576171875,
      "activations/layer17_attention_weight_min": -46.11307907104492,
      "activations/layer18_attention_weight_max": 49.40229034423828,
      "activations/layer18_attention_weight_min": -40.913753509521484,
      "activations/layer19_attention_weight_max": 23.66607666015625,
      "activations/layer19_attention_weight_min": -21.281736373901367,
      "activations/layer1_attention_weight_max": 16.409175872802734,
      "activations/layer1_attention_weight_min": -12.487674713134766,
      "activations/layer20_attention_weight_max": 20.609392166137695,
      "activations/layer20_attention_weight_min": -20.479461669921875,
      "activations/layer21_attention_weight_max": 31.484647750854492,
      "activations/layer21_attention_weight_min": -20.81627655029297,
      "activations/layer22_attention_weight_max": 28.69219207763672,
      "activations/layer22_attention_weight_min": -25.23467254638672,
      "activations/layer23_attention_weight_max": 39.03063201904297,
      "activations/layer23_attention_weight_min": -23.333728790283203,
      "activations/layer2_attention_weight_max": 32.184505462646484,
      "activations/layer2_attention_weight_min": -29.25112533569336,
      "activations/layer3_attention_weight_max": 85.1644287109375,
      "activations/layer3_attention_weight_min": -86.7145004272461,
      "activations/layer4_attention_weight_max": 84.45842742919922,
      "activations/layer4_attention_weight_min": -85.32550048828125,
      "activations/layer5_attention_weight_max": 62.04914093017578,
      "activations/layer5_attention_weight_min": -70.44523620605469,
      "activations/layer6_attention_weight_max": 49.69297790527344,
      "activations/layer6_attention_weight_min": -50.85563278198242,
      "activations/layer7_attention_weight_max": 64.35771179199219,
      "activations/layer7_attention_weight_min": -62.193031311035156,
      "activations/layer8_attention_weight_max": 49.00206756591797,
      "activations/layer8_attention_weight_min": -52.839656829833984,
      "activations/layer9_attention_weight_max": 45.08705139160156,
      "activations/layer9_attention_weight_min": -48.610130310058594,
      "epoch": 19.2,
      "learning_rate": 2.6427272727272728e-05,
      "loss": 2.7284,
      "step": 330500
    },
    {
      "activations/layer0_attention_weight_max": 15.534683227539062,
      "activations/layer0_attention_weight_min": -13.061927795410156,
      "activations/layer10_attention_weight_max": 34.27653884887695,
      "activations/layer10_attention_weight_min": -36.763057708740234,
      "activations/layer11_attention_weight_max": 31.03348159790039,
      "activations/layer11_attention_weight_min": -36.12430953979492,
      "activations/layer12_attention_weight_max": 17.924116134643555,
      "activations/layer12_attention_weight_min": -25.471220016479492,
      "activations/layer13_attention_weight_max": 33.04698181152344,
      "activations/layer13_attention_weight_min": -29.467273712158203,
      "activations/layer14_attention_weight_max": 34.63722229003906,
      "activations/layer14_attention_weight_min": -28.27326774597168,
      "activations/layer15_attention_weight_max": 31.850967407226562,
      "activations/layer15_attention_weight_min": -29.708677291870117,
      "activations/layer16_attention_weight_max": 35.62675476074219,
      "activations/layer16_attention_weight_min": -31.468135833740234,
      "activations/layer17_attention_weight_max": 51.17319107055664,
      "activations/layer17_attention_weight_min": -43.24354934692383,
      "activations/layer18_attention_weight_max": 48.66140365600586,
      "activations/layer18_attention_weight_min": -39.27118682861328,
      "activations/layer19_attention_weight_max": 25.007736206054688,
      "activations/layer19_attention_weight_min": -22.697101593017578,
      "activations/layer1_attention_weight_max": 15.36165714263916,
      "activations/layer1_attention_weight_min": -13.28233814239502,
      "activations/layer20_attention_weight_max": 22.187461853027344,
      "activations/layer20_attention_weight_min": -20.297582626342773,
      "activations/layer21_attention_weight_max": 36.83040237426758,
      "activations/layer21_attention_weight_min": -22.44564437866211,
      "activations/layer22_attention_weight_max": 28.974708557128906,
      "activations/layer22_attention_weight_min": -24.881494522094727,
      "activations/layer23_attention_weight_max": 34.19308853149414,
      "activations/layer23_attention_weight_min": -23.736351013183594,
      "activations/layer2_attention_weight_max": 30.93227195739746,
      "activations/layer2_attention_weight_min": -30.535037994384766,
      "activations/layer3_attention_weight_max": 88.89216613769531,
      "activations/layer3_attention_weight_min": -82.81611633300781,
      "activations/layer4_attention_weight_max": 82.23429870605469,
      "activations/layer4_attention_weight_min": -78.5060043334961,
      "activations/layer5_attention_weight_max": 60.582374572753906,
      "activations/layer5_attention_weight_min": -72.39878845214844,
      "activations/layer6_attention_weight_max": 48.65808868408203,
      "activations/layer6_attention_weight_min": -49.8731575012207,
      "activations/layer7_attention_weight_max": 62.62427520751953,
      "activations/layer7_attention_weight_min": -63.93229293823242,
      "activations/layer8_attention_weight_max": 44.82746124267578,
      "activations/layer8_attention_weight_min": -46.60801315307617,
      "activations/layer9_attention_weight_max": 40.05060958862305,
      "activations/layer9_attention_weight_min": -44.87733840942383,
      "epoch": 19.21,
      "learning_rate": 2.640833333333333e-05,
      "loss": 2.7209,
      "step": 330550
    },
    {
      "activations/layer0_attention_weight_max": 15.02529525756836,
      "activations/layer0_attention_weight_min": -13.600336074829102,
      "activations/layer10_attention_weight_max": 34.725624084472656,
      "activations/layer10_attention_weight_min": -33.963287353515625,
      "activations/layer11_attention_weight_max": 32.84503936767578,
      "activations/layer11_attention_weight_min": -32.750823974609375,
      "activations/layer12_attention_weight_max": 19.289531707763672,
      "activations/layer12_attention_weight_min": -27.806766510009766,
      "activations/layer13_attention_weight_max": 34.35057067871094,
      "activations/layer13_attention_weight_min": -29.569713592529297,
      "activations/layer14_attention_weight_max": 36.4484977722168,
      "activations/layer14_attention_weight_min": -30.302167892456055,
      "activations/layer15_attention_weight_max": 35.61784744262695,
      "activations/layer15_attention_weight_min": -30.664094924926758,
      "activations/layer16_attention_weight_max": 34.85678482055664,
      "activations/layer16_attention_weight_min": -32.34836959838867,
      "activations/layer17_attention_weight_max": 57.37651443481445,
      "activations/layer17_attention_weight_min": -47.25010681152344,
      "activations/layer18_attention_weight_max": 46.85291290283203,
      "activations/layer18_attention_weight_min": -42.92292785644531,
      "activations/layer19_attention_weight_max": 26.110410690307617,
      "activations/layer19_attention_weight_min": -23.271434783935547,
      "activations/layer1_attention_weight_max": 15.681303024291992,
      "activations/layer1_attention_weight_min": -13.20205307006836,
      "activations/layer20_attention_weight_max": 23.30815887451172,
      "activations/layer20_attention_weight_min": -21.851083755493164,
      "activations/layer21_attention_weight_max": 38.23310470581055,
      "activations/layer21_attention_weight_min": -20.457284927368164,
      "activations/layer22_attention_weight_max": 32.510868072509766,
      "activations/layer22_attention_weight_min": -26.3587589263916,
      "activations/layer23_attention_weight_max": 37.081336975097656,
      "activations/layer23_attention_weight_min": -25.478748321533203,
      "activations/layer2_attention_weight_max": 32.13349914550781,
      "activations/layer2_attention_weight_min": -29.59589958190918,
      "activations/layer3_attention_weight_max": 88.54402923583984,
      "activations/layer3_attention_weight_min": -85.19133758544922,
      "activations/layer4_attention_weight_max": 80.3115463256836,
      "activations/layer4_attention_weight_min": -80.21575164794922,
      "activations/layer5_attention_weight_max": 62.123695373535156,
      "activations/layer5_attention_weight_min": -71.01585388183594,
      "activations/layer6_attention_weight_max": 49.579776763916016,
      "activations/layer6_attention_weight_min": -49.08665466308594,
      "activations/layer7_attention_weight_max": 68.32083129882812,
      "activations/layer7_attention_weight_min": -62.395023345947266,
      "activations/layer8_attention_weight_max": 45.401084899902344,
      "activations/layer8_attention_weight_min": -46.43212890625,
      "activations/layer9_attention_weight_max": 44.4887809753418,
      "activations/layer9_attention_weight_min": -46.270790100097656,
      "epoch": 19.21,
      "learning_rate": 2.6389393939393938e-05,
      "loss": 2.723,
      "step": 330600
    },
    {
      "activations/layer0_attention_weight_max": 16.059829711914062,
      "activations/layer0_attention_weight_min": -13.767745971679688,
      "activations/layer10_attention_weight_max": 33.82509231567383,
      "activations/layer10_attention_weight_min": -33.423057556152344,
      "activations/layer11_attention_weight_max": 31.198200225830078,
      "activations/layer11_attention_weight_min": -32.77632522583008,
      "activations/layer12_attention_weight_max": 19.432086944580078,
      "activations/layer12_attention_weight_min": -25.345659255981445,
      "activations/layer13_attention_weight_max": 37.75537872314453,
      "activations/layer13_attention_weight_min": -28.069232940673828,
      "activations/layer14_attention_weight_max": 37.00828170776367,
      "activations/layer14_attention_weight_min": -30.213375091552734,
      "activations/layer15_attention_weight_max": 36.26060485839844,
      "activations/layer15_attention_weight_min": -29.03232192993164,
      "activations/layer16_attention_weight_max": 34.81969451904297,
      "activations/layer16_attention_weight_min": -32.61799240112305,
      "activations/layer17_attention_weight_max": 52.60480499267578,
      "activations/layer17_attention_weight_min": -45.93783950805664,
      "activations/layer18_attention_weight_max": 54.142879486083984,
      "activations/layer18_attention_weight_min": -40.4038200378418,
      "activations/layer19_attention_weight_max": 26.480844497680664,
      "activations/layer19_attention_weight_min": -22.437822341918945,
      "activations/layer1_attention_weight_max": 16.08218765258789,
      "activations/layer1_attention_weight_min": -13.50031566619873,
      "activations/layer20_attention_weight_max": 24.73638343811035,
      "activations/layer20_attention_weight_min": -21.390501022338867,
      "activations/layer21_attention_weight_max": 36.07819747924805,
      "activations/layer21_attention_weight_min": -20.610727310180664,
      "activations/layer22_attention_weight_max": 28.973957061767578,
      "activations/layer22_attention_weight_min": -27.27229118347168,
      "activations/layer23_attention_weight_max": 34.312705993652344,
      "activations/layer23_attention_weight_min": -25.180404663085938,
      "activations/layer2_attention_weight_max": 30.504226684570312,
      "activations/layer2_attention_weight_min": -28.913166046142578,
      "activations/layer3_attention_weight_max": 81.23806762695312,
      "activations/layer3_attention_weight_min": -84.4532699584961,
      "activations/layer4_attention_weight_max": 79.96617889404297,
      "activations/layer4_attention_weight_min": -82.992919921875,
      "activations/layer5_attention_weight_max": 60.45598602294922,
      "activations/layer5_attention_weight_min": -72.69783020019531,
      "activations/layer6_attention_weight_max": 48.59297180175781,
      "activations/layer6_attention_weight_min": -49.57909393310547,
      "activations/layer7_attention_weight_max": 66.291748046875,
      "activations/layer7_attention_weight_min": -64.25672912597656,
      "activations/layer8_attention_weight_max": 44.57756423950195,
      "activations/layer8_attention_weight_min": -49.309696197509766,
      "activations/layer9_attention_weight_max": 40.78728103637695,
      "activations/layer9_attention_weight_min": -44.49602508544922,
      "epoch": 19.21,
      "learning_rate": 2.6370454545454542e-05,
      "loss": 2.7205,
      "step": 330650
    },
    {
      "activations/layer0_attention_weight_max": 15.59446907043457,
      "activations/layer0_attention_weight_min": -13.411229133605957,
      "activations/layer10_attention_weight_max": 36.17266845703125,
      "activations/layer10_attention_weight_min": -35.31434631347656,
      "activations/layer11_attention_weight_max": 32.50320053100586,
      "activations/layer11_attention_weight_min": -34.562557220458984,
      "activations/layer12_attention_weight_max": 19.65573501586914,
      "activations/layer12_attention_weight_min": -23.58127212524414,
      "activations/layer13_attention_weight_max": 34.89140319824219,
      "activations/layer13_attention_weight_min": -27.67021942138672,
      "activations/layer14_attention_weight_max": 32.38020706176758,
      "activations/layer14_attention_weight_min": -31.910436630249023,
      "activations/layer15_attention_weight_max": 32.913108825683594,
      "activations/layer15_attention_weight_min": -29.21782684326172,
      "activations/layer16_attention_weight_max": 33.496055603027344,
      "activations/layer16_attention_weight_min": -31.083097457885742,
      "activations/layer17_attention_weight_max": 49.2928352355957,
      "activations/layer17_attention_weight_min": -46.0997200012207,
      "activations/layer18_attention_weight_max": 45.98648452758789,
      "activations/layer18_attention_weight_min": -39.806575775146484,
      "activations/layer19_attention_weight_max": 22.708438873291016,
      "activations/layer19_attention_weight_min": -24.5422306060791,
      "activations/layer1_attention_weight_max": 15.218636512756348,
      "activations/layer1_attention_weight_min": -12.67298698425293,
      "activations/layer20_attention_weight_max": 20.786081314086914,
      "activations/layer20_attention_weight_min": -22.592100143432617,
      "activations/layer21_attention_weight_max": 37.12045669555664,
      "activations/layer21_attention_weight_min": -22.64261245727539,
      "activations/layer22_attention_weight_max": 32.058624267578125,
      "activations/layer22_attention_weight_min": -27.63702964782715,
      "activations/layer23_attention_weight_max": 37.09392166137695,
      "activations/layer23_attention_weight_min": -24.81354522705078,
      "activations/layer2_attention_weight_max": 30.64425277709961,
      "activations/layer2_attention_weight_min": -30.96058464050293,
      "activations/layer3_attention_weight_max": 88.83020782470703,
      "activations/layer3_attention_weight_min": -87.51461029052734,
      "activations/layer4_attention_weight_max": 83.45001220703125,
      "activations/layer4_attention_weight_min": -85.6549301147461,
      "activations/layer5_attention_weight_max": 60.952491760253906,
      "activations/layer5_attention_weight_min": -75.6092758178711,
      "activations/layer6_attention_weight_max": 50.6700325012207,
      "activations/layer6_attention_weight_min": -50.01262664794922,
      "activations/layer7_attention_weight_max": 65.51276397705078,
      "activations/layer7_attention_weight_min": -63.7352409362793,
      "activations/layer8_attention_weight_max": 45.864742279052734,
      "activations/layer8_attention_weight_min": -50.1690559387207,
      "activations/layer9_attention_weight_max": 44.92784881591797,
      "activations/layer9_attention_weight_min": -45.43573760986328,
      "epoch": 19.22,
      "learning_rate": 2.635151515151515e-05,
      "loss": 2.7332,
      "step": 330700
    },
    {
      "activations/layer0_attention_weight_max": 15.499824523925781,
      "activations/layer0_attention_weight_min": -13.146510124206543,
      "activations/layer10_attention_weight_max": 33.317359924316406,
      "activations/layer10_attention_weight_min": -33.869632720947266,
      "activations/layer11_attention_weight_max": 34.825496673583984,
      "activations/layer11_attention_weight_min": -32.80026626586914,
      "activations/layer12_attention_weight_max": 20.873188018798828,
      "activations/layer12_attention_weight_min": -26.258041381835938,
      "activations/layer13_attention_weight_max": 32.91291427612305,
      "activations/layer13_attention_weight_min": -27.882041931152344,
      "activations/layer14_attention_weight_max": 35.13393020629883,
      "activations/layer14_attention_weight_min": -28.977869033813477,
      "activations/layer15_attention_weight_max": 32.18581771850586,
      "activations/layer15_attention_weight_min": -29.006938934326172,
      "activations/layer16_attention_weight_max": 33.36700439453125,
      "activations/layer16_attention_weight_min": -29.958606719970703,
      "activations/layer17_attention_weight_max": 50.0943717956543,
      "activations/layer17_attention_weight_min": -43.354061126708984,
      "activations/layer18_attention_weight_max": 46.255672454833984,
      "activations/layer18_attention_weight_min": -38.26042938232422,
      "activations/layer19_attention_weight_max": 26.02296257019043,
      "activations/layer19_attention_weight_min": -23.000812530517578,
      "activations/layer1_attention_weight_max": 15.087295532226562,
      "activations/layer1_attention_weight_min": -12.577583312988281,
      "activations/layer20_attention_weight_max": 22.370365142822266,
      "activations/layer20_attention_weight_min": -23.34034538269043,
      "activations/layer21_attention_weight_max": 36.720550537109375,
      "activations/layer21_attention_weight_min": -21.429447174072266,
      "activations/layer22_attention_weight_max": 30.220836639404297,
      "activations/layer22_attention_weight_min": -25.08935546875,
      "activations/layer23_attention_weight_max": 35.586082458496094,
      "activations/layer23_attention_weight_min": -25.83497428894043,
      "activations/layer2_attention_weight_max": 29.27330780029297,
      "activations/layer2_attention_weight_min": -28.856990814208984,
      "activations/layer3_attention_weight_max": 83.63015747070312,
      "activations/layer3_attention_weight_min": -83.47846221923828,
      "activations/layer4_attention_weight_max": 81.67684173583984,
      "activations/layer4_attention_weight_min": -77.67354583740234,
      "activations/layer5_attention_weight_max": 62.1202392578125,
      "activations/layer5_attention_weight_min": -70.67123413085938,
      "activations/layer6_attention_weight_max": 47.727603912353516,
      "activations/layer6_attention_weight_min": -50.149173736572266,
      "activations/layer7_attention_weight_max": 64.59306335449219,
      "activations/layer7_attention_weight_min": -62.21485900878906,
      "activations/layer8_attention_weight_max": 44.49650192260742,
      "activations/layer8_attention_weight_min": -48.3426399230957,
      "activations/layer9_attention_weight_max": 47.79655075073242,
      "activations/layer9_attention_weight_min": -43.91709899902344,
      "epoch": 19.22,
      "learning_rate": 2.6332575757575752e-05,
      "loss": 2.7212,
      "step": 330750
    },
    {
      "activations/layer0_attention_weight_max": 15.407158851623535,
      "activations/layer0_attention_weight_min": -13.35433578491211,
      "activations/layer10_attention_weight_max": 37.0173225402832,
      "activations/layer10_attention_weight_min": -34.42691421508789,
      "activations/layer11_attention_weight_max": 35.87622833251953,
      "activations/layer11_attention_weight_min": -33.49712371826172,
      "activations/layer12_attention_weight_max": 19.520212173461914,
      "activations/layer12_attention_weight_min": -25.17619514465332,
      "activations/layer13_attention_weight_max": 35.77199172973633,
      "activations/layer13_attention_weight_min": -28.504913330078125,
      "activations/layer14_attention_weight_max": 38.78226089477539,
      "activations/layer14_attention_weight_min": -30.660049438476562,
      "activations/layer15_attention_weight_max": 38.45635986328125,
      "activations/layer15_attention_weight_min": -29.639694213867188,
      "activations/layer16_attention_weight_max": 35.64279556274414,
      "activations/layer16_attention_weight_min": -31.64166259765625,
      "activations/layer17_attention_weight_max": 49.9831657409668,
      "activations/layer17_attention_weight_min": -46.08881378173828,
      "activations/layer18_attention_weight_max": 46.70730209350586,
      "activations/layer18_attention_weight_min": -40.823482513427734,
      "activations/layer19_attention_weight_max": 24.600923538208008,
      "activations/layer19_attention_weight_min": -23.736328125,
      "activations/layer1_attention_weight_max": 15.750730514526367,
      "activations/layer1_attention_weight_min": -13.317954063415527,
      "activations/layer20_attention_weight_max": 23.178279876708984,
      "activations/layer20_attention_weight_min": -22.35936737060547,
      "activations/layer21_attention_weight_max": 37.51591491699219,
      "activations/layer21_attention_weight_min": -24.27970314025879,
      "activations/layer22_attention_weight_max": 30.027076721191406,
      "activations/layer22_attention_weight_min": -24.760692596435547,
      "activations/layer23_attention_weight_max": 34.918785095214844,
      "activations/layer23_attention_weight_min": -23.568782806396484,
      "activations/layer2_attention_weight_max": 31.14282989501953,
      "activations/layer2_attention_weight_min": -28.2185115814209,
      "activations/layer3_attention_weight_max": 88.1734390258789,
      "activations/layer3_attention_weight_min": -83.26435089111328,
      "activations/layer4_attention_weight_max": 84.1627426147461,
      "activations/layer4_attention_weight_min": -80.4009017944336,
      "activations/layer5_attention_weight_max": 62.53133773803711,
      "activations/layer5_attention_weight_min": -65.52159118652344,
      "activations/layer6_attention_weight_max": 49.15292739868164,
      "activations/layer6_attention_weight_min": -49.17174530029297,
      "activations/layer7_attention_weight_max": 64.93958282470703,
      "activations/layer7_attention_weight_min": -61.24934768676758,
      "activations/layer8_attention_weight_max": 49.2133674621582,
      "activations/layer8_attention_weight_min": -49.061729431152344,
      "activations/layer9_attention_weight_max": 43.77851104736328,
      "activations/layer9_attention_weight_min": -47.02548599243164,
      "epoch": 19.22,
      "learning_rate": 2.631363636363636e-05,
      "loss": 2.7296,
      "step": 330800
    },
    {
      "activations/layer0_attention_weight_max": 14.372998237609863,
      "activations/layer0_attention_weight_min": -13.375568389892578,
      "activations/layer10_attention_weight_max": 38.56608581542969,
      "activations/layer10_attention_weight_min": -38.51439666748047,
      "activations/layer11_attention_weight_max": 38.44572067260742,
      "activations/layer11_attention_weight_min": -34.837554931640625,
      "activations/layer12_attention_weight_max": 18.390573501586914,
      "activations/layer12_attention_weight_min": -25.428836822509766,
      "activations/layer13_attention_weight_max": 33.38810348510742,
      "activations/layer13_attention_weight_min": -28.36061668395996,
      "activations/layer14_attention_weight_max": 34.3721923828125,
      "activations/layer14_attention_weight_min": -30.85128402709961,
      "activations/layer15_attention_weight_max": 32.58600997924805,
      "activations/layer15_attention_weight_min": -30.339462280273438,
      "activations/layer16_attention_weight_max": 34.27503967285156,
      "activations/layer16_attention_weight_min": -30.803438186645508,
      "activations/layer17_attention_weight_max": 48.969539642333984,
      "activations/layer17_attention_weight_min": -44.626136779785156,
      "activations/layer18_attention_weight_max": 47.37255096435547,
      "activations/layer18_attention_weight_min": -41.63569259643555,
      "activations/layer19_attention_weight_max": 25.54178810119629,
      "activations/layer19_attention_weight_min": -22.287569046020508,
      "activations/layer1_attention_weight_max": 15.934828758239746,
      "activations/layer1_attention_weight_min": -13.579747200012207,
      "activations/layer20_attention_weight_max": 25.187564849853516,
      "activations/layer20_attention_weight_min": -21.726238250732422,
      "activations/layer21_attention_weight_max": 38.701656341552734,
      "activations/layer21_attention_weight_min": -25.0808162689209,
      "activations/layer22_attention_weight_max": 28.28372573852539,
      "activations/layer22_attention_weight_min": -26.469221115112305,
      "activations/layer23_attention_weight_max": 36.265106201171875,
      "activations/layer23_attention_weight_min": -24.082141876220703,
      "activations/layer2_attention_weight_max": 31.15729331970215,
      "activations/layer2_attention_weight_min": -30.802135467529297,
      "activations/layer3_attention_weight_max": 90.25130462646484,
      "activations/layer3_attention_weight_min": -90.15998077392578,
      "activations/layer4_attention_weight_max": 84.92047882080078,
      "activations/layer4_attention_weight_min": -85.12828826904297,
      "activations/layer5_attention_weight_max": 62.555397033691406,
      "activations/layer5_attention_weight_min": -70.69540405273438,
      "activations/layer6_attention_weight_max": 50.18033218383789,
      "activations/layer6_attention_weight_min": -50.60697937011719,
      "activations/layer7_attention_weight_max": 76.15119934082031,
      "activations/layer7_attention_weight_min": -68.02814483642578,
      "activations/layer8_attention_weight_max": 53.50469207763672,
      "activations/layer8_attention_weight_min": -49.12104797363281,
      "activations/layer9_attention_weight_max": 60.23346710205078,
      "activations/layer9_attention_weight_min": -50.614501953125,
      "epoch": 19.22,
      "learning_rate": 2.6294696969696965e-05,
      "loss": 2.7244,
      "step": 330850
    },
    {
      "activations/layer0_attention_weight_max": 15.10689926147461,
      "activations/layer0_attention_weight_min": -13.45790958404541,
      "activations/layer10_attention_weight_max": 35.505836486816406,
      "activations/layer10_attention_weight_min": -38.23474884033203,
      "activations/layer11_attention_weight_max": 33.172908782958984,
      "activations/layer11_attention_weight_min": -34.078128814697266,
      "activations/layer12_attention_weight_max": 18.101037979125977,
      "activations/layer12_attention_weight_min": -30.191837310791016,
      "activations/layer13_attention_weight_max": 33.87761688232422,
      "activations/layer13_attention_weight_min": -28.133007049560547,
      "activations/layer14_attention_weight_max": 34.80182647705078,
      "activations/layer14_attention_weight_min": -31.25474739074707,
      "activations/layer15_attention_weight_max": 32.49543380737305,
      "activations/layer15_attention_weight_min": -30.441638946533203,
      "activations/layer16_attention_weight_max": 34.19023132324219,
      "activations/layer16_attention_weight_min": -32.558311462402344,
      "activations/layer17_attention_weight_max": 51.92716979980469,
      "activations/layer17_attention_weight_min": -47.2592658996582,
      "activations/layer18_attention_weight_max": 49.07124710083008,
      "activations/layer18_attention_weight_min": -42.69647216796875,
      "activations/layer19_attention_weight_max": 26.97016716003418,
      "activations/layer19_attention_weight_min": -22.722986221313477,
      "activations/layer1_attention_weight_max": 14.944812774658203,
      "activations/layer1_attention_weight_min": -12.502405166625977,
      "activations/layer20_attention_weight_max": 21.985950469970703,
      "activations/layer20_attention_weight_min": -24.81182861328125,
      "activations/layer21_attention_weight_max": 34.9267578125,
      "activations/layer21_attention_weight_min": -20.838464736938477,
      "activations/layer22_attention_weight_max": 29.054927825927734,
      "activations/layer22_attention_weight_min": -25.49228858947754,
      "activations/layer23_attention_weight_max": 35.860252380371094,
      "activations/layer23_attention_weight_min": -23.40060043334961,
      "activations/layer2_attention_weight_max": 27.673717498779297,
      "activations/layer2_attention_weight_min": -26.276599884033203,
      "activations/layer3_attention_weight_max": 80.91634368896484,
      "activations/layer3_attention_weight_min": -79.62765502929688,
      "activations/layer4_attention_weight_max": 78.58727264404297,
      "activations/layer4_attention_weight_min": -78.0677490234375,
      "activations/layer5_attention_weight_max": 65.31944274902344,
      "activations/layer5_attention_weight_min": -68.36212158203125,
      "activations/layer6_attention_weight_max": 49.90653991699219,
      "activations/layer6_attention_weight_min": -52.32648468017578,
      "activations/layer7_attention_weight_max": 65.95978546142578,
      "activations/layer7_attention_weight_min": -69.45621490478516,
      "activations/layer8_attention_weight_max": 47.1418571472168,
      "activations/layer8_attention_weight_min": -51.365211486816406,
      "activations/layer9_attention_weight_max": 42.476985931396484,
      "activations/layer9_attention_weight_min": -46.97683334350586,
      "epoch": 19.23,
      "learning_rate": 2.6275757575757574e-05,
      "loss": 2.7269,
      "step": 330900
    },
    {
      "activations/layer0_attention_weight_max": 14.994324684143066,
      "activations/layer0_attention_weight_min": -13.55606460571289,
      "activations/layer10_attention_weight_max": 43.32561111450195,
      "activations/layer10_attention_weight_min": -40.05351638793945,
      "activations/layer11_attention_weight_max": 37.74738311767578,
      "activations/layer11_attention_weight_min": -34.90579605102539,
      "activations/layer12_attention_weight_max": 18.5589542388916,
      "activations/layer12_attention_weight_min": -24.948278427124023,
      "activations/layer13_attention_weight_max": 38.27099609375,
      "activations/layer13_attention_weight_min": -29.374126434326172,
      "activations/layer14_attention_weight_max": 41.151248931884766,
      "activations/layer14_attention_weight_min": -30.430017471313477,
      "activations/layer15_attention_weight_max": 43.7957649230957,
      "activations/layer15_attention_weight_min": -30.0645809173584,
      "activations/layer16_attention_weight_max": 33.340457916259766,
      "activations/layer16_attention_weight_min": -32.14304733276367,
      "activations/layer17_attention_weight_max": 49.98674011230469,
      "activations/layer17_attention_weight_min": -44.61690902709961,
      "activations/layer18_attention_weight_max": 49.986385345458984,
      "activations/layer18_attention_weight_min": -39.148067474365234,
      "activations/layer19_attention_weight_max": 24.111398696899414,
      "activations/layer19_attention_weight_min": -23.430356979370117,
      "activations/layer1_attention_weight_max": 15.886748313903809,
      "activations/layer1_attention_weight_min": -14.972606658935547,
      "activations/layer20_attention_weight_max": 24.91569709777832,
      "activations/layer20_attention_weight_min": -21.020912170410156,
      "activations/layer21_attention_weight_max": 44.43741989135742,
      "activations/layer21_attention_weight_min": -23.572025299072266,
      "activations/layer22_attention_weight_max": 29.874855041503906,
      "activations/layer22_attention_weight_min": -25.30906105041504,
      "activations/layer23_attention_weight_max": 33.642906188964844,
      "activations/layer23_attention_weight_min": -21.88863182067871,
      "activations/layer2_attention_weight_max": 31.01217269897461,
      "activations/layer2_attention_weight_min": -29.35321807861328,
      "activations/layer3_attention_weight_max": 81.97238159179688,
      "activations/layer3_attention_weight_min": -81.48957824707031,
      "activations/layer4_attention_weight_max": 80.86107635498047,
      "activations/layer4_attention_weight_min": -77.58489227294922,
      "activations/layer5_attention_weight_max": 59.91636276245117,
      "activations/layer5_attention_weight_min": -74.6009521484375,
      "activations/layer6_attention_weight_max": 47.02094650268555,
      "activations/layer6_attention_weight_min": -49.39362335205078,
      "activations/layer7_attention_weight_max": 63.343017578125,
      "activations/layer7_attention_weight_min": -65.35731506347656,
      "activations/layer8_attention_weight_max": 49.76605987548828,
      "activations/layer8_attention_weight_min": -50.96067428588867,
      "activations/layer9_attention_weight_max": 49.751338958740234,
      "activations/layer9_attention_weight_min": -49.69527053833008,
      "epoch": 19.23,
      "learning_rate": 2.625681818181818e-05,
      "loss": 2.7366,
      "step": 330950
    },
    {
      "activations/layer0_attention_weight_max": 16.131006240844727,
      "activations/layer0_attention_weight_min": -13.721396446228027,
      "activations/layer10_attention_weight_max": 37.192726135253906,
      "activations/layer10_attention_weight_min": -36.04389572143555,
      "activations/layer11_attention_weight_max": 33.814273834228516,
      "activations/layer11_attention_weight_min": -34.03083801269531,
      "activations/layer12_attention_weight_max": 20.423246383666992,
      "activations/layer12_attention_weight_min": -25.812679290771484,
      "activations/layer13_attention_weight_max": 38.71543502807617,
      "activations/layer13_attention_weight_min": -27.130407333374023,
      "activations/layer14_attention_weight_max": 38.00166320800781,
      "activations/layer14_attention_weight_min": -30.493900299072266,
      "activations/layer15_attention_weight_max": 34.845767974853516,
      "activations/layer15_attention_weight_min": -29.95913314819336,
      "activations/layer16_attention_weight_max": 36.69158172607422,
      "activations/layer16_attention_weight_min": -33.27226638793945,
      "activations/layer17_attention_weight_max": 56.037559509277344,
      "activations/layer17_attention_weight_min": -49.08363342285156,
      "activations/layer18_attention_weight_max": 53.36161804199219,
      "activations/layer18_attention_weight_min": -43.57858657836914,
      "activations/layer19_attention_weight_max": 25.01262092590332,
      "activations/layer19_attention_weight_min": -24.418928146362305,
      "activations/layer1_attention_weight_max": 15.87568473815918,
      "activations/layer1_attention_weight_min": -12.983747482299805,
      "activations/layer20_attention_weight_max": 25.738079071044922,
      "activations/layer20_attention_weight_min": -21.540407180786133,
      "activations/layer21_attention_weight_max": 41.007347106933594,
      "activations/layer21_attention_weight_min": -21.67486572265625,
      "activations/layer22_attention_weight_max": 32.430816650390625,
      "activations/layer22_attention_weight_min": -26.046092987060547,
      "activations/layer23_attention_weight_max": 42.9954948425293,
      "activations/layer23_attention_weight_min": -22.36587142944336,
      "activations/layer2_attention_weight_max": 32.71690368652344,
      "activations/layer2_attention_weight_min": -30.76454734802246,
      "activations/layer3_attention_weight_max": 87.583740234375,
      "activations/layer3_attention_weight_min": -85.59514617919922,
      "activations/layer4_attention_weight_max": 81.99098205566406,
      "activations/layer4_attention_weight_min": -82.03385925292969,
      "activations/layer5_attention_weight_max": 63.08972930908203,
      "activations/layer5_attention_weight_min": -72.08076477050781,
      "activations/layer6_attention_weight_max": 50.276039123535156,
      "activations/layer6_attention_weight_min": -52.45938491821289,
      "activations/layer7_attention_weight_max": 66.03950500488281,
      "activations/layer7_attention_weight_min": -65.0612564086914,
      "activations/layer8_attention_weight_max": 46.23291778564453,
      "activations/layer8_attention_weight_min": -49.49858856201172,
      "activations/layer9_attention_weight_max": 44.09792709350586,
      "activations/layer9_attention_weight_min": -46.698486328125,
      "epoch": 19.23,
      "learning_rate": 2.6237878787878787e-05,
      "loss": 2.7209,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.5385,
      "eval_samples_per_second": 502.901,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.5385,
      "eval_openwebtext_samples_per_second": 502.901,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0071,
      "eval_wikitext_samples_per_second": 227.196,
      "step": 331000
    },
    {
      "epoch": 19.23,
      "eval_lambada_loss": 2.490234375,
      "eval_lambada_ppl": 12.064103313334172,
      "eval_lambada_runtime": 9.5961,
      "eval_lambada_samples_per_second": 507.395,
      "step": 331000
    },
    {
      "activations/layer0_attention_weight_max": 15.198102951049805,
      "activations/layer0_attention_weight_min": -13.473614692687988,
      "activations/layer10_attention_weight_max": 34.79400634765625,
      "activations/layer10_attention_weight_min": -35.44198989868164,
      "activations/layer11_attention_weight_max": 32.29789733886719,
      "activations/layer11_attention_weight_min": -34.55576705932617,
      "activations/layer12_attention_weight_max": 19.285432815551758,
      "activations/layer12_attention_weight_min": -22.469221115112305,
      "activations/layer13_attention_weight_max": 32.67280197143555,
      "activations/layer13_attention_weight_min": -25.354360580444336,
      "activations/layer14_attention_weight_max": 35.47935485839844,
      "activations/layer14_attention_weight_min": -31.808032989501953,
      "activations/layer15_attention_weight_max": 32.6626091003418,
      "activations/layer15_attention_weight_min": -31.937198638916016,
      "activations/layer16_attention_weight_max": 37.7176399230957,
      "activations/layer16_attention_weight_min": -35.73208999633789,
      "activations/layer17_attention_weight_max": 55.66610336303711,
      "activations/layer17_attention_weight_min": -48.495967864990234,
      "activations/layer18_attention_weight_max": 51.3486213684082,
      "activations/layer18_attention_weight_min": -41.87710952758789,
      "activations/layer19_attention_weight_max": 23.462167739868164,
      "activations/layer19_attention_weight_min": -22.542295455932617,
      "activations/layer1_attention_weight_max": 15.543843269348145,
      "activations/layer1_attention_weight_min": -13.69400691986084,
      "activations/layer20_attention_weight_max": 22.898977279663086,
      "activations/layer20_attention_weight_min": -21.028032302856445,
      "activations/layer21_attention_weight_max": 43.95452880859375,
      "activations/layer21_attention_weight_min": -23.56939125061035,
      "activations/layer22_attention_weight_max": 28.1229248046875,
      "activations/layer22_attention_weight_min": -26.02402114868164,
      "activations/layer23_attention_weight_max": 38.887481689453125,
      "activations/layer23_attention_weight_min": -24.14320945739746,
      "activations/layer2_attention_weight_max": 35.02851867675781,
      "activations/layer2_attention_weight_min": -30.46438217163086,
      "activations/layer3_attention_weight_max": 82.84425354003906,
      "activations/layer3_attention_weight_min": -83.4389419555664,
      "activations/layer4_attention_weight_max": 79.63436889648438,
      "activations/layer4_attention_weight_min": -76.65428924560547,
      "activations/layer5_attention_weight_max": 58.63593292236328,
      "activations/layer5_attention_weight_min": -67.44351196289062,
      "activations/layer6_attention_weight_max": 46.84965896606445,
      "activations/layer6_attention_weight_min": -48.97945022583008,
      "activations/layer7_attention_weight_max": 61.90213394165039,
      "activations/layer7_attention_weight_min": -64.14431762695312,
      "activations/layer8_attention_weight_max": 44.80397415161133,
      "activations/layer8_attention_weight_min": -48.52121353149414,
      "activations/layer9_attention_weight_max": 41.725833892822266,
      "activations/layer9_attention_weight_min": -45.856201171875,
      "epoch": 19.24,
      "learning_rate": 2.6218939393939395e-05,
      "loss": 2.7363,
      "step": 331050
    },
    {
      "activations/layer0_attention_weight_max": 15.287471771240234,
      "activations/layer0_attention_weight_min": -13.41678524017334,
      "activations/layer10_attention_weight_max": 38.51942443847656,
      "activations/layer10_attention_weight_min": -37.05299758911133,
      "activations/layer11_attention_weight_max": 36.40375900268555,
      "activations/layer11_attention_weight_min": -34.68351364135742,
      "activations/layer12_attention_weight_max": 19.001863479614258,
      "activations/layer12_attention_weight_min": -27.10667610168457,
      "activations/layer13_attention_weight_max": 36.46794128417969,
      "activations/layer13_attention_weight_min": -29.556264877319336,
      "activations/layer14_attention_weight_max": 36.658504486083984,
      "activations/layer14_attention_weight_min": -30.107234954833984,
      "activations/layer15_attention_weight_max": 35.4810905456543,
      "activations/layer15_attention_weight_min": -31.84845542907715,
      "activations/layer16_attention_weight_max": 33.95681381225586,
      "activations/layer16_attention_weight_min": -32.925804138183594,
      "activations/layer17_attention_weight_max": 52.83701705932617,
      "activations/layer17_attention_weight_min": -49.759037017822266,
      "activations/layer18_attention_weight_max": 50.025535583496094,
      "activations/layer18_attention_weight_min": -42.04547882080078,
      "activations/layer19_attention_weight_max": 25.418014526367188,
      "activations/layer19_attention_weight_min": -25.059267044067383,
      "activations/layer1_attention_weight_max": 16.472436904907227,
      "activations/layer1_attention_weight_min": -14.569554328918457,
      "activations/layer20_attention_weight_max": 23.455995559692383,
      "activations/layer20_attention_weight_min": -22.795902252197266,
      "activations/layer21_attention_weight_max": 37.741004943847656,
      "activations/layer21_attention_weight_min": -22.038381576538086,
      "activations/layer22_attention_weight_max": 28.997426986694336,
      "activations/layer22_attention_weight_min": -26.10991096496582,
      "activations/layer23_attention_weight_max": 38.7438850402832,
      "activations/layer23_attention_weight_min": -24.841657638549805,
      "activations/layer2_attention_weight_max": 33.371665954589844,
      "activations/layer2_attention_weight_min": -30.963958740234375,
      "activations/layer3_attention_weight_max": 91.90448760986328,
      "activations/layer3_attention_weight_min": -83.6458969116211,
      "activations/layer4_attention_weight_max": 86.5424575805664,
      "activations/layer4_attention_weight_min": -84.5698471069336,
      "activations/layer5_attention_weight_max": 61.83184051513672,
      "activations/layer5_attention_weight_min": -68.54002380371094,
      "activations/layer6_attention_weight_max": 52.63930892944336,
      "activations/layer6_attention_weight_min": -53.81977081298828,
      "activations/layer7_attention_weight_max": 65.59548950195312,
      "activations/layer7_attention_weight_min": -65.50630187988281,
      "activations/layer8_attention_weight_max": 50.34884262084961,
      "activations/layer8_attention_weight_min": -51.21519470214844,
      "activations/layer9_attention_weight_max": 47.81113815307617,
      "activations/layer9_attention_weight_min": -47.059871673583984,
      "epoch": 19.24,
      "learning_rate": 2.6199999999999996e-05,
      "loss": 2.7206,
      "step": 331100
    },
    {
      "activations/layer0_attention_weight_max": 15.309287071228027,
      "activations/layer0_attention_weight_min": -13.427627563476562,
      "activations/layer10_attention_weight_max": 43.159339904785156,
      "activations/layer10_attention_weight_min": -43.45817184448242,
      "activations/layer11_attention_weight_max": 38.837345123291016,
      "activations/layer11_attention_weight_min": -39.182090759277344,
      "activations/layer12_attention_weight_max": 20.508535385131836,
      "activations/layer12_attention_weight_min": -25.176956176757812,
      "activations/layer13_attention_weight_max": 37.630859375,
      "activations/layer13_attention_weight_min": -31.27012825012207,
      "activations/layer14_attention_weight_max": 41.85417938232422,
      "activations/layer14_attention_weight_min": -33.18350601196289,
      "activations/layer15_attention_weight_max": 45.66916275024414,
      "activations/layer15_attention_weight_min": -32.542991638183594,
      "activations/layer16_attention_weight_max": 38.080631256103516,
      "activations/layer16_attention_weight_min": -34.672325134277344,
      "activations/layer17_attention_weight_max": 58.36039352416992,
      "activations/layer17_attention_weight_min": -53.115577697753906,
      "activations/layer18_attention_weight_max": 52.14494705200195,
      "activations/layer18_attention_weight_min": -47.59370803833008,
      "activations/layer19_attention_weight_max": 27.067060470581055,
      "activations/layer19_attention_weight_min": -29.281665802001953,
      "activations/layer1_attention_weight_max": 16.1405086517334,
      "activations/layer1_attention_weight_min": -12.420722961425781,
      "activations/layer20_attention_weight_max": 26.18950080871582,
      "activations/layer20_attention_weight_min": -25.187400817871094,
      "activations/layer21_attention_weight_max": 38.26575469970703,
      "activations/layer21_attention_weight_min": -30.470916748046875,
      "activations/layer22_attention_weight_max": 31.188196182250977,
      "activations/layer22_attention_weight_min": -26.3217716217041,
      "activations/layer23_attention_weight_max": 36.16291809082031,
      "activations/layer23_attention_weight_min": -25.35464096069336,
      "activations/layer2_attention_weight_max": 30.33899688720703,
      "activations/layer2_attention_weight_min": -30.27193260192871,
      "activations/layer3_attention_weight_max": 80.4079360961914,
      "activations/layer3_attention_weight_min": -82.09033203125,
      "activations/layer4_attention_weight_max": 77.47868347167969,
      "activations/layer4_attention_weight_min": -78.86347198486328,
      "activations/layer5_attention_weight_max": 57.63679122924805,
      "activations/layer5_attention_weight_min": -66.39197540283203,
      "activations/layer6_attention_weight_max": 47.06307601928711,
      "activations/layer6_attention_weight_min": -46.81938934326172,
      "activations/layer7_attention_weight_max": 67.99115753173828,
      "activations/layer7_attention_weight_min": -60.861873626708984,
      "activations/layer8_attention_weight_max": 52.17113494873047,
      "activations/layer8_attention_weight_min": -49.02782440185547,
      "activations/layer9_attention_weight_max": 55.4233512878418,
      "activations/layer9_attention_weight_min": -48.390159606933594,
      "epoch": 19.24,
      "learning_rate": 2.6181060606060605e-05,
      "loss": 2.7155,
      "step": 331150
    },
    {
      "activations/layer0_attention_weight_max": 15.132766723632812,
      "activations/layer0_attention_weight_min": -13.103259086608887,
      "activations/layer10_attention_weight_max": 38.83091735839844,
      "activations/layer10_attention_weight_min": -40.388221740722656,
      "activations/layer11_attention_weight_max": 33.42991638183594,
      "activations/layer11_attention_weight_min": -36.3126220703125,
      "activations/layer12_attention_weight_max": 20.17731285095215,
      "activations/layer12_attention_weight_min": -27.090633392333984,
      "activations/layer13_attention_weight_max": 36.292694091796875,
      "activations/layer13_attention_weight_min": -27.543413162231445,
      "activations/layer14_attention_weight_max": 36.8978385925293,
      "activations/layer14_attention_weight_min": -35.04574203491211,
      "activations/layer15_attention_weight_max": 33.80195999145508,
      "activations/layer15_attention_weight_min": -33.74232864379883,
      "activations/layer16_attention_weight_max": 37.900245666503906,
      "activations/layer16_attention_weight_min": -37.16376495361328,
      "activations/layer17_attention_weight_max": 56.241790771484375,
      "activations/layer17_attention_weight_min": -54.813961029052734,
      "activations/layer18_attention_weight_max": 53.2869987487793,
      "activations/layer18_attention_weight_min": -48.2324333190918,
      "activations/layer19_attention_weight_max": 27.336130142211914,
      "activations/layer19_attention_weight_min": -26.583251953125,
      "activations/layer1_attention_weight_max": 17.0239315032959,
      "activations/layer1_attention_weight_min": -12.341368675231934,
      "activations/layer20_attention_weight_max": 26.804039001464844,
      "activations/layer20_attention_weight_min": -23.886981964111328,
      "activations/layer21_attention_weight_max": 43.662269592285156,
      "activations/layer21_attention_weight_min": -24.157041549682617,
      "activations/layer22_attention_weight_max": 30.90912437438965,
      "activations/layer22_attention_weight_min": -28.481008529663086,
      "activations/layer23_attention_weight_max": 36.700035095214844,
      "activations/layer23_attention_weight_min": -24.51837921142578,
      "activations/layer2_attention_weight_max": 30.12630844116211,
      "activations/layer2_attention_weight_min": -28.557327270507812,
      "activations/layer3_attention_weight_max": 83.54533386230469,
      "activations/layer3_attention_weight_min": -84.84392547607422,
      "activations/layer4_attention_weight_max": 79.81298065185547,
      "activations/layer4_attention_weight_min": -83.3481674194336,
      "activations/layer5_attention_weight_max": 61.3346061706543,
      "activations/layer5_attention_weight_min": -68.76136779785156,
      "activations/layer6_attention_weight_max": 47.99817657470703,
      "activations/layer6_attention_weight_min": -49.91178894042969,
      "activations/layer7_attention_weight_max": 62.576744079589844,
      "activations/layer7_attention_weight_min": -63.45945358276367,
      "activations/layer8_attention_weight_max": 46.52880096435547,
      "activations/layer8_attention_weight_min": -51.24363327026367,
      "activations/layer9_attention_weight_max": 45.369354248046875,
      "activations/layer9_attention_weight_min": -47.44243621826172,
      "epoch": 19.24,
      "learning_rate": 2.616212121212121e-05,
      "loss": 2.7184,
      "step": 331200
    },
    {
      "activations/layer0_attention_weight_max": 14.911787986755371,
      "activations/layer0_attention_weight_min": -13.301294326782227,
      "activations/layer10_attention_weight_max": 37.223602294921875,
      "activations/layer10_attention_weight_min": -36.07576370239258,
      "activations/layer11_attention_weight_max": 37.69112777709961,
      "activations/layer11_attention_weight_min": -35.11372375488281,
      "activations/layer12_attention_weight_max": 20.118854522705078,
      "activations/layer12_attention_weight_min": -28.056859970092773,
      "activations/layer13_attention_weight_max": 37.34621047973633,
      "activations/layer13_attention_weight_min": -26.58289909362793,
      "activations/layer14_attention_weight_max": 36.39887619018555,
      "activations/layer14_attention_weight_min": -30.489116668701172,
      "activations/layer15_attention_weight_max": 33.76359558105469,
      "activations/layer15_attention_weight_min": -30.388301849365234,
      "activations/layer16_attention_weight_max": 33.55977249145508,
      "activations/layer16_attention_weight_min": -32.34457015991211,
      "activations/layer17_attention_weight_max": 50.39815139770508,
      "activations/layer17_attention_weight_min": -43.5461540222168,
      "activations/layer18_attention_weight_max": 47.229209899902344,
      "activations/layer18_attention_weight_min": -39.03246307373047,
      "activations/layer19_attention_weight_max": 24.699079513549805,
      "activations/layer19_attention_weight_min": -22.924692153930664,
      "activations/layer1_attention_weight_max": 14.652752876281738,
      "activations/layer1_attention_weight_min": -12.170769691467285,
      "activations/layer20_attention_weight_max": 21.97553825378418,
      "activations/layer20_attention_weight_min": -23.441478729248047,
      "activations/layer21_attention_weight_max": 33.27101135253906,
      "activations/layer21_attention_weight_min": -21.529613494873047,
      "activations/layer22_attention_weight_max": 28.25861930847168,
      "activations/layer22_attention_weight_min": -25.324691772460938,
      "activations/layer23_attention_weight_max": 37.07156753540039,
      "activations/layer23_attention_weight_min": -23.874113082885742,
      "activations/layer2_attention_weight_max": 29.601234436035156,
      "activations/layer2_attention_weight_min": -28.314273834228516,
      "activations/layer3_attention_weight_max": 86.90038299560547,
      "activations/layer3_attention_weight_min": -82.90995788574219,
      "activations/layer4_attention_weight_max": 82.33898162841797,
      "activations/layer4_attention_weight_min": -84.13108825683594,
      "activations/layer5_attention_weight_max": 62.62468719482422,
      "activations/layer5_attention_weight_min": -70.73648834228516,
      "activations/layer6_attention_weight_max": 49.397579193115234,
      "activations/layer6_attention_weight_min": -53.32388687133789,
      "activations/layer7_attention_weight_max": 67.64042663574219,
      "activations/layer7_attention_weight_min": -67.41365051269531,
      "activations/layer8_attention_weight_max": 49.28446578979492,
      "activations/layer8_attention_weight_min": -51.96244430541992,
      "activations/layer9_attention_weight_max": 47.010643005371094,
      "activations/layer9_attention_weight_min": -45.43383026123047,
      "epoch": 19.25,
      "learning_rate": 2.6143181818181818e-05,
      "loss": 2.7053,
      "step": 331250
    },
    {
      "activations/layer0_attention_weight_max": 15.300066947937012,
      "activations/layer0_attention_weight_min": -13.309741973876953,
      "activations/layer10_attention_weight_max": 35.779109954833984,
      "activations/layer10_attention_weight_min": -35.6938362121582,
      "activations/layer11_attention_weight_max": 31.654075622558594,
      "activations/layer11_attention_weight_min": -34.12162399291992,
      "activations/layer12_attention_weight_max": 18.505956649780273,
      "activations/layer12_attention_weight_min": -25.793672561645508,
      "activations/layer13_attention_weight_max": 29.953086853027344,
      "activations/layer13_attention_weight_min": -27.727394104003906,
      "activations/layer14_attention_weight_max": 34.89530563354492,
      "activations/layer14_attention_weight_min": -30.84807777404785,
      "activations/layer15_attention_weight_max": 31.54507827758789,
      "activations/layer15_attention_weight_min": -30.166481018066406,
      "activations/layer16_attention_weight_max": 33.178009033203125,
      "activations/layer16_attention_weight_min": -31.827892303466797,
      "activations/layer17_attention_weight_max": 52.33108139038086,
      "activations/layer17_attention_weight_min": -43.67118835449219,
      "activations/layer18_attention_weight_max": 46.260066986083984,
      "activations/layer18_attention_weight_min": -40.1072998046875,
      "activations/layer19_attention_weight_max": 25.137592315673828,
      "activations/layer19_attention_weight_min": -22.41880226135254,
      "activations/layer1_attention_weight_max": 15.556452751159668,
      "activations/layer1_attention_weight_min": -12.734816551208496,
      "activations/layer20_attention_weight_max": 20.884767532348633,
      "activations/layer20_attention_weight_min": -23.420795440673828,
      "activations/layer21_attention_weight_max": 35.86124038696289,
      "activations/layer21_attention_weight_min": -23.545866012573242,
      "activations/layer22_attention_weight_max": 27.488313674926758,
      "activations/layer22_attention_weight_min": -25.288585662841797,
      "activations/layer23_attention_weight_max": 36.575706481933594,
      "activations/layer23_attention_weight_min": -23.1942138671875,
      "activations/layer2_attention_weight_max": 31.062480926513672,
      "activations/layer2_attention_weight_min": -29.964046478271484,
      "activations/layer3_attention_weight_max": 88.03053283691406,
      "activations/layer3_attention_weight_min": -85.94549560546875,
      "activations/layer4_attention_weight_max": 82.593017578125,
      "activations/layer4_attention_weight_min": -82.44377899169922,
      "activations/layer5_attention_weight_max": 63.20408248901367,
      "activations/layer5_attention_weight_min": -69.33503723144531,
      "activations/layer6_attention_weight_max": 49.56643295288086,
      "activations/layer6_attention_weight_min": -51.16301727294922,
      "activations/layer7_attention_weight_max": 62.69361114501953,
      "activations/layer7_attention_weight_min": -68.60477447509766,
      "activations/layer8_attention_weight_max": 45.08219909667969,
      "activations/layer8_attention_weight_min": -50.0693473815918,
      "activations/layer9_attention_weight_max": 42.23720932006836,
      "activations/layer9_attention_weight_min": -47.755489349365234,
      "epoch": 19.25,
      "learning_rate": 2.6124242424242423e-05,
      "loss": 2.7209,
      "step": 331300
    },
    {
      "activations/layer0_attention_weight_max": 14.886758804321289,
      "activations/layer0_attention_weight_min": -13.856151580810547,
      "activations/layer10_attention_weight_max": 32.93894958496094,
      "activations/layer10_attention_weight_min": -33.677330017089844,
      "activations/layer11_attention_weight_max": 30.966995239257812,
      "activations/layer11_attention_weight_min": -33.883689880371094,
      "activations/layer12_attention_weight_max": 19.82413101196289,
      "activations/layer12_attention_weight_min": -23.510074615478516,
      "activations/layer13_attention_weight_max": 39.298919677734375,
      "activations/layer13_attention_weight_min": -26.1284236907959,
      "activations/layer14_attention_weight_max": 34.67078399658203,
      "activations/layer14_attention_weight_min": -30.327266693115234,
      "activations/layer15_attention_weight_max": 31.760042190551758,
      "activations/layer15_attention_weight_min": -29.87203598022461,
      "activations/layer16_attention_weight_max": 32.738983154296875,
      "activations/layer16_attention_weight_min": -32.06200408935547,
      "activations/layer17_attention_weight_max": 49.636104583740234,
      "activations/layer17_attention_weight_min": -42.95985794067383,
      "activations/layer18_attention_weight_max": 43.424034118652344,
      "activations/layer18_attention_weight_min": -37.654945373535156,
      "activations/layer19_attention_weight_max": 23.948631286621094,
      "activations/layer19_attention_weight_min": -22.912702560424805,
      "activations/layer1_attention_weight_max": 15.531352996826172,
      "activations/layer1_attention_weight_min": -13.032413482666016,
      "activations/layer20_attention_weight_max": 21.89452362060547,
      "activations/layer20_attention_weight_min": -20.62757110595703,
      "activations/layer21_attention_weight_max": 38.860565185546875,
      "activations/layer21_attention_weight_min": -22.41242790222168,
      "activations/layer22_attention_weight_max": 25.758380889892578,
      "activations/layer22_attention_weight_min": -24.731361389160156,
      "activations/layer23_attention_weight_max": 35.286216735839844,
      "activations/layer23_attention_weight_min": -23.100563049316406,
      "activations/layer2_attention_weight_max": 29.837703704833984,
      "activations/layer2_attention_weight_min": -28.179229736328125,
      "activations/layer3_attention_weight_max": 83.03933715820312,
      "activations/layer3_attention_weight_min": -78.87137603759766,
      "activations/layer4_attention_weight_max": 78.99376678466797,
      "activations/layer4_attention_weight_min": -78.28963470458984,
      "activations/layer5_attention_weight_max": 61.789546966552734,
      "activations/layer5_attention_weight_min": -74.87786102294922,
      "activations/layer6_attention_weight_max": 47.91252517700195,
      "activations/layer6_attention_weight_min": -48.86323928833008,
      "activations/layer7_attention_weight_max": 62.334556579589844,
      "activations/layer7_attention_weight_min": -63.543861389160156,
      "activations/layer8_attention_weight_max": 43.843605041503906,
      "activations/layer8_attention_weight_min": -48.32849884033203,
      "activations/layer9_attention_weight_max": 47.26555633544922,
      "activations/layer9_attention_weight_min": -45.164222717285156,
      "epoch": 19.25,
      "learning_rate": 2.6105303030303028e-05,
      "loss": 2.7219,
      "step": 331350
    },
    {
      "activations/layer0_attention_weight_max": 15.618571281433105,
      "activations/layer0_attention_weight_min": -13.609723091125488,
      "activations/layer10_attention_weight_max": 36.64533615112305,
      "activations/layer10_attention_weight_min": -35.056129455566406,
      "activations/layer11_attention_weight_max": 33.49169921875,
      "activations/layer11_attention_weight_min": -35.23735046386719,
      "activations/layer12_attention_weight_max": 20.493871688842773,
      "activations/layer12_attention_weight_min": -25.253009796142578,
      "activations/layer13_attention_weight_max": 36.14670181274414,
      "activations/layer13_attention_weight_min": -27.226539611816406,
      "activations/layer14_attention_weight_max": 38.66626739501953,
      "activations/layer14_attention_weight_min": -30.79374122619629,
      "activations/layer15_attention_weight_max": 36.12186813354492,
      "activations/layer15_attention_weight_min": -29.48198699951172,
      "activations/layer16_attention_weight_max": 37.13739776611328,
      "activations/layer16_attention_weight_min": -33.687557220458984,
      "activations/layer17_attention_weight_max": 49.60590362548828,
      "activations/layer17_attention_weight_min": -48.307464599609375,
      "activations/layer18_attention_weight_max": 44.76799011230469,
      "activations/layer18_attention_weight_min": -43.35286331176758,
      "activations/layer19_attention_weight_max": 22.667583465576172,
      "activations/layer19_attention_weight_min": -22.23091697692871,
      "activations/layer1_attention_weight_max": 15.102359771728516,
      "activations/layer1_attention_weight_min": -13.785355567932129,
      "activations/layer20_attention_weight_max": 21.74333953857422,
      "activations/layer20_attention_weight_min": -23.351003646850586,
      "activations/layer21_attention_weight_max": 32.27763748168945,
      "activations/layer21_attention_weight_min": -22.473861694335938,
      "activations/layer22_attention_weight_max": 28.910202026367188,
      "activations/layer22_attention_weight_min": -26.746185302734375,
      "activations/layer23_attention_weight_max": 38.161800384521484,
      "activations/layer23_attention_weight_min": -25.049976348876953,
      "activations/layer2_attention_weight_max": 31.391162872314453,
      "activations/layer2_attention_weight_min": -30.328205108642578,
      "activations/layer3_attention_weight_max": 84.49555969238281,
      "activations/layer3_attention_weight_min": -84.76801300048828,
      "activations/layer4_attention_weight_max": 79.47689819335938,
      "activations/layer4_attention_weight_min": -79.95447540283203,
      "activations/layer5_attention_weight_max": 60.19758224487305,
      "activations/layer5_attention_weight_min": -70.8222885131836,
      "activations/layer6_attention_weight_max": 48.792049407958984,
      "activations/layer6_attention_weight_min": -50.169925689697266,
      "activations/layer7_attention_weight_max": 64.4389877319336,
      "activations/layer7_attention_weight_min": -62.93049621582031,
      "activations/layer8_attention_weight_max": 49.367923736572266,
      "activations/layer8_attention_weight_min": -49.690696716308594,
      "activations/layer9_attention_weight_max": 42.33723068237305,
      "activations/layer9_attention_weight_min": -45.73353576660156,
      "epoch": 19.26,
      "learning_rate": 2.6086363636363633e-05,
      "loss": 2.7232,
      "step": 331400
    },
    {
      "activations/layer0_attention_weight_max": 15.618430137634277,
      "activations/layer0_attention_weight_min": -13.488736152648926,
      "activations/layer10_attention_weight_max": 35.04825210571289,
      "activations/layer10_attention_weight_min": -36.51579284667969,
      "activations/layer11_attention_weight_max": 30.909671783447266,
      "activations/layer11_attention_weight_min": -36.64291763305664,
      "activations/layer12_attention_weight_max": 18.6914005279541,
      "activations/layer12_attention_weight_min": -25.29266357421875,
      "activations/layer13_attention_weight_max": 32.70345687866211,
      "activations/layer13_attention_weight_min": -29.265920639038086,
      "activations/layer14_attention_weight_max": 34.6180534362793,
      "activations/layer14_attention_weight_min": -31.46834945678711,
      "activations/layer15_attention_weight_max": 32.841644287109375,
      "activations/layer15_attention_weight_min": -31.11937141418457,
      "activations/layer16_attention_weight_max": 32.95009994506836,
      "activations/layer16_attention_weight_min": -30.294591903686523,
      "activations/layer17_attention_weight_max": 51.59083938598633,
      "activations/layer17_attention_weight_min": -45.14577865600586,
      "activations/layer18_attention_weight_max": 44.88324737548828,
      "activations/layer18_attention_weight_min": -42.151580810546875,
      "activations/layer19_attention_weight_max": 24.553733825683594,
      "activations/layer19_attention_weight_min": -25.96207618713379,
      "activations/layer1_attention_weight_max": 15.22335147857666,
      "activations/layer1_attention_weight_min": -13.38275146484375,
      "activations/layer20_attention_weight_max": 21.048189163208008,
      "activations/layer20_attention_weight_min": -25.995559692382812,
      "activations/layer21_attention_weight_max": 35.38942337036133,
      "activations/layer21_attention_weight_min": -23.042123794555664,
      "activations/layer22_attention_weight_max": 26.77206039428711,
      "activations/layer22_attention_weight_min": -27.750703811645508,
      "activations/layer23_attention_weight_max": 34.03532409667969,
      "activations/layer23_attention_weight_min": -25.281322479248047,
      "activations/layer2_attention_weight_max": 32.696388244628906,
      "activations/layer2_attention_weight_min": -31.165098190307617,
      "activations/layer3_attention_weight_max": 90.89436340332031,
      "activations/layer3_attention_weight_min": -89.68496704101562,
      "activations/layer4_attention_weight_max": 83.02687072753906,
      "activations/layer4_attention_weight_min": -84.8924789428711,
      "activations/layer5_attention_weight_max": 64.63719177246094,
      "activations/layer5_attention_weight_min": -71.69071960449219,
      "activations/layer6_attention_weight_max": 52.665287017822266,
      "activations/layer6_attention_weight_min": -53.77922821044922,
      "activations/layer7_attention_weight_max": 65.2138442993164,
      "activations/layer7_attention_weight_min": -65.948486328125,
      "activations/layer8_attention_weight_max": 45.34245681762695,
      "activations/layer8_attention_weight_min": -51.76326370239258,
      "activations/layer9_attention_weight_max": 44.760562896728516,
      "activations/layer9_attention_weight_min": -46.60219955444336,
      "epoch": 19.26,
      "learning_rate": 2.606742424242424e-05,
      "loss": 2.7347,
      "step": 331450
    },
    {
      "activations/layer0_attention_weight_max": 14.939279556274414,
      "activations/layer0_attention_weight_min": -13.582165718078613,
      "activations/layer10_attention_weight_max": 40.16230010986328,
      "activations/layer10_attention_weight_min": -39.91005325317383,
      "activations/layer11_attention_weight_max": 35.87565612792969,
      "activations/layer11_attention_weight_min": -37.61886215209961,
      "activations/layer12_attention_weight_max": 20.79550552368164,
      "activations/layer12_attention_weight_min": -23.07551383972168,
      "activations/layer13_attention_weight_max": 40.53174591064453,
      "activations/layer13_attention_weight_min": -28.730289459228516,
      "activations/layer14_attention_weight_max": 41.418663024902344,
      "activations/layer14_attention_weight_min": -30.999588012695312,
      "activations/layer15_attention_weight_max": 37.92681884765625,
      "activations/layer15_attention_weight_min": -31.10369300842285,
      "activations/layer16_attention_weight_max": 37.568084716796875,
      "activations/layer16_attention_weight_min": -34.25965881347656,
      "activations/layer17_attention_weight_max": 59.79680252075195,
      "activations/layer17_attention_weight_min": -48.47385025024414,
      "activations/layer18_attention_weight_max": 56.601497650146484,
      "activations/layer18_attention_weight_min": -45.44546890258789,
      "activations/layer19_attention_weight_max": 29.332483291625977,
      "activations/layer19_attention_weight_min": -25.268171310424805,
      "activations/layer1_attention_weight_max": 15.4964017868042,
      "activations/layer1_attention_weight_min": -13.5031156539917,
      "activations/layer20_attention_weight_max": 27.5296573638916,
      "activations/layer20_attention_weight_min": -20.956438064575195,
      "activations/layer21_attention_weight_max": 44.02627182006836,
      "activations/layer21_attention_weight_min": -22.736867904663086,
      "activations/layer22_attention_weight_max": 34.601497650146484,
      "activations/layer22_attention_weight_min": -25.989336013793945,
      "activations/layer23_attention_weight_max": 43.95301818847656,
      "activations/layer23_attention_weight_min": -24.427818298339844,
      "activations/layer2_attention_weight_max": 32.20822525024414,
      "activations/layer2_attention_weight_min": -29.591482162475586,
      "activations/layer3_attention_weight_max": 87.84412384033203,
      "activations/layer3_attention_weight_min": -84.53941345214844,
      "activations/layer4_attention_weight_max": 81.70347595214844,
      "activations/layer4_attention_weight_min": -83.01458740234375,
      "activations/layer5_attention_weight_max": 63.6102409362793,
      "activations/layer5_attention_weight_min": -69.84918975830078,
      "activations/layer6_attention_weight_max": 51.51042938232422,
      "activations/layer6_attention_weight_min": -52.68140411376953,
      "activations/layer7_attention_weight_max": 65.3198013305664,
      "activations/layer7_attention_weight_min": -64.70282745361328,
      "activations/layer8_attention_weight_max": 50.775001525878906,
      "activations/layer8_attention_weight_min": -53.4617805480957,
      "activations/layer9_attention_weight_max": 44.93436050415039,
      "activations/layer9_attention_weight_min": -50.13595962524414,
      "epoch": 19.26,
      "learning_rate": 2.6048484848484846e-05,
      "loss": 2.7202,
      "step": 331500
    },
    {
      "activations/layer0_attention_weight_max": 16.15668296813965,
      "activations/layer0_attention_weight_min": -13.291823387145996,
      "activations/layer10_attention_weight_max": 34.652557373046875,
      "activations/layer10_attention_weight_min": -35.18007278442383,
      "activations/layer11_attention_weight_max": 32.39067459106445,
      "activations/layer11_attention_weight_min": -34.32756805419922,
      "activations/layer12_attention_weight_max": 19.213315963745117,
      "activations/layer12_attention_weight_min": -24.159128189086914,
      "activations/layer13_attention_weight_max": 32.939640045166016,
      "activations/layer13_attention_weight_min": -28.280715942382812,
      "activations/layer14_attention_weight_max": 35.28706741333008,
      "activations/layer14_attention_weight_min": -31.75652313232422,
      "activations/layer15_attention_weight_max": 33.116859436035156,
      "activations/layer15_attention_weight_min": -32.04072189331055,
      "activations/layer16_attention_weight_max": 33.83262252807617,
      "activations/layer16_attention_weight_min": -33.37690353393555,
      "activations/layer17_attention_weight_max": 54.66941452026367,
      "activations/layer17_attention_weight_min": -47.44585037231445,
      "activations/layer18_attention_weight_max": 50.06794357299805,
      "activations/layer18_attention_weight_min": -41.56953811645508,
      "activations/layer19_attention_weight_max": 26.291086196899414,
      "activations/layer19_attention_weight_min": -22.19221305847168,
      "activations/layer1_attention_weight_max": 15.420443534851074,
      "activations/layer1_attention_weight_min": -13.261163711547852,
      "activations/layer20_attention_weight_max": 23.566320419311523,
      "activations/layer20_attention_weight_min": -21.90921401977539,
      "activations/layer21_attention_weight_max": 35.209693908691406,
      "activations/layer21_attention_weight_min": -21.420015335083008,
      "activations/layer22_attention_weight_max": 30.289806365966797,
      "activations/layer22_attention_weight_min": -26.393798828125,
      "activations/layer23_attention_weight_max": 39.15948486328125,
      "activations/layer23_attention_weight_min": -25.117992401123047,
      "activations/layer2_attention_weight_max": 31.468313217163086,
      "activations/layer2_attention_weight_min": -29.372703552246094,
      "activations/layer3_attention_weight_max": 88.55589294433594,
      "activations/layer3_attention_weight_min": -83.83283996582031,
      "activations/layer4_attention_weight_max": 82.97718811035156,
      "activations/layer4_attention_weight_min": -79.8167953491211,
      "activations/layer5_attention_weight_max": 63.245872497558594,
      "activations/layer5_attention_weight_min": -74.08033752441406,
      "activations/layer6_attention_weight_max": 49.96511459350586,
      "activations/layer6_attention_weight_min": -50.29764938354492,
      "activations/layer7_attention_weight_max": 65.18647003173828,
      "activations/layer7_attention_weight_min": -64.25990295410156,
      "activations/layer8_attention_weight_max": 45.16947937011719,
      "activations/layer8_attention_weight_min": -48.70368194580078,
      "activations/layer9_attention_weight_max": 45.45591735839844,
      "activations/layer9_attention_weight_min": -45.13603591918945,
      "epoch": 19.26,
      "learning_rate": 2.6029545454545454e-05,
      "loss": 2.7394,
      "step": 331550
    },
    {
      "activations/layer0_attention_weight_max": 15.319095611572266,
      "activations/layer0_attention_weight_min": -13.822690963745117,
      "activations/layer10_attention_weight_max": 35.33555221557617,
      "activations/layer10_attention_weight_min": -36.39772415161133,
      "activations/layer11_attention_weight_max": 32.199039459228516,
      "activations/layer11_attention_weight_min": -33.96012496948242,
      "activations/layer12_attention_weight_max": 19.91962242126465,
      "activations/layer12_attention_weight_min": -24.791715621948242,
      "activations/layer13_attention_weight_max": 38.06065368652344,
      "activations/layer13_attention_weight_min": -28.300676345825195,
      "activations/layer14_attention_weight_max": 37.8987922668457,
      "activations/layer14_attention_weight_min": -30.371931076049805,
      "activations/layer15_attention_weight_max": 37.48443603515625,
      "activations/layer15_attention_weight_min": -30.8214168548584,
      "activations/layer16_attention_weight_max": 36.58908462524414,
      "activations/layer16_attention_weight_min": -33.916656494140625,
      "activations/layer17_attention_weight_max": 55.402366638183594,
      "activations/layer17_attention_weight_min": -46.697330474853516,
      "activations/layer18_attention_weight_max": 54.675331115722656,
      "activations/layer18_attention_weight_min": -44.40087127685547,
      "activations/layer19_attention_weight_max": 27.179044723510742,
      "activations/layer19_attention_weight_min": -25.68107032775879,
      "activations/layer1_attention_weight_max": 15.19957447052002,
      "activations/layer1_attention_weight_min": -13.627933502197266,
      "activations/layer20_attention_weight_max": 29.260784149169922,
      "activations/layer20_attention_weight_min": -21.542381286621094,
      "activations/layer21_attention_weight_max": 40.82197189331055,
      "activations/layer21_attention_weight_min": -24.008487701416016,
      "activations/layer22_attention_weight_max": 35.17115783691406,
      "activations/layer22_attention_weight_min": -25.17874526977539,
      "activations/layer23_attention_weight_max": 39.7827262878418,
      "activations/layer23_attention_weight_min": -24.025083541870117,
      "activations/layer2_attention_weight_max": 32.066917419433594,
      "activations/layer2_attention_weight_min": -30.592466354370117,
      "activations/layer3_attention_weight_max": 86.70093536376953,
      "activations/layer3_attention_weight_min": -85.70621490478516,
      "activations/layer4_attention_weight_max": 81.3638916015625,
      "activations/layer4_attention_weight_min": -78.32908630371094,
      "activations/layer5_attention_weight_max": 60.539390563964844,
      "activations/layer5_attention_weight_min": -72.39950561523438,
      "activations/layer6_attention_weight_max": 49.942325592041016,
      "activations/layer6_attention_weight_min": -51.22774124145508,
      "activations/layer7_attention_weight_max": 66.1639404296875,
      "activations/layer7_attention_weight_min": -64.42855834960938,
      "activations/layer8_attention_weight_max": 47.30974197387695,
      "activations/layer8_attention_weight_min": -48.90593338012695,
      "activations/layer9_attention_weight_max": 43.4030876159668,
      "activations/layer9_attention_weight_min": -47.74520492553711,
      "epoch": 19.27,
      "learning_rate": 2.6010606060606055e-05,
      "loss": 2.7287,
      "step": 331600
    },
    {
      "activations/layer0_attention_weight_max": 15.091943740844727,
      "activations/layer0_attention_weight_min": -13.571296691894531,
      "activations/layer10_attention_weight_max": 37.56559753417969,
      "activations/layer10_attention_weight_min": -37.59986114501953,
      "activations/layer11_attention_weight_max": 34.370384216308594,
      "activations/layer11_attention_weight_min": -36.89678955078125,
      "activations/layer12_attention_weight_max": 20.57999038696289,
      "activations/layer12_attention_weight_min": -26.681568145751953,
      "activations/layer13_attention_weight_max": 33.63283920288086,
      "activations/layer13_attention_weight_min": -27.72801971435547,
      "activations/layer14_attention_weight_max": 36.749202728271484,
      "activations/layer14_attention_weight_min": -30.71446418762207,
      "activations/layer15_attention_weight_max": 33.677101135253906,
      "activations/layer15_attention_weight_min": -29.811691284179688,
      "activations/layer16_attention_weight_max": 35.46118927001953,
      "activations/layer16_attention_weight_min": -33.49434280395508,
      "activations/layer17_attention_weight_max": 54.728118896484375,
      "activations/layer17_attention_weight_min": -48.09336471557617,
      "activations/layer18_attention_weight_max": 51.38096618652344,
      "activations/layer18_attention_weight_min": -42.472381591796875,
      "activations/layer19_attention_weight_max": 25.791366577148438,
      "activations/layer19_attention_weight_min": -23.982328414916992,
      "activations/layer1_attention_weight_max": 15.520731925964355,
      "activations/layer1_attention_weight_min": -13.514519691467285,
      "activations/layer20_attention_weight_max": 26.869998931884766,
      "activations/layer20_attention_weight_min": -21.839536666870117,
      "activations/layer21_attention_weight_max": 36.30806350708008,
      "activations/layer21_attention_weight_min": -22.73075294494629,
      "activations/layer22_attention_weight_max": 32.25989532470703,
      "activations/layer22_attention_weight_min": -25.959510803222656,
      "activations/layer23_attention_weight_max": 41.32296371459961,
      "activations/layer23_attention_weight_min": -22.657093048095703,
      "activations/layer2_attention_weight_max": 31.77424430847168,
      "activations/layer2_attention_weight_min": -30.209335327148438,
      "activations/layer3_attention_weight_max": 91.6410140991211,
      "activations/layer3_attention_weight_min": -88.83589172363281,
      "activations/layer4_attention_weight_max": 82.8308334350586,
      "activations/layer4_attention_weight_min": -83.58533477783203,
      "activations/layer5_attention_weight_max": 62.99658966064453,
      "activations/layer5_attention_weight_min": -72.808349609375,
      "activations/layer6_attention_weight_max": 50.52003479003906,
      "activations/layer6_attention_weight_min": -50.577510833740234,
      "activations/layer7_attention_weight_max": 64.92826080322266,
      "activations/layer7_attention_weight_min": -68.40080261230469,
      "activations/layer8_attention_weight_max": 46.87921905517578,
      "activations/layer8_attention_weight_min": -52.78840255737305,
      "activations/layer9_attention_weight_max": 46.47792053222656,
      "activations/layer9_attention_weight_min": -49.040470123291016,
      "epoch": 19.27,
      "learning_rate": 2.5991666666666664e-05,
      "loss": 2.72,
      "step": 331650
    },
    {
      "activations/layer0_attention_weight_max": 15.470919609069824,
      "activations/layer0_attention_weight_min": -13.838644981384277,
      "activations/layer10_attention_weight_max": 34.359779357910156,
      "activations/layer10_attention_weight_min": -36.867916107177734,
      "activations/layer11_attention_weight_max": 31.679481506347656,
      "activations/layer11_attention_weight_min": -33.99309539794922,
      "activations/layer12_attention_weight_max": 18.852344512939453,
      "activations/layer12_attention_weight_min": -26.970834732055664,
      "activations/layer13_attention_weight_max": 29.821123123168945,
      "activations/layer13_attention_weight_min": -25.678503036499023,
      "activations/layer14_attention_weight_max": 34.104820251464844,
      "activations/layer14_attention_weight_min": -29.904094696044922,
      "activations/layer15_attention_weight_max": 32.40266418457031,
      "activations/layer15_attention_weight_min": -30.423906326293945,
      "activations/layer16_attention_weight_max": 34.15925598144531,
      "activations/layer16_attention_weight_min": -32.78934097290039,
      "activations/layer17_attention_weight_max": 48.19429397583008,
      "activations/layer17_attention_weight_min": -44.53264236450195,
      "activations/layer18_attention_weight_max": 44.3748779296875,
      "activations/layer18_attention_weight_min": -38.63763427734375,
      "activations/layer19_attention_weight_max": 22.68450164794922,
      "activations/layer19_attention_weight_min": -22.175661087036133,
      "activations/layer1_attention_weight_max": 15.439706802368164,
      "activations/layer1_attention_weight_min": -13.337135314941406,
      "activations/layer20_attention_weight_max": 21.52320671081543,
      "activations/layer20_attention_weight_min": -22.323545455932617,
      "activations/layer21_attention_weight_max": 32.720458984375,
      "activations/layer21_attention_weight_min": -19.849945068359375,
      "activations/layer22_attention_weight_max": 28.28255844116211,
      "activations/layer22_attention_weight_min": -24.22107696533203,
      "activations/layer23_attention_weight_max": 34.912986755371094,
      "activations/layer23_attention_weight_min": -22.663869857788086,
      "activations/layer2_attention_weight_max": 31.823497772216797,
      "activations/layer2_attention_weight_min": -30.70509147644043,
      "activations/layer3_attention_weight_max": 85.18624114990234,
      "activations/layer3_attention_weight_min": -84.40747833251953,
      "activations/layer4_attention_weight_max": 84.43009185791016,
      "activations/layer4_attention_weight_min": -82.22700500488281,
      "activations/layer5_attention_weight_max": 65.35122680664062,
      "activations/layer5_attention_weight_min": -73.42431640625,
      "activations/layer6_attention_weight_max": 50.14193344116211,
      "activations/layer6_attention_weight_min": -53.247501373291016,
      "activations/layer7_attention_weight_max": 71.22473907470703,
      "activations/layer7_attention_weight_min": -63.6512565612793,
      "activations/layer8_attention_weight_max": 44.76841354370117,
      "activations/layer8_attention_weight_min": -48.47822189331055,
      "activations/layer9_attention_weight_max": 43.45734786987305,
      "activations/layer9_attention_weight_min": -50.10696029663086,
      "epoch": 19.27,
      "learning_rate": 2.597272727272727e-05,
      "loss": 2.7122,
      "step": 331700
    },
    {
      "activations/layer0_attention_weight_max": 15.858491897583008,
      "activations/layer0_attention_weight_min": -13.482561111450195,
      "activations/layer10_attention_weight_max": 34.0128059387207,
      "activations/layer10_attention_weight_min": -36.12482833862305,
      "activations/layer11_attention_weight_max": 33.00450897216797,
      "activations/layer11_attention_weight_min": -33.86558532714844,
      "activations/layer12_attention_weight_max": 17.591039657592773,
      "activations/layer12_attention_weight_min": -33.88444519042969,
      "activations/layer13_attention_weight_max": 32.41199493408203,
      "activations/layer13_attention_weight_min": -29.98711395263672,
      "activations/layer14_attention_weight_max": 35.82758712768555,
      "activations/layer14_attention_weight_min": -31.205041885375977,
      "activations/layer15_attention_weight_max": 34.52212142944336,
      "activations/layer15_attention_weight_min": -32.15127944946289,
      "activations/layer16_attention_weight_max": 32.88840866088867,
      "activations/layer16_attention_weight_min": -31.432783126831055,
      "activations/layer17_attention_weight_max": 53.06541061401367,
      "activations/layer17_attention_weight_min": -46.92526626586914,
      "activations/layer18_attention_weight_max": 46.055301666259766,
      "activations/layer18_attention_weight_min": -40.83287811279297,
      "activations/layer19_attention_weight_max": 24.638235092163086,
      "activations/layer19_attention_weight_min": -23.37257957458496,
      "activations/layer1_attention_weight_max": 17.40833282470703,
      "activations/layer1_attention_weight_min": -13.307918548583984,
      "activations/layer20_attention_weight_max": 22.05499267578125,
      "activations/layer20_attention_weight_min": -22.063013076782227,
      "activations/layer21_attention_weight_max": 37.3292121887207,
      "activations/layer21_attention_weight_min": -23.325973510742188,
      "activations/layer22_attention_weight_max": 28.71172332763672,
      "activations/layer22_attention_weight_min": -27.099735260009766,
      "activations/layer23_attention_weight_max": 36.58099365234375,
      "activations/layer23_attention_weight_min": -23.077774047851562,
      "activations/layer2_attention_weight_max": 32.66032028198242,
      "activations/layer2_attention_weight_min": -28.977699279785156,
      "activations/layer3_attention_weight_max": 89.1600570678711,
      "activations/layer3_attention_weight_min": -84.30046844482422,
      "activations/layer4_attention_weight_max": 81.5499267578125,
      "activations/layer4_attention_weight_min": -82.11308288574219,
      "activations/layer5_attention_weight_max": 58.383846282958984,
      "activations/layer5_attention_weight_min": -68.28611755371094,
      "activations/layer6_attention_weight_max": 46.94868469238281,
      "activations/layer6_attention_weight_min": -51.77686309814453,
      "activations/layer7_attention_weight_max": 62.60700988769531,
      "activations/layer7_attention_weight_min": -60.05669021606445,
      "activations/layer8_attention_weight_max": 43.7375373840332,
      "activations/layer8_attention_weight_min": -46.544776916503906,
      "activations/layer9_attention_weight_max": 45.94777297973633,
      "activations/layer9_attention_weight_min": -44.220977783203125,
      "epoch": 19.28,
      "learning_rate": 2.5953787878787877e-05,
      "loss": 2.7228,
      "step": 331750
    },
    {
      "activations/layer0_attention_weight_max": 14.71672248840332,
      "activations/layer0_attention_weight_min": -13.347668647766113,
      "activations/layer10_attention_weight_max": 38.17082595825195,
      "activations/layer10_attention_weight_min": -37.223182678222656,
      "activations/layer11_attention_weight_max": 34.687896728515625,
      "activations/layer11_attention_weight_min": -34.592037200927734,
      "activations/layer12_attention_weight_max": 19.052623748779297,
      "activations/layer12_attention_weight_min": -25.48137855529785,
      "activations/layer13_attention_weight_max": 35.635738372802734,
      "activations/layer13_attention_weight_min": -28.371776580810547,
      "activations/layer14_attention_weight_max": 35.90364456176758,
      "activations/layer14_attention_weight_min": -28.190853118896484,
      "activations/layer15_attention_weight_max": 33.077396392822266,
      "activations/layer15_attention_weight_min": -28.496349334716797,
      "activations/layer16_attention_weight_max": 35.033905029296875,
      "activations/layer16_attention_weight_min": -32.16495132446289,
      "activations/layer17_attention_weight_max": 54.60148620605469,
      "activations/layer17_attention_weight_min": -44.37086868286133,
      "activations/layer18_attention_weight_max": 48.0305061340332,
      "activations/layer18_attention_weight_min": -38.67913055419922,
      "activations/layer19_attention_weight_max": 21.738096237182617,
      "activations/layer19_attention_weight_min": -21.593584060668945,
      "activations/layer1_attention_weight_max": 15.038304328918457,
      "activations/layer1_attention_weight_min": -13.147189140319824,
      "activations/layer20_attention_weight_max": 19.519865036010742,
      "activations/layer20_attention_weight_min": -24.838239669799805,
      "activations/layer21_attention_weight_max": 31.682373046875,
      "activations/layer21_attention_weight_min": -18.41358757019043,
      "activations/layer22_attention_weight_max": 25.7154483795166,
      "activations/layer22_attention_weight_min": -24.285295486450195,
      "activations/layer23_attention_weight_max": 35.497833251953125,
      "activations/layer23_attention_weight_min": -26.420988082885742,
      "activations/layer2_attention_weight_max": 31.523839950561523,
      "activations/layer2_attention_weight_min": -29.54813003540039,
      "activations/layer3_attention_weight_max": 89.0264892578125,
      "activations/layer3_attention_weight_min": -84.78438568115234,
      "activations/layer4_attention_weight_max": 83.89180755615234,
      "activations/layer4_attention_weight_min": -81.82909393310547,
      "activations/layer5_attention_weight_max": 64.31219482421875,
      "activations/layer5_attention_weight_min": -74.24066925048828,
      "activations/layer6_attention_weight_max": 49.18449783325195,
      "activations/layer6_attention_weight_min": -50.03999328613281,
      "activations/layer7_attention_weight_max": 66.02400970458984,
      "activations/layer7_attention_weight_min": -62.769771575927734,
      "activations/layer8_attention_weight_max": 48.701271057128906,
      "activations/layer8_attention_weight_min": -50.967220306396484,
      "activations/layer9_attention_weight_max": 45.55133056640625,
      "activations/layer9_attention_weight_min": -45.82147216796875,
      "epoch": 19.28,
      "learning_rate": 2.5934848484848482e-05,
      "loss": 2.7237,
      "step": 331800
    },
    {
      "activations/layer0_attention_weight_max": 14.501642227172852,
      "activations/layer0_attention_weight_min": -13.537185668945312,
      "activations/layer10_attention_weight_max": 34.968170166015625,
      "activations/layer10_attention_weight_min": -35.0907096862793,
      "activations/layer11_attention_weight_max": 33.685935974121094,
      "activations/layer11_attention_weight_min": -33.47222900390625,
      "activations/layer12_attention_weight_max": 18.415205001831055,
      "activations/layer12_attention_weight_min": -25.24563980102539,
      "activations/layer13_attention_weight_max": 37.26043701171875,
      "activations/layer13_attention_weight_min": -26.89357566833496,
      "activations/layer14_attention_weight_max": 37.71842956542969,
      "activations/layer14_attention_weight_min": -32.411922454833984,
      "activations/layer15_attention_weight_max": 33.71352005004883,
      "activations/layer15_attention_weight_min": -32.86076736450195,
      "activations/layer16_attention_weight_max": 35.37520217895508,
      "activations/layer16_attention_weight_min": -35.222694396972656,
      "activations/layer17_attention_weight_max": 49.549964904785156,
      "activations/layer17_attention_weight_min": -48.046974182128906,
      "activations/layer18_attention_weight_max": 46.34945297241211,
      "activations/layer18_attention_weight_min": -44.72196578979492,
      "activations/layer19_attention_weight_max": 24.47089385986328,
      "activations/layer19_attention_weight_min": -25.15412712097168,
      "activations/layer1_attention_weight_max": 15.76616096496582,
      "activations/layer1_attention_weight_min": -13.680713653564453,
      "activations/layer20_attention_weight_max": 24.007591247558594,
      "activations/layer20_attention_weight_min": -23.634641647338867,
      "activations/layer21_attention_weight_max": 33.16044616699219,
      "activations/layer21_attention_weight_min": -23.16929054260254,
      "activations/layer22_attention_weight_max": 32.028465270996094,
      "activations/layer22_attention_weight_min": -29.716779708862305,
      "activations/layer23_attention_weight_max": 34.23983383178711,
      "activations/layer23_attention_weight_min": -24.94197654724121,
      "activations/layer2_attention_weight_max": 30.645751953125,
      "activations/layer2_attention_weight_min": -29.84735679626465,
      "activations/layer3_attention_weight_max": 85.5722427368164,
      "activations/layer3_attention_weight_min": -83.8914566040039,
      "activations/layer4_attention_weight_max": 85.27296447753906,
      "activations/layer4_attention_weight_min": -83.20780944824219,
      "activations/layer5_attention_weight_max": 59.435401916503906,
      "activations/layer5_attention_weight_min": -76.64580535888672,
      "activations/layer6_attention_weight_max": 48.48636245727539,
      "activations/layer6_attention_weight_min": -52.88349533081055,
      "activations/layer7_attention_weight_max": 63.6639289855957,
      "activations/layer7_attention_weight_min": -65.00040435791016,
      "activations/layer8_attention_weight_max": 48.8187141418457,
      "activations/layer8_attention_weight_min": -52.54082107543945,
      "activations/layer9_attention_weight_max": 47.51796340942383,
      "activations/layer9_attention_weight_min": -46.43912124633789,
      "epoch": 19.28,
      "learning_rate": 2.591590909090909e-05,
      "loss": 2.7262,
      "step": 331850
    },
    {
      "activations/layer0_attention_weight_max": 14.455183982849121,
      "activations/layer0_attention_weight_min": -13.663961410522461,
      "activations/layer10_attention_weight_max": 38.44511795043945,
      "activations/layer10_attention_weight_min": -37.20014190673828,
      "activations/layer11_attention_weight_max": 35.423004150390625,
      "activations/layer11_attention_weight_min": -36.51148986816406,
      "activations/layer12_attention_weight_max": 21.9035587310791,
      "activations/layer12_attention_weight_min": -24.20598602294922,
      "activations/layer13_attention_weight_max": 38.49647521972656,
      "activations/layer13_attention_weight_min": -28.544631958007812,
      "activations/layer14_attention_weight_max": 36.26268005371094,
      "activations/layer14_attention_weight_min": -31.57497787475586,
      "activations/layer15_attention_weight_max": 33.599910736083984,
      "activations/layer15_attention_weight_min": -31.43866539001465,
      "activations/layer16_attention_weight_max": 36.136234283447266,
      "activations/layer16_attention_weight_min": -34.49671173095703,
      "activations/layer17_attention_weight_max": 52.637271881103516,
      "activations/layer17_attention_weight_min": -46.2135124206543,
      "activations/layer18_attention_weight_max": 48.833675384521484,
      "activations/layer18_attention_weight_min": -39.965545654296875,
      "activations/layer19_attention_weight_max": 25.037599563598633,
      "activations/layer19_attention_weight_min": -23.53896141052246,
      "activations/layer1_attention_weight_max": 15.535881042480469,
      "activations/layer1_attention_weight_min": -13.09343433380127,
      "activations/layer20_attention_weight_max": 23.530580520629883,
      "activations/layer20_attention_weight_min": -22.66742515563965,
      "activations/layer21_attention_weight_max": 41.771461486816406,
      "activations/layer21_attention_weight_min": -20.217905044555664,
      "activations/layer22_attention_weight_max": 31.946273803710938,
      "activations/layer22_attention_weight_min": -25.858762741088867,
      "activations/layer23_attention_weight_max": 39.52446746826172,
      "activations/layer23_attention_weight_min": -24.708187103271484,
      "activations/layer2_attention_weight_max": 30.711181640625,
      "activations/layer2_attention_weight_min": -29.068103790283203,
      "activations/layer3_attention_weight_max": 86.75395202636719,
      "activations/layer3_attention_weight_min": -85.55785369873047,
      "activations/layer4_attention_weight_max": 85.00519561767578,
      "activations/layer4_attention_weight_min": -84.611328125,
      "activations/layer5_attention_weight_max": 65.51396179199219,
      "activations/layer5_attention_weight_min": -70.13532257080078,
      "activations/layer6_attention_weight_max": 48.86677169799805,
      "activations/layer6_attention_weight_min": -54.5819206237793,
      "activations/layer7_attention_weight_max": 71.10555267333984,
      "activations/layer7_attention_weight_min": -67.87528991699219,
      "activations/layer8_attention_weight_max": 46.533145904541016,
      "activations/layer8_attention_weight_min": -50.00188064575195,
      "activations/layer9_attention_weight_max": 45.34192657470703,
      "activations/layer9_attention_weight_min": -47.51777648925781,
      "epoch": 19.29,
      "learning_rate": 2.5896969696969695e-05,
      "loss": 2.7351,
      "step": 331900
    },
    {
      "activations/layer0_attention_weight_max": 16.719581604003906,
      "activations/layer0_attention_weight_min": -13.442361831665039,
      "activations/layer10_attention_weight_max": 38.01071548461914,
      "activations/layer10_attention_weight_min": -38.26213073730469,
      "activations/layer11_attention_weight_max": 34.19942855834961,
      "activations/layer11_attention_weight_min": -37.15174865722656,
      "activations/layer12_attention_weight_max": 17.891845703125,
      "activations/layer12_attention_weight_min": -26.84422492980957,
      "activations/layer13_attention_weight_max": 34.50777816772461,
      "activations/layer13_attention_weight_min": -27.92691993713379,
      "activations/layer14_attention_weight_max": 35.60749053955078,
      "activations/layer14_attention_weight_min": -30.245332717895508,
      "activations/layer15_attention_weight_max": 33.091976165771484,
      "activations/layer15_attention_weight_min": -29.20836639404297,
      "activations/layer16_attention_weight_max": 32.33335494995117,
      "activations/layer16_attention_weight_min": -30.82457160949707,
      "activations/layer17_attention_weight_max": 49.567867279052734,
      "activations/layer17_attention_weight_min": -44.362579345703125,
      "activations/layer18_attention_weight_max": 47.35450744628906,
      "activations/layer18_attention_weight_min": -39.78715515136719,
      "activations/layer19_attention_weight_max": 24.069250106811523,
      "activations/layer19_attention_weight_min": -24.590896606445312,
      "activations/layer1_attention_weight_max": 15.841069221496582,
      "activations/layer1_attention_weight_min": -13.654643058776855,
      "activations/layer20_attention_weight_max": 21.894792556762695,
      "activations/layer20_attention_weight_min": -22.617734909057617,
      "activations/layer21_attention_weight_max": 38.270286560058594,
      "activations/layer21_attention_weight_min": -20.9935245513916,
      "activations/layer22_attention_weight_max": 28.12883186340332,
      "activations/layer22_attention_weight_min": -25.85114288330078,
      "activations/layer23_attention_weight_max": 37.51189041137695,
      "activations/layer23_attention_weight_min": -23.839719772338867,
      "activations/layer2_attention_weight_max": 31.785757064819336,
      "activations/layer2_attention_weight_min": -30.335166931152344,
      "activations/layer3_attention_weight_max": 89.01028442382812,
      "activations/layer3_attention_weight_min": -86.57856750488281,
      "activations/layer4_attention_weight_max": 85.24166107177734,
      "activations/layer4_attention_weight_min": -83.50666046142578,
      "activations/layer5_attention_weight_max": 62.30540084838867,
      "activations/layer5_attention_weight_min": -70.0933837890625,
      "activations/layer6_attention_weight_max": 51.023006439208984,
      "activations/layer6_attention_weight_min": -53.82347106933594,
      "activations/layer7_attention_weight_max": 65.68109893798828,
      "activations/layer7_attention_weight_min": -67.29688262939453,
      "activations/layer8_attention_weight_max": 50.51296615600586,
      "activations/layer8_attention_weight_min": -51.18562698364258,
      "activations/layer9_attention_weight_max": 47.02421569824219,
      "activations/layer9_attention_weight_min": -47.2713508605957,
      "epoch": 19.29,
      "learning_rate": 2.58780303030303e-05,
      "loss": 2.7339,
      "step": 331950
    },
    {
      "activations/layer0_attention_weight_max": 14.927072525024414,
      "activations/layer0_attention_weight_min": -13.607603073120117,
      "activations/layer10_attention_weight_max": 40.11751174926758,
      "activations/layer10_attention_weight_min": -38.99025344848633,
      "activations/layer11_attention_weight_max": 36.32038879394531,
      "activations/layer11_attention_weight_min": -37.24119186401367,
      "activations/layer12_attention_weight_max": 18.081623077392578,
      "activations/layer12_attention_weight_min": -33.09497833251953,
      "activations/layer13_attention_weight_max": 35.24121856689453,
      "activations/layer13_attention_weight_min": -28.82887840270996,
      "activations/layer14_attention_weight_max": 36.53813934326172,
      "activations/layer14_attention_weight_min": -30.07212257385254,
      "activations/layer15_attention_weight_max": 35.609214782714844,
      "activations/layer15_attention_weight_min": -30.43531036376953,
      "activations/layer16_attention_weight_max": 35.668251037597656,
      "activations/layer16_attention_weight_min": -32.53968048095703,
      "activations/layer17_attention_weight_max": 53.56544876098633,
      "activations/layer17_attention_weight_min": -46.766510009765625,
      "activations/layer18_attention_weight_max": 51.47875213623047,
      "activations/layer18_attention_weight_min": -42.35932540893555,
      "activations/layer19_attention_weight_max": 26.27084732055664,
      "activations/layer19_attention_weight_min": -24.63954734802246,
      "activations/layer1_attention_weight_max": 15.426595687866211,
      "activations/layer1_attention_weight_min": -12.81878662109375,
      "activations/layer20_attention_weight_max": 25.51924705505371,
      "activations/layer20_attention_weight_min": -23.110078811645508,
      "activations/layer21_attention_weight_max": 40.12649917602539,
      "activations/layer21_attention_weight_min": -23.62852668762207,
      "activations/layer22_attention_weight_max": 31.560028076171875,
      "activations/layer22_attention_weight_min": -26.951047897338867,
      "activations/layer23_attention_weight_max": 40.27699279785156,
      "activations/layer23_attention_weight_min": -24.305227279663086,
      "activations/layer2_attention_weight_max": 33.75457000732422,
      "activations/layer2_attention_weight_min": -30.9566707611084,
      "activations/layer3_attention_weight_max": 89.34992218017578,
      "activations/layer3_attention_weight_min": -87.86201477050781,
      "activations/layer4_attention_weight_max": 82.02098846435547,
      "activations/layer4_attention_weight_min": -81.59526062011719,
      "activations/layer5_attention_weight_max": 65.02620697021484,
      "activations/layer5_attention_weight_min": -69.12166595458984,
      "activations/layer6_attention_weight_max": 49.93671417236328,
      "activations/layer6_attention_weight_min": -49.32955551147461,
      "activations/layer7_attention_weight_max": 62.61552047729492,
      "activations/layer7_attention_weight_min": -70.98350524902344,
      "activations/layer8_attention_weight_max": 49.481346130371094,
      "activations/layer8_attention_weight_min": -52.72298049926758,
      "activations/layer9_attention_weight_max": 49.065589904785156,
      "activations/layer9_attention_weight_min": -47.157466888427734,
      "epoch": 19.29,
      "learning_rate": 2.5859090909090908e-05,
      "loss": 2.7241,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.5579,
      "eval_samples_per_second": 501.758,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.5579,
      "eval_openwebtext_samples_per_second": 501.758,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 2.0172,
      "eval_wikitext_samples_per_second": 226.06,
      "step": 332000
    },
    {
      "epoch": 19.29,
      "eval_lambada_loss": 2.470703125,
      "eval_lambada_ppl": 11.830762432692795,
      "eval_lambada_runtime": 9.6317,
      "eval_lambada_samples_per_second": 505.516,
      "step": 332000
    },
    {
      "activations/layer0_attention_weight_max": 14.871061325073242,
      "activations/layer0_attention_weight_min": -13.506950378417969,
      "activations/layer10_attention_weight_max": 39.74586868286133,
      "activations/layer10_attention_weight_min": -36.65663528442383,
      "activations/layer11_attention_weight_max": 36.58056640625,
      "activations/layer11_attention_weight_min": -36.577903747558594,
      "activations/layer12_attention_weight_max": 18.905881881713867,
      "activations/layer12_attention_weight_min": -22.617755889892578,
      "activations/layer13_attention_weight_max": 36.791072845458984,
      "activations/layer13_attention_weight_min": -27.41295623779297,
      "activations/layer14_attention_weight_max": 34.573158264160156,
      "activations/layer14_attention_weight_min": -31.25367546081543,
      "activations/layer15_attention_weight_max": 33.00104522705078,
      "activations/layer15_attention_weight_min": -31.521207809448242,
      "activations/layer16_attention_weight_max": 34.22527313232422,
      "activations/layer16_attention_weight_min": -31.677698135375977,
      "activations/layer17_attention_weight_max": 52.22014236450195,
      "activations/layer17_attention_weight_min": -44.08219528198242,
      "activations/layer18_attention_weight_max": 46.76639175415039,
      "activations/layer18_attention_weight_min": -39.195045471191406,
      "activations/layer19_attention_weight_max": 24.301856994628906,
      "activations/layer19_attention_weight_min": -22.1925106048584,
      "activations/layer1_attention_weight_max": 15.269919395446777,
      "activations/layer1_attention_weight_min": -12.765639305114746,
      "activations/layer20_attention_weight_max": 22.213401794433594,
      "activations/layer20_attention_weight_min": -23.18563461303711,
      "activations/layer21_attention_weight_max": 38.00648880004883,
      "activations/layer21_attention_weight_min": -24.379098892211914,
      "activations/layer22_attention_weight_max": 29.64681053161621,
      "activations/layer22_attention_weight_min": -26.610580444335938,
      "activations/layer23_attention_weight_max": 36.69526672363281,
      "activations/layer23_attention_weight_min": -26.440229415893555,
      "activations/layer2_attention_weight_max": 32.38983917236328,
      "activations/layer2_attention_weight_min": -31.049972534179688,
      "activations/layer3_attention_weight_max": 89.4767837524414,
      "activations/layer3_attention_weight_min": -88.13558197021484,
      "activations/layer4_attention_weight_max": 84.1187515258789,
      "activations/layer4_attention_weight_min": -83.67362976074219,
      "activations/layer5_attention_weight_max": 63.335933685302734,
      "activations/layer5_attention_weight_min": -72.39557647705078,
      "activations/layer6_attention_weight_max": 49.70392608642578,
      "activations/layer6_attention_weight_min": -52.52418899536133,
      "activations/layer7_attention_weight_max": 64.48033905029297,
      "activations/layer7_attention_weight_min": -68.5362548828125,
      "activations/layer8_attention_weight_max": 47.04847717285156,
      "activations/layer8_attention_weight_min": -52.34756851196289,
      "activations/layer9_attention_weight_max": 45.09791946411133,
      "activations/layer9_attention_weight_min": -45.485877990722656,
      "epoch": 19.29,
      "learning_rate": 2.5840151515151513e-05,
      "loss": 2.737,
      "step": 332050
    },
    {
      "activations/layer0_attention_weight_max": 15.472610473632812,
      "activations/layer0_attention_weight_min": -13.710948944091797,
      "activations/layer10_attention_weight_max": 38.922119140625,
      "activations/layer10_attention_weight_min": -38.3847541809082,
      "activations/layer11_attention_weight_max": 38.85072326660156,
      "activations/layer11_attention_weight_min": -36.57059097290039,
      "activations/layer12_attention_weight_max": 19.436254501342773,
      "activations/layer12_attention_weight_min": -25.54551887512207,
      "activations/layer13_attention_weight_max": 33.60835266113281,
      "activations/layer13_attention_weight_min": -26.55192756652832,
      "activations/layer14_attention_weight_max": 35.659080505371094,
      "activations/layer14_attention_weight_min": -30.942209243774414,
      "activations/layer15_attention_weight_max": 32.70005798339844,
      "activations/layer15_attention_weight_min": -32.45964813232422,
      "activations/layer16_attention_weight_max": 32.686485290527344,
      "activations/layer16_attention_weight_min": -30.830175399780273,
      "activations/layer17_attention_weight_max": 49.105140686035156,
      "activations/layer17_attention_weight_min": -45.6239128112793,
      "activations/layer18_attention_weight_max": 45.43852996826172,
      "activations/layer18_attention_weight_min": -43.85007858276367,
      "activations/layer19_attention_weight_max": 23.5522518157959,
      "activations/layer19_attention_weight_min": -25.1704044342041,
      "activations/layer1_attention_weight_max": 15.852118492126465,
      "activations/layer1_attention_weight_min": -13.571054458618164,
      "activations/layer20_attention_weight_max": 23.378612518310547,
      "activations/layer20_attention_weight_min": -22.058395385742188,
      "activations/layer21_attention_weight_max": 38.56988525390625,
      "activations/layer21_attention_weight_min": -24.752647399902344,
      "activations/layer22_attention_weight_max": 26.685117721557617,
      "activations/layer22_attention_weight_min": -25.843544006347656,
      "activations/layer23_attention_weight_max": 34.42711639404297,
      "activations/layer23_attention_weight_min": -22.818592071533203,
      "activations/layer2_attention_weight_max": 32.40021896362305,
      "activations/layer2_attention_weight_min": -31.088851928710938,
      "activations/layer3_attention_weight_max": 87.4439697265625,
      "activations/layer3_attention_weight_min": -90.3821792602539,
      "activations/layer4_attention_weight_max": 85.77949523925781,
      "activations/layer4_attention_weight_min": -90.280029296875,
      "activations/layer5_attention_weight_max": 60.38010787963867,
      "activations/layer5_attention_weight_min": -71.74040222167969,
      "activations/layer6_attention_weight_max": 50.14295196533203,
      "activations/layer6_attention_weight_min": -52.1196403503418,
      "activations/layer7_attention_weight_max": 70.88236999511719,
      "activations/layer7_attention_weight_min": -68.12062072753906,
      "activations/layer8_attention_weight_max": 50.507957458496094,
      "activations/layer8_attention_weight_min": -53.08159255981445,
      "activations/layer9_attention_weight_max": 50.75429916381836,
      "activations/layer9_attention_weight_min": -49.941829681396484,
      "epoch": 19.3,
      "learning_rate": 2.582121212121212e-05,
      "loss": 2.7353,
      "step": 332100
    },
    {
      "activations/layer0_attention_weight_max": 15.407904624938965,
      "activations/layer0_attention_weight_min": -13.613957405090332,
      "activations/layer10_attention_weight_max": 37.14680862426758,
      "activations/layer10_attention_weight_min": -36.590579986572266,
      "activations/layer11_attention_weight_max": 34.95356750488281,
      "activations/layer11_attention_weight_min": -36.1533317565918,
      "activations/layer12_attention_weight_max": 18.124923706054688,
      "activations/layer12_attention_weight_min": -28.630783081054688,
      "activations/layer13_attention_weight_max": 34.15311813354492,
      "activations/layer13_attention_weight_min": -27.55333137512207,
      "activations/layer14_attention_weight_max": 35.34586715698242,
      "activations/layer14_attention_weight_min": -31.35082244873047,
      "activations/layer15_attention_weight_max": 33.89792251586914,
      "activations/layer15_attention_weight_min": -33.01118850708008,
      "activations/layer16_attention_weight_max": 35.38526153564453,
      "activations/layer16_attention_weight_min": -33.71024703979492,
      "activations/layer17_attention_weight_max": 56.568138122558594,
      "activations/layer17_attention_weight_min": -46.42058181762695,
      "activations/layer18_attention_weight_max": 53.78530502319336,
      "activations/layer18_attention_weight_min": -40.36279296875,
      "activations/layer19_attention_weight_max": 26.214065551757812,
      "activations/layer19_attention_weight_min": -22.301851272583008,
      "activations/layer1_attention_weight_max": 15.938392639160156,
      "activations/layer1_attention_weight_min": -15.158074378967285,
      "activations/layer20_attention_weight_max": 24.820043563842773,
      "activations/layer20_attention_weight_min": -22.666627883911133,
      "activations/layer21_attention_weight_max": 39.142391204833984,
      "activations/layer21_attention_weight_min": -25.12689208984375,
      "activations/layer22_attention_weight_max": 32.4834098815918,
      "activations/layer22_attention_weight_min": -25.29728889465332,
      "activations/layer23_attention_weight_max": 36.284461975097656,
      "activations/layer23_attention_weight_min": -24.58441162109375,
      "activations/layer2_attention_weight_max": 32.05160903930664,
      "activations/layer2_attention_weight_min": -29.977615356445312,
      "activations/layer3_attention_weight_max": 90.0599365234375,
      "activations/layer3_attention_weight_min": -85.2267074584961,
      "activations/layer4_attention_weight_max": 85.95851135253906,
      "activations/layer4_attention_weight_min": -80.09761810302734,
      "activations/layer5_attention_weight_max": 63.47393798828125,
      "activations/layer5_attention_weight_min": -76.2869873046875,
      "activations/layer6_attention_weight_max": 51.984657287597656,
      "activations/layer6_attention_weight_min": -50.00083923339844,
      "activations/layer7_attention_weight_max": 67.314208984375,
      "activations/layer7_attention_weight_min": -65.0822525024414,
      "activations/layer8_attention_weight_max": 46.933170318603516,
      "activations/layer8_attention_weight_min": -49.53926467895508,
      "activations/layer9_attention_weight_max": 47.216339111328125,
      "activations/layer9_attention_weight_min": -44.82705307006836,
      "epoch": 19.3,
      "learning_rate": 2.5802272727272723e-05,
      "loss": 2.7274,
      "step": 332150
    },
    {
      "activations/layer0_attention_weight_max": 14.946492195129395,
      "activations/layer0_attention_weight_min": -13.279935836791992,
      "activations/layer10_attention_weight_max": 34.923580169677734,
      "activations/layer10_attention_weight_min": -34.37691116333008,
      "activations/layer11_attention_weight_max": 32.76514434814453,
      "activations/layer11_attention_weight_min": -34.220428466796875,
      "activations/layer12_attention_weight_max": 20.42249870300293,
      "activations/layer12_attention_weight_min": -24.221439361572266,
      "activations/layer13_attention_weight_max": 33.441261291503906,
      "activations/layer13_attention_weight_min": -29.27001190185547,
      "activations/layer14_attention_weight_max": 36.72456359863281,
      "activations/layer14_attention_weight_min": -28.381458282470703,
      "activations/layer15_attention_weight_max": 32.318809509277344,
      "activations/layer15_attention_weight_min": -29.596311569213867,
      "activations/layer16_attention_weight_max": 37.088802337646484,
      "activations/layer16_attention_weight_min": -35.56784439086914,
      "activations/layer17_attention_weight_max": 54.65481185913086,
      "activations/layer17_attention_weight_min": -45.6793098449707,
      "activations/layer18_attention_weight_max": 53.74966812133789,
      "activations/layer18_attention_weight_min": -42.663124084472656,
      "activations/layer19_attention_weight_max": 23.070362091064453,
      "activations/layer19_attention_weight_min": -24.444242477416992,
      "activations/layer1_attention_weight_max": 15.759748458862305,
      "activations/layer1_attention_weight_min": -13.908675193786621,
      "activations/layer20_attention_weight_max": 23.30412483215332,
      "activations/layer20_attention_weight_min": -22.43492889404297,
      "activations/layer21_attention_weight_max": 32.8425178527832,
      "activations/layer21_attention_weight_min": -21.95975685119629,
      "activations/layer22_attention_weight_max": 30.34602165222168,
      "activations/layer22_attention_weight_min": -28.532434463500977,
      "activations/layer23_attention_weight_max": 37.393638610839844,
      "activations/layer23_attention_weight_min": -25.133102416992188,
      "activations/layer2_attention_weight_max": 32.20378112792969,
      "activations/layer2_attention_weight_min": -29.79857635498047,
      "activations/layer3_attention_weight_max": 88.51567840576172,
      "activations/layer3_attention_weight_min": -87.161865234375,
      "activations/layer4_attention_weight_max": 79.81438446044922,
      "activations/layer4_attention_weight_min": -80.66368865966797,
      "activations/layer5_attention_weight_max": 62.452552795410156,
      "activations/layer5_attention_weight_min": -67.21241760253906,
      "activations/layer6_attention_weight_max": 46.9431266784668,
      "activations/layer6_attention_weight_min": -47.975276947021484,
      "activations/layer7_attention_weight_max": 64.2386703491211,
      "activations/layer7_attention_weight_min": -60.49905776977539,
      "activations/layer8_attention_weight_max": 45.04063415527344,
      "activations/layer8_attention_weight_min": -51.1744384765625,
      "activations/layer9_attention_weight_max": 46.5717887878418,
      "activations/layer9_attention_weight_min": -44.01987838745117,
      "epoch": 19.3,
      "learning_rate": 2.578333333333333e-05,
      "loss": 2.7185,
      "step": 332200
    },
    {
      "activations/layer0_attention_weight_max": 15.080862998962402,
      "activations/layer0_attention_weight_min": -13.85509967803955,
      "activations/layer10_attention_weight_max": 38.05849838256836,
      "activations/layer10_attention_weight_min": -37.586158752441406,
      "activations/layer11_attention_weight_max": 34.57072448730469,
      "activations/layer11_attention_weight_min": -36.02229309082031,
      "activations/layer12_attention_weight_max": 18.38177490234375,
      "activations/layer12_attention_weight_min": -24.295854568481445,
      "activations/layer13_attention_weight_max": 32.83772277832031,
      "activations/layer13_attention_weight_min": -26.433977127075195,
      "activations/layer14_attention_weight_max": 33.94846725463867,
      "activations/layer14_attention_weight_min": -28.832548141479492,
      "activations/layer15_attention_weight_max": 33.483280181884766,
      "activations/layer15_attention_weight_min": -30.75119972229004,
      "activations/layer16_attention_weight_max": 34.076087951660156,
      "activations/layer16_attention_weight_min": -31.07185935974121,
      "activations/layer17_attention_weight_max": 49.8752555847168,
      "activations/layer17_attention_weight_min": -43.88823318481445,
      "activations/layer18_attention_weight_max": 45.331214904785156,
      "activations/layer18_attention_weight_min": -39.9048957824707,
      "activations/layer19_attention_weight_max": 24.032644271850586,
      "activations/layer19_attention_weight_min": -21.721473693847656,
      "activations/layer1_attention_weight_max": 16.46982192993164,
      "activations/layer1_attention_weight_min": -13.647797584533691,
      "activations/layer20_attention_weight_max": 23.83315658569336,
      "activations/layer20_attention_weight_min": -20.202945709228516,
      "activations/layer21_attention_weight_max": 30.808568954467773,
      "activations/layer21_attention_weight_min": -21.415712356567383,
      "activations/layer22_attention_weight_max": 27.556312561035156,
      "activations/layer22_attention_weight_min": -23.59400177001953,
      "activations/layer23_attention_weight_max": 34.58289337158203,
      "activations/layer23_attention_weight_min": -22.92560386657715,
      "activations/layer2_attention_weight_max": 30.292699813842773,
      "activations/layer2_attention_weight_min": -28.376768112182617,
      "activations/layer3_attention_weight_max": 88.1195297241211,
      "activations/layer3_attention_weight_min": -84.66110229492188,
      "activations/layer4_attention_weight_max": 86.07457733154297,
      "activations/layer4_attention_weight_min": -83.68377685546875,
      "activations/layer5_attention_weight_max": 59.04494857788086,
      "activations/layer5_attention_weight_min": -70.37926483154297,
      "activations/layer6_attention_weight_max": 47.24883270263672,
      "activations/layer6_attention_weight_min": -49.90312194824219,
      "activations/layer7_attention_weight_max": 67.02759552001953,
      "activations/layer7_attention_weight_min": -66.63142395019531,
      "activations/layer8_attention_weight_max": 47.30796813964844,
      "activations/layer8_attention_weight_min": -50.04700469970703,
      "activations/layer9_attention_weight_max": 48.446163177490234,
      "activations/layer9_attention_weight_min": -45.3651237487793,
      "epoch": 19.31,
      "learning_rate": 2.5764393939393936e-05,
      "loss": 2.7215,
      "step": 332250
    },
    {
      "activations/layer0_attention_weight_max": 15.457196235656738,
      "activations/layer0_attention_weight_min": -14.054784774780273,
      "activations/layer10_attention_weight_max": 36.1258544921875,
      "activations/layer10_attention_weight_min": -36.05952072143555,
      "activations/layer11_attention_weight_max": 32.885196685791016,
      "activations/layer11_attention_weight_min": -34.55384826660156,
      "activations/layer12_attention_weight_max": 19.024381637573242,
      "activations/layer12_attention_weight_min": -22.99897003173828,
      "activations/layer13_attention_weight_max": 31.12411117553711,
      "activations/layer13_attention_weight_min": -26.497983932495117,
      "activations/layer14_attention_weight_max": 34.73707580566406,
      "activations/layer14_attention_weight_min": -29.96001625061035,
      "activations/layer15_attention_weight_max": 30.346553802490234,
      "activations/layer15_attention_weight_min": -30.160411834716797,
      "activations/layer16_attention_weight_max": 32.35319519042969,
      "activations/layer16_attention_weight_min": -29.96324920654297,
      "activations/layer17_attention_weight_max": 49.25324249267578,
      "activations/layer17_attention_weight_min": -43.875999450683594,
      "activations/layer18_attention_weight_max": 44.24252700805664,
      "activations/layer18_attention_weight_min": -39.55519104003906,
      "activations/layer19_attention_weight_max": 24.320512771606445,
      "activations/layer19_attention_weight_min": -22.374616622924805,
      "activations/layer1_attention_weight_max": 14.740189552307129,
      "activations/layer1_attention_weight_min": -15.051901817321777,
      "activations/layer20_attention_weight_max": 21.661930084228516,
      "activations/layer20_attention_weight_min": -21.97399139404297,
      "activations/layer21_attention_weight_max": 38.85912322998047,
      "activations/layer21_attention_weight_min": -25.5688533782959,
      "activations/layer22_attention_weight_max": 27.844093322753906,
      "activations/layer22_attention_weight_min": -24.49648666381836,
      "activations/layer23_attention_weight_max": 38.95142364501953,
      "activations/layer23_attention_weight_min": -26.15253257751465,
      "activations/layer2_attention_weight_max": 34.30632400512695,
      "activations/layer2_attention_weight_min": -31.32947540283203,
      "activations/layer3_attention_weight_max": 93.93904876708984,
      "activations/layer3_attention_weight_min": -92.49473571777344,
      "activations/layer4_attention_weight_max": 89.8067398071289,
      "activations/layer4_attention_weight_min": -86.04722595214844,
      "activations/layer5_attention_weight_max": 63.35050582885742,
      "activations/layer5_attention_weight_min": -70.06366729736328,
      "activations/layer6_attention_weight_max": 51.27769470214844,
      "activations/layer6_attention_weight_min": -51.536441802978516,
      "activations/layer7_attention_weight_max": 65.40084838867188,
      "activations/layer7_attention_weight_min": -68.44905090332031,
      "activations/layer8_attention_weight_max": 46.531063079833984,
      "activations/layer8_attention_weight_min": -48.6190071105957,
      "activations/layer9_attention_weight_max": 42.688987731933594,
      "activations/layer9_attention_weight_min": -45.230979919433594,
      "epoch": 19.31,
      "learning_rate": 2.5745454545454544e-05,
      "loss": 2.7286,
      "step": 332300
    },
    {
      "activations/layer0_attention_weight_max": 16.978466033935547,
      "activations/layer0_attention_weight_min": -13.972543716430664,
      "activations/layer10_attention_weight_max": 32.94783401489258,
      "activations/layer10_attention_weight_min": -35.37852478027344,
      "activations/layer11_attention_weight_max": 31.527097702026367,
      "activations/layer11_attention_weight_min": -34.3935661315918,
      "activations/layer12_attention_weight_max": 18.788684844970703,
      "activations/layer12_attention_weight_min": -32.78255081176758,
      "activations/layer13_attention_weight_max": 29.56552505493164,
      "activations/layer13_attention_weight_min": -25.89932632446289,
      "activations/layer14_attention_weight_max": 33.34664535522461,
      "activations/layer14_attention_weight_min": -28.159543991088867,
      "activations/layer15_attention_weight_max": 29.405658721923828,
      "activations/layer15_attention_weight_min": -28.967927932739258,
      "activations/layer16_attention_weight_max": 30.513200759887695,
      "activations/layer16_attention_weight_min": -29.95685386657715,
      "activations/layer17_attention_weight_max": 49.725440979003906,
      "activations/layer17_attention_weight_min": -41.601165771484375,
      "activations/layer18_attention_weight_max": 41.82959747314453,
      "activations/layer18_attention_weight_min": -36.520530700683594,
      "activations/layer19_attention_weight_max": 23.97241973876953,
      "activations/layer19_attention_weight_min": -22.046371459960938,
      "activations/layer1_attention_weight_max": 15.551774024963379,
      "activations/layer1_attention_weight_min": -12.842928886413574,
      "activations/layer20_attention_weight_max": 20.989086151123047,
      "activations/layer20_attention_weight_min": -20.71175765991211,
      "activations/layer21_attention_weight_max": 31.12771224975586,
      "activations/layer21_attention_weight_min": -21.33553123474121,
      "activations/layer22_attention_weight_max": 30.028127670288086,
      "activations/layer22_attention_weight_min": -25.469419479370117,
      "activations/layer23_attention_weight_max": 38.211952209472656,
      "activations/layer23_attention_weight_min": -23.830114364624023,
      "activations/layer2_attention_weight_max": 31.191390991210938,
      "activations/layer2_attention_weight_min": -29.586009979248047,
      "activations/layer3_attention_weight_max": 81.51451873779297,
      "activations/layer3_attention_weight_min": -77.40380096435547,
      "activations/layer4_attention_weight_max": 80.1284408569336,
      "activations/layer4_attention_weight_min": -73.94429779052734,
      "activations/layer5_attention_weight_max": 56.91280746459961,
      "activations/layer5_attention_weight_min": -69.86968994140625,
      "activations/layer6_attention_weight_max": 44.09678268432617,
      "activations/layer6_attention_weight_min": -44.92220687866211,
      "activations/layer7_attention_weight_max": 60.73395538330078,
      "activations/layer7_attention_weight_min": -61.2845344543457,
      "activations/layer8_attention_weight_max": 41.60029983520508,
      "activations/layer8_attention_weight_min": -47.53437042236328,
      "activations/layer9_attention_weight_max": 44.19818878173828,
      "activations/layer9_attention_weight_min": -43.13303756713867,
      "epoch": 19.31,
      "learning_rate": 2.572651515151515e-05,
      "loss": 2.7391,
      "step": 332350
    },
    {
      "activations/layer0_attention_weight_max": 15.287459373474121,
      "activations/layer0_attention_weight_min": -14.074460983276367,
      "activations/layer10_attention_weight_max": 38.025054931640625,
      "activations/layer10_attention_weight_min": -37.61920928955078,
      "activations/layer11_attention_weight_max": 35.23141860961914,
      "activations/layer11_attention_weight_min": -36.027679443359375,
      "activations/layer12_attention_weight_max": 19.18533706665039,
      "activations/layer12_attention_weight_min": -24.27869415283203,
      "activations/layer13_attention_weight_max": 36.41332244873047,
      "activations/layer13_attention_weight_min": -28.12405776977539,
      "activations/layer14_attention_weight_max": 38.94563674926758,
      "activations/layer14_attention_weight_min": -29.768722534179688,
      "activations/layer15_attention_weight_max": 36.18235397338867,
      "activations/layer15_attention_weight_min": -32.21744918823242,
      "activations/layer16_attention_weight_max": 36.52005386352539,
      "activations/layer16_attention_weight_min": -33.3742561340332,
      "activations/layer17_attention_weight_max": 59.26404571533203,
      "activations/layer17_attention_weight_min": -47.28479766845703,
      "activations/layer18_attention_weight_max": 55.836151123046875,
      "activations/layer18_attention_weight_min": -40.89889144897461,
      "activations/layer19_attention_weight_max": 26.934125900268555,
      "activations/layer19_attention_weight_min": -23.02073860168457,
      "activations/layer1_attention_weight_max": 15.56128215789795,
      "activations/layer1_attention_weight_min": -14.050127029418945,
      "activations/layer20_attention_weight_max": 25.660863876342773,
      "activations/layer20_attention_weight_min": -22.195781707763672,
      "activations/layer21_attention_weight_max": 37.604549407958984,
      "activations/layer21_attention_weight_min": -22.547317504882812,
      "activations/layer22_attention_weight_max": 29.781932830810547,
      "activations/layer22_attention_weight_min": -27.09760856628418,
      "activations/layer23_attention_weight_max": 37.110801696777344,
      "activations/layer23_attention_weight_min": -22.596233367919922,
      "activations/layer2_attention_weight_max": 32.65174102783203,
      "activations/layer2_attention_weight_min": -31.06014633178711,
      "activations/layer3_attention_weight_max": 89.35197448730469,
      "activations/layer3_attention_weight_min": -87.88333129882812,
      "activations/layer4_attention_weight_max": 89.58782196044922,
      "activations/layer4_attention_weight_min": -86.22219848632812,
      "activations/layer5_attention_weight_max": 60.465850830078125,
      "activations/layer5_attention_weight_min": -69.80787658691406,
      "activations/layer6_attention_weight_max": 49.66654968261719,
      "activations/layer6_attention_weight_min": -51.07096481323242,
      "activations/layer7_attention_weight_max": 65.83550262451172,
      "activations/layer7_attention_weight_min": -66.58282470703125,
      "activations/layer8_attention_weight_max": 51.768882751464844,
      "activations/layer8_attention_weight_min": -55.45528030395508,
      "activations/layer9_attention_weight_max": 54.04619216918945,
      "activations/layer9_attention_weight_min": -49.95344161987305,
      "epoch": 19.31,
      "learning_rate": 2.5707575757575757e-05,
      "loss": 2.7338,
      "step": 332400
    },
    {
      "activations/layer0_attention_weight_max": 15.248652458190918,
      "activations/layer0_attention_weight_min": -14.012540817260742,
      "activations/layer10_attention_weight_max": 31.90336036682129,
      "activations/layer10_attention_weight_min": -33.41014862060547,
      "activations/layer11_attention_weight_max": 30.187307357788086,
      "activations/layer11_attention_weight_min": -33.315067291259766,
      "activations/layer12_attention_weight_max": 19.59084701538086,
      "activations/layer12_attention_weight_min": -25.853498458862305,
      "activations/layer13_attention_weight_max": 32.437644958496094,
      "activations/layer13_attention_weight_min": -27.068071365356445,
      "activations/layer14_attention_weight_max": 33.27543258666992,
      "activations/layer14_attention_weight_min": -31.706310272216797,
      "activations/layer15_attention_weight_max": 32.07633590698242,
      "activations/layer15_attention_weight_min": -31.207406997680664,
      "activations/layer16_attention_weight_max": 33.72862243652344,
      "activations/layer16_attention_weight_min": -31.707231521606445,
      "activations/layer17_attention_weight_max": 51.18682861328125,
      "activations/layer17_attention_weight_min": -45.28935241699219,
      "activations/layer18_attention_weight_max": 47.253395080566406,
      "activations/layer18_attention_weight_min": -40.43559265136719,
      "activations/layer19_attention_weight_max": 25.528593063354492,
      "activations/layer19_attention_weight_min": -24.062498092651367,
      "activations/layer1_attention_weight_max": 15.937478065490723,
      "activations/layer1_attention_weight_min": -13.175070762634277,
      "activations/layer20_attention_weight_max": 22.441068649291992,
      "activations/layer20_attention_weight_min": -23.327617645263672,
      "activations/layer21_attention_weight_max": 33.201236724853516,
      "activations/layer21_attention_weight_min": -23.6586856842041,
      "activations/layer22_attention_weight_max": 29.489336013793945,
      "activations/layer22_attention_weight_min": -26.39622688293457,
      "activations/layer23_attention_weight_max": 34.126766204833984,
      "activations/layer23_attention_weight_min": -24.50558853149414,
      "activations/layer2_attention_weight_max": 31.97887420654297,
      "activations/layer2_attention_weight_min": -29.299766540527344,
      "activations/layer3_attention_weight_max": 84.75270080566406,
      "activations/layer3_attention_weight_min": -82.41390991210938,
      "activations/layer4_attention_weight_max": 80.80152130126953,
      "activations/layer4_attention_weight_min": -80.2010498046875,
      "activations/layer5_attention_weight_max": 61.67906951904297,
      "activations/layer5_attention_weight_min": -69.09909057617188,
      "activations/layer6_attention_weight_max": 47.35811233520508,
      "activations/layer6_attention_weight_min": -45.635990142822266,
      "activations/layer7_attention_weight_max": 65.95325469970703,
      "activations/layer7_attention_weight_min": -60.55931854248047,
      "activations/layer8_attention_weight_max": 42.81649398803711,
      "activations/layer8_attention_weight_min": -46.74462127685547,
      "activations/layer9_attention_weight_max": 41.017494201660156,
      "activations/layer9_attention_weight_min": -44.78620147705078,
      "epoch": 19.32,
      "learning_rate": 2.568863636363636e-05,
      "loss": 2.7247,
      "step": 332450
    },
    {
      "activations/layer0_attention_weight_max": 16.09639549255371,
      "activations/layer0_attention_weight_min": -14.036370277404785,
      "activations/layer10_attention_weight_max": 37.698429107666016,
      "activations/layer10_attention_weight_min": -38.01729202270508,
      "activations/layer11_attention_weight_max": 34.890602111816406,
      "activations/layer11_attention_weight_min": -35.67779541015625,
      "activations/layer12_attention_weight_max": 19.331253051757812,
      "activations/layer12_attention_weight_min": -32.59734344482422,
      "activations/layer13_attention_weight_max": 32.58254623413086,
      "activations/layer13_attention_weight_min": -27.50145149230957,
      "activations/layer14_attention_weight_max": 35.61104202270508,
      "activations/layer14_attention_weight_min": -29.212154388427734,
      "activations/layer15_attention_weight_max": 32.09525680541992,
      "activations/layer15_attention_weight_min": -30.703584671020508,
      "activations/layer16_attention_weight_max": 32.02914047241211,
      "activations/layer16_attention_weight_min": -30.631839752197266,
      "activations/layer17_attention_weight_max": 51.418270111083984,
      "activations/layer17_attention_weight_min": -41.74715042114258,
      "activations/layer18_attention_weight_max": 43.17496109008789,
      "activations/layer18_attention_weight_min": -39.02198791503906,
      "activations/layer19_attention_weight_max": 23.324525833129883,
      "activations/layer19_attention_weight_min": -22.783506393432617,
      "activations/layer1_attention_weight_max": 15.774081230163574,
      "activations/layer1_attention_weight_min": -13.392292976379395,
      "activations/layer20_attention_weight_max": 22.444021224975586,
      "activations/layer20_attention_weight_min": -22.798606872558594,
      "activations/layer21_attention_weight_max": 36.773284912109375,
      "activations/layer21_attention_weight_min": -23.50031089782715,
      "activations/layer22_attention_weight_max": 31.117877960205078,
      "activations/layer22_attention_weight_min": -28.24374771118164,
      "activations/layer23_attention_weight_max": 37.07451629638672,
      "activations/layer23_attention_weight_min": -24.368492126464844,
      "activations/layer2_attention_weight_max": 29.817508697509766,
      "activations/layer2_attention_weight_min": -29.63946533203125,
      "activations/layer3_attention_weight_max": 83.94331359863281,
      "activations/layer3_attention_weight_min": -83.3960189819336,
      "activations/layer4_attention_weight_max": 83.1095199584961,
      "activations/layer4_attention_weight_min": -82.8288803100586,
      "activations/layer5_attention_weight_max": 61.890602111816406,
      "activations/layer5_attention_weight_min": -69.06327056884766,
      "activations/layer6_attention_weight_max": 52.02720260620117,
      "activations/layer6_attention_weight_min": -49.391746520996094,
      "activations/layer7_attention_weight_max": 65.50926208496094,
      "activations/layer7_attention_weight_min": -68.09365844726562,
      "activations/layer8_attention_weight_max": 53.1154899597168,
      "activations/layer8_attention_weight_min": -51.662574768066406,
      "activations/layer9_attention_weight_max": 44.96073532104492,
      "activations/layer9_attention_weight_min": -48.10354232788086,
      "epoch": 19.32,
      "learning_rate": 2.5669696969696967e-05,
      "loss": 2.7156,
      "step": 332500
    },
    {
      "activations/layer0_attention_weight_max": 15.285160064697266,
      "activations/layer0_attention_weight_min": -13.939736366271973,
      "activations/layer10_attention_weight_max": 33.97028732299805,
      "activations/layer10_attention_weight_min": -35.36782455444336,
      "activations/layer11_attention_weight_max": 34.33347702026367,
      "activations/layer11_attention_weight_min": -33.843570709228516,
      "activations/layer12_attention_weight_max": 18.19182777404785,
      "activations/layer12_attention_weight_min": -27.016969680786133,
      "activations/layer13_attention_weight_max": 30.02520179748535,
      "activations/layer13_attention_weight_min": -26.531450271606445,
      "activations/layer14_attention_weight_max": 36.37443161010742,
      "activations/layer14_attention_weight_min": -33.13764190673828,
      "activations/layer15_attention_weight_max": 32.50217819213867,
      "activations/layer15_attention_weight_min": -32.45917510986328,
      "activations/layer16_attention_weight_max": 34.73826599121094,
      "activations/layer16_attention_weight_min": -33.33728790283203,
      "activations/layer17_attention_weight_max": 54.57720184326172,
      "activations/layer17_attention_weight_min": -44.2191276550293,
      "activations/layer18_attention_weight_max": 50.6409797668457,
      "activations/layer18_attention_weight_min": -39.844581604003906,
      "activations/layer19_attention_weight_max": 26.053354263305664,
      "activations/layer19_attention_weight_min": -25.511966705322266,
      "activations/layer1_attention_weight_max": 16.269166946411133,
      "activations/layer1_attention_weight_min": -13.53931999206543,
      "activations/layer20_attention_weight_max": 22.23668098449707,
      "activations/layer20_attention_weight_min": -22.925241470336914,
      "activations/layer21_attention_weight_max": 33.158424377441406,
      "activations/layer21_attention_weight_min": -22.69806480407715,
      "activations/layer22_attention_weight_max": 26.866344451904297,
      "activations/layer22_attention_weight_min": -27.02364730834961,
      "activations/layer23_attention_weight_max": 32.984981536865234,
      "activations/layer23_attention_weight_min": -25.38813591003418,
      "activations/layer2_attention_weight_max": 30.97411346435547,
      "activations/layer2_attention_weight_min": -28.52637481689453,
      "activations/layer3_attention_weight_max": 87.71564483642578,
      "activations/layer3_attention_weight_min": -86.31146240234375,
      "activations/layer4_attention_weight_max": 86.62850189208984,
      "activations/layer4_attention_weight_min": -82.88945770263672,
      "activations/layer5_attention_weight_max": 64.09768676757812,
      "activations/layer5_attention_weight_min": -70.28565216064453,
      "activations/layer6_attention_weight_max": 49.80738067626953,
      "activations/layer6_attention_weight_min": -51.673194885253906,
      "activations/layer7_attention_weight_max": 67.78671264648438,
      "activations/layer7_attention_weight_min": -65.21344757080078,
      "activations/layer8_attention_weight_max": 47.19218826293945,
      "activations/layer8_attention_weight_min": -52.03561782836914,
      "activations/layer9_attention_weight_max": 43.9296760559082,
      "activations/layer9_attention_weight_min": -46.403953552246094,
      "epoch": 19.32,
      "learning_rate": 2.5650757575757572e-05,
      "loss": 2.7403,
      "step": 332550
    },
    {
      "activations/layer0_attention_weight_max": 15.381673812866211,
      "activations/layer0_attention_weight_min": -13.896646499633789,
      "activations/layer10_attention_weight_max": 35.82635498046875,
      "activations/layer10_attention_weight_min": -35.12567901611328,
      "activations/layer11_attention_weight_max": 32.99169158935547,
      "activations/layer11_attention_weight_min": -36.41639709472656,
      "activations/layer12_attention_weight_max": 20.315052032470703,
      "activations/layer12_attention_weight_min": -29.172351837158203,
      "activations/layer13_attention_weight_max": 34.633331298828125,
      "activations/layer13_attention_weight_min": -28.551462173461914,
      "activations/layer14_attention_weight_max": 35.47787094116211,
      "activations/layer14_attention_weight_min": -31.73871612548828,
      "activations/layer15_attention_weight_max": 34.80646896362305,
      "activations/layer15_attention_weight_min": -31.205440521240234,
      "activations/layer16_attention_weight_max": 38.42376708984375,
      "activations/layer16_attention_weight_min": -34.81401443481445,
      "activations/layer17_attention_weight_max": 56.486934661865234,
      "activations/layer17_attention_weight_min": -48.269161224365234,
      "activations/layer18_attention_weight_max": 53.729576110839844,
      "activations/layer18_attention_weight_min": -45.04883575439453,
      "activations/layer19_attention_weight_max": 25.934585571289062,
      "activations/layer19_attention_weight_min": -24.302734375,
      "activations/layer1_attention_weight_max": 17.131834030151367,
      "activations/layer1_attention_weight_min": -14.015450477600098,
      "activations/layer20_attention_weight_max": 26.388425827026367,
      "activations/layer20_attention_weight_min": -19.401927947998047,
      "activations/layer21_attention_weight_max": 39.63945770263672,
      "activations/layer21_attention_weight_min": -20.51374626159668,
      "activations/layer22_attention_weight_max": 29.08138084411621,
      "activations/layer22_attention_weight_min": -24.122041702270508,
      "activations/layer23_attention_weight_max": 40.79025650024414,
      "activations/layer23_attention_weight_min": -22.597990036010742,
      "activations/layer2_attention_weight_max": 32.8156852722168,
      "activations/layer2_attention_weight_min": -30.07205581665039,
      "activations/layer3_attention_weight_max": 88.83009338378906,
      "activations/layer3_attention_weight_min": -86.99093627929688,
      "activations/layer4_attention_weight_max": 83.67487335205078,
      "activations/layer4_attention_weight_min": -83.56739044189453,
      "activations/layer5_attention_weight_max": 64.48341369628906,
      "activations/layer5_attention_weight_min": -67.68470001220703,
      "activations/layer6_attention_weight_max": 49.188758850097656,
      "activations/layer6_attention_weight_min": -50.2133903503418,
      "activations/layer7_attention_weight_max": 65.32884979248047,
      "activations/layer7_attention_weight_min": -69.02540588378906,
      "activations/layer8_attention_weight_max": 49.43159103393555,
      "activations/layer8_attention_weight_min": -49.50786209106445,
      "activations/layer9_attention_weight_max": 46.53831100463867,
      "activations/layer9_attention_weight_min": -45.06877517700195,
      "epoch": 19.33,
      "learning_rate": 2.563181818181818e-05,
      "loss": 2.7176,
      "step": 332600
    },
    {
      "activations/layer0_attention_weight_max": 15.884292602539062,
      "activations/layer0_attention_weight_min": -14.157264709472656,
      "activations/layer10_attention_weight_max": 39.75017166137695,
      "activations/layer10_attention_weight_min": -37.5948600769043,
      "activations/layer11_attention_weight_max": 39.71216583251953,
      "activations/layer11_attention_weight_min": -35.98622131347656,
      "activations/layer12_attention_weight_max": 19.467695236206055,
      "activations/layer12_attention_weight_min": -23.998817443847656,
      "activations/layer13_attention_weight_max": 40.72046661376953,
      "activations/layer13_attention_weight_min": -29.584686279296875,
      "activations/layer14_attention_weight_max": 41.186676025390625,
      "activations/layer14_attention_weight_min": -33.16509246826172,
      "activations/layer15_attention_weight_max": 40.97716522216797,
      "activations/layer15_attention_weight_min": -32.06917953491211,
      "activations/layer16_attention_weight_max": 40.96103286743164,
      "activations/layer16_attention_weight_min": -34.19801330566406,
      "activations/layer17_attention_weight_max": 62.72704315185547,
      "activations/layer17_attention_weight_min": -52.3102912902832,
      "activations/layer18_attention_weight_max": 57.4016227722168,
      "activations/layer18_attention_weight_min": -49.237091064453125,
      "activations/layer19_attention_weight_max": 30.441370010375977,
      "activations/layer19_attention_weight_min": -28.942230224609375,
      "activations/layer1_attention_weight_max": 14.728002548217773,
      "activations/layer1_attention_weight_min": -13.202789306640625,
      "activations/layer20_attention_weight_max": 29.698251724243164,
      "activations/layer20_attention_weight_min": -26.599021911621094,
      "activations/layer21_attention_weight_max": 49.22944259643555,
      "activations/layer21_attention_weight_min": -24.32179832458496,
      "activations/layer22_attention_weight_max": 36.0167121887207,
      "activations/layer22_attention_weight_min": -30.21331787109375,
      "activations/layer23_attention_weight_max": 37.541378021240234,
      "activations/layer23_attention_weight_min": -25.93289566040039,
      "activations/layer2_attention_weight_max": 33.72630310058594,
      "activations/layer2_attention_weight_min": -31.84857177734375,
      "activations/layer3_attention_weight_max": 88.31515502929688,
      "activations/layer3_attention_weight_min": -83.16839599609375,
      "activations/layer4_attention_weight_max": 84.46243286132812,
      "activations/layer4_attention_weight_min": -79.83505249023438,
      "activations/layer5_attention_weight_max": 62.848995208740234,
      "activations/layer5_attention_weight_min": -70.41171264648438,
      "activations/layer6_attention_weight_max": 48.75682830810547,
      "activations/layer6_attention_weight_min": -48.43913650512695,
      "activations/layer7_attention_weight_max": 66.03034210205078,
      "activations/layer7_attention_weight_min": -63.46697235107422,
      "activations/layer8_attention_weight_max": 48.07352066040039,
      "activations/layer8_attention_weight_min": -49.26320266723633,
      "activations/layer9_attention_weight_max": 49.48420333862305,
      "activations/layer9_attention_weight_min": -45.51003646850586,
      "epoch": 19.33,
      "learning_rate": 2.5612878787878785e-05,
      "loss": 2.7293,
      "step": 332650
    },
    {
      "activations/layer0_attention_weight_max": 15.159987449645996,
      "activations/layer0_attention_weight_min": -14.029133796691895,
      "activations/layer10_attention_weight_max": 33.87169647216797,
      "activations/layer10_attention_weight_min": -32.263885498046875,
      "activations/layer11_attention_weight_max": 32.465614318847656,
      "activations/layer11_attention_weight_min": -32.6060791015625,
      "activations/layer12_attention_weight_max": 19.578184127807617,
      "activations/layer12_attention_weight_min": -26.50777816772461,
      "activations/layer13_attention_weight_max": 33.02346420288086,
      "activations/layer13_attention_weight_min": -28.94169044494629,
      "activations/layer14_attention_weight_max": 36.08937454223633,
      "activations/layer14_attention_weight_min": -30.66181755065918,
      "activations/layer15_attention_weight_max": 32.32054138183594,
      "activations/layer15_attention_weight_min": -30.993730545043945,
      "activations/layer16_attention_weight_max": 35.780174255371094,
      "activations/layer16_attention_weight_min": -33.78645706176758,
      "activations/layer17_attention_weight_max": 50.513389587402344,
      "activations/layer17_attention_weight_min": -45.5194091796875,
      "activations/layer18_attention_weight_max": 48.33866882324219,
      "activations/layer18_attention_weight_min": -39.91109085083008,
      "activations/layer19_attention_weight_max": 24.66796112060547,
      "activations/layer19_attention_weight_min": -23.923446655273438,
      "activations/layer1_attention_weight_max": 14.658429145812988,
      "activations/layer1_attention_weight_min": -13.25804615020752,
      "activations/layer20_attention_weight_max": 23.693687438964844,
      "activations/layer20_attention_weight_min": -22.434656143188477,
      "activations/layer21_attention_weight_max": 42.45912551879883,
      "activations/layer21_attention_weight_min": -24.89633560180664,
      "activations/layer22_attention_weight_max": 30.020816802978516,
      "activations/layer22_attention_weight_min": -25.74740219116211,
      "activations/layer23_attention_weight_max": 35.167564392089844,
      "activations/layer23_attention_weight_min": -23.641098022460938,
      "activations/layer2_attention_weight_max": 31.72817611694336,
      "activations/layer2_attention_weight_min": -30.264328002929688,
      "activations/layer3_attention_weight_max": 86.23540496826172,
      "activations/layer3_attention_weight_min": -86.05188751220703,
      "activations/layer4_attention_weight_max": 81.45419311523438,
      "activations/layer4_attention_weight_min": -79.9605941772461,
      "activations/layer5_attention_weight_max": 61.307430267333984,
      "activations/layer5_attention_weight_min": -72.01420593261719,
      "activations/layer6_attention_weight_max": 47.681053161621094,
      "activations/layer6_attention_weight_min": -47.22214889526367,
      "activations/layer7_attention_weight_max": 65.04264068603516,
      "activations/layer7_attention_weight_min": -62.12613296508789,
      "activations/layer8_attention_weight_max": 43.4517822265625,
      "activations/layer8_attention_weight_min": -49.492984771728516,
      "activations/layer9_attention_weight_max": 39.87619400024414,
      "activations/layer9_attention_weight_min": -43.89243698120117,
      "epoch": 19.33,
      "learning_rate": 2.559393939393939e-05,
      "loss": 2.734,
      "step": 332700
    },
    {
      "activations/layer0_attention_weight_max": 15.427153587341309,
      "activations/layer0_attention_weight_min": -14.196008682250977,
      "activations/layer10_attention_weight_max": 34.12346267700195,
      "activations/layer10_attention_weight_min": -34.123111724853516,
      "activations/layer11_attention_weight_max": 32.94322204589844,
      "activations/layer11_attention_weight_min": -32.11561584472656,
      "activations/layer12_attention_weight_max": 20.2914981842041,
      "activations/layer12_attention_weight_min": -27.137657165527344,
      "activations/layer13_attention_weight_max": 32.27710723876953,
      "activations/layer13_attention_weight_min": -29.185375213623047,
      "activations/layer14_attention_weight_max": 35.78302001953125,
      "activations/layer14_attention_weight_min": -30.819015502929688,
      "activations/layer15_attention_weight_max": 30.71221351623535,
      "activations/layer15_attention_weight_min": -29.411563873291016,
      "activations/layer16_attention_weight_max": 33.51405334472656,
      "activations/layer16_attention_weight_min": -32.70744705200195,
      "activations/layer17_attention_weight_max": 49.83684539794922,
      "activations/layer17_attention_weight_min": -43.78340148925781,
      "activations/layer18_attention_weight_max": 47.056182861328125,
      "activations/layer18_attention_weight_min": -39.43585968017578,
      "activations/layer19_attention_weight_max": 24.751386642456055,
      "activations/layer19_attention_weight_min": -22.589519500732422,
      "activations/layer1_attention_weight_max": 16.22589874267578,
      "activations/layer1_attention_weight_min": -13.241155624389648,
      "activations/layer20_attention_weight_max": 23.246015548706055,
      "activations/layer20_attention_weight_min": -24.419164657592773,
      "activations/layer21_attention_weight_max": 40.78841781616211,
      "activations/layer21_attention_weight_min": -23.492643356323242,
      "activations/layer22_attention_weight_max": 27.475481033325195,
      "activations/layer22_attention_weight_min": -25.75538444519043,
      "activations/layer23_attention_weight_max": 38.1420783996582,
      "activations/layer23_attention_weight_min": -24.138458251953125,
      "activations/layer2_attention_weight_max": 31.606565475463867,
      "activations/layer2_attention_weight_min": -29.97903060913086,
      "activations/layer3_attention_weight_max": 84.06532287597656,
      "activations/layer3_attention_weight_min": -83.54679107666016,
      "activations/layer4_attention_weight_max": 78.17782592773438,
      "activations/layer4_attention_weight_min": -80.22682189941406,
      "activations/layer5_attention_weight_max": 59.563899993896484,
      "activations/layer5_attention_weight_min": -65.6692886352539,
      "activations/layer6_attention_weight_max": 45.99423599243164,
      "activations/layer6_attention_weight_min": -48.184730529785156,
      "activations/layer7_attention_weight_max": 60.75532150268555,
      "activations/layer7_attention_weight_min": -60.68080139160156,
      "activations/layer8_attention_weight_max": 42.889923095703125,
      "activations/layer8_attention_weight_min": -48.39506149291992,
      "activations/layer9_attention_weight_max": 40.47626495361328,
      "activations/layer9_attention_weight_min": -43.65017318725586,
      "epoch": 19.33,
      "learning_rate": 2.5575378787878784e-05,
      "loss": 2.7274,
      "step": 332750
    },
    {
      "activations/layer0_attention_weight_max": 15.259512901306152,
      "activations/layer0_attention_weight_min": -14.22519588470459,
      "activations/layer10_attention_weight_max": 36.44102096557617,
      "activations/layer10_attention_weight_min": -35.12263488769531,
      "activations/layer11_attention_weight_max": 35.392608642578125,
      "activations/layer11_attention_weight_min": -33.6094970703125,
      "activations/layer12_attention_weight_max": 18.872995376586914,
      "activations/layer12_attention_weight_min": -24.489734649658203,
      "activations/layer13_attention_weight_max": 31.96895408630371,
      "activations/layer13_attention_weight_min": -29.761722564697266,
      "activations/layer14_attention_weight_max": 35.90048599243164,
      "activations/layer14_attention_weight_min": -31.14995765686035,
      "activations/layer15_attention_weight_max": 32.41637420654297,
      "activations/layer15_attention_weight_min": -32.50598907470703,
      "activations/layer16_attention_weight_max": 32.69130325317383,
      "activations/layer16_attention_weight_min": -33.105796813964844,
      "activations/layer17_attention_weight_max": 51.12199020385742,
      "activations/layer17_attention_weight_min": -46.315948486328125,
      "activations/layer18_attention_weight_max": 45.97358322143555,
      "activations/layer18_attention_weight_min": -40.39004135131836,
      "activations/layer19_attention_weight_max": 23.86945915222168,
      "activations/layer19_attention_weight_min": -23.91136932373047,
      "activations/layer1_attention_weight_max": 16.274612426757812,
      "activations/layer1_attention_weight_min": -12.519502639770508,
      "activations/layer20_attention_weight_max": 22.164453506469727,
      "activations/layer20_attention_weight_min": -25.115337371826172,
      "activations/layer21_attention_weight_max": 34.26597213745117,
      "activations/layer21_attention_weight_min": -20.169551849365234,
      "activations/layer22_attention_weight_max": 27.953184127807617,
      "activations/layer22_attention_weight_min": -25.83095359802246,
      "activations/layer23_attention_weight_max": 37.89662170410156,
      "activations/layer23_attention_weight_min": -25.37649917602539,
      "activations/layer2_attention_weight_max": 30.887664794921875,
      "activations/layer2_attention_weight_min": -31.161922454833984,
      "activations/layer3_attention_weight_max": 83.56946563720703,
      "activations/layer3_attention_weight_min": -87.27079772949219,
      "activations/layer4_attention_weight_max": 79.96195983886719,
      "activations/layer4_attention_weight_min": -78.340087890625,
      "activations/layer5_attention_weight_max": 57.366432189941406,
      "activations/layer5_attention_weight_min": -68.16120910644531,
      "activations/layer6_attention_weight_max": 47.18048858642578,
      "activations/layer6_attention_weight_min": -51.91157150268555,
      "activations/layer7_attention_weight_max": 60.86223602294922,
      "activations/layer7_attention_weight_min": -63.706138610839844,
      "activations/layer8_attention_weight_max": 44.85615921020508,
      "activations/layer8_attention_weight_min": -46.68354415893555,
      "activations/layer9_attention_weight_max": 43.08673858642578,
      "activations/layer9_attention_weight_min": -43.10239791870117,
      "epoch": 19.34,
      "learning_rate": 2.5556439393939392e-05,
      "loss": 2.7244,
      "step": 332800
    },
    {
      "activations/layer0_attention_weight_max": 15.333203315734863,
      "activations/layer0_attention_weight_min": -14.3252534866333,
      "activations/layer10_attention_weight_max": 35.345458984375,
      "activations/layer10_attention_weight_min": -35.551612854003906,
      "activations/layer11_attention_weight_max": 33.7056884765625,
      "activations/layer11_attention_weight_min": -35.537818908691406,
      "activations/layer12_attention_weight_max": 19.150442123413086,
      "activations/layer12_attention_weight_min": -25.07713508605957,
      "activations/layer13_attention_weight_max": 41.68227005004883,
      "activations/layer13_attention_weight_min": -27.744998931884766,
      "activations/layer14_attention_weight_max": 33.290283203125,
      "activations/layer14_attention_weight_min": -29.671783447265625,
      "activations/layer15_attention_weight_max": 32.32902145385742,
      "activations/layer15_attention_weight_min": -31.645538330078125,
      "activations/layer16_attention_weight_max": 34.4119758605957,
      "activations/layer16_attention_weight_min": -33.166534423828125,
      "activations/layer17_attention_weight_max": 49.374900817871094,
      "activations/layer17_attention_weight_min": -45.178375244140625,
      "activations/layer18_attention_weight_max": 45.643455505371094,
      "activations/layer18_attention_weight_min": -43.620445251464844,
      "activations/layer19_attention_weight_max": 24.016908645629883,
      "activations/layer19_attention_weight_min": -24.256010055541992,
      "activations/layer1_attention_weight_max": 15.049015998840332,
      "activations/layer1_attention_weight_min": -16.139911651611328,
      "activations/layer20_attention_weight_max": 23.032194137573242,
      "activations/layer20_attention_weight_min": -23.198057174682617,
      "activations/layer21_attention_weight_max": 32.8245849609375,
      "activations/layer21_attention_weight_min": -23.464752197265625,
      "activations/layer22_attention_weight_max": 29.415185928344727,
      "activations/layer22_attention_weight_min": -26.38948631286621,
      "activations/layer23_attention_weight_max": 35.15144729614258,
      "activations/layer23_attention_weight_min": -26.149829864501953,
      "activations/layer2_attention_weight_max": 32.237457275390625,
      "activations/layer2_attention_weight_min": -29.549510955810547,
      "activations/layer3_attention_weight_max": 88.40770721435547,
      "activations/layer3_attention_weight_min": -84.7604751586914,
      "activations/layer4_attention_weight_max": 85.83080291748047,
      "activations/layer4_attention_weight_min": -78.58759307861328,
      "activations/layer5_attention_weight_max": 62.17741394042969,
      "activations/layer5_attention_weight_min": -76.80294799804688,
      "activations/layer6_attention_weight_max": 50.14691925048828,
      "activations/layer6_attention_weight_min": -53.37688446044922,
      "activations/layer7_attention_weight_max": 65.8004379272461,
      "activations/layer7_attention_weight_min": -67.16380310058594,
      "activations/layer8_attention_weight_max": 47.113441467285156,
      "activations/layer8_attention_weight_min": -49.9521598815918,
      "activations/layer9_attention_weight_max": 44.95223617553711,
      "activations/layer9_attention_weight_min": -45.977256774902344,
      "epoch": 19.34,
      "learning_rate": 2.55375e-05,
      "loss": 2.7251,
      "step": 332850
    },
    {
      "activations/layer0_attention_weight_max": 16.33304786682129,
      "activations/layer0_attention_weight_min": -14.254731178283691,
      "activations/layer10_attention_weight_max": 34.86640930175781,
      "activations/layer10_attention_weight_min": -35.853843688964844,
      "activations/layer11_attention_weight_max": 33.0176887512207,
      "activations/layer11_attention_weight_min": -33.55563735961914,
      "activations/layer12_attention_weight_max": 19.03299903869629,
      "activations/layer12_attention_weight_min": -35.04985427856445,
      "activations/layer13_attention_weight_max": 32.69610595703125,
      "activations/layer13_attention_weight_min": -25.17432403564453,
      "activations/layer14_attention_weight_max": 33.14594650268555,
      "activations/layer14_attention_weight_min": -29.038114547729492,
      "activations/layer15_attention_weight_max": 31.249174118041992,
      "activations/layer15_attention_weight_min": -27.296905517578125,
      "activations/layer16_attention_weight_max": 32.818721771240234,
      "activations/layer16_attention_weight_min": -30.714473724365234,
      "activations/layer17_attention_weight_max": 46.8679084777832,
      "activations/layer17_attention_weight_min": -43.28117752075195,
      "activations/layer18_attention_weight_max": 44.105072021484375,
      "activations/layer18_attention_weight_min": -38.008636474609375,
      "activations/layer19_attention_weight_max": 22.42011260986328,
      "activations/layer19_attention_weight_min": -21.13652229309082,
      "activations/layer1_attention_weight_max": 15.738912582397461,
      "activations/layer1_attention_weight_min": -12.975732803344727,
      "activations/layer20_attention_weight_max": 21.82750129699707,
      "activations/layer20_attention_weight_min": -19.072690963745117,
      "activations/layer21_attention_weight_max": 29.723384857177734,
      "activations/layer21_attention_weight_min": -19.104816436767578,
      "activations/layer22_attention_weight_max": 28.695960998535156,
      "activations/layer22_attention_weight_min": -27.363861083984375,
      "activations/layer23_attention_weight_max": 33.50497055053711,
      "activations/layer23_attention_weight_min": -22.841909408569336,
      "activations/layer2_attention_weight_max": 30.321317672729492,
      "activations/layer2_attention_weight_min": -30.522037506103516,
      "activations/layer3_attention_weight_max": 85.02421569824219,
      "activations/layer3_attention_weight_min": -88.12936401367188,
      "activations/layer4_attention_weight_max": 80.03316497802734,
      "activations/layer4_attention_weight_min": -81.15953826904297,
      "activations/layer5_attention_weight_max": 61.34759521484375,
      "activations/layer5_attention_weight_min": -70.00137329101562,
      "activations/layer6_attention_weight_max": 48.120113372802734,
      "activations/layer6_attention_weight_min": -49.175960540771484,
      "activations/layer7_attention_weight_max": 61.76322555541992,
      "activations/layer7_attention_weight_min": -62.31316375732422,
      "activations/layer8_attention_weight_max": 46.66243362426758,
      "activations/layer8_attention_weight_min": -48.98139572143555,
      "activations/layer9_attention_weight_max": 45.688785552978516,
      "activations/layer9_attention_weight_min": -44.01814270019531,
      "epoch": 19.34,
      "learning_rate": 2.5518560606060602e-05,
      "loss": 2.7316,
      "step": 332900
    },
    {
      "activations/layer0_attention_weight_max": 15.542798042297363,
      "activations/layer0_attention_weight_min": -14.291722297668457,
      "activations/layer10_attention_weight_max": 36.45244598388672,
      "activations/layer10_attention_weight_min": -36.09794616699219,
      "activations/layer11_attention_weight_max": 34.06767272949219,
      "activations/layer11_attention_weight_min": -33.59693145751953,
      "activations/layer12_attention_weight_max": 19.738693237304688,
      "activations/layer12_attention_weight_min": -24.572935104370117,
      "activations/layer13_attention_weight_max": 32.484954833984375,
      "activations/layer13_attention_weight_min": -30.83579444885254,
      "activations/layer14_attention_weight_max": 32.82994842529297,
      "activations/layer14_attention_weight_min": -31.575395584106445,
      "activations/layer15_attention_weight_max": 32.50218200683594,
      "activations/layer15_attention_weight_min": -29.448564529418945,
      "activations/layer16_attention_weight_max": 34.8095817565918,
      "activations/layer16_attention_weight_min": -31.744678497314453,
      "activations/layer17_attention_weight_max": 50.18851089477539,
      "activations/layer17_attention_weight_min": -43.425559997558594,
      "activations/layer18_attention_weight_max": 45.55069351196289,
      "activations/layer18_attention_weight_min": -40.79509353637695,
      "activations/layer19_attention_weight_max": 22.901681900024414,
      "activations/layer19_attention_weight_min": -23.258127212524414,
      "activations/layer1_attention_weight_max": 15.062260627746582,
      "activations/layer1_attention_weight_min": -13.772522926330566,
      "activations/layer20_attention_weight_max": 23.026159286499023,
      "activations/layer20_attention_weight_min": -21.707355499267578,
      "activations/layer21_attention_weight_max": 34.6912956237793,
      "activations/layer21_attention_weight_min": -24.038888931274414,
      "activations/layer22_attention_weight_max": 32.09602355957031,
      "activations/layer22_attention_weight_min": -24.598752975463867,
      "activations/layer23_attention_weight_max": 34.999244689941406,
      "activations/layer23_attention_weight_min": -27.571529388427734,
      "activations/layer2_attention_weight_max": 32.06175231933594,
      "activations/layer2_attention_weight_min": -30.55299186706543,
      "activations/layer3_attention_weight_max": 84.481689453125,
      "activations/layer3_attention_weight_min": -86.01515197753906,
      "activations/layer4_attention_weight_max": 83.32671356201172,
      "activations/layer4_attention_weight_min": -81.3238754272461,
      "activations/layer5_attention_weight_max": 59.68482971191406,
      "activations/layer5_attention_weight_min": -71.34721374511719,
      "activations/layer6_attention_weight_max": 50.24394607543945,
      "activations/layer6_attention_weight_min": -51.19255065917969,
      "activations/layer7_attention_weight_max": 66.01570129394531,
      "activations/layer7_attention_weight_min": -65.1621322631836,
      "activations/layer8_attention_weight_max": 47.33292770385742,
      "activations/layer8_attention_weight_min": -48.63409423828125,
      "activations/layer9_attention_weight_max": 43.877357482910156,
      "activations/layer9_attention_weight_min": -45.92976760864258,
      "epoch": 19.35,
      "learning_rate": 2.549962121212121e-05,
      "loss": 2.7298,
      "step": 332950
    },
    {
      "activations/layer0_attention_weight_max": 15.687002182006836,
      "activations/layer0_attention_weight_min": -14.455881118774414,
      "activations/layer10_attention_weight_max": 38.983158111572266,
      "activations/layer10_attention_weight_min": -36.90754318237305,
      "activations/layer11_attention_weight_max": 36.565147399902344,
      "activations/layer11_attention_weight_min": -35.606319427490234,
      "activations/layer12_attention_weight_max": 20.108030319213867,
      "activations/layer12_attention_weight_min": -22.637393951416016,
      "activations/layer13_attention_weight_max": 36.99602508544922,
      "activations/layer13_attention_weight_min": -29.08664321899414,
      "activations/layer14_attention_weight_max": 37.28661346435547,
      "activations/layer14_attention_weight_min": -33.49711227416992,
      "activations/layer15_attention_weight_max": 34.19982147216797,
      "activations/layer15_attention_weight_min": -31.660127639770508,
      "activations/layer16_attention_weight_max": 33.123451232910156,
      "activations/layer16_attention_weight_min": -31.996963500976562,
      "activations/layer17_attention_weight_max": 51.2374267578125,
      "activations/layer17_attention_weight_min": -45.9725227355957,
      "activations/layer18_attention_weight_max": 46.344390869140625,
      "activations/layer18_attention_weight_min": -41.13200378417969,
      "activations/layer19_attention_weight_max": 23.714096069335938,
      "activations/layer19_attention_weight_min": -23.630022048950195,
      "activations/layer1_attention_weight_max": 15.68387508392334,
      "activations/layer1_attention_weight_min": -13.274688720703125,
      "activations/layer20_attention_weight_max": 23.03768539428711,
      "activations/layer20_attention_weight_min": -20.84891128540039,
      "activations/layer21_attention_weight_max": 40.23053741455078,
      "activations/layer21_attention_weight_min": -20.609731674194336,
      "activations/layer22_attention_weight_max": 30.4226131439209,
      "activations/layer22_attention_weight_min": -25.454723358154297,
      "activations/layer23_attention_weight_max": 39.84193420410156,
      "activations/layer23_attention_weight_min": -23.83266830444336,
      "activations/layer2_attention_weight_max": 30.368762969970703,
      "activations/layer2_attention_weight_min": -29.97321128845215,
      "activations/layer3_attention_weight_max": 87.45491027832031,
      "activations/layer3_attention_weight_min": -85.2564697265625,
      "activations/layer4_attention_weight_max": 79.92881774902344,
      "activations/layer4_attention_weight_min": -77.5361557006836,
      "activations/layer5_attention_weight_max": 63.5915641784668,
      "activations/layer5_attention_weight_min": -71.44070434570312,
      "activations/layer6_attention_weight_max": 48.84896469116211,
      "activations/layer6_attention_weight_min": -49.140323638916016,
      "activations/layer7_attention_weight_max": 64.97406768798828,
      "activations/layer7_attention_weight_min": -64.24344635009766,
      "activations/layer8_attention_weight_max": 49.71025466918945,
      "activations/layer8_attention_weight_min": -51.428245544433594,
      "activations/layer9_attention_weight_max": 45.9954948425293,
      "activations/layer9_attention_weight_min": -46.99435043334961,
      "epoch": 19.35,
      "learning_rate": 2.5480681818181815e-05,
      "loss": 2.7272,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.5565,
      "eval_samples_per_second": 501.84,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.5565,
      "eval_openwebtext_samples_per_second": 501.84,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 2.0347,
      "eval_wikitext_samples_per_second": 224.116,
      "step": 333000
    },
    {
      "epoch": 19.35,
      "eval_lambada_loss": 2.4453125,
      "eval_lambada_ppl": 11.534153458662256,
      "eval_lambada_runtime": 9.6181,
      "eval_lambada_samples_per_second": 506.235,
      "step": 333000
    },
    {
      "activations/layer0_attention_weight_max": 15.524419784545898,
      "activations/layer0_attention_weight_min": -14.318334579467773,
      "activations/layer10_attention_weight_max": 35.74675369262695,
      "activations/layer10_attention_weight_min": -34.526275634765625,
      "activations/layer11_attention_weight_max": 33.143943786621094,
      "activations/layer11_attention_weight_min": -34.39888000488281,
      "activations/layer12_attention_weight_max": 18.86002540588379,
      "activations/layer12_attention_weight_min": -25.55779457092285,
      "activations/layer13_attention_weight_max": 34.23735809326172,
      "activations/layer13_attention_weight_min": -29.388505935668945,
      "activations/layer14_attention_weight_max": 36.26237487792969,
      "activations/layer14_attention_weight_min": -29.944259643554688,
      "activations/layer15_attention_weight_max": 33.84638595581055,
      "activations/layer15_attention_weight_min": -28.2366943359375,
      "activations/layer16_attention_weight_max": 32.99327850341797,
      "activations/layer16_attention_weight_min": -32.018741607666016,
      "activations/layer17_attention_weight_max": 49.62744140625,
      "activations/layer17_attention_weight_min": -44.68894577026367,
      "activations/layer18_attention_weight_max": 44.64492416381836,
      "activations/layer18_attention_weight_min": -36.35770797729492,
      "activations/layer19_attention_weight_max": 23.933029174804688,
      "activations/layer19_attention_weight_min": -20.686504364013672,
      "activations/layer1_attention_weight_max": 16.056655883789062,
      "activations/layer1_attention_weight_min": -12.673652648925781,
      "activations/layer20_attention_weight_max": 22.517854690551758,
      "activations/layer20_attention_weight_min": -20.766857147216797,
      "activations/layer21_attention_weight_max": 34.30213165283203,
      "activations/layer21_attention_weight_min": -20.062946319580078,
      "activations/layer22_attention_weight_max": 28.94484519958496,
      "activations/layer22_attention_weight_min": -25.113672256469727,
      "activations/layer23_attention_weight_max": 38.3260498046875,
      "activations/layer23_attention_weight_min": -22.75330924987793,
      "activations/layer2_attention_weight_max": 30.723722457885742,
      "activations/layer2_attention_weight_min": -28.683523178100586,
      "activations/layer3_attention_weight_max": 87.63684844970703,
      "activations/layer3_attention_weight_min": -79.60477447509766,
      "activations/layer4_attention_weight_max": 80.2855453491211,
      "activations/layer4_attention_weight_min": -79.73473358154297,
      "activations/layer5_attention_weight_max": 63.86248016357422,
      "activations/layer5_attention_weight_min": -69.93193054199219,
      "activations/layer6_attention_weight_max": 50.35813903808594,
      "activations/layer6_attention_weight_min": -50.45307159423828,
      "activations/layer7_attention_weight_max": 67.0053482055664,
      "activations/layer7_attention_weight_min": -64.68529510498047,
      "activations/layer8_attention_weight_max": 46.72871398925781,
      "activations/layer8_attention_weight_min": -52.05248260498047,
      "activations/layer9_attention_weight_max": 41.08487319946289,
      "activations/layer9_attention_weight_min": -46.9709587097168,
      "epoch": 19.35,
      "learning_rate": 2.5461742424242424e-05,
      "loss": 2.711,
      "step": 333050
    },
    {
      "activations/layer0_attention_weight_max": 15.27768611907959,
      "activations/layer0_attention_weight_min": -14.551539421081543,
      "activations/layer10_attention_weight_max": 35.66810607910156,
      "activations/layer10_attention_weight_min": -35.701847076416016,
      "activations/layer11_attention_weight_max": 32.5792236328125,
      "activations/layer11_attention_weight_min": -33.267940521240234,
      "activations/layer12_attention_weight_max": 18.88572120666504,
      "activations/layer12_attention_weight_min": -24.040327072143555,
      "activations/layer13_attention_weight_max": 30.35498809814453,
      "activations/layer13_attention_weight_min": -28.209854125976562,
      "activations/layer14_attention_weight_max": 34.312278747558594,
      "activations/layer14_attention_weight_min": -29.6168212890625,
      "activations/layer15_attention_weight_max": 32.943572998046875,
      "activations/layer15_attention_weight_min": -31.807804107666016,
      "activations/layer16_attention_weight_max": 33.73091125488281,
      "activations/layer16_attention_weight_min": -35.572322845458984,
      "activations/layer17_attention_weight_max": 51.77558898925781,
      "activations/layer17_attention_weight_min": -46.93122863769531,
      "activations/layer18_attention_weight_max": 48.74468994140625,
      "activations/layer18_attention_weight_min": -44.14756393432617,
      "activations/layer19_attention_weight_max": 23.895465850830078,
      "activations/layer19_attention_weight_min": -23.714866638183594,
      "activations/layer1_attention_weight_max": 15.300389289855957,
      "activations/layer1_attention_weight_min": -11.944214820861816,
      "activations/layer20_attention_weight_max": 23.839473724365234,
      "activations/layer20_attention_weight_min": -21.822797775268555,
      "activations/layer21_attention_weight_max": 37.63496017456055,
      "activations/layer21_attention_weight_min": -22.714052200317383,
      "activations/layer22_attention_weight_max": 30.60698699951172,
      "activations/layer22_attention_weight_min": -26.82054901123047,
      "activations/layer23_attention_weight_max": 37.046104431152344,
      "activations/layer23_attention_weight_min": -23.734634399414062,
      "activations/layer2_attention_weight_max": 29.2122802734375,
      "activations/layer2_attention_weight_min": -29.087881088256836,
      "activations/layer3_attention_weight_max": 87.3770523071289,
      "activations/layer3_attention_weight_min": -81.62224578857422,
      "activations/layer4_attention_weight_max": 80.97205352783203,
      "activations/layer4_attention_weight_min": -78.35060119628906,
      "activations/layer5_attention_weight_max": 64.48875427246094,
      "activations/layer5_attention_weight_min": -69.77423095703125,
      "activations/layer6_attention_weight_max": 48.67338943481445,
      "activations/layer6_attention_weight_min": -50.473331451416016,
      "activations/layer7_attention_weight_max": 61.1179084777832,
      "activations/layer7_attention_weight_min": -64.14537811279297,
      "activations/layer8_attention_weight_max": 48.169132232666016,
      "activations/layer8_attention_weight_min": -51.39849090576172,
      "activations/layer9_attention_weight_max": 43.85923385620117,
      "activations/layer9_attention_weight_min": -46.78879928588867,
      "epoch": 19.36,
      "learning_rate": 2.544280303030303e-05,
      "loss": 2.7287,
      "step": 333100
    },
    {
      "activations/layer0_attention_weight_max": 15.144651412963867,
      "activations/layer0_attention_weight_min": -14.337109565734863,
      "activations/layer10_attention_weight_max": 38.4168701171875,
      "activations/layer10_attention_weight_min": -38.577667236328125,
      "activations/layer11_attention_weight_max": 35.01679611206055,
      "activations/layer11_attention_weight_min": -36.491790771484375,
      "activations/layer12_attention_weight_max": 18.940092086791992,
      "activations/layer12_attention_weight_min": -23.769683837890625,
      "activations/layer13_attention_weight_max": 30.97654914855957,
      "activations/layer13_attention_weight_min": -26.85878562927246,
      "activations/layer14_attention_weight_max": 33.98987579345703,
      "activations/layer14_attention_weight_min": -29.179231643676758,
      "activations/layer15_attention_weight_max": 29.898168563842773,
      "activations/layer15_attention_weight_min": -29.425151824951172,
      "activations/layer16_attention_weight_max": 32.05039978027344,
      "activations/layer16_attention_weight_min": -29.85687255859375,
      "activations/layer17_attention_weight_max": 47.75871658325195,
      "activations/layer17_attention_weight_min": -42.88109588623047,
      "activations/layer18_attention_weight_max": 40.41572952270508,
      "activations/layer18_attention_weight_min": -38.214019775390625,
      "activations/layer19_attention_weight_max": 22.23362922668457,
      "activations/layer19_attention_weight_min": -22.94795036315918,
      "activations/layer1_attention_weight_max": 15.561239242553711,
      "activations/layer1_attention_weight_min": -13.057271957397461,
      "activations/layer20_attention_weight_max": 20.85030746459961,
      "activations/layer20_attention_weight_min": -21.458433151245117,
      "activations/layer21_attention_weight_max": 33.36099624633789,
      "activations/layer21_attention_weight_min": -21.84433364868164,
      "activations/layer22_attention_weight_max": 27.01186752319336,
      "activations/layer22_attention_weight_min": -26.61859703063965,
      "activations/layer23_attention_weight_max": 37.75007247924805,
      "activations/layer23_attention_weight_min": -24.748371124267578,
      "activations/layer2_attention_weight_max": 29.260414123535156,
      "activations/layer2_attention_weight_min": -27.501220703125,
      "activations/layer3_attention_weight_max": 79.53008270263672,
      "activations/layer3_attention_weight_min": -79.8991470336914,
      "activations/layer4_attention_weight_max": 77.32250213623047,
      "activations/layer4_attention_weight_min": -79.10516357421875,
      "activations/layer5_attention_weight_max": 58.614845275878906,
      "activations/layer5_attention_weight_min": -68.65502166748047,
      "activations/layer6_attention_weight_max": 46.8978385925293,
      "activations/layer6_attention_weight_min": -48.38441467285156,
      "activations/layer7_attention_weight_max": 60.49156951904297,
      "activations/layer7_attention_weight_min": -59.96711730957031,
      "activations/layer8_attention_weight_max": 43.946868896484375,
      "activations/layer8_attention_weight_min": -49.15983200073242,
      "activations/layer9_attention_weight_max": 42.13465118408203,
      "activations/layer9_attention_weight_min": -45.74469757080078,
      "epoch": 19.36,
      "learning_rate": 2.5423863636363637e-05,
      "loss": 2.736,
      "step": 333150
    },
    {
      "activations/layer0_attention_weight_max": 15.480737686157227,
      "activations/layer0_attention_weight_min": -14.054845809936523,
      "activations/layer10_attention_weight_max": 36.04843521118164,
      "activations/layer10_attention_weight_min": -36.61591339111328,
      "activations/layer11_attention_weight_max": 34.01925277709961,
      "activations/layer11_attention_weight_min": -37.184722900390625,
      "activations/layer12_attention_weight_max": 17.363880157470703,
      "activations/layer12_attention_weight_min": -26.7727108001709,
      "activations/layer13_attention_weight_max": 32.81395721435547,
      "activations/layer13_attention_weight_min": -25.88929557800293,
      "activations/layer14_attention_weight_max": 34.690711975097656,
      "activations/layer14_attention_weight_min": -28.52913475036621,
      "activations/layer15_attention_weight_max": 31.65382957458496,
      "activations/layer15_attention_weight_min": -29.846174240112305,
      "activations/layer16_attention_weight_max": 32.47172546386719,
      "activations/layer16_attention_weight_min": -31.20012092590332,
      "activations/layer17_attention_weight_max": 47.80192565917969,
      "activations/layer17_attention_weight_min": -42.35757064819336,
      "activations/layer18_attention_weight_max": 46.49525833129883,
      "activations/layer18_attention_weight_min": -37.833953857421875,
      "activations/layer19_attention_weight_max": 22.822969436645508,
      "activations/layer19_attention_weight_min": -23.11008071899414,
      "activations/layer1_attention_weight_max": 15.571039199829102,
      "activations/layer1_attention_weight_min": -13.7438325881958,
      "activations/layer20_attention_weight_max": 22.45330047607422,
      "activations/layer20_attention_weight_min": -21.373746871948242,
      "activations/layer21_attention_weight_max": 35.59626770019531,
      "activations/layer21_attention_weight_min": -21.58795738220215,
      "activations/layer22_attention_weight_max": 26.481918334960938,
      "activations/layer22_attention_weight_min": -24.66458511352539,
      "activations/layer23_attention_weight_max": 36.990211486816406,
      "activations/layer23_attention_weight_min": -23.639984130859375,
      "activations/layer2_attention_weight_max": 31.706363677978516,
      "activations/layer2_attention_weight_min": -30.496746063232422,
      "activations/layer3_attention_weight_max": 86.07809448242188,
      "activations/layer3_attention_weight_min": -83.8497085571289,
      "activations/layer4_attention_weight_max": 81.67218780517578,
      "activations/layer4_attention_weight_min": -83.3991470336914,
      "activations/layer5_attention_weight_max": 62.690635681152344,
      "activations/layer5_attention_weight_min": -73.01337432861328,
      "activations/layer6_attention_weight_max": 51.394840240478516,
      "activations/layer6_attention_weight_min": -55.222686767578125,
      "activations/layer7_attention_weight_max": 66.0669174194336,
      "activations/layer7_attention_weight_min": -67.66709899902344,
      "activations/layer8_attention_weight_max": 46.79465866088867,
      "activations/layer8_attention_weight_min": -50.54812240600586,
      "activations/layer9_attention_weight_max": 44.513946533203125,
      "activations/layer9_attention_weight_min": -46.71061325073242,
      "epoch": 19.36,
      "learning_rate": 2.5404924242424238e-05,
      "loss": 2.7248,
      "step": 333200
    },
    {
      "activations/layer0_attention_weight_max": 14.966726303100586,
      "activations/layer0_attention_weight_min": -13.927145004272461,
      "activations/layer10_attention_weight_max": 34.59196853637695,
      "activations/layer10_attention_weight_min": -34.42787170410156,
      "activations/layer11_attention_weight_max": 32.19744110107422,
      "activations/layer11_attention_weight_min": -34.594688415527344,
      "activations/layer12_attention_weight_max": 18.658588409423828,
      "activations/layer12_attention_weight_min": -24.14421272277832,
      "activations/layer13_attention_weight_max": 35.08952713012695,
      "activations/layer13_attention_weight_min": -27.570688247680664,
      "activations/layer14_attention_weight_max": 33.850887298583984,
      "activations/layer14_attention_weight_min": -30.470802307128906,
      "activations/layer15_attention_weight_max": 31.63796043395996,
      "activations/layer15_attention_weight_min": -29.895660400390625,
      "activations/layer16_attention_weight_max": 32.888893127441406,
      "activations/layer16_attention_weight_min": -31.747512817382812,
      "activations/layer17_attention_weight_max": 50.8719367980957,
      "activations/layer17_attention_weight_min": -42.73973083496094,
      "activations/layer18_attention_weight_max": 44.06326675415039,
      "activations/layer18_attention_weight_min": -39.642059326171875,
      "activations/layer19_attention_weight_max": 21.109941482543945,
      "activations/layer19_attention_weight_min": -22.572967529296875,
      "activations/layer1_attention_weight_max": 15.418238639831543,
      "activations/layer1_attention_weight_min": -13.60688591003418,
      "activations/layer20_attention_weight_max": 23.48613166809082,
      "activations/layer20_attention_weight_min": -20.735116958618164,
      "activations/layer21_attention_weight_max": 32.6657600402832,
      "activations/layer21_attention_weight_min": -20.958778381347656,
      "activations/layer22_attention_weight_max": 30.922279357910156,
      "activations/layer22_attention_weight_min": -24.289949417114258,
      "activations/layer23_attention_weight_max": 38.04650115966797,
      "activations/layer23_attention_weight_min": -23.852989196777344,
      "activations/layer2_attention_weight_max": 30.100406646728516,
      "activations/layer2_attention_weight_min": -28.604032516479492,
      "activations/layer3_attention_weight_max": 85.38996124267578,
      "activations/layer3_attention_weight_min": -85.38494873046875,
      "activations/layer4_attention_weight_max": 81.56107330322266,
      "activations/layer4_attention_weight_min": -82.59779357910156,
      "activations/layer5_attention_weight_max": 60.45292282104492,
      "activations/layer5_attention_weight_min": -70.96381378173828,
      "activations/layer6_attention_weight_max": 48.69248962402344,
      "activations/layer6_attention_weight_min": -50.16413116455078,
      "activations/layer7_attention_weight_max": 62.439697265625,
      "activations/layer7_attention_weight_min": -63.818668365478516,
      "activations/layer8_attention_weight_max": 48.2064208984375,
      "activations/layer8_attention_weight_min": -48.8458366394043,
      "activations/layer9_attention_weight_max": 43.03034210205078,
      "activations/layer9_attention_weight_min": -45.65770721435547,
      "epoch": 19.36,
      "learning_rate": 2.5385984848484847e-05,
      "loss": 2.7452,
      "step": 333250
    },
    {
      "activations/layer0_attention_weight_max": 14.8954439163208,
      "activations/layer0_attention_weight_min": -13.96165657043457,
      "activations/layer10_attention_weight_max": 34.046512603759766,
      "activations/layer10_attention_weight_min": -37.48874282836914,
      "activations/layer11_attention_weight_max": 31.781042098999023,
      "activations/layer11_attention_weight_min": -34.61568069458008,
      "activations/layer12_attention_weight_max": 18.095670700073242,
      "activations/layer12_attention_weight_min": -25.232620239257812,
      "activations/layer13_attention_weight_max": 36.774654388427734,
      "activations/layer13_attention_weight_min": -27.982439041137695,
      "activations/layer14_attention_weight_max": 36.10619354248047,
      "activations/layer14_attention_weight_min": -31.5969295501709,
      "activations/layer15_attention_weight_max": 33.349430084228516,
      "activations/layer15_attention_weight_min": -33.31205749511719,
      "activations/layer16_attention_weight_max": 36.2850227355957,
      "activations/layer16_attention_weight_min": -34.7624397277832,
      "activations/layer17_attention_weight_max": 54.90120315551758,
      "activations/layer17_attention_weight_min": -48.72343826293945,
      "activations/layer18_attention_weight_max": 49.765960693359375,
      "activations/layer18_attention_weight_min": -42.01920700073242,
      "activations/layer19_attention_weight_max": 23.590665817260742,
      "activations/layer19_attention_weight_min": -23.78099250793457,
      "activations/layer1_attention_weight_max": 15.761405944824219,
      "activations/layer1_attention_weight_min": -14.065241813659668,
      "activations/layer20_attention_weight_max": 24.09194564819336,
      "activations/layer20_attention_weight_min": -20.124418258666992,
      "activations/layer21_attention_weight_max": 34.82346725463867,
      "activations/layer21_attention_weight_min": -20.68919563293457,
      "activations/layer22_attention_weight_max": 27.10491943359375,
      "activations/layer22_attention_weight_min": -25.151140213012695,
      "activations/layer23_attention_weight_max": 36.21451950073242,
      "activations/layer23_attention_weight_min": -23.830062866210938,
      "activations/layer2_attention_weight_max": 30.54030418395996,
      "activations/layer2_attention_weight_min": -28.8585262298584,
      "activations/layer3_attention_weight_max": 84.5369873046875,
      "activations/layer3_attention_weight_min": -83.17903900146484,
      "activations/layer4_attention_weight_max": 77.40994262695312,
      "activations/layer4_attention_weight_min": -78.99185943603516,
      "activations/layer5_attention_weight_max": 61.25697326660156,
      "activations/layer5_attention_weight_min": -72.11713409423828,
      "activations/layer6_attention_weight_max": 49.092018127441406,
      "activations/layer6_attention_weight_min": -49.42567443847656,
      "activations/layer7_attention_weight_max": 64.45643615722656,
      "activations/layer7_attention_weight_min": -65.62045288085938,
      "activations/layer8_attention_weight_max": 44.50788497924805,
      "activations/layer8_attention_weight_min": -50.58417510986328,
      "activations/layer9_attention_weight_max": 42.504268646240234,
      "activations/layer9_attention_weight_min": -46.084877014160156,
      "epoch": 19.37,
      "learning_rate": 2.536704545454545e-05,
      "loss": 2.7067,
      "step": 333300
    },
    {
      "activations/layer0_attention_weight_max": 14.840606689453125,
      "activations/layer0_attention_weight_min": -13.89734172821045,
      "activations/layer10_attention_weight_max": 35.90441131591797,
      "activations/layer10_attention_weight_min": -35.13418197631836,
      "activations/layer11_attention_weight_max": 32.35663604736328,
      "activations/layer11_attention_weight_min": -32.86390686035156,
      "activations/layer12_attention_weight_max": 19.036924362182617,
      "activations/layer12_attention_weight_min": -23.48089599609375,
      "activations/layer13_attention_weight_max": 32.045467376708984,
      "activations/layer13_attention_weight_min": -26.971065521240234,
      "activations/layer14_attention_weight_max": 37.75260543823242,
      "activations/layer14_attention_weight_min": -32.734779357910156,
      "activations/layer15_attention_weight_max": 34.177772521972656,
      "activations/layer15_attention_weight_min": -32.02899169921875,
      "activations/layer16_attention_weight_max": 38.498558044433594,
      "activations/layer16_attention_weight_min": -35.92934036254883,
      "activations/layer17_attention_weight_max": 58.14241409301758,
      "activations/layer17_attention_weight_min": -53.53707504272461,
      "activations/layer18_attention_weight_max": 55.493247985839844,
      "activations/layer18_attention_weight_min": -45.00388717651367,
      "activations/layer19_attention_weight_max": 27.063644409179688,
      "activations/layer19_attention_weight_min": -26.848691940307617,
      "activations/layer1_attention_weight_max": 15.600860595703125,
      "activations/layer1_attention_weight_min": -13.53780746459961,
      "activations/layer20_attention_weight_max": 25.5936222076416,
      "activations/layer20_attention_weight_min": -24.866592407226562,
      "activations/layer21_attention_weight_max": 46.939693450927734,
      "activations/layer21_attention_weight_min": -22.730958938598633,
      "activations/layer22_attention_weight_max": 29.607725143432617,
      "activations/layer22_attention_weight_min": -26.39965057373047,
      "activations/layer23_attention_weight_max": 37.43693161010742,
      "activations/layer23_attention_weight_min": -25.75635528564453,
      "activations/layer2_attention_weight_max": 30.899978637695312,
      "activations/layer2_attention_weight_min": -29.717796325683594,
      "activations/layer3_attention_weight_max": 87.83065032958984,
      "activations/layer3_attention_weight_min": -91.36241149902344,
      "activations/layer4_attention_weight_max": 82.30826568603516,
      "activations/layer4_attention_weight_min": -83.41790008544922,
      "activations/layer5_attention_weight_max": 57.640438079833984,
      "activations/layer5_attention_weight_min": -71.71453094482422,
      "activations/layer6_attention_weight_max": 46.734310150146484,
      "activations/layer6_attention_weight_min": -47.594486236572266,
      "activations/layer7_attention_weight_max": 60.36787414550781,
      "activations/layer7_attention_weight_min": -63.04524230957031,
      "activations/layer8_attention_weight_max": 43.54957580566406,
      "activations/layer8_attention_weight_min": -47.323883056640625,
      "activations/layer9_attention_weight_max": 40.940879821777344,
      "activations/layer9_attention_weight_min": -45.69149398803711,
      "epoch": 19.37,
      "learning_rate": 2.534810606060606e-05,
      "loss": 2.7353,
      "step": 333350
    },
    {
      "activations/layer0_attention_weight_max": 15.767623901367188,
      "activations/layer0_attention_weight_min": -14.038958549499512,
      "activations/layer10_attention_weight_max": 37.606414794921875,
      "activations/layer10_attention_weight_min": -36.10976028442383,
      "activations/layer11_attention_weight_max": 36.071533203125,
      "activations/layer11_attention_weight_min": -33.542850494384766,
      "activations/layer12_attention_weight_max": 19.24814224243164,
      "activations/layer12_attention_weight_min": -26.75609588623047,
      "activations/layer13_attention_weight_max": 39.23012924194336,
      "activations/layer13_attention_weight_min": -29.13216781616211,
      "activations/layer14_attention_weight_max": 42.54549789428711,
      "activations/layer14_attention_weight_min": -31.370180130004883,
      "activations/layer15_attention_weight_max": 39.75112533569336,
      "activations/layer15_attention_weight_min": -30.165420532226562,
      "activations/layer16_attention_weight_max": 33.40856170654297,
      "activations/layer16_attention_weight_min": -32.27815628051758,
      "activations/layer17_attention_weight_max": 51.75422286987305,
      "activations/layer17_attention_weight_min": -44.15460205078125,
      "activations/layer18_attention_weight_max": 46.574546813964844,
      "activations/layer18_attention_weight_min": -39.947784423828125,
      "activations/layer19_attention_weight_max": 22.720869064331055,
      "activations/layer19_attention_weight_min": -21.510740280151367,
      "activations/layer1_attention_weight_max": 17.6719970703125,
      "activations/layer1_attention_weight_min": -13.858683586120605,
      "activations/layer20_attention_weight_max": 22.8415470123291,
      "activations/layer20_attention_weight_min": -21.776447296142578,
      "activations/layer21_attention_weight_max": 35.45112991333008,
      "activations/layer21_attention_weight_min": -23.903606414794922,
      "activations/layer22_attention_weight_max": 27.87999153137207,
      "activations/layer22_attention_weight_min": -25.27597427368164,
      "activations/layer23_attention_weight_max": 37.68100357055664,
      "activations/layer23_attention_weight_min": -27.531848907470703,
      "activations/layer2_attention_weight_max": 33.595863342285156,
      "activations/layer2_attention_weight_min": -31.518169403076172,
      "activations/layer3_attention_weight_max": 85.57149505615234,
      "activations/layer3_attention_weight_min": -85.00434875488281,
      "activations/layer4_attention_weight_max": 80.79387664794922,
      "activations/layer4_attention_weight_min": -80.32115936279297,
      "activations/layer5_attention_weight_max": 58.58793640136719,
      "activations/layer5_attention_weight_min": -76.35713958740234,
      "activations/layer6_attention_weight_max": 50.909278869628906,
      "activations/layer6_attention_weight_min": -51.023799896240234,
      "activations/layer7_attention_weight_max": 67.19212341308594,
      "activations/layer7_attention_weight_min": -65.00559997558594,
      "activations/layer8_attention_weight_max": 49.08635330200195,
      "activations/layer8_attention_weight_min": -51.19460678100586,
      "activations/layer9_attention_weight_max": 55.0355224609375,
      "activations/layer9_attention_weight_min": -45.41103744506836,
      "epoch": 19.37,
      "learning_rate": 2.5329166666666665e-05,
      "loss": 2.7109,
      "step": 333400
    },
    {
      "activations/layer0_attention_weight_max": 15.841837882995605,
      "activations/layer0_attention_weight_min": -14.024457931518555,
      "activations/layer10_attention_weight_max": 36.667972564697266,
      "activations/layer10_attention_weight_min": -34.2357063293457,
      "activations/layer11_attention_weight_max": 32.7547607421875,
      "activations/layer11_attention_weight_min": -33.74463653564453,
      "activations/layer12_attention_weight_max": 18.597286224365234,
      "activations/layer12_attention_weight_min": -26.228696823120117,
      "activations/layer13_attention_weight_max": 32.69984436035156,
      "activations/layer13_attention_weight_min": -28.2105712890625,
      "activations/layer14_attention_weight_max": 35.5013542175293,
      "activations/layer14_attention_weight_min": -31.023088455200195,
      "activations/layer15_attention_weight_max": 32.63157653808594,
      "activations/layer15_attention_weight_min": -30.281143188476562,
      "activations/layer16_attention_weight_max": 35.557777404785156,
      "activations/layer16_attention_weight_min": -34.03400421142578,
      "activations/layer17_attention_weight_max": 53.39213943481445,
      "activations/layer17_attention_weight_min": -47.92564010620117,
      "activations/layer18_attention_weight_max": 49.61479949951172,
      "activations/layer18_attention_weight_min": -43.771663665771484,
      "activations/layer19_attention_weight_max": 26.748098373413086,
      "activations/layer19_attention_weight_min": -24.748525619506836,
      "activations/layer1_attention_weight_max": 16.032024383544922,
      "activations/layer1_attention_weight_min": -14.194470405578613,
      "activations/layer20_attention_weight_max": 23.549823760986328,
      "activations/layer20_attention_weight_min": -23.74151039123535,
      "activations/layer21_attention_weight_max": 38.034297943115234,
      "activations/layer21_attention_weight_min": -21.791105270385742,
      "activations/layer22_attention_weight_max": 27.67853546142578,
      "activations/layer22_attention_weight_min": -25.12550163269043,
      "activations/layer23_attention_weight_max": 34.92860794067383,
      "activations/layer23_attention_weight_min": -21.572988510131836,
      "activations/layer2_attention_weight_max": 31.573768615722656,
      "activations/layer2_attention_weight_min": -27.919483184814453,
      "activations/layer3_attention_weight_max": 85.40400695800781,
      "activations/layer3_attention_weight_min": -80.86820983886719,
      "activations/layer4_attention_weight_max": 81.4401626586914,
      "activations/layer4_attention_weight_min": -77.94071960449219,
      "activations/layer5_attention_weight_max": 58.84178924560547,
      "activations/layer5_attention_weight_min": -66.83611297607422,
      "activations/layer6_attention_weight_max": 47.70431900024414,
      "activations/layer6_attention_weight_min": -47.574344635009766,
      "activations/layer7_attention_weight_max": 60.516624450683594,
      "activations/layer7_attention_weight_min": -62.92119216918945,
      "activations/layer8_attention_weight_max": 45.788543701171875,
      "activations/layer8_attention_weight_min": -50.44331741333008,
      "activations/layer9_attention_weight_max": 43.1296272277832,
      "activations/layer9_attention_weight_min": -44.5006217956543,
      "epoch": 19.38,
      "learning_rate": 2.531022727272727e-05,
      "loss": 2.7511,
      "step": 333450
    },
    {
      "activations/layer0_attention_weight_max": 15.25144100189209,
      "activations/layer0_attention_weight_min": -14.177696228027344,
      "activations/layer10_attention_weight_max": 35.99726104736328,
      "activations/layer10_attention_weight_min": -35.667816162109375,
      "activations/layer11_attention_weight_max": 32.04463577270508,
      "activations/layer11_attention_weight_min": -32.22959899902344,
      "activations/layer12_attention_weight_max": 17.394250869750977,
      "activations/layer12_attention_weight_min": -25.35484504699707,
      "activations/layer13_attention_weight_max": 30.60504722595215,
      "activations/layer13_attention_weight_min": -27.12964630126953,
      "activations/layer14_attention_weight_max": 33.85145950317383,
      "activations/layer14_attention_weight_min": -27.977359771728516,
      "activations/layer15_attention_weight_max": 32.43562316894531,
      "activations/layer15_attention_weight_min": -29.660837173461914,
      "activations/layer16_attention_weight_max": 33.65733337402344,
      "activations/layer16_attention_weight_min": -31.132108688354492,
      "activations/layer17_attention_weight_max": 48.38454055786133,
      "activations/layer17_attention_weight_min": -42.75929641723633,
      "activations/layer18_attention_weight_max": 47.76900100708008,
      "activations/layer18_attention_weight_min": -40.35023498535156,
      "activations/layer19_attention_weight_max": 24.79865074157715,
      "activations/layer19_attention_weight_min": -21.57831382751465,
      "activations/layer1_attention_weight_max": 15.070934295654297,
      "activations/layer1_attention_weight_min": -14.583927154541016,
      "activations/layer20_attention_weight_max": 23.143173217773438,
      "activations/layer20_attention_weight_min": -21.380943298339844,
      "activations/layer21_attention_weight_max": 40.96384048461914,
      "activations/layer21_attention_weight_min": -23.16347312927246,
      "activations/layer22_attention_weight_max": 28.014455795288086,
      "activations/layer22_attention_weight_min": -24.435380935668945,
      "activations/layer23_attention_weight_max": 39.37535858154297,
      "activations/layer23_attention_weight_min": -23.07738494873047,
      "activations/layer2_attention_weight_max": 31.621849060058594,
      "activations/layer2_attention_weight_min": -29.261425018310547,
      "activations/layer3_attention_weight_max": 85.20218658447266,
      "activations/layer3_attention_weight_min": -81.98506927490234,
      "activations/layer4_attention_weight_max": 79.03866577148438,
      "activations/layer4_attention_weight_min": -77.1798095703125,
      "activations/layer5_attention_weight_max": 62.28811264038086,
      "activations/layer5_attention_weight_min": -75.37260437011719,
      "activations/layer6_attention_weight_max": 45.7278938293457,
      "activations/layer6_attention_weight_min": -48.108097076416016,
      "activations/layer7_attention_weight_max": 63.89689636230469,
      "activations/layer7_attention_weight_min": -62.95754623413086,
      "activations/layer8_attention_weight_max": 42.41242218017578,
      "activations/layer8_attention_weight_min": -46.37617874145508,
      "activations/layer9_attention_weight_max": 43.65415954589844,
      "activations/layer9_attention_weight_min": -42.453609466552734,
      "epoch": 19.38,
      "learning_rate": 2.5291287878787874e-05,
      "loss": 2.7131,
      "step": 333500
    },
    {
      "activations/layer0_attention_weight_max": 15.464964866638184,
      "activations/layer0_attention_weight_min": -14.206445693969727,
      "activations/layer10_attention_weight_max": 36.697364807128906,
      "activations/layer10_attention_weight_min": -37.68195343017578,
      "activations/layer11_attention_weight_max": 32.59214782714844,
      "activations/layer11_attention_weight_min": -34.73029327392578,
      "activations/layer12_attention_weight_max": 18.349212646484375,
      "activations/layer12_attention_weight_min": -24.44196891784668,
      "activations/layer13_attention_weight_max": 29.8565616607666,
      "activations/layer13_attention_weight_min": -28.8382511138916,
      "activations/layer14_attention_weight_max": 34.38816833496094,
      "activations/layer14_attention_weight_min": -32.94239807128906,
      "activations/layer15_attention_weight_max": 33.90742492675781,
      "activations/layer15_attention_weight_min": -32.40024948120117,
      "activations/layer16_attention_weight_max": 32.74148941040039,
      "activations/layer16_attention_weight_min": -33.463043212890625,
      "activations/layer17_attention_weight_max": 47.42768096923828,
      "activations/layer17_attention_weight_min": -44.44670486450195,
      "activations/layer18_attention_weight_max": 49.935359954833984,
      "activations/layer18_attention_weight_min": -42.42179870605469,
      "activations/layer19_attention_weight_max": 23.580890655517578,
      "activations/layer19_attention_weight_min": -23.848480224609375,
      "activations/layer1_attention_weight_max": 15.108641624450684,
      "activations/layer1_attention_weight_min": -14.839702606201172,
      "activations/layer20_attention_weight_max": 22.277088165283203,
      "activations/layer20_attention_weight_min": -22.737548828125,
      "activations/layer21_attention_weight_max": 34.78668212890625,
      "activations/layer21_attention_weight_min": -21.431425094604492,
      "activations/layer22_attention_weight_max": 27.653337478637695,
      "activations/layer22_attention_weight_min": -25.9879150390625,
      "activations/layer23_attention_weight_max": 35.8124885559082,
      "activations/layer23_attention_weight_min": -23.042892456054688,
      "activations/layer2_attention_weight_max": 30.705753326416016,
      "activations/layer2_attention_weight_min": -30.63726043701172,
      "activations/layer3_attention_weight_max": 83.07689666748047,
      "activations/layer3_attention_weight_min": -86.09895324707031,
      "activations/layer4_attention_weight_max": 80.73326873779297,
      "activations/layer4_attention_weight_min": -79.62617492675781,
      "activations/layer5_attention_weight_max": 62.274444580078125,
      "activations/layer5_attention_weight_min": -69.7285385131836,
      "activations/layer6_attention_weight_max": 49.07957458496094,
      "activations/layer6_attention_weight_min": -52.68373489379883,
      "activations/layer7_attention_weight_max": 67.88691711425781,
      "activations/layer7_attention_weight_min": -63.0447883605957,
      "activations/layer8_attention_weight_max": 45.55004119873047,
      "activations/layer8_attention_weight_min": -49.799137115478516,
      "activations/layer9_attention_weight_max": 43.93547439575195,
      "activations/layer9_attention_weight_min": -46.09238815307617,
      "epoch": 19.38,
      "learning_rate": 2.5272348484848483e-05,
      "loss": 2.7399,
      "step": 333550
    },
    {
      "activations/layer0_attention_weight_max": 16.117916107177734,
      "activations/layer0_attention_weight_min": -14.072577476501465,
      "activations/layer10_attention_weight_max": 35.72625732421875,
      "activations/layer10_attention_weight_min": -36.66118621826172,
      "activations/layer11_attention_weight_max": 33.87843704223633,
      "activations/layer11_attention_weight_min": -34.226707458496094,
      "activations/layer12_attention_weight_max": 20.45545196533203,
      "activations/layer12_attention_weight_min": -24.094390869140625,
      "activations/layer13_attention_weight_max": 37.08429718017578,
      "activations/layer13_attention_weight_min": -28.50217056274414,
      "activations/layer14_attention_weight_max": 34.729026794433594,
      "activations/layer14_attention_weight_min": -30.828149795532227,
      "activations/layer15_attention_weight_max": 32.0728645324707,
      "activations/layer15_attention_weight_min": -29.656688690185547,
      "activations/layer16_attention_weight_max": 33.81101989746094,
      "activations/layer16_attention_weight_min": -32.23789596557617,
      "activations/layer17_attention_weight_max": 48.647518157958984,
      "activations/layer17_attention_weight_min": -43.35072708129883,
      "activations/layer18_attention_weight_max": 46.00579833984375,
      "activations/layer18_attention_weight_min": -38.219947814941406,
      "activations/layer19_attention_weight_max": 22.145490646362305,
      "activations/layer19_attention_weight_min": -22.187589645385742,
      "activations/layer1_attention_weight_max": 16.573991775512695,
      "activations/layer1_attention_weight_min": -15.138185501098633,
      "activations/layer20_attention_weight_max": 21.411712646484375,
      "activations/layer20_attention_weight_min": -20.989421844482422,
      "activations/layer21_attention_weight_max": 38.44266128540039,
      "activations/layer21_attention_weight_min": -25.2022762298584,
      "activations/layer22_attention_weight_max": 29.36332893371582,
      "activations/layer22_attention_weight_min": -26.283647537231445,
      "activations/layer23_attention_weight_max": 39.092891693115234,
      "activations/layer23_attention_weight_min": -24.511730194091797,
      "activations/layer2_attention_weight_max": 32.63347244262695,
      "activations/layer2_attention_weight_min": -30.572864532470703,
      "activations/layer3_attention_weight_max": 85.30087280273438,
      "activations/layer3_attention_weight_min": -82.76500701904297,
      "activations/layer4_attention_weight_max": 82.38480377197266,
      "activations/layer4_attention_weight_min": -81.71127319335938,
      "activations/layer5_attention_weight_max": 59.55274963378906,
      "activations/layer5_attention_weight_min": -70.16568756103516,
      "activations/layer6_attention_weight_max": 48.56095886230469,
      "activations/layer6_attention_weight_min": -49.18180465698242,
      "activations/layer7_attention_weight_max": 61.358055114746094,
      "activations/layer7_attention_weight_min": -63.961029052734375,
      "activations/layer8_attention_weight_max": 46.81391906738281,
      "activations/layer8_attention_weight_min": -53.79411315917969,
      "activations/layer9_attention_weight_max": 42.573219299316406,
      "activations/layer9_attention_weight_min": -45.84720993041992,
      "epoch": 19.38,
      "learning_rate": 2.5253409090909087e-05,
      "loss": 2.7176,
      "step": 333600
    },
    {
      "activations/layer0_attention_weight_max": 17.349720001220703,
      "activations/layer0_attention_weight_min": -13.988115310668945,
      "activations/layer10_attention_weight_max": 35.42110824584961,
      "activations/layer10_attention_weight_min": -36.487796783447266,
      "activations/layer11_attention_weight_max": 33.04563522338867,
      "activations/layer11_attention_weight_min": -35.27766036987305,
      "activations/layer12_attention_weight_max": 20.25098419189453,
      "activations/layer12_attention_weight_min": -28.141096115112305,
      "activations/layer13_attention_weight_max": 36.6401481628418,
      "activations/layer13_attention_weight_min": -28.514101028442383,
      "activations/layer14_attention_weight_max": 40.16603088378906,
      "activations/layer14_attention_weight_min": -31.355571746826172,
      "activations/layer15_attention_weight_max": 35.95661544799805,
      "activations/layer15_attention_weight_min": -30.821611404418945,
      "activations/layer16_attention_weight_max": 39.065834045410156,
      "activations/layer16_attention_weight_min": -33.18803024291992,
      "activations/layer17_attention_weight_max": 58.26005172729492,
      "activations/layer17_attention_weight_min": -45.95938491821289,
      "activations/layer18_attention_weight_max": 57.6120719909668,
      "activations/layer18_attention_weight_min": -41.2244987487793,
      "activations/layer19_attention_weight_max": 24.802536010742188,
      "activations/layer19_attention_weight_min": -22.470504760742188,
      "activations/layer1_attention_weight_max": 15.058032989501953,
      "activations/layer1_attention_weight_min": -14.069144248962402,
      "activations/layer20_attention_weight_max": 23.759672164916992,
      "activations/layer20_attention_weight_min": -22.712425231933594,
      "activations/layer21_attention_weight_max": 38.15285873413086,
      "activations/layer21_attention_weight_min": -22.21368980407715,
      "activations/layer22_attention_weight_max": 36.053524017333984,
      "activations/layer22_attention_weight_min": -25.2099609375,
      "activations/layer23_attention_weight_max": 42.870521545410156,
      "activations/layer23_attention_weight_min": -24.090726852416992,
      "activations/layer2_attention_weight_max": 30.964191436767578,
      "activations/layer2_attention_weight_min": -30.038434982299805,
      "activations/layer3_attention_weight_max": 83.17101287841797,
      "activations/layer3_attention_weight_min": -78.92454528808594,
      "activations/layer4_attention_weight_max": 80.3548812866211,
      "activations/layer4_attention_weight_min": -79.3164291381836,
      "activations/layer5_attention_weight_max": 65.15833282470703,
      "activations/layer5_attention_weight_min": -67.40252685546875,
      "activations/layer6_attention_weight_max": 49.251346588134766,
      "activations/layer6_attention_weight_min": -49.884422302246094,
      "activations/layer7_attention_weight_max": 67.06436920166016,
      "activations/layer7_attention_weight_min": -62.361122131347656,
      "activations/layer8_attention_weight_max": 46.41484832763672,
      "activations/layer8_attention_weight_min": -48.46855545043945,
      "activations/layer9_attention_weight_max": 43.94865036010742,
      "activations/layer9_attention_weight_min": -46.79689025878906,
      "epoch": 19.39,
      "learning_rate": 2.5234469696969696e-05,
      "loss": 2.7209,
      "step": 333650
    },
    {
      "activations/layer0_attention_weight_max": 16.288022994995117,
      "activations/layer0_attention_weight_min": -14.0479154586792,
      "activations/layer10_attention_weight_max": 36.695106506347656,
      "activations/layer10_attention_weight_min": -35.789363861083984,
      "activations/layer11_attention_weight_max": 34.110252380371094,
      "activations/layer11_attention_weight_min": -39.91111755371094,
      "activations/layer12_attention_weight_max": 20.662044525146484,
      "activations/layer12_attention_weight_min": -24.980152130126953,
      "activations/layer13_attention_weight_max": 38.03934860229492,
      "activations/layer13_attention_weight_min": -30.00767707824707,
      "activations/layer14_attention_weight_max": 41.99977111816406,
      "activations/layer14_attention_weight_min": -32.6229248046875,
      "activations/layer15_attention_weight_max": 38.379051208496094,
      "activations/layer15_attention_weight_min": -31.234241485595703,
      "activations/layer16_attention_weight_max": 39.872196197509766,
      "activations/layer16_attention_weight_min": -37.09238052368164,
      "activations/layer17_attention_weight_max": 60.52730941772461,
      "activations/layer17_attention_weight_min": -55.76728820800781,
      "activations/layer18_attention_weight_max": 55.46841049194336,
      "activations/layer18_attention_weight_min": -47.815147399902344,
      "activations/layer19_attention_weight_max": 29.047773361206055,
      "activations/layer19_attention_weight_min": -28.182188034057617,
      "activations/layer1_attention_weight_max": 15.803597450256348,
      "activations/layer1_attention_weight_min": -14.906638145446777,
      "activations/layer20_attention_weight_max": 27.287086486816406,
      "activations/layer20_attention_weight_min": -23.691709518432617,
      "activations/layer21_attention_weight_max": 38.97114944458008,
      "activations/layer21_attention_weight_min": -23.255779266357422,
      "activations/layer22_attention_weight_max": 29.7158203125,
      "activations/layer22_attention_weight_min": -26.83758544921875,
      "activations/layer23_attention_weight_max": 39.39551544189453,
      "activations/layer23_attention_weight_min": -23.109176635742188,
      "activations/layer2_attention_weight_max": 32.215858459472656,
      "activations/layer2_attention_weight_min": -31.7144832611084,
      "activations/layer3_attention_weight_max": 83.7881851196289,
      "activations/layer3_attention_weight_min": -87.24495697021484,
      "activations/layer4_attention_weight_max": 83.58506774902344,
      "activations/layer4_attention_weight_min": -84.5676498413086,
      "activations/layer5_attention_weight_max": 61.903465270996094,
      "activations/layer5_attention_weight_min": -68.25721740722656,
      "activations/layer6_attention_weight_max": 49.70051956176758,
      "activations/layer6_attention_weight_min": -51.870933532714844,
      "activations/layer7_attention_weight_max": 67.23332214355469,
      "activations/layer7_attention_weight_min": -67.52772521972656,
      "activations/layer8_attention_weight_max": 48.842140197753906,
      "activations/layer8_attention_weight_min": -52.69485092163086,
      "activations/layer9_attention_weight_max": 45.007957458496094,
      "activations/layer9_attention_weight_min": -46.774837493896484,
      "epoch": 19.39,
      "learning_rate": 2.5215530303030304e-05,
      "loss": 2.7131,
      "step": 333700
    },
    {
      "activations/layer0_attention_weight_max": 15.466378211975098,
      "activations/layer0_attention_weight_min": -13.612524032592773,
      "activations/layer10_attention_weight_max": 35.87955093383789,
      "activations/layer10_attention_weight_min": -36.255592346191406,
      "activations/layer11_attention_weight_max": 33.34223175048828,
      "activations/layer11_attention_weight_min": -33.77458572387695,
      "activations/layer12_attention_weight_max": 19.198810577392578,
      "activations/layer12_attention_weight_min": -23.10236930847168,
      "activations/layer13_attention_weight_max": 34.9825553894043,
      "activations/layer13_attention_weight_min": -29.547060012817383,
      "activations/layer14_attention_weight_max": 35.56875228881836,
      "activations/layer14_attention_weight_min": -32.49665451049805,
      "activations/layer15_attention_weight_max": 33.73551559448242,
      "activations/layer15_attention_weight_min": -31.89154815673828,
      "activations/layer16_attention_weight_max": 34.971675872802734,
      "activations/layer16_attention_weight_min": -33.598453521728516,
      "activations/layer17_attention_weight_max": 54.115665435791016,
      "activations/layer17_attention_weight_min": -46.434547424316406,
      "activations/layer18_attention_weight_max": 51.307376861572266,
      "activations/layer18_attention_weight_min": -41.190216064453125,
      "activations/layer19_attention_weight_max": 25.268281936645508,
      "activations/layer19_attention_weight_min": -25.468948364257812,
      "activations/layer1_attention_weight_max": 15.32898235321045,
      "activations/layer1_attention_weight_min": -14.06835651397705,
      "activations/layer20_attention_weight_max": 24.66888999938965,
      "activations/layer20_attention_weight_min": -22.170602798461914,
      "activations/layer21_attention_weight_max": 40.80591583251953,
      "activations/layer21_attention_weight_min": -23.84916877746582,
      "activations/layer22_attention_weight_max": 30.55034065246582,
      "activations/layer22_attention_weight_min": -26.61908721923828,
      "activations/layer23_attention_weight_max": 40.43790817260742,
      "activations/layer23_attention_weight_min": -24.962928771972656,
      "activations/layer2_attention_weight_max": 30.727123260498047,
      "activations/layer2_attention_weight_min": -29.420940399169922,
      "activations/layer3_attention_weight_max": 83.55050659179688,
      "activations/layer3_attention_weight_min": -88.21080017089844,
      "activations/layer4_attention_weight_max": 82.75179290771484,
      "activations/layer4_attention_weight_min": -80.91590118408203,
      "activations/layer5_attention_weight_max": 61.560638427734375,
      "activations/layer5_attention_weight_min": -71.15179443359375,
      "activations/layer6_attention_weight_max": 52.569786071777344,
      "activations/layer6_attention_weight_min": -53.171470642089844,
      "activations/layer7_attention_weight_max": 62.354122161865234,
      "activations/layer7_attention_weight_min": -67.15435028076172,
      "activations/layer8_attention_weight_max": 47.40309524536133,
      "activations/layer8_attention_weight_min": -51.62105941772461,
      "activations/layer9_attention_weight_max": 46.468536376953125,
      "activations/layer9_attention_weight_min": -46.97608184814453,
      "epoch": 19.39,
      "learning_rate": 2.5196590909090906e-05,
      "loss": 2.7406,
      "step": 333750
    },
    {
      "activations/layer0_attention_weight_max": 15.228045463562012,
      "activations/layer0_attention_weight_min": -14.214859962463379,
      "activations/layer10_attention_weight_max": 35.96210479736328,
      "activations/layer10_attention_weight_min": -34.91242599487305,
      "activations/layer11_attention_weight_max": 32.53774642944336,
      "activations/layer11_attention_weight_min": -33.74982452392578,
      "activations/layer12_attention_weight_max": 18.67375373840332,
      "activations/layer12_attention_weight_min": -27.74042320251465,
      "activations/layer13_attention_weight_max": 26.958293914794922,
      "activations/layer13_attention_weight_min": -27.18764305114746,
      "activations/layer14_attention_weight_max": 32.04771423339844,
      "activations/layer14_attention_weight_min": -29.75934600830078,
      "activations/layer15_attention_weight_max": 30.072233200073242,
      "activations/layer15_attention_weight_min": -31.479198455810547,
      "activations/layer16_attention_weight_max": 33.25690841674805,
      "activations/layer16_attention_weight_min": -33.436546325683594,
      "activations/layer17_attention_weight_max": 49.43980407714844,
      "activations/layer17_attention_weight_min": -48.928287506103516,
      "activations/layer18_attention_weight_max": 47.36789321899414,
      "activations/layer18_attention_weight_min": -41.33798599243164,
      "activations/layer19_attention_weight_max": 23.557302474975586,
      "activations/layer19_attention_weight_min": -23.814294815063477,
      "activations/layer1_attention_weight_max": 15.291565895080566,
      "activations/layer1_attention_weight_min": -13.603257179260254,
      "activations/layer20_attention_weight_max": 20.88858985900879,
      "activations/layer20_attention_weight_min": -22.85329246520996,
      "activations/layer21_attention_weight_max": 31.652475357055664,
      "activations/layer21_attention_weight_min": -22.49788475036621,
      "activations/layer22_attention_weight_max": 25.63048553466797,
      "activations/layer22_attention_weight_min": -26.60967254638672,
      "activations/layer23_attention_weight_max": 35.73167419433594,
      "activations/layer23_attention_weight_min": -25.775434494018555,
      "activations/layer2_attention_weight_max": 26.3177490234375,
      "activations/layer2_attention_weight_min": -26.275951385498047,
      "activations/layer3_attention_weight_max": 71.86296081542969,
      "activations/layer3_attention_weight_min": -76.22209167480469,
      "activations/layer4_attention_weight_max": 73.693115234375,
      "activations/layer4_attention_weight_min": -76.76271057128906,
      "activations/layer5_attention_weight_max": 57.470741271972656,
      "activations/layer5_attention_weight_min": -73.61119842529297,
      "activations/layer6_attention_weight_max": 45.742549896240234,
      "activations/layer6_attention_weight_min": -49.590145111083984,
      "activations/layer7_attention_weight_max": 60.46711730957031,
      "activations/layer7_attention_weight_min": -67.07714080810547,
      "activations/layer8_attention_weight_max": 42.71436309814453,
      "activations/layer8_attention_weight_min": -47.71676254272461,
      "activations/layer9_attention_weight_max": 40.17033004760742,
      "activations/layer9_attention_weight_min": -44.72282791137695,
      "epoch": 19.4,
      "learning_rate": 2.5177651515151514e-05,
      "loss": 2.7336,
      "step": 333800
    },
    {
      "activations/layer0_attention_weight_max": 15.837207794189453,
      "activations/layer0_attention_weight_min": -13.967455863952637,
      "activations/layer10_attention_weight_max": 36.31715774536133,
      "activations/layer10_attention_weight_min": -37.21846389770508,
      "activations/layer11_attention_weight_max": 33.52288055419922,
      "activations/layer11_attention_weight_min": -35.153053283691406,
      "activations/layer12_attention_weight_max": 18.841646194458008,
      "activations/layer12_attention_weight_min": -24.942977905273438,
      "activations/layer13_attention_weight_max": 31.011829376220703,
      "activations/layer13_attention_weight_min": -28.459369659423828,
      "activations/layer14_attention_weight_max": 35.27058410644531,
      "activations/layer14_attention_weight_min": -30.73165512084961,
      "activations/layer15_attention_weight_max": 32.34755325317383,
      "activations/layer15_attention_weight_min": -33.383270263671875,
      "activations/layer16_attention_weight_max": 35.04454040527344,
      "activations/layer16_attention_weight_min": -33.54905319213867,
      "activations/layer17_attention_weight_max": 52.95750045776367,
      "activations/layer17_attention_weight_min": -47.63307189941406,
      "activations/layer18_attention_weight_max": 48.28807830810547,
      "activations/layer18_attention_weight_min": -44.933353424072266,
      "activations/layer19_attention_weight_max": 24.080873489379883,
      "activations/layer19_attention_weight_min": -22.839168548583984,
      "activations/layer1_attention_weight_max": 15.013504028320312,
      "activations/layer1_attention_weight_min": -12.550369262695312,
      "activations/layer20_attention_weight_max": 22.376317977905273,
      "activations/layer20_attention_weight_min": -21.005538940429688,
      "activations/layer21_attention_weight_max": 33.9435920715332,
      "activations/layer21_attention_weight_min": -21.6019344329834,
      "activations/layer22_attention_weight_max": 28.6441707611084,
      "activations/layer22_attention_weight_min": -27.6540584564209,
      "activations/layer23_attention_weight_max": 36.591835021972656,
      "activations/layer23_attention_weight_min": -25.377361297607422,
      "activations/layer2_attention_weight_max": 31.376874923706055,
      "activations/layer2_attention_weight_min": -30.65290641784668,
      "activations/layer3_attention_weight_max": 87.2085189819336,
      "activations/layer3_attention_weight_min": -86.15328216552734,
      "activations/layer4_attention_weight_max": 81.42940521240234,
      "activations/layer4_attention_weight_min": -81.7501220703125,
      "activations/layer5_attention_weight_max": 62.31707000732422,
      "activations/layer5_attention_weight_min": -74.18415832519531,
      "activations/layer6_attention_weight_max": 48.26414489746094,
      "activations/layer6_attention_weight_min": -48.449649810791016,
      "activations/layer7_attention_weight_max": 63.45966339111328,
      "activations/layer7_attention_weight_min": -63.71107482910156,
      "activations/layer8_attention_weight_max": 47.85525131225586,
      "activations/layer8_attention_weight_min": -50.21659851074219,
      "activations/layer9_attention_weight_max": 44.8044319152832,
      "activations/layer9_attention_weight_min": -45.352664947509766,
      "epoch": 19.4,
      "learning_rate": 2.515871212121212e-05,
      "loss": 2.7411,
      "step": 333850
    },
    {
      "activations/layer0_attention_weight_max": 16.04501724243164,
      "activations/layer0_attention_weight_min": -13.846702575683594,
      "activations/layer10_attention_weight_max": 37.87080383300781,
      "activations/layer10_attention_weight_min": -38.52183532714844,
      "activations/layer11_attention_weight_max": 35.970619201660156,
      "activations/layer11_attention_weight_min": -34.72095489501953,
      "activations/layer12_attention_weight_max": 19.084293365478516,
      "activations/layer12_attention_weight_min": -23.99003028869629,
      "activations/layer13_attention_weight_max": 34.501712799072266,
      "activations/layer13_attention_weight_min": -27.262678146362305,
      "activations/layer14_attention_weight_max": 34.99531936645508,
      "activations/layer14_attention_weight_min": -28.918710708618164,
      "activations/layer15_attention_weight_max": 31.682363510131836,
      "activations/layer15_attention_weight_min": -29.65983772277832,
      "activations/layer16_attention_weight_max": 34.723289489746094,
      "activations/layer16_attention_weight_min": -34.05345153808594,
      "activations/layer17_attention_weight_max": 52.91039276123047,
      "activations/layer17_attention_weight_min": -47.97828674316406,
      "activations/layer18_attention_weight_max": 46.95127487182617,
      "activations/layer18_attention_weight_min": -40.76681900024414,
      "activations/layer19_attention_weight_max": 25.984088897705078,
      "activations/layer19_attention_weight_min": -23.03520965576172,
      "activations/layer1_attention_weight_max": 15.30853271484375,
      "activations/layer1_attention_weight_min": -12.472258567810059,
      "activations/layer20_attention_weight_max": 21.66185188293457,
      "activations/layer20_attention_weight_min": -22.671281814575195,
      "activations/layer21_attention_weight_max": 36.33824920654297,
      "activations/layer21_attention_weight_min": -25.044742584228516,
      "activations/layer22_attention_weight_max": 28.166690826416016,
      "activations/layer22_attention_weight_min": -24.580904006958008,
      "activations/layer23_attention_weight_max": 37.279991149902344,
      "activations/layer23_attention_weight_min": -25.14238739013672,
      "activations/layer2_attention_weight_max": 30.088882446289062,
      "activations/layer2_attention_weight_min": -29.052806854248047,
      "activations/layer3_attention_weight_max": 86.5147476196289,
      "activations/layer3_attention_weight_min": -84.31919860839844,
      "activations/layer4_attention_weight_max": 83.27178192138672,
      "activations/layer4_attention_weight_min": -83.26775360107422,
      "activations/layer5_attention_weight_max": 60.116943359375,
      "activations/layer5_attention_weight_min": -72.85823059082031,
      "activations/layer6_attention_weight_max": 48.33736801147461,
      "activations/layer6_attention_weight_min": -48.87218475341797,
      "activations/layer7_attention_weight_max": 65.24028015136719,
      "activations/layer7_attention_weight_min": -68.40687561035156,
      "activations/layer8_attention_weight_max": 48.373558044433594,
      "activations/layer8_attention_weight_min": -49.634193420410156,
      "activations/layer9_attention_weight_max": 47.023887634277344,
      "activations/layer9_attention_weight_min": -46.71291732788086,
      "epoch": 19.4,
      "learning_rate": 2.5140151515151513e-05,
      "loss": 2.7206,
      "step": 333900
    },
    {
      "activations/layer0_attention_weight_max": 16.45355796813965,
      "activations/layer0_attention_weight_min": -13.91514778137207,
      "activations/layer10_attention_weight_max": 35.558719635009766,
      "activations/layer10_attention_weight_min": -35.6142463684082,
      "activations/layer11_attention_weight_max": 30.814590454101562,
      "activations/layer11_attention_weight_min": -32.97327423095703,
      "activations/layer12_attention_weight_max": 17.25370979309082,
      "activations/layer12_attention_weight_min": -26.577571868896484,
      "activations/layer13_attention_weight_max": 30.61196517944336,
      "activations/layer13_attention_weight_min": -27.854686737060547,
      "activations/layer14_attention_weight_max": 32.70054626464844,
      "activations/layer14_attention_weight_min": -30.712726593017578,
      "activations/layer15_attention_weight_max": 30.863933563232422,
      "activations/layer15_attention_weight_min": -31.91802978515625,
      "activations/layer16_attention_weight_max": 32.3710823059082,
      "activations/layer16_attention_weight_min": -31.856042861938477,
      "activations/layer17_attention_weight_max": 48.3146858215332,
      "activations/layer17_attention_weight_min": -44.83394241333008,
      "activations/layer18_attention_weight_max": 46.437007904052734,
      "activations/layer18_attention_weight_min": -40.424156188964844,
      "activations/layer19_attention_weight_max": 22.340538024902344,
      "activations/layer19_attention_weight_min": -22.422311782836914,
      "activations/layer1_attention_weight_max": 16.30747413635254,
      "activations/layer1_attention_weight_min": -13.779160499572754,
      "activations/layer20_attention_weight_max": 22.14447784423828,
      "activations/layer20_attention_weight_min": -24.16054916381836,
      "activations/layer21_attention_weight_max": 30.407604217529297,
      "activations/layer21_attention_weight_min": -21.545486450195312,
      "activations/layer22_attention_weight_max": 28.538312911987305,
      "activations/layer22_attention_weight_min": -29.16954803466797,
      "activations/layer23_attention_weight_max": 35.11865997314453,
      "activations/layer23_attention_weight_min": -26.20401382446289,
      "activations/layer2_attention_weight_max": 31.960254669189453,
      "activations/layer2_attention_weight_min": -28.689863204956055,
      "activations/layer3_attention_weight_max": 85.24823760986328,
      "activations/layer3_attention_weight_min": -81.18071746826172,
      "activations/layer4_attention_weight_max": 85.07079315185547,
      "activations/layer4_attention_weight_min": -84.29070281982422,
      "activations/layer5_attention_weight_max": 62.422237396240234,
      "activations/layer5_attention_weight_min": -68.77643585205078,
      "activations/layer6_attention_weight_max": 48.68024444580078,
      "activations/layer6_attention_weight_min": -51.47952651977539,
      "activations/layer7_attention_weight_max": 66.4492416381836,
      "activations/layer7_attention_weight_min": -63.06644821166992,
      "activations/layer8_attention_weight_max": 45.2391242980957,
      "activations/layer8_attention_weight_min": -49.741207122802734,
      "activations/layer9_attention_weight_max": 48.63372039794922,
      "activations/layer9_attention_weight_min": -46.44965362548828,
      "epoch": 19.4,
      "learning_rate": 2.5121212121212118e-05,
      "loss": 2.7245,
      "step": 333950
    },
    {
      "activations/layer0_attention_weight_max": 14.980592727661133,
      "activations/layer0_attention_weight_min": -14.13250732421875,
      "activations/layer10_attention_weight_max": 40.14588928222656,
      "activations/layer10_attention_weight_min": -39.997623443603516,
      "activations/layer11_attention_weight_max": 35.26174545288086,
      "activations/layer11_attention_weight_min": -36.809967041015625,
      "activations/layer12_attention_weight_max": 20.839250564575195,
      "activations/layer12_attention_weight_min": -25.78106117248535,
      "activations/layer13_attention_weight_max": 36.2500114440918,
      "activations/layer13_attention_weight_min": -29.6207275390625,
      "activations/layer14_attention_weight_max": 36.754432678222656,
      "activations/layer14_attention_weight_min": -32.64261245727539,
      "activations/layer15_attention_weight_max": 34.00934600830078,
      "activations/layer15_attention_weight_min": -28.595500946044922,
      "activations/layer16_attention_weight_max": 36.248287200927734,
      "activations/layer16_attention_weight_min": -34.2892951965332,
      "activations/layer17_attention_weight_max": 52.65910720825195,
      "activations/layer17_attention_weight_min": -46.45021438598633,
      "activations/layer18_attention_weight_max": 48.77519989013672,
      "activations/layer18_attention_weight_min": -41.88760757446289,
      "activations/layer19_attention_weight_max": 25.558303833007812,
      "activations/layer19_attention_weight_min": -27.19180679321289,
      "activations/layer1_attention_weight_max": 15.759041786193848,
      "activations/layer1_attention_weight_min": -13.266427040100098,
      "activations/layer20_attention_weight_max": 23.81309700012207,
      "activations/layer20_attention_weight_min": -22.107933044433594,
      "activations/layer21_attention_weight_max": 35.56595993041992,
      "activations/layer21_attention_weight_min": -24.913236618041992,
      "activations/layer22_attention_weight_max": 29.35855484008789,
      "activations/layer22_attention_weight_min": -25.30963134765625,
      "activations/layer23_attention_weight_max": 37.284019470214844,
      "activations/layer23_attention_weight_min": -25.40717124938965,
      "activations/layer2_attention_weight_max": 32.77537536621094,
      "activations/layer2_attention_weight_min": -30.880903244018555,
      "activations/layer3_attention_weight_max": 90.60955810546875,
      "activations/layer3_attention_weight_min": -88.03434753417969,
      "activations/layer4_attention_weight_max": 91.14002990722656,
      "activations/layer4_attention_weight_min": -84.82913970947266,
      "activations/layer5_attention_weight_max": 69.50146484375,
      "activations/layer5_attention_weight_min": -73.02839660644531,
      "activations/layer6_attention_weight_max": 53.364280700683594,
      "activations/layer6_attention_weight_min": -55.83205032348633,
      "activations/layer7_attention_weight_max": 66.63513946533203,
      "activations/layer7_attention_weight_min": -66.8926773071289,
      "activations/layer8_attention_weight_max": 52.296783447265625,
      "activations/layer8_attention_weight_min": -53.04160690307617,
      "activations/layer9_attention_weight_max": 49.61592102050781,
      "activations/layer9_attention_weight_min": -50.259578704833984,
      "epoch": 19.41,
      "learning_rate": 2.5102272727272726e-05,
      "loss": 2.7285,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.5311,
      "eval_samples_per_second": 503.337,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.5311,
      "eval_openwebtext_samples_per_second": 503.337,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 2.0173,
      "eval_wikitext_samples_per_second": 226.05,
      "step": 334000
    },
    {
      "epoch": 19.41,
      "eval_lambada_loss": 2.474609375,
      "eval_lambada_ppl": 11.877066727642635,
      "eval_lambada_runtime": 9.5951,
      "eval_lambada_samples_per_second": 507.446,
      "step": 334000
    },
    {
      "activations/layer0_attention_weight_max": 15.522608757019043,
      "activations/layer0_attention_weight_min": -14.068167686462402,
      "activations/layer10_attention_weight_max": 33.60511016845703,
      "activations/layer10_attention_weight_min": -34.52077102661133,
      "activations/layer11_attention_weight_max": 31.23642349243164,
      "activations/layer11_attention_weight_min": -32.35209274291992,
      "activations/layer12_attention_weight_max": 18.410842895507812,
      "activations/layer12_attention_weight_min": -24.531341552734375,
      "activations/layer13_attention_weight_max": 31.468360900878906,
      "activations/layer13_attention_weight_min": -26.305034637451172,
      "activations/layer14_attention_weight_max": 33.62234115600586,
      "activations/layer14_attention_weight_min": -28.91886329650879,
      "activations/layer15_attention_weight_max": 32.50257110595703,
      "activations/layer15_attention_weight_min": -29.038787841796875,
      "activations/layer16_attention_weight_max": 33.63564682006836,
      "activations/layer16_attention_weight_min": -32.38656234741211,
      "activations/layer17_attention_weight_max": 48.91167068481445,
      "activations/layer17_attention_weight_min": -42.8914680480957,
      "activations/layer18_attention_weight_max": 47.66486358642578,
      "activations/layer18_attention_weight_min": -40.44624710083008,
      "activations/layer19_attention_weight_max": 25.372608184814453,
      "activations/layer19_attention_weight_min": -22.558279037475586,
      "activations/layer1_attention_weight_max": 15.902251243591309,
      "activations/layer1_attention_weight_min": -12.959859848022461,
      "activations/layer20_attention_weight_max": 24.13080406188965,
      "activations/layer20_attention_weight_min": -20.687623977661133,
      "activations/layer21_attention_weight_max": 35.96088790893555,
      "activations/layer21_attention_weight_min": -20.539093017578125,
      "activations/layer22_attention_weight_max": 31.42438316345215,
      "activations/layer22_attention_weight_min": -26.127532958984375,
      "activations/layer23_attention_weight_max": 39.310081481933594,
      "activations/layer23_attention_weight_min": -24.7686767578125,
      "activations/layer2_attention_weight_max": 27.782272338867188,
      "activations/layer2_attention_weight_min": -27.0880184173584,
      "activations/layer3_attention_weight_max": 76.71309661865234,
      "activations/layer3_attention_weight_min": -78.11735534667969,
      "activations/layer4_attention_weight_max": 79.35935974121094,
      "activations/layer4_attention_weight_min": -77.96211242675781,
      "activations/layer5_attention_weight_max": 59.22964096069336,
      "activations/layer5_attention_weight_min": -68.91459655761719,
      "activations/layer6_attention_weight_max": 47.45366668701172,
      "activations/layer6_attention_weight_min": -48.721981048583984,
      "activations/layer7_attention_weight_max": 61.57731246948242,
      "activations/layer7_attention_weight_min": -63.18107986450195,
      "activations/layer8_attention_weight_max": 44.11286163330078,
      "activations/layer8_attention_weight_min": -50.532470703125,
      "activations/layer9_attention_weight_max": 40.62810134887695,
      "activations/layer9_attention_weight_min": -43.615482330322266,
      "epoch": 19.41,
      "learning_rate": 2.508333333333333e-05,
      "loss": 2.7226,
      "step": 334050
    },
    {
      "activations/layer0_attention_weight_max": 15.654738426208496,
      "activations/layer0_attention_weight_min": -14.241691589355469,
      "activations/layer10_attention_weight_max": 36.3568115234375,
      "activations/layer10_attention_weight_min": -36.168209075927734,
      "activations/layer11_attention_weight_max": 33.053688049316406,
      "activations/layer11_attention_weight_min": -35.148956298828125,
      "activations/layer12_attention_weight_max": 18.423633575439453,
      "activations/layer12_attention_weight_min": -32.87104034423828,
      "activations/layer13_attention_weight_max": 32.647613525390625,
      "activations/layer13_attention_weight_min": -26.853029251098633,
      "activations/layer14_attention_weight_max": 34.8863525390625,
      "activations/layer14_attention_weight_min": -29.114604949951172,
      "activations/layer15_attention_weight_max": 34.43635940551758,
      "activations/layer15_attention_weight_min": -29.42615509033203,
      "activations/layer16_attention_weight_max": 35.51834487915039,
      "activations/layer16_attention_weight_min": -32.508766174316406,
      "activations/layer17_attention_weight_max": 51.36014175415039,
      "activations/layer17_attention_weight_min": -44.76952362060547,
      "activations/layer18_attention_weight_max": 52.32038497924805,
      "activations/layer18_attention_weight_min": -40.5329704284668,
      "activations/layer19_attention_weight_max": 25.43031120300293,
      "activations/layer19_attention_weight_min": -22.911128997802734,
      "activations/layer1_attention_weight_max": 15.796643257141113,
      "activations/layer1_attention_weight_min": -12.94660472869873,
      "activations/layer20_attention_weight_max": 24.901748657226562,
      "activations/layer20_attention_weight_min": -21.445219039916992,
      "activations/layer21_attention_weight_max": 37.170928955078125,
      "activations/layer21_attention_weight_min": -22.669504165649414,
      "activations/layer22_attention_weight_max": 30.17784309387207,
      "activations/layer22_attention_weight_min": -24.24477195739746,
      "activations/layer23_attention_weight_max": 36.926368713378906,
      "activations/layer23_attention_weight_min": -23.265913009643555,
      "activations/layer2_attention_weight_max": 31.035717010498047,
      "activations/layer2_attention_weight_min": -29.225515365600586,
      "activations/layer3_attention_weight_max": 88.48677825927734,
      "activations/layer3_attention_weight_min": -82.67279052734375,
      "activations/layer4_attention_weight_max": 84.37079620361328,
      "activations/layer4_attention_weight_min": -83.67916870117188,
      "activations/layer5_attention_weight_max": 63.44462203979492,
      "activations/layer5_attention_weight_min": -72.54020690917969,
      "activations/layer6_attention_weight_max": 49.55449295043945,
      "activations/layer6_attention_weight_min": -49.91168975830078,
      "activations/layer7_attention_weight_max": 68.02153015136719,
      "activations/layer7_attention_weight_min": -64.21546173095703,
      "activations/layer8_attention_weight_max": 46.7004280090332,
      "activations/layer8_attention_weight_min": -50.29497146606445,
      "activations/layer9_attention_weight_max": 44.96430206298828,
      "activations/layer9_attention_weight_min": -46.016544342041016,
      "epoch": 19.41,
      "learning_rate": 2.506439393939394e-05,
      "loss": 2.7443,
      "step": 334100
    },
    {
      "activations/layer0_attention_weight_max": 15.3065824508667,
      "activations/layer0_attention_weight_min": -14.077452659606934,
      "activations/layer10_attention_weight_max": 31.704626083374023,
      "activations/layer10_attention_weight_min": -33.18342208862305,
      "activations/layer11_attention_weight_max": 30.590679168701172,
      "activations/layer11_attention_weight_min": -32.71841049194336,
      "activations/layer12_attention_weight_max": 18.40185546875,
      "activations/layer12_attention_weight_min": -22.04764747619629,
      "activations/layer13_attention_weight_max": 36.0286750793457,
      "activations/layer13_attention_weight_min": -26.368513107299805,
      "activations/layer14_attention_weight_max": 34.774070739746094,
      "activations/layer14_attention_weight_min": -29.771154403686523,
      "activations/layer15_attention_weight_max": 32.480430603027344,
      "activations/layer15_attention_weight_min": -30.719907760620117,
      "activations/layer16_attention_weight_max": 33.901451110839844,
      "activations/layer16_attention_weight_min": -31.221887588500977,
      "activations/layer17_attention_weight_max": 50.52472686767578,
      "activations/layer17_attention_weight_min": -42.33774948120117,
      "activations/layer18_attention_weight_max": 43.816707611083984,
      "activations/layer18_attention_weight_min": -39.83999252319336,
      "activations/layer19_attention_weight_max": 23.840038299560547,
      "activations/layer19_attention_weight_min": -23.423412322998047,
      "activations/layer1_attention_weight_max": 15.504598617553711,
      "activations/layer1_attention_weight_min": -13.35866641998291,
      "activations/layer20_attention_weight_max": 21.178739547729492,
      "activations/layer20_attention_weight_min": -21.635343551635742,
      "activations/layer21_attention_weight_max": 33.42362976074219,
      "activations/layer21_attention_weight_min": -23.386215209960938,
      "activations/layer22_attention_weight_max": 28.8112850189209,
      "activations/layer22_attention_weight_min": -26.715173721313477,
      "activations/layer23_attention_weight_max": 33.491905212402344,
      "activations/layer23_attention_weight_min": -24.575536727905273,
      "activations/layer2_attention_weight_max": 31.020994186401367,
      "activations/layer2_attention_weight_min": -28.62392234802246,
      "activations/layer3_attention_weight_max": 80.56790924072266,
      "activations/layer3_attention_weight_min": -78.85845184326172,
      "activations/layer4_attention_weight_max": 78.32000732421875,
      "activations/layer4_attention_weight_min": -79.02489471435547,
      "activations/layer5_attention_weight_max": 62.15595626831055,
      "activations/layer5_attention_weight_min": -70.20448303222656,
      "activations/layer6_attention_weight_max": 46.19752502441406,
      "activations/layer6_attention_weight_min": -47.06359100341797,
      "activations/layer7_attention_weight_max": 62.37661361694336,
      "activations/layer7_attention_weight_min": -60.78810501098633,
      "activations/layer8_attention_weight_max": 44.82270431518555,
      "activations/layer8_attention_weight_min": -45.701759338378906,
      "activations/layer9_attention_weight_max": 39.77842712402344,
      "activations/layer9_attention_weight_min": -44.2274055480957,
      "epoch": 19.42,
      "learning_rate": 2.504545454545454e-05,
      "loss": 2.7067,
      "step": 334150
    },
    {
      "activations/layer0_attention_weight_max": 15.707362174987793,
      "activations/layer0_attention_weight_min": -14.185381889343262,
      "activations/layer10_attention_weight_max": 36.557613372802734,
      "activations/layer10_attention_weight_min": -36.49544143676758,
      "activations/layer11_attention_weight_max": 34.76744842529297,
      "activations/layer11_attention_weight_min": -35.47463607788086,
      "activations/layer12_attention_weight_max": 20.098224639892578,
      "activations/layer12_attention_weight_min": -24.7540225982666,
      "activations/layer13_attention_weight_max": 32.17275619506836,
      "activations/layer13_attention_weight_min": -26.248559951782227,
      "activations/layer14_attention_weight_max": 34.53573989868164,
      "activations/layer14_attention_weight_min": -30.100019454956055,
      "activations/layer15_attention_weight_max": 31.177326202392578,
      "activations/layer15_attention_weight_min": -29.687089920043945,
      "activations/layer16_attention_weight_max": 31.97699737548828,
      "activations/layer16_attention_weight_min": -30.285049438476562,
      "activations/layer17_attention_weight_max": 50.94957733154297,
      "activations/layer17_attention_weight_min": -42.46228790283203,
      "activations/layer18_attention_weight_max": 43.36775207519531,
      "activations/layer18_attention_weight_min": -38.62153244018555,
      "activations/layer19_attention_weight_max": 22.949193954467773,
      "activations/layer19_attention_weight_min": -22.61896324157715,
      "activations/layer1_attention_weight_max": 15.113323211669922,
      "activations/layer1_attention_weight_min": -14.184903144836426,
      "activations/layer20_attention_weight_max": 23.299848556518555,
      "activations/layer20_attention_weight_min": -21.82292366027832,
      "activations/layer21_attention_weight_max": 30.589366912841797,
      "activations/layer21_attention_weight_min": -18.883230209350586,
      "activations/layer22_attention_weight_max": 29.739099502563477,
      "activations/layer22_attention_weight_min": -23.886777877807617,
      "activations/layer23_attention_weight_max": 33.75989532470703,
      "activations/layer23_attention_weight_min": -22.653133392333984,
      "activations/layer2_attention_weight_max": 29.7695369720459,
      "activations/layer2_attention_weight_min": -28.757118225097656,
      "activations/layer3_attention_weight_max": 82.62893676757812,
      "activations/layer3_attention_weight_min": -81.92835998535156,
      "activations/layer4_attention_weight_max": 81.8250961303711,
      "activations/layer4_attention_weight_min": -82.26899719238281,
      "activations/layer5_attention_weight_max": 62.214195251464844,
      "activations/layer5_attention_weight_min": -71.23147583007812,
      "activations/layer6_attention_weight_max": 48.13153839111328,
      "activations/layer6_attention_weight_min": -48.90085220336914,
      "activations/layer7_attention_weight_max": 67.9478530883789,
      "activations/layer7_attention_weight_min": -64.62842559814453,
      "activations/layer8_attention_weight_max": 47.763763427734375,
      "activations/layer8_attention_weight_min": -51.43805694580078,
      "activations/layer9_attention_weight_max": 45.67057800292969,
      "activations/layer9_attention_weight_min": -48.31959915161133,
      "epoch": 19.42,
      "learning_rate": 2.502651515151515e-05,
      "loss": 2.7173,
      "step": 334200
    },
    {
      "activations/layer0_attention_weight_max": 15.640037536621094,
      "activations/layer0_attention_weight_min": -13.835878372192383,
      "activations/layer10_attention_weight_max": 48.71477127075195,
      "activations/layer10_attention_weight_min": -45.0413818359375,
      "activations/layer11_attention_weight_max": 47.958030700683594,
      "activations/layer11_attention_weight_min": -43.616737365722656,
      "activations/layer12_attention_weight_max": 22.26544952392578,
      "activations/layer12_attention_weight_min": -24.348812103271484,
      "activations/layer13_attention_weight_max": 42.00579833984375,
      "activations/layer13_attention_weight_min": -32.8625602722168,
      "activations/layer14_attention_weight_max": 46.980831146240234,
      "activations/layer14_attention_weight_min": -36.57350540161133,
      "activations/layer15_attention_weight_max": 46.76799774169922,
      "activations/layer15_attention_weight_min": -34.026634216308594,
      "activations/layer16_attention_weight_max": 39.44086837768555,
      "activations/layer16_attention_weight_min": -35.28120422363281,
      "activations/layer17_attention_weight_max": 58.0711669921875,
      "activations/layer17_attention_weight_min": -51.578285217285156,
      "activations/layer18_attention_weight_max": 59.46924591064453,
      "activations/layer18_attention_weight_min": -48.91054916381836,
      "activations/layer19_attention_weight_max": 28.776296615600586,
      "activations/layer19_attention_weight_min": -28.294570922851562,
      "activations/layer1_attention_weight_max": 16.212066650390625,
      "activations/layer1_attention_weight_min": -13.046282768249512,
      "activations/layer20_attention_weight_max": 30.1787166595459,
      "activations/layer20_attention_weight_min": -28.417888641357422,
      "activations/layer21_attention_weight_max": 58.327266693115234,
      "activations/layer21_attention_weight_min": -31.810428619384766,
      "activations/layer22_attention_weight_max": 33.19322204589844,
      "activations/layer22_attention_weight_min": -26.650423049926758,
      "activations/layer23_attention_weight_max": 42.74506378173828,
      "activations/layer23_attention_weight_min": -27.111652374267578,
      "activations/layer2_attention_weight_max": 33.30857849121094,
      "activations/layer2_attention_weight_min": -29.91915512084961,
      "activations/layer3_attention_weight_max": 89.09687805175781,
      "activations/layer3_attention_weight_min": -84.049072265625,
      "activations/layer4_attention_weight_max": 87.10994720458984,
      "activations/layer4_attention_weight_min": -82.73274993896484,
      "activations/layer5_attention_weight_max": 61.90472412109375,
      "activations/layer5_attention_weight_min": -71.97099304199219,
      "activations/layer6_attention_weight_max": 49.46522903442383,
      "activations/layer6_attention_weight_min": -51.59407043457031,
      "activations/layer7_attention_weight_max": 74.32242584228516,
      "activations/layer7_attention_weight_min": -73.4944076538086,
      "activations/layer8_attention_weight_max": 59.1546630859375,
      "activations/layer8_attention_weight_min": -62.2630615234375,
      "activations/layer9_attention_weight_max": 61.509765625,
      "activations/layer9_attention_weight_min": -55.54193115234375,
      "epoch": 19.42,
      "learning_rate": 2.5007575757575754e-05,
      "loss": 2.7295,
      "step": 334250
    },
    {
      "activations/layer0_attention_weight_max": 16.492088317871094,
      "activations/layer0_attention_weight_min": -14.054987907409668,
      "activations/layer10_attention_weight_max": 35.32762908935547,
      "activations/layer10_attention_weight_min": -36.09115219116211,
      "activations/layer11_attention_weight_max": 33.18840789794922,
      "activations/layer11_attention_weight_min": -34.61283874511719,
      "activations/layer12_attention_weight_max": 19.099267959594727,
      "activations/layer12_attention_weight_min": -25.410964965820312,
      "activations/layer13_attention_weight_max": 33.66886520385742,
      "activations/layer13_attention_weight_min": -26.239831924438477,
      "activations/layer14_attention_weight_max": 35.69966506958008,
      "activations/layer14_attention_weight_min": -30.25493049621582,
      "activations/layer15_attention_weight_max": 32.63088607788086,
      "activations/layer15_attention_weight_min": -30.360158920288086,
      "activations/layer16_attention_weight_max": 33.24881362915039,
      "activations/layer16_attention_weight_min": -32.149349212646484,
      "activations/layer17_attention_weight_max": 50.77452850341797,
      "activations/layer17_attention_weight_min": -43.637672424316406,
      "activations/layer18_attention_weight_max": 45.469932556152344,
      "activations/layer18_attention_weight_min": -39.49571990966797,
      "activations/layer19_attention_weight_max": 25.560895919799805,
      "activations/layer19_attention_weight_min": -24.001102447509766,
      "activations/layer1_attention_weight_max": 17.326032638549805,
      "activations/layer1_attention_weight_min": -12.973262786865234,
      "activations/layer20_attention_weight_max": 24.236587524414062,
      "activations/layer20_attention_weight_min": -23.425975799560547,
      "activations/layer21_attention_weight_max": 34.376800537109375,
      "activations/layer21_attention_weight_min": -22.967130661010742,
      "activations/layer22_attention_weight_max": 31.149906158447266,
      "activations/layer22_attention_weight_min": -25.14628791809082,
      "activations/layer23_attention_weight_max": 35.72308349609375,
      "activations/layer23_attention_weight_min": -23.560171127319336,
      "activations/layer2_attention_weight_max": 30.367300033569336,
      "activations/layer2_attention_weight_min": -29.574325561523438,
      "activations/layer3_attention_weight_max": 86.33929443359375,
      "activations/layer3_attention_weight_min": -81.68621063232422,
      "activations/layer4_attention_weight_max": 80.1559829711914,
      "activations/layer4_attention_weight_min": -80.35411834716797,
      "activations/layer5_attention_weight_max": 65.79857635498047,
      "activations/layer5_attention_weight_min": -69.17355346679688,
      "activations/layer6_attention_weight_max": 49.14909362792969,
      "activations/layer6_attention_weight_min": -49.43272018432617,
      "activations/layer7_attention_weight_max": 67.33271789550781,
      "activations/layer7_attention_weight_min": -66.5555419921875,
      "activations/layer8_attention_weight_max": 46.54008865356445,
      "activations/layer8_attention_weight_min": -50.244144439697266,
      "activations/layer9_attention_weight_max": 43.69769287109375,
      "activations/layer9_attention_weight_min": -45.57415008544922,
      "epoch": 19.42,
      "learning_rate": 2.4988636363636362e-05,
      "loss": 2.7304,
      "step": 334300
    },
    {
      "activations/layer0_attention_weight_max": 15.880478858947754,
      "activations/layer0_attention_weight_min": -13.705995559692383,
      "activations/layer10_attention_weight_max": 40.657081604003906,
      "activations/layer10_attention_weight_min": -40.427635192871094,
      "activations/layer11_attention_weight_max": 38.63222122192383,
      "activations/layer11_attention_weight_min": -37.5882568359375,
      "activations/layer12_attention_weight_max": 21.354124069213867,
      "activations/layer12_attention_weight_min": -25.453229904174805,
      "activations/layer13_attention_weight_max": 38.25963592529297,
      "activations/layer13_attention_weight_min": -32.508758544921875,
      "activations/layer14_attention_weight_max": 41.286041259765625,
      "activations/layer14_attention_weight_min": -31.80840301513672,
      "activations/layer15_attention_weight_max": 38.24140930175781,
      "activations/layer15_attention_weight_min": -31.705522537231445,
      "activations/layer16_attention_weight_max": 36.62354278564453,
      "activations/layer16_attention_weight_min": -34.16733169555664,
      "activations/layer17_attention_weight_max": 56.012935638427734,
      "activations/layer17_attention_weight_min": -51.106327056884766,
      "activations/layer18_attention_weight_max": 51.32305908203125,
      "activations/layer18_attention_weight_min": -43.72392272949219,
      "activations/layer19_attention_weight_max": 25.71504783630371,
      "activations/layer19_attention_weight_min": -25.765399932861328,
      "activations/layer1_attention_weight_max": 16.50139617919922,
      "activations/layer1_attention_weight_min": -13.192893981933594,
      "activations/layer20_attention_weight_max": 24.395915985107422,
      "activations/layer20_attention_weight_min": -25.594430923461914,
      "activations/layer21_attention_weight_max": 43.82896041870117,
      "activations/layer21_attention_weight_min": -30.11039161682129,
      "activations/layer22_attention_weight_max": 33.336483001708984,
      "activations/layer22_attention_weight_min": -26.60074806213379,
      "activations/layer23_attention_weight_max": 44.18253707885742,
      "activations/layer23_attention_weight_min": -23.637554168701172,
      "activations/layer2_attention_weight_max": 31.574697494506836,
      "activations/layer2_attention_weight_min": -29.558895111083984,
      "activations/layer3_attention_weight_max": 85.50045013427734,
      "activations/layer3_attention_weight_min": -84.13330078125,
      "activations/layer4_attention_weight_max": 84.35265350341797,
      "activations/layer4_attention_weight_min": -83.4300308227539,
      "activations/layer5_attention_weight_max": 63.241294860839844,
      "activations/layer5_attention_weight_min": -73.20887756347656,
      "activations/layer6_attention_weight_max": 54.379249572753906,
      "activations/layer6_attention_weight_min": -53.41139221191406,
      "activations/layer7_attention_weight_max": 69.3067626953125,
      "activations/layer7_attention_weight_min": -68.52411651611328,
      "activations/layer8_attention_weight_max": 56.07394027709961,
      "activations/layer8_attention_weight_min": -56.28007888793945,
      "activations/layer9_attention_weight_max": 54.59299850463867,
      "activations/layer9_attention_weight_min": -47.484066009521484,
      "epoch": 19.43,
      "learning_rate": 2.4969696969696967e-05,
      "loss": 2.7311,
      "step": 334350
    },
    {
      "activations/layer0_attention_weight_max": 14.84002685546875,
      "activations/layer0_attention_weight_min": -13.712428092956543,
      "activations/layer10_attention_weight_max": 35.93133544921875,
      "activations/layer10_attention_weight_min": -36.96891784667969,
      "activations/layer11_attention_weight_max": 32.034584045410156,
      "activations/layer11_attention_weight_min": -32.96002960205078,
      "activations/layer12_attention_weight_max": 19.94830322265625,
      "activations/layer12_attention_weight_min": -24.400232315063477,
      "activations/layer13_attention_weight_max": 40.30378341674805,
      "activations/layer13_attention_weight_min": -31.453460693359375,
      "activations/layer14_attention_weight_max": 37.0848274230957,
      "activations/layer14_attention_weight_min": -32.93995666503906,
      "activations/layer15_attention_weight_max": 37.90346145629883,
      "activations/layer15_attention_weight_min": -31.93456268310547,
      "activations/layer16_attention_weight_max": 34.33306884765625,
      "activations/layer16_attention_weight_min": -34.6186637878418,
      "activations/layer17_attention_weight_max": 51.74037551879883,
      "activations/layer17_attention_weight_min": -51.080047607421875,
      "activations/layer18_attention_weight_max": 47.84846115112305,
      "activations/layer18_attention_weight_min": -45.72498321533203,
      "activations/layer19_attention_weight_max": 26.79827880859375,
      "activations/layer19_attention_weight_min": -29.65102195739746,
      "activations/layer1_attention_weight_max": 15.629144668579102,
      "activations/layer1_attention_weight_min": -12.90278434753418,
      "activations/layer20_attention_weight_max": 28.157140731811523,
      "activations/layer20_attention_weight_min": -28.306663513183594,
      "activations/layer21_attention_weight_max": 42.55509567260742,
      "activations/layer21_attention_weight_min": -24.756864547729492,
      "activations/layer22_attention_weight_max": 28.02841567993164,
      "activations/layer22_attention_weight_min": -27.621566772460938,
      "activations/layer23_attention_weight_max": 36.17134094238281,
      "activations/layer23_attention_weight_min": -26.50354766845703,
      "activations/layer2_attention_weight_max": 30.25679588317871,
      "activations/layer2_attention_weight_min": -27.687904357910156,
      "activations/layer3_attention_weight_max": 81.30486297607422,
      "activations/layer3_attention_weight_min": -82.46965026855469,
      "activations/layer4_attention_weight_max": 80.08492279052734,
      "activations/layer4_attention_weight_min": -83.18521118164062,
      "activations/layer5_attention_weight_max": 64.12202453613281,
      "activations/layer5_attention_weight_min": -76.36015319824219,
      "activations/layer6_attention_weight_max": 47.05085372924805,
      "activations/layer6_attention_weight_min": -48.1491584777832,
      "activations/layer7_attention_weight_max": 65.42069244384766,
      "activations/layer7_attention_weight_min": -66.87076568603516,
      "activations/layer8_attention_weight_max": 46.7049560546875,
      "activations/layer8_attention_weight_min": -49.60481262207031,
      "activations/layer9_attention_weight_max": 44.990142822265625,
      "activations/layer9_attention_weight_min": -46.73044967651367,
      "epoch": 19.43,
      "learning_rate": 2.4950757575757575e-05,
      "loss": 2.7249,
      "step": 334400
    },
    {
      "activations/layer0_attention_weight_max": 14.904176712036133,
      "activations/layer0_attention_weight_min": -13.513678550720215,
      "activations/layer10_attention_weight_max": 36.60204315185547,
      "activations/layer10_attention_weight_min": -38.698333740234375,
      "activations/layer11_attention_weight_max": 33.567665100097656,
      "activations/layer11_attention_weight_min": -35.80567169189453,
      "activations/layer12_attention_weight_max": 19.887001037597656,
      "activations/layer12_attention_weight_min": -22.393613815307617,
      "activations/layer13_attention_weight_max": 34.494529724121094,
      "activations/layer13_attention_weight_min": -30.082704544067383,
      "activations/layer14_attention_weight_max": 36.79428482055664,
      "activations/layer14_attention_weight_min": -29.488168716430664,
      "activations/layer15_attention_weight_max": 36.4691162109375,
      "activations/layer15_attention_weight_min": -30.036046981811523,
      "activations/layer16_attention_weight_max": 34.23222351074219,
      "activations/layer16_attention_weight_min": -30.256914138793945,
      "activations/layer17_attention_weight_max": 55.51929473876953,
      "activations/layer17_attention_weight_min": -44.304901123046875,
      "activations/layer18_attention_weight_max": 51.1224479675293,
      "activations/layer18_attention_weight_min": -38.0236930847168,
      "activations/layer19_attention_weight_max": 25.348447799682617,
      "activations/layer19_attention_weight_min": -21.813385009765625,
      "activations/layer1_attention_weight_max": 16.885948181152344,
      "activations/layer1_attention_weight_min": -12.804227828979492,
      "activations/layer20_attention_weight_max": 25.08736801147461,
      "activations/layer20_attention_weight_min": -21.318681716918945,
      "activations/layer21_attention_weight_max": 40.23004150390625,
      "activations/layer21_attention_weight_min": -22.18896484375,
      "activations/layer22_attention_weight_max": 31.391944885253906,
      "activations/layer22_attention_weight_min": -23.803152084350586,
      "activations/layer23_attention_weight_max": 38.882266998291016,
      "activations/layer23_attention_weight_min": -23.972196578979492,
      "activations/layer2_attention_weight_max": 29.735576629638672,
      "activations/layer2_attention_weight_min": -29.173919677734375,
      "activations/layer3_attention_weight_max": 78.74919891357422,
      "activations/layer3_attention_weight_min": -83.17992401123047,
      "activations/layer4_attention_weight_max": 77.58739471435547,
      "activations/layer4_attention_weight_min": -80.88607788085938,
      "activations/layer5_attention_weight_max": 59.351707458496094,
      "activations/layer5_attention_weight_min": -71.08875274658203,
      "activations/layer6_attention_weight_max": 47.06776428222656,
      "activations/layer6_attention_weight_min": -47.45830154418945,
      "activations/layer7_attention_weight_max": 61.39761734008789,
      "activations/layer7_attention_weight_min": -63.759620666503906,
      "activations/layer8_attention_weight_max": 44.146488189697266,
      "activations/layer8_attention_weight_min": -49.78732681274414,
      "activations/layer9_attention_weight_max": 43.334712982177734,
      "activations/layer9_attention_weight_min": -45.6588020324707,
      "epoch": 19.43,
      "learning_rate": 2.4931818181818177e-05,
      "loss": 2.7257,
      "step": 334450
    },
    {
      "activations/layer0_attention_weight_max": 15.748834609985352,
      "activations/layer0_attention_weight_min": -13.916568756103516,
      "activations/layer10_attention_weight_max": 37.045528411865234,
      "activations/layer10_attention_weight_min": -36.00815200805664,
      "activations/layer11_attention_weight_max": 39.02079772949219,
      "activations/layer11_attention_weight_min": -37.5263671875,
      "activations/layer12_attention_weight_max": 19.670135498046875,
      "activations/layer12_attention_weight_min": -29.00357437133789,
      "activations/layer13_attention_weight_max": 39.41337966918945,
      "activations/layer13_attention_weight_min": -30.49264144897461,
      "activations/layer14_attention_weight_max": 34.8864631652832,
      "activations/layer14_attention_weight_min": -31.71820068359375,
      "activations/layer15_attention_weight_max": 33.25466537475586,
      "activations/layer15_attention_weight_min": -30.479270935058594,
      "activations/layer16_attention_weight_max": 32.854888916015625,
      "activations/layer16_attention_weight_min": -32.35813903808594,
      "activations/layer17_attention_weight_max": 54.69332504272461,
      "activations/layer17_attention_weight_min": -48.644588470458984,
      "activations/layer18_attention_weight_max": 47.36235809326172,
      "activations/layer18_attention_weight_min": -44.05686569213867,
      "activations/layer19_attention_weight_max": 24.056556701660156,
      "activations/layer19_attention_weight_min": -23.10194969177246,
      "activations/layer1_attention_weight_max": 15.69123363494873,
      "activations/layer1_attention_weight_min": -13.599945068359375,
      "activations/layer20_attention_weight_max": 22.51487922668457,
      "activations/layer20_attention_weight_min": -21.9700984954834,
      "activations/layer21_attention_weight_max": 37.54310607910156,
      "activations/layer21_attention_weight_min": -22.978118896484375,
      "activations/layer22_attention_weight_max": 25.862031936645508,
      "activations/layer22_attention_weight_min": -26.74755859375,
      "activations/layer23_attention_weight_max": 32.476280212402344,
      "activations/layer23_attention_weight_min": -24.49370574951172,
      "activations/layer2_attention_weight_max": 34.413909912109375,
      "activations/layer2_attention_weight_min": -32.42349624633789,
      "activations/layer3_attention_weight_max": 94.79389953613281,
      "activations/layer3_attention_weight_min": -87.59070587158203,
      "activations/layer4_attention_weight_max": 86.0009994506836,
      "activations/layer4_attention_weight_min": -84.25944519042969,
      "activations/layer5_attention_weight_max": 63.2854118347168,
      "activations/layer5_attention_weight_min": -72.40882873535156,
      "activations/layer6_attention_weight_max": 52.15485382080078,
      "activations/layer6_attention_weight_min": -53.22724151611328,
      "activations/layer7_attention_weight_max": 67.83765411376953,
      "activations/layer7_attention_weight_min": -65.42134857177734,
      "activations/layer8_attention_weight_max": 49.144901275634766,
      "activations/layer8_attention_weight_min": -52.819515228271484,
      "activations/layer9_attention_weight_max": 47.986392974853516,
      "activations/layer9_attention_weight_min": -47.055320739746094,
      "epoch": 19.44,
      "learning_rate": 2.4912878787878785e-05,
      "loss": 2.7347,
      "step": 334500
    },
    {
      "activations/layer0_attention_weight_max": 14.975973129272461,
      "activations/layer0_attention_weight_min": -13.856487274169922,
      "activations/layer10_attention_weight_max": 37.5125846862793,
      "activations/layer10_attention_weight_min": -37.40355682373047,
      "activations/layer11_attention_weight_max": 33.57162094116211,
      "activations/layer11_attention_weight_min": -35.34382629394531,
      "activations/layer12_attention_weight_max": 18.7227725982666,
      "activations/layer12_attention_weight_min": -28.624914169311523,
      "activations/layer13_attention_weight_max": 35.358642578125,
      "activations/layer13_attention_weight_min": -26.950185775756836,
      "activations/layer14_attention_weight_max": 36.52764129638672,
      "activations/layer14_attention_weight_min": -31.220619201660156,
      "activations/layer15_attention_weight_max": 35.29496765136719,
      "activations/layer15_attention_weight_min": -30.992488861083984,
      "activations/layer16_attention_weight_max": 36.4129524230957,
      "activations/layer16_attention_weight_min": -33.171661376953125,
      "activations/layer17_attention_weight_max": 55.51240921020508,
      "activations/layer17_attention_weight_min": -47.740631103515625,
      "activations/layer18_attention_weight_max": 51.2917366027832,
      "activations/layer18_attention_weight_min": -41.915504455566406,
      "activations/layer19_attention_weight_max": 21.778255462646484,
      "activations/layer19_attention_weight_min": -22.500879287719727,
      "activations/layer1_attention_weight_max": 15.814729690551758,
      "activations/layer1_attention_weight_min": -13.505016326904297,
      "activations/layer20_attention_weight_max": 22.09442138671875,
      "activations/layer20_attention_weight_min": -21.20589828491211,
      "activations/layer21_attention_weight_max": 31.18231201171875,
      "activations/layer21_attention_weight_min": -21.110137939453125,
      "activations/layer22_attention_weight_max": 28.2100772857666,
      "activations/layer22_attention_weight_min": -26.910921096801758,
      "activations/layer23_attention_weight_max": 36.653106689453125,
      "activations/layer23_attention_weight_min": -24.13319206237793,
      "activations/layer2_attention_weight_max": 32.3281135559082,
      "activations/layer2_attention_weight_min": -31.32001495361328,
      "activations/layer3_attention_weight_max": 87.9562759399414,
      "activations/layer3_attention_weight_min": -86.10012817382812,
      "activations/layer4_attention_weight_max": 84.87254333496094,
      "activations/layer4_attention_weight_min": -81.86308288574219,
      "activations/layer5_attention_weight_max": 60.19548797607422,
      "activations/layer5_attention_weight_min": -66.43850708007812,
      "activations/layer6_attention_weight_max": 49.63237762451172,
      "activations/layer6_attention_weight_min": -50.03103256225586,
      "activations/layer7_attention_weight_max": 60.77179718017578,
      "activations/layer7_attention_weight_min": -64.09790802001953,
      "activations/layer8_attention_weight_max": 49.94169998168945,
      "activations/layer8_attention_weight_min": -52.78694152832031,
      "activations/layer9_attention_weight_max": 48.33689498901367,
      "activations/layer9_attention_weight_min": -45.70187759399414,
      "epoch": 19.44,
      "learning_rate": 2.489393939393939e-05,
      "loss": 2.7266,
      "step": 334550
    },
    {
      "activations/layer0_attention_weight_max": 15.095842361450195,
      "activations/layer0_attention_weight_min": -13.92545223236084,
      "activations/layer10_attention_weight_max": 36.288841247558594,
      "activations/layer10_attention_weight_min": -36.336334228515625,
      "activations/layer11_attention_weight_max": 34.49136734008789,
      "activations/layer11_attention_weight_min": -34.21302795410156,
      "activations/layer12_attention_weight_max": 20.61260414123535,
      "activations/layer12_attention_weight_min": -26.744943618774414,
      "activations/layer13_attention_weight_max": 34.061363220214844,
      "activations/layer13_attention_weight_min": -27.976367950439453,
      "activations/layer14_attention_weight_max": 33.440406799316406,
      "activations/layer14_attention_weight_min": -29.00014877319336,
      "activations/layer15_attention_weight_max": 32.39406967163086,
      "activations/layer15_attention_weight_min": -29.269092559814453,
      "activations/layer16_attention_weight_max": 31.814510345458984,
      "activations/layer16_attention_weight_min": -29.403629302978516,
      "activations/layer17_attention_weight_max": 48.31224060058594,
      "activations/layer17_attention_weight_min": -44.34700012207031,
      "activations/layer18_attention_weight_max": 46.72917938232422,
      "activations/layer18_attention_weight_min": -39.49727249145508,
      "activations/layer19_attention_weight_max": 22.35089874267578,
      "activations/layer19_attention_weight_min": -25.54879379272461,
      "activations/layer1_attention_weight_max": 15.747757911682129,
      "activations/layer1_attention_weight_min": -13.077168464660645,
      "activations/layer20_attention_weight_max": 21.381885528564453,
      "activations/layer20_attention_weight_min": -26.638591766357422,
      "activations/layer21_attention_weight_max": 32.488807678222656,
      "activations/layer21_attention_weight_min": -25.380107879638672,
      "activations/layer22_attention_weight_max": 27.241928100585938,
      "activations/layer22_attention_weight_min": -24.89290428161621,
      "activations/layer23_attention_weight_max": 35.29575729370117,
      "activations/layer23_attention_weight_min": -24.246877670288086,
      "activations/layer2_attention_weight_max": 29.93878173828125,
      "activations/layer2_attention_weight_min": -28.61440086364746,
      "activations/layer3_attention_weight_max": 81.8633041381836,
      "activations/layer3_attention_weight_min": -83.42975616455078,
      "activations/layer4_attention_weight_max": 81.65459442138672,
      "activations/layer4_attention_weight_min": -82.13641357421875,
      "activations/layer5_attention_weight_max": 62.225608825683594,
      "activations/layer5_attention_weight_min": -71.98175048828125,
      "activations/layer6_attention_weight_max": 48.463130950927734,
      "activations/layer6_attention_weight_min": -50.41948318481445,
      "activations/layer7_attention_weight_max": 65.761962890625,
      "activations/layer7_attention_weight_min": -67.40589904785156,
      "activations/layer8_attention_weight_max": 48.06485366821289,
      "activations/layer8_attention_weight_min": -48.693511962890625,
      "activations/layer9_attention_weight_max": 46.67964172363281,
      "activations/layer9_attention_weight_min": -47.05065155029297,
      "epoch": 19.44,
      "learning_rate": 2.4874999999999998e-05,
      "loss": 2.7277,
      "step": 334600
    },
    {
      "activations/layer0_attention_weight_max": 15.198326110839844,
      "activations/layer0_attention_weight_min": -14.6215181350708,
      "activations/layer10_attention_weight_max": 31.37247085571289,
      "activations/layer10_attention_weight_min": -33.87642288208008,
      "activations/layer11_attention_weight_max": 28.5299129486084,
      "activations/layer11_attention_weight_min": -31.854984283447266,
      "activations/layer12_attention_weight_max": 18.207653045654297,
      "activations/layer12_attention_weight_min": -23.286054611206055,
      "activations/layer13_attention_weight_max": 37.75120544433594,
      "activations/layer13_attention_weight_min": -24.85087013244629,
      "activations/layer14_attention_weight_max": 38.15898895263672,
      "activations/layer14_attention_weight_min": -28.99275016784668,
      "activations/layer15_attention_weight_max": 35.5364875793457,
      "activations/layer15_attention_weight_min": -31.00020980834961,
      "activations/layer16_attention_weight_max": 34.673797607421875,
      "activations/layer16_attention_weight_min": -32.574527740478516,
      "activations/layer17_attention_weight_max": 52.057823181152344,
      "activations/layer17_attention_weight_min": -43.72853469848633,
      "activations/layer18_attention_weight_max": 50.621456146240234,
      "activations/layer18_attention_weight_min": -40.58896255493164,
      "activations/layer19_attention_weight_max": 23.950439453125,
      "activations/layer19_attention_weight_min": -23.815326690673828,
      "activations/layer1_attention_weight_max": 15.595086097717285,
      "activations/layer1_attention_weight_min": -12.522149085998535,
      "activations/layer20_attention_weight_max": 23.224454879760742,
      "activations/layer20_attention_weight_min": -21.25405502319336,
      "activations/layer21_attention_weight_max": 36.69063186645508,
      "activations/layer21_attention_weight_min": -21.1197452545166,
      "activations/layer22_attention_weight_max": 28.834672927856445,
      "activations/layer22_attention_weight_min": -24.8970947265625,
      "activations/layer23_attention_weight_max": 36.5855712890625,
      "activations/layer23_attention_weight_min": -26.725048065185547,
      "activations/layer2_attention_weight_max": 30.245376586914062,
      "activations/layer2_attention_weight_min": -28.124439239501953,
      "activations/layer3_attention_weight_max": 81.2414779663086,
      "activations/layer3_attention_weight_min": -83.97301483154297,
      "activations/layer4_attention_weight_max": 80.1860580444336,
      "activations/layer4_attention_weight_min": -80.75786590576172,
      "activations/layer5_attention_weight_max": 61.57209777832031,
      "activations/layer5_attention_weight_min": -69.0759506225586,
      "activations/layer6_attention_weight_max": 45.31218719482422,
      "activations/layer6_attention_weight_min": -46.774051666259766,
      "activations/layer7_attention_weight_max": 62.6632080078125,
      "activations/layer7_attention_weight_min": -56.642269134521484,
      "activations/layer8_attention_weight_max": 41.51127243041992,
      "activations/layer8_attention_weight_min": -48.71095657348633,
      "activations/layer9_attention_weight_max": 37.42790603637695,
      "activations/layer9_attention_weight_min": -42.725589752197266,
      "epoch": 19.45,
      "learning_rate": 2.4856060606060606e-05,
      "loss": 2.7288,
      "step": 334650
    },
    {
      "activations/layer0_attention_weight_max": 16.413312911987305,
      "activations/layer0_attention_weight_min": -14.12807846069336,
      "activations/layer10_attention_weight_max": 34.61213302612305,
      "activations/layer10_attention_weight_min": -35.583717346191406,
      "activations/layer11_attention_weight_max": 33.7602653503418,
      "activations/layer11_attention_weight_min": -33.76641082763672,
      "activations/layer12_attention_weight_max": 18.47258949279785,
      "activations/layer12_attention_weight_min": -22.51583480834961,
      "activations/layer13_attention_weight_max": 37.23474884033203,
      "activations/layer13_attention_weight_min": -28.201683044433594,
      "activations/layer14_attention_weight_max": 35.25695037841797,
      "activations/layer14_attention_weight_min": -31.08030891418457,
      "activations/layer15_attention_weight_max": 34.49428176879883,
      "activations/layer15_attention_weight_min": -30.6766300201416,
      "activations/layer16_attention_weight_max": 37.65142822265625,
      "activations/layer16_attention_weight_min": -35.06435012817383,
      "activations/layer17_attention_weight_max": 54.06520462036133,
      "activations/layer17_attention_weight_min": -46.9340705871582,
      "activations/layer18_attention_weight_max": 51.04352951049805,
      "activations/layer18_attention_weight_min": -43.079593658447266,
      "activations/layer19_attention_weight_max": 27.205717086791992,
      "activations/layer19_attention_weight_min": -25.285797119140625,
      "activations/layer1_attention_weight_max": 16.223852157592773,
      "activations/layer1_attention_weight_min": -13.414874076843262,
      "activations/layer20_attention_weight_max": 26.96333885192871,
      "activations/layer20_attention_weight_min": -22.44915199279785,
      "activations/layer21_attention_weight_max": 37.79378128051758,
      "activations/layer21_attention_weight_min": -22.86960792541504,
      "activations/layer22_attention_weight_max": 29.501070022583008,
      "activations/layer22_attention_weight_min": -25.72715950012207,
      "activations/layer23_attention_weight_max": 42.82407760620117,
      "activations/layer23_attention_weight_min": -24.355152130126953,
      "activations/layer2_attention_weight_max": 29.961894989013672,
      "activations/layer2_attention_weight_min": -30.503292083740234,
      "activations/layer3_attention_weight_max": 83.83937072753906,
      "activations/layer3_attention_weight_min": -83.7757797241211,
      "activations/layer4_attention_weight_max": 78.58943939208984,
      "activations/layer4_attention_weight_min": -81.20337677001953,
      "activations/layer5_attention_weight_max": 59.342491149902344,
      "activations/layer5_attention_weight_min": -68.88253021240234,
      "activations/layer6_attention_weight_max": 45.36888122558594,
      "activations/layer6_attention_weight_min": -50.75343704223633,
      "activations/layer7_attention_weight_max": 60.8042106628418,
      "activations/layer7_attention_weight_min": -64.6508560180664,
      "activations/layer8_attention_weight_max": 45.995758056640625,
      "activations/layer8_attention_weight_min": -50.39006042480469,
      "activations/layer9_attention_weight_max": 43.71611404418945,
      "activations/layer9_attention_weight_min": -45.05105972290039,
      "epoch": 19.45,
      "learning_rate": 2.4837121212121208e-05,
      "loss": 2.7161,
      "step": 334700
    },
    {
      "activations/layer0_attention_weight_max": 14.948200225830078,
      "activations/layer0_attention_weight_min": -14.406817436218262,
      "activations/layer10_attention_weight_max": 35.51009750366211,
      "activations/layer10_attention_weight_min": -33.299564361572266,
      "activations/layer11_attention_weight_max": 33.407981872558594,
      "activations/layer11_attention_weight_min": -34.61583709716797,
      "activations/layer12_attention_weight_max": 18.032011032104492,
      "activations/layer12_attention_weight_min": -25.643171310424805,
      "activations/layer13_attention_weight_max": 31.25959014892578,
      "activations/layer13_attention_weight_min": -25.417097091674805,
      "activations/layer14_attention_weight_max": 32.46175765991211,
      "activations/layer14_attention_weight_min": -29.001087188720703,
      "activations/layer15_attention_weight_max": 29.66100311279297,
      "activations/layer15_attention_weight_min": -30.01350975036621,
      "activations/layer16_attention_weight_max": 31.93379783630371,
      "activations/layer16_attention_weight_min": -30.233312606811523,
      "activations/layer17_attention_weight_max": 47.603904724121094,
      "activations/layer17_attention_weight_min": -42.26570510864258,
      "activations/layer18_attention_weight_max": 43.17479705810547,
      "activations/layer18_attention_weight_min": -39.08140563964844,
      "activations/layer19_attention_weight_max": 22.617412567138672,
      "activations/layer19_attention_weight_min": -24.196096420288086,
      "activations/layer1_attention_weight_max": 15.6129732131958,
      "activations/layer1_attention_weight_min": -12.823054313659668,
      "activations/layer20_attention_weight_max": 19.94353485107422,
      "activations/layer20_attention_weight_min": -22.37655258178711,
      "activations/layer21_attention_weight_max": 33.580596923828125,
      "activations/layer21_attention_weight_min": -21.586210250854492,
      "activations/layer22_attention_weight_max": 26.9405517578125,
      "activations/layer22_attention_weight_min": -25.570371627807617,
      "activations/layer23_attention_weight_max": 34.112892150878906,
      "activations/layer23_attention_weight_min": -26.397541046142578,
      "activations/layer2_attention_weight_max": 30.93838882446289,
      "activations/layer2_attention_weight_min": -29.19393539428711,
      "activations/layer3_attention_weight_max": 84.20487213134766,
      "activations/layer3_attention_weight_min": -80.57693481445312,
      "activations/layer4_attention_weight_max": 80.74076080322266,
      "activations/layer4_attention_weight_min": -77.3198471069336,
      "activations/layer5_attention_weight_max": 60.97201156616211,
      "activations/layer5_attention_weight_min": -68.91456604003906,
      "activations/layer6_attention_weight_max": 46.49931716918945,
      "activations/layer6_attention_weight_min": -45.357994079589844,
      "activations/layer7_attention_weight_max": 63.11630630493164,
      "activations/layer7_attention_weight_min": -60.612857818603516,
      "activations/layer8_attention_weight_max": 45.399940490722656,
      "activations/layer8_attention_weight_min": -46.27814865112305,
      "activations/layer9_attention_weight_max": 43.6697883605957,
      "activations/layer9_attention_weight_min": -44.46336364746094,
      "epoch": 19.45,
      "learning_rate": 2.4818181818181816e-05,
      "loss": 2.7251,
      "step": 334750
    },
    {
      "activations/layer0_attention_weight_max": 15.174056053161621,
      "activations/layer0_attention_weight_min": -14.061910629272461,
      "activations/layer10_attention_weight_max": 43.92939376831055,
      "activations/layer10_attention_weight_min": -38.50871276855469,
      "activations/layer11_attention_weight_max": 41.56392288208008,
      "activations/layer11_attention_weight_min": -37.50458526611328,
      "activations/layer12_attention_weight_max": 18.41214942932129,
      "activations/layer12_attention_weight_min": -26.618207931518555,
      "activations/layer13_attention_weight_max": 50.70684051513672,
      "activations/layer13_attention_weight_min": -29.37444305419922,
      "activations/layer14_attention_weight_max": 39.82133483886719,
      "activations/layer14_attention_weight_min": -30.132516860961914,
      "activations/layer15_attention_weight_max": 42.05087661743164,
      "activations/layer15_attention_weight_min": -31.58707618713379,
      "activations/layer16_attention_weight_max": 37.87992477416992,
      "activations/layer16_attention_weight_min": -34.23447036743164,
      "activations/layer17_attention_weight_max": 58.73196029663086,
      "activations/layer17_attention_weight_min": -53.64390182495117,
      "activations/layer18_attention_weight_max": 49.78157424926758,
      "activations/layer18_attention_weight_min": -48.880943298339844,
      "activations/layer19_attention_weight_max": 25.545448303222656,
      "activations/layer19_attention_weight_min": -24.851539611816406,
      "activations/layer1_attention_weight_max": 16.54205322265625,
      "activations/layer1_attention_weight_min": -15.363799095153809,
      "activations/layer20_attention_weight_max": 25.639402389526367,
      "activations/layer20_attention_weight_min": -23.34019660949707,
      "activations/layer21_attention_weight_max": 40.65643310546875,
      "activations/layer21_attention_weight_min": -21.07969856262207,
      "activations/layer22_attention_weight_max": 31.398042678833008,
      "activations/layer22_attention_weight_min": -24.05732536315918,
      "activations/layer23_attention_weight_max": 34.79508972167969,
      "activations/layer23_attention_weight_min": -23.361608505249023,
      "activations/layer2_attention_weight_max": 33.533382415771484,
      "activations/layer2_attention_weight_min": -29.858661651611328,
      "activations/layer3_attention_weight_max": 89.08243560791016,
      "activations/layer3_attention_weight_min": -88.4485855102539,
      "activations/layer4_attention_weight_max": 84.20179748535156,
      "activations/layer4_attention_weight_min": -81.89627075195312,
      "activations/layer5_attention_weight_max": 66.13768005371094,
      "activations/layer5_attention_weight_min": -69.55111694335938,
      "activations/layer6_attention_weight_max": 51.792808532714844,
      "activations/layer6_attention_weight_min": -50.37529754638672,
      "activations/layer7_attention_weight_max": 76.63072967529297,
      "activations/layer7_attention_weight_min": -69.34770965576172,
      "activations/layer8_attention_weight_max": 54.20292663574219,
      "activations/layer8_attention_weight_min": -50.21061706542969,
      "activations/layer9_attention_weight_max": 55.32783126831055,
      "activations/layer9_attention_weight_min": -51.933631896972656,
      "epoch": 19.45,
      "learning_rate": 2.479924242424242e-05,
      "loss": 2.733,
      "step": 334800
    },
    {
      "activations/layer0_attention_weight_max": 14.735770225524902,
      "activations/layer0_attention_weight_min": -13.891277313232422,
      "activations/layer10_attention_weight_max": 38.29176712036133,
      "activations/layer10_attention_weight_min": -34.782196044921875,
      "activations/layer11_attention_weight_max": 32.254127502441406,
      "activations/layer11_attention_weight_min": -34.10304260253906,
      "activations/layer12_attention_weight_max": 17.463359832763672,
      "activations/layer12_attention_weight_min": -25.228511810302734,
      "activations/layer13_attention_weight_max": 31.994056701660156,
      "activations/layer13_attention_weight_min": -26.097307205200195,
      "activations/layer14_attention_weight_max": 37.070167541503906,
      "activations/layer14_attention_weight_min": -29.750566482543945,
      "activations/layer15_attention_weight_max": 33.73375701904297,
      "activations/layer15_attention_weight_min": -30.807600021362305,
      "activations/layer16_attention_weight_max": 34.822757720947266,
      "activations/layer16_attention_weight_min": -32.78314208984375,
      "activations/layer17_attention_weight_max": 53.72997283935547,
      "activations/layer17_attention_weight_min": -47.754913330078125,
      "activations/layer18_attention_weight_max": 49.024383544921875,
      "activations/layer18_attention_weight_min": -40.30409240722656,
      "activations/layer19_attention_weight_max": 25.66841697692871,
      "activations/layer19_attention_weight_min": -22.335718154907227,
      "activations/layer1_attention_weight_max": 16.874454498291016,
      "activations/layer1_attention_weight_min": -13.0916748046875,
      "activations/layer20_attention_weight_max": 22.078815460205078,
      "activations/layer20_attention_weight_min": -22.96843719482422,
      "activations/layer21_attention_weight_max": 35.903167724609375,
      "activations/layer21_attention_weight_min": -27.22987937927246,
      "activations/layer22_attention_weight_max": 33.091854095458984,
      "activations/layer22_attention_weight_min": -25.733003616333008,
      "activations/layer23_attention_weight_max": 38.96833419799805,
      "activations/layer23_attention_weight_min": -24.01058006286621,
      "activations/layer2_attention_weight_max": 29.86566162109375,
      "activations/layer2_attention_weight_min": -28.441923141479492,
      "activations/layer3_attention_weight_max": 83.51141357421875,
      "activations/layer3_attention_weight_min": -81.65604400634766,
      "activations/layer4_attention_weight_max": 82.34574890136719,
      "activations/layer4_attention_weight_min": -78.66869354248047,
      "activations/layer5_attention_weight_max": 60.561126708984375,
      "activations/layer5_attention_weight_min": -68.96952056884766,
      "activations/layer6_attention_weight_max": 48.61104202270508,
      "activations/layer6_attention_weight_min": -47.434688568115234,
      "activations/layer7_attention_weight_max": 67.19893646240234,
      "activations/layer7_attention_weight_min": -60.81170654296875,
      "activations/layer8_attention_weight_max": 46.56569290161133,
      "activations/layer8_attention_weight_min": -50.08072280883789,
      "activations/layer9_attention_weight_max": 46.74560546875,
      "activations/layer9_attention_weight_min": -44.05250930786133,
      "epoch": 19.46,
      "learning_rate": 2.478030303030303e-05,
      "loss": 2.7206,
      "step": 334850
    },
    {
      "activations/layer0_attention_weight_max": 15.002351760864258,
      "activations/layer0_attention_weight_min": -13.947820663452148,
      "activations/layer10_attention_weight_max": 35.397666931152344,
      "activations/layer10_attention_weight_min": -34.30658721923828,
      "activations/layer11_attention_weight_max": 33.15890884399414,
      "activations/layer11_attention_weight_min": -32.459083557128906,
      "activations/layer12_attention_weight_max": 17.67594337463379,
      "activations/layer12_attention_weight_min": -24.92850685119629,
      "activations/layer13_attention_weight_max": 33.43058776855469,
      "activations/layer13_attention_weight_min": -29.729454040527344,
      "activations/layer14_attention_weight_max": 34.62777328491211,
      "activations/layer14_attention_weight_min": -30.882205963134766,
      "activations/layer15_attention_weight_max": 31.14722442626953,
      "activations/layer15_attention_weight_min": -29.309839248657227,
      "activations/layer16_attention_weight_max": 35.352684020996094,
      "activations/layer16_attention_weight_min": -32.77863693237305,
      "activations/layer17_attention_weight_max": 52.229591369628906,
      "activations/layer17_attention_weight_min": -45.88719177246094,
      "activations/layer18_attention_weight_max": 48.84491729736328,
      "activations/layer18_attention_weight_min": -39.563297271728516,
      "activations/layer19_attention_weight_max": 25.00713348388672,
      "activations/layer19_attention_weight_min": -20.2719783782959,
      "activations/layer1_attention_weight_max": 15.81752872467041,
      "activations/layer1_attention_weight_min": -13.324640274047852,
      "activations/layer20_attention_weight_max": 22.441165924072266,
      "activations/layer20_attention_weight_min": -20.460346221923828,
      "activations/layer21_attention_weight_max": 34.230224609375,
      "activations/layer21_attention_weight_min": -21.825237274169922,
      "activations/layer22_attention_weight_max": 29.92570686340332,
      "activations/layer22_attention_weight_min": -26.255735397338867,
      "activations/layer23_attention_weight_max": 35.792076110839844,
      "activations/layer23_attention_weight_min": -25.13189697265625,
      "activations/layer2_attention_weight_max": 31.402202606201172,
      "activations/layer2_attention_weight_min": -29.783544540405273,
      "activations/layer3_attention_weight_max": 85.46564483642578,
      "activations/layer3_attention_weight_min": -85.33318328857422,
      "activations/layer4_attention_weight_max": 82.81233978271484,
      "activations/layer4_attention_weight_min": -85.20481872558594,
      "activations/layer5_attention_weight_max": 61.52033233642578,
      "activations/layer5_attention_weight_min": -72.10537719726562,
      "activations/layer6_attention_weight_max": 47.54610824584961,
      "activations/layer6_attention_weight_min": -48.12302780151367,
      "activations/layer7_attention_weight_max": 64.6359634399414,
      "activations/layer7_attention_weight_min": -59.86281204223633,
      "activations/layer8_attention_weight_max": 45.437767028808594,
      "activations/layer8_attention_weight_min": -49.88752746582031,
      "activations/layer9_attention_weight_max": 43.15670394897461,
      "activations/layer9_attention_weight_min": -41.49048614501953,
      "epoch": 19.46,
      "learning_rate": 2.4761363636363634e-05,
      "loss": 2.7246,
      "step": 334900
    },
    {
      "activations/layer0_attention_weight_max": 15.425942420959473,
      "activations/layer0_attention_weight_min": -13.927226066589355,
      "activations/layer10_attention_weight_max": 34.32752227783203,
      "activations/layer10_attention_weight_min": -38.54783630371094,
      "activations/layer11_attention_weight_max": 34.14292526245117,
      "activations/layer11_attention_weight_min": -38.39188766479492,
      "activations/layer12_attention_weight_max": 20.27963638305664,
      "activations/layer12_attention_weight_min": -27.607309341430664,
      "activations/layer13_attention_weight_max": 34.52994918823242,
      "activations/layer13_attention_weight_min": -28.780845642089844,
      "activations/layer14_attention_weight_max": 37.211204528808594,
      "activations/layer14_attention_weight_min": -34.29058837890625,
      "activations/layer15_attention_weight_max": 35.958595275878906,
      "activations/layer15_attention_weight_min": -32.612998962402344,
      "activations/layer16_attention_weight_max": 36.2473258972168,
      "activations/layer16_attention_weight_min": -35.99363327026367,
      "activations/layer17_attention_weight_max": 53.53191375732422,
      "activations/layer17_attention_weight_min": -47.26393508911133,
      "activations/layer18_attention_weight_max": 51.31956100463867,
      "activations/layer18_attention_weight_min": -44.69426727294922,
      "activations/layer19_attention_weight_max": 25.340734481811523,
      "activations/layer19_attention_weight_min": -25.042558670043945,
      "activations/layer1_attention_weight_max": 14.33697509765625,
      "activations/layer1_attention_weight_min": -15.28338623046875,
      "activations/layer20_attention_weight_max": 23.948637008666992,
      "activations/layer20_attention_weight_min": -23.820289611816406,
      "activations/layer21_attention_weight_max": 38.82852554321289,
      "activations/layer21_attention_weight_min": -24.418804168701172,
      "activations/layer22_attention_weight_max": 29.012483596801758,
      "activations/layer22_attention_weight_min": -29.812110900878906,
      "activations/layer23_attention_weight_max": 34.14567184448242,
      "activations/layer23_attention_weight_min": -30.241928100585938,
      "activations/layer2_attention_weight_max": 30.825773239135742,
      "activations/layer2_attention_weight_min": -29.266677856445312,
      "activations/layer3_attention_weight_max": 85.07170867919922,
      "activations/layer3_attention_weight_min": -85.98912048339844,
      "activations/layer4_attention_weight_max": 81.87633514404297,
      "activations/layer4_attention_weight_min": -80.65135192871094,
      "activations/layer5_attention_weight_max": 63.714744567871094,
      "activations/layer5_attention_weight_min": -68.29420471191406,
      "activations/layer6_attention_weight_max": 48.74848175048828,
      "activations/layer6_attention_weight_min": -48.135868072509766,
      "activations/layer7_attention_weight_max": 64.89518737792969,
      "activations/layer7_attention_weight_min": -60.572017669677734,
      "activations/layer8_attention_weight_max": 47.124942779541016,
      "activations/layer8_attention_weight_min": -48.390296936035156,
      "activations/layer9_attention_weight_max": 45.77296829223633,
      "activations/layer9_attention_weight_min": -44.474605560302734,
      "epoch": 19.46,
      "learning_rate": 2.4742424242424242e-05,
      "loss": 2.7201,
      "step": 334950
    },
    {
      "activations/layer0_attention_weight_max": 15.553678512573242,
      "activations/layer0_attention_weight_min": -14.051469802856445,
      "activations/layer10_attention_weight_max": 35.701324462890625,
      "activations/layer10_attention_weight_min": -34.89735412597656,
      "activations/layer11_attention_weight_max": 31.253477096557617,
      "activations/layer11_attention_weight_min": -33.292396545410156,
      "activations/layer12_attention_weight_max": 18.082279205322266,
      "activations/layer12_attention_weight_min": -31.219932556152344,
      "activations/layer13_attention_weight_max": 33.338314056396484,
      "activations/layer13_attention_weight_min": -27.92276382446289,
      "activations/layer14_attention_weight_max": 33.92234802246094,
      "activations/layer14_attention_weight_min": -31.14789581298828,
      "activations/layer15_attention_weight_max": 31.198274612426758,
      "activations/layer15_attention_weight_min": -32.080745697021484,
      "activations/layer16_attention_weight_max": 32.976654052734375,
      "activations/layer16_attention_weight_min": -32.68016815185547,
      "activations/layer17_attention_weight_max": 47.8497200012207,
      "activations/layer17_attention_weight_min": -45.107566833496094,
      "activations/layer18_attention_weight_max": 44.04512023925781,
      "activations/layer18_attention_weight_min": -40.4370231628418,
      "activations/layer19_attention_weight_max": 22.71369743347168,
      "activations/layer19_attention_weight_min": -24.237764358520508,
      "activations/layer1_attention_weight_max": 15.260926246643066,
      "activations/layer1_attention_weight_min": -13.222126960754395,
      "activations/layer20_attention_weight_max": 21.51563262939453,
      "activations/layer20_attention_weight_min": -22.518081665039062,
      "activations/layer21_attention_weight_max": 34.2227783203125,
      "activations/layer21_attention_weight_min": -23.078243255615234,
      "activations/layer22_attention_weight_max": 25.9177303314209,
      "activations/layer22_attention_weight_min": -24.257003784179688,
      "activations/layer23_attention_weight_max": 33.78248977661133,
      "activations/layer23_attention_weight_min": -22.339744567871094,
      "activations/layer2_attention_weight_max": 32.767730712890625,
      "activations/layer2_attention_weight_min": -30.283851623535156,
      "activations/layer3_attention_weight_max": 84.64757537841797,
      "activations/layer3_attention_weight_min": -85.82022857666016,
      "activations/layer4_attention_weight_max": 82.14204406738281,
      "activations/layer4_attention_weight_min": -80.19426727294922,
      "activations/layer5_attention_weight_max": 63.224456787109375,
      "activations/layer5_attention_weight_min": -67.70057678222656,
      "activations/layer6_attention_weight_max": 50.40102005004883,
      "activations/layer6_attention_weight_min": -50.436241149902344,
      "activations/layer7_attention_weight_max": 66.4551010131836,
      "activations/layer7_attention_weight_min": -66.1172866821289,
      "activations/layer8_attention_weight_max": 47.73960876464844,
      "activations/layer8_attention_weight_min": -50.801513671875,
      "activations/layer9_attention_weight_max": 47.567222595214844,
      "activations/layer9_attention_weight_min": -45.16731643676758,
      "epoch": 19.47,
      "learning_rate": 2.4723484848484844e-05,
      "loss": 2.7307,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.556,
      "eval_samples_per_second": 501.867,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.556,
      "eval_openwebtext_samples_per_second": 501.867,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.0491,
      "eval_wikitext_samples_per_second": 222.532,
      "step": 335000
    },
    {
      "epoch": 19.47,
      "eval_lambada_loss": 2.416015625,
      "eval_lambada_ppl": 11.201140741117625,
      "eval_lambada_runtime": 9.6201,
      "eval_lambada_samples_per_second": 506.13,
      "step": 335000
    },
    {
      "activations/layer0_attention_weight_max": 14.924273490905762,
      "activations/layer0_attention_weight_min": -13.76063060760498,
      "activations/layer10_attention_weight_max": 35.554176330566406,
      "activations/layer10_attention_weight_min": -35.876373291015625,
      "activations/layer11_attention_weight_max": 32.306617736816406,
      "activations/layer11_attention_weight_min": -33.396488189697266,
      "activations/layer12_attention_weight_max": 18.136028289794922,
      "activations/layer12_attention_weight_min": -23.177701950073242,
      "activations/layer13_attention_weight_max": 33.23469924926758,
      "activations/layer13_attention_weight_min": -28.812442779541016,
      "activations/layer14_attention_weight_max": 33.57762145996094,
      "activations/layer14_attention_weight_min": -29.120277404785156,
      "activations/layer15_attention_weight_max": 32.435707092285156,
      "activations/layer15_attention_weight_min": -31.153303146362305,
      "activations/layer16_attention_weight_max": 33.357391357421875,
      "activations/layer16_attention_weight_min": -33.57949447631836,
      "activations/layer17_attention_weight_max": 47.686805725097656,
      "activations/layer17_attention_weight_min": -42.97198486328125,
      "activations/layer18_attention_weight_max": 44.04814910888672,
      "activations/layer18_attention_weight_min": -40.341033935546875,
      "activations/layer19_attention_weight_max": 23.508583068847656,
      "activations/layer19_attention_weight_min": -22.864561080932617,
      "activations/layer1_attention_weight_max": 15.236732482910156,
      "activations/layer1_attention_weight_min": -13.146313667297363,
      "activations/layer20_attention_weight_max": 21.480220794677734,
      "activations/layer20_attention_weight_min": -21.97564697265625,
      "activations/layer21_attention_weight_max": 32.088382720947266,
      "activations/layer21_attention_weight_min": -24.177453994750977,
      "activations/layer22_attention_weight_max": 27.306360244750977,
      "activations/layer22_attention_weight_min": -24.8563232421875,
      "activations/layer23_attention_weight_max": 32.98242950439453,
      "activations/layer23_attention_weight_min": -23.886999130249023,
      "activations/layer2_attention_weight_max": 30.986343383789062,
      "activations/layer2_attention_weight_min": -30.364898681640625,
      "activations/layer3_attention_weight_max": 83.99701690673828,
      "activations/layer3_attention_weight_min": -84.83847045898438,
      "activations/layer4_attention_weight_max": 82.61614227294922,
      "activations/layer4_attention_weight_min": -82.54025268554688,
      "activations/layer5_attention_weight_max": 60.921607971191406,
      "activations/layer5_attention_weight_min": -69.20967102050781,
      "activations/layer6_attention_weight_max": 48.44928741455078,
      "activations/layer6_attention_weight_min": -50.930076599121094,
      "activations/layer7_attention_weight_max": 62.80588912963867,
      "activations/layer7_attention_weight_min": -63.74555206298828,
      "activations/layer8_attention_weight_max": 45.43096160888672,
      "activations/layer8_attention_weight_min": -49.06916809082031,
      "activations/layer9_attention_weight_max": 40.55133056640625,
      "activations/layer9_attention_weight_min": -44.78486251831055,
      "epoch": 19.47,
      "learning_rate": 2.4704545454545452e-05,
      "loss": 2.7188,
      "step": 335050
    },
    {
      "activations/layer0_attention_weight_max": 15.276259422302246,
      "activations/layer0_attention_weight_min": -13.912317276000977,
      "activations/layer10_attention_weight_max": 37.79754638671875,
      "activations/layer10_attention_weight_min": -38.05557632446289,
      "activations/layer11_attention_weight_max": 34.23206329345703,
      "activations/layer11_attention_weight_min": -37.51990509033203,
      "activations/layer12_attention_weight_max": 19.974578857421875,
      "activations/layer12_attention_weight_min": -24.26500129699707,
      "activations/layer13_attention_weight_max": 36.49079513549805,
      "activations/layer13_attention_weight_min": -27.30679702758789,
      "activations/layer14_attention_weight_max": 37.64702224731445,
      "activations/layer14_attention_weight_min": -30.98183822631836,
      "activations/layer15_attention_weight_max": 34.238990783691406,
      "activations/layer15_attention_weight_min": -31.25041389465332,
      "activations/layer16_attention_weight_max": 37.89177703857422,
      "activations/layer16_attention_weight_min": -35.23412322998047,
      "activations/layer17_attention_weight_max": 52.40168762207031,
      "activations/layer17_attention_weight_min": -48.86741638183594,
      "activations/layer18_attention_weight_max": 49.57293701171875,
      "activations/layer18_attention_weight_min": -44.66526412963867,
      "activations/layer19_attention_weight_max": 25.12519073486328,
      "activations/layer19_attention_weight_min": -23.358999252319336,
      "activations/layer1_attention_weight_max": 15.331178665161133,
      "activations/layer1_attention_weight_min": -13.9297456741333,
      "activations/layer20_attention_weight_max": 24.994672775268555,
      "activations/layer20_attention_weight_min": -21.16356658935547,
      "activations/layer21_attention_weight_max": 37.0673828125,
      "activations/layer21_attention_weight_min": -22.556936264038086,
      "activations/layer22_attention_weight_max": 29.74599838256836,
      "activations/layer22_attention_weight_min": -27.87040138244629,
      "activations/layer23_attention_weight_max": 37.24925231933594,
      "activations/layer23_attention_weight_min": -24.45269012451172,
      "activations/layer2_attention_weight_max": 31.214445114135742,
      "activations/layer2_attention_weight_min": -31.6029052734375,
      "activations/layer3_attention_weight_max": 86.24131774902344,
      "activations/layer3_attention_weight_min": -87.90859985351562,
      "activations/layer4_attention_weight_max": 83.99798583984375,
      "activations/layer4_attention_weight_min": -82.93441009521484,
      "activations/layer5_attention_weight_max": 63.365501403808594,
      "activations/layer5_attention_weight_min": -72.8856430053711,
      "activations/layer6_attention_weight_max": 51.038692474365234,
      "activations/layer6_attention_weight_min": -51.604515075683594,
      "activations/layer7_attention_weight_max": 71.74506378173828,
      "activations/layer7_attention_weight_min": -71.2527847290039,
      "activations/layer8_attention_weight_max": 52.581058502197266,
      "activations/layer8_attention_weight_min": -56.79082489013672,
      "activations/layer9_attention_weight_max": 50.25464630126953,
      "activations/layer9_attention_weight_min": -51.76886749267578,
      "epoch": 19.47,
      "learning_rate": 2.4685606060606057e-05,
      "loss": 2.7318,
      "step": 335100
    },
    {
      "activations/layer0_attention_weight_max": 15.68873119354248,
      "activations/layer0_attention_weight_min": -13.999038696289062,
      "activations/layer10_attention_weight_max": 35.8275032043457,
      "activations/layer10_attention_weight_min": -35.503597259521484,
      "activations/layer11_attention_weight_max": 31.750608444213867,
      "activations/layer11_attention_weight_min": -34.06330108642578,
      "activations/layer12_attention_weight_max": 18.6802978515625,
      "activations/layer12_attention_weight_min": -28.089744567871094,
      "activations/layer13_attention_weight_max": 32.31207275390625,
      "activations/layer13_attention_weight_min": -27.84374237060547,
      "activations/layer14_attention_weight_max": 34.351009368896484,
      "activations/layer14_attention_weight_min": -30.96762466430664,
      "activations/layer15_attention_weight_max": 30.469770431518555,
      "activations/layer15_attention_weight_min": -29.53481674194336,
      "activations/layer16_attention_weight_max": 33.6399040222168,
      "activations/layer16_attention_weight_min": -32.4362678527832,
      "activations/layer17_attention_weight_max": 49.15794372558594,
      "activations/layer17_attention_weight_min": -44.34150314331055,
      "activations/layer18_attention_weight_max": 46.28820037841797,
      "activations/layer18_attention_weight_min": -42.64228439331055,
      "activations/layer19_attention_weight_max": 23.691186904907227,
      "activations/layer19_attention_weight_min": -23.14100456237793,
      "activations/layer1_attention_weight_max": 16.568262100219727,
      "activations/layer1_attention_weight_min": -13.109382629394531,
      "activations/layer20_attention_weight_max": 22.42002296447754,
      "activations/layer20_attention_weight_min": -22.087594985961914,
      "activations/layer21_attention_weight_max": 43.4959602355957,
      "activations/layer21_attention_weight_min": -24.285940170288086,
      "activations/layer22_attention_weight_max": 28.746383666992188,
      "activations/layer22_attention_weight_min": -26.392108917236328,
      "activations/layer23_attention_weight_max": 37.66303253173828,
      "activations/layer23_attention_weight_min": -26.861751556396484,
      "activations/layer2_attention_weight_max": 30.60749626159668,
      "activations/layer2_attention_weight_min": -29.518470764160156,
      "activations/layer3_attention_weight_max": 83.92546081542969,
      "activations/layer3_attention_weight_min": -81.75098419189453,
      "activations/layer4_attention_weight_max": 82.76006317138672,
      "activations/layer4_attention_weight_min": -80.79059600830078,
      "activations/layer5_attention_weight_max": 60.61820983886719,
      "activations/layer5_attention_weight_min": -68.66608428955078,
      "activations/layer6_attention_weight_max": 49.748172760009766,
      "activations/layer6_attention_weight_min": -51.26240158081055,
      "activations/layer7_attention_weight_max": 69.4210205078125,
      "activations/layer7_attention_weight_min": -65.90009307861328,
      "activations/layer8_attention_weight_max": 47.678558349609375,
      "activations/layer8_attention_weight_min": -51.00676345825195,
      "activations/layer9_attention_weight_max": 43.4519157409668,
      "activations/layer9_attention_weight_min": -45.56205368041992,
      "epoch": 19.47,
      "learning_rate": 2.4666666666666665e-05,
      "loss": 2.7323,
      "step": 335150
    },
    {
      "activations/layer0_attention_weight_max": 14.285122871398926,
      "activations/layer0_attention_weight_min": -13.99892520904541,
      "activations/layer10_attention_weight_max": 38.13703155517578,
      "activations/layer10_attention_weight_min": -37.50594711303711,
      "activations/layer11_attention_weight_max": 35.0072021484375,
      "activations/layer11_attention_weight_min": -37.15593719482422,
      "activations/layer12_attention_weight_max": 19.916173934936523,
      "activations/layer12_attention_weight_min": -26.14157485961914,
      "activations/layer13_attention_weight_max": 34.89554977416992,
      "activations/layer13_attention_weight_min": -29.030261993408203,
      "activations/layer14_attention_weight_max": 34.64339828491211,
      "activations/layer14_attention_weight_min": -29.14019203186035,
      "activations/layer15_attention_weight_max": 33.7778434753418,
      "activations/layer15_attention_weight_min": -30.35163116455078,
      "activations/layer16_attention_weight_max": 34.023468017578125,
      "activations/layer16_attention_weight_min": -31.12502098083496,
      "activations/layer17_attention_weight_max": 48.58526611328125,
      "activations/layer17_attention_weight_min": -45.18498611450195,
      "activations/layer18_attention_weight_max": 46.59562683105469,
      "activations/layer18_attention_weight_min": -37.995208740234375,
      "activations/layer19_attention_weight_max": 23.64435577392578,
      "activations/layer19_attention_weight_min": -22.305845260620117,
      "activations/layer1_attention_weight_max": 14.88009262084961,
      "activations/layer1_attention_weight_min": -12.633727073669434,
      "activations/layer20_attention_weight_max": 21.099740982055664,
      "activations/layer20_attention_weight_min": -21.008583068847656,
      "activations/layer21_attention_weight_max": 33.44829177856445,
      "activations/layer21_attention_weight_min": -21.163650512695312,
      "activations/layer22_attention_weight_max": 31.23290252685547,
      "activations/layer22_attention_weight_min": -24.77427101135254,
      "activations/layer23_attention_weight_max": 38.036773681640625,
      "activations/layer23_attention_weight_min": -23.551788330078125,
      "activations/layer2_attention_weight_max": 28.294471740722656,
      "activations/layer2_attention_weight_min": -27.88007926940918,
      "activations/layer3_attention_weight_max": 81.06118774414062,
      "activations/layer3_attention_weight_min": -83.04205322265625,
      "activations/layer4_attention_weight_max": 76.98027801513672,
      "activations/layer4_attention_weight_min": -76.28031921386719,
      "activations/layer5_attention_weight_max": 58.33302307128906,
      "activations/layer5_attention_weight_min": -69.70462036132812,
      "activations/layer6_attention_weight_max": 49.332786560058594,
      "activations/layer6_attention_weight_min": -48.70188522338867,
      "activations/layer7_attention_weight_max": 64.13890838623047,
      "activations/layer7_attention_weight_min": -64.45524597167969,
      "activations/layer8_attention_weight_max": 45.29751968383789,
      "activations/layer8_attention_weight_min": -50.26057052612305,
      "activations/layer9_attention_weight_max": 44.02725601196289,
      "activations/layer9_attention_weight_min": -45.42325973510742,
      "epoch": 19.48,
      "learning_rate": 2.464772727272727e-05,
      "loss": 2.7188,
      "step": 335200
    },
    {
      "activations/layer0_attention_weight_max": 15.830558776855469,
      "activations/layer0_attention_weight_min": -13.882708549499512,
      "activations/layer10_attention_weight_max": 36.19552230834961,
      "activations/layer10_attention_weight_min": -36.27149200439453,
      "activations/layer11_attention_weight_max": 31.869426727294922,
      "activations/layer11_attention_weight_min": -35.55534744262695,
      "activations/layer12_attention_weight_max": 19.78708839416504,
      "activations/layer12_attention_weight_min": -24.173477172851562,
      "activations/layer13_attention_weight_max": 40.064022064208984,
      "activations/layer13_attention_weight_min": -30.03030014038086,
      "activations/layer14_attention_weight_max": 45.30550003051758,
      "activations/layer14_attention_weight_min": -34.38544845581055,
      "activations/layer15_attention_weight_max": 37.616004943847656,
      "activations/layer15_attention_weight_min": -33.535335540771484,
      "activations/layer16_attention_weight_max": 39.27914810180664,
      "activations/layer16_attention_weight_min": -35.60082244873047,
      "activations/layer17_attention_weight_max": 56.83460235595703,
      "activations/layer17_attention_weight_min": -50.690093994140625,
      "activations/layer18_attention_weight_max": 52.63916015625,
      "activations/layer18_attention_weight_min": -44.06589126586914,
      "activations/layer19_attention_weight_max": 25.474868774414062,
      "activations/layer19_attention_weight_min": -21.952680587768555,
      "activations/layer1_attention_weight_max": 15.588391304016113,
      "activations/layer1_attention_weight_min": -13.147035598754883,
      "activations/layer20_attention_weight_max": 24.271827697753906,
      "activations/layer20_attention_weight_min": -21.940757751464844,
      "activations/layer21_attention_weight_max": 38.5987548828125,
      "activations/layer21_attention_weight_min": -23.62137222290039,
      "activations/layer22_attention_weight_max": 30.23920440673828,
      "activations/layer22_attention_weight_min": -25.262571334838867,
      "activations/layer23_attention_weight_max": 37.41516876220703,
      "activations/layer23_attention_weight_min": -23.113605499267578,
      "activations/layer2_attention_weight_max": 30.214445114135742,
      "activations/layer2_attention_weight_min": -27.94249725341797,
      "activations/layer3_attention_weight_max": 80.7293472290039,
      "activations/layer3_attention_weight_min": -81.6999740600586,
      "activations/layer4_attention_weight_max": 79.35575866699219,
      "activations/layer4_attention_weight_min": -79.23125457763672,
      "activations/layer5_attention_weight_max": 61.14253234863281,
      "activations/layer5_attention_weight_min": -68.3206558227539,
      "activations/layer6_attention_weight_max": 49.72227096557617,
      "activations/layer6_attention_weight_min": -49.069881439208984,
      "activations/layer7_attention_weight_max": 67.44644165039062,
      "activations/layer7_attention_weight_min": -67.47616577148438,
      "activations/layer8_attention_weight_max": 46.36879348754883,
      "activations/layer8_attention_weight_min": -50.22031784057617,
      "activations/layer9_attention_weight_max": 44.83115768432617,
      "activations/layer9_attention_weight_min": -48.872867584228516,
      "epoch": 19.48,
      "learning_rate": 2.462878787878788e-05,
      "loss": 2.725,
      "step": 335250
    },
    {
      "activations/layer0_attention_weight_max": 16.642391204833984,
      "activations/layer0_attention_weight_min": -13.596614837646484,
      "activations/layer10_attention_weight_max": 34.998748779296875,
      "activations/layer10_attention_weight_min": -35.21064758300781,
      "activations/layer11_attention_weight_max": 37.576847076416016,
      "activations/layer11_attention_weight_min": -33.77470016479492,
      "activations/layer12_attention_weight_max": 17.68694496154785,
      "activations/layer12_attention_weight_min": -27.623611450195312,
      "activations/layer13_attention_weight_max": 29.858489990234375,
      "activations/layer13_attention_weight_min": -28.114837646484375,
      "activations/layer14_attention_weight_max": 32.4031982421875,
      "activations/layer14_attention_weight_min": -30.030048370361328,
      "activations/layer15_attention_weight_max": 31.35434913635254,
      "activations/layer15_attention_weight_min": -29.27863121032715,
      "activations/layer16_attention_weight_max": 32.23698043823242,
      "activations/layer16_attention_weight_min": -31.755338668823242,
      "activations/layer17_attention_weight_max": 47.41634750366211,
      "activations/layer17_attention_weight_min": -43.0872802734375,
      "activations/layer18_attention_weight_max": 43.96874237060547,
      "activations/layer18_attention_weight_min": -42.5365104675293,
      "activations/layer19_attention_weight_max": 24.82105255126953,
      "activations/layer19_attention_weight_min": -26.9822940826416,
      "activations/layer1_attention_weight_max": 16.33995819091797,
      "activations/layer1_attention_weight_min": -14.408414840698242,
      "activations/layer20_attention_weight_max": 22.783374786376953,
      "activations/layer20_attention_weight_min": -23.103099822998047,
      "activations/layer21_attention_weight_max": 35.517791748046875,
      "activations/layer21_attention_weight_min": -20.95897674560547,
      "activations/layer22_attention_weight_max": 29.855716705322266,
      "activations/layer22_attention_weight_min": -27.828590393066406,
      "activations/layer23_attention_weight_max": 39.545196533203125,
      "activations/layer23_attention_weight_min": -23.70763397216797,
      "activations/layer2_attention_weight_max": 32.95064926147461,
      "activations/layer2_attention_weight_min": -30.89513397216797,
      "activations/layer3_attention_weight_max": 83.55098724365234,
      "activations/layer3_attention_weight_min": -80.42093658447266,
      "activations/layer4_attention_weight_max": 80.14336395263672,
      "activations/layer4_attention_weight_min": -79.52082824707031,
      "activations/layer5_attention_weight_max": 65.11494445800781,
      "activations/layer5_attention_weight_min": -68.50935363769531,
      "activations/layer6_attention_weight_max": 47.25499725341797,
      "activations/layer6_attention_weight_min": -49.75534439086914,
      "activations/layer7_attention_weight_max": 73.79108428955078,
      "activations/layer7_attention_weight_min": -65.50434875488281,
      "activations/layer8_attention_weight_max": 49.31209945678711,
      "activations/layer8_attention_weight_min": -48.086517333984375,
      "activations/layer9_attention_weight_max": 59.52798080444336,
      "activations/layer9_attention_weight_min": -46.03690719604492,
      "epoch": 19.48,
      "learning_rate": 2.460984848484848e-05,
      "loss": 2.7386,
      "step": 335300
    },
    {
      "activations/layer0_attention_weight_max": 14.77122688293457,
      "activations/layer0_attention_weight_min": -13.662221908569336,
      "activations/layer10_attention_weight_max": 34.642669677734375,
      "activations/layer10_attention_weight_min": -36.297298431396484,
      "activations/layer11_attention_weight_max": 31.263702392578125,
      "activations/layer11_attention_weight_min": -34.633079528808594,
      "activations/layer12_attention_weight_max": 20.503639221191406,
      "activations/layer12_attention_weight_min": -22.780487060546875,
      "activations/layer13_attention_weight_max": 33.56086730957031,
      "activations/layer13_attention_weight_min": -28.409143447875977,
      "activations/layer14_attention_weight_max": 34.919471740722656,
      "activations/layer14_attention_weight_min": -31.559513092041016,
      "activations/layer15_attention_weight_max": 31.039321899414062,
      "activations/layer15_attention_weight_min": -29.78536033630371,
      "activations/layer16_attention_weight_max": 32.65777587890625,
      "activations/layer16_attention_weight_min": -34.22526550292969,
      "activations/layer17_attention_weight_max": 52.315773010253906,
      "activations/layer17_attention_weight_min": -43.93696594238281,
      "activations/layer18_attention_weight_max": 45.330997467041016,
      "activations/layer18_attention_weight_min": -39.55846405029297,
      "activations/layer19_attention_weight_max": 23.531757354736328,
      "activations/layer19_attention_weight_min": -24.23069953918457,
      "activations/layer1_attention_weight_max": 15.526175498962402,
      "activations/layer1_attention_weight_min": -13.24177074432373,
      "activations/layer20_attention_weight_max": 21.49205207824707,
      "activations/layer20_attention_weight_min": -22.749271392822266,
      "activations/layer21_attention_weight_max": 37.66086196899414,
      "activations/layer21_attention_weight_min": -22.50311851501465,
      "activations/layer22_attention_weight_max": 27.2719783782959,
      "activations/layer22_attention_weight_min": -25.77011489868164,
      "activations/layer23_attention_weight_max": 36.059478759765625,
      "activations/layer23_attention_weight_min": -24.297863006591797,
      "activations/layer2_attention_weight_max": 30.207740783691406,
      "activations/layer2_attention_weight_min": -29.673789978027344,
      "activations/layer3_attention_weight_max": 85.60595703125,
      "activations/layer3_attention_weight_min": -82.76698303222656,
      "activations/layer4_attention_weight_max": 82.0283432006836,
      "activations/layer4_attention_weight_min": -78.26310729980469,
      "activations/layer5_attention_weight_max": 60.443580627441406,
      "activations/layer5_attention_weight_min": -67.77315521240234,
      "activations/layer6_attention_weight_max": 48.25540542602539,
      "activations/layer6_attention_weight_min": -49.108394622802734,
      "activations/layer7_attention_weight_max": 66.62411499023438,
      "activations/layer7_attention_weight_min": -62.964534759521484,
      "activations/layer8_attention_weight_max": 45.07979965209961,
      "activations/layer8_attention_weight_min": -47.21786117553711,
      "activations/layer9_attention_weight_max": 41.2421989440918,
      "activations/layer9_attention_weight_min": -45.86692810058594,
      "epoch": 19.49,
      "learning_rate": 2.459090909090909e-05,
      "loss": 2.7562,
      "step": 335350
    },
    {
      "activations/layer0_attention_weight_max": 14.811179161071777,
      "activations/layer0_attention_weight_min": -13.845929145812988,
      "activations/layer10_attention_weight_max": 37.495426177978516,
      "activations/layer10_attention_weight_min": -37.7210807800293,
      "activations/layer11_attention_weight_max": 33.40602493286133,
      "activations/layer11_attention_weight_min": -35.12438201904297,
      "activations/layer12_attention_weight_max": 19.743669509887695,
      "activations/layer12_attention_weight_min": -22.13341522216797,
      "activations/layer13_attention_weight_max": 35.2288818359375,
      "activations/layer13_attention_weight_min": -26.82223892211914,
      "activations/layer14_attention_weight_max": 36.020912170410156,
      "activations/layer14_attention_weight_min": -31.946672439575195,
      "activations/layer15_attention_weight_max": 32.9929084777832,
      "activations/layer15_attention_weight_min": -30.42831802368164,
      "activations/layer16_attention_weight_max": 35.8262939453125,
      "activations/layer16_attention_weight_min": -33.30331039428711,
      "activations/layer17_attention_weight_max": 55.577415466308594,
      "activations/layer17_attention_weight_min": -46.72861862182617,
      "activations/layer18_attention_weight_max": 52.55778884887695,
      "activations/layer18_attention_weight_min": -41.2275505065918,
      "activations/layer19_attention_weight_max": 28.65934181213379,
      "activations/layer19_attention_weight_min": -22.60771369934082,
      "activations/layer1_attention_weight_max": 15.73671817779541,
      "activations/layer1_attention_weight_min": -13.962601661682129,
      "activations/layer20_attention_weight_max": 26.36432456970215,
      "activations/layer20_attention_weight_min": -20.251628875732422,
      "activations/layer21_attention_weight_max": 37.690826416015625,
      "activations/layer21_attention_weight_min": -21.531902313232422,
      "activations/layer22_attention_weight_max": 32.33428955078125,
      "activations/layer22_attention_weight_min": -26.50848388671875,
      "activations/layer23_attention_weight_max": 41.45538330078125,
      "activations/layer23_attention_weight_min": -22.654346466064453,
      "activations/layer2_attention_weight_max": 31.075590133666992,
      "activations/layer2_attention_weight_min": -30.75243377685547,
      "activations/layer3_attention_weight_max": 86.27366638183594,
      "activations/layer3_attention_weight_min": -88.14234924316406,
      "activations/layer4_attention_weight_max": 86.47798919677734,
      "activations/layer4_attention_weight_min": -87.01000213623047,
      "activations/layer5_attention_weight_max": 65.03091430664062,
      "activations/layer5_attention_weight_min": -71.61488342285156,
      "activations/layer6_attention_weight_max": 52.191200256347656,
      "activations/layer6_attention_weight_min": -51.52745819091797,
      "activations/layer7_attention_weight_max": 66.83915710449219,
      "activations/layer7_attention_weight_min": -68.46068572998047,
      "activations/layer8_attention_weight_max": 49.32785415649414,
      "activations/layer8_attention_weight_min": -51.04618835449219,
      "activations/layer9_attention_weight_max": 44.42753219604492,
      "activations/layer9_attention_weight_min": -49.640621185302734,
      "epoch": 19.49,
      "learning_rate": 2.4571969696969693e-05,
      "loss": 2.7165,
      "step": 335400
    },
    {
      "activations/layer0_attention_weight_max": 16.4285945892334,
      "activations/layer0_attention_weight_min": -14.290160179138184,
      "activations/layer10_attention_weight_max": 36.22804260253906,
      "activations/layer10_attention_weight_min": -37.1072998046875,
      "activations/layer11_attention_weight_max": 34.66184616088867,
      "activations/layer11_attention_weight_min": -34.99697494506836,
      "activations/layer12_attention_weight_max": 17.59799575805664,
      "activations/layer12_attention_weight_min": -24.26397132873535,
      "activations/layer13_attention_weight_max": 36.59348678588867,
      "activations/layer13_attention_weight_min": -26.992290496826172,
      "activations/layer14_attention_weight_max": 32.55148696899414,
      "activations/layer14_attention_weight_min": -29.65031623840332,
      "activations/layer15_attention_weight_max": 31.43041229248047,
      "activations/layer15_attention_weight_min": -31.06759262084961,
      "activations/layer16_attention_weight_max": 33.88906478881836,
      "activations/layer16_attention_weight_min": -32.82133865356445,
      "activations/layer17_attention_weight_max": 48.810813903808594,
      "activations/layer17_attention_weight_min": -47.41847229003906,
      "activations/layer18_attention_weight_max": 45.991432189941406,
      "activations/layer18_attention_weight_min": -42.829345703125,
      "activations/layer19_attention_weight_max": 26.30678367614746,
      "activations/layer19_attention_weight_min": -23.903043746948242,
      "activations/layer1_attention_weight_max": 15.829911231994629,
      "activations/layer1_attention_weight_min": -13.42514705657959,
      "activations/layer20_attention_weight_max": 22.75591278076172,
      "activations/layer20_attention_weight_min": -23.940710067749023,
      "activations/layer21_attention_weight_max": 37.028560638427734,
      "activations/layer21_attention_weight_min": -20.957815170288086,
      "activations/layer22_attention_weight_max": 27.54265022277832,
      "activations/layer22_attention_weight_min": -28.427433013916016,
      "activations/layer23_attention_weight_max": 32.947898864746094,
      "activations/layer23_attention_weight_min": -24.457122802734375,
      "activations/layer2_attention_weight_max": 31.027992248535156,
      "activations/layer2_attention_weight_min": -29.761993408203125,
      "activations/layer3_attention_weight_max": 91.54531860351562,
      "activations/layer3_attention_weight_min": -87.22677612304688,
      "activations/layer4_attention_weight_max": 84.12789154052734,
      "activations/layer4_attention_weight_min": -80.24398040771484,
      "activations/layer5_attention_weight_max": 61.66828155517578,
      "activations/layer5_attention_weight_min": -70.47300720214844,
      "activations/layer6_attention_weight_max": 48.1028938293457,
      "activations/layer6_attention_weight_min": -49.96195983886719,
      "activations/layer7_attention_weight_max": 64.64283752441406,
      "activations/layer7_attention_weight_min": -63.19213104248047,
      "activations/layer8_attention_weight_max": 48.952579498291016,
      "activations/layer8_attention_weight_min": -49.32011032104492,
      "activations/layer9_attention_weight_max": 44.266014099121094,
      "activations/layer9_attention_weight_min": -44.84439468383789,
      "epoch": 19.49,
      "learning_rate": 2.45530303030303e-05,
      "loss": 2.714,
      "step": 335450
    },
    {
      "activations/layer0_attention_weight_max": 15.715243339538574,
      "activations/layer0_attention_weight_min": -14.18171501159668,
      "activations/layer10_attention_weight_max": 37.96333694458008,
      "activations/layer10_attention_weight_min": -37.47194290161133,
      "activations/layer11_attention_weight_max": 36.096317291259766,
      "activations/layer11_attention_weight_min": -35.95016860961914,
      "activations/layer12_attention_weight_max": 22.115018844604492,
      "activations/layer12_attention_weight_min": -22.94426155090332,
      "activations/layer13_attention_weight_max": 35.60934066772461,
      "activations/layer13_attention_weight_min": -30.157989501953125,
      "activations/layer14_attention_weight_max": 36.78123092651367,
      "activations/layer14_attention_weight_min": -33.35174560546875,
      "activations/layer15_attention_weight_max": 35.75458908081055,
      "activations/layer15_attention_weight_min": -30.890409469604492,
      "activations/layer16_attention_weight_max": 35.8921012878418,
      "activations/layer16_attention_weight_min": -32.43764877319336,
      "activations/layer17_attention_weight_max": 56.0548095703125,
      "activations/layer17_attention_weight_min": -46.923851013183594,
      "activations/layer18_attention_weight_max": 49.0145149230957,
      "activations/layer18_attention_weight_min": -38.8945426940918,
      "activations/layer19_attention_weight_max": 24.2302303314209,
      "activations/layer19_attention_weight_min": -21.799015045166016,
      "activations/layer1_attention_weight_max": 15.251690864562988,
      "activations/layer1_attention_weight_min": -13.348906517028809,
      "activations/layer20_attention_weight_max": 22.173851013183594,
      "activations/layer20_attention_weight_min": -22.11697006225586,
      "activations/layer21_attention_weight_max": 35.96771240234375,
      "activations/layer21_attention_weight_min": -22.314977645874023,
      "activations/layer22_attention_weight_max": 27.07314109802246,
      "activations/layer22_attention_weight_min": -27.2047176361084,
      "activations/layer23_attention_weight_max": 35.895973205566406,
      "activations/layer23_attention_weight_min": -23.527374267578125,
      "activations/layer2_attention_weight_max": 27.48827362060547,
      "activations/layer2_attention_weight_min": -26.522037506103516,
      "activations/layer3_attention_weight_max": 78.94060516357422,
      "activations/layer3_attention_weight_min": -79.43126678466797,
      "activations/layer4_attention_weight_max": 83.38775634765625,
      "activations/layer4_attention_weight_min": -78.97048950195312,
      "activations/layer5_attention_weight_max": 62.39851379394531,
      "activations/layer5_attention_weight_min": -71.91580200195312,
      "activations/layer6_attention_weight_max": 51.55747604370117,
      "activations/layer6_attention_weight_min": -49.644954681396484,
      "activations/layer7_attention_weight_max": 70.5874252319336,
      "activations/layer7_attention_weight_min": -64.42384338378906,
      "activations/layer8_attention_weight_max": 47.21704864501953,
      "activations/layer8_attention_weight_min": -52.818912506103516,
      "activations/layer9_attention_weight_max": 45.94850158691406,
      "activations/layer9_attention_weight_min": -46.80216598510742,
      "epoch": 19.49,
      "learning_rate": 2.453409090909091e-05,
      "loss": 2.7173,
      "step": 335500
    },
    {
      "activations/layer0_attention_weight_max": 15.004005432128906,
      "activations/layer0_attention_weight_min": -13.861900329589844,
      "activations/layer10_attention_weight_max": 36.08867645263672,
      "activations/layer10_attention_weight_min": -37.121315002441406,
      "activations/layer11_attention_weight_max": 34.254920959472656,
      "activations/layer11_attention_weight_min": -35.17047882080078,
      "activations/layer12_attention_weight_max": 20.298084259033203,
      "activations/layer12_attention_weight_min": -26.0303897857666,
      "activations/layer13_attention_weight_max": 34.91542053222656,
      "activations/layer13_attention_weight_min": -27.553489685058594,
      "activations/layer14_attention_weight_max": 36.45984649658203,
      "activations/layer14_attention_weight_min": -30.674468994140625,
      "activations/layer15_attention_weight_max": 32.52383041381836,
      "activations/layer15_attention_weight_min": -30.425405502319336,
      "activations/layer16_attention_weight_max": 33.94336700439453,
      "activations/layer16_attention_weight_min": -35.720645904541016,
      "activations/layer17_attention_weight_max": 50.47187042236328,
      "activations/layer17_attention_weight_min": -52.148799896240234,
      "activations/layer18_attention_weight_max": 46.57075119018555,
      "activations/layer18_attention_weight_min": -48.64455795288086,
      "activations/layer19_attention_weight_max": 25.18284034729004,
      "activations/layer19_attention_weight_min": -25.6352481842041,
      "activations/layer1_attention_weight_max": 15.457027435302734,
      "activations/layer1_attention_weight_min": -13.0728178024292,
      "activations/layer20_attention_weight_max": 22.112430572509766,
      "activations/layer20_attention_weight_min": -22.722074508666992,
      "activations/layer21_attention_weight_max": 32.00920486450195,
      "activations/layer21_attention_weight_min": -23.224102020263672,
      "activations/layer22_attention_weight_max": 28.35567283630371,
      "activations/layer22_attention_weight_min": -25.712299346923828,
      "activations/layer23_attention_weight_max": 38.91201400756836,
      "activations/layer23_attention_weight_min": -24.71829605102539,
      "activations/layer2_attention_weight_max": 31.549381256103516,
      "activations/layer2_attention_weight_min": -30.3963623046875,
      "activations/layer3_attention_weight_max": 86.01980590820312,
      "activations/layer3_attention_weight_min": -88.49054718017578,
      "activations/layer4_attention_weight_max": 87.33830261230469,
      "activations/layer4_attention_weight_min": -87.7757797241211,
      "activations/layer5_attention_weight_max": 61.706321716308594,
      "activations/layer5_attention_weight_min": -73.73869323730469,
      "activations/layer6_attention_weight_max": 50.05088424682617,
      "activations/layer6_attention_weight_min": -51.35976791381836,
      "activations/layer7_attention_weight_max": 63.04029083251953,
      "activations/layer7_attention_weight_min": -66.29022979736328,
      "activations/layer8_attention_weight_max": 46.55974578857422,
      "activations/layer8_attention_weight_min": -49.43244934082031,
      "activations/layer9_attention_weight_max": 43.28721237182617,
      "activations/layer9_attention_weight_min": -45.07700729370117,
      "epoch": 19.5,
      "learning_rate": 2.451515151515151e-05,
      "loss": 2.7269,
      "step": 335550
    },
    {
      "activations/layer0_attention_weight_max": 15.002920150756836,
      "activations/layer0_attention_weight_min": -13.777067184448242,
      "activations/layer10_attention_weight_max": 35.08354187011719,
      "activations/layer10_attention_weight_min": -36.986087799072266,
      "activations/layer11_attention_weight_max": 32.33615493774414,
      "activations/layer11_attention_weight_min": -33.48625564575195,
      "activations/layer12_attention_weight_max": 18.48186492919922,
      "activations/layer12_attention_weight_min": -26.0701847076416,
      "activations/layer13_attention_weight_max": 32.70600128173828,
      "activations/layer13_attention_weight_min": -29.137271881103516,
      "activations/layer14_attention_weight_max": 32.8541145324707,
      "activations/layer14_attention_weight_min": -29.633113861083984,
      "activations/layer15_attention_weight_max": 31.10810089111328,
      "activations/layer15_attention_weight_min": -30.158241271972656,
      "activations/layer16_attention_weight_max": 32.16315841674805,
      "activations/layer16_attention_weight_min": -30.313814163208008,
      "activations/layer17_attention_weight_max": 51.14116287231445,
      "activations/layer17_attention_weight_min": -44.5202522277832,
      "activations/layer18_attention_weight_max": 44.519309997558594,
      "activations/layer18_attention_weight_min": -41.582157135009766,
      "activations/layer19_attention_weight_max": 24.508520126342773,
      "activations/layer19_attention_weight_min": -22.822498321533203,
      "activations/layer1_attention_weight_max": 16.329694747924805,
      "activations/layer1_attention_weight_min": -13.170990943908691,
      "activations/layer20_attention_weight_max": 24.646860122680664,
      "activations/layer20_attention_weight_min": -20.899328231811523,
      "activations/layer21_attention_weight_max": 41.35338592529297,
      "activations/layer21_attention_weight_min": -23.60325050354004,
      "activations/layer22_attention_weight_max": 27.860891342163086,
      "activations/layer22_attention_weight_min": -26.216156005859375,
      "activations/layer23_attention_weight_max": 39.86180877685547,
      "activations/layer23_attention_weight_min": -23.537193298339844,
      "activations/layer2_attention_weight_max": 31.590126037597656,
      "activations/layer2_attention_weight_min": -30.468647003173828,
      "activations/layer3_attention_weight_max": 83.97048950195312,
      "activations/layer3_attention_weight_min": -87.72236633300781,
      "activations/layer4_attention_weight_max": 79.92251586914062,
      "activations/layer4_attention_weight_min": -78.15707397460938,
      "activations/layer5_attention_weight_max": 61.51703643798828,
      "activations/layer5_attention_weight_min": -67.18827819824219,
      "activations/layer6_attention_weight_max": 48.81406784057617,
      "activations/layer6_attention_weight_min": -49.9257698059082,
      "activations/layer7_attention_weight_max": 65.67768859863281,
      "activations/layer7_attention_weight_min": -60.96898651123047,
      "activations/layer8_attention_weight_max": 44.17511749267578,
      "activations/layer8_attention_weight_min": -49.66514205932617,
      "activations/layer9_attention_weight_max": 42.71384048461914,
      "activations/layer9_attention_weight_min": -44.59282302856445,
      "epoch": 19.5,
      "learning_rate": 2.449621212121212e-05,
      "loss": 2.728,
      "step": 335600
    },
    {
      "activations/layer0_attention_weight_max": 15.281428337097168,
      "activations/layer0_attention_weight_min": -14.0175199508667,
      "activations/layer10_attention_weight_max": 34.10527420043945,
      "activations/layer10_attention_weight_min": -33.38529968261719,
      "activations/layer11_attention_weight_max": 36.62592315673828,
      "activations/layer11_attention_weight_min": -34.20475769042969,
      "activations/layer12_attention_weight_max": 20.809141159057617,
      "activations/layer12_attention_weight_min": -23.851152420043945,
      "activations/layer13_attention_weight_max": 33.520896911621094,
      "activations/layer13_attention_weight_min": -25.96882438659668,
      "activations/layer14_attention_weight_max": 38.69913101196289,
      "activations/layer14_attention_weight_min": -29.56878089904785,
      "activations/layer15_attention_weight_max": 36.462520599365234,
      "activations/layer15_attention_weight_min": -29.842164993286133,
      "activations/layer16_attention_weight_max": 40.25410842895508,
      "activations/layer16_attention_weight_min": -35.972862243652344,
      "activations/layer17_attention_weight_max": 57.06662368774414,
      "activations/layer17_attention_weight_min": -49.42653274536133,
      "activations/layer18_attention_weight_max": 49.77536392211914,
      "activations/layer18_attention_weight_min": -43.27635955810547,
      "activations/layer19_attention_weight_max": 26.838266372680664,
      "activations/layer19_attention_weight_min": -25.356245040893555,
      "activations/layer1_attention_weight_max": 15.738869667053223,
      "activations/layer1_attention_weight_min": -13.429588317871094,
      "activations/layer20_attention_weight_max": 24.830081939697266,
      "activations/layer20_attention_weight_min": -24.14898681640625,
      "activations/layer21_attention_weight_max": 47.37129592895508,
      "activations/layer21_attention_weight_min": -24.118345260620117,
      "activations/layer22_attention_weight_max": 31.509458541870117,
      "activations/layer22_attention_weight_min": -30.660276412963867,
      "activations/layer23_attention_weight_max": 40.23033905029297,
      "activations/layer23_attention_weight_min": -26.2242488861084,
      "activations/layer2_attention_weight_max": 29.891523361206055,
      "activations/layer2_attention_weight_min": -29.38282012939453,
      "activations/layer3_attention_weight_max": 85.38803100585938,
      "activations/layer3_attention_weight_min": -86.71529388427734,
      "activations/layer4_attention_weight_max": 82.99290466308594,
      "activations/layer4_attention_weight_min": -84.1694564819336,
      "activations/layer5_attention_weight_max": 62.43708419799805,
      "activations/layer5_attention_weight_min": -70.98440551757812,
      "activations/layer6_attention_weight_max": 49.131649017333984,
      "activations/layer6_attention_weight_min": -49.666263580322266,
      "activations/layer7_attention_weight_max": 62.47248077392578,
      "activations/layer7_attention_weight_min": -64.10404205322266,
      "activations/layer8_attention_weight_max": 45.95068359375,
      "activations/layer8_attention_weight_min": -48.22167205810547,
      "activations/layer9_attention_weight_max": 43.87045669555664,
      "activations/layer9_attention_weight_min": -44.746307373046875,
      "epoch": 19.5,
      "learning_rate": 2.4477272727272724e-05,
      "loss": 2.7315,
      "step": 335650
    },
    {
      "activations/layer0_attention_weight_max": 15.202760696411133,
      "activations/layer0_attention_weight_min": -13.913122177124023,
      "activations/layer10_attention_weight_max": 34.090328216552734,
      "activations/layer10_attention_weight_min": -36.06853485107422,
      "activations/layer11_attention_weight_max": 31.827259063720703,
      "activations/layer11_attention_weight_min": -33.02789306640625,
      "activations/layer12_attention_weight_max": 20.06373405456543,
      "activations/layer12_attention_weight_min": -27.235849380493164,
      "activations/layer13_attention_weight_max": 37.975196838378906,
      "activations/layer13_attention_weight_min": -28.45806312561035,
      "activations/layer14_attention_weight_max": 38.113563537597656,
      "activations/layer14_attention_weight_min": -31.650049209594727,
      "activations/layer15_attention_weight_max": 35.08226013183594,
      "activations/layer15_attention_weight_min": -31.252422332763672,
      "activations/layer16_attention_weight_max": 35.91809844970703,
      "activations/layer16_attention_weight_min": -33.550315856933594,
      "activations/layer17_attention_weight_max": 54.55583572387695,
      "activations/layer17_attention_weight_min": -47.650367736816406,
      "activations/layer18_attention_weight_max": 50.234012603759766,
      "activations/layer18_attention_weight_min": -43.512184143066406,
      "activations/layer19_attention_weight_max": 26.10886573791504,
      "activations/layer19_attention_weight_min": -23.535114288330078,
      "activations/layer1_attention_weight_max": 15.500648498535156,
      "activations/layer1_attention_weight_min": -14.712666511535645,
      "activations/layer20_attention_weight_max": 23.14033317565918,
      "activations/layer20_attention_weight_min": -20.71531867980957,
      "activations/layer21_attention_weight_max": 36.40612030029297,
      "activations/layer21_attention_weight_min": -22.463411331176758,
      "activations/layer22_attention_weight_max": 28.85312843322754,
      "activations/layer22_attention_weight_min": -27.557403564453125,
      "activations/layer23_attention_weight_max": 36.856834411621094,
      "activations/layer23_attention_weight_min": -24.777725219726562,
      "activations/layer2_attention_weight_max": 30.870649337768555,
      "activations/layer2_attention_weight_min": -29.586650848388672,
      "activations/layer3_attention_weight_max": 81.83248138427734,
      "activations/layer3_attention_weight_min": -78.5289077758789,
      "activations/layer4_attention_weight_max": 79.48268127441406,
      "activations/layer4_attention_weight_min": -81.25237274169922,
      "activations/layer5_attention_weight_max": 60.83097457885742,
      "activations/layer5_attention_weight_min": -69.45777893066406,
      "activations/layer6_attention_weight_max": 48.91451644897461,
      "activations/layer6_attention_weight_min": -50.796653747558594,
      "activations/layer7_attention_weight_max": 63.04964065551758,
      "activations/layer7_attention_weight_min": -66.7934799194336,
      "activations/layer8_attention_weight_max": 48.3331184387207,
      "activations/layer8_attention_weight_min": -49.6964225769043,
      "activations/layer9_attention_weight_max": 44.944129943847656,
      "activations/layer9_attention_weight_min": -45.2109489440918,
      "epoch": 19.51,
      "learning_rate": 2.4458333333333333e-05,
      "loss": 2.7191,
      "step": 335700
    },
    {
      "activations/layer0_attention_weight_max": 15.757609367370605,
      "activations/layer0_attention_weight_min": -15.381440162658691,
      "activations/layer10_attention_weight_max": 33.4092903137207,
      "activations/layer10_attention_weight_min": -35.224056243896484,
      "activations/layer11_attention_weight_max": 31.904422760009766,
      "activations/layer11_attention_weight_min": -34.16852569580078,
      "activations/layer12_attention_weight_max": 17.834178924560547,
      "activations/layer12_attention_weight_min": -25.17656707763672,
      "activations/layer13_attention_weight_max": 32.96871566772461,
      "activations/layer13_attention_weight_min": -27.539581298828125,
      "activations/layer14_attention_weight_max": 33.26765441894531,
      "activations/layer14_attention_weight_min": -30.410694122314453,
      "activations/layer15_attention_weight_max": 31.744173049926758,
      "activations/layer15_attention_weight_min": -30.122934341430664,
      "activations/layer16_attention_weight_max": 35.14529800415039,
      "activations/layer16_attention_weight_min": -30.3153076171875,
      "activations/layer17_attention_weight_max": 49.766212463378906,
      "activations/layer17_attention_weight_min": -42.078731536865234,
      "activations/layer18_attention_weight_max": 47.83164978027344,
      "activations/layer18_attention_weight_min": -39.71984100341797,
      "activations/layer19_attention_weight_max": 25.64615821838379,
      "activations/layer19_attention_weight_min": -22.20210075378418,
      "activations/layer1_attention_weight_max": 15.034323692321777,
      "activations/layer1_attention_weight_min": -13.061077117919922,
      "activations/layer20_attention_weight_max": 26.10592269897461,
      "activations/layer20_attention_weight_min": -21.121191024780273,
      "activations/layer21_attention_weight_max": 41.49810791015625,
      "activations/layer21_attention_weight_min": -24.04552459716797,
      "activations/layer22_attention_weight_max": 31.54107666015625,
      "activations/layer22_attention_weight_min": -24.296894073486328,
      "activations/layer23_attention_weight_max": 40.730140686035156,
      "activations/layer23_attention_weight_min": -25.62128257751465,
      "activations/layer2_attention_weight_max": 30.779998779296875,
      "activations/layer2_attention_weight_min": -29.931537628173828,
      "activations/layer3_attention_weight_max": 90.35887145996094,
      "activations/layer3_attention_weight_min": -84.88548278808594,
      "activations/layer4_attention_weight_max": 88.7553939819336,
      "activations/layer4_attention_weight_min": -83.65718841552734,
      "activations/layer5_attention_weight_max": 64.11028289794922,
      "activations/layer5_attention_weight_min": -73.20267486572266,
      "activations/layer6_attention_weight_max": 49.17690658569336,
      "activations/layer6_attention_weight_min": -50.082313537597656,
      "activations/layer7_attention_weight_max": 66.93743896484375,
      "activations/layer7_attention_weight_min": -62.269168853759766,
      "activations/layer8_attention_weight_max": 45.74397659301758,
      "activations/layer8_attention_weight_min": -48.41982650756836,
      "activations/layer9_attention_weight_max": 40.61199951171875,
      "activations/layer9_attention_weight_min": -42.929256439208984,
      "epoch": 19.51,
      "learning_rate": 2.4439393939393938e-05,
      "loss": 2.7211,
      "step": 335750
    },
    {
      "activations/layer0_attention_weight_max": 15.889533042907715,
      "activations/layer0_attention_weight_min": -14.037124633789062,
      "activations/layer10_attention_weight_max": 35.697303771972656,
      "activations/layer10_attention_weight_min": -35.842411041259766,
      "activations/layer11_attention_weight_max": 33.78091812133789,
      "activations/layer11_attention_weight_min": -35.961952209472656,
      "activations/layer12_attention_weight_max": 19.734596252441406,
      "activations/layer12_attention_weight_min": -24.040931701660156,
      "activations/layer13_attention_weight_max": 35.254173278808594,
      "activations/layer13_attention_weight_min": -28.961055755615234,
      "activations/layer14_attention_weight_max": 36.664363861083984,
      "activations/layer14_attention_weight_min": -30.406909942626953,
      "activations/layer15_attention_weight_max": 35.21918487548828,
      "activations/layer15_attention_weight_min": -31.82961654663086,
      "activations/layer16_attention_weight_max": 35.0963020324707,
      "activations/layer16_attention_weight_min": -35.71169662475586,
      "activations/layer17_attention_weight_max": 53.90904235839844,
      "activations/layer17_attention_weight_min": -47.45671463012695,
      "activations/layer18_attention_weight_max": 52.75786590576172,
      "activations/layer18_attention_weight_min": -44.06683349609375,
      "activations/layer19_attention_weight_max": 24.868192672729492,
      "activations/layer19_attention_weight_min": -27.383686065673828,
      "activations/layer1_attention_weight_max": 16.523590087890625,
      "activations/layer1_attention_weight_min": -14.550623893737793,
      "activations/layer20_attention_weight_max": 22.966447830200195,
      "activations/layer20_attention_weight_min": -23.294397354125977,
      "activations/layer21_attention_weight_max": 37.772212982177734,
      "activations/layer21_attention_weight_min": -23.094486236572266,
      "activations/layer22_attention_weight_max": 29.483112335205078,
      "activations/layer22_attention_weight_min": -26.863996505737305,
      "activations/layer23_attention_weight_max": 36.41026306152344,
      "activations/layer23_attention_weight_min": -23.892833709716797,
      "activations/layer2_attention_weight_max": 30.596939086914062,
      "activations/layer2_attention_weight_min": -29.532150268554688,
      "activations/layer3_attention_weight_max": 86.28106689453125,
      "activations/layer3_attention_weight_min": -86.57501983642578,
      "activations/layer4_attention_weight_max": 85.4384765625,
      "activations/layer4_attention_weight_min": -82.73201751708984,
      "activations/layer5_attention_weight_max": 59.39817810058594,
      "activations/layer5_attention_weight_min": -71.31094360351562,
      "activations/layer6_attention_weight_max": 49.95170593261719,
      "activations/layer6_attention_weight_min": -49.70473098754883,
      "activations/layer7_attention_weight_max": 62.383785247802734,
      "activations/layer7_attention_weight_min": -65.92618560791016,
      "activations/layer8_attention_weight_max": 46.348716735839844,
      "activations/layer8_attention_weight_min": -52.1643180847168,
      "activations/layer9_attention_weight_max": 43.9698371887207,
      "activations/layer9_attention_weight_min": -45.638511657714844,
      "epoch": 19.51,
      "learning_rate": 2.4420454545454546e-05,
      "loss": 2.7183,
      "step": 335800
    },
    {
      "activations/layer0_attention_weight_max": 15.60796070098877,
      "activations/layer0_attention_weight_min": -13.662969589233398,
      "activations/layer10_attention_weight_max": 34.40811538696289,
      "activations/layer10_attention_weight_min": -34.87308120727539,
      "activations/layer11_attention_weight_max": 30.870559692382812,
      "activations/layer11_attention_weight_min": -34.406105041503906,
      "activations/layer12_attention_weight_max": 17.727144241333008,
      "activations/layer12_attention_weight_min": -25.082481384277344,
      "activations/layer13_attention_weight_max": 34.021148681640625,
      "activations/layer13_attention_weight_min": -28.77826690673828,
      "activations/layer14_attention_weight_max": 32.024658203125,
      "activations/layer14_attention_weight_min": -30.415706634521484,
      "activations/layer15_attention_weight_max": 32.351932525634766,
      "activations/layer15_attention_weight_min": -30.92251968383789,
      "activations/layer16_attention_weight_max": 31.994916915893555,
      "activations/layer16_attention_weight_min": -32.952178955078125,
      "activations/layer17_attention_weight_max": 51.54499816894531,
      "activations/layer17_attention_weight_min": -41.5434455871582,
      "activations/layer18_attention_weight_max": 49.207740783691406,
      "activations/layer18_attention_weight_min": -36.80104064941406,
      "activations/layer19_attention_weight_max": 25.94044303894043,
      "activations/layer19_attention_weight_min": -23.0869140625,
      "activations/layer1_attention_weight_max": 15.403511047363281,
      "activations/layer1_attention_weight_min": -13.00673770904541,
      "activations/layer20_attention_weight_max": 24.563331604003906,
      "activations/layer20_attention_weight_min": -22.49638557434082,
      "activations/layer21_attention_weight_max": 32.09998321533203,
      "activations/layer21_attention_weight_min": -21.630834579467773,
      "activations/layer22_attention_weight_max": 27.05889892578125,
      "activations/layer22_attention_weight_min": -24.91071319580078,
      "activations/layer23_attention_weight_max": 34.555511474609375,
      "activations/layer23_attention_weight_min": -23.902557373046875,
      "activations/layer2_attention_weight_max": 31.3734130859375,
      "activations/layer2_attention_weight_min": -27.82648277282715,
      "activations/layer3_attention_weight_max": 84.53165435791016,
      "activations/layer3_attention_weight_min": -80.12484741210938,
      "activations/layer4_attention_weight_max": 80.4511947631836,
      "activations/layer4_attention_weight_min": -77.34886169433594,
      "activations/layer5_attention_weight_max": 59.69932556152344,
      "activations/layer5_attention_weight_min": -73.54547119140625,
      "activations/layer6_attention_weight_max": 47.66966247558594,
      "activations/layer6_attention_weight_min": -50.46813201904297,
      "activations/layer7_attention_weight_max": 65.24433898925781,
      "activations/layer7_attention_weight_min": -61.163414001464844,
      "activations/layer8_attention_weight_max": 45.172889709472656,
      "activations/layer8_attention_weight_min": -48.37517166137695,
      "activations/layer9_attention_weight_max": 44.4604377746582,
      "activations/layer9_attention_weight_min": -43.69043731689453,
      "epoch": 19.51,
      "learning_rate": 2.4401515151515147e-05,
      "loss": 2.7289,
      "step": 335850
    },
    {
      "activations/layer0_attention_weight_max": 15.178245544433594,
      "activations/layer0_attention_weight_min": -14.046594619750977,
      "activations/layer10_attention_weight_max": 34.513668060302734,
      "activations/layer10_attention_weight_min": -33.90256881713867,
      "activations/layer11_attention_weight_max": 31.684614181518555,
      "activations/layer11_attention_weight_min": -32.800437927246094,
      "activations/layer12_attention_weight_max": 17.72686195373535,
      "activations/layer12_attention_weight_min": -24.372201919555664,
      "activations/layer13_attention_weight_max": 35.89780807495117,
      "activations/layer13_attention_weight_min": -29.802574157714844,
      "activations/layer14_attention_weight_max": 35.56642532348633,
      "activations/layer14_attention_weight_min": -29.47992706298828,
      "activations/layer15_attention_weight_max": 34.2323112487793,
      "activations/layer15_attention_weight_min": -28.884281158447266,
      "activations/layer16_attention_weight_max": 33.39877700805664,
      "activations/layer16_attention_weight_min": -30.85626983642578,
      "activations/layer17_attention_weight_max": 53.389183044433594,
      "activations/layer17_attention_weight_min": -44.38654327392578,
      "activations/layer18_attention_weight_max": 49.51386642456055,
      "activations/layer18_attention_weight_min": -41.25438690185547,
      "activations/layer19_attention_weight_max": 23.923831939697266,
      "activations/layer19_attention_weight_min": -23.289470672607422,
      "activations/layer1_attention_weight_max": 15.146400451660156,
      "activations/layer1_attention_weight_min": -12.93686294555664,
      "activations/layer20_attention_weight_max": 24.357709884643555,
      "activations/layer20_attention_weight_min": -23.469396591186523,
      "activations/layer21_attention_weight_max": 37.15464401245117,
      "activations/layer21_attention_weight_min": -23.366703033447266,
      "activations/layer22_attention_weight_max": 32.01997375488281,
      "activations/layer22_attention_weight_min": -26.584829330444336,
      "activations/layer23_attention_weight_max": 41.32526779174805,
      "activations/layer23_attention_weight_min": -24.281164169311523,
      "activations/layer2_attention_weight_max": 30.975433349609375,
      "activations/layer2_attention_weight_min": -30.78469467163086,
      "activations/layer3_attention_weight_max": 82.90125274658203,
      "activations/layer3_attention_weight_min": -81.54158782958984,
      "activations/layer4_attention_weight_max": 83.13331604003906,
      "activations/layer4_attention_weight_min": -80.22245025634766,
      "activations/layer5_attention_weight_max": 62.037147521972656,
      "activations/layer5_attention_weight_min": -71.55361938476562,
      "activations/layer6_attention_weight_max": 47.79314422607422,
      "activations/layer6_attention_weight_min": -50.493736267089844,
      "activations/layer7_attention_weight_max": 65.22879028320312,
      "activations/layer7_attention_weight_min": -62.97964859008789,
      "activations/layer8_attention_weight_max": 45.20914077758789,
      "activations/layer8_attention_weight_min": -51.26362609863281,
      "activations/layer9_attention_weight_max": 40.83775329589844,
      "activations/layer9_attention_weight_min": -43.591617584228516,
      "epoch": 19.52,
      "learning_rate": 2.4382575757575756e-05,
      "loss": 2.7209,
      "step": 335900
    },
    {
      "activations/layer0_attention_weight_max": 15.059988975524902,
      "activations/layer0_attention_weight_min": -14.236565589904785,
      "activations/layer10_attention_weight_max": 35.115997314453125,
      "activations/layer10_attention_weight_min": -35.06806945800781,
      "activations/layer11_attention_weight_max": 33.1946907043457,
      "activations/layer11_attention_weight_min": -33.667293548583984,
      "activations/layer12_attention_weight_max": 19.265295028686523,
      "activations/layer12_attention_weight_min": -22.522844314575195,
      "activations/layer13_attention_weight_max": 36.34185028076172,
      "activations/layer13_attention_weight_min": -28.069477081298828,
      "activations/layer14_attention_weight_max": 37.0847053527832,
      "activations/layer14_attention_weight_min": -31.86389923095703,
      "activations/layer15_attention_weight_max": 36.59413528442383,
      "activations/layer15_attention_weight_min": -30.602027893066406,
      "activations/layer16_attention_weight_max": 35.62750244140625,
      "activations/layer16_attention_weight_min": -33.88138198852539,
      "activations/layer17_attention_weight_max": 53.18342208862305,
      "activations/layer17_attention_weight_min": -44.04096603393555,
      "activations/layer18_attention_weight_max": 48.92353439331055,
      "activations/layer18_attention_weight_min": -39.31637191772461,
      "activations/layer19_attention_weight_max": 25.583913803100586,
      "activations/layer19_attention_weight_min": -23.886993408203125,
      "activations/layer1_attention_weight_max": 15.215649604797363,
      "activations/layer1_attention_weight_min": -13.056912422180176,
      "activations/layer20_attention_weight_max": 25.669267654418945,
      "activations/layer20_attention_weight_min": -22.505521774291992,
      "activations/layer21_attention_weight_max": 39.589725494384766,
      "activations/layer21_attention_weight_min": -23.773218154907227,
      "activations/layer22_attention_weight_max": 28.548717498779297,
      "activations/layer22_attention_weight_min": -26.73211097717285,
      "activations/layer23_attention_weight_max": 38.88579559326172,
      "activations/layer23_attention_weight_min": -22.7125301361084,
      "activations/layer2_attention_weight_max": 31.104446411132812,
      "activations/layer2_attention_weight_min": -30.250152587890625,
      "activations/layer3_attention_weight_max": 86.29522705078125,
      "activations/layer3_attention_weight_min": -84.0020980834961,
      "activations/layer4_attention_weight_max": 82.76658630371094,
      "activations/layer4_attention_weight_min": -82.08695983886719,
      "activations/layer5_attention_weight_max": 59.26165008544922,
      "activations/layer5_attention_weight_min": -78.61453247070312,
      "activations/layer6_attention_weight_max": 49.91199493408203,
      "activations/layer6_attention_weight_min": -51.24704360961914,
      "activations/layer7_attention_weight_max": 62.51200866699219,
      "activations/layer7_attention_weight_min": -65.8408203125,
      "activations/layer8_attention_weight_max": 45.774497985839844,
      "activations/layer8_attention_weight_min": -50.4312858581543,
      "activations/layer9_attention_weight_max": 42.671871185302734,
      "activations/layer9_attention_weight_min": -45.10357666015625,
      "epoch": 19.52,
      "learning_rate": 2.436363636363636e-05,
      "loss": 2.7085,
      "step": 335950
    },
    {
      "activations/layer0_attention_weight_max": 15.369970321655273,
      "activations/layer0_attention_weight_min": -13.908891677856445,
      "activations/layer10_attention_weight_max": 39.16890335083008,
      "activations/layer10_attention_weight_min": -38.49568176269531,
      "activations/layer11_attention_weight_max": 36.89308547973633,
      "activations/layer11_attention_weight_min": -37.35199737548828,
      "activations/layer12_attention_weight_max": 18.885055541992188,
      "activations/layer12_attention_weight_min": -22.95220375061035,
      "activations/layer13_attention_weight_max": 33.68485641479492,
      "activations/layer13_attention_weight_min": -28.39749526977539,
      "activations/layer14_attention_weight_max": 38.59220886230469,
      "activations/layer14_attention_weight_min": -30.847850799560547,
      "activations/layer15_attention_weight_max": 35.496761322021484,
      "activations/layer15_attention_weight_min": -30.244592666625977,
      "activations/layer16_attention_weight_max": 37.19019317626953,
      "activations/layer16_attention_weight_min": -33.1040153503418,
      "activations/layer17_attention_weight_max": 56.81825637817383,
      "activations/layer17_attention_weight_min": -46.572601318359375,
      "activations/layer18_attention_weight_max": 47.43826675415039,
      "activations/layer18_attention_weight_min": -41.20842361450195,
      "activations/layer19_attention_weight_max": 24.03912353515625,
      "activations/layer19_attention_weight_min": -24.345226287841797,
      "activations/layer1_attention_weight_max": 15.300820350646973,
      "activations/layer1_attention_weight_min": -13.241499900817871,
      "activations/layer20_attention_weight_max": 22.9063777923584,
      "activations/layer20_attention_weight_min": -22.891712188720703,
      "activations/layer21_attention_weight_max": 40.51642608642578,
      "activations/layer21_attention_weight_min": -24.1599063873291,
      "activations/layer22_attention_weight_max": 31.720333099365234,
      "activations/layer22_attention_weight_min": -27.60916519165039,
      "activations/layer23_attention_weight_max": 39.14414978027344,
      "activations/layer23_attention_weight_min": -26.281980514526367,
      "activations/layer2_attention_weight_max": 32.05321502685547,
      "activations/layer2_attention_weight_min": -31.99060821533203,
      "activations/layer3_attention_weight_max": 85.20246887207031,
      "activations/layer3_attention_weight_min": -86.5014877319336,
      "activations/layer4_attention_weight_max": 82.87538146972656,
      "activations/layer4_attention_weight_min": -85.36060333251953,
      "activations/layer5_attention_weight_max": 60.545936584472656,
      "activations/layer5_attention_weight_min": -74.5888900756836,
      "activations/layer6_attention_weight_max": 49.56490707397461,
      "activations/layer6_attention_weight_min": -54.53237533569336,
      "activations/layer7_attention_weight_max": 64.43025207519531,
      "activations/layer7_attention_weight_min": -74.83859252929688,
      "activations/layer8_attention_weight_max": 50.56728744506836,
      "activations/layer8_attention_weight_min": -54.597808837890625,
      "activations/layer9_attention_weight_max": 49.68788146972656,
      "activations/layer9_attention_weight_min": -50.11629867553711,
      "epoch": 19.52,
      "learning_rate": 2.434469696969697e-05,
      "loss": 2.7379,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_loss": 2.685546875,
      "eval_runtime": 8.5363,
      "eval_samples_per_second": 503.026,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_openwebtext_loss": 2.685546875,
      "eval_openwebtext_ppl": 14.666219776472907,
      "eval_openwebtext_runtime": 8.5363,
      "eval_openwebtext_samples_per_second": 503.026,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0325,
      "eval_wikitext_samples_per_second": 224.353,
      "step": 336000
    },
    {
      "epoch": 19.52,
      "eval_lambada_loss": 2.427734375,
      "eval_lambada_ppl": 11.333176243677455,
      "eval_lambada_runtime": 9.6423,
      "eval_lambada_samples_per_second": 504.964,
      "step": 336000
    },
    {
      "activations/layer0_attention_weight_max": 15.546541213989258,
      "activations/layer0_attention_weight_min": -14.168207168579102,
      "activations/layer10_attention_weight_max": 33.9197998046875,
      "activations/layer10_attention_weight_min": -34.91891098022461,
      "activations/layer11_attention_weight_max": 32.20787811279297,
      "activations/layer11_attention_weight_min": -35.187477111816406,
      "activations/layer12_attention_weight_max": 21.6313533782959,
      "activations/layer12_attention_weight_min": -22.721689224243164,
      "activations/layer13_attention_weight_max": 38.67424774169922,
      "activations/layer13_attention_weight_min": -30.392250061035156,
      "activations/layer14_attention_weight_max": 39.03548812866211,
      "activations/layer14_attention_weight_min": -31.634740829467773,
      "activations/layer15_attention_weight_max": 36.41669464111328,
      "activations/layer15_attention_weight_min": -31.659008026123047,
      "activations/layer16_attention_weight_max": 35.43212890625,
      "activations/layer16_attention_weight_min": -34.263423919677734,
      "activations/layer17_attention_weight_max": 51.62688064575195,
      "activations/layer17_attention_weight_min": -44.13351058959961,
      "activations/layer18_attention_weight_max": 49.07759094238281,
      "activations/layer18_attention_weight_min": -38.99230194091797,
      "activations/layer19_attention_weight_max": 26.94437026977539,
      "activations/layer19_attention_weight_min": -22.26286506652832,
      "activations/layer1_attention_weight_max": 14.66296672821045,
      "activations/layer1_attention_weight_min": -13.59213924407959,
      "activations/layer20_attention_weight_max": 23.963380813598633,
      "activations/layer20_attention_weight_min": -20.97130012512207,
      "activations/layer21_attention_weight_max": 38.48371124267578,
      "activations/layer21_attention_weight_min": -24.49283218383789,
      "activations/layer22_attention_weight_max": 31.60230255126953,
      "activations/layer22_attention_weight_min": -26.9582576751709,
      "activations/layer23_attention_weight_max": 38.01719665527344,
      "activations/layer23_attention_weight_min": -24.201955795288086,
      "activations/layer2_attention_weight_max": 30.863536834716797,
      "activations/layer2_attention_weight_min": -30.540771484375,
      "activations/layer3_attention_weight_max": 83.4034423828125,
      "activations/layer3_attention_weight_min": -84.7676010131836,
      "activations/layer4_attention_weight_max": 80.43315887451172,
      "activations/layer4_attention_weight_min": -81.37093353271484,
      "activations/layer5_attention_weight_max": 60.99383544921875,
      "activations/layer5_attention_weight_min": -70.45182800292969,
      "activations/layer6_attention_weight_max": 50.10111618041992,
      "activations/layer6_attention_weight_min": -51.37419891357422,
      "activations/layer7_attention_weight_max": 68.78951263427734,
      "activations/layer7_attention_weight_min": -62.275779724121094,
      "activations/layer8_attention_weight_max": 45.07518768310547,
      "activations/layer8_attention_weight_min": -48.59696578979492,
      "activations/layer9_attention_weight_max": 44.61980438232422,
      "activations/layer9_attention_weight_min": -46.11219787597656,
      "epoch": 19.53,
      "learning_rate": 2.4325757575757574e-05,
      "loss": 2.7367,
      "step": 336050
    },
    {
      "activations/layer0_attention_weight_max": 15.493022918701172,
      "activations/layer0_attention_weight_min": -13.916573524475098,
      "activations/layer10_attention_weight_max": 33.45771026611328,
      "activations/layer10_attention_weight_min": -35.00636672973633,
      "activations/layer11_attention_weight_max": 30.788074493408203,
      "activations/layer11_attention_weight_min": -33.381202697753906,
      "activations/layer12_attention_weight_max": 18.018768310546875,
      "activations/layer12_attention_weight_min": -24.412336349487305,
      "activations/layer13_attention_weight_max": 31.915048599243164,
      "activations/layer13_attention_weight_min": -27.377294540405273,
      "activations/layer14_attention_weight_max": 33.762123107910156,
      "activations/layer14_attention_weight_min": -28.750280380249023,
      "activations/layer15_attention_weight_max": 32.90252685546875,
      "activations/layer15_attention_weight_min": -29.922462463378906,
      "activations/layer16_attention_weight_max": 32.60904312133789,
      "activations/layer16_attention_weight_min": -31.680919647216797,
      "activations/layer17_attention_weight_max": 48.89063262939453,
      "activations/layer17_attention_weight_min": -43.9495849609375,
      "activations/layer18_attention_weight_max": 48.16316223144531,
      "activations/layer18_attention_weight_min": -38.26132583618164,
      "activations/layer19_attention_weight_max": 23.622167587280273,
      "activations/layer19_attention_weight_min": -22.563289642333984,
      "activations/layer1_attention_weight_max": 15.445685386657715,
      "activations/layer1_attention_weight_min": -13.335930824279785,
      "activations/layer20_attention_weight_max": 23.313005447387695,
      "activations/layer20_attention_weight_min": -22.824665069580078,
      "activations/layer21_attention_weight_max": 38.68643569946289,
      "activations/layer21_attention_weight_min": -20.859914779663086,
      "activations/layer22_attention_weight_max": 29.478635787963867,
      "activations/layer22_attention_weight_min": -25.216102600097656,
      "activations/layer23_attention_weight_max": 36.1588134765625,
      "activations/layer23_attention_weight_min": -22.75182342529297,
      "activations/layer2_attention_weight_max": 31.199481964111328,
      "activations/layer2_attention_weight_min": -29.203060150146484,
      "activations/layer3_attention_weight_max": 83.90950012207031,
      "activations/layer3_attention_weight_min": -79.61722564697266,
      "activations/layer4_attention_weight_max": 76.69314575195312,
      "activations/layer4_attention_weight_min": -77.11495208740234,
      "activations/layer5_attention_weight_max": 57.41534423828125,
      "activations/layer5_attention_weight_min": -68.57658386230469,
      "activations/layer6_attention_weight_max": 48.971580505371094,
      "activations/layer6_attention_weight_min": -46.474998474121094,
      "activations/layer7_attention_weight_max": 62.237220764160156,
      "activations/layer7_attention_weight_min": -63.854339599609375,
      "activations/layer8_attention_weight_max": 43.93284225463867,
      "activations/layer8_attention_weight_min": -46.736087799072266,
      "activations/layer9_attention_weight_max": 42.88943099975586,
      "activations/layer9_attention_weight_min": -45.864410400390625,
      "epoch": 19.53,
      "learning_rate": 2.430681818181818e-05,
      "loss": 2.7241,
      "step": 336100
    },
    {
      "activations/layer0_attention_weight_max": 14.924312591552734,
      "activations/layer0_attention_weight_min": -13.799166679382324,
      "activations/layer10_attention_weight_max": 37.429222106933594,
      "activations/layer10_attention_weight_min": -35.21660232543945,
      "activations/layer11_attention_weight_max": 32.646995544433594,
      "activations/layer11_attention_weight_min": -34.47624969482422,
      "activations/layer12_attention_weight_max": 19.974075317382812,
      "activations/layer12_attention_weight_min": -22.760231018066406,
      "activations/layer13_attention_weight_max": 36.6631965637207,
      "activations/layer13_attention_weight_min": -28.019733428955078,
      "activations/layer14_attention_weight_max": 37.16487121582031,
      "activations/layer14_attention_weight_min": -31.09903335571289,
      "activations/layer15_attention_weight_max": 36.30483627319336,
      "activations/layer15_attention_weight_min": -31.333797454833984,
      "activations/layer16_attention_weight_max": 36.70045471191406,
      "activations/layer16_attention_weight_min": -33.63733673095703,
      "activations/layer17_attention_weight_max": 53.98569107055664,
      "activations/layer17_attention_weight_min": -46.699283599853516,
      "activations/layer18_attention_weight_max": 50.8504753112793,
      "activations/layer18_attention_weight_min": -40.71052169799805,
      "activations/layer19_attention_weight_max": 26.7271728515625,
      "activations/layer19_attention_weight_min": -24.921262741088867,
      "activations/layer1_attention_weight_max": 16.201147079467773,
      "activations/layer1_attention_weight_min": -13.819601058959961,
      "activations/layer20_attention_weight_max": 24.948596954345703,
      "activations/layer20_attention_weight_min": -23.773561477661133,
      "activations/layer21_attention_weight_max": 40.174781799316406,
      "activations/layer21_attention_weight_min": -20.962671279907227,
      "activations/layer22_attention_weight_max": 32.64768981933594,
      "activations/layer22_attention_weight_min": -28.063175201416016,
      "activations/layer23_attention_weight_max": 42.57722473144531,
      "activations/layer23_attention_weight_min": -24.282033920288086,
      "activations/layer2_attention_weight_max": 31.056041717529297,
      "activations/layer2_attention_weight_min": -29.944679260253906,
      "activations/layer3_attention_weight_max": 80.8613052368164,
      "activations/layer3_attention_weight_min": -78.7127914428711,
      "activations/layer4_attention_weight_max": 75.32955169677734,
      "activations/layer4_attention_weight_min": -76.3469009399414,
      "activations/layer5_attention_weight_max": 57.6123046875,
      "activations/layer5_attention_weight_min": -73.1620864868164,
      "activations/layer6_attention_weight_max": 44.74052047729492,
      "activations/layer6_attention_weight_min": -48.78144836425781,
      "activations/layer7_attention_weight_max": 65.72968292236328,
      "activations/layer7_attention_weight_min": -59.144630432128906,
      "activations/layer8_attention_weight_max": 44.6581916809082,
      "activations/layer8_attention_weight_min": -48.173858642578125,
      "activations/layer9_attention_weight_max": 40.3258056640625,
      "activations/layer9_attention_weight_min": -43.878353118896484,
      "epoch": 19.53,
      "learning_rate": 2.4288257575757573e-05,
      "loss": 2.7178,
      "step": 336150
    },
    {
      "activations/layer0_attention_weight_max": 15.562783241271973,
      "activations/layer0_attention_weight_min": -13.994903564453125,
      "activations/layer10_attention_weight_max": 39.98597717285156,
      "activations/layer10_attention_weight_min": -39.61610794067383,
      "activations/layer11_attention_weight_max": 35.64502716064453,
      "activations/layer11_attention_weight_min": -37.611778259277344,
      "activations/layer12_attention_weight_max": 21.309894561767578,
      "activations/layer12_attention_weight_min": -24.35546875,
      "activations/layer13_attention_weight_max": 32.658103942871094,
      "activations/layer13_attention_weight_min": -28.613048553466797,
      "activations/layer14_attention_weight_max": 36.915470123291016,
      "activations/layer14_attention_weight_min": -33.70684814453125,
      "activations/layer15_attention_weight_max": 34.17664337158203,
      "activations/layer15_attention_weight_min": -33.857032775878906,
      "activations/layer16_attention_weight_max": 39.56836700439453,
      "activations/layer16_attention_weight_min": -36.640750885009766,
      "activations/layer17_attention_weight_max": 54.39134979248047,
      "activations/layer17_attention_weight_min": -48.27158737182617,
      "activations/layer18_attention_weight_max": 51.34429168701172,
      "activations/layer18_attention_weight_min": -43.43009948730469,
      "activations/layer19_attention_weight_max": 27.007293701171875,
      "activations/layer19_attention_weight_min": -25.21632194519043,
      "activations/layer1_attention_weight_max": 15.782923698425293,
      "activations/layer1_attention_weight_min": -14.638466835021973,
      "activations/layer20_attention_weight_max": 24.5646915435791,
      "activations/layer20_attention_weight_min": -24.7584285736084,
      "activations/layer21_attention_weight_max": 42.226016998291016,
      "activations/layer21_attention_weight_min": -22.204755783081055,
      "activations/layer22_attention_weight_max": 33.65437698364258,
      "activations/layer22_attention_weight_min": -29.010435104370117,
      "activations/layer23_attention_weight_max": 38.14970779418945,
      "activations/layer23_attention_weight_min": -25.868093490600586,
      "activations/layer2_attention_weight_max": 31.371387481689453,
      "activations/layer2_attention_weight_min": -31.30592155456543,
      "activations/layer3_attention_weight_max": 82.35993194580078,
      "activations/layer3_attention_weight_min": -82.89015197753906,
      "activations/layer4_attention_weight_max": 81.73345184326172,
      "activations/layer4_attention_weight_min": -82.67980194091797,
      "activations/layer5_attention_weight_max": 59.12732696533203,
      "activations/layer5_attention_weight_min": -71.31238555908203,
      "activations/layer6_attention_weight_max": 50.720279693603516,
      "activations/layer6_attention_weight_min": -52.36725997924805,
      "activations/layer7_attention_weight_max": 64.80126190185547,
      "activations/layer7_attention_weight_min": -69.51336669921875,
      "activations/layer8_attention_weight_max": 48.974586486816406,
      "activations/layer8_attention_weight_min": -54.64618682861328,
      "activations/layer9_attention_weight_max": 45.06666564941406,
      "activations/layer9_attention_weight_min": -47.89986038208008,
      "epoch": 19.54,
      "learning_rate": 2.426931818181818e-05,
      "loss": 2.7394,
      "step": 336200
    },
    {
      "activations/layer0_attention_weight_max": 15.695029258728027,
      "activations/layer0_attention_weight_min": -14.36552906036377,
      "activations/layer10_attention_weight_max": 37.65380859375,
      "activations/layer10_attention_weight_min": -37.7132453918457,
      "activations/layer11_attention_weight_max": 34.287391662597656,
      "activations/layer11_attention_weight_min": -33.91901779174805,
      "activations/layer12_attention_weight_max": 18.300352096557617,
      "activations/layer12_attention_weight_min": -29.09366798400879,
      "activations/layer13_attention_weight_max": 39.619441986083984,
      "activations/layer13_attention_weight_min": -28.78957176208496,
      "activations/layer14_attention_weight_max": 40.35177993774414,
      "activations/layer14_attention_weight_min": -28.395809173583984,
      "activations/layer15_attention_weight_max": 37.380332946777344,
      "activations/layer15_attention_weight_min": -28.155776977539062,
      "activations/layer16_attention_weight_max": 32.617759704589844,
      "activations/layer16_attention_weight_min": -31.691333770751953,
      "activations/layer17_attention_weight_max": 45.86785888671875,
      "activations/layer17_attention_weight_min": -43.3202018737793,
      "activations/layer18_attention_weight_max": 43.93741226196289,
      "activations/layer18_attention_weight_min": -39.28569412231445,
      "activations/layer19_attention_weight_max": 21.539844512939453,
      "activations/layer19_attention_weight_min": -22.57697105407715,
      "activations/layer1_attention_weight_max": 16.69396209716797,
      "activations/layer1_attention_weight_min": -13.706984519958496,
      "activations/layer20_attention_weight_max": 19.802553176879883,
      "activations/layer20_attention_weight_min": -21.825992584228516,
      "activations/layer21_attention_weight_max": 29.293060302734375,
      "activations/layer21_attention_weight_min": -20.166051864624023,
      "activations/layer22_attention_weight_max": 25.83078384399414,
      "activations/layer22_attention_weight_min": -24.322368621826172,
      "activations/layer23_attention_weight_max": 31.10448455810547,
      "activations/layer23_attention_weight_min": -23.466611862182617,
      "activations/layer2_attention_weight_max": 31.76361083984375,
      "activations/layer2_attention_weight_min": -30.929533004760742,
      "activations/layer3_attention_weight_max": 85.27364349365234,
      "activations/layer3_attention_weight_min": -89.0542984008789,
      "activations/layer4_attention_weight_max": 83.8668441772461,
      "activations/layer4_attention_weight_min": -81.88531494140625,
      "activations/layer5_attention_weight_max": 61.60117721557617,
      "activations/layer5_attention_weight_min": -71.19926452636719,
      "activations/layer6_attention_weight_max": 49.35532760620117,
      "activations/layer6_attention_weight_min": -49.84687423706055,
      "activations/layer7_attention_weight_max": 69.61549377441406,
      "activations/layer7_attention_weight_min": -65.78070831298828,
      "activations/layer8_attention_weight_max": 48.79951477050781,
      "activations/layer8_attention_weight_min": -52.10881423950195,
      "activations/layer9_attention_weight_max": 49.311527252197266,
      "activations/layer9_attention_weight_min": -45.65169143676758,
      "epoch": 19.54,
      "learning_rate": 2.4250378787878782e-05,
      "loss": 2.7293,
      "step": 336250
    },
    {
      "activations/layer0_attention_weight_max": 15.420851707458496,
      "activations/layer0_attention_weight_min": -14.474102020263672,
      "activations/layer10_attention_weight_max": 38.68323516845703,
      "activations/layer10_attention_weight_min": -36.504913330078125,
      "activations/layer11_attention_weight_max": 34.18194580078125,
      "activations/layer11_attention_weight_min": -33.763427734375,
      "activations/layer12_attention_weight_max": 19.850305557250977,
      "activations/layer12_attention_weight_min": -25.620088577270508,
      "activations/layer13_attention_weight_max": 35.143802642822266,
      "activations/layer13_attention_weight_min": -27.761789321899414,
      "activations/layer14_attention_weight_max": 36.67353820800781,
      "activations/layer14_attention_weight_min": -33.604164123535156,
      "activations/layer15_attention_weight_max": 35.32402038574219,
      "activations/layer15_attention_weight_min": -32.09361267089844,
      "activations/layer16_attention_weight_max": 35.79630661010742,
      "activations/layer16_attention_weight_min": -35.40987777709961,
      "activations/layer17_attention_weight_max": 54.532135009765625,
      "activations/layer17_attention_weight_min": -51.2439079284668,
      "activations/layer18_attention_weight_max": 54.00849533081055,
      "activations/layer18_attention_weight_min": -47.67108917236328,
      "activations/layer19_attention_weight_max": 25.398563385009766,
      "activations/layer19_attention_weight_min": -25.011686325073242,
      "activations/layer1_attention_weight_max": 15.486084938049316,
      "activations/layer1_attention_weight_min": -14.909183502197266,
      "activations/layer20_attention_weight_max": 23.078582763671875,
      "activations/layer20_attention_weight_min": -22.15244483947754,
      "activations/layer21_attention_weight_max": 38.18342971801758,
      "activations/layer21_attention_weight_min": -24.279178619384766,
      "activations/layer22_attention_weight_max": 28.764944076538086,
      "activations/layer22_attention_weight_min": -27.301620483398438,
      "activations/layer23_attention_weight_max": 37.48094940185547,
      "activations/layer23_attention_weight_min": -24.109596252441406,
      "activations/layer2_attention_weight_max": 32.0157470703125,
      "activations/layer2_attention_weight_min": -32.26420211791992,
      "activations/layer3_attention_weight_max": 85.73014831542969,
      "activations/layer3_attention_weight_min": -89.59280395507812,
      "activations/layer4_attention_weight_max": 82.7835922241211,
      "activations/layer4_attention_weight_min": -83.17250061035156,
      "activations/layer5_attention_weight_max": 66.30647277832031,
      "activations/layer5_attention_weight_min": -75.27739715576172,
      "activations/layer6_attention_weight_max": 49.13893127441406,
      "activations/layer6_attention_weight_min": -49.920902252197266,
      "activations/layer7_attention_weight_max": 65.09562683105469,
      "activations/layer7_attention_weight_min": -63.833492279052734,
      "activations/layer8_attention_weight_max": 50.90757369995117,
      "activations/layer8_attention_weight_min": -50.7330207824707,
      "activations/layer9_attention_weight_max": 46.24498748779297,
      "activations/layer9_attention_weight_min": -46.03346252441406,
      "epoch": 19.54,
      "learning_rate": 2.423143939393939e-05,
      "loss": 2.7321,
      "step": 336300
    },
    {
      "activations/layer0_attention_weight_max": 15.100085258483887,
      "activations/layer0_attention_weight_min": -14.487235069274902,
      "activations/layer10_attention_weight_max": 40.33662796020508,
      "activations/layer10_attention_weight_min": -36.59376525878906,
      "activations/layer11_attention_weight_max": 38.62109375,
      "activations/layer11_attention_weight_min": -34.105472564697266,
      "activations/layer12_attention_weight_max": 17.38288688659668,
      "activations/layer12_attention_weight_min": -24.758752822875977,
      "activations/layer13_attention_weight_max": 34.72393035888672,
      "activations/layer13_attention_weight_min": -29.45097541809082,
      "activations/layer14_attention_weight_max": 32.74189376831055,
      "activations/layer14_attention_weight_min": -29.17020034790039,
      "activations/layer15_attention_weight_max": 33.17048645019531,
      "activations/layer15_attention_weight_min": -29.765409469604492,
      "activations/layer16_attention_weight_max": 33.341129302978516,
      "activations/layer16_attention_weight_min": -30.086700439453125,
      "activations/layer17_attention_weight_max": 50.84052276611328,
      "activations/layer17_attention_weight_min": -43.621681213378906,
      "activations/layer18_attention_weight_max": 46.57072448730469,
      "activations/layer18_attention_weight_min": -37.59022903442383,
      "activations/layer19_attention_weight_max": 25.116796493530273,
      "activations/layer19_attention_weight_min": -21.892555236816406,
      "activations/layer1_attention_weight_max": 15.458431243896484,
      "activations/layer1_attention_weight_min": -12.827421188354492,
      "activations/layer20_attention_weight_max": 22.153409957885742,
      "activations/layer20_attention_weight_min": -21.33696746826172,
      "activations/layer21_attention_weight_max": 36.554134368896484,
      "activations/layer21_attention_weight_min": -22.55848503112793,
      "activations/layer22_attention_weight_max": 28.668386459350586,
      "activations/layer22_attention_weight_min": -24.840343475341797,
      "activations/layer23_attention_weight_max": 38.46897888183594,
      "activations/layer23_attention_weight_min": -23.526378631591797,
      "activations/layer2_attention_weight_max": 33.44581604003906,
      "activations/layer2_attention_weight_min": -30.77750015258789,
      "activations/layer3_attention_weight_max": 91.85403442382812,
      "activations/layer3_attention_weight_min": -94.10321807861328,
      "activations/layer4_attention_weight_max": 82.35051727294922,
      "activations/layer4_attention_weight_min": -83.57598114013672,
      "activations/layer5_attention_weight_max": 60.300350189208984,
      "activations/layer5_attention_weight_min": -77.84506225585938,
      "activations/layer6_attention_weight_max": 47.18779373168945,
      "activations/layer6_attention_weight_min": -48.4911003112793,
      "activations/layer7_attention_weight_max": 71.84512329101562,
      "activations/layer7_attention_weight_min": -69.20918273925781,
      "activations/layer8_attention_weight_max": 53.5951042175293,
      "activations/layer8_attention_weight_min": -48.28440475463867,
      "activations/layer9_attention_weight_max": 55.31611251831055,
      "activations/layer9_attention_weight_min": -48.64146423339844,
      "epoch": 19.54,
      "learning_rate": 2.42125e-05,
      "loss": 2.7218,
      "step": 336350
    },
    {
      "activations/layer0_attention_weight_max": 16.16082191467285,
      "activations/layer0_attention_weight_min": -14.052607536315918,
      "activations/layer10_attention_weight_max": 34.95572280883789,
      "activations/layer10_attention_weight_min": -35.165740966796875,
      "activations/layer11_attention_weight_max": 33.02650451660156,
      "activations/layer11_attention_weight_min": -35.35342025756836,
      "activations/layer12_attention_weight_max": 17.745595932006836,
      "activations/layer12_attention_weight_min": -24.2039794921875,
      "activations/layer13_attention_weight_max": 30.98877716064453,
      "activations/layer13_attention_weight_min": -27.636653900146484,
      "activations/layer14_attention_weight_max": 34.73066711425781,
      "activations/layer14_attention_weight_min": -31.169891357421875,
      "activations/layer15_attention_weight_max": 31.190988540649414,
      "activations/layer15_attention_weight_min": -31.135770797729492,
      "activations/layer16_attention_weight_max": 32.140350341796875,
      "activations/layer16_attention_weight_min": -32.217044830322266,
      "activations/layer17_attention_weight_max": 48.46581268310547,
      "activations/layer17_attention_weight_min": -48.991641998291016,
      "activations/layer18_attention_weight_max": 43.6076774597168,
      "activations/layer18_attention_weight_min": -41.727325439453125,
      "activations/layer19_attention_weight_max": 24.142881393432617,
      "activations/layer19_attention_weight_min": -22.673898696899414,
      "activations/layer1_attention_weight_max": 16.26895523071289,
      "activations/layer1_attention_weight_min": -13.761054992675781,
      "activations/layer20_attention_weight_max": 22.193273544311523,
      "activations/layer20_attention_weight_min": -23.37799644470215,
      "activations/layer21_attention_weight_max": 32.01298904418945,
      "activations/layer21_attention_weight_min": -24.27117156982422,
      "activations/layer22_attention_weight_max": 27.04142951965332,
      "activations/layer22_attention_weight_min": -27.023420333862305,
      "activations/layer23_attention_weight_max": 35.38951110839844,
      "activations/layer23_attention_weight_min": -23.612876892089844,
      "activations/layer2_attention_weight_max": 29.89902114868164,
      "activations/layer2_attention_weight_min": -29.757436752319336,
      "activations/layer3_attention_weight_max": 81.20796966552734,
      "activations/layer3_attention_weight_min": -83.42757415771484,
      "activations/layer4_attention_weight_max": 79.47354125976562,
      "activations/layer4_attention_weight_min": -82.1050033569336,
      "activations/layer5_attention_weight_max": 60.25901412963867,
      "activations/layer5_attention_weight_min": -68.21084594726562,
      "activations/layer6_attention_weight_max": 47.16023635864258,
      "activations/layer6_attention_weight_min": -49.92501449584961,
      "activations/layer7_attention_weight_max": 63.34735107421875,
      "activations/layer7_attention_weight_min": -64.55497741699219,
      "activations/layer8_attention_weight_max": 45.49899673461914,
      "activations/layer8_attention_weight_min": -48.695796966552734,
      "activations/layer9_attention_weight_max": 43.71818923950195,
      "activations/layer9_attention_weight_min": -45.80247116088867,
      "epoch": 19.55,
      "learning_rate": 2.4193560606060604e-05,
      "loss": 2.711,
      "step": 336400
    },
    {
      "activations/layer0_attention_weight_max": 15.616826057434082,
      "activations/layer0_attention_weight_min": -14.270764350891113,
      "activations/layer10_attention_weight_max": 38.163509368896484,
      "activations/layer10_attention_weight_min": -36.27315139770508,
      "activations/layer11_attention_weight_max": 36.519996643066406,
      "activations/layer11_attention_weight_min": -35.79308319091797,
      "activations/layer12_attention_weight_max": 20.629369735717773,
      "activations/layer12_attention_weight_min": -27.101049423217773,
      "activations/layer13_attention_weight_max": 32.494632720947266,
      "activations/layer13_attention_weight_min": -28.512277603149414,
      "activations/layer14_attention_weight_max": 33.85910415649414,
      "activations/layer14_attention_weight_min": -30.423870086669922,
      "activations/layer15_attention_weight_max": 31.16021728515625,
      "activations/layer15_attention_weight_min": -31.21367073059082,
      "activations/layer16_attention_weight_max": 33.089256286621094,
      "activations/layer16_attention_weight_min": -34.092018127441406,
      "activations/layer17_attention_weight_max": 53.83292770385742,
      "activations/layer17_attention_weight_min": -47.501670837402344,
      "activations/layer18_attention_weight_max": 44.56687927246094,
      "activations/layer18_attention_weight_min": -42.36174774169922,
      "activations/layer19_attention_weight_max": 22.905094146728516,
      "activations/layer19_attention_weight_min": -22.53985023498535,
      "activations/layer1_attention_weight_max": 15.339468002319336,
      "activations/layer1_attention_weight_min": -12.785897254943848,
      "activations/layer20_attention_weight_max": 21.913372039794922,
      "activations/layer20_attention_weight_min": -20.727642059326172,
      "activations/layer21_attention_weight_max": 38.6049690246582,
      "activations/layer21_attention_weight_min": -23.131929397583008,
      "activations/layer22_attention_weight_max": 29.81653594970703,
      "activations/layer22_attention_weight_min": -26.399274826049805,
      "activations/layer23_attention_weight_max": 35.84048843383789,
      "activations/layer23_attention_weight_min": -22.579967498779297,
      "activations/layer2_attention_weight_max": 31.56319236755371,
      "activations/layer2_attention_weight_min": -29.67493438720703,
      "activations/layer3_attention_weight_max": 89.18980407714844,
      "activations/layer3_attention_weight_min": -86.6312484741211,
      "activations/layer4_attention_weight_max": 85.86156463623047,
      "activations/layer4_attention_weight_min": -80.18840026855469,
      "activations/layer5_attention_weight_max": 64.0634765625,
      "activations/layer5_attention_weight_min": -70.15577697753906,
      "activations/layer6_attention_weight_max": 48.80287170410156,
      "activations/layer6_attention_weight_min": -50.468013763427734,
      "activations/layer7_attention_weight_max": 64.63819885253906,
      "activations/layer7_attention_weight_min": -65.63471221923828,
      "activations/layer8_attention_weight_max": 49.334983825683594,
      "activations/layer8_attention_weight_min": -53.04595947265625,
      "activations/layer9_attention_weight_max": 50.33100128173828,
      "activations/layer9_attention_weight_min": -46.694759368896484,
      "epoch": 19.55,
      "learning_rate": 2.4174621212121212e-05,
      "loss": 2.7256,
      "step": 336450
    },
    {
      "activations/layer0_attention_weight_max": 15.187260627746582,
      "activations/layer0_attention_weight_min": -14.960416793823242,
      "activations/layer10_attention_weight_max": 37.93497848510742,
      "activations/layer10_attention_weight_min": -37.73612976074219,
      "activations/layer11_attention_weight_max": 35.625694274902344,
      "activations/layer11_attention_weight_min": -34.10969924926758,
      "activations/layer12_attention_weight_max": 18.441482543945312,
      "activations/layer12_attention_weight_min": -25.445493698120117,
      "activations/layer13_attention_weight_max": 35.487281799316406,
      "activations/layer13_attention_weight_min": -29.10101318359375,
      "activations/layer14_attention_weight_max": 36.61051559448242,
      "activations/layer14_attention_weight_min": -29.4892578125,
      "activations/layer15_attention_weight_max": 34.96578598022461,
      "activations/layer15_attention_weight_min": -28.61269187927246,
      "activations/layer16_attention_weight_max": 35.25356674194336,
      "activations/layer16_attention_weight_min": -30.857303619384766,
      "activations/layer17_attention_weight_max": 54.08366775512695,
      "activations/layer17_attention_weight_min": -46.645423889160156,
      "activations/layer18_attention_weight_max": 52.66884994506836,
      "activations/layer18_attention_weight_min": -45.6494255065918,
      "activations/layer19_attention_weight_max": 23.38644027709961,
      "activations/layer19_attention_weight_min": -22.811317443847656,
      "activations/layer1_attention_weight_max": 17.109540939331055,
      "activations/layer1_attention_weight_min": -13.63279914855957,
      "activations/layer20_attention_weight_max": 22.604633331298828,
      "activations/layer20_attention_weight_min": -21.002294540405273,
      "activations/layer21_attention_weight_max": 40.48841094970703,
      "activations/layer21_attention_weight_min": -23.97406005859375,
      "activations/layer22_attention_weight_max": 27.95192527770996,
      "activations/layer22_attention_weight_min": -24.278554916381836,
      "activations/layer23_attention_weight_max": 35.83805847167969,
      "activations/layer23_attention_weight_min": -24.231243133544922,
      "activations/layer2_attention_weight_max": 32.86726760864258,
      "activations/layer2_attention_weight_min": -31.770803451538086,
      "activations/layer3_attention_weight_max": 86.08618927001953,
      "activations/layer3_attention_weight_min": -85.78117370605469,
      "activations/layer4_attention_weight_max": 83.51727294921875,
      "activations/layer4_attention_weight_min": -84.45478057861328,
      "activations/layer5_attention_weight_max": 59.98168182373047,
      "activations/layer5_attention_weight_min": -67.74272918701172,
      "activations/layer6_attention_weight_max": 48.95804977416992,
      "activations/layer6_attention_weight_min": -52.51204299926758,
      "activations/layer7_attention_weight_max": 64.53395080566406,
      "activations/layer7_attention_weight_min": -66.27513122558594,
      "activations/layer8_attention_weight_max": 48.537353515625,
      "activations/layer8_attention_weight_min": -53.39946365356445,
      "activations/layer9_attention_weight_max": 48.341712951660156,
      "activations/layer9_attention_weight_min": -46.87095260620117,
      "epoch": 19.55,
      "learning_rate": 2.4155681818181817e-05,
      "loss": 2.7337,
      "step": 336500
    },
    {
      "activations/layer0_attention_weight_max": 16.103225708007812,
      "activations/layer0_attention_weight_min": -14.110969543457031,
      "activations/layer10_attention_weight_max": 37.12900161743164,
      "activations/layer10_attention_weight_min": -36.02930450439453,
      "activations/layer11_attention_weight_max": 36.67344284057617,
      "activations/layer11_attention_weight_min": -34.73197937011719,
      "activations/layer12_attention_weight_max": 19.334556579589844,
      "activations/layer12_attention_weight_min": -28.035846710205078,
      "activations/layer13_attention_weight_max": 38.70916748046875,
      "activations/layer13_attention_weight_min": -29.01007080078125,
      "activations/layer14_attention_weight_max": 38.957359313964844,
      "activations/layer14_attention_weight_min": -33.613243103027344,
      "activations/layer15_attention_weight_max": 37.959381103515625,
      "activations/layer15_attention_weight_min": -34.90485763549805,
      "activations/layer16_attention_weight_max": 36.87205123901367,
      "activations/layer16_attention_weight_min": -34.037105560302734,
      "activations/layer17_attention_weight_max": 53.79094314575195,
      "activations/layer17_attention_weight_min": -48.13180160522461,
      "activations/layer18_attention_weight_max": 49.43011474609375,
      "activations/layer18_attention_weight_min": -41.546531677246094,
      "activations/layer19_attention_weight_max": 26.762266159057617,
      "activations/layer19_attention_weight_min": -22.980016708374023,
      "activations/layer1_attention_weight_max": 15.662850379943848,
      "activations/layer1_attention_weight_min": -12.475732803344727,
      "activations/layer20_attention_weight_max": 26.362930297851562,
      "activations/layer20_attention_weight_min": -22.424076080322266,
      "activations/layer21_attention_weight_max": 43.83677291870117,
      "activations/layer21_attention_weight_min": -24.123838424682617,
      "activations/layer22_attention_weight_max": 29.347606658935547,
      "activations/layer22_attention_weight_min": -25.60422706604004,
      "activations/layer23_attention_weight_max": 39.02415466308594,
      "activations/layer23_attention_weight_min": -22.57095718383789,
      "activations/layer2_attention_weight_max": 29.90735626220703,
      "activations/layer2_attention_weight_min": -26.693500518798828,
      "activations/layer3_attention_weight_max": 84.78309631347656,
      "activations/layer3_attention_weight_min": -79.90647888183594,
      "activations/layer4_attention_weight_max": 86.5609359741211,
      "activations/layer4_attention_weight_min": -80.84114837646484,
      "activations/layer5_attention_weight_max": 62.727378845214844,
      "activations/layer5_attention_weight_min": -70.07263946533203,
      "activations/layer6_attention_weight_max": 49.649600982666016,
      "activations/layer6_attention_weight_min": -51.1823616027832,
      "activations/layer7_attention_weight_max": 66.41239929199219,
      "activations/layer7_attention_weight_min": -66.11373901367188,
      "activations/layer8_attention_weight_max": 48.43245315551758,
      "activations/layer8_attention_weight_min": -52.51216125488281,
      "activations/layer9_attention_weight_max": 48.88283920288086,
      "activations/layer9_attention_weight_min": -48.9908561706543,
      "epoch": 19.56,
      "learning_rate": 2.4136742424242425e-05,
      "loss": 2.7362,
      "step": 336550
    },
    {
      "activations/layer0_attention_weight_max": 15.42713451385498,
      "activations/layer0_attention_weight_min": -14.133649826049805,
      "activations/layer10_attention_weight_max": 35.53957748413086,
      "activations/layer10_attention_weight_min": -36.128902435302734,
      "activations/layer11_attention_weight_max": 33.70051574707031,
      "activations/layer11_attention_weight_min": -35.73506164550781,
      "activations/layer12_attention_weight_max": 18.852352142333984,
      "activations/layer12_attention_weight_min": -25.354021072387695,
      "activations/layer13_attention_weight_max": 34.991188049316406,
      "activations/layer13_attention_weight_min": -27.84490966796875,
      "activations/layer14_attention_weight_max": 37.449562072753906,
      "activations/layer14_attention_weight_min": -32.7412109375,
      "activations/layer15_attention_weight_max": 32.39523696899414,
      "activations/layer15_attention_weight_min": -32.29486083984375,
      "activations/layer16_attention_weight_max": 35.039615631103516,
      "activations/layer16_attention_weight_min": -32.248199462890625,
      "activations/layer17_attention_weight_max": 52.405059814453125,
      "activations/layer17_attention_weight_min": -45.81252670288086,
      "activations/layer18_attention_weight_max": 45.57244110107422,
      "activations/layer18_attention_weight_min": -39.72478103637695,
      "activations/layer19_attention_weight_max": 23.6375789642334,
      "activations/layer19_attention_weight_min": -24.0850887298584,
      "activations/layer1_attention_weight_max": 15.915141105651855,
      "activations/layer1_attention_weight_min": -13.511062622070312,
      "activations/layer20_attention_weight_max": 23.008670806884766,
      "activations/layer20_attention_weight_min": -20.929502487182617,
      "activations/layer21_attention_weight_max": 35.74656677246094,
      "activations/layer21_attention_weight_min": -23.12551498413086,
      "activations/layer22_attention_weight_max": 29.343727111816406,
      "activations/layer22_attention_weight_min": -28.332422256469727,
      "activations/layer23_attention_weight_max": 35.95025634765625,
      "activations/layer23_attention_weight_min": -24.812604904174805,
      "activations/layer2_attention_weight_max": 31.527379989624023,
      "activations/layer2_attention_weight_min": -29.40692710876465,
      "activations/layer3_attention_weight_max": 80.67952728271484,
      "activations/layer3_attention_weight_min": -81.80826568603516,
      "activations/layer4_attention_weight_max": 79.34010314941406,
      "activations/layer4_attention_weight_min": -78.64207458496094,
      "activations/layer5_attention_weight_max": 58.55196762084961,
      "activations/layer5_attention_weight_min": -67.48469543457031,
      "activations/layer6_attention_weight_max": 47.56065368652344,
      "activations/layer6_attention_weight_min": -48.424598693847656,
      "activations/layer7_attention_weight_max": 65.87074279785156,
      "activations/layer7_attention_weight_min": -62.596534729003906,
      "activations/layer8_attention_weight_max": 42.700965881347656,
      "activations/layer8_attention_weight_min": -47.24351501464844,
      "activations/layer9_attention_weight_max": 42.33150863647461,
      "activations/layer9_attention_weight_min": -45.1722526550293,
      "epoch": 19.56,
      "learning_rate": 2.4117803030303027e-05,
      "loss": 2.7335,
      "step": 336600
    },
    {
      "activations/layer0_attention_weight_max": 16.232234954833984,
      "activations/layer0_attention_weight_min": -14.346659660339355,
      "activations/layer10_attention_weight_max": 34.3664436340332,
      "activations/layer10_attention_weight_min": -36.832393646240234,
      "activations/layer11_attention_weight_max": 32.343997955322266,
      "activations/layer11_attention_weight_min": -37.833045959472656,
      "activations/layer12_attention_weight_max": 19.813125610351562,
      "activations/layer12_attention_weight_min": -26.25888442993164,
      "activations/layer13_attention_weight_max": 29.597070693969727,
      "activations/layer13_attention_weight_min": -26.623098373413086,
      "activations/layer14_attention_weight_max": 33.81141662597656,
      "activations/layer14_attention_weight_min": -29.62223243713379,
      "activations/layer15_attention_weight_max": 33.2769889831543,
      "activations/layer15_attention_weight_min": -30.21632194519043,
      "activations/layer16_attention_weight_max": 35.09675979614258,
      "activations/layer16_attention_weight_min": -33.532833099365234,
      "activations/layer17_attention_weight_max": 51.7745246887207,
      "activations/layer17_attention_weight_min": -45.46049880981445,
      "activations/layer18_attention_weight_max": 48.16033172607422,
      "activations/layer18_attention_weight_min": -39.5315055847168,
      "activations/layer19_attention_weight_max": 24.438486099243164,
      "activations/layer19_attention_weight_min": -23.2550048828125,
      "activations/layer1_attention_weight_max": 15.993897438049316,
      "activations/layer1_attention_weight_min": -13.49294376373291,
      "activations/layer20_attention_weight_max": 22.806650161743164,
      "activations/layer20_attention_weight_min": -23.901805877685547,
      "activations/layer21_attention_weight_max": 35.85992431640625,
      "activations/layer21_attention_weight_min": -22.373159408569336,
      "activations/layer22_attention_weight_max": 30.957563400268555,
      "activations/layer22_attention_weight_min": -26.90032958984375,
      "activations/layer23_attention_weight_max": 35.221107482910156,
      "activations/layer23_attention_weight_min": -22.792648315429688,
      "activations/layer2_attention_weight_max": 29.269086837768555,
      "activations/layer2_attention_weight_min": -27.836952209472656,
      "activations/layer3_attention_weight_max": 84.37540435791016,
      "activations/layer3_attention_weight_min": -79.7959213256836,
      "activations/layer4_attention_weight_max": 82.37913513183594,
      "activations/layer4_attention_weight_min": -79.41938018798828,
      "activations/layer5_attention_weight_max": 60.2142333984375,
      "activations/layer5_attention_weight_min": -70.67733764648438,
      "activations/layer6_attention_weight_max": 48.5005989074707,
      "activations/layer6_attention_weight_min": -50.128963470458984,
      "activations/layer7_attention_weight_max": 63.098289489746094,
      "activations/layer7_attention_weight_min": -65.7260513305664,
      "activations/layer8_attention_weight_max": 45.826622009277344,
      "activations/layer8_attention_weight_min": -50.781105041503906,
      "activations/layer9_attention_weight_max": 44.15641403198242,
      "activations/layer9_attention_weight_min": -47.71620559692383,
      "epoch": 19.56,
      "learning_rate": 2.4098863636363635e-05,
      "loss": 2.7239,
      "step": 336650
    },
    {
      "activations/layer0_attention_weight_max": 15.82608699798584,
      "activations/layer0_attention_weight_min": -14.033134460449219,
      "activations/layer10_attention_weight_max": 32.90107727050781,
      "activations/layer10_attention_weight_min": -35.469017028808594,
      "activations/layer11_attention_weight_max": 30.354801177978516,
      "activations/layer11_attention_weight_min": -33.70635986328125,
      "activations/layer12_attention_weight_max": 19.0593204498291,
      "activations/layer12_attention_weight_min": -23.158098220825195,
      "activations/layer13_attention_weight_max": 33.413414001464844,
      "activations/layer13_attention_weight_min": -27.630510330200195,
      "activations/layer14_attention_weight_max": 33.76985549926758,
      "activations/layer14_attention_weight_min": -30.988584518432617,
      "activations/layer15_attention_weight_max": 31.439956665039062,
      "activations/layer15_attention_weight_min": -29.698453903198242,
      "activations/layer16_attention_weight_max": 32.79771423339844,
      "activations/layer16_attention_weight_min": -30.735721588134766,
      "activations/layer17_attention_weight_max": 48.73466110229492,
      "activations/layer17_attention_weight_min": -44.30598831176758,
      "activations/layer18_attention_weight_max": 45.79526138305664,
      "activations/layer18_attention_weight_min": -41.16874694824219,
      "activations/layer19_attention_weight_max": 24.461193084716797,
      "activations/layer19_attention_weight_min": -23.78681755065918,
      "activations/layer1_attention_weight_max": 15.55952262878418,
      "activations/layer1_attention_weight_min": -14.508316993713379,
      "activations/layer20_attention_weight_max": 22.991914749145508,
      "activations/layer20_attention_weight_min": -22.43282127380371,
      "activations/layer21_attention_weight_max": 35.018211364746094,
      "activations/layer21_attention_weight_min": -21.998802185058594,
      "activations/layer22_attention_weight_max": 28.73971176147461,
      "activations/layer22_attention_weight_min": -26.155305862426758,
      "activations/layer23_attention_weight_max": 32.956634521484375,
      "activations/layer23_attention_weight_min": -23.30975341796875,
      "activations/layer2_attention_weight_max": 30.139930725097656,
      "activations/layer2_attention_weight_min": -28.76959991455078,
      "activations/layer3_attention_weight_max": 83.9758071899414,
      "activations/layer3_attention_weight_min": -86.93827819824219,
      "activations/layer4_attention_weight_max": 82.47425079345703,
      "activations/layer4_attention_weight_min": -80.97221374511719,
      "activations/layer5_attention_weight_max": 60.361671447753906,
      "activations/layer5_attention_weight_min": -71.65856170654297,
      "activations/layer6_attention_weight_max": 48.37641525268555,
      "activations/layer6_attention_weight_min": -50.49943923950195,
      "activations/layer7_attention_weight_max": 64.06641387939453,
      "activations/layer7_attention_weight_min": -64.2473373413086,
      "activations/layer8_attention_weight_max": 47.56348419189453,
      "activations/layer8_attention_weight_min": -48.60681915283203,
      "activations/layer9_attention_weight_max": 41.9619026184082,
      "activations/layer9_attention_weight_min": -45.056060791015625,
      "epoch": 19.56,
      "learning_rate": 2.407992424242424e-05,
      "loss": 2.7308,
      "step": 336700
    },
    {
      "activations/layer0_attention_weight_max": 16.6181640625,
      "activations/layer0_attention_weight_min": -13.933534622192383,
      "activations/layer10_attention_weight_max": 42.06843948364258,
      "activations/layer10_attention_weight_min": -40.78255844116211,
      "activations/layer11_attention_weight_max": 41.58190155029297,
      "activations/layer11_attention_weight_min": -40.01856231689453,
      "activations/layer12_attention_weight_max": 22.47073745727539,
      "activations/layer12_attention_weight_min": -28.99478530883789,
      "activations/layer13_attention_weight_max": 35.84832000732422,
      "activations/layer13_attention_weight_min": -31.23350715637207,
      "activations/layer14_attention_weight_max": 39.234920501708984,
      "activations/layer14_attention_weight_min": -34.41111755371094,
      "activations/layer15_attention_weight_max": 36.01786422729492,
      "activations/layer15_attention_weight_min": -33.7480354309082,
      "activations/layer16_attention_weight_max": 41.686851501464844,
      "activations/layer16_attention_weight_min": -38.6141471862793,
      "activations/layer17_attention_weight_max": 53.328956604003906,
      "activations/layer17_attention_weight_min": -49.979732513427734,
      "activations/layer18_attention_weight_max": 49.15009689331055,
      "activations/layer18_attention_weight_min": -44.22997283935547,
      "activations/layer19_attention_weight_max": 26.240087509155273,
      "activations/layer19_attention_weight_min": -25.538908004760742,
      "activations/layer1_attention_weight_max": 15.470097541809082,
      "activations/layer1_attention_weight_min": -14.237343788146973,
      "activations/layer20_attention_weight_max": 23.760643005371094,
      "activations/layer20_attention_weight_min": -21.623563766479492,
      "activations/layer21_attention_weight_max": 39.528831481933594,
      "activations/layer21_attention_weight_min": -21.147010803222656,
      "activations/layer22_attention_weight_max": 34.51609420776367,
      "activations/layer22_attention_weight_min": -28.039772033691406,
      "activations/layer23_attention_weight_max": 37.77056121826172,
      "activations/layer23_attention_weight_min": -23.093151092529297,
      "activations/layer2_attention_weight_max": 31.96926498413086,
      "activations/layer2_attention_weight_min": -28.58674430847168,
      "activations/layer3_attention_weight_max": 88.72309112548828,
      "activations/layer3_attention_weight_min": -90.48290252685547,
      "activations/layer4_attention_weight_max": 84.76225280761719,
      "activations/layer4_attention_weight_min": -84.72962951660156,
      "activations/layer5_attention_weight_max": 59.04004669189453,
      "activations/layer5_attention_weight_min": -72.4024429321289,
      "activations/layer6_attention_weight_max": 51.294288635253906,
      "activations/layer6_attention_weight_min": -50.996150970458984,
      "activations/layer7_attention_weight_max": 69.79804992675781,
      "activations/layer7_attention_weight_min": -71.90138244628906,
      "activations/layer8_attention_weight_max": 56.96873092651367,
      "activations/layer8_attention_weight_min": -55.210323333740234,
      "activations/layer9_attention_weight_max": 53.44812774658203,
      "activations/layer9_attention_weight_min": -52.09083938598633,
      "epoch": 19.57,
      "learning_rate": 2.4060984848484848e-05,
      "loss": 2.7234,
      "step": 336750
    },
    {
      "activations/layer0_attention_weight_max": 15.581062316894531,
      "activations/layer0_attention_weight_min": -13.962658882141113,
      "activations/layer10_attention_weight_max": 35.590614318847656,
      "activations/layer10_attention_weight_min": -36.8978271484375,
      "activations/layer11_attention_weight_max": 33.35083770751953,
      "activations/layer11_attention_weight_min": -33.525455474853516,
      "activations/layer12_attention_weight_max": 20.25868034362793,
      "activations/layer12_attention_weight_min": -26.157560348510742,
      "activations/layer13_attention_weight_max": 42.65705108642578,
      "activations/layer13_attention_weight_min": -28.75438690185547,
      "activations/layer14_attention_weight_max": 36.799991607666016,
      "activations/layer14_attention_weight_min": -31.138500213623047,
      "activations/layer15_attention_weight_max": 35.087493896484375,
      "activations/layer15_attention_weight_min": -31.105268478393555,
      "activations/layer16_attention_weight_max": 36.27121353149414,
      "activations/layer16_attention_weight_min": -32.961299896240234,
      "activations/layer17_attention_weight_max": 54.76707458496094,
      "activations/layer17_attention_weight_min": -47.97262954711914,
      "activations/layer18_attention_weight_max": 49.234580993652344,
      "activations/layer18_attention_weight_min": -41.53272247314453,
      "activations/layer19_attention_weight_max": 25.388019561767578,
      "activations/layer19_attention_weight_min": -21.28394317626953,
      "activations/layer1_attention_weight_max": 16.98615074157715,
      "activations/layer1_attention_weight_min": -12.760863304138184,
      "activations/layer20_attention_weight_max": 22.621864318847656,
      "activations/layer20_attention_weight_min": -19.98094367980957,
      "activations/layer21_attention_weight_max": 37.0966796875,
      "activations/layer21_attention_weight_min": -21.41257095336914,
      "activations/layer22_attention_weight_max": 29.541149139404297,
      "activations/layer22_attention_weight_min": -24.831787109375,
      "activations/layer23_attention_weight_max": 37.01854705810547,
      "activations/layer23_attention_weight_min": -23.920772552490234,
      "activations/layer2_attention_weight_max": 31.176206588745117,
      "activations/layer2_attention_weight_min": -28.178478240966797,
      "activations/layer3_attention_weight_max": 83.74455261230469,
      "activations/layer3_attention_weight_min": -83.3299789428711,
      "activations/layer4_attention_weight_max": 83.51895904541016,
      "activations/layer4_attention_weight_min": -82.14158630371094,
      "activations/layer5_attention_weight_max": 58.627986907958984,
      "activations/layer5_attention_weight_min": -69.6205062866211,
      "activations/layer6_attention_weight_max": 46.12295913696289,
      "activations/layer6_attention_weight_min": -49.97773361206055,
      "activations/layer7_attention_weight_max": 66.88327026367188,
      "activations/layer7_attention_weight_min": -64.24386596679688,
      "activations/layer8_attention_weight_max": 48.069095611572266,
      "activations/layer8_attention_weight_min": -50.45542526245117,
      "activations/layer9_attention_weight_max": 49.29523849487305,
      "activations/layer9_attention_weight_min": -45.44016647338867,
      "epoch": 19.57,
      "learning_rate": 2.404204545454545e-05,
      "loss": 2.7384,
      "step": 336800
    },
    {
      "activations/layer0_attention_weight_max": 14.657842636108398,
      "activations/layer0_attention_weight_min": -14.041760444641113,
      "activations/layer10_attention_weight_max": 34.2844123840332,
      "activations/layer10_attention_weight_min": -36.11754608154297,
      "activations/layer11_attention_weight_max": 32.58856964111328,
      "activations/layer11_attention_weight_min": -33.668739318847656,
      "activations/layer12_attention_weight_max": 20.530242919921875,
      "activations/layer12_attention_weight_min": -26.687135696411133,
      "activations/layer13_attention_weight_max": 38.03731155395508,
      "activations/layer13_attention_weight_min": -25.41712188720703,
      "activations/layer14_attention_weight_max": 35.0219841003418,
      "activations/layer14_attention_weight_min": -31.220531463623047,
      "activations/layer15_attention_weight_max": 32.07368469238281,
      "activations/layer15_attention_weight_min": -32.03826904296875,
      "activations/layer16_attention_weight_max": 34.362327575683594,
      "activations/layer16_attention_weight_min": -32.04117202758789,
      "activations/layer17_attention_weight_max": 51.79999923706055,
      "activations/layer17_attention_weight_min": -45.15871047973633,
      "activations/layer18_attention_weight_max": 47.65590286254883,
      "activations/layer18_attention_weight_min": -38.65315628051758,
      "activations/layer19_attention_weight_max": 26.353307723999023,
      "activations/layer19_attention_weight_min": -22.70823860168457,
      "activations/layer1_attention_weight_max": 15.563559532165527,
      "activations/layer1_attention_weight_min": -13.220205307006836,
      "activations/layer20_attention_weight_max": 25.387067794799805,
      "activations/layer20_attention_weight_min": -21.474933624267578,
      "activations/layer21_attention_weight_max": 37.65781784057617,
      "activations/layer21_attention_weight_min": -22.069040298461914,
      "activations/layer22_attention_weight_max": 33.427425384521484,
      "activations/layer22_attention_weight_min": -27.366418838500977,
      "activations/layer23_attention_weight_max": 42.563514709472656,
      "activations/layer23_attention_weight_min": -25.835952758789062,
      "activations/layer2_attention_weight_max": 30.398595809936523,
      "activations/layer2_attention_weight_min": -28.997642517089844,
      "activations/layer3_attention_weight_max": 80.26544952392578,
      "activations/layer3_attention_weight_min": -82.28355407714844,
      "activations/layer4_attention_weight_max": 82.84944915771484,
      "activations/layer4_attention_weight_min": -86.62840270996094,
      "activations/layer5_attention_weight_max": 59.70919418334961,
      "activations/layer5_attention_weight_min": -72.9836654663086,
      "activations/layer6_attention_weight_max": 48.9243278503418,
      "activations/layer6_attention_weight_min": -50.87934112548828,
      "activations/layer7_attention_weight_max": 67.36371612548828,
      "activations/layer7_attention_weight_min": -63.044559478759766,
      "activations/layer8_attention_weight_max": 45.305145263671875,
      "activations/layer8_attention_weight_min": -47.627445220947266,
      "activations/layer9_attention_weight_max": 42.49062728881836,
      "activations/layer9_attention_weight_min": -47.8960075378418,
      "epoch": 19.57,
      "learning_rate": 2.4023106060606058e-05,
      "loss": 2.7226,
      "step": 336850
    },
    {
      "activations/layer0_attention_weight_max": 15.305765151977539,
      "activations/layer0_attention_weight_min": -14.500162124633789,
      "activations/layer10_attention_weight_max": 37.72173309326172,
      "activations/layer10_attention_weight_min": -37.37786865234375,
      "activations/layer11_attention_weight_max": 33.57434844970703,
      "activations/layer11_attention_weight_min": -34.946434020996094,
      "activations/layer12_attention_weight_max": 18.892553329467773,
      "activations/layer12_attention_weight_min": -26.024131774902344,
      "activations/layer13_attention_weight_max": 33.68375778198242,
      "activations/layer13_attention_weight_min": -27.09640121459961,
      "activations/layer14_attention_weight_max": 34.4250602722168,
      "activations/layer14_attention_weight_min": -29.281707763671875,
      "activations/layer15_attention_weight_max": 30.47987174987793,
      "activations/layer15_attention_weight_min": -29.53758430480957,
      "activations/layer16_attention_weight_max": 32.04640197753906,
      "activations/layer16_attention_weight_min": -32.81679153442383,
      "activations/layer17_attention_weight_max": 48.53229522705078,
      "activations/layer17_attention_weight_min": -44.77904510498047,
      "activations/layer18_attention_weight_max": 44.836341857910156,
      "activations/layer18_attention_weight_min": -41.60686492919922,
      "activations/layer19_attention_weight_max": 22.727903366088867,
      "activations/layer19_attention_weight_min": -24.124223709106445,
      "activations/layer1_attention_weight_max": 15.719425201416016,
      "activations/layer1_attention_weight_min": -13.199911117553711,
      "activations/layer20_attention_weight_max": 22.38995361328125,
      "activations/layer20_attention_weight_min": -22.28891944885254,
      "activations/layer21_attention_weight_max": 33.23567199707031,
      "activations/layer21_attention_weight_min": -19.654071807861328,
      "activations/layer22_attention_weight_max": 29.48818016052246,
      "activations/layer22_attention_weight_min": -27.9604549407959,
      "activations/layer23_attention_weight_max": 38.70656967163086,
      "activations/layer23_attention_weight_min": -24.207069396972656,
      "activations/layer2_attention_weight_max": 31.52227020263672,
      "activations/layer2_attention_weight_min": -28.401927947998047,
      "activations/layer3_attention_weight_max": 86.21260833740234,
      "activations/layer3_attention_weight_min": -81.8482437133789,
      "activations/layer4_attention_weight_max": 82.7592544555664,
      "activations/layer4_attention_weight_min": -82.04723358154297,
      "activations/layer5_attention_weight_max": 62.17976379394531,
      "activations/layer5_attention_weight_min": -68.51030731201172,
      "activations/layer6_attention_weight_max": 49.578067779541016,
      "activations/layer6_attention_weight_min": -51.20325469970703,
      "activations/layer7_attention_weight_max": 63.705448150634766,
      "activations/layer7_attention_weight_min": -65.9891128540039,
      "activations/layer8_attention_weight_max": 46.45845413208008,
      "activations/layer8_attention_weight_min": -51.47502517700195,
      "activations/layer9_attention_weight_max": 44.35538864135742,
      "activations/layer9_attention_weight_min": -49.102989196777344,
      "epoch": 19.58,
      "learning_rate": 2.4004166666666663e-05,
      "loss": 2.728,
      "step": 336900
    },
    {
      "activations/layer0_attention_weight_max": 15.826306343078613,
      "activations/layer0_attention_weight_min": -14.753644943237305,
      "activations/layer10_attention_weight_max": 34.01960754394531,
      "activations/layer10_attention_weight_min": -34.42671585083008,
      "activations/layer11_attention_weight_max": 32.536956787109375,
      "activations/layer11_attention_weight_min": -34.21263122558594,
      "activations/layer12_attention_weight_max": 18.586231231689453,
      "activations/layer12_attention_weight_min": -28.248476028442383,
      "activations/layer13_attention_weight_max": 30.23834991455078,
      "activations/layer13_attention_weight_min": -26.622018814086914,
      "activations/layer14_attention_weight_max": 34.64505386352539,
      "activations/layer14_attention_weight_min": -32.106971740722656,
      "activations/layer15_attention_weight_max": 31.526906967163086,
      "activations/layer15_attention_weight_min": -33.906517028808594,
      "activations/layer16_attention_weight_max": 33.34107208251953,
      "activations/layer16_attention_weight_min": -32.19154739379883,
      "activations/layer17_attention_weight_max": 50.51405715942383,
      "activations/layer17_attention_weight_min": -46.233821868896484,
      "activations/layer18_attention_weight_max": 48.134254455566406,
      "activations/layer18_attention_weight_min": -40.54767990112305,
      "activations/layer19_attention_weight_max": 22.236007690429688,
      "activations/layer19_attention_weight_min": -20.487131118774414,
      "activations/layer1_attention_weight_max": 15.979376792907715,
      "activations/layer1_attention_weight_min": -13.534862518310547,
      "activations/layer20_attention_weight_max": 21.179912567138672,
      "activations/layer20_attention_weight_min": -21.88249397277832,
      "activations/layer21_attention_weight_max": 33.025936126708984,
      "activations/layer21_attention_weight_min": -22.314502716064453,
      "activations/layer22_attention_weight_max": 29.705413818359375,
      "activations/layer22_attention_weight_min": -24.76368522644043,
      "activations/layer23_attention_weight_max": 32.01076126098633,
      "activations/layer23_attention_weight_min": -22.515745162963867,
      "activations/layer2_attention_weight_max": 29.9366455078125,
      "activations/layer2_attention_weight_min": -28.240856170654297,
      "activations/layer3_attention_weight_max": 80.05804443359375,
      "activations/layer3_attention_weight_min": -81.84929656982422,
      "activations/layer4_attention_weight_max": 80.01303100585938,
      "activations/layer4_attention_weight_min": -79.53437042236328,
      "activations/layer5_attention_weight_max": 62.443267822265625,
      "activations/layer5_attention_weight_min": -66.72381591796875,
      "activations/layer6_attention_weight_max": 50.24734878540039,
      "activations/layer6_attention_weight_min": -47.964786529541016,
      "activations/layer7_attention_weight_max": 63.01810073852539,
      "activations/layer7_attention_weight_min": -60.82890319824219,
      "activations/layer8_attention_weight_max": 43.61107635498047,
      "activations/layer8_attention_weight_min": -47.308101654052734,
      "activations/layer9_attention_weight_max": 42.23341751098633,
      "activations/layer9_attention_weight_min": -43.248531341552734,
      "epoch": 19.58,
      "learning_rate": 2.398522727272727e-05,
      "loss": 2.7342,
      "step": 336950
    },
    {
      "activations/layer0_attention_weight_max": 15.58363151550293,
      "activations/layer0_attention_weight_min": -14.465254783630371,
      "activations/layer10_attention_weight_max": 41.9849967956543,
      "activations/layer10_attention_weight_min": -41.05769729614258,
      "activations/layer11_attention_weight_max": 40.45957946777344,
      "activations/layer11_attention_weight_min": -34.86661148071289,
      "activations/layer12_attention_weight_max": 18.498273849487305,
      "activations/layer12_attention_weight_min": -25.460880279541016,
      "activations/layer13_attention_weight_max": 39.511940002441406,
      "activations/layer13_attention_weight_min": -26.81947898864746,
      "activations/layer14_attention_weight_max": 35.74757766723633,
      "activations/layer14_attention_weight_min": -29.719907760620117,
      "activations/layer15_attention_weight_max": 36.47138977050781,
      "activations/layer15_attention_weight_min": -30.253665924072266,
      "activations/layer16_attention_weight_max": 31.819543838500977,
      "activations/layer16_attention_weight_min": -31.270572662353516,
      "activations/layer17_attention_weight_max": 47.179439544677734,
      "activations/layer17_attention_weight_min": -45.98965072631836,
      "activations/layer18_attention_weight_max": 43.92882537841797,
      "activations/layer18_attention_weight_min": -42.27164077758789,
      "activations/layer19_attention_weight_max": 22.179994583129883,
      "activations/layer19_attention_weight_min": -24.019638061523438,
      "activations/layer1_attention_weight_max": 16.008338928222656,
      "activations/layer1_attention_weight_min": -12.335770606994629,
      "activations/layer20_attention_weight_max": 21.54193878173828,
      "activations/layer20_attention_weight_min": -22.618345260620117,
      "activations/layer21_attention_weight_max": 35.559364318847656,
      "activations/layer21_attention_weight_min": -21.165401458740234,
      "activations/layer22_attention_weight_max": 28.826244354248047,
      "activations/layer22_attention_weight_min": -26.974851608276367,
      "activations/layer23_attention_weight_max": 33.29672622680664,
      "activations/layer23_attention_weight_min": -24.14409637451172,
      "activations/layer2_attention_weight_max": 29.90972900390625,
      "activations/layer2_attention_weight_min": -27.751205444335938,
      "activations/layer3_attention_weight_max": 81.20235443115234,
      "activations/layer3_attention_weight_min": -81.33544158935547,
      "activations/layer4_attention_weight_max": 78.43476867675781,
      "activations/layer4_attention_weight_min": -78.18608856201172,
      "activations/layer5_attention_weight_max": 61.36186218261719,
      "activations/layer5_attention_weight_min": -67.09220886230469,
      "activations/layer6_attention_weight_max": 48.5633659362793,
      "activations/layer6_attention_weight_min": -48.82719039916992,
      "activations/layer7_attention_weight_max": 76.99295043945312,
      "activations/layer7_attention_weight_min": -60.875308990478516,
      "activations/layer8_attention_weight_max": 56.11818313598633,
      "activations/layer8_attention_weight_min": -49.17595291137695,
      "activations/layer9_attention_weight_max": 60.32883834838867,
      "activations/layer9_attention_weight_min": -45.88651657104492,
      "epoch": 19.58,
      "learning_rate": 2.3966287878787876e-05,
      "loss": 2.7268,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.5297,
      "eval_samples_per_second": 503.416,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.5297,
      "eval_openwebtext_samples_per_second": 503.416,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0333,
      "eval_wikitext_samples_per_second": 224.261,
      "step": 337000
    },
    {
      "epoch": 19.58,
      "eval_lambada_loss": 2.44140625,
      "eval_lambada_ppl": 11.489186055851967,
      "eval_lambada_runtime": 9.6117,
      "eval_lambada_samples_per_second": 506.571,
      "step": 337000
    },
    {
      "activations/layer0_attention_weight_max": 15.206878662109375,
      "activations/layer0_attention_weight_min": -14.204893112182617,
      "activations/layer10_attention_weight_max": 36.028778076171875,
      "activations/layer10_attention_weight_min": -34.74896240234375,
      "activations/layer11_attention_weight_max": 34.46120834350586,
      "activations/layer11_attention_weight_min": -34.38981628417969,
      "activations/layer12_attention_weight_max": 17.964813232421875,
      "activations/layer12_attention_weight_min": -25.10364532470703,
      "activations/layer13_attention_weight_max": 34.89323425292969,
      "activations/layer13_attention_weight_min": -25.897289276123047,
      "activations/layer14_attention_weight_max": 36.97398376464844,
      "activations/layer14_attention_weight_min": -31.407785415649414,
      "activations/layer15_attention_weight_max": 35.562618255615234,
      "activations/layer15_attention_weight_min": -31.373849868774414,
      "activations/layer16_attention_weight_max": 37.56150817871094,
      "activations/layer16_attention_weight_min": -34.985252380371094,
      "activations/layer17_attention_weight_max": 54.62699508666992,
      "activations/layer17_attention_weight_min": -50.04296112060547,
      "activations/layer18_attention_weight_max": 52.320953369140625,
      "activations/layer18_attention_weight_min": -45.25556564331055,
      "activations/layer19_attention_weight_max": 26.43165397644043,
      "activations/layer19_attention_weight_min": -26.341276168823242,
      "activations/layer1_attention_weight_max": 15.907990455627441,
      "activations/layer1_attention_weight_min": -13.108850479125977,
      "activations/layer20_attention_weight_max": 26.53075408935547,
      "activations/layer20_attention_weight_min": -22.06964683532715,
      "activations/layer21_attention_weight_max": 42.445823669433594,
      "activations/layer21_attention_weight_min": -23.832061767578125,
      "activations/layer22_attention_weight_max": 36.43641662597656,
      "activations/layer22_attention_weight_min": -26.25434112548828,
      "activations/layer23_attention_weight_max": 41.47932434082031,
      "activations/layer23_attention_weight_min": -25.215816497802734,
      "activations/layer2_attention_weight_max": 29.564979553222656,
      "activations/layer2_attention_weight_min": -28.019102096557617,
      "activations/layer3_attention_weight_max": 80.30326080322266,
      "activations/layer3_attention_weight_min": -78.04242706298828,
      "activations/layer4_attention_weight_max": 78.91908264160156,
      "activations/layer4_attention_weight_min": -77.4830093383789,
      "activations/layer5_attention_weight_max": 58.684139251708984,
      "activations/layer5_attention_weight_min": -75.36310577392578,
      "activations/layer6_attention_weight_max": 48.14067459106445,
      "activations/layer6_attention_weight_min": -47.50049591064453,
      "activations/layer7_attention_weight_max": 65.55397033691406,
      "activations/layer7_attention_weight_min": -63.79005813598633,
      "activations/layer8_attention_weight_max": 44.195865631103516,
      "activations/layer8_attention_weight_min": -46.76234436035156,
      "activations/layer9_attention_weight_max": 46.68068313598633,
      "activations/layer9_attention_weight_min": -47.65668869018555,
      "epoch": 19.58,
      "learning_rate": 2.3947348484848484e-05,
      "loss": 2.7249,
      "step": 337050
    },
    {
      "activations/layer0_attention_weight_max": 15.043692588806152,
      "activations/layer0_attention_weight_min": -14.510825157165527,
      "activations/layer10_attention_weight_max": 38.3357048034668,
      "activations/layer10_attention_weight_min": -35.869239807128906,
      "activations/layer11_attention_weight_max": 35.13484191894531,
      "activations/layer11_attention_weight_min": -35.07904815673828,
      "activations/layer12_attention_weight_max": 18.904455184936523,
      "activations/layer12_attention_weight_min": -22.472518920898438,
      "activations/layer13_attention_weight_max": 34.194183349609375,
      "activations/layer13_attention_weight_min": -28.69986343383789,
      "activations/layer14_attention_weight_max": 38.21770095825195,
      "activations/layer14_attention_weight_min": -32.736053466796875,
      "activations/layer15_attention_weight_max": 34.46199417114258,
      "activations/layer15_attention_weight_min": -32.58283615112305,
      "activations/layer16_attention_weight_max": 37.61329650878906,
      "activations/layer16_attention_weight_min": -37.4901123046875,
      "activations/layer17_attention_weight_max": 52.550113677978516,
      "activations/layer17_attention_weight_min": -50.61715316772461,
      "activations/layer18_attention_weight_max": 48.85496520996094,
      "activations/layer18_attention_weight_min": -42.94074249267578,
      "activations/layer19_attention_weight_max": 25.659473419189453,
      "activations/layer19_attention_weight_min": -23.782400131225586,
      "activations/layer1_attention_weight_max": 15.645406723022461,
      "activations/layer1_attention_weight_min": -13.083063125610352,
      "activations/layer20_attention_weight_max": 26.367713928222656,
      "activations/layer20_attention_weight_min": -20.101444244384766,
      "activations/layer21_attention_weight_max": 39.11289978027344,
      "activations/layer21_attention_weight_min": -22.008520126342773,
      "activations/layer22_attention_weight_max": 32.48842239379883,
      "activations/layer22_attention_weight_min": -25.689680099487305,
      "activations/layer23_attention_weight_max": 44.474998474121094,
      "activations/layer23_attention_weight_min": -26.116056442260742,
      "activations/layer2_attention_weight_max": 29.03146743774414,
      "activations/layer2_attention_weight_min": -28.451622009277344,
      "activations/layer3_attention_weight_max": 82.87230682373047,
      "activations/layer3_attention_weight_min": -79.66751861572266,
      "activations/layer4_attention_weight_max": 79.87580108642578,
      "activations/layer4_attention_weight_min": -80.90168762207031,
      "activations/layer5_attention_weight_max": 64.04084777832031,
      "activations/layer5_attention_weight_min": -71.6382064819336,
      "activations/layer6_attention_weight_max": 49.89824295043945,
      "activations/layer6_attention_weight_min": -49.354286193847656,
      "activations/layer7_attention_weight_max": 64.72354888916016,
      "activations/layer7_attention_weight_min": -63.67779541015625,
      "activations/layer8_attention_weight_max": 49.308841705322266,
      "activations/layer8_attention_weight_min": -51.157657623291016,
      "activations/layer9_attention_weight_max": 45.21543502807617,
      "activations/layer9_attention_weight_min": -47.040645599365234,
      "epoch": 19.59,
      "learning_rate": 2.3928409090909086e-05,
      "loss": 2.7354,
      "step": 337100
    },
    {
      "activations/layer0_attention_weight_max": 15.173110961914062,
      "activations/layer0_attention_weight_min": -14.659592628479004,
      "activations/layer10_attention_weight_max": 34.92338943481445,
      "activations/layer10_attention_weight_min": -35.81528091430664,
      "activations/layer11_attention_weight_max": 34.06996154785156,
      "activations/layer11_attention_weight_min": -35.75388717651367,
      "activations/layer12_attention_weight_max": 20.98112678527832,
      "activations/layer12_attention_weight_min": -26.375680923461914,
      "activations/layer13_attention_weight_max": 34.574928283691406,
      "activations/layer13_attention_weight_min": -27.79149627685547,
      "activations/layer14_attention_weight_max": 37.325435638427734,
      "activations/layer14_attention_weight_min": -30.734495162963867,
      "activations/layer15_attention_weight_max": 36.057525634765625,
      "activations/layer15_attention_weight_min": -29.43670654296875,
      "activations/layer16_attention_weight_max": 35.787353515625,
      "activations/layer16_attention_weight_min": -31.84132957458496,
      "activations/layer17_attention_weight_max": 54.10483169555664,
      "activations/layer17_attention_weight_min": -46.63249969482422,
      "activations/layer18_attention_weight_max": 50.083984375,
      "activations/layer18_attention_weight_min": -40.96205520629883,
      "activations/layer19_attention_weight_max": 26.470901489257812,
      "activations/layer19_attention_weight_min": -23.199655532836914,
      "activations/layer1_attention_weight_max": 15.55008602142334,
      "activations/layer1_attention_weight_min": -14.214359283447266,
      "activations/layer20_attention_weight_max": 24.99025535583496,
      "activations/layer20_attention_weight_min": -21.50939178466797,
      "activations/layer21_attention_weight_max": 39.59931945800781,
      "activations/layer21_attention_weight_min": -20.580299377441406,
      "activations/layer22_attention_weight_max": 29.30912208557129,
      "activations/layer22_attention_weight_min": -25.97536849975586,
      "activations/layer23_attention_weight_max": 35.932884216308594,
      "activations/layer23_attention_weight_min": -23.831748962402344,
      "activations/layer2_attention_weight_max": 30.88684844970703,
      "activations/layer2_attention_weight_min": -29.228017807006836,
      "activations/layer3_attention_weight_max": 85.90309143066406,
      "activations/layer3_attention_weight_min": -83.50863647460938,
      "activations/layer4_attention_weight_max": 79.79131317138672,
      "activations/layer4_attention_weight_min": -78.21219635009766,
      "activations/layer5_attention_weight_max": 58.342037200927734,
      "activations/layer5_attention_weight_min": -67.19308471679688,
      "activations/layer6_attention_weight_max": 48.07242202758789,
      "activations/layer6_attention_weight_min": -47.21892166137695,
      "activations/layer7_attention_weight_max": 67.44117736816406,
      "activations/layer7_attention_weight_min": -61.08131408691406,
      "activations/layer8_attention_weight_max": 45.28515625,
      "activations/layer8_attention_weight_min": -47.87172317504883,
      "activations/layer9_attention_weight_max": 46.591793060302734,
      "activations/layer9_attention_weight_min": -46.73479080200195,
      "epoch": 19.59,
      "learning_rate": 2.3909469696969694e-05,
      "loss": 2.7326,
      "step": 337150
    },
    {
      "activations/layer0_attention_weight_max": 14.858824729919434,
      "activations/layer0_attention_weight_min": -14.45854663848877,
      "activations/layer10_attention_weight_max": 34.31050109863281,
      "activations/layer10_attention_weight_min": -35.582374572753906,
      "activations/layer11_attention_weight_max": 34.00740051269531,
      "activations/layer11_attention_weight_min": -33.09526062011719,
      "activations/layer12_attention_weight_max": 19.49801254272461,
      "activations/layer12_attention_weight_min": -26.36963653564453,
      "activations/layer13_attention_weight_max": 37.062923431396484,
      "activations/layer13_attention_weight_min": -29.285476684570312,
      "activations/layer14_attention_weight_max": 34.65354919433594,
      "activations/layer14_attention_weight_min": -29.50969886779785,
      "activations/layer15_attention_weight_max": 33.26652908325195,
      "activations/layer15_attention_weight_min": -29.463016510009766,
      "activations/layer16_attention_weight_max": 34.69778823852539,
      "activations/layer16_attention_weight_min": -32.700706481933594,
      "activations/layer17_attention_weight_max": 50.09956741333008,
      "activations/layer17_attention_weight_min": -43.49836730957031,
      "activations/layer18_attention_weight_max": 45.42403793334961,
      "activations/layer18_attention_weight_min": -38.54487228393555,
      "activations/layer19_attention_weight_max": 25.51417350769043,
      "activations/layer19_attention_weight_min": -21.220718383789062,
      "activations/layer1_attention_weight_max": 15.28984260559082,
      "activations/layer1_attention_weight_min": -13.782086372375488,
      "activations/layer20_attention_weight_max": 24.199674606323242,
      "activations/layer20_attention_weight_min": -20.1319580078125,
      "activations/layer21_attention_weight_max": 37.04692840576172,
      "activations/layer21_attention_weight_min": -22.938940048217773,
      "activations/layer22_attention_weight_max": 30.86994171142578,
      "activations/layer22_attention_weight_min": -25.582172393798828,
      "activations/layer23_attention_weight_max": 36.182456970214844,
      "activations/layer23_attention_weight_min": -26.33971405029297,
      "activations/layer2_attention_weight_max": 31.272863388061523,
      "activations/layer2_attention_weight_min": -29.07632827758789,
      "activations/layer3_attention_weight_max": 81.73092651367188,
      "activations/layer3_attention_weight_min": -85.96491241455078,
      "activations/layer4_attention_weight_max": 80.97620391845703,
      "activations/layer4_attention_weight_min": -80.16336822509766,
      "activations/layer5_attention_weight_max": 59.13191223144531,
      "activations/layer5_attention_weight_min": -69.6970443725586,
      "activations/layer6_attention_weight_max": 48.20808792114258,
      "activations/layer6_attention_weight_min": -49.19511795043945,
      "activations/layer7_attention_weight_max": 62.70730209350586,
      "activations/layer7_attention_weight_min": -62.65990447998047,
      "activations/layer8_attention_weight_max": 45.04086685180664,
      "activations/layer8_attention_weight_min": -48.3976936340332,
      "activations/layer9_attention_weight_max": 44.85765075683594,
      "activations/layer9_attention_weight_min": -46.06934356689453,
      "epoch": 19.59,
      "learning_rate": 2.38905303030303e-05,
      "loss": 2.7414,
      "step": 337200
    },
    {
      "activations/layer0_attention_weight_max": 15.200273513793945,
      "activations/layer0_attention_weight_min": -14.162164688110352,
      "activations/layer10_attention_weight_max": 30.150161743164062,
      "activations/layer10_attention_weight_min": -31.270915985107422,
      "activations/layer11_attention_weight_max": 30.119375228881836,
      "activations/layer11_attention_weight_min": -31.05849266052246,
      "activations/layer12_attention_weight_max": 19.154373168945312,
      "activations/layer12_attention_weight_min": -25.875797271728516,
      "activations/layer13_attention_weight_max": 32.34197998046875,
      "activations/layer13_attention_weight_min": -26.70108985900879,
      "activations/layer14_attention_weight_max": 33.503658294677734,
      "activations/layer14_attention_weight_min": -29.148786544799805,
      "activations/layer15_attention_weight_max": 30.265052795410156,
      "activations/layer15_attention_weight_min": -28.73062515258789,
      "activations/layer16_attention_weight_max": 31.44584083557129,
      "activations/layer16_attention_weight_min": -31.947954177856445,
      "activations/layer17_attention_weight_max": 48.879722595214844,
      "activations/layer17_attention_weight_min": -44.55510330200195,
      "activations/layer18_attention_weight_max": 40.9410514831543,
      "activations/layer18_attention_weight_min": -40.232086181640625,
      "activations/layer19_attention_weight_max": 25.208402633666992,
      "activations/layer19_attention_weight_min": -24.299671173095703,
      "activations/layer1_attention_weight_max": 15.76811695098877,
      "activations/layer1_attention_weight_min": -13.72990894317627,
      "activations/layer20_attention_weight_max": 20.746809005737305,
      "activations/layer20_attention_weight_min": -24.765636444091797,
      "activations/layer21_attention_weight_max": 35.77083206176758,
      "activations/layer21_attention_weight_min": -24.674362182617188,
      "activations/layer22_attention_weight_max": 30.45130157470703,
      "activations/layer22_attention_weight_min": -26.53992462158203,
      "activations/layer23_attention_weight_max": 33.79993438720703,
      "activations/layer23_attention_weight_min": -24.665321350097656,
      "activations/layer2_attention_weight_max": 32.24464797973633,
      "activations/layer2_attention_weight_min": -28.283390045166016,
      "activations/layer3_attention_weight_max": 83.7378921508789,
      "activations/layer3_attention_weight_min": -79.33444213867188,
      "activations/layer4_attention_weight_max": 75.01184844970703,
      "activations/layer4_attention_weight_min": -74.54182434082031,
      "activations/layer5_attention_weight_max": 58.62736511230469,
      "activations/layer5_attention_weight_min": -68.26953887939453,
      "activations/layer6_attention_weight_max": 45.04275131225586,
      "activations/layer6_attention_weight_min": -46.61530303955078,
      "activations/layer7_attention_weight_max": 60.62016677856445,
      "activations/layer7_attention_weight_min": -63.8907585144043,
      "activations/layer8_attention_weight_max": 40.49407958984375,
      "activations/layer8_attention_weight_min": -45.81949234008789,
      "activations/layer9_attention_weight_max": 41.49323272705078,
      "activations/layer9_attention_weight_min": -46.015594482421875,
      "epoch": 19.6,
      "learning_rate": 2.3871590909090907e-05,
      "loss": 2.7113,
      "step": 337250
    },
    {
      "activations/layer0_attention_weight_max": 16.442514419555664,
      "activations/layer0_attention_weight_min": -14.445195198059082,
      "activations/layer10_attention_weight_max": 32.47996520996094,
      "activations/layer10_attention_weight_min": -35.13380432128906,
      "activations/layer11_attention_weight_max": 31.432559967041016,
      "activations/layer11_attention_weight_min": -33.91485595703125,
      "activations/layer12_attention_weight_max": 19.390480041503906,
      "activations/layer12_attention_weight_min": -26.234167098999023,
      "activations/layer13_attention_weight_max": 30.178394317626953,
      "activations/layer13_attention_weight_min": -29.501445770263672,
      "activations/layer14_attention_weight_max": 32.31011962890625,
      "activations/layer14_attention_weight_min": -32.394287109375,
      "activations/layer15_attention_weight_max": 29.89812469482422,
      "activations/layer15_attention_weight_min": -31.133337020874023,
      "activations/layer16_attention_weight_max": 34.02614974975586,
      "activations/layer16_attention_weight_min": -33.87445831298828,
      "activations/layer17_attention_weight_max": 52.320289611816406,
      "activations/layer17_attention_weight_min": -49.1247673034668,
      "activations/layer18_attention_weight_max": 46.166900634765625,
      "activations/layer18_attention_weight_min": -42.77751922607422,
      "activations/layer19_attention_weight_max": 22.784379959106445,
      "activations/layer19_attention_weight_min": -24.058670043945312,
      "activations/layer1_attention_weight_max": 15.717933654785156,
      "activations/layer1_attention_weight_min": -13.498138427734375,
      "activations/layer20_attention_weight_max": 20.802724838256836,
      "activations/layer20_attention_weight_min": -24.69780921936035,
      "activations/layer21_attention_weight_max": 37.68796920776367,
      "activations/layer21_attention_weight_min": -24.995458602905273,
      "activations/layer22_attention_weight_max": 27.71954345703125,
      "activations/layer22_attention_weight_min": -28.706573486328125,
      "activations/layer23_attention_weight_max": 32.4666633605957,
      "activations/layer23_attention_weight_min": -25.379497528076172,
      "activations/layer2_attention_weight_max": 31.515472412109375,
      "activations/layer2_attention_weight_min": -29.643362045288086,
      "activations/layer3_attention_weight_max": 81.2883071899414,
      "activations/layer3_attention_weight_min": -82.64527893066406,
      "activations/layer4_attention_weight_max": 81.98745727539062,
      "activations/layer4_attention_weight_min": -80.63460540771484,
      "activations/layer5_attention_weight_max": 65.5318603515625,
      "activations/layer5_attention_weight_min": -73.1027603149414,
      "activations/layer6_attention_weight_max": 47.439632415771484,
      "activations/layer6_attention_weight_min": -47.25265121459961,
      "activations/layer7_attention_weight_max": 66.80430603027344,
      "activations/layer7_attention_weight_min": -63.83030700683594,
      "activations/layer8_attention_weight_max": 42.843109130859375,
      "activations/layer8_attention_weight_min": -46.91447448730469,
      "activations/layer9_attention_weight_max": 42.2589225769043,
      "activations/layer9_attention_weight_min": -43.192893981933594,
      "epoch": 19.6,
      "learning_rate": 2.3852651515151515e-05,
      "loss": 2.7412,
      "step": 337300
    },
    {
      "activations/layer0_attention_weight_max": 16.433067321777344,
      "activations/layer0_attention_weight_min": -14.041210174560547,
      "activations/layer10_attention_weight_max": 37.825767517089844,
      "activations/layer10_attention_weight_min": -37.27036666870117,
      "activations/layer11_attention_weight_max": 35.49390411376953,
      "activations/layer11_attention_weight_min": -36.52403259277344,
      "activations/layer12_attention_weight_max": 18.60760498046875,
      "activations/layer12_attention_weight_min": -25.185447692871094,
      "activations/layer13_attention_weight_max": 35.54905700683594,
      "activations/layer13_attention_weight_min": -28.47939109802246,
      "activations/layer14_attention_weight_max": 33.08187484741211,
      "activations/layer14_attention_weight_min": -29.605012893676758,
      "activations/layer15_attention_weight_max": 31.879793167114258,
      "activations/layer15_attention_weight_min": -29.92888832092285,
      "activations/layer16_attention_weight_max": 32.379520416259766,
      "activations/layer16_attention_weight_min": -31.92218780517578,
      "activations/layer17_attention_weight_max": 52.3474006652832,
      "activations/layer17_attention_weight_min": -44.959815979003906,
      "activations/layer18_attention_weight_max": 46.768611907958984,
      "activations/layer18_attention_weight_min": -40.51660919189453,
      "activations/layer19_attention_weight_max": 24.48751449584961,
      "activations/layer19_attention_weight_min": -22.618858337402344,
      "activations/layer1_attention_weight_max": 15.941178321838379,
      "activations/layer1_attention_weight_min": -14.150081634521484,
      "activations/layer20_attention_weight_max": 24.05324935913086,
      "activations/layer20_attention_weight_min": -20.15745735168457,
      "activations/layer21_attention_weight_max": 40.80125045776367,
      "activations/layer21_attention_weight_min": -21.7940731048584,
      "activations/layer22_attention_weight_max": 27.768709182739258,
      "activations/layer22_attention_weight_min": -26.848800659179688,
      "activations/layer23_attention_weight_max": 39.55893325805664,
      "activations/layer23_attention_weight_min": -23.858970642089844,
      "activations/layer2_attention_weight_max": 31.115612030029297,
      "activations/layer2_attention_weight_min": -28.972545623779297,
      "activations/layer3_attention_weight_max": 85.17203521728516,
      "activations/layer3_attention_weight_min": -86.59000396728516,
      "activations/layer4_attention_weight_max": 80.90821075439453,
      "activations/layer4_attention_weight_min": -81.08692932128906,
      "activations/layer5_attention_weight_max": 60.98565673828125,
      "activations/layer5_attention_weight_min": -70.39005279541016,
      "activations/layer6_attention_weight_max": 49.60764694213867,
      "activations/layer6_attention_weight_min": -51.171207427978516,
      "activations/layer7_attention_weight_max": 66.63196563720703,
      "activations/layer7_attention_weight_min": -64.4537582397461,
      "activations/layer8_attention_weight_max": 48.014923095703125,
      "activations/layer8_attention_weight_min": -52.97425079345703,
      "activations/layer9_attention_weight_max": 47.31513977050781,
      "activations/layer9_attention_weight_min": -48.388851165771484,
      "epoch": 19.6,
      "learning_rate": 2.383371212121212e-05,
      "loss": 2.7365,
      "step": 337350
    },
    {
      "activations/layer0_attention_weight_max": 16.263900756835938,
      "activations/layer0_attention_weight_min": -14.133188247680664,
      "activations/layer10_attention_weight_max": 33.3950080871582,
      "activations/layer10_attention_weight_min": -34.17671585083008,
      "activations/layer11_attention_weight_max": 31.750625610351562,
      "activations/layer11_attention_weight_min": -34.22595977783203,
      "activations/layer12_attention_weight_max": 19.697162628173828,
      "activations/layer12_attention_weight_min": -26.892860412597656,
      "activations/layer13_attention_weight_max": 45.81142807006836,
      "activations/layer13_attention_weight_min": -28.096529006958008,
      "activations/layer14_attention_weight_max": 37.86470413208008,
      "activations/layer14_attention_weight_min": -34.15468215942383,
      "activations/layer15_attention_weight_max": 34.0665397644043,
      "activations/layer15_attention_weight_min": -31.925188064575195,
      "activations/layer16_attention_weight_max": 36.07761001586914,
      "activations/layer16_attention_weight_min": -32.30386734008789,
      "activations/layer17_attention_weight_max": 53.5515251159668,
      "activations/layer17_attention_weight_min": -45.14067459106445,
      "activations/layer18_attention_weight_max": 51.15876388549805,
      "activations/layer18_attention_weight_min": -41.92792892456055,
      "activations/layer19_attention_weight_max": 25.41663360595703,
      "activations/layer19_attention_weight_min": -24.443927764892578,
      "activations/layer1_attention_weight_max": 15.754768371582031,
      "activations/layer1_attention_weight_min": -13.643279075622559,
      "activations/layer20_attention_weight_max": 23.61141586303711,
      "activations/layer20_attention_weight_min": -24.265146255493164,
      "activations/layer21_attention_weight_max": 37.78373718261719,
      "activations/layer21_attention_weight_min": -23.042686462402344,
      "activations/layer22_attention_weight_max": 33.46608352661133,
      "activations/layer22_attention_weight_min": -27.376638412475586,
      "activations/layer23_attention_weight_max": 37.5883903503418,
      "activations/layer23_attention_weight_min": -23.85213279724121,
      "activations/layer2_attention_weight_max": 31.664281845092773,
      "activations/layer2_attention_weight_min": -29.153602600097656,
      "activations/layer3_attention_weight_max": 88.95659637451172,
      "activations/layer3_attention_weight_min": -89.51871490478516,
      "activations/layer4_attention_weight_max": 84.62779998779297,
      "activations/layer4_attention_weight_min": -82.41924285888672,
      "activations/layer5_attention_weight_max": 61.650264739990234,
      "activations/layer5_attention_weight_min": -72.61985778808594,
      "activations/layer6_attention_weight_max": 49.86262130737305,
      "activations/layer6_attention_weight_min": -51.07279586791992,
      "activations/layer7_attention_weight_max": 66.5782699584961,
      "activations/layer7_attention_weight_min": -67.05522918701172,
      "activations/layer8_attention_weight_max": 45.21377182006836,
      "activations/layer8_attention_weight_min": -49.14008712768555,
      "activations/layer9_attention_weight_max": 43.7319450378418,
      "activations/layer9_attention_weight_min": -44.55115509033203,
      "epoch": 19.6,
      "learning_rate": 2.3814772727272725e-05,
      "loss": 2.7374,
      "step": 337400
    },
    {
      "activations/layer0_attention_weight_max": 15.574400901794434,
      "activations/layer0_attention_weight_min": -14.14547348022461,
      "activations/layer10_attention_weight_max": 36.53982162475586,
      "activations/layer10_attention_weight_min": -34.649314880371094,
      "activations/layer11_attention_weight_max": 32.524532318115234,
      "activations/layer11_attention_weight_min": -34.77729034423828,
      "activations/layer12_attention_weight_max": 20.818986892700195,
      "activations/layer12_attention_weight_min": -26.025846481323242,
      "activations/layer13_attention_weight_max": 38.43025588989258,
      "activations/layer13_attention_weight_min": -31.080190658569336,
      "activations/layer14_attention_weight_max": 39.471839904785156,
      "activations/layer14_attention_weight_min": -32.90165328979492,
      "activations/layer15_attention_weight_max": 36.97533416748047,
      "activations/layer15_attention_weight_min": -32.995208740234375,
      "activations/layer16_attention_weight_max": 39.8944091796875,
      "activations/layer16_attention_weight_min": -35.19935989379883,
      "activations/layer17_attention_weight_max": 56.35588073730469,
      "activations/layer17_attention_weight_min": -50.05043411254883,
      "activations/layer18_attention_weight_max": 56.661865234375,
      "activations/layer18_attention_weight_min": -44.5351676940918,
      "activations/layer19_attention_weight_max": 28.52902603149414,
      "activations/layer19_attention_weight_min": -26.0848331451416,
      "activations/layer1_attention_weight_max": 15.216560363769531,
      "activations/layer1_attention_weight_min": -13.597496032714844,
      "activations/layer20_attention_weight_max": 27.6887264251709,
      "activations/layer20_attention_weight_min": -22.48676109313965,
      "activations/layer21_attention_weight_max": 40.86857223510742,
      "activations/layer21_attention_weight_min": -22.805381774902344,
      "activations/layer22_attention_weight_max": 35.498130798339844,
      "activations/layer22_attention_weight_min": -27.01414680480957,
      "activations/layer23_attention_weight_max": 40.39942169189453,
      "activations/layer23_attention_weight_min": -26.665672302246094,
      "activations/layer2_attention_weight_max": 31.504619598388672,
      "activations/layer2_attention_weight_min": -29.214109420776367,
      "activations/layer3_attention_weight_max": 85.83639526367188,
      "activations/layer3_attention_weight_min": -90.27485656738281,
      "activations/layer4_attention_weight_max": 86.32707214355469,
      "activations/layer4_attention_weight_min": -82.01898956298828,
      "activations/layer5_attention_weight_max": 64.81082153320312,
      "activations/layer5_attention_weight_min": -74.37796020507812,
      "activations/layer6_attention_weight_max": 50.57963562011719,
      "activations/layer6_attention_weight_min": -49.13479995727539,
      "activations/layer7_attention_weight_max": 66.41299438476562,
      "activations/layer7_attention_weight_min": -65.80792236328125,
      "activations/layer8_attention_weight_max": 47.44034194946289,
      "activations/layer8_attention_weight_min": -51.3425407409668,
      "activations/layer9_attention_weight_max": 44.04987716674805,
      "activations/layer9_attention_weight_min": -45.700294494628906,
      "epoch": 19.61,
      "learning_rate": 2.379583333333333e-05,
      "loss": 2.7097,
      "step": 337450
    },
    {
      "activations/layer0_attention_weight_max": 15.058831214904785,
      "activations/layer0_attention_weight_min": -13.832649230957031,
      "activations/layer10_attention_weight_max": 36.148651123046875,
      "activations/layer10_attention_weight_min": -37.357391357421875,
      "activations/layer11_attention_weight_max": 35.427085876464844,
      "activations/layer11_attention_weight_min": -34.844482421875,
      "activations/layer12_attention_weight_max": 18.696630477905273,
      "activations/layer12_attention_weight_min": -28.92216682434082,
      "activations/layer13_attention_weight_max": 33.582244873046875,
      "activations/layer13_attention_weight_min": -28.444276809692383,
      "activations/layer14_attention_weight_max": 33.74888229370117,
      "activations/layer14_attention_weight_min": -29.158830642700195,
      "activations/layer15_attention_weight_max": 32.99242401123047,
      "activations/layer15_attention_weight_min": -30.299209594726562,
      "activations/layer16_attention_weight_max": 34.42397689819336,
      "activations/layer16_attention_weight_min": -33.91302490234375,
      "activations/layer17_attention_weight_max": 51.225948333740234,
      "activations/layer17_attention_weight_min": -47.918800354003906,
      "activations/layer18_attention_weight_max": 45.58323669433594,
      "activations/layer18_attention_weight_min": -41.2838020324707,
      "activations/layer19_attention_weight_max": 23.082944869995117,
      "activations/layer19_attention_weight_min": -25.061073303222656,
      "activations/layer1_attention_weight_max": 16.67635154724121,
      "activations/layer1_attention_weight_min": -13.426094055175781,
      "activations/layer20_attention_weight_max": 22.247028350830078,
      "activations/layer20_attention_weight_min": -21.515119552612305,
      "activations/layer21_attention_weight_max": 38.74433898925781,
      "activations/layer21_attention_weight_min": -24.834495544433594,
      "activations/layer22_attention_weight_max": 28.783498764038086,
      "activations/layer22_attention_weight_min": -25.968828201293945,
      "activations/layer23_attention_weight_max": 34.065330505371094,
      "activations/layer23_attention_weight_min": -25.248088836669922,
      "activations/layer2_attention_weight_max": 31.39997100830078,
      "activations/layer2_attention_weight_min": -30.51173973083496,
      "activations/layer3_attention_weight_max": 88.35718536376953,
      "activations/layer3_attention_weight_min": -89.86344146728516,
      "activations/layer4_attention_weight_max": 81.3598861694336,
      "activations/layer4_attention_weight_min": -81.23601531982422,
      "activations/layer5_attention_weight_max": 60.02738952636719,
      "activations/layer5_attention_weight_min": -72.18694305419922,
      "activations/layer6_attention_weight_max": 48.211727142333984,
      "activations/layer6_attention_weight_min": -53.318092346191406,
      "activations/layer7_attention_weight_max": 63.41753005981445,
      "activations/layer7_attention_weight_min": -66.6514892578125,
      "activations/layer8_attention_weight_max": 47.24569320678711,
      "activations/layer8_attention_weight_min": -50.06557846069336,
      "activations/layer9_attention_weight_max": 44.22208786010742,
      "activations/layer9_attention_weight_min": -49.30033493041992,
      "epoch": 19.61,
      "learning_rate": 2.377689393939394e-05,
      "loss": 2.725,
      "step": 337500
    },
    {
      "activations/layer0_attention_weight_max": 15.189546585083008,
      "activations/layer0_attention_weight_min": -14.144997596740723,
      "activations/layer10_attention_weight_max": 36.46168899536133,
      "activations/layer10_attention_weight_min": -36.85736846923828,
      "activations/layer11_attention_weight_max": 33.03821563720703,
      "activations/layer11_attention_weight_min": -35.86212158203125,
      "activations/layer12_attention_weight_max": 18.8233642578125,
      "activations/layer12_attention_weight_min": -25.66783332824707,
      "activations/layer13_attention_weight_max": 33.08530044555664,
      "activations/layer13_attention_weight_min": -28.506187438964844,
      "activations/layer14_attention_weight_max": 38.479820251464844,
      "activations/layer14_attention_weight_min": -30.56238555908203,
      "activations/layer15_attention_weight_max": 34.51161193847656,
      "activations/layer15_attention_weight_min": -31.4419002532959,
      "activations/layer16_attention_weight_max": 35.33536148071289,
      "activations/layer16_attention_weight_min": -32.30192565917969,
      "activations/layer17_attention_weight_max": 53.18024444580078,
      "activations/layer17_attention_weight_min": -47.83223342895508,
      "activations/layer18_attention_weight_max": 50.262939453125,
      "activations/layer18_attention_weight_min": -44.05758285522461,
      "activations/layer19_attention_weight_max": 26.89893913269043,
      "activations/layer19_attention_weight_min": -22.54102897644043,
      "activations/layer1_attention_weight_max": 16.887454986572266,
      "activations/layer1_attention_weight_min": -14.052184104919434,
      "activations/layer20_attention_weight_max": 24.453907012939453,
      "activations/layer20_attention_weight_min": -21.64653205871582,
      "activations/layer21_attention_weight_max": 36.76436233520508,
      "activations/layer21_attention_weight_min": -25.6953182220459,
      "activations/layer22_attention_weight_max": 27.66249656677246,
      "activations/layer22_attention_weight_min": -25.43937873840332,
      "activations/layer23_attention_weight_max": 38.99353790283203,
      "activations/layer23_attention_weight_min": -25.43659782409668,
      "activations/layer2_attention_weight_max": 33.78857421875,
      "activations/layer2_attention_weight_min": -30.66640281677246,
      "activations/layer3_attention_weight_max": 83.19898986816406,
      "activations/layer3_attention_weight_min": -82.705322265625,
      "activations/layer4_attention_weight_max": 83.4804916381836,
      "activations/layer4_attention_weight_min": -79.4602279663086,
      "activations/layer5_attention_weight_max": 61.97480392456055,
      "activations/layer5_attention_weight_min": -69.15513610839844,
      "activations/layer6_attention_weight_max": 48.8146858215332,
      "activations/layer6_attention_weight_min": -52.11601638793945,
      "activations/layer7_attention_weight_max": 64.78507995605469,
      "activations/layer7_attention_weight_min": -65.68453979492188,
      "activations/layer8_attention_weight_max": 46.500675201416016,
      "activations/layer8_attention_weight_min": -49.90237045288086,
      "activations/layer9_attention_weight_max": 49.03861999511719,
      "activations/layer9_attention_weight_min": -48.10301208496094,
      "epoch": 19.61,
      "learning_rate": 2.3757954545454543e-05,
      "loss": 2.7284,
      "step": 337550
    },
    {
      "activations/layer0_attention_weight_max": 15.444395065307617,
      "activations/layer0_attention_weight_min": -14.443392753601074,
      "activations/layer10_attention_weight_max": 40.072731018066406,
      "activations/layer10_attention_weight_min": -38.76979064941406,
      "activations/layer11_attention_weight_max": 36.28363800048828,
      "activations/layer11_attention_weight_min": -36.10185623168945,
      "activations/layer12_attention_weight_max": 25.71666145324707,
      "activations/layer12_attention_weight_min": -23.114532470703125,
      "activations/layer13_attention_weight_max": 35.7527961730957,
      "activations/layer13_attention_weight_min": -29.139719009399414,
      "activations/layer14_attention_weight_max": 38.22722244262695,
      "activations/layer14_attention_weight_min": -31.873321533203125,
      "activations/layer15_attention_weight_max": 33.17082595825195,
      "activations/layer15_attention_weight_min": -31.155582427978516,
      "activations/layer16_attention_weight_max": 34.144775390625,
      "activations/layer16_attention_weight_min": -34.50436782836914,
      "activations/layer17_attention_weight_max": 52.541099548339844,
      "activations/layer17_attention_weight_min": -46.78730392456055,
      "activations/layer18_attention_weight_max": 48.46331024169922,
      "activations/layer18_attention_weight_min": -38.58606719970703,
      "activations/layer19_attention_weight_max": 24.1165828704834,
      "activations/layer19_attention_weight_min": -24.155685424804688,
      "activations/layer1_attention_weight_max": 15.089027404785156,
      "activations/layer1_attention_weight_min": -13.36694049835205,
      "activations/layer20_attention_weight_max": 23.152589797973633,
      "activations/layer20_attention_weight_min": -22.047279357910156,
      "activations/layer21_attention_weight_max": 32.579681396484375,
      "activations/layer21_attention_weight_min": -20.570926666259766,
      "activations/layer22_attention_weight_max": 28.95969581604004,
      "activations/layer22_attention_weight_min": -25.690650939941406,
      "activations/layer23_attention_weight_max": 38.64817428588867,
      "activations/layer23_attention_weight_min": -23.831396102905273,
      "activations/layer2_attention_weight_max": 31.42140769958496,
      "activations/layer2_attention_weight_min": -28.859291076660156,
      "activations/layer3_attention_weight_max": 84.31727600097656,
      "activations/layer3_attention_weight_min": -84.3164291381836,
      "activations/layer4_attention_weight_max": 83.1041030883789,
      "activations/layer4_attention_weight_min": -84.0147705078125,
      "activations/layer5_attention_weight_max": 61.56991195678711,
      "activations/layer5_attention_weight_min": -77.3691635131836,
      "activations/layer6_attention_weight_max": 49.056095123291016,
      "activations/layer6_attention_weight_min": -52.592037200927734,
      "activations/layer7_attention_weight_max": 69.09916687011719,
      "activations/layer7_attention_weight_min": -66.3412094116211,
      "activations/layer8_attention_weight_max": 46.60969543457031,
      "activations/layer8_attention_weight_min": -51.414730072021484,
      "activations/layer9_attention_weight_max": 43.824398040771484,
      "activations/layer9_attention_weight_min": -47.73124694824219,
      "epoch": 19.62,
      "learning_rate": 2.373901515151515e-05,
      "loss": 2.7315,
      "step": 337600
    },
    {
      "activations/layer0_attention_weight_max": 15.57455062866211,
      "activations/layer0_attention_weight_min": -14.225470542907715,
      "activations/layer10_attention_weight_max": 33.48214340209961,
      "activations/layer10_attention_weight_min": -34.87394714355469,
      "activations/layer11_attention_weight_max": 32.351810455322266,
      "activations/layer11_attention_weight_min": -33.94272994995117,
      "activations/layer12_attention_weight_max": 20.620752334594727,
      "activations/layer12_attention_weight_min": -25.985132217407227,
      "activations/layer13_attention_weight_max": 36.407814025878906,
      "activations/layer13_attention_weight_min": -29.92673683166504,
      "activations/layer14_attention_weight_max": 38.91337966918945,
      "activations/layer14_attention_weight_min": -33.22811508178711,
      "activations/layer15_attention_weight_max": 36.65822219848633,
      "activations/layer15_attention_weight_min": -33.54534912109375,
      "activations/layer16_attention_weight_max": 38.22276306152344,
      "activations/layer16_attention_weight_min": -35.21976089477539,
      "activations/layer17_attention_weight_max": 56.77277755737305,
      "activations/layer17_attention_weight_min": -49.711402893066406,
      "activations/layer18_attention_weight_max": 53.12957000732422,
      "activations/layer18_attention_weight_min": -41.12498092651367,
      "activations/layer19_attention_weight_max": 26.183212280273438,
      "activations/layer19_attention_weight_min": -23.206037521362305,
      "activations/layer1_attention_weight_max": 15.4932861328125,
      "activations/layer1_attention_weight_min": -12.864469528198242,
      "activations/layer20_attention_weight_max": 23.44325828552246,
      "activations/layer20_attention_weight_min": -21.836597442626953,
      "activations/layer21_attention_weight_max": 35.53718185424805,
      "activations/layer21_attention_weight_min": -23.112154006958008,
      "activations/layer22_attention_weight_max": 31.44501495361328,
      "activations/layer22_attention_weight_min": -27.664846420288086,
      "activations/layer23_attention_weight_max": 37.97685241699219,
      "activations/layer23_attention_weight_min": -25.339794158935547,
      "activations/layer2_attention_weight_max": 29.91023826599121,
      "activations/layer2_attention_weight_min": -28.704505920410156,
      "activations/layer3_attention_weight_max": 79.42362976074219,
      "activations/layer3_attention_weight_min": -84.30241394042969,
      "activations/layer4_attention_weight_max": 78.85621643066406,
      "activations/layer4_attention_weight_min": -78.95919036865234,
      "activations/layer5_attention_weight_max": 65.84187316894531,
      "activations/layer5_attention_weight_min": -70.78467559814453,
      "activations/layer6_attention_weight_max": 48.98308181762695,
      "activations/layer6_attention_weight_min": -48.21645736694336,
      "activations/layer7_attention_weight_max": 64.36476135253906,
      "activations/layer7_attention_weight_min": -61.96969985961914,
      "activations/layer8_attention_weight_max": 44.89543533325195,
      "activations/layer8_attention_weight_min": -48.92066955566406,
      "activations/layer9_attention_weight_max": 40.0787467956543,
      "activations/layer9_attention_weight_min": -43.738792419433594,
      "epoch": 19.62,
      "learning_rate": 2.3720075757575753e-05,
      "loss": 2.7165,
      "step": 337650
    },
    {
      "activations/layer0_attention_weight_max": 15.409445762634277,
      "activations/layer0_attention_weight_min": -14.010902404785156,
      "activations/layer10_attention_weight_max": 43.66108322143555,
      "activations/layer10_attention_weight_min": -39.83575439453125,
      "activations/layer11_attention_weight_max": 41.213890075683594,
      "activations/layer11_attention_weight_min": -37.965003967285156,
      "activations/layer12_attention_weight_max": 19.388986587524414,
      "activations/layer12_attention_weight_min": -24.737680435180664,
      "activations/layer13_attention_weight_max": 38.43257141113281,
      "activations/layer13_attention_weight_min": -31.346920013427734,
      "activations/layer14_attention_weight_max": 38.986331939697266,
      "activations/layer14_attention_weight_min": -30.783809661865234,
      "activations/layer15_attention_weight_max": 37.752071380615234,
      "activations/layer15_attention_weight_min": -32.06684875488281,
      "activations/layer16_attention_weight_max": 35.003868103027344,
      "activations/layer16_attention_weight_min": -32.4215202331543,
      "activations/layer17_attention_weight_max": 51.1063117980957,
      "activations/layer17_attention_weight_min": -49.0323371887207,
      "activations/layer18_attention_weight_max": 48.37808609008789,
      "activations/layer18_attention_weight_min": -42.15757369995117,
      "activations/layer19_attention_weight_max": 24.926387786865234,
      "activations/layer19_attention_weight_min": -23.667051315307617,
      "activations/layer1_attention_weight_max": 15.864602088928223,
      "activations/layer1_attention_weight_min": -13.972858428955078,
      "activations/layer20_attention_weight_max": 25.854957580566406,
      "activations/layer20_attention_weight_min": -21.44322395324707,
      "activations/layer21_attention_weight_max": 42.675445556640625,
      "activations/layer21_attention_weight_min": -23.723613739013672,
      "activations/layer22_attention_weight_max": 29.12289047241211,
      "activations/layer22_attention_weight_min": -28.87584686279297,
      "activations/layer23_attention_weight_max": 35.63282775878906,
      "activations/layer23_attention_weight_min": -24.359838485717773,
      "activations/layer2_attention_weight_max": 32.204612731933594,
      "activations/layer2_attention_weight_min": -31.44036102294922,
      "activations/layer3_attention_weight_max": 88.58770751953125,
      "activations/layer3_attention_weight_min": -91.03864288330078,
      "activations/layer4_attention_weight_max": 89.13558959960938,
      "activations/layer4_attention_weight_min": -83.57585906982422,
      "activations/layer5_attention_weight_max": 64.48726654052734,
      "activations/layer5_attention_weight_min": -71.18695068359375,
      "activations/layer6_attention_weight_max": 52.1905403137207,
      "activations/layer6_attention_weight_min": -51.03337097167969,
      "activations/layer7_attention_weight_max": 80.60576629638672,
      "activations/layer7_attention_weight_min": -68.920654296875,
      "activations/layer8_attention_weight_max": 57.63371658325195,
      "activations/layer8_attention_weight_min": -52.75178527832031,
      "activations/layer9_attention_weight_max": 55.1930046081543,
      "activations/layer9_attention_weight_min": -49.9429931640625,
      "epoch": 19.62,
      "learning_rate": 2.370151515151515e-05,
      "loss": 2.7259,
      "step": 337700
    },
    {
      "activations/layer0_attention_weight_max": 15.335712432861328,
      "activations/layer0_attention_weight_min": -14.155143737792969,
      "activations/layer10_attention_weight_max": 35.579200744628906,
      "activations/layer10_attention_weight_min": -35.79158020019531,
      "activations/layer11_attention_weight_max": 33.468101501464844,
      "activations/layer11_attention_weight_min": -33.84986877441406,
      "activations/layer12_attention_weight_max": 19.263700485229492,
      "activations/layer12_attention_weight_min": -23.92459487915039,
      "activations/layer13_attention_weight_max": 38.3868522644043,
      "activations/layer13_attention_weight_min": -29.368934631347656,
      "activations/layer14_attention_weight_max": 37.53980255126953,
      "activations/layer14_attention_weight_min": -32.07896041870117,
      "activations/layer15_attention_weight_max": 35.40869140625,
      "activations/layer15_attention_weight_min": -30.97359848022461,
      "activations/layer16_attention_weight_max": 35.549678802490234,
      "activations/layer16_attention_weight_min": -33.095703125,
      "activations/layer17_attention_weight_max": 53.879173278808594,
      "activations/layer17_attention_weight_min": -48.556610107421875,
      "activations/layer18_attention_weight_max": 50.86268615722656,
      "activations/layer18_attention_weight_min": -46.00510787963867,
      "activations/layer19_attention_weight_max": 25.57018280029297,
      "activations/layer19_attention_weight_min": -24.265209197998047,
      "activations/layer1_attention_weight_max": 16.471765518188477,
      "activations/layer1_attention_weight_min": -14.11950397491455,
      "activations/layer20_attention_weight_max": 23.67143440246582,
      "activations/layer20_attention_weight_min": -20.677793502807617,
      "activations/layer21_attention_weight_max": 38.57304382324219,
      "activations/layer21_attention_weight_min": -22.83208656311035,
      "activations/layer22_attention_weight_max": 32.31254196166992,
      "activations/layer22_attention_weight_min": -24.949909210205078,
      "activations/layer23_attention_weight_max": 40.129356384277344,
      "activations/layer23_attention_weight_min": -25.085973739624023,
      "activations/layer2_attention_weight_max": 29.697635650634766,
      "activations/layer2_attention_weight_min": -28.68002700805664,
      "activations/layer3_attention_weight_max": 82.3597412109375,
      "activations/layer3_attention_weight_min": -81.52745056152344,
      "activations/layer4_attention_weight_max": 81.75637817382812,
      "activations/layer4_attention_weight_min": -79.63391876220703,
      "activations/layer5_attention_weight_max": 59.858421325683594,
      "activations/layer5_attention_weight_min": -69.51193237304688,
      "activations/layer6_attention_weight_max": 47.92000961303711,
      "activations/layer6_attention_weight_min": -51.20491027832031,
      "activations/layer7_attention_weight_max": 65.08409118652344,
      "activations/layer7_attention_weight_min": -63.03660583496094,
      "activations/layer8_attention_weight_max": 45.44428634643555,
      "activations/layer8_attention_weight_min": -47.89027786254883,
      "activations/layer9_attention_weight_max": 50.14469528198242,
      "activations/layer9_attention_weight_min": -46.54990005493164,
      "epoch": 19.63,
      "learning_rate": 2.3682575757575755e-05,
      "loss": 2.7296,
      "step": 337750
    },
    {
      "activations/layer0_attention_weight_max": 15.691694259643555,
      "activations/layer0_attention_weight_min": -13.64255428314209,
      "activations/layer10_attention_weight_max": 36.87652587890625,
      "activations/layer10_attention_weight_min": -35.39259338378906,
      "activations/layer11_attention_weight_max": 32.81101989746094,
      "activations/layer11_attention_weight_min": -34.290374755859375,
      "activations/layer12_attention_weight_max": 19.01066780090332,
      "activations/layer12_attention_weight_min": -24.09726905822754,
      "activations/layer13_attention_weight_max": 34.25236511230469,
      "activations/layer13_attention_weight_min": -26.196651458740234,
      "activations/layer14_attention_weight_max": 34.63393783569336,
      "activations/layer14_attention_weight_min": -30.55569076538086,
      "activations/layer15_attention_weight_max": 32.81043243408203,
      "activations/layer15_attention_weight_min": -30.95497703552246,
      "activations/layer16_attention_weight_max": 33.71095275878906,
      "activations/layer16_attention_weight_min": -32.74433135986328,
      "activations/layer17_attention_weight_max": 50.018550872802734,
      "activations/layer17_attention_weight_min": -48.314212799072266,
      "activations/layer18_attention_weight_max": 46.49663162231445,
      "activations/layer18_attention_weight_min": -42.21820068359375,
      "activations/layer19_attention_weight_max": 24.55500602722168,
      "activations/layer19_attention_weight_min": -23.006664276123047,
      "activations/layer1_attention_weight_max": 15.071447372436523,
      "activations/layer1_attention_weight_min": -13.229606628417969,
      "activations/layer20_attention_weight_max": 22.27149200439453,
      "activations/layer20_attention_weight_min": -21.039384841918945,
      "activations/layer21_attention_weight_max": 37.07305908203125,
      "activations/layer21_attention_weight_min": -23.816381454467773,
      "activations/layer22_attention_weight_max": 31.749387741088867,
      "activations/layer22_attention_weight_min": -26.725358963012695,
      "activations/layer23_attention_weight_max": 35.260765075683594,
      "activations/layer23_attention_weight_min": -24.303714752197266,
      "activations/layer2_attention_weight_max": 32.07373046875,
      "activations/layer2_attention_weight_min": -29.282161712646484,
      "activations/layer3_attention_weight_max": 87.00650024414062,
      "activations/layer3_attention_weight_min": -82.97852325439453,
      "activations/layer4_attention_weight_max": 87.74837493896484,
      "activations/layer4_attention_weight_min": -81.5064697265625,
      "activations/layer5_attention_weight_max": 62.18354797363281,
      "activations/layer5_attention_weight_min": -68.52133178710938,
      "activations/layer6_attention_weight_max": 51.54374313354492,
      "activations/layer6_attention_weight_min": -49.46413040161133,
      "activations/layer7_attention_weight_max": 63.538673400878906,
      "activations/layer7_attention_weight_min": -61.28843688964844,
      "activations/layer8_attention_weight_max": 50.72345733642578,
      "activations/layer8_attention_weight_min": -52.296592712402344,
      "activations/layer9_attention_weight_max": 47.24215316772461,
      "activations/layer9_attention_weight_min": -45.84000015258789,
      "epoch": 19.63,
      "learning_rate": 2.3663636363636364e-05,
      "loss": 2.7248,
      "step": 337800
    },
    {
      "activations/layer0_attention_weight_max": 15.850296974182129,
      "activations/layer0_attention_weight_min": -14.053919792175293,
      "activations/layer10_attention_weight_max": 33.974365234375,
      "activations/layer10_attention_weight_min": -35.65359115600586,
      "activations/layer11_attention_weight_max": 32.91860580444336,
      "activations/layer11_attention_weight_min": -34.06241226196289,
      "activations/layer12_attention_weight_max": 20.64160919189453,
      "activations/layer12_attention_weight_min": -27.130949020385742,
      "activations/layer13_attention_weight_max": 33.47773742675781,
      "activations/layer13_attention_weight_min": -29.28269386291504,
      "activations/layer14_attention_weight_max": 33.91136932373047,
      "activations/layer14_attention_weight_min": -29.148006439208984,
      "activations/layer15_attention_weight_max": 31.984718322753906,
      "activations/layer15_attention_weight_min": -29.567251205444336,
      "activations/layer16_attention_weight_max": 35.12252426147461,
      "activations/layer16_attention_weight_min": -34.50550842285156,
      "activations/layer17_attention_weight_max": 51.02309799194336,
      "activations/layer17_attention_weight_min": -44.73482131958008,
      "activations/layer18_attention_weight_max": 50.22486114501953,
      "activations/layer18_attention_weight_min": -39.86070251464844,
      "activations/layer19_attention_weight_max": 23.861536026000977,
      "activations/layer19_attention_weight_min": -22.845054626464844,
      "activations/layer1_attention_weight_max": 17.259111404418945,
      "activations/layer1_attention_weight_min": -14.056364059448242,
      "activations/layer20_attention_weight_max": 24.811954498291016,
      "activations/layer20_attention_weight_min": -22.703828811645508,
      "activations/layer21_attention_weight_max": 31.44759178161621,
      "activations/layer21_attention_weight_min": -23.595748901367188,
      "activations/layer22_attention_weight_max": 26.99100112915039,
      "activations/layer22_attention_weight_min": -25.798337936401367,
      "activations/layer23_attention_weight_max": 32.96361541748047,
      "activations/layer23_attention_weight_min": -25.769569396972656,
      "activations/layer2_attention_weight_max": 29.426862716674805,
      "activations/layer2_attention_weight_min": -28.929500579833984,
      "activations/layer3_attention_weight_max": 79.77991485595703,
      "activations/layer3_attention_weight_min": -79.22135925292969,
      "activations/layer4_attention_weight_max": 77.41494750976562,
      "activations/layer4_attention_weight_min": -78.5592041015625,
      "activations/layer5_attention_weight_max": 56.24165344238281,
      "activations/layer5_attention_weight_min": -73.02305603027344,
      "activations/layer6_attention_weight_max": 45.357120513916016,
      "activations/layer6_attention_weight_min": -47.07557678222656,
      "activations/layer7_attention_weight_max": 63.839637756347656,
      "activations/layer7_attention_weight_min": -61.845802307128906,
      "activations/layer8_attention_weight_max": 44.76140594482422,
      "activations/layer8_attention_weight_min": -47.048484802246094,
      "activations/layer9_attention_weight_max": 48.73066329956055,
      "activations/layer9_attention_weight_min": -45.55158615112305,
      "epoch": 19.63,
      "learning_rate": 2.3645075757575755e-05,
      "loss": 2.7147,
      "step": 337850
    },
    {
      "activations/layer0_attention_weight_max": 15.180069923400879,
      "activations/layer0_attention_weight_min": -13.825297355651855,
      "activations/layer10_attention_weight_max": 33.47713088989258,
      "activations/layer10_attention_weight_min": -33.94441604614258,
      "activations/layer11_attention_weight_max": 32.43865966796875,
      "activations/layer11_attention_weight_min": -33.404396057128906,
      "activations/layer12_attention_weight_max": 18.353219985961914,
      "activations/layer12_attention_weight_min": -26.356889724731445,
      "activations/layer13_attention_weight_max": 29.850574493408203,
      "activations/layer13_attention_weight_min": -25.566816329956055,
      "activations/layer14_attention_weight_max": 35.12763595581055,
      "activations/layer14_attention_weight_min": -28.934301376342773,
      "activations/layer15_attention_weight_max": 31.84259605407715,
      "activations/layer15_attention_weight_min": -29.42022132873535,
      "activations/layer16_attention_weight_max": 33.073570251464844,
      "activations/layer16_attention_weight_min": -31.472366333007812,
      "activations/layer17_attention_weight_max": 50.54767608642578,
      "activations/layer17_attention_weight_min": -43.87089920043945,
      "activations/layer18_attention_weight_max": 47.298519134521484,
      "activations/layer18_attention_weight_min": -39.77766036987305,
      "activations/layer19_attention_weight_max": 23.577930450439453,
      "activations/layer19_attention_weight_min": -23.404767990112305,
      "activations/layer1_attention_weight_max": 15.14324951171875,
      "activations/layer1_attention_weight_min": -12.564739227294922,
      "activations/layer20_attention_weight_max": 20.290021896362305,
      "activations/layer20_attention_weight_min": -21.749412536621094,
      "activations/layer21_attention_weight_max": 29.80086326599121,
      "activations/layer21_attention_weight_min": -19.73963737487793,
      "activations/layer22_attention_weight_max": 27.477109909057617,
      "activations/layer22_attention_weight_min": -26.318689346313477,
      "activations/layer23_attention_weight_max": 32.02698516845703,
      "activations/layer23_attention_weight_min": -25.289854049682617,
      "activations/layer2_attention_weight_max": 30.64098358154297,
      "activations/layer2_attention_weight_min": -28.60458755493164,
      "activations/layer3_attention_weight_max": 82.74848175048828,
      "activations/layer3_attention_weight_min": -85.76820373535156,
      "activations/layer4_attention_weight_max": 80.6137466430664,
      "activations/layer4_attention_weight_min": -77.122802734375,
      "activations/layer5_attention_weight_max": 59.53968811035156,
      "activations/layer5_attention_weight_min": -71.23759460449219,
      "activations/layer6_attention_weight_max": 45.36132049560547,
      "activations/layer6_attention_weight_min": -49.493202209472656,
      "activations/layer7_attention_weight_max": 59.867950439453125,
      "activations/layer7_attention_weight_min": -63.40096664428711,
      "activations/layer8_attention_weight_max": 43.992958068847656,
      "activations/layer8_attention_weight_min": -45.28383255004883,
      "activations/layer9_attention_weight_max": 41.24906539916992,
      "activations/layer9_attention_weight_min": -43.419677734375,
      "epoch": 19.63,
      "learning_rate": 2.3626136363636363e-05,
      "loss": 2.7349,
      "step": 337900
    },
    {
      "activations/layer0_attention_weight_max": 15.289077758789062,
      "activations/layer0_attention_weight_min": -13.888219833374023,
      "activations/layer10_attention_weight_max": 36.28009796142578,
      "activations/layer10_attention_weight_min": -36.79777908325195,
      "activations/layer11_attention_weight_max": 32.8409309387207,
      "activations/layer11_attention_weight_min": -34.77870559692383,
      "activations/layer12_attention_weight_max": 19.019180297851562,
      "activations/layer12_attention_weight_min": -24.744110107421875,
      "activations/layer13_attention_weight_max": 29.756418228149414,
      "activations/layer13_attention_weight_min": -27.071855545043945,
      "activations/layer14_attention_weight_max": 33.06621170043945,
      "activations/layer14_attention_weight_min": -29.9227294921875,
      "activations/layer15_attention_weight_max": 30.901500701904297,
      "activations/layer15_attention_weight_min": -30.38588523864746,
      "activations/layer16_attention_weight_max": 33.46077346801758,
      "activations/layer16_attention_weight_min": -32.77363586425781,
      "activations/layer17_attention_weight_max": 48.15898513793945,
      "activations/layer17_attention_weight_min": -45.486019134521484,
      "activations/layer18_attention_weight_max": 44.25863265991211,
      "activations/layer18_attention_weight_min": -38.60287094116211,
      "activations/layer19_attention_weight_max": 23.563261032104492,
      "activations/layer19_attention_weight_min": -24.013029098510742,
      "activations/layer1_attention_weight_max": 16.110321044921875,
      "activations/layer1_attention_weight_min": -14.151541709899902,
      "activations/layer20_attention_weight_max": 21.491897583007812,
      "activations/layer20_attention_weight_min": -25.115903854370117,
      "activations/layer21_attention_weight_max": 33.15629196166992,
      "activations/layer21_attention_weight_min": -27.74312400817871,
      "activations/layer22_attention_weight_max": 28.41689682006836,
      "activations/layer22_attention_weight_min": -27.5257511138916,
      "activations/layer23_attention_weight_max": 36.8372917175293,
      "activations/layer23_attention_weight_min": -25.435855865478516,
      "activations/layer2_attention_weight_max": 29.713987350463867,
      "activations/layer2_attention_weight_min": -28.89289665222168,
      "activations/layer3_attention_weight_max": 80.93785095214844,
      "activations/layer3_attention_weight_min": -80.90840148925781,
      "activations/layer4_attention_weight_max": 80.32842254638672,
      "activations/layer4_attention_weight_min": -79.60662078857422,
      "activations/layer5_attention_weight_max": 60.417388916015625,
      "activations/layer5_attention_weight_min": -72.56425476074219,
      "activations/layer6_attention_weight_max": 47.647987365722656,
      "activations/layer6_attention_weight_min": -48.75891876220703,
      "activations/layer7_attention_weight_max": 62.87541961669922,
      "activations/layer7_attention_weight_min": -64.83049774169922,
      "activations/layer8_attention_weight_max": 47.9731559753418,
      "activations/layer8_attention_weight_min": -50.03032302856445,
      "activations/layer9_attention_weight_max": 42.74388885498047,
      "activations/layer9_attention_weight_min": -44.96582794189453,
      "epoch": 19.64,
      "learning_rate": 2.3607196969696968e-05,
      "loss": 2.7235,
      "step": 337950
    },
    {
      "activations/layer0_attention_weight_max": 15.209874153137207,
      "activations/layer0_attention_weight_min": -13.835916519165039,
      "activations/layer10_attention_weight_max": 37.16279220581055,
      "activations/layer10_attention_weight_min": -35.61286926269531,
      "activations/layer11_attention_weight_max": 32.61811828613281,
      "activations/layer11_attention_weight_min": -36.1148681640625,
      "activations/layer12_attention_weight_max": 18.828210830688477,
      "activations/layer12_attention_weight_min": -22.934253692626953,
      "activations/layer13_attention_weight_max": 37.48158264160156,
      "activations/layer13_attention_weight_min": -29.42483901977539,
      "activations/layer14_attention_weight_max": 34.84769821166992,
      "activations/layer14_attention_weight_min": -32.364742279052734,
      "activations/layer15_attention_weight_max": 36.99637985229492,
      "activations/layer15_attention_weight_min": -32.798580169677734,
      "activations/layer16_attention_weight_max": 33.71811294555664,
      "activations/layer16_attention_weight_min": -31.95103645324707,
      "activations/layer17_attention_weight_max": 47.2887077331543,
      "activations/layer17_attention_weight_min": -43.83028030395508,
      "activations/layer18_attention_weight_max": 46.858985900878906,
      "activations/layer18_attention_weight_min": -37.77611541748047,
      "activations/layer19_attention_weight_max": 26.619205474853516,
      "activations/layer19_attention_weight_min": -22.37493896484375,
      "activations/layer1_attention_weight_max": 15.469110488891602,
      "activations/layer1_attention_weight_min": -15.435467720031738,
      "activations/layer20_attention_weight_max": 25.83203125,
      "activations/layer20_attention_weight_min": -21.28845977783203,
      "activations/layer21_attention_weight_max": 35.43680953979492,
      "activations/layer21_attention_weight_min": -21.473527908325195,
      "activations/layer22_attention_weight_max": 28.997501373291016,
      "activations/layer22_attention_weight_min": -25.025279998779297,
      "activations/layer23_attention_weight_max": 35.45234680175781,
      "activations/layer23_attention_weight_min": -24.324203491210938,
      "activations/layer2_attention_weight_max": 30.505117416381836,
      "activations/layer2_attention_weight_min": -28.32625961303711,
      "activations/layer3_attention_weight_max": 85.85400390625,
      "activations/layer3_attention_weight_min": -81.81884765625,
      "activations/layer4_attention_weight_max": 82.28670501708984,
      "activations/layer4_attention_weight_min": -80.6963119506836,
      "activations/layer5_attention_weight_max": 59.890560150146484,
      "activations/layer5_attention_weight_min": -73.75115966796875,
      "activations/layer6_attention_weight_max": 49.22496032714844,
      "activations/layer6_attention_weight_min": -52.772274017333984,
      "activations/layer7_attention_weight_max": 64.7452392578125,
      "activations/layer7_attention_weight_min": -64.10493469238281,
      "activations/layer8_attention_weight_max": 46.20905303955078,
      "activations/layer8_attention_weight_min": -50.25629425048828,
      "activations/layer9_attention_weight_max": 50.582557678222656,
      "activations/layer9_attention_weight_min": -47.56821823120117,
      "epoch": 19.64,
      "learning_rate": 2.3588257575757573e-05,
      "loss": 2.7085,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_loss": 2.68359375,
      "eval_runtime": 8.5168,
      "eval_samples_per_second": 504.177,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_openwebtext_loss": 2.68359375,
      "eval_openwebtext_ppl": 14.637602771363136,
      "eval_openwebtext_runtime": 8.5168,
      "eval_openwebtext_samples_per_second": 504.177,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 2.0089,
      "eval_wikitext_samples_per_second": 226.987,
      "step": 338000
    },
    {
      "epoch": 19.64,
      "eval_lambada_loss": 2.45703125,
      "eval_lambada_ppl": 11.670114409757737,
      "eval_lambada_runtime": 9.6125,
      "eval_lambada_samples_per_second": 506.528,
      "step": 338000
    },
    {
      "activations/layer0_attention_weight_max": 15.345271110534668,
      "activations/layer0_attention_weight_min": -13.6277437210083,
      "activations/layer10_attention_weight_max": 40.46922302246094,
      "activations/layer10_attention_weight_min": -37.23051834106445,
      "activations/layer11_attention_weight_max": 35.420257568359375,
      "activations/layer11_attention_weight_min": -36.043819427490234,
      "activations/layer12_attention_weight_max": 20.663042068481445,
      "activations/layer12_attention_weight_min": -26.00371551513672,
      "activations/layer13_attention_weight_max": 44.00088882446289,
      "activations/layer13_attention_weight_min": -39.47633361816406,
      "activations/layer14_attention_weight_max": 49.209678649902344,
      "activations/layer14_attention_weight_min": -36.93856430053711,
      "activations/layer15_attention_weight_max": 48.85862731933594,
      "activations/layer15_attention_weight_min": -37.44424057006836,
      "activations/layer16_attention_weight_max": 39.57550048828125,
      "activations/layer16_attention_weight_min": -36.07231521606445,
      "activations/layer17_attention_weight_max": 58.19567108154297,
      "activations/layer17_attention_weight_min": -49.43999099731445,
      "activations/layer18_attention_weight_max": 53.46717071533203,
      "activations/layer18_attention_weight_min": -44.33344268798828,
      "activations/layer19_attention_weight_max": 30.17376708984375,
      "activations/layer19_attention_weight_min": -22.770614624023438,
      "activations/layer1_attention_weight_max": 16.30622673034668,
      "activations/layer1_attention_weight_min": -13.652186393737793,
      "activations/layer20_attention_weight_max": 27.435272216796875,
      "activations/layer20_attention_weight_min": -23.87569236755371,
      "activations/layer21_attention_weight_max": 42.93319320678711,
      "activations/layer21_attention_weight_min": -25.6995792388916,
      "activations/layer22_attention_weight_max": 36.18527603149414,
      "activations/layer22_attention_weight_min": -25.606719970703125,
      "activations/layer23_attention_weight_max": 47.02656555175781,
      "activations/layer23_attention_weight_min": -24.702077865600586,
      "activations/layer2_attention_weight_max": 31.30415153503418,
      "activations/layer2_attention_weight_min": -30.658414840698242,
      "activations/layer3_attention_weight_max": 79.63618469238281,
      "activations/layer3_attention_weight_min": -79.71907043457031,
      "activations/layer4_attention_weight_max": 78.80068969726562,
      "activations/layer4_attention_weight_min": -78.61640167236328,
      "activations/layer5_attention_weight_max": 61.156272888183594,
      "activations/layer5_attention_weight_min": -69.18921661376953,
      "activations/layer6_attention_weight_max": 47.51358413696289,
      "activations/layer6_attention_weight_min": -50.31914138793945,
      "activations/layer7_attention_weight_max": 66.59210968017578,
      "activations/layer7_attention_weight_min": -65.59977722167969,
      "activations/layer8_attention_weight_max": 48.110816955566406,
      "activations/layer8_attention_weight_min": -51.12947082519531,
      "activations/layer9_attention_weight_max": 50.655330657958984,
      "activations/layer9_attention_weight_min": -45.57254409790039,
      "epoch": 19.64,
      "learning_rate": 2.3569318181818178e-05,
      "loss": 2.7027,
      "step": 338050
    },
    {
      "activations/layer0_attention_weight_max": 15.776830673217773,
      "activations/layer0_attention_weight_min": -13.799650192260742,
      "activations/layer10_attention_weight_max": 37.36650466918945,
      "activations/layer10_attention_weight_min": -36.822113037109375,
      "activations/layer11_attention_weight_max": 33.202274322509766,
      "activations/layer11_attention_weight_min": -35.23329162597656,
      "activations/layer12_attention_weight_max": 19.9537296295166,
      "activations/layer12_attention_weight_min": -26.830490112304688,
      "activations/layer13_attention_weight_max": 39.896236419677734,
      "activations/layer13_attention_weight_min": -26.627201080322266,
      "activations/layer14_attention_weight_max": 38.214717864990234,
      "activations/layer14_attention_weight_min": -29.91428565979004,
      "activations/layer15_attention_weight_max": 35.57228469848633,
      "activations/layer15_attention_weight_min": -30.109813690185547,
      "activations/layer16_attention_weight_max": 39.63815689086914,
      "activations/layer16_attention_weight_min": -33.718875885009766,
      "activations/layer17_attention_weight_max": 58.68886947631836,
      "activations/layer17_attention_weight_min": -50.239036560058594,
      "activations/layer18_attention_weight_max": 55.858978271484375,
      "activations/layer18_attention_weight_min": -44.47975540161133,
      "activations/layer19_attention_weight_max": 27.52170753479004,
      "activations/layer19_attention_weight_min": -24.689733505249023,
      "activations/layer1_attention_weight_max": 16.953184127807617,
      "activations/layer1_attention_weight_min": -13.276737213134766,
      "activations/layer20_attention_weight_max": 26.391948699951172,
      "activations/layer20_attention_weight_min": -21.014324188232422,
      "activations/layer21_attention_weight_max": 42.986541748046875,
      "activations/layer21_attention_weight_min": -24.371124267578125,
      "activations/layer22_attention_weight_max": 33.65720748901367,
      "activations/layer22_attention_weight_min": -24.966718673706055,
      "activations/layer23_attention_weight_max": 47.04087448120117,
      "activations/layer23_attention_weight_min": -21.412763595581055,
      "activations/layer2_attention_weight_max": 30.38222885131836,
      "activations/layer2_attention_weight_min": -29.792129516601562,
      "activations/layer3_attention_weight_max": 85.5846939086914,
      "activations/layer3_attention_weight_min": -79.46063232421875,
      "activations/layer4_attention_weight_max": 80.6460189819336,
      "activations/layer4_attention_weight_min": -79.84735870361328,
      "activations/layer5_attention_weight_max": 59.914939880371094,
      "activations/layer5_attention_weight_min": -73.27166748046875,
      "activations/layer6_attention_weight_max": 50.75760269165039,
      "activations/layer6_attention_weight_min": -49.34187698364258,
      "activations/layer7_attention_weight_max": 66.62449645996094,
      "activations/layer7_attention_weight_min": -64.7769775390625,
      "activations/layer8_attention_weight_max": 50.73166275024414,
      "activations/layer8_attention_weight_min": -48.86831283569336,
      "activations/layer9_attention_weight_max": 45.288631439208984,
      "activations/layer9_attention_weight_min": -47.36267852783203,
      "epoch": 19.65,
      "learning_rate": 2.3550378787878786e-05,
      "loss": 2.74,
      "step": 338100
    },
    {
      "activations/layer0_attention_weight_max": 14.639313697814941,
      "activations/layer0_attention_weight_min": -13.764554023742676,
      "activations/layer10_attention_weight_max": 32.2181510925293,
      "activations/layer10_attention_weight_min": -33.995304107666016,
      "activations/layer11_attention_weight_max": 30.908369064331055,
      "activations/layer11_attention_weight_min": -34.71604537963867,
      "activations/layer12_attention_weight_max": 18.89167022705078,
      "activations/layer12_attention_weight_min": -27.89653205871582,
      "activations/layer13_attention_weight_max": 34.91099166870117,
      "activations/layer13_attention_weight_min": -29.423372268676758,
      "activations/layer14_attention_weight_max": 41.6950569152832,
      "activations/layer14_attention_weight_min": -30.980628967285156,
      "activations/layer15_attention_weight_max": 37.10468292236328,
      "activations/layer15_attention_weight_min": -32.139404296875,
      "activations/layer16_attention_weight_max": 35.00054931640625,
      "activations/layer16_attention_weight_min": -33.631805419921875,
      "activations/layer17_attention_weight_max": 53.81517028808594,
      "activations/layer17_attention_weight_min": -52.23427963256836,
      "activations/layer18_attention_weight_max": 50.517887115478516,
      "activations/layer18_attention_weight_min": -44.47886657714844,
      "activations/layer19_attention_weight_max": 25.601730346679688,
      "activations/layer19_attention_weight_min": -25.5665283203125,
      "activations/layer1_attention_weight_max": 15.966007232666016,
      "activations/layer1_attention_weight_min": -14.159552574157715,
      "activations/layer20_attention_weight_max": 26.374332427978516,
      "activations/layer20_attention_weight_min": -24.098464965820312,
      "activations/layer21_attention_weight_max": 39.912723541259766,
      "activations/layer21_attention_weight_min": -24.071849822998047,
      "activations/layer22_attention_weight_max": 30.81554412841797,
      "activations/layer22_attention_weight_min": -27.0924015045166,
      "activations/layer23_attention_weight_max": 37.92205810546875,
      "activations/layer23_attention_weight_min": -24.069992065429688,
      "activations/layer2_attention_weight_max": 29.144351959228516,
      "activations/layer2_attention_weight_min": -28.16332244873047,
      "activations/layer3_attention_weight_max": 78.59461975097656,
      "activations/layer3_attention_weight_min": -77.94245910644531,
      "activations/layer4_attention_weight_max": 77.20394134521484,
      "activations/layer4_attention_weight_min": -77.80172729492188,
      "activations/layer5_attention_weight_max": 59.89424133300781,
      "activations/layer5_attention_weight_min": -73.82432556152344,
      "activations/layer6_attention_weight_max": 45.784568786621094,
      "activations/layer6_attention_weight_min": -46.886287689208984,
      "activations/layer7_attention_weight_max": 62.50230407714844,
      "activations/layer7_attention_weight_min": -62.32761001586914,
      "activations/layer8_attention_weight_max": 44.5571174621582,
      "activations/layer8_attention_weight_min": -47.272518157958984,
      "activations/layer9_attention_weight_max": 39.208065032958984,
      "activations/layer9_attention_weight_min": -44.665035247802734,
      "epoch": 19.65,
      "learning_rate": 2.353143939393939e-05,
      "loss": 2.724,
      "step": 338150
    },
    {
      "activations/layer0_attention_weight_max": 14.800459861755371,
      "activations/layer0_attention_weight_min": -13.66963005065918,
      "activations/layer10_attention_weight_max": 41.84083938598633,
      "activations/layer10_attention_weight_min": -40.418487548828125,
      "activations/layer11_attention_weight_max": 39.90213394165039,
      "activations/layer11_attention_weight_min": -38.70475769042969,
      "activations/layer12_attention_weight_max": 19.606916427612305,
      "activations/layer12_attention_weight_min": -27.757158279418945,
      "activations/layer13_attention_weight_max": 39.28820037841797,
      "activations/layer13_attention_weight_min": -27.20585823059082,
      "activations/layer14_attention_weight_max": 33.35890579223633,
      "activations/layer14_attention_weight_min": -29.9104061126709,
      "activations/layer15_attention_weight_max": 37.73394012451172,
      "activations/layer15_attention_weight_min": -30.159828186035156,
      "activations/layer16_attention_weight_max": 32.763545989990234,
      "activations/layer16_attention_weight_min": -29.93709373474121,
      "activations/layer17_attention_weight_max": 49.444671630859375,
      "activations/layer17_attention_weight_min": -44.01933670043945,
      "activations/layer18_attention_weight_max": 43.866851806640625,
      "activations/layer18_attention_weight_min": -39.348541259765625,
      "activations/layer19_attention_weight_max": 25.948205947875977,
      "activations/layer19_attention_weight_min": -24.509538650512695,
      "activations/layer1_attention_weight_max": 15.869400024414062,
      "activations/layer1_attention_weight_min": -13.508700370788574,
      "activations/layer20_attention_weight_max": 24.832557678222656,
      "activations/layer20_attention_weight_min": -22.315473556518555,
      "activations/layer21_attention_weight_max": 36.53009796142578,
      "activations/layer21_attention_weight_min": -27.74307632446289,
      "activations/layer22_attention_weight_max": 28.47934913635254,
      "activations/layer22_attention_weight_min": -24.823768615722656,
      "activations/layer23_attention_weight_max": 35.636131286621094,
      "activations/layer23_attention_weight_min": -24.87075424194336,
      "activations/layer2_attention_weight_max": 29.657506942749023,
      "activations/layer2_attention_weight_min": -28.618419647216797,
      "activations/layer3_attention_weight_max": 77.92649841308594,
      "activations/layer3_attention_weight_min": -78.42095947265625,
      "activations/layer4_attention_weight_max": 76.55535888671875,
      "activations/layer4_attention_weight_min": -76.01053619384766,
      "activations/layer5_attention_weight_max": 56.90436935424805,
      "activations/layer5_attention_weight_min": -66.44851684570312,
      "activations/layer6_attention_weight_max": 44.38764572143555,
      "activations/layer6_attention_weight_min": -46.68330764770508,
      "activations/layer7_attention_weight_max": 73.25071716308594,
      "activations/layer7_attention_weight_min": -64.80252838134766,
      "activations/layer8_attention_weight_max": 53.155052185058594,
      "activations/layer8_attention_weight_min": -52.47023391723633,
      "activations/layer9_attention_weight_max": 56.518104553222656,
      "activations/layer9_attention_weight_min": -49.314945220947266,
      "epoch": 19.65,
      "learning_rate": 2.35125e-05,
      "loss": 2.7276,
      "step": 338200
    },
    {
      "activations/layer0_attention_weight_max": 15.73531723022461,
      "activations/layer0_attention_weight_min": -13.706655502319336,
      "activations/layer10_attention_weight_max": 36.552040100097656,
      "activations/layer10_attention_weight_min": -36.460845947265625,
      "activations/layer11_attention_weight_max": 34.894649505615234,
      "activations/layer11_attention_weight_min": -36.20771026611328,
      "activations/layer12_attention_weight_max": 19.676172256469727,
      "activations/layer12_attention_weight_min": -23.81242561340332,
      "activations/layer13_attention_weight_max": 39.65446090698242,
      "activations/layer13_attention_weight_min": -31.248971939086914,
      "activations/layer14_attention_weight_max": 39.07487106323242,
      "activations/layer14_attention_weight_min": -30.437637329101562,
      "activations/layer15_attention_weight_max": 35.27882766723633,
      "activations/layer15_attention_weight_min": -31.73948097229004,
      "activations/layer16_attention_weight_max": 34.95234680175781,
      "activations/layer16_attention_weight_min": -32.64424514770508,
      "activations/layer17_attention_weight_max": 53.76411819458008,
      "activations/layer17_attention_weight_min": -46.35553741455078,
      "activations/layer18_attention_weight_max": 47.43360137939453,
      "activations/layer18_attention_weight_min": -44.22683334350586,
      "activations/layer19_attention_weight_max": 26.505008697509766,
      "activations/layer19_attention_weight_min": -26.097278594970703,
      "activations/layer1_attention_weight_max": 15.409088134765625,
      "activations/layer1_attention_weight_min": -13.999174118041992,
      "activations/layer20_attention_weight_max": 22.074480056762695,
      "activations/layer20_attention_weight_min": -25.597999572753906,
      "activations/layer21_attention_weight_max": 37.01932144165039,
      "activations/layer21_attention_weight_min": -28.81085205078125,
      "activations/layer22_attention_weight_max": 31.072628021240234,
      "activations/layer22_attention_weight_min": -27.370075225830078,
      "activations/layer23_attention_weight_max": 41.53825759887695,
      "activations/layer23_attention_weight_min": -28.297317504882812,
      "activations/layer2_attention_weight_max": 32.02157211303711,
      "activations/layer2_attention_weight_min": -30.113439559936523,
      "activations/layer3_attention_weight_max": 87.82398223876953,
      "activations/layer3_attention_weight_min": -86.93061828613281,
      "activations/layer4_attention_weight_max": 83.89928436279297,
      "activations/layer4_attention_weight_min": -84.94515228271484,
      "activations/layer5_attention_weight_max": 62.72694396972656,
      "activations/layer5_attention_weight_min": -77.12142944335938,
      "activations/layer6_attention_weight_max": 49.91542434692383,
      "activations/layer6_attention_weight_min": -53.23466873168945,
      "activations/layer7_attention_weight_max": 65.43621826171875,
      "activations/layer7_attention_weight_min": -63.79665756225586,
      "activations/layer8_attention_weight_max": 48.72637176513672,
      "activations/layer8_attention_weight_min": -53.0102424621582,
      "activations/layer9_attention_weight_max": 46.37824249267578,
      "activations/layer9_attention_weight_min": -46.88664245605469,
      "epoch": 19.65,
      "learning_rate": 2.3493560606060607e-05,
      "loss": 2.73,
      "step": 338250
    },
    {
      "activations/layer0_attention_weight_max": 15.212804794311523,
      "activations/layer0_attention_weight_min": -13.975201606750488,
      "activations/layer10_attention_weight_max": 35.836708068847656,
      "activations/layer10_attention_weight_min": -36.75837326049805,
      "activations/layer11_attention_weight_max": 32.79890060424805,
      "activations/layer11_attention_weight_min": -33.80908966064453,
      "activations/layer12_attention_weight_max": 19.13151741027832,
      "activations/layer12_attention_weight_min": -25.220029830932617,
      "activations/layer13_attention_weight_max": 31.046730041503906,
      "activations/layer13_attention_weight_min": -27.232423782348633,
      "activations/layer14_attention_weight_max": 35.024505615234375,
      "activations/layer14_attention_weight_min": -31.436107635498047,
      "activations/layer15_attention_weight_max": 33.08695602416992,
      "activations/layer15_attention_weight_min": -31.86897850036621,
      "activations/layer16_attention_weight_max": 34.4622802734375,
      "activations/layer16_attention_weight_min": -33.23512268066406,
      "activations/layer17_attention_weight_max": 53.20014953613281,
      "activations/layer17_attention_weight_min": -50.33873748779297,
      "activations/layer18_attention_weight_max": 47.464866638183594,
      "activations/layer18_attention_weight_min": -44.62823486328125,
      "activations/layer19_attention_weight_max": 23.678720474243164,
      "activations/layer19_attention_weight_min": -23.270212173461914,
      "activations/layer1_attention_weight_max": 15.172816276550293,
      "activations/layer1_attention_weight_min": -14.13367748260498,
      "activations/layer20_attention_weight_max": 23.016202926635742,
      "activations/layer20_attention_weight_min": -21.989641189575195,
      "activations/layer21_attention_weight_max": 34.28086853027344,
      "activations/layer21_attention_weight_min": -23.08574104309082,
      "activations/layer22_attention_weight_max": 29.515304565429688,
      "activations/layer22_attention_weight_min": -26.544452667236328,
      "activations/layer23_attention_weight_max": 35.54972839355469,
      "activations/layer23_attention_weight_min": -26.521604537963867,
      "activations/layer2_attention_weight_max": 30.802730560302734,
      "activations/layer2_attention_weight_min": -28.249313354492188,
      "activations/layer3_attention_weight_max": 86.3023681640625,
      "activations/layer3_attention_weight_min": -82.24855041503906,
      "activations/layer4_attention_weight_max": 80.57292938232422,
      "activations/layer4_attention_weight_min": -80.81304931640625,
      "activations/layer5_attention_weight_max": 62.31734848022461,
      "activations/layer5_attention_weight_min": -71.18077850341797,
      "activations/layer6_attention_weight_max": 50.706214904785156,
      "activations/layer6_attention_weight_min": -48.70497512817383,
      "activations/layer7_attention_weight_max": 66.30038452148438,
      "activations/layer7_attention_weight_min": -63.22584915161133,
      "activations/layer8_attention_weight_max": 47.4434814453125,
      "activations/layer8_attention_weight_min": -48.741268157958984,
      "activations/layer9_attention_weight_max": 43.30661392211914,
      "activations/layer9_attention_weight_min": -46.95389938354492,
      "epoch": 19.66,
      "learning_rate": 2.347462121212121e-05,
      "loss": 2.741,
      "step": 338300
    },
    {
      "activations/layer0_attention_weight_max": 15.414325714111328,
      "activations/layer0_attention_weight_min": -13.903365135192871,
      "activations/layer10_attention_weight_max": 36.39579391479492,
      "activations/layer10_attention_weight_min": -36.73222732543945,
      "activations/layer11_attention_weight_max": 34.082000732421875,
      "activations/layer11_attention_weight_min": -34.89300537109375,
      "activations/layer12_attention_weight_max": 18.704130172729492,
      "activations/layer12_attention_weight_min": -26.932527542114258,
      "activations/layer13_attention_weight_max": 44.76006317138672,
      "activations/layer13_attention_weight_min": -31.700828552246094,
      "activations/layer14_attention_weight_max": 36.39095687866211,
      "activations/layer14_attention_weight_min": -34.92884063720703,
      "activations/layer15_attention_weight_max": 34.236412048339844,
      "activations/layer15_attention_weight_min": -34.301971435546875,
      "activations/layer16_attention_weight_max": 36.86317443847656,
      "activations/layer16_attention_weight_min": -35.35323715209961,
      "activations/layer17_attention_weight_max": 57.31804656982422,
      "activations/layer17_attention_weight_min": -52.53531265258789,
      "activations/layer18_attention_weight_max": 52.141700744628906,
      "activations/layer18_attention_weight_min": -44.29294967651367,
      "activations/layer19_attention_weight_max": 24.854557037353516,
      "activations/layer19_attention_weight_min": -22.840892791748047,
      "activations/layer1_attention_weight_max": 14.887609481811523,
      "activations/layer1_attention_weight_min": -13.896307945251465,
      "activations/layer20_attention_weight_max": 24.80367660522461,
      "activations/layer20_attention_weight_min": -23.422447204589844,
      "activations/layer21_attention_weight_max": 38.845211029052734,
      "activations/layer21_attention_weight_min": -24.146329879760742,
      "activations/layer22_attention_weight_max": 30.60678482055664,
      "activations/layer22_attention_weight_min": -27.337846755981445,
      "activations/layer23_attention_weight_max": 36.44621276855469,
      "activations/layer23_attention_weight_min": -26.937149047851562,
      "activations/layer2_attention_weight_max": 30.836475372314453,
      "activations/layer2_attention_weight_min": -29.49591064453125,
      "activations/layer3_attention_weight_max": 84.92371368408203,
      "activations/layer3_attention_weight_min": -85.46273040771484,
      "activations/layer4_attention_weight_max": 81.9959945678711,
      "activations/layer4_attention_weight_min": -81.08966064453125,
      "activations/layer5_attention_weight_max": 61.204803466796875,
      "activations/layer5_attention_weight_min": -69.37776947021484,
      "activations/layer6_attention_weight_max": 48.95551300048828,
      "activations/layer6_attention_weight_min": -49.09354019165039,
      "activations/layer7_attention_weight_max": 65.30571746826172,
      "activations/layer7_attention_weight_min": -65.49881744384766,
      "activations/layer8_attention_weight_max": 50.310028076171875,
      "activations/layer8_attention_weight_min": -52.96382522583008,
      "activations/layer9_attention_weight_max": 49.66968536376953,
      "activations/layer9_attention_weight_min": -48.82670593261719,
      "epoch": 19.66,
      "learning_rate": 2.3455681818181817e-05,
      "loss": 2.71,
      "step": 338350
    },
    {
      "activations/layer0_attention_weight_max": 15.982614517211914,
      "activations/layer0_attention_weight_min": -14.253666877746582,
      "activations/layer10_attention_weight_max": 37.4307861328125,
      "activations/layer10_attention_weight_min": -37.01863098144531,
      "activations/layer11_attention_weight_max": 35.19607925415039,
      "activations/layer11_attention_weight_min": -36.73310089111328,
      "activations/layer12_attention_weight_max": 19.19447898864746,
      "activations/layer12_attention_weight_min": -22.710853576660156,
      "activations/layer13_attention_weight_max": 34.976219177246094,
      "activations/layer13_attention_weight_min": -29.26320457458496,
      "activations/layer14_attention_weight_max": 38.5919303894043,
      "activations/layer14_attention_weight_min": -32.57501983642578,
      "activations/layer15_attention_weight_max": 36.61735153198242,
      "activations/layer15_attention_weight_min": -31.672544479370117,
      "activations/layer16_attention_weight_max": 37.06424331665039,
      "activations/layer16_attention_weight_min": -34.22218322753906,
      "activations/layer17_attention_weight_max": 54.297420501708984,
      "activations/layer17_attention_weight_min": -45.68007278442383,
      "activations/layer18_attention_weight_max": 48.098079681396484,
      "activations/layer18_attention_weight_min": -40.59170150756836,
      "activations/layer19_attention_weight_max": 24.98607063293457,
      "activations/layer19_attention_weight_min": -23.354604721069336,
      "activations/layer1_attention_weight_max": 16.05547332763672,
      "activations/layer1_attention_weight_min": -13.380212783813477,
      "activations/layer20_attention_weight_max": 24.98784065246582,
      "activations/layer20_attention_weight_min": -22.815685272216797,
      "activations/layer21_attention_weight_max": 36.40764236450195,
      "activations/layer21_attention_weight_min": -23.09670066833496,
      "activations/layer22_attention_weight_max": 29.60088348388672,
      "activations/layer22_attention_weight_min": -26.0501651763916,
      "activations/layer23_attention_weight_max": 39.240570068359375,
      "activations/layer23_attention_weight_min": -23.969106674194336,
      "activations/layer2_attention_weight_max": 30.070478439331055,
      "activations/layer2_attention_weight_min": -29.160568237304688,
      "activations/layer3_attention_weight_max": 80.03863525390625,
      "activations/layer3_attention_weight_min": -80.97264099121094,
      "activations/layer4_attention_weight_max": 77.90691375732422,
      "activations/layer4_attention_weight_min": -78.81917572021484,
      "activations/layer5_attention_weight_max": 60.34243392944336,
      "activations/layer5_attention_weight_min": -67.15099334716797,
      "activations/layer6_attention_weight_max": 47.23561096191406,
      "activations/layer6_attention_weight_min": -48.077552795410156,
      "activations/layer7_attention_weight_max": 62.85055160522461,
      "activations/layer7_attention_weight_min": -66.79615783691406,
      "activations/layer8_attention_weight_max": 46.54523468017578,
      "activations/layer8_attention_weight_min": -52.394195556640625,
      "activations/layer9_attention_weight_max": 44.05209732055664,
      "activations/layer9_attention_weight_min": -48.52486801147461,
      "epoch": 19.66,
      "learning_rate": 2.3436742424242422e-05,
      "loss": 2.7177,
      "step": 338400
    },
    {
      "activations/layer0_attention_weight_max": 15.188713073730469,
      "activations/layer0_attention_weight_min": -13.988927841186523,
      "activations/layer10_attention_weight_max": 35.325931549072266,
      "activations/layer10_attention_weight_min": -35.86496353149414,
      "activations/layer11_attention_weight_max": 33.28373336791992,
      "activations/layer11_attention_weight_min": -35.634037017822266,
      "activations/layer12_attention_weight_max": 18.60665512084961,
      "activations/layer12_attention_weight_min": -22.32438850402832,
      "activations/layer13_attention_weight_max": 31.922807693481445,
      "activations/layer13_attention_weight_min": -27.08083724975586,
      "activations/layer14_attention_weight_max": 32.869720458984375,
      "activations/layer14_attention_weight_min": -29.18512725830078,
      "activations/layer15_attention_weight_max": 31.83432388305664,
      "activations/layer15_attention_weight_min": -29.737258911132812,
      "activations/layer16_attention_weight_max": 33.657920837402344,
      "activations/layer16_attention_weight_min": -30.445466995239258,
      "activations/layer17_attention_weight_max": 50.51853942871094,
      "activations/layer17_attention_weight_min": -47.34506607055664,
      "activations/layer18_attention_weight_max": 50.7028694152832,
      "activations/layer18_attention_weight_min": -39.795875549316406,
      "activations/layer19_attention_weight_max": 23.034273147583008,
      "activations/layer19_attention_weight_min": -22.652145385742188,
      "activations/layer1_attention_weight_max": 15.185234069824219,
      "activations/layer1_attention_weight_min": -13.201743125915527,
      "activations/layer20_attention_weight_max": 22.547801971435547,
      "activations/layer20_attention_weight_min": -22.888755798339844,
      "activations/layer21_attention_weight_max": 33.160587310791016,
      "activations/layer21_attention_weight_min": -21.271242141723633,
      "activations/layer22_attention_weight_max": 29.84435272216797,
      "activations/layer22_attention_weight_min": -24.33444595336914,
      "activations/layer23_attention_weight_max": 35.60972595214844,
      "activations/layer23_attention_weight_min": -24.277074813842773,
      "activations/layer2_attention_weight_max": 29.225500106811523,
      "activations/layer2_attention_weight_min": -30.279930114746094,
      "activations/layer3_attention_weight_max": 82.702392578125,
      "activations/layer3_attention_weight_min": -86.34223937988281,
      "activations/layer4_attention_weight_max": 78.76395416259766,
      "activations/layer4_attention_weight_min": -82.41732025146484,
      "activations/layer5_attention_weight_max": 57.07262420654297,
      "activations/layer5_attention_weight_min": -71.12796020507812,
      "activations/layer6_attention_weight_max": 46.046871185302734,
      "activations/layer6_attention_weight_min": -48.92875289916992,
      "activations/layer7_attention_weight_max": 63.96316146850586,
      "activations/layer7_attention_weight_min": -70.85375213623047,
      "activations/layer8_attention_weight_max": 43.616477966308594,
      "activations/layer8_attention_weight_min": -49.09994888305664,
      "activations/layer9_attention_weight_max": 40.82527542114258,
      "activations/layer9_attention_weight_min": -46.300479888916016,
      "epoch": 19.67,
      "learning_rate": 2.341780303030303e-05,
      "loss": 2.7173,
      "step": 338450
    },
    {
      "activations/layer0_attention_weight_max": 17.081689834594727,
      "activations/layer0_attention_weight_min": -14.079902648925781,
      "activations/layer10_attention_weight_max": 35.47081756591797,
      "activations/layer10_attention_weight_min": -36.63737869262695,
      "activations/layer11_attention_weight_max": 33.77657699584961,
      "activations/layer11_attention_weight_min": -36.58851623535156,
      "activations/layer12_attention_weight_max": 19.227527618408203,
      "activations/layer12_attention_weight_min": -25.804807662963867,
      "activations/layer13_attention_weight_max": 36.2000732421875,
      "activations/layer13_attention_weight_min": -28.144235610961914,
      "activations/layer14_attention_weight_max": 35.257381439208984,
      "activations/layer14_attention_weight_min": -29.731334686279297,
      "activations/layer15_attention_weight_max": 35.09821319580078,
      "activations/layer15_attention_weight_min": -32.43437194824219,
      "activations/layer16_attention_weight_max": 37.42752456665039,
      "activations/layer16_attention_weight_min": -36.25816345214844,
      "activations/layer17_attention_weight_max": 53.5798225402832,
      "activations/layer17_attention_weight_min": -48.71107482910156,
      "activations/layer18_attention_weight_max": 48.896697998046875,
      "activations/layer18_attention_weight_min": -40.92228698730469,
      "activations/layer19_attention_weight_max": 26.612533569335938,
      "activations/layer19_attention_weight_min": -25.619247436523438,
      "activations/layer1_attention_weight_max": 14.856234550476074,
      "activations/layer1_attention_weight_min": -13.871867179870605,
      "activations/layer20_attention_weight_max": 24.69903564453125,
      "activations/layer20_attention_weight_min": -22.477317810058594,
      "activations/layer21_attention_weight_max": 39.187713623046875,
      "activations/layer21_attention_weight_min": -24.54393768310547,
      "activations/layer22_attention_weight_max": 29.70884132385254,
      "activations/layer22_attention_weight_min": -26.16834831237793,
      "activations/layer23_attention_weight_max": 38.992164611816406,
      "activations/layer23_attention_weight_min": -25.133079528808594,
      "activations/layer2_attention_weight_max": 31.85863494873047,
      "activations/layer2_attention_weight_min": -29.344139099121094,
      "activations/layer3_attention_weight_max": 82.38461303710938,
      "activations/layer3_attention_weight_min": -81.69910430908203,
      "activations/layer4_attention_weight_max": 81.49739837646484,
      "activations/layer4_attention_weight_min": -82.28899383544922,
      "activations/layer5_attention_weight_max": 60.174617767333984,
      "activations/layer5_attention_weight_min": -72.41735076904297,
      "activations/layer6_attention_weight_max": 48.09787368774414,
      "activations/layer6_attention_weight_min": -48.405418395996094,
      "activations/layer7_attention_weight_max": 60.17402648925781,
      "activations/layer7_attention_weight_min": -67.32959747314453,
      "activations/layer8_attention_weight_max": 49.41488265991211,
      "activations/layer8_attention_weight_min": -52.91373825073242,
      "activations/layer9_attention_weight_max": 45.22578430175781,
      "activations/layer9_attention_weight_min": -46.99806213378906,
      "epoch": 19.67,
      "learning_rate": 2.3398863636363635e-05,
      "loss": 2.7184,
      "step": 338500
    },
    {
      "activations/layer0_attention_weight_max": 14.917548179626465,
      "activations/layer0_attention_weight_min": -14.17082405090332,
      "activations/layer10_attention_weight_max": 44.21953201293945,
      "activations/layer10_attention_weight_min": -42.19204330444336,
      "activations/layer11_attention_weight_max": 39.025142669677734,
      "activations/layer11_attention_weight_min": -39.791446685791016,
      "activations/layer12_attention_weight_max": 21.23431396484375,
      "activations/layer12_attention_weight_min": -25.50459098815918,
      "activations/layer13_attention_weight_max": 37.56288146972656,
      "activations/layer13_attention_weight_min": -31.729175567626953,
      "activations/layer14_attention_weight_max": 41.45452880859375,
      "activations/layer14_attention_weight_min": -31.721033096313477,
      "activations/layer15_attention_weight_max": 38.78620910644531,
      "activations/layer15_attention_weight_min": -31.97303009033203,
      "activations/layer16_attention_weight_max": 39.93681716918945,
      "activations/layer16_attention_weight_min": -36.17842102050781,
      "activations/layer17_attention_weight_max": 57.87358474731445,
      "activations/layer17_attention_weight_min": -48.15180587768555,
      "activations/layer18_attention_weight_max": 53.12346267700195,
      "activations/layer18_attention_weight_min": -44.08662414550781,
      "activations/layer19_attention_weight_max": 26.337966918945312,
      "activations/layer19_attention_weight_min": -25.417043685913086,
      "activations/layer1_attention_weight_max": 15.617660522460938,
      "activations/layer1_attention_weight_min": -13.740275382995605,
      "activations/layer20_attention_weight_max": 26.17559814453125,
      "activations/layer20_attention_weight_min": -22.879173278808594,
      "activations/layer21_attention_weight_max": 41.73372268676758,
      "activations/layer21_attention_weight_min": -26.968822479248047,
      "activations/layer22_attention_weight_max": 31.701459884643555,
      "activations/layer22_attention_weight_min": -25.16846466064453,
      "activations/layer23_attention_weight_max": 42.28376388549805,
      "activations/layer23_attention_weight_min": -24.861286163330078,
      "activations/layer2_attention_weight_max": 31.032695770263672,
      "activations/layer2_attention_weight_min": -30.359832763671875,
      "activations/layer3_attention_weight_max": 85.00103759765625,
      "activations/layer3_attention_weight_min": -81.72235107421875,
      "activations/layer4_attention_weight_max": 83.2965316772461,
      "activations/layer4_attention_weight_min": -79.28690338134766,
      "activations/layer5_attention_weight_max": 62.15742111206055,
      "activations/layer5_attention_weight_min": -72.35586547851562,
      "activations/layer6_attention_weight_max": 50.85197830200195,
      "activations/layer6_attention_weight_min": -54.660377502441406,
      "activations/layer7_attention_weight_max": 69.363525390625,
      "activations/layer7_attention_weight_min": -66.4676284790039,
      "activations/layer8_attention_weight_max": 54.21904373168945,
      "activations/layer8_attention_weight_min": -55.7164421081543,
      "activations/layer9_attention_weight_max": 53.168373107910156,
      "activations/layer9_attention_weight_min": -51.39594268798828,
      "epoch": 19.67,
      "learning_rate": 2.3379924242424243e-05,
      "loss": 2.7268,
      "step": 338550
    },
    {
      "activations/layer0_attention_weight_max": 16.552278518676758,
      "activations/layer0_attention_weight_min": -14.27192497253418,
      "activations/layer10_attention_weight_max": 36.695213317871094,
      "activations/layer10_attention_weight_min": -36.37902069091797,
      "activations/layer11_attention_weight_max": 35.17555236816406,
      "activations/layer11_attention_weight_min": -34.71382141113281,
      "activations/layer12_attention_weight_max": 18.868595123291016,
      "activations/layer12_attention_weight_min": -24.277122497558594,
      "activations/layer13_attention_weight_max": 32.60335922241211,
      "activations/layer13_attention_weight_min": -26.54177474975586,
      "activations/layer14_attention_weight_max": 35.98994445800781,
      "activations/layer14_attention_weight_min": -31.498645782470703,
      "activations/layer15_attention_weight_max": 32.321651458740234,
      "activations/layer15_attention_weight_min": -32.20235824584961,
      "activations/layer16_attention_weight_max": 35.10654830932617,
      "activations/layer16_attention_weight_min": -34.29373550415039,
      "activations/layer17_attention_weight_max": 53.17549514770508,
      "activations/layer17_attention_weight_min": -49.241729736328125,
      "activations/layer18_attention_weight_max": 47.578861236572266,
      "activations/layer18_attention_weight_min": -45.20635223388672,
      "activations/layer19_attention_weight_max": 25.046079635620117,
      "activations/layer19_attention_weight_min": -27.4461727142334,
      "activations/layer1_attention_weight_max": 15.248695373535156,
      "activations/layer1_attention_weight_min": -13.658031463623047,
      "activations/layer20_attention_weight_max": 21.442319869995117,
      "activations/layer20_attention_weight_min": -23.35403060913086,
      "activations/layer21_attention_weight_max": 33.17166519165039,
      "activations/layer21_attention_weight_min": -23.691736221313477,
      "activations/layer22_attention_weight_max": 27.9217472076416,
      "activations/layer22_attention_weight_min": -26.742578506469727,
      "activations/layer23_attention_weight_max": 35.26167297363281,
      "activations/layer23_attention_weight_min": -25.546428680419922,
      "activations/layer2_attention_weight_max": 32.5866584777832,
      "activations/layer2_attention_weight_min": -29.47623062133789,
      "activations/layer3_attention_weight_max": 86.6228256225586,
      "activations/layer3_attention_weight_min": -86.37931823730469,
      "activations/layer4_attention_weight_max": 84.02864837646484,
      "activations/layer4_attention_weight_min": -82.8185043334961,
      "activations/layer5_attention_weight_max": 63.50212097167969,
      "activations/layer5_attention_weight_min": -69.35537719726562,
      "activations/layer6_attention_weight_max": 49.64259719848633,
      "activations/layer6_attention_weight_min": -51.677852630615234,
      "activations/layer7_attention_weight_max": 65.13114929199219,
      "activations/layer7_attention_weight_min": -66.19134521484375,
      "activations/layer8_attention_weight_max": 50.772857666015625,
      "activations/layer8_attention_weight_min": -52.567970275878906,
      "activations/layer9_attention_weight_max": 45.22414016723633,
      "activations/layer9_attention_weight_min": -48.57563781738281,
      "epoch": 19.67,
      "learning_rate": 2.3360984848484845e-05,
      "loss": 2.7337,
      "step": 338600
    },
    {
      "activations/layer0_attention_weight_max": 15.067817687988281,
      "activations/layer0_attention_weight_min": -14.148205757141113,
      "activations/layer10_attention_weight_max": 37.53120422363281,
      "activations/layer10_attention_weight_min": -37.31019973754883,
      "activations/layer11_attention_weight_max": 33.05082321166992,
      "activations/layer11_attention_weight_min": -35.4683837890625,
      "activations/layer12_attention_weight_max": 21.393827438354492,
      "activations/layer12_attention_weight_min": -24.0291805267334,
      "activations/layer13_attention_weight_max": 33.865875244140625,
      "activations/layer13_attention_weight_min": -30.971803665161133,
      "activations/layer14_attention_weight_max": 35.06489562988281,
      "activations/layer14_attention_weight_min": -33.219356536865234,
      "activations/layer15_attention_weight_max": 33.74028015136719,
      "activations/layer15_attention_weight_min": -33.012386322021484,
      "activations/layer16_attention_weight_max": 35.73553466796875,
      "activations/layer16_attention_weight_min": -34.61408996582031,
      "activations/layer17_attention_weight_max": 53.73109817504883,
      "activations/layer17_attention_weight_min": -49.16010665893555,
      "activations/layer18_attention_weight_max": 48.57511520385742,
      "activations/layer18_attention_weight_min": -42.03020477294922,
      "activations/layer19_attention_weight_max": 25.84512710571289,
      "activations/layer19_attention_weight_min": -25.464139938354492,
      "activations/layer1_attention_weight_max": 17.00455665588379,
      "activations/layer1_attention_weight_min": -13.89479923248291,
      "activations/layer20_attention_weight_max": 23.91425132751465,
      "activations/layer20_attention_weight_min": -23.322362899780273,
      "activations/layer21_attention_weight_max": 35.52320098876953,
      "activations/layer21_attention_weight_min": -19.946163177490234,
      "activations/layer22_attention_weight_max": 28.319414138793945,
      "activations/layer22_attention_weight_min": -29.065916061401367,
      "activations/layer23_attention_weight_max": 39.13043975830078,
      "activations/layer23_attention_weight_min": -24.152645111083984,
      "activations/layer2_attention_weight_max": 30.24734115600586,
      "activations/layer2_attention_weight_min": -28.56856346130371,
      "activations/layer3_attention_weight_max": 82.18033599853516,
      "activations/layer3_attention_weight_min": -85.85423278808594,
      "activations/layer4_attention_weight_max": 80.74044799804688,
      "activations/layer4_attention_weight_min": -79.0854721069336,
      "activations/layer5_attention_weight_max": 63.23753356933594,
      "activations/layer5_attention_weight_min": -70.19322204589844,
      "activations/layer6_attention_weight_max": 48.77986145019531,
      "activations/layer6_attention_weight_min": -50.96189880371094,
      "activations/layer7_attention_weight_max": 63.68190002441406,
      "activations/layer7_attention_weight_min": -62.179054260253906,
      "activations/layer8_attention_weight_max": 46.68366622924805,
      "activations/layer8_attention_weight_min": -49.309906005859375,
      "activations/layer9_attention_weight_max": 43.2456169128418,
      "activations/layer9_attention_weight_min": -45.254486083984375,
      "epoch": 19.68,
      "learning_rate": 2.3342045454545453e-05,
      "loss": 2.7063,
      "step": 338650
    },
    {
      "activations/layer0_attention_weight_max": 15.540874481201172,
      "activations/layer0_attention_weight_min": -14.038512229919434,
      "activations/layer10_attention_weight_max": 36.515655517578125,
      "activations/layer10_attention_weight_min": -37.67430877685547,
      "activations/layer11_attention_weight_max": 34.811737060546875,
      "activations/layer11_attention_weight_min": -35.84605407714844,
      "activations/layer12_attention_weight_max": 19.157432556152344,
      "activations/layer12_attention_weight_min": -22.33587646484375,
      "activations/layer13_attention_weight_max": 40.42229080200195,
      "activations/layer13_attention_weight_min": -29.468374252319336,
      "activations/layer14_attention_weight_max": 33.92379379272461,
      "activations/layer14_attention_weight_min": -29.316965103149414,
      "activations/layer15_attention_weight_max": 31.45404624938965,
      "activations/layer15_attention_weight_min": -31.517826080322266,
      "activations/layer16_attention_weight_max": 33.50535202026367,
      "activations/layer16_attention_weight_min": -32.56830596923828,
      "activations/layer17_attention_weight_max": 50.09540557861328,
      "activations/layer17_attention_weight_min": -43.83269500732422,
      "activations/layer18_attention_weight_max": 45.023414611816406,
      "activations/layer18_attention_weight_min": -38.746402740478516,
      "activations/layer19_attention_weight_max": 25.406713485717773,
      "activations/layer19_attention_weight_min": -21.8138484954834,
      "activations/layer1_attention_weight_max": 15.115995407104492,
      "activations/layer1_attention_weight_min": -14.21422004699707,
      "activations/layer20_attention_weight_max": 23.399517059326172,
      "activations/layer20_attention_weight_min": -20.831707000732422,
      "activations/layer21_attention_weight_max": 32.78389358520508,
      "activations/layer21_attention_weight_min": -22.28649139404297,
      "activations/layer22_attention_weight_max": 31.49989128112793,
      "activations/layer22_attention_weight_min": -25.7698974609375,
      "activations/layer23_attention_weight_max": 33.64134216308594,
      "activations/layer23_attention_weight_min": -23.701820373535156,
      "activations/layer2_attention_weight_max": 30.39667510986328,
      "activations/layer2_attention_weight_min": -29.683902740478516,
      "activations/layer3_attention_weight_max": 87.9326171875,
      "activations/layer3_attention_weight_min": -90.70833587646484,
      "activations/layer4_attention_weight_max": 88.31982421875,
      "activations/layer4_attention_weight_min": -86.66699981689453,
      "activations/layer5_attention_weight_max": 68.67121887207031,
      "activations/layer5_attention_weight_min": -73.54716491699219,
      "activations/layer6_attention_weight_max": 53.728275299072266,
      "activations/layer6_attention_weight_min": -53.2545051574707,
      "activations/layer7_attention_weight_max": 71.59319305419922,
      "activations/layer7_attention_weight_min": -70.7197265625,
      "activations/layer8_attention_weight_max": 48.88581466674805,
      "activations/layer8_attention_weight_min": -51.81443405151367,
      "activations/layer9_attention_weight_max": 44.4182014465332,
      "activations/layer9_attention_weight_min": -48.438472747802734,
      "epoch": 19.68,
      "learning_rate": 2.3323106060606058e-05,
      "loss": 2.7263,
      "step": 338700
    },
    {
      "activations/layer0_attention_weight_max": 15.002163887023926,
      "activations/layer0_attention_weight_min": -14.19870376586914,
      "activations/layer10_attention_weight_max": 40.20608139038086,
      "activations/layer10_attention_weight_min": -38.56658935546875,
      "activations/layer11_attention_weight_max": 39.135009765625,
      "activations/layer11_attention_weight_min": -37.4493293762207,
      "activations/layer12_attention_weight_max": 17.415964126586914,
      "activations/layer12_attention_weight_min": -24.921688079833984,
      "activations/layer13_attention_weight_max": 34.66506576538086,
      "activations/layer13_attention_weight_min": -27.076173782348633,
      "activations/layer14_attention_weight_max": 31.373472213745117,
      "activations/layer14_attention_weight_min": -30.71708869934082,
      "activations/layer15_attention_weight_max": 32.24214172363281,
      "activations/layer15_attention_weight_min": -28.013917922973633,
      "activations/layer16_attention_weight_max": 30.96513557434082,
      "activations/layer16_attention_weight_min": -29.31293296813965,
      "activations/layer17_attention_weight_max": 47.28842544555664,
      "activations/layer17_attention_weight_min": -42.527374267578125,
      "activations/layer18_attention_weight_max": 42.60953140258789,
      "activations/layer18_attention_weight_min": -36.880043029785156,
      "activations/layer19_attention_weight_max": 21.095138549804688,
      "activations/layer19_attention_weight_min": -20.059703826904297,
      "activations/layer1_attention_weight_max": 15.742029190063477,
      "activations/layer1_attention_weight_min": -14.088495254516602,
      "activations/layer20_attention_weight_max": 22.246408462524414,
      "activations/layer20_attention_weight_min": -21.841466903686523,
      "activations/layer21_attention_weight_max": 33.01826858520508,
      "activations/layer21_attention_weight_min": -21.466148376464844,
      "activations/layer22_attention_weight_max": 26.183435440063477,
      "activations/layer22_attention_weight_min": -23.709810256958008,
      "activations/layer23_attention_weight_max": 32.58731460571289,
      "activations/layer23_attention_weight_min": -24.065860748291016,
      "activations/layer2_attention_weight_max": 32.036865234375,
      "activations/layer2_attention_weight_min": -30.99014663696289,
      "activations/layer3_attention_weight_max": 83.36371612548828,
      "activations/layer3_attention_weight_min": -92.86203002929688,
      "activations/layer4_attention_weight_max": 83.84516906738281,
      "activations/layer4_attention_weight_min": -81.55199432373047,
      "activations/layer5_attention_weight_max": 63.31748580932617,
      "activations/layer5_attention_weight_min": -74.69168090820312,
      "activations/layer6_attention_weight_max": 49.23407745361328,
      "activations/layer6_attention_weight_min": -50.81849670410156,
      "activations/layer7_attention_weight_max": 68.43563079833984,
      "activations/layer7_attention_weight_min": -66.2352066040039,
      "activations/layer8_attention_weight_max": 48.38307571411133,
      "activations/layer8_attention_weight_min": -51.854244232177734,
      "activations/layer9_attention_weight_max": 47.88749313354492,
      "activations/layer9_attention_weight_min": -50.18690490722656,
      "epoch": 19.68,
      "learning_rate": 2.3304166666666666e-05,
      "loss": 2.7278,
      "step": 338750
    },
    {
      "activations/layer0_attention_weight_max": 15.407571792602539,
      "activations/layer0_attention_weight_min": -13.884156227111816,
      "activations/layer10_attention_weight_max": 37.206485748291016,
      "activations/layer10_attention_weight_min": -37.02549743652344,
      "activations/layer11_attention_weight_max": 33.306846618652344,
      "activations/layer11_attention_weight_min": -33.795387268066406,
      "activations/layer12_attention_weight_max": 19.865758895874023,
      "activations/layer12_attention_weight_min": -25.101158142089844,
      "activations/layer13_attention_weight_max": 31.840614318847656,
      "activations/layer13_attention_weight_min": -29.099218368530273,
      "activations/layer14_attention_weight_max": 33.205078125,
      "activations/layer14_attention_weight_min": -30.24620246887207,
      "activations/layer15_attention_weight_max": 31.993404388427734,
      "activations/layer15_attention_weight_min": -30.825693130493164,
      "activations/layer16_attention_weight_max": 33.29973602294922,
      "activations/layer16_attention_weight_min": -33.26923370361328,
      "activations/layer17_attention_weight_max": 50.13252639770508,
      "activations/layer17_attention_weight_min": -47.921634674072266,
      "activations/layer18_attention_weight_max": 47.56397247314453,
      "activations/layer18_attention_weight_min": -41.53083419799805,
      "activations/layer19_attention_weight_max": 23.717823028564453,
      "activations/layer19_attention_weight_min": -23.3106689453125,
      "activations/layer1_attention_weight_max": 16.676965713500977,
      "activations/layer1_attention_weight_min": -16.11540412902832,
      "activations/layer20_attention_weight_max": 23.271574020385742,
      "activations/layer20_attention_weight_min": -23.420127868652344,
      "activations/layer21_attention_weight_max": 35.27605056762695,
      "activations/layer21_attention_weight_min": -23.788747787475586,
      "activations/layer22_attention_weight_max": 31.72772216796875,
      "activations/layer22_attention_weight_min": -27.07688331604004,
      "activations/layer23_attention_weight_max": 37.819358825683594,
      "activations/layer23_attention_weight_min": -23.75239372253418,
      "activations/layer2_attention_weight_max": 32.10142517089844,
      "activations/layer2_attention_weight_min": -30.813873291015625,
      "activations/layer3_attention_weight_max": 86.78661346435547,
      "activations/layer3_attention_weight_min": -88.1713638305664,
      "activations/layer4_attention_weight_max": 83.99501037597656,
      "activations/layer4_attention_weight_min": -85.01692199707031,
      "activations/layer5_attention_weight_max": 65.4744873046875,
      "activations/layer5_attention_weight_min": -70.74440002441406,
      "activations/layer6_attention_weight_max": 49.922664642333984,
      "activations/layer6_attention_weight_min": -51.06289291381836,
      "activations/layer7_attention_weight_max": 65.3102798461914,
      "activations/layer7_attention_weight_min": -67.47743225097656,
      "activations/layer8_attention_weight_max": 47.99589920043945,
      "activations/layer8_attention_weight_min": -52.30012893676758,
      "activations/layer9_attention_weight_max": 44.87824249267578,
      "activations/layer9_attention_weight_min": -50.40066146850586,
      "epoch": 19.69,
      "learning_rate": 2.3285227272727268e-05,
      "loss": 2.719,
      "step": 338800
    },
    {
      "activations/layer0_attention_weight_max": 15.47022533416748,
      "activations/layer0_attention_weight_min": -13.931352615356445,
      "activations/layer10_attention_weight_max": 35.45232391357422,
      "activations/layer10_attention_weight_min": -34.78667449951172,
      "activations/layer11_attention_weight_max": 32.86294937133789,
      "activations/layer11_attention_weight_min": -33.76112747192383,
      "activations/layer12_attention_weight_max": 17.927213668823242,
      "activations/layer12_attention_weight_min": -24.690467834472656,
      "activations/layer13_attention_weight_max": 35.51357650756836,
      "activations/layer13_attention_weight_min": -28.921945571899414,
      "activations/layer14_attention_weight_max": 36.60603713989258,
      "activations/layer14_attention_weight_min": -33.5400505065918,
      "activations/layer15_attention_weight_max": 33.38416290283203,
      "activations/layer15_attention_weight_min": -32.49349594116211,
      "activations/layer16_attention_weight_max": 36.096771240234375,
      "activations/layer16_attention_weight_min": -35.51784133911133,
      "activations/layer17_attention_weight_max": 51.921913146972656,
      "activations/layer17_attention_weight_min": -48.80015182495117,
      "activations/layer18_attention_weight_max": 47.24080276489258,
      "activations/layer18_attention_weight_min": -43.7182502746582,
      "activations/layer19_attention_weight_max": 24.006290435791016,
      "activations/layer19_attention_weight_min": -25.672306060791016,
      "activations/layer1_attention_weight_max": 15.640076637268066,
      "activations/layer1_attention_weight_min": -14.962197303771973,
      "activations/layer20_attention_weight_max": 22.096189498901367,
      "activations/layer20_attention_weight_min": -22.647672653198242,
      "activations/layer21_attention_weight_max": 36.33181381225586,
      "activations/layer21_attention_weight_min": -24.629528045654297,
      "activations/layer22_attention_weight_max": 29.34918975830078,
      "activations/layer22_attention_weight_min": -29.22658920288086,
      "activations/layer23_attention_weight_max": 34.5384521484375,
      "activations/layer23_attention_weight_min": -26.902099609375,
      "activations/layer2_attention_weight_max": 32.53746032714844,
      "activations/layer2_attention_weight_min": -32.303504943847656,
      "activations/layer3_attention_weight_max": 84.5441665649414,
      "activations/layer3_attention_weight_min": -81.86421966552734,
      "activations/layer4_attention_weight_max": 84.49736022949219,
      "activations/layer4_attention_weight_min": -82.6408920288086,
      "activations/layer5_attention_weight_max": 63.51984786987305,
      "activations/layer5_attention_weight_min": -70.98432159423828,
      "activations/layer6_attention_weight_max": 49.38850784301758,
      "activations/layer6_attention_weight_min": -50.04933166503906,
      "activations/layer7_attention_weight_max": 62.357093811035156,
      "activations/layer7_attention_weight_min": -64.87511444091797,
      "activations/layer8_attention_weight_max": 44.9463996887207,
      "activations/layer8_attention_weight_min": -47.68632125854492,
      "activations/layer9_attention_weight_max": 40.48167037963867,
      "activations/layer9_attention_weight_min": -44.888671875,
      "epoch": 19.69,
      "learning_rate": 2.3266287878787876e-05,
      "loss": 2.7223,
      "step": 338850
    },
    {
      "activations/layer0_attention_weight_max": 16.61817741394043,
      "activations/layer0_attention_weight_min": -13.74439811706543,
      "activations/layer10_attention_weight_max": 33.29693603515625,
      "activations/layer10_attention_weight_min": -35.194190979003906,
      "activations/layer11_attention_weight_max": 32.60960388183594,
      "activations/layer11_attention_weight_min": -35.41571044921875,
      "activations/layer12_attention_weight_max": 19.087535858154297,
      "activations/layer12_attention_weight_min": -25.299640655517578,
      "activations/layer13_attention_weight_max": 31.936594009399414,
      "activations/layer13_attention_weight_min": -26.394224166870117,
      "activations/layer14_attention_weight_max": 33.76354217529297,
      "activations/layer14_attention_weight_min": -29.77887725830078,
      "activations/layer15_attention_weight_max": 31.188602447509766,
      "activations/layer15_attention_weight_min": -30.840744018554688,
      "activations/layer16_attention_weight_max": 33.713279724121094,
      "activations/layer16_attention_weight_min": -31.601591110229492,
      "activations/layer17_attention_weight_max": 49.241573333740234,
      "activations/layer17_attention_weight_min": -43.74875259399414,
      "activations/layer18_attention_weight_max": 46.90179443359375,
      "activations/layer18_attention_weight_min": -41.10507583618164,
      "activations/layer19_attention_weight_max": 24.6544246673584,
      "activations/layer19_attention_weight_min": -25.425352096557617,
      "activations/layer1_attention_weight_max": 15.951332092285156,
      "activations/layer1_attention_weight_min": -14.5875883102417,
      "activations/layer20_attention_weight_max": 24.24790382385254,
      "activations/layer20_attention_weight_min": -21.431482315063477,
      "activations/layer21_attention_weight_max": 36.28550720214844,
      "activations/layer21_attention_weight_min": -20.919509887695312,
      "activations/layer22_attention_weight_max": 28.779369354248047,
      "activations/layer22_attention_weight_min": -26.405794143676758,
      "activations/layer23_attention_weight_max": 35.578857421875,
      "activations/layer23_attention_weight_min": -26.492626190185547,
      "activations/layer2_attention_weight_max": 30.798437118530273,
      "activations/layer2_attention_weight_min": -30.459117889404297,
      "activations/layer3_attention_weight_max": 82.02992248535156,
      "activations/layer3_attention_weight_min": -85.47010040283203,
      "activations/layer4_attention_weight_max": 77.63197326660156,
      "activations/layer4_attention_weight_min": -79.50984954833984,
      "activations/layer5_attention_weight_max": 60.37699890136719,
      "activations/layer5_attention_weight_min": -70.59515380859375,
      "activations/layer6_attention_weight_max": 47.71524429321289,
      "activations/layer6_attention_weight_min": -47.44242858886719,
      "activations/layer7_attention_weight_max": 63.565303802490234,
      "activations/layer7_attention_weight_min": -62.65623474121094,
      "activations/layer8_attention_weight_max": 44.53181838989258,
      "activations/layer8_attention_weight_min": -48.29327392578125,
      "activations/layer9_attention_weight_max": 42.90256881713867,
      "activations/layer9_attention_weight_min": -44.9889030456543,
      "epoch": 19.69,
      "learning_rate": 2.324734848484848e-05,
      "loss": 2.73,
      "step": 338900
    },
    {
      "activations/layer0_attention_weight_max": 15.464755058288574,
      "activations/layer0_attention_weight_min": -14.035134315490723,
      "activations/layer10_attention_weight_max": 33.940643310546875,
      "activations/layer10_attention_weight_min": -35.10904312133789,
      "activations/layer11_attention_weight_max": 32.37615966796875,
      "activations/layer11_attention_weight_min": -35.63258361816406,
      "activations/layer12_attention_weight_max": 20.14717674255371,
      "activations/layer12_attention_weight_min": -25.788827896118164,
      "activations/layer13_attention_weight_max": 31.64078712463379,
      "activations/layer13_attention_weight_min": -27.228784561157227,
      "activations/layer14_attention_weight_max": 34.357810974121094,
      "activations/layer14_attention_weight_min": -30.40920066833496,
      "activations/layer15_attention_weight_max": 32.808128356933594,
      "activations/layer15_attention_weight_min": -29.744911193847656,
      "activations/layer16_attention_weight_max": 34.61948013305664,
      "activations/layer16_attention_weight_min": -33.43043518066406,
      "activations/layer17_attention_weight_max": 50.47800827026367,
      "activations/layer17_attention_weight_min": -45.387123107910156,
      "activations/layer18_attention_weight_max": 47.3657341003418,
      "activations/layer18_attention_weight_min": -40.76773452758789,
      "activations/layer19_attention_weight_max": 25.777124404907227,
      "activations/layer19_attention_weight_min": -23.692075729370117,
      "activations/layer1_attention_weight_max": 16.095462799072266,
      "activations/layer1_attention_weight_min": -14.780160903930664,
      "activations/layer20_attention_weight_max": 25.384748458862305,
      "activations/layer20_attention_weight_min": -21.152141571044922,
      "activations/layer21_attention_weight_max": 36.160064697265625,
      "activations/layer21_attention_weight_min": -21.995525360107422,
      "activations/layer22_attention_weight_max": 28.826217651367188,
      "activations/layer22_attention_weight_min": -25.98194122314453,
      "activations/layer23_attention_weight_max": 39.70905303955078,
      "activations/layer23_attention_weight_min": -23.28417205810547,
      "activations/layer2_attention_weight_max": 31.182140350341797,
      "activations/layer2_attention_weight_min": -29.825477600097656,
      "activations/layer3_attention_weight_max": 85.34618377685547,
      "activations/layer3_attention_weight_min": -90.0914535522461,
      "activations/layer4_attention_weight_max": 80.65685272216797,
      "activations/layer4_attention_weight_min": -82.40129852294922,
      "activations/layer5_attention_weight_max": 65.08439636230469,
      "activations/layer5_attention_weight_min": -68.21480560302734,
      "activations/layer6_attention_weight_max": 47.62092590332031,
      "activations/layer6_attention_weight_min": -51.769893646240234,
      "activations/layer7_attention_weight_max": 63.800743103027344,
      "activations/layer7_attention_weight_min": -65.11820220947266,
      "activations/layer8_attention_weight_max": 46.596744537353516,
      "activations/layer8_attention_weight_min": -49.419593811035156,
      "activations/layer9_attention_weight_max": 43.49528884887695,
      "activations/layer9_attention_weight_min": -47.60689163208008,
      "epoch": 19.69,
      "learning_rate": 2.322840909090909e-05,
      "loss": 2.7323,
      "step": 338950
    },
    {
      "activations/layer0_attention_weight_max": 15.311647415161133,
      "activations/layer0_attention_weight_min": -13.912637710571289,
      "activations/layer10_attention_weight_max": 34.830810546875,
      "activations/layer10_attention_weight_min": -34.829463958740234,
      "activations/layer11_attention_weight_max": 34.41014862060547,
      "activations/layer11_attention_weight_min": -36.267662048339844,
      "activations/layer12_attention_weight_max": 19.290693283081055,
      "activations/layer12_attention_weight_min": -23.234115600585938,
      "activations/layer13_attention_weight_max": 31.75635528564453,
      "activations/layer13_attention_weight_min": -28.01764678955078,
      "activations/layer14_attention_weight_max": 32.69867706298828,
      "activations/layer14_attention_weight_min": -30.028594970703125,
      "activations/layer15_attention_weight_max": 32.40249252319336,
      "activations/layer15_attention_weight_min": -31.926969528198242,
      "activations/layer16_attention_weight_max": 32.57077407836914,
      "activations/layer16_attention_weight_min": -30.980525970458984,
      "activations/layer17_attention_weight_max": 49.168888092041016,
      "activations/layer17_attention_weight_min": -42.8618278503418,
      "activations/layer18_attention_weight_max": 46.69136428833008,
      "activations/layer18_attention_weight_min": -40.49140548706055,
      "activations/layer19_attention_weight_max": 24.17413902282715,
      "activations/layer19_attention_weight_min": -21.796964645385742,
      "activations/layer1_attention_weight_max": 15.57121753692627,
      "activations/layer1_attention_weight_min": -12.985718727111816,
      "activations/layer20_attention_weight_max": 22.508132934570312,
      "activations/layer20_attention_weight_min": -22.025344848632812,
      "activations/layer21_attention_weight_max": 34.9708366394043,
      "activations/layer21_attention_weight_min": -24.13749122619629,
      "activations/layer22_attention_weight_max": 28.685144424438477,
      "activations/layer22_attention_weight_min": -24.74721336364746,
      "activations/layer23_attention_weight_max": 36.1744384765625,
      "activations/layer23_attention_weight_min": -24.479228973388672,
      "activations/layer2_attention_weight_max": 33.616634368896484,
      "activations/layer2_attention_weight_min": -30.18221664428711,
      "activations/layer3_attention_weight_max": 89.13141632080078,
      "activations/layer3_attention_weight_min": -87.66169738769531,
      "activations/layer4_attention_weight_max": 84.3457260131836,
      "activations/layer4_attention_weight_min": -84.34066009521484,
      "activations/layer5_attention_weight_max": 63.57884979248047,
      "activations/layer5_attention_weight_min": -71.02379608154297,
      "activations/layer6_attention_weight_max": 50.76994705200195,
      "activations/layer6_attention_weight_min": -53.96126174926758,
      "activations/layer7_attention_weight_max": 64.52128601074219,
      "activations/layer7_attention_weight_min": -65.9448013305664,
      "activations/layer8_attention_weight_max": 48.74027633666992,
      "activations/layer8_attention_weight_min": -49.514034271240234,
      "activations/layer9_attention_weight_max": 49.933895111083984,
      "activations/layer9_attention_weight_min": -46.85654830932617,
      "epoch": 19.7,
      "learning_rate": 2.3209469696969694e-05,
      "loss": 2.7363,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.5658,
      "eval_samples_per_second": 501.294,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.5658,
      "eval_openwebtext_samples_per_second": 501.294,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.056,
      "eval_wikitext_samples_per_second": 221.789,
      "step": 339000
    },
    {
      "epoch": 19.7,
      "eval_lambada_loss": 2.4375,
      "eval_lambada_ppl": 11.444393964331121,
      "eval_lambada_runtime": 9.6185,
      "eval_lambada_samples_per_second": 506.212,
      "step": 339000
    },
    {
      "activations/layer0_attention_weight_max": 14.991559982299805,
      "activations/layer0_attention_weight_min": -14.240277290344238,
      "activations/layer10_attention_weight_max": 33.291969299316406,
      "activations/layer10_attention_weight_min": -35.33208084106445,
      "activations/layer11_attention_weight_max": 30.800640106201172,
      "activations/layer11_attention_weight_min": -33.525665283203125,
      "activations/layer12_attention_weight_max": 18.898202896118164,
      "activations/layer12_attention_weight_min": -25.540353775024414,
      "activations/layer13_attention_weight_max": 33.903682708740234,
      "activations/layer13_attention_weight_min": -27.517234802246094,
      "activations/layer14_attention_weight_max": 33.535850524902344,
      "activations/layer14_attention_weight_min": -30.685609817504883,
      "activations/layer15_attention_weight_max": 31.82602882385254,
      "activations/layer15_attention_weight_min": -31.468557357788086,
      "activations/layer16_attention_weight_max": 32.872440338134766,
      "activations/layer16_attention_weight_min": -32.22916030883789,
      "activations/layer17_attention_weight_max": 50.141597747802734,
      "activations/layer17_attention_weight_min": -45.881134033203125,
      "activations/layer18_attention_weight_max": 46.338253021240234,
      "activations/layer18_attention_weight_min": -40.86647033691406,
      "activations/layer19_attention_weight_max": 23.61485481262207,
      "activations/layer19_attention_weight_min": -23.175281524658203,
      "activations/layer1_attention_weight_max": 15.663105964660645,
      "activations/layer1_attention_weight_min": -13.035736083984375,
      "activations/layer20_attention_weight_max": 23.242952346801758,
      "activations/layer20_attention_weight_min": -21.51736068725586,
      "activations/layer21_attention_weight_max": 33.88224411010742,
      "activations/layer21_attention_weight_min": -22.3902530670166,
      "activations/layer22_attention_weight_max": 28.89378547668457,
      "activations/layer22_attention_weight_min": -26.140899658203125,
      "activations/layer23_attention_weight_max": 35.0141487121582,
      "activations/layer23_attention_weight_min": -23.671688079833984,
      "activations/layer2_attention_weight_max": 30.271198272705078,
      "activations/layer2_attention_weight_min": -29.58797264099121,
      "activations/layer3_attention_weight_max": 82.28954315185547,
      "activations/layer3_attention_weight_min": -85.85299682617188,
      "activations/layer4_attention_weight_max": 83.66535186767578,
      "activations/layer4_attention_weight_min": -81.0667953491211,
      "activations/layer5_attention_weight_max": 61.462745666503906,
      "activations/layer5_attention_weight_min": -69.40062713623047,
      "activations/layer6_attention_weight_max": 47.52732849121094,
      "activations/layer6_attention_weight_min": -51.58948516845703,
      "activations/layer7_attention_weight_max": 63.70376968383789,
      "activations/layer7_attention_weight_min": -64.5600357055664,
      "activations/layer8_attention_weight_max": 43.17845916748047,
      "activations/layer8_attention_weight_min": -49.955360412597656,
      "activations/layer9_attention_weight_max": 42.554866790771484,
      "activations/layer9_attention_weight_min": -49.79440689086914,
      "epoch": 19.7,
      "learning_rate": 2.3190530303030302e-05,
      "loss": 2.7377,
      "step": 339050
    },
    {
      "activations/layer0_attention_weight_max": 15.41346549987793,
      "activations/layer0_attention_weight_min": -14.334614753723145,
      "activations/layer10_attention_weight_max": 37.17292785644531,
      "activations/layer10_attention_weight_min": -37.34470748901367,
      "activations/layer11_attention_weight_max": 33.280067443847656,
      "activations/layer11_attention_weight_min": -34.37352752685547,
      "activations/layer12_attention_weight_max": 17.136199951171875,
      "activations/layer12_attention_weight_min": -25.046659469604492,
      "activations/layer13_attention_weight_max": 35.164058685302734,
      "activations/layer13_attention_weight_min": -29.622777938842773,
      "activations/layer14_attention_weight_max": 34.73771286010742,
      "activations/layer14_attention_weight_min": -31.590553283691406,
      "activations/layer15_attention_weight_max": 32.82130432128906,
      "activations/layer15_attention_weight_min": -32.104427337646484,
      "activations/layer16_attention_weight_max": 36.097816467285156,
      "activations/layer16_attention_weight_min": -31.7886962890625,
      "activations/layer17_attention_weight_max": 48.93724822998047,
      "activations/layer17_attention_weight_min": -48.201786041259766,
      "activations/layer18_attention_weight_max": 45.54365539550781,
      "activations/layer18_attention_weight_min": -44.30148696899414,
      "activations/layer19_attention_weight_max": 24.657970428466797,
      "activations/layer19_attention_weight_min": -22.524574279785156,
      "activations/layer1_attention_weight_max": 15.813583374023438,
      "activations/layer1_attention_weight_min": -14.24095630645752,
      "activations/layer20_attention_weight_max": 24.23051643371582,
      "activations/layer20_attention_weight_min": -21.902816772460938,
      "activations/layer21_attention_weight_max": 38.53342819213867,
      "activations/layer21_attention_weight_min": -22.621747970581055,
      "activations/layer22_attention_weight_max": 31.330198287963867,
      "activations/layer22_attention_weight_min": -25.05145835876465,
      "activations/layer23_attention_weight_max": 40.93344497680664,
      "activations/layer23_attention_weight_min": -23.27539825439453,
      "activations/layer2_attention_weight_max": 32.79534912109375,
      "activations/layer2_attention_weight_min": -30.796306610107422,
      "activations/layer3_attention_weight_max": 92.33906555175781,
      "activations/layer3_attention_weight_min": -88.44954681396484,
      "activations/layer4_attention_weight_max": 84.69414520263672,
      "activations/layer4_attention_weight_min": -84.18782043457031,
      "activations/layer5_attention_weight_max": 60.690101623535156,
      "activations/layer5_attention_weight_min": -73.48603820800781,
      "activations/layer6_attention_weight_max": 52.104637145996094,
      "activations/layer6_attention_weight_min": -52.932987213134766,
      "activations/layer7_attention_weight_max": 64.92411804199219,
      "activations/layer7_attention_weight_min": -64.45924377441406,
      "activations/layer8_attention_weight_max": 49.35210037231445,
      "activations/layer8_attention_weight_min": -50.974449157714844,
      "activations/layer9_attention_weight_max": 45.557491302490234,
      "activations/layer9_attention_weight_min": -47.9918327331543,
      "epoch": 19.7,
      "learning_rate": 2.317159090909091e-05,
      "loss": 2.7271,
      "step": 339100
    },
    {
      "activations/layer0_attention_weight_max": 14.868056297302246,
      "activations/layer0_attention_weight_min": -13.96632194519043,
      "activations/layer10_attention_weight_max": 38.584999084472656,
      "activations/layer10_attention_weight_min": -35.97368240356445,
      "activations/layer11_attention_weight_max": 37.02429962158203,
      "activations/layer11_attention_weight_min": -35.103363037109375,
      "activations/layer12_attention_weight_max": 19.18147087097168,
      "activations/layer12_attention_weight_min": -27.650712966918945,
      "activations/layer13_attention_weight_max": 39.152626037597656,
      "activations/layer13_attention_weight_min": -29.753108978271484,
      "activations/layer14_attention_weight_max": 37.15647888183594,
      "activations/layer14_attention_weight_min": -31.437891006469727,
      "activations/layer15_attention_weight_max": 36.74528121948242,
      "activations/layer15_attention_weight_min": -30.255836486816406,
      "activations/layer16_attention_weight_max": 34.117393493652344,
      "activations/layer16_attention_weight_min": -30.73326301574707,
      "activations/layer17_attention_weight_max": 51.5833854675293,
      "activations/layer17_attention_weight_min": -43.5728874206543,
      "activations/layer18_attention_weight_max": 51.32571792602539,
      "activations/layer18_attention_weight_min": -39.361759185791016,
      "activations/layer19_attention_weight_max": 23.824575424194336,
      "activations/layer19_attention_weight_min": -23.8548641204834,
      "activations/layer1_attention_weight_max": 15.47543716430664,
      "activations/layer1_attention_weight_min": -14.687240600585938,
      "activations/layer20_attention_weight_max": 23.31289291381836,
      "activations/layer20_attention_weight_min": -23.627317428588867,
      "activations/layer21_attention_weight_max": 33.268310546875,
      "activations/layer21_attention_weight_min": -24.137842178344727,
      "activations/layer22_attention_weight_max": 26.592710494995117,
      "activations/layer22_attention_weight_min": -27.644346237182617,
      "activations/layer23_attention_weight_max": 34.31153869628906,
      "activations/layer23_attention_weight_min": -24.578968048095703,
      "activations/layer2_attention_weight_max": 34.455440521240234,
      "activations/layer2_attention_weight_min": -30.6091365814209,
      "activations/layer3_attention_weight_max": 89.68386840820312,
      "activations/layer3_attention_weight_min": -93.5248794555664,
      "activations/layer4_attention_weight_max": 83.0374984741211,
      "activations/layer4_attention_weight_min": -83.33390808105469,
      "activations/layer5_attention_weight_max": 65.68128204345703,
      "activations/layer5_attention_weight_min": -71.41314697265625,
      "activations/layer6_attention_weight_max": 51.51181411743164,
      "activations/layer6_attention_weight_min": -50.851566314697266,
      "activations/layer7_attention_weight_max": 72.79759979248047,
      "activations/layer7_attention_weight_min": -64.35314178466797,
      "activations/layer8_attention_weight_max": 54.245670318603516,
      "activations/layer8_attention_weight_min": -51.0845947265625,
      "activations/layer9_attention_weight_max": 54.927154541015625,
      "activations/layer9_attention_weight_min": -49.7171745300293,
      "epoch": 19.71,
      "learning_rate": 2.3152651515151512e-05,
      "loss": 2.7217,
      "step": 339150
    },
    {
      "activations/layer0_attention_weight_max": 16.275470733642578,
      "activations/layer0_attention_weight_min": -13.72624683380127,
      "activations/layer10_attention_weight_max": 35.251949310302734,
      "activations/layer10_attention_weight_min": -35.16591262817383,
      "activations/layer11_attention_weight_max": 34.03759002685547,
      "activations/layer11_attention_weight_min": -34.169776916503906,
      "activations/layer12_attention_weight_max": 19.53194236755371,
      "activations/layer12_attention_weight_min": -24.18608856201172,
      "activations/layer13_attention_weight_max": 34.05522918701172,
      "activations/layer13_attention_weight_min": -27.994966506958008,
      "activations/layer14_attention_weight_max": 36.40313720703125,
      "activations/layer14_attention_weight_min": -31.882448196411133,
      "activations/layer15_attention_weight_max": 34.072120666503906,
      "activations/layer15_attention_weight_min": -30.678220748901367,
      "activations/layer16_attention_weight_max": 32.99637985229492,
      "activations/layer16_attention_weight_min": -33.96403503417969,
      "activations/layer17_attention_weight_max": 53.55234909057617,
      "activations/layer17_attention_weight_min": -46.68196105957031,
      "activations/layer18_attention_weight_max": 48.33467483520508,
      "activations/layer18_attention_weight_min": -44.034114837646484,
      "activations/layer19_attention_weight_max": 23.88764762878418,
      "activations/layer19_attention_weight_min": -26.589078903198242,
      "activations/layer1_attention_weight_max": 15.557096481323242,
      "activations/layer1_attention_weight_min": -13.423154830932617,
      "activations/layer20_attention_weight_max": 25.589269638061523,
      "activations/layer20_attention_weight_min": -23.483537673950195,
      "activations/layer21_attention_weight_max": 38.584938049316406,
      "activations/layer21_attention_weight_min": -22.66018295288086,
      "activations/layer22_attention_weight_max": 27.9539794921875,
      "activations/layer22_attention_weight_min": -26.66033172607422,
      "activations/layer23_attention_weight_max": 38.00553894042969,
      "activations/layer23_attention_weight_min": -24.594242095947266,
      "activations/layer2_attention_weight_max": 31.543136596679688,
      "activations/layer2_attention_weight_min": -29.683561325073242,
      "activations/layer3_attention_weight_max": 85.75188446044922,
      "activations/layer3_attention_weight_min": -84.77261352539062,
      "activations/layer4_attention_weight_max": 83.28591918945312,
      "activations/layer4_attention_weight_min": -78.96141052246094,
      "activations/layer5_attention_weight_max": 61.93693161010742,
      "activations/layer5_attention_weight_min": -70.26439666748047,
      "activations/layer6_attention_weight_max": 53.64828872680664,
      "activations/layer6_attention_weight_min": -50.71078872680664,
      "activations/layer7_attention_weight_max": 62.846405029296875,
      "activations/layer7_attention_weight_min": -64.81735229492188,
      "activations/layer8_attention_weight_max": 45.21345520019531,
      "activations/layer8_attention_weight_min": -49.28249740600586,
      "activations/layer9_attention_weight_max": 41.621768951416016,
      "activations/layer9_attention_weight_min": -43.76078796386719,
      "epoch": 19.71,
      "learning_rate": 2.313371212121212e-05,
      "loss": 2.7265,
      "step": 339200
    },
    {
      "activations/layer0_attention_weight_max": 14.597193717956543,
      "activations/layer0_attention_weight_min": -14.027772903442383,
      "activations/layer10_attention_weight_max": 35.86247253417969,
      "activations/layer10_attention_weight_min": -34.40391159057617,
      "activations/layer11_attention_weight_max": 32.90799331665039,
      "activations/layer11_attention_weight_min": -33.987510681152344,
      "activations/layer12_attention_weight_max": 19.289730072021484,
      "activations/layer12_attention_weight_min": -28.46951675415039,
      "activations/layer13_attention_weight_max": 38.03510284423828,
      "activations/layer13_attention_weight_min": -29.47825813293457,
      "activations/layer14_attention_weight_max": 37.931941986083984,
      "activations/layer14_attention_weight_min": -31.97671890258789,
      "activations/layer15_attention_weight_max": 35.71204376220703,
      "activations/layer15_attention_weight_min": -31.109018325805664,
      "activations/layer16_attention_weight_max": 38.715858459472656,
      "activations/layer16_attention_weight_min": -35.35443115234375,
      "activations/layer17_attention_weight_max": 56.69255828857422,
      "activations/layer17_attention_weight_min": -48.821006774902344,
      "activations/layer18_attention_weight_max": 52.91793441772461,
      "activations/layer18_attention_weight_min": -43.1737174987793,
      "activations/layer19_attention_weight_max": 25.76265525817871,
      "activations/layer19_attention_weight_min": -24.24001121520996,
      "activations/layer1_attention_weight_max": 16.08439064025879,
      "activations/layer1_attention_weight_min": -13.752965927124023,
      "activations/layer20_attention_weight_max": 26.151439666748047,
      "activations/layer20_attention_weight_min": -24.797588348388672,
      "activations/layer21_attention_weight_max": 46.1844596862793,
      "activations/layer21_attention_weight_min": -26.992473602294922,
      "activations/layer22_attention_weight_max": 34.85928726196289,
      "activations/layer22_attention_weight_min": -27.0328369140625,
      "activations/layer23_attention_weight_max": 41.87672424316406,
      "activations/layer23_attention_weight_min": -25.754390716552734,
      "activations/layer2_attention_weight_max": 30.568058013916016,
      "activations/layer2_attention_weight_min": -28.304725646972656,
      "activations/layer3_attention_weight_max": 86.47420501708984,
      "activations/layer3_attention_weight_min": -82.01803588867188,
      "activations/layer4_attention_weight_max": 81.57489776611328,
      "activations/layer4_attention_weight_min": -76.76078033447266,
      "activations/layer5_attention_weight_max": 61.97343063354492,
      "activations/layer5_attention_weight_min": -67.73567962646484,
      "activations/layer6_attention_weight_max": 46.02168655395508,
      "activations/layer6_attention_weight_min": -50.17505645751953,
      "activations/layer7_attention_weight_max": 63.592140197753906,
      "activations/layer7_attention_weight_min": -60.943145751953125,
      "activations/layer8_attention_weight_max": 43.26326370239258,
      "activations/layer8_attention_weight_min": -46.06022644042969,
      "activations/layer9_attention_weight_max": 40.83235549926758,
      "activations/layer9_attention_weight_min": -41.978912353515625,
      "epoch": 19.71,
      "learning_rate": 2.3114772727272725e-05,
      "loss": 2.7069,
      "step": 339250
    },
    {
      "activations/layer0_attention_weight_max": 15.265228271484375,
      "activations/layer0_attention_weight_min": -13.752945899963379,
      "activations/layer10_attention_weight_max": 33.35460662841797,
      "activations/layer10_attention_weight_min": -33.47639846801758,
      "activations/layer11_attention_weight_max": 31.838775634765625,
      "activations/layer11_attention_weight_min": -31.807613372802734,
      "activations/layer12_attention_weight_max": 18.191057205200195,
      "activations/layer12_attention_weight_min": -26.40534210205078,
      "activations/layer13_attention_weight_max": 38.04956817626953,
      "activations/layer13_attention_weight_min": -27.50078010559082,
      "activations/layer14_attention_weight_max": 33.4105110168457,
      "activations/layer14_attention_weight_min": -28.732769012451172,
      "activations/layer15_attention_weight_max": 31.197540283203125,
      "activations/layer15_attention_weight_min": -29.75387954711914,
      "activations/layer16_attention_weight_max": 33.05638885498047,
      "activations/layer16_attention_weight_min": -30.199438095092773,
      "activations/layer17_attention_weight_max": 50.32230758666992,
      "activations/layer17_attention_weight_min": -43.28105545043945,
      "activations/layer18_attention_weight_max": 47.06305694580078,
      "activations/layer18_attention_weight_min": -38.333370208740234,
      "activations/layer19_attention_weight_max": 25.600473403930664,
      "activations/layer19_attention_weight_min": -21.89272689819336,
      "activations/layer1_attention_weight_max": 15.598639488220215,
      "activations/layer1_attention_weight_min": -14.025165557861328,
      "activations/layer20_attention_weight_max": 22.586828231811523,
      "activations/layer20_attention_weight_min": -20.97926902770996,
      "activations/layer21_attention_weight_max": 36.98244094848633,
      "activations/layer21_attention_weight_min": -20.58845329284668,
      "activations/layer22_attention_weight_max": 30.68431282043457,
      "activations/layer22_attention_weight_min": -25.120250701904297,
      "activations/layer23_attention_weight_max": 37.69674301147461,
      "activations/layer23_attention_weight_min": -24.046415328979492,
      "activations/layer2_attention_weight_max": 30.88970947265625,
      "activations/layer2_attention_weight_min": -29.24985122680664,
      "activations/layer3_attention_weight_max": 86.36664581298828,
      "activations/layer3_attention_weight_min": -85.96240997314453,
      "activations/layer4_attention_weight_max": 80.66009521484375,
      "activations/layer4_attention_weight_min": -76.71031951904297,
      "activations/layer5_attention_weight_max": 62.515438079833984,
      "activations/layer5_attention_weight_min": -70.07786560058594,
      "activations/layer6_attention_weight_max": 49.21662139892578,
      "activations/layer6_attention_weight_min": -48.46839904785156,
      "activations/layer7_attention_weight_max": 64.41423797607422,
      "activations/layer7_attention_weight_min": -63.00325012207031,
      "activations/layer8_attention_weight_max": 44.84251022338867,
      "activations/layer8_attention_weight_min": -49.231597900390625,
      "activations/layer9_attention_weight_max": 42.12424850463867,
      "activations/layer9_attention_weight_min": -47.98529052734375,
      "epoch": 19.72,
      "learning_rate": 2.3095833333333333e-05,
      "loss": 2.7125,
      "step": 339300
    },
    {
      "activations/layer0_attention_weight_max": 15.906440734863281,
      "activations/layer0_attention_weight_min": -14.024327278137207,
      "activations/layer10_attention_weight_max": 38.49170684814453,
      "activations/layer10_attention_weight_min": -37.986698150634766,
      "activations/layer11_attention_weight_max": 33.086212158203125,
      "activations/layer11_attention_weight_min": -34.67648696899414,
      "activations/layer12_attention_weight_max": 19.10353660583496,
      "activations/layer12_attention_weight_min": -23.343793869018555,
      "activations/layer13_attention_weight_max": 34.5874137878418,
      "activations/layer13_attention_weight_min": -29.9298038482666,
      "activations/layer14_attention_weight_max": 34.059104919433594,
      "activations/layer14_attention_weight_min": -30.592016220092773,
      "activations/layer15_attention_weight_max": 35.98770523071289,
      "activations/layer15_attention_weight_min": -30.73528289794922,
      "activations/layer16_attention_weight_max": 34.77012634277344,
      "activations/layer16_attention_weight_min": -32.926605224609375,
      "activations/layer17_attention_weight_max": 53.34135055541992,
      "activations/layer17_attention_weight_min": -47.079856872558594,
      "activations/layer18_attention_weight_max": 47.45532989501953,
      "activations/layer18_attention_weight_min": -41.86783218383789,
      "activations/layer19_attention_weight_max": 26.25734519958496,
      "activations/layer19_attention_weight_min": -22.594938278198242,
      "activations/layer1_attention_weight_max": 15.873347282409668,
      "activations/layer1_attention_weight_min": -14.458406448364258,
      "activations/layer20_attention_weight_max": 23.46898078918457,
      "activations/layer20_attention_weight_min": -25.8802547454834,
      "activations/layer21_attention_weight_max": 38.248992919921875,
      "activations/layer21_attention_weight_min": -22.16404151916504,
      "activations/layer22_attention_weight_max": 32.506072998046875,
      "activations/layer22_attention_weight_min": -25.240346908569336,
      "activations/layer23_attention_weight_max": 34.637840270996094,
      "activations/layer23_attention_weight_min": -25.232885360717773,
      "activations/layer2_attention_weight_max": 29.690824508666992,
      "activations/layer2_attention_weight_min": -29.05182456970215,
      "activations/layer3_attention_weight_max": 80.19766998291016,
      "activations/layer3_attention_weight_min": -80.63388061523438,
      "activations/layer4_attention_weight_max": 78.46764373779297,
      "activations/layer4_attention_weight_min": -78.06493377685547,
      "activations/layer5_attention_weight_max": 61.920555114746094,
      "activations/layer5_attention_weight_min": -71.94847106933594,
      "activations/layer6_attention_weight_max": 48.234012603759766,
      "activations/layer6_attention_weight_min": -51.43913269042969,
      "activations/layer7_attention_weight_max": 67.67901611328125,
      "activations/layer7_attention_weight_min": -66.88922119140625,
      "activations/layer8_attention_weight_max": 50.26227569580078,
      "activations/layer8_attention_weight_min": -52.35093307495117,
      "activations/layer9_attention_weight_max": 46.601959228515625,
      "activations/layer9_attention_weight_min": -47.02113342285156,
      "epoch": 19.72,
      "learning_rate": 2.3076893939393938e-05,
      "loss": 2.7312,
      "step": 339350
    },
    {
      "activations/layer0_attention_weight_max": 15.880010604858398,
      "activations/layer0_attention_weight_min": -14.221220016479492,
      "activations/layer10_attention_weight_max": 37.46883010864258,
      "activations/layer10_attention_weight_min": -38.24128723144531,
      "activations/layer11_attention_weight_max": 34.624351501464844,
      "activations/layer11_attention_weight_min": -36.192970275878906,
      "activations/layer12_attention_weight_max": 19.420270919799805,
      "activations/layer12_attention_weight_min": -27.268573760986328,
      "activations/layer13_attention_weight_max": 33.77257537841797,
      "activations/layer13_attention_weight_min": -26.643470764160156,
      "activations/layer14_attention_weight_max": 34.56856918334961,
      "activations/layer14_attention_weight_min": -31.530948638916016,
      "activations/layer15_attention_weight_max": 31.563926696777344,
      "activations/layer15_attention_weight_min": -29.69400978088379,
      "activations/layer16_attention_weight_max": 35.44412612915039,
      "activations/layer16_attention_weight_min": -34.88588333129883,
      "activations/layer17_attention_weight_max": 53.63959503173828,
      "activations/layer17_attention_weight_min": -48.995059967041016,
      "activations/layer18_attention_weight_max": 52.091773986816406,
      "activations/layer18_attention_weight_min": -43.05093002319336,
      "activations/layer19_attention_weight_max": 24.760221481323242,
      "activations/layer19_attention_weight_min": -23.43221092224121,
      "activations/layer1_attention_weight_max": 15.967774391174316,
      "activations/layer1_attention_weight_min": -14.226239204406738,
      "activations/layer20_attention_weight_max": 23.06610679626465,
      "activations/layer20_attention_weight_min": -24.196985244750977,
      "activations/layer21_attention_weight_max": 38.33625030517578,
      "activations/layer21_attention_weight_min": -21.7867374420166,
      "activations/layer22_attention_weight_max": 33.35282897949219,
      "activations/layer22_attention_weight_min": -26.65667724609375,
      "activations/layer23_attention_weight_max": 35.096031188964844,
      "activations/layer23_attention_weight_min": -22.801376342773438,
      "activations/layer2_attention_weight_max": 30.686429977416992,
      "activations/layer2_attention_weight_min": -29.329862594604492,
      "activations/layer3_attention_weight_max": 84.52490997314453,
      "activations/layer3_attention_weight_min": -85.63785552978516,
      "activations/layer4_attention_weight_max": 83.74983978271484,
      "activations/layer4_attention_weight_min": -85.1998291015625,
      "activations/layer5_attention_weight_max": 61.523048400878906,
      "activations/layer5_attention_weight_min": -73.67510986328125,
      "activations/layer6_attention_weight_max": 50.27152633666992,
      "activations/layer6_attention_weight_min": -51.11912536621094,
      "activations/layer7_attention_weight_max": 69.53221130371094,
      "activations/layer7_attention_weight_min": -67.41768646240234,
      "activations/layer8_attention_weight_max": 47.888240814208984,
      "activations/layer8_attention_weight_min": -53.73752975463867,
      "activations/layer9_attention_weight_max": 46.944244384765625,
      "activations/layer9_attention_weight_min": -48.90830612182617,
      "epoch": 19.72,
      "learning_rate": 2.3057954545454543e-05,
      "loss": 2.7122,
      "step": 339400
    },
    {
      "activations/layer0_attention_weight_max": 15.842100143432617,
      "activations/layer0_attention_weight_min": -14.092691421508789,
      "activations/layer10_attention_weight_max": 35.22024917602539,
      "activations/layer10_attention_weight_min": -36.688438415527344,
      "activations/layer11_attention_weight_max": 34.430084228515625,
      "activations/layer11_attention_weight_min": -34.87773513793945,
      "activations/layer12_attention_weight_max": 18.405542373657227,
      "activations/layer12_attention_weight_min": -26.039670944213867,
      "activations/layer13_attention_weight_max": 32.732521057128906,
      "activations/layer13_attention_weight_min": -30.059505462646484,
      "activations/layer14_attention_weight_max": 35.75602340698242,
      "activations/layer14_attention_weight_min": -31.017927169799805,
      "activations/layer15_attention_weight_max": 30.42025375366211,
      "activations/layer15_attention_weight_min": -29.86312484741211,
      "activations/layer16_attention_weight_max": 31.387645721435547,
      "activations/layer16_attention_weight_min": -30.80457305908203,
      "activations/layer17_attention_weight_max": 53.02522659301758,
      "activations/layer17_attention_weight_min": -44.011932373046875,
      "activations/layer18_attention_weight_max": 45.6015739440918,
      "activations/layer18_attention_weight_min": -40.45207977294922,
      "activations/layer19_attention_weight_max": 23.04134178161621,
      "activations/layer19_attention_weight_min": -25.360322952270508,
      "activations/layer1_attention_weight_max": 15.663735389709473,
      "activations/layer1_attention_weight_min": -13.088886260986328,
      "activations/layer20_attention_weight_max": 21.434322357177734,
      "activations/layer20_attention_weight_min": -24.06884002685547,
      "activations/layer21_attention_weight_max": 34.386573791503906,
      "activations/layer21_attention_weight_min": -23.38100242614746,
      "activations/layer22_attention_weight_max": 27.855459213256836,
      "activations/layer22_attention_weight_min": -27.12662696838379,
      "activations/layer23_attention_weight_max": 33.57286834716797,
      "activations/layer23_attention_weight_min": -25.535572052001953,
      "activations/layer2_attention_weight_max": 31.01769256591797,
      "activations/layer2_attention_weight_min": -29.433378219604492,
      "activations/layer3_attention_weight_max": 89.50775909423828,
      "activations/layer3_attention_weight_min": -86.98423767089844,
      "activations/layer4_attention_weight_max": 85.46993255615234,
      "activations/layer4_attention_weight_min": -83.25202178955078,
      "activations/layer5_attention_weight_max": 61.73670959472656,
      "activations/layer5_attention_weight_min": -69.31305694580078,
      "activations/layer6_attention_weight_max": 50.48030471801758,
      "activations/layer6_attention_weight_min": -51.03242874145508,
      "activations/layer7_attention_weight_max": 62.20371627807617,
      "activations/layer7_attention_weight_min": -63.40607452392578,
      "activations/layer8_attention_weight_max": 47.360103607177734,
      "activations/layer8_attention_weight_min": -52.033470153808594,
      "activations/layer9_attention_weight_max": 43.626556396484375,
      "activations/layer9_attention_weight_min": -45.775753021240234,
      "epoch": 19.72,
      "learning_rate": 2.3039015151515148e-05,
      "loss": 2.7283,
      "step": 339450
    },
    {
      "activations/layer0_attention_weight_max": 15.994100570678711,
      "activations/layer0_attention_weight_min": -13.973833084106445,
      "activations/layer10_attention_weight_max": 38.40829086303711,
      "activations/layer10_attention_weight_min": -36.5603141784668,
      "activations/layer11_attention_weight_max": 35.25105285644531,
      "activations/layer11_attention_weight_min": -36.103721618652344,
      "activations/layer12_attention_weight_max": 21.59031105041504,
      "activations/layer12_attention_weight_min": -23.84099769592285,
      "activations/layer13_attention_weight_max": 46.32455062866211,
      "activations/layer13_attention_weight_min": -28.164405822753906,
      "activations/layer14_attention_weight_max": 44.18866729736328,
      "activations/layer14_attention_weight_min": -32.713233947753906,
      "activations/layer15_attention_weight_max": 39.82670593261719,
      "activations/layer15_attention_weight_min": -33.09739303588867,
      "activations/layer16_attention_weight_max": 43.6436767578125,
      "activations/layer16_attention_weight_min": -37.14847946166992,
      "activations/layer17_attention_weight_max": 64.52521514892578,
      "activations/layer17_attention_weight_min": -50.95900344848633,
      "activations/layer18_attention_weight_max": 59.512786865234375,
      "activations/layer18_attention_weight_min": -43.118614196777344,
      "activations/layer19_attention_weight_max": 31.65730094909668,
      "activations/layer19_attention_weight_min": -25.698965072631836,
      "activations/layer1_attention_weight_max": 16.01573944091797,
      "activations/layer1_attention_weight_min": -13.55969524383545,
      "activations/layer20_attention_weight_max": 28.5407657623291,
      "activations/layer20_attention_weight_min": -22.2467098236084,
      "activations/layer21_attention_weight_max": 47.1373176574707,
      "activations/layer21_attention_weight_min": -23.49863052368164,
      "activations/layer22_attention_weight_max": 31.117305755615234,
      "activations/layer22_attention_weight_min": -24.896596908569336,
      "activations/layer23_attention_weight_max": 44.69698715209961,
      "activations/layer23_attention_weight_min": -24.66994285583496,
      "activations/layer2_attention_weight_max": 29.90875816345215,
      "activations/layer2_attention_weight_min": -29.83754539489746,
      "activations/layer3_attention_weight_max": 82.78492736816406,
      "activations/layer3_attention_weight_min": -85.03443908691406,
      "activations/layer4_attention_weight_max": 81.0101318359375,
      "activations/layer4_attention_weight_min": -79.91193389892578,
      "activations/layer5_attention_weight_max": 59.47977828979492,
      "activations/layer5_attention_weight_min": -71.09195709228516,
      "activations/layer6_attention_weight_max": 48.991233825683594,
      "activations/layer6_attention_weight_min": -52.83230209350586,
      "activations/layer7_attention_weight_max": 62.902244567871094,
      "activations/layer7_attention_weight_min": -66.4849853515625,
      "activations/layer8_attention_weight_max": 48.68498611450195,
      "activations/layer8_attention_weight_min": -52.61552810668945,
      "activations/layer9_attention_weight_max": 44.58549118041992,
      "activations/layer9_attention_weight_min": -44.78132629394531,
      "epoch": 19.73,
      "learning_rate": 2.3020075757575756e-05,
      "loss": 2.735,
      "step": 339500
    },
    {
      "activations/layer0_attention_weight_max": 15.6673002243042,
      "activations/layer0_attention_weight_min": -13.806831359863281,
      "activations/layer10_attention_weight_max": 34.18117904663086,
      "activations/layer10_attention_weight_min": -33.54563903808594,
      "activations/layer11_attention_weight_max": 30.71908950805664,
      "activations/layer11_attention_weight_min": -31.8421688079834,
      "activations/layer12_attention_weight_max": 20.5824031829834,
      "activations/layer12_attention_weight_min": -25.823511123657227,
      "activations/layer13_attention_weight_max": 35.25465774536133,
      "activations/layer13_attention_weight_min": -26.25519371032715,
      "activations/layer14_attention_weight_max": 36.726043701171875,
      "activations/layer14_attention_weight_min": -30.078426361083984,
      "activations/layer15_attention_weight_max": 34.60021209716797,
      "activations/layer15_attention_weight_min": -28.632408142089844,
      "activations/layer16_attention_weight_max": 34.759544372558594,
      "activations/layer16_attention_weight_min": -32.47848892211914,
      "activations/layer17_attention_weight_max": 51.2971076965332,
      "activations/layer17_attention_weight_min": -43.94559097290039,
      "activations/layer18_attention_weight_max": 47.14090347290039,
      "activations/layer18_attention_weight_min": -37.53264236450195,
      "activations/layer19_attention_weight_max": 24.427465438842773,
      "activations/layer19_attention_weight_min": -22.590248107910156,
      "activations/layer1_attention_weight_max": 16.146648406982422,
      "activations/layer1_attention_weight_min": -13.410734176635742,
      "activations/layer20_attention_weight_max": 23.784074783325195,
      "activations/layer20_attention_weight_min": -22.403282165527344,
      "activations/layer21_attention_weight_max": 45.125083923339844,
      "activations/layer21_attention_weight_min": -23.83347511291504,
      "activations/layer22_attention_weight_max": 30.07958984375,
      "activations/layer22_attention_weight_min": -25.58417320251465,
      "activations/layer23_attention_weight_max": 34.223079681396484,
      "activations/layer23_attention_weight_min": -24.593786239624023,
      "activations/layer2_attention_weight_max": 29.508346557617188,
      "activations/layer2_attention_weight_min": -28.504920959472656,
      "activations/layer3_attention_weight_max": 84.97989654541016,
      "activations/layer3_attention_weight_min": -82.13843536376953,
      "activations/layer4_attention_weight_max": 78.60604858398438,
      "activations/layer4_attention_weight_min": -82.18150329589844,
      "activations/layer5_attention_weight_max": 60.2463493347168,
      "activations/layer5_attention_weight_min": -71.61363983154297,
      "activations/layer6_attention_weight_max": 47.577816009521484,
      "activations/layer6_attention_weight_min": -50.14916229248047,
      "activations/layer7_attention_weight_max": 60.635398864746094,
      "activations/layer7_attention_weight_min": -62.914432525634766,
      "activations/layer8_attention_weight_max": 45.8632698059082,
      "activations/layer8_attention_weight_min": -48.781917572021484,
      "activations/layer9_attention_weight_max": 44.120758056640625,
      "activations/layer9_attention_weight_min": -44.17097091674805,
      "epoch": 19.73,
      "learning_rate": 2.300113636363636e-05,
      "loss": 2.7283,
      "step": 339550
    },
    {
      "activations/layer0_attention_weight_max": 14.864633560180664,
      "activations/layer0_attention_weight_min": -14.072440147399902,
      "activations/layer10_attention_weight_max": 36.75426483154297,
      "activations/layer10_attention_weight_min": -35.83686065673828,
      "activations/layer11_attention_weight_max": 32.71052551269531,
      "activations/layer11_attention_weight_min": -34.160682678222656,
      "activations/layer12_attention_weight_max": 21.280475616455078,
      "activations/layer12_attention_weight_min": -28.304275512695312,
      "activations/layer13_attention_weight_max": 38.171241760253906,
      "activations/layer13_attention_weight_min": -27.15056800842285,
      "activations/layer14_attention_weight_max": 39.21800994873047,
      "activations/layer14_attention_weight_min": -30.975061416625977,
      "activations/layer15_attention_weight_max": 36.557823181152344,
      "activations/layer15_attention_weight_min": -30.350317001342773,
      "activations/layer16_attention_weight_max": 38.88534164428711,
      "activations/layer16_attention_weight_min": -36.420860290527344,
      "activations/layer17_attention_weight_max": 57.09705352783203,
      "activations/layer17_attention_weight_min": -48.513954162597656,
      "activations/layer18_attention_weight_max": 55.55435562133789,
      "activations/layer18_attention_weight_min": -46.34711837768555,
      "activations/layer19_attention_weight_max": 28.20480728149414,
      "activations/layer19_attention_weight_min": -23.749412536621094,
      "activations/layer1_attention_weight_max": 16.183265686035156,
      "activations/layer1_attention_weight_min": -14.288745880126953,
      "activations/layer20_attention_weight_max": 25.726696014404297,
      "activations/layer20_attention_weight_min": -22.383956909179688,
      "activations/layer21_attention_weight_max": 38.47042465209961,
      "activations/layer21_attention_weight_min": -22.1868896484375,
      "activations/layer22_attention_weight_max": 31.890039443969727,
      "activations/layer22_attention_weight_min": -27.14145851135254,
      "activations/layer23_attention_weight_max": 37.802764892578125,
      "activations/layer23_attention_weight_min": -24.035518646240234,
      "activations/layer2_attention_weight_max": 31.850019454956055,
      "activations/layer2_attention_weight_min": -30.103683471679688,
      "activations/layer3_attention_weight_max": 83.2055435180664,
      "activations/layer3_attention_weight_min": -84.53366088867188,
      "activations/layer4_attention_weight_max": 82.23430633544922,
      "activations/layer4_attention_weight_min": -84.4549560546875,
      "activations/layer5_attention_weight_max": 63.19898223876953,
      "activations/layer5_attention_weight_min": -71.77875518798828,
      "activations/layer6_attention_weight_max": 49.04262161254883,
      "activations/layer6_attention_weight_min": -52.46491622924805,
      "activations/layer7_attention_weight_max": 68.2541732788086,
      "activations/layer7_attention_weight_min": -63.61774826049805,
      "activations/layer8_attention_weight_max": 50.2108039855957,
      "activations/layer8_attention_weight_min": -49.599815368652344,
      "activations/layer9_attention_weight_max": 44.54815673828125,
      "activations/layer9_attention_weight_min": -44.9930305480957,
      "epoch": 19.73,
      "learning_rate": 2.298219696969697e-05,
      "loss": 2.7339,
      "step": 339600
    },
    {
      "activations/layer0_attention_weight_max": 15.118205070495605,
      "activations/layer0_attention_weight_min": -14.120011329650879,
      "activations/layer10_attention_weight_max": 35.44043731689453,
      "activations/layer10_attention_weight_min": -35.69036102294922,
      "activations/layer11_attention_weight_max": 33.082069396972656,
      "activations/layer11_attention_weight_min": -34.314144134521484,
      "activations/layer12_attention_weight_max": 19.760982513427734,
      "activations/layer12_attention_weight_min": -25.477941513061523,
      "activations/layer13_attention_weight_max": 31.133432388305664,
      "activations/layer13_attention_weight_min": -28.980587005615234,
      "activations/layer14_attention_weight_max": 34.464778900146484,
      "activations/layer14_attention_weight_min": -32.80893325805664,
      "activations/layer15_attention_weight_max": 31.35482406616211,
      "activations/layer15_attention_weight_min": -32.19978713989258,
      "activations/layer16_attention_weight_max": 33.218833923339844,
      "activations/layer16_attention_weight_min": -33.039215087890625,
      "activations/layer17_attention_weight_max": 50.22734451293945,
      "activations/layer17_attention_weight_min": -46.20769119262695,
      "activations/layer18_attention_weight_max": 45.39391326904297,
      "activations/layer18_attention_weight_min": -40.504905700683594,
      "activations/layer19_attention_weight_max": 26.377988815307617,
      "activations/layer19_attention_weight_min": -22.6406307220459,
      "activations/layer1_attention_weight_max": 15.46574878692627,
      "activations/layer1_attention_weight_min": -15.12161922454834,
      "activations/layer20_attention_weight_max": 22.845714569091797,
      "activations/layer20_attention_weight_min": -22.25713348388672,
      "activations/layer21_attention_weight_max": 39.697723388671875,
      "activations/layer21_attention_weight_min": -23.5828914642334,
      "activations/layer22_attention_weight_max": 29.515363693237305,
      "activations/layer22_attention_weight_min": -26.67682647705078,
      "activations/layer23_attention_weight_max": 39.67587661743164,
      "activations/layer23_attention_weight_min": -28.765106201171875,
      "activations/layer2_attention_weight_max": 31.246871948242188,
      "activations/layer2_attention_weight_min": -30.667221069335938,
      "activations/layer3_attention_weight_max": 82.84737396240234,
      "activations/layer3_attention_weight_min": -85.55427551269531,
      "activations/layer4_attention_weight_max": 81.1985855102539,
      "activations/layer4_attention_weight_min": -84.87569427490234,
      "activations/layer5_attention_weight_max": 60.159542083740234,
      "activations/layer5_attention_weight_min": -73.4590072631836,
      "activations/layer6_attention_weight_max": 47.964054107666016,
      "activations/layer6_attention_weight_min": -48.008323669433594,
      "activations/layer7_attention_weight_max": 62.665714263916016,
      "activations/layer7_attention_weight_min": -63.15147399902344,
      "activations/layer8_attention_weight_max": 45.745540618896484,
      "activations/layer8_attention_weight_min": -48.7368049621582,
      "activations/layer9_attention_weight_max": 41.30668258666992,
      "activations/layer9_attention_weight_min": -44.38607406616211,
      "epoch": 19.74,
      "learning_rate": 2.296325757575757e-05,
      "loss": 2.7162,
      "step": 339650
    },
    {
      "activations/layer0_attention_weight_max": 15.295685768127441,
      "activations/layer0_attention_weight_min": -14.31352424621582,
      "activations/layer10_attention_weight_max": 34.825042724609375,
      "activations/layer10_attention_weight_min": -35.888519287109375,
      "activations/layer11_attention_weight_max": 32.74430465698242,
      "activations/layer11_attention_weight_min": -34.967071533203125,
      "activations/layer12_attention_weight_max": 20.699087142944336,
      "activations/layer12_attention_weight_min": -23.673856735229492,
      "activations/layer13_attention_weight_max": 34.144317626953125,
      "activations/layer13_attention_weight_min": -28.334270477294922,
      "activations/layer14_attention_weight_max": 34.99709701538086,
      "activations/layer14_attention_weight_min": -31.187480926513672,
      "activations/layer15_attention_weight_max": 31.439796447753906,
      "activations/layer15_attention_weight_min": -31.051067352294922,
      "activations/layer16_attention_weight_max": 34.41315460205078,
      "activations/layer16_attention_weight_min": -32.7374267578125,
      "activations/layer17_attention_weight_max": 49.16611099243164,
      "activations/layer17_attention_weight_min": -45.123966217041016,
      "activations/layer18_attention_weight_max": 45.83779525756836,
      "activations/layer18_attention_weight_min": -41.06093215942383,
      "activations/layer19_attention_weight_max": 23.7526912689209,
      "activations/layer19_attention_weight_min": -22.710941314697266,
      "activations/layer1_attention_weight_max": 15.391403198242188,
      "activations/layer1_attention_weight_min": -15.10163402557373,
      "activations/layer20_attention_weight_max": 22.12411117553711,
      "activations/layer20_attention_weight_min": -21.390972137451172,
      "activations/layer21_attention_weight_max": 39.799373626708984,
      "activations/layer21_attention_weight_min": -23.198732376098633,
      "activations/layer22_attention_weight_max": 29.259403228759766,
      "activations/layer22_attention_weight_min": -25.377119064331055,
      "activations/layer23_attention_weight_max": 36.01119613647461,
      "activations/layer23_attention_weight_min": -24.846845626831055,
      "activations/layer2_attention_weight_max": 31.42981719970703,
      "activations/layer2_attention_weight_min": -30.293983459472656,
      "activations/layer3_attention_weight_max": 84.48548889160156,
      "activations/layer3_attention_weight_min": -84.4972915649414,
      "activations/layer4_attention_weight_max": 82.07804870605469,
      "activations/layer4_attention_weight_min": -80.75347137451172,
      "activations/layer5_attention_weight_max": 60.443016052246094,
      "activations/layer5_attention_weight_min": -74.8412857055664,
      "activations/layer6_attention_weight_max": 50.589107513427734,
      "activations/layer6_attention_weight_min": -49.80339050292969,
      "activations/layer7_attention_weight_max": 65.59719848632812,
      "activations/layer7_attention_weight_min": -65.11099243164062,
      "activations/layer8_attention_weight_max": 47.08763122558594,
      "activations/layer8_attention_weight_min": -52.24168395996094,
      "activations/layer9_attention_weight_max": 45.47710418701172,
      "activations/layer9_attention_weight_min": -46.544437408447266,
      "epoch": 19.74,
      "learning_rate": 2.294431818181818e-05,
      "loss": 2.7236,
      "step": 339700
    },
    {
      "activations/layer0_attention_weight_max": 15.500850677490234,
      "activations/layer0_attention_weight_min": -14.143359184265137,
      "activations/layer10_attention_weight_max": 35.56145095825195,
      "activations/layer10_attention_weight_min": -35.104835510253906,
      "activations/layer11_attention_weight_max": 32.58447265625,
      "activations/layer11_attention_weight_min": -35.40840148925781,
      "activations/layer12_attention_weight_max": 18.36883544921875,
      "activations/layer12_attention_weight_min": -24.957300186157227,
      "activations/layer13_attention_weight_max": 35.36878967285156,
      "activations/layer13_attention_weight_min": -27.609302520751953,
      "activations/layer14_attention_weight_max": 34.04878616333008,
      "activations/layer14_attention_weight_min": -30.599016189575195,
      "activations/layer15_attention_weight_max": 33.3764533996582,
      "activations/layer15_attention_weight_min": -30.188133239746094,
      "activations/layer16_attention_weight_max": 35.01395034790039,
      "activations/layer16_attention_weight_min": -34.08495330810547,
      "activations/layer17_attention_weight_max": 50.54560089111328,
      "activations/layer17_attention_weight_min": -45.31893539428711,
      "activations/layer18_attention_weight_max": 46.47901916503906,
      "activations/layer18_attention_weight_min": -40.350860595703125,
      "activations/layer19_attention_weight_max": 27.220178604125977,
      "activations/layer19_attention_weight_min": -24.2161922454834,
      "activations/layer1_attention_weight_max": 16.428817749023438,
      "activations/layer1_attention_weight_min": -13.660248756408691,
      "activations/layer20_attention_weight_max": 25.354597091674805,
      "activations/layer20_attention_weight_min": -22.272916793823242,
      "activations/layer21_attention_weight_max": 36.8751106262207,
      "activations/layer21_attention_weight_min": -26.309635162353516,
      "activations/layer22_attention_weight_max": 28.70830535888672,
      "activations/layer22_attention_weight_min": -25.94369888305664,
      "activations/layer23_attention_weight_max": 35.011558532714844,
      "activations/layer23_attention_weight_min": -24.4675350189209,
      "activations/layer2_attention_weight_max": 29.83161735534668,
      "activations/layer2_attention_weight_min": -28.904834747314453,
      "activations/layer3_attention_weight_max": 86.17933654785156,
      "activations/layer3_attention_weight_min": -89.00518035888672,
      "activations/layer4_attention_weight_max": 78.67530059814453,
      "activations/layer4_attention_weight_min": -84.88903045654297,
      "activations/layer5_attention_weight_max": 58.17767333984375,
      "activations/layer5_attention_weight_min": -71.93754577636719,
      "activations/layer6_attention_weight_max": 44.27265548706055,
      "activations/layer6_attention_weight_min": -47.53822708129883,
      "activations/layer7_attention_weight_max": 63.62099838256836,
      "activations/layer7_attention_weight_min": -71.8291015625,
      "activations/layer8_attention_weight_max": 43.26932907104492,
      "activations/layer8_attention_weight_min": -47.07261657714844,
      "activations/layer9_attention_weight_max": 41.2254753112793,
      "activations/layer9_attention_weight_min": -44.72137451171875,
      "epoch": 19.74,
      "learning_rate": 2.2925378787878784e-05,
      "loss": 2.7145,
      "step": 339750
    },
    {
      "activations/layer0_attention_weight_max": 15.869483947753906,
      "activations/layer0_attention_weight_min": -13.909485816955566,
      "activations/layer10_attention_weight_max": 34.751121520996094,
      "activations/layer10_attention_weight_min": -35.80633544921875,
      "activations/layer11_attention_weight_max": 32.700782775878906,
      "activations/layer11_attention_weight_min": -33.248252868652344,
      "activations/layer12_attention_weight_max": 19.68373680114746,
      "activations/layer12_attention_weight_min": -27.225202560424805,
      "activations/layer13_attention_weight_max": 32.345436096191406,
      "activations/layer13_attention_weight_min": -30.074604034423828,
      "activations/layer14_attention_weight_max": 35.17298889160156,
      "activations/layer14_attention_weight_min": -31.033493041992188,
      "activations/layer15_attention_weight_max": 33.12894821166992,
      "activations/layer15_attention_weight_min": -31.098905563354492,
      "activations/layer16_attention_weight_max": 33.615089416503906,
      "activations/layer16_attention_weight_min": -33.16612243652344,
      "activations/layer17_attention_weight_max": 50.6988410949707,
      "activations/layer17_attention_weight_min": -44.82948684692383,
      "activations/layer18_attention_weight_max": 45.40742492675781,
      "activations/layer18_attention_weight_min": -39.06904602050781,
      "activations/layer19_attention_weight_max": 23.433551788330078,
      "activations/layer19_attention_weight_min": -22.426450729370117,
      "activations/layer1_attention_weight_max": 15.214943885803223,
      "activations/layer1_attention_weight_min": -14.34057903289795,
      "activations/layer20_attention_weight_max": 24.571765899658203,
      "activations/layer20_attention_weight_min": -22.97260093688965,
      "activations/layer21_attention_weight_max": 32.30412673950195,
      "activations/layer21_attention_weight_min": -22.9462890625,
      "activations/layer22_attention_weight_max": 27.958337783813477,
      "activations/layer22_attention_weight_min": -24.95058822631836,
      "activations/layer23_attention_weight_max": 32.75526428222656,
      "activations/layer23_attention_weight_min": -25.39309310913086,
      "activations/layer2_attention_weight_max": 29.916061401367188,
      "activations/layer2_attention_weight_min": -28.845947265625,
      "activations/layer3_attention_weight_max": 87.40081787109375,
      "activations/layer3_attention_weight_min": -86.2442398071289,
      "activations/layer4_attention_weight_max": 84.43342590332031,
      "activations/layer4_attention_weight_min": -83.54444122314453,
      "activations/layer5_attention_weight_max": 60.367305755615234,
      "activations/layer5_attention_weight_min": -74.68278503417969,
      "activations/layer6_attention_weight_max": 48.734554290771484,
      "activations/layer6_attention_weight_min": -54.343135833740234,
      "activations/layer7_attention_weight_max": 63.473514556884766,
      "activations/layer7_attention_weight_min": -64.97383880615234,
      "activations/layer8_attention_weight_max": 47.53208541870117,
      "activations/layer8_attention_weight_min": -49.8875617980957,
      "activations/layer9_attention_weight_max": 42.9366340637207,
      "activations/layer9_attention_weight_min": -45.773868560791016,
      "epoch": 19.74,
      "learning_rate": 2.2906439393939392e-05,
      "loss": 2.7202,
      "step": 339800
    },
    {
      "activations/layer0_attention_weight_max": 15.486088752746582,
      "activations/layer0_attention_weight_min": -13.982325553894043,
      "activations/layer10_attention_weight_max": 34.30217361450195,
      "activations/layer10_attention_weight_min": -36.00128173828125,
      "activations/layer11_attention_weight_max": 33.90119552612305,
      "activations/layer11_attention_weight_min": -35.29192352294922,
      "activations/layer12_attention_weight_max": 19.05514907836914,
      "activations/layer12_attention_weight_min": -25.31096076965332,
      "activations/layer13_attention_weight_max": 33.62020492553711,
      "activations/layer13_attention_weight_min": -29.8959903717041,
      "activations/layer14_attention_weight_max": 36.12861251831055,
      "activations/layer14_attention_weight_min": -31.353023529052734,
      "activations/layer15_attention_weight_max": 35.101287841796875,
      "activations/layer15_attention_weight_min": -30.48862648010254,
      "activations/layer16_attention_weight_max": 36.6345100402832,
      "activations/layer16_attention_weight_min": -35.84586715698242,
      "activations/layer17_attention_weight_max": 54.69966506958008,
      "activations/layer17_attention_weight_min": -48.77266311645508,
      "activations/layer18_attention_weight_max": 50.97125244140625,
      "activations/layer18_attention_weight_min": -42.14981460571289,
      "activations/layer19_attention_weight_max": 25.795454025268555,
      "activations/layer19_attention_weight_min": -23.595022201538086,
      "activations/layer1_attention_weight_max": 15.235669136047363,
      "activations/layer1_attention_weight_min": -14.325859069824219,
      "activations/layer20_attention_weight_max": 23.364858627319336,
      "activations/layer20_attention_weight_min": -22.175840377807617,
      "activations/layer21_attention_weight_max": 35.739295959472656,
      "activations/layer21_attention_weight_min": -23.57543182373047,
      "activations/layer22_attention_weight_max": 28.93603515625,
      "activations/layer22_attention_weight_min": -25.730838775634766,
      "activations/layer23_attention_weight_max": 38.10163116455078,
      "activations/layer23_attention_weight_min": -24.79237937927246,
      "activations/layer2_attention_weight_max": 33.59551239013672,
      "activations/layer2_attention_weight_min": -30.963043212890625,
      "activations/layer3_attention_weight_max": 86.01224517822266,
      "activations/layer3_attention_weight_min": -86.62152099609375,
      "activations/layer4_attention_weight_max": 84.05052185058594,
      "activations/layer4_attention_weight_min": -82.57615661621094,
      "activations/layer5_attention_weight_max": 67.71284484863281,
      "activations/layer5_attention_weight_min": -75.82100677490234,
      "activations/layer6_attention_weight_max": 49.26772689819336,
      "activations/layer6_attention_weight_min": -51.1815299987793,
      "activations/layer7_attention_weight_max": 66.0042495727539,
      "activations/layer7_attention_weight_min": -67.3538589477539,
      "activations/layer8_attention_weight_max": 45.95033645629883,
      "activations/layer8_attention_weight_min": -49.19731521606445,
      "activations/layer9_attention_weight_max": 44.88180160522461,
      "activations/layer9_attention_weight_min": -48.023128509521484,
      "epoch": 19.75,
      "learning_rate": 2.2887499999999997e-05,
      "loss": 2.7193,
      "step": 339850
    },
    {
      "activations/layer0_attention_weight_max": 16.34235954284668,
      "activations/layer0_attention_weight_min": -14.152143478393555,
      "activations/layer10_attention_weight_max": 40.587852478027344,
      "activations/layer10_attention_weight_min": -40.40936279296875,
      "activations/layer11_attention_weight_max": 36.5859260559082,
      "activations/layer11_attention_weight_min": -37.912967681884766,
      "activations/layer12_attention_weight_max": 21.83269691467285,
      "activations/layer12_attention_weight_min": -27.57219696044922,
      "activations/layer13_attention_weight_max": 36.856956481933594,
      "activations/layer13_attention_weight_min": -28.398151397705078,
      "activations/layer14_attention_weight_max": 38.30183792114258,
      "activations/layer14_attention_weight_min": -30.8592529296875,
      "activations/layer15_attention_weight_max": 38.783226013183594,
      "activations/layer15_attention_weight_min": -30.581117630004883,
      "activations/layer16_attention_weight_max": 38.134159088134766,
      "activations/layer16_attention_weight_min": -35.20351028442383,
      "activations/layer17_attention_weight_max": 58.23483657836914,
      "activations/layer17_attention_weight_min": -49.70159149169922,
      "activations/layer18_attention_weight_max": 52.47193145751953,
      "activations/layer18_attention_weight_min": -42.60495376586914,
      "activations/layer19_attention_weight_max": 25.700103759765625,
      "activations/layer19_attention_weight_min": -24.820470809936523,
      "activations/layer1_attention_weight_max": 15.517380714416504,
      "activations/layer1_attention_weight_min": -14.620989799499512,
      "activations/layer20_attention_weight_max": 24.469959259033203,
      "activations/layer20_attention_weight_min": -21.530925750732422,
      "activations/layer21_attention_weight_max": 39.943660736083984,
      "activations/layer21_attention_weight_min": -23.344804763793945,
      "activations/layer22_attention_weight_max": 31.73653221130371,
      "activations/layer22_attention_weight_min": -25.27803611755371,
      "activations/layer23_attention_weight_max": 39.640602111816406,
      "activations/layer23_attention_weight_min": -22.33895492553711,
      "activations/layer2_attention_weight_max": 30.79492950439453,
      "activations/layer2_attention_weight_min": -29.77118682861328,
      "activations/layer3_attention_weight_max": 87.01336669921875,
      "activations/layer3_attention_weight_min": -84.50067901611328,
      "activations/layer4_attention_weight_max": 84.60763549804688,
      "activations/layer4_attention_weight_min": -83.69445037841797,
      "activations/layer5_attention_weight_max": 64.07273864746094,
      "activations/layer5_attention_weight_min": -71.10602569580078,
      "activations/layer6_attention_weight_max": 48.94707107543945,
      "activations/layer6_attention_weight_min": -50.046363830566406,
      "activations/layer7_attention_weight_max": 65.65804290771484,
      "activations/layer7_attention_weight_min": -66.82220458984375,
      "activations/layer8_attention_weight_max": 47.33358383178711,
      "activations/layer8_attention_weight_min": -52.762332916259766,
      "activations/layer9_attention_weight_max": 49.393123626708984,
      "activations/layer9_attention_weight_min": -52.335609436035156,
      "epoch": 19.75,
      "learning_rate": 2.2868560606060606e-05,
      "loss": 2.7206,
      "step": 339900
    },
    {
      "activations/layer0_attention_weight_max": 15.248234748840332,
      "activations/layer0_attention_weight_min": -14.156637191772461,
      "activations/layer10_attention_weight_max": 39.35731506347656,
      "activations/layer10_attention_weight_min": -36.03941345214844,
      "activations/layer11_attention_weight_max": 33.273704528808594,
      "activations/layer11_attention_weight_min": -36.435874938964844,
      "activations/layer12_attention_weight_max": 19.928668975830078,
      "activations/layer12_attention_weight_min": -31.202241897583008,
      "activations/layer13_attention_weight_max": 35.64850997924805,
      "activations/layer13_attention_weight_min": -26.717327117919922,
      "activations/layer14_attention_weight_max": 37.58816909790039,
      "activations/layer14_attention_weight_min": -30.6710262298584,
      "activations/layer15_attention_weight_max": 34.70785903930664,
      "activations/layer15_attention_weight_min": -30.65312957763672,
      "activations/layer16_attention_weight_max": 34.070289611816406,
      "activations/layer16_attention_weight_min": -31.29838752746582,
      "activations/layer17_attention_weight_max": 52.05704116821289,
      "activations/layer17_attention_weight_min": -47.02532958984375,
      "activations/layer18_attention_weight_max": 54.46154022216797,
      "activations/layer18_attention_weight_min": -39.71722412109375,
      "activations/layer19_attention_weight_max": 26.176050186157227,
      "activations/layer19_attention_weight_min": -23.697425842285156,
      "activations/layer1_attention_weight_max": 15.771230697631836,
      "activations/layer1_attention_weight_min": -14.450583457946777,
      "activations/layer20_attention_weight_max": 27.52374839782715,
      "activations/layer20_attention_weight_min": -22.48192596435547,
      "activations/layer21_attention_weight_max": 44.47259521484375,
      "activations/layer21_attention_weight_min": -25.827545166015625,
      "activations/layer22_attention_weight_max": 31.782896041870117,
      "activations/layer22_attention_weight_min": -26.07845687866211,
      "activations/layer23_attention_weight_max": 39.247745513916016,
      "activations/layer23_attention_weight_min": -23.807788848876953,
      "activations/layer2_attention_weight_max": 32.189239501953125,
      "activations/layer2_attention_weight_min": -29.79200553894043,
      "activations/layer3_attention_weight_max": 93.73168182373047,
      "activations/layer3_attention_weight_min": -92.61183166503906,
      "activations/layer4_attention_weight_max": 89.62114715576172,
      "activations/layer4_attention_weight_min": -82.76168823242188,
      "activations/layer5_attention_weight_max": 62.91035079956055,
      "activations/layer5_attention_weight_min": -72.50444793701172,
      "activations/layer6_attention_weight_max": 52.153812408447266,
      "activations/layer6_attention_weight_min": -50.09162521362305,
      "activations/layer7_attention_weight_max": 66.60404205322266,
      "activations/layer7_attention_weight_min": -67.47129821777344,
      "activations/layer8_attention_weight_max": 51.35338592529297,
      "activations/layer8_attention_weight_min": -53.37153625488281,
      "activations/layer9_attention_weight_max": 45.71802520751953,
      "activations/layer9_attention_weight_min": -46.75940704345703,
      "epoch": 19.75,
      "learning_rate": 2.2849621212121214e-05,
      "loss": 2.7131,
      "step": 339950
    },
    {
      "activations/layer0_attention_weight_max": 15.865726470947266,
      "activations/layer0_attention_weight_min": -14.055015563964844,
      "activations/layer10_attention_weight_max": 35.509613037109375,
      "activations/layer10_attention_weight_min": -33.99589920043945,
      "activations/layer11_attention_weight_max": 32.30772399902344,
      "activations/layer11_attention_weight_min": -33.33735275268555,
      "activations/layer12_attention_weight_max": 19.82448387145996,
      "activations/layer12_attention_weight_min": -25.504228591918945,
      "activations/layer13_attention_weight_max": 40.60067367553711,
      "activations/layer13_attention_weight_min": -29.123859405517578,
      "activations/layer14_attention_weight_max": 38.299652099609375,
      "activations/layer14_attention_weight_min": -31.92618751525879,
      "activations/layer15_attention_weight_max": 34.25263595581055,
      "activations/layer15_attention_weight_min": -30.449098587036133,
      "activations/layer16_attention_weight_max": 33.52456283569336,
      "activations/layer16_attention_weight_min": -31.38060188293457,
      "activations/layer17_attention_weight_max": 50.13383483886719,
      "activations/layer17_attention_weight_min": -47.924320220947266,
      "activations/layer18_attention_weight_max": 46.01251220703125,
      "activations/layer18_attention_weight_min": -44.522037506103516,
      "activations/layer19_attention_weight_max": 21.504337310791016,
      "activations/layer19_attention_weight_min": -24.644033432006836,
      "activations/layer1_attention_weight_max": 15.909530639648438,
      "activations/layer1_attention_weight_min": -14.516374588012695,
      "activations/layer20_attention_weight_max": 21.910879135131836,
      "activations/layer20_attention_weight_min": -23.920320510864258,
      "activations/layer21_attention_weight_max": 38.24641418457031,
      "activations/layer21_attention_weight_min": -22.331212997436523,
      "activations/layer22_attention_weight_max": 26.41484832763672,
      "activations/layer22_attention_weight_min": -25.69722557067871,
      "activations/layer23_attention_weight_max": 32.216644287109375,
      "activations/layer23_attention_weight_min": -24.649051666259766,
      "activations/layer2_attention_weight_max": 30.58934783935547,
      "activations/layer2_attention_weight_min": -28.781414031982422,
      "activations/layer3_attention_weight_max": 86.46212005615234,
      "activations/layer3_attention_weight_min": -85.8251724243164,
      "activations/layer4_attention_weight_max": 84.73346710205078,
      "activations/layer4_attention_weight_min": -83.37568664550781,
      "activations/layer5_attention_weight_max": 61.651763916015625,
      "activations/layer5_attention_weight_min": -70.7813949584961,
      "activations/layer6_attention_weight_max": 48.912818908691406,
      "activations/layer6_attention_weight_min": -50.94103240966797,
      "activations/layer7_attention_weight_max": 64.62418365478516,
      "activations/layer7_attention_weight_min": -60.77619552612305,
      "activations/layer8_attention_weight_max": 46.1300048828125,
      "activations/layer8_attention_weight_min": -47.90818786621094,
      "activations/layer9_attention_weight_max": 42.3491096496582,
      "activations/layer9_attention_weight_min": -44.02363204956055,
      "epoch": 19.76,
      "learning_rate": 2.2830681818181815e-05,
      "loss": 2.7275,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.5197,
      "eval_samples_per_second": 504.01,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.5197,
      "eval_openwebtext_samples_per_second": 504.01,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0017,
      "eval_wikitext_samples_per_second": 227.801,
      "step": 340000
    },
    {
      "epoch": 19.76,
      "eval_lambada_loss": 2.404296875,
      "eval_lambada_ppl": 11.070643498756118,
      "eval_lambada_runtime": 9.5952,
      "eval_lambada_samples_per_second": 507.439,
      "step": 340000
    },
    {
      "activations/layer0_attention_weight_max": 15.203157424926758,
      "activations/layer0_attention_weight_min": -14.209957122802734,
      "activations/layer10_attention_weight_max": 40.15758514404297,
      "activations/layer10_attention_weight_min": -38.2731819152832,
      "activations/layer11_attention_weight_max": 37.61106872558594,
      "activations/layer11_attention_weight_min": -35.18482971191406,
      "activations/layer12_attention_weight_max": 19.332706451416016,
      "activations/layer12_attention_weight_min": -25.435230255126953,
      "activations/layer13_attention_weight_max": 41.47379684448242,
      "activations/layer13_attention_weight_min": -28.652305603027344,
      "activations/layer14_attention_weight_max": 39.6700325012207,
      "activations/layer14_attention_weight_min": -32.88811111450195,
      "activations/layer15_attention_weight_max": 36.69938659667969,
      "activations/layer15_attention_weight_min": -32.84947204589844,
      "activations/layer16_attention_weight_max": 39.219173431396484,
      "activations/layer16_attention_weight_min": -35.18025207519531,
      "activations/layer17_attention_weight_max": 57.37146759033203,
      "activations/layer17_attention_weight_min": -49.76471710205078,
      "activations/layer18_attention_weight_max": 54.21790313720703,
      "activations/layer18_attention_weight_min": -42.84319305419922,
      "activations/layer19_attention_weight_max": 27.2469539642334,
      "activations/layer19_attention_weight_min": -23.487123489379883,
      "activations/layer1_attention_weight_max": 15.135623931884766,
      "activations/layer1_attention_weight_min": -14.983492851257324,
      "activations/layer20_attention_weight_max": 24.918926239013672,
      "activations/layer20_attention_weight_min": -21.956918716430664,
      "activations/layer21_attention_weight_max": 41.75907516479492,
      "activations/layer21_attention_weight_min": -22.48435401916504,
      "activations/layer22_attention_weight_max": 34.0579719543457,
      "activations/layer22_attention_weight_min": -27.26422691345215,
      "activations/layer23_attention_weight_max": 41.81414031982422,
      "activations/layer23_attention_weight_min": -23.57882308959961,
      "activations/layer2_attention_weight_max": 32.53962707519531,
      "activations/layer2_attention_weight_min": -30.866806030273438,
      "activations/layer3_attention_weight_max": 88.09947204589844,
      "activations/layer3_attention_weight_min": -84.47360229492188,
      "activations/layer4_attention_weight_max": 83.1484603881836,
      "activations/layer4_attention_weight_min": -83.4476089477539,
      "activations/layer5_attention_weight_max": 62.69306564331055,
      "activations/layer5_attention_weight_min": -68.52969360351562,
      "activations/layer6_attention_weight_max": 50.681190490722656,
      "activations/layer6_attention_weight_min": -51.11735916137695,
      "activations/layer7_attention_weight_max": 68.32443237304688,
      "activations/layer7_attention_weight_min": -63.67978286743164,
      "activations/layer8_attention_weight_max": 48.74653625488281,
      "activations/layer8_attention_weight_min": -53.08039855957031,
      "activations/layer9_attention_weight_max": 47.169189453125,
      "activations/layer9_attention_weight_min": -47.718719482421875,
      "epoch": 19.76,
      "learning_rate": 2.2811742424242424e-05,
      "loss": 2.7135,
      "step": 340050
    },
    {
      "activations/layer0_attention_weight_max": 15.484573364257812,
      "activations/layer0_attention_weight_min": -14.051656723022461,
      "activations/layer10_attention_weight_max": 35.50441360473633,
      "activations/layer10_attention_weight_min": -37.2775764465332,
      "activations/layer11_attention_weight_max": 32.56401824951172,
      "activations/layer11_attention_weight_min": -33.46372604370117,
      "activations/layer12_attention_weight_max": 19.89909553527832,
      "activations/layer12_attention_weight_min": -25.375802993774414,
      "activations/layer13_attention_weight_max": 37.072715759277344,
      "activations/layer13_attention_weight_min": -26.186819076538086,
      "activations/layer14_attention_weight_max": 40.355674743652344,
      "activations/layer14_attention_weight_min": -30.579120635986328,
      "activations/layer15_attention_weight_max": 37.56504440307617,
      "activations/layer15_attention_weight_min": -30.801164627075195,
      "activations/layer16_attention_weight_max": 34.37858581542969,
      "activations/layer16_attention_weight_min": -30.79667091369629,
      "activations/layer17_attention_weight_max": 50.451419830322266,
      "activations/layer17_attention_weight_min": -44.74113845825195,
      "activations/layer18_attention_weight_max": 47.99469757080078,
      "activations/layer18_attention_weight_min": -38.41048049926758,
      "activations/layer19_attention_weight_max": 25.888742446899414,
      "activations/layer19_attention_weight_min": -22.53466033935547,
      "activations/layer1_attention_weight_max": 16.32175064086914,
      "activations/layer1_attention_weight_min": -14.546929359436035,
      "activations/layer20_attention_weight_max": 23.384309768676758,
      "activations/layer20_attention_weight_min": -19.494190216064453,
      "activations/layer21_attention_weight_max": 34.72168731689453,
      "activations/layer21_attention_weight_min": -20.241912841796875,
      "activations/layer22_attention_weight_max": 28.776351928710938,
      "activations/layer22_attention_weight_min": -24.321369171142578,
      "activations/layer23_attention_weight_max": 36.18345642089844,
      "activations/layer23_attention_weight_min": -23.48297119140625,
      "activations/layer2_attention_weight_max": 29.373586654663086,
      "activations/layer2_attention_weight_min": -30.24932861328125,
      "activations/layer3_attention_weight_max": 81.73899841308594,
      "activations/layer3_attention_weight_min": -86.51275634765625,
      "activations/layer4_attention_weight_max": 79.08415222167969,
      "activations/layer4_attention_weight_min": -82.05361938476562,
      "activations/layer5_attention_weight_max": 62.85392379760742,
      "activations/layer5_attention_weight_min": -69.28866577148438,
      "activations/layer6_attention_weight_max": 47.60506057739258,
      "activations/layer6_attention_weight_min": -48.035587310791016,
      "activations/layer7_attention_weight_max": 63.32503128051758,
      "activations/layer7_attention_weight_min": -63.19929122924805,
      "activations/layer8_attention_weight_max": 44.871463775634766,
      "activations/layer8_attention_weight_min": -49.10666275024414,
      "activations/layer9_attention_weight_max": 43.20759963989258,
      "activations/layer9_attention_weight_min": -44.28644943237305,
      "epoch": 19.76,
      "learning_rate": 2.279280303030303e-05,
      "loss": 2.7122,
      "step": 340100
    },
    {
      "activations/layer0_attention_weight_max": 14.745433807373047,
      "activations/layer0_attention_weight_min": -13.855094909667969,
      "activations/layer10_attention_weight_max": 36.6732063293457,
      "activations/layer10_attention_weight_min": -36.88641357421875,
      "activations/layer11_attention_weight_max": 33.99876403808594,
      "activations/layer11_attention_weight_min": -35.108154296875,
      "activations/layer12_attention_weight_max": 18.191818237304688,
      "activations/layer12_attention_weight_min": -25.562158584594727,
      "activations/layer13_attention_weight_max": 35.300289154052734,
      "activations/layer13_attention_weight_min": -28.1037654876709,
      "activations/layer14_attention_weight_max": 35.024810791015625,
      "activations/layer14_attention_weight_min": -31.647682189941406,
      "activations/layer15_attention_weight_max": 33.441810607910156,
      "activations/layer15_attention_weight_min": -31.534934997558594,
      "activations/layer16_attention_weight_max": 36.71588897705078,
      "activations/layer16_attention_weight_min": -35.15598678588867,
      "activations/layer17_attention_weight_max": 54.96503448486328,
      "activations/layer17_attention_weight_min": -48.951576232910156,
      "activations/layer18_attention_weight_max": 51.1488151550293,
      "activations/layer18_attention_weight_min": -47.1119270324707,
      "activations/layer19_attention_weight_max": 25.330045700073242,
      "activations/layer19_attention_weight_min": -24.15349769592285,
      "activations/layer1_attention_weight_max": 16.21333122253418,
      "activations/layer1_attention_weight_min": -14.789664268493652,
      "activations/layer20_attention_weight_max": 22.575801849365234,
      "activations/layer20_attention_weight_min": -23.263280868530273,
      "activations/layer21_attention_weight_max": 38.14227294921875,
      "activations/layer21_attention_weight_min": -23.165197372436523,
      "activations/layer22_attention_weight_max": 30.111223220825195,
      "activations/layer22_attention_weight_min": -29.721094131469727,
      "activations/layer23_attention_weight_max": 36.33501052856445,
      "activations/layer23_attention_weight_min": -27.552879333496094,
      "activations/layer2_attention_weight_max": 30.084033966064453,
      "activations/layer2_attention_weight_min": -28.566429138183594,
      "activations/layer3_attention_weight_max": 84.30486297607422,
      "activations/layer3_attention_weight_min": -85.59835815429688,
      "activations/layer4_attention_weight_max": 80.29024505615234,
      "activations/layer4_attention_weight_min": -80.36991882324219,
      "activations/layer5_attention_weight_max": 61.142547607421875,
      "activations/layer5_attention_weight_min": -71.04632568359375,
      "activations/layer6_attention_weight_max": 46.62501907348633,
      "activations/layer6_attention_weight_min": -49.18903350830078,
      "activations/layer7_attention_weight_max": 65.1110610961914,
      "activations/layer7_attention_weight_min": -61.2458610534668,
      "activations/layer8_attention_weight_max": 44.613922119140625,
      "activations/layer8_attention_weight_min": -49.26342010498047,
      "activations/layer9_attention_weight_max": 41.237491607666016,
      "activations/layer9_attention_weight_min": -47.718196868896484,
      "epoch": 19.76,
      "learning_rate": 2.2773863636363637e-05,
      "loss": 2.733,
      "step": 340150
    },
    {
      "activations/layer0_attention_weight_max": 15.238839149475098,
      "activations/layer0_attention_weight_min": -13.979575157165527,
      "activations/layer10_attention_weight_max": 37.45427703857422,
      "activations/layer10_attention_weight_min": -38.796112060546875,
      "activations/layer11_attention_weight_max": 36.628902435302734,
      "activations/layer11_attention_weight_min": -36.02819061279297,
      "activations/layer12_attention_weight_max": 20.167598724365234,
      "activations/layer12_attention_weight_min": -24.791208267211914,
      "activations/layer13_attention_weight_max": 34.53758239746094,
      "activations/layer13_attention_weight_min": -28.446537017822266,
      "activations/layer14_attention_weight_max": 36.18205642700195,
      "activations/layer14_attention_weight_min": -32.40453338623047,
      "activations/layer15_attention_weight_max": 34.909732818603516,
      "activations/layer15_attention_weight_min": -30.737884521484375,
      "activations/layer16_attention_weight_max": 37.759456634521484,
      "activations/layer16_attention_weight_min": -32.39701843261719,
      "activations/layer17_attention_weight_max": 51.52879333496094,
      "activations/layer17_attention_weight_min": -45.840396881103516,
      "activations/layer18_attention_weight_max": 47.40859603881836,
      "activations/layer18_attention_weight_min": -39.5921745300293,
      "activations/layer19_attention_weight_max": 26.823049545288086,
      "activations/layer19_attention_weight_min": -22.45271873474121,
      "activations/layer1_attention_weight_max": 15.40684700012207,
      "activations/layer1_attention_weight_min": -14.005057334899902,
      "activations/layer20_attention_weight_max": 25.009082794189453,
      "activations/layer20_attention_weight_min": -20.135801315307617,
      "activations/layer21_attention_weight_max": 43.17981719970703,
      "activations/layer21_attention_weight_min": -21.354412078857422,
      "activations/layer22_attention_weight_max": 31.206539154052734,
      "activations/layer22_attention_weight_min": -24.779340744018555,
      "activations/layer23_attention_weight_max": 38.09174346923828,
      "activations/layer23_attention_weight_min": -24.057376861572266,
      "activations/layer2_attention_weight_max": 30.964874267578125,
      "activations/layer2_attention_weight_min": -29.745389938354492,
      "activations/layer3_attention_weight_max": 84.43408966064453,
      "activations/layer3_attention_weight_min": -86.02141571044922,
      "activations/layer4_attention_weight_max": 85.21007537841797,
      "activations/layer4_attention_weight_min": -80.80095672607422,
      "activations/layer5_attention_weight_max": 70.21464538574219,
      "activations/layer5_attention_weight_min": -76.33318328857422,
      "activations/layer6_attention_weight_max": 50.58998107910156,
      "activations/layer6_attention_weight_min": -51.03595733642578,
      "activations/layer7_attention_weight_max": 64.38034057617188,
      "activations/layer7_attention_weight_min": -64.0805435180664,
      "activations/layer8_attention_weight_max": 51.62427520751953,
      "activations/layer8_attention_weight_min": -53.454811096191406,
      "activations/layer9_attention_weight_max": 46.82462692260742,
      "activations/layer9_attention_weight_min": -47.96162414550781,
      "epoch": 19.77,
      "learning_rate": 2.2754924242424238e-05,
      "loss": 2.7294,
      "step": 340200
    },
    {
      "activations/layer0_attention_weight_max": 15.817268371582031,
      "activations/layer0_attention_weight_min": -13.900115966796875,
      "activations/layer10_attention_weight_max": 37.12112045288086,
      "activations/layer10_attention_weight_min": -36.337425231933594,
      "activations/layer11_attention_weight_max": 38.55593490600586,
      "activations/layer11_attention_weight_min": -36.23557662963867,
      "activations/layer12_attention_weight_max": 18.626802444458008,
      "activations/layer12_attention_weight_min": -34.46064758300781,
      "activations/layer13_attention_weight_max": 42.017940521240234,
      "activations/layer13_attention_weight_min": -31.0329647064209,
      "activations/layer14_attention_weight_max": 37.62362289428711,
      "activations/layer14_attention_weight_min": -30.283565521240234,
      "activations/layer15_attention_weight_max": 38.177642822265625,
      "activations/layer15_attention_weight_min": -32.059730529785156,
      "activations/layer16_attention_weight_max": 37.60816955566406,
      "activations/layer16_attention_weight_min": -33.898231506347656,
      "activations/layer17_attention_weight_max": 57.14040756225586,
      "activations/layer17_attention_weight_min": -46.49754333496094,
      "activations/layer18_attention_weight_max": 52.33857727050781,
      "activations/layer18_attention_weight_min": -41.64958190917969,
      "activations/layer19_attention_weight_max": 27.745908737182617,
      "activations/layer19_attention_weight_min": -21.708139419555664,
      "activations/layer1_attention_weight_max": 15.81417465209961,
      "activations/layer1_attention_weight_min": -14.063848495483398,
      "activations/layer20_attention_weight_max": 25.506380081176758,
      "activations/layer20_attention_weight_min": -21.39322853088379,
      "activations/layer21_attention_weight_max": 42.839298248291016,
      "activations/layer21_attention_weight_min": -22.15631103515625,
      "activations/layer22_attention_weight_max": 32.181427001953125,
      "activations/layer22_attention_weight_min": -25.856048583984375,
      "activations/layer23_attention_weight_max": 35.957786560058594,
      "activations/layer23_attention_weight_min": -23.148435592651367,
      "activations/layer2_attention_weight_max": 34.81119155883789,
      "activations/layer2_attention_weight_min": -32.35260772705078,
      "activations/layer3_attention_weight_max": 95.68746948242188,
      "activations/layer3_attention_weight_min": -93.50576782226562,
      "activations/layer4_attention_weight_max": 87.2553939819336,
      "activations/layer4_attention_weight_min": -85.67720794677734,
      "activations/layer5_attention_weight_max": 64.580322265625,
      "activations/layer5_attention_weight_min": -70.47130584716797,
      "activations/layer6_attention_weight_max": 50.618431091308594,
      "activations/layer6_attention_weight_min": -51.70832061767578,
      "activations/layer7_attention_weight_max": 66.1052017211914,
      "activations/layer7_attention_weight_min": -64.13636016845703,
      "activations/layer8_attention_weight_max": 48.8309440612793,
      "activations/layer8_attention_weight_min": -49.17741775512695,
      "activations/layer9_attention_weight_max": 47.93307113647461,
      "activations/layer9_attention_weight_min": -46.26939010620117,
      "epoch": 19.77,
      "learning_rate": 2.2735984848484847e-05,
      "loss": 2.7334,
      "step": 340250
    },
    {
      "activations/layer0_attention_weight_max": 14.904656410217285,
      "activations/layer0_attention_weight_min": -14.06661605834961,
      "activations/layer10_attention_weight_max": 34.926918029785156,
      "activations/layer10_attention_weight_min": -35.948726654052734,
      "activations/layer11_attention_weight_max": 32.86951446533203,
      "activations/layer11_attention_weight_min": -37.4727897644043,
      "activations/layer12_attention_weight_max": 18.09264373779297,
      "activations/layer12_attention_weight_min": -24.2457332611084,
      "activations/layer13_attention_weight_max": 32.8734245300293,
      "activations/layer13_attention_weight_min": -26.116289138793945,
      "activations/layer14_attention_weight_max": 34.01970672607422,
      "activations/layer14_attention_weight_min": -27.817020416259766,
      "activations/layer15_attention_weight_max": 31.64487648010254,
      "activations/layer15_attention_weight_min": -29.10104751586914,
      "activations/layer16_attention_weight_max": 33.886497497558594,
      "activations/layer16_attention_weight_min": -31.95404815673828,
      "activations/layer17_attention_weight_max": 49.964534759521484,
      "activations/layer17_attention_weight_min": -45.35947036743164,
      "activations/layer18_attention_weight_max": 44.428409576416016,
      "activations/layer18_attention_weight_min": -41.079002380371094,
      "activations/layer19_attention_weight_max": 24.31976890563965,
      "activations/layer19_attention_weight_min": -22.93062400817871,
      "activations/layer1_attention_weight_max": 15.108041763305664,
      "activations/layer1_attention_weight_min": -13.642756462097168,
      "activations/layer20_attention_weight_max": 22.38383674621582,
      "activations/layer20_attention_weight_min": -21.122190475463867,
      "activations/layer21_attention_weight_max": 36.241573333740234,
      "activations/layer21_attention_weight_min": -20.881343841552734,
      "activations/layer22_attention_weight_max": 28.800678253173828,
      "activations/layer22_attention_weight_min": -26.795665740966797,
      "activations/layer23_attention_weight_max": 38.514991760253906,
      "activations/layer23_attention_weight_min": -25.486974716186523,
      "activations/layer2_attention_weight_max": 31.68389892578125,
      "activations/layer2_attention_weight_min": -32.59122085571289,
      "activations/layer3_attention_weight_max": 89.85596466064453,
      "activations/layer3_attention_weight_min": -90.36157989501953,
      "activations/layer4_attention_weight_max": 83.44564056396484,
      "activations/layer4_attention_weight_min": -81.09536743164062,
      "activations/layer5_attention_weight_max": 61.03887176513672,
      "activations/layer5_attention_weight_min": -66.03721618652344,
      "activations/layer6_attention_weight_max": 48.025474548339844,
      "activations/layer6_attention_weight_min": -48.10098648071289,
      "activations/layer7_attention_weight_max": 64.12649536132812,
      "activations/layer7_attention_weight_min": -61.57146072387695,
      "activations/layer8_attention_weight_max": 47.510196685791016,
      "activations/layer8_attention_weight_min": -49.00899124145508,
      "activations/layer9_attention_weight_max": 42.25254821777344,
      "activations/layer9_attention_weight_min": -47.10311508178711,
      "epoch": 19.77,
      "learning_rate": 2.271704545454545e-05,
      "loss": 2.7176,
      "step": 340300
    },
    {
      "activations/layer0_attention_weight_max": 15.757028579711914,
      "activations/layer0_attention_weight_min": -14.359171867370605,
      "activations/layer10_attention_weight_max": 33.102195739746094,
      "activations/layer10_attention_weight_min": -33.50741958618164,
      "activations/layer11_attention_weight_max": 29.507495880126953,
      "activations/layer11_attention_weight_min": -32.678932189941406,
      "activations/layer12_attention_weight_max": 18.704105377197266,
      "activations/layer12_attention_weight_min": -24.609970092773438,
      "activations/layer13_attention_weight_max": 36.18585968017578,
      "activations/layer13_attention_weight_min": -26.946300506591797,
      "activations/layer14_attention_weight_max": 37.47150421142578,
      "activations/layer14_attention_weight_min": -28.915849685668945,
      "activations/layer15_attention_weight_max": 34.58448028564453,
      "activations/layer15_attention_weight_min": -29.30743980407715,
      "activations/layer16_attention_weight_max": 36.141387939453125,
      "activations/layer16_attention_weight_min": -32.01634979248047,
      "activations/layer17_attention_weight_max": 51.602256774902344,
      "activations/layer17_attention_weight_min": -43.051856994628906,
      "activations/layer18_attention_weight_max": 49.86378479003906,
      "activations/layer18_attention_weight_min": -38.44917678833008,
      "activations/layer19_attention_weight_max": 26.742694854736328,
      "activations/layer19_attention_weight_min": -23.2684383392334,
      "activations/layer1_attention_weight_max": 15.70937442779541,
      "activations/layer1_attention_weight_min": -14.89685344696045,
      "activations/layer20_attention_weight_max": 24.41276741027832,
      "activations/layer20_attention_weight_min": -21.859556198120117,
      "activations/layer21_attention_weight_max": 38.854644775390625,
      "activations/layer21_attention_weight_min": -21.26072120666504,
      "activations/layer22_attention_weight_max": 29.405426025390625,
      "activations/layer22_attention_weight_min": -24.202606201171875,
      "activations/layer23_attention_weight_max": 36.44934844970703,
      "activations/layer23_attention_weight_min": -23.48880386352539,
      "activations/layer2_attention_weight_max": 32.52299499511719,
      "activations/layer2_attention_weight_min": -30.572011947631836,
      "activations/layer3_attention_weight_max": 85.88436889648438,
      "activations/layer3_attention_weight_min": -87.71012115478516,
      "activations/layer4_attention_weight_max": 78.5547866821289,
      "activations/layer4_attention_weight_min": -81.74121856689453,
      "activations/layer5_attention_weight_max": 59.39472579956055,
      "activations/layer5_attention_weight_min": -69.80390930175781,
      "activations/layer6_attention_weight_max": 45.73429489135742,
      "activations/layer6_attention_weight_min": -49.997215270996094,
      "activations/layer7_attention_weight_max": 59.315711975097656,
      "activations/layer7_attention_weight_min": -60.64413070678711,
      "activations/layer8_attention_weight_max": 46.30792999267578,
      "activations/layer8_attention_weight_min": -47.06053161621094,
      "activations/layer9_attention_weight_max": 40.44569778442383,
      "activations/layer9_attention_weight_min": -43.08560562133789,
      "epoch": 19.78,
      "learning_rate": 2.269810606060606e-05,
      "loss": 2.7283,
      "step": 340350
    },
    {
      "activations/layer0_attention_weight_max": 15.677785873413086,
      "activations/layer0_attention_weight_min": -14.10273551940918,
      "activations/layer10_attention_weight_max": 36.925018310546875,
      "activations/layer10_attention_weight_min": -35.185184478759766,
      "activations/layer11_attention_weight_max": 32.564788818359375,
      "activations/layer11_attention_weight_min": -33.32581329345703,
      "activations/layer12_attention_weight_max": 17.560760498046875,
      "activations/layer12_attention_weight_min": -23.1651668548584,
      "activations/layer13_attention_weight_max": 31.347557067871094,
      "activations/layer13_attention_weight_min": -28.010366439819336,
      "activations/layer14_attention_weight_max": 33.95048904418945,
      "activations/layer14_attention_weight_min": -29.04593276977539,
      "activations/layer15_attention_weight_max": 31.837228775024414,
      "activations/layer15_attention_weight_min": -28.828432083129883,
      "activations/layer16_attention_weight_max": 30.947351455688477,
      "activations/layer16_attention_weight_min": -29.947675704956055,
      "activations/layer17_attention_weight_max": 46.790340423583984,
      "activations/layer17_attention_weight_min": -43.02391052246094,
      "activations/layer18_attention_weight_max": 44.18182373046875,
      "activations/layer18_attention_weight_min": -38.00312042236328,
      "activations/layer19_attention_weight_max": 26.835098266601562,
      "activations/layer19_attention_weight_min": -20.13905143737793,
      "activations/layer1_attention_weight_max": 15.988014221191406,
      "activations/layer1_attention_weight_min": -14.022558212280273,
      "activations/layer20_attention_weight_max": 20.95431137084961,
      "activations/layer20_attention_weight_min": -19.706905364990234,
      "activations/layer21_attention_weight_max": 30.708852767944336,
      "activations/layer21_attention_weight_min": -24.477994918823242,
      "activations/layer22_attention_weight_max": 29.828487396240234,
      "activations/layer22_attention_weight_min": -24.28907585144043,
      "activations/layer23_attention_weight_max": 33.17063903808594,
      "activations/layer23_attention_weight_min": -23.64047622680664,
      "activations/layer2_attention_weight_max": 33.13160705566406,
      "activations/layer2_attention_weight_min": -33.3410758972168,
      "activations/layer3_attention_weight_max": 90.37276458740234,
      "activations/layer3_attention_weight_min": -95.11547088623047,
      "activations/layer4_attention_weight_max": 82.26100158691406,
      "activations/layer4_attention_weight_min": -88.82830047607422,
      "activations/layer5_attention_weight_max": 61.027427673339844,
      "activations/layer5_attention_weight_min": -69.38996887207031,
      "activations/layer6_attention_weight_max": 48.80720901489258,
      "activations/layer6_attention_weight_min": -52.415828704833984,
      "activations/layer7_attention_weight_max": 63.81709289550781,
      "activations/layer7_attention_weight_min": -61.8164176940918,
      "activations/layer8_attention_weight_max": 45.702301025390625,
      "activations/layer8_attention_weight_min": -46.48052978515625,
      "activations/layer9_attention_weight_max": 42.761417388916016,
      "activations/layer9_attention_weight_min": -46.478668212890625,
      "epoch": 19.78,
      "learning_rate": 2.2679166666666665e-05,
      "loss": 2.7092,
      "step": 340400
    },
    {
      "activations/layer0_attention_weight_max": 14.556747436523438,
      "activations/layer0_attention_weight_min": -14.061640739440918,
      "activations/layer10_attention_weight_max": 38.740623474121094,
      "activations/layer10_attention_weight_min": -37.885555267333984,
      "activations/layer11_attention_weight_max": 36.607505798339844,
      "activations/layer11_attention_weight_min": -35.67912292480469,
      "activations/layer12_attention_weight_max": 21.04183006286621,
      "activations/layer12_attention_weight_min": -26.540544509887695,
      "activations/layer13_attention_weight_max": 34.764400482177734,
      "activations/layer13_attention_weight_min": -29.156646728515625,
      "activations/layer14_attention_weight_max": 40.82683563232422,
      "activations/layer14_attention_weight_min": -32.24589157104492,
      "activations/layer15_attention_weight_max": 38.49005889892578,
      "activations/layer15_attention_weight_min": -32.801143646240234,
      "activations/layer16_attention_weight_max": 39.570411682128906,
      "activations/layer16_attention_weight_min": -36.01779556274414,
      "activations/layer17_attention_weight_max": 57.59197235107422,
      "activations/layer17_attention_weight_min": -47.23660659790039,
      "activations/layer18_attention_weight_max": 46.81542205810547,
      "activations/layer18_attention_weight_min": -42.45879364013672,
      "activations/layer19_attention_weight_max": 23.769094467163086,
      "activations/layer19_attention_weight_min": -25.233495712280273,
      "activations/layer1_attention_weight_max": 15.077592849731445,
      "activations/layer1_attention_weight_min": -14.267134666442871,
      "activations/layer20_attention_weight_max": 21.75609016418457,
      "activations/layer20_attention_weight_min": -23.294170379638672,
      "activations/layer21_attention_weight_max": 34.61797332763672,
      "activations/layer21_attention_weight_min": -22.051847457885742,
      "activations/layer22_attention_weight_max": 28.563955307006836,
      "activations/layer22_attention_weight_min": -26.13097381591797,
      "activations/layer23_attention_weight_max": 37.72906494140625,
      "activations/layer23_attention_weight_min": -27.253936767578125,
      "activations/layer2_attention_weight_max": 31.520732879638672,
      "activations/layer2_attention_weight_min": -29.872913360595703,
      "activations/layer3_attention_weight_max": 84.19129943847656,
      "activations/layer3_attention_weight_min": -89.02677917480469,
      "activations/layer4_attention_weight_max": 81.49730682373047,
      "activations/layer4_attention_weight_min": -82.77037811279297,
      "activations/layer5_attention_weight_max": 64.21958923339844,
      "activations/layer5_attention_weight_min": -72.11880493164062,
      "activations/layer6_attention_weight_max": 48.41511917114258,
      "activations/layer6_attention_weight_min": -49.92106246948242,
      "activations/layer7_attention_weight_max": 65.26864624023438,
      "activations/layer7_attention_weight_min": -61.21622848510742,
      "activations/layer8_attention_weight_max": 47.740478515625,
      "activations/layer8_attention_weight_min": -51.82787322998047,
      "activations/layer9_attention_weight_max": 45.65028762817383,
      "activations/layer9_attention_weight_min": -48.668060302734375,
      "epoch": 19.78,
      "learning_rate": 2.2660227272727273e-05,
      "loss": 2.7384,
      "step": 340450
    },
    {
      "activations/layer0_attention_weight_max": 15.711507797241211,
      "activations/layer0_attention_weight_min": -14.130402565002441,
      "activations/layer10_attention_weight_max": 35.18879318237305,
      "activations/layer10_attention_weight_min": -34.17724609375,
      "activations/layer11_attention_weight_max": 31.46242332458496,
      "activations/layer11_attention_weight_min": -32.44026184082031,
      "activations/layer12_attention_weight_max": 19.070112228393555,
      "activations/layer12_attention_weight_min": -24.696128845214844,
      "activations/layer13_attention_weight_max": 31.301347732543945,
      "activations/layer13_attention_weight_min": -27.81241798400879,
      "activations/layer14_attention_weight_max": 34.19916915893555,
      "activations/layer14_attention_weight_min": -29.167688369750977,
      "activations/layer15_attention_weight_max": 31.608640670776367,
      "activations/layer15_attention_weight_min": -31.880165100097656,
      "activations/layer16_attention_weight_max": 33.15708541870117,
      "activations/layer16_attention_weight_min": -31.996414184570312,
      "activations/layer17_attention_weight_max": 50.93500900268555,
      "activations/layer17_attention_weight_min": -44.5129508972168,
      "activations/layer18_attention_weight_max": 44.4427604675293,
      "activations/layer18_attention_weight_min": -37.63744354248047,
      "activations/layer19_attention_weight_max": 23.353025436401367,
      "activations/layer19_attention_weight_min": -24.756040573120117,
      "activations/layer1_attention_weight_max": 14.768908500671387,
      "activations/layer1_attention_weight_min": -15.43172836303711,
      "activations/layer20_attention_weight_max": 22.83710479736328,
      "activations/layer20_attention_weight_min": -21.885677337646484,
      "activations/layer21_attention_weight_max": 33.60894012451172,
      "activations/layer21_attention_weight_min": -20.095062255859375,
      "activations/layer22_attention_weight_max": 26.9070987701416,
      "activations/layer22_attention_weight_min": -25.46364974975586,
      "activations/layer23_attention_weight_max": 37.44463348388672,
      "activations/layer23_attention_weight_min": -25.128528594970703,
      "activations/layer2_attention_weight_max": 31.971847534179688,
      "activations/layer2_attention_weight_min": -30.437198638916016,
      "activations/layer3_attention_weight_max": 85.51002502441406,
      "activations/layer3_attention_weight_min": -87.92098999023438,
      "activations/layer4_attention_weight_max": 82.05792999267578,
      "activations/layer4_attention_weight_min": -84.63397216796875,
      "activations/layer5_attention_weight_max": 61.638710021972656,
      "activations/layer5_attention_weight_min": -69.66300964355469,
      "activations/layer6_attention_weight_max": 50.71977996826172,
      "activations/layer6_attention_weight_min": -49.24336624145508,
      "activations/layer7_attention_weight_max": 65.41438293457031,
      "activations/layer7_attention_weight_min": -62.21665573120117,
      "activations/layer8_attention_weight_max": 47.10296630859375,
      "activations/layer8_attention_weight_min": -48.731040954589844,
      "activations/layer9_attention_weight_max": 43.5462760925293,
      "activations/layer9_attention_weight_min": -45.78657150268555,
      "epoch": 19.78,
      "learning_rate": 2.2641287878787874e-05,
      "loss": 2.7166,
      "step": 340500
    },
    {
      "activations/layer0_attention_weight_max": 15.698609352111816,
      "activations/layer0_attention_weight_min": -14.218381881713867,
      "activations/layer10_attention_weight_max": 31.821529388427734,
      "activations/layer10_attention_weight_min": -33.715606689453125,
      "activations/layer11_attention_weight_max": 30.18911361694336,
      "activations/layer11_attention_weight_min": -34.07253646850586,
      "activations/layer12_attention_weight_max": 19.5115909576416,
      "activations/layer12_attention_weight_min": -23.995609283447266,
      "activations/layer13_attention_weight_max": 37.924278259277344,
      "activations/layer13_attention_weight_min": -27.111661911010742,
      "activations/layer14_attention_weight_max": 38.031959533691406,
      "activations/layer14_attention_weight_min": -28.75873565673828,
      "activations/layer15_attention_weight_max": 36.47841262817383,
      "activations/layer15_attention_weight_min": -29.753021240234375,
      "activations/layer16_attention_weight_max": 36.2159423828125,
      "activations/layer16_attention_weight_min": -32.27022171020508,
      "activations/layer17_attention_weight_max": 54.95283889770508,
      "activations/layer17_attention_weight_min": -48.08472442626953,
      "activations/layer18_attention_weight_max": 50.239933013916016,
      "activations/layer18_attention_weight_min": -42.360618591308594,
      "activations/layer19_attention_weight_max": 24.247737884521484,
      "activations/layer19_attention_weight_min": -24.890005111694336,
      "activations/layer1_attention_weight_max": 14.54096508026123,
      "activations/layer1_attention_weight_min": -13.626998901367188,
      "activations/layer20_attention_weight_max": 23.505435943603516,
      "activations/layer20_attention_weight_min": -27.25865936279297,
      "activations/layer21_attention_weight_max": 34.28980255126953,
      "activations/layer21_attention_weight_min": -26.37360191345215,
      "activations/layer22_attention_weight_max": 29.670806884765625,
      "activations/layer22_attention_weight_min": -26.26314353942871,
      "activations/layer23_attention_weight_max": 36.96349334716797,
      "activations/layer23_attention_weight_min": -24.04283905029297,
      "activations/layer2_attention_weight_max": 33.880767822265625,
      "activations/layer2_attention_weight_min": -32.701438903808594,
      "activations/layer3_attention_weight_max": 90.05156707763672,
      "activations/layer3_attention_weight_min": -88.25936126708984,
      "activations/layer4_attention_weight_max": 83.13819122314453,
      "activations/layer4_attention_weight_min": -79.52255249023438,
      "activations/layer5_attention_weight_max": 61.247459411621094,
      "activations/layer5_attention_weight_min": -68.49794006347656,
      "activations/layer6_attention_weight_max": 49.260005950927734,
      "activations/layer6_attention_weight_min": -48.52744674682617,
      "activations/layer7_attention_weight_max": 63.98169708251953,
      "activations/layer7_attention_weight_min": -60.24373245239258,
      "activations/layer8_attention_weight_max": 44.95079040527344,
      "activations/layer8_attention_weight_min": -47.002376556396484,
      "activations/layer9_attention_weight_max": 42.32318878173828,
      "activations/layer9_attention_weight_min": -43.45818328857422,
      "epoch": 19.79,
      "learning_rate": 2.2622348484848483e-05,
      "loss": 2.7287,
      "step": 340550
    },
    {
      "activations/layer0_attention_weight_max": 14.800821304321289,
      "activations/layer0_attention_weight_min": -14.514150619506836,
      "activations/layer10_attention_weight_max": 32.579830169677734,
      "activations/layer10_attention_weight_min": -33.94953155517578,
      "activations/layer11_attention_weight_max": 30.591793060302734,
      "activations/layer11_attention_weight_min": -34.274417877197266,
      "activations/layer12_attention_weight_max": 18.071584701538086,
      "activations/layer12_attention_weight_min": -25.893033981323242,
      "activations/layer13_attention_weight_max": 42.0788459777832,
      "activations/layer13_attention_weight_min": -33.952877044677734,
      "activations/layer14_attention_weight_max": 43.54438781738281,
      "activations/layer14_attention_weight_min": -33.042205810546875,
      "activations/layer15_attention_weight_max": 44.88294982910156,
      "activations/layer15_attention_weight_min": -30.656068801879883,
      "activations/layer16_attention_weight_max": 35.469810485839844,
      "activations/layer16_attention_weight_min": -33.499942779541016,
      "activations/layer17_attention_weight_max": 52.906864166259766,
      "activations/layer17_attention_weight_min": -48.27257537841797,
      "activations/layer18_attention_weight_max": 48.088706970214844,
      "activations/layer18_attention_weight_min": -47.07915496826172,
      "activations/layer19_attention_weight_max": 26.11379051208496,
      "activations/layer19_attention_weight_min": -24.479692459106445,
      "activations/layer1_attention_weight_max": 14.976719856262207,
      "activations/layer1_attention_weight_min": -13.691803932189941,
      "activations/layer20_attention_weight_max": 27.073150634765625,
      "activations/layer20_attention_weight_min": -21.07051658630371,
      "activations/layer21_attention_weight_max": 43.25530242919922,
      "activations/layer21_attention_weight_min": -22.78168296813965,
      "activations/layer22_attention_weight_max": 28.72412109375,
      "activations/layer22_attention_weight_min": -25.86353302001953,
      "activations/layer23_attention_weight_max": 35.68505859375,
      "activations/layer23_attention_weight_min": -24.99224090576172,
      "activations/layer2_attention_weight_max": 30.688743591308594,
      "activations/layer2_attention_weight_min": -28.778575897216797,
      "activations/layer3_attention_weight_max": 83.04905700683594,
      "activations/layer3_attention_weight_min": -84.00438690185547,
      "activations/layer4_attention_weight_max": 80.10871887207031,
      "activations/layer4_attention_weight_min": -80.41820526123047,
      "activations/layer5_attention_weight_max": 61.834754943847656,
      "activations/layer5_attention_weight_min": -73.62080383300781,
      "activations/layer6_attention_weight_max": 47.72190856933594,
      "activations/layer6_attention_weight_min": -47.16887664794922,
      "activations/layer7_attention_weight_max": 64.171142578125,
      "activations/layer7_attention_weight_min": -62.66149139404297,
      "activations/layer8_attention_weight_max": 42.46553039550781,
      "activations/layer8_attention_weight_min": -46.94647979736328,
      "activations/layer9_attention_weight_max": 43.514827728271484,
      "activations/layer9_attention_weight_min": -42.56235885620117,
      "epoch": 19.79,
      "learning_rate": 2.2603409090909087e-05,
      "loss": 2.7223,
      "step": 340600
    },
    {
      "activations/layer0_attention_weight_max": 15.586162567138672,
      "activations/layer0_attention_weight_min": -14.390706062316895,
      "activations/layer10_attention_weight_max": 33.9345703125,
      "activations/layer10_attention_weight_min": -37.46847152709961,
      "activations/layer11_attention_weight_max": 31.684486389160156,
      "activations/layer11_attention_weight_min": -34.52893829345703,
      "activations/layer12_attention_weight_max": 19.17738151550293,
      "activations/layer12_attention_weight_min": -25.426862716674805,
      "activations/layer13_attention_weight_max": 32.19715118408203,
      "activations/layer13_attention_weight_min": -26.146894454956055,
      "activations/layer14_attention_weight_max": 32.50297927856445,
      "activations/layer14_attention_weight_min": -29.803083419799805,
      "activations/layer15_attention_weight_max": 31.999940872192383,
      "activations/layer15_attention_weight_min": -32.565486907958984,
      "activations/layer16_attention_weight_max": 34.102725982666016,
      "activations/layer16_attention_weight_min": -32.390167236328125,
      "activations/layer17_attention_weight_max": 55.13644790649414,
      "activations/layer17_attention_weight_min": -45.83159255981445,
      "activations/layer18_attention_weight_max": 46.451934814453125,
      "activations/layer18_attention_weight_min": -39.4162483215332,
      "activations/layer19_attention_weight_max": 23.658214569091797,
      "activations/layer19_attention_weight_min": -22.27583122253418,
      "activations/layer1_attention_weight_max": 16.985807418823242,
      "activations/layer1_attention_weight_min": -14.359030723571777,
      "activations/layer20_attention_weight_max": 23.697309494018555,
      "activations/layer20_attention_weight_min": -21.53959846496582,
      "activations/layer21_attention_weight_max": 35.17747116088867,
      "activations/layer21_attention_weight_min": -23.744470596313477,
      "activations/layer22_attention_weight_max": 27.139442443847656,
      "activations/layer22_attention_weight_min": -25.22601318359375,
      "activations/layer23_attention_weight_max": 34.21952819824219,
      "activations/layer23_attention_weight_min": -26.323514938354492,
      "activations/layer2_attention_weight_max": 31.860401153564453,
      "activations/layer2_attention_weight_min": -30.364093780517578,
      "activations/layer3_attention_weight_max": 88.04185485839844,
      "activations/layer3_attention_weight_min": -90.29774475097656,
      "activations/layer4_attention_weight_max": 82.24488830566406,
      "activations/layer4_attention_weight_min": -80.40300750732422,
      "activations/layer5_attention_weight_max": 63.89360046386719,
      "activations/layer5_attention_weight_min": -68.78202819824219,
      "activations/layer6_attention_weight_max": 46.70064163208008,
      "activations/layer6_attention_weight_min": -49.8143424987793,
      "activations/layer7_attention_weight_max": 65.54904174804688,
      "activations/layer7_attention_weight_min": -62.686336517333984,
      "activations/layer8_attention_weight_max": 44.55633544921875,
      "activations/layer8_attention_weight_min": -45.97963333129883,
      "activations/layer9_attention_weight_max": 39.77404022216797,
      "activations/layer9_attention_weight_min": -43.104000091552734,
      "epoch": 19.79,
      "learning_rate": 2.2584469696969696e-05,
      "loss": 2.73,
      "step": 340650
    },
    {
      "activations/layer0_attention_weight_max": 15.45677661895752,
      "activations/layer0_attention_weight_min": -14.083159446716309,
      "activations/layer10_attention_weight_max": 40.11679458618164,
      "activations/layer10_attention_weight_min": -38.77241134643555,
      "activations/layer11_attention_weight_max": 34.70490264892578,
      "activations/layer11_attention_weight_min": -34.97972869873047,
      "activations/layer12_attention_weight_max": 18.607412338256836,
      "activations/layer12_attention_weight_min": -24.21202850341797,
      "activations/layer13_attention_weight_max": 38.2628173828125,
      "activations/layer13_attention_weight_min": -27.093393325805664,
      "activations/layer14_attention_weight_max": 35.31505584716797,
      "activations/layer14_attention_weight_min": -29.29206085205078,
      "activations/layer15_attention_weight_max": 33.69490432739258,
      "activations/layer15_attention_weight_min": -28.42474937438965,
      "activations/layer16_attention_weight_max": 34.123531341552734,
      "activations/layer16_attention_weight_min": -31.72406005859375,
      "activations/layer17_attention_weight_max": 51.36539077758789,
      "activations/layer17_attention_weight_min": -44.78797149658203,
      "activations/layer18_attention_weight_max": 46.96076202392578,
      "activations/layer18_attention_weight_min": -39.96512985229492,
      "activations/layer19_attention_weight_max": 25.0458984375,
      "activations/layer19_attention_weight_min": -20.470224380493164,
      "activations/layer1_attention_weight_max": 15.562175750732422,
      "activations/layer1_attention_weight_min": -13.899422645568848,
      "activations/layer20_attention_weight_max": 23.12502670288086,
      "activations/layer20_attention_weight_min": -22.259183883666992,
      "activations/layer21_attention_weight_max": 41.0205078125,
      "activations/layer21_attention_weight_min": -23.345151901245117,
      "activations/layer22_attention_weight_max": 33.27067947387695,
      "activations/layer22_attention_weight_min": -26.60049819946289,
      "activations/layer23_attention_weight_max": 42.45435333251953,
      "activations/layer23_attention_weight_min": -25.88251495361328,
      "activations/layer2_attention_weight_max": 31.075864791870117,
      "activations/layer2_attention_weight_min": -30.564115524291992,
      "activations/layer3_attention_weight_max": 89.26661682128906,
      "activations/layer3_attention_weight_min": -90.28992462158203,
      "activations/layer4_attention_weight_max": 88.10942840576172,
      "activations/layer4_attention_weight_min": -86.77597045898438,
      "activations/layer5_attention_weight_max": 62.451416015625,
      "activations/layer5_attention_weight_min": -71.73164367675781,
      "activations/layer6_attention_weight_max": 52.46460723876953,
      "activations/layer6_attention_weight_min": -51.170936584472656,
      "activations/layer7_attention_weight_max": 64.30378723144531,
      "activations/layer7_attention_weight_min": -66.12532043457031,
      "activations/layer8_attention_weight_max": 49.7427978515625,
      "activations/layer8_attention_weight_min": -51.227901458740234,
      "activations/layer9_attention_weight_max": 46.80450439453125,
      "activations/layer9_attention_weight_min": -47.946189880371094,
      "epoch": 19.8,
      "learning_rate": 2.25655303030303e-05,
      "loss": 2.7165,
      "step": 340700
    },
    {
      "activations/layer0_attention_weight_max": 15.678022384643555,
      "activations/layer0_attention_weight_min": -14.07924747467041,
      "activations/layer10_attention_weight_max": 35.08443832397461,
      "activations/layer10_attention_weight_min": -33.7696647644043,
      "activations/layer11_attention_weight_max": 33.132102966308594,
      "activations/layer11_attention_weight_min": -35.48661422729492,
      "activations/layer12_attention_weight_max": 18.058847427368164,
      "activations/layer12_attention_weight_min": -25.238426208496094,
      "activations/layer13_attention_weight_max": 43.55244445800781,
      "activations/layer13_attention_weight_min": -28.348690032958984,
      "activations/layer14_attention_weight_max": 36.43626403808594,
      "activations/layer14_attention_weight_min": -31.23418617248535,
      "activations/layer15_attention_weight_max": 34.01618576049805,
      "activations/layer15_attention_weight_min": -29.22877311706543,
      "activations/layer16_attention_weight_max": 34.2411003112793,
      "activations/layer16_attention_weight_min": -31.108484268188477,
      "activations/layer17_attention_weight_max": 53.57196044921875,
      "activations/layer17_attention_weight_min": -43.0631217956543,
      "activations/layer18_attention_weight_max": 51.645721435546875,
      "activations/layer18_attention_weight_min": -41.05659866333008,
      "activations/layer19_attention_weight_max": 25.273338317871094,
      "activations/layer19_attention_weight_min": -23.12082290649414,
      "activations/layer1_attention_weight_max": 16.28514862060547,
      "activations/layer1_attention_weight_min": -14.686464309692383,
      "activations/layer20_attention_weight_max": 27.105945587158203,
      "activations/layer20_attention_weight_min": -21.691883087158203,
      "activations/layer21_attention_weight_max": 39.8924674987793,
      "activations/layer21_attention_weight_min": -23.155075073242188,
      "activations/layer22_attention_weight_max": 31.7308349609375,
      "activations/layer22_attention_weight_min": -27.306455612182617,
      "activations/layer23_attention_weight_max": 36.945289611816406,
      "activations/layer23_attention_weight_min": -24.39917755126953,
      "activations/layer2_attention_weight_max": 31.37910270690918,
      "activations/layer2_attention_weight_min": -30.621137619018555,
      "activations/layer3_attention_weight_max": 84.3524169921875,
      "activations/layer3_attention_weight_min": -87.6946792602539,
      "activations/layer4_attention_weight_max": 84.18248748779297,
      "activations/layer4_attention_weight_min": -79.39745330810547,
      "activations/layer5_attention_weight_max": 60.33618927001953,
      "activations/layer5_attention_weight_min": -68.62276458740234,
      "activations/layer6_attention_weight_max": 46.78046798706055,
      "activations/layer6_attention_weight_min": -49.018733978271484,
      "activations/layer7_attention_weight_max": 62.63789367675781,
      "activations/layer7_attention_weight_min": -67.6659164428711,
      "activations/layer8_attention_weight_max": 43.34072494506836,
      "activations/layer8_attention_weight_min": -48.42595291137695,
      "activations/layer9_attention_weight_max": 44.065303802490234,
      "activations/layer9_attention_weight_min": -43.333797454833984,
      "epoch": 19.8,
      "learning_rate": 2.254659090909091e-05,
      "loss": 2.7339,
      "step": 340750
    },
    {
      "activations/layer0_attention_weight_max": 15.836737632751465,
      "activations/layer0_attention_weight_min": -14.366690635681152,
      "activations/layer10_attention_weight_max": 40.17748260498047,
      "activations/layer10_attention_weight_min": -36.80500030517578,
      "activations/layer11_attention_weight_max": 35.6459846496582,
      "activations/layer11_attention_weight_min": -36.70177459716797,
      "activations/layer12_attention_weight_max": 19.183263778686523,
      "activations/layer12_attention_weight_min": -21.377079010009766,
      "activations/layer13_attention_weight_max": 35.05772399902344,
      "activations/layer13_attention_weight_min": -26.80038070678711,
      "activations/layer14_attention_weight_max": 34.963409423828125,
      "activations/layer14_attention_weight_min": -29.558853149414062,
      "activations/layer15_attention_weight_max": 32.84522247314453,
      "activations/layer15_attention_weight_min": -27.817598342895508,
      "activations/layer16_attention_weight_max": 35.537227630615234,
      "activations/layer16_attention_weight_min": -31.866727828979492,
      "activations/layer17_attention_weight_max": 53.62704086303711,
      "activations/layer17_attention_weight_min": -48.61479949951172,
      "activations/layer18_attention_weight_max": 49.35504913330078,
      "activations/layer18_attention_weight_min": -42.23876190185547,
      "activations/layer19_attention_weight_max": 23.716800689697266,
      "activations/layer19_attention_weight_min": -22.05190658569336,
      "activations/layer1_attention_weight_max": 16.12433433532715,
      "activations/layer1_attention_weight_min": -13.967387199401855,
      "activations/layer20_attention_weight_max": 21.72614097595215,
      "activations/layer20_attention_weight_min": -23.4317684173584,
      "activations/layer21_attention_weight_max": 37.44206237792969,
      "activations/layer21_attention_weight_min": -22.945627212524414,
      "activations/layer22_attention_weight_max": 29.685781478881836,
      "activations/layer22_attention_weight_min": -23.325639724731445,
      "activations/layer23_attention_weight_max": 42.01670455932617,
      "activations/layer23_attention_weight_min": -23.726476669311523,
      "activations/layer2_attention_weight_max": 31.551006317138672,
      "activations/layer2_attention_weight_min": -30.382450103759766,
      "activations/layer3_attention_weight_max": 86.25092315673828,
      "activations/layer3_attention_weight_min": -88.05274200439453,
      "activations/layer4_attention_weight_max": 84.12578582763672,
      "activations/layer4_attention_weight_min": -85.6399917602539,
      "activations/layer5_attention_weight_max": 62.772850036621094,
      "activations/layer5_attention_weight_min": -76.71443176269531,
      "activations/layer6_attention_weight_max": 51.48735427856445,
      "activations/layer6_attention_weight_min": -52.327579498291016,
      "activations/layer7_attention_weight_max": 64.12609100341797,
      "activations/layer7_attention_weight_min": -62.38780212402344,
      "activations/layer8_attention_weight_max": 48.12124252319336,
      "activations/layer8_attention_weight_min": -51.839107513427734,
      "activations/layer9_attention_weight_max": 46.54556655883789,
      "activations/layer9_attention_weight_min": -45.296478271484375,
      "epoch": 19.8,
      "learning_rate": 2.2527651515151514e-05,
      "loss": 2.7181,
      "step": 340800
    },
    {
      "activations/layer0_attention_weight_max": 15.704841613769531,
      "activations/layer0_attention_weight_min": -14.21990966796875,
      "activations/layer10_attention_weight_max": 38.50486373901367,
      "activations/layer10_attention_weight_min": -37.89888381958008,
      "activations/layer11_attention_weight_max": 33.940826416015625,
      "activations/layer11_attention_weight_min": -35.758995056152344,
      "activations/layer12_attention_weight_max": 18.384658813476562,
      "activations/layer12_attention_weight_min": -29.754243850708008,
      "activations/layer13_attention_weight_max": 38.832977294921875,
      "activations/layer13_attention_weight_min": -28.515100479125977,
      "activations/layer14_attention_weight_max": 44.188636779785156,
      "activations/layer14_attention_weight_min": -30.070322036743164,
      "activations/layer15_attention_weight_max": 43.70896530151367,
      "activations/layer15_attention_weight_min": -30.65789794921875,
      "activations/layer16_attention_weight_max": 33.8112907409668,
      "activations/layer16_attention_weight_min": -33.68152618408203,
      "activations/layer17_attention_weight_max": 51.433082580566406,
      "activations/layer17_attention_weight_min": -49.50090408325195,
      "activations/layer18_attention_weight_max": 48.39195251464844,
      "activations/layer18_attention_weight_min": -44.906253814697266,
      "activations/layer19_attention_weight_max": 28.315826416015625,
      "activations/layer19_attention_weight_min": -25.803253173828125,
      "activations/layer1_attention_weight_max": 16.770254135131836,
      "activations/layer1_attention_weight_min": -13.788599967956543,
      "activations/layer20_attention_weight_max": 24.9382381439209,
      "activations/layer20_attention_weight_min": -22.379924774169922,
      "activations/layer21_attention_weight_max": 52.09894943237305,
      "activations/layer21_attention_weight_min": -27.028623580932617,
      "activations/layer22_attention_weight_max": 30.690853118896484,
      "activations/layer22_attention_weight_min": -25.705909729003906,
      "activations/layer23_attention_weight_max": 33.38806915283203,
      "activations/layer23_attention_weight_min": -25.04804801940918,
      "activations/layer2_attention_weight_max": 33.01170349121094,
      "activations/layer2_attention_weight_min": -30.532140731811523,
      "activations/layer3_attention_weight_max": 88.31786346435547,
      "activations/layer3_attention_weight_min": -86.1209487915039,
      "activations/layer4_attention_weight_max": 85.2863998413086,
      "activations/layer4_attention_weight_min": -81.26788330078125,
      "activations/layer5_attention_weight_max": 65.3389892578125,
      "activations/layer5_attention_weight_min": -72.19271087646484,
      "activations/layer6_attention_weight_max": 50.58143997192383,
      "activations/layer6_attention_weight_min": -51.43771743774414,
      "activations/layer7_attention_weight_max": 68.22434997558594,
      "activations/layer7_attention_weight_min": -64.7380142211914,
      "activations/layer8_attention_weight_max": 50.40852737426758,
      "activations/layer8_attention_weight_min": -51.35405349731445,
      "activations/layer9_attention_weight_max": 49.758968353271484,
      "activations/layer9_attention_weight_min": -46.372039794921875,
      "epoch": 19.81,
      "learning_rate": 2.2509090909090908e-05,
      "loss": 2.7208,
      "step": 340850
    },
    {
      "activations/layer0_attention_weight_max": 15.57453727722168,
      "activations/layer0_attention_weight_min": -14.081912994384766,
      "activations/layer10_attention_weight_max": 37.737884521484375,
      "activations/layer10_attention_weight_min": -35.022613525390625,
      "activations/layer11_attention_weight_max": 35.056541442871094,
      "activations/layer11_attention_weight_min": -34.27783203125,
      "activations/layer12_attention_weight_max": 19.143150329589844,
      "activations/layer12_attention_weight_min": -23.894418716430664,
      "activations/layer13_attention_weight_max": 36.49399185180664,
      "activations/layer13_attention_weight_min": -28.26012420654297,
      "activations/layer14_attention_weight_max": 39.30708694458008,
      "activations/layer14_attention_weight_min": -30.206697463989258,
      "activations/layer15_attention_weight_max": 36.04526901245117,
      "activations/layer15_attention_weight_min": -28.77712631225586,
      "activations/layer16_attention_weight_max": 35.632991790771484,
      "activations/layer16_attention_weight_min": -30.397794723510742,
      "activations/layer17_attention_weight_max": 56.32999801635742,
      "activations/layer17_attention_weight_min": -43.19767379760742,
      "activations/layer18_attention_weight_max": 52.33016586303711,
      "activations/layer18_attention_weight_min": -37.83904266357422,
      "activations/layer19_attention_weight_max": 24.183218002319336,
      "activations/layer19_attention_weight_min": -22.711021423339844,
      "activations/layer1_attention_weight_max": 15.551216125488281,
      "activations/layer1_attention_weight_min": -13.568008422851562,
      "activations/layer20_attention_weight_max": 22.403614044189453,
      "activations/layer20_attention_weight_min": -23.016454696655273,
      "activations/layer21_attention_weight_max": 34.14727783203125,
      "activations/layer21_attention_weight_min": -20.083459854125977,
      "activations/layer22_attention_weight_max": 29.357486724853516,
      "activations/layer22_attention_weight_min": -23.90057373046875,
      "activations/layer23_attention_weight_max": 32.446022033691406,
      "activations/layer23_attention_weight_min": -24.054670333862305,
      "activations/layer2_attention_weight_max": 31.813812255859375,
      "activations/layer2_attention_weight_min": -29.026721954345703,
      "activations/layer3_attention_weight_max": 86.17786407470703,
      "activations/layer3_attention_weight_min": -85.7987060546875,
      "activations/layer4_attention_weight_max": 82.5836181640625,
      "activations/layer4_attention_weight_min": -80.0919418334961,
      "activations/layer5_attention_weight_max": 60.911468505859375,
      "activations/layer5_attention_weight_min": -67.52587890625,
      "activations/layer6_attention_weight_max": 48.91460037231445,
      "activations/layer6_attention_weight_min": -50.61972427368164,
      "activations/layer7_attention_weight_max": 63.207096099853516,
      "activations/layer7_attention_weight_min": -61.52653503417969,
      "activations/layer8_attention_weight_max": 45.8625373840332,
      "activations/layer8_attention_weight_min": -48.782569885253906,
      "activations/layer9_attention_weight_max": 44.08795928955078,
      "activations/layer9_attention_weight_min": -44.713077545166016,
      "epoch": 19.81,
      "learning_rate": 2.2490151515151516e-05,
      "loss": 2.7239,
      "step": 340900
    },
    {
      "activations/layer0_attention_weight_max": 15.419683456420898,
      "activations/layer0_attention_weight_min": -14.126102447509766,
      "activations/layer10_attention_weight_max": 36.23933792114258,
      "activations/layer10_attention_weight_min": -36.32756423950195,
      "activations/layer11_attention_weight_max": 33.15398025512695,
      "activations/layer11_attention_weight_min": -34.459449768066406,
      "activations/layer12_attention_weight_max": 19.307104110717773,
      "activations/layer12_attention_weight_min": -24.81747055053711,
      "activations/layer13_attention_weight_max": 34.44073486328125,
      "activations/layer13_attention_weight_min": -28.351703643798828,
      "activations/layer14_attention_weight_max": 35.309730529785156,
      "activations/layer14_attention_weight_min": -30.06068229675293,
      "activations/layer15_attention_weight_max": 32.96052169799805,
      "activations/layer15_attention_weight_min": -31.24968719482422,
      "activations/layer16_attention_weight_max": 35.51262664794922,
      "activations/layer16_attention_weight_min": -34.964969635009766,
      "activations/layer17_attention_weight_max": 55.51778793334961,
      "activations/layer17_attention_weight_min": -53.06398391723633,
      "activations/layer18_attention_weight_max": 47.286312103271484,
      "activations/layer18_attention_weight_min": -44.410552978515625,
      "activations/layer19_attention_weight_max": 23.87806510925293,
      "activations/layer19_attention_weight_min": -23.676057815551758,
      "activations/layer1_attention_weight_max": 15.071534156799316,
      "activations/layer1_attention_weight_min": -13.903449058532715,
      "activations/layer20_attention_weight_max": 22.946735382080078,
      "activations/layer20_attention_weight_min": -22.06000518798828,
      "activations/layer21_attention_weight_max": 38.58898162841797,
      "activations/layer21_attention_weight_min": -19.57057762145996,
      "activations/layer22_attention_weight_max": 28.54556655883789,
      "activations/layer22_attention_weight_min": -25.25814437866211,
      "activations/layer23_attention_weight_max": 36.3392333984375,
      "activations/layer23_attention_weight_min": -24.668630599975586,
      "activations/layer2_attention_weight_max": 31.31146240234375,
      "activations/layer2_attention_weight_min": -30.70197296142578,
      "activations/layer3_attention_weight_max": 87.69778442382812,
      "activations/layer3_attention_weight_min": -88.65400695800781,
      "activations/layer4_attention_weight_max": 81.75765228271484,
      "activations/layer4_attention_weight_min": -85.51780700683594,
      "activations/layer5_attention_weight_max": 59.43215560913086,
      "activations/layer5_attention_weight_min": -69.52415466308594,
      "activations/layer6_attention_weight_max": 47.976932525634766,
      "activations/layer6_attention_weight_min": -50.0122184753418,
      "activations/layer7_attention_weight_max": 66.8141098022461,
      "activations/layer7_attention_weight_min": -63.32528305053711,
      "activations/layer8_attention_weight_max": 47.97359085083008,
      "activations/layer8_attention_weight_min": -51.650962829589844,
      "activations/layer9_attention_weight_max": 45.60741424560547,
      "activations/layer9_attention_weight_min": -45.81013107299805,
      "epoch": 19.81,
      "learning_rate": 2.2471212121212118e-05,
      "loss": 2.7301,
      "step": 340950
    },
    {
      "activations/layer0_attention_weight_max": 15.327128410339355,
      "activations/layer0_attention_weight_min": -14.380078315734863,
      "activations/layer10_attention_weight_max": 36.84088134765625,
      "activations/layer10_attention_weight_min": -36.82489013671875,
      "activations/layer11_attention_weight_max": 34.101436614990234,
      "activations/layer11_attention_weight_min": -34.94568634033203,
      "activations/layer12_attention_weight_max": 19.976200103759766,
      "activations/layer12_attention_weight_min": -25.344690322875977,
      "activations/layer13_attention_weight_max": 35.149742126464844,
      "activations/layer13_attention_weight_min": -29.656883239746094,
      "activations/layer14_attention_weight_max": 36.512725830078125,
      "activations/layer14_attention_weight_min": -32.14827346801758,
      "activations/layer15_attention_weight_max": 33.031681060791016,
      "activations/layer15_attention_weight_min": -30.189599990844727,
      "activations/layer16_attention_weight_max": 36.15964126586914,
      "activations/layer16_attention_weight_min": -34.18980407714844,
      "activations/layer17_attention_weight_max": 52.78184127807617,
      "activations/layer17_attention_weight_min": -44.056461334228516,
      "activations/layer18_attention_weight_max": 51.242774963378906,
      "activations/layer18_attention_weight_min": -40.52157974243164,
      "activations/layer19_attention_weight_max": 26.726104736328125,
      "activations/layer19_attention_weight_min": -22.302574157714844,
      "activations/layer1_attention_weight_max": 15.22847843170166,
      "activations/layer1_attention_weight_min": -13.401899337768555,
      "activations/layer20_attention_weight_max": 26.576433181762695,
      "activations/layer20_attention_weight_min": -20.542362213134766,
      "activations/layer21_attention_weight_max": 38.455509185791016,
      "activations/layer21_attention_weight_min": -24.746238708496094,
      "activations/layer22_attention_weight_max": 30.72800636291504,
      "activations/layer22_attention_weight_min": -24.887847900390625,
      "activations/layer23_attention_weight_max": 36.397911071777344,
      "activations/layer23_attention_weight_min": -26.290691375732422,
      "activations/layer2_attention_weight_max": 31.75977325439453,
      "activations/layer2_attention_weight_min": -29.774396896362305,
      "activations/layer3_attention_weight_max": 84.93463134765625,
      "activations/layer3_attention_weight_min": -86.84339904785156,
      "activations/layer4_attention_weight_max": 84.917236328125,
      "activations/layer4_attention_weight_min": -85.5955810546875,
      "activations/layer5_attention_weight_max": 64.16400909423828,
      "activations/layer5_attention_weight_min": -72.67353820800781,
      "activations/layer6_attention_weight_max": 50.94407653808594,
      "activations/layer6_attention_weight_min": -50.345027923583984,
      "activations/layer7_attention_weight_max": 72.28287506103516,
      "activations/layer7_attention_weight_min": -64.219970703125,
      "activations/layer8_attention_weight_max": 47.14510726928711,
      "activations/layer8_attention_weight_min": -49.28316116333008,
      "activations/layer9_attention_weight_max": 43.599300384521484,
      "activations/layer9_attention_weight_min": -45.74689865112305,
      "epoch": 19.81,
      "learning_rate": 2.2452272727272726e-05,
      "loss": 2.711,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.5386,
      "eval_samples_per_second": 502.895,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.5386,
      "eval_openwebtext_samples_per_second": 502.895,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 2.0256,
      "eval_wikitext_samples_per_second": 225.117,
      "step": 341000
    },
    {
      "epoch": 19.81,
      "eval_lambada_loss": 2.4140625,
      "eval_lambada_ppl": 11.17928486368573,
      "eval_lambada_runtime": 9.6132,
      "eval_lambada_samples_per_second": 506.491,
      "step": 341000
    },
    {
      "activations/layer0_attention_weight_max": 15.516927719116211,
      "activations/layer0_attention_weight_min": -14.114824295043945,
      "activations/layer10_attention_weight_max": 36.89513397216797,
      "activations/layer10_attention_weight_min": -35.531822204589844,
      "activations/layer11_attention_weight_max": 33.22290802001953,
      "activations/layer11_attention_weight_min": -34.67313003540039,
      "activations/layer12_attention_weight_max": 18.689105987548828,
      "activations/layer12_attention_weight_min": -25.49346923828125,
      "activations/layer13_attention_weight_max": 33.326175689697266,
      "activations/layer13_attention_weight_min": -27.51972007751465,
      "activations/layer14_attention_weight_max": 36.95535659790039,
      "activations/layer14_attention_weight_min": -30.585601806640625,
      "activations/layer15_attention_weight_max": 34.90151596069336,
      "activations/layer15_attention_weight_min": -29.612462997436523,
      "activations/layer16_attention_weight_max": 38.07101058959961,
      "activations/layer16_attention_weight_min": -33.53501892089844,
      "activations/layer17_attention_weight_max": 57.05129623413086,
      "activations/layer17_attention_weight_min": -47.76701736450195,
      "activations/layer18_attention_weight_max": 53.71870040893555,
      "activations/layer18_attention_weight_min": -43.554134368896484,
      "activations/layer19_attention_weight_max": 26.794885635375977,
      "activations/layer19_attention_weight_min": -25.6629695892334,
      "activations/layer1_attention_weight_max": 15.010411262512207,
      "activations/layer1_attention_weight_min": -14.364134788513184,
      "activations/layer20_attention_weight_max": 26.970617294311523,
      "activations/layer20_attention_weight_min": -22.888635635375977,
      "activations/layer21_attention_weight_max": 41.76826095581055,
      "activations/layer21_attention_weight_min": -21.699918746948242,
      "activations/layer22_attention_weight_max": 32.5164680480957,
      "activations/layer22_attention_weight_min": -25.243019104003906,
      "activations/layer23_attention_weight_max": 44.20115661621094,
      "activations/layer23_attention_weight_min": -24.48486328125,
      "activations/layer2_attention_weight_max": 32.30078887939453,
      "activations/layer2_attention_weight_min": -31.506620407104492,
      "activations/layer3_attention_weight_max": 88.10047912597656,
      "activations/layer3_attention_weight_min": -91.99449157714844,
      "activations/layer4_attention_weight_max": 82.0744857788086,
      "activations/layer4_attention_weight_min": -84.37916564941406,
      "activations/layer5_attention_weight_max": 61.602813720703125,
      "activations/layer5_attention_weight_min": -76.15667724609375,
      "activations/layer6_attention_weight_max": 49.08866500854492,
      "activations/layer6_attention_weight_min": -50.631935119628906,
      "activations/layer7_attention_weight_max": 64.54527282714844,
      "activations/layer7_attention_weight_min": -63.21394729614258,
      "activations/layer8_attention_weight_max": 46.53026580810547,
      "activations/layer8_attention_weight_min": -50.148345947265625,
      "activations/layer9_attention_weight_max": 43.86481475830078,
      "activations/layer9_attention_weight_min": -44.801055908203125,
      "epoch": 19.82,
      "learning_rate": 2.243333333333333e-05,
      "loss": 2.7296,
      "step": 341050
    },
    {
      "activations/layer0_attention_weight_max": 15.904996871948242,
      "activations/layer0_attention_weight_min": -13.851540565490723,
      "activations/layer10_attention_weight_max": 36.19317626953125,
      "activations/layer10_attention_weight_min": -37.13489532470703,
      "activations/layer11_attention_weight_max": 34.10081100463867,
      "activations/layer11_attention_weight_min": -35.74993896484375,
      "activations/layer12_attention_weight_max": 19.126375198364258,
      "activations/layer12_attention_weight_min": -24.6744384765625,
      "activations/layer13_attention_weight_max": 37.42283630371094,
      "activations/layer13_attention_weight_min": -29.67931365966797,
      "activations/layer14_attention_weight_max": 36.075096130371094,
      "activations/layer14_attention_weight_min": -33.712554931640625,
      "activations/layer15_attention_weight_max": 33.17271423339844,
      "activations/layer15_attention_weight_min": -33.689048767089844,
      "activations/layer16_attention_weight_max": 36.19593811035156,
      "activations/layer16_attention_weight_min": -34.230674743652344,
      "activations/layer17_attention_weight_max": 50.43147659301758,
      "activations/layer17_attention_weight_min": -46.980281829833984,
      "activations/layer18_attention_weight_max": 46.70930099487305,
      "activations/layer18_attention_weight_min": -39.86473846435547,
      "activations/layer19_attention_weight_max": 25.050212860107422,
      "activations/layer19_attention_weight_min": -21.690824508666992,
      "activations/layer1_attention_weight_max": 15.227992057800293,
      "activations/layer1_attention_weight_min": -14.46015453338623,
      "activations/layer20_attention_weight_max": 22.99073028564453,
      "activations/layer20_attention_weight_min": -21.527244567871094,
      "activations/layer21_attention_weight_max": 33.851261138916016,
      "activations/layer21_attention_weight_min": -22.606355667114258,
      "activations/layer22_attention_weight_max": 30.70826530456543,
      "activations/layer22_attention_weight_min": -25.157087326049805,
      "activations/layer23_attention_weight_max": 38.62360382080078,
      "activations/layer23_attention_weight_min": -25.27926254272461,
      "activations/layer2_attention_weight_max": 30.914207458496094,
      "activations/layer2_attention_weight_min": -29.443408966064453,
      "activations/layer3_attention_weight_max": 81.9368667602539,
      "activations/layer3_attention_weight_min": -83.5480728149414,
      "activations/layer4_attention_weight_max": 83.9246826171875,
      "activations/layer4_attention_weight_min": -80.49715423583984,
      "activations/layer5_attention_weight_max": 60.52523422241211,
      "activations/layer5_attention_weight_min": -71.47972869873047,
      "activations/layer6_attention_weight_max": 49.69736099243164,
      "activations/layer6_attention_weight_min": -48.651710510253906,
      "activations/layer7_attention_weight_max": 67.59893798828125,
      "activations/layer7_attention_weight_min": -63.176902770996094,
      "activations/layer8_attention_weight_max": 44.73585510253906,
      "activations/layer8_attention_weight_min": -47.8748664855957,
      "activations/layer9_attention_weight_max": 42.055477142333984,
      "activations/layer9_attention_weight_min": -45.13551330566406,
      "epoch": 19.82,
      "learning_rate": 2.241439393939394e-05,
      "loss": 2.7098,
      "step": 341100
    },
    {
      "activations/layer0_attention_weight_max": 15.751964569091797,
      "activations/layer0_attention_weight_min": -14.333272933959961,
      "activations/layer10_attention_weight_max": 34.21346664428711,
      "activations/layer10_attention_weight_min": -34.12906265258789,
      "activations/layer11_attention_weight_max": 31.718982696533203,
      "activations/layer11_attention_weight_min": -32.61667251586914,
      "activations/layer12_attention_weight_max": 18.49913787841797,
      "activations/layer12_attention_weight_min": -27.110536575317383,
      "activations/layer13_attention_weight_max": 32.05583572387695,
      "activations/layer13_attention_weight_min": -28.24705696105957,
      "activations/layer14_attention_weight_max": 36.12907409667969,
      "activations/layer14_attention_weight_min": -31.607675552368164,
      "activations/layer15_attention_weight_max": 34.31038284301758,
      "activations/layer15_attention_weight_min": -31.30460548400879,
      "activations/layer16_attention_weight_max": 33.34577178955078,
      "activations/layer16_attention_weight_min": -32.37380599975586,
      "activations/layer17_attention_weight_max": 52.61420822143555,
      "activations/layer17_attention_weight_min": -44.086448669433594,
      "activations/layer18_attention_weight_max": 46.1046257019043,
      "activations/layer18_attention_weight_min": -40.6490364074707,
      "activations/layer19_attention_weight_max": 23.072162628173828,
      "activations/layer19_attention_weight_min": -23.075843811035156,
      "activations/layer1_attention_weight_max": 14.81632137298584,
      "activations/layer1_attention_weight_min": -13.770944595336914,
      "activations/layer20_attention_weight_max": 22.35795021057129,
      "activations/layer20_attention_weight_min": -22.31009292602539,
      "activations/layer21_attention_weight_max": 38.6334228515625,
      "activations/layer21_attention_weight_min": -22.069576263427734,
      "activations/layer22_attention_weight_max": 25.7775936126709,
      "activations/layer22_attention_weight_min": -25.561399459838867,
      "activations/layer23_attention_weight_max": 35.8468017578125,
      "activations/layer23_attention_weight_min": -26.345890045166016,
      "activations/layer2_attention_weight_max": 32.05846405029297,
      "activations/layer2_attention_weight_min": -30.905393600463867,
      "activations/layer3_attention_weight_max": 90.759521484375,
      "activations/layer3_attention_weight_min": -93.58102416992188,
      "activations/layer4_attention_weight_max": 83.87886810302734,
      "activations/layer4_attention_weight_min": -83.5232925415039,
      "activations/layer5_attention_weight_max": 60.19488525390625,
      "activations/layer5_attention_weight_min": -73.13327026367188,
      "activations/layer6_attention_weight_max": 48.441184997558594,
      "activations/layer6_attention_weight_min": -53.05099868774414,
      "activations/layer7_attention_weight_max": 62.971622467041016,
      "activations/layer7_attention_weight_min": -62.62095260620117,
      "activations/layer8_attention_weight_max": 44.90810775756836,
      "activations/layer8_attention_weight_min": -48.757747650146484,
      "activations/layer9_attention_weight_max": 43.33283615112305,
      "activations/layer9_attention_weight_min": -43.22416305541992,
      "epoch": 19.82,
      "learning_rate": 2.2395454545454544e-05,
      "loss": 2.726,
      "step": 341150
    },
    {
      "activations/layer0_attention_weight_max": 15.809808731079102,
      "activations/layer0_attention_weight_min": -14.246147155761719,
      "activations/layer10_attention_weight_max": 36.21943664550781,
      "activations/layer10_attention_weight_min": -34.79374694824219,
      "activations/layer11_attention_weight_max": 32.36561584472656,
      "activations/layer11_attention_weight_min": -34.48619842529297,
      "activations/layer12_attention_weight_max": 19.51473617553711,
      "activations/layer12_attention_weight_min": -23.471094131469727,
      "activations/layer13_attention_weight_max": 31.99108123779297,
      "activations/layer13_attention_weight_min": -26.99478530883789,
      "activations/layer14_attention_weight_max": 36.036067962646484,
      "activations/layer14_attention_weight_min": -28.450634002685547,
      "activations/layer15_attention_weight_max": 33.173831939697266,
      "activations/layer15_attention_weight_min": -29.93876838684082,
      "activations/layer16_attention_weight_max": 33.594486236572266,
      "activations/layer16_attention_weight_min": -31.78049659729004,
      "activations/layer17_attention_weight_max": 52.57429504394531,
      "activations/layer17_attention_weight_min": -42.33220291137695,
      "activations/layer18_attention_weight_max": 48.96363067626953,
      "activations/layer18_attention_weight_min": -38.2108154296875,
      "activations/layer19_attention_weight_max": 22.862794876098633,
      "activations/layer19_attention_weight_min": -20.716447830200195,
      "activations/layer1_attention_weight_max": 14.865499496459961,
      "activations/layer1_attention_weight_min": -13.666936874389648,
      "activations/layer20_attention_weight_max": 20.97581672668457,
      "activations/layer20_attention_weight_min": -21.408489227294922,
      "activations/layer21_attention_weight_max": 33.68080139160156,
      "activations/layer21_attention_weight_min": -19.884326934814453,
      "activations/layer22_attention_weight_max": 28.750160217285156,
      "activations/layer22_attention_weight_min": -24.504344940185547,
      "activations/layer23_attention_weight_max": 32.749454498291016,
      "activations/layer23_attention_weight_min": -22.5919246673584,
      "activations/layer2_attention_weight_max": 31.67668914794922,
      "activations/layer2_attention_weight_min": -31.200698852539062,
      "activations/layer3_attention_weight_max": 86.453125,
      "activations/layer3_attention_weight_min": -88.38746643066406,
      "activations/layer4_attention_weight_max": 81.20740509033203,
      "activations/layer4_attention_weight_min": -79.14434051513672,
      "activations/layer5_attention_weight_max": 60.658321380615234,
      "activations/layer5_attention_weight_min": -68.62899780273438,
      "activations/layer6_attention_weight_max": 48.20267868041992,
      "activations/layer6_attention_weight_min": -46.716407775878906,
      "activations/layer7_attention_weight_max": 66.22577667236328,
      "activations/layer7_attention_weight_min": -61.70561981201172,
      "activations/layer8_attention_weight_max": 44.31071853637695,
      "activations/layer8_attention_weight_min": -48.277244567871094,
      "activations/layer9_attention_weight_max": 42.0525016784668,
      "activations/layer9_attention_weight_min": -44.15106964111328,
      "epoch": 19.83,
      "learning_rate": 2.2376515151515152e-05,
      "loss": 2.7333,
      "step": 341200
    },
    {
      "activations/layer0_attention_weight_max": 15.08708667755127,
      "activations/layer0_attention_weight_min": -14.274961471557617,
      "activations/layer10_attention_weight_max": 35.131874084472656,
      "activations/layer10_attention_weight_min": -35.616905212402344,
      "activations/layer11_attention_weight_max": 32.15168380737305,
      "activations/layer11_attention_weight_min": -33.95431900024414,
      "activations/layer12_attention_weight_max": 18.155576705932617,
      "activations/layer12_attention_weight_min": -32.80869674682617,
      "activations/layer13_attention_weight_max": 38.099952697753906,
      "activations/layer13_attention_weight_min": -27.092906951904297,
      "activations/layer14_attention_weight_max": 33.84055709838867,
      "activations/layer14_attention_weight_min": -29.155681610107422,
      "activations/layer15_attention_weight_max": 36.418548583984375,
      "activations/layer15_attention_weight_min": -30.701204299926758,
      "activations/layer16_attention_weight_max": 33.56659698486328,
      "activations/layer16_attention_weight_min": -32.2288932800293,
      "activations/layer17_attention_weight_max": 50.604286193847656,
      "activations/layer17_attention_weight_min": -43.27628707885742,
      "activations/layer18_attention_weight_max": 47.60857391357422,
      "activations/layer18_attention_weight_min": -39.52538299560547,
      "activations/layer19_attention_weight_max": 24.160070419311523,
      "activations/layer19_attention_weight_min": -21.23941421508789,
      "activations/layer1_attention_weight_max": 15.74557113647461,
      "activations/layer1_attention_weight_min": -13.468006134033203,
      "activations/layer20_attention_weight_max": 20.168071746826172,
      "activations/layer20_attention_weight_min": -23.037967681884766,
      "activations/layer21_attention_weight_max": 35.242820739746094,
      "activations/layer21_attention_weight_min": -20.173837661743164,
      "activations/layer22_attention_weight_max": 27.220796585083008,
      "activations/layer22_attention_weight_min": -24.724905014038086,
      "activations/layer23_attention_weight_max": 32.35364532470703,
      "activations/layer23_attention_weight_min": -25.186613082885742,
      "activations/layer2_attention_weight_max": 32.139469146728516,
      "activations/layer2_attention_weight_min": -29.745826721191406,
      "activations/layer3_attention_weight_max": 87.57782745361328,
      "activations/layer3_attention_weight_min": -90.28794860839844,
      "activations/layer4_attention_weight_max": 85.33299255371094,
      "activations/layer4_attention_weight_min": -81.71773529052734,
      "activations/layer5_attention_weight_max": 60.469879150390625,
      "activations/layer5_attention_weight_min": -68.9066162109375,
      "activations/layer6_attention_weight_max": 48.156795501708984,
      "activations/layer6_attention_weight_min": -49.04475402832031,
      "activations/layer7_attention_weight_max": 64.08995056152344,
      "activations/layer7_attention_weight_min": -63.64990234375,
      "activations/layer8_attention_weight_max": 44.5892219543457,
      "activations/layer8_attention_weight_min": -48.25129699707031,
      "activations/layer9_attention_weight_max": 45.846092224121094,
      "activations/layer9_attention_weight_min": -44.66887664794922,
      "epoch": 19.83,
      "learning_rate": 2.2357575757575754e-05,
      "loss": 2.7203,
      "step": 341250
    },
    {
      "activations/layer0_attention_weight_max": 15.708195686340332,
      "activations/layer0_attention_weight_min": -14.470870018005371,
      "activations/layer10_attention_weight_max": 36.556495666503906,
      "activations/layer10_attention_weight_min": -35.2933235168457,
      "activations/layer11_attention_weight_max": 35.111270904541016,
      "activations/layer11_attention_weight_min": -35.13922119140625,
      "activations/layer12_attention_weight_max": 18.262989044189453,
      "activations/layer12_attention_weight_min": -28.375484466552734,
      "activations/layer13_attention_weight_max": 34.23607635498047,
      "activations/layer13_attention_weight_min": -28.456092834472656,
      "activations/layer14_attention_weight_max": 34.27125930786133,
      "activations/layer14_attention_weight_min": -30.842147827148438,
      "activations/layer15_attention_weight_max": 34.137413024902344,
      "activations/layer15_attention_weight_min": -31.202611923217773,
      "activations/layer16_attention_weight_max": 33.05666732788086,
      "activations/layer16_attention_weight_min": -32.487571716308594,
      "activations/layer17_attention_weight_max": 51.360328674316406,
      "activations/layer17_attention_weight_min": -45.06877517700195,
      "activations/layer18_attention_weight_max": 49.554649353027344,
      "activations/layer18_attention_weight_min": -42.55586242675781,
      "activations/layer19_attention_weight_max": 27.422870635986328,
      "activations/layer19_attention_weight_min": -23.079349517822266,
      "activations/layer1_attention_weight_max": 15.791168212890625,
      "activations/layer1_attention_weight_min": -13.070754051208496,
      "activations/layer20_attention_weight_max": 22.69719696044922,
      "activations/layer20_attention_weight_min": -22.572690963745117,
      "activations/layer21_attention_weight_max": 36.79531478881836,
      "activations/layer21_attention_weight_min": -23.95212745666504,
      "activations/layer22_attention_weight_max": 29.169017791748047,
      "activations/layer22_attention_weight_min": -27.06386947631836,
      "activations/layer23_attention_weight_max": 36.45094680786133,
      "activations/layer23_attention_weight_min": -26.01860809326172,
      "activations/layer2_attention_weight_max": 31.94348907470703,
      "activations/layer2_attention_weight_min": -30.1583309173584,
      "activations/layer3_attention_weight_max": 89.00529479980469,
      "activations/layer3_attention_weight_min": -90.24185943603516,
      "activations/layer4_attention_weight_max": 84.79840087890625,
      "activations/layer4_attention_weight_min": -84.25999450683594,
      "activations/layer5_attention_weight_max": 61.019588470458984,
      "activations/layer5_attention_weight_min": -70.82179260253906,
      "activations/layer6_attention_weight_max": 51.66926956176758,
      "activations/layer6_attention_weight_min": -50.17886734008789,
      "activations/layer7_attention_weight_max": 69.52266693115234,
      "activations/layer7_attention_weight_min": -66.7818832397461,
      "activations/layer8_attention_weight_max": 49.810848236083984,
      "activations/layer8_attention_weight_min": -52.052860260009766,
      "activations/layer9_attention_weight_max": 47.40766525268555,
      "activations/layer9_attention_weight_min": -45.019901275634766,
      "epoch": 19.83,
      "learning_rate": 2.2338636363636362e-05,
      "loss": 2.7178,
      "step": 341300
    },
    {
      "activations/layer0_attention_weight_max": 16.734392166137695,
      "activations/layer0_attention_weight_min": -14.458479881286621,
      "activations/layer10_attention_weight_max": 33.76201248168945,
      "activations/layer10_attention_weight_min": -34.90658187866211,
      "activations/layer11_attention_weight_max": 32.88027572631836,
      "activations/layer11_attention_weight_min": -34.06028366088867,
      "activations/layer12_attention_weight_max": 17.71398162841797,
      "activations/layer12_attention_weight_min": -23.560632705688477,
      "activations/layer13_attention_weight_max": 29.953205108642578,
      "activations/layer13_attention_weight_min": -31.50324058532715,
      "activations/layer14_attention_weight_max": 31.501445770263672,
      "activations/layer14_attention_weight_min": -30.154632568359375,
      "activations/layer15_attention_weight_max": 29.753013610839844,
      "activations/layer15_attention_weight_min": -29.497940063476562,
      "activations/layer16_attention_weight_max": 32.092227935791016,
      "activations/layer16_attention_weight_min": -31.41477394104004,
      "activations/layer17_attention_weight_max": 50.647315979003906,
      "activations/layer17_attention_weight_min": -45.17345428466797,
      "activations/layer18_attention_weight_max": 43.97134780883789,
      "activations/layer18_attention_weight_min": -38.97806930541992,
      "activations/layer19_attention_weight_max": 23.733423233032227,
      "activations/layer19_attention_weight_min": -23.672088623046875,
      "activations/layer1_attention_weight_max": 15.78386402130127,
      "activations/layer1_attention_weight_min": -13.176740646362305,
      "activations/layer20_attention_weight_max": 20.851375579833984,
      "activations/layer20_attention_weight_min": -21.060049057006836,
      "activations/layer21_attention_weight_max": 33.8514289855957,
      "activations/layer21_attention_weight_min": -23.5970401763916,
      "activations/layer22_attention_weight_max": 27.5137939453125,
      "activations/layer22_attention_weight_min": -24.170978546142578,
      "activations/layer23_attention_weight_max": 32.92356872558594,
      "activations/layer23_attention_weight_min": -24.381574630737305,
      "activations/layer2_attention_weight_max": 33.72494125366211,
      "activations/layer2_attention_weight_min": -31.058517456054688,
      "activations/layer3_attention_weight_max": 88.9097671508789,
      "activations/layer3_attention_weight_min": -89.0722427368164,
      "activations/layer4_attention_weight_max": 87.45614624023438,
      "activations/layer4_attention_weight_min": -85.54496765136719,
      "activations/layer5_attention_weight_max": 65.48240661621094,
      "activations/layer5_attention_weight_min": -73.0235366821289,
      "activations/layer6_attention_weight_max": 49.28903579711914,
      "activations/layer6_attention_weight_min": -50.552085876464844,
      "activations/layer7_attention_weight_max": 64.41629028320312,
      "activations/layer7_attention_weight_min": -64.59796905517578,
      "activations/layer8_attention_weight_max": 45.61444091796875,
      "activations/layer8_attention_weight_min": -49.4874267578125,
      "activations/layer9_attention_weight_max": 45.348690032958984,
      "activations/layer9_attention_weight_min": -46.32266616821289,
      "epoch": 19.83,
      "learning_rate": 2.2319696969696967e-05,
      "loss": 2.7279,
      "step": 341350
    },
    {
      "activations/layer0_attention_weight_max": 16.31010627746582,
      "activations/layer0_attention_weight_min": -14.317587852478027,
      "activations/layer10_attention_weight_max": 34.81878662109375,
      "activations/layer10_attention_weight_min": -35.10340118408203,
      "activations/layer11_attention_weight_max": 30.519987106323242,
      "activations/layer11_attention_weight_min": -35.172611236572266,
      "activations/layer12_attention_weight_max": 19.511430740356445,
      "activations/layer12_attention_weight_min": -28.035463333129883,
      "activations/layer13_attention_weight_max": 31.239416122436523,
      "activations/layer13_attention_weight_min": -27.997085571289062,
      "activations/layer14_attention_weight_max": 33.60519027709961,
      "activations/layer14_attention_weight_min": -32.50629806518555,
      "activations/layer15_attention_weight_max": 31.478673934936523,
      "activations/layer15_attention_weight_min": -30.609073638916016,
      "activations/layer16_attention_weight_max": 31.893447875976562,
      "activations/layer16_attention_weight_min": -31.623262405395508,
      "activations/layer17_attention_weight_max": 50.65572738647461,
      "activations/layer17_attention_weight_min": -43.9326286315918,
      "activations/layer18_attention_weight_max": 44.95564651489258,
      "activations/layer18_attention_weight_min": -38.76457977294922,
      "activations/layer19_attention_weight_max": 26.00290298461914,
      "activations/layer19_attention_weight_min": -21.638669967651367,
      "activations/layer1_attention_weight_max": 15.093476295471191,
      "activations/layer1_attention_weight_min": -13.950745582580566,
      "activations/layer20_attention_weight_max": 21.911527633666992,
      "activations/layer20_attention_weight_min": -21.696876525878906,
      "activations/layer21_attention_weight_max": 32.8996696472168,
      "activations/layer21_attention_weight_min": -20.553251266479492,
      "activations/layer22_attention_weight_max": 30.570323944091797,
      "activations/layer22_attention_weight_min": -27.913999557495117,
      "activations/layer23_attention_weight_max": 36.44337463378906,
      "activations/layer23_attention_weight_min": -23.132938385009766,
      "activations/layer2_attention_weight_max": 30.600181579589844,
      "activations/layer2_attention_weight_min": -28.572975158691406,
      "activations/layer3_attention_weight_max": 82.98259735107422,
      "activations/layer3_attention_weight_min": -84.654296875,
      "activations/layer4_attention_weight_max": 83.00950622558594,
      "activations/layer4_attention_weight_min": -80.29157257080078,
      "activations/layer5_attention_weight_max": 61.399574279785156,
      "activations/layer5_attention_weight_min": -71.2811508178711,
      "activations/layer6_attention_weight_max": 48.92915344238281,
      "activations/layer6_attention_weight_min": -49.14518356323242,
      "activations/layer7_attention_weight_max": 61.22827911376953,
      "activations/layer7_attention_weight_min": -62.6137580871582,
      "activations/layer8_attention_weight_max": 44.85499954223633,
      "activations/layer8_attention_weight_min": -49.599998474121094,
      "activations/layer9_attention_weight_max": 42.16396713256836,
      "activations/layer9_attention_weight_min": -46.95829391479492,
      "epoch": 19.84,
      "learning_rate": 2.2300757575757575e-05,
      "loss": 2.7315,
      "step": 341400
    },
    {
      "activations/layer0_attention_weight_max": 15.952348709106445,
      "activations/layer0_attention_weight_min": -14.090959548950195,
      "activations/layer10_attention_weight_max": 36.110374450683594,
      "activations/layer10_attention_weight_min": -35.95520782470703,
      "activations/layer11_attention_weight_max": 33.54972457885742,
      "activations/layer11_attention_weight_min": -34.39055633544922,
      "activations/layer12_attention_weight_max": 20.088659286499023,
      "activations/layer12_attention_weight_min": -26.658784866333008,
      "activations/layer13_attention_weight_max": 29.67913246154785,
      "activations/layer13_attention_weight_min": -26.33894157409668,
      "activations/layer14_attention_weight_max": 34.78292465209961,
      "activations/layer14_attention_weight_min": -30.519311904907227,
      "activations/layer15_attention_weight_max": 31.8868465423584,
      "activations/layer15_attention_weight_min": -29.305131912231445,
      "activations/layer16_attention_weight_max": 35.4590950012207,
      "activations/layer16_attention_weight_min": -32.82208251953125,
      "activations/layer17_attention_weight_max": 51.537017822265625,
      "activations/layer17_attention_weight_min": -44.20841598510742,
      "activations/layer18_attention_weight_max": 49.87735366821289,
      "activations/layer18_attention_weight_min": -42.27364730834961,
      "activations/layer19_attention_weight_max": 25.199934005737305,
      "activations/layer19_attention_weight_min": -23.90675163269043,
      "activations/layer1_attention_weight_max": 15.524189949035645,
      "activations/layer1_attention_weight_min": -13.421952247619629,
      "activations/layer20_attention_weight_max": 22.724685668945312,
      "activations/layer20_attention_weight_min": -24.454387664794922,
      "activations/layer21_attention_weight_max": 32.47262954711914,
      "activations/layer21_attention_weight_min": -22.83313751220703,
      "activations/layer22_attention_weight_max": 29.559289932250977,
      "activations/layer22_attention_weight_min": -27.35356330871582,
      "activations/layer23_attention_weight_max": 35.622901916503906,
      "activations/layer23_attention_weight_min": -25.59809684753418,
      "activations/layer2_attention_weight_max": 29.725364685058594,
      "activations/layer2_attention_weight_min": -30.271038055419922,
      "activations/layer3_attention_weight_max": 83.01319122314453,
      "activations/layer3_attention_weight_min": -88.4590072631836,
      "activations/layer4_attention_weight_max": 83.65744018554688,
      "activations/layer4_attention_weight_min": -82.51118469238281,
      "activations/layer5_attention_weight_max": 66.0498275756836,
      "activations/layer5_attention_weight_min": -78.17301177978516,
      "activations/layer6_attention_weight_max": 48.95900344848633,
      "activations/layer6_attention_weight_min": -48.096561431884766,
      "activations/layer7_attention_weight_max": 64.15375518798828,
      "activations/layer7_attention_weight_min": -62.73033905029297,
      "activations/layer8_attention_weight_max": 45.27836608886719,
      "activations/layer8_attention_weight_min": -49.92527389526367,
      "activations/layer9_attention_weight_max": 44.20442199707031,
      "activations/layer9_attention_weight_min": -43.90433883666992,
      "epoch": 19.84,
      "learning_rate": 2.228181818181818e-05,
      "loss": 2.7229,
      "step": 341450
    },
    {
      "activations/layer0_attention_weight_max": 15.923666954040527,
      "activations/layer0_attention_weight_min": -13.98215103149414,
      "activations/layer10_attention_weight_max": 34.73114776611328,
      "activations/layer10_attention_weight_min": -36.12248611450195,
      "activations/layer11_attention_weight_max": 33.46955871582031,
      "activations/layer11_attention_weight_min": -33.88389587402344,
      "activations/layer12_attention_weight_max": 19.65026092529297,
      "activations/layer12_attention_weight_min": -26.154062271118164,
      "activations/layer13_attention_weight_max": 35.608154296875,
      "activations/layer13_attention_weight_min": -30.157238006591797,
      "activations/layer14_attention_weight_max": 34.794349670410156,
      "activations/layer14_attention_weight_min": -30.89635467529297,
      "activations/layer15_attention_weight_max": 32.61619186401367,
      "activations/layer15_attention_weight_min": -31.47705078125,
      "activations/layer16_attention_weight_max": 34.842533111572266,
      "activations/layer16_attention_weight_min": -32.61384582519531,
      "activations/layer17_attention_weight_max": 51.40962600708008,
      "activations/layer17_attention_weight_min": -45.29352951049805,
      "activations/layer18_attention_weight_max": 47.453460693359375,
      "activations/layer18_attention_weight_min": -42.22941589355469,
      "activations/layer19_attention_weight_max": 23.32326889038086,
      "activations/layer19_attention_weight_min": -25.180675506591797,
      "activations/layer1_attention_weight_max": 15.591937065124512,
      "activations/layer1_attention_weight_min": -15.298336029052734,
      "activations/layer20_attention_weight_max": 22.71381187438965,
      "activations/layer20_attention_weight_min": -23.7148494720459,
      "activations/layer21_attention_weight_max": 36.7127685546875,
      "activations/layer21_attention_weight_min": -24.59348487854004,
      "activations/layer22_attention_weight_max": 29.67431640625,
      "activations/layer22_attention_weight_min": -26.968095779418945,
      "activations/layer23_attention_weight_max": 37.038719177246094,
      "activations/layer23_attention_weight_min": -25.180309295654297,
      "activations/layer2_attention_weight_max": 30.370994567871094,
      "activations/layer2_attention_weight_min": -29.082202911376953,
      "activations/layer3_attention_weight_max": 84.28034973144531,
      "activations/layer3_attention_weight_min": -82.425048828125,
      "activations/layer4_attention_weight_max": 80.50794982910156,
      "activations/layer4_attention_weight_min": -81.68277740478516,
      "activations/layer5_attention_weight_max": 62.70295333862305,
      "activations/layer5_attention_weight_min": -70.9199447631836,
      "activations/layer6_attention_weight_max": 47.3104362487793,
      "activations/layer6_attention_weight_min": -50.01399230957031,
      "activations/layer7_attention_weight_max": 67.85726928710938,
      "activations/layer7_attention_weight_min": -63.97359085083008,
      "activations/layer8_attention_weight_max": 49.3696174621582,
      "activations/layer8_attention_weight_min": -48.740474700927734,
      "activations/layer9_attention_weight_max": 46.00001907348633,
      "activations/layer9_attention_weight_min": -44.25962448120117,
      "epoch": 19.84,
      "learning_rate": 2.2262878787878785e-05,
      "loss": 2.724,
      "step": 341500
    },
    {
      "activations/layer0_attention_weight_max": 15.364348411560059,
      "activations/layer0_attention_weight_min": -14.094669342041016,
      "activations/layer10_attention_weight_max": 35.93384552001953,
      "activations/layer10_attention_weight_min": -34.440887451171875,
      "activations/layer11_attention_weight_max": 32.92709732055664,
      "activations/layer11_attention_weight_min": -33.62879180908203,
      "activations/layer12_attention_weight_max": 17.749296188354492,
      "activations/layer12_attention_weight_min": -26.01121711730957,
      "activations/layer13_attention_weight_max": 33.134185791015625,
      "activations/layer13_attention_weight_min": -28.753366470336914,
      "activations/layer14_attention_weight_max": 37.187034606933594,
      "activations/layer14_attention_weight_min": -31.20599937438965,
      "activations/layer15_attention_weight_max": 36.72517013549805,
      "activations/layer15_attention_weight_min": -34.15957260131836,
      "activations/layer16_attention_weight_max": 38.29023361206055,
      "activations/layer16_attention_weight_min": -33.81187438964844,
      "activations/layer17_attention_weight_max": 52.665863037109375,
      "activations/layer17_attention_weight_min": -48.20048522949219,
      "activations/layer18_attention_weight_max": 47.250762939453125,
      "activations/layer18_attention_weight_min": -46.305137634277344,
      "activations/layer19_attention_weight_max": 26.58418846130371,
      "activations/layer19_attention_weight_min": -22.7963924407959,
      "activations/layer1_attention_weight_max": 15.636013984680176,
      "activations/layer1_attention_weight_min": -13.446192741394043,
      "activations/layer20_attention_weight_max": 25.387962341308594,
      "activations/layer20_attention_weight_min": -21.929283142089844,
      "activations/layer21_attention_weight_max": 37.233829498291016,
      "activations/layer21_attention_weight_min": -25.12155532836914,
      "activations/layer22_attention_weight_max": 30.93187141418457,
      "activations/layer22_attention_weight_min": -25.89605140686035,
      "activations/layer23_attention_weight_max": 37.94059753417969,
      "activations/layer23_attention_weight_min": -26.382434844970703,
      "activations/layer2_attention_weight_max": 29.444435119628906,
      "activations/layer2_attention_weight_min": -29.025541305541992,
      "activations/layer3_attention_weight_max": 83.7807846069336,
      "activations/layer3_attention_weight_min": -84.9833984375,
      "activations/layer4_attention_weight_max": 81.10967254638672,
      "activations/layer4_attention_weight_min": -80.17008972167969,
      "activations/layer5_attention_weight_max": 60.029579162597656,
      "activations/layer5_attention_weight_min": -75.00286865234375,
      "activations/layer6_attention_weight_max": 46.41164016723633,
      "activations/layer6_attention_weight_min": -49.25064468383789,
      "activations/layer7_attention_weight_max": 67.41431427001953,
      "activations/layer7_attention_weight_min": -64.78404998779297,
      "activations/layer8_attention_weight_max": 46.30137252807617,
      "activations/layer8_attention_weight_min": -48.77291488647461,
      "activations/layer9_attention_weight_max": 48.701202392578125,
      "activations/layer9_attention_weight_min": -48.8687858581543,
      "epoch": 19.85,
      "learning_rate": 2.224393939393939e-05,
      "loss": 2.7285,
      "step": 341550
    },
    {
      "activations/layer0_attention_weight_max": 15.321310043334961,
      "activations/layer0_attention_weight_min": -13.764240264892578,
      "activations/layer10_attention_weight_max": 33.879539489746094,
      "activations/layer10_attention_weight_min": -33.03443145751953,
      "activations/layer11_attention_weight_max": 32.1104736328125,
      "activations/layer11_attention_weight_min": -32.72428894042969,
      "activations/layer12_attention_weight_max": 20.037189483642578,
      "activations/layer12_attention_weight_min": -24.053754806518555,
      "activations/layer13_attention_weight_max": 30.597326278686523,
      "activations/layer13_attention_weight_min": -28.44652557373047,
      "activations/layer14_attention_weight_max": 33.36652374267578,
      "activations/layer14_attention_weight_min": -30.505992889404297,
      "activations/layer15_attention_weight_max": 31.751218795776367,
      "activations/layer15_attention_weight_min": -29.809789657592773,
      "activations/layer16_attention_weight_max": 31.42365264892578,
      "activations/layer16_attention_weight_min": -29.320064544677734,
      "activations/layer17_attention_weight_max": 50.59400177001953,
      "activations/layer17_attention_weight_min": -42.09261703491211,
      "activations/layer18_attention_weight_max": 43.88682556152344,
      "activations/layer18_attention_weight_min": -37.63365936279297,
      "activations/layer19_attention_weight_max": 23.658201217651367,
      "activations/layer19_attention_weight_min": -22.793514251708984,
      "activations/layer1_attention_weight_max": 16.38382911682129,
      "activations/layer1_attention_weight_min": -14.481266975402832,
      "activations/layer20_attention_weight_max": 22.511463165283203,
      "activations/layer20_attention_weight_min": -22.963895797729492,
      "activations/layer21_attention_weight_max": 31.37462615966797,
      "activations/layer21_attention_weight_min": -22.92754364013672,
      "activations/layer22_attention_weight_max": 25.338336944580078,
      "activations/layer22_attention_weight_min": -23.90226173400879,
      "activations/layer23_attention_weight_max": 34.06285858154297,
      "activations/layer23_attention_weight_min": -24.39946746826172,
      "activations/layer2_attention_weight_max": 32.93968963623047,
      "activations/layer2_attention_weight_min": -31.532310485839844,
      "activations/layer3_attention_weight_max": 83.35128784179688,
      "activations/layer3_attention_weight_min": -86.23320770263672,
      "activations/layer4_attention_weight_max": 77.90709686279297,
      "activations/layer4_attention_weight_min": -76.95269775390625,
      "activations/layer5_attention_weight_max": 58.62792205810547,
      "activations/layer5_attention_weight_min": -66.6135025024414,
      "activations/layer6_attention_weight_max": 46.581844329833984,
      "activations/layer6_attention_weight_min": -46.84758377075195,
      "activations/layer7_attention_weight_max": 60.500205993652344,
      "activations/layer7_attention_weight_min": -60.300254821777344,
      "activations/layer8_attention_weight_max": 43.262386322021484,
      "activations/layer8_attention_weight_min": -47.13637924194336,
      "activations/layer9_attention_weight_max": 40.85500717163086,
      "activations/layer9_attention_weight_min": -43.11695861816406,
      "epoch": 19.85,
      "learning_rate": 2.2224999999999998e-05,
      "loss": 2.7021,
      "step": 341600
    },
    {
      "activations/layer0_attention_weight_max": 16.02721405029297,
      "activations/layer0_attention_weight_min": -13.886695861816406,
      "activations/layer10_attention_weight_max": 36.37735366821289,
      "activations/layer10_attention_weight_min": -38.09442138671875,
      "activations/layer11_attention_weight_max": 33.48263931274414,
      "activations/layer11_attention_weight_min": -34.93937683105469,
      "activations/layer12_attention_weight_max": 18.64409637451172,
      "activations/layer12_attention_weight_min": -29.124305725097656,
      "activations/layer13_attention_weight_max": 31.123292922973633,
      "activations/layer13_attention_weight_min": -27.391780853271484,
      "activations/layer14_attention_weight_max": 34.693580627441406,
      "activations/layer14_attention_weight_min": -29.12961196899414,
      "activations/layer15_attention_weight_max": 32.72088623046875,
      "activations/layer15_attention_weight_min": -29.329090118408203,
      "activations/layer16_attention_weight_max": 33.463768005371094,
      "activations/layer16_attention_weight_min": -31.54088592529297,
      "activations/layer17_attention_weight_max": 51.80708312988281,
      "activations/layer17_attention_weight_min": -44.8272590637207,
      "activations/layer18_attention_weight_max": 49.92598342895508,
      "activations/layer18_attention_weight_min": -38.43757247924805,
      "activations/layer19_attention_weight_max": 27.628713607788086,
      "activations/layer19_attention_weight_min": -21.634201049804688,
      "activations/layer1_attention_weight_max": 17.02228355407715,
      "activations/layer1_attention_weight_min": -14.521958351135254,
      "activations/layer20_attention_weight_max": 22.09832000732422,
      "activations/layer20_attention_weight_min": -20.436704635620117,
      "activations/layer21_attention_weight_max": 35.014976501464844,
      "activations/layer21_attention_weight_min": -21.53167724609375,
      "activations/layer22_attention_weight_max": 29.97787857055664,
      "activations/layer22_attention_weight_min": -26.00697898864746,
      "activations/layer23_attention_weight_max": 36.60942840576172,
      "activations/layer23_attention_weight_min": -23.0665283203125,
      "activations/layer2_attention_weight_max": 29.141904830932617,
      "activations/layer2_attention_weight_min": -28.060300827026367,
      "activations/layer3_attention_weight_max": 77.87843322753906,
      "activations/layer3_attention_weight_min": -78.43092346191406,
      "activations/layer4_attention_weight_max": 79.10511016845703,
      "activations/layer4_attention_weight_min": -77.8083724975586,
      "activations/layer5_attention_weight_max": 59.96912384033203,
      "activations/layer5_attention_weight_min": -69.84500885009766,
      "activations/layer6_attention_weight_max": 45.19710922241211,
      "activations/layer6_attention_weight_min": -47.012001037597656,
      "activations/layer7_attention_weight_max": 60.258792877197266,
      "activations/layer7_attention_weight_min": -59.98640441894531,
      "activations/layer8_attention_weight_max": 42.6124153137207,
      "activations/layer8_attention_weight_min": -48.301734924316406,
      "activations/layer9_attention_weight_max": 41.1633415222168,
      "activations/layer9_attention_weight_min": -44.37800216674805,
      "epoch": 19.85,
      "learning_rate": 2.2206060606060603e-05,
      "loss": 2.7287,
      "step": 341650
    },
    {
      "activations/layer0_attention_weight_max": 16.02571678161621,
      "activations/layer0_attention_weight_min": -13.612825393676758,
      "activations/layer10_attention_weight_max": 32.82645034790039,
      "activations/layer10_attention_weight_min": -37.247806549072266,
      "activations/layer11_attention_weight_max": 32.91850280761719,
      "activations/layer11_attention_weight_min": -33.45580291748047,
      "activations/layer12_attention_weight_max": 18.772619247436523,
      "activations/layer12_attention_weight_min": -24.219192504882812,
      "activations/layer13_attention_weight_max": 32.006553649902344,
      "activations/layer13_attention_weight_min": -29.993698120117188,
      "activations/layer14_attention_weight_max": 32.1007194519043,
      "activations/layer14_attention_weight_min": -29.953588485717773,
      "activations/layer15_attention_weight_max": 31.2137451171875,
      "activations/layer15_attention_weight_min": -29.545217514038086,
      "activations/layer16_attention_weight_max": 30.805391311645508,
      "activations/layer16_attention_weight_min": -31.479833602905273,
      "activations/layer17_attention_weight_max": 49.13327407836914,
      "activations/layer17_attention_weight_min": -44.68434143066406,
      "activations/layer18_attention_weight_max": 48.17575454711914,
      "activations/layer18_attention_weight_min": -41.48790740966797,
      "activations/layer19_attention_weight_max": 26.483600616455078,
      "activations/layer19_attention_weight_min": -22.86647605895996,
      "activations/layer1_attention_weight_max": 15.046242713928223,
      "activations/layer1_attention_weight_min": -13.342513084411621,
      "activations/layer20_attention_weight_max": 21.942665100097656,
      "activations/layer20_attention_weight_min": -22.49087905883789,
      "activations/layer21_attention_weight_max": 33.073726654052734,
      "activations/layer21_attention_weight_min": -24.08637046813965,
      "activations/layer22_attention_weight_max": 30.85663414001465,
      "activations/layer22_attention_weight_min": -25.90688133239746,
      "activations/layer23_attention_weight_max": 35.629913330078125,
      "activations/layer23_attention_weight_min": -25.374282836914062,
      "activations/layer2_attention_weight_max": 29.410606384277344,
      "activations/layer2_attention_weight_min": -27.46466827392578,
      "activations/layer3_attention_weight_max": 78.25482940673828,
      "activations/layer3_attention_weight_min": -80.00607299804688,
      "activations/layer4_attention_weight_max": 78.69329071044922,
      "activations/layer4_attention_weight_min": -78.6081314086914,
      "activations/layer5_attention_weight_max": 58.9351806640625,
      "activations/layer5_attention_weight_min": -71.61306762695312,
      "activations/layer6_attention_weight_max": 46.01780700683594,
      "activations/layer6_attention_weight_min": -47.62627029418945,
      "activations/layer7_attention_weight_max": 61.32534408569336,
      "activations/layer7_attention_weight_min": -65.92767333984375,
      "activations/layer8_attention_weight_max": 43.31272506713867,
      "activations/layer8_attention_weight_min": -48.34824752807617,
      "activations/layer9_attention_weight_max": 40.24835205078125,
      "activations/layer9_attention_weight_min": -47.28346252441406,
      "epoch": 19.85,
      "learning_rate": 2.218712121212121e-05,
      "loss": 2.7287,
      "step": 341700
    },
    {
      "activations/layer0_attention_weight_max": 15.915129661560059,
      "activations/layer0_attention_weight_min": -14.465876579284668,
      "activations/layer10_attention_weight_max": 37.531776428222656,
      "activations/layer10_attention_weight_min": -36.242679595947266,
      "activations/layer11_attention_weight_max": 34.41565704345703,
      "activations/layer11_attention_weight_min": -34.575096130371094,
      "activations/layer12_attention_weight_max": 18.490785598754883,
      "activations/layer12_attention_weight_min": -22.785978317260742,
      "activations/layer13_attention_weight_max": 36.752037048339844,
      "activations/layer13_attention_weight_min": -29.261497497558594,
      "activations/layer14_attention_weight_max": 37.1941032409668,
      "activations/layer14_attention_weight_min": -30.48479652404785,
      "activations/layer15_attention_weight_max": 35.04359817504883,
      "activations/layer15_attention_weight_min": -33.71427917480469,
      "activations/layer16_attention_weight_max": 35.89385986328125,
      "activations/layer16_attention_weight_min": -35.41685485839844,
      "activations/layer17_attention_weight_max": 54.07149887084961,
      "activations/layer17_attention_weight_min": -46.94108963012695,
      "activations/layer18_attention_weight_max": 51.50267791748047,
      "activations/layer18_attention_weight_min": -41.086666107177734,
      "activations/layer19_attention_weight_max": 26.00766944885254,
      "activations/layer19_attention_weight_min": -22.0678768157959,
      "activations/layer1_attention_weight_max": 15.740257263183594,
      "activations/layer1_attention_weight_min": -13.261427879333496,
      "activations/layer20_attention_weight_max": 24.889747619628906,
      "activations/layer20_attention_weight_min": -21.888877868652344,
      "activations/layer21_attention_weight_max": 41.58954620361328,
      "activations/layer21_attention_weight_min": -24.960927963256836,
      "activations/layer22_attention_weight_max": 30.370515823364258,
      "activations/layer22_attention_weight_min": -24.884490966796875,
      "activations/layer23_attention_weight_max": 42.18568801879883,
      "activations/layer23_attention_weight_min": -24.84719467163086,
      "activations/layer2_attention_weight_max": 31.878795623779297,
      "activations/layer2_attention_weight_min": -30.970455169677734,
      "activations/layer3_attention_weight_max": 86.87995910644531,
      "activations/layer3_attention_weight_min": -89.02127075195312,
      "activations/layer4_attention_weight_max": 84.68623352050781,
      "activations/layer4_attention_weight_min": -84.62745666503906,
      "activations/layer5_attention_weight_max": 64.18531036376953,
      "activations/layer5_attention_weight_min": -70.86529541015625,
      "activations/layer6_attention_weight_max": 50.86095428466797,
      "activations/layer6_attention_weight_min": -51.89369583129883,
      "activations/layer7_attention_weight_max": 66.05255889892578,
      "activations/layer7_attention_weight_min": -66.37445831298828,
      "activations/layer8_attention_weight_max": 49.91826248168945,
      "activations/layer8_attention_weight_min": -51.15229034423828,
      "activations/layer9_attention_weight_max": 44.980979919433594,
      "activations/layer9_attention_weight_min": -49.40140914916992,
      "epoch": 19.86,
      "learning_rate": 2.216818181818182e-05,
      "loss": 2.7181,
      "step": 341750
    },
    {
      "activations/layer0_attention_weight_max": 16.36553955078125,
      "activations/layer0_attention_weight_min": -13.936392784118652,
      "activations/layer10_attention_weight_max": 35.7605094909668,
      "activations/layer10_attention_weight_min": -36.179443359375,
      "activations/layer11_attention_weight_max": 35.41140365600586,
      "activations/layer11_attention_weight_min": -36.15477752685547,
      "activations/layer12_attention_weight_max": 18.774333953857422,
      "activations/layer12_attention_weight_min": -23.79436492919922,
      "activations/layer13_attention_weight_max": 37.215599060058594,
      "activations/layer13_attention_weight_min": -28.78289794921875,
      "activations/layer14_attention_weight_max": 34.412193298339844,
      "activations/layer14_attention_weight_min": -32.51058578491211,
      "activations/layer15_attention_weight_max": 32.01413345336914,
      "activations/layer15_attention_weight_min": -31.793912887573242,
      "activations/layer16_attention_weight_max": 35.50168228149414,
      "activations/layer16_attention_weight_min": -33.61756134033203,
      "activations/layer17_attention_weight_max": 54.625431060791016,
      "activations/layer17_attention_weight_min": -51.128265380859375,
      "activations/layer18_attention_weight_max": 51.62470626831055,
      "activations/layer18_attention_weight_min": -47.427818298339844,
      "activations/layer19_attention_weight_max": 25.94964027404785,
      "activations/layer19_attention_weight_min": -26.940763473510742,
      "activations/layer1_attention_weight_max": 16.04313850402832,
      "activations/layer1_attention_weight_min": -13.817832946777344,
      "activations/layer20_attention_weight_max": 23.857372283935547,
      "activations/layer20_attention_weight_min": -24.790569305419922,
      "activations/layer21_attention_weight_max": 35.86221694946289,
      "activations/layer21_attention_weight_min": -23.897930145263672,
      "activations/layer22_attention_weight_max": 31.11163902282715,
      "activations/layer22_attention_weight_min": -27.19625473022461,
      "activations/layer23_attention_weight_max": 39.18818664550781,
      "activations/layer23_attention_weight_min": -24.52338409423828,
      "activations/layer2_attention_weight_max": 29.631084442138672,
      "activations/layer2_attention_weight_min": -27.500896453857422,
      "activations/layer3_attention_weight_max": 82.44783020019531,
      "activations/layer3_attention_weight_min": -82.5103759765625,
      "activations/layer4_attention_weight_max": 80.3025131225586,
      "activations/layer4_attention_weight_min": -79.92823028564453,
      "activations/layer5_attention_weight_max": 59.26455307006836,
      "activations/layer5_attention_weight_min": -68.32809448242188,
      "activations/layer6_attention_weight_max": 48.83165740966797,
      "activations/layer6_attention_weight_min": -50.45697784423828,
      "activations/layer7_attention_weight_max": 64.49832153320312,
      "activations/layer7_attention_weight_min": -64.8994140625,
      "activations/layer8_attention_weight_max": 46.942630767822266,
      "activations/layer8_attention_weight_min": -51.50307846069336,
      "activations/layer9_attention_weight_max": 44.43131637573242,
      "activations/layer9_attention_weight_min": -44.43285369873047,
      "epoch": 19.86,
      "learning_rate": 2.214924242424242e-05,
      "loss": 2.7376,
      "step": 341800
    },
    {
      "activations/layer0_attention_weight_max": 15.54127311706543,
      "activations/layer0_attention_weight_min": -13.656485557556152,
      "activations/layer10_attention_weight_max": 36.81761932373047,
      "activations/layer10_attention_weight_min": -37.17786407470703,
      "activations/layer11_attention_weight_max": 32.83213806152344,
      "activations/layer11_attention_weight_min": -36.28089904785156,
      "activations/layer12_attention_weight_max": 19.984582901000977,
      "activations/layer12_attention_weight_min": -23.81911849975586,
      "activations/layer13_attention_weight_max": 36.66607666015625,
      "activations/layer13_attention_weight_min": -28.627965927124023,
      "activations/layer14_attention_weight_max": 36.84097671508789,
      "activations/layer14_attention_weight_min": -30.49523162841797,
      "activations/layer15_attention_weight_max": 33.16987991333008,
      "activations/layer15_attention_weight_min": -30.277830123901367,
      "activations/layer16_attention_weight_max": 33.6574592590332,
      "activations/layer16_attention_weight_min": -31.821590423583984,
      "activations/layer17_attention_weight_max": 50.36262130737305,
      "activations/layer17_attention_weight_min": -44.11448669433594,
      "activations/layer18_attention_weight_max": 47.391056060791016,
      "activations/layer18_attention_weight_min": -40.27177047729492,
      "activations/layer19_attention_weight_max": 28.194721221923828,
      "activations/layer19_attention_weight_min": -24.773061752319336,
      "activations/layer1_attention_weight_max": 15.39748477935791,
      "activations/layer1_attention_weight_min": -13.837376594543457,
      "activations/layer20_attention_weight_max": 26.327512741088867,
      "activations/layer20_attention_weight_min": -23.349260330200195,
      "activations/layer21_attention_weight_max": 39.1646728515625,
      "activations/layer21_attention_weight_min": -26.527873992919922,
      "activations/layer22_attention_weight_max": 29.380971908569336,
      "activations/layer22_attention_weight_min": -27.783512115478516,
      "activations/layer23_attention_weight_max": 36.62415313720703,
      "activations/layer23_attention_weight_min": -24.621902465820312,
      "activations/layer2_attention_weight_max": 29.396286010742188,
      "activations/layer2_attention_weight_min": -29.99913787841797,
      "activations/layer3_attention_weight_max": 83.810302734375,
      "activations/layer3_attention_weight_min": -89.4938735961914,
      "activations/layer4_attention_weight_max": 80.00438690185547,
      "activations/layer4_attention_weight_min": -82.07794189453125,
      "activations/layer5_attention_weight_max": 57.708290100097656,
      "activations/layer5_attention_weight_min": -72.73155975341797,
      "activations/layer6_attention_weight_max": 47.58327865600586,
      "activations/layer6_attention_weight_min": -50.68180465698242,
      "activations/layer7_attention_weight_max": 62.3986701965332,
      "activations/layer7_attention_weight_min": -64.66327667236328,
      "activations/layer8_attention_weight_max": 44.24454116821289,
      "activations/layer8_attention_weight_min": -50.92012405395508,
      "activations/layer9_attention_weight_max": 42.87885665893555,
      "activations/layer9_attention_weight_min": -47.743873596191406,
      "epoch": 19.86,
      "learning_rate": 2.213068181818182e-05,
      "loss": 2.7214,
      "step": 341850
    },
    {
      "activations/layer0_attention_weight_max": 15.319170951843262,
      "activations/layer0_attention_weight_min": -13.997947692871094,
      "activations/layer10_attention_weight_max": 36.93144226074219,
      "activations/layer10_attention_weight_min": -37.687110900878906,
      "activations/layer11_attention_weight_max": 35.31402587890625,
      "activations/layer11_attention_weight_min": -34.980934143066406,
      "activations/layer12_attention_weight_max": 19.069822311401367,
      "activations/layer12_attention_weight_min": -24.7844181060791,
      "activations/layer13_attention_weight_max": 34.44651412963867,
      "activations/layer13_attention_weight_min": -27.008312225341797,
      "activations/layer14_attention_weight_max": 33.19112014770508,
      "activations/layer14_attention_weight_min": -29.423828125,
      "activations/layer15_attention_weight_max": 32.10922622680664,
      "activations/layer15_attention_weight_min": -32.46670913696289,
      "activations/layer16_attention_weight_max": 32.763607025146484,
      "activations/layer16_attention_weight_min": -31.836633682250977,
      "activations/layer17_attention_weight_max": 50.25092697143555,
      "activations/layer17_attention_weight_min": -45.14619445800781,
      "activations/layer18_attention_weight_max": 44.67711639404297,
      "activations/layer18_attention_weight_min": -39.46607971191406,
      "activations/layer19_attention_weight_max": 24.156789779663086,
      "activations/layer19_attention_weight_min": -23.186710357666016,
      "activations/layer1_attention_weight_max": 15.114200592041016,
      "activations/layer1_attention_weight_min": -14.895467758178711,
      "activations/layer20_attention_weight_max": 22.340866088867188,
      "activations/layer20_attention_weight_min": -20.074644088745117,
      "activations/layer21_attention_weight_max": 39.88210678100586,
      "activations/layer21_attention_weight_min": -24.023733139038086,
      "activations/layer22_attention_weight_max": 32.14776611328125,
      "activations/layer22_attention_weight_min": -25.641864776611328,
      "activations/layer23_attention_weight_max": 34.87297821044922,
      "activations/layer23_attention_weight_min": -25.456960678100586,
      "activations/layer2_attention_weight_max": 30.93138885498047,
      "activations/layer2_attention_weight_min": -29.597328186035156,
      "activations/layer3_attention_weight_max": 82.09264373779297,
      "activations/layer3_attention_weight_min": -79.59748840332031,
      "activations/layer4_attention_weight_max": 82.21476745605469,
      "activations/layer4_attention_weight_min": -80.01022338867188,
      "activations/layer5_attention_weight_max": 62.173736572265625,
      "activations/layer5_attention_weight_min": -69.85465240478516,
      "activations/layer6_attention_weight_max": 47.32218551635742,
      "activations/layer6_attention_weight_min": -49.35801696777344,
      "activations/layer7_attention_weight_max": 65.68827819824219,
      "activations/layer7_attention_weight_min": -66.38970184326172,
      "activations/layer8_attention_weight_max": 46.47922897338867,
      "activations/layer8_attention_weight_min": -48.573097229003906,
      "activations/layer9_attention_weight_max": 49.01008987426758,
      "activations/layer9_attention_weight_min": -47.322879791259766,
      "epoch": 19.87,
      "learning_rate": 2.2111742424242424e-05,
      "loss": 2.7315,
      "step": 341900
    },
    {
      "activations/layer0_attention_weight_max": 16.421903610229492,
      "activations/layer0_attention_weight_min": -13.871063232421875,
      "activations/layer10_attention_weight_max": 32.99846267700195,
      "activations/layer10_attention_weight_min": -33.347930908203125,
      "activations/layer11_attention_weight_max": 33.27399444580078,
      "activations/layer11_attention_weight_min": -33.13090515136719,
      "activations/layer12_attention_weight_max": 18.426572799682617,
      "activations/layer12_attention_weight_min": -30.803726196289062,
      "activations/layer13_attention_weight_max": 30.7189884185791,
      "activations/layer13_attention_weight_min": -27.40921401977539,
      "activations/layer14_attention_weight_max": 32.856937408447266,
      "activations/layer14_attention_weight_min": -29.18829917907715,
      "activations/layer15_attention_weight_max": 29.756059646606445,
      "activations/layer15_attention_weight_min": -30.024293899536133,
      "activations/layer16_attention_weight_max": 33.16653823852539,
      "activations/layer16_attention_weight_min": -31.501676559448242,
      "activations/layer17_attention_weight_max": 47.76242446899414,
      "activations/layer17_attention_weight_min": -43.5204963684082,
      "activations/layer18_attention_weight_max": 46.93779373168945,
      "activations/layer18_attention_weight_min": -38.78718566894531,
      "activations/layer19_attention_weight_max": 21.71770477294922,
      "activations/layer19_attention_weight_min": -21.48579978942871,
      "activations/layer1_attention_weight_max": 14.901311874389648,
      "activations/layer1_attention_weight_min": -14.22462272644043,
      "activations/layer20_attention_weight_max": 20.38033103942871,
      "activations/layer20_attention_weight_min": -21.9164981842041,
      "activations/layer21_attention_weight_max": 33.909271240234375,
      "activations/layer21_attention_weight_min": -20.683523178100586,
      "activations/layer22_attention_weight_max": 28.670915603637695,
      "activations/layer22_attention_weight_min": -23.827547073364258,
      "activations/layer23_attention_weight_max": 36.06574249267578,
      "activations/layer23_attention_weight_min": -24.533321380615234,
      "activations/layer2_attention_weight_max": 30.81014060974121,
      "activations/layer2_attention_weight_min": -28.197410583496094,
      "activations/layer3_attention_weight_max": 90.93423461914062,
      "activations/layer3_attention_weight_min": -82.32256317138672,
      "activations/layer4_attention_weight_max": 81.23822784423828,
      "activations/layer4_attention_weight_min": -78.94248962402344,
      "activations/layer5_attention_weight_max": 56.42387390136719,
      "activations/layer5_attention_weight_min": -68.31088256835938,
      "activations/layer6_attention_weight_max": 49.42148971557617,
      "activations/layer6_attention_weight_min": -47.264408111572266,
      "activations/layer7_attention_weight_max": 61.15499496459961,
      "activations/layer7_attention_weight_min": -61.96474075317383,
      "activations/layer8_attention_weight_max": 43.18041229248047,
      "activations/layer8_attention_weight_min": -46.658599853515625,
      "activations/layer9_attention_weight_max": 48.743343353271484,
      "activations/layer9_attention_weight_min": -45.32005310058594,
      "epoch": 19.87,
      "learning_rate": 2.2092803030303032e-05,
      "loss": 2.7469,
      "step": 341950
    },
    {
      "activations/layer0_attention_weight_max": 15.47391414642334,
      "activations/layer0_attention_weight_min": -14.129408836364746,
      "activations/layer10_attention_weight_max": 36.10072326660156,
      "activations/layer10_attention_weight_min": -37.92805862426758,
      "activations/layer11_attention_weight_max": 35.14409637451172,
      "activations/layer11_attention_weight_min": -34.78678894042969,
      "activations/layer12_attention_weight_max": 18.026020050048828,
      "activations/layer12_attention_weight_min": -27.25905990600586,
      "activations/layer13_attention_weight_max": 37.27849197387695,
      "activations/layer13_attention_weight_min": -33.4600830078125,
      "activations/layer14_attention_weight_max": 37.95798873901367,
      "activations/layer14_attention_weight_min": -33.901756286621094,
      "activations/layer15_attention_weight_max": 37.221717834472656,
      "activations/layer15_attention_weight_min": -33.909095764160156,
      "activations/layer16_attention_weight_max": 35.426300048828125,
      "activations/layer16_attention_weight_min": -32.536617279052734,
      "activations/layer17_attention_weight_max": 52.72018051147461,
      "activations/layer17_attention_weight_min": -46.18289566040039,
      "activations/layer18_attention_weight_max": 50.62399673461914,
      "activations/layer18_attention_weight_min": -41.078956604003906,
      "activations/layer19_attention_weight_max": 25.95899772644043,
      "activations/layer19_attention_weight_min": -21.559812545776367,
      "activations/layer1_attention_weight_max": 15.825936317443848,
      "activations/layer1_attention_weight_min": -12.665031433105469,
      "activations/layer20_attention_weight_max": 22.377193450927734,
      "activations/layer20_attention_weight_min": -22.755386352539062,
      "activations/layer21_attention_weight_max": 39.83766555786133,
      "activations/layer21_attention_weight_min": -24.488304138183594,
      "activations/layer22_attention_weight_max": 29.098527908325195,
      "activations/layer22_attention_weight_min": -25.496356964111328,
      "activations/layer23_attention_weight_max": 39.62150573730469,
      "activations/layer23_attention_weight_min": -25.007503509521484,
      "activations/layer2_attention_weight_max": 31.410072326660156,
      "activations/layer2_attention_weight_min": -29.01117515563965,
      "activations/layer3_attention_weight_max": 84.92414855957031,
      "activations/layer3_attention_weight_min": -83.93984985351562,
      "activations/layer4_attention_weight_max": 83.21299743652344,
      "activations/layer4_attention_weight_min": -85.09575653076172,
      "activations/layer5_attention_weight_max": 63.195709228515625,
      "activations/layer5_attention_weight_min": -72.58341979980469,
      "activations/layer6_attention_weight_max": 49.57353210449219,
      "activations/layer6_attention_weight_min": -49.077171325683594,
      "activations/layer7_attention_weight_max": 62.41130065917969,
      "activations/layer7_attention_weight_min": -67.17435455322266,
      "activations/layer8_attention_weight_max": 47.33681106567383,
      "activations/layer8_attention_weight_min": -50.59048080444336,
      "activations/layer9_attention_weight_max": 43.506832122802734,
      "activations/layer9_attention_weight_min": -46.79391098022461,
      "epoch": 19.87,
      "learning_rate": 2.2073863636363633e-05,
      "loss": 2.7322,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.5499,
      "eval_samples_per_second": 502.227,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.5499,
      "eval_openwebtext_samples_per_second": 502.227,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 2.0303,
      "eval_wikitext_samples_per_second": 224.599,
      "step": 342000
    },
    {
      "epoch": 19.87,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.5865,
      "eval_lambada_samples_per_second": 507.903,
      "step": 342000
    },
    {
      "activations/layer0_attention_weight_max": 15.515480995178223,
      "activations/layer0_attention_weight_min": -14.602926254272461,
      "activations/layer10_attention_weight_max": 35.20625686645508,
      "activations/layer10_attention_weight_min": -36.037841796875,
      "activations/layer11_attention_weight_max": 32.27488708496094,
      "activations/layer11_attention_weight_min": -36.21240234375,
      "activations/layer12_attention_weight_max": 20.0313663482666,
      "activations/layer12_attention_weight_min": -23.182634353637695,
      "activations/layer13_attention_weight_max": 42.32636260986328,
      "activations/layer13_attention_weight_min": -32.36888122558594,
      "activations/layer14_attention_weight_max": 36.38882064819336,
      "activations/layer14_attention_weight_min": -30.95988655090332,
      "activations/layer15_attention_weight_max": 36.03392791748047,
      "activations/layer15_attention_weight_min": -31.905969619750977,
      "activations/layer16_attention_weight_max": 35.925804138183594,
      "activations/layer16_attention_weight_min": -34.091670989990234,
      "activations/layer17_attention_weight_max": 55.547401428222656,
      "activations/layer17_attention_weight_min": -50.44295883178711,
      "activations/layer18_attention_weight_max": 50.26185607910156,
      "activations/layer18_attention_weight_min": -42.90982437133789,
      "activations/layer19_attention_weight_max": 25.376983642578125,
      "activations/layer19_attention_weight_min": -23.540708541870117,
      "activations/layer1_attention_weight_max": 15.426965713500977,
      "activations/layer1_attention_weight_min": -13.198262214660645,
      "activations/layer20_attention_weight_max": 23.80443000793457,
      "activations/layer20_attention_weight_min": -22.616718292236328,
      "activations/layer21_attention_weight_max": 40.02967834472656,
      "activations/layer21_attention_weight_min": -23.924596786499023,
      "activations/layer22_attention_weight_max": 31.76384162902832,
      "activations/layer22_attention_weight_min": -26.63595199584961,
      "activations/layer23_attention_weight_max": 42.95254135131836,
      "activations/layer23_attention_weight_min": -24.411788940429688,
      "activations/layer2_attention_weight_max": 28.936323165893555,
      "activations/layer2_attention_weight_min": -29.29871368408203,
      "activations/layer3_attention_weight_max": 79.48009490966797,
      "activations/layer3_attention_weight_min": -79.48514556884766,
      "activations/layer4_attention_weight_max": 79.77110290527344,
      "activations/layer4_attention_weight_min": -80.73625946044922,
      "activations/layer5_attention_weight_max": 60.83340072631836,
      "activations/layer5_attention_weight_min": -70.64244079589844,
      "activations/layer6_attention_weight_max": 46.61162567138672,
      "activations/layer6_attention_weight_min": -47.755924224853516,
      "activations/layer7_attention_weight_max": 68.46330261230469,
      "activations/layer7_attention_weight_min": -64.91942596435547,
      "activations/layer8_attention_weight_max": 45.424739837646484,
      "activations/layer8_attention_weight_min": -50.2044563293457,
      "activations/layer9_attention_weight_max": 41.318599700927734,
      "activations/layer9_attention_weight_min": -45.950294494628906,
      "epoch": 19.88,
      "learning_rate": 2.205492424242424e-05,
      "loss": 2.723,
      "step": 342050
    },
    {
      "activations/layer0_attention_weight_max": 15.311705589294434,
      "activations/layer0_attention_weight_min": -13.952692985534668,
      "activations/layer10_attention_weight_max": 36.20181655883789,
      "activations/layer10_attention_weight_min": -37.404273986816406,
      "activations/layer11_attention_weight_max": 36.00541687011719,
      "activations/layer11_attention_weight_min": -34.350074768066406,
      "activations/layer12_attention_weight_max": 21.10144805908203,
      "activations/layer12_attention_weight_min": -26.592084884643555,
      "activations/layer13_attention_weight_max": 31.425010681152344,
      "activations/layer13_attention_weight_min": -25.849973678588867,
      "activations/layer14_attention_weight_max": 34.05835723876953,
      "activations/layer14_attention_weight_min": -30.023710250854492,
      "activations/layer15_attention_weight_max": 32.3642578125,
      "activations/layer15_attention_weight_min": -31.373401641845703,
      "activations/layer16_attention_weight_max": 34.63056182861328,
      "activations/layer16_attention_weight_min": -32.41162872314453,
      "activations/layer17_attention_weight_max": 54.74480056762695,
      "activations/layer17_attention_weight_min": -46.62953567504883,
      "activations/layer18_attention_weight_max": 48.00274658203125,
      "activations/layer18_attention_weight_min": -40.90370559692383,
      "activations/layer19_attention_weight_max": 21.45357894897461,
      "activations/layer19_attention_weight_min": -22.01783561706543,
      "activations/layer1_attention_weight_max": 15.475081443786621,
      "activations/layer1_attention_weight_min": -14.959674835205078,
      "activations/layer20_attention_weight_max": 19.553434371948242,
      "activations/layer20_attention_weight_min": -21.373565673828125,
      "activations/layer21_attention_weight_max": 31.02924156188965,
      "activations/layer21_attention_weight_min": -21.82779884338379,
      "activations/layer22_attention_weight_max": 26.71733283996582,
      "activations/layer22_attention_weight_min": -27.3782901763916,
      "activations/layer23_attention_weight_max": 34.617950439453125,
      "activations/layer23_attention_weight_min": -27.582683563232422,
      "activations/layer2_attention_weight_max": 31.765409469604492,
      "activations/layer2_attention_weight_min": -29.974586486816406,
      "activations/layer3_attention_weight_max": 83.63514709472656,
      "activations/layer3_attention_weight_min": -84.23384857177734,
      "activations/layer4_attention_weight_max": 82.11473846435547,
      "activations/layer4_attention_weight_min": -83.9981918334961,
      "activations/layer5_attention_weight_max": 60.70984649658203,
      "activations/layer5_attention_weight_min": -70.31470489501953,
      "activations/layer6_attention_weight_max": 48.447513580322266,
      "activations/layer6_attention_weight_min": -51.991275787353516,
      "activations/layer7_attention_weight_max": 66.40416717529297,
      "activations/layer7_attention_weight_min": -63.50006103515625,
      "activations/layer8_attention_weight_max": 45.34230041503906,
      "activations/layer8_attention_weight_min": -49.5765495300293,
      "activations/layer9_attention_weight_max": 46.43284225463867,
      "activations/layer9_attention_weight_min": -48.16217041015625,
      "epoch": 19.88,
      "learning_rate": 2.2035984848484846e-05,
      "loss": 2.7247,
      "step": 342100
    },
    {
      "activations/layer0_attention_weight_max": 15.196837425231934,
      "activations/layer0_attention_weight_min": -14.467403411865234,
      "activations/layer10_attention_weight_max": 41.38361740112305,
      "activations/layer10_attention_weight_min": -39.98667526245117,
      "activations/layer11_attention_weight_max": 40.01850509643555,
      "activations/layer11_attention_weight_min": -39.00273132324219,
      "activations/layer12_attention_weight_max": 21.201507568359375,
      "activations/layer12_attention_weight_min": -23.500816345214844,
      "activations/layer13_attention_weight_max": 34.87664031982422,
      "activations/layer13_attention_weight_min": -27.619014739990234,
      "activations/layer14_attention_weight_max": 34.48028564453125,
      "activations/layer14_attention_weight_min": -30.4525089263916,
      "activations/layer15_attention_weight_max": 37.934814453125,
      "activations/layer15_attention_weight_min": -30.560409545898438,
      "activations/layer16_attention_weight_max": 36.77227020263672,
      "activations/layer16_attention_weight_min": -35.21358108520508,
      "activations/layer17_attention_weight_max": 51.29320526123047,
      "activations/layer17_attention_weight_min": -46.689083099365234,
      "activations/layer18_attention_weight_max": 49.08817672729492,
      "activations/layer18_attention_weight_min": -43.09996795654297,
      "activations/layer19_attention_weight_max": 26.442096710205078,
      "activations/layer19_attention_weight_min": -24.607702255249023,
      "activations/layer1_attention_weight_max": 16.160215377807617,
      "activations/layer1_attention_weight_min": -15.144448280334473,
      "activations/layer20_attention_weight_max": 25.814489364624023,
      "activations/layer20_attention_weight_min": -24.016393661499023,
      "activations/layer21_attention_weight_max": 33.867008209228516,
      "activations/layer21_attention_weight_min": -24.122234344482422,
      "activations/layer22_attention_weight_max": 28.485214233398438,
      "activations/layer22_attention_weight_min": -27.396583557128906,
      "activations/layer23_attention_weight_max": 37.09716033935547,
      "activations/layer23_attention_weight_min": -24.304588317871094,
      "activations/layer2_attention_weight_max": 31.698734283447266,
      "activations/layer2_attention_weight_min": -30.86469078063965,
      "activations/layer3_attention_weight_max": 85.5636978149414,
      "activations/layer3_attention_weight_min": -87.16326141357422,
      "activations/layer4_attention_weight_max": 83.83421325683594,
      "activations/layer4_attention_weight_min": -83.2806625366211,
      "activations/layer5_attention_weight_max": 63.59630584716797,
      "activations/layer5_attention_weight_min": -69.20578002929688,
      "activations/layer6_attention_weight_max": 48.07386016845703,
      "activations/layer6_attention_weight_min": -49.17924118041992,
      "activations/layer7_attention_weight_max": 75.75497436523438,
      "activations/layer7_attention_weight_min": -70.31092071533203,
      "activations/layer8_attention_weight_max": 52.49872970581055,
      "activations/layer8_attention_weight_min": -53.025482177734375,
      "activations/layer9_attention_weight_max": 60.658668518066406,
      "activations/layer9_attention_weight_min": -50.14904022216797,
      "epoch": 19.88,
      "learning_rate": 2.2017045454545455e-05,
      "loss": 2.7263,
      "step": 342150
    },
    {
      "activations/layer0_attention_weight_max": 16.0048828125,
      "activations/layer0_attention_weight_min": -13.946905136108398,
      "activations/layer10_attention_weight_max": 44.52450180053711,
      "activations/layer10_attention_weight_min": -42.475006103515625,
      "activations/layer11_attention_weight_max": 40.19080352783203,
      "activations/layer11_attention_weight_min": -39.40205383300781,
      "activations/layer12_attention_weight_max": 21.861427307128906,
      "activations/layer12_attention_weight_min": -25.848621368408203,
      "activations/layer13_attention_weight_max": 39.32219696044922,
      "activations/layer13_attention_weight_min": -34.53831481933594,
      "activations/layer14_attention_weight_max": 40.89003372192383,
      "activations/layer14_attention_weight_min": -37.33737564086914,
      "activations/layer15_attention_weight_max": 37.27739334106445,
      "activations/layer15_attention_weight_min": -33.35483169555664,
      "activations/layer16_attention_weight_max": 38.699092864990234,
      "activations/layer16_attention_weight_min": -36.64246368408203,
      "activations/layer17_attention_weight_max": 58.52899932861328,
      "activations/layer17_attention_weight_min": -51.91324996948242,
      "activations/layer18_attention_weight_max": 51.659629821777344,
      "activations/layer18_attention_weight_min": -47.59074020385742,
      "activations/layer19_attention_weight_max": 25.8376407623291,
      "activations/layer19_attention_weight_min": -24.814870834350586,
      "activations/layer1_attention_weight_max": 16.24245262145996,
      "activations/layer1_attention_weight_min": -14.3975248336792,
      "activations/layer20_attention_weight_max": 24.99190330505371,
      "activations/layer20_attention_weight_min": -22.843223571777344,
      "activations/layer21_attention_weight_max": 35.02994918823242,
      "activations/layer21_attention_weight_min": -23.749834060668945,
      "activations/layer22_attention_weight_max": 27.29946517944336,
      "activations/layer22_attention_weight_min": -25.60472869873047,
      "activations/layer23_attention_weight_max": 37.411415100097656,
      "activations/layer23_attention_weight_min": -24.966629028320312,
      "activations/layer2_attention_weight_max": 31.50821304321289,
      "activations/layer2_attention_weight_min": -30.549468994140625,
      "activations/layer3_attention_weight_max": 85.4401626586914,
      "activations/layer3_attention_weight_min": -87.56310272216797,
      "activations/layer4_attention_weight_max": 78.3057632446289,
      "activations/layer4_attention_weight_min": -81.0630111694336,
      "activations/layer5_attention_weight_max": 61.92607116699219,
      "activations/layer5_attention_weight_min": -66.64075469970703,
      "activations/layer6_attention_weight_max": 47.25825881958008,
      "activations/layer6_attention_weight_min": -49.01699447631836,
      "activations/layer7_attention_weight_max": 74.74005126953125,
      "activations/layer7_attention_weight_min": -63.30131912231445,
      "activations/layer8_attention_weight_max": 54.86590576171875,
      "activations/layer8_attention_weight_min": -55.29863357543945,
      "activations/layer9_attention_weight_max": 55.36832809448242,
      "activations/layer9_attention_weight_min": -48.51728439331055,
      "epoch": 19.88,
      "learning_rate": 2.1998106060606056e-05,
      "loss": 2.7382,
      "step": 342200
    },
    {
      "activations/layer0_attention_weight_max": 15.326919555664062,
      "activations/layer0_attention_weight_min": -14.150529861450195,
      "activations/layer10_attention_weight_max": 33.39986038208008,
      "activations/layer10_attention_weight_min": -35.75238037109375,
      "activations/layer11_attention_weight_max": 34.64330291748047,
      "activations/layer11_attention_weight_min": -33.45973587036133,
      "activations/layer12_attention_weight_max": 18.22772216796875,
      "activations/layer12_attention_weight_min": -25.921171188354492,
      "activations/layer13_attention_weight_max": 33.05043029785156,
      "activations/layer13_attention_weight_min": -26.31511116027832,
      "activations/layer14_attention_weight_max": 33.22512435913086,
      "activations/layer14_attention_weight_min": -28.36311149597168,
      "activations/layer15_attention_weight_max": 33.87514877319336,
      "activations/layer15_attention_weight_min": -29.959646224975586,
      "activations/layer16_attention_weight_max": 34.35317611694336,
      "activations/layer16_attention_weight_min": -31.046268463134766,
      "activations/layer17_attention_weight_max": 57.8412971496582,
      "activations/layer17_attention_weight_min": -48.19857406616211,
      "activations/layer18_attention_weight_max": 49.29727554321289,
      "activations/layer18_attention_weight_min": -41.67953872680664,
      "activations/layer19_attention_weight_max": 23.86127281188965,
      "activations/layer19_attention_weight_min": -23.15964698791504,
      "activations/layer1_attention_weight_max": 16.251670837402344,
      "activations/layer1_attention_weight_min": -13.590869903564453,
      "activations/layer20_attention_weight_max": 25.624046325683594,
      "activations/layer20_attention_weight_min": -22.88321304321289,
      "activations/layer21_attention_weight_max": 35.19572448730469,
      "activations/layer21_attention_weight_min": -21.907102584838867,
      "activations/layer22_attention_weight_max": 32.652286529541016,
      "activations/layer22_attention_weight_min": -26.7468318939209,
      "activations/layer23_attention_weight_max": 36.63499450683594,
      "activations/layer23_attention_weight_min": -24.639198303222656,
      "activations/layer2_attention_weight_max": 31.060440063476562,
      "activations/layer2_attention_weight_min": -31.206188201904297,
      "activations/layer3_attention_weight_max": 89.04075622558594,
      "activations/layer3_attention_weight_min": -89.17737579345703,
      "activations/layer4_attention_weight_max": 82.36263275146484,
      "activations/layer4_attention_weight_min": -83.65765380859375,
      "activations/layer5_attention_weight_max": 59.978607177734375,
      "activations/layer5_attention_weight_min": -69.6807632446289,
      "activations/layer6_attention_weight_max": 46.78376388549805,
      "activations/layer6_attention_weight_min": -47.847286224365234,
      "activations/layer7_attention_weight_max": 62.236385345458984,
      "activations/layer7_attention_weight_min": -64.7584457397461,
      "activations/layer8_attention_weight_max": 46.39338302612305,
      "activations/layer8_attention_weight_min": -47.50077819824219,
      "activations/layer9_attention_weight_max": 48.1560173034668,
      "activations/layer9_attention_weight_min": -46.34664535522461,
      "epoch": 19.89,
      "learning_rate": 2.1979166666666664e-05,
      "loss": 2.744,
      "step": 342250
    },
    {
      "activations/layer0_attention_weight_max": 14.933406829833984,
      "activations/layer0_attention_weight_min": -13.940937042236328,
      "activations/layer10_attention_weight_max": 34.381561279296875,
      "activations/layer10_attention_weight_min": -34.569313049316406,
      "activations/layer11_attention_weight_max": 34.720672607421875,
      "activations/layer11_attention_weight_min": -34.34343719482422,
      "activations/layer12_attention_weight_max": 20.017566680908203,
      "activations/layer12_attention_weight_min": -25.621498107910156,
      "activations/layer13_attention_weight_max": 29.881996154785156,
      "activations/layer13_attention_weight_min": -26.325929641723633,
      "activations/layer14_attention_weight_max": 34.7755126953125,
      "activations/layer14_attention_weight_min": -28.593889236450195,
      "activations/layer15_attention_weight_max": 33.06382751464844,
      "activations/layer15_attention_weight_min": -29.17200469970703,
      "activations/layer16_attention_weight_max": 35.41166305541992,
      "activations/layer16_attention_weight_min": -34.61098098754883,
      "activations/layer17_attention_weight_max": 48.612300872802734,
      "activations/layer17_attention_weight_min": -45.782676696777344,
      "activations/layer18_attention_weight_max": 45.15523910522461,
      "activations/layer18_attention_weight_min": -39.83609390258789,
      "activations/layer19_attention_weight_max": 22.9163875579834,
      "activations/layer19_attention_weight_min": -23.881319046020508,
      "activations/layer1_attention_weight_max": 15.665172576904297,
      "activations/layer1_attention_weight_min": -12.523014068603516,
      "activations/layer20_attention_weight_max": 20.96564292907715,
      "activations/layer20_attention_weight_min": -22.012771606445312,
      "activations/layer21_attention_weight_max": 31.68791389465332,
      "activations/layer21_attention_weight_min": -20.929340362548828,
      "activations/layer22_attention_weight_max": 29.58725929260254,
      "activations/layer22_attention_weight_min": -28.787153244018555,
      "activations/layer23_attention_weight_max": 34.70835494995117,
      "activations/layer23_attention_weight_min": -22.884702682495117,
      "activations/layer2_attention_weight_max": 30.532718658447266,
      "activations/layer2_attention_weight_min": -28.84222984313965,
      "activations/layer3_attention_weight_max": 84.45337677001953,
      "activations/layer3_attention_weight_min": -84.37915802001953,
      "activations/layer4_attention_weight_max": 80.81787872314453,
      "activations/layer4_attention_weight_min": -81.6341781616211,
      "activations/layer5_attention_weight_max": 57.27212142944336,
      "activations/layer5_attention_weight_min": -69.88523864746094,
      "activations/layer6_attention_weight_max": 44.93431091308594,
      "activations/layer6_attention_weight_min": -47.59929656982422,
      "activations/layer7_attention_weight_max": 59.860572814941406,
      "activations/layer7_attention_weight_min": -62.93262481689453,
      "activations/layer8_attention_weight_max": 45.746150970458984,
      "activations/layer8_attention_weight_min": -48.622005462646484,
      "activations/layer9_attention_weight_max": 43.74905776977539,
      "activations/layer9_attention_weight_min": -44.85123825073242,
      "epoch": 19.89,
      "learning_rate": 2.196022727272727e-05,
      "loss": 2.7331,
      "step": 342300
    },
    {
      "activations/layer0_attention_weight_max": 15.750718116760254,
      "activations/layer0_attention_weight_min": -14.594008445739746,
      "activations/layer10_attention_weight_max": 34.917503356933594,
      "activations/layer10_attention_weight_min": -35.80760192871094,
      "activations/layer11_attention_weight_max": 32.180885314941406,
      "activations/layer11_attention_weight_min": -34.823326110839844,
      "activations/layer12_attention_weight_max": 17.78096580505371,
      "activations/layer12_attention_weight_min": -29.116167068481445,
      "activations/layer13_attention_weight_max": 29.136707305908203,
      "activations/layer13_attention_weight_min": -24.98662757873535,
      "activations/layer14_attention_weight_max": 33.13970184326172,
      "activations/layer14_attention_weight_min": -28.18379020690918,
      "activations/layer15_attention_weight_max": 31.29873275756836,
      "activations/layer15_attention_weight_min": -30.037145614624023,
      "activations/layer16_attention_weight_max": 32.40170669555664,
      "activations/layer16_attention_weight_min": -32.032447814941406,
      "activations/layer17_attention_weight_max": 50.44654846191406,
      "activations/layer17_attention_weight_min": -43.723976135253906,
      "activations/layer18_attention_weight_max": 45.876041412353516,
      "activations/layer18_attention_weight_min": -37.608924865722656,
      "activations/layer19_attention_weight_max": 21.739213943481445,
      "activations/layer19_attention_weight_min": -22.14369773864746,
      "activations/layer1_attention_weight_max": 16.108537673950195,
      "activations/layer1_attention_weight_min": -13.91882610321045,
      "activations/layer20_attention_weight_max": 20.69748878479004,
      "activations/layer20_attention_weight_min": -20.345809936523438,
      "activations/layer21_attention_weight_max": 29.93094825744629,
      "activations/layer21_attention_weight_min": -18.85123062133789,
      "activations/layer22_attention_weight_max": 26.41719627380371,
      "activations/layer22_attention_weight_min": -25.361074447631836,
      "activations/layer23_attention_weight_max": 33.62400817871094,
      "activations/layer23_attention_weight_min": -21.6740779876709,
      "activations/layer2_attention_weight_max": 32.208248138427734,
      "activations/layer2_attention_weight_min": -31.489486694335938,
      "activations/layer3_attention_weight_max": 83.63156127929688,
      "activations/layer3_attention_weight_min": -85.84480285644531,
      "activations/layer4_attention_weight_max": 80.14064025878906,
      "activations/layer4_attention_weight_min": -79.69542694091797,
      "activations/layer5_attention_weight_max": 62.97675323486328,
      "activations/layer5_attention_weight_min": -69.68775939941406,
      "activations/layer6_attention_weight_max": 46.633583068847656,
      "activations/layer6_attention_weight_min": -46.60649108886719,
      "activations/layer7_attention_weight_max": 62.892757415771484,
      "activations/layer7_attention_weight_min": -59.402896881103516,
      "activations/layer8_attention_weight_max": 46.43773651123047,
      "activations/layer8_attention_weight_min": -51.60314178466797,
      "activations/layer9_attention_weight_max": 46.60770797729492,
      "activations/layer9_attention_weight_min": -46.35441207885742,
      "epoch": 19.89,
      "learning_rate": 2.1941287878787878e-05,
      "loss": 2.7239,
      "step": 342350
    },
    {
      "activations/layer0_attention_weight_max": 15.645033836364746,
      "activations/layer0_attention_weight_min": -14.24227237701416,
      "activations/layer10_attention_weight_max": 37.29408645629883,
      "activations/layer10_attention_weight_min": -36.328311920166016,
      "activations/layer11_attention_weight_max": 33.08690643310547,
      "activations/layer11_attention_weight_min": -34.066749572753906,
      "activations/layer12_attention_weight_max": 19.727298736572266,
      "activations/layer12_attention_weight_min": -24.81207847595215,
      "activations/layer13_attention_weight_max": 35.701053619384766,
      "activations/layer13_attention_weight_min": -29.955257415771484,
      "activations/layer14_attention_weight_max": 37.19501876831055,
      "activations/layer14_attention_weight_min": -32.10084533691406,
      "activations/layer15_attention_weight_max": 35.43328857421875,
      "activations/layer15_attention_weight_min": -31.400821685791016,
      "activations/layer16_attention_weight_max": 36.304969787597656,
      "activations/layer16_attention_weight_min": -35.53129959106445,
      "activations/layer17_attention_weight_max": 54.58877182006836,
      "activations/layer17_attention_weight_min": -49.15703201293945,
      "activations/layer18_attention_weight_max": 51.62386703491211,
      "activations/layer18_attention_weight_min": -41.77098846435547,
      "activations/layer19_attention_weight_max": 26.231082916259766,
      "activations/layer19_attention_weight_min": -22.3169002532959,
      "activations/layer1_attention_weight_max": 15.503511428833008,
      "activations/layer1_attention_weight_min": -13.592930793762207,
      "activations/layer20_attention_weight_max": 24.661638259887695,
      "activations/layer20_attention_weight_min": -22.838191986083984,
      "activations/layer21_attention_weight_max": 42.25774383544922,
      "activations/layer21_attention_weight_min": -25.129528045654297,
      "activations/layer22_attention_weight_max": 31.03362274169922,
      "activations/layer22_attention_weight_min": -25.787385940551758,
      "activations/layer23_attention_weight_max": 36.946388244628906,
      "activations/layer23_attention_weight_min": -24.147293090820312,
      "activations/layer2_attention_weight_max": 30.835363388061523,
      "activations/layer2_attention_weight_min": -30.95404815673828,
      "activations/layer3_attention_weight_max": 85.7026596069336,
      "activations/layer3_attention_weight_min": -85.64324951171875,
      "activations/layer4_attention_weight_max": 85.22416687011719,
      "activations/layer4_attention_weight_min": -80.63938903808594,
      "activations/layer5_attention_weight_max": 65.6076889038086,
      "activations/layer5_attention_weight_min": -68.014404296875,
      "activations/layer6_attention_weight_max": 48.707027435302734,
      "activations/layer6_attention_weight_min": -51.141090393066406,
      "activations/layer7_attention_weight_max": 64.71632385253906,
      "activations/layer7_attention_weight_min": -62.81200408935547,
      "activations/layer8_attention_weight_max": 45.48585891723633,
      "activations/layer8_attention_weight_min": -48.27959060668945,
      "activations/layer9_attention_weight_max": 43.79234313964844,
      "activations/layer9_attention_weight_min": -45.843074798583984,
      "epoch": 19.9,
      "learning_rate": 2.1922348484848482e-05,
      "loss": 2.7444,
      "step": 342400
    },
    {
      "activations/layer0_attention_weight_max": 15.419093132019043,
      "activations/layer0_attention_weight_min": -14.222779273986816,
      "activations/layer10_attention_weight_max": 35.54487991333008,
      "activations/layer10_attention_weight_min": -35.10980224609375,
      "activations/layer11_attention_weight_max": 35.36943054199219,
      "activations/layer11_attention_weight_min": -35.07427978515625,
      "activations/layer12_attention_weight_max": 20.181116104125977,
      "activations/layer12_attention_weight_min": -27.61068344116211,
      "activations/layer13_attention_weight_max": 39.09479904174805,
      "activations/layer13_attention_weight_min": -28.39446449279785,
      "activations/layer14_attention_weight_max": 52.280670166015625,
      "activations/layer14_attention_weight_min": -29.97015380859375,
      "activations/layer15_attention_weight_max": 48.26502227783203,
      "activations/layer15_attention_weight_min": -31.389129638671875,
      "activations/layer16_attention_weight_max": 34.96114730834961,
      "activations/layer16_attention_weight_min": -29.790149688720703,
      "activations/layer17_attention_weight_max": 51.90681838989258,
      "activations/layer17_attention_weight_min": -45.02720260620117,
      "activations/layer18_attention_weight_max": 46.99523162841797,
      "activations/layer18_attention_weight_min": -40.78084182739258,
      "activations/layer19_attention_weight_max": 26.223270416259766,
      "activations/layer19_attention_weight_min": -26.301626205444336,
      "activations/layer1_attention_weight_max": 15.230207443237305,
      "activations/layer1_attention_weight_min": -13.554620742797852,
      "activations/layer20_attention_weight_max": 28.011146545410156,
      "activations/layer20_attention_weight_min": -24.056772232055664,
      "activations/layer21_attention_weight_max": 44.91672134399414,
      "activations/layer21_attention_weight_min": -24.03507423400879,
      "activations/layer22_attention_weight_max": 30.66653060913086,
      "activations/layer22_attention_weight_min": -25.597436904907227,
      "activations/layer23_attention_weight_max": 36.86821365356445,
      "activations/layer23_attention_weight_min": -23.633705139160156,
      "activations/layer2_attention_weight_max": 32.741844177246094,
      "activations/layer2_attention_weight_min": -30.10968589782715,
      "activations/layer3_attention_weight_max": 88.67972564697266,
      "activations/layer3_attention_weight_min": -85.11872100830078,
      "activations/layer4_attention_weight_max": 81.16743469238281,
      "activations/layer4_attention_weight_min": -80.65769958496094,
      "activations/layer5_attention_weight_max": 59.464759826660156,
      "activations/layer5_attention_weight_min": -74.6075210571289,
      "activations/layer6_attention_weight_max": 50.433982849121094,
      "activations/layer6_attention_weight_min": -47.94974136352539,
      "activations/layer7_attention_weight_max": 67.39590454101562,
      "activations/layer7_attention_weight_min": -69.11581420898438,
      "activations/layer8_attention_weight_max": 54.092323303222656,
      "activations/layer8_attention_weight_min": -48.12018966674805,
      "activations/layer9_attention_weight_max": 51.46297836303711,
      "activations/layer9_attention_weight_min": -46.4468879699707,
      "epoch": 19.9,
      "learning_rate": 2.190340909090909e-05,
      "loss": 2.725,
      "step": 342450
    },
    {
      "activations/layer0_attention_weight_max": 16.616729736328125,
      "activations/layer0_attention_weight_min": -14.245857238769531,
      "activations/layer10_attention_weight_max": 41.05433654785156,
      "activations/layer10_attention_weight_min": -35.818965911865234,
      "activations/layer11_attention_weight_max": 36.29497528076172,
      "activations/layer11_attention_weight_min": -37.19291687011719,
      "activations/layer12_attention_weight_max": 19.553442001342773,
      "activations/layer12_attention_weight_min": -25.794692993164062,
      "activations/layer13_attention_weight_max": 41.390960693359375,
      "activations/layer13_attention_weight_min": -30.356914520263672,
      "activations/layer14_attention_weight_max": 38.13313293457031,
      "activations/layer14_attention_weight_min": -30.862276077270508,
      "activations/layer15_attention_weight_max": 36.56535720825195,
      "activations/layer15_attention_weight_min": -31.498184204101562,
      "activations/layer16_attention_weight_max": 36.95787048339844,
      "activations/layer16_attention_weight_min": -34.85703659057617,
      "activations/layer17_attention_weight_max": 57.3812141418457,
      "activations/layer17_attention_weight_min": -50.341331481933594,
      "activations/layer18_attention_weight_max": 54.73436737060547,
      "activations/layer18_attention_weight_min": -46.955810546875,
      "activations/layer19_attention_weight_max": 29.345783233642578,
      "activations/layer19_attention_weight_min": -24.81222915649414,
      "activations/layer1_attention_weight_max": 16.064878463745117,
      "activations/layer1_attention_weight_min": -12.796306610107422,
      "activations/layer20_attention_weight_max": 25.0091552734375,
      "activations/layer20_attention_weight_min": -25.517240524291992,
      "activations/layer21_attention_weight_max": 40.528785705566406,
      "activations/layer21_attention_weight_min": -23.51552391052246,
      "activations/layer22_attention_weight_max": 28.592147827148438,
      "activations/layer22_attention_weight_min": -27.218326568603516,
      "activations/layer23_attention_weight_max": 37.91374206542969,
      "activations/layer23_attention_weight_min": -25.798202514648438,
      "activations/layer2_attention_weight_max": 31.92420768737793,
      "activations/layer2_attention_weight_min": -31.228918075561523,
      "activations/layer3_attention_weight_max": 83.26598358154297,
      "activations/layer3_attention_weight_min": -85.47205352783203,
      "activations/layer4_attention_weight_max": 81.28459930419922,
      "activations/layer4_attention_weight_min": -81.71902465820312,
      "activations/layer5_attention_weight_max": 61.188758850097656,
      "activations/layer5_attention_weight_min": -69.03790283203125,
      "activations/layer6_attention_weight_max": 50.74443435668945,
      "activations/layer6_attention_weight_min": -50.94176483154297,
      "activations/layer7_attention_weight_max": 65.4722900390625,
      "activations/layer7_attention_weight_min": -68.29776000976562,
      "activations/layer8_attention_weight_max": 50.0304069519043,
      "activations/layer8_attention_weight_min": -52.13560485839844,
      "activations/layer9_attention_weight_max": 45.9100456237793,
      "activations/layer9_attention_weight_min": -46.34585189819336,
      "epoch": 19.9,
      "learning_rate": 2.1884469696969692e-05,
      "loss": 2.7287,
      "step": 342500
    },
    {
      "activations/layer0_attention_weight_max": 16.15545654296875,
      "activations/layer0_attention_weight_min": -14.392378807067871,
      "activations/layer10_attention_weight_max": 37.0146598815918,
      "activations/layer10_attention_weight_min": -38.16880798339844,
      "activations/layer11_attention_weight_max": 35.13846206665039,
      "activations/layer11_attention_weight_min": -37.079063415527344,
      "activations/layer12_attention_weight_max": 19.073415756225586,
      "activations/layer12_attention_weight_min": -23.682071685791016,
      "activations/layer13_attention_weight_max": 34.910030364990234,
      "activations/layer13_attention_weight_min": -27.331501007080078,
      "activations/layer14_attention_weight_max": 35.75679016113281,
      "activations/layer14_attention_weight_min": -32.723236083984375,
      "activations/layer15_attention_weight_max": 33.24483108520508,
      "activations/layer15_attention_weight_min": -31.185930252075195,
      "activations/layer16_attention_weight_max": 34.3538932800293,
      "activations/layer16_attention_weight_min": -32.691898345947266,
      "activations/layer17_attention_weight_max": 52.8425178527832,
      "activations/layer17_attention_weight_min": -45.00050735473633,
      "activations/layer18_attention_weight_max": 48.530784606933594,
      "activations/layer18_attention_weight_min": -39.87637710571289,
      "activations/layer19_attention_weight_max": 27.910879135131836,
      "activations/layer19_attention_weight_min": -22.48158073425293,
      "activations/layer1_attention_weight_max": 15.362420082092285,
      "activations/layer1_attention_weight_min": -13.876930236816406,
      "activations/layer20_attention_weight_max": 23.924970626831055,
      "activations/layer20_attention_weight_min": -21.53367805480957,
      "activations/layer21_attention_weight_max": 37.30996322631836,
      "activations/layer21_attention_weight_min": -22.797073364257812,
      "activations/layer22_attention_weight_max": 26.9329833984375,
      "activations/layer22_attention_weight_min": -25.805410385131836,
      "activations/layer23_attention_weight_max": 37.84642791748047,
      "activations/layer23_attention_weight_min": -22.669933319091797,
      "activations/layer2_attention_weight_max": 32.7534294128418,
      "activations/layer2_attention_weight_min": -31.1239070892334,
      "activations/layer3_attention_weight_max": 85.66446685791016,
      "activations/layer3_attention_weight_min": -89.5560073852539,
      "activations/layer4_attention_weight_max": 82.8659439086914,
      "activations/layer4_attention_weight_min": -84.14430236816406,
      "activations/layer5_attention_weight_max": 62.91511535644531,
      "activations/layer5_attention_weight_min": -67.85453796386719,
      "activations/layer6_attention_weight_max": 50.03952407836914,
      "activations/layer6_attention_weight_min": -51.298343658447266,
      "activations/layer7_attention_weight_max": 74.64020538330078,
      "activations/layer7_attention_weight_min": -64.12997436523438,
      "activations/layer8_attention_weight_max": 49.51820755004883,
      "activations/layer8_attention_weight_min": -49.847740173339844,
      "activations/layer9_attention_weight_max": 51.58454513549805,
      "activations/layer9_attention_weight_min": -49.14547348022461,
      "epoch": 19.9,
      "learning_rate": 2.18655303030303e-05,
      "loss": 2.7197,
      "step": 342550
    },
    {
      "activations/layer0_attention_weight_max": 15.416193008422852,
      "activations/layer0_attention_weight_min": -13.72850227355957,
      "activations/layer10_attention_weight_max": 39.94873809814453,
      "activations/layer10_attention_weight_min": -37.5489501953125,
      "activations/layer11_attention_weight_max": 36.53163146972656,
      "activations/layer11_attention_weight_min": -37.200199127197266,
      "activations/layer12_attention_weight_max": 19.76749038696289,
      "activations/layer12_attention_weight_min": -28.182714462280273,
      "activations/layer13_attention_weight_max": 34.123714447021484,
      "activations/layer13_attention_weight_min": -26.029666900634766,
      "activations/layer14_attention_weight_max": 34.76243209838867,
      "activations/layer14_attention_weight_min": -29.362089157104492,
      "activations/layer15_attention_weight_max": 34.30125045776367,
      "activations/layer15_attention_weight_min": -29.83649444580078,
      "activations/layer16_attention_weight_max": 34.05986022949219,
      "activations/layer16_attention_weight_min": -30.941112518310547,
      "activations/layer17_attention_weight_max": 52.23374557495117,
      "activations/layer17_attention_weight_min": -46.49069595336914,
      "activations/layer18_attention_weight_max": 47.6253776550293,
      "activations/layer18_attention_weight_min": -41.09528732299805,
      "activations/layer19_attention_weight_max": 26.352270126342773,
      "activations/layer19_attention_weight_min": -21.686628341674805,
      "activations/layer1_attention_weight_max": 16.158437728881836,
      "activations/layer1_attention_weight_min": -13.317307472229004,
      "activations/layer20_attention_weight_max": 25.073076248168945,
      "activations/layer20_attention_weight_min": -21.18507194519043,
      "activations/layer21_attention_weight_max": 40.85462188720703,
      "activations/layer21_attention_weight_min": -23.1951847076416,
      "activations/layer22_attention_weight_max": 30.606157302856445,
      "activations/layer22_attention_weight_min": -24.357940673828125,
      "activations/layer23_attention_weight_max": 36.77768325805664,
      "activations/layer23_attention_weight_min": -23.33039093017578,
      "activations/layer2_attention_weight_max": 31.974271774291992,
      "activations/layer2_attention_weight_min": -31.087207794189453,
      "activations/layer3_attention_weight_max": 89.75749969482422,
      "activations/layer3_attention_weight_min": -90.8357925415039,
      "activations/layer4_attention_weight_max": 82.72310638427734,
      "activations/layer4_attention_weight_min": -85.95770263671875,
      "activations/layer5_attention_weight_max": 62.294464111328125,
      "activations/layer5_attention_weight_min": -70.7676773071289,
      "activations/layer6_attention_weight_max": 51.595943450927734,
      "activations/layer6_attention_weight_min": -51.58871841430664,
      "activations/layer7_attention_weight_max": 71.24015808105469,
      "activations/layer7_attention_weight_min": -64.94827270507812,
      "activations/layer8_attention_weight_max": 48.57172393798828,
      "activations/layer8_attention_weight_min": -51.45980453491211,
      "activations/layer9_attention_weight_max": 51.276119232177734,
      "activations/layer9_attention_weight_min": -47.13300704956055,
      "epoch": 19.91,
      "learning_rate": 2.1846969696969695e-05,
      "loss": 2.7442,
      "step": 342600
    },
    {
      "activations/layer0_attention_weight_max": 15.7344331741333,
      "activations/layer0_attention_weight_min": -14.364335060119629,
      "activations/layer10_attention_weight_max": 34.86674499511719,
      "activations/layer10_attention_weight_min": -34.57048416137695,
      "activations/layer11_attention_weight_max": 32.007808685302734,
      "activations/layer11_attention_weight_min": -34.12982940673828,
      "activations/layer12_attention_weight_max": 21.59524917602539,
      "activations/layer12_attention_weight_min": -26.12767219543457,
      "activations/layer13_attention_weight_max": 38.2547721862793,
      "activations/layer13_attention_weight_min": -31.314544677734375,
      "activations/layer14_attention_weight_max": 41.021732330322266,
      "activations/layer14_attention_weight_min": -33.68915939331055,
      "activations/layer15_attention_weight_max": 43.232791900634766,
      "activations/layer15_attention_weight_min": -32.66255187988281,
      "activations/layer16_attention_weight_max": 42.57331085205078,
      "activations/layer16_attention_weight_min": -36.93710708618164,
      "activations/layer17_attention_weight_max": 60.97929763793945,
      "activations/layer17_attention_weight_min": -49.68379592895508,
      "activations/layer18_attention_weight_max": 60.097412109375,
      "activations/layer18_attention_weight_min": -41.9747200012207,
      "activations/layer19_attention_weight_max": 28.934329986572266,
      "activations/layer19_attention_weight_min": -24.219289779663086,
      "activations/layer1_attention_weight_max": 14.807287216186523,
      "activations/layer1_attention_weight_min": -14.136509895324707,
      "activations/layer20_attention_weight_max": 29.017900466918945,
      "activations/layer20_attention_weight_min": -23.013843536376953,
      "activations/layer21_attention_weight_max": 43.6096076965332,
      "activations/layer21_attention_weight_min": -22.67275619506836,
      "activations/layer22_attention_weight_max": 35.179378509521484,
      "activations/layer22_attention_weight_min": -25.10968589782715,
      "activations/layer23_attention_weight_max": 41.096431732177734,
      "activations/layer23_attention_weight_min": -22.862722396850586,
      "activations/layer2_attention_weight_max": 30.755430221557617,
      "activations/layer2_attention_weight_min": -30.166044235229492,
      "activations/layer3_attention_weight_max": 81.96971893310547,
      "activations/layer3_attention_weight_min": -82.58379364013672,
      "activations/layer4_attention_weight_max": 78.97374725341797,
      "activations/layer4_attention_weight_min": -78.23540496826172,
      "activations/layer5_attention_weight_max": 57.09949493408203,
      "activations/layer5_attention_weight_min": -70.0290298461914,
      "activations/layer6_attention_weight_max": 45.24647521972656,
      "activations/layer6_attention_weight_min": -48.41127014160156,
      "activations/layer7_attention_weight_max": 64.44581604003906,
      "activations/layer7_attention_weight_min": -61.656578063964844,
      "activations/layer8_attention_weight_max": 45.74806594848633,
      "activations/layer8_attention_weight_min": -47.802555084228516,
      "activations/layer9_attention_weight_max": 42.44773483276367,
      "activations/layer9_attention_weight_min": -45.03636932373047,
      "epoch": 19.91,
      "learning_rate": 2.1828030303030303e-05,
      "loss": 2.7279,
      "step": 342650
    },
    {
      "activations/layer0_attention_weight_max": 16.038917541503906,
      "activations/layer0_attention_weight_min": -15.28672981262207,
      "activations/layer10_attention_weight_max": 37.77508544921875,
      "activations/layer10_attention_weight_min": -40.7103157043457,
      "activations/layer11_attention_weight_max": 32.44596481323242,
      "activations/layer11_attention_weight_min": -36.68134307861328,
      "activations/layer12_attention_weight_max": 19.71126365661621,
      "activations/layer12_attention_weight_min": -26.998741149902344,
      "activations/layer13_attention_weight_max": 35.67953109741211,
      "activations/layer13_attention_weight_min": -27.066801071166992,
      "activations/layer14_attention_weight_max": 34.66469192504883,
      "activations/layer14_attention_weight_min": -30.134170532226562,
      "activations/layer15_attention_weight_max": 32.69865036010742,
      "activations/layer15_attention_weight_min": -31.461572647094727,
      "activations/layer16_attention_weight_max": 36.60427474975586,
      "activations/layer16_attention_weight_min": -33.587120056152344,
      "activations/layer17_attention_weight_max": 58.00330352783203,
      "activations/layer17_attention_weight_min": -50.17257308959961,
      "activations/layer18_attention_weight_max": 47.061458587646484,
      "activations/layer18_attention_weight_min": -43.51897048950195,
      "activations/layer19_attention_weight_max": 24.254310607910156,
      "activations/layer19_attention_weight_min": -22.28921127319336,
      "activations/layer1_attention_weight_max": 17.074615478515625,
      "activations/layer1_attention_weight_min": -13.301538467407227,
      "activations/layer20_attention_weight_max": 22.6977596282959,
      "activations/layer20_attention_weight_min": -22.518516540527344,
      "activations/layer21_attention_weight_max": 36.87727737426758,
      "activations/layer21_attention_weight_min": -23.397037506103516,
      "activations/layer22_attention_weight_max": 27.61252212524414,
      "activations/layer22_attention_weight_min": -26.90259552001953,
      "activations/layer23_attention_weight_max": 37.333641052246094,
      "activations/layer23_attention_weight_min": -23.189544677734375,
      "activations/layer2_attention_weight_max": 30.614004135131836,
      "activations/layer2_attention_weight_min": -29.413105010986328,
      "activations/layer3_attention_weight_max": 82.88642883300781,
      "activations/layer3_attention_weight_min": -85.5817642211914,
      "activations/layer4_attention_weight_max": 80.14727020263672,
      "activations/layer4_attention_weight_min": -81.4880599975586,
      "activations/layer5_attention_weight_max": 61.63845443725586,
      "activations/layer5_attention_weight_min": -67.39278411865234,
      "activations/layer6_attention_weight_max": 46.74907302856445,
      "activations/layer6_attention_weight_min": -49.43642044067383,
      "activations/layer7_attention_weight_max": 60.559120178222656,
      "activations/layer7_attention_weight_min": -67.47742462158203,
      "activations/layer8_attention_weight_max": 46.73036193847656,
      "activations/layer8_attention_weight_min": -49.905113220214844,
      "activations/layer9_attention_weight_max": 46.64786911010742,
      "activations/layer9_attention_weight_min": -50.58890151977539,
      "epoch": 19.91,
      "learning_rate": 2.1809090909090904e-05,
      "loss": 2.7399,
      "step": 342700
    },
    {
      "activations/layer0_attention_weight_max": 15.38570785522461,
      "activations/layer0_attention_weight_min": -13.898918151855469,
      "activations/layer10_attention_weight_max": 32.951255798339844,
      "activations/layer10_attention_weight_min": -34.01512908935547,
      "activations/layer11_attention_weight_max": 31.01491355895996,
      "activations/layer11_attention_weight_min": -33.850616455078125,
      "activations/layer12_attention_weight_max": 20.238636016845703,
      "activations/layer12_attention_weight_min": -25.082813262939453,
      "activations/layer13_attention_weight_max": 33.65879821777344,
      "activations/layer13_attention_weight_min": -31.607534408569336,
      "activations/layer14_attention_weight_max": 38.412322998046875,
      "activations/layer14_attention_weight_min": -29.250957489013672,
      "activations/layer15_attention_weight_max": 33.90836715698242,
      "activations/layer15_attention_weight_min": -31.10651397705078,
      "activations/layer16_attention_weight_max": 34.549156188964844,
      "activations/layer16_attention_weight_min": -33.868534088134766,
      "activations/layer17_attention_weight_max": 51.06953811645508,
      "activations/layer17_attention_weight_min": -47.48884963989258,
      "activations/layer18_attention_weight_max": 47.90323257446289,
      "activations/layer18_attention_weight_min": -40.69557571411133,
      "activations/layer19_attention_weight_max": 27.21463966369629,
      "activations/layer19_attention_weight_min": -25.102859497070312,
      "activations/layer1_attention_weight_max": 15.522112846374512,
      "activations/layer1_attention_weight_min": -12.27521800994873,
      "activations/layer20_attention_weight_max": 23.535015106201172,
      "activations/layer20_attention_weight_min": -24.279300689697266,
      "activations/layer21_attention_weight_max": 36.48400115966797,
      "activations/layer21_attention_weight_min": -24.699365615844727,
      "activations/layer22_attention_weight_max": 31.021244049072266,
      "activations/layer22_attention_weight_min": -25.129657745361328,
      "activations/layer23_attention_weight_max": 36.476806640625,
      "activations/layer23_attention_weight_min": -25.506271362304688,
      "activations/layer2_attention_weight_max": 28.862401962280273,
      "activations/layer2_attention_weight_min": -27.336828231811523,
      "activations/layer3_attention_weight_max": 80.59034729003906,
      "activations/layer3_attention_weight_min": -78.92768859863281,
      "activations/layer4_attention_weight_max": 77.87653350830078,
      "activations/layer4_attention_weight_min": -79.60894775390625,
      "activations/layer5_attention_weight_max": 61.1449089050293,
      "activations/layer5_attention_weight_min": -72.47171783447266,
      "activations/layer6_attention_weight_max": 46.7071647644043,
      "activations/layer6_attention_weight_min": -47.41178512573242,
      "activations/layer7_attention_weight_max": 60.8900032043457,
      "activations/layer7_attention_weight_min": -61.3106575012207,
      "activations/layer8_attention_weight_max": 44.48093032836914,
      "activations/layer8_attention_weight_min": -46.090728759765625,
      "activations/layer9_attention_weight_max": 42.579345703125,
      "activations/layer9_attention_weight_min": -41.66303634643555,
      "epoch": 19.92,
      "learning_rate": 2.1790151515151513e-05,
      "loss": 2.7376,
      "step": 342750
    },
    {
      "activations/layer0_attention_weight_max": 14.886698722839355,
      "activations/layer0_attention_weight_min": -13.886868476867676,
      "activations/layer10_attention_weight_max": 38.732086181640625,
      "activations/layer10_attention_weight_min": -36.821781158447266,
      "activations/layer11_attention_weight_max": 37.86775207519531,
      "activations/layer11_attention_weight_min": -36.313541412353516,
      "activations/layer12_attention_weight_max": 18.544401168823242,
      "activations/layer12_attention_weight_min": -24.059736251831055,
      "activations/layer13_attention_weight_max": 35.42421340942383,
      "activations/layer13_attention_weight_min": -29.20735740661621,
      "activations/layer14_attention_weight_max": 35.568443298339844,
      "activations/layer14_attention_weight_min": -30.37675666809082,
      "activations/layer15_attention_weight_max": 35.731319427490234,
      "activations/layer15_attention_weight_min": -30.248916625976562,
      "activations/layer16_attention_weight_max": 38.15488052368164,
      "activations/layer16_attention_weight_min": -35.14470672607422,
      "activations/layer17_attention_weight_max": 58.04548645019531,
      "activations/layer17_attention_weight_min": -50.435997009277344,
      "activations/layer18_attention_weight_max": 47.465126037597656,
      "activations/layer18_attention_weight_min": -40.639503479003906,
      "activations/layer19_attention_weight_max": 24.18157196044922,
      "activations/layer19_attention_weight_min": -21.18147087097168,
      "activations/layer1_attention_weight_max": 15.934329986572266,
      "activations/layer1_attention_weight_min": -13.854575157165527,
      "activations/layer20_attention_weight_max": 23.780418395996094,
      "activations/layer20_attention_weight_min": -20.933258056640625,
      "activations/layer21_attention_weight_max": 32.84511947631836,
      "activations/layer21_attention_weight_min": -20.456167221069336,
      "activations/layer22_attention_weight_max": 30.456262588500977,
      "activations/layer22_attention_weight_min": -25.277069091796875,
      "activations/layer23_attention_weight_max": 34.037940979003906,
      "activations/layer23_attention_weight_min": -22.89191436767578,
      "activations/layer2_attention_weight_max": 30.777149200439453,
      "activations/layer2_attention_weight_min": -29.833534240722656,
      "activations/layer3_attention_weight_max": 81.51834106445312,
      "activations/layer3_attention_weight_min": -84.47233581542969,
      "activations/layer4_attention_weight_max": 80.6241683959961,
      "activations/layer4_attention_weight_min": -79.19332122802734,
      "activations/layer5_attention_weight_max": 62.42548751831055,
      "activations/layer5_attention_weight_min": -71.6212158203125,
      "activations/layer6_attention_weight_max": 49.35248947143555,
      "activations/layer6_attention_weight_min": -48.796958923339844,
      "activations/layer7_attention_weight_max": 71.40095520019531,
      "activations/layer7_attention_weight_min": -64.48858642578125,
      "activations/layer8_attention_weight_max": 51.16298294067383,
      "activations/layer8_attention_weight_min": -49.47671890258789,
      "activations/layer9_attention_weight_max": 50.423282623291016,
      "activations/layer9_attention_weight_min": -45.46691131591797,
      "epoch": 19.92,
      "learning_rate": 2.177121212121212e-05,
      "loss": 2.7418,
      "step": 342800
    },
    {
      "activations/layer0_attention_weight_max": 15.412808418273926,
      "activations/layer0_attention_weight_min": -13.854555130004883,
      "activations/layer10_attention_weight_max": 34.33981704711914,
      "activations/layer10_attention_weight_min": -36.99048614501953,
      "activations/layer11_attention_weight_max": 32.226356506347656,
      "activations/layer11_attention_weight_min": -34.486228942871094,
      "activations/layer12_attention_weight_max": 18.794574737548828,
      "activations/layer12_attention_weight_min": -23.633777618408203,
      "activations/layer13_attention_weight_max": 38.37541198730469,
      "activations/layer13_attention_weight_min": -26.042043685913086,
      "activations/layer14_attention_weight_max": 32.581031799316406,
      "activations/layer14_attention_weight_min": -31.220455169677734,
      "activations/layer15_attention_weight_max": 31.45339012145996,
      "activations/layer15_attention_weight_min": -30.627534866333008,
      "activations/layer16_attention_weight_max": 32.928924560546875,
      "activations/layer16_attention_weight_min": -32.626976013183594,
      "activations/layer17_attention_weight_max": 50.64515686035156,
      "activations/layer17_attention_weight_min": -45.280784606933594,
      "activations/layer18_attention_weight_max": 45.702152252197266,
      "activations/layer18_attention_weight_min": -40.41826629638672,
      "activations/layer19_attention_weight_max": 25.93129539489746,
      "activations/layer19_attention_weight_min": -24.676197052001953,
      "activations/layer1_attention_weight_max": 15.12259292602539,
      "activations/layer1_attention_weight_min": -13.606517791748047,
      "activations/layer20_attention_weight_max": 22.598234176635742,
      "activations/layer20_attention_weight_min": -21.93515396118164,
      "activations/layer21_attention_weight_max": 35.622100830078125,
      "activations/layer21_attention_weight_min": -20.379884719848633,
      "activations/layer22_attention_weight_max": 30.311460494995117,
      "activations/layer22_attention_weight_min": -26.110416412353516,
      "activations/layer23_attention_weight_max": 36.881126403808594,
      "activations/layer23_attention_weight_min": -22.847278594970703,
      "activations/layer2_attention_weight_max": 30.121578216552734,
      "activations/layer2_attention_weight_min": -29.32051658630371,
      "activations/layer3_attention_weight_max": 80.93657684326172,
      "activations/layer3_attention_weight_min": -84.39119720458984,
      "activations/layer4_attention_weight_max": 80.58513641357422,
      "activations/layer4_attention_weight_min": -81.3771743774414,
      "activations/layer5_attention_weight_max": 59.90641784667969,
      "activations/layer5_attention_weight_min": -67.12998962402344,
      "activations/layer6_attention_weight_max": 46.810672760009766,
      "activations/layer6_attention_weight_min": -48.94218063354492,
      "activations/layer7_attention_weight_max": 64.02339935302734,
      "activations/layer7_attention_weight_min": -62.677364349365234,
      "activations/layer8_attention_weight_max": 46.39674758911133,
      "activations/layer8_attention_weight_min": -49.644439697265625,
      "activations/layer9_attention_weight_max": 42.53084182739258,
      "activations/layer9_attention_weight_min": -42.54928207397461,
      "epoch": 19.92,
      "learning_rate": 2.1752272727272726e-05,
      "loss": 2.7253,
      "step": 342850
    },
    {
      "activations/layer0_attention_weight_max": 15.578696250915527,
      "activations/layer0_attention_weight_min": -14.501132011413574,
      "activations/layer10_attention_weight_max": 34.85383987426758,
      "activations/layer10_attention_weight_min": -36.280372619628906,
      "activations/layer11_attention_weight_max": 30.95551300048828,
      "activations/layer11_attention_weight_min": -34.05520248413086,
      "activations/layer12_attention_weight_max": 18.754405975341797,
      "activations/layer12_attention_weight_min": -25.760412216186523,
      "activations/layer13_attention_weight_max": 30.10643768310547,
      "activations/layer13_attention_weight_min": -26.87285041809082,
      "activations/layer14_attention_weight_max": 31.277484893798828,
      "activations/layer14_attention_weight_min": -30.303020477294922,
      "activations/layer15_attention_weight_max": 31.50365447998047,
      "activations/layer15_attention_weight_min": -32.543914794921875,
      "activations/layer16_attention_weight_max": 30.808345794677734,
      "activations/layer16_attention_weight_min": -31.149829864501953,
      "activations/layer17_attention_weight_max": 50.42471694946289,
      "activations/layer17_attention_weight_min": -45.24151611328125,
      "activations/layer18_attention_weight_max": 43.9466552734375,
      "activations/layer18_attention_weight_min": -40.5545768737793,
      "activations/layer19_attention_weight_max": 23.1632137298584,
      "activations/layer19_attention_weight_min": -21.839590072631836,
      "activations/layer1_attention_weight_max": 15.653814315795898,
      "activations/layer1_attention_weight_min": -12.053384780883789,
      "activations/layer20_attention_weight_max": 21.340593338012695,
      "activations/layer20_attention_weight_min": -23.125181198120117,
      "activations/layer21_attention_weight_max": 30.954118728637695,
      "activations/layer21_attention_weight_min": -21.251657485961914,
      "activations/layer22_attention_weight_max": 28.411231994628906,
      "activations/layer22_attention_weight_min": -27.326284408569336,
      "activations/layer23_attention_weight_max": 32.15843200683594,
      "activations/layer23_attention_weight_min": -23.383291244506836,
      "activations/layer2_attention_weight_max": 29.04347038269043,
      "activations/layer2_attention_weight_min": -28.996967315673828,
      "activations/layer3_attention_weight_max": 81.47306823730469,
      "activations/layer3_attention_weight_min": -82.2986068725586,
      "activations/layer4_attention_weight_max": 79.35853576660156,
      "activations/layer4_attention_weight_min": -80.46976470947266,
      "activations/layer5_attention_weight_max": 57.925453186035156,
      "activations/layer5_attention_weight_min": -70.46087646484375,
      "activations/layer6_attention_weight_max": 46.979148864746094,
      "activations/layer6_attention_weight_min": -48.22258758544922,
      "activations/layer7_attention_weight_max": 60.752220153808594,
      "activations/layer7_attention_weight_min": -59.98868179321289,
      "activations/layer8_attention_weight_max": 45.24691390991211,
      "activations/layer8_attention_weight_min": -50.24685287475586,
      "activations/layer9_attention_weight_max": 41.956905364990234,
      "activations/layer9_attention_weight_min": -46.454341888427734,
      "epoch": 19.92,
      "learning_rate": 2.1733333333333334e-05,
      "loss": 2.7108,
      "step": 342900
    },
    {
      "activations/layer0_attention_weight_max": 15.420441627502441,
      "activations/layer0_attention_weight_min": -14.21292781829834,
      "activations/layer10_attention_weight_max": 38.89821243286133,
      "activations/layer10_attention_weight_min": -38.99384307861328,
      "activations/layer11_attention_weight_max": 34.71736526489258,
      "activations/layer11_attention_weight_min": -36.304656982421875,
      "activations/layer12_attention_weight_max": 18.488754272460938,
      "activations/layer12_attention_weight_min": -25.251890182495117,
      "activations/layer13_attention_weight_max": 29.40891456604004,
      "activations/layer13_attention_weight_min": -28.947738647460938,
      "activations/layer14_attention_weight_max": 32.4530029296875,
      "activations/layer14_attention_weight_min": -31.288551330566406,
      "activations/layer15_attention_weight_max": 30.84395980834961,
      "activations/layer15_attention_weight_min": -30.669143676757812,
      "activations/layer16_attention_weight_max": 32.52100372314453,
      "activations/layer16_attention_weight_min": -32.132347106933594,
      "activations/layer17_attention_weight_max": 50.710243225097656,
      "activations/layer17_attention_weight_min": -47.19568634033203,
      "activations/layer18_attention_weight_max": 45.216758728027344,
      "activations/layer18_attention_weight_min": -39.32943344116211,
      "activations/layer19_attention_weight_max": 24.493438720703125,
      "activations/layer19_attention_weight_min": -24.631750106811523,
      "activations/layer1_attention_weight_max": 15.927230834960938,
      "activations/layer1_attention_weight_min": -14.677735328674316,
      "activations/layer20_attention_weight_max": 22.258792877197266,
      "activations/layer20_attention_weight_min": -23.50236701965332,
      "activations/layer21_attention_weight_max": 31.259984970092773,
      "activations/layer21_attention_weight_min": -26.47737693786621,
      "activations/layer22_attention_weight_max": 27.734447479248047,
      "activations/layer22_attention_weight_min": -25.848596572875977,
      "activations/layer23_attention_weight_max": 33.53435134887695,
      "activations/layer23_attention_weight_min": -25.556194305419922,
      "activations/layer2_attention_weight_max": 31.349498748779297,
      "activations/layer2_attention_weight_min": -30.59292221069336,
      "activations/layer3_attention_weight_max": 82.84246826171875,
      "activations/layer3_attention_weight_min": -85.65426635742188,
      "activations/layer4_attention_weight_max": 80.63362884521484,
      "activations/layer4_attention_weight_min": -81.0957260131836,
      "activations/layer5_attention_weight_max": 60.73966979980469,
      "activations/layer5_attention_weight_min": -68.446044921875,
      "activations/layer6_attention_weight_max": 49.27650451660156,
      "activations/layer6_attention_weight_min": -51.92185592651367,
      "activations/layer7_attention_weight_max": 62.13112258911133,
      "activations/layer7_attention_weight_min": -63.367122650146484,
      "activations/layer8_attention_weight_max": 47.619510650634766,
      "activations/layer8_attention_weight_min": -50.279808044433594,
      "activations/layer9_attention_weight_max": 44.75413131713867,
      "activations/layer9_attention_weight_min": -47.34602355957031,
      "epoch": 19.93,
      "learning_rate": 2.1714393939393936e-05,
      "loss": 2.7111,
      "step": 342950
    },
    {
      "activations/layer0_attention_weight_max": 15.113603591918945,
      "activations/layer0_attention_weight_min": -13.936712265014648,
      "activations/layer10_attention_weight_max": 34.929039001464844,
      "activations/layer10_attention_weight_min": -36.52137756347656,
      "activations/layer11_attention_weight_max": 32.80712127685547,
      "activations/layer11_attention_weight_min": -35.31220245361328,
      "activations/layer12_attention_weight_max": 18.63395118713379,
      "activations/layer12_attention_weight_min": -26.467668533325195,
      "activations/layer13_attention_weight_max": 31.743812561035156,
      "activations/layer13_attention_weight_min": -28.509902954101562,
      "activations/layer14_attention_weight_max": 32.28410720825195,
      "activations/layer14_attention_weight_min": -30.267658233642578,
      "activations/layer15_attention_weight_max": 30.825014114379883,
      "activations/layer15_attention_weight_min": -30.15563201904297,
      "activations/layer16_attention_weight_max": 32.36478805541992,
      "activations/layer16_attention_weight_min": -31.54172706604004,
      "activations/layer17_attention_weight_max": 51.46284866333008,
      "activations/layer17_attention_weight_min": -46.258880615234375,
      "activations/layer18_attention_weight_max": 44.282615661621094,
      "activations/layer18_attention_weight_min": -40.34735107421875,
      "activations/layer19_attention_weight_max": 23.037118911743164,
      "activations/layer19_attention_weight_min": -22.846128463745117,
      "activations/layer1_attention_weight_max": 14.97630786895752,
      "activations/layer1_attention_weight_min": -13.218966484069824,
      "activations/layer20_attention_weight_max": 22.069232940673828,
      "activations/layer20_attention_weight_min": -22.517940521240234,
      "activations/layer21_attention_weight_max": 34.988319396972656,
      "activations/layer21_attention_weight_min": -21.360109329223633,
      "activations/layer22_attention_weight_max": 28.897851943969727,
      "activations/layer22_attention_weight_min": -25.59490966796875,
      "activations/layer23_attention_weight_max": 34.79977798461914,
      "activations/layer23_attention_weight_min": -24.18423080444336,
      "activations/layer2_attention_weight_max": 30.903217315673828,
      "activations/layer2_attention_weight_min": -29.696866989135742,
      "activations/layer3_attention_weight_max": 83.6995620727539,
      "activations/layer3_attention_weight_min": -83.39756774902344,
      "activations/layer4_attention_weight_max": 79.91643524169922,
      "activations/layer4_attention_weight_min": -80.86487579345703,
      "activations/layer5_attention_weight_max": 59.20891571044922,
      "activations/layer5_attention_weight_min": -70.73966979980469,
      "activations/layer6_attention_weight_max": 48.73225021362305,
      "activations/layer6_attention_weight_min": -49.73761749267578,
      "activations/layer7_attention_weight_max": 61.93268585205078,
      "activations/layer7_attention_weight_min": -63.096309661865234,
      "activations/layer8_attention_weight_max": 46.72407531738281,
      "activations/layer8_attention_weight_min": -48.810707092285156,
      "activations/layer9_attention_weight_max": 44.0216064453125,
      "activations/layer9_attention_weight_min": -48.42222213745117,
      "epoch": 19.93,
      "learning_rate": 2.1695454545454544e-05,
      "loss": 2.7279,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.133,
      "eval_samples_per_second": 527.973,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.133,
      "eval_openwebtext_samples_per_second": 527.973,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_wikitext_loss": 2.884765625,
      "eval_wikitext_ppl": 17.89937193856263,
      "eval_wikitext_runtime": 2.0491,
      "eval_wikitext_samples_per_second": 222.541,
      "step": 343000
    },
    {
      "epoch": 19.93,
      "eval_lambada_loss": 2.46484375,
      "eval_lambada_ppl": 11.761644251483835,
      "eval_lambada_runtime": 9.6125,
      "eval_lambada_samples_per_second": 506.526,
      "step": 343000
    },
    {
      "activations/layer0_attention_weight_max": 15.795611381530762,
      "activations/layer0_attention_weight_min": -14.310120582580566,
      "activations/layer10_attention_weight_max": 36.05166244506836,
      "activations/layer10_attention_weight_min": -35.759559631347656,
      "activations/layer11_attention_weight_max": 32.2137336730957,
      "activations/layer11_attention_weight_min": -33.695960998535156,
      "activations/layer12_attention_weight_max": 17.894468307495117,
      "activations/layer12_attention_weight_min": -25.05306625366211,
      "activations/layer13_attention_weight_max": 33.030330657958984,
      "activations/layer13_attention_weight_min": -26.83121109008789,
      "activations/layer14_attention_weight_max": 36.20266342163086,
      "activations/layer14_attention_weight_min": -29.80681800842285,
      "activations/layer15_attention_weight_max": 31.20752716064453,
      "activations/layer15_attention_weight_min": -30.869661331176758,
      "activations/layer16_attention_weight_max": 32.10310363769531,
      "activations/layer16_attention_weight_min": -31.48827362060547,
      "activations/layer17_attention_weight_max": 49.10204315185547,
      "activations/layer17_attention_weight_min": -46.94989776611328,
      "activations/layer18_attention_weight_max": 47.58018112182617,
      "activations/layer18_attention_weight_min": -42.77014923095703,
      "activations/layer19_attention_weight_max": 24.538158416748047,
      "activations/layer19_attention_weight_min": -22.47540855407715,
      "activations/layer1_attention_weight_max": 16.465803146362305,
      "activations/layer1_attention_weight_min": -12.883232116699219,
      "activations/layer20_attention_weight_max": 22.448360443115234,
      "activations/layer20_attention_weight_min": -21.832792282104492,
      "activations/layer21_attention_weight_max": 38.2425651550293,
      "activations/layer21_attention_weight_min": -24.71795082092285,
      "activations/layer22_attention_weight_max": 29.34865379333496,
      "activations/layer22_attention_weight_min": -25.25522232055664,
      "activations/layer23_attention_weight_max": 35.65404510498047,
      "activations/layer23_attention_weight_min": -23.13844108581543,
      "activations/layer2_attention_weight_max": 31.221134185791016,
      "activations/layer2_attention_weight_min": -30.236143112182617,
      "activations/layer3_attention_weight_max": 83.7508773803711,
      "activations/layer3_attention_weight_min": -86.42353057861328,
      "activations/layer4_attention_weight_max": 80.30245208740234,
      "activations/layer4_attention_weight_min": -85.32561492919922,
      "activations/layer5_attention_weight_max": 62.630332946777344,
      "activations/layer5_attention_weight_min": -71.55876922607422,
      "activations/layer6_attention_weight_max": 47.98930740356445,
      "activations/layer6_attention_weight_min": -51.04304885864258,
      "activations/layer7_attention_weight_max": 65.1482925415039,
      "activations/layer7_attention_weight_min": -62.564170837402344,
      "activations/layer8_attention_weight_max": 46.30575180053711,
      "activations/layer8_attention_weight_min": -47.865562438964844,
      "activations/layer9_attention_weight_max": 49.97403335571289,
      "activations/layer9_attention_weight_min": -45.93404769897461,
      "epoch": 19.93,
      "learning_rate": 2.167651515151515e-05,
      "loss": 2.7135,
      "step": 343050
    },
    {
      "activations/layer0_attention_weight_max": 15.14338207244873,
      "activations/layer0_attention_weight_min": -14.469635963439941,
      "activations/layer10_attention_weight_max": 34.67984390258789,
      "activations/layer10_attention_weight_min": -37.929107666015625,
      "activations/layer11_attention_weight_max": 35.265602111816406,
      "activations/layer11_attention_weight_min": -36.035335540771484,
      "activations/layer12_attention_weight_max": 18.2352237701416,
      "activations/layer12_attention_weight_min": -22.799108505249023,
      "activations/layer13_attention_weight_max": 36.05378723144531,
      "activations/layer13_attention_weight_min": -30.94613265991211,
      "activations/layer14_attention_weight_max": 35.89834213256836,
      "activations/layer14_attention_weight_min": -30.786483764648438,
      "activations/layer15_attention_weight_max": 33.03995895385742,
      "activations/layer15_attention_weight_min": -31.447938919067383,
      "activations/layer16_attention_weight_max": 33.04955291748047,
      "activations/layer16_attention_weight_min": -31.450862884521484,
      "activations/layer17_attention_weight_max": 49.42471694946289,
      "activations/layer17_attention_weight_min": -45.42562484741211,
      "activations/layer18_attention_weight_max": 46.384300231933594,
      "activations/layer18_attention_weight_min": -41.31295394897461,
      "activations/layer19_attention_weight_max": 22.279155731201172,
      "activations/layer19_attention_weight_min": -23.185258865356445,
      "activations/layer1_attention_weight_max": 15.610986709594727,
      "activations/layer1_attention_weight_min": -12.604413032531738,
      "activations/layer20_attention_weight_max": 21.99962043762207,
      "activations/layer20_attention_weight_min": -21.925731658935547,
      "activations/layer21_attention_weight_max": 33.340816497802734,
      "activations/layer21_attention_weight_min": -20.627567291259766,
      "activations/layer22_attention_weight_max": 29.968379974365234,
      "activations/layer22_attention_weight_min": -25.038389205932617,
      "activations/layer23_attention_weight_max": 34.84028625488281,
      "activations/layer23_attention_weight_min": -24.347412109375,
      "activations/layer2_attention_weight_max": 31.249900817871094,
      "activations/layer2_attention_weight_min": -29.447834014892578,
      "activations/layer3_attention_weight_max": 81.73100280761719,
      "activations/layer3_attention_weight_min": -85.95819854736328,
      "activations/layer4_attention_weight_max": 81.60747528076172,
      "activations/layer4_attention_weight_min": -77.9298095703125,
      "activations/layer5_attention_weight_max": 60.494720458984375,
      "activations/layer5_attention_weight_min": -72.5688705444336,
      "activations/layer6_attention_weight_max": 48.36852264404297,
      "activations/layer6_attention_weight_min": -49.44312286376953,
      "activations/layer7_attention_weight_max": 71.70230865478516,
      "activations/layer7_attention_weight_min": -63.73674011230469,
      "activations/layer8_attention_weight_max": 48.58985137939453,
      "activations/layer8_attention_weight_min": -50.15755844116211,
      "activations/layer9_attention_weight_max": 53.60361862182617,
      "activations/layer9_attention_weight_min": -45.77520751953125,
      "epoch": 19.94,
      "learning_rate": 2.1657575757575757e-05,
      "loss": 2.7342,
      "step": 343100
    },
    {
      "activations/layer0_attention_weight_max": 14.90202522277832,
      "activations/layer0_attention_weight_min": -14.559459686279297,
      "activations/layer10_attention_weight_max": 36.8557243347168,
      "activations/layer10_attention_weight_min": -38.16169738769531,
      "activations/layer11_attention_weight_max": 36.49406433105469,
      "activations/layer11_attention_weight_min": -36.963809967041016,
      "activations/layer12_attention_weight_max": 20.539865493774414,
      "activations/layer12_attention_weight_min": -37.171546936035156,
      "activations/layer13_attention_weight_max": 32.690670013427734,
      "activations/layer13_attention_weight_min": -27.1920108795166,
      "activations/layer14_attention_weight_max": 36.0063362121582,
      "activations/layer14_attention_weight_min": -30.736894607543945,
      "activations/layer15_attention_weight_max": 34.39096450805664,
      "activations/layer15_attention_weight_min": -29.575969696044922,
      "activations/layer16_attention_weight_max": 38.88773727416992,
      "activations/layer16_attention_weight_min": -35.62168502807617,
      "activations/layer17_attention_weight_max": 56.32783889770508,
      "activations/layer17_attention_weight_min": -49.62773513793945,
      "activations/layer18_attention_weight_max": 51.30207443237305,
      "activations/layer18_attention_weight_min": -40.10440444946289,
      "activations/layer19_attention_weight_max": 23.24028778076172,
      "activations/layer19_attention_weight_min": -21.77999496459961,
      "activations/layer1_attention_weight_max": 15.430055618286133,
      "activations/layer1_attention_weight_min": -13.849291801452637,
      "activations/layer20_attention_weight_max": 21.146020889282227,
      "activations/layer20_attention_weight_min": -21.23192024230957,
      "activations/layer21_attention_weight_max": 36.785247802734375,
      "activations/layer21_attention_weight_min": -23.997570037841797,
      "activations/layer22_attention_weight_max": 29.25262451171875,
      "activations/layer22_attention_weight_min": -26.47908592224121,
      "activations/layer23_attention_weight_max": 36.8908805847168,
      "activations/layer23_attention_weight_min": -24.958181381225586,
      "activations/layer2_attention_weight_max": 31.56829833984375,
      "activations/layer2_attention_weight_min": -30.978012084960938,
      "activations/layer3_attention_weight_max": 90.90623474121094,
      "activations/layer3_attention_weight_min": -87.89366912841797,
      "activations/layer4_attention_weight_max": 88.5342025756836,
      "activations/layer4_attention_weight_min": -83.37340545654297,
      "activations/layer5_attention_weight_max": 64.76805114746094,
      "activations/layer5_attention_weight_min": -77.65815734863281,
      "activations/layer6_attention_weight_max": 51.68325424194336,
      "activations/layer6_attention_weight_min": -52.33903884887695,
      "activations/layer7_attention_weight_max": 68.35957336425781,
      "activations/layer7_attention_weight_min": -67.72510528564453,
      "activations/layer8_attention_weight_max": 47.94301223754883,
      "activations/layer8_attention_weight_min": -52.52186965942383,
      "activations/layer9_attention_weight_max": 45.20694351196289,
      "activations/layer9_attention_weight_min": -47.113433837890625,
      "epoch": 19.94,
      "learning_rate": 2.1639015151515148e-05,
      "loss": 2.7138,
      "step": 343150
    },
    {
      "activations/layer0_attention_weight_max": 15.679184913635254,
      "activations/layer0_attention_weight_min": -14.042205810546875,
      "activations/layer10_attention_weight_max": 42.342689514160156,
      "activations/layer10_attention_weight_min": -42.696834564208984,
      "activations/layer11_attention_weight_max": 40.77857971191406,
      "activations/layer11_attention_weight_min": -38.40742874145508,
      "activations/layer12_attention_weight_max": 20.94919776916504,
      "activations/layer12_attention_weight_min": -24.209400177001953,
      "activations/layer13_attention_weight_max": 58.32166290283203,
      "activations/layer13_attention_weight_min": -31.961502075195312,
      "activations/layer14_attention_weight_max": 67.22648620605469,
      "activations/layer14_attention_weight_min": -33.31188201904297,
      "activations/layer15_attention_weight_max": 58.584136962890625,
      "activations/layer15_attention_weight_min": -33.083641052246094,
      "activations/layer16_attention_weight_max": 43.96269607543945,
      "activations/layer16_attention_weight_min": -35.877174377441406,
      "activations/layer17_attention_weight_max": 63.85350799560547,
      "activations/layer17_attention_weight_min": -54.05619812011719,
      "activations/layer18_attention_weight_max": 61.954505920410156,
      "activations/layer18_attention_weight_min": -48.64438247680664,
      "activations/layer19_attention_weight_max": 33.95338821411133,
      "activations/layer19_attention_weight_min": -28.607473373413086,
      "activations/layer1_attention_weight_max": 15.707579612731934,
      "activations/layer1_attention_weight_min": -14.872291564941406,
      "activations/layer20_attention_weight_max": 34.69518280029297,
      "activations/layer20_attention_weight_min": -26.970863342285156,
      "activations/layer21_attention_weight_max": 55.76082229614258,
      "activations/layer21_attention_weight_min": -28.864761352539062,
      "activations/layer22_attention_weight_max": 39.25796890258789,
      "activations/layer22_attention_weight_min": -29.168285369873047,
      "activations/layer23_attention_weight_max": 39.25662612915039,
      "activations/layer23_attention_weight_min": -25.264606475830078,
      "activations/layer2_attention_weight_max": 33.942138671875,
      "activations/layer2_attention_weight_min": -33.072933197021484,
      "activations/layer3_attention_weight_max": 88.5305404663086,
      "activations/layer3_attention_weight_min": -94.4979476928711,
      "activations/layer4_attention_weight_max": 86.06896209716797,
      "activations/layer4_attention_weight_min": -86.90153503417969,
      "activations/layer5_attention_weight_max": 61.083370208740234,
      "activations/layer5_attention_weight_min": -68.05361938476562,
      "activations/layer6_attention_weight_max": 50.521095275878906,
      "activations/layer6_attention_weight_min": -51.6934700012207,
      "activations/layer7_attention_weight_max": 74.43792724609375,
      "activations/layer7_attention_weight_min": -75.92475891113281,
      "activations/layer8_attention_weight_max": 57.13954162597656,
      "activations/layer8_attention_weight_min": -58.07970428466797,
      "activations/layer9_attention_weight_max": 60.034393310546875,
      "activations/layer9_attention_weight_min": -55.547420501708984,
      "epoch": 19.94,
      "learning_rate": 2.1620075757575756e-05,
      "loss": 2.7289,
      "step": 343200
    },
    {
      "activations/layer0_attention_weight_max": 17.498897552490234,
      "activations/layer0_attention_weight_min": -14.065760612487793,
      "activations/layer10_attention_weight_max": 33.02611541748047,
      "activations/layer10_attention_weight_min": -33.514244079589844,
      "activations/layer11_attention_weight_max": 31.378849029541016,
      "activations/layer11_attention_weight_min": -32.226715087890625,
      "activations/layer12_attention_weight_max": 18.174081802368164,
      "activations/layer12_attention_weight_min": -25.743404388427734,
      "activations/layer13_attention_weight_max": 39.75929641723633,
      "activations/layer13_attention_weight_min": -29.67136001586914,
      "activations/layer14_attention_weight_max": 33.06235122680664,
      "activations/layer14_attention_weight_min": -30.632762908935547,
      "activations/layer15_attention_weight_max": 31.434234619140625,
      "activations/layer15_attention_weight_min": -30.20628547668457,
      "activations/layer16_attention_weight_max": 32.44915008544922,
      "activations/layer16_attention_weight_min": -32.58755874633789,
      "activations/layer17_attention_weight_max": 47.90581130981445,
      "activations/layer17_attention_weight_min": -49.6754035949707,
      "activations/layer18_attention_weight_max": 44.18470001220703,
      "activations/layer18_attention_weight_min": -45.4952507019043,
      "activations/layer19_attention_weight_max": 25.036725997924805,
      "activations/layer19_attention_weight_min": -23.946796417236328,
      "activations/layer1_attention_weight_max": 15.078936576843262,
      "activations/layer1_attention_weight_min": -13.95324420928955,
      "activations/layer20_attention_weight_max": 25.034154891967773,
      "activations/layer20_attention_weight_min": -23.514942169189453,
      "activations/layer21_attention_weight_max": 32.1268424987793,
      "activations/layer21_attention_weight_min": -25.76883888244629,
      "activations/layer22_attention_weight_max": 30.292064666748047,
      "activations/layer22_attention_weight_min": -25.29309844970703,
      "activations/layer23_attention_weight_max": 36.37879943847656,
      "activations/layer23_attention_weight_min": -23.142765045166016,
      "activations/layer2_attention_weight_max": 30.25829315185547,
      "activations/layer2_attention_weight_min": -29.91716766357422,
      "activations/layer3_attention_weight_max": 80.09347534179688,
      "activations/layer3_attention_weight_min": -80.56719970703125,
      "activations/layer4_attention_weight_max": 75.78340148925781,
      "activations/layer4_attention_weight_min": -76.51688385009766,
      "activations/layer5_attention_weight_max": 56.27581787109375,
      "activations/layer5_attention_weight_min": -64.69940948486328,
      "activations/layer6_attention_weight_max": 44.929622650146484,
      "activations/layer6_attention_weight_min": -46.10445785522461,
      "activations/layer7_attention_weight_max": 67.00379943847656,
      "activations/layer7_attention_weight_min": -59.10090637207031,
      "activations/layer8_attention_weight_max": 42.47975540161133,
      "activations/layer8_attention_weight_min": -44.303443908691406,
      "activations/layer9_attention_weight_max": 47.543880462646484,
      "activations/layer9_attention_weight_min": -41.8269157409668,
      "epoch": 19.94,
      "learning_rate": 2.160113636363636e-05,
      "loss": 2.7184,
      "step": 343250
    },
    {
      "activations/layer0_attention_weight_max": 15.833287239074707,
      "activations/layer0_attention_weight_min": -14.210639953613281,
      "activations/layer10_attention_weight_max": 35.31037902832031,
      "activations/layer10_attention_weight_min": -35.18848419189453,
      "activations/layer11_attention_weight_max": 32.05464172363281,
      "activations/layer11_attention_weight_min": -33.87421417236328,
      "activations/layer12_attention_weight_max": 18.798715591430664,
      "activations/layer12_attention_weight_min": -24.20180892944336,
      "activations/layer13_attention_weight_max": 35.05586624145508,
      "activations/layer13_attention_weight_min": -29.938318252563477,
      "activations/layer14_attention_weight_max": 34.75569152832031,
      "activations/layer14_attention_weight_min": -29.67798614501953,
      "activations/layer15_attention_weight_max": 34.08976364135742,
      "activations/layer15_attention_weight_min": -28.645977020263672,
      "activations/layer16_attention_weight_max": 35.41286087036133,
      "activations/layer16_attention_weight_min": -31.466506958007812,
      "activations/layer17_attention_weight_max": 50.862510681152344,
      "activations/layer17_attention_weight_min": -42.23855209350586,
      "activations/layer18_attention_weight_max": 46.107810974121094,
      "activations/layer18_attention_weight_min": -35.83688735961914,
      "activations/layer19_attention_weight_max": 22.325557708740234,
      "activations/layer19_attention_weight_min": -20.147733688354492,
      "activations/layer1_attention_weight_max": 15.960403442382812,
      "activations/layer1_attention_weight_min": -13.391061782836914,
      "activations/layer20_attention_weight_max": 22.79774284362793,
      "activations/layer20_attention_weight_min": -19.474468231201172,
      "activations/layer21_attention_weight_max": 29.901561737060547,
      "activations/layer21_attention_weight_min": -21.074769973754883,
      "activations/layer22_attention_weight_max": 27.549701690673828,
      "activations/layer22_attention_weight_min": -24.181108474731445,
      "activations/layer23_attention_weight_max": 32.99040222167969,
      "activations/layer23_attention_weight_min": -24.240102767944336,
      "activations/layer2_attention_weight_max": 31.597431182861328,
      "activations/layer2_attention_weight_min": -30.64336395263672,
      "activations/layer3_attention_weight_max": 84.81102752685547,
      "activations/layer3_attention_weight_min": -86.42298889160156,
      "activations/layer4_attention_weight_max": 80.85275268554688,
      "activations/layer4_attention_weight_min": -83.46710205078125,
      "activations/layer5_attention_weight_max": 61.14736557006836,
      "activations/layer5_attention_weight_min": -74.48233032226562,
      "activations/layer6_attention_weight_max": 49.94587707519531,
      "activations/layer6_attention_weight_min": -49.5996208190918,
      "activations/layer7_attention_weight_max": 71.5051498413086,
      "activations/layer7_attention_weight_min": -61.19047546386719,
      "activations/layer8_attention_weight_max": 48.75762176513672,
      "activations/layer8_attention_weight_min": -49.72904968261719,
      "activations/layer9_attention_weight_max": 43.802490234375,
      "activations/layer9_attention_weight_min": -45.47053146362305,
      "epoch": 19.95,
      "learning_rate": 2.158219696969697e-05,
      "loss": 2.7354,
      "step": 343300
    },
    {
      "activations/layer0_attention_weight_max": 15.741923332214355,
      "activations/layer0_attention_weight_min": -13.92207145690918,
      "activations/layer10_attention_weight_max": 32.8411979675293,
      "activations/layer10_attention_weight_min": -35.523529052734375,
      "activations/layer11_attention_weight_max": 31.76124382019043,
      "activations/layer11_attention_weight_min": -36.06655502319336,
      "activations/layer12_attention_weight_max": 18.09937858581543,
      "activations/layer12_attention_weight_min": -24.09785270690918,
      "activations/layer13_attention_weight_max": 33.520626068115234,
      "activations/layer13_attention_weight_min": -26.874650955200195,
      "activations/layer14_attention_weight_max": 33.73842239379883,
      "activations/layer14_attention_weight_min": -30.394412994384766,
      "activations/layer15_attention_weight_max": 33.532814025878906,
      "activations/layer15_attention_weight_min": -30.203027725219727,
      "activations/layer16_attention_weight_max": 35.32737731933594,
      "activations/layer16_attention_weight_min": -32.499507904052734,
      "activations/layer17_attention_weight_max": 53.98123550415039,
      "activations/layer17_attention_weight_min": -45.57160949707031,
      "activations/layer18_attention_weight_max": 48.11048126220703,
      "activations/layer18_attention_weight_min": -40.04835510253906,
      "activations/layer19_attention_weight_max": 25.868955612182617,
      "activations/layer19_attention_weight_min": -25.695911407470703,
      "activations/layer1_attention_weight_max": 15.116049766540527,
      "activations/layer1_attention_weight_min": -13.623514175415039,
      "activations/layer20_attention_weight_max": 25.043283462524414,
      "activations/layer20_attention_weight_min": -23.161895751953125,
      "activations/layer21_attention_weight_max": 39.4867057800293,
      "activations/layer21_attention_weight_min": -20.4326114654541,
      "activations/layer22_attention_weight_max": 33.19595718383789,
      "activations/layer22_attention_weight_min": -26.24412727355957,
      "activations/layer23_attention_weight_max": 38.30889129638672,
      "activations/layer23_attention_weight_min": -23.584630966186523,
      "activations/layer2_attention_weight_max": 30.219791412353516,
      "activations/layer2_attention_weight_min": -30.54465103149414,
      "activations/layer3_attention_weight_max": 83.0351333618164,
      "activations/layer3_attention_weight_min": -79.56629180908203,
      "activations/layer4_attention_weight_max": 83.12834930419922,
      "activations/layer4_attention_weight_min": -80.0516586303711,
      "activations/layer5_attention_weight_max": 58.681297302246094,
      "activations/layer5_attention_weight_min": -71.60635375976562,
      "activations/layer6_attention_weight_max": 48.67391586303711,
      "activations/layer6_attention_weight_min": -49.02741622924805,
      "activations/layer7_attention_weight_max": 64.16547393798828,
      "activations/layer7_attention_weight_min": -61.746856689453125,
      "activations/layer8_attention_weight_max": 45.8713264465332,
      "activations/layer8_attention_weight_min": -48.8157958984375,
      "activations/layer9_attention_weight_max": 41.54537582397461,
      "activations/layer9_attention_weight_min": -43.88961410522461,
      "epoch": 19.95,
      "learning_rate": 2.1563257575757574e-05,
      "loss": 2.7351,
      "step": 343350
    },
    {
      "activations/layer0_attention_weight_max": 15.414056777954102,
      "activations/layer0_attention_weight_min": -14.134880065917969,
      "activations/layer10_attention_weight_max": 38.472747802734375,
      "activations/layer10_attention_weight_min": -37.79899597167969,
      "activations/layer11_attention_weight_max": 38.20920181274414,
      "activations/layer11_attention_weight_min": -37.14603805541992,
      "activations/layer12_attention_weight_max": 19.564157485961914,
      "activations/layer12_attention_weight_min": -26.718286514282227,
      "activations/layer13_attention_weight_max": 33.95166015625,
      "activations/layer13_attention_weight_min": -27.551795959472656,
      "activations/layer14_attention_weight_max": 36.22548294067383,
      "activations/layer14_attention_weight_min": -32.993492126464844,
      "activations/layer15_attention_weight_max": 34.283390045166016,
      "activations/layer15_attention_weight_min": -31.277498245239258,
      "activations/layer16_attention_weight_max": 35.51674270629883,
      "activations/layer16_attention_weight_min": -35.15532684326172,
      "activations/layer17_attention_weight_max": 54.37509536743164,
      "activations/layer17_attention_weight_min": -48.11322021484375,
      "activations/layer18_attention_weight_max": 48.387962341308594,
      "activations/layer18_attention_weight_min": -42.02294921875,
      "activations/layer19_attention_weight_max": 28.800357818603516,
      "activations/layer19_attention_weight_min": -24.495136260986328,
      "activations/layer1_attention_weight_max": 15.64818286895752,
      "activations/layer1_attention_weight_min": -13.153374671936035,
      "activations/layer20_attention_weight_max": 23.921409606933594,
      "activations/layer20_attention_weight_min": -22.80091667175293,
      "activations/layer21_attention_weight_max": 34.45308303833008,
      "activations/layer21_attention_weight_min": -22.1744327545166,
      "activations/layer22_attention_weight_max": 33.97040939331055,
      "activations/layer22_attention_weight_min": -26.37114715576172,
      "activations/layer23_attention_weight_max": 36.653568267822266,
      "activations/layer23_attention_weight_min": -23.764127731323242,
      "activations/layer2_attention_weight_max": 30.756925582885742,
      "activations/layer2_attention_weight_min": -30.00313377380371,
      "activations/layer3_attention_weight_max": 90.66747283935547,
      "activations/layer3_attention_weight_min": -87.36100769042969,
      "activations/layer4_attention_weight_max": 86.95069122314453,
      "activations/layer4_attention_weight_min": -83.92436218261719,
      "activations/layer5_attention_weight_max": 63.612510681152344,
      "activations/layer5_attention_weight_min": -70.77766418457031,
      "activations/layer6_attention_weight_max": 49.11817169189453,
      "activations/layer6_attention_weight_min": -49.62427520751953,
      "activations/layer7_attention_weight_max": 69.04824829101562,
      "activations/layer7_attention_weight_min": -64.13606262207031,
      "activations/layer8_attention_weight_max": 50.02507400512695,
      "activations/layer8_attention_weight_min": -50.82356262207031,
      "activations/layer9_attention_weight_max": 46.52216720581055,
      "activations/layer9_attention_weight_min": -47.80592346191406,
      "epoch": 19.95,
      "learning_rate": 2.154431818181818e-05,
      "loss": 2.7298,
      "step": 343400
    },
    {
      "activations/layer0_attention_weight_max": 15.311074256896973,
      "activations/layer0_attention_weight_min": -14.025128364562988,
      "activations/layer10_attention_weight_max": 34.062747955322266,
      "activations/layer10_attention_weight_min": -33.681434631347656,
      "activations/layer11_attention_weight_max": 32.58142852783203,
      "activations/layer11_attention_weight_min": -33.33134078979492,
      "activations/layer12_attention_weight_max": 18.04031753540039,
      "activations/layer12_attention_weight_min": -22.637144088745117,
      "activations/layer13_attention_weight_max": 36.09615707397461,
      "activations/layer13_attention_weight_min": -28.058361053466797,
      "activations/layer14_attention_weight_max": 37.44017028808594,
      "activations/layer14_attention_weight_min": -30.389385223388672,
      "activations/layer15_attention_weight_max": 36.170345306396484,
      "activations/layer15_attention_weight_min": -30.171100616455078,
      "activations/layer16_attention_weight_max": 37.529296875,
      "activations/layer16_attention_weight_min": -37.11421203613281,
      "activations/layer17_attention_weight_max": 55.98760223388672,
      "activations/layer17_attention_weight_min": -50.8443489074707,
      "activations/layer18_attention_weight_max": 53.62543869018555,
      "activations/layer18_attention_weight_min": -43.44085693359375,
      "activations/layer19_attention_weight_max": 29.191268920898438,
      "activations/layer19_attention_weight_min": -23.46895980834961,
      "activations/layer1_attention_weight_max": 15.8911714553833,
      "activations/layer1_attention_weight_min": -13.642093658447266,
      "activations/layer20_attention_weight_max": 24.456987380981445,
      "activations/layer20_attention_weight_min": -21.07333755493164,
      "activations/layer21_attention_weight_max": 37.688453674316406,
      "activations/layer21_attention_weight_min": -22.871217727661133,
      "activations/layer22_attention_weight_max": 35.358497619628906,
      "activations/layer22_attention_weight_min": -27.499170303344727,
      "activations/layer23_attention_weight_max": 51.10376739501953,
      "activations/layer23_attention_weight_min": -23.3251953125,
      "activations/layer2_attention_weight_max": 29.448795318603516,
      "activations/layer2_attention_weight_min": -29.055057525634766,
      "activations/layer3_attention_weight_max": 82.78641510009766,
      "activations/layer3_attention_weight_min": -85.1143569946289,
      "activations/layer4_attention_weight_max": 79.73958587646484,
      "activations/layer4_attention_weight_min": -82.97105407714844,
      "activations/layer5_attention_weight_max": 59.70613098144531,
      "activations/layer5_attention_weight_min": -75.80379486083984,
      "activations/layer6_attention_weight_max": 47.58662414550781,
      "activations/layer6_attention_weight_min": -48.87302780151367,
      "activations/layer7_attention_weight_max": 63.76411056518555,
      "activations/layer7_attention_weight_min": -65.29545593261719,
      "activations/layer8_attention_weight_max": 43.34069061279297,
      "activations/layer8_attention_weight_min": -47.313480377197266,
      "activations/layer9_attention_weight_max": 43.288330078125,
      "activations/layer9_attention_weight_min": -47.08799362182617,
      "epoch": 19.96,
      "learning_rate": 2.1525378787878784e-05,
      "loss": 2.7284,
      "step": 343450
    },
    {
      "activations/layer0_attention_weight_max": 16.4134521484375,
      "activations/layer0_attention_weight_min": -14.581217765808105,
      "activations/layer10_attention_weight_max": 38.130008697509766,
      "activations/layer10_attention_weight_min": -36.57403564453125,
      "activations/layer11_attention_weight_max": 36.2743034362793,
      "activations/layer11_attention_weight_min": -34.91507339477539,
      "activations/layer12_attention_weight_max": 18.466747283935547,
      "activations/layer12_attention_weight_min": -28.39076042175293,
      "activations/layer13_attention_weight_max": 50.71242141723633,
      "activations/layer13_attention_weight_min": -28.180273056030273,
      "activations/layer14_attention_weight_max": 47.519710540771484,
      "activations/layer14_attention_weight_min": -33.727596282958984,
      "activations/layer15_attention_weight_max": 48.440486907958984,
      "activations/layer15_attention_weight_min": -35.5611686706543,
      "activations/layer16_attention_weight_max": 37.12006378173828,
      "activations/layer16_attention_weight_min": -34.546913146972656,
      "activations/layer17_attention_weight_max": 53.246551513671875,
      "activations/layer17_attention_weight_min": -52.354652404785156,
      "activations/layer18_attention_weight_max": 51.42333221435547,
      "activations/layer18_attention_weight_min": -49.8223991394043,
      "activations/layer19_attention_weight_max": 26.832839965820312,
      "activations/layer19_attention_weight_min": -29.6075382232666,
      "activations/layer1_attention_weight_max": 15.905778884887695,
      "activations/layer1_attention_weight_min": -13.09570598602295,
      "activations/layer20_attention_weight_max": 27.683263778686523,
      "activations/layer20_attention_weight_min": -25.03325843811035,
      "activations/layer21_attention_weight_max": 54.45292282104492,
      "activations/layer21_attention_weight_min": -30.730581283569336,
      "activations/layer22_attention_weight_max": 35.4686393737793,
      "activations/layer22_attention_weight_min": -27.737777709960938,
      "activations/layer23_attention_weight_max": 36.955108642578125,
      "activations/layer23_attention_weight_min": -24.207996368408203,
      "activations/layer2_attention_weight_max": 33.43595886230469,
      "activations/layer2_attention_weight_min": -31.121490478515625,
      "activations/layer3_attention_weight_max": 85.83466339111328,
      "activations/layer3_attention_weight_min": -89.0620346069336,
      "activations/layer4_attention_weight_max": 80.37205505371094,
      "activations/layer4_attention_weight_min": -81.16283416748047,
      "activations/layer5_attention_weight_max": 59.7137565612793,
      "activations/layer5_attention_weight_min": -68.0592269897461,
      "activations/layer6_attention_weight_max": 46.9838752746582,
      "activations/layer6_attention_weight_min": -47.83619689941406,
      "activations/layer7_attention_weight_max": 69.6353988647461,
      "activations/layer7_attention_weight_min": -62.24890899658203,
      "activations/layer8_attention_weight_max": 47.70517349243164,
      "activations/layer8_attention_weight_min": -47.70915222167969,
      "activations/layer9_attention_weight_max": 54.07955551147461,
      "activations/layer9_attention_weight_min": -44.56386184692383,
      "epoch": 19.96,
      "learning_rate": 2.1506439393939392e-05,
      "loss": 2.7109,
      "step": 343500
    },
    {
      "activations/layer0_attention_weight_max": 15.52729320526123,
      "activations/layer0_attention_weight_min": -14.06036376953125,
      "activations/layer10_attention_weight_max": 35.08131790161133,
      "activations/layer10_attention_weight_min": -35.86537551879883,
      "activations/layer11_attention_weight_max": 32.097503662109375,
      "activations/layer11_attention_weight_min": -32.76476287841797,
      "activations/layer12_attention_weight_max": 20.424793243408203,
      "activations/layer12_attention_weight_min": -23.913013458251953,
      "activations/layer13_attention_weight_max": 32.73091506958008,
      "activations/layer13_attention_weight_min": -27.694393157958984,
      "activations/layer14_attention_weight_max": 34.764259338378906,
      "activations/layer14_attention_weight_min": -29.601490020751953,
      "activations/layer15_attention_weight_max": 33.077606201171875,
      "activations/layer15_attention_weight_min": -30.760141372680664,
      "activations/layer16_attention_weight_max": 36.88841247558594,
      "activations/layer16_attention_weight_min": -32.42348098754883,
      "activations/layer17_attention_weight_max": 55.21208190917969,
      "activations/layer17_attention_weight_min": -44.444644927978516,
      "activations/layer18_attention_weight_max": 52.308082580566406,
      "activations/layer18_attention_weight_min": -40.7108154296875,
      "activations/layer19_attention_weight_max": 25.59601402282715,
      "activations/layer19_attention_weight_min": -23.63022232055664,
      "activations/layer1_attention_weight_max": 15.524507522583008,
      "activations/layer1_attention_weight_min": -13.961997032165527,
      "activations/layer20_attention_weight_max": 24.899240493774414,
      "activations/layer20_attention_weight_min": -21.767139434814453,
      "activations/layer21_attention_weight_max": 34.31211853027344,
      "activations/layer21_attention_weight_min": -23.949838638305664,
      "activations/layer22_attention_weight_max": 27.714895248413086,
      "activations/layer22_attention_weight_min": -25.442285537719727,
      "activations/layer23_attention_weight_max": 35.93017578125,
      "activations/layer23_attention_weight_min": -24.435134887695312,
      "activations/layer2_attention_weight_max": 32.33180236816406,
      "activations/layer2_attention_weight_min": -31.152341842651367,
      "activations/layer3_attention_weight_max": 81.08706665039062,
      "activations/layer3_attention_weight_min": -85.9621353149414,
      "activations/layer4_attention_weight_max": 82.94808197021484,
      "activations/layer4_attention_weight_min": -85.6910400390625,
      "activations/layer5_attention_weight_max": 59.113243103027344,
      "activations/layer5_attention_weight_min": -71.45592498779297,
      "activations/layer6_attention_weight_max": 49.40507125854492,
      "activations/layer6_attention_weight_min": -50.4505729675293,
      "activations/layer7_attention_weight_max": 62.862003326416016,
      "activations/layer7_attention_weight_min": -67.74249267578125,
      "activations/layer8_attention_weight_max": 46.90983200073242,
      "activations/layer8_attention_weight_min": -53.57709503173828,
      "activations/layer9_attention_weight_max": 44.974430084228516,
      "activations/layer9_attention_weight_min": -47.33437728881836,
      "epoch": 19.96,
      "learning_rate": 2.1487499999999997e-05,
      "loss": 2.7231,
      "step": 343550
    },
    {
      "activations/layer0_attention_weight_max": 14.592016220092773,
      "activations/layer0_attention_weight_min": -13.934813499450684,
      "activations/layer10_attention_weight_max": 35.843345642089844,
      "activations/layer10_attention_weight_min": -36.79004669189453,
      "activations/layer11_attention_weight_max": 32.95906448364258,
      "activations/layer11_attention_weight_min": -35.306068420410156,
      "activations/layer12_attention_weight_max": 19.12411117553711,
      "activations/layer12_attention_weight_min": -32.02703094482422,
      "activations/layer13_attention_weight_max": 32.181060791015625,
      "activations/layer13_attention_weight_min": -27.97776222229004,
      "activations/layer14_attention_weight_max": 35.32674789428711,
      "activations/layer14_attention_weight_min": -31.282869338989258,
      "activations/layer15_attention_weight_max": 31.93684959411621,
      "activations/layer15_attention_weight_min": -31.12814712524414,
      "activations/layer16_attention_weight_max": 31.83518409729004,
      "activations/layer16_attention_weight_min": -31.265962600708008,
      "activations/layer17_attention_weight_max": 47.967288970947266,
      "activations/layer17_attention_weight_min": -45.10103988647461,
      "activations/layer18_attention_weight_max": 42.50627899169922,
      "activations/layer18_attention_weight_min": -40.46050262451172,
      "activations/layer19_attention_weight_max": 23.93890380859375,
      "activations/layer19_attention_weight_min": -24.95452308654785,
      "activations/layer1_attention_weight_max": 16.1510066986084,
      "activations/layer1_attention_weight_min": -13.502171516418457,
      "activations/layer20_attention_weight_max": 21.366411209106445,
      "activations/layer20_attention_weight_min": -26.0436954498291,
      "activations/layer21_attention_weight_max": 32.87393569946289,
      "activations/layer21_attention_weight_min": -22.8377628326416,
      "activations/layer22_attention_weight_max": 29.26123046875,
      "activations/layer22_attention_weight_min": -27.33104133605957,
      "activations/layer23_attention_weight_max": 35.25391387939453,
      "activations/layer23_attention_weight_min": -24.178686141967773,
      "activations/layer2_attention_weight_max": 30.02679443359375,
      "activations/layer2_attention_weight_min": -31.011302947998047,
      "activations/layer3_attention_weight_max": 85.65723419189453,
      "activations/layer3_attention_weight_min": -89.8262710571289,
      "activations/layer4_attention_weight_max": 84.87747955322266,
      "activations/layer4_attention_weight_min": -83.82025909423828,
      "activations/layer5_attention_weight_max": 64.21490478515625,
      "activations/layer5_attention_weight_min": -72.24290466308594,
      "activations/layer6_attention_weight_max": 48.382503509521484,
      "activations/layer6_attention_weight_min": -50.42921447753906,
      "activations/layer7_attention_weight_max": 64.223388671875,
      "activations/layer7_attention_weight_min": -60.9156608581543,
      "activations/layer8_attention_weight_max": 46.748924255371094,
      "activations/layer8_attention_weight_min": -48.50868606567383,
      "activations/layer9_attention_weight_max": 44.57497787475586,
      "activations/layer9_attention_weight_min": -45.66809844970703,
      "epoch": 19.97,
      "learning_rate": 2.1468560606060605e-05,
      "loss": 2.7056,
      "step": 343600
    },
    {
      "activations/layer0_attention_weight_max": 15.681512832641602,
      "activations/layer0_attention_weight_min": -14.792951583862305,
      "activations/layer10_attention_weight_max": 35.65066909790039,
      "activations/layer10_attention_weight_min": -34.6983528137207,
      "activations/layer11_attention_weight_max": 32.84595489501953,
      "activations/layer11_attention_weight_min": -33.632843017578125,
      "activations/layer12_attention_weight_max": 19.35862159729004,
      "activations/layer12_attention_weight_min": -26.92271614074707,
      "activations/layer13_attention_weight_max": 34.98456573486328,
      "activations/layer13_attention_weight_min": -27.958030700683594,
      "activations/layer14_attention_weight_max": 36.487831115722656,
      "activations/layer14_attention_weight_min": -29.345155715942383,
      "activations/layer15_attention_weight_max": 31.705154418945312,
      "activations/layer15_attention_weight_min": -29.15999412536621,
      "activations/layer16_attention_weight_max": 33.136131286621094,
      "activations/layer16_attention_weight_min": -30.87024688720703,
      "activations/layer17_attention_weight_max": 50.411067962646484,
      "activations/layer17_attention_weight_min": -44.2630500793457,
      "activations/layer18_attention_weight_max": 46.53823471069336,
      "activations/layer18_attention_weight_min": -40.64472579956055,
      "activations/layer19_attention_weight_max": 23.158018112182617,
      "activations/layer19_attention_weight_min": -23.197629928588867,
      "activations/layer1_attention_weight_max": 15.263622283935547,
      "activations/layer1_attention_weight_min": -13.144122123718262,
      "activations/layer20_attention_weight_max": 22.9110164642334,
      "activations/layer20_attention_weight_min": -21.674165725708008,
      "activations/layer21_attention_weight_max": 38.063838958740234,
      "activations/layer21_attention_weight_min": -23.06338882446289,
      "activations/layer22_attention_weight_max": 29.697649002075195,
      "activations/layer22_attention_weight_min": -26.175006866455078,
      "activations/layer23_attention_weight_max": 35.600284576416016,
      "activations/layer23_attention_weight_min": -24.768325805664062,
      "activations/layer2_attention_weight_max": 30.104957580566406,
      "activations/layer2_attention_weight_min": -29.4534912109375,
      "activations/layer3_attention_weight_max": 83.93638610839844,
      "activations/layer3_attention_weight_min": -84.06938171386719,
      "activations/layer4_attention_weight_max": 79.50758361816406,
      "activations/layer4_attention_weight_min": -77.00835418701172,
      "activations/layer5_attention_weight_max": 57.7642936706543,
      "activations/layer5_attention_weight_min": -73.53267669677734,
      "activations/layer6_attention_weight_max": 47.41254806518555,
      "activations/layer6_attention_weight_min": -48.749427795410156,
      "activations/layer7_attention_weight_max": 65.66730499267578,
      "activations/layer7_attention_weight_min": -62.563377380371094,
      "activations/layer8_attention_weight_max": 43.905418395996094,
      "activations/layer8_attention_weight_min": -48.814456939697266,
      "activations/layer9_attention_weight_max": 46.08576965332031,
      "activations/layer9_attention_weight_min": -43.800865173339844,
      "epoch": 19.97,
      "learning_rate": 2.1449621212121207e-05,
      "loss": 2.7281,
      "step": 343650
    },
    {
      "activations/layer0_attention_weight_max": 15.559853553771973,
      "activations/layer0_attention_weight_min": -14.195043563842773,
      "activations/layer10_attention_weight_max": 37.756134033203125,
      "activations/layer10_attention_weight_min": -36.94838333129883,
      "activations/layer11_attention_weight_max": 33.40126037597656,
      "activations/layer11_attention_weight_min": -35.018760681152344,
      "activations/layer12_attention_weight_max": 18.674516677856445,
      "activations/layer12_attention_weight_min": -26.156862258911133,
      "activations/layer13_attention_weight_max": 37.161102294921875,
      "activations/layer13_attention_weight_min": -28.34243392944336,
      "activations/layer14_attention_weight_max": 35.67570495605469,
      "activations/layer14_attention_weight_min": -30.334177017211914,
      "activations/layer15_attention_weight_max": 32.25082778930664,
      "activations/layer15_attention_weight_min": -30.68282699584961,
      "activations/layer16_attention_weight_max": 36.88986587524414,
      "activations/layer16_attention_weight_min": -33.855831146240234,
      "activations/layer17_attention_weight_max": 52.41984558105469,
      "activations/layer17_attention_weight_min": -49.4719352722168,
      "activations/layer18_attention_weight_max": 47.666709899902344,
      "activations/layer18_attention_weight_min": -42.37395477294922,
      "activations/layer19_attention_weight_max": 25.6030330657959,
      "activations/layer19_attention_weight_min": -25.73090934753418,
      "activations/layer1_attention_weight_max": 15.926305770874023,
      "activations/layer1_attention_weight_min": -14.838500022888184,
      "activations/layer20_attention_weight_max": 24.91067123413086,
      "activations/layer20_attention_weight_min": -24.261266708374023,
      "activations/layer21_attention_weight_max": 42.40949630737305,
      "activations/layer21_attention_weight_min": -25.225265502929688,
      "activations/layer22_attention_weight_max": 29.901811599731445,
      "activations/layer22_attention_weight_min": -28.184350967407227,
      "activations/layer23_attention_weight_max": 39.38642883300781,
      "activations/layer23_attention_weight_min": -25.22181510925293,
      "activations/layer2_attention_weight_max": 30.250890731811523,
      "activations/layer2_attention_weight_min": -30.420246124267578,
      "activations/layer3_attention_weight_max": 85.671875,
      "activations/layer3_attention_weight_min": -86.77029418945312,
      "activations/layer4_attention_weight_max": 79.19209289550781,
      "activations/layer4_attention_weight_min": -78.8798828125,
      "activations/layer5_attention_weight_max": 67.002685546875,
      "activations/layer5_attention_weight_min": -67.0809097290039,
      "activations/layer6_attention_weight_max": 49.62647247314453,
      "activations/layer6_attention_weight_min": -47.280757904052734,
      "activations/layer7_attention_weight_max": 63.120208740234375,
      "activations/layer7_attention_weight_min": -64.64099884033203,
      "activations/layer8_attention_weight_max": 48.446022033691406,
      "activations/layer8_attention_weight_min": -52.873680114746094,
      "activations/layer9_attention_weight_max": 44.338077545166016,
      "activations/layer9_attention_weight_min": -45.95026779174805,
      "epoch": 19.97,
      "learning_rate": 2.1430681818181815e-05,
      "loss": 2.7359,
      "step": 343700
    },
    {
      "activations/layer0_attention_weight_max": 16.76615333557129,
      "activations/layer0_attention_weight_min": -14.667401313781738,
      "activations/layer10_attention_weight_max": 36.17523956298828,
      "activations/layer10_attention_weight_min": -34.57863998413086,
      "activations/layer11_attention_weight_max": 31.593727111816406,
      "activations/layer11_attention_weight_min": -33.08125305175781,
      "activations/layer12_attention_weight_max": 18.645124435424805,
      "activations/layer12_attention_weight_min": -27.4865665435791,
      "activations/layer13_attention_weight_max": 32.73847961425781,
      "activations/layer13_attention_weight_min": -28.313663482666016,
      "activations/layer14_attention_weight_max": 33.967918395996094,
      "activations/layer14_attention_weight_min": -29.074024200439453,
      "activations/layer15_attention_weight_max": 29.919612884521484,
      "activations/layer15_attention_weight_min": -31.56596565246582,
      "activations/layer16_attention_weight_max": 31.95359230041504,
      "activations/layer16_attention_weight_min": -32.0802116394043,
      "activations/layer17_attention_weight_max": 48.631866455078125,
      "activations/layer17_attention_weight_min": -41.93272018432617,
      "activations/layer18_attention_weight_max": 42.70358657836914,
      "activations/layer18_attention_weight_min": -38.89271545410156,
      "activations/layer19_attention_weight_max": 24.28173065185547,
      "activations/layer19_attention_weight_min": -22.896146774291992,
      "activations/layer1_attention_weight_max": 15.712912559509277,
      "activations/layer1_attention_weight_min": -13.782720565795898,
      "activations/layer20_attention_weight_max": 26.20509910583496,
      "activations/layer20_attention_weight_min": -23.075700759887695,
      "activations/layer21_attention_weight_max": 36.9057731628418,
      "activations/layer21_attention_weight_min": -22.764545440673828,
      "activations/layer22_attention_weight_max": 31.766937255859375,
      "activations/layer22_attention_weight_min": -24.762361526489258,
      "activations/layer23_attention_weight_max": 36.769126892089844,
      "activations/layer23_attention_weight_min": -24.455902099609375,
      "activations/layer2_attention_weight_max": 29.834110260009766,
      "activations/layer2_attention_weight_min": -29.67706871032715,
      "activations/layer3_attention_weight_max": 84.88416290283203,
      "activations/layer3_attention_weight_min": -84.50517272949219,
      "activations/layer4_attention_weight_max": 81.20166015625,
      "activations/layer4_attention_weight_min": -80.69795227050781,
      "activations/layer5_attention_weight_max": 61.97439193725586,
      "activations/layer5_attention_weight_min": -71.89743041992188,
      "activations/layer6_attention_weight_max": 50.5034065246582,
      "activations/layer6_attention_weight_min": -48.895347595214844,
      "activations/layer7_attention_weight_max": 62.747989654541016,
      "activations/layer7_attention_weight_min": -62.90208053588867,
      "activations/layer8_attention_weight_max": 47.43092727661133,
      "activations/layer8_attention_weight_min": -49.20438766479492,
      "activations/layer9_attention_weight_max": 43.55657196044922,
      "activations/layer9_attention_weight_min": -44.58679962158203,
      "epoch": 19.97,
      "learning_rate": 2.1411742424242423e-05,
      "loss": 2.7052,
      "step": 343750
    },
    {
      "activations/layer0_attention_weight_max": 16.344438552856445,
      "activations/layer0_attention_weight_min": -14.582954406738281,
      "activations/layer10_attention_weight_max": 35.44110107421875,
      "activations/layer10_attention_weight_min": -35.42438507080078,
      "activations/layer11_attention_weight_max": 33.73090362548828,
      "activations/layer11_attention_weight_min": -34.956024169921875,
      "activations/layer12_attention_weight_max": 20.024133682250977,
      "activations/layer12_attention_weight_min": -27.488487243652344,
      "activations/layer13_attention_weight_max": 32.405582427978516,
      "activations/layer13_attention_weight_min": -27.97459602355957,
      "activations/layer14_attention_weight_max": 35.59980773925781,
      "activations/layer14_attention_weight_min": -31.554235458374023,
      "activations/layer15_attention_weight_max": 35.93505859375,
      "activations/layer15_attention_weight_min": -30.166399002075195,
      "activations/layer16_attention_weight_max": 37.64196014404297,
      "activations/layer16_attention_weight_min": -35.57884979248047,
      "activations/layer17_attention_weight_max": 55.88402557373047,
      "activations/layer17_attention_weight_min": -51.41048812866211,
      "activations/layer18_attention_weight_max": 51.8451042175293,
      "activations/layer18_attention_weight_min": -45.0890998840332,
      "activations/layer19_attention_weight_max": 28.026147842407227,
      "activations/layer19_attention_weight_min": -26.020475387573242,
      "activations/layer1_attention_weight_max": 15.81517505645752,
      "activations/layer1_attention_weight_min": -13.77669906616211,
      "activations/layer20_attention_weight_max": 25.273452758789062,
      "activations/layer20_attention_weight_min": -21.655412673950195,
      "activations/layer21_attention_weight_max": 31.425756454467773,
      "activations/layer21_attention_weight_min": -21.48548698425293,
      "activations/layer22_attention_weight_max": 30.523090362548828,
      "activations/layer22_attention_weight_min": -24.100894927978516,
      "activations/layer23_attention_weight_max": 35.23783874511719,
      "activations/layer23_attention_weight_min": -24.11795425415039,
      "activations/layer2_attention_weight_max": 30.943347930908203,
      "activations/layer2_attention_weight_min": -29.375890731811523,
      "activations/layer3_attention_weight_max": 89.51187896728516,
      "activations/layer3_attention_weight_min": -87.60865783691406,
      "activations/layer4_attention_weight_max": 83.27145385742188,
      "activations/layer4_attention_weight_min": -82.76576232910156,
      "activations/layer5_attention_weight_max": 63.73175811767578,
      "activations/layer5_attention_weight_min": -72.70152282714844,
      "activations/layer6_attention_weight_max": 49.020626068115234,
      "activations/layer6_attention_weight_min": -51.451881408691406,
      "activations/layer7_attention_weight_max": 65.0179214477539,
      "activations/layer7_attention_weight_min": -67.11955261230469,
      "activations/layer8_attention_weight_max": 45.08835983276367,
      "activations/layer8_attention_weight_min": -51.05791473388672,
      "activations/layer9_attention_weight_max": 43.68301010131836,
      "activations/layer9_attention_weight_min": -44.262447357177734,
      "epoch": 19.98,
      "learning_rate": 2.139280303030303e-05,
      "loss": 2.7345,
      "step": 343800
    },
    {
      "activations/layer0_attention_weight_max": 15.673308372497559,
      "activations/layer0_attention_weight_min": -14.225444793701172,
      "activations/layer10_attention_weight_max": 37.19908142089844,
      "activations/layer10_attention_weight_min": -36.813175201416016,
      "activations/layer11_attention_weight_max": 36.442115783691406,
      "activations/layer11_attention_weight_min": -35.885833740234375,
      "activations/layer12_attention_weight_max": 18.9991455078125,
      "activations/layer12_attention_weight_min": -25.37328338623047,
      "activations/layer13_attention_weight_max": 35.20595169067383,
      "activations/layer13_attention_weight_min": -31.584962844848633,
      "activations/layer14_attention_weight_max": 35.31764602661133,
      "activations/layer14_attention_weight_min": -30.68189239501953,
      "activations/layer15_attention_weight_max": 35.349464416503906,
      "activations/layer15_attention_weight_min": -32.28286361694336,
      "activations/layer16_attention_weight_max": 36.112728118896484,
      "activations/layer16_attention_weight_min": -34.09952163696289,
      "activations/layer17_attention_weight_max": 55.04401397705078,
      "activations/layer17_attention_weight_min": -53.88178634643555,
      "activations/layer18_attention_weight_max": 47.54234313964844,
      "activations/layer18_attention_weight_min": -48.272743225097656,
      "activations/layer19_attention_weight_max": 25.152616500854492,
      "activations/layer19_attention_weight_min": -26.95083999633789,
      "activations/layer1_attention_weight_max": 15.448539733886719,
      "activations/layer1_attention_weight_min": -13.813610076904297,
      "activations/layer20_attention_weight_max": 24.827096939086914,
      "activations/layer20_attention_weight_min": -23.93285369873047,
      "activations/layer21_attention_weight_max": 42.943843841552734,
      "activations/layer21_attention_weight_min": -28.47136116027832,
      "activations/layer22_attention_weight_max": 29.33489227294922,
      "activations/layer22_attention_weight_min": -26.70323944091797,
      "activations/layer23_attention_weight_max": 37.74571990966797,
      "activations/layer23_attention_weight_min": -23.350370407104492,
      "activations/layer2_attention_weight_max": 30.584434509277344,
      "activations/layer2_attention_weight_min": -29.242019653320312,
      "activations/layer3_attention_weight_max": 84.38370513916016,
      "activations/layer3_attention_weight_min": -87.31397247314453,
      "activations/layer4_attention_weight_max": 82.59481048583984,
      "activations/layer4_attention_weight_min": -84.80547332763672,
      "activations/layer5_attention_weight_max": 61.40121078491211,
      "activations/layer5_attention_weight_min": -71.20279693603516,
      "activations/layer6_attention_weight_max": 49.16763687133789,
      "activations/layer6_attention_weight_min": -49.807132720947266,
      "activations/layer7_attention_weight_max": 64.89702606201172,
      "activations/layer7_attention_weight_min": -66.42189025878906,
      "activations/layer8_attention_weight_max": 48.59551239013672,
      "activations/layer8_attention_weight_min": -50.42240524291992,
      "activations/layer9_attention_weight_max": 47.05422592163086,
      "activations/layer9_attention_weight_min": -49.616031646728516,
      "epoch": 19.98,
      "learning_rate": 2.1373863636363637e-05,
      "loss": 2.7241,
      "step": 343850
    },
    {
      "activations/layer0_attention_weight_max": 16.996068954467773,
      "activations/layer0_attention_weight_min": -14.775105476379395,
      "activations/layer10_attention_weight_max": 33.25904083251953,
      "activations/layer10_attention_weight_min": -34.67734909057617,
      "activations/layer11_attention_weight_max": 34.552860260009766,
      "activations/layer11_attention_weight_min": -33.552886962890625,
      "activations/layer12_attention_weight_max": 19.162538528442383,
      "activations/layer12_attention_weight_min": -35.64712142944336,
      "activations/layer13_attention_weight_max": 33.119693756103516,
      "activations/layer13_attention_weight_min": -24.439912796020508,
      "activations/layer14_attention_weight_max": 33.45177459716797,
      "activations/layer14_attention_weight_min": -30.22621726989746,
      "activations/layer15_attention_weight_max": 30.845054626464844,
      "activations/layer15_attention_weight_min": -29.980833053588867,
      "activations/layer16_attention_weight_max": 30.974124908447266,
      "activations/layer16_attention_weight_min": -29.492977142333984,
      "activations/layer17_attention_weight_max": 47.63057327270508,
      "activations/layer17_attention_weight_min": -41.84940719604492,
      "activations/layer18_attention_weight_max": 40.908260345458984,
      "activations/layer18_attention_weight_min": -38.03046798706055,
      "activations/layer19_attention_weight_max": 22.80965805053711,
      "activations/layer19_attention_weight_min": -21.941755294799805,
      "activations/layer1_attention_weight_max": 15.344481468200684,
      "activations/layer1_attention_weight_min": -12.67091178894043,
      "activations/layer20_attention_weight_max": 22.269468307495117,
      "activations/layer20_attention_weight_min": -20.408573150634766,
      "activations/layer21_attention_weight_max": 37.84444046020508,
      "activations/layer21_attention_weight_min": -21.838891983032227,
      "activations/layer22_attention_weight_max": 27.31471061706543,
      "activations/layer22_attention_weight_min": -24.68280792236328,
      "activations/layer23_attention_weight_max": 34.25455856323242,
      "activations/layer23_attention_weight_min": -21.769664764404297,
      "activations/layer2_attention_weight_max": 28.793136596679688,
      "activations/layer2_attention_weight_min": -27.212146759033203,
      "activations/layer3_attention_weight_max": 78.76284790039062,
      "activations/layer3_attention_weight_min": -80.201416015625,
      "activations/layer4_attention_weight_max": 79.08422088623047,
      "activations/layer4_attention_weight_min": -76.01721954345703,
      "activations/layer5_attention_weight_max": 58.91535949707031,
      "activations/layer5_attention_weight_min": -67.80101013183594,
      "activations/layer6_attention_weight_max": 45.401126861572266,
      "activations/layer6_attention_weight_min": -46.04545593261719,
      "activations/layer7_attention_weight_max": 69.37772369384766,
      "activations/layer7_attention_weight_min": -60.99553680419922,
      "activations/layer8_attention_weight_max": 44.45271682739258,
      "activations/layer8_attention_weight_min": -46.891212463378906,
      "activations/layer9_attention_weight_max": 50.48358154296875,
      "activations/layer9_attention_weight_min": -49.990474700927734,
      "epoch": 19.98,
      "learning_rate": 2.135492424242424e-05,
      "loss": 2.7227,
      "step": 343900
    },
    {
      "activations/layer0_attention_weight_max": 16.124866485595703,
      "activations/layer0_attention_weight_min": -14.392672538757324,
      "activations/layer10_attention_weight_max": 34.193382263183594,
      "activations/layer10_attention_weight_min": -35.07081985473633,
      "activations/layer11_attention_weight_max": 32.47626876831055,
      "activations/layer11_attention_weight_min": -35.148712158203125,
      "activations/layer12_attention_weight_max": 18.92242431640625,
      "activations/layer12_attention_weight_min": -26.083948135375977,
      "activations/layer13_attention_weight_max": 35.5118293762207,
      "activations/layer13_attention_weight_min": -27.612192153930664,
      "activations/layer14_attention_weight_max": 40.601016998291016,
      "activations/layer14_attention_weight_min": -28.86811637878418,
      "activations/layer15_attention_weight_max": 38.21864700317383,
      "activations/layer15_attention_weight_min": -29.390201568603516,
      "activations/layer16_attention_weight_max": 36.47654342651367,
      "activations/layer16_attention_weight_min": -31.773176193237305,
      "activations/layer17_attention_weight_max": 55.34831619262695,
      "activations/layer17_attention_weight_min": -45.58639144897461,
      "activations/layer18_attention_weight_max": 49.87213897705078,
      "activations/layer18_attention_weight_min": -39.4911003112793,
      "activations/layer19_attention_weight_max": 24.37356948852539,
      "activations/layer19_attention_weight_min": -23.369115829467773,
      "activations/layer1_attention_weight_max": 14.479887962341309,
      "activations/layer1_attention_weight_min": -12.705042839050293,
      "activations/layer20_attention_weight_max": 23.612810134887695,
      "activations/layer20_attention_weight_min": -19.836198806762695,
      "activations/layer21_attention_weight_max": 33.10601806640625,
      "activations/layer21_attention_weight_min": -20.41419219970703,
      "activations/layer22_attention_weight_max": 27.35466766357422,
      "activations/layer22_attention_weight_min": -22.941505432128906,
      "activations/layer23_attention_weight_max": 36.53184127807617,
      "activations/layer23_attention_weight_min": -22.428861618041992,
      "activations/layer2_attention_weight_max": 31.8461971282959,
      "activations/layer2_attention_weight_min": -31.934696197509766,
      "activations/layer3_attention_weight_max": 86.1445083618164,
      "activations/layer3_attention_weight_min": -93.68885803222656,
      "activations/layer4_attention_weight_max": 82.64006042480469,
      "activations/layer4_attention_weight_min": -83.9566650390625,
      "activations/layer5_attention_weight_max": 64.40461730957031,
      "activations/layer5_attention_weight_min": -73.32492065429688,
      "activations/layer6_attention_weight_max": 50.03157424926758,
      "activations/layer6_attention_weight_min": -51.841556549072266,
      "activations/layer7_attention_weight_max": 69.04216003417969,
      "activations/layer7_attention_weight_min": -64.87189483642578,
      "activations/layer8_attention_weight_max": 46.78654098510742,
      "activations/layer8_attention_weight_min": -51.37617874145508,
      "activations/layer9_attention_weight_max": 45.70174789428711,
      "activations/layer9_attention_weight_min": -46.6031494140625,
      "epoch": 19.99,
      "learning_rate": 2.133598484848485e-05,
      "loss": 2.7265,
      "step": 343950
    },
    {
      "activations/layer0_attention_weight_max": 14.77354907989502,
      "activations/layer0_attention_weight_min": -14.548782348632812,
      "activations/layer10_attention_weight_max": 37.5903205871582,
      "activations/layer10_attention_weight_min": -37.868709564208984,
      "activations/layer11_attention_weight_max": 36.257957458496094,
      "activations/layer11_attention_weight_min": -35.967681884765625,
      "activations/layer12_attention_weight_max": 18.498868942260742,
      "activations/layer12_attention_weight_min": -27.413957595825195,
      "activations/layer13_attention_weight_max": 33.13651657104492,
      "activations/layer13_attention_weight_min": -26.977169036865234,
      "activations/layer14_attention_weight_max": 34.475948333740234,
      "activations/layer14_attention_weight_min": -30.00354766845703,
      "activations/layer15_attention_weight_max": 34.19195556640625,
      "activations/layer15_attention_weight_min": -29.64208221435547,
      "activations/layer16_attention_weight_max": 35.91447830200195,
      "activations/layer16_attention_weight_min": -32.4807243347168,
      "activations/layer17_attention_weight_max": 55.21704864501953,
      "activations/layer17_attention_weight_min": -45.30387878417969,
      "activations/layer18_attention_weight_max": 51.10292053222656,
      "activations/layer18_attention_weight_min": -41.676971435546875,
      "activations/layer19_attention_weight_max": 24.868541717529297,
      "activations/layer19_attention_weight_min": -22.93488121032715,
      "activations/layer1_attention_weight_max": 15.837505340576172,
      "activations/layer1_attention_weight_min": -13.708699226379395,
      "activations/layer20_attention_weight_max": 23.15420150756836,
      "activations/layer20_attention_weight_min": -20.828628540039062,
      "activations/layer21_attention_weight_max": 38.45329666137695,
      "activations/layer21_attention_weight_min": -22.938674926757812,
      "activations/layer22_attention_weight_max": 28.561996459960938,
      "activations/layer22_attention_weight_min": -24.872983932495117,
      "activations/layer23_attention_weight_max": 37.37506103515625,
      "activations/layer23_attention_weight_min": -22.138145446777344,
      "activations/layer2_attention_weight_max": 31.202970504760742,
      "activations/layer2_attention_weight_min": -30.69150161743164,
      "activations/layer3_attention_weight_max": 86.8264389038086,
      "activations/layer3_attention_weight_min": -88.06315612792969,
      "activations/layer4_attention_weight_max": 84.94612884521484,
      "activations/layer4_attention_weight_min": -83.91483306884766,
      "activations/layer5_attention_weight_max": 62.85472106933594,
      "activations/layer5_attention_weight_min": -71.46914672851562,
      "activations/layer6_attention_weight_max": 47.89406967163086,
      "activations/layer6_attention_weight_min": -50.33112716674805,
      "activations/layer7_attention_weight_max": 65.92190551757812,
      "activations/layer7_attention_weight_min": -63.253089904785156,
      "activations/layer8_attention_weight_max": 48.596412658691406,
      "activations/layer8_attention_weight_min": -51.143768310546875,
      "activations/layer9_attention_weight_max": 48.05976104736328,
      "activations/layer9_attention_weight_min": -46.41359329223633,
      "epoch": 19.99,
      "learning_rate": 2.131704545454545e-05,
      "loss": 2.6983,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.5445,
      "eval_samples_per_second": 502.543,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.5445,
      "eval_openwebtext_samples_per_second": 502.543,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 1.706,
      "eval_wikitext_samples_per_second": 267.298,
      "step": 344000
    },
    {
      "epoch": 19.99,
      "eval_lambada_loss": 2.4921875,
      "eval_lambada_ppl": 12.087689040557068,
      "eval_lambada_runtime": 9.3452,
      "eval_lambada_samples_per_second": 521.019,
      "step": 344000
    },
    {
      "activations/layer0_attention_weight_max": 16.250072479248047,
      "activations/layer0_attention_weight_min": -14.146130561828613,
      "activations/layer10_attention_weight_max": 36.433876037597656,
      "activations/layer10_attention_weight_min": -35.470985412597656,
      "activations/layer11_attention_weight_max": 33.96473693847656,
      "activations/layer11_attention_weight_min": -34.119930267333984,
      "activations/layer12_attention_weight_max": 19.719894409179688,
      "activations/layer12_attention_weight_min": -24.467206954956055,
      "activations/layer13_attention_weight_max": 35.23879623413086,
      "activations/layer13_attention_weight_min": -28.8693790435791,
      "activations/layer14_attention_weight_max": 35.283077239990234,
      "activations/layer14_attention_weight_min": -29.311107635498047,
      "activations/layer15_attention_weight_max": 32.60618209838867,
      "activations/layer15_attention_weight_min": -30.2802734375,
      "activations/layer16_attention_weight_max": 34.70668029785156,
      "activations/layer16_attention_weight_min": -33.12386703491211,
      "activations/layer17_attention_weight_max": 52.75370788574219,
      "activations/layer17_attention_weight_min": -44.68378448486328,
      "activations/layer18_attention_weight_max": 48.484901428222656,
      "activations/layer18_attention_weight_min": -38.824501037597656,
      "activations/layer19_attention_weight_max": 26.915067672729492,
      "activations/layer19_attention_weight_min": -22.677095413208008,
      "activations/layer1_attention_weight_max": 15.223804473876953,
      "activations/layer1_attention_weight_min": -13.821090698242188,
      "activations/layer20_attention_weight_max": 23.513614654541016,
      "activations/layer20_attention_weight_min": -22.351903915405273,
      "activations/layer21_attention_weight_max": 32.91017532348633,
      "activations/layer21_attention_weight_min": -21.89829444885254,
      "activations/layer22_attention_weight_max": 26.833099365234375,
      "activations/layer22_attention_weight_min": -23.37466812133789,
      "activations/layer23_attention_weight_max": 35.16059112548828,
      "activations/layer23_attention_weight_min": -24.13446617126465,
      "activations/layer2_attention_weight_max": 32.23372268676758,
      "activations/layer2_attention_weight_min": -31.00225830078125,
      "activations/layer3_attention_weight_max": 86.3558349609375,
      "activations/layer3_attention_weight_min": -91.15096282958984,
      "activations/layer4_attention_weight_max": 87.16883087158203,
      "activations/layer4_attention_weight_min": -86.17044830322266,
      "activations/layer5_attention_weight_max": 63.58579635620117,
      "activations/layer5_attention_weight_min": -71.72196960449219,
      "activations/layer6_attention_weight_max": 51.65361022949219,
      "activations/layer6_attention_weight_min": -51.802547454833984,
      "activations/layer7_attention_weight_max": 68.80265808105469,
      "activations/layer7_attention_weight_min": -63.75871276855469,
      "activations/layer8_attention_weight_max": 48.1602668762207,
      "activations/layer8_attention_weight_min": -51.455448150634766,
      "activations/layer9_attention_weight_max": 43.54522705078125,
      "activations/layer9_attention_weight_min": -46.02185821533203,
      "epoch": 19.99,
      "learning_rate": 2.129810606060606e-05,
      "loss": 2.7266,
      "step": 344050
    },
    {
      "activations/layer0_attention_weight_max": 14.675357818603516,
      "activations/layer0_attention_weight_min": -14.28726577758789,
      "activations/layer10_attention_weight_max": 35.41559982299805,
      "activations/layer10_attention_weight_min": -36.57973098754883,
      "activations/layer11_attention_weight_max": 33.094215393066406,
      "activations/layer11_attention_weight_min": -38.805152893066406,
      "activations/layer12_attention_weight_max": 20.89175796508789,
      "activations/layer12_attention_weight_min": -22.673601150512695,
      "activations/layer13_attention_weight_max": 50.669776916503906,
      "activations/layer13_attention_weight_min": -32.11937713623047,
      "activations/layer14_attention_weight_max": 48.97733688354492,
      "activations/layer14_attention_weight_min": -34.051780700683594,
      "activations/layer15_attention_weight_max": 46.559783935546875,
      "activations/layer15_attention_weight_min": -34.29515075683594,
      "activations/layer16_attention_weight_max": 39.03987121582031,
      "activations/layer16_attention_weight_min": -36.119728088378906,
      "activations/layer17_attention_weight_max": 57.46903610229492,
      "activations/layer17_attention_weight_min": -55.732017517089844,
      "activations/layer18_attention_weight_max": 56.007991790771484,
      "activations/layer18_attention_weight_min": -50.237735748291016,
      "activations/layer19_attention_weight_max": 32.39939498901367,
      "activations/layer19_attention_weight_min": -29.05525016784668,
      "activations/layer1_attention_weight_max": 15.698214530944824,
      "activations/layer1_attention_weight_min": -13.86984634399414,
      "activations/layer20_attention_weight_max": 30.14146614074707,
      "activations/layer20_attention_weight_min": -26.757699966430664,
      "activations/layer21_attention_weight_max": 45.57114791870117,
      "activations/layer21_attention_weight_min": -24.03093147277832,
      "activations/layer22_attention_weight_max": 34.50935363769531,
      "activations/layer22_attention_weight_min": -28.57244110107422,
      "activations/layer23_attention_weight_max": 41.65978240966797,
      "activations/layer23_attention_weight_min": -25.327926635742188,
      "activations/layer2_attention_weight_max": 32.39281463623047,
      "activations/layer2_attention_weight_min": -30.78744125366211,
      "activations/layer3_attention_weight_max": 87.9662857055664,
      "activations/layer3_attention_weight_min": -85.25272369384766,
      "activations/layer4_attention_weight_max": 84.46788787841797,
      "activations/layer4_attention_weight_min": -81.73265838623047,
      "activations/layer5_attention_weight_max": 64.47798156738281,
      "activations/layer5_attention_weight_min": -72.9493179321289,
      "activations/layer6_attention_weight_max": 50.10468673706055,
      "activations/layer6_attention_weight_min": -50.082332611083984,
      "activations/layer7_attention_weight_max": 65.65091705322266,
      "activations/layer7_attention_weight_min": -62.91361999511719,
      "activations/layer8_attention_weight_max": 49.427242279052734,
      "activations/layer8_attention_weight_min": -51.84210205078125,
      "activations/layer9_attention_weight_max": 45.7618522644043,
      "activations/layer9_attention_weight_min": -46.160980224609375,
      "epoch": 19.99,
      "learning_rate": 2.1279166666666664e-05,
      "loss": 2.7203,
      "step": 344100
    },
    {
      "activations/layer0_attention_weight_max": 15.358627319335938,
      "activations/layer0_attention_weight_min": -14.494504928588867,
      "activations/layer10_attention_weight_max": 32.21369552612305,
      "activations/layer10_attention_weight_min": -35.508453369140625,
      "activations/layer11_attention_weight_max": 34.21843719482422,
      "activations/layer11_attention_weight_min": -33.01455307006836,
      "activations/layer12_attention_weight_max": 20.3735294342041,
      "activations/layer12_attention_weight_min": -31.09850311279297,
      "activations/layer13_attention_weight_max": 33.34902572631836,
      "activations/layer13_attention_weight_min": -27.84366226196289,
      "activations/layer14_attention_weight_max": 39.038822174072266,
      "activations/layer14_attention_weight_min": -29.651935577392578,
      "activations/layer15_attention_weight_max": 35.70866012573242,
      "activations/layer15_attention_weight_min": -29.866987228393555,
      "activations/layer16_attention_weight_max": 32.350120544433594,
      "activations/layer16_attention_weight_min": -32.234798431396484,
      "activations/layer17_attention_weight_max": 50.520957946777344,
      "activations/layer17_attention_weight_min": -46.24945068359375,
      "activations/layer18_attention_weight_max": 47.01698303222656,
      "activations/layer18_attention_weight_min": -41.18358612060547,
      "activations/layer19_attention_weight_max": 25.036701202392578,
      "activations/layer19_attention_weight_min": -23.27420997619629,
      "activations/layer1_attention_weight_max": 15.907194137573242,
      "activations/layer1_attention_weight_min": -13.057632446289062,
      "activations/layer20_attention_weight_max": 24.719621658325195,
      "activations/layer20_attention_weight_min": -21.771934509277344,
      "activations/layer21_attention_weight_max": 36.97123718261719,
      "activations/layer21_attention_weight_min": -25.18383026123047,
      "activations/layer22_attention_weight_max": 28.158729553222656,
      "activations/layer22_attention_weight_min": -26.88801383972168,
      "activations/layer23_attention_weight_max": 36.42338562011719,
      "activations/layer23_attention_weight_min": -25.12713623046875,
      "activations/layer2_attention_weight_max": 31.270933151245117,
      "activations/layer2_attention_weight_min": -29.605205535888672,
      "activations/layer3_attention_weight_max": 82.6285171508789,
      "activations/layer3_attention_weight_min": -83.89810943603516,
      "activations/layer4_attention_weight_max": 80.26984405517578,
      "activations/layer4_attention_weight_min": -78.67522430419922,
      "activations/layer5_attention_weight_max": 61.57557678222656,
      "activations/layer5_attention_weight_min": -70.94145202636719,
      "activations/layer6_attention_weight_max": 46.995365142822266,
      "activations/layer6_attention_weight_min": -48.395835876464844,
      "activations/layer7_attention_weight_max": 63.057037353515625,
      "activations/layer7_attention_weight_min": -60.764007568359375,
      "activations/layer8_attention_weight_max": 43.95669937133789,
      "activations/layer8_attention_weight_min": -47.8544921875,
      "activations/layer9_attention_weight_max": 51.40229034423828,
      "activations/layer9_attention_weight_min": -43.741451263427734,
      "epoch": 20.0,
      "learning_rate": 2.1260227272727273e-05,
      "loss": 2.7161,
      "step": 344150
    },
    {
      "activations/layer0_attention_weight_max": 15.76258373260498,
      "activations/layer0_attention_weight_min": -14.360136985778809,
      "activations/layer10_attention_weight_max": 34.60236358642578,
      "activations/layer10_attention_weight_min": -35.904579162597656,
      "activations/layer11_attention_weight_max": 31.840085983276367,
      "activations/layer11_attention_weight_min": -32.577392578125,
      "activations/layer12_attention_weight_max": 19.042919158935547,
      "activations/layer12_attention_weight_min": -23.606521606445312,
      "activations/layer13_attention_weight_max": 31.368215560913086,
      "activations/layer13_attention_weight_min": -26.773399353027344,
      "activations/layer14_attention_weight_max": 35.18641662597656,
      "activations/layer14_attention_weight_min": -29.276992797851562,
      "activations/layer15_attention_weight_max": 35.44310760498047,
      "activations/layer15_attention_weight_min": -29.475936889648438,
      "activations/layer16_attention_weight_max": 33.5517463684082,
      "activations/layer16_attention_weight_min": -31.202184677124023,
      "activations/layer17_attention_weight_max": 53.1092414855957,
      "activations/layer17_attention_weight_min": -44.65653610229492,
      "activations/layer18_attention_weight_max": 50.42460250854492,
      "activations/layer18_attention_weight_min": -43.313472747802734,
      "activations/layer19_attention_weight_max": 25.455055236816406,
      "activations/layer19_attention_weight_min": -25.543928146362305,
      "activations/layer1_attention_weight_max": 15.818476676940918,
      "activations/layer1_attention_weight_min": -14.330267906188965,
      "activations/layer20_attention_weight_max": 25.370140075683594,
      "activations/layer20_attention_weight_min": -22.752227783203125,
      "activations/layer21_attention_weight_max": 35.64262390136719,
      "activations/layer21_attention_weight_min": -23.06278419494629,
      "activations/layer22_attention_weight_max": 30.58124351501465,
      "activations/layer22_attention_weight_min": -26.26565170288086,
      "activations/layer23_attention_weight_max": 36.685890197753906,
      "activations/layer23_attention_weight_min": -23.697187423706055,
      "activations/layer2_attention_weight_max": 32.505332946777344,
      "activations/layer2_attention_weight_min": -29.440275192260742,
      "activations/layer3_attention_weight_max": 84.46672821044922,
      "activations/layer3_attention_weight_min": -82.22218322753906,
      "activations/layer4_attention_weight_max": 79.26187133789062,
      "activations/layer4_attention_weight_min": -80.37866973876953,
      "activations/layer5_attention_weight_max": 62.88259506225586,
      "activations/layer5_attention_weight_min": -70.27103424072266,
      "activations/layer6_attention_weight_max": 47.59563064575195,
      "activations/layer6_attention_weight_min": -48.30946731567383,
      "activations/layer7_attention_weight_max": 61.713600158691406,
      "activations/layer7_attention_weight_min": -63.130455017089844,
      "activations/layer8_attention_weight_max": 43.74028396606445,
      "activations/layer8_attention_weight_min": -48.42842483520508,
      "activations/layer9_attention_weight_max": 40.27619552612305,
      "activations/layer9_attention_weight_min": -43.97539520263672,
      "epoch": 20.0,
      "learning_rate": 2.1241287878787874e-05,
      "loss": 2.7458,
      "step": 344200
    },
    {
      "activations/layer0_attention_weight_max": 15.182788848876953,
      "activations/layer0_attention_weight_min": -14.11555004119873,
      "activations/layer10_attention_weight_max": 35.55329895019531,
      "activations/layer10_attention_weight_min": -35.298282623291016,
      "activations/layer11_attention_weight_max": 34.15522766113281,
      "activations/layer11_attention_weight_min": -33.081295013427734,
      "activations/layer12_attention_weight_max": 18.335054397583008,
      "activations/layer12_attention_weight_min": -24.989356994628906,
      "activations/layer13_attention_weight_max": 35.436702728271484,
      "activations/layer13_attention_weight_min": -26.742679595947266,
      "activations/layer14_attention_weight_max": 33.237525939941406,
      "activations/layer14_attention_weight_min": -30.002592086791992,
      "activations/layer15_attention_weight_max": 32.590782165527344,
      "activations/layer15_attention_weight_min": -29.195858001708984,
      "activations/layer16_attention_weight_max": 34.27812576293945,
      "activations/layer16_attention_weight_min": -32.18462371826172,
      "activations/layer17_attention_weight_max": 49.508243560791016,
      "activations/layer17_attention_weight_min": -45.90620803833008,
      "activations/layer18_attention_weight_max": 44.61384201049805,
      "activations/layer18_attention_weight_min": -42.04221725463867,
      "activations/layer19_attention_weight_max": 22.937339782714844,
      "activations/layer19_attention_weight_min": -22.08048439025879,
      "activations/layer1_attention_weight_max": 16.1628475189209,
      "activations/layer1_attention_weight_min": -13.14534854888916,
      "activations/layer20_attention_weight_max": 22.49713897705078,
      "activations/layer20_attention_weight_min": -21.648845672607422,
      "activations/layer21_attention_weight_max": 37.623497009277344,
      "activations/layer21_attention_weight_min": -21.764883041381836,
      "activations/layer22_attention_weight_max": 29.68381118774414,
      "activations/layer22_attention_weight_min": -25.01059341430664,
      "activations/layer23_attention_weight_max": 38.39766311645508,
      "activations/layer23_attention_weight_min": -25.288345336914062,
      "activations/layer2_attention_weight_max": 30.63613510131836,
      "activations/layer2_attention_weight_min": -28.101722717285156,
      "activations/layer3_attention_weight_max": 80.67216491699219,
      "activations/layer3_attention_weight_min": -81.11310577392578,
      "activations/layer4_attention_weight_max": 79.42967987060547,
      "activations/layer4_attention_weight_min": -79.4318618774414,
      "activations/layer5_attention_weight_max": 62.50025939941406,
      "activations/layer5_attention_weight_min": -70.36491394042969,
      "activations/layer6_attention_weight_max": 46.71244812011719,
      "activations/layer6_attention_weight_min": -47.117794036865234,
      "activations/layer7_attention_weight_max": 61.03998565673828,
      "activations/layer7_attention_weight_min": -63.281402587890625,
      "activations/layer8_attention_weight_max": 45.548492431640625,
      "activations/layer8_attention_weight_min": -48.38612365722656,
      "activations/layer9_attention_weight_max": 45.14521789550781,
      "activations/layer9_attention_weight_min": -46.359127044677734,
      "epoch": 20.0,
      "learning_rate": 2.1222348484848482e-05,
      "loss": 2.7606,
      "step": 344250
    },
    {
      "activations/layer0_attention_weight_max": 15.68829345703125,
      "activations/layer0_attention_weight_min": -14.274816513061523,
      "activations/layer10_attention_weight_max": 35.009422302246094,
      "activations/layer10_attention_weight_min": -35.73092269897461,
      "activations/layer11_attention_weight_max": 32.01598358154297,
      "activations/layer11_attention_weight_min": -35.48487091064453,
      "activations/layer12_attention_weight_max": 17.965869903564453,
      "activations/layer12_attention_weight_min": -24.46561622619629,
      "activations/layer13_attention_weight_max": 30.49146270751953,
      "activations/layer13_attention_weight_min": -27.800365447998047,
      "activations/layer14_attention_weight_max": 34.323055267333984,
      "activations/layer14_attention_weight_min": -28.77783203125,
      "activations/layer15_attention_weight_max": 31.094074249267578,
      "activations/layer15_attention_weight_min": -29.973974227905273,
      "activations/layer16_attention_weight_max": 33.14064407348633,
      "activations/layer16_attention_weight_min": -31.635778427124023,
      "activations/layer17_attention_weight_max": 55.1855583190918,
      "activations/layer17_attention_weight_min": -46.94707489013672,
      "activations/layer18_attention_weight_max": 46.528987884521484,
      "activations/layer18_attention_weight_min": -40.46173858642578,
      "activations/layer19_attention_weight_max": 24.45265769958496,
      "activations/layer19_attention_weight_min": -23.56053924560547,
      "activations/layer1_attention_weight_max": 16.199430465698242,
      "activations/layer1_attention_weight_min": -13.60543155670166,
      "activations/layer20_attention_weight_max": 22.678443908691406,
      "activations/layer20_attention_weight_min": -25.252330780029297,
      "activations/layer21_attention_weight_max": 35.67948913574219,
      "activations/layer21_attention_weight_min": -23.749353408813477,
      "activations/layer22_attention_weight_max": 29.117544174194336,
      "activations/layer22_attention_weight_min": -27.743453979492188,
      "activations/layer23_attention_weight_max": 34.69188690185547,
      "activations/layer23_attention_weight_min": -22.992658615112305,
      "activations/layer2_attention_weight_max": 31.525815963745117,
      "activations/layer2_attention_weight_min": -28.328033447265625,
      "activations/layer3_attention_weight_max": 83.51616668701172,
      "activations/layer3_attention_weight_min": -81.49988555908203,
      "activations/layer4_attention_weight_max": 78.78782653808594,
      "activations/layer4_attention_weight_min": -78.82781219482422,
      "activations/layer5_attention_weight_max": 62.15143585205078,
      "activations/layer5_attention_weight_min": -66.8333511352539,
      "activations/layer6_attention_weight_max": 46.988121032714844,
      "activations/layer6_attention_weight_min": -47.8263053894043,
      "activations/layer7_attention_weight_max": 63.61504364013672,
      "activations/layer7_attention_weight_min": -60.10566711425781,
      "activations/layer8_attention_weight_max": 43.29225540161133,
      "activations/layer8_attention_weight_min": -48.42137908935547,
      "activations/layer9_attention_weight_max": 41.47771453857422,
      "activations/layer9_attention_weight_min": -43.60654067993164,
      "epoch": 20.01,
      "learning_rate": 2.1203409090909087e-05,
      "loss": 2.7098,
      "step": 344300
    },
    {
      "activations/layer0_attention_weight_max": 16.895362854003906,
      "activations/layer0_attention_weight_min": -14.443819999694824,
      "activations/layer10_attention_weight_max": 33.98714828491211,
      "activations/layer10_attention_weight_min": -34.203025817871094,
      "activations/layer11_attention_weight_max": 32.932647705078125,
      "activations/layer11_attention_weight_min": -34.17278289794922,
      "activations/layer12_attention_weight_max": 17.682363510131836,
      "activations/layer12_attention_weight_min": -25.649620056152344,
      "activations/layer13_attention_weight_max": 35.708229064941406,
      "activations/layer13_attention_weight_min": -27.941118240356445,
      "activations/layer14_attention_weight_max": 34.38161087036133,
      "activations/layer14_attention_weight_min": -30.439146041870117,
      "activations/layer15_attention_weight_max": 32.86320877075195,
      "activations/layer15_attention_weight_min": -31.145706176757812,
      "activations/layer16_attention_weight_max": 34.62117385864258,
      "activations/layer16_attention_weight_min": -33.120582580566406,
      "activations/layer17_attention_weight_max": 50.908145904541016,
      "activations/layer17_attention_weight_min": -44.46747589111328,
      "activations/layer18_attention_weight_max": 47.18230056762695,
      "activations/layer18_attention_weight_min": -38.93605422973633,
      "activations/layer19_attention_weight_max": 24.16694450378418,
      "activations/layer19_attention_weight_min": -20.049135208129883,
      "activations/layer1_attention_weight_max": 15.321419715881348,
      "activations/layer1_attention_weight_min": -13.452598571777344,
      "activations/layer20_attention_weight_max": 24.218889236450195,
      "activations/layer20_attention_weight_min": -18.784202575683594,
      "activations/layer21_attention_weight_max": 38.689605712890625,
      "activations/layer21_attention_weight_min": -21.40760612487793,
      "activations/layer22_attention_weight_max": 29.213274002075195,
      "activations/layer22_attention_weight_min": -26.395444869995117,
      "activations/layer23_attention_weight_max": 43.47339630126953,
      "activations/layer23_attention_weight_min": -24.015098571777344,
      "activations/layer2_attention_weight_max": 29.266225814819336,
      "activations/layer2_attention_weight_min": -27.97975730895996,
      "activations/layer3_attention_weight_max": 84.37311553955078,
      "activations/layer3_attention_weight_min": -83.86217498779297,
      "activations/layer4_attention_weight_max": 80.98213195800781,
      "activations/layer4_attention_weight_min": -79.57833862304688,
      "activations/layer5_attention_weight_max": 60.96718978881836,
      "activations/layer5_attention_weight_min": -71.41361999511719,
      "activations/layer6_attention_weight_max": 47.92238998413086,
      "activations/layer6_attention_weight_min": -50.10350036621094,
      "activations/layer7_attention_weight_max": 63.00421905517578,
      "activations/layer7_attention_weight_min": -63.38609313964844,
      "activations/layer8_attention_weight_max": 46.758872985839844,
      "activations/layer8_attention_weight_min": -50.36530685424805,
      "activations/layer9_attention_weight_max": 44.08535385131836,
      "activations/layer9_attention_weight_min": -48.663509368896484,
      "epoch": 20.01,
      "learning_rate": 2.1184469696969696e-05,
      "loss": 2.7135,
      "step": 344350
    },
    {
      "activations/layer0_attention_weight_max": 15.203838348388672,
      "activations/layer0_attention_weight_min": -14.195384979248047,
      "activations/layer10_attention_weight_max": 37.793434143066406,
      "activations/layer10_attention_weight_min": -37.85976028442383,
      "activations/layer11_attention_weight_max": 33.984378814697266,
      "activations/layer11_attention_weight_min": -33.95994567871094,
      "activations/layer12_attention_weight_max": 18.936004638671875,
      "activations/layer12_attention_weight_min": -24.319074630737305,
      "activations/layer13_attention_weight_max": 31.271881103515625,
      "activations/layer13_attention_weight_min": -29.441675186157227,
      "activations/layer14_attention_weight_max": 32.96713638305664,
      "activations/layer14_attention_weight_min": -32.976348876953125,
      "activations/layer15_attention_weight_max": 30.576833724975586,
      "activations/layer15_attention_weight_min": -31.573122024536133,
      "activations/layer16_attention_weight_max": 33.20912551879883,
      "activations/layer16_attention_weight_min": -32.67839813232422,
      "activations/layer17_attention_weight_max": 50.41933059692383,
      "activations/layer17_attention_weight_min": -45.30732345581055,
      "activations/layer18_attention_weight_max": 45.817771911621094,
      "activations/layer18_attention_weight_min": -40.48268127441406,
      "activations/layer19_attention_weight_max": 23.05420684814453,
      "activations/layer19_attention_weight_min": -25.20985221862793,
      "activations/layer1_attention_weight_max": 15.138384819030762,
      "activations/layer1_attention_weight_min": -13.047613143920898,
      "activations/layer20_attention_weight_max": 22.78338623046875,
      "activations/layer20_attention_weight_min": -24.02796173095703,
      "activations/layer21_attention_weight_max": 37.74463653564453,
      "activations/layer21_attention_weight_min": -22.371614456176758,
      "activations/layer22_attention_weight_max": 28.032386779785156,
      "activations/layer22_attention_weight_min": -25.85770606994629,
      "activations/layer23_attention_weight_max": 34.75220489501953,
      "activations/layer23_attention_weight_min": -25.100078582763672,
      "activations/layer2_attention_weight_max": 31.11416244506836,
      "activations/layer2_attention_weight_min": -30.36138153076172,
      "activations/layer3_attention_weight_max": 84.14472961425781,
      "activations/layer3_attention_weight_min": -88.1690673828125,
      "activations/layer4_attention_weight_max": 86.19448852539062,
      "activations/layer4_attention_weight_min": -79.96338653564453,
      "activations/layer5_attention_weight_max": 59.71723556518555,
      "activations/layer5_attention_weight_min": -70.89900207519531,
      "activations/layer6_attention_weight_max": 46.25063705444336,
      "activations/layer6_attention_weight_min": -48.846763610839844,
      "activations/layer7_attention_weight_max": 64.2061767578125,
      "activations/layer7_attention_weight_min": -64.50310516357422,
      "activations/layer8_attention_weight_max": 46.546485900878906,
      "activations/layer8_attention_weight_min": -50.17522430419922,
      "activations/layer9_attention_weight_max": 45.10593795776367,
      "activations/layer9_attention_weight_min": -47.246055603027344,
      "epoch": 20.01,
      "learning_rate": 2.11655303030303e-05,
      "loss": 2.7181,
      "step": 344400
    },
    {
      "activations/layer0_attention_weight_max": 16.44590187072754,
      "activations/layer0_attention_weight_min": -14.429642677307129,
      "activations/layer10_attention_weight_max": 36.54616165161133,
      "activations/layer10_attention_weight_min": -35.21247100830078,
      "activations/layer11_attention_weight_max": 32.84679412841797,
      "activations/layer11_attention_weight_min": -34.03593444824219,
      "activations/layer12_attention_weight_max": 19.5130672454834,
      "activations/layer12_attention_weight_min": -29.394498825073242,
      "activations/layer13_attention_weight_max": 35.35238265991211,
      "activations/layer13_attention_weight_min": -29.838918685913086,
      "activations/layer14_attention_weight_max": 37.232975006103516,
      "activations/layer14_attention_weight_min": -30.13460922241211,
      "activations/layer15_attention_weight_max": 33.29645919799805,
      "activations/layer15_attention_weight_min": -30.589004516601562,
      "activations/layer16_attention_weight_max": 35.27699661254883,
      "activations/layer16_attention_weight_min": -33.249855041503906,
      "activations/layer17_attention_weight_max": 50.502105712890625,
      "activations/layer17_attention_weight_min": -46.784271240234375,
      "activations/layer18_attention_weight_max": 47.29460906982422,
      "activations/layer18_attention_weight_min": -41.652835845947266,
      "activations/layer19_attention_weight_max": 24.851728439331055,
      "activations/layer19_attention_weight_min": -24.35303497314453,
      "activations/layer1_attention_weight_max": 15.2731294631958,
      "activations/layer1_attention_weight_min": -13.214066505432129,
      "activations/layer20_attention_weight_max": 21.977962493896484,
      "activations/layer20_attention_weight_min": -24.40281867980957,
      "activations/layer21_attention_weight_max": 42.29981994628906,
      "activations/layer21_attention_weight_min": -24.97018814086914,
      "activations/layer22_attention_weight_max": 28.21201515197754,
      "activations/layer22_attention_weight_min": -25.25522804260254,
      "activations/layer23_attention_weight_max": 36.098297119140625,
      "activations/layer23_attention_weight_min": -23.751537322998047,
      "activations/layer2_attention_weight_max": 30.30984115600586,
      "activations/layer2_attention_weight_min": -30.65209197998047,
      "activations/layer3_attention_weight_max": 88.56470489501953,
      "activations/layer3_attention_weight_min": -88.75103759765625,
      "activations/layer4_attention_weight_max": 80.87686920166016,
      "activations/layer4_attention_weight_min": -80.16060638427734,
      "activations/layer5_attention_weight_max": 61.497676849365234,
      "activations/layer5_attention_weight_min": -68.20712280273438,
      "activations/layer6_attention_weight_max": 48.77607727050781,
      "activations/layer6_attention_weight_min": -48.08784866333008,
      "activations/layer7_attention_weight_max": 62.87641525268555,
      "activations/layer7_attention_weight_min": -61.88576889038086,
      "activations/layer8_attention_weight_max": 46.98817825317383,
      "activations/layer8_attention_weight_min": -52.106258392333984,
      "activations/layer9_attention_weight_max": 43.378963470458984,
      "activations/layer9_attention_weight_min": -45.41421127319336,
      "epoch": 20.01,
      "learning_rate": 2.114659090909091e-05,
      "loss": 2.7146,
      "step": 344450
    },
    {
      "activations/layer0_attention_weight_max": 15.62281322479248,
      "activations/layer0_attention_weight_min": -14.531109809875488,
      "activations/layer10_attention_weight_max": 39.08451843261719,
      "activations/layer10_attention_weight_min": -36.363922119140625,
      "activations/layer11_attention_weight_max": 36.38490295410156,
      "activations/layer11_attention_weight_min": -35.523216247558594,
      "activations/layer12_attention_weight_max": 19.056272506713867,
      "activations/layer12_attention_weight_min": -25.128128051757812,
      "activations/layer13_attention_weight_max": 41.77416229248047,
      "activations/layer13_attention_weight_min": -28.814403533935547,
      "activations/layer14_attention_weight_max": 43.0015869140625,
      "activations/layer14_attention_weight_min": -32.19484329223633,
      "activations/layer15_attention_weight_max": 42.333675384521484,
      "activations/layer15_attention_weight_min": -31.44234848022461,
      "activations/layer16_attention_weight_max": 39.53714370727539,
      "activations/layer16_attention_weight_min": -38.138465881347656,
      "activations/layer17_attention_weight_max": 54.52882766723633,
      "activations/layer17_attention_weight_min": -53.78109359741211,
      "activations/layer18_attention_weight_max": 48.891990661621094,
      "activations/layer18_attention_weight_min": -46.61594009399414,
      "activations/layer19_attention_weight_max": 22.63988494873047,
      "activations/layer19_attention_weight_min": -22.80684471130371,
      "activations/layer1_attention_weight_max": 15.970677375793457,
      "activations/layer1_attention_weight_min": -14.398798942565918,
      "activations/layer20_attention_weight_max": 22.098960876464844,
      "activations/layer20_attention_weight_min": -22.062671661376953,
      "activations/layer21_attention_weight_max": 33.94231033325195,
      "activations/layer21_attention_weight_min": -23.366477966308594,
      "activations/layer22_attention_weight_max": 28.10747718811035,
      "activations/layer22_attention_weight_min": -26.33462142944336,
      "activations/layer23_attention_weight_max": 33.97854232788086,
      "activations/layer23_attention_weight_min": -25.016399383544922,
      "activations/layer2_attention_weight_max": 31.502334594726562,
      "activations/layer2_attention_weight_min": -31.777036666870117,
      "activations/layer3_attention_weight_max": 83.40145111083984,
      "activations/layer3_attention_weight_min": -85.64309692382812,
      "activations/layer4_attention_weight_max": 83.81231689453125,
      "activations/layer4_attention_weight_min": -81.48200988769531,
      "activations/layer5_attention_weight_max": 59.67557907104492,
      "activations/layer5_attention_weight_min": -71.56767272949219,
      "activations/layer6_attention_weight_max": 48.21088409423828,
      "activations/layer6_attention_weight_min": -52.22273635864258,
      "activations/layer7_attention_weight_max": 59.96160125732422,
      "activations/layer7_attention_weight_min": -64.32523345947266,
      "activations/layer8_attention_weight_max": 46.988861083984375,
      "activations/layer8_attention_weight_min": -48.19572067260742,
      "activations/layer9_attention_weight_max": 43.816566467285156,
      "activations/layer9_attention_weight_min": -47.107784271240234,
      "epoch": 20.02,
      "learning_rate": 2.112765151515151e-05,
      "loss": 2.7184,
      "step": 344500
    },
    {
      "activations/layer0_attention_weight_max": 15.305033683776855,
      "activations/layer0_attention_weight_min": -14.076681137084961,
      "activations/layer10_attention_weight_max": 41.273834228515625,
      "activations/layer10_attention_weight_min": -37.21989822387695,
      "activations/layer11_attention_weight_max": 35.21324920654297,
      "activations/layer11_attention_weight_min": -35.0675163269043,
      "activations/layer12_attention_weight_max": 20.836767196655273,
      "activations/layer12_attention_weight_min": -24.43535804748535,
      "activations/layer13_attention_weight_max": 50.12214660644531,
      "activations/layer13_attention_weight_min": -26.19049835205078,
      "activations/layer14_attention_weight_max": 51.073486328125,
      "activations/layer14_attention_weight_min": -28.82658576965332,
      "activations/layer15_attention_weight_max": 43.05944061279297,
      "activations/layer15_attention_weight_min": -29.958194732666016,
      "activations/layer16_attention_weight_max": 34.414031982421875,
      "activations/layer16_attention_weight_min": -31.825654983520508,
      "activations/layer17_attention_weight_max": 54.32093048095703,
      "activations/layer17_attention_weight_min": -45.2247200012207,
      "activations/layer18_attention_weight_max": 52.85543441772461,
      "activations/layer18_attention_weight_min": -42.30413818359375,
      "activations/layer19_attention_weight_max": 26.733169555664062,
      "activations/layer19_attention_weight_min": -26.035398483276367,
      "activations/layer1_attention_weight_max": 16.455230712890625,
      "activations/layer1_attention_weight_min": -13.84237003326416,
      "activations/layer20_attention_weight_max": 27.613439559936523,
      "activations/layer20_attention_weight_min": -22.461591720581055,
      "activations/layer21_attention_weight_max": 38.442474365234375,
      "activations/layer21_attention_weight_min": -21.280263900756836,
      "activations/layer22_attention_weight_max": 31.54207992553711,
      "activations/layer22_attention_weight_min": -24.644325256347656,
      "activations/layer23_attention_weight_max": 38.90740203857422,
      "activations/layer23_attention_weight_min": -22.68994903564453,
      "activations/layer2_attention_weight_max": 32.750572204589844,
      "activations/layer2_attention_weight_min": -32.071495056152344,
      "activations/layer3_attention_weight_max": 88.65946197509766,
      "activations/layer3_attention_weight_min": -88.50308990478516,
      "activations/layer4_attention_weight_max": 82.81830596923828,
      "activations/layer4_attention_weight_min": -84.88165283203125,
      "activations/layer5_attention_weight_max": 64.09513854980469,
      "activations/layer5_attention_weight_min": -73.67576599121094,
      "activations/layer6_attention_weight_max": 48.80884552001953,
      "activations/layer6_attention_weight_min": -49.91422653198242,
      "activations/layer7_attention_weight_max": 67.14314270019531,
      "activations/layer7_attention_weight_min": -64.4300765991211,
      "activations/layer8_attention_weight_max": 50.25885772705078,
      "activations/layer8_attention_weight_min": -50.42954635620117,
      "activations/layer9_attention_weight_max": 52.311927795410156,
      "activations/layer9_attention_weight_min": -45.89251708984375,
      "epoch": 20.02,
      "learning_rate": 2.110871212121212e-05,
      "loss": 2.726,
      "step": 344550
    },
    {
      "activations/layer0_attention_weight_max": 15.704264640808105,
      "activations/layer0_attention_weight_min": -14.115438461303711,
      "activations/layer10_attention_weight_max": 39.73063659667969,
      "activations/layer10_attention_weight_min": -38.041996002197266,
      "activations/layer11_attention_weight_max": 36.63359832763672,
      "activations/layer11_attention_weight_min": -36.560848236083984,
      "activations/layer12_attention_weight_max": 19.956642150878906,
      "activations/layer12_attention_weight_min": -27.192615509033203,
      "activations/layer13_attention_weight_max": 35.868072509765625,
      "activations/layer13_attention_weight_min": -29.751171112060547,
      "activations/layer14_attention_weight_max": 38.93661880493164,
      "activations/layer14_attention_weight_min": -30.997665405273438,
      "activations/layer15_attention_weight_max": 35.698265075683594,
      "activations/layer15_attention_weight_min": -30.5804443359375,
      "activations/layer16_attention_weight_max": 34.37846755981445,
      "activations/layer16_attention_weight_min": -34.67467498779297,
      "activations/layer17_attention_weight_max": 51.6182975769043,
      "activations/layer17_attention_weight_min": -45.334678649902344,
      "activations/layer18_attention_weight_max": 46.01636505126953,
      "activations/layer18_attention_weight_min": -39.794578552246094,
      "activations/layer19_attention_weight_max": 24.0435848236084,
      "activations/layer19_attention_weight_min": -23.574159622192383,
      "activations/layer1_attention_weight_max": 15.545971870422363,
      "activations/layer1_attention_weight_min": -13.430521011352539,
      "activations/layer20_attention_weight_max": 22.654830932617188,
      "activations/layer20_attention_weight_min": -21.175548553466797,
      "activations/layer21_attention_weight_max": 38.23845291137695,
      "activations/layer21_attention_weight_min": -25.87602424621582,
      "activations/layer22_attention_weight_max": 30.129703521728516,
      "activations/layer22_attention_weight_min": -26.09722900390625,
      "activations/layer23_attention_weight_max": 37.65182876586914,
      "activations/layer23_attention_weight_min": -23.527809143066406,
      "activations/layer2_attention_weight_max": 32.09437942504883,
      "activations/layer2_attention_weight_min": -31.77802848815918,
      "activations/layer3_attention_weight_max": 87.97945404052734,
      "activations/layer3_attention_weight_min": -86.27984619140625,
      "activations/layer4_attention_weight_max": 83.79524993896484,
      "activations/layer4_attention_weight_min": -85.2237548828125,
      "activations/layer5_attention_weight_max": 64.15853881835938,
      "activations/layer5_attention_weight_min": -75.47042846679688,
      "activations/layer6_attention_weight_max": 49.788475036621094,
      "activations/layer6_attention_weight_min": -53.53897476196289,
      "activations/layer7_attention_weight_max": 69.37787628173828,
      "activations/layer7_attention_weight_min": -65.37088775634766,
      "activations/layer8_attention_weight_max": 49.171470642089844,
      "activations/layer8_attention_weight_min": -51.09861755371094,
      "activations/layer9_attention_weight_max": 52.10554885864258,
      "activations/layer9_attention_weight_min": -47.17275619506836,
      "epoch": 20.02,
      "learning_rate": 2.1089772727272727e-05,
      "loss": 2.706,
      "step": 344600
    },
    {
      "activations/layer0_attention_weight_max": 15.77146053314209,
      "activations/layer0_attention_weight_min": -14.301302909851074,
      "activations/layer10_attention_weight_max": 37.07188034057617,
      "activations/layer10_attention_weight_min": -36.02458953857422,
      "activations/layer11_attention_weight_max": 35.69721603393555,
      "activations/layer11_attention_weight_min": -33.69639587402344,
      "activations/layer12_attention_weight_max": 18.792688369750977,
      "activations/layer12_attention_weight_min": -27.10997772216797,
      "activations/layer13_attention_weight_max": 32.92954635620117,
      "activations/layer13_attention_weight_min": -28.718759536743164,
      "activations/layer14_attention_weight_max": 38.585182189941406,
      "activations/layer14_attention_weight_min": -29.562055587768555,
      "activations/layer15_attention_weight_max": 35.01459503173828,
      "activations/layer15_attention_weight_min": -30.533828735351562,
      "activations/layer16_attention_weight_max": 37.53413391113281,
      "activations/layer16_attention_weight_min": -33.956871032714844,
      "activations/layer17_attention_weight_max": 52.982666015625,
      "activations/layer17_attention_weight_min": -45.702571868896484,
      "activations/layer18_attention_weight_max": 47.40882110595703,
      "activations/layer18_attention_weight_min": -40.5382080078125,
      "activations/layer19_attention_weight_max": 23.544538497924805,
      "activations/layer19_attention_weight_min": -24.60863494873047,
      "activations/layer1_attention_weight_max": 15.7138032913208,
      "activations/layer1_attention_weight_min": -12.696611404418945,
      "activations/layer20_attention_weight_max": 22.06424903869629,
      "activations/layer20_attention_weight_min": -25.737049102783203,
      "activations/layer21_attention_weight_max": 35.735740661621094,
      "activations/layer21_attention_weight_min": -26.94578742980957,
      "activations/layer22_attention_weight_max": 28.520586013793945,
      "activations/layer22_attention_weight_min": -30.64289093017578,
      "activations/layer23_attention_weight_max": 35.623207092285156,
      "activations/layer23_attention_weight_min": -26.812416076660156,
      "activations/layer2_attention_weight_max": 30.245647430419922,
      "activations/layer2_attention_weight_min": -30.124502182006836,
      "activations/layer3_attention_weight_max": 82.6585464477539,
      "activations/layer3_attention_weight_min": -87.22254180908203,
      "activations/layer4_attention_weight_max": 79.77080535888672,
      "activations/layer4_attention_weight_min": -80.25137329101562,
      "activations/layer5_attention_weight_max": 61.30942916870117,
      "activations/layer5_attention_weight_min": -67.47589874267578,
      "activations/layer6_attention_weight_max": 46.58926773071289,
      "activations/layer6_attention_weight_min": -47.588294982910156,
      "activations/layer7_attention_weight_max": 61.48455810546875,
      "activations/layer7_attention_weight_min": -63.7281608581543,
      "activations/layer8_attention_weight_max": 47.2818717956543,
      "activations/layer8_attention_weight_min": -54.3719596862793,
      "activations/layer9_attention_weight_max": 49.53636932373047,
      "activations/layer9_attention_weight_min": -45.16949462890625,
      "epoch": 20.03,
      "learning_rate": 2.107083333333333e-05,
      "loss": 2.7222,
      "step": 344650
    },
    {
      "activations/layer0_attention_weight_max": 16.313871383666992,
      "activations/layer0_attention_weight_min": -13.881170272827148,
      "activations/layer10_attention_weight_max": 36.12749099731445,
      "activations/layer10_attention_weight_min": -35.38481521606445,
      "activations/layer11_attention_weight_max": 33.05803680419922,
      "activations/layer11_attention_weight_min": -33.83740997314453,
      "activations/layer12_attention_weight_max": 19.7340145111084,
      "activations/layer12_attention_weight_min": -26.56705665588379,
      "activations/layer13_attention_weight_max": 32.34946823120117,
      "activations/layer13_attention_weight_min": -26.354816436767578,
      "activations/layer14_attention_weight_max": 32.82861328125,
      "activations/layer14_attention_weight_min": -29.85771369934082,
      "activations/layer15_attention_weight_max": 31.69271469116211,
      "activations/layer15_attention_weight_min": -32.86043167114258,
      "activations/layer16_attention_weight_max": 33.11532211303711,
      "activations/layer16_attention_weight_min": -31.807735443115234,
      "activations/layer17_attention_weight_max": 49.26836013793945,
      "activations/layer17_attention_weight_min": -43.151268005371094,
      "activations/layer18_attention_weight_max": 46.606842041015625,
      "activations/layer18_attention_weight_min": -39.77375411987305,
      "activations/layer19_attention_weight_max": 24.117671966552734,
      "activations/layer19_attention_weight_min": -22.335847854614258,
      "activations/layer1_attention_weight_max": 15.169466018676758,
      "activations/layer1_attention_weight_min": -13.813311576843262,
      "activations/layer20_attention_weight_max": 22.5880184173584,
      "activations/layer20_attention_weight_min": -24.130115509033203,
      "activations/layer21_attention_weight_max": 37.9248161315918,
      "activations/layer21_attention_weight_min": -22.451171875,
      "activations/layer22_attention_weight_max": 30.46910858154297,
      "activations/layer22_attention_weight_min": -26.389814376831055,
      "activations/layer23_attention_weight_max": 36.11137008666992,
      "activations/layer23_attention_weight_min": -25.176349639892578,
      "activations/layer2_attention_weight_max": 33.41017150878906,
      "activations/layer2_attention_weight_min": -29.910551071166992,
      "activations/layer3_attention_weight_max": 85.49539184570312,
      "activations/layer3_attention_weight_min": -87.01396942138672,
      "activations/layer4_attention_weight_max": 81.37751007080078,
      "activations/layer4_attention_weight_min": -79.55107879638672,
      "activations/layer5_attention_weight_max": 62.312744140625,
      "activations/layer5_attention_weight_min": -69.47024536132812,
      "activations/layer6_attention_weight_max": 48.774105072021484,
      "activations/layer6_attention_weight_min": -49.70388412475586,
      "activations/layer7_attention_weight_max": 63.81006622314453,
      "activations/layer7_attention_weight_min": -62.05118942260742,
      "activations/layer8_attention_weight_max": 51.065887451171875,
      "activations/layer8_attention_weight_min": -49.855777740478516,
      "activations/layer9_attention_weight_max": 42.403133392333984,
      "activations/layer9_attention_weight_min": -44.58744430541992,
      "epoch": 20.03,
      "learning_rate": 2.105189393939394e-05,
      "loss": 2.7242,
      "step": 344700
    },
    {
      "activations/layer0_attention_weight_max": 15.326865196228027,
      "activations/layer0_attention_weight_min": -14.411201477050781,
      "activations/layer10_attention_weight_max": 34.68013381958008,
      "activations/layer10_attention_weight_min": -36.508392333984375,
      "activations/layer11_attention_weight_max": 33.7454833984375,
      "activations/layer11_attention_weight_min": -34.069190979003906,
      "activations/layer12_attention_weight_max": 19.2375431060791,
      "activations/layer12_attention_weight_min": -27.00269889831543,
      "activations/layer13_attention_weight_max": 32.7860107421875,
      "activations/layer13_attention_weight_min": -26.79184341430664,
      "activations/layer14_attention_weight_max": 36.41071319580078,
      "activations/layer14_attention_weight_min": -30.293302536010742,
      "activations/layer15_attention_weight_max": 34.90209197998047,
      "activations/layer15_attention_weight_min": -31.1412410736084,
      "activations/layer16_attention_weight_max": 38.64142608642578,
      "activations/layer16_attention_weight_min": -33.13619613647461,
      "activations/layer17_attention_weight_max": 62.68263626098633,
      "activations/layer17_attention_weight_min": -54.2443733215332,
      "activations/layer18_attention_weight_max": 57.926483154296875,
      "activations/layer18_attention_weight_min": -43.564456939697266,
      "activations/layer19_attention_weight_max": 24.258024215698242,
      "activations/layer19_attention_weight_min": -22.79874038696289,
      "activations/layer1_attention_weight_max": 16.088808059692383,
      "activations/layer1_attention_weight_min": -14.656225204467773,
      "activations/layer20_attention_weight_max": 24.3441104888916,
      "activations/layer20_attention_weight_min": -22.507963180541992,
      "activations/layer21_attention_weight_max": 37.75432205200195,
      "activations/layer21_attention_weight_min": -21.944623947143555,
      "activations/layer22_attention_weight_max": 26.2707576751709,
      "activations/layer22_attention_weight_min": -24.70235252380371,
      "activations/layer23_attention_weight_max": 32.89610290527344,
      "activations/layer23_attention_weight_min": -24.023099899291992,
      "activations/layer2_attention_weight_max": 30.957927703857422,
      "activations/layer2_attention_weight_min": -30.82115936279297,
      "activations/layer3_attention_weight_max": 83.91568756103516,
      "activations/layer3_attention_weight_min": -88.04083251953125,
      "activations/layer4_attention_weight_max": 78.06718444824219,
      "activations/layer4_attention_weight_min": -75.36544799804688,
      "activations/layer5_attention_weight_max": 59.24105453491211,
      "activations/layer5_attention_weight_min": -75.21844482421875,
      "activations/layer6_attention_weight_max": 45.295692443847656,
      "activations/layer6_attention_weight_min": -47.42282485961914,
      "activations/layer7_attention_weight_max": 68.68109893798828,
      "activations/layer7_attention_weight_min": -61.502960205078125,
      "activations/layer8_attention_weight_max": 45.86421585083008,
      "activations/layer8_attention_weight_min": -50.238563537597656,
      "activations/layer9_attention_weight_max": 50.955718994140625,
      "activations/layer9_attention_weight_min": -45.790775299072266,
      "epoch": 20.03,
      "learning_rate": 2.1032954545454545e-05,
      "loss": 2.7266,
      "step": 344750
    },
    {
      "activations/layer0_attention_weight_max": 15.325311660766602,
      "activations/layer0_attention_weight_min": -14.1512451171875,
      "activations/layer10_attention_weight_max": 35.43082809448242,
      "activations/layer10_attention_weight_min": -37.527183532714844,
      "activations/layer11_attention_weight_max": 32.315452575683594,
      "activations/layer11_attention_weight_min": -38.935604095458984,
      "activations/layer12_attention_weight_max": 18.927824020385742,
      "activations/layer12_attention_weight_min": -22.707284927368164,
      "activations/layer13_attention_weight_max": 31.86119842529297,
      "activations/layer13_attention_weight_min": -27.83766746520996,
      "activations/layer14_attention_weight_max": 34.659889221191406,
      "activations/layer14_attention_weight_min": -31.26165008544922,
      "activations/layer15_attention_weight_max": 31.461084365844727,
      "activations/layer15_attention_weight_min": -31.940275192260742,
      "activations/layer16_attention_weight_max": 33.39640808105469,
      "activations/layer16_attention_weight_min": -32.36365509033203,
      "activations/layer17_attention_weight_max": 53.01766586303711,
      "activations/layer17_attention_weight_min": -46.57849884033203,
      "activations/layer18_attention_weight_max": 46.98362731933594,
      "activations/layer18_attention_weight_min": -41.31745910644531,
      "activations/layer19_attention_weight_max": 23.389842987060547,
      "activations/layer19_attention_weight_min": -23.164587020874023,
      "activations/layer1_attention_weight_max": 16.246915817260742,
      "activations/layer1_attention_weight_min": -13.856691360473633,
      "activations/layer20_attention_weight_max": 21.706241607666016,
      "activations/layer20_attention_weight_min": -23.8067569732666,
      "activations/layer21_attention_weight_max": 32.422607421875,
      "activations/layer21_attention_weight_min": -21.514522552490234,
      "activations/layer22_attention_weight_max": 30.66270637512207,
      "activations/layer22_attention_weight_min": -24.64607810974121,
      "activations/layer23_attention_weight_max": 35.22039031982422,
      "activations/layer23_attention_weight_min": -24.44937515258789,
      "activations/layer2_attention_weight_max": 30.313024520874023,
      "activations/layer2_attention_weight_min": -30.289371490478516,
      "activations/layer3_attention_weight_max": 83.78355407714844,
      "activations/layer3_attention_weight_min": -86.13639831542969,
      "activations/layer4_attention_weight_max": 80.70800018310547,
      "activations/layer4_attention_weight_min": -82.6308822631836,
      "activations/layer5_attention_weight_max": 63.87846374511719,
      "activations/layer5_attention_weight_min": -73.1810302734375,
      "activations/layer6_attention_weight_max": 47.919883728027344,
      "activations/layer6_attention_weight_min": -52.54201126098633,
      "activations/layer7_attention_weight_max": 68.16087341308594,
      "activations/layer7_attention_weight_min": -64.20665740966797,
      "activations/layer8_attention_weight_max": 46.1679573059082,
      "activations/layer8_attention_weight_min": -49.11273193359375,
      "activations/layer9_attention_weight_max": 42.195194244384766,
      "activations/layer9_attention_weight_min": -46.76480484008789,
      "epoch": 20.03,
      "learning_rate": 2.101401515151515e-05,
      "loss": 2.7244,
      "step": 344800
    },
    {
      "activations/layer0_attention_weight_max": 14.522666931152344,
      "activations/layer0_attention_weight_min": -13.990791320800781,
      "activations/layer10_attention_weight_max": 35.35670471191406,
      "activations/layer10_attention_weight_min": -37.27273178100586,
      "activations/layer11_attention_weight_max": 33.30998992919922,
      "activations/layer11_attention_weight_min": -36.704795837402344,
      "activations/layer12_attention_weight_max": 18.067798614501953,
      "activations/layer12_attention_weight_min": -24.706802368164062,
      "activations/layer13_attention_weight_max": 31.06756591796875,
      "activations/layer13_attention_weight_min": -28.5501708984375,
      "activations/layer14_attention_weight_max": 36.988739013671875,
      "activations/layer14_attention_weight_min": -29.83374786376953,
      "activations/layer15_attention_weight_max": 33.96645736694336,
      "activations/layer15_attention_weight_min": -30.102325439453125,
      "activations/layer16_attention_weight_max": 33.8441276550293,
      "activations/layer16_attention_weight_min": -35.25782012939453,
      "activations/layer17_attention_weight_max": 51.32252883911133,
      "activations/layer17_attention_weight_min": -47.48588943481445,
      "activations/layer18_attention_weight_max": 46.97472381591797,
      "activations/layer18_attention_weight_min": -41.012168884277344,
      "activations/layer19_attention_weight_max": 23.069210052490234,
      "activations/layer19_attention_weight_min": -23.148508071899414,
      "activations/layer1_attention_weight_max": 16.264848709106445,
      "activations/layer1_attention_weight_min": -13.892162322998047,
      "activations/layer20_attention_weight_max": 19.797542572021484,
      "activations/layer20_attention_weight_min": -22.106664657592773,
      "activations/layer21_attention_weight_max": 30.36642074584961,
      "activations/layer21_attention_weight_min": -21.041492462158203,
      "activations/layer22_attention_weight_max": 27.845441818237305,
      "activations/layer22_attention_weight_min": -26.471906661987305,
      "activations/layer23_attention_weight_max": 33.230712890625,
      "activations/layer23_attention_weight_min": -26.00170135498047,
      "activations/layer2_attention_weight_max": 33.72047424316406,
      "activations/layer2_attention_weight_min": -30.776569366455078,
      "activations/layer3_attention_weight_max": 90.83562469482422,
      "activations/layer3_attention_weight_min": -89.13797760009766,
      "activations/layer4_attention_weight_max": 89.26535034179688,
      "activations/layer4_attention_weight_min": -87.10769653320312,
      "activations/layer5_attention_weight_max": 68.16838073730469,
      "activations/layer5_attention_weight_min": -71.6974105834961,
      "activations/layer6_attention_weight_max": 52.60322570800781,
      "activations/layer6_attention_weight_min": -51.152488708496094,
      "activations/layer7_attention_weight_max": 68.95836639404297,
      "activations/layer7_attention_weight_min": -68.99413299560547,
      "activations/layer8_attention_weight_max": 50.32757568359375,
      "activations/layer8_attention_weight_min": -51.2380485534668,
      "activations/layer9_attention_weight_max": 52.01961898803711,
      "activations/layer9_attention_weight_min": -49.61048126220703,
      "epoch": 20.04,
      "learning_rate": 2.0995075757575755e-05,
      "loss": 2.7275,
      "step": 344850
    },
    {
      "activations/layer0_attention_weight_max": 15.53071403503418,
      "activations/layer0_attention_weight_min": -14.378472328186035,
      "activations/layer10_attention_weight_max": 35.45909881591797,
      "activations/layer10_attention_weight_min": -35.423828125,
      "activations/layer11_attention_weight_max": 35.18623733520508,
      "activations/layer11_attention_weight_min": -34.74275207519531,
      "activations/layer12_attention_weight_max": 20.699951171875,
      "activations/layer12_attention_weight_min": -23.581787109375,
      "activations/layer13_attention_weight_max": 34.12908935546875,
      "activations/layer13_attention_weight_min": -29.689254760742188,
      "activations/layer14_attention_weight_max": 37.84257125854492,
      "activations/layer14_attention_weight_min": -31.498027801513672,
      "activations/layer15_attention_weight_max": 33.365074157714844,
      "activations/layer15_attention_weight_min": -31.503238677978516,
      "activations/layer16_attention_weight_max": 34.084598541259766,
      "activations/layer16_attention_weight_min": -32.35068130493164,
      "activations/layer17_attention_weight_max": 52.19401550292969,
      "activations/layer17_attention_weight_min": -49.06983184814453,
      "activations/layer18_attention_weight_max": 52.77216339111328,
      "activations/layer18_attention_weight_min": -44.393798828125,
      "activations/layer19_attention_weight_max": 26.61716651916504,
      "activations/layer19_attention_weight_min": -26.162906646728516,
      "activations/layer1_attention_weight_max": 15.69117546081543,
      "activations/layer1_attention_weight_min": -14.511937141418457,
      "activations/layer20_attention_weight_max": 24.831769943237305,
      "activations/layer20_attention_weight_min": -23.626535415649414,
      "activations/layer21_attention_weight_max": 36.58685302734375,
      "activations/layer21_attention_weight_min": -22.806171417236328,
      "activations/layer22_attention_weight_max": 37.45286178588867,
      "activations/layer22_attention_weight_min": -27.593204498291016,
      "activations/layer23_attention_weight_max": 40.39170837402344,
      "activations/layer23_attention_weight_min": -24.14533042907715,
      "activations/layer2_attention_weight_max": 31.14171600341797,
      "activations/layer2_attention_weight_min": -30.656272888183594,
      "activations/layer3_attention_weight_max": 86.55634307861328,
      "activations/layer3_attention_weight_min": -84.71509552001953,
      "activations/layer4_attention_weight_max": 84.85688018798828,
      "activations/layer4_attention_weight_min": -81.71929931640625,
      "activations/layer5_attention_weight_max": 64.80458068847656,
      "activations/layer5_attention_weight_min": -70.26948547363281,
      "activations/layer6_attention_weight_max": 49.151222229003906,
      "activations/layer6_attention_weight_min": -48.78200149536133,
      "activations/layer7_attention_weight_max": 64.92940521240234,
      "activations/layer7_attention_weight_min": -64.86286926269531,
      "activations/layer8_attention_weight_max": 47.58675765991211,
      "activations/layer8_attention_weight_min": -50.19783401489258,
      "activations/layer9_attention_weight_max": 44.545467376708984,
      "activations/layer9_attention_weight_min": -47.03814697265625,
      "epoch": 20.04,
      "learning_rate": 2.0976136363636363e-05,
      "loss": 2.7419,
      "step": 344900
    },
    {
      "activations/layer0_attention_weight_max": 15.069480895996094,
      "activations/layer0_attention_weight_min": -14.039560317993164,
      "activations/layer10_attention_weight_max": 35.05485534667969,
      "activations/layer10_attention_weight_min": -33.28189468383789,
      "activations/layer11_attention_weight_max": 30.358671188354492,
      "activations/layer11_attention_weight_min": -33.61506652832031,
      "activations/layer12_attention_weight_max": 18.736509323120117,
      "activations/layer12_attention_weight_min": -23.3159236907959,
      "activations/layer13_attention_weight_max": 33.6201286315918,
      "activations/layer13_attention_weight_min": -28.796825408935547,
      "activations/layer14_attention_weight_max": 34.75766372680664,
      "activations/layer14_attention_weight_min": -30.177949905395508,
      "activations/layer15_attention_weight_max": 33.4465217590332,
      "activations/layer15_attention_weight_min": -29.594003677368164,
      "activations/layer16_attention_weight_max": 34.898380279541016,
      "activations/layer16_attention_weight_min": -32.37422180175781,
      "activations/layer17_attention_weight_max": 50.28128433227539,
      "activations/layer17_attention_weight_min": -43.717586517333984,
      "activations/layer18_attention_weight_max": 46.88182067871094,
      "activations/layer18_attention_weight_min": -40.089717864990234,
      "activations/layer19_attention_weight_max": 25.282180786132812,
      "activations/layer19_attention_weight_min": -22.940378189086914,
      "activations/layer1_attention_weight_max": 15.400729179382324,
      "activations/layer1_attention_weight_min": -13.166474342346191,
      "activations/layer20_attention_weight_max": 21.282228469848633,
      "activations/layer20_attention_weight_min": -25.138532638549805,
      "activations/layer21_attention_weight_max": 35.8687629699707,
      "activations/layer21_attention_weight_min": -22.984376907348633,
      "activations/layer22_attention_weight_max": 29.23362159729004,
      "activations/layer22_attention_weight_min": -26.15697479248047,
      "activations/layer23_attention_weight_max": 36.570106506347656,
      "activations/layer23_attention_weight_min": -25.389389038085938,
      "activations/layer2_attention_weight_max": 30.06896209716797,
      "activations/layer2_attention_weight_min": -29.319442749023438,
      "activations/layer3_attention_weight_max": 84.65515899658203,
      "activations/layer3_attention_weight_min": -82.98017120361328,
      "activations/layer4_attention_weight_max": 75.42708587646484,
      "activations/layer4_attention_weight_min": -77.06163024902344,
      "activations/layer5_attention_weight_max": 58.20383834838867,
      "activations/layer5_attention_weight_min": -68.57722473144531,
      "activations/layer6_attention_weight_max": 46.30304718017578,
      "activations/layer6_attention_weight_min": -49.68344497680664,
      "activations/layer7_attention_weight_max": 62.00143051147461,
      "activations/layer7_attention_weight_min": -60.101680755615234,
      "activations/layer8_attention_weight_max": 43.66169357299805,
      "activations/layer8_attention_weight_min": -45.452919006347656,
      "activations/layer9_attention_weight_max": 44.971309661865234,
      "activations/layer9_attention_weight_min": -43.17995071411133,
      "epoch": 20.04,
      "learning_rate": 2.0957196969696968e-05,
      "loss": 2.6919,
      "step": 344950
    },
    {
      "activations/layer0_attention_weight_max": 15.177855491638184,
      "activations/layer0_attention_weight_min": -14.250849723815918,
      "activations/layer10_attention_weight_max": 34.67287826538086,
      "activations/layer10_attention_weight_min": -36.626094818115234,
      "activations/layer11_attention_weight_max": 34.67801284790039,
      "activations/layer11_attention_weight_min": -35.724609375,
      "activations/layer12_attention_weight_max": 19.4993896484375,
      "activations/layer12_attention_weight_min": -22.117671966552734,
      "activations/layer13_attention_weight_max": 30.51926612854004,
      "activations/layer13_attention_weight_min": -29.495513916015625,
      "activations/layer14_attention_weight_max": 33.84931182861328,
      "activations/layer14_attention_weight_min": -30.055971145629883,
      "activations/layer15_attention_weight_max": 31.979188919067383,
      "activations/layer15_attention_weight_min": -31.29580307006836,
      "activations/layer16_attention_weight_max": 35.86732864379883,
      "activations/layer16_attention_weight_min": -34.3027229309082,
      "activations/layer17_attention_weight_max": 50.34455108642578,
      "activations/layer17_attention_weight_min": -46.120235443115234,
      "activations/layer18_attention_weight_max": 43.789066314697266,
      "activations/layer18_attention_weight_min": -38.693153381347656,
      "activations/layer19_attention_weight_max": 22.54151725769043,
      "activations/layer19_attention_weight_min": -24.13497543334961,
      "activations/layer1_attention_weight_max": 14.970499038696289,
      "activations/layer1_attention_weight_min": -13.559829711914062,
      "activations/layer20_attention_weight_max": 20.797527313232422,
      "activations/layer20_attention_weight_min": -21.960432052612305,
      "activations/layer21_attention_weight_max": 37.255523681640625,
      "activations/layer21_attention_weight_min": -20.26068115234375,
      "activations/layer22_attention_weight_max": 26.74078369140625,
      "activations/layer22_attention_weight_min": -26.201488494873047,
      "activations/layer23_attention_weight_max": 35.72971725463867,
      "activations/layer23_attention_weight_min": -23.302711486816406,
      "activations/layer2_attention_weight_max": 30.270374298095703,
      "activations/layer2_attention_weight_min": -28.89252471923828,
      "activations/layer3_attention_weight_max": 83.63264465332031,
      "activations/layer3_attention_weight_min": -85.08344268798828,
      "activations/layer4_attention_weight_max": 81.27713012695312,
      "activations/layer4_attention_weight_min": -82.56517028808594,
      "activations/layer5_attention_weight_max": 63.53774642944336,
      "activations/layer5_attention_weight_min": -71.03025817871094,
      "activations/layer6_attention_weight_max": 47.402713775634766,
      "activations/layer6_attention_weight_min": -50.49026107788086,
      "activations/layer7_attention_weight_max": 67.93408966064453,
      "activations/layer7_attention_weight_min": -67.26703643798828,
      "activations/layer8_attention_weight_max": 48.61824417114258,
      "activations/layer8_attention_weight_min": -51.02649688720703,
      "activations/layer9_attention_weight_max": 45.68616485595703,
      "activations/layer9_attention_weight_min": -48.85020065307617,
      "epoch": 20.05,
      "learning_rate": 2.0938257575757576e-05,
      "loss": 2.7303,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_loss": 2.681640625,
      "eval_runtime": 8.5511,
      "eval_samples_per_second": 502.157,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_openwebtext_loss": 2.681640625,
      "eval_openwebtext_ppl": 14.609041604294383,
      "eval_openwebtext_runtime": 8.5511,
      "eval_openwebtext_samples_per_second": 502.157,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 2.0344,
      "eval_wikitext_samples_per_second": 224.142,
      "step": 345000
    },
    {
      "epoch": 20.05,
      "eval_lambada_loss": 2.443359375,
      "eval_lambada_ppl": 11.511647800524438,
      "eval_lambada_runtime": 9.6591,
      "eval_lambada_samples_per_second": 504.087,
      "step": 345000
    },
    {
      "activations/layer0_attention_weight_max": 16.24603271484375,
      "activations/layer0_attention_weight_min": -13.837522506713867,
      "activations/layer10_attention_weight_max": 34.722564697265625,
      "activations/layer10_attention_weight_min": -37.703163146972656,
      "activations/layer11_attention_weight_max": 33.64088439941406,
      "activations/layer11_attention_weight_min": -33.67186737060547,
      "activations/layer12_attention_weight_max": 19.401153564453125,
      "activations/layer12_attention_weight_min": -30.43643569946289,
      "activations/layer13_attention_weight_max": 33.15486526489258,
      "activations/layer13_attention_weight_min": -30.56638526916504,
      "activations/layer14_attention_weight_max": 34.95787048339844,
      "activations/layer14_attention_weight_min": -29.71409034729004,
      "activations/layer15_attention_weight_max": 32.26797866821289,
      "activations/layer15_attention_weight_min": -29.053813934326172,
      "activations/layer16_attention_weight_max": 32.02546310424805,
      "activations/layer16_attention_weight_min": -31.38576316833496,
      "activations/layer17_attention_weight_max": 48.4085578918457,
      "activations/layer17_attention_weight_min": -43.96628952026367,
      "activations/layer18_attention_weight_max": 45.90550994873047,
      "activations/layer18_attention_weight_min": -39.578426361083984,
      "activations/layer19_attention_weight_max": 23.881000518798828,
      "activations/layer19_attention_weight_min": -24.250795364379883,
      "activations/layer1_attention_weight_max": 15.890191078186035,
      "activations/layer1_attention_weight_min": -14.333191871643066,
      "activations/layer20_attention_weight_max": 23.447311401367188,
      "activations/layer20_attention_weight_min": -23.841590881347656,
      "activations/layer21_attention_weight_max": 42.1442985534668,
      "activations/layer21_attention_weight_min": -21.476484298706055,
      "activations/layer22_attention_weight_max": 31.177942276000977,
      "activations/layer22_attention_weight_min": -27.33094024658203,
      "activations/layer23_attention_weight_max": 39.67926788330078,
      "activations/layer23_attention_weight_min": -25.746158599853516,
      "activations/layer2_attention_weight_max": 31.717510223388672,
      "activations/layer2_attention_weight_min": -30.546165466308594,
      "activations/layer3_attention_weight_max": 88.85071563720703,
      "activations/layer3_attention_weight_min": -86.2519302368164,
      "activations/layer4_attention_weight_max": 88.0828628540039,
      "activations/layer4_attention_weight_min": -81.61858367919922,
      "activations/layer5_attention_weight_max": 60.12514877319336,
      "activations/layer5_attention_weight_min": -66.80865478515625,
      "activations/layer6_attention_weight_max": 46.545249938964844,
      "activations/layer6_attention_weight_min": -46.362449645996094,
      "activations/layer7_attention_weight_max": 65.73661804199219,
      "activations/layer7_attention_weight_min": -59.618690490722656,
      "activations/layer8_attention_weight_max": 45.17742919921875,
      "activations/layer8_attention_weight_min": -47.79568099975586,
      "activations/layer9_attention_weight_max": 52.246150970458984,
      "activations/layer9_attention_weight_min": -45.35728454589844,
      "epoch": 20.05,
      "learning_rate": 2.0919318181818177e-05,
      "loss": 2.7246,
      "step": 345050
    },
    {
      "activations/layer0_attention_weight_max": 16.426115036010742,
      "activations/layer0_attention_weight_min": -13.938282012939453,
      "activations/layer10_attention_weight_max": 33.41651916503906,
      "activations/layer10_attention_weight_min": -35.00426483154297,
      "activations/layer11_attention_weight_max": 30.705921173095703,
      "activations/layer11_attention_weight_min": -33.67826461791992,
      "activations/layer12_attention_weight_max": 18.52171516418457,
      "activations/layer12_attention_weight_min": -23.79550552368164,
      "activations/layer13_attention_weight_max": 31.545705795288086,
      "activations/layer13_attention_weight_min": -27.303462982177734,
      "activations/layer14_attention_weight_max": 31.251676559448242,
      "activations/layer14_attention_weight_min": -28.945186614990234,
      "activations/layer15_attention_weight_max": 29.037826538085938,
      "activations/layer15_attention_weight_min": -28.681875228881836,
      "activations/layer16_attention_weight_max": 32.32789611816406,
      "activations/layer16_attention_weight_min": -29.445289611816406,
      "activations/layer17_attention_weight_max": 46.86339569091797,
      "activations/layer17_attention_weight_min": -41.04622268676758,
      "activations/layer18_attention_weight_max": 42.456016540527344,
      "activations/layer18_attention_weight_min": -37.63398361206055,
      "activations/layer19_attention_weight_max": 22.661521911621094,
      "activations/layer19_attention_weight_min": -22.957509994506836,
      "activations/layer1_attention_weight_max": 15.04876708984375,
      "activations/layer1_attention_weight_min": -14.837718963623047,
      "activations/layer20_attention_weight_max": 20.71428680419922,
      "activations/layer20_attention_weight_min": -22.737197875976562,
      "activations/layer21_attention_weight_max": 33.26335525512695,
      "activations/layer21_attention_weight_min": -19.955341339111328,
      "activations/layer22_attention_weight_max": 27.89251708984375,
      "activations/layer22_attention_weight_min": -25.84147834777832,
      "activations/layer23_attention_weight_max": 35.903526306152344,
      "activations/layer23_attention_weight_min": -25.0954647064209,
      "activations/layer2_attention_weight_max": 31.69972801208496,
      "activations/layer2_attention_weight_min": -30.78828239440918,
      "activations/layer3_attention_weight_max": 82.36193084716797,
      "activations/layer3_attention_weight_min": -85.8717041015625,
      "activations/layer4_attention_weight_max": 78.21862030029297,
      "activations/layer4_attention_weight_min": -75.87444305419922,
      "activations/layer5_attention_weight_max": 57.3408203125,
      "activations/layer5_attention_weight_min": -67.87519073486328,
      "activations/layer6_attention_weight_max": 45.15152359008789,
      "activations/layer6_attention_weight_min": -44.873451232910156,
      "activations/layer7_attention_weight_max": 60.706443786621094,
      "activations/layer7_attention_weight_min": -61.403175354003906,
      "activations/layer8_attention_weight_max": 42.51167678833008,
      "activations/layer8_attention_weight_min": -45.52608108520508,
      "activations/layer9_attention_weight_max": 39.0705680847168,
      "activations/layer9_attention_weight_min": -42.67692565917969,
      "epoch": 20.05,
      "learning_rate": 2.0900378787878786e-05,
      "loss": 2.7247,
      "step": 345100
    },
    {
      "activations/layer0_attention_weight_max": 17.105573654174805,
      "activations/layer0_attention_weight_min": -14.062929153442383,
      "activations/layer10_attention_weight_max": 41.9501953125,
      "activations/layer10_attention_weight_min": -38.66618728637695,
      "activations/layer11_attention_weight_max": 36.61231231689453,
      "activations/layer11_attention_weight_min": -35.07714080810547,
      "activations/layer12_attention_weight_max": 18.18572235107422,
      "activations/layer12_attention_weight_min": -29.23651885986328,
      "activations/layer13_attention_weight_max": 38.87675476074219,
      "activations/layer13_attention_weight_min": -27.929725646972656,
      "activations/layer14_attention_weight_max": 39.92344665527344,
      "activations/layer14_attention_weight_min": -29.337575912475586,
      "activations/layer15_attention_weight_max": 41.22272491455078,
      "activations/layer15_attention_weight_min": -31.531761169433594,
      "activations/layer16_attention_weight_max": 34.97983932495117,
      "activations/layer16_attention_weight_min": -33.23219680786133,
      "activations/layer17_attention_weight_max": 50.37575912475586,
      "activations/layer17_attention_weight_min": -48.53921890258789,
      "activations/layer18_attention_weight_max": 44.516868591308594,
      "activations/layer18_attention_weight_min": -43.279029846191406,
      "activations/layer19_attention_weight_max": 24.57137680053711,
      "activations/layer19_attention_weight_min": -23.674158096313477,
      "activations/layer1_attention_weight_max": 15.381438255310059,
      "activations/layer1_attention_weight_min": -13.697883605957031,
      "activations/layer20_attention_weight_max": 22.304622650146484,
      "activations/layer20_attention_weight_min": -22.65761947631836,
      "activations/layer21_attention_weight_max": 29.95622444152832,
      "activations/layer21_attention_weight_min": -21.34049415588379,
      "activations/layer22_attention_weight_max": 26.1864013671875,
      "activations/layer22_attention_weight_min": -25.275592803955078,
      "activations/layer23_attention_weight_max": 33.86259460449219,
      "activations/layer23_attention_weight_min": -24.651981353759766,
      "activations/layer2_attention_weight_max": 31.265880584716797,
      "activations/layer2_attention_weight_min": -30.56747055053711,
      "activations/layer3_attention_weight_max": 89.27600860595703,
      "activations/layer3_attention_weight_min": -85.30793762207031,
      "activations/layer4_attention_weight_max": 82.4134750366211,
      "activations/layer4_attention_weight_min": -78.98509979248047,
      "activations/layer5_attention_weight_max": 60.196678161621094,
      "activations/layer5_attention_weight_min": -68.0049057006836,
      "activations/layer6_attention_weight_max": 48.54550552368164,
      "activations/layer6_attention_weight_min": -48.494632720947266,
      "activations/layer7_attention_weight_max": 67.16181945800781,
      "activations/layer7_attention_weight_min": -64.74163818359375,
      "activations/layer8_attention_weight_max": 52.39097595214844,
      "activations/layer8_attention_weight_min": -54.08918380737305,
      "activations/layer9_attention_weight_max": 51.9088134765625,
      "activations/layer9_attention_weight_min": -48.41865158081055,
      "epoch": 20.06,
      "learning_rate": 2.088143939393939e-05,
      "loss": 2.7183,
      "step": 345150
    },
    {
      "activations/layer0_attention_weight_max": 16.314716339111328,
      "activations/layer0_attention_weight_min": -13.897844314575195,
      "activations/layer10_attention_weight_max": 36.697242736816406,
      "activations/layer10_attention_weight_min": -36.69502639770508,
      "activations/layer11_attention_weight_max": 33.9974365234375,
      "activations/layer11_attention_weight_min": -36.16949462890625,
      "activations/layer12_attention_weight_max": 19.196964263916016,
      "activations/layer12_attention_weight_min": -26.620765686035156,
      "activations/layer13_attention_weight_max": 36.304962158203125,
      "activations/layer13_attention_weight_min": -28.3658504486084,
      "activations/layer14_attention_weight_max": 39.53977584838867,
      "activations/layer14_attention_weight_min": -29.79006004333496,
      "activations/layer15_attention_weight_max": 34.403892517089844,
      "activations/layer15_attention_weight_min": -31.809722900390625,
      "activations/layer16_attention_weight_max": 35.71164321899414,
      "activations/layer16_attention_weight_min": -33.575416564941406,
      "activations/layer17_attention_weight_max": 52.635711669921875,
      "activations/layer17_attention_weight_min": -46.334938049316406,
      "activations/layer18_attention_weight_max": 47.3111572265625,
      "activations/layer18_attention_weight_min": -43.20730209350586,
      "activations/layer19_attention_weight_max": 23.787540435791016,
      "activations/layer19_attention_weight_min": -22.47273826599121,
      "activations/layer1_attention_weight_max": 15.308971405029297,
      "activations/layer1_attention_weight_min": -15.828315734863281,
      "activations/layer20_attention_weight_max": 21.813800811767578,
      "activations/layer20_attention_weight_min": -24.299829483032227,
      "activations/layer21_attention_weight_max": 36.89332962036133,
      "activations/layer21_attention_weight_min": -25.132469177246094,
      "activations/layer22_attention_weight_max": 26.967453002929688,
      "activations/layer22_attention_weight_min": -26.49216651916504,
      "activations/layer23_attention_weight_max": 36.72612762451172,
      "activations/layer23_attention_weight_min": -25.651737213134766,
      "activations/layer2_attention_weight_max": 32.64404296875,
      "activations/layer2_attention_weight_min": -32.413673400878906,
      "activations/layer3_attention_weight_max": 91.98189544677734,
      "activations/layer3_attention_weight_min": -87.29764556884766,
      "activations/layer4_attention_weight_max": 83.43968963623047,
      "activations/layer4_attention_weight_min": -79.4626693725586,
      "activations/layer5_attention_weight_max": 64.46356201171875,
      "activations/layer5_attention_weight_min": -73.6551284790039,
      "activations/layer6_attention_weight_max": 48.4890022277832,
      "activations/layer6_attention_weight_min": -49.238380432128906,
      "activations/layer7_attention_weight_max": 71.12947082519531,
      "activations/layer7_attention_weight_min": -60.891719818115234,
      "activations/layer8_attention_weight_max": 48.04075622558594,
      "activations/layer8_attention_weight_min": -49.204833984375,
      "activations/layer9_attention_weight_max": 55.31422805786133,
      "activations/layer9_attention_weight_min": -48.47626876831055,
      "epoch": 20.06,
      "learning_rate": 2.08625e-05,
      "loss": 2.7197,
      "step": 345200
    },
    {
      "activations/layer0_attention_weight_max": 15.055537223815918,
      "activations/layer0_attention_weight_min": -13.97966480255127,
      "activations/layer10_attention_weight_max": 33.73616409301758,
      "activations/layer10_attention_weight_min": -35.7983283996582,
      "activations/layer11_attention_weight_max": 33.164520263671875,
      "activations/layer11_attention_weight_min": -36.22630310058594,
      "activations/layer12_attention_weight_max": 21.36785316467285,
      "activations/layer12_attention_weight_min": -25.975543975830078,
      "activations/layer13_attention_weight_max": 32.25153350830078,
      "activations/layer13_attention_weight_min": -28.43280601501465,
      "activations/layer14_attention_weight_max": 35.95245361328125,
      "activations/layer14_attention_weight_min": -30.404508590698242,
      "activations/layer15_attention_weight_max": 34.71707534790039,
      "activations/layer15_attention_weight_min": -32.21314239501953,
      "activations/layer16_attention_weight_max": 35.75785446166992,
      "activations/layer16_attention_weight_min": -31.97539520263672,
      "activations/layer17_attention_weight_max": 54.475589752197266,
      "activations/layer17_attention_weight_min": -47.40617752075195,
      "activations/layer18_attention_weight_max": 51.64183044433594,
      "activations/layer18_attention_weight_min": -41.43113708496094,
      "activations/layer19_attention_weight_max": 26.507583618164062,
      "activations/layer19_attention_weight_min": -25.546125411987305,
      "activations/layer1_attention_weight_max": 14.861330032348633,
      "activations/layer1_attention_weight_min": -15.093561172485352,
      "activations/layer20_attention_weight_max": 28.252880096435547,
      "activations/layer20_attention_weight_min": -25.49184799194336,
      "activations/layer21_attention_weight_max": 46.08106231689453,
      "activations/layer21_attention_weight_min": -27.593290328979492,
      "activations/layer22_attention_weight_max": 31.789562225341797,
      "activations/layer22_attention_weight_min": -24.99240493774414,
      "activations/layer23_attention_weight_max": 38.454376220703125,
      "activations/layer23_attention_weight_min": -24.38667106628418,
      "activations/layer2_attention_weight_max": 31.103607177734375,
      "activations/layer2_attention_weight_min": -29.62450408935547,
      "activations/layer3_attention_weight_max": 84.33892059326172,
      "activations/layer3_attention_weight_min": -88.335205078125,
      "activations/layer4_attention_weight_max": 84.6938705444336,
      "activations/layer4_attention_weight_min": -85.03784942626953,
      "activations/layer5_attention_weight_max": 64.8248291015625,
      "activations/layer5_attention_weight_min": -69.02085876464844,
      "activations/layer6_attention_weight_max": 48.09040451049805,
      "activations/layer6_attention_weight_min": -48.964500427246094,
      "activations/layer7_attention_weight_max": 67.7269515991211,
      "activations/layer7_attention_weight_min": -64.28966522216797,
      "activations/layer8_attention_weight_max": 48.0653190612793,
      "activations/layer8_attention_weight_min": -48.55174255371094,
      "activations/layer9_attention_weight_max": 41.160213470458984,
      "activations/layer9_attention_weight_min": -43.102291107177734,
      "epoch": 20.06,
      "learning_rate": 2.0843560606060604e-05,
      "loss": 2.7149,
      "step": 345250
    },
    {
      "activations/layer0_attention_weight_max": 15.687074661254883,
      "activations/layer0_attention_weight_min": -13.918947219848633,
      "activations/layer10_attention_weight_max": 39.62973403930664,
      "activations/layer10_attention_weight_min": -37.46879577636719,
      "activations/layer11_attention_weight_max": 36.75558853149414,
      "activations/layer11_attention_weight_min": -35.54264450073242,
      "activations/layer12_attention_weight_max": 20.500173568725586,
      "activations/layer12_attention_weight_min": -26.137510299682617,
      "activations/layer13_attention_weight_max": 51.798683166503906,
      "activations/layer13_attention_weight_min": -30.0127010345459,
      "activations/layer14_attention_weight_max": 46.95917892456055,
      "activations/layer14_attention_weight_min": -31.69047737121582,
      "activations/layer15_attention_weight_max": 40.97995376586914,
      "activations/layer15_attention_weight_min": -32.90143585205078,
      "activations/layer16_attention_weight_max": 38.152469635009766,
      "activations/layer16_attention_weight_min": -35.22236251831055,
      "activations/layer17_attention_weight_max": 53.82094192504883,
      "activations/layer17_attention_weight_min": -49.76165008544922,
      "activations/layer18_attention_weight_max": 54.9776611328125,
      "activations/layer18_attention_weight_min": -47.96515655517578,
      "activations/layer19_attention_weight_max": 28.60812759399414,
      "activations/layer19_attention_weight_min": -25.121145248413086,
      "activations/layer1_attention_weight_max": 16.225284576416016,
      "activations/layer1_attention_weight_min": -14.539170265197754,
      "activations/layer20_attention_weight_max": 26.74443817138672,
      "activations/layer20_attention_weight_min": -21.554597854614258,
      "activations/layer21_attention_weight_max": 45.7777214050293,
      "activations/layer21_attention_weight_min": -24.491819381713867,
      "activations/layer22_attention_weight_max": 30.68144989013672,
      "activations/layer22_attention_weight_min": -26.68921661376953,
      "activations/layer23_attention_weight_max": 44.39097595214844,
      "activations/layer23_attention_weight_min": -25.166107177734375,
      "activations/layer2_attention_weight_max": 30.285945892333984,
      "activations/layer2_attention_weight_min": -29.429218292236328,
      "activations/layer3_attention_weight_max": 86.04940795898438,
      "activations/layer3_attention_weight_min": -86.24757385253906,
      "activations/layer4_attention_weight_max": 81.73277282714844,
      "activations/layer4_attention_weight_min": -80.65081787109375,
      "activations/layer5_attention_weight_max": 62.594024658203125,
      "activations/layer5_attention_weight_min": -68.72502899169922,
      "activations/layer6_attention_weight_max": 48.7359504699707,
      "activations/layer6_attention_weight_min": -51.83039093017578,
      "activations/layer7_attention_weight_max": 67.14563751220703,
      "activations/layer7_attention_weight_min": -64.99746704101562,
      "activations/layer8_attention_weight_max": 47.91779327392578,
      "activations/layer8_attention_weight_min": -50.34316635131836,
      "activations/layer9_attention_weight_max": 50.90060043334961,
      "activations/layer9_attention_weight_min": -45.424720764160156,
      "epoch": 20.06,
      "learning_rate": 2.0824621212121212e-05,
      "loss": 2.7143,
      "step": 345300
    },
    {
      "activations/layer0_attention_weight_max": 16.294801712036133,
      "activations/layer0_attention_weight_min": -13.923311233520508,
      "activations/layer10_attention_weight_max": 32.29861068725586,
      "activations/layer10_attention_weight_min": -33.62017822265625,
      "activations/layer11_attention_weight_max": 29.74073600769043,
      "activations/layer11_attention_weight_min": -33.2113151550293,
      "activations/layer12_attention_weight_max": 19.17619514465332,
      "activations/layer12_attention_weight_min": -23.49018096923828,
      "activations/layer13_attention_weight_max": 39.47981643676758,
      "activations/layer13_attention_weight_min": -25.90593719482422,
      "activations/layer14_attention_weight_max": 35.100975036621094,
      "activations/layer14_attention_weight_min": -30.930788040161133,
      "activations/layer15_attention_weight_max": 33.384002685546875,
      "activations/layer15_attention_weight_min": -29.711854934692383,
      "activations/layer16_attention_weight_max": 32.461204528808594,
      "activations/layer16_attention_weight_min": -30.89928436279297,
      "activations/layer17_attention_weight_max": 47.553955078125,
      "activations/layer17_attention_weight_min": -43.12942886352539,
      "activations/layer18_attention_weight_max": 43.707374572753906,
      "activations/layer18_attention_weight_min": -38.59483337402344,
      "activations/layer19_attention_weight_max": 23.089555740356445,
      "activations/layer19_attention_weight_min": -22.35811424255371,
      "activations/layer1_attention_weight_max": 16.065006256103516,
      "activations/layer1_attention_weight_min": -13.06004810333252,
      "activations/layer20_attention_weight_max": 21.941349029541016,
      "activations/layer20_attention_weight_min": -20.992713928222656,
      "activations/layer21_attention_weight_max": 32.579063415527344,
      "activations/layer21_attention_weight_min": -23.68753433227539,
      "activations/layer22_attention_weight_max": 26.979124069213867,
      "activations/layer22_attention_weight_min": -23.743928909301758,
      "activations/layer23_attention_weight_max": 35.886749267578125,
      "activations/layer23_attention_weight_min": -24.87769889831543,
      "activations/layer2_attention_weight_max": 30.221303939819336,
      "activations/layer2_attention_weight_min": -28.583587646484375,
      "activations/layer3_attention_weight_max": 78.79066467285156,
      "activations/layer3_attention_weight_min": -83.7132797241211,
      "activations/layer4_attention_weight_max": 77.67218017578125,
      "activations/layer4_attention_weight_min": -77.7887954711914,
      "activations/layer5_attention_weight_max": 58.171722412109375,
      "activations/layer5_attention_weight_min": -73.29248809814453,
      "activations/layer6_attention_weight_max": 45.02357482910156,
      "activations/layer6_attention_weight_min": -45.11225509643555,
      "activations/layer7_attention_weight_max": 64.99323272705078,
      "activations/layer7_attention_weight_min": -60.95245361328125,
      "activations/layer8_attention_weight_max": 43.541709899902344,
      "activations/layer8_attention_weight_min": -45.98123550415039,
      "activations/layer9_attention_weight_max": 40.55887985229492,
      "activations/layer9_attention_weight_min": -42.102046966552734,
      "epoch": 20.07,
      "learning_rate": 2.0805681818181814e-05,
      "loss": 2.7253,
      "step": 345350
    },
    {
      "activations/layer0_attention_weight_max": 14.812688827514648,
      "activations/layer0_attention_weight_min": -13.59243392944336,
      "activations/layer10_attention_weight_max": 36.017333984375,
      "activations/layer10_attention_weight_min": -36.25042724609375,
      "activations/layer11_attention_weight_max": 34.40702819824219,
      "activations/layer11_attention_weight_min": -34.10453796386719,
      "activations/layer12_attention_weight_max": 17.88311195373535,
      "activations/layer12_attention_weight_min": -32.592166900634766,
      "activations/layer13_attention_weight_max": 29.79193115234375,
      "activations/layer13_attention_weight_min": -26.26060676574707,
      "activations/layer14_attention_weight_max": 31.94805145263672,
      "activations/layer14_attention_weight_min": -29.205852508544922,
      "activations/layer15_attention_weight_max": 30.31248664855957,
      "activations/layer15_attention_weight_min": -29.240184783935547,
      "activations/layer16_attention_weight_max": 32.34882736206055,
      "activations/layer16_attention_weight_min": -31.432580947875977,
      "activations/layer17_attention_weight_max": 48.268959045410156,
      "activations/layer17_attention_weight_min": -43.670448303222656,
      "activations/layer18_attention_weight_max": 42.333038330078125,
      "activations/layer18_attention_weight_min": -39.430747985839844,
      "activations/layer19_attention_weight_max": 23.55030632019043,
      "activations/layer19_attention_weight_min": -22.488628387451172,
      "activations/layer1_attention_weight_max": 14.70889949798584,
      "activations/layer1_attention_weight_min": -14.261838912963867,
      "activations/layer20_attention_weight_max": 22.10507583618164,
      "activations/layer20_attention_weight_min": -20.80975914001465,
      "activations/layer21_attention_weight_max": 33.46108627319336,
      "activations/layer21_attention_weight_min": -21.782106399536133,
      "activations/layer22_attention_weight_max": 28.30647087097168,
      "activations/layer22_attention_weight_min": -25.81470489501953,
      "activations/layer23_attention_weight_max": 32.94480514526367,
      "activations/layer23_attention_weight_min": -23.607328414916992,
      "activations/layer2_attention_weight_max": 31.629737854003906,
      "activations/layer2_attention_weight_min": -29.922468185424805,
      "activations/layer3_attention_weight_max": 88.09591674804688,
      "activations/layer3_attention_weight_min": -86.41878509521484,
      "activations/layer4_attention_weight_max": 87.2972640991211,
      "activations/layer4_attention_weight_min": -80.56820678710938,
      "activations/layer5_attention_weight_max": 60.4151725769043,
      "activations/layer5_attention_weight_min": -72.38838195800781,
      "activations/layer6_attention_weight_max": 47.03117752075195,
      "activations/layer6_attention_weight_min": -48.014442443847656,
      "activations/layer7_attention_weight_max": 64.9398193359375,
      "activations/layer7_attention_weight_min": -63.228694915771484,
      "activations/layer8_attention_weight_max": 46.3191032409668,
      "activations/layer8_attention_weight_min": -48.881046295166016,
      "activations/layer9_attention_weight_max": 47.568782806396484,
      "activations/layer9_attention_weight_min": -46.284393310546875,
      "epoch": 20.07,
      "learning_rate": 2.0786742424242422e-05,
      "loss": 2.7148,
      "step": 345400
    },
    {
      "activations/layer0_attention_weight_max": 15.694767951965332,
      "activations/layer0_attention_weight_min": -13.858281135559082,
      "activations/layer10_attention_weight_max": 34.022308349609375,
      "activations/layer10_attention_weight_min": -34.97063064575195,
      "activations/layer11_attention_weight_max": 33.40837860107422,
      "activations/layer11_attention_weight_min": -35.85411834716797,
      "activations/layer12_attention_weight_max": 20.633054733276367,
      "activations/layer12_attention_weight_min": -23.193376541137695,
      "activations/layer13_attention_weight_max": 32.14756774902344,
      "activations/layer13_attention_weight_min": -26.692707061767578,
      "activations/layer14_attention_weight_max": 37.225093841552734,
      "activations/layer14_attention_weight_min": -32.24064636230469,
      "activations/layer15_attention_weight_max": 34.32792282104492,
      "activations/layer15_attention_weight_min": -30.764484405517578,
      "activations/layer16_attention_weight_max": 37.71155548095703,
      "activations/layer16_attention_weight_min": -34.63236999511719,
      "activations/layer17_attention_weight_max": 55.348609924316406,
      "activations/layer17_attention_weight_min": -45.750091552734375,
      "activations/layer18_attention_weight_max": 52.2075080871582,
      "activations/layer18_attention_weight_min": -43.21796798706055,
      "activations/layer19_attention_weight_max": 24.91755485534668,
      "activations/layer19_attention_weight_min": -26.295841217041016,
      "activations/layer1_attention_weight_max": 15.886453628540039,
      "activations/layer1_attention_weight_min": -14.00014877319336,
      "activations/layer20_attention_weight_max": 22.705385208129883,
      "activations/layer20_attention_weight_min": -22.449726104736328,
      "activations/layer21_attention_weight_max": 33.07735061645508,
      "activations/layer21_attention_weight_min": -21.643484115600586,
      "activations/layer22_attention_weight_max": 29.35673713684082,
      "activations/layer22_attention_weight_min": -26.03377342224121,
      "activations/layer23_attention_weight_max": 37.44739532470703,
      "activations/layer23_attention_weight_min": -26.196468353271484,
      "activations/layer2_attention_weight_max": 31.646528244018555,
      "activations/layer2_attention_weight_min": -29.288047790527344,
      "activations/layer3_attention_weight_max": 81.53435516357422,
      "activations/layer3_attention_weight_min": -84.42080688476562,
      "activations/layer4_attention_weight_max": 78.85738372802734,
      "activations/layer4_attention_weight_min": -82.05172729492188,
      "activations/layer5_attention_weight_max": 58.90313720703125,
      "activations/layer5_attention_weight_min": -69.59896850585938,
      "activations/layer6_attention_weight_max": 47.35771560668945,
      "activations/layer6_attention_weight_min": -48.9313850402832,
      "activations/layer7_attention_weight_max": 64.6192626953125,
      "activations/layer7_attention_weight_min": -64.61066436767578,
      "activations/layer8_attention_weight_max": 43.63430404663086,
      "activations/layer8_attention_weight_min": -53.13360595703125,
      "activations/layer9_attention_weight_max": 42.29722213745117,
      "activations/layer9_attention_weight_min": -44.75516891479492,
      "epoch": 20.07,
      "learning_rate": 2.076780303030303e-05,
      "loss": 2.7191,
      "step": 345450
    },
    {
      "activations/layer0_attention_weight_max": 15.517122268676758,
      "activations/layer0_attention_weight_min": -13.915452003479004,
      "activations/layer10_attention_weight_max": 35.53257751464844,
      "activations/layer10_attention_weight_min": -36.33347702026367,
      "activations/layer11_attention_weight_max": 33.756431579589844,
      "activations/layer11_attention_weight_min": -34.76591491699219,
      "activations/layer12_attention_weight_max": 18.57319450378418,
      "activations/layer12_attention_weight_min": -23.64647674560547,
      "activations/layer13_attention_weight_max": 31.446863174438477,
      "activations/layer13_attention_weight_min": -25.710779190063477,
      "activations/layer14_attention_weight_max": 36.62968444824219,
      "activations/layer14_attention_weight_min": -30.53216552734375,
      "activations/layer15_attention_weight_max": 31.540000915527344,
      "activations/layer15_attention_weight_min": -31.175928115844727,
      "activations/layer16_attention_weight_max": 32.854915618896484,
      "activations/layer16_attention_weight_min": -31.380578994750977,
      "activations/layer17_attention_weight_max": 48.755374908447266,
      "activations/layer17_attention_weight_min": -44.599205017089844,
      "activations/layer18_attention_weight_max": 47.24177169799805,
      "activations/layer18_attention_weight_min": -39.5120964050293,
      "activations/layer19_attention_weight_max": 25.241838455200195,
      "activations/layer19_attention_weight_min": -24.59136390686035,
      "activations/layer1_attention_weight_max": 15.281848907470703,
      "activations/layer1_attention_weight_min": -15.03182601928711,
      "activations/layer20_attention_weight_max": 21.708417892456055,
      "activations/layer20_attention_weight_min": -22.157114028930664,
      "activations/layer21_attention_weight_max": 34.57142639160156,
      "activations/layer21_attention_weight_min": -20.810802459716797,
      "activations/layer22_attention_weight_max": 27.7718505859375,
      "activations/layer22_attention_weight_min": -25.943450927734375,
      "activations/layer23_attention_weight_max": 37.322139739990234,
      "activations/layer23_attention_weight_min": -24.499588012695312,
      "activations/layer2_attention_weight_max": 31.254165649414062,
      "activations/layer2_attention_weight_min": -30.12135887145996,
      "activations/layer3_attention_weight_max": 83.90380096435547,
      "activations/layer3_attention_weight_min": -83.9595718383789,
      "activations/layer4_attention_weight_max": 79.51488494873047,
      "activations/layer4_attention_weight_min": -83.35087585449219,
      "activations/layer5_attention_weight_max": 59.93209457397461,
      "activations/layer5_attention_weight_min": -65.43667602539062,
      "activations/layer6_attention_weight_max": 47.07236862182617,
      "activations/layer6_attention_weight_min": -49.49374008178711,
      "activations/layer7_attention_weight_max": 65.39459228515625,
      "activations/layer7_attention_weight_min": -64.46383666992188,
      "activations/layer8_attention_weight_max": 46.68218994140625,
      "activations/layer8_attention_weight_min": -49.91029357910156,
      "activations/layer9_attention_weight_max": 41.491764068603516,
      "activations/layer9_attention_weight_min": -44.61682891845703,
      "epoch": 20.08,
      "learning_rate": 2.0748863636363635e-05,
      "loss": 2.7188,
      "step": 345500
    },
    {
      "activations/layer0_attention_weight_max": 16.2125244140625,
      "activations/layer0_attention_weight_min": -14.052456855773926,
      "activations/layer10_attention_weight_max": 33.813514709472656,
      "activations/layer10_attention_weight_min": -36.527435302734375,
      "activations/layer11_attention_weight_max": 34.421024322509766,
      "activations/layer11_attention_weight_min": -34.242897033691406,
      "activations/layer12_attention_weight_max": 20.357316970825195,
      "activations/layer12_attention_weight_min": -25.13239097595215,
      "activations/layer13_attention_weight_max": 33.82308578491211,
      "activations/layer13_attention_weight_min": -27.392240524291992,
      "activations/layer14_attention_weight_max": 33.13691711425781,
      "activations/layer14_attention_weight_min": -30.493059158325195,
      "activations/layer15_attention_weight_max": 32.05287551879883,
      "activations/layer15_attention_weight_min": -31.55963897705078,
      "activations/layer16_attention_weight_max": 36.398109436035156,
      "activations/layer16_attention_weight_min": -32.41213607788086,
      "activations/layer17_attention_weight_max": 53.12605667114258,
      "activations/layer17_attention_weight_min": -46.30525588989258,
      "activations/layer18_attention_weight_max": 47.90615463256836,
      "activations/layer18_attention_weight_min": -43.29146957397461,
      "activations/layer19_attention_weight_max": 28.243627548217773,
      "activations/layer19_attention_weight_min": -24.92184066772461,
      "activations/layer1_attention_weight_max": 15.541425704956055,
      "activations/layer1_attention_weight_min": -13.986992835998535,
      "activations/layer20_attention_weight_max": 23.775043487548828,
      "activations/layer20_attention_weight_min": -25.339754104614258,
      "activations/layer21_attention_weight_max": 36.51432418823242,
      "activations/layer21_attention_weight_min": -23.899444580078125,
      "activations/layer22_attention_weight_max": 28.733091354370117,
      "activations/layer22_attention_weight_min": -25.61997413635254,
      "activations/layer23_attention_weight_max": 39.431724548339844,
      "activations/layer23_attention_weight_min": -26.798274993896484,
      "activations/layer2_attention_weight_max": 30.787412643432617,
      "activations/layer2_attention_weight_min": -28.853763580322266,
      "activations/layer3_attention_weight_max": 82.10406494140625,
      "activations/layer3_attention_weight_min": -81.10685729980469,
      "activations/layer4_attention_weight_max": 78.46187591552734,
      "activations/layer4_attention_weight_min": -76.94062042236328,
      "activations/layer5_attention_weight_max": 57.8256950378418,
      "activations/layer5_attention_weight_min": -67.68244171142578,
      "activations/layer6_attention_weight_max": 45.869361877441406,
      "activations/layer6_attention_weight_min": -48.557125091552734,
      "activations/layer7_attention_weight_max": 63.09721374511719,
      "activations/layer7_attention_weight_min": -64.17162322998047,
      "activations/layer8_attention_weight_max": 43.760284423828125,
      "activations/layer8_attention_weight_min": -48.44123840332031,
      "activations/layer9_attention_weight_max": 42.127017974853516,
      "activations/layer9_attention_weight_min": -44.286983489990234,
      "epoch": 20.08,
      "learning_rate": 2.0729924242424243e-05,
      "loss": 2.7163,
      "step": 345550
    },
    {
      "activations/layer0_attention_weight_max": 16.071657180786133,
      "activations/layer0_attention_weight_min": -13.93514633178711,
      "activations/layer10_attention_weight_max": 32.94245147705078,
      "activations/layer10_attention_weight_min": -33.74165725708008,
      "activations/layer11_attention_weight_max": 31.24407958984375,
      "activations/layer11_attention_weight_min": -32.27423858642578,
      "activations/layer12_attention_weight_max": 19.447803497314453,
      "activations/layer12_attention_weight_min": -27.107820510864258,
      "activations/layer13_attention_weight_max": 35.752593994140625,
      "activations/layer13_attention_weight_min": -25.92856788635254,
      "activations/layer14_attention_weight_max": 35.20428466796875,
      "activations/layer14_attention_weight_min": -29.383089065551758,
      "activations/layer15_attention_weight_max": 31.853839874267578,
      "activations/layer15_attention_weight_min": -31.071697235107422,
      "activations/layer16_attention_weight_max": 32.06370162963867,
      "activations/layer16_attention_weight_min": -30.06534767150879,
      "activations/layer17_attention_weight_max": 48.68941116333008,
      "activations/layer17_attention_weight_min": -42.80476760864258,
      "activations/layer18_attention_weight_max": 42.748783111572266,
      "activations/layer18_attention_weight_min": -38.12068176269531,
      "activations/layer19_attention_weight_max": 23.402984619140625,
      "activations/layer19_attention_weight_min": -21.81733512878418,
      "activations/layer1_attention_weight_max": 16.73160743713379,
      "activations/layer1_attention_weight_min": -14.535233497619629,
      "activations/layer20_attention_weight_max": 20.927799224853516,
      "activations/layer20_attention_weight_min": -23.460603713989258,
      "activations/layer21_attention_weight_max": 31.85451889038086,
      "activations/layer21_attention_weight_min": -20.766170501708984,
      "activations/layer22_attention_weight_max": 29.97901153564453,
      "activations/layer22_attention_weight_min": -26.48185920715332,
      "activations/layer23_attention_weight_max": 34.51158142089844,
      "activations/layer23_attention_weight_min": -25.37513542175293,
      "activations/layer2_attention_weight_max": 31.937885284423828,
      "activations/layer2_attention_weight_min": -29.574987411499023,
      "activations/layer3_attention_weight_max": 84.89366912841797,
      "activations/layer3_attention_weight_min": -85.33837890625,
      "activations/layer4_attention_weight_max": 83.24559020996094,
      "activations/layer4_attention_weight_min": -81.94258880615234,
      "activations/layer5_attention_weight_max": 62.574378967285156,
      "activations/layer5_attention_weight_min": -68.51348114013672,
      "activations/layer6_attention_weight_max": 45.57370376586914,
      "activations/layer6_attention_weight_min": -45.8925895690918,
      "activations/layer7_attention_weight_max": 61.143516540527344,
      "activations/layer7_attention_weight_min": -57.297279357910156,
      "activations/layer8_attention_weight_max": 42.939456939697266,
      "activations/layer8_attention_weight_min": -49.3320198059082,
      "activations/layer9_attention_weight_max": 41.95115661621094,
      "activations/layer9_attention_weight_min": -42.77655029296875,
      "epoch": 20.08,
      "learning_rate": 2.0710984848484845e-05,
      "loss": 2.7199,
      "step": 345600
    },
    {
      "activations/layer0_attention_weight_max": 15.680678367614746,
      "activations/layer0_attention_weight_min": -14.03066349029541,
      "activations/layer10_attention_weight_max": 36.15970230102539,
      "activations/layer10_attention_weight_min": -37.50141143798828,
      "activations/layer11_attention_weight_max": 33.9479866027832,
      "activations/layer11_attention_weight_min": -35.46745300292969,
      "activations/layer12_attention_weight_max": 18.43288803100586,
      "activations/layer12_attention_weight_min": -25.986988067626953,
      "activations/layer13_attention_weight_max": 30.9490966796875,
      "activations/layer13_attention_weight_min": -27.389867782592773,
      "activations/layer14_attention_weight_max": 34.25047302246094,
      "activations/layer14_attention_weight_min": -30.418521881103516,
      "activations/layer15_attention_weight_max": 30.67283058166504,
      "activations/layer15_attention_weight_min": -30.643247604370117,
      "activations/layer16_attention_weight_max": 33.45043182373047,
      "activations/layer16_attention_weight_min": -31.624755859375,
      "activations/layer17_attention_weight_max": 52.79435348510742,
      "activations/layer17_attention_weight_min": -48.331825256347656,
      "activations/layer18_attention_weight_max": 50.40232467651367,
      "activations/layer18_attention_weight_min": -40.89773941040039,
      "activations/layer19_attention_weight_max": 24.394800186157227,
      "activations/layer19_attention_weight_min": -23.137466430664062,
      "activations/layer1_attention_weight_max": 16.350008010864258,
      "activations/layer1_attention_weight_min": -12.579107284545898,
      "activations/layer20_attention_weight_max": 21.304668426513672,
      "activations/layer20_attention_weight_min": -21.24948501586914,
      "activations/layer21_attention_weight_max": 33.2889404296875,
      "activations/layer21_attention_weight_min": -23.813316345214844,
      "activations/layer22_attention_weight_max": 29.92929458618164,
      "activations/layer22_attention_weight_min": -24.31295394897461,
      "activations/layer23_attention_weight_max": 37.597076416015625,
      "activations/layer23_attention_weight_min": -24.21765899658203,
      "activations/layer2_attention_weight_max": 29.949596405029297,
      "activations/layer2_attention_weight_min": -28.991954803466797,
      "activations/layer3_attention_weight_max": 85.24665832519531,
      "activations/layer3_attention_weight_min": -86.24940490722656,
      "activations/layer4_attention_weight_max": 82.9320297241211,
      "activations/layer4_attention_weight_min": -81.49895477294922,
      "activations/layer5_attention_weight_max": 61.625972747802734,
      "activations/layer5_attention_weight_min": -72.1617660522461,
      "activations/layer6_attention_weight_max": 50.642799377441406,
      "activations/layer6_attention_weight_min": -52.608192443847656,
      "activations/layer7_attention_weight_max": 69.48811340332031,
      "activations/layer7_attention_weight_min": -66.93382263183594,
      "activations/layer8_attention_weight_max": 46.385807037353516,
      "activations/layer8_attention_weight_min": -53.149147033691406,
      "activations/layer9_attention_weight_max": 45.775508880615234,
      "activations/layer9_attention_weight_min": -47.83620834350586,
      "epoch": 20.08,
      "learning_rate": 2.0692045454545453e-05,
      "loss": 2.7313,
      "step": 345650
    },
    {
      "activations/layer0_attention_weight_max": 15.899014472961426,
      "activations/layer0_attention_weight_min": -13.740859985351562,
      "activations/layer10_attention_weight_max": 34.85883331298828,
      "activations/layer10_attention_weight_min": -35.089576721191406,
      "activations/layer11_attention_weight_max": 33.039459228515625,
      "activations/layer11_attention_weight_min": -34.692718505859375,
      "activations/layer12_attention_weight_max": 21.431228637695312,
      "activations/layer12_attention_weight_min": -24.468612670898438,
      "activations/layer13_attention_weight_max": 50.99129104614258,
      "activations/layer13_attention_weight_min": -28.885698318481445,
      "activations/layer14_attention_weight_max": 35.67099380493164,
      "activations/layer14_attention_weight_min": -30.50404167175293,
      "activations/layer15_attention_weight_max": 32.201656341552734,
      "activations/layer15_attention_weight_min": -31.111064910888672,
      "activations/layer16_attention_weight_max": 35.16667938232422,
      "activations/layer16_attention_weight_min": -33.63443374633789,
      "activations/layer17_attention_weight_max": 51.4797248840332,
      "activations/layer17_attention_weight_min": -47.53458023071289,
      "activations/layer18_attention_weight_max": 48.097652435302734,
      "activations/layer18_attention_weight_min": -45.48948669433594,
      "activations/layer19_attention_weight_max": 25.8272705078125,
      "activations/layer19_attention_weight_min": -26.169570922851562,
      "activations/layer1_attention_weight_max": 15.646016120910645,
      "activations/layer1_attention_weight_min": -13.590861320495605,
      "activations/layer20_attention_weight_max": 23.735692977905273,
      "activations/layer20_attention_weight_min": -24.713239669799805,
      "activations/layer21_attention_weight_max": 34.46236038208008,
      "activations/layer21_attention_weight_min": -23.953760147094727,
      "activations/layer22_attention_weight_max": 29.59998893737793,
      "activations/layer22_attention_weight_min": -28.66425895690918,
      "activations/layer23_attention_weight_max": 33.637149810791016,
      "activations/layer23_attention_weight_min": -28.255321502685547,
      "activations/layer2_attention_weight_max": 32.99176025390625,
      "activations/layer2_attention_weight_min": -29.75372886657715,
      "activations/layer3_attention_weight_max": 81.34148406982422,
      "activations/layer3_attention_weight_min": -82.4972915649414,
      "activations/layer4_attention_weight_max": 79.99674224853516,
      "activations/layer4_attention_weight_min": -81.3980712890625,
      "activations/layer5_attention_weight_max": 60.570030212402344,
      "activations/layer5_attention_weight_min": -67.87754821777344,
      "activations/layer6_attention_weight_max": 46.69905471801758,
      "activations/layer6_attention_weight_min": -46.835044860839844,
      "activations/layer7_attention_weight_max": 66.2491226196289,
      "activations/layer7_attention_weight_min": -64.83076477050781,
      "activations/layer8_attention_weight_max": 48.3159294128418,
      "activations/layer8_attention_weight_min": -48.303836822509766,
      "activations/layer9_attention_weight_max": 48.342323303222656,
      "activations/layer9_attention_weight_min": -45.23489761352539,
      "epoch": 20.09,
      "learning_rate": 2.0673106060606058e-05,
      "loss": 2.7302,
      "step": 345700
    },
    {
      "activations/layer0_attention_weight_max": 16.23796272277832,
      "activations/layer0_attention_weight_min": -13.815975189208984,
      "activations/layer10_attention_weight_max": 33.704139709472656,
      "activations/layer10_attention_weight_min": -35.163028717041016,
      "activations/layer11_attention_weight_max": 33.39287567138672,
      "activations/layer11_attention_weight_min": -33.72414779663086,
      "activations/layer12_attention_weight_max": 20.594274520874023,
      "activations/layer12_attention_weight_min": -24.208398818969727,
      "activations/layer13_attention_weight_max": 32.4132194519043,
      "activations/layer13_attention_weight_min": -30.05852699279785,
      "activations/layer14_attention_weight_max": 33.77790832519531,
      "activations/layer14_attention_weight_min": -33.11979293823242,
      "activations/layer15_attention_weight_max": 32.637088775634766,
      "activations/layer15_attention_weight_min": -32.05234146118164,
      "activations/layer16_attention_weight_max": 33.94914245605469,
      "activations/layer16_attention_weight_min": -33.74961853027344,
      "activations/layer17_attention_weight_max": 53.06533432006836,
      "activations/layer17_attention_weight_min": -49.245513916015625,
      "activations/layer18_attention_weight_max": 45.34652328491211,
      "activations/layer18_attention_weight_min": -42.096134185791016,
      "activations/layer19_attention_weight_max": 23.496826171875,
      "activations/layer19_attention_weight_min": -24.283700942993164,
      "activations/layer1_attention_weight_max": 15.32990550994873,
      "activations/layer1_attention_weight_min": -14.013299942016602,
      "activations/layer20_attention_weight_max": 23.6373233795166,
      "activations/layer20_attention_weight_min": -24.770679473876953,
      "activations/layer21_attention_weight_max": 39.62007141113281,
      "activations/layer21_attention_weight_min": -25.641233444213867,
      "activations/layer22_attention_weight_max": 27.371601104736328,
      "activations/layer22_attention_weight_min": -25.190950393676758,
      "activations/layer23_attention_weight_max": 35.0383186340332,
      "activations/layer23_attention_weight_min": -22.685272216796875,
      "activations/layer2_attention_weight_max": 29.8515625,
      "activations/layer2_attention_weight_min": -28.457019805908203,
      "activations/layer3_attention_weight_max": 79.1955795288086,
      "activations/layer3_attention_weight_min": -81.86530303955078,
      "activations/layer4_attention_weight_max": 76.38932800292969,
      "activations/layer4_attention_weight_min": -79.96178436279297,
      "activations/layer5_attention_weight_max": 59.81224822998047,
      "activations/layer5_attention_weight_min": -69.76896667480469,
      "activations/layer6_attention_weight_max": 48.62577438354492,
      "activations/layer6_attention_weight_min": -50.23393249511719,
      "activations/layer7_attention_weight_max": 65.17308807373047,
      "activations/layer7_attention_weight_min": -62.79338455200195,
      "activations/layer8_attention_weight_max": 45.55411148071289,
      "activations/layer8_attention_weight_min": -47.9100456237793,
      "activations/layer9_attention_weight_max": 45.0184211730957,
      "activations/layer9_attention_weight_min": -44.77528762817383,
      "epoch": 20.09,
      "learning_rate": 2.0654166666666666e-05,
      "loss": 2.7213,
      "step": 345750
    },
    {
      "activations/layer0_attention_weight_max": 16.56053924560547,
      "activations/layer0_attention_weight_min": -13.84790325164795,
      "activations/layer10_attention_weight_max": 42.911582946777344,
      "activations/layer10_attention_weight_min": -39.64478302001953,
      "activations/layer11_attention_weight_max": 37.95651626586914,
      "activations/layer11_attention_weight_min": -37.31434631347656,
      "activations/layer12_attention_weight_max": 20.265501022338867,
      "activations/layer12_attention_weight_min": -29.046871185302734,
      "activations/layer13_attention_weight_max": 39.107391357421875,
      "activations/layer13_attention_weight_min": -27.536293029785156,
      "activations/layer14_attention_weight_max": 36.42766189575195,
      "activations/layer14_attention_weight_min": -34.623897552490234,
      "activations/layer15_attention_weight_max": 33.789268493652344,
      "activations/layer15_attention_weight_min": -32.661075592041016,
      "activations/layer16_attention_weight_max": 34.435394287109375,
      "activations/layer16_attention_weight_min": -35.12898254394531,
      "activations/layer17_attention_weight_max": 53.23039627075195,
      "activations/layer17_attention_weight_min": -47.29679489135742,
      "activations/layer18_attention_weight_max": 48.21060562133789,
      "activations/layer18_attention_weight_min": -42.63337707519531,
      "activations/layer19_attention_weight_max": 23.489591598510742,
      "activations/layer19_attention_weight_min": -23.654190063476562,
      "activations/layer1_attention_weight_max": 14.654449462890625,
      "activations/layer1_attention_weight_min": -14.015161514282227,
      "activations/layer20_attention_weight_max": 23.355621337890625,
      "activations/layer20_attention_weight_min": -22.2880916595459,
      "activations/layer21_attention_weight_max": 37.01317596435547,
      "activations/layer21_attention_weight_min": -23.23801612854004,
      "activations/layer22_attention_weight_max": 32.351924896240234,
      "activations/layer22_attention_weight_min": -27.208955764770508,
      "activations/layer23_attention_weight_max": 40.69451904296875,
      "activations/layer23_attention_weight_min": -24.762653350830078,
      "activations/layer2_attention_weight_max": 33.83406066894531,
      "activations/layer2_attention_weight_min": -32.70783233642578,
      "activations/layer3_attention_weight_max": 96.82717895507812,
      "activations/layer3_attention_weight_min": -90.52313995361328,
      "activations/layer4_attention_weight_max": 95.48069763183594,
      "activations/layer4_attention_weight_min": -93.20246124267578,
      "activations/layer5_attention_weight_max": 75.08199310302734,
      "activations/layer5_attention_weight_min": -78.33863830566406,
      "activations/layer6_attention_weight_max": 55.855506896972656,
      "activations/layer6_attention_weight_min": -57.12977600097656,
      "activations/layer7_attention_weight_max": 75.71613311767578,
      "activations/layer7_attention_weight_min": -70.32781982421875,
      "activations/layer8_attention_weight_max": 53.38222122192383,
      "activations/layer8_attention_weight_min": -56.1326789855957,
      "activations/layer9_attention_weight_max": 49.86033248901367,
      "activations/layer9_attention_weight_min": -50.33158493041992,
      "epoch": 20.09,
      "learning_rate": 2.063522727272727e-05,
      "loss": 2.7151,
      "step": 345800
    },
    {
      "activations/layer0_attention_weight_max": 15.286169052124023,
      "activations/layer0_attention_weight_min": -13.458008766174316,
      "activations/layer10_attention_weight_max": 38.00452423095703,
      "activations/layer10_attention_weight_min": -35.34196472167969,
      "activations/layer11_attention_weight_max": 35.11886215209961,
      "activations/layer11_attention_weight_min": -33.492576599121094,
      "activations/layer12_attention_weight_max": 18.538471221923828,
      "activations/layer12_attention_weight_min": -25.792137145996094,
      "activations/layer13_attention_weight_max": 36.23526382446289,
      "activations/layer13_attention_weight_min": -28.61394691467285,
      "activations/layer14_attention_weight_max": 36.38624954223633,
      "activations/layer14_attention_weight_min": -30.111215591430664,
      "activations/layer15_attention_weight_max": 36.6907958984375,
      "activations/layer15_attention_weight_min": -29.08294677734375,
      "activations/layer16_attention_weight_max": 33.167236328125,
      "activations/layer16_attention_weight_min": -31.68968391418457,
      "activations/layer17_attention_weight_max": 53.07704544067383,
      "activations/layer17_attention_weight_min": -43.800357818603516,
      "activations/layer18_attention_weight_max": 47.59840774536133,
      "activations/layer18_attention_weight_min": -37.79624557495117,
      "activations/layer19_attention_weight_max": 22.212265014648438,
      "activations/layer19_attention_weight_min": -21.28594207763672,
      "activations/layer1_attention_weight_max": 14.842387199401855,
      "activations/layer1_attention_weight_min": -12.925688743591309,
      "activations/layer20_attention_weight_max": 20.611528396606445,
      "activations/layer20_attention_weight_min": -23.358137130737305,
      "activations/layer21_attention_weight_max": 31.611448287963867,
      "activations/layer21_attention_weight_min": -19.273311614990234,
      "activations/layer22_attention_weight_max": 28.469135284423828,
      "activations/layer22_attention_weight_min": -23.741270065307617,
      "activations/layer23_attention_weight_max": 37.666358947753906,
      "activations/layer23_attention_weight_min": -23.40497398376465,
      "activations/layer2_attention_weight_max": 29.026744842529297,
      "activations/layer2_attention_weight_min": -27.60759735107422,
      "activations/layer3_attention_weight_max": 79.61005401611328,
      "activations/layer3_attention_weight_min": -81.98799896240234,
      "activations/layer4_attention_weight_max": 80.1205062866211,
      "activations/layer4_attention_weight_min": -79.93099975585938,
      "activations/layer5_attention_weight_max": 61.5388298034668,
      "activations/layer5_attention_weight_min": -74.22161865234375,
      "activations/layer6_attention_weight_max": 46.39280700683594,
      "activations/layer6_attention_weight_min": -48.41126251220703,
      "activations/layer7_attention_weight_max": 63.3173713684082,
      "activations/layer7_attention_weight_min": -64.13246154785156,
      "activations/layer8_attention_weight_max": 45.95684814453125,
      "activations/layer8_attention_weight_min": -54.194026947021484,
      "activations/layer9_attention_weight_max": 43.39826583862305,
      "activations/layer9_attention_weight_min": -45.171897888183594,
      "epoch": 20.1,
      "learning_rate": 2.061628787878788e-05,
      "loss": 2.721,
      "step": 345850
    },
    {
      "activations/layer0_attention_weight_max": 15.587559700012207,
      "activations/layer0_attention_weight_min": -13.63395881652832,
      "activations/layer10_attention_weight_max": 34.976802825927734,
      "activations/layer10_attention_weight_min": -35.214717864990234,
      "activations/layer11_attention_weight_max": 32.87963104248047,
      "activations/layer11_attention_weight_min": -32.596248626708984,
      "activations/layer12_attention_weight_max": 20.04271125793457,
      "activations/layer12_attention_weight_min": -25.39988136291504,
      "activations/layer13_attention_weight_max": 30.484336853027344,
      "activations/layer13_attention_weight_min": -27.182575225830078,
      "activations/layer14_attention_weight_max": 37.242305755615234,
      "activations/layer14_attention_weight_min": -30.390947341918945,
      "activations/layer15_attention_weight_max": 33.613407135009766,
      "activations/layer15_attention_weight_min": -30.122289657592773,
      "activations/layer16_attention_weight_max": 38.457603454589844,
      "activations/layer16_attention_weight_min": -35.40303421020508,
      "activations/layer17_attention_weight_max": 58.1395263671875,
      "activations/layer17_attention_weight_min": -49.28819274902344,
      "activations/layer18_attention_weight_max": 52.67497634887695,
      "activations/layer18_attention_weight_min": -43.84709548950195,
      "activations/layer19_attention_weight_max": 25.536544799804688,
      "activations/layer19_attention_weight_min": -23.969900131225586,
      "activations/layer1_attention_weight_max": 16.275915145874023,
      "activations/layer1_attention_weight_min": -15.2307767868042,
      "activations/layer20_attention_weight_max": 24.280166625976562,
      "activations/layer20_attention_weight_min": -22.861574172973633,
      "activations/layer21_attention_weight_max": 40.285491943359375,
      "activations/layer21_attention_weight_min": -23.02048683166504,
      "activations/layer22_attention_weight_max": 32.596290588378906,
      "activations/layer22_attention_weight_min": -28.837278366088867,
      "activations/layer23_attention_weight_max": 37.53965377807617,
      "activations/layer23_attention_weight_min": -25.905927658081055,
      "activations/layer2_attention_weight_max": 33.61924362182617,
      "activations/layer2_attention_weight_min": -31.96759796142578,
      "activations/layer3_attention_weight_max": 88.72479248046875,
      "activations/layer3_attention_weight_min": -86.88951110839844,
      "activations/layer4_attention_weight_max": 81.38814544677734,
      "activations/layer4_attention_weight_min": -75.59467315673828,
      "activations/layer5_attention_weight_max": 63.22944259643555,
      "activations/layer5_attention_weight_min": -71.69683837890625,
      "activations/layer6_attention_weight_max": 51.19831085205078,
      "activations/layer6_attention_weight_min": -48.126338958740234,
      "activations/layer7_attention_weight_max": 73.10183715820312,
      "activations/layer7_attention_weight_min": -62.05665969848633,
      "activations/layer8_attention_weight_max": 44.54305648803711,
      "activations/layer8_attention_weight_min": -49.051239013671875,
      "activations/layer9_attention_weight_max": 42.30402755737305,
      "activations/layer9_attention_weight_min": -45.286556243896484,
      "epoch": 20.1,
      "learning_rate": 2.059772727272727e-05,
      "loss": 2.7334,
      "step": 345900
    },
    {
      "activations/layer0_attention_weight_max": 16.065628051757812,
      "activations/layer0_attention_weight_min": -14.025940895080566,
      "activations/layer10_attention_weight_max": 34.988372802734375,
      "activations/layer10_attention_weight_min": -38.011451721191406,
      "activations/layer11_attention_weight_max": 31.67904281616211,
      "activations/layer11_attention_weight_min": -34.32704544067383,
      "activations/layer12_attention_weight_max": 18.605880737304688,
      "activations/layer12_attention_weight_min": -26.371477127075195,
      "activations/layer13_attention_weight_max": 37.34467697143555,
      "activations/layer13_attention_weight_min": -28.645021438598633,
      "activations/layer14_attention_weight_max": 34.67081069946289,
      "activations/layer14_attention_weight_min": -30.28688621520996,
      "activations/layer15_attention_weight_max": 34.20819091796875,
      "activations/layer15_attention_weight_min": -29.80399513244629,
      "activations/layer16_attention_weight_max": 34.44633102416992,
      "activations/layer16_attention_weight_min": -33.22633361816406,
      "activations/layer17_attention_weight_max": 53.15736389160156,
      "activations/layer17_attention_weight_min": -46.915985107421875,
      "activations/layer18_attention_weight_max": 47.37747573852539,
      "activations/layer18_attention_weight_min": -38.89983367919922,
      "activations/layer19_attention_weight_max": 24.148990631103516,
      "activations/layer19_attention_weight_min": -22.12926483154297,
      "activations/layer1_attention_weight_max": 15.727852821350098,
      "activations/layer1_attention_weight_min": -14.341949462890625,
      "activations/layer20_attention_weight_max": 22.06093406677246,
      "activations/layer20_attention_weight_min": -21.273557662963867,
      "activations/layer21_attention_weight_max": 38.407711029052734,
      "activations/layer21_attention_weight_min": -26.094274520874023,
      "activations/layer22_attention_weight_max": 27.8004207611084,
      "activations/layer22_attention_weight_min": -24.681917190551758,
      "activations/layer23_attention_weight_max": 36.1643180847168,
      "activations/layer23_attention_weight_min": -25.402923583984375,
      "activations/layer2_attention_weight_max": 32.03514099121094,
      "activations/layer2_attention_weight_min": -29.34048080444336,
      "activations/layer3_attention_weight_max": 85.45774841308594,
      "activations/layer3_attention_weight_min": -84.02554321289062,
      "activations/layer4_attention_weight_max": 84.01659393310547,
      "activations/layer4_attention_weight_min": -82.3234634399414,
      "activations/layer5_attention_weight_max": 61.370155334472656,
      "activations/layer5_attention_weight_min": -69.32879638671875,
      "activations/layer6_attention_weight_max": 47.98508071899414,
      "activations/layer6_attention_weight_min": -50.17163848876953,
      "activations/layer7_attention_weight_max": 63.67695617675781,
      "activations/layer7_attention_weight_min": -62.71228790283203,
      "activations/layer8_attention_weight_max": 46.617618560791016,
      "activations/layer8_attention_weight_min": -51.31710433959961,
      "activations/layer9_attention_weight_max": 43.07662582397461,
      "activations/layer9_attention_weight_min": -47.168983459472656,
      "epoch": 20.1,
      "learning_rate": 2.057878787878788e-05,
      "loss": 2.7333,
      "step": 345950
    },
    {
      "activations/layer0_attention_weight_max": 15.397037506103516,
      "activations/layer0_attention_weight_min": -13.89780330657959,
      "activations/layer10_attention_weight_max": 38.09953308105469,
      "activations/layer10_attention_weight_min": -35.72402572631836,
      "activations/layer11_attention_weight_max": 34.485469818115234,
      "activations/layer11_attention_weight_min": -35.08746337890625,
      "activations/layer12_attention_weight_max": 19.864986419677734,
      "activations/layer12_attention_weight_min": -24.656620025634766,
      "activations/layer13_attention_weight_max": 39.28758239746094,
      "activations/layer13_attention_weight_min": -30.877723693847656,
      "activations/layer14_attention_weight_max": 38.27653884887695,
      "activations/layer14_attention_weight_min": -31.372617721557617,
      "activations/layer15_attention_weight_max": 38.89069366455078,
      "activations/layer15_attention_weight_min": -30.954998016357422,
      "activations/layer16_attention_weight_max": 37.64459228515625,
      "activations/layer16_attention_weight_min": -34.02259063720703,
      "activations/layer17_attention_weight_max": 58.10393524169922,
      "activations/layer17_attention_weight_min": -47.759525299072266,
      "activations/layer18_attention_weight_max": 50.31156921386719,
      "activations/layer18_attention_weight_min": -43.7336311340332,
      "activations/layer19_attention_weight_max": 26.093917846679688,
      "activations/layer19_attention_weight_min": -25.107995986938477,
      "activations/layer1_attention_weight_max": 17.495750427246094,
      "activations/layer1_attention_weight_min": -12.738239288330078,
      "activations/layer20_attention_weight_max": 23.367074966430664,
      "activations/layer20_attention_weight_min": -22.926267623901367,
      "activations/layer21_attention_weight_max": 41.23222732543945,
      "activations/layer21_attention_weight_min": -23.930320739746094,
      "activations/layer22_attention_weight_max": 30.37165641784668,
      "activations/layer22_attention_weight_min": -26.36390495300293,
      "activations/layer23_attention_weight_max": 39.50787353515625,
      "activations/layer23_attention_weight_min": -24.823467254638672,
      "activations/layer2_attention_weight_max": 29.738584518432617,
      "activations/layer2_attention_weight_min": -27.874526977539062,
      "activations/layer3_attention_weight_max": 87.05903625488281,
      "activations/layer3_attention_weight_min": -86.15642547607422,
      "activations/layer4_attention_weight_max": 84.84593963623047,
      "activations/layer4_attention_weight_min": -83.15696716308594,
      "activations/layer5_attention_weight_max": 63.801666259765625,
      "activations/layer5_attention_weight_min": -75.29264831542969,
      "activations/layer6_attention_weight_max": 51.35102844238281,
      "activations/layer6_attention_weight_min": -50.24638366699219,
      "activations/layer7_attention_weight_max": 67.57386016845703,
      "activations/layer7_attention_weight_min": -66.46563720703125,
      "activations/layer8_attention_weight_max": 48.3149299621582,
      "activations/layer8_attention_weight_min": -50.50907516479492,
      "activations/layer9_attention_weight_max": 44.52235412597656,
      "activations/layer9_attention_weight_min": -46.538265228271484,
      "epoch": 20.1,
      "learning_rate": 2.0559848484848483e-05,
      "loss": 2.7323,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.5335,
      "eval_samples_per_second": 503.196,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.5335,
      "eval_openwebtext_samples_per_second": 503.196,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_wikitext_loss": 2.916015625,
      "eval_wikitext_ppl": 18.467558995986426,
      "eval_wikitext_runtime": 2.0158,
      "eval_wikitext_samples_per_second": 226.214,
      "step": 346000
    },
    {
      "epoch": 20.1,
      "eval_lambada_loss": 2.369140625,
      "eval_lambada_ppl": 10.688203161633492,
      "eval_lambada_runtime": 9.6361,
      "eval_lambada_samples_per_second": 505.287,
      "step": 346000
    },
    {
      "activations/layer0_attention_weight_max": 16.020654678344727,
      "activations/layer0_attention_weight_min": -13.872476577758789,
      "activations/layer10_attention_weight_max": 36.57828903198242,
      "activations/layer10_attention_weight_min": -36.41200637817383,
      "activations/layer11_attention_weight_max": 32.08198547363281,
      "activations/layer11_attention_weight_min": -34.15204620361328,
      "activations/layer12_attention_weight_max": 20.493383407592773,
      "activations/layer12_attention_weight_min": -23.42466163635254,
      "activations/layer13_attention_weight_max": 35.70530700683594,
      "activations/layer13_attention_weight_min": -27.581130981445312,
      "activations/layer14_attention_weight_max": 36.0836067199707,
      "activations/layer14_attention_weight_min": -32.07822036743164,
      "activations/layer15_attention_weight_max": 35.346004486083984,
      "activations/layer15_attention_weight_min": -32.496238708496094,
      "activations/layer16_attention_weight_max": 39.01996994018555,
      "activations/layer16_attention_weight_min": -33.515750885009766,
      "activations/layer17_attention_weight_max": 54.068138122558594,
      "activations/layer17_attention_weight_min": -47.4892692565918,
      "activations/layer18_attention_weight_max": 47.2783203125,
      "activations/layer18_attention_weight_min": -47.18790054321289,
      "activations/layer19_attention_weight_max": 25.402341842651367,
      "activations/layer19_attention_weight_min": -26.853567123413086,
      "activations/layer1_attention_weight_max": 14.900522232055664,
      "activations/layer1_attention_weight_min": -13.432442665100098,
      "activations/layer20_attention_weight_max": 22.80771827697754,
      "activations/layer20_attention_weight_min": -21.748249053955078,
      "activations/layer21_attention_weight_max": 36.4271125793457,
      "activations/layer21_attention_weight_min": -25.58403778076172,
      "activations/layer22_attention_weight_max": 25.99237632751465,
      "activations/layer22_attention_weight_min": -25.901123046875,
      "activations/layer23_attention_weight_max": 36.48653030395508,
      "activations/layer23_attention_weight_min": -24.341224670410156,
      "activations/layer2_attention_weight_max": 31.41262435913086,
      "activations/layer2_attention_weight_min": -30.441848754882812,
      "activations/layer3_attention_weight_max": 84.3529052734375,
      "activations/layer3_attention_weight_min": -82.51695251464844,
      "activations/layer4_attention_weight_max": 85.2870101928711,
      "activations/layer4_attention_weight_min": -79.954833984375,
      "activations/layer5_attention_weight_max": 63.474205017089844,
      "activations/layer5_attention_weight_min": -71.35572052001953,
      "activations/layer6_attention_weight_max": 47.40066146850586,
      "activations/layer6_attention_weight_min": -49.55314254760742,
      "activations/layer7_attention_weight_max": 64.24726104736328,
      "activations/layer7_attention_weight_min": -60.60271072387695,
      "activations/layer8_attention_weight_max": 49.11701583862305,
      "activations/layer8_attention_weight_min": -49.852901458740234,
      "activations/layer9_attention_weight_max": 45.06538009643555,
      "activations/layer9_attention_weight_min": -44.342918395996094,
      "epoch": 20.11,
      "learning_rate": 2.0540909090909088e-05,
      "loss": 2.7085,
      "step": 346050
    },
    {
      "activations/layer0_attention_weight_max": 16.28839683532715,
      "activations/layer0_attention_weight_min": -13.998363494873047,
      "activations/layer10_attention_weight_max": 34.81658935546875,
      "activations/layer10_attention_weight_min": -35.01808547973633,
      "activations/layer11_attention_weight_max": 33.01934051513672,
      "activations/layer11_attention_weight_min": -36.54157257080078,
      "activations/layer12_attention_weight_max": 18.241596221923828,
      "activations/layer12_attention_weight_min": -25.354516983032227,
      "activations/layer13_attention_weight_max": 33.13909149169922,
      "activations/layer13_attention_weight_min": -26.51705551147461,
      "activations/layer14_attention_weight_max": 33.0091438293457,
      "activations/layer14_attention_weight_min": -31.821619033813477,
      "activations/layer15_attention_weight_max": 31.470752716064453,
      "activations/layer15_attention_weight_min": -30.95861053466797,
      "activations/layer16_attention_weight_max": 33.7259635925293,
      "activations/layer16_attention_weight_min": -31.68260383605957,
      "activations/layer17_attention_weight_max": 47.86960983276367,
      "activations/layer17_attention_weight_min": -45.80881881713867,
      "activations/layer18_attention_weight_max": 44.21530532836914,
      "activations/layer18_attention_weight_min": -39.04308319091797,
      "activations/layer19_attention_weight_max": 26.085935592651367,
      "activations/layer19_attention_weight_min": -22.531030654907227,
      "activations/layer1_attention_weight_max": 15.438288688659668,
      "activations/layer1_attention_weight_min": -13.722511291503906,
      "activations/layer20_attention_weight_max": 24.087772369384766,
      "activations/layer20_attention_weight_min": -21.343429565429688,
      "activations/layer21_attention_weight_max": 36.550559997558594,
      "activations/layer21_attention_weight_min": -20.84740447998047,
      "activations/layer22_attention_weight_max": 28.176193237304688,
      "activations/layer22_attention_weight_min": -26.596357345581055,
      "activations/layer23_attention_weight_max": 34.7779541015625,
      "activations/layer23_attention_weight_min": -23.82848358154297,
      "activations/layer2_attention_weight_max": 29.08406639099121,
      "activations/layer2_attention_weight_min": -26.718351364135742,
      "activations/layer3_attention_weight_max": 79.1668472290039,
      "activations/layer3_attention_weight_min": -79.64301300048828,
      "activations/layer4_attention_weight_max": 78.18281555175781,
      "activations/layer4_attention_weight_min": -74.58906555175781,
      "activations/layer5_attention_weight_max": 62.51832580566406,
      "activations/layer5_attention_weight_min": -71.20987701416016,
      "activations/layer6_attention_weight_max": 46.97462844848633,
      "activations/layer6_attention_weight_min": -47.022987365722656,
      "activations/layer7_attention_weight_max": 65.34488677978516,
      "activations/layer7_attention_weight_min": -64.72710418701172,
      "activations/layer8_attention_weight_max": 45.45992660522461,
      "activations/layer8_attention_weight_min": -47.71088409423828,
      "activations/layer9_attention_weight_max": 41.10338592529297,
      "activations/layer9_attention_weight_min": -46.87584686279297,
      "epoch": 20.11,
      "learning_rate": 2.0521969696969693e-05,
      "loss": 2.7139,
      "step": 346100
    },
    {
      "activations/layer0_attention_weight_max": 15.487071990966797,
      "activations/layer0_attention_weight_min": -13.834946632385254,
      "activations/layer10_attention_weight_max": 33.67884826660156,
      "activations/layer10_attention_weight_min": -35.418006896972656,
      "activations/layer11_attention_weight_max": 30.760330200195312,
      "activations/layer11_attention_weight_min": -33.10261154174805,
      "activations/layer12_attention_weight_max": 18.733688354492188,
      "activations/layer12_attention_weight_min": -27.356576919555664,
      "activations/layer13_attention_weight_max": 34.22054672241211,
      "activations/layer13_attention_weight_min": -29.2225341796875,
      "activations/layer14_attention_weight_max": 33.5647087097168,
      "activations/layer14_attention_weight_min": -29.38574981689453,
      "activations/layer15_attention_weight_max": 30.87897491455078,
      "activations/layer15_attention_weight_min": -29.901464462280273,
      "activations/layer16_attention_weight_max": 31.88036346435547,
      "activations/layer16_attention_weight_min": -31.81877899169922,
      "activations/layer17_attention_weight_max": 45.93095397949219,
      "activations/layer17_attention_weight_min": -42.41957473754883,
      "activations/layer18_attention_weight_max": 42.24924087524414,
      "activations/layer18_attention_weight_min": -38.47806930541992,
      "activations/layer19_attention_weight_max": 24.692584991455078,
      "activations/layer19_attention_weight_min": -23.014768600463867,
      "activations/layer1_attention_weight_max": 14.987509727478027,
      "activations/layer1_attention_weight_min": -13.193967819213867,
      "activations/layer20_attention_weight_max": 24.682836532592773,
      "activations/layer20_attention_weight_min": -23.551986694335938,
      "activations/layer21_attention_weight_max": 33.71276092529297,
      "activations/layer21_attention_weight_min": -28.277610778808594,
      "activations/layer22_attention_weight_max": 27.875627517700195,
      "activations/layer22_attention_weight_min": -25.538068771362305,
      "activations/layer23_attention_weight_max": 33.4832649230957,
      "activations/layer23_attention_weight_min": -24.035045623779297,
      "activations/layer2_attention_weight_max": 31.385608673095703,
      "activations/layer2_attention_weight_min": -29.313962936401367,
      "activations/layer3_attention_weight_max": 83.41238403320312,
      "activations/layer3_attention_weight_min": -90.85820770263672,
      "activations/layer4_attention_weight_max": 86.1001205444336,
      "activations/layer4_attention_weight_min": -79.5848159790039,
      "activations/layer5_attention_weight_max": 62.52130126953125,
      "activations/layer5_attention_weight_min": -66.84367370605469,
      "activations/layer6_attention_weight_max": 50.7238883972168,
      "activations/layer6_attention_weight_min": -47.57773971557617,
      "activations/layer7_attention_weight_max": 61.85382080078125,
      "activations/layer7_attention_weight_min": -62.07528305053711,
      "activations/layer8_attention_weight_max": 47.18836975097656,
      "activations/layer8_attention_weight_min": -50.06413650512695,
      "activations/layer9_attention_weight_max": 42.28667068481445,
      "activations/layer9_attention_weight_min": -44.8071174621582,
      "epoch": 20.11,
      "learning_rate": 2.05030303030303e-05,
      "loss": 2.7166,
      "step": 346150
    },
    {
      "activations/layer0_attention_weight_max": 15.657217979431152,
      "activations/layer0_attention_weight_min": -14.358061790466309,
      "activations/layer10_attention_weight_max": 36.28473663330078,
      "activations/layer10_attention_weight_min": -33.870365142822266,
      "activations/layer11_attention_weight_max": 32.81792068481445,
      "activations/layer11_attention_weight_min": -34.413856506347656,
      "activations/layer12_attention_weight_max": 19.11324119567871,
      "activations/layer12_attention_weight_min": -27.838407516479492,
      "activations/layer13_attention_weight_max": 37.84681701660156,
      "activations/layer13_attention_weight_min": -27.48531723022461,
      "activations/layer14_attention_weight_max": 36.859500885009766,
      "activations/layer14_attention_weight_min": -31.42664337158203,
      "activations/layer15_attention_weight_max": 35.459835052490234,
      "activations/layer15_attention_weight_min": -31.668663024902344,
      "activations/layer16_attention_weight_max": 36.24752426147461,
      "activations/layer16_attention_weight_min": -33.759132385253906,
      "activations/layer17_attention_weight_max": 55.21652603149414,
      "activations/layer17_attention_weight_min": -49.65590286254883,
      "activations/layer18_attention_weight_max": 49.84928894042969,
      "activations/layer18_attention_weight_min": -41.47954559326172,
      "activations/layer19_attention_weight_max": 24.613500595092773,
      "activations/layer19_attention_weight_min": -24.0213565826416,
      "activations/layer1_attention_weight_max": 15.751091003417969,
      "activations/layer1_attention_weight_min": -13.748848915100098,
      "activations/layer20_attention_weight_max": 23.239219665527344,
      "activations/layer20_attention_weight_min": -22.860002517700195,
      "activations/layer21_attention_weight_max": 38.51813507080078,
      "activations/layer21_attention_weight_min": -25.381956100463867,
      "activations/layer22_attention_weight_max": 30.365970611572266,
      "activations/layer22_attention_weight_min": -28.528318405151367,
      "activations/layer23_attention_weight_max": 34.64920425415039,
      "activations/layer23_attention_weight_min": -26.806381225585938,
      "activations/layer2_attention_weight_max": 30.26612663269043,
      "activations/layer2_attention_weight_min": -29.677330017089844,
      "activations/layer3_attention_weight_max": 84.45673370361328,
      "activations/layer3_attention_weight_min": -84.61113739013672,
      "activations/layer4_attention_weight_max": 80.0589828491211,
      "activations/layer4_attention_weight_min": -77.4736557006836,
      "activations/layer5_attention_weight_max": 61.38360595703125,
      "activations/layer5_attention_weight_min": -69.233154296875,
      "activations/layer6_attention_weight_max": 47.886104583740234,
      "activations/layer6_attention_weight_min": -48.50078582763672,
      "activations/layer7_attention_weight_max": 63.762935638427734,
      "activations/layer7_attention_weight_min": -63.95967483520508,
      "activations/layer8_attention_weight_max": 45.47137451171875,
      "activations/layer8_attention_weight_min": -48.375160217285156,
      "activations/layer9_attention_weight_max": 44.30001449584961,
      "activations/layer9_attention_weight_min": -44.998348236083984,
      "epoch": 20.12,
      "learning_rate": 2.0484090909090906e-05,
      "loss": 2.7249,
      "step": 346200
    },
    {
      "activations/layer0_attention_weight_max": 15.72577953338623,
      "activations/layer0_attention_weight_min": -14.689116477966309,
      "activations/layer10_attention_weight_max": 38.821109771728516,
      "activations/layer10_attention_weight_min": -35.50710678100586,
      "activations/layer11_attention_weight_max": 36.157371520996094,
      "activations/layer11_attention_weight_min": -35.379638671875,
      "activations/layer12_attention_weight_max": 19.916879653930664,
      "activations/layer12_attention_weight_min": -23.20225715637207,
      "activations/layer13_attention_weight_max": 31.224504470825195,
      "activations/layer13_attention_weight_min": -26.935091018676758,
      "activations/layer14_attention_weight_max": 34.01262283325195,
      "activations/layer14_attention_weight_min": -33.25468826293945,
      "activations/layer15_attention_weight_max": 32.24834442138672,
      "activations/layer15_attention_weight_min": -30.406665802001953,
      "activations/layer16_attention_weight_max": 35.87194061279297,
      "activations/layer16_attention_weight_min": -32.70387268066406,
      "activations/layer17_attention_weight_max": 53.30855178833008,
      "activations/layer17_attention_weight_min": -45.2225227355957,
      "activations/layer18_attention_weight_max": 49.12726593017578,
      "activations/layer18_attention_weight_min": -43.16054153442383,
      "activations/layer19_attention_weight_max": 25.486862182617188,
      "activations/layer19_attention_weight_min": -26.866104125976562,
      "activations/layer1_attention_weight_max": 15.22680950164795,
      "activations/layer1_attention_weight_min": -12.907585144042969,
      "activations/layer20_attention_weight_max": 22.694955825805664,
      "activations/layer20_attention_weight_min": -23.424333572387695,
      "activations/layer21_attention_weight_max": 34.54779815673828,
      "activations/layer21_attention_weight_min": -23.523977279663086,
      "activations/layer22_attention_weight_max": 27.41806983947754,
      "activations/layer22_attention_weight_min": -27.274677276611328,
      "activations/layer23_attention_weight_max": 37.44206237792969,
      "activations/layer23_attention_weight_min": -24.63299560546875,
      "activations/layer2_attention_weight_max": 31.723806381225586,
      "activations/layer2_attention_weight_min": -29.70248031616211,
      "activations/layer3_attention_weight_max": 86.68254852294922,
      "activations/layer3_attention_weight_min": -87.47377014160156,
      "activations/layer4_attention_weight_max": 85.21580505371094,
      "activations/layer4_attention_weight_min": -83.16947937011719,
      "activations/layer5_attention_weight_max": 64.0298080444336,
      "activations/layer5_attention_weight_min": -74.97130584716797,
      "activations/layer6_attention_weight_max": 53.413536071777344,
      "activations/layer6_attention_weight_min": -52.05622100830078,
      "activations/layer7_attention_weight_max": 65.86817169189453,
      "activations/layer7_attention_weight_min": -67.8409423828125,
      "activations/layer8_attention_weight_max": 51.865516662597656,
      "activations/layer8_attention_weight_min": -54.736549377441406,
      "activations/layer9_attention_weight_max": 45.754032135009766,
      "activations/layer9_attention_weight_min": -48.91617965698242,
      "epoch": 20.12,
      "learning_rate": 2.0465151515151514e-05,
      "loss": 2.7087,
      "step": 346250
    },
    {
      "activations/layer0_attention_weight_max": 16.17884635925293,
      "activations/layer0_attention_weight_min": -14.21976375579834,
      "activations/layer10_attention_weight_max": 36.471866607666016,
      "activations/layer10_attention_weight_min": -35.069435119628906,
      "activations/layer11_attention_weight_max": 33.363243103027344,
      "activations/layer11_attention_weight_min": -35.842979431152344,
      "activations/layer12_attention_weight_max": 18.909936904907227,
      "activations/layer12_attention_weight_min": -27.7379093170166,
      "activations/layer13_attention_weight_max": 34.100032806396484,
      "activations/layer13_attention_weight_min": -28.442127227783203,
      "activations/layer14_attention_weight_max": 35.66414260864258,
      "activations/layer14_attention_weight_min": -31.543087005615234,
      "activations/layer15_attention_weight_max": 34.29051208496094,
      "activations/layer15_attention_weight_min": -31.3018798828125,
      "activations/layer16_attention_weight_max": 32.12506866455078,
      "activations/layer16_attention_weight_min": -31.492341995239258,
      "activations/layer17_attention_weight_max": 50.12905502319336,
      "activations/layer17_attention_weight_min": -45.599342346191406,
      "activations/layer18_attention_weight_max": 47.5040283203125,
      "activations/layer18_attention_weight_min": -42.042022705078125,
      "activations/layer19_attention_weight_max": 26.311767578125,
      "activations/layer19_attention_weight_min": -23.95009422302246,
      "activations/layer1_attention_weight_max": 15.483898162841797,
      "activations/layer1_attention_weight_min": -12.76176643371582,
      "activations/layer20_attention_weight_max": 22.67069435119629,
      "activations/layer20_attention_weight_min": -21.54143524169922,
      "activations/layer21_attention_weight_max": 35.59825897216797,
      "activations/layer21_attention_weight_min": -21.026750564575195,
      "activations/layer22_attention_weight_max": 28.38758087158203,
      "activations/layer22_attention_weight_min": -25.231252670288086,
      "activations/layer23_attention_weight_max": 36.145416259765625,
      "activations/layer23_attention_weight_min": -24.385860443115234,
      "activations/layer2_attention_weight_max": 30.489185333251953,
      "activations/layer2_attention_weight_min": -29.857023239135742,
      "activations/layer3_attention_weight_max": 85.38082122802734,
      "activations/layer3_attention_weight_min": -84.23450469970703,
      "activations/layer4_attention_weight_max": 82.81991577148438,
      "activations/layer4_attention_weight_min": -78.4165267944336,
      "activations/layer5_attention_weight_max": 61.72233200073242,
      "activations/layer5_attention_weight_min": -67.258056640625,
      "activations/layer6_attention_weight_max": 51.45487976074219,
      "activations/layer6_attention_weight_min": -48.276912689208984,
      "activations/layer7_attention_weight_max": 68.79191589355469,
      "activations/layer7_attention_weight_min": -62.54764938354492,
      "activations/layer8_attention_weight_max": 46.113277435302734,
      "activations/layer8_attention_weight_min": -46.5001220703125,
      "activations/layer9_attention_weight_max": 45.00388717651367,
      "activations/layer9_attention_weight_min": -44.90656280517578,
      "epoch": 20.12,
      "learning_rate": 2.0446212121212116e-05,
      "loss": 2.7316,
      "step": 346300
    },
    {
      "activations/layer0_attention_weight_max": 15.432756423950195,
      "activations/layer0_attention_weight_min": -14.46660041809082,
      "activations/layer10_attention_weight_max": 36.44795608520508,
      "activations/layer10_attention_weight_min": -36.099613189697266,
      "activations/layer11_attention_weight_max": 32.52742385864258,
      "activations/layer11_attention_weight_min": -34.16102600097656,
      "activations/layer12_attention_weight_max": 20.41793441772461,
      "activations/layer12_attention_weight_min": -24.20656967163086,
      "activations/layer13_attention_weight_max": 35.083152770996094,
      "activations/layer13_attention_weight_min": -28.330032348632812,
      "activations/layer14_attention_weight_max": 37.46134948730469,
      "activations/layer14_attention_weight_min": -31.35800552368164,
      "activations/layer15_attention_weight_max": 33.799827575683594,
      "activations/layer15_attention_weight_min": -32.16783142089844,
      "activations/layer16_attention_weight_max": 36.158287048339844,
      "activations/layer16_attention_weight_min": -34.21005630493164,
      "activations/layer17_attention_weight_max": 53.3939094543457,
      "activations/layer17_attention_weight_min": -46.025169372558594,
      "activations/layer18_attention_weight_max": 49.12487030029297,
      "activations/layer18_attention_weight_min": -43.72907638549805,
      "activations/layer19_attention_weight_max": 25.917463302612305,
      "activations/layer19_attention_weight_min": -26.373779296875,
      "activations/layer1_attention_weight_max": 15.787894248962402,
      "activations/layer1_attention_weight_min": -12.483251571655273,
      "activations/layer20_attention_weight_max": 22.936992645263672,
      "activations/layer20_attention_weight_min": -24.042051315307617,
      "activations/layer21_attention_weight_max": 37.32542037963867,
      "activations/layer21_attention_weight_min": -22.963327407836914,
      "activations/layer22_attention_weight_max": 28.748281478881836,
      "activations/layer22_attention_weight_min": -29.256834030151367,
      "activations/layer23_attention_weight_max": 41.855445861816406,
      "activations/layer23_attention_weight_min": -25.7259578704834,
      "activations/layer2_attention_weight_max": 30.47235679626465,
      "activations/layer2_attention_weight_min": -29.409881591796875,
      "activations/layer3_attention_weight_max": 83.6150131225586,
      "activations/layer3_attention_weight_min": -81.82548522949219,
      "activations/layer4_attention_weight_max": 79.43408203125,
      "activations/layer4_attention_weight_min": -77.59199523925781,
      "activations/layer5_attention_weight_max": 60.732688903808594,
      "activations/layer5_attention_weight_min": -69.09895324707031,
      "activations/layer6_attention_weight_max": 47.84707260131836,
      "activations/layer6_attention_weight_min": -48.10031509399414,
      "activations/layer7_attention_weight_max": 64.07034301757812,
      "activations/layer7_attention_weight_min": -64.08145904541016,
      "activations/layer8_attention_weight_max": 47.43578338623047,
      "activations/layer8_attention_weight_min": -49.92537307739258,
      "activations/layer9_attention_weight_max": 46.51045227050781,
      "activations/layer9_attention_weight_min": -45.84041213989258,
      "epoch": 20.12,
      "learning_rate": 2.0427272727272724e-05,
      "loss": 2.7193,
      "step": 346350
    },
    {
      "activations/layer0_attention_weight_max": 16.506999969482422,
      "activations/layer0_attention_weight_min": -14.013821601867676,
      "activations/layer10_attention_weight_max": 37.06549072265625,
      "activations/layer10_attention_weight_min": -35.989986419677734,
      "activations/layer11_attention_weight_max": 33.66140365600586,
      "activations/layer11_attention_weight_min": -34.410438537597656,
      "activations/layer12_attention_weight_max": 19.345808029174805,
      "activations/layer12_attention_weight_min": -23.063047409057617,
      "activations/layer13_attention_weight_max": 34.07501983642578,
      "activations/layer13_attention_weight_min": -26.23227882385254,
      "activations/layer14_attention_weight_max": 34.51564025878906,
      "activations/layer14_attention_weight_min": -29.07691192626953,
      "activations/layer15_attention_weight_max": 33.79000473022461,
      "activations/layer15_attention_weight_min": -29.536664962768555,
      "activations/layer16_attention_weight_max": 34.842994689941406,
      "activations/layer16_attention_weight_min": -32.90567398071289,
      "activations/layer17_attention_weight_max": 50.822635650634766,
      "activations/layer17_attention_weight_min": -43.7488899230957,
      "activations/layer18_attention_weight_max": 45.90683364868164,
      "activations/layer18_attention_weight_min": -37.394317626953125,
      "activations/layer19_attention_weight_max": 23.833837509155273,
      "activations/layer19_attention_weight_min": -21.23392105102539,
      "activations/layer1_attention_weight_max": 15.492751121520996,
      "activations/layer1_attention_weight_min": -14.352794647216797,
      "activations/layer20_attention_weight_max": 21.904451370239258,
      "activations/layer20_attention_weight_min": -19.5567626953125,
      "activations/layer21_attention_weight_max": 32.91716003417969,
      "activations/layer21_attention_weight_min": -21.63909339904785,
      "activations/layer22_attention_weight_max": 27.563352584838867,
      "activations/layer22_attention_weight_min": -25.66674041748047,
      "activations/layer23_attention_weight_max": 40.39030456542969,
      "activations/layer23_attention_weight_min": -24.559070587158203,
      "activations/layer2_attention_weight_max": 30.87939453125,
      "activations/layer2_attention_weight_min": -29.861248016357422,
      "activations/layer3_attention_weight_max": 83.9353256225586,
      "activations/layer3_attention_weight_min": -85.74006652832031,
      "activations/layer4_attention_weight_max": 81.22022247314453,
      "activations/layer4_attention_weight_min": -81.97058868408203,
      "activations/layer5_attention_weight_max": 62.382362365722656,
      "activations/layer5_attention_weight_min": -72.89482879638672,
      "activations/layer6_attention_weight_max": 50.098812103271484,
      "activations/layer6_attention_weight_min": -49.83015823364258,
      "activations/layer7_attention_weight_max": 61.92393112182617,
      "activations/layer7_attention_weight_min": -68.3338394165039,
      "activations/layer8_attention_weight_max": 47.0440559387207,
      "activations/layer8_attention_weight_min": -49.63096618652344,
      "activations/layer9_attention_weight_max": 43.70612716674805,
      "activations/layer9_attention_weight_min": -46.798316955566406,
      "epoch": 20.13,
      "learning_rate": 2.0408333333333332e-05,
      "loss": 2.702,
      "step": 346400
    },
    {
      "activations/layer0_attention_weight_max": 16.274032592773438,
      "activations/layer0_attention_weight_min": -14.225081443786621,
      "activations/layer10_attention_weight_max": 39.89794921875,
      "activations/layer10_attention_weight_min": -36.0135383605957,
      "activations/layer11_attention_weight_max": 35.14972686767578,
      "activations/layer11_attention_weight_min": -34.83179473876953,
      "activations/layer12_attention_weight_max": 19.781896591186523,
      "activations/layer12_attention_weight_min": -25.660120010375977,
      "activations/layer13_attention_weight_max": 48.32231140136719,
      "activations/layer13_attention_weight_min": -28.58194923400879,
      "activations/layer14_attention_weight_max": 50.854774475097656,
      "activations/layer14_attention_weight_min": -33.46875762939453,
      "activations/layer15_attention_weight_max": 48.6074333190918,
      "activations/layer15_attention_weight_min": -31.968915939331055,
      "activations/layer16_attention_weight_max": 40.24215316772461,
      "activations/layer16_attention_weight_min": -33.87313461303711,
      "activations/layer17_attention_weight_max": 59.57430648803711,
      "activations/layer17_attention_weight_min": -48.49917221069336,
      "activations/layer18_attention_weight_max": 59.70954895019531,
      "activations/layer18_attention_weight_min": -42.042415618896484,
      "activations/layer19_attention_weight_max": 27.337711334228516,
      "activations/layer19_attention_weight_min": -24.9390926361084,
      "activations/layer1_attention_weight_max": 15.6224946975708,
      "activations/layer1_attention_weight_min": -13.170654296875,
      "activations/layer20_attention_weight_max": 26.393810272216797,
      "activations/layer20_attention_weight_min": -23.187597274780273,
      "activations/layer21_attention_weight_max": 43.139259338378906,
      "activations/layer21_attention_weight_min": -23.457250595092773,
      "activations/layer22_attention_weight_max": 33.43656539916992,
      "activations/layer22_attention_weight_min": -26.4959716796875,
      "activations/layer23_attention_weight_max": 36.18727493286133,
      "activations/layer23_attention_weight_min": -25.891441345214844,
      "activations/layer2_attention_weight_max": 31.017391204833984,
      "activations/layer2_attention_weight_min": -29.598648071289062,
      "activations/layer3_attention_weight_max": 81.16002655029297,
      "activations/layer3_attention_weight_min": -83.220947265625,
      "activations/layer4_attention_weight_max": 77.40874481201172,
      "activations/layer4_attention_weight_min": -78.95101928710938,
      "activations/layer5_attention_weight_max": 60.979949951171875,
      "activations/layer5_attention_weight_min": -67.68368530273438,
      "activations/layer6_attention_weight_max": 45.31035232543945,
      "activations/layer6_attention_weight_min": -46.66917419433594,
      "activations/layer7_attention_weight_max": 62.377952575683594,
      "activations/layer7_attention_weight_min": -62.08169937133789,
      "activations/layer8_attention_weight_max": 46.81035614013672,
      "activations/layer8_attention_weight_min": -46.86073684692383,
      "activations/layer9_attention_weight_max": 41.32360076904297,
      "activations/layer9_attention_weight_min": -43.4228630065918,
      "epoch": 20.13,
      "learning_rate": 2.0389393939393937e-05,
      "loss": 2.7092,
      "step": 346450
    },
    {
      "activations/layer0_attention_weight_max": 15.541351318359375,
      "activations/layer0_attention_weight_min": -14.109612464904785,
      "activations/layer10_attention_weight_max": 34.19197082519531,
      "activations/layer10_attention_weight_min": -35.097938537597656,
      "activations/layer11_attention_weight_max": 33.32098388671875,
      "activations/layer11_attention_weight_min": -36.24048614501953,
      "activations/layer12_attention_weight_max": 18.68305778503418,
      "activations/layer12_attention_weight_min": -25.5172119140625,
      "activations/layer13_attention_weight_max": 34.90641784667969,
      "activations/layer13_attention_weight_min": -27.666521072387695,
      "activations/layer14_attention_weight_max": 37.32111358642578,
      "activations/layer14_attention_weight_min": -30.08500099182129,
      "activations/layer15_attention_weight_max": 36.225948333740234,
      "activations/layer15_attention_weight_min": -30.765216827392578,
      "activations/layer16_attention_weight_max": 35.81003189086914,
      "activations/layer16_attention_weight_min": -33.478885650634766,
      "activations/layer17_attention_weight_max": 59.56173324584961,
      "activations/layer17_attention_weight_min": -49.148616790771484,
      "activations/layer18_attention_weight_max": 53.032554626464844,
      "activations/layer18_attention_weight_min": -41.58528518676758,
      "activations/layer19_attention_weight_max": 25.488706588745117,
      "activations/layer19_attention_weight_min": -23.462255477905273,
      "activations/layer1_attention_weight_max": 15.820277214050293,
      "activations/layer1_attention_weight_min": -12.517010688781738,
      "activations/layer20_attention_weight_max": 24.049724578857422,
      "activations/layer20_attention_weight_min": -21.81540870666504,
      "activations/layer21_attention_weight_max": 45.505828857421875,
      "activations/layer21_attention_weight_min": -23.7844181060791,
      "activations/layer22_attention_weight_max": 29.732175827026367,
      "activations/layer22_attention_weight_min": -25.769330978393555,
      "activations/layer23_attention_weight_max": 39.342166900634766,
      "activations/layer23_attention_weight_min": -24.10773468017578,
      "activations/layer2_attention_weight_max": 29.77308464050293,
      "activations/layer2_attention_weight_min": -29.413358688354492,
      "activations/layer3_attention_weight_max": 81.2073745727539,
      "activations/layer3_attention_weight_min": -81.48728942871094,
      "activations/layer4_attention_weight_max": 79.87151336669922,
      "activations/layer4_attention_weight_min": -79.29686737060547,
      "activations/layer5_attention_weight_max": 57.83028793334961,
      "activations/layer5_attention_weight_min": -72.52415466308594,
      "activations/layer6_attention_weight_max": 45.61260223388672,
      "activations/layer6_attention_weight_min": -48.85334014892578,
      "activations/layer7_attention_weight_max": 69.68496704101562,
      "activations/layer7_attention_weight_min": -66.31926727294922,
      "activations/layer8_attention_weight_max": 45.881961822509766,
      "activations/layer8_attention_weight_min": -50.71223068237305,
      "activations/layer9_attention_weight_max": 53.1773796081543,
      "activations/layer9_attention_weight_min": -47.03407669067383,
      "epoch": 20.13,
      "learning_rate": 2.0370454545454546e-05,
      "loss": 2.7167,
      "step": 346500
    },
    {
      "activations/layer0_attention_weight_max": 15.075615882873535,
      "activations/layer0_attention_weight_min": -14.265260696411133,
      "activations/layer10_attention_weight_max": 37.00356674194336,
      "activations/layer10_attention_weight_min": -34.835899353027344,
      "activations/layer11_attention_weight_max": 32.610191345214844,
      "activations/layer11_attention_weight_min": -34.20515441894531,
      "activations/layer12_attention_weight_max": 18.720979690551758,
      "activations/layer12_attention_weight_min": -24.282569885253906,
      "activations/layer13_attention_weight_max": 35.56964874267578,
      "activations/layer13_attention_weight_min": -27.781047821044922,
      "activations/layer14_attention_weight_max": 36.04038619995117,
      "activations/layer14_attention_weight_min": -30.717527389526367,
      "activations/layer15_attention_weight_max": 33.09746170043945,
      "activations/layer15_attention_weight_min": -32.131778717041016,
      "activations/layer16_attention_weight_max": 34.00598907470703,
      "activations/layer16_attention_weight_min": -34.226806640625,
      "activations/layer17_attention_weight_max": 52.17415237426758,
      "activations/layer17_attention_weight_min": -45.851871490478516,
      "activations/layer18_attention_weight_max": 50.53521728515625,
      "activations/layer18_attention_weight_min": -40.380680084228516,
      "activations/layer19_attention_weight_max": 28.649250030517578,
      "activations/layer19_attention_weight_min": -21.349567413330078,
      "activations/layer1_attention_weight_max": 14.994328498840332,
      "activations/layer1_attention_weight_min": -14.215763092041016,
      "activations/layer20_attention_weight_max": 21.416202545166016,
      "activations/layer20_attention_weight_min": -22.463863372802734,
      "activations/layer21_attention_weight_max": 31.119033813476562,
      "activations/layer21_attention_weight_min": -22.50878143310547,
      "activations/layer22_attention_weight_max": 28.633686065673828,
      "activations/layer22_attention_weight_min": -27.490142822265625,
      "activations/layer23_attention_weight_max": 36.092430114746094,
      "activations/layer23_attention_weight_min": -27.658111572265625,
      "activations/layer2_attention_weight_max": 29.820053100585938,
      "activations/layer2_attention_weight_min": -28.29646873474121,
      "activations/layer3_attention_weight_max": 83.13343811035156,
      "activations/layer3_attention_weight_min": -79.6633529663086,
      "activations/layer4_attention_weight_max": 79.31603240966797,
      "activations/layer4_attention_weight_min": -77.49549102783203,
      "activations/layer5_attention_weight_max": 63.845272064208984,
      "activations/layer5_attention_weight_min": -69.96847534179688,
      "activations/layer6_attention_weight_max": 47.49187469482422,
      "activations/layer6_attention_weight_min": -51.535221099853516,
      "activations/layer7_attention_weight_max": 65.4570083618164,
      "activations/layer7_attention_weight_min": -64.51106262207031,
      "activations/layer8_attention_weight_max": 48.510658264160156,
      "activations/layer8_attention_weight_min": -51.595760345458984,
      "activations/layer9_attention_weight_max": 44.256736755371094,
      "activations/layer9_attention_weight_min": -46.88934326171875,
      "epoch": 20.14,
      "learning_rate": 2.035151515151515e-05,
      "loss": 2.7224,
      "step": 346550
    },
    {
      "activations/layer0_attention_weight_max": 18.21219825744629,
      "activations/layer0_attention_weight_min": -13.975064277648926,
      "activations/layer10_attention_weight_max": 35.16098403930664,
      "activations/layer10_attention_weight_min": -36.9930305480957,
      "activations/layer11_attention_weight_max": 32.41698455810547,
      "activations/layer11_attention_weight_min": -35.07029724121094,
      "activations/layer12_attention_weight_max": 20.176355361938477,
      "activations/layer12_attention_weight_min": -23.6439151763916,
      "activations/layer13_attention_weight_max": 38.18775177001953,
      "activations/layer13_attention_weight_min": -28.202476501464844,
      "activations/layer14_attention_weight_max": 37.19904327392578,
      "activations/layer14_attention_weight_min": -31.822338104248047,
      "activations/layer15_attention_weight_max": 36.05490493774414,
      "activations/layer15_attention_weight_min": -30.982120513916016,
      "activations/layer16_attention_weight_max": 37.81494140625,
      "activations/layer16_attention_weight_min": -34.68912887573242,
      "activations/layer17_attention_weight_max": 55.288475036621094,
      "activations/layer17_attention_weight_min": -47.10499954223633,
      "activations/layer18_attention_weight_max": 52.262123107910156,
      "activations/layer18_attention_weight_min": -42.81486511230469,
      "activations/layer19_attention_weight_max": 24.53220558166504,
      "activations/layer19_attention_weight_min": -24.94074058532715,
      "activations/layer1_attention_weight_max": 16.39511489868164,
      "activations/layer1_attention_weight_min": -14.272468566894531,
      "activations/layer20_attention_weight_max": 23.062379837036133,
      "activations/layer20_attention_weight_min": -23.9683895111084,
      "activations/layer21_attention_weight_max": 38.1318359375,
      "activations/layer21_attention_weight_min": -22.552812576293945,
      "activations/layer22_attention_weight_max": 31.722736358642578,
      "activations/layer22_attention_weight_min": -26.1883544921875,
      "activations/layer23_attention_weight_max": 42.916725158691406,
      "activations/layer23_attention_weight_min": -22.68086051940918,
      "activations/layer2_attention_weight_max": 30.652976989746094,
      "activations/layer2_attention_weight_min": -28.638113021850586,
      "activations/layer3_attention_weight_max": 77.27650451660156,
      "activations/layer3_attention_weight_min": -78.21805572509766,
      "activations/layer4_attention_weight_max": 78.05644226074219,
      "activations/layer4_attention_weight_min": -79.95966339111328,
      "activations/layer5_attention_weight_max": 60.868858337402344,
      "activations/layer5_attention_weight_min": -67.63227844238281,
      "activations/layer6_attention_weight_max": 45.276851654052734,
      "activations/layer6_attention_weight_min": -47.72932052612305,
      "activations/layer7_attention_weight_max": 65.4617691040039,
      "activations/layer7_attention_weight_min": -61.081947326660156,
      "activations/layer8_attention_weight_max": 44.9781608581543,
      "activations/layer8_attention_weight_min": -49.888038635253906,
      "activations/layer9_attention_weight_max": 45.62842559814453,
      "activations/layer9_attention_weight_min": -46.980003356933594,
      "epoch": 20.14,
      "learning_rate": 2.033257575757576e-05,
      "loss": 2.7216,
      "step": 346600
    },
    {
      "activations/layer0_attention_weight_max": 15.452228546142578,
      "activations/layer0_attention_weight_min": -14.282463073730469,
      "activations/layer10_attention_weight_max": 35.7939453125,
      "activations/layer10_attention_weight_min": -36.583343505859375,
      "activations/layer11_attention_weight_max": 33.79383087158203,
      "activations/layer11_attention_weight_min": -35.74629592895508,
      "activations/layer12_attention_weight_max": 19.26765251159668,
      "activations/layer12_attention_weight_min": -24.473581314086914,
      "activations/layer13_attention_weight_max": 34.62355041503906,
      "activations/layer13_attention_weight_min": -27.60980796813965,
      "activations/layer14_attention_weight_max": 36.32115936279297,
      "activations/layer14_attention_weight_min": -30.856616973876953,
      "activations/layer15_attention_weight_max": 34.36495590209961,
      "activations/layer15_attention_weight_min": -30.37095832824707,
      "activations/layer16_attention_weight_max": 34.24934005737305,
      "activations/layer16_attention_weight_min": -32.693912506103516,
      "activations/layer17_attention_weight_max": 48.89554214477539,
      "activations/layer17_attention_weight_min": -45.35540771484375,
      "activations/layer18_attention_weight_max": 46.18130874633789,
      "activations/layer18_attention_weight_min": -38.427433013916016,
      "activations/layer19_attention_weight_max": 27.530500411987305,
      "activations/layer19_attention_weight_min": -22.14201545715332,
      "activations/layer1_attention_weight_max": 14.715890884399414,
      "activations/layer1_attention_weight_min": -14.257948875427246,
      "activations/layer20_attention_weight_max": 22.847719192504883,
      "activations/layer20_attention_weight_min": -20.51396942138672,
      "activations/layer21_attention_weight_max": 36.42595672607422,
      "activations/layer21_attention_weight_min": -23.178813934326172,
      "activations/layer22_attention_weight_max": 29.066829681396484,
      "activations/layer22_attention_weight_min": -23.72539520263672,
      "activations/layer23_attention_weight_max": 39.30836486816406,
      "activations/layer23_attention_weight_min": -23.087322235107422,
      "activations/layer2_attention_weight_max": 32.16960525512695,
      "activations/layer2_attention_weight_min": -30.040924072265625,
      "activations/layer3_attention_weight_max": 87.07771301269531,
      "activations/layer3_attention_weight_min": -82.7574691772461,
      "activations/layer4_attention_weight_max": 82.19563293457031,
      "activations/layer4_attention_weight_min": -79.79354095458984,
      "activations/layer5_attention_weight_max": 67.4312744140625,
      "activations/layer5_attention_weight_min": -72.80341339111328,
      "activations/layer6_attention_weight_max": 50.713993072509766,
      "activations/layer6_attention_weight_min": -50.08610153198242,
      "activations/layer7_attention_weight_max": 65.44564819335938,
      "activations/layer7_attention_weight_min": -63.53361892700195,
      "activations/layer8_attention_weight_max": 46.38751983642578,
      "activations/layer8_attention_weight_min": -51.9291877746582,
      "activations/layer9_attention_weight_max": 46.68431091308594,
      "activations/layer9_attention_weight_min": -44.44794464111328,
      "epoch": 20.14,
      "learning_rate": 2.031363636363636e-05,
      "loss": 2.741,
      "step": 346650
    },
    {
      "activations/layer0_attention_weight_max": 15.34708309173584,
      "activations/layer0_attention_weight_min": -13.905966758728027,
      "activations/layer10_attention_weight_max": 37.369972229003906,
      "activations/layer10_attention_weight_min": -37.986328125,
      "activations/layer11_attention_weight_max": 32.76488494873047,
      "activations/layer11_attention_weight_min": -35.030487060546875,
      "activations/layer12_attention_weight_max": 19.05003547668457,
      "activations/layer12_attention_weight_min": -23.91486358642578,
      "activations/layer13_attention_weight_max": 36.44658660888672,
      "activations/layer13_attention_weight_min": -28.20989227294922,
      "activations/layer14_attention_weight_max": 34.65564727783203,
      "activations/layer14_attention_weight_min": -31.330989837646484,
      "activations/layer15_attention_weight_max": 34.29051971435547,
      "activations/layer15_attention_weight_min": -31.80052947998047,
      "activations/layer16_attention_weight_max": 36.25848388671875,
      "activations/layer16_attention_weight_min": -34.35048294067383,
      "activations/layer17_attention_weight_max": 53.76731872558594,
      "activations/layer17_attention_weight_min": -48.09955978393555,
      "activations/layer18_attention_weight_max": 51.547000885009766,
      "activations/layer18_attention_weight_min": -42.98326873779297,
      "activations/layer19_attention_weight_max": 26.7888126373291,
      "activations/layer19_attention_weight_min": -25.125507354736328,
      "activations/layer1_attention_weight_max": 16.77634048461914,
      "activations/layer1_attention_weight_min": -13.738183975219727,
      "activations/layer20_attention_weight_max": 26.247722625732422,
      "activations/layer20_attention_weight_min": -21.58654022216797,
      "activations/layer21_attention_weight_max": 38.80963897705078,
      "activations/layer21_attention_weight_min": -21.478822708129883,
      "activations/layer22_attention_weight_max": 33.09923553466797,
      "activations/layer22_attention_weight_min": -26.831872940063477,
      "activations/layer23_attention_weight_max": 37.17720413208008,
      "activations/layer23_attention_weight_min": -25.815933227539062,
      "activations/layer2_attention_weight_max": 31.98257827758789,
      "activations/layer2_attention_weight_min": -29.630796432495117,
      "activations/layer3_attention_weight_max": 81.79962158203125,
      "activations/layer3_attention_weight_min": -85.31153869628906,
      "activations/layer4_attention_weight_max": 80.17008972167969,
      "activations/layer4_attention_weight_min": -83.11673736572266,
      "activations/layer5_attention_weight_max": 59.681026458740234,
      "activations/layer5_attention_weight_min": -70.58930969238281,
      "activations/layer6_attention_weight_max": 46.64583206176758,
      "activations/layer6_attention_weight_min": -48.476932525634766,
      "activations/layer7_attention_weight_max": 59.678043365478516,
      "activations/layer7_attention_weight_min": -62.94390869140625,
      "activations/layer8_attention_weight_max": 43.62406539916992,
      "activations/layer8_attention_weight_min": -51.737953186035156,
      "activations/layer9_attention_weight_max": 41.24616622924805,
      "activations/layer9_attention_weight_min": -45.02104187011719,
      "epoch": 20.15,
      "learning_rate": 2.029469696969697e-05,
      "loss": 2.7211,
      "step": 346700
    },
    {
      "activations/layer0_attention_weight_max": 15.039685249328613,
      "activations/layer0_attention_weight_min": -13.941178321838379,
      "activations/layer10_attention_weight_max": 32.3384895324707,
      "activations/layer10_attention_weight_min": -34.20330810546875,
      "activations/layer11_attention_weight_max": 31.207481384277344,
      "activations/layer11_attention_weight_min": -35.31858444213867,
      "activations/layer12_attention_weight_max": 17.9578914642334,
      "activations/layer12_attention_weight_min": -24.36544418334961,
      "activations/layer13_attention_weight_max": 33.102272033691406,
      "activations/layer13_attention_weight_min": -25.32970428466797,
      "activations/layer14_attention_weight_max": 31.343568801879883,
      "activations/layer14_attention_weight_min": -29.1978702545166,
      "activations/layer15_attention_weight_max": 31.76072120666504,
      "activations/layer15_attention_weight_min": -30.516098022460938,
      "activations/layer16_attention_weight_max": 31.61659812927246,
      "activations/layer16_attention_weight_min": -31.731136322021484,
      "activations/layer17_attention_weight_max": 48.8505744934082,
      "activations/layer17_attention_weight_min": -44.54763412475586,
      "activations/layer18_attention_weight_max": 44.44047927856445,
      "activations/layer18_attention_weight_min": -43.431373596191406,
      "activations/layer19_attention_weight_max": 24.17814064025879,
      "activations/layer19_attention_weight_min": -24.95879554748535,
      "activations/layer1_attention_weight_max": 16.18763542175293,
      "activations/layer1_attention_weight_min": -13.832074165344238,
      "activations/layer20_attention_weight_max": 23.428081512451172,
      "activations/layer20_attention_weight_min": -23.62889289855957,
      "activations/layer21_attention_weight_max": 38.0469970703125,
      "activations/layer21_attention_weight_min": -23.67059326171875,
      "activations/layer22_attention_weight_max": 28.992414474487305,
      "activations/layer22_attention_weight_min": -27.189918518066406,
      "activations/layer23_attention_weight_max": 32.60554504394531,
      "activations/layer23_attention_weight_min": -24.47738265991211,
      "activations/layer2_attention_weight_max": 32.61989974975586,
      "activations/layer2_attention_weight_min": -30.682889938354492,
      "activations/layer3_attention_weight_max": 85.89387512207031,
      "activations/layer3_attention_weight_min": -86.78306579589844,
      "activations/layer4_attention_weight_max": 83.24192810058594,
      "activations/layer4_attention_weight_min": -83.11454010009766,
      "activations/layer5_attention_weight_max": 59.86405563354492,
      "activations/layer5_attention_weight_min": -73.72704315185547,
      "activations/layer6_attention_weight_max": 46.255802154541016,
      "activations/layer6_attention_weight_min": -49.97138214111328,
      "activations/layer7_attention_weight_max": 59.715877532958984,
      "activations/layer7_attention_weight_min": -65.83537292480469,
      "activations/layer8_attention_weight_max": 43.72905731201172,
      "activations/layer8_attention_weight_min": -51.095890045166016,
      "activations/layer9_attention_weight_max": 40.35786056518555,
      "activations/layer9_attention_weight_min": -47.5919303894043,
      "epoch": 20.15,
      "learning_rate": 2.0276136363636363e-05,
      "loss": 2.7031,
      "step": 346750
    },
    {
      "activations/layer0_attention_weight_max": 15.061644554138184,
      "activations/layer0_attention_weight_min": -14.362905502319336,
      "activations/layer10_attention_weight_max": 39.30500411987305,
      "activations/layer10_attention_weight_min": -38.72393035888672,
      "activations/layer11_attention_weight_max": 36.373863220214844,
      "activations/layer11_attention_weight_min": -35.542869567871094,
      "activations/layer12_attention_weight_max": 19.554779052734375,
      "activations/layer12_attention_weight_min": -24.095985412597656,
      "activations/layer13_attention_weight_max": 35.18163299560547,
      "activations/layer13_attention_weight_min": -31.239633560180664,
      "activations/layer14_attention_weight_max": 36.958126068115234,
      "activations/layer14_attention_weight_min": -29.984134674072266,
      "activations/layer15_attention_weight_max": 33.72377395629883,
      "activations/layer15_attention_weight_min": -28.927982330322266,
      "activations/layer16_attention_weight_max": 35.747379302978516,
      "activations/layer16_attention_weight_min": -32.66242980957031,
      "activations/layer17_attention_weight_max": 51.68661117553711,
      "activations/layer17_attention_weight_min": -43.37915802001953,
      "activations/layer18_attention_weight_max": 47.41792297363281,
      "activations/layer18_attention_weight_min": -39.359737396240234,
      "activations/layer19_attention_weight_max": 25.077232360839844,
      "activations/layer19_attention_weight_min": -23.1337833404541,
      "activations/layer1_attention_weight_max": 14.86168384552002,
      "activations/layer1_attention_weight_min": -12.821815490722656,
      "activations/layer20_attention_weight_max": 23.626853942871094,
      "activations/layer20_attention_weight_min": -20.817623138427734,
      "activations/layer21_attention_weight_max": 33.41016387939453,
      "activations/layer21_attention_weight_min": -23.21760368347168,
      "activations/layer22_attention_weight_max": 29.09309196472168,
      "activations/layer22_attention_weight_min": -26.96731948852539,
      "activations/layer23_attention_weight_max": 37.96360778808594,
      "activations/layer23_attention_weight_min": -22.200902938842773,
      "activations/layer2_attention_weight_max": 31.88116455078125,
      "activations/layer2_attention_weight_min": -29.504291534423828,
      "activations/layer3_attention_weight_max": 83.00711822509766,
      "activations/layer3_attention_weight_min": -81.86189270019531,
      "activations/layer4_attention_weight_max": 82.10167694091797,
      "activations/layer4_attention_weight_min": -80.96984100341797,
      "activations/layer5_attention_weight_max": 61.990413665771484,
      "activations/layer5_attention_weight_min": -70.47001647949219,
      "activations/layer6_attention_weight_max": 50.16276168823242,
      "activations/layer6_attention_weight_min": -50.26215362548828,
      "activations/layer7_attention_weight_max": 67.87655639648438,
      "activations/layer7_attention_weight_min": -66.28765869140625,
      "activations/layer8_attention_weight_max": 49.0139045715332,
      "activations/layer8_attention_weight_min": -54.71639633178711,
      "activations/layer9_attention_weight_max": 47.3639030456543,
      "activations/layer9_attention_weight_min": -49.21553039550781,
      "epoch": 20.15,
      "learning_rate": 2.0257196969696968e-05,
      "loss": 2.7386,
      "step": 346800
    },
    {
      "activations/layer0_attention_weight_max": 15.105072975158691,
      "activations/layer0_attention_weight_min": -14.234941482543945,
      "activations/layer10_attention_weight_max": 39.2381591796875,
      "activations/layer10_attention_weight_min": -39.14293670654297,
      "activations/layer11_attention_weight_max": 39.883243560791016,
      "activations/layer11_attention_weight_min": -34.82877731323242,
      "activations/layer12_attention_weight_max": 18.090606689453125,
      "activations/layer12_attention_weight_min": -27.497983932495117,
      "activations/layer13_attention_weight_max": 32.664188385009766,
      "activations/layer13_attention_weight_min": -27.671724319458008,
      "activations/layer14_attention_weight_max": 39.88619613647461,
      "activations/layer14_attention_weight_min": -29.091567993164062,
      "activations/layer15_attention_weight_max": 36.02085876464844,
      "activations/layer15_attention_weight_min": -29.318485260009766,
      "activations/layer16_attention_weight_max": 31.58841896057129,
      "activations/layer16_attention_weight_min": -30.707096099853516,
      "activations/layer17_attention_weight_max": 47.97385787963867,
      "activations/layer17_attention_weight_min": -44.49842834472656,
      "activations/layer18_attention_weight_max": 45.06064224243164,
      "activations/layer18_attention_weight_min": -39.641204833984375,
      "activations/layer19_attention_weight_max": 22.36788558959961,
      "activations/layer19_attention_weight_min": -22.07101821899414,
      "activations/layer1_attention_weight_max": 15.179292678833008,
      "activations/layer1_attention_weight_min": -12.894922256469727,
      "activations/layer20_attention_weight_max": 20.969444274902344,
      "activations/layer20_attention_weight_min": -22.605369567871094,
      "activations/layer21_attention_weight_max": 39.30762481689453,
      "activations/layer21_attention_weight_min": -26.32066535949707,
      "activations/layer22_attention_weight_max": 28.790037155151367,
      "activations/layer22_attention_weight_min": -25.65167808532715,
      "activations/layer23_attention_weight_max": 32.15380859375,
      "activations/layer23_attention_weight_min": -24.510862350463867,
      "activations/layer2_attention_weight_max": 31.628631591796875,
      "activations/layer2_attention_weight_min": -29.557647705078125,
      "activations/layer3_attention_weight_max": 82.4540023803711,
      "activations/layer3_attention_weight_min": -82.12451934814453,
      "activations/layer4_attention_weight_max": 77.80719757080078,
      "activations/layer4_attention_weight_min": -74.05491638183594,
      "activations/layer5_attention_weight_max": 64.8688735961914,
      "activations/layer5_attention_weight_min": -65.6900634765625,
      "activations/layer6_attention_weight_max": 46.151790618896484,
      "activations/layer6_attention_weight_min": -47.92222595214844,
      "activations/layer7_attention_weight_max": 70.09113311767578,
      "activations/layer7_attention_weight_min": -65.8555679321289,
      "activations/layer8_attention_weight_max": 50.901641845703125,
      "activations/layer8_attention_weight_min": -51.27677917480469,
      "activations/layer9_attention_weight_max": 56.60395431518555,
      "activations/layer9_attention_weight_min": -49.41695022583008,
      "epoch": 20.15,
      "learning_rate": 2.0238257575757573e-05,
      "loss": 2.7051,
      "step": 346850
    },
    {
      "activations/layer0_attention_weight_max": 14.76646614074707,
      "activations/layer0_attention_weight_min": -14.453638076782227,
      "activations/layer10_attention_weight_max": 37.2407112121582,
      "activations/layer10_attention_weight_min": -38.77849197387695,
      "activations/layer11_attention_weight_max": 35.06970977783203,
      "activations/layer11_attention_weight_min": -38.38850021362305,
      "activations/layer12_attention_weight_max": 18.224349975585938,
      "activations/layer12_attention_weight_min": -32.71016311645508,
      "activations/layer13_attention_weight_max": 30.433557510375977,
      "activations/layer13_attention_weight_min": -25.492918014526367,
      "activations/layer14_attention_weight_max": 33.64225387573242,
      "activations/layer14_attention_weight_min": -30.366830825805664,
      "activations/layer15_attention_weight_max": 32.65536117553711,
      "activations/layer15_attention_weight_min": -32.44430160522461,
      "activations/layer16_attention_weight_max": 34.413700103759766,
      "activations/layer16_attention_weight_min": -31.831005096435547,
      "activations/layer17_attention_weight_max": 55.087684631347656,
      "activations/layer17_attention_weight_min": -44.89769744873047,
      "activations/layer18_attention_weight_max": 46.33808135986328,
      "activations/layer18_attention_weight_min": -41.15629196166992,
      "activations/layer19_attention_weight_max": 22.61017608642578,
      "activations/layer19_attention_weight_min": -23.277219772338867,
      "activations/layer1_attention_weight_max": 16.256526947021484,
      "activations/layer1_attention_weight_min": -13.256436347961426,
      "activations/layer20_attention_weight_max": 20.261093139648438,
      "activations/layer20_attention_weight_min": -21.208051681518555,
      "activations/layer21_attention_weight_max": 32.077796936035156,
      "activations/layer21_attention_weight_min": -24.30283546447754,
      "activations/layer22_attention_weight_max": 26.635318756103516,
      "activations/layer22_attention_weight_min": -25.34740447998047,
      "activations/layer23_attention_weight_max": 35.07768630981445,
      "activations/layer23_attention_weight_min": -23.18634033203125,
      "activations/layer2_attention_weight_max": 31.534406661987305,
      "activations/layer2_attention_weight_min": -30.613845825195312,
      "activations/layer3_attention_weight_max": 86.03252410888672,
      "activations/layer3_attention_weight_min": -94.54144287109375,
      "activations/layer4_attention_weight_max": 84.56114196777344,
      "activations/layer4_attention_weight_min": -85.18350219726562,
      "activations/layer5_attention_weight_max": 64.02622985839844,
      "activations/layer5_attention_weight_min": -69.71707916259766,
      "activations/layer6_attention_weight_max": 49.377017974853516,
      "activations/layer6_attention_weight_min": -51.48771667480469,
      "activations/layer7_attention_weight_max": 64.50231170654297,
      "activations/layer7_attention_weight_min": -65.89281463623047,
      "activations/layer8_attention_weight_max": 49.48530960083008,
      "activations/layer8_attention_weight_min": -53.77484893798828,
      "activations/layer9_attention_weight_max": 47.39347839355469,
      "activations/layer9_attention_weight_min": -48.610076904296875,
      "epoch": 20.16,
      "learning_rate": 2.021931818181818e-05,
      "loss": 2.7309,
      "step": 346900
    },
    {
      "activations/layer0_attention_weight_max": 15.290284156799316,
      "activations/layer0_attention_weight_min": -14.035911560058594,
      "activations/layer10_attention_weight_max": 35.50477600097656,
      "activations/layer10_attention_weight_min": -35.84203338623047,
      "activations/layer11_attention_weight_max": 34.48146438598633,
      "activations/layer11_attention_weight_min": -34.89604949951172,
      "activations/layer12_attention_weight_max": 18.764440536499023,
      "activations/layer12_attention_weight_min": -25.9711971282959,
      "activations/layer13_attention_weight_max": 35.452327728271484,
      "activations/layer13_attention_weight_min": -26.76706314086914,
      "activations/layer14_attention_weight_max": 35.77132034301758,
      "activations/layer14_attention_weight_min": -30.919109344482422,
      "activations/layer15_attention_weight_max": 36.095035552978516,
      "activations/layer15_attention_weight_min": -30.373998641967773,
      "activations/layer16_attention_weight_max": 35.00096893310547,
      "activations/layer16_attention_weight_min": -32.20738220214844,
      "activations/layer17_attention_weight_max": 50.38336181640625,
      "activations/layer17_attention_weight_min": -44.36411666870117,
      "activations/layer18_attention_weight_max": 45.57093811035156,
      "activations/layer18_attention_weight_min": -39.893245697021484,
      "activations/layer19_attention_weight_max": 22.784944534301758,
      "activations/layer19_attention_weight_min": -23.370668411254883,
      "activations/layer1_attention_weight_max": 15.40971851348877,
      "activations/layer1_attention_weight_min": -14.415903091430664,
      "activations/layer20_attention_weight_max": 21.246231079101562,
      "activations/layer20_attention_weight_min": -23.827062606811523,
      "activations/layer21_attention_weight_max": 38.86822509765625,
      "activations/layer21_attention_weight_min": -22.137948989868164,
      "activations/layer22_attention_weight_max": 30.788293838500977,
      "activations/layer22_attention_weight_min": -26.266754150390625,
      "activations/layer23_attention_weight_max": 35.52043151855469,
      "activations/layer23_attention_weight_min": -24.809993743896484,
      "activations/layer2_attention_weight_max": 31.62146759033203,
      "activations/layer2_attention_weight_min": -29.994991302490234,
      "activations/layer3_attention_weight_max": 86.51225280761719,
      "activations/layer3_attention_weight_min": -84.47906494140625,
      "activations/layer4_attention_weight_max": 82.5202865600586,
      "activations/layer4_attention_weight_min": -82.84642791748047,
      "activations/layer5_attention_weight_max": 60.787742614746094,
      "activations/layer5_attention_weight_min": -68.44905090332031,
      "activations/layer6_attention_weight_max": 49.18732833862305,
      "activations/layer6_attention_weight_min": -49.87968444824219,
      "activations/layer7_attention_weight_max": 65.27455139160156,
      "activations/layer7_attention_weight_min": -66.15353393554688,
      "activations/layer8_attention_weight_max": 46.551185607910156,
      "activations/layer8_attention_weight_min": -49.44593811035156,
      "activations/layer9_attention_weight_max": 42.29008102416992,
      "activations/layer9_attention_weight_min": -45.411598205566406,
      "epoch": 20.16,
      "learning_rate": 2.0200378787878786e-05,
      "loss": 2.7169,
      "step": 346950
    },
    {
      "activations/layer0_attention_weight_max": 15.58233642578125,
      "activations/layer0_attention_weight_min": -14.318982124328613,
      "activations/layer10_attention_weight_max": 38.780311584472656,
      "activations/layer10_attention_weight_min": -36.99568557739258,
      "activations/layer11_attention_weight_max": 35.4150505065918,
      "activations/layer11_attention_weight_min": -36.978431701660156,
      "activations/layer12_attention_weight_max": 18.392723083496094,
      "activations/layer12_attention_weight_min": -27.828922271728516,
      "activations/layer13_attention_weight_max": 33.74673843383789,
      "activations/layer13_attention_weight_min": -29.426198959350586,
      "activations/layer14_attention_weight_max": 35.4754753112793,
      "activations/layer14_attention_weight_min": -32.12135696411133,
      "activations/layer15_attention_weight_max": 33.58124542236328,
      "activations/layer15_attention_weight_min": -30.69586181640625,
      "activations/layer16_attention_weight_max": 34.20696258544922,
      "activations/layer16_attention_weight_min": -32.440853118896484,
      "activations/layer17_attention_weight_max": 49.26228713989258,
      "activations/layer17_attention_weight_min": -43.80914306640625,
      "activations/layer18_attention_weight_max": 45.31782150268555,
      "activations/layer18_attention_weight_min": -39.31760025024414,
      "activations/layer19_attention_weight_max": 23.852787017822266,
      "activations/layer19_attention_weight_min": -22.52064323425293,
      "activations/layer1_attention_weight_max": 15.118595123291016,
      "activations/layer1_attention_weight_min": -13.347626686096191,
      "activations/layer20_attention_weight_max": 20.754379272460938,
      "activations/layer20_attention_weight_min": -21.534717559814453,
      "activations/layer21_attention_weight_max": 34.966243743896484,
      "activations/layer21_attention_weight_min": -24.638750076293945,
      "activations/layer22_attention_weight_max": 27.66969871520996,
      "activations/layer22_attention_weight_min": -25.489423751831055,
      "activations/layer23_attention_weight_max": 33.40898513793945,
      "activations/layer23_attention_weight_min": -24.54378890991211,
      "activations/layer2_attention_weight_max": 31.141775131225586,
      "activations/layer2_attention_weight_min": -30.31178092956543,
      "activations/layer3_attention_weight_max": 83.94384002685547,
      "activations/layer3_attention_weight_min": -85.36088562011719,
      "activations/layer4_attention_weight_max": 84.59313201904297,
      "activations/layer4_attention_weight_min": -87.5731201171875,
      "activations/layer5_attention_weight_max": 67.03427124023438,
      "activations/layer5_attention_weight_min": -75.73713684082031,
      "activations/layer6_attention_weight_max": 51.50574493408203,
      "activations/layer6_attention_weight_min": -55.939857482910156,
      "activations/layer7_attention_weight_max": 70.09515380859375,
      "activations/layer7_attention_weight_min": -67.09119415283203,
      "activations/layer8_attention_weight_max": 50.38167953491211,
      "activations/layer8_attention_weight_min": -58.05833053588867,
      "activations/layer9_attention_weight_max": 46.43561935424805,
      "activations/layer9_attention_weight_min": -50.986087799072266,
      "epoch": 20.16,
      "learning_rate": 2.0181439393939394e-05,
      "loss": 2.722,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.5346,
      "eval_samples_per_second": 503.129,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.5346,
      "eval_openwebtext_samples_per_second": 503.129,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 2.0302,
      "eval_wikitext_samples_per_second": 224.607,
      "step": 347000
    },
    {
      "epoch": 20.16,
      "eval_lambada_loss": 2.3828125,
      "eval_lambada_ppl": 10.835334427562556,
      "eval_lambada_runtime": 9.6099,
      "eval_lambada_samples_per_second": 506.666,
      "step": 347000
    },
    {
      "activations/layer0_attention_weight_max": 16.054546356201172,
      "activations/layer0_attention_weight_min": -14.19949722290039,
      "activations/layer10_attention_weight_max": 34.74851608276367,
      "activations/layer10_attention_weight_min": -35.29412841796875,
      "activations/layer11_attention_weight_max": 32.977142333984375,
      "activations/layer11_attention_weight_min": -33.3873176574707,
      "activations/layer12_attention_weight_max": 17.68684196472168,
      "activations/layer12_attention_weight_min": -25.788623809814453,
      "activations/layer13_attention_weight_max": 35.728050231933594,
      "activations/layer13_attention_weight_min": -26.644479751586914,
      "activations/layer14_attention_weight_max": 34.41532897949219,
      "activations/layer14_attention_weight_min": -28.688018798828125,
      "activations/layer15_attention_weight_max": 30.83614730834961,
      "activations/layer15_attention_weight_min": -29.045969009399414,
      "activations/layer16_attention_weight_max": 33.054931640625,
      "activations/layer16_attention_weight_min": -30.653520584106445,
      "activations/layer17_attention_weight_max": 48.787010192871094,
      "activations/layer17_attention_weight_min": -43.61506271362305,
      "activations/layer18_attention_weight_max": 44.197479248046875,
      "activations/layer18_attention_weight_min": -39.89972686767578,
      "activations/layer19_attention_weight_max": 22.862247467041016,
      "activations/layer19_attention_weight_min": -23.276472091674805,
      "activations/layer1_attention_weight_max": 15.871209144592285,
      "activations/layer1_attention_weight_min": -15.013141632080078,
      "activations/layer20_attention_weight_max": 20.474687576293945,
      "activations/layer20_attention_weight_min": -22.929229736328125,
      "activations/layer21_attention_weight_max": 31.443069458007812,
      "activations/layer21_attention_weight_min": -20.313608169555664,
      "activations/layer22_attention_weight_max": 27.179325103759766,
      "activations/layer22_attention_weight_min": -26.143159866333008,
      "activations/layer23_attention_weight_max": 33.7386474609375,
      "activations/layer23_attention_weight_min": -27.467178344726562,
      "activations/layer2_attention_weight_max": 32.64145278930664,
      "activations/layer2_attention_weight_min": -32.30022048950195,
      "activations/layer3_attention_weight_max": 84.27781677246094,
      "activations/layer3_attention_weight_min": -88.39192199707031,
      "activations/layer4_attention_weight_max": 84.9724349975586,
      "activations/layer4_attention_weight_min": -83.19991302490234,
      "activations/layer5_attention_weight_max": 59.454925537109375,
      "activations/layer5_attention_weight_min": -66.72755432128906,
      "activations/layer6_attention_weight_max": 48.258079528808594,
      "activations/layer6_attention_weight_min": -50.35578918457031,
      "activations/layer7_attention_weight_max": 68.16536712646484,
      "activations/layer7_attention_weight_min": -68.24848937988281,
      "activations/layer8_attention_weight_max": 46.243064880371094,
      "activations/layer8_attention_weight_min": -51.98506546020508,
      "activations/layer9_attention_weight_max": 42.71084976196289,
      "activations/layer9_attention_weight_min": -44.65180206298828,
      "epoch": 20.17,
      "learning_rate": 2.0162499999999995e-05,
      "loss": 2.7098,
      "step": 347050
    },
    {
      "activations/layer0_attention_weight_max": 15.012430191040039,
      "activations/layer0_attention_weight_min": -13.926568031311035,
      "activations/layer10_attention_weight_max": 35.486778259277344,
      "activations/layer10_attention_weight_min": -37.9228515625,
      "activations/layer11_attention_weight_max": 32.673057556152344,
      "activations/layer11_attention_weight_min": -35.02476119995117,
      "activations/layer12_attention_weight_max": 19.07470703125,
      "activations/layer12_attention_weight_min": -26.089685440063477,
      "activations/layer13_attention_weight_max": 31.35738754272461,
      "activations/layer13_attention_weight_min": -27.773841857910156,
      "activations/layer14_attention_weight_max": 36.457122802734375,
      "activations/layer14_attention_weight_min": -31.363496780395508,
      "activations/layer15_attention_weight_max": 34.97637176513672,
      "activations/layer15_attention_weight_min": -30.478639602661133,
      "activations/layer16_attention_weight_max": 36.494876861572266,
      "activations/layer16_attention_weight_min": -32.92103576660156,
      "activations/layer17_attention_weight_max": 56.67985916137695,
      "activations/layer17_attention_weight_min": -45.77167892456055,
      "activations/layer18_attention_weight_max": 49.96965789794922,
      "activations/layer18_attention_weight_min": -41.05376052856445,
      "activations/layer19_attention_weight_max": 26.73849868774414,
      "activations/layer19_attention_weight_min": -23.940279006958008,
      "activations/layer1_attention_weight_max": 15.812638282775879,
      "activations/layer1_attention_weight_min": -12.816362380981445,
      "activations/layer20_attention_weight_max": 26.188770294189453,
      "activations/layer20_attention_weight_min": -20.885953903198242,
      "activations/layer21_attention_weight_max": 35.221519470214844,
      "activations/layer21_attention_weight_min": -21.047006607055664,
      "activations/layer22_attention_weight_max": 32.360286712646484,
      "activations/layer22_attention_weight_min": -25.55572509765625,
      "activations/layer23_attention_weight_max": 38.188209533691406,
      "activations/layer23_attention_weight_min": -24.447021484375,
      "activations/layer2_attention_weight_max": 30.253700256347656,
      "activations/layer2_attention_weight_min": -28.9344482421875,
      "activations/layer3_attention_weight_max": 85.21648406982422,
      "activations/layer3_attention_weight_min": -85.84416198730469,
      "activations/layer4_attention_weight_max": 84.92963409423828,
      "activations/layer4_attention_weight_min": -83.40977478027344,
      "activations/layer5_attention_weight_max": 62.767303466796875,
      "activations/layer5_attention_weight_min": -75.97777557373047,
      "activations/layer6_attention_weight_max": 50.4485969543457,
      "activations/layer6_attention_weight_min": -51.79140090942383,
      "activations/layer7_attention_weight_max": 67.1642837524414,
      "activations/layer7_attention_weight_min": -66.89031219482422,
      "activations/layer8_attention_weight_max": 45.94514465332031,
      "activations/layer8_attention_weight_min": -49.446983337402344,
      "activations/layer9_attention_weight_max": 43.63190460205078,
      "activations/layer9_attention_weight_min": -46.80790328979492,
      "epoch": 20.17,
      "learning_rate": 2.0143560606060604e-05,
      "loss": 2.7143,
      "step": 347100
    },
    {
      "activations/layer0_attention_weight_max": 15.50145435333252,
      "activations/layer0_attention_weight_min": -14.067753791809082,
      "activations/layer10_attention_weight_max": 38.2579345703125,
      "activations/layer10_attention_weight_min": -37.10026931762695,
      "activations/layer11_attention_weight_max": 35.375038146972656,
      "activations/layer11_attention_weight_min": -36.1720085144043,
      "activations/layer12_attention_weight_max": 18.578327178955078,
      "activations/layer12_attention_weight_min": -26.638460159301758,
      "activations/layer13_attention_weight_max": 34.922279357910156,
      "activations/layer13_attention_weight_min": -29.091150283813477,
      "activations/layer14_attention_weight_max": 34.346614837646484,
      "activations/layer14_attention_weight_min": -32.636085510253906,
      "activations/layer15_attention_weight_max": 32.07767105102539,
      "activations/layer15_attention_weight_min": -31.28874969482422,
      "activations/layer16_attention_weight_max": 33.55264663696289,
      "activations/layer16_attention_weight_min": -34.11394500732422,
      "activations/layer17_attention_weight_max": 51.01417541503906,
      "activations/layer17_attention_weight_min": -48.33209991455078,
      "activations/layer18_attention_weight_max": 47.486751556396484,
      "activations/layer18_attention_weight_min": -41.23998260498047,
      "activations/layer19_attention_weight_max": 25.673847198486328,
      "activations/layer19_attention_weight_min": -22.048574447631836,
      "activations/layer1_attention_weight_max": 15.506049156188965,
      "activations/layer1_attention_weight_min": -15.822532653808594,
      "activations/layer20_attention_weight_max": 22.646343231201172,
      "activations/layer20_attention_weight_min": -21.774465560913086,
      "activations/layer21_attention_weight_max": 36.06654357910156,
      "activations/layer21_attention_weight_min": -22.487762451171875,
      "activations/layer22_attention_weight_max": 29.06747817993164,
      "activations/layer22_attention_weight_min": -25.628639221191406,
      "activations/layer23_attention_weight_max": 34.08226776123047,
      "activations/layer23_attention_weight_min": -23.28152847290039,
      "activations/layer2_attention_weight_max": 35.206703186035156,
      "activations/layer2_attention_weight_min": -33.98841857910156,
      "activations/layer3_attention_weight_max": 88.57930755615234,
      "activations/layer3_attention_weight_min": -91.66322326660156,
      "activations/layer4_attention_weight_max": 83.18785095214844,
      "activations/layer4_attention_weight_min": -81.96101379394531,
      "activations/layer5_attention_weight_max": 59.01188278198242,
      "activations/layer5_attention_weight_min": -70.9421157836914,
      "activations/layer6_attention_weight_max": 48.12490463256836,
      "activations/layer6_attention_weight_min": -48.93058776855469,
      "activations/layer7_attention_weight_max": 64.3483657836914,
      "activations/layer7_attention_weight_min": -65.38963317871094,
      "activations/layer8_attention_weight_max": 46.300331115722656,
      "activations/layer8_attention_weight_min": -48.424583435058594,
      "activations/layer9_attention_weight_max": 45.03108215332031,
      "activations/layer9_attention_weight_min": -45.54298400878906,
      "epoch": 20.17,
      "learning_rate": 2.012462121212121e-05,
      "loss": 2.7205,
      "step": 347150
    },
    {
      "activations/layer0_attention_weight_max": 16.148210525512695,
      "activations/layer0_attention_weight_min": -14.16071891784668,
      "activations/layer10_attention_weight_max": 36.12653350830078,
      "activations/layer10_attention_weight_min": -37.15108108520508,
      "activations/layer11_attention_weight_max": 33.95859909057617,
      "activations/layer11_attention_weight_min": -34.898250579833984,
      "activations/layer12_attention_weight_max": 19.2741756439209,
      "activations/layer12_attention_weight_min": -27.38532066345215,
      "activations/layer13_attention_weight_max": 31.957138061523438,
      "activations/layer13_attention_weight_min": -27.401029586791992,
      "activations/layer14_attention_weight_max": 35.3626594543457,
      "activations/layer14_attention_weight_min": -31.173309326171875,
      "activations/layer15_attention_weight_max": 33.79608154296875,
      "activations/layer15_attention_weight_min": -31.358509063720703,
      "activations/layer16_attention_weight_max": 36.75469970703125,
      "activations/layer16_attention_weight_min": -33.42995834350586,
      "activations/layer17_attention_weight_max": 52.30818557739258,
      "activations/layer17_attention_weight_min": -48.302978515625,
      "activations/layer18_attention_weight_max": 47.97975540161133,
      "activations/layer18_attention_weight_min": -41.11601638793945,
      "activations/layer19_attention_weight_max": 24.448558807373047,
      "activations/layer19_attention_weight_min": -23.339685440063477,
      "activations/layer1_attention_weight_max": 15.570333480834961,
      "activations/layer1_attention_weight_min": -13.5891752243042,
      "activations/layer20_attention_weight_max": 23.455080032348633,
      "activations/layer20_attention_weight_min": -24.3900146484375,
      "activations/layer21_attention_weight_max": 38.533199310302734,
      "activations/layer21_attention_weight_min": -21.708477020263672,
      "activations/layer22_attention_weight_max": 31.05888557434082,
      "activations/layer22_attention_weight_min": -28.55385398864746,
      "activations/layer23_attention_weight_max": 40.77516555786133,
      "activations/layer23_attention_weight_min": -24.961193084716797,
      "activations/layer2_attention_weight_max": 31.512128829956055,
      "activations/layer2_attention_weight_min": -29.684173583984375,
      "activations/layer3_attention_weight_max": 79.12405395507812,
      "activations/layer3_attention_weight_min": -80.04029846191406,
      "activations/layer4_attention_weight_max": 80.227783203125,
      "activations/layer4_attention_weight_min": -81.08084869384766,
      "activations/layer5_attention_weight_max": 60.13810729980469,
      "activations/layer5_attention_weight_min": -75.53570556640625,
      "activations/layer6_attention_weight_max": 48.91521453857422,
      "activations/layer6_attention_weight_min": -49.034305572509766,
      "activations/layer7_attention_weight_max": 63.803916931152344,
      "activations/layer7_attention_weight_min": -62.80305480957031,
      "activations/layer8_attention_weight_max": 48.76207733154297,
      "activations/layer8_attention_weight_min": -53.25807189941406,
      "activations/layer9_attention_weight_max": 43.767391204833984,
      "activations/layer9_attention_weight_min": -45.54193878173828,
      "epoch": 20.17,
      "learning_rate": 2.0105681818181817e-05,
      "loss": 2.7096,
      "step": 347200
    },
    {
      "activations/layer0_attention_weight_max": 17.22612190246582,
      "activations/layer0_attention_weight_min": -13.731070518493652,
      "activations/layer10_attention_weight_max": 34.36101531982422,
      "activations/layer10_attention_weight_min": -35.570613861083984,
      "activations/layer11_attention_weight_max": 31.508045196533203,
      "activations/layer11_attention_weight_min": -35.091251373291016,
      "activations/layer12_attention_weight_max": 18.461626052856445,
      "activations/layer12_attention_weight_min": -27.753402709960938,
      "activations/layer13_attention_weight_max": 32.275962829589844,
      "activations/layer13_attention_weight_min": -27.235980987548828,
      "activations/layer14_attention_weight_max": 34.18118667602539,
      "activations/layer14_attention_weight_min": -28.927566528320312,
      "activations/layer15_attention_weight_max": 30.580482482910156,
      "activations/layer15_attention_weight_min": -31.15837860107422,
      "activations/layer16_attention_weight_max": 32.96969985961914,
      "activations/layer16_attention_weight_min": -30.35820198059082,
      "activations/layer17_attention_weight_max": 47.184173583984375,
      "activations/layer17_attention_weight_min": -44.09734344482422,
      "activations/layer18_attention_weight_max": 44.651546478271484,
      "activations/layer18_attention_weight_min": -42.064491271972656,
      "activations/layer19_attention_weight_max": 24.713483810424805,
      "activations/layer19_attention_weight_min": -26.404993057250977,
      "activations/layer1_attention_weight_max": 15.391608238220215,
      "activations/layer1_attention_weight_min": -13.794212341308594,
      "activations/layer20_attention_weight_max": 22.43352508544922,
      "activations/layer20_attention_weight_min": -26.426494598388672,
      "activations/layer21_attention_weight_max": 32.50970458984375,
      "activations/layer21_attention_weight_min": -26.392972946166992,
      "activations/layer22_attention_weight_max": 29.05352783203125,
      "activations/layer22_attention_weight_min": -27.713363647460938,
      "activations/layer23_attention_weight_max": 35.30055236816406,
      "activations/layer23_attention_weight_min": -25.32709312438965,
      "activations/layer2_attention_weight_max": 32.99749755859375,
      "activations/layer2_attention_weight_min": -30.224681854248047,
      "activations/layer3_attention_weight_max": 85.86231994628906,
      "activations/layer3_attention_weight_min": -85.5627670288086,
      "activations/layer4_attention_weight_max": 80.92373657226562,
      "activations/layer4_attention_weight_min": -83.15608215332031,
      "activations/layer5_attention_weight_max": 62.212501525878906,
      "activations/layer5_attention_weight_min": -72.87812805175781,
      "activations/layer6_attention_weight_max": 46.94295883178711,
      "activations/layer6_attention_weight_min": -48.30840301513672,
      "activations/layer7_attention_weight_max": 61.869869232177734,
      "activations/layer7_attention_weight_min": -62.66924285888672,
      "activations/layer8_attention_weight_max": 44.645137786865234,
      "activations/layer8_attention_weight_min": -52.458900451660156,
      "activations/layer9_attention_weight_max": 43.6265983581543,
      "activations/layer9_attention_weight_min": -45.154563903808594,
      "epoch": 20.18,
      "learning_rate": 2.0086742424242422e-05,
      "loss": 2.7271,
      "step": 347250
    },
    {
      "activations/layer0_attention_weight_max": 15.469941139221191,
      "activations/layer0_attention_weight_min": -14.06664752960205,
      "activations/layer10_attention_weight_max": 40.215824127197266,
      "activations/layer10_attention_weight_min": -39.58498001098633,
      "activations/layer11_attention_weight_max": 36.250335693359375,
      "activations/layer11_attention_weight_min": -35.58380126953125,
      "activations/layer12_attention_weight_max": 20.308609008789062,
      "activations/layer12_attention_weight_min": -24.22173500061035,
      "activations/layer13_attention_weight_max": 43.99210739135742,
      "activations/layer13_attention_weight_min": -31.736774444580078,
      "activations/layer14_attention_weight_max": 42.76293182373047,
      "activations/layer14_attention_weight_min": -32.55599594116211,
      "activations/layer15_attention_weight_max": 43.30513381958008,
      "activations/layer15_attention_weight_min": -30.543746948242188,
      "activations/layer16_attention_weight_max": 37.335845947265625,
      "activations/layer16_attention_weight_min": -34.552391052246094,
      "activations/layer17_attention_weight_max": 55.77070236206055,
      "activations/layer17_attention_weight_min": -49.374267578125,
      "activations/layer18_attention_weight_max": 52.05259323120117,
      "activations/layer18_attention_weight_min": -41.858524322509766,
      "activations/layer19_attention_weight_max": 27.8753719329834,
      "activations/layer19_attention_weight_min": -25.10554313659668,
      "activations/layer1_attention_weight_max": 14.695240020751953,
      "activations/layer1_attention_weight_min": -14.510807037353516,
      "activations/layer20_attention_weight_max": 24.47283363342285,
      "activations/layer20_attention_weight_min": -24.517370223999023,
      "activations/layer21_attention_weight_max": 41.7567138671875,
      "activations/layer21_attention_weight_min": -27.55015754699707,
      "activations/layer22_attention_weight_max": 32.42884826660156,
      "activations/layer22_attention_weight_min": -29.142927169799805,
      "activations/layer23_attention_weight_max": 37.3139762878418,
      "activations/layer23_attention_weight_min": -25.41499137878418,
      "activations/layer2_attention_weight_max": 31.384708404541016,
      "activations/layer2_attention_weight_min": -31.8968505859375,
      "activations/layer3_attention_weight_max": 86.20923614501953,
      "activations/layer3_attention_weight_min": -83.9687728881836,
      "activations/layer4_attention_weight_max": 83.53031921386719,
      "activations/layer4_attention_weight_min": -84.79589080810547,
      "activations/layer5_attention_weight_max": 62.519073486328125,
      "activations/layer5_attention_weight_min": -70.15058898925781,
      "activations/layer6_attention_weight_max": 50.582515716552734,
      "activations/layer6_attention_weight_min": -50.74839782714844,
      "activations/layer7_attention_weight_max": 69.00228118896484,
      "activations/layer7_attention_weight_min": -67.94609069824219,
      "activations/layer8_attention_weight_max": 54.09904098510742,
      "activations/layer8_attention_weight_min": -53.13524627685547,
      "activations/layer9_attention_weight_max": 57.183712005615234,
      "activations/layer9_attention_weight_min": -51.76514434814453,
      "epoch": 20.18,
      "learning_rate": 2.006780303030303e-05,
      "loss": 2.7175,
      "step": 347300
    },
    {
      "activations/layer0_attention_weight_max": 15.83169174194336,
      "activations/layer0_attention_weight_min": -13.879866600036621,
      "activations/layer10_attention_weight_max": 34.2831916809082,
      "activations/layer10_attention_weight_min": -33.921630859375,
      "activations/layer11_attention_weight_max": 32.34693908691406,
      "activations/layer11_attention_weight_min": -33.97795104980469,
      "activations/layer12_attention_weight_max": 21.42642593383789,
      "activations/layer12_attention_weight_min": -27.29629135131836,
      "activations/layer13_attention_weight_max": 36.64613342285156,
      "activations/layer13_attention_weight_min": -28.65193748474121,
      "activations/layer14_attention_weight_max": 33.9787712097168,
      "activations/layer14_attention_weight_min": -31.10588836669922,
      "activations/layer15_attention_weight_max": 31.74306297302246,
      "activations/layer15_attention_weight_min": -30.686704635620117,
      "activations/layer16_attention_weight_max": 32.48432159423828,
      "activations/layer16_attention_weight_min": -31.341833114624023,
      "activations/layer17_attention_weight_max": 48.255943298339844,
      "activations/layer17_attention_weight_min": -43.18748474121094,
      "activations/layer18_attention_weight_max": 44.90363693237305,
      "activations/layer18_attention_weight_min": -38.46664810180664,
      "activations/layer19_attention_weight_max": 23.666183471679688,
      "activations/layer19_attention_weight_min": -21.27460479736328,
      "activations/layer1_attention_weight_max": 15.317420959472656,
      "activations/layer1_attention_weight_min": -12.832072257995605,
      "activations/layer20_attention_weight_max": 21.025455474853516,
      "activations/layer20_attention_weight_min": -20.67411994934082,
      "activations/layer21_attention_weight_max": 36.523468017578125,
      "activations/layer21_attention_weight_min": -25.549949645996094,
      "activations/layer22_attention_weight_max": 28.120344161987305,
      "activations/layer22_attention_weight_min": -26.5726261138916,
      "activations/layer23_attention_weight_max": 33.59834289550781,
      "activations/layer23_attention_weight_min": -28.239269256591797,
      "activations/layer2_attention_weight_max": 29.18793296813965,
      "activations/layer2_attention_weight_min": -28.21169090270996,
      "activations/layer3_attention_weight_max": 80.743408203125,
      "activations/layer3_attention_weight_min": -80.27828979492188,
      "activations/layer4_attention_weight_max": 79.2202377319336,
      "activations/layer4_attention_weight_min": -79.6688461303711,
      "activations/layer5_attention_weight_max": 61.76643371582031,
      "activations/layer5_attention_weight_min": -69.96912384033203,
      "activations/layer6_attention_weight_max": 48.296321868896484,
      "activations/layer6_attention_weight_min": -50.8708381652832,
      "activations/layer7_attention_weight_max": 70.19725036621094,
      "activations/layer7_attention_weight_min": -62.320552825927734,
      "activations/layer8_attention_weight_max": 47.1132698059082,
      "activations/layer8_attention_weight_min": -49.121681213378906,
      "activations/layer9_attention_weight_max": 41.73653793334961,
      "activations/layer9_attention_weight_min": -44.02900314331055,
      "epoch": 20.18,
      "learning_rate": 2.0048863636363635e-05,
      "loss": 2.721,
      "step": 347350
    },
    {
      "activations/layer0_attention_weight_max": 16.114227294921875,
      "activations/layer0_attention_weight_min": -13.850826263427734,
      "activations/layer10_attention_weight_max": 43.58219528198242,
      "activations/layer10_attention_weight_min": -37.90334701538086,
      "activations/layer11_attention_weight_max": 39.849891662597656,
      "activations/layer11_attention_weight_min": -35.97686767578125,
      "activations/layer12_attention_weight_max": 20.106592178344727,
      "activations/layer12_attention_weight_min": -23.209232330322266,
      "activations/layer13_attention_weight_max": 35.98708724975586,
      "activations/layer13_attention_weight_min": -29.765625,
      "activations/layer14_attention_weight_max": 38.973838806152344,
      "activations/layer14_attention_weight_min": -31.48240089416504,
      "activations/layer15_attention_weight_max": 35.91791534423828,
      "activations/layer15_attention_weight_min": -31.5278263092041,
      "activations/layer16_attention_weight_max": 40.001914978027344,
      "activations/layer16_attention_weight_min": -36.822750091552734,
      "activations/layer17_attention_weight_max": 57.56126022338867,
      "activations/layer17_attention_weight_min": -49.79401397705078,
      "activations/layer18_attention_weight_max": 51.21649932861328,
      "activations/layer18_attention_weight_min": -43.346858978271484,
      "activations/layer19_attention_weight_max": 25.873233795166016,
      "activations/layer19_attention_weight_min": -25.059080123901367,
      "activations/layer1_attention_weight_max": 14.856801986694336,
      "activations/layer1_attention_weight_min": -13.727121353149414,
      "activations/layer20_attention_weight_max": 26.570573806762695,
      "activations/layer20_attention_weight_min": -24.6003475189209,
      "activations/layer21_attention_weight_max": 37.034324645996094,
      "activations/layer21_attention_weight_min": -24.472532272338867,
      "activations/layer22_attention_weight_max": 29.32699203491211,
      "activations/layer22_attention_weight_min": -27.43707275390625,
      "activations/layer23_attention_weight_max": 36.001502990722656,
      "activations/layer23_attention_weight_min": -23.53650665283203,
      "activations/layer2_attention_weight_max": 34.15193557739258,
      "activations/layer2_attention_weight_min": -32.279014587402344,
      "activations/layer3_attention_weight_max": 90.7991714477539,
      "activations/layer3_attention_weight_min": -95.26568603515625,
      "activations/layer4_attention_weight_max": 85.41065216064453,
      "activations/layer4_attention_weight_min": -85.5632095336914,
      "activations/layer5_attention_weight_max": 65.54419708251953,
      "activations/layer5_attention_weight_min": -70.13571166992188,
      "activations/layer6_attention_weight_max": 50.89707946777344,
      "activations/layer6_attention_weight_min": -52.66877746582031,
      "activations/layer7_attention_weight_max": 64.91015625,
      "activations/layer7_attention_weight_min": -66.05784606933594,
      "activations/layer8_attention_weight_max": 52.46141052246094,
      "activations/layer8_attention_weight_min": -51.6605110168457,
      "activations/layer9_attention_weight_max": 53.903316497802734,
      "activations/layer9_attention_weight_min": -47.66312789916992,
      "epoch": 20.19,
      "learning_rate": 2.002992424242424e-05,
      "loss": 2.714,
      "step": 347400
    },
    {
      "activations/layer0_attention_weight_max": 15.340448379516602,
      "activations/layer0_attention_weight_min": -13.712874412536621,
      "activations/layer10_attention_weight_max": 36.694244384765625,
      "activations/layer10_attention_weight_min": -36.769989013671875,
      "activations/layer11_attention_weight_max": 34.44487380981445,
      "activations/layer11_attention_weight_min": -35.1611213684082,
      "activations/layer12_attention_weight_max": 18.754941940307617,
      "activations/layer12_attention_weight_min": -22.474679946899414,
      "activations/layer13_attention_weight_max": 30.79865264892578,
      "activations/layer13_attention_weight_min": -26.473087310791016,
      "activations/layer14_attention_weight_max": 38.684017181396484,
      "activations/layer14_attention_weight_min": -32.46240997314453,
      "activations/layer15_attention_weight_max": 33.76634979248047,
      "activations/layer15_attention_weight_min": -32.327789306640625,
      "activations/layer16_attention_weight_max": 34.94715881347656,
      "activations/layer16_attention_weight_min": -33.280906677246094,
      "activations/layer17_attention_weight_max": 52.42139434814453,
      "activations/layer17_attention_weight_min": -45.17177200317383,
      "activations/layer18_attention_weight_max": 49.208621978759766,
      "activations/layer18_attention_weight_min": -39.52741241455078,
      "activations/layer19_attention_weight_max": 24.17439079284668,
      "activations/layer19_attention_weight_min": -23.67302703857422,
      "activations/layer1_attention_weight_max": 15.157709121704102,
      "activations/layer1_attention_weight_min": -11.938868522644043,
      "activations/layer20_attention_weight_max": 22.574542999267578,
      "activations/layer20_attention_weight_min": -22.952470779418945,
      "activations/layer21_attention_weight_max": 35.286834716796875,
      "activations/layer21_attention_weight_min": -23.30814552307129,
      "activations/layer22_attention_weight_max": 30.46523094177246,
      "activations/layer22_attention_weight_min": -24.486440658569336,
      "activations/layer23_attention_weight_max": 39.91267776489258,
      "activations/layer23_attention_weight_min": -23.257919311523438,
      "activations/layer2_attention_weight_max": 29.873249053955078,
      "activations/layer2_attention_weight_min": -27.899490356445312,
      "activations/layer3_attention_weight_max": 82.18443298339844,
      "activations/layer3_attention_weight_min": -82.92224884033203,
      "activations/layer4_attention_weight_max": 81.04359436035156,
      "activations/layer4_attention_weight_min": -85.75965118408203,
      "activations/layer5_attention_weight_max": 63.96607208251953,
      "activations/layer5_attention_weight_min": -71.69390106201172,
      "activations/layer6_attention_weight_max": 49.74711990356445,
      "activations/layer6_attention_weight_min": -50.51077651977539,
      "activations/layer7_attention_weight_max": 68.65955352783203,
      "activations/layer7_attention_weight_min": -64.36940002441406,
      "activations/layer8_attention_weight_max": 49.70106506347656,
      "activations/layer8_attention_weight_min": -52.173057556152344,
      "activations/layer9_attention_weight_max": 44.186431884765625,
      "activations/layer9_attention_weight_min": -46.069942474365234,
      "epoch": 20.19,
      "learning_rate": 2.0010984848484848e-05,
      "loss": 2.7198,
      "step": 347450
    },
    {
      "activations/layer0_attention_weight_max": 15.044066429138184,
      "activations/layer0_attention_weight_min": -13.771867752075195,
      "activations/layer10_attention_weight_max": 38.579132080078125,
      "activations/layer10_attention_weight_min": -37.811729431152344,
      "activations/layer11_attention_weight_max": 32.968990325927734,
      "activations/layer11_attention_weight_min": -36.16850280761719,
      "activations/layer12_attention_weight_max": 19.677169799804688,
      "activations/layer12_attention_weight_min": -25.33824920654297,
      "activations/layer13_attention_weight_max": 33.008480072021484,
      "activations/layer13_attention_weight_min": -28.090517044067383,
      "activations/layer14_attention_weight_max": 37.615013122558594,
      "activations/layer14_attention_weight_min": -29.97721290588379,
      "activations/layer15_attention_weight_max": 36.18635940551758,
      "activations/layer15_attention_weight_min": -30.340587615966797,
      "activations/layer16_attention_weight_max": 37.70444107055664,
      "activations/layer16_attention_weight_min": -34.52067565917969,
      "activations/layer17_attention_weight_max": 55.65394592285156,
      "activations/layer17_attention_weight_min": -48.74857711791992,
      "activations/layer18_attention_weight_max": 49.4194450378418,
      "activations/layer18_attention_weight_min": -44.86524963378906,
      "activations/layer19_attention_weight_max": 22.07480812072754,
      "activations/layer19_attention_weight_min": -23.81496238708496,
      "activations/layer1_attention_weight_max": 15.211196899414062,
      "activations/layer1_attention_weight_min": -14.235835075378418,
      "activations/layer20_attention_weight_max": 22.3400936126709,
      "activations/layer20_attention_weight_min": -21.31559944152832,
      "activations/layer21_attention_weight_max": 34.34669876098633,
      "activations/layer21_attention_weight_min": -20.472583770751953,
      "activations/layer22_attention_weight_max": 26.38359260559082,
      "activations/layer22_attention_weight_min": -23.98790168762207,
      "activations/layer23_attention_weight_max": 35.95600891113281,
      "activations/layer23_attention_weight_min": -22.08155632019043,
      "activations/layer2_attention_weight_max": 31.9979305267334,
      "activations/layer2_attention_weight_min": -30.00588035583496,
      "activations/layer3_attention_weight_max": 82.99838256835938,
      "activations/layer3_attention_weight_min": -83.99571228027344,
      "activations/layer4_attention_weight_max": 79.90011596679688,
      "activations/layer4_attention_weight_min": -78.92977905273438,
      "activations/layer5_attention_weight_max": 61.019866943359375,
      "activations/layer5_attention_weight_min": -68.41612243652344,
      "activations/layer6_attention_weight_max": 48.53196716308594,
      "activations/layer6_attention_weight_min": -48.911746978759766,
      "activations/layer7_attention_weight_max": 64.95478057861328,
      "activations/layer7_attention_weight_min": -62.81071853637695,
      "activations/layer8_attention_weight_max": 48.323516845703125,
      "activations/layer8_attention_weight_min": -53.04786682128906,
      "activations/layer9_attention_weight_max": 45.94670104980469,
      "activations/layer9_attention_weight_min": -46.21822738647461,
      "epoch": 20.19,
      "learning_rate": 1.9992045454545453e-05,
      "loss": 2.7126,
      "step": 347500
    },
    {
      "activations/layer0_attention_weight_max": 17.17597770690918,
      "activations/layer0_attention_weight_min": -14.088956832885742,
      "activations/layer10_attention_weight_max": 34.51093673706055,
      "activations/layer10_attention_weight_min": -33.870033264160156,
      "activations/layer11_attention_weight_max": 33.66722869873047,
      "activations/layer11_attention_weight_min": -32.454925537109375,
      "activations/layer12_attention_weight_max": 18.189115524291992,
      "activations/layer12_attention_weight_min": -25.27014923095703,
      "activations/layer13_attention_weight_max": 33.749183654785156,
      "activations/layer13_attention_weight_min": -26.66700553894043,
      "activations/layer14_attention_weight_max": 34.900001525878906,
      "activations/layer14_attention_weight_min": -29.07147789001465,
      "activations/layer15_attention_weight_max": 32.656394958496094,
      "activations/layer15_attention_weight_min": -29.366294860839844,
      "activations/layer16_attention_weight_max": 34.45584487915039,
      "activations/layer16_attention_weight_min": -34.31998825073242,
      "activations/layer17_attention_weight_max": 50.97806930541992,
      "activations/layer17_attention_weight_min": -44.39521026611328,
      "activations/layer18_attention_weight_max": 46.97221755981445,
      "activations/layer18_attention_weight_min": -39.438289642333984,
      "activations/layer19_attention_weight_max": 25.1994571685791,
      "activations/layer19_attention_weight_min": -21.166433334350586,
      "activations/layer1_attention_weight_max": 16.094823837280273,
      "activations/layer1_attention_weight_min": -12.331071853637695,
      "activations/layer20_attention_weight_max": 23.495412826538086,
      "activations/layer20_attention_weight_min": -20.619918823242188,
      "activations/layer21_attention_weight_max": 34.583106994628906,
      "activations/layer21_attention_weight_min": -24.600828170776367,
      "activations/layer22_attention_weight_max": 28.894332885742188,
      "activations/layer22_attention_weight_min": -26.539812088012695,
      "activations/layer23_attention_weight_max": 36.29742431640625,
      "activations/layer23_attention_weight_min": -22.641658782958984,
      "activations/layer2_attention_weight_max": 29.700775146484375,
      "activations/layer2_attention_weight_min": -27.668045043945312,
      "activations/layer3_attention_weight_max": 78.93656158447266,
      "activations/layer3_attention_weight_min": -84.3102035522461,
      "activations/layer4_attention_weight_max": 76.62512969970703,
      "activations/layer4_attention_weight_min": -80.00179290771484,
      "activations/layer5_attention_weight_max": 56.93482971191406,
      "activations/layer5_attention_weight_min": -71.85594177246094,
      "activations/layer6_attention_weight_max": 46.40131378173828,
      "activations/layer6_attention_weight_min": -45.60715103149414,
      "activations/layer7_attention_weight_max": 60.434791564941406,
      "activations/layer7_attention_weight_min": -62.513206481933594,
      "activations/layer8_attention_weight_max": 45.68101501464844,
      "activations/layer8_attention_weight_min": -47.60070037841797,
      "activations/layer9_attention_weight_max": 42.250152587890625,
      "activations/layer9_attention_weight_min": -43.02216720581055,
      "epoch": 20.19,
      "learning_rate": 1.997310606060606e-05,
      "loss": 2.7315,
      "step": 347550
    },
    {
      "activations/layer0_attention_weight_max": 15.678820610046387,
      "activations/layer0_attention_weight_min": -13.707695007324219,
      "activations/layer10_attention_weight_max": 37.289833068847656,
      "activations/layer10_attention_weight_min": -36.6020393371582,
      "activations/layer11_attention_weight_max": 32.36406326293945,
      "activations/layer11_attention_weight_min": -35.57673645019531,
      "activations/layer12_attention_weight_max": 19.897104263305664,
      "activations/layer12_attention_weight_min": -23.946460723876953,
      "activations/layer13_attention_weight_max": 33.04372024536133,
      "activations/layer13_attention_weight_min": -27.602663040161133,
      "activations/layer14_attention_weight_max": 34.747344970703125,
      "activations/layer14_attention_weight_min": -31.55862808227539,
      "activations/layer15_attention_weight_max": 33.818565368652344,
      "activations/layer15_attention_weight_min": -31.355722427368164,
      "activations/layer16_attention_weight_max": 32.72196578979492,
      "activations/layer16_attention_weight_min": -32.53510665893555,
      "activations/layer17_attention_weight_max": 50.31558609008789,
      "activations/layer17_attention_weight_min": -44.37928009033203,
      "activations/layer18_attention_weight_max": 44.77627944946289,
      "activations/layer18_attention_weight_min": -40.15423583984375,
      "activations/layer19_attention_weight_max": 23.075374603271484,
      "activations/layer19_attention_weight_min": -24.317052841186523,
      "activations/layer1_attention_weight_max": 15.112615585327148,
      "activations/layer1_attention_weight_min": -12.957244873046875,
      "activations/layer20_attention_weight_max": 22.740541458129883,
      "activations/layer20_attention_weight_min": -23.19308090209961,
      "activations/layer21_attention_weight_max": 32.30260467529297,
      "activations/layer21_attention_weight_min": -23.07558250427246,
      "activations/layer22_attention_weight_max": 28.54256248474121,
      "activations/layer22_attention_weight_min": -25.663118362426758,
      "activations/layer23_attention_weight_max": 35.65062713623047,
      "activations/layer23_attention_weight_min": -24.905780792236328,
      "activations/layer2_attention_weight_max": 31.012388229370117,
      "activations/layer2_attention_weight_min": -28.27301597595215,
      "activations/layer3_attention_weight_max": 86.36749267578125,
      "activations/layer3_attention_weight_min": -81.27095794677734,
      "activations/layer4_attention_weight_max": 80.19799041748047,
      "activations/layer4_attention_weight_min": -79.5477523803711,
      "activations/layer5_attention_weight_max": 60.47045135498047,
      "activations/layer5_attention_weight_min": -66.57978057861328,
      "activations/layer6_attention_weight_max": 46.982418060302734,
      "activations/layer6_attention_weight_min": -47.585655212402344,
      "activations/layer7_attention_weight_max": 62.73371505737305,
      "activations/layer7_attention_weight_min": -61.71451950073242,
      "activations/layer8_attention_weight_max": 46.554771423339844,
      "activations/layer8_attention_weight_min": -49.00433349609375,
      "activations/layer9_attention_weight_max": 42.003990173339844,
      "activations/layer9_attention_weight_min": -44.48383331298828,
      "epoch": 20.2,
      "learning_rate": 1.9954166666666663e-05,
      "loss": 2.7315,
      "step": 347600
    },
    {
      "activations/layer0_attention_weight_max": 15.962347984313965,
      "activations/layer0_attention_weight_min": -14.77763843536377,
      "activations/layer10_attention_weight_max": 38.577919006347656,
      "activations/layer10_attention_weight_min": -37.147865295410156,
      "activations/layer11_attention_weight_max": 33.90193176269531,
      "activations/layer11_attention_weight_min": -35.696266174316406,
      "activations/layer12_attention_weight_max": 20.89567756652832,
      "activations/layer12_attention_weight_min": -24.966896057128906,
      "activations/layer13_attention_weight_max": 39.405372619628906,
      "activations/layer13_attention_weight_min": -27.652345657348633,
      "activations/layer14_attention_weight_max": 37.57535171508789,
      "activations/layer14_attention_weight_min": -32.948646545410156,
      "activations/layer15_attention_weight_max": 35.86522674560547,
      "activations/layer15_attention_weight_min": -31.52625274658203,
      "activations/layer16_attention_weight_max": 36.62137985229492,
      "activations/layer16_attention_weight_min": -35.5636100769043,
      "activations/layer17_attention_weight_max": 53.91703414916992,
      "activations/layer17_attention_weight_min": -50.4448356628418,
      "activations/layer18_attention_weight_max": 49.93975830078125,
      "activations/layer18_attention_weight_min": -46.03311538696289,
      "activations/layer19_attention_weight_max": 26.937728881835938,
      "activations/layer19_attention_weight_min": -26.00359535217285,
      "activations/layer1_attention_weight_max": 15.749517440795898,
      "activations/layer1_attention_weight_min": -12.974340438842773,
      "activations/layer20_attention_weight_max": 26.381694793701172,
      "activations/layer20_attention_weight_min": -24.250633239746094,
      "activations/layer21_attention_weight_max": 38.87331008911133,
      "activations/layer21_attention_weight_min": -24.25799560546875,
      "activations/layer22_attention_weight_max": 31.479307174682617,
      "activations/layer22_attention_weight_min": -29.44643783569336,
      "activations/layer23_attention_weight_max": 40.09417724609375,
      "activations/layer23_attention_weight_min": -26.8344783782959,
      "activations/layer2_attention_weight_max": 30.066587448120117,
      "activations/layer2_attention_weight_min": -28.46695327758789,
      "activations/layer3_attention_weight_max": 81.37230682373047,
      "activations/layer3_attention_weight_min": -82.81346130371094,
      "activations/layer4_attention_weight_max": 80.99147033691406,
      "activations/layer4_attention_weight_min": -79.30648803710938,
      "activations/layer5_attention_weight_max": 59.08618927001953,
      "activations/layer5_attention_weight_min": -73.3156967163086,
      "activations/layer6_attention_weight_max": 49.609554290771484,
      "activations/layer6_attention_weight_min": -50.02084732055664,
      "activations/layer7_attention_weight_max": 66.03634643554688,
      "activations/layer7_attention_weight_min": -67.7499008178711,
      "activations/layer8_attention_weight_max": 47.740962982177734,
      "activations/layer8_attention_weight_min": -52.28776168823242,
      "activations/layer9_attention_weight_max": 45.3087043762207,
      "activations/layer9_attention_weight_min": -48.40382766723633,
      "epoch": 20.2,
      "learning_rate": 1.993560606060606e-05,
      "loss": 2.726,
      "step": 347650
    },
    {
      "activations/layer0_attention_weight_max": 15.498905181884766,
      "activations/layer0_attention_weight_min": -13.766772270202637,
      "activations/layer10_attention_weight_max": 34.618858337402344,
      "activations/layer10_attention_weight_min": -35.48750686645508,
      "activations/layer11_attention_weight_max": 34.04883575439453,
      "activations/layer11_attention_weight_min": -36.94572067260742,
      "activations/layer12_attention_weight_max": 18.706357955932617,
      "activations/layer12_attention_weight_min": -26.98802375793457,
      "activations/layer13_attention_weight_max": 30.872936248779297,
      "activations/layer13_attention_weight_min": -26.58437156677246,
      "activations/layer14_attention_weight_max": 32.944549560546875,
      "activations/layer14_attention_weight_min": -30.20178985595703,
      "activations/layer15_attention_weight_max": 33.74625778198242,
      "activations/layer15_attention_weight_min": -30.87477684020996,
      "activations/layer16_attention_weight_max": 32.14045715332031,
      "activations/layer16_attention_weight_min": -31.259965896606445,
      "activations/layer17_attention_weight_max": 48.475860595703125,
      "activations/layer17_attention_weight_min": -44.835784912109375,
      "activations/layer18_attention_weight_max": 46.4050407409668,
      "activations/layer18_attention_weight_min": -42.78775405883789,
      "activations/layer19_attention_weight_max": 22.893863677978516,
      "activations/layer19_attention_weight_min": -25.27444076538086,
      "activations/layer1_attention_weight_max": 17.015583038330078,
      "activations/layer1_attention_weight_min": -12.655719757080078,
      "activations/layer20_attention_weight_max": 21.992734909057617,
      "activations/layer20_attention_weight_min": -21.633953094482422,
      "activations/layer21_attention_weight_max": 31.180381774902344,
      "activations/layer21_attention_weight_min": -21.444597244262695,
      "activations/layer22_attention_weight_max": 31.203144073486328,
      "activations/layer22_attention_weight_min": -26.06029510498047,
      "activations/layer23_attention_weight_max": 38.06162643432617,
      "activations/layer23_attention_weight_min": -24.409378051757812,
      "activations/layer2_attention_weight_max": 29.842275619506836,
      "activations/layer2_attention_weight_min": -28.91456413269043,
      "activations/layer3_attention_weight_max": 81.74180603027344,
      "activations/layer3_attention_weight_min": -85.41022491455078,
      "activations/layer4_attention_weight_max": 81.40758514404297,
      "activations/layer4_attention_weight_min": -79.89775085449219,
      "activations/layer5_attention_weight_max": 61.548885345458984,
      "activations/layer5_attention_weight_min": -68.94853210449219,
      "activations/layer6_attention_weight_max": 47.56499099731445,
      "activations/layer6_attention_weight_min": -49.003257751464844,
      "activations/layer7_attention_weight_max": 64.12870025634766,
      "activations/layer7_attention_weight_min": -61.82025146484375,
      "activations/layer8_attention_weight_max": 44.94537353515625,
      "activations/layer8_attention_weight_min": -48.82767105102539,
      "activations/layer9_attention_weight_max": 42.957035064697266,
      "activations/layer9_attention_weight_min": -45.13964080810547,
      "epoch": 20.2,
      "learning_rate": 1.9916666666666665e-05,
      "loss": 2.7066,
      "step": 347700
    },
    {
      "activations/layer0_attention_weight_max": 15.832188606262207,
      "activations/layer0_attention_weight_min": -13.895153045654297,
      "activations/layer10_attention_weight_max": 35.17314910888672,
      "activations/layer10_attention_weight_min": -37.040462493896484,
      "activations/layer11_attention_weight_max": 32.96258544921875,
      "activations/layer11_attention_weight_min": -36.665443420410156,
      "activations/layer12_attention_weight_max": 18.50698471069336,
      "activations/layer12_attention_weight_min": -26.021730422973633,
      "activations/layer13_attention_weight_max": 38.25978469848633,
      "activations/layer13_attention_weight_min": -29.215639114379883,
      "activations/layer14_attention_weight_max": 38.72404479980469,
      "activations/layer14_attention_weight_min": -28.31785011291504,
      "activations/layer15_attention_weight_max": 35.66887283325195,
      "activations/layer15_attention_weight_min": -29.811857223510742,
      "activations/layer16_attention_weight_max": 34.03872299194336,
      "activations/layer16_attention_weight_min": -31.38494110107422,
      "activations/layer17_attention_weight_max": 53.153419494628906,
      "activations/layer17_attention_weight_min": -45.07021713256836,
      "activations/layer18_attention_weight_max": 49.09819793701172,
      "activations/layer18_attention_weight_min": -38.81729507446289,
      "activations/layer19_attention_weight_max": 24.925796508789062,
      "activations/layer19_attention_weight_min": -21.777938842773438,
      "activations/layer1_attention_weight_max": 15.823987007141113,
      "activations/layer1_attention_weight_min": -15.862959861755371,
      "activations/layer20_attention_weight_max": 23.905363082885742,
      "activations/layer20_attention_weight_min": -24.205554962158203,
      "activations/layer21_attention_weight_max": 36.207008361816406,
      "activations/layer21_attention_weight_min": -22.681846618652344,
      "activations/layer22_attention_weight_max": 29.10160255432129,
      "activations/layer22_attention_weight_min": -25.612825393676758,
      "activations/layer23_attention_weight_max": 36.64098358154297,
      "activations/layer23_attention_weight_min": -27.739288330078125,
      "activations/layer2_attention_weight_max": 31.242557525634766,
      "activations/layer2_attention_weight_min": -29.453815460205078,
      "activations/layer3_attention_weight_max": 86.08562469482422,
      "activations/layer3_attention_weight_min": -80.65966033935547,
      "activations/layer4_attention_weight_max": 84.3841781616211,
      "activations/layer4_attention_weight_min": -80.99374389648438,
      "activations/layer5_attention_weight_max": 61.03413009643555,
      "activations/layer5_attention_weight_min": -71.29197692871094,
      "activations/layer6_attention_weight_max": 49.35443878173828,
      "activations/layer6_attention_weight_min": -49.20402526855469,
      "activations/layer7_attention_weight_max": 63.424434661865234,
      "activations/layer7_attention_weight_min": -65.64070129394531,
      "activations/layer8_attention_weight_max": 47.03260040283203,
      "activations/layer8_attention_weight_min": -52.10930252075195,
      "activations/layer9_attention_weight_max": 46.28158950805664,
      "activations/layer9_attention_weight_min": -47.93257522583008,
      "epoch": 20.21,
      "learning_rate": 1.9897727272727273e-05,
      "loss": 2.7287,
      "step": 347750
    },
    {
      "activations/layer0_attention_weight_max": 15.021900177001953,
      "activations/layer0_attention_weight_min": -14.02873420715332,
      "activations/layer10_attention_weight_max": 36.594425201416016,
      "activations/layer10_attention_weight_min": -34.87322998046875,
      "activations/layer11_attention_weight_max": 32.814239501953125,
      "activations/layer11_attention_weight_min": -34.795494079589844,
      "activations/layer12_attention_weight_max": 18.801366806030273,
      "activations/layer12_attention_weight_min": -26.939762115478516,
      "activations/layer13_attention_weight_max": 34.247276306152344,
      "activations/layer13_attention_weight_min": -29.430408477783203,
      "activations/layer14_attention_weight_max": 34.47597885131836,
      "activations/layer14_attention_weight_min": -31.021255493164062,
      "activations/layer15_attention_weight_max": 32.860679626464844,
      "activations/layer15_attention_weight_min": -32.46053695678711,
      "activations/layer16_attention_weight_max": 35.479347229003906,
      "activations/layer16_attention_weight_min": -33.57975769042969,
      "activations/layer17_attention_weight_max": 50.8082275390625,
      "activations/layer17_attention_weight_min": -46.00700759887695,
      "activations/layer18_attention_weight_max": 45.19855880737305,
      "activations/layer18_attention_weight_min": -42.000240325927734,
      "activations/layer19_attention_weight_max": 25.599449157714844,
      "activations/layer19_attention_weight_min": -21.592220306396484,
      "activations/layer1_attention_weight_max": 16.401994705200195,
      "activations/layer1_attention_weight_min": -13.64693546295166,
      "activations/layer20_attention_weight_max": 25.71002769470215,
      "activations/layer20_attention_weight_min": -21.189481735229492,
      "activations/layer21_attention_weight_max": 40.08018112182617,
      "activations/layer21_attention_weight_min": -24.737802505493164,
      "activations/layer22_attention_weight_max": 30.672380447387695,
      "activations/layer22_attention_weight_min": -24.152851104736328,
      "activations/layer23_attention_weight_max": 36.59587860107422,
      "activations/layer23_attention_weight_min": -21.85651206970215,
      "activations/layer2_attention_weight_max": 31.595169067382812,
      "activations/layer2_attention_weight_min": -29.37653160095215,
      "activations/layer3_attention_weight_max": 82.59125518798828,
      "activations/layer3_attention_weight_min": -82.88404083251953,
      "activations/layer4_attention_weight_max": 80.35247039794922,
      "activations/layer4_attention_weight_min": -82.71659088134766,
      "activations/layer5_attention_weight_max": 59.77641296386719,
      "activations/layer5_attention_weight_min": -71.43441009521484,
      "activations/layer6_attention_weight_max": 48.343421936035156,
      "activations/layer6_attention_weight_min": -48.149330139160156,
      "activations/layer7_attention_weight_max": 65.98062896728516,
      "activations/layer7_attention_weight_min": -62.69389724731445,
      "activations/layer8_attention_weight_max": 44.89556121826172,
      "activations/layer8_attention_weight_min": -46.2091064453125,
      "activations/layer9_attention_weight_max": 44.47007369995117,
      "activations/layer9_attention_weight_min": -44.7838020324707,
      "epoch": 20.21,
      "learning_rate": 1.9878787878787875e-05,
      "loss": 2.7143,
      "step": 347800
    },
    {
      "activations/layer0_attention_weight_max": 16.066667556762695,
      "activations/layer0_attention_weight_min": -14.620147705078125,
      "activations/layer10_attention_weight_max": 34.262088775634766,
      "activations/layer10_attention_weight_min": -36.339324951171875,
      "activations/layer11_attention_weight_max": 32.393951416015625,
      "activations/layer11_attention_weight_min": -33.97290802001953,
      "activations/layer12_attention_weight_max": 19.451156616210938,
      "activations/layer12_attention_weight_min": -26.662961959838867,
      "activations/layer13_attention_weight_max": 35.80257797241211,
      "activations/layer13_attention_weight_min": -28.020652770996094,
      "activations/layer14_attention_weight_max": 34.179840087890625,
      "activations/layer14_attention_weight_min": -28.942922592163086,
      "activations/layer15_attention_weight_max": 31.379684448242188,
      "activations/layer15_attention_weight_min": -30.373106002807617,
      "activations/layer16_attention_weight_max": 33.83631896972656,
      "activations/layer16_attention_weight_min": -31.19392967224121,
      "activations/layer17_attention_weight_max": 50.81508255004883,
      "activations/layer17_attention_weight_min": -41.4740104675293,
      "activations/layer18_attention_weight_max": 44.85387420654297,
      "activations/layer18_attention_weight_min": -37.1842155456543,
      "activations/layer19_attention_weight_max": 22.17124366760254,
      "activations/layer19_attention_weight_min": -21.419950485229492,
      "activations/layer1_attention_weight_max": 15.077445983886719,
      "activations/layer1_attention_weight_min": -14.156293869018555,
      "activations/layer20_attention_weight_max": 22.308975219726562,
      "activations/layer20_attention_weight_min": -21.10430335998535,
      "activations/layer21_attention_weight_max": 35.23044204711914,
      "activations/layer21_attention_weight_min": -22.133590698242188,
      "activations/layer22_attention_weight_max": 27.315351486206055,
      "activations/layer22_attention_weight_min": -26.4428653717041,
      "activations/layer23_attention_weight_max": 32.80128860473633,
      "activations/layer23_attention_weight_min": -23.578779220581055,
      "activations/layer2_attention_weight_max": 30.260284423828125,
      "activations/layer2_attention_weight_min": -27.717329025268555,
      "activations/layer3_attention_weight_max": 82.69884490966797,
      "activations/layer3_attention_weight_min": -80.86219024658203,
      "activations/layer4_attention_weight_max": 79.12187957763672,
      "activations/layer4_attention_weight_min": -78.1229476928711,
      "activations/layer5_attention_weight_max": 62.877098083496094,
      "activations/layer5_attention_weight_min": -68.57206726074219,
      "activations/layer6_attention_weight_max": 48.6187629699707,
      "activations/layer6_attention_weight_min": -48.51110076904297,
      "activations/layer7_attention_weight_max": 67.44852447509766,
      "activations/layer7_attention_weight_min": -60.86555480957031,
      "activations/layer8_attention_weight_max": 43.404537200927734,
      "activations/layer8_attention_weight_min": -48.94267654418945,
      "activations/layer9_attention_weight_max": 41.26768493652344,
      "activations/layer9_attention_weight_min": -46.596282958984375,
      "epoch": 20.21,
      "learning_rate": 1.9859848484848483e-05,
      "loss": 2.7244,
      "step": 347850
    },
    {
      "activations/layer0_attention_weight_max": 16.004610061645508,
      "activations/layer0_attention_weight_min": -14.535452842712402,
      "activations/layer10_attention_weight_max": 35.87688064575195,
      "activations/layer10_attention_weight_min": -35.13457107543945,
      "activations/layer11_attention_weight_max": 31.85485076904297,
      "activations/layer11_attention_weight_min": -34.661354064941406,
      "activations/layer12_attention_weight_max": 19.690868377685547,
      "activations/layer12_attention_weight_min": -29.942110061645508,
      "activations/layer13_attention_weight_max": 35.18708419799805,
      "activations/layer13_attention_weight_min": -26.575477600097656,
      "activations/layer14_attention_weight_max": 35.55755615234375,
      "activations/layer14_attention_weight_min": -30.322175979614258,
      "activations/layer15_attention_weight_max": 33.280906677246094,
      "activations/layer15_attention_weight_min": -28.68179702758789,
      "activations/layer16_attention_weight_max": 37.148319244384766,
      "activations/layer16_attention_weight_min": -34.142127990722656,
      "activations/layer17_attention_weight_max": 51.73225784301758,
      "activations/layer17_attention_weight_min": -47.399349212646484,
      "activations/layer18_attention_weight_max": 45.09318161010742,
      "activations/layer18_attention_weight_min": -39.517845153808594,
      "activations/layer19_attention_weight_max": 26.264907836914062,
      "activations/layer19_attention_weight_min": -22.81186866760254,
      "activations/layer1_attention_weight_max": 14.546451568603516,
      "activations/layer1_attention_weight_min": -14.683181762695312,
      "activations/layer20_attention_weight_max": 23.58159637451172,
      "activations/layer20_attention_weight_min": -23.651100158691406,
      "activations/layer21_attention_weight_max": 33.61842346191406,
      "activations/layer21_attention_weight_min": -20.984689712524414,
      "activations/layer22_attention_weight_max": 33.16078186035156,
      "activations/layer22_attention_weight_min": -27.393781661987305,
      "activations/layer23_attention_weight_max": 36.087432861328125,
      "activations/layer23_attention_weight_min": -24.253385543823242,
      "activations/layer2_attention_weight_max": 33.0201530456543,
      "activations/layer2_attention_weight_min": -29.47391128540039,
      "activations/layer3_attention_weight_max": 82.67488861083984,
      "activations/layer3_attention_weight_min": -83.72343444824219,
      "activations/layer4_attention_weight_max": 81.29641723632812,
      "activations/layer4_attention_weight_min": -77.74028778076172,
      "activations/layer5_attention_weight_max": 59.13834762573242,
      "activations/layer5_attention_weight_min": -72.94451141357422,
      "activations/layer6_attention_weight_max": 49.51699447631836,
      "activations/layer6_attention_weight_min": -50.537471771240234,
      "activations/layer7_attention_weight_max": 65.78108978271484,
      "activations/layer7_attention_weight_min": -63.47402572631836,
      "activations/layer8_attention_weight_max": 45.23236846923828,
      "activations/layer8_attention_weight_min": -47.66936111450195,
      "activations/layer9_attention_weight_max": 46.1194953918457,
      "activations/layer9_attention_weight_min": -44.913448333740234,
      "epoch": 20.21,
      "learning_rate": 1.9840909090909088e-05,
      "loss": 2.7205,
      "step": 347900
    },
    {
      "activations/layer0_attention_weight_max": 16.218996047973633,
      "activations/layer0_attention_weight_min": -14.059427261352539,
      "activations/layer10_attention_weight_max": 38.77465057373047,
      "activations/layer10_attention_weight_min": -38.41770935058594,
      "activations/layer11_attention_weight_max": 38.34562301635742,
      "activations/layer11_attention_weight_min": -37.462013244628906,
      "activations/layer12_attention_weight_max": 21.078458786010742,
      "activations/layer12_attention_weight_min": -23.017826080322266,
      "activations/layer13_attention_weight_max": 46.77986145019531,
      "activations/layer13_attention_weight_min": -31.350847244262695,
      "activations/layer14_attention_weight_max": 52.152523040771484,
      "activations/layer14_attention_weight_min": -35.497352600097656,
      "activations/layer15_attention_weight_max": 53.59806442260742,
      "activations/layer15_attention_weight_min": -36.8555908203125,
      "activations/layer16_attention_weight_max": 42.0913200378418,
      "activations/layer16_attention_weight_min": -36.37898635864258,
      "activations/layer17_attention_weight_max": 63.7827033996582,
      "activations/layer17_attention_weight_min": -55.535125732421875,
      "activations/layer18_attention_weight_max": 56.97880554199219,
      "activations/layer18_attention_weight_min": -46.21118927001953,
      "activations/layer19_attention_weight_max": 33.9195556640625,
      "activations/layer19_attention_weight_min": -25.753936767578125,
      "activations/layer1_attention_weight_max": 15.58607006072998,
      "activations/layer1_attention_weight_min": -13.695160865783691,
      "activations/layer20_attention_weight_max": 32.05903244018555,
      "activations/layer20_attention_weight_min": -21.402067184448242,
      "activations/layer21_attention_weight_max": 58.90813446044922,
      "activations/layer21_attention_weight_min": -27.0561580657959,
      "activations/layer22_attention_weight_max": 39.327491760253906,
      "activations/layer22_attention_weight_min": -28.23417091369629,
      "activations/layer23_attention_weight_max": 39.227561950683594,
      "activations/layer23_attention_weight_min": -25.1837215423584,
      "activations/layer2_attention_weight_max": 33.027587890625,
      "activations/layer2_attention_weight_min": -30.11977195739746,
      "activations/layer3_attention_weight_max": 88.96046447753906,
      "activations/layer3_attention_weight_min": -87.86489868164062,
      "activations/layer4_attention_weight_max": 84.49132537841797,
      "activations/layer4_attention_weight_min": -81.5110855102539,
      "activations/layer5_attention_weight_max": 61.302207946777344,
      "activations/layer5_attention_weight_min": -73.79112243652344,
      "activations/layer6_attention_weight_max": 47.2518196105957,
      "activations/layer6_attention_weight_min": -50.07233810424805,
      "activations/layer7_attention_weight_max": 66.88926696777344,
      "activations/layer7_attention_weight_min": -67.06901550292969,
      "activations/layer8_attention_weight_max": 48.65807342529297,
      "activations/layer8_attention_weight_min": -52.3508186340332,
      "activations/layer9_attention_weight_max": 56.425323486328125,
      "activations/layer9_attention_weight_min": -47.44028854370117,
      "epoch": 20.22,
      "learning_rate": 1.9821969696969696e-05,
      "loss": 2.7121,
      "step": 347950
    },
    {
      "activations/layer0_attention_weight_max": 16.509716033935547,
      "activations/layer0_attention_weight_min": -14.118423461914062,
      "activations/layer10_attention_weight_max": 38.21826171875,
      "activations/layer10_attention_weight_min": -38.940582275390625,
      "activations/layer11_attention_weight_max": 36.02637481689453,
      "activations/layer11_attention_weight_min": -35.07176971435547,
      "activations/layer12_attention_weight_max": 20.677444458007812,
      "activations/layer12_attention_weight_min": -24.522436141967773,
      "activations/layer13_attention_weight_max": 34.950496673583984,
      "activations/layer13_attention_weight_min": -26.238969802856445,
      "activations/layer14_attention_weight_max": 35.28676223754883,
      "activations/layer14_attention_weight_min": -30.341171264648438,
      "activations/layer15_attention_weight_max": 31.18903160095215,
      "activations/layer15_attention_weight_min": -30.374845504760742,
      "activations/layer16_attention_weight_max": 31.80800437927246,
      "activations/layer16_attention_weight_min": -31.856462478637695,
      "activations/layer17_attention_weight_max": 47.91072463989258,
      "activations/layer17_attention_weight_min": -43.12080001831055,
      "activations/layer18_attention_weight_max": 43.54764175415039,
      "activations/layer18_attention_weight_min": -39.2997932434082,
      "activations/layer19_attention_weight_max": 22.135488510131836,
      "activations/layer19_attention_weight_min": -22.024656295776367,
      "activations/layer1_attention_weight_max": 15.308749198913574,
      "activations/layer1_attention_weight_min": -12.736422538757324,
      "activations/layer20_attention_weight_max": 22.01325035095215,
      "activations/layer20_attention_weight_min": -20.92124366760254,
      "activations/layer21_attention_weight_max": 34.0159912109375,
      "activations/layer21_attention_weight_min": -23.559194564819336,
      "activations/layer22_attention_weight_max": 27.99159049987793,
      "activations/layer22_attention_weight_min": -25.666732788085938,
      "activations/layer23_attention_weight_max": 36.66741943359375,
      "activations/layer23_attention_weight_min": -25.305028915405273,
      "activations/layer2_attention_weight_max": 31.683652877807617,
      "activations/layer2_attention_weight_min": -29.155248641967773,
      "activations/layer3_attention_weight_max": 81.08295440673828,
      "activations/layer3_attention_weight_min": -81.09239959716797,
      "activations/layer4_attention_weight_max": 83.23226165771484,
      "activations/layer4_attention_weight_min": -81.01866149902344,
      "activations/layer5_attention_weight_max": 63.60842514038086,
      "activations/layer5_attention_weight_min": -70.61421966552734,
      "activations/layer6_attention_weight_max": 47.18016815185547,
      "activations/layer6_attention_weight_min": -50.024497985839844,
      "activations/layer7_attention_weight_max": 65.74777221679688,
      "activations/layer7_attention_weight_min": -67.2940444946289,
      "activations/layer8_attention_weight_max": 50.51567077636719,
      "activations/layer8_attention_weight_min": -51.35927200317383,
      "activations/layer9_attention_weight_max": 44.941192626953125,
      "activations/layer9_attention_weight_min": -47.052425384521484,
      "epoch": 20.22,
      "learning_rate": 1.98030303030303e-05,
      "loss": 2.7178,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4635,
      "eval_samples_per_second": 507.357,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4635,
      "eval_openwebtext_samples_per_second": 507.357,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 2.0231,
      "eval_wikitext_samples_per_second": 225.391,
      "step": 348000
    },
    {
      "epoch": 20.22,
      "eval_lambada_loss": 2.4609375,
      "eval_lambada_ppl": 11.715789946123383,
      "eval_lambada_runtime": 9.6539,
      "eval_lambada_samples_per_second": 504.354,
      "step": 348000
    },
    {
      "activations/layer0_attention_weight_max": 15.351902961730957,
      "activations/layer0_attention_weight_min": -14.005765914916992,
      "activations/layer10_attention_weight_max": 36.0471305847168,
      "activations/layer10_attention_weight_min": -35.78407669067383,
      "activations/layer11_attention_weight_max": 32.85898971557617,
      "activations/layer11_attention_weight_min": -35.68966293334961,
      "activations/layer12_attention_weight_max": 20.15389060974121,
      "activations/layer12_attention_weight_min": -27.12702178955078,
      "activations/layer13_attention_weight_max": 38.26457214355469,
      "activations/layer13_attention_weight_min": -27.744173049926758,
      "activations/layer14_attention_weight_max": 37.16380310058594,
      "activations/layer14_attention_weight_min": -29.935306549072266,
      "activations/layer15_attention_weight_max": 32.75944519042969,
      "activations/layer15_attention_weight_min": -30.10123062133789,
      "activations/layer16_attention_weight_max": 34.2421760559082,
      "activations/layer16_attention_weight_min": -34.07379150390625,
      "activations/layer17_attention_weight_max": 50.54509353637695,
      "activations/layer17_attention_weight_min": -47.381439208984375,
      "activations/layer18_attention_weight_max": 48.55257034301758,
      "activations/layer18_attention_weight_min": -44.44473648071289,
      "activations/layer19_attention_weight_max": 25.919158935546875,
      "activations/layer19_attention_weight_min": -26.384857177734375,
      "activations/layer1_attention_weight_max": 16.914470672607422,
      "activations/layer1_attention_weight_min": -13.221909523010254,
      "activations/layer20_attention_weight_max": 23.322729110717773,
      "activations/layer20_attention_weight_min": -22.096355438232422,
      "activations/layer21_attention_weight_max": 38.889984130859375,
      "activations/layer21_attention_weight_min": -25.612228393554688,
      "activations/layer22_attention_weight_max": 29.046369552612305,
      "activations/layer22_attention_weight_min": -27.30904197692871,
      "activations/layer23_attention_weight_max": 33.79352569580078,
      "activations/layer23_attention_weight_min": -26.36583709716797,
      "activations/layer2_attention_weight_max": 30.094318389892578,
      "activations/layer2_attention_weight_min": -28.471288681030273,
      "activations/layer3_attention_weight_max": 83.05300903320312,
      "activations/layer3_attention_weight_min": -80.66792297363281,
      "activations/layer4_attention_weight_max": 79.60449981689453,
      "activations/layer4_attention_weight_min": -81.93185424804688,
      "activations/layer5_attention_weight_max": 62.8281364440918,
      "activations/layer5_attention_weight_min": -67.35905456542969,
      "activations/layer6_attention_weight_max": 50.077110290527344,
      "activations/layer6_attention_weight_min": -49.577327728271484,
      "activations/layer7_attention_weight_max": 68.5283203125,
      "activations/layer7_attention_weight_min": -67.28843688964844,
      "activations/layer8_attention_weight_max": 45.65726089477539,
      "activations/layer8_attention_weight_min": -48.97505569458008,
      "activations/layer9_attention_weight_max": 41.30937194824219,
      "activations/layer9_attention_weight_min": -45.564754486083984,
      "epoch": 20.22,
      "learning_rate": 1.9784090909090906e-05,
      "loss": 2.7112,
      "step": 348050
    },
    {
      "activations/layer0_attention_weight_max": 15.764652252197266,
      "activations/layer0_attention_weight_min": -13.731046676635742,
      "activations/layer10_attention_weight_max": 32.790008544921875,
      "activations/layer10_attention_weight_min": -33.020084381103516,
      "activations/layer11_attention_weight_max": 30.644649505615234,
      "activations/layer11_attention_weight_min": -34.64092254638672,
      "activations/layer12_attention_weight_max": 17.917987823486328,
      "activations/layer12_attention_weight_min": -24.090633392333984,
      "activations/layer13_attention_weight_max": 34.63213348388672,
      "activations/layer13_attention_weight_min": -26.79132843017578,
      "activations/layer14_attention_weight_max": 37.52669906616211,
      "activations/layer14_attention_weight_min": -30.365432739257812,
      "activations/layer15_attention_weight_max": 33.13438415527344,
      "activations/layer15_attention_weight_min": -29.794265747070312,
      "activations/layer16_attention_weight_max": 33.491455078125,
      "activations/layer16_attention_weight_min": -30.11998176574707,
      "activations/layer17_attention_weight_max": 53.23242950439453,
      "activations/layer17_attention_weight_min": -45.98432540893555,
      "activations/layer18_attention_weight_max": 48.75729751586914,
      "activations/layer18_attention_weight_min": -41.41897964477539,
      "activations/layer19_attention_weight_max": 24.916955947875977,
      "activations/layer19_attention_weight_min": -22.615781784057617,
      "activations/layer1_attention_weight_max": 14.098587989807129,
      "activations/layer1_attention_weight_min": -14.04773998260498,
      "activations/layer20_attention_weight_max": 23.15581703186035,
      "activations/layer20_attention_weight_min": -21.54216194152832,
      "activations/layer21_attention_weight_max": 37.94917678833008,
      "activations/layer21_attention_weight_min": -22.928712844848633,
      "activations/layer22_attention_weight_max": 30.31325912475586,
      "activations/layer22_attention_weight_min": -25.40838623046875,
      "activations/layer23_attention_weight_max": 32.16303253173828,
      "activations/layer23_attention_weight_min": -25.700634002685547,
      "activations/layer2_attention_weight_max": 30.018875122070312,
      "activations/layer2_attention_weight_min": -28.484859466552734,
      "activations/layer3_attention_weight_max": 81.9191665649414,
      "activations/layer3_attention_weight_min": -79.53843688964844,
      "activations/layer4_attention_weight_max": 82.70890045166016,
      "activations/layer4_attention_weight_min": -78.6272964477539,
      "activations/layer5_attention_weight_max": 65.52068328857422,
      "activations/layer5_attention_weight_min": -75.45158386230469,
      "activations/layer6_attention_weight_max": 49.29580307006836,
      "activations/layer6_attention_weight_min": -49.45203399658203,
      "activations/layer7_attention_weight_max": 67.523681640625,
      "activations/layer7_attention_weight_min": -63.245662689208984,
      "activations/layer8_attention_weight_max": 43.8362922668457,
      "activations/layer8_attention_weight_min": -48.98261260986328,
      "activations/layer9_attention_weight_max": 40.43716049194336,
      "activations/layer9_attention_weight_min": -45.879150390625,
      "epoch": 20.23,
      "learning_rate": 1.976515151515151e-05,
      "loss": 2.7241,
      "step": 348100
    },
    {
      "activations/layer0_attention_weight_max": 15.899618148803711,
      "activations/layer0_attention_weight_min": -14.09832763671875,
      "activations/layer10_attention_weight_max": 35.863380432128906,
      "activations/layer10_attention_weight_min": -33.904541015625,
      "activations/layer11_attention_weight_max": 31.912822723388672,
      "activations/layer11_attention_weight_min": -34.25476837158203,
      "activations/layer12_attention_weight_max": 17.545433044433594,
      "activations/layer12_attention_weight_min": -24.584877014160156,
      "activations/layer13_attention_weight_max": 32.84006881713867,
      "activations/layer13_attention_weight_min": -28.138330459594727,
      "activations/layer14_attention_weight_max": 36.27033996582031,
      "activations/layer14_attention_weight_min": -28.758481979370117,
      "activations/layer15_attention_weight_max": 33.63761901855469,
      "activations/layer15_attention_weight_min": -30.014423370361328,
      "activations/layer16_attention_weight_max": 33.57334518432617,
      "activations/layer16_attention_weight_min": -31.016511917114258,
      "activations/layer17_attention_weight_max": 50.33843994140625,
      "activations/layer17_attention_weight_min": -44.253841400146484,
      "activations/layer18_attention_weight_max": 49.072574615478516,
      "activations/layer18_attention_weight_min": -42.00187301635742,
      "activations/layer19_attention_weight_max": 28.5963191986084,
      "activations/layer19_attention_weight_min": -22.36558723449707,
      "activations/layer1_attention_weight_max": 15.33969783782959,
      "activations/layer1_attention_weight_min": -13.704331398010254,
      "activations/layer20_attention_weight_max": 24.424848556518555,
      "activations/layer20_attention_weight_min": -21.663373947143555,
      "activations/layer21_attention_weight_max": 37.309059143066406,
      "activations/layer21_attention_weight_min": -24.1824893951416,
      "activations/layer22_attention_weight_max": 29.968992233276367,
      "activations/layer22_attention_weight_min": -25.009172439575195,
      "activations/layer23_attention_weight_max": 37.44157409667969,
      "activations/layer23_attention_weight_min": -24.160423278808594,
      "activations/layer2_attention_weight_max": 30.025638580322266,
      "activations/layer2_attention_weight_min": -26.781139373779297,
      "activations/layer3_attention_weight_max": 79.33353424072266,
      "activations/layer3_attention_weight_min": -77.67440795898438,
      "activations/layer4_attention_weight_max": 77.98300170898438,
      "activations/layer4_attention_weight_min": -83.05876159667969,
      "activations/layer5_attention_weight_max": 60.80204391479492,
      "activations/layer5_attention_weight_min": -71.74764251708984,
      "activations/layer6_attention_weight_max": 46.161949157714844,
      "activations/layer6_attention_weight_min": -48.03337478637695,
      "activations/layer7_attention_weight_max": 63.8560905456543,
      "activations/layer7_attention_weight_min": -65.19073486328125,
      "activations/layer8_attention_weight_max": 46.05123519897461,
      "activations/layer8_attention_weight_min": -49.74525451660156,
      "activations/layer9_attention_weight_max": 42.07518005371094,
      "activations/layer9_attention_weight_min": -45.45243453979492,
      "epoch": 20.23,
      "learning_rate": 1.974621212121212e-05,
      "loss": 2.7146,
      "step": 348150
    },
    {
      "activations/layer0_attention_weight_max": 15.684022903442383,
      "activations/layer0_attention_weight_min": -13.952280044555664,
      "activations/layer10_attention_weight_max": 36.97657012939453,
      "activations/layer10_attention_weight_min": -35.60102844238281,
      "activations/layer11_attention_weight_max": 34.86296844482422,
      "activations/layer11_attention_weight_min": -35.65909957885742,
      "activations/layer12_attention_weight_max": 20.36359977722168,
      "activations/layer12_attention_weight_min": -26.19513511657715,
      "activations/layer13_attention_weight_max": 35.35981750488281,
      "activations/layer13_attention_weight_min": -27.709436416625977,
      "activations/layer14_attention_weight_max": 36.63700866699219,
      "activations/layer14_attention_weight_min": -29.895572662353516,
      "activations/layer15_attention_weight_max": 34.43819046020508,
      "activations/layer15_attention_weight_min": -30.296615600585938,
      "activations/layer16_attention_weight_max": 32.99604797363281,
      "activations/layer16_attention_weight_min": -31.047252655029297,
      "activations/layer17_attention_weight_max": 48.010494232177734,
      "activations/layer17_attention_weight_min": -42.416847229003906,
      "activations/layer18_attention_weight_max": 46.4782829284668,
      "activations/layer18_attention_weight_min": -39.4481086730957,
      "activations/layer19_attention_weight_max": 23.249082565307617,
      "activations/layer19_attention_weight_min": -23.209962844848633,
      "activations/layer1_attention_weight_max": 15.570514678955078,
      "activations/layer1_attention_weight_min": -14.170324325561523,
      "activations/layer20_attention_weight_max": 23.141183853149414,
      "activations/layer20_attention_weight_min": -25.421232223510742,
      "activations/layer21_attention_weight_max": 33.81361770629883,
      "activations/layer21_attention_weight_min": -22.13498878479004,
      "activations/layer22_attention_weight_max": 31.050445556640625,
      "activations/layer22_attention_weight_min": -26.703298568725586,
      "activations/layer23_attention_weight_max": 37.49098205566406,
      "activations/layer23_attention_weight_min": -24.672096252441406,
      "activations/layer2_attention_weight_max": 29.928634643554688,
      "activations/layer2_attention_weight_min": -28.14806365966797,
      "activations/layer3_attention_weight_max": 83.94359588623047,
      "activations/layer3_attention_weight_min": -80.57038116455078,
      "activations/layer4_attention_weight_max": 79.67680358886719,
      "activations/layer4_attention_weight_min": -81.08323669433594,
      "activations/layer5_attention_weight_max": 58.78074264526367,
      "activations/layer5_attention_weight_min": -71.22662353515625,
      "activations/layer6_attention_weight_max": 46.43124771118164,
      "activations/layer6_attention_weight_min": -48.04753494262695,
      "activations/layer7_attention_weight_max": 61.67102813720703,
      "activations/layer7_attention_weight_min": -62.975406646728516,
      "activations/layer8_attention_weight_max": 46.39570999145508,
      "activations/layer8_attention_weight_min": -49.14540481567383,
      "activations/layer9_attention_weight_max": 43.73395919799805,
      "activations/layer9_attention_weight_min": -45.71866989135742,
      "epoch": 20.23,
      "learning_rate": 1.9727272727272724e-05,
      "loss": 2.7057,
      "step": 348200
    },
    {
      "activations/layer0_attention_weight_max": 15.90244197845459,
      "activations/layer0_attention_weight_min": -14.223230361938477,
      "activations/layer10_attention_weight_max": 37.331573486328125,
      "activations/layer10_attention_weight_min": -38.68577575683594,
      "activations/layer11_attention_weight_max": 35.88981628417969,
      "activations/layer11_attention_weight_min": -36.2901611328125,
      "activations/layer12_attention_weight_max": 21.02492904663086,
      "activations/layer12_attention_weight_min": -30.957860946655273,
      "activations/layer13_attention_weight_max": 33.69664001464844,
      "activations/layer13_attention_weight_min": -28.14274024963379,
      "activations/layer14_attention_weight_max": 36.08283233642578,
      "activations/layer14_attention_weight_min": -31.907310485839844,
      "activations/layer15_attention_weight_max": 33.503761291503906,
      "activations/layer15_attention_weight_min": -32.53926467895508,
      "activations/layer16_attention_weight_max": 37.06120681762695,
      "activations/layer16_attention_weight_min": -33.76687240600586,
      "activations/layer17_attention_weight_max": 53.52768325805664,
      "activations/layer17_attention_weight_min": -48.37216567993164,
      "activations/layer18_attention_weight_max": 53.61106872558594,
      "activations/layer18_attention_weight_min": -46.44545364379883,
      "activations/layer19_attention_weight_max": 24.991844177246094,
      "activations/layer19_attention_weight_min": -25.01646614074707,
      "activations/layer1_attention_weight_max": 15.504714012145996,
      "activations/layer1_attention_weight_min": -14.599227905273438,
      "activations/layer20_attention_weight_max": 23.750804901123047,
      "activations/layer20_attention_weight_min": -21.35405158996582,
      "activations/layer21_attention_weight_max": 34.30058288574219,
      "activations/layer21_attention_weight_min": -23.53936004638672,
      "activations/layer22_attention_weight_max": 32.1776008605957,
      "activations/layer22_attention_weight_min": -26.91090202331543,
      "activations/layer23_attention_weight_max": 42.678749084472656,
      "activations/layer23_attention_weight_min": -26.08051300048828,
      "activations/layer2_attention_weight_max": 32.37123107910156,
      "activations/layer2_attention_weight_min": -29.689403533935547,
      "activations/layer3_attention_weight_max": 86.83280181884766,
      "activations/layer3_attention_weight_min": -85.82827758789062,
      "activations/layer4_attention_weight_max": 84.72762298583984,
      "activations/layer4_attention_weight_min": -85.36978912353516,
      "activations/layer5_attention_weight_max": 62.35254669189453,
      "activations/layer5_attention_weight_min": -71.39825439453125,
      "activations/layer6_attention_weight_max": 49.49797821044922,
      "activations/layer6_attention_weight_min": -52.14146041870117,
      "activations/layer7_attention_weight_max": 64.58533477783203,
      "activations/layer7_attention_weight_min": -64.356201171875,
      "activations/layer8_attention_weight_max": 47.393280029296875,
      "activations/layer8_attention_weight_min": -50.23954391479492,
      "activations/layer9_attention_weight_max": 44.04484939575195,
      "activations/layer9_attention_weight_min": -46.867095947265625,
      "epoch": 20.24,
      "learning_rate": 1.9708333333333332e-05,
      "loss": 2.7223,
      "step": 348250
    },
    {
      "activations/layer0_attention_weight_max": 17.13131332397461,
      "activations/layer0_attention_weight_min": -14.268814086914062,
      "activations/layer10_attention_weight_max": 38.5954475402832,
      "activations/layer10_attention_weight_min": -36.41161346435547,
      "activations/layer11_attention_weight_max": 40.351314544677734,
      "activations/layer11_attention_weight_min": -37.23738098144531,
      "activations/layer12_attention_weight_max": 17.74517822265625,
      "activations/layer12_attention_weight_min": -26.92922592163086,
      "activations/layer13_attention_weight_max": 38.85945510864258,
      "activations/layer13_attention_weight_min": -26.608366012573242,
      "activations/layer14_attention_weight_max": 41.399967193603516,
      "activations/layer14_attention_weight_min": -29.480871200561523,
      "activations/layer15_attention_weight_max": 41.575870513916016,
      "activations/layer15_attention_weight_min": -29.445249557495117,
      "activations/layer16_attention_weight_max": 34.08283615112305,
      "activations/layer16_attention_weight_min": -31.613113403320312,
      "activations/layer17_attention_weight_max": 48.13090133666992,
      "activations/layer17_attention_weight_min": -43.527584075927734,
      "activations/layer18_attention_weight_max": 44.35137176513672,
      "activations/layer18_attention_weight_min": -39.70143127441406,
      "activations/layer19_attention_weight_max": 23.192081451416016,
      "activations/layer19_attention_weight_min": -23.07682228088379,
      "activations/layer1_attention_weight_max": 15.992819786071777,
      "activations/layer1_attention_weight_min": -12.950661659240723,
      "activations/layer20_attention_weight_max": 22.684614181518555,
      "activations/layer20_attention_weight_min": -22.397201538085938,
      "activations/layer21_attention_weight_max": 36.798439025878906,
      "activations/layer21_attention_weight_min": -21.723630905151367,
      "activations/layer22_attention_weight_max": 30.56473159790039,
      "activations/layer22_attention_weight_min": -24.684194564819336,
      "activations/layer23_attention_weight_max": 32.85144805908203,
      "activations/layer23_attention_weight_min": -21.844219207763672,
      "activations/layer2_attention_weight_max": 32.785621643066406,
      "activations/layer2_attention_weight_min": -30.514034271240234,
      "activations/layer3_attention_weight_max": 85.52986907958984,
      "activations/layer3_attention_weight_min": -85.24654388427734,
      "activations/layer4_attention_weight_max": 81.52030181884766,
      "activations/layer4_attention_weight_min": -81.66893768310547,
      "activations/layer5_attention_weight_max": 61.551876068115234,
      "activations/layer5_attention_weight_min": -72.09368896484375,
      "activations/layer6_attention_weight_max": 47.80055236816406,
      "activations/layer6_attention_weight_min": -49.12427520751953,
      "activations/layer7_attention_weight_max": 68.75094604492188,
      "activations/layer7_attention_weight_min": -63.362972259521484,
      "activations/layer8_attention_weight_max": 46.444828033447266,
      "activations/layer8_attention_weight_min": -48.56269073486328,
      "activations/layer9_attention_weight_max": 53.693389892578125,
      "activations/layer9_attention_weight_min": -46.10878372192383,
      "epoch": 20.24,
      "learning_rate": 1.968939393939394e-05,
      "loss": 2.7229,
      "step": 348300
    },
    {
      "activations/layer0_attention_weight_max": 15.499283790588379,
      "activations/layer0_attention_weight_min": -14.024696350097656,
      "activations/layer10_attention_weight_max": 36.563682556152344,
      "activations/layer10_attention_weight_min": -36.219661712646484,
      "activations/layer11_attention_weight_max": 36.472206115722656,
      "activations/layer11_attention_weight_min": -36.429019927978516,
      "activations/layer12_attention_weight_max": 18.4265193939209,
      "activations/layer12_attention_weight_min": -23.77328109741211,
      "activations/layer13_attention_weight_max": 35.097557067871094,
      "activations/layer13_attention_weight_min": -28.30125617980957,
      "activations/layer14_attention_weight_max": 33.432952880859375,
      "activations/layer14_attention_weight_min": -29.215923309326172,
      "activations/layer15_attention_weight_max": 30.782135009765625,
      "activations/layer15_attention_weight_min": -30.57940673828125,
      "activations/layer16_attention_weight_max": 33.6473503112793,
      "activations/layer16_attention_weight_min": -31.318126678466797,
      "activations/layer17_attention_weight_max": 53.24421310424805,
      "activations/layer17_attention_weight_min": -46.684234619140625,
      "activations/layer18_attention_weight_max": 47.666717529296875,
      "activations/layer18_attention_weight_min": -45.82833480834961,
      "activations/layer19_attention_weight_max": 23.648046493530273,
      "activations/layer19_attention_weight_min": -22.544017791748047,
      "activations/layer1_attention_weight_max": 15.804203033447266,
      "activations/layer1_attention_weight_min": -13.45250129699707,
      "activations/layer20_attention_weight_max": 19.975072860717773,
      "activations/layer20_attention_weight_min": -20.373760223388672,
      "activations/layer21_attention_weight_max": 35.16106414794922,
      "activations/layer21_attention_weight_min": -20.389545440673828,
      "activations/layer22_attention_weight_max": 30.41804313659668,
      "activations/layer22_attention_weight_min": -24.42961311340332,
      "activations/layer23_attention_weight_max": 34.27647399902344,
      "activations/layer23_attention_weight_min": -26.946826934814453,
      "activations/layer2_attention_weight_max": 31.25495147705078,
      "activations/layer2_attention_weight_min": -29.479551315307617,
      "activations/layer3_attention_weight_max": 84.68029022216797,
      "activations/layer3_attention_weight_min": -84.89061737060547,
      "activations/layer4_attention_weight_max": 82.96451568603516,
      "activations/layer4_attention_weight_min": -84.51299285888672,
      "activations/layer5_attention_weight_max": 60.3054313659668,
      "activations/layer5_attention_weight_min": -68.02598571777344,
      "activations/layer6_attention_weight_max": 47.393287658691406,
      "activations/layer6_attention_weight_min": -48.899559020996094,
      "activations/layer7_attention_weight_max": 68.61256408691406,
      "activations/layer7_attention_weight_min": -72.53765869140625,
      "activations/layer8_attention_weight_max": 46.29446029663086,
      "activations/layer8_attention_weight_min": -51.52714538574219,
      "activations/layer9_attention_weight_max": 46.732723236083984,
      "activations/layer9_attention_weight_min": -46.651939392089844,
      "epoch": 20.24,
      "learning_rate": 1.9670454545454542e-05,
      "loss": 2.7225,
      "step": 348350
    },
    {
      "activations/layer0_attention_weight_max": 15.136119842529297,
      "activations/layer0_attention_weight_min": -14.437002182006836,
      "activations/layer10_attention_weight_max": 35.60134506225586,
      "activations/layer10_attention_weight_min": -34.379154205322266,
      "activations/layer11_attention_weight_max": 31.816144943237305,
      "activations/layer11_attention_weight_min": -32.89623260498047,
      "activations/layer12_attention_weight_max": 18.741548538208008,
      "activations/layer12_attention_weight_min": -25.49327278137207,
      "activations/layer13_attention_weight_max": 32.376914978027344,
      "activations/layer13_attention_weight_min": -28.478702545166016,
      "activations/layer14_attention_weight_max": 33.600189208984375,
      "activations/layer14_attention_weight_min": -33.070823669433594,
      "activations/layer15_attention_weight_max": 30.436359405517578,
      "activations/layer15_attention_weight_min": -32.76164627075195,
      "activations/layer16_attention_weight_max": 33.62546920776367,
      "activations/layer16_attention_weight_min": -31.58147430419922,
      "activations/layer17_attention_weight_max": 51.22749710083008,
      "activations/layer17_attention_weight_min": -44.381080627441406,
      "activations/layer18_attention_weight_max": 44.54941940307617,
      "activations/layer18_attention_weight_min": -40.93020248413086,
      "activations/layer19_attention_weight_max": 22.683900833129883,
      "activations/layer19_attention_weight_min": -24.140592575073242,
      "activations/layer1_attention_weight_max": 15.576688766479492,
      "activations/layer1_attention_weight_min": -14.929410934448242,
      "activations/layer20_attention_weight_max": 22.987581253051758,
      "activations/layer20_attention_weight_min": -24.775196075439453,
      "activations/layer21_attention_weight_max": 33.585025787353516,
      "activations/layer21_attention_weight_min": -24.361328125,
      "activations/layer22_attention_weight_max": 28.36899185180664,
      "activations/layer22_attention_weight_min": -26.097349166870117,
      "activations/layer23_attention_weight_max": 31.86408805847168,
      "activations/layer23_attention_weight_min": -25.089818954467773,
      "activations/layer2_attention_weight_max": 32.67724609375,
      "activations/layer2_attention_weight_min": -30.40347671508789,
      "activations/layer3_attention_weight_max": 86.12249755859375,
      "activations/layer3_attention_weight_min": -85.88023376464844,
      "activations/layer4_attention_weight_max": 81.84233093261719,
      "activations/layer4_attention_weight_min": -81.69501495361328,
      "activations/layer5_attention_weight_max": 62.365474700927734,
      "activations/layer5_attention_weight_min": -70.74273681640625,
      "activations/layer6_attention_weight_max": 47.95185470581055,
      "activations/layer6_attention_weight_min": -49.53313446044922,
      "activations/layer7_attention_weight_max": 65.25762176513672,
      "activations/layer7_attention_weight_min": -65.24263763427734,
      "activations/layer8_attention_weight_max": 45.58169937133789,
      "activations/layer8_attention_weight_min": -49.832820892333984,
      "activations/layer9_attention_weight_max": 45.97915267944336,
      "activations/layer9_attention_weight_min": -45.111976623535156,
      "epoch": 20.24,
      "learning_rate": 1.965151515151515e-05,
      "loss": 2.7308,
      "step": 348400
    },
    {
      "activations/layer0_attention_weight_max": 15.970407485961914,
      "activations/layer0_attention_weight_min": -13.8749418258667,
      "activations/layer10_attention_weight_max": 33.323341369628906,
      "activations/layer10_attention_weight_min": -33.593109130859375,
      "activations/layer11_attention_weight_max": 31.320323944091797,
      "activations/layer11_attention_weight_min": -32.99883270263672,
      "activations/layer12_attention_weight_max": 19.13874626159668,
      "activations/layer12_attention_weight_min": -22.77361297607422,
      "activations/layer13_attention_weight_max": 33.2628288269043,
      "activations/layer13_attention_weight_min": -25.902555465698242,
      "activations/layer14_attention_weight_max": 33.59589385986328,
      "activations/layer14_attention_weight_min": -29.853368759155273,
      "activations/layer15_attention_weight_max": 31.818401336669922,
      "activations/layer15_attention_weight_min": -30.187515258789062,
      "activations/layer16_attention_weight_max": 30.97327423095703,
      "activations/layer16_attention_weight_min": -31.43718147277832,
      "activations/layer17_attention_weight_max": 48.54225540161133,
      "activations/layer17_attention_weight_min": -46.85189437866211,
      "activations/layer18_attention_weight_max": 44.963436126708984,
      "activations/layer18_attention_weight_min": -41.8690071105957,
      "activations/layer19_attention_weight_max": 28.28803825378418,
      "activations/layer19_attention_weight_min": -22.76045799255371,
      "activations/layer1_attention_weight_max": 16.125608444213867,
      "activations/layer1_attention_weight_min": -13.938610076904297,
      "activations/layer20_attention_weight_max": 23.887693405151367,
      "activations/layer20_attention_weight_min": -19.704914093017578,
      "activations/layer21_attention_weight_max": 33.36626052856445,
      "activations/layer21_attention_weight_min": -22.88890838623047,
      "activations/layer22_attention_weight_max": 26.89225196838379,
      "activations/layer22_attention_weight_min": -24.46539878845215,
      "activations/layer23_attention_weight_max": 36.51337814331055,
      "activations/layer23_attention_weight_min": -26.77424430847168,
      "activations/layer2_attention_weight_max": 31.740272521972656,
      "activations/layer2_attention_weight_min": -29.90741729736328,
      "activations/layer3_attention_weight_max": 84.01296997070312,
      "activations/layer3_attention_weight_min": -84.6500473022461,
      "activations/layer4_attention_weight_max": 83.62776947021484,
      "activations/layer4_attention_weight_min": -82.0693359375,
      "activations/layer5_attention_weight_max": 59.65070343017578,
      "activations/layer5_attention_weight_min": -72.70481872558594,
      "activations/layer6_attention_weight_max": 50.887001037597656,
      "activations/layer6_attention_weight_min": -47.7232551574707,
      "activations/layer7_attention_weight_max": 64.30200958251953,
      "activations/layer7_attention_weight_min": -62.485286712646484,
      "activations/layer8_attention_weight_max": 44.15523147583008,
      "activations/layer8_attention_weight_min": -47.9582633972168,
      "activations/layer9_attention_weight_max": 40.78850173950195,
      "activations/layer9_attention_weight_min": -42.89848327636719,
      "epoch": 20.25,
      "learning_rate": 1.9632575757575755e-05,
      "loss": 2.7268,
      "step": 348450
    },
    {
      "activations/layer0_attention_weight_max": 15.674660682678223,
      "activations/layer0_attention_weight_min": -14.099828720092773,
      "activations/layer10_attention_weight_max": 35.12141418457031,
      "activations/layer10_attention_weight_min": -35.917816162109375,
      "activations/layer11_attention_weight_max": 31.77670669555664,
      "activations/layer11_attention_weight_min": -34.683326721191406,
      "activations/layer12_attention_weight_max": 18.975858688354492,
      "activations/layer12_attention_weight_min": -24.550174713134766,
      "activations/layer13_attention_weight_max": 35.430809020996094,
      "activations/layer13_attention_weight_min": -28.843006134033203,
      "activations/layer14_attention_weight_max": 34.84952163696289,
      "activations/layer14_attention_weight_min": -31.25814437866211,
      "activations/layer15_attention_weight_max": 32.06886672973633,
      "activations/layer15_attention_weight_min": -31.101791381835938,
      "activations/layer16_attention_weight_max": 35.47853088378906,
      "activations/layer16_attention_weight_min": -33.74769592285156,
      "activations/layer17_attention_weight_max": 48.97005081176758,
      "activations/layer17_attention_weight_min": -44.962928771972656,
      "activations/layer18_attention_weight_max": 45.575008392333984,
      "activations/layer18_attention_weight_min": -39.856727600097656,
      "activations/layer19_attention_weight_max": 24.597471237182617,
      "activations/layer19_attention_weight_min": -22.238759994506836,
      "activations/layer1_attention_weight_max": 15.284904479980469,
      "activations/layer1_attention_weight_min": -13.194786071777344,
      "activations/layer20_attention_weight_max": 22.783573150634766,
      "activations/layer20_attention_weight_min": -22.499122619628906,
      "activations/layer21_attention_weight_max": 36.02435302734375,
      "activations/layer21_attention_weight_min": -23.48822021484375,
      "activations/layer22_attention_weight_max": 27.388900756835938,
      "activations/layer22_attention_weight_min": -24.745813369750977,
      "activations/layer23_attention_weight_max": 36.36276626586914,
      "activations/layer23_attention_weight_min": -23.55416488647461,
      "activations/layer2_attention_weight_max": 31.745147705078125,
      "activations/layer2_attention_weight_min": -31.664398193359375,
      "activations/layer3_attention_weight_max": 82.2703628540039,
      "activations/layer3_attention_weight_min": -83.97943878173828,
      "activations/layer4_attention_weight_max": 82.40181732177734,
      "activations/layer4_attention_weight_min": -82.56779479980469,
      "activations/layer5_attention_weight_max": 61.822811126708984,
      "activations/layer5_attention_weight_min": -75.28917694091797,
      "activations/layer6_attention_weight_max": 50.46870803833008,
      "activations/layer6_attention_weight_min": -50.190364837646484,
      "activations/layer7_attention_weight_max": 62.685020446777344,
      "activations/layer7_attention_weight_min": -64.03190612792969,
      "activations/layer8_attention_weight_max": 45.22916030883789,
      "activations/layer8_attention_weight_min": -49.39395523071289,
      "activations/layer9_attention_weight_max": 41.453392028808594,
      "activations/layer9_attention_weight_min": -47.195777893066406,
      "epoch": 20.25,
      "learning_rate": 1.9613636363636364e-05,
      "loss": 2.7221,
      "step": 348500
    },
    {
      "activations/layer0_attention_weight_max": 15.7044677734375,
      "activations/layer0_attention_weight_min": -14.270517349243164,
      "activations/layer10_attention_weight_max": 42.0107307434082,
      "activations/layer10_attention_weight_min": -39.245948791503906,
      "activations/layer11_attention_weight_max": 41.44158935546875,
      "activations/layer11_attention_weight_min": -36.712318420410156,
      "activations/layer12_attention_weight_max": 18.924753189086914,
      "activations/layer12_attention_weight_min": -26.282495498657227,
      "activations/layer13_attention_weight_max": 35.79419708251953,
      "activations/layer13_attention_weight_min": -30.233789443969727,
      "activations/layer14_attention_weight_max": 37.6537971496582,
      "activations/layer14_attention_weight_min": -30.21784019470215,
      "activations/layer15_attention_weight_max": 34.34856033325195,
      "activations/layer15_attention_weight_min": -30.465343475341797,
      "activations/layer16_attention_weight_max": 36.79636764526367,
      "activations/layer16_attention_weight_min": -35.02798080444336,
      "activations/layer17_attention_weight_max": 53.32680892944336,
      "activations/layer17_attention_weight_min": -48.55880355834961,
      "activations/layer18_attention_weight_max": 52.763214111328125,
      "activations/layer18_attention_weight_min": -40.80647277832031,
      "activations/layer19_attention_weight_max": 24.562538146972656,
      "activations/layer19_attention_weight_min": -23.5184268951416,
      "activations/layer1_attention_weight_max": 16.534482955932617,
      "activations/layer1_attention_weight_min": -14.433626174926758,
      "activations/layer20_attention_weight_max": 23.96256446838379,
      "activations/layer20_attention_weight_min": -21.08496856689453,
      "activations/layer21_attention_weight_max": 40.1282958984375,
      "activations/layer21_attention_weight_min": -22.273311614990234,
      "activations/layer22_attention_weight_max": 26.608964920043945,
      "activations/layer22_attention_weight_min": -24.931007385253906,
      "activations/layer23_attention_weight_max": 32.151939392089844,
      "activations/layer23_attention_weight_min": -22.717023849487305,
      "activations/layer2_attention_weight_max": 30.799039840698242,
      "activations/layer2_attention_weight_min": -30.824506759643555,
      "activations/layer3_attention_weight_max": 85.75267791748047,
      "activations/layer3_attention_weight_min": -87.05298614501953,
      "activations/layer4_attention_weight_max": 81.21891021728516,
      "activations/layer4_attention_weight_min": -84.2834701538086,
      "activations/layer5_attention_weight_max": 62.116249084472656,
      "activations/layer5_attention_weight_min": -70.70838165283203,
      "activations/layer6_attention_weight_max": 48.43754959106445,
      "activations/layer6_attention_weight_min": -50.29347229003906,
      "activations/layer7_attention_weight_max": 78.3359603881836,
      "activations/layer7_attention_weight_min": -65.00405883789062,
      "activations/layer8_attention_weight_max": 53.90834426879883,
      "activations/layer8_attention_weight_min": -50.394981384277344,
      "activations/layer9_attention_weight_max": 57.601295471191406,
      "activations/layer9_attention_weight_min": -47.107765197753906,
      "epoch": 20.25,
      "learning_rate": 1.959469696969697e-05,
      "loss": 2.7169,
      "step": 348550
    },
    {
      "activations/layer0_attention_weight_max": 16.60425567626953,
      "activations/layer0_attention_weight_min": -13.927061080932617,
      "activations/layer10_attention_weight_max": 34.52654266357422,
      "activations/layer10_attention_weight_min": -35.90166091918945,
      "activations/layer11_attention_weight_max": 32.90888214111328,
      "activations/layer11_attention_weight_min": -33.331939697265625,
      "activations/layer12_attention_weight_max": 20.001251220703125,
      "activations/layer12_attention_weight_min": -22.906246185302734,
      "activations/layer13_attention_weight_max": 36.04339599609375,
      "activations/layer13_attention_weight_min": -28.46864891052246,
      "activations/layer14_attention_weight_max": 35.75672912597656,
      "activations/layer14_attention_weight_min": -30.012096405029297,
      "activations/layer15_attention_weight_max": 33.775753021240234,
      "activations/layer15_attention_weight_min": -30.204618453979492,
      "activations/layer16_attention_weight_max": 35.379241943359375,
      "activations/layer16_attention_weight_min": -32.90430450439453,
      "activations/layer17_attention_weight_max": 52.46147918701172,
      "activations/layer17_attention_weight_min": -45.0455436706543,
      "activations/layer18_attention_weight_max": 51.48353576660156,
      "activations/layer18_attention_weight_min": -41.2347412109375,
      "activations/layer19_attention_weight_max": 26.27860450744629,
      "activations/layer19_attention_weight_min": -21.49348258972168,
      "activations/layer1_attention_weight_max": 15.48336124420166,
      "activations/layer1_attention_weight_min": -12.822991371154785,
      "activations/layer20_attention_weight_max": 21.80853271484375,
      "activations/layer20_attention_weight_min": -20.27855682373047,
      "activations/layer21_attention_weight_max": 34.5196647644043,
      "activations/layer21_attention_weight_min": -19.841053009033203,
      "activations/layer22_attention_weight_max": 29.498708724975586,
      "activations/layer22_attention_weight_min": -24.68666648864746,
      "activations/layer23_attention_weight_max": 39.98799514770508,
      "activations/layer23_attention_weight_min": -24.168704986572266,
      "activations/layer2_attention_weight_max": 27.985502243041992,
      "activations/layer2_attention_weight_min": -26.78329849243164,
      "activations/layer3_attention_weight_max": 79.44939422607422,
      "activations/layer3_attention_weight_min": -80.56934356689453,
      "activations/layer4_attention_weight_max": 77.87836456298828,
      "activations/layer4_attention_weight_min": -78.34258270263672,
      "activations/layer5_attention_weight_max": 59.33695983886719,
      "activations/layer5_attention_weight_min": -70.50801086425781,
      "activations/layer6_attention_weight_max": 46.67619705200195,
      "activations/layer6_attention_weight_min": -48.79004669189453,
      "activations/layer7_attention_weight_max": 63.954044342041016,
      "activations/layer7_attention_weight_min": -65.36419677734375,
      "activations/layer8_attention_weight_max": 46.071041107177734,
      "activations/layer8_attention_weight_min": -50.80098342895508,
      "activations/layer9_attention_weight_max": 42.73647689819336,
      "activations/layer9_attention_weight_min": -49.03021240234375,
      "epoch": 20.26,
      "learning_rate": 1.9575757575757577e-05,
      "loss": 2.7139,
      "step": 348600
    },
    {
      "activations/layer0_attention_weight_max": 16.575969696044922,
      "activations/layer0_attention_weight_min": -13.972444534301758,
      "activations/layer10_attention_weight_max": 40.12442398071289,
      "activations/layer10_attention_weight_min": -36.617835998535156,
      "activations/layer11_attention_weight_max": 35.89323425292969,
      "activations/layer11_attention_weight_min": -35.223297119140625,
      "activations/layer12_attention_weight_max": 18.935977935791016,
      "activations/layer12_attention_weight_min": -26.363868713378906,
      "activations/layer13_attention_weight_max": 43.63725662231445,
      "activations/layer13_attention_weight_min": -26.11728286743164,
      "activations/layer14_attention_weight_max": 42.20941925048828,
      "activations/layer14_attention_weight_min": -30.413116455078125,
      "activations/layer15_attention_weight_max": 42.805076599121094,
      "activations/layer15_attention_weight_min": -31.045785903930664,
      "activations/layer16_attention_weight_max": 35.03885269165039,
      "activations/layer16_attention_weight_min": -32.33292007446289,
      "activations/layer17_attention_weight_max": 50.790138244628906,
      "activations/layer17_attention_weight_min": -45.65239715576172,
      "activations/layer18_attention_weight_max": 50.113975524902344,
      "activations/layer18_attention_weight_min": -44.1794548034668,
      "activations/layer19_attention_weight_max": 23.28907585144043,
      "activations/layer19_attention_weight_min": -23.857948303222656,
      "activations/layer1_attention_weight_max": 16.63655662536621,
      "activations/layer1_attention_weight_min": -14.475822448730469,
      "activations/layer20_attention_weight_max": 23.08212661743164,
      "activations/layer20_attention_weight_min": -24.24323844909668,
      "activations/layer21_attention_weight_max": 33.295589447021484,
      "activations/layer21_attention_weight_min": -30.09176254272461,
      "activations/layer22_attention_weight_max": 29.10115623474121,
      "activations/layer22_attention_weight_min": -25.40665054321289,
      "activations/layer23_attention_weight_max": 34.79399108886719,
      "activations/layer23_attention_weight_min": -33.05440139770508,
      "activations/layer2_attention_weight_max": 31.091426849365234,
      "activations/layer2_attention_weight_min": -27.527448654174805,
      "activations/layer3_attention_weight_max": 84.35922241210938,
      "activations/layer3_attention_weight_min": -82.68757629394531,
      "activations/layer4_attention_weight_max": 86.06442260742188,
      "activations/layer4_attention_weight_min": -81.01295471191406,
      "activations/layer5_attention_weight_max": 60.74419403076172,
      "activations/layer5_attention_weight_min": -69.67072296142578,
      "activations/layer6_attention_weight_max": 48.38502502441406,
      "activations/layer6_attention_weight_min": -49.142757415771484,
      "activations/layer7_attention_weight_max": 71.55684661865234,
      "activations/layer7_attention_weight_min": -69.08967590332031,
      "activations/layer8_attention_weight_max": 48.73330307006836,
      "activations/layer8_attention_weight_min": -51.20337677001953,
      "activations/layer9_attention_weight_max": 51.34914779663086,
      "activations/layer9_attention_weight_min": -46.280189514160156,
      "epoch": 20.26,
      "learning_rate": 1.9556818181818178e-05,
      "loss": 2.7171,
      "step": 348650
    },
    {
      "activations/layer0_attention_weight_max": 16.052474975585938,
      "activations/layer0_attention_weight_min": -14.004321098327637,
      "activations/layer10_attention_weight_max": 37.94478225708008,
      "activations/layer10_attention_weight_min": -35.50969314575195,
      "activations/layer11_attention_weight_max": 35.57375717163086,
      "activations/layer11_attention_weight_min": -35.90972137451172,
      "activations/layer12_attention_weight_max": 18.9359130859375,
      "activations/layer12_attention_weight_min": -24.692718505859375,
      "activations/layer13_attention_weight_max": 35.028297424316406,
      "activations/layer13_attention_weight_min": -29.287094116210938,
      "activations/layer14_attention_weight_max": 36.11461639404297,
      "activations/layer14_attention_weight_min": -31.841224670410156,
      "activations/layer15_attention_weight_max": 34.870849609375,
      "activations/layer15_attention_weight_min": -31.771984100341797,
      "activations/layer16_attention_weight_max": 34.60803985595703,
      "activations/layer16_attention_weight_min": -34.34099578857422,
      "activations/layer17_attention_weight_max": 51.23303985595703,
      "activations/layer17_attention_weight_min": -43.9189338684082,
      "activations/layer18_attention_weight_max": 47.24543380737305,
      "activations/layer18_attention_weight_min": -40.39577102661133,
      "activations/layer19_attention_weight_max": 27.07758903503418,
      "activations/layer19_attention_weight_min": -22.78922462463379,
      "activations/layer1_attention_weight_max": 15.254036903381348,
      "activations/layer1_attention_weight_min": -13.696005821228027,
      "activations/layer20_attention_weight_max": 24.086572647094727,
      "activations/layer20_attention_weight_min": -19.98145294189453,
      "activations/layer21_attention_weight_max": 39.08668899536133,
      "activations/layer21_attention_weight_min": -22.543169021606445,
      "activations/layer22_attention_weight_max": 30.26653289794922,
      "activations/layer22_attention_weight_min": -25.536008834838867,
      "activations/layer23_attention_weight_max": 36.89204788208008,
      "activations/layer23_attention_weight_min": -22.751562118530273,
      "activations/layer2_attention_weight_max": 33.07147979736328,
      "activations/layer2_attention_weight_min": -29.90428924560547,
      "activations/layer3_attention_weight_max": 90.17901611328125,
      "activations/layer3_attention_weight_min": -89.73925018310547,
      "activations/layer4_attention_weight_max": 86.11629486083984,
      "activations/layer4_attention_weight_min": -86.11231994628906,
      "activations/layer5_attention_weight_max": 66.06285095214844,
      "activations/layer5_attention_weight_min": -72.54269409179688,
      "activations/layer6_attention_weight_max": 52.59580993652344,
      "activations/layer6_attention_weight_min": -50.54110336303711,
      "activations/layer7_attention_weight_max": 69.08171081542969,
      "activations/layer7_attention_weight_min": -64.25530242919922,
      "activations/layer8_attention_weight_max": 47.24508285522461,
      "activations/layer8_attention_weight_min": -50.139137268066406,
      "activations/layer9_attention_weight_max": 44.2877311706543,
      "activations/layer9_attention_weight_min": -48.84580993652344,
      "epoch": 20.26,
      "learning_rate": 1.9538257575757576e-05,
      "loss": 2.7118,
      "step": 348700
    },
    {
      "activations/layer0_attention_weight_max": 15.74356746673584,
      "activations/layer0_attention_weight_min": -13.99254322052002,
      "activations/layer10_attention_weight_max": 37.963134765625,
      "activations/layer10_attention_weight_min": -38.449371337890625,
      "activations/layer11_attention_weight_max": 34.59050369262695,
      "activations/layer11_attention_weight_min": -36.86014175415039,
      "activations/layer12_attention_weight_max": 19.597490310668945,
      "activations/layer12_attention_weight_min": -26.984901428222656,
      "activations/layer13_attention_weight_max": 36.626060485839844,
      "activations/layer13_attention_weight_min": -27.940431594848633,
      "activations/layer14_attention_weight_max": 34.65423583984375,
      "activations/layer14_attention_weight_min": -31.572587966918945,
      "activations/layer15_attention_weight_max": 33.967994689941406,
      "activations/layer15_attention_weight_min": -30.845117568969727,
      "activations/layer16_attention_weight_max": 35.54943084716797,
      "activations/layer16_attention_weight_min": -31.589033126831055,
      "activations/layer17_attention_weight_max": 52.49518585205078,
      "activations/layer17_attention_weight_min": -45.825950622558594,
      "activations/layer18_attention_weight_max": 48.07124710083008,
      "activations/layer18_attention_weight_min": -42.150184631347656,
      "activations/layer19_attention_weight_max": 25.113887786865234,
      "activations/layer19_attention_weight_min": -24.405988693237305,
      "activations/layer1_attention_weight_max": 16.464576721191406,
      "activations/layer1_attention_weight_min": -13.46389102935791,
      "activations/layer20_attention_weight_max": 22.36285972595215,
      "activations/layer20_attention_weight_min": -21.647153854370117,
      "activations/layer21_attention_weight_max": 41.10246658325195,
      "activations/layer21_attention_weight_min": -25.608184814453125,
      "activations/layer22_attention_weight_max": 28.9698486328125,
      "activations/layer22_attention_weight_min": -25.97765350341797,
      "activations/layer23_attention_weight_max": 36.230804443359375,
      "activations/layer23_attention_weight_min": -25.296262741088867,
      "activations/layer2_attention_weight_max": 31.937957763671875,
      "activations/layer2_attention_weight_min": -30.095443725585938,
      "activations/layer3_attention_weight_max": 88.68077850341797,
      "activations/layer3_attention_weight_min": -88.28907012939453,
      "activations/layer4_attention_weight_max": 83.14386749267578,
      "activations/layer4_attention_weight_min": -85.10380554199219,
      "activations/layer5_attention_weight_max": 63.62230682373047,
      "activations/layer5_attention_weight_min": -69.18879699707031,
      "activations/layer6_attention_weight_max": 49.0798225402832,
      "activations/layer6_attention_weight_min": -50.08705520629883,
      "activations/layer7_attention_weight_max": 64.85265350341797,
      "activations/layer7_attention_weight_min": -65.35127258300781,
      "activations/layer8_attention_weight_max": 47.66048049926758,
      "activations/layer8_attention_weight_min": -52.38931655883789,
      "activations/layer9_attention_weight_max": 46.25632095336914,
      "activations/layer9_attention_weight_min": -46.907470703125,
      "epoch": 20.26,
      "learning_rate": 1.951931818181818e-05,
      "loss": 2.7253,
      "step": 348750
    },
    {
      "activations/layer0_attention_weight_max": 16.352806091308594,
      "activations/layer0_attention_weight_min": -14.150087356567383,
      "activations/layer10_attention_weight_max": 37.1627082824707,
      "activations/layer10_attention_weight_min": -33.953887939453125,
      "activations/layer11_attention_weight_max": 32.708980560302734,
      "activations/layer11_attention_weight_min": -33.59144592285156,
      "activations/layer12_attention_weight_max": 18.48030662536621,
      "activations/layer12_attention_weight_min": -23.360395431518555,
      "activations/layer13_attention_weight_max": 29.311433792114258,
      "activations/layer13_attention_weight_min": -27.89595603942871,
      "activations/layer14_attention_weight_max": 31.001144409179688,
      "activations/layer14_attention_weight_min": -29.13249397277832,
      "activations/layer15_attention_weight_max": 29.561279296875,
      "activations/layer15_attention_weight_min": -28.817594528198242,
      "activations/layer16_attention_weight_max": 31.222827911376953,
      "activations/layer16_attention_weight_min": -28.259626388549805,
      "activations/layer17_attention_weight_max": 44.37805938720703,
      "activations/layer17_attention_weight_min": -40.225337982177734,
      "activations/layer18_attention_weight_max": 43.08477020263672,
      "activations/layer18_attention_weight_min": -37.02019500732422,
      "activations/layer19_attention_weight_max": 23.326324462890625,
      "activations/layer19_attention_weight_min": -21.77470588684082,
      "activations/layer1_attention_weight_max": 15.77036190032959,
      "activations/layer1_attention_weight_min": -13.227360725402832,
      "activations/layer20_attention_weight_max": 20.754947662353516,
      "activations/layer20_attention_weight_min": -21.039140701293945,
      "activations/layer21_attention_weight_max": 32.99922561645508,
      "activations/layer21_attention_weight_min": -22.28830909729004,
      "activations/layer22_attention_weight_max": 32.99656295776367,
      "activations/layer22_attention_weight_min": -24.257484436035156,
      "activations/layer23_attention_weight_max": 34.74192810058594,
      "activations/layer23_attention_weight_min": -22.427955627441406,
      "activations/layer2_attention_weight_max": 32.312801361083984,
      "activations/layer2_attention_weight_min": -28.864707946777344,
      "activations/layer3_attention_weight_max": 83.85462188720703,
      "activations/layer3_attention_weight_min": -83.9174575805664,
      "activations/layer4_attention_weight_max": 82.61443328857422,
      "activations/layer4_attention_weight_min": -82.10880279541016,
      "activations/layer5_attention_weight_max": 58.063873291015625,
      "activations/layer5_attention_weight_min": -69.64903259277344,
      "activations/layer6_attention_weight_max": 46.476463317871094,
      "activations/layer6_attention_weight_min": -49.596595764160156,
      "activations/layer7_attention_weight_max": 69.26029968261719,
      "activations/layer7_attention_weight_min": -63.81789016723633,
      "activations/layer8_attention_weight_max": 46.64299011230469,
      "activations/layer8_attention_weight_min": -47.72304153442383,
      "activations/layer9_attention_weight_max": 50.347225189208984,
      "activations/layer9_attention_weight_min": -46.2877082824707,
      "epoch": 20.27,
      "learning_rate": 1.9500378787878786e-05,
      "loss": 2.7327,
      "step": 348800
    },
    {
      "activations/layer0_attention_weight_max": 16.16912269592285,
      "activations/layer0_attention_weight_min": -13.780926704406738,
      "activations/layer10_attention_weight_max": 40.51709747314453,
      "activations/layer10_attention_weight_min": -39.33271026611328,
      "activations/layer11_attention_weight_max": 35.069740295410156,
      "activations/layer11_attention_weight_min": -36.0993766784668,
      "activations/layer12_attention_weight_max": 21.06580924987793,
      "activations/layer12_attention_weight_min": -26.61884307861328,
      "activations/layer13_attention_weight_max": 44.472225189208984,
      "activations/layer13_attention_weight_min": -31.892986297607422,
      "activations/layer14_attention_weight_max": 42.94358444213867,
      "activations/layer14_attention_weight_min": -33.00318145751953,
      "activations/layer15_attention_weight_max": 37.93861770629883,
      "activations/layer15_attention_weight_min": -32.006736755371094,
      "activations/layer16_attention_weight_max": 42.39303970336914,
      "activations/layer16_attention_weight_min": -37.8222770690918,
      "activations/layer17_attention_weight_max": 59.49888229370117,
      "activations/layer17_attention_weight_min": -52.277565002441406,
      "activations/layer18_attention_weight_max": 56.28757858276367,
      "activations/layer18_attention_weight_min": -47.47874450683594,
      "activations/layer19_attention_weight_max": 30.191925048828125,
      "activations/layer19_attention_weight_min": -28.297210693359375,
      "activations/layer1_attention_weight_max": 15.921810150146484,
      "activations/layer1_attention_weight_min": -16.467506408691406,
      "activations/layer20_attention_weight_max": 29.71407127380371,
      "activations/layer20_attention_weight_min": -24.707386016845703,
      "activations/layer21_attention_weight_max": 44.35660934448242,
      "activations/layer21_attention_weight_min": -23.910728454589844,
      "activations/layer22_attention_weight_max": 34.654903411865234,
      "activations/layer22_attention_weight_min": -27.666690826416016,
      "activations/layer23_attention_weight_max": 42.29667663574219,
      "activations/layer23_attention_weight_min": -25.6402587890625,
      "activations/layer2_attention_weight_max": 33.687625885009766,
      "activations/layer2_attention_weight_min": -32.92566680908203,
      "activations/layer3_attention_weight_max": 90.32250213623047,
      "activations/layer3_attention_weight_min": -89.8149642944336,
      "activations/layer4_attention_weight_max": 83.25982666015625,
      "activations/layer4_attention_weight_min": -83.256591796875,
      "activations/layer5_attention_weight_max": 58.87255859375,
      "activations/layer5_attention_weight_min": -71.78648376464844,
      "activations/layer6_attention_weight_max": 48.20162582397461,
      "activations/layer6_attention_weight_min": -49.43351364135742,
      "activations/layer7_attention_weight_max": 65.57234954833984,
      "activations/layer7_attention_weight_min": -64.70449829101562,
      "activations/layer8_attention_weight_max": 51.43436050415039,
      "activations/layer8_attention_weight_min": -53.133968353271484,
      "activations/layer9_attention_weight_max": 52.327972412109375,
      "activations/layer9_attention_weight_min": -47.90616989135742,
      "epoch": 20.27,
      "learning_rate": 1.948143939393939e-05,
      "loss": 2.716,
      "step": 348850
    },
    {
      "activations/layer0_attention_weight_max": 15.70982551574707,
      "activations/layer0_attention_weight_min": -14.643112182617188,
      "activations/layer10_attention_weight_max": 35.026397705078125,
      "activations/layer10_attention_weight_min": -37.015933990478516,
      "activations/layer11_attention_weight_max": 32.66754913330078,
      "activations/layer11_attention_weight_min": -36.12907409667969,
      "activations/layer12_attention_weight_max": 20.065658569335938,
      "activations/layer12_attention_weight_min": -26.59263801574707,
      "activations/layer13_attention_weight_max": 33.40290069580078,
      "activations/layer13_attention_weight_min": -28.64752197265625,
      "activations/layer14_attention_weight_max": 36.26958084106445,
      "activations/layer14_attention_weight_min": -30.731645584106445,
      "activations/layer15_attention_weight_max": 33.87298583984375,
      "activations/layer15_attention_weight_min": -31.63277816772461,
      "activations/layer16_attention_weight_max": 37.174339294433594,
      "activations/layer16_attention_weight_min": -33.00900650024414,
      "activations/layer17_attention_weight_max": 54.68136215209961,
      "activations/layer17_attention_weight_min": -45.9548225402832,
      "activations/layer18_attention_weight_max": 51.15461730957031,
      "activations/layer18_attention_weight_min": -39.928035736083984,
      "activations/layer19_attention_weight_max": 25.2447452545166,
      "activations/layer19_attention_weight_min": -23.65058708190918,
      "activations/layer1_attention_weight_max": 14.939207077026367,
      "activations/layer1_attention_weight_min": -14.317216873168945,
      "activations/layer20_attention_weight_max": 25.112197875976562,
      "activations/layer20_attention_weight_min": -21.446260452270508,
      "activations/layer21_attention_weight_max": 37.274749755859375,
      "activations/layer21_attention_weight_min": -20.260873794555664,
      "activations/layer22_attention_weight_max": 30.91141128540039,
      "activations/layer22_attention_weight_min": -23.97793197631836,
      "activations/layer23_attention_weight_max": 32.99488830566406,
      "activations/layer23_attention_weight_min": -23.247419357299805,
      "activations/layer2_attention_weight_max": 32.05707931518555,
      "activations/layer2_attention_weight_min": -30.036544799804688,
      "activations/layer3_attention_weight_max": 83.8019027709961,
      "activations/layer3_attention_weight_min": -84.29045867919922,
      "activations/layer4_attention_weight_max": 83.3203353881836,
      "activations/layer4_attention_weight_min": -84.90386199951172,
      "activations/layer5_attention_weight_max": 60.89286804199219,
      "activations/layer5_attention_weight_min": -68.94961547851562,
      "activations/layer6_attention_weight_max": 48.87082290649414,
      "activations/layer6_attention_weight_min": -51.839656829833984,
      "activations/layer7_attention_weight_max": 65.81763458251953,
      "activations/layer7_attention_weight_min": -64.28348541259766,
      "activations/layer8_attention_weight_max": 46.51942825317383,
      "activations/layer8_attention_weight_min": -51.60852813720703,
      "activations/layer9_attention_weight_max": 46.560787200927734,
      "activations/layer9_attention_weight_min": -45.93073654174805,
      "epoch": 20.27,
      "learning_rate": 1.94625e-05,
      "loss": 2.7228,
      "step": 348900
    },
    {
      "activations/layer0_attention_weight_max": 16.447622299194336,
      "activations/layer0_attention_weight_min": -13.792141914367676,
      "activations/layer10_attention_weight_max": 34.80501174926758,
      "activations/layer10_attention_weight_min": -36.010414123535156,
      "activations/layer11_attention_weight_max": 32.56757736206055,
      "activations/layer11_attention_weight_min": -33.68170166015625,
      "activations/layer12_attention_weight_max": 18.992813110351562,
      "activations/layer12_attention_weight_min": -26.86846351623535,
      "activations/layer13_attention_weight_max": 32.4852180480957,
      "activations/layer13_attention_weight_min": -26.35399627685547,
      "activations/layer14_attention_weight_max": 36.82985305786133,
      "activations/layer14_attention_weight_min": -27.96541404724121,
      "activations/layer15_attention_weight_max": 32.3050537109375,
      "activations/layer15_attention_weight_min": -28.759855270385742,
      "activations/layer16_attention_weight_max": 35.31608581542969,
      "activations/layer16_attention_weight_min": -31.192171096801758,
      "activations/layer17_attention_weight_max": 48.29072570800781,
      "activations/layer17_attention_weight_min": -40.878326416015625,
      "activations/layer18_attention_weight_max": 47.03456497192383,
      "activations/layer18_attention_weight_min": -37.99439239501953,
      "activations/layer19_attention_weight_max": 24.37526512145996,
      "activations/layer19_attention_weight_min": -20.616596221923828,
      "activations/layer1_attention_weight_max": 15.380640029907227,
      "activations/layer1_attention_weight_min": -13.973387718200684,
      "activations/layer20_attention_weight_max": 23.995147705078125,
      "activations/layer20_attention_weight_min": -19.996679306030273,
      "activations/layer21_attention_weight_max": 32.25482940673828,
      "activations/layer21_attention_weight_min": -21.811038970947266,
      "activations/layer22_attention_weight_max": 32.2546501159668,
      "activations/layer22_attention_weight_min": -25.0822811126709,
      "activations/layer23_attention_weight_max": 35.721458435058594,
      "activations/layer23_attention_weight_min": -23.28704833984375,
      "activations/layer2_attention_weight_max": 32.8206901550293,
      "activations/layer2_attention_weight_min": -30.187252044677734,
      "activations/layer3_attention_weight_max": 90.3282699584961,
      "activations/layer3_attention_weight_min": -87.0230484008789,
      "activations/layer4_attention_weight_max": 80.47270202636719,
      "activations/layer4_attention_weight_min": -79.52359008789062,
      "activations/layer5_attention_weight_max": 64.00834655761719,
      "activations/layer5_attention_weight_min": -66.41704559326172,
      "activations/layer6_attention_weight_max": 49.411441802978516,
      "activations/layer6_attention_weight_min": -49.40161895751953,
      "activations/layer7_attention_weight_max": 63.478458404541016,
      "activations/layer7_attention_weight_min": -59.915008544921875,
      "activations/layer8_attention_weight_max": 46.902645111083984,
      "activations/layer8_attention_weight_min": -49.38361358642578,
      "activations/layer9_attention_weight_max": 45.3416862487793,
      "activations/layer9_attention_weight_min": -43.25797653198242,
      "epoch": 20.28,
      "learning_rate": 1.9443560606060604e-05,
      "loss": 2.7135,
      "step": 348950
    },
    {
      "activations/layer0_attention_weight_max": 15.297192573547363,
      "activations/layer0_attention_weight_min": -13.69432258605957,
      "activations/layer10_attention_weight_max": 35.394622802734375,
      "activations/layer10_attention_weight_min": -37.44467544555664,
      "activations/layer11_attention_weight_max": 33.47821044921875,
      "activations/layer11_attention_weight_min": -34.130916595458984,
      "activations/layer12_attention_weight_max": 20.405752182006836,
      "activations/layer12_attention_weight_min": -25.88223648071289,
      "activations/layer13_attention_weight_max": 40.618019104003906,
      "activations/layer13_attention_weight_min": -29.266794204711914,
      "activations/layer14_attention_weight_max": 36.46309280395508,
      "activations/layer14_attention_weight_min": -30.347986221313477,
      "activations/layer15_attention_weight_max": 34.656436920166016,
      "activations/layer15_attention_weight_min": -31.638572692871094,
      "activations/layer16_attention_weight_max": 37.301029205322266,
      "activations/layer16_attention_weight_min": -35.68333435058594,
      "activations/layer17_attention_weight_max": 55.987632751464844,
      "activations/layer17_attention_weight_min": -50.30923080444336,
      "activations/layer18_attention_weight_max": 51.6272087097168,
      "activations/layer18_attention_weight_min": -43.82051086425781,
      "activations/layer19_attention_weight_max": 25.547849655151367,
      "activations/layer19_attention_weight_min": -24.91376495361328,
      "activations/layer1_attention_weight_max": 15.61921501159668,
      "activations/layer1_attention_weight_min": -14.117895126342773,
      "activations/layer20_attention_weight_max": 24.442405700683594,
      "activations/layer20_attention_weight_min": -21.940677642822266,
      "activations/layer21_attention_weight_max": 34.914764404296875,
      "activations/layer21_attention_weight_min": -24.22825050354004,
      "activations/layer22_attention_weight_max": 28.4815616607666,
      "activations/layer22_attention_weight_min": -25.407854080200195,
      "activations/layer23_attention_weight_max": 37.13521194458008,
      "activations/layer23_attention_weight_min": -24.281078338623047,
      "activations/layer2_attention_weight_max": 30.35616111755371,
      "activations/layer2_attention_weight_min": -29.502384185791016,
      "activations/layer3_attention_weight_max": 83.80957794189453,
      "activations/layer3_attention_weight_min": -85.12057495117188,
      "activations/layer4_attention_weight_max": 84.39794158935547,
      "activations/layer4_attention_weight_min": -84.52886199951172,
      "activations/layer5_attention_weight_max": 62.02791976928711,
      "activations/layer5_attention_weight_min": -73.30249786376953,
      "activations/layer6_attention_weight_max": 49.7178840637207,
      "activations/layer6_attention_weight_min": -49.56517028808594,
      "activations/layer7_attention_weight_max": 64.17550659179688,
      "activations/layer7_attention_weight_min": -63.4995002746582,
      "activations/layer8_attention_weight_max": 45.958003997802734,
      "activations/layer8_attention_weight_min": -50.19327163696289,
      "activations/layer9_attention_weight_max": 42.73695373535156,
      "activations/layer9_attention_weight_min": -47.537601470947266,
      "epoch": 20.28,
      "learning_rate": 1.9424621212121212e-05,
      "loss": 2.7117,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.4676,
      "eval_samples_per_second": 507.111,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.4676,
      "eval_openwebtext_samples_per_second": 507.111,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 2.0165,
      "eval_wikitext_samples_per_second": 226.14,
      "step": 349000
    },
    {
      "epoch": 20.28,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.6697,
      "eval_lambada_samples_per_second": 503.534,
      "step": 349000
    },
    {
      "activations/layer0_attention_weight_max": 15.875970840454102,
      "activations/layer0_attention_weight_min": -14.18260383605957,
      "activations/layer10_attention_weight_max": 36.397090911865234,
      "activations/layer10_attention_weight_min": -34.998191833496094,
      "activations/layer11_attention_weight_max": 36.38964080810547,
      "activations/layer11_attention_weight_min": -35.53718566894531,
      "activations/layer12_attention_weight_max": 19.60749626159668,
      "activations/layer12_attention_weight_min": -28.427228927612305,
      "activations/layer13_attention_weight_max": 37.63270568847656,
      "activations/layer13_attention_weight_min": -27.778549194335938,
      "activations/layer14_attention_weight_max": 37.82841110229492,
      "activations/layer14_attention_weight_min": -30.89100456237793,
      "activations/layer15_attention_weight_max": 34.199127197265625,
      "activations/layer15_attention_weight_min": -32.85251998901367,
      "activations/layer16_attention_weight_max": 34.804439544677734,
      "activations/layer16_attention_weight_min": -33.65775680541992,
      "activations/layer17_attention_weight_max": 51.8292121887207,
      "activations/layer17_attention_weight_min": -49.20241165161133,
      "activations/layer18_attention_weight_max": 48.5064811706543,
      "activations/layer18_attention_weight_min": -46.50289535522461,
      "activations/layer19_attention_weight_max": 23.574872970581055,
      "activations/layer19_attention_weight_min": -23.627857208251953,
      "activations/layer1_attention_weight_max": 15.344587326049805,
      "activations/layer1_attention_weight_min": -13.45003604888916,
      "activations/layer20_attention_weight_max": 23.151487350463867,
      "activations/layer20_attention_weight_min": -22.385831832885742,
      "activations/layer21_attention_weight_max": 38.122222900390625,
      "activations/layer21_attention_weight_min": -25.32244300842285,
      "activations/layer22_attention_weight_max": 28.323410034179688,
      "activations/layer22_attention_weight_min": -25.56146240234375,
      "activations/layer23_attention_weight_max": 38.406089782714844,
      "activations/layer23_attention_weight_min": -22.130407333374023,
      "activations/layer2_attention_weight_max": 31.32754898071289,
      "activations/layer2_attention_weight_min": -29.72327423095703,
      "activations/layer3_attention_weight_max": 88.27082824707031,
      "activations/layer3_attention_weight_min": -84.98169708251953,
      "activations/layer4_attention_weight_max": 79.79439544677734,
      "activations/layer4_attention_weight_min": -80.70149230957031,
      "activations/layer5_attention_weight_max": 58.362972259521484,
      "activations/layer5_attention_weight_min": -72.57080841064453,
      "activations/layer6_attention_weight_max": 46.35969924926758,
      "activations/layer6_attention_weight_min": -47.96875762939453,
      "activations/layer7_attention_weight_max": 70.72241973876953,
      "activations/layer7_attention_weight_min": -63.3499870300293,
      "activations/layer8_attention_weight_max": 48.32506561279297,
      "activations/layer8_attention_weight_min": -48.986167907714844,
      "activations/layer9_attention_weight_max": 52.90309524536133,
      "activations/layer9_attention_weight_min": -45.89316177368164,
      "epoch": 20.28,
      "learning_rate": 1.9405681818181813e-05,
      "loss": 2.7251,
      "step": 349050
    },
    {
      "activations/layer0_attention_weight_max": 15.864668846130371,
      "activations/layer0_attention_weight_min": -14.173481941223145,
      "activations/layer10_attention_weight_max": 35.96187973022461,
      "activations/layer10_attention_weight_min": -36.158538818359375,
      "activations/layer11_attention_weight_max": 33.310585021972656,
      "activations/layer11_attention_weight_min": -33.5890998840332,
      "activations/layer12_attention_weight_max": 17.71548843383789,
      "activations/layer12_attention_weight_min": -24.60531234741211,
      "activations/layer13_attention_weight_max": 31.788747787475586,
      "activations/layer13_attention_weight_min": -27.920259475708008,
      "activations/layer14_attention_weight_max": 32.73902130126953,
      "activations/layer14_attention_weight_min": -30.780717849731445,
      "activations/layer15_attention_weight_max": 29.9206600189209,
      "activations/layer15_attention_weight_min": -31.523008346557617,
      "activations/layer16_attention_weight_max": 31.096755981445312,
      "activations/layer16_attention_weight_min": -31.03247833251953,
      "activations/layer17_attention_weight_max": 47.87254333496094,
      "activations/layer17_attention_weight_min": -45.59735870361328,
      "activations/layer18_attention_weight_max": 43.34846878051758,
      "activations/layer18_attention_weight_min": -38.41136932373047,
      "activations/layer19_attention_weight_max": 24.813899993896484,
      "activations/layer19_attention_weight_min": -22.354991912841797,
      "activations/layer1_attention_weight_max": 15.899884223937988,
      "activations/layer1_attention_weight_min": -13.849699020385742,
      "activations/layer20_attention_weight_max": 22.80497169494629,
      "activations/layer20_attention_weight_min": -21.316587448120117,
      "activations/layer21_attention_weight_max": 32.87847900390625,
      "activations/layer21_attention_weight_min": -20.72206687927246,
      "activations/layer22_attention_weight_max": 29.249225616455078,
      "activations/layer22_attention_weight_min": -25.704261779785156,
      "activations/layer23_attention_weight_max": 33.819793701171875,
      "activations/layer23_attention_weight_min": -24.39480209350586,
      "activations/layer2_attention_weight_max": 31.413341522216797,
      "activations/layer2_attention_weight_min": -28.937759399414062,
      "activations/layer3_attention_weight_max": 86.1363296508789,
      "activations/layer3_attention_weight_min": -83.93727111816406,
      "activations/layer4_attention_weight_max": 83.40753173828125,
      "activations/layer4_attention_weight_min": -81.32189178466797,
      "activations/layer5_attention_weight_max": 60.49248504638672,
      "activations/layer5_attention_weight_min": -69.75479125976562,
      "activations/layer6_attention_weight_max": 47.176673889160156,
      "activations/layer6_attention_weight_min": -49.38054656982422,
      "activations/layer7_attention_weight_max": 65.71611785888672,
      "activations/layer7_attention_weight_min": -62.26057815551758,
      "activations/layer8_attention_weight_max": 46.72807693481445,
      "activations/layer8_attention_weight_min": -49.54509353637695,
      "activations/layer9_attention_weight_max": 43.232181549072266,
      "activations/layer9_attention_weight_min": -47.4732780456543,
      "epoch": 20.28,
      "learning_rate": 1.938674242424242e-05,
      "loss": 2.7271,
      "step": 349100
    },
    {
      "activations/layer0_attention_weight_max": 15.660356521606445,
      "activations/layer0_attention_weight_min": -13.950480461120605,
      "activations/layer10_attention_weight_max": 40.93711471557617,
      "activations/layer10_attention_weight_min": -38.49591827392578,
      "activations/layer11_attention_weight_max": 43.81658172607422,
      "activations/layer11_attention_weight_min": -38.653717041015625,
      "activations/layer12_attention_weight_max": 19.333284378051758,
      "activations/layer12_attention_weight_min": -23.616207122802734,
      "activations/layer13_attention_weight_max": 33.7076416015625,
      "activations/layer13_attention_weight_min": -29.789466857910156,
      "activations/layer14_attention_weight_max": 35.35359573364258,
      "activations/layer14_attention_weight_min": -30.208993911743164,
      "activations/layer15_attention_weight_max": 36.178009033203125,
      "activations/layer15_attention_weight_min": -30.273962020874023,
      "activations/layer16_attention_weight_max": 34.19976043701172,
      "activations/layer16_attention_weight_min": -33.351009368896484,
      "activations/layer17_attention_weight_max": 50.834957122802734,
      "activations/layer17_attention_weight_min": -44.32441711425781,
      "activations/layer18_attention_weight_max": 48.33414840698242,
      "activations/layer18_attention_weight_min": -40.80303955078125,
      "activations/layer19_attention_weight_max": 25.81100845336914,
      "activations/layer19_attention_weight_min": -24.142423629760742,
      "activations/layer1_attention_weight_max": 15.836469650268555,
      "activations/layer1_attention_weight_min": -13.713467597961426,
      "activations/layer20_attention_weight_max": 23.925071716308594,
      "activations/layer20_attention_weight_min": -20.851238250732422,
      "activations/layer21_attention_weight_max": 37.577491760253906,
      "activations/layer21_attention_weight_min": -22.334381103515625,
      "activations/layer22_attention_weight_max": 30.50585174560547,
      "activations/layer22_attention_weight_min": -25.89402961730957,
      "activations/layer23_attention_weight_max": 38.69518280029297,
      "activations/layer23_attention_weight_min": -25.45745849609375,
      "activations/layer2_attention_weight_max": 32.31817626953125,
      "activations/layer2_attention_weight_min": -30.41962432861328,
      "activations/layer3_attention_weight_max": 85.1830062866211,
      "activations/layer3_attention_weight_min": -87.1922607421875,
      "activations/layer4_attention_weight_max": 83.81473541259766,
      "activations/layer4_attention_weight_min": -85.70516204833984,
      "activations/layer5_attention_weight_max": 64.20368957519531,
      "activations/layer5_attention_weight_min": -69.99883270263672,
      "activations/layer6_attention_weight_max": 48.30281448364258,
      "activations/layer6_attention_weight_min": -50.78609085083008,
      "activations/layer7_attention_weight_max": 73.5988540649414,
      "activations/layer7_attention_weight_min": -66.6805419921875,
      "activations/layer8_attention_weight_max": 54.19789505004883,
      "activations/layer8_attention_weight_min": -52.56678009033203,
      "activations/layer9_attention_weight_max": 58.919952392578125,
      "activations/layer9_attention_weight_min": -46.940528869628906,
      "epoch": 20.29,
      "learning_rate": 1.9367803030303027e-05,
      "loss": 2.7274,
      "step": 349150
    },
    {
      "activations/layer0_attention_weight_max": 16.320072174072266,
      "activations/layer0_attention_weight_min": -13.926105499267578,
      "activations/layer10_attention_weight_max": 35.879608154296875,
      "activations/layer10_attention_weight_min": -35.382057189941406,
      "activations/layer11_attention_weight_max": 33.85623550415039,
      "activations/layer11_attention_weight_min": -33.86048889160156,
      "activations/layer12_attention_weight_max": 20.070951461791992,
      "activations/layer12_attention_weight_min": -25.429779052734375,
      "activations/layer13_attention_weight_max": 41.247413635253906,
      "activations/layer13_attention_weight_min": -31.96205711364746,
      "activations/layer14_attention_weight_max": 36.5741081237793,
      "activations/layer14_attention_weight_min": -28.180004119873047,
      "activations/layer15_attention_weight_max": 33.721534729003906,
      "activations/layer15_attention_weight_min": -28.42086410522461,
      "activations/layer16_attention_weight_max": 33.895755767822266,
      "activations/layer16_attention_weight_min": -32.29629135131836,
      "activations/layer17_attention_weight_max": 55.09621047973633,
      "activations/layer17_attention_weight_min": -44.19319152832031,
      "activations/layer18_attention_weight_max": 47.89940643310547,
      "activations/layer18_attention_weight_min": -37.69783020019531,
      "activations/layer19_attention_weight_max": 24.476604461669922,
      "activations/layer19_attention_weight_min": -22.029508590698242,
      "activations/layer1_attention_weight_max": 15.737319946289062,
      "activations/layer1_attention_weight_min": -14.062479972839355,
      "activations/layer20_attention_weight_max": 21.97054100036621,
      "activations/layer20_attention_weight_min": -19.66971206665039,
      "activations/layer21_attention_weight_max": 32.68220901489258,
      "activations/layer21_attention_weight_min": -20.736265182495117,
      "activations/layer22_attention_weight_max": 27.8648624420166,
      "activations/layer22_attention_weight_min": -26.1860408782959,
      "activations/layer23_attention_weight_max": 35.93940353393555,
      "activations/layer23_attention_weight_min": -23.61294937133789,
      "activations/layer2_attention_weight_max": 32.13166427612305,
      "activations/layer2_attention_weight_min": -30.651321411132812,
      "activations/layer3_attention_weight_max": 83.38957214355469,
      "activations/layer3_attention_weight_min": -86.20195007324219,
      "activations/layer4_attention_weight_max": 81.1971206665039,
      "activations/layer4_attention_weight_min": -79.77676391601562,
      "activations/layer5_attention_weight_max": 62.90804672241211,
      "activations/layer5_attention_weight_min": -66.15531158447266,
      "activations/layer6_attention_weight_max": 50.87870407104492,
      "activations/layer6_attention_weight_min": -49.45655822753906,
      "activations/layer7_attention_weight_max": 65.02049255371094,
      "activations/layer7_attention_weight_min": -65.44744873046875,
      "activations/layer8_attention_weight_max": 49.63459396362305,
      "activations/layer8_attention_weight_min": -52.99195098876953,
      "activations/layer9_attention_weight_max": 48.92724609375,
      "activations/layer9_attention_weight_min": -45.43878173828125,
      "epoch": 20.29,
      "learning_rate": 1.9348863636363635e-05,
      "loss": 2.7202,
      "step": 349200
    },
    {
      "activations/layer0_attention_weight_max": 15.807844161987305,
      "activations/layer0_attention_weight_min": -13.708503723144531,
      "activations/layer10_attention_weight_max": 42.63612747192383,
      "activations/layer10_attention_weight_min": -38.151939392089844,
      "activations/layer11_attention_weight_max": 36.55576705932617,
      "activations/layer11_attention_weight_min": -37.943580627441406,
      "activations/layer12_attention_weight_max": 20.03652572631836,
      "activations/layer12_attention_weight_min": -27.11483383178711,
      "activations/layer13_attention_weight_max": 38.55971145629883,
      "activations/layer13_attention_weight_min": -27.19202995300293,
      "activations/layer14_attention_weight_max": 39.11324691772461,
      "activations/layer14_attention_weight_min": -31.72736930847168,
      "activations/layer15_attention_weight_max": 36.68667984008789,
      "activations/layer15_attention_weight_min": -31.4760799407959,
      "activations/layer16_attention_weight_max": 36.10051345825195,
      "activations/layer16_attention_weight_min": -32.53737258911133,
      "activations/layer17_attention_weight_max": 54.6792106628418,
      "activations/layer17_attention_weight_min": -46.0074348449707,
      "activations/layer18_attention_weight_max": 49.703956604003906,
      "activations/layer18_attention_weight_min": -42.8335075378418,
      "activations/layer19_attention_weight_max": 26.1590633392334,
      "activations/layer19_attention_weight_min": -23.19231605529785,
      "activations/layer1_attention_weight_max": 15.800387382507324,
      "activations/layer1_attention_weight_min": -12.66326904296875,
      "activations/layer20_attention_weight_max": 23.470300674438477,
      "activations/layer20_attention_weight_min": -23.282947540283203,
      "activations/layer21_attention_weight_max": 35.542789459228516,
      "activations/layer21_attention_weight_min": -22.901397705078125,
      "activations/layer22_attention_weight_max": 31.420021057128906,
      "activations/layer22_attention_weight_min": -25.991926193237305,
      "activations/layer23_attention_weight_max": 37.50061798095703,
      "activations/layer23_attention_weight_min": -22.634557723999023,
      "activations/layer2_attention_weight_max": 30.359439849853516,
      "activations/layer2_attention_weight_min": -29.388690948486328,
      "activations/layer3_attention_weight_max": 87.36929321289062,
      "activations/layer3_attention_weight_min": -87.46128845214844,
      "activations/layer4_attention_weight_max": 88.1730728149414,
      "activations/layer4_attention_weight_min": -85.85366821289062,
      "activations/layer5_attention_weight_max": 63.702003479003906,
      "activations/layer5_attention_weight_min": -69.2062759399414,
      "activations/layer6_attention_weight_max": 50.3738899230957,
      "activations/layer6_attention_weight_min": -51.571861267089844,
      "activations/layer7_attention_weight_max": 71.42955780029297,
      "activations/layer7_attention_weight_min": -63.93701934814453,
      "activations/layer8_attention_weight_max": 50.31313705444336,
      "activations/layer8_attention_weight_min": -52.75735855102539,
      "activations/layer9_attention_weight_max": 46.737789154052734,
      "activations/layer9_attention_weight_min": -48.14786911010742,
      "epoch": 20.29,
      "learning_rate": 1.9329924242424243e-05,
      "loss": 2.7125,
      "step": 349250
    },
    {
      "activations/layer0_attention_weight_max": 16.464147567749023,
      "activations/layer0_attention_weight_min": -14.380047798156738,
      "activations/layer10_attention_weight_max": 35.721614837646484,
      "activations/layer10_attention_weight_min": -34.53578186035156,
      "activations/layer11_attention_weight_max": 32.37483215332031,
      "activations/layer11_attention_weight_min": -33.346473693847656,
      "activations/layer12_attention_weight_max": 18.874006271362305,
      "activations/layer12_attention_weight_min": -21.49921989440918,
      "activations/layer13_attention_weight_max": 39.85902404785156,
      "activations/layer13_attention_weight_min": -30.206363677978516,
      "activations/layer14_attention_weight_max": 36.146209716796875,
      "activations/layer14_attention_weight_min": -28.97248077392578,
      "activations/layer15_attention_weight_max": 33.19119644165039,
      "activations/layer15_attention_weight_min": -29.830175399780273,
      "activations/layer16_attention_weight_max": 35.36240005493164,
      "activations/layer16_attention_weight_min": -34.206573486328125,
      "activations/layer17_attention_weight_max": 51.60424041748047,
      "activations/layer17_attention_weight_min": -46.0123176574707,
      "activations/layer18_attention_weight_max": 46.646663665771484,
      "activations/layer18_attention_weight_min": -41.57075500488281,
      "activations/layer19_attention_weight_max": 24.26976776123047,
      "activations/layer19_attention_weight_min": -25.01851463317871,
      "activations/layer1_attention_weight_max": 15.6384916305542,
      "activations/layer1_attention_weight_min": -13.557296752929688,
      "activations/layer20_attention_weight_max": 22.06386375427246,
      "activations/layer20_attention_weight_min": -22.57622528076172,
      "activations/layer21_attention_weight_max": 33.322303771972656,
      "activations/layer21_attention_weight_min": -24.184282302856445,
      "activations/layer22_attention_weight_max": 27.24544334411621,
      "activations/layer22_attention_weight_min": -29.193065643310547,
      "activations/layer23_attention_weight_max": 35.266578674316406,
      "activations/layer23_attention_weight_min": -25.37520980834961,
      "activations/layer2_attention_weight_max": 31.539337158203125,
      "activations/layer2_attention_weight_min": -30.76773452758789,
      "activations/layer3_attention_weight_max": 86.19034576416016,
      "activations/layer3_attention_weight_min": -89.89080047607422,
      "activations/layer4_attention_weight_max": 83.46859741210938,
      "activations/layer4_attention_weight_min": -88.01436614990234,
      "activations/layer5_attention_weight_max": 62.83421325683594,
      "activations/layer5_attention_weight_min": -71.59831237792969,
      "activations/layer6_attention_weight_max": 48.026668548583984,
      "activations/layer6_attention_weight_min": -48.993289947509766,
      "activations/layer7_attention_weight_max": 63.45162582397461,
      "activations/layer7_attention_weight_min": -66.33623504638672,
      "activations/layer8_attention_weight_max": 45.37221908569336,
      "activations/layer8_attention_weight_min": -49.18509292602539,
      "activations/layer9_attention_weight_max": 42.158851623535156,
      "activations/layer9_attention_weight_min": -46.026058197021484,
      "epoch": 20.3,
      "learning_rate": 1.9310984848484848e-05,
      "loss": 2.7357,
      "step": 349300
    },
    {
      "activations/layer0_attention_weight_max": 15.62867259979248,
      "activations/layer0_attention_weight_min": -13.855774879455566,
      "activations/layer10_attention_weight_max": 33.529815673828125,
      "activations/layer10_attention_weight_min": -34.6351432800293,
      "activations/layer11_attention_weight_max": 31.653791427612305,
      "activations/layer11_attention_weight_min": -34.51515579223633,
      "activations/layer12_attention_weight_max": 19.969762802124023,
      "activations/layer12_attention_weight_min": -26.07322120666504,
      "activations/layer13_attention_weight_max": 32.89971160888672,
      "activations/layer13_attention_weight_min": -29.236530303955078,
      "activations/layer14_attention_weight_max": 37.599708557128906,
      "activations/layer14_attention_weight_min": -32.300025939941406,
      "activations/layer15_attention_weight_max": 35.75440216064453,
      "activations/layer15_attention_weight_min": -32.64201736450195,
      "activations/layer16_attention_weight_max": 38.59920883178711,
      "activations/layer16_attention_weight_min": -34.22555923461914,
      "activations/layer17_attention_weight_max": 54.87903594970703,
      "activations/layer17_attention_weight_min": -48.36582565307617,
      "activations/layer18_attention_weight_max": 51.92768859863281,
      "activations/layer18_attention_weight_min": -45.55183792114258,
      "activations/layer19_attention_weight_max": 27.11627769470215,
      "activations/layer19_attention_weight_min": -25.9134521484375,
      "activations/layer1_attention_weight_max": 14.981237411499023,
      "activations/layer1_attention_weight_min": -13.733677864074707,
      "activations/layer20_attention_weight_max": 27.4984073638916,
      "activations/layer20_attention_weight_min": -22.270936965942383,
      "activations/layer21_attention_weight_max": 35.1713981628418,
      "activations/layer21_attention_weight_min": -23.938907623291016,
      "activations/layer22_attention_weight_max": 32.09934616088867,
      "activations/layer22_attention_weight_min": -27.109235763549805,
      "activations/layer23_attention_weight_max": 38.09185028076172,
      "activations/layer23_attention_weight_min": -24.099510192871094,
      "activations/layer2_attention_weight_max": 30.305395126342773,
      "activations/layer2_attention_weight_min": -31.763263702392578,
      "activations/layer3_attention_weight_max": 86.45450592041016,
      "activations/layer3_attention_weight_min": -82.6941146850586,
      "activations/layer4_attention_weight_max": 82.43648529052734,
      "activations/layer4_attention_weight_min": -83.8265151977539,
      "activations/layer5_attention_weight_max": 59.266658782958984,
      "activations/layer5_attention_weight_min": -69.87022399902344,
      "activations/layer6_attention_weight_max": 46.17180252075195,
      "activations/layer6_attention_weight_min": -47.80636978149414,
      "activations/layer7_attention_weight_max": 63.80137634277344,
      "activations/layer7_attention_weight_min": -61.80714416503906,
      "activations/layer8_attention_weight_max": 43.00339126586914,
      "activations/layer8_attention_weight_min": -47.54930114746094,
      "activations/layer9_attention_weight_max": 41.9503173828125,
      "activations/layer9_attention_weight_min": -45.74267578125,
      "epoch": 20.3,
      "learning_rate": 1.9292045454545453e-05,
      "loss": 2.7128,
      "step": 349350
    },
    {
      "activations/layer0_attention_weight_max": 16.003625869750977,
      "activations/layer0_attention_weight_min": -13.720196723937988,
      "activations/layer10_attention_weight_max": 33.548561096191406,
      "activations/layer10_attention_weight_min": -32.80192565917969,
      "activations/layer11_attention_weight_max": 32.85075759887695,
      "activations/layer11_attention_weight_min": -32.5738525390625,
      "activations/layer12_attention_weight_max": 20.14523696899414,
      "activations/layer12_attention_weight_min": -22.07616424560547,
      "activations/layer13_attention_weight_max": 34.27574157714844,
      "activations/layer13_attention_weight_min": -28.586809158325195,
      "activations/layer14_attention_weight_max": 37.27958679199219,
      "activations/layer14_attention_weight_min": -29.27467918395996,
      "activations/layer15_attention_weight_max": 35.474178314208984,
      "activations/layer15_attention_weight_min": -29.61734962463379,
      "activations/layer16_attention_weight_max": 35.53274917602539,
      "activations/layer16_attention_weight_min": -32.820526123046875,
      "activations/layer17_attention_weight_max": 55.93791580200195,
      "activations/layer17_attention_weight_min": -45.64815902709961,
      "activations/layer18_attention_weight_max": 48.491676330566406,
      "activations/layer18_attention_weight_min": -40.70378875732422,
      "activations/layer19_attention_weight_max": 24.492446899414062,
      "activations/layer19_attention_weight_min": -23.659391403198242,
      "activations/layer1_attention_weight_max": 16.628114700317383,
      "activations/layer1_attention_weight_min": -12.230249404907227,
      "activations/layer20_attention_weight_max": 23.120861053466797,
      "activations/layer20_attention_weight_min": -21.71038055419922,
      "activations/layer21_attention_weight_max": 37.13274383544922,
      "activations/layer21_attention_weight_min": -23.167850494384766,
      "activations/layer22_attention_weight_max": 28.43012809753418,
      "activations/layer22_attention_weight_min": -26.340717315673828,
      "activations/layer23_attention_weight_max": 37.45781707763672,
      "activations/layer23_attention_weight_min": -25.73604393005371,
      "activations/layer2_attention_weight_max": 27.617469787597656,
      "activations/layer2_attention_weight_min": -27.393291473388672,
      "activations/layer3_attention_weight_max": 78.57149505615234,
      "activations/layer3_attention_weight_min": -76.31716918945312,
      "activations/layer4_attention_weight_max": 76.60466003417969,
      "activations/layer4_attention_weight_min": -74.6679916381836,
      "activations/layer5_attention_weight_max": 64.8924560546875,
      "activations/layer5_attention_weight_min": -68.04764556884766,
      "activations/layer6_attention_weight_max": 44.59428405761719,
      "activations/layer6_attention_weight_min": -45.2493782043457,
      "activations/layer7_attention_weight_max": 65.50923156738281,
      "activations/layer7_attention_weight_min": -62.519161224365234,
      "activations/layer8_attention_weight_max": 44.94118881225586,
      "activations/layer8_attention_weight_min": -47.06611251831055,
      "activations/layer9_attention_weight_max": 46.998985290527344,
      "activations/layer9_attention_weight_min": -45.3776969909668,
      "epoch": 20.3,
      "learning_rate": 1.9273106060606058e-05,
      "loss": 2.7216,
      "step": 349400
    },
    {
      "activations/layer0_attention_weight_max": 15.891250610351562,
      "activations/layer0_attention_weight_min": -13.610502243041992,
      "activations/layer10_attention_weight_max": 37.685848236083984,
      "activations/layer10_attention_weight_min": -36.30168151855469,
      "activations/layer11_attention_weight_max": 34.864051818847656,
      "activations/layer11_attention_weight_min": -34.24397277832031,
      "activations/layer12_attention_weight_max": 19.56427764892578,
      "activations/layer12_attention_weight_min": -26.450342178344727,
      "activations/layer13_attention_weight_max": 32.0390625,
      "activations/layer13_attention_weight_min": -28.737123489379883,
      "activations/layer14_attention_weight_max": 34.57155227661133,
      "activations/layer14_attention_weight_min": -29.176589965820312,
      "activations/layer15_attention_weight_max": 31.58061981201172,
      "activations/layer15_attention_weight_min": -29.29854965209961,
      "activations/layer16_attention_weight_max": 32.627464294433594,
      "activations/layer16_attention_weight_min": -34.01927185058594,
      "activations/layer17_attention_weight_max": 49.64493942260742,
      "activations/layer17_attention_weight_min": -46.2304573059082,
      "activations/layer18_attention_weight_max": 46.197635650634766,
      "activations/layer18_attention_weight_min": -41.06544876098633,
      "activations/layer19_attention_weight_max": 24.021215438842773,
      "activations/layer19_attention_weight_min": -23.238821029663086,
      "activations/layer1_attention_weight_max": 15.010675430297852,
      "activations/layer1_attention_weight_min": -14.43262767791748,
      "activations/layer20_attention_weight_max": 23.29024887084961,
      "activations/layer20_attention_weight_min": -23.92160415649414,
      "activations/layer21_attention_weight_max": 38.400054931640625,
      "activations/layer21_attention_weight_min": -24.03273582458496,
      "activations/layer22_attention_weight_max": 29.55630874633789,
      "activations/layer22_attention_weight_min": -29.436311721801758,
      "activations/layer23_attention_weight_max": 34.66109085083008,
      "activations/layer23_attention_weight_min": -25.236255645751953,
      "activations/layer2_attention_weight_max": 31.62288475036621,
      "activations/layer2_attention_weight_min": -29.50232696533203,
      "activations/layer3_attention_weight_max": 85.19890594482422,
      "activations/layer3_attention_weight_min": -85.45122528076172,
      "activations/layer4_attention_weight_max": 83.5718765258789,
      "activations/layer4_attention_weight_min": -84.2789535522461,
      "activations/layer5_attention_weight_max": 59.211341857910156,
      "activations/layer5_attention_weight_min": -68.97184753417969,
      "activations/layer6_attention_weight_max": 47.92031478881836,
      "activations/layer6_attention_weight_min": -48.34022521972656,
      "activations/layer7_attention_weight_max": 61.485984802246094,
      "activations/layer7_attention_weight_min": -65.57720947265625,
      "activations/layer8_attention_weight_max": 48.28815460205078,
      "activations/layer8_attention_weight_min": -52.993385314941406,
      "activations/layer9_attention_weight_max": 44.282264709472656,
      "activations/layer9_attention_weight_min": -44.810829162597656,
      "epoch": 20.31,
      "learning_rate": 1.9254166666666666e-05,
      "loss": 2.731,
      "step": 349450
    },
    {
      "activations/layer0_attention_weight_max": 16.02219581604004,
      "activations/layer0_attention_weight_min": -14.140480995178223,
      "activations/layer10_attention_weight_max": 31.21044921875,
      "activations/layer10_attention_weight_min": -33.24951171875,
      "activations/layer11_attention_weight_max": 33.326805114746094,
      "activations/layer11_attention_weight_min": -35.14239501953125,
      "activations/layer12_attention_weight_max": 18.086463928222656,
      "activations/layer12_attention_weight_min": -22.972354888916016,
      "activations/layer13_attention_weight_max": 34.08936309814453,
      "activations/layer13_attention_weight_min": -26.918895721435547,
      "activations/layer14_attention_weight_max": 34.53538513183594,
      "activations/layer14_attention_weight_min": -29.402029037475586,
      "activations/layer15_attention_weight_max": 32.58850860595703,
      "activations/layer15_attention_weight_min": -29.981060028076172,
      "activations/layer16_attention_weight_max": 32.46823501586914,
      "activations/layer16_attention_weight_min": -31.491487503051758,
      "activations/layer17_attention_weight_max": 49.24763488769531,
      "activations/layer17_attention_weight_min": -41.92292404174805,
      "activations/layer18_attention_weight_max": 46.97063446044922,
      "activations/layer18_attention_weight_min": -39.60499954223633,
      "activations/layer19_attention_weight_max": 23.11782455444336,
      "activations/layer19_attention_weight_min": -20.97549057006836,
      "activations/layer1_attention_weight_max": 14.858311653137207,
      "activations/layer1_attention_weight_min": -13.356329917907715,
      "activations/layer20_attention_weight_max": 22.406408309936523,
      "activations/layer20_attention_weight_min": -20.572214126586914,
      "activations/layer21_attention_weight_max": 30.39525032043457,
      "activations/layer21_attention_weight_min": -19.519317626953125,
      "activations/layer22_attention_weight_max": 31.42157745361328,
      "activations/layer22_attention_weight_min": -25.088483810424805,
      "activations/layer23_attention_weight_max": 36.48434829711914,
      "activations/layer23_attention_weight_min": -23.556419372558594,
      "activations/layer2_attention_weight_max": 31.454330444335938,
      "activations/layer2_attention_weight_min": -29.577590942382812,
      "activations/layer3_attention_weight_max": 81.72499084472656,
      "activations/layer3_attention_weight_min": -85.97554779052734,
      "activations/layer4_attention_weight_max": 81.34696960449219,
      "activations/layer4_attention_weight_min": -82.38224792480469,
      "activations/layer5_attention_weight_max": 58.8311882019043,
      "activations/layer5_attention_weight_min": -64.8355712890625,
      "activations/layer6_attention_weight_max": 45.45279312133789,
      "activations/layer6_attention_weight_min": -48.20008087158203,
      "activations/layer7_attention_weight_max": 63.92836380004883,
      "activations/layer7_attention_weight_min": -61.852630615234375,
      "activations/layer8_attention_weight_max": 43.90651321411133,
      "activations/layer8_attention_weight_min": -49.025516510009766,
      "activations/layer9_attention_weight_max": 40.5874137878418,
      "activations/layer9_attention_weight_min": -44.2453727722168,
      "epoch": 20.31,
      "learning_rate": 1.923522727272727e-05,
      "loss": 2.7167,
      "step": 349500
    },
    {
      "activations/layer0_attention_weight_max": 15.407776832580566,
      "activations/layer0_attention_weight_min": -13.634415626525879,
      "activations/layer10_attention_weight_max": 37.965877532958984,
      "activations/layer10_attention_weight_min": -37.26283264160156,
      "activations/layer11_attention_weight_max": 33.64093780517578,
      "activations/layer11_attention_weight_min": -34.038700103759766,
      "activations/layer12_attention_weight_max": 18.956134796142578,
      "activations/layer12_attention_weight_min": -23.5991153717041,
      "activations/layer13_attention_weight_max": 33.86433792114258,
      "activations/layer13_attention_weight_min": -28.044164657592773,
      "activations/layer14_attention_weight_max": 35.24399948120117,
      "activations/layer14_attention_weight_min": -30.640392303466797,
      "activations/layer15_attention_weight_max": 33.26358413696289,
      "activations/layer15_attention_weight_min": -31.581417083740234,
      "activations/layer16_attention_weight_max": 34.318538665771484,
      "activations/layer16_attention_weight_min": -32.31576919555664,
      "activations/layer17_attention_weight_max": 49.6602897644043,
      "activations/layer17_attention_weight_min": -45.25120162963867,
      "activations/layer18_attention_weight_max": 48.3782844543457,
      "activations/layer18_attention_weight_min": -42.38793182373047,
      "activations/layer19_attention_weight_max": 24.477834701538086,
      "activations/layer19_attention_weight_min": -23.69313621520996,
      "activations/layer1_attention_weight_max": 17.275693893432617,
      "activations/layer1_attention_weight_min": -14.062938690185547,
      "activations/layer20_attention_weight_max": 22.835786819458008,
      "activations/layer20_attention_weight_min": -21.34029769897461,
      "activations/layer21_attention_weight_max": 37.002140045166016,
      "activations/layer21_attention_weight_min": -22.641515731811523,
      "activations/layer22_attention_weight_max": 28.62557029724121,
      "activations/layer22_attention_weight_min": -28.04083824157715,
      "activations/layer23_attention_weight_max": 38.628623962402344,
      "activations/layer23_attention_weight_min": -25.790782928466797,
      "activations/layer2_attention_weight_max": 29.18202018737793,
      "activations/layer2_attention_weight_min": -28.269620895385742,
      "activations/layer3_attention_weight_max": 79.6965560913086,
      "activations/layer3_attention_weight_min": -83.27662658691406,
      "activations/layer4_attention_weight_max": 72.91537475585938,
      "activations/layer4_attention_weight_min": -72.87000274658203,
      "activations/layer5_attention_weight_max": 55.437599182128906,
      "activations/layer5_attention_weight_min": -65.37452697753906,
      "activations/layer6_attention_weight_max": 43.731449127197266,
      "activations/layer6_attention_weight_min": -46.27538299560547,
      "activations/layer7_attention_weight_max": 58.2601318359375,
      "activations/layer7_attention_weight_min": -60.81112289428711,
      "activations/layer8_attention_weight_max": 44.76155471801758,
      "activations/layer8_attention_weight_min": -46.382259368896484,
      "activations/layer9_attention_weight_max": 42.16594314575195,
      "activations/layer9_attention_weight_min": -45.69192123413086,
      "epoch": 20.31,
      "learning_rate": 1.921628787878788e-05,
      "loss": 2.7248,
      "step": 349550
    },
    {
      "activations/layer0_attention_weight_max": 16.603605270385742,
      "activations/layer0_attention_weight_min": -13.366889953613281,
      "activations/layer10_attention_weight_max": 32.321571350097656,
      "activations/layer10_attention_weight_min": -32.59375762939453,
      "activations/layer11_attention_weight_max": 31.821136474609375,
      "activations/layer11_attention_weight_min": -32.538665771484375,
      "activations/layer12_attention_weight_max": 18.35059928894043,
      "activations/layer12_attention_weight_min": -25.421451568603516,
      "activations/layer13_attention_weight_max": 30.815486907958984,
      "activations/layer13_attention_weight_min": -26.189096450805664,
      "activations/layer14_attention_weight_max": 34.9059944152832,
      "activations/layer14_attention_weight_min": -29.499784469604492,
      "activations/layer15_attention_weight_max": 35.663291931152344,
      "activations/layer15_attention_weight_min": -32.07792663574219,
      "activations/layer16_attention_weight_max": 33.67049789428711,
      "activations/layer16_attention_weight_min": -30.329544067382812,
      "activations/layer17_attention_weight_max": 53.92296600341797,
      "activations/layer17_attention_weight_min": -44.34922409057617,
      "activations/layer18_attention_weight_max": 45.46574020385742,
      "activations/layer18_attention_weight_min": -40.355831146240234,
      "activations/layer19_attention_weight_max": 24.26951026916504,
      "activations/layer19_attention_weight_min": -25.05094337463379,
      "activations/layer1_attention_weight_max": 15.26331901550293,
      "activations/layer1_attention_weight_min": -13.468534469604492,
      "activations/layer20_attention_weight_max": 23.37421226501465,
      "activations/layer20_attention_weight_min": -24.710783004760742,
      "activations/layer21_attention_weight_max": 36.581947326660156,
      "activations/layer21_attention_weight_min": -25.486717224121094,
      "activations/layer22_attention_weight_max": 31.1041259765625,
      "activations/layer22_attention_weight_min": -25.385028839111328,
      "activations/layer23_attention_weight_max": 34.78755187988281,
      "activations/layer23_attention_weight_min": -25.04486083984375,
      "activations/layer2_attention_weight_max": 32.52217483520508,
      "activations/layer2_attention_weight_min": -30.675212860107422,
      "activations/layer3_attention_weight_max": 88.45391845703125,
      "activations/layer3_attention_weight_min": -90.19255065917969,
      "activations/layer4_attention_weight_max": 80.12425994873047,
      "activations/layer4_attention_weight_min": -79.37145233154297,
      "activations/layer5_attention_weight_max": 59.20667266845703,
      "activations/layer5_attention_weight_min": -66.57330322265625,
      "activations/layer6_attention_weight_max": 47.00749588012695,
      "activations/layer6_attention_weight_min": -46.86473083496094,
      "activations/layer7_attention_weight_max": 63.71818161010742,
      "activations/layer7_attention_weight_min": -60.19993209838867,
      "activations/layer8_attention_weight_max": 43.44082260131836,
      "activations/layer8_attention_weight_min": -46.93355178833008,
      "activations/layer9_attention_weight_max": 43.97774887084961,
      "activations/layer9_attention_weight_min": -43.0091552734375,
      "epoch": 20.31,
      "learning_rate": 1.919734848484848e-05,
      "loss": 2.7229,
      "step": 349600
    },
    {
      "activations/layer0_attention_weight_max": 16.430137634277344,
      "activations/layer0_attention_weight_min": -13.872228622436523,
      "activations/layer10_attention_weight_max": 35.039146423339844,
      "activations/layer10_attention_weight_min": -36.523155212402344,
      "activations/layer11_attention_weight_max": 34.95941925048828,
      "activations/layer11_attention_weight_min": -35.214778900146484,
      "activations/layer12_attention_weight_max": 18.693981170654297,
      "activations/layer12_attention_weight_min": -23.95054054260254,
      "activations/layer13_attention_weight_max": 36.06131362915039,
      "activations/layer13_attention_weight_min": -28.235239028930664,
      "activations/layer14_attention_weight_max": 37.761512756347656,
      "activations/layer14_attention_weight_min": -32.60811233520508,
      "activations/layer15_attention_weight_max": 35.17913055419922,
      "activations/layer15_attention_weight_min": -31.835237503051758,
      "activations/layer16_attention_weight_max": 37.287498474121094,
      "activations/layer16_attention_weight_min": -31.974279403686523,
      "activations/layer17_attention_weight_max": 51.981143951416016,
      "activations/layer17_attention_weight_min": -43.49873733520508,
      "activations/layer18_attention_weight_max": 50.44332504272461,
      "activations/layer18_attention_weight_min": -39.653133392333984,
      "activations/layer19_attention_weight_max": 24.897985458374023,
      "activations/layer19_attention_weight_min": -22.095434188842773,
      "activations/layer1_attention_weight_max": 15.3400239944458,
      "activations/layer1_attention_weight_min": -13.558060646057129,
      "activations/layer20_attention_weight_max": 23.689193725585938,
      "activations/layer20_attention_weight_min": -21.88787269592285,
      "activations/layer21_attention_weight_max": 33.801788330078125,
      "activations/layer21_attention_weight_min": -23.2126522064209,
      "activations/layer22_attention_weight_max": 27.96303367614746,
      "activations/layer22_attention_weight_min": -25.17182731628418,
      "activations/layer23_attention_weight_max": 38.9610595703125,
      "activations/layer23_attention_weight_min": -23.39354705810547,
      "activations/layer2_attention_weight_max": 30.20545768737793,
      "activations/layer2_attention_weight_min": -28.36724853515625,
      "activations/layer3_attention_weight_max": 87.2041015625,
      "activations/layer3_attention_weight_min": -80.6522216796875,
      "activations/layer4_attention_weight_max": 82.23384857177734,
      "activations/layer4_attention_weight_min": -80.13069915771484,
      "activations/layer5_attention_weight_max": 61.39331817626953,
      "activations/layer5_attention_weight_min": -68.11493682861328,
      "activations/layer6_attention_weight_max": 47.80714797973633,
      "activations/layer6_attention_weight_min": -52.07384490966797,
      "activations/layer7_attention_weight_max": 67.95105743408203,
      "activations/layer7_attention_weight_min": -63.67404556274414,
      "activations/layer8_attention_weight_max": 48.2322883605957,
      "activations/layer8_attention_weight_min": -54.88883590698242,
      "activations/layer9_attention_weight_max": 44.06338119506836,
      "activations/layer9_attention_weight_min": -48.62253952026367,
      "epoch": 20.32,
      "learning_rate": 1.917840909090909e-05,
      "loss": 2.7267,
      "step": 349650
    },
    {
      "activations/layer0_attention_weight_max": 16.372882843017578,
      "activations/layer0_attention_weight_min": -13.623679161071777,
      "activations/layer10_attention_weight_max": 38.37016296386719,
      "activations/layer10_attention_weight_min": -39.87167739868164,
      "activations/layer11_attention_weight_max": 37.07218551635742,
      "activations/layer11_attention_weight_min": -37.579010009765625,
      "activations/layer12_attention_weight_max": 18.970308303833008,
      "activations/layer12_attention_weight_min": -23.122615814208984,
      "activations/layer13_attention_weight_max": 36.98897171020508,
      "activations/layer13_attention_weight_min": -31.267684936523438,
      "activations/layer14_attention_weight_max": 33.14836883544922,
      "activations/layer14_attention_weight_min": -30.678529739379883,
      "activations/layer15_attention_weight_max": 32.687137603759766,
      "activations/layer15_attention_weight_min": -32.11587905883789,
      "activations/layer16_attention_weight_max": 33.123680114746094,
      "activations/layer16_attention_weight_min": -30.85721778869629,
      "activations/layer17_attention_weight_max": 52.6559944152832,
      "activations/layer17_attention_weight_min": -45.06996536254883,
      "activations/layer18_attention_weight_max": 45.35798263549805,
      "activations/layer18_attention_weight_min": -40.32362365722656,
      "activations/layer19_attention_weight_max": 23.412168502807617,
      "activations/layer19_attention_weight_min": -22.865997314453125,
      "activations/layer1_attention_weight_max": 15.01024055480957,
      "activations/layer1_attention_weight_min": -13.232519149780273,
      "activations/layer20_attention_weight_max": 20.688257217407227,
      "activations/layer20_attention_weight_min": -21.1020565032959,
      "activations/layer21_attention_weight_max": 34.788055419921875,
      "activations/layer21_attention_weight_min": -23.128925323486328,
      "activations/layer22_attention_weight_max": 28.34488296508789,
      "activations/layer22_attention_weight_min": -24.48772430419922,
      "activations/layer23_attention_weight_max": 35.12269592285156,
      "activations/layer23_attention_weight_min": -25.275487899780273,
      "activations/layer2_attention_weight_max": 29.766887664794922,
      "activations/layer2_attention_weight_min": -29.623319625854492,
      "activations/layer3_attention_weight_max": 82.73728942871094,
      "activations/layer3_attention_weight_min": -85.14599609375,
      "activations/layer4_attention_weight_max": 83.42268371582031,
      "activations/layer4_attention_weight_min": -81.8722152709961,
      "activations/layer5_attention_weight_max": 65.12062072753906,
      "activations/layer5_attention_weight_min": -68.69801330566406,
      "activations/layer6_attention_weight_max": 52.18534851074219,
      "activations/layer6_attention_weight_min": -50.0667610168457,
      "activations/layer7_attention_weight_max": 68.32777404785156,
      "activations/layer7_attention_weight_min": -69.2516098022461,
      "activations/layer8_attention_weight_max": 47.575653076171875,
      "activations/layer8_attention_weight_min": -48.019630432128906,
      "activations/layer9_attention_weight_max": 46.92765426635742,
      "activations/layer9_attention_weight_min": -45.66852951049805,
      "epoch": 20.32,
      "learning_rate": 1.9159469696969694e-05,
      "loss": 2.7136,
      "step": 349700
    },
    {
      "activations/layer0_attention_weight_max": 15.813774108886719,
      "activations/layer0_attention_weight_min": -13.745420455932617,
      "activations/layer10_attention_weight_max": 36.47860336303711,
      "activations/layer10_attention_weight_min": -36.32069396972656,
      "activations/layer11_attention_weight_max": 33.776145935058594,
      "activations/layer11_attention_weight_min": -35.67795181274414,
      "activations/layer12_attention_weight_max": 20.018198013305664,
      "activations/layer12_attention_weight_min": -28.06207847595215,
      "activations/layer13_attention_weight_max": 38.42797088623047,
      "activations/layer13_attention_weight_min": -27.008365631103516,
      "activations/layer14_attention_weight_max": 35.88742446899414,
      "activations/layer14_attention_weight_min": -29.365922927856445,
      "activations/layer15_attention_weight_max": 31.156503677368164,
      "activations/layer15_attention_weight_min": -30.967660903930664,
      "activations/layer16_attention_weight_max": 31.304920196533203,
      "activations/layer16_attention_weight_min": -29.93253517150879,
      "activations/layer17_attention_weight_max": 47.88535690307617,
      "activations/layer17_attention_weight_min": -44.42301940917969,
      "activations/layer18_attention_weight_max": 45.60786056518555,
      "activations/layer18_attention_weight_min": -41.5463752746582,
      "activations/layer19_attention_weight_max": 23.41754722595215,
      "activations/layer19_attention_weight_min": -23.81393814086914,
      "activations/layer1_attention_weight_max": 16.54056739807129,
      "activations/layer1_attention_weight_min": -14.029226303100586,
      "activations/layer20_attention_weight_max": 22.838485717773438,
      "activations/layer20_attention_weight_min": -23.503759384155273,
      "activations/layer21_attention_weight_max": 34.35483169555664,
      "activations/layer21_attention_weight_min": -27.18776512145996,
      "activations/layer22_attention_weight_max": 29.668354034423828,
      "activations/layer22_attention_weight_min": -26.240299224853516,
      "activations/layer23_attention_weight_max": 35.02728271484375,
      "activations/layer23_attention_weight_min": -23.495573043823242,
      "activations/layer2_attention_weight_max": 31.728055953979492,
      "activations/layer2_attention_weight_min": -28.97357177734375,
      "activations/layer3_attention_weight_max": 88.82572937011719,
      "activations/layer3_attention_weight_min": -86.50666809082031,
      "activations/layer4_attention_weight_max": 82.8247299194336,
      "activations/layer4_attention_weight_min": -80.07665252685547,
      "activations/layer5_attention_weight_max": 63.22050476074219,
      "activations/layer5_attention_weight_min": -69.56138610839844,
      "activations/layer6_attention_weight_max": 50.54343032836914,
      "activations/layer6_attention_weight_min": -52.04399490356445,
      "activations/layer7_attention_weight_max": 71.34375,
      "activations/layer7_attention_weight_min": -64.3041000366211,
      "activations/layer8_attention_weight_max": 48.740211486816406,
      "activations/layer8_attention_weight_min": -48.737701416015625,
      "activations/layer9_attention_weight_max": 49.56599044799805,
      "activations/layer9_attention_weight_min": -46.736080169677734,
      "epoch": 20.32,
      "learning_rate": 1.9140530303030302e-05,
      "loss": 2.714,
      "step": 349750
    },
    {
      "activations/layer0_attention_weight_max": 15.86451530456543,
      "activations/layer0_attention_weight_min": -13.949029922485352,
      "activations/layer10_attention_weight_max": 34.069366455078125,
      "activations/layer10_attention_weight_min": -32.838623046875,
      "activations/layer11_attention_weight_max": 32.51688766479492,
      "activations/layer11_attention_weight_min": -34.92809295654297,
      "activations/layer12_attention_weight_max": 19.51056480407715,
      "activations/layer12_attention_weight_min": -24.52766227722168,
      "activations/layer13_attention_weight_max": 33.71454620361328,
      "activations/layer13_attention_weight_min": -26.64216423034668,
      "activations/layer14_attention_weight_max": 35.94207763671875,
      "activations/layer14_attention_weight_min": -31.853132247924805,
      "activations/layer15_attention_weight_max": 34.13651657104492,
      "activations/layer15_attention_weight_min": -30.493791580200195,
      "activations/layer16_attention_weight_max": 37.14734649658203,
      "activations/layer16_attention_weight_min": -36.497859954833984,
      "activations/layer17_attention_weight_max": 53.802120208740234,
      "activations/layer17_attention_weight_min": -49.080928802490234,
      "activations/layer18_attention_weight_max": 52.42702102661133,
      "activations/layer18_attention_weight_min": -44.67369842529297,
      "activations/layer19_attention_weight_max": 29.082202911376953,
      "activations/layer19_attention_weight_min": -25.405290603637695,
      "activations/layer1_attention_weight_max": 15.158757209777832,
      "activations/layer1_attention_weight_min": -13.393174171447754,
      "activations/layer20_attention_weight_max": 28.325044631958008,
      "activations/layer20_attention_weight_min": -22.542110443115234,
      "activations/layer21_attention_weight_max": 41.7686653137207,
      "activations/layer21_attention_weight_min": -22.76564598083496,
      "activations/layer22_attention_weight_max": 34.62037658691406,
      "activations/layer22_attention_weight_min": -28.458372116088867,
      "activations/layer23_attention_weight_max": 40.97187042236328,
      "activations/layer23_attention_weight_min": -25.490558624267578,
      "activations/layer2_attention_weight_max": 29.275014877319336,
      "activations/layer2_attention_weight_min": -29.051794052124023,
      "activations/layer3_attention_weight_max": 80.25177001953125,
      "activations/layer3_attention_weight_min": -80.49248504638672,
      "activations/layer4_attention_weight_max": 79.17893981933594,
      "activations/layer4_attention_weight_min": -79.28825378417969,
      "activations/layer5_attention_weight_max": 60.607269287109375,
      "activations/layer5_attention_weight_min": -69.76277160644531,
      "activations/layer6_attention_weight_max": 51.47587966918945,
      "activations/layer6_attention_weight_min": -48.75773239135742,
      "activations/layer7_attention_weight_max": 69.8119888305664,
      "activations/layer7_attention_weight_min": -62.27418899536133,
      "activations/layer8_attention_weight_max": 44.31855773925781,
      "activations/layer8_attention_weight_min": -49.03657150268555,
      "activations/layer9_attention_weight_max": 41.57254409790039,
      "activations/layer9_attention_weight_min": -43.72004318237305,
      "epoch": 20.33,
      "learning_rate": 1.9121590909090907e-05,
      "loss": 2.7229,
      "step": 349800
    },
    {
      "activations/layer0_attention_weight_max": 15.252143859863281,
      "activations/layer0_attention_weight_min": -13.829864501953125,
      "activations/layer10_attention_weight_max": 34.02323913574219,
      "activations/layer10_attention_weight_min": -35.36914825439453,
      "activations/layer11_attention_weight_max": 33.82686996459961,
      "activations/layer11_attention_weight_min": -35.39440155029297,
      "activations/layer12_attention_weight_max": 20.000532150268555,
      "activations/layer12_attention_weight_min": -26.30497932434082,
      "activations/layer13_attention_weight_max": 32.76970672607422,
      "activations/layer13_attention_weight_min": -25.114107131958008,
      "activations/layer14_attention_weight_max": 37.992549896240234,
      "activations/layer14_attention_weight_min": -29.510866165161133,
      "activations/layer15_attention_weight_max": 35.0537109375,
      "activations/layer15_attention_weight_min": -28.825424194335938,
      "activations/layer16_attention_weight_max": 33.64081573486328,
      "activations/layer16_attention_weight_min": -30.32546043395996,
      "activations/layer17_attention_weight_max": 53.227359771728516,
      "activations/layer17_attention_weight_min": -43.82343292236328,
      "activations/layer18_attention_weight_max": 45.647216796875,
      "activations/layer18_attention_weight_min": -39.4429931640625,
      "activations/layer19_attention_weight_max": 22.75919532775879,
      "activations/layer19_attention_weight_min": -22.341123580932617,
      "activations/layer1_attention_weight_max": 15.551405906677246,
      "activations/layer1_attention_weight_min": -13.020153999328613,
      "activations/layer20_attention_weight_max": 23.955846786499023,
      "activations/layer20_attention_weight_min": -21.458105087280273,
      "activations/layer21_attention_weight_max": 33.804237365722656,
      "activations/layer21_attention_weight_min": -21.150230407714844,
      "activations/layer22_attention_weight_max": 27.789583206176758,
      "activations/layer22_attention_weight_min": -25.599315643310547,
      "activations/layer23_attention_weight_max": 33.8021240234375,
      "activations/layer23_attention_weight_min": -25.243431091308594,
      "activations/layer2_attention_weight_max": 31.284561157226562,
      "activations/layer2_attention_weight_min": -29.14605712890625,
      "activations/layer3_attention_weight_max": 83.7582015991211,
      "activations/layer3_attention_weight_min": -87.95042419433594,
      "activations/layer4_attention_weight_max": 80.06969451904297,
      "activations/layer4_attention_weight_min": -85.9134521484375,
      "activations/layer5_attention_weight_max": 60.814002990722656,
      "activations/layer5_attention_weight_min": -70.3072509765625,
      "activations/layer6_attention_weight_max": 47.27925109863281,
      "activations/layer6_attention_weight_min": -47.86341094970703,
      "activations/layer7_attention_weight_max": 64.04570770263672,
      "activations/layer7_attention_weight_min": -60.63048553466797,
      "activations/layer8_attention_weight_max": 45.31381607055664,
      "activations/layer8_attention_weight_min": -47.479881286621094,
      "activations/layer9_attention_weight_max": 41.47621154785156,
      "activations/layer9_attention_weight_min": -44.17893981933594,
      "epoch": 20.33,
      "learning_rate": 1.9102651515151515e-05,
      "loss": 2.7091,
      "step": 349850
    },
    {
      "activations/layer0_attention_weight_max": 16.344642639160156,
      "activations/layer0_attention_weight_min": -13.646477699279785,
      "activations/layer10_attention_weight_max": 38.3941535949707,
      "activations/layer10_attention_weight_min": -38.314849853515625,
      "activations/layer11_attention_weight_max": 36.299713134765625,
      "activations/layer11_attention_weight_min": -37.773948669433594,
      "activations/layer12_attention_weight_max": 20.95109748840332,
      "activations/layer12_attention_weight_min": -23.699790954589844,
      "activations/layer13_attention_weight_max": 39.205352783203125,
      "activations/layer13_attention_weight_min": -26.945283889770508,
      "activations/layer14_attention_weight_max": 40.44091033935547,
      "activations/layer14_attention_weight_min": -30.50492286682129,
      "activations/layer15_attention_weight_max": 37.642276763916016,
      "activations/layer15_attention_weight_min": -29.924619674682617,
      "activations/layer16_attention_weight_max": 37.65922546386719,
      "activations/layer16_attention_weight_min": -32.799320220947266,
      "activations/layer17_attention_weight_max": 56.31916809082031,
      "activations/layer17_attention_weight_min": -45.843788146972656,
      "activations/layer18_attention_weight_max": 54.132266998291016,
      "activations/layer18_attention_weight_min": -41.899658203125,
      "activations/layer19_attention_weight_max": 29.795608520507812,
      "activations/layer19_attention_weight_min": -21.80718421936035,
      "activations/layer1_attention_weight_max": 14.89554500579834,
      "activations/layer1_attention_weight_min": -13.845212936401367,
      "activations/layer20_attention_weight_max": 28.470186233520508,
      "activations/layer20_attention_weight_min": -22.26681900024414,
      "activations/layer21_attention_weight_max": 49.56181335449219,
      "activations/layer21_attention_weight_min": -26.01671028137207,
      "activations/layer22_attention_weight_max": 33.411495208740234,
      "activations/layer22_attention_weight_min": -24.78955078125,
      "activations/layer23_attention_weight_max": 39.8006706237793,
      "activations/layer23_attention_weight_min": -25.44774627685547,
      "activations/layer2_attention_weight_max": 30.086414337158203,
      "activations/layer2_attention_weight_min": -29.436765670776367,
      "activations/layer3_attention_weight_max": 82.50180053710938,
      "activations/layer3_attention_weight_min": -84.55111694335938,
      "activations/layer4_attention_weight_max": 81.00206756591797,
      "activations/layer4_attention_weight_min": -81.30667877197266,
      "activations/layer5_attention_weight_max": 64.07491302490234,
      "activations/layer5_attention_weight_min": -70.02714538574219,
      "activations/layer6_attention_weight_max": 47.936912536621094,
      "activations/layer6_attention_weight_min": -50.408973693847656,
      "activations/layer7_attention_weight_max": 67.86549377441406,
      "activations/layer7_attention_weight_min": -66.84253692626953,
      "activations/layer8_attention_weight_max": 51.049861907958984,
      "activations/layer8_attention_weight_min": -53.23624801635742,
      "activations/layer9_attention_weight_max": 47.06357955932617,
      "activations/layer9_attention_weight_min": -46.88138961791992,
      "epoch": 20.33,
      "learning_rate": 1.9083712121212117e-05,
      "loss": 2.7291,
      "step": 349900
    },
    {
      "activations/layer0_attention_weight_max": 15.685953140258789,
      "activations/layer0_attention_weight_min": -13.82868766784668,
      "activations/layer10_attention_weight_max": 40.75557327270508,
      "activations/layer10_attention_weight_min": -39.573944091796875,
      "activations/layer11_attention_weight_max": 37.6937255859375,
      "activations/layer11_attention_weight_min": -37.46449661254883,
      "activations/layer12_attention_weight_max": 19.77113151550293,
      "activations/layer12_attention_weight_min": -28.667133331298828,
      "activations/layer13_attention_weight_max": 37.75468826293945,
      "activations/layer13_attention_weight_min": -29.121191024780273,
      "activations/layer14_attention_weight_max": 34.81406784057617,
      "activations/layer14_attention_weight_min": -31.177104949951172,
      "activations/layer15_attention_weight_max": 33.86113739013672,
      "activations/layer15_attention_weight_min": -31.332576751708984,
      "activations/layer16_attention_weight_max": 35.59115982055664,
      "activations/layer16_attention_weight_min": -32.0622444152832,
      "activations/layer17_attention_weight_max": 53.78921127319336,
      "activations/layer17_attention_weight_min": -47.17194366455078,
      "activations/layer18_attention_weight_max": 48.850650787353516,
      "activations/layer18_attention_weight_min": -42.51861572265625,
      "activations/layer19_attention_weight_max": 23.707548141479492,
      "activations/layer19_attention_weight_min": -25.036048889160156,
      "activations/layer1_attention_weight_max": 15.58605670928955,
      "activations/layer1_attention_weight_min": -14.459894180297852,
      "activations/layer20_attention_weight_max": 22.866168975830078,
      "activations/layer20_attention_weight_min": -22.40875244140625,
      "activations/layer21_attention_weight_max": 36.96026611328125,
      "activations/layer21_attention_weight_min": -21.964872360229492,
      "activations/layer22_attention_weight_max": 29.314905166625977,
      "activations/layer22_attention_weight_min": -26.51207160949707,
      "activations/layer23_attention_weight_max": 39.74044418334961,
      "activations/layer23_attention_weight_min": -23.75399398803711,
      "activations/layer2_attention_weight_max": 31.195945739746094,
      "activations/layer2_attention_weight_min": -29.657989501953125,
      "activations/layer3_attention_weight_max": 86.85871124267578,
      "activations/layer3_attention_weight_min": -85.14606475830078,
      "activations/layer4_attention_weight_max": 83.13359832763672,
      "activations/layer4_attention_weight_min": -83.28582763671875,
      "activations/layer5_attention_weight_max": 62.272308349609375,
      "activations/layer5_attention_weight_min": -71.81000518798828,
      "activations/layer6_attention_weight_max": 48.88274383544922,
      "activations/layer6_attention_weight_min": -51.466548919677734,
      "activations/layer7_attention_weight_max": 69.21672821044922,
      "activations/layer7_attention_weight_min": -67.09819030761719,
      "activations/layer8_attention_weight_max": 52.005863189697266,
      "activations/layer8_attention_weight_min": -57.06774139404297,
      "activations/layer9_attention_weight_max": 50.00537872314453,
      "activations/layer9_attention_weight_min": -51.2048225402832,
      "epoch": 20.33,
      "learning_rate": 1.9064772727272725e-05,
      "loss": 2.7392,
      "step": 349950
    },
    {
      "activations/layer0_attention_weight_max": 15.822797775268555,
      "activations/layer0_attention_weight_min": -14.061515808105469,
      "activations/layer10_attention_weight_max": 34.18752670288086,
      "activations/layer10_attention_weight_min": -35.48023223876953,
      "activations/layer11_attention_weight_max": 31.72291374206543,
      "activations/layer11_attention_weight_min": -34.256954193115234,
      "activations/layer12_attention_weight_max": 18.83108901977539,
      "activations/layer12_attention_weight_min": -25.476112365722656,
      "activations/layer13_attention_weight_max": 34.3848762512207,
      "activations/layer13_attention_weight_min": -29.834421157836914,
      "activations/layer14_attention_weight_max": 35.536067962646484,
      "activations/layer14_attention_weight_min": -31.483654022216797,
      "activations/layer15_attention_weight_max": 32.084251403808594,
      "activations/layer15_attention_weight_min": -32.082725524902344,
      "activations/layer16_attention_weight_max": 33.00257110595703,
      "activations/layer16_attention_weight_min": -32.68755340576172,
      "activations/layer17_attention_weight_max": 48.91858673095703,
      "activations/layer17_attention_weight_min": -43.718772888183594,
      "activations/layer18_attention_weight_max": 45.940208435058594,
      "activations/layer18_attention_weight_min": -40.358699798583984,
      "activations/layer19_attention_weight_max": 23.248449325561523,
      "activations/layer19_attention_weight_min": -23.633033752441406,
      "activations/layer1_attention_weight_max": 15.385629653930664,
      "activations/layer1_attention_weight_min": -14.404635429382324,
      "activations/layer20_attention_weight_max": 22.702587127685547,
      "activations/layer20_attention_weight_min": -21.104476928710938,
      "activations/layer21_attention_weight_max": 34.341949462890625,
      "activations/layer21_attention_weight_min": -24.551836013793945,
      "activations/layer22_attention_weight_max": 27.382179260253906,
      "activations/layer22_attention_weight_min": -26.144351959228516,
      "activations/layer23_attention_weight_max": 36.648780822753906,
      "activations/layer23_attention_weight_min": -23.389997482299805,
      "activations/layer2_attention_weight_max": 31.925769805908203,
      "activations/layer2_attention_weight_min": -30.05730438232422,
      "activations/layer3_attention_weight_max": 87.3370361328125,
      "activations/layer3_attention_weight_min": -85.6956558227539,
      "activations/layer4_attention_weight_max": 79.96347045898438,
      "activations/layer4_attention_weight_min": -80.6307601928711,
      "activations/layer5_attention_weight_max": 57.08350372314453,
      "activations/layer5_attention_weight_min": -72.14092254638672,
      "activations/layer6_attention_weight_max": 48.41206741333008,
      "activations/layer6_attention_weight_min": -48.68901443481445,
      "activations/layer7_attention_weight_max": 64.9257583618164,
      "activations/layer7_attention_weight_min": -60.97711944580078,
      "activations/layer8_attention_weight_max": 46.07014083862305,
      "activations/layer8_attention_weight_min": -49.71025466918945,
      "activations/layer9_attention_weight_max": 46.28730392456055,
      "activations/layer9_attention_weight_min": -44.99152755737305,
      "epoch": 20.34,
      "learning_rate": 1.904583333333333e-05,
      "loss": 2.7195,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_loss": 2.6796875,
      "eval_runtime": 8.5228,
      "eval_samples_per_second": 503.827,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_openwebtext_loss": 2.6796875,
      "eval_openwebtext_ppl": 14.580536166314408,
      "eval_openwebtext_runtime": 8.5228,
      "eval_openwebtext_samples_per_second": 503.827,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 2.0251,
      "eval_wikitext_samples_per_second": 225.178,
      "step": 350000
    },
    {
      "epoch": 20.34,
      "eval_lambada_loss": 2.462890625,
      "eval_lambada_ppl": 11.738694709013188,
      "eval_lambada_runtime": 9.6218,
      "eval_lambada_samples_per_second": 506.037,
      "step": 350000
    },
    {
      "activations/layer0_attention_weight_max": 15.408319473266602,
      "activations/layer0_attention_weight_min": -13.616888046264648,
      "activations/layer10_attention_weight_max": 35.264381408691406,
      "activations/layer10_attention_weight_min": -36.575531005859375,
      "activations/layer11_attention_weight_max": 33.83184814453125,
      "activations/layer11_attention_weight_min": -33.71792984008789,
      "activations/layer12_attention_weight_max": 18.944869995117188,
      "activations/layer12_attention_weight_min": -26.653053283691406,
      "activations/layer13_attention_weight_max": 31.08423614501953,
      "activations/layer13_attention_weight_min": -28.07737159729004,
      "activations/layer14_attention_weight_max": 34.972469329833984,
      "activations/layer14_attention_weight_min": -33.50826644897461,
      "activations/layer15_attention_weight_max": 31.701147079467773,
      "activations/layer15_attention_weight_min": -34.658321380615234,
      "activations/layer16_attention_weight_max": 36.47490692138672,
      "activations/layer16_attention_weight_min": -34.843292236328125,
      "activations/layer17_attention_weight_max": 56.64796447753906,
      "activations/layer17_attention_weight_min": -49.531070709228516,
      "activations/layer18_attention_weight_max": 52.608253479003906,
      "activations/layer18_attention_weight_min": -47.09703063964844,
      "activations/layer19_attention_weight_max": 25.129497528076172,
      "activations/layer19_attention_weight_min": -23.24228286743164,
      "activations/layer1_attention_weight_max": 15.025262832641602,
      "activations/layer1_attention_weight_min": -13.283676147460938,
      "activations/layer20_attention_weight_max": 22.80504035949707,
      "activations/layer20_attention_weight_min": -22.0069637298584,
      "activations/layer21_attention_weight_max": 35.99336624145508,
      "activations/layer21_attention_weight_min": -22.210500717163086,
      "activations/layer22_attention_weight_max": 30.466796875,
      "activations/layer22_attention_weight_min": -26.64637565612793,
      "activations/layer23_attention_weight_max": 33.348060607910156,
      "activations/layer23_attention_weight_min": -22.305648803710938,
      "activations/layer2_attention_weight_max": 30.378597259521484,
      "activations/layer2_attention_weight_min": -29.097509384155273,
      "activations/layer3_attention_weight_max": 83.64879608154297,
      "activations/layer3_attention_weight_min": -88.95944213867188,
      "activations/layer4_attention_weight_max": 80.72801971435547,
      "activations/layer4_attention_weight_min": -84.6529769897461,
      "activations/layer5_attention_weight_max": 59.53044891357422,
      "activations/layer5_attention_weight_min": -72.12425231933594,
      "activations/layer6_attention_weight_max": 50.54637145996094,
      "activations/layer6_attention_weight_min": -50.01717758178711,
      "activations/layer7_attention_weight_max": 65.41901397705078,
      "activations/layer7_attention_weight_min": -64.20519256591797,
      "activations/layer8_attention_weight_max": 46.26369857788086,
      "activations/layer8_attention_weight_min": -50.133514404296875,
      "activations/layer9_attention_weight_max": 48.384010314941406,
      "activations/layer9_attention_weight_min": -46.20338439941406,
      "epoch": 20.34,
      "learning_rate": 1.9026893939393938e-05,
      "loss": 2.7241,
      "step": 350050
    },
    {
      "activations/layer0_attention_weight_max": 15.270482063293457,
      "activations/layer0_attention_weight_min": -13.274429321289062,
      "activations/layer10_attention_weight_max": 34.15184020996094,
      "activations/layer10_attention_weight_min": -33.733131408691406,
      "activations/layer11_attention_weight_max": 31.896968841552734,
      "activations/layer11_attention_weight_min": -33.11349868774414,
      "activations/layer12_attention_weight_max": 18.296323776245117,
      "activations/layer12_attention_weight_min": -25.0528507232666,
      "activations/layer13_attention_weight_max": 34.31990432739258,
      "activations/layer13_attention_weight_min": -28.710594177246094,
      "activations/layer14_attention_weight_max": 37.086219787597656,
      "activations/layer14_attention_weight_min": -33.54969787597656,
      "activations/layer15_attention_weight_max": 35.74125289916992,
      "activations/layer15_attention_weight_min": -30.591751098632812,
      "activations/layer16_attention_weight_max": 39.51673889160156,
      "activations/layer16_attention_weight_min": -37.871185302734375,
      "activations/layer17_attention_weight_max": 62.42233657836914,
      "activations/layer17_attention_weight_min": -54.80799102783203,
      "activations/layer18_attention_weight_max": 58.22586441040039,
      "activations/layer18_attention_weight_min": -50.70908737182617,
      "activations/layer19_attention_weight_max": 30.754663467407227,
      "activations/layer19_attention_weight_min": -29.138513565063477,
      "activations/layer1_attention_weight_max": 15.04768180847168,
      "activations/layer1_attention_weight_min": -12.977794647216797,
      "activations/layer20_attention_weight_max": 28.566537857055664,
      "activations/layer20_attention_weight_min": -23.644020080566406,
      "activations/layer21_attention_weight_max": 47.0283203125,
      "activations/layer21_attention_weight_min": -23.874174118041992,
      "activations/layer22_attention_weight_max": 35.26990509033203,
      "activations/layer22_attention_weight_min": -27.550628662109375,
      "activations/layer23_attention_weight_max": 41.04758834838867,
      "activations/layer23_attention_weight_min": -23.431636810302734,
      "activations/layer2_attention_weight_max": 30.318931579589844,
      "activations/layer2_attention_weight_min": -29.15900230407715,
      "activations/layer3_attention_weight_max": 82.07987976074219,
      "activations/layer3_attention_weight_min": -82.961181640625,
      "activations/layer4_attention_weight_max": 79.61568450927734,
      "activations/layer4_attention_weight_min": -77.79329681396484,
      "activations/layer5_attention_weight_max": 61.54681396484375,
      "activations/layer5_attention_weight_min": -70.38604736328125,
      "activations/layer6_attention_weight_max": 47.33744812011719,
      "activations/layer6_attention_weight_min": -47.8137092590332,
      "activations/layer7_attention_weight_max": 63.89037322998047,
      "activations/layer7_attention_weight_min": -61.52488708496094,
      "activations/layer8_attention_weight_max": 43.4394416809082,
      "activations/layer8_attention_weight_min": -48.224300384521484,
      "activations/layer9_attention_weight_max": 41.41377258300781,
      "activations/layer9_attention_weight_min": -46.46833801269531,
      "epoch": 20.34,
      "learning_rate": 1.9007954545454546e-05,
      "loss": 2.7328,
      "step": 350100
    },
    {
      "activations/layer0_attention_weight_max": 15.304484367370605,
      "activations/layer0_attention_weight_min": -13.736757278442383,
      "activations/layer10_attention_weight_max": 31.497940063476562,
      "activations/layer10_attention_weight_min": -35.19190216064453,
      "activations/layer11_attention_weight_max": 32.39202117919922,
      "activations/layer11_attention_weight_min": -33.11832046508789,
      "activations/layer12_attention_weight_max": 18.537321090698242,
      "activations/layer12_attention_weight_min": -25.542083740234375,
      "activations/layer13_attention_weight_max": 31.774288177490234,
      "activations/layer13_attention_weight_min": -27.63839340209961,
      "activations/layer14_attention_weight_max": 34.07776641845703,
      "activations/layer14_attention_weight_min": -31.262617111206055,
      "activations/layer15_attention_weight_max": 30.47919464111328,
      "activations/layer15_attention_weight_min": -31.172828674316406,
      "activations/layer16_attention_weight_max": 32.586856842041016,
      "activations/layer16_attention_weight_min": -32.22490310668945,
      "activations/layer17_attention_weight_max": 46.7911262512207,
      "activations/layer17_attention_weight_min": -43.9904670715332,
      "activations/layer18_attention_weight_max": 43.97104263305664,
      "activations/layer18_attention_weight_min": -39.291114807128906,
      "activations/layer19_attention_weight_max": 22.83664321899414,
      "activations/layer19_attention_weight_min": -23.58827781677246,
      "activations/layer1_attention_weight_max": 15.077654838562012,
      "activations/layer1_attention_weight_min": -13.487154006958008,
      "activations/layer20_attention_weight_max": 21.20686149597168,
      "activations/layer20_attention_weight_min": -22.29288673400879,
      "activations/layer21_attention_weight_max": 33.45149612426758,
      "activations/layer21_attention_weight_min": -20.709684371948242,
      "activations/layer22_attention_weight_max": 27.98278045654297,
      "activations/layer22_attention_weight_min": -26.373188018798828,
      "activations/layer23_attention_weight_max": 37.96372604370117,
      "activations/layer23_attention_weight_min": -24.3538875579834,
      "activations/layer2_attention_weight_max": 30.023212432861328,
      "activations/layer2_attention_weight_min": -28.397602081298828,
      "activations/layer3_attention_weight_max": 85.50518035888672,
      "activations/layer3_attention_weight_min": -82.80225372314453,
      "activations/layer4_attention_weight_max": 78.00202941894531,
      "activations/layer4_attention_weight_min": -80.78256225585938,
      "activations/layer5_attention_weight_max": 58.6478271484375,
      "activations/layer5_attention_weight_min": -69.13455200195312,
      "activations/layer6_attention_weight_max": 48.52430725097656,
      "activations/layer6_attention_weight_min": -47.328800201416016,
      "activations/layer7_attention_weight_max": 60.19439697265625,
      "activations/layer7_attention_weight_min": -64.62406921386719,
      "activations/layer8_attention_weight_max": 44.65266799926758,
      "activations/layer8_attention_weight_min": -49.777976989746094,
      "activations/layer9_attention_weight_max": 41.724178314208984,
      "activations/layer9_attention_weight_min": -46.975582122802734,
      "epoch": 20.35,
      "learning_rate": 1.8989015151515148e-05,
      "loss": 2.7261,
      "step": 350150
    },
    {
      "activations/layer0_attention_weight_max": 15.607593536376953,
      "activations/layer0_attention_weight_min": -14.183216094970703,
      "activations/layer10_attention_weight_max": 35.28917694091797,
      "activations/layer10_attention_weight_min": -36.14970397949219,
      "activations/layer11_attention_weight_max": 32.719261169433594,
      "activations/layer11_attention_weight_min": -33.96762466430664,
      "activations/layer12_attention_weight_max": 18.623031616210938,
      "activations/layer12_attention_weight_min": -25.16897201538086,
      "activations/layer13_attention_weight_max": 37.00639724731445,
      "activations/layer13_attention_weight_min": -26.974531173706055,
      "activations/layer14_attention_weight_max": 36.536556243896484,
      "activations/layer14_attention_weight_min": -29.926889419555664,
      "activations/layer15_attention_weight_max": 34.20805358886719,
      "activations/layer15_attention_weight_min": -30.071491241455078,
      "activations/layer16_attention_weight_max": 35.39073944091797,
      "activations/layer16_attention_weight_min": -33.235809326171875,
      "activations/layer17_attention_weight_max": 54.69057846069336,
      "activations/layer17_attention_weight_min": -47.855472564697266,
      "activations/layer18_attention_weight_max": 49.73796081542969,
      "activations/layer18_attention_weight_min": -41.947994232177734,
      "activations/layer19_attention_weight_max": 25.86991310119629,
      "activations/layer19_attention_weight_min": -23.39598274230957,
      "activations/layer1_attention_weight_max": 14.610732078552246,
      "activations/layer1_attention_weight_min": -12.972458839416504,
      "activations/layer20_attention_weight_max": 26.3514404296875,
      "activations/layer20_attention_weight_min": -21.116985321044922,
      "activations/layer21_attention_weight_max": 37.10182571411133,
      "activations/layer21_attention_weight_min": -25.5637264251709,
      "activations/layer22_attention_weight_max": 30.66732406616211,
      "activations/layer22_attention_weight_min": -26.291460037231445,
      "activations/layer23_attention_weight_max": 40.523681640625,
      "activations/layer23_attention_weight_min": -25.50592613220215,
      "activations/layer2_attention_weight_max": 30.69769859313965,
      "activations/layer2_attention_weight_min": -29.864948272705078,
      "activations/layer3_attention_weight_max": 87.0553207397461,
      "activations/layer3_attention_weight_min": -85.04893493652344,
      "activations/layer4_attention_weight_max": 80.94754791259766,
      "activations/layer4_attention_weight_min": -79.27812194824219,
      "activations/layer5_attention_weight_max": 64.50099182128906,
      "activations/layer5_attention_weight_min": -71.13615417480469,
      "activations/layer6_attention_weight_max": 49.336387634277344,
      "activations/layer6_attention_weight_min": -52.013832092285156,
      "activations/layer7_attention_weight_max": 63.766326904296875,
      "activations/layer7_attention_weight_min": -62.786109924316406,
      "activations/layer8_attention_weight_max": 46.71260452270508,
      "activations/layer8_attention_weight_min": -48.33476257324219,
      "activations/layer9_attention_weight_max": 44.28145980834961,
      "activations/layer9_attention_weight_min": -45.90936279296875,
      "epoch": 20.35,
      "learning_rate": 1.8970075757575756e-05,
      "loss": 2.711,
      "step": 350200
    },
    {
      "activations/layer0_attention_weight_max": 15.415627479553223,
      "activations/layer0_attention_weight_min": -13.699098587036133,
      "activations/layer10_attention_weight_max": 36.52196502685547,
      "activations/layer10_attention_weight_min": -38.45074462890625,
      "activations/layer11_attention_weight_max": 34.737945556640625,
      "activations/layer11_attention_weight_min": -37.93083953857422,
      "activations/layer12_attention_weight_max": 20.824085235595703,
      "activations/layer12_attention_weight_min": -25.47711944580078,
      "activations/layer13_attention_weight_max": 40.02970886230469,
      "activations/layer13_attention_weight_min": -30.0670166015625,
      "activations/layer14_attention_weight_max": 39.17850112915039,
      "activations/layer14_attention_weight_min": -30.513010025024414,
      "activations/layer15_attention_weight_max": 37.171627044677734,
      "activations/layer15_attention_weight_min": -29.731401443481445,
      "activations/layer16_attention_weight_max": 36.586402893066406,
      "activations/layer16_attention_weight_min": -33.22929000854492,
      "activations/layer17_attention_weight_max": 52.1801643371582,
      "activations/layer17_attention_weight_min": -47.866119384765625,
      "activations/layer18_attention_weight_max": 49.006072998046875,
      "activations/layer18_attention_weight_min": -41.52968978881836,
      "activations/layer19_attention_weight_max": 27.385461807250977,
      "activations/layer19_attention_weight_min": -22.369802474975586,
      "activations/layer1_attention_weight_max": 15.2889986038208,
      "activations/layer1_attention_weight_min": -12.772375106811523,
      "activations/layer20_attention_weight_max": 25.80406951904297,
      "activations/layer20_attention_weight_min": -21.241300582885742,
      "activations/layer21_attention_weight_max": 42.50056076049805,
      "activations/layer21_attention_weight_min": -24.1185302734375,
      "activations/layer22_attention_weight_max": 29.672687530517578,
      "activations/layer22_attention_weight_min": -25.31657600402832,
      "activations/layer23_attention_weight_max": 37.586097717285156,
      "activations/layer23_attention_weight_min": -25.304798126220703,
      "activations/layer2_attention_weight_max": 30.532432556152344,
      "activations/layer2_attention_weight_min": -29.460866928100586,
      "activations/layer3_attention_weight_max": 87.36859893798828,
      "activations/layer3_attention_weight_min": -89.31903076171875,
      "activations/layer4_attention_weight_max": 88.66832733154297,
      "activations/layer4_attention_weight_min": -89.34795379638672,
      "activations/layer5_attention_weight_max": 67.15412902832031,
      "activations/layer5_attention_weight_min": -74.23455810546875,
      "activations/layer6_attention_weight_max": 49.09325408935547,
      "activations/layer6_attention_weight_min": -49.710533142089844,
      "activations/layer7_attention_weight_max": 65.84980773925781,
      "activations/layer7_attention_weight_min": -65.95173645019531,
      "activations/layer8_attention_weight_max": 50.01015090942383,
      "activations/layer8_attention_weight_min": -54.036319732666016,
      "activations/layer9_attention_weight_max": 46.045677185058594,
      "activations/layer9_attention_weight_min": -46.32594299316406,
      "epoch": 20.35,
      "learning_rate": 1.895113636363636e-05,
      "loss": 2.7354,
      "step": 350250
    },
    {
      "activations/layer0_attention_weight_max": 15.758166313171387,
      "activations/layer0_attention_weight_min": -13.694046020507812,
      "activations/layer10_attention_weight_max": 37.53948974609375,
      "activations/layer10_attention_weight_min": -37.27321243286133,
      "activations/layer11_attention_weight_max": 33.127872467041016,
      "activations/layer11_attention_weight_min": -36.03125762939453,
      "activations/layer12_attention_weight_max": 19.80828285217285,
      "activations/layer12_attention_weight_min": -26.136308670043945,
      "activations/layer13_attention_weight_max": 35.45389175415039,
      "activations/layer13_attention_weight_min": -28.5516300201416,
      "activations/layer14_attention_weight_max": 37.86585998535156,
      "activations/layer14_attention_weight_min": -31.640422821044922,
      "activations/layer15_attention_weight_max": 35.787498474121094,
      "activations/layer15_attention_weight_min": -30.48998260498047,
      "activations/layer16_attention_weight_max": 35.53371047973633,
      "activations/layer16_attention_weight_min": -34.46085739135742,
      "activations/layer17_attention_weight_max": 52.26020050048828,
      "activations/layer17_attention_weight_min": -49.22084045410156,
      "activations/layer18_attention_weight_max": 47.69796371459961,
      "activations/layer18_attention_weight_min": -43.99538803100586,
      "activations/layer19_attention_weight_max": 25.181873321533203,
      "activations/layer19_attention_weight_min": -24.4316463470459,
      "activations/layer1_attention_weight_max": 15.077718734741211,
      "activations/layer1_attention_weight_min": -12.588974952697754,
      "activations/layer20_attention_weight_max": 22.48729705810547,
      "activations/layer20_attention_weight_min": -22.494407653808594,
      "activations/layer21_attention_weight_max": 38.26972961425781,
      "activations/layer21_attention_weight_min": -26.3203182220459,
      "activations/layer22_attention_weight_max": 29.18442153930664,
      "activations/layer22_attention_weight_min": -24.599817276000977,
      "activations/layer23_attention_weight_max": 38.172420501708984,
      "activations/layer23_attention_weight_min": -24.142459869384766,
      "activations/layer2_attention_weight_max": 30.316884994506836,
      "activations/layer2_attention_weight_min": -29.504703521728516,
      "activations/layer3_attention_weight_max": 86.99185180664062,
      "activations/layer3_attention_weight_min": -89.15243530273438,
      "activations/layer4_attention_weight_max": 84.18144989013672,
      "activations/layer4_attention_weight_min": -84.8899917602539,
      "activations/layer5_attention_weight_max": 63.76436996459961,
      "activations/layer5_attention_weight_min": -69.8276138305664,
      "activations/layer6_attention_weight_max": 47.87173843383789,
      "activations/layer6_attention_weight_min": -52.2590217590332,
      "activations/layer7_attention_weight_max": 66.31095123291016,
      "activations/layer7_attention_weight_min": -64.29659271240234,
      "activations/layer8_attention_weight_max": 46.1493034362793,
      "activations/layer8_attention_weight_min": -53.913917541503906,
      "activations/layer9_attention_weight_max": 45.6592903137207,
      "activations/layer9_attention_weight_min": -50.76496505737305,
      "epoch": 20.35,
      "learning_rate": 1.893219696969697e-05,
      "loss": 2.7171,
      "step": 350300
    },
    {
      "activations/layer0_attention_weight_max": 15.895344734191895,
      "activations/layer0_attention_weight_min": -13.800060272216797,
      "activations/layer10_attention_weight_max": 39.70924758911133,
      "activations/layer10_attention_weight_min": -37.52164840698242,
      "activations/layer11_attention_weight_max": 35.24270248413086,
      "activations/layer11_attention_weight_min": -36.299476623535156,
      "activations/layer12_attention_weight_max": 18.60099220275879,
      "activations/layer12_attention_weight_min": -26.072336196899414,
      "activations/layer13_attention_weight_max": 34.28239822387695,
      "activations/layer13_attention_weight_min": -27.36982536315918,
      "activations/layer14_attention_weight_max": 33.939205169677734,
      "activations/layer14_attention_weight_min": -29.122007369995117,
      "activations/layer15_attention_weight_max": 33.538394927978516,
      "activations/layer15_attention_weight_min": -29.1539363861084,
      "activations/layer16_attention_weight_max": 32.18429946899414,
      "activations/layer16_attention_weight_min": -30.460657119750977,
      "activations/layer17_attention_weight_max": 47.62965774536133,
      "activations/layer17_attention_weight_min": -43.05656433105469,
      "activations/layer18_attention_weight_max": 43.04281234741211,
      "activations/layer18_attention_weight_min": -38.01015090942383,
      "activations/layer19_attention_weight_max": 22.672143936157227,
      "activations/layer19_attention_weight_min": -22.01881980895996,
      "activations/layer1_attention_weight_max": 16.139991760253906,
      "activations/layer1_attention_weight_min": -13.281561851501465,
      "activations/layer20_attention_weight_max": 19.635141372680664,
      "activations/layer20_attention_weight_min": -21.760154724121094,
      "activations/layer21_attention_weight_max": 29.92045021057129,
      "activations/layer21_attention_weight_min": -21.374067306518555,
      "activations/layer22_attention_weight_max": 27.46746063232422,
      "activations/layer22_attention_weight_min": -24.648252487182617,
      "activations/layer23_attention_weight_max": 34.22737121582031,
      "activations/layer23_attention_weight_min": -22.480772018432617,
      "activations/layer2_attention_weight_max": 31.658721923828125,
      "activations/layer2_attention_weight_min": -29.287071228027344,
      "activations/layer3_attention_weight_max": 86.58307647705078,
      "activations/layer3_attention_weight_min": -88.4225082397461,
      "activations/layer4_attention_weight_max": 86.86251068115234,
      "activations/layer4_attention_weight_min": -86.83387756347656,
      "activations/layer5_attention_weight_max": 65.52543640136719,
      "activations/layer5_attention_weight_min": -73.5513916015625,
      "activations/layer6_attention_weight_max": 50.1738395690918,
      "activations/layer6_attention_weight_min": -52.99612045288086,
      "activations/layer7_attention_weight_max": 66.79231262207031,
      "activations/layer7_attention_weight_min": -66.8606948852539,
      "activations/layer8_attention_weight_max": 50.529685974121094,
      "activations/layer8_attention_weight_min": -52.125423431396484,
      "activations/layer9_attention_weight_max": 46.325416564941406,
      "activations/layer9_attention_weight_min": -46.566017150878906,
      "epoch": 20.36,
      "learning_rate": 1.8913257575757574e-05,
      "loss": 2.7224,
      "step": 350350
    },
    {
      "activations/layer0_attention_weight_max": 15.762025833129883,
      "activations/layer0_attention_weight_min": -13.950379371643066,
      "activations/layer10_attention_weight_max": 38.06652069091797,
      "activations/layer10_attention_weight_min": -37.600311279296875,
      "activations/layer11_attention_weight_max": 35.383628845214844,
      "activations/layer11_attention_weight_min": -34.012786865234375,
      "activations/layer12_attention_weight_max": 18.544889450073242,
      "activations/layer12_attention_weight_min": -25.780860900878906,
      "activations/layer13_attention_weight_max": 32.178680419921875,
      "activations/layer13_attention_weight_min": -27.374332427978516,
      "activations/layer14_attention_weight_max": 33.893917083740234,
      "activations/layer14_attention_weight_min": -32.483551025390625,
      "activations/layer15_attention_weight_max": 31.472436904907227,
      "activations/layer15_attention_weight_min": -33.5752067565918,
      "activations/layer16_attention_weight_max": 32.575374603271484,
      "activations/layer16_attention_weight_min": -32.41514587402344,
      "activations/layer17_attention_weight_max": 50.73459243774414,
      "activations/layer17_attention_weight_min": -45.55316925048828,
      "activations/layer18_attention_weight_max": 46.1051139831543,
      "activations/layer18_attention_weight_min": -41.19851303100586,
      "activations/layer19_attention_weight_max": 25.62860679626465,
      "activations/layer19_attention_weight_min": -24.112613677978516,
      "activations/layer1_attention_weight_max": 15.464093208312988,
      "activations/layer1_attention_weight_min": -14.176786422729492,
      "activations/layer20_attention_weight_max": 22.81039810180664,
      "activations/layer20_attention_weight_min": -23.39531707763672,
      "activations/layer21_attention_weight_max": 37.38836669921875,
      "activations/layer21_attention_weight_min": -23.302824020385742,
      "activations/layer22_attention_weight_max": 28.234725952148438,
      "activations/layer22_attention_weight_min": -24.95817756652832,
      "activations/layer23_attention_weight_max": 32.76839065551758,
      "activations/layer23_attention_weight_min": -24.038679122924805,
      "activations/layer2_attention_weight_max": 28.627836227416992,
      "activations/layer2_attention_weight_min": -28.874176025390625,
      "activations/layer3_attention_weight_max": 83.24200439453125,
      "activations/layer3_attention_weight_min": -88.51742553710938,
      "activations/layer4_attention_weight_max": 79.10118865966797,
      "activations/layer4_attention_weight_min": -82.70549774169922,
      "activations/layer5_attention_weight_max": 57.34265899658203,
      "activations/layer5_attention_weight_min": -70.9853744506836,
      "activations/layer6_attention_weight_max": 46.6108512878418,
      "activations/layer6_attention_weight_min": -49.768043518066406,
      "activations/layer7_attention_weight_max": 62.217403411865234,
      "activations/layer7_attention_weight_min": -63.06169509887695,
      "activations/layer8_attention_weight_max": 46.59544372558594,
      "activations/layer8_attention_weight_min": -47.88877868652344,
      "activations/layer9_attention_weight_max": 42.45904541015625,
      "activations/layer9_attention_weight_min": -45.205650329589844,
      "epoch": 20.36,
      "learning_rate": 1.8894318181818182e-05,
      "loss": 2.7066,
      "step": 350400
    },
    {
      "activations/layer0_attention_weight_max": 16.18962860107422,
      "activations/layer0_attention_weight_min": -14.163090705871582,
      "activations/layer10_attention_weight_max": 36.29895782470703,
      "activations/layer10_attention_weight_min": -35.969017028808594,
      "activations/layer11_attention_weight_max": 32.94305419921875,
      "activations/layer11_attention_weight_min": -34.381080627441406,
      "activations/layer12_attention_weight_max": 19.308517456054688,
      "activations/layer12_attention_weight_min": -26.961837768554688,
      "activations/layer13_attention_weight_max": 35.31471633911133,
      "activations/layer13_attention_weight_min": -27.116296768188477,
      "activations/layer14_attention_weight_max": 36.16400146484375,
      "activations/layer14_attention_weight_min": -30.15114402770996,
      "activations/layer15_attention_weight_max": 36.378395080566406,
      "activations/layer15_attention_weight_min": -30.329612731933594,
      "activations/layer16_attention_weight_max": 35.04917526245117,
      "activations/layer16_attention_weight_min": -34.38948059082031,
      "activations/layer17_attention_weight_max": 50.584228515625,
      "activations/layer17_attention_weight_min": -45.93293380737305,
      "activations/layer18_attention_weight_max": 47.05731964111328,
      "activations/layer18_attention_weight_min": -42.306644439697266,
      "activations/layer19_attention_weight_max": 24.181480407714844,
      "activations/layer19_attention_weight_min": -22.282060623168945,
      "activations/layer1_attention_weight_max": 15.480854988098145,
      "activations/layer1_attention_weight_min": -13.802495956420898,
      "activations/layer20_attention_weight_max": 24.746503829956055,
      "activations/layer20_attention_weight_min": -21.604576110839844,
      "activations/layer21_attention_weight_max": 35.71989059448242,
      "activations/layer21_attention_weight_min": -24.823883056640625,
      "activations/layer22_attention_weight_max": 29.02177619934082,
      "activations/layer22_attention_weight_min": -24.511384963989258,
      "activations/layer23_attention_weight_max": 33.83744430541992,
      "activations/layer23_attention_weight_min": -23.233144760131836,
      "activations/layer2_attention_weight_max": 32.097713470458984,
      "activations/layer2_attention_weight_min": -29.28317642211914,
      "activations/layer3_attention_weight_max": 80.87815856933594,
      "activations/layer3_attention_weight_min": -83.470458984375,
      "activations/layer4_attention_weight_max": 75.9327621459961,
      "activations/layer4_attention_weight_min": -78.89037322998047,
      "activations/layer5_attention_weight_max": 59.66619110107422,
      "activations/layer5_attention_weight_min": -74.00108337402344,
      "activations/layer6_attention_weight_max": 47.404903411865234,
      "activations/layer6_attention_weight_min": -49.11947250366211,
      "activations/layer7_attention_weight_max": 63.3226203918457,
      "activations/layer7_attention_weight_min": -65.78984069824219,
      "activations/layer8_attention_weight_max": 46.44711685180664,
      "activations/layer8_attention_weight_min": -48.007503509521484,
      "activations/layer9_attention_weight_max": 47.836307525634766,
      "activations/layer9_attention_weight_min": -46.144779205322266,
      "epoch": 20.36,
      "learning_rate": 1.8875378787878784e-05,
      "loss": 2.7307,
      "step": 350450
    },
    {
      "activations/layer0_attention_weight_max": 15.444615364074707,
      "activations/layer0_attention_weight_min": -13.546144485473633,
      "activations/layer10_attention_weight_max": 40.444217681884766,
      "activations/layer10_attention_weight_min": -38.754642486572266,
      "activations/layer11_attention_weight_max": 38.530765533447266,
      "activations/layer11_attention_weight_min": -37.12187194824219,
      "activations/layer12_attention_weight_max": 19.196680068969727,
      "activations/layer12_attention_weight_min": -24.40866470336914,
      "activations/layer13_attention_weight_max": 37.73545455932617,
      "activations/layer13_attention_weight_min": -27.11378288269043,
      "activations/layer14_attention_weight_max": 38.809837341308594,
      "activations/layer14_attention_weight_min": -31.332897186279297,
      "activations/layer15_attention_weight_max": 35.07772445678711,
      "activations/layer15_attention_weight_min": -30.442346572875977,
      "activations/layer16_attention_weight_max": 38.417945861816406,
      "activations/layer16_attention_weight_min": -32.186378479003906,
      "activations/layer17_attention_weight_max": 54.00908279418945,
      "activations/layer17_attention_weight_min": -44.37171936035156,
      "activations/layer18_attention_weight_max": 47.8293342590332,
      "activations/layer18_attention_weight_min": -38.90349578857422,
      "activations/layer19_attention_weight_max": 23.9837703704834,
      "activations/layer19_attention_weight_min": -25.849594116210938,
      "activations/layer1_attention_weight_max": 14.952364921569824,
      "activations/layer1_attention_weight_min": -13.255743026733398,
      "activations/layer20_attention_weight_max": 21.536523818969727,
      "activations/layer20_attention_weight_min": -22.321584701538086,
      "activations/layer21_attention_weight_max": 32.19076919555664,
      "activations/layer21_attention_weight_min": -26.25082015991211,
      "activations/layer22_attention_weight_max": 28.035493850708008,
      "activations/layer22_attention_weight_min": -25.589399337768555,
      "activations/layer23_attention_weight_max": 33.16255187988281,
      "activations/layer23_attention_weight_min": -23.938261032104492,
      "activations/layer2_attention_weight_max": 29.74359893798828,
      "activations/layer2_attention_weight_min": -29.537914276123047,
      "activations/layer3_attention_weight_max": 87.50800323486328,
      "activations/layer3_attention_weight_min": -83.80937194824219,
      "activations/layer4_attention_weight_max": 83.5263900756836,
      "activations/layer4_attention_weight_min": -80.98707580566406,
      "activations/layer5_attention_weight_max": 59.67047119140625,
      "activations/layer5_attention_weight_min": -73.04887390136719,
      "activations/layer6_attention_weight_max": 49.34275817871094,
      "activations/layer6_attention_weight_min": -51.0560188293457,
      "activations/layer7_attention_weight_max": 66.84111785888672,
      "activations/layer7_attention_weight_min": -63.79754638671875,
      "activations/layer8_attention_weight_max": 47.87040710449219,
      "activations/layer8_attention_weight_min": -51.194942474365234,
      "activations/layer9_attention_weight_max": 45.16421127319336,
      "activations/layer9_attention_weight_min": -49.246986389160156,
      "epoch": 20.37,
      "learning_rate": 1.8856439393939392e-05,
      "loss": 2.7148,
      "step": 350500
    },
    {
      "activations/layer0_attention_weight_max": 15.376564025878906,
      "activations/layer0_attention_weight_min": -13.460906982421875,
      "activations/layer10_attention_weight_max": 35.60820770263672,
      "activations/layer10_attention_weight_min": -36.17823028564453,
      "activations/layer11_attention_weight_max": 33.4127082824707,
      "activations/layer11_attention_weight_min": -34.04261016845703,
      "activations/layer12_attention_weight_max": 18.500492095947266,
      "activations/layer12_attention_weight_min": -25.844749450683594,
      "activations/layer13_attention_weight_max": 32.02705383300781,
      "activations/layer13_attention_weight_min": -27.885517120361328,
      "activations/layer14_attention_weight_max": 34.63266372680664,
      "activations/layer14_attention_weight_min": -29.6803035736084,
      "activations/layer15_attention_weight_max": 32.455711364746094,
      "activations/layer15_attention_weight_min": -30.318470001220703,
      "activations/layer16_attention_weight_max": 33.563926696777344,
      "activations/layer16_attention_weight_min": -33.64202117919922,
      "activations/layer17_attention_weight_max": 50.00563430786133,
      "activations/layer17_attention_weight_min": -46.03984451293945,
      "activations/layer18_attention_weight_max": 44.66584777832031,
      "activations/layer18_attention_weight_min": -39.44682312011719,
      "activations/layer19_attention_weight_max": 23.41935920715332,
      "activations/layer19_attention_weight_min": -22.82980728149414,
      "activations/layer1_attention_weight_max": 15.664228439331055,
      "activations/layer1_attention_weight_min": -13.542353630065918,
      "activations/layer20_attention_weight_max": 19.986600875854492,
      "activations/layer20_attention_weight_min": -22.650493621826172,
      "activations/layer21_attention_weight_max": 34.69220733642578,
      "activations/layer21_attention_weight_min": -23.358366012573242,
      "activations/layer22_attention_weight_max": 27.07574462890625,
      "activations/layer22_attention_weight_min": -24.091205596923828,
      "activations/layer23_attention_weight_max": 33.56543731689453,
      "activations/layer23_attention_weight_min": -22.302501678466797,
      "activations/layer2_attention_weight_max": 31.111906051635742,
      "activations/layer2_attention_weight_min": -28.982975006103516,
      "activations/layer3_attention_weight_max": 84.38920593261719,
      "activations/layer3_attention_weight_min": -83.95865631103516,
      "activations/layer4_attention_weight_max": 79.7878189086914,
      "activations/layer4_attention_weight_min": -81.99981689453125,
      "activations/layer5_attention_weight_max": 57.910606384277344,
      "activations/layer5_attention_weight_min": -73.99217987060547,
      "activations/layer6_attention_weight_max": 45.83692169189453,
      "activations/layer6_attention_weight_min": -48.37944412231445,
      "activations/layer7_attention_weight_max": 62.194705963134766,
      "activations/layer7_attention_weight_min": -64.17481994628906,
      "activations/layer8_attention_weight_max": 47.979007720947266,
      "activations/layer8_attention_weight_min": -51.22599411010742,
      "activations/layer9_attention_weight_max": 43.71814727783203,
      "activations/layer9_attention_weight_min": -46.06763458251953,
      "epoch": 20.37,
      "learning_rate": 1.8837499999999997e-05,
      "loss": 2.7155,
      "step": 350550
    },
    {
      "activations/layer0_attention_weight_max": 15.901350975036621,
      "activations/layer0_attention_weight_min": -13.928342819213867,
      "activations/layer10_attention_weight_max": 33.628883361816406,
      "activations/layer10_attention_weight_min": -34.49906921386719,
      "activations/layer11_attention_weight_max": 31.843032836914062,
      "activations/layer11_attention_weight_min": -33.76939392089844,
      "activations/layer12_attention_weight_max": 18.650697708129883,
      "activations/layer12_attention_weight_min": -31.570985794067383,
      "activations/layer13_attention_weight_max": 32.01030731201172,
      "activations/layer13_attention_weight_min": -26.51746940612793,
      "activations/layer14_attention_weight_max": 35.36565399169922,
      "activations/layer14_attention_weight_min": -29.9112606048584,
      "activations/layer15_attention_weight_max": 34.57655715942383,
      "activations/layer15_attention_weight_min": -31.582223892211914,
      "activations/layer16_attention_weight_max": 35.61526107788086,
      "activations/layer16_attention_weight_min": -33.78008270263672,
      "activations/layer17_attention_weight_max": 53.43236541748047,
      "activations/layer17_attention_weight_min": -49.38405227661133,
      "activations/layer18_attention_weight_max": 51.688270568847656,
      "activations/layer18_attention_weight_min": -44.26361846923828,
      "activations/layer19_attention_weight_max": 24.292530059814453,
      "activations/layer19_attention_weight_min": -24.778606414794922,
      "activations/layer1_attention_weight_max": 15.056623458862305,
      "activations/layer1_attention_weight_min": -15.557156562805176,
      "activations/layer20_attention_weight_max": 24.793704986572266,
      "activations/layer20_attention_weight_min": -23.427988052368164,
      "activations/layer21_attention_weight_max": 44.49372863769531,
      "activations/layer21_attention_weight_min": -25.422983169555664,
      "activations/layer22_attention_weight_max": 30.822208404541016,
      "activations/layer22_attention_weight_min": -26.059772491455078,
      "activations/layer23_attention_weight_max": 42.909420013427734,
      "activations/layer23_attention_weight_min": -26.09639549255371,
      "activations/layer2_attention_weight_max": 30.982364654541016,
      "activations/layer2_attention_weight_min": -29.78462028503418,
      "activations/layer3_attention_weight_max": 84.60011291503906,
      "activations/layer3_attention_weight_min": -86.2314682006836,
      "activations/layer4_attention_weight_max": 79.88571166992188,
      "activations/layer4_attention_weight_min": -82.5995101928711,
      "activations/layer5_attention_weight_max": 58.248321533203125,
      "activations/layer5_attention_weight_min": -67.47589111328125,
      "activations/layer6_attention_weight_max": 47.8402214050293,
      "activations/layer6_attention_weight_min": -46.759586334228516,
      "activations/layer7_attention_weight_max": 66.84709930419922,
      "activations/layer7_attention_weight_min": -64.95328521728516,
      "activations/layer8_attention_weight_max": 44.27406692504883,
      "activations/layer8_attention_weight_min": -49.38990783691406,
      "activations/layer9_attention_weight_max": 46.76173782348633,
      "activations/layer9_attention_weight_min": -47.22438049316406,
      "epoch": 20.37,
      "learning_rate": 1.8818560606060605e-05,
      "loss": 2.711,
      "step": 350600
    },
    {
      "activations/layer0_attention_weight_max": 16.310007095336914,
      "activations/layer0_attention_weight_min": -14.043665885925293,
      "activations/layer10_attention_weight_max": 39.34829330444336,
      "activations/layer10_attention_weight_min": -34.77162551879883,
      "activations/layer11_attention_weight_max": 35.593814849853516,
      "activations/layer11_attention_weight_min": -33.346736907958984,
      "activations/layer12_attention_weight_max": 19.94011116027832,
      "activations/layer12_attention_weight_min": -24.52961540222168,
      "activations/layer13_attention_weight_max": 35.14881896972656,
      "activations/layer13_attention_weight_min": -28.40169334411621,
      "activations/layer14_attention_weight_max": 35.96182632446289,
      "activations/layer14_attention_weight_min": -31.527645111083984,
      "activations/layer15_attention_weight_max": 33.67228698730469,
      "activations/layer15_attention_weight_min": -31.192405700683594,
      "activations/layer16_attention_weight_max": 39.14717102050781,
      "activations/layer16_attention_weight_min": -38.262062072753906,
      "activations/layer17_attention_weight_max": 53.57365798950195,
      "activations/layer17_attention_weight_min": -50.24617385864258,
      "activations/layer18_attention_weight_max": 50.45207214355469,
      "activations/layer18_attention_weight_min": -43.17885971069336,
      "activations/layer19_attention_weight_max": 27.040136337280273,
      "activations/layer19_attention_weight_min": -25.582168579101562,
      "activations/layer1_attention_weight_max": 15.290921211242676,
      "activations/layer1_attention_weight_min": -14.329564094543457,
      "activations/layer20_attention_weight_max": 24.891550064086914,
      "activations/layer20_attention_weight_min": -24.816293716430664,
      "activations/layer21_attention_weight_max": 39.95226287841797,
      "activations/layer21_attention_weight_min": -23.835824966430664,
      "activations/layer22_attention_weight_max": 31.839550018310547,
      "activations/layer22_attention_weight_min": -26.936996459960938,
      "activations/layer23_attention_weight_max": 36.43522644042969,
      "activations/layer23_attention_weight_min": -26.4578914642334,
      "activations/layer2_attention_weight_max": 30.800186157226562,
      "activations/layer2_attention_weight_min": -31.702421188354492,
      "activations/layer3_attention_weight_max": 85.82510375976562,
      "activations/layer3_attention_weight_min": -93.29232025146484,
      "activations/layer4_attention_weight_max": 82.87227630615234,
      "activations/layer4_attention_weight_min": -86.17597198486328,
      "activations/layer5_attention_weight_max": 58.80659484863281,
      "activations/layer5_attention_weight_min": -71.23081970214844,
      "activations/layer6_attention_weight_max": 48.02568817138672,
      "activations/layer6_attention_weight_min": -48.26081466674805,
      "activations/layer7_attention_weight_max": 61.11124038696289,
      "activations/layer7_attention_weight_min": -61.41514587402344,
      "activations/layer8_attention_weight_max": 47.12733840942383,
      "activations/layer8_attention_weight_min": -46.83378982543945,
      "activations/layer9_attention_weight_max": 45.986629486083984,
      "activations/layer9_attention_weight_min": -45.22125244140625,
      "epoch": 20.37,
      "learning_rate": 1.879962121212121e-05,
      "loss": 2.706,
      "step": 350650
    },
    {
      "activations/layer0_attention_weight_max": 15.765585899353027,
      "activations/layer0_attention_weight_min": -13.9774751663208,
      "activations/layer10_attention_weight_max": 34.487091064453125,
      "activations/layer10_attention_weight_min": -36.269256591796875,
      "activations/layer11_attention_weight_max": 31.45160675048828,
      "activations/layer11_attention_weight_min": -36.29244613647461,
      "activations/layer12_attention_weight_max": 18.39083480834961,
      "activations/layer12_attention_weight_min": -25.317039489746094,
      "activations/layer13_attention_weight_max": 34.44416427612305,
      "activations/layer13_attention_weight_min": -30.71879768371582,
      "activations/layer14_attention_weight_max": 35.101558685302734,
      "activations/layer14_attention_weight_min": -33.18871307373047,
      "activations/layer15_attention_weight_max": 32.755149841308594,
      "activations/layer15_attention_weight_min": -32.70961380004883,
      "activations/layer16_attention_weight_max": 34.717594146728516,
      "activations/layer16_attention_weight_min": -32.80470275878906,
      "activations/layer17_attention_weight_max": 51.550689697265625,
      "activations/layer17_attention_weight_min": -51.47551345825195,
      "activations/layer18_attention_weight_max": 49.99333572387695,
      "activations/layer18_attention_weight_min": -44.03982925415039,
      "activations/layer19_attention_weight_max": 24.74802017211914,
      "activations/layer19_attention_weight_min": -22.56418800354004,
      "activations/layer1_attention_weight_max": 16.5783748626709,
      "activations/layer1_attention_weight_min": -13.901177406311035,
      "activations/layer20_attention_weight_max": 23.09910774230957,
      "activations/layer20_attention_weight_min": -22.67074966430664,
      "activations/layer21_attention_weight_max": 39.781005859375,
      "activations/layer21_attention_weight_min": -23.674272537231445,
      "activations/layer22_attention_weight_max": 34.22010803222656,
      "activations/layer22_attention_weight_min": -27.998111724853516,
      "activations/layer23_attention_weight_max": 35.927406311035156,
      "activations/layer23_attention_weight_min": -23.512836456298828,
      "activations/layer2_attention_weight_max": 29.694631576538086,
      "activations/layer2_attention_weight_min": -28.975017547607422,
      "activations/layer3_attention_weight_max": 81.9028549194336,
      "activations/layer3_attention_weight_min": -83.93939208984375,
      "activations/layer4_attention_weight_max": 80.21041107177734,
      "activations/layer4_attention_weight_min": -81.71163177490234,
      "activations/layer5_attention_weight_max": 64.58108520507812,
      "activations/layer5_attention_weight_min": -76.55252838134766,
      "activations/layer6_attention_weight_max": 50.18157196044922,
      "activations/layer6_attention_weight_min": -50.050418853759766,
      "activations/layer7_attention_weight_max": 64.7892074584961,
      "activations/layer7_attention_weight_min": -64.09608459472656,
      "activations/layer8_attention_weight_max": 48.33146286010742,
      "activations/layer8_attention_weight_min": -52.31321334838867,
      "activations/layer9_attention_weight_max": 45.53236389160156,
      "activations/layer9_attention_weight_min": -48.42628479003906,
      "epoch": 20.38,
      "learning_rate": 1.878068181818182e-05,
      "loss": 2.732,
      "step": 350700
    },
    {
      "activations/layer0_attention_weight_max": 16.355133056640625,
      "activations/layer0_attention_weight_min": -14.022331237792969,
      "activations/layer10_attention_weight_max": 34.45262908935547,
      "activations/layer10_attention_weight_min": -34.460330963134766,
      "activations/layer11_attention_weight_max": 34.09351348876953,
      "activations/layer11_attention_weight_min": -33.19457244873047,
      "activations/layer12_attention_weight_max": 18.580041885375977,
      "activations/layer12_attention_weight_min": -25.335412979125977,
      "activations/layer13_attention_weight_max": 33.309322357177734,
      "activations/layer13_attention_weight_min": -26.548471450805664,
      "activations/layer14_attention_weight_max": 33.15403747558594,
      "activations/layer14_attention_weight_min": -30.864849090576172,
      "activations/layer15_attention_weight_max": 32.592918395996094,
      "activations/layer15_attention_weight_min": -30.411502838134766,
      "activations/layer16_attention_weight_max": 32.57392883300781,
      "activations/layer16_attention_weight_min": -30.71089744567871,
      "activations/layer17_attention_weight_max": 46.73648452758789,
      "activations/layer17_attention_weight_min": -43.7280387878418,
      "activations/layer18_attention_weight_max": 44.84220504760742,
      "activations/layer18_attention_weight_min": -37.649497985839844,
      "activations/layer19_attention_weight_max": 22.770811080932617,
      "activations/layer19_attention_weight_min": -23.149106979370117,
      "activations/layer1_attention_weight_max": 15.476766586303711,
      "activations/layer1_attention_weight_min": -14.194353103637695,
      "activations/layer20_attention_weight_max": 21.916828155517578,
      "activations/layer20_attention_weight_min": -20.79132652282715,
      "activations/layer21_attention_weight_max": 33.7685546875,
      "activations/layer21_attention_weight_min": -23.643558502197266,
      "activations/layer22_attention_weight_max": 28.627405166625977,
      "activations/layer22_attention_weight_min": -27.23906898498535,
      "activations/layer23_attention_weight_max": 36.6989631652832,
      "activations/layer23_attention_weight_min": -23.811969757080078,
      "activations/layer2_attention_weight_max": 30.815914154052734,
      "activations/layer2_attention_weight_min": -30.498703002929688,
      "activations/layer3_attention_weight_max": 86.8751449584961,
      "activations/layer3_attention_weight_min": -87.33747100830078,
      "activations/layer4_attention_weight_max": 82.86207580566406,
      "activations/layer4_attention_weight_min": -84.12918090820312,
      "activations/layer5_attention_weight_max": 57.946800231933594,
      "activations/layer5_attention_weight_min": -69.67662048339844,
      "activations/layer6_attention_weight_max": 49.0350456237793,
      "activations/layer6_attention_weight_min": -50.29356002807617,
      "activations/layer7_attention_weight_max": 64.57243347167969,
      "activations/layer7_attention_weight_min": -66.21565246582031,
      "activations/layer8_attention_weight_max": 45.91447830200195,
      "activations/layer8_attention_weight_min": -49.86601257324219,
      "activations/layer9_attention_weight_max": 42.584312438964844,
      "activations/layer9_attention_weight_min": -45.195457458496094,
      "epoch": 20.38,
      "learning_rate": 1.876174242424242e-05,
      "loss": 2.7186,
      "step": 350750
    },
    {
      "activations/layer0_attention_weight_max": 15.623857498168945,
      "activations/layer0_attention_weight_min": -14.054560661315918,
      "activations/layer10_attention_weight_max": 35.51923370361328,
      "activations/layer10_attention_weight_min": -35.717185974121094,
      "activations/layer11_attention_weight_max": 33.067935943603516,
      "activations/layer11_attention_weight_min": -36.7482795715332,
      "activations/layer12_attention_weight_max": 19.708948135375977,
      "activations/layer12_attention_weight_min": -26.031373977661133,
      "activations/layer13_attention_weight_max": 29.417633056640625,
      "activations/layer13_attention_weight_min": -27.19356918334961,
      "activations/layer14_attention_weight_max": 34.99898147583008,
      "activations/layer14_attention_weight_min": -29.114131927490234,
      "activations/layer15_attention_weight_max": 32.04034423828125,
      "activations/layer15_attention_weight_min": -29.544092178344727,
      "activations/layer16_attention_weight_max": 31.384769439697266,
      "activations/layer16_attention_weight_min": -30.138134002685547,
      "activations/layer17_attention_weight_max": 47.85463333129883,
      "activations/layer17_attention_weight_min": -41.62998580932617,
      "activations/layer18_attention_weight_max": 43.52576446533203,
      "activations/layer18_attention_weight_min": -37.157615661621094,
      "activations/layer19_attention_weight_max": 25.35279655456543,
      "activations/layer19_attention_weight_min": -24.06989860534668,
      "activations/layer1_attention_weight_max": 15.865910530090332,
      "activations/layer1_attention_weight_min": -13.864847183227539,
      "activations/layer20_attention_weight_max": 23.344860076904297,
      "activations/layer20_attention_weight_min": -21.529539108276367,
      "activations/layer21_attention_weight_max": 37.125247955322266,
      "activations/layer21_attention_weight_min": -19.82187271118164,
      "activations/layer22_attention_weight_max": 30.98082160949707,
      "activations/layer22_attention_weight_min": -26.886899948120117,
      "activations/layer23_attention_weight_max": 38.284454345703125,
      "activations/layer23_attention_weight_min": -23.234447479248047,
      "activations/layer2_attention_weight_max": 31.567066192626953,
      "activations/layer2_attention_weight_min": -30.71335220336914,
      "activations/layer3_attention_weight_max": 83.69400787353516,
      "activations/layer3_attention_weight_min": -91.85054779052734,
      "activations/layer4_attention_weight_max": 84.17008972167969,
      "activations/layer4_attention_weight_min": -85.65325927734375,
      "activations/layer5_attention_weight_max": 62.692771911621094,
      "activations/layer5_attention_weight_min": -70.4720458984375,
      "activations/layer6_attention_weight_max": 50.40748977661133,
      "activations/layer6_attention_weight_min": -50.95711135864258,
      "activations/layer7_attention_weight_max": 66.01695251464844,
      "activations/layer7_attention_weight_min": -64.14663696289062,
      "activations/layer8_attention_weight_max": 47.02168273925781,
      "activations/layer8_attention_weight_min": -51.94578170776367,
      "activations/layer9_attention_weight_max": 44.383602142333984,
      "activations/layer9_attention_weight_min": -44.28664016723633,
      "epoch": 20.38,
      "learning_rate": 1.8742803030303028e-05,
      "loss": 2.7103,
      "step": 350800
    },
    {
      "activations/layer0_attention_weight_max": 14.99191951751709,
      "activations/layer0_attention_weight_min": -13.603569030761719,
      "activations/layer10_attention_weight_max": 36.78440475463867,
      "activations/layer10_attention_weight_min": -36.584747314453125,
      "activations/layer11_attention_weight_max": 31.839420318603516,
      "activations/layer11_attention_weight_min": -33.47140121459961,
      "activations/layer12_attention_weight_max": 18.328189849853516,
      "activations/layer12_attention_weight_min": -21.753721237182617,
      "activations/layer13_attention_weight_max": 32.123565673828125,
      "activations/layer13_attention_weight_min": -26.883676528930664,
      "activations/layer14_attention_weight_max": 34.01573944091797,
      "activations/layer14_attention_weight_min": -30.624988555908203,
      "activations/layer15_attention_weight_max": 36.268558502197266,
      "activations/layer15_attention_weight_min": -31.781829833984375,
      "activations/layer16_attention_weight_max": 33.88574981689453,
      "activations/layer16_attention_weight_min": -32.19737243652344,
      "activations/layer17_attention_weight_max": 54.66219711303711,
      "activations/layer17_attention_weight_min": -48.059783935546875,
      "activations/layer18_attention_weight_max": 47.54450225830078,
      "activations/layer18_attention_weight_min": -39.2566032409668,
      "activations/layer19_attention_weight_max": 25.71297264099121,
      "activations/layer19_attention_weight_min": -21.42070770263672,
      "activations/layer1_attention_weight_max": 15.63834285736084,
      "activations/layer1_attention_weight_min": -13.653785705566406,
      "activations/layer20_attention_weight_max": 22.112672805786133,
      "activations/layer20_attention_weight_min": -21.254222869873047,
      "activations/layer21_attention_weight_max": 34.00503158569336,
      "activations/layer21_attention_weight_min": -21.805423736572266,
      "activations/layer22_attention_weight_max": 28.61383819580078,
      "activations/layer22_attention_weight_min": -24.992267608642578,
      "activations/layer23_attention_weight_max": 36.10787582397461,
      "activations/layer23_attention_weight_min": -24.857389450073242,
      "activations/layer2_attention_weight_max": 30.700735092163086,
      "activations/layer2_attention_weight_min": -29.830440521240234,
      "activations/layer3_attention_weight_max": 82.697265625,
      "activations/layer3_attention_weight_min": -85.78665161132812,
      "activations/layer4_attention_weight_max": 81.52173614501953,
      "activations/layer4_attention_weight_min": -82.5387191772461,
      "activations/layer5_attention_weight_max": 58.54784393310547,
      "activations/layer5_attention_weight_min": -70.53394317626953,
      "activations/layer6_attention_weight_max": 47.119205474853516,
      "activations/layer6_attention_weight_min": -48.91102600097656,
      "activations/layer7_attention_weight_max": 64.47003173828125,
      "activations/layer7_attention_weight_min": -65.64544677734375,
      "activations/layer8_attention_weight_max": 48.19093322753906,
      "activations/layer8_attention_weight_min": -52.7574348449707,
      "activations/layer9_attention_weight_max": 45.320980072021484,
      "activations/layer9_attention_weight_min": -46.18391036987305,
      "epoch": 20.39,
      "learning_rate": 1.8723863636363633e-05,
      "loss": 2.715,
      "step": 350850
    },
    {
      "activations/layer0_attention_weight_max": 15.830069541931152,
      "activations/layer0_attention_weight_min": -14.08285903930664,
      "activations/layer10_attention_weight_max": 35.05403518676758,
      "activations/layer10_attention_weight_min": -34.088844299316406,
      "activations/layer11_attention_weight_max": 30.99913787841797,
      "activations/layer11_attention_weight_min": -33.675025939941406,
      "activations/layer12_attention_weight_max": 18.220569610595703,
      "activations/layer12_attention_weight_min": -22.81603240966797,
      "activations/layer13_attention_weight_max": 34.31044006347656,
      "activations/layer13_attention_weight_min": -30.227479934692383,
      "activations/layer14_attention_weight_max": 33.17192077636719,
      "activations/layer14_attention_weight_min": -31.359464645385742,
      "activations/layer15_attention_weight_max": 33.43695068359375,
      "activations/layer15_attention_weight_min": -30.703763961791992,
      "activations/layer16_attention_weight_max": 33.58919143676758,
      "activations/layer16_attention_weight_min": -35.03177261352539,
      "activations/layer17_attention_weight_max": 46.23456573486328,
      "activations/layer17_attention_weight_min": -47.342308044433594,
      "activations/layer18_attention_weight_max": 43.0414924621582,
      "activations/layer18_attention_weight_min": -44.86030578613281,
      "activations/layer19_attention_weight_max": 22.70148277282715,
      "activations/layer19_attention_weight_min": -27.610666275024414,
      "activations/layer1_attention_weight_max": 15.642692565917969,
      "activations/layer1_attention_weight_min": -14.301716804504395,
      "activations/layer20_attention_weight_max": 21.4444522857666,
      "activations/layer20_attention_weight_min": -26.685762405395508,
      "activations/layer21_attention_weight_max": 35.77022171020508,
      "activations/layer21_attention_weight_min": -24.25420379638672,
      "activations/layer22_attention_weight_max": 25.787433624267578,
      "activations/layer22_attention_weight_min": -28.17736053466797,
      "activations/layer23_attention_weight_max": 34.66688919067383,
      "activations/layer23_attention_weight_min": -27.05590057373047,
      "activations/layer2_attention_weight_max": 31.765277862548828,
      "activations/layer2_attention_weight_min": -30.907867431640625,
      "activations/layer3_attention_weight_max": 85.39276123046875,
      "activations/layer3_attention_weight_min": -87.55695343017578,
      "activations/layer4_attention_weight_max": 80.27593231201172,
      "activations/layer4_attention_weight_min": -80.05486297607422,
      "activations/layer5_attention_weight_max": 60.559776306152344,
      "activations/layer5_attention_weight_min": -66.96370697021484,
      "activations/layer6_attention_weight_max": 46.981163024902344,
      "activations/layer6_attention_weight_min": -49.49950408935547,
      "activations/layer7_attention_weight_max": 63.3961067199707,
      "activations/layer7_attention_weight_min": -60.24828338623047,
      "activations/layer8_attention_weight_max": 43.04651641845703,
      "activations/layer8_attention_weight_min": -46.5660285949707,
      "activations/layer9_attention_weight_max": 44.095516204833984,
      "activations/layer9_attention_weight_min": -41.308937072753906,
      "epoch": 20.39,
      "learning_rate": 1.870492424242424e-05,
      "loss": 2.7281,
      "step": 350900
    },
    {
      "activations/layer0_attention_weight_max": 15.186878204345703,
      "activations/layer0_attention_weight_min": -14.07422924041748,
      "activations/layer10_attention_weight_max": 35.9453010559082,
      "activations/layer10_attention_weight_min": -36.87383270263672,
      "activations/layer11_attention_weight_max": 34.070762634277344,
      "activations/layer11_attention_weight_min": -36.32063293457031,
      "activations/layer12_attention_weight_max": 19.053991317749023,
      "activations/layer12_attention_weight_min": -26.349205017089844,
      "activations/layer13_attention_weight_max": 33.5430908203125,
      "activations/layer13_attention_weight_min": -27.608449935913086,
      "activations/layer14_attention_weight_max": 36.79475784301758,
      "activations/layer14_attention_weight_min": -31.878618240356445,
      "activations/layer15_attention_weight_max": 35.17619323730469,
      "activations/layer15_attention_weight_min": -30.290775299072266,
      "activations/layer16_attention_weight_max": 36.58517837524414,
      "activations/layer16_attention_weight_min": -33.61324691772461,
      "activations/layer17_attention_weight_max": 53.224308013916016,
      "activations/layer17_attention_weight_min": -48.099029541015625,
      "activations/layer18_attention_weight_max": 48.10261917114258,
      "activations/layer18_attention_weight_min": -40.5834846496582,
      "activations/layer19_attention_weight_max": 24.203561782836914,
      "activations/layer19_attention_weight_min": -21.92786979675293,
      "activations/layer1_attention_weight_max": 15.974836349487305,
      "activations/layer1_attention_weight_min": -14.533411026000977,
      "activations/layer20_attention_weight_max": 23.63669204711914,
      "activations/layer20_attention_weight_min": -21.628997802734375,
      "activations/layer21_attention_weight_max": 35.47400665283203,
      "activations/layer21_attention_weight_min": -21.0760555267334,
      "activations/layer22_attention_weight_max": 32.94660568237305,
      "activations/layer22_attention_weight_min": -25.048480987548828,
      "activations/layer23_attention_weight_max": 35.12080383300781,
      "activations/layer23_attention_weight_min": -24.060171127319336,
      "activations/layer2_attention_weight_max": 30.054006576538086,
      "activations/layer2_attention_weight_min": -29.069684982299805,
      "activations/layer3_attention_weight_max": 82.19873809814453,
      "activations/layer3_attention_weight_min": -85.41476440429688,
      "activations/layer4_attention_weight_max": 80.86937713623047,
      "activations/layer4_attention_weight_min": -80.6680908203125,
      "activations/layer5_attention_weight_max": 59.02207946777344,
      "activations/layer5_attention_weight_min": -69.30599212646484,
      "activations/layer6_attention_weight_max": 46.5296745300293,
      "activations/layer6_attention_weight_min": -51.533668518066406,
      "activations/layer7_attention_weight_max": 64.75396728515625,
      "activations/layer7_attention_weight_min": -64.52452087402344,
      "activations/layer8_attention_weight_max": 45.43498229980469,
      "activations/layer8_attention_weight_min": -47.90425491333008,
      "activations/layer9_attention_weight_max": 40.63060760498047,
      "activations/layer9_attention_weight_min": -46.53007125854492,
      "epoch": 20.39,
      "learning_rate": 1.8685984848484846e-05,
      "loss": 2.7204,
      "step": 350950
    },
    {
      "activations/layer0_attention_weight_max": 15.371501922607422,
      "activations/layer0_attention_weight_min": -13.858641624450684,
      "activations/layer10_attention_weight_max": 34.39618682861328,
      "activations/layer10_attention_weight_min": -34.42009353637695,
      "activations/layer11_attention_weight_max": 32.83525848388672,
      "activations/layer11_attention_weight_min": -33.935482025146484,
      "activations/layer12_attention_weight_max": 17.87125015258789,
      "activations/layer12_attention_weight_min": -27.710969924926758,
      "activations/layer13_attention_weight_max": 31.46267318725586,
      "activations/layer13_attention_weight_min": -28.49593734741211,
      "activations/layer14_attention_weight_max": 31.29168701171875,
      "activations/layer14_attention_weight_min": -30.864953994750977,
      "activations/layer15_attention_weight_max": 31.08235740661621,
      "activations/layer15_attention_weight_min": -30.188709259033203,
      "activations/layer16_attention_weight_max": 33.004329681396484,
      "activations/layer16_attention_weight_min": -32.5364990234375,
      "activations/layer17_attention_weight_max": 48.95546340942383,
      "activations/layer17_attention_weight_min": -44.69585037231445,
      "activations/layer18_attention_weight_max": 47.91746520996094,
      "activations/layer18_attention_weight_min": -41.425086975097656,
      "activations/layer19_attention_weight_max": 22.3161678314209,
      "activations/layer19_attention_weight_min": -20.831022262573242,
      "activations/layer1_attention_weight_max": 16.341806411743164,
      "activations/layer1_attention_weight_min": -15.000917434692383,
      "activations/layer20_attention_weight_max": 20.752315521240234,
      "activations/layer20_attention_weight_min": -20.437583923339844,
      "activations/layer21_attention_weight_max": 35.9426383972168,
      "activations/layer21_attention_weight_min": -20.710262298583984,
      "activations/layer22_attention_weight_max": 26.502946853637695,
      "activations/layer22_attention_weight_min": -25.679338455200195,
      "activations/layer23_attention_weight_max": 34.34192657470703,
      "activations/layer23_attention_weight_min": -23.66510009765625,
      "activations/layer2_attention_weight_max": 31.139677047729492,
      "activations/layer2_attention_weight_min": -29.29438018798828,
      "activations/layer3_attention_weight_max": 85.58724212646484,
      "activations/layer3_attention_weight_min": -87.09996795654297,
      "activations/layer4_attention_weight_max": 82.97837829589844,
      "activations/layer4_attention_weight_min": -82.32933807373047,
      "activations/layer5_attention_weight_max": 57.5518913269043,
      "activations/layer5_attention_weight_min": -70.75157165527344,
      "activations/layer6_attention_weight_max": 47.54123306274414,
      "activations/layer6_attention_weight_min": -48.84702682495117,
      "activations/layer7_attention_weight_max": 63.42085647583008,
      "activations/layer7_attention_weight_min": -64.27655792236328,
      "activations/layer8_attention_weight_max": 43.28140640258789,
      "activations/layer8_attention_weight_min": -47.73732376098633,
      "activations/layer9_attention_weight_max": 40.79331970214844,
      "activations/layer9_attention_weight_min": -44.90322494506836,
      "epoch": 20.4,
      "learning_rate": 1.866704545454545e-05,
      "loss": 2.7251,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.5813,
      "eval_samples_per_second": 500.391,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.5813,
      "eval_openwebtext_samples_per_second": 500.391,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 2.0138,
      "eval_wikitext_samples_per_second": 226.435,
      "step": 351000
    },
    {
      "epoch": 20.4,
      "eval_lambada_loss": 2.416015625,
      "eval_lambada_ppl": 11.201140741117625,
      "eval_lambada_runtime": 9.6655,
      "eval_lambada_samples_per_second": 503.752,
      "step": 351000
    },
    {
      "activations/layer0_attention_weight_max": 16.052894592285156,
      "activations/layer0_attention_weight_min": -13.775228500366211,
      "activations/layer10_attention_weight_max": 35.715003967285156,
      "activations/layer10_attention_weight_min": -34.204803466796875,
      "activations/layer11_attention_weight_max": 32.424251556396484,
      "activations/layer11_attention_weight_min": -34.69614028930664,
      "activations/layer12_attention_weight_max": 18.445737838745117,
      "activations/layer12_attention_weight_min": -26.48969841003418,
      "activations/layer13_attention_weight_max": 31.76742172241211,
      "activations/layer13_attention_weight_min": -26.46701431274414,
      "activations/layer14_attention_weight_max": 35.462066650390625,
      "activations/layer14_attention_weight_min": -30.69500160217285,
      "activations/layer15_attention_weight_max": 33.38151931762695,
      "activations/layer15_attention_weight_min": -30.171875,
      "activations/layer16_attention_weight_max": 36.904869079589844,
      "activations/layer16_attention_weight_min": -31.698501586914062,
      "activations/layer17_attention_weight_max": 54.16114807128906,
      "activations/layer17_attention_weight_min": -48.072898864746094,
      "activations/layer18_attention_weight_max": 51.65571212768555,
      "activations/layer18_attention_weight_min": -39.623592376708984,
      "activations/layer19_attention_weight_max": 25.805801391601562,
      "activations/layer19_attention_weight_min": -24.38275718688965,
      "activations/layer1_attention_weight_max": 15.478216171264648,
      "activations/layer1_attention_weight_min": -12.977804183959961,
      "activations/layer20_attention_weight_max": 23.256629943847656,
      "activations/layer20_attention_weight_min": -23.147632598876953,
      "activations/layer21_attention_weight_max": 34.935516357421875,
      "activations/layer21_attention_weight_min": -22.518526077270508,
      "activations/layer22_attention_weight_max": 30.620882034301758,
      "activations/layer22_attention_weight_min": -26.75203514099121,
      "activations/layer23_attention_weight_max": 35.62812805175781,
      "activations/layer23_attention_weight_min": -24.220905303955078,
      "activations/layer2_attention_weight_max": 31.327106475830078,
      "activations/layer2_attention_weight_min": -30.39662742614746,
      "activations/layer3_attention_weight_max": 85.97077941894531,
      "activations/layer3_attention_weight_min": -85.96648406982422,
      "activations/layer4_attention_weight_max": 81.0115737915039,
      "activations/layer4_attention_weight_min": -80.39147186279297,
      "activations/layer5_attention_weight_max": 60.666114807128906,
      "activations/layer5_attention_weight_min": -74.29818725585938,
      "activations/layer6_attention_weight_max": 48.14409637451172,
      "activations/layer6_attention_weight_min": -50.116939544677734,
      "activations/layer7_attention_weight_max": 63.2364501953125,
      "activations/layer7_attention_weight_min": -67.85729217529297,
      "activations/layer8_attention_weight_max": 47.083614349365234,
      "activations/layer8_attention_weight_min": -48.52457046508789,
      "activations/layer9_attention_weight_max": 42.81148147583008,
      "activations/layer9_attention_weight_min": -44.6532096862793,
      "epoch": 20.4,
      "learning_rate": 1.864810606060606e-05,
      "loss": 2.7139,
      "step": 351050
    },
    {
      "activations/layer0_attention_weight_max": 15.671277046203613,
      "activations/layer0_attention_weight_min": -13.652741432189941,
      "activations/layer10_attention_weight_max": 39.308982849121094,
      "activations/layer10_attention_weight_min": -37.60478973388672,
      "activations/layer11_attention_weight_max": 35.4027214050293,
      "activations/layer11_attention_weight_min": -36.45719909667969,
      "activations/layer12_attention_weight_max": 19.28608512878418,
      "activations/layer12_attention_weight_min": -27.275352478027344,
      "activations/layer13_attention_weight_max": 34.8092155456543,
      "activations/layer13_attention_weight_min": -26.390552520751953,
      "activations/layer14_attention_weight_max": 34.04273223876953,
      "activations/layer14_attention_weight_min": -29.227294921875,
      "activations/layer15_attention_weight_max": 32.18495559692383,
      "activations/layer15_attention_weight_min": -30.097246170043945,
      "activations/layer16_attention_weight_max": 33.14115524291992,
      "activations/layer16_attention_weight_min": -30.96434783935547,
      "activations/layer17_attention_weight_max": 51.8554573059082,
      "activations/layer17_attention_weight_min": -44.02658462524414,
      "activations/layer18_attention_weight_max": 48.97118377685547,
      "activations/layer18_attention_weight_min": -38.553897857666016,
      "activations/layer19_attention_weight_max": 25.30620574951172,
      "activations/layer19_attention_weight_min": -21.99109649658203,
      "activations/layer1_attention_weight_max": 14.756938934326172,
      "activations/layer1_attention_weight_min": -13.049965858459473,
      "activations/layer20_attention_weight_max": 23.082914352416992,
      "activations/layer20_attention_weight_min": -22.441051483154297,
      "activations/layer21_attention_weight_max": 33.63694763183594,
      "activations/layer21_attention_weight_min": -21.61745834350586,
      "activations/layer22_attention_weight_max": 29.329803466796875,
      "activations/layer22_attention_weight_min": -25.374977111816406,
      "activations/layer23_attention_weight_max": 36.21025848388672,
      "activations/layer23_attention_weight_min": -23.18458366394043,
      "activations/layer2_attention_weight_max": 31.647693634033203,
      "activations/layer2_attention_weight_min": -31.700048446655273,
      "activations/layer3_attention_weight_max": 87.1697769165039,
      "activations/layer3_attention_weight_min": -96.66388702392578,
      "activations/layer4_attention_weight_max": 82.15560150146484,
      "activations/layer4_attention_weight_min": -84.50408172607422,
      "activations/layer5_attention_weight_max": 61.07468795776367,
      "activations/layer5_attention_weight_min": -73.26846313476562,
      "activations/layer6_attention_weight_max": 48.21842956542969,
      "activations/layer6_attention_weight_min": -51.26517868041992,
      "activations/layer7_attention_weight_max": 66.11573791503906,
      "activations/layer7_attention_weight_min": -66.95945739746094,
      "activations/layer8_attention_weight_max": 47.3260383605957,
      "activations/layer8_attention_weight_min": -52.63255310058594,
      "activations/layer9_attention_weight_max": 47.7092399597168,
      "activations/layer9_attention_weight_min": -47.11007308959961,
      "epoch": 20.4,
      "learning_rate": 1.8629166666666664e-05,
      "loss": 2.7352,
      "step": 351100
    },
    {
      "activations/layer0_attention_weight_max": 15.98455810546875,
      "activations/layer0_attention_weight_min": -13.667450904846191,
      "activations/layer10_attention_weight_max": 35.238956451416016,
      "activations/layer10_attention_weight_min": -34.75429153442383,
      "activations/layer11_attention_weight_max": 32.27123260498047,
      "activations/layer11_attention_weight_min": -34.35590362548828,
      "activations/layer12_attention_weight_max": 18.9693603515625,
      "activations/layer12_attention_weight_min": -25.802295684814453,
      "activations/layer13_attention_weight_max": 31.94130516052246,
      "activations/layer13_attention_weight_min": -26.138866424560547,
      "activations/layer14_attention_weight_max": 33.89971923828125,
      "activations/layer14_attention_weight_min": -29.497251510620117,
      "activations/layer15_attention_weight_max": 34.60472869873047,
      "activations/layer15_attention_weight_min": -29.12760353088379,
      "activations/layer16_attention_weight_max": 32.25529098510742,
      "activations/layer16_attention_weight_min": -31.158069610595703,
      "activations/layer17_attention_weight_max": 51.64204406738281,
      "activations/layer17_attention_weight_min": -45.1948356628418,
      "activations/layer18_attention_weight_max": 48.906612396240234,
      "activations/layer18_attention_weight_min": -39.58849334716797,
      "activations/layer19_attention_weight_max": 29.141864776611328,
      "activations/layer19_attention_weight_min": -24.430519104003906,
      "activations/layer1_attention_weight_max": 15.32888126373291,
      "activations/layer1_attention_weight_min": -14.341320037841797,
      "activations/layer20_attention_weight_max": 24.263553619384766,
      "activations/layer20_attention_weight_min": -23.49250030517578,
      "activations/layer21_attention_weight_max": 32.54345703125,
      "activations/layer21_attention_weight_min": -20.92080307006836,
      "activations/layer22_attention_weight_max": 31.031246185302734,
      "activations/layer22_attention_weight_min": -25.541900634765625,
      "activations/layer23_attention_weight_max": 38.55460739135742,
      "activations/layer23_attention_weight_min": -26.049449920654297,
      "activations/layer2_attention_weight_max": 31.38446807861328,
      "activations/layer2_attention_weight_min": -30.129562377929688,
      "activations/layer3_attention_weight_max": 88.28321075439453,
      "activations/layer3_attention_weight_min": -91.26702117919922,
      "activations/layer4_attention_weight_max": 82.79549407958984,
      "activations/layer4_attention_weight_min": -83.86630249023438,
      "activations/layer5_attention_weight_max": 64.50283813476562,
      "activations/layer5_attention_weight_min": -72.22330474853516,
      "activations/layer6_attention_weight_max": 46.77653884887695,
      "activations/layer6_attention_weight_min": -49.965003967285156,
      "activations/layer7_attention_weight_max": 62.698184967041016,
      "activations/layer7_attention_weight_min": -61.96937942504883,
      "activations/layer8_attention_weight_max": 45.134483337402344,
      "activations/layer8_attention_weight_min": -49.96825408935547,
      "activations/layer9_attention_weight_max": 42.935726165771484,
      "activations/layer9_attention_weight_min": -45.4210205078125,
      "epoch": 20.4,
      "learning_rate": 1.861022727272727e-05,
      "loss": 2.7225,
      "step": 351150
    },
    {
      "activations/layer0_attention_weight_max": 15.439737319946289,
      "activations/layer0_attention_weight_min": -13.735377311706543,
      "activations/layer10_attention_weight_max": 36.07551574707031,
      "activations/layer10_attention_weight_min": -35.70098114013672,
      "activations/layer11_attention_weight_max": 33.43867874145508,
      "activations/layer11_attention_weight_min": -35.93994903564453,
      "activations/layer12_attention_weight_max": 19.5034122467041,
      "activations/layer12_attention_weight_min": -22.499656677246094,
      "activations/layer13_attention_weight_max": 35.26652145385742,
      "activations/layer13_attention_weight_min": -27.499557495117188,
      "activations/layer14_attention_weight_max": 35.766292572021484,
      "activations/layer14_attention_weight_min": -28.06736946105957,
      "activations/layer15_attention_weight_max": 34.893653869628906,
      "activations/layer15_attention_weight_min": -29.7230224609375,
      "activations/layer16_attention_weight_max": 32.8708610534668,
      "activations/layer16_attention_weight_min": -29.283693313598633,
      "activations/layer17_attention_weight_max": 52.843894958496094,
      "activations/layer17_attention_weight_min": -43.68510437011719,
      "activations/layer18_attention_weight_max": 52.41276931762695,
      "activations/layer18_attention_weight_min": -37.23927688598633,
      "activations/layer19_attention_weight_max": 26.0124454498291,
      "activations/layer19_attention_weight_min": -21.100046157836914,
      "activations/layer1_attention_weight_max": 16.009674072265625,
      "activations/layer1_attention_weight_min": -12.562603950500488,
      "activations/layer20_attention_weight_max": 21.174880981445312,
      "activations/layer20_attention_weight_min": -22.694435119628906,
      "activations/layer21_attention_weight_max": 29.98539161682129,
      "activations/layer21_attention_weight_min": -21.612911224365234,
      "activations/layer22_attention_weight_max": 29.13743782043457,
      "activations/layer22_attention_weight_min": -23.249488830566406,
      "activations/layer23_attention_weight_max": 36.72023391723633,
      "activations/layer23_attention_weight_min": -24.215065002441406,
      "activations/layer2_attention_weight_max": 30.578922271728516,
      "activations/layer2_attention_weight_min": -30.52233123779297,
      "activations/layer3_attention_weight_max": 84.13530731201172,
      "activations/layer3_attention_weight_min": -85.10555267333984,
      "activations/layer4_attention_weight_max": 84.39108276367188,
      "activations/layer4_attention_weight_min": -85.21755981445312,
      "activations/layer5_attention_weight_max": 62.39324951171875,
      "activations/layer5_attention_weight_min": -70.43919372558594,
      "activations/layer6_attention_weight_max": 49.51943588256836,
      "activations/layer6_attention_weight_min": -48.94999694824219,
      "activations/layer7_attention_weight_max": 73.44223022460938,
      "activations/layer7_attention_weight_min": -66.34872436523438,
      "activations/layer8_attention_weight_max": 45.621421813964844,
      "activations/layer8_attention_weight_min": -49.6442985534668,
      "activations/layer9_attention_weight_max": 44.50096893310547,
      "activations/layer9_attention_weight_min": -45.74538803100586,
      "epoch": 20.41,
      "learning_rate": 1.8591287878787877e-05,
      "loss": 2.7356,
      "step": 351200
    },
    {
      "activations/layer0_attention_weight_max": 15.412704467773438,
      "activations/layer0_attention_weight_min": -13.786795616149902,
      "activations/layer10_attention_weight_max": 42.02924728393555,
      "activations/layer10_attention_weight_min": -42.0439453125,
      "activations/layer11_attention_weight_max": 41.27788543701172,
      "activations/layer11_attention_weight_min": -40.80054473876953,
      "activations/layer12_attention_weight_max": 20.65273094177246,
      "activations/layer12_attention_weight_min": -30.13323402404785,
      "activations/layer13_attention_weight_max": 48.34163284301758,
      "activations/layer13_attention_weight_min": -30.680240631103516,
      "activations/layer14_attention_weight_max": 53.152286529541016,
      "activations/layer14_attention_weight_min": -32.795223236083984,
      "activations/layer15_attention_weight_max": 52.946075439453125,
      "activations/layer15_attention_weight_min": -32.28460693359375,
      "activations/layer16_attention_weight_max": 37.90068817138672,
      "activations/layer16_attention_weight_min": -34.77302169799805,
      "activations/layer17_attention_weight_max": 57.31314468383789,
      "activations/layer17_attention_weight_min": -51.22673416137695,
      "activations/layer18_attention_weight_max": 49.737030029296875,
      "activations/layer18_attention_weight_min": -46.96592712402344,
      "activations/layer19_attention_weight_max": 24.360111236572266,
      "activations/layer19_attention_weight_min": -23.82436180114746,
      "activations/layer1_attention_weight_max": 15.284008026123047,
      "activations/layer1_attention_weight_min": -14.634937286376953,
      "activations/layer20_attention_weight_max": 23.069467544555664,
      "activations/layer20_attention_weight_min": -21.75033950805664,
      "activations/layer21_attention_weight_max": 39.77281188964844,
      "activations/layer21_attention_weight_min": -25.148508071899414,
      "activations/layer22_attention_weight_max": 29.86717414855957,
      "activations/layer22_attention_weight_min": -25.11137580871582,
      "activations/layer23_attention_weight_max": 33.942203521728516,
      "activations/layer23_attention_weight_min": -23.03399658203125,
      "activations/layer2_attention_weight_max": 31.923206329345703,
      "activations/layer2_attention_weight_min": -29.61166763305664,
      "activations/layer3_attention_weight_max": 85.80513763427734,
      "activations/layer3_attention_weight_min": -86.6812744140625,
      "activations/layer4_attention_weight_max": 82.91534423828125,
      "activations/layer4_attention_weight_min": -82.7085189819336,
      "activations/layer5_attention_weight_max": 61.68974304199219,
      "activations/layer5_attention_weight_min": -69.7643051147461,
      "activations/layer6_attention_weight_max": 48.87350082397461,
      "activations/layer6_attention_weight_min": -49.89462661743164,
      "activations/layer7_attention_weight_max": 75.36400604248047,
      "activations/layer7_attention_weight_min": -66.15115356445312,
      "activations/layer8_attention_weight_max": 53.88998031616211,
      "activations/layer8_attention_weight_min": -53.63144302368164,
      "activations/layer9_attention_weight_max": 57.257904052734375,
      "activations/layer9_attention_weight_min": -51.74732208251953,
      "epoch": 20.41,
      "learning_rate": 1.8572348484848482e-05,
      "loss": 2.7142,
      "step": 351250
    },
    {
      "activations/layer0_attention_weight_max": 16.24856185913086,
      "activations/layer0_attention_weight_min": -13.927519798278809,
      "activations/layer10_attention_weight_max": 37.92185974121094,
      "activations/layer10_attention_weight_min": -36.467037200927734,
      "activations/layer11_attention_weight_max": 34.899391174316406,
      "activations/layer11_attention_weight_min": -36.453086853027344,
      "activations/layer12_attention_weight_max": 17.646221160888672,
      "activations/layer12_attention_weight_min": -26.045236587524414,
      "activations/layer13_attention_weight_max": 33.652618408203125,
      "activations/layer13_attention_weight_min": -27.635990142822266,
      "activations/layer14_attention_weight_max": 34.9378662109375,
      "activations/layer14_attention_weight_min": -29.174240112304688,
      "activations/layer15_attention_weight_max": 31.962263107299805,
      "activations/layer15_attention_weight_min": -29.450237274169922,
      "activations/layer16_attention_weight_max": 33.821388244628906,
      "activations/layer16_attention_weight_min": -32.413387298583984,
      "activations/layer17_attention_weight_max": 53.80412673950195,
      "activations/layer17_attention_weight_min": -48.03110885620117,
      "activations/layer18_attention_weight_max": 46.16676330566406,
      "activations/layer18_attention_weight_min": -41.879302978515625,
      "activations/layer19_attention_weight_max": 24.145973205566406,
      "activations/layer19_attention_weight_min": -26.30291748046875,
      "activations/layer1_attention_weight_max": 16.229333877563477,
      "activations/layer1_attention_weight_min": -13.975275039672852,
      "activations/layer20_attention_weight_max": 22.02837562561035,
      "activations/layer20_attention_weight_min": -24.320568084716797,
      "activations/layer21_attention_weight_max": 37.465667724609375,
      "activations/layer21_attention_weight_min": -28.1699275970459,
      "activations/layer22_attention_weight_max": 27.14683723449707,
      "activations/layer22_attention_weight_min": -26.728527069091797,
      "activations/layer23_attention_weight_max": 35.9013557434082,
      "activations/layer23_attention_weight_min": -26.777423858642578,
      "activations/layer2_attention_weight_max": 31.644865036010742,
      "activations/layer2_attention_weight_min": -30.544532775878906,
      "activations/layer3_attention_weight_max": 86.20407104492188,
      "activations/layer3_attention_weight_min": -89.96837615966797,
      "activations/layer4_attention_weight_max": 85.43875885009766,
      "activations/layer4_attention_weight_min": -85.48055267333984,
      "activations/layer5_attention_weight_max": 62.713165283203125,
      "activations/layer5_attention_weight_min": -73.77404022216797,
      "activations/layer6_attention_weight_max": 50.18710708618164,
      "activations/layer6_attention_weight_min": -50.56382369995117,
      "activations/layer7_attention_weight_max": 67.16195678710938,
      "activations/layer7_attention_weight_min": -64.46522521972656,
      "activations/layer8_attention_weight_max": 51.29665756225586,
      "activations/layer8_attention_weight_min": -54.332942962646484,
      "activations/layer9_attention_weight_max": 46.50068664550781,
      "activations/layer9_attention_weight_min": -48.585506439208984,
      "epoch": 20.41,
      "learning_rate": 1.855340909090909e-05,
      "loss": 2.7203,
      "step": 351300
    },
    {
      "activations/layer0_attention_weight_max": 14.830923080444336,
      "activations/layer0_attention_weight_min": -13.628301620483398,
      "activations/layer10_attention_weight_max": 34.537254333496094,
      "activations/layer10_attention_weight_min": -36.55244827270508,
      "activations/layer11_attention_weight_max": 32.755001068115234,
      "activations/layer11_attention_weight_min": -36.061126708984375,
      "activations/layer12_attention_weight_max": 19.44719886779785,
      "activations/layer12_attention_weight_min": -25.698286056518555,
      "activations/layer13_attention_weight_max": 31.714805603027344,
      "activations/layer13_attention_weight_min": -27.31615447998047,
      "activations/layer14_attention_weight_max": 36.3549690246582,
      "activations/layer14_attention_weight_min": -31.27348518371582,
      "activations/layer15_attention_weight_max": 36.762386322021484,
      "activations/layer15_attention_weight_min": -30.905271530151367,
      "activations/layer16_attention_weight_max": 35.99726486206055,
      "activations/layer16_attention_weight_min": -35.0626335144043,
      "activations/layer17_attention_weight_max": 53.891204833984375,
      "activations/layer17_attention_weight_min": -49.610408782958984,
      "activations/layer18_attention_weight_max": 50.915401458740234,
      "activations/layer18_attention_weight_min": -40.476959228515625,
      "activations/layer19_attention_weight_max": 25.808713912963867,
      "activations/layer19_attention_weight_min": -22.98348617553711,
      "activations/layer1_attention_weight_max": 14.91301441192627,
      "activations/layer1_attention_weight_min": -12.184866905212402,
      "activations/layer20_attention_weight_max": 22.363426208496094,
      "activations/layer20_attention_weight_min": -22.804405212402344,
      "activations/layer21_attention_weight_max": 34.809303283691406,
      "activations/layer21_attention_weight_min": -24.01065444946289,
      "activations/layer22_attention_weight_max": 30.03702163696289,
      "activations/layer22_attention_weight_min": -25.932329177856445,
      "activations/layer23_attention_weight_max": 37.83625411987305,
      "activations/layer23_attention_weight_min": -24.728008270263672,
      "activations/layer2_attention_weight_max": 30.66861343383789,
      "activations/layer2_attention_weight_min": -29.081378936767578,
      "activations/layer3_attention_weight_max": 83.30475616455078,
      "activations/layer3_attention_weight_min": -84.77208709716797,
      "activations/layer4_attention_weight_max": 82.94844055175781,
      "activations/layer4_attention_weight_min": -81.98643493652344,
      "activations/layer5_attention_weight_max": 60.50244140625,
      "activations/layer5_attention_weight_min": -71.94993591308594,
      "activations/layer6_attention_weight_max": 48.94291687011719,
      "activations/layer6_attention_weight_min": -49.41221237182617,
      "activations/layer7_attention_weight_max": 67.25642395019531,
      "activations/layer7_attention_weight_min": -67.16883087158203,
      "activations/layer8_attention_weight_max": 46.92308044433594,
      "activations/layer8_attention_weight_min": -48.423439025878906,
      "activations/layer9_attention_weight_max": 43.11284255981445,
      "activations/layer9_attention_weight_min": -45.64886474609375,
      "epoch": 20.42,
      "learning_rate": 1.8534469696969696e-05,
      "loss": 2.7303,
      "step": 351350
    },
    {
      "activations/layer0_attention_weight_max": 15.704144477844238,
      "activations/layer0_attention_weight_min": -13.633440971374512,
      "activations/layer10_attention_weight_max": 35.73088455200195,
      "activations/layer10_attention_weight_min": -35.11842727661133,
      "activations/layer11_attention_weight_max": 31.288549423217773,
      "activations/layer11_attention_weight_min": -33.62005615234375,
      "activations/layer12_attention_weight_max": 19.658262252807617,
      "activations/layer12_attention_weight_min": -23.845481872558594,
      "activations/layer13_attention_weight_max": 34.71810531616211,
      "activations/layer13_attention_weight_min": -28.020767211914062,
      "activations/layer14_attention_weight_max": 38.926856994628906,
      "activations/layer14_attention_weight_min": -31.456436157226562,
      "activations/layer15_attention_weight_max": 35.69036102294922,
      "activations/layer15_attention_weight_min": -31.528146743774414,
      "activations/layer16_attention_weight_max": 36.68090057373047,
      "activations/layer16_attention_weight_min": -34.105804443359375,
      "activations/layer17_attention_weight_max": 53.53560256958008,
      "activations/layer17_attention_weight_min": -46.17034912109375,
      "activations/layer18_attention_weight_max": 51.87431335449219,
      "activations/layer18_attention_weight_min": -41.57011413574219,
      "activations/layer19_attention_weight_max": 24.653676986694336,
      "activations/layer19_attention_weight_min": -22.111562728881836,
      "activations/layer1_attention_weight_max": 15.229720115661621,
      "activations/layer1_attention_weight_min": -13.24850845336914,
      "activations/layer20_attention_weight_max": 22.086416244506836,
      "activations/layer20_attention_weight_min": -20.9321346282959,
      "activations/layer21_attention_weight_max": 35.43373489379883,
      "activations/layer21_attention_weight_min": -22.391183853149414,
      "activations/layer22_attention_weight_max": 29.76713752746582,
      "activations/layer22_attention_weight_min": -26.516572952270508,
      "activations/layer23_attention_weight_max": 34.663883209228516,
      "activations/layer23_attention_weight_min": -27.17413902282715,
      "activations/layer2_attention_weight_max": 29.777286529541016,
      "activations/layer2_attention_weight_min": -28.6075439453125,
      "activations/layer3_attention_weight_max": 85.67430114746094,
      "activations/layer3_attention_weight_min": -85.46061706542969,
      "activations/layer4_attention_weight_max": 81.27278900146484,
      "activations/layer4_attention_weight_min": -86.40013885498047,
      "activations/layer5_attention_weight_max": 62.993080139160156,
      "activations/layer5_attention_weight_min": -71.33503723144531,
      "activations/layer6_attention_weight_max": 48.58835983276367,
      "activations/layer6_attention_weight_min": -51.080257415771484,
      "activations/layer7_attention_weight_max": 67.57936096191406,
      "activations/layer7_attention_weight_min": -62.80580520629883,
      "activations/layer8_attention_weight_max": 47.53056716918945,
      "activations/layer8_attention_weight_min": -53.0735969543457,
      "activations/layer9_attention_weight_max": 42.95097351074219,
      "activations/layer9_attention_weight_min": -45.3948974609375,
      "epoch": 20.42,
      "learning_rate": 1.8515530303030304e-05,
      "loss": 2.721,
      "step": 351400
    },
    {
      "activations/layer0_attention_weight_max": 16.100488662719727,
      "activations/layer0_attention_weight_min": -13.789921760559082,
      "activations/layer10_attention_weight_max": 34.376441955566406,
      "activations/layer10_attention_weight_min": -35.04861831665039,
      "activations/layer11_attention_weight_max": 32.824832916259766,
      "activations/layer11_attention_weight_min": -34.965187072753906,
      "activations/layer12_attention_weight_max": 20.82172393798828,
      "activations/layer12_attention_weight_min": -23.752893447875977,
      "activations/layer13_attention_weight_max": 32.289085388183594,
      "activations/layer13_attention_weight_min": -28.851131439208984,
      "activations/layer14_attention_weight_max": 36.28252029418945,
      "activations/layer14_attention_weight_min": -32.745445251464844,
      "activations/layer15_attention_weight_max": 34.193206787109375,
      "activations/layer15_attention_weight_min": -33.13272476196289,
      "activations/layer16_attention_weight_max": 34.89652633666992,
      "activations/layer16_attention_weight_min": -34.09806442260742,
      "activations/layer17_attention_weight_max": 51.36519241333008,
      "activations/layer17_attention_weight_min": -44.712825775146484,
      "activations/layer18_attention_weight_max": 47.000030517578125,
      "activations/layer18_attention_weight_min": -40.83930969238281,
      "activations/layer19_attention_weight_max": 24.45924186706543,
      "activations/layer19_attention_weight_min": -26.964553833007812,
      "activations/layer1_attention_weight_max": 15.544520378112793,
      "activations/layer1_attention_weight_min": -13.972538948059082,
      "activations/layer20_attention_weight_max": 22.92808723449707,
      "activations/layer20_attention_weight_min": -25.3070011138916,
      "activations/layer21_attention_weight_max": 35.08116149902344,
      "activations/layer21_attention_weight_min": -22.156375885009766,
      "activations/layer22_attention_weight_max": 29.132532119750977,
      "activations/layer22_attention_weight_min": -25.882131576538086,
      "activations/layer23_attention_weight_max": 38.29667663574219,
      "activations/layer23_attention_weight_min": -24.175395965576172,
      "activations/layer2_attention_weight_max": 29.522520065307617,
      "activations/layer2_attention_weight_min": -29.228702545166016,
      "activations/layer3_attention_weight_max": 81.99414825439453,
      "activations/layer3_attention_weight_min": -83.51849365234375,
      "activations/layer4_attention_weight_max": 83.85102081298828,
      "activations/layer4_attention_weight_min": -83.4809799194336,
      "activations/layer5_attention_weight_max": 64.104736328125,
      "activations/layer5_attention_weight_min": -70.78341674804688,
      "activations/layer6_attention_weight_max": 48.946895599365234,
      "activations/layer6_attention_weight_min": -51.52372741699219,
      "activations/layer7_attention_weight_max": 63.89839172363281,
      "activations/layer7_attention_weight_min": -64.39976501464844,
      "activations/layer8_attention_weight_max": 45.678436279296875,
      "activations/layer8_attention_weight_min": -48.500823974609375,
      "activations/layer9_attention_weight_max": 43.50899887084961,
      "activations/layer9_attention_weight_min": -46.754451751708984,
      "epoch": 20.42,
      "learning_rate": 1.849659090909091e-05,
      "loss": 2.7239,
      "step": 351450
    },
    {
      "activations/layer0_attention_weight_max": 15.62574291229248,
      "activations/layer0_attention_weight_min": -13.50761604309082,
      "activations/layer10_attention_weight_max": 34.957969665527344,
      "activations/layer10_attention_weight_min": -36.49453353881836,
      "activations/layer11_attention_weight_max": 30.91864585876465,
      "activations/layer11_attention_weight_min": -34.594486236572266,
      "activations/layer12_attention_weight_max": 19.262128829956055,
      "activations/layer12_attention_weight_min": -24.04355239868164,
      "activations/layer13_attention_weight_max": 42.61088562011719,
      "activations/layer13_attention_weight_min": -29.540973663330078,
      "activations/layer14_attention_weight_max": 38.810707092285156,
      "activations/layer14_attention_weight_min": -31.61161994934082,
      "activations/layer15_attention_weight_max": 40.285072326660156,
      "activations/layer15_attention_weight_min": -33.25213623046875,
      "activations/layer16_attention_weight_max": 34.98243713378906,
      "activations/layer16_attention_weight_min": -32.78032684326172,
      "activations/layer17_attention_weight_max": 52.35170364379883,
      "activations/layer17_attention_weight_min": -45.77156448364258,
      "activations/layer18_attention_weight_max": 50.70450210571289,
      "activations/layer18_attention_weight_min": -43.95651626586914,
      "activations/layer19_attention_weight_max": 22.83698844909668,
      "activations/layer19_attention_weight_min": -24.073137283325195,
      "activations/layer1_attention_weight_max": 15.713761329650879,
      "activations/layer1_attention_weight_min": -14.003661155700684,
      "activations/layer20_attention_weight_max": 23.96929168701172,
      "activations/layer20_attention_weight_min": -23.577098846435547,
      "activations/layer21_attention_weight_max": 43.92906188964844,
      "activations/layer21_attention_weight_min": -22.00310516357422,
      "activations/layer22_attention_weight_max": 30.53580093383789,
      "activations/layer22_attention_weight_min": -26.449861526489258,
      "activations/layer23_attention_weight_max": 32.39665985107422,
      "activations/layer23_attention_weight_min": -26.308347702026367,
      "activations/layer2_attention_weight_max": 30.045059204101562,
      "activations/layer2_attention_weight_min": -28.076122283935547,
      "activations/layer3_attention_weight_max": 83.68201446533203,
      "activations/layer3_attention_weight_min": -81.14350128173828,
      "activations/layer4_attention_weight_max": 80.66289520263672,
      "activations/layer4_attention_weight_min": -79.9454574584961,
      "activations/layer5_attention_weight_max": 58.824241638183594,
      "activations/layer5_attention_weight_min": -73.90042114257812,
      "activations/layer6_attention_weight_max": 45.26033401489258,
      "activations/layer6_attention_weight_min": -47.6819953918457,
      "activations/layer7_attention_weight_max": 63.551143646240234,
      "activations/layer7_attention_weight_min": -60.94990921020508,
      "activations/layer8_attention_weight_max": 44.307254791259766,
      "activations/layer8_attention_weight_min": -48.36006546020508,
      "activations/layer9_attention_weight_max": 43.602386474609375,
      "activations/layer9_attention_weight_min": -43.592620849609375,
      "epoch": 20.42,
      "learning_rate": 1.8477651515151514e-05,
      "loss": 2.7368,
      "step": 351500
    },
    {
      "activations/layer0_attention_weight_max": 16.234338760375977,
      "activations/layer0_attention_weight_min": -13.6444730758667,
      "activations/layer10_attention_weight_max": 37.687530517578125,
      "activations/layer10_attention_weight_min": -36.682891845703125,
      "activations/layer11_attention_weight_max": 35.06361770629883,
      "activations/layer11_attention_weight_min": -34.983909606933594,
      "activations/layer12_attention_weight_max": 19.527158737182617,
      "activations/layer12_attention_weight_min": -26.049903869628906,
      "activations/layer13_attention_weight_max": 32.644317626953125,
      "activations/layer13_attention_weight_min": -26.27655601501465,
      "activations/layer14_attention_weight_max": 33.52547073364258,
      "activations/layer14_attention_weight_min": -29.534513473510742,
      "activations/layer15_attention_weight_max": 32.17936706542969,
      "activations/layer15_attention_weight_min": -29.5799617767334,
      "activations/layer16_attention_weight_max": 33.855995178222656,
      "activations/layer16_attention_weight_min": -32.74466323852539,
      "activations/layer17_attention_weight_max": 50.166534423828125,
      "activations/layer17_attention_weight_min": -43.597373962402344,
      "activations/layer18_attention_weight_max": 46.53865432739258,
      "activations/layer18_attention_weight_min": -39.987178802490234,
      "activations/layer19_attention_weight_max": 23.43663787841797,
      "activations/layer19_attention_weight_min": -20.779294967651367,
      "activations/layer1_attention_weight_max": 15.559412956237793,
      "activations/layer1_attention_weight_min": -13.358830451965332,
      "activations/layer20_attention_weight_max": 23.03385353088379,
      "activations/layer20_attention_weight_min": -19.04501724243164,
      "activations/layer21_attention_weight_max": 32.9405403137207,
      "activations/layer21_attention_weight_min": -20.87002182006836,
      "activations/layer22_attention_weight_max": 29.449222564697266,
      "activations/layer22_attention_weight_min": -24.779863357543945,
      "activations/layer23_attention_weight_max": 39.27996826171875,
      "activations/layer23_attention_weight_min": -25.53311538696289,
      "activations/layer2_attention_weight_max": 30.88995361328125,
      "activations/layer2_attention_weight_min": -29.49921417236328,
      "activations/layer3_attention_weight_max": 86.10527038574219,
      "activations/layer3_attention_weight_min": -91.25308227539062,
      "activations/layer4_attention_weight_max": 86.10152435302734,
      "activations/layer4_attention_weight_min": -86.38346862792969,
      "activations/layer5_attention_weight_max": 62.82476806640625,
      "activations/layer5_attention_weight_min": -68.15676879882812,
      "activations/layer6_attention_weight_max": 51.44193649291992,
      "activations/layer6_attention_weight_min": -51.3922004699707,
      "activations/layer7_attention_weight_max": 68.4646224975586,
      "activations/layer7_attention_weight_min": -67.98542022705078,
      "activations/layer8_attention_weight_max": 50.47804260253906,
      "activations/layer8_attention_weight_min": -52.74745559692383,
      "activations/layer9_attention_weight_max": 45.39402389526367,
      "activations/layer9_attention_weight_min": -47.063751220703125,
      "epoch": 20.43,
      "learning_rate": 1.845871212121212e-05,
      "loss": 2.7261,
      "step": 351550
    },
    {
      "activations/layer0_attention_weight_max": 15.955761909484863,
      "activations/layer0_attention_weight_min": -13.624610900878906,
      "activations/layer10_attention_weight_max": 34.72147750854492,
      "activations/layer10_attention_weight_min": -35.18621826171875,
      "activations/layer11_attention_weight_max": 33.302032470703125,
      "activations/layer11_attention_weight_min": -34.6374397277832,
      "activations/layer12_attention_weight_max": 19.65889549255371,
      "activations/layer12_attention_weight_min": -25.66783332824707,
      "activations/layer13_attention_weight_max": 32.66242980957031,
      "activations/layer13_attention_weight_min": -27.667882919311523,
      "activations/layer14_attention_weight_max": 34.399940490722656,
      "activations/layer14_attention_weight_min": -31.24163055419922,
      "activations/layer15_attention_weight_max": 33.613983154296875,
      "activations/layer15_attention_weight_min": -33.27132797241211,
      "activations/layer16_attention_weight_max": 39.131492614746094,
      "activations/layer16_attention_weight_min": -36.88977813720703,
      "activations/layer17_attention_weight_max": 53.151145935058594,
      "activations/layer17_attention_weight_min": -47.33143997192383,
      "activations/layer18_attention_weight_max": 50.72837829589844,
      "activations/layer18_attention_weight_min": -44.19780731201172,
      "activations/layer19_attention_weight_max": 26.6773624420166,
      "activations/layer19_attention_weight_min": -25.89126968383789,
      "activations/layer1_attention_weight_max": 16.7177677154541,
      "activations/layer1_attention_weight_min": -14.093779563903809,
      "activations/layer20_attention_weight_max": 25.2276554107666,
      "activations/layer20_attention_weight_min": -23.041423797607422,
      "activations/layer21_attention_weight_max": 37.00148010253906,
      "activations/layer21_attention_weight_min": -23.82777976989746,
      "activations/layer22_attention_weight_max": 27.41046905517578,
      "activations/layer22_attention_weight_min": -26.832077026367188,
      "activations/layer23_attention_weight_max": 36.101966857910156,
      "activations/layer23_attention_weight_min": -25.428998947143555,
      "activations/layer2_attention_weight_max": 31.097379684448242,
      "activations/layer2_attention_weight_min": -28.812185287475586,
      "activations/layer3_attention_weight_max": 85.02793884277344,
      "activations/layer3_attention_weight_min": -85.80862426757812,
      "activations/layer4_attention_weight_max": 81.97649383544922,
      "activations/layer4_attention_weight_min": -79.95335388183594,
      "activations/layer5_attention_weight_max": 59.346221923828125,
      "activations/layer5_attention_weight_min": -74.3896255493164,
      "activations/layer6_attention_weight_max": 48.6840934753418,
      "activations/layer6_attention_weight_min": -47.89862823486328,
      "activations/layer7_attention_weight_max": 65.90462493896484,
      "activations/layer7_attention_weight_min": -61.05900192260742,
      "activations/layer8_attention_weight_max": 44.44816207885742,
      "activations/layer8_attention_weight_min": -47.51686477661133,
      "activations/layer9_attention_weight_max": 48.36537551879883,
      "activations/layer9_attention_weight_min": -44.920955657958984,
      "epoch": 20.43,
      "learning_rate": 1.8440151515151513e-05,
      "loss": 2.7325,
      "step": 351600
    },
    {
      "activations/layer0_attention_weight_max": 15.34233570098877,
      "activations/layer0_attention_weight_min": -13.523421287536621,
      "activations/layer10_attention_weight_max": 38.862483978271484,
      "activations/layer10_attention_weight_min": -38.33463668823242,
      "activations/layer11_attention_weight_max": 34.98686218261719,
      "activations/layer11_attention_weight_min": -35.55858612060547,
      "activations/layer12_attention_weight_max": 20.277889251708984,
      "activations/layer12_attention_weight_min": -26.31169891357422,
      "activations/layer13_attention_weight_max": 34.80644989013672,
      "activations/layer13_attention_weight_min": -29.305994033813477,
      "activations/layer14_attention_weight_max": 36.3525276184082,
      "activations/layer14_attention_weight_min": -31.98618507385254,
      "activations/layer15_attention_weight_max": 36.664695739746094,
      "activations/layer15_attention_weight_min": -30.40096092224121,
      "activations/layer16_attention_weight_max": 36.20500946044922,
      "activations/layer16_attention_weight_min": -32.762168884277344,
      "activations/layer17_attention_weight_max": 56.18907165527344,
      "activations/layer17_attention_weight_min": -46.9858283996582,
      "activations/layer18_attention_weight_max": 49.01877212524414,
      "activations/layer18_attention_weight_min": -39.498870849609375,
      "activations/layer19_attention_weight_max": 22.89310073852539,
      "activations/layer19_attention_weight_min": -21.931081771850586,
      "activations/layer1_attention_weight_max": 16.143095016479492,
      "activations/layer1_attention_weight_min": -12.167619705200195,
      "activations/layer20_attention_weight_max": 22.31362533569336,
      "activations/layer20_attention_weight_min": -24.50490379333496,
      "activations/layer21_attention_weight_max": 37.17436218261719,
      "activations/layer21_attention_weight_min": -23.234167098999023,
      "activations/layer22_attention_weight_max": 29.859113693237305,
      "activations/layer22_attention_weight_min": -25.67245101928711,
      "activations/layer23_attention_weight_max": 36.951873779296875,
      "activations/layer23_attention_weight_min": -24.31226921081543,
      "activations/layer2_attention_weight_max": 31.289745330810547,
      "activations/layer2_attention_weight_min": -29.4930419921875,
      "activations/layer3_attention_weight_max": 86.7750015258789,
      "activations/layer3_attention_weight_min": -86.3924331665039,
      "activations/layer4_attention_weight_max": 80.9780044555664,
      "activations/layer4_attention_weight_min": -84.62989044189453,
      "activations/layer5_attention_weight_max": 58.817012786865234,
      "activations/layer5_attention_weight_min": -66.42434692382812,
      "activations/layer6_attention_weight_max": 50.31631088256836,
      "activations/layer6_attention_weight_min": -52.54701614379883,
      "activations/layer7_attention_weight_max": 66.88248443603516,
      "activations/layer7_attention_weight_min": -68.95198822021484,
      "activations/layer8_attention_weight_max": 48.48906326293945,
      "activations/layer8_attention_weight_min": -52.5047492980957,
      "activations/layer9_attention_weight_max": 46.97256851196289,
      "activations/layer9_attention_weight_min": -46.799171447753906,
      "epoch": 20.43,
      "learning_rate": 1.842121212121212e-05,
      "loss": 2.718,
      "step": 351650
    },
    {
      "activations/layer0_attention_weight_max": 15.502202987670898,
      "activations/layer0_attention_weight_min": -13.795670509338379,
      "activations/layer10_attention_weight_max": 37.61513137817383,
      "activations/layer10_attention_weight_min": -37.20681381225586,
      "activations/layer11_attention_weight_max": 35.891090393066406,
      "activations/layer11_attention_weight_min": -35.7227783203125,
      "activations/layer12_attention_weight_max": 21.106449127197266,
      "activations/layer12_attention_weight_min": -23.18102264404297,
      "activations/layer13_attention_weight_max": 32.29848861694336,
      "activations/layer13_attention_weight_min": -31.036434173583984,
      "activations/layer14_attention_weight_max": 36.244468688964844,
      "activations/layer14_attention_weight_min": -30.948394775390625,
      "activations/layer15_attention_weight_max": 34.67465591430664,
      "activations/layer15_attention_weight_min": -30.178165435791016,
      "activations/layer16_attention_weight_max": 37.00440979003906,
      "activations/layer16_attention_weight_min": -33.60087203979492,
      "activations/layer17_attention_weight_max": 55.37906265258789,
      "activations/layer17_attention_weight_min": -47.114402770996094,
      "activations/layer18_attention_weight_max": 51.38385772705078,
      "activations/layer18_attention_weight_min": -42.400299072265625,
      "activations/layer19_attention_weight_max": 25.688852310180664,
      "activations/layer19_attention_weight_min": -23.348215103149414,
      "activations/layer1_attention_weight_max": 16.251277923583984,
      "activations/layer1_attention_weight_min": -14.630423545837402,
      "activations/layer20_attention_weight_max": 25.47744369506836,
      "activations/layer20_attention_weight_min": -21.226524353027344,
      "activations/layer21_attention_weight_max": 39.674922943115234,
      "activations/layer21_attention_weight_min": -22.899145126342773,
      "activations/layer22_attention_weight_max": 29.273372650146484,
      "activations/layer22_attention_weight_min": -24.688753128051758,
      "activations/layer23_attention_weight_max": 38.08777618408203,
      "activations/layer23_attention_weight_min": -23.545940399169922,
      "activations/layer2_attention_weight_max": 32.75426483154297,
      "activations/layer2_attention_weight_min": -30.62117576599121,
      "activations/layer3_attention_weight_max": 86.99291229248047,
      "activations/layer3_attention_weight_min": -88.98225402832031,
      "activations/layer4_attention_weight_max": 84.34676361083984,
      "activations/layer4_attention_weight_min": -87.0923843383789,
      "activations/layer5_attention_weight_max": 61.908470153808594,
      "activations/layer5_attention_weight_min": -70.41452026367188,
      "activations/layer6_attention_weight_max": 51.80399703979492,
      "activations/layer6_attention_weight_min": -52.669742584228516,
      "activations/layer7_attention_weight_max": 65.77018737792969,
      "activations/layer7_attention_weight_min": -66.52693939208984,
      "activations/layer8_attention_weight_max": 48.3959846496582,
      "activations/layer8_attention_weight_min": -52.285255432128906,
      "activations/layer9_attention_weight_max": 47.04769515991211,
      "activations/layer9_attention_weight_min": -47.33273696899414,
      "epoch": 20.44,
      "learning_rate": 1.8402272727272726e-05,
      "loss": 2.7188,
      "step": 351700
    },
    {
      "activations/layer0_attention_weight_max": 14.97899341583252,
      "activations/layer0_attention_weight_min": -13.729730606079102,
      "activations/layer10_attention_weight_max": 35.06241989135742,
      "activations/layer10_attention_weight_min": -34.47633743286133,
      "activations/layer11_attention_weight_max": 31.452665328979492,
      "activations/layer11_attention_weight_min": -33.90331268310547,
      "activations/layer12_attention_weight_max": 21.081838607788086,
      "activations/layer12_attention_weight_min": -23.4686222076416,
      "activations/layer13_attention_weight_max": 35.59857177734375,
      "activations/layer13_attention_weight_min": -28.114778518676758,
      "activations/layer14_attention_weight_max": 38.690792083740234,
      "activations/layer14_attention_weight_min": -31.885486602783203,
      "activations/layer15_attention_weight_max": 37.360477447509766,
      "activations/layer15_attention_weight_min": -32.0052604675293,
      "activations/layer16_attention_weight_max": 37.578556060791016,
      "activations/layer16_attention_weight_min": -35.85464096069336,
      "activations/layer17_attention_weight_max": 56.94169235229492,
      "activations/layer17_attention_weight_min": -49.683448791503906,
      "activations/layer18_attention_weight_max": 47.053253173828125,
      "activations/layer18_attention_weight_min": -42.583213806152344,
      "activations/layer19_attention_weight_max": 24.48345184326172,
      "activations/layer19_attention_weight_min": -24.698505401611328,
      "activations/layer1_attention_weight_max": 15.479894638061523,
      "activations/layer1_attention_weight_min": -13.926146507263184,
      "activations/layer20_attention_weight_max": 24.658884048461914,
      "activations/layer20_attention_weight_min": -21.803173065185547,
      "activations/layer21_attention_weight_max": 35.95515823364258,
      "activations/layer21_attention_weight_min": -23.512962341308594,
      "activations/layer22_attention_weight_max": 26.940204620361328,
      "activations/layer22_attention_weight_min": -27.291027069091797,
      "activations/layer23_attention_weight_max": 35.963722229003906,
      "activations/layer23_attention_weight_min": -26.415042877197266,
      "activations/layer2_attention_weight_max": 31.5535945892334,
      "activations/layer2_attention_weight_min": -29.858625411987305,
      "activations/layer3_attention_weight_max": 85.47540283203125,
      "activations/layer3_attention_weight_min": -87.3236312866211,
      "activations/layer4_attention_weight_max": 83.76771545410156,
      "activations/layer4_attention_weight_min": -84.35704803466797,
      "activations/layer5_attention_weight_max": 62.8161735534668,
      "activations/layer5_attention_weight_min": -68.72509002685547,
      "activations/layer6_attention_weight_max": 47.50542068481445,
      "activations/layer6_attention_weight_min": -49.7888069152832,
      "activations/layer7_attention_weight_max": 62.80830764770508,
      "activations/layer7_attention_weight_min": -65.0397720336914,
      "activations/layer8_attention_weight_max": 44.30415725708008,
      "activations/layer8_attention_weight_min": -48.58100891113281,
      "activations/layer9_attention_weight_max": 40.94228744506836,
      "activations/layer9_attention_weight_min": -45.260398864746094,
      "epoch": 20.44,
      "learning_rate": 1.838333333333333e-05,
      "loss": 2.7245,
      "step": 351750
    },
    {
      "activations/layer0_attention_weight_max": 15.569073677062988,
      "activations/layer0_attention_weight_min": -13.850428581237793,
      "activations/layer10_attention_weight_max": 39.3055419921875,
      "activations/layer10_attention_weight_min": -37.78358459472656,
      "activations/layer11_attention_weight_max": 36.27700424194336,
      "activations/layer11_attention_weight_min": -36.549659729003906,
      "activations/layer12_attention_weight_max": 19.713598251342773,
      "activations/layer12_attention_weight_min": -33.1171760559082,
      "activations/layer13_attention_weight_max": 32.7218132019043,
      "activations/layer13_attention_weight_min": -27.618896484375,
      "activations/layer14_attention_weight_max": 37.0485725402832,
      "activations/layer14_attention_weight_min": -31.069053649902344,
      "activations/layer15_attention_weight_max": 35.00935745239258,
      "activations/layer15_attention_weight_min": -32.418941497802734,
      "activations/layer16_attention_weight_max": 37.452056884765625,
      "activations/layer16_attention_weight_min": -32.25553894042969,
      "activations/layer17_attention_weight_max": 57.5176887512207,
      "activations/layer17_attention_weight_min": -45.567138671875,
      "activations/layer18_attention_weight_max": 51.98893737792969,
      "activations/layer18_attention_weight_min": -42.81684875488281,
      "activations/layer19_attention_weight_max": 22.585783004760742,
      "activations/layer19_attention_weight_min": -23.02701187133789,
      "activations/layer1_attention_weight_max": 15.059333801269531,
      "activations/layer1_attention_weight_min": -13.867203712463379,
      "activations/layer20_attention_weight_max": 23.222307205200195,
      "activations/layer20_attention_weight_min": -23.089052200317383,
      "activations/layer21_attention_weight_max": 33.142486572265625,
      "activations/layer21_attention_weight_min": -19.85038185119629,
      "activations/layer22_attention_weight_max": 26.95913314819336,
      "activations/layer22_attention_weight_min": -24.92344093322754,
      "activations/layer23_attention_weight_max": 32.86528778076172,
      "activations/layer23_attention_weight_min": -22.85248374938965,
      "activations/layer2_attention_weight_max": 31.920398712158203,
      "activations/layer2_attention_weight_min": -30.346525192260742,
      "activations/layer3_attention_weight_max": 88.2585678100586,
      "activations/layer3_attention_weight_min": -88.46556091308594,
      "activations/layer4_attention_weight_max": 87.8768310546875,
      "activations/layer4_attention_weight_min": -88.08161163330078,
      "activations/layer5_attention_weight_max": 64.39804077148438,
      "activations/layer5_attention_weight_min": -71.8148193359375,
      "activations/layer6_attention_weight_max": 51.661460876464844,
      "activations/layer6_attention_weight_min": -52.24065017700195,
      "activations/layer7_attention_weight_max": 72.99559783935547,
      "activations/layer7_attention_weight_min": -67.67938232421875,
      "activations/layer8_attention_weight_max": 49.38078689575195,
      "activations/layer8_attention_weight_min": -52.398197174072266,
      "activations/layer9_attention_weight_max": 46.342037200927734,
      "activations/layer9_attention_weight_min": -47.20817947387695,
      "epoch": 20.44,
      "learning_rate": 1.836439393939394e-05,
      "loss": 2.715,
      "step": 351800
    },
    {
      "activations/layer0_attention_weight_max": 15.731154441833496,
      "activations/layer0_attention_weight_min": -14.019643783569336,
      "activations/layer10_attention_weight_max": 34.039093017578125,
      "activations/layer10_attention_weight_min": -36.96733474731445,
      "activations/layer11_attention_weight_max": 31.16163444519043,
      "activations/layer11_attention_weight_min": -34.64387893676758,
      "activations/layer12_attention_weight_max": 21.51606559753418,
      "activations/layer12_attention_weight_min": -25.91693687438965,
      "activations/layer13_attention_weight_max": 36.64744186401367,
      "activations/layer13_attention_weight_min": -27.010122299194336,
      "activations/layer14_attention_weight_max": 36.913822174072266,
      "activations/layer14_attention_weight_min": -29.311281204223633,
      "activations/layer15_attention_weight_max": 36.939292907714844,
      "activations/layer15_attention_weight_min": -29.786718368530273,
      "activations/layer16_attention_weight_max": 36.19544982910156,
      "activations/layer16_attention_weight_min": -31.680007934570312,
      "activations/layer17_attention_weight_max": 53.48814010620117,
      "activations/layer17_attention_weight_min": -46.66291809082031,
      "activations/layer18_attention_weight_max": 49.95297622680664,
      "activations/layer18_attention_weight_min": -41.515689849853516,
      "activations/layer19_attention_weight_max": 23.798072814941406,
      "activations/layer19_attention_weight_min": -23.462583541870117,
      "activations/layer1_attention_weight_max": 14.97209644317627,
      "activations/layer1_attention_weight_min": -13.888131141662598,
      "activations/layer20_attention_weight_max": 22.790510177612305,
      "activations/layer20_attention_weight_min": -21.306934356689453,
      "activations/layer21_attention_weight_max": 35.208412170410156,
      "activations/layer21_attention_weight_min": -22.916183471679688,
      "activations/layer22_attention_weight_max": 29.23126792907715,
      "activations/layer22_attention_weight_min": -26.1336669921875,
      "activations/layer23_attention_weight_max": 34.368560791015625,
      "activations/layer23_attention_weight_min": -24.753454208374023,
      "activations/layer2_attention_weight_max": 29.577194213867188,
      "activations/layer2_attention_weight_min": -29.37959098815918,
      "activations/layer3_attention_weight_max": 84.49980926513672,
      "activations/layer3_attention_weight_min": -82.78395080566406,
      "activations/layer4_attention_weight_max": 79.68114471435547,
      "activations/layer4_attention_weight_min": -79.98494720458984,
      "activations/layer5_attention_weight_max": 56.46417236328125,
      "activations/layer5_attention_weight_min": -66.86978149414062,
      "activations/layer6_attention_weight_max": 46.18845748901367,
      "activations/layer6_attention_weight_min": -46.83651351928711,
      "activations/layer7_attention_weight_max": 64.15763092041016,
      "activations/layer7_attention_weight_min": -61.02702331542969,
      "activations/layer8_attention_weight_max": 44.02656936645508,
      "activations/layer8_attention_weight_min": -46.89065933227539,
      "activations/layer9_attention_weight_max": 41.83694839477539,
      "activations/layer9_attention_weight_min": -44.35994338989258,
      "epoch": 20.44,
      "learning_rate": 1.8345454545454544e-05,
      "loss": 2.724,
      "step": 351850
    },
    {
      "activations/layer0_attention_weight_max": 14.894129753112793,
      "activations/layer0_attention_weight_min": -13.739546775817871,
      "activations/layer10_attention_weight_max": 36.503684997558594,
      "activations/layer10_attention_weight_min": -35.08570098876953,
      "activations/layer11_attention_weight_max": 33.43833541870117,
      "activations/layer11_attention_weight_min": -34.26301193237305,
      "activations/layer12_attention_weight_max": 18.39615821838379,
      "activations/layer12_attention_weight_min": -23.696685791015625,
      "activations/layer13_attention_weight_max": 35.90019226074219,
      "activations/layer13_attention_weight_min": -28.71407127380371,
      "activations/layer14_attention_weight_max": 35.017799377441406,
      "activations/layer14_attention_weight_min": -34.27626419067383,
      "activations/layer15_attention_weight_max": 31.156078338623047,
      "activations/layer15_attention_weight_min": -31.88628387451172,
      "activations/layer16_attention_weight_max": 34.14068603515625,
      "activations/layer16_attention_weight_min": -32.23942565917969,
      "activations/layer17_attention_weight_max": 46.82773208618164,
      "activations/layer17_attention_weight_min": -46.82400894165039,
      "activations/layer18_attention_weight_max": 44.34086990356445,
      "activations/layer18_attention_weight_min": -42.57282638549805,
      "activations/layer19_attention_weight_max": 22.70164680480957,
      "activations/layer19_attention_weight_min": -24.559532165527344,
      "activations/layer1_attention_weight_max": 16.106063842773438,
      "activations/layer1_attention_weight_min": -13.33210277557373,
      "activations/layer20_attention_weight_max": 21.649742126464844,
      "activations/layer20_attention_weight_min": -23.58070182800293,
      "activations/layer21_attention_weight_max": 36.463775634765625,
      "activations/layer21_attention_weight_min": -26.77098846435547,
      "activations/layer22_attention_weight_max": 26.79757308959961,
      "activations/layer22_attention_weight_min": -28.857954025268555,
      "activations/layer23_attention_weight_max": 40.196495056152344,
      "activations/layer23_attention_weight_min": -28.47443199157715,
      "activations/layer2_attention_weight_max": 32.009185791015625,
      "activations/layer2_attention_weight_min": -30.346763610839844,
      "activations/layer3_attention_weight_max": 86.76142120361328,
      "activations/layer3_attention_weight_min": -83.85822296142578,
      "activations/layer4_attention_weight_max": 85.60396575927734,
      "activations/layer4_attention_weight_min": -82.56210327148438,
      "activations/layer5_attention_weight_max": 60.30062484741211,
      "activations/layer5_attention_weight_min": -71.35144805908203,
      "activations/layer6_attention_weight_max": 51.40188980102539,
      "activations/layer6_attention_weight_min": -50.92002487182617,
      "activations/layer7_attention_weight_max": 66.17475128173828,
      "activations/layer7_attention_weight_min": -64.54891967773438,
      "activations/layer8_attention_weight_max": 49.15971374511719,
      "activations/layer8_attention_weight_min": -52.30949783325195,
      "activations/layer9_attention_weight_max": 44.488311767578125,
      "activations/layer9_attention_weight_min": -47.819725036621094,
      "epoch": 20.45,
      "learning_rate": 1.832651515151515e-05,
      "loss": 2.7153,
      "step": 351900
    },
    {
      "activations/layer0_attention_weight_max": 15.616689682006836,
      "activations/layer0_attention_weight_min": -13.777749061584473,
      "activations/layer10_attention_weight_max": 39.700645446777344,
      "activations/layer10_attention_weight_min": -39.8101806640625,
      "activations/layer11_attention_weight_max": 36.5823974609375,
      "activations/layer11_attention_weight_min": -36.71484375,
      "activations/layer12_attention_weight_max": 19.275896072387695,
      "activations/layer12_attention_weight_min": -23.682464599609375,
      "activations/layer13_attention_weight_max": 36.160091400146484,
      "activations/layer13_attention_weight_min": -28.993465423583984,
      "activations/layer14_attention_weight_max": 38.69572448730469,
      "activations/layer14_attention_weight_min": -31.77716636657715,
      "activations/layer15_attention_weight_max": 33.671138763427734,
      "activations/layer15_attention_weight_min": -32.18388366699219,
      "activations/layer16_attention_weight_max": 35.15113067626953,
      "activations/layer16_attention_weight_min": -31.17325782775879,
      "activations/layer17_attention_weight_max": 52.921390533447266,
      "activations/layer17_attention_weight_min": -45.38749313354492,
      "activations/layer18_attention_weight_max": 46.47626495361328,
      "activations/layer18_attention_weight_min": -41.50084686279297,
      "activations/layer19_attention_weight_max": 25.760425567626953,
      "activations/layer19_attention_weight_min": -25.509296417236328,
      "activations/layer1_attention_weight_max": 15.081121444702148,
      "activations/layer1_attention_weight_min": -13.5155611038208,
      "activations/layer20_attention_weight_max": 22.778261184692383,
      "activations/layer20_attention_weight_min": -23.439868927001953,
      "activations/layer21_attention_weight_max": 36.4360237121582,
      "activations/layer21_attention_weight_min": -23.54630470275879,
      "activations/layer22_attention_weight_max": 29.697093963623047,
      "activations/layer22_attention_weight_min": -27.10629653930664,
      "activations/layer23_attention_weight_max": 39.05835723876953,
      "activations/layer23_attention_weight_min": -25.624736785888672,
      "activations/layer2_attention_weight_max": 31.755084991455078,
      "activations/layer2_attention_weight_min": -31.476301193237305,
      "activations/layer3_attention_weight_max": 89.89936828613281,
      "activations/layer3_attention_weight_min": -88.51302337646484,
      "activations/layer4_attention_weight_max": 89.40111541748047,
      "activations/layer4_attention_weight_min": -89.81714630126953,
      "activations/layer5_attention_weight_max": 65.30349731445312,
      "activations/layer5_attention_weight_min": -70.93238830566406,
      "activations/layer6_attention_weight_max": 51.54987335205078,
      "activations/layer6_attention_weight_min": -54.58866882324219,
      "activations/layer7_attention_weight_max": 68.09691619873047,
      "activations/layer7_attention_weight_min": -66.58091735839844,
      "activations/layer8_attention_weight_max": 52.497249603271484,
      "activations/layer8_attention_weight_min": -60.37261962890625,
      "activations/layer9_attention_weight_max": 48.19809341430664,
      "activations/layer9_attention_weight_min": -50.060909271240234,
      "epoch": 20.45,
      "learning_rate": 1.8307575757575757e-05,
      "loss": 2.7019,
      "step": 351950
    },
    {
      "activations/layer0_attention_weight_max": 16.596097946166992,
      "activations/layer0_attention_weight_min": -14.088329315185547,
      "activations/layer10_attention_weight_max": 39.36717224121094,
      "activations/layer10_attention_weight_min": -38.86133575439453,
      "activations/layer11_attention_weight_max": 35.92640686035156,
      "activations/layer11_attention_weight_min": -37.2071647644043,
      "activations/layer12_attention_weight_max": 18.06402587890625,
      "activations/layer12_attention_weight_min": -25.163469314575195,
      "activations/layer13_attention_weight_max": 32.17715072631836,
      "activations/layer13_attention_weight_min": -29.48503303527832,
      "activations/layer14_attention_weight_max": 37.090171813964844,
      "activations/layer14_attention_weight_min": -33.2049560546875,
      "activations/layer15_attention_weight_max": 35.62895965576172,
      "activations/layer15_attention_weight_min": -32.44423294067383,
      "activations/layer16_attention_weight_max": 34.59524917602539,
      "activations/layer16_attention_weight_min": -34.05812454223633,
      "activations/layer17_attention_weight_max": 53.8245735168457,
      "activations/layer17_attention_weight_min": -50.61331558227539,
      "activations/layer18_attention_weight_max": 51.232818603515625,
      "activations/layer18_attention_weight_min": -46.231990814208984,
      "activations/layer19_attention_weight_max": 25.41983985900879,
      "activations/layer19_attention_weight_min": -23.343820571899414,
      "activations/layer1_attention_weight_max": 15.547868728637695,
      "activations/layer1_attention_weight_min": -12.700339317321777,
      "activations/layer20_attention_weight_max": 24.387727737426758,
      "activations/layer20_attention_weight_min": -24.169803619384766,
      "activations/layer21_attention_weight_max": 39.498600006103516,
      "activations/layer21_attention_weight_min": -21.65248680114746,
      "activations/layer22_attention_weight_max": 28.5064754486084,
      "activations/layer22_attention_weight_min": -24.583114624023438,
      "activations/layer23_attention_weight_max": 39.586669921875,
      "activations/layer23_attention_weight_min": -22.935585021972656,
      "activations/layer2_attention_weight_max": 31.41053009033203,
      "activations/layer2_attention_weight_min": -30.783267974853516,
      "activations/layer3_attention_weight_max": 82.74392700195312,
      "activations/layer3_attention_weight_min": -86.20903778076172,
      "activations/layer4_attention_weight_max": 84.03318786621094,
      "activations/layer4_attention_weight_min": -81.90406799316406,
      "activations/layer5_attention_weight_max": 61.87374496459961,
      "activations/layer5_attention_weight_min": -74.64778900146484,
      "activations/layer6_attention_weight_max": 49.880592346191406,
      "activations/layer6_attention_weight_min": -50.294029235839844,
      "activations/layer7_attention_weight_max": 75.04832458496094,
      "activations/layer7_attention_weight_min": -68.40194702148438,
      "activations/layer8_attention_weight_max": 49.33440399169922,
      "activations/layer8_attention_weight_min": -52.31306838989258,
      "activations/layer9_attention_weight_max": 52.1278076171875,
      "activations/layer9_attention_weight_min": -47.22542953491211,
      "epoch": 20.45,
      "learning_rate": 1.8288636363636362e-05,
      "loss": 2.723,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.5394,
      "eval_samples_per_second": 502.846,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.5394,
      "eval_openwebtext_samples_per_second": 502.846,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 2.0276,
      "eval_wikitext_samples_per_second": 224.9,
      "step": 352000
    },
    {
      "epoch": 20.45,
      "eval_lambada_loss": 2.40234375,
      "eval_lambada_ppl": 11.049042250008942,
      "eval_lambada_runtime": 9.6026,
      "eval_lambada_samples_per_second": 507.05,
      "step": 352000
    },
    {
      "activations/layer0_attention_weight_max": 14.883515357971191,
      "activations/layer0_attention_weight_min": -13.521425247192383,
      "activations/layer10_attention_weight_max": 37.56841278076172,
      "activations/layer10_attention_weight_min": -36.46205520629883,
      "activations/layer11_attention_weight_max": 35.9025993347168,
      "activations/layer11_attention_weight_min": -37.087181091308594,
      "activations/layer12_attention_weight_max": 18.49643325805664,
      "activations/layer12_attention_weight_min": -24.878711700439453,
      "activations/layer13_attention_weight_max": 40.50215148925781,
      "activations/layer13_attention_weight_min": -29.81856918334961,
      "activations/layer14_attention_weight_max": 38.178863525390625,
      "activations/layer14_attention_weight_min": -32.73427200317383,
      "activations/layer15_attention_weight_max": 35.36601257324219,
      "activations/layer15_attention_weight_min": -30.711408615112305,
      "activations/layer16_attention_weight_max": 38.08170700073242,
      "activations/layer16_attention_weight_min": -33.7700309753418,
      "activations/layer17_attention_weight_max": 56.4012336730957,
      "activations/layer17_attention_weight_min": -45.584564208984375,
      "activations/layer18_attention_weight_max": 57.29158020019531,
      "activations/layer18_attention_weight_min": -42.34807586669922,
      "activations/layer19_attention_weight_max": 28.20271873474121,
      "activations/layer19_attention_weight_min": -23.347394943237305,
      "activations/layer1_attention_weight_max": 14.767034530639648,
      "activations/layer1_attention_weight_min": -13.450284004211426,
      "activations/layer20_attention_weight_max": 27.863624572753906,
      "activations/layer20_attention_weight_min": -22.67411231994629,
      "activations/layer21_attention_weight_max": 42.85247802734375,
      "activations/layer21_attention_weight_min": -27.69223976135254,
      "activations/layer22_attention_weight_max": 35.97724914550781,
      "activations/layer22_attention_weight_min": -25.561067581176758,
      "activations/layer23_attention_weight_max": 42.312217712402344,
      "activations/layer23_attention_weight_min": -24.649436950683594,
      "activations/layer2_attention_weight_max": 32.671966552734375,
      "activations/layer2_attention_weight_min": -31.342819213867188,
      "activations/layer3_attention_weight_max": 88.49481201171875,
      "activations/layer3_attention_weight_min": -92.58415985107422,
      "activations/layer4_attention_weight_max": 86.21266174316406,
      "activations/layer4_attention_weight_min": -90.4487533569336,
      "activations/layer5_attention_weight_max": 65.28700256347656,
      "activations/layer5_attention_weight_min": -73.98611450195312,
      "activations/layer6_attention_weight_max": 52.84325408935547,
      "activations/layer6_attention_weight_min": -53.80574417114258,
      "activations/layer7_attention_weight_max": 66.68931579589844,
      "activations/layer7_attention_weight_min": -67.63735961914062,
      "activations/layer8_attention_weight_max": 48.59977340698242,
      "activations/layer8_attention_weight_min": -52.92251968383789,
      "activations/layer9_attention_weight_max": 44.47590255737305,
      "activations/layer9_attention_weight_min": -49.35030746459961,
      "epoch": 20.46,
      "learning_rate": 1.8269696969696967e-05,
      "loss": 2.7264,
      "step": 352050
    },
    {
      "activations/layer0_attention_weight_max": 16.15584373474121,
      "activations/layer0_attention_weight_min": -13.55398178100586,
      "activations/layer10_attention_weight_max": 34.152191162109375,
      "activations/layer10_attention_weight_min": -37.3724365234375,
      "activations/layer11_attention_weight_max": 32.34236526489258,
      "activations/layer11_attention_weight_min": -35.109134674072266,
      "activations/layer12_attention_weight_max": 18.53865623474121,
      "activations/layer12_attention_weight_min": -25.779788970947266,
      "activations/layer13_attention_weight_max": 34.12152099609375,
      "activations/layer13_attention_weight_min": -26.678808212280273,
      "activations/layer14_attention_weight_max": 35.311668395996094,
      "activations/layer14_attention_weight_min": -30.99737548828125,
      "activations/layer15_attention_weight_max": 32.9376335144043,
      "activations/layer15_attention_weight_min": -31.51015281677246,
      "activations/layer16_attention_weight_max": 35.036617279052734,
      "activations/layer16_attention_weight_min": -32.385101318359375,
      "activations/layer17_attention_weight_max": 52.93962097167969,
      "activations/layer17_attention_weight_min": -45.43058776855469,
      "activations/layer18_attention_weight_max": 52.66653823852539,
      "activations/layer18_attention_weight_min": -41.83597946166992,
      "activations/layer19_attention_weight_max": 26.730398178100586,
      "activations/layer19_attention_weight_min": -23.804807662963867,
      "activations/layer1_attention_weight_max": 16.594396591186523,
      "activations/layer1_attention_weight_min": -12.887612342834473,
      "activations/layer20_attention_weight_max": 28.03105354309082,
      "activations/layer20_attention_weight_min": -21.918498992919922,
      "activations/layer21_attention_weight_max": 36.394187927246094,
      "activations/layer21_attention_weight_min": -23.3906192779541,
      "activations/layer22_attention_weight_max": 31.611162185668945,
      "activations/layer22_attention_weight_min": -26.27313232421875,
      "activations/layer23_attention_weight_max": 38.8045654296875,
      "activations/layer23_attention_weight_min": -26.34137535095215,
      "activations/layer2_attention_weight_max": 30.59625244140625,
      "activations/layer2_attention_weight_min": -28.81525230407715,
      "activations/layer3_attention_weight_max": 83.92395782470703,
      "activations/layer3_attention_weight_min": -82.56130981445312,
      "activations/layer4_attention_weight_max": 78.84449005126953,
      "activations/layer4_attention_weight_min": -81.19332122802734,
      "activations/layer5_attention_weight_max": 58.54977035522461,
      "activations/layer5_attention_weight_min": -69.724853515625,
      "activations/layer6_attention_weight_max": 46.33964538574219,
      "activations/layer6_attention_weight_min": -48.458797454833984,
      "activations/layer7_attention_weight_max": 63.98639678955078,
      "activations/layer7_attention_weight_min": -61.61112976074219,
      "activations/layer8_attention_weight_max": 45.198631286621094,
      "activations/layer8_attention_weight_min": -48.87163543701172,
      "activations/layer9_attention_weight_max": 41.029808044433594,
      "activations/layer9_attention_weight_min": -44.51791763305664,
      "epoch": 20.46,
      "learning_rate": 1.8250757575757575e-05,
      "loss": 2.72,
      "step": 352100
    },
    {
      "activations/layer0_attention_weight_max": 16.39778709411621,
      "activations/layer0_attention_weight_min": -13.663928985595703,
      "activations/layer10_attention_weight_max": 35.72615432739258,
      "activations/layer10_attention_weight_min": -36.65908432006836,
      "activations/layer11_attention_weight_max": 35.399444580078125,
      "activations/layer11_attention_weight_min": -37.140113830566406,
      "activations/layer12_attention_weight_max": 18.152273178100586,
      "activations/layer12_attention_weight_min": -27.471708297729492,
      "activations/layer13_attention_weight_max": 32.33899688720703,
      "activations/layer13_attention_weight_min": -27.616172790527344,
      "activations/layer14_attention_weight_max": 34.066158294677734,
      "activations/layer14_attention_weight_min": -30.561538696289062,
      "activations/layer15_attention_weight_max": 33.9911994934082,
      "activations/layer15_attention_weight_min": -31.570865631103516,
      "activations/layer16_attention_weight_max": 33.69976806640625,
      "activations/layer16_attention_weight_min": -31.650793075561523,
      "activations/layer17_attention_weight_max": 49.94301223754883,
      "activations/layer17_attention_weight_min": -44.428123474121094,
      "activations/layer18_attention_weight_max": 46.67214584350586,
      "activations/layer18_attention_weight_min": -40.04374313354492,
      "activations/layer19_attention_weight_max": 26.05289077758789,
      "activations/layer19_attention_weight_min": -22.761816024780273,
      "activations/layer1_attention_weight_max": 15.125980377197266,
      "activations/layer1_attention_weight_min": -13.682751655578613,
      "activations/layer20_attention_weight_max": 22.724468231201172,
      "activations/layer20_attention_weight_min": -21.269145965576172,
      "activations/layer21_attention_weight_max": 35.721214294433594,
      "activations/layer21_attention_weight_min": -21.303203582763672,
      "activations/layer22_attention_weight_max": 29.148557662963867,
      "activations/layer22_attention_weight_min": -24.967681884765625,
      "activations/layer23_attention_weight_max": 35.88254928588867,
      "activations/layer23_attention_weight_min": -25.392845153808594,
      "activations/layer2_attention_weight_max": 30.839155197143555,
      "activations/layer2_attention_weight_min": -29.590126037597656,
      "activations/layer3_attention_weight_max": 87.94670867919922,
      "activations/layer3_attention_weight_min": -85.4795913696289,
      "activations/layer4_attention_weight_max": 82.47040557861328,
      "activations/layer4_attention_weight_min": -81.34639739990234,
      "activations/layer5_attention_weight_max": 61.945030212402344,
      "activations/layer5_attention_weight_min": -71.98941040039062,
      "activations/layer6_attention_weight_max": 49.610435485839844,
      "activations/layer6_attention_weight_min": -48.83422088623047,
      "activations/layer7_attention_weight_max": 67.97637939453125,
      "activations/layer7_attention_weight_min": -62.96090316772461,
      "activations/layer8_attention_weight_max": 47.959224700927734,
      "activations/layer8_attention_weight_min": -52.51034164428711,
      "activations/layer9_attention_weight_max": 45.39719772338867,
      "activations/layer9_attention_weight_min": -44.81747055053711,
      "epoch": 20.46,
      "learning_rate": 1.823181818181818e-05,
      "loss": 2.7072,
      "step": 352150
    },
    {
      "activations/layer0_attention_weight_max": 16.589921951293945,
      "activations/layer0_attention_weight_min": -13.813231468200684,
      "activations/layer10_attention_weight_max": 36.215267181396484,
      "activations/layer10_attention_weight_min": -36.620582580566406,
      "activations/layer11_attention_weight_max": 34.84800720214844,
      "activations/layer11_attention_weight_min": -34.40240478515625,
      "activations/layer12_attention_weight_max": 19.044586181640625,
      "activations/layer12_attention_weight_min": -34.67733383178711,
      "activations/layer13_attention_weight_max": 33.36537170410156,
      "activations/layer13_attention_weight_min": -25.76022720336914,
      "activations/layer14_attention_weight_max": 36.14628982543945,
      "activations/layer14_attention_weight_min": -29.01594352722168,
      "activations/layer15_attention_weight_max": 34.15976333618164,
      "activations/layer15_attention_weight_min": -29.655933380126953,
      "activations/layer16_attention_weight_max": 35.70738220214844,
      "activations/layer16_attention_weight_min": -33.894012451171875,
      "activations/layer17_attention_weight_max": 51.28609848022461,
      "activations/layer17_attention_weight_min": -47.4672966003418,
      "activations/layer18_attention_weight_max": 46.18946838378906,
      "activations/layer18_attention_weight_min": -42.16017532348633,
      "activations/layer19_attention_weight_max": 25.44477081298828,
      "activations/layer19_attention_weight_min": -23.603017807006836,
      "activations/layer1_attention_weight_max": 15.694768905639648,
      "activations/layer1_attention_weight_min": -14.382594108581543,
      "activations/layer20_attention_weight_max": 21.567365646362305,
      "activations/layer20_attention_weight_min": -23.92131233215332,
      "activations/layer21_attention_weight_max": 33.55665969848633,
      "activations/layer21_attention_weight_min": -24.79716682434082,
      "activations/layer22_attention_weight_max": 28.417804718017578,
      "activations/layer22_attention_weight_min": -26.348997116088867,
      "activations/layer23_attention_weight_max": 37.51178741455078,
      "activations/layer23_attention_weight_min": -25.88516616821289,
      "activations/layer2_attention_weight_max": 33.37384033203125,
      "activations/layer2_attention_weight_min": -31.100139617919922,
      "activations/layer3_attention_weight_max": 89.0959701538086,
      "activations/layer3_attention_weight_min": -88.36695861816406,
      "activations/layer4_attention_weight_max": 88.53857421875,
      "activations/layer4_attention_weight_min": -85.12615203857422,
      "activations/layer5_attention_weight_max": 61.965755462646484,
      "activations/layer5_attention_weight_min": -71.87158966064453,
      "activations/layer6_attention_weight_max": 52.04273223876953,
      "activations/layer6_attention_weight_min": -51.22328186035156,
      "activations/layer7_attention_weight_max": 70.2533187866211,
      "activations/layer7_attention_weight_min": -67.17684936523438,
      "activations/layer8_attention_weight_max": 48.11372756958008,
      "activations/layer8_attention_weight_min": -52.5032844543457,
      "activations/layer9_attention_weight_max": 48.80294418334961,
      "activations/layer9_attention_weight_min": -47.62400436401367,
      "epoch": 20.46,
      "learning_rate": 1.8212878787878785e-05,
      "loss": 2.7192,
      "step": 352200
    },
    {
      "activations/layer0_attention_weight_max": 15.172184944152832,
      "activations/layer0_attention_weight_min": -13.600442886352539,
      "activations/layer10_attention_weight_max": 37.30437469482422,
      "activations/layer10_attention_weight_min": -35.03070068359375,
      "activations/layer11_attention_weight_max": 35.18860626220703,
      "activations/layer11_attention_weight_min": -34.36758804321289,
      "activations/layer12_attention_weight_max": 20.715734481811523,
      "activations/layer12_attention_weight_min": -25.43052101135254,
      "activations/layer13_attention_weight_max": 33.350067138671875,
      "activations/layer13_attention_weight_min": -27.988759994506836,
      "activations/layer14_attention_weight_max": 34.052547454833984,
      "activations/layer14_attention_weight_min": -30.977874755859375,
      "activations/layer15_attention_weight_max": 32.60044860839844,
      "activations/layer15_attention_weight_min": -29.941970825195312,
      "activations/layer16_attention_weight_max": 34.100196838378906,
      "activations/layer16_attention_weight_min": -32.986907958984375,
      "activations/layer17_attention_weight_max": 51.1955451965332,
      "activations/layer17_attention_weight_min": -45.26497268676758,
      "activations/layer18_attention_weight_max": 45.89244842529297,
      "activations/layer18_attention_weight_min": -40.32019805908203,
      "activations/layer19_attention_weight_max": 23.2409725189209,
      "activations/layer19_attention_weight_min": -23.139995574951172,
      "activations/layer1_attention_weight_max": 15.045727729797363,
      "activations/layer1_attention_weight_min": -14.7034912109375,
      "activations/layer20_attention_weight_max": 21.955846786499023,
      "activations/layer20_attention_weight_min": -24.01569938659668,
      "activations/layer21_attention_weight_max": 38.91321563720703,
      "activations/layer21_attention_weight_min": -24.7995548248291,
      "activations/layer22_attention_weight_max": 27.50257110595703,
      "activations/layer22_attention_weight_min": -24.87472152709961,
      "activations/layer23_attention_weight_max": 35.64852523803711,
      "activations/layer23_attention_weight_min": -26.054344177246094,
      "activations/layer2_attention_weight_max": 31.573204040527344,
      "activations/layer2_attention_weight_min": -30.472251892089844,
      "activations/layer3_attention_weight_max": 86.423828125,
      "activations/layer3_attention_weight_min": -85.5906753540039,
      "activations/layer4_attention_weight_max": 82.93669128417969,
      "activations/layer4_attention_weight_min": -79.75668334960938,
      "activations/layer5_attention_weight_max": 60.964759826660156,
      "activations/layer5_attention_weight_min": -66.63514709472656,
      "activations/layer6_attention_weight_max": 48.59746551513672,
      "activations/layer6_attention_weight_min": -49.32628631591797,
      "activations/layer7_attention_weight_max": 69.2128677368164,
      "activations/layer7_attention_weight_min": -64.2300033569336,
      "activations/layer8_attention_weight_max": 48.13374710083008,
      "activations/layer8_attention_weight_min": -48.42705154418945,
      "activations/layer9_attention_weight_max": 43.86408233642578,
      "activations/layer9_attention_weight_min": -44.9028434753418,
      "epoch": 20.47,
      "learning_rate": 1.8193939393939393e-05,
      "loss": 2.7123,
      "step": 352250
    },
    {
      "activations/layer0_attention_weight_max": 15.227433204650879,
      "activations/layer0_attention_weight_min": -13.599095344543457,
      "activations/layer10_attention_weight_max": 37.634124755859375,
      "activations/layer10_attention_weight_min": -36.900299072265625,
      "activations/layer11_attention_weight_max": 33.904205322265625,
      "activations/layer11_attention_weight_min": -35.62013244628906,
      "activations/layer12_attention_weight_max": 18.457571029663086,
      "activations/layer12_attention_weight_min": -24.965585708618164,
      "activations/layer13_attention_weight_max": 35.0865364074707,
      "activations/layer13_attention_weight_min": -30.19139862060547,
      "activations/layer14_attention_weight_max": 35.3963623046875,
      "activations/layer14_attention_weight_min": -28.679166793823242,
      "activations/layer15_attention_weight_max": 34.677513122558594,
      "activations/layer15_attention_weight_min": -29.30447769165039,
      "activations/layer16_attention_weight_max": 34.91228485107422,
      "activations/layer16_attention_weight_min": -32.520057678222656,
      "activations/layer17_attention_weight_max": 51.34364700317383,
      "activations/layer17_attention_weight_min": -45.29878234863281,
      "activations/layer18_attention_weight_max": 45.9653205871582,
      "activations/layer18_attention_weight_min": -45.73379898071289,
      "activations/layer19_attention_weight_max": 25.483028411865234,
      "activations/layer19_attention_weight_min": -26.556303024291992,
      "activations/layer1_attention_weight_max": 15.649685859680176,
      "activations/layer1_attention_weight_min": -13.876720428466797,
      "activations/layer20_attention_weight_max": 24.223508834838867,
      "activations/layer20_attention_weight_min": -23.16233253479004,
      "activations/layer21_attention_weight_max": 33.20914077758789,
      "activations/layer21_attention_weight_min": -29.064739227294922,
      "activations/layer22_attention_weight_max": 28.23952293395996,
      "activations/layer22_attention_weight_min": -27.60747718811035,
      "activations/layer23_attention_weight_max": 38.243682861328125,
      "activations/layer23_attention_weight_min": -26.244953155517578,
      "activations/layer2_attention_weight_max": 31.76844024658203,
      "activations/layer2_attention_weight_min": -30.116992950439453,
      "activations/layer3_attention_weight_max": 91.05238342285156,
      "activations/layer3_attention_weight_min": -90.63103485107422,
      "activations/layer4_attention_weight_max": 84.6495361328125,
      "activations/layer4_attention_weight_min": -82.97398376464844,
      "activations/layer5_attention_weight_max": 62.929664611816406,
      "activations/layer5_attention_weight_min": -68.88212585449219,
      "activations/layer6_attention_weight_max": 48.493412017822266,
      "activations/layer6_attention_weight_min": -52.23640060424805,
      "activations/layer7_attention_weight_max": 66.45647430419922,
      "activations/layer7_attention_weight_min": -66.31668090820312,
      "activations/layer8_attention_weight_max": 49.0036735534668,
      "activations/layer8_attention_weight_min": -52.20830535888672,
      "activations/layer9_attention_weight_max": 48.54633712768555,
      "activations/layer9_attention_weight_min": -48.046016693115234,
      "epoch": 20.47,
      "learning_rate": 1.8174999999999998e-05,
      "loss": 2.7173,
      "step": 352300
    },
    {
      "activations/layer0_attention_weight_max": 14.872377395629883,
      "activations/layer0_attention_weight_min": -13.7603178024292,
      "activations/layer10_attention_weight_max": 35.665870666503906,
      "activations/layer10_attention_weight_min": -35.77544021606445,
      "activations/layer11_attention_weight_max": 31.928028106689453,
      "activations/layer11_attention_weight_min": -34.7900390625,
      "activations/layer12_attention_weight_max": 20.71892738342285,
      "activations/layer12_attention_weight_min": -23.42913246154785,
      "activations/layer13_attention_weight_max": 33.31782150268555,
      "activations/layer13_attention_weight_min": -29.938461303710938,
      "activations/layer14_attention_weight_max": 36.11079025268555,
      "activations/layer14_attention_weight_min": -31.43653106689453,
      "activations/layer15_attention_weight_max": 33.687477111816406,
      "activations/layer15_attention_weight_min": -31.1215763092041,
      "activations/layer16_attention_weight_max": 36.684295654296875,
      "activations/layer16_attention_weight_min": -34.19193649291992,
      "activations/layer17_attention_weight_max": 54.84696578979492,
      "activations/layer17_attention_weight_min": -47.59028625488281,
      "activations/layer18_attention_weight_max": 50.584747314453125,
      "activations/layer18_attention_weight_min": -41.86844253540039,
      "activations/layer19_attention_weight_max": 27.097442626953125,
      "activations/layer19_attention_weight_min": -23.79218864440918,
      "activations/layer1_attention_weight_max": 15.159664154052734,
      "activations/layer1_attention_weight_min": -14.413110733032227,
      "activations/layer20_attention_weight_max": 25.70208740234375,
      "activations/layer20_attention_weight_min": -21.028804779052734,
      "activations/layer21_attention_weight_max": 36.35110855102539,
      "activations/layer21_attention_weight_min": -24.17717933654785,
      "activations/layer22_attention_weight_max": 33.206966400146484,
      "activations/layer22_attention_weight_min": -27.662506103515625,
      "activations/layer23_attention_weight_max": 36.267452239990234,
      "activations/layer23_attention_weight_min": -23.54581069946289,
      "activations/layer2_attention_weight_max": 31.693113327026367,
      "activations/layer2_attention_weight_min": -30.977872848510742,
      "activations/layer3_attention_weight_max": 87.67048645019531,
      "activations/layer3_attention_weight_min": -86.06171417236328,
      "activations/layer4_attention_weight_max": 83.93733215332031,
      "activations/layer4_attention_weight_min": -83.45406341552734,
      "activations/layer5_attention_weight_max": 61.55267333984375,
      "activations/layer5_attention_weight_min": -73.61614227294922,
      "activations/layer6_attention_weight_max": 48.275489807128906,
      "activations/layer6_attention_weight_min": -50.98462677001953,
      "activations/layer7_attention_weight_max": 64.57975769042969,
      "activations/layer7_attention_weight_min": -66.32563781738281,
      "activations/layer8_attention_weight_max": 45.98774719238281,
      "activations/layer8_attention_weight_min": -52.81413650512695,
      "activations/layer9_attention_weight_max": 44.19108963012695,
      "activations/layer9_attention_weight_min": -47.74660110473633,
      "epoch": 20.47,
      "learning_rate": 1.8156060606060606e-05,
      "loss": 2.7114,
      "step": 352350
    },
    {
      "activations/layer0_attention_weight_max": 16.063365936279297,
      "activations/layer0_attention_weight_min": -13.424911499023438,
      "activations/layer10_attention_weight_max": 36.801185607910156,
      "activations/layer10_attention_weight_min": -37.69242858886719,
      "activations/layer11_attention_weight_max": 32.01142883300781,
      "activations/layer11_attention_weight_min": -36.559608459472656,
      "activations/layer12_attention_weight_max": 19.90279197692871,
      "activations/layer12_attention_weight_min": -25.822181701660156,
      "activations/layer13_attention_weight_max": 37.48017883300781,
      "activations/layer13_attention_weight_min": -27.240571975708008,
      "activations/layer14_attention_weight_max": 39.75060272216797,
      "activations/layer14_attention_weight_min": -31.60040855407715,
      "activations/layer15_attention_weight_max": 36.139244079589844,
      "activations/layer15_attention_weight_min": -31.10614585876465,
      "activations/layer16_attention_weight_max": 38.196006774902344,
      "activations/layer16_attention_weight_min": -36.0749397277832,
      "activations/layer17_attention_weight_max": 57.57657241821289,
      "activations/layer17_attention_weight_min": -49.94478225708008,
      "activations/layer18_attention_weight_max": 51.098079681396484,
      "activations/layer18_attention_weight_min": -41.158592224121094,
      "activations/layer19_attention_weight_max": 29.207386016845703,
      "activations/layer19_attention_weight_min": -21.921669006347656,
      "activations/layer1_attention_weight_max": 15.735930442810059,
      "activations/layer1_attention_weight_min": -14.412857055664062,
      "activations/layer20_attention_weight_max": 23.835905075073242,
      "activations/layer20_attention_weight_min": -23.02232551574707,
      "activations/layer21_attention_weight_max": 39.900760650634766,
      "activations/layer21_attention_weight_min": -22.230398178100586,
      "activations/layer22_attention_weight_max": 35.5204963684082,
      "activations/layer22_attention_weight_min": -24.85037612915039,
      "activations/layer23_attention_weight_max": 40.02173614501953,
      "activations/layer23_attention_weight_min": -22.672367095947266,
      "activations/layer2_attention_weight_max": 29.44867515563965,
      "activations/layer2_attention_weight_min": -28.543262481689453,
      "activations/layer3_attention_weight_max": 82.72589874267578,
      "activations/layer3_attention_weight_min": -85.37417602539062,
      "activations/layer4_attention_weight_max": 83.7429428100586,
      "activations/layer4_attention_weight_min": -86.74816131591797,
      "activations/layer5_attention_weight_max": 60.06805419921875,
      "activations/layer5_attention_weight_min": -71.60869598388672,
      "activations/layer6_attention_weight_max": 50.33325958251953,
      "activations/layer6_attention_weight_min": -50.85149383544922,
      "activations/layer7_attention_weight_max": 63.88987350463867,
      "activations/layer7_attention_weight_min": -68.33882141113281,
      "activations/layer8_attention_weight_max": 46.05291748046875,
      "activations/layer8_attention_weight_min": -49.88388442993164,
      "activations/layer9_attention_weight_max": 42.87700653076172,
      "activations/layer9_attention_weight_min": -44.61959457397461,
      "epoch": 20.48,
      "learning_rate": 1.81375e-05,
      "loss": 2.7228,
      "step": 352400
    },
    {
      "activations/layer0_attention_weight_max": 15.278724670410156,
      "activations/layer0_attention_weight_min": -13.55614948272705,
      "activations/layer10_attention_weight_max": 41.2725715637207,
      "activations/layer10_attention_weight_min": -40.23991775512695,
      "activations/layer11_attention_weight_max": 35.32373809814453,
      "activations/layer11_attention_weight_min": -36.42123794555664,
      "activations/layer12_attention_weight_max": 20.3662166595459,
      "activations/layer12_attention_weight_min": -27.698322296142578,
      "activations/layer13_attention_weight_max": 35.186519622802734,
      "activations/layer13_attention_weight_min": -29.09334373474121,
      "activations/layer14_attention_weight_max": 37.99910354614258,
      "activations/layer14_attention_weight_min": -32.79385757446289,
      "activations/layer15_attention_weight_max": 34.487178802490234,
      "activations/layer15_attention_weight_min": -29.705106735229492,
      "activations/layer16_attention_weight_max": 37.75138473510742,
      "activations/layer16_attention_weight_min": -33.965797424316406,
      "activations/layer17_attention_weight_max": 54.26523208618164,
      "activations/layer17_attention_weight_min": -48.05473327636719,
      "activations/layer18_attention_weight_max": 50.95476531982422,
      "activations/layer18_attention_weight_min": -43.53604507446289,
      "activations/layer19_attention_weight_max": 24.426746368408203,
      "activations/layer19_attention_weight_min": -25.657052993774414,
      "activations/layer1_attention_weight_max": 15.04477310180664,
      "activations/layer1_attention_weight_min": -13.900035858154297,
      "activations/layer20_attention_weight_max": 21.426769256591797,
      "activations/layer20_attention_weight_min": -22.307844161987305,
      "activations/layer21_attention_weight_max": 34.644962310791016,
      "activations/layer21_attention_weight_min": -24.419408798217773,
      "activations/layer22_attention_weight_max": 26.39011573791504,
      "activations/layer22_attention_weight_min": -28.56122398376465,
      "activations/layer23_attention_weight_max": 36.01608657836914,
      "activations/layer23_attention_weight_min": -27.436466217041016,
      "activations/layer2_attention_weight_max": 30.54742431640625,
      "activations/layer2_attention_weight_min": -29.330862045288086,
      "activations/layer3_attention_weight_max": 90.7059326171875,
      "activations/layer3_attention_weight_min": -85.17499542236328,
      "activations/layer4_attention_weight_max": 88.6585922241211,
      "activations/layer4_attention_weight_min": -86.63143920898438,
      "activations/layer5_attention_weight_max": 62.63177490234375,
      "activations/layer5_attention_weight_min": -70.98702239990234,
      "activations/layer6_attention_weight_max": 53.674903869628906,
      "activations/layer6_attention_weight_min": -52.094078063964844,
      "activations/layer7_attention_weight_max": 64.52396392822266,
      "activations/layer7_attention_weight_min": -67.25374603271484,
      "activations/layer8_attention_weight_max": 50.97030258178711,
      "activations/layer8_attention_weight_min": -54.74636459350586,
      "activations/layer9_attention_weight_max": 46.81654357910156,
      "activations/layer9_attention_weight_min": -48.234371185302734,
      "epoch": 20.48,
      "learning_rate": 1.8118560606060605e-05,
      "loss": 2.7024,
      "step": 352450
    },
    {
      "activations/layer0_attention_weight_max": 15.948019027709961,
      "activations/layer0_attention_weight_min": -13.422154426574707,
      "activations/layer10_attention_weight_max": 35.087432861328125,
      "activations/layer10_attention_weight_min": -35.4940299987793,
      "activations/layer11_attention_weight_max": 32.06789779663086,
      "activations/layer11_attention_weight_min": -35.676307678222656,
      "activations/layer12_attention_weight_max": 18.92723274230957,
      "activations/layer12_attention_weight_min": -24.532812118530273,
      "activations/layer13_attention_weight_max": 32.63686752319336,
      "activations/layer13_attention_weight_min": -28.39236831665039,
      "activations/layer14_attention_weight_max": 34.551361083984375,
      "activations/layer14_attention_weight_min": -30.515390396118164,
      "activations/layer15_attention_weight_max": 33.11791229248047,
      "activations/layer15_attention_weight_min": -30.448392868041992,
      "activations/layer16_attention_weight_max": 31.695051193237305,
      "activations/layer16_attention_weight_min": -31.826168060302734,
      "activations/layer17_attention_weight_max": 50.225704193115234,
      "activations/layer17_attention_weight_min": -44.5679931640625,
      "activations/layer18_attention_weight_max": 46.15491485595703,
      "activations/layer18_attention_weight_min": -38.59822082519531,
      "activations/layer19_attention_weight_max": 25.452091217041016,
      "activations/layer19_attention_weight_min": -22.555707931518555,
      "activations/layer1_attention_weight_max": 14.86760139465332,
      "activations/layer1_attention_weight_min": -13.323945045471191,
      "activations/layer20_attention_weight_max": 22.033735275268555,
      "activations/layer20_attention_weight_min": -22.232545852661133,
      "activations/layer21_attention_weight_max": 36.316688537597656,
      "activations/layer21_attention_weight_min": -25.22493553161621,
      "activations/layer22_attention_weight_max": 28.01136589050293,
      "activations/layer22_attention_weight_min": -23.307781219482422,
      "activations/layer23_attention_weight_max": 33.327903747558594,
      "activations/layer23_attention_weight_min": -25.347248077392578,
      "activations/layer2_attention_weight_max": 29.582107543945312,
      "activations/layer2_attention_weight_min": -31.45569610595703,
      "activations/layer3_attention_weight_max": 82.43001556396484,
      "activations/layer3_attention_weight_min": -91.61421966552734,
      "activations/layer4_attention_weight_max": 80.08274841308594,
      "activations/layer4_attention_weight_min": -84.76512145996094,
      "activations/layer5_attention_weight_max": 60.40392303466797,
      "activations/layer5_attention_weight_min": -70.98585510253906,
      "activations/layer6_attention_weight_max": 48.583499908447266,
      "activations/layer6_attention_weight_min": -51.19005584716797,
      "activations/layer7_attention_weight_max": 62.397281646728516,
      "activations/layer7_attention_weight_min": -62.80358123779297,
      "activations/layer8_attention_weight_max": 46.39303970336914,
      "activations/layer8_attention_weight_min": -49.958351135253906,
      "activations/layer9_attention_weight_max": 43.033973693847656,
      "activations/layer9_attention_weight_min": -46.369163513183594,
      "epoch": 20.48,
      "learning_rate": 1.809962121212121e-05,
      "loss": 2.7231,
      "step": 352500
    },
    {
      "activations/layer0_attention_weight_max": 15.139857292175293,
      "activations/layer0_attention_weight_min": -13.565340995788574,
      "activations/layer10_attention_weight_max": 36.356204986572266,
      "activations/layer10_attention_weight_min": -36.13560104370117,
      "activations/layer11_attention_weight_max": 32.54233932495117,
      "activations/layer11_attention_weight_min": -35.182804107666016,
      "activations/layer12_attention_weight_max": 20.265613555908203,
      "activations/layer12_attention_weight_min": -27.236215591430664,
      "activations/layer13_attention_weight_max": 35.56203842163086,
      "activations/layer13_attention_weight_min": -28.222810745239258,
      "activations/layer14_attention_weight_max": 35.33785629272461,
      "activations/layer14_attention_weight_min": -30.635583877563477,
      "activations/layer15_attention_weight_max": 34.54963684082031,
      "activations/layer15_attention_weight_min": -29.98932647705078,
      "activations/layer16_attention_weight_max": 33.895416259765625,
      "activations/layer16_attention_weight_min": -32.615692138671875,
      "activations/layer17_attention_weight_max": 52.68143081665039,
      "activations/layer17_attention_weight_min": -45.61891174316406,
      "activations/layer18_attention_weight_max": 50.2495002746582,
      "activations/layer18_attention_weight_min": -42.43271255493164,
      "activations/layer19_attention_weight_max": 27.80301284790039,
      "activations/layer19_attention_weight_min": -22.794111251831055,
      "activations/layer1_attention_weight_max": 15.214653015136719,
      "activations/layer1_attention_weight_min": -13.358488082885742,
      "activations/layer20_attention_weight_max": 24.777597427368164,
      "activations/layer20_attention_weight_min": -22.34352684020996,
      "activations/layer21_attention_weight_max": 41.28337860107422,
      "activations/layer21_attention_weight_min": -22.05750274658203,
      "activations/layer22_attention_weight_max": 30.8009090423584,
      "activations/layer22_attention_weight_min": -25.98355484008789,
      "activations/layer23_attention_weight_max": 36.153785705566406,
      "activations/layer23_attention_weight_min": -24.714733123779297,
      "activations/layer2_attention_weight_max": 30.97265625,
      "activations/layer2_attention_weight_min": -28.98145866394043,
      "activations/layer3_attention_weight_max": 85.76895904541016,
      "activations/layer3_attention_weight_min": -85.05986022949219,
      "activations/layer4_attention_weight_max": 82.27589416503906,
      "activations/layer4_attention_weight_min": -81.04090881347656,
      "activations/layer5_attention_weight_max": 60.84442138671875,
      "activations/layer5_attention_weight_min": -73.52750396728516,
      "activations/layer6_attention_weight_max": 48.62561798095703,
      "activations/layer6_attention_weight_min": -49.68785095214844,
      "activations/layer7_attention_weight_max": 67.6005630493164,
      "activations/layer7_attention_weight_min": -65.71188354492188,
      "activations/layer8_attention_weight_max": 44.34260177612305,
      "activations/layer8_attention_weight_min": -48.52248764038086,
      "activations/layer9_attention_weight_max": 41.61082077026367,
      "activations/layer9_attention_weight_min": -44.93171691894531,
      "epoch": 20.49,
      "learning_rate": 1.808068181818182e-05,
      "loss": 2.7161,
      "step": 352550
    },
    {
      "activations/layer0_attention_weight_max": 15.799599647521973,
      "activations/layer0_attention_weight_min": -13.613507270812988,
      "activations/layer10_attention_weight_max": 34.910064697265625,
      "activations/layer10_attention_weight_min": -34.72658920288086,
      "activations/layer11_attention_weight_max": 32.140769958496094,
      "activations/layer11_attention_weight_min": -34.532554626464844,
      "activations/layer12_attention_weight_max": 18.372936248779297,
      "activations/layer12_attention_weight_min": -24.818416595458984,
      "activations/layer13_attention_weight_max": 35.80344009399414,
      "activations/layer13_attention_weight_min": -28.451889038085938,
      "activations/layer14_attention_weight_max": 34.88718795776367,
      "activations/layer14_attention_weight_min": -33.66413497924805,
      "activations/layer15_attention_weight_max": 35.33237838745117,
      "activations/layer15_attention_weight_min": -32.58837890625,
      "activations/layer16_attention_weight_max": 34.99235916137695,
      "activations/layer16_attention_weight_min": -34.00119400024414,
      "activations/layer17_attention_weight_max": 50.34707260131836,
      "activations/layer17_attention_weight_min": -47.03665542602539,
      "activations/layer18_attention_weight_max": 45.964420318603516,
      "activations/layer18_attention_weight_min": -40.41251754760742,
      "activations/layer19_attention_weight_max": 23.671037673950195,
      "activations/layer19_attention_weight_min": -21.85059928894043,
      "activations/layer1_attention_weight_max": 15.560220718383789,
      "activations/layer1_attention_weight_min": -13.06857967376709,
      "activations/layer20_attention_weight_max": 22.457500457763672,
      "activations/layer20_attention_weight_min": -21.480621337890625,
      "activations/layer21_attention_weight_max": 34.15962600708008,
      "activations/layer21_attention_weight_min": -22.46116065979004,
      "activations/layer22_attention_weight_max": 27.5756893157959,
      "activations/layer22_attention_weight_min": -24.81045150756836,
      "activations/layer23_attention_weight_max": 34.53485870361328,
      "activations/layer23_attention_weight_min": -23.540571212768555,
      "activations/layer2_attention_weight_max": 30.35570526123047,
      "activations/layer2_attention_weight_min": -27.477811813354492,
      "activations/layer3_attention_weight_max": 80.26465606689453,
      "activations/layer3_attention_weight_min": -81.65147399902344,
      "activations/layer4_attention_weight_max": 80.43978118896484,
      "activations/layer4_attention_weight_min": -77.70015716552734,
      "activations/layer5_attention_weight_max": 58.21277618408203,
      "activations/layer5_attention_weight_min": -66.40388488769531,
      "activations/layer6_attention_weight_max": 50.45024871826172,
      "activations/layer6_attention_weight_min": -49.184627532958984,
      "activations/layer7_attention_weight_max": 65.8700942993164,
      "activations/layer7_attention_weight_min": -61.18495178222656,
      "activations/layer8_attention_weight_max": 48.27983474731445,
      "activations/layer8_attention_weight_min": -49.44300842285156,
      "activations/layer9_attention_weight_max": 42.330780029296875,
      "activations/layer9_attention_weight_min": -46.72116470336914,
      "epoch": 20.49,
      "learning_rate": 1.8061742424242423e-05,
      "loss": 2.7223,
      "step": 352600
    },
    {
      "activations/layer0_attention_weight_max": 15.725372314453125,
      "activations/layer0_attention_weight_min": -13.78052043914795,
      "activations/layer10_attention_weight_max": 35.4384651184082,
      "activations/layer10_attention_weight_min": -35.34206771850586,
      "activations/layer11_attention_weight_max": 33.35474395751953,
      "activations/layer11_attention_weight_min": -33.74830627441406,
      "activations/layer12_attention_weight_max": 18.65521812438965,
      "activations/layer12_attention_weight_min": -25.87421989440918,
      "activations/layer13_attention_weight_max": 42.078712463378906,
      "activations/layer13_attention_weight_min": -33.516639709472656,
      "activations/layer14_attention_weight_max": 34.10027313232422,
      "activations/layer14_attention_weight_min": -28.04575538635254,
      "activations/layer15_attention_weight_max": 32.60783004760742,
      "activations/layer15_attention_weight_min": -28.794845581054688,
      "activations/layer16_attention_weight_max": 32.185550689697266,
      "activations/layer16_attention_weight_min": -29.597747802734375,
      "activations/layer17_attention_weight_max": 50.28059005737305,
      "activations/layer17_attention_weight_min": -44.57699203491211,
      "activations/layer18_attention_weight_max": 44.811614990234375,
      "activations/layer18_attention_weight_min": -40.99789810180664,
      "activations/layer19_attention_weight_max": 25.00410270690918,
      "activations/layer19_attention_weight_min": -24.378877639770508,
      "activations/layer1_attention_weight_max": 15.317774772644043,
      "activations/layer1_attention_weight_min": -14.101374626159668,
      "activations/layer20_attention_weight_max": 23.406536102294922,
      "activations/layer20_attention_weight_min": -23.489154815673828,
      "activations/layer21_attention_weight_max": 34.47251892089844,
      "activations/layer21_attention_weight_min": -23.084280014038086,
      "activations/layer22_attention_weight_max": 26.9703426361084,
      "activations/layer22_attention_weight_min": -25.015398025512695,
      "activations/layer23_attention_weight_max": 35.1497802734375,
      "activations/layer23_attention_weight_min": -26.22162628173828,
      "activations/layer2_attention_weight_max": 32.0692138671875,
      "activations/layer2_attention_weight_min": -29.037405014038086,
      "activations/layer3_attention_weight_max": 90.00314331054688,
      "activations/layer3_attention_weight_min": -90.69908142089844,
      "activations/layer4_attention_weight_max": 85.40837097167969,
      "activations/layer4_attention_weight_min": -83.8783187866211,
      "activations/layer5_attention_weight_max": 65.17520904541016,
      "activations/layer5_attention_weight_min": -76.78656005859375,
      "activations/layer6_attention_weight_max": 48.013057708740234,
      "activations/layer6_attention_weight_min": -50.94829559326172,
      "activations/layer7_attention_weight_max": 67.1871566772461,
      "activations/layer7_attention_weight_min": -68.36251068115234,
      "activations/layer8_attention_weight_max": 48.32429885864258,
      "activations/layer8_attention_weight_min": -49.3459358215332,
      "activations/layer9_attention_weight_max": 43.98054122924805,
      "activations/layer9_attention_weight_min": -45.53070068359375,
      "epoch": 20.49,
      "learning_rate": 1.8042803030303028e-05,
      "loss": 2.7189,
      "step": 352650
    },
    {
      "activations/layer0_attention_weight_max": 15.775919914245605,
      "activations/layer0_attention_weight_min": -13.900431632995605,
      "activations/layer10_attention_weight_max": 37.637672424316406,
      "activations/layer10_attention_weight_min": -36.75145721435547,
      "activations/layer11_attention_weight_max": 34.123775482177734,
      "activations/layer11_attention_weight_min": -35.39027786254883,
      "activations/layer12_attention_weight_max": 21.8328857421875,
      "activations/layer12_attention_weight_min": -28.664880752563477,
      "activations/layer13_attention_weight_max": 38.691314697265625,
      "activations/layer13_attention_weight_min": -27.847841262817383,
      "activations/layer14_attention_weight_max": 37.63713455200195,
      "activations/layer14_attention_weight_min": -32.69807434082031,
      "activations/layer15_attention_weight_max": 37.11608123779297,
      "activations/layer15_attention_weight_min": -31.425983428955078,
      "activations/layer16_attention_weight_max": 38.471221923828125,
      "activations/layer16_attention_weight_min": -34.22148513793945,
      "activations/layer17_attention_weight_max": 58.93354034423828,
      "activations/layer17_attention_weight_min": -49.92293167114258,
      "activations/layer18_attention_weight_max": 57.62014389038086,
      "activations/layer18_attention_weight_min": -44.33016586303711,
      "activations/layer19_attention_weight_max": 28.561691284179688,
      "activations/layer19_attention_weight_min": -26.285425186157227,
      "activations/layer1_attention_weight_max": 15.083625793457031,
      "activations/layer1_attention_weight_min": -13.856040954589844,
      "activations/layer20_attention_weight_max": 26.023540496826172,
      "activations/layer20_attention_weight_min": -22.724559783935547,
      "activations/layer21_attention_weight_max": 44.04496383666992,
      "activations/layer21_attention_weight_min": -22.56744956970215,
      "activations/layer22_attention_weight_max": 32.25290298461914,
      "activations/layer22_attention_weight_min": -27.54696273803711,
      "activations/layer23_attention_weight_max": 42.93647766113281,
      "activations/layer23_attention_weight_min": -23.865943908691406,
      "activations/layer2_attention_weight_max": 31.09963035583496,
      "activations/layer2_attention_weight_min": -30.87929344177246,
      "activations/layer3_attention_weight_max": 89.2796859741211,
      "activations/layer3_attention_weight_min": -91.98059844970703,
      "activations/layer4_attention_weight_max": 90.87902069091797,
      "activations/layer4_attention_weight_min": -90.37944793701172,
      "activations/layer5_attention_weight_max": 60.559288024902344,
      "activations/layer5_attention_weight_min": -73.03191375732422,
      "activations/layer6_attention_weight_max": 51.81776428222656,
      "activations/layer6_attention_weight_min": -51.15477752685547,
      "activations/layer7_attention_weight_max": 68.76504516601562,
      "activations/layer7_attention_weight_min": -65.64298248291016,
      "activations/layer8_attention_weight_max": 50.64482879638672,
      "activations/layer8_attention_weight_min": -52.40340805053711,
      "activations/layer9_attention_weight_max": 45.74927520751953,
      "activations/layer9_attention_weight_min": -47.391475677490234,
      "epoch": 20.49,
      "learning_rate": 1.8023863636363636e-05,
      "loss": 2.7114,
      "step": 352700
    },
    {
      "activations/layer0_attention_weight_max": 15.124225616455078,
      "activations/layer0_attention_weight_min": -13.607903480529785,
      "activations/layer10_attention_weight_max": 36.823726654052734,
      "activations/layer10_attention_weight_min": -34.702701568603516,
      "activations/layer11_attention_weight_max": 33.89692687988281,
      "activations/layer11_attention_weight_min": -34.65397644042969,
      "activations/layer12_attention_weight_max": 19.86648941040039,
      "activations/layer12_attention_weight_min": -24.490047454833984,
      "activations/layer13_attention_weight_max": 32.40669250488281,
      "activations/layer13_attention_weight_min": -28.014602661132812,
      "activations/layer14_attention_weight_max": 33.24480056762695,
      "activations/layer14_attention_weight_min": -29.661651611328125,
      "activations/layer15_attention_weight_max": 30.78182029724121,
      "activations/layer15_attention_weight_min": -30.064979553222656,
      "activations/layer16_attention_weight_max": 31.99229621887207,
      "activations/layer16_attention_weight_min": -32.86508560180664,
      "activations/layer17_attention_weight_max": 47.5954475402832,
      "activations/layer17_attention_weight_min": -44.26091766357422,
      "activations/layer18_attention_weight_max": 43.830570220947266,
      "activations/layer18_attention_weight_min": -42.445247650146484,
      "activations/layer19_attention_weight_max": 21.645675659179688,
      "activations/layer19_attention_weight_min": -23.53484535217285,
      "activations/layer1_attention_weight_max": 16.236074447631836,
      "activations/layer1_attention_weight_min": -13.948801040649414,
      "activations/layer20_attention_weight_max": 21.013652801513672,
      "activations/layer20_attention_weight_min": -21.623180389404297,
      "activations/layer21_attention_weight_max": 35.64814758300781,
      "activations/layer21_attention_weight_min": -24.546789169311523,
      "activations/layer22_attention_weight_max": 27.996673583984375,
      "activations/layer22_attention_weight_min": -26.857046127319336,
      "activations/layer23_attention_weight_max": 31.576995849609375,
      "activations/layer23_attention_weight_min": -26.14087677001953,
      "activations/layer2_attention_weight_max": 30.04592514038086,
      "activations/layer2_attention_weight_min": -28.70691680908203,
      "activations/layer3_attention_weight_max": 87.52977752685547,
      "activations/layer3_attention_weight_min": -84.13211059570312,
      "activations/layer4_attention_weight_max": 82.87657928466797,
      "activations/layer4_attention_weight_min": -81.66548919677734,
      "activations/layer5_attention_weight_max": 60.839962005615234,
      "activations/layer5_attention_weight_min": -70.3678970336914,
      "activations/layer6_attention_weight_max": 47.92534255981445,
      "activations/layer6_attention_weight_min": -48.08736801147461,
      "activations/layer7_attention_weight_max": 65.69673919677734,
      "activations/layer7_attention_weight_min": -65.02745056152344,
      "activations/layer8_attention_weight_max": 46.40546417236328,
      "activations/layer8_attention_weight_min": -49.80436706542969,
      "activations/layer9_attention_weight_max": 42.974430084228516,
      "activations/layer9_attention_weight_min": -44.97442626953125,
      "epoch": 20.5,
      "learning_rate": 1.800492424242424e-05,
      "loss": 2.7102,
      "step": 352750
    },
    {
      "activations/layer0_attention_weight_max": 16.23771095275879,
      "activations/layer0_attention_weight_min": -13.215315818786621,
      "activations/layer10_attention_weight_max": 33.761322021484375,
      "activations/layer10_attention_weight_min": -34.86629104614258,
      "activations/layer11_attention_weight_max": 33.640193939208984,
      "activations/layer11_attention_weight_min": -34.75563049316406,
      "activations/layer12_attention_weight_max": 19.367839813232422,
      "activations/layer12_attention_weight_min": -25.443145751953125,
      "activations/layer13_attention_weight_max": 35.520545959472656,
      "activations/layer13_attention_weight_min": -27.955440521240234,
      "activations/layer14_attention_weight_max": 35.66900634765625,
      "activations/layer14_attention_weight_min": -31.316162109375,
      "activations/layer15_attention_weight_max": 33.107967376708984,
      "activations/layer15_attention_weight_min": -31.535249710083008,
      "activations/layer16_attention_weight_max": 34.343589782714844,
      "activations/layer16_attention_weight_min": -33.48958206176758,
      "activations/layer17_attention_weight_max": 46.15742874145508,
      "activations/layer17_attention_weight_min": -43.51355743408203,
      "activations/layer18_attention_weight_max": 43.16343307495117,
      "activations/layer18_attention_weight_min": -39.02587890625,
      "activations/layer19_attention_weight_max": 26.020856857299805,
      "activations/layer19_attention_weight_min": -22.680234909057617,
      "activations/layer1_attention_weight_max": 15.740928649902344,
      "activations/layer1_attention_weight_min": -12.878512382507324,
      "activations/layer20_attention_weight_max": 21.34144401550293,
      "activations/layer20_attention_weight_min": -22.03860092163086,
      "activations/layer21_attention_weight_max": 34.38161087036133,
      "activations/layer21_attention_weight_min": -23.51747703552246,
      "activations/layer22_attention_weight_max": 29.36457061767578,
      "activations/layer22_attention_weight_min": -25.79357147216797,
      "activations/layer23_attention_weight_max": 36.61897659301758,
      "activations/layer23_attention_weight_min": -25.937095642089844,
      "activations/layer2_attention_weight_max": 30.657514572143555,
      "activations/layer2_attention_weight_min": -28.960758209228516,
      "activations/layer3_attention_weight_max": 90.80868530273438,
      "activations/layer3_attention_weight_min": -90.95596313476562,
      "activations/layer4_attention_weight_max": 83.9723892211914,
      "activations/layer4_attention_weight_min": -84.02780151367188,
      "activations/layer5_attention_weight_max": 65.62771606445312,
      "activations/layer5_attention_weight_min": -68.93988037109375,
      "activations/layer6_attention_weight_max": 52.64044952392578,
      "activations/layer6_attention_weight_min": -48.84693908691406,
      "activations/layer7_attention_weight_max": 71.6550064086914,
      "activations/layer7_attention_weight_min": -68.16466522216797,
      "activations/layer8_attention_weight_max": 46.11140441894531,
      "activations/layer8_attention_weight_min": -48.561275482177734,
      "activations/layer9_attention_weight_max": 44.85561752319336,
      "activations/layer9_attention_weight_min": -44.98871612548828,
      "epoch": 20.5,
      "learning_rate": 1.7985984848484846e-05,
      "loss": 2.7198,
      "step": 352800
    },
    {
      "activations/layer0_attention_weight_max": 15.796573638916016,
      "activations/layer0_attention_weight_min": -14.064523696899414,
      "activations/layer10_attention_weight_max": 33.855655670166016,
      "activations/layer10_attention_weight_min": -33.989532470703125,
      "activations/layer11_attention_weight_max": 31.308086395263672,
      "activations/layer11_attention_weight_min": -33.710540771484375,
      "activations/layer12_attention_weight_max": 19.03766632080078,
      "activations/layer12_attention_weight_min": -25.697059631347656,
      "activations/layer13_attention_weight_max": 33.02867126464844,
      "activations/layer13_attention_weight_min": -27.28025245666504,
      "activations/layer14_attention_weight_max": 32.84213638305664,
      "activations/layer14_attention_weight_min": -29.720277786254883,
      "activations/layer15_attention_weight_max": 32.152976989746094,
      "activations/layer15_attention_weight_min": -29.58311653137207,
      "activations/layer16_attention_weight_max": 34.58247756958008,
      "activations/layer16_attention_weight_min": -33.23411560058594,
      "activations/layer17_attention_weight_max": 54.46975326538086,
      "activations/layer17_attention_weight_min": -46.31201171875,
      "activations/layer18_attention_weight_max": 49.396141052246094,
      "activations/layer18_attention_weight_min": -42.59956741333008,
      "activations/layer19_attention_weight_max": 24.725053787231445,
      "activations/layer19_attention_weight_min": -23.177738189697266,
      "activations/layer1_attention_weight_max": 15.694916725158691,
      "activations/layer1_attention_weight_min": -13.052237510681152,
      "activations/layer20_attention_weight_max": 23.192699432373047,
      "activations/layer20_attention_weight_min": -21.551572799682617,
      "activations/layer21_attention_weight_max": 33.48378372192383,
      "activations/layer21_attention_weight_min": -21.416339874267578,
      "activations/layer22_attention_weight_max": 27.653039932250977,
      "activations/layer22_attention_weight_min": -24.410751342773438,
      "activations/layer23_attention_weight_max": 34.49778747558594,
      "activations/layer23_attention_weight_min": -25.92482566833496,
      "activations/layer2_attention_weight_max": 30.50655746459961,
      "activations/layer2_attention_weight_min": -29.652828216552734,
      "activations/layer3_attention_weight_max": 84.59706115722656,
      "activations/layer3_attention_weight_min": -91.05145263671875,
      "activations/layer4_attention_weight_max": 81.12046813964844,
      "activations/layer4_attention_weight_min": -84.32044219970703,
      "activations/layer5_attention_weight_max": 60.94270324707031,
      "activations/layer5_attention_weight_min": -70.65530395507812,
      "activations/layer6_attention_weight_max": 48.071712493896484,
      "activations/layer6_attention_weight_min": -50.70010757446289,
      "activations/layer7_attention_weight_max": 62.433834075927734,
      "activations/layer7_attention_weight_min": -62.816768646240234,
      "activations/layer8_attention_weight_max": 44.860713958740234,
      "activations/layer8_attention_weight_min": -49.81142044067383,
      "activations/layer9_attention_weight_max": 40.26777648925781,
      "activations/layer9_attention_weight_min": -44.00022506713867,
      "epoch": 20.5,
      "learning_rate": 1.796704545454545e-05,
      "loss": 2.7379,
      "step": 352850
    },
    {
      "activations/layer0_attention_weight_max": 15.565201759338379,
      "activations/layer0_attention_weight_min": -13.760038375854492,
      "activations/layer10_attention_weight_max": 34.38418960571289,
      "activations/layer10_attention_weight_min": -34.65234375,
      "activations/layer11_attention_weight_max": 35.234283447265625,
      "activations/layer11_attention_weight_min": -34.868282318115234,
      "activations/layer12_attention_weight_max": 18.721115112304688,
      "activations/layer12_attention_weight_min": -26.11125373840332,
      "activations/layer13_attention_weight_max": 32.452144622802734,
      "activations/layer13_attention_weight_min": -27.57794952392578,
      "activations/layer14_attention_weight_max": 35.314231872558594,
      "activations/layer14_attention_weight_min": -29.667499542236328,
      "activations/layer15_attention_weight_max": 33.28428268432617,
      "activations/layer15_attention_weight_min": -29.93252182006836,
      "activations/layer16_attention_weight_max": 34.11211395263672,
      "activations/layer16_attention_weight_min": -30.20920753479004,
      "activations/layer17_attention_weight_max": 51.73457717895508,
      "activations/layer17_attention_weight_min": -42.13650894165039,
      "activations/layer18_attention_weight_max": 46.91550064086914,
      "activations/layer18_attention_weight_min": -37.39771270751953,
      "activations/layer19_attention_weight_max": 24.34779930114746,
      "activations/layer19_attention_weight_min": -20.852949142456055,
      "activations/layer1_attention_weight_max": 14.706475257873535,
      "activations/layer1_attention_weight_min": -13.950201988220215,
      "activations/layer20_attention_weight_max": 21.125560760498047,
      "activations/layer20_attention_weight_min": -19.822023391723633,
      "activations/layer21_attention_weight_max": 34.727203369140625,
      "activations/layer21_attention_weight_min": -20.235185623168945,
      "activations/layer22_attention_weight_max": 31.710832595825195,
      "activations/layer22_attention_weight_min": -24.71845817565918,
      "activations/layer23_attention_weight_max": 34.48776626586914,
      "activations/layer23_attention_weight_min": -22.619441986083984,
      "activations/layer2_attention_weight_max": 32.91361999511719,
      "activations/layer2_attention_weight_min": -30.117698669433594,
      "activations/layer3_attention_weight_max": 87.42378997802734,
      "activations/layer3_attention_weight_min": -86.30197143554688,
      "activations/layer4_attention_weight_max": 84.00361633300781,
      "activations/layer4_attention_weight_min": -85.25862884521484,
      "activations/layer5_attention_weight_max": 61.57048797607422,
      "activations/layer5_attention_weight_min": -76.06153869628906,
      "activations/layer6_attention_weight_max": 51.770484924316406,
      "activations/layer6_attention_weight_min": -52.12639236450195,
      "activations/layer7_attention_weight_max": 65.7825927734375,
      "activations/layer7_attention_weight_min": -67.99897003173828,
      "activations/layer8_attention_weight_max": 49.21662902832031,
      "activations/layer8_attention_weight_min": -49.535865783691406,
      "activations/layer9_attention_weight_max": 44.319854736328125,
      "activations/layer9_attention_weight_min": -46.36793899536133,
      "epoch": 20.51,
      "learning_rate": 1.794810606060606e-05,
      "loss": 2.7082,
      "step": 352900
    },
    {
      "activations/layer0_attention_weight_max": 15.336925506591797,
      "activations/layer0_attention_weight_min": -13.735507011413574,
      "activations/layer10_attention_weight_max": 33.294063568115234,
      "activations/layer10_attention_weight_min": -35.64546203613281,
      "activations/layer11_attention_weight_max": 31.267894744873047,
      "activations/layer11_attention_weight_min": -34.81631088256836,
      "activations/layer12_attention_weight_max": 19.111970901489258,
      "activations/layer12_attention_weight_min": -25.144163131713867,
      "activations/layer13_attention_weight_max": 43.488990783691406,
      "activations/layer13_attention_weight_min": -26.121484756469727,
      "activations/layer14_attention_weight_max": 34.132503509521484,
      "activations/layer14_attention_weight_min": -31.27100372314453,
      "activations/layer15_attention_weight_max": 33.692169189453125,
      "activations/layer15_attention_weight_min": -31.0753173828125,
      "activations/layer16_attention_weight_max": 34.59446716308594,
      "activations/layer16_attention_weight_min": -35.07430648803711,
      "activations/layer17_attention_weight_max": 51.37697982788086,
      "activations/layer17_attention_weight_min": -49.18535614013672,
      "activations/layer18_attention_weight_max": 48.55839920043945,
      "activations/layer18_attention_weight_min": -42.71226119995117,
      "activations/layer19_attention_weight_max": 26.368453979492188,
      "activations/layer19_attention_weight_min": -24.214223861694336,
      "activations/layer1_attention_weight_max": 15.88171672821045,
      "activations/layer1_attention_weight_min": -16.62131690979004,
      "activations/layer20_attention_weight_max": 23.802480697631836,
      "activations/layer20_attention_weight_min": -22.603612899780273,
      "activations/layer21_attention_weight_max": 35.83626174926758,
      "activations/layer21_attention_weight_min": -20.439634323120117,
      "activations/layer22_attention_weight_max": 30.831398010253906,
      "activations/layer22_attention_weight_min": -26.81538200378418,
      "activations/layer23_attention_weight_max": 36.78595733642578,
      "activations/layer23_attention_weight_min": -26.12521743774414,
      "activations/layer2_attention_weight_max": 29.92430877685547,
      "activations/layer2_attention_weight_min": -31.027997970581055,
      "activations/layer3_attention_weight_max": 84.21882629394531,
      "activations/layer3_attention_weight_min": -85.54602813720703,
      "activations/layer4_attention_weight_max": 81.31749725341797,
      "activations/layer4_attention_weight_min": -83.8301773071289,
      "activations/layer5_attention_weight_max": 57.086769104003906,
      "activations/layer5_attention_weight_min": -69.84184265136719,
      "activations/layer6_attention_weight_max": 45.9278678894043,
      "activations/layer6_attention_weight_min": -47.586734771728516,
      "activations/layer7_attention_weight_max": 62.849613189697266,
      "activations/layer7_attention_weight_min": -60.71963882446289,
      "activations/layer8_attention_weight_max": 42.906246185302734,
      "activations/layer8_attention_weight_min": -45.25242614746094,
      "activations/layer9_attention_weight_max": 39.53422164916992,
      "activations/layer9_attention_weight_min": -42.82685470581055,
      "epoch": 20.51,
      "learning_rate": 1.7929166666666664e-05,
      "loss": 2.7121,
      "step": 352950
    },
    {
      "activations/layer0_attention_weight_max": 16.914697647094727,
      "activations/layer0_attention_weight_min": -13.915422439575195,
      "activations/layer10_attention_weight_max": 39.15285110473633,
      "activations/layer10_attention_weight_min": -37.98261260986328,
      "activations/layer11_attention_weight_max": 36.243003845214844,
      "activations/layer11_attention_weight_min": -35.27015686035156,
      "activations/layer12_attention_weight_max": 18.862173080444336,
      "activations/layer12_attention_weight_min": -24.98453712463379,
      "activations/layer13_attention_weight_max": 44.50944519042969,
      "activations/layer13_attention_weight_min": -26.531648635864258,
      "activations/layer14_attention_weight_max": 48.47089767456055,
      "activations/layer14_attention_weight_min": -29.004087448120117,
      "activations/layer15_attention_weight_max": 44.55677032470703,
      "activations/layer15_attention_weight_min": -28.711149215698242,
      "activations/layer16_attention_weight_max": 37.0336799621582,
      "activations/layer16_attention_weight_min": -33.52260208129883,
      "activations/layer17_attention_weight_max": 56.15443801879883,
      "activations/layer17_attention_weight_min": -48.77695846557617,
      "activations/layer18_attention_weight_max": 48.393402099609375,
      "activations/layer18_attention_weight_min": -45.5112419128418,
      "activations/layer19_attention_weight_max": 22.602975845336914,
      "activations/layer19_attention_weight_min": -24.014150619506836,
      "activations/layer1_attention_weight_max": 16.134262084960938,
      "activations/layer1_attention_weight_min": -16.157211303710938,
      "activations/layer20_attention_weight_max": 24.31233787536621,
      "activations/layer20_attention_weight_min": -22.12938117980957,
      "activations/layer21_attention_weight_max": 41.31035232543945,
      "activations/layer21_attention_weight_min": -23.89927101135254,
      "activations/layer22_attention_weight_max": 28.17374610900879,
      "activations/layer22_attention_weight_min": -24.96967124938965,
      "activations/layer23_attention_weight_max": 39.64909744262695,
      "activations/layer23_attention_weight_min": -23.504661560058594,
      "activations/layer2_attention_weight_max": 31.277244567871094,
      "activations/layer2_attention_weight_min": -30.4259090423584,
      "activations/layer3_attention_weight_max": 87.33385467529297,
      "activations/layer3_attention_weight_min": -90.42962646484375,
      "activations/layer4_attention_weight_max": 84.98086547851562,
      "activations/layer4_attention_weight_min": -81.611328125,
      "activations/layer5_attention_weight_max": 59.61249923706055,
      "activations/layer5_attention_weight_min": -65.08523559570312,
      "activations/layer6_attention_weight_max": 46.2266731262207,
      "activations/layer6_attention_weight_min": -49.70455551147461,
      "activations/layer7_attention_weight_max": 66.9867172241211,
      "activations/layer7_attention_weight_min": -64.33296203613281,
      "activations/layer8_attention_weight_max": 47.81681442260742,
      "activations/layer8_attention_weight_min": -49.51307678222656,
      "activations/layer9_attention_weight_max": 50.77558517456055,
      "activations/layer9_attention_weight_min": -46.42702102661133,
      "epoch": 20.51,
      "learning_rate": 1.791022727272727e-05,
      "loss": 2.7191,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.4561,
      "eval_samples_per_second": 507.801,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.4561,
      "eval_openwebtext_samples_per_second": 507.801,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_wikitext_loss": 2.91796875,
      "eval_wikitext_ppl": 18.503663694167408,
      "eval_wikitext_runtime": 1.9916,
      "eval_wikitext_samples_per_second": 228.966,
      "step": 353000
    },
    {
      "epoch": 20.51,
      "eval_lambada_loss": 2.396484375,
      "eval_lambada_ppl": 10.984491067549964,
      "eval_lambada_runtime": 9.6076,
      "eval_lambada_samples_per_second": 506.786,
      "step": 353000
    },
    {
      "activations/layer0_attention_weight_max": 15.209184646606445,
      "activations/layer0_attention_weight_min": -13.95011043548584,
      "activations/layer10_attention_weight_max": 34.104400634765625,
      "activations/layer10_attention_weight_min": -33.91796875,
      "activations/layer11_attention_weight_max": 33.09903335571289,
      "activations/layer11_attention_weight_min": -33.46882629394531,
      "activations/layer12_attention_weight_max": 19.088083267211914,
      "activations/layer12_attention_weight_min": -26.922664642333984,
      "activations/layer13_attention_weight_max": 32.638702392578125,
      "activations/layer13_attention_weight_min": -26.51050567626953,
      "activations/layer14_attention_weight_max": 33.13020706176758,
      "activations/layer14_attention_weight_min": -30.413923263549805,
      "activations/layer15_attention_weight_max": 33.94312286376953,
      "activations/layer15_attention_weight_min": -30.4846248626709,
      "activations/layer16_attention_weight_max": 33.96832275390625,
      "activations/layer16_attention_weight_min": -33.18278503417969,
      "activations/layer17_attention_weight_max": 51.3242073059082,
      "activations/layer17_attention_weight_min": -48.05824661254883,
      "activations/layer18_attention_weight_max": 49.84347915649414,
      "activations/layer18_attention_weight_min": -44.41645050048828,
      "activations/layer19_attention_weight_max": 25.346818923950195,
      "activations/layer19_attention_weight_min": -24.81837272644043,
      "activations/layer1_attention_weight_max": 15.647892951965332,
      "activations/layer1_attention_weight_min": -14.446856498718262,
      "activations/layer20_attention_weight_max": 24.916698455810547,
      "activations/layer20_attention_weight_min": -21.584379196166992,
      "activations/layer21_attention_weight_max": 32.342918395996094,
      "activations/layer21_attention_weight_min": -21.75920867919922,
      "activations/layer22_attention_weight_max": 29.824283599853516,
      "activations/layer22_attention_weight_min": -24.83000373840332,
      "activations/layer23_attention_weight_max": 39.15879821777344,
      "activations/layer23_attention_weight_min": -23.594905853271484,
      "activations/layer2_attention_weight_max": 30.152952194213867,
      "activations/layer2_attention_weight_min": -29.124351501464844,
      "activations/layer3_attention_weight_max": 83.77526092529297,
      "activations/layer3_attention_weight_min": -83.88866424560547,
      "activations/layer4_attention_weight_max": 81.1935043334961,
      "activations/layer4_attention_weight_min": -81.46965789794922,
      "activations/layer5_attention_weight_max": 60.430580139160156,
      "activations/layer5_attention_weight_min": -69.46304321289062,
      "activations/layer6_attention_weight_max": 48.48488998413086,
      "activations/layer6_attention_weight_min": -52.70828628540039,
      "activations/layer7_attention_weight_max": 65.13201904296875,
      "activations/layer7_attention_weight_min": -65.76456451416016,
      "activations/layer8_attention_weight_max": 45.00051498413086,
      "activations/layer8_attention_weight_min": -50.63340377807617,
      "activations/layer9_attention_weight_max": 41.44523239135742,
      "activations/layer9_attention_weight_min": -46.86404037475586,
      "epoch": 20.51,
      "learning_rate": 1.7891287878787877e-05,
      "loss": 2.7138,
      "step": 353050
    },
    {
      "activations/layer0_attention_weight_max": 15.15837574005127,
      "activations/layer0_attention_weight_min": -13.980694770812988,
      "activations/layer10_attention_weight_max": 38.23492431640625,
      "activations/layer10_attention_weight_min": -38.029945373535156,
      "activations/layer11_attention_weight_max": 38.408382415771484,
      "activations/layer11_attention_weight_min": -36.007423400878906,
      "activations/layer12_attention_weight_max": 20.880069732666016,
      "activations/layer12_attention_weight_min": -24.73674201965332,
      "activations/layer13_attention_weight_max": 33.560462951660156,
      "activations/layer13_attention_weight_min": -28.36149787902832,
      "activations/layer14_attention_weight_max": 37.99099349975586,
      "activations/layer14_attention_weight_min": -30.664581298828125,
      "activations/layer15_attention_weight_max": 33.53188705444336,
      "activations/layer15_attention_weight_min": -30.34376335144043,
      "activations/layer16_attention_weight_max": 34.447452545166016,
      "activations/layer16_attention_weight_min": -33.43553924560547,
      "activations/layer17_attention_weight_max": 54.250205993652344,
      "activations/layer17_attention_weight_min": -45.45176315307617,
      "activations/layer18_attention_weight_max": 50.92054748535156,
      "activations/layer18_attention_weight_min": -39.80210876464844,
      "activations/layer19_attention_weight_max": 24.84521484375,
      "activations/layer19_attention_weight_min": -22.490680694580078,
      "activations/layer1_attention_weight_max": 15.265317916870117,
      "activations/layer1_attention_weight_min": -13.91450309753418,
      "activations/layer20_attention_weight_max": 21.196226119995117,
      "activations/layer20_attention_weight_min": -20.96576690673828,
      "activations/layer21_attention_weight_max": 34.86971664428711,
      "activations/layer21_attention_weight_min": -22.005817413330078,
      "activations/layer22_attention_weight_max": 29.15176773071289,
      "activations/layer22_attention_weight_min": -24.74488639831543,
      "activations/layer23_attention_weight_max": 38.38648223876953,
      "activations/layer23_attention_weight_min": -23.470497131347656,
      "activations/layer2_attention_weight_max": 30.979825973510742,
      "activations/layer2_attention_weight_min": -32.185874938964844,
      "activations/layer3_attention_weight_max": 87.66496276855469,
      "activations/layer3_attention_weight_min": -93.25835418701172,
      "activations/layer4_attention_weight_max": 84.73565673828125,
      "activations/layer4_attention_weight_min": -84.95194244384766,
      "activations/layer5_attention_weight_max": 63.22809982299805,
      "activations/layer5_attention_weight_min": -74.17410278320312,
      "activations/layer6_attention_weight_max": 50.49907302856445,
      "activations/layer6_attention_weight_min": -51.948692321777344,
      "activations/layer7_attention_weight_max": 68.21531677246094,
      "activations/layer7_attention_weight_min": -69.18114471435547,
      "activations/layer8_attention_weight_max": 48.51686477661133,
      "activations/layer8_attention_weight_min": -52.17878723144531,
      "activations/layer9_attention_weight_max": 48.324798583984375,
      "activations/layer9_attention_weight_min": -47.55619430541992,
      "epoch": 20.52,
      "learning_rate": 1.7872348484848482e-05,
      "loss": 2.6988,
      "step": 353100
    },
    {
      "activations/layer0_attention_weight_max": 15.14647388458252,
      "activations/layer0_attention_weight_min": -13.868844032287598,
      "activations/layer10_attention_weight_max": 39.767642974853516,
      "activations/layer10_attention_weight_min": -39.80139923095703,
      "activations/layer11_attention_weight_max": 38.13315200805664,
      "activations/layer11_attention_weight_min": -37.6929817199707,
      "activations/layer12_attention_weight_max": 18.440088272094727,
      "activations/layer12_attention_weight_min": -23.68250846862793,
      "activations/layer13_attention_weight_max": 36.62864685058594,
      "activations/layer13_attention_weight_min": -30.08427619934082,
      "activations/layer14_attention_weight_max": 39.044593811035156,
      "activations/layer14_attention_weight_min": -32.28138732910156,
      "activations/layer15_attention_weight_max": 35.165531158447266,
      "activations/layer15_attention_weight_min": -34.73085021972656,
      "activations/layer16_attention_weight_max": 34.09199142456055,
      "activations/layer16_attention_weight_min": -32.54542922973633,
      "activations/layer17_attention_weight_max": 52.39225769042969,
      "activations/layer17_attention_weight_min": -47.00688934326172,
      "activations/layer18_attention_weight_max": 48.790061950683594,
      "activations/layer18_attention_weight_min": -41.00688934326172,
      "activations/layer19_attention_weight_max": 25.912927627563477,
      "activations/layer19_attention_weight_min": -24.33734703063965,
      "activations/layer1_attention_weight_max": 15.275612831115723,
      "activations/layer1_attention_weight_min": -15.705893516540527,
      "activations/layer20_attention_weight_max": 27.142295837402344,
      "activations/layer20_attention_weight_min": -23.164770126342773,
      "activations/layer21_attention_weight_max": 34.68954086303711,
      "activations/layer21_attention_weight_min": -21.435888290405273,
      "activations/layer22_attention_weight_max": 28.326047897338867,
      "activations/layer22_attention_weight_min": -26.284584045410156,
      "activations/layer23_attention_weight_max": 35.64996337890625,
      "activations/layer23_attention_weight_min": -26.08846664428711,
      "activations/layer2_attention_weight_max": 33.008941650390625,
      "activations/layer2_attention_weight_min": -33.143226623535156,
      "activations/layer3_attention_weight_max": 93.10132598876953,
      "activations/layer3_attention_weight_min": -92.46330261230469,
      "activations/layer4_attention_weight_max": 86.25029754638672,
      "activations/layer4_attention_weight_min": -86.22964477539062,
      "activations/layer5_attention_weight_max": 63.83552169799805,
      "activations/layer5_attention_weight_min": -73.73687744140625,
      "activations/layer6_attention_weight_max": 51.44291305541992,
      "activations/layer6_attention_weight_min": -53.92567443847656,
      "activations/layer7_attention_weight_max": 68.20987701416016,
      "activations/layer7_attention_weight_min": -75.29911804199219,
      "activations/layer8_attention_weight_max": 51.63679885864258,
      "activations/layer8_attention_weight_min": -53.62637710571289,
      "activations/layer9_attention_weight_max": 49.54557418823242,
      "activations/layer9_attention_weight_min": -48.88517379760742,
      "epoch": 20.52,
      "learning_rate": 1.7853409090909087e-05,
      "loss": 2.7211,
      "step": 353150
    },
    {
      "activations/layer0_attention_weight_max": 16.16206169128418,
      "activations/layer0_attention_weight_min": -13.868847846984863,
      "activations/layer10_attention_weight_max": 37.81816482543945,
      "activations/layer10_attention_weight_min": -37.198509216308594,
      "activations/layer11_attention_weight_max": 33.78403091430664,
      "activations/layer11_attention_weight_min": -36.326171875,
      "activations/layer12_attention_weight_max": 19.68507957458496,
      "activations/layer12_attention_weight_min": -24.234882354736328,
      "activations/layer13_attention_weight_max": 30.4566593170166,
      "activations/layer13_attention_weight_min": -27.962238311767578,
      "activations/layer14_attention_weight_max": 35.90336227416992,
      "activations/layer14_attention_weight_min": -29.907405853271484,
      "activations/layer15_attention_weight_max": 33.242271423339844,
      "activations/layer15_attention_weight_min": -31.103164672851562,
      "activations/layer16_attention_weight_max": 34.94374084472656,
      "activations/layer16_attention_weight_min": -32.83237075805664,
      "activations/layer17_attention_weight_max": 50.68046951293945,
      "activations/layer17_attention_weight_min": -43.55360794067383,
      "activations/layer18_attention_weight_max": 48.07598114013672,
      "activations/layer18_attention_weight_min": -41.22378158569336,
      "activations/layer19_attention_weight_max": 25.723392486572266,
      "activations/layer19_attention_weight_min": -24.515331268310547,
      "activations/layer1_attention_weight_max": 15.223173141479492,
      "activations/layer1_attention_weight_min": -12.534233093261719,
      "activations/layer20_attention_weight_max": 23.358810424804688,
      "activations/layer20_attention_weight_min": -22.357175827026367,
      "activations/layer21_attention_weight_max": 36.36552810668945,
      "activations/layer21_attention_weight_min": -25.470678329467773,
      "activations/layer22_attention_weight_max": 29.006988525390625,
      "activations/layer22_attention_weight_min": -24.985891342163086,
      "activations/layer23_attention_weight_max": 38.932395935058594,
      "activations/layer23_attention_weight_min": -26.298179626464844,
      "activations/layer2_attention_weight_max": 30.988807678222656,
      "activations/layer2_attention_weight_min": -29.550031661987305,
      "activations/layer3_attention_weight_max": 84.012451171875,
      "activations/layer3_attention_weight_min": -82.72272491455078,
      "activations/layer4_attention_weight_max": 79.98257446289062,
      "activations/layer4_attention_weight_min": -85.05107879638672,
      "activations/layer5_attention_weight_max": 59.22541427612305,
      "activations/layer5_attention_weight_min": -71.34195709228516,
      "activations/layer6_attention_weight_max": 47.86525344848633,
      "activations/layer6_attention_weight_min": -50.42936706542969,
      "activations/layer7_attention_weight_max": 64.14503479003906,
      "activations/layer7_attention_weight_min": -62.71556854248047,
      "activations/layer8_attention_weight_max": 47.454498291015625,
      "activations/layer8_attention_weight_min": -52.09443664550781,
      "activations/layer9_attention_weight_max": 44.15420150756836,
      "activations/layer9_attention_weight_min": -48.70606994628906,
      "epoch": 20.52,
      "learning_rate": 1.7834469696969695e-05,
      "loss": 2.7204,
      "step": 353200
    },
    {
      "activations/layer0_attention_weight_max": 15.874788284301758,
      "activations/layer0_attention_weight_min": -14.019556999206543,
      "activations/layer10_attention_weight_max": 35.88504409790039,
      "activations/layer10_attention_weight_min": -34.57353591918945,
      "activations/layer11_attention_weight_max": 33.76744079589844,
      "activations/layer11_attention_weight_min": -35.73746109008789,
      "activations/layer12_attention_weight_max": 18.34432029724121,
      "activations/layer12_attention_weight_min": -24.04593276977539,
      "activations/layer13_attention_weight_max": 31.16092872619629,
      "activations/layer13_attention_weight_min": -28.2835636138916,
      "activations/layer14_attention_weight_max": 34.53471755981445,
      "activations/layer14_attention_weight_min": -30.752161026000977,
      "activations/layer15_attention_weight_max": 31.77018928527832,
      "activations/layer15_attention_weight_min": -29.437559127807617,
      "activations/layer16_attention_weight_max": 31.39305305480957,
      "activations/layer16_attention_weight_min": -29.11682891845703,
      "activations/layer17_attention_weight_max": 50.73147201538086,
      "activations/layer17_attention_weight_min": -41.524166107177734,
      "activations/layer18_attention_weight_max": 44.44145965576172,
      "activations/layer18_attention_weight_min": -37.61069869995117,
      "activations/layer19_attention_weight_max": 22.307126998901367,
      "activations/layer19_attention_weight_min": -21.577781677246094,
      "activations/layer1_attention_weight_max": 15.967977523803711,
      "activations/layer1_attention_weight_min": -14.535747528076172,
      "activations/layer20_attention_weight_max": 22.25020408630371,
      "activations/layer20_attention_weight_min": -24.81825828552246,
      "activations/layer21_attention_weight_max": 36.14556884765625,
      "activations/layer21_attention_weight_min": -24.3951416015625,
      "activations/layer22_attention_weight_max": 27.072134017944336,
      "activations/layer22_attention_weight_min": -24.611923217773438,
      "activations/layer23_attention_weight_max": 33.70267105102539,
      "activations/layer23_attention_weight_min": -24.481239318847656,
      "activations/layer2_attention_weight_max": 29.430740356445312,
      "activations/layer2_attention_weight_min": -30.357120513916016,
      "activations/layer3_attention_weight_max": 85.48200225830078,
      "activations/layer3_attention_weight_min": -89.29497528076172,
      "activations/layer4_attention_weight_max": 78.38542938232422,
      "activations/layer4_attention_weight_min": -80.70068359375,
      "activations/layer5_attention_weight_max": 58.10734176635742,
      "activations/layer5_attention_weight_min": -70.31136322021484,
      "activations/layer6_attention_weight_max": 46.048789978027344,
      "activations/layer6_attention_weight_min": -47.55265426635742,
      "activations/layer7_attention_weight_max": 63.84394073486328,
      "activations/layer7_attention_weight_min": -63.62486267089844,
      "activations/layer8_attention_weight_max": 47.19842529296875,
      "activations/layer8_attention_weight_min": -48.57788848876953,
      "activations/layer9_attention_weight_max": 42.410457611083984,
      "activations/layer9_attention_weight_min": -45.017799377441406,
      "epoch": 20.53,
      "learning_rate": 1.7815530303030304e-05,
      "loss": 2.723,
      "step": 353250
    },
    {
      "activations/layer0_attention_weight_max": 14.604727745056152,
      "activations/layer0_attention_weight_min": -13.567767143249512,
      "activations/layer10_attention_weight_max": 33.74945831298828,
      "activations/layer10_attention_weight_min": -34.38311767578125,
      "activations/layer11_attention_weight_max": 32.27410125732422,
      "activations/layer11_attention_weight_min": -34.98471450805664,
      "activations/layer12_attention_weight_max": 18.737735748291016,
      "activations/layer12_attention_weight_min": -28.79332733154297,
      "activations/layer13_attention_weight_max": 33.74494171142578,
      "activations/layer13_attention_weight_min": -26.32955551147461,
      "activations/layer14_attention_weight_max": 36.13502883911133,
      "activations/layer14_attention_weight_min": -31.250778198242188,
      "activations/layer15_attention_weight_max": 35.309043884277344,
      "activations/layer15_attention_weight_min": -31.034818649291992,
      "activations/layer16_attention_weight_max": 34.34730911254883,
      "activations/layer16_attention_weight_min": -31.956844329833984,
      "activations/layer17_attention_weight_max": 52.65836715698242,
      "activations/layer17_attention_weight_min": -46.46643829345703,
      "activations/layer18_attention_weight_max": 48.799137115478516,
      "activations/layer18_attention_weight_min": -42.73246383666992,
      "activations/layer19_attention_weight_max": 25.347576141357422,
      "activations/layer19_attention_weight_min": -24.527626037597656,
      "activations/layer1_attention_weight_max": 15.162299156188965,
      "activations/layer1_attention_weight_min": -13.302865982055664,
      "activations/layer20_attention_weight_max": 24.45866584777832,
      "activations/layer20_attention_weight_min": -22.17158317565918,
      "activations/layer21_attention_weight_max": 42.738433837890625,
      "activations/layer21_attention_weight_min": -23.596622467041016,
      "activations/layer22_attention_weight_max": 31.63219451904297,
      "activations/layer22_attention_weight_min": -26.04718589782715,
      "activations/layer23_attention_weight_max": 41.261783599853516,
      "activations/layer23_attention_weight_min": -24.14270782470703,
      "activations/layer2_attention_weight_max": 30.224424362182617,
      "activations/layer2_attention_weight_min": -28.879323959350586,
      "activations/layer3_attention_weight_max": 86.8345718383789,
      "activations/layer3_attention_weight_min": -87.79507446289062,
      "activations/layer4_attention_weight_max": 82.37499237060547,
      "activations/layer4_attention_weight_min": -83.07362365722656,
      "activations/layer5_attention_weight_max": 61.32733154296875,
      "activations/layer5_attention_weight_min": -69.96067810058594,
      "activations/layer6_attention_weight_max": 47.69156265258789,
      "activations/layer6_attention_weight_min": -51.376686096191406,
      "activations/layer7_attention_weight_max": 65.33995819091797,
      "activations/layer7_attention_weight_min": -63.41678237915039,
      "activations/layer8_attention_weight_max": 45.1915168762207,
      "activations/layer8_attention_weight_min": -50.6209602355957,
      "activations/layer9_attention_weight_max": 42.3555793762207,
      "activations/layer9_attention_weight_min": -45.70913314819336,
      "epoch": 20.53,
      "learning_rate": 1.779659090909091e-05,
      "loss": 2.714,
      "step": 353300
    },
    {
      "activations/layer0_attention_weight_max": 15.09028148651123,
      "activations/layer0_attention_weight_min": -14.04882526397705,
      "activations/layer10_attention_weight_max": 35.808231353759766,
      "activations/layer10_attention_weight_min": -35.680877685546875,
      "activations/layer11_attention_weight_max": 33.684478759765625,
      "activations/layer11_attention_weight_min": -36.69313049316406,
      "activations/layer12_attention_weight_max": 18.944000244140625,
      "activations/layer12_attention_weight_min": -25.082895278930664,
      "activations/layer13_attention_weight_max": 36.4648551940918,
      "activations/layer13_attention_weight_min": -28.981494903564453,
      "activations/layer14_attention_weight_max": 33.35131072998047,
      "activations/layer14_attention_weight_min": -29.877565383911133,
      "activations/layer15_attention_weight_max": 33.90419387817383,
      "activations/layer15_attention_weight_min": -29.581777572631836,
      "activations/layer16_attention_weight_max": 34.30961990356445,
      "activations/layer16_attention_weight_min": -31.208227157592773,
      "activations/layer17_attention_weight_max": 51.356258392333984,
      "activations/layer17_attention_weight_min": -44.2869987487793,
      "activations/layer18_attention_weight_max": 49.082725524902344,
      "activations/layer18_attention_weight_min": -41.93279266357422,
      "activations/layer19_attention_weight_max": 24.344194412231445,
      "activations/layer19_attention_weight_min": -23.931838989257812,
      "activations/layer1_attention_weight_max": 15.211723327636719,
      "activations/layer1_attention_weight_min": -14.974413871765137,
      "activations/layer20_attention_weight_max": 23.01884651184082,
      "activations/layer20_attention_weight_min": -22.246877670288086,
      "activations/layer21_attention_weight_max": 34.52164077758789,
      "activations/layer21_attention_weight_min": -22.92566680908203,
      "activations/layer22_attention_weight_max": 28.504316329956055,
      "activations/layer22_attention_weight_min": -26.315765380859375,
      "activations/layer23_attention_weight_max": 36.492069244384766,
      "activations/layer23_attention_weight_min": -23.751537322998047,
      "activations/layer2_attention_weight_max": 31.716421127319336,
      "activations/layer2_attention_weight_min": -29.78803825378418,
      "activations/layer3_attention_weight_max": 89.62039184570312,
      "activations/layer3_attention_weight_min": -87.22695922851562,
      "activations/layer4_attention_weight_max": 85.11123657226562,
      "activations/layer4_attention_weight_min": -88.2987289428711,
      "activations/layer5_attention_weight_max": 59.49102020263672,
      "activations/layer5_attention_weight_min": -71.38459777832031,
      "activations/layer6_attention_weight_max": 47.96064376831055,
      "activations/layer6_attention_weight_min": -49.2652702331543,
      "activations/layer7_attention_weight_max": 62.251861572265625,
      "activations/layer7_attention_weight_min": -69.04714965820312,
      "activations/layer8_attention_weight_max": 47.2772216796875,
      "activations/layer8_attention_weight_min": -51.38508224487305,
      "activations/layer9_attention_weight_max": 43.72999572753906,
      "activations/layer9_attention_weight_min": -46.58488464355469,
      "epoch": 20.53,
      "learning_rate": 1.7777651515151513e-05,
      "loss": 2.7195,
      "step": 353350
    },
    {
      "activations/layer0_attention_weight_max": 15.520634651184082,
      "activations/layer0_attention_weight_min": -13.687906265258789,
      "activations/layer10_attention_weight_max": 45.859615325927734,
      "activations/layer10_attention_weight_min": -44.037353515625,
      "activations/layer11_attention_weight_max": 45.281856536865234,
      "activations/layer11_attention_weight_min": -42.24652099609375,
      "activations/layer12_attention_weight_max": 20.464752197265625,
      "activations/layer12_attention_weight_min": -24.226531982421875,
      "activations/layer13_attention_weight_max": 50.999027252197266,
      "activations/layer13_attention_weight_min": -30.09847068786621,
      "activations/layer14_attention_weight_max": 47.71712875366211,
      "activations/layer14_attention_weight_min": -33.47022247314453,
      "activations/layer15_attention_weight_max": 46.0984992980957,
      "activations/layer15_attention_weight_min": -32.5920524597168,
      "activations/layer16_attention_weight_max": 36.088165283203125,
      "activations/layer16_attention_weight_min": -35.01012420654297,
      "activations/layer17_attention_weight_max": 54.87126541137695,
      "activations/layer17_attention_weight_min": -50.92932891845703,
      "activations/layer18_attention_weight_max": 52.30105972290039,
      "activations/layer18_attention_weight_min": -53.04849624633789,
      "activations/layer19_attention_weight_max": 27.086746215820312,
      "activations/layer19_attention_weight_min": -27.833892822265625,
      "activations/layer1_attention_weight_max": 15.408042907714844,
      "activations/layer1_attention_weight_min": -14.855386734008789,
      "activations/layer20_attention_weight_max": 27.61055564880371,
      "activations/layer20_attention_weight_min": -25.79165267944336,
      "activations/layer21_attention_weight_max": 49.13420486450195,
      "activations/layer21_attention_weight_min": -24.917068481445312,
      "activations/layer22_attention_weight_max": 35.422752380371094,
      "activations/layer22_attention_weight_min": -31.55544090270996,
      "activations/layer23_attention_weight_max": 36.912071228027344,
      "activations/layer23_attention_weight_min": -26.37820053100586,
      "activations/layer2_attention_weight_max": 34.152122497558594,
      "activations/layer2_attention_weight_min": -33.039344787597656,
      "activations/layer3_attention_weight_max": 94.44599914550781,
      "activations/layer3_attention_weight_min": -93.02193450927734,
      "activations/layer4_attention_weight_max": 97.88981628417969,
      "activations/layer4_attention_weight_min": -91.68629455566406,
      "activations/layer5_attention_weight_max": 69.68539428710938,
      "activations/layer5_attention_weight_min": -68.93476867675781,
      "activations/layer6_attention_weight_max": 53.79328536987305,
      "activations/layer6_attention_weight_min": -51.90235900878906,
      "activations/layer7_attention_weight_max": 77.07404327392578,
      "activations/layer7_attention_weight_min": -69.82808685302734,
      "activations/layer8_attention_weight_max": 57.72249984741211,
      "activations/layer8_attention_weight_min": -55.0872688293457,
      "activations/layer9_attention_weight_max": 58.82117462158203,
      "activations/layer9_attention_weight_min": -51.811737060546875,
      "epoch": 20.53,
      "learning_rate": 1.7758712121212122e-05,
      "loss": 2.7186,
      "step": 353400
    },
    {
      "activations/layer0_attention_weight_max": 15.527729988098145,
      "activations/layer0_attention_weight_min": -13.684220314025879,
      "activations/layer10_attention_weight_max": 34.94551086425781,
      "activations/layer10_attention_weight_min": -34.928836822509766,
      "activations/layer11_attention_weight_max": 32.90663528442383,
      "activations/layer11_attention_weight_min": -34.529510498046875,
      "activations/layer12_attention_weight_max": 18.86471939086914,
      "activations/layer12_attention_weight_min": -25.723499298095703,
      "activations/layer13_attention_weight_max": 32.774986267089844,
      "activations/layer13_attention_weight_min": -28.122941970825195,
      "activations/layer14_attention_weight_max": 35.89536666870117,
      "activations/layer14_attention_weight_min": -31.961013793945312,
      "activations/layer15_attention_weight_max": 34.805850982666016,
      "activations/layer15_attention_weight_min": -32.128944396972656,
      "activations/layer16_attention_weight_max": 37.3937873840332,
      "activations/layer16_attention_weight_min": -35.51337432861328,
      "activations/layer17_attention_weight_max": 54.46002197265625,
      "activations/layer17_attention_weight_min": -47.64936447143555,
      "activations/layer18_attention_weight_max": 53.71990203857422,
      "activations/layer18_attention_weight_min": -44.36738967895508,
      "activations/layer19_attention_weight_max": 25.803625106811523,
      "activations/layer19_attention_weight_min": -24.202844619750977,
      "activations/layer1_attention_weight_max": 16.12224769592285,
      "activations/layer1_attention_weight_min": -14.235004425048828,
      "activations/layer20_attention_weight_max": 26.948989868164062,
      "activations/layer20_attention_weight_min": -20.662912368774414,
      "activations/layer21_attention_weight_max": 37.69306182861328,
      "activations/layer21_attention_weight_min": -24.381120681762695,
      "activations/layer22_attention_weight_max": 31.492843627929688,
      "activations/layer22_attention_weight_min": -26.268354415893555,
      "activations/layer23_attention_weight_max": 39.92632293701172,
      "activations/layer23_attention_weight_min": -24.94172477722168,
      "activations/layer2_attention_weight_max": 30.163145065307617,
      "activations/layer2_attention_weight_min": -29.5810546875,
      "activations/layer3_attention_weight_max": 85.24250793457031,
      "activations/layer3_attention_weight_min": -85.60688018798828,
      "activations/layer4_attention_weight_max": 81.73283386230469,
      "activations/layer4_attention_weight_min": -82.48507690429688,
      "activations/layer5_attention_weight_max": 60.7605094909668,
      "activations/layer5_attention_weight_min": -75.93954467773438,
      "activations/layer6_attention_weight_max": 48.30039596557617,
      "activations/layer6_attention_weight_min": -52.36811828613281,
      "activations/layer7_attention_weight_max": 66.22077178955078,
      "activations/layer7_attention_weight_min": -62.582664489746094,
      "activations/layer8_attention_weight_max": 46.155235290527344,
      "activations/layer8_attention_weight_min": -51.002464294433594,
      "activations/layer9_attention_weight_max": 43.021114349365234,
      "activations/layer9_attention_weight_min": -44.74224853515625,
      "epoch": 20.54,
      "learning_rate": 1.7739772727272727e-05,
      "loss": 2.7237,
      "step": 353450
    },
    {
      "activations/layer0_attention_weight_max": 15.591361045837402,
      "activations/layer0_attention_weight_min": -13.72374439239502,
      "activations/layer10_attention_weight_max": 34.06613540649414,
      "activations/layer10_attention_weight_min": -34.19587326049805,
      "activations/layer11_attention_weight_max": 32.3118896484375,
      "activations/layer11_attention_weight_min": -34.52714920043945,
      "activations/layer12_attention_weight_max": 20.67334747314453,
      "activations/layer12_attention_weight_min": -23.7316951751709,
      "activations/layer13_attention_weight_max": 34.22804260253906,
      "activations/layer13_attention_weight_min": -26.70538902282715,
      "activations/layer14_attention_weight_max": 36.58762741088867,
      "activations/layer14_attention_weight_min": -30.220834732055664,
      "activations/layer15_attention_weight_max": 33.79729461669922,
      "activations/layer15_attention_weight_min": -30.8709659576416,
      "activations/layer16_attention_weight_max": 33.79890060424805,
      "activations/layer16_attention_weight_min": -32.30425262451172,
      "activations/layer17_attention_weight_max": 52.074615478515625,
      "activations/layer17_attention_weight_min": -47.20582962036133,
      "activations/layer18_attention_weight_max": 51.02743911743164,
      "activations/layer18_attention_weight_min": -41.338016510009766,
      "activations/layer19_attention_weight_max": 28.109922409057617,
      "activations/layer19_attention_weight_min": -24.931974411010742,
      "activations/layer1_attention_weight_max": 14.94714641571045,
      "activations/layer1_attention_weight_min": -14.59289264678955,
      "activations/layer20_attention_weight_max": 23.933006286621094,
      "activations/layer20_attention_weight_min": -23.31244468688965,
      "activations/layer21_attention_weight_max": 35.59275436401367,
      "activations/layer21_attention_weight_min": -22.90757179260254,
      "activations/layer22_attention_weight_max": 30.849782943725586,
      "activations/layer22_attention_weight_min": -25.212549209594727,
      "activations/layer23_attention_weight_max": 39.56317901611328,
      "activations/layer23_attention_weight_min": -23.152029037475586,
      "activations/layer2_attention_weight_max": 30.113994598388672,
      "activations/layer2_attention_weight_min": -30.162586212158203,
      "activations/layer3_attention_weight_max": 83.90184783935547,
      "activations/layer3_attention_weight_min": -87.83548736572266,
      "activations/layer4_attention_weight_max": 81.64397430419922,
      "activations/layer4_attention_weight_min": -86.46379852294922,
      "activations/layer5_attention_weight_max": 65.85015869140625,
      "activations/layer5_attention_weight_min": -70.54309844970703,
      "activations/layer6_attention_weight_max": 48.86235809326172,
      "activations/layer6_attention_weight_min": -48.973018646240234,
      "activations/layer7_attention_weight_max": 62.0783805847168,
      "activations/layer7_attention_weight_min": -61.844478607177734,
      "activations/layer8_attention_weight_max": 44.58135986328125,
      "activations/layer8_attention_weight_min": -48.6707649230957,
      "activations/layer9_attention_weight_max": 41.99729537963867,
      "activations/layer9_attention_weight_min": -44.94456481933594,
      "epoch": 20.54,
      "learning_rate": 1.772083333333333e-05,
      "loss": 2.7201,
      "step": 353500
    },
    {
      "activations/layer0_attention_weight_max": 15.541059494018555,
      "activations/layer0_attention_weight_min": -14.137836456298828,
      "activations/layer10_attention_weight_max": 36.371551513671875,
      "activations/layer10_attention_weight_min": -34.99081039428711,
      "activations/layer11_attention_weight_max": 33.3662109375,
      "activations/layer11_attention_weight_min": -33.694313049316406,
      "activations/layer12_attention_weight_max": 18.54454803466797,
      "activations/layer12_attention_weight_min": -34.91425704956055,
      "activations/layer13_attention_weight_max": 33.18449020385742,
      "activations/layer13_attention_weight_min": -27.881750106811523,
      "activations/layer14_attention_weight_max": 33.117103576660156,
      "activations/layer14_attention_weight_min": -29.057832717895508,
      "activations/layer15_attention_weight_max": 33.22954177856445,
      "activations/layer15_attention_weight_min": -30.12681007385254,
      "activations/layer16_attention_weight_max": 33.81178283691406,
      "activations/layer16_attention_weight_min": -31.366544723510742,
      "activations/layer17_attention_weight_max": 47.41037368774414,
      "activations/layer17_attention_weight_min": -43.5527229309082,
      "activations/layer18_attention_weight_max": 44.73497009277344,
      "activations/layer18_attention_weight_min": -39.00110626220703,
      "activations/layer19_attention_weight_max": 22.440271377563477,
      "activations/layer19_attention_weight_min": -23.238489151000977,
      "activations/layer1_attention_weight_max": 15.488903045654297,
      "activations/layer1_attention_weight_min": -12.998832702636719,
      "activations/layer20_attention_weight_max": 22.620800018310547,
      "activations/layer20_attention_weight_min": -21.02157974243164,
      "activations/layer21_attention_weight_max": 41.616729736328125,
      "activations/layer21_attention_weight_min": -22.110841751098633,
      "activations/layer22_attention_weight_max": 29.13641357421875,
      "activations/layer22_attention_weight_min": -25.03447151184082,
      "activations/layer23_attention_weight_max": 35.06635284423828,
      "activations/layer23_attention_weight_min": -22.77597427368164,
      "activations/layer2_attention_weight_max": 30.34345054626465,
      "activations/layer2_attention_weight_min": -29.11365509033203,
      "activations/layer3_attention_weight_max": 83.7271957397461,
      "activations/layer3_attention_weight_min": -83.3182373046875,
      "activations/layer4_attention_weight_max": 79.95948028564453,
      "activations/layer4_attention_weight_min": -81.73223876953125,
      "activations/layer5_attention_weight_max": 57.003135681152344,
      "activations/layer5_attention_weight_min": -77.15138244628906,
      "activations/layer6_attention_weight_max": 46.185935974121094,
      "activations/layer6_attention_weight_min": -49.78925323486328,
      "activations/layer7_attention_weight_max": 62.72932052612305,
      "activations/layer7_attention_weight_min": -63.90059280395508,
      "activations/layer8_attention_weight_max": 43.58778762817383,
      "activations/layer8_attention_weight_min": -47.163108825683594,
      "activations/layer9_attention_weight_max": 44.53415298461914,
      "activations/layer9_attention_weight_min": -44.91419219970703,
      "epoch": 20.54,
      "learning_rate": 1.7701893939393936e-05,
      "loss": 2.7211,
      "step": 353550
    },
    {
      "activations/layer0_attention_weight_max": 15.640036582946777,
      "activations/layer0_attention_weight_min": -14.085053443908691,
      "activations/layer10_attention_weight_max": 36.51730728149414,
      "activations/layer10_attention_weight_min": -37.23665237426758,
      "activations/layer11_attention_weight_max": 33.83167266845703,
      "activations/layer11_attention_weight_min": -35.54161834716797,
      "activations/layer12_attention_weight_max": 20.172107696533203,
      "activations/layer12_attention_weight_min": -29.39121437072754,
      "activations/layer13_attention_weight_max": 34.65563201904297,
      "activations/layer13_attention_weight_min": -27.73365020751953,
      "activations/layer14_attention_weight_max": 36.24098205566406,
      "activations/layer14_attention_weight_min": -31.686717987060547,
      "activations/layer15_attention_weight_max": 34.670204162597656,
      "activations/layer15_attention_weight_min": -29.892934799194336,
      "activations/layer16_attention_weight_max": 36.56698989868164,
      "activations/layer16_attention_weight_min": -34.57681655883789,
      "activations/layer17_attention_weight_max": 57.10299301147461,
      "activations/layer17_attention_weight_min": -48.01283645629883,
      "activations/layer18_attention_weight_max": 49.328956604003906,
      "activations/layer18_attention_weight_min": -41.26947784423828,
      "activations/layer19_attention_weight_max": 25.386018753051758,
      "activations/layer19_attention_weight_min": -22.990631103515625,
      "activations/layer1_attention_weight_max": 14.919034957885742,
      "activations/layer1_attention_weight_min": -13.641307830810547,
      "activations/layer20_attention_weight_max": 23.13526725769043,
      "activations/layer20_attention_weight_min": -22.582353591918945,
      "activations/layer21_attention_weight_max": 37.10436248779297,
      "activations/layer21_attention_weight_min": -22.139785766601562,
      "activations/layer22_attention_weight_max": 30.75400161743164,
      "activations/layer22_attention_weight_min": -26.258394241333008,
      "activations/layer23_attention_weight_max": 38.105201721191406,
      "activations/layer23_attention_weight_min": -24.515348434448242,
      "activations/layer2_attention_weight_max": 28.031951904296875,
      "activations/layer2_attention_weight_min": -27.562583923339844,
      "activations/layer3_attention_weight_max": 82.78700256347656,
      "activations/layer3_attention_weight_min": -80.80841827392578,
      "activations/layer4_attention_weight_max": 78.04960632324219,
      "activations/layer4_attention_weight_min": -78.9354476928711,
      "activations/layer5_attention_weight_max": 58.61576461791992,
      "activations/layer5_attention_weight_min": -67.2629165649414,
      "activations/layer6_attention_weight_max": 46.51583480834961,
      "activations/layer6_attention_weight_min": -48.99421310424805,
      "activations/layer7_attention_weight_max": 65.5570068359375,
      "activations/layer7_attention_weight_min": -62.00844192504883,
      "activations/layer8_attention_weight_max": 45.671241760253906,
      "activations/layer8_attention_weight_min": -50.961570739746094,
      "activations/layer9_attention_weight_max": 43.9737434387207,
      "activations/layer9_attention_weight_min": -47.00082015991211,
      "epoch": 20.55,
      "learning_rate": 1.7682954545454545e-05,
      "loss": 2.718,
      "step": 353600
    },
    {
      "activations/layer0_attention_weight_max": 14.947734832763672,
      "activations/layer0_attention_weight_min": -13.867524147033691,
      "activations/layer10_attention_weight_max": 37.64693832397461,
      "activations/layer10_attention_weight_min": -36.29628372192383,
      "activations/layer11_attention_weight_max": 34.60778045654297,
      "activations/layer11_attention_weight_min": -34.35260772705078,
      "activations/layer12_attention_weight_max": 19.602096557617188,
      "activations/layer12_attention_weight_min": -24.21442222595215,
      "activations/layer13_attention_weight_max": 30.688451766967773,
      "activations/layer13_attention_weight_min": -28.62506675720215,
      "activations/layer14_attention_weight_max": 36.2309684753418,
      "activations/layer14_attention_weight_min": -33.612144470214844,
      "activations/layer15_attention_weight_max": 32.58438491821289,
      "activations/layer15_attention_weight_min": -32.37273025512695,
      "activations/layer16_attention_weight_max": 36.97359085083008,
      "activations/layer16_attention_weight_min": -36.575260162353516,
      "activations/layer17_attention_weight_max": 58.97834014892578,
      "activations/layer17_attention_weight_min": -53.591609954833984,
      "activations/layer18_attention_weight_max": 52.043155670166016,
      "activations/layer18_attention_weight_min": -46.26865005493164,
      "activations/layer19_attention_weight_max": 24.12081527709961,
      "activations/layer19_attention_weight_min": -25.877408981323242,
      "activations/layer1_attention_weight_max": 15.777203559875488,
      "activations/layer1_attention_weight_min": -14.694534301757812,
      "activations/layer20_attention_weight_max": 23.19743537902832,
      "activations/layer20_attention_weight_min": -21.78482437133789,
      "activations/layer21_attention_weight_max": 37.550201416015625,
      "activations/layer21_attention_weight_min": -22.374731063842773,
      "activations/layer22_attention_weight_max": 30.564992904663086,
      "activations/layer22_attention_weight_min": -25.74498748779297,
      "activations/layer23_attention_weight_max": 35.9645881652832,
      "activations/layer23_attention_weight_min": -23.84842300415039,
      "activations/layer2_attention_weight_max": 30.438383102416992,
      "activations/layer2_attention_weight_min": -29.102924346923828,
      "activations/layer3_attention_weight_max": 88.43101501464844,
      "activations/layer3_attention_weight_min": -88.37866973876953,
      "activations/layer4_attention_weight_max": 87.46310424804688,
      "activations/layer4_attention_weight_min": -84.06990814208984,
      "activations/layer5_attention_weight_max": 63.34314727783203,
      "activations/layer5_attention_weight_min": -75.14215087890625,
      "activations/layer6_attention_weight_max": 49.2903938293457,
      "activations/layer6_attention_weight_min": -50.439613342285156,
      "activations/layer7_attention_weight_max": 67.88609313964844,
      "activations/layer7_attention_weight_min": -69.29907989501953,
      "activations/layer8_attention_weight_max": 49.159454345703125,
      "activations/layer8_attention_weight_min": -54.44674301147461,
      "activations/layer9_attention_weight_max": 45.336788177490234,
      "activations/layer9_attention_weight_min": -50.47612762451172,
      "epoch": 20.55,
      "learning_rate": 1.766401515151515e-05,
      "loss": 2.7356,
      "step": 353650
    },
    {
      "activations/layer0_attention_weight_max": 14.955127716064453,
      "activations/layer0_attention_weight_min": -14.489814758300781,
      "activations/layer10_attention_weight_max": 35.045204162597656,
      "activations/layer10_attention_weight_min": -34.56147766113281,
      "activations/layer11_attention_weight_max": 31.586097717285156,
      "activations/layer11_attention_weight_min": -34.93501281738281,
      "activations/layer12_attention_weight_max": 18.999691009521484,
      "activations/layer12_attention_weight_min": -27.3518009185791,
      "activations/layer13_attention_weight_max": 33.96985626220703,
      "activations/layer13_attention_weight_min": -28.840923309326172,
      "activations/layer14_attention_weight_max": 35.88752746582031,
      "activations/layer14_attention_weight_min": -33.123958587646484,
      "activations/layer15_attention_weight_max": 34.42048645019531,
      "activations/layer15_attention_weight_min": -32.41408157348633,
      "activations/layer16_attention_weight_max": 35.75955581665039,
      "activations/layer16_attention_weight_min": -35.72039031982422,
      "activations/layer17_attention_weight_max": 56.2912483215332,
      "activations/layer17_attention_weight_min": -47.34798049926758,
      "activations/layer18_attention_weight_max": 50.281314849853516,
      "activations/layer18_attention_weight_min": -44.66411209106445,
      "activations/layer19_attention_weight_max": 25.448881149291992,
      "activations/layer19_attention_weight_min": -24.162683486938477,
      "activations/layer1_attention_weight_max": 16.199602127075195,
      "activations/layer1_attention_weight_min": -14.508334159851074,
      "activations/layer20_attention_weight_max": 23.48381233215332,
      "activations/layer20_attention_weight_min": -20.883384704589844,
      "activations/layer21_attention_weight_max": 37.23675537109375,
      "activations/layer21_attention_weight_min": -22.17669677734375,
      "activations/layer22_attention_weight_max": 29.42424201965332,
      "activations/layer22_attention_weight_min": -26.730804443359375,
      "activations/layer23_attention_weight_max": 34.623016357421875,
      "activations/layer23_attention_weight_min": -23.784719467163086,
      "activations/layer2_attention_weight_max": 30.84283447265625,
      "activations/layer2_attention_weight_min": -29.885852813720703,
      "activations/layer3_attention_weight_max": 86.96769714355469,
      "activations/layer3_attention_weight_min": -86.7785415649414,
      "activations/layer4_attention_weight_max": 83.06160736083984,
      "activations/layer4_attention_weight_min": -82.8676986694336,
      "activations/layer5_attention_weight_max": 60.64136505126953,
      "activations/layer5_attention_weight_min": -74.71553039550781,
      "activations/layer6_attention_weight_max": 50.300228118896484,
      "activations/layer6_attention_weight_min": -50.94053649902344,
      "activations/layer7_attention_weight_max": 65.16197967529297,
      "activations/layer7_attention_weight_min": -64.22881317138672,
      "activations/layer8_attention_weight_max": 48.21937942504883,
      "activations/layer8_attention_weight_min": -49.699623107910156,
      "activations/layer9_attention_weight_max": 43.600582122802734,
      "activations/layer9_attention_weight_min": -46.13231658935547,
      "epoch": 20.55,
      "learning_rate": 1.7645075757575754e-05,
      "loss": 2.7315,
      "step": 353700
    },
    {
      "activations/layer0_attention_weight_max": 15.141548156738281,
      "activations/layer0_attention_weight_min": -13.699556350708008,
      "activations/layer10_attention_weight_max": 35.24245834350586,
      "activations/layer10_attention_weight_min": -35.96006393432617,
      "activations/layer11_attention_weight_max": 33.30233383178711,
      "activations/layer11_attention_weight_min": -34.6972770690918,
      "activations/layer12_attention_weight_max": 18.723997116088867,
      "activations/layer12_attention_weight_min": -25.97635269165039,
      "activations/layer13_attention_weight_max": 34.76141357421875,
      "activations/layer13_attention_weight_min": -26.55915069580078,
      "activations/layer14_attention_weight_max": 36.120567321777344,
      "activations/layer14_attention_weight_min": -29.028779983520508,
      "activations/layer15_attention_weight_max": 35.393638610839844,
      "activations/layer15_attention_weight_min": -28.662954330444336,
      "activations/layer16_attention_weight_max": 37.84733581542969,
      "activations/layer16_attention_weight_min": -34.87710952758789,
      "activations/layer17_attention_weight_max": 56.0858154296875,
      "activations/layer17_attention_weight_min": -45.98887252807617,
      "activations/layer18_attention_weight_max": 51.26103591918945,
      "activations/layer18_attention_weight_min": -40.9914665222168,
      "activations/layer19_attention_weight_max": 26.880756378173828,
      "activations/layer19_attention_weight_min": -21.997709274291992,
      "activations/layer1_attention_weight_max": 15.877069473266602,
      "activations/layer1_attention_weight_min": -14.98072338104248,
      "activations/layer20_attention_weight_max": 25.08501625061035,
      "activations/layer20_attention_weight_min": -21.348203659057617,
      "activations/layer21_attention_weight_max": 34.77724838256836,
      "activations/layer21_attention_weight_min": -20.432762145996094,
      "activations/layer22_attention_weight_max": 29.472248077392578,
      "activations/layer22_attention_weight_min": -26.022417068481445,
      "activations/layer23_attention_weight_max": 38.44202423095703,
      "activations/layer23_attention_weight_min": -24.74951171875,
      "activations/layer2_attention_weight_max": 31.76155662536621,
      "activations/layer2_attention_weight_min": -31.137449264526367,
      "activations/layer3_attention_weight_max": 89.34542083740234,
      "activations/layer3_attention_weight_min": -89.49739837646484,
      "activations/layer4_attention_weight_max": 84.42540740966797,
      "activations/layer4_attention_weight_min": -83.02440643310547,
      "activations/layer5_attention_weight_max": 59.75641632080078,
      "activations/layer5_attention_weight_min": -69.17977142333984,
      "activations/layer6_attention_weight_max": 46.893856048583984,
      "activations/layer6_attention_weight_min": -50.3360481262207,
      "activations/layer7_attention_weight_max": 63.31356430053711,
      "activations/layer7_attention_weight_min": -67.54962158203125,
      "activations/layer8_attention_weight_max": 49.47626876831055,
      "activations/layer8_attention_weight_min": -51.7515754699707,
      "activations/layer9_attention_weight_max": 43.63379669189453,
      "activations/layer9_attention_weight_min": -43.660640716552734,
      "epoch": 20.55,
      "learning_rate": 1.7626136363636363e-05,
      "loss": 2.7094,
      "step": 353750
    },
    {
      "activations/layer0_attention_weight_max": 16.49854850769043,
      "activations/layer0_attention_weight_min": -14.347268104553223,
      "activations/layer10_attention_weight_max": 34.97299575805664,
      "activations/layer10_attention_weight_min": -33.94623947143555,
      "activations/layer11_attention_weight_max": 31.856231689453125,
      "activations/layer11_attention_weight_min": -34.22308349609375,
      "activations/layer12_attention_weight_max": 18.675373077392578,
      "activations/layer12_attention_weight_min": -25.578184127807617,
      "activations/layer13_attention_weight_max": 42.37702178955078,
      "activations/layer13_attention_weight_min": -29.145742416381836,
      "activations/layer14_attention_weight_max": 33.76348876953125,
      "activations/layer14_attention_weight_min": -30.619709014892578,
      "activations/layer15_attention_weight_max": 31.468341827392578,
      "activations/layer15_attention_weight_min": -30.538589477539062,
      "activations/layer16_attention_weight_max": 31.725955963134766,
      "activations/layer16_attention_weight_min": -30.3370361328125,
      "activations/layer17_attention_weight_max": 49.67427444458008,
      "activations/layer17_attention_weight_min": -42.58184051513672,
      "activations/layer18_attention_weight_max": 43.820621490478516,
      "activations/layer18_attention_weight_min": -38.5831298828125,
      "activations/layer19_attention_weight_max": 23.924863815307617,
      "activations/layer19_attention_weight_min": -21.725751876831055,
      "activations/layer1_attention_weight_max": 15.664334297180176,
      "activations/layer1_attention_weight_min": -14.008431434631348,
      "activations/layer20_attention_weight_max": 19.562055587768555,
      "activations/layer20_attention_weight_min": -22.20213508605957,
      "activations/layer21_attention_weight_max": 30.316390991210938,
      "activations/layer21_attention_weight_min": -20.254690170288086,
      "activations/layer22_attention_weight_max": 25.01956558227539,
      "activations/layer22_attention_weight_min": -23.72657585144043,
      "activations/layer23_attention_weight_max": 34.043270111083984,
      "activations/layer23_attention_weight_min": -21.654937744140625,
      "activations/layer2_attention_weight_max": 30.571380615234375,
      "activations/layer2_attention_weight_min": -28.77054214477539,
      "activations/layer3_attention_weight_max": 86.18180847167969,
      "activations/layer3_attention_weight_min": -85.37400817871094,
      "activations/layer4_attention_weight_max": 82.68634796142578,
      "activations/layer4_attention_weight_min": -78.0204086303711,
      "activations/layer5_attention_weight_max": 63.80736541748047,
      "activations/layer5_attention_weight_min": -64.99250793457031,
      "activations/layer6_attention_weight_max": 46.63809585571289,
      "activations/layer6_attention_weight_min": -50.004337310791016,
      "activations/layer7_attention_weight_max": 64.81398010253906,
      "activations/layer7_attention_weight_min": -63.02254867553711,
      "activations/layer8_attention_weight_max": 48.4704704284668,
      "activations/layer8_attention_weight_min": -54.9608039855957,
      "activations/layer9_attention_weight_max": 44.91504669189453,
      "activations/layer9_attention_weight_min": -44.871089935302734,
      "epoch": 20.56,
      "learning_rate": 1.7607196969696968e-05,
      "loss": 2.7148,
      "step": 353800
    },
    {
      "activations/layer0_attention_weight_max": 15.41728401184082,
      "activations/layer0_attention_weight_min": -13.928511619567871,
      "activations/layer10_attention_weight_max": 36.12242126464844,
      "activations/layer10_attention_weight_min": -34.757965087890625,
      "activations/layer11_attention_weight_max": 33.365196228027344,
      "activations/layer11_attention_weight_min": -33.89448928833008,
      "activations/layer12_attention_weight_max": 19.98133659362793,
      "activations/layer12_attention_weight_min": -24.003192901611328,
      "activations/layer13_attention_weight_max": 39.77533721923828,
      "activations/layer13_attention_weight_min": -28.131446838378906,
      "activations/layer14_attention_weight_max": 36.44414138793945,
      "activations/layer14_attention_weight_min": -30.87812614440918,
      "activations/layer15_attention_weight_max": 35.30234146118164,
      "activations/layer15_attention_weight_min": -32.38312530517578,
      "activations/layer16_attention_weight_max": 39.05427169799805,
      "activations/layer16_attention_weight_min": -37.363075256347656,
      "activations/layer17_attention_weight_max": 57.83806228637695,
      "activations/layer17_attention_weight_min": -54.23365020751953,
      "activations/layer18_attention_weight_max": 51.468955993652344,
      "activations/layer18_attention_weight_min": -47.0056037902832,
      "activations/layer19_attention_weight_max": 23.73586654663086,
      "activations/layer19_attention_weight_min": -24.4067440032959,
      "activations/layer1_attention_weight_max": 15.597939491271973,
      "activations/layer1_attention_weight_min": -14.189559936523438,
      "activations/layer20_attention_weight_max": 23.189834594726562,
      "activations/layer20_attention_weight_min": -22.837013244628906,
      "activations/layer21_attention_weight_max": 36.39234161376953,
      "activations/layer21_attention_weight_min": -23.773550033569336,
      "activations/layer22_attention_weight_max": 29.147336959838867,
      "activations/layer22_attention_weight_min": -26.46047592163086,
      "activations/layer23_attention_weight_max": 35.33927917480469,
      "activations/layer23_attention_weight_min": -23.44508934020996,
      "activations/layer2_attention_weight_max": 29.678123474121094,
      "activations/layer2_attention_weight_min": -28.07117462158203,
      "activations/layer3_attention_weight_max": 82.89508819580078,
      "activations/layer3_attention_weight_min": -80.05675506591797,
      "activations/layer4_attention_weight_max": 79.67713928222656,
      "activations/layer4_attention_weight_min": -77.95958709716797,
      "activations/layer5_attention_weight_max": 59.79292297363281,
      "activations/layer5_attention_weight_min": -69.69285583496094,
      "activations/layer6_attention_weight_max": 47.82810592651367,
      "activations/layer6_attention_weight_min": -48.31632995605469,
      "activations/layer7_attention_weight_max": 65.29790496826172,
      "activations/layer7_attention_weight_min": -61.512489318847656,
      "activations/layer8_attention_weight_max": 45.65139389038086,
      "activations/layer8_attention_weight_min": -48.37830352783203,
      "activations/layer9_attention_weight_max": 43.326141357421875,
      "activations/layer9_attention_weight_min": -43.34054946899414,
      "epoch": 20.56,
      "learning_rate": 1.7588257575757572e-05,
      "loss": 2.7191,
      "step": 353850
    },
    {
      "activations/layer0_attention_weight_max": 15.409534454345703,
      "activations/layer0_attention_weight_min": -13.86929702758789,
      "activations/layer10_attention_weight_max": 36.37342071533203,
      "activations/layer10_attention_weight_min": -35.896812438964844,
      "activations/layer11_attention_weight_max": 36.497596740722656,
      "activations/layer11_attention_weight_min": -35.47962951660156,
      "activations/layer12_attention_weight_max": 20.06019401550293,
      "activations/layer12_attention_weight_min": -27.8643856048584,
      "activations/layer13_attention_weight_max": 34.94976043701172,
      "activations/layer13_attention_weight_min": -28.426998138427734,
      "activations/layer14_attention_weight_max": 37.60993576049805,
      "activations/layer14_attention_weight_min": -34.1744499206543,
      "activations/layer15_attention_weight_max": 37.029964447021484,
      "activations/layer15_attention_weight_min": -32.543949127197266,
      "activations/layer16_attention_weight_max": 36.5830192565918,
      "activations/layer16_attention_weight_min": -34.71670913696289,
      "activations/layer17_attention_weight_max": 52.99742889404297,
      "activations/layer17_attention_weight_min": -47.30701446533203,
      "activations/layer18_attention_weight_max": 47.909034729003906,
      "activations/layer18_attention_weight_min": -40.48717498779297,
      "activations/layer19_attention_weight_max": 26.9999942779541,
      "activations/layer19_attention_weight_min": -22.76350212097168,
      "activations/layer1_attention_weight_max": 15.458976745605469,
      "activations/layer1_attention_weight_min": -13.505911827087402,
      "activations/layer20_attention_weight_max": 27.85162353515625,
      "activations/layer20_attention_weight_min": -22.744718551635742,
      "activations/layer21_attention_weight_max": 45.70654296875,
      "activations/layer21_attention_weight_min": -23.802898406982422,
      "activations/layer22_attention_weight_max": 33.822383880615234,
      "activations/layer22_attention_weight_min": -27.140745162963867,
      "activations/layer23_attention_weight_max": 43.62849426269531,
      "activations/layer23_attention_weight_min": -24.021121978759766,
      "activations/layer2_attention_weight_max": 32.20954513549805,
      "activations/layer2_attention_weight_min": -28.913681030273438,
      "activations/layer3_attention_weight_max": 85.463623046875,
      "activations/layer3_attention_weight_min": -83.32109832763672,
      "activations/layer4_attention_weight_max": 84.94485473632812,
      "activations/layer4_attention_weight_min": -82.4394760131836,
      "activations/layer5_attention_weight_max": 59.09005355834961,
      "activations/layer5_attention_weight_min": -72.79122161865234,
      "activations/layer6_attention_weight_max": 47.69731521606445,
      "activations/layer6_attention_weight_min": -51.25850296020508,
      "activations/layer7_attention_weight_max": 64.95879364013672,
      "activations/layer7_attention_weight_min": -66.03071594238281,
      "activations/layer8_attention_weight_max": 48.23302459716797,
      "activations/layer8_attention_weight_min": -54.8900032043457,
      "activations/layer9_attention_weight_max": 47.125484466552734,
      "activations/layer9_attention_weight_min": -46.66139602661133,
      "epoch": 20.56,
      "learning_rate": 1.756931818181818e-05,
      "loss": 2.7033,
      "step": 353900
    },
    {
      "activations/layer0_attention_weight_max": 15.615341186523438,
      "activations/layer0_attention_weight_min": -13.954599380493164,
      "activations/layer10_attention_weight_max": 39.45840072631836,
      "activations/layer10_attention_weight_min": -35.92778396606445,
      "activations/layer11_attention_weight_max": 42.463966369628906,
      "activations/layer11_attention_weight_min": -34.394039154052734,
      "activations/layer12_attention_weight_max": 18.645090103149414,
      "activations/layer12_attention_weight_min": -26.463241577148438,
      "activations/layer13_attention_weight_max": 31.582286834716797,
      "activations/layer13_attention_weight_min": -27.825382232666016,
      "activations/layer14_attention_weight_max": 35.2999267578125,
      "activations/layer14_attention_weight_min": -30.83042335510254,
      "activations/layer15_attention_weight_max": 35.07399368286133,
      "activations/layer15_attention_weight_min": -31.575889587402344,
      "activations/layer16_attention_weight_max": 37.42171096801758,
      "activations/layer16_attention_weight_min": -35.105918884277344,
      "activations/layer17_attention_weight_max": 57.82821273803711,
      "activations/layer17_attention_weight_min": -48.360015869140625,
      "activations/layer18_attention_weight_max": 51.43785095214844,
      "activations/layer18_attention_weight_min": -45.59532165527344,
      "activations/layer19_attention_weight_max": 25.722427368164062,
      "activations/layer19_attention_weight_min": -26.156728744506836,
      "activations/layer1_attention_weight_max": 14.731801986694336,
      "activations/layer1_attention_weight_min": -13.198934555053711,
      "activations/layer20_attention_weight_max": 22.60587501525879,
      "activations/layer20_attention_weight_min": -22.963224411010742,
      "activations/layer21_attention_weight_max": 32.601219177246094,
      "activations/layer21_attention_weight_min": -21.480384826660156,
      "activations/layer22_attention_weight_max": 27.956220626831055,
      "activations/layer22_attention_weight_min": -26.616201400756836,
      "activations/layer23_attention_weight_max": 36.48776626586914,
      "activations/layer23_attention_weight_min": -23.709211349487305,
      "activations/layer2_attention_weight_max": 32.93220138549805,
      "activations/layer2_attention_weight_min": -30.261821746826172,
      "activations/layer3_attention_weight_max": 88.51366424560547,
      "activations/layer3_attention_weight_min": -88.75717163085938,
      "activations/layer4_attention_weight_max": 87.73184967041016,
      "activations/layer4_attention_weight_min": -84.13916778564453,
      "activations/layer5_attention_weight_max": 65.56977081298828,
      "activations/layer5_attention_weight_min": -73.66392517089844,
      "activations/layer6_attention_weight_max": 49.395103454589844,
      "activations/layer6_attention_weight_min": -49.14781188964844,
      "activations/layer7_attention_weight_max": 80.31843566894531,
      "activations/layer7_attention_weight_min": -67.78893280029297,
      "activations/layer8_attention_weight_max": 52.38991165161133,
      "activations/layer8_attention_weight_min": -49.68863296508789,
      "activations/layer9_attention_weight_max": 61.33475875854492,
      "activations/layer9_attention_weight_min": -46.81868362426758,
      "epoch": 20.57,
      "learning_rate": 1.7550378787878786e-05,
      "loss": 2.721,
      "step": 353950
    },
    {
      "activations/layer0_attention_weight_max": 15.365906715393066,
      "activations/layer0_attention_weight_min": -13.637385368347168,
      "activations/layer10_attention_weight_max": 35.50362014770508,
      "activations/layer10_attention_weight_min": -35.24748229980469,
      "activations/layer11_attention_weight_max": 31.539817810058594,
      "activations/layer11_attention_weight_min": -33.31416320800781,
      "activations/layer12_attention_weight_max": 18.838611602783203,
      "activations/layer12_attention_weight_min": -26.1120662689209,
      "activations/layer13_attention_weight_max": 34.323448181152344,
      "activations/layer13_attention_weight_min": -27.798803329467773,
      "activations/layer14_attention_weight_max": 36.78321075439453,
      "activations/layer14_attention_weight_min": -31.25023078918457,
      "activations/layer15_attention_weight_max": 34.4660758972168,
      "activations/layer15_attention_weight_min": -32.79996109008789,
      "activations/layer16_attention_weight_max": 35.378143310546875,
      "activations/layer16_attention_weight_min": -35.379783630371094,
      "activations/layer17_attention_weight_max": 50.792484283447266,
      "activations/layer17_attention_weight_min": -49.38282775878906,
      "activations/layer18_attention_weight_max": 47.53392791748047,
      "activations/layer18_attention_weight_min": -43.58405303955078,
      "activations/layer19_attention_weight_max": 26.969928741455078,
      "activations/layer19_attention_weight_min": -23.412351608276367,
      "activations/layer1_attention_weight_max": 15.044611930847168,
      "activations/layer1_attention_weight_min": -13.177488327026367,
      "activations/layer20_attention_weight_max": 23.300315856933594,
      "activations/layer20_attention_weight_min": -21.05091667175293,
      "activations/layer21_attention_weight_max": 33.88201141357422,
      "activations/layer21_attention_weight_min": -21.580707550048828,
      "activations/layer22_attention_weight_max": 30.27596092224121,
      "activations/layer22_attention_weight_min": -26.24625015258789,
      "activations/layer23_attention_weight_max": 39.211090087890625,
      "activations/layer23_attention_weight_min": -25.17432403564453,
      "activations/layer2_attention_weight_max": 31.777931213378906,
      "activations/layer2_attention_weight_min": -30.486799240112305,
      "activations/layer3_attention_weight_max": 86.5206527709961,
      "activations/layer3_attention_weight_min": -86.27652740478516,
      "activations/layer4_attention_weight_max": 85.63811492919922,
      "activations/layer4_attention_weight_min": -83.19712829589844,
      "activations/layer5_attention_weight_max": 60.34841537475586,
      "activations/layer5_attention_weight_min": -72.81044006347656,
      "activations/layer6_attention_weight_max": 49.58380889892578,
      "activations/layer6_attention_weight_min": -51.174198150634766,
      "activations/layer7_attention_weight_max": 67.5119400024414,
      "activations/layer7_attention_weight_min": -64.80558013916016,
      "activations/layer8_attention_weight_max": 47.73067092895508,
      "activations/layer8_attention_weight_min": -52.87547302246094,
      "activations/layer9_attention_weight_max": 49.23469161987305,
      "activations/layer9_attention_weight_min": -47.2066535949707,
      "epoch": 20.57,
      "learning_rate": 1.753143939393939e-05,
      "loss": 2.7102,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_loss": 2.677734375,
      "eval_runtime": 8.5828,
      "eval_samples_per_second": 500.301,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_openwebtext_loss": 2.677734375,
      "eval_openwebtext_ppl": 14.552086348683558,
      "eval_openwebtext_runtime": 8.5828,
      "eval_openwebtext_samples_per_second": 500.301,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_wikitext_loss": 2.919921875,
      "eval_wikitext_ppl": 18.539838978246124,
      "eval_wikitext_runtime": 2.0179,
      "eval_wikitext_samples_per_second": 225.978,
      "step": 354000
    },
    {
      "epoch": 20.57,
      "eval_lambada_loss": 2.423828125,
      "eval_lambada_ppl": 11.288992376773315,
      "eval_lambada_runtime": 9.6893,
      "eval_lambada_samples_per_second": 502.515,
      "step": 354000
    },
    {
      "activations/layer0_attention_weight_max": 15.413576126098633,
      "activations/layer0_attention_weight_min": -13.632366180419922,
      "activations/layer10_attention_weight_max": 37.64906311035156,
      "activations/layer10_attention_weight_min": -37.896278381347656,
      "activations/layer11_attention_weight_max": 33.83175277709961,
      "activations/layer11_attention_weight_min": -37.35988235473633,
      "activations/layer12_attention_weight_max": 20.136829376220703,
      "activations/layer12_attention_weight_min": -25.46209716796875,
      "activations/layer13_attention_weight_max": 33.583152770996094,
      "activations/layer13_attention_weight_min": -30.697729110717773,
      "activations/layer14_attention_weight_max": 35.28593444824219,
      "activations/layer14_attention_weight_min": -30.07628059387207,
      "activations/layer15_attention_weight_max": 33.39281463623047,
      "activations/layer15_attention_weight_min": -32.21775817871094,
      "activations/layer16_attention_weight_max": 33.89847183227539,
      "activations/layer16_attention_weight_min": -33.92148971557617,
      "activations/layer17_attention_weight_max": 52.2457275390625,
      "activations/layer17_attention_weight_min": -45.1899528503418,
      "activations/layer18_attention_weight_max": 49.73250198364258,
      "activations/layer18_attention_weight_min": -42.09025192260742,
      "activations/layer19_attention_weight_max": 24.95476531982422,
      "activations/layer19_attention_weight_min": -22.79293441772461,
      "activations/layer1_attention_weight_max": 15.543485641479492,
      "activations/layer1_attention_weight_min": -13.759308815002441,
      "activations/layer20_attention_weight_max": 23.999069213867188,
      "activations/layer20_attention_weight_min": -22.239595413208008,
      "activations/layer21_attention_weight_max": 33.664306640625,
      "activations/layer21_attention_weight_min": -22.72935676574707,
      "activations/layer22_attention_weight_max": 28.74645233154297,
      "activations/layer22_attention_weight_min": -24.5528564453125,
      "activations/layer23_attention_weight_max": 38.536678314208984,
      "activations/layer23_attention_weight_min": -26.576416015625,
      "activations/layer2_attention_weight_max": 31.753032684326172,
      "activations/layer2_attention_weight_min": -31.398433685302734,
      "activations/layer3_attention_weight_max": 86.4700927734375,
      "activations/layer3_attention_weight_min": -87.35501861572266,
      "activations/layer4_attention_weight_max": 88.3782730102539,
      "activations/layer4_attention_weight_min": -91.92161560058594,
      "activations/layer5_attention_weight_max": 65.1973876953125,
      "activations/layer5_attention_weight_min": -73.01835632324219,
      "activations/layer6_attention_weight_max": 51.14430236816406,
      "activations/layer6_attention_weight_min": -52.18138122558594,
      "activations/layer7_attention_weight_max": 69.74251556396484,
      "activations/layer7_attention_weight_min": -65.1158447265625,
      "activations/layer8_attention_weight_max": 47.420692443847656,
      "activations/layer8_attention_weight_min": -51.16229248046875,
      "activations/layer9_attention_weight_max": 45.758358001708984,
      "activations/layer9_attention_weight_min": -50.33098602294922,
      "epoch": 20.57,
      "learning_rate": 1.75125e-05,
      "loss": 2.7126,
      "step": 354050
    },
    {
      "activations/layer0_attention_weight_max": 15.857961654663086,
      "activations/layer0_attention_weight_min": -13.543134689331055,
      "activations/layer10_attention_weight_max": 33.39859390258789,
      "activations/layer10_attention_weight_min": -35.54375076293945,
      "activations/layer11_attention_weight_max": 31.87270164489746,
      "activations/layer11_attention_weight_min": -33.911293029785156,
      "activations/layer12_attention_weight_max": 19.117462158203125,
      "activations/layer12_attention_weight_min": -25.24427604675293,
      "activations/layer13_attention_weight_max": 36.1662712097168,
      "activations/layer13_attention_weight_min": -27.667421340942383,
      "activations/layer14_attention_weight_max": 33.51003646850586,
      "activations/layer14_attention_weight_min": -29.87310791015625,
      "activations/layer15_attention_weight_max": 30.814470291137695,
      "activations/layer15_attention_weight_min": -32.017948150634766,
      "activations/layer16_attention_weight_max": 33.734310150146484,
      "activations/layer16_attention_weight_min": -32.430110931396484,
      "activations/layer17_attention_weight_max": 48.43155288696289,
      "activations/layer17_attention_weight_min": -43.53043746948242,
      "activations/layer18_attention_weight_max": 42.86312484741211,
      "activations/layer18_attention_weight_min": -37.3370246887207,
      "activations/layer19_attention_weight_max": 24.183237075805664,
      "activations/layer19_attention_weight_min": -23.81954002380371,
      "activations/layer1_attention_weight_max": 15.446321487426758,
      "activations/layer1_attention_weight_min": -12.526796340942383,
      "activations/layer20_attention_weight_max": 20.949403762817383,
      "activations/layer20_attention_weight_min": -23.880962371826172,
      "activations/layer21_attention_weight_max": 36.18416213989258,
      "activations/layer21_attention_weight_min": -23.288772583007812,
      "activations/layer22_attention_weight_max": 26.576698303222656,
      "activations/layer22_attention_weight_min": -26.507762908935547,
      "activations/layer23_attention_weight_max": 36.50354766845703,
      "activations/layer23_attention_weight_min": -24.79981231689453,
      "activations/layer2_attention_weight_max": 30.025617599487305,
      "activations/layer2_attention_weight_min": -28.272634506225586,
      "activations/layer3_attention_weight_max": 80.64116668701172,
      "activations/layer3_attention_weight_min": -82.21353149414062,
      "activations/layer4_attention_weight_max": 79.50727844238281,
      "activations/layer4_attention_weight_min": -79.44226837158203,
      "activations/layer5_attention_weight_max": 59.15595626831055,
      "activations/layer5_attention_weight_min": -69.66621398925781,
      "activations/layer6_attention_weight_max": 46.42512512207031,
      "activations/layer6_attention_weight_min": -47.07061004638672,
      "activations/layer7_attention_weight_max": 61.313392639160156,
      "activations/layer7_attention_weight_min": -65.79642486572266,
      "activations/layer8_attention_weight_max": 46.102317810058594,
      "activations/layer8_attention_weight_min": -51.03895950317383,
      "activations/layer9_attention_weight_max": 42.024925231933594,
      "activations/layer9_attention_weight_min": -44.638980865478516,
      "epoch": 20.58,
      "learning_rate": 1.7493560606060607e-05,
      "loss": 2.742,
      "step": 354100
    },
    {
      "activations/layer0_attention_weight_max": 15.269232749938965,
      "activations/layer0_attention_weight_min": -14.141366958618164,
      "activations/layer10_attention_weight_max": 34.46311950683594,
      "activations/layer10_attention_weight_min": -34.95216369628906,
      "activations/layer11_attention_weight_max": 34.20358657836914,
      "activations/layer11_attention_weight_min": -33.59221267700195,
      "activations/layer12_attention_weight_max": 19.461267471313477,
      "activations/layer12_attention_weight_min": -24.73063087463379,
      "activations/layer13_attention_weight_max": 37.1245002746582,
      "activations/layer13_attention_weight_min": -25.65801429748535,
      "activations/layer14_attention_weight_max": 36.423667907714844,
      "activations/layer14_attention_weight_min": -30.96512794494629,
      "activations/layer15_attention_weight_max": 33.748992919921875,
      "activations/layer15_attention_weight_min": -31.157554626464844,
      "activations/layer16_attention_weight_max": 36.83588409423828,
      "activations/layer16_attention_weight_min": -32.45716857910156,
      "activations/layer17_attention_weight_max": 53.651397705078125,
      "activations/layer17_attention_weight_min": -47.681976318359375,
      "activations/layer18_attention_weight_max": 49.76912307739258,
      "activations/layer18_attention_weight_min": -44.27660369873047,
      "activations/layer19_attention_weight_max": 26.37540054321289,
      "activations/layer19_attention_weight_min": -25.679101943969727,
      "activations/layer1_attention_weight_max": 15.565281867980957,
      "activations/layer1_attention_weight_min": -12.970710754394531,
      "activations/layer20_attention_weight_max": 23.82246208190918,
      "activations/layer20_attention_weight_min": -25.739444732666016,
      "activations/layer21_attention_weight_max": 35.8193473815918,
      "activations/layer21_attention_weight_min": -22.74286651611328,
      "activations/layer22_attention_weight_max": 29.823503494262695,
      "activations/layer22_attention_weight_min": -28.585697174072266,
      "activations/layer23_attention_weight_max": 34.15376281738281,
      "activations/layer23_attention_weight_min": -25.632431030273438,
      "activations/layer2_attention_weight_max": 28.959096908569336,
      "activations/layer2_attention_weight_min": -27.536588668823242,
      "activations/layer3_attention_weight_max": 82.95684051513672,
      "activations/layer3_attention_weight_min": -81.65576171875,
      "activations/layer4_attention_weight_max": 79.76940155029297,
      "activations/layer4_attention_weight_min": -81.06411743164062,
      "activations/layer5_attention_weight_max": 59.0141716003418,
      "activations/layer5_attention_weight_min": -70.98981475830078,
      "activations/layer6_attention_weight_max": 48.45032501220703,
      "activations/layer6_attention_weight_min": -49.70854568481445,
      "activations/layer7_attention_weight_max": 66.03853607177734,
      "activations/layer7_attention_weight_min": -65.86830139160156,
      "activations/layer8_attention_weight_max": 46.04916000366211,
      "activations/layer8_attention_weight_min": -51.411537170410156,
      "activations/layer9_attention_weight_max": 44.20018005371094,
      "activations/layer9_attention_weight_min": -48.00659942626953,
      "epoch": 20.58,
      "learning_rate": 1.7474621212121212e-05,
      "loss": 2.7202,
      "step": 354150
    },
    {
      "activations/layer0_attention_weight_max": 15.563369750976562,
      "activations/layer0_attention_weight_min": -13.737600326538086,
      "activations/layer10_attention_weight_max": 34.81348419189453,
      "activations/layer10_attention_weight_min": -33.68697738647461,
      "activations/layer11_attention_weight_max": 32.89105224609375,
      "activations/layer11_attention_weight_min": -33.19575881958008,
      "activations/layer12_attention_weight_max": 20.425479888916016,
      "activations/layer12_attention_weight_min": -24.231359481811523,
      "activations/layer13_attention_weight_max": 33.42915344238281,
      "activations/layer13_attention_weight_min": -27.345918655395508,
      "activations/layer14_attention_weight_max": 36.58711624145508,
      "activations/layer14_attention_weight_min": -31.12104034423828,
      "activations/layer15_attention_weight_max": 33.73195266723633,
      "activations/layer15_attention_weight_min": -31.060468673706055,
      "activations/layer16_attention_weight_max": 38.181400299072266,
      "activations/layer16_attention_weight_min": -34.75897979736328,
      "activations/layer17_attention_weight_max": 57.23793411254883,
      "activations/layer17_attention_weight_min": -50.24675369262695,
      "activations/layer18_attention_weight_max": 51.952354431152344,
      "activations/layer18_attention_weight_min": -41.72772979736328,
      "activations/layer19_attention_weight_max": 24.807878494262695,
      "activations/layer19_attention_weight_min": -25.40363883972168,
      "activations/layer1_attention_weight_max": 16.488117218017578,
      "activations/layer1_attention_weight_min": -15.540197372436523,
      "activations/layer20_attention_weight_max": 21.902326583862305,
      "activations/layer20_attention_weight_min": -22.263410568237305,
      "activations/layer21_attention_weight_max": 37.956932067871094,
      "activations/layer21_attention_weight_min": -23.186159133911133,
      "activations/layer22_attention_weight_max": 29.21961212158203,
      "activations/layer22_attention_weight_min": -26.397905349731445,
      "activations/layer23_attention_weight_max": 37.465110778808594,
      "activations/layer23_attention_weight_min": -25.40529441833496,
      "activations/layer2_attention_weight_max": 30.64105796813965,
      "activations/layer2_attention_weight_min": -31.99855613708496,
      "activations/layer3_attention_weight_max": 85.0299072265625,
      "activations/layer3_attention_weight_min": -86.4669189453125,
      "activations/layer4_attention_weight_max": 83.93843841552734,
      "activations/layer4_attention_weight_min": -80.7647705078125,
      "activations/layer5_attention_weight_max": 58.6496696472168,
      "activations/layer5_attention_weight_min": -69.46153259277344,
      "activations/layer6_attention_weight_max": 47.124168395996094,
      "activations/layer6_attention_weight_min": -49.04113006591797,
      "activations/layer7_attention_weight_max": 63.649620056152344,
      "activations/layer7_attention_weight_min": -63.631622314453125,
      "activations/layer8_attention_weight_max": 47.026123046875,
      "activations/layer8_attention_weight_min": -48.34780502319336,
      "activations/layer9_attention_weight_max": 46.93521499633789,
      "activations/layer9_attention_weight_min": -45.724082946777344,
      "epoch": 20.58,
      "learning_rate": 1.7455681818181817e-05,
      "loss": 2.7206,
      "step": 354200
    },
    {
      "activations/layer0_attention_weight_max": 15.228248596191406,
      "activations/layer0_attention_weight_min": -14.08693790435791,
      "activations/layer10_attention_weight_max": 34.93805694580078,
      "activations/layer10_attention_weight_min": -35.730499267578125,
      "activations/layer11_attention_weight_max": 32.70558166503906,
      "activations/layer11_attention_weight_min": -34.08589553833008,
      "activations/layer12_attention_weight_max": 20.382421493530273,
      "activations/layer12_attention_weight_min": -24.234514236450195,
      "activations/layer13_attention_weight_max": 34.72182846069336,
      "activations/layer13_attention_weight_min": -27.87920570373535,
      "activations/layer14_attention_weight_max": 37.29743576049805,
      "activations/layer14_attention_weight_min": -30.049291610717773,
      "activations/layer15_attention_weight_max": 36.186180114746094,
      "activations/layer15_attention_weight_min": -29.801868438720703,
      "activations/layer16_attention_weight_max": 37.51313400268555,
      "activations/layer16_attention_weight_min": -33.21720504760742,
      "activations/layer17_attention_weight_max": 52.62514877319336,
      "activations/layer17_attention_weight_min": -44.715476989746094,
      "activations/layer18_attention_weight_max": 48.97758483886719,
      "activations/layer18_attention_weight_min": -40.336517333984375,
      "activations/layer19_attention_weight_max": 26.453840255737305,
      "activations/layer19_attention_weight_min": -24.680727005004883,
      "activations/layer1_attention_weight_max": 15.592853546142578,
      "activations/layer1_attention_weight_min": -14.978873252868652,
      "activations/layer20_attention_weight_max": 23.066112518310547,
      "activations/layer20_attention_weight_min": -22.296850204467773,
      "activations/layer21_attention_weight_max": 33.461246490478516,
      "activations/layer21_attention_weight_min": -22.65504264831543,
      "activations/layer22_attention_weight_max": 28.594758987426758,
      "activations/layer22_attention_weight_min": -23.157394409179688,
      "activations/layer23_attention_weight_max": 37.15612030029297,
      "activations/layer23_attention_weight_min": -23.462833404541016,
      "activations/layer2_attention_weight_max": 31.079818725585938,
      "activations/layer2_attention_weight_min": -29.617454528808594,
      "activations/layer3_attention_weight_max": 88.96935272216797,
      "activations/layer3_attention_weight_min": -86.93572998046875,
      "activations/layer4_attention_weight_max": 83.0523681640625,
      "activations/layer4_attention_weight_min": -85.08624267578125,
      "activations/layer5_attention_weight_max": 59.655147552490234,
      "activations/layer5_attention_weight_min": -69.17586517333984,
      "activations/layer6_attention_weight_max": 47.416378021240234,
      "activations/layer6_attention_weight_min": -50.1502799987793,
      "activations/layer7_attention_weight_max": 65.6693344116211,
      "activations/layer7_attention_weight_min": -63.07718276977539,
      "activations/layer8_attention_weight_max": 45.08641052246094,
      "activations/layer8_attention_weight_min": -48.413814544677734,
      "activations/layer9_attention_weight_max": 42.25535202026367,
      "activations/layer9_attention_weight_min": -45.069393157958984,
      "epoch": 20.58,
      "learning_rate": 1.743674242424242e-05,
      "loss": 2.7304,
      "step": 354250
    },
    {
      "activations/layer0_attention_weight_max": 15.6912260055542,
      "activations/layer0_attention_weight_min": -14.099348068237305,
      "activations/layer10_attention_weight_max": 34.14127731323242,
      "activations/layer10_attention_weight_min": -35.29678726196289,
      "activations/layer11_attention_weight_max": 31.771591186523438,
      "activations/layer11_attention_weight_min": -34.891563415527344,
      "activations/layer12_attention_weight_max": 20.970205307006836,
      "activations/layer12_attention_weight_min": -23.51020622253418,
      "activations/layer13_attention_weight_max": 33.324886322021484,
      "activations/layer13_attention_weight_min": -27.42770004272461,
      "activations/layer14_attention_weight_max": 34.67254638671875,
      "activations/layer14_attention_weight_min": -31.400827407836914,
      "activations/layer15_attention_weight_max": 33.72406768798828,
      "activations/layer15_attention_weight_min": -31.57568359375,
      "activations/layer16_attention_weight_max": 34.693634033203125,
      "activations/layer16_attention_weight_min": -33.41380310058594,
      "activations/layer17_attention_weight_max": 55.42996597290039,
      "activations/layer17_attention_weight_min": -45.668663024902344,
      "activations/layer18_attention_weight_max": 51.385868072509766,
      "activations/layer18_attention_weight_min": -42.67449188232422,
      "activations/layer19_attention_weight_max": 26.015060424804688,
      "activations/layer19_attention_weight_min": -24.19145965576172,
      "activations/layer1_attention_weight_max": 15.456315994262695,
      "activations/layer1_attention_weight_min": -14.671112060546875,
      "activations/layer20_attention_weight_max": 23.581762313842773,
      "activations/layer20_attention_weight_min": -22.884262084960938,
      "activations/layer21_attention_weight_max": 38.8699836730957,
      "activations/layer21_attention_weight_min": -23.189056396484375,
      "activations/layer22_attention_weight_max": 33.79538345336914,
      "activations/layer22_attention_weight_min": -27.633922576904297,
      "activations/layer23_attention_weight_max": 34.39231872558594,
      "activations/layer23_attention_weight_min": -24.3210506439209,
      "activations/layer2_attention_weight_max": 31.970565795898438,
      "activations/layer2_attention_weight_min": -29.88405990600586,
      "activations/layer3_attention_weight_max": 88.2498550415039,
      "activations/layer3_attention_weight_min": -87.6872787475586,
      "activations/layer4_attention_weight_max": 87.71822357177734,
      "activations/layer4_attention_weight_min": -85.77803039550781,
      "activations/layer5_attention_weight_max": 63.63874053955078,
      "activations/layer5_attention_weight_min": -69.86456298828125,
      "activations/layer6_attention_weight_max": 52.025020599365234,
      "activations/layer6_attention_weight_min": -51.124698638916016,
      "activations/layer7_attention_weight_max": 70.94903564453125,
      "activations/layer7_attention_weight_min": -66.55560302734375,
      "activations/layer8_attention_weight_max": 46.700138092041016,
      "activations/layer8_attention_weight_min": -48.90477752685547,
      "activations/layer9_attention_weight_max": 44.385833740234375,
      "activations/layer9_attention_weight_min": -46.443153381347656,
      "epoch": 20.59,
      "learning_rate": 1.741780303030303e-05,
      "loss": 2.7248,
      "step": 354300
    },
    {
      "activations/layer0_attention_weight_max": 17.30082893371582,
      "activations/layer0_attention_weight_min": -13.902305603027344,
      "activations/layer10_attention_weight_max": 37.0394287109375,
      "activations/layer10_attention_weight_min": -37.86896896362305,
      "activations/layer11_attention_weight_max": 33.3620491027832,
      "activations/layer11_attention_weight_min": -37.75225830078125,
      "activations/layer12_attention_weight_max": 18.03065299987793,
      "activations/layer12_attention_weight_min": -29.773639678955078,
      "activations/layer13_attention_weight_max": 33.14495086669922,
      "activations/layer13_attention_weight_min": -25.367748260498047,
      "activations/layer14_attention_weight_max": 33.23469543457031,
      "activations/layer14_attention_weight_min": -28.52593421936035,
      "activations/layer15_attention_weight_max": 31.525487899780273,
      "activations/layer15_attention_weight_min": -29.05528450012207,
      "activations/layer16_attention_weight_max": 34.623416900634766,
      "activations/layer16_attention_weight_min": -31.624813079833984,
      "activations/layer17_attention_weight_max": 50.35702133178711,
      "activations/layer17_attention_weight_min": -43.233455657958984,
      "activations/layer18_attention_weight_max": 45.137847900390625,
      "activations/layer18_attention_weight_min": -37.28494644165039,
      "activations/layer19_attention_weight_max": 24.80214500427246,
      "activations/layer19_attention_weight_min": -20.472299575805664,
      "activations/layer1_attention_weight_max": 15.491361618041992,
      "activations/layer1_attention_weight_min": -14.73735523223877,
      "activations/layer20_attention_weight_max": 21.048643112182617,
      "activations/layer20_attention_weight_min": -21.632055282592773,
      "activations/layer21_attention_weight_max": 34.612144470214844,
      "activations/layer21_attention_weight_min": -23.534074783325195,
      "activations/layer22_attention_weight_max": 29.688419342041016,
      "activations/layer22_attention_weight_min": -24.103160858154297,
      "activations/layer23_attention_weight_max": 36.3015022277832,
      "activations/layer23_attention_weight_min": -24.068180084228516,
      "activations/layer2_attention_weight_max": 32.61979675292969,
      "activations/layer2_attention_weight_min": -31.67743682861328,
      "activations/layer3_attention_weight_max": 89.38768005371094,
      "activations/layer3_attention_weight_min": -88.68537139892578,
      "activations/layer4_attention_weight_max": 82.80272674560547,
      "activations/layer4_attention_weight_min": -82.60880279541016,
      "activations/layer5_attention_weight_max": 60.587623596191406,
      "activations/layer5_attention_weight_min": -72.20916748046875,
      "activations/layer6_attention_weight_max": 45.758155822753906,
      "activations/layer6_attention_weight_min": -54.15937423706055,
      "activations/layer7_attention_weight_max": 65.8282241821289,
      "activations/layer7_attention_weight_min": -67.31854248046875,
      "activations/layer8_attention_weight_max": 50.18749237060547,
      "activations/layer8_attention_weight_min": -59.59326934814453,
      "activations/layer9_attention_weight_max": 43.29794692993164,
      "activations/layer9_attention_weight_min": -45.524436950683594,
      "epoch": 20.59,
      "learning_rate": 1.7398863636363635e-05,
      "loss": 2.7245,
      "step": 354350
    },
    {
      "activations/layer0_attention_weight_max": 15.630608558654785,
      "activations/layer0_attention_weight_min": -13.961918830871582,
      "activations/layer10_attention_weight_max": 33.97747039794922,
      "activations/layer10_attention_weight_min": -35.88959884643555,
      "activations/layer11_attention_weight_max": 32.307071685791016,
      "activations/layer11_attention_weight_min": -34.05485534667969,
      "activations/layer12_attention_weight_max": 19.106266021728516,
      "activations/layer12_attention_weight_min": -26.176956176757812,
      "activations/layer13_attention_weight_max": 45.12960433959961,
      "activations/layer13_attention_weight_min": -28.60567283630371,
      "activations/layer14_attention_weight_max": 37.52544021606445,
      "activations/layer14_attention_weight_min": -31.100120544433594,
      "activations/layer15_attention_weight_max": 37.67071533203125,
      "activations/layer15_attention_weight_min": -31.74462127685547,
      "activations/layer16_attention_weight_max": 34.50520706176758,
      "activations/layer16_attention_weight_min": -34.7120361328125,
      "activations/layer17_attention_weight_max": 52.77375411987305,
      "activations/layer17_attention_weight_min": -47.4094123840332,
      "activations/layer18_attention_weight_max": 48.964107513427734,
      "activations/layer18_attention_weight_min": -43.1254768371582,
      "activations/layer19_attention_weight_max": 24.114225387573242,
      "activations/layer19_attention_weight_min": -25.547468185424805,
      "activations/layer1_attention_weight_max": 15.141343116760254,
      "activations/layer1_attention_weight_min": -13.307600021362305,
      "activations/layer20_attention_weight_max": 21.762100219726562,
      "activations/layer20_attention_weight_min": -26.76348876953125,
      "activations/layer21_attention_weight_max": 31.99776268005371,
      "activations/layer21_attention_weight_min": -21.170166015625,
      "activations/layer22_attention_weight_max": 31.00425148010254,
      "activations/layer22_attention_weight_min": -25.422504425048828,
      "activations/layer23_attention_weight_max": 32.618125915527344,
      "activations/layer23_attention_weight_min": -23.561115264892578,
      "activations/layer2_attention_weight_max": 30.110759735107422,
      "activations/layer2_attention_weight_min": -28.337377548217773,
      "activations/layer3_attention_weight_max": 82.81102752685547,
      "activations/layer3_attention_weight_min": -83.08106231689453,
      "activations/layer4_attention_weight_max": 87.507080078125,
      "activations/layer4_attention_weight_min": -80.34349822998047,
      "activations/layer5_attention_weight_max": 62.161949157714844,
      "activations/layer5_attention_weight_min": -69.2721176147461,
      "activations/layer6_attention_weight_max": 50.048282623291016,
      "activations/layer6_attention_weight_min": -51.04762268066406,
      "activations/layer7_attention_weight_max": 67.01272583007812,
      "activations/layer7_attention_weight_min": -64.443603515625,
      "activations/layer8_attention_weight_max": 44.868167877197266,
      "activations/layer8_attention_weight_min": -50.26609420776367,
      "activations/layer9_attention_weight_max": 43.81587600708008,
      "activations/layer9_attention_weight_min": -46.90220642089844,
      "epoch": 20.59,
      "learning_rate": 1.737992424242424e-05,
      "loss": 2.7213,
      "step": 354400
    },
    {
      "activations/layer0_attention_weight_max": 16.058870315551758,
      "activations/layer0_attention_weight_min": -14.003838539123535,
      "activations/layer10_attention_weight_max": 38.20952224731445,
      "activations/layer10_attention_weight_min": -36.98566818237305,
      "activations/layer11_attention_weight_max": 35.89669418334961,
      "activations/layer11_attention_weight_min": -36.931495666503906,
      "activations/layer12_attention_weight_max": 19.160730361938477,
      "activations/layer12_attention_weight_min": -31.360246658325195,
      "activations/layer13_attention_weight_max": 33.33525085449219,
      "activations/layer13_attention_weight_min": -28.783166885375977,
      "activations/layer14_attention_weight_max": 37.25227737426758,
      "activations/layer14_attention_weight_min": -32.736873626708984,
      "activations/layer15_attention_weight_max": 35.85377883911133,
      "activations/layer15_attention_weight_min": -32.54892349243164,
      "activations/layer16_attention_weight_max": 38.07257843017578,
      "activations/layer16_attention_weight_min": -35.396175384521484,
      "activations/layer17_attention_weight_max": 58.34502029418945,
      "activations/layer17_attention_weight_min": -48.07063674926758,
      "activations/layer18_attention_weight_max": 53.88868713378906,
      "activations/layer18_attention_weight_min": -43.28654479980469,
      "activations/layer19_attention_weight_max": 27.513219833374023,
      "activations/layer19_attention_weight_min": -23.176729202270508,
      "activations/layer1_attention_weight_max": 15.528359413146973,
      "activations/layer1_attention_weight_min": -14.256284713745117,
      "activations/layer20_attention_weight_max": 25.419071197509766,
      "activations/layer20_attention_weight_min": -21.937437057495117,
      "activations/layer21_attention_weight_max": 39.69820022583008,
      "activations/layer21_attention_weight_min": -24.8215389251709,
      "activations/layer22_attention_weight_max": 29.63934326171875,
      "activations/layer22_attention_weight_min": -26.943880081176758,
      "activations/layer23_attention_weight_max": 38.883567810058594,
      "activations/layer23_attention_weight_min": -25.05544662475586,
      "activations/layer2_attention_weight_max": 33.240238189697266,
      "activations/layer2_attention_weight_min": -30.826902389526367,
      "activations/layer3_attention_weight_max": 86.2166519165039,
      "activations/layer3_attention_weight_min": -84.7787094116211,
      "activations/layer4_attention_weight_max": 83.29345703125,
      "activations/layer4_attention_weight_min": -84.04389190673828,
      "activations/layer5_attention_weight_max": 59.246604919433594,
      "activations/layer5_attention_weight_min": -70.01226806640625,
      "activations/layer6_attention_weight_max": 49.70793533325195,
      "activations/layer6_attention_weight_min": -50.29312515258789,
      "activations/layer7_attention_weight_max": 64.90953063964844,
      "activations/layer7_attention_weight_min": -64.0441665649414,
      "activations/layer8_attention_weight_max": 49.66871643066406,
      "activations/layer8_attention_weight_min": -54.0985221862793,
      "activations/layer9_attention_weight_max": 44.20225524902344,
      "activations/layer9_attention_weight_min": -46.937721252441406,
      "epoch": 20.6,
      "learning_rate": 1.7360984848484848e-05,
      "loss": 2.6945,
      "step": 354450
    },
    {
      "activations/layer0_attention_weight_max": 15.117798805236816,
      "activations/layer0_attention_weight_min": -14.064478874206543,
      "activations/layer10_attention_weight_max": 34.240882873535156,
      "activations/layer10_attention_weight_min": -34.26540756225586,
      "activations/layer11_attention_weight_max": 31.202083587646484,
      "activations/layer11_attention_weight_min": -34.035831451416016,
      "activations/layer12_attention_weight_max": 21.6645565032959,
      "activations/layer12_attention_weight_min": -24.856529235839844,
      "activations/layer13_attention_weight_max": 40.03611373901367,
      "activations/layer13_attention_weight_min": -28.565879821777344,
      "activations/layer14_attention_weight_max": 43.33823776245117,
      "activations/layer14_attention_weight_min": -35.52838897705078,
      "activations/layer15_attention_weight_max": 40.04293441772461,
      "activations/layer15_attention_weight_min": -33.89423751831055,
      "activations/layer16_attention_weight_max": 41.34677505493164,
      "activations/layer16_attention_weight_min": -37.579689025878906,
      "activations/layer17_attention_weight_max": 61.48286437988281,
      "activations/layer17_attention_weight_min": -52.805419921875,
      "activations/layer18_attention_weight_max": 59.2640266418457,
      "activations/layer18_attention_weight_min": -46.50033950805664,
      "activations/layer19_attention_weight_max": 29.863937377929688,
      "activations/layer19_attention_weight_min": -25.007266998291016,
      "activations/layer1_attention_weight_max": 15.399026870727539,
      "activations/layer1_attention_weight_min": -12.985967636108398,
      "activations/layer20_attention_weight_max": 29.170452117919922,
      "activations/layer20_attention_weight_min": -20.71469497680664,
      "activations/layer21_attention_weight_max": 43.01075744628906,
      "activations/layer21_attention_weight_min": -22.728004455566406,
      "activations/layer22_attention_weight_max": 34.84770584106445,
      "activations/layer22_attention_weight_min": -26.629249572753906,
      "activations/layer23_attention_weight_max": 45.393341064453125,
      "activations/layer23_attention_weight_min": -25.582447052001953,
      "activations/layer2_attention_weight_max": 28.846874237060547,
      "activations/layer2_attention_weight_min": -29.441038131713867,
      "activations/layer3_attention_weight_max": 82.3067398071289,
      "activations/layer3_attention_weight_min": -84.03097534179688,
      "activations/layer4_attention_weight_max": 79.24169158935547,
      "activations/layer4_attention_weight_min": -81.64923858642578,
      "activations/layer5_attention_weight_max": 57.87755584716797,
      "activations/layer5_attention_weight_min": -72.44087219238281,
      "activations/layer6_attention_weight_max": 44.12735366821289,
      "activations/layer6_attention_weight_min": -48.095191955566406,
      "activations/layer7_attention_weight_max": 64.4061279296875,
      "activations/layer7_attention_weight_min": -63.47701644897461,
      "activations/layer8_attention_weight_max": 43.76751708984375,
      "activations/layer8_attention_weight_min": -49.59670639038086,
      "activations/layer9_attention_weight_max": 40.8127555847168,
      "activations/layer9_attention_weight_min": -44.17660140991211,
      "epoch": 20.6,
      "learning_rate": 1.7342045454545453e-05,
      "loss": 2.7187,
      "step": 354500
    },
    {
      "activations/layer0_attention_weight_max": 14.98577880859375,
      "activations/layer0_attention_weight_min": -14.533336639404297,
      "activations/layer10_attention_weight_max": 36.606990814208984,
      "activations/layer10_attention_weight_min": -37.26791000366211,
      "activations/layer11_attention_weight_max": 33.67401123046875,
      "activations/layer11_attention_weight_min": -37.672203063964844,
      "activations/layer12_attention_weight_max": 20.205245971679688,
      "activations/layer12_attention_weight_min": -24.045557022094727,
      "activations/layer13_attention_weight_max": 34.3728141784668,
      "activations/layer13_attention_weight_min": -29.429473876953125,
      "activations/layer14_attention_weight_max": 37.06269836425781,
      "activations/layer14_attention_weight_min": -31.37741470336914,
      "activations/layer15_attention_weight_max": 34.941673278808594,
      "activations/layer15_attention_weight_min": -33.24229431152344,
      "activations/layer16_attention_weight_max": 35.48970413208008,
      "activations/layer16_attention_weight_min": -33.06853485107422,
      "activations/layer17_attention_weight_max": 51.809906005859375,
      "activations/layer17_attention_weight_min": -48.66547775268555,
      "activations/layer18_attention_weight_max": 48.68500900268555,
      "activations/layer18_attention_weight_min": -40.633365631103516,
      "activations/layer19_attention_weight_max": 24.91560935974121,
      "activations/layer19_attention_weight_min": -23.13231658935547,
      "activations/layer1_attention_weight_max": 15.812541961669922,
      "activations/layer1_attention_weight_min": -13.534698486328125,
      "activations/layer20_attention_weight_max": 22.546688079833984,
      "activations/layer20_attention_weight_min": -21.858379364013672,
      "activations/layer21_attention_weight_max": 38.71427536010742,
      "activations/layer21_attention_weight_min": -23.725969314575195,
      "activations/layer22_attention_weight_max": 32.0657958984375,
      "activations/layer22_attention_weight_min": -26.439931869506836,
      "activations/layer23_attention_weight_max": 37.82516860961914,
      "activations/layer23_attention_weight_min": -23.49652099609375,
      "activations/layer2_attention_weight_max": 32.284629821777344,
      "activations/layer2_attention_weight_min": -29.720924377441406,
      "activations/layer3_attention_weight_max": 89.14048767089844,
      "activations/layer3_attention_weight_min": -87.44868469238281,
      "activations/layer4_attention_weight_max": 86.21289825439453,
      "activations/layer4_attention_weight_min": -85.7769775390625,
      "activations/layer5_attention_weight_max": 64.99961853027344,
      "activations/layer5_attention_weight_min": -72.35038757324219,
      "activations/layer6_attention_weight_max": 50.95366668701172,
      "activations/layer6_attention_weight_min": -52.82368087768555,
      "activations/layer7_attention_weight_max": 67.98474884033203,
      "activations/layer7_attention_weight_min": -69.51473236083984,
      "activations/layer8_attention_weight_max": 48.381160736083984,
      "activations/layer8_attention_weight_min": -56.82667922973633,
      "activations/layer9_attention_weight_max": 45.70771789550781,
      "activations/layer9_attention_weight_min": -50.93743896484375,
      "epoch": 20.6,
      "learning_rate": 1.7323106060606058e-05,
      "loss": 2.7445,
      "step": 354550
    },
    {
      "activations/layer0_attention_weight_max": 15.279775619506836,
      "activations/layer0_attention_weight_min": -14.343246459960938,
      "activations/layer10_attention_weight_max": 34.794593811035156,
      "activations/layer10_attention_weight_min": -35.7945442199707,
      "activations/layer11_attention_weight_max": 32.112815856933594,
      "activations/layer11_attention_weight_min": -35.836265563964844,
      "activations/layer12_attention_weight_max": 19.26251792907715,
      "activations/layer12_attention_weight_min": -25.504825592041016,
      "activations/layer13_attention_weight_max": 35.400917053222656,
      "activations/layer13_attention_weight_min": -27.952529907226562,
      "activations/layer14_attention_weight_max": 41.38493347167969,
      "activations/layer14_attention_weight_min": -30.228803634643555,
      "activations/layer15_attention_weight_max": 35.896331787109375,
      "activations/layer15_attention_weight_min": -30.33211326599121,
      "activations/layer16_attention_weight_max": 37.469810485839844,
      "activations/layer16_attention_weight_min": -33.99551773071289,
      "activations/layer17_attention_weight_max": 57.04823684692383,
      "activations/layer17_attention_weight_min": -49.4195556640625,
      "activations/layer18_attention_weight_max": 53.51156234741211,
      "activations/layer18_attention_weight_min": -44.88750076293945,
      "activations/layer19_attention_weight_max": 26.964130401611328,
      "activations/layer19_attention_weight_min": -25.121707916259766,
      "activations/layer1_attention_weight_max": 15.206864356994629,
      "activations/layer1_attention_weight_min": -12.774541854858398,
      "activations/layer20_attention_weight_max": 25.80995750427246,
      "activations/layer20_attention_weight_min": -22.96661949157715,
      "activations/layer21_attention_weight_max": 45.89024353027344,
      "activations/layer21_attention_weight_min": -24.67453384399414,
      "activations/layer22_attention_weight_max": 35.3058967590332,
      "activations/layer22_attention_weight_min": -26.42289924621582,
      "activations/layer23_attention_weight_max": 41.274208068847656,
      "activations/layer23_attention_weight_min": -25.362911224365234,
      "activations/layer2_attention_weight_max": 30.9024715423584,
      "activations/layer2_attention_weight_min": -31.199872970581055,
      "activations/layer3_attention_weight_max": 90.1279525756836,
      "activations/layer3_attention_weight_min": -93.55868530273438,
      "activations/layer4_attention_weight_max": 87.5368881225586,
      "activations/layer4_attention_weight_min": -83.58855438232422,
      "activations/layer5_attention_weight_max": 64.71849060058594,
      "activations/layer5_attention_weight_min": -73.01421356201172,
      "activations/layer6_attention_weight_max": 48.34296417236328,
      "activations/layer6_attention_weight_min": -49.15225601196289,
      "activations/layer7_attention_weight_max": 65.8413314819336,
      "activations/layer7_attention_weight_min": -64.86286926269531,
      "activations/layer8_attention_weight_max": 46.270606994628906,
      "activations/layer8_attention_weight_min": -48.40605545043945,
      "activations/layer9_attention_weight_max": 44.47938919067383,
      "activations/layer9_attention_weight_min": -44.799495697021484,
      "epoch": 20.6,
      "learning_rate": 1.7304166666666666e-05,
      "loss": 2.7279,
      "step": 354600
    },
    {
      "activations/layer0_attention_weight_max": 15.743561744689941,
      "activations/layer0_attention_weight_min": -14.305434226989746,
      "activations/layer10_attention_weight_max": 36.81819534301758,
      "activations/layer10_attention_weight_min": -36.89166259765625,
      "activations/layer11_attention_weight_max": 35.88239288330078,
      "activations/layer11_attention_weight_min": -36.70367431640625,
      "activations/layer12_attention_weight_max": 18.568201065063477,
      "activations/layer12_attention_weight_min": -25.011938095092773,
      "activations/layer13_attention_weight_max": 31.162498474121094,
      "activations/layer13_attention_weight_min": -26.22315216064453,
      "activations/layer14_attention_weight_max": 33.78366470336914,
      "activations/layer14_attention_weight_min": -30.021596908569336,
      "activations/layer15_attention_weight_max": 31.803987503051758,
      "activations/layer15_attention_weight_min": -29.962125778198242,
      "activations/layer16_attention_weight_max": 34.627105712890625,
      "activations/layer16_attention_weight_min": -32.45685577392578,
      "activations/layer17_attention_weight_max": 49.55174255371094,
      "activations/layer17_attention_weight_min": -44.64604949951172,
      "activations/layer18_attention_weight_max": 45.75896072387695,
      "activations/layer18_attention_weight_min": -38.704166412353516,
      "activations/layer19_attention_weight_max": 26.59401512145996,
      "activations/layer19_attention_weight_min": -24.039640426635742,
      "activations/layer1_attention_weight_max": 14.85377025604248,
      "activations/layer1_attention_weight_min": -14.135555267333984,
      "activations/layer20_attention_weight_max": 24.281877517700195,
      "activations/layer20_attention_weight_min": -21.06479263305664,
      "activations/layer21_attention_weight_max": 35.54545974731445,
      "activations/layer21_attention_weight_min": -20.709692001342773,
      "activations/layer22_attention_weight_max": 29.265058517456055,
      "activations/layer22_attention_weight_min": -24.908283233642578,
      "activations/layer23_attention_weight_max": 39.213218688964844,
      "activations/layer23_attention_weight_min": -24.709980010986328,
      "activations/layer2_attention_weight_max": 32.499603271484375,
      "activations/layer2_attention_weight_min": -30.369613647460938,
      "activations/layer3_attention_weight_max": 88.65033721923828,
      "activations/layer3_attention_weight_min": -88.22417449951172,
      "activations/layer4_attention_weight_max": 84.4925765991211,
      "activations/layer4_attention_weight_min": -83.1656723022461,
      "activations/layer5_attention_weight_max": 61.92595672607422,
      "activations/layer5_attention_weight_min": -73.4139175415039,
      "activations/layer6_attention_weight_max": 49.465946197509766,
      "activations/layer6_attention_weight_min": -50.18280792236328,
      "activations/layer7_attention_weight_max": 67.68329620361328,
      "activations/layer7_attention_weight_min": -64.61925506591797,
      "activations/layer8_attention_weight_max": 47.6983528137207,
      "activations/layer8_attention_weight_min": -49.66565704345703,
      "activations/layer9_attention_weight_max": 43.58078384399414,
      "activations/layer9_attention_weight_min": -43.97060775756836,
      "epoch": 20.61,
      "learning_rate": 1.728522727272727e-05,
      "loss": 2.7138,
      "step": 354650
    },
    {
      "activations/layer0_attention_weight_max": 15.377799987792969,
      "activations/layer0_attention_weight_min": -14.253498077392578,
      "activations/layer10_attention_weight_max": 35.206756591796875,
      "activations/layer10_attention_weight_min": -35.74506759643555,
      "activations/layer11_attention_weight_max": 32.82561492919922,
      "activations/layer11_attention_weight_min": -33.80754089355469,
      "activations/layer12_attention_weight_max": 18.953266143798828,
      "activations/layer12_attention_weight_min": -23.95461654663086,
      "activations/layer13_attention_weight_max": 35.884403228759766,
      "activations/layer13_attention_weight_min": -27.94244956970215,
      "activations/layer14_attention_weight_max": 39.453338623046875,
      "activations/layer14_attention_weight_min": -28.71552276611328,
      "activations/layer15_attention_weight_max": 35.350467681884766,
      "activations/layer15_attention_weight_min": -28.797927856445312,
      "activations/layer16_attention_weight_max": 35.49966812133789,
      "activations/layer16_attention_weight_min": -32.09162521362305,
      "activations/layer17_attention_weight_max": 52.22077178955078,
      "activations/layer17_attention_weight_min": -42.95305252075195,
      "activations/layer18_attention_weight_max": 48.000423431396484,
      "activations/layer18_attention_weight_min": -37.920326232910156,
      "activations/layer19_attention_weight_max": 25.8409481048584,
      "activations/layer19_attention_weight_min": -23.26187515258789,
      "activations/layer1_attention_weight_max": 15.088926315307617,
      "activations/layer1_attention_weight_min": -13.062615394592285,
      "activations/layer20_attention_weight_max": 25.19112205505371,
      "activations/layer20_attention_weight_min": -19.649019241333008,
      "activations/layer21_attention_weight_max": 36.34523391723633,
      "activations/layer21_attention_weight_min": -23.2257080078125,
      "activations/layer22_attention_weight_max": 31.117626190185547,
      "activations/layer22_attention_weight_min": -23.94632339477539,
      "activations/layer23_attention_weight_max": 41.22930145263672,
      "activations/layer23_attention_weight_min": -23.339033126831055,
      "activations/layer2_attention_weight_max": 30.75123405456543,
      "activations/layer2_attention_weight_min": -29.218955993652344,
      "activations/layer3_attention_weight_max": 90.0957260131836,
      "activations/layer3_attention_weight_min": -88.96533203125,
      "activations/layer4_attention_weight_max": 85.43255615234375,
      "activations/layer4_attention_weight_min": -82.47421264648438,
      "activations/layer5_attention_weight_max": 62.46596908569336,
      "activations/layer5_attention_weight_min": -72.10430145263672,
      "activations/layer6_attention_weight_max": 49.064186096191406,
      "activations/layer6_attention_weight_min": -48.738121032714844,
      "activations/layer7_attention_weight_max": 67.14797973632812,
      "activations/layer7_attention_weight_min": -66.4976806640625,
      "activations/layer8_attention_weight_max": 47.06032180786133,
      "activations/layer8_attention_weight_min": -48.802242279052734,
      "activations/layer9_attention_weight_max": 47.59792709350586,
      "activations/layer9_attention_weight_min": -46.686920166015625,
      "epoch": 20.61,
      "learning_rate": 1.7266287878787876e-05,
      "loss": 2.7087,
      "step": 354700
    },
    {
      "activations/layer0_attention_weight_max": 15.073533058166504,
      "activations/layer0_attention_weight_min": -14.330362319946289,
      "activations/layer10_attention_weight_max": 50.46979522705078,
      "activations/layer10_attention_weight_min": -49.506282806396484,
      "activations/layer11_attention_weight_max": 50.734283447265625,
      "activations/layer11_attention_weight_min": -47.98571014404297,
      "activations/layer12_attention_weight_max": 24.690933227539062,
      "activations/layer12_attention_weight_min": -26.183565139770508,
      "activations/layer13_attention_weight_max": 63.402015686035156,
      "activations/layer13_attention_weight_min": -36.00175857543945,
      "activations/layer14_attention_weight_max": 64.73772430419922,
      "activations/layer14_attention_weight_min": -35.46211242675781,
      "activations/layer15_attention_weight_max": 55.273193359375,
      "activations/layer15_attention_weight_min": -34.81614303588867,
      "activations/layer16_attention_weight_max": 42.81338882446289,
      "activations/layer16_attention_weight_min": -39.202789306640625,
      "activations/layer17_attention_weight_max": 63.8926887512207,
      "activations/layer17_attention_weight_min": -56.7986946105957,
      "activations/layer18_attention_weight_max": 56.4533576965332,
      "activations/layer18_attention_weight_min": -48.752952575683594,
      "activations/layer19_attention_weight_max": 29.711322784423828,
      "activations/layer19_attention_weight_min": -27.6834774017334,
      "activations/layer1_attention_weight_max": 15.73076343536377,
      "activations/layer1_attention_weight_min": -15.493698120117188,
      "activations/layer20_attention_weight_max": 30.6718692779541,
      "activations/layer20_attention_weight_min": -22.753135681152344,
      "activations/layer21_attention_weight_max": 57.8905029296875,
      "activations/layer21_attention_weight_min": -28.020797729492188,
      "activations/layer22_attention_weight_max": 33.52808380126953,
      "activations/layer22_attention_weight_min": -26.808332443237305,
      "activations/layer23_attention_weight_max": 38.5826530456543,
      "activations/layer23_attention_weight_min": -26.555946350097656,
      "activations/layer2_attention_weight_max": 33.19050598144531,
      "activations/layer2_attention_weight_min": -31.44036102294922,
      "activations/layer3_attention_weight_max": 95.2271957397461,
      "activations/layer3_attention_weight_min": -95.16893005371094,
      "activations/layer4_attention_weight_max": 88.54991149902344,
      "activations/layer4_attention_weight_min": -87.56291961669922,
      "activations/layer5_attention_weight_max": 62.174617767333984,
      "activations/layer5_attention_weight_min": -69.09507751464844,
      "activations/layer6_attention_weight_max": 52.10817337036133,
      "activations/layer6_attention_weight_min": -53.729740142822266,
      "activations/layer7_attention_weight_max": 74.21846771240234,
      "activations/layer7_attention_weight_min": -72.7446060180664,
      "activations/layer8_attention_weight_max": 60.73297882080078,
      "activations/layer8_attention_weight_min": -61.16096496582031,
      "activations/layer9_attention_weight_max": 61.72265625,
      "activations/layer9_attention_weight_min": -56.181880950927734,
      "epoch": 20.61,
      "learning_rate": 1.7247348484848484e-05,
      "loss": 2.7125,
      "step": 354750
    },
    {
      "activations/layer0_attention_weight_max": 15.603262901306152,
      "activations/layer0_attention_weight_min": -14.010834693908691,
      "activations/layer10_attention_weight_max": 37.12184524536133,
      "activations/layer10_attention_weight_min": -37.702919006347656,
      "activations/layer11_attention_weight_max": 35.178855895996094,
      "activations/layer11_attention_weight_min": -36.489227294921875,
      "activations/layer12_attention_weight_max": 20.75320053100586,
      "activations/layer12_attention_weight_min": -27.1299991607666,
      "activations/layer13_attention_weight_max": 35.90685272216797,
      "activations/layer13_attention_weight_min": -28.48832130432129,
      "activations/layer14_attention_weight_max": 34.649566650390625,
      "activations/layer14_attention_weight_min": -31.16520118713379,
      "activations/layer15_attention_weight_max": 34.86492919921875,
      "activations/layer15_attention_weight_min": -31.18705940246582,
      "activations/layer16_attention_weight_max": 34.92617416381836,
      "activations/layer16_attention_weight_min": -33.02451705932617,
      "activations/layer17_attention_weight_max": 53.29530334472656,
      "activations/layer17_attention_weight_min": -46.76482391357422,
      "activations/layer18_attention_weight_max": 50.330894470214844,
      "activations/layer18_attention_weight_min": -42.152767181396484,
      "activations/layer19_attention_weight_max": 24.548965454101562,
      "activations/layer19_attention_weight_min": -23.133161544799805,
      "activations/layer1_attention_weight_max": 15.846202850341797,
      "activations/layer1_attention_weight_min": -13.950396537780762,
      "activations/layer20_attention_weight_max": 22.5434627532959,
      "activations/layer20_attention_weight_min": -20.251811981201172,
      "activations/layer21_attention_weight_max": 35.76129913330078,
      "activations/layer21_attention_weight_min": -22.506282806396484,
      "activations/layer22_attention_weight_max": 29.581562042236328,
      "activations/layer22_attention_weight_min": -29.836090087890625,
      "activations/layer23_attention_weight_max": 35.294708251953125,
      "activations/layer23_attention_weight_min": -25.10932159423828,
      "activations/layer2_attention_weight_max": 31.562965393066406,
      "activations/layer2_attention_weight_min": -30.095115661621094,
      "activations/layer3_attention_weight_max": 86.63916778564453,
      "activations/layer3_attention_weight_min": -92.7497787475586,
      "activations/layer4_attention_weight_max": 84.08503723144531,
      "activations/layer4_attention_weight_min": -83.6301498413086,
      "activations/layer5_attention_weight_max": 60.37571716308594,
      "activations/layer5_attention_weight_min": -68.58335876464844,
      "activations/layer6_attention_weight_max": 47.988853454589844,
      "activations/layer6_attention_weight_min": -49.490272521972656,
      "activations/layer7_attention_weight_max": 64.784912109375,
      "activations/layer7_attention_weight_min": -68.28327178955078,
      "activations/layer8_attention_weight_max": 48.04300308227539,
      "activations/layer8_attention_weight_min": -50.52939224243164,
      "activations/layer9_attention_weight_max": 47.11195373535156,
      "activations/layer9_attention_weight_min": -47.35958480834961,
      "epoch": 20.62,
      "learning_rate": 1.722840909090909e-05,
      "loss": 2.7089,
      "step": 354800
    },
    {
      "activations/layer0_attention_weight_max": 15.015277862548828,
      "activations/layer0_attention_weight_min": -14.815648078918457,
      "activations/layer10_attention_weight_max": 37.88747787475586,
      "activations/layer10_attention_weight_min": -36.77785110473633,
      "activations/layer11_attention_weight_max": 35.57769012451172,
      "activations/layer11_attention_weight_min": -37.48748779296875,
      "activations/layer12_attention_weight_max": 19.11798858642578,
      "activations/layer12_attention_weight_min": -26.47955894470215,
      "activations/layer13_attention_weight_max": 33.48876190185547,
      "activations/layer13_attention_weight_min": -28.551469802856445,
      "activations/layer14_attention_weight_max": 36.8309211730957,
      "activations/layer14_attention_weight_min": -32.103431701660156,
      "activations/layer15_attention_weight_max": 35.96845245361328,
      "activations/layer15_attention_weight_min": -32.4874382019043,
      "activations/layer16_attention_weight_max": 37.2048454284668,
      "activations/layer16_attention_weight_min": -34.05306625366211,
      "activations/layer17_attention_weight_max": 53.08015060424805,
      "activations/layer17_attention_weight_min": -47.93071365356445,
      "activations/layer18_attention_weight_max": 48.21620559692383,
      "activations/layer18_attention_weight_min": -44.50086975097656,
      "activations/layer19_attention_weight_max": 24.235570907592773,
      "activations/layer19_attention_weight_min": -24.50727653503418,
      "activations/layer1_attention_weight_max": 16.694787979125977,
      "activations/layer1_attention_weight_min": -13.063876152038574,
      "activations/layer20_attention_weight_max": 22.8221378326416,
      "activations/layer20_attention_weight_min": -22.9386043548584,
      "activations/layer21_attention_weight_max": 36.785438537597656,
      "activations/layer21_attention_weight_min": -22.48594856262207,
      "activations/layer22_attention_weight_max": 27.17141342163086,
      "activations/layer22_attention_weight_min": -28.164047241210938,
      "activations/layer23_attention_weight_max": 35.345855712890625,
      "activations/layer23_attention_weight_min": -24.5550537109375,
      "activations/layer2_attention_weight_max": 32.92420196533203,
      "activations/layer2_attention_weight_min": -30.302762985229492,
      "activations/layer3_attention_weight_max": 93.6565170288086,
      "activations/layer3_attention_weight_min": -89.0159683227539,
      "activations/layer4_attention_weight_max": 87.01255798339844,
      "activations/layer4_attention_weight_min": -85.1676025390625,
      "activations/layer5_attention_weight_max": 60.11195373535156,
      "activations/layer5_attention_weight_min": -71.01898956298828,
      "activations/layer6_attention_weight_max": 51.60788345336914,
      "activations/layer6_attention_weight_min": -50.04551696777344,
      "activations/layer7_attention_weight_max": 65.57539367675781,
      "activations/layer7_attention_weight_min": -65.41405487060547,
      "activations/layer8_attention_weight_max": 49.018001556396484,
      "activations/layer8_attention_weight_min": -53.209083557128906,
      "activations/layer9_attention_weight_max": 46.27078628540039,
      "activations/layer9_attention_weight_min": -49.75423812866211,
      "epoch": 20.62,
      "learning_rate": 1.7209469696969694e-05,
      "loss": 2.719,
      "step": 354850
    },
    {
      "activations/layer0_attention_weight_max": 15.73257064819336,
      "activations/layer0_attention_weight_min": -14.378737449645996,
      "activations/layer10_attention_weight_max": 37.77079772949219,
      "activations/layer10_attention_weight_min": -35.37030029296875,
      "activations/layer11_attention_weight_max": 33.91925048828125,
      "activations/layer11_attention_weight_min": -33.85063171386719,
      "activations/layer12_attention_weight_max": 19.208293914794922,
      "activations/layer12_attention_weight_min": -24.66609001159668,
      "activations/layer13_attention_weight_max": 33.20093536376953,
      "activations/layer13_attention_weight_min": -28.247961044311523,
      "activations/layer14_attention_weight_max": 35.71197509765625,
      "activations/layer14_attention_weight_min": -30.32115364074707,
      "activations/layer15_attention_weight_max": 34.31606674194336,
      "activations/layer15_attention_weight_min": -31.105175018310547,
      "activations/layer16_attention_weight_max": 35.73821258544922,
      "activations/layer16_attention_weight_min": -33.03018569946289,
      "activations/layer17_attention_weight_max": 54.85994338989258,
      "activations/layer17_attention_weight_min": -46.24384307861328,
      "activations/layer18_attention_weight_max": 50.3648567199707,
      "activations/layer18_attention_weight_min": -41.904808044433594,
      "activations/layer19_attention_weight_max": 24.187536239624023,
      "activations/layer19_attention_weight_min": -23.77411460876465,
      "activations/layer1_attention_weight_max": 16.233243942260742,
      "activations/layer1_attention_weight_min": -14.796660423278809,
      "activations/layer20_attention_weight_max": 24.491167068481445,
      "activations/layer20_attention_weight_min": -22.040294647216797,
      "activations/layer21_attention_weight_max": 32.53790283203125,
      "activations/layer21_attention_weight_min": -25.2197265625,
      "activations/layer22_attention_weight_max": 27.835357666015625,
      "activations/layer22_attention_weight_min": -24.959619522094727,
      "activations/layer23_attention_weight_max": 38.0140266418457,
      "activations/layer23_attention_weight_min": -22.892730712890625,
      "activations/layer2_attention_weight_max": 32.345970153808594,
      "activations/layer2_attention_weight_min": -31.11425018310547,
      "activations/layer3_attention_weight_max": 90.67243194580078,
      "activations/layer3_attention_weight_min": -88.95532989501953,
      "activations/layer4_attention_weight_max": 87.16732025146484,
      "activations/layer4_attention_weight_min": -84.52565002441406,
      "activations/layer5_attention_weight_max": 68.78144836425781,
      "activations/layer5_attention_weight_min": -72.62268829345703,
      "activations/layer6_attention_weight_max": 50.70368576049805,
      "activations/layer6_attention_weight_min": -54.862335205078125,
      "activations/layer7_attention_weight_max": 69.90686798095703,
      "activations/layer7_attention_weight_min": -65.63377380371094,
      "activations/layer8_attention_weight_max": 47.791873931884766,
      "activations/layer8_attention_weight_min": -51.618289947509766,
      "activations/layer9_attention_weight_max": 45.217586517333984,
      "activations/layer9_attention_weight_min": -46.358924865722656,
      "epoch": 20.62,
      "learning_rate": 1.7190530303030302e-05,
      "loss": 2.7122,
      "step": 354900
    },
    {
      "activations/layer0_attention_weight_max": 14.921133041381836,
      "activations/layer0_attention_weight_min": -14.034323692321777,
      "activations/layer10_attention_weight_max": 39.04139709472656,
      "activations/layer10_attention_weight_min": -35.620765686035156,
      "activations/layer11_attention_weight_max": 35.85346984863281,
      "activations/layer11_attention_weight_min": -35.74055480957031,
      "activations/layer12_attention_weight_max": 18.562902450561523,
      "activations/layer12_attention_weight_min": -23.000120162963867,
      "activations/layer13_attention_weight_max": 33.045989990234375,
      "activations/layer13_attention_weight_min": -26.395292282104492,
      "activations/layer14_attention_weight_max": 32.82391357421875,
      "activations/layer14_attention_weight_min": -29.721284866333008,
      "activations/layer15_attention_weight_max": 30.838069915771484,
      "activations/layer15_attention_weight_min": -29.993968963623047,
      "activations/layer16_attention_weight_max": 31.87760353088379,
      "activations/layer16_attention_weight_min": -31.10502052307129,
      "activations/layer17_attention_weight_max": 49.41334533691406,
      "activations/layer17_attention_weight_min": -43.36052703857422,
      "activations/layer18_attention_weight_max": 45.27281188964844,
      "activations/layer18_attention_weight_min": -41.83649826049805,
      "activations/layer19_attention_weight_max": 22.75580596923828,
      "activations/layer19_attention_weight_min": -24.1452693939209,
      "activations/layer1_attention_weight_max": 16.983125686645508,
      "activations/layer1_attention_weight_min": -16.06986427307129,
      "activations/layer20_attention_weight_max": 21.158418655395508,
      "activations/layer20_attention_weight_min": -24.64948081970215,
      "activations/layer21_attention_weight_max": 32.92781448364258,
      "activations/layer21_attention_weight_min": -28.309507369995117,
      "activations/layer22_attention_weight_max": 28.123905181884766,
      "activations/layer22_attention_weight_min": -25.883590698242188,
      "activations/layer23_attention_weight_max": 35.80533218383789,
      "activations/layer23_attention_weight_min": -26.32721710205078,
      "activations/layer2_attention_weight_max": 31.526792526245117,
      "activations/layer2_attention_weight_min": -30.016666412353516,
      "activations/layer3_attention_weight_max": 89.79142761230469,
      "activations/layer3_attention_weight_min": -88.12682342529297,
      "activations/layer4_attention_weight_max": 87.7015609741211,
      "activations/layer4_attention_weight_min": -86.59584045410156,
      "activations/layer5_attention_weight_max": 62.9569091796875,
      "activations/layer5_attention_weight_min": -72.8708267211914,
      "activations/layer6_attention_weight_max": 50.76675033569336,
      "activations/layer6_attention_weight_min": -51.68161392211914,
      "activations/layer7_attention_weight_max": 66.98804473876953,
      "activations/layer7_attention_weight_min": -65.57454681396484,
      "activations/layer8_attention_weight_max": 51.287681579589844,
      "activations/layer8_attention_weight_min": -56.066558837890625,
      "activations/layer9_attention_weight_max": 49.41597366333008,
      "activations/layer9_attention_weight_min": -47.552364349365234,
      "epoch": 20.62,
      "learning_rate": 1.7171590909090907e-05,
      "loss": 2.7318,
      "step": 354950
    },
    {
      "activations/layer0_attention_weight_max": 15.46180534362793,
      "activations/layer0_attention_weight_min": -13.900585174560547,
      "activations/layer10_attention_weight_max": 37.82351303100586,
      "activations/layer10_attention_weight_min": -36.75172424316406,
      "activations/layer11_attention_weight_max": 33.85325622558594,
      "activations/layer11_attention_weight_min": -36.81034851074219,
      "activations/layer12_attention_weight_max": 19.470327377319336,
      "activations/layer12_attention_weight_min": -23.678760528564453,
      "activations/layer13_attention_weight_max": 34.83470916748047,
      "activations/layer13_attention_weight_min": -31.325674057006836,
      "activations/layer14_attention_weight_max": 38.42699432373047,
      "activations/layer14_attention_weight_min": -31.36172103881836,
      "activations/layer15_attention_weight_max": 35.05330276489258,
      "activations/layer15_attention_weight_min": -30.994678497314453,
      "activations/layer16_attention_weight_max": 36.82992172241211,
      "activations/layer16_attention_weight_min": -34.14236068725586,
      "activations/layer17_attention_weight_max": 54.78633117675781,
      "activations/layer17_attention_weight_min": -48.64023971557617,
      "activations/layer18_attention_weight_max": 53.056854248046875,
      "activations/layer18_attention_weight_min": -41.4034538269043,
      "activations/layer19_attention_weight_max": 25.24205207824707,
      "activations/layer19_attention_weight_min": -21.662988662719727,
      "activations/layer1_attention_weight_max": 14.977835655212402,
      "activations/layer1_attention_weight_min": -14.632508277893066,
      "activations/layer20_attention_weight_max": 24.293682098388672,
      "activations/layer20_attention_weight_min": -21.139720916748047,
      "activations/layer21_attention_weight_max": 33.40174865722656,
      "activations/layer21_attention_weight_min": -24.355131149291992,
      "activations/layer22_attention_weight_max": 31.077621459960938,
      "activations/layer22_attention_weight_min": -26.40443229675293,
      "activations/layer23_attention_weight_max": 41.91869354248047,
      "activations/layer23_attention_weight_min": -25.65145492553711,
      "activations/layer2_attention_weight_max": 30.64194107055664,
      "activations/layer2_attention_weight_min": -29.97368621826172,
      "activations/layer3_attention_weight_max": 86.06953430175781,
      "activations/layer3_attention_weight_min": -86.06685638427734,
      "activations/layer4_attention_weight_max": 83.78588104248047,
      "activations/layer4_attention_weight_min": -83.3170394897461,
      "activations/layer5_attention_weight_max": 66.93710327148438,
      "activations/layer5_attention_weight_min": -71.84124755859375,
      "activations/layer6_attention_weight_max": 49.39458465576172,
      "activations/layer6_attention_weight_min": -51.93570327758789,
      "activations/layer7_attention_weight_max": 63.93880081176758,
      "activations/layer7_attention_weight_min": -67.13742065429688,
      "activations/layer8_attention_weight_max": 47.219764709472656,
      "activations/layer8_attention_weight_min": -52.88379669189453,
      "activations/layer9_attention_weight_max": 43.65620040893555,
      "activations/layer9_attention_weight_min": -45.894775390625,
      "epoch": 20.63,
      "learning_rate": 1.7152651515151515e-05,
      "loss": 2.736,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.5534,
      "eval_samples_per_second": 502.024,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.5534,
      "eval_openwebtext_samples_per_second": 502.024,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.991,
      "eval_wikitext_samples_per_second": 229.036,
      "step": 355000
    },
    {
      "epoch": 20.63,
      "eval_lambada_loss": 2.416015625,
      "eval_lambada_ppl": 11.201140741117625,
      "eval_lambada_runtime": 9.5984,
      "eval_lambada_samples_per_second": 507.273,
      "step": 355000
    },
    {
      "activations/layer0_attention_weight_max": 15.830717086791992,
      "activations/layer0_attention_weight_min": -14.070663452148438,
      "activations/layer10_attention_weight_max": 37.12567901611328,
      "activations/layer10_attention_weight_min": -35.380645751953125,
      "activations/layer11_attention_weight_max": 33.58208465576172,
      "activations/layer11_attention_weight_min": -34.38315200805664,
      "activations/layer12_attention_weight_max": 19.102109909057617,
      "activations/layer12_attention_weight_min": -25.827083587646484,
      "activations/layer13_attention_weight_max": 33.697994232177734,
      "activations/layer13_attention_weight_min": -29.985965728759766,
      "activations/layer14_attention_weight_max": 41.154319763183594,
      "activations/layer14_attention_weight_min": -30.5961971282959,
      "activations/layer15_attention_weight_max": 39.68397521972656,
      "activations/layer15_attention_weight_min": -30.741243362426758,
      "activations/layer16_attention_weight_max": 33.82461166381836,
      "activations/layer16_attention_weight_min": -31.371862411499023,
      "activations/layer17_attention_weight_max": 49.545753479003906,
      "activations/layer17_attention_weight_min": -46.3010368347168,
      "activations/layer18_attention_weight_max": 46.060203552246094,
      "activations/layer18_attention_weight_min": -44.10382843017578,
      "activations/layer19_attention_weight_max": 25.6023006439209,
      "activations/layer19_attention_weight_min": -27.634077072143555,
      "activations/layer1_attention_weight_max": 15.24322509765625,
      "activations/layer1_attention_weight_min": -14.973220825195312,
      "activations/layer20_attention_weight_max": 27.660545349121094,
      "activations/layer20_attention_weight_min": -21.381813049316406,
      "activations/layer21_attention_weight_max": 39.10594177246094,
      "activations/layer21_attention_weight_min": -24.67545509338379,
      "activations/layer22_attention_weight_max": 31.017011642456055,
      "activations/layer22_attention_weight_min": -25.47317123413086,
      "activations/layer23_attention_weight_max": 35.031402587890625,
      "activations/layer23_attention_weight_min": -24.41522216796875,
      "activations/layer2_attention_weight_max": 31.930845260620117,
      "activations/layer2_attention_weight_min": -31.376968383789062,
      "activations/layer3_attention_weight_max": 89.38509368896484,
      "activations/layer3_attention_weight_min": -90.23253631591797,
      "activations/layer4_attention_weight_max": 88.9379653930664,
      "activations/layer4_attention_weight_min": -84.62329864501953,
      "activations/layer5_attention_weight_max": 62.90697479248047,
      "activations/layer5_attention_weight_min": -74.4864730834961,
      "activations/layer6_attention_weight_max": 49.498023986816406,
      "activations/layer6_attention_weight_min": -50.87562561035156,
      "activations/layer7_attention_weight_max": 66.32125091552734,
      "activations/layer7_attention_weight_min": -64.49454498291016,
      "activations/layer8_attention_weight_max": 45.214969635009766,
      "activations/layer8_attention_weight_min": -50.86579895019531,
      "activations/layer9_attention_weight_max": 46.052207946777344,
      "activations/layer9_attention_weight_min": -47.41762924194336,
      "epoch": 20.63,
      "learning_rate": 1.713371212121212e-05,
      "loss": 2.6991,
      "step": 355050
    },
    {
      "activations/layer0_attention_weight_max": 16.062034606933594,
      "activations/layer0_attention_weight_min": -14.01561450958252,
      "activations/layer10_attention_weight_max": 35.670562744140625,
      "activations/layer10_attention_weight_min": -35.094581604003906,
      "activations/layer11_attention_weight_max": 36.53031539916992,
      "activations/layer11_attention_weight_min": -34.00398635864258,
      "activations/layer12_attention_weight_max": 16.779991149902344,
      "activations/layer12_attention_weight_min": -27.09246253967285,
      "activations/layer13_attention_weight_max": 32.18232345581055,
      "activations/layer13_attention_weight_min": -27.82073402404785,
      "activations/layer14_attention_weight_max": 32.447906494140625,
      "activations/layer14_attention_weight_min": -28.21074104309082,
      "activations/layer15_attention_weight_max": 30.973285675048828,
      "activations/layer15_attention_weight_min": -29.875272750854492,
      "activations/layer16_attention_weight_max": 31.235782623291016,
      "activations/layer16_attention_weight_min": -30.28784942626953,
      "activations/layer17_attention_weight_max": 44.79301834106445,
      "activations/layer17_attention_weight_min": -43.05430603027344,
      "activations/layer18_attention_weight_max": 43.826316833496094,
      "activations/layer18_attention_weight_min": -38.994163513183594,
      "activations/layer19_attention_weight_max": 22.115842819213867,
      "activations/layer19_attention_weight_min": -22.761585235595703,
      "activations/layer1_attention_weight_max": 14.953192710876465,
      "activations/layer1_attention_weight_min": -13.501351356506348,
      "activations/layer20_attention_weight_max": 22.691484451293945,
      "activations/layer20_attention_weight_min": -21.362831115722656,
      "activations/layer21_attention_weight_max": 32.67509078979492,
      "activations/layer21_attention_weight_min": -23.766830444335938,
      "activations/layer22_attention_weight_max": 24.938339233398438,
      "activations/layer22_attention_weight_min": -26.23356056213379,
      "activations/layer23_attention_weight_max": 34.60601043701172,
      "activations/layer23_attention_weight_min": -23.131208419799805,
      "activations/layer2_attention_weight_max": 31.855037689208984,
      "activations/layer2_attention_weight_min": -29.729251861572266,
      "activations/layer3_attention_weight_max": 90.35890197753906,
      "activations/layer3_attention_weight_min": -86.19012451171875,
      "activations/layer4_attention_weight_max": 87.0255355834961,
      "activations/layer4_attention_weight_min": -82.4626235961914,
      "activations/layer5_attention_weight_max": 65.08482360839844,
      "activations/layer5_attention_weight_min": -71.80490112304688,
      "activations/layer6_attention_weight_max": 49.836952209472656,
      "activations/layer6_attention_weight_min": -47.46583938598633,
      "activations/layer7_attention_weight_max": 67.11180877685547,
      "activations/layer7_attention_weight_min": -62.89314651489258,
      "activations/layer8_attention_weight_max": 45.315887451171875,
      "activations/layer8_attention_weight_min": -48.133968353271484,
      "activations/layer9_attention_weight_max": 41.191097259521484,
      "activations/layer9_attention_weight_min": -45.0201301574707,
      "epoch": 20.63,
      "learning_rate": 1.7114772727272725e-05,
      "loss": 2.7161,
      "step": 355100
    },
    {
      "activations/layer0_attention_weight_max": 15.833176612854004,
      "activations/layer0_attention_weight_min": -13.958697319030762,
      "activations/layer10_attention_weight_max": 36.66705322265625,
      "activations/layer10_attention_weight_min": -33.843692779541016,
      "activations/layer11_attention_weight_max": 33.22389602661133,
      "activations/layer11_attention_weight_min": -34.622398376464844,
      "activations/layer12_attention_weight_max": 18.83314323425293,
      "activations/layer12_attention_weight_min": -29.164222717285156,
      "activations/layer13_attention_weight_max": 34.73509216308594,
      "activations/layer13_attention_weight_min": -26.616178512573242,
      "activations/layer14_attention_weight_max": 39.35210037231445,
      "activations/layer14_attention_weight_min": -29.863500595092773,
      "activations/layer15_attention_weight_max": 39.21748733520508,
      "activations/layer15_attention_weight_min": -29.98680877685547,
      "activations/layer16_attention_weight_max": 35.455169677734375,
      "activations/layer16_attention_weight_min": -32.13804244995117,
      "activations/layer17_attention_weight_max": 49.970306396484375,
      "activations/layer17_attention_weight_min": -44.78157424926758,
      "activations/layer18_attention_weight_max": 48.78912353515625,
      "activations/layer18_attention_weight_min": -40.037078857421875,
      "activations/layer19_attention_weight_max": 24.17265510559082,
      "activations/layer19_attention_weight_min": -22.427173614501953,
      "activations/layer1_attention_weight_max": 16.03831672668457,
      "activations/layer1_attention_weight_min": -15.585124969482422,
      "activations/layer20_attention_weight_max": 22.052478790283203,
      "activations/layer20_attention_weight_min": -21.717485427856445,
      "activations/layer21_attention_weight_max": 38.3310546875,
      "activations/layer21_attention_weight_min": -23.55765151977539,
      "activations/layer22_attention_weight_max": 28.29947853088379,
      "activations/layer22_attention_weight_min": -28.23346710205078,
      "activations/layer23_attention_weight_max": 37.15937042236328,
      "activations/layer23_attention_weight_min": -24.896289825439453,
      "activations/layer2_attention_weight_max": 30.863086700439453,
      "activations/layer2_attention_weight_min": -29.553159713745117,
      "activations/layer3_attention_weight_max": 86.5005111694336,
      "activations/layer3_attention_weight_min": -85.67464447021484,
      "activations/layer4_attention_weight_max": 83.61365509033203,
      "activations/layer4_attention_weight_min": -83.88823699951172,
      "activations/layer5_attention_weight_max": 61.304012298583984,
      "activations/layer5_attention_weight_min": -74.36018371582031,
      "activations/layer6_attention_weight_max": 50.585853576660156,
      "activations/layer6_attention_weight_min": -49.576717376708984,
      "activations/layer7_attention_weight_max": 62.32398986816406,
      "activations/layer7_attention_weight_min": -65.176025390625,
      "activations/layer8_attention_weight_max": 48.718910217285156,
      "activations/layer8_attention_weight_min": -48.19554138183594,
      "activations/layer9_attention_weight_max": 43.32944869995117,
      "activations/layer9_attention_weight_min": -44.46091842651367,
      "epoch": 20.64,
      "learning_rate": 1.7095833333333333e-05,
      "loss": 2.7142,
      "step": 355150
    },
    {
      "activations/layer0_attention_weight_max": 15.113859176635742,
      "activations/layer0_attention_weight_min": -13.968605041503906,
      "activations/layer10_attention_weight_max": 35.9248046875,
      "activations/layer10_attention_weight_min": -35.627418518066406,
      "activations/layer11_attention_weight_max": 32.64314270019531,
      "activations/layer11_attention_weight_min": -34.424163818359375,
      "activations/layer12_attention_weight_max": 18.26481056213379,
      "activations/layer12_attention_weight_min": -24.648038864135742,
      "activations/layer13_attention_weight_max": 30.93728256225586,
      "activations/layer13_attention_weight_min": -30.06422996520996,
      "activations/layer14_attention_weight_max": 36.24452209472656,
      "activations/layer14_attention_weight_min": -32.73714065551758,
      "activations/layer15_attention_weight_max": 36.013999938964844,
      "activations/layer15_attention_weight_min": -31.71310806274414,
      "activations/layer16_attention_weight_max": 35.2396354675293,
      "activations/layer16_attention_weight_min": -33.80164337158203,
      "activations/layer17_attention_weight_max": 53.38090133666992,
      "activations/layer17_attention_weight_min": -49.66926956176758,
      "activations/layer18_attention_weight_max": 51.08281707763672,
      "activations/layer18_attention_weight_min": -43.33983612060547,
      "activations/layer19_attention_weight_max": 27.567432403564453,
      "activations/layer19_attention_weight_min": -24.218252182006836,
      "activations/layer1_attention_weight_max": 16.129457473754883,
      "activations/layer1_attention_weight_min": -13.910493850708008,
      "activations/layer20_attention_weight_max": 25.862308502197266,
      "activations/layer20_attention_weight_min": -22.139741897583008,
      "activations/layer21_attention_weight_max": 38.93391418457031,
      "activations/layer21_attention_weight_min": -19.570066452026367,
      "activations/layer22_attention_weight_max": 36.03494644165039,
      "activations/layer22_attention_weight_min": -25.547801971435547,
      "activations/layer23_attention_weight_max": 36.4570198059082,
      "activations/layer23_attention_weight_min": -23.893028259277344,
      "activations/layer2_attention_weight_max": 31.07428550720215,
      "activations/layer2_attention_weight_min": -28.72100067138672,
      "activations/layer3_attention_weight_max": 87.57543182373047,
      "activations/layer3_attention_weight_min": -86.49443817138672,
      "activations/layer4_attention_weight_max": 82.9427490234375,
      "activations/layer4_attention_weight_min": -80.81473541259766,
      "activations/layer5_attention_weight_max": 60.55259704589844,
      "activations/layer5_attention_weight_min": -70.72525787353516,
      "activations/layer6_attention_weight_max": 47.21934509277344,
      "activations/layer6_attention_weight_min": -48.59498977661133,
      "activations/layer7_attention_weight_max": 68.86837005615234,
      "activations/layer7_attention_weight_min": -64.78527069091797,
      "activations/layer8_attention_weight_max": 46.13218688964844,
      "activations/layer8_attention_weight_min": -49.322818756103516,
      "activations/layer9_attention_weight_max": 42.258331298828125,
      "activations/layer9_attention_weight_min": -46.234127044677734,
      "epoch": 20.64,
      "learning_rate": 1.7076893939393938e-05,
      "loss": 2.7217,
      "step": 355200
    },
    {
      "activations/layer0_attention_weight_max": 15.904459953308105,
      "activations/layer0_attention_weight_min": -14.296245574951172,
      "activations/layer10_attention_weight_max": 35.483341217041016,
      "activations/layer10_attention_weight_min": -34.05973434448242,
      "activations/layer11_attention_weight_max": 31.583091735839844,
      "activations/layer11_attention_weight_min": -34.317161560058594,
      "activations/layer12_attention_weight_max": 20.36065673828125,
      "activations/layer12_attention_weight_min": -23.982242584228516,
      "activations/layer13_attention_weight_max": 32.443851470947266,
      "activations/layer13_attention_weight_min": -28.408199310302734,
      "activations/layer14_attention_weight_max": 33.221221923828125,
      "activations/layer14_attention_weight_min": -31.859371185302734,
      "activations/layer15_attention_weight_max": 30.731258392333984,
      "activations/layer15_attention_weight_min": -32.409183502197266,
      "activations/layer16_attention_weight_max": 33.256553649902344,
      "activations/layer16_attention_weight_min": -32.6397819519043,
      "activations/layer17_attention_weight_max": 50.47724151611328,
      "activations/layer17_attention_weight_min": -45.33900833129883,
      "activations/layer18_attention_weight_max": 45.282222747802734,
      "activations/layer18_attention_weight_min": -41.57460403442383,
      "activations/layer19_attention_weight_max": 22.134197235107422,
      "activations/layer19_attention_weight_min": -24.18114471435547,
      "activations/layer1_attention_weight_max": 15.074644088745117,
      "activations/layer1_attention_weight_min": -13.619842529296875,
      "activations/layer20_attention_weight_max": 20.419912338256836,
      "activations/layer20_attention_weight_min": -23.33873748779297,
      "activations/layer21_attention_weight_max": 33.47135925292969,
      "activations/layer21_attention_weight_min": -19.587265014648438,
      "activations/layer22_attention_weight_max": 28.042776107788086,
      "activations/layer22_attention_weight_min": -26.028642654418945,
      "activations/layer23_attention_weight_max": 32.02149963378906,
      "activations/layer23_attention_weight_min": -24.486530303955078,
      "activations/layer2_attention_weight_max": 31.130245208740234,
      "activations/layer2_attention_weight_min": -29.351865768432617,
      "activations/layer3_attention_weight_max": 85.41088104248047,
      "activations/layer3_attention_weight_min": -87.14447021484375,
      "activations/layer4_attention_weight_max": 80.70341491699219,
      "activations/layer4_attention_weight_min": -82.07064819335938,
      "activations/layer5_attention_weight_max": 63.63246154785156,
      "activations/layer5_attention_weight_min": -71.77177429199219,
      "activations/layer6_attention_weight_max": 47.82707214355469,
      "activations/layer6_attention_weight_min": -49.159507751464844,
      "activations/layer7_attention_weight_max": 66.4896469116211,
      "activations/layer7_attention_weight_min": -62.37933349609375,
      "activations/layer8_attention_weight_max": 44.03911590576172,
      "activations/layer8_attention_weight_min": -47.36553955078125,
      "activations/layer9_attention_weight_max": 41.51462173461914,
      "activations/layer9_attention_weight_min": -42.31679916381836,
      "epoch": 20.64,
      "learning_rate": 1.7057954545454543e-05,
      "loss": 2.7241,
      "step": 355250
    },
    {
      "activations/layer0_attention_weight_max": 14.819246292114258,
      "activations/layer0_attention_weight_min": -14.21360969543457,
      "activations/layer10_attention_weight_max": 37.42266845703125,
      "activations/layer10_attention_weight_min": -35.591331481933594,
      "activations/layer11_attention_weight_max": 37.417057037353516,
      "activations/layer11_attention_weight_min": -34.84117126464844,
      "activations/layer12_attention_weight_max": 18.543960571289062,
      "activations/layer12_attention_weight_min": -24.63701057434082,
      "activations/layer13_attention_weight_max": 33.39094543457031,
      "activations/layer13_attention_weight_min": -32.17603302001953,
      "activations/layer14_attention_weight_max": 38.323211669921875,
      "activations/layer14_attention_weight_min": -34.58937454223633,
      "activations/layer15_attention_weight_max": 38.75692367553711,
      "activations/layer15_attention_weight_min": -35.01508712768555,
      "activations/layer16_attention_weight_max": 37.93223190307617,
      "activations/layer16_attention_weight_min": -35.107582092285156,
      "activations/layer17_attention_weight_max": 55.63254165649414,
      "activations/layer17_attention_weight_min": -50.60640335083008,
      "activations/layer18_attention_weight_max": 53.90134048461914,
      "activations/layer18_attention_weight_min": -42.62629699707031,
      "activations/layer19_attention_weight_max": 27.30029296875,
      "activations/layer19_attention_weight_min": -24.648962020874023,
      "activations/layer1_attention_weight_max": 15.039194107055664,
      "activations/layer1_attention_weight_min": -14.41597843170166,
      "activations/layer20_attention_weight_max": 26.53966522216797,
      "activations/layer20_attention_weight_min": -23.469778060913086,
      "activations/layer21_attention_weight_max": 37.78516387939453,
      "activations/layer21_attention_weight_min": -23.926843643188477,
      "activations/layer22_attention_weight_max": 30.265094757080078,
      "activations/layer22_attention_weight_min": -28.42045021057129,
      "activations/layer23_attention_weight_max": 41.0168342590332,
      "activations/layer23_attention_weight_min": -24.23780059814453,
      "activations/layer2_attention_weight_max": 30.838993072509766,
      "activations/layer2_attention_weight_min": -29.97272300720215,
      "activations/layer3_attention_weight_max": 86.8447265625,
      "activations/layer3_attention_weight_min": -87.56620025634766,
      "activations/layer4_attention_weight_max": 85.90475463867188,
      "activations/layer4_attention_weight_min": -83.96959686279297,
      "activations/layer5_attention_weight_max": 62.18632507324219,
      "activations/layer5_attention_weight_min": -73.77458190917969,
      "activations/layer6_attention_weight_max": 52.23677444458008,
      "activations/layer6_attention_weight_min": -55.14549255371094,
      "activations/layer7_attention_weight_max": 67.98442077636719,
      "activations/layer7_attention_weight_min": -69.92095947265625,
      "activations/layer8_attention_weight_max": 49.512577056884766,
      "activations/layer8_attention_weight_min": -53.37177276611328,
      "activations/layer9_attention_weight_max": 45.582305908203125,
      "activations/layer9_attention_weight_min": -47.88743591308594,
      "epoch": 20.64,
      "learning_rate": 1.703901515151515e-05,
      "loss": 2.7014,
      "step": 355300
    },
    {
      "activations/layer0_attention_weight_max": 15.865391731262207,
      "activations/layer0_attention_weight_min": -14.720675468444824,
      "activations/layer10_attention_weight_max": 36.00768280029297,
      "activations/layer10_attention_weight_min": -36.43097686767578,
      "activations/layer11_attention_weight_max": 35.283599853515625,
      "activations/layer11_attention_weight_min": -34.957740783691406,
      "activations/layer12_attention_weight_max": 22.178226470947266,
      "activations/layer12_attention_weight_min": -24.340097427368164,
      "activations/layer13_attention_weight_max": 33.616573333740234,
      "activations/layer13_attention_weight_min": -28.088953018188477,
      "activations/layer14_attention_weight_max": 35.07553482055664,
      "activations/layer14_attention_weight_min": -28.977155685424805,
      "activations/layer15_attention_weight_max": 33.3712272644043,
      "activations/layer15_attention_weight_min": -30.563302993774414,
      "activations/layer16_attention_weight_max": 33.86153030395508,
      "activations/layer16_attention_weight_min": -32.302791595458984,
      "activations/layer17_attention_weight_max": 50.18324661254883,
      "activations/layer17_attention_weight_min": -42.64873123168945,
      "activations/layer18_attention_weight_max": 45.45549392700195,
      "activations/layer18_attention_weight_min": -39.376468658447266,
      "activations/layer19_attention_weight_max": 21.645368576049805,
      "activations/layer19_attention_weight_min": -22.301023483276367,
      "activations/layer1_attention_weight_max": 15.802115440368652,
      "activations/layer1_attention_weight_min": -13.900897979736328,
      "activations/layer20_attention_weight_max": 20.35460662841797,
      "activations/layer20_attention_weight_min": -21.815776824951172,
      "activations/layer21_attention_weight_max": 33.1980094909668,
      "activations/layer21_attention_weight_min": -20.245471954345703,
      "activations/layer22_attention_weight_max": 26.844432830810547,
      "activations/layer22_attention_weight_min": -24.54439926147461,
      "activations/layer23_attention_weight_max": 33.347930908203125,
      "activations/layer23_attention_weight_min": -26.866971969604492,
      "activations/layer2_attention_weight_max": 32.6561279296875,
      "activations/layer2_attention_weight_min": -29.511510848999023,
      "activations/layer3_attention_weight_max": 86.806640625,
      "activations/layer3_attention_weight_min": -88.3582992553711,
      "activations/layer4_attention_weight_max": 82.95096588134766,
      "activations/layer4_attention_weight_min": -83.6239013671875,
      "activations/layer5_attention_weight_max": 60.30117416381836,
      "activations/layer5_attention_weight_min": -73.36177062988281,
      "activations/layer6_attention_weight_max": 49.11934280395508,
      "activations/layer6_attention_weight_min": -50.11193084716797,
      "activations/layer7_attention_weight_max": 63.56355667114258,
      "activations/layer7_attention_weight_min": -66.47798919677734,
      "activations/layer8_attention_weight_max": 47.10358428955078,
      "activations/layer8_attention_weight_min": -49.552486419677734,
      "activations/layer9_attention_weight_max": 44.0791015625,
      "activations/layer9_attention_weight_min": -45.15973663330078,
      "epoch": 20.65,
      "learning_rate": 1.7020075757575756e-05,
      "loss": 2.712,
      "step": 355350
    },
    {
      "activations/layer0_attention_weight_max": 15.887396812438965,
      "activations/layer0_attention_weight_min": -13.982510566711426,
      "activations/layer10_attention_weight_max": 38.10789108276367,
      "activations/layer10_attention_weight_min": -36.53416061401367,
      "activations/layer11_attention_weight_max": 33.28230285644531,
      "activations/layer11_attention_weight_min": -37.09373474121094,
      "activations/layer12_attention_weight_max": 19.656177520751953,
      "activations/layer12_attention_weight_min": -23.659622192382812,
      "activations/layer13_attention_weight_max": 37.51097106933594,
      "activations/layer13_attention_weight_min": -33.16371154785156,
      "activations/layer14_attention_weight_max": 35.10918045043945,
      "activations/layer14_attention_weight_min": -31.254119873046875,
      "activations/layer15_attention_weight_max": 32.2368278503418,
      "activations/layer15_attention_weight_min": -31.288227081298828,
      "activations/layer16_attention_weight_max": 34.36270523071289,
      "activations/layer16_attention_weight_min": -32.701072692871094,
      "activations/layer17_attention_weight_max": 53.99964904785156,
      "activations/layer17_attention_weight_min": -47.02434158325195,
      "activations/layer18_attention_weight_max": 47.228633880615234,
      "activations/layer18_attention_weight_min": -45.03995132446289,
      "activations/layer19_attention_weight_max": 25.54305648803711,
      "activations/layer19_attention_weight_min": -25.600515365600586,
      "activations/layer1_attention_weight_max": 15.36178970336914,
      "activations/layer1_attention_weight_min": -14.4525146484375,
      "activations/layer20_attention_weight_max": 25.47627830505371,
      "activations/layer20_attention_weight_min": -24.470365524291992,
      "activations/layer21_attention_weight_max": 34.383567810058594,
      "activations/layer21_attention_weight_min": -23.00979995727539,
      "activations/layer22_attention_weight_max": 30.933292388916016,
      "activations/layer22_attention_weight_min": -26.421031951904297,
      "activations/layer23_attention_weight_max": 38.365753173828125,
      "activations/layer23_attention_weight_min": -28.688182830810547,
      "activations/layer2_attention_weight_max": 31.38182830810547,
      "activations/layer2_attention_weight_min": -30.800071716308594,
      "activations/layer3_attention_weight_max": 91.2425308227539,
      "activations/layer3_attention_weight_min": -93.99474334716797,
      "activations/layer4_attention_weight_max": 86.23058319091797,
      "activations/layer4_attention_weight_min": -85.05760955810547,
      "activations/layer5_attention_weight_max": 64.1227035522461,
      "activations/layer5_attention_weight_min": -71.54110717773438,
      "activations/layer6_attention_weight_max": 49.95561599731445,
      "activations/layer6_attention_weight_min": -51.594181060791016,
      "activations/layer7_attention_weight_max": 66.96146392822266,
      "activations/layer7_attention_weight_min": -67.13460540771484,
      "activations/layer8_attention_weight_max": 49.79317092895508,
      "activations/layer8_attention_weight_min": -52.03416061401367,
      "activations/layer9_attention_weight_max": 49.99052810668945,
      "activations/layer9_attention_weight_min": -48.11357116699219,
      "epoch": 20.65,
      "learning_rate": 1.700113636363636e-05,
      "loss": 2.7101,
      "step": 355400
    },
    {
      "activations/layer0_attention_weight_max": 15.977508544921875,
      "activations/layer0_attention_weight_min": -14.27155590057373,
      "activations/layer10_attention_weight_max": 39.12139892578125,
      "activations/layer10_attention_weight_min": -38.2091064453125,
      "activations/layer11_attention_weight_max": 36.05422592163086,
      "activations/layer11_attention_weight_min": -37.01493835449219,
      "activations/layer12_attention_weight_max": 18.171091079711914,
      "activations/layer12_attention_weight_min": -27.89019203186035,
      "activations/layer13_attention_weight_max": 37.36750411987305,
      "activations/layer13_attention_weight_min": -29.163551330566406,
      "activations/layer14_attention_weight_max": 34.62523651123047,
      "activations/layer14_attention_weight_min": -30.90381622314453,
      "activations/layer15_attention_weight_max": 36.15182876586914,
      "activations/layer15_attention_weight_min": -30.877254486083984,
      "activations/layer16_attention_weight_max": 33.90952682495117,
      "activations/layer16_attention_weight_min": -32.194637298583984,
      "activations/layer17_attention_weight_max": 51.6334342956543,
      "activations/layer17_attention_weight_min": -47.1856575012207,
      "activations/layer18_attention_weight_max": 45.945213317871094,
      "activations/layer18_attention_weight_min": -41.90058135986328,
      "activations/layer19_attention_weight_max": 23.91075325012207,
      "activations/layer19_attention_weight_min": -23.920148849487305,
      "activations/layer1_attention_weight_max": 16.3321475982666,
      "activations/layer1_attention_weight_min": -13.25098991394043,
      "activations/layer20_attention_weight_max": 22.505083084106445,
      "activations/layer20_attention_weight_min": -23.403928756713867,
      "activations/layer21_attention_weight_max": 37.30303955078125,
      "activations/layer21_attention_weight_min": -24.275609970092773,
      "activations/layer22_attention_weight_max": 25.92140007019043,
      "activations/layer22_attention_weight_min": -24.52523422241211,
      "activations/layer23_attention_weight_max": 36.69529724121094,
      "activations/layer23_attention_weight_min": -23.424163818359375,
      "activations/layer2_attention_weight_max": 31.090587615966797,
      "activations/layer2_attention_weight_min": -30.814123153686523,
      "activations/layer3_attention_weight_max": 88.87151336669922,
      "activations/layer3_attention_weight_min": -90.01301574707031,
      "activations/layer4_attention_weight_max": 84.93840789794922,
      "activations/layer4_attention_weight_min": -86.72637939453125,
      "activations/layer5_attention_weight_max": 62.64745330810547,
      "activations/layer5_attention_weight_min": -68.46054077148438,
      "activations/layer6_attention_weight_max": 49.19180679321289,
      "activations/layer6_attention_weight_min": -50.768943786621094,
      "activations/layer7_attention_weight_max": 63.68723678588867,
      "activations/layer7_attention_weight_min": -65.00230407714844,
      "activations/layer8_attention_weight_max": 48.278743743896484,
      "activations/layer8_attention_weight_min": -50.80780792236328,
      "activations/layer9_attention_weight_max": 47.52140808105469,
      "activations/layer9_attention_weight_min": -45.701290130615234,
      "epoch": 20.65,
      "learning_rate": 1.698219696969697e-05,
      "loss": 2.7094,
      "step": 355450
    },
    {
      "activations/layer0_attention_weight_max": 15.730752944946289,
      "activations/layer0_attention_weight_min": -14.133378028869629,
      "activations/layer10_attention_weight_max": 34.229644775390625,
      "activations/layer10_attention_weight_min": -35.092159271240234,
      "activations/layer11_attention_weight_max": 32.871768951416016,
      "activations/layer11_attention_weight_min": -34.081871032714844,
      "activations/layer12_attention_weight_max": 17.460756301879883,
      "activations/layer12_attention_weight_min": -28.022005081176758,
      "activations/layer13_attention_weight_max": 31.443592071533203,
      "activations/layer13_attention_weight_min": -26.865488052368164,
      "activations/layer14_attention_weight_max": 33.66618728637695,
      "activations/layer14_attention_weight_min": -32.113948822021484,
      "activations/layer15_attention_weight_max": 31.145307540893555,
      "activations/layer15_attention_weight_min": -31.772775650024414,
      "activations/layer16_attention_weight_max": 31.335506439208984,
      "activations/layer16_attention_weight_min": -33.36032485961914,
      "activations/layer17_attention_weight_max": 47.634979248046875,
      "activations/layer17_attention_weight_min": -43.74162673950195,
      "activations/layer18_attention_weight_max": 45.9517936706543,
      "activations/layer18_attention_weight_min": -39.288368225097656,
      "activations/layer19_attention_weight_max": 23.611631393432617,
      "activations/layer19_attention_weight_min": -23.1606502532959,
      "activations/layer1_attention_weight_max": 16.13767433166504,
      "activations/layer1_attention_weight_min": -13.747835159301758,
      "activations/layer20_attention_weight_max": 23.05866050720215,
      "activations/layer20_attention_weight_min": -20.09303092956543,
      "activations/layer21_attention_weight_max": 35.995643615722656,
      "activations/layer21_attention_weight_min": -21.98420524597168,
      "activations/layer22_attention_weight_max": 28.86992645263672,
      "activations/layer22_attention_weight_min": -24.435964584350586,
      "activations/layer23_attention_weight_max": 33.791778564453125,
      "activations/layer23_attention_weight_min": -24.531230926513672,
      "activations/layer2_attention_weight_max": 30.634052276611328,
      "activations/layer2_attention_weight_min": -29.56848907470703,
      "activations/layer3_attention_weight_max": 88.9874038696289,
      "activations/layer3_attention_weight_min": -84.95053100585938,
      "activations/layer4_attention_weight_max": 83.24852752685547,
      "activations/layer4_attention_weight_min": -81.71550750732422,
      "activations/layer5_attention_weight_max": 60.62722396850586,
      "activations/layer5_attention_weight_min": -71.2503662109375,
      "activations/layer6_attention_weight_max": 47.08634567260742,
      "activations/layer6_attention_weight_min": -49.70665740966797,
      "activations/layer7_attention_weight_max": 64.0611572265625,
      "activations/layer7_attention_weight_min": -64.25464630126953,
      "activations/layer8_attention_weight_max": 46.39023971557617,
      "activations/layer8_attention_weight_min": -50.78254318237305,
      "activations/layer9_attention_weight_max": 43.648956298828125,
      "activations/layer9_attention_weight_min": -46.833370208740234,
      "epoch": 20.66,
      "learning_rate": 1.6963257575757574e-05,
      "loss": 2.7029,
      "step": 355500
    },
    {
      "activations/layer0_attention_weight_max": 15.876042366027832,
      "activations/layer0_attention_weight_min": -14.204756736755371,
      "activations/layer10_attention_weight_max": 50.86286544799805,
      "activations/layer10_attention_weight_min": -48.153106689453125,
      "activations/layer11_attention_weight_max": 49.643516540527344,
      "activations/layer11_attention_weight_min": -45.053924560546875,
      "activations/layer12_attention_weight_max": 23.74527931213379,
      "activations/layer12_attention_weight_min": -25.97324562072754,
      "activations/layer13_attention_weight_max": 52.693965911865234,
      "activations/layer13_attention_weight_min": -31.53183364868164,
      "activations/layer14_attention_weight_max": 54.74496841430664,
      "activations/layer14_attention_weight_min": -35.713619232177734,
      "activations/layer15_attention_weight_max": 60.45563888549805,
      "activations/layer15_attention_weight_min": -35.1644172668457,
      "activations/layer16_attention_weight_max": 43.01236343383789,
      "activations/layer16_attention_weight_min": -38.38044738769531,
      "activations/layer17_attention_weight_max": 67.4307861328125,
      "activations/layer17_attention_weight_min": -58.36426544189453,
      "activations/layer18_attention_weight_max": 56.9815559387207,
      "activations/layer18_attention_weight_min": -52.44391632080078,
      "activations/layer19_attention_weight_max": 28.75082778930664,
      "activations/layer19_attention_weight_min": -26.47736930847168,
      "activations/layer1_attention_weight_max": 15.432579040527344,
      "activations/layer1_attention_weight_min": -14.321547508239746,
      "activations/layer20_attention_weight_max": 30.037158966064453,
      "activations/layer20_attention_weight_min": -23.41570281982422,
      "activations/layer21_attention_weight_max": 49.19594192504883,
      "activations/layer21_attention_weight_min": -23.200204849243164,
      "activations/layer22_attention_weight_max": 36.567039489746094,
      "activations/layer22_attention_weight_min": -26.26040267944336,
      "activations/layer23_attention_weight_max": 38.58478546142578,
      "activations/layer23_attention_weight_min": -23.7918701171875,
      "activations/layer2_attention_weight_max": 33.74128723144531,
      "activations/layer2_attention_weight_min": -31.649044036865234,
      "activations/layer3_attention_weight_max": 100.8221435546875,
      "activations/layer3_attention_weight_min": -100.0896224975586,
      "activations/layer4_attention_weight_max": 94.59634399414062,
      "activations/layer4_attention_weight_min": -86.23332977294922,
      "activations/layer5_attention_weight_max": 68.82142639160156,
      "activations/layer5_attention_weight_min": -71.78752136230469,
      "activations/layer6_attention_weight_max": 50.33279037475586,
      "activations/layer6_attention_weight_min": -50.90696334838867,
      "activations/layer7_attention_weight_max": 84.83291625976562,
      "activations/layer7_attention_weight_min": -72.1708755493164,
      "activations/layer8_attention_weight_max": 64.398193359375,
      "activations/layer8_attention_weight_min": -59.65917205810547,
      "activations/layer9_attention_weight_max": 66.53185272216797,
      "activations/layer9_attention_weight_min": -57.38155746459961,
      "epoch": 20.66,
      "learning_rate": 1.694431818181818e-05,
      "loss": 2.7044,
      "step": 355550
    },
    {
      "activations/layer0_attention_weight_max": 15.41922664642334,
      "activations/layer0_attention_weight_min": -14.17471981048584,
      "activations/layer10_attention_weight_max": 36.65923309326172,
      "activations/layer10_attention_weight_min": -36.91890335083008,
      "activations/layer11_attention_weight_max": 32.46784210205078,
      "activations/layer11_attention_weight_min": -35.768592834472656,
      "activations/layer12_attention_weight_max": 19.268836975097656,
      "activations/layer12_attention_weight_min": -25.679445266723633,
      "activations/layer13_attention_weight_max": 32.984588623046875,
      "activations/layer13_attention_weight_min": -26.726484298706055,
      "activations/layer14_attention_weight_max": 35.44541549682617,
      "activations/layer14_attention_weight_min": -30.735994338989258,
      "activations/layer15_attention_weight_max": 32.530357360839844,
      "activations/layer15_attention_weight_min": -31.661157608032227,
      "activations/layer16_attention_weight_max": 35.42799377441406,
      "activations/layer16_attention_weight_min": -32.608543395996094,
      "activations/layer17_attention_weight_max": 51.61014175415039,
      "activations/layer17_attention_weight_min": -44.9460334777832,
      "activations/layer18_attention_weight_max": 45.92782974243164,
      "activations/layer18_attention_weight_min": -39.34098815917969,
      "activations/layer19_attention_weight_max": 25.10227394104004,
      "activations/layer19_attention_weight_min": -22.403959274291992,
      "activations/layer1_attention_weight_max": 15.401243209838867,
      "activations/layer1_attention_weight_min": -13.69801139831543,
      "activations/layer20_attention_weight_max": 22.91544532775879,
      "activations/layer20_attention_weight_min": -21.56414031982422,
      "activations/layer21_attention_weight_max": 35.99711990356445,
      "activations/layer21_attention_weight_min": -22.17437171936035,
      "activations/layer22_attention_weight_max": 32.959556579589844,
      "activations/layer22_attention_weight_min": -25.116071701049805,
      "activations/layer23_attention_weight_max": 36.84928894042969,
      "activations/layer23_attention_weight_min": -23.26553726196289,
      "activations/layer2_attention_weight_max": 30.34439468383789,
      "activations/layer2_attention_weight_min": -29.67696762084961,
      "activations/layer3_attention_weight_max": 87.51606750488281,
      "activations/layer3_attention_weight_min": -91.34964752197266,
      "activations/layer4_attention_weight_max": 83.21805572509766,
      "activations/layer4_attention_weight_min": -84.6589584350586,
      "activations/layer5_attention_weight_max": 61.078670501708984,
      "activations/layer5_attention_weight_min": -73.11322784423828,
      "activations/layer6_attention_weight_max": 48.91322708129883,
      "activations/layer6_attention_weight_min": -53.11320495605469,
      "activations/layer7_attention_weight_max": 64.19964599609375,
      "activations/layer7_attention_weight_min": -66.11792755126953,
      "activations/layer8_attention_weight_max": 46.598628997802734,
      "activations/layer8_attention_weight_min": -50.26557540893555,
      "activations/layer9_attention_weight_max": 44.49279022216797,
      "activations/layer9_attention_weight_min": -48.2418098449707,
      "epoch": 20.66,
      "learning_rate": 1.6925378787878787e-05,
      "loss": 2.7173,
      "step": 355600
    },
    {
      "activations/layer0_attention_weight_max": 15.403678894042969,
      "activations/layer0_attention_weight_min": -13.977231979370117,
      "activations/layer10_attention_weight_max": 38.37727355957031,
      "activations/layer10_attention_weight_min": -35.59248733520508,
      "activations/layer11_attention_weight_max": 33.117530822753906,
      "activations/layer11_attention_weight_min": -35.034423828125,
      "activations/layer12_attention_weight_max": 20.06147575378418,
      "activations/layer12_attention_weight_min": -23.676530838012695,
      "activations/layer13_attention_weight_max": 34.349178314208984,
      "activations/layer13_attention_weight_min": -30.35359764099121,
      "activations/layer14_attention_weight_max": 35.974365234375,
      "activations/layer14_attention_weight_min": -33.855899810791016,
      "activations/layer15_attention_weight_max": 32.88663101196289,
      "activations/layer15_attention_weight_min": -31.767623901367188,
      "activations/layer16_attention_weight_max": 37.60440444946289,
      "activations/layer16_attention_weight_min": -34.28501892089844,
      "activations/layer17_attention_weight_max": 53.53207015991211,
      "activations/layer17_attention_weight_min": -48.43119430541992,
      "activations/layer18_attention_weight_max": 46.7478141784668,
      "activations/layer18_attention_weight_min": -42.615867614746094,
      "activations/layer19_attention_weight_max": 23.222543716430664,
      "activations/layer19_attention_weight_min": -22.432790756225586,
      "activations/layer1_attention_weight_max": 15.819067001342773,
      "activations/layer1_attention_weight_min": -14.264392852783203,
      "activations/layer20_attention_weight_max": 23.409496307373047,
      "activations/layer20_attention_weight_min": -21.678918838500977,
      "activations/layer21_attention_weight_max": 31.72959327697754,
      "activations/layer21_attention_weight_min": -20.395164489746094,
      "activations/layer22_attention_weight_max": 28.246606826782227,
      "activations/layer22_attention_weight_min": -25.122610092163086,
      "activations/layer23_attention_weight_max": 39.991050720214844,
      "activations/layer23_attention_weight_min": -24.11676597595215,
      "activations/layer2_attention_weight_max": 31.035263061523438,
      "activations/layer2_attention_weight_min": -29.53374481201172,
      "activations/layer3_attention_weight_max": 87.83354949951172,
      "activations/layer3_attention_weight_min": -87.12651062011719,
      "activations/layer4_attention_weight_max": 84.19905853271484,
      "activations/layer4_attention_weight_min": -84.04948425292969,
      "activations/layer5_attention_weight_max": 62.01795959472656,
      "activations/layer5_attention_weight_min": -73.56907653808594,
      "activations/layer6_attention_weight_max": 49.12773513793945,
      "activations/layer6_attention_weight_min": -52.22632598876953,
      "activations/layer7_attention_weight_max": 67.9442138671875,
      "activations/layer7_attention_weight_min": -68.58354187011719,
      "activations/layer8_attention_weight_max": 49.22138977050781,
      "activations/layer8_attention_weight_min": -54.746620178222656,
      "activations/layer9_attention_weight_max": 46.74362564086914,
      "activations/layer9_attention_weight_min": -46.74650573730469,
      "epoch": 20.67,
      "learning_rate": 1.6906439393939392e-05,
      "loss": 2.7299,
      "step": 355650
    },
    {
      "activations/layer0_attention_weight_max": 16.015743255615234,
      "activations/layer0_attention_weight_min": -13.97766399383545,
      "activations/layer10_attention_weight_max": 36.4590950012207,
      "activations/layer10_attention_weight_min": -35.200965881347656,
      "activations/layer11_attention_weight_max": 33.01996612548828,
      "activations/layer11_attention_weight_min": -34.38776779174805,
      "activations/layer12_attention_weight_max": 19.572097778320312,
      "activations/layer12_attention_weight_min": -24.66352653503418,
      "activations/layer13_attention_weight_max": 36.799903869628906,
      "activations/layer13_attention_weight_min": -28.336074829101562,
      "activations/layer14_attention_weight_max": 35.140926361083984,
      "activations/layer14_attention_weight_min": -29.258705139160156,
      "activations/layer15_attention_weight_max": 34.15818786621094,
      "activations/layer15_attention_weight_min": -29.799190521240234,
      "activations/layer16_attention_weight_max": 32.48240661621094,
      "activations/layer16_attention_weight_min": -31.4000244140625,
      "activations/layer17_attention_weight_max": 49.5937385559082,
      "activations/layer17_attention_weight_min": -44.691383361816406,
      "activations/layer18_attention_weight_max": 48.01942825317383,
      "activations/layer18_attention_weight_min": -43.14698791503906,
      "activations/layer19_attention_weight_max": 23.313644409179688,
      "activations/layer19_attention_weight_min": -24.017494201660156,
      "activations/layer1_attention_weight_max": 15.566800117492676,
      "activations/layer1_attention_weight_min": -13.827391624450684,
      "activations/layer20_attention_weight_max": 21.88273811340332,
      "activations/layer20_attention_weight_min": -21.607303619384766,
      "activations/layer21_attention_weight_max": 38.757598876953125,
      "activations/layer21_attention_weight_min": -22.105615615844727,
      "activations/layer22_attention_weight_max": 26.313751220703125,
      "activations/layer22_attention_weight_min": -25.38567543029785,
      "activations/layer23_attention_weight_max": 36.30018997192383,
      "activations/layer23_attention_weight_min": -24.23078155517578,
      "activations/layer2_attention_weight_max": 31.262571334838867,
      "activations/layer2_attention_weight_min": -30.153945922851562,
      "activations/layer3_attention_weight_max": 85.64678955078125,
      "activations/layer3_attention_weight_min": -89.2146224975586,
      "activations/layer4_attention_weight_max": 84.73281860351562,
      "activations/layer4_attention_weight_min": -86.67819213867188,
      "activations/layer5_attention_weight_max": 63.27550506591797,
      "activations/layer5_attention_weight_min": -73.08696746826172,
      "activations/layer6_attention_weight_max": 49.94397735595703,
      "activations/layer6_attention_weight_min": -49.691768646240234,
      "activations/layer7_attention_weight_max": 69.16285705566406,
      "activations/layer7_attention_weight_min": -66.74151611328125,
      "activations/layer8_attention_weight_max": 47.52287673950195,
      "activations/layer8_attention_weight_min": -48.48089599609375,
      "activations/layer9_attention_weight_max": 44.505531311035156,
      "activations/layer9_attention_weight_min": -44.903717041015625,
      "epoch": 20.67,
      "learning_rate": 1.6887499999999997e-05,
      "loss": 2.7338,
      "step": 355700
    },
    {
      "activations/layer0_attention_weight_max": 15.67936897277832,
      "activations/layer0_attention_weight_min": -13.929916381835938,
      "activations/layer10_attention_weight_max": 35.9658203125,
      "activations/layer10_attention_weight_min": -36.24396514892578,
      "activations/layer11_attention_weight_max": 32.886192321777344,
      "activations/layer11_attention_weight_min": -33.87116622924805,
      "activations/layer12_attention_weight_max": 19.662582397460938,
      "activations/layer12_attention_weight_min": -25.71289825439453,
      "activations/layer13_attention_weight_max": 33.24458694458008,
      "activations/layer13_attention_weight_min": -27.865524291992188,
      "activations/layer14_attention_weight_max": 37.48635482788086,
      "activations/layer14_attention_weight_min": -30.778230667114258,
      "activations/layer15_attention_weight_max": 35.7907600402832,
      "activations/layer15_attention_weight_min": -29.578855514526367,
      "activations/layer16_attention_weight_max": 36.69251251220703,
      "activations/layer16_attention_weight_min": -33.577064514160156,
      "activations/layer17_attention_weight_max": 52.51674270629883,
      "activations/layer17_attention_weight_min": -44.669227600097656,
      "activations/layer18_attention_weight_max": 47.7273063659668,
      "activations/layer18_attention_weight_min": -40.792842864990234,
      "activations/layer19_attention_weight_max": 25.142038345336914,
      "activations/layer19_attention_weight_min": -23.00245475769043,
      "activations/layer1_attention_weight_max": 16.137813568115234,
      "activations/layer1_attention_weight_min": -14.310864448547363,
      "activations/layer20_attention_weight_max": 21.339570999145508,
      "activations/layer20_attention_weight_min": -22.580463409423828,
      "activations/layer21_attention_weight_max": 37.60765075683594,
      "activations/layer21_attention_weight_min": -21.814233779907227,
      "activations/layer22_attention_weight_max": 29.177978515625,
      "activations/layer22_attention_weight_min": -24.878070831298828,
      "activations/layer23_attention_weight_max": 32.88132858276367,
      "activations/layer23_attention_weight_min": -23.408645629882812,
      "activations/layer2_attention_weight_max": 32.394683837890625,
      "activations/layer2_attention_weight_min": -30.25311279296875,
      "activations/layer3_attention_weight_max": 86.27189636230469,
      "activations/layer3_attention_weight_min": -83.70604705810547,
      "activations/layer4_attention_weight_max": 80.3294906616211,
      "activations/layer4_attention_weight_min": -83.80524444580078,
      "activations/layer5_attention_weight_max": 61.468719482421875,
      "activations/layer5_attention_weight_min": -70.67138671875,
      "activations/layer6_attention_weight_max": 49.58116149902344,
      "activations/layer6_attention_weight_min": -49.475467681884766,
      "activations/layer7_attention_weight_max": 65.00590515136719,
      "activations/layer7_attention_weight_min": -65.26468658447266,
      "activations/layer8_attention_weight_max": 45.9944953918457,
      "activations/layer8_attention_weight_min": -51.543724060058594,
      "activations/layer9_attention_weight_max": 42.09792709350586,
      "activations/layer9_attention_weight_min": -50.2083854675293,
      "epoch": 20.67,
      "learning_rate": 1.6868560606060605e-05,
      "loss": 2.7164,
      "step": 355750
    },
    {
      "activations/layer0_attention_weight_max": 15.93254280090332,
      "activations/layer0_attention_weight_min": -14.006869316101074,
      "activations/layer10_attention_weight_max": 40.12834167480469,
      "activations/layer10_attention_weight_min": -37.8941650390625,
      "activations/layer11_attention_weight_max": 37.67892074584961,
      "activations/layer11_attention_weight_min": -36.19827651977539,
      "activations/layer12_attention_weight_max": 19.64441680908203,
      "activations/layer12_attention_weight_min": -23.359119415283203,
      "activations/layer13_attention_weight_max": 35.18474578857422,
      "activations/layer13_attention_weight_min": -28.6419734954834,
      "activations/layer14_attention_weight_max": 35.52204132080078,
      "activations/layer14_attention_weight_min": -30.097002029418945,
      "activations/layer15_attention_weight_max": 34.5798225402832,
      "activations/layer15_attention_weight_min": -30.963491439819336,
      "activations/layer16_attention_weight_max": 36.04364776611328,
      "activations/layer16_attention_weight_min": -32.961856842041016,
      "activations/layer17_attention_weight_max": 55.471744537353516,
      "activations/layer17_attention_weight_min": -49.45564651489258,
      "activations/layer18_attention_weight_max": 50.296630859375,
      "activations/layer18_attention_weight_min": -43.52414321899414,
      "activations/layer19_attention_weight_max": 24.762739181518555,
      "activations/layer19_attention_weight_min": -24.720129013061523,
      "activations/layer1_attention_weight_max": 16.24424171447754,
      "activations/layer1_attention_weight_min": -14.318821907043457,
      "activations/layer20_attention_weight_max": 22.941547393798828,
      "activations/layer20_attention_weight_min": -21.880699157714844,
      "activations/layer21_attention_weight_max": 38.7322883605957,
      "activations/layer21_attention_weight_min": -23.87091064453125,
      "activations/layer22_attention_weight_max": 29.46428108215332,
      "activations/layer22_attention_weight_min": -26.782686233520508,
      "activations/layer23_attention_weight_max": 39.14632797241211,
      "activations/layer23_attention_weight_min": -27.185583114624023,
      "activations/layer2_attention_weight_max": 30.93633270263672,
      "activations/layer2_attention_weight_min": -31.037860870361328,
      "activations/layer3_attention_weight_max": 85.92949676513672,
      "activations/layer3_attention_weight_min": -89.3661880493164,
      "activations/layer4_attention_weight_max": 85.5868911743164,
      "activations/layer4_attention_weight_min": -83.67839813232422,
      "activations/layer5_attention_weight_max": 63.58307647705078,
      "activations/layer5_attention_weight_min": -70.39549255371094,
      "activations/layer6_attention_weight_max": 49.688148498535156,
      "activations/layer6_attention_weight_min": -51.945865631103516,
      "activations/layer7_attention_weight_max": 66.48339080810547,
      "activations/layer7_attention_weight_min": -67.8070297241211,
      "activations/layer8_attention_weight_max": 50.1939811706543,
      "activations/layer8_attention_weight_min": -52.397850036621094,
      "activations/layer9_attention_weight_max": 48.387203216552734,
      "activations/layer9_attention_weight_min": -47.502193450927734,
      "epoch": 20.67,
      "learning_rate": 1.684962121212121e-05,
      "loss": 2.6997,
      "step": 355800
    },
    {
      "activations/layer0_attention_weight_max": 15.438828468322754,
      "activations/layer0_attention_weight_min": -14.345111846923828,
      "activations/layer10_attention_weight_max": 36.898231506347656,
      "activations/layer10_attention_weight_min": -37.720123291015625,
      "activations/layer11_attention_weight_max": 34.725589752197266,
      "activations/layer11_attention_weight_min": -37.17692184448242,
      "activations/layer12_attention_weight_max": 20.362638473510742,
      "activations/layer12_attention_weight_min": -24.829811096191406,
      "activations/layer13_attention_weight_max": 34.05103302001953,
      "activations/layer13_attention_weight_min": -28.592281341552734,
      "activations/layer14_attention_weight_max": 35.89699172973633,
      "activations/layer14_attention_weight_min": -30.08191680908203,
      "activations/layer15_attention_weight_max": 33.73211669921875,
      "activations/layer15_attention_weight_min": -30.737390518188477,
      "activations/layer16_attention_weight_max": 37.43782043457031,
      "activations/layer16_attention_weight_min": -37.11529541015625,
      "activations/layer17_attention_weight_max": 54.491355895996094,
      "activations/layer17_attention_weight_min": -48.99263381958008,
      "activations/layer18_attention_weight_max": 48.84212875366211,
      "activations/layer18_attention_weight_min": -43.42732238769531,
      "activations/layer19_attention_weight_max": 23.002412796020508,
      "activations/layer19_attention_weight_min": -23.401426315307617,
      "activations/layer1_attention_weight_max": 15.074629783630371,
      "activations/layer1_attention_weight_min": -13.987603187561035,
      "activations/layer20_attention_weight_max": 21.781410217285156,
      "activations/layer20_attention_weight_min": -21.507604598999023,
      "activations/layer21_attention_weight_max": 32.58567810058594,
      "activations/layer21_attention_weight_min": -20.752824783325195,
      "activations/layer22_attention_weight_max": 27.73946189880371,
      "activations/layer22_attention_weight_min": -27.609529495239258,
      "activations/layer23_attention_weight_max": 35.815757751464844,
      "activations/layer23_attention_weight_min": -25.712520599365234,
      "activations/layer2_attention_weight_max": 32.0581169128418,
      "activations/layer2_attention_weight_min": -30.308048248291016,
      "activations/layer3_attention_weight_max": 86.77686309814453,
      "activations/layer3_attention_weight_min": -87.26348114013672,
      "activations/layer4_attention_weight_max": 82.11544799804688,
      "activations/layer4_attention_weight_min": -84.55238342285156,
      "activations/layer5_attention_weight_max": 62.900718688964844,
      "activations/layer5_attention_weight_min": -73.13111877441406,
      "activations/layer6_attention_weight_max": 50.36795425415039,
      "activations/layer6_attention_weight_min": -51.53054428100586,
      "activations/layer7_attention_weight_max": 66.80951690673828,
      "activations/layer7_attention_weight_min": -66.45538330078125,
      "activations/layer8_attention_weight_max": 50.224632263183594,
      "activations/layer8_attention_weight_min": -51.14692687988281,
      "activations/layer9_attention_weight_max": 45.388946533203125,
      "activations/layer9_attention_weight_min": -49.668724060058594,
      "epoch": 20.68,
      "learning_rate": 1.683068181818182e-05,
      "loss": 2.7202,
      "step": 355850
    },
    {
      "activations/layer0_attention_weight_max": 16.30059814453125,
      "activations/layer0_attention_weight_min": -14.167569160461426,
      "activations/layer10_attention_weight_max": 36.9206657409668,
      "activations/layer10_attention_weight_min": -36.66946029663086,
      "activations/layer11_attention_weight_max": 33.597877502441406,
      "activations/layer11_attention_weight_min": -34.83652877807617,
      "activations/layer12_attention_weight_max": 19.21283531188965,
      "activations/layer12_attention_weight_min": -23.01254653930664,
      "activations/layer13_attention_weight_max": 33.20996856689453,
      "activations/layer13_attention_weight_min": -27.149919509887695,
      "activations/layer14_attention_weight_max": 35.3886604309082,
      "activations/layer14_attention_weight_min": -31.589649200439453,
      "activations/layer15_attention_weight_max": 33.614845275878906,
      "activations/layer15_attention_weight_min": -30.710464477539062,
      "activations/layer16_attention_weight_max": 34.08005905151367,
      "activations/layer16_attention_weight_min": -32.78133773803711,
      "activations/layer17_attention_weight_max": 51.43447494506836,
      "activations/layer17_attention_weight_min": -44.54182052612305,
      "activations/layer18_attention_weight_max": 49.99213790893555,
      "activations/layer18_attention_weight_min": -40.22999954223633,
      "activations/layer19_attention_weight_max": 24.75811767578125,
      "activations/layer19_attention_weight_min": -24.16413688659668,
      "activations/layer1_attention_weight_max": 15.387794494628906,
      "activations/layer1_attention_weight_min": -14.373373031616211,
      "activations/layer20_attention_weight_max": 23.34618377685547,
      "activations/layer20_attention_weight_min": -21.44456672668457,
      "activations/layer21_attention_weight_max": 40.14181900024414,
      "activations/layer21_attention_weight_min": -22.47056770324707,
      "activations/layer22_attention_weight_max": 31.65732765197754,
      "activations/layer22_attention_weight_min": -24.30092430114746,
      "activations/layer23_attention_weight_max": 33.31875228881836,
      "activations/layer23_attention_weight_min": -24.17946434020996,
      "activations/layer2_attention_weight_max": 30.6265869140625,
      "activations/layer2_attention_weight_min": -29.002792358398438,
      "activations/layer3_attention_weight_max": 86.5198974609375,
      "activations/layer3_attention_weight_min": -85.98892974853516,
      "activations/layer4_attention_weight_max": 83.06521606445312,
      "activations/layer4_attention_weight_min": -85.36034393310547,
      "activations/layer5_attention_weight_max": 61.79005432128906,
      "activations/layer5_attention_weight_min": -70.14644622802734,
      "activations/layer6_attention_weight_max": 48.31338882446289,
      "activations/layer6_attention_weight_min": -52.44990921020508,
      "activations/layer7_attention_weight_max": 66.3525390625,
      "activations/layer7_attention_weight_min": -65.39922332763672,
      "activations/layer8_attention_weight_max": 47.615196228027344,
      "activations/layer8_attention_weight_min": -53.795650482177734,
      "activations/layer9_attention_weight_max": 46.5792350769043,
      "activations/layer9_attention_weight_min": -48.11516189575195,
      "epoch": 20.68,
      "learning_rate": 1.6811742424242423e-05,
      "loss": 2.7332,
      "step": 355900
    },
    {
      "activations/layer0_attention_weight_max": 15.707805633544922,
      "activations/layer0_attention_weight_min": -14.145596504211426,
      "activations/layer10_attention_weight_max": 36.463436126708984,
      "activations/layer10_attention_weight_min": -35.04751968383789,
      "activations/layer11_attention_weight_max": 33.74922180175781,
      "activations/layer11_attention_weight_min": -35.153533935546875,
      "activations/layer12_attention_weight_max": 18.795373916625977,
      "activations/layer12_attention_weight_min": -25.635770797729492,
      "activations/layer13_attention_weight_max": 35.57294464111328,
      "activations/layer13_attention_weight_min": -25.147146224975586,
      "activations/layer14_attention_weight_max": 33.35108184814453,
      "activations/layer14_attention_weight_min": -28.348461151123047,
      "activations/layer15_attention_weight_max": 30.530126571655273,
      "activations/layer15_attention_weight_min": -30.43900489807129,
      "activations/layer16_attention_weight_max": 32.653560638427734,
      "activations/layer16_attention_weight_min": -31.007165908813477,
      "activations/layer17_attention_weight_max": 51.01994705200195,
      "activations/layer17_attention_weight_min": -44.312644958496094,
      "activations/layer18_attention_weight_max": 47.7785758972168,
      "activations/layer18_attention_weight_min": -38.34508514404297,
      "activations/layer19_attention_weight_max": 25.164592742919922,
      "activations/layer19_attention_weight_min": -21.42697525024414,
      "activations/layer1_attention_weight_max": 15.517820358276367,
      "activations/layer1_attention_weight_min": -13.682510375976562,
      "activations/layer20_attention_weight_max": 22.039121627807617,
      "activations/layer20_attention_weight_min": -21.25603675842285,
      "activations/layer21_attention_weight_max": 31.543216705322266,
      "activations/layer21_attention_weight_min": -22.019933700561523,
      "activations/layer22_attention_weight_max": 29.23896598815918,
      "activations/layer22_attention_weight_min": -25.049463272094727,
      "activations/layer23_attention_weight_max": 37.62985610961914,
      "activations/layer23_attention_weight_min": -26.058141708374023,
      "activations/layer2_attention_weight_max": 29.536659240722656,
      "activations/layer2_attention_weight_min": -30.856826782226562,
      "activations/layer3_attention_weight_max": 85.14130401611328,
      "activations/layer3_attention_weight_min": -89.00707244873047,
      "activations/layer4_attention_weight_max": 79.97415924072266,
      "activations/layer4_attention_weight_min": -80.05148315429688,
      "activations/layer5_attention_weight_max": 60.77342987060547,
      "activations/layer5_attention_weight_min": -68.55619812011719,
      "activations/layer6_attention_weight_max": 47.020423889160156,
      "activations/layer6_attention_weight_min": -48.21482467651367,
      "activations/layer7_attention_weight_max": 62.77387619018555,
      "activations/layer7_attention_weight_min": -61.791412353515625,
      "activations/layer8_attention_weight_max": 45.17827606201172,
      "activations/layer8_attention_weight_min": -50.723087310791016,
      "activations/layer9_attention_weight_max": 44.08706283569336,
      "activations/layer9_attention_weight_min": -45.46883010864258,
      "epoch": 20.68,
      "learning_rate": 1.679280303030303e-05,
      "loss": 2.7166,
      "step": 355950
    },
    {
      "activations/layer0_attention_weight_max": 15.632172584533691,
      "activations/layer0_attention_weight_min": -14.115361213684082,
      "activations/layer10_attention_weight_max": 36.736785888671875,
      "activations/layer10_attention_weight_min": -35.47181701660156,
      "activations/layer11_attention_weight_max": 36.2767333984375,
      "activations/layer11_attention_weight_min": -35.78790283203125,
      "activations/layer12_attention_weight_max": 18.872079849243164,
      "activations/layer12_attention_weight_min": -23.314552307128906,
      "activations/layer13_attention_weight_max": 36.20866012573242,
      "activations/layer13_attention_weight_min": -31.972429275512695,
      "activations/layer14_attention_weight_max": 35.31865310668945,
      "activations/layer14_attention_weight_min": -31.320659637451172,
      "activations/layer15_attention_weight_max": 36.270538330078125,
      "activations/layer15_attention_weight_min": -31.153282165527344,
      "activations/layer16_attention_weight_max": 35.36587905883789,
      "activations/layer16_attention_weight_min": -35.60287857055664,
      "activations/layer17_attention_weight_max": 56.4087028503418,
      "activations/layer17_attention_weight_min": -52.10908889770508,
      "activations/layer18_attention_weight_max": 49.11606216430664,
      "activations/layer18_attention_weight_min": -48.419429779052734,
      "activations/layer19_attention_weight_max": 27.210947036743164,
      "activations/layer19_attention_weight_min": -28.455432891845703,
      "activations/layer1_attention_weight_max": 15.08918571472168,
      "activations/layer1_attention_weight_min": -16.09136199951172,
      "activations/layer20_attention_weight_max": 26.916990280151367,
      "activations/layer20_attention_weight_min": -24.526201248168945,
      "activations/layer21_attention_weight_max": 40.451961517333984,
      "activations/layer21_attention_weight_min": -25.03287696838379,
      "activations/layer22_attention_weight_max": 33.33803176879883,
      "activations/layer22_attention_weight_min": -27.227760314941406,
      "activations/layer23_attention_weight_max": 35.64202117919922,
      "activations/layer23_attention_weight_min": -23.083555221557617,
      "activations/layer2_attention_weight_max": 30.210729598999023,
      "activations/layer2_attention_weight_min": -29.536663055419922,
      "activations/layer3_attention_weight_max": 85.73577880859375,
      "activations/layer3_attention_weight_min": -86.28646087646484,
      "activations/layer4_attention_weight_max": 85.34752655029297,
      "activations/layer4_attention_weight_min": -84.47254943847656,
      "activations/layer5_attention_weight_max": 58.46924591064453,
      "activations/layer5_attention_weight_min": -72.08651733398438,
      "activations/layer6_attention_weight_max": 50.97576141357422,
      "activations/layer6_attention_weight_min": -51.39559555053711,
      "activations/layer7_attention_weight_max": 70.54327392578125,
      "activations/layer7_attention_weight_min": -70.13838958740234,
      "activations/layer8_attention_weight_max": 50.41925048828125,
      "activations/layer8_attention_weight_min": -49.85914993286133,
      "activations/layer9_attention_weight_max": 48.190574645996094,
      "activations/layer9_attention_weight_min": -47.3474235534668,
      "epoch": 20.69,
      "learning_rate": 1.6773863636363637e-05,
      "loss": 2.705,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.5391,
      "eval_samples_per_second": 502.864,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.5391,
      "eval_openwebtext_samples_per_second": 502.864,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_wikitext_loss": 2.9140625,
      "eval_wikitext_ppl": 18.431524745974645,
      "eval_wikitext_runtime": 2.0216,
      "eval_wikitext_samples_per_second": 225.562,
      "step": 356000
    },
    {
      "epoch": 20.69,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.6138,
      "eval_lambada_samples_per_second": 506.458,
      "step": 356000
    },
    {
      "activations/layer0_attention_weight_max": 15.173277854919434,
      "activations/layer0_attention_weight_min": -14.253345489501953,
      "activations/layer10_attention_weight_max": 40.270503997802734,
      "activations/layer10_attention_weight_min": -37.85293197631836,
      "activations/layer11_attention_weight_max": 36.59637451171875,
      "activations/layer11_attention_weight_min": -35.4622688293457,
      "activations/layer12_attention_weight_max": 19.205026626586914,
      "activations/layer12_attention_weight_min": -27.364540100097656,
      "activations/layer13_attention_weight_max": 38.218101501464844,
      "activations/layer13_attention_weight_min": -28.74011993408203,
      "activations/layer14_attention_weight_max": 36.97174835205078,
      "activations/layer14_attention_weight_min": -31.478343963623047,
      "activations/layer15_attention_weight_max": 34.59776306152344,
      "activations/layer15_attention_weight_min": -31.445358276367188,
      "activations/layer16_attention_weight_max": 36.54972839355469,
      "activations/layer16_attention_weight_min": -33.41460418701172,
      "activations/layer17_attention_weight_max": 54.638916015625,
      "activations/layer17_attention_weight_min": -46.69161605834961,
      "activations/layer18_attention_weight_max": 53.58812713623047,
      "activations/layer18_attention_weight_min": -40.78144454956055,
      "activations/layer19_attention_weight_max": 27.50632095336914,
      "activations/layer19_attention_weight_min": -21.918699264526367,
      "activations/layer1_attention_weight_max": 14.486669540405273,
      "activations/layer1_attention_weight_min": -13.507495880126953,
      "activations/layer20_attention_weight_max": 25.522520065307617,
      "activations/layer20_attention_weight_min": -20.697477340698242,
      "activations/layer21_attention_weight_max": 34.00772476196289,
      "activations/layer21_attention_weight_min": -24.426809310913086,
      "activations/layer22_attention_weight_max": 27.454132080078125,
      "activations/layer22_attention_weight_min": -26.899112701416016,
      "activations/layer23_attention_weight_max": 34.30890655517578,
      "activations/layer23_attention_weight_min": -25.045820236206055,
      "activations/layer2_attention_weight_max": 31.882503509521484,
      "activations/layer2_attention_weight_min": -30.72011947631836,
      "activations/layer3_attention_weight_max": 89.2551498413086,
      "activations/layer3_attention_weight_min": -90.64688110351562,
      "activations/layer4_attention_weight_max": 86.03868103027344,
      "activations/layer4_attention_weight_min": -89.70012664794922,
      "activations/layer5_attention_weight_max": 61.448829650878906,
      "activations/layer5_attention_weight_min": -75.28150177001953,
      "activations/layer6_attention_weight_max": 51.51799774169922,
      "activations/layer6_attention_weight_min": -54.24191665649414,
      "activations/layer7_attention_weight_max": 66.23579406738281,
      "activations/layer7_attention_weight_min": -68.63297271728516,
      "activations/layer8_attention_weight_max": 49.40410232543945,
      "activations/layer8_attention_weight_min": -53.96731185913086,
      "activations/layer9_attention_weight_max": 47.008628845214844,
      "activations/layer9_attention_weight_min": -47.586158752441406,
      "epoch": 20.69,
      "learning_rate": 1.675530303030303e-05,
      "loss": 2.7146,
      "step": 356050
    },
    {
      "activations/layer0_attention_weight_max": 15.134101867675781,
      "activations/layer0_attention_weight_min": -14.076700210571289,
      "activations/layer10_attention_weight_max": 36.79182052612305,
      "activations/layer10_attention_weight_min": -36.70941925048828,
      "activations/layer11_attention_weight_max": 33.33705139160156,
      "activations/layer11_attention_weight_min": -35.1973876953125,
      "activations/layer12_attention_weight_max": 19.211750030517578,
      "activations/layer12_attention_weight_min": -26.15140724182129,
      "activations/layer13_attention_weight_max": 39.331756591796875,
      "activations/layer13_attention_weight_min": -28.910823822021484,
      "activations/layer14_attention_weight_max": 40.42604446411133,
      "activations/layer14_attention_weight_min": -30.44709587097168,
      "activations/layer15_attention_weight_max": 38.265724182128906,
      "activations/layer15_attention_weight_min": -30.314790725708008,
      "activations/layer16_attention_weight_max": 38.6106071472168,
      "activations/layer16_attention_weight_min": -34.610355377197266,
      "activations/layer17_attention_weight_max": 57.24354934692383,
      "activations/layer17_attention_weight_min": -48.02479553222656,
      "activations/layer18_attention_weight_max": 55.776092529296875,
      "activations/layer18_attention_weight_min": -46.114131927490234,
      "activations/layer19_attention_weight_max": 31.095998764038086,
      "activations/layer19_attention_weight_min": -25.985122680664062,
      "activations/layer1_attention_weight_max": 15.955055236816406,
      "activations/layer1_attention_weight_min": -14.17671012878418,
      "activations/layer20_attention_weight_max": 31.1119441986084,
      "activations/layer20_attention_weight_min": -23.146461486816406,
      "activations/layer21_attention_weight_max": 61.52268600463867,
      "activations/layer21_attention_weight_min": -26.998992919921875,
      "activations/layer22_attention_weight_max": 35.1079216003418,
      "activations/layer22_attention_weight_min": -24.778907775878906,
      "activations/layer23_attention_weight_max": 46.61138916015625,
      "activations/layer23_attention_weight_min": -25.589101791381836,
      "activations/layer2_attention_weight_max": 30.044281005859375,
      "activations/layer2_attention_weight_min": -29.969152450561523,
      "activations/layer3_attention_weight_max": 86.19422149658203,
      "activations/layer3_attention_weight_min": -86.83267974853516,
      "activations/layer4_attention_weight_max": 82.32162475585938,
      "activations/layer4_attention_weight_min": -85.23422241210938,
      "activations/layer5_attention_weight_max": 59.76588439941406,
      "activations/layer5_attention_weight_min": -69.14363861083984,
      "activations/layer6_attention_weight_max": 48.95978546142578,
      "activations/layer6_attention_weight_min": -50.19670104980469,
      "activations/layer7_attention_weight_max": 70.4394760131836,
      "activations/layer7_attention_weight_min": -64.76914978027344,
      "activations/layer8_attention_weight_max": 47.626407623291016,
      "activations/layer8_attention_weight_min": -55.20246124267578,
      "activations/layer9_attention_weight_max": 43.95073318481445,
      "activations/layer9_attention_weight_min": -46.7550163269043,
      "epoch": 20.69,
      "learning_rate": 1.6736363636363636e-05,
      "loss": 2.7112,
      "step": 356100
    },
    {
      "activations/layer0_attention_weight_max": 15.29164981842041,
      "activations/layer0_attention_weight_min": -14.4640531539917,
      "activations/layer10_attention_weight_max": 40.2960090637207,
      "activations/layer10_attention_weight_min": -38.85191345214844,
      "activations/layer11_attention_weight_max": 35.79185485839844,
      "activations/layer11_attention_weight_min": -37.041404724121094,
      "activations/layer12_attention_weight_max": 20.393768310546875,
      "activations/layer12_attention_weight_min": -21.97043228149414,
      "activations/layer13_attention_weight_max": 41.197776794433594,
      "activations/layer13_attention_weight_min": -29.509817123413086,
      "activations/layer14_attention_weight_max": 35.98089599609375,
      "activations/layer14_attention_weight_min": -30.456985473632812,
      "activations/layer15_attention_weight_max": 34.541751861572266,
      "activations/layer15_attention_weight_min": -30.318603515625,
      "activations/layer16_attention_weight_max": 34.84180450439453,
      "activations/layer16_attention_weight_min": -31.841434478759766,
      "activations/layer17_attention_weight_max": 50.84666442871094,
      "activations/layer17_attention_weight_min": -42.584068298339844,
      "activations/layer18_attention_weight_max": 45.8381233215332,
      "activations/layer18_attention_weight_min": -39.854576110839844,
      "activations/layer19_attention_weight_max": 25.35919952392578,
      "activations/layer19_attention_weight_min": -24.770048141479492,
      "activations/layer1_attention_weight_max": 15.343971252441406,
      "activations/layer1_attention_weight_min": -15.047932624816895,
      "activations/layer20_attention_weight_max": 26.079851150512695,
      "activations/layer20_attention_weight_min": -21.494510650634766,
      "activations/layer21_attention_weight_max": 35.59705352783203,
      "activations/layer21_attention_weight_min": -22.648601531982422,
      "activations/layer22_attention_weight_max": 29.56767463684082,
      "activations/layer22_attention_weight_min": -26.914506912231445,
      "activations/layer23_attention_weight_max": 44.29637908935547,
      "activations/layer23_attention_weight_min": -27.860992431640625,
      "activations/layer2_attention_weight_max": 30.46878433227539,
      "activations/layer2_attention_weight_min": -30.372953414916992,
      "activations/layer3_attention_weight_max": 90.5003433227539,
      "activations/layer3_attention_weight_min": -88.85800170898438,
      "activations/layer4_attention_weight_max": 87.44084930419922,
      "activations/layer4_attention_weight_min": -85.94578552246094,
      "activations/layer5_attention_weight_max": 64.36601257324219,
      "activations/layer5_attention_weight_min": -76.5704345703125,
      "activations/layer6_attention_weight_max": 53.92317581176758,
      "activations/layer6_attention_weight_min": -52.10349655151367,
      "activations/layer7_attention_weight_max": 67.92695617675781,
      "activations/layer7_attention_weight_min": -66.46102905273438,
      "activations/layer8_attention_weight_max": 51.37282943725586,
      "activations/layer8_attention_weight_min": -53.67182159423828,
      "activations/layer9_attention_weight_max": 48.460086822509766,
      "activations/layer9_attention_weight_min": -46.4631233215332,
      "epoch": 20.69,
      "learning_rate": 1.671742424242424e-05,
      "loss": 2.7316,
      "step": 356150
    },
    {
      "activations/layer0_attention_weight_max": 15.330249786376953,
      "activations/layer0_attention_weight_min": -14.206247329711914,
      "activations/layer10_attention_weight_max": 34.54829788208008,
      "activations/layer10_attention_weight_min": -35.22188186645508,
      "activations/layer11_attention_weight_max": 32.361717224121094,
      "activations/layer11_attention_weight_min": -34.52482986450195,
      "activations/layer12_attention_weight_max": 19.56877899169922,
      "activations/layer12_attention_weight_min": -25.095199584960938,
      "activations/layer13_attention_weight_max": 30.76996612548828,
      "activations/layer13_attention_weight_min": -27.999431610107422,
      "activations/layer14_attention_weight_max": 32.48051452636719,
      "activations/layer14_attention_weight_min": -31.325407028198242,
      "activations/layer15_attention_weight_max": 30.773340225219727,
      "activations/layer15_attention_weight_min": -30.93202018737793,
      "activations/layer16_attention_weight_max": 31.06127166748047,
      "activations/layer16_attention_weight_min": -32.475765228271484,
      "activations/layer17_attention_weight_max": 49.57052993774414,
      "activations/layer17_attention_weight_min": -45.038326263427734,
      "activations/layer18_attention_weight_max": 42.84502029418945,
      "activations/layer18_attention_weight_min": -41.027854919433594,
      "activations/layer19_attention_weight_max": 25.184324264526367,
      "activations/layer19_attention_weight_min": -24.541160583496094,
      "activations/layer1_attention_weight_max": 15.41141128540039,
      "activations/layer1_attention_weight_min": -13.605799674987793,
      "activations/layer20_attention_weight_max": 21.223421096801758,
      "activations/layer20_attention_weight_min": -22.24763298034668,
      "activations/layer21_attention_weight_max": 28.88688087463379,
      "activations/layer21_attention_weight_min": -21.602453231811523,
      "activations/layer22_attention_weight_max": 26.709884643554688,
      "activations/layer22_attention_weight_min": -24.823015213012695,
      "activations/layer23_attention_weight_max": 33.02421188354492,
      "activations/layer23_attention_weight_min": -24.395177841186523,
      "activations/layer2_attention_weight_max": 30.9960994720459,
      "activations/layer2_attention_weight_min": -29.167266845703125,
      "activations/layer3_attention_weight_max": 87.71676635742188,
      "activations/layer3_attention_weight_min": -90.89505004882812,
      "activations/layer4_attention_weight_max": 82.21068572998047,
      "activations/layer4_attention_weight_min": -85.68282318115234,
      "activations/layer5_attention_weight_max": 58.433250427246094,
      "activations/layer5_attention_weight_min": -71.65673828125,
      "activations/layer6_attention_weight_max": 49.63271713256836,
      "activations/layer6_attention_weight_min": -50.044349670410156,
      "activations/layer7_attention_weight_max": 65.2843017578125,
      "activations/layer7_attention_weight_min": -64.93217468261719,
      "activations/layer8_attention_weight_max": 48.635162353515625,
      "activations/layer8_attention_weight_min": -52.06988525390625,
      "activations/layer9_attention_weight_max": 43.03097152709961,
      "activations/layer9_attention_weight_min": -47.70594787597656,
      "epoch": 20.7,
      "learning_rate": 1.669848484848485e-05,
      "loss": 2.7402,
      "step": 356200
    },
    {
      "activations/layer0_attention_weight_max": 15.043511390686035,
      "activations/layer0_attention_weight_min": -14.166308403015137,
      "activations/layer10_attention_weight_max": 37.269813537597656,
      "activations/layer10_attention_weight_min": -35.19744110107422,
      "activations/layer11_attention_weight_max": 37.47916793823242,
      "activations/layer11_attention_weight_min": -34.950286865234375,
      "activations/layer12_attention_weight_max": 20.46630859375,
      "activations/layer12_attention_weight_min": -28.685714721679688,
      "activations/layer13_attention_weight_max": 34.91695785522461,
      "activations/layer13_attention_weight_min": -28.42208480834961,
      "activations/layer14_attention_weight_max": 37.08728790283203,
      "activations/layer14_attention_weight_min": -29.69875144958496,
      "activations/layer15_attention_weight_max": 36.242794036865234,
      "activations/layer15_attention_weight_min": -32.513729095458984,
      "activations/layer16_attention_weight_max": 33.93549346923828,
      "activations/layer16_attention_weight_min": -31.42539405822754,
      "activations/layer17_attention_weight_max": 51.90934753417969,
      "activations/layer17_attention_weight_min": -45.60334777832031,
      "activations/layer18_attention_weight_max": 48.37161636352539,
      "activations/layer18_attention_weight_min": -39.51041793823242,
      "activations/layer19_attention_weight_max": 28.221689224243164,
      "activations/layer19_attention_weight_min": -23.18730354309082,
      "activations/layer1_attention_weight_max": 15.352205276489258,
      "activations/layer1_attention_weight_min": -14.027697563171387,
      "activations/layer20_attention_weight_max": 22.683002471923828,
      "activations/layer20_attention_weight_min": -20.81306266784668,
      "activations/layer21_attention_weight_max": 36.242313385009766,
      "activations/layer21_attention_weight_min": -26.471288681030273,
      "activations/layer22_attention_weight_max": 31.808963775634766,
      "activations/layer22_attention_weight_min": -25.59427261352539,
      "activations/layer23_attention_weight_max": 40.01361846923828,
      "activations/layer23_attention_weight_min": -23.179344177246094,
      "activations/layer2_attention_weight_max": 31.23883819580078,
      "activations/layer2_attention_weight_min": -30.33704376220703,
      "activations/layer3_attention_weight_max": 88.96460723876953,
      "activations/layer3_attention_weight_min": -93.50374603271484,
      "activations/layer4_attention_weight_max": 84.21253967285156,
      "activations/layer4_attention_weight_min": -82.39596557617188,
      "activations/layer5_attention_weight_max": 63.30430221557617,
      "activations/layer5_attention_weight_min": -70.77161407470703,
      "activations/layer6_attention_weight_max": 49.88999938964844,
      "activations/layer6_attention_weight_min": -51.26546859741211,
      "activations/layer7_attention_weight_max": 64.66397094726562,
      "activations/layer7_attention_weight_min": -69.75473022460938,
      "activations/layer8_attention_weight_max": 50.158416748046875,
      "activations/layer8_attention_weight_min": -51.782554626464844,
      "activations/layer9_attention_weight_max": 45.12101364135742,
      "activations/layer9_attention_weight_min": -46.89502716064453,
      "epoch": 20.7,
      "learning_rate": 1.6679545454545454e-05,
      "loss": 2.7234,
      "step": 356250
    },
    {
      "activations/layer0_attention_weight_max": 15.78377628326416,
      "activations/layer0_attention_weight_min": -14.138702392578125,
      "activations/layer10_attention_weight_max": 43.25,
      "activations/layer10_attention_weight_min": -41.25584411621094,
      "activations/layer11_attention_weight_max": 40.008689880371094,
      "activations/layer11_attention_weight_min": -36.81889343261719,
      "activations/layer12_attention_weight_max": 20.451494216918945,
      "activations/layer12_attention_weight_min": -24.91545867919922,
      "activations/layer13_attention_weight_max": 44.73830795288086,
      "activations/layer13_attention_weight_min": -30.46662139892578,
      "activations/layer14_attention_weight_max": 64.40754699707031,
      "activations/layer14_attention_weight_min": -31.97410011291504,
      "activations/layer15_attention_weight_max": 59.63673782348633,
      "activations/layer15_attention_weight_min": -38.81369400024414,
      "activations/layer16_attention_weight_max": 42.673248291015625,
      "activations/layer16_attention_weight_min": -36.71003723144531,
      "activations/layer17_attention_weight_max": 66.81489562988281,
      "activations/layer17_attention_weight_min": -58.52700424194336,
      "activations/layer18_attention_weight_max": 61.37413787841797,
      "activations/layer18_attention_weight_min": -50.25918197631836,
      "activations/layer19_attention_weight_max": 33.184120178222656,
      "activations/layer19_attention_weight_min": -28.637405395507812,
      "activations/layer1_attention_weight_max": 15.781782150268555,
      "activations/layer1_attention_weight_min": -14.175829887390137,
      "activations/layer20_attention_weight_max": 33.334503173828125,
      "activations/layer20_attention_weight_min": -26.0614013671875,
      "activations/layer21_attention_weight_max": 58.64884948730469,
      "activations/layer21_attention_weight_min": -31.952444076538086,
      "activations/layer22_attention_weight_max": 35.82223892211914,
      "activations/layer22_attention_weight_min": -30.148828506469727,
      "activations/layer23_attention_weight_max": 36.778472900390625,
      "activations/layer23_attention_weight_min": -30.089218139648438,
      "activations/layer2_attention_weight_max": 34.22225570678711,
      "activations/layer2_attention_weight_min": -32.4749870300293,
      "activations/layer3_attention_weight_max": 97.21862030029297,
      "activations/layer3_attention_weight_min": -94.59130096435547,
      "activations/layer4_attention_weight_max": 89.54085540771484,
      "activations/layer4_attention_weight_min": -89.15484619140625,
      "activations/layer5_attention_weight_max": 61.731990814208984,
      "activations/layer5_attention_weight_min": -70.64753723144531,
      "activations/layer6_attention_weight_max": 51.532466888427734,
      "activations/layer6_attention_weight_min": -51.0576286315918,
      "activations/layer7_attention_weight_max": 71.32058715820312,
      "activations/layer7_attention_weight_min": -69.2347412109375,
      "activations/layer8_attention_weight_max": 50.95673370361328,
      "activations/layer8_attention_weight_min": -50.79454040527344,
      "activations/layer9_attention_weight_max": 56.93239212036133,
      "activations/layer9_attention_weight_min": -50.77818298339844,
      "epoch": 20.7,
      "learning_rate": 1.666060606060606e-05,
      "loss": 2.7073,
      "step": 356300
    },
    {
      "activations/layer0_attention_weight_max": 15.502365112304688,
      "activations/layer0_attention_weight_min": -14.054813385009766,
      "activations/layer10_attention_weight_max": 35.28553771972656,
      "activations/layer10_attention_weight_min": -36.566314697265625,
      "activations/layer11_attention_weight_max": 32.911319732666016,
      "activations/layer11_attention_weight_min": -35.46961975097656,
      "activations/layer12_attention_weight_max": 18.5212345123291,
      "activations/layer12_attention_weight_min": -26.579980850219727,
      "activations/layer13_attention_weight_max": 33.08616638183594,
      "activations/layer13_attention_weight_min": -25.716737747192383,
      "activations/layer14_attention_weight_max": 35.607032775878906,
      "activations/layer14_attention_weight_min": -28.448810577392578,
      "activations/layer15_attention_weight_max": 32.78387451171875,
      "activations/layer15_attention_weight_min": -30.354602813720703,
      "activations/layer16_attention_weight_max": 33.28361129760742,
      "activations/layer16_attention_weight_min": -31.551834106445312,
      "activations/layer17_attention_weight_max": 49.727298736572266,
      "activations/layer17_attention_weight_min": -47.87105178833008,
      "activations/layer18_attention_weight_max": 48.9486198425293,
      "activations/layer18_attention_weight_min": -45.195770263671875,
      "activations/layer19_attention_weight_max": 24.727985382080078,
      "activations/layer19_attention_weight_min": -20.927968978881836,
      "activations/layer1_attention_weight_max": 15.946372985839844,
      "activations/layer1_attention_weight_min": -14.070815086364746,
      "activations/layer20_attention_weight_max": 23.576433181762695,
      "activations/layer20_attention_weight_min": -21.522785186767578,
      "activations/layer21_attention_weight_max": 32.76905822753906,
      "activations/layer21_attention_weight_min": -19.61630630493164,
      "activations/layer22_attention_weight_max": 32.611690521240234,
      "activations/layer22_attention_weight_min": -25.174333572387695,
      "activations/layer23_attention_weight_max": 35.28882598876953,
      "activations/layer23_attention_weight_min": -24.249967575073242,
      "activations/layer2_attention_weight_max": 30.89065933227539,
      "activations/layer2_attention_weight_min": -29.32074546813965,
      "activations/layer3_attention_weight_max": 89.70890808105469,
      "activations/layer3_attention_weight_min": -88.17813110351562,
      "activations/layer4_attention_weight_max": 86.77400970458984,
      "activations/layer4_attention_weight_min": -89.2805404663086,
      "activations/layer5_attention_weight_max": 63.512489318847656,
      "activations/layer5_attention_weight_min": -70.6758804321289,
      "activations/layer6_attention_weight_max": 50.12144470214844,
      "activations/layer6_attention_weight_min": -51.384727478027344,
      "activations/layer7_attention_weight_max": 66.40601348876953,
      "activations/layer7_attention_weight_min": -66.25273895263672,
      "activations/layer8_attention_weight_max": 45.75615310668945,
      "activations/layer8_attention_weight_min": -50.29536819458008,
      "activations/layer9_attention_weight_max": 44.0883903503418,
      "activations/layer9_attention_weight_min": -46.11075210571289,
      "epoch": 20.71,
      "learning_rate": 1.6641666666666663e-05,
      "loss": 2.732,
      "step": 356350
    },
    {
      "activations/layer0_attention_weight_max": 15.855998992919922,
      "activations/layer0_attention_weight_min": -14.018821716308594,
      "activations/layer10_attention_weight_max": 37.04170227050781,
      "activations/layer10_attention_weight_min": -36.941871643066406,
      "activations/layer11_attention_weight_max": 34.256935119628906,
      "activations/layer11_attention_weight_min": -35.64348602294922,
      "activations/layer12_attention_weight_max": 22.312814712524414,
      "activations/layer12_attention_weight_min": -26.330659866333008,
      "activations/layer13_attention_weight_max": 32.81065368652344,
      "activations/layer13_attention_weight_min": -28.216196060180664,
      "activations/layer14_attention_weight_max": 33.98091125488281,
      "activations/layer14_attention_weight_min": -31.82756233215332,
      "activations/layer15_attention_weight_max": 32.51546859741211,
      "activations/layer15_attention_weight_min": -32.657135009765625,
      "activations/layer16_attention_weight_max": 31.97145652770996,
      "activations/layer16_attention_weight_min": -32.3192138671875,
      "activations/layer17_attention_weight_max": 49.46979522705078,
      "activations/layer17_attention_weight_min": -47.71269226074219,
      "activations/layer18_attention_weight_max": 45.75847244262695,
      "activations/layer18_attention_weight_min": -41.09549331665039,
      "activations/layer19_attention_weight_max": 26.04972267150879,
      "activations/layer19_attention_weight_min": -22.000410079956055,
      "activations/layer1_attention_weight_max": 15.218674659729004,
      "activations/layer1_attention_weight_min": -17.304603576660156,
      "activations/layer20_attention_weight_max": 23.0017032623291,
      "activations/layer20_attention_weight_min": -22.695749282836914,
      "activations/layer21_attention_weight_max": 33.2408332824707,
      "activations/layer21_attention_weight_min": -22.402482986450195,
      "activations/layer22_attention_weight_max": 28.304462432861328,
      "activations/layer22_attention_weight_min": -26.589128494262695,
      "activations/layer23_attention_weight_max": 37.1827392578125,
      "activations/layer23_attention_weight_min": -27.72249412536621,
      "activations/layer2_attention_weight_max": 31.61058807373047,
      "activations/layer2_attention_weight_min": -30.559242248535156,
      "activations/layer3_attention_weight_max": 90.74807739257812,
      "activations/layer3_attention_weight_min": -90.21102142333984,
      "activations/layer4_attention_weight_max": 88.41838836669922,
      "activations/layer4_attention_weight_min": -85.46349334716797,
      "activations/layer5_attention_weight_max": 63.249168395996094,
      "activations/layer5_attention_weight_min": -69.14531707763672,
      "activations/layer6_attention_weight_max": 48.25194549560547,
      "activations/layer6_attention_weight_min": -50.035579681396484,
      "activations/layer7_attention_weight_max": 66.46212768554688,
      "activations/layer7_attention_weight_min": -66.08356475830078,
      "activations/layer8_attention_weight_max": 49.618690490722656,
      "activations/layer8_attention_weight_min": -53.0520133972168,
      "activations/layer9_attention_weight_max": 45.74467086791992,
      "activations/layer9_attention_weight_min": -45.70048522949219,
      "epoch": 20.71,
      "learning_rate": 1.6622727272727272e-05,
      "loss": 2.7304,
      "step": 356400
    },
    {
      "activations/layer0_attention_weight_max": 16.55472755432129,
      "activations/layer0_attention_weight_min": -13.937834739685059,
      "activations/layer10_attention_weight_max": 36.710609436035156,
      "activations/layer10_attention_weight_min": -35.126617431640625,
      "activations/layer11_attention_weight_max": 33.79201126098633,
      "activations/layer11_attention_weight_min": -34.61080551147461,
      "activations/layer12_attention_weight_max": 18.492908477783203,
      "activations/layer12_attention_weight_min": -24.793704986572266,
      "activations/layer13_attention_weight_max": 33.14898681640625,
      "activations/layer13_attention_weight_min": -27.812110900878906,
      "activations/layer14_attention_weight_max": 37.84156799316406,
      "activations/layer14_attention_weight_min": -31.554325103759766,
      "activations/layer15_attention_weight_max": 35.97025680541992,
      "activations/layer15_attention_weight_min": -32.78171920776367,
      "activations/layer16_attention_weight_max": 38.407875061035156,
      "activations/layer16_attention_weight_min": -36.48186111450195,
      "activations/layer17_attention_weight_max": 54.81147766113281,
      "activations/layer17_attention_weight_min": -49.03845977783203,
      "activations/layer18_attention_weight_max": 51.35859298706055,
      "activations/layer18_attention_weight_min": -44.482337951660156,
      "activations/layer19_attention_weight_max": 24.694244384765625,
      "activations/layer19_attention_weight_min": -25.163864135742188,
      "activations/layer1_attention_weight_max": 14.808342933654785,
      "activations/layer1_attention_weight_min": -14.900177001953125,
      "activations/layer20_attention_weight_max": 25.336326599121094,
      "activations/layer20_attention_weight_min": -22.28436851501465,
      "activations/layer21_attention_weight_max": 42.77040100097656,
      "activations/layer21_attention_weight_min": -24.125534057617188,
      "activations/layer22_attention_weight_max": 29.796459197998047,
      "activations/layer22_attention_weight_min": -25.60352897644043,
      "activations/layer23_attention_weight_max": 37.694766998291016,
      "activations/layer23_attention_weight_min": -22.56893539428711,
      "activations/layer2_attention_weight_max": 29.047473907470703,
      "activations/layer2_attention_weight_min": -28.71356201171875,
      "activations/layer3_attention_weight_max": 86.56816864013672,
      "activations/layer3_attention_weight_min": -85.78704071044922,
      "activations/layer4_attention_weight_max": 84.91981506347656,
      "activations/layer4_attention_weight_min": -85.23165893554688,
      "activations/layer5_attention_weight_max": 63.06724548339844,
      "activations/layer5_attention_weight_min": -73.29794311523438,
      "activations/layer6_attention_weight_max": 51.38068389892578,
      "activations/layer6_attention_weight_min": -50.85322189331055,
      "activations/layer7_attention_weight_max": 66.81745147705078,
      "activations/layer7_attention_weight_min": -66.28081512451172,
      "activations/layer8_attention_weight_max": 45.79314041137695,
      "activations/layer8_attention_weight_min": -49.743804931640625,
      "activations/layer9_attention_weight_max": 42.688201904296875,
      "activations/layer9_attention_weight_min": -47.910125732421875,
      "epoch": 20.71,
      "learning_rate": 1.6603787878787877e-05,
      "loss": 2.724,
      "step": 356450
    },
    {
      "activations/layer0_attention_weight_max": 16.19769859313965,
      "activations/layer0_attention_weight_min": -13.729536056518555,
      "activations/layer10_attention_weight_max": 34.78438186645508,
      "activations/layer10_attention_weight_min": -38.4830322265625,
      "activations/layer11_attention_weight_max": 35.05664825439453,
      "activations/layer11_attention_weight_min": -35.750946044921875,
      "activations/layer12_attention_weight_max": 19.88777732849121,
      "activations/layer12_attention_weight_min": -27.21723747253418,
      "activations/layer13_attention_weight_max": 31.404155731201172,
      "activations/layer13_attention_weight_min": -28.073923110961914,
      "activations/layer14_attention_weight_max": 35.901058197021484,
      "activations/layer14_attention_weight_min": -31.885766983032227,
      "activations/layer15_attention_weight_max": 32.76259231567383,
      "activations/layer15_attention_weight_min": -32.60973358154297,
      "activations/layer16_attention_weight_max": 37.442787170410156,
      "activations/layer16_attention_weight_min": -36.64919662475586,
      "activations/layer17_attention_weight_max": 53.51106643676758,
      "activations/layer17_attention_weight_min": -51.676475524902344,
      "activations/layer18_attention_weight_max": 48.60493087768555,
      "activations/layer18_attention_weight_min": -49.13414764404297,
      "activations/layer19_attention_weight_max": 24.704477310180664,
      "activations/layer19_attention_weight_min": -29.15723991394043,
      "activations/layer1_attention_weight_max": 16.76955795288086,
      "activations/layer1_attention_weight_min": -14.31749153137207,
      "activations/layer20_attention_weight_max": 21.86654281616211,
      "activations/layer20_attention_weight_min": -25.540666580200195,
      "activations/layer21_attention_weight_max": 38.3179931640625,
      "activations/layer21_attention_weight_min": -23.630077362060547,
      "activations/layer22_attention_weight_max": 29.732027053833008,
      "activations/layer22_attention_weight_min": -29.796899795532227,
      "activations/layer23_attention_weight_max": 36.99215316772461,
      "activations/layer23_attention_weight_min": -24.6229248046875,
      "activations/layer2_attention_weight_max": 31.181438446044922,
      "activations/layer2_attention_weight_min": -31.36932373046875,
      "activations/layer3_attention_weight_max": 85.50631713867188,
      "activations/layer3_attention_weight_min": -86.75812530517578,
      "activations/layer4_attention_weight_max": 82.46476745605469,
      "activations/layer4_attention_weight_min": -80.79093170166016,
      "activations/layer5_attention_weight_max": 62.153846740722656,
      "activations/layer5_attention_weight_min": -67.43412017822266,
      "activations/layer6_attention_weight_max": 48.98099136352539,
      "activations/layer6_attention_weight_min": -47.60874557495117,
      "activations/layer7_attention_weight_max": 68.13176727294922,
      "activations/layer7_attention_weight_min": -61.714115142822266,
      "activations/layer8_attention_weight_max": 45.4996223449707,
      "activations/layer8_attention_weight_min": -48.142311096191406,
      "activations/layer9_attention_weight_max": 56.4599494934082,
      "activations/layer9_attention_weight_min": -48.108642578125,
      "epoch": 20.71,
      "learning_rate": 1.658484848484848e-05,
      "loss": 2.7135,
      "step": 356500
    },
    {
      "activations/layer0_attention_weight_max": 15.379411697387695,
      "activations/layer0_attention_weight_min": -13.796534538269043,
      "activations/layer10_attention_weight_max": 38.7755012512207,
      "activations/layer10_attention_weight_min": -36.63064956665039,
      "activations/layer11_attention_weight_max": 36.11711502075195,
      "activations/layer11_attention_weight_min": -35.225624084472656,
      "activations/layer12_attention_weight_max": 20.83119010925293,
      "activations/layer12_attention_weight_min": -23.87063980102539,
      "activations/layer13_attention_weight_max": 35.9121208190918,
      "activations/layer13_attention_weight_min": -28.9964599609375,
      "activations/layer14_attention_weight_max": 35.89329528808594,
      "activations/layer14_attention_weight_min": -32.12553024291992,
      "activations/layer15_attention_weight_max": 35.12525177001953,
      "activations/layer15_attention_weight_min": -31.46278953552246,
      "activations/layer16_attention_weight_max": 35.73379898071289,
      "activations/layer16_attention_weight_min": -35.22779846191406,
      "activations/layer17_attention_weight_max": 54.0451545715332,
      "activations/layer17_attention_weight_min": -47.65336990356445,
      "activations/layer18_attention_weight_max": 51.694297790527344,
      "activations/layer18_attention_weight_min": -44.213993072509766,
      "activations/layer19_attention_weight_max": 25.567928314208984,
      "activations/layer19_attention_weight_min": -26.451175689697266,
      "activations/layer1_attention_weight_max": 15.24073600769043,
      "activations/layer1_attention_weight_min": -14.146807670593262,
      "activations/layer20_attention_weight_max": 24.66344451904297,
      "activations/layer20_attention_weight_min": -24.134172439575195,
      "activations/layer21_attention_weight_max": 35.08864212036133,
      "activations/layer21_attention_weight_min": -23.92735481262207,
      "activations/layer22_attention_weight_max": 29.567272186279297,
      "activations/layer22_attention_weight_min": -28.05224609375,
      "activations/layer23_attention_weight_max": 35.055198669433594,
      "activations/layer23_attention_weight_min": -25.188461303710938,
      "activations/layer2_attention_weight_max": 30.088603973388672,
      "activations/layer2_attention_weight_min": -29.440093994140625,
      "activations/layer3_attention_weight_max": 86.15003967285156,
      "activations/layer3_attention_weight_min": -87.93154907226562,
      "activations/layer4_attention_weight_max": 85.04872131347656,
      "activations/layer4_attention_weight_min": -84.39371490478516,
      "activations/layer5_attention_weight_max": 60.15605163574219,
      "activations/layer5_attention_weight_min": -73.43065643310547,
      "activations/layer6_attention_weight_max": 51.58582305908203,
      "activations/layer6_attention_weight_min": -51.10562515258789,
      "activations/layer7_attention_weight_max": 67.45867919921875,
      "activations/layer7_attention_weight_min": -64.81519317626953,
      "activations/layer8_attention_weight_max": 49.5496940612793,
      "activations/layer8_attention_weight_min": -49.16200637817383,
      "activations/layer9_attention_weight_max": 45.80261993408203,
      "activations/layer9_attention_weight_min": -46.32368469238281,
      "epoch": 20.72,
      "learning_rate": 1.656590909090909e-05,
      "loss": 2.7346,
      "step": 356550
    },
    {
      "activations/layer0_attention_weight_max": 15.87896728515625,
      "activations/layer0_attention_weight_min": -14.002267837524414,
      "activations/layer10_attention_weight_max": 35.517921447753906,
      "activations/layer10_attention_weight_min": -35.28194046020508,
      "activations/layer11_attention_weight_max": 33.42647171020508,
      "activations/layer11_attention_weight_min": -35.37770080566406,
      "activations/layer12_attention_weight_max": 21.89238739013672,
      "activations/layer12_attention_weight_min": -25.009016036987305,
      "activations/layer13_attention_weight_max": 32.98650360107422,
      "activations/layer13_attention_weight_min": -25.039583206176758,
      "activations/layer14_attention_weight_max": 35.19003677368164,
      "activations/layer14_attention_weight_min": -28.883188247680664,
      "activations/layer15_attention_weight_max": 33.40922927856445,
      "activations/layer15_attention_weight_min": -30.099994659423828,
      "activations/layer16_attention_weight_max": 35.45654296875,
      "activations/layer16_attention_weight_min": -31.632339477539062,
      "activations/layer17_attention_weight_max": 53.29998779296875,
      "activations/layer17_attention_weight_min": -44.216552734375,
      "activations/layer18_attention_weight_max": 48.153072357177734,
      "activations/layer18_attention_weight_min": -39.90190505981445,
      "activations/layer19_attention_weight_max": 25.69183921813965,
      "activations/layer19_attention_weight_min": -23.426496505737305,
      "activations/layer1_attention_weight_max": 15.330920219421387,
      "activations/layer1_attention_weight_min": -15.254999160766602,
      "activations/layer20_attention_weight_max": 24.6875057220459,
      "activations/layer20_attention_weight_min": -20.526893615722656,
      "activations/layer21_attention_weight_max": 36.181419372558594,
      "activations/layer21_attention_weight_min": -22.688268661499023,
      "activations/layer22_attention_weight_max": 32.135772705078125,
      "activations/layer22_attention_weight_min": -26.93057632446289,
      "activations/layer23_attention_weight_max": 37.00994110107422,
      "activations/layer23_attention_weight_min": -29.525798797607422,
      "activations/layer2_attention_weight_max": 31.583837509155273,
      "activations/layer2_attention_weight_min": -29.888530731201172,
      "activations/layer3_attention_weight_max": 85.76612091064453,
      "activations/layer3_attention_weight_min": -88.49744415283203,
      "activations/layer4_attention_weight_max": 83.77498626708984,
      "activations/layer4_attention_weight_min": -85.06957244873047,
      "activations/layer5_attention_weight_max": 62.00338363647461,
      "activations/layer5_attention_weight_min": -70.38459777832031,
      "activations/layer6_attention_weight_max": 48.73336410522461,
      "activations/layer6_attention_weight_min": -52.11842727661133,
      "activations/layer7_attention_weight_max": 66.77140045166016,
      "activations/layer7_attention_weight_min": -70.46615600585938,
      "activations/layer8_attention_weight_max": 46.86765670776367,
      "activations/layer8_attention_weight_min": -55.16194534301758,
      "activations/layer9_attention_weight_max": 44.14404296875,
      "activations/layer9_attention_weight_min": -49.6169548034668,
      "epoch": 20.72,
      "learning_rate": 1.6546969696969695e-05,
      "loss": 2.7074,
      "step": 356600
    },
    {
      "activations/layer0_attention_weight_max": 15.62576675415039,
      "activations/layer0_attention_weight_min": -14.169306755065918,
      "activations/layer10_attention_weight_max": 39.33011245727539,
      "activations/layer10_attention_weight_min": -38.0592041015625,
      "activations/layer11_attention_weight_max": 37.991004943847656,
      "activations/layer11_attention_weight_min": -34.42550277709961,
      "activations/layer12_attention_weight_max": 21.21067237854004,
      "activations/layer12_attention_weight_min": -25.009296417236328,
      "activations/layer13_attention_weight_max": 49.647911071777344,
      "activations/layer13_attention_weight_min": -29.015893936157227,
      "activations/layer14_attention_weight_max": 41.41407012939453,
      "activations/layer14_attention_weight_min": -30.085323333740234,
      "activations/layer15_attention_weight_max": 41.74704360961914,
      "activations/layer15_attention_weight_min": -32.4384651184082,
      "activations/layer16_attention_weight_max": 37.38235092163086,
      "activations/layer16_attention_weight_min": -32.43991470336914,
      "activations/layer17_attention_weight_max": 55.170936584472656,
      "activations/layer17_attention_weight_min": -44.65971755981445,
      "activations/layer18_attention_weight_max": 51.1075325012207,
      "activations/layer18_attention_weight_min": -40.255950927734375,
      "activations/layer19_attention_weight_max": 28.000900268554688,
      "activations/layer19_attention_weight_min": -24.816003799438477,
      "activations/layer1_attention_weight_max": 16.064002990722656,
      "activations/layer1_attention_weight_min": -13.65914535522461,
      "activations/layer20_attention_weight_max": 28.848432540893555,
      "activations/layer20_attention_weight_min": -20.913728713989258,
      "activations/layer21_attention_weight_max": 47.108123779296875,
      "activations/layer21_attention_weight_min": -26.256969451904297,
      "activations/layer22_attention_weight_max": 30.448057174682617,
      "activations/layer22_attention_weight_min": -26.05894660949707,
      "activations/layer23_attention_weight_max": 42.790191650390625,
      "activations/layer23_attention_weight_min": -26.200119018554688,
      "activations/layer2_attention_weight_max": 31.540937423706055,
      "activations/layer2_attention_weight_min": -31.460927963256836,
      "activations/layer3_attention_weight_max": 89.72554779052734,
      "activations/layer3_attention_weight_min": -86.35944366455078,
      "activations/layer4_attention_weight_max": 83.39629364013672,
      "activations/layer4_attention_weight_min": -83.73412322998047,
      "activations/layer5_attention_weight_max": 62.39461898803711,
      "activations/layer5_attention_weight_min": -72.04914093017578,
      "activations/layer6_attention_weight_max": 49.3187141418457,
      "activations/layer6_attention_weight_min": -48.536006927490234,
      "activations/layer7_attention_weight_max": 69.31349182128906,
      "activations/layer7_attention_weight_min": -64.40335845947266,
      "activations/layer8_attention_weight_max": 48.17257308959961,
      "activations/layer8_attention_weight_min": -48.26010513305664,
      "activations/layer9_attention_weight_max": 53.67083740234375,
      "activations/layer9_attention_weight_min": -45.579833984375,
      "epoch": 20.72,
      "learning_rate": 1.65280303030303e-05,
      "loss": 2.7268,
      "step": 356650
    },
    {
      "activations/layer0_attention_weight_max": 16.307937622070312,
      "activations/layer0_attention_weight_min": -14.00446891784668,
      "activations/layer10_attention_weight_max": 35.05741500854492,
      "activations/layer10_attention_weight_min": -35.5003547668457,
      "activations/layer11_attention_weight_max": 37.333770751953125,
      "activations/layer11_attention_weight_min": -35.712772369384766,
      "activations/layer12_attention_weight_max": 19.06773567199707,
      "activations/layer12_attention_weight_min": -26.21997833251953,
      "activations/layer13_attention_weight_max": 31.866750717163086,
      "activations/layer13_attention_weight_min": -27.748538970947266,
      "activations/layer14_attention_weight_max": 36.571128845214844,
      "activations/layer14_attention_weight_min": -31.537826538085938,
      "activations/layer15_attention_weight_max": 34.775455474853516,
      "activations/layer15_attention_weight_min": -30.744352340698242,
      "activations/layer16_attention_weight_max": 35.07684326171875,
      "activations/layer16_attention_weight_min": -31.394710540771484,
      "activations/layer17_attention_weight_max": 51.83720397949219,
      "activations/layer17_attention_weight_min": -46.488433837890625,
      "activations/layer18_attention_weight_max": 49.42595291137695,
      "activations/layer18_attention_weight_min": -40.60906219482422,
      "activations/layer19_attention_weight_max": 25.89131736755371,
      "activations/layer19_attention_weight_min": -20.311677932739258,
      "activations/layer1_attention_weight_max": 15.098204612731934,
      "activations/layer1_attention_weight_min": -16.157514572143555,
      "activations/layer20_attention_weight_max": 23.1868839263916,
      "activations/layer20_attention_weight_min": -21.356735229492188,
      "activations/layer21_attention_weight_max": 44.1032600402832,
      "activations/layer21_attention_weight_min": -21.77562713623047,
      "activations/layer22_attention_weight_max": 33.1541633605957,
      "activations/layer22_attention_weight_min": -25.26296615600586,
      "activations/layer23_attention_weight_max": 43.21135711669922,
      "activations/layer23_attention_weight_min": -27.647544860839844,
      "activations/layer2_attention_weight_max": 31.681570053100586,
      "activations/layer2_attention_weight_min": -29.26845932006836,
      "activations/layer3_attention_weight_max": 89.7669677734375,
      "activations/layer3_attention_weight_min": -88.0047836303711,
      "activations/layer4_attention_weight_max": 85.65373992919922,
      "activations/layer4_attention_weight_min": -83.0161361694336,
      "activations/layer5_attention_weight_max": 62.7408561706543,
      "activations/layer5_attention_weight_min": -71.23648071289062,
      "activations/layer6_attention_weight_max": 47.64540481567383,
      "activations/layer6_attention_weight_min": -50.23911666870117,
      "activations/layer7_attention_weight_max": 66.61701965332031,
      "activations/layer7_attention_weight_min": -63.88289260864258,
      "activations/layer8_attention_weight_max": 48.3834228515625,
      "activations/layer8_attention_weight_min": -50.35817337036133,
      "activations/layer9_attention_weight_max": 49.25687026977539,
      "activations/layer9_attention_weight_min": -45.20909881591797,
      "epoch": 20.73,
      "learning_rate": 1.6509090909090908e-05,
      "loss": 2.7159,
      "step": 356700
    },
    {
      "activations/layer0_attention_weight_max": 16.13383674621582,
      "activations/layer0_attention_weight_min": -13.783111572265625,
      "activations/layer10_attention_weight_max": 34.36558532714844,
      "activations/layer10_attention_weight_min": -34.05048751831055,
      "activations/layer11_attention_weight_max": 32.153160095214844,
      "activations/layer11_attention_weight_min": -33.890167236328125,
      "activations/layer12_attention_weight_max": 19.00094985961914,
      "activations/layer12_attention_weight_min": -29.685014724731445,
      "activations/layer13_attention_weight_max": 33.576725006103516,
      "activations/layer13_attention_weight_min": -28.00999641418457,
      "activations/layer14_attention_weight_max": 35.48680114746094,
      "activations/layer14_attention_weight_min": -30.19866371154785,
      "activations/layer15_attention_weight_max": 31.913774490356445,
      "activations/layer15_attention_weight_min": -30.05255126953125,
      "activations/layer16_attention_weight_max": 34.738731384277344,
      "activations/layer16_attention_weight_min": -32.75811767578125,
      "activations/layer17_attention_weight_max": 51.75319290161133,
      "activations/layer17_attention_weight_min": -43.502830505371094,
      "activations/layer18_attention_weight_max": 47.68062210083008,
      "activations/layer18_attention_weight_min": -39.664188385009766,
      "activations/layer19_attention_weight_max": 24.740407943725586,
      "activations/layer19_attention_weight_min": -23.747570037841797,
      "activations/layer1_attention_weight_max": 16.153217315673828,
      "activations/layer1_attention_weight_min": -14.836652755737305,
      "activations/layer20_attention_weight_max": 23.519067764282227,
      "activations/layer20_attention_weight_min": -21.451345443725586,
      "activations/layer21_attention_weight_max": 37.425052642822266,
      "activations/layer21_attention_weight_min": -24.142906188964844,
      "activations/layer22_attention_weight_max": 28.4461727142334,
      "activations/layer22_attention_weight_min": -25.523088455200195,
      "activations/layer23_attention_weight_max": 35.989688873291016,
      "activations/layer23_attention_weight_min": -26.811695098876953,
      "activations/layer2_attention_weight_max": 31.75153923034668,
      "activations/layer2_attention_weight_min": -31.362491607666016,
      "activations/layer3_attention_weight_max": 90.06143951416016,
      "activations/layer3_attention_weight_min": -91.44538879394531,
      "activations/layer4_attention_weight_max": 84.23250579833984,
      "activations/layer4_attention_weight_min": -81.7728042602539,
      "activations/layer5_attention_weight_max": 60.38946533203125,
      "activations/layer5_attention_weight_min": -70.12136840820312,
      "activations/layer6_attention_weight_max": 49.9979362487793,
      "activations/layer6_attention_weight_min": -49.59677505493164,
      "activations/layer7_attention_weight_max": 65.34640502929688,
      "activations/layer7_attention_weight_min": -62.09777069091797,
      "activations/layer8_attention_weight_max": 46.93898010253906,
      "activations/layer8_attention_weight_min": -49.73871994018555,
      "activations/layer9_attention_weight_max": 41.39971923828125,
      "activations/layer9_attention_weight_min": -43.98701095581055,
      "epoch": 20.73,
      "learning_rate": 1.6490151515151516e-05,
      "loss": 2.7081,
      "step": 356750
    },
    {
      "activations/layer0_attention_weight_max": 15.771281242370605,
      "activations/layer0_attention_weight_min": -13.895173072814941,
      "activations/layer10_attention_weight_max": 35.4725341796875,
      "activations/layer10_attention_weight_min": -35.54029846191406,
      "activations/layer11_attention_weight_max": 31.28468132019043,
      "activations/layer11_attention_weight_min": -36.13031005859375,
      "activations/layer12_attention_weight_max": 17.556804656982422,
      "activations/layer12_attention_weight_min": -24.485868453979492,
      "activations/layer13_attention_weight_max": 33.474945068359375,
      "activations/layer13_attention_weight_min": -26.430644989013672,
      "activations/layer14_attention_weight_max": 32.698062896728516,
      "activations/layer14_attention_weight_min": -28.822036743164062,
      "activations/layer15_attention_weight_max": 30.552194595336914,
      "activations/layer15_attention_weight_min": -29.475492477416992,
      "activations/layer16_attention_weight_max": 36.015533447265625,
      "activations/layer16_attention_weight_min": -33.71152877807617,
      "activations/layer17_attention_weight_max": 52.880680084228516,
      "activations/layer17_attention_weight_min": -46.103092193603516,
      "activations/layer18_attention_weight_max": 47.19873046875,
      "activations/layer18_attention_weight_min": -40.72765350341797,
      "activations/layer19_attention_weight_max": 25.192310333251953,
      "activations/layer19_attention_weight_min": -24.86359977722168,
      "activations/layer1_attention_weight_max": 15.54178237915039,
      "activations/layer1_attention_weight_min": -14.315173149108887,
      "activations/layer20_attention_weight_max": 22.74193572998047,
      "activations/layer20_attention_weight_min": -22.38644790649414,
      "activations/layer21_attention_weight_max": 36.359466552734375,
      "activations/layer21_attention_weight_min": -22.954620361328125,
      "activations/layer22_attention_weight_max": 29.24776840209961,
      "activations/layer22_attention_weight_min": -26.79258918762207,
      "activations/layer23_attention_weight_max": 39.882972717285156,
      "activations/layer23_attention_weight_min": -23.239383697509766,
      "activations/layer2_attention_weight_max": 31.142017364501953,
      "activations/layer2_attention_weight_min": -31.256065368652344,
      "activations/layer3_attention_weight_max": 88.20065307617188,
      "activations/layer3_attention_weight_min": -87.8227310180664,
      "activations/layer4_attention_weight_max": 84.08129119873047,
      "activations/layer4_attention_weight_min": -83.11548614501953,
      "activations/layer5_attention_weight_max": 59.380279541015625,
      "activations/layer5_attention_weight_min": -75.7952880859375,
      "activations/layer6_attention_weight_max": 48.52779769897461,
      "activations/layer6_attention_weight_min": -49.64674758911133,
      "activations/layer7_attention_weight_max": 65.484375,
      "activations/layer7_attention_weight_min": -69.02520751953125,
      "activations/layer8_attention_weight_max": 46.94898223876953,
      "activations/layer8_attention_weight_min": -50.74611282348633,
      "activations/layer9_attention_weight_max": 42.36826705932617,
      "activations/layer9_attention_weight_min": -46.809688568115234,
      "epoch": 20.73,
      "learning_rate": 1.647121212121212e-05,
      "loss": 2.7245,
      "step": 356800
    },
    {
      "activations/layer0_attention_weight_max": 15.59903621673584,
      "activations/layer0_attention_weight_min": -13.735185623168945,
      "activations/layer10_attention_weight_max": 37.07305908203125,
      "activations/layer10_attention_weight_min": -39.43158721923828,
      "activations/layer11_attention_weight_max": 34.08271026611328,
      "activations/layer11_attention_weight_min": -37.54079055786133,
      "activations/layer12_attention_weight_max": 20.039472579956055,
      "activations/layer12_attention_weight_min": -28.540502548217773,
      "activations/layer13_attention_weight_max": 34.75212097167969,
      "activations/layer13_attention_weight_min": -28.051254272460938,
      "activations/layer14_attention_weight_max": 37.41072463989258,
      "activations/layer14_attention_weight_min": -30.890850067138672,
      "activations/layer15_attention_weight_max": 34.59829330444336,
      "activations/layer15_attention_weight_min": -29.72435188293457,
      "activations/layer16_attention_weight_max": 35.23589324951172,
      "activations/layer16_attention_weight_min": -33.151607513427734,
      "activations/layer17_attention_weight_max": 50.58204650878906,
      "activations/layer17_attention_weight_min": -46.28798294067383,
      "activations/layer18_attention_weight_max": 47.005638122558594,
      "activations/layer18_attention_weight_min": -39.72736740112305,
      "activations/layer19_attention_weight_max": 22.86919593811035,
      "activations/layer19_attention_weight_min": -22.668210983276367,
      "activations/layer1_attention_weight_max": 15.663652420043945,
      "activations/layer1_attention_weight_min": -14.768566131591797,
      "activations/layer20_attention_weight_max": 22.638259887695312,
      "activations/layer20_attention_weight_min": -21.668628692626953,
      "activations/layer21_attention_weight_max": 41.516395568847656,
      "activations/layer21_attention_weight_min": -24.9024600982666,
      "activations/layer22_attention_weight_max": 28.197195053100586,
      "activations/layer22_attention_weight_min": -26.71698760986328,
      "activations/layer23_attention_weight_max": 35.302650451660156,
      "activations/layer23_attention_weight_min": -29.755386352539062,
      "activations/layer2_attention_weight_max": 32.60518264770508,
      "activations/layer2_attention_weight_min": -30.14865493774414,
      "activations/layer3_attention_weight_max": 90.7475814819336,
      "activations/layer3_attention_weight_min": -89.2464370727539,
      "activations/layer4_attention_weight_max": 84.54142761230469,
      "activations/layer4_attention_weight_min": -88.82027435302734,
      "activations/layer5_attention_weight_max": 60.72602844238281,
      "activations/layer5_attention_weight_min": -68.70237731933594,
      "activations/layer6_attention_weight_max": 51.30855178833008,
      "activations/layer6_attention_weight_min": -52.93074417114258,
      "activations/layer7_attention_weight_max": 65.22703552246094,
      "activations/layer7_attention_weight_min": -66.735107421875,
      "activations/layer8_attention_weight_max": 46.86261749267578,
      "activations/layer8_attention_weight_min": -52.66645050048828,
      "activations/layer9_attention_weight_max": 42.5727424621582,
      "activations/layer9_attention_weight_min": -47.25776672363281,
      "epoch": 20.74,
      "learning_rate": 1.6452272727272726e-05,
      "loss": 2.7108,
      "step": 356850
    },
    {
      "activations/layer0_attention_weight_max": 16.465539932250977,
      "activations/layer0_attention_weight_min": -13.836140632629395,
      "activations/layer10_attention_weight_max": 42.03056716918945,
      "activations/layer10_attention_weight_min": -41.51643753051758,
      "activations/layer11_attention_weight_max": 41.822513580322266,
      "activations/layer11_attention_weight_min": -40.47300720214844,
      "activations/layer12_attention_weight_max": 21.894710540771484,
      "activations/layer12_attention_weight_min": -29.167692184448242,
      "activations/layer13_attention_weight_max": 38.91621398925781,
      "activations/layer13_attention_weight_min": -34.109466552734375,
      "activations/layer14_attention_weight_max": 36.376220703125,
      "activations/layer14_attention_weight_min": -32.73969268798828,
      "activations/layer15_attention_weight_max": 33.330787658691406,
      "activations/layer15_attention_weight_min": -29.5307559967041,
      "activations/layer16_attention_weight_max": 35.77079772949219,
      "activations/layer16_attention_weight_min": -34.17350769042969,
      "activations/layer17_attention_weight_max": 53.778236389160156,
      "activations/layer17_attention_weight_min": -46.732521057128906,
      "activations/layer18_attention_weight_max": 46.83906555175781,
      "activations/layer18_attention_weight_min": -40.053035736083984,
      "activations/layer19_attention_weight_max": 25.943002700805664,
      "activations/layer19_attention_weight_min": -22.342395782470703,
      "activations/layer1_attention_weight_max": 15.516969680786133,
      "activations/layer1_attention_weight_min": -13.42661190032959,
      "activations/layer20_attention_weight_max": 24.174449920654297,
      "activations/layer20_attention_weight_min": -21.990463256835938,
      "activations/layer21_attention_weight_max": 36.62113952636719,
      "activations/layer21_attention_weight_min": -22.24272346496582,
      "activations/layer22_attention_weight_max": 30.966033935546875,
      "activations/layer22_attention_weight_min": -26.158206939697266,
      "activations/layer23_attention_weight_max": 37.05613327026367,
      "activations/layer23_attention_weight_min": -23.299039840698242,
      "activations/layer2_attention_weight_max": 31.533498764038086,
      "activations/layer2_attention_weight_min": -30.74874496459961,
      "activations/layer3_attention_weight_max": 88.91719055175781,
      "activations/layer3_attention_weight_min": -92.48866271972656,
      "activations/layer4_attention_weight_max": 87.39261627197266,
      "activations/layer4_attention_weight_min": -85.8743667602539,
      "activations/layer5_attention_weight_max": 65.27194213867188,
      "activations/layer5_attention_weight_min": -71.23011016845703,
      "activations/layer6_attention_weight_max": 52.71572494506836,
      "activations/layer6_attention_weight_min": -52.927818298339844,
      "activations/layer7_attention_weight_max": 68.53948974609375,
      "activations/layer7_attention_weight_min": -70.20349884033203,
      "activations/layer8_attention_weight_max": 51.7441291809082,
      "activations/layer8_attention_weight_min": -56.715362548828125,
      "activations/layer9_attention_weight_max": 49.796791076660156,
      "activations/layer9_attention_weight_min": -50.221397399902344,
      "epoch": 20.74,
      "learning_rate": 1.643333333333333e-05,
      "loss": 2.7139,
      "step": 356900
    },
    {
      "activations/layer0_attention_weight_max": 15.661201477050781,
      "activations/layer0_attention_weight_min": -13.974435806274414,
      "activations/layer10_attention_weight_max": 36.376953125,
      "activations/layer10_attention_weight_min": -35.565589904785156,
      "activations/layer11_attention_weight_max": 32.55479431152344,
      "activations/layer11_attention_weight_min": -34.610015869140625,
      "activations/layer12_attention_weight_max": 18.854873657226562,
      "activations/layer12_attention_weight_min": -26.13556671142578,
      "activations/layer13_attention_weight_max": 35.676910400390625,
      "activations/layer13_attention_weight_min": -28.116348266601562,
      "activations/layer14_attention_weight_max": 37.52193069458008,
      "activations/layer14_attention_weight_min": -31.053970336914062,
      "activations/layer15_attention_weight_max": 35.19618606567383,
      "activations/layer15_attention_weight_min": -31.83207130432129,
      "activations/layer16_attention_weight_max": 35.61525344848633,
      "activations/layer16_attention_weight_min": -33.9971809387207,
      "activations/layer17_attention_weight_max": 52.645179748535156,
      "activations/layer17_attention_weight_min": -48.68460464477539,
      "activations/layer18_attention_weight_max": 48.26767349243164,
      "activations/layer18_attention_weight_min": -41.716182708740234,
      "activations/layer19_attention_weight_max": 26.242294311523438,
      "activations/layer19_attention_weight_min": -23.6539306640625,
      "activations/layer1_attention_weight_max": 15.177318572998047,
      "activations/layer1_attention_weight_min": -14.652321815490723,
      "activations/layer20_attention_weight_max": 25.1248722076416,
      "activations/layer20_attention_weight_min": -21.950227737426758,
      "activations/layer21_attention_weight_max": 43.90078353881836,
      "activations/layer21_attention_weight_min": -26.46209716796875,
      "activations/layer22_attention_weight_max": 30.6590633392334,
      "activations/layer22_attention_weight_min": -25.50563621520996,
      "activations/layer23_attention_weight_max": 36.99833297729492,
      "activations/layer23_attention_weight_min": -25.432418823242188,
      "activations/layer2_attention_weight_max": 31.550132751464844,
      "activations/layer2_attention_weight_min": -31.175350189208984,
      "activations/layer3_attention_weight_max": 88.67979431152344,
      "activations/layer3_attention_weight_min": -88.9148178100586,
      "activations/layer4_attention_weight_max": 81.8646240234375,
      "activations/layer4_attention_weight_min": -81.8916244506836,
      "activations/layer5_attention_weight_max": 60.45332336425781,
      "activations/layer5_attention_weight_min": -73.26824951171875,
      "activations/layer6_attention_weight_max": 47.62491226196289,
      "activations/layer6_attention_weight_min": -50.642494201660156,
      "activations/layer7_attention_weight_max": 67.084228515625,
      "activations/layer7_attention_weight_min": -64.6418685913086,
      "activations/layer8_attention_weight_max": 48.117530822753906,
      "activations/layer8_attention_weight_min": -51.20671844482422,
      "activations/layer9_attention_weight_max": 43.488433837890625,
      "activations/layer9_attention_weight_min": -45.74189758300781,
      "epoch": 20.74,
      "learning_rate": 1.641439393939394e-05,
      "loss": 2.7117,
      "step": 356950
    },
    {
      "activations/layer0_attention_weight_max": 15.652005195617676,
      "activations/layer0_attention_weight_min": -13.907078742980957,
      "activations/layer10_attention_weight_max": 36.86486053466797,
      "activations/layer10_attention_weight_min": -36.27350616455078,
      "activations/layer11_attention_weight_max": 33.12108612060547,
      "activations/layer11_attention_weight_min": -34.771148681640625,
      "activations/layer12_attention_weight_max": 17.412202835083008,
      "activations/layer12_attention_weight_min": -23.977466583251953,
      "activations/layer13_attention_weight_max": 34.07613754272461,
      "activations/layer13_attention_weight_min": -31.199560165405273,
      "activations/layer14_attention_weight_max": 37.39064025878906,
      "activations/layer14_attention_weight_min": -32.506282806396484,
      "activations/layer15_attention_weight_max": 34.02141189575195,
      "activations/layer15_attention_weight_min": -30.702375411987305,
      "activations/layer16_attention_weight_max": 33.496826171875,
      "activations/layer16_attention_weight_min": -34.597129821777344,
      "activations/layer17_attention_weight_max": 52.05160140991211,
      "activations/layer17_attention_weight_min": -46.22432327270508,
      "activations/layer18_attention_weight_max": 47.506221771240234,
      "activations/layer18_attention_weight_min": -41.94427490234375,
      "activations/layer19_attention_weight_max": 23.64459800720215,
      "activations/layer19_attention_weight_min": -22.255842208862305,
      "activations/layer1_attention_weight_max": 15.181944847106934,
      "activations/layer1_attention_weight_min": -12.372605323791504,
      "activations/layer20_attention_weight_max": 23.191911697387695,
      "activations/layer20_attention_weight_min": -21.828683853149414,
      "activations/layer21_attention_weight_max": 32.67073059082031,
      "activations/layer21_attention_weight_min": -21.569089889526367,
      "activations/layer22_attention_weight_max": 29.53776741027832,
      "activations/layer22_attention_weight_min": -25.599140167236328,
      "activations/layer23_attention_weight_max": 40.90668869018555,
      "activations/layer23_attention_weight_min": -24.46385955810547,
      "activations/layer2_attention_weight_max": 33.09668731689453,
      "activations/layer2_attention_weight_min": -32.19880676269531,
      "activations/layer3_attention_weight_max": 93.26393127441406,
      "activations/layer3_attention_weight_min": -94.86966705322266,
      "activations/layer4_attention_weight_max": 88.46631622314453,
      "activations/layer4_attention_weight_min": -87.64401245117188,
      "activations/layer5_attention_weight_max": 62.41239547729492,
      "activations/layer5_attention_weight_min": -70.70600128173828,
      "activations/layer6_attention_weight_max": 53.8979377746582,
      "activations/layer6_attention_weight_min": -55.008995056152344,
      "activations/layer7_attention_weight_max": 73.12322235107422,
      "activations/layer7_attention_weight_min": -68.73416137695312,
      "activations/layer8_attention_weight_max": 52.72935104370117,
      "activations/layer8_attention_weight_min": -52.869773864746094,
      "activations/layer9_attention_weight_max": 43.405906677246094,
      "activations/layer9_attention_weight_min": -46.21687316894531,
      "epoch": 20.74,
      "learning_rate": 1.6395454545454544e-05,
      "loss": 2.718,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.6159,
      "eval_samples_per_second": 498.379,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.6159,
      "eval_openwebtext_samples_per_second": 498.379,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_wikitext_loss": 2.923828125,
      "eval_wikitext_ppl": 18.61240185635776,
      "eval_wikitext_runtime": 2.0227,
      "eval_wikitext_samples_per_second": 225.447,
      "step": 357000
    },
    {
      "epoch": 20.74,
      "eval_lambada_loss": 2.435546875,
      "eval_lambada_ppl": 11.422063446614425,
      "eval_lambada_runtime": 9.6241,
      "eval_lambada_samples_per_second": 505.918,
      "step": 357000
    },
    {
      "activations/layer0_attention_weight_max": 15.163351058959961,
      "activations/layer0_attention_weight_min": -13.658060073852539,
      "activations/layer10_attention_weight_max": 34.644710540771484,
      "activations/layer10_attention_weight_min": -35.9439697265625,
      "activations/layer11_attention_weight_max": 32.728214263916016,
      "activations/layer11_attention_weight_min": -37.390708923339844,
      "activations/layer12_attention_weight_max": 17.883085250854492,
      "activations/layer12_attention_weight_min": -28.453763961791992,
      "activations/layer13_attention_weight_max": 37.001522064208984,
      "activations/layer13_attention_weight_min": -29.530916213989258,
      "activations/layer14_attention_weight_max": 42.86587905883789,
      "activations/layer14_attention_weight_min": -31.377046585083008,
      "activations/layer15_attention_weight_max": 42.86869430541992,
      "activations/layer15_attention_weight_min": -32.44248580932617,
      "activations/layer16_attention_weight_max": 44.63681411743164,
      "activations/layer16_attention_weight_min": -38.84906768798828,
      "activations/layer17_attention_weight_max": 65.56487274169922,
      "activations/layer17_attention_weight_min": -53.77632522583008,
      "activations/layer18_attention_weight_max": 63.342838287353516,
      "activations/layer18_attention_weight_min": -46.533565521240234,
      "activations/layer19_attention_weight_max": 33.194026947021484,
      "activations/layer19_attention_weight_min": -28.496694564819336,
      "activations/layer1_attention_weight_max": 15.398262023925781,
      "activations/layer1_attention_weight_min": -14.499482154846191,
      "activations/layer20_attention_weight_max": 34.82408142089844,
      "activations/layer20_attention_weight_min": -24.216215133666992,
      "activations/layer21_attention_weight_max": 58.39064407348633,
      "activations/layer21_attention_weight_min": -24.69482421875,
      "activations/layer22_attention_weight_max": 39.03184127807617,
      "activations/layer22_attention_weight_min": -27.38622283935547,
      "activations/layer23_attention_weight_max": 43.72552490234375,
      "activations/layer23_attention_weight_min": -24.819561004638672,
      "activations/layer2_attention_weight_max": 34.83327102661133,
      "activations/layer2_attention_weight_min": -33.892250061035156,
      "activations/layer3_attention_weight_max": 102.51971435546875,
      "activations/layer3_attention_weight_min": -95.46683502197266,
      "activations/layer4_attention_weight_max": 87.56038665771484,
      "activations/layer4_attention_weight_min": -86.41305541992188,
      "activations/layer5_attention_weight_max": 60.50225830078125,
      "activations/layer5_attention_weight_min": -73.62044525146484,
      "activations/layer6_attention_weight_max": 46.70441818237305,
      "activations/layer6_attention_weight_min": -48.33360290527344,
      "activations/layer7_attention_weight_max": 70.06004333496094,
      "activations/layer7_attention_weight_min": -66.7784652709961,
      "activations/layer8_attention_weight_max": 45.72643280029297,
      "activations/layer8_attention_weight_min": -49.2344856262207,
      "activations/layer9_attention_weight_max": 42.58946990966797,
      "activations/layer9_attention_weight_min": -44.10176086425781,
      "epoch": 20.75,
      "learning_rate": 1.637651515151515e-05,
      "loss": 2.7298,
      "step": 357050
    },
    {
      "activations/layer0_attention_weight_max": 16.01601791381836,
      "activations/layer0_attention_weight_min": -14.000325202941895,
      "activations/layer10_attention_weight_max": 40.1749382019043,
      "activations/layer10_attention_weight_min": -37.523681640625,
      "activations/layer11_attention_weight_max": 38.551734924316406,
      "activations/layer11_attention_weight_min": -36.55470657348633,
      "activations/layer12_attention_weight_max": 21.129831314086914,
      "activations/layer12_attention_weight_min": -23.52579116821289,
      "activations/layer13_attention_weight_max": 40.53154373168945,
      "activations/layer13_attention_weight_min": -28.73655128479004,
      "activations/layer14_attention_weight_max": 37.95103454589844,
      "activations/layer14_attention_weight_min": -34.14503860473633,
      "activations/layer15_attention_weight_max": 37.226436614990234,
      "activations/layer15_attention_weight_min": -30.784067153930664,
      "activations/layer16_attention_weight_max": 38.73587417602539,
      "activations/layer16_attention_weight_min": -34.57958221435547,
      "activations/layer17_attention_weight_max": 59.11185073852539,
      "activations/layer17_attention_weight_min": -47.28703308105469,
      "activations/layer18_attention_weight_max": 55.07787322998047,
      "activations/layer18_attention_weight_min": -43.22509002685547,
      "activations/layer19_attention_weight_max": 28.121143341064453,
      "activations/layer19_attention_weight_min": -24.988142013549805,
      "activations/layer1_attention_weight_max": 16.642976760864258,
      "activations/layer1_attention_weight_min": -15.606170654296875,
      "activations/layer20_attention_weight_max": 27.918039321899414,
      "activations/layer20_attention_weight_min": -20.895280838012695,
      "activations/layer21_attention_weight_max": 47.13853073120117,
      "activations/layer21_attention_weight_min": -24.517221450805664,
      "activations/layer22_attention_weight_max": 37.721004486083984,
      "activations/layer22_attention_weight_min": -26.53364372253418,
      "activations/layer23_attention_weight_max": 41.32375717163086,
      "activations/layer23_attention_weight_min": -24.186918258666992,
      "activations/layer2_attention_weight_max": 33.19166946411133,
      "activations/layer2_attention_weight_min": -31.813962936401367,
      "activations/layer3_attention_weight_max": 89.59489440917969,
      "activations/layer3_attention_weight_min": -87.76888275146484,
      "activations/layer4_attention_weight_max": 82.0472183227539,
      "activations/layer4_attention_weight_min": -83.30545806884766,
      "activations/layer5_attention_weight_max": 63.15134811401367,
      "activations/layer5_attention_weight_min": -72.11555480957031,
      "activations/layer6_attention_weight_max": 47.819271087646484,
      "activations/layer6_attention_weight_min": -53.396297454833984,
      "activations/layer7_attention_weight_max": 67.44100952148438,
      "activations/layer7_attention_weight_min": -65.45994567871094,
      "activations/layer8_attention_weight_max": 51.794734954833984,
      "activations/layer8_attention_weight_min": -53.480472564697266,
      "activations/layer9_attention_weight_max": 49.25267028808594,
      "activations/layer9_attention_weight_min": -49.03384017944336,
      "epoch": 20.75,
      "learning_rate": 1.6357575757575757e-05,
      "loss": 2.7006,
      "step": 357100
    },
    {
      "activations/layer0_attention_weight_max": 15.305929183959961,
      "activations/layer0_attention_weight_min": -13.96777629852295,
      "activations/layer10_attention_weight_max": 35.74025344848633,
      "activations/layer10_attention_weight_min": -33.83063888549805,
      "activations/layer11_attention_weight_max": 32.135475158691406,
      "activations/layer11_attention_weight_min": -35.71635055541992,
      "activations/layer12_attention_weight_max": 17.940876007080078,
      "activations/layer12_attention_weight_min": -24.656578063964844,
      "activations/layer13_attention_weight_max": 31.6943359375,
      "activations/layer13_attention_weight_min": -25.283498764038086,
      "activations/layer14_attention_weight_max": 34.62372970581055,
      "activations/layer14_attention_weight_min": -29.541378021240234,
      "activations/layer15_attention_weight_max": 33.923954010009766,
      "activations/layer15_attention_weight_min": -30.092527389526367,
      "activations/layer16_attention_weight_max": 33.30366897583008,
      "activations/layer16_attention_weight_min": -31.54615020751953,
      "activations/layer17_attention_weight_max": 49.23804473876953,
      "activations/layer17_attention_weight_min": -44.24797439575195,
      "activations/layer18_attention_weight_max": 43.97308349609375,
      "activations/layer18_attention_weight_min": -40.47099685668945,
      "activations/layer19_attention_weight_max": 24.882719039916992,
      "activations/layer19_attention_weight_min": -22.7911376953125,
      "activations/layer1_attention_weight_max": 16.228723526000977,
      "activations/layer1_attention_weight_min": -12.332571983337402,
      "activations/layer20_attention_weight_max": 21.067279815673828,
      "activations/layer20_attention_weight_min": -21.033611297607422,
      "activations/layer21_attention_weight_max": 34.13576889038086,
      "activations/layer21_attention_weight_min": -22.543996810913086,
      "activations/layer22_attention_weight_max": 28.31500244140625,
      "activations/layer22_attention_weight_min": -25.621931076049805,
      "activations/layer23_attention_weight_max": 32.97578048706055,
      "activations/layer23_attention_weight_min": -24.984527587890625,
      "activations/layer2_attention_weight_max": 28.137226104736328,
      "activations/layer2_attention_weight_min": -29.1060848236084,
      "activations/layer3_attention_weight_max": 85.06880187988281,
      "activations/layer3_attention_weight_min": -86.78951263427734,
      "activations/layer4_attention_weight_max": 82.24124908447266,
      "activations/layer4_attention_weight_min": -82.51683044433594,
      "activations/layer5_attention_weight_max": 60.21966552734375,
      "activations/layer5_attention_weight_min": -70.70487976074219,
      "activations/layer6_attention_weight_max": 48.289798736572266,
      "activations/layer6_attention_weight_min": -49.88972473144531,
      "activations/layer7_attention_weight_max": 66.46674346923828,
      "activations/layer7_attention_weight_min": -62.75547409057617,
      "activations/layer8_attention_weight_max": 45.34090042114258,
      "activations/layer8_attention_weight_min": -48.9894905090332,
      "activations/layer9_attention_weight_max": 45.22825622558594,
      "activations/layer9_attention_weight_min": -46.60969161987305,
      "epoch": 20.75,
      "learning_rate": 1.6338636363636362e-05,
      "loss": 2.7211,
      "step": 357150
    },
    {
      "activations/layer0_attention_weight_max": 15.64639949798584,
      "activations/layer0_attention_weight_min": -13.87820053100586,
      "activations/layer10_attention_weight_max": 35.1917724609375,
      "activations/layer10_attention_weight_min": -36.66157913208008,
      "activations/layer11_attention_weight_max": 32.531951904296875,
      "activations/layer11_attention_weight_min": -35.16458511352539,
      "activations/layer12_attention_weight_max": 20.252573013305664,
      "activations/layer12_attention_weight_min": -23.981544494628906,
      "activations/layer13_attention_weight_max": 34.23739242553711,
      "activations/layer13_attention_weight_min": -26.30060577392578,
      "activations/layer14_attention_weight_max": 34.81477737426758,
      "activations/layer14_attention_weight_min": -30.051897048950195,
      "activations/layer15_attention_weight_max": 31.601659774780273,
      "activations/layer15_attention_weight_min": -30.755863189697266,
      "activations/layer16_attention_weight_max": 34.05802536010742,
      "activations/layer16_attention_weight_min": -31.57079315185547,
      "activations/layer17_attention_weight_max": 49.9487190246582,
      "activations/layer17_attention_weight_min": -49.860198974609375,
      "activations/layer18_attention_weight_max": 45.55885314941406,
      "activations/layer18_attention_weight_min": -41.696502685546875,
      "activations/layer19_attention_weight_max": 24.3818302154541,
      "activations/layer19_attention_weight_min": -24.58486557006836,
      "activations/layer1_attention_weight_max": 15.774118423461914,
      "activations/layer1_attention_weight_min": -13.727883338928223,
      "activations/layer20_attention_weight_max": 26.450477600097656,
      "activations/layer20_attention_weight_min": -21.19607925415039,
      "activations/layer21_attention_weight_max": 33.19864273071289,
      "activations/layer21_attention_weight_min": -21.07904052734375,
      "activations/layer22_attention_weight_max": 32.25343704223633,
      "activations/layer22_attention_weight_min": -25.546796798706055,
      "activations/layer23_attention_weight_max": 40.47545623779297,
      "activations/layer23_attention_weight_min": -24.251211166381836,
      "activations/layer2_attention_weight_max": 31.253604888916016,
      "activations/layer2_attention_weight_min": -29.321592330932617,
      "activations/layer3_attention_weight_max": 91.0158462524414,
      "activations/layer3_attention_weight_min": -85.93266296386719,
      "activations/layer4_attention_weight_max": 86.03421020507812,
      "activations/layer4_attention_weight_min": -82.57123565673828,
      "activations/layer5_attention_weight_max": 67.03294372558594,
      "activations/layer5_attention_weight_min": -75.90251922607422,
      "activations/layer6_attention_weight_max": 50.48997116088867,
      "activations/layer6_attention_weight_min": -52.05241394042969,
      "activations/layer7_attention_weight_max": 69.3302993774414,
      "activations/layer7_attention_weight_min": -68.19571685791016,
      "activations/layer8_attention_weight_max": 49.4151611328125,
      "activations/layer8_attention_weight_min": -50.70703125,
      "activations/layer9_attention_weight_max": 42.92338943481445,
      "activations/layer9_attention_weight_min": -46.269161224365234,
      "epoch": 20.76,
      "learning_rate": 1.6319696969696967e-05,
      "loss": 2.7013,
      "step": 357200
    },
    {
      "activations/layer0_attention_weight_max": 15.258105278015137,
      "activations/layer0_attention_weight_min": -13.84996223449707,
      "activations/layer10_attention_weight_max": 36.53260803222656,
      "activations/layer10_attention_weight_min": -35.2279167175293,
      "activations/layer11_attention_weight_max": 33.212074279785156,
      "activations/layer11_attention_weight_min": -33.605106353759766,
      "activations/layer12_attention_weight_max": 19.332441329956055,
      "activations/layer12_attention_weight_min": -21.932418823242188,
      "activations/layer13_attention_weight_max": 32.97983169555664,
      "activations/layer13_attention_weight_min": -27.008291244506836,
      "activations/layer14_attention_weight_max": 35.46760940551758,
      "activations/layer14_attention_weight_min": -29.217180252075195,
      "activations/layer15_attention_weight_max": 32.409244537353516,
      "activations/layer15_attention_weight_min": -30.732206344604492,
      "activations/layer16_attention_weight_max": 32.766571044921875,
      "activations/layer16_attention_weight_min": -32.16257095336914,
      "activations/layer17_attention_weight_max": 48.7762336730957,
      "activations/layer17_attention_weight_min": -45.83325958251953,
      "activations/layer18_attention_weight_max": 45.99066925048828,
      "activations/layer18_attention_weight_min": -40.60601806640625,
      "activations/layer19_attention_weight_max": 23.475309371948242,
      "activations/layer19_attention_weight_min": -20.390100479125977,
      "activations/layer1_attention_weight_max": 15.489243507385254,
      "activations/layer1_attention_weight_min": -13.77656364440918,
      "activations/layer20_attention_weight_max": 21.81668472290039,
      "activations/layer20_attention_weight_min": -21.853078842163086,
      "activations/layer21_attention_weight_max": 33.94685745239258,
      "activations/layer21_attention_weight_min": -24.3846492767334,
      "activations/layer22_attention_weight_max": 27.764163970947266,
      "activations/layer22_attention_weight_min": -27.21236228942871,
      "activations/layer23_attention_weight_max": 37.47400665283203,
      "activations/layer23_attention_weight_min": -24.315624237060547,
      "activations/layer2_attention_weight_max": 31.5058650970459,
      "activations/layer2_attention_weight_min": -30.392593383789062,
      "activations/layer3_attention_weight_max": 87.48894500732422,
      "activations/layer3_attention_weight_min": -86.90477752685547,
      "activations/layer4_attention_weight_max": 83.8108139038086,
      "activations/layer4_attention_weight_min": -83.70841217041016,
      "activations/layer5_attention_weight_max": 60.191768646240234,
      "activations/layer5_attention_weight_min": -72.01538848876953,
      "activations/layer6_attention_weight_max": 49.291160583496094,
      "activations/layer6_attention_weight_min": -49.6564826965332,
      "activations/layer7_attention_weight_max": 64.41212463378906,
      "activations/layer7_attention_weight_min": -65.45985412597656,
      "activations/layer8_attention_weight_max": 50.50010681152344,
      "activations/layer8_attention_weight_min": -49.797672271728516,
      "activations/layer9_attention_weight_max": 43.8809700012207,
      "activations/layer9_attention_weight_min": -47.676029205322266,
      "epoch": 20.76,
      "learning_rate": 1.630113636363636e-05,
      "loss": 2.7122,
      "step": 357250
    },
    {
      "activations/layer0_attention_weight_max": 16.20622444152832,
      "activations/layer0_attention_weight_min": -13.843095779418945,
      "activations/layer10_attention_weight_max": 35.116390228271484,
      "activations/layer10_attention_weight_min": -35.2512092590332,
      "activations/layer11_attention_weight_max": 32.970008850097656,
      "activations/layer11_attention_weight_min": -35.36006546020508,
      "activations/layer12_attention_weight_max": 19.28357696533203,
      "activations/layer12_attention_weight_min": -24.19226837158203,
      "activations/layer13_attention_weight_max": 34.603965759277344,
      "activations/layer13_attention_weight_min": -29.2988338470459,
      "activations/layer14_attention_weight_max": 38.93425750732422,
      "activations/layer14_attention_weight_min": -32.8734016418457,
      "activations/layer15_attention_weight_max": 35.59102249145508,
      "activations/layer15_attention_weight_min": -32.23175048828125,
      "activations/layer16_attention_weight_max": 35.59724426269531,
      "activations/layer16_attention_weight_min": -36.36732864379883,
      "activations/layer17_attention_weight_max": 53.76936721801758,
      "activations/layer17_attention_weight_min": -48.7586669921875,
      "activations/layer18_attention_weight_max": 48.55324172973633,
      "activations/layer18_attention_weight_min": -44.08564758300781,
      "activations/layer19_attention_weight_max": 25.4425106048584,
      "activations/layer19_attention_weight_min": -25.91037368774414,
      "activations/layer1_attention_weight_max": 15.243614196777344,
      "activations/layer1_attention_weight_min": -14.04274845123291,
      "activations/layer20_attention_weight_max": 23.897214889526367,
      "activations/layer20_attention_weight_min": -21.983606338500977,
      "activations/layer21_attention_weight_max": 37.43358612060547,
      "activations/layer21_attention_weight_min": -22.743026733398438,
      "activations/layer22_attention_weight_max": 30.92277717590332,
      "activations/layer22_attention_weight_min": -29.43145751953125,
      "activations/layer23_attention_weight_max": 37.51234817504883,
      "activations/layer23_attention_weight_min": -24.293546676635742,
      "activations/layer2_attention_weight_max": 31.833654403686523,
      "activations/layer2_attention_weight_min": -30.147762298583984,
      "activations/layer3_attention_weight_max": 86.87245178222656,
      "activations/layer3_attention_weight_min": -86.03665161132812,
      "activations/layer4_attention_weight_max": 80.15904998779297,
      "activations/layer4_attention_weight_min": -81.02859497070312,
      "activations/layer5_attention_weight_max": 59.5754508972168,
      "activations/layer5_attention_weight_min": -68.05591583251953,
      "activations/layer6_attention_weight_max": 47.75367736816406,
      "activations/layer6_attention_weight_min": -48.240543365478516,
      "activations/layer7_attention_weight_max": 65.3873291015625,
      "activations/layer7_attention_weight_min": -63.177574157714844,
      "activations/layer8_attention_weight_max": 45.400428771972656,
      "activations/layer8_attention_weight_min": -47.97636413574219,
      "activations/layer9_attention_weight_max": 44.28181076049805,
      "activations/layer9_attention_weight_min": -46.517234802246094,
      "epoch": 20.76,
      "learning_rate": 1.628219696969697e-05,
      "loss": 2.721,
      "step": 357300
    },
    {
      "activations/layer0_attention_weight_max": 15.242926597595215,
      "activations/layer0_attention_weight_min": -13.969106674194336,
      "activations/layer10_attention_weight_max": 42.12333297729492,
      "activations/layer10_attention_weight_min": -39.139923095703125,
      "activations/layer11_attention_weight_max": 39.081504821777344,
      "activations/layer11_attention_weight_min": -35.69377899169922,
      "activations/layer12_attention_weight_max": 17.887252807617188,
      "activations/layer12_attention_weight_min": -28.960391998291016,
      "activations/layer13_attention_weight_max": 35.280181884765625,
      "activations/layer13_attention_weight_min": -28.19353675842285,
      "activations/layer14_attention_weight_max": 34.029911041259766,
      "activations/layer14_attention_weight_min": -31.66388511657715,
      "activations/layer15_attention_weight_max": 31.895708084106445,
      "activations/layer15_attention_weight_min": -29.452564239501953,
      "activations/layer16_attention_weight_max": 31.819974899291992,
      "activations/layer16_attention_weight_min": -29.448307037353516,
      "activations/layer17_attention_weight_max": 49.97300338745117,
      "activations/layer17_attention_weight_min": -45.33297348022461,
      "activations/layer18_attention_weight_max": 45.08888244628906,
      "activations/layer18_attention_weight_min": -41.99953079223633,
      "activations/layer19_attention_weight_max": 23.052770614624023,
      "activations/layer19_attention_weight_min": -23.773136138916016,
      "activations/layer1_attention_weight_max": 14.706439971923828,
      "activations/layer1_attention_weight_min": -13.988213539123535,
      "activations/layer20_attention_weight_max": 22.1698055267334,
      "activations/layer20_attention_weight_min": -22.130046844482422,
      "activations/layer21_attention_weight_max": 34.70259475708008,
      "activations/layer21_attention_weight_min": -20.09543800354004,
      "activations/layer22_attention_weight_max": 29.05933952331543,
      "activations/layer22_attention_weight_min": -24.96000099182129,
      "activations/layer23_attention_weight_max": 40.5004997253418,
      "activations/layer23_attention_weight_min": -23.550472259521484,
      "activations/layer2_attention_weight_max": 34.45149230957031,
      "activations/layer2_attention_weight_min": -32.4698371887207,
      "activations/layer3_attention_weight_max": 97.35317993164062,
      "activations/layer3_attention_weight_min": -91.737060546875,
      "activations/layer4_attention_weight_max": 87.10435485839844,
      "activations/layer4_attention_weight_min": -83.32807159423828,
      "activations/layer5_attention_weight_max": 60.009578704833984,
      "activations/layer5_attention_weight_min": -70.12677001953125,
      "activations/layer6_attention_weight_max": 51.06716537475586,
      "activations/layer6_attention_weight_min": -50.68783950805664,
      "activations/layer7_attention_weight_max": 73.24150085449219,
      "activations/layer7_attention_weight_min": -65.93838500976562,
      "activations/layer8_attention_weight_max": 51.03573226928711,
      "activations/layer8_attention_weight_min": -52.218746185302734,
      "activations/layer9_attention_weight_max": 52.97976303100586,
      "activations/layer9_attention_weight_min": -48.078678131103516,
      "epoch": 20.76,
      "learning_rate": 1.6263257575757574e-05,
      "loss": 2.7294,
      "step": 357350
    },
    {
      "activations/layer0_attention_weight_max": 15.847771644592285,
      "activations/layer0_attention_weight_min": -13.9370698928833,
      "activations/layer10_attention_weight_max": 34.46331787109375,
      "activations/layer10_attention_weight_min": -33.39705276489258,
      "activations/layer11_attention_weight_max": 32.358184814453125,
      "activations/layer11_attention_weight_min": -32.482887268066406,
      "activations/layer12_attention_weight_max": 18.0970401763916,
      "activations/layer12_attention_weight_min": -23.6575870513916,
      "activations/layer13_attention_weight_max": 31.819438934326172,
      "activations/layer13_attention_weight_min": -26.547218322753906,
      "activations/layer14_attention_weight_max": 33.841495513916016,
      "activations/layer14_attention_weight_min": -28.505891799926758,
      "activations/layer15_attention_weight_max": 29.803726196289062,
      "activations/layer15_attention_weight_min": -29.521310806274414,
      "activations/layer16_attention_weight_max": 29.855676651000977,
      "activations/layer16_attention_weight_min": -30.640180587768555,
      "activations/layer17_attention_weight_max": 50.29402542114258,
      "activations/layer17_attention_weight_min": -43.923500061035156,
      "activations/layer18_attention_weight_max": 41.95686721801758,
      "activations/layer18_attention_weight_min": -38.49330520629883,
      "activations/layer19_attention_weight_max": 25.320602416992188,
      "activations/layer19_attention_weight_min": -22.05609130859375,
      "activations/layer1_attention_weight_max": 15.50676441192627,
      "activations/layer1_attention_weight_min": -14.431387901306152,
      "activations/layer20_attention_weight_max": 20.73049545288086,
      "activations/layer20_attention_weight_min": -23.114185333251953,
      "activations/layer21_attention_weight_max": 31.12050437927246,
      "activations/layer21_attention_weight_min": -21.021921157836914,
      "activations/layer22_attention_weight_max": 28.160140991210938,
      "activations/layer22_attention_weight_min": -25.889713287353516,
      "activations/layer23_attention_weight_max": 35.75521469116211,
      "activations/layer23_attention_weight_min": -24.108558654785156,
      "activations/layer2_attention_weight_max": 30.696964263916016,
      "activations/layer2_attention_weight_min": -30.682546615600586,
      "activations/layer3_attention_weight_max": 85.54179382324219,
      "activations/layer3_attention_weight_min": -85.09765625,
      "activations/layer4_attention_weight_max": 79.00032806396484,
      "activations/layer4_attention_weight_min": -78.78447723388672,
      "activations/layer5_attention_weight_max": 57.25171661376953,
      "activations/layer5_attention_weight_min": -65.35624694824219,
      "activations/layer6_attention_weight_max": 51.08113098144531,
      "activations/layer6_attention_weight_min": -51.80358123779297,
      "activations/layer7_attention_weight_max": 63.83763885498047,
      "activations/layer7_attention_weight_min": -63.14033889770508,
      "activations/layer8_attention_weight_max": 45.8904914855957,
      "activations/layer8_attention_weight_min": -50.54811477661133,
      "activations/layer9_attention_weight_max": 41.553810119628906,
      "activations/layer9_attention_weight_min": -44.13297653198242,
      "epoch": 20.77,
      "learning_rate": 1.624431818181818e-05,
      "loss": 2.7181,
      "step": 357400
    },
    {
      "activations/layer0_attention_weight_max": 15.628132820129395,
      "activations/layer0_attention_weight_min": -13.565033912658691,
      "activations/layer10_attention_weight_max": 38.427433013916016,
      "activations/layer10_attention_weight_min": -36.47954559326172,
      "activations/layer11_attention_weight_max": 33.746585845947266,
      "activations/layer11_attention_weight_min": -36.57830810546875,
      "activations/layer12_attention_weight_max": 18.813949584960938,
      "activations/layer12_attention_weight_min": -21.373281478881836,
      "activations/layer13_attention_weight_max": 39.94628143310547,
      "activations/layer13_attention_weight_min": -27.051546096801758,
      "activations/layer14_attention_weight_max": 36.017608642578125,
      "activations/layer14_attention_weight_min": -30.353511810302734,
      "activations/layer15_attention_weight_max": 33.581241607666016,
      "activations/layer15_attention_weight_min": -30.704002380371094,
      "activations/layer16_attention_weight_max": 33.69602966308594,
      "activations/layer16_attention_weight_min": -32.812564849853516,
      "activations/layer17_attention_weight_max": 51.624229431152344,
      "activations/layer17_attention_weight_min": -42.48856735229492,
      "activations/layer18_attention_weight_max": 45.44477844238281,
      "activations/layer18_attention_weight_min": -38.30289840698242,
      "activations/layer19_attention_weight_max": 22.901607513427734,
      "activations/layer19_attention_weight_min": -20.963109970092773,
      "activations/layer1_attention_weight_max": 16.070362091064453,
      "activations/layer1_attention_weight_min": -14.396595001220703,
      "activations/layer20_attention_weight_max": 25.657642364501953,
      "activations/layer20_attention_weight_min": -19.069957733154297,
      "activations/layer21_attention_weight_max": 34.67275619506836,
      "activations/layer21_attention_weight_min": -22.2016658782959,
      "activations/layer22_attention_weight_max": 28.906463623046875,
      "activations/layer22_attention_weight_min": -23.67466163635254,
      "activations/layer23_attention_weight_max": 40.55131530761719,
      "activations/layer23_attention_weight_min": -22.91684341430664,
      "activations/layer2_attention_weight_max": 32.53940963745117,
      "activations/layer2_attention_weight_min": -32.245826721191406,
      "activations/layer3_attention_weight_max": 90.99295806884766,
      "activations/layer3_attention_weight_min": -96.46317291259766,
      "activations/layer4_attention_weight_max": 87.02472686767578,
      "activations/layer4_attention_weight_min": -86.27521514892578,
      "activations/layer5_attention_weight_max": 65.77491760253906,
      "activations/layer5_attention_weight_min": -67.99530029296875,
      "activations/layer6_attention_weight_max": 50.70903778076172,
      "activations/layer6_attention_weight_min": -51.6814079284668,
      "activations/layer7_attention_weight_max": 68.56086730957031,
      "activations/layer7_attention_weight_min": -66.67509460449219,
      "activations/layer8_attention_weight_max": 47.48598098754883,
      "activations/layer8_attention_weight_min": -50.826805114746094,
      "activations/layer9_attention_weight_max": 45.69113540649414,
      "activations/layer9_attention_weight_min": -46.16596603393555,
      "epoch": 20.77,
      "learning_rate": 1.6225378787878787e-05,
      "loss": 2.708,
      "step": 357450
    },
    {
      "activations/layer0_attention_weight_max": 15.140481948852539,
      "activations/layer0_attention_weight_min": -13.731532096862793,
      "activations/layer10_attention_weight_max": 35.853851318359375,
      "activations/layer10_attention_weight_min": -36.01630401611328,
      "activations/layer11_attention_weight_max": 33.69776153564453,
      "activations/layer11_attention_weight_min": -35.788307189941406,
      "activations/layer12_attention_weight_max": 19.780900955200195,
      "activations/layer12_attention_weight_min": -26.274625778198242,
      "activations/layer13_attention_weight_max": 32.47409439086914,
      "activations/layer13_attention_weight_min": -28.313175201416016,
      "activations/layer14_attention_weight_max": 35.376853942871094,
      "activations/layer14_attention_weight_min": -29.475479125976562,
      "activations/layer15_attention_weight_max": 34.195091247558594,
      "activations/layer15_attention_weight_min": -30.624963760375977,
      "activations/layer16_attention_weight_max": 33.04963302612305,
      "activations/layer16_attention_weight_min": -31.12044334411621,
      "activations/layer17_attention_weight_max": 52.19874954223633,
      "activations/layer17_attention_weight_min": -42.65829849243164,
      "activations/layer18_attention_weight_max": 49.18802261352539,
      "activations/layer18_attention_weight_min": -38.73749923706055,
      "activations/layer19_attention_weight_max": 24.23810577392578,
      "activations/layer19_attention_weight_min": -22.8635311126709,
      "activations/layer1_attention_weight_max": 16.179346084594727,
      "activations/layer1_attention_weight_min": -14.759978294372559,
      "activations/layer20_attention_weight_max": 22.964412689208984,
      "activations/layer20_attention_weight_min": -21.96889305114746,
      "activations/layer21_attention_weight_max": 37.841434478759766,
      "activations/layer21_attention_weight_min": -24.398296356201172,
      "activations/layer22_attention_weight_max": 32.123172760009766,
      "activations/layer22_attention_weight_min": -26.027570724487305,
      "activations/layer23_attention_weight_max": 32.848228454589844,
      "activations/layer23_attention_weight_min": -24.651020050048828,
      "activations/layer2_attention_weight_max": 32.13648223876953,
      "activations/layer2_attention_weight_min": -30.645177841186523,
      "activations/layer3_attention_weight_max": 86.91688537597656,
      "activations/layer3_attention_weight_min": -86.65068054199219,
      "activations/layer4_attention_weight_max": 82.27745056152344,
      "activations/layer4_attention_weight_min": -84.57305908203125,
      "activations/layer5_attention_weight_max": 59.738075256347656,
      "activations/layer5_attention_weight_min": -68.30955505371094,
      "activations/layer6_attention_weight_max": 48.49192428588867,
      "activations/layer6_attention_weight_min": -52.86247634887695,
      "activations/layer7_attention_weight_max": 64.83637237548828,
      "activations/layer7_attention_weight_min": -62.3320426940918,
      "activations/layer8_attention_weight_max": 46.327152252197266,
      "activations/layer8_attention_weight_min": -51.692535400390625,
      "activations/layer9_attention_weight_max": 42.70625305175781,
      "activations/layer9_attention_weight_min": -48.15009307861328,
      "epoch": 20.77,
      "learning_rate": 1.6206439393939392e-05,
      "loss": 2.7326,
      "step": 357500
    },
    {
      "activations/layer0_attention_weight_max": 15.053743362426758,
      "activations/layer0_attention_weight_min": -13.824117660522461,
      "activations/layer10_attention_weight_max": 34.631065368652344,
      "activations/layer10_attention_weight_min": -33.924354553222656,
      "activations/layer11_attention_weight_max": 31.783628463745117,
      "activations/layer11_attention_weight_min": -33.43769073486328,
      "activations/layer12_attention_weight_max": 19.90789222717285,
      "activations/layer12_attention_weight_min": -26.58586311340332,
      "activations/layer13_attention_weight_max": 37.383506774902344,
      "activations/layer13_attention_weight_min": -26.729963302612305,
      "activations/layer14_attention_weight_max": 37.56174850463867,
      "activations/layer14_attention_weight_min": -31.040504455566406,
      "activations/layer15_attention_weight_max": 35.135215759277344,
      "activations/layer15_attention_weight_min": -31.732629776000977,
      "activations/layer16_attention_weight_max": 36.909698486328125,
      "activations/layer16_attention_weight_min": -33.635318756103516,
      "activations/layer17_attention_weight_max": 54.27421569824219,
      "activations/layer17_attention_weight_min": -44.8552360534668,
      "activations/layer18_attention_weight_max": 55.018798828125,
      "activations/layer18_attention_weight_min": -41.440643310546875,
      "activations/layer19_attention_weight_max": 24.299726486206055,
      "activations/layer19_attention_weight_min": -22.287012100219727,
      "activations/layer1_attention_weight_max": 15.50505542755127,
      "activations/layer1_attention_weight_min": -14.459308624267578,
      "activations/layer20_attention_weight_max": 24.572782516479492,
      "activations/layer20_attention_weight_min": -20.930086135864258,
      "activations/layer21_attention_weight_max": 36.3891487121582,
      "activations/layer21_attention_weight_min": -21.829147338867188,
      "activations/layer22_attention_weight_max": 30.682830810546875,
      "activations/layer22_attention_weight_min": -26.8057861328125,
      "activations/layer23_attention_weight_max": 40.003440856933594,
      "activations/layer23_attention_weight_min": -23.35504722595215,
      "activations/layer2_attention_weight_max": 31.12442398071289,
      "activations/layer2_attention_weight_min": -30.511022567749023,
      "activations/layer3_attention_weight_max": 85.85227966308594,
      "activations/layer3_attention_weight_min": -93.17521667480469,
      "activations/layer4_attention_weight_max": 84.48815155029297,
      "activations/layer4_attention_weight_min": -87.22618103027344,
      "activations/layer5_attention_weight_max": 61.30509948730469,
      "activations/layer5_attention_weight_min": -70.99317169189453,
      "activations/layer6_attention_weight_max": 48.97932434082031,
      "activations/layer6_attention_weight_min": -49.35026931762695,
      "activations/layer7_attention_weight_max": 67.00648498535156,
      "activations/layer7_attention_weight_min": -67.11290740966797,
      "activations/layer8_attention_weight_max": 45.11058807373047,
      "activations/layer8_attention_weight_min": -52.2481803894043,
      "activations/layer9_attention_weight_max": 44.15898513793945,
      "activations/layer9_attention_weight_min": -46.17920684814453,
      "epoch": 20.78,
      "learning_rate": 1.6187499999999997e-05,
      "loss": 2.7255,
      "step": 357550
    },
    {
      "activations/layer0_attention_weight_max": 15.54731273651123,
      "activations/layer0_attention_weight_min": -13.917435646057129,
      "activations/layer10_attention_weight_max": 36.418174743652344,
      "activations/layer10_attention_weight_min": -35.999755859375,
      "activations/layer11_attention_weight_max": 32.808956146240234,
      "activations/layer11_attention_weight_min": -34.735443115234375,
      "activations/layer12_attention_weight_max": 18.558700561523438,
      "activations/layer12_attention_weight_min": -28.4293270111084,
      "activations/layer13_attention_weight_max": 34.3994026184082,
      "activations/layer13_attention_weight_min": -31.135635375976562,
      "activations/layer14_attention_weight_max": 35.99993133544922,
      "activations/layer14_attention_weight_min": -32.88302230834961,
      "activations/layer15_attention_weight_max": 34.694847106933594,
      "activations/layer15_attention_weight_min": -29.8409366607666,
      "activations/layer16_attention_weight_max": 34.83625793457031,
      "activations/layer16_attention_weight_min": -33.717742919921875,
      "activations/layer17_attention_weight_max": 49.95101547241211,
      "activations/layer17_attention_weight_min": -43.83469772338867,
      "activations/layer18_attention_weight_max": 44.86258316040039,
      "activations/layer18_attention_weight_min": -40.36611557006836,
      "activations/layer19_attention_weight_max": 26.486509323120117,
      "activations/layer19_attention_weight_min": -23.096614837646484,
      "activations/layer1_attention_weight_max": 15.770535469055176,
      "activations/layer1_attention_weight_min": -15.23364543914795,
      "activations/layer20_attention_weight_max": 23.155620574951172,
      "activations/layer20_attention_weight_min": -22.451871871948242,
      "activations/layer21_attention_weight_max": 43.07551193237305,
      "activations/layer21_attention_weight_min": -24.446142196655273,
      "activations/layer22_attention_weight_max": 29.103837966918945,
      "activations/layer22_attention_weight_min": -27.514354705810547,
      "activations/layer23_attention_weight_max": 42.315330505371094,
      "activations/layer23_attention_weight_min": -24.21392822265625,
      "activations/layer2_attention_weight_max": 31.422338485717773,
      "activations/layer2_attention_weight_min": -30.95217514038086,
      "activations/layer3_attention_weight_max": 91.17269897460938,
      "activations/layer3_attention_weight_min": -91.68167877197266,
      "activations/layer4_attention_weight_max": 87.83895111083984,
      "activations/layer4_attention_weight_min": -84.34889221191406,
      "activations/layer5_attention_weight_max": 65.60307312011719,
      "activations/layer5_attention_weight_min": -73.17561340332031,
      "activations/layer6_attention_weight_max": 50.6104621887207,
      "activations/layer6_attention_weight_min": -51.122413635253906,
      "activations/layer7_attention_weight_max": 69.06470489501953,
      "activations/layer7_attention_weight_min": -65.32074737548828,
      "activations/layer8_attention_weight_max": 52.04762649536133,
      "activations/layer8_attention_weight_min": -50.107662200927734,
      "activations/layer9_attention_weight_max": 48.28547286987305,
      "activations/layer9_attention_weight_min": -46.82432174682617,
      "epoch": 20.78,
      "learning_rate": 1.6168560606060605e-05,
      "loss": 2.7118,
      "step": 357600
    },
    {
      "activations/layer0_attention_weight_max": 16.05519676208496,
      "activations/layer0_attention_weight_min": -13.972832679748535,
      "activations/layer10_attention_weight_max": 34.99623489379883,
      "activations/layer10_attention_weight_min": -37.403568267822266,
      "activations/layer11_attention_weight_max": 34.16389465332031,
      "activations/layer11_attention_weight_min": -39.06685256958008,
      "activations/layer12_attention_weight_max": 18.87660026550293,
      "activations/layer12_attention_weight_min": -29.909772872924805,
      "activations/layer13_attention_weight_max": 33.2271842956543,
      "activations/layer13_attention_weight_min": -26.65680503845215,
      "activations/layer14_attention_weight_max": 32.4841423034668,
      "activations/layer14_attention_weight_min": -29.259244918823242,
      "activations/layer15_attention_weight_max": 31.518394470214844,
      "activations/layer15_attention_weight_min": -30.00527572631836,
      "activations/layer16_attention_weight_max": 32.27884292602539,
      "activations/layer16_attention_weight_min": -30.468671798706055,
      "activations/layer17_attention_weight_max": 50.14509582519531,
      "activations/layer17_attention_weight_min": -43.4039192199707,
      "activations/layer18_attention_weight_max": 46.54098129272461,
      "activations/layer18_attention_weight_min": -37.870697021484375,
      "activations/layer19_attention_weight_max": 24.684589385986328,
      "activations/layer19_attention_weight_min": -22.711957931518555,
      "activations/layer1_attention_weight_max": 16.850217819213867,
      "activations/layer1_attention_weight_min": -14.486329078674316,
      "activations/layer20_attention_weight_max": 24.632118225097656,
      "activations/layer20_attention_weight_min": -22.26384735107422,
      "activations/layer21_attention_weight_max": 38.988346099853516,
      "activations/layer21_attention_weight_min": -25.492034912109375,
      "activations/layer22_attention_weight_max": 30.07907485961914,
      "activations/layer22_attention_weight_min": -24.92224884033203,
      "activations/layer23_attention_weight_max": 34.52867889404297,
      "activations/layer23_attention_weight_min": -23.18453598022461,
      "activations/layer2_attention_weight_max": 31.089523315429688,
      "activations/layer2_attention_weight_min": -32.977622985839844,
      "activations/layer3_attention_weight_max": 88.17708587646484,
      "activations/layer3_attention_weight_min": -87.1007308959961,
      "activations/layer4_attention_weight_max": 81.92179870605469,
      "activations/layer4_attention_weight_min": -82.55126190185547,
      "activations/layer5_attention_weight_max": 59.91878890991211,
      "activations/layer5_attention_weight_min": -68.09996032714844,
      "activations/layer6_attention_weight_max": 50.607948303222656,
      "activations/layer6_attention_weight_min": -51.005924224853516,
      "activations/layer7_attention_weight_max": 64.18377685546875,
      "activations/layer7_attention_weight_min": -68.11676025390625,
      "activations/layer8_attention_weight_max": 48.59483337402344,
      "activations/layer8_attention_weight_min": -49.17300796508789,
      "activations/layer9_attention_weight_max": 46.9190788269043,
      "activations/layer9_attention_weight_min": -46.4802131652832,
      "epoch": 20.78,
      "learning_rate": 1.614962121212121e-05,
      "loss": 2.7336,
      "step": 357650
    },
    {
      "activations/layer0_attention_weight_max": 14.965961456298828,
      "activations/layer0_attention_weight_min": -13.93747329711914,
      "activations/layer10_attention_weight_max": 35.86029052734375,
      "activations/layer10_attention_weight_min": -35.90662384033203,
      "activations/layer11_attention_weight_max": 35.13922119140625,
      "activations/layer11_attention_weight_min": -36.06101989746094,
      "activations/layer12_attention_weight_max": 19.254650115966797,
      "activations/layer12_attention_weight_min": -28.727018356323242,
      "activations/layer13_attention_weight_max": 36.65120315551758,
      "activations/layer13_attention_weight_min": -26.870512008666992,
      "activations/layer14_attention_weight_max": 37.063385009765625,
      "activations/layer14_attention_weight_min": -28.91439437866211,
      "activations/layer15_attention_weight_max": 35.9924201965332,
      "activations/layer15_attention_weight_min": -30.161897659301758,
      "activations/layer16_attention_weight_max": 36.78395462036133,
      "activations/layer16_attention_weight_min": -33.634490966796875,
      "activations/layer17_attention_weight_max": 54.11152267456055,
      "activations/layer17_attention_weight_min": -45.92196273803711,
      "activations/layer18_attention_weight_max": 53.704837799072266,
      "activations/layer18_attention_weight_min": -40.70856857299805,
      "activations/layer19_attention_weight_max": 26.086347579956055,
      "activations/layer19_attention_weight_min": -22.657264709472656,
      "activations/layer1_attention_weight_max": 15.506781578063965,
      "activations/layer1_attention_weight_min": -14.068198204040527,
      "activations/layer20_attention_weight_max": 24.92763900756836,
      "activations/layer20_attention_weight_min": -19.84103775024414,
      "activations/layer21_attention_weight_max": 38.50954055786133,
      "activations/layer21_attention_weight_min": -20.82721519470215,
      "activations/layer22_attention_weight_max": 31.725069046020508,
      "activations/layer22_attention_weight_min": -23.41776466369629,
      "activations/layer23_attention_weight_max": 35.84535217285156,
      "activations/layer23_attention_weight_min": -24.400983810424805,
      "activations/layer2_attention_weight_max": 32.5504264831543,
      "activations/layer2_attention_weight_min": -30.385976791381836,
      "activations/layer3_attention_weight_max": 92.85585021972656,
      "activations/layer3_attention_weight_min": -88.79780578613281,
      "activations/layer4_attention_weight_max": 91.32939910888672,
      "activations/layer4_attention_weight_min": -84.32085418701172,
      "activations/layer5_attention_weight_max": 66.20101928710938,
      "activations/layer5_attention_weight_min": -71.8704605102539,
      "activations/layer6_attention_weight_max": 52.10774612426758,
      "activations/layer6_attention_weight_min": -52.55302810668945,
      "activations/layer7_attention_weight_max": 67.60980987548828,
      "activations/layer7_attention_weight_min": -65.86756896972656,
      "activations/layer8_attention_weight_max": 46.96494674682617,
      "activations/layer8_attention_weight_min": -50.972877502441406,
      "activations/layer9_attention_weight_max": 44.70081329345703,
      "activations/layer9_attention_weight_min": -46.89827346801758,
      "epoch": 20.78,
      "learning_rate": 1.613068181818182e-05,
      "loss": 2.7208,
      "step": 357700
    },
    {
      "activations/layer0_attention_weight_max": 15.025873184204102,
      "activations/layer0_attention_weight_min": -14.319061279296875,
      "activations/layer10_attention_weight_max": 42.81151580810547,
      "activations/layer10_attention_weight_min": -41.14064407348633,
      "activations/layer11_attention_weight_max": 41.398746490478516,
      "activations/layer11_attention_weight_min": -40.55875015258789,
      "activations/layer12_attention_weight_max": 21.252059936523438,
      "activations/layer12_attention_weight_min": -27.918136596679688,
      "activations/layer13_attention_weight_max": 36.951988220214844,
      "activations/layer13_attention_weight_min": -30.811752319335938,
      "activations/layer14_attention_weight_max": 36.69566345214844,
      "activations/layer14_attention_weight_min": -32.340476989746094,
      "activations/layer15_attention_weight_max": 34.46632385253906,
      "activations/layer15_attention_weight_min": -31.61264991760254,
      "activations/layer16_attention_weight_max": 35.880859375,
      "activations/layer16_attention_weight_min": -33.411231994628906,
      "activations/layer17_attention_weight_max": 55.83187484741211,
      "activations/layer17_attention_weight_min": -46.97589874267578,
      "activations/layer18_attention_weight_max": 49.57157897949219,
      "activations/layer18_attention_weight_min": -44.13218688964844,
      "activations/layer19_attention_weight_max": 26.47015380859375,
      "activations/layer19_attention_weight_min": -24.76234245300293,
      "activations/layer1_attention_weight_max": 16.397485733032227,
      "activations/layer1_attention_weight_min": -14.15700626373291,
      "activations/layer20_attention_weight_max": 23.391077041625977,
      "activations/layer20_attention_weight_min": -22.814939498901367,
      "activations/layer21_attention_weight_max": 37.51683044433594,
      "activations/layer21_attention_weight_min": -22.683420181274414,
      "activations/layer22_attention_weight_max": 30.1167049407959,
      "activations/layer22_attention_weight_min": -26.881759643554688,
      "activations/layer23_attention_weight_max": 36.86699676513672,
      "activations/layer23_attention_weight_min": -25.930938720703125,
      "activations/layer2_attention_weight_max": 32.87749481201172,
      "activations/layer2_attention_weight_min": -31.839048385620117,
      "activations/layer3_attention_weight_max": 98.14783477783203,
      "activations/layer3_attention_weight_min": -93.4954833984375,
      "activations/layer4_attention_weight_max": 92.53536224365234,
      "activations/layer4_attention_weight_min": -91.71864318847656,
      "activations/layer5_attention_weight_max": 65.03446960449219,
      "activations/layer5_attention_weight_min": -70.04217529296875,
      "activations/layer6_attention_weight_max": 55.33488845825195,
      "activations/layer6_attention_weight_min": -56.75324630737305,
      "activations/layer7_attention_weight_max": 73.06828308105469,
      "activations/layer7_attention_weight_min": -74.28131866455078,
      "activations/layer8_attention_weight_max": 56.082054138183594,
      "activations/layer8_attention_weight_min": -58.11442184448242,
      "activations/layer9_attention_weight_max": 51.03937530517578,
      "activations/layer9_attention_weight_min": -50.449520111083984,
      "epoch": 20.79,
      "learning_rate": 1.6111742424242423e-05,
      "loss": 2.7153,
      "step": 357750
    },
    {
      "activations/layer0_attention_weight_max": 15.164084434509277,
      "activations/layer0_attention_weight_min": -13.86496639251709,
      "activations/layer10_attention_weight_max": 35.4901008605957,
      "activations/layer10_attention_weight_min": -35.29058837890625,
      "activations/layer11_attention_weight_max": 31.91944122314453,
      "activations/layer11_attention_weight_min": -34.430362701416016,
      "activations/layer12_attention_weight_max": 19.113704681396484,
      "activations/layer12_attention_weight_min": -24.554336547851562,
      "activations/layer13_attention_weight_max": 38.57978439331055,
      "activations/layer13_attention_weight_min": -30.2083683013916,
      "activations/layer14_attention_weight_max": 41.17435073852539,
      "activations/layer14_attention_weight_min": -31.191070556640625,
      "activations/layer15_attention_weight_max": 37.88018035888672,
      "activations/layer15_attention_weight_min": -31.89031410217285,
      "activations/layer16_attention_weight_max": 37.13275909423828,
      "activations/layer16_attention_weight_min": -35.89839172363281,
      "activations/layer17_attention_weight_max": 54.158992767333984,
      "activations/layer17_attention_weight_min": -46.3598518371582,
      "activations/layer18_attention_weight_max": 54.021820068359375,
      "activations/layer18_attention_weight_min": -42.03035354614258,
      "activations/layer19_attention_weight_max": 29.107046127319336,
      "activations/layer19_attention_weight_min": -24.84621238708496,
      "activations/layer1_attention_weight_max": 15.342453956604004,
      "activations/layer1_attention_weight_min": -15.299271583557129,
      "activations/layer20_attention_weight_max": 31.02415657043457,
      "activations/layer20_attention_weight_min": -24.848093032836914,
      "activations/layer21_attention_weight_max": 44.045894622802734,
      "activations/layer21_attention_weight_min": -23.861968994140625,
      "activations/layer22_attention_weight_max": 31.40744972229004,
      "activations/layer22_attention_weight_min": -26.771284103393555,
      "activations/layer23_attention_weight_max": 42.70390319824219,
      "activations/layer23_attention_weight_min": -25.483280181884766,
      "activations/layer2_attention_weight_max": 31.83100700378418,
      "activations/layer2_attention_weight_min": -30.5964412689209,
      "activations/layer3_attention_weight_max": 87.11605834960938,
      "activations/layer3_attention_weight_min": -88.54277038574219,
      "activations/layer4_attention_weight_max": 83.44377899169922,
      "activations/layer4_attention_weight_min": -83.46939086914062,
      "activations/layer5_attention_weight_max": 61.229461669921875,
      "activations/layer5_attention_weight_min": -71.71288299560547,
      "activations/layer6_attention_weight_max": 50.909122467041016,
      "activations/layer6_attention_weight_min": -49.85274887084961,
      "activations/layer7_attention_weight_max": 66.83089447021484,
      "activations/layer7_attention_weight_min": -63.63657760620117,
      "activations/layer8_attention_weight_max": 49.58237838745117,
      "activations/layer8_attention_weight_min": -50.04080581665039,
      "activations/layer9_attention_weight_max": 44.33885192871094,
      "activations/layer9_attention_weight_min": -46.05402755737305,
      "epoch": 20.79,
      "learning_rate": 1.6092803030303028e-05,
      "loss": 2.7027,
      "step": 357800
    },
    {
      "activations/layer0_attention_weight_max": 15.519986152648926,
      "activations/layer0_attention_weight_min": -14.12858772277832,
      "activations/layer10_attention_weight_max": 35.12822341918945,
      "activations/layer10_attention_weight_min": -35.23810577392578,
      "activations/layer11_attention_weight_max": 36.02511215209961,
      "activations/layer11_attention_weight_min": -33.909873962402344,
      "activations/layer12_attention_weight_max": 19.454988479614258,
      "activations/layer12_attention_weight_min": -26.84349822998047,
      "activations/layer13_attention_weight_max": 33.28044128417969,
      "activations/layer13_attention_weight_min": -26.893468856811523,
      "activations/layer14_attention_weight_max": 35.91738510131836,
      "activations/layer14_attention_weight_min": -30.511674880981445,
      "activations/layer15_attention_weight_max": 33.74089431762695,
      "activations/layer15_attention_weight_min": -31.228235244750977,
      "activations/layer16_attention_weight_max": 36.31825256347656,
      "activations/layer16_attention_weight_min": -34.80669403076172,
      "activations/layer17_attention_weight_max": 52.879920959472656,
      "activations/layer17_attention_weight_min": -48.21505355834961,
      "activations/layer18_attention_weight_max": 47.916954040527344,
      "activations/layer18_attention_weight_min": -43.209228515625,
      "activations/layer19_attention_weight_max": 26.017669677734375,
      "activations/layer19_attention_weight_min": -21.205190658569336,
      "activations/layer1_attention_weight_max": 15.566686630249023,
      "activations/layer1_attention_weight_min": -14.687422752380371,
      "activations/layer20_attention_weight_max": 23.517776489257812,
      "activations/layer20_attention_weight_min": -20.990720748901367,
      "activations/layer21_attention_weight_max": 33.619022369384766,
      "activations/layer21_attention_weight_min": -21.941625595092773,
      "activations/layer22_attention_weight_max": 28.576711654663086,
      "activations/layer22_attention_weight_min": -25.37311363220215,
      "activations/layer23_attention_weight_max": 35.29458999633789,
      "activations/layer23_attention_weight_min": -26.09889793395996,
      "activations/layer2_attention_weight_max": 32.64707565307617,
      "activations/layer2_attention_weight_min": -30.595060348510742,
      "activations/layer3_attention_weight_max": 87.72876739501953,
      "activations/layer3_attention_weight_min": -87.2579574584961,
      "activations/layer4_attention_weight_max": 81.7393569946289,
      "activations/layer4_attention_weight_min": -81.23358154296875,
      "activations/layer5_attention_weight_max": 60.140113830566406,
      "activations/layer5_attention_weight_min": -70.00917053222656,
      "activations/layer6_attention_weight_max": 50.739112854003906,
      "activations/layer6_attention_weight_min": -50.099430084228516,
      "activations/layer7_attention_weight_max": 68.1561279296875,
      "activations/layer7_attention_weight_min": -63.07500076293945,
      "activations/layer8_attention_weight_max": 47.2015495300293,
      "activations/layer8_attention_weight_min": -49.39885330200195,
      "activations/layer9_attention_weight_max": 46.505958557128906,
      "activations/layer9_attention_weight_min": -44.34912109375,
      "epoch": 20.79,
      "learning_rate": 1.6073863636363637e-05,
      "loss": 2.7185,
      "step": 357850
    },
    {
      "activations/layer0_attention_weight_max": 15.309552192687988,
      "activations/layer0_attention_weight_min": -13.83660888671875,
      "activations/layer10_attention_weight_max": 36.15757369995117,
      "activations/layer10_attention_weight_min": -36.51809310913086,
      "activations/layer11_attention_weight_max": 35.173004150390625,
      "activations/layer11_attention_weight_min": -36.645545959472656,
      "activations/layer12_attention_weight_max": 19.228694915771484,
      "activations/layer12_attention_weight_min": -26.678646087646484,
      "activations/layer13_attention_weight_max": 35.558807373046875,
      "activations/layer13_attention_weight_min": -28.295602798461914,
      "activations/layer14_attention_weight_max": 36.84867858886719,
      "activations/layer14_attention_weight_min": -32.38835906982422,
      "activations/layer15_attention_weight_max": 36.39982223510742,
      "activations/layer15_attention_weight_min": -31.456775665283203,
      "activations/layer16_attention_weight_max": 38.95452117919922,
      "activations/layer16_attention_weight_min": -37.97089767456055,
      "activations/layer17_attention_weight_max": 56.01811599731445,
      "activations/layer17_attention_weight_min": -48.621402740478516,
      "activations/layer18_attention_weight_max": 53.41477966308594,
      "activations/layer18_attention_weight_min": -44.03380584716797,
      "activations/layer19_attention_weight_max": 27.872533798217773,
      "activations/layer19_attention_weight_min": -25.820159912109375,
      "activations/layer1_attention_weight_max": 15.6544828414917,
      "activations/layer1_attention_weight_min": -14.433065414428711,
      "activations/layer20_attention_weight_max": 30.804527282714844,
      "activations/layer20_attention_weight_min": -21.867921829223633,
      "activations/layer21_attention_weight_max": 41.49411392211914,
      "activations/layer21_attention_weight_min": -23.146770477294922,
      "activations/layer22_attention_weight_max": 37.1029167175293,
      "activations/layer22_attention_weight_min": -24.68255043029785,
      "activations/layer23_attention_weight_max": 43.253692626953125,
      "activations/layer23_attention_weight_min": -23.91737937927246,
      "activations/layer2_attention_weight_max": 31.809494018554688,
      "activations/layer2_attention_weight_min": -32.04253005981445,
      "activations/layer3_attention_weight_max": 88.81277465820312,
      "activations/layer3_attention_weight_min": -88.74768829345703,
      "activations/layer4_attention_weight_max": 82.90583801269531,
      "activations/layer4_attention_weight_min": -85.37879180908203,
      "activations/layer5_attention_weight_max": 61.61414337158203,
      "activations/layer5_attention_weight_min": -72.93544006347656,
      "activations/layer6_attention_weight_max": 48.30586624145508,
      "activations/layer6_attention_weight_min": -51.07188415527344,
      "activations/layer7_attention_weight_max": 65.53050994873047,
      "activations/layer7_attention_weight_min": -63.26702117919922,
      "activations/layer8_attention_weight_max": 47.069358825683594,
      "activations/layer8_attention_weight_min": -50.856170654296875,
      "activations/layer9_attention_weight_max": 44.90147018432617,
      "activations/layer9_attention_weight_min": -47.23725128173828,
      "epoch": 20.8,
      "learning_rate": 1.605492424242424e-05,
      "loss": 2.7173,
      "step": 357900
    },
    {
      "activations/layer0_attention_weight_max": 14.977493286132812,
      "activations/layer0_attention_weight_min": -13.947726249694824,
      "activations/layer10_attention_weight_max": 36.347442626953125,
      "activations/layer10_attention_weight_min": -33.66497802734375,
      "activations/layer11_attention_weight_max": 33.59124755859375,
      "activations/layer11_attention_weight_min": -33.05542755126953,
      "activations/layer12_attention_weight_max": 18.17031478881836,
      "activations/layer12_attention_weight_min": -25.329477310180664,
      "activations/layer13_attention_weight_max": 35.348628997802734,
      "activations/layer13_attention_weight_min": -25.995792388916016,
      "activations/layer14_attention_weight_max": 35.4344596862793,
      "activations/layer14_attention_weight_min": -32.062992095947266,
      "activations/layer15_attention_weight_max": 33.74051284790039,
      "activations/layer15_attention_weight_min": -31.881240844726562,
      "activations/layer16_attention_weight_max": 33.53594970703125,
      "activations/layer16_attention_weight_min": -32.82525634765625,
      "activations/layer17_attention_weight_max": 50.13551330566406,
      "activations/layer17_attention_weight_min": -45.52676773071289,
      "activations/layer18_attention_weight_max": 46.72105407714844,
      "activations/layer18_attention_weight_min": -40.16753387451172,
      "activations/layer19_attention_weight_max": 25.85223388671875,
      "activations/layer19_attention_weight_min": -22.565481185913086,
      "activations/layer1_attention_weight_max": 15.00877571105957,
      "activations/layer1_attention_weight_min": -15.037129402160645,
      "activations/layer20_attention_weight_max": 23.164657592773438,
      "activations/layer20_attention_weight_min": -21.410888671875,
      "activations/layer21_attention_weight_max": 40.024147033691406,
      "activations/layer21_attention_weight_min": -21.158950805664062,
      "activations/layer22_attention_weight_max": 29.499250411987305,
      "activations/layer22_attention_weight_min": -24.465627670288086,
      "activations/layer23_attention_weight_max": 37.71058654785156,
      "activations/layer23_attention_weight_min": -24.357221603393555,
      "activations/layer2_attention_weight_max": 32.52914810180664,
      "activations/layer2_attention_weight_min": -31.57778549194336,
      "activations/layer3_attention_weight_max": 89.44093322753906,
      "activations/layer3_attention_weight_min": -88.6114273071289,
      "activations/layer4_attention_weight_max": 85.04143524169922,
      "activations/layer4_attention_weight_min": -82.01851654052734,
      "activations/layer5_attention_weight_max": 61.9537467956543,
      "activations/layer5_attention_weight_min": -70.56304931640625,
      "activations/layer6_attention_weight_max": 48.51913070678711,
      "activations/layer6_attention_weight_min": -50.181636810302734,
      "activations/layer7_attention_weight_max": 64.45156860351562,
      "activations/layer7_attention_weight_min": -66.69886779785156,
      "activations/layer8_attention_weight_max": 50.33922576904297,
      "activations/layer8_attention_weight_min": -49.91196823120117,
      "activations/layer9_attention_weight_max": 42.69916915893555,
      "activations/layer9_attention_weight_min": -45.17850875854492,
      "epoch": 20.8,
      "learning_rate": 1.6035984848484846e-05,
      "loss": 2.7191,
      "step": 357950
    },
    {
      "activations/layer0_attention_weight_max": 14.98027515411377,
      "activations/layer0_attention_weight_min": -13.808958053588867,
      "activations/layer10_attention_weight_max": 36.50413513183594,
      "activations/layer10_attention_weight_min": -38.13117599487305,
      "activations/layer11_attention_weight_max": 34.349090576171875,
      "activations/layer11_attention_weight_min": -36.317874908447266,
      "activations/layer12_attention_weight_max": 20.181400299072266,
      "activations/layer12_attention_weight_min": -29.853153228759766,
      "activations/layer13_attention_weight_max": 34.541114807128906,
      "activations/layer13_attention_weight_min": -27.695106506347656,
      "activations/layer14_attention_weight_max": 36.93659973144531,
      "activations/layer14_attention_weight_min": -30.77983856201172,
      "activations/layer15_attention_weight_max": 35.91859436035156,
      "activations/layer15_attention_weight_min": -30.601707458496094,
      "activations/layer16_attention_weight_max": 37.19065856933594,
      "activations/layer16_attention_weight_min": -33.48615646362305,
      "activations/layer17_attention_weight_max": 54.483856201171875,
      "activations/layer17_attention_weight_min": -45.79384231567383,
      "activations/layer18_attention_weight_max": 50.3429069519043,
      "activations/layer18_attention_weight_min": -40.54894256591797,
      "activations/layer19_attention_weight_max": 26.078115463256836,
      "activations/layer19_attention_weight_min": -23.105941772460938,
      "activations/layer1_attention_weight_max": 14.926756858825684,
      "activations/layer1_attention_weight_min": -13.005973815917969,
      "activations/layer20_attention_weight_max": 26.15057373046875,
      "activations/layer20_attention_weight_min": -20.848400115966797,
      "activations/layer21_attention_weight_max": 37.61366653442383,
      "activations/layer21_attention_weight_min": -22.42916488647461,
      "activations/layer22_attention_weight_max": 31.538991928100586,
      "activations/layer22_attention_weight_min": -25.11314582824707,
      "activations/layer23_attention_weight_max": 40.599342346191406,
      "activations/layer23_attention_weight_min": -23.660972595214844,
      "activations/layer2_attention_weight_max": 31.827911376953125,
      "activations/layer2_attention_weight_min": -30.026683807373047,
      "activations/layer3_attention_weight_max": 90.88671875,
      "activations/layer3_attention_weight_min": -87.4641342163086,
      "activations/layer4_attention_weight_max": 81.23351287841797,
      "activations/layer4_attention_weight_min": -82.568603515625,
      "activations/layer5_attention_weight_max": 58.676666259765625,
      "activations/layer5_attention_weight_min": -70.11024475097656,
      "activations/layer6_attention_weight_max": 47.071022033691406,
      "activations/layer6_attention_weight_min": -53.261680603027344,
      "activations/layer7_attention_weight_max": 62.41612243652344,
      "activations/layer7_attention_weight_min": -64.02108764648438,
      "activations/layer8_attention_weight_max": 46.65121078491211,
      "activations/layer8_attention_weight_min": -52.016117095947266,
      "activations/layer9_attention_weight_max": 49.821746826171875,
      "activations/layer9_attention_weight_min": -46.9621696472168,
      "epoch": 20.8,
      "learning_rate": 1.6017045454545455e-05,
      "loss": 2.7247,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_loss": 2.67578125,
      "eval_runtime": 8.5495,
      "eval_samples_per_second": 502.249,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_openwebtext_loss": 2.67578125,
      "eval_openwebtext_ppl": 14.523692042874359,
      "eval_openwebtext_runtime": 8.5495,
      "eval_openwebtext_samples_per_second": 502.249,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_wikitext_loss": 2.912109375,
      "eval_wikitext_ppl": 18.39556080667227,
      "eval_wikitext_runtime": 2.0308,
      "eval_wikitext_samples_per_second": 224.54,
      "step": 358000
    },
    {
      "epoch": 20.8,
      "eval_lambada_loss": 2.39453125,
      "eval_lambada_ppl": 10.963057921054194,
      "eval_lambada_runtime": 9.6466,
      "eval_lambada_samples_per_second": 504.736,
      "step": 358000
    },
    {
      "activations/layer0_attention_weight_max": 16.56207847595215,
      "activations/layer0_attention_weight_min": -14.179027557373047,
      "activations/layer10_attention_weight_max": 34.7099609375,
      "activations/layer10_attention_weight_min": -34.696510314941406,
      "activations/layer11_attention_weight_max": 33.30430603027344,
      "activations/layer11_attention_weight_min": -33.48491287231445,
      "activations/layer12_attention_weight_max": 16.648754119873047,
      "activations/layer12_attention_weight_min": -29.51676368713379,
      "activations/layer13_attention_weight_max": 33.304351806640625,
      "activations/layer13_attention_weight_min": -26.351703643798828,
      "activations/layer14_attention_weight_max": 35.6009635925293,
      "activations/layer14_attention_weight_min": -28.041837692260742,
      "activations/layer15_attention_weight_max": 32.99460983276367,
      "activations/layer15_attention_weight_min": -28.51999282836914,
      "activations/layer16_attention_weight_max": 34.616554260253906,
      "activations/layer16_attention_weight_min": -31.577632904052734,
      "activations/layer17_attention_weight_max": 55.11137771606445,
      "activations/layer17_attention_weight_min": -46.588809967041016,
      "activations/layer18_attention_weight_max": 47.54209518432617,
      "activations/layer18_attention_weight_min": -38.662200927734375,
      "activations/layer19_attention_weight_max": 23.741052627563477,
      "activations/layer19_attention_weight_min": -19.949283599853516,
      "activations/layer1_attention_weight_max": 16.358152389526367,
      "activations/layer1_attention_weight_min": -13.256902694702148,
      "activations/layer20_attention_weight_max": 21.97444725036621,
      "activations/layer20_attention_weight_min": -19.784650802612305,
      "activations/layer21_attention_weight_max": 40.76580047607422,
      "activations/layer21_attention_weight_min": -21.890060424804688,
      "activations/layer22_attention_weight_max": 27.491363525390625,
      "activations/layer22_attention_weight_min": -22.738393783569336,
      "activations/layer23_attention_weight_max": 34.30706024169922,
      "activations/layer23_attention_weight_min": -22.50984764099121,
      "activations/layer2_attention_weight_max": 32.625335693359375,
      "activations/layer2_attention_weight_min": -32.384368896484375,
      "activations/layer3_attention_weight_max": 87.50841522216797,
      "activations/layer3_attention_weight_min": -88.50080108642578,
      "activations/layer4_attention_weight_max": 82.55193328857422,
      "activations/layer4_attention_weight_min": -85.52303314208984,
      "activations/layer5_attention_weight_max": 58.57933807373047,
      "activations/layer5_attention_weight_min": -69.78988647460938,
      "activations/layer6_attention_weight_max": 50.82139587402344,
      "activations/layer6_attention_weight_min": -52.32174301147461,
      "activations/layer7_attention_weight_max": 62.9266357421875,
      "activations/layer7_attention_weight_min": -64.54009246826172,
      "activations/layer8_attention_weight_max": 46.23442459106445,
      "activations/layer8_attention_weight_min": -48.09320831298828,
      "activations/layer9_attention_weight_max": 45.70795440673828,
      "activations/layer9_attention_weight_min": -44.19987106323242,
      "epoch": 20.8,
      "learning_rate": 1.599810606060606e-05,
      "loss": 2.7008,
      "step": 358050
    },
    {
      "activations/layer0_attention_weight_max": 15.290792465209961,
      "activations/layer0_attention_weight_min": -14.13154125213623,
      "activations/layer10_attention_weight_max": 36.47869110107422,
      "activations/layer10_attention_weight_min": -36.895904541015625,
      "activations/layer11_attention_weight_max": 32.796085357666016,
      "activations/layer11_attention_weight_min": -35.216148376464844,
      "activations/layer12_attention_weight_max": 18.306318283081055,
      "activations/layer12_attention_weight_min": -24.533714294433594,
      "activations/layer13_attention_weight_max": 33.83486557006836,
      "activations/layer13_attention_weight_min": -29.880653381347656,
      "activations/layer14_attention_weight_max": 33.988128662109375,
      "activations/layer14_attention_weight_min": -32.003265380859375,
      "activations/layer15_attention_weight_max": 32.23379135131836,
      "activations/layer15_attention_weight_min": -33.62897872924805,
      "activations/layer16_attention_weight_max": 33.1749382019043,
      "activations/layer16_attention_weight_min": -32.63826370239258,
      "activations/layer17_attention_weight_max": 47.657470703125,
      "activations/layer17_attention_weight_min": -46.130191802978516,
      "activations/layer18_attention_weight_max": 46.31476593017578,
      "activations/layer18_attention_weight_min": -40.98823165893555,
      "activations/layer19_attention_weight_max": 26.420644760131836,
      "activations/layer19_attention_weight_min": -22.535886764526367,
      "activations/layer1_attention_weight_max": 15.594870567321777,
      "activations/layer1_attention_weight_min": -14.30944538116455,
      "activations/layer20_attention_weight_max": 22.599031448364258,
      "activations/layer20_attention_weight_min": -21.317888259887695,
      "activations/layer21_attention_weight_max": 38.54161834716797,
      "activations/layer21_attention_weight_min": -23.099285125732422,
      "activations/layer22_attention_weight_max": 29.212169647216797,
      "activations/layer22_attention_weight_min": -25.92353057861328,
      "activations/layer23_attention_weight_max": 35.052330017089844,
      "activations/layer23_attention_weight_min": -24.22388458251953,
      "activations/layer2_attention_weight_max": 31.833303451538086,
      "activations/layer2_attention_weight_min": -31.623130798339844,
      "activations/layer3_attention_weight_max": 87.18561553955078,
      "activations/layer3_attention_weight_min": -90.39360046386719,
      "activations/layer4_attention_weight_max": 83.3467788696289,
      "activations/layer4_attention_weight_min": -85.80988311767578,
      "activations/layer5_attention_weight_max": 64.49827575683594,
      "activations/layer5_attention_weight_min": -74.00128936767578,
      "activations/layer6_attention_weight_max": 48.61250686645508,
      "activations/layer6_attention_weight_min": -51.6135139465332,
      "activations/layer7_attention_weight_max": 67.92864227294922,
      "activations/layer7_attention_weight_min": -65.12051391601562,
      "activations/layer8_attention_weight_max": 47.693359375,
      "activations/layer8_attention_weight_min": -52.746280670166016,
      "activations/layer9_attention_weight_max": 45.40582275390625,
      "activations/layer9_attention_weight_min": -47.9668083190918,
      "epoch": 20.81,
      "learning_rate": 1.5979166666666664e-05,
      "loss": 2.7316,
      "step": 358100
    },
    {
      "activations/layer0_attention_weight_max": 15.284924507141113,
      "activations/layer0_attention_weight_min": -14.176973342895508,
      "activations/layer10_attention_weight_max": 38.46298599243164,
      "activations/layer10_attention_weight_min": -36.84880828857422,
      "activations/layer11_attention_weight_max": 35.38190841674805,
      "activations/layer11_attention_weight_min": -35.91581726074219,
      "activations/layer12_attention_weight_max": 19.66690444946289,
      "activations/layer12_attention_weight_min": -22.638628005981445,
      "activations/layer13_attention_weight_max": 37.20886993408203,
      "activations/layer13_attention_weight_min": -30.650558471679688,
      "activations/layer14_attention_weight_max": 39.164283752441406,
      "activations/layer14_attention_weight_min": -32.368465423583984,
      "activations/layer15_attention_weight_max": 37.113651275634766,
      "activations/layer15_attention_weight_min": -31.32056427001953,
      "activations/layer16_attention_weight_max": 40.72845458984375,
      "activations/layer16_attention_weight_min": -37.14219665527344,
      "activations/layer17_attention_weight_max": 55.64838790893555,
      "activations/layer17_attention_weight_min": -46.459007263183594,
      "activations/layer18_attention_weight_max": 50.823020935058594,
      "activations/layer18_attention_weight_min": -42.53984451293945,
      "activations/layer19_attention_weight_max": 28.70368766784668,
      "activations/layer19_attention_weight_min": -23.248228073120117,
      "activations/layer1_attention_weight_max": 15.117480278015137,
      "activations/layer1_attention_weight_min": -13.852309226989746,
      "activations/layer20_attention_weight_max": 26.48785972595215,
      "activations/layer20_attention_weight_min": -21.282276153564453,
      "activations/layer21_attention_weight_max": 36.24256134033203,
      "activations/layer21_attention_weight_min": -22.171815872192383,
      "activations/layer22_attention_weight_max": 31.292722702026367,
      "activations/layer22_attention_weight_min": -25.72339630126953,
      "activations/layer23_attention_weight_max": 39.40129852294922,
      "activations/layer23_attention_weight_min": -23.173912048339844,
      "activations/layer2_attention_weight_max": 32.848541259765625,
      "activations/layer2_attention_weight_min": -32.872318267822266,
      "activations/layer3_attention_weight_max": 89.07151794433594,
      "activations/layer3_attention_weight_min": -90.40509796142578,
      "activations/layer4_attention_weight_max": 85.06230926513672,
      "activations/layer4_attention_weight_min": -85.85713958740234,
      "activations/layer5_attention_weight_max": 63.390663146972656,
      "activations/layer5_attention_weight_min": -75.36541748046875,
      "activations/layer6_attention_weight_max": 50.85597610473633,
      "activations/layer6_attention_weight_min": -51.09052276611328,
      "activations/layer7_attention_weight_max": 66.81563568115234,
      "activations/layer7_attention_weight_min": -70.08584594726562,
      "activations/layer8_attention_weight_max": 47.89200210571289,
      "activations/layer8_attention_weight_min": -49.77080154418945,
      "activations/layer9_attention_weight_max": 46.7965202331543,
      "activations/layer9_attention_weight_min": -47.92655944824219,
      "epoch": 20.81,
      "learning_rate": 1.5960227272727273e-05,
      "loss": 2.7119,
      "step": 358150
    },
    {
      "activations/layer0_attention_weight_max": 15.625509262084961,
      "activations/layer0_attention_weight_min": -13.969059944152832,
      "activations/layer10_attention_weight_max": 34.565467834472656,
      "activations/layer10_attention_weight_min": -36.073150634765625,
      "activations/layer11_attention_weight_max": 33.24641418457031,
      "activations/layer11_attention_weight_min": -36.15191650390625,
      "activations/layer12_attention_weight_max": 19.641925811767578,
      "activations/layer12_attention_weight_min": -23.261777877807617,
      "activations/layer13_attention_weight_max": 38.732139587402344,
      "activations/layer13_attention_weight_min": -29.28350257873535,
      "activations/layer14_attention_weight_max": 39.18428039550781,
      "activations/layer14_attention_weight_min": -31.735576629638672,
      "activations/layer15_attention_weight_max": 37.41982650756836,
      "activations/layer15_attention_weight_min": -30.519983291625977,
      "activations/layer16_attention_weight_max": 41.473846435546875,
      "activations/layer16_attention_weight_min": -36.856719970703125,
      "activations/layer17_attention_weight_max": 59.10934066772461,
      "activations/layer17_attention_weight_min": -48.34690856933594,
      "activations/layer18_attention_weight_max": 53.40806198120117,
      "activations/layer18_attention_weight_min": -41.96977996826172,
      "activations/layer19_attention_weight_max": 24.99677085876465,
      "activations/layer19_attention_weight_min": -23.9039249420166,
      "activations/layer1_attention_weight_max": 16.245826721191406,
      "activations/layer1_attention_weight_min": -13.594866752624512,
      "activations/layer20_attention_weight_max": 24.198545455932617,
      "activations/layer20_attention_weight_min": -19.800033569335938,
      "activations/layer21_attention_weight_max": 38.77027130126953,
      "activations/layer21_attention_weight_min": -22.25879669189453,
      "activations/layer22_attention_weight_max": 30.30145263671875,
      "activations/layer22_attention_weight_min": -25.70819854736328,
      "activations/layer23_attention_weight_max": 35.92601013183594,
      "activations/layer23_attention_weight_min": -22.83741569519043,
      "activations/layer2_attention_weight_max": 31.55870819091797,
      "activations/layer2_attention_weight_min": -31.52639389038086,
      "activations/layer3_attention_weight_max": 88.46183776855469,
      "activations/layer3_attention_weight_min": -90.471923828125,
      "activations/layer4_attention_weight_max": 83.09024810791016,
      "activations/layer4_attention_weight_min": -83.26573944091797,
      "activations/layer5_attention_weight_max": 62.63945770263672,
      "activations/layer5_attention_weight_min": -69.84122467041016,
      "activations/layer6_attention_weight_max": 49.303646087646484,
      "activations/layer6_attention_weight_min": -51.21052551269531,
      "activations/layer7_attention_weight_max": 66.05909729003906,
      "activations/layer7_attention_weight_min": -66.65142822265625,
      "activations/layer8_attention_weight_max": 46.831329345703125,
      "activations/layer8_attention_weight_min": -50.68169021606445,
      "activations/layer9_attention_weight_max": 48.07149887084961,
      "activations/layer9_attention_weight_min": -47.24482727050781,
      "epoch": 20.81,
      "learning_rate": 1.5941287878787877e-05,
      "loss": 2.7173,
      "step": 358200
    },
    {
      "activations/layer0_attention_weight_max": 15.966227531433105,
      "activations/layer0_attention_weight_min": -13.970316886901855,
      "activations/layer10_attention_weight_max": 37.351051330566406,
      "activations/layer10_attention_weight_min": -37.690948486328125,
      "activations/layer11_attention_weight_max": 35.361942291259766,
      "activations/layer11_attention_weight_min": -35.69053649902344,
      "activations/layer12_attention_weight_max": 19.87584114074707,
      "activations/layer12_attention_weight_min": -22.308927536010742,
      "activations/layer13_attention_weight_max": 36.52766799926758,
      "activations/layer13_attention_weight_min": -28.07617950439453,
      "activations/layer14_attention_weight_max": 37.693580627441406,
      "activations/layer14_attention_weight_min": -31.878210067749023,
      "activations/layer15_attention_weight_max": 37.46498107910156,
      "activations/layer15_attention_weight_min": -32.140769958496094,
      "activations/layer16_attention_weight_max": 38.00287628173828,
      "activations/layer16_attention_weight_min": -34.10983657836914,
      "activations/layer17_attention_weight_max": 58.63286590576172,
      "activations/layer17_attention_weight_min": -47.19984436035156,
      "activations/layer18_attention_weight_max": 51.083377838134766,
      "activations/layer18_attention_weight_min": -40.03813552856445,
      "activations/layer19_attention_weight_max": 24.796119689941406,
      "activations/layer19_attention_weight_min": -23.193113327026367,
      "activations/layer1_attention_weight_max": 15.167203903198242,
      "activations/layer1_attention_weight_min": -13.7217435836792,
      "activations/layer20_attention_weight_max": 23.280717849731445,
      "activations/layer20_attention_weight_min": -21.155174255371094,
      "activations/layer21_attention_weight_max": 36.9384880065918,
      "activations/layer21_attention_weight_min": -22.2131404876709,
      "activations/layer22_attention_weight_max": 29.538482666015625,
      "activations/layer22_attention_weight_min": -25.19477081298828,
      "activations/layer23_attention_weight_max": 41.00220489501953,
      "activations/layer23_attention_weight_min": -26.08338165283203,
      "activations/layer2_attention_weight_max": 31.93756103515625,
      "activations/layer2_attention_weight_min": -31.723134994506836,
      "activations/layer3_attention_weight_max": 91.49517822265625,
      "activations/layer3_attention_weight_min": -92.83805084228516,
      "activations/layer4_attention_weight_max": 86.92440795898438,
      "activations/layer4_attention_weight_min": -87.23082733154297,
      "activations/layer5_attention_weight_max": 61.09992218017578,
      "activations/layer5_attention_weight_min": -71.49296569824219,
      "activations/layer6_attention_weight_max": 52.03318786621094,
      "activations/layer6_attention_weight_min": -54.22798538208008,
      "activations/layer7_attention_weight_max": 68.6574478149414,
      "activations/layer7_attention_weight_min": -69.14185333251953,
      "activations/layer8_attention_weight_max": 51.830116271972656,
      "activations/layer8_attention_weight_min": -55.112586975097656,
      "activations/layer9_attention_weight_max": 45.13858413696289,
      "activations/layer9_attention_weight_min": -49.241024017333984,
      "epoch": 20.82,
      "learning_rate": 1.5922348484848482e-05,
      "loss": 2.7269,
      "step": 358250
    },
    {
      "activations/layer0_attention_weight_max": 16.685279846191406,
      "activations/layer0_attention_weight_min": -14.051578521728516,
      "activations/layer10_attention_weight_max": 41.48236083984375,
      "activations/layer10_attention_weight_min": -41.412872314453125,
      "activations/layer11_attention_weight_max": 37.894874572753906,
      "activations/layer11_attention_weight_min": -38.107391357421875,
      "activations/layer12_attention_weight_max": 22.134408950805664,
      "activations/layer12_attention_weight_min": -24.22233772277832,
      "activations/layer13_attention_weight_max": 36.54716873168945,
      "activations/layer13_attention_weight_min": -29.40471839904785,
      "activations/layer14_attention_weight_max": 34.6180534362793,
      "activations/layer14_attention_weight_min": -31.629989624023438,
      "activations/layer15_attention_weight_max": 34.86189651489258,
      "activations/layer15_attention_weight_min": -30.742996215820312,
      "activations/layer16_attention_weight_max": 36.9886474609375,
      "activations/layer16_attention_weight_min": -36.192623138427734,
      "activations/layer17_attention_weight_max": 54.19554138183594,
      "activations/layer17_attention_weight_min": -49.33467102050781,
      "activations/layer18_attention_weight_max": 51.21187973022461,
      "activations/layer18_attention_weight_min": -44.25786590576172,
      "activations/layer19_attention_weight_max": 25.610870361328125,
      "activations/layer19_attention_weight_min": -26.531875610351562,
      "activations/layer1_attention_weight_max": 16.26311492919922,
      "activations/layer1_attention_weight_min": -17.03556251525879,
      "activations/layer20_attention_weight_max": 22.565589904785156,
      "activations/layer20_attention_weight_min": -22.43405532836914,
      "activations/layer21_attention_weight_max": 33.22650909423828,
      "activations/layer21_attention_weight_min": -22.88833999633789,
      "activations/layer22_attention_weight_max": 27.12310791015625,
      "activations/layer22_attention_weight_min": -27.103357315063477,
      "activations/layer23_attention_weight_max": 35.36915969848633,
      "activations/layer23_attention_weight_min": -26.76058578491211,
      "activations/layer2_attention_weight_max": 35.75061798095703,
      "activations/layer2_attention_weight_min": -31.36557388305664,
      "activations/layer3_attention_weight_max": 100.95148468017578,
      "activations/layer3_attention_weight_min": -97.61941528320312,
      "activations/layer4_attention_weight_max": 91.32341766357422,
      "activations/layer4_attention_weight_min": -85.98188781738281,
      "activations/layer5_attention_weight_max": 61.5975341796875,
      "activations/layer5_attention_weight_min": -67.98247528076172,
      "activations/layer6_attention_weight_max": 50.46630096435547,
      "activations/layer6_attention_weight_min": -53.99750518798828,
      "activations/layer7_attention_weight_max": 71.03685760498047,
      "activations/layer7_attention_weight_min": -67.96000671386719,
      "activations/layer8_attention_weight_max": 51.08020782470703,
      "activations/layer8_attention_weight_min": -52.92570495605469,
      "activations/layer9_attention_weight_max": 49.10634994506836,
      "activations/layer9_attention_weight_min": -48.21147918701172,
      "epoch": 20.82,
      "learning_rate": 1.5903409090909087e-05,
      "loss": 2.7255,
      "step": 358300
    },
    {
      "activations/layer0_attention_weight_max": 15.294845581054688,
      "activations/layer0_attention_weight_min": -14.043794631958008,
      "activations/layer10_attention_weight_max": 39.119834899902344,
      "activations/layer10_attention_weight_min": -37.28339767456055,
      "activations/layer11_attention_weight_max": 35.989906311035156,
      "activations/layer11_attention_weight_min": -35.76128387451172,
      "activations/layer12_attention_weight_max": 18.5120849609375,
      "activations/layer12_attention_weight_min": -26.66033363342285,
      "activations/layer13_attention_weight_max": 36.579891204833984,
      "activations/layer13_attention_weight_min": -28.152385711669922,
      "activations/layer14_attention_weight_max": 36.50885009765625,
      "activations/layer14_attention_weight_min": -32.75578308105469,
      "activations/layer15_attention_weight_max": 35.16181182861328,
      "activations/layer15_attention_weight_min": -33.30309295654297,
      "activations/layer16_attention_weight_max": 35.859886169433594,
      "activations/layer16_attention_weight_min": -33.43626403808594,
      "activations/layer17_attention_weight_max": 53.61102294921875,
      "activations/layer17_attention_weight_min": -49.12918472290039,
      "activations/layer18_attention_weight_max": 47.932376861572266,
      "activations/layer18_attention_weight_min": -40.92757797241211,
      "activations/layer19_attention_weight_max": 27.655561447143555,
      "activations/layer19_attention_weight_min": -23.62596321105957,
      "activations/layer1_attention_weight_max": 15.569660186767578,
      "activations/layer1_attention_weight_min": -14.354280471801758,
      "activations/layer20_attention_weight_max": 25.211978912353516,
      "activations/layer20_attention_weight_min": -24.113859176635742,
      "activations/layer21_attention_weight_max": 35.70097351074219,
      "activations/layer21_attention_weight_min": -24.195518493652344,
      "activations/layer22_attention_weight_max": 32.63807678222656,
      "activations/layer22_attention_weight_min": -28.09445571899414,
      "activations/layer23_attention_weight_max": 38.65007019042969,
      "activations/layer23_attention_weight_min": -23.7641658782959,
      "activations/layer2_attention_weight_max": 32.34584045410156,
      "activations/layer2_attention_weight_min": -30.712980270385742,
      "activations/layer3_attention_weight_max": 89.72416687011719,
      "activations/layer3_attention_weight_min": -88.57695007324219,
      "activations/layer4_attention_weight_max": 84.23826599121094,
      "activations/layer4_attention_weight_min": -84.5557861328125,
      "activations/layer5_attention_weight_max": 61.892574310302734,
      "activations/layer5_attention_weight_min": -67.58707427978516,
      "activations/layer6_attention_weight_max": 50.077693939208984,
      "activations/layer6_attention_weight_min": -51.7892951965332,
      "activations/layer7_attention_weight_max": 65.66694641113281,
      "activations/layer7_attention_weight_min": -67.93692779541016,
      "activations/layer8_attention_weight_max": 50.0460319519043,
      "activations/layer8_attention_weight_min": -53.68067932128906,
      "activations/layer9_attention_weight_max": 45.78302764892578,
      "activations/layer9_attention_weight_min": -47.90690231323242,
      "epoch": 20.82,
      "learning_rate": 1.5884469696969695e-05,
      "loss": 2.6993,
      "step": 358350
    },
    {
      "activations/layer0_attention_weight_max": 15.741790771484375,
      "activations/layer0_attention_weight_min": -14.08578872680664,
      "activations/layer10_attention_weight_max": 35.1317253112793,
      "activations/layer10_attention_weight_min": -37.101707458496094,
      "activations/layer11_attention_weight_max": 33.91399383544922,
      "activations/layer11_attention_weight_min": -34.76352310180664,
      "activations/layer12_attention_weight_max": 18.255069732666016,
      "activations/layer12_attention_weight_min": -26.007905960083008,
      "activations/layer13_attention_weight_max": 32.332767486572266,
      "activations/layer13_attention_weight_min": -29.73444366455078,
      "activations/layer14_attention_weight_max": 33.393638610839844,
      "activations/layer14_attention_weight_min": -29.62726593017578,
      "activations/layer15_attention_weight_max": 32.67672348022461,
      "activations/layer15_attention_weight_min": -30.531326293945312,
      "activations/layer16_attention_weight_max": 34.179840087890625,
      "activations/layer16_attention_weight_min": -34.23798370361328,
      "activations/layer17_attention_weight_max": 50.353031158447266,
      "activations/layer17_attention_weight_min": -46.95644760131836,
      "activations/layer18_attention_weight_max": 50.49523162841797,
      "activations/layer18_attention_weight_min": -44.05562973022461,
      "activations/layer19_attention_weight_max": 23.052640914916992,
      "activations/layer19_attention_weight_min": -25.23383903503418,
      "activations/layer1_attention_weight_max": 15.908552169799805,
      "activations/layer1_attention_weight_min": -14.59848690032959,
      "activations/layer20_attention_weight_max": 23.80021858215332,
      "activations/layer20_attention_weight_min": -23.763626098632812,
      "activations/layer21_attention_weight_max": 38.05929946899414,
      "activations/layer21_attention_weight_min": -26.096036911010742,
      "activations/layer22_attention_weight_max": 29.586017608642578,
      "activations/layer22_attention_weight_min": -25.037351608276367,
      "activations/layer23_attention_weight_max": 35.611045837402344,
      "activations/layer23_attention_weight_min": -26.5236873626709,
      "activations/layer2_attention_weight_max": 32.73616027832031,
      "activations/layer2_attention_weight_min": -31.356544494628906,
      "activations/layer3_attention_weight_max": 92.19404602050781,
      "activations/layer3_attention_weight_min": -94.45575714111328,
      "activations/layer4_attention_weight_max": 84.0730209350586,
      "activations/layer4_attention_weight_min": -84.95088195800781,
      "activations/layer5_attention_weight_max": 62.392494201660156,
      "activations/layer5_attention_weight_min": -71.09832763671875,
      "activations/layer6_attention_weight_max": 48.45469284057617,
      "activations/layer6_attention_weight_min": -50.75346755981445,
      "activations/layer7_attention_weight_max": 68.32010650634766,
      "activations/layer7_attention_weight_min": -68.16481018066406,
      "activations/layer8_attention_weight_max": 47.82093811035156,
      "activations/layer8_attention_weight_min": -51.292015075683594,
      "activations/layer9_attention_weight_max": 47.648868560791016,
      "activations/layer9_attention_weight_min": -46.5538444519043,
      "epoch": 20.83,
      "learning_rate": 1.58655303030303e-05,
      "loss": 2.7184,
      "step": 358400
    },
    {
      "activations/layer0_attention_weight_max": 16.192041397094727,
      "activations/layer0_attention_weight_min": -14.040689468383789,
      "activations/layer10_attention_weight_max": 37.1192512512207,
      "activations/layer10_attention_weight_min": -36.364051818847656,
      "activations/layer11_attention_weight_max": 35.139766693115234,
      "activations/layer11_attention_weight_min": -36.53514862060547,
      "activations/layer12_attention_weight_max": 20.97371482849121,
      "activations/layer12_attention_weight_min": -27.118030548095703,
      "activations/layer13_attention_weight_max": 37.813079833984375,
      "activations/layer13_attention_weight_min": -28.761390686035156,
      "activations/layer14_attention_weight_max": 38.65208435058594,
      "activations/layer14_attention_weight_min": -31.169193267822266,
      "activations/layer15_attention_weight_max": 35.57878112792969,
      "activations/layer15_attention_weight_min": -28.89386558532715,
      "activations/layer16_attention_weight_max": 35.740562438964844,
      "activations/layer16_attention_weight_min": -33.72653579711914,
      "activations/layer17_attention_weight_max": 53.647987365722656,
      "activations/layer17_attention_weight_min": -48.457557678222656,
      "activations/layer18_attention_weight_max": 52.16480255126953,
      "activations/layer18_attention_weight_min": -41.37972640991211,
      "activations/layer19_attention_weight_max": 26.859668731689453,
      "activations/layer19_attention_weight_min": -23.47051429748535,
      "activations/layer1_attention_weight_max": 15.254558563232422,
      "activations/layer1_attention_weight_min": -14.22637939453125,
      "activations/layer20_attention_weight_max": 26.144145965576172,
      "activations/layer20_attention_weight_min": -23.531129837036133,
      "activations/layer21_attention_weight_max": 43.14407730102539,
      "activations/layer21_attention_weight_min": -22.563318252563477,
      "activations/layer22_attention_weight_max": 32.25423049926758,
      "activations/layer22_attention_weight_min": -24.747541427612305,
      "activations/layer23_attention_weight_max": 42.936946868896484,
      "activations/layer23_attention_weight_min": -24.987934112548828,
      "activations/layer2_attention_weight_max": 33.16988754272461,
      "activations/layer2_attention_weight_min": -31.055103302001953,
      "activations/layer3_attention_weight_max": 87.55543518066406,
      "activations/layer3_attention_weight_min": -91.86478424072266,
      "activations/layer4_attention_weight_max": 83.35871887207031,
      "activations/layer4_attention_weight_min": -82.66493225097656,
      "activations/layer5_attention_weight_max": 61.99980163574219,
      "activations/layer5_attention_weight_min": -72.5923843383789,
      "activations/layer6_attention_weight_max": 49.05269241333008,
      "activations/layer6_attention_weight_min": -49.35063552856445,
      "activations/layer7_attention_weight_max": 64.55936431884766,
      "activations/layer7_attention_weight_min": -67.66471099853516,
      "activations/layer8_attention_weight_max": 46.76139831542969,
      "activations/layer8_attention_weight_min": -53.669639587402344,
      "activations/layer9_attention_weight_max": 47.178470611572266,
      "activations/layer9_attention_weight_min": -47.11600875854492,
      "epoch": 20.83,
      "learning_rate": 1.5846590909090905e-05,
      "loss": 2.7257,
      "step": 358450
    },
    {
      "activations/layer0_attention_weight_max": 16.515357971191406,
      "activations/layer0_attention_weight_min": -14.01567554473877,
      "activations/layer10_attention_weight_max": 35.673927307128906,
      "activations/layer10_attention_weight_min": -35.488563537597656,
      "activations/layer11_attention_weight_max": 32.75432586669922,
      "activations/layer11_attention_weight_min": -35.88919448852539,
      "activations/layer12_attention_weight_max": 18.92161750793457,
      "activations/layer12_attention_weight_min": -24.54814338684082,
      "activations/layer13_attention_weight_max": 33.44270706176758,
      "activations/layer13_attention_weight_min": -26.634593963623047,
      "activations/layer14_attention_weight_max": 31.82341766357422,
      "activations/layer14_attention_weight_min": -30.355722427368164,
      "activations/layer15_attention_weight_max": 31.236793518066406,
      "activations/layer15_attention_weight_min": -30.77703285217285,
      "activations/layer16_attention_weight_max": 33.24195861816406,
      "activations/layer16_attention_weight_min": -32.78548812866211,
      "activations/layer17_attention_weight_max": 48.507503509521484,
      "activations/layer17_attention_weight_min": -44.1546745300293,
      "activations/layer18_attention_weight_max": 45.08373260498047,
      "activations/layer18_attention_weight_min": -39.761009216308594,
      "activations/layer19_attention_weight_max": 23.21068572998047,
      "activations/layer19_attention_weight_min": -23.105024337768555,
      "activations/layer1_attention_weight_max": 17.265995025634766,
      "activations/layer1_attention_weight_min": -13.478314399719238,
      "activations/layer20_attention_weight_max": 22.89665985107422,
      "activations/layer20_attention_weight_min": -22.31245231628418,
      "activations/layer21_attention_weight_max": 34.56409454345703,
      "activations/layer21_attention_weight_min": -23.612356185913086,
      "activations/layer22_attention_weight_max": 27.185712814331055,
      "activations/layer22_attention_weight_min": -25.20012855529785,
      "activations/layer23_attention_weight_max": 32.46451950073242,
      "activations/layer23_attention_weight_min": -23.000974655151367,
      "activations/layer2_attention_weight_max": 30.85725212097168,
      "activations/layer2_attention_weight_min": -30.096717834472656,
      "activations/layer3_attention_weight_max": 87.75962829589844,
      "activations/layer3_attention_weight_min": -84.31774139404297,
      "activations/layer4_attention_weight_max": 81.10930633544922,
      "activations/layer4_attention_weight_min": -82.8366928100586,
      "activations/layer5_attention_weight_max": 62.92324447631836,
      "activations/layer5_attention_weight_min": -73.78197479248047,
      "activations/layer6_attention_weight_max": 49.345088958740234,
      "activations/layer6_attention_weight_min": -49.81636047363281,
      "activations/layer7_attention_weight_max": 65.9389419555664,
      "activations/layer7_attention_weight_min": -66.57574462890625,
      "activations/layer8_attention_weight_max": 45.96351623535156,
      "activations/layer8_attention_weight_min": -52.68407440185547,
      "activations/layer9_attention_weight_max": 43.91858673095703,
      "activations/layer9_attention_weight_min": -46.705467224121094,
      "epoch": 20.83,
      "learning_rate": 1.5827651515151514e-05,
      "loss": 2.7081,
      "step": 358500
    },
    {
      "activations/layer0_attention_weight_max": 15.120009422302246,
      "activations/layer0_attention_weight_min": -13.880327224731445,
      "activations/layer10_attention_weight_max": 36.622745513916016,
      "activations/layer10_attention_weight_min": -39.926025390625,
      "activations/layer11_attention_weight_max": 35.66905212402344,
      "activations/layer11_attention_weight_min": -38.816993713378906,
      "activations/layer12_attention_weight_max": 20.7269229888916,
      "activations/layer12_attention_weight_min": -25.832447052001953,
      "activations/layer13_attention_weight_max": 29.706356048583984,
      "activations/layer13_attention_weight_min": -27.347158432006836,
      "activations/layer14_attention_weight_max": 36.03548049926758,
      "activations/layer14_attention_weight_min": -30.643320083618164,
      "activations/layer15_attention_weight_max": 33.229862213134766,
      "activations/layer15_attention_weight_min": -30.436687469482422,
      "activations/layer16_attention_weight_max": 38.881290435791016,
      "activations/layer16_attention_weight_min": -36.12186813354492,
      "activations/layer17_attention_weight_max": 55.00177001953125,
      "activations/layer17_attention_weight_min": -49.582279205322266,
      "activations/layer18_attention_weight_max": 49.549034118652344,
      "activations/layer18_attention_weight_min": -44.10739517211914,
      "activations/layer19_attention_weight_max": 25.4659423828125,
      "activations/layer19_attention_weight_min": -24.43880271911621,
      "activations/layer1_attention_weight_max": 15.113452911376953,
      "activations/layer1_attention_weight_min": -14.2939453125,
      "activations/layer20_attention_weight_max": 22.29513168334961,
      "activations/layer20_attention_weight_min": -23.53563690185547,
      "activations/layer21_attention_weight_max": 33.70183181762695,
      "activations/layer21_attention_weight_min": -21.8466796875,
      "activations/layer22_attention_weight_max": 26.372365951538086,
      "activations/layer22_attention_weight_min": -25.64242935180664,
      "activations/layer23_attention_weight_max": 38.404022216796875,
      "activations/layer23_attention_weight_min": -24.876148223876953,
      "activations/layer2_attention_weight_max": 31.8587646484375,
      "activations/layer2_attention_weight_min": -31.717144012451172,
      "activations/layer3_attention_weight_max": 87.35855102539062,
      "activations/layer3_attention_weight_min": -86.47260284423828,
      "activations/layer4_attention_weight_max": 83.2370834350586,
      "activations/layer4_attention_weight_min": -83.95536041259766,
      "activations/layer5_attention_weight_max": 59.54869079589844,
      "activations/layer5_attention_weight_min": -72.14390563964844,
      "activations/layer6_attention_weight_max": 47.94990921020508,
      "activations/layer6_attention_weight_min": -51.901981353759766,
      "activations/layer7_attention_weight_max": 61.69681930541992,
      "activations/layer7_attention_weight_min": -64.81036376953125,
      "activations/layer8_attention_weight_max": 45.65177917480469,
      "activations/layer8_attention_weight_min": -53.60405731201172,
      "activations/layer9_attention_weight_max": 45.64783477783203,
      "activations/layer9_attention_weight_min": -49.819007873535156,
      "epoch": 20.83,
      "learning_rate": 1.5808712121212122e-05,
      "loss": 2.7107,
      "step": 358550
    },
    {
      "activations/layer0_attention_weight_max": 14.524576187133789,
      "activations/layer0_attention_weight_min": -13.867387771606445,
      "activations/layer10_attention_weight_max": 34.52642822265625,
      "activations/layer10_attention_weight_min": -37.244529724121094,
      "activations/layer11_attention_weight_max": 31.97449493408203,
      "activations/layer11_attention_weight_min": -33.07563781738281,
      "activations/layer12_attention_weight_max": 19.329696655273438,
      "activations/layer12_attention_weight_min": -24.052303314208984,
      "activations/layer13_attention_weight_max": 32.233070373535156,
      "activations/layer13_attention_weight_min": -29.725391387939453,
      "activations/layer14_attention_weight_max": 34.58454513549805,
      "activations/layer14_attention_weight_min": -30.39120864868164,
      "activations/layer15_attention_weight_max": 33.76827621459961,
      "activations/layer15_attention_weight_min": -29.130083084106445,
      "activations/layer16_attention_weight_max": 35.22373580932617,
      "activations/layer16_attention_weight_min": -32.69245910644531,
      "activations/layer17_attention_weight_max": 51.66801834106445,
      "activations/layer17_attention_weight_min": -43.245697021484375,
      "activations/layer18_attention_weight_max": 48.63970184326172,
      "activations/layer18_attention_weight_min": -39.66717529296875,
      "activations/layer19_attention_weight_max": 23.432649612426758,
      "activations/layer19_attention_weight_min": -23.483293533325195,
      "activations/layer1_attention_weight_max": 15.639152526855469,
      "activations/layer1_attention_weight_min": -15.020767211914062,
      "activations/layer20_attention_weight_max": 21.889257431030273,
      "activations/layer20_attention_weight_min": -22.12801742553711,
      "activations/layer21_attention_weight_max": 33.825469970703125,
      "activations/layer21_attention_weight_min": -24.847532272338867,
      "activations/layer22_attention_weight_max": 29.6298828125,
      "activations/layer22_attention_weight_min": -27.10401153564453,
      "activations/layer23_attention_weight_max": 36.96659851074219,
      "activations/layer23_attention_weight_min": -25.324932098388672,
      "activations/layer2_attention_weight_max": 31.664993286132812,
      "activations/layer2_attention_weight_min": -30.64592742919922,
      "activations/layer3_attention_weight_max": 87.99745178222656,
      "activations/layer3_attention_weight_min": -87.07278442382812,
      "activations/layer4_attention_weight_max": 80.34162902832031,
      "activations/layer4_attention_weight_min": -82.57254791259766,
      "activations/layer5_attention_weight_max": 57.862213134765625,
      "activations/layer5_attention_weight_min": -69.21011352539062,
      "activations/layer6_attention_weight_max": 48.24394989013672,
      "activations/layer6_attention_weight_min": -50.38774108886719,
      "activations/layer7_attention_weight_max": 65.11561584472656,
      "activations/layer7_attention_weight_min": -66.05113220214844,
      "activations/layer8_attention_weight_max": 47.08785629272461,
      "activations/layer8_attention_weight_min": -51.30802536010742,
      "activations/layer9_attention_weight_max": 42.84316635131836,
      "activations/layer9_attention_weight_min": -45.85784149169922,
      "epoch": 20.84,
      "learning_rate": 1.5789772727272727e-05,
      "loss": 2.7092,
      "step": 358600
    },
    {
      "activations/layer0_attention_weight_max": 16.16612434387207,
      "activations/layer0_attention_weight_min": -13.891500473022461,
      "activations/layer10_attention_weight_max": 34.626434326171875,
      "activations/layer10_attention_weight_min": -33.927974700927734,
      "activations/layer11_attention_weight_max": 33.85100173950195,
      "activations/layer11_attention_weight_min": -34.88258361816406,
      "activations/layer12_attention_weight_max": 19.00589370727539,
      "activations/layer12_attention_weight_min": -27.376806259155273,
      "activations/layer13_attention_weight_max": 33.17523956298828,
      "activations/layer13_attention_weight_min": -25.856945037841797,
      "activations/layer14_attention_weight_max": 36.22459411621094,
      "activations/layer14_attention_weight_min": -28.447681427001953,
      "activations/layer15_attention_weight_max": 32.40364074707031,
      "activations/layer15_attention_weight_min": -29.16242027282715,
      "activations/layer16_attention_weight_max": 36.64102554321289,
      "activations/layer16_attention_weight_min": -32.01732635498047,
      "activations/layer17_attention_weight_max": 50.30987548828125,
      "activations/layer17_attention_weight_min": -44.68540573120117,
      "activations/layer18_attention_weight_max": 47.79094314575195,
      "activations/layer18_attention_weight_min": -40.7653923034668,
      "activations/layer19_attention_weight_max": 25.66179847717285,
      "activations/layer19_attention_weight_min": -21.7698917388916,
      "activations/layer1_attention_weight_max": 15.793986320495605,
      "activations/layer1_attention_weight_min": -14.057849884033203,
      "activations/layer20_attention_weight_max": 20.910184860229492,
      "activations/layer20_attention_weight_min": -21.1774959564209,
      "activations/layer21_attention_weight_max": 41.48445510864258,
      "activations/layer21_attention_weight_min": -23.650266647338867,
      "activations/layer22_attention_weight_max": 29.412839889526367,
      "activations/layer22_attention_weight_min": -24.07325553894043,
      "activations/layer23_attention_weight_max": 38.944759368896484,
      "activations/layer23_attention_weight_min": -22.716995239257812,
      "activations/layer2_attention_weight_max": 31.8076229095459,
      "activations/layer2_attention_weight_min": -32.55896759033203,
      "activations/layer3_attention_weight_max": 87.4346923828125,
      "activations/layer3_attention_weight_min": -88.50126647949219,
      "activations/layer4_attention_weight_max": 86.30757904052734,
      "activations/layer4_attention_weight_min": -81.97041320800781,
      "activations/layer5_attention_weight_max": 58.64091110229492,
      "activations/layer5_attention_weight_min": -69.49811553955078,
      "activations/layer6_attention_weight_max": 48.35994338989258,
      "activations/layer6_attention_weight_min": -48.96580505371094,
      "activations/layer7_attention_weight_max": 64.8022689819336,
      "activations/layer7_attention_weight_min": -62.06886672973633,
      "activations/layer8_attention_weight_max": 45.87275695800781,
      "activations/layer8_attention_weight_min": -50.75376510620117,
      "activations/layer9_attention_weight_max": 43.614532470703125,
      "activations/layer9_attention_weight_min": -43.805240631103516,
      "epoch": 20.84,
      "learning_rate": 1.577083333333333e-05,
      "loss": 2.7172,
      "step": 358650
    },
    {
      "activations/layer0_attention_weight_max": 16.01640510559082,
      "activations/layer0_attention_weight_min": -14.769282341003418,
      "activations/layer10_attention_weight_max": 36.21512222290039,
      "activations/layer10_attention_weight_min": -35.87464904785156,
      "activations/layer11_attention_weight_max": 33.5357666015625,
      "activations/layer11_attention_weight_min": -35.68338394165039,
      "activations/layer12_attention_weight_max": 19.52428436279297,
      "activations/layer12_attention_weight_min": -29.947587966918945,
      "activations/layer13_attention_weight_max": 37.362220764160156,
      "activations/layer13_attention_weight_min": -26.736927032470703,
      "activations/layer14_attention_weight_max": 36.61259078979492,
      "activations/layer14_attention_weight_min": -29.394607543945312,
      "activations/layer15_attention_weight_max": 32.50194549560547,
      "activations/layer15_attention_weight_min": -30.521814346313477,
      "activations/layer16_attention_weight_max": 33.7248420715332,
      "activations/layer16_attention_weight_min": -30.82316780090332,
      "activations/layer17_attention_weight_max": 54.13325881958008,
      "activations/layer17_attention_weight_min": -44.795921325683594,
      "activations/layer18_attention_weight_max": 46.662132263183594,
      "activations/layer18_attention_weight_min": -39.82794952392578,
      "activations/layer19_attention_weight_max": 23.611249923706055,
      "activations/layer19_attention_weight_min": -22.13459014892578,
      "activations/layer1_attention_weight_max": 15.167925834655762,
      "activations/layer1_attention_weight_min": -15.512202262878418,
      "activations/layer20_attention_weight_max": 23.006275177001953,
      "activations/layer20_attention_weight_min": -23.11097526550293,
      "activations/layer21_attention_weight_max": 36.2098503112793,
      "activations/layer21_attention_weight_min": -22.933486938476562,
      "activations/layer22_attention_weight_max": 27.722904205322266,
      "activations/layer22_attention_weight_min": -25.0220947265625,
      "activations/layer23_attention_weight_max": 35.197574615478516,
      "activations/layer23_attention_weight_min": -25.220067977905273,
      "activations/layer2_attention_weight_max": 32.203033447265625,
      "activations/layer2_attention_weight_min": -31.28750991821289,
      "activations/layer3_attention_weight_max": 86.37601470947266,
      "activations/layer3_attention_weight_min": -87.05634307861328,
      "activations/layer4_attention_weight_max": 83.61410522460938,
      "activations/layer4_attention_weight_min": -80.83746337890625,
      "activations/layer5_attention_weight_max": 59.333213806152344,
      "activations/layer5_attention_weight_min": -70.17021179199219,
      "activations/layer6_attention_weight_max": 49.48353576660156,
      "activations/layer6_attention_weight_min": -49.65156173706055,
      "activations/layer7_attention_weight_max": 63.341880798339844,
      "activations/layer7_attention_weight_min": -66.028076171875,
      "activations/layer8_attention_weight_max": 49.6360969543457,
      "activations/layer8_attention_weight_min": -53.56639099121094,
      "activations/layer9_attention_weight_max": 44.38204574584961,
      "activations/layer9_attention_weight_min": -45.13003921508789,
      "epoch": 20.84,
      "learning_rate": 1.575189393939394e-05,
      "loss": 2.7228,
      "step": 358700
    },
    {
      "activations/layer0_attention_weight_max": 15.126664161682129,
      "activations/layer0_attention_weight_min": -13.950756072998047,
      "activations/layer10_attention_weight_max": 35.938751220703125,
      "activations/layer10_attention_weight_min": -39.202796936035156,
      "activations/layer11_attention_weight_max": 33.53525161743164,
      "activations/layer11_attention_weight_min": -35.53168869018555,
      "activations/layer12_attention_weight_max": 20.813745498657227,
      "activations/layer12_attention_weight_min": -24.53396987915039,
      "activations/layer13_attention_weight_max": 41.771968841552734,
      "activations/layer13_attention_weight_min": -30.131624221801758,
      "activations/layer14_attention_weight_max": 41.861358642578125,
      "activations/layer14_attention_weight_min": -33.347530364990234,
      "activations/layer15_attention_weight_max": 37.97640609741211,
      "activations/layer15_attention_weight_min": -32.49559783935547,
      "activations/layer16_attention_weight_max": 36.82219314575195,
      "activations/layer16_attention_weight_min": -35.84294509887695,
      "activations/layer17_attention_weight_max": 57.02490234375,
      "activations/layer17_attention_weight_min": -49.363990783691406,
      "activations/layer18_attention_weight_max": 58.17934799194336,
      "activations/layer18_attention_weight_min": -45.7003173828125,
      "activations/layer19_attention_weight_max": 30.05861473083496,
      "activations/layer19_attention_weight_min": -23.37898826599121,
      "activations/layer1_attention_weight_max": 16.0870304107666,
      "activations/layer1_attention_weight_min": -15.143909454345703,
      "activations/layer20_attention_weight_max": 27.852943420410156,
      "activations/layer20_attention_weight_min": -22.5626220703125,
      "activations/layer21_attention_weight_max": 41.50925827026367,
      "activations/layer21_attention_weight_min": -23.965110778808594,
      "activations/layer22_attention_weight_max": 32.047157287597656,
      "activations/layer22_attention_weight_min": -28.94525909423828,
      "activations/layer23_attention_weight_max": 39.736305236816406,
      "activations/layer23_attention_weight_min": -24.40046501159668,
      "activations/layer2_attention_weight_max": 31.34207534790039,
      "activations/layer2_attention_weight_min": -31.735265731811523,
      "activations/layer3_attention_weight_max": 89.42906951904297,
      "activations/layer3_attention_weight_min": -88.46836853027344,
      "activations/layer4_attention_weight_max": 84.123291015625,
      "activations/layer4_attention_weight_min": -80.68866729736328,
      "activations/layer5_attention_weight_max": 59.96464538574219,
      "activations/layer5_attention_weight_min": -70.58322143554688,
      "activations/layer6_attention_weight_max": 46.90458679199219,
      "activations/layer6_attention_weight_min": -50.281429290771484,
      "activations/layer7_attention_weight_max": 65.6238021850586,
      "activations/layer7_attention_weight_min": -63.606021881103516,
      "activations/layer8_attention_weight_max": 45.13479232788086,
      "activations/layer8_attention_weight_min": -48.85789108276367,
      "activations/layer9_attention_weight_max": 47.470394134521484,
      "activations/layer9_attention_weight_min": -45.82963180541992,
      "epoch": 20.85,
      "learning_rate": 1.5732954545454545e-05,
      "loss": 2.7123,
      "step": 358750
    },
    {
      "activations/layer0_attention_weight_max": 15.393942832946777,
      "activations/layer0_attention_weight_min": -13.973190307617188,
      "activations/layer10_attention_weight_max": 39.642581939697266,
      "activations/layer10_attention_weight_min": -37.71396255493164,
      "activations/layer11_attention_weight_max": 37.45966339111328,
      "activations/layer11_attention_weight_min": -40.703857421875,
      "activations/layer12_attention_weight_max": 20.223142623901367,
      "activations/layer12_attention_weight_min": -25.938228607177734,
      "activations/layer13_attention_weight_max": 35.27281951904297,
      "activations/layer13_attention_weight_min": -28.55550193786621,
      "activations/layer14_attention_weight_max": 40.36750030517578,
      "activations/layer14_attention_weight_min": -31.263139724731445,
      "activations/layer15_attention_weight_max": 36.27964782714844,
      "activations/layer15_attention_weight_min": -31.958545684814453,
      "activations/layer16_attention_weight_max": 38.282230377197266,
      "activations/layer16_attention_weight_min": -33.946598052978516,
      "activations/layer17_attention_weight_max": 58.7523078918457,
      "activations/layer17_attention_weight_min": -46.40043258666992,
      "activations/layer18_attention_weight_max": 54.844886779785156,
      "activations/layer18_attention_weight_min": -41.0075569152832,
      "activations/layer19_attention_weight_max": 26.439929962158203,
      "activations/layer19_attention_weight_min": -24.311599731445312,
      "activations/layer1_attention_weight_max": 16.08454132080078,
      "activations/layer1_attention_weight_min": -15.023481369018555,
      "activations/layer20_attention_weight_max": 25.01784324645996,
      "activations/layer20_attention_weight_min": -22.19462776184082,
      "activations/layer21_attention_weight_max": 41.01414489746094,
      "activations/layer21_attention_weight_min": -23.48283576965332,
      "activations/layer22_attention_weight_max": 30.694026947021484,
      "activations/layer22_attention_weight_min": -25.875919342041016,
      "activations/layer23_attention_weight_max": 43.48063659667969,
      "activations/layer23_attention_weight_min": -26.951793670654297,
      "activations/layer2_attention_weight_max": 31.856916427612305,
      "activations/layer2_attention_weight_min": -32.135623931884766,
      "activations/layer3_attention_weight_max": 94.8024673461914,
      "activations/layer3_attention_weight_min": -94.83043670654297,
      "activations/layer4_attention_weight_max": 90.31194305419922,
      "activations/layer4_attention_weight_min": -88.2733383178711,
      "activations/layer5_attention_weight_max": 64.78034973144531,
      "activations/layer5_attention_weight_min": -72.84408569335938,
      "activations/layer6_attention_weight_max": 51.80965805053711,
      "activations/layer6_attention_weight_min": -52.37479019165039,
      "activations/layer7_attention_weight_max": 74.885009765625,
      "activations/layer7_attention_weight_min": -71.80560302734375,
      "activations/layer8_attention_weight_max": 51.99020767211914,
      "activations/layer8_attention_weight_min": -53.55315017700195,
      "activations/layer9_attention_weight_max": 47.43484115600586,
      "activations/layer9_attention_weight_min": -48.764381408691406,
      "epoch": 20.85,
      "learning_rate": 1.571401515151515e-05,
      "loss": 2.7102,
      "step": 358800
    },
    {
      "activations/layer0_attention_weight_max": 16.08868980407715,
      "activations/layer0_attention_weight_min": -14.296887397766113,
      "activations/layer10_attention_weight_max": 37.19569778442383,
      "activations/layer10_attention_weight_min": -37.06674575805664,
      "activations/layer11_attention_weight_max": 34.350120544433594,
      "activations/layer11_attention_weight_min": -36.21990966796875,
      "activations/layer12_attention_weight_max": 20.078298568725586,
      "activations/layer12_attention_weight_min": -24.76020050048828,
      "activations/layer13_attention_weight_max": 34.516685485839844,
      "activations/layer13_attention_weight_min": -27.665803909301758,
      "activations/layer14_attention_weight_max": 37.39021301269531,
      "activations/layer14_attention_weight_min": -31.234256744384766,
      "activations/layer15_attention_weight_max": 34.714759826660156,
      "activations/layer15_attention_weight_min": -31.64523696899414,
      "activations/layer16_attention_weight_max": 34.957576751708984,
      "activations/layer16_attention_weight_min": -32.504634857177734,
      "activations/layer17_attention_weight_max": 51.19959259033203,
      "activations/layer17_attention_weight_min": -45.36357498168945,
      "activations/layer18_attention_weight_max": 48.003944396972656,
      "activations/layer18_attention_weight_min": -39.328895568847656,
      "activations/layer19_attention_weight_max": 25.472631454467773,
      "activations/layer19_attention_weight_min": -21.849504470825195,
      "activations/layer1_attention_weight_max": 15.40988540649414,
      "activations/layer1_attention_weight_min": -14.497422218322754,
      "activations/layer20_attention_weight_max": 24.747974395751953,
      "activations/layer20_attention_weight_min": -19.67573356628418,
      "activations/layer21_attention_weight_max": 34.6148681640625,
      "activations/layer21_attention_weight_min": -23.887781143188477,
      "activations/layer22_attention_weight_max": 28.911062240600586,
      "activations/layer22_attention_weight_min": -24.917165756225586,
      "activations/layer23_attention_weight_max": 37.89192581176758,
      "activations/layer23_attention_weight_min": -24.921817779541016,
      "activations/layer2_attention_weight_max": 32.05546569824219,
      "activations/layer2_attention_weight_min": -30.66815948486328,
      "activations/layer3_attention_weight_max": 91.34415435791016,
      "activations/layer3_attention_weight_min": -89.53436279296875,
      "activations/layer4_attention_weight_max": 85.83614349365234,
      "activations/layer4_attention_weight_min": -84.17041015625,
      "activations/layer5_attention_weight_max": 61.605316162109375,
      "activations/layer5_attention_weight_min": -70.03623962402344,
      "activations/layer6_attention_weight_max": 51.81184005737305,
      "activations/layer6_attention_weight_min": -51.506832122802734,
      "activations/layer7_attention_weight_max": 64.6765365600586,
      "activations/layer7_attention_weight_min": -67.29737854003906,
      "activations/layer8_attention_weight_max": 50.85354232788086,
      "activations/layer8_attention_weight_min": -51.48052978515625,
      "activations/layer9_attention_weight_max": 46.05547332763672,
      "activations/layer9_attention_weight_min": -46.15166091918945,
      "epoch": 20.85,
      "learning_rate": 1.5695075757575758e-05,
      "loss": 2.7008,
      "step": 358850
    },
    {
      "activations/layer0_attention_weight_max": 16.150203704833984,
      "activations/layer0_attention_weight_min": -13.944611549377441,
      "activations/layer10_attention_weight_max": 37.53082275390625,
      "activations/layer10_attention_weight_min": -35.61030578613281,
      "activations/layer11_attention_weight_max": 36.708648681640625,
      "activations/layer11_attention_weight_min": -35.80543899536133,
      "activations/layer12_attention_weight_max": 18.658803939819336,
      "activations/layer12_attention_weight_min": -27.722150802612305,
      "activations/layer13_attention_weight_max": 47.454185485839844,
      "activations/layer13_attention_weight_min": -26.583702087402344,
      "activations/layer14_attention_weight_max": 47.065223693847656,
      "activations/layer14_attention_weight_min": -30.394973754882812,
      "activations/layer15_attention_weight_max": 43.290245056152344,
      "activations/layer15_attention_weight_min": -31.1888484954834,
      "activations/layer16_attention_weight_max": 38.74410629272461,
      "activations/layer16_attention_weight_min": -32.46889114379883,
      "activations/layer17_attention_weight_max": 59.10729217529297,
      "activations/layer17_attention_weight_min": -50.64518356323242,
      "activations/layer18_attention_weight_max": 54.63716125488281,
      "activations/layer18_attention_weight_min": -43.051292419433594,
      "activations/layer19_attention_weight_max": 33.87467956542969,
      "activations/layer19_attention_weight_min": -26.0220947265625,
      "activations/layer1_attention_weight_max": 15.033823013305664,
      "activations/layer1_attention_weight_min": -13.917269706726074,
      "activations/layer20_attention_weight_max": 33.47368621826172,
      "activations/layer20_attention_weight_min": -23.60872459411621,
      "activations/layer21_attention_weight_max": 55.68878936767578,
      "activations/layer21_attention_weight_min": -28.30357551574707,
      "activations/layer22_attention_weight_max": 36.45878219604492,
      "activations/layer22_attention_weight_min": -28.35148811340332,
      "activations/layer23_attention_weight_max": 32.18526840209961,
      "activations/layer23_attention_weight_min": -23.80714988708496,
      "activations/layer2_attention_weight_max": 31.750259399414062,
      "activations/layer2_attention_weight_min": -29.552099227905273,
      "activations/layer3_attention_weight_max": 90.0686264038086,
      "activations/layer3_attention_weight_min": -86.67048645019531,
      "activations/layer4_attention_weight_max": 83.4245834350586,
      "activations/layer4_attention_weight_min": -81.65135955810547,
      "activations/layer5_attention_weight_max": 60.736602783203125,
      "activations/layer5_attention_weight_min": -73.87008666992188,
      "activations/layer6_attention_weight_max": 46.68180847167969,
      "activations/layer6_attention_weight_min": -50.70087432861328,
      "activations/layer7_attention_weight_max": 66.58319091796875,
      "activations/layer7_attention_weight_min": -66.32859802246094,
      "activations/layer8_attention_weight_max": 48.49297332763672,
      "activations/layer8_attention_weight_min": -49.82949447631836,
      "activations/layer9_attention_weight_max": 49.63594055175781,
      "activations/layer9_attention_weight_min": -49.02449035644531,
      "epoch": 20.85,
      "learning_rate": 1.5676136363636363e-05,
      "loss": 2.7182,
      "step": 358900
    },
    {
      "activations/layer0_attention_weight_max": 15.195042610168457,
      "activations/layer0_attention_weight_min": -14.142383575439453,
      "activations/layer10_attention_weight_max": 36.195289611816406,
      "activations/layer10_attention_weight_min": -35.51216506958008,
      "activations/layer11_attention_weight_max": 33.133697509765625,
      "activations/layer11_attention_weight_min": -34.044185638427734,
      "activations/layer12_attention_weight_max": 19.074180603027344,
      "activations/layer12_attention_weight_min": -23.549579620361328,
      "activations/layer13_attention_weight_max": 35.40882110595703,
      "activations/layer13_attention_weight_min": -26.424476623535156,
      "activations/layer14_attention_weight_max": 37.69458770751953,
      "activations/layer14_attention_weight_min": -32.31744384765625,
      "activations/layer15_attention_weight_max": 36.08293533325195,
      "activations/layer15_attention_weight_min": -33.24525451660156,
      "activations/layer16_attention_weight_max": 37.54169845581055,
      "activations/layer16_attention_weight_min": -35.627437591552734,
      "activations/layer17_attention_weight_max": 52.33775329589844,
      "activations/layer17_attention_weight_min": -50.20616912841797,
      "activations/layer18_attention_weight_max": 50.25462341308594,
      "activations/layer18_attention_weight_min": -43.03616714477539,
      "activations/layer19_attention_weight_max": 22.981847763061523,
      "activations/layer19_attention_weight_min": -21.769535064697266,
      "activations/layer1_attention_weight_max": 16.40151596069336,
      "activations/layer1_attention_weight_min": -14.895612716674805,
      "activations/layer20_attention_weight_max": 23.063983917236328,
      "activations/layer20_attention_weight_min": -21.53046226501465,
      "activations/layer21_attention_weight_max": 33.91725158691406,
      "activations/layer21_attention_weight_min": -19.613168716430664,
      "activations/layer22_attention_weight_max": 29.20615005493164,
      "activations/layer22_attention_weight_min": -25.58480453491211,
      "activations/layer23_attention_weight_max": 34.840126037597656,
      "activations/layer23_attention_weight_min": -22.516029357910156,
      "activations/layer2_attention_weight_max": 31.480995178222656,
      "activations/layer2_attention_weight_min": -30.839338302612305,
      "activations/layer3_attention_weight_max": 85.72384643554688,
      "activations/layer3_attention_weight_min": -90.55525970458984,
      "activations/layer4_attention_weight_max": 84.66653442382812,
      "activations/layer4_attention_weight_min": -85.16159057617188,
      "activations/layer5_attention_weight_max": 61.77953338623047,
      "activations/layer5_attention_weight_min": -74.04989624023438,
      "activations/layer6_attention_weight_max": 51.21195602416992,
      "activations/layer6_attention_weight_min": -48.73991394042969,
      "activations/layer7_attention_weight_max": 65.24837493896484,
      "activations/layer7_attention_weight_min": -63.94872283935547,
      "activations/layer8_attention_weight_max": 47.32106399536133,
      "activations/layer8_attention_weight_min": -50.244590759277344,
      "activations/layer9_attention_weight_max": 43.39139938354492,
      "activations/layer9_attention_weight_min": -46.11279296875,
      "epoch": 20.86,
      "learning_rate": 1.5657196969696968e-05,
      "loss": 2.7166,
      "step": 358950
    },
    {
      "activations/layer0_attention_weight_max": 15.620210647583008,
      "activations/layer0_attention_weight_min": -13.948911666870117,
      "activations/layer10_attention_weight_max": 34.71978759765625,
      "activations/layer10_attention_weight_min": -34.920841217041016,
      "activations/layer11_attention_weight_max": 34.51980209350586,
      "activations/layer11_attention_weight_min": -33.955413818359375,
      "activations/layer12_attention_weight_max": 18.793338775634766,
      "activations/layer12_attention_weight_min": -26.197853088378906,
      "activations/layer13_attention_weight_max": 33.72608947753906,
      "activations/layer13_attention_weight_min": -25.76615333557129,
      "activations/layer14_attention_weight_max": 35.77975845336914,
      "activations/layer14_attention_weight_min": -28.970745086669922,
      "activations/layer15_attention_weight_max": 37.78792190551758,
      "activations/layer15_attention_weight_min": -31.159351348876953,
      "activations/layer16_attention_weight_max": 39.33880615234375,
      "activations/layer16_attention_weight_min": -32.48622131347656,
      "activations/layer17_attention_weight_max": 53.440185546875,
      "activations/layer17_attention_weight_min": -45.60392761230469,
      "activations/layer18_attention_weight_max": 51.716983795166016,
      "activations/layer18_attention_weight_min": -38.41340637207031,
      "activations/layer19_attention_weight_max": 23.90693473815918,
      "activations/layer19_attention_weight_min": -20.39753532409668,
      "activations/layer1_attention_weight_max": 14.908196449279785,
      "activations/layer1_attention_weight_min": -15.194978713989258,
      "activations/layer20_attention_weight_max": 21.076932907104492,
      "activations/layer20_attention_weight_min": -20.31946563720703,
      "activations/layer21_attention_weight_max": 34.10694122314453,
      "activations/layer21_attention_weight_min": -20.04977035522461,
      "activations/layer22_attention_weight_max": 30.840425491333008,
      "activations/layer22_attention_weight_min": -23.087244033813477,
      "activations/layer23_attention_weight_max": 35.363807678222656,
      "activations/layer23_attention_weight_min": -26.065946578979492,
      "activations/layer2_attention_weight_max": 31.739721298217773,
      "activations/layer2_attention_weight_min": -31.14892578125,
      "activations/layer3_attention_weight_max": 88.83283233642578,
      "activations/layer3_attention_weight_min": -90.11245727539062,
      "activations/layer4_attention_weight_max": 82.67728424072266,
      "activations/layer4_attention_weight_min": -83.68582153320312,
      "activations/layer5_attention_weight_max": 58.77922058105469,
      "activations/layer5_attention_weight_min": -71.19041442871094,
      "activations/layer6_attention_weight_max": 48.884159088134766,
      "activations/layer6_attention_weight_min": -50.38648223876953,
      "activations/layer7_attention_weight_max": 63.64298629760742,
      "activations/layer7_attention_weight_min": -66.73182678222656,
      "activations/layer8_attention_weight_max": 47.625091552734375,
      "activations/layer8_attention_weight_min": -51.173274993896484,
      "activations/layer9_attention_weight_max": 42.08051300048828,
      "activations/layer9_attention_weight_min": -43.873226165771484,
      "epoch": 20.86,
      "learning_rate": 1.5638257575757573e-05,
      "loss": 2.7006,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.5544,
      "eval_samples_per_second": 501.965,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.5544,
      "eval_openwebtext_samples_per_second": 501.965,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9858,
      "eval_wikitext_samples_per_second": 229.63,
      "step": 359000
    },
    {
      "epoch": 20.86,
      "eval_lambada_loss": 2.44140625,
      "eval_lambada_ppl": 11.489186055851967,
      "eval_lambada_runtime": 9.5782,
      "eval_lambada_samples_per_second": 508.344,
      "step": 359000
    },
    {
      "activations/layer0_attention_weight_max": 16.3348388671875,
      "activations/layer0_attention_weight_min": -14.072242736816406,
      "activations/layer10_attention_weight_max": 38.207515716552734,
      "activations/layer10_attention_weight_min": -39.267311096191406,
      "activations/layer11_attention_weight_max": 34.94859313964844,
      "activations/layer11_attention_weight_min": -37.63187789916992,
      "activations/layer12_attention_weight_max": 19.523826599121094,
      "activations/layer12_attention_weight_min": -26.100540161132812,
      "activations/layer13_attention_weight_max": 35.101417541503906,
      "activations/layer13_attention_weight_min": -31.685272216796875,
      "activations/layer14_attention_weight_max": 36.72400665283203,
      "activations/layer14_attention_weight_min": -33.088314056396484,
      "activations/layer15_attention_weight_max": 35.13682556152344,
      "activations/layer15_attention_weight_min": -31.169673919677734,
      "activations/layer16_attention_weight_max": 36.08385467529297,
      "activations/layer16_attention_weight_min": -33.58478927612305,
      "activations/layer17_attention_weight_max": 50.80160903930664,
      "activations/layer17_attention_weight_min": -47.90288162231445,
      "activations/layer18_attention_weight_max": 45.6517448425293,
      "activations/layer18_attention_weight_min": -43.35407638549805,
      "activations/layer19_attention_weight_max": 23.697956085205078,
      "activations/layer19_attention_weight_min": -23.81863021850586,
      "activations/layer1_attention_weight_max": 15.207112312316895,
      "activations/layer1_attention_weight_min": -14.937840461730957,
      "activations/layer20_attention_weight_max": 22.19609260559082,
      "activations/layer20_attention_weight_min": -22.623790740966797,
      "activations/layer21_attention_weight_max": 37.14992141723633,
      "activations/layer21_attention_weight_min": -25.315458297729492,
      "activations/layer22_attention_weight_max": 28.554481506347656,
      "activations/layer22_attention_weight_min": -27.48750877380371,
      "activations/layer23_attention_weight_max": 35.837249755859375,
      "activations/layer23_attention_weight_min": -25.330352783203125,
      "activations/layer2_attention_weight_max": 31.299785614013672,
      "activations/layer2_attention_weight_min": -29.58125877380371,
      "activations/layer3_attention_weight_max": 87.3453369140625,
      "activations/layer3_attention_weight_min": -87.34651947021484,
      "activations/layer4_attention_weight_max": 83.73684692382812,
      "activations/layer4_attention_weight_min": -84.82351684570312,
      "activations/layer5_attention_weight_max": 60.79259490966797,
      "activations/layer5_attention_weight_min": -70.29754638671875,
      "activations/layer6_attention_weight_max": 48.970924377441406,
      "activations/layer6_attention_weight_min": -50.46475601196289,
      "activations/layer7_attention_weight_max": 66.18778228759766,
      "activations/layer7_attention_weight_min": -68.47590637207031,
      "activations/layer8_attention_weight_max": 46.01813507080078,
      "activations/layer8_attention_weight_min": -51.318363189697266,
      "activations/layer9_attention_weight_max": 44.279197692871094,
      "activations/layer9_attention_weight_min": -47.02031707763672,
      "epoch": 20.86,
      "learning_rate": 1.561931818181818e-05,
      "loss": 2.7056,
      "step": 359050
    },
    {
      "activations/layer0_attention_weight_max": 15.091748237609863,
      "activations/layer0_attention_weight_min": -14.167205810546875,
      "activations/layer10_attention_weight_max": 36.30971145629883,
      "activations/layer10_attention_weight_min": -36.292476654052734,
      "activations/layer11_attention_weight_max": 34.36432647705078,
      "activations/layer11_attention_weight_min": -34.2653694152832,
      "activations/layer12_attention_weight_max": 19.571903228759766,
      "activations/layer12_attention_weight_min": -23.461679458618164,
      "activations/layer13_attention_weight_max": 32.60889434814453,
      "activations/layer13_attention_weight_min": -26.873470306396484,
      "activations/layer14_attention_weight_max": 39.153385162353516,
      "activations/layer14_attention_weight_min": -29.81277084350586,
      "activations/layer15_attention_weight_max": 35.70672607421875,
      "activations/layer15_attention_weight_min": -30.020883560180664,
      "activations/layer16_attention_weight_max": 33.92924118041992,
      "activations/layer16_attention_weight_min": -32.66838836669922,
      "activations/layer17_attention_weight_max": 54.70417022705078,
      "activations/layer17_attention_weight_min": -47.18410110473633,
      "activations/layer18_attention_weight_max": 48.19819641113281,
      "activations/layer18_attention_weight_min": -41.383201599121094,
      "activations/layer19_attention_weight_max": 24.332195281982422,
      "activations/layer19_attention_weight_min": -24.603755950927734,
      "activations/layer1_attention_weight_max": 15.746835708618164,
      "activations/layer1_attention_weight_min": -13.275334358215332,
      "activations/layer20_attention_weight_max": 23.53031349182129,
      "activations/layer20_attention_weight_min": -22.411516189575195,
      "activations/layer21_attention_weight_max": 36.54936981201172,
      "activations/layer21_attention_weight_min": -22.677377700805664,
      "activations/layer22_attention_weight_max": 30.99308967590332,
      "activations/layer22_attention_weight_min": -27.074251174926758,
      "activations/layer23_attention_weight_max": 38.44480895996094,
      "activations/layer23_attention_weight_min": -25.8249568939209,
      "activations/layer2_attention_weight_max": 32.672874450683594,
      "activations/layer2_attention_weight_min": -30.024810791015625,
      "activations/layer3_attention_weight_max": 88.55353546142578,
      "activations/layer3_attention_weight_min": -89.55001831054688,
      "activations/layer4_attention_weight_max": 84.59661865234375,
      "activations/layer4_attention_weight_min": -84.27874755859375,
      "activations/layer5_attention_weight_max": 62.69918441772461,
      "activations/layer5_attention_weight_min": -74.81303405761719,
      "activations/layer6_attention_weight_max": 49.95185852050781,
      "activations/layer6_attention_weight_min": -51.82689666748047,
      "activations/layer7_attention_weight_max": 66.04601287841797,
      "activations/layer7_attention_weight_min": -65.43595123291016,
      "activations/layer8_attention_weight_max": 48.2319221496582,
      "activations/layer8_attention_weight_min": -50.674617767333984,
      "activations/layer9_attention_weight_max": 44.48771286010742,
      "activations/layer9_attention_weight_min": -46.444889068603516,
      "epoch": 20.87,
      "learning_rate": 1.5600378787878786e-05,
      "loss": 2.6951,
      "step": 359100
    },
    {
      "activations/layer0_attention_weight_max": 16.20469856262207,
      "activations/layer0_attention_weight_min": -14.029159545898438,
      "activations/layer10_attention_weight_max": 32.302528381347656,
      "activations/layer10_attention_weight_min": -34.610740661621094,
      "activations/layer11_attention_weight_max": 32.84607696533203,
      "activations/layer11_attention_weight_min": -34.55696487426758,
      "activations/layer12_attention_weight_max": 19.08584213256836,
      "activations/layer12_attention_weight_min": -24.7032470703125,
      "activations/layer13_attention_weight_max": 39.306365966796875,
      "activations/layer13_attention_weight_min": -28.280954360961914,
      "activations/layer14_attention_weight_max": 33.72539138793945,
      "activations/layer14_attention_weight_min": -30.538801193237305,
      "activations/layer15_attention_weight_max": 31.719274520874023,
      "activations/layer15_attention_weight_min": -30.274267196655273,
      "activations/layer16_attention_weight_max": 33.78879165649414,
      "activations/layer16_attention_weight_min": -32.6351432800293,
      "activations/layer17_attention_weight_max": 50.62554931640625,
      "activations/layer17_attention_weight_min": -44.0900764465332,
      "activations/layer18_attention_weight_max": 48.09779357910156,
      "activations/layer18_attention_weight_min": -38.89039993286133,
      "activations/layer19_attention_weight_max": 23.6451473236084,
      "activations/layer19_attention_weight_min": -23.682188034057617,
      "activations/layer1_attention_weight_max": 15.221266746520996,
      "activations/layer1_attention_weight_min": -14.541248321533203,
      "activations/layer20_attention_weight_max": 22.94980239868164,
      "activations/layer20_attention_weight_min": -23.4903564453125,
      "activations/layer21_attention_weight_max": 36.980987548828125,
      "activations/layer21_attention_weight_min": -20.34634780883789,
      "activations/layer22_attention_weight_max": 29.049043655395508,
      "activations/layer22_attention_weight_min": -27.610260009765625,
      "activations/layer23_attention_weight_max": 39.14054870605469,
      "activations/layer23_attention_weight_min": -23.4991512298584,
      "activations/layer2_attention_weight_max": 32.088890075683594,
      "activations/layer2_attention_weight_min": -29.708560943603516,
      "activations/layer3_attention_weight_max": 88.90541076660156,
      "activations/layer3_attention_weight_min": -88.01712799072266,
      "activations/layer4_attention_weight_max": 81.20123291015625,
      "activations/layer4_attention_weight_min": -84.01396179199219,
      "activations/layer5_attention_weight_max": 60.95283508300781,
      "activations/layer5_attention_weight_min": -72.57231140136719,
      "activations/layer6_attention_weight_max": 46.13779067993164,
      "activations/layer6_attention_weight_min": -48.98242950439453,
      "activations/layer7_attention_weight_max": 65.43665313720703,
      "activations/layer7_attention_weight_min": -66.53778839111328,
      "activations/layer8_attention_weight_max": 44.47079086303711,
      "activations/layer8_attention_weight_min": -46.9362678527832,
      "activations/layer9_attention_weight_max": 47.11357498168945,
      "activations/layer9_attention_weight_min": -44.56180953979492,
      "epoch": 20.87,
      "learning_rate": 1.558143939393939e-05,
      "loss": 2.717,
      "step": 359150
    },
    {
      "activations/layer0_attention_weight_max": 15.609490394592285,
      "activations/layer0_attention_weight_min": -14.272505760192871,
      "activations/layer10_attention_weight_max": 41.20060729980469,
      "activations/layer10_attention_weight_min": -38.00310134887695,
      "activations/layer11_attention_weight_max": 37.39772415161133,
      "activations/layer11_attention_weight_min": -35.54698181152344,
      "activations/layer12_attention_weight_max": 18.462675094604492,
      "activations/layer12_attention_weight_min": -25.473703384399414,
      "activations/layer13_attention_weight_max": 32.22734451293945,
      "activations/layer13_attention_weight_min": -27.963844299316406,
      "activations/layer14_attention_weight_max": 33.964290618896484,
      "activations/layer14_attention_weight_min": -30.99662971496582,
      "activations/layer15_attention_weight_max": 32.09138107299805,
      "activations/layer15_attention_weight_min": -29.83365821838379,
      "activations/layer16_attention_weight_max": 35.46329116821289,
      "activations/layer16_attention_weight_min": -33.56513214111328,
      "activations/layer17_attention_weight_max": 53.6699104309082,
      "activations/layer17_attention_weight_min": -47.335018157958984,
      "activations/layer18_attention_weight_max": 47.33953094482422,
      "activations/layer18_attention_weight_min": -41.758811950683594,
      "activations/layer19_attention_weight_max": 25.2858943939209,
      "activations/layer19_attention_weight_min": -24.20477294921875,
      "activations/layer1_attention_weight_max": 15.72515869140625,
      "activations/layer1_attention_weight_min": -13.95043659210205,
      "activations/layer20_attention_weight_max": 23.41950035095215,
      "activations/layer20_attention_weight_min": -23.49128532409668,
      "activations/layer21_attention_weight_max": 36.484291076660156,
      "activations/layer21_attention_weight_min": -24.585166931152344,
      "activations/layer22_attention_weight_max": 30.025917053222656,
      "activations/layer22_attention_weight_min": -27.02233123779297,
      "activations/layer23_attention_weight_max": 37.63957977294922,
      "activations/layer23_attention_weight_min": -25.351425170898438,
      "activations/layer2_attention_weight_max": 32.676475524902344,
      "activations/layer2_attention_weight_min": -30.340106964111328,
      "activations/layer3_attention_weight_max": 92.43904113769531,
      "activations/layer3_attention_weight_min": -93.48617553710938,
      "activations/layer4_attention_weight_max": 86.24124908447266,
      "activations/layer4_attention_weight_min": -89.25696563720703,
      "activations/layer5_attention_weight_max": 63.885887145996094,
      "activations/layer5_attention_weight_min": -73.51885223388672,
      "activations/layer6_attention_weight_max": 50.59242630004883,
      "activations/layer6_attention_weight_min": -51.26545715332031,
      "activations/layer7_attention_weight_max": 66.2737808227539,
      "activations/layer7_attention_weight_min": -66.03915405273438,
      "activations/layer8_attention_weight_max": 50.691619873046875,
      "activations/layer8_attention_weight_min": -51.33989715576172,
      "activations/layer9_attention_weight_max": 46.89314270019531,
      "activations/layer9_attention_weight_min": -47.52249526977539,
      "epoch": 20.87,
      "learning_rate": 1.55625e-05,
      "loss": 2.7067,
      "step": 359200
    },
    {
      "activations/layer0_attention_weight_max": 15.768474578857422,
      "activations/layer0_attention_weight_min": -14.037961959838867,
      "activations/layer10_attention_weight_max": 40.109291076660156,
      "activations/layer10_attention_weight_min": -37.56523513793945,
      "activations/layer11_attention_weight_max": 36.179439544677734,
      "activations/layer11_attention_weight_min": -36.54395294189453,
      "activations/layer12_attention_weight_max": 20.067291259765625,
      "activations/layer12_attention_weight_min": -22.90630531311035,
      "activations/layer13_attention_weight_max": 36.34574508666992,
      "activations/layer13_attention_weight_min": -29.25600814819336,
      "activations/layer14_attention_weight_max": 38.61384963989258,
      "activations/layer14_attention_weight_min": -31.237207412719727,
      "activations/layer15_attention_weight_max": 36.91984176635742,
      "activations/layer15_attention_weight_min": -31.308958053588867,
      "activations/layer16_attention_weight_max": 36.95059585571289,
      "activations/layer16_attention_weight_min": -33.21637725830078,
      "activations/layer17_attention_weight_max": 53.713470458984375,
      "activations/layer17_attention_weight_min": -44.70132827758789,
      "activations/layer18_attention_weight_max": 50.072532653808594,
      "activations/layer18_attention_weight_min": -38.91434097290039,
      "activations/layer19_attention_weight_max": 26.194849014282227,
      "activations/layer19_attention_weight_min": -20.47947120666504,
      "activations/layer1_attention_weight_max": 14.979655265808105,
      "activations/layer1_attention_weight_min": -14.21795654296875,
      "activations/layer20_attention_weight_max": 21.03331756591797,
      "activations/layer20_attention_weight_min": -20.367332458496094,
      "activations/layer21_attention_weight_max": 31.863834381103516,
      "activations/layer21_attention_weight_min": -20.189878463745117,
      "activations/layer22_attention_weight_max": 28.64305877685547,
      "activations/layer22_attention_weight_min": -24.065279006958008,
      "activations/layer23_attention_weight_max": 36.75468826293945,
      "activations/layer23_attention_weight_min": -23.116661071777344,
      "activations/layer2_attention_weight_max": 31.489906311035156,
      "activations/layer2_attention_weight_min": -30.897550582885742,
      "activations/layer3_attention_weight_max": 91.6679916381836,
      "activations/layer3_attention_weight_min": -91.81140899658203,
      "activations/layer4_attention_weight_max": 89.25670623779297,
      "activations/layer4_attention_weight_min": -89.93678283691406,
      "activations/layer5_attention_weight_max": 63.01313400268555,
      "activations/layer5_attention_weight_min": -72.7049789428711,
      "activations/layer6_attention_weight_max": 50.6260871887207,
      "activations/layer6_attention_weight_min": -51.8620491027832,
      "activations/layer7_attention_weight_max": 67.02452087402344,
      "activations/layer7_attention_weight_min": -69.51676177978516,
      "activations/layer8_attention_weight_max": 48.63853073120117,
      "activations/layer8_attention_weight_min": -51.91295623779297,
      "activations/layer9_attention_weight_max": 46.93826675415039,
      "activations/layer9_attention_weight_min": -50.03121566772461,
      "epoch": 20.87,
      "learning_rate": 1.5543560606060604e-05,
      "loss": 2.7129,
      "step": 359250
    },
    {
      "activations/layer0_attention_weight_max": 15.67223834991455,
      "activations/layer0_attention_weight_min": -13.825507164001465,
      "activations/layer10_attention_weight_max": 35.1924934387207,
      "activations/layer10_attention_weight_min": -37.722801208496094,
      "activations/layer11_attention_weight_max": 33.61274719238281,
      "activations/layer11_attention_weight_min": -37.136192321777344,
      "activations/layer12_attention_weight_max": 20.224212646484375,
      "activations/layer12_attention_weight_min": -24.432382583618164,
      "activations/layer13_attention_weight_max": 33.914161682128906,
      "activations/layer13_attention_weight_min": -27.37854766845703,
      "activations/layer14_attention_weight_max": 36.976646423339844,
      "activations/layer14_attention_weight_min": -31.81589126586914,
      "activations/layer15_attention_weight_max": 34.757930755615234,
      "activations/layer15_attention_weight_min": -32.292724609375,
      "activations/layer16_attention_weight_max": 35.124324798583984,
      "activations/layer16_attention_weight_min": -32.48112869262695,
      "activations/layer17_attention_weight_max": 52.60967254638672,
      "activations/layer17_attention_weight_min": -45.04638671875,
      "activations/layer18_attention_weight_max": 47.240272521972656,
      "activations/layer18_attention_weight_min": -39.22404861450195,
      "activations/layer19_attention_weight_max": 22.21498680114746,
      "activations/layer19_attention_weight_min": -23.240873336791992,
      "activations/layer1_attention_weight_max": 15.4775390625,
      "activations/layer1_attention_weight_min": -15.39649486541748,
      "activations/layer20_attention_weight_max": 21.6026554107666,
      "activations/layer20_attention_weight_min": -21.334125518798828,
      "activations/layer21_attention_weight_max": 35.15179443359375,
      "activations/layer21_attention_weight_min": -22.197607040405273,
      "activations/layer22_attention_weight_max": 26.443939208984375,
      "activations/layer22_attention_weight_min": -25.9439754486084,
      "activations/layer23_attention_weight_max": 35.841407775878906,
      "activations/layer23_attention_weight_min": -23.94240951538086,
      "activations/layer2_attention_weight_max": 32.06599426269531,
      "activations/layer2_attention_weight_min": -31.966978073120117,
      "activations/layer3_attention_weight_max": 92.54033660888672,
      "activations/layer3_attention_weight_min": -91.3533935546875,
      "activations/layer4_attention_weight_max": 86.52421569824219,
      "activations/layer4_attention_weight_min": -88.50029754638672,
      "activations/layer5_attention_weight_max": 61.80738067626953,
      "activations/layer5_attention_weight_min": -79.295166015625,
      "activations/layer6_attention_weight_max": 50.81098556518555,
      "activations/layer6_attention_weight_min": -53.715030670166016,
      "activations/layer7_attention_weight_max": 68.06532287597656,
      "activations/layer7_attention_weight_min": -68.48270416259766,
      "activations/layer8_attention_weight_max": 48.11821365356445,
      "activations/layer8_attention_weight_min": -57.72078323364258,
      "activations/layer9_attention_weight_max": 43.57277297973633,
      "activations/layer9_attention_weight_min": -46.586761474609375,
      "epoch": 20.88,
      "learning_rate": 1.552462121212121e-05,
      "loss": 2.72,
      "step": 359300
    },
    {
      "activations/layer0_attention_weight_max": 16.092802047729492,
      "activations/layer0_attention_weight_min": -14.000105857849121,
      "activations/layer10_attention_weight_max": 38.98371505737305,
      "activations/layer10_attention_weight_min": -36.276275634765625,
      "activations/layer11_attention_weight_max": 34.45225524902344,
      "activations/layer11_attention_weight_min": -35.610530853271484,
      "activations/layer12_attention_weight_max": 20.4810791015625,
      "activations/layer12_attention_weight_min": -26.44548225402832,
      "activations/layer13_attention_weight_max": 35.190147399902344,
      "activations/layer13_attention_weight_min": -29.894189834594727,
      "activations/layer14_attention_weight_max": 35.747154235839844,
      "activations/layer14_attention_weight_min": -29.78477668762207,
      "activations/layer15_attention_weight_max": 31.829797744750977,
      "activations/layer15_attention_weight_min": -31.18492889404297,
      "activations/layer16_attention_weight_max": 35.266693115234375,
      "activations/layer16_attention_weight_min": -34.722015380859375,
      "activations/layer17_attention_weight_max": 54.528438568115234,
      "activations/layer17_attention_weight_min": -47.285980224609375,
      "activations/layer18_attention_weight_max": 49.8143310546875,
      "activations/layer18_attention_weight_min": -41.2627067565918,
      "activations/layer19_attention_weight_max": 26.084491729736328,
      "activations/layer19_attention_weight_min": -25.066661834716797,
      "activations/layer1_attention_weight_max": 15.959585189819336,
      "activations/layer1_attention_weight_min": -15.207226753234863,
      "activations/layer20_attention_weight_max": 25.214702606201172,
      "activations/layer20_attention_weight_min": -23.137144088745117,
      "activations/layer21_attention_weight_max": 37.71762466430664,
      "activations/layer21_attention_weight_min": -24.612380981445312,
      "activations/layer22_attention_weight_max": 28.89773178100586,
      "activations/layer22_attention_weight_min": -26.285261154174805,
      "activations/layer23_attention_weight_max": 35.33262252807617,
      "activations/layer23_attention_weight_min": -25.5156307220459,
      "activations/layer2_attention_weight_max": 32.64780807495117,
      "activations/layer2_attention_weight_min": -32.02614212036133,
      "activations/layer3_attention_weight_max": 89.04856872558594,
      "activations/layer3_attention_weight_min": -87.66857147216797,
      "activations/layer4_attention_weight_max": 85.62423706054688,
      "activations/layer4_attention_weight_min": -85.48558044433594,
      "activations/layer5_attention_weight_max": 66.02436828613281,
      "activations/layer5_attention_weight_min": -72.31864166259766,
      "activations/layer6_attention_weight_max": 51.96847152709961,
      "activations/layer6_attention_weight_min": -52.298728942871094,
      "activations/layer7_attention_weight_max": 67.96199035644531,
      "activations/layer7_attention_weight_min": -68.34169006347656,
      "activations/layer8_attention_weight_max": 50.25094223022461,
      "activations/layer8_attention_weight_min": -52.04922103881836,
      "activations/layer9_attention_weight_max": 49.31863021850586,
      "activations/layer9_attention_weight_min": -47.4982795715332,
      "epoch": 20.88,
      "learning_rate": 1.5506060606060603e-05,
      "loss": 2.7215,
      "step": 359350
    },
    {
      "activations/layer0_attention_weight_max": 16.55610466003418,
      "activations/layer0_attention_weight_min": -13.82645034790039,
      "activations/layer10_attention_weight_max": 35.35918045043945,
      "activations/layer10_attention_weight_min": -34.74580764770508,
      "activations/layer11_attention_weight_max": 34.723079681396484,
      "activations/layer11_attention_weight_min": -33.48101043701172,
      "activations/layer12_attention_weight_max": 18.95280647277832,
      "activations/layer12_attention_weight_min": -25.666040420532227,
      "activations/layer13_attention_weight_max": 34.97288513183594,
      "activations/layer13_attention_weight_min": -28.37616729736328,
      "activations/layer14_attention_weight_max": 36.65876007080078,
      "activations/layer14_attention_weight_min": -30.52730941772461,
      "activations/layer15_attention_weight_max": 34.08916473388672,
      "activations/layer15_attention_weight_min": -31.087068557739258,
      "activations/layer16_attention_weight_max": 33.731258392333984,
      "activations/layer16_attention_weight_min": -32.54178237915039,
      "activations/layer17_attention_weight_max": 52.76917266845703,
      "activations/layer17_attention_weight_min": -45.92287063598633,
      "activations/layer18_attention_weight_max": 51.4636344909668,
      "activations/layer18_attention_weight_min": -40.4614143371582,
      "activations/layer19_attention_weight_max": 27.467060089111328,
      "activations/layer19_attention_weight_min": -21.52248764038086,
      "activations/layer1_attention_weight_max": 14.94924545288086,
      "activations/layer1_attention_weight_min": -14.001829147338867,
      "activations/layer20_attention_weight_max": 23.943748474121094,
      "activations/layer20_attention_weight_min": -22.860597610473633,
      "activations/layer21_attention_weight_max": 36.627220153808594,
      "activations/layer21_attention_weight_min": -21.394290924072266,
      "activations/layer22_attention_weight_max": 29.83042335510254,
      "activations/layer22_attention_weight_min": -25.743703842163086,
      "activations/layer23_attention_weight_max": 37.80712127685547,
      "activations/layer23_attention_weight_min": -22.96320915222168,
      "activations/layer2_attention_weight_max": 30.345256805419922,
      "activations/layer2_attention_weight_min": -29.452444076538086,
      "activations/layer3_attention_weight_max": 88.02787017822266,
      "activations/layer3_attention_weight_min": -86.44210052490234,
      "activations/layer4_attention_weight_max": 80.3028793334961,
      "activations/layer4_attention_weight_min": -82.55950164794922,
      "activations/layer5_attention_weight_max": 59.84212875366211,
      "activations/layer5_attention_weight_min": -76.61730194091797,
      "activations/layer6_attention_weight_max": 47.62751770019531,
      "activations/layer6_attention_weight_min": -49.20746994018555,
      "activations/layer7_attention_weight_max": 66.40383911132812,
      "activations/layer7_attention_weight_min": -68.12450408935547,
      "activations/layer8_attention_weight_max": 44.441932678222656,
      "activations/layer8_attention_weight_min": -50.67061233520508,
      "activations/layer9_attention_weight_max": 44.24003982543945,
      "activations/layer9_attention_weight_min": -47.45466995239258,
      "epoch": 20.88,
      "learning_rate": 1.548712121212121e-05,
      "loss": 2.7261,
      "step": 359400
    },
    {
      "activations/layer0_attention_weight_max": 16.488510131835938,
      "activations/layer0_attention_weight_min": -13.717569351196289,
      "activations/layer10_attention_weight_max": 37.1507682800293,
      "activations/layer10_attention_weight_min": -37.61328125,
      "activations/layer11_attention_weight_max": 34.54426574707031,
      "activations/layer11_attention_weight_min": -38.58383560180664,
      "activations/layer12_attention_weight_max": 20.93511390686035,
      "activations/layer12_attention_weight_min": -24.121139526367188,
      "activations/layer13_attention_weight_max": 33.41364288330078,
      "activations/layer13_attention_weight_min": -27.84151268005371,
      "activations/layer14_attention_weight_max": 39.85820007324219,
      "activations/layer14_attention_weight_min": -36.43782424926758,
      "activations/layer15_attention_weight_max": 35.7393684387207,
      "activations/layer15_attention_weight_min": -32.7805290222168,
      "activations/layer16_attention_weight_max": 37.21565628051758,
      "activations/layer16_attention_weight_min": -35.24919509887695,
      "activations/layer17_attention_weight_max": 53.57743453979492,
      "activations/layer17_attention_weight_min": -47.67238998413086,
      "activations/layer18_attention_weight_max": 52.70555877685547,
      "activations/layer18_attention_weight_min": -42.52428436279297,
      "activations/layer19_attention_weight_max": 25.095029830932617,
      "activations/layer19_attention_weight_min": -24.481260299682617,
      "activations/layer1_attention_weight_max": 16.330087661743164,
      "activations/layer1_attention_weight_min": -16.069305419921875,
      "activations/layer20_attention_weight_max": 23.3736629486084,
      "activations/layer20_attention_weight_min": -21.573381423950195,
      "activations/layer21_attention_weight_max": 35.62262725830078,
      "activations/layer21_attention_weight_min": -23.650968551635742,
      "activations/layer22_attention_weight_max": 29.279483795166016,
      "activations/layer22_attention_weight_min": -26.75749397277832,
      "activations/layer23_attention_weight_max": 38.26655960083008,
      "activations/layer23_attention_weight_min": -26.05147933959961,
      "activations/layer2_attention_weight_max": 31.731719970703125,
      "activations/layer2_attention_weight_min": -29.85047149658203,
      "activations/layer3_attention_weight_max": 89.43181610107422,
      "activations/layer3_attention_weight_min": -87.67149353027344,
      "activations/layer4_attention_weight_max": 86.4894027709961,
      "activations/layer4_attention_weight_min": -86.25076293945312,
      "activations/layer5_attention_weight_max": 62.58187484741211,
      "activations/layer5_attention_weight_min": -70.78553771972656,
      "activations/layer6_attention_weight_max": 49.939064025878906,
      "activations/layer6_attention_weight_min": -53.019737243652344,
      "activations/layer7_attention_weight_max": 73.88975524902344,
      "activations/layer7_attention_weight_min": -66.04672241210938,
      "activations/layer8_attention_weight_max": 48.32206726074219,
      "activations/layer8_attention_weight_min": -53.65608596801758,
      "activations/layer9_attention_weight_max": 45.802059173583984,
      "activations/layer9_attention_weight_min": -47.115909576416016,
      "epoch": 20.89,
      "learning_rate": 1.546818181818182e-05,
      "loss": 2.7149,
      "step": 359450
    },
    {
      "activations/layer0_attention_weight_max": 15.134642601013184,
      "activations/layer0_attention_weight_min": -13.827784538269043,
      "activations/layer10_attention_weight_max": 39.1949348449707,
      "activations/layer10_attention_weight_min": -37.97438430786133,
      "activations/layer11_attention_weight_max": 35.580692291259766,
      "activations/layer11_attention_weight_min": -33.32281494140625,
      "activations/layer12_attention_weight_max": 19.507770538330078,
      "activations/layer12_attention_weight_min": -26.68335723876953,
      "activations/layer13_attention_weight_max": 31.23288345336914,
      "activations/layer13_attention_weight_min": -26.666227340698242,
      "activations/layer14_attention_weight_max": 36.39839553833008,
      "activations/layer14_attention_weight_min": -31.27445411682129,
      "activations/layer15_attention_weight_max": 33.81800079345703,
      "activations/layer15_attention_weight_min": -33.243804931640625,
      "activations/layer16_attention_weight_max": 34.5905647277832,
      "activations/layer16_attention_weight_min": -33.14439010620117,
      "activations/layer17_attention_weight_max": 51.809268951416016,
      "activations/layer17_attention_weight_min": -45.88297653198242,
      "activations/layer18_attention_weight_max": 47.734317779541016,
      "activations/layer18_attention_weight_min": -39.72175598144531,
      "activations/layer19_attention_weight_max": 25.685998916625977,
      "activations/layer19_attention_weight_min": -23.70705795288086,
      "activations/layer1_attention_weight_max": 15.896122932434082,
      "activations/layer1_attention_weight_min": -14.765896797180176,
      "activations/layer20_attention_weight_max": 24.311552047729492,
      "activations/layer20_attention_weight_min": -24.348278045654297,
      "activations/layer21_attention_weight_max": 33.657955169677734,
      "activations/layer21_attention_weight_min": -22.403413772583008,
      "activations/layer22_attention_weight_max": 28.11274528503418,
      "activations/layer22_attention_weight_min": -24.625919342041016,
      "activations/layer23_attention_weight_max": 34.492313385009766,
      "activations/layer23_attention_weight_min": -21.959686279296875,
      "activations/layer2_attention_weight_max": 33.013465881347656,
      "activations/layer2_attention_weight_min": -31.884780883789062,
      "activations/layer3_attention_weight_max": 90.59043884277344,
      "activations/layer3_attention_weight_min": -87.1138916015625,
      "activations/layer4_attention_weight_max": 81.1567611694336,
      "activations/layer4_attention_weight_min": -83.88939666748047,
      "activations/layer5_attention_weight_max": 59.93014144897461,
      "activations/layer5_attention_weight_min": -72.89598083496094,
      "activations/layer6_attention_weight_max": 47.33908462524414,
      "activations/layer6_attention_weight_min": -51.04556655883789,
      "activations/layer7_attention_weight_max": 69.20155334472656,
      "activations/layer7_attention_weight_min": -66.46312713623047,
      "activations/layer8_attention_weight_max": 46.53483581542969,
      "activations/layer8_attention_weight_min": -49.62151336669922,
      "activations/layer9_attention_weight_max": 46.044219970703125,
      "activations/layer9_attention_weight_min": -45.12372970581055,
      "epoch": 20.89,
      "learning_rate": 1.5449242424242424e-05,
      "loss": 2.7147,
      "step": 359500
    },
    {
      "activations/layer0_attention_weight_max": 15.688041687011719,
      "activations/layer0_attention_weight_min": -14.000483512878418,
      "activations/layer10_attention_weight_max": 34.392433166503906,
      "activations/layer10_attention_weight_min": -36.707786560058594,
      "activations/layer11_attention_weight_max": 31.66890525817871,
      "activations/layer11_attention_weight_min": -33.885128021240234,
      "activations/layer12_attention_weight_max": 19.94167709350586,
      "activations/layer12_attention_weight_min": -26.821151733398438,
      "activations/layer13_attention_weight_max": 35.80607604980469,
      "activations/layer13_attention_weight_min": -27.958946228027344,
      "activations/layer14_attention_weight_max": 36.28411865234375,
      "activations/layer14_attention_weight_min": -30.973512649536133,
      "activations/layer15_attention_weight_max": 37.45750045776367,
      "activations/layer15_attention_weight_min": -32.52992248535156,
      "activations/layer16_attention_weight_max": 32.194610595703125,
      "activations/layer16_attention_weight_min": -31.020633697509766,
      "activations/layer17_attention_weight_max": 51.62438201904297,
      "activations/layer17_attention_weight_min": -43.599853515625,
      "activations/layer18_attention_weight_max": 48.70260238647461,
      "activations/layer18_attention_weight_min": -38.472869873046875,
      "activations/layer19_attention_weight_max": 25.022592544555664,
      "activations/layer19_attention_weight_min": -21.19727897644043,
      "activations/layer1_attention_weight_max": 15.772795677185059,
      "activations/layer1_attention_weight_min": -13.121387481689453,
      "activations/layer20_attention_weight_max": 22.506567001342773,
      "activations/layer20_attention_weight_min": -22.42860984802246,
      "activations/layer21_attention_weight_max": 32.7252082824707,
      "activations/layer21_attention_weight_min": -21.14523696899414,
      "activations/layer22_attention_weight_max": 28.732717514038086,
      "activations/layer22_attention_weight_min": -23.41327476501465,
      "activations/layer23_attention_weight_max": 33.933746337890625,
      "activations/layer23_attention_weight_min": -25.482444763183594,
      "activations/layer2_attention_weight_max": 29.367835998535156,
      "activations/layer2_attention_weight_min": -29.66387176513672,
      "activations/layer3_attention_weight_max": 84.2186279296875,
      "activations/layer3_attention_weight_min": -84.962890625,
      "activations/layer4_attention_weight_max": 79.44107818603516,
      "activations/layer4_attention_weight_min": -79.40691375732422,
      "activations/layer5_attention_weight_max": 59.490779876708984,
      "activations/layer5_attention_weight_min": -70.30142211914062,
      "activations/layer6_attention_weight_max": 46.78329849243164,
      "activations/layer6_attention_weight_min": -49.84938430786133,
      "activations/layer7_attention_weight_max": 62.55078887939453,
      "activations/layer7_attention_weight_min": -60.387577056884766,
      "activations/layer8_attention_weight_max": 44.29336929321289,
      "activations/layer8_attention_weight_min": -49.283241271972656,
      "activations/layer9_attention_weight_max": 42.61113357543945,
      "activations/layer9_attention_weight_min": -44.88124084472656,
      "epoch": 20.89,
      "learning_rate": 1.543030303030303e-05,
      "loss": 2.7168,
      "step": 359550
    },
    {
      "activations/layer0_attention_weight_max": 15.781312942504883,
      "activations/layer0_attention_weight_min": -13.929369926452637,
      "activations/layer10_attention_weight_max": 32.67255401611328,
      "activations/layer10_attention_weight_min": -34.350616455078125,
      "activations/layer11_attention_weight_max": 30.521942138671875,
      "activations/layer11_attention_weight_min": -36.968414306640625,
      "activations/layer12_attention_weight_max": 19.642189025878906,
      "activations/layer12_attention_weight_min": -25.80893325805664,
      "activations/layer13_attention_weight_max": 33.17444610595703,
      "activations/layer13_attention_weight_min": -26.46413803100586,
      "activations/layer14_attention_weight_max": 35.193809509277344,
      "activations/layer14_attention_weight_min": -31.642122268676758,
      "activations/layer15_attention_weight_max": 33.77848815917969,
      "activations/layer15_attention_weight_min": -32.46678924560547,
      "activations/layer16_attention_weight_max": 33.04929733276367,
      "activations/layer16_attention_weight_min": -32.46487808227539,
      "activations/layer17_attention_weight_max": 50.34074401855469,
      "activations/layer17_attention_weight_min": -44.99667739868164,
      "activations/layer18_attention_weight_max": 47.93306350708008,
      "activations/layer18_attention_weight_min": -38.70255661010742,
      "activations/layer19_attention_weight_max": 27.465391159057617,
      "activations/layer19_attention_weight_min": -22.475309371948242,
      "activations/layer1_attention_weight_max": 15.840299606323242,
      "activations/layer1_attention_weight_min": -14.118786811828613,
      "activations/layer20_attention_weight_max": 24.122167587280273,
      "activations/layer20_attention_weight_min": -21.695335388183594,
      "activations/layer21_attention_weight_max": 40.510005950927734,
      "activations/layer21_attention_weight_min": -26.674564361572266,
      "activations/layer22_attention_weight_max": 29.926435470581055,
      "activations/layer22_attention_weight_min": -25.454715728759766,
      "activations/layer23_attention_weight_max": 35.262939453125,
      "activations/layer23_attention_weight_min": -25.833219528198242,
      "activations/layer2_attention_weight_max": 32.507991790771484,
      "activations/layer2_attention_weight_min": -29.897998809814453,
      "activations/layer3_attention_weight_max": 86.6458969116211,
      "activations/layer3_attention_weight_min": -86.034423828125,
      "activations/layer4_attention_weight_max": 86.05609130859375,
      "activations/layer4_attention_weight_min": -83.27995300292969,
      "activations/layer5_attention_weight_max": 61.836116790771484,
      "activations/layer5_attention_weight_min": -74.55377197265625,
      "activations/layer6_attention_weight_max": 48.67939376831055,
      "activations/layer6_attention_weight_min": -48.954837799072266,
      "activations/layer7_attention_weight_max": 68.7642822265625,
      "activations/layer7_attention_weight_min": -70.54754638671875,
      "activations/layer8_attention_weight_max": 45.95853805541992,
      "activations/layer8_attention_weight_min": -49.60553741455078,
      "activations/layer9_attention_weight_max": 41.59989547729492,
      "activations/layer9_attention_weight_min": -44.13496017456055,
      "epoch": 20.89,
      "learning_rate": 1.5411363636363634e-05,
      "loss": 2.7282,
      "step": 359600
    },
    {
      "activations/layer0_attention_weight_max": 16.099042892456055,
      "activations/layer0_attention_weight_min": -13.801663398742676,
      "activations/layer10_attention_weight_max": 34.85546875,
      "activations/layer10_attention_weight_min": -35.27438735961914,
      "activations/layer11_attention_weight_max": 31.82831382751465,
      "activations/layer11_attention_weight_min": -33.250308990478516,
      "activations/layer12_attention_weight_max": 20.070520401000977,
      "activations/layer12_attention_weight_min": -23.799028396606445,
      "activations/layer13_attention_weight_max": 34.35594177246094,
      "activations/layer13_attention_weight_min": -28.370071411132812,
      "activations/layer14_attention_weight_max": 39.31877136230469,
      "activations/layer14_attention_weight_min": -31.218727111816406,
      "activations/layer15_attention_weight_max": 35.569427490234375,
      "activations/layer15_attention_weight_min": -31.191730499267578,
      "activations/layer16_attention_weight_max": 36.613189697265625,
      "activations/layer16_attention_weight_min": -32.749122619628906,
      "activations/layer17_attention_weight_max": 54.33739471435547,
      "activations/layer17_attention_weight_min": -46.362674713134766,
      "activations/layer18_attention_weight_max": 48.32411193847656,
      "activations/layer18_attention_weight_min": -41.38047409057617,
      "activations/layer19_attention_weight_max": 25.639493942260742,
      "activations/layer19_attention_weight_min": -25.26593589782715,
      "activations/layer1_attention_weight_max": 15.0712308883667,
      "activations/layer1_attention_weight_min": -14.00495433807373,
      "activations/layer20_attention_weight_max": 24.390148162841797,
      "activations/layer20_attention_weight_min": -23.022571563720703,
      "activations/layer21_attention_weight_max": 34.858673095703125,
      "activations/layer21_attention_weight_min": -22.53444480895996,
      "activations/layer22_attention_weight_max": 29.122055053710938,
      "activations/layer22_attention_weight_min": -25.8773136138916,
      "activations/layer23_attention_weight_max": 39.86376953125,
      "activations/layer23_attention_weight_min": -25.58513641357422,
      "activations/layer2_attention_weight_max": 31.053592681884766,
      "activations/layer2_attention_weight_min": -29.339508056640625,
      "activations/layer3_attention_weight_max": 87.5636978149414,
      "activations/layer3_attention_weight_min": -85.10757446289062,
      "activations/layer4_attention_weight_max": 85.96989440917969,
      "activations/layer4_attention_weight_min": -84.66474151611328,
      "activations/layer5_attention_weight_max": 59.94627380371094,
      "activations/layer5_attention_weight_min": -70.3064193725586,
      "activations/layer6_attention_weight_max": 48.48705291748047,
      "activations/layer6_attention_weight_min": -49.61594772338867,
      "activations/layer7_attention_weight_max": 63.14067840576172,
      "activations/layer7_attention_weight_min": -66.32032012939453,
      "activations/layer8_attention_weight_max": 46.15629959106445,
      "activations/layer8_attention_weight_min": -51.59769058227539,
      "activations/layer9_attention_weight_max": 41.913455963134766,
      "activations/layer9_attention_weight_min": -45.69729232788086,
      "epoch": 20.9,
      "learning_rate": 1.5392424242424242e-05,
      "loss": 2.7163,
      "step": 359650
    },
    {
      "activations/layer0_attention_weight_max": 15.836639404296875,
      "activations/layer0_attention_weight_min": -13.941608428955078,
      "activations/layer10_attention_weight_max": 38.72358703613281,
      "activations/layer10_attention_weight_min": -38.460426330566406,
      "activations/layer11_attention_weight_max": 38.125213623046875,
      "activations/layer11_attention_weight_min": -35.378684997558594,
      "activations/layer12_attention_weight_max": 21.51968765258789,
      "activations/layer12_attention_weight_min": -27.109590530395508,
      "activations/layer13_attention_weight_max": 37.08588409423828,
      "activations/layer13_attention_weight_min": -28.924999237060547,
      "activations/layer14_attention_weight_max": 38.58245086669922,
      "activations/layer14_attention_weight_min": -31.847841262817383,
      "activations/layer15_attention_weight_max": 36.396732330322266,
      "activations/layer15_attention_weight_min": -31.376916885375977,
      "activations/layer16_attention_weight_max": 38.740020751953125,
      "activations/layer16_attention_weight_min": -37.22296905517578,
      "activations/layer17_attention_weight_max": 60.04344940185547,
      "activations/layer17_attention_weight_min": -49.27724075317383,
      "activations/layer18_attention_weight_max": 51.64795684814453,
      "activations/layer18_attention_weight_min": -42.119964599609375,
      "activations/layer19_attention_weight_max": 24.724340438842773,
      "activations/layer19_attention_weight_min": -21.619050979614258,
      "activations/layer1_attention_weight_max": 15.006268501281738,
      "activations/layer1_attention_weight_min": -13.107503890991211,
      "activations/layer20_attention_weight_max": 23.62244415283203,
      "activations/layer20_attention_weight_min": -21.121856689453125,
      "activations/layer21_attention_weight_max": 36.27475357055664,
      "activations/layer21_attention_weight_min": -19.18599510192871,
      "activations/layer22_attention_weight_max": 30.007720947265625,
      "activations/layer22_attention_weight_min": -24.46212387084961,
      "activations/layer23_attention_weight_max": 37.69752502441406,
      "activations/layer23_attention_weight_min": -23.034902572631836,
      "activations/layer2_attention_weight_max": 31.38248062133789,
      "activations/layer2_attention_weight_min": -30.192394256591797,
      "activations/layer3_attention_weight_max": 91.04934692382812,
      "activations/layer3_attention_weight_min": -87.27190399169922,
      "activations/layer4_attention_weight_max": 85.42334747314453,
      "activations/layer4_attention_weight_min": -81.96133422851562,
      "activations/layer5_attention_weight_max": 61.43330383300781,
      "activations/layer5_attention_weight_min": -65.58654022216797,
      "activations/layer6_attention_weight_max": 50.64046859741211,
      "activations/layer6_attention_weight_min": -50.05164337158203,
      "activations/layer7_attention_weight_max": 71.33757019042969,
      "activations/layer7_attention_weight_min": -67.49317169189453,
      "activations/layer8_attention_weight_max": 51.52649688720703,
      "activations/layer8_attention_weight_min": -51.04789733886719,
      "activations/layer9_attention_weight_max": 54.517269134521484,
      "activations/layer9_attention_weight_min": -51.15045166015625,
      "epoch": 20.9,
      "learning_rate": 1.5373484848484847e-05,
      "loss": 2.7027,
      "step": 359700
    },
    {
      "activations/layer0_attention_weight_max": 15.694427490234375,
      "activations/layer0_attention_weight_min": -14.072822570800781,
      "activations/layer10_attention_weight_max": 36.42473602294922,
      "activations/layer10_attention_weight_min": -36.99312210083008,
      "activations/layer11_attention_weight_max": 32.221099853515625,
      "activations/layer11_attention_weight_min": -35.624534606933594,
      "activations/layer12_attention_weight_max": 19.08751678466797,
      "activations/layer12_attention_weight_min": -23.65178680419922,
      "activations/layer13_attention_weight_max": 35.62738800048828,
      "activations/layer13_attention_weight_min": -28.16550636291504,
      "activations/layer14_attention_weight_max": 35.305824279785156,
      "activations/layer14_attention_weight_min": -30.09995460510254,
      "activations/layer15_attention_weight_max": 31.162071228027344,
      "activations/layer15_attention_weight_min": -28.620447158813477,
      "activations/layer16_attention_weight_max": 33.61503219604492,
      "activations/layer16_attention_weight_min": -31.300092697143555,
      "activations/layer17_attention_weight_max": 51.134925842285156,
      "activations/layer17_attention_weight_min": -44.88905715942383,
      "activations/layer18_attention_weight_max": 46.38058853149414,
      "activations/layer18_attention_weight_min": -38.26549530029297,
      "activations/layer19_attention_weight_max": 23.998432159423828,
      "activations/layer19_attention_weight_min": -21.129941940307617,
      "activations/layer1_attention_weight_max": 15.178933143615723,
      "activations/layer1_attention_weight_min": -13.609628677368164,
      "activations/layer20_attention_weight_max": 21.61587142944336,
      "activations/layer20_attention_weight_min": -22.33978843688965,
      "activations/layer21_attention_weight_max": 33.437461853027344,
      "activations/layer21_attention_weight_min": -22.39514923095703,
      "activations/layer22_attention_weight_max": 30.17851448059082,
      "activations/layer22_attention_weight_min": -25.02867317199707,
      "activations/layer23_attention_weight_max": 36.5150146484375,
      "activations/layer23_attention_weight_min": -22.6735897064209,
      "activations/layer2_attention_weight_max": 30.018009185791016,
      "activations/layer2_attention_weight_min": -29.930118560791016,
      "activations/layer3_attention_weight_max": 85.01280212402344,
      "activations/layer3_attention_weight_min": -86.97647094726562,
      "activations/layer4_attention_weight_max": 80.97405242919922,
      "activations/layer4_attention_weight_min": -82.9231185913086,
      "activations/layer5_attention_weight_max": 58.993995666503906,
      "activations/layer5_attention_weight_min": -69.85420227050781,
      "activations/layer6_attention_weight_max": 47.33529281616211,
      "activations/layer6_attention_weight_min": -48.85051727294922,
      "activations/layer7_attention_weight_max": 64.68441772460938,
      "activations/layer7_attention_weight_min": -63.979347229003906,
      "activations/layer8_attention_weight_max": 46.252803802490234,
      "activations/layer8_attention_weight_min": -53.775611877441406,
      "activations/layer9_attention_weight_max": 45.6143913269043,
      "activations/layer9_attention_weight_min": -48.791160583496094,
      "epoch": 20.9,
      "learning_rate": 1.5354545454545452e-05,
      "loss": 2.6941,
      "step": 359750
    },
    {
      "activations/layer0_attention_weight_max": 16.07931900024414,
      "activations/layer0_attention_weight_min": -13.83974838256836,
      "activations/layer10_attention_weight_max": 36.49089431762695,
      "activations/layer10_attention_weight_min": -35.724273681640625,
      "activations/layer11_attention_weight_max": 32.54344940185547,
      "activations/layer11_attention_weight_min": -35.226318359375,
      "activations/layer12_attention_weight_max": 19.544986724853516,
      "activations/layer12_attention_weight_min": -27.536344528198242,
      "activations/layer13_attention_weight_max": 36.47743225097656,
      "activations/layer13_attention_weight_min": -29.48365020751953,
      "activations/layer14_attention_weight_max": 35.293243408203125,
      "activations/layer14_attention_weight_min": -29.03563117980957,
      "activations/layer15_attention_weight_max": 32.26165008544922,
      "activations/layer15_attention_weight_min": -29.91971206665039,
      "activations/layer16_attention_weight_max": 32.41877746582031,
      "activations/layer16_attention_weight_min": -32.232505798339844,
      "activations/layer17_attention_weight_max": 51.609893798828125,
      "activations/layer17_attention_weight_min": -44.37514114379883,
      "activations/layer18_attention_weight_max": 44.38302993774414,
      "activations/layer18_attention_weight_min": -37.97163391113281,
      "activations/layer19_attention_weight_max": 26.148502349853516,
      "activations/layer19_attention_weight_min": -21.72503662109375,
      "activations/layer1_attention_weight_max": 14.732794761657715,
      "activations/layer1_attention_weight_min": -13.485217094421387,
      "activations/layer20_attention_weight_max": 21.207246780395508,
      "activations/layer20_attention_weight_min": -23.93218994140625,
      "activations/layer21_attention_weight_max": 31.22772216796875,
      "activations/layer21_attention_weight_min": -19.793134689331055,
      "activations/layer22_attention_weight_max": 34.77911376953125,
      "activations/layer22_attention_weight_min": -25.56121063232422,
      "activations/layer23_attention_weight_max": 36.98847961425781,
      "activations/layer23_attention_weight_min": -24.168020248413086,
      "activations/layer2_attention_weight_max": 31.094810485839844,
      "activations/layer2_attention_weight_min": -30.49060821533203,
      "activations/layer3_attention_weight_max": 90.60165405273438,
      "activations/layer3_attention_weight_min": -88.13621520996094,
      "activations/layer4_attention_weight_max": 83.75255584716797,
      "activations/layer4_attention_weight_min": -84.2452621459961,
      "activations/layer5_attention_weight_max": 57.94109344482422,
      "activations/layer5_attention_weight_min": -72.01150512695312,
      "activations/layer6_attention_weight_max": 49.921634674072266,
      "activations/layer6_attention_weight_min": -51.531856536865234,
      "activations/layer7_attention_weight_max": 66.78343963623047,
      "activations/layer7_attention_weight_min": -66.8549575805664,
      "activations/layer8_attention_weight_max": 51.98373794555664,
      "activations/layer8_attention_weight_min": -51.1341667175293,
      "activations/layer9_attention_weight_max": 43.64765167236328,
      "activations/layer9_attention_weight_min": -46.6368293762207,
      "epoch": 20.91,
      "learning_rate": 1.533560606060606e-05,
      "loss": 2.7022,
      "step": 359800
    },
    {
      "activations/layer0_attention_weight_max": 15.28196907043457,
      "activations/layer0_attention_weight_min": -13.86897087097168,
      "activations/layer10_attention_weight_max": 32.824153900146484,
      "activations/layer10_attention_weight_min": -34.1883430480957,
      "activations/layer11_attention_weight_max": 31.985248565673828,
      "activations/layer11_attention_weight_min": -33.17914962768555,
      "activations/layer12_attention_weight_max": 18.09781265258789,
      "activations/layer12_attention_weight_min": -24.209054946899414,
      "activations/layer13_attention_weight_max": 32.02479553222656,
      "activations/layer13_attention_weight_min": -26.473390579223633,
      "activations/layer14_attention_weight_max": 33.62492370605469,
      "activations/layer14_attention_weight_min": -29.61438751220703,
      "activations/layer15_attention_weight_max": 35.41475296020508,
      "activations/layer15_attention_weight_min": -30.26634979248047,
      "activations/layer16_attention_weight_max": 33.9984016418457,
      "activations/layer16_attention_weight_min": -31.548828125,
      "activations/layer17_attention_weight_max": 48.886558532714844,
      "activations/layer17_attention_weight_min": -45.43991470336914,
      "activations/layer18_attention_weight_max": 47.276153564453125,
      "activations/layer18_attention_weight_min": -41.69462203979492,
      "activations/layer19_attention_weight_max": 23.334976196289062,
      "activations/layer19_attention_weight_min": -24.030498504638672,
      "activations/layer1_attention_weight_max": 15.269306182861328,
      "activations/layer1_attention_weight_min": -14.150053977966309,
      "activations/layer20_attention_weight_max": 22.780519485473633,
      "activations/layer20_attention_weight_min": -22.10757064819336,
      "activations/layer21_attention_weight_max": 36.02717971801758,
      "activations/layer21_attention_weight_min": -23.99979591369629,
      "activations/layer22_attention_weight_max": 29.168413162231445,
      "activations/layer22_attention_weight_min": -27.177087783813477,
      "activations/layer23_attention_weight_max": 35.00759506225586,
      "activations/layer23_attention_weight_min": -25.249624252319336,
      "activations/layer2_attention_weight_max": 32.09812927246094,
      "activations/layer2_attention_weight_min": -30.73468017578125,
      "activations/layer3_attention_weight_max": 88.3395004272461,
      "activations/layer3_attention_weight_min": -88.11589050292969,
      "activations/layer4_attention_weight_max": 80.30036163330078,
      "activations/layer4_attention_weight_min": -80.32488250732422,
      "activations/layer5_attention_weight_max": 60.793983459472656,
      "activations/layer5_attention_weight_min": -69.47716522216797,
      "activations/layer6_attention_weight_max": 49.63195037841797,
      "activations/layer6_attention_weight_min": -51.90495300292969,
      "activations/layer7_attention_weight_max": 67.60244750976562,
      "activations/layer7_attention_weight_min": -63.52210235595703,
      "activations/layer8_attention_weight_max": 45.74177932739258,
      "activations/layer8_attention_weight_min": -49.88496780395508,
      "activations/layer9_attention_weight_max": 44.34565353393555,
      "activations/layer9_attention_weight_min": -43.95323181152344,
      "epoch": 20.91,
      "learning_rate": 1.5316666666666665e-05,
      "loss": 2.7153,
      "step": 359850
    },
    {
      "activations/layer0_attention_weight_max": 15.443225860595703,
      "activations/layer0_attention_weight_min": -13.857369422912598,
      "activations/layer10_attention_weight_max": 35.93413543701172,
      "activations/layer10_attention_weight_min": -35.433807373046875,
      "activations/layer11_attention_weight_max": 32.60538101196289,
      "activations/layer11_attention_weight_min": -34.42704772949219,
      "activations/layer12_attention_weight_max": 18.216176986694336,
      "activations/layer12_attention_weight_min": -25.22160530090332,
      "activations/layer13_attention_weight_max": 33.93017578125,
      "activations/layer13_attention_weight_min": -27.627731323242188,
      "activations/layer14_attention_weight_max": 33.58636474609375,
      "activations/layer14_attention_weight_min": -30.591890335083008,
      "activations/layer15_attention_weight_max": 33.18760681152344,
      "activations/layer15_attention_weight_min": -29.86095428466797,
      "activations/layer16_attention_weight_max": 36.2795295715332,
      "activations/layer16_attention_weight_min": -33.51963424682617,
      "activations/layer17_attention_weight_max": 54.171268463134766,
      "activations/layer17_attention_weight_min": -46.80508804321289,
      "activations/layer18_attention_weight_max": 49.05842971801758,
      "activations/layer18_attention_weight_min": -41.46063232421875,
      "activations/layer19_attention_weight_max": 27.010290145874023,
      "activations/layer19_attention_weight_min": -21.95050621032715,
      "activations/layer1_attention_weight_max": 15.413156509399414,
      "activations/layer1_attention_weight_min": -14.985477447509766,
      "activations/layer20_attention_weight_max": 25.07439422607422,
      "activations/layer20_attention_weight_min": -22.11669921875,
      "activations/layer21_attention_weight_max": 32.586891174316406,
      "activations/layer21_attention_weight_min": -20.835739135742188,
      "activations/layer22_attention_weight_max": 32.818111419677734,
      "activations/layer22_attention_weight_min": -27.691335678100586,
      "activations/layer23_attention_weight_max": 40.25723648071289,
      "activations/layer23_attention_weight_min": -24.456144332885742,
      "activations/layer2_attention_weight_max": 31.579174041748047,
      "activations/layer2_attention_weight_min": -30.770763397216797,
      "activations/layer3_attention_weight_max": 87.99571228027344,
      "activations/layer3_attention_weight_min": -90.18132019042969,
      "activations/layer4_attention_weight_max": 84.81233978271484,
      "activations/layer4_attention_weight_min": -82.89714050292969,
      "activations/layer5_attention_weight_max": 62.86164855957031,
      "activations/layer5_attention_weight_min": -77.76457977294922,
      "activations/layer6_attention_weight_max": 50.64241027832031,
      "activations/layer6_attention_weight_min": -51.121376037597656,
      "activations/layer7_attention_weight_max": 66.82665252685547,
      "activations/layer7_attention_weight_min": -65.9803695678711,
      "activations/layer8_attention_weight_max": 50.11867141723633,
      "activations/layer8_attention_weight_min": -51.33290100097656,
      "activations/layer9_attention_weight_max": 43.132171630859375,
      "activations/layer9_attention_weight_min": -46.47841262817383,
      "epoch": 20.91,
      "learning_rate": 1.529772727272727e-05,
      "loss": 2.7257,
      "step": 359900
    },
    {
      "activations/layer0_attention_weight_max": 15.320466995239258,
      "activations/layer0_attention_weight_min": -14.41254997253418,
      "activations/layer10_attention_weight_max": 36.67603302001953,
      "activations/layer10_attention_weight_min": -36.474056243896484,
      "activations/layer11_attention_weight_max": 35.541297912597656,
      "activations/layer11_attention_weight_min": -38.388675689697266,
      "activations/layer12_attention_weight_max": 18.553773880004883,
      "activations/layer12_attention_weight_min": -27.524965286254883,
      "activations/layer13_attention_weight_max": 34.13170623779297,
      "activations/layer13_attention_weight_min": -27.241167068481445,
      "activations/layer14_attention_weight_max": 33.96221160888672,
      "activations/layer14_attention_weight_min": -30.48102378845215,
      "activations/layer15_attention_weight_max": 32.15974044799805,
      "activations/layer15_attention_weight_min": -31.257448196411133,
      "activations/layer16_attention_weight_max": 32.83354949951172,
      "activations/layer16_attention_weight_min": -31.44530487060547,
      "activations/layer17_attention_weight_max": 49.437522888183594,
      "activations/layer17_attention_weight_min": -44.4451789855957,
      "activations/layer18_attention_weight_max": 47.5405387878418,
      "activations/layer18_attention_weight_min": -42.7539176940918,
      "activations/layer19_attention_weight_max": 23.612489700317383,
      "activations/layer19_attention_weight_min": -23.28417205810547,
      "activations/layer1_attention_weight_max": 15.331720352172852,
      "activations/layer1_attention_weight_min": -14.3240327835083,
      "activations/layer20_attention_weight_max": 23.109371185302734,
      "activations/layer20_attention_weight_min": -21.98431968688965,
      "activations/layer21_attention_weight_max": 36.96381759643555,
      "activations/layer21_attention_weight_min": -22.975067138671875,
      "activations/layer22_attention_weight_max": 28.4126033782959,
      "activations/layer22_attention_weight_min": -25.458404541015625,
      "activations/layer23_attention_weight_max": 35.25569152832031,
      "activations/layer23_attention_weight_min": -24.42095947265625,
      "activations/layer2_attention_weight_max": 32.66321563720703,
      "activations/layer2_attention_weight_min": -30.374242782592773,
      "activations/layer3_attention_weight_max": 90.417236328125,
      "activations/layer3_attention_weight_min": -87.33233642578125,
      "activations/layer4_attention_weight_max": 84.67338562011719,
      "activations/layer4_attention_weight_min": -79.95277404785156,
      "activations/layer5_attention_weight_max": 61.80675506591797,
      "activations/layer5_attention_weight_min": -65.91584014892578,
      "activations/layer6_attention_weight_max": 49.07376480102539,
      "activations/layer6_attention_weight_min": -50.02006149291992,
      "activations/layer7_attention_weight_max": 67.73658752441406,
      "activations/layer7_attention_weight_min": -68.46658325195312,
      "activations/layer8_attention_weight_max": 48.65370559692383,
      "activations/layer8_attention_weight_min": -51.6170539855957,
      "activations/layer9_attention_weight_max": 49.91634750366211,
      "activations/layer9_attention_weight_min": -44.538692474365234,
      "epoch": 20.92,
      "learning_rate": 1.5278787878787878e-05,
      "loss": 2.7295,
      "step": 359950
    },
    {
      "activations/layer0_attention_weight_max": 16.199949264526367,
      "activations/layer0_attention_weight_min": -13.866470336914062,
      "activations/layer10_attention_weight_max": 37.302955627441406,
      "activations/layer10_attention_weight_min": -35.28521728515625,
      "activations/layer11_attention_weight_max": 33.79329299926758,
      "activations/layer11_attention_weight_min": -34.78709411621094,
      "activations/layer12_attention_weight_max": 19.613521575927734,
      "activations/layer12_attention_weight_min": -31.962936401367188,
      "activations/layer13_attention_weight_max": 33.6326904296875,
      "activations/layer13_attention_weight_min": -29.348474502563477,
      "activations/layer14_attention_weight_max": 36.88325500488281,
      "activations/layer14_attention_weight_min": -33.264488220214844,
      "activations/layer15_attention_weight_max": 35.63931655883789,
      "activations/layer15_attention_weight_min": -33.63817596435547,
      "activations/layer16_attention_weight_max": 38.153236389160156,
      "activations/layer16_attention_weight_min": -35.17714309692383,
      "activations/layer17_attention_weight_max": 53.9729118347168,
      "activations/layer17_attention_weight_min": -49.46775436401367,
      "activations/layer18_attention_weight_max": 52.32217788696289,
      "activations/layer18_attention_weight_min": -41.05840301513672,
      "activations/layer19_attention_weight_max": 25.803268432617188,
      "activations/layer19_attention_weight_min": -24.272109985351562,
      "activations/layer1_attention_weight_max": 16.46531867980957,
      "activations/layer1_attention_weight_min": -14.591552734375,
      "activations/layer20_attention_weight_max": 24.782798767089844,
      "activations/layer20_attention_weight_min": -22.61992645263672,
      "activations/layer21_attention_weight_max": 39.745361328125,
      "activations/layer21_attention_weight_min": -21.967348098754883,
      "activations/layer22_attention_weight_max": 29.314640045166016,
      "activations/layer22_attention_weight_min": -27.148035049438477,
      "activations/layer23_attention_weight_max": 34.066375732421875,
      "activations/layer23_attention_weight_min": -25.5697021484375,
      "activations/layer2_attention_weight_max": 33.86024475097656,
      "activations/layer2_attention_weight_min": -31.03106689453125,
      "activations/layer3_attention_weight_max": 90.67951965332031,
      "activations/layer3_attention_weight_min": -87.25602722167969,
      "activations/layer4_attention_weight_max": 87.85282135009766,
      "activations/layer4_attention_weight_min": -85.73828887939453,
      "activations/layer5_attention_weight_max": 59.8596305847168,
      "activations/layer5_attention_weight_min": -75.48600769042969,
      "activations/layer6_attention_weight_max": 49.4262809753418,
      "activations/layer6_attention_weight_min": -51.09675979614258,
      "activations/layer7_attention_weight_max": 66.7909164428711,
      "activations/layer7_attention_weight_min": -67.4195327758789,
      "activations/layer8_attention_weight_max": 48.47006607055664,
      "activations/layer8_attention_weight_min": -50.426727294921875,
      "activations/layer9_attention_weight_max": 46.59740447998047,
      "activations/layer9_attention_weight_min": -45.461891174316406,
      "epoch": 20.92,
      "learning_rate": 1.5259848484848483e-05,
      "loss": 2.7319,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4477,
      "eval_samples_per_second": 508.303,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4477,
      "eval_openwebtext_samples_per_second": 508.303,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.98,
      "eval_wikitext_samples_per_second": 230.305,
      "step": 360000
    },
    {
      "epoch": 20.92,
      "eval_lambada_loss": 2.453125,
      "eval_lambada_ppl": 11.624616945432633,
      "eval_lambada_runtime": 9.5809,
      "eval_lambada_samples_per_second": 508.197,
      "step": 360000
    },
    {
      "activations/layer0_attention_weight_max": 15.550482749938965,
      "activations/layer0_attention_weight_min": -13.931078910827637,
      "activations/layer10_attention_weight_max": 34.51505661010742,
      "activations/layer10_attention_weight_min": -36.76523971557617,
      "activations/layer11_attention_weight_max": 32.569664001464844,
      "activations/layer11_attention_weight_min": -33.8985595703125,
      "activations/layer12_attention_weight_max": 18.16892433166504,
      "activations/layer12_attention_weight_min": -27.162351608276367,
      "activations/layer13_attention_weight_max": 35.61410903930664,
      "activations/layer13_attention_weight_min": -27.44808578491211,
      "activations/layer14_attention_weight_max": 36.84423065185547,
      "activations/layer14_attention_weight_min": -28.714332580566406,
      "activations/layer15_attention_weight_max": 34.377532958984375,
      "activations/layer15_attention_weight_min": -29.75424575805664,
      "activations/layer16_attention_weight_max": 34.86867904663086,
      "activations/layer16_attention_weight_min": -31.62491798400879,
      "activations/layer17_attention_weight_max": 53.68867111206055,
      "activations/layer17_attention_weight_min": -43.765132904052734,
      "activations/layer18_attention_weight_max": 45.29930877685547,
      "activations/layer18_attention_weight_min": -38.53715515136719,
      "activations/layer19_attention_weight_max": 26.06587028503418,
      "activations/layer19_attention_weight_min": -21.597328186035156,
      "activations/layer1_attention_weight_max": 15.848759651184082,
      "activations/layer1_attention_weight_min": -15.057794570922852,
      "activations/layer20_attention_weight_max": 26.278526306152344,
      "activations/layer20_attention_weight_min": -21.895858764648438,
      "activations/layer21_attention_weight_max": 34.706298828125,
      "activations/layer21_attention_weight_min": -21.572546005249023,
      "activations/layer22_attention_weight_max": 28.050251007080078,
      "activations/layer22_attention_weight_min": -25.822025299072266,
      "activations/layer23_attention_weight_max": 35.578895568847656,
      "activations/layer23_attention_weight_min": -22.970409393310547,
      "activations/layer2_attention_weight_max": 33.94941711425781,
      "activations/layer2_attention_weight_min": -33.96314239501953,
      "activations/layer3_attention_weight_max": 88.50227355957031,
      "activations/layer3_attention_weight_min": -90.04755401611328,
      "activations/layer4_attention_weight_max": 87.17630004882812,
      "activations/layer4_attention_weight_min": -88.16312408447266,
      "activations/layer5_attention_weight_max": 62.04064178466797,
      "activations/layer5_attention_weight_min": -69.2307357788086,
      "activations/layer6_attention_weight_max": 51.066429138183594,
      "activations/layer6_attention_weight_min": -51.95616149902344,
      "activations/layer7_attention_weight_max": 64.96263122558594,
      "activations/layer7_attention_weight_min": -67.29898834228516,
      "activations/layer8_attention_weight_max": 44.34960174560547,
      "activations/layer8_attention_weight_min": -48.87578582763672,
      "activations/layer9_attention_weight_max": 46.480926513671875,
      "activations/layer9_attention_weight_min": -45.654300689697266,
      "epoch": 20.92,
      "learning_rate": 1.524090909090909e-05,
      "loss": 2.6998,
      "step": 360050
    },
    {
      "activations/layer0_attention_weight_max": 15.151544570922852,
      "activations/layer0_attention_weight_min": -14.056835174560547,
      "activations/layer10_attention_weight_max": 34.880496978759766,
      "activations/layer10_attention_weight_min": -36.8447265625,
      "activations/layer11_attention_weight_max": 31.677947998046875,
      "activations/layer11_attention_weight_min": -34.885772705078125,
      "activations/layer12_attention_weight_max": 19.144344329833984,
      "activations/layer12_attention_weight_min": -34.02587127685547,
      "activations/layer13_attention_weight_max": 34.93404006958008,
      "activations/layer13_attention_weight_min": -29.752756118774414,
      "activations/layer14_attention_weight_max": 40.171775817871094,
      "activations/layer14_attention_weight_min": -32.35081481933594,
      "activations/layer15_attention_weight_max": 36.67231750488281,
      "activations/layer15_attention_weight_min": -31.72850227355957,
      "activations/layer16_attention_weight_max": 37.4792594909668,
      "activations/layer16_attention_weight_min": -36.5622673034668,
      "activations/layer17_attention_weight_max": 56.5057258605957,
      "activations/layer17_attention_weight_min": -50.0080680847168,
      "activations/layer18_attention_weight_max": 52.4517936706543,
      "activations/layer18_attention_weight_min": -42.26711654663086,
      "activations/layer19_attention_weight_max": 25.67189598083496,
      "activations/layer19_attention_weight_min": -23.509849548339844,
      "activations/layer1_attention_weight_max": 15.222234725952148,
      "activations/layer1_attention_weight_min": -14.439336776733398,
      "activations/layer20_attention_weight_max": 24.02324104309082,
      "activations/layer20_attention_weight_min": -22.134258270263672,
      "activations/layer21_attention_weight_max": 34.26871871948242,
      "activations/layer21_attention_weight_min": -23.322141647338867,
      "activations/layer22_attention_weight_max": 30.091604232788086,
      "activations/layer22_attention_weight_min": -27.538801193237305,
      "activations/layer23_attention_weight_max": 37.647403717041016,
      "activations/layer23_attention_weight_min": -29.381309509277344,
      "activations/layer2_attention_weight_max": 29.536359786987305,
      "activations/layer2_attention_weight_min": -30.385480880737305,
      "activations/layer3_attention_weight_max": 84.19728088378906,
      "activations/layer3_attention_weight_min": -88.16670227050781,
      "activations/layer4_attention_weight_max": 85.07616424560547,
      "activations/layer4_attention_weight_min": -83.17504119873047,
      "activations/layer5_attention_weight_max": 64.2474594116211,
      "activations/layer5_attention_weight_min": -73.2947998046875,
      "activations/layer6_attention_weight_max": 50.43323516845703,
      "activations/layer6_attention_weight_min": -49.322757720947266,
      "activations/layer7_attention_weight_max": 68.62202453613281,
      "activations/layer7_attention_weight_min": -68.34444427490234,
      "activations/layer8_attention_weight_max": 46.995845794677734,
      "activations/layer8_attention_weight_min": -50.1502799987793,
      "activations/layer9_attention_weight_max": 42.80955123901367,
      "activations/layer9_attention_weight_min": -48.111724853515625,
      "epoch": 20.92,
      "learning_rate": 1.5221969696969695e-05,
      "loss": 2.7248,
      "step": 360100
    },
    {
      "activations/layer0_attention_weight_max": 15.597220420837402,
      "activations/layer0_attention_weight_min": -14.06952953338623,
      "activations/layer10_attention_weight_max": 34.725196838378906,
      "activations/layer10_attention_weight_min": -38.02295684814453,
      "activations/layer11_attention_weight_max": 32.251121520996094,
      "activations/layer11_attention_weight_min": -37.489654541015625,
      "activations/layer12_attention_weight_max": 20.16814613342285,
      "activations/layer12_attention_weight_min": -26.445621490478516,
      "activations/layer13_attention_weight_max": 37.16536331176758,
      "activations/layer13_attention_weight_min": -28.020174026489258,
      "activations/layer14_attention_weight_max": 36.32612609863281,
      "activations/layer14_attention_weight_min": -30.357511520385742,
      "activations/layer15_attention_weight_max": 34.693878173828125,
      "activations/layer15_attention_weight_min": -30.92711639404297,
      "activations/layer16_attention_weight_max": 34.964290618896484,
      "activations/layer16_attention_weight_min": -32.224761962890625,
      "activations/layer17_attention_weight_max": 53.57146072387695,
      "activations/layer17_attention_weight_min": -45.37604904174805,
      "activations/layer18_attention_weight_max": 49.19347381591797,
      "activations/layer18_attention_weight_min": -39.965274810791016,
      "activations/layer19_attention_weight_max": 24.493200302124023,
      "activations/layer19_attention_weight_min": -23.64972496032715,
      "activations/layer1_attention_weight_max": 15.011211395263672,
      "activations/layer1_attention_weight_min": -13.761844635009766,
      "activations/layer20_attention_weight_max": 23.701147079467773,
      "activations/layer20_attention_weight_min": -21.116636276245117,
      "activations/layer21_attention_weight_max": 41.24319839477539,
      "activations/layer21_attention_weight_min": -24.584739685058594,
      "activations/layer22_attention_weight_max": 31.443458557128906,
      "activations/layer22_attention_weight_min": -25.66238021850586,
      "activations/layer23_attention_weight_max": 36.33161544799805,
      "activations/layer23_attention_weight_min": -23.844078063964844,
      "activations/layer2_attention_weight_max": 31.862789154052734,
      "activations/layer2_attention_weight_min": -29.921354293823242,
      "activations/layer3_attention_weight_max": 90.72233581542969,
      "activations/layer3_attention_weight_min": -92.0327377319336,
      "activations/layer4_attention_weight_max": 88.72750854492188,
      "activations/layer4_attention_weight_min": -87.0373306274414,
      "activations/layer5_attention_weight_max": 63.146480560302734,
      "activations/layer5_attention_weight_min": -75.46223449707031,
      "activations/layer6_attention_weight_max": 54.72447967529297,
      "activations/layer6_attention_weight_min": -53.58650588989258,
      "activations/layer7_attention_weight_max": 68.232666015625,
      "activations/layer7_attention_weight_min": -67.82038116455078,
      "activations/layer8_attention_weight_max": 48.068077087402344,
      "activations/layer8_attention_weight_min": -52.66511917114258,
      "activations/layer9_attention_weight_max": 44.9608268737793,
      "activations/layer9_attention_weight_min": -47.96015548706055,
      "epoch": 20.93,
      "learning_rate": 1.5203030303030301e-05,
      "loss": 2.7197,
      "step": 360150
    },
    {
      "activations/layer0_attention_weight_max": 15.070051193237305,
      "activations/layer0_attention_weight_min": -13.886102676391602,
      "activations/layer10_attention_weight_max": 36.13629913330078,
      "activations/layer10_attention_weight_min": -36.14323806762695,
      "activations/layer11_attention_weight_max": 33.278106689453125,
      "activations/layer11_attention_weight_min": -35.31268310546875,
      "activations/layer12_attention_weight_max": 18.64704704284668,
      "activations/layer12_attention_weight_min": -26.430322647094727,
      "activations/layer13_attention_weight_max": 31.543502807617188,
      "activations/layer13_attention_weight_min": -28.432710647583008,
      "activations/layer14_attention_weight_max": 37.3785285949707,
      "activations/layer14_attention_weight_min": -29.65351104736328,
      "activations/layer15_attention_weight_max": 35.58029556274414,
      "activations/layer15_attention_weight_min": -30.085708618164062,
      "activations/layer16_attention_weight_max": 38.696781158447266,
      "activations/layer16_attention_weight_min": -34.37979507446289,
      "activations/layer17_attention_weight_max": 54.50862121582031,
      "activations/layer17_attention_weight_min": -48.600582122802734,
      "activations/layer18_attention_weight_max": 49.530006408691406,
      "activations/layer18_attention_weight_min": -40.55949020385742,
      "activations/layer19_attention_weight_max": 25.841007232666016,
      "activations/layer19_attention_weight_min": -24.327226638793945,
      "activations/layer1_attention_weight_max": 15.20287036895752,
      "activations/layer1_attention_weight_min": -12.960783004760742,
      "activations/layer20_attention_weight_max": 21.923542022705078,
      "activations/layer20_attention_weight_min": -21.10186195373535,
      "activations/layer21_attention_weight_max": 37.97240447998047,
      "activations/layer21_attention_weight_min": -23.75840187072754,
      "activations/layer22_attention_weight_max": 29.34307861328125,
      "activations/layer22_attention_weight_min": -26.208621978759766,
      "activations/layer23_attention_weight_max": 38.07753372192383,
      "activations/layer23_attention_weight_min": -23.01929473876953,
      "activations/layer2_attention_weight_max": 31.558765411376953,
      "activations/layer2_attention_weight_min": -29.795143127441406,
      "activations/layer3_attention_weight_max": 86.8868408203125,
      "activations/layer3_attention_weight_min": -88.67623138427734,
      "activations/layer4_attention_weight_max": 83.54269409179688,
      "activations/layer4_attention_weight_min": -84.5482177734375,
      "activations/layer5_attention_weight_max": 60.52722930908203,
      "activations/layer5_attention_weight_min": -74.7044677734375,
      "activations/layer6_attention_weight_max": 47.90846633911133,
      "activations/layer6_attention_weight_min": -48.66938400268555,
      "activations/layer7_attention_weight_max": 66.05204010009766,
      "activations/layer7_attention_weight_min": -65.61079406738281,
      "activations/layer8_attention_weight_max": 47.03107452392578,
      "activations/layer8_attention_weight_min": -50.98672103881836,
      "activations/layer9_attention_weight_max": 44.02813720703125,
      "activations/layer9_attention_weight_min": -45.659420013427734,
      "epoch": 20.93,
      "learning_rate": 1.5184090909090906e-05,
      "loss": 2.7136,
      "step": 360200
    },
    {
      "activations/layer0_attention_weight_max": 15.32690715789795,
      "activations/layer0_attention_weight_min": -13.984536170959473,
      "activations/layer10_attention_weight_max": 38.626651763916016,
      "activations/layer10_attention_weight_min": -37.53848648071289,
      "activations/layer11_attention_weight_max": 33.18764877319336,
      "activations/layer11_attention_weight_min": -37.41107940673828,
      "activations/layer12_attention_weight_max": 18.60126304626465,
      "activations/layer12_attention_weight_min": -25.020675659179688,
      "activations/layer13_attention_weight_max": 35.11463165283203,
      "activations/layer13_attention_weight_min": -27.761995315551758,
      "activations/layer14_attention_weight_max": 37.79999923706055,
      "activations/layer14_attention_weight_min": -29.727558135986328,
      "activations/layer15_attention_weight_max": 33.34934616088867,
      "activations/layer15_attention_weight_min": -29.568923950195312,
      "activations/layer16_attention_weight_max": 34.64930725097656,
      "activations/layer16_attention_weight_min": -31.580867767333984,
      "activations/layer17_attention_weight_max": 51.92668533325195,
      "activations/layer17_attention_weight_min": -46.57807540893555,
      "activations/layer18_attention_weight_max": 46.657684326171875,
      "activations/layer18_attention_weight_min": -38.793025970458984,
      "activations/layer19_attention_weight_max": 23.2424373626709,
      "activations/layer19_attention_weight_min": -21.50139808654785,
      "activations/layer1_attention_weight_max": 15.383970260620117,
      "activations/layer1_attention_weight_min": -13.349261283874512,
      "activations/layer20_attention_weight_max": 21.580900192260742,
      "activations/layer20_attention_weight_min": -23.95823860168457,
      "activations/layer21_attention_weight_max": 33.67412567138672,
      "activations/layer21_attention_weight_min": -19.99616050720215,
      "activations/layer22_attention_weight_max": 27.751075744628906,
      "activations/layer22_attention_weight_min": -25.395004272460938,
      "activations/layer23_attention_weight_max": 35.571128845214844,
      "activations/layer23_attention_weight_min": -23.004642486572266,
      "activations/layer2_attention_weight_max": 31.617534637451172,
      "activations/layer2_attention_weight_min": -30.353851318359375,
      "activations/layer3_attention_weight_max": 87.61563110351562,
      "activations/layer3_attention_weight_min": -86.96979522705078,
      "activations/layer4_attention_weight_max": 82.7019271850586,
      "activations/layer4_attention_weight_min": -83.38663482666016,
      "activations/layer5_attention_weight_max": 58.769126892089844,
      "activations/layer5_attention_weight_min": -69.6527328491211,
      "activations/layer6_attention_weight_max": 52.20844268798828,
      "activations/layer6_attention_weight_min": -50.056785583496094,
      "activations/layer7_attention_weight_max": 73.14693450927734,
      "activations/layer7_attention_weight_min": -64.38907623291016,
      "activations/layer8_attention_weight_max": 48.43397521972656,
      "activations/layer8_attention_weight_min": -50.01231002807617,
      "activations/layer9_attention_weight_max": 43.63568878173828,
      "activations/layer9_attention_weight_min": -45.3803596496582,
      "epoch": 20.93,
      "learning_rate": 1.5165151515151513e-05,
      "loss": 2.7055,
      "step": 360250
    },
    {
      "activations/layer0_attention_weight_max": 15.304328918457031,
      "activations/layer0_attention_weight_min": -14.076252937316895,
      "activations/layer10_attention_weight_max": 38.881526947021484,
      "activations/layer10_attention_weight_min": -37.61438751220703,
      "activations/layer11_attention_weight_max": 34.82189178466797,
      "activations/layer11_attention_weight_min": -36.04107666015625,
      "activations/layer12_attention_weight_max": 20.726789474487305,
      "activations/layer12_attention_weight_min": -27.61095428466797,
      "activations/layer13_attention_weight_max": 39.86285400390625,
      "activations/layer13_attention_weight_min": -28.1519718170166,
      "activations/layer14_attention_weight_max": 41.47637176513672,
      "activations/layer14_attention_weight_min": -33.49211502075195,
      "activations/layer15_attention_weight_max": 39.973384857177734,
      "activations/layer15_attention_weight_min": -32.900699615478516,
      "activations/layer16_attention_weight_max": 42.912086486816406,
      "activations/layer16_attention_weight_min": -38.05988693237305,
      "activations/layer17_attention_weight_max": 63.0258674621582,
      "activations/layer17_attention_weight_min": -51.85053634643555,
      "activations/layer18_attention_weight_max": 60.58677291870117,
      "activations/layer18_attention_weight_min": -45.250648498535156,
      "activations/layer19_attention_weight_max": 34.98057556152344,
      "activations/layer19_attention_weight_min": -27.28244400024414,
      "activations/layer1_attention_weight_max": 16.054777145385742,
      "activations/layer1_attention_weight_min": -13.547876358032227,
      "activations/layer20_attention_weight_max": 30.0892391204834,
      "activations/layer20_attention_weight_min": -23.627084732055664,
      "activations/layer21_attention_weight_max": 44.781253814697266,
      "activations/layer21_attention_weight_min": -22.539506912231445,
      "activations/layer22_attention_weight_max": 34.072330474853516,
      "activations/layer22_attention_weight_min": -26.37006378173828,
      "activations/layer23_attention_weight_max": 44.396888732910156,
      "activations/layer23_attention_weight_min": -25.115188598632812,
      "activations/layer2_attention_weight_max": 31.876516342163086,
      "activations/layer2_attention_weight_min": -30.86362075805664,
      "activations/layer3_attention_weight_max": 91.88301086425781,
      "activations/layer3_attention_weight_min": -92.40107727050781,
      "activations/layer4_attention_weight_max": 86.96691131591797,
      "activations/layer4_attention_weight_min": -88.56587982177734,
      "activations/layer5_attention_weight_max": 60.896766662597656,
      "activations/layer5_attention_weight_min": -74.02523040771484,
      "activations/layer6_attention_weight_max": 49.21935272216797,
      "activations/layer6_attention_weight_min": -50.62897491455078,
      "activations/layer7_attention_weight_max": 66.43411254882812,
      "activations/layer7_attention_weight_min": -66.06985473632812,
      "activations/layer8_attention_weight_max": 48.52055740356445,
      "activations/layer8_attention_weight_min": -50.81880187988281,
      "activations/layer9_attention_weight_max": 47.513736724853516,
      "activations/layer9_attention_weight_min": -48.73355484008789,
      "epoch": 20.94,
      "learning_rate": 1.5146212121212121e-05,
      "loss": 2.7403,
      "step": 360300
    },
    {
      "activations/layer0_attention_weight_max": 15.892797470092773,
      "activations/layer0_attention_weight_min": -13.962067604064941,
      "activations/layer10_attention_weight_max": 35.50537872314453,
      "activations/layer10_attention_weight_min": -37.4063835144043,
      "activations/layer11_attention_weight_max": 34.466094970703125,
      "activations/layer11_attention_weight_min": -34.85676956176758,
      "activations/layer12_attention_weight_max": 21.07316780090332,
      "activations/layer12_attention_weight_min": -21.604230880737305,
      "activations/layer13_attention_weight_max": 45.58285903930664,
      "activations/layer13_attention_weight_min": -30.579601287841797,
      "activations/layer14_attention_weight_max": 36.094337463378906,
      "activations/layer14_attention_weight_min": -32.329654693603516,
      "activations/layer15_attention_weight_max": 33.861167907714844,
      "activations/layer15_attention_weight_min": -34.67238235473633,
      "activations/layer16_attention_weight_max": 35.86590576171875,
      "activations/layer16_attention_weight_min": -35.98933792114258,
      "activations/layer17_attention_weight_max": 53.18031692504883,
      "activations/layer17_attention_weight_min": -50.4581184387207,
      "activations/layer18_attention_weight_max": 51.94176483154297,
      "activations/layer18_attention_weight_min": -49.6666374206543,
      "activations/layer19_attention_weight_max": 25.714521408081055,
      "activations/layer19_attention_weight_min": -26.774587631225586,
      "activations/layer1_attention_weight_max": 15.263833999633789,
      "activations/layer1_attention_weight_min": -12.960716247558594,
      "activations/layer20_attention_weight_max": 23.34012222290039,
      "activations/layer20_attention_weight_min": -20.914365768432617,
      "activations/layer21_attention_weight_max": 36.48146438598633,
      "activations/layer21_attention_weight_min": -26.117656707763672,
      "activations/layer22_attention_weight_max": 25.851465225219727,
      "activations/layer22_attention_weight_min": -25.60296058654785,
      "activations/layer23_attention_weight_max": 36.027183532714844,
      "activations/layer23_attention_weight_min": -24.32457733154297,
      "activations/layer2_attention_weight_max": 31.699199676513672,
      "activations/layer2_attention_weight_min": -31.387832641601562,
      "activations/layer3_attention_weight_max": 92.63285064697266,
      "activations/layer3_attention_weight_min": -91.19271850585938,
      "activations/layer4_attention_weight_max": 83.95040130615234,
      "activations/layer4_attention_weight_min": -84.45038604736328,
      "activations/layer5_attention_weight_max": 60.34313201904297,
      "activations/layer5_attention_weight_min": -69.43616485595703,
      "activations/layer6_attention_weight_max": 51.22391128540039,
      "activations/layer6_attention_weight_min": -50.15408706665039,
      "activations/layer7_attention_weight_max": 67.5311508178711,
      "activations/layer7_attention_weight_min": -67.02910614013672,
      "activations/layer8_attention_weight_max": 48.12872314453125,
      "activations/layer8_attention_weight_min": -52.26663589477539,
      "activations/layer9_attention_weight_max": 47.15470886230469,
      "activations/layer9_attention_weight_min": -46.47407531738281,
      "epoch": 20.94,
      "learning_rate": 1.5127272727272728e-05,
      "loss": 2.7172,
      "step": 360350
    },
    {
      "activations/layer0_attention_weight_max": 15.304611206054688,
      "activations/layer0_attention_weight_min": -14.314202308654785,
      "activations/layer10_attention_weight_max": 35.31489944458008,
      "activations/layer10_attention_weight_min": -35.61818313598633,
      "activations/layer11_attention_weight_max": 34.19037628173828,
      "activations/layer11_attention_weight_min": -33.067413330078125,
      "activations/layer12_attention_weight_max": 18.8353328704834,
      "activations/layer12_attention_weight_min": -27.535240173339844,
      "activations/layer13_attention_weight_max": 34.23625183105469,
      "activations/layer13_attention_weight_min": -28.313661575317383,
      "activations/layer14_attention_weight_max": 33.02861022949219,
      "activations/layer14_attention_weight_min": -29.10195541381836,
      "activations/layer15_attention_weight_max": 30.36651039123535,
      "activations/layer15_attention_weight_min": -29.79020881652832,
      "activations/layer16_attention_weight_max": 30.173627853393555,
      "activations/layer16_attention_weight_min": -30.322359085083008,
      "activations/layer17_attention_weight_max": 51.643035888671875,
      "activations/layer17_attention_weight_min": -46.26655960083008,
      "activations/layer18_attention_weight_max": 40.52116394042969,
      "activations/layer18_attention_weight_min": -39.72850036621094,
      "activations/layer19_attention_weight_max": 25.62445068359375,
      "activations/layer19_attention_weight_min": -24.5592098236084,
      "activations/layer1_attention_weight_max": 15.242979049682617,
      "activations/layer1_attention_weight_min": -14.22197437286377,
      "activations/layer20_attention_weight_max": 22.30101203918457,
      "activations/layer20_attention_weight_min": -25.11589241027832,
      "activations/layer21_attention_weight_max": 32.11468505859375,
      "activations/layer21_attention_weight_min": -25.630626678466797,
      "activations/layer22_attention_weight_max": 25.83904457092285,
      "activations/layer22_attention_weight_min": -25.653589248657227,
      "activations/layer23_attention_weight_max": 36.34400939941406,
      "activations/layer23_attention_weight_min": -25.5398006439209,
      "activations/layer2_attention_weight_max": 31.356969833374023,
      "activations/layer2_attention_weight_min": -29.880287170410156,
      "activations/layer3_attention_weight_max": 85.20463562011719,
      "activations/layer3_attention_weight_min": -83.13390350341797,
      "activations/layer4_attention_weight_max": 81.81136322021484,
      "activations/layer4_attention_weight_min": -81.5190200805664,
      "activations/layer5_attention_weight_max": 63.2474479675293,
      "activations/layer5_attention_weight_min": -70.57523345947266,
      "activations/layer6_attention_weight_max": 50.7280158996582,
      "activations/layer6_attention_weight_min": -50.05624771118164,
      "activations/layer7_attention_weight_max": 66.9466781616211,
      "activations/layer7_attention_weight_min": -64.98966979980469,
      "activations/layer8_attention_weight_max": 46.07820129394531,
      "activations/layer8_attention_weight_min": -50.39065933227539,
      "activations/layer9_attention_weight_max": 42.676841735839844,
      "activations/layer9_attention_weight_min": -46.23286819458008,
      "epoch": 20.94,
      "learning_rate": 1.510871212121212e-05,
      "loss": 2.7276,
      "step": 360400
    },
    {
      "activations/layer0_attention_weight_max": 15.66983699798584,
      "activations/layer0_attention_weight_min": -13.694189071655273,
      "activations/layer10_attention_weight_max": 38.05393600463867,
      "activations/layer10_attention_weight_min": -36.71110153198242,
      "activations/layer11_attention_weight_max": 36.545692443847656,
      "activations/layer11_attention_weight_min": -33.96138000488281,
      "activations/layer12_attention_weight_max": 18.030485153198242,
      "activations/layer12_attention_weight_min": -25.950651168823242,
      "activations/layer13_attention_weight_max": 38.268890380859375,
      "activations/layer13_attention_weight_min": -27.940649032592773,
      "activations/layer14_attention_weight_max": 36.31377029418945,
      "activations/layer14_attention_weight_min": -30.156696319580078,
      "activations/layer15_attention_weight_max": 34.619300842285156,
      "activations/layer15_attention_weight_min": -31.077857971191406,
      "activations/layer16_attention_weight_max": 33.006343841552734,
      "activations/layer16_attention_weight_min": -32.155277252197266,
      "activations/layer17_attention_weight_max": 50.200679779052734,
      "activations/layer17_attention_weight_min": -45.75538635253906,
      "activations/layer18_attention_weight_max": 48.27389144897461,
      "activations/layer18_attention_weight_min": -39.73834991455078,
      "activations/layer19_attention_weight_max": 23.6644287109375,
      "activations/layer19_attention_weight_min": -25.03407096862793,
      "activations/layer1_attention_weight_max": 16.22524642944336,
      "activations/layer1_attention_weight_min": -15.570866584777832,
      "activations/layer20_attention_weight_max": 22.514787673950195,
      "activations/layer20_attention_weight_min": -23.496196746826172,
      "activations/layer21_attention_weight_max": 37.44785690307617,
      "activations/layer21_attention_weight_min": -24.928850173950195,
      "activations/layer22_attention_weight_max": 30.68802833557129,
      "activations/layer22_attention_weight_min": -25.572574615478516,
      "activations/layer23_attention_weight_max": 34.894432067871094,
      "activations/layer23_attention_weight_min": -26.478750228881836,
      "activations/layer2_attention_weight_max": 33.389923095703125,
      "activations/layer2_attention_weight_min": -31.825122833251953,
      "activations/layer3_attention_weight_max": 88.89849090576172,
      "activations/layer3_attention_weight_min": -89.75975799560547,
      "activations/layer4_attention_weight_max": 84.6219253540039,
      "activations/layer4_attention_weight_min": -87.10765838623047,
      "activations/layer5_attention_weight_max": 63.39805221557617,
      "activations/layer5_attention_weight_min": -69.98343658447266,
      "activations/layer6_attention_weight_max": 49.533424377441406,
      "activations/layer6_attention_weight_min": -54.39280319213867,
      "activations/layer7_attention_weight_max": 69.65775299072266,
      "activations/layer7_attention_weight_min": -68.67994689941406,
      "activations/layer8_attention_weight_max": 47.83013153076172,
      "activations/layer8_attention_weight_min": -54.844261169433594,
      "activations/layer9_attention_weight_max": 54.899383544921875,
      "activations/layer9_attention_weight_min": -45.93817138671875,
      "epoch": 20.94,
      "learning_rate": 1.5089772727272727e-05,
      "loss": 2.724,
      "step": 360450
    },
    {
      "activations/layer0_attention_weight_max": 15.430322647094727,
      "activations/layer0_attention_weight_min": -14.094854354858398,
      "activations/layer10_attention_weight_max": 33.8907356262207,
      "activations/layer10_attention_weight_min": -34.75618362426758,
      "activations/layer11_attention_weight_max": 32.773719787597656,
      "activations/layer11_attention_weight_min": -33.6717414855957,
      "activations/layer12_attention_weight_max": 18.892532348632812,
      "activations/layer12_attention_weight_min": -21.972021102905273,
      "activations/layer13_attention_weight_max": 31.90810203552246,
      "activations/layer13_attention_weight_min": -26.78724479675293,
      "activations/layer14_attention_weight_max": 32.271263122558594,
      "activations/layer14_attention_weight_min": -29.923791885375977,
      "activations/layer15_attention_weight_max": 31.300569534301758,
      "activations/layer15_attention_weight_min": -30.47176742553711,
      "activations/layer16_attention_weight_max": 32.191349029541016,
      "activations/layer16_attention_weight_min": -30.84417724609375,
      "activations/layer17_attention_weight_max": 45.65733337402344,
      "activations/layer17_attention_weight_min": -43.51711654663086,
      "activations/layer18_attention_weight_max": 42.89427947998047,
      "activations/layer18_attention_weight_min": -38.324275970458984,
      "activations/layer19_attention_weight_max": 24.83070182800293,
      "activations/layer19_attention_weight_min": -21.627683639526367,
      "activations/layer1_attention_weight_max": 15.449976921081543,
      "activations/layer1_attention_weight_min": -13.985952377319336,
      "activations/layer20_attention_weight_max": 23.5064697265625,
      "activations/layer20_attention_weight_min": -20.974973678588867,
      "activations/layer21_attention_weight_max": 33.25244140625,
      "activations/layer21_attention_weight_min": -21.19073486328125,
      "activations/layer22_attention_weight_max": 30.14993667602539,
      "activations/layer22_attention_weight_min": -23.664480209350586,
      "activations/layer23_attention_weight_max": 34.73740005493164,
      "activations/layer23_attention_weight_min": -23.698461532592773,
      "activations/layer2_attention_weight_max": 30.694988250732422,
      "activations/layer2_attention_weight_min": -30.406009674072266,
      "activations/layer3_attention_weight_max": 86.81620788574219,
      "activations/layer3_attention_weight_min": -89.38981628417969,
      "activations/layer4_attention_weight_max": 82.3216781616211,
      "activations/layer4_attention_weight_min": -82.97730255126953,
      "activations/layer5_attention_weight_max": 59.87484359741211,
      "activations/layer5_attention_weight_min": -68.20027160644531,
      "activations/layer6_attention_weight_max": 49.21907424926758,
      "activations/layer6_attention_weight_min": -50.673831939697266,
      "activations/layer7_attention_weight_max": 64.05274200439453,
      "activations/layer7_attention_weight_min": -66.9397964477539,
      "activations/layer8_attention_weight_max": 47.28375244140625,
      "activations/layer8_attention_weight_min": -50.56898880004883,
      "activations/layer9_attention_weight_max": 42.9405403137207,
      "activations/layer9_attention_weight_min": -45.061771392822266,
      "epoch": 20.95,
      "learning_rate": 1.5070833333333333e-05,
      "loss": 2.73,
      "step": 360500
    },
    {
      "activations/layer0_attention_weight_max": 15.695918083190918,
      "activations/layer0_attention_weight_min": -14.146306037902832,
      "activations/layer10_attention_weight_max": 39.34575271606445,
      "activations/layer10_attention_weight_min": -40.13832473754883,
      "activations/layer11_attention_weight_max": 37.84596252441406,
      "activations/layer11_attention_weight_min": -36.897247314453125,
      "activations/layer12_attention_weight_max": 19.17221450805664,
      "activations/layer12_attention_weight_min": -29.48850440979004,
      "activations/layer13_attention_weight_max": 33.95516586303711,
      "activations/layer13_attention_weight_min": -30.07172203063965,
      "activations/layer14_attention_weight_max": 37.53061294555664,
      "activations/layer14_attention_weight_min": -29.600475311279297,
      "activations/layer15_attention_weight_max": 34.52056884765625,
      "activations/layer15_attention_weight_min": -30.698780059814453,
      "activations/layer16_attention_weight_max": 34.888275146484375,
      "activations/layer16_attention_weight_min": -34.170074462890625,
      "activations/layer17_attention_weight_max": 54.03391647338867,
      "activations/layer17_attention_weight_min": -45.45966720581055,
      "activations/layer18_attention_weight_max": 45.86166000366211,
      "activations/layer18_attention_weight_min": -42.08955383300781,
      "activations/layer19_attention_weight_max": 22.610776901245117,
      "activations/layer19_attention_weight_min": -22.55604362487793,
      "activations/layer1_attention_weight_max": 16.034202575683594,
      "activations/layer1_attention_weight_min": -13.389144897460938,
      "activations/layer20_attention_weight_max": 22.162803649902344,
      "activations/layer20_attention_weight_min": -23.26365852355957,
      "activations/layer21_attention_weight_max": 31.309263229370117,
      "activations/layer21_attention_weight_min": -22.991287231445312,
      "activations/layer22_attention_weight_max": 25.7921142578125,
      "activations/layer22_attention_weight_min": -25.76206398010254,
      "activations/layer23_attention_weight_max": 33.501243591308594,
      "activations/layer23_attention_weight_min": -25.735931396484375,
      "activations/layer2_attention_weight_max": 34.49351119995117,
      "activations/layer2_attention_weight_min": -30.3787841796875,
      "activations/layer3_attention_weight_max": 89.02711486816406,
      "activations/layer3_attention_weight_min": -85.8208236694336,
      "activations/layer4_attention_weight_max": 83.15397644042969,
      "activations/layer4_attention_weight_min": -85.1340103149414,
      "activations/layer5_attention_weight_max": 62.813812255859375,
      "activations/layer5_attention_weight_min": -71.42483520507812,
      "activations/layer6_attention_weight_max": 50.45506286621094,
      "activations/layer6_attention_weight_min": -51.04661560058594,
      "activations/layer7_attention_weight_max": 67.20594787597656,
      "activations/layer7_attention_weight_min": -69.00756072998047,
      "activations/layer8_attention_weight_max": 50.3452262878418,
      "activations/layer8_attention_weight_min": -52.27274703979492,
      "activations/layer9_attention_weight_max": 50.62847900390625,
      "activations/layer9_attention_weight_min": -46.04523849487305,
      "epoch": 20.95,
      "learning_rate": 1.5052272727272727e-05,
      "loss": 2.7238,
      "step": 360550
    },
    {
      "activations/layer0_attention_weight_max": 15.60338306427002,
      "activations/layer0_attention_weight_min": -13.855072021484375,
      "activations/layer10_attention_weight_max": 34.31619644165039,
      "activations/layer10_attention_weight_min": -34.896278381347656,
      "activations/layer11_attention_weight_max": 31.964889526367188,
      "activations/layer11_attention_weight_min": -35.03829574584961,
      "activations/layer12_attention_weight_max": 19.06878662109375,
      "activations/layer12_attention_weight_min": -24.390995025634766,
      "activations/layer13_attention_weight_max": 37.82664108276367,
      "activations/layer13_attention_weight_min": -27.2952823638916,
      "activations/layer14_attention_weight_max": 35.60015106201172,
      "activations/layer14_attention_weight_min": -29.50904655456543,
      "activations/layer15_attention_weight_max": 32.6188850402832,
      "activations/layer15_attention_weight_min": -30.375213623046875,
      "activations/layer16_attention_weight_max": 33.41156768798828,
      "activations/layer16_attention_weight_min": -32.13901138305664,
      "activations/layer17_attention_weight_max": 51.04558181762695,
      "activations/layer17_attention_weight_min": -45.29032897949219,
      "activations/layer18_attention_weight_max": 44.350914001464844,
      "activations/layer18_attention_weight_min": -38.237945556640625,
      "activations/layer19_attention_weight_max": 23.046600341796875,
      "activations/layer19_attention_weight_min": -21.776796340942383,
      "activations/layer1_attention_weight_max": 15.163728713989258,
      "activations/layer1_attention_weight_min": -14.938786506652832,
      "activations/layer20_attention_weight_max": 22.118547439575195,
      "activations/layer20_attention_weight_min": -21.421051025390625,
      "activations/layer21_attention_weight_max": 35.39864730834961,
      "activations/layer21_attention_weight_min": -24.987548828125,
      "activations/layer22_attention_weight_max": 27.484130859375,
      "activations/layer22_attention_weight_min": -25.326034545898438,
      "activations/layer23_attention_weight_max": 39.274227142333984,
      "activations/layer23_attention_weight_min": -24.88355827331543,
      "activations/layer2_attention_weight_max": 31.35633087158203,
      "activations/layer2_attention_weight_min": -30.684001922607422,
      "activations/layer3_attention_weight_max": 90.84471893310547,
      "activations/layer3_attention_weight_min": -89.494873046875,
      "activations/layer4_attention_weight_max": 82.34680938720703,
      "activations/layer4_attention_weight_min": -82.20793151855469,
      "activations/layer5_attention_weight_max": 61.98772430419922,
      "activations/layer5_attention_weight_min": -69.70660400390625,
      "activations/layer6_attention_weight_max": 48.60389709472656,
      "activations/layer6_attention_weight_min": -50.28868865966797,
      "activations/layer7_attention_weight_max": 66.35769653320312,
      "activations/layer7_attention_weight_min": -64.34783172607422,
      "activations/layer8_attention_weight_max": 45.216651916503906,
      "activations/layer8_attention_weight_min": -49.17176055908203,
      "activations/layer9_attention_weight_max": 41.337886810302734,
      "activations/layer9_attention_weight_min": -44.396907806396484,
      "epoch": 20.95,
      "learning_rate": 1.5033333333333332e-05,
      "loss": 2.7041,
      "step": 360600
    },
    {
      "activations/layer0_attention_weight_max": 15.563133239746094,
      "activations/layer0_attention_weight_min": -13.98701000213623,
      "activations/layer10_attention_weight_max": 36.76571273803711,
      "activations/layer10_attention_weight_min": -36.791263580322266,
      "activations/layer11_attention_weight_max": 35.56544494628906,
      "activations/layer11_attention_weight_min": -35.443756103515625,
      "activations/layer12_attention_weight_max": 19.845008850097656,
      "activations/layer12_attention_weight_min": -29.174312591552734,
      "activations/layer13_attention_weight_max": 38.80675506591797,
      "activations/layer13_attention_weight_min": -28.62181282043457,
      "activations/layer14_attention_weight_max": 36.71275329589844,
      "activations/layer14_attention_weight_min": -29.84126853942871,
      "activations/layer15_attention_weight_max": 36.47197723388672,
      "activations/layer15_attention_weight_min": -31.65683937072754,
      "activations/layer16_attention_weight_max": 35.457801818847656,
      "activations/layer16_attention_weight_min": -32.815670013427734,
      "activations/layer17_attention_weight_max": 53.2859992980957,
      "activations/layer17_attention_weight_min": -47.68385696411133,
      "activations/layer18_attention_weight_max": 49.00775146484375,
      "activations/layer18_attention_weight_min": -42.978851318359375,
      "activations/layer19_attention_weight_max": 24.11830711364746,
      "activations/layer19_attention_weight_min": -24.306406021118164,
      "activations/layer1_attention_weight_max": 15.012202262878418,
      "activations/layer1_attention_weight_min": -14.664246559143066,
      "activations/layer20_attention_weight_max": 23.577604293823242,
      "activations/layer20_attention_weight_min": -22.222196578979492,
      "activations/layer21_attention_weight_max": 32.64278030395508,
      "activations/layer21_attention_weight_min": -22.250015258789062,
      "activations/layer22_attention_weight_max": 29.33324432373047,
      "activations/layer22_attention_weight_min": -26.77167510986328,
      "activations/layer23_attention_weight_max": 36.54012680053711,
      "activations/layer23_attention_weight_min": -25.47121810913086,
      "activations/layer2_attention_weight_max": 33.026329040527344,
      "activations/layer2_attention_weight_min": -30.145709991455078,
      "activations/layer3_attention_weight_max": 93.30408477783203,
      "activations/layer3_attention_weight_min": -88.97377014160156,
      "activations/layer4_attention_weight_max": 80.38621520996094,
      "activations/layer4_attention_weight_min": -82.01420593261719,
      "activations/layer5_attention_weight_max": 57.6234245300293,
      "activations/layer5_attention_weight_min": -73.04472351074219,
      "activations/layer6_attention_weight_max": 46.42982482910156,
      "activations/layer6_attention_weight_min": -49.25833511352539,
      "activations/layer7_attention_weight_max": 69.2313232421875,
      "activations/layer7_attention_weight_min": -69.86552429199219,
      "activations/layer8_attention_weight_max": 48.625118255615234,
      "activations/layer8_attention_weight_min": -51.52507400512695,
      "activations/layer9_attention_weight_max": 48.303836822509766,
      "activations/layer9_attention_weight_min": -46.83172607421875,
      "epoch": 20.96,
      "learning_rate": 1.5014393939393939e-05,
      "loss": 2.7136,
      "step": 360650
    },
    {
      "activations/layer0_attention_weight_max": 15.460823059082031,
      "activations/layer0_attention_weight_min": -14.081180572509766,
      "activations/layer10_attention_weight_max": 37.341583251953125,
      "activations/layer10_attention_weight_min": -37.39400863647461,
      "activations/layer11_attention_weight_max": 34.287757873535156,
      "activations/layer11_attention_weight_min": -35.21746826171875,
      "activations/layer12_attention_weight_max": 19.78876304626465,
      "activations/layer12_attention_weight_min": -29.68781089782715,
      "activations/layer13_attention_weight_max": 34.787506103515625,
      "activations/layer13_attention_weight_min": -31.304983139038086,
      "activations/layer14_attention_weight_max": 36.121456146240234,
      "activations/layer14_attention_weight_min": -31.8212890625,
      "activations/layer15_attention_weight_max": 37.135948181152344,
      "activations/layer15_attention_weight_min": -33.17839813232422,
      "activations/layer16_attention_weight_max": 35.593502044677734,
      "activations/layer16_attention_weight_min": -34.13799285888672,
      "activations/layer17_attention_weight_max": 52.7238883972168,
      "activations/layer17_attention_weight_min": -47.46844482421875,
      "activations/layer18_attention_weight_max": 51.633426666259766,
      "activations/layer18_attention_weight_min": -41.76022720336914,
      "activations/layer19_attention_weight_max": 27.234994888305664,
      "activations/layer19_attention_weight_min": -22.7425594329834,
      "activations/layer1_attention_weight_max": 15.602767944335938,
      "activations/layer1_attention_weight_min": -14.93850040435791,
      "activations/layer20_attention_weight_max": 28.526626586914062,
      "activations/layer20_attention_weight_min": -20.250511169433594,
      "activations/layer21_attention_weight_max": 36.815711975097656,
      "activations/layer21_attention_weight_min": -20.643451690673828,
      "activations/layer22_attention_weight_max": 32.56435775756836,
      "activations/layer22_attention_weight_min": -24.962366104125977,
      "activations/layer23_attention_weight_max": 38.893985748291016,
      "activations/layer23_attention_weight_min": -25.201396942138672,
      "activations/layer2_attention_weight_max": 32.23048782348633,
      "activations/layer2_attention_weight_min": -30.63863754272461,
      "activations/layer3_attention_weight_max": 88.73591613769531,
      "activations/layer3_attention_weight_min": -89.13359069824219,
      "activations/layer4_attention_weight_max": 83.19108581542969,
      "activations/layer4_attention_weight_min": -84.6568374633789,
      "activations/layer5_attention_weight_max": 63.00349426269531,
      "activations/layer5_attention_weight_min": -76.75858306884766,
      "activations/layer6_attention_weight_max": 49.61588668823242,
      "activations/layer6_attention_weight_min": -50.818172454833984,
      "activations/layer7_attention_weight_max": 68.13877868652344,
      "activations/layer7_attention_weight_min": -66.62837219238281,
      "activations/layer8_attention_weight_max": 46.4551887512207,
      "activations/layer8_attention_weight_min": -51.336883544921875,
      "activations/layer9_attention_weight_max": 47.87174606323242,
      "activations/layer9_attention_weight_min": -48.38277816772461,
      "epoch": 20.96,
      "learning_rate": 1.4995454545454544e-05,
      "loss": 2.7165,
      "step": 360700
    },
    {
      "activations/layer0_attention_weight_max": 16.166767120361328,
      "activations/layer0_attention_weight_min": -14.116914749145508,
      "activations/layer10_attention_weight_max": 37.46332550048828,
      "activations/layer10_attention_weight_min": -35.62798309326172,
      "activations/layer11_attention_weight_max": 33.356483459472656,
      "activations/layer11_attention_weight_min": -35.31003952026367,
      "activations/layer12_attention_weight_max": 18.976177215576172,
      "activations/layer12_attention_weight_min": -23.138675689697266,
      "activations/layer13_attention_weight_max": 33.79885482788086,
      "activations/layer13_attention_weight_min": -27.28954315185547,
      "activations/layer14_attention_weight_max": 36.134765625,
      "activations/layer14_attention_weight_min": -29.579980850219727,
      "activations/layer15_attention_weight_max": 31.967008590698242,
      "activations/layer15_attention_weight_min": -31.507854461669922,
      "activations/layer16_attention_weight_max": 35.54220962524414,
      "activations/layer16_attention_weight_min": -31.050508499145508,
      "activations/layer17_attention_weight_max": 48.89280319213867,
      "activations/layer17_attention_weight_min": -42.90306091308594,
      "activations/layer18_attention_weight_max": 48.95549774169922,
      "activations/layer18_attention_weight_min": -39.46119689941406,
      "activations/layer19_attention_weight_max": 23.5181827545166,
      "activations/layer19_attention_weight_min": -21.651884078979492,
      "activations/layer1_attention_weight_max": 15.528162956237793,
      "activations/layer1_attention_weight_min": -14.580645561218262,
      "activations/layer20_attention_weight_max": 21.076078414916992,
      "activations/layer20_attention_weight_min": -20.28119659423828,
      "activations/layer21_attention_weight_max": 35.30892562866211,
      "activations/layer21_attention_weight_min": -23.355154037475586,
      "activations/layer22_attention_weight_max": 29.243059158325195,
      "activations/layer22_attention_weight_min": -23.71744155883789,
      "activations/layer23_attention_weight_max": 35.410316467285156,
      "activations/layer23_attention_weight_min": -24.18834686279297,
      "activations/layer2_attention_weight_max": 31.327903747558594,
      "activations/layer2_attention_weight_min": -30.923873901367188,
      "activations/layer3_attention_weight_max": 89.89607238769531,
      "activations/layer3_attention_weight_min": -88.08580017089844,
      "activations/layer4_attention_weight_max": 84.27193450927734,
      "activations/layer4_attention_weight_min": -85.43155670166016,
      "activations/layer5_attention_weight_max": 65.92821502685547,
      "activations/layer5_attention_weight_min": -73.78020477294922,
      "activations/layer6_attention_weight_max": 51.24907302856445,
      "activations/layer6_attention_weight_min": -52.52033996582031,
      "activations/layer7_attention_weight_max": 70.54148864746094,
      "activations/layer7_attention_weight_min": -70.12925720214844,
      "activations/layer8_attention_weight_max": 47.68463897705078,
      "activations/layer8_attention_weight_min": -48.585453033447266,
      "activations/layer9_attention_weight_max": 44.434532165527344,
      "activations/layer9_attention_weight_min": -46.77193832397461,
      "epoch": 20.96,
      "learning_rate": 1.497651515151515e-05,
      "loss": 2.7119,
      "step": 360750
    },
    {
      "activations/layer0_attention_weight_max": 15.882891654968262,
      "activations/layer0_attention_weight_min": -14.067037582397461,
      "activations/layer10_attention_weight_max": 39.596092224121094,
      "activations/layer10_attention_weight_min": -38.362953186035156,
      "activations/layer11_attention_weight_max": 34.49384689331055,
      "activations/layer11_attention_weight_min": -34.963104248046875,
      "activations/layer12_attention_weight_max": 20.314443588256836,
      "activations/layer12_attention_weight_min": -31.60430335998535,
      "activations/layer13_attention_weight_max": 35.49651336669922,
      "activations/layer13_attention_weight_min": -28.533119201660156,
      "activations/layer14_attention_weight_max": 37.918331146240234,
      "activations/layer14_attention_weight_min": -31.205730438232422,
      "activations/layer15_attention_weight_max": 37.13685607910156,
      "activations/layer15_attention_weight_min": -31.02550506591797,
      "activations/layer16_attention_weight_max": 38.537113189697266,
      "activations/layer16_attention_weight_min": -37.2784309387207,
      "activations/layer17_attention_weight_max": 61.28352355957031,
      "activations/layer17_attention_weight_min": -54.158119201660156,
      "activations/layer18_attention_weight_max": 55.933265686035156,
      "activations/layer18_attention_weight_min": -43.54942321777344,
      "activations/layer19_attention_weight_max": 23.69232177734375,
      "activations/layer19_attention_weight_min": -23.770593643188477,
      "activations/layer1_attention_weight_max": 15.598374366760254,
      "activations/layer1_attention_weight_min": -14.711609840393066,
      "activations/layer20_attention_weight_max": 22.102519989013672,
      "activations/layer20_attention_weight_min": -22.92608070373535,
      "activations/layer21_attention_weight_max": 37.06945037841797,
      "activations/layer21_attention_weight_min": -23.358139038085938,
      "activations/layer22_attention_weight_max": 35.04145812988281,
      "activations/layer22_attention_weight_min": -24.919654846191406,
      "activations/layer23_attention_weight_max": 39.026039123535156,
      "activations/layer23_attention_weight_min": -24.141170501708984,
      "activations/layer2_attention_weight_max": 31.606809616088867,
      "activations/layer2_attention_weight_min": -31.872282028198242,
      "activations/layer3_attention_weight_max": 94.29072570800781,
      "activations/layer3_attention_weight_min": -94.82135772705078,
      "activations/layer4_attention_weight_max": 91.45711517333984,
      "activations/layer4_attention_weight_min": -91.08980560302734,
      "activations/layer5_attention_weight_max": 67.55415344238281,
      "activations/layer5_attention_weight_min": -71.03849029541016,
      "activations/layer6_attention_weight_max": 51.91874313354492,
      "activations/layer6_attention_weight_min": -53.8547248840332,
      "activations/layer7_attention_weight_max": 74.22647094726562,
      "activations/layer7_attention_weight_min": -72.91657257080078,
      "activations/layer8_attention_weight_max": 53.14461135864258,
      "activations/layer8_attention_weight_min": -53.629615783691406,
      "activations/layer9_attention_weight_max": 49.07600021362305,
      "activations/layer9_attention_weight_min": -50.10946273803711,
      "epoch": 20.96,
      "learning_rate": 1.4957575757575757e-05,
      "loss": 2.7115,
      "step": 360800
    },
    {
      "activations/layer0_attention_weight_max": 15.743879318237305,
      "activations/layer0_attention_weight_min": -13.979438781738281,
      "activations/layer10_attention_weight_max": 36.849342346191406,
      "activations/layer10_attention_weight_min": -36.942222595214844,
      "activations/layer11_attention_weight_max": 34.060302734375,
      "activations/layer11_attention_weight_min": -36.060546875,
      "activations/layer12_attention_weight_max": 18.078882217407227,
      "activations/layer12_attention_weight_min": -26.81974220275879,
      "activations/layer13_attention_weight_max": 32.59493637084961,
      "activations/layer13_attention_weight_min": -27.13689613342285,
      "activations/layer14_attention_weight_max": 36.106651306152344,
      "activations/layer14_attention_weight_min": -31.806377410888672,
      "activations/layer15_attention_weight_max": 35.02114486694336,
      "activations/layer15_attention_weight_min": -30.952774047851562,
      "activations/layer16_attention_weight_max": 35.7695198059082,
      "activations/layer16_attention_weight_min": -35.76890563964844,
      "activations/layer17_attention_weight_max": 53.80746078491211,
      "activations/layer17_attention_weight_min": -49.89175033569336,
      "activations/layer18_attention_weight_max": 48.95932388305664,
      "activations/layer18_attention_weight_min": -43.703250885009766,
      "activations/layer19_attention_weight_max": 22.57850456237793,
      "activations/layer19_attention_weight_min": -21.801477432250977,
      "activations/layer1_attention_weight_max": 15.565077781677246,
      "activations/layer1_attention_weight_min": -15.28051471710205,
      "activations/layer20_attention_weight_max": 20.78742218017578,
      "activations/layer20_attention_weight_min": -20.015575408935547,
      "activations/layer21_attention_weight_max": 34.28224182128906,
      "activations/layer21_attention_weight_min": -20.90814781188965,
      "activations/layer22_attention_weight_max": 30.959110260009766,
      "activations/layer22_attention_weight_min": -24.1405029296875,
      "activations/layer23_attention_weight_max": 35.80628967285156,
      "activations/layer23_attention_weight_min": -22.983007431030273,
      "activations/layer2_attention_weight_max": 32.04718017578125,
      "activations/layer2_attention_weight_min": -30.869441986083984,
      "activations/layer3_attention_weight_max": 90.70645904541016,
      "activations/layer3_attention_weight_min": -88.47697448730469,
      "activations/layer4_attention_weight_max": 85.41806030273438,
      "activations/layer4_attention_weight_min": -84.87279510498047,
      "activations/layer5_attention_weight_max": 65.81135559082031,
      "activations/layer5_attention_weight_min": -75.79319763183594,
      "activations/layer6_attention_weight_max": 47.739749908447266,
      "activations/layer6_attention_weight_min": -49.835289001464844,
      "activations/layer7_attention_weight_max": 62.85470962524414,
      "activations/layer7_attention_weight_min": -65.09697723388672,
      "activations/layer8_attention_weight_max": 46.95503616333008,
      "activations/layer8_attention_weight_min": -49.064903259277344,
      "activations/layer9_attention_weight_max": 43.173805236816406,
      "activations/layer9_attention_weight_min": -45.2614860534668,
      "epoch": 20.97,
      "learning_rate": 1.4938636363636362e-05,
      "loss": 2.7072,
      "step": 360850
    },
    {
      "activations/layer0_attention_weight_max": 16.101747512817383,
      "activations/layer0_attention_weight_min": -14.094145774841309,
      "activations/layer10_attention_weight_max": 38.050880432128906,
      "activations/layer10_attention_weight_min": -38.86531066894531,
      "activations/layer11_attention_weight_max": 40.13286590576172,
      "activations/layer11_attention_weight_min": -37.245361328125,
      "activations/layer12_attention_weight_max": 19.839948654174805,
      "activations/layer12_attention_weight_min": -30.00796127319336,
      "activations/layer13_attention_weight_max": 39.06390380859375,
      "activations/layer13_attention_weight_min": -28.595247268676758,
      "activations/layer14_attention_weight_max": 45.329261779785156,
      "activations/layer14_attention_weight_min": -34.12321472167969,
      "activations/layer15_attention_weight_max": 41.256752014160156,
      "activations/layer15_attention_weight_min": -33.05081558227539,
      "activations/layer16_attention_weight_max": 39.819801330566406,
      "activations/layer16_attention_weight_min": -36.59097671508789,
      "activations/layer17_attention_weight_max": 55.257537841796875,
      "activations/layer17_attention_weight_min": -49.263916015625,
      "activations/layer18_attention_weight_max": 49.69859313964844,
      "activations/layer18_attention_weight_min": -43.30316162109375,
      "activations/layer19_attention_weight_max": 26.997150421142578,
      "activations/layer19_attention_weight_min": -23.584861755371094,
      "activations/layer1_attention_weight_max": 15.235267639160156,
      "activations/layer1_attention_weight_min": -13.957136154174805,
      "activations/layer20_attention_weight_max": 22.8004207611084,
      "activations/layer20_attention_weight_min": -19.997732162475586,
      "activations/layer21_attention_weight_max": 45.035987854003906,
      "activations/layer21_attention_weight_min": -22.613323211669922,
      "activations/layer22_attention_weight_max": 31.569978713989258,
      "activations/layer22_attention_weight_min": -23.043731689453125,
      "activations/layer23_attention_weight_max": 36.320899963378906,
      "activations/layer23_attention_weight_min": -24.612892150878906,
      "activations/layer2_attention_weight_max": 34.72874069213867,
      "activations/layer2_attention_weight_min": -31.390090942382812,
      "activations/layer3_attention_weight_max": 93.89287567138672,
      "activations/layer3_attention_weight_min": -95.19313049316406,
      "activations/layer4_attention_weight_max": 84.2233657836914,
      "activations/layer4_attention_weight_min": -84.93389129638672,
      "activations/layer5_attention_weight_max": 59.986236572265625,
      "activations/layer5_attention_weight_min": -74.80574035644531,
      "activations/layer6_attention_weight_max": 49.26123809814453,
      "activations/layer6_attention_weight_min": -50.56663513183594,
      "activations/layer7_attention_weight_max": 66.68762969970703,
      "activations/layer7_attention_weight_min": -68.33279418945312,
      "activations/layer8_attention_weight_max": 48.90595626831055,
      "activations/layer8_attention_weight_min": -53.72487258911133,
      "activations/layer9_attention_weight_max": 51.05056381225586,
      "activations/layer9_attention_weight_min": -47.58562469482422,
      "epoch": 20.97,
      "learning_rate": 1.4919696969696968e-05,
      "loss": 2.7231,
      "step": 360900
    },
    {
      "activations/layer0_attention_weight_max": 15.877991676330566,
      "activations/layer0_attention_weight_min": -14.06356143951416,
      "activations/layer10_attention_weight_max": 37.22473907470703,
      "activations/layer10_attention_weight_min": -36.93992233276367,
      "activations/layer11_attention_weight_max": 31.793209075927734,
      "activations/layer11_attention_weight_min": -33.314369201660156,
      "activations/layer12_attention_weight_max": 19.334596633911133,
      "activations/layer12_attention_weight_min": -23.028640747070312,
      "activations/layer13_attention_weight_max": 34.914005279541016,
      "activations/layer13_attention_weight_min": -30.115367889404297,
      "activations/layer14_attention_weight_max": 35.44858169555664,
      "activations/layer14_attention_weight_min": -30.565692901611328,
      "activations/layer15_attention_weight_max": 34.55718231201172,
      "activations/layer15_attention_weight_min": -32.09501647949219,
      "activations/layer16_attention_weight_max": 35.65941619873047,
      "activations/layer16_attention_weight_min": -32.32422637939453,
      "activations/layer17_attention_weight_max": 53.516212463378906,
      "activations/layer17_attention_weight_min": -46.27782440185547,
      "activations/layer18_attention_weight_max": 48.66746520996094,
      "activations/layer18_attention_weight_min": -46.15388107299805,
      "activations/layer19_attention_weight_max": 27.070898056030273,
      "activations/layer19_attention_weight_min": -23.787826538085938,
      "activations/layer1_attention_weight_max": 15.699217796325684,
      "activations/layer1_attention_weight_min": -14.426299095153809,
      "activations/layer20_attention_weight_max": 23.9947509765625,
      "activations/layer20_attention_weight_min": -24.051410675048828,
      "activations/layer21_attention_weight_max": 37.09759521484375,
      "activations/layer21_attention_weight_min": -26.291412353515625,
      "activations/layer22_attention_weight_max": 30.195858001708984,
      "activations/layer22_attention_weight_min": -27.182329177856445,
      "activations/layer23_attention_weight_max": 35.402835845947266,
      "activations/layer23_attention_weight_min": -24.71919822692871,
      "activations/layer2_attention_weight_max": 30.39814567565918,
      "activations/layer2_attention_weight_min": -30.348424911499023,
      "activations/layer3_attention_weight_max": 87.0176773071289,
      "activations/layer3_attention_weight_min": -87.97309875488281,
      "activations/layer4_attention_weight_max": 85.32447814941406,
      "activations/layer4_attention_weight_min": -85.12842559814453,
      "activations/layer5_attention_weight_max": 62.41621398925781,
      "activations/layer5_attention_weight_min": -72.45357513427734,
      "activations/layer6_attention_weight_max": 49.74433898925781,
      "activations/layer6_attention_weight_min": -51.87738800048828,
      "activations/layer7_attention_weight_max": 68.85333251953125,
      "activations/layer7_attention_weight_min": -66.45309448242188,
      "activations/layer8_attention_weight_max": 49.59092330932617,
      "activations/layer8_attention_weight_min": -51.6799430847168,
      "activations/layer9_attention_weight_max": 46.82699203491211,
      "activations/layer9_attention_weight_min": -48.13284683227539,
      "epoch": 20.97,
      "learning_rate": 1.4900757575757575e-05,
      "loss": 2.7039,
      "step": 360950
    },
    {
      "activations/layer0_attention_weight_max": 15.760607719421387,
      "activations/layer0_attention_weight_min": -13.793773651123047,
      "activations/layer10_attention_weight_max": 36.980796813964844,
      "activations/layer10_attention_weight_min": -38.082820892333984,
      "activations/layer11_attention_weight_max": 32.29245376586914,
      "activations/layer11_attention_weight_min": -36.61280822753906,
      "activations/layer12_attention_weight_max": 19.570798873901367,
      "activations/layer12_attention_weight_min": -26.173662185668945,
      "activations/layer13_attention_weight_max": 32.80542755126953,
      "activations/layer13_attention_weight_min": -27.662189483642578,
      "activations/layer14_attention_weight_max": 38.37611389160156,
      "activations/layer14_attention_weight_min": -29.944795608520508,
      "activations/layer15_attention_weight_max": 36.889644622802734,
      "activations/layer15_attention_weight_min": -29.41049575805664,
      "activations/layer16_attention_weight_max": 39.01346206665039,
      "activations/layer16_attention_weight_min": -35.66305923461914,
      "activations/layer17_attention_weight_max": 56.29970169067383,
      "activations/layer17_attention_weight_min": -46.51217269897461,
      "activations/layer18_attention_weight_max": 56.258609771728516,
      "activations/layer18_attention_weight_min": -42.4514045715332,
      "activations/layer19_attention_weight_max": 25.701940536499023,
      "activations/layer19_attention_weight_min": -24.160907745361328,
      "activations/layer1_attention_weight_max": 15.919709205627441,
      "activations/layer1_attention_weight_min": -13.793333053588867,
      "activations/layer20_attention_weight_max": 23.10839080810547,
      "activations/layer20_attention_weight_min": -22.49640655517578,
      "activations/layer21_attention_weight_max": 39.170555114746094,
      "activations/layer21_attention_weight_min": -21.964067459106445,
      "activations/layer22_attention_weight_max": 28.896875381469727,
      "activations/layer22_attention_weight_min": -24.63437271118164,
      "activations/layer23_attention_weight_max": 37.756378173828125,
      "activations/layer23_attention_weight_min": -24.45821762084961,
      "activations/layer2_attention_weight_max": 32.40559387207031,
      "activations/layer2_attention_weight_min": -30.031179428100586,
      "activations/layer3_attention_weight_max": 87.83541107177734,
      "activations/layer3_attention_weight_min": -87.51322937011719,
      "activations/layer4_attention_weight_max": 84.46544647216797,
      "activations/layer4_attention_weight_min": -81.50559997558594,
      "activations/layer5_attention_weight_max": 60.972286224365234,
      "activations/layer5_attention_weight_min": -73.9014892578125,
      "activations/layer6_attention_weight_max": 49.30316925048828,
      "activations/layer6_attention_weight_min": -52.02421188354492,
      "activations/layer7_attention_weight_max": 66.69575500488281,
      "activations/layer7_attention_weight_min": -65.2419662475586,
      "activations/layer8_attention_weight_max": 46.166229248046875,
      "activations/layer8_attention_weight_min": -52.370967864990234,
      "activations/layer9_attention_weight_max": 46.19816207885742,
      "activations/layer9_attention_weight_min": -47.78322982788086,
      "epoch": 20.98,
      "learning_rate": 1.488181818181818e-05,
      "loss": 2.6999,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4702,
      "eval_samples_per_second": 506.955,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4702,
      "eval_openwebtext_samples_per_second": 506.955,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9862,
      "eval_wikitext_samples_per_second": 229.583,
      "step": 361000
    },
    {
      "epoch": 20.98,
      "eval_lambada_loss": 2.416015625,
      "eval_lambada_ppl": 11.201140741117625,
      "eval_lambada_runtime": 9.5235,
      "eval_lambada_samples_per_second": 511.26,
      "step": 361000
    },
    {
      "activations/layer0_attention_weight_max": 15.502067565917969,
      "activations/layer0_attention_weight_min": -14.838543891906738,
      "activations/layer10_attention_weight_max": 37.409175872802734,
      "activations/layer10_attention_weight_min": -36.94837951660156,
      "activations/layer11_attention_weight_max": 34.30014419555664,
      "activations/layer11_attention_weight_min": -34.61273956298828,
      "activations/layer12_attention_weight_max": 21.1456298828125,
      "activations/layer12_attention_weight_min": -26.510480880737305,
      "activations/layer13_attention_weight_max": 38.3338737487793,
      "activations/layer13_attention_weight_min": -28.49228286743164,
      "activations/layer14_attention_weight_max": 38.994964599609375,
      "activations/layer14_attention_weight_min": -32.6931266784668,
      "activations/layer15_attention_weight_max": 38.12118911743164,
      "activations/layer15_attention_weight_min": -32.73186492919922,
      "activations/layer16_attention_weight_max": 37.163883209228516,
      "activations/layer16_attention_weight_min": -34.43233108520508,
      "activations/layer17_attention_weight_max": 56.799373626708984,
      "activations/layer17_attention_weight_min": -46.362491607666016,
      "activations/layer18_attention_weight_max": 54.265830993652344,
      "activations/layer18_attention_weight_min": -43.4147834777832,
      "activations/layer19_attention_weight_max": 32.393680572509766,
      "activations/layer19_attention_weight_min": -25.38257598876953,
      "activations/layer1_attention_weight_max": 15.21186637878418,
      "activations/layer1_attention_weight_min": -14.879380226135254,
      "activations/layer20_attention_weight_max": 28.083410263061523,
      "activations/layer20_attention_weight_min": -20.553659439086914,
      "activations/layer21_attention_weight_max": 42.67135238647461,
      "activations/layer21_attention_weight_min": -23.494705200195312,
      "activations/layer22_attention_weight_max": 34.14780044555664,
      "activations/layer22_attention_weight_min": -26.781675338745117,
      "activations/layer23_attention_weight_max": 45.65058135986328,
      "activations/layer23_attention_weight_min": -23.546451568603516,
      "activations/layer2_attention_weight_max": 32.95854568481445,
      "activations/layer2_attention_weight_min": -31.404218673706055,
      "activations/layer3_attention_weight_max": 89.5234375,
      "activations/layer3_attention_weight_min": -88.93280792236328,
      "activations/layer4_attention_weight_max": 83.29288482666016,
      "activations/layer4_attention_weight_min": -83.46773529052734,
      "activations/layer5_attention_weight_max": 63.65667724609375,
      "activations/layer5_attention_weight_min": -71.09407043457031,
      "activations/layer6_attention_weight_max": 50.457237243652344,
      "activations/layer6_attention_weight_min": -49.750633239746094,
      "activations/layer7_attention_weight_max": 65.97715759277344,
      "activations/layer7_attention_weight_min": -68.82947540283203,
      "activations/layer8_attention_weight_max": 47.74795913696289,
      "activations/layer8_attention_weight_min": -50.83729934692383,
      "activations/layer9_attention_weight_max": 47.656410217285156,
      "activations/layer9_attention_weight_min": -47.763404846191406,
      "epoch": 20.98,
      "learning_rate": 1.4862878787878786e-05,
      "loss": 2.723,
      "step": 361050
    },
    {
      "activations/layer0_attention_weight_max": 16.169788360595703,
      "activations/layer0_attention_weight_min": -14.399101257324219,
      "activations/layer10_attention_weight_max": 39.53451919555664,
      "activations/layer10_attention_weight_min": -38.010536193847656,
      "activations/layer11_attention_weight_max": 34.65660858154297,
      "activations/layer11_attention_weight_min": -35.93098831176758,
      "activations/layer12_attention_weight_max": 19.10338020324707,
      "activations/layer12_attention_weight_min": -24.14891815185547,
      "activations/layer13_attention_weight_max": 33.6172981262207,
      "activations/layer13_attention_weight_min": -31.999317169189453,
      "activations/layer14_attention_weight_max": 34.32866287231445,
      "activations/layer14_attention_weight_min": -32.121726989746094,
      "activations/layer15_attention_weight_max": 32.4233283996582,
      "activations/layer15_attention_weight_min": -30.530357360839844,
      "activations/layer16_attention_weight_max": 34.7168083190918,
      "activations/layer16_attention_weight_min": -34.54168701171875,
      "activations/layer17_attention_weight_max": 51.188907623291016,
      "activations/layer17_attention_weight_min": -47.210819244384766,
      "activations/layer18_attention_weight_max": 50.89237976074219,
      "activations/layer18_attention_weight_min": -43.08176040649414,
      "activations/layer19_attention_weight_max": 26.152570724487305,
      "activations/layer19_attention_weight_min": -23.950927734375,
      "activations/layer1_attention_weight_max": 14.816637992858887,
      "activations/layer1_attention_weight_min": -15.517603874206543,
      "activations/layer20_attention_weight_max": 24.475950241088867,
      "activations/layer20_attention_weight_min": -22.10923957824707,
      "activations/layer21_attention_weight_max": 33.98350524902344,
      "activations/layer21_attention_weight_min": -23.979398727416992,
      "activations/layer22_attention_weight_max": 29.052583694458008,
      "activations/layer22_attention_weight_min": -25.690441131591797,
      "activations/layer23_attention_weight_max": 39.59588623046875,
      "activations/layer23_attention_weight_min": -28.41014862060547,
      "activations/layer2_attention_weight_max": 32.57542037963867,
      "activations/layer2_attention_weight_min": -31.564617156982422,
      "activations/layer3_attention_weight_max": 91.58271789550781,
      "activations/layer3_attention_weight_min": -95.78453826904297,
      "activations/layer4_attention_weight_max": 89.01793670654297,
      "activations/layer4_attention_weight_min": -90.32720184326172,
      "activations/layer5_attention_weight_max": 62.355018615722656,
      "activations/layer5_attention_weight_min": -73.96707153320312,
      "activations/layer6_attention_weight_max": 52.86581802368164,
      "activations/layer6_attention_weight_min": -55.56669235229492,
      "activations/layer7_attention_weight_max": 67.5925521850586,
      "activations/layer7_attention_weight_min": -67.11333465576172,
      "activations/layer8_attention_weight_max": 49.15021896362305,
      "activations/layer8_attention_weight_min": -52.06787109375,
      "activations/layer9_attention_weight_max": 46.493873596191406,
      "activations/layer9_attention_weight_min": -48.269508361816406,
      "epoch": 20.98,
      "learning_rate": 1.4843939393939391e-05,
      "loss": 2.7065,
      "step": 361100
    },
    {
      "activations/layer0_attention_weight_max": 16.50414276123047,
      "activations/layer0_attention_weight_min": -14.041863441467285,
      "activations/layer10_attention_weight_max": 35.0438232421875,
      "activations/layer10_attention_weight_min": -33.25140380859375,
      "activations/layer11_attention_weight_max": 32.320716857910156,
      "activations/layer11_attention_weight_min": -35.07035446166992,
      "activations/layer12_attention_weight_max": 17.857507705688477,
      "activations/layer12_attention_weight_min": -25.79043960571289,
      "activations/layer13_attention_weight_max": 29.570392608642578,
      "activations/layer13_attention_weight_min": -24.936925888061523,
      "activations/layer14_attention_weight_max": 33.37544250488281,
      "activations/layer14_attention_weight_min": -29.58162498474121,
      "activations/layer15_attention_weight_max": 31.30647087097168,
      "activations/layer15_attention_weight_min": -29.434446334838867,
      "activations/layer16_attention_weight_max": 32.63864517211914,
      "activations/layer16_attention_weight_min": -31.302885055541992,
      "activations/layer17_attention_weight_max": 48.03916549682617,
      "activations/layer17_attention_weight_min": -42.113746643066406,
      "activations/layer18_attention_weight_max": 45.257957458496094,
      "activations/layer18_attention_weight_min": -37.90412139892578,
      "activations/layer19_attention_weight_max": 22.533525466918945,
      "activations/layer19_attention_weight_min": -20.44679069519043,
      "activations/layer1_attention_weight_max": 16.073436737060547,
      "activations/layer1_attention_weight_min": -15.78466510772705,
      "activations/layer20_attention_weight_max": 22.48581314086914,
      "activations/layer20_attention_weight_min": -19.873931884765625,
      "activations/layer21_attention_weight_max": 32.41274642944336,
      "activations/layer21_attention_weight_min": -21.65265655517578,
      "activations/layer22_attention_weight_max": 27.156524658203125,
      "activations/layer22_attention_weight_min": -24.056671142578125,
      "activations/layer23_attention_weight_max": 32.086021423339844,
      "activations/layer23_attention_weight_min": -23.183181762695312,
      "activations/layer2_attention_weight_max": 32.1616096496582,
      "activations/layer2_attention_weight_min": -31.810773849487305,
      "activations/layer3_attention_weight_max": 88.9421157836914,
      "activations/layer3_attention_weight_min": -91.03041076660156,
      "activations/layer4_attention_weight_max": 81.52107238769531,
      "activations/layer4_attention_weight_min": -83.5800552368164,
      "activations/layer5_attention_weight_max": 59.87416076660156,
      "activations/layer5_attention_weight_min": -68.12553405761719,
      "activations/layer6_attention_weight_max": 46.92036437988281,
      "activations/layer6_attention_weight_min": -47.960880279541016,
      "activations/layer7_attention_weight_max": 65.07376098632812,
      "activations/layer7_attention_weight_min": -65.46955108642578,
      "activations/layer8_attention_weight_max": 43.71491622924805,
      "activations/layer8_attention_weight_min": -50.76433181762695,
      "activations/layer9_attention_weight_max": 41.56716537475586,
      "activations/layer9_attention_weight_min": -46.270626068115234,
      "epoch": 20.98,
      "learning_rate": 1.4824999999999998e-05,
      "loss": 2.7234,
      "step": 361150
    },
    {
      "activations/layer0_attention_weight_max": 15.608861923217773,
      "activations/layer0_attention_weight_min": -14.041757583618164,
      "activations/layer10_attention_weight_max": 37.70988845825195,
      "activations/layer10_attention_weight_min": -38.418128967285156,
      "activations/layer11_attention_weight_max": 35.73758316040039,
      "activations/layer11_attention_weight_min": -36.12804412841797,
      "activations/layer12_attention_weight_max": 20.206838607788086,
      "activations/layer12_attention_weight_min": -22.540348052978516,
      "activations/layer13_attention_weight_max": 35.88985824584961,
      "activations/layer13_attention_weight_min": -30.72188377380371,
      "activations/layer14_attention_weight_max": 36.19612121582031,
      "activations/layer14_attention_weight_min": -33.233150482177734,
      "activations/layer15_attention_weight_max": 36.38690185546875,
      "activations/layer15_attention_weight_min": -33.649322509765625,
      "activations/layer16_attention_weight_max": 37.64749526977539,
      "activations/layer16_attention_weight_min": -33.859657287597656,
      "activations/layer17_attention_weight_max": 53.715248107910156,
      "activations/layer17_attention_weight_min": -48.7391242980957,
      "activations/layer18_attention_weight_max": 52.03409957885742,
      "activations/layer18_attention_weight_min": -45.01693344116211,
      "activations/layer19_attention_weight_max": 26.269947052001953,
      "activations/layer19_attention_weight_min": -25.623098373413086,
      "activations/layer1_attention_weight_max": 16.006532669067383,
      "activations/layer1_attention_weight_min": -15.689437866210938,
      "activations/layer20_attention_weight_max": 23.567684173583984,
      "activations/layer20_attention_weight_min": -22.385154724121094,
      "activations/layer21_attention_weight_max": 44.19940185546875,
      "activations/layer21_attention_weight_min": -24.71086883544922,
      "activations/layer22_attention_weight_max": 28.667226791381836,
      "activations/layer22_attention_weight_min": -25.89053726196289,
      "activations/layer23_attention_weight_max": 36.312660217285156,
      "activations/layer23_attention_weight_min": -25.089630126953125,
      "activations/layer2_attention_weight_max": 33.28751754760742,
      "activations/layer2_attention_weight_min": -31.05543327331543,
      "activations/layer3_attention_weight_max": 86.88554382324219,
      "activations/layer3_attention_weight_min": -83.47372436523438,
      "activations/layer4_attention_weight_max": 85.3691177368164,
      "activations/layer4_attention_weight_min": -82.10183715820312,
      "activations/layer5_attention_weight_max": 61.31809997558594,
      "activations/layer5_attention_weight_min": -73.05796813964844,
      "activations/layer6_attention_weight_max": 49.60983657836914,
      "activations/layer6_attention_weight_min": -50.492332458496094,
      "activations/layer7_attention_weight_max": 68.85803985595703,
      "activations/layer7_attention_weight_min": -66.0207748413086,
      "activations/layer8_attention_weight_max": 49.02737808227539,
      "activations/layer8_attention_weight_min": -50.61759948730469,
      "activations/layer9_attention_weight_max": 46.235408782958984,
      "activations/layer9_attention_weight_min": -48.176177978515625,
      "epoch": 20.99,
      "learning_rate": 1.4806060606060604e-05,
      "loss": 2.7094,
      "step": 361200
    },
    {
      "activations/layer0_attention_weight_max": 16.260046005249023,
      "activations/layer0_attention_weight_min": -13.940437316894531,
      "activations/layer10_attention_weight_max": 34.53816604614258,
      "activations/layer10_attention_weight_min": -34.3633918762207,
      "activations/layer11_attention_weight_max": 32.05754852294922,
      "activations/layer11_attention_weight_min": -33.24857711791992,
      "activations/layer12_attention_weight_max": 19.50775146484375,
      "activations/layer12_attention_weight_min": -26.75596809387207,
      "activations/layer13_attention_weight_max": 32.518516540527344,
      "activations/layer13_attention_weight_min": -28.192981719970703,
      "activations/layer14_attention_weight_max": 35.39968490600586,
      "activations/layer14_attention_weight_min": -32.5676383972168,
      "activations/layer15_attention_weight_max": 34.01436996459961,
      "activations/layer15_attention_weight_min": -33.69011306762695,
      "activations/layer16_attention_weight_max": 34.86302185058594,
      "activations/layer16_attention_weight_min": -35.563350677490234,
      "activations/layer17_attention_weight_max": 49.81495666503906,
      "activations/layer17_attention_weight_min": -51.3177375793457,
      "activations/layer18_attention_weight_max": 46.877471923828125,
      "activations/layer18_attention_weight_min": -48.68995666503906,
      "activations/layer19_attention_weight_max": 25.8083553314209,
      "activations/layer19_attention_weight_min": -28.05536460876465,
      "activations/layer1_attention_weight_max": 15.990950584411621,
      "activations/layer1_attention_weight_min": -13.869885444641113,
      "activations/layer20_attention_weight_max": 23.1676082611084,
      "activations/layer20_attention_weight_min": -24.753372192382812,
      "activations/layer21_attention_weight_max": 38.818904876708984,
      "activations/layer21_attention_weight_min": -23.599393844604492,
      "activations/layer22_attention_weight_max": 30.616044998168945,
      "activations/layer22_attention_weight_min": -26.46681022644043,
      "activations/layer23_attention_weight_max": 34.913047790527344,
      "activations/layer23_attention_weight_min": -23.32712173461914,
      "activations/layer2_attention_weight_max": 32.018775939941406,
      "activations/layer2_attention_weight_min": -31.718276977539062,
      "activations/layer3_attention_weight_max": 88.6611099243164,
      "activations/layer3_attention_weight_min": -91.05858612060547,
      "activations/layer4_attention_weight_max": 83.53515625,
      "activations/layer4_attention_weight_min": -84.3141098022461,
      "activations/layer5_attention_weight_max": 59.60470199584961,
      "activations/layer5_attention_weight_min": -69.13739013671875,
      "activations/layer6_attention_weight_max": 48.96995544433594,
      "activations/layer6_attention_weight_min": -51.273033142089844,
      "activations/layer7_attention_weight_max": 63.27008056640625,
      "activations/layer7_attention_weight_min": -64.28821563720703,
      "activations/layer8_attention_weight_max": 45.28717803955078,
      "activations/layer8_attention_weight_min": -52.86579513549805,
      "activations/layer9_attention_weight_max": 43.964874267578125,
      "activations/layer9_attention_weight_min": -44.82711410522461,
      "epoch": 20.99,
      "learning_rate": 1.478712121212121e-05,
      "loss": 2.7084,
      "step": 361250
    },
    {
      "activations/layer0_attention_weight_max": 16.05609130859375,
      "activations/layer0_attention_weight_min": -14.032055854797363,
      "activations/layer10_attention_weight_max": 36.586700439453125,
      "activations/layer10_attention_weight_min": -35.54121017456055,
      "activations/layer11_attention_weight_max": 34.10517883300781,
      "activations/layer11_attention_weight_min": -34.16449737548828,
      "activations/layer12_attention_weight_max": 18.249542236328125,
      "activations/layer12_attention_weight_min": -22.298160552978516,
      "activations/layer13_attention_weight_max": 36.754695892333984,
      "activations/layer13_attention_weight_min": -28.05325698852539,
      "activations/layer14_attention_weight_max": 34.974361419677734,
      "activations/layer14_attention_weight_min": -30.5938663482666,
      "activations/layer15_attention_weight_max": 32.89342498779297,
      "activations/layer15_attention_weight_min": -30.67389488220215,
      "activations/layer16_attention_weight_max": 33.36473846435547,
      "activations/layer16_attention_weight_min": -32.37283706665039,
      "activations/layer17_attention_weight_max": 52.596431732177734,
      "activations/layer17_attention_weight_min": -48.355491638183594,
      "activations/layer18_attention_weight_max": 46.36978530883789,
      "activations/layer18_attention_weight_min": -41.633026123046875,
      "activations/layer19_attention_weight_max": 25.41705322265625,
      "activations/layer19_attention_weight_min": -21.953535079956055,
      "activations/layer1_attention_weight_max": 15.681634902954102,
      "activations/layer1_attention_weight_min": -13.831045150756836,
      "activations/layer20_attention_weight_max": 22.66680908203125,
      "activations/layer20_attention_weight_min": -23.028911590576172,
      "activations/layer21_attention_weight_max": 32.44947052001953,
      "activations/layer21_attention_weight_min": -24.37630271911621,
      "activations/layer22_attention_weight_max": 29.203550338745117,
      "activations/layer22_attention_weight_min": -26.896711349487305,
      "activations/layer23_attention_weight_max": 37.55046463012695,
      "activations/layer23_attention_weight_min": -29.367534637451172,
      "activations/layer2_attention_weight_max": 31.974102020263672,
      "activations/layer2_attention_weight_min": -29.829429626464844,
      "activations/layer3_attention_weight_max": 89.21532440185547,
      "activations/layer3_attention_weight_min": -92.81285095214844,
      "activations/layer4_attention_weight_max": 86.40521240234375,
      "activations/layer4_attention_weight_min": -88.15524291992188,
      "activations/layer5_attention_weight_max": 63.8083610534668,
      "activations/layer5_attention_weight_min": -71.17301940917969,
      "activations/layer6_attention_weight_max": 50.53115463256836,
      "activations/layer6_attention_weight_min": -53.149452209472656,
      "activations/layer7_attention_weight_max": 68.521484375,
      "activations/layer7_attention_weight_min": -64.00344848632812,
      "activations/layer8_attention_weight_max": 47.791751861572266,
      "activations/layer8_attention_weight_min": -52.8997917175293,
      "activations/layer9_attention_weight_max": 44.75032424926758,
      "activations/layer9_attention_weight_min": -46.83686447143555,
      "epoch": 20.99,
      "learning_rate": 1.4768181818181818e-05,
      "loss": 2.7141,
      "step": 361300
    },
    {
      "activations/layer0_attention_weight_max": 16.38048553466797,
      "activations/layer0_attention_weight_min": -13.799169540405273,
      "activations/layer10_attention_weight_max": 36.583717346191406,
      "activations/layer10_attention_weight_min": -35.549190521240234,
      "activations/layer11_attention_weight_max": 34.935768127441406,
      "activations/layer11_attention_weight_min": -36.720252990722656,
      "activations/layer12_attention_weight_max": 19.79205894470215,
      "activations/layer12_attention_weight_min": -25.387969970703125,
      "activations/layer13_attention_weight_max": 47.76679992675781,
      "activations/layer13_attention_weight_min": -31.256967544555664,
      "activations/layer14_attention_weight_max": 37.153778076171875,
      "activations/layer14_attention_weight_min": -31.099056243896484,
      "activations/layer15_attention_weight_max": 35.464420318603516,
      "activations/layer15_attention_weight_min": -30.765201568603516,
      "activations/layer16_attention_weight_max": 35.90852737426758,
      "activations/layer16_attention_weight_min": -34.39969253540039,
      "activations/layer17_attention_weight_max": 52.78571319580078,
      "activations/layer17_attention_weight_min": -46.39483642578125,
      "activations/layer18_attention_weight_max": 49.82358169555664,
      "activations/layer18_attention_weight_min": -40.032527923583984,
      "activations/layer19_attention_weight_max": 27.805044174194336,
      "activations/layer19_attention_weight_min": -22.298368453979492,
      "activations/layer1_attention_weight_max": 15.550482749938965,
      "activations/layer1_attention_weight_min": -15.167431831359863,
      "activations/layer20_attention_weight_max": 23.428647994995117,
      "activations/layer20_attention_weight_min": -23.892356872558594,
      "activations/layer21_attention_weight_max": 45.77206039428711,
      "activations/layer21_attention_weight_min": -26.668216705322266,
      "activations/layer22_attention_weight_max": 32.6057014465332,
      "activations/layer22_attention_weight_min": -25.80621910095215,
      "activations/layer23_attention_weight_max": 38.659423828125,
      "activations/layer23_attention_weight_min": -25.59541130065918,
      "activations/layer2_attention_weight_max": 32.74545669555664,
      "activations/layer2_attention_weight_min": -31.448471069335938,
      "activations/layer3_attention_weight_max": 86.14300537109375,
      "activations/layer3_attention_weight_min": -87.13858032226562,
      "activations/layer4_attention_weight_max": 83.31483459472656,
      "activations/layer4_attention_weight_min": -80.8745346069336,
      "activations/layer5_attention_weight_max": 61.26568603515625,
      "activations/layer5_attention_weight_min": -70.33071899414062,
      "activations/layer6_attention_weight_max": 49.89800262451172,
      "activations/layer6_attention_weight_min": -49.4932746887207,
      "activations/layer7_attention_weight_max": 68.10601806640625,
      "activations/layer7_attention_weight_min": -64.31565856933594,
      "activations/layer8_attention_weight_max": 48.546348571777344,
      "activations/layer8_attention_weight_min": -52.946434020996094,
      "activations/layer9_attention_weight_max": 45.41703414916992,
      "activations/layer9_attention_weight_min": -45.27436447143555,
      "epoch": 21.0,
      "learning_rate": 1.4749242424242424e-05,
      "loss": 2.7283,
      "step": 361350
    },
    {
      "activations/layer0_attention_weight_max": 16.031618118286133,
      "activations/layer0_attention_weight_min": -13.970609664916992,
      "activations/layer10_attention_weight_max": 36.807106018066406,
      "activations/layer10_attention_weight_min": -34.8853759765625,
      "activations/layer11_attention_weight_max": 33.051177978515625,
      "activations/layer11_attention_weight_min": -33.430938720703125,
      "activations/layer12_attention_weight_max": 19.5068416595459,
      "activations/layer12_attention_weight_min": -22.0595760345459,
      "activations/layer13_attention_weight_max": 36.23635482788086,
      "activations/layer13_attention_weight_min": -25.62496566772461,
      "activations/layer14_attention_weight_max": 37.90962219238281,
      "activations/layer14_attention_weight_min": -29.693166732788086,
      "activations/layer15_attention_weight_max": 33.43592834472656,
      "activations/layer15_attention_weight_min": -29.968914031982422,
      "activations/layer16_attention_weight_max": 33.531803131103516,
      "activations/layer16_attention_weight_min": -31.241796493530273,
      "activations/layer17_attention_weight_max": 50.450077056884766,
      "activations/layer17_attention_weight_min": -44.12690353393555,
      "activations/layer18_attention_weight_max": 49.39021682739258,
      "activations/layer18_attention_weight_min": -40.18793487548828,
      "activations/layer19_attention_weight_max": 24.828222274780273,
      "activations/layer19_attention_weight_min": -23.268220901489258,
      "activations/layer1_attention_weight_max": 16.09746742248535,
      "activations/layer1_attention_weight_min": -13.556872367858887,
      "activations/layer20_attention_weight_max": 24.620328903198242,
      "activations/layer20_attention_weight_min": -22.05565643310547,
      "activations/layer21_attention_weight_max": 36.02583694458008,
      "activations/layer21_attention_weight_min": -21.200546264648438,
      "activations/layer22_attention_weight_max": 29.445770263671875,
      "activations/layer22_attention_weight_min": -25.05030632019043,
      "activations/layer23_attention_weight_max": 35.472076416015625,
      "activations/layer23_attention_weight_min": -25.440799713134766,
      "activations/layer2_attention_weight_max": 32.53683090209961,
      "activations/layer2_attention_weight_min": -29.84442138671875,
      "activations/layer3_attention_weight_max": 86.49150848388672,
      "activations/layer3_attention_weight_min": -84.91062927246094,
      "activations/layer4_attention_weight_max": 85.59745788574219,
      "activations/layer4_attention_weight_min": -81.38703918457031,
      "activations/layer5_attention_weight_max": 58.820823669433594,
      "activations/layer5_attention_weight_min": -70.52449798583984,
      "activations/layer6_attention_weight_max": 47.609867095947266,
      "activations/layer6_attention_weight_min": -50.56059646606445,
      "activations/layer7_attention_weight_max": 67.173583984375,
      "activations/layer7_attention_weight_min": -66.38575744628906,
      "activations/layer8_attention_weight_max": 44.23740005493164,
      "activations/layer8_attention_weight_min": -51.19934844970703,
      "activations/layer9_attention_weight_max": 41.92214584350586,
      "activations/layer9_attention_weight_min": -46.682804107666016,
      "epoch": 21.0,
      "learning_rate": 1.4730681818181818e-05,
      "loss": 2.7245,
      "step": 361400
    },
    {
      "activations/layer0_attention_weight_max": 15.580482482910156,
      "activations/layer0_attention_weight_min": -13.722957611083984,
      "activations/layer10_attention_weight_max": 35.126007080078125,
      "activations/layer10_attention_weight_min": -36.16023635864258,
      "activations/layer11_attention_weight_max": 34.073768615722656,
      "activations/layer11_attention_weight_min": -34.30664825439453,
      "activations/layer12_attention_weight_max": 19.569406509399414,
      "activations/layer12_attention_weight_min": -25.03277015686035,
      "activations/layer13_attention_weight_max": 44.14934539794922,
      "activations/layer13_attention_weight_min": -26.643770217895508,
      "activations/layer14_attention_weight_max": 37.22854995727539,
      "activations/layer14_attention_weight_min": -30.382251739501953,
      "activations/layer15_attention_weight_max": 36.211456298828125,
      "activations/layer15_attention_weight_min": -31.15721321105957,
      "activations/layer16_attention_weight_max": 38.12187194824219,
      "activations/layer16_attention_weight_min": -35.04348373413086,
      "activations/layer17_attention_weight_max": 57.64515686035156,
      "activations/layer17_attention_weight_min": -48.21367645263672,
      "activations/layer18_attention_weight_max": 55.777164459228516,
      "activations/layer18_attention_weight_min": -42.990535736083984,
      "activations/layer19_attention_weight_max": 29.7310848236084,
      "activations/layer19_attention_weight_min": -23.165143966674805,
      "activations/layer1_attention_weight_max": 15.63134765625,
      "activations/layer1_attention_weight_min": -13.905306816101074,
      "activations/layer20_attention_weight_max": 28.724586486816406,
      "activations/layer20_attention_weight_min": -22.742517471313477,
      "activations/layer21_attention_weight_max": 46.74878692626953,
      "activations/layer21_attention_weight_min": -23.881816864013672,
      "activations/layer22_attention_weight_max": 37.3615837097168,
      "activations/layer22_attention_weight_min": -28.291093826293945,
      "activations/layer23_attention_weight_max": 44.87108612060547,
      "activations/layer23_attention_weight_min": -24.691938400268555,
      "activations/layer2_attention_weight_max": 31.704307556152344,
      "activations/layer2_attention_weight_min": -29.274700164794922,
      "activations/layer3_attention_weight_max": 88.3637466430664,
      "activations/layer3_attention_weight_min": -88.43080139160156,
      "activations/layer4_attention_weight_max": 83.76901245117188,
      "activations/layer4_attention_weight_min": -83.45850372314453,
      "activations/layer5_attention_weight_max": 58.549861907958984,
      "activations/layer5_attention_weight_min": -67.9536361694336,
      "activations/layer6_attention_weight_max": 50.21173858642578,
      "activations/layer6_attention_weight_min": -52.02009963989258,
      "activations/layer7_attention_weight_max": 70.0125503540039,
      "activations/layer7_attention_weight_min": -67.68486785888672,
      "activations/layer8_attention_weight_max": 47.82374954223633,
      "activations/layer8_attention_weight_min": -52.405059814453125,
      "activations/layer9_attention_weight_max": 47.52585983276367,
      "activations/layer9_attention_weight_min": -45.41812515258789,
      "epoch": 21.0,
      "learning_rate": 1.4711742424242423e-05,
      "loss": 2.7764,
      "step": 361450
    },
    {
      "activations/layer0_attention_weight_max": 15.511012077331543,
      "activations/layer0_attention_weight_min": -13.875937461853027,
      "activations/layer10_attention_weight_max": 36.6125373840332,
      "activations/layer10_attention_weight_min": -35.89162826538086,
      "activations/layer11_attention_weight_max": 35.95701599121094,
      "activations/layer11_attention_weight_min": -34.92509460449219,
      "activations/layer12_attention_weight_max": 21.188613891601562,
      "activations/layer12_attention_weight_min": -28.383472442626953,
      "activations/layer13_attention_weight_max": 37.107295989990234,
      "activations/layer13_attention_weight_min": -26.952497482299805,
      "activations/layer14_attention_weight_max": 38.859195709228516,
      "activations/layer14_attention_weight_min": -28.709980010986328,
      "activations/layer15_attention_weight_max": 39.75089645385742,
      "activations/layer15_attention_weight_min": -28.56485939025879,
      "activations/layer16_attention_weight_max": 34.610965728759766,
      "activations/layer16_attention_weight_min": -31.284650802612305,
      "activations/layer17_attention_weight_max": 49.282901763916016,
      "activations/layer17_attention_weight_min": -44.98124694824219,
      "activations/layer18_attention_weight_max": 45.764930725097656,
      "activations/layer18_attention_weight_min": -42.31878662109375,
      "activations/layer19_attention_weight_max": 24.226301193237305,
      "activations/layer19_attention_weight_min": -22.42494773864746,
      "activations/layer1_attention_weight_max": 14.734582901000977,
      "activations/layer1_attention_weight_min": -13.710776329040527,
      "activations/layer20_attention_weight_max": 23.520349502563477,
      "activations/layer20_attention_weight_min": -20.933774948120117,
      "activations/layer21_attention_weight_max": 35.4922981262207,
      "activations/layer21_attention_weight_min": -22.144067764282227,
      "activations/layer22_attention_weight_max": 31.210996627807617,
      "activations/layer22_attention_weight_min": -27.085126876831055,
      "activations/layer23_attention_weight_max": 38.417030334472656,
      "activations/layer23_attention_weight_min": -25.61079216003418,
      "activations/layer2_attention_weight_max": 32.29559326171875,
      "activations/layer2_attention_weight_min": -32.657596588134766,
      "activations/layer3_attention_weight_max": 93.73822784423828,
      "activations/layer3_attention_weight_min": -93.95401000976562,
      "activations/layer4_attention_weight_max": 85.77672576904297,
      "activations/layer4_attention_weight_min": -87.63217163085938,
      "activations/layer5_attention_weight_max": 60.22035217285156,
      "activations/layer5_attention_weight_min": -72.76766967773438,
      "activations/layer6_attention_weight_max": 48.354915618896484,
      "activations/layer6_attention_weight_min": -51.07725143432617,
      "activations/layer7_attention_weight_max": 69.52705383300781,
      "activations/layer7_attention_weight_min": -67.96243286132812,
      "activations/layer8_attention_weight_max": 53.11979293823242,
      "activations/layer8_attention_weight_min": -52.03065490722656,
      "activations/layer9_attention_weight_max": 51.31990432739258,
      "activations/layer9_attention_weight_min": -49.16120529174805,
      "epoch": 21.01,
      "learning_rate": 1.469280303030303e-05,
      "loss": 2.6932,
      "step": 361500
    },
    {
      "activations/layer0_attention_weight_max": 15.838066101074219,
      "activations/layer0_attention_weight_min": -13.819743156433105,
      "activations/layer10_attention_weight_max": 32.99690628051758,
      "activations/layer10_attention_weight_min": -34.58994674682617,
      "activations/layer11_attention_weight_max": 30.921201705932617,
      "activations/layer11_attention_weight_min": -31.766904830932617,
      "activations/layer12_attention_weight_max": 18.329166412353516,
      "activations/layer12_attention_weight_min": -24.38551139831543,
      "activations/layer13_attention_weight_max": 33.937889099121094,
      "activations/layer13_attention_weight_min": -28.52886390686035,
      "activations/layer14_attention_weight_max": 36.12613296508789,
      "activations/layer14_attention_weight_min": -32.02489471435547,
      "activations/layer15_attention_weight_max": 33.86545181274414,
      "activations/layer15_attention_weight_min": -31.586475372314453,
      "activations/layer16_attention_weight_max": 34.18783950805664,
      "activations/layer16_attention_weight_min": -33.42717742919922,
      "activations/layer17_attention_weight_max": 52.91717529296875,
      "activations/layer17_attention_weight_min": -46.335472106933594,
      "activations/layer18_attention_weight_max": 48.41588592529297,
      "activations/layer18_attention_weight_min": -43.64759063720703,
      "activations/layer19_attention_weight_max": 24.66866683959961,
      "activations/layer19_attention_weight_min": -23.693851470947266,
      "activations/layer1_attention_weight_max": 15.02596664428711,
      "activations/layer1_attention_weight_min": -14.45273208618164,
      "activations/layer20_attention_weight_max": 23.060646057128906,
      "activations/layer20_attention_weight_min": -22.96315574645996,
      "activations/layer21_attention_weight_max": 35.64723587036133,
      "activations/layer21_attention_weight_min": -21.478214263916016,
      "activations/layer22_attention_weight_max": 32.549072265625,
      "activations/layer22_attention_weight_min": -26.35683822631836,
      "activations/layer23_attention_weight_max": 35.45326232910156,
      "activations/layer23_attention_weight_min": -24.77215576171875,
      "activations/layer2_attention_weight_max": 31.4948787689209,
      "activations/layer2_attention_weight_min": -28.386394500732422,
      "activations/layer3_attention_weight_max": 86.29708862304688,
      "activations/layer3_attention_weight_min": -89.63678741455078,
      "activations/layer4_attention_weight_max": 80.83196258544922,
      "activations/layer4_attention_weight_min": -82.61910247802734,
      "activations/layer5_attention_weight_max": 62.47426986694336,
      "activations/layer5_attention_weight_min": -69.79444885253906,
      "activations/layer6_attention_weight_max": 50.031002044677734,
      "activations/layer6_attention_weight_min": -50.310951232910156,
      "activations/layer7_attention_weight_max": 70.59272003173828,
      "activations/layer7_attention_weight_min": -67.12822723388672,
      "activations/layer8_attention_weight_max": 49.02680587768555,
      "activations/layer8_attention_weight_min": -47.74885940551758,
      "activations/layer9_attention_weight_max": 44.23666763305664,
      "activations/layer9_attention_weight_min": -46.69468688964844,
      "epoch": 21.01,
      "learning_rate": 1.4673863636363636e-05,
      "loss": 2.7006,
      "step": 361550
    },
    {
      "activations/layer0_attention_weight_max": 15.413041114807129,
      "activations/layer0_attention_weight_min": -14.085007667541504,
      "activations/layer10_attention_weight_max": 35.022037506103516,
      "activations/layer10_attention_weight_min": -36.92993927001953,
      "activations/layer11_attention_weight_max": 34.69142150878906,
      "activations/layer11_attention_weight_min": -34.36192321777344,
      "activations/layer12_attention_weight_max": 21.975584030151367,
      "activations/layer12_attention_weight_min": -26.272903442382812,
      "activations/layer13_attention_weight_max": 42.77268981933594,
      "activations/layer13_attention_weight_min": -29.65853500366211,
      "activations/layer14_attention_weight_max": 38.346561431884766,
      "activations/layer14_attention_weight_min": -30.94995880126953,
      "activations/layer15_attention_weight_max": 37.77888107299805,
      "activations/layer15_attention_weight_min": -31.5771541595459,
      "activations/layer16_attention_weight_max": 36.83497619628906,
      "activations/layer16_attention_weight_min": -32.89198303222656,
      "activations/layer17_attention_weight_max": 56.63182067871094,
      "activations/layer17_attention_weight_min": -46.87379837036133,
      "activations/layer18_attention_weight_max": 52.716331481933594,
      "activations/layer18_attention_weight_min": -41.57526779174805,
      "activations/layer19_attention_weight_max": 28.364654541015625,
      "activations/layer19_attention_weight_min": -23.995891571044922,
      "activations/layer1_attention_weight_max": 16.122943878173828,
      "activations/layer1_attention_weight_min": -14.124502182006836,
      "activations/layer20_attention_weight_max": 25.959062576293945,
      "activations/layer20_attention_weight_min": -22.319969177246094,
      "activations/layer21_attention_weight_max": 42.196842193603516,
      "activations/layer21_attention_weight_min": -26.36720848083496,
      "activations/layer22_attention_weight_max": 32.12522506713867,
      "activations/layer22_attention_weight_min": -25.505870819091797,
      "activations/layer23_attention_weight_max": 37.20924377441406,
      "activations/layer23_attention_weight_min": -25.131507873535156,
      "activations/layer2_attention_weight_max": 31.703001022338867,
      "activations/layer2_attention_weight_min": -29.832448959350586,
      "activations/layer3_attention_weight_max": 89.4963150024414,
      "activations/layer3_attention_weight_min": -87.49488830566406,
      "activations/layer4_attention_weight_max": 83.02247619628906,
      "activations/layer4_attention_weight_min": -82.65559387207031,
      "activations/layer5_attention_weight_max": 61.34324264526367,
      "activations/layer5_attention_weight_min": -70.68168640136719,
      "activations/layer6_attention_weight_max": 46.80647659301758,
      "activations/layer6_attention_weight_min": -48.86235427856445,
      "activations/layer7_attention_weight_max": 70.41905975341797,
      "activations/layer7_attention_weight_min": -66.44690704345703,
      "activations/layer8_attention_weight_max": 46.30709457397461,
      "activations/layer8_attention_weight_min": -48.37003707885742,
      "activations/layer9_attention_weight_max": 42.88786697387695,
      "activations/layer9_attention_weight_min": -45.72861099243164,
      "epoch": 21.01,
      "learning_rate": 1.4654924242424241e-05,
      "loss": 2.7191,
      "step": 361600
    },
    {
      "activations/layer0_attention_weight_max": 15.383577346801758,
      "activations/layer0_attention_weight_min": -13.696112632751465,
      "activations/layer10_attention_weight_max": 38.738990783691406,
      "activations/layer10_attention_weight_min": -38.444244384765625,
      "activations/layer11_attention_weight_max": 35.94221496582031,
      "activations/layer11_attention_weight_min": -35.59214782714844,
      "activations/layer12_attention_weight_max": 20.356460571289062,
      "activations/layer12_attention_weight_min": -24.42345428466797,
      "activations/layer13_attention_weight_max": 37.158607482910156,
      "activations/layer13_attention_weight_min": -28.92203140258789,
      "activations/layer14_attention_weight_max": 37.029685974121094,
      "activations/layer14_attention_weight_min": -31.277965545654297,
      "activations/layer15_attention_weight_max": 35.77993392944336,
      "activations/layer15_attention_weight_min": -31.543548583984375,
      "activations/layer16_attention_weight_max": 35.78291702270508,
      "activations/layer16_attention_weight_min": -32.98654556274414,
      "activations/layer17_attention_weight_max": 52.76767349243164,
      "activations/layer17_attention_weight_min": -45.58655548095703,
      "activations/layer18_attention_weight_max": 52.430179595947266,
      "activations/layer18_attention_weight_min": -41.15230178833008,
      "activations/layer19_attention_weight_max": 30.59952735900879,
      "activations/layer19_attention_weight_min": -24.6058406829834,
      "activations/layer1_attention_weight_max": 14.962824821472168,
      "activations/layer1_attention_weight_min": -15.06304931640625,
      "activations/layer20_attention_weight_max": 25.51509666442871,
      "activations/layer20_attention_weight_min": -23.452552795410156,
      "activations/layer21_attention_weight_max": 42.97333908081055,
      "activations/layer21_attention_weight_min": -21.291357040405273,
      "activations/layer22_attention_weight_max": 36.55272674560547,
      "activations/layer22_attention_weight_min": -26.14913558959961,
      "activations/layer23_attention_weight_max": 45.403987884521484,
      "activations/layer23_attention_weight_min": -24.94649314880371,
      "activations/layer2_attention_weight_max": 30.459266662597656,
      "activations/layer2_attention_weight_min": -31.737659454345703,
      "activations/layer3_attention_weight_max": 92.06565856933594,
      "activations/layer3_attention_weight_min": -90.98423767089844,
      "activations/layer4_attention_weight_max": 84.74065399169922,
      "activations/layer4_attention_weight_min": -86.32786560058594,
      "activations/layer5_attention_weight_max": 58.996761322021484,
      "activations/layer5_attention_weight_min": -67.53334045410156,
      "activations/layer6_attention_weight_max": 49.64630889892578,
      "activations/layer6_attention_weight_min": -51.343692779541016,
      "activations/layer7_attention_weight_max": 63.54515075683594,
      "activations/layer7_attention_weight_min": -66.2505874633789,
      "activations/layer8_attention_weight_max": 47.137413024902344,
      "activations/layer8_attention_weight_min": -51.29197311401367,
      "activations/layer9_attention_weight_max": 45.67930603027344,
      "activations/layer9_attention_weight_min": -48.878684997558594,
      "epoch": 21.01,
      "learning_rate": 1.4635984848484848e-05,
      "loss": 2.7195,
      "step": 361650
    },
    {
      "activations/layer0_attention_weight_max": 15.836403846740723,
      "activations/layer0_attention_weight_min": -13.784955978393555,
      "activations/layer10_attention_weight_max": 39.08330535888672,
      "activations/layer10_attention_weight_min": -36.9072265625,
      "activations/layer11_attention_weight_max": 33.43463897705078,
      "activations/layer11_attention_weight_min": -33.806480407714844,
      "activations/layer12_attention_weight_max": 20.953031539916992,
      "activations/layer12_attention_weight_min": -23.80473518371582,
      "activations/layer13_attention_weight_max": 34.927772521972656,
      "activations/layer13_attention_weight_min": -26.569456100463867,
      "activations/layer14_attention_weight_max": 37.55409622192383,
      "activations/layer14_attention_weight_min": -30.5766658782959,
      "activations/layer15_attention_weight_max": 34.82718276977539,
      "activations/layer15_attention_weight_min": -30.706418991088867,
      "activations/layer16_attention_weight_max": 36.196834564208984,
      "activations/layer16_attention_weight_min": -34.578643798828125,
      "activations/layer17_attention_weight_max": 50.40707015991211,
      "activations/layer17_attention_weight_min": -44.693580627441406,
      "activations/layer18_attention_weight_max": 48.30646896362305,
      "activations/layer18_attention_weight_min": -40.51700210571289,
      "activations/layer19_attention_weight_max": 26.093225479125977,
      "activations/layer19_attention_weight_min": -21.9638729095459,
      "activations/layer1_attention_weight_max": 15.183964729309082,
      "activations/layer1_attention_weight_min": -14.648138046264648,
      "activations/layer20_attention_weight_max": 24.439579010009766,
      "activations/layer20_attention_weight_min": -22.349376678466797,
      "activations/layer21_attention_weight_max": 40.4033088684082,
      "activations/layer21_attention_weight_min": -23.286039352416992,
      "activations/layer22_attention_weight_max": 29.38221549987793,
      "activations/layer22_attention_weight_min": -25.211957931518555,
      "activations/layer23_attention_weight_max": 36.74945068359375,
      "activations/layer23_attention_weight_min": -26.095062255859375,
      "activations/layer2_attention_weight_max": 30.891971588134766,
      "activations/layer2_attention_weight_min": -30.37056541442871,
      "activations/layer3_attention_weight_max": 87.36280822753906,
      "activations/layer3_attention_weight_min": -87.81171417236328,
      "activations/layer4_attention_weight_max": 84.90872955322266,
      "activations/layer4_attention_weight_min": -85.41149139404297,
      "activations/layer5_attention_weight_max": 63.49249267578125,
      "activations/layer5_attention_weight_min": -72.80926513671875,
      "activations/layer6_attention_weight_max": 50.52250671386719,
      "activations/layer6_attention_weight_min": -50.88533401489258,
      "activations/layer7_attention_weight_max": 66.50225830078125,
      "activations/layer7_attention_weight_min": -68.96697235107422,
      "activations/layer8_attention_weight_max": 46.27281188964844,
      "activations/layer8_attention_weight_min": -54.17238998413086,
      "activations/layer9_attention_weight_max": 44.81637954711914,
      "activations/layer9_attention_weight_min": -50.398406982421875,
      "epoch": 21.02,
      "learning_rate": 1.4617045454545453e-05,
      "loss": 2.7112,
      "step": 361700
    },
    {
      "activations/layer0_attention_weight_max": 15.344593048095703,
      "activations/layer0_attention_weight_min": -13.92736530303955,
      "activations/layer10_attention_weight_max": 35.32911682128906,
      "activations/layer10_attention_weight_min": -36.17776107788086,
      "activations/layer11_attention_weight_max": 33.105735778808594,
      "activations/layer11_attention_weight_min": -33.32221984863281,
      "activations/layer12_attention_weight_max": 17.57839012145996,
      "activations/layer12_attention_weight_min": -26.0169620513916,
      "activations/layer13_attention_weight_max": 35.380062103271484,
      "activations/layer13_attention_weight_min": -29.395238876342773,
      "activations/layer14_attention_weight_max": 33.09292984008789,
      "activations/layer14_attention_weight_min": -28.996843338012695,
      "activations/layer15_attention_weight_max": 33.548187255859375,
      "activations/layer15_attention_weight_min": -30.56700325012207,
      "activations/layer16_attention_weight_max": 33.78214645385742,
      "activations/layer16_attention_weight_min": -31.88592529296875,
      "activations/layer17_attention_weight_max": 51.949798583984375,
      "activations/layer17_attention_weight_min": -46.16228103637695,
      "activations/layer18_attention_weight_max": 48.62184143066406,
      "activations/layer18_attention_weight_min": -42.32743453979492,
      "activations/layer19_attention_weight_max": 23.8775634765625,
      "activations/layer19_attention_weight_min": -23.574861526489258,
      "activations/layer1_attention_weight_max": 16.135318756103516,
      "activations/layer1_attention_weight_min": -15.266831398010254,
      "activations/layer20_attention_weight_max": 22.540231704711914,
      "activations/layer20_attention_weight_min": -22.293920516967773,
      "activations/layer21_attention_weight_max": 34.46234130859375,
      "activations/layer21_attention_weight_min": -21.40690040588379,
      "activations/layer22_attention_weight_max": 27.700275421142578,
      "activations/layer22_attention_weight_min": -26.93461799621582,
      "activations/layer23_attention_weight_max": 33.36304473876953,
      "activations/layer23_attention_weight_min": -24.861038208007812,
      "activations/layer2_attention_weight_max": 30.723194122314453,
      "activations/layer2_attention_weight_min": -31.065322875976562,
      "activations/layer3_attention_weight_max": 84.9408950805664,
      "activations/layer3_attention_weight_min": -88.34532165527344,
      "activations/layer4_attention_weight_max": 81.14482879638672,
      "activations/layer4_attention_weight_min": -81.11573791503906,
      "activations/layer5_attention_weight_max": 58.586334228515625,
      "activations/layer5_attention_weight_min": -68.41375732421875,
      "activations/layer6_attention_weight_max": 48.25593185424805,
      "activations/layer6_attention_weight_min": -49.13154220581055,
      "activations/layer7_attention_weight_max": 63.04230880737305,
      "activations/layer7_attention_weight_min": -62.48984909057617,
      "activations/layer8_attention_weight_max": 46.295963287353516,
      "activations/layer8_attention_weight_min": -49.868350982666016,
      "activations/layer9_attention_weight_max": 41.42259979248047,
      "activations/layer9_attention_weight_min": -44.383785247802734,
      "epoch": 21.02,
      "learning_rate": 1.459810606060606e-05,
      "loss": 2.717,
      "step": 361750
    },
    {
      "activations/layer0_attention_weight_max": 16.846424102783203,
      "activations/layer0_attention_weight_min": -15.766149520874023,
      "activations/layer10_attention_weight_max": 36.25148010253906,
      "activations/layer10_attention_weight_min": -37.41278839111328,
      "activations/layer11_attention_weight_max": 35.42210388183594,
      "activations/layer11_attention_weight_min": -36.636566162109375,
      "activations/layer12_attention_weight_max": 19.75665283203125,
      "activations/layer12_attention_weight_min": -26.16460418701172,
      "activations/layer13_attention_weight_max": 38.08887481689453,
      "activations/layer13_attention_weight_min": -28.299402236938477,
      "activations/layer14_attention_weight_max": 37.68171310424805,
      "activations/layer14_attention_weight_min": -32.00404739379883,
      "activations/layer15_attention_weight_max": 33.128543853759766,
      "activations/layer15_attention_weight_min": -31.983003616333008,
      "activations/layer16_attention_weight_max": 35.16681671142578,
      "activations/layer16_attention_weight_min": -33.91339111328125,
      "activations/layer17_attention_weight_max": 53.892555236816406,
      "activations/layer17_attention_weight_min": -44.81816482543945,
      "activations/layer18_attention_weight_max": 47.92695999145508,
      "activations/layer18_attention_weight_min": -40.579803466796875,
      "activations/layer19_attention_weight_max": 24.458984375,
      "activations/layer19_attention_weight_min": -23.727426528930664,
      "activations/layer1_attention_weight_max": 15.748054504394531,
      "activations/layer1_attention_weight_min": -15.371854782104492,
      "activations/layer20_attention_weight_max": 23.344919204711914,
      "activations/layer20_attention_weight_min": -21.433500289916992,
      "activations/layer21_attention_weight_max": 37.64023971557617,
      "activations/layer21_attention_weight_min": -23.23581314086914,
      "activations/layer22_attention_weight_max": 30.55613899230957,
      "activations/layer22_attention_weight_min": -25.13125228881836,
      "activations/layer23_attention_weight_max": 34.46809387207031,
      "activations/layer23_attention_weight_min": -27.216449737548828,
      "activations/layer2_attention_weight_max": 31.377120971679688,
      "activations/layer2_attention_weight_min": -30.405380249023438,
      "activations/layer3_attention_weight_max": 87.6669921875,
      "activations/layer3_attention_weight_min": -88.66895294189453,
      "activations/layer4_attention_weight_max": 85.22801971435547,
      "activations/layer4_attention_weight_min": -85.65132904052734,
      "activations/layer5_attention_weight_max": 59.556087493896484,
      "activations/layer5_attention_weight_min": -72.79594421386719,
      "activations/layer6_attention_weight_max": 50.63557434082031,
      "activations/layer6_attention_weight_min": -51.05754852294922,
      "activations/layer7_attention_weight_max": 66.20120239257812,
      "activations/layer7_attention_weight_min": -64.0193099975586,
      "activations/layer8_attention_weight_max": 48.29768753051758,
      "activations/layer8_attention_weight_min": -49.95572280883789,
      "activations/layer9_attention_weight_max": 43.13690948486328,
      "activations/layer9_attention_weight_min": -46.11139678955078,
      "epoch": 21.02,
      "learning_rate": 1.4579166666666666e-05,
      "loss": 2.723,
      "step": 361800
    },
    {
      "activations/layer0_attention_weight_max": 16.125782012939453,
      "activations/layer0_attention_weight_min": -13.909111022949219,
      "activations/layer10_attention_weight_max": 35.00315856933594,
      "activations/layer10_attention_weight_min": -35.13264846801758,
      "activations/layer11_attention_weight_max": 33.04925537109375,
      "activations/layer11_attention_weight_min": -34.603515625,
      "activations/layer12_attention_weight_max": 19.534198760986328,
      "activations/layer12_attention_weight_min": -23.051002502441406,
      "activations/layer13_attention_weight_max": 33.133846282958984,
      "activations/layer13_attention_weight_min": -31.076969146728516,
      "activations/layer14_attention_weight_max": 34.00642776489258,
      "activations/layer14_attention_weight_min": -33.094337463378906,
      "activations/layer15_attention_weight_max": 32.46881866455078,
      "activations/layer15_attention_weight_min": -36.2994270324707,
      "activations/layer16_attention_weight_max": 34.700599670410156,
      "activations/layer16_attention_weight_min": -34.411537170410156,
      "activations/layer17_attention_weight_max": 50.17176055908203,
      "activations/layer17_attention_weight_min": -47.363975524902344,
      "activations/layer18_attention_weight_max": 45.37263870239258,
      "activations/layer18_attention_weight_min": -41.1342658996582,
      "activations/layer19_attention_weight_max": 23.322904586791992,
      "activations/layer19_attention_weight_min": -22.354917526245117,
      "activations/layer1_attention_weight_max": 15.543097496032715,
      "activations/layer1_attention_weight_min": -14.262879371643066,
      "activations/layer20_attention_weight_max": 23.779146194458008,
      "activations/layer20_attention_weight_min": -23.103899002075195,
      "activations/layer21_attention_weight_max": 33.11833572387695,
      "activations/layer21_attention_weight_min": -22.703081130981445,
      "activations/layer22_attention_weight_max": 26.493350982666016,
      "activations/layer22_attention_weight_min": -27.094879150390625,
      "activations/layer23_attention_weight_max": 33.70799255371094,
      "activations/layer23_attention_weight_min": -25.052648544311523,
      "activations/layer2_attention_weight_max": 31.532629013061523,
      "activations/layer2_attention_weight_min": -31.081403732299805,
      "activations/layer3_attention_weight_max": 88.76951599121094,
      "activations/layer3_attention_weight_min": -86.25163269042969,
      "activations/layer4_attention_weight_max": 84.02625274658203,
      "activations/layer4_attention_weight_min": -86.5913314819336,
      "activations/layer5_attention_weight_max": 63.831356048583984,
      "activations/layer5_attention_weight_min": -70.38907623291016,
      "activations/layer6_attention_weight_max": 50.68248748779297,
      "activations/layer6_attention_weight_min": -53.71695327758789,
      "activations/layer7_attention_weight_max": 69.52570343017578,
      "activations/layer7_attention_weight_min": -68.00823974609375,
      "activations/layer8_attention_weight_max": 47.26042938232422,
      "activations/layer8_attention_weight_min": -50.74264144897461,
      "activations/layer9_attention_weight_max": 45.36832046508789,
      "activations/layer9_attention_weight_min": -46.3922233581543,
      "epoch": 21.03,
      "learning_rate": 1.456022727272727e-05,
      "loss": 2.6964,
      "step": 361850
    },
    {
      "activations/layer0_attention_weight_max": 15.826760292053223,
      "activations/layer0_attention_weight_min": -14.250853538513184,
      "activations/layer10_attention_weight_max": 33.92982864379883,
      "activations/layer10_attention_weight_min": -37.165218353271484,
      "activations/layer11_attention_weight_max": 32.14317321777344,
      "activations/layer11_attention_weight_min": -34.52056121826172,
      "activations/layer12_attention_weight_max": 16.26422119140625,
      "activations/layer12_attention_weight_min": -27.69219207763672,
      "activations/layer13_attention_weight_max": 33.269737243652344,
      "activations/layer13_attention_weight_min": -27.720632553100586,
      "activations/layer14_attention_weight_max": 30.269651412963867,
      "activations/layer14_attention_weight_min": -29.095579147338867,
      "activations/layer15_attention_weight_max": 29.14432144165039,
      "activations/layer15_attention_weight_min": -29.2012939453125,
      "activations/layer16_attention_weight_max": 29.822593688964844,
      "activations/layer16_attention_weight_min": -30.568115234375,
      "activations/layer17_attention_weight_max": 45.5933723449707,
      "activations/layer17_attention_weight_min": -45.36137771606445,
      "activations/layer18_attention_weight_max": 45.16875076293945,
      "activations/layer18_attention_weight_min": -41.19266891479492,
      "activations/layer19_attention_weight_max": 22.24551010131836,
      "activations/layer19_attention_weight_min": -23.32910919189453,
      "activations/layer1_attention_weight_max": 15.588780403137207,
      "activations/layer1_attention_weight_min": -14.02613639831543,
      "activations/layer20_attention_weight_max": 22.114439010620117,
      "activations/layer20_attention_weight_min": -23.808238983154297,
      "activations/layer21_attention_weight_max": 32.65602111816406,
      "activations/layer21_attention_weight_min": -25.37639808654785,
      "activations/layer22_attention_weight_max": 27.71082305908203,
      "activations/layer22_attention_weight_min": -24.998525619506836,
      "activations/layer23_attention_weight_max": 33.03308868408203,
      "activations/layer23_attention_weight_min": -25.414020538330078,
      "activations/layer2_attention_weight_max": 28.912193298339844,
      "activations/layer2_attention_weight_min": -30.642837524414062,
      "activations/layer3_attention_weight_max": 83.24032592773438,
      "activations/layer3_attention_weight_min": -86.44163513183594,
      "activations/layer4_attention_weight_max": 83.25428771972656,
      "activations/layer4_attention_weight_min": -80.93058776855469,
      "activations/layer5_attention_weight_max": 60.10902404785156,
      "activations/layer5_attention_weight_min": -74.10911560058594,
      "activations/layer6_attention_weight_max": 51.76935577392578,
      "activations/layer6_attention_weight_min": -54.132083892822266,
      "activations/layer7_attention_weight_max": 70.33387756347656,
      "activations/layer7_attention_weight_min": -67.85792541503906,
      "activations/layer8_attention_weight_max": 47.40190505981445,
      "activations/layer8_attention_weight_min": -50.75881576538086,
      "activations/layer9_attention_weight_max": 46.77233123779297,
      "activations/layer9_attention_weight_min": -48.33328628540039,
      "epoch": 21.03,
      "learning_rate": 1.4541287878787877e-05,
      "loss": 2.7083,
      "step": 361900
    },
    {
      "activations/layer0_attention_weight_max": 15.8456449508667,
      "activations/layer0_attention_weight_min": -14.111074447631836,
      "activations/layer10_attention_weight_max": 36.62972640991211,
      "activations/layer10_attention_weight_min": -37.08070755004883,
      "activations/layer11_attention_weight_max": 33.53186798095703,
      "activations/layer11_attention_weight_min": -35.259063720703125,
      "activations/layer12_attention_weight_max": 20.82400131225586,
      "activations/layer12_attention_weight_min": -26.183307647705078,
      "activations/layer13_attention_weight_max": 35.61883544921875,
      "activations/layer13_attention_weight_min": -31.70852279663086,
      "activations/layer14_attention_weight_max": 38.27981185913086,
      "activations/layer14_attention_weight_min": -34.717201232910156,
      "activations/layer15_attention_weight_max": 34.28837203979492,
      "activations/layer15_attention_weight_min": -35.6096076965332,
      "activations/layer16_attention_weight_max": 36.53557586669922,
      "activations/layer16_attention_weight_min": -35.053672790527344,
      "activations/layer17_attention_weight_max": 53.333457946777344,
      "activations/layer17_attention_weight_min": -50.25006103515625,
      "activations/layer18_attention_weight_max": 48.77344512939453,
      "activations/layer18_attention_weight_min": -43.175777435302734,
      "activations/layer19_attention_weight_max": 23.496999740600586,
      "activations/layer19_attention_weight_min": -23.355052947998047,
      "activations/layer1_attention_weight_max": 16.960845947265625,
      "activations/layer1_attention_weight_min": -13.961563110351562,
      "activations/layer20_attention_weight_max": 20.675050735473633,
      "activations/layer20_attention_weight_min": -23.027847290039062,
      "activations/layer21_attention_weight_max": 31.49362564086914,
      "activations/layer21_attention_weight_min": -22.416561126708984,
      "activations/layer22_attention_weight_max": 24.953794479370117,
      "activations/layer22_attention_weight_min": -25.797204971313477,
      "activations/layer23_attention_weight_max": 33.98678970336914,
      "activations/layer23_attention_weight_min": -24.52859878540039,
      "activations/layer2_attention_weight_max": 29.866466522216797,
      "activations/layer2_attention_weight_min": -29.439022064208984,
      "activations/layer3_attention_weight_max": 87.85029602050781,
      "activations/layer3_attention_weight_min": -87.41368865966797,
      "activations/layer4_attention_weight_max": 80.60124969482422,
      "activations/layer4_attention_weight_min": -80.66768646240234,
      "activations/layer5_attention_weight_max": 60.429229736328125,
      "activations/layer5_attention_weight_min": -70.52596282958984,
      "activations/layer6_attention_weight_max": 47.99464416503906,
      "activations/layer6_attention_weight_min": -50.86463165283203,
      "activations/layer7_attention_weight_max": 66.24246978759766,
      "activations/layer7_attention_weight_min": -65.16815948486328,
      "activations/layer8_attention_weight_max": 46.58806228637695,
      "activations/layer8_attention_weight_min": -50.915889739990234,
      "activations/layer9_attention_weight_max": 42.83573532104492,
      "activations/layer9_attention_weight_min": -46.59647750854492,
      "epoch": 21.03,
      "learning_rate": 1.4522348484848484e-05,
      "loss": 2.7106,
      "step": 361950
    },
    {
      "activations/layer0_attention_weight_max": 15.536249160766602,
      "activations/layer0_attention_weight_min": -14.331497192382812,
      "activations/layer10_attention_weight_max": 37.22719955444336,
      "activations/layer10_attention_weight_min": -36.07628631591797,
      "activations/layer11_attention_weight_max": 34.87358856201172,
      "activations/layer11_attention_weight_min": -33.18052291870117,
      "activations/layer12_attention_weight_max": 19.684438705444336,
      "activations/layer12_attention_weight_min": -24.111997604370117,
      "activations/layer13_attention_weight_max": 34.54476547241211,
      "activations/layer13_attention_weight_min": -25.93742561340332,
      "activations/layer14_attention_weight_max": 37.40689468383789,
      "activations/layer14_attention_weight_min": -30.297924041748047,
      "activations/layer15_attention_weight_max": 38.26744079589844,
      "activations/layer15_attention_weight_min": -29.711055755615234,
      "activations/layer16_attention_weight_max": 37.445030212402344,
      "activations/layer16_attention_weight_min": -33.192359924316406,
      "activations/layer17_attention_weight_max": 57.10710525512695,
      "activations/layer17_attention_weight_min": -45.00319290161133,
      "activations/layer18_attention_weight_max": 52.48713684082031,
      "activations/layer18_attention_weight_min": -39.96584701538086,
      "activations/layer19_attention_weight_max": 27.32333755493164,
      "activations/layer19_attention_weight_min": -25.11807632446289,
      "activations/layer1_attention_weight_max": 15.269330024719238,
      "activations/layer1_attention_weight_min": -13.766204833984375,
      "activations/layer20_attention_weight_max": 26.9785213470459,
      "activations/layer20_attention_weight_min": -20.961942672729492,
      "activations/layer21_attention_weight_max": 40.416847229003906,
      "activations/layer21_attention_weight_min": -21.500255584716797,
      "activations/layer22_attention_weight_max": 32.975250244140625,
      "activations/layer22_attention_weight_min": -23.74578285217285,
      "activations/layer23_attention_weight_max": 44.21098327636719,
      "activations/layer23_attention_weight_min": -21.941810607910156,
      "activations/layer2_attention_weight_max": 30.75780487060547,
      "activations/layer2_attention_weight_min": -29.304466247558594,
      "activations/layer3_attention_weight_max": 90.46219635009766,
      "activations/layer3_attention_weight_min": -86.31749725341797,
      "activations/layer4_attention_weight_max": 85.28294372558594,
      "activations/layer4_attention_weight_min": -82.1627426147461,
      "activations/layer5_attention_weight_max": 62.378047943115234,
      "activations/layer5_attention_weight_min": -70.14412689208984,
      "activations/layer6_attention_weight_max": 48.494930267333984,
      "activations/layer6_attention_weight_min": -50.148460388183594,
      "activations/layer7_attention_weight_max": 68.92707824707031,
      "activations/layer7_attention_weight_min": -68.80067443847656,
      "activations/layer8_attention_weight_max": 45.98731994628906,
      "activations/layer8_attention_weight_min": -48.60344314575195,
      "activations/layer9_attention_weight_max": 47.28817367553711,
      "activations/layer9_attention_weight_min": -46.80162811279297,
      "epoch": 21.03,
      "learning_rate": 1.4503409090909089e-05,
      "loss": 2.7068,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4624,
      "eval_samples_per_second": 507.423,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4624,
      "eval_openwebtext_samples_per_second": 507.423,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9745,
      "eval_wikitext_samples_per_second": 230.941,
      "step": 362000
    },
    {
      "epoch": 21.03,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.6285,
      "eval_lambada_samples_per_second": 505.685,
      "step": 362000
    },
    {
      "activations/layer0_attention_weight_max": 15.969401359558105,
      "activations/layer0_attention_weight_min": -14.134446144104004,
      "activations/layer10_attention_weight_max": 37.498313903808594,
      "activations/layer10_attention_weight_min": -35.361480712890625,
      "activations/layer11_attention_weight_max": 34.03496551513672,
      "activations/layer11_attention_weight_min": -37.14752197265625,
      "activations/layer12_attention_weight_max": 19.290525436401367,
      "activations/layer12_attention_weight_min": -25.90682601928711,
      "activations/layer13_attention_weight_max": 38.11370849609375,
      "activations/layer13_attention_weight_min": -26.92490005493164,
      "activations/layer14_attention_weight_max": 38.45024871826172,
      "activations/layer14_attention_weight_min": -29.980085372924805,
      "activations/layer15_attention_weight_max": 37.44303512573242,
      "activations/layer15_attention_weight_min": -28.85146713256836,
      "activations/layer16_attention_weight_max": 38.481719970703125,
      "activations/layer16_attention_weight_min": -35.45376968383789,
      "activations/layer17_attention_weight_max": 55.1606330871582,
      "activations/layer17_attention_weight_min": -47.793060302734375,
      "activations/layer18_attention_weight_max": 50.46593475341797,
      "activations/layer18_attention_weight_min": -41.917137145996094,
      "activations/layer19_attention_weight_max": 25.792837142944336,
      "activations/layer19_attention_weight_min": -23.117040634155273,
      "activations/layer1_attention_weight_max": 16.03470230102539,
      "activations/layer1_attention_weight_min": -15.335115432739258,
      "activations/layer20_attention_weight_max": 23.63675308227539,
      "activations/layer20_attention_weight_min": -20.813623428344727,
      "activations/layer21_attention_weight_max": 39.728031158447266,
      "activations/layer21_attention_weight_min": -22.75811004638672,
      "activations/layer22_attention_weight_max": 30.086822509765625,
      "activations/layer22_attention_weight_min": -23.971210479736328,
      "activations/layer23_attention_weight_max": 40.39592361450195,
      "activations/layer23_attention_weight_min": -24.22574234008789,
      "activations/layer2_attention_weight_max": 31.886436462402344,
      "activations/layer2_attention_weight_min": -29.861652374267578,
      "activations/layer3_attention_weight_max": 93.7293472290039,
      "activations/layer3_attention_weight_min": -89.67349243164062,
      "activations/layer4_attention_weight_max": 83.82506561279297,
      "activations/layer4_attention_weight_min": -87.1717300415039,
      "activations/layer5_attention_weight_max": 60.89963912963867,
      "activations/layer5_attention_weight_min": -70.5246353149414,
      "activations/layer6_attention_weight_max": 50.410362243652344,
      "activations/layer6_attention_weight_min": -53.00994110107422,
      "activations/layer7_attention_weight_max": 69.4151840209961,
      "activations/layer7_attention_weight_min": -66.57650756835938,
      "activations/layer8_attention_weight_max": 46.425804138183594,
      "activations/layer8_attention_weight_min": -48.7677001953125,
      "activations/layer9_attention_weight_max": 43.738616943359375,
      "activations/layer9_attention_weight_min": -46.53659439086914,
      "epoch": 21.04,
      "learning_rate": 1.4484469696969695e-05,
      "loss": 2.7108,
      "step": 362050
    },
    {
      "activations/layer0_attention_weight_max": 16.514299392700195,
      "activations/layer0_attention_weight_min": -14.03693675994873,
      "activations/layer10_attention_weight_max": 39.287986755371094,
      "activations/layer10_attention_weight_min": -36.90630340576172,
      "activations/layer11_attention_weight_max": 35.16276168823242,
      "activations/layer11_attention_weight_min": -36.58744430541992,
      "activations/layer12_attention_weight_max": 20.01740074157715,
      "activations/layer12_attention_weight_min": -22.198341369628906,
      "activations/layer13_attention_weight_max": 35.32366943359375,
      "activations/layer13_attention_weight_min": -27.36787986755371,
      "activations/layer14_attention_weight_max": 36.59567642211914,
      "activations/layer14_attention_weight_min": -28.688756942749023,
      "activations/layer15_attention_weight_max": 34.712562561035156,
      "activations/layer15_attention_weight_min": -29.8247127532959,
      "activations/layer16_attention_weight_max": 33.311126708984375,
      "activations/layer16_attention_weight_min": -31.718835830688477,
      "activations/layer17_attention_weight_max": 53.29773712158203,
      "activations/layer17_attention_weight_min": -47.991764068603516,
      "activations/layer18_attention_weight_max": 48.912784576416016,
      "activations/layer18_attention_weight_min": -41.577674865722656,
      "activations/layer19_attention_weight_max": 28.789459228515625,
      "activations/layer19_attention_weight_min": -25.23554801940918,
      "activations/layer1_attention_weight_max": 14.880632400512695,
      "activations/layer1_attention_weight_min": -13.983720779418945,
      "activations/layer20_attention_weight_max": 24.52920150756836,
      "activations/layer20_attention_weight_min": -24.812461853027344,
      "activations/layer21_attention_weight_max": 38.803375244140625,
      "activations/layer21_attention_weight_min": -20.145740509033203,
      "activations/layer22_attention_weight_max": 36.426979064941406,
      "activations/layer22_attention_weight_min": -26.604984283447266,
      "activations/layer23_attention_weight_max": 42.560157775878906,
      "activations/layer23_attention_weight_min": -24.34864044189453,
      "activations/layer2_attention_weight_max": 30.155765533447266,
      "activations/layer2_attention_weight_min": -30.4639892578125,
      "activations/layer3_attention_weight_max": 89.96703338623047,
      "activations/layer3_attention_weight_min": -88.0806655883789,
      "activations/layer4_attention_weight_max": 82.0494384765625,
      "activations/layer4_attention_weight_min": -83.2036361694336,
      "activations/layer5_attention_weight_max": 60.82804870605469,
      "activations/layer5_attention_weight_min": -72.94110107421875,
      "activations/layer6_attention_weight_max": 50.377445220947266,
      "activations/layer6_attention_weight_min": -50.718875885009766,
      "activations/layer7_attention_weight_max": 69.77790832519531,
      "activations/layer7_attention_weight_min": -69.0018539428711,
      "activations/layer8_attention_weight_max": 51.219932556152344,
      "activations/layer8_attention_weight_min": -50.2803955078125,
      "activations/layer9_attention_weight_max": 46.48514938354492,
      "activations/layer9_attention_weight_min": -47.582557678222656,
      "epoch": 21.04,
      "learning_rate": 1.44655303030303e-05,
      "loss": 2.7158,
      "step": 362100
    },
    {
      "activations/layer0_attention_weight_max": 16.338945388793945,
      "activations/layer0_attention_weight_min": -14.164992332458496,
      "activations/layer10_attention_weight_max": 35.649078369140625,
      "activations/layer10_attention_weight_min": -36.11532211303711,
      "activations/layer11_attention_weight_max": 33.45734405517578,
      "activations/layer11_attention_weight_min": -34.955108642578125,
      "activations/layer12_attention_weight_max": 19.88690948486328,
      "activations/layer12_attention_weight_min": -25.21708106994629,
      "activations/layer13_attention_weight_max": 33.010772705078125,
      "activations/layer13_attention_weight_min": -26.237037658691406,
      "activations/layer14_attention_weight_max": 35.69671630859375,
      "activations/layer14_attention_weight_min": -33.83144760131836,
      "activations/layer15_attention_weight_max": 34.11468505859375,
      "activations/layer15_attention_weight_min": -35.10052490234375,
      "activations/layer16_attention_weight_max": 35.84915542602539,
      "activations/layer16_attention_weight_min": -34.99030685424805,
      "activations/layer17_attention_weight_max": 56.08586502075195,
      "activations/layer17_attention_weight_min": -47.87306594848633,
      "activations/layer18_attention_weight_max": 53.092079162597656,
      "activations/layer18_attention_weight_min": -42.20021057128906,
      "activations/layer19_attention_weight_max": 27.4947566986084,
      "activations/layer19_attention_weight_min": -25.330612182617188,
      "activations/layer1_attention_weight_max": 17.204439163208008,
      "activations/layer1_attention_weight_min": -14.520994186401367,
      "activations/layer20_attention_weight_max": 27.10248374938965,
      "activations/layer20_attention_weight_min": -22.251848220825195,
      "activations/layer21_attention_weight_max": 40.16062545776367,
      "activations/layer21_attention_weight_min": -22.99421501159668,
      "activations/layer22_attention_weight_max": 31.37521743774414,
      "activations/layer22_attention_weight_min": -22.885087966918945,
      "activations/layer23_attention_weight_max": 41.13861083984375,
      "activations/layer23_attention_weight_min": -23.096250534057617,
      "activations/layer2_attention_weight_max": 30.652517318725586,
      "activations/layer2_attention_weight_min": -29.681175231933594,
      "activations/layer3_attention_weight_max": 84.82466888427734,
      "activations/layer3_attention_weight_min": -83.38370513916016,
      "activations/layer4_attention_weight_max": 81.00559997558594,
      "activations/layer4_attention_weight_min": -83.22891998291016,
      "activations/layer5_attention_weight_max": 60.55056381225586,
      "activations/layer5_attention_weight_min": -71.45268249511719,
      "activations/layer6_attention_weight_max": 47.7800407409668,
      "activations/layer6_attention_weight_min": -48.48777389526367,
      "activations/layer7_attention_weight_max": 68.13797760009766,
      "activations/layer7_attention_weight_min": -64.612548828125,
      "activations/layer8_attention_weight_max": 46.25932693481445,
      "activations/layer8_attention_weight_min": -48.70988082885742,
      "activations/layer9_attention_weight_max": 44.769039154052734,
      "activations/layer9_attention_weight_min": -45.55410385131836,
      "epoch": 21.04,
      "learning_rate": 1.4446590909090907e-05,
      "loss": 2.7005,
      "step": 362150
    },
    {
      "activations/layer0_attention_weight_max": 15.94876480102539,
      "activations/layer0_attention_weight_min": -13.87954330444336,
      "activations/layer10_attention_weight_max": 35.69762420654297,
      "activations/layer10_attention_weight_min": -34.89942169189453,
      "activations/layer11_attention_weight_max": 32.82133865356445,
      "activations/layer11_attention_weight_min": -35.333580017089844,
      "activations/layer12_attention_weight_max": 17.0606746673584,
      "activations/layer12_attention_weight_min": -26.645034790039062,
      "activations/layer13_attention_weight_max": 28.511873245239258,
      "activations/layer13_attention_weight_min": -26.05959129333496,
      "activations/layer14_attention_weight_max": 35.471736907958984,
      "activations/layer14_attention_weight_min": -29.45977210998535,
      "activations/layer15_attention_weight_max": 32.152793884277344,
      "activations/layer15_attention_weight_min": -29.758546829223633,
      "activations/layer16_attention_weight_max": 33.94224548339844,
      "activations/layer16_attention_weight_min": -32.918434143066406,
      "activations/layer17_attention_weight_max": 52.99729537963867,
      "activations/layer17_attention_weight_min": -45.50811767578125,
      "activations/layer18_attention_weight_max": 48.563072204589844,
      "activations/layer18_attention_weight_min": -40.813079833984375,
      "activations/layer19_attention_weight_max": 23.745609283447266,
      "activations/layer19_attention_weight_min": -20.86707878112793,
      "activations/layer1_attention_weight_max": 15.449559211730957,
      "activations/layer1_attention_weight_min": -15.819090843200684,
      "activations/layer20_attention_weight_max": 21.665647506713867,
      "activations/layer20_attention_weight_min": -22.04277229309082,
      "activations/layer21_attention_weight_max": 32.33263397216797,
      "activations/layer21_attention_weight_min": -20.794479370117188,
      "activations/layer22_attention_weight_max": 27.055892944335938,
      "activations/layer22_attention_weight_min": -24.687885284423828,
      "activations/layer23_attention_weight_max": 32.529258728027344,
      "activations/layer23_attention_weight_min": -21.935237884521484,
      "activations/layer2_attention_weight_max": 31.371726989746094,
      "activations/layer2_attention_weight_min": -30.325361251831055,
      "activations/layer3_attention_weight_max": 94.95085144042969,
      "activations/layer3_attention_weight_min": -91.13311767578125,
      "activations/layer4_attention_weight_max": 87.18240356445312,
      "activations/layer4_attention_weight_min": -84.86006927490234,
      "activations/layer5_attention_weight_max": 61.55011749267578,
      "activations/layer5_attention_weight_min": -70.93464660644531,
      "activations/layer6_attention_weight_max": 50.894508361816406,
      "activations/layer6_attention_weight_min": -53.28209686279297,
      "activations/layer7_attention_weight_max": 66.80570220947266,
      "activations/layer7_attention_weight_min": -67.34793090820312,
      "activations/layer8_attention_weight_max": 46.404624938964844,
      "activations/layer8_attention_weight_min": -48.78758239746094,
      "activations/layer9_attention_weight_max": 43.96935272216797,
      "activations/layer9_attention_weight_min": -45.605010986328125,
      "epoch": 21.05,
      "learning_rate": 1.4427651515151513e-05,
      "loss": 2.7168,
      "step": 362200
    },
    {
      "activations/layer0_attention_weight_max": 15.108871459960938,
      "activations/layer0_attention_weight_min": -13.853741645812988,
      "activations/layer10_attention_weight_max": 48.540889739990234,
      "activations/layer10_attention_weight_min": -45.90422821044922,
      "activations/layer11_attention_weight_max": 42.94404602050781,
      "activations/layer11_attention_weight_min": -42.98295593261719,
      "activations/layer12_attention_weight_max": 23.26911163330078,
      "activations/layer12_attention_weight_min": -24.958511352539062,
      "activations/layer13_attention_weight_max": 54.656314849853516,
      "activations/layer13_attention_weight_min": -33.65682601928711,
      "activations/layer14_attention_weight_max": 69.05300903320312,
      "activations/layer14_attention_weight_min": -39.622188568115234,
      "activations/layer15_attention_weight_max": 69.89446258544922,
      "activations/layer15_attention_weight_min": -41.6067008972168,
      "activations/layer16_attention_weight_max": 49.054988861083984,
      "activations/layer16_attention_weight_min": -43.0135498046875,
      "activations/layer17_attention_weight_max": 72.55975341796875,
      "activations/layer17_attention_weight_min": -61.05204391479492,
      "activations/layer18_attention_weight_max": 74.41177368164062,
      "activations/layer18_attention_weight_min": -57.6041259765625,
      "activations/layer19_attention_weight_max": 43.55657196044922,
      "activations/layer19_attention_weight_min": -36.126060485839844,
      "activations/layer1_attention_weight_max": 15.328031539916992,
      "activations/layer1_attention_weight_min": -14.704276084899902,
      "activations/layer20_attention_weight_max": 46.44950866699219,
      "activations/layer20_attention_weight_min": -29.164690017700195,
      "activations/layer21_attention_weight_max": 84.25362396240234,
      "activations/layer21_attention_weight_min": -32.95903778076172,
      "activations/layer22_attention_weight_max": 53.530975341796875,
      "activations/layer22_attention_weight_min": -36.33852005004883,
      "activations/layer23_attention_weight_max": 58.660011291503906,
      "activations/layer23_attention_weight_min": -31.898178100585938,
      "activations/layer2_attention_weight_max": 31.51811981201172,
      "activations/layer2_attention_weight_min": -29.091007232666016,
      "activations/layer3_attention_weight_max": 89.43080139160156,
      "activations/layer3_attention_weight_min": -90.37045288085938,
      "activations/layer4_attention_weight_max": 80.47404479980469,
      "activations/layer4_attention_weight_min": -82.36474609375,
      "activations/layer5_attention_weight_max": 59.14445495605469,
      "activations/layer5_attention_weight_min": -70.89726257324219,
      "activations/layer6_attention_weight_max": 49.12550735473633,
      "activations/layer6_attention_weight_min": -49.829444885253906,
      "activations/layer7_attention_weight_max": 73.86593627929688,
      "activations/layer7_attention_weight_min": -66.25585174560547,
      "activations/layer8_attention_weight_max": 55.52346420288086,
      "activations/layer8_attention_weight_min": -53.4948616027832,
      "activations/layer9_attention_weight_max": 55.53396224975586,
      "activations/layer9_attention_weight_min": -50.3422966003418,
      "epoch": 21.05,
      "learning_rate": 1.4408712121212122e-05,
      "loss": 2.7249,
      "step": 362250
    },
    {
      "activations/layer0_attention_weight_max": 16.37041473388672,
      "activations/layer0_attention_weight_min": -13.878300666809082,
      "activations/layer10_attention_weight_max": 39.89982604980469,
      "activations/layer10_attention_weight_min": -37.07918930053711,
      "activations/layer11_attention_weight_max": 36.66446304321289,
      "activations/layer11_attention_weight_min": -35.917694091796875,
      "activations/layer12_attention_weight_max": 20.445270538330078,
      "activations/layer12_attention_weight_min": -25.120004653930664,
      "activations/layer13_attention_weight_max": 33.58969497680664,
      "activations/layer13_attention_weight_min": -28.61178970336914,
      "activations/layer14_attention_weight_max": 34.69565200805664,
      "activations/layer14_attention_weight_min": -30.715911865234375,
      "activations/layer15_attention_weight_max": 32.39814376831055,
      "activations/layer15_attention_weight_min": -31.72841453552246,
      "activations/layer16_attention_weight_max": 33.98876190185547,
      "activations/layer16_attention_weight_min": -31.853836059570312,
      "activations/layer17_attention_weight_max": 50.28687286376953,
      "activations/layer17_attention_weight_min": -45.09611892700195,
      "activations/layer18_attention_weight_max": 46.8446044921875,
      "activations/layer18_attention_weight_min": -40.602569580078125,
      "activations/layer19_attention_weight_max": 25.377120971679688,
      "activations/layer19_attention_weight_min": -23.812944412231445,
      "activations/layer1_attention_weight_max": 15.850943565368652,
      "activations/layer1_attention_weight_min": -14.590692520141602,
      "activations/layer20_attention_weight_max": 25.804912567138672,
      "activations/layer20_attention_weight_min": -24.06937026977539,
      "activations/layer21_attention_weight_max": 35.80479431152344,
      "activations/layer21_attention_weight_min": -24.961179733276367,
      "activations/layer22_attention_weight_max": 29.765872955322266,
      "activations/layer22_attention_weight_min": -26.40897560119629,
      "activations/layer23_attention_weight_max": 36.186805725097656,
      "activations/layer23_attention_weight_min": -23.45825958251953,
      "activations/layer2_attention_weight_max": 33.24790954589844,
      "activations/layer2_attention_weight_min": -31.642547607421875,
      "activations/layer3_attention_weight_max": 95.02745819091797,
      "activations/layer3_attention_weight_min": -90.3753433227539,
      "activations/layer4_attention_weight_max": 87.77371215820312,
      "activations/layer4_attention_weight_min": -83.637939453125,
      "activations/layer5_attention_weight_max": 62.74343490600586,
      "activations/layer5_attention_weight_min": -74.5398178100586,
      "activations/layer6_attention_weight_max": 50.60047149658203,
      "activations/layer6_attention_weight_min": -53.12519454956055,
      "activations/layer7_attention_weight_max": 69.122314453125,
      "activations/layer7_attention_weight_min": -66.8386001586914,
      "activations/layer8_attention_weight_max": 48.841373443603516,
      "activations/layer8_attention_weight_min": -52.25967025756836,
      "activations/layer9_attention_weight_max": 47.859352111816406,
      "activations/layer9_attention_weight_min": -49.11436080932617,
      "epoch": 21.05,
      "learning_rate": 1.4389772727272727e-05,
      "loss": 2.7245,
      "step": 362300
    },
    {
      "activations/layer0_attention_weight_max": 15.717779159545898,
      "activations/layer0_attention_weight_min": -14.699051856994629,
      "activations/layer10_attention_weight_max": 34.94377136230469,
      "activations/layer10_attention_weight_min": -35.45602798461914,
      "activations/layer11_attention_weight_max": 33.43208312988281,
      "activations/layer11_attention_weight_min": -33.29637908935547,
      "activations/layer12_attention_weight_max": 20.5567569732666,
      "activations/layer12_attention_weight_min": -27.65052032470703,
      "activations/layer13_attention_weight_max": 34.57542419433594,
      "activations/layer13_attention_weight_min": -24.86812400817871,
      "activations/layer14_attention_weight_max": 36.25096130371094,
      "activations/layer14_attention_weight_min": -29.34307289123535,
      "activations/layer15_attention_weight_max": 34.706058502197266,
      "activations/layer15_attention_weight_min": -29.689558029174805,
      "activations/layer16_attention_weight_max": 36.70653533935547,
      "activations/layer16_attention_weight_min": -34.67209243774414,
      "activations/layer17_attention_weight_max": 54.42488098144531,
      "activations/layer17_attention_weight_min": -46.9629020690918,
      "activations/layer18_attention_weight_max": 48.40144729614258,
      "activations/layer18_attention_weight_min": -43.18440628051758,
      "activations/layer19_attention_weight_max": 25.572874069213867,
      "activations/layer19_attention_weight_min": -25.050661087036133,
      "activations/layer1_attention_weight_max": 15.146099090576172,
      "activations/layer1_attention_weight_min": -12.921758651733398,
      "activations/layer20_attention_weight_max": 23.106529235839844,
      "activations/layer20_attention_weight_min": -21.952226638793945,
      "activations/layer21_attention_weight_max": 40.611785888671875,
      "activations/layer21_attention_weight_min": -24.79729652404785,
      "activations/layer22_attention_weight_max": 29.750608444213867,
      "activations/layer22_attention_weight_min": -26.508087158203125,
      "activations/layer23_attention_weight_max": 39.56605529785156,
      "activations/layer23_attention_weight_min": -26.088985443115234,
      "activations/layer2_attention_weight_max": 29.85239028930664,
      "activations/layer2_attention_weight_min": -28.003501892089844,
      "activations/layer3_attention_weight_max": 85.84314727783203,
      "activations/layer3_attention_weight_min": -81.02005767822266,
      "activations/layer4_attention_weight_max": 85.49994659423828,
      "activations/layer4_attention_weight_min": -80.11871337890625,
      "activations/layer5_attention_weight_max": 62.180938720703125,
      "activations/layer5_attention_weight_min": -73.31851196289062,
      "activations/layer6_attention_weight_max": 48.43053436279297,
      "activations/layer6_attention_weight_min": -49.10541915893555,
      "activations/layer7_attention_weight_max": 66.97581481933594,
      "activations/layer7_attention_weight_min": -63.44889450073242,
      "activations/layer8_attention_weight_max": 44.91965866088867,
      "activations/layer8_attention_weight_min": -50.03487014770508,
      "activations/layer9_attention_weight_max": 42.39561080932617,
      "activations/layer9_attention_weight_min": -43.900814056396484,
      "epoch": 21.05,
      "learning_rate": 1.4370833333333333e-05,
      "loss": 2.709,
      "step": 362350
    },
    {
      "activations/layer0_attention_weight_max": 15.522809982299805,
      "activations/layer0_attention_weight_min": -14.062322616577148,
      "activations/layer10_attention_weight_max": 34.740135192871094,
      "activations/layer10_attention_weight_min": -35.797218322753906,
      "activations/layer11_attention_weight_max": 31.86139678955078,
      "activations/layer11_attention_weight_min": -33.64933395385742,
      "activations/layer12_attention_weight_max": 18.877227783203125,
      "activations/layer12_attention_weight_min": -25.84166717529297,
      "activations/layer13_attention_weight_max": 29.976741790771484,
      "activations/layer13_attention_weight_min": -27.73529815673828,
      "activations/layer14_attention_weight_max": 34.06000900268555,
      "activations/layer14_attention_weight_min": -30.186283111572266,
      "activations/layer15_attention_weight_max": 32.87980270385742,
      "activations/layer15_attention_weight_min": -31.47317886352539,
      "activations/layer16_attention_weight_max": 34.37757873535156,
      "activations/layer16_attention_weight_min": -32.24448013305664,
      "activations/layer17_attention_weight_max": 50.319419860839844,
      "activations/layer17_attention_weight_min": -45.561424255371094,
      "activations/layer18_attention_weight_max": 46.310768127441406,
      "activations/layer18_attention_weight_min": -39.42963790893555,
      "activations/layer19_attention_weight_max": 22.648345947265625,
      "activations/layer19_attention_weight_min": -22.39955711364746,
      "activations/layer1_attention_weight_max": 15.502157211303711,
      "activations/layer1_attention_weight_min": -14.283323287963867,
      "activations/layer20_attention_weight_max": 20.144145965576172,
      "activations/layer20_attention_weight_min": -20.792827606201172,
      "activations/layer21_attention_weight_max": 30.157487869262695,
      "activations/layer21_attention_weight_min": -20.16091537475586,
      "activations/layer22_attention_weight_max": 27.29537010192871,
      "activations/layer22_attention_weight_min": -22.91826057434082,
      "activations/layer23_attention_weight_max": 33.30655288696289,
      "activations/layer23_attention_weight_min": -24.88884735107422,
      "activations/layer2_attention_weight_max": 31.043590545654297,
      "activations/layer2_attention_weight_min": -30.13348960876465,
      "activations/layer3_attention_weight_max": 86.12725830078125,
      "activations/layer3_attention_weight_min": -83.81207275390625,
      "activations/layer4_attention_weight_max": 82.1559066772461,
      "activations/layer4_attention_weight_min": -82.22020721435547,
      "activations/layer5_attention_weight_max": 58.895042419433594,
      "activations/layer5_attention_weight_min": -70.98065948486328,
      "activations/layer6_attention_weight_max": 48.904396057128906,
      "activations/layer6_attention_weight_min": -49.27338409423828,
      "activations/layer7_attention_weight_max": 64.48648071289062,
      "activations/layer7_attention_weight_min": -64.54754638671875,
      "activations/layer8_attention_weight_max": 46.678409576416016,
      "activations/layer8_attention_weight_min": -50.64786911010742,
      "activations/layer9_attention_weight_max": 42.99285125732422,
      "activations/layer9_attention_weight_min": -45.82761764526367,
      "epoch": 21.06,
      "learning_rate": 1.4351893939393938e-05,
      "loss": 2.7063,
      "step": 362400
    },
    {
      "activations/layer0_attention_weight_max": 16.41880989074707,
      "activations/layer0_attention_weight_min": -14.078360557556152,
      "activations/layer10_attention_weight_max": 42.03845977783203,
      "activations/layer10_attention_weight_min": -37.68584060668945,
      "activations/layer11_attention_weight_max": 38.04330825805664,
      "activations/layer11_attention_weight_min": -36.30402374267578,
      "activations/layer12_attention_weight_max": 21.529232025146484,
      "activations/layer12_attention_weight_min": -24.897186279296875,
      "activations/layer13_attention_weight_max": 49.55986022949219,
      "activations/layer13_attention_weight_min": -32.5138053894043,
      "activations/layer14_attention_weight_max": 43.2025146484375,
      "activations/layer14_attention_weight_min": -33.510765075683594,
      "activations/layer15_attention_weight_max": 38.72312545776367,
      "activations/layer15_attention_weight_min": -35.03008270263672,
      "activations/layer16_attention_weight_max": 40.87981033325195,
      "activations/layer16_attention_weight_min": -35.37608337402344,
      "activations/layer17_attention_weight_max": 59.06791305541992,
      "activations/layer17_attention_weight_min": -50.649051666259766,
      "activations/layer18_attention_weight_max": 56.6861572265625,
      "activations/layer18_attention_weight_min": -46.3869514465332,
      "activations/layer19_attention_weight_max": 29.88885498046875,
      "activations/layer19_attention_weight_min": -25.96181297302246,
      "activations/layer1_attention_weight_max": 15.987764358520508,
      "activations/layer1_attention_weight_min": -13.496072769165039,
      "activations/layer20_attention_weight_max": 27.464336395263672,
      "activations/layer20_attention_weight_min": -24.50596809387207,
      "activations/layer21_attention_weight_max": 45.9838981628418,
      "activations/layer21_attention_weight_min": -22.751203536987305,
      "activations/layer22_attention_weight_max": 31.455530166625977,
      "activations/layer22_attention_weight_min": -26.313901901245117,
      "activations/layer23_attention_weight_max": 43.76312255859375,
      "activations/layer23_attention_weight_min": -25.078405380249023,
      "activations/layer2_attention_weight_max": 31.483993530273438,
      "activations/layer2_attention_weight_min": -29.621315002441406,
      "activations/layer3_attention_weight_max": 88.37478637695312,
      "activations/layer3_attention_weight_min": -85.24522399902344,
      "activations/layer4_attention_weight_max": 88.03339385986328,
      "activations/layer4_attention_weight_min": -81.83112335205078,
      "activations/layer5_attention_weight_max": 64.83753967285156,
      "activations/layer5_attention_weight_min": -68.80168151855469,
      "activations/layer6_attention_weight_max": 51.16570281982422,
      "activations/layer6_attention_weight_min": -52.28901672363281,
      "activations/layer7_attention_weight_max": 70.39515686035156,
      "activations/layer7_attention_weight_min": -67.298095703125,
      "activations/layer8_attention_weight_max": 53.257225036621094,
      "activations/layer8_attention_weight_min": -54.35407638549805,
      "activations/layer9_attention_weight_max": 52.722408294677734,
      "activations/layer9_attention_weight_min": -46.5849494934082,
      "epoch": 21.06,
      "learning_rate": 1.4332954545454545e-05,
      "loss": 2.7141,
      "step": 362450
    },
    {
      "activations/layer0_attention_weight_max": 15.56667709350586,
      "activations/layer0_attention_weight_min": -14.195801734924316,
      "activations/layer10_attention_weight_max": 43.85261154174805,
      "activations/layer10_attention_weight_min": -45.316558837890625,
      "activations/layer11_attention_weight_max": 40.38746643066406,
      "activations/layer11_attention_weight_min": -43.06319808959961,
      "activations/layer12_attention_weight_max": 21.05419921875,
      "activations/layer12_attention_weight_min": -27.4327335357666,
      "activations/layer13_attention_weight_max": 54.337196350097656,
      "activations/layer13_attention_weight_min": -35.59346389770508,
      "activations/layer14_attention_weight_max": 50.18326950073242,
      "activations/layer14_attention_weight_min": -36.11845016479492,
      "activations/layer15_attention_weight_max": 47.25234603881836,
      "activations/layer15_attention_weight_min": -37.45545196533203,
      "activations/layer16_attention_weight_max": 46.30393981933594,
      "activations/layer16_attention_weight_min": -45.048282623291016,
      "activations/layer17_attention_weight_max": 63.75520324707031,
      "activations/layer17_attention_weight_min": -61.21600341796875,
      "activations/layer18_attention_weight_max": 57.38869094848633,
      "activations/layer18_attention_weight_min": -53.12094497680664,
      "activations/layer19_attention_weight_max": 26.04572105407715,
      "activations/layer19_attention_weight_min": -27.695043563842773,
      "activations/layer1_attention_weight_max": 15.606213569641113,
      "activations/layer1_attention_weight_min": -13.545156478881836,
      "activations/layer20_attention_weight_max": 23.270118713378906,
      "activations/layer20_attention_weight_min": -22.79511070251465,
      "activations/layer21_attention_weight_max": 39.79054641723633,
      "activations/layer21_attention_weight_min": -25.12190818786621,
      "activations/layer22_attention_weight_max": 32.56452560424805,
      "activations/layer22_attention_weight_min": -26.377241134643555,
      "activations/layer23_attention_weight_max": 37.693145751953125,
      "activations/layer23_attention_weight_min": -29.073219299316406,
      "activations/layer2_attention_weight_max": 33.00330352783203,
      "activations/layer2_attention_weight_min": -30.715160369873047,
      "activations/layer3_attention_weight_max": 94.11699676513672,
      "activations/layer3_attention_weight_min": -92.08733367919922,
      "activations/layer4_attention_weight_max": 89.33015441894531,
      "activations/layer4_attention_weight_min": -85.84500885009766,
      "activations/layer5_attention_weight_max": 61.730072021484375,
      "activations/layer5_attention_weight_min": -72.70106506347656,
      "activations/layer6_attention_weight_max": 57.24714279174805,
      "activations/layer6_attention_weight_min": -55.65578079223633,
      "activations/layer7_attention_weight_max": 67.40402221679688,
      "activations/layer7_attention_weight_min": -70.0005874633789,
      "activations/layer8_attention_weight_max": 53.170196533203125,
      "activations/layer8_attention_weight_min": -57.04749298095703,
      "activations/layer9_attention_weight_max": 51.438385009765625,
      "activations/layer9_attention_weight_min": -57.8736686706543,
      "epoch": 21.06,
      "learning_rate": 1.4314015151515151e-05,
      "loss": 2.7262,
      "step": 362500
    },
    {
      "activations/layer0_attention_weight_max": 15.83503246307373,
      "activations/layer0_attention_weight_min": -14.023416519165039,
      "activations/layer10_attention_weight_max": 36.15016555786133,
      "activations/layer10_attention_weight_min": -38.00519943237305,
      "activations/layer11_attention_weight_max": 34.14619445800781,
      "activations/layer11_attention_weight_min": -34.15459060668945,
      "activations/layer12_attention_weight_max": 18.699838638305664,
      "activations/layer12_attention_weight_min": -30.896255493164062,
      "activations/layer13_attention_weight_max": 35.65550994873047,
      "activations/layer13_attention_weight_min": -27.87412452697754,
      "activations/layer14_attention_weight_max": 36.838436126708984,
      "activations/layer14_attention_weight_min": -31.25530242919922,
      "activations/layer15_attention_weight_max": 35.155391693115234,
      "activations/layer15_attention_weight_min": -30.826757431030273,
      "activations/layer16_attention_weight_max": 32.81593322753906,
      "activations/layer16_attention_weight_min": -31.744245529174805,
      "activations/layer17_attention_weight_max": 49.88782501220703,
      "activations/layer17_attention_weight_min": -45.2254753112793,
      "activations/layer18_attention_weight_max": 43.93260955810547,
      "activations/layer18_attention_weight_min": -39.73777770996094,
      "activations/layer19_attention_weight_max": 23.038375854492188,
      "activations/layer19_attention_weight_min": -21.482341766357422,
      "activations/layer1_attention_weight_max": 15.055695533752441,
      "activations/layer1_attention_weight_min": -13.191962242126465,
      "activations/layer20_attention_weight_max": 21.21050262451172,
      "activations/layer20_attention_weight_min": -20.177207946777344,
      "activations/layer21_attention_weight_max": 40.601051330566406,
      "activations/layer21_attention_weight_min": -23.9177303314209,
      "activations/layer22_attention_weight_max": 29.780487060546875,
      "activations/layer22_attention_weight_min": -26.58929443359375,
      "activations/layer23_attention_weight_max": 32.76127243041992,
      "activations/layer23_attention_weight_min": -25.178159713745117,
      "activations/layer2_attention_weight_max": 32.24332809448242,
      "activations/layer2_attention_weight_min": -30.7314510345459,
      "activations/layer3_attention_weight_max": 87.17863464355469,
      "activations/layer3_attention_weight_min": -87.8709945678711,
      "activations/layer4_attention_weight_max": 83.79659271240234,
      "activations/layer4_attention_weight_min": -81.32661437988281,
      "activations/layer5_attention_weight_max": 61.94641876220703,
      "activations/layer5_attention_weight_min": -70.87350463867188,
      "activations/layer6_attention_weight_max": 49.47526168823242,
      "activations/layer6_attention_weight_min": -53.58806228637695,
      "activations/layer7_attention_weight_max": 66.45033264160156,
      "activations/layer7_attention_weight_min": -65.98575592041016,
      "activations/layer8_attention_weight_max": 48.36623001098633,
      "activations/layer8_attention_weight_min": -51.95698547363281,
      "activations/layer9_attention_weight_max": 44.44642639160156,
      "activations/layer9_attention_weight_min": -47.35597229003906,
      "epoch": 21.07,
      "learning_rate": 1.4295075757575756e-05,
      "loss": 2.726,
      "step": 362550
    },
    {
      "activations/layer0_attention_weight_max": 15.748917579650879,
      "activations/layer0_attention_weight_min": -14.135516166687012,
      "activations/layer10_attention_weight_max": 38.3448486328125,
      "activations/layer10_attention_weight_min": -37.94353103637695,
      "activations/layer11_attention_weight_max": 35.65407943725586,
      "activations/layer11_attention_weight_min": -34.67074966430664,
      "activations/layer12_attention_weight_max": 19.39544677734375,
      "activations/layer12_attention_weight_min": -29.421716690063477,
      "activations/layer13_attention_weight_max": 49.411354064941406,
      "activations/layer13_attention_weight_min": -30.279150009155273,
      "activations/layer14_attention_weight_max": 43.602149963378906,
      "activations/layer14_attention_weight_min": -35.81890869140625,
      "activations/layer15_attention_weight_max": 42.10715866088867,
      "activations/layer15_attention_weight_min": -35.100406646728516,
      "activations/layer16_attention_weight_max": 43.34561538696289,
      "activations/layer16_attention_weight_min": -39.691925048828125,
      "activations/layer17_attention_weight_max": 62.80144119262695,
      "activations/layer17_attention_weight_min": -55.03019332885742,
      "activations/layer18_attention_weight_max": 58.47431945800781,
      "activations/layer18_attention_weight_min": -49.392601013183594,
      "activations/layer19_attention_weight_max": 29.954940795898438,
      "activations/layer19_attention_weight_min": -27.357534408569336,
      "activations/layer1_attention_weight_max": 16.48356056213379,
      "activations/layer1_attention_weight_min": -13.39518928527832,
      "activations/layer20_attention_weight_max": 30.504438400268555,
      "activations/layer20_attention_weight_min": -23.231237411499023,
      "activations/layer21_attention_weight_max": 52.205711364746094,
      "activations/layer21_attention_weight_min": -26.916107177734375,
      "activations/layer22_attention_weight_max": 38.63306427001953,
      "activations/layer22_attention_weight_min": -27.79564666748047,
      "activations/layer23_attention_weight_max": 47.236549377441406,
      "activations/layer23_attention_weight_min": -22.498023986816406,
      "activations/layer2_attention_weight_max": 31.0653076171875,
      "activations/layer2_attention_weight_min": -29.952369689941406,
      "activations/layer3_attention_weight_max": 83.39981842041016,
      "activations/layer3_attention_weight_min": -85.83212280273438,
      "activations/layer4_attention_weight_max": 79.0334243774414,
      "activations/layer4_attention_weight_min": -81.33184814453125,
      "activations/layer5_attention_weight_max": 61.904052734375,
      "activations/layer5_attention_weight_min": -69.95860290527344,
      "activations/layer6_attention_weight_max": 49.3473014831543,
      "activations/layer6_attention_weight_min": -48.99068832397461,
      "activations/layer7_attention_weight_max": 67.66808319091797,
      "activations/layer7_attention_weight_min": -65.25104522705078,
      "activations/layer8_attention_weight_max": 48.37638473510742,
      "activations/layer8_attention_weight_min": -51.637149810791016,
      "activations/layer9_attention_weight_max": 48.57689666748047,
      "activations/layer9_attention_weight_min": -43.50243377685547,
      "epoch": 21.07,
      "learning_rate": 1.4276136363636363e-05,
      "loss": 2.7113,
      "step": 362600
    },
    {
      "activations/layer0_attention_weight_max": 15.312546730041504,
      "activations/layer0_attention_weight_min": -14.43192195892334,
      "activations/layer10_attention_weight_max": 34.065895080566406,
      "activations/layer10_attention_weight_min": -35.87910842895508,
      "activations/layer11_attention_weight_max": 32.22483825683594,
      "activations/layer11_attention_weight_min": -35.076534271240234,
      "activations/layer12_attention_weight_max": 19.62361717224121,
      "activations/layer12_attention_weight_min": -24.24839210510254,
      "activations/layer13_attention_weight_max": 38.25434112548828,
      "activations/layer13_attention_weight_min": -28.26764488220215,
      "activations/layer14_attention_weight_max": 37.773597717285156,
      "activations/layer14_attention_weight_min": -30.73204803466797,
      "activations/layer15_attention_weight_max": 35.86022186279297,
      "activations/layer15_attention_weight_min": -31.39142417907715,
      "activations/layer16_attention_weight_max": 35.19316101074219,
      "activations/layer16_attention_weight_min": -32.1312141418457,
      "activations/layer17_attention_weight_max": 51.18989562988281,
      "activations/layer17_attention_weight_min": -45.121131896972656,
      "activations/layer18_attention_weight_max": 48.37504577636719,
      "activations/layer18_attention_weight_min": -39.21825408935547,
      "activations/layer19_attention_weight_max": 25.664663314819336,
      "activations/layer19_attention_weight_min": -24.14646339416504,
      "activations/layer1_attention_weight_max": 15.122675895690918,
      "activations/layer1_attention_weight_min": -13.261762619018555,
      "activations/layer20_attention_weight_max": 22.66021156311035,
      "activations/layer20_attention_weight_min": -21.993932723999023,
      "activations/layer21_attention_weight_max": 37.81329345703125,
      "activations/layer21_attention_weight_min": -23.461429595947266,
      "activations/layer22_attention_weight_max": 28.3718318939209,
      "activations/layer22_attention_weight_min": -26.153629302978516,
      "activations/layer23_attention_weight_max": 34.832725524902344,
      "activations/layer23_attention_weight_min": -24.390865325927734,
      "activations/layer2_attention_weight_max": 29.733739852905273,
      "activations/layer2_attention_weight_min": -28.951656341552734,
      "activations/layer3_attention_weight_max": 82.39212036132812,
      "activations/layer3_attention_weight_min": -84.2704849243164,
      "activations/layer4_attention_weight_max": 78.73843383789062,
      "activations/layer4_attention_weight_min": -79.89561462402344,
      "activations/layer5_attention_weight_max": 60.78351593017578,
      "activations/layer5_attention_weight_min": -72.68375396728516,
      "activations/layer6_attention_weight_max": 47.95669174194336,
      "activations/layer6_attention_weight_min": -51.64229965209961,
      "activations/layer7_attention_weight_max": 66.6007308959961,
      "activations/layer7_attention_weight_min": -69.19512939453125,
      "activations/layer8_attention_weight_max": 44.55199432373047,
      "activations/layer8_attention_weight_min": -50.48951721191406,
      "activations/layer9_attention_weight_max": 43.467437744140625,
      "activations/layer9_attention_weight_min": -46.08948516845703,
      "epoch": 21.07,
      "learning_rate": 1.425719696969697e-05,
      "loss": 2.7106,
      "step": 362650
    },
    {
      "activations/layer0_attention_weight_max": 15.708043098449707,
      "activations/layer0_attention_weight_min": -14.197237968444824,
      "activations/layer10_attention_weight_max": 36.278079986572266,
      "activations/layer10_attention_weight_min": -36.1225700378418,
      "activations/layer11_attention_weight_max": 33.69898986816406,
      "activations/layer11_attention_weight_min": -35.510597229003906,
      "activations/layer12_attention_weight_max": 17.89620018005371,
      "activations/layer12_attention_weight_min": -23.579002380371094,
      "activations/layer13_attention_weight_max": 33.63530731201172,
      "activations/layer13_attention_weight_min": -29.372522354125977,
      "activations/layer14_attention_weight_max": 35.31106948852539,
      "activations/layer14_attention_weight_min": -29.715688705444336,
      "activations/layer15_attention_weight_max": 35.68281936645508,
      "activations/layer15_attention_weight_min": -30.992958068847656,
      "activations/layer16_attention_weight_max": 34.085548400878906,
      "activations/layer16_attention_weight_min": -31.923322677612305,
      "activations/layer17_attention_weight_max": 51.07998275756836,
      "activations/layer17_attention_weight_min": -48.36921691894531,
      "activations/layer18_attention_weight_max": 44.93002700805664,
      "activations/layer18_attention_weight_min": -44.31349182128906,
      "activations/layer19_attention_weight_max": 27.25771141052246,
      "activations/layer19_attention_weight_min": -24.0846004486084,
      "activations/layer1_attention_weight_max": 15.647134780883789,
      "activations/layer1_attention_weight_min": -13.026396751403809,
      "activations/layer20_attention_weight_max": 23.22905731201172,
      "activations/layer20_attention_weight_min": -23.564599990844727,
      "activations/layer21_attention_weight_max": 32.54631042480469,
      "activations/layer21_attention_weight_min": -21.88504409790039,
      "activations/layer22_attention_weight_max": 28.60605812072754,
      "activations/layer22_attention_weight_min": -27.70578384399414,
      "activations/layer23_attention_weight_max": 33.12461471557617,
      "activations/layer23_attention_weight_min": -25.87096405029297,
      "activations/layer2_attention_weight_max": 31.211538314819336,
      "activations/layer2_attention_weight_min": -29.852615356445312,
      "activations/layer3_attention_weight_max": 83.40292358398438,
      "activations/layer3_attention_weight_min": -82.34497833251953,
      "activations/layer4_attention_weight_max": 80.5274887084961,
      "activations/layer4_attention_weight_min": -81.74078369140625,
      "activations/layer5_attention_weight_max": 60.240882873535156,
      "activations/layer5_attention_weight_min": -72.07816314697266,
      "activations/layer6_attention_weight_max": 49.6530647277832,
      "activations/layer6_attention_weight_min": -52.109676361083984,
      "activations/layer7_attention_weight_max": 66.02714538574219,
      "activations/layer7_attention_weight_min": -63.672977447509766,
      "activations/layer8_attention_weight_max": 45.36725616455078,
      "activations/layer8_attention_weight_min": -51.11916732788086,
      "activations/layer9_attention_weight_max": 43.43792724609375,
      "activations/layer9_attention_weight_min": -44.15534973144531,
      "epoch": 21.07,
      "learning_rate": 1.4238257575757574e-05,
      "loss": 2.7088,
      "step": 362700
    },
    {
      "activations/layer0_attention_weight_max": 16.670942306518555,
      "activations/layer0_attention_weight_min": -14.158143043518066,
      "activations/layer10_attention_weight_max": 34.910587310791016,
      "activations/layer10_attention_weight_min": -35.75977325439453,
      "activations/layer11_attention_weight_max": 33.2621955871582,
      "activations/layer11_attention_weight_min": -34.842472076416016,
      "activations/layer12_attention_weight_max": 19.63957977294922,
      "activations/layer12_attention_weight_min": -27.07352638244629,
      "activations/layer13_attention_weight_max": 36.055335998535156,
      "activations/layer13_attention_weight_min": -27.85591697692871,
      "activations/layer14_attention_weight_max": 38.6469612121582,
      "activations/layer14_attention_weight_min": -30.32251739501953,
      "activations/layer15_attention_weight_max": 36.00983810424805,
      "activations/layer15_attention_weight_min": -32.088096618652344,
      "activations/layer16_attention_weight_max": 36.79001235961914,
      "activations/layer16_attention_weight_min": -34.18927001953125,
      "activations/layer17_attention_weight_max": 55.41277313232422,
      "activations/layer17_attention_weight_min": -46.17626190185547,
      "activations/layer18_attention_weight_max": 47.266170501708984,
      "activations/layer18_attention_weight_min": -41.286312103271484,
      "activations/layer19_attention_weight_max": 27.042240142822266,
      "activations/layer19_attention_weight_min": -23.546194076538086,
      "activations/layer1_attention_weight_max": 15.353463172912598,
      "activations/layer1_attention_weight_min": -13.98355484008789,
      "activations/layer20_attention_weight_max": 24.817485809326172,
      "activations/layer20_attention_weight_min": -22.965538024902344,
      "activations/layer21_attention_weight_max": 39.436256408691406,
      "activations/layer21_attention_weight_min": -21.742431640625,
      "activations/layer22_attention_weight_max": 29.736026763916016,
      "activations/layer22_attention_weight_min": -26.22325897216797,
      "activations/layer23_attention_weight_max": 33.84352111816406,
      "activations/layer23_attention_weight_min": -23.8215389251709,
      "activations/layer2_attention_weight_max": 32.89490509033203,
      "activations/layer2_attention_weight_min": -31.944602966308594,
      "activations/layer3_attention_weight_max": 92.98079681396484,
      "activations/layer3_attention_weight_min": -88.6468505859375,
      "activations/layer4_attention_weight_max": 86.68338775634766,
      "activations/layer4_attention_weight_min": -81.88555145263672,
      "activations/layer5_attention_weight_max": 63.11875534057617,
      "activations/layer5_attention_weight_min": -68.74931335449219,
      "activations/layer6_attention_weight_max": 48.54897689819336,
      "activations/layer6_attention_weight_min": -49.30043411254883,
      "activations/layer7_attention_weight_max": 66.89234924316406,
      "activations/layer7_attention_weight_min": -63.671627044677734,
      "activations/layer8_attention_weight_max": 48.0452995300293,
      "activations/layer8_attention_weight_min": -49.241493225097656,
      "activations/layer9_attention_weight_max": 45.541988372802734,
      "activations/layer9_attention_weight_min": -46.764404296875,
      "epoch": 21.08,
      "learning_rate": 1.421931818181818e-05,
      "loss": 2.6996,
      "step": 362750
    },
    {
      "activations/layer0_attention_weight_max": 15.715420722961426,
      "activations/layer0_attention_weight_min": -14.220197677612305,
      "activations/layer10_attention_weight_max": 33.95997619628906,
      "activations/layer10_attention_weight_min": -34.76580810546875,
      "activations/layer11_attention_weight_max": 31.10812759399414,
      "activations/layer11_attention_weight_min": -35.48332977294922,
      "activations/layer12_attention_weight_max": 19.56606101989746,
      "activations/layer12_attention_weight_min": -23.05596160888672,
      "activations/layer13_attention_weight_max": 31.24167251586914,
      "activations/layer13_attention_weight_min": -25.223207473754883,
      "activations/layer14_attention_weight_max": 36.104736328125,
      "activations/layer14_attention_weight_min": -29.05875587463379,
      "activations/layer15_attention_weight_max": 33.21822738647461,
      "activations/layer15_attention_weight_min": -28.43709373474121,
      "activations/layer16_attention_weight_max": 34.40046691894531,
      "activations/layer16_attention_weight_min": -32.49421691894531,
      "activations/layer17_attention_weight_max": 49.88262939453125,
      "activations/layer17_attention_weight_min": -48.00569534301758,
      "activations/layer18_attention_weight_max": 47.90877151489258,
      "activations/layer18_attention_weight_min": -39.82123947143555,
      "activations/layer19_attention_weight_max": 27.423486709594727,
      "activations/layer19_attention_weight_min": -21.030851364135742,
      "activations/layer1_attention_weight_max": 15.482709884643555,
      "activations/layer1_attention_weight_min": -14.225422859191895,
      "activations/layer20_attention_weight_max": 26.693269729614258,
      "activations/layer20_attention_weight_min": -20.10242462158203,
      "activations/layer21_attention_weight_max": 33.693302154541016,
      "activations/layer21_attention_weight_min": -23.51051902770996,
      "activations/layer22_attention_weight_max": 28.83811378479004,
      "activations/layer22_attention_weight_min": -25.05046844482422,
      "activations/layer23_attention_weight_max": 36.551483154296875,
      "activations/layer23_attention_weight_min": -24.39285659790039,
      "activations/layer2_attention_weight_max": 32.69581985473633,
      "activations/layer2_attention_weight_min": -32.32253646850586,
      "activations/layer3_attention_weight_max": 87.93543243408203,
      "activations/layer3_attention_weight_min": -89.08282470703125,
      "activations/layer4_attention_weight_max": 82.22074127197266,
      "activations/layer4_attention_weight_min": -80.81237030029297,
      "activations/layer5_attention_weight_max": 60.68083190917969,
      "activations/layer5_attention_weight_min": -72.41932678222656,
      "activations/layer6_attention_weight_max": 52.5786247253418,
      "activations/layer6_attention_weight_min": -49.53816223144531,
      "activations/layer7_attention_weight_max": 63.12074279785156,
      "activations/layer7_attention_weight_min": -64.98367309570312,
      "activations/layer8_attention_weight_max": 42.986846923828125,
      "activations/layer8_attention_weight_min": -47.13823318481445,
      "activations/layer9_attention_weight_max": 39.123653411865234,
      "activations/layer9_attention_weight_min": -43.87266159057617,
      "epoch": 21.08,
      "learning_rate": 1.4200378787878786e-05,
      "loss": 2.6997,
      "step": 362800
    },
    {
      "activations/layer0_attention_weight_max": 16.585798263549805,
      "activations/layer0_attention_weight_min": -14.39105224609375,
      "activations/layer10_attention_weight_max": 36.371341705322266,
      "activations/layer10_attention_weight_min": -36.2427978515625,
      "activations/layer11_attention_weight_max": 35.43169403076172,
      "activations/layer11_attention_weight_min": -36.66632843017578,
      "activations/layer12_attention_weight_max": 19.76398468017578,
      "activations/layer12_attention_weight_min": -22.698226928710938,
      "activations/layer13_attention_weight_max": 35.06370544433594,
      "activations/layer13_attention_weight_min": -28.50800323486328,
      "activations/layer14_attention_weight_max": 36.39336395263672,
      "activations/layer14_attention_weight_min": -29.789560317993164,
      "activations/layer15_attention_weight_max": 33.681068420410156,
      "activations/layer15_attention_weight_min": -29.656282424926758,
      "activations/layer16_attention_weight_max": 35.51597213745117,
      "activations/layer16_attention_weight_min": -32.662681579589844,
      "activations/layer17_attention_weight_max": 54.352542877197266,
      "activations/layer17_attention_weight_min": -46.9238166809082,
      "activations/layer18_attention_weight_max": 46.796016693115234,
      "activations/layer18_attention_weight_min": -39.061805725097656,
      "activations/layer19_attention_weight_max": 25.31524085998535,
      "activations/layer19_attention_weight_min": -21.13003158569336,
      "activations/layer1_attention_weight_max": 15.341168403625488,
      "activations/layer1_attention_weight_min": -13.51582145690918,
      "activations/layer20_attention_weight_max": 24.84075355529785,
      "activations/layer20_attention_weight_min": -21.130918502807617,
      "activations/layer21_attention_weight_max": 35.9310188293457,
      "activations/layer21_attention_weight_min": -21.214521408081055,
      "activations/layer22_attention_weight_max": 29.966150283813477,
      "activations/layer22_attention_weight_min": -25.019102096557617,
      "activations/layer23_attention_weight_max": 35.815773010253906,
      "activations/layer23_attention_weight_min": -25.522825241088867,
      "activations/layer2_attention_weight_max": 31.32650375366211,
      "activations/layer2_attention_weight_min": -30.578800201416016,
      "activations/layer3_attention_weight_max": 85.07234954833984,
      "activations/layer3_attention_weight_min": -88.45021057128906,
      "activations/layer4_attention_weight_max": 79.20384979248047,
      "activations/layer4_attention_weight_min": -83.17818450927734,
      "activations/layer5_attention_weight_max": 61.84882736206055,
      "activations/layer5_attention_weight_min": -75.87120819091797,
      "activations/layer6_attention_weight_max": 47.716453552246094,
      "activations/layer6_attention_weight_min": -50.422481536865234,
      "activations/layer7_attention_weight_max": 66.86775207519531,
      "activations/layer7_attention_weight_min": -64.97074127197266,
      "activations/layer8_attention_weight_max": 44.84345626831055,
      "activations/layer8_attention_weight_min": -50.568153381347656,
      "activations/layer9_attention_weight_max": 46.479331970214844,
      "activations/layer9_attention_weight_min": -45.018524169921875,
      "epoch": 21.08,
      "learning_rate": 1.4181439393939392e-05,
      "loss": 2.6942,
      "step": 362850
    },
    {
      "activations/layer0_attention_weight_max": 16.854639053344727,
      "activations/layer0_attention_weight_min": -14.04873275756836,
      "activations/layer10_attention_weight_max": 34.95142364501953,
      "activations/layer10_attention_weight_min": -36.12439727783203,
      "activations/layer11_attention_weight_max": 32.70215606689453,
      "activations/layer11_attention_weight_min": -35.92153549194336,
      "activations/layer12_attention_weight_max": 19.150426864624023,
      "activations/layer12_attention_weight_min": -29.555126190185547,
      "activations/layer13_attention_weight_max": 32.52392578125,
      "activations/layer13_attention_weight_min": -27.998132705688477,
      "activations/layer14_attention_weight_max": 34.99781799316406,
      "activations/layer14_attention_weight_min": -29.91902732849121,
      "activations/layer15_attention_weight_max": 32.577144622802734,
      "activations/layer15_attention_weight_min": -32.064208984375,
      "activations/layer16_attention_weight_max": 34.552066802978516,
      "activations/layer16_attention_weight_min": -31.130569458007812,
      "activations/layer17_attention_weight_max": 54.01887893676758,
      "activations/layer17_attention_weight_min": -48.34288787841797,
      "activations/layer18_attention_weight_max": 48.8790397644043,
      "activations/layer18_attention_weight_min": -42.21290588378906,
      "activations/layer19_attention_weight_max": 24.2717342376709,
      "activations/layer19_attention_weight_min": -24.32813262939453,
      "activations/layer1_attention_weight_max": 15.369071006774902,
      "activations/layer1_attention_weight_min": -13.400758743286133,
      "activations/layer20_attention_weight_max": 23.540884017944336,
      "activations/layer20_attention_weight_min": -23.96807098388672,
      "activations/layer21_attention_weight_max": 42.480621337890625,
      "activations/layer21_attention_weight_min": -26.946971893310547,
      "activations/layer22_attention_weight_max": 30.898250579833984,
      "activations/layer22_attention_weight_min": -28.50864028930664,
      "activations/layer23_attention_weight_max": 38.69500732421875,
      "activations/layer23_attention_weight_min": -26.374000549316406,
      "activations/layer2_attention_weight_max": 31.611623764038086,
      "activations/layer2_attention_weight_min": -30.026309967041016,
      "activations/layer3_attention_weight_max": 85.93821716308594,
      "activations/layer3_attention_weight_min": -84.54685974121094,
      "activations/layer4_attention_weight_max": 79.81565856933594,
      "activations/layer4_attention_weight_min": -82.15308380126953,
      "activations/layer5_attention_weight_max": 59.814815521240234,
      "activations/layer5_attention_weight_min": -74.44548034667969,
      "activations/layer6_attention_weight_max": 46.313655853271484,
      "activations/layer6_attention_weight_min": -50.67261505126953,
      "activations/layer7_attention_weight_max": 62.319950103759766,
      "activations/layer7_attention_weight_min": -64.0750503540039,
      "activations/layer8_attention_weight_max": 45.237098693847656,
      "activations/layer8_attention_weight_min": -52.20035934448242,
      "activations/layer9_attention_weight_max": 42.403873443603516,
      "activations/layer9_attention_weight_min": -45.59331512451172,
      "epoch": 21.09,
      "learning_rate": 1.4162499999999999e-05,
      "loss": 2.7169,
      "step": 362900
    },
    {
      "activations/layer0_attention_weight_max": 15.381648063659668,
      "activations/layer0_attention_weight_min": -14.128525733947754,
      "activations/layer10_attention_weight_max": 37.70244216918945,
      "activations/layer10_attention_weight_min": -37.83079147338867,
      "activations/layer11_attention_weight_max": 33.79387664794922,
      "activations/layer11_attention_weight_min": -35.17388153076172,
      "activations/layer12_attention_weight_max": 19.827728271484375,
      "activations/layer12_attention_weight_min": -23.50724220275879,
      "activations/layer13_attention_weight_max": 32.74406433105469,
      "activations/layer13_attention_weight_min": -27.37333106994629,
      "activations/layer14_attention_weight_max": 35.706207275390625,
      "activations/layer14_attention_weight_min": -31.08832550048828,
      "activations/layer15_attention_weight_max": 35.44623947143555,
      "activations/layer15_attention_weight_min": -31.531099319458008,
      "activations/layer16_attention_weight_max": 37.882076263427734,
      "activations/layer16_attention_weight_min": -34.97000503540039,
      "activations/layer17_attention_weight_max": 55.36603927612305,
      "activations/layer17_attention_weight_min": -45.34032440185547,
      "activations/layer18_attention_weight_max": 52.00440216064453,
      "activations/layer18_attention_weight_min": -39.91606903076172,
      "activations/layer19_attention_weight_max": 24.129396438598633,
      "activations/layer19_attention_weight_min": -23.154027938842773,
      "activations/layer1_attention_weight_max": 15.161465644836426,
      "activations/layer1_attention_weight_min": -15.850628852844238,
      "activations/layer20_attention_weight_max": 23.953643798828125,
      "activations/layer20_attention_weight_min": -23.080406188964844,
      "activations/layer21_attention_weight_max": 36.336273193359375,
      "activations/layer21_attention_weight_min": -22.450122833251953,
      "activations/layer22_attention_weight_max": 28.052148818969727,
      "activations/layer22_attention_weight_min": -24.307231903076172,
      "activations/layer23_attention_weight_max": 35.03789138793945,
      "activations/layer23_attention_weight_min": -24.109785079956055,
      "activations/layer2_attention_weight_max": 32.90473175048828,
      "activations/layer2_attention_weight_min": -29.340579986572266,
      "activations/layer3_attention_weight_max": 86.55781555175781,
      "activations/layer3_attention_weight_min": -85.28179931640625,
      "activations/layer4_attention_weight_max": 80.34957122802734,
      "activations/layer4_attention_weight_min": -83.41981506347656,
      "activations/layer5_attention_weight_max": 59.55854797363281,
      "activations/layer5_attention_weight_min": -71.99183654785156,
      "activations/layer6_attention_weight_max": 49.81007385253906,
      "activations/layer6_attention_weight_min": -51.48653030395508,
      "activations/layer7_attention_weight_max": 66.60016632080078,
      "activations/layer7_attention_weight_min": -65.58590698242188,
      "activations/layer8_attention_weight_max": 47.89999008178711,
      "activations/layer8_attention_weight_min": -53.356292724609375,
      "activations/layer9_attention_weight_max": 45.08506393432617,
      "activations/layer9_attention_weight_min": -49.80179977416992,
      "epoch": 21.09,
      "learning_rate": 1.4143560606060604e-05,
      "loss": 2.7204,
      "step": 362950
    },
    {
      "activations/layer0_attention_weight_max": 16.38640594482422,
      "activations/layer0_attention_weight_min": -14.081210136413574,
      "activations/layer10_attention_weight_max": 38.13124465942383,
      "activations/layer10_attention_weight_min": -39.052833557128906,
      "activations/layer11_attention_weight_max": 37.50205993652344,
      "activations/layer11_attention_weight_min": -38.6247673034668,
      "activations/layer12_attention_weight_max": 19.232019424438477,
      "activations/layer12_attention_weight_min": -31.08415412902832,
      "activations/layer13_attention_weight_max": 39.033447265625,
      "activations/layer13_attention_weight_min": -30.372520446777344,
      "activations/layer14_attention_weight_max": 30.84284210205078,
      "activations/layer14_attention_weight_min": -30.989774703979492,
      "activations/layer15_attention_weight_max": 29.900096893310547,
      "activations/layer15_attention_weight_min": -30.112428665161133,
      "activations/layer16_attention_weight_max": 30.418933868408203,
      "activations/layer16_attention_weight_min": -30.76175880432129,
      "activations/layer17_attention_weight_max": 50.01108932495117,
      "activations/layer17_attention_weight_min": -48.51441192626953,
      "activations/layer18_attention_weight_max": 46.825416564941406,
      "activations/layer18_attention_weight_min": -51.28788757324219,
      "activations/layer19_attention_weight_max": 23.34992027282715,
      "activations/layer19_attention_weight_min": -26.918155670166016,
      "activations/layer1_attention_weight_max": 15.87398910522461,
      "activations/layer1_attention_weight_min": -14.887845039367676,
      "activations/layer20_attention_weight_max": 23.399633407592773,
      "activations/layer20_attention_weight_min": -24.036893844604492,
      "activations/layer21_attention_weight_max": 32.82810592651367,
      "activations/layer21_attention_weight_min": -26.036394119262695,
      "activations/layer22_attention_weight_max": 28.75440216064453,
      "activations/layer22_attention_weight_min": -26.276824951171875,
      "activations/layer23_attention_weight_max": 34.90886688232422,
      "activations/layer23_attention_weight_min": -25.516569137573242,
      "activations/layer2_attention_weight_max": 32.37417984008789,
      "activations/layer2_attention_weight_min": -30.132850646972656,
      "activations/layer3_attention_weight_max": 97.06358337402344,
      "activations/layer3_attention_weight_min": -84.67047882080078,
      "activations/layer4_attention_weight_max": 84.2878189086914,
      "activations/layer4_attention_weight_min": -79.55927276611328,
      "activations/layer5_attention_weight_max": 59.10136413574219,
      "activations/layer5_attention_weight_min": -73.21046447753906,
      "activations/layer6_attention_weight_max": 48.91446304321289,
      "activations/layer6_attention_weight_min": -47.461116790771484,
      "activations/layer7_attention_weight_max": 70.16072845458984,
      "activations/layer7_attention_weight_min": -71.98123931884766,
      "activations/layer8_attention_weight_max": 51.07261276245117,
      "activations/layer8_attention_weight_min": -51.54209518432617,
      "activations/layer9_attention_weight_max": 52.059791564941406,
      "activations/layer9_attention_weight_min": -51.43742752075195,
      "epoch": 21.09,
      "learning_rate": 1.412462121212121e-05,
      "loss": 2.7157,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.469,
      "eval_samples_per_second": 507.025,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.469,
      "eval_openwebtext_samples_per_second": 507.025,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9967,
      "eval_wikitext_samples_per_second": 228.378,
      "step": 363000
    },
    {
      "epoch": 21.09,
      "eval_lambada_loss": 2.396484375,
      "eval_lambada_ppl": 10.984491067549964,
      "eval_lambada_runtime": 9.5982,
      "eval_lambada_samples_per_second": 507.284,
      "step": 363000
    },
    {
      "activations/layer0_attention_weight_max": 16.332910537719727,
      "activations/layer0_attention_weight_min": -14.56937026977539,
      "activations/layer10_attention_weight_max": 45.79008865356445,
      "activations/layer10_attention_weight_min": -40.16984558105469,
      "activations/layer11_attention_weight_max": 45.494293212890625,
      "activations/layer11_attention_weight_min": -40.10190963745117,
      "activations/layer12_attention_weight_max": 20.30307388305664,
      "activations/layer12_attention_weight_min": -22.87557601928711,
      "activations/layer13_attention_weight_max": 56.32622528076172,
      "activations/layer13_attention_weight_min": -33.22296905517578,
      "activations/layer14_attention_weight_max": 62.75605773925781,
      "activations/layer14_attention_weight_min": -34.16347122192383,
      "activations/layer15_attention_weight_max": 53.51105499267578,
      "activations/layer15_attention_weight_min": -34.109256744384766,
      "activations/layer16_attention_weight_max": 38.40335464477539,
      "activations/layer16_attention_weight_min": -36.05598449707031,
      "activations/layer17_attention_weight_max": 56.68523406982422,
      "activations/layer17_attention_weight_min": -50.846771240234375,
      "activations/layer18_attention_weight_max": 55.41868209838867,
      "activations/layer18_attention_weight_min": -47.508174896240234,
      "activations/layer19_attention_weight_max": 27.514554977416992,
      "activations/layer19_attention_weight_min": -27.818470001220703,
      "activations/layer1_attention_weight_max": 15.825569152832031,
      "activations/layer1_attention_weight_min": -15.280390739440918,
      "activations/layer20_attention_weight_max": 28.1737060546875,
      "activations/layer20_attention_weight_min": -24.13062858581543,
      "activations/layer21_attention_weight_max": 49.855220794677734,
      "activations/layer21_attention_weight_min": -24.640844345092773,
      "activations/layer22_attention_weight_max": 34.07710266113281,
      "activations/layer22_attention_weight_min": -27.748579025268555,
      "activations/layer23_attention_weight_max": 40.211387634277344,
      "activations/layer23_attention_weight_min": -24.198528289794922,
      "activations/layer2_attention_weight_max": 30.681570053100586,
      "activations/layer2_attention_weight_min": -31.748680114746094,
      "activations/layer3_attention_weight_max": 87.72042083740234,
      "activations/layer3_attention_weight_min": -86.66364288330078,
      "activations/layer4_attention_weight_max": 80.99394226074219,
      "activations/layer4_attention_weight_min": -84.4999008178711,
      "activations/layer5_attention_weight_max": 64.83662414550781,
      "activations/layer5_attention_weight_min": -72.95755004882812,
      "activations/layer6_attention_weight_max": 47.040714263916016,
      "activations/layer6_attention_weight_min": -50.398685455322266,
      "activations/layer7_attention_weight_max": 74.44972229003906,
      "activations/layer7_attention_weight_min": -68.27367401123047,
      "activations/layer8_attention_weight_max": 56.76692199707031,
      "activations/layer8_attention_weight_min": -49.866127014160156,
      "activations/layer9_attention_weight_max": 58.85789108276367,
      "activations/layer9_attention_weight_min": -48.822505950927734,
      "epoch": 21.1,
      "learning_rate": 1.4105681818181817e-05,
      "loss": 2.7152,
      "step": 363050
    },
    {
      "activations/layer0_attention_weight_max": 15.979938507080078,
      "activations/layer0_attention_weight_min": -13.887372016906738,
      "activations/layer10_attention_weight_max": 40.8973388671875,
      "activations/layer10_attention_weight_min": -41.2297477722168,
      "activations/layer11_attention_weight_max": 37.24651336669922,
      "activations/layer11_attention_weight_min": -37.802494049072266,
      "activations/layer12_attention_weight_max": 20.8453369140625,
      "activations/layer12_attention_weight_min": -28.263912200927734,
      "activations/layer13_attention_weight_max": 47.167884826660156,
      "activations/layer13_attention_weight_min": -32.54383850097656,
      "activations/layer14_attention_weight_max": 46.137535095214844,
      "activations/layer14_attention_weight_min": -32.761051177978516,
      "activations/layer15_attention_weight_max": 45.71491622924805,
      "activations/layer15_attention_weight_min": -31.596973419189453,
      "activations/layer16_attention_weight_max": 40.285247802734375,
      "activations/layer16_attention_weight_min": -35.67204284667969,
      "activations/layer17_attention_weight_max": 61.918212890625,
      "activations/layer17_attention_weight_min": -53.562435150146484,
      "activations/layer18_attention_weight_max": 56.29051971435547,
      "activations/layer18_attention_weight_min": -46.62982177734375,
      "activations/layer19_attention_weight_max": 28.49094009399414,
      "activations/layer19_attention_weight_min": -25.534772872924805,
      "activations/layer1_attention_weight_max": 15.160134315490723,
      "activations/layer1_attention_weight_min": -13.101151466369629,
      "activations/layer20_attention_weight_max": 29.0062198638916,
      "activations/layer20_attention_weight_min": -21.322532653808594,
      "activations/layer21_attention_weight_max": 52.577449798583984,
      "activations/layer21_attention_weight_min": -25.360525131225586,
      "activations/layer22_attention_weight_max": 36.420658111572266,
      "activations/layer22_attention_weight_min": -27.314044952392578,
      "activations/layer23_attention_weight_max": 44.038909912109375,
      "activations/layer23_attention_weight_min": -22.148618698120117,
      "activations/layer2_attention_weight_max": 31.96869659423828,
      "activations/layer2_attention_weight_min": -30.337955474853516,
      "activations/layer3_attention_weight_max": 89.00641632080078,
      "activations/layer3_attention_weight_min": -86.93917083740234,
      "activations/layer4_attention_weight_max": 85.6848373413086,
      "activations/layer4_attention_weight_min": -82.96336364746094,
      "activations/layer5_attention_weight_max": 64.35006713867188,
      "activations/layer5_attention_weight_min": -73.2381820678711,
      "activations/layer6_attention_weight_max": 51.39186096191406,
      "activations/layer6_attention_weight_min": -51.14289474487305,
      "activations/layer7_attention_weight_max": 67.04217529296875,
      "activations/layer7_attention_weight_min": -70.85111236572266,
      "activations/layer8_attention_weight_max": 54.15255355834961,
      "activations/layer8_attention_weight_min": -55.81736373901367,
      "activations/layer9_attention_weight_max": 50.79513931274414,
      "activations/layer9_attention_weight_min": -51.44110870361328,
      "epoch": 21.1,
      "learning_rate": 1.4086742424242423e-05,
      "loss": 2.7434,
      "step": 363100
    },
    {
      "activations/layer0_attention_weight_max": 15.281764030456543,
      "activations/layer0_attention_weight_min": -14.410270690917969,
      "activations/layer10_attention_weight_max": 35.338199615478516,
      "activations/layer10_attention_weight_min": -35.911468505859375,
      "activations/layer11_attention_weight_max": 35.491966247558594,
      "activations/layer11_attention_weight_min": -35.188385009765625,
      "activations/layer12_attention_weight_max": 18.951587677001953,
      "activations/layer12_attention_weight_min": -24.68718719482422,
      "activations/layer13_attention_weight_max": 34.45429611206055,
      "activations/layer13_attention_weight_min": -28.569095611572266,
      "activations/layer14_attention_weight_max": 35.80839538574219,
      "activations/layer14_attention_weight_min": -31.58665657043457,
      "activations/layer15_attention_weight_max": 32.95372772216797,
      "activations/layer15_attention_weight_min": -31.73267936706543,
      "activations/layer16_attention_weight_max": 34.6851921081543,
      "activations/layer16_attention_weight_min": -33.505985260009766,
      "activations/layer17_attention_weight_max": 52.63579177856445,
      "activations/layer17_attention_weight_min": -45.201332092285156,
      "activations/layer18_attention_weight_max": 49.685943603515625,
      "activations/layer18_attention_weight_min": -40.50151824951172,
      "activations/layer19_attention_weight_max": 24.477275848388672,
      "activations/layer19_attention_weight_min": -23.491195678710938,
      "activations/layer1_attention_weight_max": 15.906891822814941,
      "activations/layer1_attention_weight_min": -14.319085121154785,
      "activations/layer20_attention_weight_max": 25.06871223449707,
      "activations/layer20_attention_weight_min": -22.65546226501465,
      "activations/layer21_attention_weight_max": 38.1743049621582,
      "activations/layer21_attention_weight_min": -25.11357307434082,
      "activations/layer22_attention_weight_max": 28.69645881652832,
      "activations/layer22_attention_weight_min": -24.668787002563477,
      "activations/layer23_attention_weight_max": 34.09132385253906,
      "activations/layer23_attention_weight_min": -23.65984344482422,
      "activations/layer2_attention_weight_max": 31.568614959716797,
      "activations/layer2_attention_weight_min": -29.80411148071289,
      "activations/layer3_attention_weight_max": 85.43948364257812,
      "activations/layer3_attention_weight_min": -82.36857604980469,
      "activations/layer4_attention_weight_max": 82.9088363647461,
      "activations/layer4_attention_weight_min": -82.14762115478516,
      "activations/layer5_attention_weight_max": 66.67557525634766,
      "activations/layer5_attention_weight_min": -76.66358947753906,
      "activations/layer6_attention_weight_max": 50.08797836303711,
      "activations/layer6_attention_weight_min": -51.75186538696289,
      "activations/layer7_attention_weight_max": 66.0306625366211,
      "activations/layer7_attention_weight_min": -69.54200744628906,
      "activations/layer8_attention_weight_max": 46.62662887573242,
      "activations/layer8_attention_weight_min": -49.60469436645508,
      "activations/layer9_attention_weight_max": 41.70976257324219,
      "activations/layer9_attention_weight_min": -45.227848052978516,
      "epoch": 21.1,
      "learning_rate": 1.406780303030303e-05,
      "loss": 2.7027,
      "step": 363150
    },
    {
      "activations/layer0_attention_weight_max": 15.504420280456543,
      "activations/layer0_attention_weight_min": -14.10584831237793,
      "activations/layer10_attention_weight_max": 39.71662139892578,
      "activations/layer10_attention_weight_min": -39.02465057373047,
      "activations/layer11_attention_weight_max": 36.78068161010742,
      "activations/layer11_attention_weight_min": -36.652305603027344,
      "activations/layer12_attention_weight_max": 19.288667678833008,
      "activations/layer12_attention_weight_min": -27.997093200683594,
      "activations/layer13_attention_weight_max": 39.282249450683594,
      "activations/layer13_attention_weight_min": -28.517976760864258,
      "activations/layer14_attention_weight_max": 37.83783721923828,
      "activations/layer14_attention_weight_min": -29.926589965820312,
      "activations/layer15_attention_weight_max": 35.34951400756836,
      "activations/layer15_attention_weight_min": -30.77254295349121,
      "activations/layer16_attention_weight_max": 36.286170959472656,
      "activations/layer16_attention_weight_min": -30.85447883605957,
      "activations/layer17_attention_weight_max": 58.04998016357422,
      "activations/layer17_attention_weight_min": -44.32185745239258,
      "activations/layer18_attention_weight_max": 52.860626220703125,
      "activations/layer18_attention_weight_min": -38.71340560913086,
      "activations/layer19_attention_weight_max": 26.149703979492188,
      "activations/layer19_attention_weight_min": -24.037336349487305,
      "activations/layer1_attention_weight_max": 16.44330406188965,
      "activations/layer1_attention_weight_min": -15.090729713439941,
      "activations/layer20_attention_weight_max": 24.58116912841797,
      "activations/layer20_attention_weight_min": -20.337215423583984,
      "activations/layer21_attention_weight_max": 37.10759353637695,
      "activations/layer21_attention_weight_min": -22.267560958862305,
      "activations/layer22_attention_weight_max": 28.87926483154297,
      "activations/layer22_attention_weight_min": -25.391035079956055,
      "activations/layer23_attention_weight_max": 39.54279708862305,
      "activations/layer23_attention_weight_min": -26.571868896484375,
      "activations/layer2_attention_weight_max": 30.990467071533203,
      "activations/layer2_attention_weight_min": -31.193038940429688,
      "activations/layer3_attention_weight_max": 87.60256958007812,
      "activations/layer3_attention_weight_min": -87.82948303222656,
      "activations/layer4_attention_weight_max": 84.48091888427734,
      "activations/layer4_attention_weight_min": -87.21251678466797,
      "activations/layer5_attention_weight_max": 62.34891891479492,
      "activations/layer5_attention_weight_min": -68.7303237915039,
      "activations/layer6_attention_weight_max": 49.1751594543457,
      "activations/layer6_attention_weight_min": -52.74922561645508,
      "activations/layer7_attention_weight_max": 74.10775756835938,
      "activations/layer7_attention_weight_min": -68.15101623535156,
      "activations/layer8_attention_weight_max": 50.07998275756836,
      "activations/layer8_attention_weight_min": -52.13865661621094,
      "activations/layer9_attention_weight_max": 48.54225540161133,
      "activations/layer9_attention_weight_min": -48.19324493408203,
      "epoch": 21.1,
      "learning_rate": 1.4048863636363636e-05,
      "loss": 2.7129,
      "step": 363200
    },
    {
      "activations/layer0_attention_weight_max": 16.371248245239258,
      "activations/layer0_attention_weight_min": -14.215585708618164,
      "activations/layer10_attention_weight_max": 36.04525375366211,
      "activations/layer10_attention_weight_min": -40.08989334106445,
      "activations/layer11_attention_weight_max": 34.14623260498047,
      "activations/layer11_attention_weight_min": -38.488590240478516,
      "activations/layer12_attention_weight_max": 19.538232803344727,
      "activations/layer12_attention_weight_min": -29.638240814208984,
      "activations/layer13_attention_weight_max": 33.38694763183594,
      "activations/layer13_attention_weight_min": -27.706880569458008,
      "activations/layer14_attention_weight_max": 32.647274017333984,
      "activations/layer14_attention_weight_min": -31.90447235107422,
      "activations/layer15_attention_weight_max": 32.24656677246094,
      "activations/layer15_attention_weight_min": -30.801803588867188,
      "activations/layer16_attention_weight_max": 33.85906219482422,
      "activations/layer16_attention_weight_min": -32.655704498291016,
      "activations/layer17_attention_weight_max": 48.920799255371094,
      "activations/layer17_attention_weight_min": -43.420677185058594,
      "activations/layer18_attention_weight_max": 45.72126388549805,
      "activations/layer18_attention_weight_min": -41.66666793823242,
      "activations/layer19_attention_weight_max": 24.799556732177734,
      "activations/layer19_attention_weight_min": -23.7838134765625,
      "activations/layer1_attention_weight_max": 15.680804252624512,
      "activations/layer1_attention_weight_min": -12.73169231414795,
      "activations/layer20_attention_weight_max": 22.048978805541992,
      "activations/layer20_attention_weight_min": -22.481679916381836,
      "activations/layer21_attention_weight_max": 37.259056091308594,
      "activations/layer21_attention_weight_min": -21.97212791442871,
      "activations/layer22_attention_weight_max": 29.770769119262695,
      "activations/layer22_attention_weight_min": -25.910011291503906,
      "activations/layer23_attention_weight_max": 36.88096618652344,
      "activations/layer23_attention_weight_min": -23.720333099365234,
      "activations/layer2_attention_weight_max": 31.218807220458984,
      "activations/layer2_attention_weight_min": -29.727628707885742,
      "activations/layer3_attention_weight_max": 90.19139862060547,
      "activations/layer3_attention_weight_min": -86.55072784423828,
      "activations/layer4_attention_weight_max": 86.3591537475586,
      "activations/layer4_attention_weight_min": -86.53429412841797,
      "activations/layer5_attention_weight_max": 62.44761657714844,
      "activations/layer5_attention_weight_min": -77.26376342773438,
      "activations/layer6_attention_weight_max": 48.95820617675781,
      "activations/layer6_attention_weight_min": -50.10932922363281,
      "activations/layer7_attention_weight_max": 65.4066162109375,
      "activations/layer7_attention_weight_min": -67.86028289794922,
      "activations/layer8_attention_weight_max": 47.84238052368164,
      "activations/layer8_attention_weight_min": -51.481117248535156,
      "activations/layer9_attention_weight_max": 43.948020935058594,
      "activations/layer9_attention_weight_min": -48.371097564697266,
      "epoch": 21.11,
      "learning_rate": 1.4029924242424241e-05,
      "loss": 2.7215,
      "step": 363250
    },
    {
      "activations/layer0_attention_weight_max": 14.803771018981934,
      "activations/layer0_attention_weight_min": -14.088191032409668,
      "activations/layer10_attention_weight_max": 40.0711555480957,
      "activations/layer10_attention_weight_min": -40.420772552490234,
      "activations/layer11_attention_weight_max": 37.59445571899414,
      "activations/layer11_attention_weight_min": -37.54111099243164,
      "activations/layer12_attention_weight_max": 23.74985694885254,
      "activations/layer12_attention_weight_min": -30.414464950561523,
      "activations/layer13_attention_weight_max": 33.09920883178711,
      "activations/layer13_attention_weight_min": -27.071834564208984,
      "activations/layer14_attention_weight_max": 38.9738655090332,
      "activations/layer14_attention_weight_min": -32.364280700683594,
      "activations/layer15_attention_weight_max": 37.70468521118164,
      "activations/layer15_attention_weight_min": -32.47986602783203,
      "activations/layer16_attention_weight_max": 38.70669937133789,
      "activations/layer16_attention_weight_min": -34.48560333251953,
      "activations/layer17_attention_weight_max": 55.076438903808594,
      "activations/layer17_attention_weight_min": -49.54253005981445,
      "activations/layer18_attention_weight_max": 56.15046310424805,
      "activations/layer18_attention_weight_min": -46.3853874206543,
      "activations/layer19_attention_weight_max": 27.074392318725586,
      "activations/layer19_attention_weight_min": -25.983869552612305,
      "activations/layer1_attention_weight_max": 15.570746421813965,
      "activations/layer1_attention_weight_min": -13.655839920043945,
      "activations/layer20_attention_weight_max": 24.447649002075195,
      "activations/layer20_attention_weight_min": -24.702669143676758,
      "activations/layer21_attention_weight_max": 38.47233963012695,
      "activations/layer21_attention_weight_min": -23.52872657775879,
      "activations/layer22_attention_weight_max": 34.501182556152344,
      "activations/layer22_attention_weight_min": -29.696857452392578,
      "activations/layer23_attention_weight_max": 42.89019775390625,
      "activations/layer23_attention_weight_min": -29.063613891601562,
      "activations/layer2_attention_weight_max": 31.89938735961914,
      "activations/layer2_attention_weight_min": -31.378355026245117,
      "activations/layer3_attention_weight_max": 91.8982925415039,
      "activations/layer3_attention_weight_min": -92.18717193603516,
      "activations/layer4_attention_weight_max": 86.65322875976562,
      "activations/layer4_attention_weight_min": -86.6951675415039,
      "activations/layer5_attention_weight_max": 61.74494934082031,
      "activations/layer5_attention_weight_min": -71.56961059570312,
      "activations/layer6_attention_weight_max": 51.94516372680664,
      "activations/layer6_attention_weight_min": -50.99701690673828,
      "activations/layer7_attention_weight_max": 69.72418975830078,
      "activations/layer7_attention_weight_min": -67.3152084350586,
      "activations/layer8_attention_weight_max": 50.34243392944336,
      "activations/layer8_attention_weight_min": -53.42763900756836,
      "activations/layer9_attention_weight_max": 49.16515350341797,
      "activations/layer9_attention_weight_min": -47.853111267089844,
      "epoch": 21.11,
      "learning_rate": 1.4010984848484848e-05,
      "loss": 2.7097,
      "step": 363300
    },
    {
      "activations/layer0_attention_weight_max": 15.432832717895508,
      "activations/layer0_attention_weight_min": -14.221725463867188,
      "activations/layer10_attention_weight_max": 36.473697662353516,
      "activations/layer10_attention_weight_min": -39.15941619873047,
      "activations/layer11_attention_weight_max": 35.879737854003906,
      "activations/layer11_attention_weight_min": -35.56308364868164,
      "activations/layer12_attention_weight_max": 19.779083251953125,
      "activations/layer12_attention_weight_min": -27.73816680908203,
      "activations/layer13_attention_weight_max": 47.19336700439453,
      "activations/layer13_attention_weight_min": -32.40400314331055,
      "activations/layer14_attention_weight_max": 42.8772087097168,
      "activations/layer14_attention_weight_min": -33.08327865600586,
      "activations/layer15_attention_weight_max": 41.61759567260742,
      "activations/layer15_attention_weight_min": -32.291786193847656,
      "activations/layer16_attention_weight_max": 39.300880432128906,
      "activations/layer16_attention_weight_min": -36.66407775878906,
      "activations/layer17_attention_weight_max": 63.60621643066406,
      "activations/layer17_attention_weight_min": -50.4409294128418,
      "activations/layer18_attention_weight_max": 59.592220306396484,
      "activations/layer18_attention_weight_min": -49.65030288696289,
      "activations/layer19_attention_weight_max": 31.637998580932617,
      "activations/layer19_attention_weight_min": -26.383747100830078,
      "activations/layer1_attention_weight_max": 15.258598327636719,
      "activations/layer1_attention_weight_min": -14.314395904541016,
      "activations/layer20_attention_weight_max": 29.029857635498047,
      "activations/layer20_attention_weight_min": -22.233421325683594,
      "activations/layer21_attention_weight_max": 46.6641731262207,
      "activations/layer21_attention_weight_min": -30.178619384765625,
      "activations/layer22_attention_weight_max": 32.46366500854492,
      "activations/layer22_attention_weight_min": -27.73352813720703,
      "activations/layer23_attention_weight_max": 43.265380859375,
      "activations/layer23_attention_weight_min": -25.410350799560547,
      "activations/layer2_attention_weight_max": 31.039852142333984,
      "activations/layer2_attention_weight_min": -30.119529724121094,
      "activations/layer3_attention_weight_max": 88.93457794189453,
      "activations/layer3_attention_weight_min": -85.97854614257812,
      "activations/layer4_attention_weight_max": 83.1668930053711,
      "activations/layer4_attention_weight_min": -82.10530853271484,
      "activations/layer5_attention_weight_max": 61.69715881347656,
      "activations/layer5_attention_weight_min": -71.21556091308594,
      "activations/layer6_attention_weight_max": 50.11960220336914,
      "activations/layer6_attention_weight_min": -51.444583892822266,
      "activations/layer7_attention_weight_max": 66.48654174804688,
      "activations/layer7_attention_weight_min": -69.99300384521484,
      "activations/layer8_attention_weight_max": 46.900482177734375,
      "activations/layer8_attention_weight_min": -54.841331481933594,
      "activations/layer9_attention_weight_max": 46.93792724609375,
      "activations/layer9_attention_weight_min": -50.6834831237793,
      "epoch": 21.11,
      "learning_rate": 1.3992045454545454e-05,
      "loss": 2.7075,
      "step": 363350
    },
    {
      "activations/layer0_attention_weight_max": 15.452775001525879,
      "activations/layer0_attention_weight_min": -14.05573558807373,
      "activations/layer10_attention_weight_max": 37.648704528808594,
      "activations/layer10_attention_weight_min": -36.69984817504883,
      "activations/layer11_attention_weight_max": 35.681034088134766,
      "activations/layer11_attention_weight_min": -37.881187438964844,
      "activations/layer12_attention_weight_max": 20.24346923828125,
      "activations/layer12_attention_weight_min": -25.560897827148438,
      "activations/layer13_attention_weight_max": 31.72455596923828,
      "activations/layer13_attention_weight_min": -29.54292869567871,
      "activations/layer14_attention_weight_max": 37.2364616394043,
      "activations/layer14_attention_weight_min": -32.6563720703125,
      "activations/layer15_attention_weight_max": 35.559452056884766,
      "activations/layer15_attention_weight_min": -33.34629440307617,
      "activations/layer16_attention_weight_max": 38.36216354370117,
      "activations/layer16_attention_weight_min": -34.33043670654297,
      "activations/layer17_attention_weight_max": 56.43982696533203,
      "activations/layer17_attention_weight_min": -47.27885055541992,
      "activations/layer18_attention_weight_max": 49.09114456176758,
      "activations/layer18_attention_weight_min": -43.22000503540039,
      "activations/layer19_attention_weight_max": 26.434524536132812,
      "activations/layer19_attention_weight_min": -25.948740005493164,
      "activations/layer1_attention_weight_max": 15.660296440124512,
      "activations/layer1_attention_weight_min": -14.444314002990723,
      "activations/layer20_attention_weight_max": 25.173351287841797,
      "activations/layer20_attention_weight_min": -24.22740936279297,
      "activations/layer21_attention_weight_max": 39.54957962036133,
      "activations/layer21_attention_weight_min": -23.089311599731445,
      "activations/layer22_attention_weight_max": 34.257991790771484,
      "activations/layer22_attention_weight_min": -28.27608871459961,
      "activations/layer23_attention_weight_max": 40.193763732910156,
      "activations/layer23_attention_weight_min": -23.01911735534668,
      "activations/layer2_attention_weight_max": 31.737545013427734,
      "activations/layer2_attention_weight_min": -30.6634521484375,
      "activations/layer3_attention_weight_max": 87.35680389404297,
      "activations/layer3_attention_weight_min": -86.79769134521484,
      "activations/layer4_attention_weight_max": 86.26305389404297,
      "activations/layer4_attention_weight_min": -84.92390441894531,
      "activations/layer5_attention_weight_max": 62.12943649291992,
      "activations/layer5_attention_weight_min": -74.86062622070312,
      "activations/layer6_attention_weight_max": 49.01502990722656,
      "activations/layer6_attention_weight_min": -52.10995864868164,
      "activations/layer7_attention_weight_max": 66.40645599365234,
      "activations/layer7_attention_weight_min": -67.043212890625,
      "activations/layer8_attention_weight_max": 48.46609878540039,
      "activations/layer8_attention_weight_min": -50.30839538574219,
      "activations/layer9_attention_weight_max": 47.12250518798828,
      "activations/layer9_attention_weight_min": -45.40379333496094,
      "epoch": 21.12,
      "learning_rate": 1.397310606060606e-05,
      "loss": 2.7176,
      "step": 363400
    },
    {
      "activations/layer0_attention_weight_max": 15.919984817504883,
      "activations/layer0_attention_weight_min": -14.264517784118652,
      "activations/layer10_attention_weight_max": 37.035343170166016,
      "activations/layer10_attention_weight_min": -38.95206832885742,
      "activations/layer11_attention_weight_max": 34.01038360595703,
      "activations/layer11_attention_weight_min": -33.800628662109375,
      "activations/layer12_attention_weight_max": 18.060697555541992,
      "activations/layer12_attention_weight_min": -24.49577522277832,
      "activations/layer13_attention_weight_max": 34.58265686035156,
      "activations/layer13_attention_weight_min": -27.016910552978516,
      "activations/layer14_attention_weight_max": 38.214656829833984,
      "activations/layer14_attention_weight_min": -29.683387756347656,
      "activations/layer15_attention_weight_max": 35.89411163330078,
      "activations/layer15_attention_weight_min": -32.02066421508789,
      "activations/layer16_attention_weight_max": 36.63990020751953,
      "activations/layer16_attention_weight_min": -32.89547348022461,
      "activations/layer17_attention_weight_max": 60.31719970703125,
      "activations/layer17_attention_weight_min": -47.949913024902344,
      "activations/layer18_attention_weight_max": 53.66427993774414,
      "activations/layer18_attention_weight_min": -40.720279693603516,
      "activations/layer19_attention_weight_max": 26.622629165649414,
      "activations/layer19_attention_weight_min": -22.7042293548584,
      "activations/layer1_attention_weight_max": 15.914800643920898,
      "activations/layer1_attention_weight_min": -13.025495529174805,
      "activations/layer20_attention_weight_max": 25.286880493164062,
      "activations/layer20_attention_weight_min": -19.03764533996582,
      "activations/layer21_attention_weight_max": 39.5037727355957,
      "activations/layer21_attention_weight_min": -20.876266479492188,
      "activations/layer22_attention_weight_max": 29.96821403503418,
      "activations/layer22_attention_weight_min": -24.30622100830078,
      "activations/layer23_attention_weight_max": 39.98450469970703,
      "activations/layer23_attention_weight_min": -25.25355339050293,
      "activations/layer2_attention_weight_max": 28.947662353515625,
      "activations/layer2_attention_weight_min": -28.885486602783203,
      "activations/layer3_attention_weight_max": 85.36083984375,
      "activations/layer3_attention_weight_min": -84.46110534667969,
      "activations/layer4_attention_weight_max": 80.78438568115234,
      "activations/layer4_attention_weight_min": -82.36901092529297,
      "activations/layer5_attention_weight_max": 62.30683898925781,
      "activations/layer5_attention_weight_min": -69.15437316894531,
      "activations/layer6_attention_weight_max": 51.21753692626953,
      "activations/layer6_attention_weight_min": -51.38481140136719,
      "activations/layer7_attention_weight_max": 69.388671875,
      "activations/layer7_attention_weight_min": -67.11864471435547,
      "activations/layer8_attention_weight_max": 49.151329040527344,
      "activations/layer8_attention_weight_min": -54.813255310058594,
      "activations/layer9_attention_weight_max": 45.97923278808594,
      "activations/layer9_attention_weight_min": -49.13545608520508,
      "epoch": 21.12,
      "learning_rate": 1.3954166666666666e-05,
      "loss": 2.7158,
      "step": 363450
    },
    {
      "activations/layer0_attention_weight_max": 15.853297233581543,
      "activations/layer0_attention_weight_min": -14.038609504699707,
      "activations/layer10_attention_weight_max": 32.95503616333008,
      "activations/layer10_attention_weight_min": -33.666385650634766,
      "activations/layer11_attention_weight_max": 30.9199275970459,
      "activations/layer11_attention_weight_min": -34.226409912109375,
      "activations/layer12_attention_weight_max": 18.70663833618164,
      "activations/layer12_attention_weight_min": -27.59307098388672,
      "activations/layer13_attention_weight_max": 35.880794525146484,
      "activations/layer13_attention_weight_min": -26.238832473754883,
      "activations/layer14_attention_weight_max": 35.21420669555664,
      "activations/layer14_attention_weight_min": -29.50254249572754,
      "activations/layer15_attention_weight_max": 33.17418670654297,
      "activations/layer15_attention_weight_min": -29.689945220947266,
      "activations/layer16_attention_weight_max": 34.7886962890625,
      "activations/layer16_attention_weight_min": -31.23338508605957,
      "activations/layer17_attention_weight_max": 48.53157424926758,
      "activations/layer17_attention_weight_min": -44.92891311645508,
      "activations/layer18_attention_weight_max": 46.25453186035156,
      "activations/layer18_attention_weight_min": -39.396060943603516,
      "activations/layer19_attention_weight_max": 26.748056411743164,
      "activations/layer19_attention_weight_min": -22.694766998291016,
      "activations/layer1_attention_weight_max": 14.94255542755127,
      "activations/layer1_attention_weight_min": -13.959131240844727,
      "activations/layer20_attention_weight_max": 26.3083438873291,
      "activations/layer20_attention_weight_min": -21.399009704589844,
      "activations/layer21_attention_weight_max": 38.727088928222656,
      "activations/layer21_attention_weight_min": -22.865442276000977,
      "activations/layer22_attention_weight_max": 27.698877334594727,
      "activations/layer22_attention_weight_min": -24.571918487548828,
      "activations/layer23_attention_weight_max": 38.788848876953125,
      "activations/layer23_attention_weight_min": -23.13036346435547,
      "activations/layer2_attention_weight_max": 30.730419158935547,
      "activations/layer2_attention_weight_min": -30.01865005493164,
      "activations/layer3_attention_weight_max": 84.06403350830078,
      "activations/layer3_attention_weight_min": -85.40895080566406,
      "activations/layer4_attention_weight_max": 80.697998046875,
      "activations/layer4_attention_weight_min": -81.69859313964844,
      "activations/layer5_attention_weight_max": 60.17018127441406,
      "activations/layer5_attention_weight_min": -70.56890869140625,
      "activations/layer6_attention_weight_max": 49.74485778808594,
      "activations/layer6_attention_weight_min": -49.788936614990234,
      "activations/layer7_attention_weight_max": 66.20020294189453,
      "activations/layer7_attention_weight_min": -64.27217864990234,
      "activations/layer8_attention_weight_max": 45.23685836791992,
      "activations/layer8_attention_weight_min": -49.77521896362305,
      "activations/layer9_attention_weight_max": 42.06780242919922,
      "activations/layer9_attention_weight_min": -45.3541259765625,
      "epoch": 21.12,
      "learning_rate": 1.393522727272727e-05,
      "loss": 2.7242,
      "step": 363500
    },
    {
      "activations/layer0_attention_weight_max": 15.841846466064453,
      "activations/layer0_attention_weight_min": -14.111692428588867,
      "activations/layer10_attention_weight_max": 34.1666145324707,
      "activations/layer10_attention_weight_min": -34.438873291015625,
      "activations/layer11_attention_weight_max": 33.36013412475586,
      "activations/layer11_attention_weight_min": -35.48053741455078,
      "activations/layer12_attention_weight_max": 19.792896270751953,
      "activations/layer12_attention_weight_min": -23.810216903686523,
      "activations/layer13_attention_weight_max": 37.426513671875,
      "activations/layer13_attention_weight_min": -28.874038696289062,
      "activations/layer14_attention_weight_max": 35.16301345825195,
      "activations/layer14_attention_weight_min": -30.61310386657715,
      "activations/layer15_attention_weight_max": 33.97393035888672,
      "activations/layer15_attention_weight_min": -32.56195068359375,
      "activations/layer16_attention_weight_max": 35.79065704345703,
      "activations/layer16_attention_weight_min": -34.11537551879883,
      "activations/layer17_attention_weight_max": 53.90599822998047,
      "activations/layer17_attention_weight_min": -47.86008834838867,
      "activations/layer18_attention_weight_max": 50.47878646850586,
      "activations/layer18_attention_weight_min": -43.54815673828125,
      "activations/layer19_attention_weight_max": 25.029767990112305,
      "activations/layer19_attention_weight_min": -27.648792266845703,
      "activations/layer1_attention_weight_max": 15.244359016418457,
      "activations/layer1_attention_weight_min": -13.776260375976562,
      "activations/layer20_attention_weight_max": 23.75020408630371,
      "activations/layer20_attention_weight_min": -22.880821228027344,
      "activations/layer21_attention_weight_max": 39.76678466796875,
      "activations/layer21_attention_weight_min": -25.626995086669922,
      "activations/layer22_attention_weight_max": 31.198566436767578,
      "activations/layer22_attention_weight_min": -28.22821044921875,
      "activations/layer23_attention_weight_max": 38.61594009399414,
      "activations/layer23_attention_weight_min": -22.433853149414062,
      "activations/layer2_attention_weight_max": 30.612043380737305,
      "activations/layer2_attention_weight_min": -29.529556274414062,
      "activations/layer3_attention_weight_max": 85.26213836669922,
      "activations/layer3_attention_weight_min": -85.40193176269531,
      "activations/layer4_attention_weight_max": 80.80296325683594,
      "activations/layer4_attention_weight_min": -79.5609359741211,
      "activations/layer5_attention_weight_max": 59.33789825439453,
      "activations/layer5_attention_weight_min": -71.01802825927734,
      "activations/layer6_attention_weight_max": 47.52429962158203,
      "activations/layer6_attention_weight_min": -47.415138244628906,
      "activations/layer7_attention_weight_max": 64.76383209228516,
      "activations/layer7_attention_weight_min": -62.93309020996094,
      "activations/layer8_attention_weight_max": 43.61977005004883,
      "activations/layer8_attention_weight_min": -46.449520111083984,
      "activations/layer9_attention_weight_max": 41.37777328491211,
      "activations/layer9_attention_weight_min": -43.207305908203125,
      "epoch": 21.12,
      "learning_rate": 1.3916287878787877e-05,
      "loss": 2.7136,
      "step": 363550
    },
    {
      "activations/layer0_attention_weight_max": 15.506216049194336,
      "activations/layer0_attention_weight_min": -14.368086814880371,
      "activations/layer10_attention_weight_max": 33.311588287353516,
      "activations/layer10_attention_weight_min": -33.62465286254883,
      "activations/layer11_attention_weight_max": 30.970008850097656,
      "activations/layer11_attention_weight_min": -33.654327392578125,
      "activations/layer12_attention_weight_max": 18.52583885192871,
      "activations/layer12_attention_weight_min": -23.69015884399414,
      "activations/layer13_attention_weight_max": 31.42779541015625,
      "activations/layer13_attention_weight_min": -27.716075897216797,
      "activations/layer14_attention_weight_max": 35.450435638427734,
      "activations/layer14_attention_weight_min": -28.7500057220459,
      "activations/layer15_attention_weight_max": 31.891883850097656,
      "activations/layer15_attention_weight_min": -29.234407424926758,
      "activations/layer16_attention_weight_max": 31.193740844726562,
      "activations/layer16_attention_weight_min": -31.73862648010254,
      "activations/layer17_attention_weight_max": 47.54253005981445,
      "activations/layer17_attention_weight_min": -44.98286056518555,
      "activations/layer18_attention_weight_max": 44.150733947753906,
      "activations/layer18_attention_weight_min": -40.868133544921875,
      "activations/layer19_attention_weight_max": 24.291101455688477,
      "activations/layer19_attention_weight_min": -23.56873893737793,
      "activations/layer1_attention_weight_max": 14.883413314819336,
      "activations/layer1_attention_weight_min": -13.920379638671875,
      "activations/layer20_attention_weight_max": 21.223112106323242,
      "activations/layer20_attention_weight_min": -23.137914657592773,
      "activations/layer21_attention_weight_max": 35.004913330078125,
      "activations/layer21_attention_weight_min": -22.44013023376465,
      "activations/layer22_attention_weight_max": 26.999713897705078,
      "activations/layer22_attention_weight_min": -25.68230628967285,
      "activations/layer23_attention_weight_max": 35.94794464111328,
      "activations/layer23_attention_weight_min": -24.927799224853516,
      "activations/layer2_attention_weight_max": 30.248626708984375,
      "activations/layer2_attention_weight_min": -31.84162712097168,
      "activations/layer3_attention_weight_max": 89.09764099121094,
      "activations/layer3_attention_weight_min": -89.72235107421875,
      "activations/layer4_attention_weight_max": 80.6993637084961,
      "activations/layer4_attention_weight_min": -81.47876739501953,
      "activations/layer5_attention_weight_max": 62.70299530029297,
      "activations/layer5_attention_weight_min": -67.23136901855469,
      "activations/layer6_attention_weight_max": 48.48387908935547,
      "activations/layer6_attention_weight_min": -48.74394989013672,
      "activations/layer7_attention_weight_max": 67.92662811279297,
      "activations/layer7_attention_weight_min": -62.9857292175293,
      "activations/layer8_attention_weight_max": 44.242191314697266,
      "activations/layer8_attention_weight_min": -48.72024154663086,
      "activations/layer9_attention_weight_max": 43.655967712402344,
      "activations/layer9_attention_weight_min": -46.394325256347656,
      "epoch": 21.13,
      "learning_rate": 1.3897348484848484e-05,
      "loss": 2.7231,
      "step": 363600
    },
    {
      "activations/layer0_attention_weight_max": 14.773575782775879,
      "activations/layer0_attention_weight_min": -14.216381072998047,
      "activations/layer10_attention_weight_max": 36.2965087890625,
      "activations/layer10_attention_weight_min": -35.64990234375,
      "activations/layer11_attention_weight_max": 33.65301513671875,
      "activations/layer11_attention_weight_min": -35.037540435791016,
      "activations/layer12_attention_weight_max": 18.729045867919922,
      "activations/layer12_attention_weight_min": -30.29753303527832,
      "activations/layer13_attention_weight_max": 33.626243591308594,
      "activations/layer13_attention_weight_min": -28.353862762451172,
      "activations/layer14_attention_weight_max": 35.794334411621094,
      "activations/layer14_attention_weight_min": -29.6475830078125,
      "activations/layer15_attention_weight_max": 32.36682891845703,
      "activations/layer15_attention_weight_min": -30.781293869018555,
      "activations/layer16_attention_weight_max": 32.3519401550293,
      "activations/layer16_attention_weight_min": -30.946870803833008,
      "activations/layer17_attention_weight_max": 51.91464614868164,
      "activations/layer17_attention_weight_min": -44.755672454833984,
      "activations/layer18_attention_weight_max": 46.80064010620117,
      "activations/layer18_attention_weight_min": -40.973541259765625,
      "activations/layer19_attention_weight_max": 25.411062240600586,
      "activations/layer19_attention_weight_min": -22.175344467163086,
      "activations/layer1_attention_weight_max": 14.692037582397461,
      "activations/layer1_attention_weight_min": -12.847044944763184,
      "activations/layer20_attention_weight_max": 24.055147171020508,
      "activations/layer20_attention_weight_min": -22.114534378051758,
      "activations/layer21_attention_weight_max": 35.18217849731445,
      "activations/layer21_attention_weight_min": -22.574432373046875,
      "activations/layer22_attention_weight_max": 28.43872833251953,
      "activations/layer22_attention_weight_min": -25.876136779785156,
      "activations/layer23_attention_weight_max": 37.617095947265625,
      "activations/layer23_attention_weight_min": -22.981794357299805,
      "activations/layer2_attention_weight_max": 30.504865646362305,
      "activations/layer2_attention_weight_min": -29.146862030029297,
      "activations/layer3_attention_weight_max": 89.66724395751953,
      "activations/layer3_attention_weight_min": -88.20085144042969,
      "activations/layer4_attention_weight_max": 85.80696868896484,
      "activations/layer4_attention_weight_min": -88.27325439453125,
      "activations/layer5_attention_weight_max": 64.48062133789062,
      "activations/layer5_attention_weight_min": -72.64350891113281,
      "activations/layer6_attention_weight_max": 50.41110610961914,
      "activations/layer6_attention_weight_min": -52.17942810058594,
      "activations/layer7_attention_weight_max": 67.43207550048828,
      "activations/layer7_attention_weight_min": -67.6960678100586,
      "activations/layer8_attention_weight_max": 48.03823471069336,
      "activations/layer8_attention_weight_min": -51.522239685058594,
      "activations/layer9_attention_weight_max": 46.29783248901367,
      "activations/layer9_attention_weight_min": -45.759498596191406,
      "epoch": 21.13,
      "learning_rate": 1.3878787878787878e-05,
      "loss": 2.7246,
      "step": 363650
    },
    {
      "activations/layer0_attention_weight_max": 16.205860137939453,
      "activations/layer0_attention_weight_min": -14.536508560180664,
      "activations/layer10_attention_weight_max": 35.783668518066406,
      "activations/layer10_attention_weight_min": -34.839637756347656,
      "activations/layer11_attention_weight_max": 31.98566436767578,
      "activations/layer11_attention_weight_min": -35.624969482421875,
      "activations/layer12_attention_weight_max": 19.777456283569336,
      "activations/layer12_attention_weight_min": -25.259916305541992,
      "activations/layer13_attention_weight_max": 40.813812255859375,
      "activations/layer13_attention_weight_min": -26.960386276245117,
      "activations/layer14_attention_weight_max": 38.64609146118164,
      "activations/layer14_attention_weight_min": -30.464994430541992,
      "activations/layer15_attention_weight_max": 37.57859420776367,
      "activations/layer15_attention_weight_min": -30.27354621887207,
      "activations/layer16_attention_weight_max": 37.29913330078125,
      "activations/layer16_attention_weight_min": -34.98806381225586,
      "activations/layer17_attention_weight_max": 54.69529342651367,
      "activations/layer17_attention_weight_min": -50.74225997924805,
      "activations/layer18_attention_weight_max": 53.06223678588867,
      "activations/layer18_attention_weight_min": -44.473697662353516,
      "activations/layer19_attention_weight_max": 27.321836471557617,
      "activations/layer19_attention_weight_min": -23.322019577026367,
      "activations/layer1_attention_weight_max": 16.17928123474121,
      "activations/layer1_attention_weight_min": -13.539929389953613,
      "activations/layer20_attention_weight_max": 24.270299911499023,
      "activations/layer20_attention_weight_min": -21.7182674407959,
      "activations/layer21_attention_weight_max": 46.66703414916992,
      "activations/layer21_attention_weight_min": -21.889551162719727,
      "activations/layer22_attention_weight_max": 31.612621307373047,
      "activations/layer22_attention_weight_min": -24.151880264282227,
      "activations/layer23_attention_weight_max": 40.738887786865234,
      "activations/layer23_attention_weight_min": -23.59842300415039,
      "activations/layer2_attention_weight_max": 31.703285217285156,
      "activations/layer2_attention_weight_min": -30.350109100341797,
      "activations/layer3_attention_weight_max": 87.03577423095703,
      "activations/layer3_attention_weight_min": -86.6918716430664,
      "activations/layer4_attention_weight_max": 79.67510223388672,
      "activations/layer4_attention_weight_min": -79.53888702392578,
      "activations/layer5_attention_weight_max": 58.20243453979492,
      "activations/layer5_attention_weight_min": -69.53572082519531,
      "activations/layer6_attention_weight_max": 48.9810676574707,
      "activations/layer6_attention_weight_min": -50.1097526550293,
      "activations/layer7_attention_weight_max": 64.33153533935547,
      "activations/layer7_attention_weight_min": -65.87823486328125,
      "activations/layer8_attention_weight_max": 45.41927719116211,
      "activations/layer8_attention_weight_min": -50.788143157958984,
      "activations/layer9_attention_weight_max": 44.06800079345703,
      "activations/layer9_attention_weight_min": -45.382225036621094,
      "epoch": 21.13,
      "learning_rate": 1.3859848484848483e-05,
      "loss": 2.7357,
      "step": 363700
    },
    {
      "activations/layer0_attention_weight_max": 14.772842407226562,
      "activations/layer0_attention_weight_min": -14.364302635192871,
      "activations/layer10_attention_weight_max": 35.69860076904297,
      "activations/layer10_attention_weight_min": -35.187049865722656,
      "activations/layer11_attention_weight_max": 31.343406677246094,
      "activations/layer11_attention_weight_min": -34.24603271484375,
      "activations/layer12_attention_weight_max": 18.43399429321289,
      "activations/layer12_attention_weight_min": -29.12291145324707,
      "activations/layer13_attention_weight_max": 31.253307342529297,
      "activations/layer13_attention_weight_min": -26.614850997924805,
      "activations/layer14_attention_weight_max": 32.611480712890625,
      "activations/layer14_attention_weight_min": -28.113611221313477,
      "activations/layer15_attention_weight_max": 33.15625,
      "activations/layer15_attention_weight_min": -29.326330184936523,
      "activations/layer16_attention_weight_max": 33.83625030517578,
      "activations/layer16_attention_weight_min": -31.93869972229004,
      "activations/layer17_attention_weight_max": 50.576148986816406,
      "activations/layer17_attention_weight_min": -44.8281135559082,
      "activations/layer18_attention_weight_max": 44.645751953125,
      "activations/layer18_attention_weight_min": -39.985130310058594,
      "activations/layer19_attention_weight_max": 24.436866760253906,
      "activations/layer19_attention_weight_min": -22.385196685791016,
      "activations/layer1_attention_weight_max": 15.014948844909668,
      "activations/layer1_attention_weight_min": -13.570514678955078,
      "activations/layer20_attention_weight_max": 20.356536865234375,
      "activations/layer20_attention_weight_min": -23.445646286010742,
      "activations/layer21_attention_weight_max": 35.17790603637695,
      "activations/layer21_attention_weight_min": -23.28177833557129,
      "activations/layer22_attention_weight_max": 28.809886932373047,
      "activations/layer22_attention_weight_min": -24.571138381958008,
      "activations/layer23_attention_weight_max": 35.701717376708984,
      "activations/layer23_attention_weight_min": -22.663562774658203,
      "activations/layer2_attention_weight_max": 32.12982940673828,
      "activations/layer2_attention_weight_min": -30.00394058227539,
      "activations/layer3_attention_weight_max": 88.92431640625,
      "activations/layer3_attention_weight_min": -88.68919372558594,
      "activations/layer4_attention_weight_max": 84.3947982788086,
      "activations/layer4_attention_weight_min": -84.70498657226562,
      "activations/layer5_attention_weight_max": 61.93467330932617,
      "activations/layer5_attention_weight_min": -75.52912902832031,
      "activations/layer6_attention_weight_max": 48.95114517211914,
      "activations/layer6_attention_weight_min": -52.214012145996094,
      "activations/layer7_attention_weight_max": 63.81113052368164,
      "activations/layer7_attention_weight_min": -67.36186218261719,
      "activations/layer8_attention_weight_max": 50.944068908691406,
      "activations/layer8_attention_weight_min": -53.90198516845703,
      "activations/layer9_attention_weight_max": 42.78237533569336,
      "activations/layer9_attention_weight_min": -46.512046813964844,
      "epoch": 21.14,
      "learning_rate": 1.3841287878787877e-05,
      "loss": 2.7148,
      "step": 363750
    },
    {
      "activations/layer0_attention_weight_max": 16.419187545776367,
      "activations/layer0_attention_weight_min": -14.458279609680176,
      "activations/layer10_attention_weight_max": 34.325103759765625,
      "activations/layer10_attention_weight_min": -34.88044738769531,
      "activations/layer11_attention_weight_max": 31.591773986816406,
      "activations/layer11_attention_weight_min": -33.91728973388672,
      "activations/layer12_attention_weight_max": 19.646451950073242,
      "activations/layer12_attention_weight_min": -24.45854949951172,
      "activations/layer13_attention_weight_max": 35.22419357299805,
      "activations/layer13_attention_weight_min": -27.76416015625,
      "activations/layer14_attention_weight_max": 35.47913360595703,
      "activations/layer14_attention_weight_min": -30.982362747192383,
      "activations/layer15_attention_weight_max": 34.638912200927734,
      "activations/layer15_attention_weight_min": -31.060997009277344,
      "activations/layer16_attention_weight_max": 34.75750732421875,
      "activations/layer16_attention_weight_min": -34.085296630859375,
      "activations/layer17_attention_weight_max": 51.3235969543457,
      "activations/layer17_attention_weight_min": -45.8343620300293,
      "activations/layer18_attention_weight_max": 47.55732345581055,
      "activations/layer18_attention_weight_min": -42.15545654296875,
      "activations/layer19_attention_weight_max": 27.291330337524414,
      "activations/layer19_attention_weight_min": -23.79347038269043,
      "activations/layer1_attention_weight_max": 14.927315711975098,
      "activations/layer1_attention_weight_min": -13.754483222961426,
      "activations/layer20_attention_weight_max": 23.538211822509766,
      "activations/layer20_attention_weight_min": -22.16394805908203,
      "activations/layer21_attention_weight_max": 38.506553649902344,
      "activations/layer21_attention_weight_min": -22.79718780517578,
      "activations/layer22_attention_weight_max": 29.51113510131836,
      "activations/layer22_attention_weight_min": -25.616777420043945,
      "activations/layer23_attention_weight_max": 34.41712951660156,
      "activations/layer23_attention_weight_min": -24.850637435913086,
      "activations/layer2_attention_weight_max": 31.886489868164062,
      "activations/layer2_attention_weight_min": -30.846778869628906,
      "activations/layer3_attention_weight_max": 91.67826080322266,
      "activations/layer3_attention_weight_min": -89.05066680908203,
      "activations/layer4_attention_weight_max": 84.54866790771484,
      "activations/layer4_attention_weight_min": -83.48656463623047,
      "activations/layer5_attention_weight_max": 62.82630157470703,
      "activations/layer5_attention_weight_min": -69.49848937988281,
      "activations/layer6_attention_weight_max": 50.85200500488281,
      "activations/layer6_attention_weight_min": -48.95149612426758,
      "activations/layer7_attention_weight_max": 65.74278259277344,
      "activations/layer7_attention_weight_min": -63.53297805786133,
      "activations/layer8_attention_weight_max": 47.54319381713867,
      "activations/layer8_attention_weight_min": -49.00466537475586,
      "activations/layer9_attention_weight_max": 43.902671813964844,
      "activations/layer9_attention_weight_min": -43.76321029663086,
      "epoch": 21.14,
      "learning_rate": 1.3822348484848484e-05,
      "loss": 2.7209,
      "step": 363800
    },
    {
      "activations/layer0_attention_weight_max": 15.648198127746582,
      "activations/layer0_attention_weight_min": -14.555764198303223,
      "activations/layer10_attention_weight_max": 34.34306335449219,
      "activations/layer10_attention_weight_min": -35.50334548950195,
      "activations/layer11_attention_weight_max": 32.15174102783203,
      "activations/layer11_attention_weight_min": -35.29421615600586,
      "activations/layer12_attention_weight_max": 18.43007469177246,
      "activations/layer12_attention_weight_min": -25.404821395874023,
      "activations/layer13_attention_weight_max": 30.245960235595703,
      "activations/layer13_attention_weight_min": -27.752756118774414,
      "activations/layer14_attention_weight_max": 33.57212829589844,
      "activations/layer14_attention_weight_min": -31.88839340209961,
      "activations/layer15_attention_weight_max": 29.81049346923828,
      "activations/layer15_attention_weight_min": -31.06499481201172,
      "activations/layer16_attention_weight_max": 33.555538177490234,
      "activations/layer16_attention_weight_min": -31.669189453125,
      "activations/layer17_attention_weight_max": 47.60515213012695,
      "activations/layer17_attention_weight_min": -43.752647399902344,
      "activations/layer18_attention_weight_max": 44.28679275512695,
      "activations/layer18_attention_weight_min": -36.49183654785156,
      "activations/layer19_attention_weight_max": 23.278026580810547,
      "activations/layer19_attention_weight_min": -22.548206329345703,
      "activations/layer1_attention_weight_max": 15.342743873596191,
      "activations/layer1_attention_weight_min": -12.878664016723633,
      "activations/layer20_attention_weight_max": 22.692731857299805,
      "activations/layer20_attention_weight_min": -22.770719528198242,
      "activations/layer21_attention_weight_max": 34.612815856933594,
      "activations/layer21_attention_weight_min": -22.13121795654297,
      "activations/layer22_attention_weight_max": 27.18753433227539,
      "activations/layer22_attention_weight_min": -23.41278839111328,
      "activations/layer23_attention_weight_max": 30.516080856323242,
      "activations/layer23_attention_weight_min": -20.98495864868164,
      "activations/layer2_attention_weight_max": 32.18383026123047,
      "activations/layer2_attention_weight_min": -32.00169372558594,
      "activations/layer3_attention_weight_max": 92.79304504394531,
      "activations/layer3_attention_weight_min": -91.06542205810547,
      "activations/layer4_attention_weight_max": 81.1242446899414,
      "activations/layer4_attention_weight_min": -81.95860290527344,
      "activations/layer5_attention_weight_max": 61.083343505859375,
      "activations/layer5_attention_weight_min": -72.25277709960938,
      "activations/layer6_attention_weight_max": 47.76505661010742,
      "activations/layer6_attention_weight_min": -48.286964416503906,
      "activations/layer7_attention_weight_max": 63.931358337402344,
      "activations/layer7_attention_weight_min": -65.6463394165039,
      "activations/layer8_attention_weight_max": 45.57035446166992,
      "activations/layer8_attention_weight_min": -48.90926742553711,
      "activations/layer9_attention_weight_max": 45.78269577026367,
      "activations/layer9_attention_weight_min": -45.69814682006836,
      "epoch": 21.14,
      "learning_rate": 1.3803409090909089e-05,
      "loss": 2.7085,
      "step": 363850
    },
    {
      "activations/layer0_attention_weight_max": 15.602387428283691,
      "activations/layer0_attention_weight_min": -14.932071685791016,
      "activations/layer10_attention_weight_max": 35.496246337890625,
      "activations/layer10_attention_weight_min": -34.745609283447266,
      "activations/layer11_attention_weight_max": 33.17178726196289,
      "activations/layer11_attention_weight_min": -33.968353271484375,
      "activations/layer12_attention_weight_max": 20.568710327148438,
      "activations/layer12_attention_weight_min": -23.956390380859375,
      "activations/layer13_attention_weight_max": 33.642269134521484,
      "activations/layer13_attention_weight_min": -28.16103172302246,
      "activations/layer14_attention_weight_max": 37.54402542114258,
      "activations/layer14_attention_weight_min": -30.127653121948242,
      "activations/layer15_attention_weight_max": 32.59654235839844,
      "activations/layer15_attention_weight_min": -30.19919776916504,
      "activations/layer16_attention_weight_max": 37.235286712646484,
      "activations/layer16_attention_weight_min": -33.53757095336914,
      "activations/layer17_attention_weight_max": 55.00346374511719,
      "activations/layer17_attention_weight_min": -46.9009895324707,
      "activations/layer18_attention_weight_max": 49.47439956665039,
      "activations/layer18_attention_weight_min": -40.339599609375,
      "activations/layer19_attention_weight_max": 24.912504196166992,
      "activations/layer19_attention_weight_min": -23.24786376953125,
      "activations/layer1_attention_weight_max": 15.339912414550781,
      "activations/layer1_attention_weight_min": -14.598767280578613,
      "activations/layer20_attention_weight_max": 23.04448699951172,
      "activations/layer20_attention_weight_min": -21.826780319213867,
      "activations/layer21_attention_weight_max": 36.628517150878906,
      "activations/layer21_attention_weight_min": -23.10111427307129,
      "activations/layer22_attention_weight_max": 26.89341926574707,
      "activations/layer22_attention_weight_min": -26.91476821899414,
      "activations/layer23_attention_weight_max": 34.156795501708984,
      "activations/layer23_attention_weight_min": -24.0347957611084,
      "activations/layer2_attention_weight_max": 31.632781982421875,
      "activations/layer2_attention_weight_min": -30.365798950195312,
      "activations/layer3_attention_weight_max": 87.85549926757812,
      "activations/layer3_attention_weight_min": -91.74163055419922,
      "activations/layer4_attention_weight_max": 81.4741439819336,
      "activations/layer4_attention_weight_min": -79.4944839477539,
      "activations/layer5_attention_weight_max": 60.779693603515625,
      "activations/layer5_attention_weight_min": -70.21355438232422,
      "activations/layer6_attention_weight_max": 49.45539093017578,
      "activations/layer6_attention_weight_min": -51.73618698120117,
      "activations/layer7_attention_weight_max": 66.36878967285156,
      "activations/layer7_attention_weight_min": -68.94242858886719,
      "activations/layer8_attention_weight_max": 46.76649856567383,
      "activations/layer8_attention_weight_min": -49.80583190917969,
      "activations/layer9_attention_weight_max": 45.61079406738281,
      "activations/layer9_attention_weight_min": -44.58441162109375,
      "epoch": 21.14,
      "learning_rate": 1.3784469696969695e-05,
      "loss": 2.71,
      "step": 363900
    },
    {
      "activations/layer0_attention_weight_max": 16.036096572875977,
      "activations/layer0_attention_weight_min": -14.376419067382812,
      "activations/layer10_attention_weight_max": 33.539833068847656,
      "activations/layer10_attention_weight_min": -37.151092529296875,
      "activations/layer11_attention_weight_max": 31.58513641357422,
      "activations/layer11_attention_weight_min": -36.053627014160156,
      "activations/layer12_attention_weight_max": 18.644865036010742,
      "activations/layer12_attention_weight_min": -24.475976943969727,
      "activations/layer13_attention_weight_max": 33.0557975769043,
      "activations/layer13_attention_weight_min": -27.972871780395508,
      "activations/layer14_attention_weight_max": 33.53696823120117,
      "activations/layer14_attention_weight_min": -31.30734634399414,
      "activations/layer15_attention_weight_max": 34.584678649902344,
      "activations/layer15_attention_weight_min": -31.954387664794922,
      "activations/layer16_attention_weight_max": 33.566871643066406,
      "activations/layer16_attention_weight_min": -33.70392990112305,
      "activations/layer17_attention_weight_max": 49.7799186706543,
      "activations/layer17_attention_weight_min": -50.48103713989258,
      "activations/layer18_attention_weight_max": 46.37411880493164,
      "activations/layer18_attention_weight_min": -46.32872772216797,
      "activations/layer19_attention_weight_max": 25.308765411376953,
      "activations/layer19_attention_weight_min": -27.287099838256836,
      "activations/layer1_attention_weight_max": 17.55068016052246,
      "activations/layer1_attention_weight_min": -14.605119705200195,
      "activations/layer20_attention_weight_max": 23.372257232666016,
      "activations/layer20_attention_weight_min": -22.070457458496094,
      "activations/layer21_attention_weight_max": 36.03911209106445,
      "activations/layer21_attention_weight_min": -25.905399322509766,
      "activations/layer22_attention_weight_max": 31.99752426147461,
      "activations/layer22_attention_weight_min": -26.479692459106445,
      "activations/layer23_attention_weight_max": 37.09538269042969,
      "activations/layer23_attention_weight_min": -24.46356773376465,
      "activations/layer2_attention_weight_max": 30.59015464782715,
      "activations/layer2_attention_weight_min": -29.260955810546875,
      "activations/layer3_attention_weight_max": 88.77665710449219,
      "activations/layer3_attention_weight_min": -84.72987365722656,
      "activations/layer4_attention_weight_max": 82.52008056640625,
      "activations/layer4_attention_weight_min": -80.16928100585938,
      "activations/layer5_attention_weight_max": 63.11780548095703,
      "activations/layer5_attention_weight_min": -69.88539123535156,
      "activations/layer6_attention_weight_max": 48.80464172363281,
      "activations/layer6_attention_weight_min": -51.1403923034668,
      "activations/layer7_attention_weight_max": 70.17899322509766,
      "activations/layer7_attention_weight_min": -67.13060760498047,
      "activations/layer8_attention_weight_max": 42.91447067260742,
      "activations/layer8_attention_weight_min": -47.699161529541016,
      "activations/layer9_attention_weight_max": 42.72726821899414,
      "activations/layer9_attention_weight_min": -45.789363861083984,
      "epoch": 21.15,
      "learning_rate": 1.3765530303030302e-05,
      "loss": 2.7063,
      "step": 363950
    },
    {
      "activations/layer0_attention_weight_max": 15.202068328857422,
      "activations/layer0_attention_weight_min": -14.501992225646973,
      "activations/layer10_attention_weight_max": 36.809669494628906,
      "activations/layer10_attention_weight_min": -36.49150466918945,
      "activations/layer11_attention_weight_max": 34.50432586669922,
      "activations/layer11_attention_weight_min": -36.091217041015625,
      "activations/layer12_attention_weight_max": 20.4838924407959,
      "activations/layer12_attention_weight_min": -23.04222297668457,
      "activations/layer13_attention_weight_max": 34.70557403564453,
      "activations/layer13_attention_weight_min": -27.47881507873535,
      "activations/layer14_attention_weight_max": 36.19023132324219,
      "activations/layer14_attention_weight_min": -31.222688674926758,
      "activations/layer15_attention_weight_max": 32.3707160949707,
      "activations/layer15_attention_weight_min": -31.7748966217041,
      "activations/layer16_attention_weight_max": 35.7504997253418,
      "activations/layer16_attention_weight_min": -33.936092376708984,
      "activations/layer17_attention_weight_max": 50.048641204833984,
      "activations/layer17_attention_weight_min": -47.097572326660156,
      "activations/layer18_attention_weight_max": 46.72430419921875,
      "activations/layer18_attention_weight_min": -40.57243728637695,
      "activations/layer19_attention_weight_max": 25.14027214050293,
      "activations/layer19_attention_weight_min": -23.315523147583008,
      "activations/layer1_attention_weight_max": 14.4931001663208,
      "activations/layer1_attention_weight_min": -14.254558563232422,
      "activations/layer20_attention_weight_max": 24.762083053588867,
      "activations/layer20_attention_weight_min": -22.04826545715332,
      "activations/layer21_attention_weight_max": 36.91545867919922,
      "activations/layer21_attention_weight_min": -23.362268447875977,
      "activations/layer22_attention_weight_max": 30.98151397705078,
      "activations/layer22_attention_weight_min": -24.50228500366211,
      "activations/layer23_attention_weight_max": 33.333702087402344,
      "activations/layer23_attention_weight_min": -25.138586044311523,
      "activations/layer2_attention_weight_max": 30.82257652282715,
      "activations/layer2_attention_weight_min": -30.693317413330078,
      "activations/layer3_attention_weight_max": 85.09950256347656,
      "activations/layer3_attention_weight_min": -88.40834045410156,
      "activations/layer4_attention_weight_max": 79.6658935546875,
      "activations/layer4_attention_weight_min": -82.81626892089844,
      "activations/layer5_attention_weight_max": 59.095054626464844,
      "activations/layer5_attention_weight_min": -78.2535171508789,
      "activations/layer6_attention_weight_max": 50.306453704833984,
      "activations/layer6_attention_weight_min": -49.336158752441406,
      "activations/layer7_attention_weight_max": 63.209312438964844,
      "activations/layer7_attention_weight_min": -66.92829895019531,
      "activations/layer8_attention_weight_max": 44.51750946044922,
      "activations/layer8_attention_weight_min": -48.43154525756836,
      "activations/layer9_attention_weight_max": 42.364341735839844,
      "activations/layer9_attention_weight_min": -47.933162689208984,
      "epoch": 21.15,
      "learning_rate": 1.3746590909090907e-05,
      "loss": 2.7179,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.2253,
      "eval_samples_per_second": 522.046,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.2253,
      "eval_openwebtext_samples_per_second": 522.046,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9482,
      "eval_wikitext_samples_per_second": 234.059,
      "step": 364000
    },
    {
      "epoch": 21.15,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.5572,
      "eval_lambada_samples_per_second": 509.461,
      "step": 364000
    },
    {
      "activations/layer0_attention_weight_max": 15.33302116394043,
      "activations/layer0_attention_weight_min": -14.529142379760742,
      "activations/layer10_attention_weight_max": 35.3333625793457,
      "activations/layer10_attention_weight_min": -35.39063262939453,
      "activations/layer11_attention_weight_max": 35.13886260986328,
      "activations/layer11_attention_weight_min": -35.72792434692383,
      "activations/layer12_attention_weight_max": 18.910171508789062,
      "activations/layer12_attention_weight_min": -25.03386878967285,
      "activations/layer13_attention_weight_max": 37.465755462646484,
      "activations/layer13_attention_weight_min": -28.447208404541016,
      "activations/layer14_attention_weight_max": 37.02368927001953,
      "activations/layer14_attention_weight_min": -32.69724655151367,
      "activations/layer15_attention_weight_max": 35.42753219604492,
      "activations/layer15_attention_weight_min": -31.26667594909668,
      "activations/layer16_attention_weight_max": 35.15125274658203,
      "activations/layer16_attention_weight_min": -32.22077178955078,
      "activations/layer17_attention_weight_max": 53.90617752075195,
      "activations/layer17_attention_weight_min": -44.43840789794922,
      "activations/layer18_attention_weight_max": 50.84825897216797,
      "activations/layer18_attention_weight_min": -40.92689514160156,
      "activations/layer19_attention_weight_max": 24.812450408935547,
      "activations/layer19_attention_weight_min": -20.636816024780273,
      "activations/layer1_attention_weight_max": 15.4295072555542,
      "activations/layer1_attention_weight_min": -13.857781410217285,
      "activations/layer20_attention_weight_max": 24.056745529174805,
      "activations/layer20_attention_weight_min": -18.77483367919922,
      "activations/layer21_attention_weight_max": 40.70098876953125,
      "activations/layer21_attention_weight_min": -23.17080307006836,
      "activations/layer22_attention_weight_max": 30.079877853393555,
      "activations/layer22_attention_weight_min": -24.228113174438477,
      "activations/layer23_attention_weight_max": 39.93943786621094,
      "activations/layer23_attention_weight_min": -22.68633270263672,
      "activations/layer2_attention_weight_max": 31.170555114746094,
      "activations/layer2_attention_weight_min": -30.9348201751709,
      "activations/layer3_attention_weight_max": 88.37220764160156,
      "activations/layer3_attention_weight_min": -89.9404525756836,
      "activations/layer4_attention_weight_max": 81.9803695678711,
      "activations/layer4_attention_weight_min": -82.50206756591797,
      "activations/layer5_attention_weight_max": 64.80516052246094,
      "activations/layer5_attention_weight_min": -71.62911987304688,
      "activations/layer6_attention_weight_max": 49.63975524902344,
      "activations/layer6_attention_weight_min": -48.35991668701172,
      "activations/layer7_attention_weight_max": 68.24583435058594,
      "activations/layer7_attention_weight_min": -67.30860137939453,
      "activations/layer8_attention_weight_max": 45.611106872558594,
      "activations/layer8_attention_weight_min": -49.046146392822266,
      "activations/layer9_attention_weight_max": 44.924686431884766,
      "activations/layer9_attention_weight_min": -44.815399169921875,
      "epoch": 21.15,
      "learning_rate": 1.3727651515151513e-05,
      "loss": 2.7169,
      "step": 364050
    },
    {
      "activations/layer0_attention_weight_max": 15.431806564331055,
      "activations/layer0_attention_weight_min": -14.419194221496582,
      "activations/layer10_attention_weight_max": 36.409912109375,
      "activations/layer10_attention_weight_min": -38.80109405517578,
      "activations/layer11_attention_weight_max": 33.1423225402832,
      "activations/layer11_attention_weight_min": -35.429725646972656,
      "activations/layer12_attention_weight_max": 18.1193790435791,
      "activations/layer12_attention_weight_min": -24.029809951782227,
      "activations/layer13_attention_weight_max": 35.69100570678711,
      "activations/layer13_attention_weight_min": -29.91778564453125,
      "activations/layer14_attention_weight_max": 37.98650360107422,
      "activations/layer14_attention_weight_min": -31.52166175842285,
      "activations/layer15_attention_weight_max": 34.14282989501953,
      "activations/layer15_attention_weight_min": -33.37350082397461,
      "activations/layer16_attention_weight_max": 34.577980041503906,
      "activations/layer16_attention_weight_min": -34.09621810913086,
      "activations/layer17_attention_weight_max": 52.68624496459961,
      "activations/layer17_attention_weight_min": -46.4309196472168,
      "activations/layer18_attention_weight_max": 50.16364288330078,
      "activations/layer18_attention_weight_min": -40.39655303955078,
      "activations/layer19_attention_weight_max": 25.624588012695312,
      "activations/layer19_attention_weight_min": -22.643714904785156,
      "activations/layer1_attention_weight_max": 14.712976455688477,
      "activations/layer1_attention_weight_min": -14.070591926574707,
      "activations/layer20_attention_weight_max": 21.5709285736084,
      "activations/layer20_attention_weight_min": -21.75006866455078,
      "activations/layer21_attention_weight_max": 37.22157669067383,
      "activations/layer21_attention_weight_min": -24.443559646606445,
      "activations/layer22_attention_weight_max": 26.115015029907227,
      "activations/layer22_attention_weight_min": -26.775163650512695,
      "activations/layer23_attention_weight_max": 33.52233123779297,
      "activations/layer23_attention_weight_min": -23.543933868408203,
      "activations/layer2_attention_weight_max": 31.393268585205078,
      "activations/layer2_attention_weight_min": -29.972715377807617,
      "activations/layer3_attention_weight_max": 88.8473892211914,
      "activations/layer3_attention_weight_min": -91.5460433959961,
      "activations/layer4_attention_weight_max": 81.85979461669922,
      "activations/layer4_attention_weight_min": -82.82149505615234,
      "activations/layer5_attention_weight_max": 63.68006134033203,
      "activations/layer5_attention_weight_min": -73.42575073242188,
      "activations/layer6_attention_weight_max": 49.30253601074219,
      "activations/layer6_attention_weight_min": -52.327903747558594,
      "activations/layer7_attention_weight_max": 62.09806823730469,
      "activations/layer7_attention_weight_min": -70.97349548339844,
      "activations/layer8_attention_weight_max": 48.10018539428711,
      "activations/layer8_attention_weight_min": -54.11471939086914,
      "activations/layer9_attention_weight_max": 43.151649475097656,
      "activations/layer9_attention_weight_min": -46.88065719604492,
      "epoch": 21.16,
      "learning_rate": 1.3708712121212118e-05,
      "loss": 2.7234,
      "step": 364100
    },
    {
      "activations/layer0_attention_weight_max": 15.877668380737305,
      "activations/layer0_attention_weight_min": -15.104795455932617,
      "activations/layer10_attention_weight_max": 35.33441162109375,
      "activations/layer10_attention_weight_min": -35.089290618896484,
      "activations/layer11_attention_weight_max": 33.55934143066406,
      "activations/layer11_attention_weight_min": -34.560882568359375,
      "activations/layer12_attention_weight_max": 18.2230167388916,
      "activations/layer12_attention_weight_min": -28.38775062561035,
      "activations/layer13_attention_weight_max": 33.20648956298828,
      "activations/layer13_attention_weight_min": -27.364133834838867,
      "activations/layer14_attention_weight_max": 34.94194793701172,
      "activations/layer14_attention_weight_min": -30.27960968017578,
      "activations/layer15_attention_weight_max": 33.71711349487305,
      "activations/layer15_attention_weight_min": -29.492109298706055,
      "activations/layer16_attention_weight_max": 31.369287490844727,
      "activations/layer16_attention_weight_min": -33.691585540771484,
      "activations/layer17_attention_weight_max": 47.967254638671875,
      "activations/layer17_attention_weight_min": -43.86094284057617,
      "activations/layer18_attention_weight_max": 40.83133316040039,
      "activations/layer18_attention_weight_min": -38.33130645751953,
      "activations/layer19_attention_weight_max": 22.619537353515625,
      "activations/layer19_attention_weight_min": -21.701677322387695,
      "activations/layer1_attention_weight_max": 15.140899658203125,
      "activations/layer1_attention_weight_min": -14.299835205078125,
      "activations/layer20_attention_weight_max": 22.373186111450195,
      "activations/layer20_attention_weight_min": -22.182842254638672,
      "activations/layer21_attention_weight_max": 32.775508880615234,
      "activations/layer21_attention_weight_min": -21.301651000976562,
      "activations/layer22_attention_weight_max": 28.179834365844727,
      "activations/layer22_attention_weight_min": -23.7414608001709,
      "activations/layer23_attention_weight_max": 36.650352478027344,
      "activations/layer23_attention_weight_min": -24.337299346923828,
      "activations/layer2_attention_weight_max": 31.050003051757812,
      "activations/layer2_attention_weight_min": -30.987266540527344,
      "activations/layer3_attention_weight_max": 86.62281799316406,
      "activations/layer3_attention_weight_min": -88.54092407226562,
      "activations/layer4_attention_weight_max": 81.50201416015625,
      "activations/layer4_attention_weight_min": -81.73726654052734,
      "activations/layer5_attention_weight_max": 64.1589584350586,
      "activations/layer5_attention_weight_min": -70.73420715332031,
      "activations/layer6_attention_weight_max": 47.98594284057617,
      "activations/layer6_attention_weight_min": -50.49066925048828,
      "activations/layer7_attention_weight_max": 66.93611907958984,
      "activations/layer7_attention_weight_min": -70.93923950195312,
      "activations/layer8_attention_weight_max": 45.59823989868164,
      "activations/layer8_attention_weight_min": -50.187095642089844,
      "activations/layer9_attention_weight_max": 42.3931999206543,
      "activations/layer9_attention_weight_min": -46.382320404052734,
      "epoch": 21.16,
      "learning_rate": 1.3689772727272726e-05,
      "loss": 2.706,
      "step": 364150
    },
    {
      "activations/layer0_attention_weight_max": 15.769268989562988,
      "activations/layer0_attention_weight_min": -14.770475387573242,
      "activations/layer10_attention_weight_max": 37.594879150390625,
      "activations/layer10_attention_weight_min": -38.71619415283203,
      "activations/layer11_attention_weight_max": 35.06270980834961,
      "activations/layer11_attention_weight_min": -37.64112854003906,
      "activations/layer12_attention_weight_max": 19.966976165771484,
      "activations/layer12_attention_weight_min": -37.43684768676758,
      "activations/layer13_attention_weight_max": 37.46022415161133,
      "activations/layer13_attention_weight_min": -30.727733612060547,
      "activations/layer14_attention_weight_max": 35.5673942565918,
      "activations/layer14_attention_weight_min": -29.68832778930664,
      "activations/layer15_attention_weight_max": 34.080665588378906,
      "activations/layer15_attention_weight_min": -30.035261154174805,
      "activations/layer16_attention_weight_max": 34.29652404785156,
      "activations/layer16_attention_weight_min": -32.47371292114258,
      "activations/layer17_attention_weight_max": 49.59414291381836,
      "activations/layer17_attention_weight_min": -43.831016540527344,
      "activations/layer18_attention_weight_max": 45.40732955932617,
      "activations/layer18_attention_weight_min": -39.7866325378418,
      "activations/layer19_attention_weight_max": 25.690692901611328,
      "activations/layer19_attention_weight_min": -24.205551147460938,
      "activations/layer1_attention_weight_max": 16.125370025634766,
      "activations/layer1_attention_weight_min": -13.76313591003418,
      "activations/layer20_attention_weight_max": 23.97381591796875,
      "activations/layer20_attention_weight_min": -22.83824920654297,
      "activations/layer21_attention_weight_max": 35.16023635864258,
      "activations/layer21_attention_weight_min": -25.849960327148438,
      "activations/layer22_attention_weight_max": 28.49534034729004,
      "activations/layer22_attention_weight_min": -26.098989486694336,
      "activations/layer23_attention_weight_max": 37.631595611572266,
      "activations/layer23_attention_weight_min": -24.378355026245117,
      "activations/layer2_attention_weight_max": 31.76803207397461,
      "activations/layer2_attention_weight_min": -30.04067611694336,
      "activations/layer3_attention_weight_max": 83.54828643798828,
      "activations/layer3_attention_weight_min": -82.33760833740234,
      "activations/layer4_attention_weight_max": 77.77703094482422,
      "activations/layer4_attention_weight_min": -77.8203353881836,
      "activations/layer5_attention_weight_max": 60.20469665527344,
      "activations/layer5_attention_weight_min": -68.15422821044922,
      "activations/layer6_attention_weight_max": 48.126216888427734,
      "activations/layer6_attention_weight_min": -47.59994125366211,
      "activations/layer7_attention_weight_max": 71.8769302368164,
      "activations/layer7_attention_weight_min": -63.260719299316406,
      "activations/layer8_attention_weight_max": 52.236175537109375,
      "activations/layer8_attention_weight_min": -46.04143524169922,
      "activations/layer9_attention_weight_max": 56.59482955932617,
      "activations/layer9_attention_weight_min": -45.79539108276367,
      "epoch": 21.16,
      "learning_rate": 1.3670833333333333e-05,
      "loss": 2.7123,
      "step": 364200
    },
    {
      "activations/layer0_attention_weight_max": 15.700663566589355,
      "activations/layer0_attention_weight_min": -14.893589973449707,
      "activations/layer10_attention_weight_max": 36.919898986816406,
      "activations/layer10_attention_weight_min": -36.225921630859375,
      "activations/layer11_attention_weight_max": 33.95534133911133,
      "activations/layer11_attention_weight_min": -34.67101287841797,
      "activations/layer12_attention_weight_max": 18.24047088623047,
      "activations/layer12_attention_weight_min": -25.87366485595703,
      "activations/layer13_attention_weight_max": 40.02979278564453,
      "activations/layer13_attention_weight_min": -27.581531524658203,
      "activations/layer14_attention_weight_max": 34.4105339050293,
      "activations/layer14_attention_weight_min": -30.364940643310547,
      "activations/layer15_attention_weight_max": 30.43659782409668,
      "activations/layer15_attention_weight_min": -30.821346282958984,
      "activations/layer16_attention_weight_max": 33.263179779052734,
      "activations/layer16_attention_weight_min": -33.18821334838867,
      "activations/layer17_attention_weight_max": 49.113468170166016,
      "activations/layer17_attention_weight_min": -47.246910095214844,
      "activations/layer18_attention_weight_max": 46.52994155883789,
      "activations/layer18_attention_weight_min": -39.195762634277344,
      "activations/layer19_attention_weight_max": 22.056785583496094,
      "activations/layer19_attention_weight_min": -21.775632858276367,
      "activations/layer1_attention_weight_max": 15.044900894165039,
      "activations/layer1_attention_weight_min": -12.621001243591309,
      "activations/layer20_attention_weight_max": 21.79063606262207,
      "activations/layer20_attention_weight_min": -23.39464569091797,
      "activations/layer21_attention_weight_max": 31.180309295654297,
      "activations/layer21_attention_weight_min": -20.439977645874023,
      "activations/layer22_attention_weight_max": 29.836917877197266,
      "activations/layer22_attention_weight_min": -25.23423194885254,
      "activations/layer23_attention_weight_max": 33.926673889160156,
      "activations/layer23_attention_weight_min": -24.861616134643555,
      "activations/layer2_attention_weight_max": 28.423995971679688,
      "activations/layer2_attention_weight_min": -28.57198143005371,
      "activations/layer3_attention_weight_max": 81.42333221435547,
      "activations/layer3_attention_weight_min": -81.64630126953125,
      "activations/layer4_attention_weight_max": 81.06158447265625,
      "activations/layer4_attention_weight_min": -79.27288055419922,
      "activations/layer5_attention_weight_max": 61.37120819091797,
      "activations/layer5_attention_weight_min": -68.81043243408203,
      "activations/layer6_attention_weight_max": 47.30716323852539,
      "activations/layer6_attention_weight_min": -47.96175765991211,
      "activations/layer7_attention_weight_max": 64.31120300292969,
      "activations/layer7_attention_weight_min": -64.21995544433594,
      "activations/layer8_attention_weight_max": 48.07242202758789,
      "activations/layer8_attention_weight_min": -49.31299591064453,
      "activations/layer9_attention_weight_max": 46.141963958740234,
      "activations/layer9_attention_weight_min": -46.86366653442383,
      "epoch": 21.17,
      "learning_rate": 1.365189393939394e-05,
      "loss": 2.7317,
      "step": 364250
    },
    {
      "activations/layer0_attention_weight_max": 15.051026344299316,
      "activations/layer0_attention_weight_min": -14.648043632507324,
      "activations/layer10_attention_weight_max": 35.099327087402344,
      "activations/layer10_attention_weight_min": -36.77669143676758,
      "activations/layer11_attention_weight_max": 33.928741455078125,
      "activations/layer11_attention_weight_min": -34.86500930786133,
      "activations/layer12_attention_weight_max": 19.15471839904785,
      "activations/layer12_attention_weight_min": -23.276382446289062,
      "activations/layer13_attention_weight_max": 33.757503509521484,
      "activations/layer13_attention_weight_min": -25.717275619506836,
      "activations/layer14_attention_weight_max": 34.61845397949219,
      "activations/layer14_attention_weight_min": -31.21672821044922,
      "activations/layer15_attention_weight_max": 32.79191970825195,
      "activations/layer15_attention_weight_min": -29.853084564208984,
      "activations/layer16_attention_weight_max": 34.0258903503418,
      "activations/layer16_attention_weight_min": -30.876249313354492,
      "activations/layer17_attention_weight_max": 50.7205696105957,
      "activations/layer17_attention_weight_min": -43.210994720458984,
      "activations/layer18_attention_weight_max": 47.829689025878906,
      "activations/layer18_attention_weight_min": -38.87506866455078,
      "activations/layer19_attention_weight_max": 24.21076774597168,
      "activations/layer19_attention_weight_min": -21.68915557861328,
      "activations/layer1_attention_weight_max": 14.918606758117676,
      "activations/layer1_attention_weight_min": -13.263463973999023,
      "activations/layer20_attention_weight_max": 23.17330551147461,
      "activations/layer20_attention_weight_min": -20.287763595581055,
      "activations/layer21_attention_weight_max": 34.25566482543945,
      "activations/layer21_attention_weight_min": -22.242610931396484,
      "activations/layer22_attention_weight_max": 28.470783233642578,
      "activations/layer22_attention_weight_min": -24.19878578186035,
      "activations/layer23_attention_weight_max": 37.539188385009766,
      "activations/layer23_attention_weight_min": -23.338857650756836,
      "activations/layer2_attention_weight_max": 30.839750289916992,
      "activations/layer2_attention_weight_min": -29.532257080078125,
      "activations/layer3_attention_weight_max": 86.78720092773438,
      "activations/layer3_attention_weight_min": -82.15540313720703,
      "activations/layer4_attention_weight_max": 80.7221450805664,
      "activations/layer4_attention_weight_min": -77.95892333984375,
      "activations/layer5_attention_weight_max": 61.5492057800293,
      "activations/layer5_attention_weight_min": -72.08853149414062,
      "activations/layer6_attention_weight_max": 49.76918029785156,
      "activations/layer6_attention_weight_min": -49.41268539428711,
      "activations/layer7_attention_weight_max": 69.0488052368164,
      "activations/layer7_attention_weight_min": -63.016719818115234,
      "activations/layer8_attention_weight_max": 46.54107666015625,
      "activations/layer8_attention_weight_min": -48.47587203979492,
      "activations/layer9_attention_weight_max": 44.544532775878906,
      "activations/layer9_attention_weight_min": -43.94807052612305,
      "epoch": 21.17,
      "learning_rate": 1.3632954545454544e-05,
      "loss": 2.7194,
      "step": 364300
    },
    {
      "activations/layer0_attention_weight_max": 15.88174057006836,
      "activations/layer0_attention_weight_min": -14.517433166503906,
      "activations/layer10_attention_weight_max": 35.541080474853516,
      "activations/layer10_attention_weight_min": -33.75251770019531,
      "activations/layer11_attention_weight_max": 30.74403953552246,
      "activations/layer11_attention_weight_min": -34.206504821777344,
      "activations/layer12_attention_weight_max": 18.490116119384766,
      "activations/layer12_attention_weight_min": -29.41225814819336,
      "activations/layer13_attention_weight_max": 33.85173797607422,
      "activations/layer13_attention_weight_min": -26.67972183227539,
      "activations/layer14_attention_weight_max": 34.97456359863281,
      "activations/layer14_attention_weight_min": -30.7955322265625,
      "activations/layer15_attention_weight_max": 34.33243942260742,
      "activations/layer15_attention_weight_min": -30.599241256713867,
      "activations/layer16_attention_weight_max": 35.032283782958984,
      "activations/layer16_attention_weight_min": -33.816001892089844,
      "activations/layer17_attention_weight_max": 53.611907958984375,
      "activations/layer17_attention_weight_min": -46.08021926879883,
      "activations/layer18_attention_weight_max": 45.87751770019531,
      "activations/layer18_attention_weight_min": -41.923763275146484,
      "activations/layer19_attention_weight_max": 26.31117820739746,
      "activations/layer19_attention_weight_min": -23.387731552124023,
      "activations/layer1_attention_weight_max": 16.709321975708008,
      "activations/layer1_attention_weight_min": -13.58198356628418,
      "activations/layer20_attention_weight_max": 21.674413681030273,
      "activations/layer20_attention_weight_min": -21.81342887878418,
      "activations/layer21_attention_weight_max": 34.018741607666016,
      "activations/layer21_attention_weight_min": -21.83234214782715,
      "activations/layer22_attention_weight_max": 26.667898178100586,
      "activations/layer22_attention_weight_min": -25.76932716369629,
      "activations/layer23_attention_weight_max": 33.19424057006836,
      "activations/layer23_attention_weight_min": -23.833099365234375,
      "activations/layer2_attention_weight_max": 31.073728561401367,
      "activations/layer2_attention_weight_min": -29.626094818115234,
      "activations/layer3_attention_weight_max": 83.8954086303711,
      "activations/layer3_attention_weight_min": -82.39189147949219,
      "activations/layer4_attention_weight_max": 79.32626342773438,
      "activations/layer4_attention_weight_min": -81.12110900878906,
      "activations/layer5_attention_weight_max": 61.06985092163086,
      "activations/layer5_attention_weight_min": -72.81104278564453,
      "activations/layer6_attention_weight_max": 48.045997619628906,
      "activations/layer6_attention_weight_min": -52.136070251464844,
      "activations/layer7_attention_weight_max": 63.78556442260742,
      "activations/layer7_attention_weight_min": -66.14784240722656,
      "activations/layer8_attention_weight_max": 43.53264236450195,
      "activations/layer8_attention_weight_min": -47.509925842285156,
      "activations/layer9_attention_weight_max": 40.81343078613281,
      "activations/layer9_attention_weight_min": -44.337772369384766,
      "epoch": 21.17,
      "learning_rate": 1.3614015151515151e-05,
      "loss": 2.7064,
      "step": 364350
    },
    {
      "activations/layer0_attention_weight_max": 16.176494598388672,
      "activations/layer0_attention_weight_min": -14.663864135742188,
      "activations/layer10_attention_weight_max": 36.19860076904297,
      "activations/layer10_attention_weight_min": -34.95721435546875,
      "activations/layer11_attention_weight_max": 33.35036087036133,
      "activations/layer11_attention_weight_min": -34.407936096191406,
      "activations/layer12_attention_weight_max": 20.08955192565918,
      "activations/layer12_attention_weight_min": -23.409271240234375,
      "activations/layer13_attention_weight_max": 34.29702377319336,
      "activations/layer13_attention_weight_min": -26.417139053344727,
      "activations/layer14_attention_weight_max": 35.31561279296875,
      "activations/layer14_attention_weight_min": -30.670860290527344,
      "activations/layer15_attention_weight_max": 34.32389450073242,
      "activations/layer15_attention_weight_min": -30.78824806213379,
      "activations/layer16_attention_weight_max": 35.08174133300781,
      "activations/layer16_attention_weight_min": -32.91227340698242,
      "activations/layer17_attention_weight_max": 55.15851593017578,
      "activations/layer17_attention_weight_min": -45.2677116394043,
      "activations/layer18_attention_weight_max": 50.53325653076172,
      "activations/layer18_attention_weight_min": -41.8748893737793,
      "activations/layer19_attention_weight_max": 26.2851619720459,
      "activations/layer19_attention_weight_min": -23.745176315307617,
      "activations/layer1_attention_weight_max": 14.924433708190918,
      "activations/layer1_attention_weight_min": -13.870460510253906,
      "activations/layer20_attention_weight_max": 25.779319763183594,
      "activations/layer20_attention_weight_min": -20.367948532104492,
      "activations/layer21_attention_weight_max": 36.61382293701172,
      "activations/layer21_attention_weight_min": -22.512664794921875,
      "activations/layer22_attention_weight_max": 30.100324630737305,
      "activations/layer22_attention_weight_min": -26.585792541503906,
      "activations/layer23_attention_weight_max": 35.978302001953125,
      "activations/layer23_attention_weight_min": -25.847835540771484,
      "activations/layer2_attention_weight_max": 31.86910629272461,
      "activations/layer2_attention_weight_min": -30.41501235961914,
      "activations/layer3_attention_weight_max": 89.38023376464844,
      "activations/layer3_attention_weight_min": -93.2901840209961,
      "activations/layer4_attention_weight_max": 80.03932189941406,
      "activations/layer4_attention_weight_min": -80.69318389892578,
      "activations/layer5_attention_weight_max": 60.82018280029297,
      "activations/layer5_attention_weight_min": -69.3318099975586,
      "activations/layer6_attention_weight_max": 47.107234954833984,
      "activations/layer6_attention_weight_min": -48.4109992980957,
      "activations/layer7_attention_weight_max": 64.15399169921875,
      "activations/layer7_attention_weight_min": -63.503257751464844,
      "activations/layer8_attention_weight_max": 45.162601470947266,
      "activations/layer8_attention_weight_min": -49.01808547973633,
      "activations/layer9_attention_weight_max": 45.52140426635742,
      "activations/layer9_attention_weight_min": -44.98286819458008,
      "epoch": 21.17,
      "learning_rate": 1.3595075757575756e-05,
      "loss": 2.7089,
      "step": 364400
    },
    {
      "activations/layer0_attention_weight_max": 16.553823471069336,
      "activations/layer0_attention_weight_min": -14.416121482849121,
      "activations/layer10_attention_weight_max": 33.7073860168457,
      "activations/layer10_attention_weight_min": -35.42231750488281,
      "activations/layer11_attention_weight_max": 33.323631286621094,
      "activations/layer11_attention_weight_min": -35.209110260009766,
      "activations/layer12_attention_weight_max": 18.86757469177246,
      "activations/layer12_attention_weight_min": -25.449222564697266,
      "activations/layer13_attention_weight_max": 31.04649543762207,
      "activations/layer13_attention_weight_min": -27.613834381103516,
      "activations/layer14_attention_weight_max": 33.032291412353516,
      "activations/layer14_attention_weight_min": -28.934751510620117,
      "activations/layer15_attention_weight_max": 31.805999755859375,
      "activations/layer15_attention_weight_min": -29.293725967407227,
      "activations/layer16_attention_weight_max": 33.05714416503906,
      "activations/layer16_attention_weight_min": -31.82938575744629,
      "activations/layer17_attention_weight_max": 48.3110237121582,
      "activations/layer17_attention_weight_min": -45.24542999267578,
      "activations/layer18_attention_weight_max": 46.00373458862305,
      "activations/layer18_attention_weight_min": -41.52688217163086,
      "activations/layer19_attention_weight_max": 24.727807998657227,
      "activations/layer19_attention_weight_min": -24.458791732788086,
      "activations/layer1_attention_weight_max": 14.841663360595703,
      "activations/layer1_attention_weight_min": -13.346342086791992,
      "activations/layer20_attention_weight_max": 24.121002197265625,
      "activations/layer20_attention_weight_min": -23.082082748413086,
      "activations/layer21_attention_weight_max": 38.36140823364258,
      "activations/layer21_attention_weight_min": -25.743633270263672,
      "activations/layer22_attention_weight_max": 26.27433967590332,
      "activations/layer22_attention_weight_min": -25.50162696838379,
      "activations/layer23_attention_weight_max": 34.972564697265625,
      "activations/layer23_attention_weight_min": -25.642065048217773,
      "activations/layer2_attention_weight_max": 30.94316864013672,
      "activations/layer2_attention_weight_min": -30.8807430267334,
      "activations/layer3_attention_weight_max": 83.45195770263672,
      "activations/layer3_attention_weight_min": -87.34040832519531,
      "activations/layer4_attention_weight_max": 79.89366912841797,
      "activations/layer4_attention_weight_min": -79.95462799072266,
      "activations/layer5_attention_weight_max": 57.25438690185547,
      "activations/layer5_attention_weight_min": -70.35697174072266,
      "activations/layer6_attention_weight_max": 47.7899284362793,
      "activations/layer6_attention_weight_min": -48.038761138916016,
      "activations/layer7_attention_weight_max": 66.86278533935547,
      "activations/layer7_attention_weight_min": -63.701969146728516,
      "activations/layer8_attention_weight_max": 43.59814453125,
      "activations/layer8_attention_weight_min": -48.7060661315918,
      "activations/layer9_attention_weight_max": 44.31679916381836,
      "activations/layer9_attention_weight_min": -43.06626892089844,
      "epoch": 21.18,
      "learning_rate": 1.3576136363636363e-05,
      "loss": 2.7249,
      "step": 364450
    },
    {
      "activations/layer0_attention_weight_max": 15.820013999938965,
      "activations/layer0_attention_weight_min": -16.09585189819336,
      "activations/layer10_attention_weight_max": 36.46146774291992,
      "activations/layer10_attention_weight_min": -37.095703125,
      "activations/layer11_attention_weight_max": 34.954803466796875,
      "activations/layer11_attention_weight_min": -35.367244720458984,
      "activations/layer12_attention_weight_max": 19.64182472229004,
      "activations/layer12_attention_weight_min": -24.057815551757812,
      "activations/layer13_attention_weight_max": 31.45244789123535,
      "activations/layer13_attention_weight_min": -28.437944412231445,
      "activations/layer14_attention_weight_max": 36.298736572265625,
      "activations/layer14_attention_weight_min": -32.835731506347656,
      "activations/layer15_attention_weight_max": 33.0374641418457,
      "activations/layer15_attention_weight_min": -33.49850082397461,
      "activations/layer16_attention_weight_max": 36.29844284057617,
      "activations/layer16_attention_weight_min": -33.867496490478516,
      "activations/layer17_attention_weight_max": 51.58851623535156,
      "activations/layer17_attention_weight_min": -44.57583236694336,
      "activations/layer18_attention_weight_max": 47.78795623779297,
      "activations/layer18_attention_weight_min": -40.2237548828125,
      "activations/layer19_attention_weight_max": 22.44666290283203,
      "activations/layer19_attention_weight_min": -22.64716148376465,
      "activations/layer1_attention_weight_max": 16.403676986694336,
      "activations/layer1_attention_weight_min": -15.610655784606934,
      "activations/layer20_attention_weight_max": 20.247129440307617,
      "activations/layer20_attention_weight_min": -23.33420181274414,
      "activations/layer21_attention_weight_max": 32.266578674316406,
      "activations/layer21_attention_weight_min": -22.37040138244629,
      "activations/layer22_attention_weight_max": 28.471773147583008,
      "activations/layer22_attention_weight_min": -27.480148315429688,
      "activations/layer23_attention_weight_max": 34.16749572753906,
      "activations/layer23_attention_weight_min": -24.426502227783203,
      "activations/layer2_attention_weight_max": 30.826099395751953,
      "activations/layer2_attention_weight_min": -29.745075225830078,
      "activations/layer3_attention_weight_max": 89.7525405883789,
      "activations/layer3_attention_weight_min": -90.0783462524414,
      "activations/layer4_attention_weight_max": 84.50137329101562,
      "activations/layer4_attention_weight_min": -86.07120513916016,
      "activations/layer5_attention_weight_max": 61.212127685546875,
      "activations/layer5_attention_weight_min": -74.62407684326172,
      "activations/layer6_attention_weight_max": 50.51976776123047,
      "activations/layer6_attention_weight_min": -51.34303283691406,
      "activations/layer7_attention_weight_max": 68.24977111816406,
      "activations/layer7_attention_weight_min": -70.57020568847656,
      "activations/layer8_attention_weight_max": 48.67167663574219,
      "activations/layer8_attention_weight_min": -52.63695526123047,
      "activations/layer9_attention_weight_max": 46.148468017578125,
      "activations/layer9_attention_weight_min": -47.74273681640625,
      "epoch": 21.18,
      "learning_rate": 1.3557196969696969e-05,
      "loss": 2.7298,
      "step": 364500
    },
    {
      "activations/layer0_attention_weight_max": 15.099584579467773,
      "activations/layer0_attention_weight_min": -14.217366218566895,
      "activations/layer10_attention_weight_max": 36.0317268371582,
      "activations/layer10_attention_weight_min": -36.163394927978516,
      "activations/layer11_attention_weight_max": 35.920257568359375,
      "activations/layer11_attention_weight_min": -36.54346466064453,
      "activations/layer12_attention_weight_max": 19.4669132232666,
      "activations/layer12_attention_weight_min": -24.24470329284668,
      "activations/layer13_attention_weight_max": 33.71070098876953,
      "activations/layer13_attention_weight_min": -26.315359115600586,
      "activations/layer14_attention_weight_max": 36.62566375732422,
      "activations/layer14_attention_weight_min": -29.62167739868164,
      "activations/layer15_attention_weight_max": 34.14796829223633,
      "activations/layer15_attention_weight_min": -29.42746353149414,
      "activations/layer16_attention_weight_max": 35.55026626586914,
      "activations/layer16_attention_weight_min": -33.36186599731445,
      "activations/layer17_attention_weight_max": 52.98983383178711,
      "activations/layer17_attention_weight_min": -44.8951530456543,
      "activations/layer18_attention_weight_max": 48.57924270629883,
      "activations/layer18_attention_weight_min": -38.88421630859375,
      "activations/layer19_attention_weight_max": 28.463193893432617,
      "activations/layer19_attention_weight_min": -23.446762084960938,
      "activations/layer1_attention_weight_max": 15.323128700256348,
      "activations/layer1_attention_weight_min": -14.226364135742188,
      "activations/layer20_attention_weight_max": 25.208690643310547,
      "activations/layer20_attention_weight_min": -23.86884880065918,
      "activations/layer21_attention_weight_max": 41.8323974609375,
      "activations/layer21_attention_weight_min": -24.940298080444336,
      "activations/layer22_attention_weight_max": 28.707857131958008,
      "activations/layer22_attention_weight_min": -29.683504104614258,
      "activations/layer23_attention_weight_max": 33.87739562988281,
      "activations/layer23_attention_weight_min": -26.640005111694336,
      "activations/layer2_attention_weight_max": 31.252517700195312,
      "activations/layer2_attention_weight_min": -30.002490997314453,
      "activations/layer3_attention_weight_max": 86.49008178710938,
      "activations/layer3_attention_weight_min": -89.48628997802734,
      "activations/layer4_attention_weight_max": 83.97229766845703,
      "activations/layer4_attention_weight_min": -82.44578552246094,
      "activations/layer5_attention_weight_max": 58.61064910888672,
      "activations/layer5_attention_weight_min": -69.8567123413086,
      "activations/layer6_attention_weight_max": 52.31418228149414,
      "activations/layer6_attention_weight_min": -52.622718811035156,
      "activations/layer7_attention_weight_max": 66.21344757080078,
      "activations/layer7_attention_weight_min": -65.96256256103516,
      "activations/layer8_attention_weight_max": 46.66896438598633,
      "activations/layer8_attention_weight_min": -52.0231819152832,
      "activations/layer9_attention_weight_max": 47.224422454833984,
      "activations/layer9_attention_weight_min": -47.89242172241211,
      "epoch": 21.18,
      "learning_rate": 1.3538257575757574e-05,
      "loss": 2.7025,
      "step": 364550
    },
    {
      "activations/layer0_attention_weight_max": 16.42756462097168,
      "activations/layer0_attention_weight_min": -14.26114273071289,
      "activations/layer10_attention_weight_max": 37.216796875,
      "activations/layer10_attention_weight_min": -36.43320846557617,
      "activations/layer11_attention_weight_max": 34.774749755859375,
      "activations/layer11_attention_weight_min": -35.95677947998047,
      "activations/layer12_attention_weight_max": 18.891481399536133,
      "activations/layer12_attention_weight_min": -25.215482711791992,
      "activations/layer13_attention_weight_max": 35.534332275390625,
      "activations/layer13_attention_weight_min": -27.682876586914062,
      "activations/layer14_attention_weight_max": 35.51367950439453,
      "activations/layer14_attention_weight_min": -29.87704086303711,
      "activations/layer15_attention_weight_max": 38.3741455078125,
      "activations/layer15_attention_weight_min": -30.685260772705078,
      "activations/layer16_attention_weight_max": 38.41807174682617,
      "activations/layer16_attention_weight_min": -34.21782684326172,
      "activations/layer17_attention_weight_max": 56.670997619628906,
      "activations/layer17_attention_weight_min": -47.711822509765625,
      "activations/layer18_attention_weight_max": 53.031925201416016,
      "activations/layer18_attention_weight_min": -40.58979415893555,
      "activations/layer19_attention_weight_max": 30.21923828125,
      "activations/layer19_attention_weight_min": -23.578319549560547,
      "activations/layer1_attention_weight_max": 15.529054641723633,
      "activations/layer1_attention_weight_min": -13.481609344482422,
      "activations/layer20_attention_weight_max": 26.160356521606445,
      "activations/layer20_attention_weight_min": -22.19448471069336,
      "activations/layer21_attention_weight_max": 37.191314697265625,
      "activations/layer21_attention_weight_min": -21.87643051147461,
      "activations/layer22_attention_weight_max": 33.13163757324219,
      "activations/layer22_attention_weight_min": -25.44903564453125,
      "activations/layer23_attention_weight_max": 40.312889099121094,
      "activations/layer23_attention_weight_min": -26.22726821899414,
      "activations/layer2_attention_weight_max": 30.406898498535156,
      "activations/layer2_attention_weight_min": -31.019283294677734,
      "activations/layer3_attention_weight_max": 89.15562438964844,
      "activations/layer3_attention_weight_min": -91.31573486328125,
      "activations/layer4_attention_weight_max": 86.51717376708984,
      "activations/layer4_attention_weight_min": -86.54812622070312,
      "activations/layer5_attention_weight_max": 61.633766174316406,
      "activations/layer5_attention_weight_min": -71.79425048828125,
      "activations/layer6_attention_weight_max": 50.49165725708008,
      "activations/layer6_attention_weight_min": -51.162296295166016,
      "activations/layer7_attention_weight_max": 67.66288757324219,
      "activations/layer7_attention_weight_min": -64.31665802001953,
      "activations/layer8_attention_weight_max": 46.56070327758789,
      "activations/layer8_attention_weight_min": -52.10502243041992,
      "activations/layer9_attention_weight_max": 43.955780029296875,
      "activations/layer9_attention_weight_min": -45.21373748779297,
      "epoch": 21.19,
      "learning_rate": 1.351931818181818e-05,
      "loss": 2.6884,
      "step": 364600
    },
    {
      "activations/layer0_attention_weight_max": 14.69357681274414,
      "activations/layer0_attention_weight_min": -14.27010440826416,
      "activations/layer10_attention_weight_max": 37.130859375,
      "activations/layer10_attention_weight_min": -36.560916900634766,
      "activations/layer11_attention_weight_max": 36.603416442871094,
      "activations/layer11_attention_weight_min": -37.201927185058594,
      "activations/layer12_attention_weight_max": 19.515275955200195,
      "activations/layer12_attention_weight_min": -23.466415405273438,
      "activations/layer13_attention_weight_max": 44.72794723510742,
      "activations/layer13_attention_weight_min": -29.730907440185547,
      "activations/layer14_attention_weight_max": 49.16733932495117,
      "activations/layer14_attention_weight_min": -31.96842384338379,
      "activations/layer15_attention_weight_max": 42.39076614379883,
      "activations/layer15_attention_weight_min": -31.044050216674805,
      "activations/layer16_attention_weight_max": 34.70980453491211,
      "activations/layer16_attention_weight_min": -32.65065002441406,
      "activations/layer17_attention_weight_max": 51.88115310668945,
      "activations/layer17_attention_weight_min": -46.98128128051758,
      "activations/layer18_attention_weight_max": 49.182247161865234,
      "activations/layer18_attention_weight_min": -44.84152603149414,
      "activations/layer19_attention_weight_max": 28.378860473632812,
      "activations/layer19_attention_weight_min": -23.76190757751465,
      "activations/layer1_attention_weight_max": 14.571370124816895,
      "activations/layer1_attention_weight_min": -15.116220474243164,
      "activations/layer20_attention_weight_max": 21.75892448425293,
      "activations/layer20_attention_weight_min": -21.461286544799805,
      "activations/layer21_attention_weight_max": 35.63231658935547,
      "activations/layer21_attention_weight_min": -21.84037971496582,
      "activations/layer22_attention_weight_max": 27.24807357788086,
      "activations/layer22_attention_weight_min": -25.214157104492188,
      "activations/layer23_attention_weight_max": 33.23472213745117,
      "activations/layer23_attention_weight_min": -24.518033981323242,
      "activations/layer2_attention_weight_max": 31.972135543823242,
      "activations/layer2_attention_weight_min": -31.41805648803711,
      "activations/layer3_attention_weight_max": 89.47025299072266,
      "activations/layer3_attention_weight_min": -87.62810516357422,
      "activations/layer4_attention_weight_max": 83.38693237304688,
      "activations/layer4_attention_weight_min": -85.03321075439453,
      "activations/layer5_attention_weight_max": 59.20201873779297,
      "activations/layer5_attention_weight_min": -68.91213989257812,
      "activations/layer6_attention_weight_max": 49.85159683227539,
      "activations/layer6_attention_weight_min": -51.01494598388672,
      "activations/layer7_attention_weight_max": 69.72425842285156,
      "activations/layer7_attention_weight_min": -68.18877410888672,
      "activations/layer8_attention_weight_max": 49.82583999633789,
      "activations/layer8_attention_weight_min": -52.95448684692383,
      "activations/layer9_attention_weight_max": 48.789180755615234,
      "activations/layer9_attention_weight_min": -48.78218460083008,
      "epoch": 21.19,
      "learning_rate": 1.3500378787878787e-05,
      "loss": 2.7074,
      "step": 364650
    },
    {
      "activations/layer0_attention_weight_max": 16.926116943359375,
      "activations/layer0_attention_weight_min": -14.579172134399414,
      "activations/layer10_attention_weight_max": 39.610755920410156,
      "activations/layer10_attention_weight_min": -37.358333587646484,
      "activations/layer11_attention_weight_max": 37.00761795043945,
      "activations/layer11_attention_weight_min": -34.56718063354492,
      "activations/layer12_attention_weight_max": 20.475187301635742,
      "activations/layer12_attention_weight_min": -23.199914932250977,
      "activations/layer13_attention_weight_max": 37.17965316772461,
      "activations/layer13_attention_weight_min": -29.464113235473633,
      "activations/layer14_attention_weight_max": 35.5346794128418,
      "activations/layer14_attention_weight_min": -32.810943603515625,
      "activations/layer15_attention_weight_max": 33.442115783691406,
      "activations/layer15_attention_weight_min": -31.994495391845703,
      "activations/layer16_attention_weight_max": 34.70942687988281,
      "activations/layer16_attention_weight_min": -33.4614143371582,
      "activations/layer17_attention_weight_max": 53.302642822265625,
      "activations/layer17_attention_weight_min": -46.62628173828125,
      "activations/layer18_attention_weight_max": 49.5815315246582,
      "activations/layer18_attention_weight_min": -43.38814163208008,
      "activations/layer19_attention_weight_max": 23.75857925415039,
      "activations/layer19_attention_weight_min": -23.211149215698242,
      "activations/layer1_attention_weight_max": 14.995004653930664,
      "activations/layer1_attention_weight_min": -13.984519958496094,
      "activations/layer20_attention_weight_max": 23.463407516479492,
      "activations/layer20_attention_weight_min": -23.867950439453125,
      "activations/layer21_attention_weight_max": 34.47724151611328,
      "activations/layer21_attention_weight_min": -22.132883071899414,
      "activations/layer22_attention_weight_max": 29.790735244750977,
      "activations/layer22_attention_weight_min": -26.093889236450195,
      "activations/layer23_attention_weight_max": 40.01783752441406,
      "activations/layer23_attention_weight_min": -25.606422424316406,
      "activations/layer2_attention_weight_max": 30.51221466064453,
      "activations/layer2_attention_weight_min": -29.525571823120117,
      "activations/layer3_attention_weight_max": 86.84427642822266,
      "activations/layer3_attention_weight_min": -83.12345123291016,
      "activations/layer4_attention_weight_max": 82.04125213623047,
      "activations/layer4_attention_weight_min": -80.76506805419922,
      "activations/layer5_attention_weight_max": 61.842308044433594,
      "activations/layer5_attention_weight_min": -67.30517578125,
      "activations/layer6_attention_weight_max": 48.7961540222168,
      "activations/layer6_attention_weight_min": -50.80696487426758,
      "activations/layer7_attention_weight_max": 64.04723358154297,
      "activations/layer7_attention_weight_min": -65.23109436035156,
      "activations/layer8_attention_weight_max": 50.939823150634766,
      "activations/layer8_attention_weight_min": -50.290313720703125,
      "activations/layer9_attention_weight_max": 46.3723030090332,
      "activations/layer9_attention_weight_min": -45.27309036254883,
      "epoch": 21.19,
      "learning_rate": 1.3481439393939392e-05,
      "loss": 2.7132,
      "step": 364700
    },
    {
      "activations/layer0_attention_weight_max": 15.986247062683105,
      "activations/layer0_attention_weight_min": -14.277121543884277,
      "activations/layer10_attention_weight_max": 34.540199279785156,
      "activations/layer10_attention_weight_min": -34.00460433959961,
      "activations/layer11_attention_weight_max": 30.88489532470703,
      "activations/layer11_attention_weight_min": -33.45427703857422,
      "activations/layer12_attention_weight_max": 18.615779876708984,
      "activations/layer12_attention_weight_min": -25.575702667236328,
      "activations/layer13_attention_weight_max": 34.390254974365234,
      "activations/layer13_attention_weight_min": -27.579408645629883,
      "activations/layer14_attention_weight_max": 37.49837875366211,
      "activations/layer14_attention_weight_min": -31.814224243164062,
      "activations/layer15_attention_weight_max": 34.9174690246582,
      "activations/layer15_attention_weight_min": -30.90813636779785,
      "activations/layer16_attention_weight_max": 37.64515686035156,
      "activations/layer16_attention_weight_min": -33.35805892944336,
      "activations/layer17_attention_weight_max": 53.3456916809082,
      "activations/layer17_attention_weight_min": -49.961814880371094,
      "activations/layer18_attention_weight_max": 50.28009796142578,
      "activations/layer18_attention_weight_min": -43.22751235961914,
      "activations/layer19_attention_weight_max": 25.457868576049805,
      "activations/layer19_attention_weight_min": -22.501266479492188,
      "activations/layer1_attention_weight_max": 15.156508445739746,
      "activations/layer1_attention_weight_min": -13.830870628356934,
      "activations/layer20_attention_weight_max": 24.464021682739258,
      "activations/layer20_attention_weight_min": -22.513853073120117,
      "activations/layer21_attention_weight_max": 44.16743850708008,
      "activations/layer21_attention_weight_min": -24.264036178588867,
      "activations/layer22_attention_weight_max": 33.228172302246094,
      "activations/layer22_attention_weight_min": -27.5402774810791,
      "activations/layer23_attention_weight_max": 41.36429214477539,
      "activations/layer23_attention_weight_min": -22.791297912597656,
      "activations/layer2_attention_weight_max": 30.7728328704834,
      "activations/layer2_attention_weight_min": -30.276626586914062,
      "activations/layer3_attention_weight_max": 84.30284881591797,
      "activations/layer3_attention_weight_min": -85.1749038696289,
      "activations/layer4_attention_weight_max": 78.20748138427734,
      "activations/layer4_attention_weight_min": -77.85767364501953,
      "activations/layer5_attention_weight_max": 55.69829559326172,
      "activations/layer5_attention_weight_min": -68.88016510009766,
      "activations/layer6_attention_weight_max": 45.059993743896484,
      "activations/layer6_attention_weight_min": -46.043582916259766,
      "activations/layer7_attention_weight_max": 64.15103912353516,
      "activations/layer7_attention_weight_min": -64.64250946044922,
      "activations/layer8_attention_weight_max": 42.31235885620117,
      "activations/layer8_attention_weight_min": -46.18163299560547,
      "activations/layer9_attention_weight_max": 42.30602264404297,
      "activations/layer9_attention_weight_min": -43.1154899597168,
      "epoch": 21.19,
      "learning_rate": 1.3462499999999999e-05,
      "loss": 2.7196,
      "step": 364750
    },
    {
      "activations/layer0_attention_weight_max": 15.166522979736328,
      "activations/layer0_attention_weight_min": -14.397355079650879,
      "activations/layer10_attention_weight_max": 34.44654846191406,
      "activations/layer10_attention_weight_min": -35.873207092285156,
      "activations/layer11_attention_weight_max": 31.458829879760742,
      "activations/layer11_attention_weight_min": -35.54203414916992,
      "activations/layer12_attention_weight_max": 18.707290649414062,
      "activations/layer12_attention_weight_min": -23.705890655517578,
      "activations/layer13_attention_weight_max": 30.73198699951172,
      "activations/layer13_attention_weight_min": -26.788822174072266,
      "activations/layer14_attention_weight_max": 35.55899429321289,
      "activations/layer14_attention_weight_min": -30.861005783081055,
      "activations/layer15_attention_weight_max": 31.676971435546875,
      "activations/layer15_attention_weight_min": -29.83930778503418,
      "activations/layer16_attention_weight_max": 33.37723159790039,
      "activations/layer16_attention_weight_min": -31.735761642456055,
      "activations/layer17_attention_weight_max": 55.05302429199219,
      "activations/layer17_attention_weight_min": -45.83573913574219,
      "activations/layer18_attention_weight_max": 49.35906219482422,
      "activations/layer18_attention_weight_min": -40.2636833190918,
      "activations/layer19_attention_weight_max": 22.402292251586914,
      "activations/layer19_attention_weight_min": -22.67746925354004,
      "activations/layer1_attention_weight_max": 16.031660079956055,
      "activations/layer1_attention_weight_min": -15.149116516113281,
      "activations/layer20_attention_weight_max": 19.794389724731445,
      "activations/layer20_attention_weight_min": -22.132963180541992,
      "activations/layer21_attention_weight_max": 33.46760940551758,
      "activations/layer21_attention_weight_min": -20.828161239624023,
      "activations/layer22_attention_weight_max": 28.412689208984375,
      "activations/layer22_attention_weight_min": -25.43663787841797,
      "activations/layer23_attention_weight_max": 36.03892517089844,
      "activations/layer23_attention_weight_min": -23.613758087158203,
      "activations/layer2_attention_weight_max": 33.683502197265625,
      "activations/layer2_attention_weight_min": -31.6837215423584,
      "activations/layer3_attention_weight_max": 91.88509368896484,
      "activations/layer3_attention_weight_min": -90.52059173583984,
      "activations/layer4_attention_weight_max": 86.15132141113281,
      "activations/layer4_attention_weight_min": -84.7227783203125,
      "activations/layer5_attention_weight_max": 64.20555114746094,
      "activations/layer5_attention_weight_min": -72.31507110595703,
      "activations/layer6_attention_weight_max": 48.49455261230469,
      "activations/layer6_attention_weight_min": -49.46039962768555,
      "activations/layer7_attention_weight_max": 68.46922302246094,
      "activations/layer7_attention_weight_min": -64.48046875,
      "activations/layer8_attention_weight_max": 44.20885467529297,
      "activations/layer8_attention_weight_min": -47.79509735107422,
      "activations/layer9_attention_weight_max": 40.59022521972656,
      "activations/layer9_attention_weight_min": -45.2791748046875,
      "epoch": 21.2,
      "learning_rate": 1.3443560606060603e-05,
      "loss": 2.6999,
      "step": 364800
    },
    {
      "activations/layer0_attention_weight_max": 15.779149055480957,
      "activations/layer0_attention_weight_min": -14.0897216796875,
      "activations/layer10_attention_weight_max": 37.23038101196289,
      "activations/layer10_attention_weight_min": -36.3346061706543,
      "activations/layer11_attention_weight_max": 35.3403434753418,
      "activations/layer11_attention_weight_min": -34.40154266357422,
      "activations/layer12_attention_weight_max": 20.5401668548584,
      "activations/layer12_attention_weight_min": -31.90069007873535,
      "activations/layer13_attention_weight_max": 39.807159423828125,
      "activations/layer13_attention_weight_min": -29.886987686157227,
      "activations/layer14_attention_weight_max": 41.14817810058594,
      "activations/layer14_attention_weight_min": -33.43831253051758,
      "activations/layer15_attention_weight_max": 40.94841003417969,
      "activations/layer15_attention_weight_min": -33.048580169677734,
      "activations/layer16_attention_weight_max": 38.523136138916016,
      "activations/layer16_attention_weight_min": -34.37245178222656,
      "activations/layer17_attention_weight_max": 58.81003952026367,
      "activations/layer17_attention_weight_min": -52.548187255859375,
      "activations/layer18_attention_weight_max": 55.370784759521484,
      "activations/layer18_attention_weight_min": -47.56224822998047,
      "activations/layer19_attention_weight_max": 26.851104736328125,
      "activations/layer19_attention_weight_min": -24.782636642456055,
      "activations/layer1_attention_weight_max": 15.611979484558105,
      "activations/layer1_attention_weight_min": -13.984769821166992,
      "activations/layer20_attention_weight_max": 25.337919235229492,
      "activations/layer20_attention_weight_min": -23.07782554626465,
      "activations/layer21_attention_weight_max": 44.2934455871582,
      "activations/layer21_attention_weight_min": -24.458885192871094,
      "activations/layer22_attention_weight_max": 29.540321350097656,
      "activations/layer22_attention_weight_min": -24.4191837310791,
      "activations/layer23_attention_weight_max": 37.75078582763672,
      "activations/layer23_attention_weight_min": -24.8990478515625,
      "activations/layer2_attention_weight_max": 31.70339012145996,
      "activations/layer2_attention_weight_min": -31.51691436767578,
      "activations/layer3_attention_weight_max": 85.92933654785156,
      "activations/layer3_attention_weight_min": -86.97407531738281,
      "activations/layer4_attention_weight_max": 79.91028594970703,
      "activations/layer4_attention_weight_min": -78.57368469238281,
      "activations/layer5_attention_weight_max": 58.16899108886719,
      "activations/layer5_attention_weight_min": -68.17645263671875,
      "activations/layer6_attention_weight_max": 48.067264556884766,
      "activations/layer6_attention_weight_min": -46.977256774902344,
      "activations/layer7_attention_weight_max": 73.46165466308594,
      "activations/layer7_attention_weight_min": -65.95537567138672,
      "activations/layer8_attention_weight_max": 48.67774963378906,
      "activations/layer8_attention_weight_min": -47.28572463989258,
      "activations/layer9_attention_weight_max": 54.16781234741211,
      "activations/layer9_attention_weight_min": -44.34052658081055,
      "epoch": 21.2,
      "learning_rate": 1.342462121212121e-05,
      "loss": 2.7034,
      "step": 364850
    },
    {
      "activations/layer0_attention_weight_max": 15.629429817199707,
      "activations/layer0_attention_weight_min": -14.717966079711914,
      "activations/layer10_attention_weight_max": 34.33370590209961,
      "activations/layer10_attention_weight_min": -35.673065185546875,
      "activations/layer11_attention_weight_max": 33.643943786621094,
      "activations/layer11_attention_weight_min": -35.57942581176758,
      "activations/layer12_attention_weight_max": 19.173433303833008,
      "activations/layer12_attention_weight_min": -28.1444034576416,
      "activations/layer13_attention_weight_max": 32.44459533691406,
      "activations/layer13_attention_weight_min": -27.84897804260254,
      "activations/layer14_attention_weight_max": 31.474971771240234,
      "activations/layer14_attention_weight_min": -32.249725341796875,
      "activations/layer15_attention_weight_max": 31.87662124633789,
      "activations/layer15_attention_weight_min": -32.76921463012695,
      "activations/layer16_attention_weight_max": 31.983850479125977,
      "activations/layer16_attention_weight_min": -32.636024475097656,
      "activations/layer17_attention_weight_max": 49.0546875,
      "activations/layer17_attention_weight_min": -47.915348052978516,
      "activations/layer18_attention_weight_max": 45.51929473876953,
      "activations/layer18_attention_weight_min": -42.811927795410156,
      "activations/layer19_attention_weight_max": 22.84638023376465,
      "activations/layer19_attention_weight_min": -25.949939727783203,
      "activations/layer1_attention_weight_max": 14.462824821472168,
      "activations/layer1_attention_weight_min": -15.987540245056152,
      "activations/layer20_attention_weight_max": 22.035810470581055,
      "activations/layer20_attention_weight_min": -25.718616485595703,
      "activations/layer21_attention_weight_max": 37.88399124145508,
      "activations/layer21_attention_weight_min": -20.834457397460938,
      "activations/layer22_attention_weight_max": 26.399127960205078,
      "activations/layer22_attention_weight_min": -27.487958908081055,
      "activations/layer23_attention_weight_max": 33.8011474609375,
      "activations/layer23_attention_weight_min": -23.86870574951172,
      "activations/layer2_attention_weight_max": 32.250179290771484,
      "activations/layer2_attention_weight_min": -30.802658081054688,
      "activations/layer3_attention_weight_max": 88.22781372070312,
      "activations/layer3_attention_weight_min": -87.27082824707031,
      "activations/layer4_attention_weight_max": 82.83806610107422,
      "activations/layer4_attention_weight_min": -81.75997161865234,
      "activations/layer5_attention_weight_max": 61.35883331298828,
      "activations/layer5_attention_weight_min": -67.99447631835938,
      "activations/layer6_attention_weight_max": 49.42634201049805,
      "activations/layer6_attention_weight_min": -52.40903091430664,
      "activations/layer7_attention_weight_max": 69.9891586303711,
      "activations/layer7_attention_weight_min": -63.96147537231445,
      "activations/layer8_attention_weight_max": 45.71745681762695,
      "activations/layer8_attention_weight_min": -50.60881423950195,
      "activations/layer9_attention_weight_max": 42.44905090332031,
      "activations/layer9_attention_weight_min": -44.9141960144043,
      "epoch": 21.2,
      "learning_rate": 1.3405681818181817e-05,
      "loss": 2.7007,
      "step": 364900
    },
    {
      "activations/layer0_attention_weight_max": 14.973470687866211,
      "activations/layer0_attention_weight_min": -14.485011100769043,
      "activations/layer10_attention_weight_max": 37.0284423828125,
      "activations/layer10_attention_weight_min": -36.021297454833984,
      "activations/layer11_attention_weight_max": 34.599056243896484,
      "activations/layer11_attention_weight_min": -35.08345031738281,
      "activations/layer12_attention_weight_max": 18.693233489990234,
      "activations/layer12_attention_weight_min": -27.537240982055664,
      "activations/layer13_attention_weight_max": 33.092830657958984,
      "activations/layer13_attention_weight_min": -26.588289260864258,
      "activations/layer14_attention_weight_max": 34.33165740966797,
      "activations/layer14_attention_weight_min": -29.652584075927734,
      "activations/layer15_attention_weight_max": 32.33250427246094,
      "activations/layer15_attention_weight_min": -29.351789474487305,
      "activations/layer16_attention_weight_max": 34.396785736083984,
      "activations/layer16_attention_weight_min": -34.278358459472656,
      "activations/layer17_attention_weight_max": 51.44736862182617,
      "activations/layer17_attention_weight_min": -44.45159149169922,
      "activations/layer18_attention_weight_max": 47.00023651123047,
      "activations/layer18_attention_weight_min": -39.843685150146484,
      "activations/layer19_attention_weight_max": 24.66029167175293,
      "activations/layer19_attention_weight_min": -21.4898738861084,
      "activations/layer1_attention_weight_max": 15.08146858215332,
      "activations/layer1_attention_weight_min": -14.18946647644043,
      "activations/layer20_attention_weight_max": 21.06421661376953,
      "activations/layer20_attention_weight_min": -20.27280616760254,
      "activations/layer21_attention_weight_max": 34.047054290771484,
      "activations/layer21_attention_weight_min": -22.807945251464844,
      "activations/layer22_attention_weight_max": 27.561771392822266,
      "activations/layer22_attention_weight_min": -24.74380874633789,
      "activations/layer23_attention_weight_max": 35.01543045043945,
      "activations/layer23_attention_weight_min": -25.536128997802734,
      "activations/layer2_attention_weight_max": 30.375564575195312,
      "activations/layer2_attention_weight_min": -31.395971298217773,
      "activations/layer3_attention_weight_max": 86.2486572265625,
      "activations/layer3_attention_weight_min": -90.4802017211914,
      "activations/layer4_attention_weight_max": 85.38688659667969,
      "activations/layer4_attention_weight_min": -84.58248138427734,
      "activations/layer5_attention_weight_max": 60.0587272644043,
      "activations/layer5_attention_weight_min": -74.35201263427734,
      "activations/layer6_attention_weight_max": 48.969303131103516,
      "activations/layer6_attention_weight_min": -51.16099548339844,
      "activations/layer7_attention_weight_max": 65.67796325683594,
      "activations/layer7_attention_weight_min": -72.1930160522461,
      "activations/layer8_attention_weight_max": 47.374332427978516,
      "activations/layer8_attention_weight_min": -53.07623291015625,
      "activations/layer9_attention_weight_max": 45.20124053955078,
      "activations/layer9_attention_weight_min": -47.102840423583984,
      "epoch": 21.21,
      "learning_rate": 1.3386742424242421e-05,
      "loss": 2.7206,
      "step": 364950
    },
    {
      "activations/layer0_attention_weight_max": 15.21633243560791,
      "activations/layer0_attention_weight_min": -14.415101051330566,
      "activations/layer10_attention_weight_max": 35.0602912902832,
      "activations/layer10_attention_weight_min": -35.1493034362793,
      "activations/layer11_attention_weight_max": 32.865638732910156,
      "activations/layer11_attention_weight_min": -34.487449645996094,
      "activations/layer12_attention_weight_max": 23.131935119628906,
      "activations/layer12_attention_weight_min": -25.659711837768555,
      "activations/layer13_attention_weight_max": 33.63508224487305,
      "activations/layer13_attention_weight_min": -28.40524673461914,
      "activations/layer14_attention_weight_max": 35.802345275878906,
      "activations/layer14_attention_weight_min": -32.814964294433594,
      "activations/layer15_attention_weight_max": 33.796287536621094,
      "activations/layer15_attention_weight_min": -33.28114318847656,
      "activations/layer16_attention_weight_max": 34.88581085205078,
      "activations/layer16_attention_weight_min": -34.497684478759766,
      "activations/layer17_attention_weight_max": 52.96522903442383,
      "activations/layer17_attention_weight_min": -49.47999954223633,
      "activations/layer18_attention_weight_max": 49.04085922241211,
      "activations/layer18_attention_weight_min": -43.61423110961914,
      "activations/layer19_attention_weight_max": 25.11081314086914,
      "activations/layer19_attention_weight_min": -23.59614372253418,
      "activations/layer1_attention_weight_max": 15.703958511352539,
      "activations/layer1_attention_weight_min": -14.429755210876465,
      "activations/layer20_attention_weight_max": 23.224632263183594,
      "activations/layer20_attention_weight_min": -22.066043853759766,
      "activations/layer21_attention_weight_max": 37.615657806396484,
      "activations/layer21_attention_weight_min": -22.147092819213867,
      "activations/layer22_attention_weight_max": 29.946475982666016,
      "activations/layer22_attention_weight_min": -25.84958839416504,
      "activations/layer23_attention_weight_max": 35.89824295043945,
      "activations/layer23_attention_weight_min": -25.08119010925293,
      "activations/layer2_attention_weight_max": 30.999135971069336,
      "activations/layer2_attention_weight_min": -30.043651580810547,
      "activations/layer3_attention_weight_max": 86.509765625,
      "activations/layer3_attention_weight_min": -85.8265151977539,
      "activations/layer4_attention_weight_max": 83.39794158935547,
      "activations/layer4_attention_weight_min": -81.78069305419922,
      "activations/layer5_attention_weight_max": 63.150428771972656,
      "activations/layer5_attention_weight_min": -70.05401611328125,
      "activations/layer6_attention_weight_max": 49.32752227783203,
      "activations/layer6_attention_weight_min": -49.39938735961914,
      "activations/layer7_attention_weight_max": 66.80683135986328,
      "activations/layer7_attention_weight_min": -67.32892608642578,
      "activations/layer8_attention_weight_max": 45.99034118652344,
      "activations/layer8_attention_weight_min": -48.34019088745117,
      "activations/layer9_attention_weight_max": 43.50722885131836,
      "activations/layer9_attention_weight_min": -43.60080337524414,
      "epoch": 21.21,
      "learning_rate": 1.336780303030303e-05,
      "loss": 2.7263,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4694,
      "eval_samples_per_second": 507.002,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4694,
      "eval_openwebtext_samples_per_second": 507.002,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9603,
      "eval_wikitext_samples_per_second": 232.613,
      "step": 365000
    },
    {
      "epoch": 21.21,
      "eval_lambada_loss": 2.419921875,
      "eval_lambada_ppl": 11.244980766441616,
      "eval_lambada_runtime": 9.5247,
      "eval_lambada_samples_per_second": 511.197,
      "step": 365000
    },
    {
      "activations/layer0_attention_weight_max": 15.8933687210083,
      "activations/layer0_attention_weight_min": -14.481196403503418,
      "activations/layer10_attention_weight_max": 35.903770446777344,
      "activations/layer10_attention_weight_min": -36.98671340942383,
      "activations/layer11_attention_weight_max": 34.04184341430664,
      "activations/layer11_attention_weight_min": -35.180580139160156,
      "activations/layer12_attention_weight_max": 18.49045753479004,
      "activations/layer12_attention_weight_min": -25.606470108032227,
      "activations/layer13_attention_weight_max": 33.69736862182617,
      "activations/layer13_attention_weight_min": -28.243663787841797,
      "activations/layer14_attention_weight_max": 34.38924026489258,
      "activations/layer14_attention_weight_min": -30.58050537109375,
      "activations/layer15_attention_weight_max": 32.39347457885742,
      "activations/layer15_attention_weight_min": -32.20180130004883,
      "activations/layer16_attention_weight_max": 34.2397575378418,
      "activations/layer16_attention_weight_min": -33.67984390258789,
      "activations/layer17_attention_weight_max": 51.201725006103516,
      "activations/layer17_attention_weight_min": -47.37248611450195,
      "activations/layer18_attention_weight_max": 48.20785903930664,
      "activations/layer18_attention_weight_min": -42.59169387817383,
      "activations/layer19_attention_weight_max": 22.903493881225586,
      "activations/layer19_attention_weight_min": -24.069791793823242,
      "activations/layer1_attention_weight_max": 15.022547721862793,
      "activations/layer1_attention_weight_min": -14.681056022644043,
      "activations/layer20_attention_weight_max": 20.062284469604492,
      "activations/layer20_attention_weight_min": -21.616662979125977,
      "activations/layer21_attention_weight_max": 45.372894287109375,
      "activations/layer21_attention_weight_min": -24.291885375976562,
      "activations/layer22_attention_weight_max": 26.7974910736084,
      "activations/layer22_attention_weight_min": -25.16683006286621,
      "activations/layer23_attention_weight_max": 36.70293045043945,
      "activations/layer23_attention_weight_min": -23.334430694580078,
      "activations/layer2_attention_weight_max": 33.145111083984375,
      "activations/layer2_attention_weight_min": -31.460988998413086,
      "activations/layer3_attention_weight_max": 85.25202941894531,
      "activations/layer3_attention_weight_min": -89.25201416015625,
      "activations/layer4_attention_weight_max": 80.66656494140625,
      "activations/layer4_attention_weight_min": -86.28716278076172,
      "activations/layer5_attention_weight_max": 60.26921844482422,
      "activations/layer5_attention_weight_min": -69.9386978149414,
      "activations/layer6_attention_weight_max": 48.26622009277344,
      "activations/layer6_attention_weight_min": -50.30868148803711,
      "activations/layer7_attention_weight_max": 64.27659606933594,
      "activations/layer7_attention_weight_min": -69.4719467163086,
      "activations/layer8_attention_weight_max": 46.880435943603516,
      "activations/layer8_attention_weight_min": -49.16195297241211,
      "activations/layer9_attention_weight_max": 44.12734603881836,
      "activations/layer9_attention_weight_min": -48.41195297241211,
      "epoch": 21.21,
      "learning_rate": 1.3348863636363636e-05,
      "loss": 2.6995,
      "step": 365050
    },
    {
      "activations/layer0_attention_weight_max": 16.537771224975586,
      "activations/layer0_attention_weight_min": -14.262556076049805,
      "activations/layer10_attention_weight_max": 35.300540924072266,
      "activations/layer10_attention_weight_min": -34.3593635559082,
      "activations/layer11_attention_weight_max": 33.84050750732422,
      "activations/layer11_attention_weight_min": -33.96843719482422,
      "activations/layer12_attention_weight_max": 18.381383895874023,
      "activations/layer12_attention_weight_min": -25.170923233032227,
      "activations/layer13_attention_weight_max": 35.318077087402344,
      "activations/layer13_attention_weight_min": -26.796533584594727,
      "activations/layer14_attention_weight_max": 34.8166618347168,
      "activations/layer14_attention_weight_min": -28.61617660522461,
      "activations/layer15_attention_weight_max": 33.43328094482422,
      "activations/layer15_attention_weight_min": -28.875574111938477,
      "activations/layer16_attention_weight_max": 31.33329200744629,
      "activations/layer16_attention_weight_min": -29.848827362060547,
      "activations/layer17_attention_weight_max": 49.556758880615234,
      "activations/layer17_attention_weight_min": -42.867530822753906,
      "activations/layer18_attention_weight_max": 44.53730392456055,
      "activations/layer18_attention_weight_min": -44.535430908203125,
      "activations/layer19_attention_weight_max": 23.02218246459961,
      "activations/layer19_attention_weight_min": -23.841726303100586,
      "activations/layer1_attention_weight_max": 15.42236042022705,
      "activations/layer1_attention_weight_min": -14.932449340820312,
      "activations/layer20_attention_weight_max": 22.47955894470215,
      "activations/layer20_attention_weight_min": -22.291423797607422,
      "activations/layer21_attention_weight_max": 44.01751708984375,
      "activations/layer21_attention_weight_min": -30.26181983947754,
      "activations/layer22_attention_weight_max": 28.274892807006836,
      "activations/layer22_attention_weight_min": -27.481966018676758,
      "activations/layer23_attention_weight_max": 33.018001556396484,
      "activations/layer23_attention_weight_min": -22.44347381591797,
      "activations/layer2_attention_weight_max": 32.377601623535156,
      "activations/layer2_attention_weight_min": -31.64278793334961,
      "activations/layer3_attention_weight_max": 91.41985321044922,
      "activations/layer3_attention_weight_min": -92.17807006835938,
      "activations/layer4_attention_weight_max": 82.7883071899414,
      "activations/layer4_attention_weight_min": -82.76839447021484,
      "activations/layer5_attention_weight_max": 62.63273620605469,
      "activations/layer5_attention_weight_min": -71.72972106933594,
      "activations/layer6_attention_weight_max": 50.14029312133789,
      "activations/layer6_attention_weight_min": -49.39780807495117,
      "activations/layer7_attention_weight_max": 71.61817932128906,
      "activations/layer7_attention_weight_min": -64.3752670288086,
      "activations/layer8_attention_weight_max": 48.37799835205078,
      "activations/layer8_attention_weight_min": -48.252193450927734,
      "activations/layer9_attention_weight_max": 50.25297164916992,
      "activations/layer9_attention_weight_min": -44.360652923583984,
      "epoch": 21.21,
      "learning_rate": 1.3329924242424241e-05,
      "loss": 2.7129,
      "step": 365100
    },
    {
      "activations/layer0_attention_weight_max": 15.298883438110352,
      "activations/layer0_attention_weight_min": -14.45051383972168,
      "activations/layer10_attention_weight_max": 35.604225158691406,
      "activations/layer10_attention_weight_min": -35.20687484741211,
      "activations/layer11_attention_weight_max": 32.15373992919922,
      "activations/layer11_attention_weight_min": -35.3785514831543,
      "activations/layer12_attention_weight_max": 18.750822067260742,
      "activations/layer12_attention_weight_min": -23.458789825439453,
      "activations/layer13_attention_weight_max": 36.34672927856445,
      "activations/layer13_attention_weight_min": -27.305583953857422,
      "activations/layer14_attention_weight_max": 34.55305480957031,
      "activations/layer14_attention_weight_min": -30.481367111206055,
      "activations/layer15_attention_weight_max": 33.194053649902344,
      "activations/layer15_attention_weight_min": -32.17707061767578,
      "activations/layer16_attention_weight_max": 34.88532257080078,
      "activations/layer16_attention_weight_min": -32.2640495300293,
      "activations/layer17_attention_weight_max": 52.28582763671875,
      "activations/layer17_attention_weight_min": -43.4677619934082,
      "activations/layer18_attention_weight_max": 46.27960968017578,
      "activations/layer18_attention_weight_min": -40.07798767089844,
      "activations/layer19_attention_weight_max": 22.661846160888672,
      "activations/layer19_attention_weight_min": -22.011356353759766,
      "activations/layer1_attention_weight_max": 14.712869644165039,
      "activations/layer1_attention_weight_min": -13.98965072631836,
      "activations/layer20_attention_weight_max": 19.926958084106445,
      "activations/layer20_attention_weight_min": -19.117868423461914,
      "activations/layer21_attention_weight_max": 38.08409881591797,
      "activations/layer21_attention_weight_min": -22.296554565429688,
      "activations/layer22_attention_weight_max": 29.113048553466797,
      "activations/layer22_attention_weight_min": -24.808130264282227,
      "activations/layer23_attention_weight_max": 35.40426254272461,
      "activations/layer23_attention_weight_min": -23.05715560913086,
      "activations/layer2_attention_weight_max": 30.861186981201172,
      "activations/layer2_attention_weight_min": -30.302635192871094,
      "activations/layer3_attention_weight_max": 87.1650161743164,
      "activations/layer3_attention_weight_min": -88.19464874267578,
      "activations/layer4_attention_weight_max": 79.21380615234375,
      "activations/layer4_attention_weight_min": -83.10678100585938,
      "activations/layer5_attention_weight_max": 57.844966888427734,
      "activations/layer5_attention_weight_min": -70.65287017822266,
      "activations/layer6_attention_weight_max": 49.20903778076172,
      "activations/layer6_attention_weight_min": -51.34668731689453,
      "activations/layer7_attention_weight_max": 64.97505950927734,
      "activations/layer7_attention_weight_min": -66.10862731933594,
      "activations/layer8_attention_weight_max": 47.102535247802734,
      "activations/layer8_attention_weight_min": -51.69147872924805,
      "activations/layer9_attention_weight_max": 44.4703254699707,
      "activations/layer9_attention_weight_min": -46.8283805847168,
      "epoch": 21.22,
      "learning_rate": 1.3310984848484848e-05,
      "loss": 2.7093,
      "step": 365150
    },
    {
      "activations/layer0_attention_weight_max": 15.56988525390625,
      "activations/layer0_attention_weight_min": -14.364548683166504,
      "activations/layer10_attention_weight_max": 33.453041076660156,
      "activations/layer10_attention_weight_min": -35.57160949707031,
      "activations/layer11_attention_weight_max": 32.35881805419922,
      "activations/layer11_attention_weight_min": -35.110599517822266,
      "activations/layer12_attention_weight_max": 19.826202392578125,
      "activations/layer12_attention_weight_min": -26.747817993164062,
      "activations/layer13_attention_weight_max": 34.09061813354492,
      "activations/layer13_attention_weight_min": -26.51249885559082,
      "activations/layer14_attention_weight_max": 36.62089920043945,
      "activations/layer14_attention_weight_min": -29.954736709594727,
      "activations/layer15_attention_weight_max": 33.58983612060547,
      "activations/layer15_attention_weight_min": -30.356874465942383,
      "activations/layer16_attention_weight_max": 37.37427520751953,
      "activations/layer16_attention_weight_min": -33.802772521972656,
      "activations/layer17_attention_weight_max": 52.39707565307617,
      "activations/layer17_attention_weight_min": -46.50796890258789,
      "activations/layer18_attention_weight_max": 47.638710021972656,
      "activations/layer18_attention_weight_min": -40.66851043701172,
      "activations/layer19_attention_weight_max": 27.42005729675293,
      "activations/layer19_attention_weight_min": -23.040908813476562,
      "activations/layer1_attention_weight_max": 14.990714073181152,
      "activations/layer1_attention_weight_min": -14.23962116241455,
      "activations/layer20_attention_weight_max": 25.22963523864746,
      "activations/layer20_attention_weight_min": -22.28917694091797,
      "activations/layer21_attention_weight_max": 40.85923385620117,
      "activations/layer21_attention_weight_min": -25.938732147216797,
      "activations/layer22_attention_weight_max": 28.705381393432617,
      "activations/layer22_attention_weight_min": -27.03057098388672,
      "activations/layer23_attention_weight_max": 33.866981506347656,
      "activations/layer23_attention_weight_min": -24.87714195251465,
      "activations/layer2_attention_weight_max": 32.00210952758789,
      "activations/layer2_attention_weight_min": -31.15155029296875,
      "activations/layer3_attention_weight_max": 86.12205505371094,
      "activations/layer3_attention_weight_min": -87.97834014892578,
      "activations/layer4_attention_weight_max": 80.54939270019531,
      "activations/layer4_attention_weight_min": -82.28082275390625,
      "activations/layer5_attention_weight_max": 58.45071029663086,
      "activations/layer5_attention_weight_min": -68.62654876708984,
      "activations/layer6_attention_weight_max": 49.08584976196289,
      "activations/layer6_attention_weight_min": -50.586246490478516,
      "activations/layer7_attention_weight_max": 63.241397857666016,
      "activations/layer7_attention_weight_min": -64.37034606933594,
      "activations/layer8_attention_weight_max": 44.430912017822266,
      "activations/layer8_attention_weight_min": -50.37800216674805,
      "activations/layer9_attention_weight_max": 44.00482940673828,
      "activations/layer9_attention_weight_min": -47.88555908203125,
      "epoch": 21.22,
      "learning_rate": 1.3292045454545454e-05,
      "loss": 2.7212,
      "step": 365200
    },
    {
      "activations/layer0_attention_weight_max": 15.411653518676758,
      "activations/layer0_attention_weight_min": -14.327765464782715,
      "activations/layer10_attention_weight_max": 36.49346160888672,
      "activations/layer10_attention_weight_min": -35.60370635986328,
      "activations/layer11_attention_weight_max": 31.696056365966797,
      "activations/layer11_attention_weight_min": -34.2818603515625,
      "activations/layer12_attention_weight_max": 19.099727630615234,
      "activations/layer12_attention_weight_min": -27.35735511779785,
      "activations/layer13_attention_weight_max": 32.43303298950195,
      "activations/layer13_attention_weight_min": -25.92838478088379,
      "activations/layer14_attention_weight_max": 34.15271759033203,
      "activations/layer14_attention_weight_min": -31.708393096923828,
      "activations/layer15_attention_weight_max": 30.93301773071289,
      "activations/layer15_attention_weight_min": -30.44115447998047,
      "activations/layer16_attention_weight_max": 32.06440734863281,
      "activations/layer16_attention_weight_min": -31.564327239990234,
      "activations/layer17_attention_weight_max": 47.11623001098633,
      "activations/layer17_attention_weight_min": -43.74765396118164,
      "activations/layer18_attention_weight_max": 44.51355743408203,
      "activations/layer18_attention_weight_min": -38.6959114074707,
      "activations/layer19_attention_weight_max": 26.58208656311035,
      "activations/layer19_attention_weight_min": -23.415992736816406,
      "activations/layer1_attention_weight_max": 14.611546516418457,
      "activations/layer1_attention_weight_min": -14.335430145263672,
      "activations/layer20_attention_weight_max": 25.777660369873047,
      "activations/layer20_attention_weight_min": -22.98153305053711,
      "activations/layer21_attention_weight_max": 40.475929260253906,
      "activations/layer21_attention_weight_min": -24.8253173828125,
      "activations/layer22_attention_weight_max": 29.82088851928711,
      "activations/layer22_attention_weight_min": -27.19783592224121,
      "activations/layer23_attention_weight_max": 36.5868034362793,
      "activations/layer23_attention_weight_min": -24.93572235107422,
      "activations/layer2_attention_weight_max": 32.474769592285156,
      "activations/layer2_attention_weight_min": -30.087202072143555,
      "activations/layer3_attention_weight_max": 88.47713470458984,
      "activations/layer3_attention_weight_min": -91.90821838378906,
      "activations/layer4_attention_weight_max": 87.24779510498047,
      "activations/layer4_attention_weight_min": -84.44927215576172,
      "activations/layer5_attention_weight_max": 60.16795349121094,
      "activations/layer5_attention_weight_min": -74.79315948486328,
      "activations/layer6_attention_weight_max": 50.693336486816406,
      "activations/layer6_attention_weight_min": -51.612483978271484,
      "activations/layer7_attention_weight_max": 67.30964660644531,
      "activations/layer7_attention_weight_min": -66.99659729003906,
      "activations/layer8_attention_weight_max": 48.16028594970703,
      "activations/layer8_attention_weight_min": -52.74884796142578,
      "activations/layer9_attention_weight_max": 42.46445846557617,
      "activations/layer9_attention_weight_min": -45.91001892089844,
      "epoch": 21.22,
      "learning_rate": 1.327310606060606e-05,
      "loss": 2.7171,
      "step": 365250
    },
    {
      "activations/layer0_attention_weight_max": 15.858787536621094,
      "activations/layer0_attention_weight_min": -14.26202392578125,
      "activations/layer10_attention_weight_max": 38.081974029541016,
      "activations/layer10_attention_weight_min": -36.32646560668945,
      "activations/layer11_attention_weight_max": 37.33905792236328,
      "activations/layer11_attention_weight_min": -36.00157928466797,
      "activations/layer12_attention_weight_max": 19.688251495361328,
      "activations/layer12_attention_weight_min": -26.53516960144043,
      "activations/layer13_attention_weight_max": 35.211360931396484,
      "activations/layer13_attention_weight_min": -26.18437385559082,
      "activations/layer14_attention_weight_max": 37.57957077026367,
      "activations/layer14_attention_weight_min": -29.55802345275879,
      "activations/layer15_attention_weight_max": 37.044944763183594,
      "activations/layer15_attention_weight_min": -30.399566650390625,
      "activations/layer16_attention_weight_max": 34.640438079833984,
      "activations/layer16_attention_weight_min": -31.027725219726562,
      "activations/layer17_attention_weight_max": 54.96052551269531,
      "activations/layer17_attention_weight_min": -44.617977142333984,
      "activations/layer18_attention_weight_max": 51.46915054321289,
      "activations/layer18_attention_weight_min": -39.61305618286133,
      "activations/layer19_attention_weight_max": 25.724050521850586,
      "activations/layer19_attention_weight_min": -22.814359664916992,
      "activations/layer1_attention_weight_max": 16.72900390625,
      "activations/layer1_attention_weight_min": -14.892534255981445,
      "activations/layer20_attention_weight_max": 22.41746711730957,
      "activations/layer20_attention_weight_min": -22.092025756835938,
      "activations/layer21_attention_weight_max": 36.95692825317383,
      "activations/layer21_attention_weight_min": -22.09599494934082,
      "activations/layer22_attention_weight_max": 28.83854103088379,
      "activations/layer22_attention_weight_min": -24.764617919921875,
      "activations/layer23_attention_weight_max": 37.251773834228516,
      "activations/layer23_attention_weight_min": -23.690250396728516,
      "activations/layer2_attention_weight_max": 30.536075592041016,
      "activations/layer2_attention_weight_min": -29.146024703979492,
      "activations/layer3_attention_weight_max": 87.92938995361328,
      "activations/layer3_attention_weight_min": -88.62288665771484,
      "activations/layer4_attention_weight_max": 88.3589859008789,
      "activations/layer4_attention_weight_min": -83.8680191040039,
      "activations/layer5_attention_weight_max": 64.19357299804688,
      "activations/layer5_attention_weight_min": -70.07171630859375,
      "activations/layer6_attention_weight_max": 49.32052993774414,
      "activations/layer6_attention_weight_min": -49.34364318847656,
      "activations/layer7_attention_weight_max": 68.6715087890625,
      "activations/layer7_attention_weight_min": -66.34963989257812,
      "activations/layer8_attention_weight_max": 46.50852584838867,
      "activations/layer8_attention_weight_min": -49.905330657958984,
      "activations/layer9_attention_weight_max": 49.56542205810547,
      "activations/layer9_attention_weight_min": -44.698524475097656,
      "epoch": 21.23,
      "learning_rate": 1.3254166666666666e-05,
      "loss": 2.7181,
      "step": 365300
    },
    {
      "activations/layer0_attention_weight_max": 15.92426872253418,
      "activations/layer0_attention_weight_min": -14.62862491607666,
      "activations/layer10_attention_weight_max": 34.78531265258789,
      "activations/layer10_attention_weight_min": -36.95463943481445,
      "activations/layer11_attention_weight_max": 32.35779571533203,
      "activations/layer11_attention_weight_min": -36.300750732421875,
      "activations/layer12_attention_weight_max": 18.358095169067383,
      "activations/layer12_attention_weight_min": -24.903499603271484,
      "activations/layer13_attention_weight_max": 38.324249267578125,
      "activations/layer13_attention_weight_min": -28.171459197998047,
      "activations/layer14_attention_weight_max": 32.908782958984375,
      "activations/layer14_attention_weight_min": -31.267423629760742,
      "activations/layer15_attention_weight_max": 31.047361373901367,
      "activations/layer15_attention_weight_min": -31.75469398498535,
      "activations/layer16_attention_weight_max": 35.5283317565918,
      "activations/layer16_attention_weight_min": -33.805206298828125,
      "activations/layer17_attention_weight_max": 49.64518356323242,
      "activations/layer17_attention_weight_min": -48.072608947753906,
      "activations/layer18_attention_weight_max": 44.80374526977539,
      "activations/layer18_attention_weight_min": -42.97830581665039,
      "activations/layer19_attention_weight_max": 23.661415100097656,
      "activations/layer19_attention_weight_min": -22.738162994384766,
      "activations/layer1_attention_weight_max": 15.57540225982666,
      "activations/layer1_attention_weight_min": -14.411455154418945,
      "activations/layer20_attention_weight_max": 22.274450302124023,
      "activations/layer20_attention_weight_min": -21.3034610748291,
      "activations/layer21_attention_weight_max": 39.18031692504883,
      "activations/layer21_attention_weight_min": -24.17947769165039,
      "activations/layer22_attention_weight_max": 28.59600830078125,
      "activations/layer22_attention_weight_min": -26.47909927368164,
      "activations/layer23_attention_weight_max": 37.22364807128906,
      "activations/layer23_attention_weight_min": -23.306671142578125,
      "activations/layer2_attention_weight_max": 30.5030517578125,
      "activations/layer2_attention_weight_min": -29.805831909179688,
      "activations/layer3_attention_weight_max": 85.54828643798828,
      "activations/layer3_attention_weight_min": -87.29450988769531,
      "activations/layer4_attention_weight_max": 82.01798248291016,
      "activations/layer4_attention_weight_min": -82.4327621459961,
      "activations/layer5_attention_weight_max": 61.22776412963867,
      "activations/layer5_attention_weight_min": -69.4771499633789,
      "activations/layer6_attention_weight_max": 47.919898986816406,
      "activations/layer6_attention_weight_min": -48.009361267089844,
      "activations/layer7_attention_weight_max": 69.02667236328125,
      "activations/layer7_attention_weight_min": -64.06038665771484,
      "activations/layer8_attention_weight_max": 44.89116668701172,
      "activations/layer8_attention_weight_min": -45.072792053222656,
      "activations/layer9_attention_weight_max": 43.385765075683594,
      "activations/layer9_attention_weight_min": -44.23929977416992,
      "epoch": 21.23,
      "learning_rate": 1.3235227272727272e-05,
      "loss": 2.7041,
      "step": 365350
    },
    {
      "activations/layer0_attention_weight_max": 15.406147956848145,
      "activations/layer0_attention_weight_min": -14.591121673583984,
      "activations/layer10_attention_weight_max": 34.92298889160156,
      "activations/layer10_attention_weight_min": -34.8046875,
      "activations/layer11_attention_weight_max": 32.207088470458984,
      "activations/layer11_attention_weight_min": -36.619110107421875,
      "activations/layer12_attention_weight_max": 19.10941505432129,
      "activations/layer12_attention_weight_min": -23.85474967956543,
      "activations/layer13_attention_weight_max": 33.48889923095703,
      "activations/layer13_attention_weight_min": -27.634685516357422,
      "activations/layer14_attention_weight_max": 36.81790542602539,
      "activations/layer14_attention_weight_min": -30.424348831176758,
      "activations/layer15_attention_weight_max": 34.217987060546875,
      "activations/layer15_attention_weight_min": -30.90178871154785,
      "activations/layer16_attention_weight_max": 35.29281997680664,
      "activations/layer16_attention_weight_min": -31.88066291809082,
      "activations/layer17_attention_weight_max": 51.36333084106445,
      "activations/layer17_attention_weight_min": -45.30914306640625,
      "activations/layer18_attention_weight_max": 47.45751953125,
      "activations/layer18_attention_weight_min": -40.059898376464844,
      "activations/layer19_attention_weight_max": 24.676815032958984,
      "activations/layer19_attention_weight_min": -22.18316650390625,
      "activations/layer1_attention_weight_max": 16.61132049560547,
      "activations/layer1_attention_weight_min": -14.10229206085205,
      "activations/layer20_attention_weight_max": 22.45875358581543,
      "activations/layer20_attention_weight_min": -20.944854736328125,
      "activations/layer21_attention_weight_max": 38.69739532470703,
      "activations/layer21_attention_weight_min": -22.98822021484375,
      "activations/layer22_attention_weight_max": 31.77623176574707,
      "activations/layer22_attention_weight_min": -24.869504928588867,
      "activations/layer23_attention_weight_max": 42.55741882324219,
      "activations/layer23_attention_weight_min": -24.97562026977539,
      "activations/layer2_attention_weight_max": 33.08592224121094,
      "activations/layer2_attention_weight_min": -30.211406707763672,
      "activations/layer3_attention_weight_max": 91.25341796875,
      "activations/layer3_attention_weight_min": -88.45886993408203,
      "activations/layer4_attention_weight_max": 84.70816040039062,
      "activations/layer4_attention_weight_min": -87.3455581665039,
      "activations/layer5_attention_weight_max": 62.946861267089844,
      "activations/layer5_attention_weight_min": -73.57272338867188,
      "activations/layer6_attention_weight_max": 49.811241149902344,
      "activations/layer6_attention_weight_min": -51.04378890991211,
      "activations/layer7_attention_weight_max": 67.75335693359375,
      "activations/layer7_attention_weight_min": -67.37345123291016,
      "activations/layer8_attention_weight_max": 49.679012298583984,
      "activations/layer8_attention_weight_min": -55.7310676574707,
      "activations/layer9_attention_weight_max": 43.7911262512207,
      "activations/layer9_attention_weight_min": -46.727840423583984,
      "epoch": 21.23,
      "learning_rate": 1.3216287878787877e-05,
      "loss": 2.7126,
      "step": 365400
    },
    {
      "activations/layer0_attention_weight_max": 15.221640586853027,
      "activations/layer0_attention_weight_min": -14.465078353881836,
      "activations/layer10_attention_weight_max": 36.01559829711914,
      "activations/layer10_attention_weight_min": -37.1951904296875,
      "activations/layer11_attention_weight_max": 33.48552322387695,
      "activations/layer11_attention_weight_min": -36.17416000366211,
      "activations/layer12_attention_weight_max": 19.631195068359375,
      "activations/layer12_attention_weight_min": -22.589492797851562,
      "activations/layer13_attention_weight_max": 34.570404052734375,
      "activations/layer13_attention_weight_min": -28.715402603149414,
      "activations/layer14_attention_weight_max": 34.46397018432617,
      "activations/layer14_attention_weight_min": -32.79790496826172,
      "activations/layer15_attention_weight_max": 36.23849105834961,
      "activations/layer15_attention_weight_min": -34.714542388916016,
      "activations/layer16_attention_weight_max": 35.21342468261719,
      "activations/layer16_attention_weight_min": -34.51455307006836,
      "activations/layer17_attention_weight_max": 53.43932342529297,
      "activations/layer17_attention_weight_min": -51.68776321411133,
      "activations/layer18_attention_weight_max": 47.34663391113281,
      "activations/layer18_attention_weight_min": -50.63412857055664,
      "activations/layer19_attention_weight_max": 25.606006622314453,
      "activations/layer19_attention_weight_min": -28.484189987182617,
      "activations/layer1_attention_weight_max": 15.02925968170166,
      "activations/layer1_attention_weight_min": -13.973151206970215,
      "activations/layer20_attention_weight_max": 24.5828800201416,
      "activations/layer20_attention_weight_min": -26.709566116333008,
      "activations/layer21_attention_weight_max": 35.882320404052734,
      "activations/layer21_attention_weight_min": -27.9886474609375,
      "activations/layer22_attention_weight_max": 28.50486946105957,
      "activations/layer22_attention_weight_min": -28.103290557861328,
      "activations/layer23_attention_weight_max": 35.30200958251953,
      "activations/layer23_attention_weight_min": -25.20850944519043,
      "activations/layer2_attention_weight_max": 32.17498016357422,
      "activations/layer2_attention_weight_min": -31.257673263549805,
      "activations/layer3_attention_weight_max": 85.976318359375,
      "activations/layer3_attention_weight_min": -86.78636169433594,
      "activations/layer4_attention_weight_max": 79.68019104003906,
      "activations/layer4_attention_weight_min": -80.35466766357422,
      "activations/layer5_attention_weight_max": 58.27180099487305,
      "activations/layer5_attention_weight_min": -70.68482208251953,
      "activations/layer6_attention_weight_max": 46.83415985107422,
      "activations/layer6_attention_weight_min": -49.29951477050781,
      "activations/layer7_attention_weight_max": 63.74855422973633,
      "activations/layer7_attention_weight_min": -68.84718322753906,
      "activations/layer8_attention_weight_max": 46.16762161254883,
      "activations/layer8_attention_weight_min": -52.2609748840332,
      "activations/layer9_attention_weight_max": 46.77531051635742,
      "activations/layer9_attention_weight_min": -48.104454040527344,
      "epoch": 21.23,
      "learning_rate": 1.3197348484848484e-05,
      "loss": 2.7006,
      "step": 365450
    },
    {
      "activations/layer0_attention_weight_max": 14.723396301269531,
      "activations/layer0_attention_weight_min": -14.818729400634766,
      "activations/layer10_attention_weight_max": 36.39674758911133,
      "activations/layer10_attention_weight_min": -37.06489944458008,
      "activations/layer11_attention_weight_max": 34.096282958984375,
      "activations/layer11_attention_weight_min": -37.21385192871094,
      "activations/layer12_attention_weight_max": 19.203876495361328,
      "activations/layer12_attention_weight_min": -24.67348861694336,
      "activations/layer13_attention_weight_max": 35.813175201416016,
      "activations/layer13_attention_weight_min": -27.941612243652344,
      "activations/layer14_attention_weight_max": 36.40488815307617,
      "activations/layer14_attention_weight_min": -30.068368911743164,
      "activations/layer15_attention_weight_max": 36.194026947021484,
      "activations/layer15_attention_weight_min": -30.232208251953125,
      "activations/layer16_attention_weight_max": 36.1855354309082,
      "activations/layer16_attention_weight_min": -33.27215576171875,
      "activations/layer17_attention_weight_max": 55.63235092163086,
      "activations/layer17_attention_weight_min": -47.484493255615234,
      "activations/layer18_attention_weight_max": 51.13478469848633,
      "activations/layer18_attention_weight_min": -44.01518630981445,
      "activations/layer19_attention_weight_max": 30.750213623046875,
      "activations/layer19_attention_weight_min": -24.461206436157227,
      "activations/layer1_attention_weight_max": 15.4308443069458,
      "activations/layer1_attention_weight_min": -14.667808532714844,
      "activations/layer20_attention_weight_max": 25.36296272277832,
      "activations/layer20_attention_weight_min": -22.757144927978516,
      "activations/layer21_attention_weight_max": 40.45659255981445,
      "activations/layer21_attention_weight_min": -22.707584381103516,
      "activations/layer22_attention_weight_max": 33.9431037902832,
      "activations/layer22_attention_weight_min": -24.627107620239258,
      "activations/layer23_attention_weight_max": 40.14938735961914,
      "activations/layer23_attention_weight_min": -23.693572998046875,
      "activations/layer2_attention_weight_max": 32.48298645019531,
      "activations/layer2_attention_weight_min": -30.18055534362793,
      "activations/layer3_attention_weight_max": 90.1881103515625,
      "activations/layer3_attention_weight_min": -89.06716918945312,
      "activations/layer4_attention_weight_max": 84.74388122558594,
      "activations/layer4_attention_weight_min": -83.57857513427734,
      "activations/layer5_attention_weight_max": 63.29712677001953,
      "activations/layer5_attention_weight_min": -76.39607238769531,
      "activations/layer6_attention_weight_max": 50.08390426635742,
      "activations/layer6_attention_weight_min": -52.25711441040039,
      "activations/layer7_attention_weight_max": 70.01030731201172,
      "activations/layer7_attention_weight_min": -66.0965347290039,
      "activations/layer8_attention_weight_max": 49.154541015625,
      "activations/layer8_attention_weight_min": -54.75680923461914,
      "activations/layer9_attention_weight_max": 45.5490608215332,
      "activations/layer9_attention_weight_min": -45.08021545410156,
      "epoch": 21.24,
      "learning_rate": 1.3178409090909089e-05,
      "loss": 2.7089,
      "step": 365500
    },
    {
      "activations/layer0_attention_weight_max": 18.377193450927734,
      "activations/layer0_attention_weight_min": -14.934706687927246,
      "activations/layer10_attention_weight_max": 38.387306213378906,
      "activations/layer10_attention_weight_min": -39.0390625,
      "activations/layer11_attention_weight_max": 35.851863861083984,
      "activations/layer11_attention_weight_min": -37.38603210449219,
      "activations/layer12_attention_weight_max": 18.29677391052246,
      "activations/layer12_attention_weight_min": -25.242576599121094,
      "activations/layer13_attention_weight_max": 33.596981048583984,
      "activations/layer13_attention_weight_min": -26.943761825561523,
      "activations/layer14_attention_weight_max": 36.843318939208984,
      "activations/layer14_attention_weight_min": -30.94832992553711,
      "activations/layer15_attention_weight_max": 34.79954147338867,
      "activations/layer15_attention_weight_min": -33.2696533203125,
      "activations/layer16_attention_weight_max": 39.65206527709961,
      "activations/layer16_attention_weight_min": -35.677520751953125,
      "activations/layer17_attention_weight_max": 54.55034255981445,
      "activations/layer17_attention_weight_min": -46.49915313720703,
      "activations/layer18_attention_weight_max": 50.991207122802734,
      "activations/layer18_attention_weight_min": -40.85386276245117,
      "activations/layer19_attention_weight_max": 26.14971923828125,
      "activations/layer19_attention_weight_min": -23.067251205444336,
      "activations/layer1_attention_weight_max": 15.378840446472168,
      "activations/layer1_attention_weight_min": -16.107805252075195,
      "activations/layer20_attention_weight_max": 23.837276458740234,
      "activations/layer20_attention_weight_min": -20.9848575592041,
      "activations/layer21_attention_weight_max": 34.97859573364258,
      "activations/layer21_attention_weight_min": -24.094894409179688,
      "activations/layer22_attention_weight_max": 30.479381561279297,
      "activations/layer22_attention_weight_min": -24.96812629699707,
      "activations/layer23_attention_weight_max": 38.75901794433594,
      "activations/layer23_attention_weight_min": -26.29851531982422,
      "activations/layer2_attention_weight_max": 33.48505401611328,
      "activations/layer2_attention_weight_min": -31.477375030517578,
      "activations/layer3_attention_weight_max": 86.73250579833984,
      "activations/layer3_attention_weight_min": -91.66376495361328,
      "activations/layer4_attention_weight_max": 82.95343017578125,
      "activations/layer4_attention_weight_min": -84.46387481689453,
      "activations/layer5_attention_weight_max": 62.70571517944336,
      "activations/layer5_attention_weight_min": -73.15278625488281,
      "activations/layer6_attention_weight_max": 50.787513732910156,
      "activations/layer6_attention_weight_min": -51.47101593017578,
      "activations/layer7_attention_weight_max": 67.8984603881836,
      "activations/layer7_attention_weight_min": -71.50202178955078,
      "activations/layer8_attention_weight_max": 49.988555908203125,
      "activations/layer8_attention_weight_min": -54.362430572509766,
      "activations/layer9_attention_weight_max": 46.44802474975586,
      "activations/layer9_attention_weight_min": -47.019676208496094,
      "epoch": 21.24,
      "learning_rate": 1.3159469696969695e-05,
      "loss": 2.7129,
      "step": 365550
    },
    {
      "activations/layer0_attention_weight_max": 15.066366195678711,
      "activations/layer0_attention_weight_min": -14.599409103393555,
      "activations/layer10_attention_weight_max": 37.24713134765625,
      "activations/layer10_attention_weight_min": -35.272762298583984,
      "activations/layer11_attention_weight_max": 36.98270797729492,
      "activations/layer11_attention_weight_min": -35.55461120605469,
      "activations/layer12_attention_weight_max": 18.580190658569336,
      "activations/layer12_attention_weight_min": -28.68964385986328,
      "activations/layer13_attention_weight_max": 36.890411376953125,
      "activations/layer13_attention_weight_min": -28.52031898498535,
      "activations/layer14_attention_weight_max": 36.15753173828125,
      "activations/layer14_attention_weight_min": -30.350698471069336,
      "activations/layer15_attention_weight_max": 32.879398345947266,
      "activations/layer15_attention_weight_min": -31.678136825561523,
      "activations/layer16_attention_weight_max": 33.92928695678711,
      "activations/layer16_attention_weight_min": -32.516700744628906,
      "activations/layer17_attention_weight_max": 49.7830696105957,
      "activations/layer17_attention_weight_min": -47.2518196105957,
      "activations/layer18_attention_weight_max": 47.86734390258789,
      "activations/layer18_attention_weight_min": -41.694236755371094,
      "activations/layer19_attention_weight_max": 23.6273136138916,
      "activations/layer19_attention_weight_min": -24.047773361206055,
      "activations/layer1_attention_weight_max": 14.846832275390625,
      "activations/layer1_attention_weight_min": -13.591343879699707,
      "activations/layer20_attention_weight_max": 23.04466438293457,
      "activations/layer20_attention_weight_min": -25.475255966186523,
      "activations/layer21_attention_weight_max": 42.94028854370117,
      "activations/layer21_attention_weight_min": -25.614364624023438,
      "activations/layer22_attention_weight_max": 29.380191802978516,
      "activations/layer22_attention_weight_min": -27.545944213867188,
      "activations/layer23_attention_weight_max": 37.85710144042969,
      "activations/layer23_attention_weight_min": -24.069854736328125,
      "activations/layer2_attention_weight_max": 31.5282039642334,
      "activations/layer2_attention_weight_min": -30.30952262878418,
      "activations/layer3_attention_weight_max": 87.0150146484375,
      "activations/layer3_attention_weight_min": -93.28422546386719,
      "activations/layer4_attention_weight_max": 79.98908996582031,
      "activations/layer4_attention_weight_min": -81.69910430908203,
      "activations/layer5_attention_weight_max": 59.877464294433594,
      "activations/layer5_attention_weight_min": -67.32716369628906,
      "activations/layer6_attention_weight_max": 47.36768341064453,
      "activations/layer6_attention_weight_min": -49.713130950927734,
      "activations/layer7_attention_weight_max": 64.53402709960938,
      "activations/layer7_attention_weight_min": -66.26884460449219,
      "activations/layer8_attention_weight_max": 44.84453582763672,
      "activations/layer8_attention_weight_min": -47.924034118652344,
      "activations/layer9_attention_weight_max": 43.18270492553711,
      "activations/layer9_attention_weight_min": -46.34162521362305,
      "epoch": 21.24,
      "learning_rate": 1.3140530303030302e-05,
      "loss": 2.716,
      "step": 365600
    },
    {
      "activations/layer0_attention_weight_max": 15.379380226135254,
      "activations/layer0_attention_weight_min": -14.693744659423828,
      "activations/layer10_attention_weight_max": 38.439544677734375,
      "activations/layer10_attention_weight_min": -36.324195861816406,
      "activations/layer11_attention_weight_max": 34.33909606933594,
      "activations/layer11_attention_weight_min": -34.17312240600586,
      "activations/layer12_attention_weight_max": 20.321590423583984,
      "activations/layer12_attention_weight_min": -22.424638748168945,
      "activations/layer13_attention_weight_max": 37.49609375,
      "activations/layer13_attention_weight_min": -28.071414947509766,
      "activations/layer14_attention_weight_max": 37.449703216552734,
      "activations/layer14_attention_weight_min": -30.217885971069336,
      "activations/layer15_attention_weight_max": 37.48737716674805,
      "activations/layer15_attention_weight_min": -30.023029327392578,
      "activations/layer16_attention_weight_max": 36.34878158569336,
      "activations/layer16_attention_weight_min": -34.4838981628418,
      "activations/layer17_attention_weight_max": 55.99518966674805,
      "activations/layer17_attention_weight_min": -47.74165725708008,
      "activations/layer18_attention_weight_max": 55.66032409667969,
      "activations/layer18_attention_weight_min": -44.61001968383789,
      "activations/layer19_attention_weight_max": 27.765424728393555,
      "activations/layer19_attention_weight_min": -24.28346824645996,
      "activations/layer1_attention_weight_max": 16.303443908691406,
      "activations/layer1_attention_weight_min": -14.589688301086426,
      "activations/layer20_attention_weight_max": 22.598787307739258,
      "activations/layer20_attention_weight_min": -21.680936813354492,
      "activations/layer21_attention_weight_max": 42.00039291381836,
      "activations/layer21_attention_weight_min": -22.78159523010254,
      "activations/layer22_attention_weight_max": 30.0627498626709,
      "activations/layer22_attention_weight_min": -24.49904441833496,
      "activations/layer23_attention_weight_max": 40.043067932128906,
      "activations/layer23_attention_weight_min": -23.19172477722168,
      "activations/layer2_attention_weight_max": 32.1849365234375,
      "activations/layer2_attention_weight_min": -30.44611930847168,
      "activations/layer3_attention_weight_max": 83.16488647460938,
      "activations/layer3_attention_weight_min": -86.56163024902344,
      "activations/layer4_attention_weight_max": 82.34380340576172,
      "activations/layer4_attention_weight_min": -83.44307708740234,
      "activations/layer5_attention_weight_max": 60.738037109375,
      "activations/layer5_attention_weight_min": -69.97885131835938,
      "activations/layer6_attention_weight_max": 48.47213363647461,
      "activations/layer6_attention_weight_min": -50.14031219482422,
      "activations/layer7_attention_weight_max": 66.19808959960938,
      "activations/layer7_attention_weight_min": -66.3360366821289,
      "activations/layer8_attention_weight_max": 49.99205017089844,
      "activations/layer8_attention_weight_min": -51.4362678527832,
      "activations/layer9_attention_weight_max": 44.877262115478516,
      "activations/layer9_attention_weight_min": -47.787471771240234,
      "epoch": 21.25,
      "learning_rate": 1.3121590909090907e-05,
      "loss": 2.7103,
      "step": 365650
    },
    {
      "activations/layer0_attention_weight_max": 16.155826568603516,
      "activations/layer0_attention_weight_min": -14.621426582336426,
      "activations/layer10_attention_weight_max": 36.40888977050781,
      "activations/layer10_attention_weight_min": -36.73323440551758,
      "activations/layer11_attention_weight_max": 32.90789794921875,
      "activations/layer11_attention_weight_min": -33.96343994140625,
      "activations/layer12_attention_weight_max": 21.773319244384766,
      "activations/layer12_attention_weight_min": -26.967435836791992,
      "activations/layer13_attention_weight_max": 41.98174285888672,
      "activations/layer13_attention_weight_min": -28.327367782592773,
      "activations/layer14_attention_weight_max": 39.44007110595703,
      "activations/layer14_attention_weight_min": -31.2657413482666,
      "activations/layer15_attention_weight_max": 36.161712646484375,
      "activations/layer15_attention_weight_min": -31.48763084411621,
      "activations/layer16_attention_weight_max": 37.14704132080078,
      "activations/layer16_attention_weight_min": -32.09389114379883,
      "activations/layer17_attention_weight_max": 52.237083435058594,
      "activations/layer17_attention_weight_min": -45.32608413696289,
      "activations/layer18_attention_weight_max": 54.64980697631836,
      "activations/layer18_attention_weight_min": -43.640174865722656,
      "activations/layer19_attention_weight_max": 28.102127075195312,
      "activations/layer19_attention_weight_min": -25.010276794433594,
      "activations/layer1_attention_weight_max": 15.412606239318848,
      "activations/layer1_attention_weight_min": -13.934494018554688,
      "activations/layer20_attention_weight_max": 28.373815536499023,
      "activations/layer20_attention_weight_min": -23.72215461730957,
      "activations/layer21_attention_weight_max": 45.06083297729492,
      "activations/layer21_attention_weight_min": -22.29130744934082,
      "activations/layer22_attention_weight_max": 32.78904342651367,
      "activations/layer22_attention_weight_min": -26.977996826171875,
      "activations/layer23_attention_weight_max": 41.433738708496094,
      "activations/layer23_attention_weight_min": -25.095327377319336,
      "activations/layer2_attention_weight_max": 30.366981506347656,
      "activations/layer2_attention_weight_min": -30.874866485595703,
      "activations/layer3_attention_weight_max": 88.04914093017578,
      "activations/layer3_attention_weight_min": -89.40321350097656,
      "activations/layer4_attention_weight_max": 82.72477722167969,
      "activations/layer4_attention_weight_min": -83.36434936523438,
      "activations/layer5_attention_weight_max": 60.506309509277344,
      "activations/layer5_attention_weight_min": -67.26106262207031,
      "activations/layer6_attention_weight_max": 48.7977409362793,
      "activations/layer6_attention_weight_min": -52.062931060791016,
      "activations/layer7_attention_weight_max": 69.1689224243164,
      "activations/layer7_attention_weight_min": -65.1119155883789,
      "activations/layer8_attention_weight_max": 48.60673522949219,
      "activations/layer8_attention_weight_min": -52.2899284362793,
      "activations/layer9_attention_weight_max": 49.17599105834961,
      "activations/layer9_attention_weight_min": -47.250755310058594,
      "epoch": 21.25,
      "learning_rate": 1.3102651515151513e-05,
      "loss": 2.7034,
      "step": 365700
    },
    {
      "activations/layer0_attention_weight_max": 15.487462997436523,
      "activations/layer0_attention_weight_min": -14.774456024169922,
      "activations/layer10_attention_weight_max": 46.52944564819336,
      "activations/layer10_attention_weight_min": -44.577754974365234,
      "activations/layer11_attention_weight_max": 44.6688232421875,
      "activations/layer11_attention_weight_min": -40.362403869628906,
      "activations/layer12_attention_weight_max": 20.654905319213867,
      "activations/layer12_attention_weight_min": -25.783039093017578,
      "activations/layer13_attention_weight_max": 45.53604507446289,
      "activations/layer13_attention_weight_min": -30.019046783447266,
      "activations/layer14_attention_weight_max": 40.27263259887695,
      "activations/layer14_attention_weight_min": -32.07596969604492,
      "activations/layer15_attention_weight_max": 40.21586608886719,
      "activations/layer15_attention_weight_min": -31.265756607055664,
      "activations/layer16_attention_weight_max": 37.17035675048828,
      "activations/layer16_attention_weight_min": -33.745140075683594,
      "activations/layer17_attention_weight_max": 56.64961624145508,
      "activations/layer17_attention_weight_min": -52.340797424316406,
      "activations/layer18_attention_weight_max": 48.98333740234375,
      "activations/layer18_attention_weight_min": -45.75020980834961,
      "activations/layer19_attention_weight_max": 25.32086944580078,
      "activations/layer19_attention_weight_min": -23.290973663330078,
      "activations/layer1_attention_weight_max": 16.478679656982422,
      "activations/layer1_attention_weight_min": -15.3810396194458,
      "activations/layer20_attention_weight_max": 22.2773494720459,
      "activations/layer20_attention_weight_min": -20.450603485107422,
      "activations/layer21_attention_weight_max": 35.501617431640625,
      "activations/layer21_attention_weight_min": -26.078033447265625,
      "activations/layer22_attention_weight_max": 31.32267189025879,
      "activations/layer22_attention_weight_min": -25.1772518157959,
      "activations/layer23_attention_weight_max": 35.010494232177734,
      "activations/layer23_attention_weight_min": -22.1993465423584,
      "activations/layer2_attention_weight_max": 31.090023040771484,
      "activations/layer2_attention_weight_min": -30.151386260986328,
      "activations/layer3_attention_weight_max": 88.25053405761719,
      "activations/layer3_attention_weight_min": -90.67870330810547,
      "activations/layer4_attention_weight_max": 87.14300537109375,
      "activations/layer4_attention_weight_min": -85.44983673095703,
      "activations/layer5_attention_weight_max": 70.49981689453125,
      "activations/layer5_attention_weight_min": -72.61984252929688,
      "activations/layer6_attention_weight_max": 50.26723861694336,
      "activations/layer6_attention_weight_min": -50.29152297973633,
      "activations/layer7_attention_weight_max": 85.28620147705078,
      "activations/layer7_attention_weight_min": -72.7231216430664,
      "activations/layer8_attention_weight_max": 60.19400405883789,
      "activations/layer8_attention_weight_min": -53.85165786743164,
      "activations/layer9_attention_weight_max": 65.03445434570312,
      "activations/layer9_attention_weight_min": -55.632476806640625,
      "epoch": 21.25,
      "learning_rate": 1.308371212121212e-05,
      "loss": 2.7149,
      "step": 365750
    },
    {
      "activations/layer0_attention_weight_max": 15.79000186920166,
      "activations/layer0_attention_weight_min": -14.378545761108398,
      "activations/layer10_attention_weight_max": 36.602840423583984,
      "activations/layer10_attention_weight_min": -34.62672424316406,
      "activations/layer11_attention_weight_max": 32.292057037353516,
      "activations/layer11_attention_weight_min": -35.503536224365234,
      "activations/layer12_attention_weight_max": 18.633682250976562,
      "activations/layer12_attention_weight_min": -23.456586837768555,
      "activations/layer13_attention_weight_max": 30.641572952270508,
      "activations/layer13_attention_weight_min": -25.837446212768555,
      "activations/layer14_attention_weight_max": 31.28520965576172,
      "activations/layer14_attention_weight_min": -29.055713653564453,
      "activations/layer15_attention_weight_max": 30.476953506469727,
      "activations/layer15_attention_weight_min": -28.244173049926758,
      "activations/layer16_attention_weight_max": 29.5695743560791,
      "activations/layer16_attention_weight_min": -29.647363662719727,
      "activations/layer17_attention_weight_max": 44.37553787231445,
      "activations/layer17_attention_weight_min": -42.69234085083008,
      "activations/layer18_attention_weight_max": 42.82902908325195,
      "activations/layer18_attention_weight_min": -36.97585678100586,
      "activations/layer19_attention_weight_max": 21.484418869018555,
      "activations/layer19_attention_weight_min": -20.91944694519043,
      "activations/layer1_attention_weight_max": 15.146025657653809,
      "activations/layer1_attention_weight_min": -13.929740905761719,
      "activations/layer20_attention_weight_max": 20.416759490966797,
      "activations/layer20_attention_weight_min": -20.608787536621094,
      "activations/layer21_attention_weight_max": 27.205583572387695,
      "activations/layer21_attention_weight_min": -19.687419891357422,
      "activations/layer22_attention_weight_max": 25.41408348083496,
      "activations/layer22_attention_weight_min": -24.418609619140625,
      "activations/layer23_attention_weight_max": 31.161178588867188,
      "activations/layer23_attention_weight_min": -23.580198287963867,
      "activations/layer2_attention_weight_max": 31.19048309326172,
      "activations/layer2_attention_weight_min": -31.684873580932617,
      "activations/layer3_attention_weight_max": 89.30487823486328,
      "activations/layer3_attention_weight_min": -90.0230484008789,
      "activations/layer4_attention_weight_max": 83.59056854248047,
      "activations/layer4_attention_weight_min": -83.50725555419922,
      "activations/layer5_attention_weight_max": 62.81067657470703,
      "activations/layer5_attention_weight_min": -68.01736450195312,
      "activations/layer6_attention_weight_max": 49.32059860229492,
      "activations/layer6_attention_weight_min": -54.51990509033203,
      "activations/layer7_attention_weight_max": 66.25218963623047,
      "activations/layer7_attention_weight_min": -66.17127990722656,
      "activations/layer8_attention_weight_max": 45.5745849609375,
      "activations/layer8_attention_weight_min": -50.5785026550293,
      "activations/layer9_attention_weight_max": 42.80361557006836,
      "activations/layer9_attention_weight_min": -46.64961242675781,
      "epoch": 21.26,
      "learning_rate": 1.3064772727272725e-05,
      "loss": 2.7267,
      "step": 365800
    },
    {
      "activations/layer0_attention_weight_max": 15.20068645477295,
      "activations/layer0_attention_weight_min": -14.26484203338623,
      "activations/layer10_attention_weight_max": 37.34062576293945,
      "activations/layer10_attention_weight_min": -36.466796875,
      "activations/layer11_attention_weight_max": 35.456905364990234,
      "activations/layer11_attention_weight_min": -34.52333450317383,
      "activations/layer12_attention_weight_max": 19.967763900756836,
      "activations/layer12_attention_weight_min": -22.361005783081055,
      "activations/layer13_attention_weight_max": 35.39854049682617,
      "activations/layer13_attention_weight_min": -29.996522903442383,
      "activations/layer14_attention_weight_max": 37.61045837402344,
      "activations/layer14_attention_weight_min": -32.90244674682617,
      "activations/layer15_attention_weight_max": 34.413230895996094,
      "activations/layer15_attention_weight_min": -31.659685134887695,
      "activations/layer16_attention_weight_max": 35.53130340576172,
      "activations/layer16_attention_weight_min": -32.20773696899414,
      "activations/layer17_attention_weight_max": 53.91477584838867,
      "activations/layer17_attention_weight_min": -44.56597137451172,
      "activations/layer18_attention_weight_max": 48.06241989135742,
      "activations/layer18_attention_weight_min": -40.953346252441406,
      "activations/layer19_attention_weight_max": 25.500137329101562,
      "activations/layer19_attention_weight_min": -24.41202735900879,
      "activations/layer1_attention_weight_max": 15.496312141418457,
      "activations/layer1_attention_weight_min": -14.508504867553711,
      "activations/layer20_attention_weight_max": 22.38847541809082,
      "activations/layer20_attention_weight_min": -22.9863338470459,
      "activations/layer21_attention_weight_max": 37.43376541137695,
      "activations/layer21_attention_weight_min": -22.455495834350586,
      "activations/layer22_attention_weight_max": 29.139089584350586,
      "activations/layer22_attention_weight_min": -26.33963966369629,
      "activations/layer23_attention_weight_max": 37.70408630371094,
      "activations/layer23_attention_weight_min": -23.909421920776367,
      "activations/layer2_attention_weight_max": 31.643476486206055,
      "activations/layer2_attention_weight_min": -31.28716278076172,
      "activations/layer3_attention_weight_max": 87.08059692382812,
      "activations/layer3_attention_weight_min": -88.1263198852539,
      "activations/layer4_attention_weight_max": 81.98077392578125,
      "activations/layer4_attention_weight_min": -82.6405258178711,
      "activations/layer5_attention_weight_max": 58.38456344604492,
      "activations/layer5_attention_weight_min": -72.62928771972656,
      "activations/layer6_attention_weight_max": 47.16647720336914,
      "activations/layer6_attention_weight_min": -48.31547927856445,
      "activations/layer7_attention_weight_max": 64.42047882080078,
      "activations/layer7_attention_weight_min": -65.9654312133789,
      "activations/layer8_attention_weight_max": 45.84225082397461,
      "activations/layer8_attention_weight_min": -50.72926712036133,
      "activations/layer9_attention_weight_max": 47.40074920654297,
      "activations/layer9_attention_weight_min": -47.67066192626953,
      "epoch": 21.26,
      "learning_rate": 1.3045833333333333e-05,
      "loss": 2.7167,
      "step": 365850
    },
    {
      "activations/layer0_attention_weight_max": 17.11563491821289,
      "activations/layer0_attention_weight_min": -14.452820777893066,
      "activations/layer10_attention_weight_max": 34.797664642333984,
      "activations/layer10_attention_weight_min": -36.48945999145508,
      "activations/layer11_attention_weight_max": 32.88290023803711,
      "activations/layer11_attention_weight_min": -35.306095123291016,
      "activations/layer12_attention_weight_max": 18.765470504760742,
      "activations/layer12_attention_weight_min": -24.078142166137695,
      "activations/layer13_attention_weight_max": 32.600860595703125,
      "activations/layer13_attention_weight_min": -26.08870506286621,
      "activations/layer14_attention_weight_max": 33.789581298828125,
      "activations/layer14_attention_weight_min": -29.414077758789062,
      "activations/layer15_attention_weight_max": 34.0976676940918,
      "activations/layer15_attention_weight_min": -30.1885986328125,
      "activations/layer16_attention_weight_max": 34.20115661621094,
      "activations/layer16_attention_weight_min": -34.22927474975586,
      "activations/layer17_attention_weight_max": 52.172523498535156,
      "activations/layer17_attention_weight_min": -46.008445739746094,
      "activations/layer18_attention_weight_max": 46.7055778503418,
      "activations/layer18_attention_weight_min": -40.21623992919922,
      "activations/layer19_attention_weight_max": 23.783918380737305,
      "activations/layer19_attention_weight_min": -21.30560302734375,
      "activations/layer1_attention_weight_max": 14.81731128692627,
      "activations/layer1_attention_weight_min": -12.8043212890625,
      "activations/layer20_attention_weight_max": 22.21182632446289,
      "activations/layer20_attention_weight_min": -19.740629196166992,
      "activations/layer21_attention_weight_max": 36.22993850708008,
      "activations/layer21_attention_weight_min": -22.272626876831055,
      "activations/layer22_attention_weight_max": 32.19002151489258,
      "activations/layer22_attention_weight_min": -25.80015754699707,
      "activations/layer23_attention_weight_max": 37.432273864746094,
      "activations/layer23_attention_weight_min": -23.83111572265625,
      "activations/layer2_attention_weight_max": 29.52924919128418,
      "activations/layer2_attention_weight_min": -30.275936126708984,
      "activations/layer3_attention_weight_max": 87.60517120361328,
      "activations/layer3_attention_weight_min": -88.29364776611328,
      "activations/layer4_attention_weight_max": 82.90935516357422,
      "activations/layer4_attention_weight_min": -82.4130630493164,
      "activations/layer5_attention_weight_max": 57.91814422607422,
      "activations/layer5_attention_weight_min": -72.30106353759766,
      "activations/layer6_attention_weight_max": 45.99935531616211,
      "activations/layer6_attention_weight_min": -49.922996520996094,
      "activations/layer7_attention_weight_max": 64.78784942626953,
      "activations/layer7_attention_weight_min": -66.30253601074219,
      "activations/layer8_attention_weight_max": 45.74135971069336,
      "activations/layer8_attention_weight_min": -49.87708282470703,
      "activations/layer9_attention_weight_max": 42.69968032836914,
      "activations/layer9_attention_weight_min": -46.418277740478516,
      "epoch": 21.26,
      "learning_rate": 1.302689393939394e-05,
      "loss": 2.7141,
      "step": 365900
    },
    {
      "activations/layer0_attention_weight_max": 16.402250289916992,
      "activations/layer0_attention_weight_min": -14.716376304626465,
      "activations/layer10_attention_weight_max": 37.015419006347656,
      "activations/layer10_attention_weight_min": -38.75061798095703,
      "activations/layer11_attention_weight_max": 34.28824996948242,
      "activations/layer11_attention_weight_min": -35.24350357055664,
      "activations/layer12_attention_weight_max": 19.556364059448242,
      "activations/layer12_attention_weight_min": -28.201757431030273,
      "activations/layer13_attention_weight_max": 33.74160385131836,
      "activations/layer13_attention_weight_min": -28.86701011657715,
      "activations/layer14_attention_weight_max": 36.73310089111328,
      "activations/layer14_attention_weight_min": -31.552845001220703,
      "activations/layer15_attention_weight_max": 32.23372268676758,
      "activations/layer15_attention_weight_min": -31.525915145874023,
      "activations/layer16_attention_weight_max": 32.66704177856445,
      "activations/layer16_attention_weight_min": -31.880809783935547,
      "activations/layer17_attention_weight_max": 49.79164505004883,
      "activations/layer17_attention_weight_min": -47.25537109375,
      "activations/layer18_attention_weight_max": 45.88268280029297,
      "activations/layer18_attention_weight_min": -40.83349609375,
      "activations/layer19_attention_weight_max": 24.474674224853516,
      "activations/layer19_attention_weight_min": -22.897789001464844,
      "activations/layer1_attention_weight_max": 15.655229568481445,
      "activations/layer1_attention_weight_min": -14.95127010345459,
      "activations/layer20_attention_weight_max": 22.480981826782227,
      "activations/layer20_attention_weight_min": -23.064359664916992,
      "activations/layer21_attention_weight_max": 35.539024353027344,
      "activations/layer21_attention_weight_min": -27.09027671813965,
      "activations/layer22_attention_weight_max": 27.454389572143555,
      "activations/layer22_attention_weight_min": -25.294086456298828,
      "activations/layer23_attention_weight_max": 35.929569244384766,
      "activations/layer23_attention_weight_min": -24.081188201904297,
      "activations/layer2_attention_weight_max": 33.19286346435547,
      "activations/layer2_attention_weight_min": -32.76219940185547,
      "activations/layer3_attention_weight_max": 90.745361328125,
      "activations/layer3_attention_weight_min": -86.8521728515625,
      "activations/layer4_attention_weight_max": 86.74022674560547,
      "activations/layer4_attention_weight_min": -82.77157592773438,
      "activations/layer5_attention_weight_max": 58.54557800292969,
      "activations/layer5_attention_weight_min": -69.04862976074219,
      "activations/layer6_attention_weight_max": 50.31938171386719,
      "activations/layer6_attention_weight_min": -49.522987365722656,
      "activations/layer7_attention_weight_max": 66.3839111328125,
      "activations/layer7_attention_weight_min": -66.30125427246094,
      "activations/layer8_attention_weight_max": 52.025634765625,
      "activations/layer8_attention_weight_min": -50.30850601196289,
      "activations/layer9_attention_weight_max": 45.942203521728516,
      "activations/layer9_attention_weight_min": -45.286285400390625,
      "epoch": 21.26,
      "learning_rate": 1.3008333333333334e-05,
      "loss": 2.6988,
      "step": 365950
    },
    {
      "activations/layer0_attention_weight_max": 15.381980895996094,
      "activations/layer0_attention_weight_min": -14.684555053710938,
      "activations/layer10_attention_weight_max": 35.05686950683594,
      "activations/layer10_attention_weight_min": -36.69487380981445,
      "activations/layer11_attention_weight_max": 31.6195125579834,
      "activations/layer11_attention_weight_min": -35.3504753112793,
      "activations/layer12_attention_weight_max": 20.74810791015625,
      "activations/layer12_attention_weight_min": -22.675329208374023,
      "activations/layer13_attention_weight_max": 32.55423355102539,
      "activations/layer13_attention_weight_min": -26.493202209472656,
      "activations/layer14_attention_weight_max": 35.765968322753906,
      "activations/layer14_attention_weight_min": -31.302549362182617,
      "activations/layer15_attention_weight_max": 35.052734375,
      "activations/layer15_attention_weight_min": -29.478940963745117,
      "activations/layer16_attention_weight_max": 34.45487594604492,
      "activations/layer16_attention_weight_min": -32.423728942871094,
      "activations/layer17_attention_weight_max": 51.52438735961914,
      "activations/layer17_attention_weight_min": -45.35560989379883,
      "activations/layer18_attention_weight_max": 47.51492691040039,
      "activations/layer18_attention_weight_min": -41.47686004638672,
      "activations/layer19_attention_weight_max": 25.706390380859375,
      "activations/layer19_attention_weight_min": -23.169523239135742,
      "activations/layer1_attention_weight_max": 15.548686981201172,
      "activations/layer1_attention_weight_min": -14.43026351928711,
      "activations/layer20_attention_weight_max": 23.629575729370117,
      "activations/layer20_attention_weight_min": -22.80776023864746,
      "activations/layer21_attention_weight_max": 33.77885818481445,
      "activations/layer21_attention_weight_min": -20.167898178100586,
      "activations/layer22_attention_weight_max": 27.364168167114258,
      "activations/layer22_attention_weight_min": -24.61046028137207,
      "activations/layer23_attention_weight_max": 33.59379959106445,
      "activations/layer23_attention_weight_min": -23.693084716796875,
      "activations/layer2_attention_weight_max": 32.24165344238281,
      "activations/layer2_attention_weight_min": -32.146324157714844,
      "activations/layer3_attention_weight_max": 91.06722259521484,
      "activations/layer3_attention_weight_min": -96.46308898925781,
      "activations/layer4_attention_weight_max": 83.64605712890625,
      "activations/layer4_attention_weight_min": -87.98129272460938,
      "activations/layer5_attention_weight_max": 59.7686882019043,
      "activations/layer5_attention_weight_min": -67.68299102783203,
      "activations/layer6_attention_weight_max": 50.40359115600586,
      "activations/layer6_attention_weight_min": -51.32170104980469,
      "activations/layer7_attention_weight_max": 65.12891387939453,
      "activations/layer7_attention_weight_min": -67.67813110351562,
      "activations/layer8_attention_weight_max": 45.9467658996582,
      "activations/layer8_attention_weight_min": -51.87107849121094,
      "activations/layer9_attention_weight_max": 43.77164077758789,
      "activations/layer9_attention_weight_min": -50.59251403808594,
      "epoch": 21.27,
      "learning_rate": 1.2989393939393939e-05,
      "loss": 2.7249,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_loss": 2.673828125,
      "eval_runtime": 8.4246,
      "eval_samples_per_second": 509.699,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_openwebtext_loss": 2.673828125,
      "eval_openwebtext_ppl": 14.495353140571094,
      "eval_openwebtext_runtime": 8.4246,
      "eval_openwebtext_samples_per_second": 509.699,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9875,
      "eval_wikitext_samples_per_second": 229.432,
      "step": 366000
    },
    {
      "epoch": 21.27,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.5486,
      "eval_lambada_samples_per_second": 509.92,
      "step": 366000
    },
    {
      "activations/layer0_attention_weight_max": 15.453841209411621,
      "activations/layer0_attention_weight_min": -14.548429489135742,
      "activations/layer10_attention_weight_max": 35.84235763549805,
      "activations/layer10_attention_weight_min": -36.0921745300293,
      "activations/layer11_attention_weight_max": 34.57129669189453,
      "activations/layer11_attention_weight_min": -35.655799865722656,
      "activations/layer12_attention_weight_max": 19.443002700805664,
      "activations/layer12_attention_weight_min": -24.47918128967285,
      "activations/layer13_attention_weight_max": 33.59703063964844,
      "activations/layer13_attention_weight_min": -29.894868850708008,
      "activations/layer14_attention_weight_max": 35.8736572265625,
      "activations/layer14_attention_weight_min": -32.55734634399414,
      "activations/layer15_attention_weight_max": 34.94813537597656,
      "activations/layer15_attention_weight_min": -33.45404052734375,
      "activations/layer16_attention_weight_max": 35.78166198730469,
      "activations/layer16_attention_weight_min": -34.5078239440918,
      "activations/layer17_attention_weight_max": 57.29888153076172,
      "activations/layer17_attention_weight_min": -49.57018280029297,
      "activations/layer18_attention_weight_max": 51.71707534790039,
      "activations/layer18_attention_weight_min": -43.65825653076172,
      "activations/layer19_attention_weight_max": 25.645538330078125,
      "activations/layer19_attention_weight_min": -24.924985885620117,
      "activations/layer1_attention_weight_max": 15.321418762207031,
      "activations/layer1_attention_weight_min": -15.433396339416504,
      "activations/layer20_attention_weight_max": 25.80797576904297,
      "activations/layer20_attention_weight_min": -24.031158447265625,
      "activations/layer21_attention_weight_max": 39.2253303527832,
      "activations/layer21_attention_weight_min": -24.998764038085938,
      "activations/layer22_attention_weight_max": 31.63043785095215,
      "activations/layer22_attention_weight_min": -26.259742736816406,
      "activations/layer23_attention_weight_max": 42.54893112182617,
      "activations/layer23_attention_weight_min": -25.206073760986328,
      "activations/layer2_attention_weight_max": 31.997289657592773,
      "activations/layer2_attention_weight_min": -31.232147216796875,
      "activations/layer3_attention_weight_max": 91.02305603027344,
      "activations/layer3_attention_weight_min": -90.33501434326172,
      "activations/layer4_attention_weight_max": 85.3424301147461,
      "activations/layer4_attention_weight_min": -83.59249114990234,
      "activations/layer5_attention_weight_max": 62.153255462646484,
      "activations/layer5_attention_weight_min": -72.97300720214844,
      "activations/layer6_attention_weight_max": 50.098262786865234,
      "activations/layer6_attention_weight_min": -52.19303894042969,
      "activations/layer7_attention_weight_max": 64.22160339355469,
      "activations/layer7_attention_weight_min": -66.61136627197266,
      "activations/layer8_attention_weight_max": 49.8101921081543,
      "activations/layer8_attention_weight_min": -50.19163131713867,
      "activations/layer9_attention_weight_max": 44.87212371826172,
      "activations/layer9_attention_weight_min": -46.90646743774414,
      "epoch": 21.27,
      "learning_rate": 1.2970454545454545e-05,
      "loss": 2.7259,
      "step": 366050
    },
    {
      "activations/layer0_attention_weight_max": 15.776247024536133,
      "activations/layer0_attention_weight_min": -14.786794662475586,
      "activations/layer10_attention_weight_max": 34.448486328125,
      "activations/layer10_attention_weight_min": -36.32731246948242,
      "activations/layer11_attention_weight_max": 33.98099136352539,
      "activations/layer11_attention_weight_min": -37.60697937011719,
      "activations/layer12_attention_weight_max": 19.596677780151367,
      "activations/layer12_attention_weight_min": -28.072690963745117,
      "activations/layer13_attention_weight_max": 34.07463073730469,
      "activations/layer13_attention_weight_min": -27.28951072692871,
      "activations/layer14_attention_weight_max": 34.300045013427734,
      "activations/layer14_attention_weight_min": -29.726545333862305,
      "activations/layer15_attention_weight_max": 32.621761322021484,
      "activations/layer15_attention_weight_min": -28.906944274902344,
      "activations/layer16_attention_weight_max": 33.77055740356445,
      "activations/layer16_attention_weight_min": -31.94938087463379,
      "activations/layer17_attention_weight_max": 52.6917610168457,
      "activations/layer17_attention_weight_min": -44.02223587036133,
      "activations/layer18_attention_weight_max": 47.10850143432617,
      "activations/layer18_attention_weight_min": -39.85301971435547,
      "activations/layer19_attention_weight_max": 21.425748825073242,
      "activations/layer19_attention_weight_min": -21.1663875579834,
      "activations/layer1_attention_weight_max": 15.276148796081543,
      "activations/layer1_attention_weight_min": -14.17578125,
      "activations/layer20_attention_weight_max": 20.783849716186523,
      "activations/layer20_attention_weight_min": -21.361461639404297,
      "activations/layer21_attention_weight_max": 30.430803298950195,
      "activations/layer21_attention_weight_min": -22.097074508666992,
      "activations/layer22_attention_weight_max": 26.506242752075195,
      "activations/layer22_attention_weight_min": -24.40412139892578,
      "activations/layer23_attention_weight_max": 34.02463150024414,
      "activations/layer23_attention_weight_min": -22.174137115478516,
      "activations/layer2_attention_weight_max": 30.771656036376953,
      "activations/layer2_attention_weight_min": -30.215932846069336,
      "activations/layer3_attention_weight_max": 91.66450500488281,
      "activations/layer3_attention_weight_min": -91.50525665283203,
      "activations/layer4_attention_weight_max": 83.20143127441406,
      "activations/layer4_attention_weight_min": -82.61681365966797,
      "activations/layer5_attention_weight_max": 63.4427490234375,
      "activations/layer5_attention_weight_min": -71.36312103271484,
      "activations/layer6_attention_weight_max": 51.13345718383789,
      "activations/layer6_attention_weight_min": -50.65077209472656,
      "activations/layer7_attention_weight_max": 71.1552505493164,
      "activations/layer7_attention_weight_min": -67.58894348144531,
      "activations/layer8_attention_weight_max": 46.52577590942383,
      "activations/layer8_attention_weight_min": -49.923553466796875,
      "activations/layer9_attention_weight_max": 45.36777114868164,
      "activations/layer9_attention_weight_min": -47.319454193115234,
      "epoch": 21.27,
      "learning_rate": 1.295151515151515e-05,
      "loss": 2.7168,
      "step": 366100
    },
    {
      "activations/layer0_attention_weight_max": 15.459770202636719,
      "activations/layer0_attention_weight_min": -14.426291465759277,
      "activations/layer10_attention_weight_max": 38.45090866088867,
      "activations/layer10_attention_weight_min": -37.971160888671875,
      "activations/layer11_attention_weight_max": 38.10781478881836,
      "activations/layer11_attention_weight_min": -38.53681945800781,
      "activations/layer12_attention_weight_max": 18.51931381225586,
      "activations/layer12_attention_weight_min": -25.90363311767578,
      "activations/layer13_attention_weight_max": 33.06477355957031,
      "activations/layer13_attention_weight_min": -27.731185913085938,
      "activations/layer14_attention_weight_max": 35.02064514160156,
      "activations/layer14_attention_weight_min": -30.19753646850586,
      "activations/layer15_attention_weight_max": 30.73873519897461,
      "activations/layer15_attention_weight_min": -29.810182571411133,
      "activations/layer16_attention_weight_max": 31.388748168945312,
      "activations/layer16_attention_weight_min": -30.94402313232422,
      "activations/layer17_attention_weight_max": 46.82841491699219,
      "activations/layer17_attention_weight_min": -43.33781814575195,
      "activations/layer18_attention_weight_max": 43.85807800292969,
      "activations/layer18_attention_weight_min": -38.029422760009766,
      "activations/layer19_attention_weight_max": 24.295379638671875,
      "activations/layer19_attention_weight_min": -21.241626739501953,
      "activations/layer1_attention_weight_max": 14.882101058959961,
      "activations/layer1_attention_weight_min": -13.948426246643066,
      "activations/layer20_attention_weight_max": 23.543031692504883,
      "activations/layer20_attention_weight_min": -20.49469757080078,
      "activations/layer21_attention_weight_max": 35.1434440612793,
      "activations/layer21_attention_weight_min": -21.137624740600586,
      "activations/layer22_attention_weight_max": 27.50210189819336,
      "activations/layer22_attention_weight_min": -25.02051544189453,
      "activations/layer23_attention_weight_max": 34.75001525878906,
      "activations/layer23_attention_weight_min": -23.790205001831055,
      "activations/layer2_attention_weight_max": 30.34687614440918,
      "activations/layer2_attention_weight_min": -30.044742584228516,
      "activations/layer3_attention_weight_max": 88.18988037109375,
      "activations/layer3_attention_weight_min": -90.9403305053711,
      "activations/layer4_attention_weight_max": 86.87609100341797,
      "activations/layer4_attention_weight_min": -87.3601303100586,
      "activations/layer5_attention_weight_max": 60.901268005371094,
      "activations/layer5_attention_weight_min": -73.69181823730469,
      "activations/layer6_attention_weight_max": 52.74457550048828,
      "activations/layer6_attention_weight_min": -51.592647552490234,
      "activations/layer7_attention_weight_max": 68.63568115234375,
      "activations/layer7_attention_weight_min": -68.98465728759766,
      "activations/layer8_attention_weight_max": 49.85021209716797,
      "activations/layer8_attention_weight_min": -52.18144226074219,
      "activations/layer9_attention_weight_max": 47.53608322143555,
      "activations/layer9_attention_weight_min": -47.732154846191406,
      "epoch": 21.28,
      "learning_rate": 1.2932575757575757e-05,
      "loss": 2.6995,
      "step": 366150
    },
    {
      "activations/layer0_attention_weight_max": 15.100485801696777,
      "activations/layer0_attention_weight_min": -14.79816722869873,
      "activations/layer10_attention_weight_max": 35.27059555053711,
      "activations/layer10_attention_weight_min": -34.613338470458984,
      "activations/layer11_attention_weight_max": 32.053314208984375,
      "activations/layer11_attention_weight_min": -33.82105255126953,
      "activations/layer12_attention_weight_max": 19.246814727783203,
      "activations/layer12_attention_weight_min": -25.60337257385254,
      "activations/layer13_attention_weight_max": 32.54698181152344,
      "activations/layer13_attention_weight_min": -25.96306610107422,
      "activations/layer14_attention_weight_max": 35.14237976074219,
      "activations/layer14_attention_weight_min": -29.372941970825195,
      "activations/layer15_attention_weight_max": 33.407039642333984,
      "activations/layer15_attention_weight_min": -30.546865463256836,
      "activations/layer16_attention_weight_max": 34.13853454589844,
      "activations/layer16_attention_weight_min": -32.94130325317383,
      "activations/layer17_attention_weight_max": 52.283233642578125,
      "activations/layer17_attention_weight_min": -46.802703857421875,
      "activations/layer18_attention_weight_max": 48.574886322021484,
      "activations/layer18_attention_weight_min": -42.52249526977539,
      "activations/layer19_attention_weight_max": 24.295278549194336,
      "activations/layer19_attention_weight_min": -26.850997924804688,
      "activations/layer1_attention_weight_max": 14.698562622070312,
      "activations/layer1_attention_weight_min": -13.688676834106445,
      "activations/layer20_attention_weight_max": 23.669572830200195,
      "activations/layer20_attention_weight_min": -23.83661460876465,
      "activations/layer21_attention_weight_max": 36.39983367919922,
      "activations/layer21_attention_weight_min": -24.717771530151367,
      "activations/layer22_attention_weight_max": 28.810836791992188,
      "activations/layer22_attention_weight_min": -26.437328338623047,
      "activations/layer23_attention_weight_max": 37.08209228515625,
      "activations/layer23_attention_weight_min": -25.822385787963867,
      "activations/layer2_attention_weight_max": 30.472951889038086,
      "activations/layer2_attention_weight_min": -29.940128326416016,
      "activations/layer3_attention_weight_max": 89.09424591064453,
      "activations/layer3_attention_weight_min": -88.09062957763672,
      "activations/layer4_attention_weight_max": 83.0197982788086,
      "activations/layer4_attention_weight_min": -86.399658203125,
      "activations/layer5_attention_weight_max": 59.4012451171875,
      "activations/layer5_attention_weight_min": -69.54605102539062,
      "activations/layer6_attention_weight_max": 48.321510314941406,
      "activations/layer6_attention_weight_min": -50.90143585205078,
      "activations/layer7_attention_weight_max": 65.86901092529297,
      "activations/layer7_attention_weight_min": -65.97901153564453,
      "activations/layer8_attention_weight_max": 46.48525619506836,
      "activations/layer8_attention_weight_min": -49.73857116699219,
      "activations/layer9_attention_weight_max": 41.00944137573242,
      "activations/layer9_attention_weight_min": -47.89277648925781,
      "epoch": 21.28,
      "learning_rate": 1.2913636363636363e-05,
      "loss": 2.7,
      "step": 366200
    },
    {
      "activations/layer0_attention_weight_max": 15.626952171325684,
      "activations/layer0_attention_weight_min": -14.392411231994629,
      "activations/layer10_attention_weight_max": 34.03140640258789,
      "activations/layer10_attention_weight_min": -35.15932083129883,
      "activations/layer11_attention_weight_max": 33.11093521118164,
      "activations/layer11_attention_weight_min": -35.452110290527344,
      "activations/layer12_attention_weight_max": 19.160614013671875,
      "activations/layer12_attention_weight_min": -23.278703689575195,
      "activations/layer13_attention_weight_max": 42.75364685058594,
      "activations/layer13_attention_weight_min": -27.483257293701172,
      "activations/layer14_attention_weight_max": 37.87258529663086,
      "activations/layer14_attention_weight_min": -31.336015701293945,
      "activations/layer15_attention_weight_max": 37.04133224487305,
      "activations/layer15_attention_weight_min": -31.622739791870117,
      "activations/layer16_attention_weight_max": 38.8867073059082,
      "activations/layer16_attention_weight_min": -37.16324234008789,
      "activations/layer17_attention_weight_max": 53.5927734375,
      "activations/layer17_attention_weight_min": -44.90587615966797,
      "activations/layer18_attention_weight_max": 47.68860626220703,
      "activations/layer18_attention_weight_min": -41.2193603515625,
      "activations/layer19_attention_weight_max": 25.6938419342041,
      "activations/layer19_attention_weight_min": -23.29842758178711,
      "activations/layer1_attention_weight_max": 14.992523193359375,
      "activations/layer1_attention_weight_min": -13.895843505859375,
      "activations/layer20_attention_weight_max": 23.258506774902344,
      "activations/layer20_attention_weight_min": -22.724422454833984,
      "activations/layer21_attention_weight_max": 35.363834381103516,
      "activations/layer21_attention_weight_min": -24.812103271484375,
      "activations/layer22_attention_weight_max": 28.253498077392578,
      "activations/layer22_attention_weight_min": -28.45655632019043,
      "activations/layer23_attention_weight_max": 38.59290313720703,
      "activations/layer23_attention_weight_min": -25.747440338134766,
      "activations/layer2_attention_weight_max": 30.317535400390625,
      "activations/layer2_attention_weight_min": -28.853775024414062,
      "activations/layer3_attention_weight_max": 85.09503173828125,
      "activations/layer3_attention_weight_min": -83.81786346435547,
      "activations/layer4_attention_weight_max": 83.55506134033203,
      "activations/layer4_attention_weight_min": -82.8775863647461,
      "activations/layer5_attention_weight_max": 64.83102416992188,
      "activations/layer5_attention_weight_min": -68.8598403930664,
      "activations/layer6_attention_weight_max": 48.07218933105469,
      "activations/layer6_attention_weight_min": -50.51254653930664,
      "activations/layer7_attention_weight_max": 68.36239624023438,
      "activations/layer7_attention_weight_min": -61.84824752807617,
      "activations/layer8_attention_weight_max": 44.249298095703125,
      "activations/layer8_attention_weight_min": -49.60319137573242,
      "activations/layer9_attention_weight_max": 43.81075668334961,
      "activations/layer9_attention_weight_min": -45.23170852661133,
      "epoch": 21.28,
      "learning_rate": 1.2894696969696968e-05,
      "loss": 2.726,
      "step": 366250
    },
    {
      "activations/layer0_attention_weight_max": 15.508480072021484,
      "activations/layer0_attention_weight_min": -14.724031448364258,
      "activations/layer10_attention_weight_max": 35.359161376953125,
      "activations/layer10_attention_weight_min": -37.16925811767578,
      "activations/layer11_attention_weight_max": 34.59941482543945,
      "activations/layer11_attention_weight_min": -34.83489990234375,
      "activations/layer12_attention_weight_max": 18.503690719604492,
      "activations/layer12_attention_weight_min": -25.088260650634766,
      "activations/layer13_attention_weight_max": 29.863710403442383,
      "activations/layer13_attention_weight_min": -29.821292877197266,
      "activations/layer14_attention_weight_max": 32.68037033081055,
      "activations/layer14_attention_weight_min": -29.40815544128418,
      "activations/layer15_attention_weight_max": 29.70640754699707,
      "activations/layer15_attention_weight_min": -31.175554275512695,
      "activations/layer16_attention_weight_max": 31.947620391845703,
      "activations/layer16_attention_weight_min": -33.100257873535156,
      "activations/layer17_attention_weight_max": 49.63766098022461,
      "activations/layer17_attention_weight_min": -47.5620231628418,
      "activations/layer18_attention_weight_max": 42.35086441040039,
      "activations/layer18_attention_weight_min": -42.95762634277344,
      "activations/layer19_attention_weight_max": 22.365890502929688,
      "activations/layer19_attention_weight_min": -24.572410583496094,
      "activations/layer1_attention_weight_max": 14.898597717285156,
      "activations/layer1_attention_weight_min": -14.87209701538086,
      "activations/layer20_attention_weight_max": 20.469646453857422,
      "activations/layer20_attention_weight_min": -22.57476043701172,
      "activations/layer21_attention_weight_max": 31.47453498840332,
      "activations/layer21_attention_weight_min": -24.2287540435791,
      "activations/layer22_attention_weight_max": 27.33110809326172,
      "activations/layer22_attention_weight_min": -26.537975311279297,
      "activations/layer23_attention_weight_max": 35.611244201660156,
      "activations/layer23_attention_weight_min": -26.3046875,
      "activations/layer2_attention_weight_max": 32.35517501831055,
      "activations/layer2_attention_weight_min": -30.940872192382812,
      "activations/layer3_attention_weight_max": 89.28235626220703,
      "activations/layer3_attention_weight_min": -88.40381622314453,
      "activations/layer4_attention_weight_max": 84.74828338623047,
      "activations/layer4_attention_weight_min": -82.1199951171875,
      "activations/layer5_attention_weight_max": 61.651939392089844,
      "activations/layer5_attention_weight_min": -69.8523941040039,
      "activations/layer6_attention_weight_max": 50.287776947021484,
      "activations/layer6_attention_weight_min": -49.491844177246094,
      "activations/layer7_attention_weight_max": 68.24137115478516,
      "activations/layer7_attention_weight_min": -67.1449203491211,
      "activations/layer8_attention_weight_max": 47.233699798583984,
      "activations/layer8_attention_weight_min": -47.974056243896484,
      "activations/layer9_attention_weight_max": 42.24471664428711,
      "activations/layer9_attention_weight_min": -46.053688049316406,
      "epoch": 21.28,
      "learning_rate": 1.2875757575757575e-05,
      "loss": 2.7248,
      "step": 366300
    },
    {
      "activations/layer0_attention_weight_max": 15.878680229187012,
      "activations/layer0_attention_weight_min": -14.751303672790527,
      "activations/layer10_attention_weight_max": 35.873573303222656,
      "activations/layer10_attention_weight_min": -35.70793533325195,
      "activations/layer11_attention_weight_max": 35.44518280029297,
      "activations/layer11_attention_weight_min": -35.774658203125,
      "activations/layer12_attention_weight_max": 20.94772720336914,
      "activations/layer12_attention_weight_min": -23.649471282958984,
      "activations/layer13_attention_weight_max": 36.09894943237305,
      "activations/layer13_attention_weight_min": -27.318845748901367,
      "activations/layer14_attention_weight_max": 40.21284866333008,
      "activations/layer14_attention_weight_min": -34.937286376953125,
      "activations/layer15_attention_weight_max": 38.13987350463867,
      "activations/layer15_attention_weight_min": -33.248985290527344,
      "activations/layer16_attention_weight_max": 39.939796447753906,
      "activations/layer16_attention_weight_min": -39.38748550415039,
      "activations/layer17_attention_weight_max": 55.21160125732422,
      "activations/layer17_attention_weight_min": -51.72903823852539,
      "activations/layer18_attention_weight_max": 50.306907653808594,
      "activations/layer18_attention_weight_min": -48.55765914916992,
      "activations/layer19_attention_weight_max": 26.332538604736328,
      "activations/layer19_attention_weight_min": -26.268136978149414,
      "activations/layer1_attention_weight_max": 14.858540534973145,
      "activations/layer1_attention_weight_min": -16.647260665893555,
      "activations/layer20_attention_weight_max": 24.95027732849121,
      "activations/layer20_attention_weight_min": -24.308788299560547,
      "activations/layer21_attention_weight_max": 39.002227783203125,
      "activations/layer21_attention_weight_min": -24.128061294555664,
      "activations/layer22_attention_weight_max": 31.53774070739746,
      "activations/layer22_attention_weight_min": -25.658981323242188,
      "activations/layer23_attention_weight_max": 43.424644470214844,
      "activations/layer23_attention_weight_min": -22.90964126586914,
      "activations/layer2_attention_weight_max": 31.890560150146484,
      "activations/layer2_attention_weight_min": -30.21377944946289,
      "activations/layer3_attention_weight_max": 87.93396759033203,
      "activations/layer3_attention_weight_min": -89.84164428710938,
      "activations/layer4_attention_weight_max": 85.6448745727539,
      "activations/layer4_attention_weight_min": -86.96647644042969,
      "activations/layer5_attention_weight_max": 60.79228973388672,
      "activations/layer5_attention_weight_min": -73.80789947509766,
      "activations/layer6_attention_weight_max": 52.87661361694336,
      "activations/layer6_attention_weight_min": -51.78528594970703,
      "activations/layer7_attention_weight_max": 65.27778625488281,
      "activations/layer7_attention_weight_min": -66.4954833984375,
      "activations/layer8_attention_weight_max": 49.01088333129883,
      "activations/layer8_attention_weight_min": -52.51669692993164,
      "activations/layer9_attention_weight_max": 44.135066986083984,
      "activations/layer9_attention_weight_min": -47.560943603515625,
      "epoch": 21.29,
      "learning_rate": 1.2856818181818181e-05,
      "loss": 2.6996,
      "step": 366350
    },
    {
      "activations/layer0_attention_weight_max": 15.325745582580566,
      "activations/layer0_attention_weight_min": -14.538822174072266,
      "activations/layer10_attention_weight_max": 34.897987365722656,
      "activations/layer10_attention_weight_min": -36.38469314575195,
      "activations/layer11_attention_weight_max": 33.25522994995117,
      "activations/layer11_attention_weight_min": -34.59568786621094,
      "activations/layer12_attention_weight_max": 19.941953659057617,
      "activations/layer12_attention_weight_min": -29.38111686706543,
      "activations/layer13_attention_weight_max": 36.115509033203125,
      "activations/layer13_attention_weight_min": -28.125743865966797,
      "activations/layer14_attention_weight_max": 36.213314056396484,
      "activations/layer14_attention_weight_min": -28.9796142578125,
      "activations/layer15_attention_weight_max": 32.326515197753906,
      "activations/layer15_attention_weight_min": -29.212045669555664,
      "activations/layer16_attention_weight_max": 35.36098861694336,
      "activations/layer16_attention_weight_min": -31.96337127685547,
      "activations/layer17_attention_weight_max": 53.34661865234375,
      "activations/layer17_attention_weight_min": -44.95930480957031,
      "activations/layer18_attention_weight_max": 48.069766998291016,
      "activations/layer18_attention_weight_min": -40.15401077270508,
      "activations/layer19_attention_weight_max": 25.5938663482666,
      "activations/layer19_attention_weight_min": -24.212141036987305,
      "activations/layer1_attention_weight_max": 15.0054349899292,
      "activations/layer1_attention_weight_min": -13.67034912109375,
      "activations/layer20_attention_weight_max": 24.069293975830078,
      "activations/layer20_attention_weight_min": -20.071269989013672,
      "activations/layer21_attention_weight_max": 33.70967483520508,
      "activations/layer21_attention_weight_min": -21.436969757080078,
      "activations/layer22_attention_weight_max": 29.361738204956055,
      "activations/layer22_attention_weight_min": -23.4671630859375,
      "activations/layer23_attention_weight_max": 36.09941101074219,
      "activations/layer23_attention_weight_min": -23.496366500854492,
      "activations/layer2_attention_weight_max": 32.92873764038086,
      "activations/layer2_attention_weight_min": -29.55476188659668,
      "activations/layer3_attention_weight_max": 90.25436401367188,
      "activations/layer3_attention_weight_min": -87.64676666259766,
      "activations/layer4_attention_weight_max": 84.1872329711914,
      "activations/layer4_attention_weight_min": -82.76820373535156,
      "activations/layer5_attention_weight_max": 61.35154724121094,
      "activations/layer5_attention_weight_min": -69.40928649902344,
      "activations/layer6_attention_weight_max": 49.788063049316406,
      "activations/layer6_attention_weight_min": -51.7795524597168,
      "activations/layer7_attention_weight_max": 71.39678955078125,
      "activations/layer7_attention_weight_min": -68.16072082519531,
      "activations/layer8_attention_weight_max": 47.85698318481445,
      "activations/layer8_attention_weight_min": -52.019649505615234,
      "activations/layer9_attention_weight_max": 46.41373825073242,
      "activations/layer9_attention_weight_min": -45.55208969116211,
      "epoch": 21.29,
      "learning_rate": 1.2837878787878786e-05,
      "loss": 2.7149,
      "step": 366400
    },
    {
      "activations/layer0_attention_weight_max": 16.67838478088379,
      "activations/layer0_attention_weight_min": -14.17054271697998,
      "activations/layer10_attention_weight_max": 37.39478302001953,
      "activations/layer10_attention_weight_min": -37.75720977783203,
      "activations/layer11_attention_weight_max": 37.351322174072266,
      "activations/layer11_attention_weight_min": -36.600120544433594,
      "activations/layer12_attention_weight_max": 19.6387996673584,
      "activations/layer12_attention_weight_min": -30.981348037719727,
      "activations/layer13_attention_weight_max": 39.72432327270508,
      "activations/layer13_attention_weight_min": -29.453807830810547,
      "activations/layer14_attention_weight_max": 43.380523681640625,
      "activations/layer14_attention_weight_min": -33.323265075683594,
      "activations/layer15_attention_weight_max": 45.372676849365234,
      "activations/layer15_attention_weight_min": -32.49946975708008,
      "activations/layer16_attention_weight_max": 40.74128723144531,
      "activations/layer16_attention_weight_min": -34.93703842163086,
      "activations/layer17_attention_weight_max": 60.34210205078125,
      "activations/layer17_attention_weight_min": -53.73759841918945,
      "activations/layer18_attention_weight_max": 60.9805793762207,
      "activations/layer18_attention_weight_min": -48.64362335205078,
      "activations/layer19_attention_weight_max": 33.55238342285156,
      "activations/layer19_attention_weight_min": -27.45412254333496,
      "activations/layer1_attention_weight_max": 15.083276748657227,
      "activations/layer1_attention_weight_min": -13.589553833007812,
      "activations/layer20_attention_weight_max": 29.336585998535156,
      "activations/layer20_attention_weight_min": -23.170122146606445,
      "activations/layer21_attention_weight_max": 52.61246109008789,
      "activations/layer21_attention_weight_min": -26.251665115356445,
      "activations/layer22_attention_weight_max": 33.04328536987305,
      "activations/layer22_attention_weight_min": -26.94611358642578,
      "activations/layer23_attention_weight_max": 42.595645904541016,
      "activations/layer23_attention_weight_min": -24.80671501159668,
      "activations/layer2_attention_weight_max": 33.35811996459961,
      "activations/layer2_attention_weight_min": -31.61066436767578,
      "activations/layer3_attention_weight_max": 93.6788101196289,
      "activations/layer3_attention_weight_min": -92.5833969116211,
      "activations/layer4_attention_weight_max": 88.55160522460938,
      "activations/layer4_attention_weight_min": -86.7529525756836,
      "activations/layer5_attention_weight_max": 61.51361083984375,
      "activations/layer5_attention_weight_min": -75.91011810302734,
      "activations/layer6_attention_weight_max": 49.306556701660156,
      "activations/layer6_attention_weight_min": -54.87943649291992,
      "activations/layer7_attention_weight_max": 69.0215835571289,
      "activations/layer7_attention_weight_min": -68.91626739501953,
      "activations/layer8_attention_weight_max": 48.94081497192383,
      "activations/layer8_attention_weight_min": -51.74135971069336,
      "activations/layer9_attention_weight_max": 47.50783920288086,
      "activations/layer9_attention_weight_min": -50.62704086303711,
      "epoch": 21.29,
      "learning_rate": 1.2818939393939393e-05,
      "loss": 2.7023,
      "step": 366450
    },
    {
      "activations/layer0_attention_weight_max": 15.69048023223877,
      "activations/layer0_attention_weight_min": -14.623753547668457,
      "activations/layer10_attention_weight_max": 37.31028366088867,
      "activations/layer10_attention_weight_min": -37.744789123535156,
      "activations/layer11_attention_weight_max": 35.094696044921875,
      "activations/layer11_attention_weight_min": -36.05854415893555,
      "activations/layer12_attention_weight_max": 20.8093204498291,
      "activations/layer12_attention_weight_min": -24.909820556640625,
      "activations/layer13_attention_weight_max": 35.11872863769531,
      "activations/layer13_attention_weight_min": -28.469219207763672,
      "activations/layer14_attention_weight_max": 37.59121322631836,
      "activations/layer14_attention_weight_min": -29.86781883239746,
      "activations/layer15_attention_weight_max": 34.35362243652344,
      "activations/layer15_attention_weight_min": -29.514785766601562,
      "activations/layer16_attention_weight_max": 35.577762603759766,
      "activations/layer16_attention_weight_min": -32.83616256713867,
      "activations/layer17_attention_weight_max": 51.81401443481445,
      "activations/layer17_attention_weight_min": -44.163509368896484,
      "activations/layer18_attention_weight_max": 49.54489517211914,
      "activations/layer18_attention_weight_min": -39.724708557128906,
      "activations/layer19_attention_weight_max": 24.26057243347168,
      "activations/layer19_attention_weight_min": -21.98546600341797,
      "activations/layer1_attention_weight_max": 15.57901382446289,
      "activations/layer1_attention_weight_min": -15.714373588562012,
      "activations/layer20_attention_weight_max": 23.2314453125,
      "activations/layer20_attention_weight_min": -20.147518157958984,
      "activations/layer21_attention_weight_max": 32.41337966918945,
      "activations/layer21_attention_weight_min": -22.31591796875,
      "activations/layer22_attention_weight_max": 30.01593780517578,
      "activations/layer22_attention_weight_min": -23.93968963623047,
      "activations/layer23_attention_weight_max": 36.002864837646484,
      "activations/layer23_attention_weight_min": -24.647695541381836,
      "activations/layer2_attention_weight_max": 33.12747573852539,
      "activations/layer2_attention_weight_min": -32.60914611816406,
      "activations/layer3_attention_weight_max": 90.43981170654297,
      "activations/layer3_attention_weight_min": -91.62963104248047,
      "activations/layer4_attention_weight_max": 86.97704315185547,
      "activations/layer4_attention_weight_min": -89.44530487060547,
      "activations/layer5_attention_weight_max": 59.84735870361328,
      "activations/layer5_attention_weight_min": -75.71162414550781,
      "activations/layer6_attention_weight_max": 51.85652542114258,
      "activations/layer6_attention_weight_min": -55.13832473754883,
      "activations/layer7_attention_weight_max": 67.33901977539062,
      "activations/layer7_attention_weight_min": -68.4127197265625,
      "activations/layer8_attention_weight_max": 48.02622604370117,
      "activations/layer8_attention_weight_min": -52.687461853027344,
      "activations/layer9_attention_weight_max": 45.69017791748047,
      "activations/layer9_attention_weight_min": -48.847293853759766,
      "epoch": 21.3,
      "learning_rate": 1.2799999999999998e-05,
      "loss": 2.7153,
      "step": 366500
    },
    {
      "activations/layer0_attention_weight_max": 15.307846069335938,
      "activations/layer0_attention_weight_min": -14.523993492126465,
      "activations/layer10_attention_weight_max": 37.49605178833008,
      "activations/layer10_attention_weight_min": -34.253936767578125,
      "activations/layer11_attention_weight_max": 31.991140365600586,
      "activations/layer11_attention_weight_min": -33.09356689453125,
      "activations/layer12_attention_weight_max": 18.12534523010254,
      "activations/layer12_attention_weight_min": -25.962238311767578,
      "activations/layer13_attention_weight_max": 31.605030059814453,
      "activations/layer13_attention_weight_min": -25.55681610107422,
      "activations/layer14_attention_weight_max": 35.09999465942383,
      "activations/layer14_attention_weight_min": -29.140764236450195,
      "activations/layer15_attention_weight_max": 32.66646194458008,
      "activations/layer15_attention_weight_min": -28.662689208984375,
      "activations/layer16_attention_weight_max": 34.74826431274414,
      "activations/layer16_attention_weight_min": -32.78211975097656,
      "activations/layer17_attention_weight_max": 52.557010650634766,
      "activations/layer17_attention_weight_min": -43.84076690673828,
      "activations/layer18_attention_weight_max": 47.46002197265625,
      "activations/layer18_attention_weight_min": -39.85249710083008,
      "activations/layer19_attention_weight_max": 23.165931701660156,
      "activations/layer19_attention_weight_min": -22.00934410095215,
      "activations/layer1_attention_weight_max": 15.752092361450195,
      "activations/layer1_attention_weight_min": -14.467455863952637,
      "activations/layer20_attention_weight_max": 24.535728454589844,
      "activations/layer20_attention_weight_min": -23.891357421875,
      "activations/layer21_attention_weight_max": 39.011070251464844,
      "activations/layer21_attention_weight_min": -20.95950698852539,
      "activations/layer22_attention_weight_max": 30.331100463867188,
      "activations/layer22_attention_weight_min": -25.92605972290039,
      "activations/layer23_attention_weight_max": 35.254783630371094,
      "activations/layer23_attention_weight_min": -23.280658721923828,
      "activations/layer2_attention_weight_max": 29.380477905273438,
      "activations/layer2_attention_weight_min": -28.156539916992188,
      "activations/layer3_attention_weight_max": 82.01590728759766,
      "activations/layer3_attention_weight_min": -81.51689147949219,
      "activations/layer4_attention_weight_max": 76.87812805175781,
      "activations/layer4_attention_weight_min": -79.42748260498047,
      "activations/layer5_attention_weight_max": 57.854026794433594,
      "activations/layer5_attention_weight_min": -67.4525146484375,
      "activations/layer6_attention_weight_max": 45.26893615722656,
      "activations/layer6_attention_weight_min": -45.822959899902344,
      "activations/layer7_attention_weight_max": 63.74925231933594,
      "activations/layer7_attention_weight_min": -64.94390869140625,
      "activations/layer8_attention_weight_max": 42.726348876953125,
      "activations/layer8_attention_weight_min": -46.21051788330078,
      "activations/layer9_attention_weight_max": 47.0298957824707,
      "activations/layer9_attention_weight_min": -43.0971794128418,
      "epoch": 21.3,
      "learning_rate": 1.2781060606060604e-05,
      "loss": 2.7166,
      "step": 366550
    },
    {
      "activations/layer0_attention_weight_max": 17.45142936706543,
      "activations/layer0_attention_weight_min": -14.684624671936035,
      "activations/layer10_attention_weight_max": 35.064239501953125,
      "activations/layer10_attention_weight_min": -34.15932083129883,
      "activations/layer11_attention_weight_max": 31.938594818115234,
      "activations/layer11_attention_weight_min": -34.33386993408203,
      "activations/layer12_attention_weight_max": 18.58162498474121,
      "activations/layer12_attention_weight_min": -24.13787269592285,
      "activations/layer13_attention_weight_max": 34.2879638671875,
      "activations/layer13_attention_weight_min": -27.16084098815918,
      "activations/layer14_attention_weight_max": 34.84614181518555,
      "activations/layer14_attention_weight_min": -30.28857421875,
      "activations/layer15_attention_weight_max": 31.42597198486328,
      "activations/layer15_attention_weight_min": -30.307170867919922,
      "activations/layer16_attention_weight_max": 34.99494552612305,
      "activations/layer16_attention_weight_min": -33.212257385253906,
      "activations/layer17_attention_weight_max": 55.49757766723633,
      "activations/layer17_attention_weight_min": -46.019020080566406,
      "activations/layer18_attention_weight_max": 46.846927642822266,
      "activations/layer18_attention_weight_min": -39.66015625,
      "activations/layer19_attention_weight_max": 24.85983657836914,
      "activations/layer19_attention_weight_min": -25.100154876708984,
      "activations/layer1_attention_weight_max": 15.531780242919922,
      "activations/layer1_attention_weight_min": -15.72236442565918,
      "activations/layer20_attention_weight_max": 21.696514129638672,
      "activations/layer20_attention_weight_min": -21.44445037841797,
      "activations/layer21_attention_weight_max": 37.40787124633789,
      "activations/layer21_attention_weight_min": -22.362924575805664,
      "activations/layer22_attention_weight_max": 26.913360595703125,
      "activations/layer22_attention_weight_min": -25.08881187438965,
      "activations/layer23_attention_weight_max": 35.5125732421875,
      "activations/layer23_attention_weight_min": -25.1457576751709,
      "activations/layer2_attention_weight_max": 34.924659729003906,
      "activations/layer2_attention_weight_min": -29.461294174194336,
      "activations/layer3_attention_weight_max": 93.34150695800781,
      "activations/layer3_attention_weight_min": -89.45543670654297,
      "activations/layer4_attention_weight_max": 89.06353759765625,
      "activations/layer4_attention_weight_min": -89.91796875,
      "activations/layer5_attention_weight_max": 61.16605758666992,
      "activations/layer5_attention_weight_min": -73.85958862304688,
      "activations/layer6_attention_weight_max": 51.67362594604492,
      "activations/layer6_attention_weight_min": -52.45133590698242,
      "activations/layer7_attention_weight_max": 69.68707275390625,
      "activations/layer7_attention_weight_min": -66.62812805175781,
      "activations/layer8_attention_weight_max": 47.22300338745117,
      "activations/layer8_attention_weight_min": -50.72561264038086,
      "activations/layer9_attention_weight_max": 43.609397888183594,
      "activations/layer9_attention_weight_min": -45.152957916259766,
      "epoch": 21.3,
      "learning_rate": 1.2762121212121211e-05,
      "loss": 2.7113,
      "step": 366600
    },
    {
      "activations/layer0_attention_weight_max": 14.67260456085205,
      "activations/layer0_attention_weight_min": -14.275471687316895,
      "activations/layer10_attention_weight_max": 34.79909896850586,
      "activations/layer10_attention_weight_min": -36.85681915283203,
      "activations/layer11_attention_weight_max": 30.816892623901367,
      "activations/layer11_attention_weight_min": -34.651519775390625,
      "activations/layer12_attention_weight_max": 20.52437973022461,
      "activations/layer12_attention_weight_min": -25.98249626159668,
      "activations/layer13_attention_weight_max": 36.71110153198242,
      "activations/layer13_attention_weight_min": -29.230186462402344,
      "activations/layer14_attention_weight_max": 40.271881103515625,
      "activations/layer14_attention_weight_min": -30.276859283447266,
      "activations/layer15_attention_weight_max": 34.79773712158203,
      "activations/layer15_attention_weight_min": -30.133323669433594,
      "activations/layer16_attention_weight_max": 35.29880142211914,
      "activations/layer16_attention_weight_min": -34.19748306274414,
      "activations/layer17_attention_weight_max": 51.210689544677734,
      "activations/layer17_attention_weight_min": -45.44084548950195,
      "activations/layer18_attention_weight_max": 47.5515022277832,
      "activations/layer18_attention_weight_min": -42.65848159790039,
      "activations/layer19_attention_weight_max": 25.26273536682129,
      "activations/layer19_attention_weight_min": -22.710689544677734,
      "activations/layer1_attention_weight_max": 15.591191291809082,
      "activations/layer1_attention_weight_min": -13.691608428955078,
      "activations/layer20_attention_weight_max": 23.45198631286621,
      "activations/layer20_attention_weight_min": -21.42930793762207,
      "activations/layer21_attention_weight_max": 43.0346794128418,
      "activations/layer21_attention_weight_min": -23.32848358154297,
      "activations/layer22_attention_weight_max": 30.2078914642334,
      "activations/layer22_attention_weight_min": -25.450145721435547,
      "activations/layer23_attention_weight_max": 38.056358337402344,
      "activations/layer23_attention_weight_min": -24.555418014526367,
      "activations/layer2_attention_weight_max": 29.846586227416992,
      "activations/layer2_attention_weight_min": -29.212154388427734,
      "activations/layer3_attention_weight_max": 89.75867462158203,
      "activations/layer3_attention_weight_min": -90.34053802490234,
      "activations/layer4_attention_weight_max": 82.54067993164062,
      "activations/layer4_attention_weight_min": -86.27469635009766,
      "activations/layer5_attention_weight_max": 62.43408966064453,
      "activations/layer5_attention_weight_min": -72.7242431640625,
      "activations/layer6_attention_weight_max": 50.45315933227539,
      "activations/layer6_attention_weight_min": -53.37412643432617,
      "activations/layer7_attention_weight_max": 65.04325103759766,
      "activations/layer7_attention_weight_min": -66.34383392333984,
      "activations/layer8_attention_weight_max": 46.45118713378906,
      "activations/layer8_attention_weight_min": -51.976497650146484,
      "activations/layer9_attention_weight_max": 48.91050338745117,
      "activations/layer9_attention_weight_min": -47.4689826965332,
      "epoch": 21.3,
      "learning_rate": 1.2743181818181816e-05,
      "loss": 2.6942,
      "step": 366650
    },
    {
      "activations/layer0_attention_weight_max": 15.604148864746094,
      "activations/layer0_attention_weight_min": -14.63058853149414,
      "activations/layer10_attention_weight_max": 37.5114631652832,
      "activations/layer10_attention_weight_min": -36.134490966796875,
      "activations/layer11_attention_weight_max": 33.11109924316406,
      "activations/layer11_attention_weight_min": -34.84225082397461,
      "activations/layer12_attention_weight_max": 19.16963768005371,
      "activations/layer12_attention_weight_min": -26.445453643798828,
      "activations/layer13_attention_weight_max": 32.00183868408203,
      "activations/layer13_attention_weight_min": -26.368654251098633,
      "activations/layer14_attention_weight_max": 33.93791961669922,
      "activations/layer14_attention_weight_min": -30.396942138671875,
      "activations/layer15_attention_weight_max": 33.50353240966797,
      "activations/layer15_attention_weight_min": -31.322498321533203,
      "activations/layer16_attention_weight_max": 35.95798110961914,
      "activations/layer16_attention_weight_min": -34.79617691040039,
      "activations/layer17_attention_weight_max": 55.03927230834961,
      "activations/layer17_attention_weight_min": -46.9086799621582,
      "activations/layer18_attention_weight_max": 48.36479949951172,
      "activations/layer18_attention_weight_min": -40.60845947265625,
      "activations/layer19_attention_weight_max": 23.233224868774414,
      "activations/layer19_attention_weight_min": -20.914119720458984,
      "activations/layer1_attention_weight_max": 15.246818542480469,
      "activations/layer1_attention_weight_min": -13.764300346374512,
      "activations/layer20_attention_weight_max": 21.96619987487793,
      "activations/layer20_attention_weight_min": -21.654380798339844,
      "activations/layer21_attention_weight_max": 31.05051040649414,
      "activations/layer21_attention_weight_min": -20.21778106689453,
      "activations/layer22_attention_weight_max": 27.911693572998047,
      "activations/layer22_attention_weight_min": -25.463485717773438,
      "activations/layer23_attention_weight_max": 34.915924072265625,
      "activations/layer23_attention_weight_min": -22.12611198425293,
      "activations/layer2_attention_weight_max": 31.621456146240234,
      "activations/layer2_attention_weight_min": -31.15666961669922,
      "activations/layer3_attention_weight_max": 88.54498291015625,
      "activations/layer3_attention_weight_min": -88.345947265625,
      "activations/layer4_attention_weight_max": 83.19502258300781,
      "activations/layer4_attention_weight_min": -84.37557983398438,
      "activations/layer5_attention_weight_max": 60.607295989990234,
      "activations/layer5_attention_weight_min": -69.06426239013672,
      "activations/layer6_attention_weight_max": 49.353843688964844,
      "activations/layer6_attention_weight_min": -49.2487678527832,
      "activations/layer7_attention_weight_max": 69.86405944824219,
      "activations/layer7_attention_weight_min": -64.03167724609375,
      "activations/layer8_attention_weight_max": 50.05678176879883,
      "activations/layer8_attention_weight_min": -52.81145095825195,
      "activations/layer9_attention_weight_max": 42.732662200927734,
      "activations/layer9_attention_weight_min": -45.03664016723633,
      "epoch": 21.31,
      "learning_rate": 1.2724242424242422e-05,
      "loss": 2.6942,
      "step": 366700
    },
    {
      "activations/layer0_attention_weight_max": 15.968640327453613,
      "activations/layer0_attention_weight_min": -14.426798820495605,
      "activations/layer10_attention_weight_max": 37.46318817138672,
      "activations/layer10_attention_weight_min": -39.31464767456055,
      "activations/layer11_attention_weight_max": 35.854949951171875,
      "activations/layer11_attention_weight_min": -36.89808654785156,
      "activations/layer12_attention_weight_max": 19.85479164123535,
      "activations/layer12_attention_weight_min": -25.728788375854492,
      "activations/layer13_attention_weight_max": 36.45713806152344,
      "activations/layer13_attention_weight_min": -27.92873191833496,
      "activations/layer14_attention_weight_max": 38.62739181518555,
      "activations/layer14_attention_weight_min": -33.29136276245117,
      "activations/layer15_attention_weight_max": 37.7832145690918,
      "activations/layer15_attention_weight_min": -30.58491325378418,
      "activations/layer16_attention_weight_max": 38.38676452636719,
      "activations/layer16_attention_weight_min": -34.26679992675781,
      "activations/layer17_attention_weight_max": 55.10219192504883,
      "activations/layer17_attention_weight_min": -46.60823440551758,
      "activations/layer18_attention_weight_max": 51.2363395690918,
      "activations/layer18_attention_weight_min": -41.52404022216797,
      "activations/layer19_attention_weight_max": 26.04965591430664,
      "activations/layer19_attention_weight_min": -23.444110870361328,
      "activations/layer1_attention_weight_max": 15.917389869689941,
      "activations/layer1_attention_weight_min": -13.526116371154785,
      "activations/layer20_attention_weight_max": 24.86882781982422,
      "activations/layer20_attention_weight_min": -20.635053634643555,
      "activations/layer21_attention_weight_max": 38.35129928588867,
      "activations/layer21_attention_weight_min": -22.49380874633789,
      "activations/layer22_attention_weight_max": 32.36963653564453,
      "activations/layer22_attention_weight_min": -25.26949119567871,
      "activations/layer23_attention_weight_max": 40.61944580078125,
      "activations/layer23_attention_weight_min": -23.251937866210938,
      "activations/layer2_attention_weight_max": 32.00294494628906,
      "activations/layer2_attention_weight_min": -30.3447265625,
      "activations/layer3_attention_weight_max": 97.22071838378906,
      "activations/layer3_attention_weight_min": -96.93128967285156,
      "activations/layer4_attention_weight_max": 92.08148956298828,
      "activations/layer4_attention_weight_min": -85.20487976074219,
      "activations/layer5_attention_weight_max": 62.696205139160156,
      "activations/layer5_attention_weight_min": -73.1399917602539,
      "activations/layer6_attention_weight_max": 52.67668533325195,
      "activations/layer6_attention_weight_min": -52.83889389038086,
      "activations/layer7_attention_weight_max": 70.21406555175781,
      "activations/layer7_attention_weight_min": -69.58263397216797,
      "activations/layer8_attention_weight_max": 51.35164260864258,
      "activations/layer8_attention_weight_min": -57.38768768310547,
      "activations/layer9_attention_weight_max": 47.58948516845703,
      "activations/layer9_attention_weight_min": -50.81690216064453,
      "epoch": 21.31,
      "learning_rate": 1.2705303030303029e-05,
      "loss": 2.7168,
      "step": 366750
    },
    {
      "activations/layer0_attention_weight_max": 16.859169006347656,
      "activations/layer0_attention_weight_min": -14.16680908203125,
      "activations/layer10_attention_weight_max": 32.99626922607422,
      "activations/layer10_attention_weight_min": -35.106910705566406,
      "activations/layer11_attention_weight_max": 31.215099334716797,
      "activations/layer11_attention_weight_min": -33.73350524902344,
      "activations/layer12_attention_weight_max": 17.90456771850586,
      "activations/layer12_attention_weight_min": -26.297697067260742,
      "activations/layer13_attention_weight_max": 30.213661193847656,
      "activations/layer13_attention_weight_min": -26.25594711303711,
      "activations/layer14_attention_weight_max": 33.74137496948242,
      "activations/layer14_attention_weight_min": -27.845491409301758,
      "activations/layer15_attention_weight_max": 31.891300201416016,
      "activations/layer15_attention_weight_min": -28.43581771850586,
      "activations/layer16_attention_weight_max": 33.25205993652344,
      "activations/layer16_attention_weight_min": -31.58580207824707,
      "activations/layer17_attention_weight_max": 49.444358825683594,
      "activations/layer17_attention_weight_min": -44.276329040527344,
      "activations/layer18_attention_weight_max": 46.09385299682617,
      "activations/layer18_attention_weight_min": -39.471229553222656,
      "activations/layer19_attention_weight_max": 25.668493270874023,
      "activations/layer19_attention_weight_min": -23.462810516357422,
      "activations/layer1_attention_weight_max": 15.328827857971191,
      "activations/layer1_attention_weight_min": -14.136117935180664,
      "activations/layer20_attention_weight_max": 23.32124137878418,
      "activations/layer20_attention_weight_min": -22.318763732910156,
      "activations/layer21_attention_weight_max": 36.50564956665039,
      "activations/layer21_attention_weight_min": -23.987045288085938,
      "activations/layer22_attention_weight_max": 27.40803337097168,
      "activations/layer22_attention_weight_min": -25.153100967407227,
      "activations/layer23_attention_weight_max": 34.688194274902344,
      "activations/layer23_attention_weight_min": -27.230876922607422,
      "activations/layer2_attention_weight_max": 33.00413513183594,
      "activations/layer2_attention_weight_min": -30.71396255493164,
      "activations/layer3_attention_weight_max": 85.4727783203125,
      "activations/layer3_attention_weight_min": -84.65380859375,
      "activations/layer4_attention_weight_max": 84.1686019897461,
      "activations/layer4_attention_weight_min": -83.1189193725586,
      "activations/layer5_attention_weight_max": 59.61012268066406,
      "activations/layer5_attention_weight_min": -68.81204223632812,
      "activations/layer6_attention_weight_max": 47.42246627807617,
      "activations/layer6_attention_weight_min": -48.520423889160156,
      "activations/layer7_attention_weight_max": 62.0533561706543,
      "activations/layer7_attention_weight_min": -65.34188079833984,
      "activations/layer8_attention_weight_max": 46.464717864990234,
      "activations/layer8_attention_weight_min": -46.672645568847656,
      "activations/layer9_attention_weight_max": 41.54948043823242,
      "activations/layer9_attention_weight_min": -44.259559631347656,
      "epoch": 21.31,
      "learning_rate": 1.2686363636363635e-05,
      "loss": 2.6895,
      "step": 366800
    },
    {
      "activations/layer0_attention_weight_max": 15.014066696166992,
      "activations/layer0_attention_weight_min": -14.49853229522705,
      "activations/layer10_attention_weight_max": 35.275184631347656,
      "activations/layer10_attention_weight_min": -36.15251159667969,
      "activations/layer11_attention_weight_max": 34.541107177734375,
      "activations/layer11_attention_weight_min": -34.036136627197266,
      "activations/layer12_attention_weight_max": 20.1345272064209,
      "activations/layer12_attention_weight_min": -22.4406795501709,
      "activations/layer13_attention_weight_max": 36.9645881652832,
      "activations/layer13_attention_weight_min": -29.36318016052246,
      "activations/layer14_attention_weight_max": 40.81480026245117,
      "activations/layer14_attention_weight_min": -31.06367301940918,
      "activations/layer15_attention_weight_max": 36.43617248535156,
      "activations/layer15_attention_weight_min": -32.23135757446289,
      "activations/layer16_attention_weight_max": 35.88117980957031,
      "activations/layer16_attention_weight_min": -35.78040313720703,
      "activations/layer17_attention_weight_max": 53.95534133911133,
      "activations/layer17_attention_weight_min": -47.319480895996094,
      "activations/layer18_attention_weight_max": 55.00199890136719,
      "activations/layer18_attention_weight_min": -42.097572326660156,
      "activations/layer19_attention_weight_max": 25.15117835998535,
      "activations/layer19_attention_weight_min": -22.668996810913086,
      "activations/layer1_attention_weight_max": 15.028800964355469,
      "activations/layer1_attention_weight_min": -13.766860961914062,
      "activations/layer20_attention_weight_max": 24.597761154174805,
      "activations/layer20_attention_weight_min": -20.172239303588867,
      "activations/layer21_attention_weight_max": 47.084983825683594,
      "activations/layer21_attention_weight_min": -21.771835327148438,
      "activations/layer22_attention_weight_max": 30.37921142578125,
      "activations/layer22_attention_weight_min": -24.669559478759766,
      "activations/layer23_attention_weight_max": 41.34598922729492,
      "activations/layer23_attention_weight_min": -22.380292892456055,
      "activations/layer2_attention_weight_max": 30.914297103881836,
      "activations/layer2_attention_weight_min": -30.322349548339844,
      "activations/layer3_attention_weight_max": 90.56828308105469,
      "activations/layer3_attention_weight_min": -90.3780746459961,
      "activations/layer4_attention_weight_max": 82.726806640625,
      "activations/layer4_attention_weight_min": -81.3258285522461,
      "activations/layer5_attention_weight_max": 59.755977630615234,
      "activations/layer5_attention_weight_min": -68.14518737792969,
      "activations/layer6_attention_weight_max": 49.22636032104492,
      "activations/layer6_attention_weight_min": -48.105323791503906,
      "activations/layer7_attention_weight_max": 70.4356460571289,
      "activations/layer7_attention_weight_min": -64.90510559082031,
      "activations/layer8_attention_weight_max": 50.308692932128906,
      "activations/layer8_attention_weight_min": -50.23908996582031,
      "activations/layer9_attention_weight_max": 53.03120040893555,
      "activations/layer9_attention_weight_min": -46.38396453857422,
      "epoch": 21.32,
      "learning_rate": 1.2667424242424242e-05,
      "loss": 2.713,
      "step": 366850
    },
    {
      "activations/layer0_attention_weight_max": 16.391868591308594,
      "activations/layer0_attention_weight_min": -14.61690616607666,
      "activations/layer10_attention_weight_max": 34.1936149597168,
      "activations/layer10_attention_weight_min": -35.86410903930664,
      "activations/layer11_attention_weight_max": 32.926429748535156,
      "activations/layer11_attention_weight_min": -34.54776382446289,
      "activations/layer12_attention_weight_max": 19.336284637451172,
      "activations/layer12_attention_weight_min": -22.645965576171875,
      "activations/layer13_attention_weight_max": 31.619129180908203,
      "activations/layer13_attention_weight_min": -25.564502716064453,
      "activations/layer14_attention_weight_max": 35.701072692871094,
      "activations/layer14_attention_weight_min": -28.87240219116211,
      "activations/layer15_attention_weight_max": 35.84165573120117,
      "activations/layer15_attention_weight_min": -29.638063430786133,
      "activations/layer16_attention_weight_max": 34.46022033691406,
      "activations/layer16_attention_weight_min": -32.213382720947266,
      "activations/layer17_attention_weight_max": 56.98394775390625,
      "activations/layer17_attention_weight_min": -45.89640426635742,
      "activations/layer18_attention_weight_max": 46.73209762573242,
      "activations/layer18_attention_weight_min": -40.954402923583984,
      "activations/layer19_attention_weight_max": 25.36286735534668,
      "activations/layer19_attention_weight_min": -21.464277267456055,
      "activations/layer1_attention_weight_max": 16.33687400817871,
      "activations/layer1_attention_weight_min": -13.797151565551758,
      "activations/layer20_attention_weight_max": 22.144149780273438,
      "activations/layer20_attention_weight_min": -21.381298065185547,
      "activations/layer21_attention_weight_max": 31.83067512512207,
      "activations/layer21_attention_weight_min": -21.283483505249023,
      "activations/layer22_attention_weight_max": 27.875797271728516,
      "activations/layer22_attention_weight_min": -25.93950653076172,
      "activations/layer23_attention_weight_max": 37.67216491699219,
      "activations/layer23_attention_weight_min": -22.848175048828125,
      "activations/layer2_attention_weight_max": 31.800384521484375,
      "activations/layer2_attention_weight_min": -29.78191375732422,
      "activations/layer3_attention_weight_max": 84.1227035522461,
      "activations/layer3_attention_weight_min": -83.03535461425781,
      "activations/layer4_attention_weight_max": 82.74168395996094,
      "activations/layer4_attention_weight_min": -79.94049072265625,
      "activations/layer5_attention_weight_max": 61.779380798339844,
      "activations/layer5_attention_weight_min": -70.80499267578125,
      "activations/layer6_attention_weight_max": 48.59829330444336,
      "activations/layer6_attention_weight_min": -47.849830627441406,
      "activations/layer7_attention_weight_max": 65.5849380493164,
      "activations/layer7_attention_weight_min": -64.34321594238281,
      "activations/layer8_attention_weight_max": 47.23158645629883,
      "activations/layer8_attention_weight_min": -48.049560546875,
      "activations/layer9_attention_weight_max": 41.442691802978516,
      "activations/layer9_attention_weight_min": -44.45263671875,
      "epoch": 21.32,
      "learning_rate": 1.2648484848484849e-05,
      "loss": 2.7218,
      "step": 366900
    },
    {
      "activations/layer0_attention_weight_max": 15.467111587524414,
      "activations/layer0_attention_weight_min": -14.160340309143066,
      "activations/layer10_attention_weight_max": 38.35592269897461,
      "activations/layer10_attention_weight_min": -38.4914436340332,
      "activations/layer11_attention_weight_max": 35.92045974731445,
      "activations/layer11_attention_weight_min": -37.487388610839844,
      "activations/layer12_attention_weight_max": 19.970661163330078,
      "activations/layer12_attention_weight_min": -23.771133422851562,
      "activations/layer13_attention_weight_max": 39.67611312866211,
      "activations/layer13_attention_weight_min": -28.040313720703125,
      "activations/layer14_attention_weight_max": 37.07398986816406,
      "activations/layer14_attention_weight_min": -31.41374969482422,
      "activations/layer15_attention_weight_max": 36.24665832519531,
      "activations/layer15_attention_weight_min": -31.501676559448242,
      "activations/layer16_attention_weight_max": 37.35938262939453,
      "activations/layer16_attention_weight_min": -34.332244873046875,
      "activations/layer17_attention_weight_max": 53.61341094970703,
      "activations/layer17_attention_weight_min": -47.269840240478516,
      "activations/layer18_attention_weight_max": 49.77863311767578,
      "activations/layer18_attention_weight_min": -42.17788314819336,
      "activations/layer19_attention_weight_max": 24.162256240844727,
      "activations/layer19_attention_weight_min": -25.549726486206055,
      "activations/layer1_attention_weight_max": 15.34031867980957,
      "activations/layer1_attention_weight_min": -14.193865776062012,
      "activations/layer20_attention_weight_max": 23.20208740234375,
      "activations/layer20_attention_weight_min": -25.521329879760742,
      "activations/layer21_attention_weight_max": 41.27230453491211,
      "activations/layer21_attention_weight_min": -24.60063362121582,
      "activations/layer22_attention_weight_max": 29.352863311767578,
      "activations/layer22_attention_weight_min": -27.919477462768555,
      "activations/layer23_attention_weight_max": 35.93446731567383,
      "activations/layer23_attention_weight_min": -24.83997344970703,
      "activations/layer2_attention_weight_max": 32.47909927368164,
      "activations/layer2_attention_weight_min": -30.775379180908203,
      "activations/layer3_attention_weight_max": 89.54106903076172,
      "activations/layer3_attention_weight_min": -87.96575927734375,
      "activations/layer4_attention_weight_max": 86.33109283447266,
      "activations/layer4_attention_weight_min": -84.75108337402344,
      "activations/layer5_attention_weight_max": 62.896324157714844,
      "activations/layer5_attention_weight_min": -69.99554443359375,
      "activations/layer6_attention_weight_max": 51.76345443725586,
      "activations/layer6_attention_weight_min": -53.89826965332031,
      "activations/layer7_attention_weight_max": 67.21129608154297,
      "activations/layer7_attention_weight_min": -69.81401062011719,
      "activations/layer8_attention_weight_max": 49.551177978515625,
      "activations/layer8_attention_weight_min": -53.23007583618164,
      "activations/layer9_attention_weight_max": 47.343353271484375,
      "activations/layer9_attention_weight_min": -49.238929748535156,
      "epoch": 21.32,
      "learning_rate": 1.2629545454545454e-05,
      "loss": 2.7215,
      "step": 366950
    },
    {
      "activations/layer0_attention_weight_max": 15.575501441955566,
      "activations/layer0_attention_weight_min": -14.136598587036133,
      "activations/layer10_attention_weight_max": 34.73002243041992,
      "activations/layer10_attention_weight_min": -35.550384521484375,
      "activations/layer11_attention_weight_max": 31.830747604370117,
      "activations/layer11_attention_weight_min": -35.475799560546875,
      "activations/layer12_attention_weight_max": 19.51434326171875,
      "activations/layer12_attention_weight_min": -26.29541778564453,
      "activations/layer13_attention_weight_max": 32.89504623413086,
      "activations/layer13_attention_weight_min": -26.034757614135742,
      "activations/layer14_attention_weight_max": 33.493221282958984,
      "activations/layer14_attention_weight_min": -28.283369064331055,
      "activations/layer15_attention_weight_max": 36.18701171875,
      "activations/layer15_attention_weight_min": -28.84600830078125,
      "activations/layer16_attention_weight_max": 33.8670654296875,
      "activations/layer16_attention_weight_min": -31.542064666748047,
      "activations/layer17_attention_weight_max": 51.313262939453125,
      "activations/layer17_attention_weight_min": -44.409053802490234,
      "activations/layer18_attention_weight_max": 48.43598937988281,
      "activations/layer18_attention_weight_min": -43.57209777832031,
      "activations/layer19_attention_weight_max": 24.9781551361084,
      "activations/layer19_attention_weight_min": -22.8200740814209,
      "activations/layer1_attention_weight_max": 15.728243827819824,
      "activations/layer1_attention_weight_min": -14.474300384521484,
      "activations/layer20_attention_weight_max": 22.114336013793945,
      "activations/layer20_attention_weight_min": -21.76447296142578,
      "activations/layer21_attention_weight_max": 35.52882385253906,
      "activations/layer21_attention_weight_min": -24.70230484008789,
      "activations/layer22_attention_weight_max": 28.469329833984375,
      "activations/layer22_attention_weight_min": -25.174104690551758,
      "activations/layer23_attention_weight_max": 39.570011138916016,
      "activations/layer23_attention_weight_min": -24.54568099975586,
      "activations/layer2_attention_weight_max": 31.61513900756836,
      "activations/layer2_attention_weight_min": -29.997055053710938,
      "activations/layer3_attention_weight_max": 85.77555847167969,
      "activations/layer3_attention_weight_min": -84.93109893798828,
      "activations/layer4_attention_weight_max": 86.42118835449219,
      "activations/layer4_attention_weight_min": -84.05657196044922,
      "activations/layer5_attention_weight_max": 60.215450286865234,
      "activations/layer5_attention_weight_min": -73.0347671508789,
      "activations/layer6_attention_weight_max": 48.818111419677734,
      "activations/layer6_attention_weight_min": -51.02571487426758,
      "activations/layer7_attention_weight_max": 64.92254638671875,
      "activations/layer7_attention_weight_min": -64.17803955078125,
      "activations/layer8_attention_weight_max": 45.61516571044922,
      "activations/layer8_attention_weight_min": -49.26707458496094,
      "activations/layer9_attention_weight_max": 42.29740905761719,
      "activations/layer9_attention_weight_min": -44.238853454589844,
      "epoch": 21.32,
      "learning_rate": 1.261060606060606e-05,
      "loss": 2.7036,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4428,
      "eval_samples_per_second": 508.601,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4428,
      "eval_openwebtext_samples_per_second": 508.601,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9446,
      "eval_wikitext_samples_per_second": 234.499,
      "step": 367000
    },
    {
      "epoch": 21.32,
      "eval_lambada_loss": 2.392578125,
      "eval_lambada_ppl": 10.941666595318791,
      "eval_lambada_runtime": 9.5739,
      "eval_lambada_samples_per_second": 508.57,
      "step": 367000
    },
    {
      "activations/layer0_attention_weight_max": 15.382929801940918,
      "activations/layer0_attention_weight_min": -14.535103797912598,
      "activations/layer10_attention_weight_max": 33.71421432495117,
      "activations/layer10_attention_weight_min": -35.1301155090332,
      "activations/layer11_attention_weight_max": 30.934616088867188,
      "activations/layer11_attention_weight_min": -34.39038848876953,
      "activations/layer12_attention_weight_max": 20.439529418945312,
      "activations/layer12_attention_weight_min": -26.21393394470215,
      "activations/layer13_attention_weight_max": 33.6794548034668,
      "activations/layer13_attention_weight_min": -27.912921905517578,
      "activations/layer14_attention_weight_max": 35.133331298828125,
      "activations/layer14_attention_weight_min": -30.23032569885254,
      "activations/layer15_attention_weight_max": 32.370033264160156,
      "activations/layer15_attention_weight_min": -31.213611602783203,
      "activations/layer16_attention_weight_max": 35.018028259277344,
      "activations/layer16_attention_weight_min": -34.69874954223633,
      "activations/layer17_attention_weight_max": 53.19379425048828,
      "activations/layer17_attention_weight_min": -45.782440185546875,
      "activations/layer18_attention_weight_max": 47.37451934814453,
      "activations/layer18_attention_weight_min": -41.705814361572266,
      "activations/layer19_attention_weight_max": 25.817737579345703,
      "activations/layer19_attention_weight_min": -24.25462532043457,
      "activations/layer1_attention_weight_max": 16.134199142456055,
      "activations/layer1_attention_weight_min": -14.667325019836426,
      "activations/layer20_attention_weight_max": 25.96035385131836,
      "activations/layer20_attention_weight_min": -24.90575408935547,
      "activations/layer21_attention_weight_max": 37.4931755065918,
      "activations/layer21_attention_weight_min": -25.82388687133789,
      "activations/layer22_attention_weight_max": 29.124149322509766,
      "activations/layer22_attention_weight_min": -25.91480827331543,
      "activations/layer23_attention_weight_max": 34.42835235595703,
      "activations/layer23_attention_weight_min": -24.234939575195312,
      "activations/layer2_attention_weight_max": 29.64557456970215,
      "activations/layer2_attention_weight_min": -28.636734008789062,
      "activations/layer3_attention_weight_max": 88.46015930175781,
      "activations/layer3_attention_weight_min": -86.55889129638672,
      "activations/layer4_attention_weight_max": 81.73867797851562,
      "activations/layer4_attention_weight_min": -83.65452575683594,
      "activations/layer5_attention_weight_max": 59.05341720581055,
      "activations/layer5_attention_weight_min": -72.29126739501953,
      "activations/layer6_attention_weight_max": 46.6599235534668,
      "activations/layer6_attention_weight_min": -49.260616302490234,
      "activations/layer7_attention_weight_max": 63.84657669067383,
      "activations/layer7_attention_weight_min": -65.13426208496094,
      "activations/layer8_attention_weight_max": 45.35419464111328,
      "activations/layer8_attention_weight_min": -51.1578369140625,
      "activations/layer9_attention_weight_max": 40.19350814819336,
      "activations/layer9_attention_weight_min": -44.335811614990234,
      "epoch": 21.33,
      "learning_rate": 1.2591666666666667e-05,
      "loss": 2.7158,
      "step": 367050
    },
    {
      "activations/layer0_attention_weight_max": 15.794391632080078,
      "activations/layer0_attention_weight_min": -14.667206764221191,
      "activations/layer10_attention_weight_max": 34.245872497558594,
      "activations/layer10_attention_weight_min": -35.600093841552734,
      "activations/layer11_attention_weight_max": 32.764854431152344,
      "activations/layer11_attention_weight_min": -35.43838882446289,
      "activations/layer12_attention_weight_max": 19.819570541381836,
      "activations/layer12_attention_weight_min": -26.209213256835938,
      "activations/layer13_attention_weight_max": 33.87917709350586,
      "activations/layer13_attention_weight_min": -27.509780883789062,
      "activations/layer14_attention_weight_max": 35.092864990234375,
      "activations/layer14_attention_weight_min": -32.55502700805664,
      "activations/layer15_attention_weight_max": 33.063209533691406,
      "activations/layer15_attention_weight_min": -31.399112701416016,
      "activations/layer16_attention_weight_max": 33.82122802734375,
      "activations/layer16_attention_weight_min": -31.2490234375,
      "activations/layer17_attention_weight_max": 51.223602294921875,
      "activations/layer17_attention_weight_min": -44.169063568115234,
      "activations/layer18_attention_weight_max": 47.92864227294922,
      "activations/layer18_attention_weight_min": -41.046451568603516,
      "activations/layer19_attention_weight_max": 26.407339096069336,
      "activations/layer19_attention_weight_min": -23.95201873779297,
      "activations/layer1_attention_weight_max": 14.657392501831055,
      "activations/layer1_attention_weight_min": -12.586922645568848,
      "activations/layer20_attention_weight_max": 22.75776481628418,
      "activations/layer20_attention_weight_min": -22.03141975402832,
      "activations/layer21_attention_weight_max": 35.353515625,
      "activations/layer21_attention_weight_min": -23.652490615844727,
      "activations/layer22_attention_weight_max": 27.26413345336914,
      "activations/layer22_attention_weight_min": -26.754955291748047,
      "activations/layer23_attention_weight_max": 34.48200988769531,
      "activations/layer23_attention_weight_min": -30.213857650756836,
      "activations/layer2_attention_weight_max": 29.067129135131836,
      "activations/layer2_attention_weight_min": -28.277957916259766,
      "activations/layer3_attention_weight_max": 83.2847900390625,
      "activations/layer3_attention_weight_min": -84.30197143554688,
      "activations/layer4_attention_weight_max": 80.7252426147461,
      "activations/layer4_attention_weight_min": -80.33621215820312,
      "activations/layer5_attention_weight_max": 59.564483642578125,
      "activations/layer5_attention_weight_min": -72.34744262695312,
      "activations/layer6_attention_weight_max": 48.99259567260742,
      "activations/layer6_attention_weight_min": -48.515804290771484,
      "activations/layer7_attention_weight_max": 67.27997589111328,
      "activations/layer7_attention_weight_min": -66.748291015625,
      "activations/layer8_attention_weight_max": 46.91376495361328,
      "activations/layer8_attention_weight_min": -49.45295333862305,
      "activations/layer9_attention_weight_max": 42.13239288330078,
      "activations/layer9_attention_weight_min": -45.6672477722168,
      "epoch": 21.33,
      "learning_rate": 1.257310606060606e-05,
      "loss": 2.7208,
      "step": 367100
    },
    {
      "activations/layer0_attention_weight_max": 16.3325252532959,
      "activations/layer0_attention_weight_min": -14.614741325378418,
      "activations/layer10_attention_weight_max": 35.6546745300293,
      "activations/layer10_attention_weight_min": -36.434539794921875,
      "activations/layer11_attention_weight_max": 34.966880798339844,
      "activations/layer11_attention_weight_min": -34.74479675292969,
      "activations/layer12_attention_weight_max": 19.253372192382812,
      "activations/layer12_attention_weight_min": -23.8162841796875,
      "activations/layer13_attention_weight_max": 35.57036590576172,
      "activations/layer13_attention_weight_min": -29.78716278076172,
      "activations/layer14_attention_weight_max": 35.93531799316406,
      "activations/layer14_attention_weight_min": -32.50246047973633,
      "activations/layer15_attention_weight_max": 32.808650970458984,
      "activations/layer15_attention_weight_min": -33.07725524902344,
      "activations/layer16_attention_weight_max": 38.8629035949707,
      "activations/layer16_attention_weight_min": -38.70138168334961,
      "activations/layer17_attention_weight_max": 52.34789276123047,
      "activations/layer17_attention_weight_min": -50.71368408203125,
      "activations/layer18_attention_weight_max": 46.08492660522461,
      "activations/layer18_attention_weight_min": -44.82917404174805,
      "activations/layer19_attention_weight_max": 23.67007827758789,
      "activations/layer19_attention_weight_min": -25.958189010620117,
      "activations/layer1_attention_weight_max": 15.206032752990723,
      "activations/layer1_attention_weight_min": -14.117687225341797,
      "activations/layer20_attention_weight_max": 22.59789276123047,
      "activations/layer20_attention_weight_min": -22.80497932434082,
      "activations/layer21_attention_weight_max": 38.385135650634766,
      "activations/layer21_attention_weight_min": -25.964481353759766,
      "activations/layer22_attention_weight_max": 31.733444213867188,
      "activations/layer22_attention_weight_min": -27.00478744506836,
      "activations/layer23_attention_weight_max": 34.262855529785156,
      "activations/layer23_attention_weight_min": -24.68355369567871,
      "activations/layer2_attention_weight_max": 29.831649780273438,
      "activations/layer2_attention_weight_min": -28.828983306884766,
      "activations/layer3_attention_weight_max": 83.92324829101562,
      "activations/layer3_attention_weight_min": -86.8888931274414,
      "activations/layer4_attention_weight_max": 81.34276580810547,
      "activations/layer4_attention_weight_min": -81.73308563232422,
      "activations/layer5_attention_weight_max": 61.16779327392578,
      "activations/layer5_attention_weight_min": -70.19567108154297,
      "activations/layer6_attention_weight_max": 50.203304290771484,
      "activations/layer6_attention_weight_min": -51.99193572998047,
      "activations/layer7_attention_weight_max": 75.20692443847656,
      "activations/layer7_attention_weight_min": -68.48356628417969,
      "activations/layer8_attention_weight_max": 48.7154655456543,
      "activations/layer8_attention_weight_min": -50.942176818847656,
      "activations/layer9_attention_weight_max": 48.407291412353516,
      "activations/layer9_attention_weight_min": -50.54221725463867,
      "epoch": 21.33,
      "learning_rate": 1.2554166666666666e-05,
      "loss": 2.7107,
      "step": 367150
    },
    {
      "activations/layer0_attention_weight_max": 15.748076438903809,
      "activations/layer0_attention_weight_min": -14.100813865661621,
      "activations/layer10_attention_weight_max": 34.255680084228516,
      "activations/layer10_attention_weight_min": -35.016807556152344,
      "activations/layer11_attention_weight_max": 33.03599548339844,
      "activations/layer11_attention_weight_min": -35.204933166503906,
      "activations/layer12_attention_weight_max": 18.033817291259766,
      "activations/layer12_attention_weight_min": -25.050207138061523,
      "activations/layer13_attention_weight_max": 31.49698829650879,
      "activations/layer13_attention_weight_min": -29.210403442382812,
      "activations/layer14_attention_weight_max": 33.57914352416992,
      "activations/layer14_attention_weight_min": -29.500337600708008,
      "activations/layer15_attention_weight_max": 31.565738677978516,
      "activations/layer15_attention_weight_min": -29.52625846862793,
      "activations/layer16_attention_weight_max": 31.463788986206055,
      "activations/layer16_attention_weight_min": -30.285884857177734,
      "activations/layer17_attention_weight_max": 46.40007019042969,
      "activations/layer17_attention_weight_min": -42.48180389404297,
      "activations/layer18_attention_weight_max": 45.4695930480957,
      "activations/layer18_attention_weight_min": -39.24215316772461,
      "activations/layer19_attention_weight_max": 21.428213119506836,
      "activations/layer19_attention_weight_min": -21.727275848388672,
      "activations/layer1_attention_weight_max": 14.771963119506836,
      "activations/layer1_attention_weight_min": -13.824450492858887,
      "activations/layer20_attention_weight_max": 20.05841064453125,
      "activations/layer20_attention_weight_min": -20.559144973754883,
      "activations/layer21_attention_weight_max": 34.67591094970703,
      "activations/layer21_attention_weight_min": -21.84463119506836,
      "activations/layer22_attention_weight_max": 29.756669998168945,
      "activations/layer22_attention_weight_min": -24.192556381225586,
      "activations/layer23_attention_weight_max": 32.645599365234375,
      "activations/layer23_attention_weight_min": -25.90933609008789,
      "activations/layer2_attention_weight_max": 32.230064392089844,
      "activations/layer2_attention_weight_min": -31.19685935974121,
      "activations/layer3_attention_weight_max": 89.80060577392578,
      "activations/layer3_attention_weight_min": -90.31168365478516,
      "activations/layer4_attention_weight_max": 86.27024841308594,
      "activations/layer4_attention_weight_min": -85.75757598876953,
      "activations/layer5_attention_weight_max": 64.18631744384766,
      "activations/layer5_attention_weight_min": -76.37959289550781,
      "activations/layer6_attention_weight_max": 50.10645294189453,
      "activations/layer6_attention_weight_min": -50.91081619262695,
      "activations/layer7_attention_weight_max": 73.00931549072266,
      "activations/layer7_attention_weight_min": -67.43792724609375,
      "activations/layer8_attention_weight_max": 46.97477340698242,
      "activations/layer8_attention_weight_min": -49.501708984375,
      "activations/layer9_attention_weight_max": 43.37022018432617,
      "activations/layer9_attention_weight_min": -47.3128662109375,
      "epoch": 21.34,
      "learning_rate": 1.2535227272727272e-05,
      "loss": 2.7053,
      "step": 367200
    },
    {
      "activations/layer0_attention_weight_max": 16.0473690032959,
      "activations/layer0_attention_weight_min": -14.437469482421875,
      "activations/layer10_attention_weight_max": 33.913330078125,
      "activations/layer10_attention_weight_min": -34.688961029052734,
      "activations/layer11_attention_weight_max": 33.049072265625,
      "activations/layer11_attention_weight_min": -33.41578674316406,
      "activations/layer12_attention_weight_max": 19.091663360595703,
      "activations/layer12_attention_weight_min": -27.5262393951416,
      "activations/layer13_attention_weight_max": 34.87832260131836,
      "activations/layer13_attention_weight_min": -27.320226669311523,
      "activations/layer14_attention_weight_max": 36.48256301879883,
      "activations/layer14_attention_weight_min": -29.975391387939453,
      "activations/layer15_attention_weight_max": 34.2005615234375,
      "activations/layer15_attention_weight_min": -31.2906436920166,
      "activations/layer16_attention_weight_max": 34.244117736816406,
      "activations/layer16_attention_weight_min": -34.05348587036133,
      "activations/layer17_attention_weight_max": 53.493247985839844,
      "activations/layer17_attention_weight_min": -46.59891891479492,
      "activations/layer18_attention_weight_max": 47.01749038696289,
      "activations/layer18_attention_weight_min": -38.44254684448242,
      "activations/layer19_attention_weight_max": 24.580371856689453,
      "activations/layer19_attention_weight_min": -22.626171112060547,
      "activations/layer1_attention_weight_max": 15.869140625,
      "activations/layer1_attention_weight_min": -13.558196067810059,
      "activations/layer20_attention_weight_max": 23.036725997924805,
      "activations/layer20_attention_weight_min": -21.33922004699707,
      "activations/layer21_attention_weight_max": 32.83169937133789,
      "activations/layer21_attention_weight_min": -23.335660934448242,
      "activations/layer22_attention_weight_max": 27.17050552368164,
      "activations/layer22_attention_weight_min": -24.974498748779297,
      "activations/layer23_attention_weight_max": 36.260345458984375,
      "activations/layer23_attention_weight_min": -23.654361724853516,
      "activations/layer2_attention_weight_max": 31.21314811706543,
      "activations/layer2_attention_weight_min": -29.14598846435547,
      "activations/layer3_attention_weight_max": 89.1371078491211,
      "activations/layer3_attention_weight_min": -83.87763214111328,
      "activations/layer4_attention_weight_max": 83.30899047851562,
      "activations/layer4_attention_weight_min": -83.01512908935547,
      "activations/layer5_attention_weight_max": 60.32160568237305,
      "activations/layer5_attention_weight_min": -68.34611511230469,
      "activations/layer6_attention_weight_max": 48.095924377441406,
      "activations/layer6_attention_weight_min": -51.70210647583008,
      "activations/layer7_attention_weight_max": 70.02568054199219,
      "activations/layer7_attention_weight_min": -67.25238800048828,
      "activations/layer8_attention_weight_max": 48.476104736328125,
      "activations/layer8_attention_weight_min": -50.674617767333984,
      "activations/layer9_attention_weight_max": 44.8405647277832,
      "activations/layer9_attention_weight_min": -47.751220703125,
      "epoch": 21.34,
      "learning_rate": 1.2516287878787877e-05,
      "loss": 2.7013,
      "step": 367250
    },
    {
      "activations/layer0_attention_weight_max": 15.696906089782715,
      "activations/layer0_attention_weight_min": -14.121594429016113,
      "activations/layer10_attention_weight_max": 38.07304000854492,
      "activations/layer10_attention_weight_min": -38.45024490356445,
      "activations/layer11_attention_weight_max": 35.34154510498047,
      "activations/layer11_attention_weight_min": -33.60411834716797,
      "activations/layer12_attention_weight_max": 20.052488327026367,
      "activations/layer12_attention_weight_min": -23.52720832824707,
      "activations/layer13_attention_weight_max": 38.935211181640625,
      "activations/layer13_attention_weight_min": -28.447423934936523,
      "activations/layer14_attention_weight_max": 36.941558837890625,
      "activations/layer14_attention_weight_min": -30.648576736450195,
      "activations/layer15_attention_weight_max": 36.247798919677734,
      "activations/layer15_attention_weight_min": -31.01793670654297,
      "activations/layer16_attention_weight_max": 36.84269332885742,
      "activations/layer16_attention_weight_min": -35.78459930419922,
      "activations/layer17_attention_weight_max": 56.0797119140625,
      "activations/layer17_attention_weight_min": -47.156158447265625,
      "activations/layer18_attention_weight_max": 51.49478530883789,
      "activations/layer18_attention_weight_min": -40.93606948852539,
      "activations/layer19_attention_weight_max": 28.192230224609375,
      "activations/layer19_attention_weight_min": -23.492679595947266,
      "activations/layer1_attention_weight_max": 15.916993141174316,
      "activations/layer1_attention_weight_min": -14.359346389770508,
      "activations/layer20_attention_weight_max": 27.486881256103516,
      "activations/layer20_attention_weight_min": -20.856876373291016,
      "activations/layer21_attention_weight_max": 41.206417083740234,
      "activations/layer21_attention_weight_min": -23.18393898010254,
      "activations/layer22_attention_weight_max": 28.887420654296875,
      "activations/layer22_attention_weight_min": -24.3843994140625,
      "activations/layer23_attention_weight_max": 40.11175537109375,
      "activations/layer23_attention_weight_min": -22.64484405517578,
      "activations/layer2_attention_weight_max": 31.244190216064453,
      "activations/layer2_attention_weight_min": -29.34475326538086,
      "activations/layer3_attention_weight_max": 89.31871032714844,
      "activations/layer3_attention_weight_min": -84.70001220703125,
      "activations/layer4_attention_weight_max": 84.87808227539062,
      "activations/layer4_attention_weight_min": -81.5355224609375,
      "activations/layer5_attention_weight_max": 63.18891906738281,
      "activations/layer5_attention_weight_min": -68.01176452636719,
      "activations/layer6_attention_weight_max": 48.7950553894043,
      "activations/layer6_attention_weight_min": -50.253414154052734,
      "activations/layer7_attention_weight_max": 70.47160339355469,
      "activations/layer7_attention_weight_min": -66.3210678100586,
      "activations/layer8_attention_weight_max": 47.53529357910156,
      "activations/layer8_attention_weight_min": -51.236305236816406,
      "activations/layer9_attention_weight_max": 47.637290954589844,
      "activations/layer9_attention_weight_min": -48.113059997558594,
      "epoch": 21.34,
      "learning_rate": 1.2497348484848484e-05,
      "loss": 2.7107,
      "step": 367300
    },
    {
      "activations/layer0_attention_weight_max": 16.24515724182129,
      "activations/layer0_attention_weight_min": -14.072484970092773,
      "activations/layer10_attention_weight_max": 35.359004974365234,
      "activations/layer10_attention_weight_min": -37.840232849121094,
      "activations/layer11_attention_weight_max": 32.849639892578125,
      "activations/layer11_attention_weight_min": -36.815399169921875,
      "activations/layer12_attention_weight_max": 20.729875564575195,
      "activations/layer12_attention_weight_min": -26.52859115600586,
      "activations/layer13_attention_weight_max": 33.40806198120117,
      "activations/layer13_attention_weight_min": -27.66585922241211,
      "activations/layer14_attention_weight_max": 34.03298568725586,
      "activations/layer14_attention_weight_min": -29.54771614074707,
      "activations/layer15_attention_weight_max": 32.108062744140625,
      "activations/layer15_attention_weight_min": -30.087881088256836,
      "activations/layer16_attention_weight_max": 34.03248977661133,
      "activations/layer16_attention_weight_min": -32.38188552856445,
      "activations/layer17_attention_weight_max": 48.82780838012695,
      "activations/layer17_attention_weight_min": -46.633583068847656,
      "activations/layer18_attention_weight_max": 47.23613739013672,
      "activations/layer18_attention_weight_min": -41.70923614501953,
      "activations/layer19_attention_weight_max": 26.40570640563965,
      "activations/layer19_attention_weight_min": -26.238744735717773,
      "activations/layer1_attention_weight_max": 15.811795234680176,
      "activations/layer1_attention_weight_min": -14.468791961669922,
      "activations/layer20_attention_weight_max": 24.23064613342285,
      "activations/layer20_attention_weight_min": -22.612079620361328,
      "activations/layer21_attention_weight_max": 36.95556640625,
      "activations/layer21_attention_weight_min": -23.634666442871094,
      "activations/layer22_attention_weight_max": 28.232995986938477,
      "activations/layer22_attention_weight_min": -25.231138229370117,
      "activations/layer23_attention_weight_max": 40.137088775634766,
      "activations/layer23_attention_weight_min": -23.31678009033203,
      "activations/layer2_attention_weight_max": 31.411273956298828,
      "activations/layer2_attention_weight_min": -31.673110961914062,
      "activations/layer3_attention_weight_max": 84.41639709472656,
      "activations/layer3_attention_weight_min": -91.30744171142578,
      "activations/layer4_attention_weight_max": 80.48384094238281,
      "activations/layer4_attention_weight_min": -84.87858581542969,
      "activations/layer5_attention_weight_max": 60.11191940307617,
      "activations/layer5_attention_weight_min": -68.25320434570312,
      "activations/layer6_attention_weight_max": 47.635921478271484,
      "activations/layer6_attention_weight_min": -50.69438552856445,
      "activations/layer7_attention_weight_max": 66.21930694580078,
      "activations/layer7_attention_weight_min": -66.33624267578125,
      "activations/layer8_attention_weight_max": 44.69035720825195,
      "activations/layer8_attention_weight_min": -47.22872543334961,
      "activations/layer9_attention_weight_max": 42.768768310546875,
      "activations/layer9_attention_weight_min": -44.91219711303711,
      "epoch": 21.35,
      "learning_rate": 1.247840909090909e-05,
      "loss": 2.7144,
      "step": 367350
    },
    {
      "activations/layer0_attention_weight_max": 15.823296546936035,
      "activations/layer0_attention_weight_min": -14.233260154724121,
      "activations/layer10_attention_weight_max": 35.11930465698242,
      "activations/layer10_attention_weight_min": -35.95620346069336,
      "activations/layer11_attention_weight_max": 34.61418151855469,
      "activations/layer11_attention_weight_min": -35.96835708618164,
      "activations/layer12_attention_weight_max": 19.019155502319336,
      "activations/layer12_attention_weight_min": -22.756214141845703,
      "activations/layer13_attention_weight_max": 34.566959381103516,
      "activations/layer13_attention_weight_min": -28.17119598388672,
      "activations/layer14_attention_weight_max": 37.42542266845703,
      "activations/layer14_attention_weight_min": -34.96976852416992,
      "activations/layer15_attention_weight_max": 35.16060256958008,
      "activations/layer15_attention_weight_min": -32.130672454833984,
      "activations/layer16_attention_weight_max": 35.33053970336914,
      "activations/layer16_attention_weight_min": -35.05659103393555,
      "activations/layer17_attention_weight_max": 53.173728942871094,
      "activations/layer17_attention_weight_min": -46.934940338134766,
      "activations/layer18_attention_weight_max": 48.32614517211914,
      "activations/layer18_attention_weight_min": -42.006839752197266,
      "activations/layer19_attention_weight_max": 25.179738998413086,
      "activations/layer19_attention_weight_min": -22.06433868408203,
      "activations/layer1_attention_weight_max": 16.012107849121094,
      "activations/layer1_attention_weight_min": -14.35705280303955,
      "activations/layer20_attention_weight_max": 22.639087677001953,
      "activations/layer20_attention_weight_min": -21.68382453918457,
      "activations/layer21_attention_weight_max": 36.36835861206055,
      "activations/layer21_attention_weight_min": -23.42156982421875,
      "activations/layer22_attention_weight_max": 31.934749603271484,
      "activations/layer22_attention_weight_min": -25.18052101135254,
      "activations/layer23_attention_weight_max": 36.73309326171875,
      "activations/layer23_attention_weight_min": -23.74915885925293,
      "activations/layer2_attention_weight_max": 31.826190948486328,
      "activations/layer2_attention_weight_min": -31.64003562927246,
      "activations/layer3_attention_weight_max": 96.19854736328125,
      "activations/layer3_attention_weight_min": -92.46542358398438,
      "activations/layer4_attention_weight_max": 87.55260467529297,
      "activations/layer4_attention_weight_min": -85.96585083007812,
      "activations/layer5_attention_weight_max": 65.7190170288086,
      "activations/layer5_attention_weight_min": -71.15958404541016,
      "activations/layer6_attention_weight_max": 50.268043518066406,
      "activations/layer6_attention_weight_min": -52.06914138793945,
      "activations/layer7_attention_weight_max": 69.40155029296875,
      "activations/layer7_attention_weight_min": -70.3034896850586,
      "activations/layer8_attention_weight_max": 50.72523880004883,
      "activations/layer8_attention_weight_min": -51.34409713745117,
      "activations/layer9_attention_weight_max": 42.739158630371094,
      "activations/layer9_attention_weight_min": -46.813472747802734,
      "epoch": 21.35,
      "learning_rate": 1.2459469696969695e-05,
      "loss": 2.7243,
      "step": 367400
    },
    {
      "activations/layer0_attention_weight_max": 16.462488174438477,
      "activations/layer0_attention_weight_min": -14.1734619140625,
      "activations/layer10_attention_weight_max": 35.71502685546875,
      "activations/layer10_attention_weight_min": -33.90264129638672,
      "activations/layer11_attention_weight_max": 33.25569152832031,
      "activations/layer11_attention_weight_min": -33.39733123779297,
      "activations/layer12_attention_weight_max": 19.137540817260742,
      "activations/layer12_attention_weight_min": -25.001235961914062,
      "activations/layer13_attention_weight_max": 34.86363220214844,
      "activations/layer13_attention_weight_min": -26.917865753173828,
      "activations/layer14_attention_weight_max": 36.08646774291992,
      "activations/layer14_attention_weight_min": -31.26515007019043,
      "activations/layer15_attention_weight_max": 33.814579010009766,
      "activations/layer15_attention_weight_min": -31.15638542175293,
      "activations/layer16_attention_weight_max": 35.01259231567383,
      "activations/layer16_attention_weight_min": -30.48250389099121,
      "activations/layer17_attention_weight_max": 49.126014709472656,
      "activations/layer17_attention_weight_min": -42.965694427490234,
      "activations/layer18_attention_weight_max": 45.50995635986328,
      "activations/layer18_attention_weight_min": -37.96718978881836,
      "activations/layer19_attention_weight_max": 23.58656883239746,
      "activations/layer19_attention_weight_min": -23.6181697845459,
      "activations/layer1_attention_weight_max": 15.777594566345215,
      "activations/layer1_attention_weight_min": -15.747206687927246,
      "activations/layer20_attention_weight_max": 22.50457191467285,
      "activations/layer20_attention_weight_min": -21.275901794433594,
      "activations/layer21_attention_weight_max": 34.93048095703125,
      "activations/layer21_attention_weight_min": -23.001346588134766,
      "activations/layer22_attention_weight_max": 28.36475372314453,
      "activations/layer22_attention_weight_min": -26.006916046142578,
      "activations/layer23_attention_weight_max": 35.61376953125,
      "activations/layer23_attention_weight_min": -23.637584686279297,
      "activations/layer2_attention_weight_max": 31.730100631713867,
      "activations/layer2_attention_weight_min": -30.330829620361328,
      "activations/layer3_attention_weight_max": 89.86991882324219,
      "activations/layer3_attention_weight_min": -87.38462829589844,
      "activations/layer4_attention_weight_max": 80.83784484863281,
      "activations/layer4_attention_weight_min": -82.37335205078125,
      "activations/layer5_attention_weight_max": 59.928611755371094,
      "activations/layer5_attention_weight_min": -68.49824523925781,
      "activations/layer6_attention_weight_max": 48.30907440185547,
      "activations/layer6_attention_weight_min": -46.87019348144531,
      "activations/layer7_attention_weight_max": 67.11949920654297,
      "activations/layer7_attention_weight_min": -63.24825668334961,
      "activations/layer8_attention_weight_max": 46.52964401245117,
      "activations/layer8_attention_weight_min": -48.42459487915039,
      "activations/layer9_attention_weight_max": 44.088050842285156,
      "activations/layer9_attention_weight_min": -44.41748809814453,
      "epoch": 21.35,
      "learning_rate": 1.2440530303030302e-05,
      "loss": 2.709,
      "step": 367450
    },
    {
      "activations/layer0_attention_weight_max": 15.760272026062012,
      "activations/layer0_attention_weight_min": -14.26024341583252,
      "activations/layer10_attention_weight_max": 40.22397232055664,
      "activations/layer10_attention_weight_min": -38.483890533447266,
      "activations/layer11_attention_weight_max": 36.74121856689453,
      "activations/layer11_attention_weight_min": -36.74732971191406,
      "activations/layer12_attention_weight_max": 22.533103942871094,
      "activations/layer12_attention_weight_min": -29.28777313232422,
      "activations/layer13_attention_weight_max": 36.0387077331543,
      "activations/layer13_attention_weight_min": -31.504796981811523,
      "activations/layer14_attention_weight_max": 37.64094924926758,
      "activations/layer14_attention_weight_min": -30.088153839111328,
      "activations/layer15_attention_weight_max": 34.628360748291016,
      "activations/layer15_attention_weight_min": -29.925296783447266,
      "activations/layer16_attention_weight_max": 38.31923294067383,
      "activations/layer16_attention_weight_min": -32.58926773071289,
      "activations/layer17_attention_weight_max": 53.690879821777344,
      "activations/layer17_attention_weight_min": -45.82377243041992,
      "activations/layer18_attention_weight_max": 50.52009963989258,
      "activations/layer18_attention_weight_min": -41.99250411987305,
      "activations/layer19_attention_weight_max": 26.142669677734375,
      "activations/layer19_attention_weight_min": -26.046049118041992,
      "activations/layer1_attention_weight_max": 16.732973098754883,
      "activations/layer1_attention_weight_min": -15.8953857421875,
      "activations/layer20_attention_weight_max": 26.652751922607422,
      "activations/layer20_attention_weight_min": -22.197660446166992,
      "activations/layer21_attention_weight_max": 39.44526672363281,
      "activations/layer21_attention_weight_min": -23.216278076171875,
      "activations/layer22_attention_weight_max": 30.833045959472656,
      "activations/layer22_attention_weight_min": -25.55617332458496,
      "activations/layer23_attention_weight_max": 44.248199462890625,
      "activations/layer23_attention_weight_min": -24.94363021850586,
      "activations/layer2_attention_weight_max": 32.09951400756836,
      "activations/layer2_attention_weight_min": -29.789274215698242,
      "activations/layer3_attention_weight_max": 89.05892181396484,
      "activations/layer3_attention_weight_min": -87.9945297241211,
      "activations/layer4_attention_weight_max": 83.15169525146484,
      "activations/layer4_attention_weight_min": -83.74176025390625,
      "activations/layer5_attention_weight_max": 60.396026611328125,
      "activations/layer5_attention_weight_min": -71.446533203125,
      "activations/layer6_attention_weight_max": 49.51461410522461,
      "activations/layer6_attention_weight_min": -51.41399383544922,
      "activations/layer7_attention_weight_max": 67.77106475830078,
      "activations/layer7_attention_weight_min": -69.5235824584961,
      "activations/layer8_attention_weight_max": 52.38979721069336,
      "activations/layer8_attention_weight_min": -56.68949890136719,
      "activations/layer9_attention_weight_max": 46.35009765625,
      "activations/layer9_attention_weight_min": -49.60354232788086,
      "epoch": 21.35,
      "learning_rate": 1.2421590909090907e-05,
      "loss": 2.7161,
      "step": 367500
    },
    {
      "activations/layer0_attention_weight_max": 15.559213638305664,
      "activations/layer0_attention_weight_min": -14.021016120910645,
      "activations/layer10_attention_weight_max": 34.315513610839844,
      "activations/layer10_attention_weight_min": -37.893028259277344,
      "activations/layer11_attention_weight_max": 33.86172866821289,
      "activations/layer11_attention_weight_min": -37.13044357299805,
      "activations/layer12_attention_weight_max": 19.27198600769043,
      "activations/layer12_attention_weight_min": -27.683731079101562,
      "activations/layer13_attention_weight_max": 30.2996883392334,
      "activations/layer13_attention_weight_min": -29.798707962036133,
      "activations/layer14_attention_weight_max": 33.308006286621094,
      "activations/layer14_attention_weight_min": -30.200593948364258,
      "activations/layer15_attention_weight_max": 30.96718978881836,
      "activations/layer15_attention_weight_min": -31.534404754638672,
      "activations/layer16_attention_weight_max": 34.47242736816406,
      "activations/layer16_attention_weight_min": -32.00300216674805,
      "activations/layer17_attention_weight_max": 50.30498123168945,
      "activations/layer17_attention_weight_min": -46.263267517089844,
      "activations/layer18_attention_weight_max": 44.65519332885742,
      "activations/layer18_attention_weight_min": -39.450469970703125,
      "activations/layer19_attention_weight_max": 24.492483139038086,
      "activations/layer19_attention_weight_min": -23.594083786010742,
      "activations/layer1_attention_weight_max": 15.30366039276123,
      "activations/layer1_attention_weight_min": -13.601571083068848,
      "activations/layer20_attention_weight_max": 20.743192672729492,
      "activations/layer20_attention_weight_min": -24.939115524291992,
      "activations/layer21_attention_weight_max": 33.093082427978516,
      "activations/layer21_attention_weight_min": -23.060745239257812,
      "activations/layer22_attention_weight_max": 27.08290672302246,
      "activations/layer22_attention_weight_min": -26.10808563232422,
      "activations/layer23_attention_weight_max": 32.94727325439453,
      "activations/layer23_attention_weight_min": -24.579017639160156,
      "activations/layer2_attention_weight_max": 32.708168029785156,
      "activations/layer2_attention_weight_min": -30.69232177734375,
      "activations/layer3_attention_weight_max": 89.09443664550781,
      "activations/layer3_attention_weight_min": -88.38565826416016,
      "activations/layer4_attention_weight_max": 83.53079223632812,
      "activations/layer4_attention_weight_min": -87.43294525146484,
      "activations/layer5_attention_weight_max": 60.374420166015625,
      "activations/layer5_attention_weight_min": -74.064697265625,
      "activations/layer6_attention_weight_max": 49.34770965576172,
      "activations/layer6_attention_weight_min": -51.25529098510742,
      "activations/layer7_attention_weight_max": 67.70306396484375,
      "activations/layer7_attention_weight_min": -67.2073974609375,
      "activations/layer8_attention_weight_max": 48.06011199951172,
      "activations/layer8_attention_weight_min": -50.35488510131836,
      "activations/layer9_attention_weight_max": 42.969539642333984,
      "activations/layer9_attention_weight_min": -45.73356246948242,
      "epoch": 21.36,
      "learning_rate": 1.2402651515151513e-05,
      "loss": 2.7035,
      "step": 367550
    },
    {
      "activations/layer0_attention_weight_max": 16.000093460083008,
      "activations/layer0_attention_weight_min": -14.205005645751953,
      "activations/layer10_attention_weight_max": 40.337928771972656,
      "activations/layer10_attention_weight_min": -39.17185592651367,
      "activations/layer11_attention_weight_max": 39.44615936279297,
      "activations/layer11_attention_weight_min": -38.04212188720703,
      "activations/layer12_attention_weight_max": 21.320255279541016,
      "activations/layer12_attention_weight_min": -25.3514461517334,
      "activations/layer13_attention_weight_max": 38.72657775878906,
      "activations/layer13_attention_weight_min": -28.978294372558594,
      "activations/layer14_attention_weight_max": 36.90021514892578,
      "activations/layer14_attention_weight_min": -30.92949676513672,
      "activations/layer15_attention_weight_max": 36.98477554321289,
      "activations/layer15_attention_weight_min": -31.434354782104492,
      "activations/layer16_attention_weight_max": 38.683414459228516,
      "activations/layer16_attention_weight_min": -36.41773223876953,
      "activations/layer17_attention_weight_max": 60.04497528076172,
      "activations/layer17_attention_weight_min": -53.05989456176758,
      "activations/layer18_attention_weight_max": 54.71652603149414,
      "activations/layer18_attention_weight_min": -46.187583923339844,
      "activations/layer19_attention_weight_max": 25.446481704711914,
      "activations/layer19_attention_weight_min": -22.857572555541992,
      "activations/layer1_attention_weight_max": 15.644813537597656,
      "activations/layer1_attention_weight_min": -14.131754875183105,
      "activations/layer20_attention_weight_max": 24.22688102722168,
      "activations/layer20_attention_weight_min": -21.351526260375977,
      "activations/layer21_attention_weight_max": 39.52608871459961,
      "activations/layer21_attention_weight_min": -21.146102905273438,
      "activations/layer22_attention_weight_max": 33.09989547729492,
      "activations/layer22_attention_weight_min": -24.31667709350586,
      "activations/layer23_attention_weight_max": 41.63166046142578,
      "activations/layer23_attention_weight_min": -24.777616500854492,
      "activations/layer2_attention_weight_max": 31.559864044189453,
      "activations/layer2_attention_weight_min": -31.331130981445312,
      "activations/layer3_attention_weight_max": 88.92957305908203,
      "activations/layer3_attention_weight_min": -91.15747833251953,
      "activations/layer4_attention_weight_max": 86.62179565429688,
      "activations/layer4_attention_weight_min": -87.05341339111328,
      "activations/layer5_attention_weight_max": 62.893714904785156,
      "activations/layer5_attention_weight_min": -70.28791046142578,
      "activations/layer6_attention_weight_max": 50.95315933227539,
      "activations/layer6_attention_weight_min": -54.8117790222168,
      "activations/layer7_attention_weight_max": 72.54830169677734,
      "activations/layer7_attention_weight_min": -66.69528198242188,
      "activations/layer8_attention_weight_max": 48.86518096923828,
      "activations/layer8_attention_weight_min": -51.24054718017578,
      "activations/layer9_attention_weight_max": 48.65388107299805,
      "activations/layer9_attention_weight_min": -46.36977005004883,
      "epoch": 21.36,
      "learning_rate": 1.238371212121212e-05,
      "loss": 2.7067,
      "step": 367600
    },
    {
      "activations/layer0_attention_weight_max": 16.580135345458984,
      "activations/layer0_attention_weight_min": -14.15504264831543,
      "activations/layer10_attention_weight_max": 34.97692108154297,
      "activations/layer10_attention_weight_min": -35.88477325439453,
      "activations/layer11_attention_weight_max": 34.26961135864258,
      "activations/layer11_attention_weight_min": -33.685855865478516,
      "activations/layer12_attention_weight_max": 19.620004653930664,
      "activations/layer12_attention_weight_min": -27.506120681762695,
      "activations/layer13_attention_weight_max": 34.6698112487793,
      "activations/layer13_attention_weight_min": -26.2829532623291,
      "activations/layer14_attention_weight_max": 36.786441802978516,
      "activations/layer14_attention_weight_min": -30.246761322021484,
      "activations/layer15_attention_weight_max": 34.90644836425781,
      "activations/layer15_attention_weight_min": -30.571182250976562,
      "activations/layer16_attention_weight_max": 37.62443542480469,
      "activations/layer16_attention_weight_min": -35.62821960449219,
      "activations/layer17_attention_weight_max": 55.36228942871094,
      "activations/layer17_attention_weight_min": -48.20101547241211,
      "activations/layer18_attention_weight_max": 52.313663482666016,
      "activations/layer18_attention_weight_min": -42.78257369995117,
      "activations/layer19_attention_weight_max": 25.73052406311035,
      "activations/layer19_attention_weight_min": -24.658618927001953,
      "activations/layer1_attention_weight_max": 19.602001190185547,
      "activations/layer1_attention_weight_min": -15.491450309753418,
      "activations/layer20_attention_weight_max": 23.466270446777344,
      "activations/layer20_attention_weight_min": -21.64094352722168,
      "activations/layer21_attention_weight_max": 32.748046875,
      "activations/layer21_attention_weight_min": -26.096891403198242,
      "activations/layer22_attention_weight_max": 30.22989273071289,
      "activations/layer22_attention_weight_min": -26.9090633392334,
      "activations/layer23_attention_weight_max": 34.31772232055664,
      "activations/layer23_attention_weight_min": -24.092830657958984,
      "activations/layer2_attention_weight_max": 31.30563735961914,
      "activations/layer2_attention_weight_min": -31.19683074951172,
      "activations/layer3_attention_weight_max": 90.95848846435547,
      "activations/layer3_attention_weight_min": -88.99095916748047,
      "activations/layer4_attention_weight_max": 85.59893035888672,
      "activations/layer4_attention_weight_min": -83.16655731201172,
      "activations/layer5_attention_weight_max": 60.879722595214844,
      "activations/layer5_attention_weight_min": -71.6238784790039,
      "activations/layer6_attention_weight_max": 50.0568962097168,
      "activations/layer6_attention_weight_min": -50.08911895751953,
      "activations/layer7_attention_weight_max": 65.78895568847656,
      "activations/layer7_attention_weight_min": -66.67813873291016,
      "activations/layer8_attention_weight_max": 46.34489822387695,
      "activations/layer8_attention_weight_min": -47.42311096191406,
      "activations/layer9_attention_weight_max": 44.8735466003418,
      "activations/layer9_attention_weight_min": -42.257328033447266,
      "epoch": 21.36,
      "learning_rate": 1.2364772727272725e-05,
      "loss": 2.706,
      "step": 367650
    },
    {
      "activations/layer0_attention_weight_max": 15.10788631439209,
      "activations/layer0_attention_weight_min": -14.373259544372559,
      "activations/layer10_attention_weight_max": 35.73273468017578,
      "activations/layer10_attention_weight_min": -36.132293701171875,
      "activations/layer11_attention_weight_max": 33.267242431640625,
      "activations/layer11_attention_weight_min": -35.663597106933594,
      "activations/layer12_attention_weight_max": 20.63730239868164,
      "activations/layer12_attention_weight_min": -27.088703155517578,
      "activations/layer13_attention_weight_max": 32.545440673828125,
      "activations/layer13_attention_weight_min": -28.1655330657959,
      "activations/layer14_attention_weight_max": 38.332191467285156,
      "activations/layer14_attention_weight_min": -30.721981048583984,
      "activations/layer15_attention_weight_max": 35.2364616394043,
      "activations/layer15_attention_weight_min": -31.237295150756836,
      "activations/layer16_attention_weight_max": 37.4560432434082,
      "activations/layer16_attention_weight_min": -35.04149627685547,
      "activations/layer17_attention_weight_max": 53.81086730957031,
      "activations/layer17_attention_weight_min": -47.41011047363281,
      "activations/layer18_attention_weight_max": 51.49230194091797,
      "activations/layer18_attention_weight_min": -41.604713439941406,
      "activations/layer19_attention_weight_max": 25.70000648498535,
      "activations/layer19_attention_weight_min": -23.577184677124023,
      "activations/layer1_attention_weight_max": 15.158479690551758,
      "activations/layer1_attention_weight_min": -14.711654663085938,
      "activations/layer20_attention_weight_max": 23.561473846435547,
      "activations/layer20_attention_weight_min": -22.104270935058594,
      "activations/layer21_attention_weight_max": 37.33967208862305,
      "activations/layer21_attention_weight_min": -23.712575912475586,
      "activations/layer22_attention_weight_max": 27.445072174072266,
      "activations/layer22_attention_weight_min": -25.352882385253906,
      "activations/layer23_attention_weight_max": 34.106834411621094,
      "activations/layer23_attention_weight_min": -26.04080581665039,
      "activations/layer2_attention_weight_max": 30.62324333190918,
      "activations/layer2_attention_weight_min": -29.521495819091797,
      "activations/layer3_attention_weight_max": 89.07474517822266,
      "activations/layer3_attention_weight_min": -87.37711334228516,
      "activations/layer4_attention_weight_max": 80.23316192626953,
      "activations/layer4_attention_weight_min": -84.05724334716797,
      "activations/layer5_attention_weight_max": 58.8109245300293,
      "activations/layer5_attention_weight_min": -69.35899353027344,
      "activations/layer6_attention_weight_max": 47.64694595336914,
      "activations/layer6_attention_weight_min": -50.7445182800293,
      "activations/layer7_attention_weight_max": 66.16534423828125,
      "activations/layer7_attention_weight_min": -68.90974426269531,
      "activations/layer8_attention_weight_max": 45.70061111450195,
      "activations/layer8_attention_weight_min": -51.12167739868164,
      "activations/layer9_attention_weight_max": 42.599239349365234,
      "activations/layer9_attention_weight_min": -46.297943115234375,
      "epoch": 21.37,
      "learning_rate": 1.2345833333333333e-05,
      "loss": 2.7171,
      "step": 367700
    },
    {
      "activations/layer0_attention_weight_max": 15.827940940856934,
      "activations/layer0_attention_weight_min": -14.319677352905273,
      "activations/layer10_attention_weight_max": 38.652862548828125,
      "activations/layer10_attention_weight_min": -38.493072509765625,
      "activations/layer11_attention_weight_max": 36.02445983886719,
      "activations/layer11_attention_weight_min": -35.52049255371094,
      "activations/layer12_attention_weight_max": 23.568958282470703,
      "activations/layer12_attention_weight_min": -24.308286666870117,
      "activations/layer13_attention_weight_max": 38.262508392333984,
      "activations/layer13_attention_weight_min": -29.197940826416016,
      "activations/layer14_attention_weight_max": 37.9160041809082,
      "activations/layer14_attention_weight_min": -31.09978675842285,
      "activations/layer15_attention_weight_max": 37.47624969482422,
      "activations/layer15_attention_weight_min": -31.95208168029785,
      "activations/layer16_attention_weight_max": 38.41638946533203,
      "activations/layer16_attention_weight_min": -34.72885513305664,
      "activations/layer17_attention_weight_max": 57.49155807495117,
      "activations/layer17_attention_weight_min": -49.000244140625,
      "activations/layer18_attention_weight_max": 53.51188659667969,
      "activations/layer18_attention_weight_min": -44.27832794189453,
      "activations/layer19_attention_weight_max": 28.8572940826416,
      "activations/layer19_attention_weight_min": -27.432342529296875,
      "activations/layer1_attention_weight_max": 17.00080108642578,
      "activations/layer1_attention_weight_min": -14.628649711608887,
      "activations/layer20_attention_weight_max": 27.482669830322266,
      "activations/layer20_attention_weight_min": -21.719097137451172,
      "activations/layer21_attention_weight_max": 43.35691452026367,
      "activations/layer21_attention_weight_min": -23.282724380493164,
      "activations/layer22_attention_weight_max": 35.09122085571289,
      "activations/layer22_attention_weight_min": -26.66343879699707,
      "activations/layer23_attention_weight_max": 41.580299377441406,
      "activations/layer23_attention_weight_min": -24.156721115112305,
      "activations/layer2_attention_weight_max": 31.199527740478516,
      "activations/layer2_attention_weight_min": -30.594913482666016,
      "activations/layer3_attention_weight_max": 90.95713806152344,
      "activations/layer3_attention_weight_min": -90.85081481933594,
      "activations/layer4_attention_weight_max": 84.37648010253906,
      "activations/layer4_attention_weight_min": -84.81851959228516,
      "activations/layer5_attention_weight_max": 62.585201263427734,
      "activations/layer5_attention_weight_min": -70.825927734375,
      "activations/layer6_attention_weight_max": 50.45090866088867,
      "activations/layer6_attention_weight_min": -52.758522033691406,
      "activations/layer7_attention_weight_max": 75.57259368896484,
      "activations/layer7_attention_weight_min": -67.31192779541016,
      "activations/layer8_attention_weight_max": 50.87226104736328,
      "activations/layer8_attention_weight_min": -53.73619842529297,
      "activations/layer9_attention_weight_max": 47.31290054321289,
      "activations/layer9_attention_weight_min": -49.1474494934082,
      "epoch": 21.37,
      "learning_rate": 1.232689393939394e-05,
      "loss": 2.7081,
      "step": 367750
    },
    {
      "activations/layer0_attention_weight_max": 16.92670249938965,
      "activations/layer0_attention_weight_min": -14.071676254272461,
      "activations/layer10_attention_weight_max": 39.78886413574219,
      "activations/layer10_attention_weight_min": -40.0543327331543,
      "activations/layer11_attention_weight_max": 35.78596115112305,
      "activations/layer11_attention_weight_min": -35.95099639892578,
      "activations/layer12_attention_weight_max": 19.86865997314453,
      "activations/layer12_attention_weight_min": -25.500898361206055,
      "activations/layer13_attention_weight_max": 38.407081604003906,
      "activations/layer13_attention_weight_min": -29.52532386779785,
      "activations/layer14_attention_weight_max": 36.35435485839844,
      "activations/layer14_attention_weight_min": -32.12070083618164,
      "activations/layer15_attention_weight_max": 34.69537353515625,
      "activations/layer15_attention_weight_min": -32.79375457763672,
      "activations/layer16_attention_weight_max": 41.00836944580078,
      "activations/layer16_attention_weight_min": -39.04041290283203,
      "activations/layer17_attention_weight_max": 57.54219436645508,
      "activations/layer17_attention_weight_min": -54.51988220214844,
      "activations/layer18_attention_weight_max": 53.880577087402344,
      "activations/layer18_attention_weight_min": -46.75056076049805,
      "activations/layer19_attention_weight_max": 26.411020278930664,
      "activations/layer19_attention_weight_min": -23.702587127685547,
      "activations/layer1_attention_weight_max": 16.777660369873047,
      "activations/layer1_attention_weight_min": -14.018301963806152,
      "activations/layer20_attention_weight_max": 24.328641891479492,
      "activations/layer20_attention_weight_min": -20.72260093688965,
      "activations/layer21_attention_weight_max": 46.47285461425781,
      "activations/layer21_attention_weight_min": -22.075878143310547,
      "activations/layer22_attention_weight_max": 27.951494216918945,
      "activations/layer22_attention_weight_min": -23.77385902404785,
      "activations/layer23_attention_weight_max": 35.93351745605469,
      "activations/layer23_attention_weight_min": -24.88776397705078,
      "activations/layer2_attention_weight_max": 29.85230255126953,
      "activations/layer2_attention_weight_min": -29.372974395751953,
      "activations/layer3_attention_weight_max": 93.24028778076172,
      "activations/layer3_attention_weight_min": -88.60682678222656,
      "activations/layer4_attention_weight_max": 86.54667663574219,
      "activations/layer4_attention_weight_min": -84.6208724975586,
      "activations/layer5_attention_weight_max": 61.207759857177734,
      "activations/layer5_attention_weight_min": -72.91783905029297,
      "activations/layer6_attention_weight_max": 52.34235382080078,
      "activations/layer6_attention_weight_min": -52.22120666503906,
      "activations/layer7_attention_weight_max": 69.76284790039062,
      "activations/layer7_attention_weight_min": -69.50010681152344,
      "activations/layer8_attention_weight_max": 52.35907745361328,
      "activations/layer8_attention_weight_min": -54.31559753417969,
      "activations/layer9_attention_weight_max": 47.21297836303711,
      "activations/layer9_attention_weight_min": -49.5147705078125,
      "epoch": 21.37,
      "learning_rate": 1.2307954545454544e-05,
      "loss": 2.7103,
      "step": 367800
    },
    {
      "activations/layer0_attention_weight_max": 15.505807876586914,
      "activations/layer0_attention_weight_min": -14.306221961975098,
      "activations/layer10_attention_weight_max": 37.56364059448242,
      "activations/layer10_attention_weight_min": -35.11937713623047,
      "activations/layer11_attention_weight_max": 33.971343994140625,
      "activations/layer11_attention_weight_min": -34.854557037353516,
      "activations/layer12_attention_weight_max": 17.92289161682129,
      "activations/layer12_attention_weight_min": -24.92938995361328,
      "activations/layer13_attention_weight_max": 34.65476989746094,
      "activations/layer13_attention_weight_min": -26.825786590576172,
      "activations/layer14_attention_weight_max": 36.19965744018555,
      "activations/layer14_attention_weight_min": -29.358840942382812,
      "activations/layer15_attention_weight_max": 33.398006439208984,
      "activations/layer15_attention_weight_min": -29.54300308227539,
      "activations/layer16_attention_weight_max": 32.37067794799805,
      "activations/layer16_attention_weight_min": -31.670183181762695,
      "activations/layer17_attention_weight_max": 56.0833625793457,
      "activations/layer17_attention_weight_min": -45.374271392822266,
      "activations/layer18_attention_weight_max": 51.10590362548828,
      "activations/layer18_attention_weight_min": -39.38938903808594,
      "activations/layer19_attention_weight_max": 26.178070068359375,
      "activations/layer19_attention_weight_min": -22.229333877563477,
      "activations/layer1_attention_weight_max": 16.32914924621582,
      "activations/layer1_attention_weight_min": -15.260549545288086,
      "activations/layer20_attention_weight_max": 25.196542739868164,
      "activations/layer20_attention_weight_min": -19.898561477661133,
      "activations/layer21_attention_weight_max": 37.90434646606445,
      "activations/layer21_attention_weight_min": -22.196365356445312,
      "activations/layer22_attention_weight_max": 29.0679931640625,
      "activations/layer22_attention_weight_min": -24.674489974975586,
      "activations/layer23_attention_weight_max": 36.180389404296875,
      "activations/layer23_attention_weight_min": -25.179931640625,
      "activations/layer2_attention_weight_max": 32.20196533203125,
      "activations/layer2_attention_weight_min": -30.11449432373047,
      "activations/layer3_attention_weight_max": 90.05186462402344,
      "activations/layer3_attention_weight_min": -86.37413787841797,
      "activations/layer4_attention_weight_max": 84.10454559326172,
      "activations/layer4_attention_weight_min": -84.13362121582031,
      "activations/layer5_attention_weight_max": 60.11170959472656,
      "activations/layer5_attention_weight_min": -70.31819152832031,
      "activations/layer6_attention_weight_max": 49.791160583496094,
      "activations/layer6_attention_weight_min": -51.35822296142578,
      "activations/layer7_attention_weight_max": 67.24137115478516,
      "activations/layer7_attention_weight_min": -64.62963104248047,
      "activations/layer8_attention_weight_max": 48.61900329589844,
      "activations/layer8_attention_weight_min": -50.78445053100586,
      "activations/layer9_attention_weight_max": 43.87119674682617,
      "activations/layer9_attention_weight_min": -43.79648971557617,
      "epoch": 21.37,
      "learning_rate": 1.2289015151515151e-05,
      "loss": 2.7241,
      "step": 367850
    },
    {
      "activations/layer0_attention_weight_max": 16.012191772460938,
      "activations/layer0_attention_weight_min": -14.130146026611328,
      "activations/layer10_attention_weight_max": 35.77177810668945,
      "activations/layer10_attention_weight_min": -36.975921630859375,
      "activations/layer11_attention_weight_max": 34.251564025878906,
      "activations/layer11_attention_weight_min": -35.835689544677734,
      "activations/layer12_attention_weight_max": 17.661205291748047,
      "activations/layer12_attention_weight_min": -26.490657806396484,
      "activations/layer13_attention_weight_max": 31.949092864990234,
      "activations/layer13_attention_weight_min": -26.859882354736328,
      "activations/layer14_attention_weight_max": 35.27031326293945,
      "activations/layer14_attention_weight_min": -29.747514724731445,
      "activations/layer15_attention_weight_max": 35.101009368896484,
      "activations/layer15_attention_weight_min": -29.758678436279297,
      "activations/layer16_attention_weight_max": 33.77394104003906,
      "activations/layer16_attention_weight_min": -33.41751480102539,
      "activations/layer17_attention_weight_max": 51.360870361328125,
      "activations/layer17_attention_weight_min": -45.8116455078125,
      "activations/layer18_attention_weight_max": 47.531307220458984,
      "activations/layer18_attention_weight_min": -40.23176574707031,
      "activations/layer19_attention_weight_max": 24.029428482055664,
      "activations/layer19_attention_weight_min": -24.457700729370117,
      "activations/layer1_attention_weight_max": 14.993446350097656,
      "activations/layer1_attention_weight_min": -14.731080055236816,
      "activations/layer20_attention_weight_max": 21.858482360839844,
      "activations/layer20_attention_weight_min": -21.14281463623047,
      "activations/layer21_attention_weight_max": 35.3675537109375,
      "activations/layer21_attention_weight_min": -22.075336456298828,
      "activations/layer22_attention_weight_max": 33.155235290527344,
      "activations/layer22_attention_weight_min": -25.444007873535156,
      "activations/layer23_attention_weight_max": 37.556236267089844,
      "activations/layer23_attention_weight_min": -23.83291244506836,
      "activations/layer2_attention_weight_max": 32.921974182128906,
      "activations/layer2_attention_weight_min": -30.986961364746094,
      "activations/layer3_attention_weight_max": 93.56076049804688,
      "activations/layer3_attention_weight_min": -93.77709197998047,
      "activations/layer4_attention_weight_max": 86.52559661865234,
      "activations/layer4_attention_weight_min": -85.32659149169922,
      "activations/layer5_attention_weight_max": 60.320030212402344,
      "activations/layer5_attention_weight_min": -69.09474182128906,
      "activations/layer6_attention_weight_max": 51.167781829833984,
      "activations/layer6_attention_weight_min": -52.90962219238281,
      "activations/layer7_attention_weight_max": 66.87430572509766,
      "activations/layer7_attention_weight_min": -67.44734954833984,
      "activations/layer8_attention_weight_max": 48.03278732299805,
      "activations/layer8_attention_weight_min": -50.448184967041016,
      "activations/layer9_attention_weight_max": 45.73089599609375,
      "activations/layer9_attention_weight_min": -46.38276290893555,
      "epoch": 21.38,
      "learning_rate": 1.2270075757575758e-05,
      "loss": 2.7178,
      "step": 367900
    },
    {
      "activations/layer0_attention_weight_max": 15.778295516967773,
      "activations/layer0_attention_weight_min": -14.172554969787598,
      "activations/layer10_attention_weight_max": 36.10415267944336,
      "activations/layer10_attention_weight_min": -37.396087646484375,
      "activations/layer11_attention_weight_max": 36.05621337890625,
      "activations/layer11_attention_weight_min": -38.77346420288086,
      "activations/layer12_attention_weight_max": 19.555015563964844,
      "activations/layer12_attention_weight_min": -24.2995662689209,
      "activations/layer13_attention_weight_max": 41.42720413208008,
      "activations/layer13_attention_weight_min": -28.580791473388672,
      "activations/layer14_attention_weight_max": 38.92079544067383,
      "activations/layer14_attention_weight_min": -30.943851470947266,
      "activations/layer15_attention_weight_max": 39.49726867675781,
      "activations/layer15_attention_weight_min": -32.4566764831543,
      "activations/layer16_attention_weight_max": 39.63358688354492,
      "activations/layer16_attention_weight_min": -35.99365234375,
      "activations/layer17_attention_weight_max": 57.85588455200195,
      "activations/layer17_attention_weight_min": -49.03092956542969,
      "activations/layer18_attention_weight_max": 51.99078369140625,
      "activations/layer18_attention_weight_min": -43.47093963623047,
      "activations/layer19_attention_weight_max": 24.113279342651367,
      "activations/layer19_attention_weight_min": -22.5449161529541,
      "activations/layer1_attention_weight_max": 15.406189918518066,
      "activations/layer1_attention_weight_min": -14.408364295959473,
      "activations/layer20_attention_weight_max": 22.41156768798828,
      "activations/layer20_attention_weight_min": -23.728609085083008,
      "activations/layer21_attention_weight_max": 38.69162368774414,
      "activations/layer21_attention_weight_min": -23.56920623779297,
      "activations/layer22_attention_weight_max": 29.573904037475586,
      "activations/layer22_attention_weight_min": -26.27543830871582,
      "activations/layer23_attention_weight_max": 36.220909118652344,
      "activations/layer23_attention_weight_min": -25.091018676757812,
      "activations/layer2_attention_weight_max": 30.444217681884766,
      "activations/layer2_attention_weight_min": -29.59651756286621,
      "activations/layer3_attention_weight_max": 89.23123168945312,
      "activations/layer3_attention_weight_min": -89.66217803955078,
      "activations/layer4_attention_weight_max": 84.97521209716797,
      "activations/layer4_attention_weight_min": -87.05675506591797,
      "activations/layer5_attention_weight_max": 65.0821533203125,
      "activations/layer5_attention_weight_min": -73.04696655273438,
      "activations/layer6_attention_weight_max": 53.816810607910156,
      "activations/layer6_attention_weight_min": -52.71809768676758,
      "activations/layer7_attention_weight_max": 72.38639831542969,
      "activations/layer7_attention_weight_min": -68.48162078857422,
      "activations/layer8_attention_weight_max": 46.48653030395508,
      "activations/layer8_attention_weight_min": -52.947975158691406,
      "activations/layer9_attention_weight_max": 45.15053939819336,
      "activations/layer9_attention_weight_min": -48.72760772705078,
      "epoch": 21.38,
      "learning_rate": 1.2251136363636362e-05,
      "loss": 2.7081,
      "step": 367950
    },
    {
      "activations/layer0_attention_weight_max": 16.123445510864258,
      "activations/layer0_attention_weight_min": -14.143248558044434,
      "activations/layer10_attention_weight_max": 34.744163513183594,
      "activations/layer10_attention_weight_min": -33.61415481567383,
      "activations/layer11_attention_weight_max": 34.13603210449219,
      "activations/layer11_attention_weight_min": -34.929107666015625,
      "activations/layer12_attention_weight_max": 19.251747131347656,
      "activations/layer12_attention_weight_min": -25.93675422668457,
      "activations/layer13_attention_weight_max": 30.124347686767578,
      "activations/layer13_attention_weight_min": -26.43886947631836,
      "activations/layer14_attention_weight_max": 33.890846252441406,
      "activations/layer14_attention_weight_min": -30.50309181213379,
      "activations/layer15_attention_weight_max": 32.07231903076172,
      "activations/layer15_attention_weight_min": -31.4116268157959,
      "activations/layer16_attention_weight_max": 35.396453857421875,
      "activations/layer16_attention_weight_min": -34.03965377807617,
      "activations/layer17_attention_weight_max": 50.07386779785156,
      "activations/layer17_attention_weight_min": -46.67721176147461,
      "activations/layer18_attention_weight_max": 43.7069206237793,
      "activations/layer18_attention_weight_min": -41.32426834106445,
      "activations/layer19_attention_weight_max": 24.498353958129883,
      "activations/layer19_attention_weight_min": -24.01727867126465,
      "activations/layer1_attention_weight_max": 15.313275337219238,
      "activations/layer1_attention_weight_min": -14.678257942199707,
      "activations/layer20_attention_weight_max": 24.295156478881836,
      "activations/layer20_attention_weight_min": -21.264238357543945,
      "activations/layer21_attention_weight_max": 34.02957534790039,
      "activations/layer21_attention_weight_min": -20.995210647583008,
      "activations/layer22_attention_weight_max": 28.575212478637695,
      "activations/layer22_attention_weight_min": -25.96234130859375,
      "activations/layer23_attention_weight_max": 37.3387565612793,
      "activations/layer23_attention_weight_min": -26.94476890563965,
      "activations/layer2_attention_weight_max": 31.237850189208984,
      "activations/layer2_attention_weight_min": -30.652061462402344,
      "activations/layer3_attention_weight_max": 86.96904754638672,
      "activations/layer3_attention_weight_min": -84.65860748291016,
      "activations/layer4_attention_weight_max": 83.37854766845703,
      "activations/layer4_attention_weight_min": -83.63997650146484,
      "activations/layer5_attention_weight_max": 60.237266540527344,
      "activations/layer5_attention_weight_min": -76.73529052734375,
      "activations/layer6_attention_weight_max": 50.40568161010742,
      "activations/layer6_attention_weight_min": -53.87344741821289,
      "activations/layer7_attention_weight_max": 68.65605163574219,
      "activations/layer7_attention_weight_min": -65.69668579101562,
      "activations/layer8_attention_weight_max": 45.12437438964844,
      "activations/layer8_attention_weight_min": -49.2076530456543,
      "activations/layer9_attention_weight_max": 42.70023727416992,
      "activations/layer9_attention_weight_min": -46.43873596191406,
      "epoch": 21.38,
      "learning_rate": 1.2232196969696969e-05,
      "loss": 2.7085,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_loss": 2.671875,
      "eval_runtime": 8.4656,
      "eval_samples_per_second": 507.23,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.4656,
      "eval_openwebtext_samples_per_second": 507.23,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_wikitext_loss": 2.91015625,
      "eval_wikitext_ppl": 18.359667040887704,
      "eval_wikitext_runtime": 1.9664,
      "eval_wikitext_samples_per_second": 231.901,
      "step": 368000
    },
    {
      "epoch": 21.38,
      "eval_lambada_loss": 2.390625,
      "eval_lambada_ppl": 10.920317008742302,
      "eval_lambada_runtime": 9.5378,
      "eval_lambada_samples_per_second": 510.496,
      "step": 368000
    },
    {
      "activations/layer0_attention_weight_max": 15.596531867980957,
      "activations/layer0_attention_weight_min": -14.232522964477539,
      "activations/layer10_attention_weight_max": 37.70832443237305,
      "activations/layer10_attention_weight_min": -36.941162109375,
      "activations/layer11_attention_weight_max": 35.84229278564453,
      "activations/layer11_attention_weight_min": -36.14788818359375,
      "activations/layer12_attention_weight_max": 19.511564254760742,
      "activations/layer12_attention_weight_min": -27.980058670043945,
      "activations/layer13_attention_weight_max": 34.50651168823242,
      "activations/layer13_attention_weight_min": -27.34233856201172,
      "activations/layer14_attention_weight_max": 39.461917877197266,
      "activations/layer14_attention_weight_min": -29.824918746948242,
      "activations/layer15_attention_weight_max": 37.167972564697266,
      "activations/layer15_attention_weight_min": -30.924659729003906,
      "activations/layer16_attention_weight_max": 35.64337158203125,
      "activations/layer16_attention_weight_min": -32.58218765258789,
      "activations/layer17_attention_weight_max": 56.765480041503906,
      "activations/layer17_attention_weight_min": -47.91404724121094,
      "activations/layer18_attention_weight_max": 51.124637603759766,
      "activations/layer18_attention_weight_min": -43.73176574707031,
      "activations/layer19_attention_weight_max": 28.789249420166016,
      "activations/layer19_attention_weight_min": -23.674257278442383,
      "activations/layer1_attention_weight_max": 15.3829345703125,
      "activations/layer1_attention_weight_min": -14.214871406555176,
      "activations/layer20_attention_weight_max": 27.98818588256836,
      "activations/layer20_attention_weight_min": -23.88878631591797,
      "activations/layer21_attention_weight_max": 40.371124267578125,
      "activations/layer21_attention_weight_min": -23.086013793945312,
      "activations/layer22_attention_weight_max": 30.764984130859375,
      "activations/layer22_attention_weight_min": -26.237709045410156,
      "activations/layer23_attention_weight_max": 37.89492416381836,
      "activations/layer23_attention_weight_min": -25.417720794677734,
      "activations/layer2_attention_weight_max": 33.68280029296875,
      "activations/layer2_attention_weight_min": -31.84619903564453,
      "activations/layer3_attention_weight_max": 90.2901382446289,
      "activations/layer3_attention_weight_min": -88.36331176757812,
      "activations/layer4_attention_weight_max": 86.51838684082031,
      "activations/layer4_attention_weight_min": -84.14613342285156,
      "activations/layer5_attention_weight_max": 60.638362884521484,
      "activations/layer5_attention_weight_min": -71.96912384033203,
      "activations/layer6_attention_weight_max": 51.71955871582031,
      "activations/layer6_attention_weight_min": -53.55113983154297,
      "activations/layer7_attention_weight_max": 69.95118713378906,
      "activations/layer7_attention_weight_min": -70.11402130126953,
      "activations/layer8_attention_weight_max": 49.9754524230957,
      "activations/layer8_attention_weight_min": -51.825984954833984,
      "activations/layer9_attention_weight_max": 49.13188934326172,
      "activations/layer9_attention_weight_min": -50.487911224365234,
      "epoch": 21.39,
      "learning_rate": 1.2213257575757576e-05,
      "loss": 2.7043,
      "step": 368050
    },
    {
      "activations/layer0_attention_weight_max": 15.978209495544434,
      "activations/layer0_attention_weight_min": -14.22140884399414,
      "activations/layer10_attention_weight_max": 36.861724853515625,
      "activations/layer10_attention_weight_min": -35.76503372192383,
      "activations/layer11_attention_weight_max": 34.98289489746094,
      "activations/layer11_attention_weight_min": -34.27180480957031,
      "activations/layer12_attention_weight_max": 18.92291259765625,
      "activations/layer12_attention_weight_min": -23.49406623840332,
      "activations/layer13_attention_weight_max": 34.34830856323242,
      "activations/layer13_attention_weight_min": -26.55685806274414,
      "activations/layer14_attention_weight_max": 33.18523025512695,
      "activations/layer14_attention_weight_min": -28.759078979492188,
      "activations/layer15_attention_weight_max": 32.48527145385742,
      "activations/layer15_attention_weight_min": -28.908565521240234,
      "activations/layer16_attention_weight_max": 31.606815338134766,
      "activations/layer16_attention_weight_min": -29.818849563598633,
      "activations/layer17_attention_weight_max": 48.376747131347656,
      "activations/layer17_attention_weight_min": -42.64373779296875,
      "activations/layer18_attention_weight_max": 45.426326751708984,
      "activations/layer18_attention_weight_min": -38.22019577026367,
      "activations/layer19_attention_weight_max": 26.338790893554688,
      "activations/layer19_attention_weight_min": -22.602781295776367,
      "activations/layer1_attention_weight_max": 14.802655220031738,
      "activations/layer1_attention_weight_min": -15.087120056152344,
      "activations/layer20_attention_weight_max": 21.89998435974121,
      "activations/layer20_attention_weight_min": -20.96506118774414,
      "activations/layer21_attention_weight_max": 35.582122802734375,
      "activations/layer21_attention_weight_min": -26.953323364257812,
      "activations/layer22_attention_weight_max": 29.67560577392578,
      "activations/layer22_attention_weight_min": -24.34810447692871,
      "activations/layer23_attention_weight_max": 35.42292022705078,
      "activations/layer23_attention_weight_min": -23.9692440032959,
      "activations/layer2_attention_weight_max": 30.844017028808594,
      "activations/layer2_attention_weight_min": -29.47882080078125,
      "activations/layer3_attention_weight_max": 87.8865737915039,
      "activations/layer3_attention_weight_min": -85.58485412597656,
      "activations/layer4_attention_weight_max": 86.20877838134766,
      "activations/layer4_attention_weight_min": -83.02699279785156,
      "activations/layer5_attention_weight_max": 60.89581298828125,
      "activations/layer5_attention_weight_min": -68.73646545410156,
      "activations/layer6_attention_weight_max": 54.02803039550781,
      "activations/layer6_attention_weight_min": -52.93266677856445,
      "activations/layer7_attention_weight_max": 68.81917572021484,
      "activations/layer7_attention_weight_min": -70.3365707397461,
      "activations/layer8_attention_weight_max": 48.44887924194336,
      "activations/layer8_attention_weight_min": -52.57429885864258,
      "activations/layer9_attention_weight_max": 45.428775787353516,
      "activations/layer9_attention_weight_min": -45.432857513427734,
      "epoch": 21.39,
      "learning_rate": 1.219431818181818e-05,
      "loss": 2.7292,
      "step": 368100
    },
    {
      "activations/layer0_attention_weight_max": 15.555268287658691,
      "activations/layer0_attention_weight_min": -14.053007125854492,
      "activations/layer10_attention_weight_max": 39.22041320800781,
      "activations/layer10_attention_weight_min": -37.614925384521484,
      "activations/layer11_attention_weight_max": 35.137115478515625,
      "activations/layer11_attention_weight_min": -36.062320709228516,
      "activations/layer12_attention_weight_max": 19.670068740844727,
      "activations/layer12_attention_weight_min": -22.722471237182617,
      "activations/layer13_attention_weight_max": 37.58181381225586,
      "activations/layer13_attention_weight_min": -28.7313289642334,
      "activations/layer14_attention_weight_max": 37.524452209472656,
      "activations/layer14_attention_weight_min": -30.55925178527832,
      "activations/layer15_attention_weight_max": 33.813941955566406,
      "activations/layer15_attention_weight_min": -30.226028442382812,
      "activations/layer16_attention_weight_max": 34.79629135131836,
      "activations/layer16_attention_weight_min": -33.60113525390625,
      "activations/layer17_attention_weight_max": 51.9905891418457,
      "activations/layer17_attention_weight_min": -45.67552185058594,
      "activations/layer18_attention_weight_max": 51.78924560546875,
      "activations/layer18_attention_weight_min": -41.577850341796875,
      "activations/layer19_attention_weight_max": 23.893665313720703,
      "activations/layer19_attention_weight_min": -21.537565231323242,
      "activations/layer1_attention_weight_max": 15.753731727600098,
      "activations/layer1_attention_weight_min": -13.290999412536621,
      "activations/layer20_attention_weight_max": 21.97954750061035,
      "activations/layer20_attention_weight_min": -20.96099853515625,
      "activations/layer21_attention_weight_max": 37.215576171875,
      "activations/layer21_attention_weight_min": -21.525062561035156,
      "activations/layer22_attention_weight_max": 26.199983596801758,
      "activations/layer22_attention_weight_min": -25.58429527282715,
      "activations/layer23_attention_weight_max": 34.520477294921875,
      "activations/layer23_attention_weight_min": -22.93344497680664,
      "activations/layer2_attention_weight_max": 31.122879028320312,
      "activations/layer2_attention_weight_min": -30.44439697265625,
      "activations/layer3_attention_weight_max": 88.41032409667969,
      "activations/layer3_attention_weight_min": -91.04312133789062,
      "activations/layer4_attention_weight_max": 86.12389373779297,
      "activations/layer4_attention_weight_min": -84.16378021240234,
      "activations/layer5_attention_weight_max": 63.37842559814453,
      "activations/layer5_attention_weight_min": -68.93023681640625,
      "activations/layer6_attention_weight_max": 49.778892517089844,
      "activations/layer6_attention_weight_min": -51.817440032958984,
      "activations/layer7_attention_weight_max": 67.91905212402344,
      "activations/layer7_attention_weight_min": -69.73246765136719,
      "activations/layer8_attention_weight_max": 50.20415496826172,
      "activations/layer8_attention_weight_min": -52.72332763671875,
      "activations/layer9_attention_weight_max": 48.13679122924805,
      "activations/layer9_attention_weight_min": -48.794498443603516,
      "epoch": 21.39,
      "learning_rate": 1.2175378787878787e-05,
      "loss": 2.7228,
      "step": 368150
    },
    {
      "activations/layer0_attention_weight_max": 15.096784591674805,
      "activations/layer0_attention_weight_min": -14.158934593200684,
      "activations/layer10_attention_weight_max": 35.24239730834961,
      "activations/layer10_attention_weight_min": -33.424560546875,
      "activations/layer11_attention_weight_max": 30.85848617553711,
      "activations/layer11_attention_weight_min": -33.25330352783203,
      "activations/layer12_attention_weight_max": 18.646011352539062,
      "activations/layer12_attention_weight_min": -26.575044631958008,
      "activations/layer13_attention_weight_max": 34.60896301269531,
      "activations/layer13_attention_weight_min": -27.9337158203125,
      "activations/layer14_attention_weight_max": 39.69150924682617,
      "activations/layer14_attention_weight_min": -30.826229095458984,
      "activations/layer15_attention_weight_max": 37.13458251953125,
      "activations/layer15_attention_weight_min": -32.08433151245117,
      "activations/layer16_attention_weight_max": 36.44076156616211,
      "activations/layer16_attention_weight_min": -33.284423828125,
      "activations/layer17_attention_weight_max": 56.41249465942383,
      "activations/layer17_attention_weight_min": -47.17784881591797,
      "activations/layer18_attention_weight_max": 55.14194869995117,
      "activations/layer18_attention_weight_min": -41.031925201416016,
      "activations/layer19_attention_weight_max": 24.952730178833008,
      "activations/layer19_attention_weight_min": -21.177581787109375,
      "activations/layer1_attention_weight_max": 15.84754753112793,
      "activations/layer1_attention_weight_min": -13.91301441192627,
      "activations/layer20_attention_weight_max": 22.568809509277344,
      "activations/layer20_attention_weight_min": -21.141883850097656,
      "activations/layer21_attention_weight_max": 29.850265502929688,
      "activations/layer21_attention_weight_min": -19.81901741027832,
      "activations/layer22_attention_weight_max": 29.217031478881836,
      "activations/layer22_attention_weight_min": -23.222614288330078,
      "activations/layer23_attention_weight_max": 34.38102340698242,
      "activations/layer23_attention_weight_min": -22.9085693359375,
      "activations/layer2_attention_weight_max": 32.42061996459961,
      "activations/layer2_attention_weight_min": -31.200756072998047,
      "activations/layer3_attention_weight_max": 87.53595733642578,
      "activations/layer3_attention_weight_min": -86.23118591308594,
      "activations/layer4_attention_weight_max": 83.28695678710938,
      "activations/layer4_attention_weight_min": -82.43724822998047,
      "activations/layer5_attention_weight_max": 61.54922103881836,
      "activations/layer5_attention_weight_min": -70.30516052246094,
      "activations/layer6_attention_weight_max": 47.720829010009766,
      "activations/layer6_attention_weight_min": -48.85431671142578,
      "activations/layer7_attention_weight_max": 73.44743347167969,
      "activations/layer7_attention_weight_min": -66.6668701171875,
      "activations/layer8_attention_weight_max": 44.68254089355469,
      "activations/layer8_attention_weight_min": -50.25918960571289,
      "activations/layer9_attention_weight_max": 44.79041290283203,
      "activations/layer9_attention_weight_min": -46.78673553466797,
      "epoch": 21.39,
      "learning_rate": 1.2156439393939392e-05,
      "loss": 2.7102,
      "step": 368200
    },
    {
      "activations/layer0_attention_weight_max": 15.29246997833252,
      "activations/layer0_attention_weight_min": -14.177755355834961,
      "activations/layer10_attention_weight_max": 36.070674896240234,
      "activations/layer10_attention_weight_min": -38.890380859375,
      "activations/layer11_attention_weight_max": 32.6907844543457,
      "activations/layer11_attention_weight_min": -34.957763671875,
      "activations/layer12_attention_weight_max": 20.504045486450195,
      "activations/layer12_attention_weight_min": -23.155807495117188,
      "activations/layer13_attention_weight_max": 38.60335922241211,
      "activations/layer13_attention_weight_min": -27.614334106445312,
      "activations/layer14_attention_weight_max": 40.18106460571289,
      "activations/layer14_attention_weight_min": -30.8173770904541,
      "activations/layer15_attention_weight_max": 39.265323638916016,
      "activations/layer15_attention_weight_min": -31.45802116394043,
      "activations/layer16_attention_weight_max": 38.76576232910156,
      "activations/layer16_attention_weight_min": -34.56718444824219,
      "activations/layer17_attention_weight_max": 55.26167678833008,
      "activations/layer17_attention_weight_min": -50.157901763916016,
      "activations/layer18_attention_weight_max": 50.48184585571289,
      "activations/layer18_attention_weight_min": -45.56874084472656,
      "activations/layer19_attention_weight_max": 25.105764389038086,
      "activations/layer19_attention_weight_min": -23.997068405151367,
      "activations/layer1_attention_weight_max": 15.087427139282227,
      "activations/layer1_attention_weight_min": -15.94865608215332,
      "activations/layer20_attention_weight_max": 23.254741668701172,
      "activations/layer20_attention_weight_min": -21.18899154663086,
      "activations/layer21_attention_weight_max": 35.32900619506836,
      "activations/layer21_attention_weight_min": -20.921855926513672,
      "activations/layer22_attention_weight_max": 30.505701065063477,
      "activations/layer22_attention_weight_min": -26.406038284301758,
      "activations/layer23_attention_weight_max": 36.25794982910156,
      "activations/layer23_attention_weight_min": -23.367507934570312,
      "activations/layer2_attention_weight_max": 30.95105743408203,
      "activations/layer2_attention_weight_min": -31.37203025817871,
      "activations/layer3_attention_weight_max": 86.94999694824219,
      "activations/layer3_attention_weight_min": -92.44220733642578,
      "activations/layer4_attention_weight_max": 81.98265838623047,
      "activations/layer4_attention_weight_min": -87.1860580444336,
      "activations/layer5_attention_weight_max": 60.28240203857422,
      "activations/layer5_attention_weight_min": -71.37921142578125,
      "activations/layer6_attention_weight_max": 49.847225189208984,
      "activations/layer6_attention_weight_min": -55.43586349487305,
      "activations/layer7_attention_weight_max": 65.82450866699219,
      "activations/layer7_attention_weight_min": -65.743408203125,
      "activations/layer8_attention_weight_max": 49.39436340332031,
      "activations/layer8_attention_weight_min": -55.59593200683594,
      "activations/layer9_attention_weight_max": 45.06070327758789,
      "activations/layer9_attention_weight_min": -46.44101333618164,
      "epoch": 21.4,
      "learning_rate": 1.2137499999999999e-05,
      "loss": 2.7173,
      "step": 368250
    },
    {
      "activations/layer0_attention_weight_max": 16.3038387298584,
      "activations/layer0_attention_weight_min": -14.11473274230957,
      "activations/layer10_attention_weight_max": 33.97532272338867,
      "activations/layer10_attention_weight_min": -35.90960693359375,
      "activations/layer11_attention_weight_max": 34.474334716796875,
      "activations/layer11_attention_weight_min": -33.82756042480469,
      "activations/layer12_attention_weight_max": 18.502777099609375,
      "activations/layer12_attention_weight_min": -25.411029815673828,
      "activations/layer13_attention_weight_max": 35.1376953125,
      "activations/layer13_attention_weight_min": -27.10445213317871,
      "activations/layer14_attention_weight_max": 32.86888885498047,
      "activations/layer14_attention_weight_min": -29.563447952270508,
      "activations/layer15_attention_weight_max": 34.303226470947266,
      "activations/layer15_attention_weight_min": -29.396841049194336,
      "activations/layer16_attention_weight_max": 32.173858642578125,
      "activations/layer16_attention_weight_min": -30.971981048583984,
      "activations/layer17_attention_weight_max": 48.09940719604492,
      "activations/layer17_attention_weight_min": -45.301265716552734,
      "activations/layer18_attention_weight_max": 46.25183868408203,
      "activations/layer18_attention_weight_min": -42.49790954589844,
      "activations/layer19_attention_weight_max": 24.12717056274414,
      "activations/layer19_attention_weight_min": -22.58576011657715,
      "activations/layer1_attention_weight_max": 14.995372772216797,
      "activations/layer1_attention_weight_min": -15.889741897583008,
      "activations/layer20_attention_weight_max": 20.66580581665039,
      "activations/layer20_attention_weight_min": -20.831954956054688,
      "activations/layer21_attention_weight_max": 34.36945724487305,
      "activations/layer21_attention_weight_min": -21.4219970703125,
      "activations/layer22_attention_weight_max": 28.940013885498047,
      "activations/layer22_attention_weight_min": -26.260591506958008,
      "activations/layer23_attention_weight_max": 37.41167068481445,
      "activations/layer23_attention_weight_min": -25.416812896728516,
      "activations/layer2_attention_weight_max": 32.57270050048828,
      "activations/layer2_attention_weight_min": -30.779279708862305,
      "activations/layer3_attention_weight_max": 90.84693908691406,
      "activations/layer3_attention_weight_min": -88.3099594116211,
      "activations/layer4_attention_weight_max": 83.67874145507812,
      "activations/layer4_attention_weight_min": -83.287841796875,
      "activations/layer5_attention_weight_max": 61.91614532470703,
      "activations/layer5_attention_weight_min": -72.58246612548828,
      "activations/layer6_attention_weight_max": 51.42201232910156,
      "activations/layer6_attention_weight_min": -50.608943939208984,
      "activations/layer7_attention_weight_max": 64.97588348388672,
      "activations/layer7_attention_weight_min": -65.45246887207031,
      "activations/layer8_attention_weight_max": 46.24872970581055,
      "activations/layer8_attention_weight_min": -50.857112884521484,
      "activations/layer9_attention_weight_max": 44.180908203125,
      "activations/layer9_attention_weight_min": -43.40713119506836,
      "epoch": 21.4,
      "learning_rate": 1.2118939393939393e-05,
      "loss": 2.7115,
      "step": 368300
    },
    {
      "activations/layer0_attention_weight_max": 16.10173797607422,
      "activations/layer0_attention_weight_min": -14.338594436645508,
      "activations/layer10_attention_weight_max": 37.34790802001953,
      "activations/layer10_attention_weight_min": -37.624385833740234,
      "activations/layer11_attention_weight_max": 34.75438690185547,
      "activations/layer11_attention_weight_min": -37.876007080078125,
      "activations/layer12_attention_weight_max": 20.219764709472656,
      "activations/layer12_attention_weight_min": -25.280729293823242,
      "activations/layer13_attention_weight_max": 38.380035400390625,
      "activations/layer13_attention_weight_min": -27.757661819458008,
      "activations/layer14_attention_weight_max": 36.11882019042969,
      "activations/layer14_attention_weight_min": -30.552135467529297,
      "activations/layer15_attention_weight_max": 32.9665641784668,
      "activations/layer15_attention_weight_min": -33.07938003540039,
      "activations/layer16_attention_weight_max": 35.00279235839844,
      "activations/layer16_attention_weight_min": -32.8185920715332,
      "activations/layer17_attention_weight_max": 54.088409423828125,
      "activations/layer17_attention_weight_min": -50.55237579345703,
      "activations/layer18_attention_weight_max": 50.96100616455078,
      "activations/layer18_attention_weight_min": -47.7331657409668,
      "activations/layer19_attention_weight_max": 26.382810592651367,
      "activations/layer19_attention_weight_min": -27.89491081237793,
      "activations/layer1_attention_weight_max": 15.413793563842773,
      "activations/layer1_attention_weight_min": -13.78666877746582,
      "activations/layer20_attention_weight_max": 23.595561981201172,
      "activations/layer20_attention_weight_min": -26.243562698364258,
      "activations/layer21_attention_weight_max": 34.2073860168457,
      "activations/layer21_attention_weight_min": -24.7185115814209,
      "activations/layer22_attention_weight_max": 32.4074592590332,
      "activations/layer22_attention_weight_min": -28.82067108154297,
      "activations/layer23_attention_weight_max": 35.71628952026367,
      "activations/layer23_attention_weight_min": -25.56855010986328,
      "activations/layer2_attention_weight_max": 33.08504104614258,
      "activations/layer2_attention_weight_min": -30.949424743652344,
      "activations/layer3_attention_weight_max": 91.23743438720703,
      "activations/layer3_attention_weight_min": -94.0843734741211,
      "activations/layer4_attention_weight_max": 84.15509033203125,
      "activations/layer4_attention_weight_min": -86.0440902709961,
      "activations/layer5_attention_weight_max": 61.13908386230469,
      "activations/layer5_attention_weight_min": -77.37107849121094,
      "activations/layer6_attention_weight_max": 51.43805694580078,
      "activations/layer6_attention_weight_min": -51.75880432128906,
      "activations/layer7_attention_weight_max": 70.62530517578125,
      "activations/layer7_attention_weight_min": -66.57890319824219,
      "activations/layer8_attention_weight_max": 48.66679763793945,
      "activations/layer8_attention_weight_min": -52.17189407348633,
      "activations/layer9_attention_weight_max": 46.535499572753906,
      "activations/layer9_attention_weight_min": -46.47085189819336,
      "epoch": 21.4,
      "learning_rate": 1.21e-05,
      "loss": 2.7111,
      "step": 368350
    },
    {
      "activations/layer0_attention_weight_max": 16.340444564819336,
      "activations/layer0_attention_weight_min": -14.240313529968262,
      "activations/layer10_attention_weight_max": 35.6935920715332,
      "activations/layer10_attention_weight_min": -35.1435546875,
      "activations/layer11_attention_weight_max": 32.22517013549805,
      "activations/layer11_attention_weight_min": -34.49208068847656,
      "activations/layer12_attention_weight_max": 19.14930534362793,
      "activations/layer12_attention_weight_min": -27.76869010925293,
      "activations/layer13_attention_weight_max": 36.478084564208984,
      "activations/layer13_attention_weight_min": -28.429271697998047,
      "activations/layer14_attention_weight_max": 41.975914001464844,
      "activations/layer14_attention_weight_min": -30.720203399658203,
      "activations/layer15_attention_weight_max": 36.13148498535156,
      "activations/layer15_attention_weight_min": -30.552967071533203,
      "activations/layer16_attention_weight_max": 35.438411712646484,
      "activations/layer16_attention_weight_min": -34.292137145996094,
      "activations/layer17_attention_weight_max": 52.65997314453125,
      "activations/layer17_attention_weight_min": -49.66697692871094,
      "activations/layer18_attention_weight_max": 48.84837341308594,
      "activations/layer18_attention_weight_min": -46.8399658203125,
      "activations/layer19_attention_weight_max": 27.564496994018555,
      "activations/layer19_attention_weight_min": -27.19593620300293,
      "activations/layer1_attention_weight_max": 15.35990047454834,
      "activations/layer1_attention_weight_min": -13.878174781799316,
      "activations/layer20_attention_weight_max": 28.300188064575195,
      "activations/layer20_attention_weight_min": -22.567277908325195,
      "activations/layer21_attention_weight_max": 44.526649475097656,
      "activations/layer21_attention_weight_min": -24.71190643310547,
      "activations/layer22_attention_weight_max": 31.601003646850586,
      "activations/layer22_attention_weight_min": -29.601573944091797,
      "activations/layer23_attention_weight_max": 38.19804000854492,
      "activations/layer23_attention_weight_min": -26.189786911010742,
      "activations/layer2_attention_weight_max": 33.45570373535156,
      "activations/layer2_attention_weight_min": -31.957210540771484,
      "activations/layer3_attention_weight_max": 91.92719268798828,
      "activations/layer3_attention_weight_min": -93.44805908203125,
      "activations/layer4_attention_weight_max": 87.09963989257812,
      "activations/layer4_attention_weight_min": -83.81246185302734,
      "activations/layer5_attention_weight_max": 60.383087158203125,
      "activations/layer5_attention_weight_min": -71.65291595458984,
      "activations/layer6_attention_weight_max": 50.57352066040039,
      "activations/layer6_attention_weight_min": -51.237220764160156,
      "activations/layer7_attention_weight_max": 68.50828552246094,
      "activations/layer7_attention_weight_min": -66.0276870727539,
      "activations/layer8_attention_weight_max": 47.66762924194336,
      "activations/layer8_attention_weight_min": -47.10236358642578,
      "activations/layer9_attention_weight_max": 43.832454681396484,
      "activations/layer9_attention_weight_min": -46.1750602722168,
      "epoch": 21.41,
      "learning_rate": 1.2081060606060604e-05,
      "loss": 2.7049,
      "step": 368400
    },
    {
      "activations/layer0_attention_weight_max": 16.14000701904297,
      "activations/layer0_attention_weight_min": -13.961069107055664,
      "activations/layer10_attention_weight_max": 47.61460876464844,
      "activations/layer10_attention_weight_min": -48.767269134521484,
      "activations/layer11_attention_weight_max": 46.35539245605469,
      "activations/layer11_attention_weight_min": -42.82587432861328,
      "activations/layer12_attention_weight_max": 20.013036727905273,
      "activations/layer12_attention_weight_min": -25.34482192993164,
      "activations/layer13_attention_weight_max": 43.97909927368164,
      "activations/layer13_attention_weight_min": -32.148529052734375,
      "activations/layer14_attention_weight_max": 53.19317626953125,
      "activations/layer14_attention_weight_min": -31.54531478881836,
      "activations/layer15_attention_weight_max": 50.62955856323242,
      "activations/layer15_attention_weight_min": -32.15754699707031,
      "activations/layer16_attention_weight_max": 36.285526275634766,
      "activations/layer16_attention_weight_min": -33.094749450683594,
      "activations/layer17_attention_weight_max": 56.95730972290039,
      "activations/layer17_attention_weight_min": -49.397804260253906,
      "activations/layer18_attention_weight_max": 45.93749237060547,
      "activations/layer18_attention_weight_min": -43.24080276489258,
      "activations/layer19_attention_weight_max": 25.465110778808594,
      "activations/layer19_attention_weight_min": -24.446439743041992,
      "activations/layer1_attention_weight_max": 16.006086349487305,
      "activations/layer1_attention_weight_min": -14.446782112121582,
      "activations/layer20_attention_weight_max": 25.78335189819336,
      "activations/layer20_attention_weight_min": -24.114322662353516,
      "activations/layer21_attention_weight_max": 37.23997116088867,
      "activations/layer21_attention_weight_min": -23.676504135131836,
      "activations/layer22_attention_weight_max": 28.89600944519043,
      "activations/layer22_attention_weight_min": -28.15061378479004,
      "activations/layer23_attention_weight_max": 32.04883575439453,
      "activations/layer23_attention_weight_min": -25.982158660888672,
      "activations/layer2_attention_weight_max": 32.61064529418945,
      "activations/layer2_attention_weight_min": -31.837852478027344,
      "activations/layer3_attention_weight_max": 90.99140167236328,
      "activations/layer3_attention_weight_min": -88.22595977783203,
      "activations/layer4_attention_weight_max": 88.09927368164062,
      "activations/layer4_attention_weight_min": -84.24248504638672,
      "activations/layer5_attention_weight_max": 67.2759017944336,
      "activations/layer5_attention_weight_min": -66.42372131347656,
      "activations/layer6_attention_weight_max": 56.35557556152344,
      "activations/layer6_attention_weight_min": -53.745235443115234,
      "activations/layer7_attention_weight_max": 77.22892761230469,
      "activations/layer7_attention_weight_min": -71.53459930419922,
      "activations/layer8_attention_weight_max": 57.42653274536133,
      "activations/layer8_attention_weight_min": -57.3134651184082,
      "activations/layer9_attention_weight_max": 62.91536331176758,
      "activations/layer9_attention_weight_min": -54.63477325439453,
      "epoch": 21.41,
      "learning_rate": 1.206212121212121e-05,
      "loss": 2.6989,
      "step": 368450
    },
    {
      "activations/layer0_attention_weight_max": 15.923471450805664,
      "activations/layer0_attention_weight_min": -14.158254623413086,
      "activations/layer10_attention_weight_max": 37.63087844848633,
      "activations/layer10_attention_weight_min": -36.44477844238281,
      "activations/layer11_attention_weight_max": 33.85733413696289,
      "activations/layer11_attention_weight_min": -35.76325988769531,
      "activations/layer12_attention_weight_max": 18.09700584411621,
      "activations/layer12_attention_weight_min": -24.501953125,
      "activations/layer13_attention_weight_max": 32.6666259765625,
      "activations/layer13_attention_weight_min": -28.611806869506836,
      "activations/layer14_attention_weight_max": 36.53922653198242,
      "activations/layer14_attention_weight_min": -30.248138427734375,
      "activations/layer15_attention_weight_max": 31.85739517211914,
      "activations/layer15_attention_weight_min": -30.092195510864258,
      "activations/layer16_attention_weight_max": 32.803958892822266,
      "activations/layer16_attention_weight_min": -32.65489196777344,
      "activations/layer17_attention_weight_max": 52.42515182495117,
      "activations/layer17_attention_weight_min": -46.01467514038086,
      "activations/layer18_attention_weight_max": 45.063507080078125,
      "activations/layer18_attention_weight_min": -40.22046661376953,
      "activations/layer19_attention_weight_max": 23.2779598236084,
      "activations/layer19_attention_weight_min": -21.244070053100586,
      "activations/layer1_attention_weight_max": 14.685795783996582,
      "activations/layer1_attention_weight_min": -14.860669136047363,
      "activations/layer20_attention_weight_max": 19.994476318359375,
      "activations/layer20_attention_weight_min": -21.394472122192383,
      "activations/layer21_attention_weight_max": 31.769668579101562,
      "activations/layer21_attention_weight_min": -22.74074363708496,
      "activations/layer22_attention_weight_max": 26.921586990356445,
      "activations/layer22_attention_weight_min": -25.35173225402832,
      "activations/layer23_attention_weight_max": 35.90747833251953,
      "activations/layer23_attention_weight_min": -24.065933227539062,
      "activations/layer2_attention_weight_max": 32.73710250854492,
      "activations/layer2_attention_weight_min": -30.750690460205078,
      "activations/layer3_attention_weight_max": 88.7785873413086,
      "activations/layer3_attention_weight_min": -90.27914428710938,
      "activations/layer4_attention_weight_max": 83.4697265625,
      "activations/layer4_attention_weight_min": -82.37120056152344,
      "activations/layer5_attention_weight_max": 59.43504333496094,
      "activations/layer5_attention_weight_min": -70.58885192871094,
      "activations/layer6_attention_weight_max": 50.24622344970703,
      "activations/layer6_attention_weight_min": -49.80665588378906,
      "activations/layer7_attention_weight_max": 66.51241302490234,
      "activations/layer7_attention_weight_min": -68.52426147460938,
      "activations/layer8_attention_weight_max": 47.76866149902344,
      "activations/layer8_attention_weight_min": -52.4939079284668,
      "activations/layer9_attention_weight_max": 42.9251708984375,
      "activations/layer9_attention_weight_min": -48.32969284057617,
      "epoch": 21.41,
      "learning_rate": 1.2043181818181816e-05,
      "loss": 2.7008,
      "step": 368500
    },
    {
      "activations/layer0_attention_weight_max": 16.859128952026367,
      "activations/layer0_attention_weight_min": -14.575267791748047,
      "activations/layer10_attention_weight_max": 37.56549072265625,
      "activations/layer10_attention_weight_min": -35.95305252075195,
      "activations/layer11_attention_weight_max": 34.323265075683594,
      "activations/layer11_attention_weight_min": -35.65228271484375,
      "activations/layer12_attention_weight_max": 18.963605880737305,
      "activations/layer12_attention_weight_min": -24.7552547454834,
      "activations/layer13_attention_weight_max": 32.63187789916992,
      "activations/layer13_attention_weight_min": -27.787385940551758,
      "activations/layer14_attention_weight_max": 34.128265380859375,
      "activations/layer14_attention_weight_min": -29.61347007751465,
      "activations/layer15_attention_weight_max": 32.48626708984375,
      "activations/layer15_attention_weight_min": -29.56665802001953,
      "activations/layer16_attention_weight_max": 34.24368667602539,
      "activations/layer16_attention_weight_min": -31.20098114013672,
      "activations/layer17_attention_weight_max": 50.58907699584961,
      "activations/layer17_attention_weight_min": -44.74620819091797,
      "activations/layer18_attention_weight_max": 45.61835861206055,
      "activations/layer18_attention_weight_min": -40.187225341796875,
      "activations/layer19_attention_weight_max": 23.754304885864258,
      "activations/layer19_attention_weight_min": -23.82286262512207,
      "activations/layer1_attention_weight_max": 14.831632614135742,
      "activations/layer1_attention_weight_min": -14.506921768188477,
      "activations/layer20_attention_weight_max": 21.961292266845703,
      "activations/layer20_attention_weight_min": -23.409650802612305,
      "activations/layer21_attention_weight_max": 33.52964782714844,
      "activations/layer21_attention_weight_min": -21.367338180541992,
      "activations/layer22_attention_weight_max": 29.927040100097656,
      "activations/layer22_attention_weight_min": -24.90532875061035,
      "activations/layer23_attention_weight_max": 36.673301696777344,
      "activations/layer23_attention_weight_min": -24.032962799072266,
      "activations/layer2_attention_weight_max": 32.447959899902344,
      "activations/layer2_attention_weight_min": -31.917081832885742,
      "activations/layer3_attention_weight_max": 83.9073486328125,
      "activations/layer3_attention_weight_min": -86.98589324951172,
      "activations/layer4_attention_weight_max": 79.80538177490234,
      "activations/layer4_attention_weight_min": -83.82238006591797,
      "activations/layer5_attention_weight_max": 56.8551139831543,
      "activations/layer5_attention_weight_min": -70.81407165527344,
      "activations/layer6_attention_weight_max": 47.81973648071289,
      "activations/layer6_attention_weight_min": -51.766273498535156,
      "activations/layer7_attention_weight_max": 66.65164947509766,
      "activations/layer7_attention_weight_min": -67.97132110595703,
      "activations/layer8_attention_weight_max": 47.39436721801758,
      "activations/layer8_attention_weight_min": -51.17189025878906,
      "activations/layer9_attention_weight_max": 44.45587921142578,
      "activations/layer9_attention_weight_min": -49.03855514526367,
      "epoch": 21.41,
      "learning_rate": 1.2024242424242422e-05,
      "loss": 2.701,
      "step": 368550
    },
    {
      "activations/layer0_attention_weight_max": 15.603534698486328,
      "activations/layer0_attention_weight_min": -14.397013664245605,
      "activations/layer10_attention_weight_max": 34.70579528808594,
      "activations/layer10_attention_weight_min": -33.907291412353516,
      "activations/layer11_attention_weight_max": 33.17295837402344,
      "activations/layer11_attention_weight_min": -33.98906707763672,
      "activations/layer12_attention_weight_max": 20.50741195678711,
      "activations/layer12_attention_weight_min": -25.996198654174805,
      "activations/layer13_attention_weight_max": 30.096221923828125,
      "activations/layer13_attention_weight_min": -26.549772262573242,
      "activations/layer14_attention_weight_max": 33.925079345703125,
      "activations/layer14_attention_weight_min": -30.626115798950195,
      "activations/layer15_attention_weight_max": 35.19635772705078,
      "activations/layer15_attention_weight_min": -30.53636360168457,
      "activations/layer16_attention_weight_max": 34.51328659057617,
      "activations/layer16_attention_weight_min": -32.80363082885742,
      "activations/layer17_attention_weight_max": 49.42829513549805,
      "activations/layer17_attention_weight_min": -45.742034912109375,
      "activations/layer18_attention_weight_max": 47.17059326171875,
      "activations/layer18_attention_weight_min": -41.72085952758789,
      "activations/layer19_attention_weight_max": 27.5964412689209,
      "activations/layer19_attention_weight_min": -25.711700439453125,
      "activations/layer1_attention_weight_max": 15.742273330688477,
      "activations/layer1_attention_weight_min": -14.278498649597168,
      "activations/layer20_attention_weight_max": 24.596961975097656,
      "activations/layer20_attention_weight_min": -25.294994354248047,
      "activations/layer21_attention_weight_max": 35.73602294921875,
      "activations/layer21_attention_weight_min": -24.48819923400879,
      "activations/layer22_attention_weight_max": 32.21759796142578,
      "activations/layer22_attention_weight_min": -28.55060577392578,
      "activations/layer23_attention_weight_max": 36.339542388916016,
      "activations/layer23_attention_weight_min": -25.798498153686523,
      "activations/layer2_attention_weight_max": 30.30815887451172,
      "activations/layer2_attention_weight_min": -31.827848434448242,
      "activations/layer3_attention_weight_max": 83.80995178222656,
      "activations/layer3_attention_weight_min": -84.32101440429688,
      "activations/layer4_attention_weight_max": 81.80839538574219,
      "activations/layer4_attention_weight_min": -79.21240234375,
      "activations/layer5_attention_weight_max": 60.008453369140625,
      "activations/layer5_attention_weight_min": -70.09559631347656,
      "activations/layer6_attention_weight_max": 48.0044059753418,
      "activations/layer6_attention_weight_min": -50.279659271240234,
      "activations/layer7_attention_weight_max": 65.12645721435547,
      "activations/layer7_attention_weight_min": -66.36688232421875,
      "activations/layer8_attention_weight_max": 45.03904724121094,
      "activations/layer8_attention_weight_min": -49.48463439941406,
      "activations/layer9_attention_weight_max": 42.991268157958984,
      "activations/layer9_attention_weight_min": -45.49246597290039,
      "epoch": 21.42,
      "learning_rate": 1.2005303030303029e-05,
      "loss": 2.7103,
      "step": 368600
    },
    {
      "activations/layer0_attention_weight_max": 16.04669952392578,
      "activations/layer0_attention_weight_min": -14.729296684265137,
      "activations/layer10_attention_weight_max": 35.58623504638672,
      "activations/layer10_attention_weight_min": -36.48052215576172,
      "activations/layer11_attention_weight_max": 32.911170959472656,
      "activations/layer11_attention_weight_min": -37.0152473449707,
      "activations/layer12_attention_weight_max": 20.016950607299805,
      "activations/layer12_attention_weight_min": -23.628259658813477,
      "activations/layer13_attention_weight_max": 31.23335838317871,
      "activations/layer13_attention_weight_min": -26.008466720581055,
      "activations/layer14_attention_weight_max": 35.5587043762207,
      "activations/layer14_attention_weight_min": -30.988903045654297,
      "activations/layer15_attention_weight_max": 34.84769821166992,
      "activations/layer15_attention_weight_min": -32.02989959716797,
      "activations/layer16_attention_weight_max": 38.58732986450195,
      "activations/layer16_attention_weight_min": -35.19150161743164,
      "activations/layer17_attention_weight_max": 55.22239685058594,
      "activations/layer17_attention_weight_min": -48.02045822143555,
      "activations/layer18_attention_weight_max": 50.265621185302734,
      "activations/layer18_attention_weight_min": -42.10789108276367,
      "activations/layer19_attention_weight_max": 25.575450897216797,
      "activations/layer19_attention_weight_min": -25.06064224243164,
      "activations/layer1_attention_weight_max": 14.84196949005127,
      "activations/layer1_attention_weight_min": -15.000670433044434,
      "activations/layer20_attention_weight_max": 23.57080078125,
      "activations/layer20_attention_weight_min": -24.965085983276367,
      "activations/layer21_attention_weight_max": 35.306068420410156,
      "activations/layer21_attention_weight_min": -25.17236328125,
      "activations/layer22_attention_weight_max": 27.84282684326172,
      "activations/layer22_attention_weight_min": -26.551361083984375,
      "activations/layer23_attention_weight_max": 35.32464599609375,
      "activations/layer23_attention_weight_min": -24.243907928466797,
      "activations/layer2_attention_weight_max": 30.918928146362305,
      "activations/layer2_attention_weight_min": -31.633495330810547,
      "activations/layer3_attention_weight_max": 86.814208984375,
      "activations/layer3_attention_weight_min": -86.67347717285156,
      "activations/layer4_attention_weight_max": 86.55370330810547,
      "activations/layer4_attention_weight_min": -84.91484069824219,
      "activations/layer5_attention_weight_max": 60.960792541503906,
      "activations/layer5_attention_weight_min": -71.48187255859375,
      "activations/layer6_attention_weight_max": 49.552120208740234,
      "activations/layer6_attention_weight_min": -51.02604675292969,
      "activations/layer7_attention_weight_max": 67.7945785522461,
      "activations/layer7_attention_weight_min": -66.76741790771484,
      "activations/layer8_attention_weight_max": 48.0052375793457,
      "activations/layer8_attention_weight_min": -51.22561264038086,
      "activations/layer9_attention_weight_max": 43.688716888427734,
      "activations/layer9_attention_weight_min": -50.49825668334961,
      "epoch": 21.42,
      "learning_rate": 1.1986363636363637e-05,
      "loss": 2.7248,
      "step": 368650
    },
    {
      "activations/layer0_attention_weight_max": 16.46059226989746,
      "activations/layer0_attention_weight_min": -14.113245964050293,
      "activations/layer10_attention_weight_max": 34.951942443847656,
      "activations/layer10_attention_weight_min": -35.120147705078125,
      "activations/layer11_attention_weight_max": 32.386592864990234,
      "activations/layer11_attention_weight_min": -35.44902420043945,
      "activations/layer12_attention_weight_max": 19.841934204101562,
      "activations/layer12_attention_weight_min": -23.402790069580078,
      "activations/layer13_attention_weight_max": 32.1235466003418,
      "activations/layer13_attention_weight_min": -28.06357192993164,
      "activations/layer14_attention_weight_max": 35.59559631347656,
      "activations/layer14_attention_weight_min": -30.823108673095703,
      "activations/layer15_attention_weight_max": 31.726646423339844,
      "activations/layer15_attention_weight_min": -29.90317726135254,
      "activations/layer16_attention_weight_max": 35.24382781982422,
      "activations/layer16_attention_weight_min": -33.842369079589844,
      "activations/layer17_attention_weight_max": 50.706398010253906,
      "activations/layer17_attention_weight_min": -46.71199417114258,
      "activations/layer18_attention_weight_max": 52.92880630493164,
      "activations/layer18_attention_weight_min": -43.3455810546875,
      "activations/layer19_attention_weight_max": 27.051605224609375,
      "activations/layer19_attention_weight_min": -24.73215675354004,
      "activations/layer1_attention_weight_max": 14.932485580444336,
      "activations/layer1_attention_weight_min": -14.645524024963379,
      "activations/layer20_attention_weight_max": 25.060148239135742,
      "activations/layer20_attention_weight_min": -23.127883911132812,
      "activations/layer21_attention_weight_max": 39.209983825683594,
      "activations/layer21_attention_weight_min": -22.156475067138672,
      "activations/layer22_attention_weight_max": 29.902286529541016,
      "activations/layer22_attention_weight_min": -28.386674880981445,
      "activations/layer23_attention_weight_max": 35.421695709228516,
      "activations/layer23_attention_weight_min": -22.046857833862305,
      "activations/layer2_attention_weight_max": 30.370710372924805,
      "activations/layer2_attention_weight_min": -30.500648498535156,
      "activations/layer3_attention_weight_max": 87.65789031982422,
      "activations/layer3_attention_weight_min": -84.43244934082031,
      "activations/layer4_attention_weight_max": 80.33478546142578,
      "activations/layer4_attention_weight_min": -81.31480407714844,
      "activations/layer5_attention_weight_max": 58.774959564208984,
      "activations/layer5_attention_weight_min": -68.67129516601562,
      "activations/layer6_attention_weight_max": 48.69837188720703,
      "activations/layer6_attention_weight_min": -51.33931350708008,
      "activations/layer7_attention_weight_max": 67.3204574584961,
      "activations/layer7_attention_weight_min": -65.33228302001953,
      "activations/layer8_attention_weight_max": 47.750755310058594,
      "activations/layer8_attention_weight_min": -50.19424819946289,
      "activations/layer9_attention_weight_max": 42.70213317871094,
      "activations/layer9_attention_weight_min": -45.48385238647461,
      "epoch": 21.42,
      "learning_rate": 1.1967424242424242e-05,
      "loss": 2.7093,
      "step": 368700
    },
    {
      "activations/layer0_attention_weight_max": 15.646086692810059,
      "activations/layer0_attention_weight_min": -15.021135330200195,
      "activations/layer10_attention_weight_max": 36.54789352416992,
      "activations/layer10_attention_weight_min": -37.26411819458008,
      "activations/layer11_attention_weight_max": 32.67148208618164,
      "activations/layer11_attention_weight_min": -34.86219024658203,
      "activations/layer12_attention_weight_max": 19.46061897277832,
      "activations/layer12_attention_weight_min": -22.836400985717773,
      "activations/layer13_attention_weight_max": 38.11339569091797,
      "activations/layer13_attention_weight_min": -28.539283752441406,
      "activations/layer14_attention_weight_max": 36.07077407836914,
      "activations/layer14_attention_weight_min": -30.20332145690918,
      "activations/layer15_attention_weight_max": 35.55357360839844,
      "activations/layer15_attention_weight_min": -30.6224422454834,
      "activations/layer16_attention_weight_max": 36.04531478881836,
      "activations/layer16_attention_weight_min": -32.21723175048828,
      "activations/layer17_attention_weight_max": 52.746761322021484,
      "activations/layer17_attention_weight_min": -43.24267578125,
      "activations/layer18_attention_weight_max": 49.573726654052734,
      "activations/layer18_attention_weight_min": -40.82720947265625,
      "activations/layer19_attention_weight_max": 24.24304962158203,
      "activations/layer19_attention_weight_min": -23.693681716918945,
      "activations/layer1_attention_weight_max": 15.5861177444458,
      "activations/layer1_attention_weight_min": -15.34216022491455,
      "activations/layer20_attention_weight_max": 22.868175506591797,
      "activations/layer20_attention_weight_min": -21.661449432373047,
      "activations/layer21_attention_weight_max": 34.784446716308594,
      "activations/layer21_attention_weight_min": -24.153892517089844,
      "activations/layer22_attention_weight_max": 27.202526092529297,
      "activations/layer22_attention_weight_min": -25.38356590270996,
      "activations/layer23_attention_weight_max": 38.462562561035156,
      "activations/layer23_attention_weight_min": -25.82900619506836,
      "activations/layer2_attention_weight_max": 32.463462829589844,
      "activations/layer2_attention_weight_min": -31.096050262451172,
      "activations/layer3_attention_weight_max": 91.02872467041016,
      "activations/layer3_attention_weight_min": -87.8951416015625,
      "activations/layer4_attention_weight_max": 86.42001342773438,
      "activations/layer4_attention_weight_min": -86.49309539794922,
      "activations/layer5_attention_weight_max": 63.09963607788086,
      "activations/layer5_attention_weight_min": -75.6801986694336,
      "activations/layer6_attention_weight_max": 52.549720764160156,
      "activations/layer6_attention_weight_min": -52.623783111572266,
      "activations/layer7_attention_weight_max": 71.99481201171875,
      "activations/layer7_attention_weight_min": -68.38172149658203,
      "activations/layer8_attention_weight_max": 48.282135009765625,
      "activations/layer8_attention_weight_min": -51.72135925292969,
      "activations/layer9_attention_weight_max": 46.98305892944336,
      "activations/layer9_attention_weight_min": -47.67217254638672,
      "epoch": 21.43,
      "learning_rate": 1.1948484848484849e-05,
      "loss": 2.7021,
      "step": 368750
    },
    {
      "activations/layer0_attention_weight_max": 16.231369018554688,
      "activations/layer0_attention_weight_min": -14.3042631149292,
      "activations/layer10_attention_weight_max": 33.181396484375,
      "activations/layer10_attention_weight_min": -36.22255325317383,
      "activations/layer11_attention_weight_max": 30.971044540405273,
      "activations/layer11_attention_weight_min": -36.191123962402344,
      "activations/layer12_attention_weight_max": 17.929292678833008,
      "activations/layer12_attention_weight_min": -24.176380157470703,
      "activations/layer13_attention_weight_max": 31.59177017211914,
      "activations/layer13_attention_weight_min": -26.708059310913086,
      "activations/layer14_attention_weight_max": 33.740089416503906,
      "activations/layer14_attention_weight_min": -28.5919246673584,
      "activations/layer15_attention_weight_max": 31.718097686767578,
      "activations/layer15_attention_weight_min": -29.506851196289062,
      "activations/layer16_attention_weight_max": 34.765830993652344,
      "activations/layer16_attention_weight_min": -32.18429946899414,
      "activations/layer17_attention_weight_max": 51.66727066040039,
      "activations/layer17_attention_weight_min": -49.47260665893555,
      "activations/layer18_attention_weight_max": 45.42558288574219,
      "activations/layer18_attention_weight_min": -40.75356674194336,
      "activations/layer19_attention_weight_max": 23.767057418823242,
      "activations/layer19_attention_weight_min": -23.626476287841797,
      "activations/layer1_attention_weight_max": 15.764968872070312,
      "activations/layer1_attention_weight_min": -14.610215187072754,
      "activations/layer20_attention_weight_max": 23.33948516845703,
      "activations/layer20_attention_weight_min": -21.208017349243164,
      "activations/layer21_attention_weight_max": 36.259765625,
      "activations/layer21_attention_weight_min": -21.38730239868164,
      "activations/layer22_attention_weight_max": 27.263832092285156,
      "activations/layer22_attention_weight_min": -25.03056526184082,
      "activations/layer23_attention_weight_max": 34.053138732910156,
      "activations/layer23_attention_weight_min": -26.120223999023438,
      "activations/layer2_attention_weight_max": 32.23633575439453,
      "activations/layer2_attention_weight_min": -32.85133361816406,
      "activations/layer3_attention_weight_max": 90.85092163085938,
      "activations/layer3_attention_weight_min": -85.64225769042969,
      "activations/layer4_attention_weight_max": 83.79924011230469,
      "activations/layer4_attention_weight_min": -82.28705596923828,
      "activations/layer5_attention_weight_max": 61.20427703857422,
      "activations/layer5_attention_weight_min": -72.03107452392578,
      "activations/layer6_attention_weight_max": 48.64400863647461,
      "activations/layer6_attention_weight_min": -48.664100646972656,
      "activations/layer7_attention_weight_max": 64.73271179199219,
      "activations/layer7_attention_weight_min": -67.0381088256836,
      "activations/layer8_attention_weight_max": 43.29921340942383,
      "activations/layer8_attention_weight_min": -47.84935760498047,
      "activations/layer9_attention_weight_max": 39.99643325805664,
      "activations/layer9_attention_weight_min": -43.910423278808594,
      "epoch": 21.43,
      "learning_rate": 1.1929545454545453e-05,
      "loss": 2.7207,
      "step": 368800
    },
    {
      "activations/layer0_attention_weight_max": 16.009233474731445,
      "activations/layer0_attention_weight_min": -14.251960754394531,
      "activations/layer10_attention_weight_max": 34.58843231201172,
      "activations/layer10_attention_weight_min": -35.60740661621094,
      "activations/layer11_attention_weight_max": 31.808462142944336,
      "activations/layer11_attention_weight_min": -35.07134246826172,
      "activations/layer12_attention_weight_max": 18.580913543701172,
      "activations/layer12_attention_weight_min": -22.799423217773438,
      "activations/layer13_attention_weight_max": 40.60111999511719,
      "activations/layer13_attention_weight_min": -29.191579818725586,
      "activations/layer14_attention_weight_max": 36.08380126953125,
      "activations/layer14_attention_weight_min": -28.851224899291992,
      "activations/layer15_attention_weight_max": 35.5423469543457,
      "activations/layer15_attention_weight_min": -29.334056854248047,
      "activations/layer16_attention_weight_max": 36.66727066040039,
      "activations/layer16_attention_weight_min": -31.796903610229492,
      "activations/layer17_attention_weight_max": 56.789615631103516,
      "activations/layer17_attention_weight_min": -43.075294494628906,
      "activations/layer18_attention_weight_max": 51.25157928466797,
      "activations/layer18_attention_weight_min": -40.209442138671875,
      "activations/layer19_attention_weight_max": 29.228666305541992,
      "activations/layer19_attention_weight_min": -24.196590423583984,
      "activations/layer1_attention_weight_max": 16.050052642822266,
      "activations/layer1_attention_weight_min": -13.974690437316895,
      "activations/layer20_attention_weight_max": 27.914905548095703,
      "activations/layer20_attention_weight_min": -22.658870697021484,
      "activations/layer21_attention_weight_max": 33.5871467590332,
      "activations/layer21_attention_weight_min": -21.25493812561035,
      "activations/layer22_attention_weight_max": 28.161794662475586,
      "activations/layer22_attention_weight_min": -23.1921443939209,
      "activations/layer23_attention_weight_max": 36.00385284423828,
      "activations/layer23_attention_weight_min": -23.15589141845703,
      "activations/layer2_attention_weight_max": 32.130104064941406,
      "activations/layer2_attention_weight_min": -31.479721069335938,
      "activations/layer3_attention_weight_max": 88.80231475830078,
      "activations/layer3_attention_weight_min": -88.29329681396484,
      "activations/layer4_attention_weight_max": 84.07815551757812,
      "activations/layer4_attention_weight_min": -85.9687728881836,
      "activations/layer5_attention_weight_max": 61.33186340332031,
      "activations/layer5_attention_weight_min": -71.79228973388672,
      "activations/layer6_attention_weight_max": 49.03510284423828,
      "activations/layer6_attention_weight_min": -50.24924850463867,
      "activations/layer7_attention_weight_max": 67.93418884277344,
      "activations/layer7_attention_weight_min": -65.34623718261719,
      "activations/layer8_attention_weight_max": 45.870140075683594,
      "activations/layer8_attention_weight_min": -50.68977355957031,
      "activations/layer9_attention_weight_max": 44.14016342163086,
      "activations/layer9_attention_weight_min": -46.67470169067383,
      "epoch": 21.43,
      "learning_rate": 1.191060606060606e-05,
      "loss": 2.7112,
      "step": 368850
    },
    {
      "activations/layer0_attention_weight_max": 15.967884063720703,
      "activations/layer0_attention_weight_min": -14.069600105285645,
      "activations/layer10_attention_weight_max": 34.037086486816406,
      "activations/layer10_attention_weight_min": -34.460838317871094,
      "activations/layer11_attention_weight_max": 32.51531219482422,
      "activations/layer11_attention_weight_min": -35.54875183105469,
      "activations/layer12_attention_weight_max": 19.40142822265625,
      "activations/layer12_attention_weight_min": -24.770221710205078,
      "activations/layer13_attention_weight_max": 32.97119140625,
      "activations/layer13_attention_weight_min": -28.264867782592773,
      "activations/layer14_attention_weight_max": 36.97612380981445,
      "activations/layer14_attention_weight_min": -32.98848342895508,
      "activations/layer15_attention_weight_max": 33.49684143066406,
      "activations/layer15_attention_weight_min": -30.775211334228516,
      "activations/layer16_attention_weight_max": 35.797786712646484,
      "activations/layer16_attention_weight_min": -35.41452407836914,
      "activations/layer17_attention_weight_max": 53.647987365722656,
      "activations/layer17_attention_weight_min": -48.18525695800781,
      "activations/layer18_attention_weight_max": 52.82170486450195,
      "activations/layer18_attention_weight_min": -43.6103630065918,
      "activations/layer19_attention_weight_max": 27.25198745727539,
      "activations/layer19_attention_weight_min": -24.641054153442383,
      "activations/layer1_attention_weight_max": 16.037668228149414,
      "activations/layer1_attention_weight_min": -13.616599082946777,
      "activations/layer20_attention_weight_max": 25.98717498779297,
      "activations/layer20_attention_weight_min": -21.203275680541992,
      "activations/layer21_attention_weight_max": 42.25168991088867,
      "activations/layer21_attention_weight_min": -24.884613037109375,
      "activations/layer22_attention_weight_max": 32.8416633605957,
      "activations/layer22_attention_weight_min": -27.463701248168945,
      "activations/layer23_attention_weight_max": 35.5015869140625,
      "activations/layer23_attention_weight_min": -26.341238021850586,
      "activations/layer2_attention_weight_max": 30.25250244140625,
      "activations/layer2_attention_weight_min": -29.587453842163086,
      "activations/layer3_attention_weight_max": 84.41710662841797,
      "activations/layer3_attention_weight_min": -84.69782257080078,
      "activations/layer4_attention_weight_max": 81.6514663696289,
      "activations/layer4_attention_weight_min": -82.38174438476562,
      "activations/layer5_attention_weight_max": 61.63267135620117,
      "activations/layer5_attention_weight_min": -71.37461853027344,
      "activations/layer6_attention_weight_max": 48.14149856567383,
      "activations/layer6_attention_weight_min": -48.40227508544922,
      "activations/layer7_attention_weight_max": 67.42876434326172,
      "activations/layer7_attention_weight_min": -70.53863525390625,
      "activations/layer8_attention_weight_max": 45.127315521240234,
      "activations/layer8_attention_weight_min": -49.27287292480469,
      "activations/layer9_attention_weight_max": 42.7047233581543,
      "activations/layer9_attention_weight_min": -48.53325271606445,
      "epoch": 21.44,
      "learning_rate": 1.1891666666666667e-05,
      "loss": 2.7186,
      "step": 368900
    },
    {
      "activations/layer0_attention_weight_max": 16.06348991394043,
      "activations/layer0_attention_weight_min": -14.247066497802734,
      "activations/layer10_attention_weight_max": 34.43393325805664,
      "activations/layer10_attention_weight_min": -34.001441955566406,
      "activations/layer11_attention_weight_max": 31.5142822265625,
      "activations/layer11_attention_weight_min": -33.55974578857422,
      "activations/layer12_attention_weight_max": 19.194385528564453,
      "activations/layer12_attention_weight_min": -27.09781265258789,
      "activations/layer13_attention_weight_max": 31.621715545654297,
      "activations/layer13_attention_weight_min": -26.301536560058594,
      "activations/layer14_attention_weight_max": 33.84174346923828,
      "activations/layer14_attention_weight_min": -30.058757781982422,
      "activations/layer15_attention_weight_max": 32.96961212158203,
      "activations/layer15_attention_weight_min": -31.10655975341797,
      "activations/layer16_attention_weight_max": 36.85927200317383,
      "activations/layer16_attention_weight_min": -33.89632034301758,
      "activations/layer17_attention_weight_max": 51.087608337402344,
      "activations/layer17_attention_weight_min": -46.40160369873047,
      "activations/layer18_attention_weight_max": 49.14682388305664,
      "activations/layer18_attention_weight_min": -40.9459342956543,
      "activations/layer19_attention_weight_max": 24.56770133972168,
      "activations/layer19_attention_weight_min": -23.80588722229004,
      "activations/layer1_attention_weight_max": 15.17527961730957,
      "activations/layer1_attention_weight_min": -13.582746505737305,
      "activations/layer20_attention_weight_max": 21.423662185668945,
      "activations/layer20_attention_weight_min": -22.04008674621582,
      "activations/layer21_attention_weight_max": 34.517120361328125,
      "activations/layer21_attention_weight_min": -22.854581832885742,
      "activations/layer22_attention_weight_max": 28.1007080078125,
      "activations/layer22_attention_weight_min": -25.92805290222168,
      "activations/layer23_attention_weight_max": 33.39488983154297,
      "activations/layer23_attention_weight_min": -24.75672149658203,
      "activations/layer2_attention_weight_max": 31.00969696044922,
      "activations/layer2_attention_weight_min": -30.29467010498047,
      "activations/layer3_attention_weight_max": 88.38785552978516,
      "activations/layer3_attention_weight_min": -87.28144836425781,
      "activations/layer4_attention_weight_max": 82.84587860107422,
      "activations/layer4_attention_weight_min": -85.5064926147461,
      "activations/layer5_attention_weight_max": 63.538177490234375,
      "activations/layer5_attention_weight_min": -70.1466064453125,
      "activations/layer6_attention_weight_max": 49.323917388916016,
      "activations/layer6_attention_weight_min": -50.64687728881836,
      "activations/layer7_attention_weight_max": 69.33715057373047,
      "activations/layer7_attention_weight_min": -64.93805694580078,
      "activations/layer8_attention_weight_max": 46.15205383300781,
      "activations/layer8_attention_weight_min": -46.99308395385742,
      "activations/layer9_attention_weight_max": 41.397178649902344,
      "activations/layer9_attention_weight_min": -46.26792526245117,
      "epoch": 21.44,
      "learning_rate": 1.1872727272727271e-05,
      "loss": 2.7104,
      "step": 368950
    },
    {
      "activations/layer0_attention_weight_max": 15.258527755737305,
      "activations/layer0_attention_weight_min": -14.464441299438477,
      "activations/layer10_attention_weight_max": 44.16997528076172,
      "activations/layer10_attention_weight_min": -43.611328125,
      "activations/layer11_attention_weight_max": 39.065128326416016,
      "activations/layer11_attention_weight_min": -40.195777893066406,
      "activations/layer12_attention_weight_max": 20.653234481811523,
      "activations/layer12_attention_weight_min": -26.237958908081055,
      "activations/layer13_attention_weight_max": 39.657318115234375,
      "activations/layer13_attention_weight_min": -32.77164077758789,
      "activations/layer14_attention_weight_max": 39.78306198120117,
      "activations/layer14_attention_weight_min": -36.100860595703125,
      "activations/layer15_attention_weight_max": 36.80740737915039,
      "activations/layer15_attention_weight_min": -35.406368255615234,
      "activations/layer16_attention_weight_max": 38.84584426879883,
      "activations/layer16_attention_weight_min": -37.2072868347168,
      "activations/layer17_attention_weight_max": 54.27646255493164,
      "activations/layer17_attention_weight_min": -54.9921989440918,
      "activations/layer18_attention_weight_max": 50.80474090576172,
      "activations/layer18_attention_weight_min": -48.166263580322266,
      "activations/layer19_attention_weight_max": 26.550888061523438,
      "activations/layer19_attention_weight_min": -27.260160446166992,
      "activations/layer1_attention_weight_max": 15.891719818115234,
      "activations/layer1_attention_weight_min": -14.616076469421387,
      "activations/layer20_attention_weight_max": 24.38534164428711,
      "activations/layer20_attention_weight_min": -21.928146362304688,
      "activations/layer21_attention_weight_max": 37.73143005371094,
      "activations/layer21_attention_weight_min": -22.2651309967041,
      "activations/layer22_attention_weight_max": 31.235151290893555,
      "activations/layer22_attention_weight_min": -28.0435733795166,
      "activations/layer23_attention_weight_max": 37.22272491455078,
      "activations/layer23_attention_weight_min": -24.59815788269043,
      "activations/layer2_attention_weight_max": 31.616779327392578,
      "activations/layer2_attention_weight_min": -33.11311721801758,
      "activations/layer3_attention_weight_max": 92.3843765258789,
      "activations/layer3_attention_weight_min": -90.37520599365234,
      "activations/layer4_attention_weight_max": 86.19345092773438,
      "activations/layer4_attention_weight_min": -90.08906555175781,
      "activations/layer5_attention_weight_max": 61.741432189941406,
      "activations/layer5_attention_weight_min": -71.29154968261719,
      "activations/layer6_attention_weight_max": 50.393211364746094,
      "activations/layer6_attention_weight_min": -53.48030090332031,
      "activations/layer7_attention_weight_max": 73.68943786621094,
      "activations/layer7_attention_weight_min": -74.74012756347656,
      "activations/layer8_attention_weight_max": 55.45822525024414,
      "activations/layer8_attention_weight_min": -58.76167297363281,
      "activations/layer9_attention_weight_max": 50.18954086303711,
      "activations/layer9_attention_weight_min": -53.211097717285156,
      "epoch": 21.44,
      "learning_rate": 1.1853787878787878e-05,
      "loss": 2.713,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_loss": 2.671875,
      "eval_runtime": 8.495,
      "eval_samples_per_second": 505.473,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.495,
      "eval_openwebtext_samples_per_second": 505.473,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.966,
      "eval_wikitext_samples_per_second": 231.945,
      "step": 369000
    },
    {
      "epoch": 21.44,
      "eval_lambada_loss": 2.416015625,
      "eval_lambada_ppl": 11.201140741117625,
      "eval_lambada_runtime": 9.5654,
      "eval_lambada_samples_per_second": 509.022,
      "step": 369000
    },
    {
      "activations/layer0_attention_weight_max": 16.34841537475586,
      "activations/layer0_attention_weight_min": -14.406675338745117,
      "activations/layer10_attention_weight_max": 35.012916564941406,
      "activations/layer10_attention_weight_min": -34.59160232543945,
      "activations/layer11_attention_weight_max": 33.08062744140625,
      "activations/layer11_attention_weight_min": -35.176204681396484,
      "activations/layer12_attention_weight_max": 20.273107528686523,
      "activations/layer12_attention_weight_min": -24.229631423950195,
      "activations/layer13_attention_weight_max": 29.90629005432129,
      "activations/layer13_attention_weight_min": -27.53628158569336,
      "activations/layer14_attention_weight_max": 32.52212142944336,
      "activations/layer14_attention_weight_min": -27.61759376525879,
      "activations/layer15_attention_weight_max": 30.920330047607422,
      "activations/layer15_attention_weight_min": -28.389202117919922,
      "activations/layer16_attention_weight_max": 33.703285217285156,
      "activations/layer16_attention_weight_min": -31.44767189025879,
      "activations/layer17_attention_weight_max": 47.054771423339844,
      "activations/layer17_attention_weight_min": -43.982421875,
      "activations/layer18_attention_weight_max": 44.5764045715332,
      "activations/layer18_attention_weight_min": -40.58264923095703,
      "activations/layer19_attention_weight_max": 23.10969352722168,
      "activations/layer19_attention_weight_min": -23.608850479125977,
      "activations/layer1_attention_weight_max": 14.891595840454102,
      "activations/layer1_attention_weight_min": -14.622371673583984,
      "activations/layer20_attention_weight_max": 20.479045867919922,
      "activations/layer20_attention_weight_min": -21.792905807495117,
      "activations/layer21_attention_weight_max": 33.10508728027344,
      "activations/layer21_attention_weight_min": -23.794578552246094,
      "activations/layer22_attention_weight_max": 27.458269119262695,
      "activations/layer22_attention_weight_min": -25.447792053222656,
      "activations/layer23_attention_weight_max": 36.578399658203125,
      "activations/layer23_attention_weight_min": -24.2626953125,
      "activations/layer2_attention_weight_max": 31.68417739868164,
      "activations/layer2_attention_weight_min": -30.09490966796875,
      "activations/layer3_attention_weight_max": 87.12723541259766,
      "activations/layer3_attention_weight_min": -87.19625091552734,
      "activations/layer4_attention_weight_max": 83.11579132080078,
      "activations/layer4_attention_weight_min": -82.32369232177734,
      "activations/layer5_attention_weight_max": 59.119140625,
      "activations/layer5_attention_weight_min": -70.0416259765625,
      "activations/layer6_attention_weight_max": 45.624794006347656,
      "activations/layer6_attention_weight_min": -49.76251983642578,
      "activations/layer7_attention_weight_max": 65.82988739013672,
      "activations/layer7_attention_weight_min": -70.03064727783203,
      "activations/layer8_attention_weight_max": 44.709190368652344,
      "activations/layer8_attention_weight_min": -48.65896987915039,
      "activations/layer9_attention_weight_max": 42.09369659423828,
      "activations/layer9_attention_weight_min": -47.571346282958984,
      "epoch": 21.44,
      "learning_rate": 1.1834848484848485e-05,
      "loss": 2.6979,
      "step": 369050
    },
    {
      "activations/layer0_attention_weight_max": 16.182764053344727,
      "activations/layer0_attention_weight_min": -14.505941390991211,
      "activations/layer10_attention_weight_max": 35.43367385864258,
      "activations/layer10_attention_weight_min": -35.787803649902344,
      "activations/layer11_attention_weight_max": 33.065338134765625,
      "activations/layer11_attention_weight_min": -34.78668975830078,
      "activations/layer12_attention_weight_max": 19.214733123779297,
      "activations/layer12_attention_weight_min": -26.968690872192383,
      "activations/layer13_attention_weight_max": 34.33454513549805,
      "activations/layer13_attention_weight_min": -28.851490020751953,
      "activations/layer14_attention_weight_max": 35.21532440185547,
      "activations/layer14_attention_weight_min": -31.066070556640625,
      "activations/layer15_attention_weight_max": 32.93680191040039,
      "activations/layer15_attention_weight_min": -29.263309478759766,
      "activations/layer16_attention_weight_max": 35.485443115234375,
      "activations/layer16_attention_weight_min": -33.60451126098633,
      "activations/layer17_attention_weight_max": 51.962608337402344,
      "activations/layer17_attention_weight_min": -45.66813278198242,
      "activations/layer18_attention_weight_max": 48.83591079711914,
      "activations/layer18_attention_weight_min": -40.813377380371094,
      "activations/layer19_attention_weight_max": 25.590579986572266,
      "activations/layer19_attention_weight_min": -21.90028190612793,
      "activations/layer1_attention_weight_max": 15.684591293334961,
      "activations/layer1_attention_weight_min": -13.774791717529297,
      "activations/layer20_attention_weight_max": 25.060379028320312,
      "activations/layer20_attention_weight_min": -22.201709747314453,
      "activations/layer21_attention_weight_max": 34.252864837646484,
      "activations/layer21_attention_weight_min": -20.010440826416016,
      "activations/layer22_attention_weight_max": 31.92562484741211,
      "activations/layer22_attention_weight_min": -25.1934871673584,
      "activations/layer23_attention_weight_max": 37.50836944580078,
      "activations/layer23_attention_weight_min": -24.158588409423828,
      "activations/layer2_attention_weight_max": 31.427579879760742,
      "activations/layer2_attention_weight_min": -30.160818099975586,
      "activations/layer3_attention_weight_max": 86.87033081054688,
      "activations/layer3_attention_weight_min": -86.78263854980469,
      "activations/layer4_attention_weight_max": 81.96932983398438,
      "activations/layer4_attention_weight_min": -81.53502655029297,
      "activations/layer5_attention_weight_max": 60.47396469116211,
      "activations/layer5_attention_weight_min": -74.55049896240234,
      "activations/layer6_attention_weight_max": 50.106204986572266,
      "activations/layer6_attention_weight_min": -50.0772590637207,
      "activations/layer7_attention_weight_max": 68.86421203613281,
      "activations/layer7_attention_weight_min": -65.81720733642578,
      "activations/layer8_attention_weight_max": 47.68828201293945,
      "activations/layer8_attention_weight_min": -49.00859451293945,
      "activations/layer9_attention_weight_max": 41.79124069213867,
      "activations/layer9_attention_weight_min": -47.26615905761719,
      "epoch": 21.45,
      "learning_rate": 1.181590909090909e-05,
      "loss": 2.7177,
      "step": 369100
    },
    {
      "activations/layer0_attention_weight_max": 15.98247241973877,
      "activations/layer0_attention_weight_min": -14.172264099121094,
      "activations/layer10_attention_weight_max": 37.56480026245117,
      "activations/layer10_attention_weight_min": -37.5337028503418,
      "activations/layer11_attention_weight_max": 33.96982955932617,
      "activations/layer11_attention_weight_min": -35.94023513793945,
      "activations/layer12_attention_weight_max": 19.057476043701172,
      "activations/layer12_attention_weight_min": -23.58732032775879,
      "activations/layer13_attention_weight_max": 32.70655822753906,
      "activations/layer13_attention_weight_min": -27.591386795043945,
      "activations/layer14_attention_weight_max": 35.64406967163086,
      "activations/layer14_attention_weight_min": -30.825504302978516,
      "activations/layer15_attention_weight_max": 32.52494430541992,
      "activations/layer15_attention_weight_min": -31.703638076782227,
      "activations/layer16_attention_weight_max": 33.15695571899414,
      "activations/layer16_attention_weight_min": -31.743120193481445,
      "activations/layer17_attention_weight_max": 51.26149368286133,
      "activations/layer17_attention_weight_min": -43.52238082885742,
      "activations/layer18_attention_weight_max": 51.73606491088867,
      "activations/layer18_attention_weight_min": -40.829254150390625,
      "activations/layer19_attention_weight_max": 24.817689895629883,
      "activations/layer19_attention_weight_min": -21.721010208129883,
      "activations/layer1_attention_weight_max": 15.0682373046875,
      "activations/layer1_attention_weight_min": -14.265650749206543,
      "activations/layer20_attention_weight_max": 22.746965408325195,
      "activations/layer20_attention_weight_min": -21.625675201416016,
      "activations/layer21_attention_weight_max": 39.92934036254883,
      "activations/layer21_attention_weight_min": -24.942636489868164,
      "activations/layer22_attention_weight_max": 29.67359161376953,
      "activations/layer22_attention_weight_min": -24.518199920654297,
      "activations/layer23_attention_weight_max": 39.558353424072266,
      "activations/layer23_attention_weight_min": -23.156330108642578,
      "activations/layer2_attention_weight_max": 30.535314559936523,
      "activations/layer2_attention_weight_min": -30.078479766845703,
      "activations/layer3_attention_weight_max": 88.74515533447266,
      "activations/layer3_attention_weight_min": -88.41732025146484,
      "activations/layer4_attention_weight_max": 88.44916534423828,
      "activations/layer4_attention_weight_min": -87.30391693115234,
      "activations/layer5_attention_weight_max": 62.393550872802734,
      "activations/layer5_attention_weight_min": -71.5805892944336,
      "activations/layer6_attention_weight_max": 50.06873321533203,
      "activations/layer6_attention_weight_min": -54.180946350097656,
      "activations/layer7_attention_weight_max": 69.54154205322266,
      "activations/layer7_attention_weight_min": -68.95056915283203,
      "activations/layer8_attention_weight_max": 46.94682693481445,
      "activations/layer8_attention_weight_min": -52.285675048828125,
      "activations/layer9_attention_weight_max": 44.07585906982422,
      "activations/layer9_attention_weight_min": -46.63312530517578,
      "epoch": 21.45,
      "learning_rate": 1.1796969696969696e-05,
      "loss": 2.7033,
      "step": 369150
    },
    {
      "activations/layer0_attention_weight_max": 16.205835342407227,
      "activations/layer0_attention_weight_min": -14.22922420501709,
      "activations/layer10_attention_weight_max": 37.0898323059082,
      "activations/layer10_attention_weight_min": -36.57722854614258,
      "activations/layer11_attention_weight_max": 35.760841369628906,
      "activations/layer11_attention_weight_min": -37.02362823486328,
      "activations/layer12_attention_weight_max": 20.69576072692871,
      "activations/layer12_attention_weight_min": -25.939306259155273,
      "activations/layer13_attention_weight_max": 38.0090217590332,
      "activations/layer13_attention_weight_min": -29.35688018798828,
      "activations/layer14_attention_weight_max": 39.37250518798828,
      "activations/layer14_attention_weight_min": -34.25606155395508,
      "activations/layer15_attention_weight_max": 38.054168701171875,
      "activations/layer15_attention_weight_min": -33.16107177734375,
      "activations/layer16_attention_weight_max": 39.757232666015625,
      "activations/layer16_attention_weight_min": -36.73047637939453,
      "activations/layer17_attention_weight_max": 58.861572265625,
      "activations/layer17_attention_weight_min": -52.876121520996094,
      "activations/layer18_attention_weight_max": 57.49769973754883,
      "activations/layer18_attention_weight_min": -45.864654541015625,
      "activations/layer19_attention_weight_max": 29.161590576171875,
      "activations/layer19_attention_weight_min": -26.304601669311523,
      "activations/layer1_attention_weight_max": 15.68500804901123,
      "activations/layer1_attention_weight_min": -16.059059143066406,
      "activations/layer20_attention_weight_max": 24.67150115966797,
      "activations/layer20_attention_weight_min": -23.130451202392578,
      "activations/layer21_attention_weight_max": 45.809181213378906,
      "activations/layer21_attention_weight_min": -24.47795867919922,
      "activations/layer22_attention_weight_max": 32.79402542114258,
      "activations/layer22_attention_weight_min": -27.51436996459961,
      "activations/layer23_attention_weight_max": 40.834861755371094,
      "activations/layer23_attention_weight_min": -26.539785385131836,
      "activations/layer2_attention_weight_max": 31.51019859313965,
      "activations/layer2_attention_weight_min": -31.156164169311523,
      "activations/layer3_attention_weight_max": 87.80841827392578,
      "activations/layer3_attention_weight_min": -85.5096435546875,
      "activations/layer4_attention_weight_max": 85.58977508544922,
      "activations/layer4_attention_weight_min": -86.90094757080078,
      "activations/layer5_attention_weight_max": 59.1729850769043,
      "activations/layer5_attention_weight_min": -74.801513671875,
      "activations/layer6_attention_weight_max": 50.64534378051758,
      "activations/layer6_attention_weight_min": -49.755184173583984,
      "activations/layer7_attention_weight_max": 66.67881774902344,
      "activations/layer7_attention_weight_min": -68.42953491210938,
      "activations/layer8_attention_weight_max": 44.3016242980957,
      "activations/layer8_attention_weight_min": -51.26198196411133,
      "activations/layer9_attention_weight_max": 42.024967193603516,
      "activations/layer9_attention_weight_min": -46.49031448364258,
      "epoch": 21.45,
      "learning_rate": 1.1778030303030301e-05,
      "loss": 2.6945,
      "step": 369200
    },
    {
      "activations/layer0_attention_weight_max": 16.210975646972656,
      "activations/layer0_attention_weight_min": -14.368526458740234,
      "activations/layer10_attention_weight_max": 36.349769592285156,
      "activations/layer10_attention_weight_min": -37.06781005859375,
      "activations/layer11_attention_weight_max": 34.947383880615234,
      "activations/layer11_attention_weight_min": -35.398590087890625,
      "activations/layer12_attention_weight_max": 18.445968627929688,
      "activations/layer12_attention_weight_min": -22.63556671142578,
      "activations/layer13_attention_weight_max": 34.64353561401367,
      "activations/layer13_attention_weight_min": -26.965282440185547,
      "activations/layer14_attention_weight_max": 33.441009521484375,
      "activations/layer14_attention_weight_min": -29.241931915283203,
      "activations/layer15_attention_weight_max": 31.064367294311523,
      "activations/layer15_attention_weight_min": -29.906681060791016,
      "activations/layer16_attention_weight_max": 31.82712745666504,
      "activations/layer16_attention_weight_min": -31.868961334228516,
      "activations/layer17_attention_weight_max": 49.566925048828125,
      "activations/layer17_attention_weight_min": -43.844661712646484,
      "activations/layer18_attention_weight_max": 41.92304611206055,
      "activations/layer18_attention_weight_min": -37.376155853271484,
      "activations/layer19_attention_weight_max": 23.26350975036621,
      "activations/layer19_attention_weight_min": -20.5024471282959,
      "activations/layer1_attention_weight_max": 14.881363868713379,
      "activations/layer1_attention_weight_min": -14.137903213500977,
      "activations/layer20_attention_weight_max": 19.904338836669922,
      "activations/layer20_attention_weight_min": -21.86270523071289,
      "activations/layer21_attention_weight_max": 32.38593292236328,
      "activations/layer21_attention_weight_min": -20.828699111938477,
      "activations/layer22_attention_weight_max": 26.95737648010254,
      "activations/layer22_attention_weight_min": -25.179182052612305,
      "activations/layer23_attention_weight_max": 33.96311569213867,
      "activations/layer23_attention_weight_min": -22.422679901123047,
      "activations/layer2_attention_weight_max": 31.83620262145996,
      "activations/layer2_attention_weight_min": -30.211605072021484,
      "activations/layer3_attention_weight_max": 93.55904388427734,
      "activations/layer3_attention_weight_min": -88.40018463134766,
      "activations/layer4_attention_weight_max": 82.5562973022461,
      "activations/layer4_attention_weight_min": -82.41899871826172,
      "activations/layer5_attention_weight_max": 58.79331970214844,
      "activations/layer5_attention_weight_min": -75.41030883789062,
      "activations/layer6_attention_weight_max": 48.89195251464844,
      "activations/layer6_attention_weight_min": -50.23824691772461,
      "activations/layer7_attention_weight_max": 67.36925506591797,
      "activations/layer7_attention_weight_min": -65.9595718383789,
      "activations/layer8_attention_weight_max": 50.22732162475586,
      "activations/layer8_attention_weight_min": -49.10874938964844,
      "activations/layer9_attention_weight_max": 50.172359466552734,
      "activations/layer9_attention_weight_min": -47.9260368347168,
      "epoch": 21.46,
      "learning_rate": 1.1759090909090908e-05,
      "loss": 2.712,
      "step": 369250
    },
    {
      "activations/layer0_attention_weight_max": 15.643736839294434,
      "activations/layer0_attention_weight_min": -13.938163757324219,
      "activations/layer10_attention_weight_max": 37.35762023925781,
      "activations/layer10_attention_weight_min": -36.08695602416992,
      "activations/layer11_attention_weight_max": 34.71512985229492,
      "activations/layer11_attention_weight_min": -35.73521041870117,
      "activations/layer12_attention_weight_max": 22.246240615844727,
      "activations/layer12_attention_weight_min": -23.45590591430664,
      "activations/layer13_attention_weight_max": 34.607208251953125,
      "activations/layer13_attention_weight_min": -28.569137573242188,
      "activations/layer14_attention_weight_max": 39.22212600708008,
      "activations/layer14_attention_weight_min": -34.57490921020508,
      "activations/layer15_attention_weight_max": 36.71965408325195,
      "activations/layer15_attention_weight_min": -29.901058197021484,
      "activations/layer16_attention_weight_max": 36.57640838623047,
      "activations/layer16_attention_weight_min": -34.098697662353516,
      "activations/layer17_attention_weight_max": 53.918487548828125,
      "activations/layer17_attention_weight_min": -45.86577224731445,
      "activations/layer18_attention_weight_max": 50.0603141784668,
      "activations/layer18_attention_weight_min": -41.294403076171875,
      "activations/layer19_attention_weight_max": 23.7605037689209,
      "activations/layer19_attention_weight_min": -23.907848358154297,
      "activations/layer1_attention_weight_max": 15.07144832611084,
      "activations/layer1_attention_weight_min": -15.52096939086914,
      "activations/layer20_attention_weight_max": 23.17571449279785,
      "activations/layer20_attention_weight_min": -21.707530975341797,
      "activations/layer21_attention_weight_max": 33.78220748901367,
      "activations/layer21_attention_weight_min": -20.197481155395508,
      "activations/layer22_attention_weight_max": 30.869037628173828,
      "activations/layer22_attention_weight_min": -26.23649024963379,
      "activations/layer23_attention_weight_max": 37.478450775146484,
      "activations/layer23_attention_weight_min": -22.458175659179688,
      "activations/layer2_attention_weight_max": 32.84889221191406,
      "activations/layer2_attention_weight_min": -32.934993743896484,
      "activations/layer3_attention_weight_max": 89.1016616821289,
      "activations/layer3_attention_weight_min": -91.62346649169922,
      "activations/layer4_attention_weight_max": 84.89287567138672,
      "activations/layer4_attention_weight_min": -84.56444549560547,
      "activations/layer5_attention_weight_max": 60.812679290771484,
      "activations/layer5_attention_weight_min": -72.14311981201172,
      "activations/layer6_attention_weight_max": 53.0660285949707,
      "activations/layer6_attention_weight_min": -54.25672149658203,
      "activations/layer7_attention_weight_max": 71.81132507324219,
      "activations/layer7_attention_weight_min": -68.1453857421875,
      "activations/layer8_attention_weight_max": 47.24037551879883,
      "activations/layer8_attention_weight_min": -50.84737014770508,
      "activations/layer9_attention_weight_max": 44.00228500366211,
      "activations/layer9_attention_weight_min": -49.17207717895508,
      "epoch": 21.46,
      "learning_rate": 1.1740151515151514e-05,
      "loss": 2.7055,
      "step": 369300
    },
    {
      "activations/layer0_attention_weight_max": 16.217809677124023,
      "activations/layer0_attention_weight_min": -14.16818904876709,
      "activations/layer10_attention_weight_max": 39.50578308105469,
      "activations/layer10_attention_weight_min": -37.009891510009766,
      "activations/layer11_attention_weight_max": 35.6845703125,
      "activations/layer11_attention_weight_min": -34.811180114746094,
      "activations/layer12_attention_weight_max": 20.325794219970703,
      "activations/layer12_attention_weight_min": -25.37630844116211,
      "activations/layer13_attention_weight_max": 47.1689338684082,
      "activations/layer13_attention_weight_min": -29.452722549438477,
      "activations/layer14_attention_weight_max": 39.57564926147461,
      "activations/layer14_attention_weight_min": -29.880413055419922,
      "activations/layer15_attention_weight_max": 46.03467559814453,
      "activations/layer15_attention_weight_min": -30.8748779296875,
      "activations/layer16_attention_weight_max": 36.90032958984375,
      "activations/layer16_attention_weight_min": -34.96597671508789,
      "activations/layer17_attention_weight_max": 53.779685974121094,
      "activations/layer17_attention_weight_min": -47.836761474609375,
      "activations/layer18_attention_weight_max": 46.92266082763672,
      "activations/layer18_attention_weight_min": -43.273521423339844,
      "activations/layer19_attention_weight_max": 27.17806053161621,
      "activations/layer19_attention_weight_min": -23.719449996948242,
      "activations/layer1_attention_weight_max": 16.84479331970215,
      "activations/layer1_attention_weight_min": -14.881916046142578,
      "activations/layer20_attention_weight_max": 24.85131072998047,
      "activations/layer20_attention_weight_min": -21.70663833618164,
      "activations/layer21_attention_weight_max": 38.26027297973633,
      "activations/layer21_attention_weight_min": -22.08258628845215,
      "activations/layer22_attention_weight_max": 30.09657859802246,
      "activations/layer22_attention_weight_min": -24.6055908203125,
      "activations/layer23_attention_weight_max": 38.834083557128906,
      "activations/layer23_attention_weight_min": -24.694639205932617,
      "activations/layer2_attention_weight_max": 31.46411895751953,
      "activations/layer2_attention_weight_min": -29.864530563354492,
      "activations/layer3_attention_weight_max": 89.62761688232422,
      "activations/layer3_attention_weight_min": -87.4286117553711,
      "activations/layer4_attention_weight_max": 83.09746551513672,
      "activations/layer4_attention_weight_min": -85.25165557861328,
      "activations/layer5_attention_weight_max": 60.651405334472656,
      "activations/layer5_attention_weight_min": -75.08811950683594,
      "activations/layer6_attention_weight_max": 49.56978988647461,
      "activations/layer6_attention_weight_min": -50.58539581298828,
      "activations/layer7_attention_weight_max": 73.1782455444336,
      "activations/layer7_attention_weight_min": -66.75233459472656,
      "activations/layer8_attention_weight_max": 49.01835632324219,
      "activations/layer8_attention_weight_min": -50.743988037109375,
      "activations/layer9_attention_weight_max": 51.64811325073242,
      "activations/layer9_attention_weight_min": -46.86636734008789,
      "epoch": 21.46,
      "learning_rate": 1.1721590909090908e-05,
      "loss": 2.714,
      "step": 369350
    },
    {
      "activations/layer0_attention_weight_max": 16.002065658569336,
      "activations/layer0_attention_weight_min": -13.988387107849121,
      "activations/layer10_attention_weight_max": 34.65146255493164,
      "activations/layer10_attention_weight_min": -34.08490753173828,
      "activations/layer11_attention_weight_max": 31.83283042907715,
      "activations/layer11_attention_weight_min": -33.11646270751953,
      "activations/layer12_attention_weight_max": 19.823057174682617,
      "activations/layer12_attention_weight_min": -27.29716682434082,
      "activations/layer13_attention_weight_max": 33.68162536621094,
      "activations/layer13_attention_weight_min": -26.626300811767578,
      "activations/layer14_attention_weight_max": 35.04800796508789,
      "activations/layer14_attention_weight_min": -29.889785766601562,
      "activations/layer15_attention_weight_max": 33.5694580078125,
      "activations/layer15_attention_weight_min": -30.013015747070312,
      "activations/layer16_attention_weight_max": 38.03469467163086,
      "activations/layer16_attention_weight_min": -33.58314514160156,
      "activations/layer17_attention_weight_max": 52.3543701171875,
      "activations/layer17_attention_weight_min": -47.93661880493164,
      "activations/layer18_attention_weight_max": 51.204437255859375,
      "activations/layer18_attention_weight_min": -42.28325653076172,
      "activations/layer19_attention_weight_max": 24.099294662475586,
      "activations/layer19_attention_weight_min": -25.81337547302246,
      "activations/layer1_attention_weight_max": 15.491947174072266,
      "activations/layer1_attention_weight_min": -15.101131439208984,
      "activations/layer20_attention_weight_max": 24.466941833496094,
      "activations/layer20_attention_weight_min": -23.959726333618164,
      "activations/layer21_attention_weight_max": 38.42218780517578,
      "activations/layer21_attention_weight_min": -23.28593635559082,
      "activations/layer22_attention_weight_max": 28.5941104888916,
      "activations/layer22_attention_weight_min": -25.6295223236084,
      "activations/layer23_attention_weight_max": 37.75228500366211,
      "activations/layer23_attention_weight_min": -24.87629508972168,
      "activations/layer2_attention_weight_max": 30.827842712402344,
      "activations/layer2_attention_weight_min": -31.94710350036621,
      "activations/layer3_attention_weight_max": 88.98160552978516,
      "activations/layer3_attention_weight_min": -93.71636962890625,
      "activations/layer4_attention_weight_max": 85.89962005615234,
      "activations/layer4_attention_weight_min": -82.1310043334961,
      "activations/layer5_attention_weight_max": 58.83300018310547,
      "activations/layer5_attention_weight_min": -71.42864990234375,
      "activations/layer6_attention_weight_max": 50.186458587646484,
      "activations/layer6_attention_weight_min": -50.308048248291016,
      "activations/layer7_attention_weight_max": 65.43849182128906,
      "activations/layer7_attention_weight_min": -66.04425048828125,
      "activations/layer8_attention_weight_max": 44.331233978271484,
      "activations/layer8_attention_weight_min": -48.13823318481445,
      "activations/layer9_attention_weight_max": 41.27214431762695,
      "activations/layer9_attention_weight_min": -45.306087493896484,
      "epoch": 21.46,
      "learning_rate": 1.1702651515151513e-05,
      "loss": 2.7288,
      "step": 369400
    },
    {
      "activations/layer0_attention_weight_max": 15.979083061218262,
      "activations/layer0_attention_weight_min": -13.973355293273926,
      "activations/layer10_attention_weight_max": 34.067771911621094,
      "activations/layer10_attention_weight_min": -35.84957504272461,
      "activations/layer11_attention_weight_max": 31.117795944213867,
      "activations/layer11_attention_weight_min": -36.54048156738281,
      "activations/layer12_attention_weight_max": 18.719493865966797,
      "activations/layer12_attention_weight_min": -29.203365325927734,
      "activations/layer13_attention_weight_max": 37.19758224487305,
      "activations/layer13_attention_weight_min": -29.262819290161133,
      "activations/layer14_attention_weight_max": 39.86941909790039,
      "activations/layer14_attention_weight_min": -34.306148529052734,
      "activations/layer15_attention_weight_max": 38.839027404785156,
      "activations/layer15_attention_weight_min": -32.24225997924805,
      "activations/layer16_attention_weight_max": 38.47076416015625,
      "activations/layer16_attention_weight_min": -37.874786376953125,
      "activations/layer17_attention_weight_max": 57.21344757080078,
      "activations/layer17_attention_weight_min": -52.292503356933594,
      "activations/layer18_attention_weight_max": 58.14188003540039,
      "activations/layer18_attention_weight_min": -47.82392501831055,
      "activations/layer19_attention_weight_max": 28.63361167907715,
      "activations/layer19_attention_weight_min": -22.97235107421875,
      "activations/layer1_attention_weight_max": 15.535917282104492,
      "activations/layer1_attention_weight_min": -14.473068237304688,
      "activations/layer20_attention_weight_max": 25.94378662109375,
      "activations/layer20_attention_weight_min": -22.236406326293945,
      "activations/layer21_attention_weight_max": 42.82318878173828,
      "activations/layer21_attention_weight_min": -19.645315170288086,
      "activations/layer22_attention_weight_max": 34.16989517211914,
      "activations/layer22_attention_weight_min": -24.752914428710938,
      "activations/layer23_attention_weight_max": 40.70466613769531,
      "activations/layer23_attention_weight_min": -26.77896499633789,
      "activations/layer2_attention_weight_max": 31.094274520874023,
      "activations/layer2_attention_weight_min": -29.819351196289062,
      "activations/layer3_attention_weight_max": 85.88616180419922,
      "activations/layer3_attention_weight_min": -85.390625,
      "activations/layer4_attention_weight_max": 85.83946990966797,
      "activations/layer4_attention_weight_min": -83.73006439208984,
      "activations/layer5_attention_weight_max": 62.88241195678711,
      "activations/layer5_attention_weight_min": -75.2935791015625,
      "activations/layer6_attention_weight_max": 49.234825134277344,
      "activations/layer6_attention_weight_min": -51.4800910949707,
      "activations/layer7_attention_weight_max": 71.05009460449219,
      "activations/layer7_attention_weight_min": -69.07180786132812,
      "activations/layer8_attention_weight_max": 46.48969268798828,
      "activations/layer8_attention_weight_min": -51.15602111816406,
      "activations/layer9_attention_weight_max": 44.277099609375,
      "activations/layer9_attention_weight_min": -47.0345573425293,
      "epoch": 21.47,
      "learning_rate": 1.168371212121212e-05,
      "loss": 2.7259,
      "step": 369450
    },
    {
      "activations/layer0_attention_weight_max": 15.362276077270508,
      "activations/layer0_attention_weight_min": -13.877626419067383,
      "activations/layer10_attention_weight_max": 37.03394317626953,
      "activations/layer10_attention_weight_min": -36.32169723510742,
      "activations/layer11_attention_weight_max": 32.15850830078125,
      "activations/layer11_attention_weight_min": -36.83407974243164,
      "activations/layer12_attention_weight_max": 19.53780174255371,
      "activations/layer12_attention_weight_min": -24.8450927734375,
      "activations/layer13_attention_weight_max": 35.06668472290039,
      "activations/layer13_attention_weight_min": -28.675800323486328,
      "activations/layer14_attention_weight_max": 36.586158752441406,
      "activations/layer14_attention_weight_min": -30.507736206054688,
      "activations/layer15_attention_weight_max": 35.85960006713867,
      "activations/layer15_attention_weight_min": -31.1219425201416,
      "activations/layer16_attention_weight_max": 35.68519973754883,
      "activations/layer16_attention_weight_min": -33.75724411010742,
      "activations/layer17_attention_weight_max": 52.98503494262695,
      "activations/layer17_attention_weight_min": -46.69852066040039,
      "activations/layer18_attention_weight_max": 48.98179626464844,
      "activations/layer18_attention_weight_min": -41.29733657836914,
      "activations/layer19_attention_weight_max": 23.766164779663086,
      "activations/layer19_attention_weight_min": -21.978731155395508,
      "activations/layer1_attention_weight_max": 15.197010040283203,
      "activations/layer1_attention_weight_min": -13.584985733032227,
      "activations/layer20_attention_weight_max": 22.335783004760742,
      "activations/layer20_attention_weight_min": -21.987382888793945,
      "activations/layer21_attention_weight_max": 36.60451126098633,
      "activations/layer21_attention_weight_min": -20.465688705444336,
      "activations/layer22_attention_weight_max": 30.11524772644043,
      "activations/layer22_attention_weight_min": -25.834096908569336,
      "activations/layer23_attention_weight_max": 32.20888137817383,
      "activations/layer23_attention_weight_min": -23.85324478149414,
      "activations/layer2_attention_weight_max": 30.452198028564453,
      "activations/layer2_attention_weight_min": -29.9440975189209,
      "activations/layer3_attention_weight_max": 91.42598724365234,
      "activations/layer3_attention_weight_min": -87.71232604980469,
      "activations/layer4_attention_weight_max": 84.5335693359375,
      "activations/layer4_attention_weight_min": -86.24585723876953,
      "activations/layer5_attention_weight_max": 64.36093139648438,
      "activations/layer5_attention_weight_min": -71.77070617675781,
      "activations/layer6_attention_weight_max": 50.07900619506836,
      "activations/layer6_attention_weight_min": -51.562984466552734,
      "activations/layer7_attention_weight_max": 66.55740356445312,
      "activations/layer7_attention_weight_min": -66.87371063232422,
      "activations/layer8_attention_weight_max": 46.51417922973633,
      "activations/layer8_attention_weight_min": -49.59418869018555,
      "activations/layer9_attention_weight_max": 44.16830062866211,
      "activations/layer9_attention_weight_min": -48.31096649169922,
      "epoch": 21.47,
      "learning_rate": 1.1664772727272725e-05,
      "loss": 2.7183,
      "step": 369500
    },
    {
      "activations/layer0_attention_weight_max": 14.857698440551758,
      "activations/layer0_attention_weight_min": -14.230643272399902,
      "activations/layer10_attention_weight_max": 39.16858673095703,
      "activations/layer10_attention_weight_min": -38.972572326660156,
      "activations/layer11_attention_weight_max": 38.92726516723633,
      "activations/layer11_attention_weight_min": -38.584136962890625,
      "activations/layer12_attention_weight_max": 19.376707077026367,
      "activations/layer12_attention_weight_min": -22.62847137451172,
      "activations/layer13_attention_weight_max": 36.79531478881836,
      "activations/layer13_attention_weight_min": -27.38393211364746,
      "activations/layer14_attention_weight_max": 40.73841857910156,
      "activations/layer14_attention_weight_min": -29.953018188476562,
      "activations/layer15_attention_weight_max": 36.28898239135742,
      "activations/layer15_attention_weight_min": -30.57762336730957,
      "activations/layer16_attention_weight_max": 37.38027572631836,
      "activations/layer16_attention_weight_min": -34.51676559448242,
      "activations/layer17_attention_weight_max": 52.70103454589844,
      "activations/layer17_attention_weight_min": -46.64326477050781,
      "activations/layer18_attention_weight_max": 46.40668869018555,
      "activations/layer18_attention_weight_min": -39.83115005493164,
      "activations/layer19_attention_weight_max": 24.58745002746582,
      "activations/layer19_attention_weight_min": -22.724424362182617,
      "activations/layer1_attention_weight_max": 14.989694595336914,
      "activations/layer1_attention_weight_min": -14.148162841796875,
      "activations/layer20_attention_weight_max": 24.224607467651367,
      "activations/layer20_attention_weight_min": -20.297332763671875,
      "activations/layer21_attention_weight_max": 39.794097900390625,
      "activations/layer21_attention_weight_min": -20.40323257446289,
      "activations/layer22_attention_weight_max": 31.110700607299805,
      "activations/layer22_attention_weight_min": -24.595069885253906,
      "activations/layer23_attention_weight_max": 40.63752365112305,
      "activations/layer23_attention_weight_min": -24.60755729675293,
      "activations/layer2_attention_weight_max": 30.504905700683594,
      "activations/layer2_attention_weight_min": -29.17593002319336,
      "activations/layer3_attention_weight_max": 88.63080596923828,
      "activations/layer3_attention_weight_min": -87.21947479248047,
      "activations/layer4_attention_weight_max": 85.60814666748047,
      "activations/layer4_attention_weight_min": -86.12586212158203,
      "activations/layer5_attention_weight_max": 64.92558288574219,
      "activations/layer5_attention_weight_min": -74.01301574707031,
      "activations/layer6_attention_weight_max": 51.45771789550781,
      "activations/layer6_attention_weight_min": -53.54421615600586,
      "activations/layer7_attention_weight_max": 74.30674743652344,
      "activations/layer7_attention_weight_min": -70.30449676513672,
      "activations/layer8_attention_weight_max": 48.6667594909668,
      "activations/layer8_attention_weight_min": -52.386741638183594,
      "activations/layer9_attention_weight_max": 46.74589538574219,
      "activations/layer9_attention_weight_min": -48.99103546142578,
      "epoch": 21.47,
      "learning_rate": 1.1645833333333331e-05,
      "loss": 2.7101,
      "step": 369550
    },
    {
      "activations/layer0_attention_weight_max": 16.15523338317871,
      "activations/layer0_attention_weight_min": -13.828594207763672,
      "activations/layer10_attention_weight_max": 34.957557678222656,
      "activations/layer10_attention_weight_min": -35.22206497192383,
      "activations/layer11_attention_weight_max": 33.504695892333984,
      "activations/layer11_attention_weight_min": -36.165618896484375,
      "activations/layer12_attention_weight_max": 19.392248153686523,
      "activations/layer12_attention_weight_min": -25.579505920410156,
      "activations/layer13_attention_weight_max": 34.22352600097656,
      "activations/layer13_attention_weight_min": -29.291566848754883,
      "activations/layer14_attention_weight_max": 34.52085494995117,
      "activations/layer14_attention_weight_min": -30.202457427978516,
      "activations/layer15_attention_weight_max": 33.78302764892578,
      "activations/layer15_attention_weight_min": -30.523601531982422,
      "activations/layer16_attention_weight_max": 34.00223159790039,
      "activations/layer16_attention_weight_min": -30.59932518005371,
      "activations/layer17_attention_weight_max": 51.97994613647461,
      "activations/layer17_attention_weight_min": -43.72744369506836,
      "activations/layer18_attention_weight_max": 44.66352844238281,
      "activations/layer18_attention_weight_min": -38.09244155883789,
      "activations/layer19_attention_weight_max": 22.410301208496094,
      "activations/layer19_attention_weight_min": -24.069169998168945,
      "activations/layer1_attention_weight_max": 15.391047477722168,
      "activations/layer1_attention_weight_min": -13.537491798400879,
      "activations/layer20_attention_weight_max": 22.831655502319336,
      "activations/layer20_attention_weight_min": -21.959880828857422,
      "activations/layer21_attention_weight_max": 32.69916534423828,
      "activations/layer21_attention_weight_min": -22.183147430419922,
      "activations/layer22_attention_weight_max": 27.312606811523438,
      "activations/layer22_attention_weight_min": -25.50391960144043,
      "activations/layer23_attention_weight_max": 34.126895904541016,
      "activations/layer23_attention_weight_min": -22.26970672607422,
      "activations/layer2_attention_weight_max": 32.41504669189453,
      "activations/layer2_attention_weight_min": -30.305585861206055,
      "activations/layer3_attention_weight_max": 89.4871826171875,
      "activations/layer3_attention_weight_min": -86.65702819824219,
      "activations/layer4_attention_weight_max": 84.51445007324219,
      "activations/layer4_attention_weight_min": -81.1273193359375,
      "activations/layer5_attention_weight_max": 63.08917236328125,
      "activations/layer5_attention_weight_min": -71.6209716796875,
      "activations/layer6_attention_weight_max": 49.044578552246094,
      "activations/layer6_attention_weight_min": -48.79451370239258,
      "activations/layer7_attention_weight_max": 66.5653076171875,
      "activations/layer7_attention_weight_min": -66.8837661743164,
      "activations/layer8_attention_weight_max": 45.340423583984375,
      "activations/layer8_attention_weight_min": -49.35163497924805,
      "activations/layer9_attention_weight_max": 42.61748504638672,
      "activations/layer9_attention_weight_min": -46.01333236694336,
      "epoch": 21.48,
      "learning_rate": 1.162689393939394e-05,
      "loss": 2.7097,
      "step": 369600
    },
    {
      "activations/layer0_attention_weight_max": 15.907505989074707,
      "activations/layer0_attention_weight_min": -14.010891914367676,
      "activations/layer10_attention_weight_max": 34.095985412597656,
      "activations/layer10_attention_weight_min": -36.138572692871094,
      "activations/layer11_attention_weight_max": 32.427162170410156,
      "activations/layer11_attention_weight_min": -33.86914825439453,
      "activations/layer12_attention_weight_max": 18.81829833984375,
      "activations/layer12_attention_weight_min": -25.834379196166992,
      "activations/layer13_attention_weight_max": 32.10020065307617,
      "activations/layer13_attention_weight_min": -27.42780303955078,
      "activations/layer14_attention_weight_max": 34.68946075439453,
      "activations/layer14_attention_weight_min": -29.243080139160156,
      "activations/layer15_attention_weight_max": 32.167423248291016,
      "activations/layer15_attention_weight_min": -31.024539947509766,
      "activations/layer16_attention_weight_max": 32.760047912597656,
      "activations/layer16_attention_weight_min": -31.76639747619629,
      "activations/layer17_attention_weight_max": 50.81205749511719,
      "activations/layer17_attention_weight_min": -44.76049041748047,
      "activations/layer18_attention_weight_max": 46.141292572021484,
      "activations/layer18_attention_weight_min": -39.13953399658203,
      "activations/layer19_attention_weight_max": 24.880353927612305,
      "activations/layer19_attention_weight_min": -21.16928482055664,
      "activations/layer1_attention_weight_max": 16.03350257873535,
      "activations/layer1_attention_weight_min": -15.800341606140137,
      "activations/layer20_attention_weight_max": 22.458852767944336,
      "activations/layer20_attention_weight_min": -21.302532196044922,
      "activations/layer21_attention_weight_max": 32.25946044921875,
      "activations/layer21_attention_weight_min": -19.971399307250977,
      "activations/layer22_attention_weight_max": 26.787500381469727,
      "activations/layer22_attention_weight_min": -26.613143920898438,
      "activations/layer23_attention_weight_max": 35.47068405151367,
      "activations/layer23_attention_weight_min": -24.260038375854492,
      "activations/layer2_attention_weight_max": 31.771276473999023,
      "activations/layer2_attention_weight_min": -31.81882095336914,
      "activations/layer3_attention_weight_max": 92.69823455810547,
      "activations/layer3_attention_weight_min": -87.6310806274414,
      "activations/layer4_attention_weight_max": 88.1430435180664,
      "activations/layer4_attention_weight_min": -83.726806640625,
      "activations/layer5_attention_weight_max": 63.66502380371094,
      "activations/layer5_attention_weight_min": -77.20166015625,
      "activations/layer6_attention_weight_max": 49.52529525756836,
      "activations/layer6_attention_weight_min": -51.72267532348633,
      "activations/layer7_attention_weight_max": 72.01473999023438,
      "activations/layer7_attention_weight_min": -69.58155059814453,
      "activations/layer8_attention_weight_max": 49.206993103027344,
      "activations/layer8_attention_weight_min": -51.22852325439453,
      "activations/layer9_attention_weight_max": 49.04718780517578,
      "activations/layer9_attention_weight_min": -49.859886169433594,
      "epoch": 21.48,
      "learning_rate": 1.1607954545454546e-05,
      "loss": 2.7032,
      "step": 369650
    },
    {
      "activations/layer0_attention_weight_max": 15.590133666992188,
      "activations/layer0_attention_weight_min": -14.371532440185547,
      "activations/layer10_attention_weight_max": 35.04174041748047,
      "activations/layer10_attention_weight_min": -37.76607131958008,
      "activations/layer11_attention_weight_max": 34.805152893066406,
      "activations/layer11_attention_weight_min": -39.424217224121094,
      "activations/layer12_attention_weight_max": 19.701770782470703,
      "activations/layer12_attention_weight_min": -24.977392196655273,
      "activations/layer13_attention_weight_max": 29.866836547851562,
      "activations/layer13_attention_weight_min": -28.832820892333984,
      "activations/layer14_attention_weight_max": 37.827781677246094,
      "activations/layer14_attention_weight_min": -32.426979064941406,
      "activations/layer15_attention_weight_max": 31.8266544342041,
      "activations/layer15_attention_weight_min": -32.0512809753418,
      "activations/layer16_attention_weight_max": 33.30490493774414,
      "activations/layer16_attention_weight_min": -32.612464904785156,
      "activations/layer17_attention_weight_max": 52.897579193115234,
      "activations/layer17_attention_weight_min": -44.44459533691406,
      "activations/layer18_attention_weight_max": 49.35200881958008,
      "activations/layer18_attention_weight_min": -39.79107666015625,
      "activations/layer19_attention_weight_max": 23.504175186157227,
      "activations/layer19_attention_weight_min": -24.831329345703125,
      "activations/layer1_attention_weight_max": 16.35471534729004,
      "activations/layer1_attention_weight_min": -14.575881958007812,
      "activations/layer20_attention_weight_max": 25.030424118041992,
      "activations/layer20_attention_weight_min": -22.670141220092773,
      "activations/layer21_attention_weight_max": 38.117408752441406,
      "activations/layer21_attention_weight_min": -22.617250442504883,
      "activations/layer22_attention_weight_max": 27.82154655456543,
      "activations/layer22_attention_weight_min": -24.995956420898438,
      "activations/layer23_attention_weight_max": 33.84420394897461,
      "activations/layer23_attention_weight_min": -24.94806671142578,
      "activations/layer2_attention_weight_max": 32.33985900878906,
      "activations/layer2_attention_weight_min": -31.440351486206055,
      "activations/layer3_attention_weight_max": 88.60746765136719,
      "activations/layer3_attention_weight_min": -88.12859344482422,
      "activations/layer4_attention_weight_max": 83.29047393798828,
      "activations/layer4_attention_weight_min": -86.0508041381836,
      "activations/layer5_attention_weight_max": 63.97654342651367,
      "activations/layer5_attention_weight_min": -71.79669189453125,
      "activations/layer6_attention_weight_max": 51.38467788696289,
      "activations/layer6_attention_weight_min": -52.417442321777344,
      "activations/layer7_attention_weight_max": 68.33198547363281,
      "activations/layer7_attention_weight_min": -65.70429992675781,
      "activations/layer8_attention_weight_max": 48.274940490722656,
      "activations/layer8_attention_weight_min": -47.96370315551758,
      "activations/layer9_attention_weight_max": 52.58182907104492,
      "activations/layer9_attention_weight_min": -46.178871154785156,
      "epoch": 21.48,
      "learning_rate": 1.1589015151515151e-05,
      "loss": 2.692,
      "step": 369700
    },
    {
      "activations/layer0_attention_weight_max": 16.885820388793945,
      "activations/layer0_attention_weight_min": -14.0170259475708,
      "activations/layer10_attention_weight_max": 37.07763671875,
      "activations/layer10_attention_weight_min": -35.6413688659668,
      "activations/layer11_attention_weight_max": 35.06606674194336,
      "activations/layer11_attention_weight_min": -34.90989303588867,
      "activations/layer12_attention_weight_max": 21.821252822875977,
      "activations/layer12_attention_weight_min": -25.409650802612305,
      "activations/layer13_attention_weight_max": 52.18350601196289,
      "activations/layer13_attention_weight_min": -30.958423614501953,
      "activations/layer14_attention_weight_max": 35.70457077026367,
      "activations/layer14_attention_weight_min": -30.898374557495117,
      "activations/layer15_attention_weight_max": 32.697322845458984,
      "activations/layer15_attention_weight_min": -31.288299560546875,
      "activations/layer16_attention_weight_max": 34.67154312133789,
      "activations/layer16_attention_weight_min": -34.3152961730957,
      "activations/layer17_attention_weight_max": 51.77251052856445,
      "activations/layer17_attention_weight_min": -48.61473083496094,
      "activations/layer18_attention_weight_max": 49.94004440307617,
      "activations/layer18_attention_weight_min": -43.73964309692383,
      "activations/layer19_attention_weight_max": 23.575956344604492,
      "activations/layer19_attention_weight_min": -23.193998336791992,
      "activations/layer1_attention_weight_max": 16.239879608154297,
      "activations/layer1_attention_weight_min": -14.46361255645752,
      "activations/layer20_attention_weight_max": 22.967618942260742,
      "activations/layer20_attention_weight_min": -26.11031723022461,
      "activations/layer21_attention_weight_max": 37.27465057373047,
      "activations/layer21_attention_weight_min": -23.727157592773438,
      "activations/layer22_attention_weight_max": 27.56063461303711,
      "activations/layer22_attention_weight_min": -24.807039260864258,
      "activations/layer23_attention_weight_max": 36.55706787109375,
      "activations/layer23_attention_weight_min": -24.02220916748047,
      "activations/layer2_attention_weight_max": 32.56755828857422,
      "activations/layer2_attention_weight_min": -31.974735260009766,
      "activations/layer3_attention_weight_max": 90.67635345458984,
      "activations/layer3_attention_weight_min": -90.61767578125,
      "activations/layer4_attention_weight_max": 83.68569946289062,
      "activations/layer4_attention_weight_min": -85.08855438232422,
      "activations/layer5_attention_weight_max": 62.40391159057617,
      "activations/layer5_attention_weight_min": -76.54998016357422,
      "activations/layer6_attention_weight_max": 49.14202117919922,
      "activations/layer6_attention_weight_min": -50.390567779541016,
      "activations/layer7_attention_weight_max": 67.55307006835938,
      "activations/layer7_attention_weight_min": -69.09553527832031,
      "activations/layer8_attention_weight_max": 48.759456634521484,
      "activations/layer8_attention_weight_min": -51.713871002197266,
      "activations/layer9_attention_weight_max": 45.762325286865234,
      "activations/layer9_attention_weight_min": -48.244720458984375,
      "epoch": 21.48,
      "learning_rate": 1.1570075757575758e-05,
      "loss": 2.7203,
      "step": 369750
    },
    {
      "activations/layer0_attention_weight_max": 18.720905303955078,
      "activations/layer0_attention_weight_min": -13.854913711547852,
      "activations/layer10_attention_weight_max": 36.820579528808594,
      "activations/layer10_attention_weight_min": -36.224395751953125,
      "activations/layer11_attention_weight_max": 33.16352081298828,
      "activations/layer11_attention_weight_min": -34.959266662597656,
      "activations/layer12_attention_weight_max": 19.627912521362305,
      "activations/layer12_attention_weight_min": -25.104698181152344,
      "activations/layer13_attention_weight_max": 31.85012435913086,
      "activations/layer13_attention_weight_min": -27.71218490600586,
      "activations/layer14_attention_weight_max": 33.718528747558594,
      "activations/layer14_attention_weight_min": -30.954938888549805,
      "activations/layer15_attention_weight_max": 32.09419631958008,
      "activations/layer15_attention_weight_min": -32.320716857910156,
      "activations/layer16_attention_weight_max": 33.27402114868164,
      "activations/layer16_attention_weight_min": -32.34808349609375,
      "activations/layer17_attention_weight_max": 50.06968307495117,
      "activations/layer17_attention_weight_min": -46.99797821044922,
      "activations/layer18_attention_weight_max": 45.54221725463867,
      "activations/layer18_attention_weight_min": -43.060821533203125,
      "activations/layer19_attention_weight_max": 23.495649337768555,
      "activations/layer19_attention_weight_min": -25.22287368774414,
      "activations/layer1_attention_weight_max": 15.729217529296875,
      "activations/layer1_attention_weight_min": -14.788126945495605,
      "activations/layer20_attention_weight_max": 22.447898864746094,
      "activations/layer20_attention_weight_min": -23.788551330566406,
      "activations/layer21_attention_weight_max": 33.22857666015625,
      "activations/layer21_attention_weight_min": -23.232280731201172,
      "activations/layer22_attention_weight_max": 26.685382843017578,
      "activations/layer22_attention_weight_min": -27.358381271362305,
      "activations/layer23_attention_weight_max": 37.188026428222656,
      "activations/layer23_attention_weight_min": -26.38433074951172,
      "activations/layer2_attention_weight_max": 31.663908004760742,
      "activations/layer2_attention_weight_min": -30.187335968017578,
      "activations/layer3_attention_weight_max": 91.03821563720703,
      "activations/layer3_attention_weight_min": -84.96329498291016,
      "activations/layer4_attention_weight_max": 84.8658447265625,
      "activations/layer4_attention_weight_min": -81.96858215332031,
      "activations/layer5_attention_weight_max": 58.567710876464844,
      "activations/layer5_attention_weight_min": -71.79562377929688,
      "activations/layer6_attention_weight_max": 49.49577713012695,
      "activations/layer6_attention_weight_min": -52.83147430419922,
      "activations/layer7_attention_weight_max": 67.99781036376953,
      "activations/layer7_attention_weight_min": -65.74528503417969,
      "activations/layer8_attention_weight_max": 47.02705383300781,
      "activations/layer8_attention_weight_min": -50.04243850708008,
      "activations/layer9_attention_weight_max": 42.446258544921875,
      "activations/layer9_attention_weight_min": -47.18972396850586,
      "epoch": 21.49,
      "learning_rate": 1.1551136363636362e-05,
      "loss": 2.7106,
      "step": 369800
    },
    {
      "activations/layer0_attention_weight_max": 15.585996627807617,
      "activations/layer0_attention_weight_min": -14.380019187927246,
      "activations/layer10_attention_weight_max": 33.665687561035156,
      "activations/layer10_attention_weight_min": -35.58885192871094,
      "activations/layer11_attention_weight_max": 33.26521301269531,
      "activations/layer11_attention_weight_min": -34.39459228515625,
      "activations/layer12_attention_weight_max": 19.576234817504883,
      "activations/layer12_attention_weight_min": -24.47995376586914,
      "activations/layer13_attention_weight_max": 43.62813186645508,
      "activations/layer13_attention_weight_min": -28.319711685180664,
      "activations/layer14_attention_weight_max": 35.73920440673828,
      "activations/layer14_attention_weight_min": -29.540409088134766,
      "activations/layer15_attention_weight_max": 33.783409118652344,
      "activations/layer15_attention_weight_min": -30.788352966308594,
      "activations/layer16_attention_weight_max": 35.16855239868164,
      "activations/layer16_attention_weight_min": -35.64188766479492,
      "activations/layer17_attention_weight_max": 53.417747497558594,
      "activations/layer17_attention_weight_min": -47.67441177368164,
      "activations/layer18_attention_weight_max": 46.97530746459961,
      "activations/layer18_attention_weight_min": -45.14949417114258,
      "activations/layer19_attention_weight_max": 24.138809204101562,
      "activations/layer19_attention_weight_min": -24.870563507080078,
      "activations/layer1_attention_weight_max": 16.49582290649414,
      "activations/layer1_attention_weight_min": -13.468426704406738,
      "activations/layer20_attention_weight_max": 22.05906867980957,
      "activations/layer20_attention_weight_min": -24.55119514465332,
      "activations/layer21_attention_weight_max": 33.01969909667969,
      "activations/layer21_attention_weight_min": -23.073328018188477,
      "activations/layer22_attention_weight_max": 29.45347785949707,
      "activations/layer22_attention_weight_min": -25.466333389282227,
      "activations/layer23_attention_weight_max": 34.00194549560547,
      "activations/layer23_attention_weight_min": -25.09311294555664,
      "activations/layer2_attention_weight_max": 31.382633209228516,
      "activations/layer2_attention_weight_min": -31.882261276245117,
      "activations/layer3_attention_weight_max": 87.77521514892578,
      "activations/layer3_attention_weight_min": -89.92215728759766,
      "activations/layer4_attention_weight_max": 81.88414764404297,
      "activations/layer4_attention_weight_min": -84.38868713378906,
      "activations/layer5_attention_weight_max": 62.034297943115234,
      "activations/layer5_attention_weight_min": -70.09359741210938,
      "activations/layer6_attention_weight_max": 48.50013732910156,
      "activations/layer6_attention_weight_min": -50.06589889526367,
      "activations/layer7_attention_weight_max": 65.08776092529297,
      "activations/layer7_attention_weight_min": -67.51183319091797,
      "activations/layer8_attention_weight_max": 44.88289260864258,
      "activations/layer8_attention_weight_min": -50.33700180053711,
      "activations/layer9_attention_weight_max": 44.65121078491211,
      "activations/layer9_attention_weight_min": -45.380706787109375,
      "epoch": 21.49,
      "learning_rate": 1.1532196969696969e-05,
      "loss": 2.7247,
      "step": 369850
    },
    {
      "activations/layer0_attention_weight_max": 15.741430282592773,
      "activations/layer0_attention_weight_min": -14.139049530029297,
      "activations/layer10_attention_weight_max": 34.6224250793457,
      "activations/layer10_attention_weight_min": -34.474769592285156,
      "activations/layer11_attention_weight_max": 32.64224624633789,
      "activations/layer11_attention_weight_min": -33.042762756347656,
      "activations/layer12_attention_weight_max": 19.37663459777832,
      "activations/layer12_attention_weight_min": -25.41233253479004,
      "activations/layer13_attention_weight_max": 38.10426330566406,
      "activations/layer13_attention_weight_min": -27.32498550415039,
      "activations/layer14_attention_weight_max": 34.33833694458008,
      "activations/layer14_attention_weight_min": -30.526308059692383,
      "activations/layer15_attention_weight_max": 32.03204345703125,
      "activations/layer15_attention_weight_min": -30.607080459594727,
      "activations/layer16_attention_weight_max": 34.39365768432617,
      "activations/layer16_attention_weight_min": -31.864551544189453,
      "activations/layer17_attention_weight_max": 53.125450134277344,
      "activations/layer17_attention_weight_min": -43.29384994506836,
      "activations/layer18_attention_weight_max": 48.19699478149414,
      "activations/layer18_attention_weight_min": -39.991397857666016,
      "activations/layer19_attention_weight_max": 24.862234115600586,
      "activations/layer19_attention_weight_min": -22.856407165527344,
      "activations/layer1_attention_weight_max": 15.18920612335205,
      "activations/layer1_attention_weight_min": -14.628862380981445,
      "activations/layer20_attention_weight_max": 22.222078323364258,
      "activations/layer20_attention_weight_min": -21.593021392822266,
      "activations/layer21_attention_weight_max": 36.98221206665039,
      "activations/layer21_attention_weight_min": -22.613990783691406,
      "activations/layer22_attention_weight_max": 29.888200759887695,
      "activations/layer22_attention_weight_min": -24.74576759338379,
      "activations/layer23_attention_weight_max": 36.63556671142578,
      "activations/layer23_attention_weight_min": -24.706830978393555,
      "activations/layer2_attention_weight_max": 31.12319564819336,
      "activations/layer2_attention_weight_min": -30.389623641967773,
      "activations/layer3_attention_weight_max": 86.01156616210938,
      "activations/layer3_attention_weight_min": -89.65129089355469,
      "activations/layer4_attention_weight_max": 82.52046966552734,
      "activations/layer4_attention_weight_min": -82.37570190429688,
      "activations/layer5_attention_weight_max": 62.22158432006836,
      "activations/layer5_attention_weight_min": -70.99272155761719,
      "activations/layer6_attention_weight_max": 47.3884162902832,
      "activations/layer6_attention_weight_min": -51.6496696472168,
      "activations/layer7_attention_weight_max": 70.51371002197266,
      "activations/layer7_attention_weight_min": -65.58765411376953,
      "activations/layer8_attention_weight_max": 45.86580276489258,
      "activations/layer8_attention_weight_min": -50.027889251708984,
      "activations/layer9_attention_weight_max": 44.19071578979492,
      "activations/layer9_attention_weight_min": -44.33991622924805,
      "epoch": 21.49,
      "learning_rate": 1.1513257575757576e-05,
      "loss": 2.7047,
      "step": 369900
    },
    {
      "activations/layer0_attention_weight_max": 15.782917976379395,
      "activations/layer0_attention_weight_min": -14.164567947387695,
      "activations/layer10_attention_weight_max": 37.10075378417969,
      "activations/layer10_attention_weight_min": -37.44865798950195,
      "activations/layer11_attention_weight_max": 33.302345275878906,
      "activations/layer11_attention_weight_min": -34.19951629638672,
      "activations/layer12_attention_weight_max": 19.5267276763916,
      "activations/layer12_attention_weight_min": -27.02823829650879,
      "activations/layer13_attention_weight_max": 36.050140380859375,
      "activations/layer13_attention_weight_min": -31.299856185913086,
      "activations/layer14_attention_weight_max": 35.6435661315918,
      "activations/layer14_attention_weight_min": -29.69456672668457,
      "activations/layer15_attention_weight_max": 31.461233139038086,
      "activations/layer15_attention_weight_min": -31.368566513061523,
      "activations/layer16_attention_weight_max": 33.27015686035156,
      "activations/layer16_attention_weight_min": -34.23942184448242,
      "activations/layer17_attention_weight_max": 52.84676742553711,
      "activations/layer17_attention_weight_min": -44.74965286254883,
      "activations/layer18_attention_weight_max": 52.15170669555664,
      "activations/layer18_attention_weight_min": -39.77696990966797,
      "activations/layer19_attention_weight_max": 26.113340377807617,
      "activations/layer19_attention_weight_min": -23.134822845458984,
      "activations/layer1_attention_weight_max": 14.953178405761719,
      "activations/layer1_attention_weight_min": -14.982843399047852,
      "activations/layer20_attention_weight_max": 23.646760940551758,
      "activations/layer20_attention_weight_min": -20.436492919921875,
      "activations/layer21_attention_weight_max": 32.56288528442383,
      "activations/layer21_attention_weight_min": -21.481489181518555,
      "activations/layer22_attention_weight_max": 27.21311378479004,
      "activations/layer22_attention_weight_min": -25.94891929626465,
      "activations/layer23_attention_weight_max": 33.09129333496094,
      "activations/layer23_attention_weight_min": -24.248287200927734,
      "activations/layer2_attention_weight_max": 32.20061111450195,
      "activations/layer2_attention_weight_min": -31.5865478515625,
      "activations/layer3_attention_weight_max": 90.81866455078125,
      "activations/layer3_attention_weight_min": -89.73511505126953,
      "activations/layer4_attention_weight_max": 86.25801849365234,
      "activations/layer4_attention_weight_min": -87.7068099975586,
      "activations/layer5_attention_weight_max": 60.93623352050781,
      "activations/layer5_attention_weight_min": -68.25582122802734,
      "activations/layer6_attention_weight_max": 50.12160110473633,
      "activations/layer6_attention_weight_min": -54.09465026855469,
      "activations/layer7_attention_weight_max": 69.71966552734375,
      "activations/layer7_attention_weight_min": -67.93222045898438,
      "activations/layer8_attention_weight_max": 47.571197509765625,
      "activations/layer8_attention_weight_min": -50.767738342285156,
      "activations/layer9_attention_weight_max": 46.53181076049805,
      "activations/layer9_attention_weight_min": -48.3459587097168,
      "epoch": 21.5,
      "learning_rate": 1.149431818181818e-05,
      "loss": 2.7108,
      "step": 369950
    },
    {
      "activations/layer0_attention_weight_max": 15.256648063659668,
      "activations/layer0_attention_weight_min": -14.522811889648438,
      "activations/layer10_attention_weight_max": 33.039615631103516,
      "activations/layer10_attention_weight_min": -34.623905181884766,
      "activations/layer11_attention_weight_max": 33.23885726928711,
      "activations/layer11_attention_weight_min": -34.09308624267578,
      "activations/layer12_attention_weight_max": 18.054590225219727,
      "activations/layer12_attention_weight_min": -24.767024993896484,
      "activations/layer13_attention_weight_max": 36.42438888549805,
      "activations/layer13_attention_weight_min": -26.67574691772461,
      "activations/layer14_attention_weight_max": 34.48469924926758,
      "activations/layer14_attention_weight_min": -28.825166702270508,
      "activations/layer15_attention_weight_max": 32.07411193847656,
      "activations/layer15_attention_weight_min": -28.98564338684082,
      "activations/layer16_attention_weight_max": 32.44773864746094,
      "activations/layer16_attention_weight_min": -30.645963668823242,
      "activations/layer17_attention_weight_max": 49.92389678955078,
      "activations/layer17_attention_weight_min": -43.09761047363281,
      "activations/layer18_attention_weight_max": 44.550376892089844,
      "activations/layer18_attention_weight_min": -37.87596893310547,
      "activations/layer19_attention_weight_max": 23.968456268310547,
      "activations/layer19_attention_weight_min": -21.40336799621582,
      "activations/layer1_attention_weight_max": 16.4849796295166,
      "activations/layer1_attention_weight_min": -13.970649719238281,
      "activations/layer20_attention_weight_max": 23.401945114135742,
      "activations/layer20_attention_weight_min": -22.456321716308594,
      "activations/layer21_attention_weight_max": 34.02095413208008,
      "activations/layer21_attention_weight_min": -20.997177124023438,
      "activations/layer22_attention_weight_max": 28.965362548828125,
      "activations/layer22_attention_weight_min": -27.458587646484375,
      "activations/layer23_attention_weight_max": 40.2379150390625,
      "activations/layer23_attention_weight_min": -23.846519470214844,
      "activations/layer2_attention_weight_max": 31.853229522705078,
      "activations/layer2_attention_weight_min": -30.305591583251953,
      "activations/layer3_attention_weight_max": 88.37372589111328,
      "activations/layer3_attention_weight_min": -87.22875213623047,
      "activations/layer4_attention_weight_max": 82.03520202636719,
      "activations/layer4_attention_weight_min": -81.70050811767578,
      "activations/layer5_attention_weight_max": 59.97210693359375,
      "activations/layer5_attention_weight_min": -67.77659606933594,
      "activations/layer6_attention_weight_max": 47.20052719116211,
      "activations/layer6_attention_weight_min": -47.26484298706055,
      "activations/layer7_attention_weight_max": 67.80976104736328,
      "activations/layer7_attention_weight_min": -71.85261535644531,
      "activations/layer8_attention_weight_max": 45.37562942504883,
      "activations/layer8_attention_weight_min": -49.21912384033203,
      "activations/layer9_attention_weight_max": 46.801971435546875,
      "activations/layer9_attention_weight_min": -51.2347526550293,
      "epoch": 21.5,
      "learning_rate": 1.1475378787878787e-05,
      "loss": 2.7241,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_loss": 2.671875,
      "eval_runtime": 8.495,
      "eval_samples_per_second": 505.472,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.495,
      "eval_openwebtext_samples_per_second": 505.472,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9552,
      "eval_wikitext_samples_per_second": 233.223,
      "step": 370000
    },
    {
      "epoch": 21.5,
      "eval_lambada_loss": 2.41015625,
      "eval_lambada_ppl": 11.135700962413912,
      "eval_lambada_runtime": 9.5862,
      "eval_lambada_samples_per_second": 507.916,
      "step": 370000
    },
    {
      "activations/layer0_attention_weight_max": 15.961407661437988,
      "activations/layer0_attention_weight_min": -14.38101577758789,
      "activations/layer10_attention_weight_max": 36.776145935058594,
      "activations/layer10_attention_weight_min": -35.2209358215332,
      "activations/layer11_attention_weight_max": 32.68043518066406,
      "activations/layer11_attention_weight_min": -33.835540771484375,
      "activations/layer12_attention_weight_max": 19.36358070373535,
      "activations/layer12_attention_weight_min": -23.393274307250977,
      "activations/layer13_attention_weight_max": 29.736900329589844,
      "activations/layer13_attention_weight_min": -28.238296508789062,
      "activations/layer14_attention_weight_max": 32.61764144897461,
      "activations/layer14_attention_weight_min": -30.845748901367188,
      "activations/layer15_attention_weight_max": 30.981657028198242,
      "activations/layer15_attention_weight_min": -29.86809730529785,
      "activations/layer16_attention_weight_max": 33.765472412109375,
      "activations/layer16_attention_weight_min": -32.07484817504883,
      "activations/layer17_attention_weight_max": 48.45549392700195,
      "activations/layer17_attention_weight_min": -43.707305908203125,
      "activations/layer18_attention_weight_max": 44.92578887939453,
      "activations/layer18_attention_weight_min": -40.02687454223633,
      "activations/layer19_attention_weight_max": 25.984411239624023,
      "activations/layer19_attention_weight_min": -24.38776206970215,
      "activations/layer1_attention_weight_max": 14.564817428588867,
      "activations/layer1_attention_weight_min": -15.469619750976562,
      "activations/layer20_attention_weight_max": 22.983705520629883,
      "activations/layer20_attention_weight_min": -22.063573837280273,
      "activations/layer21_attention_weight_max": 31.284587860107422,
      "activations/layer21_attention_weight_min": -22.838525772094727,
      "activations/layer22_attention_weight_max": 29.483688354492188,
      "activations/layer22_attention_weight_min": -25.050703048706055,
      "activations/layer23_attention_weight_max": 33.94439697265625,
      "activations/layer23_attention_weight_min": -23.768470764160156,
      "activations/layer2_attention_weight_max": 31.64657974243164,
      "activations/layer2_attention_weight_min": -30.668811798095703,
      "activations/layer3_attention_weight_max": 89.1716537475586,
      "activations/layer3_attention_weight_min": -86.69082641601562,
      "activations/layer4_attention_weight_max": 80.44227600097656,
      "activations/layer4_attention_weight_min": -82.67272186279297,
      "activations/layer5_attention_weight_max": 59.997222900390625,
      "activations/layer5_attention_weight_min": -70.55780792236328,
      "activations/layer6_attention_weight_max": 48.03303909301758,
      "activations/layer6_attention_weight_min": -50.09545135498047,
      "activations/layer7_attention_weight_max": 68.8838119506836,
      "activations/layer7_attention_weight_min": -66.71532440185547,
      "activations/layer8_attention_weight_max": 46.556617736816406,
      "activations/layer8_attention_weight_min": -48.422550201416016,
      "activations/layer9_attention_weight_max": 42.699798583984375,
      "activations/layer9_attention_weight_min": -45.9198112487793,
      "epoch": 21.5,
      "learning_rate": 1.1456439393939394e-05,
      "loss": 2.709,
      "step": 370050
    },
    {
      "activations/layer0_attention_weight_max": 15.953411102294922,
      "activations/layer0_attention_weight_min": -14.151054382324219,
      "activations/layer10_attention_weight_max": 35.90533447265625,
      "activations/layer10_attention_weight_min": -37.56403350830078,
      "activations/layer11_attention_weight_max": 34.06597137451172,
      "activations/layer11_attention_weight_min": -35.17598342895508,
      "activations/layer12_attention_weight_max": 19.622976303100586,
      "activations/layer12_attention_weight_min": -25.842832565307617,
      "activations/layer13_attention_weight_max": 35.27537155151367,
      "activations/layer13_attention_weight_min": -30.94317626953125,
      "activations/layer14_attention_weight_max": 35.963600158691406,
      "activations/layer14_attention_weight_min": -31.25981330871582,
      "activations/layer15_attention_weight_max": 35.88020324707031,
      "activations/layer15_attention_weight_min": -30.87867546081543,
      "activations/layer16_attention_weight_max": 40.11320495605469,
      "activations/layer16_attention_weight_min": -37.52201461791992,
      "activations/layer17_attention_weight_max": 59.43380355834961,
      "activations/layer17_attention_weight_min": -49.109737396240234,
      "activations/layer18_attention_weight_max": 56.377986907958984,
      "activations/layer18_attention_weight_min": -45.182220458984375,
      "activations/layer19_attention_weight_max": 26.5123233795166,
      "activations/layer19_attention_weight_min": -25.877490997314453,
      "activations/layer1_attention_weight_max": 14.657532691955566,
      "activations/layer1_attention_weight_min": -13.021065711975098,
      "activations/layer20_attention_weight_max": 25.04973602294922,
      "activations/layer20_attention_weight_min": -22.366466522216797,
      "activations/layer21_attention_weight_max": 42.3516731262207,
      "activations/layer21_attention_weight_min": -22.267847061157227,
      "activations/layer22_attention_weight_max": 34.569541931152344,
      "activations/layer22_attention_weight_min": -27.118183135986328,
      "activations/layer23_attention_weight_max": 49.50919723510742,
      "activations/layer23_attention_weight_min": -28.40560531616211,
      "activations/layer2_attention_weight_max": 30.62667465209961,
      "activations/layer2_attention_weight_min": -30.752975463867188,
      "activations/layer3_attention_weight_max": 88.84410858154297,
      "activations/layer3_attention_weight_min": -91.29784393310547,
      "activations/layer4_attention_weight_max": 86.05876922607422,
      "activations/layer4_attention_weight_min": -87.28186798095703,
      "activations/layer5_attention_weight_max": 61.08701705932617,
      "activations/layer5_attention_weight_min": -70.57523345947266,
      "activations/layer6_attention_weight_max": 50.972129821777344,
      "activations/layer6_attention_weight_min": -51.10270690917969,
      "activations/layer7_attention_weight_max": 72.11392211914062,
      "activations/layer7_attention_weight_min": -66.77078247070312,
      "activations/layer8_attention_weight_max": 46.93782043457031,
      "activations/layer8_attention_weight_min": -52.382965087890625,
      "activations/layer9_attention_weight_max": 46.06867599487305,
      "activations/layer9_attention_weight_min": -46.82560348510742,
      "epoch": 21.5,
      "learning_rate": 1.1437499999999998e-05,
      "loss": 2.7137,
      "step": 370100
    },
    {
      "activations/layer0_attention_weight_max": 16.41472053527832,
      "activations/layer0_attention_weight_min": -14.026779174804688,
      "activations/layer10_attention_weight_max": 34.232025146484375,
      "activations/layer10_attention_weight_min": -36.02207565307617,
      "activations/layer11_attention_weight_max": 34.10036849975586,
      "activations/layer11_attention_weight_min": -35.85142135620117,
      "activations/layer12_attention_weight_max": 18.49966049194336,
      "activations/layer12_attention_weight_min": -28.555131912231445,
      "activations/layer13_attention_weight_max": 32.047142028808594,
      "activations/layer13_attention_weight_min": -26.88590431213379,
      "activations/layer14_attention_weight_max": 34.05436325073242,
      "activations/layer14_attention_weight_min": -30.35174560546875,
      "activations/layer15_attention_weight_max": 32.789451599121094,
      "activations/layer15_attention_weight_min": -31.101760864257812,
      "activations/layer16_attention_weight_max": 32.477474212646484,
      "activations/layer16_attention_weight_min": -32.159889221191406,
      "activations/layer17_attention_weight_max": 48.036617279052734,
      "activations/layer17_attention_weight_min": -45.37479019165039,
      "activations/layer18_attention_weight_max": 45.351295471191406,
      "activations/layer18_attention_weight_min": -40.303077697753906,
      "activations/layer19_attention_weight_max": 25.960102081298828,
      "activations/layer19_attention_weight_min": -21.96164894104004,
      "activations/layer1_attention_weight_max": 14.857532501220703,
      "activations/layer1_attention_weight_min": -14.404723167419434,
      "activations/layer20_attention_weight_max": 23.127471923828125,
      "activations/layer20_attention_weight_min": -25.28286361694336,
      "activations/layer21_attention_weight_max": 29.892757415771484,
      "activations/layer21_attention_weight_min": -20.001815795898438,
      "activations/layer22_attention_weight_max": 28.4813232421875,
      "activations/layer22_attention_weight_min": -29.3043212890625,
      "activations/layer23_attention_weight_max": 34.752037048339844,
      "activations/layer23_attention_weight_min": -24.741422653198242,
      "activations/layer2_attention_weight_max": 32.68412780761719,
      "activations/layer2_attention_weight_min": -29.79689598083496,
      "activations/layer3_attention_weight_max": 86.75466918945312,
      "activations/layer3_attention_weight_min": -86.61500549316406,
      "activations/layer4_attention_weight_max": 81.21094512939453,
      "activations/layer4_attention_weight_min": -81.72579193115234,
      "activations/layer5_attention_weight_max": 60.71278381347656,
      "activations/layer5_attention_weight_min": -72.09835815429688,
      "activations/layer6_attention_weight_max": 50.626625061035156,
      "activations/layer6_attention_weight_min": -49.569435119628906,
      "activations/layer7_attention_weight_max": 68.97550964355469,
      "activations/layer7_attention_weight_min": -68.04866027832031,
      "activations/layer8_attention_weight_max": 45.88910675048828,
      "activations/layer8_attention_weight_min": -49.68385314941406,
      "activations/layer9_attention_weight_max": 43.37401580810547,
      "activations/layer9_attention_weight_min": -47.14925003051758,
      "epoch": 21.51,
      "learning_rate": 1.1418560606060605e-05,
      "loss": 2.71,
      "step": 370150
    },
    {
      "activations/layer0_attention_weight_max": 15.380358695983887,
      "activations/layer0_attention_weight_min": -14.053359031677246,
      "activations/layer10_attention_weight_max": 37.13686752319336,
      "activations/layer10_attention_weight_min": -36.065887451171875,
      "activations/layer11_attention_weight_max": 36.09758377075195,
      "activations/layer11_attention_weight_min": -37.328575134277344,
      "activations/layer12_attention_weight_max": 20.184457778930664,
      "activations/layer12_attention_weight_min": -23.17283058166504,
      "activations/layer13_attention_weight_max": 39.613197326660156,
      "activations/layer13_attention_weight_min": -27.37357521057129,
      "activations/layer14_attention_weight_max": 39.53095626831055,
      "activations/layer14_attention_weight_min": -31.086116790771484,
      "activations/layer15_attention_weight_max": 37.02116012573242,
      "activations/layer15_attention_weight_min": -31.008590698242188,
      "activations/layer16_attention_weight_max": 38.932193756103516,
      "activations/layer16_attention_weight_min": -36.83247375488281,
      "activations/layer17_attention_weight_max": 58.2352180480957,
      "activations/layer17_attention_weight_min": -52.931522369384766,
      "activations/layer18_attention_weight_max": 56.16652297973633,
      "activations/layer18_attention_weight_min": -42.65135955810547,
      "activations/layer19_attention_weight_max": 26.703500747680664,
      "activations/layer19_attention_weight_min": -24.497726440429688,
      "activations/layer1_attention_weight_max": 15.227749824523926,
      "activations/layer1_attention_weight_min": -13.79061508178711,
      "activations/layer20_attention_weight_max": 24.69335174560547,
      "activations/layer20_attention_weight_min": -20.796443939208984,
      "activations/layer21_attention_weight_max": 35.4415168762207,
      "activations/layer21_attention_weight_min": -22.366601943969727,
      "activations/layer22_attention_weight_max": 33.186317443847656,
      "activations/layer22_attention_weight_min": -25.633224487304688,
      "activations/layer23_attention_weight_max": 41.416038513183594,
      "activations/layer23_attention_weight_min": -22.835981369018555,
      "activations/layer2_attention_weight_max": 32.53158950805664,
      "activations/layer2_attention_weight_min": -31.08582305908203,
      "activations/layer3_attention_weight_max": 91.76504516601562,
      "activations/layer3_attention_weight_min": -88.38402557373047,
      "activations/layer4_attention_weight_max": 88.51439666748047,
      "activations/layer4_attention_weight_min": -86.21825408935547,
      "activations/layer5_attention_weight_max": 63.22700500488281,
      "activations/layer5_attention_weight_min": -73.938720703125,
      "activations/layer6_attention_weight_max": 50.611759185791016,
      "activations/layer6_attention_weight_min": -52.44493103027344,
      "activations/layer7_attention_weight_max": 71.9701919555664,
      "activations/layer7_attention_weight_min": -68.69908905029297,
      "activations/layer8_attention_weight_max": 48.95968246459961,
      "activations/layer8_attention_weight_min": -53.78725814819336,
      "activations/layer9_attention_weight_max": 46.09056854248047,
      "activations/layer9_attention_weight_min": -47.927894592285156,
      "epoch": 21.51,
      "learning_rate": 1.139962121212121e-05,
      "loss": 2.7236,
      "step": 370200
    },
    {
      "activations/layer0_attention_weight_max": 15.927567481994629,
      "activations/layer0_attention_weight_min": -14.118303298950195,
      "activations/layer10_attention_weight_max": 36.54988479614258,
      "activations/layer10_attention_weight_min": -34.54944610595703,
      "activations/layer11_attention_weight_max": 32.42451095581055,
      "activations/layer11_attention_weight_min": -32.37310791015625,
      "activations/layer12_attention_weight_max": 18.442718505859375,
      "activations/layer12_attention_weight_min": -30.824796676635742,
      "activations/layer13_attention_weight_max": 31.449190139770508,
      "activations/layer13_attention_weight_min": -27.28810691833496,
      "activations/layer14_attention_weight_max": 33.36931610107422,
      "activations/layer14_attention_weight_min": -28.748092651367188,
      "activations/layer15_attention_weight_max": 30.453916549682617,
      "activations/layer15_attention_weight_min": -30.359102249145508,
      "activations/layer16_attention_weight_max": 32.18368911743164,
      "activations/layer16_attention_weight_min": -31.13462257385254,
      "activations/layer17_attention_weight_max": 49.96986389160156,
      "activations/layer17_attention_weight_min": -44.159664154052734,
      "activations/layer18_attention_weight_max": 45.51848220825195,
      "activations/layer18_attention_weight_min": -38.6684684753418,
      "activations/layer19_attention_weight_max": 25.02198028564453,
      "activations/layer19_attention_weight_min": -21.216537475585938,
      "activations/layer1_attention_weight_max": 15.710333824157715,
      "activations/layer1_attention_weight_min": -14.081452369689941,
      "activations/layer20_attention_weight_max": 21.258304595947266,
      "activations/layer20_attention_weight_min": -19.90781021118164,
      "activations/layer21_attention_weight_max": 30.123716354370117,
      "activations/layer21_attention_weight_min": -20.7711181640625,
      "activations/layer22_attention_weight_max": 26.315311431884766,
      "activations/layer22_attention_weight_min": -25.838151931762695,
      "activations/layer23_attention_weight_max": 32.07780456542969,
      "activations/layer23_attention_weight_min": -24.581220626831055,
      "activations/layer2_attention_weight_max": 31.847766876220703,
      "activations/layer2_attention_weight_min": -31.823833465576172,
      "activations/layer3_attention_weight_max": 92.12945556640625,
      "activations/layer3_attention_weight_min": -90.29444885253906,
      "activations/layer4_attention_weight_max": 85.91085052490234,
      "activations/layer4_attention_weight_min": -89.5974349975586,
      "activations/layer5_attention_weight_max": 61.96735763549805,
      "activations/layer5_attention_weight_min": -69.86866760253906,
      "activations/layer6_attention_weight_max": 48.48700714111328,
      "activations/layer6_attention_weight_min": -50.42733383178711,
      "activations/layer7_attention_weight_max": 65.71258544921875,
      "activations/layer7_attention_weight_min": -66.79842376708984,
      "activations/layer8_attention_weight_max": 51.5543098449707,
      "activations/layer8_attention_weight_min": -50.79193878173828,
      "activations/layer9_attention_weight_max": 47.420166015625,
      "activations/layer9_attention_weight_min": -43.33115768432617,
      "epoch": 21.51,
      "learning_rate": 1.1380681818181817e-05,
      "loss": 2.7178,
      "step": 370250
    },
    {
      "activations/layer0_attention_weight_max": 15.568201065063477,
      "activations/layer0_attention_weight_min": -14.157367706298828,
      "activations/layer10_attention_weight_max": 37.48070526123047,
      "activations/layer10_attention_weight_min": -40.2873649597168,
      "activations/layer11_attention_weight_max": 37.571022033691406,
      "activations/layer11_attention_weight_min": -37.01495361328125,
      "activations/layer12_attention_weight_max": 18.543684005737305,
      "activations/layer12_attention_weight_min": -24.632055282592773,
      "activations/layer13_attention_weight_max": 33.836734771728516,
      "activations/layer13_attention_weight_min": -26.24279022216797,
      "activations/layer14_attention_weight_max": 36.43792724609375,
      "activations/layer14_attention_weight_min": -30.399921417236328,
      "activations/layer15_attention_weight_max": 31.756771087646484,
      "activations/layer15_attention_weight_min": -30.326128005981445,
      "activations/layer16_attention_weight_max": 33.6090202331543,
      "activations/layer16_attention_weight_min": -32.0416259765625,
      "activations/layer17_attention_weight_max": 53.83694076538086,
      "activations/layer17_attention_weight_min": -45.49751663208008,
      "activations/layer18_attention_weight_max": 47.65465545654297,
      "activations/layer18_attention_weight_min": -41.02363967895508,
      "activations/layer19_attention_weight_max": 25.863779067993164,
      "activations/layer19_attention_weight_min": -23.501737594604492,
      "activations/layer1_attention_weight_max": 15.527900695800781,
      "activations/layer1_attention_weight_min": -13.858450889587402,
      "activations/layer20_attention_weight_max": 22.28533363342285,
      "activations/layer20_attention_weight_min": -22.1375675201416,
      "activations/layer21_attention_weight_max": 36.137516021728516,
      "activations/layer21_attention_weight_min": -24.10647964477539,
      "activations/layer22_attention_weight_max": 31.21583366394043,
      "activations/layer22_attention_weight_min": -24.6812801361084,
      "activations/layer23_attention_weight_max": 33.591468811035156,
      "activations/layer23_attention_weight_min": -24.241748809814453,
      "activations/layer2_attention_weight_max": 31.552770614624023,
      "activations/layer2_attention_weight_min": -30.899131774902344,
      "activations/layer3_attention_weight_max": 92.94690704345703,
      "activations/layer3_attention_weight_min": -88.24042510986328,
      "activations/layer4_attention_weight_max": 90.94759368896484,
      "activations/layer4_attention_weight_min": -86.83056640625,
      "activations/layer5_attention_weight_max": 63.78202819824219,
      "activations/layer5_attention_weight_min": -72.0670394897461,
      "activations/layer6_attention_weight_max": 51.91750717163086,
      "activations/layer6_attention_weight_min": -53.20403289794922,
      "activations/layer7_attention_weight_max": 69.72550964355469,
      "activations/layer7_attention_weight_min": -72.51567077636719,
      "activations/layer8_attention_weight_max": 51.940792083740234,
      "activations/layer8_attention_weight_min": -51.554443359375,
      "activations/layer9_attention_weight_max": 47.024078369140625,
      "activations/layer9_attention_weight_min": -48.79960632324219,
      "epoch": 21.52,
      "learning_rate": 1.1361742424242423e-05,
      "loss": 2.7319,
      "step": 370300
    },
    {
      "activations/layer0_attention_weight_max": 16.365169525146484,
      "activations/layer0_attention_weight_min": -14.146045684814453,
      "activations/layer10_attention_weight_max": 45.90850067138672,
      "activations/layer10_attention_weight_min": -44.78837966918945,
      "activations/layer11_attention_weight_max": 43.85255432128906,
      "activations/layer11_attention_weight_min": -40.901161193847656,
      "activations/layer12_attention_weight_max": 21.000946044921875,
      "activations/layer12_attention_weight_min": -35.17668914794922,
      "activations/layer13_attention_weight_max": 45.08797073364258,
      "activations/layer13_attention_weight_min": -34.78988265991211,
      "activations/layer14_attention_weight_max": 48.06061935424805,
      "activations/layer14_attention_weight_min": -40.46010208129883,
      "activations/layer15_attention_weight_max": 45.45503234863281,
      "activations/layer15_attention_weight_min": -36.30607604980469,
      "activations/layer16_attention_weight_max": 41.775299072265625,
      "activations/layer16_attention_weight_min": -40.30793380737305,
      "activations/layer17_attention_weight_max": 61.97517776489258,
      "activations/layer17_attention_weight_min": -61.35694885253906,
      "activations/layer18_attention_weight_max": 59.85169219970703,
      "activations/layer18_attention_weight_min": -59.38498306274414,
      "activations/layer19_attention_weight_max": 32.368953704833984,
      "activations/layer19_attention_weight_min": -31.114917755126953,
      "activations/layer1_attention_weight_max": 15.3667631149292,
      "activations/layer1_attention_weight_min": -14.153200149536133,
      "activations/layer20_attention_weight_max": 32.21925354003906,
      "activations/layer20_attention_weight_min": -25.56584930419922,
      "activations/layer21_attention_weight_max": 60.374755859375,
      "activations/layer21_attention_weight_min": -34.424400329589844,
      "activations/layer22_attention_weight_max": 38.01594543457031,
      "activations/layer22_attention_weight_min": -29.21499252319336,
      "activations/layer23_attention_weight_max": 38.02247619628906,
      "activations/layer23_attention_weight_min": -23.426313400268555,
      "activations/layer2_attention_weight_max": 33.16194152832031,
      "activations/layer2_attention_weight_min": -31.921939849853516,
      "activations/layer3_attention_weight_max": 95.78789520263672,
      "activations/layer3_attention_weight_min": -92.76215362548828,
      "activations/layer4_attention_weight_max": 89.95079803466797,
      "activations/layer4_attention_weight_min": -86.63134765625,
      "activations/layer5_attention_weight_max": 64.9365234375,
      "activations/layer5_attention_weight_min": -75.7369384765625,
      "activations/layer6_attention_weight_max": 57.54667282104492,
      "activations/layer6_attention_weight_min": -55.005645751953125,
      "activations/layer7_attention_weight_max": 79.3306884765625,
      "activations/layer7_attention_weight_min": -76.90325164794922,
      "activations/layer8_attention_weight_max": 65.65005493164062,
      "activations/layer8_attention_weight_min": -60.20510482788086,
      "activations/layer9_attention_weight_max": 63.10628890991211,
      "activations/layer9_attention_weight_min": -54.51266098022461,
      "epoch": 21.52,
      "learning_rate": 1.1342803030303028e-05,
      "loss": 2.7161,
      "step": 370350
    },
    {
      "activations/layer0_attention_weight_max": 15.716960906982422,
      "activations/layer0_attention_weight_min": -13.876981735229492,
      "activations/layer10_attention_weight_max": 40.935943603515625,
      "activations/layer10_attention_weight_min": -39.88330841064453,
      "activations/layer11_attention_weight_max": 38.66320037841797,
      "activations/layer11_attention_weight_min": -37.570438385009766,
      "activations/layer12_attention_weight_max": 18.18788719177246,
      "activations/layer12_attention_weight_min": -25.928205490112305,
      "activations/layer13_attention_weight_max": 36.262428283691406,
      "activations/layer13_attention_weight_min": -30.400768280029297,
      "activations/layer14_attention_weight_max": 36.06732940673828,
      "activations/layer14_attention_weight_min": -34.4427375793457,
      "activations/layer15_attention_weight_max": 34.14680862426758,
      "activations/layer15_attention_weight_min": -34.80036926269531,
      "activations/layer16_attention_weight_max": 37.245079040527344,
      "activations/layer16_attention_weight_min": -35.84297180175781,
      "activations/layer17_attention_weight_max": 53.39570236206055,
      "activations/layer17_attention_weight_min": -53.14104080200195,
      "activations/layer18_attention_weight_max": 53.14017868041992,
      "activations/layer18_attention_weight_min": -49.50553894042969,
      "activations/layer19_attention_weight_max": 27.924734115600586,
      "activations/layer19_attention_weight_min": -26.04095458984375,
      "activations/layer1_attention_weight_max": 15.563655853271484,
      "activations/layer1_attention_weight_min": -15.10529899597168,
      "activations/layer20_attention_weight_max": 27.950855255126953,
      "activations/layer20_attention_weight_min": -24.431371688842773,
      "activations/layer21_attention_weight_max": 40.07509231567383,
      "activations/layer21_attention_weight_min": -20.452285766601562,
      "activations/layer22_attention_weight_max": 31.55056381225586,
      "activations/layer22_attention_weight_min": -26.03093719482422,
      "activations/layer23_attention_weight_max": 42.69047164916992,
      "activations/layer23_attention_weight_min": -24.07770347595215,
      "activations/layer2_attention_weight_max": 32.39623260498047,
      "activations/layer2_attention_weight_min": -31.952346801757812,
      "activations/layer3_attention_weight_max": 90.61343383789062,
      "activations/layer3_attention_weight_min": -87.70220184326172,
      "activations/layer4_attention_weight_max": 90.03489685058594,
      "activations/layer4_attention_weight_min": -87.0007553100586,
      "activations/layer5_attention_weight_max": 66.29872131347656,
      "activations/layer5_attention_weight_min": -75.93367767333984,
      "activations/layer6_attention_weight_max": 51.27539825439453,
      "activations/layer6_attention_weight_min": -53.065895080566406,
      "activations/layer7_attention_weight_max": 76.99655151367188,
      "activations/layer7_attention_weight_min": -69.04597473144531,
      "activations/layer8_attention_weight_max": 51.96133041381836,
      "activations/layer8_attention_weight_min": -54.759521484375,
      "activations/layer9_attention_weight_max": 52.440303802490234,
      "activations/layer9_attention_weight_min": -50.65775680541992,
      "epoch": 21.52,
      "learning_rate": 1.1323863636363635e-05,
      "loss": 2.7171,
      "step": 370400
    },
    {
      "activations/layer0_attention_weight_max": 16.16839599609375,
      "activations/layer0_attention_weight_min": -14.087409973144531,
      "activations/layer10_attention_weight_max": 36.13112258911133,
      "activations/layer10_attention_weight_min": -36.54536819458008,
      "activations/layer11_attention_weight_max": 31.866025924682617,
      "activations/layer11_attention_weight_min": -35.554168701171875,
      "activations/layer12_attention_weight_max": 18.241098403930664,
      "activations/layer12_attention_weight_min": -29.213829040527344,
      "activations/layer13_attention_weight_max": 36.07684326171875,
      "activations/layer13_attention_weight_min": -26.665693283081055,
      "activations/layer14_attention_weight_max": 35.49202346801758,
      "activations/layer14_attention_weight_min": -29.39406394958496,
      "activations/layer15_attention_weight_max": 31.15730094909668,
      "activations/layer15_attention_weight_min": -28.87772560119629,
      "activations/layer16_attention_weight_max": 34.599090576171875,
      "activations/layer16_attention_weight_min": -32.420204162597656,
      "activations/layer17_attention_weight_max": 49.339683532714844,
      "activations/layer17_attention_weight_min": -42.64705276489258,
      "activations/layer18_attention_weight_max": 49.07943344116211,
      "activations/layer18_attention_weight_min": -39.33498001098633,
      "activations/layer19_attention_weight_max": 23.947614669799805,
      "activations/layer19_attention_weight_min": -23.045696258544922,
      "activations/layer1_attention_weight_max": 14.444151878356934,
      "activations/layer1_attention_weight_min": -13.818378448486328,
      "activations/layer20_attention_weight_max": 21.067333221435547,
      "activations/layer20_attention_weight_min": -23.93648338317871,
      "activations/layer21_attention_weight_max": 35.27421951293945,
      "activations/layer21_attention_weight_min": -21.099157333374023,
      "activations/layer22_attention_weight_max": 27.171310424804688,
      "activations/layer22_attention_weight_min": -27.180267333984375,
      "activations/layer23_attention_weight_max": 35.00567626953125,
      "activations/layer23_attention_weight_min": -26.548397064208984,
      "activations/layer2_attention_weight_max": 30.583881378173828,
      "activations/layer2_attention_weight_min": -29.025571823120117,
      "activations/layer3_attention_weight_max": 88.17152404785156,
      "activations/layer3_attention_weight_min": -87.32908630371094,
      "activations/layer4_attention_weight_max": 82.70027923583984,
      "activations/layer4_attention_weight_min": -83.69757080078125,
      "activations/layer5_attention_weight_max": 61.92794418334961,
      "activations/layer5_attention_weight_min": -69.33045959472656,
      "activations/layer6_attention_weight_max": 50.23046112060547,
      "activations/layer6_attention_weight_min": -53.55287551879883,
      "activations/layer7_attention_weight_max": 72.53462219238281,
      "activations/layer7_attention_weight_min": -65.20278930664062,
      "activations/layer8_attention_weight_max": 48.352294921875,
      "activations/layer8_attention_weight_min": -54.86281204223633,
      "activations/layer9_attention_weight_max": 45.013492584228516,
      "activations/layer9_attention_weight_min": -49.017181396484375,
      "epoch": 21.53,
      "learning_rate": 1.1305303030303029e-05,
      "loss": 2.7099,
      "step": 370450
    },
    {
      "activations/layer0_attention_weight_max": 16.905216217041016,
      "activations/layer0_attention_weight_min": -14.088743209838867,
      "activations/layer10_attention_weight_max": 36.90262222290039,
      "activations/layer10_attention_weight_min": -35.00445556640625,
      "activations/layer11_attention_weight_max": 33.268714904785156,
      "activations/layer11_attention_weight_min": -35.61223220825195,
      "activations/layer12_attention_weight_max": 20.912338256835938,
      "activations/layer12_attention_weight_min": -26.645164489746094,
      "activations/layer13_attention_weight_max": 35.33453369140625,
      "activations/layer13_attention_weight_min": -27.18732452392578,
      "activations/layer14_attention_weight_max": 38.21061325073242,
      "activations/layer14_attention_weight_min": -33.05942916870117,
      "activations/layer15_attention_weight_max": 37.0713996887207,
      "activations/layer15_attention_weight_min": -33.49711227416992,
      "activations/layer16_attention_weight_max": 37.01044464111328,
      "activations/layer16_attention_weight_min": -34.770713806152344,
      "activations/layer17_attention_weight_max": 57.31282043457031,
      "activations/layer17_attention_weight_min": -50.07817459106445,
      "activations/layer18_attention_weight_max": 53.73162078857422,
      "activations/layer18_attention_weight_min": -44.58069610595703,
      "activations/layer19_attention_weight_max": 29.22040367126465,
      "activations/layer19_attention_weight_min": -27.069076538085938,
      "activations/layer1_attention_weight_max": 14.717434883117676,
      "activations/layer1_attention_weight_min": -14.066543579101562,
      "activations/layer20_attention_weight_max": 27.01763343811035,
      "activations/layer20_attention_weight_min": -21.50395393371582,
      "activations/layer21_attention_weight_max": 41.17072677612305,
      "activations/layer21_attention_weight_min": -22.48992347717285,
      "activations/layer22_attention_weight_max": 32.475826263427734,
      "activations/layer22_attention_weight_min": -26.411449432373047,
      "activations/layer23_attention_weight_max": 39.10380172729492,
      "activations/layer23_attention_weight_min": -24.1396484375,
      "activations/layer2_attention_weight_max": 31.746816635131836,
      "activations/layer2_attention_weight_min": -31.05501365661621,
      "activations/layer3_attention_weight_max": 89.95026397705078,
      "activations/layer3_attention_weight_min": -89.52273559570312,
      "activations/layer4_attention_weight_max": 82.76595306396484,
      "activations/layer4_attention_weight_min": -83.45178985595703,
      "activations/layer5_attention_weight_max": 62.032135009765625,
      "activations/layer5_attention_weight_min": -79.827392578125,
      "activations/layer6_attention_weight_max": 49.8764762878418,
      "activations/layer6_attention_weight_min": -49.2204475402832,
      "activations/layer7_attention_weight_max": 67.5336685180664,
      "activations/layer7_attention_weight_min": -70.402099609375,
      "activations/layer8_attention_weight_max": 46.170440673828125,
      "activations/layer8_attention_weight_min": -48.97141647338867,
      "activations/layer9_attention_weight_max": 44.28139114379883,
      "activations/layer9_attention_weight_min": -47.225643157958984,
      "epoch": 21.53,
      "learning_rate": 1.1286363636363634e-05,
      "loss": 2.7297,
      "step": 370500
    },
    {
      "activations/layer0_attention_weight_max": 18.48419952392578,
      "activations/layer0_attention_weight_min": -14.09311580657959,
      "activations/layer10_attention_weight_max": 37.86668395996094,
      "activations/layer10_attention_weight_min": -36.0234260559082,
      "activations/layer11_attention_weight_max": 34.67163848876953,
      "activations/layer11_attention_weight_min": -34.51060485839844,
      "activations/layer12_attention_weight_max": 18.93553924560547,
      "activations/layer12_attention_weight_min": -25.34271240234375,
      "activations/layer13_attention_weight_max": 35.299903869628906,
      "activations/layer13_attention_weight_min": -28.235801696777344,
      "activations/layer14_attention_weight_max": 35.99509048461914,
      "activations/layer14_attention_weight_min": -30.183671951293945,
      "activations/layer15_attention_weight_max": 34.317691802978516,
      "activations/layer15_attention_weight_min": -30.43992042541504,
      "activations/layer16_attention_weight_max": 36.47357940673828,
      "activations/layer16_attention_weight_min": -32.7591438293457,
      "activations/layer17_attention_weight_max": 53.516448974609375,
      "activations/layer17_attention_weight_min": -48.0135612487793,
      "activations/layer18_attention_weight_max": 46.34160614013672,
      "activations/layer18_attention_weight_min": -44.25236511230469,
      "activations/layer19_attention_weight_max": 30.88764762878418,
      "activations/layer19_attention_weight_min": -26.89528465270996,
      "activations/layer1_attention_weight_max": 15.653160095214844,
      "activations/layer1_attention_weight_min": -15.945087432861328,
      "activations/layer20_attention_weight_max": 28.11358642578125,
      "activations/layer20_attention_weight_min": -21.99334144592285,
      "activations/layer21_attention_weight_max": 39.3568115234375,
      "activations/layer21_attention_weight_min": -23.701244354248047,
      "activations/layer22_attention_weight_max": 30.710309982299805,
      "activations/layer22_attention_weight_min": -24.595436096191406,
      "activations/layer23_attention_weight_max": 34.56462097167969,
      "activations/layer23_attention_weight_min": -22.67177391052246,
      "activations/layer2_attention_weight_max": 31.57659149169922,
      "activations/layer2_attention_weight_min": -30.490631103515625,
      "activations/layer3_attention_weight_max": 87.95510864257812,
      "activations/layer3_attention_weight_min": -88.2836685180664,
      "activations/layer4_attention_weight_max": 83.59667205810547,
      "activations/layer4_attention_weight_min": -84.55865478515625,
      "activations/layer5_attention_weight_max": 60.08750915527344,
      "activations/layer5_attention_weight_min": -78.45858764648438,
      "activations/layer6_attention_weight_max": 48.30308532714844,
      "activations/layer6_attention_weight_min": -50.31550598144531,
      "activations/layer7_attention_weight_max": 67.31170654296875,
      "activations/layer7_attention_weight_min": -67.05323791503906,
      "activations/layer8_attention_weight_max": 49.21017074584961,
      "activations/layer8_attention_weight_min": -53.23640823364258,
      "activations/layer9_attention_weight_max": 42.930030822753906,
      "activations/layer9_attention_weight_min": -47.47718048095703,
      "epoch": 21.53,
      "learning_rate": 1.1267424242424242e-05,
      "loss": 2.7089,
      "step": 370550
    },
    {
      "activations/layer0_attention_weight_max": 16.873857498168945,
      "activations/layer0_attention_weight_min": -14.098450660705566,
      "activations/layer10_attention_weight_max": 41.070953369140625,
      "activations/layer10_attention_weight_min": -39.375343322753906,
      "activations/layer11_attention_weight_max": 38.78416442871094,
      "activations/layer11_attention_weight_min": -38.91695022583008,
      "activations/layer12_attention_weight_max": 20.045801162719727,
      "activations/layer12_attention_weight_min": -25.36861801147461,
      "activations/layer13_attention_weight_max": 39.59663009643555,
      "activations/layer13_attention_weight_min": -29.31707763671875,
      "activations/layer14_attention_weight_max": 41.79966735839844,
      "activations/layer14_attention_weight_min": -33.11804962158203,
      "activations/layer15_attention_weight_max": 39.848628997802734,
      "activations/layer15_attention_weight_min": -33.34339141845703,
      "activations/layer16_attention_weight_max": 39.64179611206055,
      "activations/layer16_attention_weight_min": -35.7180290222168,
      "activations/layer17_attention_weight_max": 58.75168991088867,
      "activations/layer17_attention_weight_min": -50.65275573730469,
      "activations/layer18_attention_weight_max": 56.519866943359375,
      "activations/layer18_attention_weight_min": -43.936824798583984,
      "activations/layer19_attention_weight_max": 27.942218780517578,
      "activations/layer19_attention_weight_min": -23.758249282836914,
      "activations/layer1_attention_weight_max": 15.304100036621094,
      "activations/layer1_attention_weight_min": -14.028641700744629,
      "activations/layer20_attention_weight_max": 28.26813316345215,
      "activations/layer20_attention_weight_min": -21.112993240356445,
      "activations/layer21_attention_weight_max": 38.47943115234375,
      "activations/layer21_attention_weight_min": -24.47093391418457,
      "activations/layer22_attention_weight_max": 32.992958068847656,
      "activations/layer22_attention_weight_min": -25.043420791625977,
      "activations/layer23_attention_weight_max": 40.778587341308594,
      "activations/layer23_attention_weight_min": -24.073204040527344,
      "activations/layer2_attention_weight_max": 31.272809982299805,
      "activations/layer2_attention_weight_min": -30.287918090820312,
      "activations/layer3_attention_weight_max": 88.86953735351562,
      "activations/layer3_attention_weight_min": -88.68721008300781,
      "activations/layer4_attention_weight_max": 84.7491683959961,
      "activations/layer4_attention_weight_min": -90.72420501708984,
      "activations/layer5_attention_weight_max": 64.92572021484375,
      "activations/layer5_attention_weight_min": -68.31109619140625,
      "activations/layer6_attention_weight_max": 50.87427520751953,
      "activations/layer6_attention_weight_min": -52.8409309387207,
      "activations/layer7_attention_weight_max": 73.34993743896484,
      "activations/layer7_attention_weight_min": -71.28532409667969,
      "activations/layer8_attention_weight_max": 52.56970977783203,
      "activations/layer8_attention_weight_min": -58.02518844604492,
      "activations/layer9_attention_weight_max": 50.3047981262207,
      "activations/layer9_attention_weight_min": -52.57257080078125,
      "epoch": 21.53,
      "learning_rate": 1.1248484848484848e-05,
      "loss": 2.7172,
      "step": 370600
    },
    {
      "activations/layer0_attention_weight_max": 15.417192459106445,
      "activations/layer0_attention_weight_min": -14.186366081237793,
      "activations/layer10_attention_weight_max": 37.4285888671875,
      "activations/layer10_attention_weight_min": -37.317626953125,
      "activations/layer11_attention_weight_max": 34.89886474609375,
      "activations/layer11_attention_weight_min": -35.391578674316406,
      "activations/layer12_attention_weight_max": 18.754898071289062,
      "activations/layer12_attention_weight_min": -28.021398544311523,
      "activations/layer13_attention_weight_max": 40.26613998413086,
      "activations/layer13_attention_weight_min": -30.32408905029297,
      "activations/layer14_attention_weight_max": 35.65001678466797,
      "activations/layer14_attention_weight_min": -31.194297790527344,
      "activations/layer15_attention_weight_max": 37.807708740234375,
      "activations/layer15_attention_weight_min": -32.21906661987305,
      "activations/layer16_attention_weight_max": 41.99704360961914,
      "activations/layer16_attention_weight_min": -40.62436294555664,
      "activations/layer17_attention_weight_max": 54.93941879272461,
      "activations/layer17_attention_weight_min": -51.49832534790039,
      "activations/layer18_attention_weight_max": 48.26565933227539,
      "activations/layer18_attention_weight_min": -46.20175552368164,
      "activations/layer19_attention_weight_max": 24.79227066040039,
      "activations/layer19_attention_weight_min": -24.14374351501465,
      "activations/layer1_attention_weight_max": 15.800994873046875,
      "activations/layer1_attention_weight_min": -15.591212272644043,
      "activations/layer20_attention_weight_max": 25.58771324157715,
      "activations/layer20_attention_weight_min": -20.99078369140625,
      "activations/layer21_attention_weight_max": 34.15930938720703,
      "activations/layer21_attention_weight_min": -19.70073890686035,
      "activations/layer22_attention_weight_max": 33.052024841308594,
      "activations/layer22_attention_weight_min": -25.8038272857666,
      "activations/layer23_attention_weight_max": 39.907920837402344,
      "activations/layer23_attention_weight_min": -23.299571990966797,
      "activations/layer2_attention_weight_max": 31.949180603027344,
      "activations/layer2_attention_weight_min": -30.35704803466797,
      "activations/layer3_attention_weight_max": 90.4913330078125,
      "activations/layer3_attention_weight_min": -89.76667785644531,
      "activations/layer4_attention_weight_max": 85.50008392333984,
      "activations/layer4_attention_weight_min": -83.41437530517578,
      "activations/layer5_attention_weight_max": 64.90519714355469,
      "activations/layer5_attention_weight_min": -72.0592041015625,
      "activations/layer6_attention_weight_max": 49.656639099121094,
      "activations/layer6_attention_weight_min": -48.22780227661133,
      "activations/layer7_attention_weight_max": 68.59029388427734,
      "activations/layer7_attention_weight_min": -66.52042388916016,
      "activations/layer8_attention_weight_max": 48.71227264404297,
      "activations/layer8_attention_weight_min": -48.028961181640625,
      "activations/layer9_attention_weight_max": 47.750877380371094,
      "activations/layer9_attention_weight_min": -50.23115539550781,
      "epoch": 21.54,
      "learning_rate": 1.1229545454545455e-05,
      "loss": 2.7186,
      "step": 370650
    },
    {
      "activations/layer0_attention_weight_max": 14.940183639526367,
      "activations/layer0_attention_weight_min": -14.278081893920898,
      "activations/layer10_attention_weight_max": 37.973907470703125,
      "activations/layer10_attention_weight_min": -38.1888313293457,
      "activations/layer11_attention_weight_max": 36.22307586669922,
      "activations/layer11_attention_weight_min": -35.55012893676758,
      "activations/layer12_attention_weight_max": 20.76858901977539,
      "activations/layer12_attention_weight_min": -25.502090454101562,
      "activations/layer13_attention_weight_max": 43.554405212402344,
      "activations/layer13_attention_weight_min": -28.834741592407227,
      "activations/layer14_attention_weight_max": 41.74494934082031,
      "activations/layer14_attention_weight_min": -32.77281951904297,
      "activations/layer15_attention_weight_max": 38.993404388427734,
      "activations/layer15_attention_weight_min": -32.48655700683594,
      "activations/layer16_attention_weight_max": 39.9315185546875,
      "activations/layer16_attention_weight_min": -35.50220489501953,
      "activations/layer17_attention_weight_max": 62.21072769165039,
      "activations/layer17_attention_weight_min": -51.16377258300781,
      "activations/layer18_attention_weight_max": 59.1491584777832,
      "activations/layer18_attention_weight_min": -46.32111740112305,
      "activations/layer19_attention_weight_max": 32.24265670776367,
      "activations/layer19_attention_weight_min": -25.193660736083984,
      "activations/layer1_attention_weight_max": 15.464188575744629,
      "activations/layer1_attention_weight_min": -13.749755859375,
      "activations/layer20_attention_weight_max": 28.961856842041016,
      "activations/layer20_attention_weight_min": -21.42410659790039,
      "activations/layer21_attention_weight_max": 45.790008544921875,
      "activations/layer21_attention_weight_min": -23.250228881835938,
      "activations/layer22_attention_weight_max": 34.426082611083984,
      "activations/layer22_attention_weight_min": -23.815427780151367,
      "activations/layer23_attention_weight_max": 43.36152648925781,
      "activations/layer23_attention_weight_min": -21.738929748535156,
      "activations/layer2_attention_weight_max": 30.21735954284668,
      "activations/layer2_attention_weight_min": -29.409765243530273,
      "activations/layer3_attention_weight_max": 90.27113342285156,
      "activations/layer3_attention_weight_min": -86.66107940673828,
      "activations/layer4_attention_weight_max": 86.0152359008789,
      "activations/layer4_attention_weight_min": -85.84122467041016,
      "activations/layer5_attention_weight_max": 61.600948333740234,
      "activations/layer5_attention_weight_min": -69.35176086425781,
      "activations/layer6_attention_weight_max": 52.91270065307617,
      "activations/layer6_attention_weight_min": -52.24808120727539,
      "activations/layer7_attention_weight_max": 70.499755859375,
      "activations/layer7_attention_weight_min": -69.08534240722656,
      "activations/layer8_attention_weight_max": 50.91855239868164,
      "activations/layer8_attention_weight_min": -54.29780960083008,
      "activations/layer9_attention_weight_max": 48.833595275878906,
      "activations/layer9_attention_weight_min": -48.26246643066406,
      "epoch": 21.54,
      "learning_rate": 1.121060606060606e-05,
      "loss": 2.7116,
      "step": 370700
    },
    {
      "activations/layer0_attention_weight_max": 15.75556755065918,
      "activations/layer0_attention_weight_min": -13.979175567626953,
      "activations/layer10_attention_weight_max": 35.438941955566406,
      "activations/layer10_attention_weight_min": -37.55042266845703,
      "activations/layer11_attention_weight_max": 32.966835021972656,
      "activations/layer11_attention_weight_min": -35.832496643066406,
      "activations/layer12_attention_weight_max": 20.27467918395996,
      "activations/layer12_attention_weight_min": -24.58562660217285,
      "activations/layer13_attention_weight_max": 32.71678161621094,
      "activations/layer13_attention_weight_min": -27.59840965270996,
      "activations/layer14_attention_weight_max": 36.05388641357422,
      "activations/layer14_attention_weight_min": -31.933691024780273,
      "activations/layer15_attention_weight_max": 32.88935852050781,
      "activations/layer15_attention_weight_min": -31.429738998413086,
      "activations/layer16_attention_weight_max": 33.96238708496094,
      "activations/layer16_attention_weight_min": -32.71650314331055,
      "activations/layer17_attention_weight_max": 51.541656494140625,
      "activations/layer17_attention_weight_min": -45.149410247802734,
      "activations/layer18_attention_weight_max": 47.08656311035156,
      "activations/layer18_attention_weight_min": -39.0196533203125,
      "activations/layer19_attention_weight_max": 24.735788345336914,
      "activations/layer19_attention_weight_min": -23.932340621948242,
      "activations/layer1_attention_weight_max": 16.008886337280273,
      "activations/layer1_attention_weight_min": -13.56950569152832,
      "activations/layer20_attention_weight_max": 22.85152816772461,
      "activations/layer20_attention_weight_min": -21.437000274658203,
      "activations/layer21_attention_weight_max": 33.47441482543945,
      "activations/layer21_attention_weight_min": -22.25089454650879,
      "activations/layer22_attention_weight_max": 30.78662872314453,
      "activations/layer22_attention_weight_min": -25.24135398864746,
      "activations/layer23_attention_weight_max": 35.3216438293457,
      "activations/layer23_attention_weight_min": -26.66341781616211,
      "activations/layer2_attention_weight_max": 30.073089599609375,
      "activations/layer2_attention_weight_min": -29.320697784423828,
      "activations/layer3_attention_weight_max": 83.74308776855469,
      "activations/layer3_attention_weight_min": -89.85474395751953,
      "activations/layer4_attention_weight_max": 81.2197265625,
      "activations/layer4_attention_weight_min": -84.15017700195312,
      "activations/layer5_attention_weight_max": 65.42388916015625,
      "activations/layer5_attention_weight_min": -69.11244201660156,
      "activations/layer6_attention_weight_max": 47.79389953613281,
      "activations/layer6_attention_weight_min": -48.56711196899414,
      "activations/layer7_attention_weight_max": 71.87261962890625,
      "activations/layer7_attention_weight_min": -63.416900634765625,
      "activations/layer8_attention_weight_max": 45.1746940612793,
      "activations/layer8_attention_weight_min": -49.02997970581055,
      "activations/layer9_attention_weight_max": 45.10295867919922,
      "activations/layer9_attention_weight_min": -48.8188362121582,
      "epoch": 21.54,
      "learning_rate": 1.1191666666666667e-05,
      "loss": 2.7086,
      "step": 370750
    },
    {
      "activations/layer0_attention_weight_max": 16.610837936401367,
      "activations/layer0_attention_weight_min": -14.33619213104248,
      "activations/layer10_attention_weight_max": 37.279640197753906,
      "activations/layer10_attention_weight_min": -37.079280853271484,
      "activations/layer11_attention_weight_max": 33.419342041015625,
      "activations/layer11_attention_weight_min": -34.226409912109375,
      "activations/layer12_attention_weight_max": 17.777921676635742,
      "activations/layer12_attention_weight_min": -23.7471981048584,
      "activations/layer13_attention_weight_max": 31.71022605895996,
      "activations/layer13_attention_weight_min": -27.850038528442383,
      "activations/layer14_attention_weight_max": 32.83405303955078,
      "activations/layer14_attention_weight_min": -29.56314468383789,
      "activations/layer15_attention_weight_max": 31.81136703491211,
      "activations/layer15_attention_weight_min": -28.784082412719727,
      "activations/layer16_attention_weight_max": 32.76566696166992,
      "activations/layer16_attention_weight_min": -31.77455711364746,
      "activations/layer17_attention_weight_max": 52.3031005859375,
      "activations/layer17_attention_weight_min": -43.67143249511719,
      "activations/layer18_attention_weight_max": 46.62439727783203,
      "activations/layer18_attention_weight_min": -37.94688034057617,
      "activations/layer19_attention_weight_max": 23.71400260925293,
      "activations/layer19_attention_weight_min": -22.862627029418945,
      "activations/layer1_attention_weight_max": 14.633455276489258,
      "activations/layer1_attention_weight_min": -14.855518341064453,
      "activations/layer20_attention_weight_max": 21.917007446289062,
      "activations/layer20_attention_weight_min": -21.018810272216797,
      "activations/layer21_attention_weight_max": 33.82007598876953,
      "activations/layer21_attention_weight_min": -20.624462127685547,
      "activations/layer22_attention_weight_max": 30.045991897583008,
      "activations/layer22_attention_weight_min": -26.52765655517578,
      "activations/layer23_attention_weight_max": 32.8834114074707,
      "activations/layer23_attention_weight_min": -23.957271575927734,
      "activations/layer2_attention_weight_max": 30.548429489135742,
      "activations/layer2_attention_weight_min": -29.40235710144043,
      "activations/layer3_attention_weight_max": 89.4407958984375,
      "activations/layer3_attention_weight_min": -87.01802825927734,
      "activations/layer4_attention_weight_max": 84.92688751220703,
      "activations/layer4_attention_weight_min": -83.8108139038086,
      "activations/layer5_attention_weight_max": 61.350486755371094,
      "activations/layer5_attention_weight_min": -76.39144897460938,
      "activations/layer6_attention_weight_max": 52.78742218017578,
      "activations/layer6_attention_weight_min": -50.293739318847656,
      "activations/layer7_attention_weight_max": 67.31587219238281,
      "activations/layer7_attention_weight_min": -67.13753509521484,
      "activations/layer8_attention_weight_max": 48.111515045166016,
      "activations/layer8_attention_weight_min": -48.64445877075195,
      "activations/layer9_attention_weight_max": 43.03110885620117,
      "activations/layer9_attention_weight_min": -47.59698486328125,
      "epoch": 21.55,
      "learning_rate": 1.1172727272727271e-05,
      "loss": 2.704,
      "step": 370800
    },
    {
      "activations/layer0_attention_weight_max": 16.72480010986328,
      "activations/layer0_attention_weight_min": -14.267498016357422,
      "activations/layer10_attention_weight_max": 36.85602569580078,
      "activations/layer10_attention_weight_min": -37.54399490356445,
      "activations/layer11_attention_weight_max": 35.5590705871582,
      "activations/layer11_attention_weight_min": -36.156211853027344,
      "activations/layer12_attention_weight_max": 18.244903564453125,
      "activations/layer12_attention_weight_min": -26.324508666992188,
      "activations/layer13_attention_weight_max": 39.15681076049805,
      "activations/layer13_attention_weight_min": -29.683656692504883,
      "activations/layer14_attention_weight_max": 34.88895034790039,
      "activations/layer14_attention_weight_min": -29.953611373901367,
      "activations/layer15_attention_weight_max": 33.19620132446289,
      "activations/layer15_attention_weight_min": -29.948482513427734,
      "activations/layer16_attention_weight_max": 34.343910217285156,
      "activations/layer16_attention_weight_min": -32.294734954833984,
      "activations/layer17_attention_weight_max": 48.58815383911133,
      "activations/layer17_attention_weight_min": -45.19758605957031,
      "activations/layer18_attention_weight_max": 46.38668441772461,
      "activations/layer18_attention_weight_min": -41.68899154663086,
      "activations/layer19_attention_weight_max": 23.176542282104492,
      "activations/layer19_attention_weight_min": -21.895231246948242,
      "activations/layer1_attention_weight_max": 15.191819190979004,
      "activations/layer1_attention_weight_min": -13.772796630859375,
      "activations/layer20_attention_weight_max": 20.932353973388672,
      "activations/layer20_attention_weight_min": -20.949295043945312,
      "activations/layer21_attention_weight_max": 32.712642669677734,
      "activations/layer21_attention_weight_min": -21.268301010131836,
      "activations/layer22_attention_weight_max": 26.53580093383789,
      "activations/layer22_attention_weight_min": -24.292192459106445,
      "activations/layer23_attention_weight_max": 34.289764404296875,
      "activations/layer23_attention_weight_min": -24.189603805541992,
      "activations/layer2_attention_weight_max": 31.45387840270996,
      "activations/layer2_attention_weight_min": -29.855548858642578,
      "activations/layer3_attention_weight_max": 87.72242736816406,
      "activations/layer3_attention_weight_min": -88.40828704833984,
      "activations/layer4_attention_weight_max": 85.67752838134766,
      "activations/layer4_attention_weight_min": -85.80474090576172,
      "activations/layer5_attention_weight_max": 66.00437927246094,
      "activations/layer5_attention_weight_min": -71.21717834472656,
      "activations/layer6_attention_weight_max": 51.64780807495117,
      "activations/layer6_attention_weight_min": -51.01615524291992,
      "activations/layer7_attention_weight_max": 67.7450942993164,
      "activations/layer7_attention_weight_min": -67.29925537109375,
      "activations/layer8_attention_weight_max": 49.41867446899414,
      "activations/layer8_attention_weight_min": -52.77816390991211,
      "activations/layer9_attention_weight_max": 46.9270133972168,
      "activations/layer9_attention_weight_min": -46.577171325683594,
      "epoch": 21.55,
      "learning_rate": 1.1154545454545455e-05,
      "loss": 2.7095,
      "step": 370850
    },
    {
      "activations/layer0_attention_weight_max": 16.025720596313477,
      "activations/layer0_attention_weight_min": -14.421433448791504,
      "activations/layer10_attention_weight_max": 38.75680923461914,
      "activations/layer10_attention_weight_min": -37.85972213745117,
      "activations/layer11_attention_weight_max": 34.311058044433594,
      "activations/layer11_attention_weight_min": -34.90166473388672,
      "activations/layer12_attention_weight_max": 20.101831436157227,
      "activations/layer12_attention_weight_min": -33.100467681884766,
      "activations/layer13_attention_weight_max": 35.15321731567383,
      "activations/layer13_attention_weight_min": -29.80014419555664,
      "activations/layer14_attention_weight_max": 39.388832092285156,
      "activations/layer14_attention_weight_min": -32.353599548339844,
      "activations/layer15_attention_weight_max": 34.23075485229492,
      "activations/layer15_attention_weight_min": -32.865291595458984,
      "activations/layer16_attention_weight_max": 37.0164794921875,
      "activations/layer16_attention_weight_min": -34.07276916503906,
      "activations/layer17_attention_weight_max": 54.39836883544922,
      "activations/layer17_attention_weight_min": -45.944026947021484,
      "activations/layer18_attention_weight_max": 49.54094314575195,
      "activations/layer18_attention_weight_min": -41.7115592956543,
      "activations/layer19_attention_weight_max": 28.359643936157227,
      "activations/layer19_attention_weight_min": -26.074783325195312,
      "activations/layer1_attention_weight_max": 15.295780181884766,
      "activations/layer1_attention_weight_min": -14.24593448638916,
      "activations/layer20_attention_weight_max": 25.929574966430664,
      "activations/layer20_attention_weight_min": -21.450353622436523,
      "activations/layer21_attention_weight_max": 37.50822830200195,
      "activations/layer21_attention_weight_min": -20.705827713012695,
      "activations/layer22_attention_weight_max": 28.84516716003418,
      "activations/layer22_attention_weight_min": -23.934181213378906,
      "activations/layer23_attention_weight_max": 34.79016876220703,
      "activations/layer23_attention_weight_min": -21.7584285736084,
      "activations/layer2_attention_weight_max": 31.52503204345703,
      "activations/layer2_attention_weight_min": -29.693462371826172,
      "activations/layer3_attention_weight_max": 88.78289794921875,
      "activations/layer3_attention_weight_min": -85.42567443847656,
      "activations/layer4_attention_weight_max": 85.61986541748047,
      "activations/layer4_attention_weight_min": -84.1419906616211,
      "activations/layer5_attention_weight_max": 63.115352630615234,
      "activations/layer5_attention_weight_min": -69.34123992919922,
      "activations/layer6_attention_weight_max": 50.77790451049805,
      "activations/layer6_attention_weight_min": -49.68550491333008,
      "activations/layer7_attention_weight_max": 69.51081848144531,
      "activations/layer7_attention_weight_min": -68.7558364868164,
      "activations/layer8_attention_weight_max": 48.59098434448242,
      "activations/layer8_attention_weight_min": -52.54072189331055,
      "activations/layer9_attention_weight_max": 45.553680419921875,
      "activations/layer9_attention_weight_min": -47.533790588378906,
      "epoch": 21.55,
      "learning_rate": 1.113560606060606e-05,
      "loss": 2.7244,
      "step": 370900
    },
    {
      "activations/layer0_attention_weight_max": 16.978076934814453,
      "activations/layer0_attention_weight_min": -14.443587303161621,
      "activations/layer10_attention_weight_max": 35.28718185424805,
      "activations/layer10_attention_weight_min": -38.77690124511719,
      "activations/layer11_attention_weight_max": 35.50958251953125,
      "activations/layer11_attention_weight_min": -34.94553756713867,
      "activations/layer12_attention_weight_max": 24.15410041809082,
      "activations/layer12_attention_weight_min": -25.052793502807617,
      "activations/layer13_attention_weight_max": 33.654273986816406,
      "activations/layer13_attention_weight_min": -27.779760360717773,
      "activations/layer14_attention_weight_max": 34.89227294921875,
      "activations/layer14_attention_weight_min": -28.764263153076172,
      "activations/layer15_attention_weight_max": 34.035125732421875,
      "activations/layer15_attention_weight_min": -30.151811599731445,
      "activations/layer16_attention_weight_max": 34.56278610229492,
      "activations/layer16_attention_weight_min": -32.98970413208008,
      "activations/layer17_attention_weight_max": 51.87983322143555,
      "activations/layer17_attention_weight_min": -46.84678649902344,
      "activations/layer18_attention_weight_max": 48.77534103393555,
      "activations/layer18_attention_weight_min": -42.43315887451172,
      "activations/layer19_attention_weight_max": 23.32147216796875,
      "activations/layer19_attention_weight_min": -23.886859893798828,
      "activations/layer1_attention_weight_max": 15.196381568908691,
      "activations/layer1_attention_weight_min": -14.09704303741455,
      "activations/layer20_attention_weight_max": 23.06492805480957,
      "activations/layer20_attention_weight_min": -22.121482849121094,
      "activations/layer21_attention_weight_max": 36.09975814819336,
      "activations/layer21_attention_weight_min": -24.21439552307129,
      "activations/layer22_attention_weight_max": 29.66027069091797,
      "activations/layer22_attention_weight_min": -24.900468826293945,
      "activations/layer23_attention_weight_max": 35.53534698486328,
      "activations/layer23_attention_weight_min": -25.08711051940918,
      "activations/layer2_attention_weight_max": 32.26933288574219,
      "activations/layer2_attention_weight_min": -31.268535614013672,
      "activations/layer3_attention_weight_max": 91.46284484863281,
      "activations/layer3_attention_weight_min": -94.63776397705078,
      "activations/layer4_attention_weight_max": 88.88762664794922,
      "activations/layer4_attention_weight_min": -89.42778778076172,
      "activations/layer5_attention_weight_max": 62.37202072143555,
      "activations/layer5_attention_weight_min": -71.54338073730469,
      "activations/layer6_attention_weight_max": 49.18928527832031,
      "activations/layer6_attention_weight_min": -51.36128234863281,
      "activations/layer7_attention_weight_max": 69.82795715332031,
      "activations/layer7_attention_weight_min": -68.08378601074219,
      "activations/layer8_attention_weight_max": 46.289424896240234,
      "activations/layer8_attention_weight_min": -52.302528381347656,
      "activations/layer9_attention_weight_max": 45.322975158691406,
      "activations/layer9_attention_weight_min": -49.36212921142578,
      "epoch": 21.55,
      "learning_rate": 1.1116666666666666e-05,
      "loss": 2.7105,
      "step": 370950
    },
    {
      "activations/layer0_attention_weight_max": 15.71286678314209,
      "activations/layer0_attention_weight_min": -13.870301246643066,
      "activations/layer10_attention_weight_max": 35.9122200012207,
      "activations/layer10_attention_weight_min": -38.19025802612305,
      "activations/layer11_attention_weight_max": 35.012596130371094,
      "activations/layer11_attention_weight_min": -37.101226806640625,
      "activations/layer12_attention_weight_max": 18.640827178955078,
      "activations/layer12_attention_weight_min": -30.834457397460938,
      "activations/layer13_attention_weight_max": 33.2630500793457,
      "activations/layer13_attention_weight_min": -26.59067726135254,
      "activations/layer14_attention_weight_max": 34.144554138183594,
      "activations/layer14_attention_weight_min": -30.44328498840332,
      "activations/layer15_attention_weight_max": 33.138729095458984,
      "activations/layer15_attention_weight_min": -30.4763126373291,
      "activations/layer16_attention_weight_max": 36.55158996582031,
      "activations/layer16_attention_weight_min": -34.58936309814453,
      "activations/layer17_attention_weight_max": 54.15683364868164,
      "activations/layer17_attention_weight_min": -45.47444534301758,
      "activations/layer18_attention_weight_max": 47.821346282958984,
      "activations/layer18_attention_weight_min": -43.8926887512207,
      "activations/layer19_attention_weight_max": 22.466650009155273,
      "activations/layer19_attention_weight_min": -23.428190231323242,
      "activations/layer1_attention_weight_max": 15.484889030456543,
      "activations/layer1_attention_weight_min": -16.059141159057617,
      "activations/layer20_attention_weight_max": 21.670902252197266,
      "activations/layer20_attention_weight_min": -21.971580505371094,
      "activations/layer21_attention_weight_max": 32.240901947021484,
      "activations/layer21_attention_weight_min": -22.23955535888672,
      "activations/layer22_attention_weight_max": 25.94434928894043,
      "activations/layer22_attention_weight_min": -26.325422286987305,
      "activations/layer23_attention_weight_max": 34.633331298828125,
      "activations/layer23_attention_weight_min": -25.640655517578125,
      "activations/layer2_attention_weight_max": 31.675336837768555,
      "activations/layer2_attention_weight_min": -31.06429100036621,
      "activations/layer3_attention_weight_max": 88.80658721923828,
      "activations/layer3_attention_weight_min": -90.36333465576172,
      "activations/layer4_attention_weight_max": 82.36382293701172,
      "activations/layer4_attention_weight_min": -82.86859893798828,
      "activations/layer5_attention_weight_max": 61.7362060546875,
      "activations/layer5_attention_weight_min": -70.93675994873047,
      "activations/layer6_attention_weight_max": 49.10958480834961,
      "activations/layer6_attention_weight_min": -51.69412612915039,
      "activations/layer7_attention_weight_max": 64.02464294433594,
      "activations/layer7_attention_weight_min": -68.28128051757812,
      "activations/layer8_attention_weight_max": 46.5069465637207,
      "activations/layer8_attention_weight_min": -53.39201736450195,
      "activations/layer9_attention_weight_max": 45.66400909423828,
      "activations/layer9_attention_weight_min": -47.1381950378418,
      "epoch": 21.56,
      "learning_rate": 1.1097727272727271e-05,
      "loss": 2.716,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4605,
      "eval_samples_per_second": 507.532,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4605,
      "eval_openwebtext_samples_per_second": 507.532,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_wikitext_loss": 2.904296875,
      "eval_wikitext_ppl": 18.2524054167373,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.291,
      "step": 371000
    },
    {
      "epoch": 21.56,
      "eval_lambada_loss": 2.388671875,
      "eval_lambada_ppl": 10.899009079882486,
      "eval_lambada_runtime": 9.5674,
      "eval_lambada_samples_per_second": 508.918,
      "step": 371000
    },
    {
      "activations/layer0_attention_weight_max": 15.900644302368164,
      "activations/layer0_attention_weight_min": -14.037163734436035,
      "activations/layer10_attention_weight_max": 35.31736373901367,
      "activations/layer10_attention_weight_min": -34.71957778930664,
      "activations/layer11_attention_weight_max": 32.68423843383789,
      "activations/layer11_attention_weight_min": -34.30854797363281,
      "activations/layer12_attention_weight_max": 20.094181060791016,
      "activations/layer12_attention_weight_min": -26.454811096191406,
      "activations/layer13_attention_weight_max": 35.68101119995117,
      "activations/layer13_attention_weight_min": -26.638607025146484,
      "activations/layer14_attention_weight_max": 34.96249008178711,
      "activations/layer14_attention_weight_min": -30.9643611907959,
      "activations/layer15_attention_weight_max": 33.29033660888672,
      "activations/layer15_attention_weight_min": -29.819589614868164,
      "activations/layer16_attention_weight_max": 34.823699951171875,
      "activations/layer16_attention_weight_min": -34.37800216674805,
      "activations/layer17_attention_weight_max": 51.786869049072266,
      "activations/layer17_attention_weight_min": -46.05131912231445,
      "activations/layer18_attention_weight_max": 49.17095184326172,
      "activations/layer18_attention_weight_min": -44.83439254760742,
      "activations/layer19_attention_weight_max": 24.191757202148438,
      "activations/layer19_attention_weight_min": -24.514455795288086,
      "activations/layer1_attention_weight_max": 15.143406867980957,
      "activations/layer1_attention_weight_min": -13.546330451965332,
      "activations/layer20_attention_weight_max": 24.611370086669922,
      "activations/layer20_attention_weight_min": -22.317785263061523,
      "activations/layer21_attention_weight_max": 36.866310119628906,
      "activations/layer21_attention_weight_min": -28.96664810180664,
      "activations/layer22_attention_weight_max": 30.541852951049805,
      "activations/layer22_attention_weight_min": -26.01671600341797,
      "activations/layer23_attention_weight_max": 37.78630065917969,
      "activations/layer23_attention_weight_min": -24.474502563476562,
      "activations/layer2_attention_weight_max": 30.561012268066406,
      "activations/layer2_attention_weight_min": -29.47517967224121,
      "activations/layer3_attention_weight_max": 87.45111846923828,
      "activations/layer3_attention_weight_min": -86.41956329345703,
      "activations/layer4_attention_weight_max": 81.22673797607422,
      "activations/layer4_attention_weight_min": -80.72035217285156,
      "activations/layer5_attention_weight_max": 60.7821044921875,
      "activations/layer5_attention_weight_min": -71.30406188964844,
      "activations/layer6_attention_weight_max": 50.67969512939453,
      "activations/layer6_attention_weight_min": -50.21358871459961,
      "activations/layer7_attention_weight_max": 68.3597640991211,
      "activations/layer7_attention_weight_min": -68.58094787597656,
      "activations/layer8_attention_weight_max": 48.89851760864258,
      "activations/layer8_attention_weight_min": -52.78658676147461,
      "activations/layer9_attention_weight_max": 44.945430755615234,
      "activations/layer9_attention_weight_min": -46.536136627197266,
      "epoch": 21.56,
      "learning_rate": 1.1078787878787878e-05,
      "loss": 2.706,
      "step": 371050
    },
    {
      "activations/layer0_attention_weight_max": 16.47713279724121,
      "activations/layer0_attention_weight_min": -14.081260681152344,
      "activations/layer10_attention_weight_max": 39.18205261230469,
      "activations/layer10_attention_weight_min": -38.08838653564453,
      "activations/layer11_attention_weight_max": 36.33075714111328,
      "activations/layer11_attention_weight_min": -36.771507263183594,
      "activations/layer12_attention_weight_max": 18.939756393432617,
      "activations/layer12_attention_weight_min": -24.000110626220703,
      "activations/layer13_attention_weight_max": 40.98212814331055,
      "activations/layer13_attention_weight_min": -29.24862289428711,
      "activations/layer14_attention_weight_max": 39.49216079711914,
      "activations/layer14_attention_weight_min": -31.79935646057129,
      "activations/layer15_attention_weight_max": 38.2245979309082,
      "activations/layer15_attention_weight_min": -31.693763732910156,
      "activations/layer16_attention_weight_max": 37.844364166259766,
      "activations/layer16_attention_weight_min": -35.80807876586914,
      "activations/layer17_attention_weight_max": 54.266231536865234,
      "activations/layer17_attention_weight_min": -48.692955017089844,
      "activations/layer18_attention_weight_max": 52.25457763671875,
      "activations/layer18_attention_weight_min": -41.63398742675781,
      "activations/layer19_attention_weight_max": 26.178516387939453,
      "activations/layer19_attention_weight_min": -21.638437271118164,
      "activations/layer1_attention_weight_max": 14.8560152053833,
      "activations/layer1_attention_weight_min": -14.04401683807373,
      "activations/layer20_attention_weight_max": 23.232004165649414,
      "activations/layer20_attention_weight_min": -21.81023597717285,
      "activations/layer21_attention_weight_max": 31.155071258544922,
      "activations/layer21_attention_weight_min": -20.002323150634766,
      "activations/layer22_attention_weight_max": 29.883602142333984,
      "activations/layer22_attention_weight_min": -25.68832778930664,
      "activations/layer23_attention_weight_max": 37.51706314086914,
      "activations/layer23_attention_weight_min": -23.95537567138672,
      "activations/layer2_attention_weight_max": 31.568004608154297,
      "activations/layer2_attention_weight_min": -29.29930877685547,
      "activations/layer3_attention_weight_max": 91.4537353515625,
      "activations/layer3_attention_weight_min": -90.07524108886719,
      "activations/layer4_attention_weight_max": 86.32730865478516,
      "activations/layer4_attention_weight_min": -87.51546478271484,
      "activations/layer5_attention_weight_max": 63.120628356933594,
      "activations/layer5_attention_weight_min": -73.90211486816406,
      "activations/layer6_attention_weight_max": 51.68983459472656,
      "activations/layer6_attention_weight_min": -56.54273223876953,
      "activations/layer7_attention_weight_max": 75.38966369628906,
      "activations/layer7_attention_weight_min": -70.92669677734375,
      "activations/layer8_attention_weight_max": 50.02653121948242,
      "activations/layer8_attention_weight_min": -54.85451889038086,
      "activations/layer9_attention_weight_max": 45.77532958984375,
      "activations/layer9_attention_weight_min": -47.74049758911133,
      "epoch": 21.56,
      "learning_rate": 1.1059848484848484e-05,
      "loss": 2.7102,
      "step": 371100
    },
    {
      "activations/layer0_attention_weight_max": 16.70392417907715,
      "activations/layer0_attention_weight_min": -14.168342590332031,
      "activations/layer10_attention_weight_max": 36.0029296875,
      "activations/layer10_attention_weight_min": -37.58154296875,
      "activations/layer11_attention_weight_max": 33.48259353637695,
      "activations/layer11_attention_weight_min": -35.193199157714844,
      "activations/layer12_attention_weight_max": 18.880401611328125,
      "activations/layer12_attention_weight_min": -22.875186920166016,
      "activations/layer13_attention_weight_max": 32.20769500732422,
      "activations/layer13_attention_weight_min": -27.43295669555664,
      "activations/layer14_attention_weight_max": 32.941017150878906,
      "activations/layer14_attention_weight_min": -29.190793991088867,
      "activations/layer15_attention_weight_max": 31.344188690185547,
      "activations/layer15_attention_weight_min": -30.902645111083984,
      "activations/layer16_attention_weight_max": 32.22293472290039,
      "activations/layer16_attention_weight_min": -31.17594337463379,
      "activations/layer17_attention_weight_max": 47.10051727294922,
      "activations/layer17_attention_weight_min": -42.6375617980957,
      "activations/layer18_attention_weight_max": 45.1539192199707,
      "activations/layer18_attention_weight_min": -38.88215255737305,
      "activations/layer19_attention_weight_max": 21.62516975402832,
      "activations/layer19_attention_weight_min": -24.129474639892578,
      "activations/layer1_attention_weight_max": 15.441484451293945,
      "activations/layer1_attention_weight_min": -15.237512588500977,
      "activations/layer20_attention_weight_max": 19.866270065307617,
      "activations/layer20_attention_weight_min": -20.74892807006836,
      "activations/layer21_attention_weight_max": 32.421329498291016,
      "activations/layer21_attention_weight_min": -21.613418579101562,
      "activations/layer22_attention_weight_max": 28.060546875,
      "activations/layer22_attention_weight_min": -25.6840763092041,
      "activations/layer23_attention_weight_max": 38.003684997558594,
      "activations/layer23_attention_weight_min": -23.926368713378906,
      "activations/layer2_attention_weight_max": 31.409465789794922,
      "activations/layer2_attention_weight_min": -30.3419246673584,
      "activations/layer3_attention_weight_max": 89.08183288574219,
      "activations/layer3_attention_weight_min": -86.36133575439453,
      "activations/layer4_attention_weight_max": 81.07485961914062,
      "activations/layer4_attention_weight_min": -84.59226989746094,
      "activations/layer5_attention_weight_max": 59.10786056518555,
      "activations/layer5_attention_weight_min": -71.31861114501953,
      "activations/layer6_attention_weight_max": 48.29526901245117,
      "activations/layer6_attention_weight_min": -51.06438446044922,
      "activations/layer7_attention_weight_max": 66.57898712158203,
      "activations/layer7_attention_weight_min": -68.83126068115234,
      "activations/layer8_attention_weight_max": 47.029029846191406,
      "activations/layer8_attention_weight_min": -52.50480651855469,
      "activations/layer9_attention_weight_max": 44.68567657470703,
      "activations/layer9_attention_weight_min": -48.802120208740234,
      "epoch": 21.57,
      "learning_rate": 1.104090909090909e-05,
      "loss": 2.7054,
      "step": 371150
    },
    {
      "activations/layer0_attention_weight_max": 16.311967849731445,
      "activations/layer0_attention_weight_min": -14.21336841583252,
      "activations/layer10_attention_weight_max": 34.47468185424805,
      "activations/layer10_attention_weight_min": -35.59186935424805,
      "activations/layer11_attention_weight_max": 31.901947021484375,
      "activations/layer11_attention_weight_min": -34.935543060302734,
      "activations/layer12_attention_weight_max": 19.5261173248291,
      "activations/layer12_attention_weight_min": -22.942827224731445,
      "activations/layer13_attention_weight_max": 33.316280364990234,
      "activations/layer13_attention_weight_min": -31.41707992553711,
      "activations/layer14_attention_weight_max": 37.99318313598633,
      "activations/layer14_attention_weight_min": -30.371427536010742,
      "activations/layer15_attention_weight_max": 33.71281433105469,
      "activations/layer15_attention_weight_min": -30.8411922454834,
      "activations/layer16_attention_weight_max": 34.76960754394531,
      "activations/layer16_attention_weight_min": -32.27488327026367,
      "activations/layer17_attention_weight_max": 53.56473922729492,
      "activations/layer17_attention_weight_min": -46.38896942138672,
      "activations/layer18_attention_weight_max": 49.98776626586914,
      "activations/layer18_attention_weight_min": -42.01761245727539,
      "activations/layer19_attention_weight_max": 24.254613876342773,
      "activations/layer19_attention_weight_min": -22.38056755065918,
      "activations/layer1_attention_weight_max": 15.167130470275879,
      "activations/layer1_attention_weight_min": -14.005988121032715,
      "activations/layer20_attention_weight_max": 22.61579132080078,
      "activations/layer20_attention_weight_min": -21.07139778137207,
      "activations/layer21_attention_weight_max": 33.15659713745117,
      "activations/layer21_attention_weight_min": -22.81158447265625,
      "activations/layer22_attention_weight_max": 30.11997413635254,
      "activations/layer22_attention_weight_min": -24.909822463989258,
      "activations/layer23_attention_weight_max": 34.086669921875,
      "activations/layer23_attention_weight_min": -22.421981811523438,
      "activations/layer2_attention_weight_max": 32.75706100463867,
      "activations/layer2_attention_weight_min": -29.62961196899414,
      "activations/layer3_attention_weight_max": 88.45685577392578,
      "activations/layer3_attention_weight_min": -86.98070526123047,
      "activations/layer4_attention_weight_max": 82.1820297241211,
      "activations/layer4_attention_weight_min": -80.5900650024414,
      "activations/layer5_attention_weight_max": 58.43788146972656,
      "activations/layer5_attention_weight_min": -74.49752807617188,
      "activations/layer6_attention_weight_max": 47.59846115112305,
      "activations/layer6_attention_weight_min": -47.8723258972168,
      "activations/layer7_attention_weight_max": 67.55233764648438,
      "activations/layer7_attention_weight_min": -65.19352722167969,
      "activations/layer8_attention_weight_max": 45.426002502441406,
      "activations/layer8_attention_weight_min": -49.069889068603516,
      "activations/layer9_attention_weight_max": 42.095027923583984,
      "activations/layer9_attention_weight_min": -44.91438293457031,
      "epoch": 21.57,
      "learning_rate": 1.1021969696969696e-05,
      "loss": 2.7172,
      "step": 371200
    },
    {
      "activations/layer0_attention_weight_max": 16.578365325927734,
      "activations/layer0_attention_weight_min": -14.302285194396973,
      "activations/layer10_attention_weight_max": 35.60100173950195,
      "activations/layer10_attention_weight_min": -36.63862991333008,
      "activations/layer11_attention_weight_max": 35.169403076171875,
      "activations/layer11_attention_weight_min": -34.445777893066406,
      "activations/layer12_attention_weight_max": 20.36200523376465,
      "activations/layer12_attention_weight_min": -23.04644203186035,
      "activations/layer13_attention_weight_max": 34.19242858886719,
      "activations/layer13_attention_weight_min": -27.944711685180664,
      "activations/layer14_attention_weight_max": 35.73849105834961,
      "activations/layer14_attention_weight_min": -28.62958526611328,
      "activations/layer15_attention_weight_max": 34.5892448425293,
      "activations/layer15_attention_weight_min": -28.632347106933594,
      "activations/layer16_attention_weight_max": 34.99508285522461,
      "activations/layer16_attention_weight_min": -33.43056106567383,
      "activations/layer17_attention_weight_max": 50.977333068847656,
      "activations/layer17_attention_weight_min": -49.45134735107422,
      "activations/layer18_attention_weight_max": 50.38239288330078,
      "activations/layer18_attention_weight_min": -43.32175827026367,
      "activations/layer19_attention_weight_max": 24.65304946899414,
      "activations/layer19_attention_weight_min": -24.51380157470703,
      "activations/layer1_attention_weight_max": 15.728315353393555,
      "activations/layer1_attention_weight_min": -13.84499454498291,
      "activations/layer20_attention_weight_max": 23.42508888244629,
      "activations/layer20_attention_weight_min": -21.693021774291992,
      "activations/layer21_attention_weight_max": 37.00762176513672,
      "activations/layer21_attention_weight_min": -23.470815658569336,
      "activations/layer22_attention_weight_max": 29.554012298583984,
      "activations/layer22_attention_weight_min": -27.2911319732666,
      "activations/layer23_attention_weight_max": 38.53730392456055,
      "activations/layer23_attention_weight_min": -24.590513229370117,
      "activations/layer2_attention_weight_max": 29.758460998535156,
      "activations/layer2_attention_weight_min": -29.49117660522461,
      "activations/layer3_attention_weight_max": 86.32575225830078,
      "activations/layer3_attention_weight_min": -84.45767974853516,
      "activations/layer4_attention_weight_max": 83.68697357177734,
      "activations/layer4_attention_weight_min": -82.94342041015625,
      "activations/layer5_attention_weight_max": 61.249794006347656,
      "activations/layer5_attention_weight_min": -69.10989379882812,
      "activations/layer6_attention_weight_max": 49.477169036865234,
      "activations/layer6_attention_weight_min": -54.075748443603516,
      "activations/layer7_attention_weight_max": 68.26309204101562,
      "activations/layer7_attention_weight_min": -66.5749740600586,
      "activations/layer8_attention_weight_max": 47.43561935424805,
      "activations/layer8_attention_weight_min": -50.78766632080078,
      "activations/layer9_attention_weight_max": 44.752506256103516,
      "activations/layer9_attention_weight_min": -46.888572692871094,
      "epoch": 21.57,
      "learning_rate": 1.1003030303030302e-05,
      "loss": 2.7047,
      "step": 371250
    },
    {
      "activations/layer0_attention_weight_max": 15.800895690917969,
      "activations/layer0_attention_weight_min": -14.110493659973145,
      "activations/layer10_attention_weight_max": 41.06748962402344,
      "activations/layer10_attention_weight_min": -41.22428512573242,
      "activations/layer11_attention_weight_max": 36.89754867553711,
      "activations/layer11_attention_weight_min": -38.929203033447266,
      "activations/layer12_attention_weight_max": 20.179533004760742,
      "activations/layer12_attention_weight_min": -25.629478454589844,
      "activations/layer13_attention_weight_max": 35.18280792236328,
      "activations/layer13_attention_weight_min": -31.828792572021484,
      "activations/layer14_attention_weight_max": 37.27257537841797,
      "activations/layer14_attention_weight_min": -33.30245590209961,
      "activations/layer15_attention_weight_max": 36.37516403198242,
      "activations/layer15_attention_weight_min": -33.09452438354492,
      "activations/layer16_attention_weight_max": 37.3474235534668,
      "activations/layer16_attention_weight_min": -33.50254821777344,
      "activations/layer17_attention_weight_max": 58.70602798461914,
      "activations/layer17_attention_weight_min": -48.14078140258789,
      "activations/layer18_attention_weight_max": 50.78838348388672,
      "activations/layer18_attention_weight_min": -43.41570281982422,
      "activations/layer19_attention_weight_max": 30.777481079101562,
      "activations/layer19_attention_weight_min": -24.848604202270508,
      "activations/layer1_attention_weight_max": 15.473445892333984,
      "activations/layer1_attention_weight_min": -15.165445327758789,
      "activations/layer20_attention_weight_max": 26.678342819213867,
      "activations/layer20_attention_weight_min": -22.598196029663086,
      "activations/layer21_attention_weight_max": 42.606658935546875,
      "activations/layer21_attention_weight_min": -25.99595832824707,
      "activations/layer22_attention_weight_max": 32.37091827392578,
      "activations/layer22_attention_weight_min": -25.83649444580078,
      "activations/layer23_attention_weight_max": 46.87717819213867,
      "activations/layer23_attention_weight_min": -25.336097717285156,
      "activations/layer2_attention_weight_max": 31.37928009033203,
      "activations/layer2_attention_weight_min": -31.724300384521484,
      "activations/layer3_attention_weight_max": 92.59182739257812,
      "activations/layer3_attention_weight_min": -88.95529174804688,
      "activations/layer4_attention_weight_max": 86.81002807617188,
      "activations/layer4_attention_weight_min": -88.83867645263672,
      "activations/layer5_attention_weight_max": 63.00286865234375,
      "activations/layer5_attention_weight_min": -69.66366577148438,
      "activations/layer6_attention_weight_max": 52.668758392333984,
      "activations/layer6_attention_weight_min": -53.75080871582031,
      "activations/layer7_attention_weight_max": 70.77096557617188,
      "activations/layer7_attention_weight_min": -70.82501983642578,
      "activations/layer8_attention_weight_max": 49.47487258911133,
      "activations/layer8_attention_weight_min": -53.657745361328125,
      "activations/layer9_attention_weight_max": 49.93572235107422,
      "activations/layer9_attention_weight_min": -51.11167526245117,
      "epoch": 21.57,
      "learning_rate": 1.0984090909090907e-05,
      "loss": 2.7113,
      "step": 371300
    },
    {
      "activations/layer0_attention_weight_max": 17.108810424804688,
      "activations/layer0_attention_weight_min": -14.340070724487305,
      "activations/layer10_attention_weight_max": 34.47578811645508,
      "activations/layer10_attention_weight_min": -37.78286361694336,
      "activations/layer11_attention_weight_max": 35.291770935058594,
      "activations/layer11_attention_weight_min": -37.747352600097656,
      "activations/layer12_attention_weight_max": 18.689117431640625,
      "activations/layer12_attention_weight_min": -23.893531799316406,
      "activations/layer13_attention_weight_max": 33.18993377685547,
      "activations/layer13_attention_weight_min": -27.814714431762695,
      "activations/layer14_attention_weight_max": 33.6784782409668,
      "activations/layer14_attention_weight_min": -32.85822677612305,
      "activations/layer15_attention_weight_max": 31.225603103637695,
      "activations/layer15_attention_weight_min": -30.330482482910156,
      "activations/layer16_attention_weight_max": 34.81230545043945,
      "activations/layer16_attention_weight_min": -32.48368453979492,
      "activations/layer17_attention_weight_max": 52.76963424682617,
      "activations/layer17_attention_weight_min": -46.259761810302734,
      "activations/layer18_attention_weight_max": 50.067874908447266,
      "activations/layer18_attention_weight_min": -40.0985107421875,
      "activations/layer19_attention_weight_max": 26.750497817993164,
      "activations/layer19_attention_weight_min": -23.25374412536621,
      "activations/layer1_attention_weight_max": 15.252970695495605,
      "activations/layer1_attention_weight_min": -14.385297775268555,
      "activations/layer20_attention_weight_max": 22.392738342285156,
      "activations/layer20_attention_weight_min": -22.74538803100586,
      "activations/layer21_attention_weight_max": 35.674957275390625,
      "activations/layer21_attention_weight_min": -21.460046768188477,
      "activations/layer22_attention_weight_max": 29.185760498046875,
      "activations/layer22_attention_weight_min": -26.523298263549805,
      "activations/layer23_attention_weight_max": 36.34185791015625,
      "activations/layer23_attention_weight_min": -23.935184478759766,
      "activations/layer2_attention_weight_max": 29.30849838256836,
      "activations/layer2_attention_weight_min": -29.605443954467773,
      "activations/layer3_attention_weight_max": 87.62289428710938,
      "activations/layer3_attention_weight_min": -89.00690460205078,
      "activations/layer4_attention_weight_max": 82.98722839355469,
      "activations/layer4_attention_weight_min": -82.1322250366211,
      "activations/layer5_attention_weight_max": 62.084495544433594,
      "activations/layer5_attention_weight_min": -70.42323303222656,
      "activations/layer6_attention_weight_max": 50.57615661621094,
      "activations/layer6_attention_weight_min": -50.88577651977539,
      "activations/layer7_attention_weight_max": 69.76998138427734,
      "activations/layer7_attention_weight_min": -66.6098861694336,
      "activations/layer8_attention_weight_max": 46.14521026611328,
      "activations/layer8_attention_weight_min": -49.29005813598633,
      "activations/layer9_attention_weight_max": 42.9737663269043,
      "activations/layer9_attention_weight_min": -45.57954025268555,
      "epoch": 21.58,
      "learning_rate": 1.0965151515151514e-05,
      "loss": 2.7097,
      "step": 371350
    },
    {
      "activations/layer0_attention_weight_max": 16.09008026123047,
      "activations/layer0_attention_weight_min": -13.923697471618652,
      "activations/layer10_attention_weight_max": 34.5411262512207,
      "activations/layer10_attention_weight_min": -35.07001495361328,
      "activations/layer11_attention_weight_max": 33.068809509277344,
      "activations/layer11_attention_weight_min": -34.1321907043457,
      "activations/layer12_attention_weight_max": 17.725122451782227,
      "activations/layer12_attention_weight_min": -26.536195755004883,
      "activations/layer13_attention_weight_max": 34.260223388671875,
      "activations/layer13_attention_weight_min": -27.359928131103516,
      "activations/layer14_attention_weight_max": 35.27082824707031,
      "activations/layer14_attention_weight_min": -31.244537353515625,
      "activations/layer15_attention_weight_max": 35.9869499206543,
      "activations/layer15_attention_weight_min": -30.572250366210938,
      "activations/layer16_attention_weight_max": 37.23025131225586,
      "activations/layer16_attention_weight_min": -34.04750061035156,
      "activations/layer17_attention_weight_max": 53.301429748535156,
      "activations/layer17_attention_weight_min": -46.118927001953125,
      "activations/layer18_attention_weight_max": 47.52708053588867,
      "activations/layer18_attention_weight_min": -42.97476577758789,
      "activations/layer19_attention_weight_max": 27.67063331604004,
      "activations/layer19_attention_weight_min": -23.828763961791992,
      "activations/layer1_attention_weight_max": 15.814596176147461,
      "activations/layer1_attention_weight_min": -14.477778434753418,
      "activations/layer20_attention_weight_max": 24.092998504638672,
      "activations/layer20_attention_weight_min": -21.26553726196289,
      "activations/layer21_attention_weight_max": 34.837303161621094,
      "activations/layer21_attention_weight_min": -21.934024810791016,
      "activations/layer22_attention_weight_max": 28.119953155517578,
      "activations/layer22_attention_weight_min": -26.404111862182617,
      "activations/layer23_attention_weight_max": 35.52497863769531,
      "activations/layer23_attention_weight_min": -23.909543991088867,
      "activations/layer2_attention_weight_max": 31.835529327392578,
      "activations/layer2_attention_weight_min": -28.698965072631836,
      "activations/layer3_attention_weight_max": 86.35847473144531,
      "activations/layer3_attention_weight_min": -86.78185272216797,
      "activations/layer4_attention_weight_max": 78.54097747802734,
      "activations/layer4_attention_weight_min": -79.78239440917969,
      "activations/layer5_attention_weight_max": 58.6905517578125,
      "activations/layer5_attention_weight_min": -69.39176177978516,
      "activations/layer6_attention_weight_max": 45.99148941040039,
      "activations/layer6_attention_weight_min": -50.7736930847168,
      "activations/layer7_attention_weight_max": 67.93334197998047,
      "activations/layer7_attention_weight_min": -67.38188934326172,
      "activations/layer8_attention_weight_max": 44.5598030090332,
      "activations/layer8_attention_weight_min": -51.67592239379883,
      "activations/layer9_attention_weight_max": 41.164913177490234,
      "activations/layer9_attention_weight_min": -44.9915885925293,
      "epoch": 21.58,
      "learning_rate": 1.0946212121212119e-05,
      "loss": 2.7134,
      "step": 371400
    },
    {
      "activations/layer0_attention_weight_max": 16.380878448486328,
      "activations/layer0_attention_weight_min": -14.19233226776123,
      "activations/layer10_attention_weight_max": 37.081214904785156,
      "activations/layer10_attention_weight_min": -36.58366775512695,
      "activations/layer11_attention_weight_max": 34.57884979248047,
      "activations/layer11_attention_weight_min": -35.937095642089844,
      "activations/layer12_attention_weight_max": 18.01474380493164,
      "activations/layer12_attention_weight_min": -23.709802627563477,
      "activations/layer13_attention_weight_max": 32.896480560302734,
      "activations/layer13_attention_weight_min": -29.402912139892578,
      "activations/layer14_attention_weight_max": 33.23750305175781,
      "activations/layer14_attention_weight_min": -31.998918533325195,
      "activations/layer15_attention_weight_max": 31.09276580810547,
      "activations/layer15_attention_weight_min": -32.71746826171875,
      "activations/layer16_attention_weight_max": 34.585079193115234,
      "activations/layer16_attention_weight_min": -33.7562255859375,
      "activations/layer17_attention_weight_max": 50.565345764160156,
      "activations/layer17_attention_weight_min": -46.803226470947266,
      "activations/layer18_attention_weight_max": 44.640933990478516,
      "activations/layer18_attention_weight_min": -39.657615661621094,
      "activations/layer19_attention_weight_max": 23.44451904296875,
      "activations/layer19_attention_weight_min": -23.808393478393555,
      "activations/layer1_attention_weight_max": 16.132490158081055,
      "activations/layer1_attention_weight_min": -13.56236743927002,
      "activations/layer20_attention_weight_max": 19.92758560180664,
      "activations/layer20_attention_weight_min": -21.449491500854492,
      "activations/layer21_attention_weight_max": 32.371826171875,
      "activations/layer21_attention_weight_min": -23.33177947998047,
      "activations/layer22_attention_weight_max": 27.394367218017578,
      "activations/layer22_attention_weight_min": -25.963634490966797,
      "activations/layer23_attention_weight_max": 35.111610412597656,
      "activations/layer23_attention_weight_min": -24.510679244995117,
      "activations/layer2_attention_weight_max": 32.714839935302734,
      "activations/layer2_attention_weight_min": -30.715431213378906,
      "activations/layer3_attention_weight_max": 89.64957427978516,
      "activations/layer3_attention_weight_min": -88.93168640136719,
      "activations/layer4_attention_weight_max": 86.1032943725586,
      "activations/layer4_attention_weight_min": -85.01663208007812,
      "activations/layer5_attention_weight_max": 61.940486907958984,
      "activations/layer5_attention_weight_min": -68.59303283691406,
      "activations/layer6_attention_weight_max": 48.46223449707031,
      "activations/layer6_attention_weight_min": -49.77421569824219,
      "activations/layer7_attention_weight_max": 66.07910919189453,
      "activations/layer7_attention_weight_min": -67.97040557861328,
      "activations/layer8_attention_weight_max": 47.191429138183594,
      "activations/layer8_attention_weight_min": -51.290733337402344,
      "activations/layer9_attention_weight_max": 46.60606002807617,
      "activations/layer9_attention_weight_min": -47.5745849609375,
      "epoch": 21.58,
      "learning_rate": 1.0927272727272725e-05,
      "loss": 2.7018,
      "step": 371450
    },
    {
      "activations/layer0_attention_weight_max": 15.639575004577637,
      "activations/layer0_attention_weight_min": -14.116284370422363,
      "activations/layer10_attention_weight_max": 39.430484771728516,
      "activations/layer10_attention_weight_min": -39.95042037963867,
      "activations/layer11_attention_weight_max": 42.70844650268555,
      "activations/layer11_attention_weight_min": -38.69209671020508,
      "activations/layer12_attention_weight_max": 17.413780212402344,
      "activations/layer12_attention_weight_min": -28.06202507019043,
      "activations/layer13_attention_weight_max": 31.6801700592041,
      "activations/layer13_attention_weight_min": -26.536701202392578,
      "activations/layer14_attention_weight_max": 34.393795013427734,
      "activations/layer14_attention_weight_min": -28.084457397460938,
      "activations/layer15_attention_weight_max": 29.91191864013672,
      "activations/layer15_attention_weight_min": -29.55084800720215,
      "activations/layer16_attention_weight_max": 31.211946487426758,
      "activations/layer16_attention_weight_min": -33.00029373168945,
      "activations/layer17_attention_weight_max": 50.08140563964844,
      "activations/layer17_attention_weight_min": -45.90564727783203,
      "activations/layer18_attention_weight_max": 44.966033935546875,
      "activations/layer18_attention_weight_min": -39.80694580078125,
      "activations/layer19_attention_weight_max": 23.84675407409668,
      "activations/layer19_attention_weight_min": -21.919721603393555,
      "activations/layer1_attention_weight_max": 15.819948196411133,
      "activations/layer1_attention_weight_min": -13.692068099975586,
      "activations/layer20_attention_weight_max": 19.26397705078125,
      "activations/layer20_attention_weight_min": -26.148719787597656,
      "activations/layer21_attention_weight_max": 32.2166862487793,
      "activations/layer21_attention_weight_min": -21.528579711914062,
      "activations/layer22_attention_weight_max": 28.461437225341797,
      "activations/layer22_attention_weight_min": -24.93100929260254,
      "activations/layer23_attention_weight_max": 34.46553421020508,
      "activations/layer23_attention_weight_min": -23.026317596435547,
      "activations/layer2_attention_weight_max": 33.061805725097656,
      "activations/layer2_attention_weight_min": -32.197776794433594,
      "activations/layer3_attention_weight_max": 93.60874938964844,
      "activations/layer3_attention_weight_min": -92.37220001220703,
      "activations/layer4_attention_weight_max": 88.56468963623047,
      "activations/layer4_attention_weight_min": -88.00251770019531,
      "activations/layer5_attention_weight_max": 67.14266967773438,
      "activations/layer5_attention_weight_min": -83.05290985107422,
      "activations/layer6_attention_weight_max": 48.34488296508789,
      "activations/layer6_attention_weight_min": -51.16203308105469,
      "activations/layer7_attention_weight_max": 76.9560546875,
      "activations/layer7_attention_weight_min": -72.59333801269531,
      "activations/layer8_attention_weight_max": 52.098384857177734,
      "activations/layer8_attention_weight_min": -51.30118942260742,
      "activations/layer9_attention_weight_max": 58.17314529418945,
      "activations/layer9_attention_weight_min": -50.861751556396484,
      "epoch": 21.59,
      "learning_rate": 1.0908333333333332e-05,
      "loss": 2.7053,
      "step": 371500
    },
    {
      "activations/layer0_attention_weight_max": 16.442358016967773,
      "activations/layer0_attention_weight_min": -14.541473388671875,
      "activations/layer10_attention_weight_max": 35.023048400878906,
      "activations/layer10_attention_weight_min": -35.49818801879883,
      "activations/layer11_attention_weight_max": 32.390140533447266,
      "activations/layer11_attention_weight_min": -34.115543365478516,
      "activations/layer12_attention_weight_max": 19.233200073242188,
      "activations/layer12_attention_weight_min": -28.488828659057617,
      "activations/layer13_attention_weight_max": 35.016136169433594,
      "activations/layer13_attention_weight_min": -24.97730827331543,
      "activations/layer14_attention_weight_max": 33.90013885498047,
      "activations/layer14_attention_weight_min": -30.181550979614258,
      "activations/layer15_attention_weight_max": 33.93901824951172,
      "activations/layer15_attention_weight_min": -30.327234268188477,
      "activations/layer16_attention_weight_max": 34.61729431152344,
      "activations/layer16_attention_weight_min": -31.046138763427734,
      "activations/layer17_attention_weight_max": 50.39867401123047,
      "activations/layer17_attention_weight_min": -44.06285095214844,
      "activations/layer18_attention_weight_max": 48.55293273925781,
      "activations/layer18_attention_weight_min": -38.18068313598633,
      "activations/layer19_attention_weight_max": 25.7576961517334,
      "activations/layer19_attention_weight_min": -21.272634506225586,
      "activations/layer1_attention_weight_max": 15.350469589233398,
      "activations/layer1_attention_weight_min": -14.264374732971191,
      "activations/layer20_attention_weight_max": 23.00316619873047,
      "activations/layer20_attention_weight_min": -21.274370193481445,
      "activations/layer21_attention_weight_max": 44.3922233581543,
      "activations/layer21_attention_weight_min": -22.26409149169922,
      "activations/layer22_attention_weight_max": 29.216550827026367,
      "activations/layer22_attention_weight_min": -25.03813934326172,
      "activations/layer23_attention_weight_max": 37.27101135253906,
      "activations/layer23_attention_weight_min": -22.950401306152344,
      "activations/layer2_attention_weight_max": 32.565025329589844,
      "activations/layer2_attention_weight_min": -30.96078872680664,
      "activations/layer3_attention_weight_max": 90.6534423828125,
      "activations/layer3_attention_weight_min": -90.34927368164062,
      "activations/layer4_attention_weight_max": 84.91917419433594,
      "activations/layer4_attention_weight_min": -82.62052154541016,
      "activations/layer5_attention_weight_max": 62.881317138671875,
      "activations/layer5_attention_weight_min": -68.40595245361328,
      "activations/layer6_attention_weight_max": 48.92751693725586,
      "activations/layer6_attention_weight_min": -50.439144134521484,
      "activations/layer7_attention_weight_max": 69.95320129394531,
      "activations/layer7_attention_weight_min": -70.9364013671875,
      "activations/layer8_attention_weight_max": 48.60622787475586,
      "activations/layer8_attention_weight_min": -50.53908920288086,
      "activations/layer9_attention_weight_max": 42.473819732666016,
      "activations/layer9_attention_weight_min": -47.44775390625,
      "epoch": 21.59,
      "learning_rate": 1.088939393939394e-05,
      "loss": 2.7108,
      "step": 371550
    },
    {
      "activations/layer0_attention_weight_max": 15.87707805633545,
      "activations/layer0_attention_weight_min": -14.155401229858398,
      "activations/layer10_attention_weight_max": 37.8085823059082,
      "activations/layer10_attention_weight_min": -40.21472930908203,
      "activations/layer11_attention_weight_max": 35.34743118286133,
      "activations/layer11_attention_weight_min": -37.50552749633789,
      "activations/layer12_attention_weight_max": 19.872913360595703,
      "activations/layer12_attention_weight_min": -23.322839736938477,
      "activations/layer13_attention_weight_max": 34.08723449707031,
      "activations/layer13_attention_weight_min": -28.634666442871094,
      "activations/layer14_attention_weight_max": 36.086448669433594,
      "activations/layer14_attention_weight_min": -30.79355239868164,
      "activations/layer15_attention_weight_max": 34.19234848022461,
      "activations/layer15_attention_weight_min": -30.283376693725586,
      "activations/layer16_attention_weight_max": 33.97105407714844,
      "activations/layer16_attention_weight_min": -32.56456756591797,
      "activations/layer17_attention_weight_max": 49.75322341918945,
      "activations/layer17_attention_weight_min": -45.66389846801758,
      "activations/layer18_attention_weight_max": 46.79625701904297,
      "activations/layer18_attention_weight_min": -43.43267059326172,
      "activations/layer19_attention_weight_max": 25.71811866760254,
      "activations/layer19_attention_weight_min": -24.63263511657715,
      "activations/layer1_attention_weight_max": 15.932780265808105,
      "activations/layer1_attention_weight_min": -16.257732391357422,
      "activations/layer20_attention_weight_max": 24.561460494995117,
      "activations/layer20_attention_weight_min": -24.429670333862305,
      "activations/layer21_attention_weight_max": 42.80061721801758,
      "activations/layer21_attention_weight_min": -24.514907836914062,
      "activations/layer22_attention_weight_max": 29.165281295776367,
      "activations/layer22_attention_weight_min": -25.28838539123535,
      "activations/layer23_attention_weight_max": 36.521400451660156,
      "activations/layer23_attention_weight_min": -23.584075927734375,
      "activations/layer2_attention_weight_max": 33.274105072021484,
      "activations/layer2_attention_weight_min": -31.613494873046875,
      "activations/layer3_attention_weight_max": 98.70567321777344,
      "activations/layer3_attention_weight_min": -95.08341979980469,
      "activations/layer4_attention_weight_max": 87.33956146240234,
      "activations/layer4_attention_weight_min": -85.81175231933594,
      "activations/layer5_attention_weight_max": 63.65312194824219,
      "activations/layer5_attention_weight_min": -69.14508056640625,
      "activations/layer6_attention_weight_max": 48.43943405151367,
      "activations/layer6_attention_weight_min": -49.77273941040039,
      "activations/layer7_attention_weight_max": 71.41908264160156,
      "activations/layer7_attention_weight_min": -65.19454956054688,
      "activations/layer8_attention_weight_max": 48.27497482299805,
      "activations/layer8_attention_weight_min": -50.28302001953125,
      "activations/layer9_attention_weight_max": 46.619773864746094,
      "activations/layer9_attention_weight_min": -46.17279815673828,
      "epoch": 21.59,
      "learning_rate": 1.0870454545454545e-05,
      "loss": 2.7045,
      "step": 371600
    },
    {
      "activations/layer0_attention_weight_max": 15.533159255981445,
      "activations/layer0_attention_weight_min": -14.360164642333984,
      "activations/layer10_attention_weight_max": 34.647743225097656,
      "activations/layer10_attention_weight_min": -36.676361083984375,
      "activations/layer11_attention_weight_max": 35.013832092285156,
      "activations/layer11_attention_weight_min": -35.65589904785156,
      "activations/layer12_attention_weight_max": 20.38694953918457,
      "activations/layer12_attention_weight_min": -26.906343460083008,
      "activations/layer13_attention_weight_max": 34.08374786376953,
      "activations/layer13_attention_weight_min": -26.565027236938477,
      "activations/layer14_attention_weight_max": 37.52063751220703,
      "activations/layer14_attention_weight_min": -31.061628341674805,
      "activations/layer15_attention_weight_max": 35.35805130004883,
      "activations/layer15_attention_weight_min": -31.81876564025879,
      "activations/layer16_attention_weight_max": 37.098087310791016,
      "activations/layer16_attention_weight_min": -33.06642532348633,
      "activations/layer17_attention_weight_max": 52.59075164794922,
      "activations/layer17_attention_weight_min": -48.37150573730469,
      "activations/layer18_attention_weight_max": 48.49608612060547,
      "activations/layer18_attention_weight_min": -41.17241668701172,
      "activations/layer19_attention_weight_max": 24.832796096801758,
      "activations/layer19_attention_weight_min": -23.664840698242188,
      "activations/layer1_attention_weight_max": 15.23344898223877,
      "activations/layer1_attention_weight_min": -14.25283432006836,
      "activations/layer20_attention_weight_max": 21.906864166259766,
      "activations/layer20_attention_weight_min": -21.899477005004883,
      "activations/layer21_attention_weight_max": 35.36767578125,
      "activations/layer21_attention_weight_min": -24.69496726989746,
      "activations/layer22_attention_weight_max": 31.519073486328125,
      "activations/layer22_attention_weight_min": -24.276811599731445,
      "activations/layer23_attention_weight_max": 37.75908660888672,
      "activations/layer23_attention_weight_min": -24.525985717773438,
      "activations/layer2_attention_weight_max": 30.523679733276367,
      "activations/layer2_attention_weight_min": -31.540607452392578,
      "activations/layer3_attention_weight_max": 88.97545623779297,
      "activations/layer3_attention_weight_min": -90.38850402832031,
      "activations/layer4_attention_weight_max": 85.90142822265625,
      "activations/layer4_attention_weight_min": -85.96609497070312,
      "activations/layer5_attention_weight_max": 65.2228775024414,
      "activations/layer5_attention_weight_min": -71.13754272460938,
      "activations/layer6_attention_weight_max": 51.18642044067383,
      "activations/layer6_attention_weight_min": -50.952632904052734,
      "activations/layer7_attention_weight_max": 66.8568115234375,
      "activations/layer7_attention_weight_min": -66.34403991699219,
      "activations/layer8_attention_weight_max": 48.73194122314453,
      "activations/layer8_attention_weight_min": -54.025630950927734,
      "activations/layer9_attention_weight_max": 44.42607879638672,
      "activations/layer9_attention_weight_min": -46.57199478149414,
      "epoch": 21.59,
      "learning_rate": 1.0851515151515152e-05,
      "loss": 2.7116,
      "step": 371650
    },
    {
      "activations/layer0_attention_weight_max": 16.429609298706055,
      "activations/layer0_attention_weight_min": -14.514827728271484,
      "activations/layer10_attention_weight_max": 33.463661193847656,
      "activations/layer10_attention_weight_min": -35.13862228393555,
      "activations/layer11_attention_weight_max": 33.352787017822266,
      "activations/layer11_attention_weight_min": -33.47027587890625,
      "activations/layer12_attention_weight_max": 18.08481216430664,
      "activations/layer12_attention_weight_min": -27.41389274597168,
      "activations/layer13_attention_weight_max": 30.57451820373535,
      "activations/layer13_attention_weight_min": -27.033138275146484,
      "activations/layer14_attention_weight_max": 34.44784927368164,
      "activations/layer14_attention_weight_min": -29.719194412231445,
      "activations/layer15_attention_weight_max": 33.45698165893555,
      "activations/layer15_attention_weight_min": -29.14816665649414,
      "activations/layer16_attention_weight_max": 33.568973541259766,
      "activations/layer16_attention_weight_min": -33.40264129638672,
      "activations/layer17_attention_weight_max": 50.9019660949707,
      "activations/layer17_attention_weight_min": -45.243202209472656,
      "activations/layer18_attention_weight_max": 46.605491638183594,
      "activations/layer18_attention_weight_min": -39.507015228271484,
      "activations/layer19_attention_weight_max": 25.97163963317871,
      "activations/layer19_attention_weight_min": -23.792417526245117,
      "activations/layer1_attention_weight_max": 16.05716896057129,
      "activations/layer1_attention_weight_min": -15.052306175231934,
      "activations/layer20_attention_weight_max": 22.5176944732666,
      "activations/layer20_attention_weight_min": -21.0268497467041,
      "activations/layer21_attention_weight_max": 34.26805877685547,
      "activations/layer21_attention_weight_min": -21.077241897583008,
      "activations/layer22_attention_weight_max": 32.68963623046875,
      "activations/layer22_attention_weight_min": -26.759010314941406,
      "activations/layer23_attention_weight_max": 36.502384185791016,
      "activations/layer23_attention_weight_min": -24.592117309570312,
      "activations/layer2_attention_weight_max": 31.613990783691406,
      "activations/layer2_attention_weight_min": -30.994924545288086,
      "activations/layer3_attention_weight_max": 91.96895599365234,
      "activations/layer3_attention_weight_min": -92.14900970458984,
      "activations/layer4_attention_weight_max": 85.21769714355469,
      "activations/layer4_attention_weight_min": -84.86798858642578,
      "activations/layer5_attention_weight_max": 60.76288604736328,
      "activations/layer5_attention_weight_min": -70.81086730957031,
      "activations/layer6_attention_weight_max": 48.3430290222168,
      "activations/layer6_attention_weight_min": -51.23463439941406,
      "activations/layer7_attention_weight_max": 69.17401885986328,
      "activations/layer7_attention_weight_min": -69.68533325195312,
      "activations/layer8_attention_weight_max": 44.40705108642578,
      "activations/layer8_attention_weight_min": -52.99076461791992,
      "activations/layer9_attention_weight_max": 49.22500991821289,
      "activations/layer9_attention_weight_min": -46.04659652709961,
      "epoch": 21.6,
      "learning_rate": 1.0832575757575757e-05,
      "loss": 2.7131,
      "step": 371700
    },
    {
      "activations/layer0_attention_weight_max": 17.394338607788086,
      "activations/layer0_attention_weight_min": -14.261332511901855,
      "activations/layer10_attention_weight_max": 38.51123809814453,
      "activations/layer10_attention_weight_min": -37.61062240600586,
      "activations/layer11_attention_weight_max": 36.672977447509766,
      "activations/layer11_attention_weight_min": -37.3157958984375,
      "activations/layer12_attention_weight_max": 19.296865463256836,
      "activations/layer12_attention_weight_min": -26.929481506347656,
      "activations/layer13_attention_weight_max": 34.58575439453125,
      "activations/layer13_attention_weight_min": -27.960243225097656,
      "activations/layer14_attention_weight_max": 36.704872131347656,
      "activations/layer14_attention_weight_min": -30.424348831176758,
      "activations/layer15_attention_weight_max": 35.08741760253906,
      "activations/layer15_attention_weight_min": -31.380714416503906,
      "activations/layer16_attention_weight_max": 35.32648468017578,
      "activations/layer16_attention_weight_min": -33.89646911621094,
      "activations/layer17_attention_weight_max": 55.55220413208008,
      "activations/layer17_attention_weight_min": -48.136234283447266,
      "activations/layer18_attention_weight_max": 52.204368591308594,
      "activations/layer18_attention_weight_min": -42.020694732666016,
      "activations/layer19_attention_weight_max": 24.36016082763672,
      "activations/layer19_attention_weight_min": -23.853069305419922,
      "activations/layer1_attention_weight_max": 15.50554084777832,
      "activations/layer1_attention_weight_min": -14.494497299194336,
      "activations/layer20_attention_weight_max": 24.421281814575195,
      "activations/layer20_attention_weight_min": -24.733043670654297,
      "activations/layer21_attention_weight_max": 34.38513946533203,
      "activations/layer21_attention_weight_min": -21.596044540405273,
      "activations/layer22_attention_weight_max": 28.80337905883789,
      "activations/layer22_attention_weight_min": -24.564804077148438,
      "activations/layer23_attention_weight_max": 36.79931640625,
      "activations/layer23_attention_weight_min": -26.37646484375,
      "activations/layer2_attention_weight_max": 31.614856719970703,
      "activations/layer2_attention_weight_min": -29.979650497436523,
      "activations/layer3_attention_weight_max": 91.05664825439453,
      "activations/layer3_attention_weight_min": -88.90752410888672,
      "activations/layer4_attention_weight_max": 85.7728500366211,
      "activations/layer4_attention_weight_min": -85.32302856445312,
      "activations/layer5_attention_weight_max": 63.38686752319336,
      "activations/layer5_attention_weight_min": -73.92984771728516,
      "activations/layer6_attention_weight_max": 49.55192947387695,
      "activations/layer6_attention_weight_min": -50.22888946533203,
      "activations/layer7_attention_weight_max": 67.9365234375,
      "activations/layer7_attention_weight_min": -68.28688049316406,
      "activations/layer8_attention_weight_max": 49.64358901977539,
      "activations/layer8_attention_weight_min": -50.662330627441406,
      "activations/layer9_attention_weight_max": 47.14926528930664,
      "activations/layer9_attention_weight_min": -48.20359420776367,
      "epoch": 21.6,
      "learning_rate": 1.0813636363636363e-05,
      "loss": 2.7028,
      "step": 371750
    },
    {
      "activations/layer0_attention_weight_max": 15.715063095092773,
      "activations/layer0_attention_weight_min": -14.115544319152832,
      "activations/layer10_attention_weight_max": 34.88690948486328,
      "activations/layer10_attention_weight_min": -35.738441467285156,
      "activations/layer11_attention_weight_max": 32.752899169921875,
      "activations/layer11_attention_weight_min": -35.108970642089844,
      "activations/layer12_attention_weight_max": 18.888294219970703,
      "activations/layer12_attention_weight_min": -21.81764030456543,
      "activations/layer13_attention_weight_max": 38.08247375488281,
      "activations/layer13_attention_weight_min": -27.945573806762695,
      "activations/layer14_attention_weight_max": 34.86695098876953,
      "activations/layer14_attention_weight_min": -28.714548110961914,
      "activations/layer15_attention_weight_max": 32.07546615600586,
      "activations/layer15_attention_weight_min": -29.419193267822266,
      "activations/layer16_attention_weight_max": 35.025272369384766,
      "activations/layer16_attention_weight_min": -32.120967864990234,
      "activations/layer17_attention_weight_max": 50.6697998046875,
      "activations/layer17_attention_weight_min": -44.73578643798828,
      "activations/layer18_attention_weight_max": 46.69756317138672,
      "activations/layer18_attention_weight_min": -40.0811653137207,
      "activations/layer19_attention_weight_max": 26.827680587768555,
      "activations/layer19_attention_weight_min": -22.30307960510254,
      "activations/layer1_attention_weight_max": 15.254435539245605,
      "activations/layer1_attention_weight_min": -13.818228721618652,
      "activations/layer20_attention_weight_max": 21.33330726623535,
      "activations/layer20_attention_weight_min": -20.42277717590332,
      "activations/layer21_attention_weight_max": 34.413917541503906,
      "activations/layer21_attention_weight_min": -23.37962532043457,
      "activations/layer22_attention_weight_max": 27.17103385925293,
      "activations/layer22_attention_weight_min": -25.88739585876465,
      "activations/layer23_attention_weight_max": 35.84503173828125,
      "activations/layer23_attention_weight_min": -21.931690216064453,
      "activations/layer2_attention_weight_max": 31.12531089782715,
      "activations/layer2_attention_weight_min": -30.235523223876953,
      "activations/layer3_attention_weight_max": 88.3713150024414,
      "activations/layer3_attention_weight_min": -94.03424072265625,
      "activations/layer4_attention_weight_max": 85.5167236328125,
      "activations/layer4_attention_weight_min": -87.09270477294922,
      "activations/layer5_attention_weight_max": 64.56678771972656,
      "activations/layer5_attention_weight_min": -77.0313720703125,
      "activations/layer6_attention_weight_max": 49.38298797607422,
      "activations/layer6_attention_weight_min": -49.190128326416016,
      "activations/layer7_attention_weight_max": 69.15241241455078,
      "activations/layer7_attention_weight_min": -68.88296508789062,
      "activations/layer8_attention_weight_max": 50.44135665893555,
      "activations/layer8_attention_weight_min": -51.124481201171875,
      "activations/layer9_attention_weight_max": 44.327022552490234,
      "activations/layer9_attention_weight_min": -44.606727600097656,
      "epoch": 21.6,
      "learning_rate": 1.079469696969697e-05,
      "loss": 2.7088,
      "step": 371800
    },
    {
      "activations/layer0_attention_weight_max": 15.982897758483887,
      "activations/layer0_attention_weight_min": -14.43048095703125,
      "activations/layer10_attention_weight_max": 36.28184127807617,
      "activations/layer10_attention_weight_min": -35.2913818359375,
      "activations/layer11_attention_weight_max": 33.338260650634766,
      "activations/layer11_attention_weight_min": -33.79823303222656,
      "activations/layer12_attention_weight_max": 18.761146545410156,
      "activations/layer12_attention_weight_min": -28.372333526611328,
      "activations/layer13_attention_weight_max": 36.49946975708008,
      "activations/layer13_attention_weight_min": -29.24728775024414,
      "activations/layer14_attention_weight_max": 36.108795166015625,
      "activations/layer14_attention_weight_min": -30.2331485748291,
      "activations/layer15_attention_weight_max": 33.90908432006836,
      "activations/layer15_attention_weight_min": -31.417707443237305,
      "activations/layer16_attention_weight_max": 33.89754104614258,
      "activations/layer16_attention_weight_min": -33.00849533081055,
      "activations/layer17_attention_weight_max": 51.62794876098633,
      "activations/layer17_attention_weight_min": -48.731266021728516,
      "activations/layer18_attention_weight_max": 46.78806686401367,
      "activations/layer18_attention_weight_min": -42.269752502441406,
      "activations/layer19_attention_weight_max": 23.453214645385742,
      "activations/layer19_attention_weight_min": -22.23075294494629,
      "activations/layer1_attention_weight_max": 14.847225189208984,
      "activations/layer1_attention_weight_min": -14.544103622436523,
      "activations/layer20_attention_weight_max": 22.47264289855957,
      "activations/layer20_attention_weight_min": -21.722423553466797,
      "activations/layer21_attention_weight_max": 32.81656265258789,
      "activations/layer21_attention_weight_min": -20.64422035217285,
      "activations/layer22_attention_weight_max": 28.0152645111084,
      "activations/layer22_attention_weight_min": -26.274940490722656,
      "activations/layer23_attention_weight_max": 35.03704071044922,
      "activations/layer23_attention_weight_min": -23.552167892456055,
      "activations/layer2_attention_weight_max": 31.790212631225586,
      "activations/layer2_attention_weight_min": -31.31031036376953,
      "activations/layer3_attention_weight_max": 87.73300170898438,
      "activations/layer3_attention_weight_min": -87.08665466308594,
      "activations/layer4_attention_weight_max": 83.21944427490234,
      "activations/layer4_attention_weight_min": -83.2683334350586,
      "activations/layer5_attention_weight_max": 62.195960998535156,
      "activations/layer5_attention_weight_min": -74.6483383178711,
      "activations/layer6_attention_weight_max": 48.74140167236328,
      "activations/layer6_attention_weight_min": -49.1522331237793,
      "activations/layer7_attention_weight_max": 71.88758087158203,
      "activations/layer7_attention_weight_min": -66.6196517944336,
      "activations/layer8_attention_weight_max": 45.13325881958008,
      "activations/layer8_attention_weight_min": -49.83671951293945,
      "activations/layer9_attention_weight_max": 46.247352600097656,
      "activations/layer9_attention_weight_min": -47.00136947631836,
      "epoch": 21.61,
      "learning_rate": 1.0775757575757575e-05,
      "loss": 2.7186,
      "step": 371850
    },
    {
      "activations/layer0_attention_weight_max": 16.118005752563477,
      "activations/layer0_attention_weight_min": -14.400504112243652,
      "activations/layer10_attention_weight_max": 36.18610763549805,
      "activations/layer10_attention_weight_min": -35.76408767700195,
      "activations/layer11_attention_weight_max": 34.841373443603516,
      "activations/layer11_attention_weight_min": -34.09428024291992,
      "activations/layer12_attention_weight_max": 18.60659408569336,
      "activations/layer12_attention_weight_min": -23.786792755126953,
      "activations/layer13_attention_weight_max": 38.77262878417969,
      "activations/layer13_attention_weight_min": -28.31090545654297,
      "activations/layer14_attention_weight_max": 42.19649124145508,
      "activations/layer14_attention_weight_min": -29.828216552734375,
      "activations/layer15_attention_weight_max": 33.730010986328125,
      "activations/layer15_attention_weight_min": -29.877010345458984,
      "activations/layer16_attention_weight_max": 36.332340240478516,
      "activations/layer16_attention_weight_min": -33.88473129272461,
      "activations/layer17_attention_weight_max": 55.10149002075195,
      "activations/layer17_attention_weight_min": -49.02290725708008,
      "activations/layer18_attention_weight_max": 53.348548889160156,
      "activations/layer18_attention_weight_min": -43.527931213378906,
      "activations/layer19_attention_weight_max": 27.51822853088379,
      "activations/layer19_attention_weight_min": -23.40597915649414,
      "activations/layer1_attention_weight_max": 17.190736770629883,
      "activations/layer1_attention_weight_min": -16.06935691833496,
      "activations/layer20_attention_weight_max": 26.269180297851562,
      "activations/layer20_attention_weight_min": -22.493274688720703,
      "activations/layer21_attention_weight_max": 39.54755783081055,
      "activations/layer21_attention_weight_min": -23.611228942871094,
      "activations/layer22_attention_weight_max": 29.648714065551758,
      "activations/layer22_attention_weight_min": -26.741168975830078,
      "activations/layer23_attention_weight_max": 37.46741485595703,
      "activations/layer23_attention_weight_min": -23.78334617614746,
      "activations/layer2_attention_weight_max": 31.837818145751953,
      "activations/layer2_attention_weight_min": -29.749969482421875,
      "activations/layer3_attention_weight_max": 87.88443756103516,
      "activations/layer3_attention_weight_min": -90.02833557128906,
      "activations/layer4_attention_weight_max": 81.63396453857422,
      "activations/layer4_attention_weight_min": -81.92254638671875,
      "activations/layer5_attention_weight_max": 59.46497344970703,
      "activations/layer5_attention_weight_min": -74.14518737792969,
      "activations/layer6_attention_weight_max": 47.73221969604492,
      "activations/layer6_attention_weight_min": -47.09980010986328,
      "activations/layer7_attention_weight_max": 67.56253814697266,
      "activations/layer7_attention_weight_min": -63.82065200805664,
      "activations/layer8_attention_weight_max": 46.45365905761719,
      "activations/layer8_attention_weight_min": -49.43659591674805,
      "activations/layer9_attention_weight_max": 48.11977005004883,
      "activations/layer9_attention_weight_min": -45.83944320678711,
      "epoch": 21.61,
      "learning_rate": 1.0756818181818181e-05,
      "loss": 2.7098,
      "step": 371900
    },
    {
      "activations/layer0_attention_weight_max": 16.388694763183594,
      "activations/layer0_attention_weight_min": -14.547858238220215,
      "activations/layer10_attention_weight_max": 37.562618255615234,
      "activations/layer10_attention_weight_min": -38.751834869384766,
      "activations/layer11_attention_weight_max": 38.457786560058594,
      "activations/layer11_attention_weight_min": -36.75666427612305,
      "activations/layer12_attention_weight_max": 19.4599552154541,
      "activations/layer12_attention_weight_min": -32.17915725708008,
      "activations/layer13_attention_weight_max": 37.866737365722656,
      "activations/layer13_attention_weight_min": -26.203256607055664,
      "activations/layer14_attention_weight_max": 35.781856536865234,
      "activations/layer14_attention_weight_min": -28.962841033935547,
      "activations/layer15_attention_weight_max": 34.57854461669922,
      "activations/layer15_attention_weight_min": -29.69210433959961,
      "activations/layer16_attention_weight_max": 33.451576232910156,
      "activations/layer16_attention_weight_min": -30.60732078552246,
      "activations/layer17_attention_weight_max": 54.682960510253906,
      "activations/layer17_attention_weight_min": -48.23126220703125,
      "activations/layer18_attention_weight_max": 48.709327697753906,
      "activations/layer18_attention_weight_min": -40.928504943847656,
      "activations/layer19_attention_weight_max": 25.568889617919922,
      "activations/layer19_attention_weight_min": -21.833127975463867,
      "activations/layer1_attention_weight_max": 15.91275405883789,
      "activations/layer1_attention_weight_min": -16.261106491088867,
      "activations/layer20_attention_weight_max": 26.373497009277344,
      "activations/layer20_attention_weight_min": -20.18987274169922,
      "activations/layer21_attention_weight_max": 34.265037536621094,
      "activations/layer21_attention_weight_min": -20.593896865844727,
      "activations/layer22_attention_weight_max": 28.400333404541016,
      "activations/layer22_attention_weight_min": -22.6253662109375,
      "activations/layer23_attention_weight_max": 35.106101989746094,
      "activations/layer23_attention_weight_min": -23.763853073120117,
      "activations/layer2_attention_weight_max": 32.584434509277344,
      "activations/layer2_attention_weight_min": -30.785215377807617,
      "activations/layer3_attention_weight_max": 89.51222229003906,
      "activations/layer3_attention_weight_min": -90.58019256591797,
      "activations/layer4_attention_weight_max": 84.0511703491211,
      "activations/layer4_attention_weight_min": -82.8692626953125,
      "activations/layer5_attention_weight_max": 61.71397018432617,
      "activations/layer5_attention_weight_min": -73.04595184326172,
      "activations/layer6_attention_weight_max": 52.65039825439453,
      "activations/layer6_attention_weight_min": -48.0700569152832,
      "activations/layer7_attention_weight_max": 68.80240631103516,
      "activations/layer7_attention_weight_min": -66.11988067626953,
      "activations/layer8_attention_weight_max": 50.04629898071289,
      "activations/layer8_attention_weight_min": -52.56087112426758,
      "activations/layer9_attention_weight_max": 49.295074462890625,
      "activations/layer9_attention_weight_min": -46.356441497802734,
      "epoch": 21.61,
      "learning_rate": 1.0737878787878788e-05,
      "loss": 2.7176,
      "step": 371950
    },
    {
      "activations/layer0_attention_weight_max": 16.55498695373535,
      "activations/layer0_attention_weight_min": -14.249861717224121,
      "activations/layer10_attention_weight_max": 37.98810577392578,
      "activations/layer10_attention_weight_min": -40.89064407348633,
      "activations/layer11_attention_weight_max": 33.82106018066406,
      "activations/layer11_attention_weight_min": -39.98411560058594,
      "activations/layer12_attention_weight_max": 19.409982681274414,
      "activations/layer12_attention_weight_min": -23.544612884521484,
      "activations/layer13_attention_weight_max": 33.6601448059082,
      "activations/layer13_attention_weight_min": -27.680828094482422,
      "activations/layer14_attention_weight_max": 34.92667007446289,
      "activations/layer14_attention_weight_min": -32.6270866394043,
      "activations/layer15_attention_weight_max": 32.478824615478516,
      "activations/layer15_attention_weight_min": -31.672847747802734,
      "activations/layer16_attention_weight_max": 36.33065414428711,
      "activations/layer16_attention_weight_min": -33.355804443359375,
      "activations/layer17_attention_weight_max": 53.29350280761719,
      "activations/layer17_attention_weight_min": -47.84868621826172,
      "activations/layer18_attention_weight_max": 47.82902908325195,
      "activations/layer18_attention_weight_min": -43.158138275146484,
      "activations/layer19_attention_weight_max": 27.048521041870117,
      "activations/layer19_attention_weight_min": -25.424232482910156,
      "activations/layer1_attention_weight_max": 14.452536582946777,
      "activations/layer1_attention_weight_min": -14.001273155212402,
      "activations/layer20_attention_weight_max": 24.745515823364258,
      "activations/layer20_attention_weight_min": -24.591691970825195,
      "activations/layer21_attention_weight_max": 36.31964874267578,
      "activations/layer21_attention_weight_min": -21.733871459960938,
      "activations/layer22_attention_weight_max": 31.28620147705078,
      "activations/layer22_attention_weight_min": -26.723203659057617,
      "activations/layer23_attention_weight_max": 38.22637176513672,
      "activations/layer23_attention_weight_min": -24.092853546142578,
      "activations/layer2_attention_weight_max": 30.53778648376465,
      "activations/layer2_attention_weight_min": -29.81142807006836,
      "activations/layer3_attention_weight_max": 90.24943542480469,
      "activations/layer3_attention_weight_min": -90.2649917602539,
      "activations/layer4_attention_weight_max": 85.57210540771484,
      "activations/layer4_attention_weight_min": -88.56621551513672,
      "activations/layer5_attention_weight_max": 59.71806335449219,
      "activations/layer5_attention_weight_min": -73.58187103271484,
      "activations/layer6_attention_weight_max": 48.65184783935547,
      "activations/layer6_attention_weight_min": -51.772274017333984,
      "activations/layer7_attention_weight_max": 66.13633728027344,
      "activations/layer7_attention_weight_min": -71.35346221923828,
      "activations/layer8_attention_weight_max": 46.46125411987305,
      "activations/layer8_attention_weight_min": -51.30668258666992,
      "activations/layer9_attention_weight_max": 43.340999603271484,
      "activations/layer9_attention_weight_min": -48.263916015625,
      "epoch": 21.62,
      "learning_rate": 1.0718939393939393e-05,
      "loss": 2.712,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_loss": 2.671875,
      "eval_runtime": 8.5018,
      "eval_samples_per_second": 505.067,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_openwebtext_loss": 2.671875,
      "eval_openwebtext_ppl": 14.467069533669397,
      "eval_openwebtext_runtime": 8.5018,
      "eval_openwebtext_samples_per_second": 505.067,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9547,
      "eval_wikitext_samples_per_second": 233.279,
      "step": 372000
    },
    {
      "epoch": 21.62,
      "eval_lambada_loss": 2.41015625,
      "eval_lambada_ppl": 11.135700962413912,
      "eval_lambada_runtime": 9.5565,
      "eval_lambada_samples_per_second": 509.495,
      "step": 372000
    },
    {
      "activations/layer0_attention_weight_max": 16.765493392944336,
      "activations/layer0_attention_weight_min": -14.333450317382812,
      "activations/layer10_attention_weight_max": 36.2184944152832,
      "activations/layer10_attention_weight_min": -36.49383544921875,
      "activations/layer11_attention_weight_max": 33.23212432861328,
      "activations/layer11_attention_weight_min": -35.98487091064453,
      "activations/layer12_attention_weight_max": 20.450122833251953,
      "activations/layer12_attention_weight_min": -26.306398391723633,
      "activations/layer13_attention_weight_max": 32.080413818359375,
      "activations/layer13_attention_weight_min": -28.32330894470215,
      "activations/layer14_attention_weight_max": 35.959224700927734,
      "activations/layer14_attention_weight_min": -30.098426818847656,
      "activations/layer15_attention_weight_max": 31.93001365661621,
      "activations/layer15_attention_weight_min": -31.835485458374023,
      "activations/layer16_attention_weight_max": 33.80833435058594,
      "activations/layer16_attention_weight_min": -33.532283782958984,
      "activations/layer17_attention_weight_max": 45.873374938964844,
      "activations/layer17_attention_weight_min": -42.883609771728516,
      "activations/layer18_attention_weight_max": 42.81658172607422,
      "activations/layer18_attention_weight_min": -40.760719299316406,
      "activations/layer19_attention_weight_max": 23.820531845092773,
      "activations/layer19_attention_weight_min": -22.75294303894043,
      "activations/layer1_attention_weight_max": 15.56796646118164,
      "activations/layer1_attention_weight_min": -16.035425186157227,
      "activations/layer20_attention_weight_max": 20.35245132446289,
      "activations/layer20_attention_weight_min": -22.558927536010742,
      "activations/layer21_attention_weight_max": 32.73918533325195,
      "activations/layer21_attention_weight_min": -24.851125717163086,
      "activations/layer22_attention_weight_max": 26.65626335144043,
      "activations/layer22_attention_weight_min": -26.89661407470703,
      "activations/layer23_attention_weight_max": 37.528892517089844,
      "activations/layer23_attention_weight_min": -24.6453857421875,
      "activations/layer2_attention_weight_max": 31.401790618896484,
      "activations/layer2_attention_weight_min": -30.78911590576172,
      "activations/layer3_attention_weight_max": 90.60153198242188,
      "activations/layer3_attention_weight_min": -94.8168716430664,
      "activations/layer4_attention_weight_max": 87.86934661865234,
      "activations/layer4_attention_weight_min": -88.00149536132812,
      "activations/layer5_attention_weight_max": 61.420345306396484,
      "activations/layer5_attention_weight_min": -70.28239440917969,
      "activations/layer6_attention_weight_max": 53.056114196777344,
      "activations/layer6_attention_weight_min": -51.53809356689453,
      "activations/layer7_attention_weight_max": 69.23416900634766,
      "activations/layer7_attention_weight_min": -72.27919006347656,
      "activations/layer8_attention_weight_max": 49.9857177734375,
      "activations/layer8_attention_weight_min": -54.84419631958008,
      "activations/layer9_attention_weight_max": 47.25309371948242,
      "activations/layer9_attention_weight_min": -49.43427658081055,
      "epoch": 21.62,
      "learning_rate": 1.07e-05,
      "loss": 2.7101,
      "step": 372050
    },
    {
      "activations/layer0_attention_weight_max": 15.458686828613281,
      "activations/layer0_attention_weight_min": -14.07204818725586,
      "activations/layer10_attention_weight_max": 49.96391296386719,
      "activations/layer10_attention_weight_min": -46.37031555175781,
      "activations/layer11_attention_weight_max": 46.120174407958984,
      "activations/layer11_attention_weight_min": -44.74921798706055,
      "activations/layer12_attention_weight_max": 19.504474639892578,
      "activations/layer12_attention_weight_min": -28.197254180908203,
      "activations/layer13_attention_weight_max": 41.66081619262695,
      "activations/layer13_attention_weight_min": -32.766876220703125,
      "activations/layer14_attention_weight_max": 46.63847732543945,
      "activations/layer14_attention_weight_min": -32.72696304321289,
      "activations/layer15_attention_weight_max": 47.91025924682617,
      "activations/layer15_attention_weight_min": -32.559593200683594,
      "activations/layer16_attention_weight_max": 35.02260971069336,
      "activations/layer16_attention_weight_min": -32.46101760864258,
      "activations/layer17_attention_weight_max": 53.36556625366211,
      "activations/layer17_attention_weight_min": -46.01267623901367,
      "activations/layer18_attention_weight_max": 49.849910736083984,
      "activations/layer18_attention_weight_min": -41.789451599121094,
      "activations/layer19_attention_weight_max": 27.153589248657227,
      "activations/layer19_attention_weight_min": -22.456623077392578,
      "activations/layer1_attention_weight_max": 15.671893119812012,
      "activations/layer1_attention_weight_min": -15.416482925415039,
      "activations/layer20_attention_weight_max": 24.51729393005371,
      "activations/layer20_attention_weight_min": -20.767173767089844,
      "activations/layer21_attention_weight_max": 37.54386520385742,
      "activations/layer21_attention_weight_min": -21.825653076171875,
      "activations/layer22_attention_weight_max": 29.28909683227539,
      "activations/layer22_attention_weight_min": -23.97058868408203,
      "activations/layer23_attention_weight_max": 35.4393310546875,
      "activations/layer23_attention_weight_min": -22.82246971130371,
      "activations/layer2_attention_weight_max": 34.39690399169922,
      "activations/layer2_attention_weight_min": -31.124343872070312,
      "activations/layer3_attention_weight_max": 96.6340560913086,
      "activations/layer3_attention_weight_min": -92.61467742919922,
      "activations/layer4_attention_weight_max": 93.8205337524414,
      "activations/layer4_attention_weight_min": -86.17252349853516,
      "activations/layer5_attention_weight_max": 64.80960083007812,
      "activations/layer5_attention_weight_min": -70.595458984375,
      "activations/layer6_attention_weight_max": 56.38158416748047,
      "activations/layer6_attention_weight_min": -53.458560943603516,
      "activations/layer7_attention_weight_max": 79.48189544677734,
      "activations/layer7_attention_weight_min": -76.8531723022461,
      "activations/layer8_attention_weight_max": 63.40675354003906,
      "activations/layer8_attention_weight_min": -59.20693588256836,
      "activations/layer9_attention_weight_max": 60.623085021972656,
      "activations/layer9_attention_weight_min": -54.342586517333984,
      "epoch": 21.62,
      "learning_rate": 1.0681060606060604e-05,
      "loss": 2.7167,
      "step": 372100
    },
    {
      "activations/layer0_attention_weight_max": 16.33670425415039,
      "activations/layer0_attention_weight_min": -14.6929349899292,
      "activations/layer10_attention_weight_max": 36.32915115356445,
      "activations/layer10_attention_weight_min": -37.945220947265625,
      "activations/layer11_attention_weight_max": 34.27873611450195,
      "activations/layer11_attention_weight_min": -36.4504508972168,
      "activations/layer12_attention_weight_max": 20.706623077392578,
      "activations/layer12_attention_weight_min": -27.874771118164062,
      "activations/layer13_attention_weight_max": 37.90644454956055,
      "activations/layer13_attention_weight_min": -28.90031623840332,
      "activations/layer14_attention_weight_max": 39.30722427368164,
      "activations/layer14_attention_weight_min": -33.183387756347656,
      "activations/layer15_attention_weight_max": 38.087562561035156,
      "activations/layer15_attention_weight_min": -32.50270462036133,
      "activations/layer16_attention_weight_max": 35.81601333618164,
      "activations/layer16_attention_weight_min": -34.75413513183594,
      "activations/layer17_attention_weight_max": 53.78250503540039,
      "activations/layer17_attention_weight_min": -49.34880065917969,
      "activations/layer18_attention_weight_max": 50.18914031982422,
      "activations/layer18_attention_weight_min": -41.92253875732422,
      "activations/layer19_attention_weight_max": 27.945377349853516,
      "activations/layer19_attention_weight_min": -24.56733512878418,
      "activations/layer1_attention_weight_max": 15.152389526367188,
      "activations/layer1_attention_weight_min": -14.77529239654541,
      "activations/layer20_attention_weight_max": 28.044464111328125,
      "activations/layer20_attention_weight_min": -21.288185119628906,
      "activations/layer21_attention_weight_max": 43.567298889160156,
      "activations/layer21_attention_weight_min": -19.79033088684082,
      "activations/layer22_attention_weight_max": 36.018611907958984,
      "activations/layer22_attention_weight_min": -25.3824462890625,
      "activations/layer23_attention_weight_max": 46.01374816894531,
      "activations/layer23_attention_weight_min": -23.118282318115234,
      "activations/layer2_attention_weight_max": 31.327505111694336,
      "activations/layer2_attention_weight_min": -31.300992965698242,
      "activations/layer3_attention_weight_max": 88.96478271484375,
      "activations/layer3_attention_weight_min": -87.69913482666016,
      "activations/layer4_attention_weight_max": 84.03211212158203,
      "activations/layer4_attention_weight_min": -83.8385238647461,
      "activations/layer5_attention_weight_max": 63.424842834472656,
      "activations/layer5_attention_weight_min": -76.4317626953125,
      "activations/layer6_attention_weight_max": 49.854164123535156,
      "activations/layer6_attention_weight_min": -50.562232971191406,
      "activations/layer7_attention_weight_max": 69.71472930908203,
      "activations/layer7_attention_weight_min": -72.41524505615234,
      "activations/layer8_attention_weight_max": 47.172645568847656,
      "activations/layer8_attention_weight_min": -50.19697570800781,
      "activations/layer9_attention_weight_max": 45.47307586669922,
      "activations/layer9_attention_weight_min": -47.43517303466797,
      "epoch": 21.62,
      "learning_rate": 1.066212121212121e-05,
      "loss": 2.7214,
      "step": 372150
    },
    {
      "activations/layer0_attention_weight_max": 16.12409210205078,
      "activations/layer0_attention_weight_min": -14.011269569396973,
      "activations/layer10_attention_weight_max": 36.38383102416992,
      "activations/layer10_attention_weight_min": -36.82172393798828,
      "activations/layer11_attention_weight_max": 31.563919067382812,
      "activations/layer11_attention_weight_min": -35.05241775512695,
      "activations/layer12_attention_weight_max": 19.506954193115234,
      "activations/layer12_attention_weight_min": -26.267086029052734,
      "activations/layer13_attention_weight_max": 32.660125732421875,
      "activations/layer13_attention_weight_min": -26.97606086730957,
      "activations/layer14_attention_weight_max": 34.932945251464844,
      "activations/layer14_attention_weight_min": -29.765350341796875,
      "activations/layer15_attention_weight_max": 34.46590805053711,
      "activations/layer15_attention_weight_min": -29.48049545288086,
      "activations/layer16_attention_weight_max": 33.8232536315918,
      "activations/layer16_attention_weight_min": -32.05815124511719,
      "activations/layer17_attention_weight_max": 51.62601852416992,
      "activations/layer17_attention_weight_min": -43.731075286865234,
      "activations/layer18_attention_weight_max": 48.328704833984375,
      "activations/layer18_attention_weight_min": -39.0416374206543,
      "activations/layer19_attention_weight_max": 26.1424503326416,
      "activations/layer19_attention_weight_min": -22.71806526184082,
      "activations/layer1_attention_weight_max": 15.647820472717285,
      "activations/layer1_attention_weight_min": -13.694448471069336,
      "activations/layer20_attention_weight_max": 23.653467178344727,
      "activations/layer20_attention_weight_min": -22.179241180419922,
      "activations/layer21_attention_weight_max": 37.00914764404297,
      "activations/layer21_attention_weight_min": -23.3101863861084,
      "activations/layer22_attention_weight_max": 27.715866088867188,
      "activations/layer22_attention_weight_min": -24.47701644897461,
      "activations/layer23_attention_weight_max": 36.11869812011719,
      "activations/layer23_attention_weight_min": -23.1671199798584,
      "activations/layer2_attention_weight_max": 31.492095947265625,
      "activations/layer2_attention_weight_min": -31.050222396850586,
      "activations/layer3_attention_weight_max": 89.14045715332031,
      "activations/layer3_attention_weight_min": -93.76403045654297,
      "activations/layer4_attention_weight_max": 84.41320037841797,
      "activations/layer4_attention_weight_min": -84.59307098388672,
      "activations/layer5_attention_weight_max": 61.405635833740234,
      "activations/layer5_attention_weight_min": -75.27635955810547,
      "activations/layer6_attention_weight_max": 48.406978607177734,
      "activations/layer6_attention_weight_min": -49.55032730102539,
      "activations/layer7_attention_weight_max": 67.7645263671875,
      "activations/layer7_attention_weight_min": -67.28092956542969,
      "activations/layer8_attention_weight_max": 45.59620666503906,
      "activations/layer8_attention_weight_min": -50.79535675048828,
      "activations/layer9_attention_weight_max": 42.0194206237793,
      "activations/layer9_attention_weight_min": -46.504634857177734,
      "epoch": 21.63,
      "learning_rate": 1.0643181818181817e-05,
      "loss": 2.7232,
      "step": 372200
    },
    {
      "activations/layer0_attention_weight_max": 16.674556732177734,
      "activations/layer0_attention_weight_min": -14.523728370666504,
      "activations/layer10_attention_weight_max": 36.83513259887695,
      "activations/layer10_attention_weight_min": -36.07501983642578,
      "activations/layer11_attention_weight_max": 33.34165954589844,
      "activations/layer11_attention_weight_min": -36.93291473388672,
      "activations/layer12_attention_weight_max": 19.65519142150879,
      "activations/layer12_attention_weight_min": -23.011926651000977,
      "activations/layer13_attention_weight_max": 34.06745147705078,
      "activations/layer13_attention_weight_min": -30.5858154296875,
      "activations/layer14_attention_weight_max": 37.463905334472656,
      "activations/layer14_attention_weight_min": -33.606109619140625,
      "activations/layer15_attention_weight_max": 35.75404357910156,
      "activations/layer15_attention_weight_min": -32.82741165161133,
      "activations/layer16_attention_weight_max": 38.0121955871582,
      "activations/layer16_attention_weight_min": -34.42508316040039,
      "activations/layer17_attention_weight_max": 55.869606018066406,
      "activations/layer17_attention_weight_min": -49.562286376953125,
      "activations/layer18_attention_weight_max": 55.28960418701172,
      "activations/layer18_attention_weight_min": -44.048667907714844,
      "activations/layer19_attention_weight_max": 27.308881759643555,
      "activations/layer19_attention_weight_min": -23.414634704589844,
      "activations/layer1_attention_weight_max": 15.665682792663574,
      "activations/layer1_attention_weight_min": -13.264117240905762,
      "activations/layer20_attention_weight_max": 24.92906379699707,
      "activations/layer20_attention_weight_min": -21.439584732055664,
      "activations/layer21_attention_weight_max": 50.00525665283203,
      "activations/layer21_attention_weight_min": -23.551603317260742,
      "activations/layer22_attention_weight_max": 32.4746208190918,
      "activations/layer22_attention_weight_min": -26.70067596435547,
      "activations/layer23_attention_weight_max": 37.22659683227539,
      "activations/layer23_attention_weight_min": -25.53682518005371,
      "activations/layer2_attention_weight_max": 31.797958374023438,
      "activations/layer2_attention_weight_min": -29.58173370361328,
      "activations/layer3_attention_weight_max": 90.47525024414062,
      "activations/layer3_attention_weight_min": -91.86759185791016,
      "activations/layer4_attention_weight_max": 84.72993469238281,
      "activations/layer4_attention_weight_min": -82.72911834716797,
      "activations/layer5_attention_weight_max": 64.15599822998047,
      "activations/layer5_attention_weight_min": -70.85162353515625,
      "activations/layer6_attention_weight_max": 51.164730072021484,
      "activations/layer6_attention_weight_min": -51.4775276184082,
      "activations/layer7_attention_weight_max": 68.5067367553711,
      "activations/layer7_attention_weight_min": -69.08274841308594,
      "activations/layer8_attention_weight_max": 45.184444427490234,
      "activations/layer8_attention_weight_min": -50.6292724609375,
      "activations/layer9_attention_weight_max": 44.455257415771484,
      "activations/layer9_attention_weight_min": -45.64787673950195,
      "epoch": 21.63,
      "learning_rate": 1.0624242424242422e-05,
      "loss": 2.7169,
      "step": 372250
    },
    {
      "activations/layer0_attention_weight_max": 16.796703338623047,
      "activations/layer0_attention_weight_min": -14.292682647705078,
      "activations/layer10_attention_weight_max": 34.430580139160156,
      "activations/layer10_attention_weight_min": -35.80117416381836,
      "activations/layer11_attention_weight_max": 32.404815673828125,
      "activations/layer11_attention_weight_min": -34.80254364013672,
      "activations/layer12_attention_weight_max": 17.65293312072754,
      "activations/layer12_attention_weight_min": -28.669225692749023,
      "activations/layer13_attention_weight_max": 33.32563400268555,
      "activations/layer13_attention_weight_min": -28.354957580566406,
      "activations/layer14_attention_weight_max": 33.90668869018555,
      "activations/layer14_attention_weight_min": -29.22105598449707,
      "activations/layer15_attention_weight_max": 31.56589698791504,
      "activations/layer15_attention_weight_min": -30.925485610961914,
      "activations/layer16_attention_weight_max": 32.24650573730469,
      "activations/layer16_attention_weight_min": -31.60506248474121,
      "activations/layer17_attention_weight_max": 49.21785354614258,
      "activations/layer17_attention_weight_min": -45.32189178466797,
      "activations/layer18_attention_weight_max": 43.7408447265625,
      "activations/layer18_attention_weight_min": -37.46149826049805,
      "activations/layer19_attention_weight_max": 21.00762939453125,
      "activations/layer19_attention_weight_min": -22.14304542541504,
      "activations/layer1_attention_weight_max": 14.652161598205566,
      "activations/layer1_attention_weight_min": -14.363121032714844,
      "activations/layer20_attention_weight_max": 19.935056686401367,
      "activations/layer20_attention_weight_min": -22.98029327392578,
      "activations/layer21_attention_weight_max": 35.56118392944336,
      "activations/layer21_attention_weight_min": -23.095760345458984,
      "activations/layer22_attention_weight_max": 24.765628814697266,
      "activations/layer22_attention_weight_min": -25.075885772705078,
      "activations/layer23_attention_weight_max": 34.735694885253906,
      "activations/layer23_attention_weight_min": -23.52484893798828,
      "activations/layer2_attention_weight_max": 32.461021423339844,
      "activations/layer2_attention_weight_min": -30.438926696777344,
      "activations/layer3_attention_weight_max": 89.54035186767578,
      "activations/layer3_attention_weight_min": -89.82071685791016,
      "activations/layer4_attention_weight_max": 84.40703582763672,
      "activations/layer4_attention_weight_min": -83.90678405761719,
      "activations/layer5_attention_weight_max": 62.39410400390625,
      "activations/layer5_attention_weight_min": -70.72698211669922,
      "activations/layer6_attention_weight_max": 52.87171936035156,
      "activations/layer6_attention_weight_min": -52.705928802490234,
      "activations/layer7_attention_weight_max": 70.85184478759766,
      "activations/layer7_attention_weight_min": -66.69097137451172,
      "activations/layer8_attention_weight_max": 46.75809097290039,
      "activations/layer8_attention_weight_min": -50.33692932128906,
      "activations/layer9_attention_weight_max": 43.28925704956055,
      "activations/layer9_attention_weight_min": -45.41558074951172,
      "epoch": 21.63,
      "learning_rate": 1.0605303030303029e-05,
      "loss": 2.714,
      "step": 372300
    },
    {
      "activations/layer0_attention_weight_max": 15.961060523986816,
      "activations/layer0_attention_weight_min": -14.240161895751953,
      "activations/layer10_attention_weight_max": 33.53226852416992,
      "activations/layer10_attention_weight_min": -34.48814010620117,
      "activations/layer11_attention_weight_max": 32.27967071533203,
      "activations/layer11_attention_weight_min": -33.205570220947266,
      "activations/layer12_attention_weight_max": 19.267845153808594,
      "activations/layer12_attention_weight_min": -23.568201065063477,
      "activations/layer13_attention_weight_max": 31.588106155395508,
      "activations/layer13_attention_weight_min": -25.686796188354492,
      "activations/layer14_attention_weight_max": 34.38562774658203,
      "activations/layer14_attention_weight_min": -29.450395584106445,
      "activations/layer15_attention_weight_max": 32.20985412597656,
      "activations/layer15_attention_weight_min": -30.12406349182129,
      "activations/layer16_attention_weight_max": 34.00366973876953,
      "activations/layer16_attention_weight_min": -32.67259979248047,
      "activations/layer17_attention_weight_max": 48.36977767944336,
      "activations/layer17_attention_weight_min": -45.75651931762695,
      "activations/layer18_attention_weight_max": 45.941715240478516,
      "activations/layer18_attention_weight_min": -43.70218276977539,
      "activations/layer19_attention_weight_max": 24.462467193603516,
      "activations/layer19_attention_weight_min": -24.200849533081055,
      "activations/layer1_attention_weight_max": 14.570269584655762,
      "activations/layer1_attention_weight_min": -14.073225975036621,
      "activations/layer20_attention_weight_max": 22.50394630432129,
      "activations/layer20_attention_weight_min": -22.027372360229492,
      "activations/layer21_attention_weight_max": 33.1150016784668,
      "activations/layer21_attention_weight_min": -21.489099502563477,
      "activations/layer22_attention_weight_max": 28.564796447753906,
      "activations/layer22_attention_weight_min": -24.46807861328125,
      "activations/layer23_attention_weight_max": 35.10954666137695,
      "activations/layer23_attention_weight_min": -23.28982925415039,
      "activations/layer2_attention_weight_max": 32.474998474121094,
      "activations/layer2_attention_weight_min": -32.550926208496094,
      "activations/layer3_attention_weight_max": 89.43099212646484,
      "activations/layer3_attention_weight_min": -89.45063018798828,
      "activations/layer4_attention_weight_max": 85.1858139038086,
      "activations/layer4_attention_weight_min": -83.96410369873047,
      "activations/layer5_attention_weight_max": 61.955711364746094,
      "activations/layer5_attention_weight_min": -75.0431137084961,
      "activations/layer6_attention_weight_max": 49.10197830200195,
      "activations/layer6_attention_weight_min": -51.44938278198242,
      "activations/layer7_attention_weight_max": 68.87300109863281,
      "activations/layer7_attention_weight_min": -65.61009216308594,
      "activations/layer8_attention_weight_max": 48.879661560058594,
      "activations/layer8_attention_weight_min": -49.65877151489258,
      "activations/layer9_attention_weight_max": 43.222129821777344,
      "activations/layer9_attention_weight_min": -44.81809616088867,
      "epoch": 21.64,
      "learning_rate": 1.0586363636363635e-05,
      "loss": 2.7045,
      "step": 372350
    },
    {
      "activations/layer0_attention_weight_max": 16.362478256225586,
      "activations/layer0_attention_weight_min": -14.163019180297852,
      "activations/layer10_attention_weight_max": 36.888343811035156,
      "activations/layer10_attention_weight_min": -37.212974548339844,
      "activations/layer11_attention_weight_max": 34.965126037597656,
      "activations/layer11_attention_weight_min": -37.138851165771484,
      "activations/layer12_attention_weight_max": 19.516803741455078,
      "activations/layer12_attention_weight_min": -23.76848030090332,
      "activations/layer13_attention_weight_max": 34.695980072021484,
      "activations/layer13_attention_weight_min": -29.074920654296875,
      "activations/layer14_attention_weight_max": 36.61061477661133,
      "activations/layer14_attention_weight_min": -32.73898696899414,
      "activations/layer15_attention_weight_max": 37.1550178527832,
      "activations/layer15_attention_weight_min": -32.17262268066406,
      "activations/layer16_attention_weight_max": 36.97623825073242,
      "activations/layer16_attention_weight_min": -36.722251892089844,
      "activations/layer17_attention_weight_max": 55.34950256347656,
      "activations/layer17_attention_weight_min": -50.70116424560547,
      "activations/layer18_attention_weight_max": 53.25640869140625,
      "activations/layer18_attention_weight_min": -43.81528091430664,
      "activations/layer19_attention_weight_max": 27.154800415039062,
      "activations/layer19_attention_weight_min": -23.801359176635742,
      "activations/layer1_attention_weight_max": 15.160490036010742,
      "activations/layer1_attention_weight_min": -13.32099437713623,
      "activations/layer20_attention_weight_max": 26.58025360107422,
      "activations/layer20_attention_weight_min": -23.432064056396484,
      "activations/layer21_attention_weight_max": 41.466064453125,
      "activations/layer21_attention_weight_min": -21.130416870117188,
      "activations/layer22_attention_weight_max": 33.534976959228516,
      "activations/layer22_attention_weight_min": -26.999788284301758,
      "activations/layer23_attention_weight_max": 37.79831314086914,
      "activations/layer23_attention_weight_min": -23.441383361816406,
      "activations/layer2_attention_weight_max": 31.782041549682617,
      "activations/layer2_attention_weight_min": -30.01087760925293,
      "activations/layer3_attention_weight_max": 87.09840393066406,
      "activations/layer3_attention_weight_min": -86.80335998535156,
      "activations/layer4_attention_weight_max": 84.29134368896484,
      "activations/layer4_attention_weight_min": -84.04347229003906,
      "activations/layer5_attention_weight_max": 60.88640594482422,
      "activations/layer5_attention_weight_min": -73.1116714477539,
      "activations/layer6_attention_weight_max": 48.83652877807617,
      "activations/layer6_attention_weight_min": -54.13497543334961,
      "activations/layer7_attention_weight_max": 68.9350814819336,
      "activations/layer7_attention_weight_min": -65.14705657958984,
      "activations/layer8_attention_weight_max": 46.61842727661133,
      "activations/layer8_attention_weight_min": -50.659645080566406,
      "activations/layer9_attention_weight_max": 43.32516098022461,
      "activations/layer9_attention_weight_min": -47.311553955078125,
      "epoch": 21.64,
      "learning_rate": 1.0567424242424242e-05,
      "loss": 2.7068,
      "step": 372400
    },
    {
      "activations/layer0_attention_weight_max": 15.605196952819824,
      "activations/layer0_attention_weight_min": -14.290769577026367,
      "activations/layer10_attention_weight_max": 37.5107536315918,
      "activations/layer10_attention_weight_min": -37.83828353881836,
      "activations/layer11_attention_weight_max": 36.164093017578125,
      "activations/layer11_attention_weight_min": -36.39707946777344,
      "activations/layer12_attention_weight_max": 19.14175796508789,
      "activations/layer12_attention_weight_min": -24.017929077148438,
      "activations/layer13_attention_weight_max": 34.10589599609375,
      "activations/layer13_attention_weight_min": -30.919015884399414,
      "activations/layer14_attention_weight_max": 37.172142028808594,
      "activations/layer14_attention_weight_min": -32.594459533691406,
      "activations/layer15_attention_weight_max": 35.01346969604492,
      "activations/layer15_attention_weight_min": -32.54331970214844,
      "activations/layer16_attention_weight_max": 35.88158416748047,
      "activations/layer16_attention_weight_min": -34.664642333984375,
      "activations/layer17_attention_weight_max": 53.644412994384766,
      "activations/layer17_attention_weight_min": -46.091678619384766,
      "activations/layer18_attention_weight_max": 47.76997375488281,
      "activations/layer18_attention_weight_min": -41.56974792480469,
      "activations/layer19_attention_weight_max": 24.23651123046875,
      "activations/layer19_attention_weight_min": -22.67093849182129,
      "activations/layer1_attention_weight_max": 14.570812225341797,
      "activations/layer1_attention_weight_min": -13.966973304748535,
      "activations/layer20_attention_weight_max": 21.091339111328125,
      "activations/layer20_attention_weight_min": -22.572341918945312,
      "activations/layer21_attention_weight_max": 38.09693145751953,
      "activations/layer21_attention_weight_min": -26.153640747070312,
      "activations/layer22_attention_weight_max": 28.442609786987305,
      "activations/layer22_attention_weight_min": -25.3238468170166,
      "activations/layer23_attention_weight_max": 32.962249755859375,
      "activations/layer23_attention_weight_min": -26.742780685424805,
      "activations/layer2_attention_weight_max": 31.52295684814453,
      "activations/layer2_attention_weight_min": -31.162242889404297,
      "activations/layer3_attention_weight_max": 87.9216079711914,
      "activations/layer3_attention_weight_min": -86.99518585205078,
      "activations/layer4_attention_weight_max": 83.8228759765625,
      "activations/layer4_attention_weight_min": -84.28609466552734,
      "activations/layer5_attention_weight_max": 60.57876205444336,
      "activations/layer5_attention_weight_min": -74.34727478027344,
      "activations/layer6_attention_weight_max": 52.551998138427734,
      "activations/layer6_attention_weight_min": -51.768611907958984,
      "activations/layer7_attention_weight_max": 68.80742645263672,
      "activations/layer7_attention_weight_min": -69.3141860961914,
      "activations/layer8_attention_weight_max": 50.24505615234375,
      "activations/layer8_attention_weight_min": -52.35969924926758,
      "activations/layer9_attention_weight_max": 47.86875534057617,
      "activations/layer9_attention_weight_min": -47.417274475097656,
      "epoch": 21.64,
      "learning_rate": 1.0548484848484848e-05,
      "loss": 2.7196,
      "step": 372450
    },
    {
      "activations/layer0_attention_weight_max": 15.499932289123535,
      "activations/layer0_attention_weight_min": -14.310440063476562,
      "activations/layer10_attention_weight_max": 37.211708068847656,
      "activations/layer10_attention_weight_min": -35.49143600463867,
      "activations/layer11_attention_weight_max": 34.89044189453125,
      "activations/layer11_attention_weight_min": -34.277748107910156,
      "activations/layer12_attention_weight_max": 17.991199493408203,
      "activations/layer12_attention_weight_min": -22.65964126586914,
      "activations/layer13_attention_weight_max": 36.40657424926758,
      "activations/layer13_attention_weight_min": -27.662025451660156,
      "activations/layer14_attention_weight_max": 32.99850082397461,
      "activations/layer14_attention_weight_min": -30.201732635498047,
      "activations/layer15_attention_weight_max": 32.06989288330078,
      "activations/layer15_attention_weight_min": -32.238502502441406,
      "activations/layer16_attention_weight_max": 34.465980529785156,
      "activations/layer16_attention_weight_min": -31.930675506591797,
      "activations/layer17_attention_weight_max": 51.280372619628906,
      "activations/layer17_attention_weight_min": -44.931602478027344,
      "activations/layer18_attention_weight_max": 49.39594650268555,
      "activations/layer18_attention_weight_min": -39.91558837890625,
      "activations/layer19_attention_weight_max": 24.675247192382812,
      "activations/layer19_attention_weight_min": -22.691654205322266,
      "activations/layer1_attention_weight_max": 15.111300468444824,
      "activations/layer1_attention_weight_min": -13.752676010131836,
      "activations/layer20_attention_weight_max": 24.885509490966797,
      "activations/layer20_attention_weight_min": -21.886940002441406,
      "activations/layer21_attention_weight_max": 37.85999298095703,
      "activations/layer21_attention_weight_min": -24.43602752685547,
      "activations/layer22_attention_weight_max": 30.950397491455078,
      "activations/layer22_attention_weight_min": -24.687641143798828,
      "activations/layer23_attention_weight_max": 35.44392395019531,
      "activations/layer23_attention_weight_min": -23.37228775024414,
      "activations/layer2_attention_weight_max": 31.30010223388672,
      "activations/layer2_attention_weight_min": -30.206722259521484,
      "activations/layer3_attention_weight_max": 89.10555267333984,
      "activations/layer3_attention_weight_min": -86.82923126220703,
      "activations/layer4_attention_weight_max": 83.57856750488281,
      "activations/layer4_attention_weight_min": -84.5572738647461,
      "activations/layer5_attention_weight_max": 58.01479721069336,
      "activations/layer5_attention_weight_min": -72.59313201904297,
      "activations/layer6_attention_weight_max": 48.22806167602539,
      "activations/layer6_attention_weight_min": -49.7827262878418,
      "activations/layer7_attention_weight_max": 65.51438903808594,
      "activations/layer7_attention_weight_min": -68.99970245361328,
      "activations/layer8_attention_weight_max": 46.79396438598633,
      "activations/layer8_attention_weight_min": -53.03940200805664,
      "activations/layer9_attention_weight_max": 43.38887023925781,
      "activations/layer9_attention_weight_min": -47.244483947753906,
      "epoch": 21.64,
      "learning_rate": 1.0529545454545455e-05,
      "loss": 2.7042,
      "step": 372500
    },
    {
      "activations/layer0_attention_weight_max": 15.887833595275879,
      "activations/layer0_attention_weight_min": -14.343731880187988,
      "activations/layer10_attention_weight_max": 38.690513610839844,
      "activations/layer10_attention_weight_min": -39.53988265991211,
      "activations/layer11_attention_weight_max": 36.63307571411133,
      "activations/layer11_attention_weight_min": -38.617286682128906,
      "activations/layer12_attention_weight_max": 19.591257095336914,
      "activations/layer12_attention_weight_min": -27.284202575683594,
      "activations/layer13_attention_weight_max": 37.82037353515625,
      "activations/layer13_attention_weight_min": -31.08171272277832,
      "activations/layer14_attention_weight_max": 38.84404754638672,
      "activations/layer14_attention_weight_min": -31.780426025390625,
      "activations/layer15_attention_weight_max": 35.60940170288086,
      "activations/layer15_attention_weight_min": -31.734081268310547,
      "activations/layer16_attention_weight_max": 35.95176315307617,
      "activations/layer16_attention_weight_min": -32.23249053955078,
      "activations/layer17_attention_weight_max": 53.159278869628906,
      "activations/layer17_attention_weight_min": -44.94121170043945,
      "activations/layer18_attention_weight_max": 49.105316162109375,
      "activations/layer18_attention_weight_min": -40.12311935424805,
      "activations/layer19_attention_weight_max": 26.335290908813477,
      "activations/layer19_attention_weight_min": -23.043153762817383,
      "activations/layer1_attention_weight_max": 15.025874137878418,
      "activations/layer1_attention_weight_min": -14.555400848388672,
      "activations/layer20_attention_weight_max": 25.859962463378906,
      "activations/layer20_attention_weight_min": -22.746498107910156,
      "activations/layer21_attention_weight_max": 36.17937469482422,
      "activations/layer21_attention_weight_min": -23.910512924194336,
      "activations/layer22_attention_weight_max": 29.652326583862305,
      "activations/layer22_attention_weight_min": -24.69341278076172,
      "activations/layer23_attention_weight_max": 40.399864196777344,
      "activations/layer23_attention_weight_min": -25.262866973876953,
      "activations/layer2_attention_weight_max": 33.17478942871094,
      "activations/layer2_attention_weight_min": -30.85770034790039,
      "activations/layer3_attention_weight_max": 96.54307556152344,
      "activations/layer3_attention_weight_min": -93.47136688232422,
      "activations/layer4_attention_weight_max": 92.46172332763672,
      "activations/layer4_attention_weight_min": -91.70069122314453,
      "activations/layer5_attention_weight_max": 65.29957580566406,
      "activations/layer5_attention_weight_min": -74.89057922363281,
      "activations/layer6_attention_weight_max": 54.82730484008789,
      "activations/layer6_attention_weight_min": -54.206199645996094,
      "activations/layer7_attention_weight_max": 76.09199523925781,
      "activations/layer7_attention_weight_min": -71.86614227294922,
      "activations/layer8_attention_weight_max": 49.92290496826172,
      "activations/layer8_attention_weight_min": -56.487735748291016,
      "activations/layer9_attention_weight_max": 48.67730712890625,
      "activations/layer9_attention_weight_min": -49.05260467529297,
      "epoch": 21.65,
      "learning_rate": 1.051060606060606e-05,
      "loss": 2.7268,
      "step": 372550
    },
    {
      "activations/layer0_attention_weight_max": 15.869980812072754,
      "activations/layer0_attention_weight_min": -14.342235565185547,
      "activations/layer10_attention_weight_max": 34.14167022705078,
      "activations/layer10_attention_weight_min": -33.69675064086914,
      "activations/layer11_attention_weight_max": 33.10651397705078,
      "activations/layer11_attention_weight_min": -33.413658142089844,
      "activations/layer12_attention_weight_max": 19.003660202026367,
      "activations/layer12_attention_weight_min": -26.20252799987793,
      "activations/layer13_attention_weight_max": 35.76030349731445,
      "activations/layer13_attention_weight_min": -30.06352424621582,
      "activations/layer14_attention_weight_max": 35.8914794921875,
      "activations/layer14_attention_weight_min": -29.744531631469727,
      "activations/layer15_attention_weight_max": 32.74117660522461,
      "activations/layer15_attention_weight_min": -31.932403564453125,
      "activations/layer16_attention_weight_max": 34.6388053894043,
      "activations/layer16_attention_weight_min": -34.42744064331055,
      "activations/layer17_attention_weight_max": 48.82524108886719,
      "activations/layer17_attention_weight_min": -46.52302169799805,
      "activations/layer18_attention_weight_max": 45.76057052612305,
      "activations/layer18_attention_weight_min": -42.842674255371094,
      "activations/layer19_attention_weight_max": 23.698261260986328,
      "activations/layer19_attention_weight_min": -24.789119720458984,
      "activations/layer1_attention_weight_max": 15.684248924255371,
      "activations/layer1_attention_weight_min": -13.662199974060059,
      "activations/layer20_attention_weight_max": 23.14164924621582,
      "activations/layer20_attention_weight_min": -23.348539352416992,
      "activations/layer21_attention_weight_max": 39.766143798828125,
      "activations/layer21_attention_weight_min": -22.639360427856445,
      "activations/layer22_attention_weight_max": 28.34027671813965,
      "activations/layer22_attention_weight_min": -27.55122184753418,
      "activations/layer23_attention_weight_max": 36.070518493652344,
      "activations/layer23_attention_weight_min": -23.778554916381836,
      "activations/layer2_attention_weight_max": 30.112796783447266,
      "activations/layer2_attention_weight_min": -30.28413200378418,
      "activations/layer3_attention_weight_max": 84.7747802734375,
      "activations/layer3_attention_weight_min": -88.7865982055664,
      "activations/layer4_attention_weight_max": 82.49650573730469,
      "activations/layer4_attention_weight_min": -82.15532684326172,
      "activations/layer5_attention_weight_max": 61.95915222167969,
      "activations/layer5_attention_weight_min": -68.27519226074219,
      "activations/layer6_attention_weight_max": 49.352962493896484,
      "activations/layer6_attention_weight_min": -50.688907623291016,
      "activations/layer7_attention_weight_max": 67.52735900878906,
      "activations/layer7_attention_weight_min": -66.13660430908203,
      "activations/layer8_attention_weight_max": 48.41368103027344,
      "activations/layer8_attention_weight_min": -52.47465133666992,
      "activations/layer9_attention_weight_max": 41.90169143676758,
      "activations/layer9_attention_weight_min": -46.77915954589844,
      "epoch": 21.65,
      "learning_rate": 1.0491666666666666e-05,
      "loss": 2.7102,
      "step": 372600
    },
    {
      "activations/layer0_attention_weight_max": 16.00030517578125,
      "activations/layer0_attention_weight_min": -13.990849494934082,
      "activations/layer10_attention_weight_max": 40.17236328125,
      "activations/layer10_attention_weight_min": -38.96383285522461,
      "activations/layer11_attention_weight_max": 37.35508728027344,
      "activations/layer11_attention_weight_min": -36.78532028198242,
      "activations/layer12_attention_weight_max": 19.272899627685547,
      "activations/layer12_attention_weight_min": -24.70829963684082,
      "activations/layer13_attention_weight_max": 34.190006256103516,
      "activations/layer13_attention_weight_min": -28.370349884033203,
      "activations/layer14_attention_weight_max": 34.48754119873047,
      "activations/layer14_attention_weight_min": -31.427566528320312,
      "activations/layer15_attention_weight_max": 32.996742248535156,
      "activations/layer15_attention_weight_min": -31.78190803527832,
      "activations/layer16_attention_weight_max": 34.941646575927734,
      "activations/layer16_attention_weight_min": -33.1119270324707,
      "activations/layer17_attention_weight_max": 55.90406799316406,
      "activations/layer17_attention_weight_min": -48.57860565185547,
      "activations/layer18_attention_weight_max": 52.849308013916016,
      "activations/layer18_attention_weight_min": -45.382083892822266,
      "activations/layer19_attention_weight_max": 25.105180740356445,
      "activations/layer19_attention_weight_min": -24.593324661254883,
      "activations/layer1_attention_weight_max": 15.673635482788086,
      "activations/layer1_attention_weight_min": -15.721844673156738,
      "activations/layer20_attention_weight_max": 23.793590545654297,
      "activations/layer20_attention_weight_min": -22.59235191345215,
      "activations/layer21_attention_weight_max": 41.00327682495117,
      "activations/layer21_attention_weight_min": -22.171260833740234,
      "activations/layer22_attention_weight_max": 29.65148162841797,
      "activations/layer22_attention_weight_min": -27.301559448242188,
      "activations/layer23_attention_weight_max": 35.50884246826172,
      "activations/layer23_attention_weight_min": -23.869176864624023,
      "activations/layer2_attention_weight_max": 31.814945220947266,
      "activations/layer2_attention_weight_min": -30.964818954467773,
      "activations/layer3_attention_weight_max": 93.33036041259766,
      "activations/layer3_attention_weight_min": -89.34722900390625,
      "activations/layer4_attention_weight_max": 86.58970642089844,
      "activations/layer4_attention_weight_min": -83.83728790283203,
      "activations/layer5_attention_weight_max": 60.966651916503906,
      "activations/layer5_attention_weight_min": -71.39752197265625,
      "activations/layer6_attention_weight_max": 52.272972106933594,
      "activations/layer6_attention_weight_min": -51.13490295410156,
      "activations/layer7_attention_weight_max": 67.92960357666016,
      "activations/layer7_attention_weight_min": -67.70823669433594,
      "activations/layer8_attention_weight_max": 52.56743621826172,
      "activations/layer8_attention_weight_min": -50.82770538330078,
      "activations/layer9_attention_weight_max": 53.224361419677734,
      "activations/layer9_attention_weight_min": -50.3797607421875,
      "epoch": 21.65,
      "learning_rate": 1.0472727272727273e-05,
      "loss": 2.6982,
      "step": 372650
    },
    {
      "activations/layer0_attention_weight_max": 16.334163665771484,
      "activations/layer0_attention_weight_min": -14.043279647827148,
      "activations/layer10_attention_weight_max": 35.98677062988281,
      "activations/layer10_attention_weight_min": -37.57588577270508,
      "activations/layer11_attention_weight_max": 33.46745300292969,
      "activations/layer11_attention_weight_min": -36.64575958251953,
      "activations/layer12_attention_weight_max": 20.376705169677734,
      "activations/layer12_attention_weight_min": -24.474252700805664,
      "activations/layer13_attention_weight_max": 36.072425842285156,
      "activations/layer13_attention_weight_min": -27.778621673583984,
      "activations/layer14_attention_weight_max": 37.29750061035156,
      "activations/layer14_attention_weight_min": -30.68057632446289,
      "activations/layer15_attention_weight_max": 34.18262481689453,
      "activations/layer15_attention_weight_min": -31.007150650024414,
      "activations/layer16_attention_weight_max": 36.079078674316406,
      "activations/layer16_attention_weight_min": -34.23657989501953,
      "activations/layer17_attention_weight_max": 53.346153259277344,
      "activations/layer17_attention_weight_min": -47.27759552001953,
      "activations/layer18_attention_weight_max": 51.75025939941406,
      "activations/layer18_attention_weight_min": -42.207801818847656,
      "activations/layer19_attention_weight_max": 26.248966217041016,
      "activations/layer19_attention_weight_min": -25.092023849487305,
      "activations/layer1_attention_weight_max": 16.89507293701172,
      "activations/layer1_attention_weight_min": -15.809566497802734,
      "activations/layer20_attention_weight_max": 27.551530838012695,
      "activations/layer20_attention_weight_min": -22.294876098632812,
      "activations/layer21_attention_weight_max": 36.12200927734375,
      "activations/layer21_attention_weight_min": -23.506284713745117,
      "activations/layer22_attention_weight_max": 30.306774139404297,
      "activations/layer22_attention_weight_min": -24.688323974609375,
      "activations/layer23_attention_weight_max": 37.57726287841797,
      "activations/layer23_attention_weight_min": -23.479366302490234,
      "activations/layer2_attention_weight_max": 30.439428329467773,
      "activations/layer2_attention_weight_min": -30.433605194091797,
      "activations/layer3_attention_weight_max": 87.07137298583984,
      "activations/layer3_attention_weight_min": -89.07508087158203,
      "activations/layer4_attention_weight_max": 82.1473159790039,
      "activations/layer4_attention_weight_min": -84.14981079101562,
      "activations/layer5_attention_weight_max": 64.19853210449219,
      "activations/layer5_attention_weight_min": -76.86480712890625,
      "activations/layer6_attention_weight_max": 47.193660736083984,
      "activations/layer6_attention_weight_min": -48.857444763183594,
      "activations/layer7_attention_weight_max": 68.44393157958984,
      "activations/layer7_attention_weight_min": -65.8757553100586,
      "activations/layer8_attention_weight_max": 46.56984329223633,
      "activations/layer8_attention_weight_min": -52.191349029541016,
      "activations/layer9_attention_weight_max": 46.13035583496094,
      "activations/layer9_attention_weight_min": -46.9777717590332,
      "epoch": 21.66,
      "learning_rate": 1.0453787878787878e-05,
      "loss": 2.7128,
      "step": 372700
    },
    {
      "activations/layer0_attention_weight_max": 16.637542724609375,
      "activations/layer0_attention_weight_min": -14.318053245544434,
      "activations/layer10_attention_weight_max": 38.100860595703125,
      "activations/layer10_attention_weight_min": -37.1910514831543,
      "activations/layer11_attention_weight_max": 35.22637939453125,
      "activations/layer11_attention_weight_min": -35.41728210449219,
      "activations/layer12_attention_weight_max": 19.881637573242188,
      "activations/layer12_attention_weight_min": -24.34433364868164,
      "activations/layer13_attention_weight_max": 34.70496368408203,
      "activations/layer13_attention_weight_min": -30.474260330200195,
      "activations/layer14_attention_weight_max": 34.89555358886719,
      "activations/layer14_attention_weight_min": -31.741010665893555,
      "activations/layer15_attention_weight_max": 32.65888595581055,
      "activations/layer15_attention_weight_min": -30.75800323486328,
      "activations/layer16_attention_weight_max": 33.197689056396484,
      "activations/layer16_attention_weight_min": -33.64157485961914,
      "activations/layer17_attention_weight_max": 50.09387969970703,
      "activations/layer17_attention_weight_min": -47.28699493408203,
      "activations/layer18_attention_weight_max": 46.28343200683594,
      "activations/layer18_attention_weight_min": -42.72807693481445,
      "activations/layer19_attention_weight_max": 24.061782836914062,
      "activations/layer19_attention_weight_min": -24.42060661315918,
      "activations/layer1_attention_weight_max": 15.675467491149902,
      "activations/layer1_attention_weight_min": -15.130393028259277,
      "activations/layer20_attention_weight_max": 22.40262222290039,
      "activations/layer20_attention_weight_min": -22.552257537841797,
      "activations/layer21_attention_weight_max": 34.02021789550781,
      "activations/layer21_attention_weight_min": -23.907272338867188,
      "activations/layer22_attention_weight_max": 28.52469825744629,
      "activations/layer22_attention_weight_min": -25.58509063720703,
      "activations/layer23_attention_weight_max": 36.73023986816406,
      "activations/layer23_attention_weight_min": -25.82254409790039,
      "activations/layer2_attention_weight_max": 32.81468963623047,
      "activations/layer2_attention_weight_min": -32.782440185546875,
      "activations/layer3_attention_weight_max": 92.67491912841797,
      "activations/layer3_attention_weight_min": -92.67955780029297,
      "activations/layer4_attention_weight_max": 86.96187591552734,
      "activations/layer4_attention_weight_min": -90.53843688964844,
      "activations/layer5_attention_weight_max": 63.660247802734375,
      "activations/layer5_attention_weight_min": -78.7525634765625,
      "activations/layer6_attention_weight_max": 51.05370330810547,
      "activations/layer6_attention_weight_min": -51.90379333496094,
      "activations/layer7_attention_weight_max": 70.80216979980469,
      "activations/layer7_attention_weight_min": -70.74595642089844,
      "activations/layer8_attention_weight_max": 50.29878234863281,
      "activations/layer8_attention_weight_min": -53.89577102661133,
      "activations/layer9_attention_weight_max": 47.68745040893555,
      "activations/layer9_attention_weight_min": -48.14469909667969,
      "epoch": 21.66,
      "learning_rate": 1.0434848484848484e-05,
      "loss": 2.7386,
      "step": 372750
    },
    {
      "activations/layer0_attention_weight_max": 15.910344123840332,
      "activations/layer0_attention_weight_min": -14.8018798828125,
      "activations/layer10_attention_weight_max": 36.708984375,
      "activations/layer10_attention_weight_min": -37.573062896728516,
      "activations/layer11_attention_weight_max": 33.24267578125,
      "activations/layer11_attention_weight_min": -34.773162841796875,
      "activations/layer12_attention_weight_max": 19.262670516967773,
      "activations/layer12_attention_weight_min": -22.95565414428711,
      "activations/layer13_attention_weight_max": 33.21599578857422,
      "activations/layer13_attention_weight_min": -25.683631896972656,
      "activations/layer14_attention_weight_max": 34.82319641113281,
      "activations/layer14_attention_weight_min": -30.30557632446289,
      "activations/layer15_attention_weight_max": 33.715843200683594,
      "activations/layer15_attention_weight_min": -29.461381912231445,
      "activations/layer16_attention_weight_max": 34.47537612915039,
      "activations/layer16_attention_weight_min": -32.665809631347656,
      "activations/layer17_attention_weight_max": 54.364952087402344,
      "activations/layer17_attention_weight_min": -43.905059814453125,
      "activations/layer18_attention_weight_max": 49.36031723022461,
      "activations/layer18_attention_weight_min": -40.01689147949219,
      "activations/layer19_attention_weight_max": 27.3259334564209,
      "activations/layer19_attention_weight_min": -23.01430892944336,
      "activations/layer1_attention_weight_max": 15.373003005981445,
      "activations/layer1_attention_weight_min": -14.604789733886719,
      "activations/layer20_attention_weight_max": 23.887630462646484,
      "activations/layer20_attention_weight_min": -22.97026252746582,
      "activations/layer21_attention_weight_max": 32.68791580200195,
      "activations/layer21_attention_weight_min": -22.995325088500977,
      "activations/layer22_attention_weight_max": 31.48454475402832,
      "activations/layer22_attention_weight_min": -24.17136573791504,
      "activations/layer23_attention_weight_max": 37.700889587402344,
      "activations/layer23_attention_weight_min": -27.368961334228516,
      "activations/layer2_attention_weight_max": 31.56661033630371,
      "activations/layer2_attention_weight_min": -29.89922332763672,
      "activations/layer3_attention_weight_max": 89.76961517333984,
      "activations/layer3_attention_weight_min": -86.85929870605469,
      "activations/layer4_attention_weight_max": 83.70532989501953,
      "activations/layer4_attention_weight_min": -84.63817596435547,
      "activations/layer5_attention_weight_max": 63.9666862487793,
      "activations/layer5_attention_weight_min": -71.73185729980469,
      "activations/layer6_attention_weight_max": 49.526493072509766,
      "activations/layer6_attention_weight_min": -50.49161148071289,
      "activations/layer7_attention_weight_max": 67.39400482177734,
      "activations/layer7_attention_weight_min": -67.58068084716797,
      "activations/layer8_attention_weight_max": 46.505985260009766,
      "activations/layer8_attention_weight_min": -49.94401931762695,
      "activations/layer9_attention_weight_max": 42.15254592895508,
      "activations/layer9_attention_weight_min": -48.423789978027344,
      "epoch": 21.66,
      "learning_rate": 1.041590909090909e-05,
      "loss": 2.7262,
      "step": 372800
    },
    {
      "activations/layer0_attention_weight_max": 15.717691421508789,
      "activations/layer0_attention_weight_min": -14.228333473205566,
      "activations/layer10_attention_weight_max": 34.919769287109375,
      "activations/layer10_attention_weight_min": -33.78862762451172,
      "activations/layer11_attention_weight_max": 31.896663665771484,
      "activations/layer11_attention_weight_min": -35.57441711425781,
      "activations/layer12_attention_weight_max": 19.352054595947266,
      "activations/layer12_attention_weight_min": -26.741003036499023,
      "activations/layer13_attention_weight_max": 32.54846954345703,
      "activations/layer13_attention_weight_min": -26.748796463012695,
      "activations/layer14_attention_weight_max": 34.404815673828125,
      "activations/layer14_attention_weight_min": -30.03792381286621,
      "activations/layer15_attention_weight_max": 32.60786819458008,
      "activations/layer15_attention_weight_min": -30.32834243774414,
      "activations/layer16_attention_weight_max": 35.85330581665039,
      "activations/layer16_attention_weight_min": -34.106842041015625,
      "activations/layer17_attention_weight_max": 52.39025115966797,
      "activations/layer17_attention_weight_min": -46.28211212158203,
      "activations/layer18_attention_weight_max": 47.02592086791992,
      "activations/layer18_attention_weight_min": -39.71273422241211,
      "activations/layer19_attention_weight_max": 25.47084617614746,
      "activations/layer19_attention_weight_min": -22.998552322387695,
      "activations/layer1_attention_weight_max": 16.495361328125,
      "activations/layer1_attention_weight_min": -15.167231559753418,
      "activations/layer20_attention_weight_max": 22.983837127685547,
      "activations/layer20_attention_weight_min": -22.1844539642334,
      "activations/layer21_attention_weight_max": 37.53705596923828,
      "activations/layer21_attention_weight_min": -30.0716552734375,
      "activations/layer22_attention_weight_max": 28.725971221923828,
      "activations/layer22_attention_weight_min": -26.066221237182617,
      "activations/layer23_attention_weight_max": 35.43082809448242,
      "activations/layer23_attention_weight_min": -23.23646354675293,
      "activations/layer2_attention_weight_max": 31.394765853881836,
      "activations/layer2_attention_weight_min": -29.814525604248047,
      "activations/layer3_attention_weight_max": 89.65541076660156,
      "activations/layer3_attention_weight_min": -87.84938049316406,
      "activations/layer4_attention_weight_max": 85.18748474121094,
      "activations/layer4_attention_weight_min": -82.4629898071289,
      "activations/layer5_attention_weight_max": 67.32805633544922,
      "activations/layer5_attention_weight_min": -70.66477966308594,
      "activations/layer6_attention_weight_max": 49.189029693603516,
      "activations/layer6_attention_weight_min": -47.94343185424805,
      "activations/layer7_attention_weight_max": 66.63151550292969,
      "activations/layer7_attention_weight_min": -64.20013427734375,
      "activations/layer8_attention_weight_max": 43.90183639526367,
      "activations/layer8_attention_weight_min": -48.639217376708984,
      "activations/layer9_attention_weight_max": 42.01244354248047,
      "activations/layer9_attention_weight_min": -45.42915725708008,
      "epoch": 21.66,
      "learning_rate": 1.0396969696969696e-05,
      "loss": 2.7365,
      "step": 372850
    },
    {
      "activations/layer0_attention_weight_max": 15.998507499694824,
      "activations/layer0_attention_weight_min": -13.986445426940918,
      "activations/layer10_attention_weight_max": 35.71407699584961,
      "activations/layer10_attention_weight_min": -36.692115783691406,
      "activations/layer11_attention_weight_max": 34.48456573486328,
      "activations/layer11_attention_weight_min": -35.71210861206055,
      "activations/layer12_attention_weight_max": 20.451446533203125,
      "activations/layer12_attention_weight_min": -24.615819931030273,
      "activations/layer13_attention_weight_max": 38.43306350708008,
      "activations/layer13_attention_weight_min": -29.297019958496094,
      "activations/layer14_attention_weight_max": 37.055747985839844,
      "activations/layer14_attention_weight_min": -31.570613861083984,
      "activations/layer15_attention_weight_max": 36.74713897705078,
      "activations/layer15_attention_weight_min": -31.682239532470703,
      "activations/layer16_attention_weight_max": 38.472625732421875,
      "activations/layer16_attention_weight_min": -35.03654861450195,
      "activations/layer17_attention_weight_max": 55.54152297973633,
      "activations/layer17_attention_weight_min": -48.66716766357422,
      "activations/layer18_attention_weight_max": 56.59197235107422,
      "activations/layer18_attention_weight_min": -43.086669921875,
      "activations/layer19_attention_weight_max": 28.489587783813477,
      "activations/layer19_attention_weight_min": -24.316150665283203,
      "activations/layer1_attention_weight_max": 16.623668670654297,
      "activations/layer1_attention_weight_min": -16.13303565979004,
      "activations/layer20_attention_weight_max": 27.77556037902832,
      "activations/layer20_attention_weight_min": -22.250043869018555,
      "activations/layer21_attention_weight_max": 48.5245246887207,
      "activations/layer21_attention_weight_min": -22.7713565826416,
      "activations/layer22_attention_weight_max": 29.84709358215332,
      "activations/layer22_attention_weight_min": -27.13701057434082,
      "activations/layer23_attention_weight_max": 37.326114654541016,
      "activations/layer23_attention_weight_min": -26.349624633789062,
      "activations/layer2_attention_weight_max": 31.62372589111328,
      "activations/layer2_attention_weight_min": -32.02231216430664,
      "activations/layer3_attention_weight_max": 89.6795425415039,
      "activations/layer3_attention_weight_min": -89.66609954833984,
      "activations/layer4_attention_weight_max": 83.81903839111328,
      "activations/layer4_attention_weight_min": -85.45970916748047,
      "activations/layer5_attention_weight_max": 60.23025131225586,
      "activations/layer5_attention_weight_min": -72.55857849121094,
      "activations/layer6_attention_weight_max": 48.94589614868164,
      "activations/layer6_attention_weight_min": -52.02265548706055,
      "activations/layer7_attention_weight_max": 66.1821060180664,
      "activations/layer7_attention_weight_min": -66.11917877197266,
      "activations/layer8_attention_weight_max": 47.0368766784668,
      "activations/layer8_attention_weight_min": -49.880435943603516,
      "activations/layer9_attention_weight_max": 43.469757080078125,
      "activations/layer9_attention_weight_min": -46.713008880615234,
      "epoch": 21.67,
      "learning_rate": 1.0378030303030303e-05,
      "loss": 2.7161,
      "step": 372900
    },
    {
      "activations/layer0_attention_weight_max": 15.778488159179688,
      "activations/layer0_attention_weight_min": -14.176390647888184,
      "activations/layer10_attention_weight_max": 36.59504699707031,
      "activations/layer10_attention_weight_min": -34.964805603027344,
      "activations/layer11_attention_weight_max": 33.60578155517578,
      "activations/layer11_attention_weight_min": -33.66825485229492,
      "activations/layer12_attention_weight_max": 19.671932220458984,
      "activations/layer12_attention_weight_min": -25.502206802368164,
      "activations/layer13_attention_weight_max": 31.502647399902344,
      "activations/layer13_attention_weight_min": -27.639984130859375,
      "activations/layer14_attention_weight_max": 35.85158157348633,
      "activations/layer14_attention_weight_min": -32.369834899902344,
      "activations/layer15_attention_weight_max": 34.41923904418945,
      "activations/layer15_attention_weight_min": -33.09682846069336,
      "activations/layer16_attention_weight_max": 34.879852294921875,
      "activations/layer16_attention_weight_min": -33.68797302246094,
      "activations/layer17_attention_weight_max": 53.160614013671875,
      "activations/layer17_attention_weight_min": -47.63112258911133,
      "activations/layer18_attention_weight_max": 49.28607177734375,
      "activations/layer18_attention_weight_min": -42.87922286987305,
      "activations/layer19_attention_weight_max": 28.680923461914062,
      "activations/layer19_attention_weight_min": -25.713220596313477,
      "activations/layer1_attention_weight_max": 15.916729927062988,
      "activations/layer1_attention_weight_min": -15.99409294128418,
      "activations/layer20_attention_weight_max": 25.431987762451172,
      "activations/layer20_attention_weight_min": -28.26837730407715,
      "activations/layer21_attention_weight_max": 35.01377868652344,
      "activations/layer21_attention_weight_min": -22.71023941040039,
      "activations/layer22_attention_weight_max": 30.071096420288086,
      "activations/layer22_attention_weight_min": -26.28837776184082,
      "activations/layer23_attention_weight_max": 35.505489349365234,
      "activations/layer23_attention_weight_min": -25.14112091064453,
      "activations/layer2_attention_weight_max": 32.70308303833008,
      "activations/layer2_attention_weight_min": -31.67633819580078,
      "activations/layer3_attention_weight_max": 93.12165069580078,
      "activations/layer3_attention_weight_min": -88.93647003173828,
      "activations/layer4_attention_weight_max": 88.23307800292969,
      "activations/layer4_attention_weight_min": -85.08518981933594,
      "activations/layer5_attention_weight_max": 64.496337890625,
      "activations/layer5_attention_weight_min": -73.9161376953125,
      "activations/layer6_attention_weight_max": 50.59817123413086,
      "activations/layer6_attention_weight_min": -49.4378776550293,
      "activations/layer7_attention_weight_max": 70.9170913696289,
      "activations/layer7_attention_weight_min": -65.85389709472656,
      "activations/layer8_attention_weight_max": 45.00863265991211,
      "activations/layer8_attention_weight_min": -51.01856231689453,
      "activations/layer9_attention_weight_max": 42.90532302856445,
      "activations/layer9_attention_weight_min": -45.77494430541992,
      "epoch": 21.67,
      "learning_rate": 1.0359469696969697e-05,
      "loss": 2.7181,
      "step": 372950
    },
    {
      "activations/layer0_attention_weight_max": 15.17528247833252,
      "activations/layer0_attention_weight_min": -14.24999713897705,
      "activations/layer10_attention_weight_max": 35.19696044921875,
      "activations/layer10_attention_weight_min": -33.94289779663086,
      "activations/layer11_attention_weight_max": 33.02348327636719,
      "activations/layer11_attention_weight_min": -34.7923469543457,
      "activations/layer12_attention_weight_max": 19.76251983642578,
      "activations/layer12_attention_weight_min": -27.3695125579834,
      "activations/layer13_attention_weight_max": 33.236324310302734,
      "activations/layer13_attention_weight_min": -28.042709350585938,
      "activations/layer14_attention_weight_max": 36.178871154785156,
      "activations/layer14_attention_weight_min": -30.857521057128906,
      "activations/layer15_attention_weight_max": 34.11430358886719,
      "activations/layer15_attention_weight_min": -31.760774612426758,
      "activations/layer16_attention_weight_max": 34.2037353515625,
      "activations/layer16_attention_weight_min": -32.39691162109375,
      "activations/layer17_attention_weight_max": 53.17516326904297,
      "activations/layer17_attention_weight_min": -44.76778030395508,
      "activations/layer18_attention_weight_max": 45.53102493286133,
      "activations/layer18_attention_weight_min": -38.70720672607422,
      "activations/layer19_attention_weight_max": 27.986541748046875,
      "activations/layer19_attention_weight_min": -21.833065032958984,
      "activations/layer1_attention_weight_max": 15.53772258758545,
      "activations/layer1_attention_weight_min": -13.713749885559082,
      "activations/layer20_attention_weight_max": 26.37047004699707,
      "activations/layer20_attention_weight_min": -21.992496490478516,
      "activations/layer21_attention_weight_max": 39.15815734863281,
      "activations/layer21_attention_weight_min": -23.01227378845215,
      "activations/layer22_attention_weight_max": 28.487058639526367,
      "activations/layer22_attention_weight_min": -25.764354705810547,
      "activations/layer23_attention_weight_max": 39.3861083984375,
      "activations/layer23_attention_weight_min": -23.26540184020996,
      "activations/layer2_attention_weight_max": 32.55647659301758,
      "activations/layer2_attention_weight_min": -29.663047790527344,
      "activations/layer3_attention_weight_max": 90.19466400146484,
      "activations/layer3_attention_weight_min": -86.49739074707031,
      "activations/layer4_attention_weight_max": 85.73772430419922,
      "activations/layer4_attention_weight_min": -84.87434387207031,
      "activations/layer5_attention_weight_max": 62.31959533691406,
      "activations/layer5_attention_weight_min": -71.36958312988281,
      "activations/layer6_attention_weight_max": 49.91028594970703,
      "activations/layer6_attention_weight_min": -49.078853607177734,
      "activations/layer7_attention_weight_max": 67.35657501220703,
      "activations/layer7_attention_weight_min": -66.88055419921875,
      "activations/layer8_attention_weight_max": 48.23480224609375,
      "activations/layer8_attention_weight_min": -50.77412796020508,
      "activations/layer9_attention_weight_max": 43.75748825073242,
      "activations/layer9_attention_weight_min": -45.97889709472656,
      "epoch": 21.67,
      "learning_rate": 1.0340530303030302e-05,
      "loss": 2.715,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4865,
      "eval_samples_per_second": 505.982,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4865,
      "eval_openwebtext_samples_per_second": 505.982,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9817,
      "eval_wikitext_samples_per_second": 230.111,
      "step": 373000
    },
    {
      "epoch": 21.67,
      "eval_lambada_loss": 2.40234375,
      "eval_lambada_ppl": 11.049042250008942,
      "eval_lambada_runtime": 9.587,
      "eval_lambada_samples_per_second": 507.877,
      "step": 373000
    },
    {
      "activations/layer0_attention_weight_max": 15.736871719360352,
      "activations/layer0_attention_weight_min": -14.0070161819458,
      "activations/layer10_attention_weight_max": 40.55648422241211,
      "activations/layer10_attention_weight_min": -38.178977966308594,
      "activations/layer11_attention_weight_max": 38.666316986083984,
      "activations/layer11_attention_weight_min": -37.37962341308594,
      "activations/layer12_attention_weight_max": 20.5192928314209,
      "activations/layer12_attention_weight_min": -23.2546443939209,
      "activations/layer13_attention_weight_max": 42.66264724731445,
      "activations/layer13_attention_weight_min": -29.90215492248535,
      "activations/layer14_attention_weight_max": 39.08137512207031,
      "activations/layer14_attention_weight_min": -30.758895874023438,
      "activations/layer15_attention_weight_max": 37.212764739990234,
      "activations/layer15_attention_weight_min": -30.3411865234375,
      "activations/layer16_attention_weight_max": 35.29180908203125,
      "activations/layer16_attention_weight_min": -34.46016311645508,
      "activations/layer17_attention_weight_max": 53.11977005004883,
      "activations/layer17_attention_weight_min": -48.84737014770508,
      "activations/layer18_attention_weight_max": 50.56245040893555,
      "activations/layer18_attention_weight_min": -40.68805694580078,
      "activations/layer19_attention_weight_max": 25.651342391967773,
      "activations/layer19_attention_weight_min": -23.22913360595703,
      "activations/layer1_attention_weight_max": 17.809249877929688,
      "activations/layer1_attention_weight_min": -16.881484985351562,
      "activations/layer20_attention_weight_max": 23.626115798950195,
      "activations/layer20_attention_weight_min": -23.073083877563477,
      "activations/layer21_attention_weight_max": 34.00519943237305,
      "activations/layer21_attention_weight_min": -20.82794189453125,
      "activations/layer22_attention_weight_max": 29.406497955322266,
      "activations/layer22_attention_weight_min": -26.37287139892578,
      "activations/layer23_attention_weight_max": 35.841552734375,
      "activations/layer23_attention_weight_min": -23.35497283935547,
      "activations/layer2_attention_weight_max": 33.33611297607422,
      "activations/layer2_attention_weight_min": -31.235862731933594,
      "activations/layer3_attention_weight_max": 94.62525939941406,
      "activations/layer3_attention_weight_min": -92.5628890991211,
      "activations/layer4_attention_weight_max": 88.56642150878906,
      "activations/layer4_attention_weight_min": -84.76510620117188,
      "activations/layer5_attention_weight_max": 62.06330490112305,
      "activations/layer5_attention_weight_min": -71.30137634277344,
      "activations/layer6_attention_weight_max": 53.18585968017578,
      "activations/layer6_attention_weight_min": -50.02470016479492,
      "activations/layer7_attention_weight_max": 70.95246124267578,
      "activations/layer7_attention_weight_min": -68.33462524414062,
      "activations/layer8_attention_weight_max": 52.724151611328125,
      "activations/layer8_attention_weight_min": -52.38040542602539,
      "activations/layer9_attention_weight_max": 51.50148391723633,
      "activations/layer9_attention_weight_min": -50.928890228271484,
      "epoch": 21.68,
      "learning_rate": 1.0321590909090908e-05,
      "loss": 2.7215,
      "step": 373050
    },
    {
      "activations/layer0_attention_weight_max": 15.479571342468262,
      "activations/layer0_attention_weight_min": -14.079411506652832,
      "activations/layer10_attention_weight_max": 37.59272766113281,
      "activations/layer10_attention_weight_min": -37.93294143676758,
      "activations/layer11_attention_weight_max": 35.7512321472168,
      "activations/layer11_attention_weight_min": -36.700191497802734,
      "activations/layer12_attention_weight_max": 20.65976333618164,
      "activations/layer12_attention_weight_min": -22.274368286132812,
      "activations/layer13_attention_weight_max": 39.463287353515625,
      "activations/layer13_attention_weight_min": -30.120159149169922,
      "activations/layer14_attention_weight_max": 36.41773986816406,
      "activations/layer14_attention_weight_min": -30.948610305786133,
      "activations/layer15_attention_weight_max": 35.41465377807617,
      "activations/layer15_attention_weight_min": -31.731977462768555,
      "activations/layer16_attention_weight_max": 38.574134826660156,
      "activations/layer16_attention_weight_min": -36.035926818847656,
      "activations/layer17_attention_weight_max": 56.01421356201172,
      "activations/layer17_attention_weight_min": -51.293209075927734,
      "activations/layer18_attention_weight_max": 55.25395965576172,
      "activations/layer18_attention_weight_min": -45.73517990112305,
      "activations/layer19_attention_weight_max": 31.66973304748535,
      "activations/layer19_attention_weight_min": -23.79566192626953,
      "activations/layer1_attention_weight_max": 19.72149085998535,
      "activations/layer1_attention_weight_min": -18.571666717529297,
      "activations/layer20_attention_weight_max": 27.34453773498535,
      "activations/layer20_attention_weight_min": -22.33098030090332,
      "activations/layer21_attention_weight_max": 41.85568618774414,
      "activations/layer21_attention_weight_min": -26.908721923828125,
      "activations/layer22_attention_weight_max": 30.70992088317871,
      "activations/layer22_attention_weight_min": -25.03168487548828,
      "activations/layer23_attention_weight_max": 38.46507263183594,
      "activations/layer23_attention_weight_min": -23.294185638427734,
      "activations/layer2_attention_weight_max": 33.15818786621094,
      "activations/layer2_attention_weight_min": -30.263097763061523,
      "activations/layer3_attention_weight_max": 91.7568359375,
      "activations/layer3_attention_weight_min": -90.36150360107422,
      "activations/layer4_attention_weight_max": 87.6936264038086,
      "activations/layer4_attention_weight_min": -89.92282104492188,
      "activations/layer5_attention_weight_max": 65.83267974853516,
      "activations/layer5_attention_weight_min": -75.74742889404297,
      "activations/layer6_attention_weight_max": 52.107627868652344,
      "activations/layer6_attention_weight_min": -53.713294982910156,
      "activations/layer7_attention_weight_max": 73.00553894042969,
      "activations/layer7_attention_weight_min": -71.68110656738281,
      "activations/layer8_attention_weight_max": 50.14430236816406,
      "activations/layer8_attention_weight_min": -53.149818420410156,
      "activations/layer9_attention_weight_max": 46.99589157104492,
      "activations/layer9_attention_weight_min": -51.10400390625,
      "epoch": 21.68,
      "learning_rate": 1.0302651515151513e-05,
      "loss": 2.6993,
      "step": 373100
    },
    {
      "activations/layer0_attention_weight_max": 16.029726028442383,
      "activations/layer0_attention_weight_min": -14.296022415161133,
      "activations/layer10_attention_weight_max": 36.08224868774414,
      "activations/layer10_attention_weight_min": -35.769405364990234,
      "activations/layer11_attention_weight_max": 34.836090087890625,
      "activations/layer11_attention_weight_min": -34.96986389160156,
      "activations/layer12_attention_weight_max": 18.08258628845215,
      "activations/layer12_attention_weight_min": -23.868064880371094,
      "activations/layer13_attention_weight_max": 41.612056732177734,
      "activations/layer13_attention_weight_min": -28.881935119628906,
      "activations/layer14_attention_weight_max": 35.56513214111328,
      "activations/layer14_attention_weight_min": -30.344682693481445,
      "activations/layer15_attention_weight_max": 36.05237579345703,
      "activations/layer15_attention_weight_min": -30.513282775878906,
      "activations/layer16_attention_weight_max": 34.943721771240234,
      "activations/layer16_attention_weight_min": -32.57413101196289,
      "activations/layer17_attention_weight_max": 51.036128997802734,
      "activations/layer17_attention_weight_min": -46.13739013671875,
      "activations/layer18_attention_weight_max": 47.21151351928711,
      "activations/layer18_attention_weight_min": -40.96329116821289,
      "activations/layer19_attention_weight_max": 26.428207397460938,
      "activations/layer19_attention_weight_min": -23.17962074279785,
      "activations/layer1_attention_weight_max": 17.054931640625,
      "activations/layer1_attention_weight_min": -17.258024215698242,
      "activations/layer20_attention_weight_max": 23.85887336730957,
      "activations/layer20_attention_weight_min": -20.4538516998291,
      "activations/layer21_attention_weight_max": 35.77404022216797,
      "activations/layer21_attention_weight_min": -21.710641860961914,
      "activations/layer22_attention_weight_max": 28.659164428710938,
      "activations/layer22_attention_weight_min": -23.969114303588867,
      "activations/layer23_attention_weight_max": 36.89382553100586,
      "activations/layer23_attention_weight_min": -23.485998153686523,
      "activations/layer2_attention_weight_max": 32.24913024902344,
      "activations/layer2_attention_weight_min": -32.124759674072266,
      "activations/layer3_attention_weight_max": 91.1906967163086,
      "activations/layer3_attention_weight_min": -90.25247955322266,
      "activations/layer4_attention_weight_max": 87.35138702392578,
      "activations/layer4_attention_weight_min": -88.74724578857422,
      "activations/layer5_attention_weight_max": 60.878108978271484,
      "activations/layer5_attention_weight_min": -76.20182800292969,
      "activations/layer6_attention_weight_max": 51.47322463989258,
      "activations/layer6_attention_weight_min": -52.067787170410156,
      "activations/layer7_attention_weight_max": 69.9380874633789,
      "activations/layer7_attention_weight_min": -69.88902282714844,
      "activations/layer8_attention_weight_max": 46.88190841674805,
      "activations/layer8_attention_weight_min": -49.02299880981445,
      "activations/layer9_attention_weight_max": 47.20423126220703,
      "activations/layer9_attention_weight_min": -44.70183181762695,
      "epoch": 21.68,
      "learning_rate": 1.028371212121212e-05,
      "loss": 2.7214,
      "step": 373150
    },
    {
      "activations/layer0_attention_weight_max": 15.902913093566895,
      "activations/layer0_attention_weight_min": -14.015839576721191,
      "activations/layer10_attention_weight_max": 34.360389709472656,
      "activations/layer10_attention_weight_min": -36.05800247192383,
      "activations/layer11_attention_weight_max": 32.91655731201172,
      "activations/layer11_attention_weight_min": -34.152767181396484,
      "activations/layer12_attention_weight_max": 18.4352970123291,
      "activations/layer12_attention_weight_min": -27.520029067993164,
      "activations/layer13_attention_weight_max": 31.081626892089844,
      "activations/layer13_attention_weight_min": -28.50718116760254,
      "activations/layer14_attention_weight_max": 34.985496520996094,
      "activations/layer14_attention_weight_min": -29.810272216796875,
      "activations/layer15_attention_weight_max": 30.295011520385742,
      "activations/layer15_attention_weight_min": -29.77067756652832,
      "activations/layer16_attention_weight_max": 32.99563217163086,
      "activations/layer16_attention_weight_min": -31.198486328125,
      "activations/layer17_attention_weight_max": 50.6370964050293,
      "activations/layer17_attention_weight_min": -44.839908599853516,
      "activations/layer18_attention_weight_max": 44.721900939941406,
      "activations/layer18_attention_weight_min": -39.15978240966797,
      "activations/layer19_attention_weight_max": 23.016244888305664,
      "activations/layer19_attention_weight_min": -22.503795623779297,
      "activations/layer1_attention_weight_max": 15.683326721191406,
      "activations/layer1_attention_weight_min": -15.340104103088379,
      "activations/layer20_attention_weight_max": 20.346113204956055,
      "activations/layer20_attention_weight_min": -22.38414764404297,
      "activations/layer21_attention_weight_max": 32.2398796081543,
      "activations/layer21_attention_weight_min": -21.974689483642578,
      "activations/layer22_attention_weight_max": 31.778289794921875,
      "activations/layer22_attention_weight_min": -26.23311424255371,
      "activations/layer23_attention_weight_max": 34.337440490722656,
      "activations/layer23_attention_weight_min": -23.36339569091797,
      "activations/layer2_attention_weight_max": 31.400390625,
      "activations/layer2_attention_weight_min": -30.164579391479492,
      "activations/layer3_attention_weight_max": 85.57767486572266,
      "activations/layer3_attention_weight_min": -84.83026123046875,
      "activations/layer4_attention_weight_max": 81.38634490966797,
      "activations/layer4_attention_weight_min": -81.93268585205078,
      "activations/layer5_attention_weight_max": 60.098289489746094,
      "activations/layer5_attention_weight_min": -69.77299499511719,
      "activations/layer6_attention_weight_max": 48.43596267700195,
      "activations/layer6_attention_weight_min": -50.5820426940918,
      "activations/layer7_attention_weight_max": 68.62089538574219,
      "activations/layer7_attention_weight_min": -64.82841491699219,
      "activations/layer8_attention_weight_max": 46.319881439208984,
      "activations/layer8_attention_weight_min": -48.33970642089844,
      "activations/layer9_attention_weight_max": 44.55839157104492,
      "activations/layer9_attention_weight_min": -46.50716781616211,
      "epoch": 21.69,
      "learning_rate": 1.0264772727272726e-05,
      "loss": 2.713,
      "step": 373200
    },
    {
      "activations/layer0_attention_weight_max": 16.332670211791992,
      "activations/layer0_attention_weight_min": -14.10144329071045,
      "activations/layer10_attention_weight_max": 36.06163024902344,
      "activations/layer10_attention_weight_min": -37.43666458129883,
      "activations/layer11_attention_weight_max": 35.411705017089844,
      "activations/layer11_attention_weight_min": -34.46910858154297,
      "activations/layer12_attention_weight_max": 20.439186096191406,
      "activations/layer12_attention_weight_min": -23.78901481628418,
      "activations/layer13_attention_weight_max": 32.79494857788086,
      "activations/layer13_attention_weight_min": -28.91286277770996,
      "activations/layer14_attention_weight_max": 34.234764099121094,
      "activations/layer14_attention_weight_min": -31.57187843322754,
      "activations/layer15_attention_weight_max": 36.10649490356445,
      "activations/layer15_attention_weight_min": -31.4215145111084,
      "activations/layer16_attention_weight_max": 34.9337158203125,
      "activations/layer16_attention_weight_min": -33.49040985107422,
      "activations/layer17_attention_weight_max": 54.58736038208008,
      "activations/layer17_attention_weight_min": -49.208343505859375,
      "activations/layer18_attention_weight_max": 47.624610900878906,
      "activations/layer18_attention_weight_min": -42.94053649902344,
      "activations/layer19_attention_weight_max": 24.493667602539062,
      "activations/layer19_attention_weight_min": -23.15830421447754,
      "activations/layer1_attention_weight_max": 15.64698314666748,
      "activations/layer1_attention_weight_min": -14.011197090148926,
      "activations/layer20_attention_weight_max": 23.040000915527344,
      "activations/layer20_attention_weight_min": -22.098417282104492,
      "activations/layer21_attention_weight_max": 33.921051025390625,
      "activations/layer21_attention_weight_min": -20.415542602539062,
      "activations/layer22_attention_weight_max": 27.8817195892334,
      "activations/layer22_attention_weight_min": -26.10915756225586,
      "activations/layer23_attention_weight_max": 32.96018981933594,
      "activations/layer23_attention_weight_min": -21.718603134155273,
      "activations/layer2_attention_weight_max": 31.164417266845703,
      "activations/layer2_attention_weight_min": -29.938167572021484,
      "activations/layer3_attention_weight_max": 85.31396484375,
      "activations/layer3_attention_weight_min": -88.29894256591797,
      "activations/layer4_attention_weight_max": 81.07939910888672,
      "activations/layer4_attention_weight_min": -83.96367645263672,
      "activations/layer5_attention_weight_max": 62.74004364013672,
      "activations/layer5_attention_weight_min": -74.20237731933594,
      "activations/layer6_attention_weight_max": 49.38800811767578,
      "activations/layer6_attention_weight_min": -50.39807891845703,
      "activations/layer7_attention_weight_max": 69.08291625976562,
      "activations/layer7_attention_weight_min": -67.96973419189453,
      "activations/layer8_attention_weight_max": 48.96053695678711,
      "activations/layer8_attention_weight_min": -54.273643493652344,
      "activations/layer9_attention_weight_max": 54.34859085083008,
      "activations/layer9_attention_weight_min": -48.110836029052734,
      "epoch": 21.69,
      "learning_rate": 1.0245833333333331e-05,
      "loss": 2.7081,
      "step": 373250
    },
    {
      "activations/layer0_attention_weight_max": 16.6248722076416,
      "activations/layer0_attention_weight_min": -14.2672700881958,
      "activations/layer10_attention_weight_max": 34.42695999145508,
      "activations/layer10_attention_weight_min": -34.90475082397461,
      "activations/layer11_attention_weight_max": 32.182044982910156,
      "activations/layer11_attention_weight_min": -34.380435943603516,
      "activations/layer12_attention_weight_max": 18.155601501464844,
      "activations/layer12_attention_weight_min": -27.046613693237305,
      "activations/layer13_attention_weight_max": 31.824037551879883,
      "activations/layer13_attention_weight_min": -27.468835830688477,
      "activations/layer14_attention_weight_max": 34.68464279174805,
      "activations/layer14_attention_weight_min": -30.256044387817383,
      "activations/layer15_attention_weight_max": 32.35694885253906,
      "activations/layer15_attention_weight_min": -30.77063751220703,
      "activations/layer16_attention_weight_max": 34.1525764465332,
      "activations/layer16_attention_weight_min": -32.123748779296875,
      "activations/layer17_attention_weight_max": 49.55561828613281,
      "activations/layer17_attention_weight_min": -48.22027587890625,
      "activations/layer18_attention_weight_max": 44.767173767089844,
      "activations/layer18_attention_weight_min": -42.81226348876953,
      "activations/layer19_attention_weight_max": 23.968900680541992,
      "activations/layer19_attention_weight_min": -24.210060119628906,
      "activations/layer1_attention_weight_max": 14.961812973022461,
      "activations/layer1_attention_weight_min": -14.570740699768066,
      "activations/layer20_attention_weight_max": 21.839733123779297,
      "activations/layer20_attention_weight_min": -23.205434799194336,
      "activations/layer21_attention_weight_max": 38.127716064453125,
      "activations/layer21_attention_weight_min": -24.048851013183594,
      "activations/layer22_attention_weight_max": 28.43342399597168,
      "activations/layer22_attention_weight_min": -28.400062561035156,
      "activations/layer23_attention_weight_max": 33.948524475097656,
      "activations/layer23_attention_weight_min": -23.863754272460938,
      "activations/layer2_attention_weight_max": 31.554874420166016,
      "activations/layer2_attention_weight_min": -32.05720520019531,
      "activations/layer3_attention_weight_max": 88.09004974365234,
      "activations/layer3_attention_weight_min": -90.78471374511719,
      "activations/layer4_attention_weight_max": 83.83380889892578,
      "activations/layer4_attention_weight_min": -85.31902313232422,
      "activations/layer5_attention_weight_max": 65.31018829345703,
      "activations/layer5_attention_weight_min": -70.31802368164062,
      "activations/layer6_attention_weight_max": 49.18193817138672,
      "activations/layer6_attention_weight_min": -49.31569290161133,
      "activations/layer7_attention_weight_max": 66.69526672363281,
      "activations/layer7_attention_weight_min": -66.78325653076172,
      "activations/layer8_attention_weight_max": 46.79469680786133,
      "activations/layer8_attention_weight_min": -48.70077896118164,
      "activations/layer9_attention_weight_max": 43.2216682434082,
      "activations/layer9_attention_weight_min": -44.25565719604492,
      "epoch": 21.69,
      "learning_rate": 1.0226893939393938e-05,
      "loss": 2.7181,
      "step": 373300
    },
    {
      "activations/layer0_attention_weight_max": 16.344757080078125,
      "activations/layer0_attention_weight_min": -14.425256729125977,
      "activations/layer10_attention_weight_max": 35.83156204223633,
      "activations/layer10_attention_weight_min": -35.880489349365234,
      "activations/layer11_attention_weight_max": 35.281898498535156,
      "activations/layer11_attention_weight_min": -33.674560546875,
      "activations/layer12_attention_weight_max": 18.564533233642578,
      "activations/layer12_attention_weight_min": -26.308191299438477,
      "activations/layer13_attention_weight_max": 33.928436279296875,
      "activations/layer13_attention_weight_min": -30.5954532623291,
      "activations/layer14_attention_weight_max": 35.13593292236328,
      "activations/layer14_attention_weight_min": -31.143400192260742,
      "activations/layer15_attention_weight_max": 33.47476577758789,
      "activations/layer15_attention_weight_min": -32.16840744018555,
      "activations/layer16_attention_weight_max": 33.78031921386719,
      "activations/layer16_attention_weight_min": -32.744140625,
      "activations/layer17_attention_weight_max": 50.33883285522461,
      "activations/layer17_attention_weight_min": -47.473934173583984,
      "activations/layer18_attention_weight_max": 48.72594451904297,
      "activations/layer18_attention_weight_min": -43.64201736450195,
      "activations/layer19_attention_weight_max": 26.130157470703125,
      "activations/layer19_attention_weight_min": -26.30527687072754,
      "activations/layer1_attention_weight_max": 16.024429321289062,
      "activations/layer1_attention_weight_min": -14.220000267028809,
      "activations/layer20_attention_weight_max": 26.94182014465332,
      "activations/layer20_attention_weight_min": -25.504356384277344,
      "activations/layer21_attention_weight_max": 37.40977096557617,
      "activations/layer21_attention_weight_min": -26.543386459350586,
      "activations/layer22_attention_weight_max": 26.978580474853516,
      "activations/layer22_attention_weight_min": -28.02229881286621,
      "activations/layer23_attention_weight_max": 33.24188995361328,
      "activations/layer23_attention_weight_min": -25.93042755126953,
      "activations/layer2_attention_weight_max": 31.875045776367188,
      "activations/layer2_attention_weight_min": -29.59200096130371,
      "activations/layer3_attention_weight_max": 87.17346954345703,
      "activations/layer3_attention_weight_min": -87.40067291259766,
      "activations/layer4_attention_weight_max": 81.54692840576172,
      "activations/layer4_attention_weight_min": -84.92764282226562,
      "activations/layer5_attention_weight_max": 59.45368957519531,
      "activations/layer5_attention_weight_min": -70.2494125366211,
      "activations/layer6_attention_weight_max": 46.41328430175781,
      "activations/layer6_attention_weight_min": -48.42279815673828,
      "activations/layer7_attention_weight_max": 70.71668243408203,
      "activations/layer7_attention_weight_min": -67.50605773925781,
      "activations/layer8_attention_weight_max": 47.65159225463867,
      "activations/layer8_attention_weight_min": -51.91883850097656,
      "activations/layer9_attention_weight_max": 44.103755950927734,
      "activations/layer9_attention_weight_min": -47.203060150146484,
      "epoch": 21.69,
      "learning_rate": 1.0207954545454546e-05,
      "loss": 2.6902,
      "step": 373350
    },
    {
      "activations/layer0_attention_weight_max": 15.002248764038086,
      "activations/layer0_attention_weight_min": -14.112947463989258,
      "activations/layer10_attention_weight_max": 36.500667572021484,
      "activations/layer10_attention_weight_min": -37.84021759033203,
      "activations/layer11_attention_weight_max": 35.912105560302734,
      "activations/layer11_attention_weight_min": -36.43279266357422,
      "activations/layer12_attention_weight_max": 19.425146102905273,
      "activations/layer12_attention_weight_min": -28.03075408935547,
      "activations/layer13_attention_weight_max": 33.06947326660156,
      "activations/layer13_attention_weight_min": -28.85402488708496,
      "activations/layer14_attention_weight_max": 42.865074157714844,
      "activations/layer14_attention_weight_min": -30.890670776367188,
      "activations/layer15_attention_weight_max": 36.82077407836914,
      "activations/layer15_attention_weight_min": -30.88345718383789,
      "activations/layer16_attention_weight_max": 33.96133804321289,
      "activations/layer16_attention_weight_min": -32.19532012939453,
      "activations/layer17_attention_weight_max": 52.22608184814453,
      "activations/layer17_attention_weight_min": -47.95772171020508,
      "activations/layer18_attention_weight_max": 46.31996536254883,
      "activations/layer18_attention_weight_min": -44.355613708496094,
      "activations/layer19_attention_weight_max": 23.635236740112305,
      "activations/layer19_attention_weight_min": -26.443822860717773,
      "activations/layer1_attention_weight_max": 16.314006805419922,
      "activations/layer1_attention_weight_min": -14.367278099060059,
      "activations/layer20_attention_weight_max": 21.926271438598633,
      "activations/layer20_attention_weight_min": -25.87665367126465,
      "activations/layer21_attention_weight_max": 38.69761657714844,
      "activations/layer21_attention_weight_min": -30.256235122680664,
      "activations/layer22_attention_weight_max": 27.714433670043945,
      "activations/layer22_attention_weight_min": -26.168428421020508,
      "activations/layer23_attention_weight_max": 34.248779296875,
      "activations/layer23_attention_weight_min": -24.405921936035156,
      "activations/layer2_attention_weight_max": 33.79084014892578,
      "activations/layer2_attention_weight_min": -30.32544708251953,
      "activations/layer3_attention_weight_max": 87.34915161132812,
      "activations/layer3_attention_weight_min": -87.41903686523438,
      "activations/layer4_attention_weight_max": 86.88516998291016,
      "activations/layer4_attention_weight_min": -88.28652954101562,
      "activations/layer5_attention_weight_max": 61.265281677246094,
      "activations/layer5_attention_weight_min": -71.5389404296875,
      "activations/layer6_attention_weight_max": 50.647178649902344,
      "activations/layer6_attention_weight_min": -52.70870590209961,
      "activations/layer7_attention_weight_max": 67.0300521850586,
      "activations/layer7_attention_weight_min": -67.76343536376953,
      "activations/layer8_attention_weight_max": 49.409358978271484,
      "activations/layer8_attention_weight_min": -52.2842903137207,
      "activations/layer9_attention_weight_max": 48.78549575805664,
      "activations/layer9_attention_weight_min": -46.85039138793945,
      "epoch": 21.7,
      "learning_rate": 1.018901515151515e-05,
      "loss": 2.7044,
      "step": 373400
    },
    {
      "activations/layer0_attention_weight_max": 16.01279067993164,
      "activations/layer0_attention_weight_min": -14.585747718811035,
      "activations/layer10_attention_weight_max": 36.00484085083008,
      "activations/layer10_attention_weight_min": -35.76897048950195,
      "activations/layer11_attention_weight_max": 32.47235870361328,
      "activations/layer11_attention_weight_min": -36.11315155029297,
      "activations/layer12_attention_weight_max": 19.077619552612305,
      "activations/layer12_attention_weight_min": -21.675704956054688,
      "activations/layer13_attention_weight_max": 30.760032653808594,
      "activations/layer13_attention_weight_min": -24.817058563232422,
      "activations/layer14_attention_weight_max": 34.41856384277344,
      "activations/layer14_attention_weight_min": -28.151456832885742,
      "activations/layer15_attention_weight_max": 30.107810974121094,
      "activations/layer15_attention_weight_min": -28.453744888305664,
      "activations/layer16_attention_weight_max": 32.61869430541992,
      "activations/layer16_attention_weight_min": -30.525175094604492,
      "activations/layer17_attention_weight_max": 49.73472595214844,
      "activations/layer17_attention_weight_min": -43.236961364746094,
      "activations/layer18_attention_weight_max": 43.2280387878418,
      "activations/layer18_attention_weight_min": -39.30476379394531,
      "activations/layer19_attention_weight_max": 23.25430679321289,
      "activations/layer19_attention_weight_min": -21.182598114013672,
      "activations/layer1_attention_weight_max": 15.172216415405273,
      "activations/layer1_attention_weight_min": -14.330339431762695,
      "activations/layer20_attention_weight_max": 20.42181396484375,
      "activations/layer20_attention_weight_min": -23.407316207885742,
      "activations/layer21_attention_weight_max": 32.32578659057617,
      "activations/layer21_attention_weight_min": -20.3408145904541,
      "activations/layer22_attention_weight_max": 27.290525436401367,
      "activations/layer22_attention_weight_min": -23.942890167236328,
      "activations/layer23_attention_weight_max": 33.32524108886719,
      "activations/layer23_attention_weight_min": -26.360815048217773,
      "activations/layer2_attention_weight_max": 32.11250305175781,
      "activations/layer2_attention_weight_min": -29.828880310058594,
      "activations/layer3_attention_weight_max": 90.17620086669922,
      "activations/layer3_attention_weight_min": -86.23529815673828,
      "activations/layer4_attention_weight_max": 85.22457885742188,
      "activations/layer4_attention_weight_min": -83.98355865478516,
      "activations/layer5_attention_weight_max": 60.940635681152344,
      "activations/layer5_attention_weight_min": -73.07562255859375,
      "activations/layer6_attention_weight_max": 50.389705657958984,
      "activations/layer6_attention_weight_min": -51.47725296020508,
      "activations/layer7_attention_weight_max": 68.76599884033203,
      "activations/layer7_attention_weight_min": -70.38780975341797,
      "activations/layer8_attention_weight_max": 47.40856170654297,
      "activations/layer8_attention_weight_min": -50.551963806152344,
      "activations/layer9_attention_weight_max": 47.80644226074219,
      "activations/layer9_attention_weight_min": -47.35285949707031,
      "epoch": 21.7,
      "learning_rate": 1.0170075757575757e-05,
      "loss": 2.7247,
      "step": 373450
    },
    {
      "activations/layer0_attention_weight_max": 17.161806106567383,
      "activations/layer0_attention_weight_min": -14.235596656799316,
      "activations/layer10_attention_weight_max": 37.827423095703125,
      "activations/layer10_attention_weight_min": -37.52190017700195,
      "activations/layer11_attention_weight_max": 34.582801818847656,
      "activations/layer11_attention_weight_min": -37.08567810058594,
      "activations/layer12_attention_weight_max": 18.45989227294922,
      "activations/layer12_attention_weight_min": -22.621944427490234,
      "activations/layer13_attention_weight_max": 35.70511245727539,
      "activations/layer13_attention_weight_min": -26.763256072998047,
      "activations/layer14_attention_weight_max": 33.7698974609375,
      "activations/layer14_attention_weight_min": -27.881427764892578,
      "activations/layer15_attention_weight_max": 32.309608459472656,
      "activations/layer15_attention_weight_min": -28.700838088989258,
      "activations/layer16_attention_weight_max": 33.319236755371094,
      "activations/layer16_attention_weight_min": -34.43996047973633,
      "activations/layer17_attention_weight_max": 50.805904388427734,
      "activations/layer17_attention_weight_min": -47.245601654052734,
      "activations/layer18_attention_weight_max": 46.40066909790039,
      "activations/layer18_attention_weight_min": -42.045005798339844,
      "activations/layer19_attention_weight_max": 22.727182388305664,
      "activations/layer19_attention_weight_min": -23.43543815612793,
      "activations/layer1_attention_weight_max": 15.316366195678711,
      "activations/layer1_attention_weight_min": -14.980219841003418,
      "activations/layer20_attention_weight_max": 21.675310134887695,
      "activations/layer20_attention_weight_min": -20.97252082824707,
      "activations/layer21_attention_weight_max": 33.476722717285156,
      "activations/layer21_attention_weight_min": -21.54442024230957,
      "activations/layer22_attention_weight_max": 27.255626678466797,
      "activations/layer22_attention_weight_min": -23.47610855102539,
      "activations/layer23_attention_weight_max": 34.63536071777344,
      "activations/layer23_attention_weight_min": -25.317096710205078,
      "activations/layer2_attention_weight_max": 32.65387725830078,
      "activations/layer2_attention_weight_min": -31.992923736572266,
      "activations/layer3_attention_weight_max": 93.68295288085938,
      "activations/layer3_attention_weight_min": -93.50080108642578,
      "activations/layer4_attention_weight_max": 88.92256927490234,
      "activations/layer4_attention_weight_min": -93.13749694824219,
      "activations/layer5_attention_weight_max": 66.16299438476562,
      "activations/layer5_attention_weight_min": -75.72390747070312,
      "activations/layer6_attention_weight_max": 53.20354461669922,
      "activations/layer6_attention_weight_min": -54.252525329589844,
      "activations/layer7_attention_weight_max": 74.87999725341797,
      "activations/layer7_attention_weight_min": -69.77420043945312,
      "activations/layer8_attention_weight_max": 50.1287956237793,
      "activations/layer8_attention_weight_min": -52.39168167114258,
      "activations/layer9_attention_weight_max": 44.52595901489258,
      "activations/layer9_attention_weight_min": -48.2137336730957,
      "epoch": 21.7,
      "learning_rate": 1.0151136363636364e-05,
      "loss": 2.7106,
      "step": 373500
    },
    {
      "activations/layer0_attention_weight_max": 16.343223571777344,
      "activations/layer0_attention_weight_min": -14.378179550170898,
      "activations/layer10_attention_weight_max": 33.18159103393555,
      "activations/layer10_attention_weight_min": -36.272071838378906,
      "activations/layer11_attention_weight_max": 31.351316452026367,
      "activations/layer11_attention_weight_min": -34.16447448730469,
      "activations/layer12_attention_weight_max": 18.11103057861328,
      "activations/layer12_attention_weight_min": -24.4039306640625,
      "activations/layer13_attention_weight_max": 29.17949867248535,
      "activations/layer13_attention_weight_min": -26.759668350219727,
      "activations/layer14_attention_weight_max": 31.41161346435547,
      "activations/layer14_attention_weight_min": -32.688934326171875,
      "activations/layer15_attention_weight_max": 30.151630401611328,
      "activations/layer15_attention_weight_min": -30.928377151489258,
      "activations/layer16_attention_weight_max": 33.00235366821289,
      "activations/layer16_attention_weight_min": -30.63690948486328,
      "activations/layer17_attention_weight_max": 46.106327056884766,
      "activations/layer17_attention_weight_min": -43.01277542114258,
      "activations/layer18_attention_weight_max": 42.407657623291016,
      "activations/layer18_attention_weight_min": -37.82109451293945,
      "activations/layer19_attention_weight_max": 21.30854606628418,
      "activations/layer19_attention_weight_min": -21.253908157348633,
      "activations/layer1_attention_weight_max": 15.420308113098145,
      "activations/layer1_attention_weight_min": -14.062538146972656,
      "activations/layer20_attention_weight_max": 21.116352081298828,
      "activations/layer20_attention_weight_min": -20.959537506103516,
      "activations/layer21_attention_weight_max": 32.74304962158203,
      "activations/layer21_attention_weight_min": -19.415815353393555,
      "activations/layer22_attention_weight_max": 26.47052574157715,
      "activations/layer22_attention_weight_min": -24.925800323486328,
      "activations/layer23_attention_weight_max": 33.28422546386719,
      "activations/layer23_attention_weight_min": -23.352834701538086,
      "activations/layer2_attention_weight_max": 30.832921981811523,
      "activations/layer2_attention_weight_min": -30.07367706298828,
      "activations/layer3_attention_weight_max": 88.33885955810547,
      "activations/layer3_attention_weight_min": -88.24739837646484,
      "activations/layer4_attention_weight_max": 86.81624603271484,
      "activations/layer4_attention_weight_min": -87.31712341308594,
      "activations/layer5_attention_weight_max": 60.95719909667969,
      "activations/layer5_attention_weight_min": -72.68196105957031,
      "activations/layer6_attention_weight_max": 50.6483268737793,
      "activations/layer6_attention_weight_min": -50.59798812866211,
      "activations/layer7_attention_weight_max": 66.56104278564453,
      "activations/layer7_attention_weight_min": -65.12773132324219,
      "activations/layer8_attention_weight_max": 45.589263916015625,
      "activations/layer8_attention_weight_min": -50.56709671020508,
      "activations/layer9_attention_weight_max": 41.04030990600586,
      "activations/layer9_attention_weight_min": -45.615318298339844,
      "epoch": 21.71,
      "learning_rate": 1.0132575757575758e-05,
      "loss": 2.7024,
      "step": 373550
    },
    {
      "activations/layer0_attention_weight_max": 16.0308895111084,
      "activations/layer0_attention_weight_min": -14.53701114654541,
      "activations/layer10_attention_weight_max": 35.4884033203125,
      "activations/layer10_attention_weight_min": -34.96782302856445,
      "activations/layer11_attention_weight_max": 33.36215591430664,
      "activations/layer11_attention_weight_min": -35.64794921875,
      "activations/layer12_attention_weight_max": 18.668689727783203,
      "activations/layer12_attention_weight_min": -24.818647384643555,
      "activations/layer13_attention_weight_max": 33.969390869140625,
      "activations/layer13_attention_weight_min": -30.319015502929688,
      "activations/layer14_attention_weight_max": 35.19630432128906,
      "activations/layer14_attention_weight_min": -30.45132827758789,
      "activations/layer15_attention_weight_max": 35.33106231689453,
      "activations/layer15_attention_weight_min": -30.410743713378906,
      "activations/layer16_attention_weight_max": 35.73044204711914,
      "activations/layer16_attention_weight_min": -33.779266357421875,
      "activations/layer17_attention_weight_max": 52.884849548339844,
      "activations/layer17_attention_weight_min": -47.0595817565918,
      "activations/layer18_attention_weight_max": 47.00099563598633,
      "activations/layer18_attention_weight_min": -40.23447799682617,
      "activations/layer19_attention_weight_max": 25.177207946777344,
      "activations/layer19_attention_weight_min": -22.713151931762695,
      "activations/layer1_attention_weight_max": 16.910810470581055,
      "activations/layer1_attention_weight_min": -14.693591117858887,
      "activations/layer20_attention_weight_max": 22.811742782592773,
      "activations/layer20_attention_weight_min": -21.421052932739258,
      "activations/layer21_attention_weight_max": 32.6624641418457,
      "activations/layer21_attention_weight_min": -24.153188705444336,
      "activations/layer22_attention_weight_max": 29.392480850219727,
      "activations/layer22_attention_weight_min": -25.684513092041016,
      "activations/layer23_attention_weight_max": 35.32673645019531,
      "activations/layer23_attention_weight_min": -22.962980270385742,
      "activations/layer2_attention_weight_max": 31.313987731933594,
      "activations/layer2_attention_weight_min": -30.309566497802734,
      "activations/layer3_attention_weight_max": 90.52272033691406,
      "activations/layer3_attention_weight_min": -95.23860168457031,
      "activations/layer4_attention_weight_max": 86.21543884277344,
      "activations/layer4_attention_weight_min": -90.1430892944336,
      "activations/layer5_attention_weight_max": 66.0146484375,
      "activations/layer5_attention_weight_min": -77.807861328125,
      "activations/layer6_attention_weight_max": 50.51178741455078,
      "activations/layer6_attention_weight_min": -51.35716247558594,
      "activations/layer7_attention_weight_max": 71.86780548095703,
      "activations/layer7_attention_weight_min": -73.71398162841797,
      "activations/layer8_attention_weight_max": 48.87865447998047,
      "activations/layer8_attention_weight_min": -50.75865173339844,
      "activations/layer9_attention_weight_max": 43.33328628540039,
      "activations/layer9_attention_weight_min": -46.59476852416992,
      "epoch": 21.71,
      "learning_rate": 1.0113636363636363e-05,
      "loss": 2.7219,
      "step": 373600
    },
    {
      "activations/layer0_attention_weight_max": 15.629368782043457,
      "activations/layer0_attention_weight_min": -14.618295669555664,
      "activations/layer10_attention_weight_max": 35.902503967285156,
      "activations/layer10_attention_weight_min": -35.61537170410156,
      "activations/layer11_attention_weight_max": 33.9434814453125,
      "activations/layer11_attention_weight_min": -35.24977111816406,
      "activations/layer12_attention_weight_max": 18.19455337524414,
      "activations/layer12_attention_weight_min": -23.26569175720215,
      "activations/layer13_attention_weight_max": 43.607059478759766,
      "activations/layer13_attention_weight_min": -28.40335464477539,
      "activations/layer14_attention_weight_max": 37.757686614990234,
      "activations/layer14_attention_weight_min": -32.04747772216797,
      "activations/layer15_attention_weight_max": 33.15340042114258,
      "activations/layer15_attention_weight_min": -32.71440505981445,
      "activations/layer16_attention_weight_max": 36.421958923339844,
      "activations/layer16_attention_weight_min": -34.63962173461914,
      "activations/layer17_attention_weight_max": 54.62586975097656,
      "activations/layer17_attention_weight_min": -47.9459114074707,
      "activations/layer18_attention_weight_max": 55.35757064819336,
      "activations/layer18_attention_weight_min": -41.00386047363281,
      "activations/layer19_attention_weight_max": 25.02326202392578,
      "activations/layer19_attention_weight_min": -23.7966251373291,
      "activations/layer1_attention_weight_max": 14.94813060760498,
      "activations/layer1_attention_weight_min": -14.721096992492676,
      "activations/layer20_attention_weight_max": 22.74173355102539,
      "activations/layer20_attention_weight_min": -24.311595916748047,
      "activations/layer21_attention_weight_max": 33.25422286987305,
      "activations/layer21_attention_weight_min": -24.559776306152344,
      "activations/layer22_attention_weight_max": 29.841569900512695,
      "activations/layer22_attention_weight_min": -24.72197914123535,
      "activations/layer23_attention_weight_max": 40.8887939453125,
      "activations/layer23_attention_weight_min": -25.77301788330078,
      "activations/layer2_attention_weight_max": 32.41551971435547,
      "activations/layer2_attention_weight_min": -30.87970733642578,
      "activations/layer3_attention_weight_max": 88.87189483642578,
      "activations/layer3_attention_weight_min": -87.45442199707031,
      "activations/layer4_attention_weight_max": 85.969482421875,
      "activations/layer4_attention_weight_min": -85.65755462646484,
      "activations/layer5_attention_weight_max": 64.00361633300781,
      "activations/layer5_attention_weight_min": -71.95757293701172,
      "activations/layer6_attention_weight_max": 51.33530807495117,
      "activations/layer6_attention_weight_min": -51.824771881103516,
      "activations/layer7_attention_weight_max": 73.83576965332031,
      "activations/layer7_attention_weight_min": -71.04833221435547,
      "activations/layer8_attention_weight_max": 48.311832427978516,
      "activations/layer8_attention_weight_min": -52.56768035888672,
      "activations/layer9_attention_weight_max": 47.568233489990234,
      "activations/layer9_attention_weight_min": -46.08890914916992,
      "epoch": 21.71,
      "learning_rate": 1.009469696969697e-05,
      "loss": 2.7101,
      "step": 373650
    },
    {
      "activations/layer0_attention_weight_max": 15.586311340332031,
      "activations/layer0_attention_weight_min": -14.315458297729492,
      "activations/layer10_attention_weight_max": 35.236000061035156,
      "activations/layer10_attention_weight_min": -35.54424285888672,
      "activations/layer11_attention_weight_max": 35.108577728271484,
      "activations/layer11_attention_weight_min": -35.14276885986328,
      "activations/layer12_attention_weight_max": 19.883432388305664,
      "activations/layer12_attention_weight_min": -26.311769485473633,
      "activations/layer13_attention_weight_max": 41.370662689208984,
      "activations/layer13_attention_weight_min": -29.964065551757812,
      "activations/layer14_attention_weight_max": 43.64732360839844,
      "activations/layer14_attention_weight_min": -35.17991256713867,
      "activations/layer15_attention_weight_max": 42.86406326293945,
      "activations/layer15_attention_weight_min": -32.738525390625,
      "activations/layer16_attention_weight_max": 42.73395919799805,
      "activations/layer16_attention_weight_min": -37.372989654541016,
      "activations/layer17_attention_weight_max": 63.30086135864258,
      "activations/layer17_attention_weight_min": -54.77121353149414,
      "activations/layer18_attention_weight_max": 58.011417388916016,
      "activations/layer18_attention_weight_min": -46.215152740478516,
      "activations/layer19_attention_weight_max": 30.137847900390625,
      "activations/layer19_attention_weight_min": -24.682876586914062,
      "activations/layer1_attention_weight_max": 16.266098022460938,
      "activations/layer1_attention_weight_min": -14.795900344848633,
      "activations/layer20_attention_weight_max": 27.755163192749023,
      "activations/layer20_attention_weight_min": -22.496374130249023,
      "activations/layer21_attention_weight_max": 46.214630126953125,
      "activations/layer21_attention_weight_min": -23.439193725585938,
      "activations/layer22_attention_weight_max": 31.383148193359375,
      "activations/layer22_attention_weight_min": -24.40907096862793,
      "activations/layer23_attention_weight_max": 36.38597869873047,
      "activations/layer23_attention_weight_min": -24.501113891601562,
      "activations/layer2_attention_weight_max": 30.96762466430664,
      "activations/layer2_attention_weight_min": -29.48469352722168,
      "activations/layer3_attention_weight_max": 89.30133819580078,
      "activations/layer3_attention_weight_min": -86.952880859375,
      "activations/layer4_attention_weight_max": 85.98030090332031,
      "activations/layer4_attention_weight_min": -86.88859558105469,
      "activations/layer5_attention_weight_max": 64.19918060302734,
      "activations/layer5_attention_weight_min": -68.16239166259766,
      "activations/layer6_attention_weight_max": 50.523902893066406,
      "activations/layer6_attention_weight_min": -51.95493698120117,
      "activations/layer7_attention_weight_max": 69.03659057617188,
      "activations/layer7_attention_weight_min": -66.49737548828125,
      "activations/layer8_attention_weight_max": 46.72030258178711,
      "activations/layer8_attention_weight_min": -54.653053283691406,
      "activations/layer9_attention_weight_max": 42.462158203125,
      "activations/layer9_attention_weight_min": -45.85697937011719,
      "epoch": 21.71,
      "learning_rate": 1.0075757575757575e-05,
      "loss": 2.7144,
      "step": 373700
    },
    {
      "activations/layer0_attention_weight_max": 15.502632141113281,
      "activations/layer0_attention_weight_min": -14.330042839050293,
      "activations/layer10_attention_weight_max": 34.850746154785156,
      "activations/layer10_attention_weight_min": -35.04362106323242,
      "activations/layer11_attention_weight_max": 32.10834884643555,
      "activations/layer11_attention_weight_min": -35.15570068359375,
      "activations/layer12_attention_weight_max": 18.411624908447266,
      "activations/layer12_attention_weight_min": -26.449541091918945,
      "activations/layer13_attention_weight_max": 34.09463119506836,
      "activations/layer13_attention_weight_min": -29.400455474853516,
      "activations/layer14_attention_weight_max": 34.834014892578125,
      "activations/layer14_attention_weight_min": -29.962690353393555,
      "activations/layer15_attention_weight_max": 32.98188400268555,
      "activations/layer15_attention_weight_min": -30.848560333251953,
      "activations/layer16_attention_weight_max": 32.93223190307617,
      "activations/layer16_attention_weight_min": -33.36112594604492,
      "activations/layer17_attention_weight_max": 49.66041946411133,
      "activations/layer17_attention_weight_min": -43.73408889770508,
      "activations/layer18_attention_weight_max": 50.504615783691406,
      "activations/layer18_attention_weight_min": -39.8739128112793,
      "activations/layer19_attention_weight_max": 26.69773292541504,
      "activations/layer19_attention_weight_min": -21.647764205932617,
      "activations/layer1_attention_weight_max": 14.303295135498047,
      "activations/layer1_attention_weight_min": -13.604340553283691,
      "activations/layer20_attention_weight_max": 24.75983238220215,
      "activations/layer20_attention_weight_min": -21.824979782104492,
      "activations/layer21_attention_weight_max": 36.868324279785156,
      "activations/layer21_attention_weight_min": -21.765329360961914,
      "activations/layer22_attention_weight_max": 29.506046295166016,
      "activations/layer22_attention_weight_min": -27.52083969116211,
      "activations/layer23_attention_weight_max": 35.98638916015625,
      "activations/layer23_attention_weight_min": -22.76666259765625,
      "activations/layer2_attention_weight_max": 29.794843673706055,
      "activations/layer2_attention_weight_min": -27.88752555847168,
      "activations/layer3_attention_weight_max": 84.01579284667969,
      "activations/layer3_attention_weight_min": -84.85514831542969,
      "activations/layer4_attention_weight_max": 83.35366821289062,
      "activations/layer4_attention_weight_min": -82.53081512451172,
      "activations/layer5_attention_weight_max": 63.37030792236328,
      "activations/layer5_attention_weight_min": -71.14768981933594,
      "activations/layer6_attention_weight_max": 48.40250015258789,
      "activations/layer6_attention_weight_min": -50.50016403198242,
      "activations/layer7_attention_weight_max": 66.42436981201172,
      "activations/layer7_attention_weight_min": -67.26272583007812,
      "activations/layer8_attention_weight_max": 45.92717361450195,
      "activations/layer8_attention_weight_min": -49.21455764770508,
      "activations/layer9_attention_weight_max": 43.423187255859375,
      "activations/layer9_attention_weight_min": -45.134056091308594,
      "epoch": 21.72,
      "learning_rate": 1.0056818181818181e-05,
      "loss": 2.6992,
      "step": 373750
    },
    {
      "activations/layer0_attention_weight_max": 16.135313034057617,
      "activations/layer0_attention_weight_min": -14.412985801696777,
      "activations/layer10_attention_weight_max": 34.369544982910156,
      "activations/layer10_attention_weight_min": -36.42542266845703,
      "activations/layer11_attention_weight_max": 30.035404205322266,
      "activations/layer11_attention_weight_min": -34.67366027832031,
      "activations/layer12_attention_weight_max": 18.051542282104492,
      "activations/layer12_attention_weight_min": -23.959007263183594,
      "activations/layer13_attention_weight_max": 31.41312026977539,
      "activations/layer13_attention_weight_min": -25.946983337402344,
      "activations/layer14_attention_weight_max": 31.477432250976562,
      "activations/layer14_attention_weight_min": -27.599061965942383,
      "activations/layer15_attention_weight_max": 31.452241897583008,
      "activations/layer15_attention_weight_min": -28.54456329345703,
      "activations/layer16_attention_weight_max": 31.96867561340332,
      "activations/layer16_attention_weight_min": -31.32493019104004,
      "activations/layer17_attention_weight_max": 48.485382080078125,
      "activations/layer17_attention_weight_min": -40.9187126159668,
      "activations/layer18_attention_weight_max": 46.213714599609375,
      "activations/layer18_attention_weight_min": -37.965003967285156,
      "activations/layer19_attention_weight_max": 25.429044723510742,
      "activations/layer19_attention_weight_min": -22.89009666442871,
      "activations/layer1_attention_weight_max": 14.642688751220703,
      "activations/layer1_attention_weight_min": -14.703612327575684,
      "activations/layer20_attention_weight_max": 22.752710342407227,
      "activations/layer20_attention_weight_min": -22.634384155273438,
      "activations/layer21_attention_weight_max": 36.9656982421875,
      "activations/layer21_attention_weight_min": -22.228757858276367,
      "activations/layer22_attention_weight_max": 28.5400333404541,
      "activations/layer22_attention_weight_min": -25.29412841796875,
      "activations/layer23_attention_weight_max": 40.18370819091797,
      "activations/layer23_attention_weight_min": -24.669078826904297,
      "activations/layer2_attention_weight_max": 31.9769344329834,
      "activations/layer2_attention_weight_min": -29.44692039489746,
      "activations/layer3_attention_weight_max": 87.79438781738281,
      "activations/layer3_attention_weight_min": -88.46522521972656,
      "activations/layer4_attention_weight_max": 81.83840942382812,
      "activations/layer4_attention_weight_min": -81.64531707763672,
      "activations/layer5_attention_weight_max": 58.68157958984375,
      "activations/layer5_attention_weight_min": -71.25506591796875,
      "activations/layer6_attention_weight_max": 47.29956817626953,
      "activations/layer6_attention_weight_min": -50.314720153808594,
      "activations/layer7_attention_weight_max": 66.15763854980469,
      "activations/layer7_attention_weight_min": -66.60507202148438,
      "activations/layer8_attention_weight_max": 46.250553131103516,
      "activations/layer8_attention_weight_min": -49.44132995605469,
      "activations/layer9_attention_weight_max": 42.41728973388672,
      "activations/layer9_attention_weight_min": -45.34225082397461,
      "epoch": 21.72,
      "learning_rate": 1.0037878787878788e-05,
      "loss": 2.7049,
      "step": 373800
    },
    {
      "activations/layer0_attention_weight_max": 16.17351722717285,
      "activations/layer0_attention_weight_min": -14.68449592590332,
      "activations/layer10_attention_weight_max": 40.69913864135742,
      "activations/layer10_attention_weight_min": -38.567543029785156,
      "activations/layer11_attention_weight_max": 39.528839111328125,
      "activations/layer11_attention_weight_min": -35.435943603515625,
      "activations/layer12_attention_weight_max": 19.1864070892334,
      "activations/layer12_attention_weight_min": -27.256332397460938,
      "activations/layer13_attention_weight_max": 33.135040283203125,
      "activations/layer13_attention_weight_min": -27.962905883789062,
      "activations/layer14_attention_weight_max": 36.816158294677734,
      "activations/layer14_attention_weight_min": -29.695629119873047,
      "activations/layer15_attention_weight_max": 32.52396011352539,
      "activations/layer15_attention_weight_min": -31.59055519104004,
      "activations/layer16_attention_weight_max": 32.72451400756836,
      "activations/layer16_attention_weight_min": -31.044706344604492,
      "activations/layer17_attention_weight_max": 51.00324249267578,
      "activations/layer17_attention_weight_min": -46.16977310180664,
      "activations/layer18_attention_weight_max": 46.27701187133789,
      "activations/layer18_attention_weight_min": -40.58692169189453,
      "activations/layer19_attention_weight_max": 24.972679138183594,
      "activations/layer19_attention_weight_min": -21.38567352294922,
      "activations/layer1_attention_weight_max": 15.774327278137207,
      "activations/layer1_attention_weight_min": -15.46015453338623,
      "activations/layer20_attention_weight_max": 21.952556610107422,
      "activations/layer20_attention_weight_min": -21.36942481994629,
      "activations/layer21_attention_weight_max": 34.1136360168457,
      "activations/layer21_attention_weight_min": -26.48746109008789,
      "activations/layer22_attention_weight_max": 29.715187072753906,
      "activations/layer22_attention_weight_min": -23.840579986572266,
      "activations/layer23_attention_weight_max": 40.13267517089844,
      "activations/layer23_attention_weight_min": -24.639556884765625,
      "activations/layer2_attention_weight_max": 32.07042694091797,
      "activations/layer2_attention_weight_min": -29.60353660583496,
      "activations/layer3_attention_weight_max": 88.07563018798828,
      "activations/layer3_attention_weight_min": -86.85094451904297,
      "activations/layer4_attention_weight_max": 86.6295394897461,
      "activations/layer4_attention_weight_min": -85.69231414794922,
      "activations/layer5_attention_weight_max": 62.656646728515625,
      "activations/layer5_attention_weight_min": -74.29520416259766,
      "activations/layer6_attention_weight_max": 49.53938293457031,
      "activations/layer6_attention_weight_min": -51.84578323364258,
      "activations/layer7_attention_weight_max": 86.3584213256836,
      "activations/layer7_attention_weight_min": -71.09488677978516,
      "activations/layer8_attention_weight_max": 54.25080108642578,
      "activations/layer8_attention_weight_min": -52.22538757324219,
      "activations/layer9_attention_weight_max": 60.53631591796875,
      "activations/layer9_attention_weight_min": -47.85319900512695,
      "epoch": 21.72,
      "learning_rate": 1.0018939393939393e-05,
      "loss": 2.7005,
      "step": 373850
    },
    {
      "activations/layer0_attention_weight_max": 15.881025314331055,
      "activations/layer0_attention_weight_min": -14.840849876403809,
      "activations/layer10_attention_weight_max": 38.485870361328125,
      "activations/layer10_attention_weight_min": -41.19997787475586,
      "activations/layer11_attention_weight_max": 36.77472686767578,
      "activations/layer11_attention_weight_min": -37.91178512573242,
      "activations/layer12_attention_weight_max": 19.579801559448242,
      "activations/layer12_attention_weight_min": -24.61862564086914,
      "activations/layer13_attention_weight_max": 38.054317474365234,
      "activations/layer13_attention_weight_min": -29.791858673095703,
      "activations/layer14_attention_weight_max": 36.94013595581055,
      "activations/layer14_attention_weight_min": -32.031070709228516,
      "activations/layer15_attention_weight_max": 34.38722610473633,
      "activations/layer15_attention_weight_min": -30.752155303955078,
      "activations/layer16_attention_weight_max": 37.628868103027344,
      "activations/layer16_attention_weight_min": -34.723052978515625,
      "activations/layer17_attention_weight_max": 55.832027435302734,
      "activations/layer17_attention_weight_min": -47.762149810791016,
      "activations/layer18_attention_weight_max": 51.93979263305664,
      "activations/layer18_attention_weight_min": -40.95861053466797,
      "activations/layer19_attention_weight_max": 24.017108917236328,
      "activations/layer19_attention_weight_min": -23.362287521362305,
      "activations/layer1_attention_weight_max": 15.503535270690918,
      "activations/layer1_attention_weight_min": -14.112526893615723,
      "activations/layer20_attention_weight_max": 22.601165771484375,
      "activations/layer20_attention_weight_min": -22.823436737060547,
      "activations/layer21_attention_weight_max": 33.9888916015625,
      "activations/layer21_attention_weight_min": -20.335834503173828,
      "activations/layer22_attention_weight_max": 35.70619201660156,
      "activations/layer22_attention_weight_min": -24.195358276367188,
      "activations/layer23_attention_weight_max": 38.95411682128906,
      "activations/layer23_attention_weight_min": -22.861783981323242,
      "activations/layer2_attention_weight_max": 32.10660934448242,
      "activations/layer2_attention_weight_min": -29.833576202392578,
      "activations/layer3_attention_weight_max": 88.63688659667969,
      "activations/layer3_attention_weight_min": -84.31206512451172,
      "activations/layer4_attention_weight_max": 84.62102508544922,
      "activations/layer4_attention_weight_min": -83.45635223388672,
      "activations/layer5_attention_weight_max": 61.38857650756836,
      "activations/layer5_attention_weight_min": -68.36085510253906,
      "activations/layer6_attention_weight_max": 51.8455696105957,
      "activations/layer6_attention_weight_min": -54.04143524169922,
      "activations/layer7_attention_weight_max": 68.52924346923828,
      "activations/layer7_attention_weight_min": -67.85529327392578,
      "activations/layer8_attention_weight_max": 51.2675895690918,
      "activations/layer8_attention_weight_min": -51.023502349853516,
      "activations/layer9_attention_weight_max": 50.72936248779297,
      "activations/layer9_attention_weight_min": -47.81496047973633,
      "epoch": 21.73,
      "learning_rate": 9.999999999999999e-06,
      "loss": 2.7121,
      "step": 373900
    },
    {
      "activations/layer0_attention_weight_max": 15.994654655456543,
      "activations/layer0_attention_weight_min": -14.524288177490234,
      "activations/layer10_attention_weight_max": 37.97723388671875,
      "activations/layer10_attention_weight_min": -37.44683837890625,
      "activations/layer11_attention_weight_max": 33.32537841796875,
      "activations/layer11_attention_weight_min": -35.69898223876953,
      "activations/layer12_attention_weight_max": 20.49307632446289,
      "activations/layer12_attention_weight_min": -24.494901657104492,
      "activations/layer13_attention_weight_max": 38.402793884277344,
      "activations/layer13_attention_weight_min": -27.315458297729492,
      "activations/layer14_attention_weight_max": 36.842201232910156,
      "activations/layer14_attention_weight_min": -32.24082946777344,
      "activations/layer15_attention_weight_max": 35.112735748291016,
      "activations/layer15_attention_weight_min": -31.695392608642578,
      "activations/layer16_attention_weight_max": 37.20407485961914,
      "activations/layer16_attention_weight_min": -36.03465270996094,
      "activations/layer17_attention_weight_max": 54.48173522949219,
      "activations/layer17_attention_weight_min": -50.8092155456543,
      "activations/layer18_attention_weight_max": 51.99404525756836,
      "activations/layer18_attention_weight_min": -47.99003601074219,
      "activations/layer19_attention_weight_max": 28.206274032592773,
      "activations/layer19_attention_weight_min": -25.191606521606445,
      "activations/layer1_attention_weight_max": 15.086477279663086,
      "activations/layer1_attention_weight_min": -14.972027778625488,
      "activations/layer20_attention_weight_max": 26.160423278808594,
      "activations/layer20_attention_weight_min": -23.123098373413086,
      "activations/layer21_attention_weight_max": 41.04802322387695,
      "activations/layer21_attention_weight_min": -22.7769718170166,
      "activations/layer22_attention_weight_max": 30.32245635986328,
      "activations/layer22_attention_weight_min": -27.560503005981445,
      "activations/layer23_attention_weight_max": 36.293426513671875,
      "activations/layer23_attention_weight_min": -25.1341552734375,
      "activations/layer2_attention_weight_max": 30.998523712158203,
      "activations/layer2_attention_weight_min": -30.374380111694336,
      "activations/layer3_attention_weight_max": 87.92876434326172,
      "activations/layer3_attention_weight_min": -88.79391479492188,
      "activations/layer4_attention_weight_max": 86.2647705078125,
      "activations/layer4_attention_weight_min": -86.11201477050781,
      "activations/layer5_attention_weight_max": 65.21994018554688,
      "activations/layer5_attention_weight_min": -72.31681823730469,
      "activations/layer6_attention_weight_max": 49.47147750854492,
      "activations/layer6_attention_weight_min": -52.579471588134766,
      "activations/layer7_attention_weight_max": 67.7337646484375,
      "activations/layer7_attention_weight_min": -68.67835235595703,
      "activations/layer8_attention_weight_max": 47.92713165283203,
      "activations/layer8_attention_weight_min": -54.10254669189453,
      "activations/layer9_attention_weight_max": 44.33517074584961,
      "activations/layer9_attention_weight_min": -48.113746643066406,
      "epoch": 21.73,
      "learning_rate": 9.981060606060606e-06,
      "loss": 2.7143,
      "step": 373950
    },
    {
      "activations/layer0_attention_weight_max": 16.123064041137695,
      "activations/layer0_attention_weight_min": -14.335796356201172,
      "activations/layer10_attention_weight_max": 35.97332763671875,
      "activations/layer10_attention_weight_min": -35.77671813964844,
      "activations/layer11_attention_weight_max": 32.234825134277344,
      "activations/layer11_attention_weight_min": -34.64512252807617,
      "activations/layer12_attention_weight_max": 18.28370475769043,
      "activations/layer12_attention_weight_min": -25.255414962768555,
      "activations/layer13_attention_weight_max": 32.5186882019043,
      "activations/layer13_attention_weight_min": -28.65522003173828,
      "activations/layer14_attention_weight_max": 35.07059860229492,
      "activations/layer14_attention_weight_min": -29.94386100769043,
      "activations/layer15_attention_weight_max": 33.02279281616211,
      "activations/layer15_attention_weight_min": -29.99559783935547,
      "activations/layer16_attention_weight_max": 34.026084899902344,
      "activations/layer16_attention_weight_min": -32.27924728393555,
      "activations/layer17_attention_weight_max": 50.1559944152832,
      "activations/layer17_attention_weight_min": -44.28839874267578,
      "activations/layer18_attention_weight_max": 51.32510757446289,
      "activations/layer18_attention_weight_min": -40.9481201171875,
      "activations/layer19_attention_weight_max": 24.176240921020508,
      "activations/layer19_attention_weight_min": -23.030494689941406,
      "activations/layer1_attention_weight_max": 15.159128189086914,
      "activations/layer1_attention_weight_min": -14.19713020324707,
      "activations/layer20_attention_weight_max": 24.51797103881836,
      "activations/layer20_attention_weight_min": -21.83246612548828,
      "activations/layer21_attention_weight_max": 37.190460205078125,
      "activations/layer21_attention_weight_min": -25.048110961914062,
      "activations/layer22_attention_weight_max": 27.96240997314453,
      "activations/layer22_attention_weight_min": -25.495752334594727,
      "activations/layer23_attention_weight_max": 38.258811950683594,
      "activations/layer23_attention_weight_min": -25.94312858581543,
      "activations/layer2_attention_weight_max": 31.819990158081055,
      "activations/layer2_attention_weight_min": -30.011089324951172,
      "activations/layer3_attention_weight_max": 89.9941177368164,
      "activations/layer3_attention_weight_min": -91.99419403076172,
      "activations/layer4_attention_weight_max": 84.8779525756836,
      "activations/layer4_attention_weight_min": -86.86898040771484,
      "activations/layer5_attention_weight_max": 60.94887924194336,
      "activations/layer5_attention_weight_min": -73.42181396484375,
      "activations/layer6_attention_weight_max": 49.78862380981445,
      "activations/layer6_attention_weight_min": -51.96672439575195,
      "activations/layer7_attention_weight_max": 68.09648132324219,
      "activations/layer7_attention_weight_min": -68.89191436767578,
      "activations/layer8_attention_weight_max": 45.816490173339844,
      "activations/layer8_attention_weight_min": -53.03034210205078,
      "activations/layer9_attention_weight_max": 44.58773422241211,
      "activations/layer9_attention_weight_min": -47.30249786376953,
      "epoch": 21.73,
      "learning_rate": 9.96212121212121e-06,
      "loss": 2.7205,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4529,
      "eval_samples_per_second": 507.99,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4529,
      "eval_openwebtext_samples_per_second": 507.99,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_wikitext_loss": 2.90625,
      "eval_wikitext_ppl": 18.28808948244362,
      "eval_wikitext_runtime": 1.9946,
      "eval_wikitext_samples_per_second": 228.616,
      "step": 374000
    },
    {
      "epoch": 21.73,
      "eval_lambada_loss": 2.44140625,
      "eval_lambada_ppl": 11.489186055851967,
      "eval_lambada_runtime": 9.5619,
      "eval_lambada_samples_per_second": 509.208,
      "step": 374000
    },
    {
      "activations/layer0_attention_weight_max": 15.489893913269043,
      "activations/layer0_attention_weight_min": -14.417186737060547,
      "activations/layer10_attention_weight_max": 39.99338150024414,
      "activations/layer10_attention_weight_min": -38.862220764160156,
      "activations/layer11_attention_weight_max": 38.570098876953125,
      "activations/layer11_attention_weight_min": -38.76898956298828,
      "activations/layer12_attention_weight_max": 19.60103988647461,
      "activations/layer12_attention_weight_min": -24.035497665405273,
      "activations/layer13_attention_weight_max": 37.216732025146484,
      "activations/layer13_attention_weight_min": -28.95311737060547,
      "activations/layer14_attention_weight_max": 37.73548889160156,
      "activations/layer14_attention_weight_min": -30.509790420532227,
      "activations/layer15_attention_weight_max": 33.85115051269531,
      "activations/layer15_attention_weight_min": -32.59569549560547,
      "activations/layer16_attention_weight_max": 35.24669647216797,
      "activations/layer16_attention_weight_min": -34.12493896484375,
      "activations/layer17_attention_weight_max": 50.59499740600586,
      "activations/layer17_attention_weight_min": -46.272621154785156,
      "activations/layer18_attention_weight_max": 49.35593795776367,
      "activations/layer18_attention_weight_min": -43.80559539794922,
      "activations/layer19_attention_weight_max": 25.08661460876465,
      "activations/layer19_attention_weight_min": -26.023330688476562,
      "activations/layer1_attention_weight_max": 15.558772087097168,
      "activations/layer1_attention_weight_min": -15.874408721923828,
      "activations/layer20_attention_weight_max": 25.19612693786621,
      "activations/layer20_attention_weight_min": -21.51375389099121,
      "activations/layer21_attention_weight_max": 34.583282470703125,
      "activations/layer21_attention_weight_min": -21.79744529724121,
      "activations/layer22_attention_weight_max": 34.54714584350586,
      "activations/layer22_attention_weight_min": -25.5338191986084,
      "activations/layer23_attention_weight_max": 37.682926177978516,
      "activations/layer23_attention_weight_min": -24.98668670654297,
      "activations/layer2_attention_weight_max": 31.891559600830078,
      "activations/layer2_attention_weight_min": -31.01502227783203,
      "activations/layer3_attention_weight_max": 90.33192443847656,
      "activations/layer3_attention_weight_min": -89.45728302001953,
      "activations/layer4_attention_weight_max": 86.03980255126953,
      "activations/layer4_attention_weight_min": -85.06489562988281,
      "activations/layer5_attention_weight_max": 59.69929122924805,
      "activations/layer5_attention_weight_min": -70.2536849975586,
      "activations/layer6_attention_weight_max": 53.86874771118164,
      "activations/layer6_attention_weight_min": -51.63465881347656,
      "activations/layer7_attention_weight_max": 74.66402435302734,
      "activations/layer7_attention_weight_min": -70.96817016601562,
      "activations/layer8_attention_weight_max": 51.51859664916992,
      "activations/layer8_attention_weight_min": -51.860260009765625,
      "activations/layer9_attention_weight_max": 49.718265533447266,
      "activations/layer9_attention_weight_min": -48.58695983886719,
      "epoch": 21.73,
      "learning_rate": 9.943181818181817e-06,
      "loss": 2.7121,
      "step": 374050
    },
    {
      "activations/layer0_attention_weight_max": 16.05838966369629,
      "activations/layer0_attention_weight_min": -14.13361644744873,
      "activations/layer10_attention_weight_max": 36.43746566772461,
      "activations/layer10_attention_weight_min": -38.69782257080078,
      "activations/layer11_attention_weight_max": 35.03315734863281,
      "activations/layer11_attention_weight_min": -40.162654876708984,
      "activations/layer12_attention_weight_max": 20.652774810791016,
      "activations/layer12_attention_weight_min": -24.853363037109375,
      "activations/layer13_attention_weight_max": 38.18608474731445,
      "activations/layer13_attention_weight_min": -30.946931838989258,
      "activations/layer14_attention_weight_max": 37.41411209106445,
      "activations/layer14_attention_weight_min": -32.07552719116211,
      "activations/layer15_attention_weight_max": 35.04424285888672,
      "activations/layer15_attention_weight_min": -32.95133590698242,
      "activations/layer16_attention_weight_max": 37.28288650512695,
      "activations/layer16_attention_weight_min": -34.48793411254883,
      "activations/layer17_attention_weight_max": 54.738162994384766,
      "activations/layer17_attention_weight_min": -47.12934494018555,
      "activations/layer18_attention_weight_max": 53.62141799926758,
      "activations/layer18_attention_weight_min": -41.82557678222656,
      "activations/layer19_attention_weight_max": 29.025785446166992,
      "activations/layer19_attention_weight_min": -24.288227081298828,
      "activations/layer1_attention_weight_max": 14.555829048156738,
      "activations/layer1_attention_weight_min": -16.819665908813477,
      "activations/layer20_attention_weight_max": 25.406526565551758,
      "activations/layer20_attention_weight_min": -23.260194778442383,
      "activations/layer21_attention_weight_max": 38.129215240478516,
      "activations/layer21_attention_weight_min": -23.0361385345459,
      "activations/layer22_attention_weight_max": 30.700071334838867,
      "activations/layer22_attention_weight_min": -27.5248966217041,
      "activations/layer23_attention_weight_max": 39.93198013305664,
      "activations/layer23_attention_weight_min": -24.56838035583496,
      "activations/layer2_attention_weight_max": 30.005163192749023,
      "activations/layer2_attention_weight_min": -29.94522476196289,
      "activations/layer3_attention_weight_max": 90.14703369140625,
      "activations/layer3_attention_weight_min": -91.76433563232422,
      "activations/layer4_attention_weight_max": 86.27526092529297,
      "activations/layer4_attention_weight_min": -84.63526916503906,
      "activations/layer5_attention_weight_max": 65.18345642089844,
      "activations/layer5_attention_weight_min": -72.36991119384766,
      "activations/layer6_attention_weight_max": 51.11562728881836,
      "activations/layer6_attention_weight_min": -50.71940612792969,
      "activations/layer7_attention_weight_max": 71.10723114013672,
      "activations/layer7_attention_weight_min": -68.19752502441406,
      "activations/layer8_attention_weight_max": 50.071258544921875,
      "activations/layer8_attention_weight_min": -52.004512786865234,
      "activations/layer9_attention_weight_max": 45.85224914550781,
      "activations/layer9_attention_weight_min": -48.572444915771484,
      "epoch": 21.74,
      "learning_rate": 9.924242424242422e-06,
      "loss": 2.7069,
      "step": 374100
    },
    {
      "activations/layer0_attention_weight_max": 15.62016773223877,
      "activations/layer0_attention_weight_min": -14.139444351196289,
      "activations/layer10_attention_weight_max": 39.846046447753906,
      "activations/layer10_attention_weight_min": -39.13858413696289,
      "activations/layer11_attention_weight_max": 36.606117248535156,
      "activations/layer11_attention_weight_min": -37.30706024169922,
      "activations/layer12_attention_weight_max": 20.886234283447266,
      "activations/layer12_attention_weight_min": -25.367122650146484,
      "activations/layer13_attention_weight_max": 34.94611358642578,
      "activations/layer13_attention_weight_min": -31.452472686767578,
      "activations/layer14_attention_weight_max": 39.49336242675781,
      "activations/layer14_attention_weight_min": -32.944175720214844,
      "activations/layer15_attention_weight_max": 35.783267974853516,
      "activations/layer15_attention_weight_min": -33.962257385253906,
      "activations/layer16_attention_weight_max": 37.55720520019531,
      "activations/layer16_attention_weight_min": -35.45844650268555,
      "activations/layer17_attention_weight_max": 55.87788009643555,
      "activations/layer17_attention_weight_min": -50.099212646484375,
      "activations/layer18_attention_weight_max": 51.554901123046875,
      "activations/layer18_attention_weight_min": -44.807220458984375,
      "activations/layer19_attention_weight_max": 25.17755126953125,
      "activations/layer19_attention_weight_min": -24.5090389251709,
      "activations/layer1_attention_weight_max": 15.380538940429688,
      "activations/layer1_attention_weight_min": -13.761285781860352,
      "activations/layer20_attention_weight_max": 24.46474838256836,
      "activations/layer20_attention_weight_min": -23.129344940185547,
      "activations/layer21_attention_weight_max": 38.29275131225586,
      "activations/layer21_attention_weight_min": -22.607385635375977,
      "activations/layer22_attention_weight_max": 31.890487670898438,
      "activations/layer22_attention_weight_min": -25.445146560668945,
      "activations/layer23_attention_weight_max": 43.975887298583984,
      "activations/layer23_attention_weight_min": -23.42719078063965,
      "activations/layer2_attention_weight_max": 31.32126235961914,
      "activations/layer2_attention_weight_min": -30.691038131713867,
      "activations/layer3_attention_weight_max": 90.85818481445312,
      "activations/layer3_attention_weight_min": -92.61885833740234,
      "activations/layer4_attention_weight_max": 85.3693618774414,
      "activations/layer4_attention_weight_min": -88.9974594116211,
      "activations/layer5_attention_weight_max": 62.864227294921875,
      "activations/layer5_attention_weight_min": -69.68476104736328,
      "activations/layer6_attention_weight_max": 51.46094512939453,
      "activations/layer6_attention_weight_min": -53.30402755737305,
      "activations/layer7_attention_weight_max": 67.03548431396484,
      "activations/layer7_attention_weight_min": -74.39049530029297,
      "activations/layer8_attention_weight_max": 48.19343566894531,
      "activations/layer8_attention_weight_min": -55.078548431396484,
      "activations/layer9_attention_weight_max": 48.083763122558594,
      "activations/layer9_attention_weight_min": -49.46259689331055,
      "epoch": 21.74,
      "learning_rate": 9.905303030303029e-06,
      "loss": 2.7271,
      "step": 374150
    },
    {
      "activations/layer0_attention_weight_max": 16.344924926757812,
      "activations/layer0_attention_weight_min": -14.494070053100586,
      "activations/layer10_attention_weight_max": 36.9437370300293,
      "activations/layer10_attention_weight_min": -37.66386795043945,
      "activations/layer11_attention_weight_max": 36.42688751220703,
      "activations/layer11_attention_weight_min": -37.09927749633789,
      "activations/layer12_attention_weight_max": 19.519765853881836,
      "activations/layer12_attention_weight_min": -26.01106071472168,
      "activations/layer13_attention_weight_max": 40.45424270629883,
      "activations/layer13_attention_weight_min": -29.409730911254883,
      "activations/layer14_attention_weight_max": 44.95372772216797,
      "activations/layer14_attention_weight_min": -30.52262306213379,
      "activations/layer15_attention_weight_max": 42.97063064575195,
      "activations/layer15_attention_weight_min": -31.966718673706055,
      "activations/layer16_attention_weight_max": 34.746952056884766,
      "activations/layer16_attention_weight_min": -34.838409423828125,
      "activations/layer17_attention_weight_max": 53.88518142700195,
      "activations/layer17_attention_weight_min": -51.608707427978516,
      "activations/layer18_attention_weight_max": 49.38311767578125,
      "activations/layer18_attention_weight_min": -47.882484436035156,
      "activations/layer19_attention_weight_max": 27.00826072692871,
      "activations/layer19_attention_weight_min": -28.084949493408203,
      "activations/layer1_attention_weight_max": 15.46663761138916,
      "activations/layer1_attention_weight_min": -14.426569938659668,
      "activations/layer20_attention_weight_max": 26.99605941772461,
      "activations/layer20_attention_weight_min": -23.322166442871094,
      "activations/layer21_attention_weight_max": 36.48828887939453,
      "activations/layer21_attention_weight_min": -26.762868881225586,
      "activations/layer22_attention_weight_max": 29.758665084838867,
      "activations/layer22_attention_weight_min": -25.27351188659668,
      "activations/layer23_attention_weight_max": 33.387229919433594,
      "activations/layer23_attention_weight_min": -24.264867782592773,
      "activations/layer2_attention_weight_max": 33.16239929199219,
      "activations/layer2_attention_weight_min": -31.845298767089844,
      "activations/layer3_attention_weight_max": 94.31022644042969,
      "activations/layer3_attention_weight_min": -90.37738800048828,
      "activations/layer4_attention_weight_max": 83.20482635498047,
      "activations/layer4_attention_weight_min": -82.06285858154297,
      "activations/layer5_attention_weight_max": 61.34011459350586,
      "activations/layer5_attention_weight_min": -67.35166931152344,
      "activations/layer6_attention_weight_max": 46.856285095214844,
      "activations/layer6_attention_weight_min": -49.29656219482422,
      "activations/layer7_attention_weight_max": 76.30923461914062,
      "activations/layer7_attention_weight_min": -69.12005615234375,
      "activations/layer8_attention_weight_max": 48.0115966796875,
      "activations/layer8_attention_weight_min": -48.88405227661133,
      "activations/layer9_attention_weight_max": 53.89612579345703,
      "activations/layer9_attention_weight_min": -46.65464782714844,
      "epoch": 21.74,
      "learning_rate": 9.886363636363635e-06,
      "loss": 2.7248,
      "step": 374200
    },
    {
      "activations/layer0_attention_weight_max": 16.029335021972656,
      "activations/layer0_attention_weight_min": -14.709954261779785,
      "activations/layer10_attention_weight_max": 35.024330139160156,
      "activations/layer10_attention_weight_min": -34.78864669799805,
      "activations/layer11_attention_weight_max": 32.58465576171875,
      "activations/layer11_attention_weight_min": -34.439903259277344,
      "activations/layer12_attention_weight_max": 19.016189575195312,
      "activations/layer12_attention_weight_min": -24.10888671875,
      "activations/layer13_attention_weight_max": 33.12721252441406,
      "activations/layer13_attention_weight_min": -30.2745418548584,
      "activations/layer14_attention_weight_max": 35.0296516418457,
      "activations/layer14_attention_weight_min": -30.49655532836914,
      "activations/layer15_attention_weight_max": 34.0119743347168,
      "activations/layer15_attention_weight_min": -30.6813907623291,
      "activations/layer16_attention_weight_max": 36.63404083251953,
      "activations/layer16_attention_weight_min": -34.84260940551758,
      "activations/layer17_attention_weight_max": 54.395225524902344,
      "activations/layer17_attention_weight_min": -51.19950485229492,
      "activations/layer18_attention_weight_max": 50.793907165527344,
      "activations/layer18_attention_weight_min": -43.69865417480469,
      "activations/layer19_attention_weight_max": 24.19990348815918,
      "activations/layer19_attention_weight_min": -23.46405601501465,
      "activations/layer1_attention_weight_max": 16.6688232421875,
      "activations/layer1_attention_weight_min": -15.600024223327637,
      "activations/layer20_attention_weight_max": 21.940963745117188,
      "activations/layer20_attention_weight_min": -21.31334686279297,
      "activations/layer21_attention_weight_max": 35.69002151489258,
      "activations/layer21_attention_weight_min": -21.23236656188965,
      "activations/layer22_attention_weight_max": 29.49100112915039,
      "activations/layer22_attention_weight_min": -25.88456916809082,
      "activations/layer23_attention_weight_max": 33.716957092285156,
      "activations/layer23_attention_weight_min": -25.29139518737793,
      "activations/layer2_attention_weight_max": 32.42140579223633,
      "activations/layer2_attention_weight_min": -30.90247344970703,
      "activations/layer3_attention_weight_max": 87.92816162109375,
      "activations/layer3_attention_weight_min": -85.09623718261719,
      "activations/layer4_attention_weight_max": 80.06694030761719,
      "activations/layer4_attention_weight_min": -80.25154876708984,
      "activations/layer5_attention_weight_max": 58.88759994506836,
      "activations/layer5_attention_weight_min": -68.35185241699219,
      "activations/layer6_attention_weight_max": 47.54729080200195,
      "activations/layer6_attention_weight_min": -48.14188003540039,
      "activations/layer7_attention_weight_max": 64.2497787475586,
      "activations/layer7_attention_weight_min": -65.19964599609375,
      "activations/layer8_attention_weight_max": 45.23114013671875,
      "activations/layer8_attention_weight_min": -49.01264572143555,
      "activations/layer9_attention_weight_max": 41.7221794128418,
      "activations/layer9_attention_weight_min": -45.12779998779297,
      "epoch": 21.75,
      "learning_rate": 9.86742424242424e-06,
      "loss": 2.7173,
      "step": 374250
    },
    {
      "activations/layer0_attention_weight_max": 15.912069320678711,
      "activations/layer0_attention_weight_min": -14.305654525756836,
      "activations/layer10_attention_weight_max": 33.70453643798828,
      "activations/layer10_attention_weight_min": -33.61492919921875,
      "activations/layer11_attention_weight_max": 32.99418258666992,
      "activations/layer11_attention_weight_min": -34.44932556152344,
      "activations/layer12_attention_weight_max": 18.125268936157227,
      "activations/layer12_attention_weight_min": -25.124984741210938,
      "activations/layer13_attention_weight_max": 33.41532516479492,
      "activations/layer13_attention_weight_min": -27.280254364013672,
      "activations/layer14_attention_weight_max": 35.71633529663086,
      "activations/layer14_attention_weight_min": -30.8184757232666,
      "activations/layer15_attention_weight_max": 31.28764533996582,
      "activations/layer15_attention_weight_min": -32.22099685668945,
      "activations/layer16_attention_weight_max": 32.597782135009766,
      "activations/layer16_attention_weight_min": -32.0694465637207,
      "activations/layer17_attention_weight_max": 51.40299606323242,
      "activations/layer17_attention_weight_min": -42.53132629394531,
      "activations/layer18_attention_weight_max": 43.24001693725586,
      "activations/layer18_attention_weight_min": -38.908470153808594,
      "activations/layer19_attention_weight_max": 23.61304473876953,
      "activations/layer19_attention_weight_min": -22.239368438720703,
      "activations/layer1_attention_weight_max": 15.114340782165527,
      "activations/layer1_attention_weight_min": -14.482110023498535,
      "activations/layer20_attention_weight_max": 21.399051666259766,
      "activations/layer20_attention_weight_min": -20.784669876098633,
      "activations/layer21_attention_weight_max": 33.14291763305664,
      "activations/layer21_attention_weight_min": -21.590261459350586,
      "activations/layer22_attention_weight_max": 28.979732513427734,
      "activations/layer22_attention_weight_min": -24.404111862182617,
      "activations/layer23_attention_weight_max": 33.133724212646484,
      "activations/layer23_attention_weight_min": -24.838491439819336,
      "activations/layer2_attention_weight_max": 31.35748291015625,
      "activations/layer2_attention_weight_min": -33.21493911743164,
      "activations/layer3_attention_weight_max": 88.1021957397461,
      "activations/layer3_attention_weight_min": -87.82218933105469,
      "activations/layer4_attention_weight_max": 81.35494232177734,
      "activations/layer4_attention_weight_min": -81.27684783935547,
      "activations/layer5_attention_weight_max": 61.652549743652344,
      "activations/layer5_attention_weight_min": -68.89285278320312,
      "activations/layer6_attention_weight_max": 47.07769012451172,
      "activations/layer6_attention_weight_min": -52.16460418701172,
      "activations/layer7_attention_weight_max": 70.33708190917969,
      "activations/layer7_attention_weight_min": -62.916568756103516,
      "activations/layer8_attention_weight_max": 46.528076171875,
      "activations/layer8_attention_weight_min": -49.51582336425781,
      "activations/layer9_attention_weight_max": 41.27977752685547,
      "activations/layer9_attention_weight_min": -45.10719680786133,
      "epoch": 21.75,
      "learning_rate": 9.848484848484848e-06,
      "loss": 2.7153,
      "step": 374300
    },
    {
      "activations/layer0_attention_weight_max": 15.989898681640625,
      "activations/layer0_attention_weight_min": -14.560523986816406,
      "activations/layer10_attention_weight_max": 33.4653205871582,
      "activations/layer10_attention_weight_min": -36.76352310180664,
      "activations/layer11_attention_weight_max": 33.86164855957031,
      "activations/layer11_attention_weight_min": -33.06497573852539,
      "activations/layer12_attention_weight_max": 20.06028175354004,
      "activations/layer12_attention_weight_min": -23.814847946166992,
      "activations/layer13_attention_weight_max": 31.758129119873047,
      "activations/layer13_attention_weight_min": -25.465171813964844,
      "activations/layer14_attention_weight_max": 34.19502639770508,
      "activations/layer14_attention_weight_min": -30.36911392211914,
      "activations/layer15_attention_weight_max": 32.42091751098633,
      "activations/layer15_attention_weight_min": -30.80560874938965,
      "activations/layer16_attention_weight_max": 33.314701080322266,
      "activations/layer16_attention_weight_min": -33.305538177490234,
      "activations/layer17_attention_weight_max": 50.619503021240234,
      "activations/layer17_attention_weight_min": -46.27231979370117,
      "activations/layer18_attention_weight_max": 48.17912673950195,
      "activations/layer18_attention_weight_min": -41.04991912841797,
      "activations/layer19_attention_weight_max": 23.423274993896484,
      "activations/layer19_attention_weight_min": -23.34986686706543,
      "activations/layer1_attention_weight_max": 15.17248821258545,
      "activations/layer1_attention_weight_min": -14.108235359191895,
      "activations/layer20_attention_weight_max": 21.190095901489258,
      "activations/layer20_attention_weight_min": -22.269695281982422,
      "activations/layer21_attention_weight_max": 31.04586410522461,
      "activations/layer21_attention_weight_min": -18.821529388427734,
      "activations/layer22_attention_weight_max": 26.76389503479004,
      "activations/layer22_attention_weight_min": -23.91141128540039,
      "activations/layer23_attention_weight_max": 35.59405517578125,
      "activations/layer23_attention_weight_min": -25.247358322143555,
      "activations/layer2_attention_weight_max": 31.47182273864746,
      "activations/layer2_attention_weight_min": -29.68598747253418,
      "activations/layer3_attention_weight_max": 85.93912506103516,
      "activations/layer3_attention_weight_min": -86.38697814941406,
      "activations/layer4_attention_weight_max": 82.05125427246094,
      "activations/layer4_attention_weight_min": -84.77483367919922,
      "activations/layer5_attention_weight_max": 60.133296966552734,
      "activations/layer5_attention_weight_min": -74.94817352294922,
      "activations/layer6_attention_weight_max": 47.14582824707031,
      "activations/layer6_attention_weight_min": -48.14438247680664,
      "activations/layer7_attention_weight_max": 63.98347473144531,
      "activations/layer7_attention_weight_min": -64.35563659667969,
      "activations/layer8_attention_weight_max": 45.830848693847656,
      "activations/layer8_attention_weight_min": -49.42469024658203,
      "activations/layer9_attention_weight_max": 42.33225631713867,
      "activations/layer9_attention_weight_min": -45.42817306518555,
      "epoch": 21.75,
      "learning_rate": 9.829545454545455e-06,
      "loss": 2.712,
      "step": 374350
    },
    {
      "activations/layer0_attention_weight_max": 17.579322814941406,
      "activations/layer0_attention_weight_min": -14.468650817871094,
      "activations/layer10_attention_weight_max": 37.25310134887695,
      "activations/layer10_attention_weight_min": -35.61460876464844,
      "activations/layer11_attention_weight_max": 33.234169006347656,
      "activations/layer11_attention_weight_min": -35.378089904785156,
      "activations/layer12_attention_weight_max": 19.90205955505371,
      "activations/layer12_attention_weight_min": -24.26327133178711,
      "activations/layer13_attention_weight_max": 36.97953414916992,
      "activations/layer13_attention_weight_min": -28.090951919555664,
      "activations/layer14_attention_weight_max": 35.596405029296875,
      "activations/layer14_attention_weight_min": -30.43111801147461,
      "activations/layer15_attention_weight_max": 34.0693473815918,
      "activations/layer15_attention_weight_min": -32.25868225097656,
      "activations/layer16_attention_weight_max": 36.01753616333008,
      "activations/layer16_attention_weight_min": -31.98581886291504,
      "activations/layer17_attention_weight_max": 54.33477783203125,
      "activations/layer17_attention_weight_min": -47.68234634399414,
      "activations/layer18_attention_weight_max": 50.9703483581543,
      "activations/layer18_attention_weight_min": -41.360992431640625,
      "activations/layer19_attention_weight_max": 25.269041061401367,
      "activations/layer19_attention_weight_min": -22.370128631591797,
      "activations/layer1_attention_weight_max": 15.71948528289795,
      "activations/layer1_attention_weight_min": -14.68105697631836,
      "activations/layer20_attention_weight_max": 26.27752113342285,
      "activations/layer20_attention_weight_min": -21.700010299682617,
      "activations/layer21_attention_weight_max": 39.756683349609375,
      "activations/layer21_attention_weight_min": -25.72635269165039,
      "activations/layer22_attention_weight_max": 30.477128982543945,
      "activations/layer22_attention_weight_min": -24.520444869995117,
      "activations/layer23_attention_weight_max": 37.237491607666016,
      "activations/layer23_attention_weight_min": -29.784900665283203,
      "activations/layer2_attention_weight_max": 31.633970260620117,
      "activations/layer2_attention_weight_min": -30.030691146850586,
      "activations/layer3_attention_weight_max": 94.98432159423828,
      "activations/layer3_attention_weight_min": -89.51512145996094,
      "activations/layer4_attention_weight_max": 87.49067687988281,
      "activations/layer4_attention_weight_min": -87.8124008178711,
      "activations/layer5_attention_weight_max": 62.87500762939453,
      "activations/layer5_attention_weight_min": -78.09481811523438,
      "activations/layer6_attention_weight_max": 51.01215362548828,
      "activations/layer6_attention_weight_min": -53.522361755371094,
      "activations/layer7_attention_weight_max": 68.63700103759766,
      "activations/layer7_attention_weight_min": -71.88809204101562,
      "activations/layer8_attention_weight_max": 49.44966506958008,
      "activations/layer8_attention_weight_min": -54.576114654541016,
      "activations/layer9_attention_weight_max": 44.55305862426758,
      "activations/layer9_attention_weight_min": -48.37545394897461,
      "epoch": 21.75,
      "learning_rate": 9.81060606060606e-06,
      "loss": 2.7271,
      "step": 374400
    },
    {
      "activations/layer0_attention_weight_max": 16.286882400512695,
      "activations/layer0_attention_weight_min": -14.481788635253906,
      "activations/layer10_attention_weight_max": 36.75470733642578,
      "activations/layer10_attention_weight_min": -37.785640716552734,
      "activations/layer11_attention_weight_max": 34.10736083984375,
      "activations/layer11_attention_weight_min": -35.94944763183594,
      "activations/layer12_attention_weight_max": 20.215255737304688,
      "activations/layer12_attention_weight_min": -25.00342559814453,
      "activations/layer13_attention_weight_max": 32.240379333496094,
      "activations/layer13_attention_weight_min": -26.588462829589844,
      "activations/layer14_attention_weight_max": 37.549617767333984,
      "activations/layer14_attention_weight_min": -29.092782974243164,
      "activations/layer15_attention_weight_max": 34.652645111083984,
      "activations/layer15_attention_weight_min": -28.80251693725586,
      "activations/layer16_attention_weight_max": 38.872493743896484,
      "activations/layer16_attention_weight_min": -34.408531188964844,
      "activations/layer17_attention_weight_max": 54.775428771972656,
      "activations/layer17_attention_weight_min": -45.71028137207031,
      "activations/layer18_attention_weight_max": 50.00009536743164,
      "activations/layer18_attention_weight_min": -41.29775619506836,
      "activations/layer19_attention_weight_max": 24.472198486328125,
      "activations/layer19_attention_weight_min": -21.910497665405273,
      "activations/layer1_attention_weight_max": 14.76971435546875,
      "activations/layer1_attention_weight_min": -13.976140975952148,
      "activations/layer20_attention_weight_max": 21.401409149169922,
      "activations/layer20_attention_weight_min": -22.966445922851562,
      "activations/layer21_attention_weight_max": 35.387046813964844,
      "activations/layer21_attention_weight_min": -19.84638023376465,
      "activations/layer22_attention_weight_max": 31.017040252685547,
      "activations/layer22_attention_weight_min": -25.47477149963379,
      "activations/layer23_attention_weight_max": 44.65400314331055,
      "activations/layer23_attention_weight_min": -23.809162139892578,
      "activations/layer2_attention_weight_max": 30.029159545898438,
      "activations/layer2_attention_weight_min": -28.615415573120117,
      "activations/layer3_attention_weight_max": 89.1844482421875,
      "activations/layer3_attention_weight_min": -89.43742370605469,
      "activations/layer4_attention_weight_max": 84.65105438232422,
      "activations/layer4_attention_weight_min": -85.41262817382812,
      "activations/layer5_attention_weight_max": 62.36339569091797,
      "activations/layer5_attention_weight_min": -72.82127380371094,
      "activations/layer6_attention_weight_max": 49.775489807128906,
      "activations/layer6_attention_weight_min": -52.166690826416016,
      "activations/layer7_attention_weight_max": 69.48212432861328,
      "activations/layer7_attention_weight_min": -67.07308959960938,
      "activations/layer8_attention_weight_max": 47.419029235839844,
      "activations/layer8_attention_weight_min": -51.98349380493164,
      "activations/layer9_attention_weight_max": 45.32511520385742,
      "activations/layer9_attention_weight_min": -47.99384689331055,
      "epoch": 21.76,
      "learning_rate": 9.791666666666666e-06,
      "loss": 2.7216,
      "step": 374450
    },
    {
      "activations/layer0_attention_weight_max": 17.07784080505371,
      "activations/layer0_attention_weight_min": -14.471129417419434,
      "activations/layer10_attention_weight_max": 37.181800842285156,
      "activations/layer10_attention_weight_min": -35.8695068359375,
      "activations/layer11_attention_weight_max": 32.64645767211914,
      "activations/layer11_attention_weight_min": -34.103370666503906,
      "activations/layer12_attention_weight_max": 17.820552825927734,
      "activations/layer12_attention_weight_min": -23.290868759155273,
      "activations/layer13_attention_weight_max": 34.33748245239258,
      "activations/layer13_attention_weight_min": -27.550989151000977,
      "activations/layer14_attention_weight_max": 33.75978469848633,
      "activations/layer14_attention_weight_min": -29.881683349609375,
      "activations/layer15_attention_weight_max": 31.15245246887207,
      "activations/layer15_attention_weight_min": -31.051054000854492,
      "activations/layer16_attention_weight_max": 34.24287414550781,
      "activations/layer16_attention_weight_min": -31.650911331176758,
      "activations/layer17_attention_weight_max": 51.08042907714844,
      "activations/layer17_attention_weight_min": -43.38982391357422,
      "activations/layer18_attention_weight_max": 45.86064910888672,
      "activations/layer18_attention_weight_min": -40.002628326416016,
      "activations/layer19_attention_weight_max": 26.721281051635742,
      "activations/layer19_attention_weight_min": -21.464645385742188,
      "activations/layer1_attention_weight_max": 15.060758590698242,
      "activations/layer1_attention_weight_min": -15.173576354980469,
      "activations/layer20_attention_weight_max": 22.8697509765625,
      "activations/layer20_attention_weight_min": -19.797924041748047,
      "activations/layer21_attention_weight_max": 33.21765899658203,
      "activations/layer21_attention_weight_min": -22.361454010009766,
      "activations/layer22_attention_weight_max": 26.747346878051758,
      "activations/layer22_attention_weight_min": -24.11275863647461,
      "activations/layer23_attention_weight_max": 32.26514434814453,
      "activations/layer23_attention_weight_min": -23.460411071777344,
      "activations/layer2_attention_weight_max": 31.55963134765625,
      "activations/layer2_attention_weight_min": -29.78901481628418,
      "activations/layer3_attention_weight_max": 88.77465057373047,
      "activations/layer3_attention_weight_min": -86.51990509033203,
      "activations/layer4_attention_weight_max": 86.7143783569336,
      "activations/layer4_attention_weight_min": -82.24018859863281,
      "activations/layer5_attention_weight_max": 62.940242767333984,
      "activations/layer5_attention_weight_min": -69.74789428710938,
      "activations/layer6_attention_weight_max": 49.56733322143555,
      "activations/layer6_attention_weight_min": -48.92277145385742,
      "activations/layer7_attention_weight_max": 66.90975952148438,
      "activations/layer7_attention_weight_min": -67.15453338623047,
      "activations/layer8_attention_weight_max": 46.79097366333008,
      "activations/layer8_attention_weight_min": -49.9422492980957,
      "activations/layer9_attention_weight_max": 45.967323303222656,
      "activations/layer9_attention_weight_min": -46.077030181884766,
      "epoch": 21.76,
      "learning_rate": 9.772727272727273e-06,
      "loss": 2.7078,
      "step": 374500
    },
    {
      "activations/layer0_attention_weight_max": 15.760000228881836,
      "activations/layer0_attention_weight_min": -14.507291793823242,
      "activations/layer10_attention_weight_max": 34.654937744140625,
      "activations/layer10_attention_weight_min": -35.07365417480469,
      "activations/layer11_attention_weight_max": 33.19695281982422,
      "activations/layer11_attention_weight_min": -33.366912841796875,
      "activations/layer12_attention_weight_max": 18.756128311157227,
      "activations/layer12_attention_weight_min": -24.173240661621094,
      "activations/layer13_attention_weight_max": 35.36467742919922,
      "activations/layer13_attention_weight_min": -29.67786407470703,
      "activations/layer14_attention_weight_max": 33.85137939453125,
      "activations/layer14_attention_weight_min": -32.27157974243164,
      "activations/layer15_attention_weight_max": 36.49822235107422,
      "activations/layer15_attention_weight_min": -33.47178649902344,
      "activations/layer16_attention_weight_max": 35.422664642333984,
      "activations/layer16_attention_weight_min": -33.390037536621094,
      "activations/layer17_attention_weight_max": 54.646942138671875,
      "activations/layer17_attention_weight_min": -52.343101501464844,
      "activations/layer18_attention_weight_max": 51.2641487121582,
      "activations/layer18_attention_weight_min": -48.276329040527344,
      "activations/layer19_attention_weight_max": 24.598649978637695,
      "activations/layer19_attention_weight_min": -26.052518844604492,
      "activations/layer1_attention_weight_max": 14.974577903747559,
      "activations/layer1_attention_weight_min": -15.075556755065918,
      "activations/layer20_attention_weight_max": 24.113311767578125,
      "activations/layer20_attention_weight_min": -25.0308837890625,
      "activations/layer21_attention_weight_max": 43.3316535949707,
      "activations/layer21_attention_weight_min": -24.67345428466797,
      "activations/layer22_attention_weight_max": 33.078285217285156,
      "activations/layer22_attention_weight_min": -27.438339233398438,
      "activations/layer23_attention_weight_max": 37.46320343017578,
      "activations/layer23_attention_weight_min": -23.72412109375,
      "activations/layer2_attention_weight_max": 30.29076385498047,
      "activations/layer2_attention_weight_min": -29.74744415283203,
      "activations/layer3_attention_weight_max": 87.63299560546875,
      "activations/layer3_attention_weight_min": -87.2440185546875,
      "activations/layer4_attention_weight_max": 84.41015625,
      "activations/layer4_attention_weight_min": -82.51262664794922,
      "activations/layer5_attention_weight_max": 62.40853500366211,
      "activations/layer5_attention_weight_min": -69.47630310058594,
      "activations/layer6_attention_weight_max": 46.99337387084961,
      "activations/layer6_attention_weight_min": -48.718082427978516,
      "activations/layer7_attention_weight_max": 67.86578369140625,
      "activations/layer7_attention_weight_min": -65.77881622314453,
      "activations/layer8_attention_weight_max": 46.95717239379883,
      "activations/layer8_attention_weight_min": -49.423912048339844,
      "activations/layer9_attention_weight_max": 44.89523696899414,
      "activations/layer9_attention_weight_min": -45.366878509521484,
      "epoch": 21.76,
      "learning_rate": 9.753787878787878e-06,
      "loss": 2.7107,
      "step": 374550
    },
    {
      "activations/layer0_attention_weight_max": 16.660966873168945,
      "activations/layer0_attention_weight_min": -14.114173889160156,
      "activations/layer10_attention_weight_max": 37.703407287597656,
      "activations/layer10_attention_weight_min": -36.760284423828125,
      "activations/layer11_attention_weight_max": 35.37958526611328,
      "activations/layer11_attention_weight_min": -34.199546813964844,
      "activations/layer12_attention_weight_max": 20.054134368896484,
      "activations/layer12_attention_weight_min": -26.135372161865234,
      "activations/layer13_attention_weight_max": 44.107688903808594,
      "activations/layer13_attention_weight_min": -28.040111541748047,
      "activations/layer14_attention_weight_max": 37.440425872802734,
      "activations/layer14_attention_weight_min": -31.1386661529541,
      "activations/layer15_attention_weight_max": 35.116416931152344,
      "activations/layer15_attention_weight_min": -30.924396514892578,
      "activations/layer16_attention_weight_max": 38.871463775634766,
      "activations/layer16_attention_weight_min": -34.5628662109375,
      "activations/layer17_attention_weight_max": 56.15009689331055,
      "activations/layer17_attention_weight_min": -51.73169708251953,
      "activations/layer18_attention_weight_max": 51.23688888549805,
      "activations/layer18_attention_weight_min": -46.301239013671875,
      "activations/layer19_attention_weight_max": 26.32862663269043,
      "activations/layer19_attention_weight_min": -25.3186092376709,
      "activations/layer1_attention_weight_max": 15.837623596191406,
      "activations/layer1_attention_weight_min": -16.053991317749023,
      "activations/layer20_attention_weight_max": 23.627090454101562,
      "activations/layer20_attention_weight_min": -22.39042854309082,
      "activations/layer21_attention_weight_max": 34.55790710449219,
      "activations/layer21_attention_weight_min": -20.983055114746094,
      "activations/layer22_attention_weight_max": 30.21571159362793,
      "activations/layer22_attention_weight_min": -25.85692024230957,
      "activations/layer23_attention_weight_max": 34.31013488769531,
      "activations/layer23_attention_weight_min": -23.535491943359375,
      "activations/layer2_attention_weight_max": 32.404457092285156,
      "activations/layer2_attention_weight_min": -31.85140609741211,
      "activations/layer3_attention_weight_max": 87.09237670898438,
      "activations/layer3_attention_weight_min": -89.06441497802734,
      "activations/layer4_attention_weight_max": 81.4988784790039,
      "activations/layer4_attention_weight_min": -81.36643981933594,
      "activations/layer5_attention_weight_max": 61.081138610839844,
      "activations/layer5_attention_weight_min": -72.56332397460938,
      "activations/layer6_attention_weight_max": 48.300209045410156,
      "activations/layer6_attention_weight_min": -50.124412536621094,
      "activations/layer7_attention_weight_max": 68.85343170166016,
      "activations/layer7_attention_weight_min": -67.37097930908203,
      "activations/layer8_attention_weight_max": 50.99732971191406,
      "activations/layer8_attention_weight_min": -53.71733856201172,
      "activations/layer9_attention_weight_max": 49.55420684814453,
      "activations/layer9_attention_weight_min": -49.48091125488281,
      "epoch": 21.77,
      "learning_rate": 9.734848484848484e-06,
      "loss": 2.7061,
      "step": 374600
    },
    {
      "activations/layer0_attention_weight_max": 17.400691986083984,
      "activations/layer0_attention_weight_min": -14.25605583190918,
      "activations/layer10_attention_weight_max": 35.68212890625,
      "activations/layer10_attention_weight_min": -36.290748596191406,
      "activations/layer11_attention_weight_max": 34.148860931396484,
      "activations/layer11_attention_weight_min": -36.979156494140625,
      "activations/layer12_attention_weight_max": 19.479589462280273,
      "activations/layer12_attention_weight_min": -23.324735641479492,
      "activations/layer13_attention_weight_max": 37.885986328125,
      "activations/layer13_attention_weight_min": -27.575645446777344,
      "activations/layer14_attention_weight_max": 38.094207763671875,
      "activations/layer14_attention_weight_min": -33.18082809448242,
      "activations/layer15_attention_weight_max": 36.231407165527344,
      "activations/layer15_attention_weight_min": -34.06722640991211,
      "activations/layer16_attention_weight_max": 36.725364685058594,
      "activations/layer16_attention_weight_min": -33.91727828979492,
      "activations/layer17_attention_weight_max": 54.323631286621094,
      "activations/layer17_attention_weight_min": -45.48844909667969,
      "activations/layer18_attention_weight_max": 51.73220443725586,
      "activations/layer18_attention_weight_min": -43.08432388305664,
      "activations/layer19_attention_weight_max": 27.44693946838379,
      "activations/layer19_attention_weight_min": -23.6727352142334,
      "activations/layer1_attention_weight_max": 14.991823196411133,
      "activations/layer1_attention_weight_min": -13.918591499328613,
      "activations/layer20_attention_weight_max": 26.708290100097656,
      "activations/layer20_attention_weight_min": -21.93539810180664,
      "activations/layer21_attention_weight_max": 40.29415512084961,
      "activations/layer21_attention_weight_min": -23.63615608215332,
      "activations/layer22_attention_weight_max": 31.55588722229004,
      "activations/layer22_attention_weight_min": -25.80781364440918,
      "activations/layer23_attention_weight_max": 40.88379669189453,
      "activations/layer23_attention_weight_min": -22.892210006713867,
      "activations/layer2_attention_weight_max": 31.231094360351562,
      "activations/layer2_attention_weight_min": -30.428813934326172,
      "activations/layer3_attention_weight_max": 90.02767181396484,
      "activations/layer3_attention_weight_min": -90.92160034179688,
      "activations/layer4_attention_weight_max": 86.49333953857422,
      "activations/layer4_attention_weight_min": -83.0841293334961,
      "activations/layer5_attention_weight_max": 62.58955001831055,
      "activations/layer5_attention_weight_min": -72.06365203857422,
      "activations/layer6_attention_weight_max": 51.40501022338867,
      "activations/layer6_attention_weight_min": -50.927066802978516,
      "activations/layer7_attention_weight_max": 67.82423400878906,
      "activations/layer7_attention_weight_min": -67.3746566772461,
      "activations/layer8_attention_weight_max": 48.0404052734375,
      "activations/layer8_attention_weight_min": -49.59366226196289,
      "activations/layer9_attention_weight_max": 44.232357025146484,
      "activations/layer9_attention_weight_min": -48.40592956542969,
      "epoch": 21.77,
      "learning_rate": 9.715909090909091e-06,
      "loss": 2.7131,
      "step": 374650
    },
    {
      "activations/layer0_attention_weight_max": 15.303263664245605,
      "activations/layer0_attention_weight_min": -14.327821731567383,
      "activations/layer10_attention_weight_max": 39.258583068847656,
      "activations/layer10_attention_weight_min": -36.11776351928711,
      "activations/layer11_attention_weight_max": 35.2591552734375,
      "activations/layer11_attention_weight_min": -36.26298522949219,
      "activations/layer12_attention_weight_max": 19.079586029052734,
      "activations/layer12_attention_weight_min": -23.740936279296875,
      "activations/layer13_attention_weight_max": 39.471397399902344,
      "activations/layer13_attention_weight_min": -29.00735855102539,
      "activations/layer14_attention_weight_max": 34.05220413208008,
      "activations/layer14_attention_weight_min": -29.735383987426758,
      "activations/layer15_attention_weight_max": 33.415218353271484,
      "activations/layer15_attention_weight_min": -30.041336059570312,
      "activations/layer16_attention_weight_max": 34.728214263916016,
      "activations/layer16_attention_weight_min": -33.28160858154297,
      "activations/layer17_attention_weight_max": 51.31743621826172,
      "activations/layer17_attention_weight_min": -43.608280181884766,
      "activations/layer18_attention_weight_max": 47.37995910644531,
      "activations/layer18_attention_weight_min": -39.07269287109375,
      "activations/layer19_attention_weight_max": 23.7901554107666,
      "activations/layer19_attention_weight_min": -22.567115783691406,
      "activations/layer1_attention_weight_max": 16.17636489868164,
      "activations/layer1_attention_weight_min": -15.262290954589844,
      "activations/layer20_attention_weight_max": 25.704740524291992,
      "activations/layer20_attention_weight_min": -22.54753303527832,
      "activations/layer21_attention_weight_max": 38.985389709472656,
      "activations/layer21_attention_weight_min": -22.743593215942383,
      "activations/layer22_attention_weight_max": 28.42209815979004,
      "activations/layer22_attention_weight_min": -25.109375,
      "activations/layer23_attention_weight_max": 36.369728088378906,
      "activations/layer23_attention_weight_min": -24.645145416259766,
      "activations/layer2_attention_weight_max": 31.35323143005371,
      "activations/layer2_attention_weight_min": -31.74024200439453,
      "activations/layer3_attention_weight_max": 95.22671508789062,
      "activations/layer3_attention_weight_min": -92.64138793945312,
      "activations/layer4_attention_weight_max": 88.58448791503906,
      "activations/layer4_attention_weight_min": -84.78059387207031,
      "activations/layer5_attention_weight_max": 65.80221557617188,
      "activations/layer5_attention_weight_min": -70.67667388916016,
      "activations/layer6_attention_weight_max": 50.16911697387695,
      "activations/layer6_attention_weight_min": -51.13087844848633,
      "activations/layer7_attention_weight_max": 68.21487426757812,
      "activations/layer7_attention_weight_min": -67.78815460205078,
      "activations/layer8_attention_weight_max": 48.28714370727539,
      "activations/layer8_attention_weight_min": -50.089351654052734,
      "activations/layer9_attention_weight_max": 46.488155364990234,
      "activations/layer9_attention_weight_min": -47.24915313720703,
      "epoch": 21.77,
      "learning_rate": 9.696969696969696e-06,
      "loss": 2.7209,
      "step": 374700
    },
    {
      "activations/layer0_attention_weight_max": 16.66539764404297,
      "activations/layer0_attention_weight_min": -14.256204605102539,
      "activations/layer10_attention_weight_max": 35.54213333129883,
      "activations/layer10_attention_weight_min": -35.88703155517578,
      "activations/layer11_attention_weight_max": 33.292457580566406,
      "activations/layer11_attention_weight_min": -34.2117919921875,
      "activations/layer12_attention_weight_max": 18.47977638244629,
      "activations/layer12_attention_weight_min": -24.82318687438965,
      "activations/layer13_attention_weight_max": 37.47541809082031,
      "activations/layer13_attention_weight_min": -28.198808670043945,
      "activations/layer14_attention_weight_max": 34.10981369018555,
      "activations/layer14_attention_weight_min": -32.11884307861328,
      "activations/layer15_attention_weight_max": 32.38134002685547,
      "activations/layer15_attention_weight_min": -32.817447662353516,
      "activations/layer16_attention_weight_max": 34.05242919921875,
      "activations/layer16_attention_weight_min": -34.399024963378906,
      "activations/layer17_attention_weight_max": 53.043025970458984,
      "activations/layer17_attention_weight_min": -49.85259246826172,
      "activations/layer18_attention_weight_max": 47.35050582885742,
      "activations/layer18_attention_weight_min": -42.7505989074707,
      "activations/layer19_attention_weight_max": 25.523258209228516,
      "activations/layer19_attention_weight_min": -21.984447479248047,
      "activations/layer1_attention_weight_max": 14.732463836669922,
      "activations/layer1_attention_weight_min": -14.262974739074707,
      "activations/layer20_attention_weight_max": 22.81953239440918,
      "activations/layer20_attention_weight_min": -20.843717575073242,
      "activations/layer21_attention_weight_max": 35.237709045410156,
      "activations/layer21_attention_weight_min": -19.850858688354492,
      "activations/layer22_attention_weight_max": 30.510223388671875,
      "activations/layer22_attention_weight_min": -25.319124221801758,
      "activations/layer23_attention_weight_max": 33.5294075012207,
      "activations/layer23_attention_weight_min": -23.092018127441406,
      "activations/layer2_attention_weight_max": 31.282669067382812,
      "activations/layer2_attention_weight_min": -29.027095794677734,
      "activations/layer3_attention_weight_max": 85.27625274658203,
      "activations/layer3_attention_weight_min": -85.83670806884766,
      "activations/layer4_attention_weight_max": 81.10308837890625,
      "activations/layer4_attention_weight_min": -86.00245666503906,
      "activations/layer5_attention_weight_max": 60.10233688354492,
      "activations/layer5_attention_weight_min": -71.36980438232422,
      "activations/layer6_attention_weight_max": 48.404903411865234,
      "activations/layer6_attention_weight_min": -51.81583023071289,
      "activations/layer7_attention_weight_max": 65.81967163085938,
      "activations/layer7_attention_weight_min": -66.03710174560547,
      "activations/layer8_attention_weight_max": 45.95970916748047,
      "activations/layer8_attention_weight_min": -51.80555725097656,
      "activations/layer9_attention_weight_max": 44.863807678222656,
      "activations/layer9_attention_weight_min": -46.722896575927734,
      "epoch": 21.78,
      "learning_rate": 9.678030303030302e-06,
      "loss": 2.7188,
      "step": 374750
    },
    {
      "activations/layer0_attention_weight_max": 16.52765464782715,
      "activations/layer0_attention_weight_min": -14.395730972290039,
      "activations/layer10_attention_weight_max": 36.79769515991211,
      "activations/layer10_attention_weight_min": -38.06943130493164,
      "activations/layer11_attention_weight_max": 37.26911163330078,
      "activations/layer11_attention_weight_min": -34.39007568359375,
      "activations/layer12_attention_weight_max": 18.134201049804688,
      "activations/layer12_attention_weight_min": -27.795433044433594,
      "activations/layer13_attention_weight_max": 31.339736938476562,
      "activations/layer13_attention_weight_min": -25.744617462158203,
      "activations/layer14_attention_weight_max": 33.02165603637695,
      "activations/layer14_attention_weight_min": -30.28327751159668,
      "activations/layer15_attention_weight_max": 29.0677433013916,
      "activations/layer15_attention_weight_min": -29.509767532348633,
      "activations/layer16_attention_weight_max": 32.770416259765625,
      "activations/layer16_attention_weight_min": -32.0184326171875,
      "activations/layer17_attention_weight_max": 47.90937805175781,
      "activations/layer17_attention_weight_min": -45.176795959472656,
      "activations/layer18_attention_weight_max": 42.974674224853516,
      "activations/layer18_attention_weight_min": -40.82501220703125,
      "activations/layer19_attention_weight_max": 24.5428524017334,
      "activations/layer19_attention_weight_min": -22.572250366210938,
      "activations/layer1_attention_weight_max": 15.738393783569336,
      "activations/layer1_attention_weight_min": -13.866323471069336,
      "activations/layer20_attention_weight_max": 21.562728881835938,
      "activations/layer20_attention_weight_min": -22.08382225036621,
      "activations/layer21_attention_weight_max": 30.097698211669922,
      "activations/layer21_attention_weight_min": -21.742162704467773,
      "activations/layer22_attention_weight_max": 27.1966495513916,
      "activations/layer22_attention_weight_min": -25.66827392578125,
      "activations/layer23_attention_weight_max": 34.67138671875,
      "activations/layer23_attention_weight_min": -25.819473266601562,
      "activations/layer2_attention_weight_max": 31.217041015625,
      "activations/layer2_attention_weight_min": -30.404821395874023,
      "activations/layer3_attention_weight_max": 87.30243682861328,
      "activations/layer3_attention_weight_min": -88.79634094238281,
      "activations/layer4_attention_weight_max": 84.43702697753906,
      "activations/layer4_attention_weight_min": -85.35871124267578,
      "activations/layer5_attention_weight_max": 60.097068786621094,
      "activations/layer5_attention_weight_min": -70.99932098388672,
      "activations/layer6_attention_weight_max": 51.18305587768555,
      "activations/layer6_attention_weight_min": -52.5186882019043,
      "activations/layer7_attention_weight_max": 74.86259460449219,
      "activations/layer7_attention_weight_min": -66.67735290527344,
      "activations/layer8_attention_weight_max": 48.15791702270508,
      "activations/layer8_attention_weight_min": -51.26485061645508,
      "activations/layer9_attention_weight_max": 47.06791687011719,
      "activations/layer9_attention_weight_min": -48.19326400756836,
      "epoch": 21.78,
      "learning_rate": 9.659090909090907e-06,
      "loss": 2.7116,
      "step": 374800
    },
    {
      "activations/layer0_attention_weight_max": 15.515936851501465,
      "activations/layer0_attention_weight_min": -14.073674201965332,
      "activations/layer10_attention_weight_max": 36.26573181152344,
      "activations/layer10_attention_weight_min": -38.379058837890625,
      "activations/layer11_attention_weight_max": 35.00925827026367,
      "activations/layer11_attention_weight_min": -37.20526885986328,
      "activations/layer12_attention_weight_max": 20.799062728881836,
      "activations/layer12_attention_weight_min": -27.672155380249023,
      "activations/layer13_attention_weight_max": 39.1923713684082,
      "activations/layer13_attention_weight_min": -32.74491500854492,
      "activations/layer14_attention_weight_max": 41.18311309814453,
      "activations/layer14_attention_weight_min": -31.78379249572754,
      "activations/layer15_attention_weight_max": 38.615203857421875,
      "activations/layer15_attention_weight_min": -34.277122497558594,
      "activations/layer16_attention_weight_max": 38.85334014892578,
      "activations/layer16_attention_weight_min": -37.72281265258789,
      "activations/layer17_attention_weight_max": 60.11151885986328,
      "activations/layer17_attention_weight_min": -50.14125061035156,
      "activations/layer18_attention_weight_max": 56.06906509399414,
      "activations/layer18_attention_weight_min": -48.078460693359375,
      "activations/layer19_attention_weight_max": 28.200010299682617,
      "activations/layer19_attention_weight_min": -26.343229293823242,
      "activations/layer1_attention_weight_max": 15.298746109008789,
      "activations/layer1_attention_weight_min": -14.715278625488281,
      "activations/layer20_attention_weight_max": 28.070981979370117,
      "activations/layer20_attention_weight_min": -21.861244201660156,
      "activations/layer21_attention_weight_max": 48.385005950927734,
      "activations/layer21_attention_weight_min": -24.681121826171875,
      "activations/layer22_attention_weight_max": 32.268802642822266,
      "activations/layer22_attention_weight_min": -26.51889419555664,
      "activations/layer23_attention_weight_max": 39.545475006103516,
      "activations/layer23_attention_weight_min": -25.472816467285156,
      "activations/layer2_attention_weight_max": 30.977405548095703,
      "activations/layer2_attention_weight_min": -30.599355697631836,
      "activations/layer3_attention_weight_max": 88.82464599609375,
      "activations/layer3_attention_weight_min": -94.8950424194336,
      "activations/layer4_attention_weight_max": 86.5387954711914,
      "activations/layer4_attention_weight_min": -83.34366607666016,
      "activations/layer5_attention_weight_max": 62.60931396484375,
      "activations/layer5_attention_weight_min": -71.55690002441406,
      "activations/layer6_attention_weight_max": 49.915218353271484,
      "activations/layer6_attention_weight_min": -50.95537185668945,
      "activations/layer7_attention_weight_max": 72.98551177978516,
      "activations/layer7_attention_weight_min": -70.64246368408203,
      "activations/layer8_attention_weight_max": 48.0535888671875,
      "activations/layer8_attention_weight_min": -50.743919372558594,
      "activations/layer9_attention_weight_max": 46.862300872802734,
      "activations/layer9_attention_weight_min": -47.30219650268555,
      "epoch": 21.78,
      "learning_rate": 9.640151515151514e-06,
      "loss": 2.7036,
      "step": 374850
    },
    {
      "activations/layer0_attention_weight_max": 15.464552879333496,
      "activations/layer0_attention_weight_min": -14.104296684265137,
      "activations/layer10_attention_weight_max": 37.348331451416016,
      "activations/layer10_attention_weight_min": -36.19266128540039,
      "activations/layer11_attention_weight_max": 35.124534606933594,
      "activations/layer11_attention_weight_min": -35.94309997558594,
      "activations/layer12_attention_weight_max": 19.21067237854004,
      "activations/layer12_attention_weight_min": -23.642515182495117,
      "activations/layer13_attention_weight_max": 34.257415771484375,
      "activations/layer13_attention_weight_min": -27.151721954345703,
      "activations/layer14_attention_weight_max": 36.680538177490234,
      "activations/layer14_attention_weight_min": -29.540102005004883,
      "activations/layer15_attention_weight_max": 33.95161056518555,
      "activations/layer15_attention_weight_min": -31.035757064819336,
      "activations/layer16_attention_weight_max": 35.82938766479492,
      "activations/layer16_attention_weight_min": -34.9384651184082,
      "activations/layer17_attention_weight_max": 51.253936767578125,
      "activations/layer17_attention_weight_min": -47.67899703979492,
      "activations/layer18_attention_weight_max": 49.80161666870117,
      "activations/layer18_attention_weight_min": -40.861427307128906,
      "activations/layer19_attention_weight_max": 25.54014778137207,
      "activations/layer19_attention_weight_min": -21.826602935791016,
      "activations/layer1_attention_weight_max": 16.278879165649414,
      "activations/layer1_attention_weight_min": -15.046673774719238,
      "activations/layer20_attention_weight_max": 24.066673278808594,
      "activations/layer20_attention_weight_min": -19.73563003540039,
      "activations/layer21_attention_weight_max": 38.33700942993164,
      "activations/layer21_attention_weight_min": -22.4338436126709,
      "activations/layer22_attention_weight_max": 32.58997344970703,
      "activations/layer22_attention_weight_min": -25.67887306213379,
      "activations/layer23_attention_weight_max": 38.20642852783203,
      "activations/layer23_attention_weight_min": -23.63789939880371,
      "activations/layer2_attention_weight_max": 31.82436180114746,
      "activations/layer2_attention_weight_min": -30.23154067993164,
      "activations/layer3_attention_weight_max": 91.24832916259766,
      "activations/layer3_attention_weight_min": -87.19947052001953,
      "activations/layer4_attention_weight_max": 86.59574127197266,
      "activations/layer4_attention_weight_min": -85.64837646484375,
      "activations/layer5_attention_weight_max": 62.22196578979492,
      "activations/layer5_attention_weight_min": -73.2211685180664,
      "activations/layer6_attention_weight_max": 50.74574279785156,
      "activations/layer6_attention_weight_min": -51.699703216552734,
      "activations/layer7_attention_weight_max": 69.49160766601562,
      "activations/layer7_attention_weight_min": -70.55406951904297,
      "activations/layer8_attention_weight_max": 48.10678482055664,
      "activations/layer8_attention_weight_min": -50.16590881347656,
      "activations/layer9_attention_weight_max": 45.66512680053711,
      "activations/layer9_attention_weight_min": -47.57073974609375,
      "epoch": 21.78,
      "learning_rate": 9.62121212121212e-06,
      "loss": 2.7236,
      "step": 374900
    },
    {
      "activations/layer0_attention_weight_max": 15.628389358520508,
      "activations/layer0_attention_weight_min": -14.327960014343262,
      "activations/layer10_attention_weight_max": 39.622127532958984,
      "activations/layer10_attention_weight_min": -37.229183197021484,
      "activations/layer11_attention_weight_max": 37.13390350341797,
      "activations/layer11_attention_weight_min": -36.868614196777344,
      "activations/layer12_attention_weight_max": 18.79938507080078,
      "activations/layer12_attention_weight_min": -26.019559860229492,
      "activations/layer13_attention_weight_max": 36.64860153198242,
      "activations/layer13_attention_weight_min": -28.6502742767334,
      "activations/layer14_attention_weight_max": 32.4033088684082,
      "activations/layer14_attention_weight_min": -31.930477142333984,
      "activations/layer15_attention_weight_max": 32.52039337158203,
      "activations/layer15_attention_weight_min": -31.5614013671875,
      "activations/layer16_attention_weight_max": 35.90242004394531,
      "activations/layer16_attention_weight_min": -32.99844741821289,
      "activations/layer17_attention_weight_max": 51.11708450317383,
      "activations/layer17_attention_weight_min": -47.52970886230469,
      "activations/layer18_attention_weight_max": 46.0274658203125,
      "activations/layer18_attention_weight_min": -41.259708404541016,
      "activations/layer19_attention_weight_max": 24.185501098632812,
      "activations/layer19_attention_weight_min": -21.883081436157227,
      "activations/layer1_attention_weight_max": 14.661308288574219,
      "activations/layer1_attention_weight_min": -14.167818069458008,
      "activations/layer20_attention_weight_max": 21.13372802734375,
      "activations/layer20_attention_weight_min": -21.045291900634766,
      "activations/layer21_attention_weight_max": 35.670135498046875,
      "activations/layer21_attention_weight_min": -22.376617431640625,
      "activations/layer22_attention_weight_max": 28.018199920654297,
      "activations/layer22_attention_weight_min": -25.255002975463867,
      "activations/layer23_attention_weight_max": 35.2936897277832,
      "activations/layer23_attention_weight_min": -22.648603439331055,
      "activations/layer2_attention_weight_max": 31.980756759643555,
      "activations/layer2_attention_weight_min": -31.2893123626709,
      "activations/layer3_attention_weight_max": 94.79781341552734,
      "activations/layer3_attention_weight_min": -91.41935729980469,
      "activations/layer4_attention_weight_max": 89.38985443115234,
      "activations/layer4_attention_weight_min": -86.7365493774414,
      "activations/layer5_attention_weight_max": 62.752464294433594,
      "activations/layer5_attention_weight_min": -73.38304138183594,
      "activations/layer6_attention_weight_max": 54.53691482543945,
      "activations/layer6_attention_weight_min": -52.06776428222656,
      "activations/layer7_attention_weight_max": 73.14219665527344,
      "activations/layer7_attention_weight_min": -68.71675872802734,
      "activations/layer8_attention_weight_max": 51.44480514526367,
      "activations/layer8_attention_weight_min": -53.16151809692383,
      "activations/layer9_attention_weight_max": 46.36991500854492,
      "activations/layer9_attention_weight_min": -49.335960388183594,
      "epoch": 21.79,
      "learning_rate": 9.602272727272725e-06,
      "loss": 2.715,
      "step": 374950
    },
    {
      "activations/layer0_attention_weight_max": 16.373611450195312,
      "activations/layer0_attention_weight_min": -14.169480323791504,
      "activations/layer10_attention_weight_max": 34.44036865234375,
      "activations/layer10_attention_weight_min": -34.42737579345703,
      "activations/layer11_attention_weight_max": 33.93692398071289,
      "activations/layer11_attention_weight_min": -34.289222717285156,
      "activations/layer12_attention_weight_max": 17.58118438720703,
      "activations/layer12_attention_weight_min": -23.746299743652344,
      "activations/layer13_attention_weight_max": 32.32731246948242,
      "activations/layer13_attention_weight_min": -25.1071834564209,
      "activations/layer14_attention_weight_max": 35.049560546875,
      "activations/layer14_attention_weight_min": -27.81150245666504,
      "activations/layer15_attention_weight_max": 31.695802688598633,
      "activations/layer15_attention_weight_min": -27.760969161987305,
      "activations/layer16_attention_weight_max": 31.995349884033203,
      "activations/layer16_attention_weight_min": -30.15445899963379,
      "activations/layer17_attention_weight_max": 48.693206787109375,
      "activations/layer17_attention_weight_min": -40.95012283325195,
      "activations/layer18_attention_weight_max": 44.66084289550781,
      "activations/layer18_attention_weight_min": -37.15964126586914,
      "activations/layer19_attention_weight_max": 25.351850509643555,
      "activations/layer19_attention_weight_min": -20.771759033203125,
      "activations/layer1_attention_weight_max": 15.20974349975586,
      "activations/layer1_attention_weight_min": -16.992305755615234,
      "activations/layer20_attention_weight_max": 22.3989315032959,
      "activations/layer20_attention_weight_min": -19.414459228515625,
      "activations/layer21_attention_weight_max": 33.6609001159668,
      "activations/layer21_attention_weight_min": -19.269224166870117,
      "activations/layer22_attention_weight_max": 30.114654541015625,
      "activations/layer22_attention_weight_min": -22.5508975982666,
      "activations/layer23_attention_weight_max": 36.8188591003418,
      "activations/layer23_attention_weight_min": -21.739803314208984,
      "activations/layer2_attention_weight_max": 31.50702667236328,
      "activations/layer2_attention_weight_min": -30.630521774291992,
      "activations/layer3_attention_weight_max": 87.22090911865234,
      "activations/layer3_attention_weight_min": -92.19805145263672,
      "activations/layer4_attention_weight_max": 84.23143005371094,
      "activations/layer4_attention_weight_min": -83.22000122070312,
      "activations/layer5_attention_weight_max": 63.117897033691406,
      "activations/layer5_attention_weight_min": -69.8830337524414,
      "activations/layer6_attention_weight_max": 51.74784851074219,
      "activations/layer6_attention_weight_min": -51.74094009399414,
      "activations/layer7_attention_weight_max": 64.88966369628906,
      "activations/layer7_attention_weight_min": -68.77328491210938,
      "activations/layer8_attention_weight_max": 47.020713806152344,
      "activations/layer8_attention_weight_min": -52.61132049560547,
      "activations/layer9_attention_weight_max": 45.91309356689453,
      "activations/layer9_attention_weight_min": -45.480613708496094,
      "epoch": 21.79,
      "learning_rate": 9.583333333333332e-06,
      "loss": 2.7227,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.4841,
      "eval_samples_per_second": 506.123,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.4841,
      "eval_openwebtext_samples_per_second": 506.123,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_wikitext_loss": 2.908203125,
      "eval_wikitext_ppl": 18.323843311697065,
      "eval_wikitext_runtime": 1.9467,
      "eval_wikitext_samples_per_second": 234.237,
      "step": 375000
    },
    {
      "epoch": 21.79,
      "eval_lambada_loss": 2.431640625,
      "eval_lambada_ppl": 11.3775330413472,
      "eval_lambada_runtime": 9.605,
      "eval_lambada_samples_per_second": 506.925,
      "step": 375000
    },
    {
      "activations/layer0_attention_weight_max": 16.22970199584961,
      "activations/layer0_attention_weight_min": -14.3436279296875,
      "activations/layer10_attention_weight_max": 34.382171630859375,
      "activations/layer10_attention_weight_min": -36.152252197265625,
      "activations/layer11_attention_weight_max": 32.56061553955078,
      "activations/layer11_attention_weight_min": -34.349403381347656,
      "activations/layer12_attention_weight_max": 19.59601402282715,
      "activations/layer12_attention_weight_min": -27.00836753845215,
      "activations/layer13_attention_weight_max": 36.643001556396484,
      "activations/layer13_attention_weight_min": -26.089574813842773,
      "activations/layer14_attention_weight_max": 35.36643600463867,
      "activations/layer14_attention_weight_min": -30.587703704833984,
      "activations/layer15_attention_weight_max": 35.49959945678711,
      "activations/layer15_attention_weight_min": -31.50301742553711,
      "activations/layer16_attention_weight_max": 39.557735443115234,
      "activations/layer16_attention_weight_min": -35.56674575805664,
      "activations/layer17_attention_weight_max": 56.638553619384766,
      "activations/layer17_attention_weight_min": -50.38216018676758,
      "activations/layer18_attention_weight_max": 54.55893325805664,
      "activations/layer18_attention_weight_min": -44.760433197021484,
      "activations/layer19_attention_weight_max": 28.262832641601562,
      "activations/layer19_attention_weight_min": -23.377107620239258,
      "activations/layer1_attention_weight_max": 15.424785614013672,
      "activations/layer1_attention_weight_min": -14.690875053405762,
      "activations/layer20_attention_weight_max": 25.642959594726562,
      "activations/layer20_attention_weight_min": -23.01565170288086,
      "activations/layer21_attention_weight_max": 44.206695556640625,
      "activations/layer21_attention_weight_min": -26.618478775024414,
      "activations/layer22_attention_weight_max": 30.91761589050293,
      "activations/layer22_attention_weight_min": -26.044097900390625,
      "activations/layer23_attention_weight_max": 42.45686721801758,
      "activations/layer23_attention_weight_min": -24.003446578979492,
      "activations/layer2_attention_weight_max": 32.285804748535156,
      "activations/layer2_attention_weight_min": -29.92243766784668,
      "activations/layer3_attention_weight_max": 91.17520904541016,
      "activations/layer3_attention_weight_min": -90.61695861816406,
      "activations/layer4_attention_weight_max": 87.03044128417969,
      "activations/layer4_attention_weight_min": -83.95923614501953,
      "activations/layer5_attention_weight_max": 61.88572692871094,
      "activations/layer5_attention_weight_min": -72.5434341430664,
      "activations/layer6_attention_weight_max": 50.648128509521484,
      "activations/layer6_attention_weight_min": -51.520484924316406,
      "activations/layer7_attention_weight_max": 73.48999786376953,
      "activations/layer7_attention_weight_min": -69.96643829345703,
      "activations/layer8_attention_weight_max": 47.049293518066406,
      "activations/layer8_attention_weight_min": -49.57817459106445,
      "activations/layer9_attention_weight_max": 43.6978874206543,
      "activations/layer9_attention_weight_min": -48.10179901123047,
      "epoch": 21.79,
      "learning_rate": 9.564393939393939e-06,
      "loss": 2.694,
      "step": 375050
    },
    {
      "activations/layer0_attention_weight_max": 16.901166915893555,
      "activations/layer0_attention_weight_min": -13.930693626403809,
      "activations/layer10_attention_weight_max": 35.52089309692383,
      "activations/layer10_attention_weight_min": -37.44742965698242,
      "activations/layer11_attention_weight_max": 33.52919006347656,
      "activations/layer11_attention_weight_min": -36.616172790527344,
      "activations/layer12_attention_weight_max": 17.90056800842285,
      "activations/layer12_attention_weight_min": -24.2227783203125,
      "activations/layer13_attention_weight_max": 32.148765563964844,
      "activations/layer13_attention_weight_min": -26.824018478393555,
      "activations/layer14_attention_weight_max": 34.81635665893555,
      "activations/layer14_attention_weight_min": -31.66415786743164,
      "activations/layer15_attention_weight_max": 33.58878707885742,
      "activations/layer15_attention_weight_min": -31.869783401489258,
      "activations/layer16_attention_weight_max": 32.88563537597656,
      "activations/layer16_attention_weight_min": -31.571216583251953,
      "activations/layer17_attention_weight_max": 49.89262390136719,
      "activations/layer17_attention_weight_min": -46.45783996582031,
      "activations/layer18_attention_weight_max": 48.53794479370117,
      "activations/layer18_attention_weight_min": -44.03010940551758,
      "activations/layer19_attention_weight_max": 26.022069931030273,
      "activations/layer19_attention_weight_min": -26.14051628112793,
      "activations/layer1_attention_weight_max": 15.550897598266602,
      "activations/layer1_attention_weight_min": -13.511382102966309,
      "activations/layer20_attention_weight_max": 23.810914993286133,
      "activations/layer20_attention_weight_min": -25.21942710876465,
      "activations/layer21_attention_weight_max": 38.06819534301758,
      "activations/layer21_attention_weight_min": -25.02141571044922,
      "activations/layer22_attention_weight_max": 30.70667839050293,
      "activations/layer22_attention_weight_min": -26.556203842163086,
      "activations/layer23_attention_weight_max": 39.80125045776367,
      "activations/layer23_attention_weight_min": -23.22943878173828,
      "activations/layer2_attention_weight_max": 30.363759994506836,
      "activations/layer2_attention_weight_min": -30.335018157958984,
      "activations/layer3_attention_weight_max": 89.78153228759766,
      "activations/layer3_attention_weight_min": -87.72412109375,
      "activations/layer4_attention_weight_max": 84.39806365966797,
      "activations/layer4_attention_weight_min": -80.38436126708984,
      "activations/layer5_attention_weight_max": 61.22472381591797,
      "activations/layer5_attention_weight_min": -77.29313659667969,
      "activations/layer6_attention_weight_max": 49.37368392944336,
      "activations/layer6_attention_weight_min": -47.5630989074707,
      "activations/layer7_attention_weight_max": 68.19483184814453,
      "activations/layer7_attention_weight_min": -62.66078567504883,
      "activations/layer8_attention_weight_max": 48.62049102783203,
      "activations/layer8_attention_weight_min": -47.92433166503906,
      "activations/layer9_attention_weight_max": 43.83451461791992,
      "activations/layer9_attention_weight_min": -44.32062530517578,
      "epoch": 21.8,
      "learning_rate": 9.545454545454543e-06,
      "loss": 2.7184,
      "step": 375100
    },
    {
      "activations/layer0_attention_weight_max": 15.33430004119873,
      "activations/layer0_attention_weight_min": -14.144886016845703,
      "activations/layer10_attention_weight_max": 34.20356369018555,
      "activations/layer10_attention_weight_min": -36.71152877807617,
      "activations/layer11_attention_weight_max": 33.56496810913086,
      "activations/layer11_attention_weight_min": -34.79779815673828,
      "activations/layer12_attention_weight_max": 18.60413932800293,
      "activations/layer12_attention_weight_min": -27.510229110717773,
      "activations/layer13_attention_weight_max": 32.78311538696289,
      "activations/layer13_attention_weight_min": -26.695980072021484,
      "activations/layer14_attention_weight_max": 35.00981903076172,
      "activations/layer14_attention_weight_min": -29.874576568603516,
      "activations/layer15_attention_weight_max": 33.26551818847656,
      "activations/layer15_attention_weight_min": -31.052927017211914,
      "activations/layer16_attention_weight_max": 33.550113677978516,
      "activations/layer16_attention_weight_min": -33.396453857421875,
      "activations/layer17_attention_weight_max": 48.94792938232422,
      "activations/layer17_attention_weight_min": -45.75408172607422,
      "activations/layer18_attention_weight_max": 46.218997955322266,
      "activations/layer18_attention_weight_min": -42.42657470703125,
      "activations/layer19_attention_weight_max": 24.9613094329834,
      "activations/layer19_attention_weight_min": -22.4639949798584,
      "activations/layer1_attention_weight_max": 14.992846488952637,
      "activations/layer1_attention_weight_min": -14.724198341369629,
      "activations/layer20_attention_weight_max": 24.415054321289062,
      "activations/layer20_attention_weight_min": -21.348936080932617,
      "activations/layer21_attention_weight_max": 37.98503875732422,
      "activations/layer21_attention_weight_min": -22.099611282348633,
      "activations/layer22_attention_weight_max": 27.550582885742188,
      "activations/layer22_attention_weight_min": -24.74250030517578,
      "activations/layer23_attention_weight_max": 35.70835876464844,
      "activations/layer23_attention_weight_min": -22.839027404785156,
      "activations/layer2_attention_weight_max": 31.806289672851562,
      "activations/layer2_attention_weight_min": -29.955902099609375,
      "activations/layer3_attention_weight_max": 93.22877502441406,
      "activations/layer3_attention_weight_min": -92.81352233886719,
      "activations/layer4_attention_weight_max": 88.20551300048828,
      "activations/layer4_attention_weight_min": -86.87838745117188,
      "activations/layer5_attention_weight_max": 64.87651824951172,
      "activations/layer5_attention_weight_min": -79.03366088867188,
      "activations/layer6_attention_weight_max": 50.225318908691406,
      "activations/layer6_attention_weight_min": -51.69996643066406,
      "activations/layer7_attention_weight_max": 71.86270904541016,
      "activations/layer7_attention_weight_min": -71.08354949951172,
      "activations/layer8_attention_weight_max": 48.52595520019531,
      "activations/layer8_attention_weight_min": -55.746910095214844,
      "activations/layer9_attention_weight_max": 46.779388427734375,
      "activations/layer9_attention_weight_min": -48.156761169433594,
      "epoch": 21.8,
      "learning_rate": 9.526515151515152e-06,
      "loss": 2.7047,
      "step": 375150
    },
    {
      "activations/layer0_attention_weight_max": 17.766794204711914,
      "activations/layer0_attention_weight_min": -13.977315902709961,
      "activations/layer10_attention_weight_max": 36.865264892578125,
      "activations/layer10_attention_weight_min": -36.96152877807617,
      "activations/layer11_attention_weight_max": 34.43592071533203,
      "activations/layer11_attention_weight_min": -34.893192291259766,
      "activations/layer12_attention_weight_max": 19.128936767578125,
      "activations/layer12_attention_weight_min": -25.0068302154541,
      "activations/layer13_attention_weight_max": 35.06407928466797,
      "activations/layer13_attention_weight_min": -28.090356826782227,
      "activations/layer14_attention_weight_max": 38.22407150268555,
      "activations/layer14_attention_weight_min": -29.61680793762207,
      "activations/layer15_attention_weight_max": 33.39911651611328,
      "activations/layer15_attention_weight_min": -31.179019927978516,
      "activations/layer16_attention_weight_max": 36.12281799316406,
      "activations/layer16_attention_weight_min": -32.9441032409668,
      "activations/layer17_attention_weight_max": 49.81319046020508,
      "activations/layer17_attention_weight_min": -48.15122985839844,
      "activations/layer18_attention_weight_max": 48.96017837524414,
      "activations/layer18_attention_weight_min": -41.25104522705078,
      "activations/layer19_attention_weight_max": 23.96039390563965,
      "activations/layer19_attention_weight_min": -22.55858039855957,
      "activations/layer1_attention_weight_max": 15.383389472961426,
      "activations/layer1_attention_weight_min": -16.72987937927246,
      "activations/layer20_attention_weight_max": 21.623348236083984,
      "activations/layer20_attention_weight_min": -21.396556854248047,
      "activations/layer21_attention_weight_max": 35.97675323486328,
      "activations/layer21_attention_weight_min": -23.9847354888916,
      "activations/layer22_attention_weight_max": 30.287757873535156,
      "activations/layer22_attention_weight_min": -25.93301010131836,
      "activations/layer23_attention_weight_max": 35.528564453125,
      "activations/layer23_attention_weight_min": -24.821144104003906,
      "activations/layer2_attention_weight_max": 32.11080551147461,
      "activations/layer2_attention_weight_min": -29.94989585876465,
      "activations/layer3_attention_weight_max": 91.00924682617188,
      "activations/layer3_attention_weight_min": -87.8753433227539,
      "activations/layer4_attention_weight_max": 83.63833618164062,
      "activations/layer4_attention_weight_min": -86.18434143066406,
      "activations/layer5_attention_weight_max": 62.085853576660156,
      "activations/layer5_attention_weight_min": -75.25592803955078,
      "activations/layer6_attention_weight_max": 50.42047882080078,
      "activations/layer6_attention_weight_min": -50.67118453979492,
      "activations/layer7_attention_weight_max": 66.82437133789062,
      "activations/layer7_attention_weight_min": -65.68936157226562,
      "activations/layer8_attention_weight_max": 48.99446105957031,
      "activations/layer8_attention_weight_min": -50.72882843017578,
      "activations/layer9_attention_weight_max": 46.86951446533203,
      "activations/layer9_attention_weight_min": -44.94902420043945,
      "epoch": 21.8,
      "learning_rate": 9.507575757575758e-06,
      "loss": 2.7127,
      "step": 375200
    },
    {
      "activations/layer0_attention_weight_max": 16.182357788085938,
      "activations/layer0_attention_weight_min": -14.255965232849121,
      "activations/layer10_attention_weight_max": 34.994808197021484,
      "activations/layer10_attention_weight_min": -35.43642044067383,
      "activations/layer11_attention_weight_max": 31.93075180053711,
      "activations/layer11_attention_weight_min": -33.63951110839844,
      "activations/layer12_attention_weight_max": 19.981124877929688,
      "activations/layer12_attention_weight_min": -26.94268035888672,
      "activations/layer13_attention_weight_max": 33.741607666015625,
      "activations/layer13_attention_weight_min": -26.679527282714844,
      "activations/layer14_attention_weight_max": 37.64751434326172,
      "activations/layer14_attention_weight_min": -32.63794708251953,
      "activations/layer15_attention_weight_max": 36.141204833984375,
      "activations/layer15_attention_weight_min": -32.766170501708984,
      "activations/layer16_attention_weight_max": 37.68892288208008,
      "activations/layer16_attention_weight_min": -35.21812438964844,
      "activations/layer17_attention_weight_max": 55.54546356201172,
      "activations/layer17_attention_weight_min": -48.779605865478516,
      "activations/layer18_attention_weight_max": 51.89182662963867,
      "activations/layer18_attention_weight_min": -44.194454193115234,
      "activations/layer19_attention_weight_max": 26.75616455078125,
      "activations/layer19_attention_weight_min": -22.49931526184082,
      "activations/layer1_attention_weight_max": 15.707895278930664,
      "activations/layer1_attention_weight_min": -14.853469848632812,
      "activations/layer20_attention_weight_max": 24.40859603881836,
      "activations/layer20_attention_weight_min": -23.72650909423828,
      "activations/layer21_attention_weight_max": 39.5090446472168,
      "activations/layer21_attention_weight_min": -19.824567794799805,
      "activations/layer22_attention_weight_max": 32.17264175415039,
      "activations/layer22_attention_weight_min": -26.12692642211914,
      "activations/layer23_attention_weight_max": 35.541866302490234,
      "activations/layer23_attention_weight_min": -24.134349822998047,
      "activations/layer2_attention_weight_max": 31.066307067871094,
      "activations/layer2_attention_weight_min": -31.04890251159668,
      "activations/layer3_attention_weight_max": 88.26759338378906,
      "activations/layer3_attention_weight_min": -88.84164428710938,
      "activations/layer4_attention_weight_max": 84.52347564697266,
      "activations/layer4_attention_weight_min": -83.98422241210938,
      "activations/layer5_attention_weight_max": 59.048152923583984,
      "activations/layer5_attention_weight_min": -71.6119155883789,
      "activations/layer6_attention_weight_max": 51.18996047973633,
      "activations/layer6_attention_weight_min": -52.66952133178711,
      "activations/layer7_attention_weight_max": 69.51864624023438,
      "activations/layer7_attention_weight_min": -62.994651794433594,
      "activations/layer8_attention_weight_max": 46.11912536621094,
      "activations/layer8_attention_weight_min": -49.73600769042969,
      "activations/layer9_attention_weight_max": 41.81483459472656,
      "activations/layer9_attention_weight_min": -45.644710540771484,
      "epoch": 21.8,
      "learning_rate": 9.488636363636363e-06,
      "loss": 2.6994,
      "step": 375250
    },
    {
      "activations/layer0_attention_weight_max": 15.804997444152832,
      "activations/layer0_attention_weight_min": -14.518622398376465,
      "activations/layer10_attention_weight_max": 37.07716369628906,
      "activations/layer10_attention_weight_min": -35.3085823059082,
      "activations/layer11_attention_weight_max": 33.368934631347656,
      "activations/layer11_attention_weight_min": -37.02831268310547,
      "activations/layer12_attention_weight_max": 19.361616134643555,
      "activations/layer12_attention_weight_min": -23.36625862121582,
      "activations/layer13_attention_weight_max": 35.416072845458984,
      "activations/layer13_attention_weight_min": -28.878089904785156,
      "activations/layer14_attention_weight_max": 35.679893493652344,
      "activations/layer14_attention_weight_min": -30.749141693115234,
      "activations/layer15_attention_weight_max": 32.47587203979492,
      "activations/layer15_attention_weight_min": -31.422632217407227,
      "activations/layer16_attention_weight_max": 35.319759368896484,
      "activations/layer16_attention_weight_min": -31.674381256103516,
      "activations/layer17_attention_weight_max": 52.22443389892578,
      "activations/layer17_attention_weight_min": -45.12908935546875,
      "activations/layer18_attention_weight_max": 50.26036834716797,
      "activations/layer18_attention_weight_min": -39.46216583251953,
      "activations/layer19_attention_weight_max": 25.0421142578125,
      "activations/layer19_attention_weight_min": -23.304519653320312,
      "activations/layer1_attention_weight_max": 14.984720230102539,
      "activations/layer1_attention_weight_min": -15.509754180908203,
      "activations/layer20_attention_weight_max": 22.535274505615234,
      "activations/layer20_attention_weight_min": -23.099945068359375,
      "activations/layer21_attention_weight_max": 34.781124114990234,
      "activations/layer21_attention_weight_min": -21.792396545410156,
      "activations/layer22_attention_weight_max": 27.609922409057617,
      "activations/layer22_attention_weight_min": -25.328813552856445,
      "activations/layer23_attention_weight_max": 41.19060134887695,
      "activations/layer23_attention_weight_min": -24.961637496948242,
      "activations/layer2_attention_weight_max": 31.218107223510742,
      "activations/layer2_attention_weight_min": -29.9776668548584,
      "activations/layer3_attention_weight_max": 91.51387023925781,
      "activations/layer3_attention_weight_min": -87.39364624023438,
      "activations/layer4_attention_weight_max": 86.12530517578125,
      "activations/layer4_attention_weight_min": -84.45616149902344,
      "activations/layer5_attention_weight_max": 62.8741569519043,
      "activations/layer5_attention_weight_min": -73.00542449951172,
      "activations/layer6_attention_weight_max": 49.87537384033203,
      "activations/layer6_attention_weight_min": -51.999446868896484,
      "activations/layer7_attention_weight_max": 67.70001220703125,
      "activations/layer7_attention_weight_min": -71.60475158691406,
      "activations/layer8_attention_weight_max": 46.30686569213867,
      "activations/layer8_attention_weight_min": -52.825870513916016,
      "activations/layer9_attention_weight_max": 43.32185363769531,
      "activations/layer9_attention_weight_min": -47.68666076660156,
      "epoch": 21.81,
      "learning_rate": 9.46969696969697e-06,
      "loss": 2.7016,
      "step": 375300
    },
    {
      "activations/layer0_attention_weight_max": 15.618915557861328,
      "activations/layer0_attention_weight_min": -14.15273380279541,
      "activations/layer10_attention_weight_max": 43.23541259765625,
      "activations/layer10_attention_weight_min": -41.13473129272461,
      "activations/layer11_attention_weight_max": 41.06230163574219,
      "activations/layer11_attention_weight_min": -39.17625427246094,
      "activations/layer12_attention_weight_max": 20.239959716796875,
      "activations/layer12_attention_weight_min": -22.930694580078125,
      "activations/layer13_attention_weight_max": 38.85447311401367,
      "activations/layer13_attention_weight_min": -27.021160125732422,
      "activations/layer14_attention_weight_max": 41.41381072998047,
      "activations/layer14_attention_weight_min": -33.75862503051758,
      "activations/layer15_attention_weight_max": 41.36651611328125,
      "activations/layer15_attention_weight_min": -32.18272018432617,
      "activations/layer16_attention_weight_max": 43.663177490234375,
      "activations/layer16_attention_weight_min": -38.393516540527344,
      "activations/layer17_attention_weight_max": 64.35330200195312,
      "activations/layer17_attention_weight_min": -51.187076568603516,
      "activations/layer18_attention_weight_max": 59.23844909667969,
      "activations/layer18_attention_weight_min": -44.94424819946289,
      "activations/layer19_attention_weight_max": 28.589984893798828,
      "activations/layer19_attention_weight_min": -24.434171676635742,
      "activations/layer1_attention_weight_max": 16.95389175415039,
      "activations/layer1_attention_weight_min": -16.92556381225586,
      "activations/layer20_attention_weight_max": 24.678409576416016,
      "activations/layer20_attention_weight_min": -21.401065826416016,
      "activations/layer21_attention_weight_max": 46.52770233154297,
      "activations/layer21_attention_weight_min": -25.20621681213379,
      "activations/layer22_attention_weight_max": 36.146785736083984,
      "activations/layer22_attention_weight_min": -25.644102096557617,
      "activations/layer23_attention_weight_max": 41.971778869628906,
      "activations/layer23_attention_weight_min": -25.277402877807617,
      "activations/layer2_attention_weight_max": 34.138404846191406,
      "activations/layer2_attention_weight_min": -31.521289825439453,
      "activations/layer3_attention_weight_max": 93.66120147705078,
      "activations/layer3_attention_weight_min": -90.36659240722656,
      "activations/layer4_attention_weight_max": 88.49048614501953,
      "activations/layer4_attention_weight_min": -85.926025390625,
      "activations/layer5_attention_weight_max": 60.15180206298828,
      "activations/layer5_attention_weight_min": -70.27124786376953,
      "activations/layer6_attention_weight_max": 51.34556198120117,
      "activations/layer6_attention_weight_min": -50.50520706176758,
      "activations/layer7_attention_weight_max": 71.09664916992188,
      "activations/layer7_attention_weight_min": -72.02528381347656,
      "activations/layer8_attention_weight_max": 56.34104537963867,
      "activations/layer8_attention_weight_min": -57.66025161743164,
      "activations/layer9_attention_weight_max": 53.966670989990234,
      "activations/layer9_attention_weight_min": -52.85237503051758,
      "epoch": 21.81,
      "learning_rate": 9.450757575757576e-06,
      "loss": 2.7057,
      "step": 375350
    },
    {
      "activations/layer0_attention_weight_max": 15.893878936767578,
      "activations/layer0_attention_weight_min": -15.058045387268066,
      "activations/layer10_attention_weight_max": 34.597164154052734,
      "activations/layer10_attention_weight_min": -36.45552444458008,
      "activations/layer11_attention_weight_max": 31.347801208496094,
      "activations/layer11_attention_weight_min": -34.59748840332031,
      "activations/layer12_attention_weight_max": 18.537763595581055,
      "activations/layer12_attention_weight_min": -24.923429489135742,
      "activations/layer13_attention_weight_max": 32.3255729675293,
      "activations/layer13_attention_weight_min": -27.424842834472656,
      "activations/layer14_attention_weight_max": 34.0414924621582,
      "activations/layer14_attention_weight_min": -29.093143463134766,
      "activations/layer15_attention_weight_max": 31.462722778320312,
      "activations/layer15_attention_weight_min": -30.15800666809082,
      "activations/layer16_attention_weight_max": 32.60182571411133,
      "activations/layer16_attention_weight_min": -30.157026290893555,
      "activations/layer17_attention_weight_max": 48.06061935424805,
      "activations/layer17_attention_weight_min": -41.32634735107422,
      "activations/layer18_attention_weight_max": 45.252769470214844,
      "activations/layer18_attention_weight_min": -39.43684005737305,
      "activations/layer19_attention_weight_max": 24.955068588256836,
      "activations/layer19_attention_weight_min": -22.619047164916992,
      "activations/layer1_attention_weight_max": 14.759699821472168,
      "activations/layer1_attention_weight_min": -15.762027740478516,
      "activations/layer20_attention_weight_max": 24.818796157836914,
      "activations/layer20_attention_weight_min": -21.5710391998291,
      "activations/layer21_attention_weight_max": 31.081310272216797,
      "activations/layer21_attention_weight_min": -23.853160858154297,
      "activations/layer22_attention_weight_max": 29.375961303710938,
      "activations/layer22_attention_weight_min": -24.155420303344727,
      "activations/layer23_attention_weight_max": 32.98516082763672,
      "activations/layer23_attention_weight_min": -24.671829223632812,
      "activations/layer2_attention_weight_max": 31.826618194580078,
      "activations/layer2_attention_weight_min": -30.6273193359375,
      "activations/layer3_attention_weight_max": 89.48615264892578,
      "activations/layer3_attention_weight_min": -87.29330444335938,
      "activations/layer4_attention_weight_max": 84.1030502319336,
      "activations/layer4_attention_weight_min": -84.43524169921875,
      "activations/layer5_attention_weight_max": 62.132972717285156,
      "activations/layer5_attention_weight_min": -70.21419525146484,
      "activations/layer6_attention_weight_max": 49.728858947753906,
      "activations/layer6_attention_weight_min": -50.185585021972656,
      "activations/layer7_attention_weight_max": 66.10832214355469,
      "activations/layer7_attention_weight_min": -67.53379821777344,
      "activations/layer8_attention_weight_max": 49.55220031738281,
      "activations/layer8_attention_weight_min": -51.78001403808594,
      "activations/layer9_attention_weight_max": 41.386817932128906,
      "activations/layer9_attention_weight_min": -45.66445541381836,
      "epoch": 21.81,
      "learning_rate": 9.431818181818181e-06,
      "loss": 2.6998,
      "step": 375400
    },
    {
      "activations/layer0_attention_weight_max": 15.613664627075195,
      "activations/layer0_attention_weight_min": -14.065800666809082,
      "activations/layer10_attention_weight_max": 35.916019439697266,
      "activations/layer10_attention_weight_min": -35.80265426635742,
      "activations/layer11_attention_weight_max": 35.77009582519531,
      "activations/layer11_attention_weight_min": -36.18815612792969,
      "activations/layer12_attention_weight_max": 19.461807250976562,
      "activations/layer12_attention_weight_min": -27.883968353271484,
      "activations/layer13_attention_weight_max": 36.26304244995117,
      "activations/layer13_attention_weight_min": -29.843425750732422,
      "activations/layer14_attention_weight_max": 35.635414123535156,
      "activations/layer14_attention_weight_min": -31.575855255126953,
      "activations/layer15_attention_weight_max": 33.61003875732422,
      "activations/layer15_attention_weight_min": -30.857168197631836,
      "activations/layer16_attention_weight_max": 33.41071701049805,
      "activations/layer16_attention_weight_min": -31.22259521484375,
      "activations/layer17_attention_weight_max": 51.858192443847656,
      "activations/layer17_attention_weight_min": -44.46907424926758,
      "activations/layer18_attention_weight_max": 47.258548736572266,
      "activations/layer18_attention_weight_min": -40.1446533203125,
      "activations/layer19_attention_weight_max": 25.208086013793945,
      "activations/layer19_attention_weight_min": -21.40997886657715,
      "activations/layer1_attention_weight_max": 15.687029838562012,
      "activations/layer1_attention_weight_min": -14.79410457611084,
      "activations/layer20_attention_weight_max": 22.291364669799805,
      "activations/layer20_attention_weight_min": -22.03672981262207,
      "activations/layer21_attention_weight_max": 35.2136344909668,
      "activations/layer21_attention_weight_min": -21.45841407775879,
      "activations/layer22_attention_weight_max": 28.28731918334961,
      "activations/layer22_attention_weight_min": -24.187030792236328,
      "activations/layer23_attention_weight_max": 35.55958557128906,
      "activations/layer23_attention_weight_min": -24.194595336914062,
      "activations/layer2_attention_weight_max": 31.18604850769043,
      "activations/layer2_attention_weight_min": -30.42884635925293,
      "activations/layer3_attention_weight_max": 89.31861877441406,
      "activations/layer3_attention_weight_min": -89.8332748413086,
      "activations/layer4_attention_weight_max": 84.32453155517578,
      "activations/layer4_attention_weight_min": -89.27303314208984,
      "activations/layer5_attention_weight_max": 59.83868408203125,
      "activations/layer5_attention_weight_min": -73.01708984375,
      "activations/layer6_attention_weight_max": 50.579288482666016,
      "activations/layer6_attention_weight_min": -51.21102523803711,
      "activations/layer7_attention_weight_max": 73.752685546875,
      "activations/layer7_attention_weight_min": -68.94605255126953,
      "activations/layer8_attention_weight_max": 46.0582160949707,
      "activations/layer8_attention_weight_min": -50.776432037353516,
      "activations/layer9_attention_weight_max": 46.28936767578125,
      "activations/layer9_attention_weight_min": -45.978763580322266,
      "epoch": 21.82,
      "learning_rate": 9.412878787878788e-06,
      "loss": 2.7078,
      "step": 375450
    },
    {
      "activations/layer0_attention_weight_max": 17.445972442626953,
      "activations/layer0_attention_weight_min": -14.387831687927246,
      "activations/layer10_attention_weight_max": 38.5911750793457,
      "activations/layer10_attention_weight_min": -39.36273193359375,
      "activations/layer11_attention_weight_max": 35.62027359008789,
      "activations/layer11_attention_weight_min": -38.82818603515625,
      "activations/layer12_attention_weight_max": 20.169523239135742,
      "activations/layer12_attention_weight_min": -23.6451416015625,
      "activations/layer13_attention_weight_max": 44.430965423583984,
      "activations/layer13_attention_weight_min": -28.922616958618164,
      "activations/layer14_attention_weight_max": 42.68794250488281,
      "activations/layer14_attention_weight_min": -33.0723876953125,
      "activations/layer15_attention_weight_max": 37.70046615600586,
      "activations/layer15_attention_weight_min": -31.75392723083496,
      "activations/layer16_attention_weight_max": 42.393619537353516,
      "activations/layer16_attention_weight_min": -37.9603271484375,
      "activations/layer17_attention_weight_max": 59.11238098144531,
      "activations/layer17_attention_weight_min": -48.74176025390625,
      "activations/layer18_attention_weight_max": 55.0123291015625,
      "activations/layer18_attention_weight_min": -43.30236053466797,
      "activations/layer19_attention_weight_max": 27.9697208404541,
      "activations/layer19_attention_weight_min": -23.99966812133789,
      "activations/layer1_attention_weight_max": 15.613877296447754,
      "activations/layer1_attention_weight_min": -14.227209091186523,
      "activations/layer20_attention_weight_max": 26.810325622558594,
      "activations/layer20_attention_weight_min": -21.585622787475586,
      "activations/layer21_attention_weight_max": 41.462921142578125,
      "activations/layer21_attention_weight_min": -22.002126693725586,
      "activations/layer22_attention_weight_max": 34.36576843261719,
      "activations/layer22_attention_weight_min": -26.63381576538086,
      "activations/layer23_attention_weight_max": 40.634544372558594,
      "activations/layer23_attention_weight_min": -22.453895568847656,
      "activations/layer2_attention_weight_max": 32.42344665527344,
      "activations/layer2_attention_weight_min": -31.137760162353516,
      "activations/layer3_attention_weight_max": 90.99616241455078,
      "activations/layer3_attention_weight_min": -87.26893615722656,
      "activations/layer4_attention_weight_max": 87.0557861328125,
      "activations/layer4_attention_weight_min": -89.59030151367188,
      "activations/layer5_attention_weight_max": 64.67522430419922,
      "activations/layer5_attention_weight_min": -77.33100891113281,
      "activations/layer6_attention_weight_max": 52.10636901855469,
      "activations/layer6_attention_weight_min": -57.05657958984375,
      "activations/layer7_attention_weight_max": 68.63706970214844,
      "activations/layer7_attention_weight_min": -74.275390625,
      "activations/layer8_attention_weight_max": 47.282222747802734,
      "activations/layer8_attention_weight_min": -53.14015197753906,
      "activations/layer9_attention_weight_max": 45.67741394042969,
      "activations/layer9_attention_weight_min": -47.32790756225586,
      "epoch": 21.82,
      "learning_rate": 9.393939393939393e-06,
      "loss": 2.7222,
      "step": 375500
    },
    {
      "activations/layer0_attention_weight_max": 16.30649757385254,
      "activations/layer0_attention_weight_min": -14.312759399414062,
      "activations/layer10_attention_weight_max": 37.3985595703125,
      "activations/layer10_attention_weight_min": -36.7078971862793,
      "activations/layer11_attention_weight_max": 35.99073028564453,
      "activations/layer11_attention_weight_min": -33.67841339111328,
      "activations/layer12_attention_weight_max": 22.6497859954834,
      "activations/layer12_attention_weight_min": -27.920141220092773,
      "activations/layer13_attention_weight_max": 42.29448318481445,
      "activations/layer13_attention_weight_min": -29.737417221069336,
      "activations/layer14_attention_weight_max": 41.72602844238281,
      "activations/layer14_attention_weight_min": -33.41939163208008,
      "activations/layer15_attention_weight_max": 39.64614486694336,
      "activations/layer15_attention_weight_min": -34.20774841308594,
      "activations/layer16_attention_weight_max": 40.483154296875,
      "activations/layer16_attention_weight_min": -37.703582763671875,
      "activations/layer17_attention_weight_max": 59.78962707519531,
      "activations/layer17_attention_weight_min": -52.05510330200195,
      "activations/layer18_attention_weight_max": 54.966522216796875,
      "activations/layer18_attention_weight_min": -46.22732925415039,
      "activations/layer19_attention_weight_max": 28.38950538635254,
      "activations/layer19_attention_weight_min": -24.777748107910156,
      "activations/layer1_attention_weight_max": 16.932308197021484,
      "activations/layer1_attention_weight_min": -15.564865112304688,
      "activations/layer20_attention_weight_max": 26.418521881103516,
      "activations/layer20_attention_weight_min": -22.655241012573242,
      "activations/layer21_attention_weight_max": 45.7819938659668,
      "activations/layer21_attention_weight_min": -24.510631561279297,
      "activations/layer22_attention_weight_max": 37.925357818603516,
      "activations/layer22_attention_weight_min": -24.73683738708496,
      "activations/layer23_attention_weight_max": 43.902339935302734,
      "activations/layer23_attention_weight_min": -23.482131958007812,
      "activations/layer2_attention_weight_max": 33.46601867675781,
      "activations/layer2_attention_weight_min": -31.444580078125,
      "activations/layer3_attention_weight_max": 88.51692962646484,
      "activations/layer3_attention_weight_min": -90.44438934326172,
      "activations/layer4_attention_weight_max": 82.16712188720703,
      "activations/layer4_attention_weight_min": -82.25906372070312,
      "activations/layer5_attention_weight_max": 59.43339157104492,
      "activations/layer5_attention_weight_min": -70.95046997070312,
      "activations/layer6_attention_weight_max": 49.053428649902344,
      "activations/layer6_attention_weight_min": -51.511375427246094,
      "activations/layer7_attention_weight_max": 68.86112976074219,
      "activations/layer7_attention_weight_min": -69.20098876953125,
      "activations/layer8_attention_weight_max": 48.51359939575195,
      "activations/layer8_attention_weight_min": -50.477439880371094,
      "activations/layer9_attention_weight_max": 44.63605499267578,
      "activations/layer9_attention_weight_min": -45.727500915527344,
      "epoch": 21.82,
      "learning_rate": 9.375e-06,
      "loss": 2.7038,
      "step": 375550
    },
    {
      "activations/layer0_attention_weight_max": 15.720946311950684,
      "activations/layer0_attention_weight_min": -15.743602752685547,
      "activations/layer10_attention_weight_max": 34.3214225769043,
      "activations/layer10_attention_weight_min": -35.23256301879883,
      "activations/layer11_attention_weight_max": 31.106252670288086,
      "activations/layer11_attention_weight_min": -34.43058776855469,
      "activations/layer12_attention_weight_max": 19.568994522094727,
      "activations/layer12_attention_weight_min": -26.75659942626953,
      "activations/layer13_attention_weight_max": 31.831722259521484,
      "activations/layer13_attention_weight_min": -26.868938446044922,
      "activations/layer14_attention_weight_max": 36.42987823486328,
      "activations/layer14_attention_weight_min": -31.17959213256836,
      "activations/layer15_attention_weight_max": 33.44921875,
      "activations/layer15_attention_weight_min": -31.061155319213867,
      "activations/layer16_attention_weight_max": 34.220130920410156,
      "activations/layer16_attention_weight_min": -33.832000732421875,
      "activations/layer17_attention_weight_max": 53.61996841430664,
      "activations/layer17_attention_weight_min": -45.89137649536133,
      "activations/layer18_attention_weight_max": 46.82745361328125,
      "activations/layer18_attention_weight_min": -40.57152557373047,
      "activations/layer19_attention_weight_max": 24.77147674560547,
      "activations/layer19_attention_weight_min": -23.16782569885254,
      "activations/layer1_attention_weight_max": 15.79383659362793,
      "activations/layer1_attention_weight_min": -13.672414779663086,
      "activations/layer20_attention_weight_max": 23.300155639648438,
      "activations/layer20_attention_weight_min": -23.256784439086914,
      "activations/layer21_attention_weight_max": 37.70238494873047,
      "activations/layer21_attention_weight_min": -21.209135055541992,
      "activations/layer22_attention_weight_max": 29.630645751953125,
      "activations/layer22_attention_weight_min": -25.25831413269043,
      "activations/layer23_attention_weight_max": 36.096214294433594,
      "activations/layer23_attention_weight_min": -22.955982208251953,
      "activations/layer2_attention_weight_max": 30.94812774658203,
      "activations/layer2_attention_weight_min": -30.40578269958496,
      "activations/layer3_attention_weight_max": 89.85368347167969,
      "activations/layer3_attention_weight_min": -89.90941619873047,
      "activations/layer4_attention_weight_max": 83.86833190917969,
      "activations/layer4_attention_weight_min": -83.50971984863281,
      "activations/layer5_attention_weight_max": 60.8297233581543,
      "activations/layer5_attention_weight_min": -72.65294647216797,
      "activations/layer6_attention_weight_max": 52.45120620727539,
      "activations/layer6_attention_weight_min": -52.63821792602539,
      "activations/layer7_attention_weight_max": 67.61408233642578,
      "activations/layer7_attention_weight_min": -66.29732513427734,
      "activations/layer8_attention_weight_max": 48.700557708740234,
      "activations/layer8_attention_weight_min": -48.837398529052734,
      "activations/layer9_attention_weight_max": 41.261043548583984,
      "activations/layer9_attention_weight_min": -44.6977653503418,
      "epoch": 21.82,
      "learning_rate": 9.356060606060606e-06,
      "loss": 2.7081,
      "step": 375600
    },
    {
      "activations/layer0_attention_weight_max": 16.727855682373047,
      "activations/layer0_attention_weight_min": -14.33166790008545,
      "activations/layer10_attention_weight_max": 38.77448272705078,
      "activations/layer10_attention_weight_min": -37.90763473510742,
      "activations/layer11_attention_weight_max": 34.286373138427734,
      "activations/layer11_attention_weight_min": -36.688812255859375,
      "activations/layer12_attention_weight_max": 19.138978958129883,
      "activations/layer12_attention_weight_min": -23.067480087280273,
      "activations/layer13_attention_weight_max": 40.34222412109375,
      "activations/layer13_attention_weight_min": -26.281599044799805,
      "activations/layer14_attention_weight_max": 34.092002868652344,
      "activations/layer14_attention_weight_min": -31.53367805480957,
      "activations/layer15_attention_weight_max": 33.222965240478516,
      "activations/layer15_attention_weight_min": -31.07839012145996,
      "activations/layer16_attention_weight_max": 33.53725814819336,
      "activations/layer16_attention_weight_min": -32.31220626831055,
      "activations/layer17_attention_weight_max": 52.20961380004883,
      "activations/layer17_attention_weight_min": -47.216529846191406,
      "activations/layer18_attention_weight_max": 49.59477233886719,
      "activations/layer18_attention_weight_min": -43.90738296508789,
      "activations/layer19_attention_weight_max": 27.243371963500977,
      "activations/layer19_attention_weight_min": -25.147768020629883,
      "activations/layer1_attention_weight_max": 15.472355842590332,
      "activations/layer1_attention_weight_min": -15.061224937438965,
      "activations/layer20_attention_weight_max": 24.831764221191406,
      "activations/layer20_attention_weight_min": -25.248943328857422,
      "activations/layer21_attention_weight_max": 38.81085205078125,
      "activations/layer21_attention_weight_min": -22.5850772857666,
      "activations/layer22_attention_weight_max": 30.24713134765625,
      "activations/layer22_attention_weight_min": -27.8541259765625,
      "activations/layer23_attention_weight_max": 36.85509490966797,
      "activations/layer23_attention_weight_min": -24.837722778320312,
      "activations/layer2_attention_weight_max": 32.667545318603516,
      "activations/layer2_attention_weight_min": -30.998126983642578,
      "activations/layer3_attention_weight_max": 92.68506622314453,
      "activations/layer3_attention_weight_min": -91.73899841308594,
      "activations/layer4_attention_weight_max": 87.21710968017578,
      "activations/layer4_attention_weight_min": -89.46379852294922,
      "activations/layer5_attention_weight_max": 62.61231994628906,
      "activations/layer5_attention_weight_min": -71.00035858154297,
      "activations/layer6_attention_weight_max": 50.77960968017578,
      "activations/layer6_attention_weight_min": -53.40855407714844,
      "activations/layer7_attention_weight_max": 67.16651916503906,
      "activations/layer7_attention_weight_min": -68.40604400634766,
      "activations/layer8_attention_weight_max": 47.816505432128906,
      "activations/layer8_attention_weight_min": -54.50673294067383,
      "activations/layer9_attention_weight_max": 44.104759216308594,
      "activations/layer9_attention_weight_min": -48.1593132019043,
      "epoch": 21.83,
      "learning_rate": 9.33712121212121e-06,
      "loss": 2.7021,
      "step": 375650
    },
    {
      "activations/layer0_attention_weight_max": 15.168688774108887,
      "activations/layer0_attention_weight_min": -14.179421424865723,
      "activations/layer10_attention_weight_max": 36.56792449951172,
      "activations/layer10_attention_weight_min": -36.7508659362793,
      "activations/layer11_attention_weight_max": 33.51789855957031,
      "activations/layer11_attention_weight_min": -36.35478591918945,
      "activations/layer12_attention_weight_max": 18.937549591064453,
      "activations/layer12_attention_weight_min": -28.00920867919922,
      "activations/layer13_attention_weight_max": 40.575347900390625,
      "activations/layer13_attention_weight_min": -30.600744247436523,
      "activations/layer14_attention_weight_max": 39.83473205566406,
      "activations/layer14_attention_weight_min": -31.88416290283203,
      "activations/layer15_attention_weight_max": 37.15039825439453,
      "activations/layer15_attention_weight_min": -32.0186882019043,
      "activations/layer16_attention_weight_max": 36.02519989013672,
      "activations/layer16_attention_weight_min": -33.87869644165039,
      "activations/layer17_attention_weight_max": 57.29593276977539,
      "activations/layer17_attention_weight_min": -48.29526901245117,
      "activations/layer18_attention_weight_max": 50.075870513916016,
      "activations/layer18_attention_weight_min": -43.96845626831055,
      "activations/layer19_attention_weight_max": 27.96921730041504,
      "activations/layer19_attention_weight_min": -23.61547088623047,
      "activations/layer1_attention_weight_max": 16.04689598083496,
      "activations/layer1_attention_weight_min": -14.055030822753906,
      "activations/layer20_attention_weight_max": 25.257080078125,
      "activations/layer20_attention_weight_min": -22.271970748901367,
      "activations/layer21_attention_weight_max": 40.88777160644531,
      "activations/layer21_attention_weight_min": -23.641836166381836,
      "activations/layer22_attention_weight_max": 31.173994064331055,
      "activations/layer22_attention_weight_min": -27.38543128967285,
      "activations/layer23_attention_weight_max": 36.46672439575195,
      "activations/layer23_attention_weight_min": -27.586132049560547,
      "activations/layer2_attention_weight_max": 33.226585388183594,
      "activations/layer2_attention_weight_min": -31.10074234008789,
      "activations/layer3_attention_weight_max": 92.06246948242188,
      "activations/layer3_attention_weight_min": -89.95384216308594,
      "activations/layer4_attention_weight_max": 85.8491439819336,
      "activations/layer4_attention_weight_min": -88.01055145263672,
      "activations/layer5_attention_weight_max": 63.742958068847656,
      "activations/layer5_attention_weight_min": -77.9781494140625,
      "activations/layer6_attention_weight_max": 49.866703033447266,
      "activations/layer6_attention_weight_min": -51.12893295288086,
      "activations/layer7_attention_weight_max": 72.59038543701172,
      "activations/layer7_attention_weight_min": -69.82157897949219,
      "activations/layer8_attention_weight_max": 48.70229721069336,
      "activations/layer8_attention_weight_min": -52.77043533325195,
      "activations/layer9_attention_weight_max": 45.916412353515625,
      "activations/layer9_attention_weight_min": -48.17938995361328,
      "epoch": 21.83,
      "learning_rate": 9.318181818181817e-06,
      "loss": 2.6928,
      "step": 375700
    },
    {
      "activations/layer0_attention_weight_max": 15.680855751037598,
      "activations/layer0_attention_weight_min": -14.310707092285156,
      "activations/layer10_attention_weight_max": 37.320274353027344,
      "activations/layer10_attention_weight_min": -36.53925323486328,
      "activations/layer11_attention_weight_max": 33.8221435546875,
      "activations/layer11_attention_weight_min": -35.66313552856445,
      "activations/layer12_attention_weight_max": 19.50130271911621,
      "activations/layer12_attention_weight_min": -23.43451499938965,
      "activations/layer13_attention_weight_max": 36.70106887817383,
      "activations/layer13_attention_weight_min": -29.914796829223633,
      "activations/layer14_attention_weight_max": 36.03883361816406,
      "activations/layer14_attention_weight_min": -31.169313430786133,
      "activations/layer15_attention_weight_max": 34.49216842651367,
      "activations/layer15_attention_weight_min": -32.08024978637695,
      "activations/layer16_attention_weight_max": 36.02313232421875,
      "activations/layer16_attention_weight_min": -33.98587417602539,
      "activations/layer17_attention_weight_max": 53.54220962524414,
      "activations/layer17_attention_weight_min": -47.06199264526367,
      "activations/layer18_attention_weight_max": 48.10040283203125,
      "activations/layer18_attention_weight_min": -44.1051139831543,
      "activations/layer19_attention_weight_max": 22.830432891845703,
      "activations/layer19_attention_weight_min": -22.444944381713867,
      "activations/layer1_attention_weight_max": 15.465703010559082,
      "activations/layer1_attention_weight_min": -14.670927047729492,
      "activations/layer20_attention_weight_max": 22.54302406311035,
      "activations/layer20_attention_weight_min": -22.770896911621094,
      "activations/layer21_attention_weight_max": 37.81536102294922,
      "activations/layer21_attention_weight_min": -23.107908248901367,
      "activations/layer22_attention_weight_max": 28.297103881835938,
      "activations/layer22_attention_weight_min": -25.59494400024414,
      "activations/layer23_attention_weight_max": 36.566673278808594,
      "activations/layer23_attention_weight_min": -24.713157653808594,
      "activations/layer2_attention_weight_max": 33.28520202636719,
      "activations/layer2_attention_weight_min": -32.05558776855469,
      "activations/layer3_attention_weight_max": 91.71757507324219,
      "activations/layer3_attention_weight_min": -89.31696319580078,
      "activations/layer4_attention_weight_max": 86.12334442138672,
      "activations/layer4_attention_weight_min": -83.06295013427734,
      "activations/layer5_attention_weight_max": 62.80255889892578,
      "activations/layer5_attention_weight_min": -69.62684631347656,
      "activations/layer6_attention_weight_max": 49.125099182128906,
      "activations/layer6_attention_weight_min": -51.6004524230957,
      "activations/layer7_attention_weight_max": 72.4506607055664,
      "activations/layer7_attention_weight_min": -66.93868255615234,
      "activations/layer8_attention_weight_max": 49.141395568847656,
      "activations/layer8_attention_weight_min": -49.955318450927734,
      "activations/layer9_attention_weight_max": 45.74706268310547,
      "activations/layer9_attention_weight_min": -46.3590202331543,
      "epoch": 21.83,
      "learning_rate": 9.299242424242422e-06,
      "loss": 2.7154,
      "step": 375750
    },
    {
      "activations/layer0_attention_weight_max": 16.837942123413086,
      "activations/layer0_attention_weight_min": -14.344804763793945,
      "activations/layer10_attention_weight_max": 35.37482452392578,
      "activations/layer10_attention_weight_min": -35.18323516845703,
      "activations/layer11_attention_weight_max": 32.824615478515625,
      "activations/layer11_attention_weight_min": -33.64009475708008,
      "activations/layer12_attention_weight_max": 19.117223739624023,
      "activations/layer12_attention_weight_min": -22.547340393066406,
      "activations/layer13_attention_weight_max": 33.39453125,
      "activations/layer13_attention_weight_min": -28.669076919555664,
      "activations/layer14_attention_weight_max": 35.11234664916992,
      "activations/layer14_attention_weight_min": -31.11605453491211,
      "activations/layer15_attention_weight_max": 36.31690216064453,
      "activations/layer15_attention_weight_min": -31.25197982788086,
      "activations/layer16_attention_weight_max": 35.5113525390625,
      "activations/layer16_attention_weight_min": -33.2149772644043,
      "activations/layer17_attention_weight_max": 50.70084762573242,
      "activations/layer17_attention_weight_min": -48.328853607177734,
      "activations/layer18_attention_weight_max": 48.99596405029297,
      "activations/layer18_attention_weight_min": -45.3266487121582,
      "activations/layer19_attention_weight_max": 24.604318618774414,
      "activations/layer19_attention_weight_min": -26.314661026000977,
      "activations/layer1_attention_weight_max": 15.046296119689941,
      "activations/layer1_attention_weight_min": -15.953866004943848,
      "activations/layer20_attention_weight_max": 22.301815032958984,
      "activations/layer20_attention_weight_min": -22.162817001342773,
      "activations/layer21_attention_weight_max": 41.9002571105957,
      "activations/layer21_attention_weight_min": -24.418832778930664,
      "activations/layer22_attention_weight_max": 28.811656951904297,
      "activations/layer22_attention_weight_min": -25.876937866210938,
      "activations/layer23_attention_weight_max": 38.7931022644043,
      "activations/layer23_attention_weight_min": -23.813398361206055,
      "activations/layer2_attention_weight_max": 31.108036041259766,
      "activations/layer2_attention_weight_min": -30.593931198120117,
      "activations/layer3_attention_weight_max": 90.31922912597656,
      "activations/layer3_attention_weight_min": -89.59101867675781,
      "activations/layer4_attention_weight_max": 87.56884002685547,
      "activations/layer4_attention_weight_min": -84.53691864013672,
      "activations/layer5_attention_weight_max": 64.92361450195312,
      "activations/layer5_attention_weight_min": -76.24872589111328,
      "activations/layer6_attention_weight_max": 50.59554672241211,
      "activations/layer6_attention_weight_min": -53.47959518432617,
      "activations/layer7_attention_weight_max": 68.19274139404297,
      "activations/layer7_attention_weight_min": -68.87406158447266,
      "activations/layer8_attention_weight_max": 47.439483642578125,
      "activations/layer8_attention_weight_min": -51.08079528808594,
      "activations/layer9_attention_weight_max": 43.66592788696289,
      "activations/layer9_attention_weight_min": -46.30487060546875,
      "epoch": 21.84,
      "learning_rate": 9.28030303030303e-06,
      "loss": 2.7017,
      "step": 375800
    },
    {
      "activations/layer0_attention_weight_max": 16.118574142456055,
      "activations/layer0_attention_weight_min": -14.212200164794922,
      "activations/layer10_attention_weight_max": 33.67548370361328,
      "activations/layer10_attention_weight_min": -33.221370697021484,
      "activations/layer11_attention_weight_max": 31.713848114013672,
      "activations/layer11_attention_weight_min": -33.360107421875,
      "activations/layer12_attention_weight_max": 18.203426361083984,
      "activations/layer12_attention_weight_min": -23.35871696472168,
      "activations/layer13_attention_weight_max": 30.470277786254883,
      "activations/layer13_attention_weight_min": -27.019390106201172,
      "activations/layer14_attention_weight_max": 33.56825637817383,
      "activations/layer14_attention_weight_min": -29.812570571899414,
      "activations/layer15_attention_weight_max": 33.80628967285156,
      "activations/layer15_attention_weight_min": -30.534297943115234,
      "activations/layer16_attention_weight_max": 32.111045837402344,
      "activations/layer16_attention_weight_min": -32.40672302246094,
      "activations/layer17_attention_weight_max": 48.52875518798828,
      "activations/layer17_attention_weight_min": -43.41541290283203,
      "activations/layer18_attention_weight_max": 42.86930465698242,
      "activations/layer18_attention_weight_min": -39.69739532470703,
      "activations/layer19_attention_weight_max": 25.276552200317383,
      "activations/layer19_attention_weight_min": -21.810537338256836,
      "activations/layer1_attention_weight_max": 15.16475772857666,
      "activations/layer1_attention_weight_min": -15.295890808105469,
      "activations/layer20_attention_weight_max": 20.856529235839844,
      "activations/layer20_attention_weight_min": -18.998350143432617,
      "activations/layer21_attention_weight_max": 31.498863220214844,
      "activations/layer21_attention_weight_min": -22.395282745361328,
      "activations/layer22_attention_weight_max": 32.3304328918457,
      "activations/layer22_attention_weight_min": -24.217082977294922,
      "activations/layer23_attention_weight_max": 37.12049102783203,
      "activations/layer23_attention_weight_min": -24.671384811401367,
      "activations/layer2_attention_weight_max": 31.68659782409668,
      "activations/layer2_attention_weight_min": -29.737667083740234,
      "activations/layer3_attention_weight_max": 87.50242614746094,
      "activations/layer3_attention_weight_min": -86.3810043334961,
      "activations/layer4_attention_weight_max": 80.04430389404297,
      "activations/layer4_attention_weight_min": -81.52172088623047,
      "activations/layer5_attention_weight_max": 65.43901062011719,
      "activations/layer5_attention_weight_min": -70.05247497558594,
      "activations/layer6_attention_weight_max": 48.800819396972656,
      "activations/layer6_attention_weight_min": -49.27157974243164,
      "activations/layer7_attention_weight_max": 67.72265625,
      "activations/layer7_attention_weight_min": -64.43589782714844,
      "activations/layer8_attention_weight_max": 44.826725006103516,
      "activations/layer8_attention_weight_min": -50.017940521240234,
      "activations/layer9_attention_weight_max": 43.35521697998047,
      "activations/layer9_attention_weight_min": -44.06260299682617,
      "epoch": 21.84,
      "learning_rate": 9.261363636363635e-06,
      "loss": 2.7036,
      "step": 375850
    },
    {
      "activations/layer0_attention_weight_max": 16.30597686767578,
      "activations/layer0_attention_weight_min": -14.665605545043945,
      "activations/layer10_attention_weight_max": 35.37287139892578,
      "activations/layer10_attention_weight_min": -34.945919036865234,
      "activations/layer11_attention_weight_max": 32.4664192199707,
      "activations/layer11_attention_weight_min": -35.65630340576172,
      "activations/layer12_attention_weight_max": 17.143747329711914,
      "activations/layer12_attention_weight_min": -23.213354110717773,
      "activations/layer13_attention_weight_max": 34.80160140991211,
      "activations/layer13_attention_weight_min": -27.39006233215332,
      "activations/layer14_attention_weight_max": 32.95718765258789,
      "activations/layer14_attention_weight_min": -30.880008697509766,
      "activations/layer15_attention_weight_max": 33.45235824584961,
      "activations/layer15_attention_weight_min": -31.252012252807617,
      "activations/layer16_attention_weight_max": 33.11682891845703,
      "activations/layer16_attention_weight_min": -32.30659484863281,
      "activations/layer17_attention_weight_max": 51.29689407348633,
      "activations/layer17_attention_weight_min": -44.695613861083984,
      "activations/layer18_attention_weight_max": 45.55834197998047,
      "activations/layer18_attention_weight_min": -38.20635223388672,
      "activations/layer19_attention_weight_max": 23.77162742614746,
      "activations/layer19_attention_weight_min": -20.635086059570312,
      "activations/layer1_attention_weight_max": 15.34166431427002,
      "activations/layer1_attention_weight_min": -15.13748550415039,
      "activations/layer20_attention_weight_max": 20.28740692138672,
      "activations/layer20_attention_weight_min": -20.852306365966797,
      "activations/layer21_attention_weight_max": 29.824138641357422,
      "activations/layer21_attention_weight_min": -20.337522506713867,
      "activations/layer22_attention_weight_max": 27.85251235961914,
      "activations/layer22_attention_weight_min": -25.19715118408203,
      "activations/layer23_attention_weight_max": 33.22977828979492,
      "activations/layer23_attention_weight_min": -23.850509643554688,
      "activations/layer2_attention_weight_max": 30.680204391479492,
      "activations/layer2_attention_weight_min": -30.187389373779297,
      "activations/layer3_attention_weight_max": 88.0125503540039,
      "activations/layer3_attention_weight_min": -91.63182067871094,
      "activations/layer4_attention_weight_max": 80.73941040039062,
      "activations/layer4_attention_weight_min": -82.50530242919922,
      "activations/layer5_attention_weight_max": 59.02273178100586,
      "activations/layer5_attention_weight_min": -75.11848449707031,
      "activations/layer6_attention_weight_max": 49.10134506225586,
      "activations/layer6_attention_weight_min": -49.068782806396484,
      "activations/layer7_attention_weight_max": 68.6636734008789,
      "activations/layer7_attention_weight_min": -67.52685546875,
      "activations/layer8_attention_weight_max": 45.99757385253906,
      "activations/layer8_attention_weight_min": -51.058876037597656,
      "activations/layer9_attention_weight_max": 42.188961029052734,
      "activations/layer9_attention_weight_min": -46.84933853149414,
      "epoch": 21.84,
      "learning_rate": 9.242424242424242e-06,
      "loss": 2.736,
      "step": 375900
    },
    {
      "activations/layer0_attention_weight_max": 15.418050765991211,
      "activations/layer0_attention_weight_min": -14.199150085449219,
      "activations/layer10_attention_weight_max": 35.0912971496582,
      "activations/layer10_attention_weight_min": -35.7180061340332,
      "activations/layer11_attention_weight_max": 31.63625717163086,
      "activations/layer11_attention_weight_min": -34.43362045288086,
      "activations/layer12_attention_weight_max": 17.80597496032715,
      "activations/layer12_attention_weight_min": -26.01227569580078,
      "activations/layer13_attention_weight_max": 28.701257705688477,
      "activations/layer13_attention_weight_min": -27.115114212036133,
      "activations/layer14_attention_weight_max": 33.76605224609375,
      "activations/layer14_attention_weight_min": -30.3465518951416,
      "activations/layer15_attention_weight_max": 32.98130416870117,
      "activations/layer15_attention_weight_min": -31.935001373291016,
      "activations/layer16_attention_weight_max": 34.75589370727539,
      "activations/layer16_attention_weight_min": -34.87055206298828,
      "activations/layer17_attention_weight_max": 52.262977600097656,
      "activations/layer17_attention_weight_min": -48.72441482543945,
      "activations/layer18_attention_weight_max": 50.33104705810547,
      "activations/layer18_attention_weight_min": -45.9367790222168,
      "activations/layer19_attention_weight_max": 27.04667091369629,
      "activations/layer19_attention_weight_min": -24.229591369628906,
      "activations/layer1_attention_weight_max": 16.351078033447266,
      "activations/layer1_attention_weight_min": -14.575790405273438,
      "activations/layer20_attention_weight_max": 24.68304443359375,
      "activations/layer20_attention_weight_min": -21.086870193481445,
      "activations/layer21_attention_weight_max": 33.49577713012695,
      "activations/layer21_attention_weight_min": -22.314605712890625,
      "activations/layer22_attention_weight_max": 29.996232986450195,
      "activations/layer22_attention_weight_min": -25.068519592285156,
      "activations/layer23_attention_weight_max": 35.43035125732422,
      "activations/layer23_attention_weight_min": -23.465538024902344,
      "activations/layer2_attention_weight_max": 32.31925964355469,
      "activations/layer2_attention_weight_min": -31.108318328857422,
      "activations/layer3_attention_weight_max": 89.29842376708984,
      "activations/layer3_attention_weight_min": -94.15824890136719,
      "activations/layer4_attention_weight_max": 82.76409149169922,
      "activations/layer4_attention_weight_min": -84.64030456542969,
      "activations/layer5_attention_weight_max": 59.288047790527344,
      "activations/layer5_attention_weight_min": -71.78910064697266,
      "activations/layer6_attention_weight_max": 49.42204284667969,
      "activations/layer6_attention_weight_min": -50.32457733154297,
      "activations/layer7_attention_weight_max": 66.55984497070312,
      "activations/layer7_attention_weight_min": -67.73653411865234,
      "activations/layer8_attention_weight_max": 47.56828689575195,
      "activations/layer8_attention_weight_min": -52.1954231262207,
      "activations/layer9_attention_weight_max": 43.800533294677734,
      "activations/layer9_attention_weight_min": -47.3217887878418,
      "epoch": 21.84,
      "learning_rate": 9.223484848484848e-06,
      "loss": 2.7167,
      "step": 375950
    },
    {
      "activations/layer0_attention_weight_max": 16.324304580688477,
      "activations/layer0_attention_weight_min": -14.188909530639648,
      "activations/layer10_attention_weight_max": 32.84428405761719,
      "activations/layer10_attention_weight_min": -34.50798416137695,
      "activations/layer11_attention_weight_max": 31.246875762939453,
      "activations/layer11_attention_weight_min": -34.49150848388672,
      "activations/layer12_attention_weight_max": 19.23030662536621,
      "activations/layer12_attention_weight_min": -25.63858985900879,
      "activations/layer13_attention_weight_max": 34.442588806152344,
      "activations/layer13_attention_weight_min": -27.133100509643555,
      "activations/layer14_attention_weight_max": 35.0035285949707,
      "activations/layer14_attention_weight_min": -30.158279418945312,
      "activations/layer15_attention_weight_max": 34.19688415527344,
      "activations/layer15_attention_weight_min": -31.3968505859375,
      "activations/layer16_attention_weight_max": 34.83213806152344,
      "activations/layer16_attention_weight_min": -34.221004486083984,
      "activations/layer17_attention_weight_max": 55.06878662109375,
      "activations/layer17_attention_weight_min": -48.22303771972656,
      "activations/layer18_attention_weight_max": 52.93983459472656,
      "activations/layer18_attention_weight_min": -45.51389694213867,
      "activations/layer19_attention_weight_max": 29.581838607788086,
      "activations/layer19_attention_weight_min": -28.267240524291992,
      "activations/layer1_attention_weight_max": 16.61989402770996,
      "activations/layer1_attention_weight_min": -16.049036026000977,
      "activations/layer20_attention_weight_max": 30.05487632751465,
      "activations/layer20_attention_weight_min": -21.94391441345215,
      "activations/layer21_attention_weight_max": 42.455177307128906,
      "activations/layer21_attention_weight_min": -24.67992401123047,
      "activations/layer22_attention_weight_max": 32.119544982910156,
      "activations/layer22_attention_weight_min": -26.708011627197266,
      "activations/layer23_attention_weight_max": 35.59154510498047,
      "activations/layer23_attention_weight_min": -24.507184982299805,
      "activations/layer2_attention_weight_max": 31.901405334472656,
      "activations/layer2_attention_weight_min": -29.901952743530273,
      "activations/layer3_attention_weight_max": 91.60011291503906,
      "activations/layer3_attention_weight_min": -89.33718872070312,
      "activations/layer4_attention_weight_max": 82.0723648071289,
      "activations/layer4_attention_weight_min": -81.5068588256836,
      "activations/layer5_attention_weight_max": 61.03080749511719,
      "activations/layer5_attention_weight_min": -67.77406311035156,
      "activations/layer6_attention_weight_max": 49.02885818481445,
      "activations/layer6_attention_weight_min": -46.71495056152344,
      "activations/layer7_attention_weight_max": 68.19612884521484,
      "activations/layer7_attention_weight_min": -63.51473617553711,
      "activations/layer8_attention_weight_max": 47.63555908203125,
      "activations/layer8_attention_weight_min": -47.90925598144531,
      "activations/layer9_attention_weight_max": 40.79206466674805,
      "activations/layer9_attention_weight_min": -41.81182861328125,
      "epoch": 21.85,
      "learning_rate": 9.204545454545453e-06,
      "loss": 2.7034,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.485,
      "eval_samples_per_second": 506.07,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.485,
      "eval_openwebtext_samples_per_second": 506.07,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9799,
      "eval_wikitext_samples_per_second": 230.316,
      "step": 376000
    },
    {
      "epoch": 21.85,
      "eval_lambada_loss": 2.3828125,
      "eval_lambada_ppl": 10.835334427562556,
      "eval_lambada_runtime": 9.5525,
      "eval_lambada_samples_per_second": 509.709,
      "step": 376000
    },
    {
      "activations/layer0_attention_weight_max": 16.919660568237305,
      "activations/layer0_attention_weight_min": -14.718462944030762,
      "activations/layer10_attention_weight_max": 35.124114990234375,
      "activations/layer10_attention_weight_min": -36.19908142089844,
      "activations/layer11_attention_weight_max": 36.07514953613281,
      "activations/layer11_attention_weight_min": -35.92797088623047,
      "activations/layer12_attention_weight_max": 19.920475006103516,
      "activations/layer12_attention_weight_min": -24.122026443481445,
      "activations/layer13_attention_weight_max": 35.854251861572266,
      "activations/layer13_attention_weight_min": -26.79645347595215,
      "activations/layer14_attention_weight_max": 32.43767547607422,
      "activations/layer14_attention_weight_min": -30.315271377563477,
      "activations/layer15_attention_weight_max": 30.900257110595703,
      "activations/layer15_attention_weight_min": -31.42367935180664,
      "activations/layer16_attention_weight_max": 32.50920486450195,
      "activations/layer16_attention_weight_min": -31.145593643188477,
      "activations/layer17_attention_weight_max": 48.562721252441406,
      "activations/layer17_attention_weight_min": -44.83732223510742,
      "activations/layer18_attention_weight_max": 47.430721282958984,
      "activations/layer18_attention_weight_min": -39.668907165527344,
      "activations/layer19_attention_weight_max": 25.53995132446289,
      "activations/layer19_attention_weight_min": -23.682384490966797,
      "activations/layer1_attention_weight_max": 15.46233081817627,
      "activations/layer1_attention_weight_min": -14.692220687866211,
      "activations/layer20_attention_weight_max": 21.095121383666992,
      "activations/layer20_attention_weight_min": -25.6032657623291,
      "activations/layer21_attention_weight_max": 38.56652069091797,
      "activations/layer21_attention_weight_min": -21.96013069152832,
      "activations/layer22_attention_weight_max": 27.875701904296875,
      "activations/layer22_attention_weight_min": -24.631103515625,
      "activations/layer23_attention_weight_max": 35.330360412597656,
      "activations/layer23_attention_weight_min": -24.765724182128906,
      "activations/layer2_attention_weight_max": 31.082355499267578,
      "activations/layer2_attention_weight_min": -30.515064239501953,
      "activations/layer3_attention_weight_max": 90.39418029785156,
      "activations/layer3_attention_weight_min": -90.66344451904297,
      "activations/layer4_attention_weight_max": 81.43797302246094,
      "activations/layer4_attention_weight_min": -84.62547302246094,
      "activations/layer5_attention_weight_max": 62.026878356933594,
      "activations/layer5_attention_weight_min": -73.17777252197266,
      "activations/layer6_attention_weight_max": 50.337425231933594,
      "activations/layer6_attention_weight_min": -49.272499084472656,
      "activations/layer7_attention_weight_max": 68.61302947998047,
      "activations/layer7_attention_weight_min": -66.66800689697266,
      "activations/layer8_attention_weight_max": 48.071163177490234,
      "activations/layer8_attention_weight_min": -49.27157974243164,
      "activations/layer9_attention_weight_max": 51.14860153198242,
      "activations/layer9_attention_weight_min": -49.05266571044922,
      "epoch": 21.85,
      "learning_rate": 9.18560606060606e-06,
      "loss": 2.7012,
      "step": 376050
    },
    {
      "activations/layer0_attention_weight_max": 16.237550735473633,
      "activations/layer0_attention_weight_min": -14.20345401763916,
      "activations/layer10_attention_weight_max": 40.71284866333008,
      "activations/layer10_attention_weight_min": -38.538787841796875,
      "activations/layer11_attention_weight_max": 40.9890022277832,
      "activations/layer11_attention_weight_min": -38.420448303222656,
      "activations/layer12_attention_weight_max": 21.270139694213867,
      "activations/layer12_attention_weight_min": -24.062986373901367,
      "activations/layer13_attention_weight_max": 35.18121337890625,
      "activations/layer13_attention_weight_min": -31.49251937866211,
      "activations/layer14_attention_weight_max": 41.86509323120117,
      "activations/layer14_attention_weight_min": -35.20487594604492,
      "activations/layer15_attention_weight_max": 36.86162185668945,
      "activations/layer15_attention_weight_min": -35.925559997558594,
      "activations/layer16_attention_weight_max": 40.11289596557617,
      "activations/layer16_attention_weight_min": -38.68388748168945,
      "activations/layer17_attention_weight_max": 54.51754379272461,
      "activations/layer17_attention_weight_min": -49.42100524902344,
      "activations/layer18_attention_weight_max": 51.41202926635742,
      "activations/layer18_attention_weight_min": -43.651206970214844,
      "activations/layer19_attention_weight_max": 25.601896286010742,
      "activations/layer19_attention_weight_min": -26.151660919189453,
      "activations/layer1_attention_weight_max": 15.637874603271484,
      "activations/layer1_attention_weight_min": -17.34335708618164,
      "activations/layer20_attention_weight_max": 24.37424087524414,
      "activations/layer20_attention_weight_min": -23.511167526245117,
      "activations/layer21_attention_weight_max": 41.19575119018555,
      "activations/layer21_attention_weight_min": -22.9320011138916,
      "activations/layer22_attention_weight_max": 31.7852783203125,
      "activations/layer22_attention_weight_min": -26.80643653869629,
      "activations/layer23_attention_weight_max": 39.8040771484375,
      "activations/layer23_attention_weight_min": -24.861936569213867,
      "activations/layer2_attention_weight_max": 33.30155944824219,
      "activations/layer2_attention_weight_min": -31.2691707611084,
      "activations/layer3_attention_weight_max": 93.67154693603516,
      "activations/layer3_attention_weight_min": -90.91043853759766,
      "activations/layer4_attention_weight_max": 88.11620330810547,
      "activations/layer4_attention_weight_min": -84.94242095947266,
      "activations/layer5_attention_weight_max": 63.420555114746094,
      "activations/layer5_attention_weight_min": -69.43959045410156,
      "activations/layer6_attention_weight_max": 51.102195739746094,
      "activations/layer6_attention_weight_min": -53.23530578613281,
      "activations/layer7_attention_weight_max": 80.261474609375,
      "activations/layer7_attention_weight_min": -71.09812927246094,
      "activations/layer8_attention_weight_max": 57.53675079345703,
      "activations/layer8_attention_weight_min": -56.51808166503906,
      "activations/layer9_attention_weight_max": 60.76450729370117,
      "activations/layer9_attention_weight_min": -49.93296813964844,
      "epoch": 21.85,
      "learning_rate": 9.166666666666665e-06,
      "loss": 2.7266,
      "step": 376100
    },
    {
      "activations/layer0_attention_weight_max": 16.236572265625,
      "activations/layer0_attention_weight_min": -14.352988243103027,
      "activations/layer10_attention_weight_max": 35.88844680786133,
      "activations/layer10_attention_weight_min": -37.009124755859375,
      "activations/layer11_attention_weight_max": 35.055938720703125,
      "activations/layer11_attention_weight_min": -38.69758605957031,
      "activations/layer12_attention_weight_max": 19.619600296020508,
      "activations/layer12_attention_weight_min": -23.6555233001709,
      "activations/layer13_attention_weight_max": 34.69550323486328,
      "activations/layer13_attention_weight_min": -29.59656524658203,
      "activations/layer14_attention_weight_max": 38.4668083190918,
      "activations/layer14_attention_weight_min": -31.82849884033203,
      "activations/layer15_attention_weight_max": 36.28770065307617,
      "activations/layer15_attention_weight_min": -32.334144592285156,
      "activations/layer16_attention_weight_max": 37.843849182128906,
      "activations/layer16_attention_weight_min": -34.96577835083008,
      "activations/layer17_attention_weight_max": 58.47789764404297,
      "activations/layer17_attention_weight_min": -49.49310302734375,
      "activations/layer18_attention_weight_max": 56.80884552001953,
      "activations/layer18_attention_weight_min": -42.754276275634766,
      "activations/layer19_attention_weight_max": 25.513635635375977,
      "activations/layer19_attention_weight_min": -23.819875717163086,
      "activations/layer1_attention_weight_max": 15.648326873779297,
      "activations/layer1_attention_weight_min": -15.93393325805664,
      "activations/layer20_attention_weight_max": 24.459190368652344,
      "activations/layer20_attention_weight_min": -21.587631225585938,
      "activations/layer21_attention_weight_max": 40.972076416015625,
      "activations/layer21_attention_weight_min": -23.644309997558594,
      "activations/layer22_attention_weight_max": 29.447635650634766,
      "activations/layer22_attention_weight_min": -27.63258934020996,
      "activations/layer23_attention_weight_max": 33.802188873291016,
      "activations/layer23_attention_weight_min": -25.221935272216797,
      "activations/layer2_attention_weight_max": 33.21584701538086,
      "activations/layer2_attention_weight_min": -32.71144104003906,
      "activations/layer3_attention_weight_max": 89.31089782714844,
      "activations/layer3_attention_weight_min": -91.7330322265625,
      "activations/layer4_attention_weight_max": 81.92877197265625,
      "activations/layer4_attention_weight_min": -86.17578125,
      "activations/layer5_attention_weight_max": 64.23973846435547,
      "activations/layer5_attention_weight_min": -71.38963317871094,
      "activations/layer6_attention_weight_max": 50.42961502075195,
      "activations/layer6_attention_weight_min": -51.594356536865234,
      "activations/layer7_attention_weight_max": 67.66242218017578,
      "activations/layer7_attention_weight_min": -72.0667953491211,
      "activations/layer8_attention_weight_max": 45.93947982788086,
      "activations/layer8_attention_weight_min": -50.28955078125,
      "activations/layer9_attention_weight_max": 42.94194412231445,
      "activations/layer9_attention_weight_min": -46.713314056396484,
      "epoch": 21.86,
      "learning_rate": 9.147727272727271e-06,
      "loss": 2.7208,
      "step": 376150
    },
    {
      "activations/layer0_attention_weight_max": 15.568525314331055,
      "activations/layer0_attention_weight_min": -14.10784912109375,
      "activations/layer10_attention_weight_max": 34.5526237487793,
      "activations/layer10_attention_weight_min": -35.02616500854492,
      "activations/layer11_attention_weight_max": 32.31341552734375,
      "activations/layer11_attention_weight_min": -34.058319091796875,
      "activations/layer12_attention_weight_max": 17.916128158569336,
      "activations/layer12_attention_weight_min": -25.509069442749023,
      "activations/layer13_attention_weight_max": 32.13878631591797,
      "activations/layer13_attention_weight_min": -26.62485694885254,
      "activations/layer14_attention_weight_max": 34.77230453491211,
      "activations/layer14_attention_weight_min": -30.532855987548828,
      "activations/layer15_attention_weight_max": 34.80306625366211,
      "activations/layer15_attention_weight_min": -32.015281677246094,
      "activations/layer16_attention_weight_max": 35.18495559692383,
      "activations/layer16_attention_weight_min": -32.13066482543945,
      "activations/layer17_attention_weight_max": 49.459739685058594,
      "activations/layer17_attention_weight_min": -46.01662063598633,
      "activations/layer18_attention_weight_max": 48.3812141418457,
      "activations/layer18_attention_weight_min": -39.79000473022461,
      "activations/layer19_attention_weight_max": 29.51198387145996,
      "activations/layer19_attention_weight_min": -23.480037689208984,
      "activations/layer1_attention_weight_max": 14.875340461730957,
      "activations/layer1_attention_weight_min": -14.983144760131836,
      "activations/layer20_attention_weight_max": 24.52814292907715,
      "activations/layer20_attention_weight_min": -22.340757369995117,
      "activations/layer21_attention_weight_max": 33.20211410522461,
      "activations/layer21_attention_weight_min": -22.596759796142578,
      "activations/layer22_attention_weight_max": 32.89118194580078,
      "activations/layer22_attention_weight_min": -25.723060607910156,
      "activations/layer23_attention_weight_max": 39.56171417236328,
      "activations/layer23_attention_weight_min": -24.401744842529297,
      "activations/layer2_attention_weight_max": 31.886192321777344,
      "activations/layer2_attention_weight_min": -29.69643783569336,
      "activations/layer3_attention_weight_max": 90.40998840332031,
      "activations/layer3_attention_weight_min": -89.68878173828125,
      "activations/layer4_attention_weight_max": 82.63065338134766,
      "activations/layer4_attention_weight_min": -83.51775360107422,
      "activations/layer5_attention_weight_max": 61.68040084838867,
      "activations/layer5_attention_weight_min": -72.652099609375,
      "activations/layer6_attention_weight_max": 47.22731018066406,
      "activations/layer6_attention_weight_min": -49.61611557006836,
      "activations/layer7_attention_weight_max": 65.88816833496094,
      "activations/layer7_attention_weight_min": -67.7567138671875,
      "activations/layer8_attention_weight_max": 47.168846130371094,
      "activations/layer8_attention_weight_min": -51.27490234375,
      "activations/layer9_attention_weight_max": 43.884063720703125,
      "activations/layer9_attention_weight_min": -45.553646087646484,
      "epoch": 21.86,
      "learning_rate": 9.128787878787878e-06,
      "loss": 2.6985,
      "step": 376200
    },
    {
      "activations/layer0_attention_weight_max": 16.701032638549805,
      "activations/layer0_attention_weight_min": -14.199352264404297,
      "activations/layer10_attention_weight_max": 37.104835510253906,
      "activations/layer10_attention_weight_min": -36.54037857055664,
      "activations/layer11_attention_weight_max": 36.53638458251953,
      "activations/layer11_attention_weight_min": -36.14828109741211,
      "activations/layer12_attention_weight_max": 19.706695556640625,
      "activations/layer12_attention_weight_min": -24.45638656616211,
      "activations/layer13_attention_weight_max": 36.823768615722656,
      "activations/layer13_attention_weight_min": -26.650175094604492,
      "activations/layer14_attention_weight_max": 34.28056335449219,
      "activations/layer14_attention_weight_min": -30.284528732299805,
      "activations/layer15_attention_weight_max": 33.64039611816406,
      "activations/layer15_attention_weight_min": -29.824546813964844,
      "activations/layer16_attention_weight_max": 33.76323699951172,
      "activations/layer16_attention_weight_min": -32.958892822265625,
      "activations/layer17_attention_weight_max": 52.717918395996094,
      "activations/layer17_attention_weight_min": -45.91621398925781,
      "activations/layer18_attention_weight_max": 46.75048065185547,
      "activations/layer18_attention_weight_min": -42.0949821472168,
      "activations/layer19_attention_weight_max": 25.461772918701172,
      "activations/layer19_attention_weight_min": -23.206811904907227,
      "activations/layer1_attention_weight_max": 14.50373363494873,
      "activations/layer1_attention_weight_min": -13.229955673217773,
      "activations/layer20_attention_weight_max": 24.46562385559082,
      "activations/layer20_attention_weight_min": -24.70415496826172,
      "activations/layer21_attention_weight_max": 37.98381423950195,
      "activations/layer21_attention_weight_min": -24.063762664794922,
      "activations/layer22_attention_weight_max": 30.547895431518555,
      "activations/layer22_attention_weight_min": -24.66117286682129,
      "activations/layer23_attention_weight_max": 37.93842315673828,
      "activations/layer23_attention_weight_min": -22.25031280517578,
      "activations/layer2_attention_weight_max": 31.212726593017578,
      "activations/layer2_attention_weight_min": -29.672536849975586,
      "activations/layer3_attention_weight_max": 90.49069213867188,
      "activations/layer3_attention_weight_min": -87.45044708251953,
      "activations/layer4_attention_weight_max": 88.52245330810547,
      "activations/layer4_attention_weight_min": -83.8258285522461,
      "activations/layer5_attention_weight_max": 61.06135559082031,
      "activations/layer5_attention_weight_min": -70.89249420166016,
      "activations/layer6_attention_weight_max": 49.32114028930664,
      "activations/layer6_attention_weight_min": -51.911441802978516,
      "activations/layer7_attention_weight_max": 66.03743743896484,
      "activations/layer7_attention_weight_min": -68.064208984375,
      "activations/layer8_attention_weight_max": 47.16838455200195,
      "activations/layer8_attention_weight_min": -52.22734069824219,
      "activations/layer9_attention_weight_max": 44.4602165222168,
      "activations/layer9_attention_weight_min": -46.72922134399414,
      "epoch": 21.86,
      "learning_rate": 9.109848484848484e-06,
      "loss": 2.7097,
      "step": 376250
    },
    {
      "activations/layer0_attention_weight_max": 15.773262977600098,
      "activations/layer0_attention_weight_min": -14.216079711914062,
      "activations/layer10_attention_weight_max": 36.14778518676758,
      "activations/layer10_attention_weight_min": -36.95620346069336,
      "activations/layer11_attention_weight_max": 33.939517974853516,
      "activations/layer11_attention_weight_min": -36.923622131347656,
      "activations/layer12_attention_weight_max": 21.03559112548828,
      "activations/layer12_attention_weight_min": -25.8100643157959,
      "activations/layer13_attention_weight_max": 51.50284957885742,
      "activations/layer13_attention_weight_min": -30.316957473754883,
      "activations/layer14_attention_weight_max": 43.46904754638672,
      "activations/layer14_attention_weight_min": -30.333301544189453,
      "activations/layer15_attention_weight_max": 41.81196212768555,
      "activations/layer15_attention_weight_min": -31.36350440979004,
      "activations/layer16_attention_weight_max": 37.93007278442383,
      "activations/layer16_attention_weight_min": -34.51082992553711,
      "activations/layer17_attention_weight_max": 56.15878677368164,
      "activations/layer17_attention_weight_min": -46.6052131652832,
      "activations/layer18_attention_weight_max": 51.59366989135742,
      "activations/layer18_attention_weight_min": -43.49764633178711,
      "activations/layer19_attention_weight_max": 25.881113052368164,
      "activations/layer19_attention_weight_min": -23.496488571166992,
      "activations/layer1_attention_weight_max": 15.432087898254395,
      "activations/layer1_attention_weight_min": -13.833635330200195,
      "activations/layer20_attention_weight_max": 24.54059410095215,
      "activations/layer20_attention_weight_min": -21.90915298461914,
      "activations/layer21_attention_weight_max": 37.53749084472656,
      "activations/layer21_attention_weight_min": -22.387367248535156,
      "activations/layer22_attention_weight_max": 29.850618362426758,
      "activations/layer22_attention_weight_min": -25.988908767700195,
      "activations/layer23_attention_weight_max": 35.680335998535156,
      "activations/layer23_attention_weight_min": -24.172691345214844,
      "activations/layer2_attention_weight_max": 32.3780517578125,
      "activations/layer2_attention_weight_min": -31.520719528198242,
      "activations/layer3_attention_weight_max": 92.07479858398438,
      "activations/layer3_attention_weight_min": -91.53370666503906,
      "activations/layer4_attention_weight_max": 85.63471984863281,
      "activations/layer4_attention_weight_min": -85.6983642578125,
      "activations/layer5_attention_weight_max": 65.31633758544922,
      "activations/layer5_attention_weight_min": -70.18412780761719,
      "activations/layer6_attention_weight_max": 52.04415512084961,
      "activations/layer6_attention_weight_min": -51.74589157104492,
      "activations/layer7_attention_weight_max": 68.32901000976562,
      "activations/layer7_attention_weight_min": -68.5024642944336,
      "activations/layer8_attention_weight_max": 50.702083587646484,
      "activations/layer8_attention_weight_min": -52.60981369018555,
      "activations/layer9_attention_weight_max": 44.267513275146484,
      "activations/layer9_attention_weight_min": -48.89789581298828,
      "epoch": 21.87,
      "learning_rate": 9.090909090909091e-06,
      "loss": 2.7126,
      "step": 376300
    },
    {
      "activations/layer0_attention_weight_max": 16.664127349853516,
      "activations/layer0_attention_weight_min": -15.004975318908691,
      "activations/layer10_attention_weight_max": 36.05181884765625,
      "activations/layer10_attention_weight_min": -37.208988189697266,
      "activations/layer11_attention_weight_max": 33.86650848388672,
      "activations/layer11_attention_weight_min": -34.934478759765625,
      "activations/layer12_attention_weight_max": 19.844690322875977,
      "activations/layer12_attention_weight_min": -28.754724502563477,
      "activations/layer13_attention_weight_max": 40.1348876953125,
      "activations/layer13_attention_weight_min": -29.697303771972656,
      "activations/layer14_attention_weight_max": 36.45381546020508,
      "activations/layer14_attention_weight_min": -32.101722717285156,
      "activations/layer15_attention_weight_max": 32.614994049072266,
      "activations/layer15_attention_weight_min": -31.756206512451172,
      "activations/layer16_attention_weight_max": 36.623634338378906,
      "activations/layer16_attention_weight_min": -33.35961151123047,
      "activations/layer17_attention_weight_max": 52.25794982910156,
      "activations/layer17_attention_weight_min": -47.147125244140625,
      "activations/layer18_attention_weight_max": 47.87603759765625,
      "activations/layer18_attention_weight_min": -41.10652160644531,
      "activations/layer19_attention_weight_max": 26.045454025268555,
      "activations/layer19_attention_weight_min": -23.353317260742188,
      "activations/layer1_attention_weight_max": 15.151204109191895,
      "activations/layer1_attention_weight_min": -14.953927993774414,
      "activations/layer20_attention_weight_max": 22.935340881347656,
      "activations/layer20_attention_weight_min": -21.449033737182617,
      "activations/layer21_attention_weight_max": 36.60935974121094,
      "activations/layer21_attention_weight_min": -21.91139793395996,
      "activations/layer22_attention_weight_max": 30.27812957763672,
      "activations/layer22_attention_weight_min": -27.183246612548828,
      "activations/layer23_attention_weight_max": 36.49435806274414,
      "activations/layer23_attention_weight_min": -23.275516510009766,
      "activations/layer2_attention_weight_max": 32.60389709472656,
      "activations/layer2_attention_weight_min": -31.167837142944336,
      "activations/layer3_attention_weight_max": 91.87104797363281,
      "activations/layer3_attention_weight_min": -91.82463073730469,
      "activations/layer4_attention_weight_max": 87.53367614746094,
      "activations/layer4_attention_weight_min": -88.32496643066406,
      "activations/layer5_attention_weight_max": 62.28186798095703,
      "activations/layer5_attention_weight_min": -74.58609771728516,
      "activations/layer6_attention_weight_max": 51.07858657836914,
      "activations/layer6_attention_weight_min": -50.9787712097168,
      "activations/layer7_attention_weight_max": 66.17781066894531,
      "activations/layer7_attention_weight_min": -66.41612243652344,
      "activations/layer8_attention_weight_max": 46.19634246826172,
      "activations/layer8_attention_weight_min": -52.02210235595703,
      "activations/layer9_attention_weight_max": 43.38945770263672,
      "activations/layer9_attention_weight_min": -49.960262298583984,
      "epoch": 21.87,
      "learning_rate": 9.071969696969696e-06,
      "loss": 2.712,
      "step": 376350
    },
    {
      "activations/layer0_attention_weight_max": 15.753823280334473,
      "activations/layer0_attention_weight_min": -14.349102973937988,
      "activations/layer10_attention_weight_max": 34.197845458984375,
      "activations/layer10_attention_weight_min": -35.90513229370117,
      "activations/layer11_attention_weight_max": 33.451812744140625,
      "activations/layer11_attention_weight_min": -35.31584930419922,
      "activations/layer12_attention_weight_max": 18.009458541870117,
      "activations/layer12_attention_weight_min": -30.207584381103516,
      "activations/layer13_attention_weight_max": 31.94424057006836,
      "activations/layer13_attention_weight_min": -26.654050827026367,
      "activations/layer14_attention_weight_max": 33.97693634033203,
      "activations/layer14_attention_weight_min": -31.14592933654785,
      "activations/layer15_attention_weight_max": 31.158405303955078,
      "activations/layer15_attention_weight_min": -31.459978103637695,
      "activations/layer16_attention_weight_max": 33.83602523803711,
      "activations/layer16_attention_weight_min": -32.896480560302734,
      "activations/layer17_attention_weight_max": 50.77958297729492,
      "activations/layer17_attention_weight_min": -46.0479850769043,
      "activations/layer18_attention_weight_max": 45.543670654296875,
      "activations/layer18_attention_weight_min": -41.47865295410156,
      "activations/layer19_attention_weight_max": 26.80580711364746,
      "activations/layer19_attention_weight_min": -24.781368255615234,
      "activations/layer1_attention_weight_max": 15.790514945983887,
      "activations/layer1_attention_weight_min": -13.473811149597168,
      "activations/layer20_attention_weight_max": 26.141075134277344,
      "activations/layer20_attention_weight_min": -23.000368118286133,
      "activations/layer21_attention_weight_max": 33.26829147338867,
      "activations/layer21_attention_weight_min": -20.43779754638672,
      "activations/layer22_attention_weight_max": 29.897111892700195,
      "activations/layer22_attention_weight_min": -26.30328369140625,
      "activations/layer23_attention_weight_max": 40.67681884765625,
      "activations/layer23_attention_weight_min": -21.776599884033203,
      "activations/layer2_attention_weight_max": 31.517559051513672,
      "activations/layer2_attention_weight_min": -29.68692398071289,
      "activations/layer3_attention_weight_max": 90.11329650878906,
      "activations/layer3_attention_weight_min": -89.60907745361328,
      "activations/layer4_attention_weight_max": 86.02606201171875,
      "activations/layer4_attention_weight_min": -88.34673309326172,
      "activations/layer5_attention_weight_max": 63.09135055541992,
      "activations/layer5_attention_weight_min": -72.54875183105469,
      "activations/layer6_attention_weight_max": 48.85932540893555,
      "activations/layer6_attention_weight_min": -49.58369827270508,
      "activations/layer7_attention_weight_max": 72.36503601074219,
      "activations/layer7_attention_weight_min": -69.97566986083984,
      "activations/layer8_attention_weight_max": 44.814979553222656,
      "activations/layer8_attention_weight_min": -47.95906448364258,
      "activations/layer9_attention_weight_max": 48.800132751464844,
      "activations/layer9_attention_weight_min": -46.98213577270508,
      "epoch": 21.87,
      "learning_rate": 9.053030303030302e-06,
      "loss": 2.7062,
      "step": 376400
    },
    {
      "activations/layer0_attention_weight_max": 15.742074012756348,
      "activations/layer0_attention_weight_min": -13.95836353302002,
      "activations/layer10_attention_weight_max": 44.940284729003906,
      "activations/layer10_attention_weight_min": -44.62392807006836,
      "activations/layer11_attention_weight_max": 42.86425018310547,
      "activations/layer11_attention_weight_min": -39.24085998535156,
      "activations/layer12_attention_weight_max": 20.371240615844727,
      "activations/layer12_attention_weight_min": -30.106599807739258,
      "activations/layer13_attention_weight_max": 36.693603515625,
      "activations/layer13_attention_weight_min": -27.761919021606445,
      "activations/layer14_attention_weight_max": 36.72224426269531,
      "activations/layer14_attention_weight_min": -30.86927604675293,
      "activations/layer15_attention_weight_max": 37.042823791503906,
      "activations/layer15_attention_weight_min": -31.767675399780273,
      "activations/layer16_attention_weight_max": 36.745513916015625,
      "activations/layer16_attention_weight_min": -33.71151351928711,
      "activations/layer17_attention_weight_max": 51.69207763671875,
      "activations/layer17_attention_weight_min": -48.33885192871094,
      "activations/layer18_attention_weight_max": 49.13298416137695,
      "activations/layer18_attention_weight_min": -42.31130599975586,
      "activations/layer19_attention_weight_max": 24.44895362854004,
      "activations/layer19_attention_weight_min": -23.105724334716797,
      "activations/layer1_attention_weight_max": 15.418313026428223,
      "activations/layer1_attention_weight_min": -13.991432189941406,
      "activations/layer20_attention_weight_max": 23.558589935302734,
      "activations/layer20_attention_weight_min": -24.34986686706543,
      "activations/layer21_attention_weight_max": 39.01247787475586,
      "activations/layer21_attention_weight_min": -23.920976638793945,
      "activations/layer22_attention_weight_max": 29.10325050354004,
      "activations/layer22_attention_weight_min": -26.134967803955078,
      "activations/layer23_attention_weight_max": 38.449981689453125,
      "activations/layer23_attention_weight_min": -24.585506439208984,
      "activations/layer2_attention_weight_max": 33.43978500366211,
      "activations/layer2_attention_weight_min": -32.23613357543945,
      "activations/layer3_attention_weight_max": 93.29595947265625,
      "activations/layer3_attention_weight_min": -91.62308502197266,
      "activations/layer4_attention_weight_max": 85.47029876708984,
      "activations/layer4_attention_weight_min": -85.08631896972656,
      "activations/layer5_attention_weight_max": 64.90776062011719,
      "activations/layer5_attention_weight_min": -78.50613403320312,
      "activations/layer6_attention_weight_max": 48.61406707763672,
      "activations/layer6_attention_weight_min": -52.30935287475586,
      "activations/layer7_attention_weight_max": 78.15637969970703,
      "activations/layer7_attention_weight_min": -70.0850601196289,
      "activations/layer8_attention_weight_max": 54.194923400878906,
      "activations/layer8_attention_weight_min": -52.27126693725586,
      "activations/layer9_attention_weight_max": 63.5277099609375,
      "activations/layer9_attention_weight_min": -50.23075485229492,
      "epoch": 21.87,
      "learning_rate": 9.034090909090907e-06,
      "loss": 2.7004,
      "step": 376450
    },
    {
      "activations/layer0_attention_weight_max": 15.63649845123291,
      "activations/layer0_attention_weight_min": -13.995820999145508,
      "activations/layer10_attention_weight_max": 33.582855224609375,
      "activations/layer10_attention_weight_min": -35.49897003173828,
      "activations/layer11_attention_weight_max": 31.622251510620117,
      "activations/layer11_attention_weight_min": -33.08421325683594,
      "activations/layer12_attention_weight_max": 19.172893524169922,
      "activations/layer12_attention_weight_min": -29.66157341003418,
      "activations/layer13_attention_weight_max": 33.180686950683594,
      "activations/layer13_attention_weight_min": -30.209169387817383,
      "activations/layer14_attention_weight_max": 34.26887893676758,
      "activations/layer14_attention_weight_min": -29.668989181518555,
      "activations/layer15_attention_weight_max": 33.77680969238281,
      "activations/layer15_attention_weight_min": -31.372726440429688,
      "activations/layer16_attention_weight_max": 37.24767303466797,
      "activations/layer16_attention_weight_min": -35.94558334350586,
      "activations/layer17_attention_weight_max": 53.87177276611328,
      "activations/layer17_attention_weight_min": -48.22578430175781,
      "activations/layer18_attention_weight_max": 49.6278190612793,
      "activations/layer18_attention_weight_min": -42.219303131103516,
      "activations/layer19_attention_weight_max": 25.685773849487305,
      "activations/layer19_attention_weight_min": -25.35097885131836,
      "activations/layer1_attention_weight_max": 15.202702522277832,
      "activations/layer1_attention_weight_min": -14.318077087402344,
      "activations/layer20_attention_weight_max": 22.82645034790039,
      "activations/layer20_attention_weight_min": -22.1632080078125,
      "activations/layer21_attention_weight_max": 40.308536529541016,
      "activations/layer21_attention_weight_min": -23.007781982421875,
      "activations/layer22_attention_weight_max": 30.27219009399414,
      "activations/layer22_attention_weight_min": -26.09992790222168,
      "activations/layer23_attention_weight_max": 35.07868957519531,
      "activations/layer23_attention_weight_min": -24.36064910888672,
      "activations/layer2_attention_weight_max": 31.097991943359375,
      "activations/layer2_attention_weight_min": -30.507774353027344,
      "activations/layer3_attention_weight_max": 87.29552459716797,
      "activations/layer3_attention_weight_min": -92.73054504394531,
      "activations/layer4_attention_weight_max": 80.7512435913086,
      "activations/layer4_attention_weight_min": -85.1729507446289,
      "activations/layer5_attention_weight_max": 59.40684127807617,
      "activations/layer5_attention_weight_min": -72.75851440429688,
      "activations/layer6_attention_weight_max": 47.97185134887695,
      "activations/layer6_attention_weight_min": -50.21836471557617,
      "activations/layer7_attention_weight_max": 65.40673065185547,
      "activations/layer7_attention_weight_min": -65.22276306152344,
      "activations/layer8_attention_weight_max": 44.27408981323242,
      "activations/layer8_attention_weight_min": -48.10383987426758,
      "activations/layer9_attention_weight_max": 41.944217681884766,
      "activations/layer9_attention_weight_min": -44.53521728515625,
      "epoch": 21.88,
      "learning_rate": 9.015530303030302e-06,
      "loss": 2.7038,
      "step": 376500
    },
    {
      "activations/layer0_attention_weight_max": 15.530316352844238,
      "activations/layer0_attention_weight_min": -14.389697074890137,
      "activations/layer10_attention_weight_max": 33.73409652709961,
      "activations/layer10_attention_weight_min": -34.96985626220703,
      "activations/layer11_attention_weight_max": 33.83525466918945,
      "activations/layer11_attention_weight_min": -36.69053649902344,
      "activations/layer12_attention_weight_max": 18.343101501464844,
      "activations/layer12_attention_weight_min": -25.51523208618164,
      "activations/layer13_attention_weight_max": 34.917076110839844,
      "activations/layer13_attention_weight_min": -27.768190383911133,
      "activations/layer14_attention_weight_max": 35.272457122802734,
      "activations/layer14_attention_weight_min": -30.148452758789062,
      "activations/layer15_attention_weight_max": 33.02471160888672,
      "activations/layer15_attention_weight_min": -31.206085205078125,
      "activations/layer16_attention_weight_max": 33.78679656982422,
      "activations/layer16_attention_weight_min": -33.01263427734375,
      "activations/layer17_attention_weight_max": 54.292938232421875,
      "activations/layer17_attention_weight_min": -43.38045120239258,
      "activations/layer18_attention_weight_max": 46.23303985595703,
      "activations/layer18_attention_weight_min": -39.57154083251953,
      "activations/layer19_attention_weight_max": 26.414356231689453,
      "activations/layer19_attention_weight_min": -21.692365646362305,
      "activations/layer1_attention_weight_max": 16.12969398498535,
      "activations/layer1_attention_weight_min": -14.830296516418457,
      "activations/layer20_attention_weight_max": 23.134069442749023,
      "activations/layer20_attention_weight_min": -21.07054328918457,
      "activations/layer21_attention_weight_max": 33.252227783203125,
      "activations/layer21_attention_weight_min": -22.735654830932617,
      "activations/layer22_attention_weight_max": 27.560497283935547,
      "activations/layer22_attention_weight_min": -24.761011123657227,
      "activations/layer23_attention_weight_max": 37.265804290771484,
      "activations/layer23_attention_weight_min": -26.654447555541992,
      "activations/layer2_attention_weight_max": 31.910247802734375,
      "activations/layer2_attention_weight_min": -30.25464630126953,
      "activations/layer3_attention_weight_max": 88.41704559326172,
      "activations/layer3_attention_weight_min": -89.80305480957031,
      "activations/layer4_attention_weight_max": 85.01077270507812,
      "activations/layer4_attention_weight_min": -83.78643035888672,
      "activations/layer5_attention_weight_max": 63.77555847167969,
      "activations/layer5_attention_weight_min": -74.07833862304688,
      "activations/layer6_attention_weight_max": 48.961097717285156,
      "activations/layer6_attention_weight_min": -49.13367462158203,
      "activations/layer7_attention_weight_max": 67.09918975830078,
      "activations/layer7_attention_weight_min": -65.88288116455078,
      "activations/layer8_attention_weight_max": 46.91650390625,
      "activations/layer8_attention_weight_min": -51.179500579833984,
      "activations/layer9_attention_weight_max": 42.542144775390625,
      "activations/layer9_attention_weight_min": -44.783084869384766,
      "epoch": 21.88,
      "learning_rate": 8.996590909090908e-06,
      "loss": 2.7129,
      "step": 376550
    },
    {
      "activations/layer0_attention_weight_max": 15.75053596496582,
      "activations/layer0_attention_weight_min": -13.970500946044922,
      "activations/layer10_attention_weight_max": 34.19123840332031,
      "activations/layer10_attention_weight_min": -35.55659866333008,
      "activations/layer11_attention_weight_max": 33.14950942993164,
      "activations/layer11_attention_weight_min": -36.99900817871094,
      "activations/layer12_attention_weight_max": 20.12629508972168,
      "activations/layer12_attention_weight_min": -25.283164978027344,
      "activations/layer13_attention_weight_max": 34.884925842285156,
      "activations/layer13_attention_weight_min": -26.89743423461914,
      "activations/layer14_attention_weight_max": 36.32642364501953,
      "activations/layer14_attention_weight_min": -32.182395935058594,
      "activations/layer15_attention_weight_max": 34.593284606933594,
      "activations/layer15_attention_weight_min": -32.218929290771484,
      "activations/layer16_attention_weight_max": 37.43231964111328,
      "activations/layer16_attention_weight_min": -36.014076232910156,
      "activations/layer17_attention_weight_max": 54.7967414855957,
      "activations/layer17_attention_weight_min": -47.772865295410156,
      "activations/layer18_attention_weight_max": 50.27959442138672,
      "activations/layer18_attention_weight_min": -41.80833053588867,
      "activations/layer19_attention_weight_max": 24.235506057739258,
      "activations/layer19_attention_weight_min": -24.036588668823242,
      "activations/layer1_attention_weight_max": 15.00900650024414,
      "activations/layer1_attention_weight_min": -13.253217697143555,
      "activations/layer20_attention_weight_max": 22.982519149780273,
      "activations/layer20_attention_weight_min": -21.731958389282227,
      "activations/layer21_attention_weight_max": 40.32009506225586,
      "activations/layer21_attention_weight_min": -23.13118553161621,
      "activations/layer22_attention_weight_max": 28.424484252929688,
      "activations/layer22_attention_weight_min": -27.317325592041016,
      "activations/layer23_attention_weight_max": 38.72038650512695,
      "activations/layer23_attention_weight_min": -24.547409057617188,
      "activations/layer2_attention_weight_max": 31.92593765258789,
      "activations/layer2_attention_weight_min": -29.866683959960938,
      "activations/layer3_attention_weight_max": 87.4931640625,
      "activations/layer3_attention_weight_min": -88.13372802734375,
      "activations/layer4_attention_weight_max": 84.39935302734375,
      "activations/layer4_attention_weight_min": -84.0138931274414,
      "activations/layer5_attention_weight_max": 63.996849060058594,
      "activations/layer5_attention_weight_min": -72.61734008789062,
      "activations/layer6_attention_weight_max": 47.29170608520508,
      "activations/layer6_attention_weight_min": -49.82700729370117,
      "activations/layer7_attention_weight_max": 65.74427032470703,
      "activations/layer7_attention_weight_min": -68.34828186035156,
      "activations/layer8_attention_weight_max": 44.64558792114258,
      "activations/layer8_attention_weight_min": -52.4003791809082,
      "activations/layer9_attention_weight_max": 42.109432220458984,
      "activations/layer9_attention_weight_min": -44.87384796142578,
      "epoch": 21.88,
      "learning_rate": 8.977651515151515e-06,
      "loss": 2.7134,
      "step": 376600
    },
    {
      "activations/layer0_attention_weight_max": 15.780191421508789,
      "activations/layer0_attention_weight_min": -13.862676620483398,
      "activations/layer10_attention_weight_max": 32.950469970703125,
      "activations/layer10_attention_weight_min": -33.970436096191406,
      "activations/layer11_attention_weight_max": 30.955625534057617,
      "activations/layer11_attention_weight_min": -33.627323150634766,
      "activations/layer12_attention_weight_max": 20.120716094970703,
      "activations/layer12_attention_weight_min": -24.296932220458984,
      "activations/layer13_attention_weight_max": 30.484228134155273,
      "activations/layer13_attention_weight_min": -26.135723114013672,
      "activations/layer14_attention_weight_max": 35.86311340332031,
      "activations/layer14_attention_weight_min": -33.06568145751953,
      "activations/layer15_attention_weight_max": 34.53270721435547,
      "activations/layer15_attention_weight_min": -34.58714294433594,
      "activations/layer16_attention_weight_max": 33.2221794128418,
      "activations/layer16_attention_weight_min": -32.414180755615234,
      "activations/layer17_attention_weight_max": 50.408485412597656,
      "activations/layer17_attention_weight_min": -47.18593215942383,
      "activations/layer18_attention_weight_max": 46.54304885864258,
      "activations/layer18_attention_weight_min": -42.67750549316406,
      "activations/layer19_attention_weight_max": 23.39112663269043,
      "activations/layer19_attention_weight_min": -24.711030960083008,
      "activations/layer1_attention_weight_max": 15.405655860900879,
      "activations/layer1_attention_weight_min": -15.222169876098633,
      "activations/layer20_attention_weight_max": 21.194766998291016,
      "activations/layer20_attention_weight_min": -25.8842830657959,
      "activations/layer21_attention_weight_max": 33.57377243041992,
      "activations/layer21_attention_weight_min": -22.727800369262695,
      "activations/layer22_attention_weight_max": 27.05376434326172,
      "activations/layer22_attention_weight_min": -26.212501525878906,
      "activations/layer23_attention_weight_max": 33.66230773925781,
      "activations/layer23_attention_weight_min": -28.406827926635742,
      "activations/layer2_attention_weight_max": 31.47296714782715,
      "activations/layer2_attention_weight_min": -31.12200927734375,
      "activations/layer3_attention_weight_max": 88.87120056152344,
      "activations/layer3_attention_weight_min": -89.57559967041016,
      "activations/layer4_attention_weight_max": 85.97901153564453,
      "activations/layer4_attention_weight_min": -81.75492858886719,
      "activations/layer5_attention_weight_max": 64.26716613769531,
      "activations/layer5_attention_weight_min": -73.47469329833984,
      "activations/layer6_attention_weight_max": 49.00617599487305,
      "activations/layer6_attention_weight_min": -49.417659759521484,
      "activations/layer7_attention_weight_max": 71.4758529663086,
      "activations/layer7_attention_weight_min": -67.11175537109375,
      "activations/layer8_attention_weight_max": 45.879493713378906,
      "activations/layer8_attention_weight_min": -48.85302734375,
      "activations/layer9_attention_weight_max": 40.60783386230469,
      "activations/layer9_attention_weight_min": -43.07689666748047,
      "epoch": 21.89,
      "learning_rate": 8.95871212121212e-06,
      "loss": 2.7107,
      "step": 376650
    },
    {
      "activations/layer0_attention_weight_max": 15.55611801147461,
      "activations/layer0_attention_weight_min": -14.087718963623047,
      "activations/layer10_attention_weight_max": 37.2497444152832,
      "activations/layer10_attention_weight_min": -37.26105499267578,
      "activations/layer11_attention_weight_max": 34.44671630859375,
      "activations/layer11_attention_weight_min": -35.136138916015625,
      "activations/layer12_attention_weight_max": 18.14891815185547,
      "activations/layer12_attention_weight_min": -23.84537696838379,
      "activations/layer13_attention_weight_max": 32.299835205078125,
      "activations/layer13_attention_weight_min": -26.682559967041016,
      "activations/layer14_attention_weight_max": 34.28765106201172,
      "activations/layer14_attention_weight_min": -29.044485092163086,
      "activations/layer15_attention_weight_max": 32.22649383544922,
      "activations/layer15_attention_weight_min": -28.829069137573242,
      "activations/layer16_attention_weight_max": 34.58745574951172,
      "activations/layer16_attention_weight_min": -32.98765182495117,
      "activations/layer17_attention_weight_max": 54.201019287109375,
      "activations/layer17_attention_weight_min": -43.4901008605957,
      "activations/layer18_attention_weight_max": 45.51730728149414,
      "activations/layer18_attention_weight_min": -38.10601043701172,
      "activations/layer19_attention_weight_max": 26.581424713134766,
      "activations/layer19_attention_weight_min": -22.430564880371094,
      "activations/layer1_attention_weight_max": 15.414268493652344,
      "activations/layer1_attention_weight_min": -13.980304718017578,
      "activations/layer20_attention_weight_max": 25.57650375366211,
      "activations/layer20_attention_weight_min": -20.992164611816406,
      "activations/layer21_attention_weight_max": 35.26344299316406,
      "activations/layer21_attention_weight_min": -20.114765167236328,
      "activations/layer22_attention_weight_max": 29.51825523376465,
      "activations/layer22_attention_weight_min": -25.446144104003906,
      "activations/layer23_attention_weight_max": 34.381595611572266,
      "activations/layer23_attention_weight_min": -22.71603775024414,
      "activations/layer2_attention_weight_max": 31.858837127685547,
      "activations/layer2_attention_weight_min": -30.462520599365234,
      "activations/layer3_attention_weight_max": 93.34842681884766,
      "activations/layer3_attention_weight_min": -90.890869140625,
      "activations/layer4_attention_weight_max": 85.06649780273438,
      "activations/layer4_attention_weight_min": -84.96100616455078,
      "activations/layer5_attention_weight_max": 62.43281173706055,
      "activations/layer5_attention_weight_min": -77.16494750976562,
      "activations/layer6_attention_weight_max": 50.45107650756836,
      "activations/layer6_attention_weight_min": -51.48886489868164,
      "activations/layer7_attention_weight_max": 70.47589111328125,
      "activations/layer7_attention_weight_min": -67.83560943603516,
      "activations/layer8_attention_weight_max": 47.921897888183594,
      "activations/layer8_attention_weight_min": -50.379600524902344,
      "activations/layer9_attention_weight_max": 47.952117919921875,
      "activations/layer9_attention_weight_min": -47.43326187133789,
      "epoch": 21.89,
      "learning_rate": 8.939772727272726e-06,
      "loss": 2.7062,
      "step": 376700
    },
    {
      "activations/layer0_attention_weight_max": 16.34744644165039,
      "activations/layer0_attention_weight_min": -14.222322463989258,
      "activations/layer10_attention_weight_max": 36.137115478515625,
      "activations/layer10_attention_weight_min": -35.058616638183594,
      "activations/layer11_attention_weight_max": 33.39406204223633,
      "activations/layer11_attention_weight_min": -35.98858642578125,
      "activations/layer12_attention_weight_max": 20.589290618896484,
      "activations/layer12_attention_weight_min": -25.790658950805664,
      "activations/layer13_attention_weight_max": 32.14271926879883,
      "activations/layer13_attention_weight_min": -26.08574676513672,
      "activations/layer14_attention_weight_max": 34.5381965637207,
      "activations/layer14_attention_weight_min": -28.643102645874023,
      "activations/layer15_attention_weight_max": 32.906028747558594,
      "activations/layer15_attention_weight_min": -29.593774795532227,
      "activations/layer16_attention_weight_max": 35.65829849243164,
      "activations/layer16_attention_weight_min": -31.490694046020508,
      "activations/layer17_attention_weight_max": 53.090694427490234,
      "activations/layer17_attention_weight_min": -46.445499420166016,
      "activations/layer18_attention_weight_max": 47.86954116821289,
      "activations/layer18_attention_weight_min": -42.54701614379883,
      "activations/layer19_attention_weight_max": 25.914663314819336,
      "activations/layer19_attention_weight_min": -22.7026309967041,
      "activations/layer1_attention_weight_max": 15.284136772155762,
      "activations/layer1_attention_weight_min": -13.754450798034668,
      "activations/layer20_attention_weight_max": 23.783166885375977,
      "activations/layer20_attention_weight_min": -23.114477157592773,
      "activations/layer21_attention_weight_max": 37.024452209472656,
      "activations/layer21_attention_weight_min": -21.530118942260742,
      "activations/layer22_attention_weight_max": 29.96657943725586,
      "activations/layer22_attention_weight_min": -26.393884658813477,
      "activations/layer23_attention_weight_max": 39.65287780761719,
      "activations/layer23_attention_weight_min": -25.290191650390625,
      "activations/layer2_attention_weight_max": 30.793556213378906,
      "activations/layer2_attention_weight_min": -29.83599090576172,
      "activations/layer3_attention_weight_max": 87.50677490234375,
      "activations/layer3_attention_weight_min": -84.3614501953125,
      "activations/layer4_attention_weight_max": 83.26580810546875,
      "activations/layer4_attention_weight_min": -82.9451904296875,
      "activations/layer5_attention_weight_max": 59.27862548828125,
      "activations/layer5_attention_weight_min": -72.56158447265625,
      "activations/layer6_attention_weight_max": 49.025535583496094,
      "activations/layer6_attention_weight_min": -50.38288497924805,
      "activations/layer7_attention_weight_max": 66.81249237060547,
      "activations/layer7_attention_weight_min": -67.60025024414062,
      "activations/layer8_attention_weight_max": 46.40523910522461,
      "activations/layer8_attention_weight_min": -48.90141677856445,
      "activations/layer9_attention_weight_max": 43.746498107910156,
      "activations/layer9_attention_weight_min": -45.68001174926758,
      "epoch": 21.89,
      "learning_rate": 8.920833333333333e-06,
      "loss": 2.7104,
      "step": 376750
    },
    {
      "activations/layer0_attention_weight_max": 15.54831600189209,
      "activations/layer0_attention_weight_min": -13.995266914367676,
      "activations/layer10_attention_weight_max": 36.897560119628906,
      "activations/layer10_attention_weight_min": -35.8929443359375,
      "activations/layer11_attention_weight_max": 35.39604187011719,
      "activations/layer11_attention_weight_min": -34.48005294799805,
      "activations/layer12_attention_weight_max": 18.34958267211914,
      "activations/layer12_attention_weight_min": -25.39577865600586,
      "activations/layer13_attention_weight_max": 33.579811096191406,
      "activations/layer13_attention_weight_min": -27.612457275390625,
      "activations/layer14_attention_weight_max": 33.879024505615234,
      "activations/layer14_attention_weight_min": -30.640045166015625,
      "activations/layer15_attention_weight_max": 31.949138641357422,
      "activations/layer15_attention_weight_min": -31.72641372680664,
      "activations/layer16_attention_weight_max": 32.65290832519531,
      "activations/layer16_attention_weight_min": -32.400390625,
      "activations/layer17_attention_weight_max": 49.47020721435547,
      "activations/layer17_attention_weight_min": -46.783966064453125,
      "activations/layer18_attention_weight_max": 47.39178466796875,
      "activations/layer18_attention_weight_min": -41.54670333862305,
      "activations/layer19_attention_weight_max": 23.17302894592285,
      "activations/layer19_attention_weight_min": -22.707006454467773,
      "activations/layer1_attention_weight_max": 14.805899620056152,
      "activations/layer1_attention_weight_min": -14.706295013427734,
      "activations/layer20_attention_weight_max": 23.603940963745117,
      "activations/layer20_attention_weight_min": -20.772428512573242,
      "activations/layer21_attention_weight_max": 33.35492706298828,
      "activations/layer21_attention_weight_min": -22.189834594726562,
      "activations/layer22_attention_weight_max": 30.409624099731445,
      "activations/layer22_attention_weight_min": -27.126951217651367,
      "activations/layer23_attention_weight_max": 35.71898651123047,
      "activations/layer23_attention_weight_min": -23.11337661743164,
      "activations/layer2_attention_weight_max": 32.36621856689453,
      "activations/layer2_attention_weight_min": -31.52129364013672,
      "activations/layer3_attention_weight_max": 92.46758270263672,
      "activations/layer3_attention_weight_min": -92.01862335205078,
      "activations/layer4_attention_weight_max": 85.24251556396484,
      "activations/layer4_attention_weight_min": -86.05648040771484,
      "activations/layer5_attention_weight_max": 60.657493591308594,
      "activations/layer5_attention_weight_min": -74.08387756347656,
      "activations/layer6_attention_weight_max": 50.55332946777344,
      "activations/layer6_attention_weight_min": -52.04713821411133,
      "activations/layer7_attention_weight_max": 67.45836639404297,
      "activations/layer7_attention_weight_min": -66.43053436279297,
      "activations/layer8_attention_weight_max": 46.6785774230957,
      "activations/layer8_attention_weight_min": -49.35655212402344,
      "activations/layer9_attention_weight_max": 42.493587493896484,
      "activations/layer9_attention_weight_min": -45.75116729736328,
      "epoch": 21.89,
      "learning_rate": 8.90189393939394e-06,
      "loss": 2.7262,
      "step": 376800
    },
    {
      "activations/layer0_attention_weight_max": 16.28275489807129,
      "activations/layer0_attention_weight_min": -14.257915496826172,
      "activations/layer10_attention_weight_max": 37.66434097290039,
      "activations/layer10_attention_weight_min": -38.75213623046875,
      "activations/layer11_attention_weight_max": 35.755615234375,
      "activations/layer11_attention_weight_min": -39.162818908691406,
      "activations/layer12_attention_weight_max": 19.06540298461914,
      "activations/layer12_attention_weight_min": -26.17107582092285,
      "activations/layer13_attention_weight_max": 36.130916595458984,
      "activations/layer13_attention_weight_min": -30.26209259033203,
      "activations/layer14_attention_weight_max": 38.900718688964844,
      "activations/layer14_attention_weight_min": -32.536285400390625,
      "activations/layer15_attention_weight_max": 39.40632629394531,
      "activations/layer15_attention_weight_min": -32.748779296875,
      "activations/layer16_attention_weight_max": 39.678321838378906,
      "activations/layer16_attention_weight_min": -38.154571533203125,
      "activations/layer17_attention_weight_max": 56.644256591796875,
      "activations/layer17_attention_weight_min": -50.30540084838867,
      "activations/layer18_attention_weight_max": 49.46680450439453,
      "activations/layer18_attention_weight_min": -42.158321380615234,
      "activations/layer19_attention_weight_max": 23.669767379760742,
      "activations/layer19_attention_weight_min": -24.424562454223633,
      "activations/layer1_attention_weight_max": 15.648992538452148,
      "activations/layer1_attention_weight_min": -14.337591171264648,
      "activations/layer20_attention_weight_max": 25.335222244262695,
      "activations/layer20_attention_weight_min": -22.509458541870117,
      "activations/layer21_attention_weight_max": 38.050933837890625,
      "activations/layer21_attention_weight_min": -23.16438865661621,
      "activations/layer22_attention_weight_max": 30.766984939575195,
      "activations/layer22_attention_weight_min": -26.25613784790039,
      "activations/layer23_attention_weight_max": 37.69207000732422,
      "activations/layer23_attention_weight_min": -22.85071563720703,
      "activations/layer2_attention_weight_max": 31.957059860229492,
      "activations/layer2_attention_weight_min": -29.833904266357422,
      "activations/layer3_attention_weight_max": 89.06773376464844,
      "activations/layer3_attention_weight_min": -85.19249725341797,
      "activations/layer4_attention_weight_max": 83.80245208740234,
      "activations/layer4_attention_weight_min": -86.38846588134766,
      "activations/layer5_attention_weight_max": 67.63471221923828,
      "activations/layer5_attention_weight_min": -75.16537475585938,
      "activations/layer6_attention_weight_max": 49.87492752075195,
      "activations/layer6_attention_weight_min": -52.437068939208984,
      "activations/layer7_attention_weight_max": 70.3477783203125,
      "activations/layer7_attention_weight_min": -73.67621612548828,
      "activations/layer8_attention_weight_max": 47.79731750488281,
      "activations/layer8_attention_weight_min": -51.86099624633789,
      "activations/layer9_attention_weight_max": 48.992530822753906,
      "activations/layer9_attention_weight_min": -49.76126480102539,
      "epoch": 21.9,
      "learning_rate": 8.882954545454544e-06,
      "loss": 2.7025,
      "step": 376850
    },
    {
      "activations/layer0_attention_weight_max": 16.097043991088867,
      "activations/layer0_attention_weight_min": -13.995039939880371,
      "activations/layer10_attention_weight_max": 33.99970245361328,
      "activations/layer10_attention_weight_min": -34.723052978515625,
      "activations/layer11_attention_weight_max": 33.2755241394043,
      "activations/layer11_attention_weight_min": -33.97920227050781,
      "activations/layer12_attention_weight_max": 20.302053451538086,
      "activations/layer12_attention_weight_min": -24.279996871948242,
      "activations/layer13_attention_weight_max": 32.11228942871094,
      "activations/layer13_attention_weight_min": -25.045454025268555,
      "activations/layer14_attention_weight_max": 34.654476165771484,
      "activations/layer14_attention_weight_min": -31.63567352294922,
      "activations/layer15_attention_weight_max": 32.67556381225586,
      "activations/layer15_attention_weight_min": -29.76161003112793,
      "activations/layer16_attention_weight_max": 34.03676986694336,
      "activations/layer16_attention_weight_min": -32.560951232910156,
      "activations/layer17_attention_weight_max": 49.36525344848633,
      "activations/layer17_attention_weight_min": -45.85143280029297,
      "activations/layer18_attention_weight_max": 45.05420684814453,
      "activations/layer18_attention_weight_min": -43.46834945678711,
      "activations/layer19_attention_weight_max": 24.839431762695312,
      "activations/layer19_attention_weight_min": -24.41986846923828,
      "activations/layer1_attention_weight_max": 14.927279472351074,
      "activations/layer1_attention_weight_min": -13.788496971130371,
      "activations/layer20_attention_weight_max": 22.222679138183594,
      "activations/layer20_attention_weight_min": -23.257421493530273,
      "activations/layer21_attention_weight_max": 32.530677795410156,
      "activations/layer21_attention_weight_min": -21.974740982055664,
      "activations/layer22_attention_weight_max": 28.643117904663086,
      "activations/layer22_attention_weight_min": -26.33066177368164,
      "activations/layer23_attention_weight_max": 35.59382629394531,
      "activations/layer23_attention_weight_min": -26.455963134765625,
      "activations/layer2_attention_weight_max": 30.477413177490234,
      "activations/layer2_attention_weight_min": -29.868846893310547,
      "activations/layer3_attention_weight_max": 86.0604019165039,
      "activations/layer3_attention_weight_min": -88.40240478515625,
      "activations/layer4_attention_weight_max": 80.17721557617188,
      "activations/layer4_attention_weight_min": -82.1908187866211,
      "activations/layer5_attention_weight_max": 59.48818588256836,
      "activations/layer5_attention_weight_min": -73.26054382324219,
      "activations/layer6_attention_weight_max": 48.105228424072266,
      "activations/layer6_attention_weight_min": -50.95248031616211,
      "activations/layer7_attention_weight_max": 66.64710235595703,
      "activations/layer7_attention_weight_min": -67.74151611328125,
      "activations/layer8_attention_weight_max": 47.66721725463867,
      "activations/layer8_attention_weight_min": -52.1535758972168,
      "activations/layer9_attention_weight_max": 42.12324523925781,
      "activations/layer9_attention_weight_min": -46.2525749206543,
      "epoch": 21.9,
      "learning_rate": 8.86401515151515e-06,
      "loss": 2.729,
      "step": 376900
    },
    {
      "activations/layer0_attention_weight_max": 15.789199829101562,
      "activations/layer0_attention_weight_min": -13.970699310302734,
      "activations/layer10_attention_weight_max": 36.38240432739258,
      "activations/layer10_attention_weight_min": -37.699031829833984,
      "activations/layer11_attention_weight_max": 34.401954650878906,
      "activations/layer11_attention_weight_min": -34.422142028808594,
      "activations/layer12_attention_weight_max": 21.433349609375,
      "activations/layer12_attention_weight_min": -24.98350715637207,
      "activations/layer13_attention_weight_max": 35.98193359375,
      "activations/layer13_attention_weight_min": -27.495325088500977,
      "activations/layer14_attention_weight_max": 37.444793701171875,
      "activations/layer14_attention_weight_min": -35.568695068359375,
      "activations/layer15_attention_weight_max": 34.348060607910156,
      "activations/layer15_attention_weight_min": -34.572349548339844,
      "activations/layer16_attention_weight_max": 36.66038513183594,
      "activations/layer16_attention_weight_min": -35.372596740722656,
      "activations/layer17_attention_weight_max": 53.30348205566406,
      "activations/layer17_attention_weight_min": -48.55663299560547,
      "activations/layer18_attention_weight_max": 50.097415924072266,
      "activations/layer18_attention_weight_min": -42.67888641357422,
      "activations/layer19_attention_weight_max": 25.906265258789062,
      "activations/layer19_attention_weight_min": -25.573457717895508,
      "activations/layer1_attention_weight_max": 15.059985160827637,
      "activations/layer1_attention_weight_min": -15.274259567260742,
      "activations/layer20_attention_weight_max": 26.312244415283203,
      "activations/layer20_attention_weight_min": -23.544921875,
      "activations/layer21_attention_weight_max": 40.83918380737305,
      "activations/layer21_attention_weight_min": -21.910202026367188,
      "activations/layer22_attention_weight_max": 28.901906967163086,
      "activations/layer22_attention_weight_min": -26.633195877075195,
      "activations/layer23_attention_weight_max": 37.43217086791992,
      "activations/layer23_attention_weight_min": -24.073528289794922,
      "activations/layer2_attention_weight_max": 31.419002532958984,
      "activations/layer2_attention_weight_min": -30.65949058532715,
      "activations/layer3_attention_weight_max": 91.73571014404297,
      "activations/layer3_attention_weight_min": -86.85057067871094,
      "activations/layer4_attention_weight_max": 88.20018768310547,
      "activations/layer4_attention_weight_min": -83.552734375,
      "activations/layer5_attention_weight_max": 64.84266662597656,
      "activations/layer5_attention_weight_min": -72.03221130371094,
      "activations/layer6_attention_weight_max": 51.23789978027344,
      "activations/layer6_attention_weight_min": -51.08991241455078,
      "activations/layer7_attention_weight_max": 69.13935852050781,
      "activations/layer7_attention_weight_min": -68.15125274658203,
      "activations/layer8_attention_weight_max": 49.50652313232422,
      "activations/layer8_attention_weight_min": -50.62346649169922,
      "activations/layer9_attention_weight_max": 44.98633575439453,
      "activations/layer9_attention_weight_min": -46.5693473815918,
      "epoch": 21.9,
      "learning_rate": 8.845075757575757e-06,
      "loss": 2.6912,
      "step": 376950
    },
    {
      "activations/layer0_attention_weight_max": 16.24736976623535,
      "activations/layer0_attention_weight_min": -13.85558795928955,
      "activations/layer10_attention_weight_max": 35.668853759765625,
      "activations/layer10_attention_weight_min": -34.56745910644531,
      "activations/layer11_attention_weight_max": 32.345787048339844,
      "activations/layer11_attention_weight_min": -33.75666046142578,
      "activations/layer12_attention_weight_max": 20.139102935791016,
      "activations/layer12_attention_weight_min": -26.816370010375977,
      "activations/layer13_attention_weight_max": 37.01702880859375,
      "activations/layer13_attention_weight_min": -26.14366912841797,
      "activations/layer14_attention_weight_max": 35.64921569824219,
      "activations/layer14_attention_weight_min": -30.118993759155273,
      "activations/layer15_attention_weight_max": 35.37152862548828,
      "activations/layer15_attention_weight_min": -28.405109405517578,
      "activations/layer16_attention_weight_max": 36.13382339477539,
      "activations/layer16_attention_weight_min": -32.502323150634766,
      "activations/layer17_attention_weight_max": 55.41645050048828,
      "activations/layer17_attention_weight_min": -44.80352020263672,
      "activations/layer18_attention_weight_max": 52.701480865478516,
      "activations/layer18_attention_weight_min": -40.313270568847656,
      "activations/layer19_attention_weight_max": 27.720212936401367,
      "activations/layer19_attention_weight_min": -22.093103408813477,
      "activations/layer1_attention_weight_max": 15.985208511352539,
      "activations/layer1_attention_weight_min": -15.883423805236816,
      "activations/layer20_attention_weight_max": 23.209117889404297,
      "activations/layer20_attention_weight_min": -20.34647560119629,
      "activations/layer21_attention_weight_max": 37.54793167114258,
      "activations/layer21_attention_weight_min": -21.62188720703125,
      "activations/layer22_attention_weight_max": 29.91899299621582,
      "activations/layer22_attention_weight_min": -23.517698287963867,
      "activations/layer23_attention_weight_max": 40.271121978759766,
      "activations/layer23_attention_weight_min": -22.007469177246094,
      "activations/layer2_attention_weight_max": 31.185888290405273,
      "activations/layer2_attention_weight_min": -30.561567306518555,
      "activations/layer3_attention_weight_max": 92.74176025390625,
      "activations/layer3_attention_weight_min": -95.0623550415039,
      "activations/layer4_attention_weight_max": 86.04364776611328,
      "activations/layer4_attention_weight_min": -85.38505554199219,
      "activations/layer5_attention_weight_max": 61.48431396484375,
      "activations/layer5_attention_weight_min": -72.73357391357422,
      "activations/layer6_attention_weight_max": 48.97431182861328,
      "activations/layer6_attention_weight_min": -52.94721603393555,
      "activations/layer7_attention_weight_max": 70.01309967041016,
      "activations/layer7_attention_weight_min": -70.25996398925781,
      "activations/layer8_attention_weight_max": 44.99419021606445,
      "activations/layer8_attention_weight_min": -49.84137725830078,
      "activations/layer9_attention_weight_max": 46.26314163208008,
      "activations/layer9_attention_weight_min": -45.79335021972656,
      "epoch": 21.91,
      "learning_rate": 8.826136363636362e-06,
      "loss": 2.6985,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_loss": 2.669921875,
      "eval_runtime": 8.47,
      "eval_samples_per_second": 506.963,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_openwebtext_loss": 2.669921875,
      "eval_openwebtext_ppl": 14.438841114275835,
      "eval_openwebtext_runtime": 8.47,
      "eval_openwebtext_samples_per_second": 506.963,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9371,
      "eval_wikitext_samples_per_second": 235.401,
      "step": 377000
    },
    {
      "epoch": 21.91,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.5185,
      "eval_lambada_samples_per_second": 511.529,
      "step": 377000
    },
    {
      "activations/layer0_attention_weight_max": 16.51365089416504,
      "activations/layer0_attention_weight_min": -14.290440559387207,
      "activations/layer10_attention_weight_max": 35.578765869140625,
      "activations/layer10_attention_weight_min": -36.41719055175781,
      "activations/layer11_attention_weight_max": 33.705894470214844,
      "activations/layer11_attention_weight_min": -34.41938781738281,
      "activations/layer12_attention_weight_max": 18.591537475585938,
      "activations/layer12_attention_weight_min": -22.951045989990234,
      "activations/layer13_attention_weight_max": 36.318504333496094,
      "activations/layer13_attention_weight_min": -31.217937469482422,
      "activations/layer14_attention_weight_max": 35.03338623046875,
      "activations/layer14_attention_weight_min": -33.41388702392578,
      "activations/layer15_attention_weight_max": 33.00763702392578,
      "activations/layer15_attention_weight_min": -32.18755340576172,
      "activations/layer16_attention_weight_max": 34.98423385620117,
      "activations/layer16_attention_weight_min": -36.286869049072266,
      "activations/layer17_attention_weight_max": 53.7601318359375,
      "activations/layer17_attention_weight_min": -49.95285415649414,
      "activations/layer18_attention_weight_max": 51.00633239746094,
      "activations/layer18_attention_weight_min": -43.998878479003906,
      "activations/layer19_attention_weight_max": 26.212615966796875,
      "activations/layer19_attention_weight_min": -23.863882064819336,
      "activations/layer1_attention_weight_max": 15.037402153015137,
      "activations/layer1_attention_weight_min": -15.309167861938477,
      "activations/layer20_attention_weight_max": 22.618986129760742,
      "activations/layer20_attention_weight_min": -21.549835205078125,
      "activations/layer21_attention_weight_max": 32.3446044921875,
      "activations/layer21_attention_weight_min": -21.612485885620117,
      "activations/layer22_attention_weight_max": 28.13825035095215,
      "activations/layer22_attention_weight_min": -25.724050521850586,
      "activations/layer23_attention_weight_max": 34.21327590942383,
      "activations/layer23_attention_weight_min": -25.195402145385742,
      "activations/layer2_attention_weight_max": 30.404821395874023,
      "activations/layer2_attention_weight_min": -29.460500717163086,
      "activations/layer3_attention_weight_max": 88.67427062988281,
      "activations/layer3_attention_weight_min": -88.019775390625,
      "activations/layer4_attention_weight_max": 88.96173095703125,
      "activations/layer4_attention_weight_min": -87.1633529663086,
      "activations/layer5_attention_weight_max": 63.55332565307617,
      "activations/layer5_attention_weight_min": -72.7402572631836,
      "activations/layer6_attention_weight_max": 49.773014068603516,
      "activations/layer6_attention_weight_min": -51.86649703979492,
      "activations/layer7_attention_weight_max": 67.50273895263672,
      "activations/layer7_attention_weight_min": -68.27954864501953,
      "activations/layer8_attention_weight_max": 48.44042205810547,
      "activations/layer8_attention_weight_min": -49.80352020263672,
      "activations/layer9_attention_weight_max": 46.555076599121094,
      "activations/layer9_attention_weight_min": -46.36922073364258,
      "epoch": 21.91,
      "learning_rate": 8.807196969696969e-06,
      "loss": 2.7294,
      "step": 377050
    },
    {
      "activations/layer0_attention_weight_max": 15.980319023132324,
      "activations/layer0_attention_weight_min": -14.218709945678711,
      "activations/layer10_attention_weight_max": 37.10971450805664,
      "activations/layer10_attention_weight_min": -35.861114501953125,
      "activations/layer11_attention_weight_max": 32.91764831542969,
      "activations/layer11_attention_weight_min": -33.73419189453125,
      "activations/layer12_attention_weight_max": 18.634445190429688,
      "activations/layer12_attention_weight_min": -26.736865997314453,
      "activations/layer13_attention_weight_max": 31.512758255004883,
      "activations/layer13_attention_weight_min": -28.30398941040039,
      "activations/layer14_attention_weight_max": 38.964569091796875,
      "activations/layer14_attention_weight_min": -30.34975814819336,
      "activations/layer15_attention_weight_max": 33.205020904541016,
      "activations/layer15_attention_weight_min": -30.027978897094727,
      "activations/layer16_attention_weight_max": 35.24242401123047,
      "activations/layer16_attention_weight_min": -34.32968521118164,
      "activations/layer17_attention_weight_max": 50.80158615112305,
      "activations/layer17_attention_weight_min": -47.24419021606445,
      "activations/layer18_attention_weight_max": 45.45047378540039,
      "activations/layer18_attention_weight_min": -41.14649963378906,
      "activations/layer19_attention_weight_max": 24.722524642944336,
      "activations/layer19_attention_weight_min": -23.63973045349121,
      "activations/layer1_attention_weight_max": 15.288317680358887,
      "activations/layer1_attention_weight_min": -13.489724159240723,
      "activations/layer20_attention_weight_max": 24.1052303314209,
      "activations/layer20_attention_weight_min": -22.190399169921875,
      "activations/layer21_attention_weight_max": 37.54792022705078,
      "activations/layer21_attention_weight_min": -24.373191833496094,
      "activations/layer22_attention_weight_max": 27.649703979492188,
      "activations/layer22_attention_weight_min": -26.415924072265625,
      "activations/layer23_attention_weight_max": 33.934356689453125,
      "activations/layer23_attention_weight_min": -25.02957534790039,
      "activations/layer2_attention_weight_max": 30.624128341674805,
      "activations/layer2_attention_weight_min": -30.08063316345215,
      "activations/layer3_attention_weight_max": 91.1375503540039,
      "activations/layer3_attention_weight_min": -87.643798828125,
      "activations/layer4_attention_weight_max": 88.09712982177734,
      "activations/layer4_attention_weight_min": -83.16122436523438,
      "activations/layer5_attention_weight_max": 64.89668273925781,
      "activations/layer5_attention_weight_min": -75.9180908203125,
      "activations/layer6_attention_weight_max": 51.52217102050781,
      "activations/layer6_attention_weight_min": -51.287410736083984,
      "activations/layer7_attention_weight_max": 69.12060546875,
      "activations/layer7_attention_weight_min": -73.03866577148438,
      "activations/layer8_attention_weight_max": 49.18765640258789,
      "activations/layer8_attention_weight_min": -55.366825103759766,
      "activations/layer9_attention_weight_max": 43.22078323364258,
      "activations/layer9_attention_weight_min": -46.403900146484375,
      "epoch": 21.91,
      "learning_rate": 8.788257575757574e-06,
      "loss": 2.7224,
      "step": 377100
    },
    {
      "activations/layer0_attention_weight_max": 16.095867156982422,
      "activations/layer0_attention_weight_min": -13.755684852600098,
      "activations/layer10_attention_weight_max": 35.14823913574219,
      "activations/layer10_attention_weight_min": -35.364837646484375,
      "activations/layer11_attention_weight_max": 32.352394104003906,
      "activations/layer11_attention_weight_min": -34.71936798095703,
      "activations/layer12_attention_weight_max": 19.535306930541992,
      "activations/layer12_attention_weight_min": -24.087459564208984,
      "activations/layer13_attention_weight_max": 31.70292854309082,
      "activations/layer13_attention_weight_min": -27.4483585357666,
      "activations/layer14_attention_weight_max": 33.46620178222656,
      "activations/layer14_attention_weight_min": -30.78110694885254,
      "activations/layer15_attention_weight_max": 30.04262351989746,
      "activations/layer15_attention_weight_min": -30.618555068969727,
      "activations/layer16_attention_weight_max": 34.02943801879883,
      "activations/layer16_attention_weight_min": -32.17333221435547,
      "activations/layer17_attention_weight_max": 51.890846252441406,
      "activations/layer17_attention_weight_min": -43.99538040161133,
      "activations/layer18_attention_weight_max": 48.075767517089844,
      "activations/layer18_attention_weight_min": -38.32018280029297,
      "activations/layer19_attention_weight_max": 25.0002384185791,
      "activations/layer19_attention_weight_min": -22.922183990478516,
      "activations/layer1_attention_weight_max": 15.904991149902344,
      "activations/layer1_attention_weight_min": -13.941864013671875,
      "activations/layer20_attention_weight_max": 23.258317947387695,
      "activations/layer20_attention_weight_min": -22.250505447387695,
      "activations/layer21_attention_weight_max": 38.083343505859375,
      "activations/layer21_attention_weight_min": -20.889659881591797,
      "activations/layer22_attention_weight_max": 28.401954650878906,
      "activations/layer22_attention_weight_min": -24.98882484436035,
      "activations/layer23_attention_weight_max": 35.69908142089844,
      "activations/layer23_attention_weight_min": -24.069461822509766,
      "activations/layer2_attention_weight_max": 29.379226684570312,
      "activations/layer2_attention_weight_min": -30.193134307861328,
      "activations/layer3_attention_weight_max": 86.08625030517578,
      "activations/layer3_attention_weight_min": -85.62638854980469,
      "activations/layer4_attention_weight_max": 81.34062957763672,
      "activations/layer4_attention_weight_min": -83.0848617553711,
      "activations/layer5_attention_weight_max": 63.55146026611328,
      "activations/layer5_attention_weight_min": -72.31817626953125,
      "activations/layer6_attention_weight_max": 49.19858932495117,
      "activations/layer6_attention_weight_min": -50.910377502441406,
      "activations/layer7_attention_weight_max": 71.1583480834961,
      "activations/layer7_attention_weight_min": -68.87133026123047,
      "activations/layer8_attention_weight_max": 44.13728332519531,
      "activations/layer8_attention_weight_min": -49.746604919433594,
      "activations/layer9_attention_weight_max": 42.18378829956055,
      "activations/layer9_attention_weight_min": -44.58216857910156,
      "epoch": 21.91,
      "learning_rate": 8.769318181818182e-06,
      "loss": 2.6944,
      "step": 377150
    },
    {
      "activations/layer0_attention_weight_max": 15.514947891235352,
      "activations/layer0_attention_weight_min": -14.130695343017578,
      "activations/layer10_attention_weight_max": 40.440792083740234,
      "activations/layer10_attention_weight_min": -39.89593505859375,
      "activations/layer11_attention_weight_max": 37.902122497558594,
      "activations/layer11_attention_weight_min": -36.75872802734375,
      "activations/layer12_attention_weight_max": 20.54496955871582,
      "activations/layer12_attention_weight_min": -26.718154907226562,
      "activations/layer13_attention_weight_max": 37.081153869628906,
      "activations/layer13_attention_weight_min": -28.31048011779785,
      "activations/layer14_attention_weight_max": 37.74417495727539,
      "activations/layer14_attention_weight_min": -31.413774490356445,
      "activations/layer15_attention_weight_max": 35.62131118774414,
      "activations/layer15_attention_weight_min": -31.124252319335938,
      "activations/layer16_attention_weight_max": 35.443931579589844,
      "activations/layer16_attention_weight_min": -33.560096740722656,
      "activations/layer17_attention_weight_max": 50.56473922729492,
      "activations/layer17_attention_weight_min": -46.03724670410156,
      "activations/layer18_attention_weight_max": 50.35480499267578,
      "activations/layer18_attention_weight_min": -40.36591720581055,
      "activations/layer19_attention_weight_max": 24.135290145874023,
      "activations/layer19_attention_weight_min": -21.89982032775879,
      "activations/layer1_attention_weight_max": 14.52021312713623,
      "activations/layer1_attention_weight_min": -13.471700668334961,
      "activations/layer20_attention_weight_max": 24.069984436035156,
      "activations/layer20_attention_weight_min": -21.525148391723633,
      "activations/layer21_attention_weight_max": 33.41107940673828,
      "activations/layer21_attention_weight_min": -20.749404907226562,
      "activations/layer22_attention_weight_max": 28.454620361328125,
      "activations/layer22_attention_weight_min": -24.600711822509766,
      "activations/layer23_attention_weight_max": 35.78013229370117,
      "activations/layer23_attention_weight_min": -24.879013061523438,
      "activations/layer2_attention_weight_max": 31.275686264038086,
      "activations/layer2_attention_weight_min": -30.57840347290039,
      "activations/layer3_attention_weight_max": 89.59634399414062,
      "activations/layer3_attention_weight_min": -91.8034896850586,
      "activations/layer4_attention_weight_max": 87.04063415527344,
      "activations/layer4_attention_weight_min": -85.16134643554688,
      "activations/layer5_attention_weight_max": 63.06407928466797,
      "activations/layer5_attention_weight_min": -73.84942626953125,
      "activations/layer6_attention_weight_max": 54.8592643737793,
      "activations/layer6_attention_weight_min": -52.94880676269531,
      "activations/layer7_attention_weight_max": 71.51664733886719,
      "activations/layer7_attention_weight_min": -72.56509399414062,
      "activations/layer8_attention_weight_max": 49.975833892822266,
      "activations/layer8_attention_weight_min": -56.60472869873047,
      "activations/layer9_attention_weight_max": 47.896724700927734,
      "activations/layer9_attention_weight_min": -48.982234954833984,
      "epoch": 21.92,
      "learning_rate": 8.750378787878787e-06,
      "loss": 2.7022,
      "step": 377200
    },
    {
      "activations/layer0_attention_weight_max": 15.982328414916992,
      "activations/layer0_attention_weight_min": -14.33935546875,
      "activations/layer10_attention_weight_max": 37.915321350097656,
      "activations/layer10_attention_weight_min": -37.116188049316406,
      "activations/layer11_attention_weight_max": 34.41875457763672,
      "activations/layer11_attention_weight_min": -35.50950241088867,
      "activations/layer12_attention_weight_max": 19.388504028320312,
      "activations/layer12_attention_weight_min": -29.464637756347656,
      "activations/layer13_attention_weight_max": 39.51979064941406,
      "activations/layer13_attention_weight_min": -29.913454055786133,
      "activations/layer14_attention_weight_max": 35.67234802246094,
      "activations/layer14_attention_weight_min": -31.422712326049805,
      "activations/layer15_attention_weight_max": 34.62320327758789,
      "activations/layer15_attention_weight_min": -32.76948928833008,
      "activations/layer16_attention_weight_max": 36.33699035644531,
      "activations/layer16_attention_weight_min": -35.53256607055664,
      "activations/layer17_attention_weight_max": 51.951141357421875,
      "activations/layer17_attention_weight_min": -50.05952835083008,
      "activations/layer18_attention_weight_max": 52.23406982421875,
      "activations/layer18_attention_weight_min": -46.69649887084961,
      "activations/layer19_attention_weight_max": 25.382862091064453,
      "activations/layer19_attention_weight_min": -25.459081649780273,
      "activations/layer1_attention_weight_max": 15.09986400604248,
      "activations/layer1_attention_weight_min": -14.946687698364258,
      "activations/layer20_attention_weight_max": 24.06448745727539,
      "activations/layer20_attention_weight_min": -25.188854217529297,
      "activations/layer21_attention_weight_max": 39.588958740234375,
      "activations/layer21_attention_weight_min": -20.652746200561523,
      "activations/layer22_attention_weight_max": 28.70346450805664,
      "activations/layer22_attention_weight_min": -25.958833694458008,
      "activations/layer23_attention_weight_max": 32.65748596191406,
      "activations/layer23_attention_weight_min": -23.693634033203125,
      "activations/layer2_attention_weight_max": 31.902006149291992,
      "activations/layer2_attention_weight_min": -30.465675354003906,
      "activations/layer3_attention_weight_max": 93.06046295166016,
      "activations/layer3_attention_weight_min": -89.2197265625,
      "activations/layer4_attention_weight_max": 90.8114242553711,
      "activations/layer4_attention_weight_min": -88.57266998291016,
      "activations/layer5_attention_weight_max": 63.182708740234375,
      "activations/layer5_attention_weight_min": -72.60560607910156,
      "activations/layer6_attention_weight_max": 53.207027435302734,
      "activations/layer6_attention_weight_min": -53.248268127441406,
      "activations/layer7_attention_weight_max": 68.05926513671875,
      "activations/layer7_attention_weight_min": -71.53031921386719,
      "activations/layer8_attention_weight_max": 48.89004135131836,
      "activations/layer8_attention_weight_min": -50.28045654296875,
      "activations/layer9_attention_weight_max": 47.03780746459961,
      "activations/layer9_attention_weight_min": -49.79084396362305,
      "epoch": 21.92,
      "learning_rate": 8.731439393939393e-06,
      "loss": 2.7071,
      "step": 377250
    },
    {
      "activations/layer0_attention_weight_max": 16.244674682617188,
      "activations/layer0_attention_weight_min": -14.058097839355469,
      "activations/layer10_attention_weight_max": 37.30916976928711,
      "activations/layer10_attention_weight_min": -37.98323059082031,
      "activations/layer11_attention_weight_max": 36.817726135253906,
      "activations/layer11_attention_weight_min": -35.370906829833984,
      "activations/layer12_attention_weight_max": 20.823970794677734,
      "activations/layer12_attention_weight_min": -27.451448440551758,
      "activations/layer13_attention_weight_max": 42.41218185424805,
      "activations/layer13_attention_weight_min": -33.56438446044922,
      "activations/layer14_attention_weight_max": 43.06864547729492,
      "activations/layer14_attention_weight_min": -33.11397933959961,
      "activations/layer15_attention_weight_max": 38.93840789794922,
      "activations/layer15_attention_weight_min": -32.148128509521484,
      "activations/layer16_attention_weight_max": 37.33357238769531,
      "activations/layer16_attention_weight_min": -35.90088653564453,
      "activations/layer17_attention_weight_max": 55.631099700927734,
      "activations/layer17_attention_weight_min": -50.690792083740234,
      "activations/layer18_attention_weight_max": 51.43384552001953,
      "activations/layer18_attention_weight_min": -44.97285079956055,
      "activations/layer19_attention_weight_max": 32.79903030395508,
      "activations/layer19_attention_weight_min": -26.9456844329834,
      "activations/layer1_attention_weight_max": 15.804808616638184,
      "activations/layer1_attention_weight_min": -15.425420761108398,
      "activations/layer20_attention_weight_max": 31.596105575561523,
      "activations/layer20_attention_weight_min": -23.724178314208984,
      "activations/layer21_attention_weight_max": 47.07322311401367,
      "activations/layer21_attention_weight_min": -23.89903450012207,
      "activations/layer22_attention_weight_max": 32.28732681274414,
      "activations/layer22_attention_weight_min": -28.545074462890625,
      "activations/layer23_attention_weight_max": 44.8221549987793,
      "activations/layer23_attention_weight_min": -25.926048278808594,
      "activations/layer2_attention_weight_max": 33.8436164855957,
      "activations/layer2_attention_weight_min": -32.27880859375,
      "activations/layer3_attention_weight_max": 89.86388397216797,
      "activations/layer3_attention_weight_min": -90.17012786865234,
      "activations/layer4_attention_weight_max": 83.8353500366211,
      "activations/layer4_attention_weight_min": -83.4553451538086,
      "activations/layer5_attention_weight_max": 63.09238815307617,
      "activations/layer5_attention_weight_min": -71.67786407470703,
      "activations/layer6_attention_weight_max": 47.87753677368164,
      "activations/layer6_attention_weight_min": -48.553871154785156,
      "activations/layer7_attention_weight_max": 70.75392150878906,
      "activations/layer7_attention_weight_min": -70.47271728515625,
      "activations/layer8_attention_weight_max": 49.685035705566406,
      "activations/layer8_attention_weight_min": -50.813987731933594,
      "activations/layer9_attention_weight_max": 52.377288818359375,
      "activations/layer9_attention_weight_min": -47.39326858520508,
      "epoch": 21.92,
      "learning_rate": 8.7125e-06,
      "loss": 2.7216,
      "step": 377300
    },
    {
      "activations/layer0_attention_weight_max": 15.566058158874512,
      "activations/layer0_attention_weight_min": -13.805870056152344,
      "activations/layer10_attention_weight_max": 45.31696701049805,
      "activations/layer10_attention_weight_min": -42.019439697265625,
      "activations/layer11_attention_weight_max": 41.39239501953125,
      "activations/layer11_attention_weight_min": -40.123775482177734,
      "activations/layer12_attention_weight_max": 20.09568977355957,
      "activations/layer12_attention_weight_min": -32.533958435058594,
      "activations/layer13_attention_weight_max": 36.931785583496094,
      "activations/layer13_attention_weight_min": -28.758554458618164,
      "activations/layer14_attention_weight_max": 39.24185562133789,
      "activations/layer14_attention_weight_min": -30.384506225585938,
      "activations/layer15_attention_weight_max": 35.78603744506836,
      "activations/layer15_attention_weight_min": -32.1656608581543,
      "activations/layer16_attention_weight_max": 35.93571090698242,
      "activations/layer16_attention_weight_min": -34.47743606567383,
      "activations/layer17_attention_weight_max": 53.578392028808594,
      "activations/layer17_attention_weight_min": -48.1887092590332,
      "activations/layer18_attention_weight_max": 50.3336296081543,
      "activations/layer18_attention_weight_min": -43.94098663330078,
      "activations/layer19_attention_weight_max": 26.088857650756836,
      "activations/layer19_attention_weight_min": -24.631654739379883,
      "activations/layer1_attention_weight_max": 16.141197204589844,
      "activations/layer1_attention_weight_min": -15.277495384216309,
      "activations/layer20_attention_weight_max": 27.563182830810547,
      "activations/layer20_attention_weight_min": -20.363908767700195,
      "activations/layer21_attention_weight_max": 40.761207580566406,
      "activations/layer21_attention_weight_min": -21.87479019165039,
      "activations/layer22_attention_weight_max": 30.5568904876709,
      "activations/layer22_attention_weight_min": -23.005653381347656,
      "activations/layer23_attention_weight_max": 34.95661926269531,
      "activations/layer23_attention_weight_min": -24.39981460571289,
      "activations/layer2_attention_weight_max": 36.68900680541992,
      "activations/layer2_attention_weight_min": -31.380096435546875,
      "activations/layer3_attention_weight_max": 93.94779968261719,
      "activations/layer3_attention_weight_min": -93.52938842773438,
      "activations/layer4_attention_weight_max": 90.54417419433594,
      "activations/layer4_attention_weight_min": -88.31242370605469,
      "activations/layer5_attention_weight_max": 65.84046936035156,
      "activations/layer5_attention_weight_min": -76.18601989746094,
      "activations/layer6_attention_weight_max": 54.74230194091797,
      "activations/layer6_attention_weight_min": -54.791526794433594,
      "activations/layer7_attention_weight_max": 76.04653930664062,
      "activations/layer7_attention_weight_min": -70.62303924560547,
      "activations/layer8_attention_weight_max": 53.41884231567383,
      "activations/layer8_attention_weight_min": -56.1671028137207,
      "activations/layer9_attention_weight_max": 51.20863723754883,
      "activations/layer9_attention_weight_min": -50.985599517822266,
      "epoch": 21.93,
      "learning_rate": 8.693560606060605e-06,
      "loss": 2.7078,
      "step": 377350
    },
    {
      "activations/layer0_attention_weight_max": 16.26974105834961,
      "activations/layer0_attention_weight_min": -14.126130104064941,
      "activations/layer10_attention_weight_max": 42.10499954223633,
      "activations/layer10_attention_weight_min": -42.013214111328125,
      "activations/layer11_attention_weight_max": 38.066162109375,
      "activations/layer11_attention_weight_min": -38.677520751953125,
      "activations/layer12_attention_weight_max": 21.5146427154541,
      "activations/layer12_attention_weight_min": -26.409584045410156,
      "activations/layer13_attention_weight_max": 40.871543884277344,
      "activations/layer13_attention_weight_min": -31.049312591552734,
      "activations/layer14_attention_weight_max": 40.433170318603516,
      "activations/layer14_attention_weight_min": -31.83659553527832,
      "activations/layer15_attention_weight_max": 38.632625579833984,
      "activations/layer15_attention_weight_min": -32.688392639160156,
      "activations/layer16_attention_weight_max": 41.12912368774414,
      "activations/layer16_attention_weight_min": -35.61000061035156,
      "activations/layer17_attention_weight_max": 57.88831329345703,
      "activations/layer17_attention_weight_min": -47.09818649291992,
      "activations/layer18_attention_weight_max": 55.61609649658203,
      "activations/layer18_attention_weight_min": -42.711143493652344,
      "activations/layer19_attention_weight_max": 25.5423583984375,
      "activations/layer19_attention_weight_min": -21.99702262878418,
      "activations/layer1_attention_weight_max": 15.074560165405273,
      "activations/layer1_attention_weight_min": -13.586179733276367,
      "activations/layer20_attention_weight_max": 23.666101455688477,
      "activations/layer20_attention_weight_min": -19.617774963378906,
      "activations/layer21_attention_weight_max": 33.09611892700195,
      "activations/layer21_attention_weight_min": -19.701309204101562,
      "activations/layer22_attention_weight_max": 28.286409378051758,
      "activations/layer22_attention_weight_min": -24.758821487426758,
      "activations/layer23_attention_weight_max": 36.423011779785156,
      "activations/layer23_attention_weight_min": -22.679807662963867,
      "activations/layer2_attention_weight_max": 33.55646514892578,
      "activations/layer2_attention_weight_min": -31.627750396728516,
      "activations/layer3_attention_weight_max": 92.468994140625,
      "activations/layer3_attention_weight_min": -92.01455688476562,
      "activations/layer4_attention_weight_max": 89.73006439208984,
      "activations/layer4_attention_weight_min": -88.30223083496094,
      "activations/layer5_attention_weight_max": 66.12176513671875,
      "activations/layer5_attention_weight_min": -77.20323944091797,
      "activations/layer6_attention_weight_max": 52.71033477783203,
      "activations/layer6_attention_weight_min": -54.26565933227539,
      "activations/layer7_attention_weight_max": 72.0799560546875,
      "activations/layer7_attention_weight_min": -72.3012924194336,
      "activations/layer8_attention_weight_max": 52.00108337402344,
      "activations/layer8_attention_weight_min": -56.80995178222656,
      "activations/layer9_attention_weight_max": 51.59522247314453,
      "activations/layer9_attention_weight_min": -51.7338752746582,
      "epoch": 21.93,
      "learning_rate": 8.674621212121211e-06,
      "loss": 2.6947,
      "step": 377400
    },
    {
      "activations/layer0_attention_weight_max": 15.579649925231934,
      "activations/layer0_attention_weight_min": -14.024706840515137,
      "activations/layer10_attention_weight_max": 37.06887435913086,
      "activations/layer10_attention_weight_min": -36.83993911743164,
      "activations/layer11_attention_weight_max": 35.91824722290039,
      "activations/layer11_attention_weight_min": -36.1461181640625,
      "activations/layer12_attention_weight_max": 20.604589462280273,
      "activations/layer12_attention_weight_min": -24.1571102142334,
      "activations/layer13_attention_weight_max": 36.19231414794922,
      "activations/layer13_attention_weight_min": -27.876235961914062,
      "activations/layer14_attention_weight_max": 36.88178253173828,
      "activations/layer14_attention_weight_min": -32.89906692504883,
      "activations/layer15_attention_weight_max": 33.30329513549805,
      "activations/layer15_attention_weight_min": -31.616613388061523,
      "activations/layer16_attention_weight_max": 34.96666717529297,
      "activations/layer16_attention_weight_min": -33.76957702636719,
      "activations/layer17_attention_weight_max": 52.25518035888672,
      "activations/layer17_attention_weight_min": -44.86561584472656,
      "activations/layer18_attention_weight_max": 47.840938568115234,
      "activations/layer18_attention_weight_min": -44.1300048828125,
      "activations/layer19_attention_weight_max": 24.748924255371094,
      "activations/layer19_attention_weight_min": -26.91432762145996,
      "activations/layer1_attention_weight_max": 16.12818717956543,
      "activations/layer1_attention_weight_min": -14.181534767150879,
      "activations/layer20_attention_weight_max": 24.989351272583008,
      "activations/layer20_attention_weight_min": -23.569231033325195,
      "activations/layer21_attention_weight_max": 36.44532012939453,
      "activations/layer21_attention_weight_min": -19.85251235961914,
      "activations/layer22_attention_weight_max": 28.932960510253906,
      "activations/layer22_attention_weight_min": -27.751665115356445,
      "activations/layer23_attention_weight_max": 37.38745880126953,
      "activations/layer23_attention_weight_min": -26.788654327392578,
      "activations/layer2_attention_weight_max": 31.870162963867188,
      "activations/layer2_attention_weight_min": -31.16766929626465,
      "activations/layer3_attention_weight_max": 90.75543975830078,
      "activations/layer3_attention_weight_min": -88.82982635498047,
      "activations/layer4_attention_weight_max": 86.51778411865234,
      "activations/layer4_attention_weight_min": -83.20513916015625,
      "activations/layer5_attention_weight_max": 62.074790954589844,
      "activations/layer5_attention_weight_min": -72.8658218383789,
      "activations/layer6_attention_weight_max": 51.05899429321289,
      "activations/layer6_attention_weight_min": -52.836891174316406,
      "activations/layer7_attention_weight_max": 66.95731353759766,
      "activations/layer7_attention_weight_min": -66.58804321289062,
      "activations/layer8_attention_weight_max": 48.63719177246094,
      "activations/layer8_attention_weight_min": -49.84801483154297,
      "activations/layer9_attention_weight_max": 43.84381103515625,
      "activations/layer9_attention_weight_min": -45.86772918701172,
      "epoch": 21.93,
      "learning_rate": 8.655681818181816e-06,
      "loss": 2.7044,
      "step": 377450
    },
    {
      "activations/layer0_attention_weight_max": 16.145742416381836,
      "activations/layer0_attention_weight_min": -14.26867961883545,
      "activations/layer10_attention_weight_max": 39.12980270385742,
      "activations/layer10_attention_weight_min": -39.10268783569336,
      "activations/layer11_attention_weight_max": 36.37300491333008,
      "activations/layer11_attention_weight_min": -37.93026351928711,
      "activations/layer12_attention_weight_max": 19.224607467651367,
      "activations/layer12_attention_weight_min": -23.790916442871094,
      "activations/layer13_attention_weight_max": 33.34838104248047,
      "activations/layer13_attention_weight_min": -26.672935485839844,
      "activations/layer14_attention_weight_max": 35.664955139160156,
      "activations/layer14_attention_weight_min": -29.238866806030273,
      "activations/layer15_attention_weight_max": 34.395225524902344,
      "activations/layer15_attention_weight_min": -29.380773544311523,
      "activations/layer16_attention_weight_max": 36.577659606933594,
      "activations/layer16_attention_weight_min": -32.33375549316406,
      "activations/layer17_attention_weight_max": 53.181724548339844,
      "activations/layer17_attention_weight_min": -45.06512451171875,
      "activations/layer18_attention_weight_max": 47.38201141357422,
      "activations/layer18_attention_weight_min": -39.0664176940918,
      "activations/layer19_attention_weight_max": 23.81150245666504,
      "activations/layer19_attention_weight_min": -23.06110954284668,
      "activations/layer1_attention_weight_max": 16.957969665527344,
      "activations/layer1_attention_weight_min": -13.880011558532715,
      "activations/layer20_attention_weight_max": 22.02220916748047,
      "activations/layer20_attention_weight_min": -21.20928955078125,
      "activations/layer21_attention_weight_max": 36.22599792480469,
      "activations/layer21_attention_weight_min": -23.41735076904297,
      "activations/layer22_attention_weight_max": 28.081453323364258,
      "activations/layer22_attention_weight_min": -22.864608764648438,
      "activations/layer23_attention_weight_max": 36.785491943359375,
      "activations/layer23_attention_weight_min": -22.2271728515625,
      "activations/layer2_attention_weight_max": 31.646251678466797,
      "activations/layer2_attention_weight_min": -31.60909652709961,
      "activations/layer3_attention_weight_max": 91.1082763671875,
      "activations/layer3_attention_weight_min": -90.2129898071289,
      "activations/layer4_attention_weight_max": 82.977783203125,
      "activations/layer4_attention_weight_min": -83.64808654785156,
      "activations/layer5_attention_weight_max": 61.52846908569336,
      "activations/layer5_attention_weight_min": -70.35569763183594,
      "activations/layer6_attention_weight_max": 51.04096603393555,
      "activations/layer6_attention_weight_min": -52.685829162597656,
      "activations/layer7_attention_weight_max": 65.17655944824219,
      "activations/layer7_attention_weight_min": -71.04096984863281,
      "activations/layer8_attention_weight_max": 50.31230545043945,
      "activations/layer8_attention_weight_min": -53.65953063964844,
      "activations/layer9_attention_weight_max": 47.286338806152344,
      "activations/layer9_attention_weight_min": -48.090023040771484,
      "epoch": 21.93,
      "learning_rate": 8.636742424242423e-06,
      "loss": 2.7061,
      "step": 377500
    },
    {
      "activations/layer0_attention_weight_max": 15.575274467468262,
      "activations/layer0_attention_weight_min": -14.030725479125977,
      "activations/layer10_attention_weight_max": 36.788856506347656,
      "activations/layer10_attention_weight_min": -35.08706283569336,
      "activations/layer11_attention_weight_max": 35.07835388183594,
      "activations/layer11_attention_weight_min": -37.77229309082031,
      "activations/layer12_attention_weight_max": 19.54364585876465,
      "activations/layer12_attention_weight_min": -24.04603385925293,
      "activations/layer13_attention_weight_max": 32.724212646484375,
      "activations/layer13_attention_weight_min": -27.856966018676758,
      "activations/layer14_attention_weight_max": 37.74903869628906,
      "activations/layer14_attention_weight_min": -33.37928009033203,
      "activations/layer15_attention_weight_max": 40.3927116394043,
      "activations/layer15_attention_weight_min": -32.987796783447266,
      "activations/layer16_attention_weight_max": 41.81291961669922,
      "activations/layer16_attention_weight_min": -38.21010971069336,
      "activations/layer17_attention_weight_max": 61.38884735107422,
      "activations/layer17_attention_weight_min": -54.09974670410156,
      "activations/layer18_attention_weight_max": 57.682655334472656,
      "activations/layer18_attention_weight_min": -49.59954071044922,
      "activations/layer19_attention_weight_max": 30.201292037963867,
      "activations/layer19_attention_weight_min": -26.7015438079834,
      "activations/layer1_attention_weight_max": 15.122015953063965,
      "activations/layer1_attention_weight_min": -13.61263656616211,
      "activations/layer20_attention_weight_max": 28.090368270874023,
      "activations/layer20_attention_weight_min": -23.912996292114258,
      "activations/layer21_attention_weight_max": 43.87546920776367,
      "activations/layer21_attention_weight_min": -21.625308990478516,
      "activations/layer22_attention_weight_max": 34.682044982910156,
      "activations/layer22_attention_weight_min": -26.135225296020508,
      "activations/layer23_attention_weight_max": 44.427490234375,
      "activations/layer23_attention_weight_min": -24.696800231933594,
      "activations/layer2_attention_weight_max": 31.609853744506836,
      "activations/layer2_attention_weight_min": -31.156749725341797,
      "activations/layer3_attention_weight_max": 87.92512512207031,
      "activations/layer3_attention_weight_min": -87.0793228149414,
      "activations/layer4_attention_weight_max": 82.84272766113281,
      "activations/layer4_attention_weight_min": -86.01618194580078,
      "activations/layer5_attention_weight_max": 61.26298141479492,
      "activations/layer5_attention_weight_min": -73.01911926269531,
      "activations/layer6_attention_weight_max": 48.634315490722656,
      "activations/layer6_attention_weight_min": -51.115989685058594,
      "activations/layer7_attention_weight_max": 68.71354675292969,
      "activations/layer7_attention_weight_min": -68.12548828125,
      "activations/layer8_attention_weight_max": 48.958003997802734,
      "activations/layer8_attention_weight_min": -51.106075286865234,
      "activations/layer9_attention_weight_max": 46.299556732177734,
      "activations/layer9_attention_weight_min": -47.68232345581055,
      "epoch": 21.94,
      "learning_rate": 8.618181818181817e-06,
      "loss": 2.7047,
      "step": 377550
    },
    {
      "activations/layer0_attention_weight_max": 16.136743545532227,
      "activations/layer0_attention_weight_min": -14.917027473449707,
      "activations/layer10_attention_weight_max": 36.69470977783203,
      "activations/layer10_attention_weight_min": -37.07200241088867,
      "activations/layer11_attention_weight_max": 33.304931640625,
      "activations/layer11_attention_weight_min": -34.844970703125,
      "activations/layer12_attention_weight_max": 19.005521774291992,
      "activations/layer12_attention_weight_min": -26.866483688354492,
      "activations/layer13_attention_weight_max": 34.25860595703125,
      "activations/layer13_attention_weight_min": -26.543001174926758,
      "activations/layer14_attention_weight_max": 39.02865982055664,
      "activations/layer14_attention_weight_min": -33.222835540771484,
      "activations/layer15_attention_weight_max": 38.13935089111328,
      "activations/layer15_attention_weight_min": -32.15024948120117,
      "activations/layer16_attention_weight_max": 39.432167053222656,
      "activations/layer16_attention_weight_min": -39.143070220947266,
      "activations/layer17_attention_weight_max": 53.97508239746094,
      "activations/layer17_attention_weight_min": -51.12004470825195,
      "activations/layer18_attention_weight_max": 46.28245162963867,
      "activations/layer18_attention_weight_min": -42.1112060546875,
      "activations/layer19_attention_weight_max": 24.6519718170166,
      "activations/layer19_attention_weight_min": -22.04056167602539,
      "activations/layer1_attention_weight_max": 16.202072143554688,
      "activations/layer1_attention_weight_min": -15.52506160736084,
      "activations/layer20_attention_weight_max": 21.43480110168457,
      "activations/layer20_attention_weight_min": -23.241947174072266,
      "activations/layer21_attention_weight_max": 34.09079360961914,
      "activations/layer21_attention_weight_min": -23.64638328552246,
      "activations/layer22_attention_weight_max": 27.39728355407715,
      "activations/layer22_attention_weight_min": -24.902862548828125,
      "activations/layer23_attention_weight_max": 31.676733016967773,
      "activations/layer23_attention_weight_min": -23.9161434173584,
      "activations/layer2_attention_weight_max": 33.27410125732422,
      "activations/layer2_attention_weight_min": -31.895172119140625,
      "activations/layer3_attention_weight_max": 96.23506164550781,
      "activations/layer3_attention_weight_min": -97.14490509033203,
      "activations/layer4_attention_weight_max": 92.83379364013672,
      "activations/layer4_attention_weight_min": -89.65447235107422,
      "activations/layer5_attention_weight_max": 64.96908569335938,
      "activations/layer5_attention_weight_min": -69.17665100097656,
      "activations/layer6_attention_weight_max": 52.75142288208008,
      "activations/layer6_attention_weight_min": -52.699546813964844,
      "activations/layer7_attention_weight_max": 74.7665023803711,
      "activations/layer7_attention_weight_min": -67.75083923339844,
      "activations/layer8_attention_weight_max": 49.10013198852539,
      "activations/layer8_attention_weight_min": -51.329124450683594,
      "activations/layer9_attention_weight_max": 44.70083236694336,
      "activations/layer9_attention_weight_min": -47.905452728271484,
      "epoch": 21.94,
      "learning_rate": 8.599242424242424e-06,
      "loss": 2.7049,
      "step": 377600
    },
    {
      "activations/layer0_attention_weight_max": 15.819829940795898,
      "activations/layer0_attention_weight_min": -14.116154670715332,
      "activations/layer10_attention_weight_max": 38.22843551635742,
      "activations/layer10_attention_weight_min": -35.815093994140625,
      "activations/layer11_attention_weight_max": 34.9758186340332,
      "activations/layer11_attention_weight_min": -36.52263641357422,
      "activations/layer12_attention_weight_max": 19.735551834106445,
      "activations/layer12_attention_weight_min": -25.454090118408203,
      "activations/layer13_attention_weight_max": 35.192474365234375,
      "activations/layer13_attention_weight_min": -27.029146194458008,
      "activations/layer14_attention_weight_max": 38.99190902709961,
      "activations/layer14_attention_weight_min": -33.33745574951172,
      "activations/layer15_attention_weight_max": 35.79700469970703,
      "activations/layer15_attention_weight_min": -31.356298446655273,
      "activations/layer16_attention_weight_max": 39.3559684753418,
      "activations/layer16_attention_weight_min": -35.36931228637695,
      "activations/layer17_attention_weight_max": 52.43986892700195,
      "activations/layer17_attention_weight_min": -47.319671630859375,
      "activations/layer18_attention_weight_max": 50.35773849487305,
      "activations/layer18_attention_weight_min": -43.45096206665039,
      "activations/layer19_attention_weight_max": 24.1355037689209,
      "activations/layer19_attention_weight_min": -21.937170028686523,
      "activations/layer1_attention_weight_max": 15.66839599609375,
      "activations/layer1_attention_weight_min": -13.759580612182617,
      "activations/layer20_attention_weight_max": 20.55022621154785,
      "activations/layer20_attention_weight_min": -22.2811279296875,
      "activations/layer21_attention_weight_max": 35.099056243896484,
      "activations/layer21_attention_weight_min": -23.87950897216797,
      "activations/layer22_attention_weight_max": 28.713850021362305,
      "activations/layer22_attention_weight_min": -24.58856773376465,
      "activations/layer23_attention_weight_max": 36.31446075439453,
      "activations/layer23_attention_weight_min": -22.5325927734375,
      "activations/layer2_attention_weight_max": 31.059843063354492,
      "activations/layer2_attention_weight_min": -29.556865692138672,
      "activations/layer3_attention_weight_max": 85.8043441772461,
      "activations/layer3_attention_weight_min": -85.32146453857422,
      "activations/layer4_attention_weight_max": 82.23181915283203,
      "activations/layer4_attention_weight_min": -82.58621215820312,
      "activations/layer5_attention_weight_max": 65.93553161621094,
      "activations/layer5_attention_weight_min": -73.42256164550781,
      "activations/layer6_attention_weight_max": 48.10990905761719,
      "activations/layer6_attention_weight_min": -49.65444564819336,
      "activations/layer7_attention_weight_max": 67.85456085205078,
      "activations/layer7_attention_weight_min": -69.72735595703125,
      "activations/layer8_attention_weight_max": 47.37644577026367,
      "activations/layer8_attention_weight_min": -49.152870178222656,
      "activations/layer9_attention_weight_max": 42.725608825683594,
      "activations/layer9_attention_weight_min": -45.98065185546875,
      "epoch": 21.94,
      "learning_rate": 8.58030303030303e-06,
      "loss": 2.6937,
      "step": 377650
    },
    {
      "activations/layer0_attention_weight_max": 16.52198028564453,
      "activations/layer0_attention_weight_min": -13.723899841308594,
      "activations/layer10_attention_weight_max": 36.0940055847168,
      "activations/layer10_attention_weight_min": -35.654327392578125,
      "activations/layer11_attention_weight_max": 33.89307403564453,
      "activations/layer11_attention_weight_min": -35.19867706298828,
      "activations/layer12_attention_weight_max": 20.64858055114746,
      "activations/layer12_attention_weight_min": -29.366287231445312,
      "activations/layer13_attention_weight_max": 34.693294525146484,
      "activations/layer13_attention_weight_min": -27.15309715270996,
      "activations/layer14_attention_weight_max": 37.168704986572266,
      "activations/layer14_attention_weight_min": -30.204708099365234,
      "activations/layer15_attention_weight_max": 35.13284683227539,
      "activations/layer15_attention_weight_min": -32.11064910888672,
      "activations/layer16_attention_weight_max": 33.70961380004883,
      "activations/layer16_attention_weight_min": -31.114013671875,
      "activations/layer17_attention_weight_max": 50.58961868286133,
      "activations/layer17_attention_weight_min": -45.04640579223633,
      "activations/layer18_attention_weight_max": 42.85409927368164,
      "activations/layer18_attention_weight_min": -41.87017822265625,
      "activations/layer19_attention_weight_max": 22.944522857666016,
      "activations/layer19_attention_weight_min": -24.96247100830078,
      "activations/layer1_attention_weight_max": 14.949014663696289,
      "activations/layer1_attention_weight_min": -14.34177017211914,
      "activations/layer20_attention_weight_max": 21.288352966308594,
      "activations/layer20_attention_weight_min": -20.897199630737305,
      "activations/layer21_attention_weight_max": 34.1207389831543,
      "activations/layer21_attention_weight_min": -21.449905395507812,
      "activations/layer22_attention_weight_max": 30.240468978881836,
      "activations/layer22_attention_weight_min": -25.771137237548828,
      "activations/layer23_attention_weight_max": 35.13408660888672,
      "activations/layer23_attention_weight_min": -26.342029571533203,
      "activations/layer2_attention_weight_max": 32.65541458129883,
      "activations/layer2_attention_weight_min": -30.16751480102539,
      "activations/layer3_attention_weight_max": 87.86417388916016,
      "activations/layer3_attention_weight_min": -88.62156677246094,
      "activations/layer4_attention_weight_max": 84.69925689697266,
      "activations/layer4_attention_weight_min": -86.05147552490234,
      "activations/layer5_attention_weight_max": 64.3675308227539,
      "activations/layer5_attention_weight_min": -72.20435333251953,
      "activations/layer6_attention_weight_max": 52.42634582519531,
      "activations/layer6_attention_weight_min": -53.86420440673828,
      "activations/layer7_attention_weight_max": 72.50689697265625,
      "activations/layer7_attention_weight_min": -67.55587005615234,
      "activations/layer8_attention_weight_max": 47.58851623535156,
      "activations/layer8_attention_weight_min": -50.638267517089844,
      "activations/layer9_attention_weight_max": 46.98866271972656,
      "activations/layer9_attention_weight_min": -46.40365219116211,
      "epoch": 21.95,
      "learning_rate": 8.561363636363635e-06,
      "loss": 2.7,
      "step": 377700
    },
    {
      "activations/layer0_attention_weight_max": 16.114953994750977,
      "activations/layer0_attention_weight_min": -14.100255966186523,
      "activations/layer10_attention_weight_max": 35.27558517456055,
      "activations/layer10_attention_weight_min": -37.34511947631836,
      "activations/layer11_attention_weight_max": 33.680049896240234,
      "activations/layer11_attention_weight_min": -34.89093780517578,
      "activations/layer12_attention_weight_max": 19.436992645263672,
      "activations/layer12_attention_weight_min": -27.75606918334961,
      "activations/layer13_attention_weight_max": 34.226219177246094,
      "activations/layer13_attention_weight_min": -27.61305809020996,
      "activations/layer14_attention_weight_max": 36.20036315917969,
      "activations/layer14_attention_weight_min": -33.141761779785156,
      "activations/layer15_attention_weight_max": 34.365966796875,
      "activations/layer15_attention_weight_min": -32.40525817871094,
      "activations/layer16_attention_weight_max": 37.881561279296875,
      "activations/layer16_attention_weight_min": -36.7474365234375,
      "activations/layer17_attention_weight_max": 55.04886245727539,
      "activations/layer17_attention_weight_min": -50.017452239990234,
      "activations/layer18_attention_weight_max": 50.42603302001953,
      "activations/layer18_attention_weight_min": -41.934139251708984,
      "activations/layer19_attention_weight_max": 25.60502052307129,
      "activations/layer19_attention_weight_min": -21.96515655517578,
      "activations/layer1_attention_weight_max": 16.385202407836914,
      "activations/layer1_attention_weight_min": -13.899706840515137,
      "activations/layer20_attention_weight_max": 24.34990119934082,
      "activations/layer20_attention_weight_min": -21.322233200073242,
      "activations/layer21_attention_weight_max": 33.69812774658203,
      "activations/layer21_attention_weight_min": -28.00802230834961,
      "activations/layer22_attention_weight_max": 30.32396697998047,
      "activations/layer22_attention_weight_min": -24.055004119873047,
      "activations/layer23_attention_weight_max": 37.92974090576172,
      "activations/layer23_attention_weight_min": -21.51312828063965,
      "activations/layer2_attention_weight_max": 30.82378387451172,
      "activations/layer2_attention_weight_min": -30.704450607299805,
      "activations/layer3_attention_weight_max": 89.23697662353516,
      "activations/layer3_attention_weight_min": -86.61949920654297,
      "activations/layer4_attention_weight_max": 82.20677947998047,
      "activations/layer4_attention_weight_min": -85.31077575683594,
      "activations/layer5_attention_weight_max": 60.59397506713867,
      "activations/layer5_attention_weight_min": -75.21538543701172,
      "activations/layer6_attention_weight_max": 50.48823547363281,
      "activations/layer6_attention_weight_min": -51.19063949584961,
      "activations/layer7_attention_weight_max": 67.40223693847656,
      "activations/layer7_attention_weight_min": -72.3056640625,
      "activations/layer8_attention_weight_max": 48.593597412109375,
      "activations/layer8_attention_weight_min": -51.99968719482422,
      "activations/layer9_attention_weight_max": 46.032405853271484,
      "activations/layer9_attention_weight_min": -47.30998611450195,
      "epoch": 21.95,
      "learning_rate": 8.542424242424242e-06,
      "loss": 2.7068,
      "step": 377750
    },
    {
      "activations/layer0_attention_weight_max": 16.124452590942383,
      "activations/layer0_attention_weight_min": -14.031179428100586,
      "activations/layer10_attention_weight_max": 37.1541748046875,
      "activations/layer10_attention_weight_min": -37.45790481567383,
      "activations/layer11_attention_weight_max": 33.097293853759766,
      "activations/layer11_attention_weight_min": -36.447265625,
      "activations/layer12_attention_weight_max": 20.5904541015625,
      "activations/layer12_attention_weight_min": -27.371889114379883,
      "activations/layer13_attention_weight_max": 42.36980056762695,
      "activations/layer13_attention_weight_min": -32.15528869628906,
      "activations/layer14_attention_weight_max": 35.8856201171875,
      "activations/layer14_attention_weight_min": -29.559326171875,
      "activations/layer15_attention_weight_max": 33.50525665283203,
      "activations/layer15_attention_weight_min": -32.44058609008789,
      "activations/layer16_attention_weight_max": 33.74278259277344,
      "activations/layer16_attention_weight_min": -32.82402420043945,
      "activations/layer17_attention_weight_max": 51.91477584838867,
      "activations/layer17_attention_weight_min": -46.28249740600586,
      "activations/layer18_attention_weight_max": 48.486392974853516,
      "activations/layer18_attention_weight_min": -40.61952209472656,
      "activations/layer19_attention_weight_max": 23.007047653198242,
      "activations/layer19_attention_weight_min": -24.62355613708496,
      "activations/layer1_attention_weight_max": 15.217134475708008,
      "activations/layer1_attention_weight_min": -16.184993743896484,
      "activations/layer20_attention_weight_max": 22.501392364501953,
      "activations/layer20_attention_weight_min": -22.585355758666992,
      "activations/layer21_attention_weight_max": 38.4296760559082,
      "activations/layer21_attention_weight_min": -23.043222427368164,
      "activations/layer22_attention_weight_max": 28.18770980834961,
      "activations/layer22_attention_weight_min": -25.671785354614258,
      "activations/layer23_attention_weight_max": 36.16261672973633,
      "activations/layer23_attention_weight_min": -24.49523162841797,
      "activations/layer2_attention_weight_max": 31.83271598815918,
      "activations/layer2_attention_weight_min": -30.743621826171875,
      "activations/layer3_attention_weight_max": 87.6328125,
      "activations/layer3_attention_weight_min": -88.31607818603516,
      "activations/layer4_attention_weight_max": 84.3287582397461,
      "activations/layer4_attention_weight_min": -84.3324203491211,
      "activations/layer5_attention_weight_max": 59.9991569519043,
      "activations/layer5_attention_weight_min": -72.3543930053711,
      "activations/layer6_attention_weight_max": 48.655311584472656,
      "activations/layer6_attention_weight_min": -48.54943084716797,
      "activations/layer7_attention_weight_max": 66.6636962890625,
      "activations/layer7_attention_weight_min": -67.18611145019531,
      "activations/layer8_attention_weight_max": 48.10049819946289,
      "activations/layer8_attention_weight_min": -49.61829376220703,
      "activations/layer9_attention_weight_max": 42.54435729980469,
      "activations/layer9_attention_weight_min": -47.58366012573242,
      "epoch": 21.95,
      "learning_rate": 8.523484848484848e-06,
      "loss": 2.7089,
      "step": 377800
    },
    {
      "activations/layer0_attention_weight_max": 17.211284637451172,
      "activations/layer0_attention_weight_min": -14.297457695007324,
      "activations/layer10_attention_weight_max": 37.08143997192383,
      "activations/layer10_attention_weight_min": -38.716888427734375,
      "activations/layer11_attention_weight_max": 34.35865783691406,
      "activations/layer11_attention_weight_min": -37.43364715576172,
      "activations/layer12_attention_weight_max": 18.750307083129883,
      "activations/layer12_attention_weight_min": -26.30902862548828,
      "activations/layer13_attention_weight_max": 34.17461395263672,
      "activations/layer13_attention_weight_min": -26.91127586364746,
      "activations/layer14_attention_weight_max": 36.96792984008789,
      "activations/layer14_attention_weight_min": -30.581573486328125,
      "activations/layer15_attention_weight_max": 37.18630599975586,
      "activations/layer15_attention_weight_min": -31.815364837646484,
      "activations/layer16_attention_weight_max": 35.34832763671875,
      "activations/layer16_attention_weight_min": -34.38412857055664,
      "activations/layer17_attention_weight_max": 51.354366302490234,
      "activations/layer17_attention_weight_min": -47.58015823364258,
      "activations/layer18_attention_weight_max": 50.416534423828125,
      "activations/layer18_attention_weight_min": -44.08008575439453,
      "activations/layer19_attention_weight_max": 26.0797119140625,
      "activations/layer19_attention_weight_min": -24.15300941467285,
      "activations/layer1_attention_weight_max": 15.932744979858398,
      "activations/layer1_attention_weight_min": -13.329797744750977,
      "activations/layer20_attention_weight_max": 23.927162170410156,
      "activations/layer20_attention_weight_min": -21.285654067993164,
      "activations/layer21_attention_weight_max": 33.00462341308594,
      "activations/layer21_attention_weight_min": -20.493484497070312,
      "activations/layer22_attention_weight_max": 28.788421630859375,
      "activations/layer22_attention_weight_min": -23.906064987182617,
      "activations/layer23_attention_weight_max": 31.89191246032715,
      "activations/layer23_attention_weight_min": -21.15129852294922,
      "activations/layer2_attention_weight_max": 33.26647186279297,
      "activations/layer2_attention_weight_min": -31.149648666381836,
      "activations/layer3_attention_weight_max": 94.8847427368164,
      "activations/layer3_attention_weight_min": -91.47732543945312,
      "activations/layer4_attention_weight_max": 87.9887924194336,
      "activations/layer4_attention_weight_min": -88.5625228881836,
      "activations/layer5_attention_weight_max": 61.49103546142578,
      "activations/layer5_attention_weight_min": -77.04411315917969,
      "activations/layer6_attention_weight_max": 49.873844146728516,
      "activations/layer6_attention_weight_min": -51.84949493408203,
      "activations/layer7_attention_weight_max": 72.22608947753906,
      "activations/layer7_attention_weight_min": -71.21472930908203,
      "activations/layer8_attention_weight_max": 48.42036819458008,
      "activations/layer8_attention_weight_min": -52.07267379760742,
      "activations/layer9_attention_weight_max": 51.615657806396484,
      "activations/layer9_attention_weight_min": -46.619258880615234,
      "epoch": 21.96,
      "learning_rate": 8.504545454545453e-06,
      "loss": 2.6968,
      "step": 377850
    },
    {
      "activations/layer0_attention_weight_max": 15.929157257080078,
      "activations/layer0_attention_weight_min": -14.182313919067383,
      "activations/layer10_attention_weight_max": 40.87588119506836,
      "activations/layer10_attention_weight_min": -38.86414337158203,
      "activations/layer11_attention_weight_max": 39.28481674194336,
      "activations/layer11_attention_weight_min": -39.023277282714844,
      "activations/layer12_attention_weight_max": 19.503252029418945,
      "activations/layer12_attention_weight_min": -27.515092849731445,
      "activations/layer13_attention_weight_max": 34.1086540222168,
      "activations/layer13_attention_weight_min": -29.237262725830078,
      "activations/layer14_attention_weight_max": 34.902191162109375,
      "activations/layer14_attention_weight_min": -32.65715408325195,
      "activations/layer15_attention_weight_max": 34.11594772338867,
      "activations/layer15_attention_weight_min": -31.29308319091797,
      "activations/layer16_attention_weight_max": 35.03704833984375,
      "activations/layer16_attention_weight_min": -35.019893646240234,
      "activations/layer17_attention_weight_max": 51.18550491333008,
      "activations/layer17_attention_weight_min": -47.650264739990234,
      "activations/layer18_attention_weight_max": 47.46905517578125,
      "activations/layer18_attention_weight_min": -44.19442367553711,
      "activations/layer19_attention_weight_max": 25.265451431274414,
      "activations/layer19_attention_weight_min": -26.31546401977539,
      "activations/layer1_attention_weight_max": 15.374394416809082,
      "activations/layer1_attention_weight_min": -14.08804988861084,
      "activations/layer20_attention_weight_max": 24.63396644592285,
      "activations/layer20_attention_weight_min": -22.978408813476562,
      "activations/layer21_attention_weight_max": 33.13715362548828,
      "activations/layer21_attention_weight_min": -22.79629135131836,
      "activations/layer22_attention_weight_max": 30.691179275512695,
      "activations/layer22_attention_weight_min": -25.729341506958008,
      "activations/layer23_attention_weight_max": 37.78376007080078,
      "activations/layer23_attention_weight_min": -23.735309600830078,
      "activations/layer2_attention_weight_max": 32.604454040527344,
      "activations/layer2_attention_weight_min": -29.991470336914062,
      "activations/layer3_attention_weight_max": 90.40079498291016,
      "activations/layer3_attention_weight_min": -92.81304931640625,
      "activations/layer4_attention_weight_max": 90.17352294921875,
      "activations/layer4_attention_weight_min": -89.26534271240234,
      "activations/layer5_attention_weight_max": 62.33796691894531,
      "activations/layer5_attention_weight_min": -72.54661560058594,
      "activations/layer6_attention_weight_max": 51.84324264526367,
      "activations/layer6_attention_weight_min": -53.75796127319336,
      "activations/layer7_attention_weight_max": 71.13817596435547,
      "activations/layer7_attention_weight_min": -75.86219024658203,
      "activations/layer8_attention_weight_max": 51.789093017578125,
      "activations/layer8_attention_weight_min": -54.685455322265625,
      "activations/layer9_attention_weight_max": 47.094970703125,
      "activations/layer9_attention_weight_min": -51.69135284423828,
      "epoch": 21.96,
      "learning_rate": 8.48560606060606e-06,
      "loss": 2.7048,
      "step": 377900
    },
    {
      "activations/layer0_attention_weight_max": 17.154836654663086,
      "activations/layer0_attention_weight_min": -14.2540864944458,
      "activations/layer10_attention_weight_max": 36.66452407836914,
      "activations/layer10_attention_weight_min": -35.3191032409668,
      "activations/layer11_attention_weight_max": 34.558631896972656,
      "activations/layer11_attention_weight_min": -34.093589782714844,
      "activations/layer12_attention_weight_max": 19.74671173095703,
      "activations/layer12_attention_weight_min": -24.92440414428711,
      "activations/layer13_attention_weight_max": 36.956687927246094,
      "activations/layer13_attention_weight_min": -26.825376510620117,
      "activations/layer14_attention_weight_max": 35.01081848144531,
      "activations/layer14_attention_weight_min": -27.7829647064209,
      "activations/layer15_attention_weight_max": 34.41055679321289,
      "activations/layer15_attention_weight_min": -29.023122787475586,
      "activations/layer16_attention_weight_max": 34.92912292480469,
      "activations/layer16_attention_weight_min": -32.28439712524414,
      "activations/layer17_attention_weight_max": 50.86147689819336,
      "activations/layer17_attention_weight_min": -44.8393440246582,
      "activations/layer18_attention_weight_max": 49.28278350830078,
      "activations/layer18_attention_weight_min": -40.25629425048828,
      "activations/layer19_attention_weight_max": 25.83966636657715,
      "activations/layer19_attention_weight_min": -22.09865951538086,
      "activations/layer1_attention_weight_max": 14.932782173156738,
      "activations/layer1_attention_weight_min": -14.020369529724121,
      "activations/layer20_attention_weight_max": 25.164627075195312,
      "activations/layer20_attention_weight_min": -19.708120346069336,
      "activations/layer21_attention_weight_max": 36.10683822631836,
      "activations/layer21_attention_weight_min": -20.32952880859375,
      "activations/layer22_attention_weight_max": 29.590959548950195,
      "activations/layer22_attention_weight_min": -24.8641414642334,
      "activations/layer23_attention_weight_max": 37.268882751464844,
      "activations/layer23_attention_weight_min": -24.41986656188965,
      "activations/layer2_attention_weight_max": 30.70693016052246,
      "activations/layer2_attention_weight_min": -29.589750289916992,
      "activations/layer3_attention_weight_max": 89.20368194580078,
      "activations/layer3_attention_weight_min": -88.88438415527344,
      "activations/layer4_attention_weight_max": 90.42645263671875,
      "activations/layer4_attention_weight_min": -83.33827209472656,
      "activations/layer5_attention_weight_max": 63.643829345703125,
      "activations/layer5_attention_weight_min": -70.74866485595703,
      "activations/layer6_attention_weight_max": 49.13646697998047,
      "activations/layer6_attention_weight_min": -50.58314895629883,
      "activations/layer7_attention_weight_max": 72.48005676269531,
      "activations/layer7_attention_weight_min": -67.501953125,
      "activations/layer8_attention_weight_max": 46.62776565551758,
      "activations/layer8_attention_weight_min": -50.18806076049805,
      "activations/layer9_attention_weight_max": 45.71992492675781,
      "activations/layer9_attention_weight_min": -45.53957748413086,
      "epoch": 21.96,
      "learning_rate": 8.466666666666666e-06,
      "loss": 2.7071,
      "step": 377950
    },
    {
      "activations/layer0_attention_weight_max": 15.316615104675293,
      "activations/layer0_attention_weight_min": -14.40302848815918,
      "activations/layer10_attention_weight_max": 36.04637145996094,
      "activations/layer10_attention_weight_min": -34.29112243652344,
      "activations/layer11_attention_weight_max": 33.00926971435547,
      "activations/layer11_attention_weight_min": -33.849037170410156,
      "activations/layer12_attention_weight_max": 19.199827194213867,
      "activations/layer12_attention_weight_min": -28.58116912841797,
      "activations/layer13_attention_weight_max": 34.16721725463867,
      "activations/layer13_attention_weight_min": -26.51656723022461,
      "activations/layer14_attention_weight_max": 35.47490310668945,
      "activations/layer14_attention_weight_min": -30.134145736694336,
      "activations/layer15_attention_weight_max": 32.15245819091797,
      "activations/layer15_attention_weight_min": -30.663724899291992,
      "activations/layer16_attention_weight_max": 34.17825698852539,
      "activations/layer16_attention_weight_min": -31.80754852294922,
      "activations/layer17_attention_weight_max": 51.66447448730469,
      "activations/layer17_attention_weight_min": -43.02387619018555,
      "activations/layer18_attention_weight_max": 46.919700622558594,
      "activations/layer18_attention_weight_min": -37.8916130065918,
      "activations/layer19_attention_weight_max": 25.838232040405273,
      "activations/layer19_attention_weight_min": -22.33190155029297,
      "activations/layer1_attention_weight_max": 15.123032569885254,
      "activations/layer1_attention_weight_min": -13.269166946411133,
      "activations/layer20_attention_weight_max": 22.13042640686035,
      "activations/layer20_attention_weight_min": -20.13199234008789,
      "activations/layer21_attention_weight_max": 32.9993896484375,
      "activations/layer21_attention_weight_min": -21.026073455810547,
      "activations/layer22_attention_weight_max": 27.77339744567871,
      "activations/layer22_attention_weight_min": -23.826555252075195,
      "activations/layer23_attention_weight_max": 35.752769470214844,
      "activations/layer23_attention_weight_min": -22.4763240814209,
      "activations/layer2_attention_weight_max": 32.16232681274414,
      "activations/layer2_attention_weight_min": -30.151138305664062,
      "activations/layer3_attention_weight_max": 90.04914855957031,
      "activations/layer3_attention_weight_min": -89.6363754272461,
      "activations/layer4_attention_weight_max": 83.73402404785156,
      "activations/layer4_attention_weight_min": -88.09638214111328,
      "activations/layer5_attention_weight_max": 63.13653564453125,
      "activations/layer5_attention_weight_min": -76.31045532226562,
      "activations/layer6_attention_weight_max": 48.758949279785156,
      "activations/layer6_attention_weight_min": -50.266727447509766,
      "activations/layer7_attention_weight_max": 70.08646392822266,
      "activations/layer7_attention_weight_min": -67.6536865234375,
      "activations/layer8_attention_weight_max": 46.83088302612305,
      "activations/layer8_attention_weight_min": -48.95469665527344,
      "activations/layer9_attention_weight_max": 43.908973693847656,
      "activations/layer9_attention_weight_min": -44.45476531982422,
      "epoch": 21.96,
      "learning_rate": 8.447727272727271e-06,
      "loss": 2.7213,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.4658,
      "eval_samples_per_second": 507.217,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.4658,
      "eval_openwebtext_samples_per_second": 507.217,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.969,
      "eval_wikitext_samples_per_second": 231.586,
      "step": 378000
    },
    {
      "epoch": 21.96,
      "eval_lambada_loss": 2.412109375,
      "eval_lambada_ppl": 11.157471631854794,
      "eval_lambada_runtime": 9.6158,
      "eval_lambada_samples_per_second": 506.354,
      "step": 378000
    },
    {
      "activations/layer0_attention_weight_max": 16.882335662841797,
      "activations/layer0_attention_weight_min": -14.294295310974121,
      "activations/layer10_attention_weight_max": 35.1212272644043,
      "activations/layer10_attention_weight_min": -36.52515411376953,
      "activations/layer11_attention_weight_max": 34.06672668457031,
      "activations/layer11_attention_weight_min": -35.84131622314453,
      "activations/layer12_attention_weight_max": 20.87009620666504,
      "activations/layer12_attention_weight_min": -23.141056060791016,
      "activations/layer13_attention_weight_max": 34.96390914916992,
      "activations/layer13_attention_weight_min": -26.6752986907959,
      "activations/layer14_attention_weight_max": 37.554561614990234,
      "activations/layer14_attention_weight_min": -30.7211971282959,
      "activations/layer15_attention_weight_max": 34.58344650268555,
      "activations/layer15_attention_weight_min": -32.589385986328125,
      "activations/layer16_attention_weight_max": 35.80767822265625,
      "activations/layer16_attention_weight_min": -34.90613555908203,
      "activations/layer17_attention_weight_max": 54.846107482910156,
      "activations/layer17_attention_weight_min": -45.66079330444336,
      "activations/layer18_attention_weight_max": 50.427249908447266,
      "activations/layer18_attention_weight_min": -41.27043914794922,
      "activations/layer19_attention_weight_max": 27.72454833984375,
      "activations/layer19_attention_weight_min": -25.742013931274414,
      "activations/layer1_attention_weight_max": 15.022784233093262,
      "activations/layer1_attention_weight_min": -13.847939491271973,
      "activations/layer20_attention_weight_max": 26.00474739074707,
      "activations/layer20_attention_weight_min": -24.76140785217285,
      "activations/layer21_attention_weight_max": 39.768714904785156,
      "activations/layer21_attention_weight_min": -22.752172470092773,
      "activations/layer22_attention_weight_max": 30.41025733947754,
      "activations/layer22_attention_weight_min": -25.466068267822266,
      "activations/layer23_attention_weight_max": 37.22601318359375,
      "activations/layer23_attention_weight_min": -25.94698143005371,
      "activations/layer2_attention_weight_max": 33.26993942260742,
      "activations/layer2_attention_weight_min": -31.42280387878418,
      "activations/layer3_attention_weight_max": 91.18122100830078,
      "activations/layer3_attention_weight_min": -94.11463165283203,
      "activations/layer4_attention_weight_max": 87.6705551147461,
      "activations/layer4_attention_weight_min": -86.65251922607422,
      "activations/layer5_attention_weight_max": 64.11117553710938,
      "activations/layer5_attention_weight_min": -72.79148864746094,
      "activations/layer6_attention_weight_max": 50.17013931274414,
      "activations/layer6_attention_weight_min": -50.57833480834961,
      "activations/layer7_attention_weight_max": 70.1727294921875,
      "activations/layer7_attention_weight_min": -67.47894287109375,
      "activations/layer8_attention_weight_max": 47.37328338623047,
      "activations/layer8_attention_weight_min": -51.27005386352539,
      "activations/layer9_attention_weight_max": 43.777217864990234,
      "activations/layer9_attention_weight_min": -46.721405029296875,
      "epoch": 21.97,
      "learning_rate": 8.428787878787878e-06,
      "loss": 2.7171,
      "step": 378050
    },
    {
      "activations/layer0_attention_weight_max": 15.677967071533203,
      "activations/layer0_attention_weight_min": -13.974055290222168,
      "activations/layer10_attention_weight_max": 36.266361236572266,
      "activations/layer10_attention_weight_min": -36.50762939453125,
      "activations/layer11_attention_weight_max": 34.06597137451172,
      "activations/layer11_attention_weight_min": -36.47654724121094,
      "activations/layer12_attention_weight_max": 20.088882446289062,
      "activations/layer12_attention_weight_min": -24.746238708496094,
      "activations/layer13_attention_weight_max": 40.60627365112305,
      "activations/layer13_attention_weight_min": -28.509279251098633,
      "activations/layer14_attention_weight_max": 39.238040924072266,
      "activations/layer14_attention_weight_min": -33.326622009277344,
      "activations/layer15_attention_weight_max": 36.964473724365234,
      "activations/layer15_attention_weight_min": -31.78410530090332,
      "activations/layer16_attention_weight_max": 39.04554748535156,
      "activations/layer16_attention_weight_min": -36.13017654418945,
      "activations/layer17_attention_weight_max": 56.90647506713867,
      "activations/layer17_attention_weight_min": -48.48101043701172,
      "activations/layer18_attention_weight_max": 52.165340423583984,
      "activations/layer18_attention_weight_min": -42.50900650024414,
      "activations/layer19_attention_weight_max": 24.37190818786621,
      "activations/layer19_attention_weight_min": -23.555021286010742,
      "activations/layer1_attention_weight_max": 15.274068832397461,
      "activations/layer1_attention_weight_min": -14.815391540527344,
      "activations/layer20_attention_weight_max": 23.062461853027344,
      "activations/layer20_attention_weight_min": -21.8834285736084,
      "activations/layer21_attention_weight_max": 39.693016052246094,
      "activations/layer21_attention_weight_min": -22.736831665039062,
      "activations/layer22_attention_weight_max": 33.497135162353516,
      "activations/layer22_attention_weight_min": -24.76728630065918,
      "activations/layer23_attention_weight_max": 38.16795349121094,
      "activations/layer23_attention_weight_min": -25.252071380615234,
      "activations/layer2_attention_weight_max": 31.850255966186523,
      "activations/layer2_attention_weight_min": -31.830114364624023,
      "activations/layer3_attention_weight_max": 90.04644775390625,
      "activations/layer3_attention_weight_min": -92.48268127441406,
      "activations/layer4_attention_weight_max": 84.35902404785156,
      "activations/layer4_attention_weight_min": -86.30094146728516,
      "activations/layer5_attention_weight_max": 63.73545837402344,
      "activations/layer5_attention_weight_min": -76.700439453125,
      "activations/layer6_attention_weight_max": 49.11730194091797,
      "activations/layer6_attention_weight_min": -51.27930450439453,
      "activations/layer7_attention_weight_max": 67.29326629638672,
      "activations/layer7_attention_weight_min": -65.76210021972656,
      "activations/layer8_attention_weight_max": 47.660911560058594,
      "activations/layer8_attention_weight_min": -49.27840042114258,
      "activations/layer9_attention_weight_max": 44.868064880371094,
      "activations/layer9_attention_weight_min": -46.52872085571289,
      "epoch": 21.97,
      "learning_rate": 8.409848484848484e-06,
      "loss": 2.7081,
      "step": 378100
    },
    {
      "activations/layer0_attention_weight_max": 15.609755516052246,
      "activations/layer0_attention_weight_min": -14.122795104980469,
      "activations/layer10_attention_weight_max": 36.4879035949707,
      "activations/layer10_attention_weight_min": -37.91850662231445,
      "activations/layer11_attention_weight_max": 34.25690460205078,
      "activations/layer11_attention_weight_min": -39.571414947509766,
      "activations/layer12_attention_weight_max": 19.3033447265625,
      "activations/layer12_attention_weight_min": -26.931297302246094,
      "activations/layer13_attention_weight_max": 34.87533187866211,
      "activations/layer13_attention_weight_min": -28.779735565185547,
      "activations/layer14_attention_weight_max": 34.42306137084961,
      "activations/layer14_attention_weight_min": -31.206605911254883,
      "activations/layer15_attention_weight_max": 33.1688117980957,
      "activations/layer15_attention_weight_min": -30.438940048217773,
      "activations/layer16_attention_weight_max": 36.421878814697266,
      "activations/layer16_attention_weight_min": -31.51247215270996,
      "activations/layer17_attention_weight_max": 53.50227355957031,
      "activations/layer17_attention_weight_min": -46.202667236328125,
      "activations/layer18_attention_weight_max": 46.3106575012207,
      "activations/layer18_attention_weight_min": -39.78831100463867,
      "activations/layer19_attention_weight_max": 26.12497329711914,
      "activations/layer19_attention_weight_min": -23.10450553894043,
      "activations/layer1_attention_weight_max": 15.468355178833008,
      "activations/layer1_attention_weight_min": -14.140267372131348,
      "activations/layer20_attention_weight_max": 26.091867446899414,
      "activations/layer20_attention_weight_min": -20.901817321777344,
      "activations/layer21_attention_weight_max": 35.38380813598633,
      "activations/layer21_attention_weight_min": -20.39657974243164,
      "activations/layer22_attention_weight_max": 29.32359504699707,
      "activations/layer22_attention_weight_min": -26.58002471923828,
      "activations/layer23_attention_weight_max": 36.0635986328125,
      "activations/layer23_attention_weight_min": -25.26852798461914,
      "activations/layer2_attention_weight_max": 32.374656677246094,
      "activations/layer2_attention_weight_min": -30.603635787963867,
      "activations/layer3_attention_weight_max": 90.95679473876953,
      "activations/layer3_attention_weight_min": -89.63613891601562,
      "activations/layer4_attention_weight_max": 86.78966522216797,
      "activations/layer4_attention_weight_min": -85.31502532958984,
      "activations/layer5_attention_weight_max": 62.04615020751953,
      "activations/layer5_attention_weight_min": -72.9674072265625,
      "activations/layer6_attention_weight_max": 52.385475158691406,
      "activations/layer6_attention_weight_min": -55.96339797973633,
      "activations/layer7_attention_weight_max": 69.543701171875,
      "activations/layer7_attention_weight_min": -71.41444396972656,
      "activations/layer8_attention_weight_max": 50.164031982421875,
      "activations/layer8_attention_weight_min": -54.38340377807617,
      "activations/layer9_attention_weight_max": 44.0019645690918,
      "activations/layer9_attention_weight_min": -48.59312438964844,
      "epoch": 21.97,
      "learning_rate": 8.390909090909091e-06,
      "loss": 2.7122,
      "step": 378150
    },
    {
      "activations/layer0_attention_weight_max": 15.601398468017578,
      "activations/layer0_attention_weight_min": -14.176185607910156,
      "activations/layer10_attention_weight_max": 34.75004577636719,
      "activations/layer10_attention_weight_min": -34.69668197631836,
      "activations/layer11_attention_weight_max": 32.77429962158203,
      "activations/layer11_attention_weight_min": -33.601375579833984,
      "activations/layer12_attention_weight_max": 17.679704666137695,
      "activations/layer12_attention_weight_min": -25.409931182861328,
      "activations/layer13_attention_weight_max": 32.49950408935547,
      "activations/layer13_attention_weight_min": -26.04498863220215,
      "activations/layer14_attention_weight_max": 35.05693817138672,
      "activations/layer14_attention_weight_min": -28.351337432861328,
      "activations/layer15_attention_weight_max": 32.03463363647461,
      "activations/layer15_attention_weight_min": -31.366012573242188,
      "activations/layer16_attention_weight_max": 35.3038215637207,
      "activations/layer16_attention_weight_min": -31.059492111206055,
      "activations/layer17_attention_weight_max": 50.15106964111328,
      "activations/layer17_attention_weight_min": -47.609375,
      "activations/layer18_attention_weight_max": 48.59564971923828,
      "activations/layer18_attention_weight_min": -38.199798583984375,
      "activations/layer19_attention_weight_max": 26.19782066345215,
      "activations/layer19_attention_weight_min": -23.29657554626465,
      "activations/layer1_attention_weight_max": 14.999731063842773,
      "activations/layer1_attention_weight_min": -14.170875549316406,
      "activations/layer20_attention_weight_max": 20.3126220703125,
      "activations/layer20_attention_weight_min": -20.766698837280273,
      "activations/layer21_attention_weight_max": 33.67964553833008,
      "activations/layer21_attention_weight_min": -24.70729637145996,
      "activations/layer22_attention_weight_max": 26.74689292907715,
      "activations/layer22_attention_weight_min": -25.119186401367188,
      "activations/layer23_attention_weight_max": 32.753116607666016,
      "activations/layer23_attention_weight_min": -23.471088409423828,
      "activations/layer2_attention_weight_max": 32.92709732055664,
      "activations/layer2_attention_weight_min": -30.74193572998047,
      "activations/layer3_attention_weight_max": 89.90106964111328,
      "activations/layer3_attention_weight_min": -88.80500030517578,
      "activations/layer4_attention_weight_max": 83.94497680664062,
      "activations/layer4_attention_weight_min": -83.83161163330078,
      "activations/layer5_attention_weight_max": 62.885128021240234,
      "activations/layer5_attention_weight_min": -72.47984313964844,
      "activations/layer6_attention_weight_max": 48.26728057861328,
      "activations/layer6_attention_weight_min": -48.756290435791016,
      "activations/layer7_attention_weight_max": 71.08934783935547,
      "activations/layer7_attention_weight_min": -63.72490310668945,
      "activations/layer8_attention_weight_max": 46.75517654418945,
      "activations/layer8_attention_weight_min": -48.7191047668457,
      "activations/layer9_attention_weight_max": 43.530006408691406,
      "activations/layer9_attention_weight_min": -44.2186279296875,
      "epoch": 21.98,
      "learning_rate": 8.371969696969696e-06,
      "loss": 2.6968,
      "step": 378200
    },
    {
      "activations/layer0_attention_weight_max": 16.08896827697754,
      "activations/layer0_attention_weight_min": -14.092714309692383,
      "activations/layer10_attention_weight_max": 36.30488204956055,
      "activations/layer10_attention_weight_min": -36.19706726074219,
      "activations/layer11_attention_weight_max": 33.47291564941406,
      "activations/layer11_attention_weight_min": -33.684898376464844,
      "activations/layer12_attention_weight_max": 19.147626876831055,
      "activations/layer12_attention_weight_min": -27.432703018188477,
      "activations/layer13_attention_weight_max": 35.365474700927734,
      "activations/layer13_attention_weight_min": -28.113969802856445,
      "activations/layer14_attention_weight_max": 37.94368362426758,
      "activations/layer14_attention_weight_min": -30.790729522705078,
      "activations/layer15_attention_weight_max": 36.062007904052734,
      "activations/layer15_attention_weight_min": -30.18254852294922,
      "activations/layer16_attention_weight_max": 36.711971282958984,
      "activations/layer16_attention_weight_min": -33.960689544677734,
      "activations/layer17_attention_weight_max": 54.25519943237305,
      "activations/layer17_attention_weight_min": -46.66481399536133,
      "activations/layer18_attention_weight_max": 50.69633865356445,
      "activations/layer18_attention_weight_min": -42.94452667236328,
      "activations/layer19_attention_weight_max": 26.737380981445312,
      "activations/layer19_attention_weight_min": -24.952762603759766,
      "activations/layer1_attention_weight_max": 15.492263793945312,
      "activations/layer1_attention_weight_min": -13.392107009887695,
      "activations/layer20_attention_weight_max": 23.83546257019043,
      "activations/layer20_attention_weight_min": -23.78683853149414,
      "activations/layer21_attention_weight_max": 41.03810501098633,
      "activations/layer21_attention_weight_min": -21.927263259887695,
      "activations/layer22_attention_weight_max": 28.179306030273438,
      "activations/layer22_attention_weight_min": -26.63981056213379,
      "activations/layer23_attention_weight_max": 35.747013092041016,
      "activations/layer23_attention_weight_min": -22.338016510009766,
      "activations/layer2_attention_weight_max": 31.236425399780273,
      "activations/layer2_attention_weight_min": -29.24879264831543,
      "activations/layer3_attention_weight_max": 89.29403686523438,
      "activations/layer3_attention_weight_min": -87.18831634521484,
      "activations/layer4_attention_weight_max": 85.8922348022461,
      "activations/layer4_attention_weight_min": -82.82665252685547,
      "activations/layer5_attention_weight_max": 62.90298080444336,
      "activations/layer5_attention_weight_min": -70.27674865722656,
      "activations/layer6_attention_weight_max": 50.563636779785156,
      "activations/layer6_attention_weight_min": -49.578529357910156,
      "activations/layer7_attention_weight_max": 68.90348815917969,
      "activations/layer7_attention_weight_min": -65.03237915039062,
      "activations/layer8_attention_weight_max": 47.61444091796875,
      "activations/layer8_attention_weight_min": -53.330322265625,
      "activations/layer9_attention_weight_max": 43.356346130371094,
      "activations/layer9_attention_weight_min": -47.154998779296875,
      "epoch": 21.98,
      "learning_rate": 8.353030303030302e-06,
      "loss": 2.7198,
      "step": 378250
    },
    {
      "activations/layer0_attention_weight_max": 16.148908615112305,
      "activations/layer0_attention_weight_min": -14.016335487365723,
      "activations/layer10_attention_weight_max": 36.06843185424805,
      "activations/layer10_attention_weight_min": -36.09315872192383,
      "activations/layer11_attention_weight_max": 35.05218505859375,
      "activations/layer11_attention_weight_min": -37.682003021240234,
      "activations/layer12_attention_weight_max": 19.770490646362305,
      "activations/layer12_attention_weight_min": -25.36298942565918,
      "activations/layer13_attention_weight_max": 37.641998291015625,
      "activations/layer13_attention_weight_min": -26.63710594177246,
      "activations/layer14_attention_weight_max": 40.43051528930664,
      "activations/layer14_attention_weight_min": -31.220909118652344,
      "activations/layer15_attention_weight_max": 36.79053497314453,
      "activations/layer15_attention_weight_min": -31.79720687866211,
      "activations/layer16_attention_weight_max": 37.7421760559082,
      "activations/layer16_attention_weight_min": -34.84030532836914,
      "activations/layer17_attention_weight_max": 55.956180572509766,
      "activations/layer17_attention_weight_min": -48.89409255981445,
      "activations/layer18_attention_weight_max": 54.0278205871582,
      "activations/layer18_attention_weight_min": -45.65753936767578,
      "activations/layer19_attention_weight_max": 28.437002182006836,
      "activations/layer19_attention_weight_min": -26.05729103088379,
      "activations/layer1_attention_weight_max": 15.185181617736816,
      "activations/layer1_attention_weight_min": -13.127829551696777,
      "activations/layer20_attention_weight_max": 30.360532760620117,
      "activations/layer20_attention_weight_min": -26.027416229248047,
      "activations/layer21_attention_weight_max": 47.5161247253418,
      "activations/layer21_attention_weight_min": -22.38027572631836,
      "activations/layer22_attention_weight_max": 33.3497200012207,
      "activations/layer22_attention_weight_min": -27.85246467590332,
      "activations/layer23_attention_weight_max": 40.38539123535156,
      "activations/layer23_attention_weight_min": -25.9453125,
      "activations/layer2_attention_weight_max": 32.64411163330078,
      "activations/layer2_attention_weight_min": -31.397602081298828,
      "activations/layer3_attention_weight_max": 91.17451477050781,
      "activations/layer3_attention_weight_min": -90.02814483642578,
      "activations/layer4_attention_weight_max": 86.55616760253906,
      "activations/layer4_attention_weight_min": -88.32545471191406,
      "activations/layer5_attention_weight_max": 67.30351257324219,
      "activations/layer5_attention_weight_min": -72.10128021240234,
      "activations/layer6_attention_weight_max": 50.06166076660156,
      "activations/layer6_attention_weight_min": -52.517147064208984,
      "activations/layer7_attention_weight_max": 69.561279296875,
      "activations/layer7_attention_weight_min": -69.93446350097656,
      "activations/layer8_attention_weight_max": 45.75526428222656,
      "activations/layer8_attention_weight_min": -54.766990661621094,
      "activations/layer9_attention_weight_max": 44.06608200073242,
      "activations/layer9_attention_weight_min": -48.133663177490234,
      "epoch": 21.98,
      "learning_rate": 8.334090909090909e-06,
      "loss": 2.7093,
      "step": 378300
    },
    {
      "activations/layer0_attention_weight_max": 16.195573806762695,
      "activations/layer0_attention_weight_min": -14.382932662963867,
      "activations/layer10_attention_weight_max": 37.318504333496094,
      "activations/layer10_attention_weight_min": -37.967506408691406,
      "activations/layer11_attention_weight_max": 34.05292510986328,
      "activations/layer11_attention_weight_min": -36.66581726074219,
      "activations/layer12_attention_weight_max": 17.713903427124023,
      "activations/layer12_attention_weight_min": -24.414777755737305,
      "activations/layer13_attention_weight_max": 33.19479751586914,
      "activations/layer13_attention_weight_min": -26.67020034790039,
      "activations/layer14_attention_weight_max": 34.40171813964844,
      "activations/layer14_attention_weight_min": -29.788978576660156,
      "activations/layer15_attention_weight_max": 34.25802230834961,
      "activations/layer15_attention_weight_min": -29.553403854370117,
      "activations/layer16_attention_weight_max": 34.21109390258789,
      "activations/layer16_attention_weight_min": -31.060338973999023,
      "activations/layer17_attention_weight_max": 50.55731201171875,
      "activations/layer17_attention_weight_min": -42.25191879272461,
      "activations/layer18_attention_weight_max": 48.185020446777344,
      "activations/layer18_attention_weight_min": -38.591556549072266,
      "activations/layer19_attention_weight_max": 23.379989624023438,
      "activations/layer19_attention_weight_min": -21.74019432067871,
      "activations/layer1_attention_weight_max": 14.70274543762207,
      "activations/layer1_attention_weight_min": -13.316431045532227,
      "activations/layer20_attention_weight_max": 21.85550308227539,
      "activations/layer20_attention_weight_min": -20.795488357543945,
      "activations/layer21_attention_weight_max": 36.672607421875,
      "activations/layer21_attention_weight_min": -21.26532745361328,
      "activations/layer22_attention_weight_max": 31.321565628051758,
      "activations/layer22_attention_weight_min": -26.00874137878418,
      "activations/layer23_attention_weight_max": 34.25932312011719,
      "activations/layer23_attention_weight_min": -24.92475700378418,
      "activations/layer2_attention_weight_max": 32.73115921020508,
      "activations/layer2_attention_weight_min": -32.22074508666992,
      "activations/layer3_attention_weight_max": 92.99736785888672,
      "activations/layer3_attention_weight_min": -89.50213623046875,
      "activations/layer4_attention_weight_max": 87.48651123046875,
      "activations/layer4_attention_weight_min": -87.1826400756836,
      "activations/layer5_attention_weight_max": 63.273250579833984,
      "activations/layer5_attention_weight_min": -74.43858337402344,
      "activations/layer6_attention_weight_max": 50.43067932128906,
      "activations/layer6_attention_weight_min": -51.83080291748047,
      "activations/layer7_attention_weight_max": 69.83819580078125,
      "activations/layer7_attention_weight_min": -69.7464599609375,
      "activations/layer8_attention_weight_max": 49.60101318359375,
      "activations/layer8_attention_weight_min": -51.49531936645508,
      "activations/layer9_attention_weight_max": 45.22932052612305,
      "activations/layer9_attention_weight_min": -48.376338958740234,
      "epoch": 21.98,
      "learning_rate": 8.315151515151514e-06,
      "loss": 2.7034,
      "step": 378350
    },
    {
      "activations/layer0_attention_weight_max": 16.251174926757812,
      "activations/layer0_attention_weight_min": -14.129823684692383,
      "activations/layer10_attention_weight_max": 35.329627990722656,
      "activations/layer10_attention_weight_min": -36.107398986816406,
      "activations/layer11_attention_weight_max": 34.15664291381836,
      "activations/layer11_attention_weight_min": -34.69685363769531,
      "activations/layer12_attention_weight_max": 19.124828338623047,
      "activations/layer12_attention_weight_min": -25.01390266418457,
      "activations/layer13_attention_weight_max": 38.531654357910156,
      "activations/layer13_attention_weight_min": -29.369827270507812,
      "activations/layer14_attention_weight_max": 36.89828109741211,
      "activations/layer14_attention_weight_min": -31.467144012451172,
      "activations/layer15_attention_weight_max": 36.01996612548828,
      "activations/layer15_attention_weight_min": -33.36909484863281,
      "activations/layer16_attention_weight_max": 36.32583999633789,
      "activations/layer16_attention_weight_min": -34.583858489990234,
      "activations/layer17_attention_weight_max": 52.5329475402832,
      "activations/layer17_attention_weight_min": -47.65106964111328,
      "activations/layer18_attention_weight_max": 51.3773307800293,
      "activations/layer18_attention_weight_min": -41.24068832397461,
      "activations/layer19_attention_weight_max": 25.832571029663086,
      "activations/layer19_attention_weight_min": -23.085968017578125,
      "activations/layer1_attention_weight_max": 14.871460914611816,
      "activations/layer1_attention_weight_min": -13.935456275939941,
      "activations/layer20_attention_weight_max": 23.259056091308594,
      "activations/layer20_attention_weight_min": -20.140199661254883,
      "activations/layer21_attention_weight_max": 37.057403564453125,
      "activations/layer21_attention_weight_min": -21.342845916748047,
      "activations/layer22_attention_weight_max": 31.394577026367188,
      "activations/layer22_attention_weight_min": -26.35650062561035,
      "activations/layer23_attention_weight_max": 37.64318084716797,
      "activations/layer23_attention_weight_min": -24.289745330810547,
      "activations/layer2_attention_weight_max": 32.36198806762695,
      "activations/layer2_attention_weight_min": -31.383970260620117,
      "activations/layer3_attention_weight_max": 89.95443725585938,
      "activations/layer3_attention_weight_min": -91.19808959960938,
      "activations/layer4_attention_weight_max": 85.4459457397461,
      "activations/layer4_attention_weight_min": -84.25084686279297,
      "activations/layer5_attention_weight_max": 62.86616134643555,
      "activations/layer5_attention_weight_min": -68.27322387695312,
      "activations/layer6_attention_weight_max": 49.43094253540039,
      "activations/layer6_attention_weight_min": -51.752235412597656,
      "activations/layer7_attention_weight_max": 67.2457275390625,
      "activations/layer7_attention_weight_min": -67.76889038085938,
      "activations/layer8_attention_weight_max": 49.5504035949707,
      "activations/layer8_attention_weight_min": -49.01082992553711,
      "activations/layer9_attention_weight_max": 43.590511322021484,
      "activations/layer9_attention_weight_min": -46.04484176635742,
      "epoch": 21.99,
      "learning_rate": 8.29621212121212e-06,
      "loss": 2.7136,
      "step": 378400
    },
    {
      "activations/layer0_attention_weight_max": 17.6846866607666,
      "activations/layer0_attention_weight_min": -14.030031204223633,
      "activations/layer10_attention_weight_max": 36.0189208984375,
      "activations/layer10_attention_weight_min": -35.420509338378906,
      "activations/layer11_attention_weight_max": 35.36408996582031,
      "activations/layer11_attention_weight_min": -38.278236389160156,
      "activations/layer12_attention_weight_max": 19.347259521484375,
      "activations/layer12_attention_weight_min": -24.98424530029297,
      "activations/layer13_attention_weight_max": 37.551841735839844,
      "activations/layer13_attention_weight_min": -28.301576614379883,
      "activations/layer14_attention_weight_max": 40.076499938964844,
      "activations/layer14_attention_weight_min": -35.23784255981445,
      "activations/layer15_attention_weight_max": 37.951663970947266,
      "activations/layer15_attention_weight_min": -35.677734375,
      "activations/layer16_attention_weight_max": 39.96975326538086,
      "activations/layer16_attention_weight_min": -36.017906188964844,
      "activations/layer17_attention_weight_max": 58.95155334472656,
      "activations/layer17_attention_weight_min": -48.94432830810547,
      "activations/layer18_attention_weight_max": 56.5191650390625,
      "activations/layer18_attention_weight_min": -42.13921356201172,
      "activations/layer19_attention_weight_max": 25.836511611938477,
      "activations/layer19_attention_weight_min": -25.310401916503906,
      "activations/layer1_attention_weight_max": 15.082932472229004,
      "activations/layer1_attention_weight_min": -14.261824607849121,
      "activations/layer20_attention_weight_max": 23.89588737487793,
      "activations/layer20_attention_weight_min": -23.460521697998047,
      "activations/layer21_attention_weight_max": 41.270973205566406,
      "activations/layer21_attention_weight_min": -22.58719253540039,
      "activations/layer22_attention_weight_max": 30.22565460205078,
      "activations/layer22_attention_weight_min": -26.524887084960938,
      "activations/layer23_attention_weight_max": 35.325111389160156,
      "activations/layer23_attention_weight_min": -24.789113998413086,
      "activations/layer2_attention_weight_max": 32.2007942199707,
      "activations/layer2_attention_weight_min": -30.436939239501953,
      "activations/layer3_attention_weight_max": 88.01847076416016,
      "activations/layer3_attention_weight_min": -89.5795669555664,
      "activations/layer4_attention_weight_max": 81.73184204101562,
      "activations/layer4_attention_weight_min": -86.97209930419922,
      "activations/layer5_attention_weight_max": 62.263404846191406,
      "activations/layer5_attention_weight_min": -73.85321044921875,
      "activations/layer6_attention_weight_max": 47.22331619262695,
      "activations/layer6_attention_weight_min": -48.9683723449707,
      "activations/layer7_attention_weight_max": 68.50552368164062,
      "activations/layer7_attention_weight_min": -68.9448471069336,
      "activations/layer8_attention_weight_max": 44.71803665161133,
      "activations/layer8_attention_weight_min": -48.23518753051758,
      "activations/layer9_attention_weight_max": 43.788360595703125,
      "activations/layer9_attention_weight_min": -46.68812942504883,
      "epoch": 21.99,
      "learning_rate": 8.277272727272725e-06,
      "loss": 2.7041,
      "step": 378450
    },
    {
      "activations/layer0_attention_weight_max": 16.136554718017578,
      "activations/layer0_attention_weight_min": -14.154838562011719,
      "activations/layer10_attention_weight_max": 33.29119873046875,
      "activations/layer10_attention_weight_min": -34.02510452270508,
      "activations/layer11_attention_weight_max": 33.175132751464844,
      "activations/layer11_attention_weight_min": -36.13562774658203,
      "activations/layer12_attention_weight_max": 18.156003952026367,
      "activations/layer12_attention_weight_min": -25.438385009765625,
      "activations/layer13_attention_weight_max": 31.93373680114746,
      "activations/layer13_attention_weight_min": -26.63929557800293,
      "activations/layer14_attention_weight_max": 34.34164047241211,
      "activations/layer14_attention_weight_min": -30.809837341308594,
      "activations/layer15_attention_weight_max": 31.94504165649414,
      "activations/layer15_attention_weight_min": -31.27680015563965,
      "activations/layer16_attention_weight_max": 33.576534271240234,
      "activations/layer16_attention_weight_min": -32.704872131347656,
      "activations/layer17_attention_weight_max": 51.69572067260742,
      "activations/layer17_attention_weight_min": -43.550262451171875,
      "activations/layer18_attention_weight_max": 45.84671401977539,
      "activations/layer18_attention_weight_min": -41.4508056640625,
      "activations/layer19_attention_weight_max": 23.088510513305664,
      "activations/layer19_attention_weight_min": -22.727619171142578,
      "activations/layer1_attention_weight_max": 16.473403930664062,
      "activations/layer1_attention_weight_min": -15.368208885192871,
      "activations/layer20_attention_weight_max": 21.504701614379883,
      "activations/layer20_attention_weight_min": -22.0636043548584,
      "activations/layer21_attention_weight_max": 32.659698486328125,
      "activations/layer21_attention_weight_min": -22.521011352539062,
      "activations/layer22_attention_weight_max": 27.983566284179688,
      "activations/layer22_attention_weight_min": -24.86415672302246,
      "activations/layer23_attention_weight_max": 33.66938400268555,
      "activations/layer23_attention_weight_min": -24.029705047607422,
      "activations/layer2_attention_weight_max": 32.25770568847656,
      "activations/layer2_attention_weight_min": -31.172651290893555,
      "activations/layer3_attention_weight_max": 89.44598388671875,
      "activations/layer3_attention_weight_min": -89.84272766113281,
      "activations/layer4_attention_weight_max": 83.3790283203125,
      "activations/layer4_attention_weight_min": -83.00608825683594,
      "activations/layer5_attention_weight_max": 60.6656494140625,
      "activations/layer5_attention_weight_min": -72.41142272949219,
      "activations/layer6_attention_weight_max": 50.123512268066406,
      "activations/layer6_attention_weight_min": -51.096614837646484,
      "activations/layer7_attention_weight_max": 68.67007446289062,
      "activations/layer7_attention_weight_min": -68.06624603271484,
      "activations/layer8_attention_weight_max": 43.43434524536133,
      "activations/layer8_attention_weight_min": -45.89339828491211,
      "activations/layer9_attention_weight_max": 39.31673812866211,
      "activations/layer9_attention_weight_min": -45.196834564208984,
      "epoch": 21.99,
      "learning_rate": 8.258333333333334e-06,
      "loss": 2.7022,
      "step": 378500
    },
    {
      "activations/layer0_attention_weight_max": 15.994224548339844,
      "activations/layer0_attention_weight_min": -14.330349922180176,
      "activations/layer10_attention_weight_max": 38.03346633911133,
      "activations/layer10_attention_weight_min": -39.3069953918457,
      "activations/layer11_attention_weight_max": 37.352970123291016,
      "activations/layer11_attention_weight_min": -37.29731750488281,
      "activations/layer12_attention_weight_max": 19.677223205566406,
      "activations/layer12_attention_weight_min": -23.836055755615234,
      "activations/layer13_attention_weight_max": 38.82476806640625,
      "activations/layer13_attention_weight_min": -28.99224090576172,
      "activations/layer14_attention_weight_max": 51.08925247192383,
      "activations/layer14_attention_weight_min": -31.67418098449707,
      "activations/layer15_attention_weight_max": 42.39143371582031,
      "activations/layer15_attention_weight_min": -32.002376556396484,
      "activations/layer16_attention_weight_max": 34.84370422363281,
      "activations/layer16_attention_weight_min": -33.50449752807617,
      "activations/layer17_attention_weight_max": 53.967742919921875,
      "activations/layer17_attention_weight_min": -48.63197326660156,
      "activations/layer18_attention_weight_max": 49.10164260864258,
      "activations/layer18_attention_weight_min": -43.128597259521484,
      "activations/layer19_attention_weight_max": 26.614593505859375,
      "activations/layer19_attention_weight_min": -25.1774959564209,
      "activations/layer1_attention_weight_max": 15.021512985229492,
      "activations/layer1_attention_weight_min": -14.521007537841797,
      "activations/layer20_attention_weight_max": 25.59105682373047,
      "activations/layer20_attention_weight_min": -21.11296844482422,
      "activations/layer21_attention_weight_max": 42.38762283325195,
      "activations/layer21_attention_weight_min": -23.591567993164062,
      "activations/layer22_attention_weight_max": 29.474382400512695,
      "activations/layer22_attention_weight_min": -25.10403823852539,
      "activations/layer23_attention_weight_max": 35.433204650878906,
      "activations/layer23_attention_weight_min": -24.60370445251465,
      "activations/layer2_attention_weight_max": 31.882747650146484,
      "activations/layer2_attention_weight_min": -30.828060150146484,
      "activations/layer3_attention_weight_max": 93.53173828125,
      "activations/layer3_attention_weight_min": -90.49067687988281,
      "activations/layer4_attention_weight_max": 89.48943328857422,
      "activations/layer4_attention_weight_min": -87.72313690185547,
      "activations/layer5_attention_weight_max": 66.12361907958984,
      "activations/layer5_attention_weight_min": -69.2601547241211,
      "activations/layer6_attention_weight_max": 51.60999298095703,
      "activations/layer6_attention_weight_min": -54.23624038696289,
      "activations/layer7_attention_weight_max": 69.27632904052734,
      "activations/layer7_attention_weight_min": -72.11320495605469,
      "activations/layer8_attention_weight_max": 50.26858139038086,
      "activations/layer8_attention_weight_min": -54.95814895629883,
      "activations/layer9_attention_weight_max": 47.45339584350586,
      "activations/layer9_attention_weight_min": -50.20203399658203,
      "epoch": 22.0,
      "learning_rate": 8.239393939393938e-06,
      "loss": 2.7102,
      "step": 378550
    },
    {
      "activations/layer0_attention_weight_max": 16.27933692932129,
      "activations/layer0_attention_weight_min": -13.998824119567871,
      "activations/layer10_attention_weight_max": 38.5914306640625,
      "activations/layer10_attention_weight_min": -37.379947662353516,
      "activations/layer11_attention_weight_max": 34.13652801513672,
      "activations/layer11_attention_weight_min": -36.948673248291016,
      "activations/layer12_attention_weight_max": 19.32088851928711,
      "activations/layer12_attention_weight_min": -22.96943473815918,
      "activations/layer13_attention_weight_max": 37.08746337890625,
      "activations/layer13_attention_weight_min": -28.50218963623047,
      "activations/layer14_attention_weight_max": 36.26893615722656,
      "activations/layer14_attention_weight_min": -29.310504913330078,
      "activations/layer15_attention_weight_max": 36.91468048095703,
      "activations/layer15_attention_weight_min": -30.465612411499023,
      "activations/layer16_attention_weight_max": 35.794349670410156,
      "activations/layer16_attention_weight_min": -32.93818283081055,
      "activations/layer17_attention_weight_max": 55.90599060058594,
      "activations/layer17_attention_weight_min": -44.24085998535156,
      "activations/layer18_attention_weight_max": 48.92694091796875,
      "activations/layer18_attention_weight_min": -39.9073371887207,
      "activations/layer19_attention_weight_max": 26.43092155456543,
      "activations/layer19_attention_weight_min": -24.09673500061035,
      "activations/layer1_attention_weight_max": 15.296414375305176,
      "activations/layer1_attention_weight_min": -14.871834754943848,
      "activations/layer20_attention_weight_max": 22.228727340698242,
      "activations/layer20_attention_weight_min": -22.45451545715332,
      "activations/layer21_attention_weight_max": 37.81952667236328,
      "activations/layer21_attention_weight_min": -23.405929565429688,
      "activations/layer22_attention_weight_max": 28.272968292236328,
      "activations/layer22_attention_weight_min": -24.849058151245117,
      "activations/layer23_attention_weight_max": 37.718605041503906,
      "activations/layer23_attention_weight_min": -25.132871627807617,
      "activations/layer2_attention_weight_max": 31.424169540405273,
      "activations/layer2_attention_weight_min": -30.64181900024414,
      "activations/layer3_attention_weight_max": 89.62384033203125,
      "activations/layer3_attention_weight_min": -90.1485595703125,
      "activations/layer4_attention_weight_max": 84.9206771850586,
      "activations/layer4_attention_weight_min": -83.70879364013672,
      "activations/layer5_attention_weight_max": 62.34537887573242,
      "activations/layer5_attention_weight_min": -69.94356536865234,
      "activations/layer6_attention_weight_max": 50.12502670288086,
      "activations/layer6_attention_weight_min": -52.43352508544922,
      "activations/layer7_attention_weight_max": 71.32618713378906,
      "activations/layer7_attention_weight_min": -67.77433776855469,
      "activations/layer8_attention_weight_max": 45.970428466796875,
      "activations/layer8_attention_weight_min": -51.47249221801758,
      "activations/layer9_attention_weight_max": 42.139244079589844,
      "activations/layer9_attention_weight_min": -47.15409851074219,
      "epoch": 22.0,
      "learning_rate": 8.220454545454545e-06,
      "loss": 2.7157,
      "step": 378600
    },
    {
      "activations/layer0_attention_weight_max": 15.829024314880371,
      "activations/layer0_attention_weight_min": -14.23633098602295,
      "activations/layer10_attention_weight_max": 35.8511848449707,
      "activations/layer10_attention_weight_min": -37.27052688598633,
      "activations/layer11_attention_weight_max": 33.567779541015625,
      "activations/layer11_attention_weight_min": -36.606971740722656,
      "activations/layer12_attention_weight_max": 19.852888107299805,
      "activations/layer12_attention_weight_min": -24.59613037109375,
      "activations/layer13_attention_weight_max": 38.710262298583984,
      "activations/layer13_attention_weight_min": -28.526025772094727,
      "activations/layer14_attention_weight_max": 35.89849090576172,
      "activations/layer14_attention_weight_min": -31.193191528320312,
      "activations/layer15_attention_weight_max": 35.437374114990234,
      "activations/layer15_attention_weight_min": -29.799182891845703,
      "activations/layer16_attention_weight_max": 35.63939666748047,
      "activations/layer16_attention_weight_min": -32.70237350463867,
      "activations/layer17_attention_weight_max": 50.25035095214844,
      "activations/layer17_attention_weight_min": -44.02996063232422,
      "activations/layer18_attention_weight_max": 48.182533264160156,
      "activations/layer18_attention_weight_min": -39.27077102661133,
      "activations/layer19_attention_weight_max": 26.26581382751465,
      "activations/layer19_attention_weight_min": -23.555517196655273,
      "activations/layer1_attention_weight_max": 15.196788787841797,
      "activations/layer1_attention_weight_min": -13.760957717895508,
      "activations/layer20_attention_weight_max": 22.69339370727539,
      "activations/layer20_attention_weight_min": -20.727262496948242,
      "activations/layer21_attention_weight_max": 32.779884338378906,
      "activations/layer21_attention_weight_min": -19.47053337097168,
      "activations/layer22_attention_weight_max": 29.53441047668457,
      "activations/layer22_attention_weight_min": -25.33210563659668,
      "activations/layer23_attention_weight_max": 36.389854431152344,
      "activations/layer23_attention_weight_min": -23.282073974609375,
      "activations/layer2_attention_weight_max": 32.19849395751953,
      "activations/layer2_attention_weight_min": -31.94815444946289,
      "activations/layer3_attention_weight_max": 92.2823257446289,
      "activations/layer3_attention_weight_min": -88.8184585571289,
      "activations/layer4_attention_weight_max": 85.9074935913086,
      "activations/layer4_attention_weight_min": -87.79448699951172,
      "activations/layer5_attention_weight_max": 61.08662414550781,
      "activations/layer5_attention_weight_min": -70.72620391845703,
      "activations/layer6_attention_weight_max": 50.02460479736328,
      "activations/layer6_attention_weight_min": -50.843833923339844,
      "activations/layer7_attention_weight_max": 69.90489959716797,
      "activations/layer7_attention_weight_min": -69.50090026855469,
      "activations/layer8_attention_weight_max": 47.46662139892578,
      "activations/layer8_attention_weight_min": -51.741310119628906,
      "activations/layer9_attention_weight_max": 46.11532211303711,
      "activations/layer9_attention_weight_min": -49.470462799072266,
      "epoch": 22.0,
      "learning_rate": 8.201136363636362e-06,
      "loss": 2.7404,
      "step": 378650
    },
    {
      "activations/layer0_attention_weight_max": 16.666505813598633,
      "activations/layer0_attention_weight_min": -14.312817573547363,
      "activations/layer10_attention_weight_max": 36.78725051879883,
      "activations/layer10_attention_weight_min": -37.152347564697266,
      "activations/layer11_attention_weight_max": 37.61053466796875,
      "activations/layer11_attention_weight_min": -35.388980865478516,
      "activations/layer12_attention_weight_max": 18.937421798706055,
      "activations/layer12_attention_weight_min": -27.228551864624023,
      "activations/layer13_attention_weight_max": 34.41538619995117,
      "activations/layer13_attention_weight_min": -27.37403678894043,
      "activations/layer14_attention_weight_max": 33.82557678222656,
      "activations/layer14_attention_weight_min": -28.668996810913086,
      "activations/layer15_attention_weight_max": 35.566184997558594,
      "activations/layer15_attention_weight_min": -28.979293823242188,
      "activations/layer16_attention_weight_max": 35.686954498291016,
      "activations/layer16_attention_weight_min": -33.83167266845703,
      "activations/layer17_attention_weight_max": 46.489234924316406,
      "activations/layer17_attention_weight_min": -43.05254364013672,
      "activations/layer18_attention_weight_max": 42.9984245300293,
      "activations/layer18_attention_weight_min": -38.69257354736328,
      "activations/layer19_attention_weight_max": 23.611452102661133,
      "activations/layer19_attention_weight_min": -22.64999771118164,
      "activations/layer1_attention_weight_max": 14.40488338470459,
      "activations/layer1_attention_weight_min": -14.148497581481934,
      "activations/layer20_attention_weight_max": 21.982641220092773,
      "activations/layer20_attention_weight_min": -20.891721725463867,
      "activations/layer21_attention_weight_max": 36.80816650390625,
      "activations/layer21_attention_weight_min": -24.630014419555664,
      "activations/layer22_attention_weight_max": 26.345029830932617,
      "activations/layer22_attention_weight_min": -23.45564079284668,
      "activations/layer23_attention_weight_max": 34.600059509277344,
      "activations/layer23_attention_weight_min": -23.971912384033203,
      "activations/layer2_attention_weight_max": 31.776885986328125,
      "activations/layer2_attention_weight_min": -32.04014587402344,
      "activations/layer3_attention_weight_max": 90.98784637451172,
      "activations/layer3_attention_weight_min": -92.19445037841797,
      "activations/layer4_attention_weight_max": 86.56866455078125,
      "activations/layer4_attention_weight_min": -85.3522720336914,
      "activations/layer5_attention_weight_max": 62.406883239746094,
      "activations/layer5_attention_weight_min": -76.41221618652344,
      "activations/layer6_attention_weight_max": 50.538333892822266,
      "activations/layer6_attention_weight_min": -51.231422424316406,
      "activations/layer7_attention_weight_max": 71.48793029785156,
      "activations/layer7_attention_weight_min": -71.75096130371094,
      "activations/layer8_attention_weight_max": 50.95083236694336,
      "activations/layer8_attention_weight_min": -50.54652786254883,
      "activations/layer9_attention_weight_max": 46.60061264038086,
      "activations/layer9_attention_weight_min": -47.06339645385742,
      "epoch": 22.0,
      "learning_rate": 8.182196969696969e-06,
      "loss": 2.7115,
      "step": 378700
    },
    {
      "activations/layer0_attention_weight_max": 17.208663940429688,
      "activations/layer0_attention_weight_min": -14.163718223571777,
      "activations/layer10_attention_weight_max": 35.08304977416992,
      "activations/layer10_attention_weight_min": -35.36076736450195,
      "activations/layer11_attention_weight_max": 32.16731262207031,
      "activations/layer11_attention_weight_min": -33.92223358154297,
      "activations/layer12_attention_weight_max": 18.48920440673828,
      "activations/layer12_attention_weight_min": -25.48368263244629,
      "activations/layer13_attention_weight_max": 30.521196365356445,
      "activations/layer13_attention_weight_min": -26.927186965942383,
      "activations/layer14_attention_weight_max": 36.60921859741211,
      "activations/layer14_attention_weight_min": -30.929000854492188,
      "activations/layer15_attention_weight_max": 34.86895751953125,
      "activations/layer15_attention_weight_min": -30.061275482177734,
      "activations/layer16_attention_weight_max": 36.9263916015625,
      "activations/layer16_attention_weight_min": -34.29785919189453,
      "activations/layer17_attention_weight_max": 57.48646926879883,
      "activations/layer17_attention_weight_min": -49.27147674560547,
      "activations/layer18_attention_weight_max": 52.612491607666016,
      "activations/layer18_attention_weight_min": -45.516441345214844,
      "activations/layer19_attention_weight_max": 28.059988021850586,
      "activations/layer19_attention_weight_min": -24.974950790405273,
      "activations/layer1_attention_weight_max": 15.697174072265625,
      "activations/layer1_attention_weight_min": -16.089523315429688,
      "activations/layer20_attention_weight_max": 27.59648895263672,
      "activations/layer20_attention_weight_min": -22.664867401123047,
      "activations/layer21_attention_weight_max": 36.39574432373047,
      "activations/layer21_attention_weight_min": -22.385833740234375,
      "activations/layer22_attention_weight_max": 29.28949737548828,
      "activations/layer22_attention_weight_min": -26.69612693786621,
      "activations/layer23_attention_weight_max": 33.762542724609375,
      "activations/layer23_attention_weight_min": -24.073436737060547,
      "activations/layer2_attention_weight_max": 31.66700553894043,
      "activations/layer2_attention_weight_min": -30.498367309570312,
      "activations/layer3_attention_weight_max": 92.59305572509766,
      "activations/layer3_attention_weight_min": -89.10400390625,
      "activations/layer4_attention_weight_max": 88.90404510498047,
      "activations/layer4_attention_weight_min": -88.16598510742188,
      "activations/layer5_attention_weight_max": 64.48847198486328,
      "activations/layer5_attention_weight_min": -73.67488098144531,
      "activations/layer6_attention_weight_max": 50.91398620605469,
      "activations/layer6_attention_weight_min": -51.03783416748047,
      "activations/layer7_attention_weight_max": 67.91535186767578,
      "activations/layer7_attention_weight_min": -69.849365234375,
      "activations/layer8_attention_weight_max": 47.50736999511719,
      "activations/layer8_attention_weight_min": -51.10954666137695,
      "activations/layer9_attention_weight_max": 45.39374542236328,
      "activations/layer9_attention_weight_min": -46.79203414916992,
      "epoch": 22.01,
      "learning_rate": 8.163257575757575e-06,
      "loss": 2.7071,
      "step": 378750
    },
    {
      "activations/layer0_attention_weight_max": 17.0832462310791,
      "activations/layer0_attention_weight_min": -14.274007797241211,
      "activations/layer10_attention_weight_max": 39.15266418457031,
      "activations/layer10_attention_weight_min": -36.416969299316406,
      "activations/layer11_attention_weight_max": 37.22267532348633,
      "activations/layer11_attention_weight_min": -35.207698822021484,
      "activations/layer12_attention_weight_max": 20.243398666381836,
      "activations/layer12_attention_weight_min": -24.56725311279297,
      "activations/layer13_attention_weight_max": 34.7943000793457,
      "activations/layer13_attention_weight_min": -27.529077529907227,
      "activations/layer14_attention_weight_max": 40.11320495605469,
      "activations/layer14_attention_weight_min": -32.20745849609375,
      "activations/layer15_attention_weight_max": 35.65668487548828,
      "activations/layer15_attention_weight_min": -31.82823371887207,
      "activations/layer16_attention_weight_max": 39.60552215576172,
      "activations/layer16_attention_weight_min": -33.36873245239258,
      "activations/layer17_attention_weight_max": 55.50107192993164,
      "activations/layer17_attention_weight_min": -46.8471565246582,
      "activations/layer18_attention_weight_max": 55.231201171875,
      "activations/layer18_attention_weight_min": -41.97789001464844,
      "activations/layer19_attention_weight_max": 30.218839645385742,
      "activations/layer19_attention_weight_min": -24.946121215820312,
      "activations/layer1_attention_weight_max": 15.29201602935791,
      "activations/layer1_attention_weight_min": -15.638467788696289,
      "activations/layer20_attention_weight_max": 27.915775299072266,
      "activations/layer20_attention_weight_min": -21.580142974853516,
      "activations/layer21_attention_weight_max": 39.697872161865234,
      "activations/layer21_attention_weight_min": -20.708831787109375,
      "activations/layer22_attention_weight_max": 32.96357345581055,
      "activations/layer22_attention_weight_min": -27.7375545501709,
      "activations/layer23_attention_weight_max": 39.5022087097168,
      "activations/layer23_attention_weight_min": -22.35871696472168,
      "activations/layer2_attention_weight_max": 32.694000244140625,
      "activations/layer2_attention_weight_min": -31.42044448852539,
      "activations/layer3_attention_weight_max": 91.57421112060547,
      "activations/layer3_attention_weight_min": -90.59207916259766,
      "activations/layer4_attention_weight_max": 90.09542083740234,
      "activations/layer4_attention_weight_min": -87.10713958740234,
      "activations/layer5_attention_weight_max": 67.38041687011719,
      "activations/layer5_attention_weight_min": -72.05278015136719,
      "activations/layer6_attention_weight_max": 52.04720687866211,
      "activations/layer6_attention_weight_min": -53.89241027832031,
      "activations/layer7_attention_weight_max": 71.5287094116211,
      "activations/layer7_attention_weight_min": -70.96867370605469,
      "activations/layer8_attention_weight_max": 50.10609436035156,
      "activations/layer8_attention_weight_min": -53.5192985534668,
      "activations/layer9_attention_weight_max": 46.26974105834961,
      "activations/layer9_attention_weight_min": -47.950225830078125,
      "epoch": 22.01,
      "learning_rate": 8.144696969696968e-06,
      "loss": 2.7022,
      "step": 378800
    },
    {
      "activations/layer0_attention_weight_max": 15.925317764282227,
      "activations/layer0_attention_weight_min": -14.253582954406738,
      "activations/layer10_attention_weight_max": 36.88175582885742,
      "activations/layer10_attention_weight_min": -35.39338684082031,
      "activations/layer11_attention_weight_max": 34.33287811279297,
      "activations/layer11_attention_weight_min": -34.56184387207031,
      "activations/layer12_attention_weight_max": 19.922229766845703,
      "activations/layer12_attention_weight_min": -25.372970581054688,
      "activations/layer13_attention_weight_max": 33.67427062988281,
      "activations/layer13_attention_weight_min": -27.98741912841797,
      "activations/layer14_attention_weight_max": 41.09673309326172,
      "activations/layer14_attention_weight_min": -33.41463851928711,
      "activations/layer15_attention_weight_max": 38.03264236450195,
      "activations/layer15_attention_weight_min": -33.170745849609375,
      "activations/layer16_attention_weight_max": 34.03527069091797,
      "activations/layer16_attention_weight_min": -32.996734619140625,
      "activations/layer17_attention_weight_max": 50.66994857788086,
      "activations/layer17_attention_weight_min": -46.9737434387207,
      "activations/layer18_attention_weight_max": 46.739707946777344,
      "activations/layer18_attention_weight_min": -42.90929412841797,
      "activations/layer19_attention_weight_max": 25.159025192260742,
      "activations/layer19_attention_weight_min": -24.44135093688965,
      "activations/layer1_attention_weight_max": 15.131237983703613,
      "activations/layer1_attention_weight_min": -14.474799156188965,
      "activations/layer20_attention_weight_max": 24.246910095214844,
      "activations/layer20_attention_weight_min": -21.596216201782227,
      "activations/layer21_attention_weight_max": 35.663055419921875,
      "activations/layer21_attention_weight_min": -22.3215389251709,
      "activations/layer22_attention_weight_max": 30.991466522216797,
      "activations/layer22_attention_weight_min": -24.861572265625,
      "activations/layer23_attention_weight_max": 33.68962478637695,
      "activations/layer23_attention_weight_min": -22.972366333007812,
      "activations/layer2_attention_weight_max": 35.752593994140625,
      "activations/layer2_attention_weight_min": -33.8465461730957,
      "activations/layer3_attention_weight_max": 91.78459930419922,
      "activations/layer3_attention_weight_min": -94.0494155883789,
      "activations/layer4_attention_weight_max": 89.1393051147461,
      "activations/layer4_attention_weight_min": -85.38471221923828,
      "activations/layer5_attention_weight_max": 60.54103088378906,
      "activations/layer5_attention_weight_min": -68.54487609863281,
      "activations/layer6_attention_weight_max": 50.291358947753906,
      "activations/layer6_attention_weight_min": -49.39346694946289,
      "activations/layer7_attention_weight_max": 71.3784408569336,
      "activations/layer7_attention_weight_min": -68.34115600585938,
      "activations/layer8_attention_weight_max": 51.85701370239258,
      "activations/layer8_attention_weight_min": -49.04176712036133,
      "activations/layer9_attention_weight_max": 48.838016510009766,
      "activations/layer9_attention_weight_min": -47.68545150756836,
      "epoch": 22.01,
      "learning_rate": 8.125757575757575e-06,
      "loss": 2.7209,
      "step": 378850
    },
    {
      "activations/layer0_attention_weight_max": 15.582842826843262,
      "activations/layer0_attention_weight_min": -14.03170108795166,
      "activations/layer10_attention_weight_max": 36.1190185546875,
      "activations/layer10_attention_weight_min": -36.48406982421875,
      "activations/layer11_attention_weight_max": 34.33983612060547,
      "activations/layer11_attention_weight_min": -35.83869934082031,
      "activations/layer12_attention_weight_max": 18.647598266601562,
      "activations/layer12_attention_weight_min": -24.51078987121582,
      "activations/layer13_attention_weight_max": 33.56159210205078,
      "activations/layer13_attention_weight_min": -26.566776275634766,
      "activations/layer14_attention_weight_max": 36.343299865722656,
      "activations/layer14_attention_weight_min": -32.16206359863281,
      "activations/layer15_attention_weight_max": 33.00832748413086,
      "activations/layer15_attention_weight_min": -32.479610443115234,
      "activations/layer16_attention_weight_max": 34.120849609375,
      "activations/layer16_attention_weight_min": -33.327144622802734,
      "activations/layer17_attention_weight_max": 50.7381477355957,
      "activations/layer17_attention_weight_min": -45.9492301940918,
      "activations/layer18_attention_weight_max": 47.100406646728516,
      "activations/layer18_attention_weight_min": -41.79438400268555,
      "activations/layer19_attention_weight_max": 25.598651885986328,
      "activations/layer19_attention_weight_min": -24.43598747253418,
      "activations/layer1_attention_weight_max": 14.650997161865234,
      "activations/layer1_attention_weight_min": -13.73966121673584,
      "activations/layer20_attention_weight_max": 22.37065887451172,
      "activations/layer20_attention_weight_min": -22.57921600341797,
      "activations/layer21_attention_weight_max": 32.42793273925781,
      "activations/layer21_attention_weight_min": -22.219614028930664,
      "activations/layer22_attention_weight_max": 26.694448471069336,
      "activations/layer22_attention_weight_min": -26.719038009643555,
      "activations/layer23_attention_weight_max": 33.73625183105469,
      "activations/layer23_attention_weight_min": -26.812530517578125,
      "activations/layer2_attention_weight_max": 32.07160568237305,
      "activations/layer2_attention_weight_min": -30.806087493896484,
      "activations/layer3_attention_weight_max": 91.3453598022461,
      "activations/layer3_attention_weight_min": -92.1468276977539,
      "activations/layer4_attention_weight_max": 85.96049499511719,
      "activations/layer4_attention_weight_min": -84.27728271484375,
      "activations/layer5_attention_weight_max": 60.943817138671875,
      "activations/layer5_attention_weight_min": -73.74183654785156,
      "activations/layer6_attention_weight_max": 52.18858337402344,
      "activations/layer6_attention_weight_min": -51.246971130371094,
      "activations/layer7_attention_weight_max": 69.08824920654297,
      "activations/layer7_attention_weight_min": -67.6331558227539,
      "activations/layer8_attention_weight_max": 51.50408172607422,
      "activations/layer8_attention_weight_min": -51.28846740722656,
      "activations/layer9_attention_weight_max": 44.69551086425781,
      "activations/layer9_attention_weight_min": -45.88747787475586,
      "epoch": 22.02,
      "learning_rate": 8.106818181818181e-06,
      "loss": 2.6962,
      "step": 378900
    },
    {
      "activations/layer0_attention_weight_max": 16.56591796875,
      "activations/layer0_attention_weight_min": -14.191265106201172,
      "activations/layer10_attention_weight_max": 39.762691497802734,
      "activations/layer10_attention_weight_min": -40.15283203125,
      "activations/layer11_attention_weight_max": 36.5268669128418,
      "activations/layer11_attention_weight_min": -37.951698303222656,
      "activations/layer12_attention_weight_max": 22.398239135742188,
      "activations/layer12_attention_weight_min": -25.921279907226562,
      "activations/layer13_attention_weight_max": 38.040035247802734,
      "activations/layer13_attention_weight_min": -30.186737060546875,
      "activations/layer14_attention_weight_max": 37.41460418701172,
      "activations/layer14_attention_weight_min": -32.91123580932617,
      "activations/layer15_attention_weight_max": 35.48185348510742,
      "activations/layer15_attention_weight_min": -32.86600112915039,
      "activations/layer16_attention_weight_max": 39.41470718383789,
      "activations/layer16_attention_weight_min": -35.95616912841797,
      "activations/layer17_attention_weight_max": 63.33283233642578,
      "activations/layer17_attention_weight_min": -53.406272888183594,
      "activations/layer18_attention_weight_max": 56.01328659057617,
      "activations/layer18_attention_weight_min": -45.957950592041016,
      "activations/layer19_attention_weight_max": 28.033889770507812,
      "activations/layer19_attention_weight_min": -24.100263595581055,
      "activations/layer1_attention_weight_max": 14.863608360290527,
      "activations/layer1_attention_weight_min": -14.4845552444458,
      "activations/layer20_attention_weight_max": 24.49236297607422,
      "activations/layer20_attention_weight_min": -21.706480026245117,
      "activations/layer21_attention_weight_max": 38.30189514160156,
      "activations/layer21_attention_weight_min": -22.823984146118164,
      "activations/layer22_attention_weight_max": 33.378326416015625,
      "activations/layer22_attention_weight_min": -24.857873916625977,
      "activations/layer23_attention_weight_max": 44.64106750488281,
      "activations/layer23_attention_weight_min": -23.35857391357422,
      "activations/layer2_attention_weight_max": 31.576581954956055,
      "activations/layer2_attention_weight_min": -31.691253662109375,
      "activations/layer3_attention_weight_max": 89.75264739990234,
      "activations/layer3_attention_weight_min": -89.10063171386719,
      "activations/layer4_attention_weight_max": 85.90035247802734,
      "activations/layer4_attention_weight_min": -86.85444641113281,
      "activations/layer5_attention_weight_max": 62.1049919128418,
      "activations/layer5_attention_weight_min": -70.64340209960938,
      "activations/layer6_attention_weight_max": 50.38056945800781,
      "activations/layer6_attention_weight_min": -49.90486526489258,
      "activations/layer7_attention_weight_max": 69.14200592041016,
      "activations/layer7_attention_weight_min": -71.9745864868164,
      "activations/layer8_attention_weight_max": 52.339256286621094,
      "activations/layer8_attention_weight_min": -52.37358474731445,
      "activations/layer9_attention_weight_max": 50.41440963745117,
      "activations/layer9_attention_weight_min": -47.243255615234375,
      "epoch": 22.02,
      "learning_rate": 8.087878787878788e-06,
      "loss": 2.6973,
      "step": 378950
    },
    {
      "activations/layer0_attention_weight_max": 15.975725173950195,
      "activations/layer0_attention_weight_min": -13.676952362060547,
      "activations/layer10_attention_weight_max": 33.82426071166992,
      "activations/layer10_attention_weight_min": -35.140052795410156,
      "activations/layer11_attention_weight_max": 33.99521255493164,
      "activations/layer11_attention_weight_min": -35.89243698120117,
      "activations/layer12_attention_weight_max": 19.689456939697266,
      "activations/layer12_attention_weight_min": -26.07406234741211,
      "activations/layer13_attention_weight_max": 35.11960983276367,
      "activations/layer13_attention_weight_min": -27.523469924926758,
      "activations/layer14_attention_weight_max": 36.43451690673828,
      "activations/layer14_attention_weight_min": -30.075408935546875,
      "activations/layer15_attention_weight_max": 35.294639587402344,
      "activations/layer15_attention_weight_min": -29.94086265563965,
      "activations/layer16_attention_weight_max": 36.10761260986328,
      "activations/layer16_attention_weight_min": -32.8721809387207,
      "activations/layer17_attention_weight_max": 54.00349044799805,
      "activations/layer17_attention_weight_min": -45.76308059692383,
      "activations/layer18_attention_weight_max": 47.783138275146484,
      "activations/layer18_attention_weight_min": -40.248931884765625,
      "activations/layer19_attention_weight_max": 25.988157272338867,
      "activations/layer19_attention_weight_min": -23.485212326049805,
      "activations/layer1_attention_weight_max": 15.49409294128418,
      "activations/layer1_attention_weight_min": -14.536551475524902,
      "activations/layer20_attention_weight_max": 24.860654830932617,
      "activations/layer20_attention_weight_min": -22.931921005249023,
      "activations/layer21_attention_weight_max": 38.91756820678711,
      "activations/layer21_attention_weight_min": -26.381589889526367,
      "activations/layer22_attention_weight_max": 28.44178009033203,
      "activations/layer22_attention_weight_min": -25.85733413696289,
      "activations/layer23_attention_weight_max": 42.48262023925781,
      "activations/layer23_attention_weight_min": -25.821701049804688,
      "activations/layer2_attention_weight_max": 32.11081314086914,
      "activations/layer2_attention_weight_min": -30.448265075683594,
      "activations/layer3_attention_weight_max": 93.74323272705078,
      "activations/layer3_attention_weight_min": -88.42647552490234,
      "activations/layer4_attention_weight_max": 88.88433074951172,
      "activations/layer4_attention_weight_min": -85.9563217163086,
      "activations/layer5_attention_weight_max": 61.51433563232422,
      "activations/layer5_attention_weight_min": -72.25714111328125,
      "activations/layer6_attention_weight_max": 48.74795913696289,
      "activations/layer6_attention_weight_min": -50.4916877746582,
      "activations/layer7_attention_weight_max": 67.741943359375,
      "activations/layer7_attention_weight_min": -67.51746368408203,
      "activations/layer8_attention_weight_max": 46.778411865234375,
      "activations/layer8_attention_weight_min": -48.57006072998047,
      "activations/layer9_attention_weight_max": 43.224998474121094,
      "activations/layer9_attention_weight_min": -44.14960861206055,
      "epoch": 22.02,
      "learning_rate": 8.068939393939394e-06,
      "loss": 2.7141,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.6059,
      "eval_samples_per_second": 498.961,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.6059,
      "eval_openwebtext_samples_per_second": 498.961,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9695,
      "eval_wikitext_samples_per_second": 231.533,
      "step": 379000
    },
    {
      "epoch": 22.02,
      "eval_lambada_loss": 2.4140625,
      "eval_lambada_ppl": 11.17928486368573,
      "eval_lambada_runtime": 9.5878,
      "eval_lambada_samples_per_second": 507.833,
      "step": 379000
    },
    {
      "activations/layer0_attention_weight_max": 16.546119689941406,
      "activations/layer0_attention_weight_min": -14.107765197753906,
      "activations/layer10_attention_weight_max": 37.39251708984375,
      "activations/layer10_attention_weight_min": -35.5111083984375,
      "activations/layer11_attention_weight_max": 34.19288635253906,
      "activations/layer11_attention_weight_min": -34.238441467285156,
      "activations/layer12_attention_weight_max": 19.03769874572754,
      "activations/layer12_attention_weight_min": -25.9009952545166,
      "activations/layer13_attention_weight_max": 33.10551834106445,
      "activations/layer13_attention_weight_min": -26.310041427612305,
      "activations/layer14_attention_weight_max": 34.81637954711914,
      "activations/layer14_attention_weight_min": -30.526216506958008,
      "activations/layer15_attention_weight_max": 34.609981536865234,
      "activations/layer15_attention_weight_min": -29.619773864746094,
      "activations/layer16_attention_weight_max": 34.52479934692383,
      "activations/layer16_attention_weight_min": -33.040626525878906,
      "activations/layer17_attention_weight_max": 51.9515495300293,
      "activations/layer17_attention_weight_min": -45.652992248535156,
      "activations/layer18_attention_weight_max": 50.859432220458984,
      "activations/layer18_attention_weight_min": -40.111549377441406,
      "activations/layer19_attention_weight_max": 28.226163864135742,
      "activations/layer19_attention_weight_min": -21.226097106933594,
      "activations/layer1_attention_weight_max": 15.039510726928711,
      "activations/layer1_attention_weight_min": -13.527214050292969,
      "activations/layer20_attention_weight_max": 22.86408042907715,
      "activations/layer20_attention_weight_min": -20.894805908203125,
      "activations/layer21_attention_weight_max": 35.75012969970703,
      "activations/layer21_attention_weight_min": -19.94345474243164,
      "activations/layer22_attention_weight_max": 30.494701385498047,
      "activations/layer22_attention_weight_min": -24.69207000732422,
      "activations/layer23_attention_weight_max": 35.82459259033203,
      "activations/layer23_attention_weight_min": -22.845935821533203,
      "activations/layer2_attention_weight_max": 30.11452865600586,
      "activations/layer2_attention_weight_min": -28.573993682861328,
      "activations/layer3_attention_weight_max": 87.1351318359375,
      "activations/layer3_attention_weight_min": -86.7478256225586,
      "activations/layer4_attention_weight_max": 82.93106842041016,
      "activations/layer4_attention_weight_min": -83.35631561279297,
      "activations/layer5_attention_weight_max": 59.96220016479492,
      "activations/layer5_attention_weight_min": -73.03572845458984,
      "activations/layer6_attention_weight_max": 48.65076446533203,
      "activations/layer6_attention_weight_min": -50.51803207397461,
      "activations/layer7_attention_weight_max": 68.81410217285156,
      "activations/layer7_attention_weight_min": -70.2136459350586,
      "activations/layer8_attention_weight_max": 44.40486526489258,
      "activations/layer8_attention_weight_min": -49.171302795410156,
      "activations/layer9_attention_weight_max": 41.47025680541992,
      "activations/layer9_attention_weight_min": -48.43631362915039,
      "epoch": 22.02,
      "learning_rate": 8.05e-06,
      "loss": 2.703,
      "step": 379050
    },
    {
      "activations/layer0_attention_weight_max": 17.07276153564453,
      "activations/layer0_attention_weight_min": -13.879637718200684,
      "activations/layer10_attention_weight_max": 34.308998107910156,
      "activations/layer10_attention_weight_min": -34.51697540283203,
      "activations/layer11_attention_weight_max": 33.21049118041992,
      "activations/layer11_attention_weight_min": -33.65386962890625,
      "activations/layer12_attention_weight_max": 20.349401473999023,
      "activations/layer12_attention_weight_min": -25.53716468811035,
      "activations/layer13_attention_weight_max": 34.90997314453125,
      "activations/layer13_attention_weight_min": -28.169275283813477,
      "activations/layer14_attention_weight_max": 36.668094635009766,
      "activations/layer14_attention_weight_min": -30.745548248291016,
      "activations/layer15_attention_weight_max": 33.6371955871582,
      "activations/layer15_attention_weight_min": -31.109817504882812,
      "activations/layer16_attention_weight_max": 38.19577407836914,
      "activations/layer16_attention_weight_min": -34.56291580200195,
      "activations/layer17_attention_weight_max": 52.049896240234375,
      "activations/layer17_attention_weight_min": -46.178680419921875,
      "activations/layer18_attention_weight_max": 51.203094482421875,
      "activations/layer18_attention_weight_min": -40.37438201904297,
      "activations/layer19_attention_weight_max": 25.351472854614258,
      "activations/layer19_attention_weight_min": -22.39991569519043,
      "activations/layer1_attention_weight_max": 15.677461624145508,
      "activations/layer1_attention_weight_min": -13.315446853637695,
      "activations/layer20_attention_weight_max": 24.98423957824707,
      "activations/layer20_attention_weight_min": -21.464378356933594,
      "activations/layer21_attention_weight_max": 37.28460693359375,
      "activations/layer21_attention_weight_min": -21.39836883544922,
      "activations/layer22_attention_weight_max": 31.00547218322754,
      "activations/layer22_attention_weight_min": -25.368711471557617,
      "activations/layer23_attention_weight_max": 36.618404388427734,
      "activations/layer23_attention_weight_min": -23.173656463623047,
      "activations/layer2_attention_weight_max": 29.85036277770996,
      "activations/layer2_attention_weight_min": -29.117156982421875,
      "activations/layer3_attention_weight_max": 88.85485076904297,
      "activations/layer3_attention_weight_min": -86.44679260253906,
      "activations/layer4_attention_weight_max": 86.33358001708984,
      "activations/layer4_attention_weight_min": -83.62594604492188,
      "activations/layer5_attention_weight_max": 62.64552307128906,
      "activations/layer5_attention_weight_min": -70.61300659179688,
      "activations/layer6_attention_weight_max": 48.09796905517578,
      "activations/layer6_attention_weight_min": -49.92036819458008,
      "activations/layer7_attention_weight_max": 70.41791534423828,
      "activations/layer7_attention_weight_min": -66.37262725830078,
      "activations/layer8_attention_weight_max": 45.32146072387695,
      "activations/layer8_attention_weight_min": -50.53763198852539,
      "activations/layer9_attention_weight_max": 42.88011169433594,
      "activations/layer9_attention_weight_min": -45.288448333740234,
      "epoch": 22.03,
      "learning_rate": 8.031060606060606e-06,
      "loss": 2.7141,
      "step": 379100
    },
    {
      "activations/layer0_attention_weight_max": 15.485286712646484,
      "activations/layer0_attention_weight_min": -14.980192184448242,
      "activations/layer10_attention_weight_max": 36.719207763671875,
      "activations/layer10_attention_weight_min": -35.52150344848633,
      "activations/layer11_attention_weight_max": 33.19015884399414,
      "activations/layer11_attention_weight_min": -35.263702392578125,
      "activations/layer12_attention_weight_max": 19.32219886779785,
      "activations/layer12_attention_weight_min": -24.61469078063965,
      "activations/layer13_attention_weight_max": 32.30686569213867,
      "activations/layer13_attention_weight_min": -27.424379348754883,
      "activations/layer14_attention_weight_max": 32.77402877807617,
      "activations/layer14_attention_weight_min": -29.519458770751953,
      "activations/layer15_attention_weight_max": 32.778038024902344,
      "activations/layer15_attention_weight_min": -30.506515502929688,
      "activations/layer16_attention_weight_max": 33.352542877197266,
      "activations/layer16_attention_weight_min": -32.56894302368164,
      "activations/layer17_attention_weight_max": 49.23060989379883,
      "activations/layer17_attention_weight_min": -43.9963493347168,
      "activations/layer18_attention_weight_max": 48.0071907043457,
      "activations/layer18_attention_weight_min": -41.627281188964844,
      "activations/layer19_attention_weight_max": 22.71782684326172,
      "activations/layer19_attention_weight_min": -23.242263793945312,
      "activations/layer1_attention_weight_max": 14.96588134765625,
      "activations/layer1_attention_weight_min": -14.52973461151123,
      "activations/layer20_attention_weight_max": 21.1699275970459,
      "activations/layer20_attention_weight_min": -23.63442039489746,
      "activations/layer21_attention_weight_max": 33.429237365722656,
      "activations/layer21_attention_weight_min": -23.00516700744629,
      "activations/layer22_attention_weight_max": 29.858686447143555,
      "activations/layer22_attention_weight_min": -25.360599517822266,
      "activations/layer23_attention_weight_max": 34.53642272949219,
      "activations/layer23_attention_weight_min": -26.943212509155273,
      "activations/layer2_attention_weight_max": 31.355527877807617,
      "activations/layer2_attention_weight_min": -32.03728485107422,
      "activations/layer3_attention_weight_max": 87.61882781982422,
      "activations/layer3_attention_weight_min": -90.10902404785156,
      "activations/layer4_attention_weight_max": 84.15858459472656,
      "activations/layer4_attention_weight_min": -83.60942077636719,
      "activations/layer5_attention_weight_max": 61.35558319091797,
      "activations/layer5_attention_weight_min": -73.31892395019531,
      "activations/layer6_attention_weight_max": 48.4238395690918,
      "activations/layer6_attention_weight_min": -50.80071258544922,
      "activations/layer7_attention_weight_max": 67.35054779052734,
      "activations/layer7_attention_weight_min": -66.34239959716797,
      "activations/layer8_attention_weight_max": 48.21803283691406,
      "activations/layer8_attention_weight_min": -50.81269454956055,
      "activations/layer9_attention_weight_max": 43.46000289916992,
      "activations/layer9_attention_weight_min": -46.50551986694336,
      "epoch": 22.03,
      "learning_rate": 8.01212121212121e-06,
      "loss": 2.6994,
      "step": 379150
    },
    {
      "activations/layer0_attention_weight_max": 16.163509368896484,
      "activations/layer0_attention_weight_min": -14.065668106079102,
      "activations/layer10_attention_weight_max": 39.643043518066406,
      "activations/layer10_attention_weight_min": -38.06898880004883,
      "activations/layer11_attention_weight_max": 37.56148147583008,
      "activations/layer11_attention_weight_min": -36.393646240234375,
      "activations/layer12_attention_weight_max": 21.045774459838867,
      "activations/layer12_attention_weight_min": -27.314725875854492,
      "activations/layer13_attention_weight_max": 33.00719451904297,
      "activations/layer13_attention_weight_min": -26.469432830810547,
      "activations/layer14_attention_weight_max": 34.88705825805664,
      "activations/layer14_attention_weight_min": -30.430185317993164,
      "activations/layer15_attention_weight_max": 33.452816009521484,
      "activations/layer15_attention_weight_min": -30.30461883544922,
      "activations/layer16_attention_weight_max": 35.58053970336914,
      "activations/layer16_attention_weight_min": -32.38392639160156,
      "activations/layer17_attention_weight_max": 55.999778747558594,
      "activations/layer17_attention_weight_min": -44.5964241027832,
      "activations/layer18_attention_weight_max": 51.833473205566406,
      "activations/layer18_attention_weight_min": -40.583927154541016,
      "activations/layer19_attention_weight_max": 25.75540542602539,
      "activations/layer19_attention_weight_min": -22.939496994018555,
      "activations/layer1_attention_weight_max": 15.31491470336914,
      "activations/layer1_attention_weight_min": -13.996685981750488,
      "activations/layer20_attention_weight_max": 25.494855880737305,
      "activations/layer20_attention_weight_min": -22.099733352661133,
      "activations/layer21_attention_weight_max": 43.00661087036133,
      "activations/layer21_attention_weight_min": -22.866668701171875,
      "activations/layer22_attention_weight_max": 30.18104362487793,
      "activations/layer22_attention_weight_min": -24.67721176147461,
      "activations/layer23_attention_weight_max": 37.77244567871094,
      "activations/layer23_attention_weight_min": -23.633350372314453,
      "activations/layer2_attention_weight_max": 32.292510986328125,
      "activations/layer2_attention_weight_min": -29.812461853027344,
      "activations/layer3_attention_weight_max": 88.2371826171875,
      "activations/layer3_attention_weight_min": -87.79386901855469,
      "activations/layer4_attention_weight_max": 84.39299011230469,
      "activations/layer4_attention_weight_min": -83.20699310302734,
      "activations/layer5_attention_weight_max": 60.61028289794922,
      "activations/layer5_attention_weight_min": -73.13872528076172,
      "activations/layer6_attention_weight_max": 51.24272918701172,
      "activations/layer6_attention_weight_min": -54.76467514038086,
      "activations/layer7_attention_weight_max": 70.76254272460938,
      "activations/layer7_attention_weight_min": -67.4985122680664,
      "activations/layer8_attention_weight_max": 51.18925857543945,
      "activations/layer8_attention_weight_min": -53.934486389160156,
      "activations/layer9_attention_weight_max": 50.72347640991211,
      "activations/layer9_attention_weight_min": -49.40921401977539,
      "epoch": 22.03,
      "learning_rate": 7.993181818181817e-06,
      "loss": 2.7134,
      "step": 379200
    },
    {
      "activations/layer0_attention_weight_max": 15.380753517150879,
      "activations/layer0_attention_weight_min": -13.858692169189453,
      "activations/layer10_attention_weight_max": 39.38745880126953,
      "activations/layer10_attention_weight_min": -39.59767150878906,
      "activations/layer11_attention_weight_max": 36.14655303955078,
      "activations/layer11_attention_weight_min": -37.18576431274414,
      "activations/layer12_attention_weight_max": 23.19354820251465,
      "activations/layer12_attention_weight_min": -25.255765914916992,
      "activations/layer13_attention_weight_max": 40.39177322387695,
      "activations/layer13_attention_weight_min": -28.64932632446289,
      "activations/layer14_attention_weight_max": 37.831756591796875,
      "activations/layer14_attention_weight_min": -32.09258270263672,
      "activations/layer15_attention_weight_max": 38.25271987915039,
      "activations/layer15_attention_weight_min": -33.997982025146484,
      "activations/layer16_attention_weight_max": 38.37919998168945,
      "activations/layer16_attention_weight_min": -34.834232330322266,
      "activations/layer17_attention_weight_max": 55.828529357910156,
      "activations/layer17_attention_weight_min": -46.90312576293945,
      "activations/layer18_attention_weight_max": 51.346195220947266,
      "activations/layer18_attention_weight_min": -41.40605926513672,
      "activations/layer19_attention_weight_max": 27.155029296875,
      "activations/layer19_attention_weight_min": -24.31764793395996,
      "activations/layer1_attention_weight_max": 14.896451950073242,
      "activations/layer1_attention_weight_min": -14.130867004394531,
      "activations/layer20_attention_weight_max": 25.178504943847656,
      "activations/layer20_attention_weight_min": -21.93158531188965,
      "activations/layer21_attention_weight_max": 37.8503532409668,
      "activations/layer21_attention_weight_min": -22.656200408935547,
      "activations/layer22_attention_weight_max": 30.150484085083008,
      "activations/layer22_attention_weight_min": -25.42246437072754,
      "activations/layer23_attention_weight_max": 39.1114501953125,
      "activations/layer23_attention_weight_min": -23.733028411865234,
      "activations/layer2_attention_weight_max": 31.139738082885742,
      "activations/layer2_attention_weight_min": -30.36196517944336,
      "activations/layer3_attention_weight_max": 88.8702621459961,
      "activations/layer3_attention_weight_min": -95.54143524169922,
      "activations/layer4_attention_weight_max": 87.41987609863281,
      "activations/layer4_attention_weight_min": -87.70037078857422,
      "activations/layer5_attention_weight_max": 67.37521362304688,
      "activations/layer5_attention_weight_min": -74.04719543457031,
      "activations/layer6_attention_weight_max": 54.737667083740234,
      "activations/layer6_attention_weight_min": -53.64569854736328,
      "activations/layer7_attention_weight_max": 72.86852264404297,
      "activations/layer7_attention_weight_min": -72.60797119140625,
      "activations/layer8_attention_weight_max": 51.05556869506836,
      "activations/layer8_attention_weight_min": -53.41288757324219,
      "activations/layer9_attention_weight_max": 47.920066833496094,
      "activations/layer9_attention_weight_min": -52.289581298828125,
      "epoch": 22.04,
      "learning_rate": 7.974242424242424e-06,
      "loss": 2.7022,
      "step": 379250
    },
    {
      "activations/layer0_attention_weight_max": 15.496018409729004,
      "activations/layer0_attention_weight_min": -13.871278762817383,
      "activations/layer10_attention_weight_max": 33.92970657348633,
      "activations/layer10_attention_weight_min": -35.0630989074707,
      "activations/layer11_attention_weight_max": 33.324954986572266,
      "activations/layer11_attention_weight_min": -33.727325439453125,
      "activations/layer12_attention_weight_max": 20.27140998840332,
      "activations/layer12_attention_weight_min": -31.222408294677734,
      "activations/layer13_attention_weight_max": 36.668148040771484,
      "activations/layer13_attention_weight_min": -27.581100463867188,
      "activations/layer14_attention_weight_max": 35.824745178222656,
      "activations/layer14_attention_weight_min": -31.84893226623535,
      "activations/layer15_attention_weight_max": 32.801448822021484,
      "activations/layer15_attention_weight_min": -30.826887130737305,
      "activations/layer16_attention_weight_max": 32.32028579711914,
      "activations/layer16_attention_weight_min": -32.65167999267578,
      "activations/layer17_attention_weight_max": 49.2548713684082,
      "activations/layer17_attention_weight_min": -43.7880744934082,
      "activations/layer18_attention_weight_max": 46.6236686706543,
      "activations/layer18_attention_weight_min": -36.88205337524414,
      "activations/layer19_attention_weight_max": 23.709983825683594,
      "activations/layer19_attention_weight_min": -21.36475372314453,
      "activations/layer1_attention_weight_max": 15.312885284423828,
      "activations/layer1_attention_weight_min": -13.555188179016113,
      "activations/layer20_attention_weight_max": 21.160728454589844,
      "activations/layer20_attention_weight_min": -23.67201805114746,
      "activations/layer21_attention_weight_max": 33.07012939453125,
      "activations/layer21_attention_weight_min": -21.107568740844727,
      "activations/layer22_attention_weight_max": 27.40813636779785,
      "activations/layer22_attention_weight_min": -24.65060806274414,
      "activations/layer23_attention_weight_max": 32.612213134765625,
      "activations/layer23_attention_weight_min": -24.04915428161621,
      "activations/layer2_attention_weight_max": 30.63496971130371,
      "activations/layer2_attention_weight_min": -29.252607345581055,
      "activations/layer3_attention_weight_max": 91.22550964355469,
      "activations/layer3_attention_weight_min": -89.29122161865234,
      "activations/layer4_attention_weight_max": 80.81877899169922,
      "activations/layer4_attention_weight_min": -80.09362030029297,
      "activations/layer5_attention_weight_max": 57.620445251464844,
      "activations/layer5_attention_weight_min": -72.2973861694336,
      "activations/layer6_attention_weight_max": 47.165916442871094,
      "activations/layer6_attention_weight_min": -47.02257537841797,
      "activations/layer7_attention_weight_max": 64.83499908447266,
      "activations/layer7_attention_weight_min": -66.562744140625,
      "activations/layer8_attention_weight_max": 44.75668716430664,
      "activations/layer8_attention_weight_min": -51.80512237548828,
      "activations/layer9_attention_weight_max": 47.03080368041992,
      "activations/layer9_attention_weight_min": -49.627723693847656,
      "epoch": 22.04,
      "learning_rate": 7.955303030303029e-06,
      "loss": 2.6861,
      "step": 379300
    },
    {
      "activations/layer0_attention_weight_max": 16.568050384521484,
      "activations/layer0_attention_weight_min": -14.272610664367676,
      "activations/layer10_attention_weight_max": 35.6585578918457,
      "activations/layer10_attention_weight_min": -39.66584014892578,
      "activations/layer11_attention_weight_max": 33.36389923095703,
      "activations/layer11_attention_weight_min": -38.846961975097656,
      "activations/layer12_attention_weight_max": 19.22093391418457,
      "activations/layer12_attention_weight_min": -23.554676055908203,
      "activations/layer13_attention_weight_max": 36.381919860839844,
      "activations/layer13_attention_weight_min": -27.171289443969727,
      "activations/layer14_attention_weight_max": 33.639644622802734,
      "activations/layer14_attention_weight_min": -30.93174934387207,
      "activations/layer15_attention_weight_max": 34.017391204833984,
      "activations/layer15_attention_weight_min": -33.56083679199219,
      "activations/layer16_attention_weight_max": 33.515560150146484,
      "activations/layer16_attention_weight_min": -32.3997688293457,
      "activations/layer17_attention_weight_max": 49.21950149536133,
      "activations/layer17_attention_weight_min": -45.81956100463867,
      "activations/layer18_attention_weight_max": 48.181358337402344,
      "activations/layer18_attention_weight_min": -42.336639404296875,
      "activations/layer19_attention_weight_max": 22.63332748413086,
      "activations/layer19_attention_weight_min": -26.187692642211914,
      "activations/layer1_attention_weight_max": 15.00162410736084,
      "activations/layer1_attention_weight_min": -13.358214378356934,
      "activations/layer20_attention_weight_max": 21.64970588684082,
      "activations/layer20_attention_weight_min": -24.264060974121094,
      "activations/layer21_attention_weight_max": 38.978214263916016,
      "activations/layer21_attention_weight_min": -23.882488250732422,
      "activations/layer22_attention_weight_max": 28.100482940673828,
      "activations/layer22_attention_weight_min": -24.75731658935547,
      "activations/layer23_attention_weight_max": 35.151145935058594,
      "activations/layer23_attention_weight_min": -24.517621994018555,
      "activations/layer2_attention_weight_max": 31.719982147216797,
      "activations/layer2_attention_weight_min": -30.108489990234375,
      "activations/layer3_attention_weight_max": 87.05530548095703,
      "activations/layer3_attention_weight_min": -86.1141586303711,
      "activations/layer4_attention_weight_max": 81.87887573242188,
      "activations/layer4_attention_weight_min": -81.23534393310547,
      "activations/layer5_attention_weight_max": 59.004356384277344,
      "activations/layer5_attention_weight_min": -71.90386962890625,
      "activations/layer6_attention_weight_max": 48.71215057373047,
      "activations/layer6_attention_weight_min": -49.376258850097656,
      "activations/layer7_attention_weight_max": 68.31439208984375,
      "activations/layer7_attention_weight_min": -66.44535064697266,
      "activations/layer8_attention_weight_max": 47.30010223388672,
      "activations/layer8_attention_weight_min": -49.55388641357422,
      "activations/layer9_attention_weight_max": 42.74702453613281,
      "activations/layer9_attention_weight_min": -46.062068939208984,
      "epoch": 22.04,
      "learning_rate": 7.936363636363637e-06,
      "loss": 2.6939,
      "step": 379350
    },
    {
      "activations/layer0_attention_weight_max": 16.105575561523438,
      "activations/layer0_attention_weight_min": -14.57986068725586,
      "activations/layer10_attention_weight_max": 34.678871154785156,
      "activations/layer10_attention_weight_min": -33.799583435058594,
      "activations/layer11_attention_weight_max": 33.20606231689453,
      "activations/layer11_attention_weight_min": -33.79376983642578,
      "activations/layer12_attention_weight_max": 19.134687423706055,
      "activations/layer12_attention_weight_min": -25.689390182495117,
      "activations/layer13_attention_weight_max": 33.680259704589844,
      "activations/layer13_attention_weight_min": -25.945049285888672,
      "activations/layer14_attention_weight_max": 35.395118713378906,
      "activations/layer14_attention_weight_min": -29.150707244873047,
      "activations/layer15_attention_weight_max": 32.938045501708984,
      "activations/layer15_attention_weight_min": -28.43564224243164,
      "activations/layer16_attention_weight_max": 34.49424362182617,
      "activations/layer16_attention_weight_min": -33.43686294555664,
      "activations/layer17_attention_weight_max": 51.45361328125,
      "activations/layer17_attention_weight_min": -44.22197723388672,
      "activations/layer18_attention_weight_max": 47.48456954956055,
      "activations/layer18_attention_weight_min": -41.529136657714844,
      "activations/layer19_attention_weight_max": 24.11003875732422,
      "activations/layer19_attention_weight_min": -21.97077751159668,
      "activations/layer1_attention_weight_max": 14.756620407104492,
      "activations/layer1_attention_weight_min": -12.885115623474121,
      "activations/layer20_attention_weight_max": 20.538150787353516,
      "activations/layer20_attention_weight_min": -21.50124740600586,
      "activations/layer21_attention_weight_max": 34.05393981933594,
      "activations/layer21_attention_weight_min": -21.015100479125977,
      "activations/layer22_attention_weight_max": 30.622373580932617,
      "activations/layer22_attention_weight_min": -27.498729705810547,
      "activations/layer23_attention_weight_max": 35.2921028137207,
      "activations/layer23_attention_weight_min": -23.46233367919922,
      "activations/layer2_attention_weight_max": 30.346342086791992,
      "activations/layer2_attention_weight_min": -29.759458541870117,
      "activations/layer3_attention_weight_max": 86.82452392578125,
      "activations/layer3_attention_weight_min": -87.6871337890625,
      "activations/layer4_attention_weight_max": 83.75524139404297,
      "activations/layer4_attention_weight_min": -82.84343719482422,
      "activations/layer5_attention_weight_max": 59.7216796875,
      "activations/layer5_attention_weight_min": -71.06404113769531,
      "activations/layer6_attention_weight_max": 49.501441955566406,
      "activations/layer6_attention_weight_min": -50.463233947753906,
      "activations/layer7_attention_weight_max": 66.58321380615234,
      "activations/layer7_attention_weight_min": -68.54421997070312,
      "activations/layer8_attention_weight_max": 47.939064025878906,
      "activations/layer8_attention_weight_min": -50.88850021362305,
      "activations/layer9_attention_weight_max": 42.70331573486328,
      "activations/layer9_attention_weight_min": -44.382110595703125,
      "epoch": 22.05,
      "learning_rate": 7.917424242424242e-06,
      "loss": 2.7055,
      "step": 379400
    },
    {
      "activations/layer0_attention_weight_max": 15.88624382019043,
      "activations/layer0_attention_weight_min": -14.004895210266113,
      "activations/layer10_attention_weight_max": 36.80414962768555,
      "activations/layer10_attention_weight_min": -38.0220947265625,
      "activations/layer11_attention_weight_max": 34.178524017333984,
      "activations/layer11_attention_weight_min": -35.90648651123047,
      "activations/layer12_attention_weight_max": 20.392719268798828,
      "activations/layer12_attention_weight_min": -23.719234466552734,
      "activations/layer13_attention_weight_max": 35.4814567565918,
      "activations/layer13_attention_weight_min": -26.65712547302246,
      "activations/layer14_attention_weight_max": 35.764522552490234,
      "activations/layer14_attention_weight_min": -30.305452346801758,
      "activations/layer15_attention_weight_max": 37.486000061035156,
      "activations/layer15_attention_weight_min": -30.087932586669922,
      "activations/layer16_attention_weight_max": 35.6392707824707,
      "activations/layer16_attention_weight_min": -31.312679290771484,
      "activations/layer17_attention_weight_max": 54.141395568847656,
      "activations/layer17_attention_weight_min": -44.41646194458008,
      "activations/layer18_attention_weight_max": 53.26119613647461,
      "activations/layer18_attention_weight_min": -42.11479568481445,
      "activations/layer19_attention_weight_max": 25.399364471435547,
      "activations/layer19_attention_weight_min": -22.329816818237305,
      "activations/layer1_attention_weight_max": 14.68731689453125,
      "activations/layer1_attention_weight_min": -14.010843276977539,
      "activations/layer20_attention_weight_max": 25.26825523376465,
      "activations/layer20_attention_weight_min": -20.373838424682617,
      "activations/layer21_attention_weight_max": 45.43462371826172,
      "activations/layer21_attention_weight_min": -25.926525115966797,
      "activations/layer22_attention_weight_max": 32.20096969604492,
      "activations/layer22_attention_weight_min": -25.58708953857422,
      "activations/layer23_attention_weight_max": 37.595458984375,
      "activations/layer23_attention_weight_min": -22.908405303955078,
      "activations/layer2_attention_weight_max": 30.9609432220459,
      "activations/layer2_attention_weight_min": -30.594425201416016,
      "activations/layer3_attention_weight_max": 87.32755279541016,
      "activations/layer3_attention_weight_min": -87.07176971435547,
      "activations/layer4_attention_weight_max": 84.22594451904297,
      "activations/layer4_attention_weight_min": -86.01531982421875,
      "activations/layer5_attention_weight_max": 60.981475830078125,
      "activations/layer5_attention_weight_min": -71.46507263183594,
      "activations/layer6_attention_weight_max": 52.25822067260742,
      "activations/layer6_attention_weight_min": -51.37370300292969,
      "activations/layer7_attention_weight_max": 70.7693099975586,
      "activations/layer7_attention_weight_min": -70.53836059570312,
      "activations/layer8_attention_weight_max": 49.68949508666992,
      "activations/layer8_attention_weight_min": -53.139259338378906,
      "activations/layer9_attention_weight_max": 47.07725524902344,
      "activations/layer9_attention_weight_min": -48.17161560058594,
      "epoch": 22.05,
      "learning_rate": 7.898484848484848e-06,
      "loss": 2.6923,
      "step": 379450
    },
    {
      "activations/layer0_attention_weight_max": 16.390777587890625,
      "activations/layer0_attention_weight_min": -14.127821922302246,
      "activations/layer10_attention_weight_max": 34.7751350402832,
      "activations/layer10_attention_weight_min": -35.73810958862305,
      "activations/layer11_attention_weight_max": 32.49248504638672,
      "activations/layer11_attention_weight_min": -33.83499526977539,
      "activations/layer12_attention_weight_max": 18.969802856445312,
      "activations/layer12_attention_weight_min": -25.88276481628418,
      "activations/layer13_attention_weight_max": 41.29606246948242,
      "activations/layer13_attention_weight_min": -29.49111557006836,
      "activations/layer14_attention_weight_max": 35.70062255859375,
      "activations/layer14_attention_weight_min": -31.31057357788086,
      "activations/layer15_attention_weight_max": 40.31306838989258,
      "activations/layer15_attention_weight_min": -32.75008773803711,
      "activations/layer16_attention_weight_max": 31.942630767822266,
      "activations/layer16_attention_weight_min": -32.03182601928711,
      "activations/layer17_attention_weight_max": 48.34121322631836,
      "activations/layer17_attention_weight_min": -48.7006721496582,
      "activations/layer18_attention_weight_max": 43.9595832824707,
      "activations/layer18_attention_weight_min": -45.78474807739258,
      "activations/layer19_attention_weight_max": 24.609182357788086,
      "activations/layer19_attention_weight_min": -24.845426559448242,
      "activations/layer1_attention_weight_max": 15.758879661560059,
      "activations/layer1_attention_weight_min": -15.03445816040039,
      "activations/layer20_attention_weight_max": 20.863887786865234,
      "activations/layer20_attention_weight_min": -27.017473220825195,
      "activations/layer21_attention_weight_max": 44.71030807495117,
      "activations/layer21_attention_weight_min": -31.398025512695312,
      "activations/layer22_attention_weight_max": 28.32242774963379,
      "activations/layer22_attention_weight_min": -27.4520320892334,
      "activations/layer23_attention_weight_max": 38.34853744506836,
      "activations/layer23_attention_weight_min": -24.879392623901367,
      "activations/layer2_attention_weight_max": 30.560760498046875,
      "activations/layer2_attention_weight_min": -30.419979095458984,
      "activations/layer3_attention_weight_max": 88.11736297607422,
      "activations/layer3_attention_weight_min": -86.57992553710938,
      "activations/layer4_attention_weight_max": 83.33879089355469,
      "activations/layer4_attention_weight_min": -85.11251831054688,
      "activations/layer5_attention_weight_max": 61.21141052246094,
      "activations/layer5_attention_weight_min": -74.40945434570312,
      "activations/layer6_attention_weight_max": 50.05441665649414,
      "activations/layer6_attention_weight_min": -50.147544860839844,
      "activations/layer7_attention_weight_max": 69.13700103759766,
      "activations/layer7_attention_weight_min": -68.8455581665039,
      "activations/layer8_attention_weight_max": 47.62276077270508,
      "activations/layer8_attention_weight_min": -48.3758659362793,
      "activations/layer9_attention_weight_max": 46.21052169799805,
      "activations/layer9_attention_weight_min": -46.1961555480957,
      "epoch": 22.05,
      "learning_rate": 7.879545454545453e-06,
      "loss": 2.6975,
      "step": 379500
    },
    {
      "activations/layer0_attention_weight_max": 15.890402793884277,
      "activations/layer0_attention_weight_min": -14.37582015991211,
      "activations/layer10_attention_weight_max": 36.13157653808594,
      "activations/layer10_attention_weight_min": -36.55244445800781,
      "activations/layer11_attention_weight_max": 33.15663528442383,
      "activations/layer11_attention_weight_min": -35.638092041015625,
      "activations/layer12_attention_weight_max": 18.622907638549805,
      "activations/layer12_attention_weight_min": -24.8704776763916,
      "activations/layer13_attention_weight_max": 32.77696990966797,
      "activations/layer13_attention_weight_min": -26.361759185791016,
      "activations/layer14_attention_weight_max": 35.24213790893555,
      "activations/layer14_attention_weight_min": -29.520566940307617,
      "activations/layer15_attention_weight_max": 34.02272415161133,
      "activations/layer15_attention_weight_min": -29.567628860473633,
      "activations/layer16_attention_weight_max": 33.780914306640625,
      "activations/layer16_attention_weight_min": -31.45599365234375,
      "activations/layer17_attention_weight_max": 51.39118957519531,
      "activations/layer17_attention_weight_min": -43.92334747314453,
      "activations/layer18_attention_weight_max": 48.39450454711914,
      "activations/layer18_attention_weight_min": -39.51803207397461,
      "activations/layer19_attention_weight_max": 26.242298126220703,
      "activations/layer19_attention_weight_min": -22.385862350463867,
      "activations/layer1_attention_weight_max": 14.83066463470459,
      "activations/layer1_attention_weight_min": -14.027486801147461,
      "activations/layer20_attention_weight_max": 22.032346725463867,
      "activations/layer20_attention_weight_min": -22.616405487060547,
      "activations/layer21_attention_weight_max": 31.898263931274414,
      "activations/layer21_attention_weight_min": -21.386653900146484,
      "activations/layer22_attention_weight_max": 27.460237503051758,
      "activations/layer22_attention_weight_min": -25.395587921142578,
      "activations/layer23_attention_weight_max": 36.135894775390625,
      "activations/layer23_attention_weight_min": -22.707866668701172,
      "activations/layer2_attention_weight_max": 31.584308624267578,
      "activations/layer2_attention_weight_min": -30.666885375976562,
      "activations/layer3_attention_weight_max": 87.69280242919922,
      "activations/layer3_attention_weight_min": -86.64334869384766,
      "activations/layer4_attention_weight_max": 83.50664520263672,
      "activations/layer4_attention_weight_min": -83.87801361083984,
      "activations/layer5_attention_weight_max": 61.18772888183594,
      "activations/layer5_attention_weight_min": -79.14744567871094,
      "activations/layer6_attention_weight_max": 48.239139556884766,
      "activations/layer6_attention_weight_min": -50.1223030090332,
      "activations/layer7_attention_weight_max": 68.46900939941406,
      "activations/layer7_attention_weight_min": -65.80529022216797,
      "activations/layer8_attention_weight_max": 44.546913146972656,
      "activations/layer8_attention_weight_min": -50.00974655151367,
      "activations/layer9_attention_weight_max": 41.89109802246094,
      "activations/layer9_attention_weight_min": -44.84186935424805,
      "epoch": 22.05,
      "learning_rate": 7.86060606060606e-06,
      "loss": 2.693,
      "step": 379550
    },
    {
      "activations/layer0_attention_weight_max": 16.241230010986328,
      "activations/layer0_attention_weight_min": -13.766934394836426,
      "activations/layer10_attention_weight_max": 33.77195358276367,
      "activations/layer10_attention_weight_min": -34.459938049316406,
      "activations/layer11_attention_weight_max": 34.51799011230469,
      "activations/layer11_attention_weight_min": -35.59990692138672,
      "activations/layer12_attention_weight_max": 19.156892776489258,
      "activations/layer12_attention_weight_min": -27.972469329833984,
      "activations/layer13_attention_weight_max": 34.77689743041992,
      "activations/layer13_attention_weight_min": -26.237613677978516,
      "activations/layer14_attention_weight_max": 34.0070915222168,
      "activations/layer14_attention_weight_min": -29.333166122436523,
      "activations/layer15_attention_weight_max": 32.819122314453125,
      "activations/layer15_attention_weight_min": -29.51549530029297,
      "activations/layer16_attention_weight_max": 33.72871017456055,
      "activations/layer16_attention_weight_min": -32.05801010131836,
      "activations/layer17_attention_weight_max": 49.07713317871094,
      "activations/layer17_attention_weight_min": -42.76668930053711,
      "activations/layer18_attention_weight_max": 42.13919448852539,
      "activations/layer18_attention_weight_min": -37.10519790649414,
      "activations/layer19_attention_weight_max": 23.384382247924805,
      "activations/layer19_attention_weight_min": -21.224348068237305,
      "activations/layer1_attention_weight_max": 15.269463539123535,
      "activations/layer1_attention_weight_min": -14.49612045288086,
      "activations/layer20_attention_weight_max": 21.6989803314209,
      "activations/layer20_attention_weight_min": -22.768339157104492,
      "activations/layer21_attention_weight_max": 37.81019592285156,
      "activations/layer21_attention_weight_min": -24.182342529296875,
      "activations/layer22_attention_weight_max": 27.57025718688965,
      "activations/layer22_attention_weight_min": -25.75443458557129,
      "activations/layer23_attention_weight_max": 33.23881149291992,
      "activations/layer23_attention_weight_min": -24.06926727294922,
      "activations/layer2_attention_weight_max": 34.27710723876953,
      "activations/layer2_attention_weight_min": -32.93462371826172,
      "activations/layer3_attention_weight_max": 96.61390686035156,
      "activations/layer3_attention_weight_min": -92.7440414428711,
      "activations/layer4_attention_weight_max": 87.83855438232422,
      "activations/layer4_attention_weight_min": -87.3907241821289,
      "activations/layer5_attention_weight_max": 62.90848922729492,
      "activations/layer5_attention_weight_min": -74.6282730102539,
      "activations/layer6_attention_weight_max": 49.813636779785156,
      "activations/layer6_attention_weight_min": -49.796775817871094,
      "activations/layer7_attention_weight_max": 69.49268341064453,
      "activations/layer7_attention_weight_min": -68.76976013183594,
      "activations/layer8_attention_weight_max": 48.723567962646484,
      "activations/layer8_attention_weight_min": -49.76314926147461,
      "activations/layer9_attention_weight_max": 48.07062911987305,
      "activations/layer9_attention_weight_min": -47.58903121948242,
      "epoch": 22.06,
      "learning_rate": 7.842045454545454e-06,
      "loss": 2.7027,
      "step": 379600
    },
    {
      "activations/layer0_attention_weight_max": 15.39453125,
      "activations/layer0_attention_weight_min": -14.098528861999512,
      "activations/layer10_attention_weight_max": 37.337432861328125,
      "activations/layer10_attention_weight_min": -37.94923400878906,
      "activations/layer11_attention_weight_max": 33.43336486816406,
      "activations/layer11_attention_weight_min": -35.05144119262695,
      "activations/layer12_attention_weight_max": 19.38388442993164,
      "activations/layer12_attention_weight_min": -24.74496078491211,
      "activations/layer13_attention_weight_max": 36.168094635009766,
      "activations/layer13_attention_weight_min": -27.396907806396484,
      "activations/layer14_attention_weight_max": 36.417293548583984,
      "activations/layer14_attention_weight_min": -31.085203170776367,
      "activations/layer15_attention_weight_max": 33.654014587402344,
      "activations/layer15_attention_weight_min": -31.688762664794922,
      "activations/layer16_attention_weight_max": 34.845943450927734,
      "activations/layer16_attention_weight_min": -34.822147369384766,
      "activations/layer17_attention_weight_max": 51.0803108215332,
      "activations/layer17_attention_weight_min": -49.6181640625,
      "activations/layer18_attention_weight_max": 49.82769775390625,
      "activations/layer18_attention_weight_min": -42.0280647277832,
      "activations/layer19_attention_weight_max": 25.767892837524414,
      "activations/layer19_attention_weight_min": -25.739912033081055,
      "activations/layer1_attention_weight_max": 15.17769718170166,
      "activations/layer1_attention_weight_min": -14.427353858947754,
      "activations/layer20_attention_weight_max": 27.401464462280273,
      "activations/layer20_attention_weight_min": -22.09824562072754,
      "activations/layer21_attention_weight_max": 43.31941604614258,
      "activations/layer21_attention_weight_min": -24.11406135559082,
      "activations/layer22_attention_weight_max": 29.006013870239258,
      "activations/layer22_attention_weight_min": -25.39348030090332,
      "activations/layer23_attention_weight_max": 37.97357177734375,
      "activations/layer23_attention_weight_min": -24.10037612915039,
      "activations/layer2_attention_weight_max": 34.91028594970703,
      "activations/layer2_attention_weight_min": -30.25351333618164,
      "activations/layer3_attention_weight_max": 89.98590850830078,
      "activations/layer3_attention_weight_min": -88.57160949707031,
      "activations/layer4_attention_weight_max": 86.42147827148438,
      "activations/layer4_attention_weight_min": -85.34972381591797,
      "activations/layer5_attention_weight_max": 65.36920166015625,
      "activations/layer5_attention_weight_min": -71.46843719482422,
      "activations/layer6_attention_weight_max": 49.829776763916016,
      "activations/layer6_attention_weight_min": -51.39127731323242,
      "activations/layer7_attention_weight_max": 67.1935806274414,
      "activations/layer7_attention_weight_min": -70.3493423461914,
      "activations/layer8_attention_weight_max": 53.051124572753906,
      "activations/layer8_attention_weight_min": -56.12445831298828,
      "activations/layer9_attention_weight_max": 45.42600631713867,
      "activations/layer9_attention_weight_min": -48.30912780761719,
      "epoch": 22.06,
      "learning_rate": 7.82310606060606e-06,
      "loss": 2.6965,
      "step": 379650
    },
    {
      "activations/layer0_attention_weight_max": 16.175020217895508,
      "activations/layer0_attention_weight_min": -13.850849151611328,
      "activations/layer10_attention_weight_max": 34.53274917602539,
      "activations/layer10_attention_weight_min": -36.4560432434082,
      "activations/layer11_attention_weight_max": 32.117828369140625,
      "activations/layer11_attention_weight_min": -36.5201530456543,
      "activations/layer12_attention_weight_max": 18.95549201965332,
      "activations/layer12_attention_weight_min": -30.004655838012695,
      "activations/layer13_attention_weight_max": 31.068986892700195,
      "activations/layer13_attention_weight_min": -26.60435676574707,
      "activations/layer14_attention_weight_max": 34.082820892333984,
      "activations/layer14_attention_weight_min": -31.007492065429688,
      "activations/layer15_attention_weight_max": 34.27244186401367,
      "activations/layer15_attention_weight_min": -32.96735382080078,
      "activations/layer16_attention_weight_max": 35.759254455566406,
      "activations/layer16_attention_weight_min": -34.31922149658203,
      "activations/layer17_attention_weight_max": 54.779075622558594,
      "activations/layer17_attention_weight_min": -50.8026008605957,
      "activations/layer18_attention_weight_max": 52.341094970703125,
      "activations/layer18_attention_weight_min": -44.24579620361328,
      "activations/layer19_attention_weight_max": 25.42752456665039,
      "activations/layer19_attention_weight_min": -26.49529457092285,
      "activations/layer1_attention_weight_max": 15.419718742370605,
      "activations/layer1_attention_weight_min": -13.549206733703613,
      "activations/layer20_attention_weight_max": 22.817296981811523,
      "activations/layer20_attention_weight_min": -24.704479217529297,
      "activations/layer21_attention_weight_max": 32.90509033203125,
      "activations/layer21_attention_weight_min": -22.929428100585938,
      "activations/layer22_attention_weight_max": 28.53648567199707,
      "activations/layer22_attention_weight_min": -27.91437530517578,
      "activations/layer23_attention_weight_max": 35.8752555847168,
      "activations/layer23_attention_weight_min": -23.996063232421875,
      "activations/layer2_attention_weight_max": 30.456266403198242,
      "activations/layer2_attention_weight_min": -29.269309997558594,
      "activations/layer3_attention_weight_max": 88.20668029785156,
      "activations/layer3_attention_weight_min": -87.18658447265625,
      "activations/layer4_attention_weight_max": 83.2341079711914,
      "activations/layer4_attention_weight_min": -84.06388854980469,
      "activations/layer5_attention_weight_max": 61.464962005615234,
      "activations/layer5_attention_weight_min": -72.15494537353516,
      "activations/layer6_attention_weight_max": 49.26137924194336,
      "activations/layer6_attention_weight_min": -51.88591384887695,
      "activations/layer7_attention_weight_max": 74.12104797363281,
      "activations/layer7_attention_weight_min": -71.1534194946289,
      "activations/layer8_attention_weight_max": 45.41133117675781,
      "activations/layer8_attention_weight_min": -50.140708923339844,
      "activations/layer9_attention_weight_max": 41.482852935791016,
      "activations/layer9_attention_weight_min": -46.084625244140625,
      "epoch": 22.06,
      "learning_rate": 7.804166666666665e-06,
      "loss": 2.6958,
      "step": 379700
    },
    {
      "activations/layer0_attention_weight_max": 16.7172908782959,
      "activations/layer0_attention_weight_min": -15.436064720153809,
      "activations/layer10_attention_weight_max": 35.338722229003906,
      "activations/layer10_attention_weight_min": -37.671424865722656,
      "activations/layer11_attention_weight_max": 33.26012420654297,
      "activations/layer11_attention_weight_min": -34.56085205078125,
      "activations/layer12_attention_weight_max": 20.179895401000977,
      "activations/layer12_attention_weight_min": -21.452585220336914,
      "activations/layer13_attention_weight_max": 34.964412689208984,
      "activations/layer13_attention_weight_min": -28.879961013793945,
      "activations/layer14_attention_weight_max": 34.9620361328125,
      "activations/layer14_attention_weight_min": -28.865449905395508,
      "activations/layer15_attention_weight_max": 34.65190505981445,
      "activations/layer15_attention_weight_min": -30.687238693237305,
      "activations/layer16_attention_weight_max": 37.33747100830078,
      "activations/layer16_attention_weight_min": -34.36857986450195,
      "activations/layer17_attention_weight_max": 58.29686737060547,
      "activations/layer17_attention_weight_min": -48.577598571777344,
      "activations/layer18_attention_weight_max": 50.719905853271484,
      "activations/layer18_attention_weight_min": -42.85226821899414,
      "activations/layer19_attention_weight_max": 25.751815795898438,
      "activations/layer19_attention_weight_min": -22.78546714782715,
      "activations/layer1_attention_weight_max": 14.574902534484863,
      "activations/layer1_attention_weight_min": -13.684353828430176,
      "activations/layer20_attention_weight_max": 24.253437042236328,
      "activations/layer20_attention_weight_min": -22.196855545043945,
      "activations/layer21_attention_weight_max": 36.21365737915039,
      "activations/layer21_attention_weight_min": -22.105459213256836,
      "activations/layer22_attention_weight_max": 31.477706909179688,
      "activations/layer22_attention_weight_min": -26.16845703125,
      "activations/layer23_attention_weight_max": 34.930850982666016,
      "activations/layer23_attention_weight_min": -25.190593719482422,
      "activations/layer2_attention_weight_max": 30.570392608642578,
      "activations/layer2_attention_weight_min": -28.968299865722656,
      "activations/layer3_attention_weight_max": 84.0907211303711,
      "activations/layer3_attention_weight_min": -84.37060546875,
      "activations/layer4_attention_weight_max": 79.9901123046875,
      "activations/layer4_attention_weight_min": -82.8255844116211,
      "activations/layer5_attention_weight_max": 61.14165496826172,
      "activations/layer5_attention_weight_min": -73.53011322021484,
      "activations/layer6_attention_weight_max": 48.703067779541016,
      "activations/layer6_attention_weight_min": -49.82257843017578,
      "activations/layer7_attention_weight_max": 64.80542755126953,
      "activations/layer7_attention_weight_min": -67.78611755371094,
      "activations/layer8_attention_weight_max": 44.70457458496094,
      "activations/layer8_attention_weight_min": -50.192466735839844,
      "activations/layer9_attention_weight_max": 42.79196548461914,
      "activations/layer9_attention_weight_min": -45.695594787597656,
      "epoch": 22.07,
      "learning_rate": 7.785227272727272e-06,
      "loss": 2.7091,
      "step": 379750
    },
    {
      "activations/layer0_attention_weight_max": 15.990431785583496,
      "activations/layer0_attention_weight_min": -14.38221549987793,
      "activations/layer10_attention_weight_max": 36.6326789855957,
      "activations/layer10_attention_weight_min": -37.35038757324219,
      "activations/layer11_attention_weight_max": 33.57811737060547,
      "activations/layer11_attention_weight_min": -34.77912139892578,
      "activations/layer12_attention_weight_max": 20.376779556274414,
      "activations/layer12_attention_weight_min": -28.49736976623535,
      "activations/layer13_attention_weight_max": 34.106746673583984,
      "activations/layer13_attention_weight_min": -32.461021423339844,
      "activations/layer14_attention_weight_max": 37.168357849121094,
      "activations/layer14_attention_weight_min": -32.35179901123047,
      "activations/layer15_attention_weight_max": 37.74705123901367,
      "activations/layer15_attention_weight_min": -32.4594841003418,
      "activations/layer16_attention_weight_max": 37.10005569458008,
      "activations/layer16_attention_weight_min": -34.25362777709961,
      "activations/layer17_attention_weight_max": 54.59800338745117,
      "activations/layer17_attention_weight_min": -50.464454650878906,
      "activations/layer18_attention_weight_max": 50.05392837524414,
      "activations/layer18_attention_weight_min": -46.64188766479492,
      "activations/layer19_attention_weight_max": 25.15818977355957,
      "activations/layer19_attention_weight_min": -25.054805755615234,
      "activations/layer1_attention_weight_max": 15.133172035217285,
      "activations/layer1_attention_weight_min": -13.252857208251953,
      "activations/layer20_attention_weight_max": 23.804431915283203,
      "activations/layer20_attention_weight_min": -24.16224479675293,
      "activations/layer21_attention_weight_max": 40.43010711669922,
      "activations/layer21_attention_weight_min": -26.42320442199707,
      "activations/layer22_attention_weight_max": 31.462081909179688,
      "activations/layer22_attention_weight_min": -27.231111526489258,
      "activations/layer23_attention_weight_max": 40.4288215637207,
      "activations/layer23_attention_weight_min": -25.80830955505371,
      "activations/layer2_attention_weight_max": 31.63058853149414,
      "activations/layer2_attention_weight_min": -30.46442222595215,
      "activations/layer3_attention_weight_max": 87.41910552978516,
      "activations/layer3_attention_weight_min": -88.32577514648438,
      "activations/layer4_attention_weight_max": 81.95674896240234,
      "activations/layer4_attention_weight_min": -85.19287872314453,
      "activations/layer5_attention_weight_max": 65.21775817871094,
      "activations/layer5_attention_weight_min": -70.8587417602539,
      "activations/layer6_attention_weight_max": 48.70134353637695,
      "activations/layer6_attention_weight_min": -51.21528625488281,
      "activations/layer7_attention_weight_max": 65.8433609008789,
      "activations/layer7_attention_weight_min": -67.95706939697266,
      "activations/layer8_attention_weight_max": 45.75676345825195,
      "activations/layer8_attention_weight_min": -50.55251693725586,
      "activations/layer9_attention_weight_max": 43.79270553588867,
      "activations/layer9_attention_weight_min": -46.860477447509766,
      "epoch": 22.07,
      "learning_rate": 7.766287878787877e-06,
      "loss": 2.7004,
      "step": 379800
    },
    {
      "activations/layer0_attention_weight_max": 15.76705551147461,
      "activations/layer0_attention_weight_min": -14.104796409606934,
      "activations/layer10_attention_weight_max": 39.27337646484375,
      "activations/layer10_attention_weight_min": -37.45707702636719,
      "activations/layer11_attention_weight_max": 38.42551803588867,
      "activations/layer11_attention_weight_min": -39.56874084472656,
      "activations/layer12_attention_weight_max": 19.60235023498535,
      "activations/layer12_attention_weight_min": -26.44574737548828,
      "activations/layer13_attention_weight_max": 33.42942810058594,
      "activations/layer13_attention_weight_min": -32.22214889526367,
      "activations/layer14_attention_weight_max": 34.25971984863281,
      "activations/layer14_attention_weight_min": -31.374427795410156,
      "activations/layer15_attention_weight_max": 34.09832763671875,
      "activations/layer15_attention_weight_min": -32.11442184448242,
      "activations/layer16_attention_weight_max": 34.513607025146484,
      "activations/layer16_attention_weight_min": -32.99952697753906,
      "activations/layer17_attention_weight_max": 50.63329315185547,
      "activations/layer17_attention_weight_min": -45.43395233154297,
      "activations/layer18_attention_weight_max": 49.68258285522461,
      "activations/layer18_attention_weight_min": -41.94835662841797,
      "activations/layer19_attention_weight_max": 25.687255859375,
      "activations/layer19_attention_weight_min": -24.05385398864746,
      "activations/layer1_attention_weight_max": 15.277329444885254,
      "activations/layer1_attention_weight_min": -14.261853218078613,
      "activations/layer20_attention_weight_max": 25.2902774810791,
      "activations/layer20_attention_weight_min": -22.052812576293945,
      "activations/layer21_attention_weight_max": 38.9578742980957,
      "activations/layer21_attention_weight_min": -22.491561889648438,
      "activations/layer22_attention_weight_max": 32.56169128417969,
      "activations/layer22_attention_weight_min": -24.750640869140625,
      "activations/layer23_attention_weight_max": 37.27935791015625,
      "activations/layer23_attention_weight_min": -25.54635238647461,
      "activations/layer2_attention_weight_max": 32.73658752441406,
      "activations/layer2_attention_weight_min": -30.74599266052246,
      "activations/layer3_attention_weight_max": 92.50203704833984,
      "activations/layer3_attention_weight_min": -91.5996322631836,
      "activations/layer4_attention_weight_max": 87.89632415771484,
      "activations/layer4_attention_weight_min": -88.64356231689453,
      "activations/layer5_attention_weight_max": 65.05928802490234,
      "activations/layer5_attention_weight_min": -79.52613830566406,
      "activations/layer6_attention_weight_max": 51.238407135009766,
      "activations/layer6_attention_weight_min": -52.15837860107422,
      "activations/layer7_attention_weight_max": 73.25013732910156,
      "activations/layer7_attention_weight_min": -68.71273040771484,
      "activations/layer8_attention_weight_max": 48.642391204833984,
      "activations/layer8_attention_weight_min": -51.079429626464844,
      "activations/layer9_attention_weight_max": 45.448421478271484,
      "activations/layer9_attention_weight_min": -49.53580093383789,
      "epoch": 22.07,
      "learning_rate": 7.747348484848484e-06,
      "loss": 2.7107,
      "step": 379850
    },
    {
      "activations/layer0_attention_weight_max": 17.705312728881836,
      "activations/layer0_attention_weight_min": -14.17641830444336,
      "activations/layer10_attention_weight_max": 40.40567398071289,
      "activations/layer10_attention_weight_min": -37.802093505859375,
      "activations/layer11_attention_weight_max": 37.03325653076172,
      "activations/layer11_attention_weight_min": -38.01839065551758,
      "activations/layer12_attention_weight_max": 19.800466537475586,
      "activations/layer12_attention_weight_min": -23.498695373535156,
      "activations/layer13_attention_weight_max": 38.081298828125,
      "activations/layer13_attention_weight_min": -28.571857452392578,
      "activations/layer14_attention_weight_max": 36.48181915283203,
      "activations/layer14_attention_weight_min": -31.49596405029297,
      "activations/layer15_attention_weight_max": 35.704715728759766,
      "activations/layer15_attention_weight_min": -31.437719345092773,
      "activations/layer16_attention_weight_max": 37.08148193359375,
      "activations/layer16_attention_weight_min": -34.993648529052734,
      "activations/layer17_attention_weight_max": 52.6431884765625,
      "activations/layer17_attention_weight_min": -46.30046463012695,
      "activations/layer18_attention_weight_max": 48.90277099609375,
      "activations/layer18_attention_weight_min": -42.480018615722656,
      "activations/layer19_attention_weight_max": 25.789295196533203,
      "activations/layer19_attention_weight_min": -24.151887893676758,
      "activations/layer1_attention_weight_max": 15.617178916931152,
      "activations/layer1_attention_weight_min": -15.410201072692871,
      "activations/layer20_attention_weight_max": 24.2186336517334,
      "activations/layer20_attention_weight_min": -22.59380340576172,
      "activations/layer21_attention_weight_max": 33.54365921020508,
      "activations/layer21_attention_weight_min": -21.382673263549805,
      "activations/layer22_attention_weight_max": 28.414588928222656,
      "activations/layer22_attention_weight_min": -27.10637092590332,
      "activations/layer23_attention_weight_max": 35.272254943847656,
      "activations/layer23_attention_weight_min": -23.98591423034668,
      "activations/layer2_attention_weight_max": 31.59830093383789,
      "activations/layer2_attention_weight_min": -31.234128952026367,
      "activations/layer3_attention_weight_max": 86.80111694335938,
      "activations/layer3_attention_weight_min": -90.81426239013672,
      "activations/layer4_attention_weight_max": 84.79918670654297,
      "activations/layer4_attention_weight_min": -85.60578155517578,
      "activations/layer5_attention_weight_max": 64.54104614257812,
      "activations/layer5_attention_weight_min": -71.04095458984375,
      "activations/layer6_attention_weight_max": 51.29473114013672,
      "activations/layer6_attention_weight_min": -52.07601547241211,
      "activations/layer7_attention_weight_max": 72.88048553466797,
      "activations/layer7_attention_weight_min": -72.53607940673828,
      "activations/layer8_attention_weight_max": 50.909706115722656,
      "activations/layer8_attention_weight_min": -52.4653205871582,
      "activations/layer9_attention_weight_max": 50.10336685180664,
      "activations/layer9_attention_weight_min": -50.396427154541016,
      "epoch": 22.07,
      "learning_rate": 7.72840909090909e-06,
      "loss": 2.7041,
      "step": 379900
    },
    {
      "activations/layer0_attention_weight_max": 16.888347625732422,
      "activations/layer0_attention_weight_min": -13.909989356994629,
      "activations/layer10_attention_weight_max": 41.6412239074707,
      "activations/layer10_attention_weight_min": -37.865116119384766,
      "activations/layer11_attention_weight_max": 41.42163848876953,
      "activations/layer11_attention_weight_min": -36.65947723388672,
      "activations/layer12_attention_weight_max": 20.37183380126953,
      "activations/layer12_attention_weight_min": -22.971786499023438,
      "activations/layer13_attention_weight_max": 57.468727111816406,
      "activations/layer13_attention_weight_min": -27.560564041137695,
      "activations/layer14_attention_weight_max": 60.24216079711914,
      "activations/layer14_attention_weight_min": -30.357051849365234,
      "activations/layer15_attention_weight_max": 53.752174377441406,
      "activations/layer15_attention_weight_min": -31.32672691345215,
      "activations/layer16_attention_weight_max": 36.264198303222656,
      "activations/layer16_attention_weight_min": -32.6070671081543,
      "activations/layer17_attention_weight_max": 57.64204406738281,
      "activations/layer17_attention_weight_min": -49.299530029296875,
      "activations/layer18_attention_weight_max": 58.83283996582031,
      "activations/layer18_attention_weight_min": -45.863704681396484,
      "activations/layer19_attention_weight_max": 33.11860275268555,
      "activations/layer19_attention_weight_min": -27.136337280273438,
      "activations/layer1_attention_weight_max": 14.945146560668945,
      "activations/layer1_attention_weight_min": -13.931221008300781,
      "activations/layer20_attention_weight_max": 33.82914733886719,
      "activations/layer20_attention_weight_min": -24.52996063232422,
      "activations/layer21_attention_weight_max": 56.51426696777344,
      "activations/layer21_attention_weight_min": -29.93267059326172,
      "activations/layer22_attention_weight_max": 36.59626770019531,
      "activations/layer22_attention_weight_min": -27.286001205444336,
      "activations/layer23_attention_weight_max": 40.930274963378906,
      "activations/layer23_attention_weight_min": -25.080724716186523,
      "activations/layer2_attention_weight_max": 31.932655334472656,
      "activations/layer2_attention_weight_min": -31.64901351928711,
      "activations/layer3_attention_weight_max": 91.0841064453125,
      "activations/layer3_attention_weight_min": -92.6076889038086,
      "activations/layer4_attention_weight_max": 88.47161102294922,
      "activations/layer4_attention_weight_min": -86.26107025146484,
      "activations/layer5_attention_weight_max": 63.381134033203125,
      "activations/layer5_attention_weight_min": -71.51907348632812,
      "activations/layer6_attention_weight_max": 51.9450798034668,
      "activations/layer6_attention_weight_min": -52.342857360839844,
      "activations/layer7_attention_weight_max": 75.78653717041016,
      "activations/layer7_attention_weight_min": -68.03495788574219,
      "activations/layer8_attention_weight_max": 53.334083557128906,
      "activations/layer8_attention_weight_min": -52.080177307128906,
      "activations/layer9_attention_weight_max": 50.06819152832031,
      "activations/layer9_attention_weight_min": -47.4495735168457,
      "epoch": 22.08,
      "learning_rate": 7.709469696969697e-06,
      "loss": 2.7004,
      "step": 379950
    },
    {
      "activations/layer0_attention_weight_max": 16.79357147216797,
      "activations/layer0_attention_weight_min": -14.188210487365723,
      "activations/layer10_attention_weight_max": 33.757240295410156,
      "activations/layer10_attention_weight_min": -34.097015380859375,
      "activations/layer11_attention_weight_max": 32.23172378540039,
      "activations/layer11_attention_weight_min": -33.68299865722656,
      "activations/layer12_attention_weight_max": 19.285192489624023,
      "activations/layer12_attention_weight_min": -24.13223648071289,
      "activations/layer13_attention_weight_max": 28.84654426574707,
      "activations/layer13_attention_weight_min": -26.128604888916016,
      "activations/layer14_attention_weight_max": 34.318145751953125,
      "activations/layer14_attention_weight_min": -28.98357582092285,
      "activations/layer15_attention_weight_max": 32.050811767578125,
      "activations/layer15_attention_weight_min": -29.433177947998047,
      "activations/layer16_attention_weight_max": 34.36285400390625,
      "activations/layer16_attention_weight_min": -32.06568145751953,
      "activations/layer17_attention_weight_max": 49.62587356567383,
      "activations/layer17_attention_weight_min": -43.8819465637207,
      "activations/layer18_attention_weight_max": 46.369327545166016,
      "activations/layer18_attention_weight_min": -39.42755126953125,
      "activations/layer19_attention_weight_max": 22.86634635925293,
      "activations/layer19_attention_weight_min": -22.47681999206543,
      "activations/layer1_attention_weight_max": 15.306418418884277,
      "activations/layer1_attention_weight_min": -15.366927146911621,
      "activations/layer20_attention_weight_max": 20.959348678588867,
      "activations/layer20_attention_weight_min": -25.532743453979492,
      "activations/layer21_attention_weight_max": 32.89407730102539,
      "activations/layer21_attention_weight_min": -19.819055557250977,
      "activations/layer22_attention_weight_max": 28.019489288330078,
      "activations/layer22_attention_weight_min": -24.903514862060547,
      "activations/layer23_attention_weight_max": 35.56673049926758,
      "activations/layer23_attention_weight_min": -22.686161041259766,
      "activations/layer2_attention_weight_max": 33.614295959472656,
      "activations/layer2_attention_weight_min": -31.002063751220703,
      "activations/layer3_attention_weight_max": 88.7789306640625,
      "activations/layer3_attention_weight_min": -88.13448333740234,
      "activations/layer4_attention_weight_max": 86.92182922363281,
      "activations/layer4_attention_weight_min": -87.46721649169922,
      "activations/layer5_attention_weight_max": 63.99317932128906,
      "activations/layer5_attention_weight_min": -73.88710021972656,
      "activations/layer6_attention_weight_max": 48.42020034790039,
      "activations/layer6_attention_weight_min": -48.62929153442383,
      "activations/layer7_attention_weight_max": 65.89175415039062,
      "activations/layer7_attention_weight_min": -66.34453582763672,
      "activations/layer8_attention_weight_max": 45.07380294799805,
      "activations/layer8_attention_weight_min": -47.523345947265625,
      "activations/layer9_attention_weight_max": 40.953453063964844,
      "activations/layer9_attention_weight_min": -43.397953033447266,
      "epoch": 22.08,
      "learning_rate": 7.690530303030303e-06,
      "loss": 2.7152,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.517,
      "eval_samples_per_second": 504.167,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.517,
      "eval_openwebtext_samples_per_second": 504.167,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9482,
      "eval_wikitext_samples_per_second": 234.068,
      "step": 380000
    },
    {
      "epoch": 22.08,
      "eval_lambada_loss": 2.38671875,
      "eval_lambada_ppl": 10.877742727456024,
      "eval_lambada_runtime": 9.5738,
      "eval_lambada_samples_per_second": 508.573,
      "step": 380000
    },
    {
      "activations/layer0_attention_weight_max": 16.369108200073242,
      "activations/layer0_attention_weight_min": -14.0360746383667,
      "activations/layer10_attention_weight_max": 37.60886764526367,
      "activations/layer10_attention_weight_min": -36.89380645751953,
      "activations/layer11_attention_weight_max": 36.333595275878906,
      "activations/layer11_attention_weight_min": -38.17041015625,
      "activations/layer12_attention_weight_max": 22.574581146240234,
      "activations/layer12_attention_weight_min": -23.646732330322266,
      "activations/layer13_attention_weight_max": 36.578216552734375,
      "activations/layer13_attention_weight_min": -29.152833938598633,
      "activations/layer14_attention_weight_max": 40.019386291503906,
      "activations/layer14_attention_weight_min": -34.951656341552734,
      "activations/layer15_attention_weight_max": 39.43930435180664,
      "activations/layer15_attention_weight_min": -34.69397735595703,
      "activations/layer16_attention_weight_max": 38.939998626708984,
      "activations/layer16_attention_weight_min": -35.75917053222656,
      "activations/layer17_attention_weight_max": 54.4661865234375,
      "activations/layer17_attention_weight_min": -48.3293342590332,
      "activations/layer18_attention_weight_max": 54.84095001220703,
      "activations/layer18_attention_weight_min": -43.451534271240234,
      "activations/layer19_attention_weight_max": 27.299163818359375,
      "activations/layer19_attention_weight_min": -25.8674373626709,
      "activations/layer1_attention_weight_max": 15.094793319702148,
      "activations/layer1_attention_weight_min": -15.441835403442383,
      "activations/layer20_attention_weight_max": 25.193479537963867,
      "activations/layer20_attention_weight_min": -22.009418487548828,
      "activations/layer21_attention_weight_max": 40.295318603515625,
      "activations/layer21_attention_weight_min": -23.67792320251465,
      "activations/layer22_attention_weight_max": 29.37221336364746,
      "activations/layer22_attention_weight_min": -25.664466857910156,
      "activations/layer23_attention_weight_max": 38.14073944091797,
      "activations/layer23_attention_weight_min": -24.841838836669922,
      "activations/layer2_attention_weight_max": 30.36549949645996,
      "activations/layer2_attention_weight_min": -30.771215438842773,
      "activations/layer3_attention_weight_max": 87.24422454833984,
      "activations/layer3_attention_weight_min": -86.5218505859375,
      "activations/layer4_attention_weight_max": 81.5395278930664,
      "activations/layer4_attention_weight_min": -85.82775115966797,
      "activations/layer5_attention_weight_max": 62.26924133300781,
      "activations/layer5_attention_weight_min": -71.58207702636719,
      "activations/layer6_attention_weight_max": 50.815311431884766,
      "activations/layer6_attention_weight_min": -53.128726959228516,
      "activations/layer7_attention_weight_max": 69.34788513183594,
      "activations/layer7_attention_weight_min": -68.88716888427734,
      "activations/layer8_attention_weight_max": 48.43168640136719,
      "activations/layer8_attention_weight_min": -50.42523956298828,
      "activations/layer9_attention_weight_max": 45.35744857788086,
      "activations/layer9_attention_weight_min": -48.14374542236328,
      "epoch": 22.08,
      "learning_rate": 7.671590909090908e-06,
      "loss": 2.696,
      "step": 380050
    },
    {
      "activations/layer0_attention_weight_max": 17.936681747436523,
      "activations/layer0_attention_weight_min": -14.254154205322266,
      "activations/layer10_attention_weight_max": 36.2741584777832,
      "activations/layer10_attention_weight_min": -36.819236755371094,
      "activations/layer11_attention_weight_max": 34.73583984375,
      "activations/layer11_attention_weight_min": -35.715301513671875,
      "activations/layer12_attention_weight_max": 20.12679100036621,
      "activations/layer12_attention_weight_min": -26.4135799407959,
      "activations/layer13_attention_weight_max": 35.472930908203125,
      "activations/layer13_attention_weight_min": -26.18021011352539,
      "activations/layer14_attention_weight_max": 34.71613311767578,
      "activations/layer14_attention_weight_min": -30.295541763305664,
      "activations/layer15_attention_weight_max": 36.295589447021484,
      "activations/layer15_attention_weight_min": -30.70687484741211,
      "activations/layer16_attention_weight_max": 37.17095184326172,
      "activations/layer16_attention_weight_min": -34.06344985961914,
      "activations/layer17_attention_weight_max": 51.78651428222656,
      "activations/layer17_attention_weight_min": -45.70055389404297,
      "activations/layer18_attention_weight_max": 49.550045013427734,
      "activations/layer18_attention_weight_min": -42.28256607055664,
      "activations/layer19_attention_weight_max": 25.701175689697266,
      "activations/layer19_attention_weight_min": -25.092485427856445,
      "activations/layer1_attention_weight_max": 14.991786003112793,
      "activations/layer1_attention_weight_min": -14.942079544067383,
      "activations/layer20_attention_weight_max": 25.261430740356445,
      "activations/layer20_attention_weight_min": -22.858407974243164,
      "activations/layer21_attention_weight_max": 41.511844635009766,
      "activations/layer21_attention_weight_min": -22.817399978637695,
      "activations/layer22_attention_weight_max": 28.86317253112793,
      "activations/layer22_attention_weight_min": -25.177997589111328,
      "activations/layer23_attention_weight_max": 35.60023880004883,
      "activations/layer23_attention_weight_min": -23.14862823486328,
      "activations/layer2_attention_weight_max": 35.2126350402832,
      "activations/layer2_attention_weight_min": -31.25798797607422,
      "activations/layer3_attention_weight_max": 96.9283447265625,
      "activations/layer3_attention_weight_min": -91.83169555664062,
      "activations/layer4_attention_weight_max": 84.69019317626953,
      "activations/layer4_attention_weight_min": -83.98778533935547,
      "activations/layer5_attention_weight_max": 59.91302490234375,
      "activations/layer5_attention_weight_min": -68.10792541503906,
      "activations/layer6_attention_weight_max": 49.18669128417969,
      "activations/layer6_attention_weight_min": -52.12504959106445,
      "activations/layer7_attention_weight_max": 66.60375213623047,
      "activations/layer7_attention_weight_min": -70.37345123291016,
      "activations/layer8_attention_weight_max": 47.388710021972656,
      "activations/layer8_attention_weight_min": -48.76607131958008,
      "activations/layer9_attention_weight_max": 47.18362808227539,
      "activations/layer9_attention_weight_min": -46.923343658447266,
      "epoch": 22.09,
      "learning_rate": 7.652651515151515e-06,
      "loss": 2.7002,
      "step": 380100
    },
    {
      "activations/layer0_attention_weight_max": 16.402450561523438,
      "activations/layer0_attention_weight_min": -13.985212326049805,
      "activations/layer10_attention_weight_max": 35.57390213012695,
      "activations/layer10_attention_weight_min": -35.86109161376953,
      "activations/layer11_attention_weight_max": 34.47499084472656,
      "activations/layer11_attention_weight_min": -36.400882720947266,
      "activations/layer12_attention_weight_max": 19.320877075195312,
      "activations/layer12_attention_weight_min": -28.333236694335938,
      "activations/layer13_attention_weight_max": 34.47034454345703,
      "activations/layer13_attention_weight_min": -27.673057556152344,
      "activations/layer14_attention_weight_max": 39.01836395263672,
      "activations/layer14_attention_weight_min": -31.359323501586914,
      "activations/layer15_attention_weight_max": 33.90488815307617,
      "activations/layer15_attention_weight_min": -30.68840980529785,
      "activations/layer16_attention_weight_max": 34.937034606933594,
      "activations/layer16_attention_weight_min": -32.5540657043457,
      "activations/layer17_attention_weight_max": 52.04982376098633,
      "activations/layer17_attention_weight_min": -46.683380126953125,
      "activations/layer18_attention_weight_max": 46.34336853027344,
      "activations/layer18_attention_weight_min": -42.358638763427734,
      "activations/layer19_attention_weight_max": 25.17704963684082,
      "activations/layer19_attention_weight_min": -24.097187042236328,
      "activations/layer1_attention_weight_max": 14.646340370178223,
      "activations/layer1_attention_weight_min": -13.5215482711792,
      "activations/layer20_attention_weight_max": 22.21331024169922,
      "activations/layer20_attention_weight_min": -22.35738182067871,
      "activations/layer21_attention_weight_max": 39.26285171508789,
      "activations/layer21_attention_weight_min": -25.48723030090332,
      "activations/layer22_attention_weight_max": 29.16963005065918,
      "activations/layer22_attention_weight_min": -24.026714324951172,
      "activations/layer23_attention_weight_max": 35.803306579589844,
      "activations/layer23_attention_weight_min": -22.39468002319336,
      "activations/layer2_attention_weight_max": 34.096099853515625,
      "activations/layer2_attention_weight_min": -32.39562225341797,
      "activations/layer3_attention_weight_max": 88.3185806274414,
      "activations/layer3_attention_weight_min": -90.65452575683594,
      "activations/layer4_attention_weight_max": 84.5868148803711,
      "activations/layer4_attention_weight_min": -85.60569763183594,
      "activations/layer5_attention_weight_max": 61.830928802490234,
      "activations/layer5_attention_weight_min": -71.90765380859375,
      "activations/layer6_attention_weight_max": 48.78828811645508,
      "activations/layer6_attention_weight_min": -51.37496566772461,
      "activations/layer7_attention_weight_max": 75.50284576416016,
      "activations/layer7_attention_weight_min": -63.93519973754883,
      "activations/layer8_attention_weight_max": 46.79949951171875,
      "activations/layer8_attention_weight_min": -48.59706497192383,
      "activations/layer9_attention_weight_max": 49.59360885620117,
      "activations/layer9_attention_weight_min": -43.10541915893555,
      "epoch": 22.09,
      "learning_rate": 7.63371212121212e-06,
      "loss": 2.7052,
      "step": 380150
    },
    {
      "activations/layer0_attention_weight_max": 15.70032024383545,
      "activations/layer0_attention_weight_min": -14.111379623413086,
      "activations/layer10_attention_weight_max": 35.251502990722656,
      "activations/layer10_attention_weight_min": -34.84856414794922,
      "activations/layer11_attention_weight_max": 31.61835289001465,
      "activations/layer11_attention_weight_min": -33.483970642089844,
      "activations/layer12_attention_weight_max": 19.541467666625977,
      "activations/layer12_attention_weight_min": -27.231678009033203,
      "activations/layer13_attention_weight_max": 33.49531936645508,
      "activations/layer13_attention_weight_min": -27.556476593017578,
      "activations/layer14_attention_weight_max": 33.12004470825195,
      "activations/layer14_attention_weight_min": -28.8655948638916,
      "activations/layer15_attention_weight_max": 30.55329132080078,
      "activations/layer15_attention_weight_min": -30.100507736206055,
      "activations/layer16_attention_weight_max": 32.659488677978516,
      "activations/layer16_attention_weight_min": -31.671064376831055,
      "activations/layer17_attention_weight_max": 47.68849182128906,
      "activations/layer17_attention_weight_min": -44.26364517211914,
      "activations/layer18_attention_weight_max": 42.57207107543945,
      "activations/layer18_attention_weight_min": -40.59714126586914,
      "activations/layer19_attention_weight_max": 22.759328842163086,
      "activations/layer19_attention_weight_min": -22.802480697631836,
      "activations/layer1_attention_weight_max": 15.185839653015137,
      "activations/layer1_attention_weight_min": -14.035842895507812,
      "activations/layer20_attention_weight_max": 19.755245208740234,
      "activations/layer20_attention_weight_min": -21.010038375854492,
      "activations/layer21_attention_weight_max": 29.26802635192871,
      "activations/layer21_attention_weight_min": -21.668319702148438,
      "activations/layer22_attention_weight_max": 28.090913772583008,
      "activations/layer22_attention_weight_min": -29.636478424072266,
      "activations/layer23_attention_weight_max": 34.37367248535156,
      "activations/layer23_attention_weight_min": -26.883481979370117,
      "activations/layer2_attention_weight_max": 30.415695190429688,
      "activations/layer2_attention_weight_min": -29.441984176635742,
      "activations/layer3_attention_weight_max": 89.0893783569336,
      "activations/layer3_attention_weight_min": -87.40111541748047,
      "activations/layer4_attention_weight_max": 82.33822631835938,
      "activations/layer4_attention_weight_min": -81.83390045166016,
      "activations/layer5_attention_weight_max": 62.42356872558594,
      "activations/layer5_attention_weight_min": -69.92437744140625,
      "activations/layer6_attention_weight_max": 46.28317642211914,
      "activations/layer6_attention_weight_min": -48.63735580444336,
      "activations/layer7_attention_weight_max": 68.9464340209961,
      "activations/layer7_attention_weight_min": -63.569969177246094,
      "activations/layer8_attention_weight_max": 47.14842224121094,
      "activations/layer8_attention_weight_min": -48.31565856933594,
      "activations/layer9_attention_weight_max": 42.875946044921875,
      "activations/layer9_attention_weight_min": -43.20396423339844,
      "epoch": 22.09,
      "learning_rate": 7.614772727272726e-06,
      "loss": 2.684,
      "step": 380200
    },
    {
      "activations/layer0_attention_weight_max": 16.157024383544922,
      "activations/layer0_attention_weight_min": -14.205120086669922,
      "activations/layer10_attention_weight_max": 39.14015579223633,
      "activations/layer10_attention_weight_min": -37.37740707397461,
      "activations/layer11_attention_weight_max": 37.15106964111328,
      "activations/layer11_attention_weight_min": -37.0047492980957,
      "activations/layer12_attention_weight_max": 19.443880081176758,
      "activations/layer12_attention_weight_min": -25.69767189025879,
      "activations/layer13_attention_weight_max": 31.568317413330078,
      "activations/layer13_attention_weight_min": -27.450830459594727,
      "activations/layer14_attention_weight_max": 34.08256530761719,
      "activations/layer14_attention_weight_min": -31.859100341796875,
      "activations/layer15_attention_weight_max": 32.91238784790039,
      "activations/layer15_attention_weight_min": -31.335824966430664,
      "activations/layer16_attention_weight_max": 34.60514450073242,
      "activations/layer16_attention_weight_min": -32.583351135253906,
      "activations/layer17_attention_weight_max": 50.314979553222656,
      "activations/layer17_attention_weight_min": -45.28199005126953,
      "activations/layer18_attention_weight_max": 47.749813079833984,
      "activations/layer18_attention_weight_min": -42.35222244262695,
      "activations/layer19_attention_weight_max": 23.678346633911133,
      "activations/layer19_attention_weight_min": -24.511999130249023,
      "activations/layer1_attention_weight_max": 15.028646469116211,
      "activations/layer1_attention_weight_min": -15.537753105163574,
      "activations/layer20_attention_weight_max": 22.56963539123535,
      "activations/layer20_attention_weight_min": -24.413545608520508,
      "activations/layer21_attention_weight_max": 34.57562255859375,
      "activations/layer21_attention_weight_min": -22.413850784301758,
      "activations/layer22_attention_weight_max": 27.1251277923584,
      "activations/layer22_attention_weight_min": -26.949739456176758,
      "activations/layer23_attention_weight_max": 33.532798767089844,
      "activations/layer23_attention_weight_min": -24.80865478515625,
      "activations/layer2_attention_weight_max": 31.733963012695312,
      "activations/layer2_attention_weight_min": -31.002012252807617,
      "activations/layer3_attention_weight_max": 94.87191009521484,
      "activations/layer3_attention_weight_min": -89.9125747680664,
      "activations/layer4_attention_weight_max": 90.2686767578125,
      "activations/layer4_attention_weight_min": -86.07524871826172,
      "activations/layer5_attention_weight_max": 65.88632202148438,
      "activations/layer5_attention_weight_min": -72.89433288574219,
      "activations/layer6_attention_weight_max": 51.80883026123047,
      "activations/layer6_attention_weight_min": -53.599552154541016,
      "activations/layer7_attention_weight_max": 71.24037170410156,
      "activations/layer7_attention_weight_min": -67.50628662109375,
      "activations/layer8_attention_weight_max": 49.1179313659668,
      "activations/layer8_attention_weight_min": -52.69752883911133,
      "activations/layer9_attention_weight_max": 45.94388198852539,
      "activations/layer9_attention_weight_min": -47.286659240722656,
      "epoch": 22.09,
      "learning_rate": 7.595833333333332e-06,
      "loss": 2.6984,
      "step": 380250
    },
    {
      "activations/layer0_attention_weight_max": 16.82263946533203,
      "activations/layer0_attention_weight_min": -13.9489107131958,
      "activations/layer10_attention_weight_max": 38.62823486328125,
      "activations/layer10_attention_weight_min": -39.544761657714844,
      "activations/layer11_attention_weight_max": 34.11408233642578,
      "activations/layer11_attention_weight_min": -35.84931182861328,
      "activations/layer12_attention_weight_max": 20.12384605407715,
      "activations/layer12_attention_weight_min": -24.436866760253906,
      "activations/layer13_attention_weight_max": 34.7366943359375,
      "activations/layer13_attention_weight_min": -28.293792724609375,
      "activations/layer14_attention_weight_max": 35.54093933105469,
      "activations/layer14_attention_weight_min": -30.89548110961914,
      "activations/layer15_attention_weight_max": 34.35893249511719,
      "activations/layer15_attention_weight_min": -30.599550247192383,
      "activations/layer16_attention_weight_max": 35.2598762512207,
      "activations/layer16_attention_weight_min": -34.145286560058594,
      "activations/layer17_attention_weight_max": 53.89837646484375,
      "activations/layer17_attention_weight_min": -45.1075325012207,
      "activations/layer18_attention_weight_max": 52.022491455078125,
      "activations/layer18_attention_weight_min": -40.99538803100586,
      "activations/layer19_attention_weight_max": 27.479612350463867,
      "activations/layer19_attention_weight_min": -25.7465877532959,
      "activations/layer1_attention_weight_max": 14.772900581359863,
      "activations/layer1_attention_weight_min": -13.70650863647461,
      "activations/layer20_attention_weight_max": 25.835493087768555,
      "activations/layer20_attention_weight_min": -24.087261199951172,
      "activations/layer21_attention_weight_max": 41.36749267578125,
      "activations/layer21_attention_weight_min": -22.79121208190918,
      "activations/layer22_attention_weight_max": 28.88302993774414,
      "activations/layer22_attention_weight_min": -25.315563201904297,
      "activations/layer23_attention_weight_max": 37.40713882446289,
      "activations/layer23_attention_weight_min": -27.782909393310547,
      "activations/layer2_attention_weight_max": 32.02676010131836,
      "activations/layer2_attention_weight_min": -30.655765533447266,
      "activations/layer3_attention_weight_max": 89.66651153564453,
      "activations/layer3_attention_weight_min": -88.39514923095703,
      "activations/layer4_attention_weight_max": 88.30378723144531,
      "activations/layer4_attention_weight_min": -85.85936737060547,
      "activations/layer5_attention_weight_max": 60.0753288269043,
      "activations/layer5_attention_weight_min": -72.0653305053711,
      "activations/layer6_attention_weight_max": 52.34391403198242,
      "activations/layer6_attention_weight_min": -52.46425247192383,
      "activations/layer7_attention_weight_max": 72.11194610595703,
      "activations/layer7_attention_weight_min": -73.47518157958984,
      "activations/layer8_attention_weight_max": 50.7100715637207,
      "activations/layer8_attention_weight_min": -53.6043586730957,
      "activations/layer9_attention_weight_max": 49.3750114440918,
      "activations/layer9_attention_weight_min": -50.186100006103516,
      "epoch": 22.1,
      "learning_rate": 7.576893939393939e-06,
      "loss": 2.705,
      "step": 380300
    },
    {
      "activations/layer0_attention_weight_max": 16.854251861572266,
      "activations/layer0_attention_weight_min": -14.09278678894043,
      "activations/layer10_attention_weight_max": 37.8294677734375,
      "activations/layer10_attention_weight_min": -38.6191291809082,
      "activations/layer11_attention_weight_max": 35.36885452270508,
      "activations/layer11_attention_weight_min": -36.13551330566406,
      "activations/layer12_attention_weight_max": 19.650028228759766,
      "activations/layer12_attention_weight_min": -26.801393508911133,
      "activations/layer13_attention_weight_max": 36.552574157714844,
      "activations/layer13_attention_weight_min": -30.76610565185547,
      "activations/layer14_attention_weight_max": 36.12153244018555,
      "activations/layer14_attention_weight_min": -32.707942962646484,
      "activations/layer15_attention_weight_max": 33.31927490234375,
      "activations/layer15_attention_weight_min": -32.52294158935547,
      "activations/layer16_attention_weight_max": 36.524658203125,
      "activations/layer16_attention_weight_min": -35.374732971191406,
      "activations/layer17_attention_weight_max": 52.817115783691406,
      "activations/layer17_attention_weight_min": -49.85955047607422,
      "activations/layer18_attention_weight_max": 47.8304557800293,
      "activations/layer18_attention_weight_min": -44.53394317626953,
      "activations/layer19_attention_weight_max": 26.41080665588379,
      "activations/layer19_attention_weight_min": -22.813650131225586,
      "activations/layer1_attention_weight_max": 14.989200592041016,
      "activations/layer1_attention_weight_min": -14.563175201416016,
      "activations/layer20_attention_weight_max": 25.78072166442871,
      "activations/layer20_attention_weight_min": -20.588335037231445,
      "activations/layer21_attention_weight_max": 37.15065383911133,
      "activations/layer21_attention_weight_min": -20.63256072998047,
      "activations/layer22_attention_weight_max": 28.373363494873047,
      "activations/layer22_attention_weight_min": -25.028385162353516,
      "activations/layer23_attention_weight_max": 34.84023666381836,
      "activations/layer23_attention_weight_min": -23.97193145751953,
      "activations/layer2_attention_weight_max": 31.841289520263672,
      "activations/layer2_attention_weight_min": -30.290996551513672,
      "activations/layer3_attention_weight_max": 88.79644775390625,
      "activations/layer3_attention_weight_min": -89.0333023071289,
      "activations/layer4_attention_weight_max": 83.12741088867188,
      "activations/layer4_attention_weight_min": -85.183837890625,
      "activations/layer5_attention_weight_max": 63.973995208740234,
      "activations/layer5_attention_weight_min": -73.49661254882812,
      "activations/layer6_attention_weight_max": 49.94707107543945,
      "activations/layer6_attention_weight_min": -52.41740036010742,
      "activations/layer7_attention_weight_max": 71.34554290771484,
      "activations/layer7_attention_weight_min": -75.55490112304688,
      "activations/layer8_attention_weight_max": 49.23653793334961,
      "activations/layer8_attention_weight_min": -54.767208099365234,
      "activations/layer9_attention_weight_max": 49.117950439453125,
      "activations/layer9_attention_weight_min": -49.19863510131836,
      "epoch": 22.1,
      "learning_rate": 7.557954545454545e-06,
      "loss": 2.6952,
      "step": 380350
    },
    {
      "activations/layer0_attention_weight_max": 16.781049728393555,
      "activations/layer0_attention_weight_min": -13.954445838928223,
      "activations/layer10_attention_weight_max": 38.49077224731445,
      "activations/layer10_attention_weight_min": -37.59362030029297,
      "activations/layer11_attention_weight_max": 36.003814697265625,
      "activations/layer11_attention_weight_min": -37.706356048583984,
      "activations/layer12_attention_weight_max": 21.232614517211914,
      "activations/layer12_attention_weight_min": -26.140316009521484,
      "activations/layer13_attention_weight_max": 38.66908264160156,
      "activations/layer13_attention_weight_min": -30.23942756652832,
      "activations/layer14_attention_weight_max": 41.332847595214844,
      "activations/layer14_attention_weight_min": -33.7253303527832,
      "activations/layer15_attention_weight_max": 37.377899169921875,
      "activations/layer15_attention_weight_min": -33.35853958129883,
      "activations/layer16_attention_weight_max": 42.05108642578125,
      "activations/layer16_attention_weight_min": -37.186767578125,
      "activations/layer17_attention_weight_max": 60.05009841918945,
      "activations/layer17_attention_weight_min": -52.10597610473633,
      "activations/layer18_attention_weight_max": 56.1679573059082,
      "activations/layer18_attention_weight_min": -46.55455780029297,
      "activations/layer19_attention_weight_max": 30.17778968811035,
      "activations/layer19_attention_weight_min": -27.185306549072266,
      "activations/layer1_attention_weight_max": 14.84915828704834,
      "activations/layer1_attention_weight_min": -14.0116548538208,
      "activations/layer20_attention_weight_max": 28.748960494995117,
      "activations/layer20_attention_weight_min": -22.86314582824707,
      "activations/layer21_attention_weight_max": 48.065364837646484,
      "activations/layer21_attention_weight_min": -25.531919479370117,
      "activations/layer22_attention_weight_max": 35.16141891479492,
      "activations/layer22_attention_weight_min": -29.061466217041016,
      "activations/layer23_attention_weight_max": 43.179100036621094,
      "activations/layer23_attention_weight_min": -25.447021484375,
      "activations/layer2_attention_weight_max": 31.533308029174805,
      "activations/layer2_attention_weight_min": -31.5644588470459,
      "activations/layer3_attention_weight_max": 90.4176254272461,
      "activations/layer3_attention_weight_min": -91.05855560302734,
      "activations/layer4_attention_weight_max": 85.46318054199219,
      "activations/layer4_attention_weight_min": -82.6697769165039,
      "activations/layer5_attention_weight_max": 60.7085075378418,
      "activations/layer5_attention_weight_min": -75.82997131347656,
      "activations/layer6_attention_weight_max": 49.939449310302734,
      "activations/layer6_attention_weight_min": -50.22414016723633,
      "activations/layer7_attention_weight_max": 68.20378875732422,
      "activations/layer7_attention_weight_min": -68.44641876220703,
      "activations/layer8_attention_weight_max": 49.716957092285156,
      "activations/layer8_attention_weight_min": -51.51365280151367,
      "activations/layer9_attention_weight_max": 45.2776985168457,
      "activations/layer9_attention_weight_min": -47.71442413330078,
      "epoch": 22.1,
      "learning_rate": 7.539015151515151e-06,
      "loss": 2.7072,
      "step": 380400
    },
    {
      "activations/layer0_attention_weight_max": 16.18079948425293,
      "activations/layer0_attention_weight_min": -14.078888893127441,
      "activations/layer10_attention_weight_max": 37.024269104003906,
      "activations/layer10_attention_weight_min": -35.11322784423828,
      "activations/layer11_attention_weight_max": 36.09959411621094,
      "activations/layer11_attention_weight_min": -35.13983154296875,
      "activations/layer12_attention_weight_max": 20.05136489868164,
      "activations/layer12_attention_weight_min": -24.950231552124023,
      "activations/layer13_attention_weight_max": 36.27817916870117,
      "activations/layer13_attention_weight_min": -27.053768157958984,
      "activations/layer14_attention_weight_max": 35.49500274658203,
      "activations/layer14_attention_weight_min": -33.392513275146484,
      "activations/layer15_attention_weight_max": 33.84833526611328,
      "activations/layer15_attention_weight_min": -31.076168060302734,
      "activations/layer16_attention_weight_max": 35.890716552734375,
      "activations/layer16_attention_weight_min": -33.071800231933594,
      "activations/layer17_attention_weight_max": 55.379520416259766,
      "activations/layer17_attention_weight_min": -45.96376419067383,
      "activations/layer18_attention_weight_max": 48.50091552734375,
      "activations/layer18_attention_weight_min": -40.3308219909668,
      "activations/layer19_attention_weight_max": 26.001615524291992,
      "activations/layer19_attention_weight_min": -24.078611373901367,
      "activations/layer1_attention_weight_max": 14.777210235595703,
      "activations/layer1_attention_weight_min": -13.659480094909668,
      "activations/layer20_attention_weight_max": 25.253833770751953,
      "activations/layer20_attention_weight_min": -21.484088897705078,
      "activations/layer21_attention_weight_max": 40.31623077392578,
      "activations/layer21_attention_weight_min": -23.73500633239746,
      "activations/layer22_attention_weight_max": 31.834341049194336,
      "activations/layer22_attention_weight_min": -26.760501861572266,
      "activations/layer23_attention_weight_max": 35.207088470458984,
      "activations/layer23_attention_weight_min": -24.61390495300293,
      "activations/layer2_attention_weight_max": 30.372852325439453,
      "activations/layer2_attention_weight_min": -28.855976104736328,
      "activations/layer3_attention_weight_max": 84.8298568725586,
      "activations/layer3_attention_weight_min": -83.56800079345703,
      "activations/layer4_attention_weight_max": 80.94807434082031,
      "activations/layer4_attention_weight_min": -79.72679138183594,
      "activations/layer5_attention_weight_max": 61.045318603515625,
      "activations/layer5_attention_weight_min": -71.97665405273438,
      "activations/layer6_attention_weight_max": 47.94633102416992,
      "activations/layer6_attention_weight_min": -49.4986457824707,
      "activations/layer7_attention_weight_max": 67.30107116699219,
      "activations/layer7_attention_weight_min": -67.74491119384766,
      "activations/layer8_attention_weight_max": 48.311058044433594,
      "activations/layer8_attention_weight_min": -51.75370407104492,
      "activations/layer9_attention_weight_max": 42.5705451965332,
      "activations/layer9_attention_weight_min": -45.620784759521484,
      "epoch": 22.11,
      "learning_rate": 7.520075757575757e-06,
      "loss": 2.6934,
      "step": 380450
    },
    {
      "activations/layer0_attention_weight_max": 16.399629592895508,
      "activations/layer0_attention_weight_min": -13.959088325500488,
      "activations/layer10_attention_weight_max": 36.82825469970703,
      "activations/layer10_attention_weight_min": -37.0211181640625,
      "activations/layer11_attention_weight_max": 33.27484130859375,
      "activations/layer11_attention_weight_min": -35.265628814697266,
      "activations/layer12_attention_weight_max": 19.277067184448242,
      "activations/layer12_attention_weight_min": -23.698368072509766,
      "activations/layer13_attention_weight_max": 34.15912628173828,
      "activations/layer13_attention_weight_min": -27.460466384887695,
      "activations/layer14_attention_weight_max": 38.54951477050781,
      "activations/layer14_attention_weight_min": -32.90443420410156,
      "activations/layer15_attention_weight_max": 37.5665168762207,
      "activations/layer15_attention_weight_min": -33.54951858520508,
      "activations/layer16_attention_weight_max": 37.83258056640625,
      "activations/layer16_attention_weight_min": -35.2989616394043,
      "activations/layer17_attention_weight_max": 55.019229888916016,
      "activations/layer17_attention_weight_min": -46.703392028808594,
      "activations/layer18_attention_weight_max": 50.07254409790039,
      "activations/layer18_attention_weight_min": -39.98838806152344,
      "activations/layer19_attention_weight_max": 25.917694091796875,
      "activations/layer19_attention_weight_min": -21.911914825439453,
      "activations/layer1_attention_weight_max": 14.664105415344238,
      "activations/layer1_attention_weight_min": -13.83061695098877,
      "activations/layer20_attention_weight_max": 24.811689376831055,
      "activations/layer20_attention_weight_min": -20.2579402923584,
      "activations/layer21_attention_weight_max": 32.989837646484375,
      "activations/layer21_attention_weight_min": -21.68275260925293,
      "activations/layer22_attention_weight_max": 31.863616943359375,
      "activations/layer22_attention_weight_min": -25.63303565979004,
      "activations/layer23_attention_weight_max": 35.56068801879883,
      "activations/layer23_attention_weight_min": -22.392047882080078,
      "activations/layer2_attention_weight_max": 31.666913986206055,
      "activations/layer2_attention_weight_min": -30.423038482666016,
      "activations/layer3_attention_weight_max": 87.39842987060547,
      "activations/layer3_attention_weight_min": -89.09236145019531,
      "activations/layer4_attention_weight_max": 83.84288787841797,
      "activations/layer4_attention_weight_min": -87.5179443359375,
      "activations/layer5_attention_weight_max": 61.28417205810547,
      "activations/layer5_attention_weight_min": -70.03142547607422,
      "activations/layer6_attention_weight_max": 49.549110412597656,
      "activations/layer6_attention_weight_min": -51.26591873168945,
      "activations/layer7_attention_weight_max": 67.56475067138672,
      "activations/layer7_attention_weight_min": -68.8931884765625,
      "activations/layer8_attention_weight_max": 48.3466682434082,
      "activations/layer8_attention_weight_min": -53.66472244262695,
      "activations/layer9_attention_weight_max": 44.814456939697266,
      "activations/layer9_attention_weight_min": -46.114925384521484,
      "epoch": 22.11,
      "learning_rate": 7.501136363636363e-06,
      "loss": 2.6957,
      "step": 380500
    },
    {
      "activations/layer0_attention_weight_max": 16.14875602722168,
      "activations/layer0_attention_weight_min": -14.0029935836792,
      "activations/layer10_attention_weight_max": 37.36907196044922,
      "activations/layer10_attention_weight_min": -36.48215103149414,
      "activations/layer11_attention_weight_max": 34.258689880371094,
      "activations/layer11_attention_weight_min": -36.45111846923828,
      "activations/layer12_attention_weight_max": 18.18327522277832,
      "activations/layer12_attention_weight_min": -24.724809646606445,
      "activations/layer13_attention_weight_max": 35.56782531738281,
      "activations/layer13_attention_weight_min": -26.960254669189453,
      "activations/layer14_attention_weight_max": 35.58723068237305,
      "activations/layer14_attention_weight_min": -29.59258460998535,
      "activations/layer15_attention_weight_max": 33.35752868652344,
      "activations/layer15_attention_weight_min": -29.33648109436035,
      "activations/layer16_attention_weight_max": 33.16092300415039,
      "activations/layer16_attention_weight_min": -32.01192855834961,
      "activations/layer17_attention_weight_max": 49.959720611572266,
      "activations/layer17_attention_weight_min": -42.76873779296875,
      "activations/layer18_attention_weight_max": 45.37187194824219,
      "activations/layer18_attention_weight_min": -39.80152130126953,
      "activations/layer19_attention_weight_max": 22.79763412475586,
      "activations/layer19_attention_weight_min": -19.762142181396484,
      "activations/layer1_attention_weight_max": 14.564953804016113,
      "activations/layer1_attention_weight_min": -13.821067810058594,
      "activations/layer20_attention_weight_max": 19.919294357299805,
      "activations/layer20_attention_weight_min": -20.535778045654297,
      "activations/layer21_attention_weight_max": 31.865163803100586,
      "activations/layer21_attention_weight_min": -20.28267478942871,
      "activations/layer22_attention_weight_max": 27.9538516998291,
      "activations/layer22_attention_weight_min": -25.831026077270508,
      "activations/layer23_attention_weight_max": 37.262351989746094,
      "activations/layer23_attention_weight_min": -23.724754333496094,
      "activations/layer2_attention_weight_max": 32.22135925292969,
      "activations/layer2_attention_weight_min": -29.710546493530273,
      "activations/layer3_attention_weight_max": 92.59577941894531,
      "activations/layer3_attention_weight_min": -90.66727447509766,
      "activations/layer4_attention_weight_max": 90.56815338134766,
      "activations/layer4_attention_weight_min": -87.37230682373047,
      "activations/layer5_attention_weight_max": 61.9329833984375,
      "activations/layer5_attention_weight_min": -75.76029968261719,
      "activations/layer6_attention_weight_max": 52.22502899169922,
      "activations/layer6_attention_weight_min": -51.02666091918945,
      "activations/layer7_attention_weight_max": 71.1728286743164,
      "activations/layer7_attention_weight_min": -67.33135223388672,
      "activations/layer8_attention_weight_max": 50.16278839111328,
      "activations/layer8_attention_weight_min": -52.44743728637695,
      "activations/layer9_attention_weight_max": 45.01757049560547,
      "activations/layer9_attention_weight_min": -45.39694595336914,
      "epoch": 22.11,
      "learning_rate": 7.482196969696969e-06,
      "loss": 2.6836,
      "step": 380550
    },
    {
      "activations/layer0_attention_weight_max": 15.940119743347168,
      "activations/layer0_attention_weight_min": -14.387109756469727,
      "activations/layer10_attention_weight_max": 35.92881774902344,
      "activations/layer10_attention_weight_min": -35.37799072265625,
      "activations/layer11_attention_weight_max": 33.174896240234375,
      "activations/layer11_attention_weight_min": -33.72308349609375,
      "activations/layer12_attention_weight_max": 20.447153091430664,
      "activations/layer12_attention_weight_min": -37.88493728637695,
      "activations/layer13_attention_weight_max": 38.05440139770508,
      "activations/layer13_attention_weight_min": -32.3331184387207,
      "activations/layer14_attention_weight_max": 39.732460021972656,
      "activations/layer14_attention_weight_min": -33.75919723510742,
      "activations/layer15_attention_weight_max": 37.08222198486328,
      "activations/layer15_attention_weight_min": -32.078125,
      "activations/layer16_attention_weight_max": 37.76971435546875,
      "activations/layer16_attention_weight_min": -35.64776611328125,
      "activations/layer17_attention_weight_max": 55.46086120605469,
      "activations/layer17_attention_weight_min": -49.605525970458984,
      "activations/layer18_attention_weight_max": 49.574806213378906,
      "activations/layer18_attention_weight_min": -46.38926696777344,
      "activations/layer19_attention_weight_max": 24.86131477355957,
      "activations/layer19_attention_weight_min": -25.011137008666992,
      "activations/layer1_attention_weight_max": 15.250391006469727,
      "activations/layer1_attention_weight_min": -14.782693862915039,
      "activations/layer20_attention_weight_max": 24.673051834106445,
      "activations/layer20_attention_weight_min": -22.188798904418945,
      "activations/layer21_attention_weight_max": 42.3779411315918,
      "activations/layer21_attention_weight_min": -22.78633689880371,
      "activations/layer22_attention_weight_max": 29.9462890625,
      "activations/layer22_attention_weight_min": -24.986406326293945,
      "activations/layer23_attention_weight_max": 35.14613723754883,
      "activations/layer23_attention_weight_min": -26.09467315673828,
      "activations/layer2_attention_weight_max": 31.811025619506836,
      "activations/layer2_attention_weight_min": -29.672107696533203,
      "activations/layer3_attention_weight_max": 87.88529205322266,
      "activations/layer3_attention_weight_min": -87.5633316040039,
      "activations/layer4_attention_weight_max": 82.48870086669922,
      "activations/layer4_attention_weight_min": -82.89134979248047,
      "activations/layer5_attention_weight_max": 62.111785888671875,
      "activations/layer5_attention_weight_min": -72.8508529663086,
      "activations/layer6_attention_weight_max": 48.17757797241211,
      "activations/layer6_attention_weight_min": -48.40373229980469,
      "activations/layer7_attention_weight_max": 70.69114685058594,
      "activations/layer7_attention_weight_min": -67.69622039794922,
      "activations/layer8_attention_weight_max": 45.87791061401367,
      "activations/layer8_attention_weight_min": -48.07000732421875,
      "activations/layer9_attention_weight_max": 50.68864440917969,
      "activations/layer9_attention_weight_min": -46.90098190307617,
      "epoch": 22.12,
      "learning_rate": 7.4632575757575745e-06,
      "loss": 2.6905,
      "step": 380600
    },
    {
      "activations/layer0_attention_weight_max": 16.85489845275879,
      "activations/layer0_attention_weight_min": -13.95231819152832,
      "activations/layer10_attention_weight_max": 34.741607666015625,
      "activations/layer10_attention_weight_min": -34.31415557861328,
      "activations/layer11_attention_weight_max": 31.27008628845215,
      "activations/layer11_attention_weight_min": -35.122589111328125,
      "activations/layer12_attention_weight_max": 19.079919815063477,
      "activations/layer12_attention_weight_min": -24.130109786987305,
      "activations/layer13_attention_weight_max": 35.737266540527344,
      "activations/layer13_attention_weight_min": -25.615156173706055,
      "activations/layer14_attention_weight_max": 33.23809814453125,
      "activations/layer14_attention_weight_min": -31.56612777709961,
      "activations/layer15_attention_weight_max": 35.107872009277344,
      "activations/layer15_attention_weight_min": -31.23590660095215,
      "activations/layer16_attention_weight_max": 34.05567169189453,
      "activations/layer16_attention_weight_min": -32.724937438964844,
      "activations/layer17_attention_weight_max": 50.183101654052734,
      "activations/layer17_attention_weight_min": -46.88935470581055,
      "activations/layer18_attention_weight_max": 46.23185729980469,
      "activations/layer18_attention_weight_min": -39.31666564941406,
      "activations/layer19_attention_weight_max": 23.926733016967773,
      "activations/layer19_attention_weight_min": -21.57520866394043,
      "activations/layer1_attention_weight_max": 15.981942176818848,
      "activations/layer1_attention_weight_min": -14.24023151397705,
      "activations/layer20_attention_weight_max": 25.713865280151367,
      "activations/layer20_attention_weight_min": -20.404033660888672,
      "activations/layer21_attention_weight_max": 36.91797637939453,
      "activations/layer21_attention_weight_min": -23.776748657226562,
      "activations/layer22_attention_weight_max": 30.47554588317871,
      "activations/layer22_attention_weight_min": -25.1793270111084,
      "activations/layer23_attention_weight_max": 38.04833221435547,
      "activations/layer23_attention_weight_min": -22.83023452758789,
      "activations/layer2_attention_weight_max": 32.4572639465332,
      "activations/layer2_attention_weight_min": -30.80846405029297,
      "activations/layer3_attention_weight_max": 88.3416519165039,
      "activations/layer3_attention_weight_min": -88.54100036621094,
      "activations/layer4_attention_weight_max": 84.21449279785156,
      "activations/layer4_attention_weight_min": -84.03369903564453,
      "activations/layer5_attention_weight_max": 61.24339294433594,
      "activations/layer5_attention_weight_min": -74.07243347167969,
      "activations/layer6_attention_weight_max": 50.795467376708984,
      "activations/layer6_attention_weight_min": -50.05635452270508,
      "activations/layer7_attention_weight_max": 67.4951400756836,
      "activations/layer7_attention_weight_min": -67.73858642578125,
      "activations/layer8_attention_weight_max": 45.920040130615234,
      "activations/layer8_attention_weight_min": -47.6949577331543,
      "activations/layer9_attention_weight_max": 43.1860466003418,
      "activations/layer9_attention_weight_min": -46.33230972290039,
      "epoch": 22.12,
      "learning_rate": 7.444318181818181e-06,
      "loss": 2.6997,
      "step": 380650
    },
    {
      "activations/layer0_attention_weight_max": 16.871267318725586,
      "activations/layer0_attention_weight_min": -14.152315139770508,
      "activations/layer10_attention_weight_max": 36.604339599609375,
      "activations/layer10_attention_weight_min": -37.84758758544922,
      "activations/layer11_attention_weight_max": 34.33760070800781,
      "activations/layer11_attention_weight_min": -34.74237823486328,
      "activations/layer12_attention_weight_max": 20.804235458374023,
      "activations/layer12_attention_weight_min": -24.326778411865234,
      "activations/layer13_attention_weight_max": 35.80634689331055,
      "activations/layer13_attention_weight_min": -30.176448822021484,
      "activations/layer14_attention_weight_max": 36.464752197265625,
      "activations/layer14_attention_weight_min": -31.837106704711914,
      "activations/layer15_attention_weight_max": 34.4063606262207,
      "activations/layer15_attention_weight_min": -29.773351669311523,
      "activations/layer16_attention_weight_max": 36.60666275024414,
      "activations/layer16_attention_weight_min": -35.56755065917969,
      "activations/layer17_attention_weight_max": 56.64194869995117,
      "activations/layer17_attention_weight_min": -52.23127746582031,
      "activations/layer18_attention_weight_max": 51.124019622802734,
      "activations/layer18_attention_weight_min": -49.552616119384766,
      "activations/layer19_attention_weight_max": 27.92649269104004,
      "activations/layer19_attention_weight_min": -25.6030330657959,
      "activations/layer1_attention_weight_max": 15.007933616638184,
      "activations/layer1_attention_weight_min": -14.46802043914795,
      "activations/layer20_attention_weight_max": 26.317052841186523,
      "activations/layer20_attention_weight_min": -22.44806480407715,
      "activations/layer21_attention_weight_max": 32.61764907836914,
      "activations/layer21_attention_weight_min": -21.374786376953125,
      "activations/layer22_attention_weight_max": 28.849634170532227,
      "activations/layer22_attention_weight_min": -25.711580276489258,
      "activations/layer23_attention_weight_max": 38.60103988647461,
      "activations/layer23_attention_weight_min": -26.438640594482422,
      "activations/layer2_attention_weight_max": 32.45586013793945,
      "activations/layer2_attention_weight_min": -31.33704376220703,
      "activations/layer3_attention_weight_max": 91.02751159667969,
      "activations/layer3_attention_weight_min": -92.08399963378906,
      "activations/layer4_attention_weight_max": 84.51545715332031,
      "activations/layer4_attention_weight_min": -87.2994613647461,
      "activations/layer5_attention_weight_max": 58.86494445800781,
      "activations/layer5_attention_weight_min": -73.37086486816406,
      "activations/layer6_attention_weight_max": 50.071510314941406,
      "activations/layer6_attention_weight_min": -53.24238586425781,
      "activations/layer7_attention_weight_max": 68.65389251708984,
      "activations/layer7_attention_weight_min": -71.88217163085938,
      "activations/layer8_attention_weight_max": 49.56929397583008,
      "activations/layer8_attention_weight_min": -55.64315414428711,
      "activations/layer9_attention_weight_max": 46.0120849609375,
      "activations/layer9_attention_weight_min": -50.12641906738281,
      "epoch": 22.12,
      "learning_rate": 7.425378787878787e-06,
      "loss": 2.7006,
      "step": 380700
    },
    {
      "activations/layer0_attention_weight_max": 16.304729461669922,
      "activations/layer0_attention_weight_min": -14.728394508361816,
      "activations/layer10_attention_weight_max": 35.84245681762695,
      "activations/layer10_attention_weight_min": -37.40144729614258,
      "activations/layer11_attention_weight_max": 35.46833419799805,
      "activations/layer11_attention_weight_min": -38.32025909423828,
      "activations/layer12_attention_weight_max": 21.15110206604004,
      "activations/layer12_attention_weight_min": -25.381078720092773,
      "activations/layer13_attention_weight_max": 35.41700744628906,
      "activations/layer13_attention_weight_min": -25.805139541625977,
      "activations/layer14_attention_weight_max": 38.167606353759766,
      "activations/layer14_attention_weight_min": -32.62696838378906,
      "activations/layer15_attention_weight_max": 37.65934753417969,
      "activations/layer15_attention_weight_min": -32.09089279174805,
      "activations/layer16_attention_weight_max": 38.18357849121094,
      "activations/layer16_attention_weight_min": -37.12434768676758,
      "activations/layer17_attention_weight_max": 56.98042678833008,
      "activations/layer17_attention_weight_min": -48.58955383300781,
      "activations/layer18_attention_weight_max": 54.9638786315918,
      "activations/layer18_attention_weight_min": -46.011436462402344,
      "activations/layer19_attention_weight_max": 30.142358779907227,
      "activations/layer19_attention_weight_min": -27.021520614624023,
      "activations/layer1_attention_weight_max": 15.250359535217285,
      "activations/layer1_attention_weight_min": -13.541454315185547,
      "activations/layer20_attention_weight_max": 26.05318260192871,
      "activations/layer20_attention_weight_min": -22.90819549560547,
      "activations/layer21_attention_weight_max": 42.09166717529297,
      "activations/layer21_attention_weight_min": -23.56151580810547,
      "activations/layer22_attention_weight_max": 34.66305923461914,
      "activations/layer22_attention_weight_min": -28.67261505126953,
      "activations/layer23_attention_weight_max": 41.43679428100586,
      "activations/layer23_attention_weight_min": -23.477468490600586,
      "activations/layer2_attention_weight_max": 30.82246971130371,
      "activations/layer2_attention_weight_min": -29.458744049072266,
      "activations/layer3_attention_weight_max": 86.3165512084961,
      "activations/layer3_attention_weight_min": -87.1852798461914,
      "activations/layer4_attention_weight_max": 83.76506805419922,
      "activations/layer4_attention_weight_min": -82.03411865234375,
      "activations/layer5_attention_weight_max": 59.83709716796875,
      "activations/layer5_attention_weight_min": -68.14376831054688,
      "activations/layer6_attention_weight_max": 48.451229095458984,
      "activations/layer6_attention_weight_min": -49.89052200317383,
      "activations/layer7_attention_weight_max": 64.69441223144531,
      "activations/layer7_attention_weight_min": -72.3333969116211,
      "activations/layer8_attention_weight_max": 46.15260696411133,
      "activations/layer8_attention_weight_min": -50.762725830078125,
      "activations/layer9_attention_weight_max": 41.999717712402344,
      "activations/layer9_attention_weight_min": -47.670101165771484,
      "epoch": 22.12,
      "learning_rate": 7.406439393939393e-06,
      "loss": 2.6949,
      "step": 380750
    },
    {
      "activations/layer0_attention_weight_max": 16.213611602783203,
      "activations/layer0_attention_weight_min": -14.04332447052002,
      "activations/layer10_attention_weight_max": 33.71807098388672,
      "activations/layer10_attention_weight_min": -36.21992492675781,
      "activations/layer11_attention_weight_max": 30.907196044921875,
      "activations/layer11_attention_weight_min": -33.436614990234375,
      "activations/layer12_attention_weight_max": 19.02744483947754,
      "activations/layer12_attention_weight_min": -28.209745407104492,
      "activations/layer13_attention_weight_max": 34.525840759277344,
      "activations/layer13_attention_weight_min": -26.47087860107422,
      "activations/layer14_attention_weight_max": 34.853736877441406,
      "activations/layer14_attention_weight_min": -29.558101654052734,
      "activations/layer15_attention_weight_max": 32.58328628540039,
      "activations/layer15_attention_weight_min": -29.35671043395996,
      "activations/layer16_attention_weight_max": 34.64453887939453,
      "activations/layer16_attention_weight_min": -33.537288665771484,
      "activations/layer17_attention_weight_max": 51.144683837890625,
      "activations/layer17_attention_weight_min": -46.03200149536133,
      "activations/layer18_attention_weight_max": 48.13783645629883,
      "activations/layer18_attention_weight_min": -42.577667236328125,
      "activations/layer19_attention_weight_max": 24.33452796936035,
      "activations/layer19_attention_weight_min": -23.473392486572266,
      "activations/layer1_attention_weight_max": 16.091148376464844,
      "activations/layer1_attention_weight_min": -15.41037654876709,
      "activations/layer20_attention_weight_max": 24.525901794433594,
      "activations/layer20_attention_weight_min": -22.162994384765625,
      "activations/layer21_attention_weight_max": 37.46828079223633,
      "activations/layer21_attention_weight_min": -24.962488174438477,
      "activations/layer22_attention_weight_max": 30.16341209411621,
      "activations/layer22_attention_weight_min": -25.406919479370117,
      "activations/layer23_attention_weight_max": 42.23221206665039,
      "activations/layer23_attention_weight_min": -23.783794403076172,
      "activations/layer2_attention_weight_max": 33.41305160522461,
      "activations/layer2_attention_weight_min": -31.286117553710938,
      "activations/layer3_attention_weight_max": 91.296630859375,
      "activations/layer3_attention_weight_min": -89.81071472167969,
      "activations/layer4_attention_weight_max": 84.71355438232422,
      "activations/layer4_attention_weight_min": -83.19368743896484,
      "activations/layer5_attention_weight_max": 59.711029052734375,
      "activations/layer5_attention_weight_min": -70.63278198242188,
      "activations/layer6_attention_weight_max": 48.74227523803711,
      "activations/layer6_attention_weight_min": -49.98744201660156,
      "activations/layer7_attention_weight_max": 68.73104095458984,
      "activations/layer7_attention_weight_min": -64.92264556884766,
      "activations/layer8_attention_weight_max": 47.239471435546875,
      "activations/layer8_attention_weight_min": -51.21696853637695,
      "activations/layer9_attention_weight_max": 42.016963958740234,
      "activations/layer9_attention_weight_min": -45.676307678222656,
      "epoch": 22.13,
      "learning_rate": 7.3875e-06,
      "loss": 2.6982,
      "step": 380800
    },
    {
      "activations/layer0_attention_weight_max": 15.962981224060059,
      "activations/layer0_attention_weight_min": -13.863578796386719,
      "activations/layer10_attention_weight_max": 36.26643753051758,
      "activations/layer10_attention_weight_min": -36.78102111816406,
      "activations/layer11_attention_weight_max": 32.26174545288086,
      "activations/layer11_attention_weight_min": -34.94200134277344,
      "activations/layer12_attention_weight_max": 18.786096572875977,
      "activations/layer12_attention_weight_min": -24.884902954101562,
      "activations/layer13_attention_weight_max": 34.280330657958984,
      "activations/layer13_attention_weight_min": -28.155738830566406,
      "activations/layer14_attention_weight_max": 39.88933563232422,
      "activations/layer14_attention_weight_min": -33.41139602661133,
      "activations/layer15_attention_weight_max": 36.372440338134766,
      "activations/layer15_attention_weight_min": -30.360105514526367,
      "activations/layer16_attention_weight_max": 38.02622985839844,
      "activations/layer16_attention_weight_min": -35.47720718383789,
      "activations/layer17_attention_weight_max": 57.61650848388672,
      "activations/layer17_attention_weight_min": -50.45934295654297,
      "activations/layer18_attention_weight_max": 52.4233512878418,
      "activations/layer18_attention_weight_min": -43.203189849853516,
      "activations/layer19_attention_weight_max": 25.047033309936523,
      "activations/layer19_attention_weight_min": -23.086668014526367,
      "activations/layer1_attention_weight_max": 14.944116592407227,
      "activations/layer1_attention_weight_min": -14.162504196166992,
      "activations/layer20_attention_weight_max": 22.509374618530273,
      "activations/layer20_attention_weight_min": -21.844554901123047,
      "activations/layer21_attention_weight_max": 39.065738677978516,
      "activations/layer21_attention_weight_min": -20.993995666503906,
      "activations/layer22_attention_weight_max": 26.775978088378906,
      "activations/layer22_attention_weight_min": -24.94285774230957,
      "activations/layer23_attention_weight_max": 35.549015045166016,
      "activations/layer23_attention_weight_min": -24.77722930908203,
      "activations/layer2_attention_weight_max": 31.677724838256836,
      "activations/layer2_attention_weight_min": -31.186687469482422,
      "activations/layer3_attention_weight_max": 89.18840789794922,
      "activations/layer3_attention_weight_min": -92.12149047851562,
      "activations/layer4_attention_weight_max": 84.94788360595703,
      "activations/layer4_attention_weight_min": -83.8887939453125,
      "activations/layer5_attention_weight_max": 60.83209228515625,
      "activations/layer5_attention_weight_min": -70.56358337402344,
      "activations/layer6_attention_weight_max": 50.169124603271484,
      "activations/layer6_attention_weight_min": -51.56618881225586,
      "activations/layer7_attention_weight_max": 65.67027282714844,
      "activations/layer7_attention_weight_min": -70.20430755615234,
      "activations/layer8_attention_weight_max": 46.084983825683594,
      "activations/layer8_attention_weight_min": -49.82326889038086,
      "activations/layer9_attention_weight_max": 41.78652572631836,
      "activations/layer9_attention_weight_min": -45.96533966064453,
      "epoch": 22.13,
      "learning_rate": 7.368560606060606e-06,
      "loss": 2.71,
      "step": 380850
    },
    {
      "activations/layer0_attention_weight_max": 15.886296272277832,
      "activations/layer0_attention_weight_min": -14.038122177124023,
      "activations/layer10_attention_weight_max": 36.79597854614258,
      "activations/layer10_attention_weight_min": -36.04792022705078,
      "activations/layer11_attention_weight_max": 33.47547912597656,
      "activations/layer11_attention_weight_min": -34.40980529785156,
      "activations/layer12_attention_weight_max": 18.781261444091797,
      "activations/layer12_attention_weight_min": -25.858320236206055,
      "activations/layer13_attention_weight_max": 33.4365119934082,
      "activations/layer13_attention_weight_min": -26.704553604125977,
      "activations/layer14_attention_weight_max": 32.74072265625,
      "activations/layer14_attention_weight_min": -30.589221954345703,
      "activations/layer15_attention_weight_max": 32.926170349121094,
      "activations/layer15_attention_weight_min": -29.95427703857422,
      "activations/layer16_attention_weight_max": 31.86334991455078,
      "activations/layer16_attention_weight_min": -31.040491104125977,
      "activations/layer17_attention_weight_max": 47.85441589355469,
      "activations/layer17_attention_weight_min": -44.490135192871094,
      "activations/layer18_attention_weight_max": 44.68614196777344,
      "activations/layer18_attention_weight_min": -39.53411102294922,
      "activations/layer19_attention_weight_max": 22.477201461791992,
      "activations/layer19_attention_weight_min": -23.277395248413086,
      "activations/layer1_attention_weight_max": 16.420000076293945,
      "activations/layer1_attention_weight_min": -15.150256156921387,
      "activations/layer20_attention_weight_max": 21.529495239257812,
      "activations/layer20_attention_weight_min": -20.109067916870117,
      "activations/layer21_attention_weight_max": 33.51193618774414,
      "activations/layer21_attention_weight_min": -21.22577476501465,
      "activations/layer22_attention_weight_max": 29.376502990722656,
      "activations/layer22_attention_weight_min": -25.592113494873047,
      "activations/layer23_attention_weight_max": 35.401344299316406,
      "activations/layer23_attention_weight_min": -23.56143569946289,
      "activations/layer2_attention_weight_max": 31.60320281982422,
      "activations/layer2_attention_weight_min": -30.327985763549805,
      "activations/layer3_attention_weight_max": 88.31204986572266,
      "activations/layer3_attention_weight_min": -89.31360626220703,
      "activations/layer4_attention_weight_max": 83.22359466552734,
      "activations/layer4_attention_weight_min": -82.3322982788086,
      "activations/layer5_attention_weight_max": 61.67763900756836,
      "activations/layer5_attention_weight_min": -76.23175048828125,
      "activations/layer6_attention_weight_max": 47.61439895629883,
      "activations/layer6_attention_weight_min": -49.85521697998047,
      "activations/layer7_attention_weight_max": 72.40042114257812,
      "activations/layer7_attention_weight_min": -66.50330352783203,
      "activations/layer8_attention_weight_max": 47.13622283935547,
      "activations/layer8_attention_weight_min": -50.78815460205078,
      "activations/layer9_attention_weight_max": 43.954795837402344,
      "activations/layer9_attention_weight_min": -44.760372161865234,
      "epoch": 22.13,
      "learning_rate": 7.3496212121212115e-06,
      "loss": 2.7089,
      "step": 380900
    },
    {
      "activations/layer0_attention_weight_max": 16.00716781616211,
      "activations/layer0_attention_weight_min": -14.30870532989502,
      "activations/layer10_attention_weight_max": 36.00253677368164,
      "activations/layer10_attention_weight_min": -36.20282745361328,
      "activations/layer11_attention_weight_max": 36.957908630371094,
      "activations/layer11_attention_weight_min": -37.757606506347656,
      "activations/layer12_attention_weight_max": 18.819904327392578,
      "activations/layer12_attention_weight_min": -24.60833740234375,
      "activations/layer13_attention_weight_max": 33.45396423339844,
      "activations/layer13_attention_weight_min": -27.5084285736084,
      "activations/layer14_attention_weight_max": 35.117149353027344,
      "activations/layer14_attention_weight_min": -32.73237609863281,
      "activations/layer15_attention_weight_max": 31.759136199951172,
      "activations/layer15_attention_weight_min": -30.673166275024414,
      "activations/layer16_attention_weight_max": 32.85212326049805,
      "activations/layer16_attention_weight_min": -32.088687896728516,
      "activations/layer17_attention_weight_max": 49.63313674926758,
      "activations/layer17_attention_weight_min": -45.109649658203125,
      "activations/layer18_attention_weight_max": 45.61465835571289,
      "activations/layer18_attention_weight_min": -41.120628356933594,
      "activations/layer19_attention_weight_max": 25.172788619995117,
      "activations/layer19_attention_weight_min": -21.835912704467773,
      "activations/layer1_attention_weight_max": 15.336091995239258,
      "activations/layer1_attention_weight_min": -15.531031608581543,
      "activations/layer20_attention_weight_max": 24.067304611206055,
      "activations/layer20_attention_weight_min": -21.24801254272461,
      "activations/layer21_attention_weight_max": 32.03095245361328,
      "activations/layer21_attention_weight_min": -21.53934669494629,
      "activations/layer22_attention_weight_max": 28.397005081176758,
      "activations/layer22_attention_weight_min": -24.93755531311035,
      "activations/layer23_attention_weight_max": 37.225685119628906,
      "activations/layer23_attention_weight_min": -25.47340202331543,
      "activations/layer2_attention_weight_max": 33.47197341918945,
      "activations/layer2_attention_weight_min": -31.515859603881836,
      "activations/layer3_attention_weight_max": 92.13690948486328,
      "activations/layer3_attention_weight_min": -90.62578582763672,
      "activations/layer4_attention_weight_max": 87.91288757324219,
      "activations/layer4_attention_weight_min": -86.3077621459961,
      "activations/layer5_attention_weight_max": 63.733978271484375,
      "activations/layer5_attention_weight_min": -78.70476531982422,
      "activations/layer6_attention_weight_max": 52.3725700378418,
      "activations/layer6_attention_weight_min": -53.028907775878906,
      "activations/layer7_attention_weight_max": 71.08344268798828,
      "activations/layer7_attention_weight_min": -73.4077377319336,
      "activations/layer8_attention_weight_max": 50.35060501098633,
      "activations/layer8_attention_weight_min": -51.44279861450195,
      "activations/layer9_attention_weight_max": 46.42051315307617,
      "activations/layer9_attention_weight_min": -48.09440231323242,
      "epoch": 22.14,
      "learning_rate": 7.330681818181817e-06,
      "loss": 2.7268,
      "step": 380950
    },
    {
      "activations/layer0_attention_weight_max": 15.96423053741455,
      "activations/layer0_attention_weight_min": -14.374035835266113,
      "activations/layer10_attention_weight_max": 36.16682434082031,
      "activations/layer10_attention_weight_min": -36.74214553833008,
      "activations/layer11_attention_weight_max": 34.030677795410156,
      "activations/layer11_attention_weight_min": -35.158226013183594,
      "activations/layer12_attention_weight_max": 19.355127334594727,
      "activations/layer12_attention_weight_min": -25.751466751098633,
      "activations/layer13_attention_weight_max": 33.2227897644043,
      "activations/layer13_attention_weight_min": -28.943893432617188,
      "activations/layer14_attention_weight_max": 33.37276077270508,
      "activations/layer14_attention_weight_min": -32.5360221862793,
      "activations/layer15_attention_weight_max": 32.206722259521484,
      "activations/layer15_attention_weight_min": -30.706958770751953,
      "activations/layer16_attention_weight_max": 33.49942398071289,
      "activations/layer16_attention_weight_min": -33.39420700073242,
      "activations/layer17_attention_weight_max": 50.23606491088867,
      "activations/layer17_attention_weight_min": -44.932945251464844,
      "activations/layer18_attention_weight_max": 48.35000991821289,
      "activations/layer18_attention_weight_min": -40.918846130371094,
      "activations/layer19_attention_weight_max": 24.040393829345703,
      "activations/layer19_attention_weight_min": -22.738149642944336,
      "activations/layer1_attention_weight_max": 14.688039779663086,
      "activations/layer1_attention_weight_min": -13.437406539916992,
      "activations/layer20_attention_weight_max": 21.94760513305664,
      "activations/layer20_attention_weight_min": -22.69889259338379,
      "activations/layer21_attention_weight_max": 39.317779541015625,
      "activations/layer21_attention_weight_min": -25.381589889526367,
      "activations/layer22_attention_weight_max": 29.635868072509766,
      "activations/layer22_attention_weight_min": -26.683687210083008,
      "activations/layer23_attention_weight_max": 35.28170394897461,
      "activations/layer23_attention_weight_min": -24.084091186523438,
      "activations/layer2_attention_weight_max": 30.909276962280273,
      "activations/layer2_attention_weight_min": -29.992584228515625,
      "activations/layer3_attention_weight_max": 85.93527221679688,
      "activations/layer3_attention_weight_min": -88.25457000732422,
      "activations/layer4_attention_weight_max": 83.22352600097656,
      "activations/layer4_attention_weight_min": -83.57196044921875,
      "activations/layer5_attention_weight_max": 63.478057861328125,
      "activations/layer5_attention_weight_min": -72.96953582763672,
      "activations/layer6_attention_weight_max": 49.38899230957031,
      "activations/layer6_attention_weight_min": -50.1529426574707,
      "activations/layer7_attention_weight_max": 72.82574462890625,
      "activations/layer7_attention_weight_min": -69.7140884399414,
      "activations/layer8_attention_weight_max": 49.31245040893555,
      "activations/layer8_attention_weight_min": -51.14152145385742,
      "activations/layer9_attention_weight_max": 42.98843002319336,
      "activations/layer9_attention_weight_min": -46.277889251708984,
      "epoch": 22.14,
      "learning_rate": 7.311742424242424e-06,
      "loss": 2.7113,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.5079,
      "eval_samples_per_second": 504.707,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.5079,
      "eval_openwebtext_samples_per_second": 504.707,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9698,
      "eval_wikitext_samples_per_second": 231.499,
      "step": 381000
    },
    {
      "epoch": 22.14,
      "eval_lambada_loss": 2.3828125,
      "eval_lambada_ppl": 10.835334427562556,
      "eval_lambada_runtime": 9.5162,
      "eval_lambada_samples_per_second": 511.655,
      "step": 381000
    },
    {
      "activations/layer0_attention_weight_max": 15.903648376464844,
      "activations/layer0_attention_weight_min": -14.045557022094727,
      "activations/layer10_attention_weight_max": 34.34400177001953,
      "activations/layer10_attention_weight_min": -35.84312438964844,
      "activations/layer11_attention_weight_max": 31.39753532409668,
      "activations/layer11_attention_weight_min": -32.63054656982422,
      "activations/layer12_attention_weight_max": 17.55359649658203,
      "activations/layer12_attention_weight_min": -22.978727340698242,
      "activations/layer13_attention_weight_max": 33.79586410522461,
      "activations/layer13_attention_weight_min": -26.533742904663086,
      "activations/layer14_attention_weight_max": 33.324432373046875,
      "activations/layer14_attention_weight_min": -30.8013973236084,
      "activations/layer15_attention_weight_max": 32.565711975097656,
      "activations/layer15_attention_weight_min": -31.12482452392578,
      "activations/layer16_attention_weight_max": 32.23211669921875,
      "activations/layer16_attention_weight_min": -32.216453552246094,
      "activations/layer17_attention_weight_max": 47.876766204833984,
      "activations/layer17_attention_weight_min": -45.629215240478516,
      "activations/layer18_attention_weight_max": 47.46078109741211,
      "activations/layer18_attention_weight_min": -40.23268508911133,
      "activations/layer19_attention_weight_max": 26.805130004882812,
      "activations/layer19_attention_weight_min": -23.08845329284668,
      "activations/layer1_attention_weight_max": 15.0782470703125,
      "activations/layer1_attention_weight_min": -13.936737060546875,
      "activations/layer20_attention_weight_max": 27.434734344482422,
      "activations/layer20_attention_weight_min": -20.934892654418945,
      "activations/layer21_attention_weight_max": 34.15828323364258,
      "activations/layer21_attention_weight_min": -21.25765037536621,
      "activations/layer22_attention_weight_max": 30.816951751708984,
      "activations/layer22_attention_weight_min": -23.927331924438477,
      "activations/layer23_attention_weight_max": 36.0457649230957,
      "activations/layer23_attention_weight_min": -22.774234771728516,
      "activations/layer2_attention_weight_max": 29.989561080932617,
      "activations/layer2_attention_weight_min": -30.06939697265625,
      "activations/layer3_attention_weight_max": 86.89702606201172,
      "activations/layer3_attention_weight_min": -87.24502563476562,
      "activations/layer4_attention_weight_max": 82.75112915039062,
      "activations/layer4_attention_weight_min": -83.58971405029297,
      "activations/layer5_attention_weight_max": 62.482513427734375,
      "activations/layer5_attention_weight_min": -74.02912902832031,
      "activations/layer6_attention_weight_max": 47.68030548095703,
      "activations/layer6_attention_weight_min": -50.518714904785156,
      "activations/layer7_attention_weight_max": 70.1325454711914,
      "activations/layer7_attention_weight_min": -67.58834838867188,
      "activations/layer8_attention_weight_max": 45.75230026245117,
      "activations/layer8_attention_weight_min": -48.25105285644531,
      "activations/layer9_attention_weight_max": 42.74008560180664,
      "activations/layer9_attention_weight_min": -47.01517868041992,
      "epoch": 22.14,
      "learning_rate": 7.2928030303030295e-06,
      "loss": 2.7092,
      "step": 381050
    },
    {
      "activations/layer0_attention_weight_max": 16.857311248779297,
      "activations/layer0_attention_weight_min": -16.644372940063477,
      "activations/layer10_attention_weight_max": 38.0984992980957,
      "activations/layer10_attention_weight_min": -35.97059631347656,
      "activations/layer11_attention_weight_max": 34.277122497558594,
      "activations/layer11_attention_weight_min": -37.385643005371094,
      "activations/layer12_attention_weight_max": 20.156885147094727,
      "activations/layer12_attention_weight_min": -26.022945404052734,
      "activations/layer13_attention_weight_max": 35.10951614379883,
      "activations/layer13_attention_weight_min": -27.91516876220703,
      "activations/layer14_attention_weight_max": 37.466983795166016,
      "activations/layer14_attention_weight_min": -30.540918350219727,
      "activations/layer15_attention_weight_max": 34.96571350097656,
      "activations/layer15_attention_weight_min": -32.56364440917969,
      "activations/layer16_attention_weight_max": 35.44921112060547,
      "activations/layer16_attention_weight_min": -32.432777404785156,
      "activations/layer17_attention_weight_max": 52.94329833984375,
      "activations/layer17_attention_weight_min": -45.63447189331055,
      "activations/layer18_attention_weight_max": 48.67655944824219,
      "activations/layer18_attention_weight_min": -40.095726013183594,
      "activations/layer19_attention_weight_max": 23.28182601928711,
      "activations/layer19_attention_weight_min": -23.899036407470703,
      "activations/layer1_attention_weight_max": 15.487690925598145,
      "activations/layer1_attention_weight_min": -13.177237510681152,
      "activations/layer20_attention_weight_max": 22.596715927124023,
      "activations/layer20_attention_weight_min": -22.432348251342773,
      "activations/layer21_attention_weight_max": 35.57276153564453,
      "activations/layer21_attention_weight_min": -24.710643768310547,
      "activations/layer22_attention_weight_max": 28.16538429260254,
      "activations/layer22_attention_weight_min": -26.48953628540039,
      "activations/layer23_attention_weight_max": 39.252628326416016,
      "activations/layer23_attention_weight_min": -25.108642578125,
      "activations/layer2_attention_weight_max": 32.83613586425781,
      "activations/layer2_attention_weight_min": -31.60232925415039,
      "activations/layer3_attention_weight_max": 92.08698272705078,
      "activations/layer3_attention_weight_min": -94.50603485107422,
      "activations/layer4_attention_weight_max": 86.56607055664062,
      "activations/layer4_attention_weight_min": -87.35381317138672,
      "activations/layer5_attention_weight_max": 63.58564376831055,
      "activations/layer5_attention_weight_min": -73.35897827148438,
      "activations/layer6_attention_weight_max": 50.89910125732422,
      "activations/layer6_attention_weight_min": -50.39963150024414,
      "activations/layer7_attention_weight_max": 69.47540283203125,
      "activations/layer7_attention_weight_min": -67.94712829589844,
      "activations/layer8_attention_weight_max": 46.303993225097656,
      "activations/layer8_attention_weight_min": -50.457984924316406,
      "activations/layer9_attention_weight_max": 41.19356918334961,
      "activations/layer9_attention_weight_min": -45.87103271484375,
      "epoch": 22.14,
      "learning_rate": 7.273863636363635e-06,
      "loss": 2.7009,
      "step": 381100
    },
    {
      "activations/layer0_attention_weight_max": 16.682132720947266,
      "activations/layer0_attention_weight_min": -14.186628341674805,
      "activations/layer10_attention_weight_max": 39.21038818359375,
      "activations/layer10_attention_weight_min": -39.3724479675293,
      "activations/layer11_attention_weight_max": 36.73689270019531,
      "activations/layer11_attention_weight_min": -39.09027862548828,
      "activations/layer12_attention_weight_max": 20.819538116455078,
      "activations/layer12_attention_weight_min": -23.982275009155273,
      "activations/layer13_attention_weight_max": 37.845367431640625,
      "activations/layer13_attention_weight_min": -28.42031478881836,
      "activations/layer14_attention_weight_max": 36.40007400512695,
      "activations/layer14_attention_weight_min": -32.14006423950195,
      "activations/layer15_attention_weight_max": 33.9623908996582,
      "activations/layer15_attention_weight_min": -31.658884048461914,
      "activations/layer16_attention_weight_max": 36.300750732421875,
      "activations/layer16_attention_weight_min": -34.46513748168945,
      "activations/layer17_attention_weight_max": 53.0718879699707,
      "activations/layer17_attention_weight_min": -48.60078430175781,
      "activations/layer18_attention_weight_max": 48.39760208129883,
      "activations/layer18_attention_weight_min": -41.94282913208008,
      "activations/layer19_attention_weight_max": 24.28860092163086,
      "activations/layer19_attention_weight_min": -22.578880310058594,
      "activations/layer1_attention_weight_max": 14.6297607421875,
      "activations/layer1_attention_weight_min": -13.677948951721191,
      "activations/layer20_attention_weight_max": 23.959020614624023,
      "activations/layer20_attention_weight_min": -20.492931365966797,
      "activations/layer21_attention_weight_max": 33.38991165161133,
      "activations/layer21_attention_weight_min": -20.828378677368164,
      "activations/layer22_attention_weight_max": 27.600929260253906,
      "activations/layer22_attention_weight_min": -26.933483123779297,
      "activations/layer23_attention_weight_max": 36.580623626708984,
      "activations/layer23_attention_weight_min": -23.12929916381836,
      "activations/layer2_attention_weight_max": 33.115116119384766,
      "activations/layer2_attention_weight_min": -32.42164993286133,
      "activations/layer3_attention_weight_max": 91.95014190673828,
      "activations/layer3_attention_weight_min": -92.40795135498047,
      "activations/layer4_attention_weight_max": 88.0700912475586,
      "activations/layer4_attention_weight_min": -88.7114486694336,
      "activations/layer5_attention_weight_max": 64.14673614501953,
      "activations/layer5_attention_weight_min": -71.95695495605469,
      "activations/layer6_attention_weight_max": 52.80640411376953,
      "activations/layer6_attention_weight_min": -52.3621711730957,
      "activations/layer7_attention_weight_max": 70.2945327758789,
      "activations/layer7_attention_weight_min": -69.92599487304688,
      "activations/layer8_attention_weight_max": 50.36484146118164,
      "activations/layer8_attention_weight_min": -54.60088348388672,
      "activations/layer9_attention_weight_max": 47.23726272583008,
      "activations/layer9_attention_weight_min": -48.95659637451172,
      "epoch": 22.15,
      "learning_rate": 7.254924242424243e-06,
      "loss": 2.706,
      "step": 381150
    },
    {
      "activations/layer0_attention_weight_max": 15.69410514831543,
      "activations/layer0_attention_weight_min": -14.351263046264648,
      "activations/layer10_attention_weight_max": 37.41415786743164,
      "activations/layer10_attention_weight_min": -38.86662673950195,
      "activations/layer11_attention_weight_max": 34.44974899291992,
      "activations/layer11_attention_weight_min": -34.55374526977539,
      "activations/layer12_attention_weight_max": 19.43550682067871,
      "activations/layer12_attention_weight_min": -27.201839447021484,
      "activations/layer13_attention_weight_max": 42.99596405029297,
      "activations/layer13_attention_weight_min": -30.67905044555664,
      "activations/layer14_attention_weight_max": 34.854522705078125,
      "activations/layer14_attention_weight_min": -29.009258270263672,
      "activations/layer15_attention_weight_max": 34.21923828125,
      "activations/layer15_attention_weight_min": -29.186508178710938,
      "activations/layer16_attention_weight_max": 33.587158203125,
      "activations/layer16_attention_weight_min": -31.57202911376953,
      "activations/layer17_attention_weight_max": 49.0825309753418,
      "activations/layer17_attention_weight_min": -44.37000274658203,
      "activations/layer18_attention_weight_max": 46.538246154785156,
      "activations/layer18_attention_weight_min": -41.072967529296875,
      "activations/layer19_attention_weight_max": 23.215145111083984,
      "activations/layer19_attention_weight_min": -21.10882568359375,
      "activations/layer1_attention_weight_max": 15.443289756774902,
      "activations/layer1_attention_weight_min": -14.393264770507812,
      "activations/layer20_attention_weight_max": 22.886281967163086,
      "activations/layer20_attention_weight_min": -22.485090255737305,
      "activations/layer21_attention_weight_max": 36.19434356689453,
      "activations/layer21_attention_weight_min": -21.531457901000977,
      "activations/layer22_attention_weight_max": 29.303382873535156,
      "activations/layer22_attention_weight_min": -25.405738830566406,
      "activations/layer23_attention_weight_max": 33.8204460144043,
      "activations/layer23_attention_weight_min": -23.732349395751953,
      "activations/layer2_attention_weight_max": 32.599666595458984,
      "activations/layer2_attention_weight_min": -30.763944625854492,
      "activations/layer3_attention_weight_max": 95.4551773071289,
      "activations/layer3_attention_weight_min": -94.2142333984375,
      "activations/layer4_attention_weight_max": 85.8725814819336,
      "activations/layer4_attention_weight_min": -85.91059112548828,
      "activations/layer5_attention_weight_max": 64.59815216064453,
      "activations/layer5_attention_weight_min": -69.90699768066406,
      "activations/layer6_attention_weight_max": 49.251834869384766,
      "activations/layer6_attention_weight_min": -49.534767150878906,
      "activations/layer7_attention_weight_max": 70.84597778320312,
      "activations/layer7_attention_weight_min": -65.26469421386719,
      "activations/layer8_attention_weight_max": 46.02521514892578,
      "activations/layer8_attention_weight_min": -48.879676818847656,
      "activations/layer9_attention_weight_max": 43.797977447509766,
      "activations/layer9_attention_weight_min": -48.96686935424805,
      "epoch": 22.15,
      "learning_rate": 7.235984848484848e-06,
      "loss": 2.6969,
      "step": 381200
    },
    {
      "activations/layer0_attention_weight_max": 16.123674392700195,
      "activations/layer0_attention_weight_min": -14.330414772033691,
      "activations/layer10_attention_weight_max": 33.76853942871094,
      "activations/layer10_attention_weight_min": -35.68042755126953,
      "activations/layer11_attention_weight_max": 31.282806396484375,
      "activations/layer11_attention_weight_min": -33.66089630126953,
      "activations/layer12_attention_weight_max": 20.746097564697266,
      "activations/layer12_attention_weight_min": -24.411781311035156,
      "activations/layer13_attention_weight_max": 32.16753005981445,
      "activations/layer13_attention_weight_min": -27.29740333557129,
      "activations/layer14_attention_weight_max": 35.116634368896484,
      "activations/layer14_attention_weight_min": -30.67473793029785,
      "activations/layer15_attention_weight_max": 34.070552825927734,
      "activations/layer15_attention_weight_min": -31.19904899597168,
      "activations/layer16_attention_weight_max": 34.018836975097656,
      "activations/layer16_attention_weight_min": -33.22414016723633,
      "activations/layer17_attention_weight_max": 51.851295471191406,
      "activations/layer17_attention_weight_min": -47.70903778076172,
      "activations/layer18_attention_weight_max": 47.87556076049805,
      "activations/layer18_attention_weight_min": -43.5962028503418,
      "activations/layer19_attention_weight_max": 25.340608596801758,
      "activations/layer19_attention_weight_min": -23.9787654876709,
      "activations/layer1_attention_weight_max": 14.792098045349121,
      "activations/layer1_attention_weight_min": -14.182101249694824,
      "activations/layer20_attention_weight_max": 26.175376892089844,
      "activations/layer20_attention_weight_min": -23.31208038330078,
      "activations/layer21_attention_weight_max": 34.47419738769531,
      "activations/layer21_attention_weight_min": -23.085527420043945,
      "activations/layer22_attention_weight_max": 33.644081115722656,
      "activations/layer22_attention_weight_min": -26.35529899597168,
      "activations/layer23_attention_weight_max": 38.085594177246094,
      "activations/layer23_attention_weight_min": -24.706762313842773,
      "activations/layer2_attention_weight_max": 31.204818725585938,
      "activations/layer2_attention_weight_min": -30.738616943359375,
      "activations/layer3_attention_weight_max": 89.34612274169922,
      "activations/layer3_attention_weight_min": -92.39463806152344,
      "activations/layer4_attention_weight_max": 86.1371078491211,
      "activations/layer4_attention_weight_min": -83.20167541503906,
      "activations/layer5_attention_weight_max": 62.0783805847168,
      "activations/layer5_attention_weight_min": -76.56143951416016,
      "activations/layer6_attention_weight_max": 49.60710144042969,
      "activations/layer6_attention_weight_min": -50.273414611816406,
      "activations/layer7_attention_weight_max": 73.06678009033203,
      "activations/layer7_attention_weight_min": -67.92036437988281,
      "activations/layer8_attention_weight_max": 46.97877502441406,
      "activations/layer8_attention_weight_min": -51.93822479248047,
      "activations/layer9_attention_weight_max": 41.65614700317383,
      "activations/layer9_attention_weight_min": -46.1175422668457,
      "epoch": 22.15,
      "learning_rate": 7.217045454545454e-06,
      "loss": 2.6876,
      "step": 381250
    },
    {
      "activations/layer0_attention_weight_max": 16.399309158325195,
      "activations/layer0_attention_weight_min": -13.976341247558594,
      "activations/layer10_attention_weight_max": 38.26912307739258,
      "activations/layer10_attention_weight_min": -38.64885711669922,
      "activations/layer11_attention_weight_max": 33.889732360839844,
      "activations/layer11_attention_weight_min": -37.42057800292969,
      "activations/layer12_attention_weight_max": 19.097749710083008,
      "activations/layer12_attention_weight_min": -25.924728393554688,
      "activations/layer13_attention_weight_max": 32.186161041259766,
      "activations/layer13_attention_weight_min": -28.04948616027832,
      "activations/layer14_attention_weight_max": 34.743709564208984,
      "activations/layer14_attention_weight_min": -30.486610412597656,
      "activations/layer15_attention_weight_max": 32.19581604003906,
      "activations/layer15_attention_weight_min": -32.07746505737305,
      "activations/layer16_attention_weight_max": 33.50680923461914,
      "activations/layer16_attention_weight_min": -32.90192794799805,
      "activations/layer17_attention_weight_max": 49.748451232910156,
      "activations/layer17_attention_weight_min": -44.3605842590332,
      "activations/layer18_attention_weight_max": 47.999237060546875,
      "activations/layer18_attention_weight_min": -41.78374099731445,
      "activations/layer19_attention_weight_max": 25.76260757446289,
      "activations/layer19_attention_weight_min": -23.28037452697754,
      "activations/layer1_attention_weight_max": 15.287503242492676,
      "activations/layer1_attention_weight_min": -14.54123592376709,
      "activations/layer20_attention_weight_max": 24.47359275817871,
      "activations/layer20_attention_weight_min": -23.04792594909668,
      "activations/layer21_attention_weight_max": 36.06266784667969,
      "activations/layer21_attention_weight_min": -24.579858779907227,
      "activations/layer22_attention_weight_max": 28.952430725097656,
      "activations/layer22_attention_weight_min": -25.75775909423828,
      "activations/layer23_attention_weight_max": 35.797786712646484,
      "activations/layer23_attention_weight_min": -26.211668014526367,
      "activations/layer2_attention_weight_max": 32.715904235839844,
      "activations/layer2_attention_weight_min": -31.617048263549805,
      "activations/layer3_attention_weight_max": 91.2308120727539,
      "activations/layer3_attention_weight_min": -90.45796966552734,
      "activations/layer4_attention_weight_max": 86.2871322631836,
      "activations/layer4_attention_weight_min": -86.51353454589844,
      "activations/layer5_attention_weight_max": 62.03495407104492,
      "activations/layer5_attention_weight_min": -75.03645324707031,
      "activations/layer6_attention_weight_max": 51.9610481262207,
      "activations/layer6_attention_weight_min": -52.51649856567383,
      "activations/layer7_attention_weight_max": 68.36146545410156,
      "activations/layer7_attention_weight_min": -71.42767333984375,
      "activations/layer8_attention_weight_max": 51.804107666015625,
      "activations/layer8_attention_weight_min": -55.5346565246582,
      "activations/layer9_attention_weight_max": 47.023399353027344,
      "activations/layer9_attention_weight_min": -46.74285125732422,
      "epoch": 22.16,
      "learning_rate": 7.19810606060606e-06,
      "loss": 2.6997,
      "step": 381300
    },
    {
      "activations/layer0_attention_weight_max": 15.742801666259766,
      "activations/layer0_attention_weight_min": -14.91435718536377,
      "activations/layer10_attention_weight_max": 40.1808967590332,
      "activations/layer10_attention_weight_min": -40.063297271728516,
      "activations/layer11_attention_weight_max": 36.905216217041016,
      "activations/layer11_attention_weight_min": -38.35582733154297,
      "activations/layer12_attention_weight_max": 20.162519454956055,
      "activations/layer12_attention_weight_min": -25.525192260742188,
      "activations/layer13_attention_weight_max": 38.65386199951172,
      "activations/layer13_attention_weight_min": -28.220497131347656,
      "activations/layer14_attention_weight_max": 36.28990173339844,
      "activations/layer14_attention_weight_min": -32.315467834472656,
      "activations/layer15_attention_weight_max": 34.125274658203125,
      "activations/layer15_attention_weight_min": -31.712642669677734,
      "activations/layer16_attention_weight_max": 35.923336029052734,
      "activations/layer16_attention_weight_min": -34.16462326049805,
      "activations/layer17_attention_weight_max": 59.63146209716797,
      "activations/layer17_attention_weight_min": -48.3359260559082,
      "activations/layer18_attention_weight_max": 55.473838806152344,
      "activations/layer18_attention_weight_min": -41.2501106262207,
      "activations/layer19_attention_weight_max": 28.294767379760742,
      "activations/layer19_attention_weight_min": -24.337234497070312,
      "activations/layer1_attention_weight_max": 15.378543853759766,
      "activations/layer1_attention_weight_min": -14.920286178588867,
      "activations/layer20_attention_weight_max": 24.432865142822266,
      "activations/layer20_attention_weight_min": -23.550113677978516,
      "activations/layer21_attention_weight_max": 35.928672790527344,
      "activations/layer21_attention_weight_min": -24.38580322265625,
      "activations/layer22_attention_weight_max": 30.349050521850586,
      "activations/layer22_attention_weight_min": -25.769306182861328,
      "activations/layer23_attention_weight_max": 37.607444763183594,
      "activations/layer23_attention_weight_min": -23.8021297454834,
      "activations/layer2_attention_weight_max": 32.486724853515625,
      "activations/layer2_attention_weight_min": -33.01927185058594,
      "activations/layer3_attention_weight_max": 91.10862731933594,
      "activations/layer3_attention_weight_min": -91.55378723144531,
      "activations/layer4_attention_weight_max": 86.63426971435547,
      "activations/layer4_attention_weight_min": -90.34558868408203,
      "activations/layer5_attention_weight_max": 63.498165130615234,
      "activations/layer5_attention_weight_min": -76.78317260742188,
      "activations/layer6_attention_weight_max": 52.51130676269531,
      "activations/layer6_attention_weight_min": -54.17626190185547,
      "activations/layer7_attention_weight_max": 71.56340026855469,
      "activations/layer7_attention_weight_min": -71.34657287597656,
      "activations/layer8_attention_weight_max": 56.4224853515625,
      "activations/layer8_attention_weight_min": -56.93440246582031,
      "activations/layer9_attention_weight_max": 55.671241760253906,
      "activations/layer9_attention_weight_min": -50.26189422607422,
      "epoch": 22.16,
      "learning_rate": 7.179166666666666e-06,
      "loss": 2.7136,
      "step": 381350
    },
    {
      "activations/layer0_attention_weight_max": 16.081134796142578,
      "activations/layer0_attention_weight_min": -14.008035659790039,
      "activations/layer10_attention_weight_max": 37.297489166259766,
      "activations/layer10_attention_weight_min": -36.30287551879883,
      "activations/layer11_attention_weight_max": 33.91621398925781,
      "activations/layer11_attention_weight_min": -37.256263732910156,
      "activations/layer12_attention_weight_max": 19.24295425415039,
      "activations/layer12_attention_weight_min": -24.26856231689453,
      "activations/layer13_attention_weight_max": 38.586483001708984,
      "activations/layer13_attention_weight_min": -27.761011123657227,
      "activations/layer14_attention_weight_max": 36.99122619628906,
      "activations/layer14_attention_weight_min": -30.066728591918945,
      "activations/layer15_attention_weight_max": 34.7783203125,
      "activations/layer15_attention_weight_min": -31.113174438476562,
      "activations/layer16_attention_weight_max": 35.07246780395508,
      "activations/layer16_attention_weight_min": -33.885807037353516,
      "activations/layer17_attention_weight_max": 52.121517181396484,
      "activations/layer17_attention_weight_min": -47.68828201293945,
      "activations/layer18_attention_weight_max": 47.46421813964844,
      "activations/layer18_attention_weight_min": -39.379791259765625,
      "activations/layer19_attention_weight_max": 22.791135787963867,
      "activations/layer19_attention_weight_min": -21.949365615844727,
      "activations/layer1_attention_weight_max": 14.982917785644531,
      "activations/layer1_attention_weight_min": -14.567732810974121,
      "activations/layer20_attention_weight_max": 22.20429801940918,
      "activations/layer20_attention_weight_min": -23.164709091186523,
      "activations/layer21_attention_weight_max": 34.3737678527832,
      "activations/layer21_attention_weight_min": -22.204524993896484,
      "activations/layer22_attention_weight_max": 28.024982452392578,
      "activations/layer22_attention_weight_min": -26.31407356262207,
      "activations/layer23_attention_weight_max": 34.53655242919922,
      "activations/layer23_attention_weight_min": -22.87440299987793,
      "activations/layer2_attention_weight_max": 31.32158660888672,
      "activations/layer2_attention_weight_min": -29.547412872314453,
      "activations/layer3_attention_weight_max": 91.52257537841797,
      "activations/layer3_attention_weight_min": -87.93668365478516,
      "activations/layer4_attention_weight_max": 87.84783172607422,
      "activations/layer4_attention_weight_min": -87.00338745117188,
      "activations/layer5_attention_weight_max": 63.33498764038086,
      "activations/layer5_attention_weight_min": -75.2642593383789,
      "activations/layer6_attention_weight_max": 50.82481002807617,
      "activations/layer6_attention_weight_min": -50.40789031982422,
      "activations/layer7_attention_weight_max": 71.43223571777344,
      "activations/layer7_attention_weight_min": -69.72489166259766,
      "activations/layer8_attention_weight_max": 48.97649002075195,
      "activations/layer8_attention_weight_min": -53.40909194946289,
      "activations/layer9_attention_weight_max": 45.867225646972656,
      "activations/layer9_attention_weight_min": -47.90873718261719,
      "epoch": 22.16,
      "learning_rate": 7.160227272727272e-06,
      "loss": 2.7102,
      "step": 381400
    },
    {
      "activations/layer0_attention_weight_max": 15.284187316894531,
      "activations/layer0_attention_weight_min": -13.972031593322754,
      "activations/layer10_attention_weight_max": 35.32749557495117,
      "activations/layer10_attention_weight_min": -36.18876266479492,
      "activations/layer11_attention_weight_max": 33.809165954589844,
      "activations/layer11_attention_weight_min": -35.1889533996582,
      "activations/layer12_attention_weight_max": 17.9261474609375,
      "activations/layer12_attention_weight_min": -28.00882911682129,
      "activations/layer13_attention_weight_max": 29.84855079650879,
      "activations/layer13_attention_weight_min": -27.388904571533203,
      "activations/layer14_attention_weight_max": 32.53180694580078,
      "activations/layer14_attention_weight_min": -30.012224197387695,
      "activations/layer15_attention_weight_max": 30.30506134033203,
      "activations/layer15_attention_weight_min": -31.999725341796875,
      "activations/layer16_attention_weight_max": 31.59992218017578,
      "activations/layer16_attention_weight_min": -30.85661506652832,
      "activations/layer17_attention_weight_max": 50.13679504394531,
      "activations/layer17_attention_weight_min": -42.61293411254883,
      "activations/layer18_attention_weight_max": 44.70869827270508,
      "activations/layer18_attention_weight_min": -39.429473876953125,
      "activations/layer19_attention_weight_max": 22.783681869506836,
      "activations/layer19_attention_weight_min": -23.291847229003906,
      "activations/layer1_attention_weight_max": 14.93115520477295,
      "activations/layer1_attention_weight_min": -15.061617851257324,
      "activations/layer20_attention_weight_max": 20.005937576293945,
      "activations/layer20_attention_weight_min": -21.656034469604492,
      "activations/layer21_attention_weight_max": 40.10269546508789,
      "activations/layer21_attention_weight_min": -23.158864974975586,
      "activations/layer22_attention_weight_max": 25.75722312927246,
      "activations/layer22_attention_weight_min": -23.975170135498047,
      "activations/layer23_attention_weight_max": 33.470741271972656,
      "activations/layer23_attention_weight_min": -22.619892120361328,
      "activations/layer2_attention_weight_max": 31.673751831054688,
      "activations/layer2_attention_weight_min": -31.304828643798828,
      "activations/layer3_attention_weight_max": 94.8221206665039,
      "activations/layer3_attention_weight_min": -89.12837219238281,
      "activations/layer4_attention_weight_max": 87.66902923583984,
      "activations/layer4_attention_weight_min": -84.3359375,
      "activations/layer5_attention_weight_max": 63.404293060302734,
      "activations/layer5_attention_weight_min": -70.17513275146484,
      "activations/layer6_attention_weight_max": 49.20295715332031,
      "activations/layer6_attention_weight_min": -51.90697479248047,
      "activations/layer7_attention_weight_max": 67.16844177246094,
      "activations/layer7_attention_weight_min": -67.93445587158203,
      "activations/layer8_attention_weight_max": 49.05345916748047,
      "activations/layer8_attention_weight_min": -54.4144172668457,
      "activations/layer9_attention_weight_max": 43.49009323120117,
      "activations/layer9_attention_weight_min": -47.56437301635742,
      "epoch": 22.16,
      "learning_rate": 7.141287878787878e-06,
      "loss": 2.6945,
      "step": 381450
    },
    {
      "activations/layer0_attention_weight_max": 16.83919334411621,
      "activations/layer0_attention_weight_min": -13.946893692016602,
      "activations/layer10_attention_weight_max": 34.7679443359375,
      "activations/layer10_attention_weight_min": -36.26980972290039,
      "activations/layer11_attention_weight_max": 32.29724884033203,
      "activations/layer11_attention_weight_min": -34.64354705810547,
      "activations/layer12_attention_weight_max": 18.035314559936523,
      "activations/layer12_attention_weight_min": -26.477659225463867,
      "activations/layer13_attention_weight_max": 32.16120147705078,
      "activations/layer13_attention_weight_min": -27.02683448791504,
      "activations/layer14_attention_weight_max": 35.615875244140625,
      "activations/layer14_attention_weight_min": -31.031574249267578,
      "activations/layer15_attention_weight_max": 35.071319580078125,
      "activations/layer15_attention_weight_min": -30.17249870300293,
      "activations/layer16_attention_weight_max": 34.0478630065918,
      "activations/layer16_attention_weight_min": -34.16707229614258,
      "activations/layer17_attention_weight_max": 53.949913024902344,
      "activations/layer17_attention_weight_min": -46.50571823120117,
      "activations/layer18_attention_weight_max": 50.143611907958984,
      "activations/layer18_attention_weight_min": -42.0559196472168,
      "activations/layer19_attention_weight_max": 24.92666244506836,
      "activations/layer19_attention_weight_min": -22.196392059326172,
      "activations/layer1_attention_weight_max": 14.860360145568848,
      "activations/layer1_attention_weight_min": -14.595826148986816,
      "activations/layer20_attention_weight_max": 24.21769905090332,
      "activations/layer20_attention_weight_min": -24.340349197387695,
      "activations/layer21_attention_weight_max": 34.43492126464844,
      "activations/layer21_attention_weight_min": -21.11946678161621,
      "activations/layer22_attention_weight_max": 27.638242721557617,
      "activations/layer22_attention_weight_min": -25.83074378967285,
      "activations/layer23_attention_weight_max": 37.06211853027344,
      "activations/layer23_attention_weight_min": -22.134571075439453,
      "activations/layer2_attention_weight_max": 32.10505294799805,
      "activations/layer2_attention_weight_min": -30.724035263061523,
      "activations/layer3_attention_weight_max": 88.53807830810547,
      "activations/layer3_attention_weight_min": -87.02471160888672,
      "activations/layer4_attention_weight_max": 80.82232666015625,
      "activations/layer4_attention_weight_min": -83.88790130615234,
      "activations/layer5_attention_weight_max": 59.629302978515625,
      "activations/layer5_attention_weight_min": -72.15428161621094,
      "activations/layer6_attention_weight_max": 47.52220916748047,
      "activations/layer6_attention_weight_min": -49.49232482910156,
      "activations/layer7_attention_weight_max": 68.2628402709961,
      "activations/layer7_attention_weight_min": -67.65180206298828,
      "activations/layer8_attention_weight_max": 46.41941452026367,
      "activations/layer8_attention_weight_min": -50.08746337890625,
      "activations/layer9_attention_weight_max": 42.8123664855957,
      "activations/layer9_attention_weight_min": -45.85065841674805,
      "epoch": 22.17,
      "learning_rate": 7.122348484848484e-06,
      "loss": 2.7151,
      "step": 381500
    },
    {
      "activations/layer0_attention_weight_max": 16.897537231445312,
      "activations/layer0_attention_weight_min": -14.245244026184082,
      "activations/layer10_attention_weight_max": 36.5400276184082,
      "activations/layer10_attention_weight_min": -38.214698791503906,
      "activations/layer11_attention_weight_max": 35.688926696777344,
      "activations/layer11_attention_weight_min": -36.43672180175781,
      "activations/layer12_attention_weight_max": 20.95832061767578,
      "activations/layer12_attention_weight_min": -27.88231086730957,
      "activations/layer13_attention_weight_max": 37.77997589111328,
      "activations/layer13_attention_weight_min": -27.161354064941406,
      "activations/layer14_attention_weight_max": 42.634063720703125,
      "activations/layer14_attention_weight_min": -31.05255699157715,
      "activations/layer15_attention_weight_max": 45.69602584838867,
      "activations/layer15_attention_weight_min": -31.790040969848633,
      "activations/layer16_attention_weight_max": 37.74116897583008,
      "activations/layer16_attention_weight_min": -33.99240493774414,
      "activations/layer17_attention_weight_max": 55.09989547729492,
      "activations/layer17_attention_weight_min": -47.42427444458008,
      "activations/layer18_attention_weight_max": 52.6291618347168,
      "activations/layer18_attention_weight_min": -43.269065856933594,
      "activations/layer19_attention_weight_max": 28.19662857055664,
      "activations/layer19_attention_weight_min": -24.713422775268555,
      "activations/layer1_attention_weight_max": 16.269399642944336,
      "activations/layer1_attention_weight_min": -14.93484115600586,
      "activations/layer20_attention_weight_max": 29.148042678833008,
      "activations/layer20_attention_weight_min": -21.30452537536621,
      "activations/layer21_attention_weight_max": 42.93672561645508,
      "activations/layer21_attention_weight_min": -23.726322174072266,
      "activations/layer22_attention_weight_max": 31.04296112060547,
      "activations/layer22_attention_weight_min": -24.74443817138672,
      "activations/layer23_attention_weight_max": 37.033355712890625,
      "activations/layer23_attention_weight_min": -24.857864379882812,
      "activations/layer2_attention_weight_max": 31.073272705078125,
      "activations/layer2_attention_weight_min": -30.37647819519043,
      "activations/layer3_attention_weight_max": 92.14958190917969,
      "activations/layer3_attention_weight_min": -94.06478118896484,
      "activations/layer4_attention_weight_max": 85.9842758178711,
      "activations/layer4_attention_weight_min": -84.24504852294922,
      "activations/layer5_attention_weight_max": 65.6802978515625,
      "activations/layer5_attention_weight_min": -72.10835266113281,
      "activations/layer6_attention_weight_max": 48.90742111206055,
      "activations/layer6_attention_weight_min": -51.864322662353516,
      "activations/layer7_attention_weight_max": 67.98721313476562,
      "activations/layer7_attention_weight_min": -70.17090606689453,
      "activations/layer8_attention_weight_max": 46.13811492919922,
      "activations/layer8_attention_weight_min": -51.34271240234375,
      "activations/layer9_attention_weight_max": 45.79804611206055,
      "activations/layer9_attention_weight_min": -45.98454666137695,
      "epoch": 22.17,
      "learning_rate": 7.10340909090909e-06,
      "loss": 2.7109,
      "step": 381550
    },
    {
      "activations/layer0_attention_weight_max": 16.120819091796875,
      "activations/layer0_attention_weight_min": -14.112881660461426,
      "activations/layer10_attention_weight_max": 34.00017547607422,
      "activations/layer10_attention_weight_min": -35.23507308959961,
      "activations/layer11_attention_weight_max": 32.54814910888672,
      "activations/layer11_attention_weight_min": -33.24309539794922,
      "activations/layer12_attention_weight_max": 17.96105194091797,
      "activations/layer12_attention_weight_min": -31.446849822998047,
      "activations/layer13_attention_weight_max": 37.90019226074219,
      "activations/layer13_attention_weight_min": -27.49628257751465,
      "activations/layer14_attention_weight_max": 35.404205322265625,
      "activations/layer14_attention_weight_min": -30.633527755737305,
      "activations/layer15_attention_weight_max": 33.52456283569336,
      "activations/layer15_attention_weight_min": -30.56729507446289,
      "activations/layer16_attention_weight_max": 34.29553985595703,
      "activations/layer16_attention_weight_min": -33.89427185058594,
      "activations/layer17_attention_weight_max": 54.390968322753906,
      "activations/layer17_attention_weight_min": -46.1712532043457,
      "activations/layer18_attention_weight_max": 47.784385681152344,
      "activations/layer18_attention_weight_min": -41.81924819946289,
      "activations/layer19_attention_weight_max": 24.405271530151367,
      "activations/layer19_attention_weight_min": -21.84759521484375,
      "activations/layer1_attention_weight_max": 14.732645034790039,
      "activations/layer1_attention_weight_min": -13.802556991577148,
      "activations/layer20_attention_weight_max": 22.01072883605957,
      "activations/layer20_attention_weight_min": -22.020036697387695,
      "activations/layer21_attention_weight_max": 33.046363830566406,
      "activations/layer21_attention_weight_min": -21.189937591552734,
      "activations/layer22_attention_weight_max": 27.347585678100586,
      "activations/layer22_attention_weight_min": -23.96022605895996,
      "activations/layer23_attention_weight_max": 35.87413787841797,
      "activations/layer23_attention_weight_min": -22.498783111572266,
      "activations/layer2_attention_weight_max": 32.57017517089844,
      "activations/layer2_attention_weight_min": -30.248342514038086,
      "activations/layer3_attention_weight_max": 88.197021484375,
      "activations/layer3_attention_weight_min": -87.42533874511719,
      "activations/layer4_attention_weight_max": 84.4035415649414,
      "activations/layer4_attention_weight_min": -83.06298065185547,
      "activations/layer5_attention_weight_max": 60.845245361328125,
      "activations/layer5_attention_weight_min": -72.17891693115234,
      "activations/layer6_attention_weight_max": 48.7137336730957,
      "activations/layer6_attention_weight_min": -49.26986312866211,
      "activations/layer7_attention_weight_max": 65.21072387695312,
      "activations/layer7_attention_weight_min": -69.95572662353516,
      "activations/layer8_attention_weight_max": 45.99026107788086,
      "activations/layer8_attention_weight_min": -49.96765899658203,
      "activations/layer9_attention_weight_max": 44.49138641357422,
      "activations/layer9_attention_weight_min": -46.14008712768555,
      "epoch": 22.17,
      "learning_rate": 7.084469696969697e-06,
      "loss": 2.7034,
      "step": 381600
    },
    {
      "activations/layer0_attention_weight_max": 16.198089599609375,
      "activations/layer0_attention_weight_min": -14.317862510681152,
      "activations/layer10_attention_weight_max": 34.347206115722656,
      "activations/layer10_attention_weight_min": -35.094154357910156,
      "activations/layer11_attention_weight_max": 32.781166076660156,
      "activations/layer11_attention_weight_min": -34.47587585449219,
      "activations/layer12_attention_weight_max": 18.339313507080078,
      "activations/layer12_attention_weight_min": -24.402027130126953,
      "activations/layer13_attention_weight_max": 33.7010612487793,
      "activations/layer13_attention_weight_min": -28.369182586669922,
      "activations/layer14_attention_weight_max": 35.91924285888672,
      "activations/layer14_attention_weight_min": -31.311182022094727,
      "activations/layer15_attention_weight_max": 32.60100173950195,
      "activations/layer15_attention_weight_min": -30.50632667541504,
      "activations/layer16_attention_weight_max": 34.182979583740234,
      "activations/layer16_attention_weight_min": -31.11958885192871,
      "activations/layer17_attention_weight_max": 51.49577331542969,
      "activations/layer17_attention_weight_min": -44.13430404663086,
      "activations/layer18_attention_weight_max": 47.467830657958984,
      "activations/layer18_attention_weight_min": -39.48524856567383,
      "activations/layer19_attention_weight_max": 26.616168975830078,
      "activations/layer19_attention_weight_min": -21.865949630737305,
      "activations/layer1_attention_weight_max": 15.973644256591797,
      "activations/layer1_attention_weight_min": -13.826722145080566,
      "activations/layer20_attention_weight_max": 22.53693962097168,
      "activations/layer20_attention_weight_min": -20.844158172607422,
      "activations/layer21_attention_weight_max": 39.593074798583984,
      "activations/layer21_attention_weight_min": -23.24306297302246,
      "activations/layer22_attention_weight_max": 31.854736328125,
      "activations/layer22_attention_weight_min": -23.9073543548584,
      "activations/layer23_attention_weight_max": 37.11502456665039,
      "activations/layer23_attention_weight_min": -23.298307418823242,
      "activations/layer2_attention_weight_max": 31.453672409057617,
      "activations/layer2_attention_weight_min": -30.241458892822266,
      "activations/layer3_attention_weight_max": 88.55529022216797,
      "activations/layer3_attention_weight_min": -86.80007934570312,
      "activations/layer4_attention_weight_max": 80.62920379638672,
      "activations/layer4_attention_weight_min": -81.40185546875,
      "activations/layer5_attention_weight_max": 63.41268539428711,
      "activations/layer5_attention_weight_min": -74.71356201171875,
      "activations/layer6_attention_weight_max": 47.51427459716797,
      "activations/layer6_attention_weight_min": -49.98859786987305,
      "activations/layer7_attention_weight_max": 72.45125579833984,
      "activations/layer7_attention_weight_min": -66.65376281738281,
      "activations/layer8_attention_weight_max": 44.731388092041016,
      "activations/layer8_attention_weight_min": -49.92290115356445,
      "activations/layer9_attention_weight_max": 46.23398971557617,
      "activations/layer9_attention_weight_min": -45.132930755615234,
      "epoch": 22.18,
      "learning_rate": 7.0655303030303025e-06,
      "loss": 2.7257,
      "step": 381650
    },
    {
      "activations/layer0_attention_weight_max": 16.38578987121582,
      "activations/layer0_attention_weight_min": -14.150022506713867,
      "activations/layer10_attention_weight_max": 34.73876953125,
      "activations/layer10_attention_weight_min": -35.19947052001953,
      "activations/layer11_attention_weight_max": 32.47576904296875,
      "activations/layer11_attention_weight_min": -35.72724151611328,
      "activations/layer12_attention_weight_max": 19.680418014526367,
      "activations/layer12_attention_weight_min": -23.817819595336914,
      "activations/layer13_attention_weight_max": 36.61151123046875,
      "activations/layer13_attention_weight_min": -29.536090850830078,
      "activations/layer14_attention_weight_max": 34.9752082824707,
      "activations/layer14_attention_weight_min": -30.38018226623535,
      "activations/layer15_attention_weight_max": 33.81044006347656,
      "activations/layer15_attention_weight_min": -29.95134162902832,
      "activations/layer16_attention_weight_max": 34.62107849121094,
      "activations/layer16_attention_weight_min": -31.436466217041016,
      "activations/layer17_attention_weight_max": 52.34103012084961,
      "activations/layer17_attention_weight_min": -44.488826751708984,
      "activations/layer18_attention_weight_max": 46.69618225097656,
      "activations/layer18_attention_weight_min": -42.56810760498047,
      "activations/layer19_attention_weight_max": 26.138334274291992,
      "activations/layer19_attention_weight_min": -23.03511619567871,
      "activations/layer1_attention_weight_max": 15.355756759643555,
      "activations/layer1_attention_weight_min": -13.62751293182373,
      "activations/layer20_attention_weight_max": 23.02037239074707,
      "activations/layer20_attention_weight_min": -22.98318862915039,
      "activations/layer21_attention_weight_max": 33.697052001953125,
      "activations/layer21_attention_weight_min": -20.77036476135254,
      "activations/layer22_attention_weight_max": 30.830583572387695,
      "activations/layer22_attention_weight_min": -26.509695053100586,
      "activations/layer23_attention_weight_max": 38.51851272583008,
      "activations/layer23_attention_weight_min": -22.931188583374023,
      "activations/layer2_attention_weight_max": 32.09783172607422,
      "activations/layer2_attention_weight_min": -29.855209350585938,
      "activations/layer3_attention_weight_max": 90.59809875488281,
      "activations/layer3_attention_weight_min": -87.7917709350586,
      "activations/layer4_attention_weight_max": 84.10787200927734,
      "activations/layer4_attention_weight_min": -83.10734558105469,
      "activations/layer5_attention_weight_max": 60.580345153808594,
      "activations/layer5_attention_weight_min": -74.05281066894531,
      "activations/layer6_attention_weight_max": 48.50598907470703,
      "activations/layer6_attention_weight_min": -50.533077239990234,
      "activations/layer7_attention_weight_max": 71.48463439941406,
      "activations/layer7_attention_weight_min": -65.45745086669922,
      "activations/layer8_attention_weight_max": 47.661888122558594,
      "activations/layer8_attention_weight_min": -49.99177932739258,
      "activations/layer9_attention_weight_max": 45.2525749206543,
      "activations/layer9_attention_weight_min": -44.98735427856445,
      "epoch": 22.18,
      "learning_rate": 7.046590909090909e-06,
      "loss": 2.7068,
      "step": 381700
    },
    {
      "activations/layer0_attention_weight_max": 15.920517921447754,
      "activations/layer0_attention_weight_min": -14.22213363647461,
      "activations/layer10_attention_weight_max": 34.69575881958008,
      "activations/layer10_attention_weight_min": -37.1665153503418,
      "activations/layer11_attention_weight_max": 33.310760498046875,
      "activations/layer11_attention_weight_min": -37.762733459472656,
      "activations/layer12_attention_weight_max": 19.49740982055664,
      "activations/layer12_attention_weight_min": -27.06002426147461,
      "activations/layer13_attention_weight_max": 34.617584228515625,
      "activations/layer13_attention_weight_min": -27.279483795166016,
      "activations/layer14_attention_weight_max": 34.78770446777344,
      "activations/layer14_attention_weight_min": -30.672021865844727,
      "activations/layer15_attention_weight_max": 33.76314926147461,
      "activations/layer15_attention_weight_min": -31.177431106567383,
      "activations/layer16_attention_weight_max": 36.2777214050293,
      "activations/layer16_attention_weight_min": -33.53718948364258,
      "activations/layer17_attention_weight_max": 52.12944793701172,
      "activations/layer17_attention_weight_min": -46.617279052734375,
      "activations/layer18_attention_weight_max": 45.494144439697266,
      "activations/layer18_attention_weight_min": -43.40692901611328,
      "activations/layer19_attention_weight_max": 23.6007137298584,
      "activations/layer19_attention_weight_min": -22.9610595703125,
      "activations/layer1_attention_weight_max": 14.50749683380127,
      "activations/layer1_attention_weight_min": -13.868650436401367,
      "activations/layer20_attention_weight_max": 21.242416381835938,
      "activations/layer20_attention_weight_min": -20.677135467529297,
      "activations/layer21_attention_weight_max": 38.59955978393555,
      "activations/layer21_attention_weight_min": -23.40050506591797,
      "activations/layer22_attention_weight_max": 27.1648006439209,
      "activations/layer22_attention_weight_min": -24.64809226989746,
      "activations/layer23_attention_weight_max": 34.47376251220703,
      "activations/layer23_attention_weight_min": -23.319374084472656,
      "activations/layer2_attention_weight_max": 31.47846031188965,
      "activations/layer2_attention_weight_min": -30.146574020385742,
      "activations/layer3_attention_weight_max": 88.57525634765625,
      "activations/layer3_attention_weight_min": -89.63119506835938,
      "activations/layer4_attention_weight_max": 85.94251251220703,
      "activations/layer4_attention_weight_min": -84.29934692382812,
      "activations/layer5_attention_weight_max": 59.45336151123047,
      "activations/layer5_attention_weight_min": -72.10276794433594,
      "activations/layer6_attention_weight_max": 48.42723846435547,
      "activations/layer6_attention_weight_min": -52.409690856933594,
      "activations/layer7_attention_weight_max": 74.61660766601562,
      "activations/layer7_attention_weight_min": -73.08609771728516,
      "activations/layer8_attention_weight_max": 47.31279754638672,
      "activations/layer8_attention_weight_min": -53.725181579589844,
      "activations/layer9_attention_weight_max": 44.08989715576172,
      "activations/layer9_attention_weight_min": -45.51228332519531,
      "epoch": 22.18,
      "learning_rate": 7.027651515151515e-06,
      "loss": 2.7009,
      "step": 381750
    },
    {
      "activations/layer0_attention_weight_max": 17.071636199951172,
      "activations/layer0_attention_weight_min": -14.348191261291504,
      "activations/layer10_attention_weight_max": 36.70277786254883,
      "activations/layer10_attention_weight_min": -35.540931701660156,
      "activations/layer11_attention_weight_max": 34.02513885498047,
      "activations/layer11_attention_weight_min": -35.96742248535156,
      "activations/layer12_attention_weight_max": 19.792142868041992,
      "activations/layer12_attention_weight_min": -25.610328674316406,
      "activations/layer13_attention_weight_max": 40.24555969238281,
      "activations/layer13_attention_weight_min": -28.180383682250977,
      "activations/layer14_attention_weight_max": 38.7551383972168,
      "activations/layer14_attention_weight_min": -29.78309440612793,
      "activations/layer15_attention_weight_max": 37.217830657958984,
      "activations/layer15_attention_weight_min": -30.405759811401367,
      "activations/layer16_attention_weight_max": 37.065547943115234,
      "activations/layer16_attention_weight_min": -34.2643928527832,
      "activations/layer17_attention_weight_max": 53.517005920410156,
      "activations/layer17_attention_weight_min": -47.582237243652344,
      "activations/layer18_attention_weight_max": 50.49481964111328,
      "activations/layer18_attention_weight_min": -43.05208206176758,
      "activations/layer19_attention_weight_max": 26.389225006103516,
      "activations/layer19_attention_weight_min": -24.9302921295166,
      "activations/layer1_attention_weight_max": 15.445863723754883,
      "activations/layer1_attention_weight_min": -14.535273551940918,
      "activations/layer20_attention_weight_max": 26.30690574645996,
      "activations/layer20_attention_weight_min": -23.25742530822754,
      "activations/layer21_attention_weight_max": 39.68766403198242,
      "activations/layer21_attention_weight_min": -23.255840301513672,
      "activations/layer22_attention_weight_max": 30.049179077148438,
      "activations/layer22_attention_weight_min": -26.904523849487305,
      "activations/layer23_attention_weight_max": 37.53813934326172,
      "activations/layer23_attention_weight_min": -25.32791519165039,
      "activations/layer2_attention_weight_max": 32.25874328613281,
      "activations/layer2_attention_weight_min": -30.28529930114746,
      "activations/layer3_attention_weight_max": 90.0864486694336,
      "activations/layer3_attention_weight_min": -88.93443298339844,
      "activations/layer4_attention_weight_max": 86.49755859375,
      "activations/layer4_attention_weight_min": -85.6880874633789,
      "activations/layer5_attention_weight_max": 64.6378173828125,
      "activations/layer5_attention_weight_min": -73.37712860107422,
      "activations/layer6_attention_weight_max": 50.51605987548828,
      "activations/layer6_attention_weight_min": -52.52177429199219,
      "activations/layer7_attention_weight_max": 73.01901245117188,
      "activations/layer7_attention_weight_min": -70.65301513671875,
      "activations/layer8_attention_weight_max": 49.51476287841797,
      "activations/layer8_attention_weight_min": -50.30031967163086,
      "activations/layer9_attention_weight_max": 42.550865173339844,
      "activations/layer9_attention_weight_min": -48.1366081237793,
      "epoch": 22.18,
      "learning_rate": 7.0087121212121205e-06,
      "loss": 2.7042,
      "step": 381800
    },
    {
      "activations/layer0_attention_weight_max": 16.28847312927246,
      "activations/layer0_attention_weight_min": -14.311325073242188,
      "activations/layer10_attention_weight_max": 34.45824432373047,
      "activations/layer10_attention_weight_min": -36.34428787231445,
      "activations/layer11_attention_weight_max": 33.7015266418457,
      "activations/layer11_attention_weight_min": -34.85163116455078,
      "activations/layer12_attention_weight_max": 18.518159866333008,
      "activations/layer12_attention_weight_min": -31.082063674926758,
      "activations/layer13_attention_weight_max": 30.939300537109375,
      "activations/layer13_attention_weight_min": -26.549325942993164,
      "activations/layer14_attention_weight_max": 31.367477416992188,
      "activations/layer14_attention_weight_min": -28.682336807250977,
      "activations/layer15_attention_weight_max": 30.809511184692383,
      "activations/layer15_attention_weight_min": -29.927852630615234,
      "activations/layer16_attention_weight_max": 33.31641387939453,
      "activations/layer16_attention_weight_min": -31.934484481811523,
      "activations/layer17_attention_weight_max": 49.33773422241211,
      "activations/layer17_attention_weight_min": -43.4788818359375,
      "activations/layer18_attention_weight_max": 46.04674530029297,
      "activations/layer18_attention_weight_min": -39.989288330078125,
      "activations/layer19_attention_weight_max": 23.596792221069336,
      "activations/layer19_attention_weight_min": -24.8477840423584,
      "activations/layer1_attention_weight_max": 15.663898468017578,
      "activations/layer1_attention_weight_min": -15.147011756896973,
      "activations/layer20_attention_weight_max": 23.04045867919922,
      "activations/layer20_attention_weight_min": -21.748308181762695,
      "activations/layer21_attention_weight_max": 33.58563232421875,
      "activations/layer21_attention_weight_min": -22.52202796936035,
      "activations/layer22_attention_weight_max": 27.10641860961914,
      "activations/layer22_attention_weight_min": -25.28756332397461,
      "activations/layer23_attention_weight_max": 32.538902282714844,
      "activations/layer23_attention_weight_min": -24.164426803588867,
      "activations/layer2_attention_weight_max": 30.915802001953125,
      "activations/layer2_attention_weight_min": -31.181026458740234,
      "activations/layer3_attention_weight_max": 88.59166717529297,
      "activations/layer3_attention_weight_min": -87.25370788574219,
      "activations/layer4_attention_weight_max": 83.7059555053711,
      "activations/layer4_attention_weight_min": -85.3564682006836,
      "activations/layer5_attention_weight_max": 59.09583282470703,
      "activations/layer5_attention_weight_min": -74.19933319091797,
      "activations/layer6_attention_weight_max": 48.532386779785156,
      "activations/layer6_attention_weight_min": -50.043312072753906,
      "activations/layer7_attention_weight_max": 69.65911865234375,
      "activations/layer7_attention_weight_min": -66.49678039550781,
      "activations/layer8_attention_weight_max": 45.70174026489258,
      "activations/layer8_attention_weight_min": -49.007774353027344,
      "activations/layer9_attention_weight_max": 50.4576530456543,
      "activations/layer9_attention_weight_min": -44.39638900756836,
      "epoch": 22.19,
      "learning_rate": 6.989772727272726e-06,
      "loss": 2.698,
      "step": 381850
    },
    {
      "activations/layer0_attention_weight_max": 15.734307289123535,
      "activations/layer0_attention_weight_min": -14.204732894897461,
      "activations/layer10_attention_weight_max": 36.26246643066406,
      "activations/layer10_attention_weight_min": -38.116607666015625,
      "activations/layer11_attention_weight_max": 35.47866439819336,
      "activations/layer11_attention_weight_min": -35.927398681640625,
      "activations/layer12_attention_weight_max": 19.795072555541992,
      "activations/layer12_attention_weight_min": -25.125,
      "activations/layer13_attention_weight_max": 40.00422668457031,
      "activations/layer13_attention_weight_min": -27.88261604309082,
      "activations/layer14_attention_weight_max": 45.28525161743164,
      "activations/layer14_attention_weight_min": -30.858993530273438,
      "activations/layer15_attention_weight_max": 37.660255432128906,
      "activations/layer15_attention_weight_min": -30.52857208251953,
      "activations/layer16_attention_weight_max": 38.63864517211914,
      "activations/layer16_attention_weight_min": -33.20159912109375,
      "activations/layer17_attention_weight_max": 54.31709671020508,
      "activations/layer17_attention_weight_min": -48.353572845458984,
      "activations/layer18_attention_weight_max": 54.600059509277344,
      "activations/layer18_attention_weight_min": -43.47818374633789,
      "activations/layer19_attention_weight_max": 29.002643585205078,
      "activations/layer19_attention_weight_min": -25.29439353942871,
      "activations/layer1_attention_weight_max": 16.225299835205078,
      "activations/layer1_attention_weight_min": -14.006365776062012,
      "activations/layer20_attention_weight_max": 29.293134689331055,
      "activations/layer20_attention_weight_min": -25.28628158569336,
      "activations/layer21_attention_weight_max": 39.28926467895508,
      "activations/layer21_attention_weight_min": -25.98575782775879,
      "activations/layer22_attention_weight_max": 38.1805305480957,
      "activations/layer22_attention_weight_min": -27.07204818725586,
      "activations/layer23_attention_weight_max": 45.85021209716797,
      "activations/layer23_attention_weight_min": -23.64631462097168,
      "activations/layer2_attention_weight_max": 31.345645904541016,
      "activations/layer2_attention_weight_min": -30.108280181884766,
      "activations/layer3_attention_weight_max": 92.63219451904297,
      "activations/layer3_attention_weight_min": -88.81278991699219,
      "activations/layer4_attention_weight_max": 86.85948944091797,
      "activations/layer4_attention_weight_min": -87.80775451660156,
      "activations/layer5_attention_weight_max": 64.69054412841797,
      "activations/layer5_attention_weight_min": -73.18327331542969,
      "activations/layer6_attention_weight_max": 49.18634796142578,
      "activations/layer6_attention_weight_min": -51.37128448486328,
      "activations/layer7_attention_weight_max": 75.01001739501953,
      "activations/layer7_attention_weight_min": -74.67647552490234,
      "activations/layer8_attention_weight_max": 50.46586227416992,
      "activations/layer8_attention_weight_min": -51.081787109375,
      "activations/layer9_attention_weight_max": 51.072540283203125,
      "activations/layer9_attention_weight_min": -50.83289337158203,
      "epoch": 22.19,
      "learning_rate": 6.970833333333333e-06,
      "loss": 2.6908,
      "step": 381900
    },
    {
      "activations/layer0_attention_weight_max": 15.711023330688477,
      "activations/layer0_attention_weight_min": -14.591371536254883,
      "activations/layer10_attention_weight_max": 35.04758834838867,
      "activations/layer10_attention_weight_min": -37.41877746582031,
      "activations/layer11_attention_weight_max": 34.40821075439453,
      "activations/layer11_attention_weight_min": -34.198219299316406,
      "activations/layer12_attention_weight_max": 19.648094177246094,
      "activations/layer12_attention_weight_min": -28.070589065551758,
      "activations/layer13_attention_weight_max": 49.489933013916016,
      "activations/layer13_attention_weight_min": -29.424068450927734,
      "activations/layer14_attention_weight_max": 38.7280387878418,
      "activations/layer14_attention_weight_min": -31.50128746032715,
      "activations/layer15_attention_weight_max": 32.47306823730469,
      "activations/layer15_attention_weight_min": -34.090423583984375,
      "activations/layer16_attention_weight_max": 35.76405715942383,
      "activations/layer16_attention_weight_min": -32.996585845947266,
      "activations/layer17_attention_weight_max": 52.353118896484375,
      "activations/layer17_attention_weight_min": -48.21241760253906,
      "activations/layer18_attention_weight_max": 52.43236541748047,
      "activations/layer18_attention_weight_min": -45.72731399536133,
      "activations/layer19_attention_weight_max": 25.85379981994629,
      "activations/layer19_attention_weight_min": -26.44890785217285,
      "activations/layer1_attention_weight_max": 15.699159622192383,
      "activations/layer1_attention_weight_min": -13.72265625,
      "activations/layer20_attention_weight_max": 23.29735565185547,
      "activations/layer20_attention_weight_min": -22.06010627746582,
      "activations/layer21_attention_weight_max": 38.643768310546875,
      "activations/layer21_attention_weight_min": -19.935625076293945,
      "activations/layer22_attention_weight_max": 28.49784278869629,
      "activations/layer22_attention_weight_min": -25.75664520263672,
      "activations/layer23_attention_weight_max": 36.25065612792969,
      "activations/layer23_attention_weight_min": -23.160863876342773,
      "activations/layer2_attention_weight_max": 30.540878295898438,
      "activations/layer2_attention_weight_min": -30.889860153198242,
      "activations/layer3_attention_weight_max": 87.3228530883789,
      "activations/layer3_attention_weight_min": -94.55040740966797,
      "activations/layer4_attention_weight_max": 84.16570281982422,
      "activations/layer4_attention_weight_min": -82.11420440673828,
      "activations/layer5_attention_weight_max": 63.187294006347656,
      "activations/layer5_attention_weight_min": -69.42155456542969,
      "activations/layer6_attention_weight_max": 48.83517837524414,
      "activations/layer6_attention_weight_min": -48.95762634277344,
      "activations/layer7_attention_weight_max": 66.50015258789062,
      "activations/layer7_attention_weight_min": -67.922607421875,
      "activations/layer8_attention_weight_max": 44.909725189208984,
      "activations/layer8_attention_weight_min": -48.74769973754883,
      "activations/layer9_attention_weight_max": 45.52907943725586,
      "activations/layer9_attention_weight_min": -46.65604019165039,
      "epoch": 22.19,
      "learning_rate": 6.9518939393939385e-06,
      "loss": 2.7061,
      "step": 381950
    },
    {
      "activations/layer0_attention_weight_max": 15.404582023620605,
      "activations/layer0_attention_weight_min": -13.946053504943848,
      "activations/layer10_attention_weight_max": 39.060791015625,
      "activations/layer10_attention_weight_min": -37.16829299926758,
      "activations/layer11_attention_weight_max": 35.84101104736328,
      "activations/layer11_attention_weight_min": -35.1196174621582,
      "activations/layer12_attention_weight_max": 19.933542251586914,
      "activations/layer12_attention_weight_min": -25.63546371459961,
      "activations/layer13_attention_weight_max": 41.61036682128906,
      "activations/layer13_attention_weight_min": -27.88389778137207,
      "activations/layer14_attention_weight_max": 36.29326629638672,
      "activations/layer14_attention_weight_min": -31.047718048095703,
      "activations/layer15_attention_weight_max": 34.34809112548828,
      "activations/layer15_attention_weight_min": -28.875995635986328,
      "activations/layer16_attention_weight_max": 36.776466369628906,
      "activations/layer16_attention_weight_min": -34.97819519042969,
      "activations/layer17_attention_weight_max": 54.27642822265625,
      "activations/layer17_attention_weight_min": -47.68858337402344,
      "activations/layer18_attention_weight_max": 51.2054557800293,
      "activations/layer18_attention_weight_min": -39.2871208190918,
      "activations/layer19_attention_weight_max": 25.665185928344727,
      "activations/layer19_attention_weight_min": -19.601205825805664,
      "activations/layer1_attention_weight_max": 14.621006965637207,
      "activations/layer1_attention_weight_min": -14.380621910095215,
      "activations/layer20_attention_weight_max": 24.15302848815918,
      "activations/layer20_attention_weight_min": -19.181825637817383,
      "activations/layer21_attention_weight_max": 30.100723266601562,
      "activations/layer21_attention_weight_min": -21.996641159057617,
      "activations/layer22_attention_weight_max": 27.10795783996582,
      "activations/layer22_attention_weight_min": -22.582988739013672,
      "activations/layer23_attention_weight_max": 35.004859924316406,
      "activations/layer23_attention_weight_min": -23.131916046142578,
      "activations/layer2_attention_weight_max": 30.95303726196289,
      "activations/layer2_attention_weight_min": -29.233505249023438,
      "activations/layer3_attention_weight_max": 90.21569061279297,
      "activations/layer3_attention_weight_min": -88.8962173461914,
      "activations/layer4_attention_weight_max": 86.44853973388672,
      "activations/layer4_attention_weight_min": -82.67176818847656,
      "activations/layer5_attention_weight_max": 59.405452728271484,
      "activations/layer5_attention_weight_min": -75.2367172241211,
      "activations/layer6_attention_weight_max": 53.61836242675781,
      "activations/layer6_attention_weight_min": -52.2685432434082,
      "activations/layer7_attention_weight_max": 73.4072036743164,
      "activations/layer7_attention_weight_min": -70.72274780273438,
      "activations/layer8_attention_weight_max": 51.9766731262207,
      "activations/layer8_attention_weight_min": -53.163875579833984,
      "activations/layer9_attention_weight_max": 46.12288284301758,
      "activations/layer9_attention_weight_min": -46.34636688232422,
      "epoch": 22.2,
      "learning_rate": 6.933333333333333e-06,
      "loss": 2.7026,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.5541,
      "eval_samples_per_second": 501.981,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.5541,
      "eval_openwebtext_samples_per_second": 501.981,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9729,
      "eval_wikitext_samples_per_second": 231.133,
      "step": 382000
    },
    {
      "epoch": 22.2,
      "eval_lambada_loss": 2.37890625,
      "eval_lambada_ppl": 10.793091461961774,
      "eval_lambada_runtime": 9.5253,
      "eval_lambada_samples_per_second": 511.167,
      "step": 382000
    },
    {
      "activations/layer0_attention_weight_max": 16.01665687561035,
      "activations/layer0_attention_weight_min": -14.73353385925293,
      "activations/layer10_attention_weight_max": 47.345298767089844,
      "activations/layer10_attention_weight_min": -48.686275482177734,
      "activations/layer11_attention_weight_max": 48.774169921875,
      "activations/layer11_attention_weight_min": -46.051658630371094,
      "activations/layer12_attention_weight_max": 20.586910247802734,
      "activations/layer12_attention_weight_min": -25.502273559570312,
      "activations/layer13_attention_weight_max": 51.542381286621094,
      "activations/layer13_attention_weight_min": -34.22538757324219,
      "activations/layer14_attention_weight_max": 66.69783020019531,
      "activations/layer14_attention_weight_min": -39.318546295166016,
      "activations/layer15_attention_weight_max": 63.61225891113281,
      "activations/layer15_attention_weight_min": -39.957176208496094,
      "activations/layer16_attention_weight_max": 44.28989028930664,
      "activations/layer16_attention_weight_min": -38.53584671020508,
      "activations/layer17_attention_weight_max": 62.87239074707031,
      "activations/layer17_attention_weight_min": -58.402587890625,
      "activations/layer18_attention_weight_max": 57.59703826904297,
      "activations/layer18_attention_weight_min": -52.85509490966797,
      "activations/layer19_attention_weight_max": 30.597639083862305,
      "activations/layer19_attention_weight_min": -28.41764259338379,
      "activations/layer1_attention_weight_max": 15.27357006072998,
      "activations/layer1_attention_weight_min": -15.00974178314209,
      "activations/layer20_attention_weight_max": 31.54511070251465,
      "activations/layer20_attention_weight_min": -23.733182907104492,
      "activations/layer21_attention_weight_max": 50.78244400024414,
      "activations/layer21_attention_weight_min": -25.48229217529297,
      "activations/layer22_attention_weight_max": 37.163448333740234,
      "activations/layer22_attention_weight_min": -29.22065544128418,
      "activations/layer23_attention_weight_max": 38.48804473876953,
      "activations/layer23_attention_weight_min": -26.75259780883789,
      "activations/layer2_attention_weight_max": 34.98666763305664,
      "activations/layer2_attention_weight_min": -32.87480163574219,
      "activations/layer3_attention_weight_max": 97.94506072998047,
      "activations/layer3_attention_weight_min": -99.1022720336914,
      "activations/layer4_attention_weight_max": 92.82183074951172,
      "activations/layer4_attention_weight_min": -84.0199203491211,
      "activations/layer5_attention_weight_max": 67.74052429199219,
      "activations/layer5_attention_weight_min": -70.3548583984375,
      "activations/layer6_attention_weight_max": 51.65349197387695,
      "activations/layer6_attention_weight_min": -55.480621337890625,
      "activations/layer7_attention_weight_max": 84.72509765625,
      "activations/layer7_attention_weight_min": -70.2673568725586,
      "activations/layer8_attention_weight_max": 58.75893020629883,
      "activations/layer8_attention_weight_min": -56.74169921875,
      "activations/layer9_attention_weight_max": 63.8779296875,
      "activations/layer9_attention_weight_min": -54.9276237487793,
      "epoch": 22.2,
      "learning_rate": 6.9143939393939385e-06,
      "loss": 2.7093,
      "step": 382050
    },
    {
      "activations/layer0_attention_weight_max": 16.663116455078125,
      "activations/layer0_attention_weight_min": -14.505792617797852,
      "activations/layer10_attention_weight_max": 36.899879455566406,
      "activations/layer10_attention_weight_min": -35.72323989868164,
      "activations/layer11_attention_weight_max": 34.011531829833984,
      "activations/layer11_attention_weight_min": -33.739952087402344,
      "activations/layer12_attention_weight_max": 17.809785842895508,
      "activations/layer12_attention_weight_min": -28.700672149658203,
      "activations/layer13_attention_weight_max": 36.789241790771484,
      "activations/layer13_attention_weight_min": -28.1373291015625,
      "activations/layer14_attention_weight_max": 34.537105560302734,
      "activations/layer14_attention_weight_min": -32.60184860229492,
      "activations/layer15_attention_weight_max": 34.424835205078125,
      "activations/layer15_attention_weight_min": -32.71074295043945,
      "activations/layer16_attention_weight_max": 36.244041442871094,
      "activations/layer16_attention_weight_min": -36.14937210083008,
      "activations/layer17_attention_weight_max": 51.47995376586914,
      "activations/layer17_attention_weight_min": -47.2341194152832,
      "activations/layer18_attention_weight_max": 49.74611282348633,
      "activations/layer18_attention_weight_min": -41.102779388427734,
      "activations/layer19_attention_weight_max": 22.62716293334961,
      "activations/layer19_attention_weight_min": -26.780338287353516,
      "activations/layer1_attention_weight_max": 14.84624195098877,
      "activations/layer1_attention_weight_min": -14.311577796936035,
      "activations/layer20_attention_weight_max": 22.202512741088867,
      "activations/layer20_attention_weight_min": -21.937829971313477,
      "activations/layer21_attention_weight_max": 32.460968017578125,
      "activations/layer21_attention_weight_min": -21.409311294555664,
      "activations/layer22_attention_weight_max": 29.666109085083008,
      "activations/layer22_attention_weight_min": -24.550256729125977,
      "activations/layer23_attention_weight_max": 34.60472869873047,
      "activations/layer23_attention_weight_min": -24.014915466308594,
      "activations/layer2_attention_weight_max": 32.240116119384766,
      "activations/layer2_attention_weight_min": -31.444942474365234,
      "activations/layer3_attention_weight_max": 89.41719055175781,
      "activations/layer3_attention_weight_min": -91.04618835449219,
      "activations/layer4_attention_weight_max": 82.4902114868164,
      "activations/layer4_attention_weight_min": -86.84648132324219,
      "activations/layer5_attention_weight_max": 63.31739044189453,
      "activations/layer5_attention_weight_min": -76.08992004394531,
      "activations/layer6_attention_weight_max": 48.753273010253906,
      "activations/layer6_attention_weight_min": -49.67234802246094,
      "activations/layer7_attention_weight_max": 76.21251678466797,
      "activations/layer7_attention_weight_min": -67.46273803710938,
      "activations/layer8_attention_weight_max": 46.58030319213867,
      "activations/layer8_attention_weight_min": -51.80263137817383,
      "activations/layer9_attention_weight_max": 50.664730072021484,
      "activations/layer9_attention_weight_min": -47.26276779174805,
      "epoch": 22.2,
      "learning_rate": 6.895454545454545e-06,
      "loss": 2.7052,
      "step": 382100
    },
    {
      "activations/layer0_attention_weight_max": 16.079479217529297,
      "activations/layer0_attention_weight_min": -13.96796989440918,
      "activations/layer10_attention_weight_max": 39.53072738647461,
      "activations/layer10_attention_weight_min": -39.90383529663086,
      "activations/layer11_attention_weight_max": 34.492794036865234,
      "activations/layer11_attention_weight_min": -37.72412872314453,
      "activations/layer12_attention_weight_max": 19.637779235839844,
      "activations/layer12_attention_weight_min": -25.949832916259766,
      "activations/layer13_attention_weight_max": 34.727718353271484,
      "activations/layer13_attention_weight_min": -28.502599716186523,
      "activations/layer14_attention_weight_max": 34.95075988769531,
      "activations/layer14_attention_weight_min": -29.925811767578125,
      "activations/layer15_attention_weight_max": 31.692651748657227,
      "activations/layer15_attention_weight_min": -29.688758850097656,
      "activations/layer16_attention_weight_max": 32.459354400634766,
      "activations/layer16_attention_weight_min": -30.917089462280273,
      "activations/layer17_attention_weight_max": 51.09825897216797,
      "activations/layer17_attention_weight_min": -43.609134674072266,
      "activations/layer18_attention_weight_max": 44.5324821472168,
      "activations/layer18_attention_weight_min": -39.20690155029297,
      "activations/layer19_attention_weight_max": 25.481348037719727,
      "activations/layer19_attention_weight_min": -20.8680419921875,
      "activations/layer1_attention_weight_max": 14.873513221740723,
      "activations/layer1_attention_weight_min": -13.431883811950684,
      "activations/layer20_attention_weight_max": 21.281803131103516,
      "activations/layer20_attention_weight_min": -20.18636703491211,
      "activations/layer21_attention_weight_max": 35.15813064575195,
      "activations/layer21_attention_weight_min": -20.29437255859375,
      "activations/layer22_attention_weight_max": 27.113325119018555,
      "activations/layer22_attention_weight_min": -25.174211502075195,
      "activations/layer23_attention_weight_max": 36.348106384277344,
      "activations/layer23_attention_weight_min": -24.979480743408203,
      "activations/layer2_attention_weight_max": 32.36460876464844,
      "activations/layer2_attention_weight_min": -32.383209228515625,
      "activations/layer3_attention_weight_max": 90.60984802246094,
      "activations/layer3_attention_weight_min": -92.5118179321289,
      "activations/layer4_attention_weight_max": 85.0827865600586,
      "activations/layer4_attention_weight_min": -85.12203216552734,
      "activations/layer5_attention_weight_max": 63.6788330078125,
      "activations/layer5_attention_weight_min": -72.50381469726562,
      "activations/layer6_attention_weight_max": 53.83527755737305,
      "activations/layer6_attention_weight_min": -53.95718002319336,
      "activations/layer7_attention_weight_max": 69.4678955078125,
      "activations/layer7_attention_weight_min": -71.41519165039062,
      "activations/layer8_attention_weight_max": 49.887855529785156,
      "activations/layer8_attention_weight_min": -55.87089157104492,
      "activations/layer9_attention_weight_max": 44.93669509887695,
      "activations/layer9_attention_weight_min": -48.04377746582031,
      "epoch": 22.21,
      "learning_rate": 6.876893939393938e-06,
      "loss": 2.7001,
      "step": 382150
    },
    {
      "activations/layer0_attention_weight_max": 15.284698486328125,
      "activations/layer0_attention_weight_min": -14.040767669677734,
      "activations/layer10_attention_weight_max": 33.70358657836914,
      "activations/layer10_attention_weight_min": -33.5242919921875,
      "activations/layer11_attention_weight_max": 31.374303817749023,
      "activations/layer11_attention_weight_min": -34.377994537353516,
      "activations/layer12_attention_weight_max": 17.75446891784668,
      "activations/layer12_attention_weight_min": -25.74017906188965,
      "activations/layer13_attention_weight_max": 33.77193832397461,
      "activations/layer13_attention_weight_min": -26.366657257080078,
      "activations/layer14_attention_weight_max": 31.67491340637207,
      "activations/layer14_attention_weight_min": -28.020801544189453,
      "activations/layer15_attention_weight_max": 30.799802780151367,
      "activations/layer15_attention_weight_min": -29.06501579284668,
      "activations/layer16_attention_weight_max": 31.173023223876953,
      "activations/layer16_attention_weight_min": -30.438447952270508,
      "activations/layer17_attention_weight_max": 46.37001419067383,
      "activations/layer17_attention_weight_min": -43.1341552734375,
      "activations/layer18_attention_weight_max": 41.72090530395508,
      "activations/layer18_attention_weight_min": -40.0572624206543,
      "activations/layer19_attention_weight_max": 23.168548583984375,
      "activations/layer19_attention_weight_min": -22.299707412719727,
      "activations/layer1_attention_weight_max": 16.165393829345703,
      "activations/layer1_attention_weight_min": -14.922039985656738,
      "activations/layer20_attention_weight_max": 20.809383392333984,
      "activations/layer20_attention_weight_min": -22.29411506652832,
      "activations/layer21_attention_weight_max": 30.737869262695312,
      "activations/layer21_attention_weight_min": -21.39206886291504,
      "activations/layer22_attention_weight_max": 24.629962921142578,
      "activations/layer22_attention_weight_min": -23.7819766998291,
      "activations/layer23_attention_weight_max": 34.559539794921875,
      "activations/layer23_attention_weight_min": -24.50525665283203,
      "activations/layer2_attention_weight_max": 32.03966522216797,
      "activations/layer2_attention_weight_min": -31.854461669921875,
      "activations/layer3_attention_weight_max": 90.32749938964844,
      "activations/layer3_attention_weight_min": -95.64351654052734,
      "activations/layer4_attention_weight_max": 84.5125961303711,
      "activations/layer4_attention_weight_min": -84.51872253417969,
      "activations/layer5_attention_weight_max": 64.03703308105469,
      "activations/layer5_attention_weight_min": -73.14961242675781,
      "activations/layer6_attention_weight_max": 49.16670227050781,
      "activations/layer6_attention_weight_min": -48.6177864074707,
      "activations/layer7_attention_weight_max": 67.05459594726562,
      "activations/layer7_attention_weight_min": -68.38996124267578,
      "activations/layer8_attention_weight_max": 46.3329963684082,
      "activations/layer8_attention_weight_min": -49.01377868652344,
      "activations/layer9_attention_weight_max": 48.720794677734375,
      "activations/layer9_attention_weight_min": -44.75664138793945,
      "epoch": 22.21,
      "learning_rate": 6.857954545454545e-06,
      "loss": 2.6986,
      "step": 382200
    },
    {
      "activations/layer0_attention_weight_max": 15.297405242919922,
      "activations/layer0_attention_weight_min": -13.881943702697754,
      "activations/layer10_attention_weight_max": 41.769691467285156,
      "activations/layer10_attention_weight_min": -44.13720703125,
      "activations/layer11_attention_weight_max": 41.410133361816406,
      "activations/layer11_attention_weight_min": -39.85798263549805,
      "activations/layer12_attention_weight_max": 20.88541030883789,
      "activations/layer12_attention_weight_min": -26.833248138427734,
      "activations/layer13_attention_weight_max": 51.7258415222168,
      "activations/layer13_attention_weight_min": -31.229480743408203,
      "activations/layer14_attention_weight_max": 46.76047134399414,
      "activations/layer14_attention_weight_min": -33.640716552734375,
      "activations/layer15_attention_weight_max": 47.385223388671875,
      "activations/layer15_attention_weight_min": -33.2383918762207,
      "activations/layer16_attention_weight_max": 42.21372604370117,
      "activations/layer16_attention_weight_min": -37.06562042236328,
      "activations/layer17_attention_weight_max": 59.64691925048828,
      "activations/layer17_attention_weight_min": -54.98891067504883,
      "activations/layer18_attention_weight_max": 58.63808059692383,
      "activations/layer18_attention_weight_min": -49.372440338134766,
      "activations/layer19_attention_weight_max": 26.4652099609375,
      "activations/layer19_attention_weight_min": -25.61475944519043,
      "activations/layer1_attention_weight_max": 15.17525863647461,
      "activations/layer1_attention_weight_min": -13.492091178894043,
      "activations/layer20_attention_weight_max": 28.411380767822266,
      "activations/layer20_attention_weight_min": -21.57854461669922,
      "activations/layer21_attention_weight_max": 46.74421310424805,
      "activations/layer21_attention_weight_min": -23.40041160583496,
      "activations/layer22_attention_weight_max": 30.97194480895996,
      "activations/layer22_attention_weight_min": -25.783039093017578,
      "activations/layer23_attention_weight_max": 37.06141662597656,
      "activations/layer23_attention_weight_min": -23.542682647705078,
      "activations/layer2_attention_weight_max": 32.29540252685547,
      "activations/layer2_attention_weight_min": -30.572078704833984,
      "activations/layer3_attention_weight_max": 92.9604721069336,
      "activations/layer3_attention_weight_min": -93.92575073242188,
      "activations/layer4_attention_weight_max": 87.42025756835938,
      "activations/layer4_attention_weight_min": -88.1044692993164,
      "activations/layer5_attention_weight_max": 65.77908325195312,
      "activations/layer5_attention_weight_min": -71.63725280761719,
      "activations/layer6_attention_weight_max": 53.33943176269531,
      "activations/layer6_attention_weight_min": -53.7509651184082,
      "activations/layer7_attention_weight_max": 77.80048370361328,
      "activations/layer7_attention_weight_min": -70.26153564453125,
      "activations/layer8_attention_weight_max": 55.984161376953125,
      "activations/layer8_attention_weight_min": -56.137081146240234,
      "activations/layer9_attention_weight_max": 54.93685531616211,
      "activations/layer9_attention_weight_min": -49.18943786621094,
      "epoch": 22.21,
      "learning_rate": 6.8390151515151516e-06,
      "loss": 2.7118,
      "step": 382250
    },
    {
      "activations/layer0_attention_weight_max": 16.064760208129883,
      "activations/layer0_attention_weight_min": -14.101362228393555,
      "activations/layer10_attention_weight_max": 37.29580307006836,
      "activations/layer10_attention_weight_min": -37.24924850463867,
      "activations/layer11_attention_weight_max": 36.289161682128906,
      "activations/layer11_attention_weight_min": -36.07551574707031,
      "activations/layer12_attention_weight_max": 20.089262008666992,
      "activations/layer12_attention_weight_min": -26.1019229888916,
      "activations/layer13_attention_weight_max": 35.8616943359375,
      "activations/layer13_attention_weight_min": -28.216463088989258,
      "activations/layer14_attention_weight_max": 35.49884033203125,
      "activations/layer14_attention_weight_min": -30.718021392822266,
      "activations/layer15_attention_weight_max": 36.139747619628906,
      "activations/layer15_attention_weight_min": -32.02061080932617,
      "activations/layer16_attention_weight_max": 36.07902908325195,
      "activations/layer16_attention_weight_min": -34.20048904418945,
      "activations/layer17_attention_weight_max": 55.86381530761719,
      "activations/layer17_attention_weight_min": -49.318275451660156,
      "activations/layer18_attention_weight_max": 50.76353073120117,
      "activations/layer18_attention_weight_min": -45.061702728271484,
      "activations/layer19_attention_weight_max": 26.31199073791504,
      "activations/layer19_attention_weight_min": -24.824756622314453,
      "activations/layer1_attention_weight_max": 14.95530891418457,
      "activations/layer1_attention_weight_min": -14.09871768951416,
      "activations/layer20_attention_weight_max": 27.060035705566406,
      "activations/layer20_attention_weight_min": -22.47148323059082,
      "activations/layer21_attention_weight_max": 41.00791931152344,
      "activations/layer21_attention_weight_min": -21.393310546875,
      "activations/layer22_attention_weight_max": 33.2259407043457,
      "activations/layer22_attention_weight_min": -24.75892448425293,
      "activations/layer23_attention_weight_max": 36.65309143066406,
      "activations/layer23_attention_weight_min": -23.61354637145996,
      "activations/layer2_attention_weight_max": 33.110984802246094,
      "activations/layer2_attention_weight_min": -34.189369201660156,
      "activations/layer3_attention_weight_max": 92.72935485839844,
      "activations/layer3_attention_weight_min": -96.19905853271484,
      "activations/layer4_attention_weight_max": 86.89769744873047,
      "activations/layer4_attention_weight_min": -88.3978271484375,
      "activations/layer5_attention_weight_max": 63.28374481201172,
      "activations/layer5_attention_weight_min": -73.06626892089844,
      "activations/layer6_attention_weight_max": 51.28688049316406,
      "activations/layer6_attention_weight_min": -51.723026275634766,
      "activations/layer7_attention_weight_max": 67.51232147216797,
      "activations/layer7_attention_weight_min": -68.9135513305664,
      "activations/layer8_attention_weight_max": 49.66996383666992,
      "activations/layer8_attention_weight_min": -51.37623977661133,
      "activations/layer9_attention_weight_max": 47.69709014892578,
      "activations/layer9_attention_weight_min": -47.82286834716797,
      "epoch": 22.21,
      "learning_rate": 6.820075757575757e-06,
      "loss": 2.7113,
      "step": 382300
    },
    {
      "activations/layer0_attention_weight_max": 15.686278343200684,
      "activations/layer0_attention_weight_min": -13.950695991516113,
      "activations/layer10_attention_weight_max": 38.02382278442383,
      "activations/layer10_attention_weight_min": -36.523338317871094,
      "activations/layer11_attention_weight_max": 36.74929428100586,
      "activations/layer11_attention_weight_min": -36.96576690673828,
      "activations/layer12_attention_weight_max": 20.325998306274414,
      "activations/layer12_attention_weight_min": -28.10847282409668,
      "activations/layer13_attention_weight_max": 40.2381591796875,
      "activations/layer13_attention_weight_min": -29.134681701660156,
      "activations/layer14_attention_weight_max": 39.77631759643555,
      "activations/layer14_attention_weight_min": -33.70826721191406,
      "activations/layer15_attention_weight_max": 38.312477111816406,
      "activations/layer15_attention_weight_min": -32.198795318603516,
      "activations/layer16_attention_weight_max": 39.877891540527344,
      "activations/layer16_attention_weight_min": -35.71385955810547,
      "activations/layer17_attention_weight_max": 60.02948760986328,
      "activations/layer17_attention_weight_min": -49.46719741821289,
      "activations/layer18_attention_weight_max": 57.39350509643555,
      "activations/layer18_attention_weight_min": -43.177467346191406,
      "activations/layer19_attention_weight_max": 27.047290802001953,
      "activations/layer19_attention_weight_min": -24.54300880432129,
      "activations/layer1_attention_weight_max": 14.351858139038086,
      "activations/layer1_attention_weight_min": -14.231621742248535,
      "activations/layer20_attention_weight_max": 24.7373046875,
      "activations/layer20_attention_weight_min": -21.567380905151367,
      "activations/layer21_attention_weight_max": 41.55399703979492,
      "activations/layer21_attention_weight_min": -23.697351455688477,
      "activations/layer22_attention_weight_max": 33.242774963378906,
      "activations/layer22_attention_weight_min": -26.98891258239746,
      "activations/layer23_attention_weight_max": 45.812347412109375,
      "activations/layer23_attention_weight_min": -24.368892669677734,
      "activations/layer2_attention_weight_max": 32.987953186035156,
      "activations/layer2_attention_weight_min": -31.602336883544922,
      "activations/layer3_attention_weight_max": 96.15074157714844,
      "activations/layer3_attention_weight_min": -93.68535614013672,
      "activations/layer4_attention_weight_max": 87.96028900146484,
      "activations/layer4_attention_weight_min": -88.68487548828125,
      "activations/layer5_attention_weight_max": 63.57879638671875,
      "activations/layer5_attention_weight_min": -78.43418884277344,
      "activations/layer6_attention_weight_max": 51.33694839477539,
      "activations/layer6_attention_weight_min": -52.417198181152344,
      "activations/layer7_attention_weight_max": 67.95283508300781,
      "activations/layer7_attention_weight_min": -69.83755493164062,
      "activations/layer8_attention_weight_max": 49.01716995239258,
      "activations/layer8_attention_weight_min": -53.18976593017578,
      "activations/layer9_attention_weight_max": 45.359066009521484,
      "activations/layer9_attention_weight_min": -49.73777770996094,
      "epoch": 22.22,
      "learning_rate": 6.801136363636363e-06,
      "loss": 2.7062,
      "step": 382350
    },
    {
      "activations/layer0_attention_weight_max": 16.872251510620117,
      "activations/layer0_attention_weight_min": -14.592199325561523,
      "activations/layer10_attention_weight_max": 40.726158142089844,
      "activations/layer10_attention_weight_min": -40.34169387817383,
      "activations/layer11_attention_weight_max": 37.869476318359375,
      "activations/layer11_attention_weight_min": -39.62633514404297,
      "activations/layer12_attention_weight_max": 19.458112716674805,
      "activations/layer12_attention_weight_min": -25.877670288085938,
      "activations/layer13_attention_weight_max": 34.35272216796875,
      "activations/layer13_attention_weight_min": -28.32598304748535,
      "activations/layer14_attention_weight_max": 43.38657760620117,
      "activations/layer14_attention_weight_min": -32.362022399902344,
      "activations/layer15_attention_weight_max": 41.28839874267578,
      "activations/layer15_attention_weight_min": -33.64035415649414,
      "activations/layer16_attention_weight_max": 38.398555755615234,
      "activations/layer16_attention_weight_min": -35.037410736083984,
      "activations/layer17_attention_weight_max": 54.31209182739258,
      "activations/layer17_attention_weight_min": -45.62767028808594,
      "activations/layer18_attention_weight_max": 54.851234436035156,
      "activations/layer18_attention_weight_min": -40.557159423828125,
      "activations/layer19_attention_weight_max": 25.692609786987305,
      "activations/layer19_attention_weight_min": -22.919958114624023,
      "activations/layer1_attention_weight_max": 16.42353630065918,
      "activations/layer1_attention_weight_min": -14.177433013916016,
      "activations/layer20_attention_weight_max": 24.541751861572266,
      "activations/layer20_attention_weight_min": -20.53695297241211,
      "activations/layer21_attention_weight_max": 41.46248245239258,
      "activations/layer21_attention_weight_min": -24.187883377075195,
      "activations/layer22_attention_weight_max": 32.14531326293945,
      "activations/layer22_attention_weight_min": -26.37884521484375,
      "activations/layer23_attention_weight_max": 37.81791305541992,
      "activations/layer23_attention_weight_min": -23.29938316345215,
      "activations/layer2_attention_weight_max": 30.46856689453125,
      "activations/layer2_attention_weight_min": -30.488224029541016,
      "activations/layer3_attention_weight_max": 88.0344467163086,
      "activations/layer3_attention_weight_min": -88.84172821044922,
      "activations/layer4_attention_weight_max": 88.89275360107422,
      "activations/layer4_attention_weight_min": -87.0107650756836,
      "activations/layer5_attention_weight_max": 63.62307357788086,
      "activations/layer5_attention_weight_min": -72.55376434326172,
      "activations/layer6_attention_weight_max": 50.75046157836914,
      "activations/layer6_attention_weight_min": -52.34092330932617,
      "activations/layer7_attention_weight_max": 68.23686218261719,
      "activations/layer7_attention_weight_min": -67.36959075927734,
      "activations/layer8_attention_weight_max": 50.42356872558594,
      "activations/layer8_attention_weight_min": -52.66044235229492,
      "activations/layer9_attention_weight_max": 49.03841781616211,
      "activations/layer9_attention_weight_min": -49.69822311401367,
      "epoch": 22.22,
      "learning_rate": 6.782196969696969e-06,
      "loss": 2.7094,
      "step": 382400
    },
    {
      "activations/layer0_attention_weight_max": 15.65752124786377,
      "activations/layer0_attention_weight_min": -14.016655921936035,
      "activations/layer10_attention_weight_max": 36.67625427246094,
      "activations/layer10_attention_weight_min": -39.140289306640625,
      "activations/layer11_attention_weight_max": 36.38641357421875,
      "activations/layer11_attention_weight_min": -39.734336853027344,
      "activations/layer12_attention_weight_max": 20.08917236328125,
      "activations/layer12_attention_weight_min": -24.354087829589844,
      "activations/layer13_attention_weight_max": 35.65001678466797,
      "activations/layer13_attention_weight_min": -28.232805252075195,
      "activations/layer14_attention_weight_max": 35.50575256347656,
      "activations/layer14_attention_weight_min": -31.502965927124023,
      "activations/layer15_attention_weight_max": 34.815940856933594,
      "activations/layer15_attention_weight_min": -31.206602096557617,
      "activations/layer16_attention_weight_max": 35.80734634399414,
      "activations/layer16_attention_weight_min": -32.49177551269531,
      "activations/layer17_attention_weight_max": 55.68195343017578,
      "activations/layer17_attention_weight_min": -47.86491012573242,
      "activations/layer18_attention_weight_max": 49.71770095825195,
      "activations/layer18_attention_weight_min": -41.58743667602539,
      "activations/layer19_attention_weight_max": 25.955991744995117,
      "activations/layer19_attention_weight_min": -22.364948272705078,
      "activations/layer1_attention_weight_max": 16.091217041015625,
      "activations/layer1_attention_weight_min": -15.313246726989746,
      "activations/layer20_attention_weight_max": 25.17778778076172,
      "activations/layer20_attention_weight_min": -22.146427154541016,
      "activations/layer21_attention_weight_max": 38.23316192626953,
      "activations/layer21_attention_weight_min": -22.46286392211914,
      "activations/layer22_attention_weight_max": 30.139577865600586,
      "activations/layer22_attention_weight_min": -26.46405029296875,
      "activations/layer23_attention_weight_max": 37.17470169067383,
      "activations/layer23_attention_weight_min": -25.517797470092773,
      "activations/layer2_attention_weight_max": 32.40619659423828,
      "activations/layer2_attention_weight_min": -30.586788177490234,
      "activations/layer3_attention_weight_max": 91.5277099609375,
      "activations/layer3_attention_weight_min": -89.05724334716797,
      "activations/layer4_attention_weight_max": 87.54509735107422,
      "activations/layer4_attention_weight_min": -88.83409881591797,
      "activations/layer5_attention_weight_max": 63.03577423095703,
      "activations/layer5_attention_weight_min": -71.02995300292969,
      "activations/layer6_attention_weight_max": 49.29330062866211,
      "activations/layer6_attention_weight_min": -52.41887664794922,
      "activations/layer7_attention_weight_max": 69.17005920410156,
      "activations/layer7_attention_weight_min": -69.35079193115234,
      "activations/layer8_attention_weight_max": 46.50116729736328,
      "activations/layer8_attention_weight_min": -51.46660232543945,
      "activations/layer9_attention_weight_max": 42.85892105102539,
      "activations/layer9_attention_weight_min": -45.894039154052734,
      "epoch": 22.22,
      "learning_rate": 6.763257575757575e-06,
      "loss": 2.7025,
      "step": 382450
    },
    {
      "activations/layer0_attention_weight_max": 15.784568786621094,
      "activations/layer0_attention_weight_min": -14.316498756408691,
      "activations/layer10_attention_weight_max": 37.034610748291016,
      "activations/layer10_attention_weight_min": -37.14234924316406,
      "activations/layer11_attention_weight_max": 34.44458770751953,
      "activations/layer11_attention_weight_min": -36.02854537963867,
      "activations/layer12_attention_weight_max": 19.974069595336914,
      "activations/layer12_attention_weight_min": -23.461227416992188,
      "activations/layer13_attention_weight_max": 31.709064483642578,
      "activations/layer13_attention_weight_min": -26.3839168548584,
      "activations/layer14_attention_weight_max": 34.85742950439453,
      "activations/layer14_attention_weight_min": -31.37272071838379,
      "activations/layer15_attention_weight_max": 32.007179260253906,
      "activations/layer15_attention_weight_min": -30.70807456970215,
      "activations/layer16_attention_weight_max": 34.431644439697266,
      "activations/layer16_attention_weight_min": -34.170040130615234,
      "activations/layer17_attention_weight_max": 53.00365447998047,
      "activations/layer17_attention_weight_min": -46.937801361083984,
      "activations/layer18_attention_weight_max": 46.70954513549805,
      "activations/layer18_attention_weight_min": -43.83463668823242,
      "activations/layer19_attention_weight_max": 23.463441848754883,
      "activations/layer19_attention_weight_min": -25.76299476623535,
      "activations/layer1_attention_weight_max": 14.903182029724121,
      "activations/layer1_attention_weight_min": -14.150199890136719,
      "activations/layer20_attention_weight_max": 22.925445556640625,
      "activations/layer20_attention_weight_min": -22.326671600341797,
      "activations/layer21_attention_weight_max": 31.677196502685547,
      "activations/layer21_attention_weight_min": -22.514930725097656,
      "activations/layer22_attention_weight_max": 30.377174377441406,
      "activations/layer22_attention_weight_min": -26.58612632751465,
      "activations/layer23_attention_weight_max": 40.722015380859375,
      "activations/layer23_attention_weight_min": -23.878032684326172,
      "activations/layer2_attention_weight_max": 33.62627410888672,
      "activations/layer2_attention_weight_min": -31.11981773376465,
      "activations/layer3_attention_weight_max": 92.60669708251953,
      "activations/layer3_attention_weight_min": -89.5743637084961,
      "activations/layer4_attention_weight_max": 88.96890258789062,
      "activations/layer4_attention_weight_min": -88.0296401977539,
      "activations/layer5_attention_weight_max": 64.31722259521484,
      "activations/layer5_attention_weight_min": -76.00264739990234,
      "activations/layer6_attention_weight_max": 51.7072639465332,
      "activations/layer6_attention_weight_min": -53.40508270263672,
      "activations/layer7_attention_weight_max": 72.51270294189453,
      "activations/layer7_attention_weight_min": -71.7270278930664,
      "activations/layer8_attention_weight_max": 48.50741195678711,
      "activations/layer8_attention_weight_min": -52.43191909790039,
      "activations/layer9_attention_weight_max": 46.84772872924805,
      "activations/layer9_attention_weight_min": -51.53883743286133,
      "epoch": 22.23,
      "learning_rate": 6.744318181818181e-06,
      "loss": 2.695,
      "step": 382500
    },
    {
      "activations/layer0_attention_weight_max": 16.882537841796875,
      "activations/layer0_attention_weight_min": -14.238635063171387,
      "activations/layer10_attention_weight_max": 33.66005325317383,
      "activations/layer10_attention_weight_min": -34.881568908691406,
      "activations/layer11_attention_weight_max": 31.846166610717773,
      "activations/layer11_attention_weight_min": -35.060768127441406,
      "activations/layer12_attention_weight_max": 18.393035888671875,
      "activations/layer12_attention_weight_min": -26.233396530151367,
      "activations/layer13_attention_weight_max": 30.354997634887695,
      "activations/layer13_attention_weight_min": -26.334171295166016,
      "activations/layer14_attention_weight_max": 34.02644729614258,
      "activations/layer14_attention_weight_min": -28.42764663696289,
      "activations/layer15_attention_weight_max": 31.90218734741211,
      "activations/layer15_attention_weight_min": -30.550615310668945,
      "activations/layer16_attention_weight_max": 33.36382293701172,
      "activations/layer16_attention_weight_min": -32.43054962158203,
      "activations/layer17_attention_weight_max": 47.9859504699707,
      "activations/layer17_attention_weight_min": -45.073177337646484,
      "activations/layer18_attention_weight_max": 43.4062385559082,
      "activations/layer18_attention_weight_min": -39.1558723449707,
      "activations/layer19_attention_weight_max": 22.070331573486328,
      "activations/layer19_attention_weight_min": -21.769773483276367,
      "activations/layer1_attention_weight_max": 14.855023384094238,
      "activations/layer1_attention_weight_min": -14.128851890563965,
      "activations/layer20_attention_weight_max": 22.77025032043457,
      "activations/layer20_attention_weight_min": -22.259624481201172,
      "activations/layer21_attention_weight_max": 31.158828735351562,
      "activations/layer21_attention_weight_min": -19.217174530029297,
      "activations/layer22_attention_weight_max": 25.84464454650879,
      "activations/layer22_attention_weight_min": -24.5924129486084,
      "activations/layer23_attention_weight_max": 34.30556106567383,
      "activations/layer23_attention_weight_min": -24.388595581054688,
      "activations/layer2_attention_weight_max": 30.993091583251953,
      "activations/layer2_attention_weight_min": -29.925006866455078,
      "activations/layer3_attention_weight_max": 89.63262939453125,
      "activations/layer3_attention_weight_min": -87.92375946044922,
      "activations/layer4_attention_weight_max": 84.53582000732422,
      "activations/layer4_attention_weight_min": -82.8791275024414,
      "activations/layer5_attention_weight_max": 64.03353881835938,
      "activations/layer5_attention_weight_min": -74.97763061523438,
      "activations/layer6_attention_weight_max": 50.40696334838867,
      "activations/layer6_attention_weight_min": -51.15345001220703,
      "activations/layer7_attention_weight_max": 71.26454162597656,
      "activations/layer7_attention_weight_min": -65.85965728759766,
      "activations/layer8_attention_weight_max": 46.3538818359375,
      "activations/layer8_attention_weight_min": -48.28901672363281,
      "activations/layer9_attention_weight_max": 43.69645690917969,
      "activations/layer9_attention_weight_min": -43.926273345947266,
      "epoch": 22.23,
      "learning_rate": 6.725378787878787e-06,
      "loss": 2.6958,
      "step": 382550
    },
    {
      "activations/layer0_attention_weight_max": 15.90415096282959,
      "activations/layer0_attention_weight_min": -14.58934497833252,
      "activations/layer10_attention_weight_max": 39.82493591308594,
      "activations/layer10_attention_weight_min": -39.789093017578125,
      "activations/layer11_attention_weight_max": 35.873687744140625,
      "activations/layer11_attention_weight_min": -36.408470153808594,
      "activations/layer12_attention_weight_max": 24.105464935302734,
      "activations/layer12_attention_weight_min": -23.26349449157715,
      "activations/layer13_attention_weight_max": 36.281890869140625,
      "activations/layer13_attention_weight_min": -30.119457244873047,
      "activations/layer14_attention_weight_max": 36.08119201660156,
      "activations/layer14_attention_weight_min": -29.87077522277832,
      "activations/layer15_attention_weight_max": 33.409908294677734,
      "activations/layer15_attention_weight_min": -29.307437896728516,
      "activations/layer16_attention_weight_max": 35.001853942871094,
      "activations/layer16_attention_weight_min": -34.952056884765625,
      "activations/layer17_attention_weight_max": 52.024410247802734,
      "activations/layer17_attention_weight_min": -46.19182586669922,
      "activations/layer18_attention_weight_max": 49.612918853759766,
      "activations/layer18_attention_weight_min": -42.690460205078125,
      "activations/layer19_attention_weight_max": 24.499046325683594,
      "activations/layer19_attention_weight_min": -24.80242156982422,
      "activations/layer1_attention_weight_max": 14.622929573059082,
      "activations/layer1_attention_weight_min": -13.975698471069336,
      "activations/layer20_attention_weight_max": 24.731494903564453,
      "activations/layer20_attention_weight_min": -21.764097213745117,
      "activations/layer21_attention_weight_max": 37.06504821777344,
      "activations/layer21_attention_weight_min": -20.471994400024414,
      "activations/layer22_attention_weight_max": 33.56174087524414,
      "activations/layer22_attention_weight_min": -26.19954490661621,
      "activations/layer23_attention_weight_max": 36.05659484863281,
      "activations/layer23_attention_weight_min": -23.774860382080078,
      "activations/layer2_attention_weight_max": 32.484092712402344,
      "activations/layer2_attention_weight_min": -31.20318603515625,
      "activations/layer3_attention_weight_max": 90.98900604248047,
      "activations/layer3_attention_weight_min": -92.91497039794922,
      "activations/layer4_attention_weight_max": 87.81513214111328,
      "activations/layer4_attention_weight_min": -89.92809295654297,
      "activations/layer5_attention_weight_max": 64.80963134765625,
      "activations/layer5_attention_weight_min": -72.25799560546875,
      "activations/layer6_attention_weight_max": 54.53483581542969,
      "activations/layer6_attention_weight_min": -53.65709686279297,
      "activations/layer7_attention_weight_max": 72.66632080078125,
      "activations/layer7_attention_weight_min": -73.36558532714844,
      "activations/layer8_attention_weight_max": 52.430301666259766,
      "activations/layer8_attention_weight_min": -55.848777770996094,
      "activations/layer9_attention_weight_max": 48.6212272644043,
      "activations/layer9_attention_weight_min": -50.7681884765625,
      "epoch": 22.23,
      "learning_rate": 6.706439393939394e-06,
      "loss": 2.7313,
      "step": 382600
    },
    {
      "activations/layer0_attention_weight_max": 16.31838035583496,
      "activations/layer0_attention_weight_min": -14.310030937194824,
      "activations/layer10_attention_weight_max": 34.80963134765625,
      "activations/layer10_attention_weight_min": -36.82291030883789,
      "activations/layer11_attention_weight_max": 33.648048400878906,
      "activations/layer11_attention_weight_min": -35.849395751953125,
      "activations/layer12_attention_weight_max": 19.359102249145508,
      "activations/layer12_attention_weight_min": -24.99828338623047,
      "activations/layer13_attention_weight_max": 35.29930877685547,
      "activations/layer13_attention_weight_min": -28.362857818603516,
      "activations/layer14_attention_weight_max": 36.74094772338867,
      "activations/layer14_attention_weight_min": -29.718456268310547,
      "activations/layer15_attention_weight_max": 34.61357116699219,
      "activations/layer15_attention_weight_min": -30.10900115966797,
      "activations/layer16_attention_weight_max": 36.12491226196289,
      "activations/layer16_attention_weight_min": -32.51913833618164,
      "activations/layer17_attention_weight_max": 50.54085922241211,
      "activations/layer17_attention_weight_min": -44.00498962402344,
      "activations/layer18_attention_weight_max": 47.995052337646484,
      "activations/layer18_attention_weight_min": -41.58818817138672,
      "activations/layer19_attention_weight_max": 24.428693771362305,
      "activations/layer19_attention_weight_min": -23.249549865722656,
      "activations/layer1_attention_weight_max": 15.078336715698242,
      "activations/layer1_attention_weight_min": -14.719518661499023,
      "activations/layer20_attention_weight_max": 22.12289047241211,
      "activations/layer20_attention_weight_min": -23.336782455444336,
      "activations/layer21_attention_weight_max": 34.65329360961914,
      "activations/layer21_attention_weight_min": -21.788894653320312,
      "activations/layer22_attention_weight_max": 27.471088409423828,
      "activations/layer22_attention_weight_min": -23.457988739013672,
      "activations/layer23_attention_weight_max": 36.401878356933594,
      "activations/layer23_attention_weight_min": -26.742300033569336,
      "activations/layer2_attention_weight_max": 32.70370101928711,
      "activations/layer2_attention_weight_min": -30.640155792236328,
      "activations/layer3_attention_weight_max": 91.5023422241211,
      "activations/layer3_attention_weight_min": -91.6152114868164,
      "activations/layer4_attention_weight_max": 85.34077453613281,
      "activations/layer4_attention_weight_min": -83.57390594482422,
      "activations/layer5_attention_weight_max": 60.559120178222656,
      "activations/layer5_attention_weight_min": -75.31074523925781,
      "activations/layer6_attention_weight_max": 49.11457061767578,
      "activations/layer6_attention_weight_min": -49.91462326049805,
      "activations/layer7_attention_weight_max": 67.65762329101562,
      "activations/layer7_attention_weight_min": -72.44973754882812,
      "activations/layer8_attention_weight_max": 46.68009948730469,
      "activations/layer8_attention_weight_min": -50.84109115600586,
      "activations/layer9_attention_weight_max": 43.998130798339844,
      "activations/layer9_attention_weight_min": -47.56092071533203,
      "epoch": 22.23,
      "learning_rate": 6.6875e-06,
      "loss": 2.7124,
      "step": 382650
    },
    {
      "activations/layer0_attention_weight_max": 16.363466262817383,
      "activations/layer0_attention_weight_min": -13.914358139038086,
      "activations/layer10_attention_weight_max": 37.65675354003906,
      "activations/layer10_attention_weight_min": -37.17665481567383,
      "activations/layer11_attention_weight_max": 35.483917236328125,
      "activations/layer11_attention_weight_min": -35.293296813964844,
      "activations/layer12_attention_weight_max": 20.333051681518555,
      "activations/layer12_attention_weight_min": -28.32134246826172,
      "activations/layer13_attention_weight_max": 44.331993103027344,
      "activations/layer13_attention_weight_min": -31.101327896118164,
      "activations/layer14_attention_weight_max": 40.484832763671875,
      "activations/layer14_attention_weight_min": -32.544925689697266,
      "activations/layer15_attention_weight_max": 37.42612075805664,
      "activations/layer15_attention_weight_min": -32.70566177368164,
      "activations/layer16_attention_weight_max": 40.641136169433594,
      "activations/layer16_attention_weight_min": -35.53757858276367,
      "activations/layer17_attention_weight_max": 58.68369674682617,
      "activations/layer17_attention_weight_min": -49.689762115478516,
      "activations/layer18_attention_weight_max": 56.961143493652344,
      "activations/layer18_attention_weight_min": -43.58023452758789,
      "activations/layer19_attention_weight_max": 29.984140396118164,
      "activations/layer19_attention_weight_min": -24.185766220092773,
      "activations/layer1_attention_weight_max": 16.211040496826172,
      "activations/layer1_attention_weight_min": -13.801609992980957,
      "activations/layer20_attention_weight_max": 26.03515625,
      "activations/layer20_attention_weight_min": -21.817054748535156,
      "activations/layer21_attention_weight_max": 40.8641242980957,
      "activations/layer21_attention_weight_min": -23.901592254638672,
      "activations/layer22_attention_weight_max": 33.479183197021484,
      "activations/layer22_attention_weight_min": -26.850637435913086,
      "activations/layer23_attention_weight_max": 47.659400939941406,
      "activations/layer23_attention_weight_min": -22.908954620361328,
      "activations/layer2_attention_weight_max": 31.241012573242188,
      "activations/layer2_attention_weight_min": -29.906681060791016,
      "activations/layer3_attention_weight_max": 89.80311584472656,
      "activations/layer3_attention_weight_min": -91.62455749511719,
      "activations/layer4_attention_weight_max": 86.9967269897461,
      "activations/layer4_attention_weight_min": -85.99406433105469,
      "activations/layer5_attention_weight_max": 63.19824981689453,
      "activations/layer5_attention_weight_min": -72.76449584960938,
      "activations/layer6_attention_weight_max": 52.380088806152344,
      "activations/layer6_attention_weight_min": -55.08414077758789,
      "activations/layer7_attention_weight_max": 75.02365112304688,
      "activations/layer7_attention_weight_min": -71.85628509521484,
      "activations/layer8_attention_weight_max": 50.63874053955078,
      "activations/layer8_attention_weight_min": -53.1253547668457,
      "activations/layer9_attention_weight_max": 54.629425048828125,
      "activations/layer9_attention_weight_min": -51.45882034301758,
      "epoch": 22.24,
      "learning_rate": 6.668560606060606e-06,
      "loss": 2.709,
      "step": 382700
    },
    {
      "activations/layer0_attention_weight_max": 15.945564270019531,
      "activations/layer0_attention_weight_min": -14.157198905944824,
      "activations/layer10_attention_weight_max": 37.06772232055664,
      "activations/layer10_attention_weight_min": -36.33284378051758,
      "activations/layer11_attention_weight_max": 36.07129669189453,
      "activations/layer11_attention_weight_min": -35.90311050415039,
      "activations/layer12_attention_weight_max": 18.141849517822266,
      "activations/layer12_attention_weight_min": -28.54086685180664,
      "activations/layer13_attention_weight_max": 32.595306396484375,
      "activations/layer13_attention_weight_min": -26.294475555419922,
      "activations/layer14_attention_weight_max": 35.07273483276367,
      "activations/layer14_attention_weight_min": -30.458866119384766,
      "activations/layer15_attention_weight_max": 32.64179229736328,
      "activations/layer15_attention_weight_min": -30.750768661499023,
      "activations/layer16_attention_weight_max": 34.88935852050781,
      "activations/layer16_attention_weight_min": -32.700714111328125,
      "activations/layer17_attention_weight_max": 54.79976272583008,
      "activations/layer17_attention_weight_min": -47.562660217285156,
      "activations/layer18_attention_weight_max": 50.908119201660156,
      "activations/layer18_attention_weight_min": -40.64158630371094,
      "activations/layer19_attention_weight_max": 27.02704429626465,
      "activations/layer19_attention_weight_min": -23.97942352294922,
      "activations/layer1_attention_weight_max": 14.896591186523438,
      "activations/layer1_attention_weight_min": -14.294721603393555,
      "activations/layer20_attention_weight_max": 26.684791564941406,
      "activations/layer20_attention_weight_min": -21.58671760559082,
      "activations/layer21_attention_weight_max": 42.19682693481445,
      "activations/layer21_attention_weight_min": -23.872207641601562,
      "activations/layer22_attention_weight_max": 36.17088317871094,
      "activations/layer22_attention_weight_min": -24.9161434173584,
      "activations/layer23_attention_weight_max": 37.131465911865234,
      "activations/layer23_attention_weight_min": -23.986370086669922,
      "activations/layer2_attention_weight_max": 32.329566955566406,
      "activations/layer2_attention_weight_min": -30.972129821777344,
      "activations/layer3_attention_weight_max": 91.6391372680664,
      "activations/layer3_attention_weight_min": -87.95259094238281,
      "activations/layer4_attention_weight_max": 89.91883850097656,
      "activations/layer4_attention_weight_min": -85.5593490600586,
      "activations/layer5_attention_weight_max": 64.09124755859375,
      "activations/layer5_attention_weight_min": -72.56087493896484,
      "activations/layer6_attention_weight_max": 52.217411041259766,
      "activations/layer6_attention_weight_min": -50.701656341552734,
      "activations/layer7_attention_weight_max": 70.32007598876953,
      "activations/layer7_attention_weight_min": -68.49010467529297,
      "activations/layer8_attention_weight_max": 54.04194259643555,
      "activations/layer8_attention_weight_min": -57.88161087036133,
      "activations/layer9_attention_weight_max": 45.77730941772461,
      "activations/layer9_attention_weight_min": -46.88571548461914,
      "epoch": 22.24,
      "learning_rate": 6.649621212121211e-06,
      "loss": 2.6968,
      "step": 382750
    },
    {
      "activations/layer0_attention_weight_max": 15.708608627319336,
      "activations/layer0_attention_weight_min": -14.335265159606934,
      "activations/layer10_attention_weight_max": 37.86784744262695,
      "activations/layer10_attention_weight_min": -39.850807189941406,
      "activations/layer11_attention_weight_max": 40.19198226928711,
      "activations/layer11_attention_weight_min": -41.998779296875,
      "activations/layer12_attention_weight_max": 19.1401424407959,
      "activations/layer12_attention_weight_min": -25.838802337646484,
      "activations/layer13_attention_weight_max": 37.48176574707031,
      "activations/layer13_attention_weight_min": -30.92885398864746,
      "activations/layer14_attention_weight_max": 46.10010528564453,
      "activations/layer14_attention_weight_min": -35.86990737915039,
      "activations/layer15_attention_weight_max": 42.789554595947266,
      "activations/layer15_attention_weight_min": -36.077049255371094,
      "activations/layer16_attention_weight_max": 39.74118423461914,
      "activations/layer16_attention_weight_min": -36.325416564941406,
      "activations/layer17_attention_weight_max": 57.63765335083008,
      "activations/layer17_attention_weight_min": -53.9402961730957,
      "activations/layer18_attention_weight_max": 55.600528717041016,
      "activations/layer18_attention_weight_min": -52.16106414794922,
      "activations/layer19_attention_weight_max": 28.875722885131836,
      "activations/layer19_attention_weight_min": -29.848806381225586,
      "activations/layer1_attention_weight_max": 14.838540077209473,
      "activations/layer1_attention_weight_min": -13.816417694091797,
      "activations/layer20_attention_weight_max": 28.687103271484375,
      "activations/layer20_attention_weight_min": -25.43352508544922,
      "activations/layer21_attention_weight_max": 40.46118927001953,
      "activations/layer21_attention_weight_min": -27.045297622680664,
      "activations/layer22_attention_weight_max": 30.651702880859375,
      "activations/layer22_attention_weight_min": -27.079288482666016,
      "activations/layer23_attention_weight_max": 39.415802001953125,
      "activations/layer23_attention_weight_min": -25.92330551147461,
      "activations/layer2_attention_weight_max": 34.37105941772461,
      "activations/layer2_attention_weight_min": -32.061431884765625,
      "activations/layer3_attention_weight_max": 92.48124694824219,
      "activations/layer3_attention_weight_min": -92.17205810546875,
      "activations/layer4_attention_weight_max": 89.87434387207031,
      "activations/layer4_attention_weight_min": -89.99272155761719,
      "activations/layer5_attention_weight_max": 64.385498046875,
      "activations/layer5_attention_weight_min": -71.18936157226562,
      "activations/layer6_attention_weight_max": 52.74552917480469,
      "activations/layer6_attention_weight_min": -53.14165115356445,
      "activations/layer7_attention_weight_max": 76.54790496826172,
      "activations/layer7_attention_weight_min": -71.37476348876953,
      "activations/layer8_attention_weight_max": 50.45641326904297,
      "activations/layer8_attention_weight_min": -52.397186279296875,
      "activations/layer9_attention_weight_max": 48.227230072021484,
      "activations/layer9_attention_weight_min": -49.042476654052734,
      "epoch": 22.24,
      "learning_rate": 6.630681818181818e-06,
      "loss": 2.7448,
      "step": 382800
    },
    {
      "activations/layer0_attention_weight_max": 15.421545028686523,
      "activations/layer0_attention_weight_min": -14.177597045898438,
      "activations/layer10_attention_weight_max": 34.53326416015625,
      "activations/layer10_attention_weight_min": -35.40421676635742,
      "activations/layer11_attention_weight_max": 31.584789276123047,
      "activations/layer11_attention_weight_min": -34.51946258544922,
      "activations/layer12_attention_weight_max": 19.51091766357422,
      "activations/layer12_attention_weight_min": -28.9389705657959,
      "activations/layer13_attention_weight_max": 32.656490325927734,
      "activations/layer13_attention_weight_min": -27.71184730529785,
      "activations/layer14_attention_weight_max": 34.8585205078125,
      "activations/layer14_attention_weight_min": -30.323274612426758,
      "activations/layer15_attention_weight_max": 32.331138610839844,
      "activations/layer15_attention_weight_min": -29.294309616088867,
      "activations/layer16_attention_weight_max": 34.092830657958984,
      "activations/layer16_attention_weight_min": -31.973995208740234,
      "activations/layer17_attention_weight_max": 52.76274108886719,
      "activations/layer17_attention_weight_min": -43.81890869140625,
      "activations/layer18_attention_weight_max": 47.78021240234375,
      "activations/layer18_attention_weight_min": -38.97257614135742,
      "activations/layer19_attention_weight_max": 23.962663650512695,
      "activations/layer19_attention_weight_min": -23.361310958862305,
      "activations/layer1_attention_weight_max": 15.11679744720459,
      "activations/layer1_attention_weight_min": -13.77229118347168,
      "activations/layer20_attention_weight_max": 21.261638641357422,
      "activations/layer20_attention_weight_min": -21.13861656188965,
      "activations/layer21_attention_weight_max": 34.601192474365234,
      "activations/layer21_attention_weight_min": -22.41179084777832,
      "activations/layer22_attention_weight_max": 27.40310287475586,
      "activations/layer22_attention_weight_min": -24.834213256835938,
      "activations/layer23_attention_weight_max": 33.68598175048828,
      "activations/layer23_attention_weight_min": -23.09991455078125,
      "activations/layer2_attention_weight_max": 32.836631774902344,
      "activations/layer2_attention_weight_min": -30.426639556884766,
      "activations/layer3_attention_weight_max": 90.42366790771484,
      "activations/layer3_attention_weight_min": -89.37007141113281,
      "activations/layer4_attention_weight_max": 87.16008758544922,
      "activations/layer4_attention_weight_min": -86.53893280029297,
      "activations/layer5_attention_weight_max": 63.49555969238281,
      "activations/layer5_attention_weight_min": -75.73995971679688,
      "activations/layer6_attention_weight_max": 50.54691696166992,
      "activations/layer6_attention_weight_min": -53.94904327392578,
      "activations/layer7_attention_weight_max": 70.92096710205078,
      "activations/layer7_attention_weight_min": -74.63436889648438,
      "activations/layer8_attention_weight_max": 50.55812454223633,
      "activations/layer8_attention_weight_min": -52.0573844909668,
      "activations/layer9_attention_weight_max": 44.41117477416992,
      "activations/layer9_attention_weight_min": -53.101287841796875,
      "epoch": 22.25,
      "learning_rate": 6.611742424242424e-06,
      "loss": 2.71,
      "step": 382850
    },
    {
      "activations/layer0_attention_weight_max": 16.732933044433594,
      "activations/layer0_attention_weight_min": -13.827232360839844,
      "activations/layer10_attention_weight_max": 43.3157844543457,
      "activations/layer10_attention_weight_min": -43.80821990966797,
      "activations/layer11_attention_weight_max": 43.23932647705078,
      "activations/layer11_attention_weight_min": -39.001441955566406,
      "activations/layer12_attention_weight_max": 17.871244430541992,
      "activations/layer12_attention_weight_min": -32.3132438659668,
      "activations/layer13_attention_weight_max": 53.862693786621094,
      "activations/layer13_attention_weight_min": -33.554691314697266,
      "activations/layer14_attention_weight_max": 52.10858917236328,
      "activations/layer14_attention_weight_min": -30.13495445251465,
      "activations/layer15_attention_weight_max": 55.14071273803711,
      "activations/layer15_attention_weight_min": -33.098167419433594,
      "activations/layer16_attention_weight_max": 35.64288330078125,
      "activations/layer16_attention_weight_min": -32.539825439453125,
      "activations/layer17_attention_weight_max": 54.55622100830078,
      "activations/layer17_attention_weight_min": -48.86954116821289,
      "activations/layer18_attention_weight_max": 46.7145881652832,
      "activations/layer18_attention_weight_min": -43.01690673828125,
      "activations/layer19_attention_weight_max": 23.93054962158203,
      "activations/layer19_attention_weight_min": -23.88397789001465,
      "activations/layer1_attention_weight_max": 15.008536338806152,
      "activations/layer1_attention_weight_min": -14.012184143066406,
      "activations/layer20_attention_weight_max": 23.941478729248047,
      "activations/layer20_attention_weight_min": -20.392972946166992,
      "activations/layer21_attention_weight_max": 32.84804153442383,
      "activations/layer21_attention_weight_min": -24.635356903076172,
      "activations/layer22_attention_weight_max": 26.63055419921875,
      "activations/layer22_attention_weight_min": -23.73455810546875,
      "activations/layer23_attention_weight_max": 32.73068618774414,
      "activations/layer23_attention_weight_min": -23.070537567138672,
      "activations/layer2_attention_weight_max": 32.25993728637695,
      "activations/layer2_attention_weight_min": -29.8564453125,
      "activations/layer3_attention_weight_max": 88.49950408935547,
      "activations/layer3_attention_weight_min": -86.0,
      "activations/layer4_attention_weight_max": 82.7033920288086,
      "activations/layer4_attention_weight_min": -83.7944564819336,
      "activations/layer5_attention_weight_max": 63.83115768432617,
      "activations/layer5_attention_weight_min": -71.54747009277344,
      "activations/layer6_attention_weight_max": 52.10860061645508,
      "activations/layer6_attention_weight_min": -50.82449722290039,
      "activations/layer7_attention_weight_max": 79.99183654785156,
      "activations/layer7_attention_weight_min": -66.18965148925781,
      "activations/layer8_attention_weight_max": 57.27730178833008,
      "activations/layer8_attention_weight_min": -52.86076736450195,
      "activations/layer9_attention_weight_max": 61.231201171875,
      "activations/layer9_attention_weight_min": -50.37919235229492,
      "epoch": 22.25,
      "learning_rate": 6.592803030303029e-06,
      "loss": 2.6958,
      "step": 382900
    },
    {
      "activations/layer0_attention_weight_max": 16.394258499145508,
      "activations/layer0_attention_weight_min": -13.934386253356934,
      "activations/layer10_attention_weight_max": 35.12265396118164,
      "activations/layer10_attention_weight_min": -36.16029739379883,
      "activations/layer11_attention_weight_max": 33.45822525024414,
      "activations/layer11_attention_weight_min": -34.82007598876953,
      "activations/layer12_attention_weight_max": 19.86919593811035,
      "activations/layer12_attention_weight_min": -25.289813995361328,
      "activations/layer13_attention_weight_max": 33.46039962768555,
      "activations/layer13_attention_weight_min": -26.191038131713867,
      "activations/layer14_attention_weight_max": 32.784210205078125,
      "activations/layer14_attention_weight_min": -31.52975082397461,
      "activations/layer15_attention_weight_max": 30.89061737060547,
      "activations/layer15_attention_weight_min": -30.241802215576172,
      "activations/layer16_attention_weight_max": 32.418155670166016,
      "activations/layer16_attention_weight_min": -33.64407730102539,
      "activations/layer17_attention_weight_max": 48.08076095581055,
      "activations/layer17_attention_weight_min": -45.351173400878906,
      "activations/layer18_attention_weight_max": 42.45737838745117,
      "activations/layer18_attention_weight_min": -39.743934631347656,
      "activations/layer19_attention_weight_max": 22.62103843688965,
      "activations/layer19_attention_weight_min": -23.214221954345703,
      "activations/layer1_attention_weight_max": 15.242576599121094,
      "activations/layer1_attention_weight_min": -14.00326156616211,
      "activations/layer20_attention_weight_max": 21.941801071166992,
      "activations/layer20_attention_weight_min": -24.063053131103516,
      "activations/layer21_attention_weight_max": 32.15354919433594,
      "activations/layer21_attention_weight_min": -19.896183013916016,
      "activations/layer22_attention_weight_max": 25.066621780395508,
      "activations/layer22_attention_weight_min": -25.52243423461914,
      "activations/layer23_attention_weight_max": 33.49932098388672,
      "activations/layer23_attention_weight_min": -22.417354583740234,
      "activations/layer2_attention_weight_max": 31.444828033447266,
      "activations/layer2_attention_weight_min": -29.977092742919922,
      "activations/layer3_attention_weight_max": 90.11722564697266,
      "activations/layer3_attention_weight_min": -86.8809585571289,
      "activations/layer4_attention_weight_max": 83.4919662475586,
      "activations/layer4_attention_weight_min": -82.54644012451172,
      "activations/layer5_attention_weight_max": 62.98887252807617,
      "activations/layer5_attention_weight_min": -73.50335693359375,
      "activations/layer6_attention_weight_max": 47.8897819519043,
      "activations/layer6_attention_weight_min": -49.72122573852539,
      "activations/layer7_attention_weight_max": 68.38314056396484,
      "activations/layer7_attention_weight_min": -65.68281555175781,
      "activations/layer8_attention_weight_max": 45.91249084472656,
      "activations/layer8_attention_weight_min": -48.41682052612305,
      "activations/layer9_attention_weight_max": 42.85454177856445,
      "activations/layer9_attention_weight_min": -44.5134391784668,
      "epoch": 22.25,
      "learning_rate": 6.573863636363635e-06,
      "loss": 2.7092,
      "step": 382950
    },
    {
      "activations/layer0_attention_weight_max": 15.974161148071289,
      "activations/layer0_attention_weight_min": -14.134138107299805,
      "activations/layer10_attention_weight_max": 38.61243438720703,
      "activations/layer10_attention_weight_min": -36.914390563964844,
      "activations/layer11_attention_weight_max": 34.102108001708984,
      "activations/layer11_attention_weight_min": -36.218196868896484,
      "activations/layer12_attention_weight_max": 19.869522094726562,
      "activations/layer12_attention_weight_min": -22.127670288085938,
      "activations/layer13_attention_weight_max": 34.69843292236328,
      "activations/layer13_attention_weight_min": -27.898462295532227,
      "activations/layer14_attention_weight_max": 37.88979721069336,
      "activations/layer14_attention_weight_min": -30.53582191467285,
      "activations/layer15_attention_weight_max": 35.80764389038086,
      "activations/layer15_attention_weight_min": -31.50430679321289,
      "activations/layer16_attention_weight_max": 36.92037582397461,
      "activations/layer16_attention_weight_min": -34.8659782409668,
      "activations/layer17_attention_weight_max": 51.67406463623047,
      "activations/layer17_attention_weight_min": -46.05217742919922,
      "activations/layer18_attention_weight_max": 50.7044792175293,
      "activations/layer18_attention_weight_min": -42.1474723815918,
      "activations/layer19_attention_weight_max": 27.68699073791504,
      "activations/layer19_attention_weight_min": -23.14832878112793,
      "activations/layer1_attention_weight_max": 14.608198165893555,
      "activations/layer1_attention_weight_min": -14.218426704406738,
      "activations/layer20_attention_weight_max": 25.94866371154785,
      "activations/layer20_attention_weight_min": -22.009355545043945,
      "activations/layer21_attention_weight_max": 42.973140716552734,
      "activations/layer21_attention_weight_min": -23.39535903930664,
      "activations/layer22_attention_weight_max": 29.75577735900879,
      "activations/layer22_attention_weight_min": -24.72519874572754,
      "activations/layer23_attention_weight_max": 45.735713958740234,
      "activations/layer23_attention_weight_min": -23.095075607299805,
      "activations/layer2_attention_weight_max": 31.47098159790039,
      "activations/layer2_attention_weight_min": -30.291669845581055,
      "activations/layer3_attention_weight_max": 92.78192138671875,
      "activations/layer3_attention_weight_min": -93.71822357177734,
      "activations/layer4_attention_weight_max": 84.3936996459961,
      "activations/layer4_attention_weight_min": -86.28260040283203,
      "activations/layer5_attention_weight_max": 62.36775207519531,
      "activations/layer5_attention_weight_min": -73.71236419677734,
      "activations/layer6_attention_weight_max": 52.64152526855469,
      "activations/layer6_attention_weight_min": -51.67911148071289,
      "activations/layer7_attention_weight_max": 72.17597198486328,
      "activations/layer7_attention_weight_min": -71.2955551147461,
      "activations/layer8_attention_weight_max": 55.30492401123047,
      "activations/layer8_attention_weight_min": -54.887210845947266,
      "activations/layer9_attention_weight_max": 50.840206146240234,
      "activations/layer9_attention_weight_min": -51.9941291809082,
      "epoch": 22.25,
      "learning_rate": 6.554924242424242e-06,
      "loss": 2.6953,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_loss": 2.66796875,
      "eval_runtime": 8.4975,
      "eval_samples_per_second": 505.324,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_openwebtext_loss": 2.66796875,
      "eval_openwebtext_ppl": 14.4106677747075,
      "eval_openwebtext_runtime": 8.4975,
      "eval_openwebtext_samples_per_second": 505.324,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_wikitext_loss": 2.90234375,
      "eval_wikitext_ppl": 18.216790978454146,
      "eval_wikitext_runtime": 1.9783,
      "eval_wikitext_samples_per_second": 230.506,
      "step": 383000
    },
    {
      "epoch": 22.25,
      "eval_lambada_loss": 2.380859375,
      "eval_lambada_ppl": 10.814192318320675,
      "eval_lambada_runtime": 9.5448,
      "eval_lambada_samples_per_second": 510.123,
      "step": 383000
    },
    {
      "activations/layer0_attention_weight_max": 16.16739845275879,
      "activations/layer0_attention_weight_min": -14.034857749938965,
      "activations/layer10_attention_weight_max": 36.00069046020508,
      "activations/layer10_attention_weight_min": -37.460723876953125,
      "activations/layer11_attention_weight_max": 32.99056625366211,
      "activations/layer11_attention_weight_min": -35.373313903808594,
      "activations/layer12_attention_weight_max": 17.68265724182129,
      "activations/layer12_attention_weight_min": -23.818540573120117,
      "activations/layer13_attention_weight_max": 34.29667282104492,
      "activations/layer13_attention_weight_min": -27.43927764892578,
      "activations/layer14_attention_weight_max": 36.36664581298828,
      "activations/layer14_attention_weight_min": -31.36185646057129,
      "activations/layer15_attention_weight_max": 33.63102722167969,
      "activations/layer15_attention_weight_min": -29.82701301574707,
      "activations/layer16_attention_weight_max": 31.49226951599121,
      "activations/layer16_attention_weight_min": -31.12888526916504,
      "activations/layer17_attention_weight_max": 48.94490051269531,
      "activations/layer17_attention_weight_min": -42.84919738769531,
      "activations/layer18_attention_weight_max": 46.25080871582031,
      "activations/layer18_attention_weight_min": -39.69492721557617,
      "activations/layer19_attention_weight_max": 25.307327270507812,
      "activations/layer19_attention_weight_min": -23.3681697845459,
      "activations/layer1_attention_weight_max": 16.3743839263916,
      "activations/layer1_attention_weight_min": -14.131429672241211,
      "activations/layer20_attention_weight_max": 21.980241775512695,
      "activations/layer20_attention_weight_min": -22.842330932617188,
      "activations/layer21_attention_weight_max": 37.59331130981445,
      "activations/layer21_attention_weight_min": -24.621667861938477,
      "activations/layer22_attention_weight_max": 28.98375129699707,
      "activations/layer22_attention_weight_min": -27.206886291503906,
      "activations/layer23_attention_weight_max": 35.36006164550781,
      "activations/layer23_attention_weight_min": -23.959964752197266,
      "activations/layer2_attention_weight_max": 32.51998519897461,
      "activations/layer2_attention_weight_min": -29.9642333984375,
      "activations/layer3_attention_weight_max": 92.86053466796875,
      "activations/layer3_attention_weight_min": -90.21288299560547,
      "activations/layer4_attention_weight_max": 87.25347900390625,
      "activations/layer4_attention_weight_min": -86.16792297363281,
      "activations/layer5_attention_weight_max": 66.09004211425781,
      "activations/layer5_attention_weight_min": -73.72754669189453,
      "activations/layer6_attention_weight_max": 49.888851165771484,
      "activations/layer6_attention_weight_min": -53.5910758972168,
      "activations/layer7_attention_weight_max": 70.53485870361328,
      "activations/layer7_attention_weight_min": -68.42780303955078,
      "activations/layer8_attention_weight_max": 48.648963928222656,
      "activations/layer8_attention_weight_min": -53.431846618652344,
      "activations/layer9_attention_weight_max": 46.14744186401367,
      "activations/layer9_attention_weight_min": -50.077239990234375,
      "epoch": 22.26,
      "learning_rate": 6.535984848484848e-06,
      "loss": 2.7035,
      "step": 383050
    },
    {
      "activations/layer0_attention_weight_max": 15.940459251403809,
      "activations/layer0_attention_weight_min": -14.31162166595459,
      "activations/layer10_attention_weight_max": 37.60992431640625,
      "activations/layer10_attention_weight_min": -36.789581298828125,
      "activations/layer11_attention_weight_max": 36.636314392089844,
      "activations/layer11_attention_weight_min": -34.66322326660156,
      "activations/layer12_attention_weight_max": 19.851165771484375,
      "activations/layer12_attention_weight_min": -29.131486892700195,
      "activations/layer13_attention_weight_max": 33.21431350708008,
      "activations/layer13_attention_weight_min": -28.643407821655273,
      "activations/layer14_attention_weight_max": 35.75711441040039,
      "activations/layer14_attention_weight_min": -30.650724411010742,
      "activations/layer15_attention_weight_max": 38.37197494506836,
      "activations/layer15_attention_weight_min": -29.327476501464844,
      "activations/layer16_attention_weight_max": 36.10594177246094,
      "activations/layer16_attention_weight_min": -33.35732650756836,
      "activations/layer17_attention_weight_max": 52.32851028442383,
      "activations/layer17_attention_weight_min": -45.278934478759766,
      "activations/layer18_attention_weight_max": 44.17093276977539,
      "activations/layer18_attention_weight_min": -44.406768798828125,
      "activations/layer19_attention_weight_max": 23.91951560974121,
      "activations/layer19_attention_weight_min": -26.64815330505371,
      "activations/layer1_attention_weight_max": 16.304428100585938,
      "activations/layer1_attention_weight_min": -14.996481895446777,
      "activations/layer20_attention_weight_max": 22.12128257751465,
      "activations/layer20_attention_weight_min": -24.03387451171875,
      "activations/layer21_attention_weight_max": 36.315940856933594,
      "activations/layer21_attention_weight_min": -23.13665199279785,
      "activations/layer22_attention_weight_max": 26.076433181762695,
      "activations/layer22_attention_weight_min": -25.001747131347656,
      "activations/layer23_attention_weight_max": 31.971355438232422,
      "activations/layer23_attention_weight_min": -22.764156341552734,
      "activations/layer2_attention_weight_max": 31.807170867919922,
      "activations/layer2_attention_weight_min": -30.396099090576172,
      "activations/layer3_attention_weight_max": 88.62418365478516,
      "activations/layer3_attention_weight_min": -89.33606719970703,
      "activations/layer4_attention_weight_max": 83.24530792236328,
      "activations/layer4_attention_weight_min": -81.2652359008789,
      "activations/layer5_attention_weight_max": 62.48588180541992,
      "activations/layer5_attention_weight_min": -70.9510498046875,
      "activations/layer6_attention_weight_max": 48.13092803955078,
      "activations/layer6_attention_weight_min": -48.15446472167969,
      "activations/layer7_attention_weight_max": 70.12183380126953,
      "activations/layer7_attention_weight_min": -64.96084594726562,
      "activations/layer8_attention_weight_max": 48.9538459777832,
      "activations/layer8_attention_weight_min": -48.76787185668945,
      "activations/layer9_attention_weight_max": 52.468482971191406,
      "activations/layer9_attention_weight_min": -44.2693977355957,
      "epoch": 22.26,
      "learning_rate": 6.517045454545454e-06,
      "loss": 2.7107,
      "step": 383100
    },
    {
      "activations/layer0_attention_weight_max": 17.246593475341797,
      "activations/layer0_attention_weight_min": -14.113835334777832,
      "activations/layer10_attention_weight_max": 38.171470642089844,
      "activations/layer10_attention_weight_min": -37.004249572753906,
      "activations/layer11_attention_weight_max": 34.077178955078125,
      "activations/layer11_attention_weight_min": -34.80869674682617,
      "activations/layer12_attention_weight_max": 19.307296752929688,
      "activations/layer12_attention_weight_min": -25.857030868530273,
      "activations/layer13_attention_weight_max": 35.44717025756836,
      "activations/layer13_attention_weight_min": -27.849441528320312,
      "activations/layer14_attention_weight_max": 33.44285583496094,
      "activations/layer14_attention_weight_min": -30.510358810424805,
      "activations/layer15_attention_weight_max": 32.867549896240234,
      "activations/layer15_attention_weight_min": -30.772802352905273,
      "activations/layer16_attention_weight_max": 34.963294982910156,
      "activations/layer16_attention_weight_min": -33.85896682739258,
      "activations/layer17_attention_weight_max": 53.88460159301758,
      "activations/layer17_attention_weight_min": -48.30231475830078,
      "activations/layer18_attention_weight_max": 47.5120735168457,
      "activations/layer18_attention_weight_min": -42.43178176879883,
      "activations/layer19_attention_weight_max": 25.903949737548828,
      "activations/layer19_attention_weight_min": -22.68895149230957,
      "activations/layer1_attention_weight_max": 14.357826232910156,
      "activations/layer1_attention_weight_min": -14.85461139678955,
      "activations/layer20_attention_weight_max": 23.842470169067383,
      "activations/layer20_attention_weight_min": -21.549623489379883,
      "activations/layer21_attention_weight_max": 33.73692321777344,
      "activations/layer21_attention_weight_min": -21.88859748840332,
      "activations/layer22_attention_weight_max": 28.020811080932617,
      "activations/layer22_attention_weight_min": -25.394243240356445,
      "activations/layer23_attention_weight_max": 35.650142669677734,
      "activations/layer23_attention_weight_min": -25.22183609008789,
      "activations/layer2_attention_weight_max": 31.853984832763672,
      "activations/layer2_attention_weight_min": -30.205230712890625,
      "activations/layer3_attention_weight_max": 87.13232421875,
      "activations/layer3_attention_weight_min": -84.6273422241211,
      "activations/layer4_attention_weight_max": 83.70382690429688,
      "activations/layer4_attention_weight_min": -79.44247436523438,
      "activations/layer5_attention_weight_max": 62.99569320678711,
      "activations/layer5_attention_weight_min": -70.67169189453125,
      "activations/layer6_attention_weight_max": 49.44581604003906,
      "activations/layer6_attention_weight_min": -49.38117980957031,
      "activations/layer7_attention_weight_max": 69.07939147949219,
      "activations/layer7_attention_weight_min": -66.23114776611328,
      "activations/layer8_attention_weight_max": 48.00762176513672,
      "activations/layer8_attention_weight_min": -50.74878692626953,
      "activations/layer9_attention_weight_max": 47.54897689819336,
      "activations/layer9_attention_weight_min": -45.31303024291992,
      "epoch": 22.26,
      "learning_rate": 6.498106060606061e-06,
      "loss": 2.7082,
      "step": 383150
    },
    {
      "activations/layer0_attention_weight_max": 17.051631927490234,
      "activations/layer0_attention_weight_min": -14.01634407043457,
      "activations/layer10_attention_weight_max": 40.81821823120117,
      "activations/layer10_attention_weight_min": -38.165218353271484,
      "activations/layer11_attention_weight_max": 38.92731475830078,
      "activations/layer11_attention_weight_min": -37.508575439453125,
      "activations/layer12_attention_weight_max": 19.3570613861084,
      "activations/layer12_attention_weight_min": -28.935482025146484,
      "activations/layer13_attention_weight_max": 38.9114990234375,
      "activations/layer13_attention_weight_min": -27.912776947021484,
      "activations/layer14_attention_weight_max": 40.54744338989258,
      "activations/layer14_attention_weight_min": -32.28618621826172,
      "activations/layer15_attention_weight_max": 42.13007354736328,
      "activations/layer15_attention_weight_min": -32.72700500488281,
      "activations/layer16_attention_weight_max": 40.199249267578125,
      "activations/layer16_attention_weight_min": -35.998817443847656,
      "activations/layer17_attention_weight_max": 58.46236801147461,
      "activations/layer17_attention_weight_min": -50.52180862426758,
      "activations/layer18_attention_weight_max": 52.67839813232422,
      "activations/layer18_attention_weight_min": -49.09141540527344,
      "activations/layer19_attention_weight_max": 26.04205894470215,
      "activations/layer19_attention_weight_min": -26.529911041259766,
      "activations/layer1_attention_weight_max": 14.985641479492188,
      "activations/layer1_attention_weight_min": -14.157903671264648,
      "activations/layer20_attention_weight_max": 27.664011001586914,
      "activations/layer20_attention_weight_min": -23.36261558532715,
      "activations/layer21_attention_weight_max": 49.24913024902344,
      "activations/layer21_attention_weight_min": -24.632848739624023,
      "activations/layer22_attention_weight_max": 34.03046417236328,
      "activations/layer22_attention_weight_min": -25.69135093688965,
      "activations/layer23_attention_weight_max": 39.05621337890625,
      "activations/layer23_attention_weight_min": -26.519681930541992,
      "activations/layer2_attention_weight_max": 33.29253005981445,
      "activations/layer2_attention_weight_min": -30.471393585205078,
      "activations/layer3_attention_weight_max": 93.31290435791016,
      "activations/layer3_attention_weight_min": -88.84373474121094,
      "activations/layer4_attention_weight_max": 82.17418670654297,
      "activations/layer4_attention_weight_min": -80.8550796508789,
      "activations/layer5_attention_weight_max": 61.3025016784668,
      "activations/layer5_attention_weight_min": -73.83873748779297,
      "activations/layer6_attention_weight_max": 46.94738006591797,
      "activations/layer6_attention_weight_min": -48.80342102050781,
      "activations/layer7_attention_weight_max": 72.33029174804688,
      "activations/layer7_attention_weight_min": -67.72904205322266,
      "activations/layer8_attention_weight_max": 51.574058532714844,
      "activations/layer8_attention_weight_min": -48.9063606262207,
      "activations/layer9_attention_weight_max": 52.64509963989258,
      "activations/layer9_attention_weight_min": -46.052059173583984,
      "epoch": 22.27,
      "learning_rate": 6.479166666666666e-06,
      "loss": 2.7067,
      "step": 383200
    },
    {
      "activations/layer0_attention_weight_max": 16.068984985351562,
      "activations/layer0_attention_weight_min": -14.423798561096191,
      "activations/layer10_attention_weight_max": 39.92707443237305,
      "activations/layer10_attention_weight_min": -37.089012145996094,
      "activations/layer11_attention_weight_max": 37.720054626464844,
      "activations/layer11_attention_weight_min": -35.18516540527344,
      "activations/layer12_attention_weight_max": 18.874074935913086,
      "activations/layer12_attention_weight_min": -24.200651168823242,
      "activations/layer13_attention_weight_max": 34.639217376708984,
      "activations/layer13_attention_weight_min": -28.15964698791504,
      "activations/layer14_attention_weight_max": 35.10380172729492,
      "activations/layer14_attention_weight_min": -31.477027893066406,
      "activations/layer15_attention_weight_max": 32.631072998046875,
      "activations/layer15_attention_weight_min": -28.797447204589844,
      "activations/layer16_attention_weight_max": 34.788448333740234,
      "activations/layer16_attention_weight_min": -31.767240524291992,
      "activations/layer17_attention_weight_max": 51.40255355834961,
      "activations/layer17_attention_weight_min": -43.675235748291016,
      "activations/layer18_attention_weight_max": 46.59917449951172,
      "activations/layer18_attention_weight_min": -40.306583404541016,
      "activations/layer19_attention_weight_max": 24.47658920288086,
      "activations/layer19_attention_weight_min": -22.34728240966797,
      "activations/layer1_attention_weight_max": 15.691658973693848,
      "activations/layer1_attention_weight_min": -14.62042236328125,
      "activations/layer20_attention_weight_max": 23.41962242126465,
      "activations/layer20_attention_weight_min": -20.826900482177734,
      "activations/layer21_attention_weight_max": 36.83077621459961,
      "activations/layer21_attention_weight_min": -23.80344009399414,
      "activations/layer22_attention_weight_max": 29.92557144165039,
      "activations/layer22_attention_weight_min": -24.27130889892578,
      "activations/layer23_attention_weight_max": 35.769405364990234,
      "activations/layer23_attention_weight_min": -24.09882354736328,
      "activations/layer2_attention_weight_max": 31.552364349365234,
      "activations/layer2_attention_weight_min": -30.092567443847656,
      "activations/layer3_attention_weight_max": 88.44020080566406,
      "activations/layer3_attention_weight_min": -92.06871795654297,
      "activations/layer4_attention_weight_max": 83.92133331298828,
      "activations/layer4_attention_weight_min": -85.58241271972656,
      "activations/layer5_attention_weight_max": 60.92643356323242,
      "activations/layer5_attention_weight_min": -69.29647827148438,
      "activations/layer6_attention_weight_max": 48.714988708496094,
      "activations/layer6_attention_weight_min": -51.25627899169922,
      "activations/layer7_attention_weight_max": 67.28956604003906,
      "activations/layer7_attention_weight_min": -72.29563903808594,
      "activations/layer8_attention_weight_max": 51.65483856201172,
      "activations/layer8_attention_weight_min": -52.48276138305664,
      "activations/layer9_attention_weight_max": 49.92788314819336,
      "activations/layer9_attention_weight_min": -47.28021240234375,
      "epoch": 22.27,
      "learning_rate": 6.460227272727272e-06,
      "loss": 2.7132,
      "step": 383250
    },
    {
      "activations/layer0_attention_weight_max": 16.088876724243164,
      "activations/layer0_attention_weight_min": -14.018135070800781,
      "activations/layer10_attention_weight_max": 37.082069396972656,
      "activations/layer10_attention_weight_min": -37.54534912109375,
      "activations/layer11_attention_weight_max": 34.61884689331055,
      "activations/layer11_attention_weight_min": -36.75950622558594,
      "activations/layer12_attention_weight_max": 17.578519821166992,
      "activations/layer12_attention_weight_min": -27.505516052246094,
      "activations/layer13_attention_weight_max": 33.116783142089844,
      "activations/layer13_attention_weight_min": -25.439992904663086,
      "activations/layer14_attention_weight_max": 35.141536712646484,
      "activations/layer14_attention_weight_min": -29.522138595581055,
      "activations/layer15_attention_weight_max": 33.94719696044922,
      "activations/layer15_attention_weight_min": -31.433935165405273,
      "activations/layer16_attention_weight_max": 34.449703216552734,
      "activations/layer16_attention_weight_min": -32.72017288208008,
      "activations/layer17_attention_weight_max": 49.33794021606445,
      "activations/layer17_attention_weight_min": -44.06983947753906,
      "activations/layer18_attention_weight_max": 46.18307876586914,
      "activations/layer18_attention_weight_min": -40.84074401855469,
      "activations/layer19_attention_weight_max": 23.405872344970703,
      "activations/layer19_attention_weight_min": -23.95100212097168,
      "activations/layer1_attention_weight_max": 14.962725639343262,
      "activations/layer1_attention_weight_min": -14.380890846252441,
      "activations/layer20_attention_weight_max": 21.060832977294922,
      "activations/layer20_attention_weight_min": -22.609371185302734,
      "activations/layer21_attention_weight_max": 31.345626831054688,
      "activations/layer21_attention_weight_min": -19.633554458618164,
      "activations/layer22_attention_weight_max": 26.854768753051758,
      "activations/layer22_attention_weight_min": -24.62154197692871,
      "activations/layer23_attention_weight_max": 33.29594421386719,
      "activations/layer23_attention_weight_min": -25.642017364501953,
      "activations/layer2_attention_weight_max": 31.781234741210938,
      "activations/layer2_attention_weight_min": -29.959836959838867,
      "activations/layer3_attention_weight_max": 90.73082733154297,
      "activations/layer3_attention_weight_min": -89.91234588623047,
      "activations/layer4_attention_weight_max": 87.02759552001953,
      "activations/layer4_attention_weight_min": -89.40809631347656,
      "activations/layer5_attention_weight_max": 63.86341857910156,
      "activations/layer5_attention_weight_min": -69.43419647216797,
      "activations/layer6_attention_weight_max": 51.31381607055664,
      "activations/layer6_attention_weight_min": -52.73726272583008,
      "activations/layer7_attention_weight_max": 69.95658874511719,
      "activations/layer7_attention_weight_min": -69.11375427246094,
      "activations/layer8_attention_weight_max": 46.0672492980957,
      "activations/layer8_attention_weight_min": -48.61609649658203,
      "activations/layer9_attention_weight_max": 44.76486587524414,
      "activations/layer9_attention_weight_min": -45.963191986083984,
      "epoch": 22.27,
      "learning_rate": 6.441287878787878e-06,
      "loss": 2.7094,
      "step": 383300
    },
    {
      "activations/layer0_attention_weight_max": 15.815053939819336,
      "activations/layer0_attention_weight_min": -14.081363677978516,
      "activations/layer10_attention_weight_max": 35.283447265625,
      "activations/layer10_attention_weight_min": -35.036376953125,
      "activations/layer11_attention_weight_max": 33.01390075683594,
      "activations/layer11_attention_weight_min": -35.06829833984375,
      "activations/layer12_attention_weight_max": 17.861074447631836,
      "activations/layer12_attention_weight_min": -26.771827697753906,
      "activations/layer13_attention_weight_max": 33.47385787963867,
      "activations/layer13_attention_weight_min": -27.808456420898438,
      "activations/layer14_attention_weight_max": 33.08624267578125,
      "activations/layer14_attention_weight_min": -30.068416595458984,
      "activations/layer15_attention_weight_max": 33.74506378173828,
      "activations/layer15_attention_weight_min": -30.953468322753906,
      "activations/layer16_attention_weight_max": 33.636417388916016,
      "activations/layer16_attention_weight_min": -31.715938568115234,
      "activations/layer17_attention_weight_max": 46.96101379394531,
      "activations/layer17_attention_weight_min": -43.075233459472656,
      "activations/layer18_attention_weight_max": 43.21516036987305,
      "activations/layer18_attention_weight_min": -39.71469497680664,
      "activations/layer19_attention_weight_max": 23.52389907836914,
      "activations/layer19_attention_weight_min": -22.349008560180664,
      "activations/layer1_attention_weight_max": 15.642404556274414,
      "activations/layer1_attention_weight_min": -14.57236099243164,
      "activations/layer20_attention_weight_max": 21.029083251953125,
      "activations/layer20_attention_weight_min": -21.372337341308594,
      "activations/layer21_attention_weight_max": 27.85525894165039,
      "activations/layer21_attention_weight_min": -20.73031997680664,
      "activations/layer22_attention_weight_max": 25.089244842529297,
      "activations/layer22_attention_weight_min": -24.13207244873047,
      "activations/layer23_attention_weight_max": 36.31459045410156,
      "activations/layer23_attention_weight_min": -22.58203887939453,
      "activations/layer2_attention_weight_max": 32.22799301147461,
      "activations/layer2_attention_weight_min": -30.931577682495117,
      "activations/layer3_attention_weight_max": 93.29827117919922,
      "activations/layer3_attention_weight_min": -88.94308471679688,
      "activations/layer4_attention_weight_max": 84.76146697998047,
      "activations/layer4_attention_weight_min": -84.44391632080078,
      "activations/layer5_attention_weight_max": 58.83163070678711,
      "activations/layer5_attention_weight_min": -67.68605041503906,
      "activations/layer6_attention_weight_max": 47.92210388183594,
      "activations/layer6_attention_weight_min": -49.40498352050781,
      "activations/layer7_attention_weight_max": 66.99703216552734,
      "activations/layer7_attention_weight_min": -67.44221496582031,
      "activations/layer8_attention_weight_max": 46.38233184814453,
      "activations/layer8_attention_weight_min": -49.7227897644043,
      "activations/layer9_attention_weight_max": 43.93132019042969,
      "activations/layer9_attention_weight_min": -45.69761276245117,
      "epoch": 22.27,
      "learning_rate": 6.422348484848484e-06,
      "loss": 2.7154,
      "step": 383350
    },
    {
      "activations/layer0_attention_weight_max": 16.256343841552734,
      "activations/layer0_attention_weight_min": -14.313566207885742,
      "activations/layer10_attention_weight_max": 36.84003448486328,
      "activations/layer10_attention_weight_min": -36.10456085205078,
      "activations/layer11_attention_weight_max": 33.16928482055664,
      "activations/layer11_attention_weight_min": -36.748680114746094,
      "activations/layer12_attention_weight_max": 19.54454803466797,
      "activations/layer12_attention_weight_min": -25.997657775878906,
      "activations/layer13_attention_weight_max": 34.65723419189453,
      "activations/layer13_attention_weight_min": -30.230318069458008,
      "activations/layer14_attention_weight_max": 33.458984375,
      "activations/layer14_attention_weight_min": -28.753189086914062,
      "activations/layer15_attention_weight_max": 34.10979080200195,
      "activations/layer15_attention_weight_min": -30.87905502319336,
      "activations/layer16_attention_weight_max": 31.321949005126953,
      "activations/layer16_attention_weight_min": -32.620948791503906,
      "activations/layer17_attention_weight_max": 48.043827056884766,
      "activations/layer17_attention_weight_min": -45.13333511352539,
      "activations/layer18_attention_weight_max": 45.657997131347656,
      "activations/layer18_attention_weight_min": -40.39893341064453,
      "activations/layer19_attention_weight_max": 24.504159927368164,
      "activations/layer19_attention_weight_min": -21.767236709594727,
      "activations/layer1_attention_weight_max": 14.991899490356445,
      "activations/layer1_attention_weight_min": -15.329010009765625,
      "activations/layer20_attention_weight_max": 23.723081588745117,
      "activations/layer20_attention_weight_min": -20.40503692626953,
      "activations/layer21_attention_weight_max": 33.676055908203125,
      "activations/layer21_attention_weight_min": -22.544921875,
      "activations/layer22_attention_weight_max": 28.625566482543945,
      "activations/layer22_attention_weight_min": -24.78693199157715,
      "activations/layer23_attention_weight_max": 39.51885223388672,
      "activations/layer23_attention_weight_min": -22.957721710205078,
      "activations/layer2_attention_weight_max": 31.089038848876953,
      "activations/layer2_attention_weight_min": -30.328697204589844,
      "activations/layer3_attention_weight_max": 85.17890930175781,
      "activations/layer3_attention_weight_min": -87.20954132080078,
      "activations/layer4_attention_weight_max": 82.33586120605469,
      "activations/layer4_attention_weight_min": -84.53404235839844,
      "activations/layer5_attention_weight_max": 58.983192443847656,
      "activations/layer5_attention_weight_min": -74.74168395996094,
      "activations/layer6_attention_weight_max": 47.23764419555664,
      "activations/layer6_attention_weight_min": -52.48435974121094,
      "activations/layer7_attention_weight_max": 66.50675201416016,
      "activations/layer7_attention_weight_min": -67.01301574707031,
      "activations/layer8_attention_weight_max": 47.83946990966797,
      "activations/layer8_attention_weight_min": -52.902767181396484,
      "activations/layer9_attention_weight_max": 44.96297836303711,
      "activations/layer9_attention_weight_min": -46.05189895629883,
      "epoch": 22.28,
      "learning_rate": 6.40340909090909e-06,
      "loss": 2.7182,
      "step": 383400
    },
    {
      "activations/layer0_attention_weight_max": 15.957625389099121,
      "activations/layer0_attention_weight_min": -14.309087753295898,
      "activations/layer10_attention_weight_max": 38.654693603515625,
      "activations/layer10_attention_weight_min": -36.77304458618164,
      "activations/layer11_attention_weight_max": 35.475624084472656,
      "activations/layer11_attention_weight_min": -37.27344512939453,
      "activations/layer12_attention_weight_max": 19.825536727905273,
      "activations/layer12_attention_weight_min": -29.819625854492188,
      "activations/layer13_attention_weight_max": 34.241180419921875,
      "activations/layer13_attention_weight_min": -29.885845184326172,
      "activations/layer14_attention_weight_max": 35.728878021240234,
      "activations/layer14_attention_weight_min": -32.09150695800781,
      "activations/layer15_attention_weight_max": 33.97180938720703,
      "activations/layer15_attention_weight_min": -32.70455551147461,
      "activations/layer16_attention_weight_max": 37.07866287231445,
      "activations/layer16_attention_weight_min": -35.93516159057617,
      "activations/layer17_attention_weight_max": 55.414459228515625,
      "activations/layer17_attention_weight_min": -49.37712478637695,
      "activations/layer18_attention_weight_max": 52.142723083496094,
      "activations/layer18_attention_weight_min": -45.96316146850586,
      "activations/layer19_attention_weight_max": 26.72178077697754,
      "activations/layer19_attention_weight_min": -23.470958709716797,
      "activations/layer1_attention_weight_max": 15.164392471313477,
      "activations/layer1_attention_weight_min": -13.905068397521973,
      "activations/layer20_attention_weight_max": 23.81090545654297,
      "activations/layer20_attention_weight_min": -22.718786239624023,
      "activations/layer21_attention_weight_max": 37.91841125488281,
      "activations/layer21_attention_weight_min": -23.7086181640625,
      "activations/layer22_attention_weight_max": 30.959104537963867,
      "activations/layer22_attention_weight_min": -25.182872772216797,
      "activations/layer23_attention_weight_max": 36.05736541748047,
      "activations/layer23_attention_weight_min": -22.922714233398438,
      "activations/layer2_attention_weight_max": 33.19871520996094,
      "activations/layer2_attention_weight_min": -30.86817169189453,
      "activations/layer3_attention_weight_max": 90.88018798828125,
      "activations/layer3_attention_weight_min": -89.81480407714844,
      "activations/layer4_attention_weight_max": 87.54344177246094,
      "activations/layer4_attention_weight_min": -86.278076171875,
      "activations/layer5_attention_weight_max": 61.96413803100586,
      "activations/layer5_attention_weight_min": -74.50190734863281,
      "activations/layer6_attention_weight_max": 51.48381042480469,
      "activations/layer6_attention_weight_min": -53.15372848510742,
      "activations/layer7_attention_weight_max": 71.36637115478516,
      "activations/layer7_attention_weight_min": -68.70667266845703,
      "activations/layer8_attention_weight_max": 50.026039123535156,
      "activations/layer8_attention_weight_min": -50.11212158203125,
      "activations/layer9_attention_weight_max": 46.64204025268555,
      "activations/layer9_attention_weight_min": -47.9626579284668,
      "epoch": 22.28,
      "learning_rate": 6.384469696969697e-06,
      "loss": 2.7016,
      "step": 383450
    },
    {
      "activations/layer0_attention_weight_max": 15.54842472076416,
      "activations/layer0_attention_weight_min": -14.20053482055664,
      "activations/layer10_attention_weight_max": 35.15184020996094,
      "activations/layer10_attention_weight_min": -34.31664276123047,
      "activations/layer11_attention_weight_max": 35.14722442626953,
      "activations/layer11_attention_weight_min": -33.45828628540039,
      "activations/layer12_attention_weight_max": 18.486942291259766,
      "activations/layer12_attention_weight_min": -24.45016098022461,
      "activations/layer13_attention_weight_max": 38.6770133972168,
      "activations/layer13_attention_weight_min": -27.32466697692871,
      "activations/layer14_attention_weight_max": 36.38119125366211,
      "activations/layer14_attention_weight_min": -29.153533935546875,
      "activations/layer15_attention_weight_max": 41.40184783935547,
      "activations/layer15_attention_weight_min": -29.060791015625,
      "activations/layer16_attention_weight_max": 31.253816604614258,
      "activations/layer16_attention_weight_min": -30.129268646240234,
      "activations/layer17_attention_weight_max": 47.88105010986328,
      "activations/layer17_attention_weight_min": -43.90872573852539,
      "activations/layer18_attention_weight_max": 43.63166046142578,
      "activations/layer18_attention_weight_min": -38.864444732666016,
      "activations/layer19_attention_weight_max": 24.183332443237305,
      "activations/layer19_attention_weight_min": -24.616931915283203,
      "activations/layer1_attention_weight_max": 15.362542152404785,
      "activations/layer1_attention_weight_min": -14.519451141357422,
      "activations/layer20_attention_weight_max": 21.265247344970703,
      "activations/layer20_attention_weight_min": -21.883697509765625,
      "activations/layer21_attention_weight_max": 32.09961700439453,
      "activations/layer21_attention_weight_min": -24.944406509399414,
      "activations/layer22_attention_weight_max": 29.542049407958984,
      "activations/layer22_attention_weight_min": -23.82652473449707,
      "activations/layer23_attention_weight_max": 34.72597885131836,
      "activations/layer23_attention_weight_min": -22.55672836303711,
      "activations/layer2_attention_weight_max": 32.4331169128418,
      "activations/layer2_attention_weight_min": -31.426908493041992,
      "activations/layer3_attention_weight_max": 90.5648422241211,
      "activations/layer3_attention_weight_min": -89.73426818847656,
      "activations/layer4_attention_weight_max": 86.38877868652344,
      "activations/layer4_attention_weight_min": -87.14778900146484,
      "activations/layer5_attention_weight_max": 61.09817123413086,
      "activations/layer5_attention_weight_min": -73.53466796875,
      "activations/layer6_attention_weight_max": 48.31040954589844,
      "activations/layer6_attention_weight_min": -51.18647384643555,
      "activations/layer7_attention_weight_max": 68.90694427490234,
      "activations/layer7_attention_weight_min": -67.55715942382812,
      "activations/layer8_attention_weight_max": 45.14236831665039,
      "activations/layer8_attention_weight_min": -49.44364547729492,
      "activations/layer9_attention_weight_max": 42.42646408081055,
      "activations/layer9_attention_weight_min": -44.55921936035156,
      "epoch": 22.28,
      "learning_rate": 6.365530303030303e-06,
      "loss": 2.707,
      "step": 383500
    },
    {
      "activations/layer0_attention_weight_max": 15.440807342529297,
      "activations/layer0_attention_weight_min": -14.1656494140625,
      "activations/layer10_attention_weight_max": 41.46812057495117,
      "activations/layer10_attention_weight_min": -39.001319885253906,
      "activations/layer11_attention_weight_max": 37.840232849121094,
      "activations/layer11_attention_weight_min": -36.79956817626953,
      "activations/layer12_attention_weight_max": 20.36387062072754,
      "activations/layer12_attention_weight_min": -25.229206085205078,
      "activations/layer13_attention_weight_max": 34.9105224609375,
      "activations/layer13_attention_weight_min": -29.198894500732422,
      "activations/layer14_attention_weight_max": 35.171173095703125,
      "activations/layer14_attention_weight_min": -30.14011573791504,
      "activations/layer15_attention_weight_max": 33.61549377441406,
      "activations/layer15_attention_weight_min": -30.16897201538086,
      "activations/layer16_attention_weight_max": 38.5307502746582,
      "activations/layer16_attention_weight_min": -36.31266403198242,
      "activations/layer17_attention_weight_max": 52.27122497558594,
      "activations/layer17_attention_weight_min": -45.609928131103516,
      "activations/layer18_attention_weight_max": 46.71054458618164,
      "activations/layer18_attention_weight_min": -41.4460334777832,
      "activations/layer19_attention_weight_max": 24.41141700744629,
      "activations/layer19_attention_weight_min": -22.312658309936523,
      "activations/layer1_attention_weight_max": 15.481502532958984,
      "activations/layer1_attention_weight_min": -14.895344734191895,
      "activations/layer20_attention_weight_max": 23.283084869384766,
      "activations/layer20_attention_weight_min": -21.74644660949707,
      "activations/layer21_attention_weight_max": 39.290916442871094,
      "activations/layer21_attention_weight_min": -23.866289138793945,
      "activations/layer22_attention_weight_max": 27.887998580932617,
      "activations/layer22_attention_weight_min": -26.8018741607666,
      "activations/layer23_attention_weight_max": 36.4697265625,
      "activations/layer23_attention_weight_min": -25.982566833496094,
      "activations/layer2_attention_weight_max": 32.9957275390625,
      "activations/layer2_attention_weight_min": -31.42382049560547,
      "activations/layer3_attention_weight_max": 94.68710327148438,
      "activations/layer3_attention_weight_min": -94.4778823852539,
      "activations/layer4_attention_weight_max": 86.82262420654297,
      "activations/layer4_attention_weight_min": -88.27547454833984,
      "activations/layer5_attention_weight_max": 62.46906661987305,
      "activations/layer5_attention_weight_min": -71.84196472167969,
      "activations/layer6_attention_weight_max": 49.79011917114258,
      "activations/layer6_attention_weight_min": -50.62444305419922,
      "activations/layer7_attention_weight_max": 74.58759307861328,
      "activations/layer7_attention_weight_min": -73.58270263671875,
      "activations/layer8_attention_weight_max": 55.82396697998047,
      "activations/layer8_attention_weight_min": -54.10042190551758,
      "activations/layer9_attention_weight_max": 54.814208984375,
      "activations/layer9_attention_weight_min": -51.12070846557617,
      "epoch": 22.29,
      "learning_rate": 6.346969696969697e-06,
      "loss": 2.7046,
      "step": 383550
    },
    {
      "activations/layer0_attention_weight_max": 15.876996994018555,
      "activations/layer0_attention_weight_min": -14.228862762451172,
      "activations/layer10_attention_weight_max": 37.54289627075195,
      "activations/layer10_attention_weight_min": -36.68975830078125,
      "activations/layer11_attention_weight_max": 34.66557312011719,
      "activations/layer11_attention_weight_min": -35.46539306640625,
      "activations/layer12_attention_weight_max": 19.398523330688477,
      "activations/layer12_attention_weight_min": -26.00800895690918,
      "activations/layer13_attention_weight_max": 38.1043586730957,
      "activations/layer13_attention_weight_min": -28.351226806640625,
      "activations/layer14_attention_weight_max": 36.965423583984375,
      "activations/layer14_attention_weight_min": -30.200172424316406,
      "activations/layer15_attention_weight_max": 37.47940444946289,
      "activations/layer15_attention_weight_min": -29.095428466796875,
      "activations/layer16_attention_weight_max": 36.51005935668945,
      "activations/layer16_attention_weight_min": -32.565643310546875,
      "activations/layer17_attention_weight_max": 55.12812805175781,
      "activations/layer17_attention_weight_min": -46.217952728271484,
      "activations/layer18_attention_weight_max": 54.26891326904297,
      "activations/layer18_attention_weight_min": -40.61351013183594,
      "activations/layer19_attention_weight_max": 28.3813533782959,
      "activations/layer19_attention_weight_min": -24.822772979736328,
      "activations/layer1_attention_weight_max": 14.808673858642578,
      "activations/layer1_attention_weight_min": -14.466456413269043,
      "activations/layer20_attention_weight_max": 28.78807830810547,
      "activations/layer20_attention_weight_min": -20.859922409057617,
      "activations/layer21_attention_weight_max": 52.36876678466797,
      "activations/layer21_attention_weight_min": -22.66744041442871,
      "activations/layer22_attention_weight_max": 32.8050651550293,
      "activations/layer22_attention_weight_min": -23.75949478149414,
      "activations/layer23_attention_weight_max": 38.504642486572266,
      "activations/layer23_attention_weight_min": -23.215070724487305,
      "activations/layer2_attention_weight_max": 32.86419677734375,
      "activations/layer2_attention_weight_min": -30.353343963623047,
      "activations/layer3_attention_weight_max": 92.45181274414062,
      "activations/layer3_attention_weight_min": -89.1756820678711,
      "activations/layer4_attention_weight_max": 87.16372680664062,
      "activations/layer4_attention_weight_min": -84.19979858398438,
      "activations/layer5_attention_weight_max": 63.64704895019531,
      "activations/layer5_attention_weight_min": -70.79299926757812,
      "activations/layer6_attention_weight_max": 51.97621536254883,
      "activations/layer6_attention_weight_min": -52.2062873840332,
      "activations/layer7_attention_weight_max": 68.39640808105469,
      "activations/layer7_attention_weight_min": -67.42626190185547,
      "activations/layer8_attention_weight_max": 50.22818374633789,
      "activations/layer8_attention_weight_min": -53.879032135009766,
      "activations/layer9_attention_weight_max": 46.06515884399414,
      "activations/layer9_attention_weight_min": -45.49445724487305,
      "epoch": 22.29,
      "learning_rate": 6.328030303030303e-06,
      "loss": 2.7001,
      "step": 383600
    },
    {
      "activations/layer0_attention_weight_max": 16.251117706298828,
      "activations/layer0_attention_weight_min": -14.071417808532715,
      "activations/layer10_attention_weight_max": 35.520172119140625,
      "activations/layer10_attention_weight_min": -35.101959228515625,
      "activations/layer11_attention_weight_max": 32.80121612548828,
      "activations/layer11_attention_weight_min": -34.14112854003906,
      "activations/layer12_attention_weight_max": 18.794652938842773,
      "activations/layer12_attention_weight_min": -25.671485900878906,
      "activations/layer13_attention_weight_max": 39.82819366455078,
      "activations/layer13_attention_weight_min": -31.429012298583984,
      "activations/layer14_attention_weight_max": 43.806358337402344,
      "activations/layer14_attention_weight_min": -30.461442947387695,
      "activations/layer15_attention_weight_max": 40.38661193847656,
      "activations/layer15_attention_weight_min": -30.050214767456055,
      "activations/layer16_attention_weight_max": 32.15204620361328,
      "activations/layer16_attention_weight_min": -31.100812911987305,
      "activations/layer17_attention_weight_max": 52.22648239135742,
      "activations/layer17_attention_weight_min": -49.46726989746094,
      "activations/layer18_attention_weight_max": 42.362918853759766,
      "activations/layer18_attention_weight_min": -42.505470275878906,
      "activations/layer19_attention_weight_max": 21.943023681640625,
      "activations/layer19_attention_weight_min": -23.654521942138672,
      "activations/layer1_attention_weight_max": 14.673054695129395,
      "activations/layer1_attention_weight_min": -14.68618106842041,
      "activations/layer20_attention_weight_max": 20.06168556213379,
      "activations/layer20_attention_weight_min": -21.274808883666992,
      "activations/layer21_attention_weight_max": 31.62868881225586,
      "activations/layer21_attention_weight_min": -24.741928100585938,
      "activations/layer22_attention_weight_max": 28.515954971313477,
      "activations/layer22_attention_weight_min": -24.902204513549805,
      "activations/layer23_attention_weight_max": 35.72025680541992,
      "activations/layer23_attention_weight_min": -23.83348846435547,
      "activations/layer2_attention_weight_max": 34.388267517089844,
      "activations/layer2_attention_weight_min": -31.442951202392578,
      "activations/layer3_attention_weight_max": 92.16519927978516,
      "activations/layer3_attention_weight_min": -90.42001342773438,
      "activations/layer4_attention_weight_max": 86.63594055175781,
      "activations/layer4_attention_weight_min": -87.49645233154297,
      "activations/layer5_attention_weight_max": 61.975563049316406,
      "activations/layer5_attention_weight_min": -71.08726501464844,
      "activations/layer6_attention_weight_max": 50.08707809448242,
      "activations/layer6_attention_weight_min": -50.66169357299805,
      "activations/layer7_attention_weight_max": 71.27304077148438,
      "activations/layer7_attention_weight_min": -69.31788635253906,
      "activations/layer8_attention_weight_max": 48.19059753417969,
      "activations/layer8_attention_weight_min": -50.603057861328125,
      "activations/layer9_attention_weight_max": 46.787933349609375,
      "activations/layer9_attention_weight_min": -45.19218826293945,
      "epoch": 22.29,
      "learning_rate": 6.309090909090909e-06,
      "loss": 2.7209,
      "step": 383650
    },
    {
      "activations/layer0_attention_weight_max": 15.95055866241455,
      "activations/layer0_attention_weight_min": -13.874015808105469,
      "activations/layer10_attention_weight_max": 39.63007354736328,
      "activations/layer10_attention_weight_min": -42.43280029296875,
      "activations/layer11_attention_weight_max": 35.355125427246094,
      "activations/layer11_attention_weight_min": -39.69889831542969,
      "activations/layer12_attention_weight_max": 18.706832885742188,
      "activations/layer12_attention_weight_min": -25.71976089477539,
      "activations/layer13_attention_weight_max": 34.248165130615234,
      "activations/layer13_attention_weight_min": -29.900108337402344,
      "activations/layer14_attention_weight_max": 35.16997146606445,
      "activations/layer14_attention_weight_min": -31.707721710205078,
      "activations/layer15_attention_weight_max": 34.75645446777344,
      "activations/layer15_attention_weight_min": -32.08881378173828,
      "activations/layer16_attention_weight_max": 33.76698684692383,
      "activations/layer16_attention_weight_min": -32.88762283325195,
      "activations/layer17_attention_weight_max": 56.20065689086914,
      "activations/layer17_attention_weight_min": -47.51809310913086,
      "activations/layer18_attention_weight_max": 48.21567153930664,
      "activations/layer18_attention_weight_min": -42.99332046508789,
      "activations/layer19_attention_weight_max": 24.293777465820312,
      "activations/layer19_attention_weight_min": -21.4233455657959,
      "activations/layer1_attention_weight_max": 15.088065147399902,
      "activations/layer1_attention_weight_min": -13.486650466918945,
      "activations/layer20_attention_weight_max": 23.47988510131836,
      "activations/layer20_attention_weight_min": -23.31726837158203,
      "activations/layer21_attention_weight_max": 33.14376449584961,
      "activations/layer21_attention_weight_min": -20.607961654663086,
      "activations/layer22_attention_weight_max": 28.663349151611328,
      "activations/layer22_attention_weight_min": -25.136343002319336,
      "activations/layer23_attention_weight_max": 36.43196105957031,
      "activations/layer23_attention_weight_min": -23.530094146728516,
      "activations/layer2_attention_weight_max": 32.88723373413086,
      "activations/layer2_attention_weight_min": -29.60761833190918,
      "activations/layer3_attention_weight_max": 86.75798797607422,
      "activations/layer3_attention_weight_min": -87.673095703125,
      "activations/layer4_attention_weight_max": 83.20242309570312,
      "activations/layer4_attention_weight_min": -87.44315338134766,
      "activations/layer5_attention_weight_max": 63.90449142456055,
      "activations/layer5_attention_weight_min": -72.38663482666016,
      "activations/layer6_attention_weight_max": 48.29874801635742,
      "activations/layer6_attention_weight_min": -50.44914627075195,
      "activations/layer7_attention_weight_max": 70.0919189453125,
      "activations/layer7_attention_weight_min": -68.28041076660156,
      "activations/layer8_attention_weight_max": 53.363651275634766,
      "activations/layer8_attention_weight_min": -51.0541877746582,
      "activations/layer9_attention_weight_max": 47.64571762084961,
      "activations/layer9_attention_weight_min": -50.33620834350586,
      "epoch": 22.3,
      "learning_rate": 6.290151515151515e-06,
      "loss": 2.7043,
      "step": 383700
    },
    {
      "activations/layer0_attention_weight_max": 16.35809326171875,
      "activations/layer0_attention_weight_min": -14.195582389831543,
      "activations/layer10_attention_weight_max": 42.01242446899414,
      "activations/layer10_attention_weight_min": -39.5758056640625,
      "activations/layer11_attention_weight_max": 41.497772216796875,
      "activations/layer11_attention_weight_min": -37.77300262451172,
      "activations/layer12_attention_weight_max": 19.80666160583496,
      "activations/layer12_attention_weight_min": -24.154123306274414,
      "activations/layer13_attention_weight_max": 39.17021560668945,
      "activations/layer13_attention_weight_min": -30.022293090820312,
      "activations/layer14_attention_weight_max": 37.84025955200195,
      "activations/layer14_attention_weight_min": -31.47381019592285,
      "activations/layer15_attention_weight_max": 32.95607376098633,
      "activations/layer15_attention_weight_min": -29.466049194335938,
      "activations/layer16_attention_weight_max": 34.87918472290039,
      "activations/layer16_attention_weight_min": -32.80012130737305,
      "activations/layer17_attention_weight_max": 51.37147903442383,
      "activations/layer17_attention_weight_min": -44.29161834716797,
      "activations/layer18_attention_weight_max": 45.974124908447266,
      "activations/layer18_attention_weight_min": -41.25068283081055,
      "activations/layer19_attention_weight_max": 28.491994857788086,
      "activations/layer19_attention_weight_min": -23.169227600097656,
      "activations/layer1_attention_weight_max": 15.11937141418457,
      "activations/layer1_attention_weight_min": -13.706686019897461,
      "activations/layer20_attention_weight_max": 26.77922821044922,
      "activations/layer20_attention_weight_min": -20.468990325927734,
      "activations/layer21_attention_weight_max": 39.49972152709961,
      "activations/layer21_attention_weight_min": -21.92010498046875,
      "activations/layer22_attention_weight_max": 30.88324737548828,
      "activations/layer22_attention_weight_min": -25.090736389160156,
      "activations/layer23_attention_weight_max": 37.352813720703125,
      "activations/layer23_attention_weight_min": -22.91335678100586,
      "activations/layer2_attention_weight_max": 32.70466995239258,
      "activations/layer2_attention_weight_min": -30.935728073120117,
      "activations/layer3_attention_weight_max": 92.44208526611328,
      "activations/layer3_attention_weight_min": -94.83855438232422,
      "activations/layer4_attention_weight_max": 85.896728515625,
      "activations/layer4_attention_weight_min": -93.63146209716797,
      "activations/layer5_attention_weight_max": 65.22360229492188,
      "activations/layer5_attention_weight_min": -72.30716705322266,
      "activations/layer6_attention_weight_max": 54.1728401184082,
      "activations/layer6_attention_weight_min": -53.0203742980957,
      "activations/layer7_attention_weight_max": 77.06720733642578,
      "activations/layer7_attention_weight_min": -70.24479675292969,
      "activations/layer8_attention_weight_max": 54.29753875732422,
      "activations/layer8_attention_weight_min": -54.50584030151367,
      "activations/layer9_attention_weight_max": 51.210693359375,
      "activations/layer9_attention_weight_min": -52.96787643432617,
      "epoch": 22.3,
      "learning_rate": 6.27121212121212e-06,
      "loss": 2.7084,
      "step": 383750
    },
    {
      "activations/layer0_attention_weight_max": 15.670297622680664,
      "activations/layer0_attention_weight_min": -14.225114822387695,
      "activations/layer10_attention_weight_max": 36.043067932128906,
      "activations/layer10_attention_weight_min": -35.892051696777344,
      "activations/layer11_attention_weight_max": 32.378135681152344,
      "activations/layer11_attention_weight_min": -35.399810791015625,
      "activations/layer12_attention_weight_max": 20.524808883666992,
      "activations/layer12_attention_weight_min": -25.678104400634766,
      "activations/layer13_attention_weight_max": 44.48830795288086,
      "activations/layer13_attention_weight_min": -27.905765533447266,
      "activations/layer14_attention_weight_max": 35.10188293457031,
      "activations/layer14_attention_weight_min": -28.707252502441406,
      "activations/layer15_attention_weight_max": 34.28712463378906,
      "activations/layer15_attention_weight_min": -29.503353118896484,
      "activations/layer16_attention_weight_max": 35.85064697265625,
      "activations/layer16_attention_weight_min": -32.367427825927734,
      "activations/layer17_attention_weight_max": 51.361331939697266,
      "activations/layer17_attention_weight_min": -41.559818267822266,
      "activations/layer18_attention_weight_max": 46.9613151550293,
      "activations/layer18_attention_weight_min": -38.3145866394043,
      "activations/layer19_attention_weight_max": 26.624298095703125,
      "activations/layer19_attention_weight_min": -20.95418357849121,
      "activations/layer1_attention_weight_max": 14.766644477844238,
      "activations/layer1_attention_weight_min": -13.337828636169434,
      "activations/layer20_attention_weight_max": 23.660226821899414,
      "activations/layer20_attention_weight_min": -20.841951370239258,
      "activations/layer21_attention_weight_max": 39.01188659667969,
      "activations/layer21_attention_weight_min": -23.790170669555664,
      "activations/layer22_attention_weight_max": 31.205121994018555,
      "activations/layer22_attention_weight_min": -22.117650985717773,
      "activations/layer23_attention_weight_max": 38.460147857666016,
      "activations/layer23_attention_weight_min": -23.179866790771484,
      "activations/layer2_attention_weight_max": 32.684383392333984,
      "activations/layer2_attention_weight_min": -30.70931053161621,
      "activations/layer3_attention_weight_max": 93.23066711425781,
      "activations/layer3_attention_weight_min": -91.57787322998047,
      "activations/layer4_attention_weight_max": 86.1794204711914,
      "activations/layer4_attention_weight_min": -84.73340606689453,
      "activations/layer5_attention_weight_max": 67.34310913085938,
      "activations/layer5_attention_weight_min": -74.96952056884766,
      "activations/layer6_attention_weight_max": 53.25412368774414,
      "activations/layer6_attention_weight_min": -54.81757736206055,
      "activations/layer7_attention_weight_max": 74.33919525146484,
      "activations/layer7_attention_weight_min": -68.39000701904297,
      "activations/layer8_attention_weight_max": 48.17219161987305,
      "activations/layer8_attention_weight_min": -50.87960433959961,
      "activations/layer9_attention_weight_max": 47.28733444213867,
      "activations/layer9_attention_weight_min": -47.98810577392578,
      "epoch": 22.3,
      "learning_rate": 6.252272727272727e-06,
      "loss": 2.6959,
      "step": 383800
    },
    {
      "activations/layer0_attention_weight_max": 15.716300010681152,
      "activations/layer0_attention_weight_min": -14.26512336730957,
      "activations/layer10_attention_weight_max": 36.011043548583984,
      "activations/layer10_attention_weight_min": -35.610755920410156,
      "activations/layer11_attention_weight_max": 33.27056884765625,
      "activations/layer11_attention_weight_min": -35.0906982421875,
      "activations/layer12_attention_weight_max": 19.768136978149414,
      "activations/layer12_attention_weight_min": -26.71113395690918,
      "activations/layer13_attention_weight_max": 42.14809036254883,
      "activations/layer13_attention_weight_min": -28.067569732666016,
      "activations/layer14_attention_weight_max": 33.64750671386719,
      "activations/layer14_attention_weight_min": -29.40182876586914,
      "activations/layer15_attention_weight_max": 30.859619140625,
      "activations/layer15_attention_weight_min": -30.955657958984375,
      "activations/layer16_attention_weight_max": 32.5943489074707,
      "activations/layer16_attention_weight_min": -31.930679321289062,
      "activations/layer17_attention_weight_max": 48.93632888793945,
      "activations/layer17_attention_weight_min": -47.36647415161133,
      "activations/layer18_attention_weight_max": 47.34184265136719,
      "activations/layer18_attention_weight_min": -42.55573654174805,
      "activations/layer19_attention_weight_max": 23.95517349243164,
      "activations/layer19_attention_weight_min": -24.93442726135254,
      "activations/layer1_attention_weight_max": 16.00922966003418,
      "activations/layer1_attention_weight_min": -13.965123176574707,
      "activations/layer20_attention_weight_max": 22.853857040405273,
      "activations/layer20_attention_weight_min": -22.312299728393555,
      "activations/layer21_attention_weight_max": 36.11362075805664,
      "activations/layer21_attention_weight_min": -23.329198837280273,
      "activations/layer22_attention_weight_max": 27.70448112487793,
      "activations/layer22_attention_weight_min": -25.998254776000977,
      "activations/layer23_attention_weight_max": 34.769195556640625,
      "activations/layer23_attention_weight_min": -25.70848846435547,
      "activations/layer2_attention_weight_max": 31.400510787963867,
      "activations/layer2_attention_weight_min": -30.48866081237793,
      "activations/layer3_attention_weight_max": 88.19502258300781,
      "activations/layer3_attention_weight_min": -87.15852355957031,
      "activations/layer4_attention_weight_max": 87.60320281982422,
      "activations/layer4_attention_weight_min": -83.40186309814453,
      "activations/layer5_attention_weight_max": 63.565731048583984,
      "activations/layer5_attention_weight_min": -76.13849639892578,
      "activations/layer6_attention_weight_max": 48.361698150634766,
      "activations/layer6_attention_weight_min": -51.369441986083984,
      "activations/layer7_attention_weight_max": 69.70976257324219,
      "activations/layer7_attention_weight_min": -67.44156646728516,
      "activations/layer8_attention_weight_max": 47.1375846862793,
      "activations/layer8_attention_weight_min": -50.922142028808594,
      "activations/layer9_attention_weight_max": 44.038475036621094,
      "activations/layer9_attention_weight_min": -48.74894714355469,
      "epoch": 22.3,
      "learning_rate": 6.233333333333333e-06,
      "loss": 2.7136,
      "step": 383850
    },
    {
      "activations/layer0_attention_weight_max": 15.151052474975586,
      "activations/layer0_attention_weight_min": -14.376445770263672,
      "activations/layer10_attention_weight_max": 37.089839935302734,
      "activations/layer10_attention_weight_min": -36.282745361328125,
      "activations/layer11_attention_weight_max": 35.25152587890625,
      "activations/layer11_attention_weight_min": -33.248741149902344,
      "activations/layer12_attention_weight_max": 19.93277931213379,
      "activations/layer12_attention_weight_min": -29.23505401611328,
      "activations/layer13_attention_weight_max": 39.0585823059082,
      "activations/layer13_attention_weight_min": -27.264009475708008,
      "activations/layer14_attention_weight_max": 35.69073486328125,
      "activations/layer14_attention_weight_min": -29.043859481811523,
      "activations/layer15_attention_weight_max": 33.97578811645508,
      "activations/layer15_attention_weight_min": -31.33152198791504,
      "activations/layer16_attention_weight_max": 35.05447769165039,
      "activations/layer16_attention_weight_min": -33.3336067199707,
      "activations/layer17_attention_weight_max": 50.61503982543945,
      "activations/layer17_attention_weight_min": -43.28672409057617,
      "activations/layer18_attention_weight_max": 48.519771575927734,
      "activations/layer18_attention_weight_min": -38.076473236083984,
      "activations/layer19_attention_weight_max": 26.884414672851562,
      "activations/layer19_attention_weight_min": -22.77760124206543,
      "activations/layer1_attention_weight_max": 14.790528297424316,
      "activations/layer1_attention_weight_min": -13.466021537780762,
      "activations/layer20_attention_weight_max": 25.754117965698242,
      "activations/layer20_attention_weight_min": -21.267215728759766,
      "activations/layer21_attention_weight_max": 35.86568832397461,
      "activations/layer21_attention_weight_min": -21.56328582763672,
      "activations/layer22_attention_weight_max": 31.721454620361328,
      "activations/layer22_attention_weight_min": -24.598352432250977,
      "activations/layer23_attention_weight_max": 33.21385955810547,
      "activations/layer23_attention_weight_min": -22.354032516479492,
      "activations/layer2_attention_weight_max": 32.30998992919922,
      "activations/layer2_attention_weight_min": -32.30213928222656,
      "activations/layer3_attention_weight_max": 90.59017944335938,
      "activations/layer3_attention_weight_min": -93.56867218017578,
      "activations/layer4_attention_weight_max": 87.02717590332031,
      "activations/layer4_attention_weight_min": -91.18721771240234,
      "activations/layer5_attention_weight_max": 61.964019775390625,
      "activations/layer5_attention_weight_min": -74.12389373779297,
      "activations/layer6_attention_weight_max": 50.652992248535156,
      "activations/layer6_attention_weight_min": -51.349063873291016,
      "activations/layer7_attention_weight_max": 70.82852935791016,
      "activations/layer7_attention_weight_min": -71.50690460205078,
      "activations/layer8_attention_weight_max": 47.65156936645508,
      "activations/layer8_attention_weight_min": -51.05282974243164,
      "activations/layer9_attention_weight_max": 46.97858810424805,
      "activations/layer9_attention_weight_min": -47.50763702392578,
      "epoch": 22.31,
      "learning_rate": 6.214393939393938e-06,
      "loss": 2.7169,
      "step": 383900
    },
    {
      "activations/layer0_attention_weight_max": 16.05561637878418,
      "activations/layer0_attention_weight_min": -14.215011596679688,
      "activations/layer10_attention_weight_max": 38.62202835083008,
      "activations/layer10_attention_weight_min": -38.0283317565918,
      "activations/layer11_attention_weight_max": 35.654823303222656,
      "activations/layer11_attention_weight_min": -36.77684783935547,
      "activations/layer12_attention_weight_max": 19.765071868896484,
      "activations/layer12_attention_weight_min": -28.128055572509766,
      "activations/layer13_attention_weight_max": 33.11174774169922,
      "activations/layer13_attention_weight_min": -25.898778915405273,
      "activations/layer14_attention_weight_max": 37.66752624511719,
      "activations/layer14_attention_weight_min": -30.395069122314453,
      "activations/layer15_attention_weight_max": 35.70184326171875,
      "activations/layer15_attention_weight_min": -31.79981231689453,
      "activations/layer16_attention_weight_max": 38.328041076660156,
      "activations/layer16_attention_weight_min": -34.83295440673828,
      "activations/layer17_attention_weight_max": 55.01222610473633,
      "activations/layer17_attention_weight_min": -47.2924690246582,
      "activations/layer18_attention_weight_max": 49.092655181884766,
      "activations/layer18_attention_weight_min": -41.462093353271484,
      "activations/layer19_attention_weight_max": 25.992965698242188,
      "activations/layer19_attention_weight_min": -22.757169723510742,
      "activations/layer1_attention_weight_max": 16.654611587524414,
      "activations/layer1_attention_weight_min": -14.587067604064941,
      "activations/layer20_attention_weight_max": 23.332698822021484,
      "activations/layer20_attention_weight_min": -22.464025497436523,
      "activations/layer21_attention_weight_max": 39.126861572265625,
      "activations/layer21_attention_weight_min": -20.803295135498047,
      "activations/layer22_attention_weight_max": 30.471210479736328,
      "activations/layer22_attention_weight_min": -26.57659912109375,
      "activations/layer23_attention_weight_max": 35.10659408569336,
      "activations/layer23_attention_weight_min": -24.354293823242188,
      "activations/layer2_attention_weight_max": 32.623924255371094,
      "activations/layer2_attention_weight_min": -31.839645385742188,
      "activations/layer3_attention_weight_max": 89.69912719726562,
      "activations/layer3_attention_weight_min": -90.78417205810547,
      "activations/layer4_attention_weight_max": 84.85049438476562,
      "activations/layer4_attention_weight_min": -86.12306213378906,
      "activations/layer5_attention_weight_max": 64.26447296142578,
      "activations/layer5_attention_weight_min": -71.9141845703125,
      "activations/layer6_attention_weight_max": 49.46278381347656,
      "activations/layer6_attention_weight_min": -52.475460052490234,
      "activations/layer7_attention_weight_max": 70.33787536621094,
      "activations/layer7_attention_weight_min": -74.62323760986328,
      "activations/layer8_attention_weight_max": 48.1552734375,
      "activations/layer8_attention_weight_min": -54.357521057128906,
      "activations/layer9_attention_weight_max": 45.516414642333984,
      "activations/layer9_attention_weight_min": -46.85110092163086,
      "epoch": 22.31,
      "learning_rate": 6.195454545454544e-06,
      "loss": 2.7001,
      "step": 383950
    },
    {
      "activations/layer0_attention_weight_max": 15.94787311553955,
      "activations/layer0_attention_weight_min": -14.293204307556152,
      "activations/layer10_attention_weight_max": 34.56328201293945,
      "activations/layer10_attention_weight_min": -34.6076774597168,
      "activations/layer11_attention_weight_max": 33.88356018066406,
      "activations/layer11_attention_weight_min": -33.9544563293457,
      "activations/layer12_attention_weight_max": 19.00241470336914,
      "activations/layer12_attention_weight_min": -23.318519592285156,
      "activations/layer13_attention_weight_max": 32.02876281738281,
      "activations/layer13_attention_weight_min": -27.77191162109375,
      "activations/layer14_attention_weight_max": 33.45923614501953,
      "activations/layer14_attention_weight_min": -28.819652557373047,
      "activations/layer15_attention_weight_max": 30.870189666748047,
      "activations/layer15_attention_weight_min": -29.502275466918945,
      "activations/layer16_attention_weight_max": 31.44159698486328,
      "activations/layer16_attention_weight_min": -30.16356658935547,
      "activations/layer17_attention_weight_max": 52.24502944946289,
      "activations/layer17_attention_weight_min": -43.60394287109375,
      "activations/layer18_attention_weight_max": 47.302093505859375,
      "activations/layer18_attention_weight_min": -37.572723388671875,
      "activations/layer19_attention_weight_max": 22.57023811340332,
      "activations/layer19_attention_weight_min": -22.416240692138672,
      "activations/layer1_attention_weight_max": 15.073749542236328,
      "activations/layer1_attention_weight_min": -13.905011177062988,
      "activations/layer20_attention_weight_max": 23.31403160095215,
      "activations/layer20_attention_weight_min": -21.881933212280273,
      "activations/layer21_attention_weight_max": 37.19725036621094,
      "activations/layer21_attention_weight_min": -22.180273056030273,
      "activations/layer22_attention_weight_max": 25.545522689819336,
      "activations/layer22_attention_weight_min": -22.885271072387695,
      "activations/layer23_attention_weight_max": 33.23642349243164,
      "activations/layer23_attention_weight_min": -23.954486846923828,
      "activations/layer2_attention_weight_max": 31.71816062927246,
      "activations/layer2_attention_weight_min": -30.030757904052734,
      "activations/layer3_attention_weight_max": 87.47264099121094,
      "activations/layer3_attention_weight_min": -90.42333221435547,
      "activations/layer4_attention_weight_max": 83.29838562011719,
      "activations/layer4_attention_weight_min": -86.6445541381836,
      "activations/layer5_attention_weight_max": 60.0570182800293,
      "activations/layer5_attention_weight_min": -68.7994384765625,
      "activations/layer6_attention_weight_max": 51.39061737060547,
      "activations/layer6_attention_weight_min": -49.83014678955078,
      "activations/layer7_attention_weight_max": 69.88665771484375,
      "activations/layer7_attention_weight_min": -66.73316192626953,
      "activations/layer8_attention_weight_max": 45.08564376831055,
      "activations/layer8_attention_weight_min": -50.02834701538086,
      "activations/layer9_attention_weight_max": 41.48283386230469,
      "activations/layer9_attention_weight_min": -45.49561309814453,
      "epoch": 22.31,
      "learning_rate": 6.1765151515151515e-06,
      "loss": 2.6969,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4767,
      "eval_samples_per_second": 506.567,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4767,
      "eval_openwebtext_samples_per_second": 506.567,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9757,
      "eval_wikitext_samples_per_second": 230.803,
      "step": 384000
    },
    {
      "epoch": 22.31,
      "eval_lambada_loss": 2.4140625,
      "eval_lambada_ppl": 11.17928486368573,
      "eval_lambada_runtime": 9.5843,
      "eval_lambada_samples_per_second": 508.02,
      "step": 384000
    },
    {
      "activations/layer0_attention_weight_max": 15.630777359008789,
      "activations/layer0_attention_weight_min": -14.11034107208252,
      "activations/layer10_attention_weight_max": 40.86589813232422,
      "activations/layer10_attention_weight_min": -39.92999267578125,
      "activations/layer11_attention_weight_max": 38.61275863647461,
      "activations/layer11_attention_weight_min": -34.9655876159668,
      "activations/layer12_attention_weight_max": 17.621885299682617,
      "activations/layer12_attention_weight_min": -25.963363647460938,
      "activations/layer13_attention_weight_max": 36.236480712890625,
      "activations/layer13_attention_weight_min": -25.575693130493164,
      "activations/layer14_attention_weight_max": 34.26649856567383,
      "activations/layer14_attention_weight_min": -29.28936195373535,
      "activations/layer15_attention_weight_max": 34.73048782348633,
      "activations/layer15_attention_weight_min": -28.785884857177734,
      "activations/layer16_attention_weight_max": 31.84580421447754,
      "activations/layer16_attention_weight_min": -30.35141944885254,
      "activations/layer17_attention_weight_max": 48.53731918334961,
      "activations/layer17_attention_weight_min": -45.17713928222656,
      "activations/layer18_attention_weight_max": 43.59724807739258,
      "activations/layer18_attention_weight_min": -39.777870178222656,
      "activations/layer19_attention_weight_max": 24.270170211791992,
      "activations/layer19_attention_weight_min": -21.241073608398438,
      "activations/layer1_attention_weight_max": 16.168188095092773,
      "activations/layer1_attention_weight_min": -14.276714324951172,
      "activations/layer20_attention_weight_max": 23.251792907714844,
      "activations/layer20_attention_weight_min": -19.54564094543457,
      "activations/layer21_attention_weight_max": 30.618370056152344,
      "activations/layer21_attention_weight_min": -21.4646053314209,
      "activations/layer22_attention_weight_max": 29.40825653076172,
      "activations/layer22_attention_weight_min": -24.252519607543945,
      "activations/layer23_attention_weight_max": 32.417137145996094,
      "activations/layer23_attention_weight_min": -23.988739013671875,
      "activations/layer2_attention_weight_max": 35.03955841064453,
      "activations/layer2_attention_weight_min": -32.30785369873047,
      "activations/layer3_attention_weight_max": 93.2646713256836,
      "activations/layer3_attention_weight_min": -92.10003662109375,
      "activations/layer4_attention_weight_max": 82.09259796142578,
      "activations/layer4_attention_weight_min": -81.29745483398438,
      "activations/layer5_attention_weight_max": 59.967315673828125,
      "activations/layer5_attention_weight_min": -70.114013671875,
      "activations/layer6_attention_weight_max": 46.296566009521484,
      "activations/layer6_attention_weight_min": -47.112327575683594,
      "activations/layer7_attention_weight_max": 82.28311920166016,
      "activations/layer7_attention_weight_min": -65.13895416259766,
      "activations/layer8_attention_weight_max": 53.475555419921875,
      "activations/layer8_attention_weight_min": -49.18791580200195,
      "activations/layer9_attention_weight_max": 59.4325065612793,
      "activations/layer9_attention_weight_min": -46.442562103271484,
      "epoch": 22.32,
      "learning_rate": 6.157575757575757e-06,
      "loss": 2.7062,
      "step": 384050
    },
    {
      "activations/layer0_attention_weight_max": 16.143281936645508,
      "activations/layer0_attention_weight_min": -14.366708755493164,
      "activations/layer10_attention_weight_max": 35.570068359375,
      "activations/layer10_attention_weight_min": -39.726375579833984,
      "activations/layer11_attention_weight_max": 36.91961669921875,
      "activations/layer11_attention_weight_min": -38.63370895385742,
      "activations/layer12_attention_weight_max": 19.60719871520996,
      "activations/layer12_attention_weight_min": -23.314966201782227,
      "activations/layer13_attention_weight_max": 35.02323913574219,
      "activations/layer13_attention_weight_min": -30.102956771850586,
      "activations/layer14_attention_weight_max": 34.849822998046875,
      "activations/layer14_attention_weight_min": -30.79935073852539,
      "activations/layer15_attention_weight_max": 32.88642883300781,
      "activations/layer15_attention_weight_min": -33.09382629394531,
      "activations/layer16_attention_weight_max": 35.581382751464844,
      "activations/layer16_attention_weight_min": -34.594398498535156,
      "activations/layer17_attention_weight_max": 55.02621078491211,
      "activations/layer17_attention_weight_min": -47.47224807739258,
      "activations/layer18_attention_weight_max": 46.66205596923828,
      "activations/layer18_attention_weight_min": -39.463260650634766,
      "activations/layer19_attention_weight_max": 23.304052352905273,
      "activations/layer19_attention_weight_min": -22.172380447387695,
      "activations/layer1_attention_weight_max": 15.0227689743042,
      "activations/layer1_attention_weight_min": -15.996563911437988,
      "activations/layer20_attention_weight_max": 21.88282585144043,
      "activations/layer20_attention_weight_min": -22.87320327758789,
      "activations/layer21_attention_weight_max": 33.69345474243164,
      "activations/layer21_attention_weight_min": -22.179054260253906,
      "activations/layer22_attention_weight_max": 26.004438400268555,
      "activations/layer22_attention_weight_min": -25.114046096801758,
      "activations/layer23_attention_weight_max": 34.6233024597168,
      "activations/layer23_attention_weight_min": -26.004905700683594,
      "activations/layer2_attention_weight_max": 31.386859893798828,
      "activations/layer2_attention_weight_min": -30.926074981689453,
      "activations/layer3_attention_weight_max": 85.94891357421875,
      "activations/layer3_attention_weight_min": -90.17398071289062,
      "activations/layer4_attention_weight_max": 80.5484848022461,
      "activations/layer4_attention_weight_min": -83.02674102783203,
      "activations/layer5_attention_weight_max": 59.65203094482422,
      "activations/layer5_attention_weight_min": -70.40475463867188,
      "activations/layer6_attention_weight_max": 49.97058868408203,
      "activations/layer6_attention_weight_min": -52.63560485839844,
      "activations/layer7_attention_weight_max": 65.37346649169922,
      "activations/layer7_attention_weight_min": -67.75081634521484,
      "activations/layer8_attention_weight_max": 45.788387298583984,
      "activations/layer8_attention_weight_min": -53.86857223510742,
      "activations/layer9_attention_weight_max": 44.59296798706055,
      "activations/layer9_attention_weight_min": -46.8664665222168,
      "epoch": 22.32,
      "learning_rate": 6.138636363636363e-06,
      "loss": 2.7111,
      "step": 384100
    },
    {
      "activations/layer0_attention_weight_max": 15.508118629455566,
      "activations/layer0_attention_weight_min": -14.217612266540527,
      "activations/layer10_attention_weight_max": 36.73308181762695,
      "activations/layer10_attention_weight_min": -34.89583206176758,
      "activations/layer11_attention_weight_max": 34.416603088378906,
      "activations/layer11_attention_weight_min": -33.672149658203125,
      "activations/layer12_attention_weight_max": 18.88384437561035,
      "activations/layer12_attention_weight_min": -25.497177124023438,
      "activations/layer13_attention_weight_max": 33.9987907409668,
      "activations/layer13_attention_weight_min": -26.18122673034668,
      "activations/layer14_attention_weight_max": 35.368141174316406,
      "activations/layer14_attention_weight_min": -29.49469566345215,
      "activations/layer15_attention_weight_max": 35.57244873046875,
      "activations/layer15_attention_weight_min": -29.50838851928711,
      "activations/layer16_attention_weight_max": 37.76952362060547,
      "activations/layer16_attention_weight_min": -32.78142166137695,
      "activations/layer17_attention_weight_max": 54.33533477783203,
      "activations/layer17_attention_weight_min": -45.86226272583008,
      "activations/layer18_attention_weight_max": 54.36648178100586,
      "activations/layer18_attention_weight_min": -41.11980056762695,
      "activations/layer19_attention_weight_max": 29.616153717041016,
      "activations/layer19_attention_weight_min": -24.92059898376465,
      "activations/layer1_attention_weight_max": 15.154557228088379,
      "activations/layer1_attention_weight_min": -13.619564056396484,
      "activations/layer20_attention_weight_max": 25.31332015991211,
      "activations/layer20_attention_weight_min": -22.29530906677246,
      "activations/layer21_attention_weight_max": 38.530521392822266,
      "activations/layer21_attention_weight_min": -23.08843994140625,
      "activations/layer22_attention_weight_max": 28.73662567138672,
      "activations/layer22_attention_weight_min": -25.949623107910156,
      "activations/layer23_attention_weight_max": 38.80710983276367,
      "activations/layer23_attention_weight_min": -24.421432495117188,
      "activations/layer2_attention_weight_max": 31.292861938476562,
      "activations/layer2_attention_weight_min": -31.244609832763672,
      "activations/layer3_attention_weight_max": 88.01114654541016,
      "activations/layer3_attention_weight_min": -87.72134399414062,
      "activations/layer4_attention_weight_max": 82.5044937133789,
      "activations/layer4_attention_weight_min": -82.00836181640625,
      "activations/layer5_attention_weight_max": 60.08915328979492,
      "activations/layer5_attention_weight_min": -69.54734802246094,
      "activations/layer6_attention_weight_max": 48.937110900878906,
      "activations/layer6_attention_weight_min": -50.522701263427734,
      "activations/layer7_attention_weight_max": 69.72405242919922,
      "activations/layer7_attention_weight_min": -66.3327407836914,
      "activations/layer8_attention_weight_max": 48.93572235107422,
      "activations/layer8_attention_weight_min": -49.64675521850586,
      "activations/layer9_attention_weight_max": 51.83736801147461,
      "activations/layer9_attention_weight_min": -47.648521423339844,
      "epoch": 22.32,
      "learning_rate": 6.1196969696969696e-06,
      "loss": 2.6998,
      "step": 384150
    },
    {
      "activations/layer0_attention_weight_max": 16.180599212646484,
      "activations/layer0_attention_weight_min": -14.081042289733887,
      "activations/layer10_attention_weight_max": 36.488094329833984,
      "activations/layer10_attention_weight_min": -37.3682975769043,
      "activations/layer11_attention_weight_max": 35.44571304321289,
      "activations/layer11_attention_weight_min": -36.34971618652344,
      "activations/layer12_attention_weight_max": 20.056045532226562,
      "activations/layer12_attention_weight_min": -27.673667907714844,
      "activations/layer13_attention_weight_max": 36.824790954589844,
      "activations/layer13_attention_weight_min": -27.006324768066406,
      "activations/layer14_attention_weight_max": 37.21261978149414,
      "activations/layer14_attention_weight_min": -29.71476936340332,
      "activations/layer15_attention_weight_max": 33.5576171875,
      "activations/layer15_attention_weight_min": -30.91681480407715,
      "activations/layer16_attention_weight_max": 34.15488815307617,
      "activations/layer16_attention_weight_min": -31.657514572143555,
      "activations/layer17_attention_weight_max": 52.37107467651367,
      "activations/layer17_attention_weight_min": -47.815120697021484,
      "activations/layer18_attention_weight_max": 44.90772247314453,
      "activations/layer18_attention_weight_min": -40.24726486206055,
      "activations/layer19_attention_weight_max": 24.332046508789062,
      "activations/layer19_attention_weight_min": -25.054780960083008,
      "activations/layer1_attention_weight_max": 14.867027282714844,
      "activations/layer1_attention_weight_min": -14.04222297668457,
      "activations/layer20_attention_weight_max": 25.90875816345215,
      "activations/layer20_attention_weight_min": -25.558164596557617,
      "activations/layer21_attention_weight_max": 34.637603759765625,
      "activations/layer21_attention_weight_min": -22.09487533569336,
      "activations/layer22_attention_weight_max": 28.912364959716797,
      "activations/layer22_attention_weight_min": -25.592771530151367,
      "activations/layer23_attention_weight_max": 34.31188201904297,
      "activations/layer23_attention_weight_min": -22.743988037109375,
      "activations/layer2_attention_weight_max": 32.239559173583984,
      "activations/layer2_attention_weight_min": -30.716014862060547,
      "activations/layer3_attention_weight_max": 90.6463623046875,
      "activations/layer3_attention_weight_min": -92.8922348022461,
      "activations/layer4_attention_weight_max": 86.1828384399414,
      "activations/layer4_attention_weight_min": -84.93938446044922,
      "activations/layer5_attention_weight_max": 60.166378021240234,
      "activations/layer5_attention_weight_min": -71.45845794677734,
      "activations/layer6_attention_weight_max": 48.94522476196289,
      "activations/layer6_attention_weight_min": -51.023338317871094,
      "activations/layer7_attention_weight_max": 68.44664001464844,
      "activations/layer7_attention_weight_min": -65.88958740234375,
      "activations/layer8_attention_weight_max": 46.549842834472656,
      "activations/layer8_attention_weight_min": -51.044803619384766,
      "activations/layer9_attention_weight_max": 43.86709213256836,
      "activations/layer9_attention_weight_min": -47.85661315917969,
      "epoch": 22.32,
      "learning_rate": 6.100757575757575e-06,
      "loss": 2.7151,
      "step": 384200
    },
    {
      "activations/layer0_attention_weight_max": 15.567339897155762,
      "activations/layer0_attention_weight_min": -14.249669075012207,
      "activations/layer10_attention_weight_max": 35.92142868041992,
      "activations/layer10_attention_weight_min": -33.33726119995117,
      "activations/layer11_attention_weight_max": 34.055965423583984,
      "activations/layer11_attention_weight_min": -33.18729782104492,
      "activations/layer12_attention_weight_max": 20.416030883789062,
      "activations/layer12_attention_weight_min": -26.131502151489258,
      "activations/layer13_attention_weight_max": 34.3619384765625,
      "activations/layer13_attention_weight_min": -26.286319732666016,
      "activations/layer14_attention_weight_max": 35.948944091796875,
      "activations/layer14_attention_weight_min": -31.47144317626953,
      "activations/layer15_attention_weight_max": 32.98241424560547,
      "activations/layer15_attention_weight_min": -31.028581619262695,
      "activations/layer16_attention_weight_max": 34.50938415527344,
      "activations/layer16_attention_weight_min": -32.89386749267578,
      "activations/layer17_attention_weight_max": 49.935115814208984,
      "activations/layer17_attention_weight_min": -44.45992660522461,
      "activations/layer18_attention_weight_max": 49.95233917236328,
      "activations/layer18_attention_weight_min": -41.70466232299805,
      "activations/layer19_attention_weight_max": 27.3504695892334,
      "activations/layer19_attention_weight_min": -24.180015563964844,
      "activations/layer1_attention_weight_max": 15.125354766845703,
      "activations/layer1_attention_weight_min": -14.079907417297363,
      "activations/layer20_attention_weight_max": 24.349227905273438,
      "activations/layer20_attention_weight_min": -20.614044189453125,
      "activations/layer21_attention_weight_max": 33.19730758666992,
      "activations/layer21_attention_weight_min": -23.858476638793945,
      "activations/layer22_attention_weight_max": 28.14459800720215,
      "activations/layer22_attention_weight_min": -26.118316650390625,
      "activations/layer23_attention_weight_max": 32.90237045288086,
      "activations/layer23_attention_weight_min": -24.671520233154297,
      "activations/layer2_attention_weight_max": 32.941654205322266,
      "activations/layer2_attention_weight_min": -29.491497039794922,
      "activations/layer3_attention_weight_max": 86.44036865234375,
      "activations/layer3_attention_weight_min": -86.15449523925781,
      "activations/layer4_attention_weight_max": 81.03670501708984,
      "activations/layer4_attention_weight_min": -81.06067657470703,
      "activations/layer5_attention_weight_max": 65.13555908203125,
      "activations/layer5_attention_weight_min": -70.4797134399414,
      "activations/layer6_attention_weight_max": 49.180625915527344,
      "activations/layer6_attention_weight_min": -48.31159210205078,
      "activations/layer7_attention_weight_max": 67.09712219238281,
      "activations/layer7_attention_weight_min": -65.44303131103516,
      "activations/layer8_attention_weight_max": 45.24527359008789,
      "activations/layer8_attention_weight_min": -49.428749084472656,
      "activations/layer9_attention_weight_max": 41.21722412109375,
      "activations/layer9_attention_weight_min": -45.54277801513672,
      "epoch": 22.33,
      "learning_rate": 6.081818181818181e-06,
      "loss": 2.7004,
      "step": 384250
    },
    {
      "activations/layer0_attention_weight_max": 16.267759323120117,
      "activations/layer0_attention_weight_min": -13.936997413635254,
      "activations/layer10_attention_weight_max": 39.54181671142578,
      "activations/layer10_attention_weight_min": -39.788509368896484,
      "activations/layer11_attention_weight_max": 37.688270568847656,
      "activations/layer11_attention_weight_min": -37.51044845581055,
      "activations/layer12_attention_weight_max": 21.963132858276367,
      "activations/layer12_attention_weight_min": -25.749061584472656,
      "activations/layer13_attention_weight_max": 39.31554412841797,
      "activations/layer13_attention_weight_min": -29.354740142822266,
      "activations/layer14_attention_weight_max": 37.76326370239258,
      "activations/layer14_attention_weight_min": -30.755586624145508,
      "activations/layer15_attention_weight_max": 36.92420196533203,
      "activations/layer15_attention_weight_min": -29.149337768554688,
      "activations/layer16_attention_weight_max": 39.202850341796875,
      "activations/layer16_attention_weight_min": -35.63847732543945,
      "activations/layer17_attention_weight_max": 54.24604034423828,
      "activations/layer17_attention_weight_min": -46.41281509399414,
      "activations/layer18_attention_weight_max": 51.26588821411133,
      "activations/layer18_attention_weight_min": -40.43687057495117,
      "activations/layer19_attention_weight_max": 26.100494384765625,
      "activations/layer19_attention_weight_min": -22.486539840698242,
      "activations/layer1_attention_weight_max": 14.323307991027832,
      "activations/layer1_attention_weight_min": -13.839288711547852,
      "activations/layer20_attention_weight_max": 23.549100875854492,
      "activations/layer20_attention_weight_min": -21.511812210083008,
      "activations/layer21_attention_weight_max": 33.968780517578125,
      "activations/layer21_attention_weight_min": -19.590625762939453,
      "activations/layer22_attention_weight_max": 30.23769760131836,
      "activations/layer22_attention_weight_min": -25.57135581970215,
      "activations/layer23_attention_weight_max": 37.16947555541992,
      "activations/layer23_attention_weight_min": -23.02227783203125,
      "activations/layer2_attention_weight_max": 31.77815818786621,
      "activations/layer2_attention_weight_min": -31.15521240234375,
      "activations/layer3_attention_weight_max": 94.01165008544922,
      "activations/layer3_attention_weight_min": -94.20713806152344,
      "activations/layer4_attention_weight_max": 88.67898559570312,
      "activations/layer4_attention_weight_min": -91.02801513671875,
      "activations/layer5_attention_weight_max": 66.55195617675781,
      "activations/layer5_attention_weight_min": -71.2342529296875,
      "activations/layer6_attention_weight_max": 51.98461151123047,
      "activations/layer6_attention_weight_min": -52.320980072021484,
      "activations/layer7_attention_weight_max": 77.50455474853516,
      "activations/layer7_attention_weight_min": -73.24170684814453,
      "activations/layer8_attention_weight_max": 51.49039077758789,
      "activations/layer8_attention_weight_min": -56.34995651245117,
      "activations/layer9_attention_weight_max": 48.77035903930664,
      "activations/layer9_attention_weight_min": -52.41716384887695,
      "epoch": 22.33,
      "learning_rate": 6.062878787878787e-06,
      "loss": 2.6988,
      "step": 384300
    },
    {
      "activations/layer0_attention_weight_max": 16.120819091796875,
      "activations/layer0_attention_weight_min": -14.008770942687988,
      "activations/layer10_attention_weight_max": 38.1241455078125,
      "activations/layer10_attention_weight_min": -37.38750076293945,
      "activations/layer11_attention_weight_max": 36.58330535888672,
      "activations/layer11_attention_weight_min": -35.12376022338867,
      "activations/layer12_attention_weight_max": 18.549596786499023,
      "activations/layer12_attention_weight_min": -23.70309829711914,
      "activations/layer13_attention_weight_max": 34.35711669921875,
      "activations/layer13_attention_weight_min": -28.889942169189453,
      "activations/layer14_attention_weight_max": 37.193885803222656,
      "activations/layer14_attention_weight_min": -29.86298942565918,
      "activations/layer15_attention_weight_max": 37.52200698852539,
      "activations/layer15_attention_weight_min": -31.94799041748047,
      "activations/layer16_attention_weight_max": 39.56415939331055,
      "activations/layer16_attention_weight_min": -37.42317581176758,
      "activations/layer17_attention_weight_max": 58.34056854248047,
      "activations/layer17_attention_weight_min": -51.593692779541016,
      "activations/layer18_attention_weight_max": 51.27796936035156,
      "activations/layer18_attention_weight_min": -44.53528594970703,
      "activations/layer19_attention_weight_max": 26.971485137939453,
      "activations/layer19_attention_weight_min": -25.146093368530273,
      "activations/layer1_attention_weight_max": 14.534562110900879,
      "activations/layer1_attention_weight_min": -13.826756477355957,
      "activations/layer20_attention_weight_max": 22.256616592407227,
      "activations/layer20_attention_weight_min": -20.080373764038086,
      "activations/layer21_attention_weight_max": 38.72214126586914,
      "activations/layer21_attention_weight_min": -20.318422317504883,
      "activations/layer22_attention_weight_max": 34.342952728271484,
      "activations/layer22_attention_weight_min": -25.747394561767578,
      "activations/layer23_attention_weight_max": 40.57524871826172,
      "activations/layer23_attention_weight_min": -27.389074325561523,
      "activations/layer2_attention_weight_max": 31.842727661132812,
      "activations/layer2_attention_weight_min": -30.464147567749023,
      "activations/layer3_attention_weight_max": 92.43305206298828,
      "activations/layer3_attention_weight_min": -91.65776824951172,
      "activations/layer4_attention_weight_max": 86.79046630859375,
      "activations/layer4_attention_weight_min": -88.47862243652344,
      "activations/layer5_attention_weight_max": 60.688331604003906,
      "activations/layer5_attention_weight_min": -70.02886962890625,
      "activations/layer6_attention_weight_max": 52.091251373291016,
      "activations/layer6_attention_weight_min": -54.05607986450195,
      "activations/layer7_attention_weight_max": 69.461669921875,
      "activations/layer7_attention_weight_min": -68.8392333984375,
      "activations/layer8_attention_weight_max": 46.98929977416992,
      "activations/layer8_attention_weight_min": -51.07088088989258,
      "activations/layer9_attention_weight_max": 48.580135345458984,
      "activations/layer9_attention_weight_min": -47.73784637451172,
      "epoch": 22.33,
      "learning_rate": 6.043939393939393e-06,
      "loss": 2.7217,
      "step": 384350
    },
    {
      "activations/layer0_attention_weight_max": 16.1915283203125,
      "activations/layer0_attention_weight_min": -14.216268539428711,
      "activations/layer10_attention_weight_max": 34.8970947265625,
      "activations/layer10_attention_weight_min": -36.49724197387695,
      "activations/layer11_attention_weight_max": 33.5635871887207,
      "activations/layer11_attention_weight_min": -36.50843811035156,
      "activations/layer12_attention_weight_max": 19.284833908081055,
      "activations/layer12_attention_weight_min": -23.812576293945312,
      "activations/layer13_attention_weight_max": 39.999813079833984,
      "activations/layer13_attention_weight_min": -28.636804580688477,
      "activations/layer14_attention_weight_max": 33.83654022216797,
      "activations/layer14_attention_weight_min": -29.320472717285156,
      "activations/layer15_attention_weight_max": 34.932334899902344,
      "activations/layer15_attention_weight_min": -29.15023422241211,
      "activations/layer16_attention_weight_max": 37.38527297973633,
      "activations/layer16_attention_weight_min": -35.01917266845703,
      "activations/layer17_attention_weight_max": 54.91496658325195,
      "activations/layer17_attention_weight_min": -46.42412185668945,
      "activations/layer18_attention_weight_max": 53.165565490722656,
      "activations/layer18_attention_weight_min": -42.12057876586914,
      "activations/layer19_attention_weight_max": 23.6523380279541,
      "activations/layer19_attention_weight_min": -21.78888511657715,
      "activations/layer1_attention_weight_max": 15.262153625488281,
      "activations/layer1_attention_weight_min": -14.307426452636719,
      "activations/layer20_attention_weight_max": 21.195354461669922,
      "activations/layer20_attention_weight_min": -21.617719650268555,
      "activations/layer21_attention_weight_max": 30.722654342651367,
      "activations/layer21_attention_weight_min": -19.455968856811523,
      "activations/layer22_attention_weight_max": 26.571802139282227,
      "activations/layer22_attention_weight_min": -27.163721084594727,
      "activations/layer23_attention_weight_max": 34.29862976074219,
      "activations/layer23_attention_weight_min": -24.451505661010742,
      "activations/layer2_attention_weight_max": 31.935596466064453,
      "activations/layer2_attention_weight_min": -30.399337768554688,
      "activations/layer3_attention_weight_max": 91.54118347167969,
      "activations/layer3_attention_weight_min": -91.84232330322266,
      "activations/layer4_attention_weight_max": 85.21337127685547,
      "activations/layer4_attention_weight_min": -86.2876205444336,
      "activations/layer5_attention_weight_max": 61.91473388671875,
      "activations/layer5_attention_weight_min": -70.22451782226562,
      "activations/layer6_attention_weight_max": 50.83649444580078,
      "activations/layer6_attention_weight_min": -51.7027702331543,
      "activations/layer7_attention_weight_max": 69.69734191894531,
      "activations/layer7_attention_weight_min": -69.01378631591797,
      "activations/layer8_attention_weight_max": 47.00390625,
      "activations/layer8_attention_weight_min": -51.7863883972168,
      "activations/layer9_attention_weight_max": 43.318355560302734,
      "activations/layer9_attention_weight_min": -46.57221603393555,
      "epoch": 22.34,
      "learning_rate": 6.025e-06,
      "loss": 2.7239,
      "step": 384400
    },
    {
      "activations/layer0_attention_weight_max": 15.303871154785156,
      "activations/layer0_attention_weight_min": -13.940999031066895,
      "activations/layer10_attention_weight_max": 35.649658203125,
      "activations/layer10_attention_weight_min": -36.799049377441406,
      "activations/layer11_attention_weight_max": 34.9876708984375,
      "activations/layer11_attention_weight_min": -36.4703254699707,
      "activations/layer12_attention_weight_max": 21.247671127319336,
      "activations/layer12_attention_weight_min": -24.822784423828125,
      "activations/layer13_attention_weight_max": 36.4373893737793,
      "activations/layer13_attention_weight_min": -31.487977981567383,
      "activations/layer14_attention_weight_max": 38.307159423828125,
      "activations/layer14_attention_weight_min": -35.28742980957031,
      "activations/layer15_attention_weight_max": 36.1716194152832,
      "activations/layer15_attention_weight_min": -35.6207275390625,
      "activations/layer16_attention_weight_max": 36.50447082519531,
      "activations/layer16_attention_weight_min": -34.455909729003906,
      "activations/layer17_attention_weight_max": 56.446495056152344,
      "activations/layer17_attention_weight_min": -52.099952697753906,
      "activations/layer18_attention_weight_max": 52.109249114990234,
      "activations/layer18_attention_weight_min": -43.70737075805664,
      "activations/layer19_attention_weight_max": 28.753910064697266,
      "activations/layer19_attention_weight_min": -23.478561401367188,
      "activations/layer1_attention_weight_max": 15.324865341186523,
      "activations/layer1_attention_weight_min": -15.366087913513184,
      "activations/layer20_attention_weight_max": 26.7811222076416,
      "activations/layer20_attention_weight_min": -21.672990798950195,
      "activations/layer21_attention_weight_max": 35.95752716064453,
      "activations/layer21_attention_weight_min": -20.96448516845703,
      "activations/layer22_attention_weight_max": 31.144866943359375,
      "activations/layer22_attention_weight_min": -23.99217414855957,
      "activations/layer23_attention_weight_max": 37.38007354736328,
      "activations/layer23_attention_weight_min": -23.616474151611328,
      "activations/layer2_attention_weight_max": 32.823760986328125,
      "activations/layer2_attention_weight_min": -30.489622116088867,
      "activations/layer3_attention_weight_max": 89.79305267333984,
      "activations/layer3_attention_weight_min": -92.18916320800781,
      "activations/layer4_attention_weight_max": 87.04192352294922,
      "activations/layer4_attention_weight_min": -88.01750183105469,
      "activations/layer5_attention_weight_max": 60.95922088623047,
      "activations/layer5_attention_weight_min": -70.83038330078125,
      "activations/layer6_attention_weight_max": 48.08405303955078,
      "activations/layer6_attention_weight_min": -52.246925354003906,
      "activations/layer7_attention_weight_max": 69.78961944580078,
      "activations/layer7_attention_weight_min": -69.05550384521484,
      "activations/layer8_attention_weight_max": 46.15250778198242,
      "activations/layer8_attention_weight_min": -53.33753967285156,
      "activations/layer9_attention_weight_max": 46.41767501831055,
      "activations/layer9_attention_weight_min": -47.95145797729492,
      "epoch": 22.34,
      "learning_rate": 6.006060606060606e-06,
      "loss": 2.7084,
      "step": 384450
    },
    {
      "activations/layer0_attention_weight_max": 16.033409118652344,
      "activations/layer0_attention_weight_min": -14.30587100982666,
      "activations/layer10_attention_weight_max": 36.269805908203125,
      "activations/layer10_attention_weight_min": -37.402835845947266,
      "activations/layer11_attention_weight_max": 34.719303131103516,
      "activations/layer11_attention_weight_min": -37.55054473876953,
      "activations/layer12_attention_weight_max": 20.604448318481445,
      "activations/layer12_attention_weight_min": -27.179113388061523,
      "activations/layer13_attention_weight_max": 36.974853515625,
      "activations/layer13_attention_weight_min": -27.39164161682129,
      "activations/layer14_attention_weight_max": 36.13019561767578,
      "activations/layer14_attention_weight_min": -30.758703231811523,
      "activations/layer15_attention_weight_max": 36.441986083984375,
      "activations/layer15_attention_weight_min": -32.318111419677734,
      "activations/layer16_attention_weight_max": 34.750850677490234,
      "activations/layer16_attention_weight_min": -33.50236511230469,
      "activations/layer17_attention_weight_max": 52.483158111572266,
      "activations/layer17_attention_weight_min": -45.83980178833008,
      "activations/layer18_attention_weight_max": 48.41905975341797,
      "activations/layer18_attention_weight_min": -40.96644973754883,
      "activations/layer19_attention_weight_max": 23.75778579711914,
      "activations/layer19_attention_weight_min": -24.464513778686523,
      "activations/layer1_attention_weight_max": 15.333212852478027,
      "activations/layer1_attention_weight_min": -14.643794059753418,
      "activations/layer20_attention_weight_max": 22.679298400878906,
      "activations/layer20_attention_weight_min": -22.571922302246094,
      "activations/layer21_attention_weight_max": 35.385215759277344,
      "activations/layer21_attention_weight_min": -21.813655853271484,
      "activations/layer22_attention_weight_max": 29.64167594909668,
      "activations/layer22_attention_weight_min": -23.66895866394043,
      "activations/layer23_attention_weight_max": 36.947265625,
      "activations/layer23_attention_weight_min": -25.849058151245117,
      "activations/layer2_attention_weight_max": 32.12529373168945,
      "activations/layer2_attention_weight_min": -30.741474151611328,
      "activations/layer3_attention_weight_max": 91.67890930175781,
      "activations/layer3_attention_weight_min": -87.1470947265625,
      "activations/layer4_attention_weight_max": 89.77882385253906,
      "activations/layer4_attention_weight_min": -83.84358978271484,
      "activations/layer5_attention_weight_max": 63.660945892333984,
      "activations/layer5_attention_weight_min": -75.04324340820312,
      "activations/layer6_attention_weight_max": 51.4188117980957,
      "activations/layer6_attention_weight_min": -52.08176803588867,
      "activations/layer7_attention_weight_max": 68.67588806152344,
      "activations/layer7_attention_weight_min": -70.3451156616211,
      "activations/layer8_attention_weight_max": 49.086185455322266,
      "activations/layer8_attention_weight_min": -53.34918975830078,
      "activations/layer9_attention_weight_max": 44.529685974121094,
      "activations/layer9_attention_weight_min": -48.05918502807617,
      "epoch": 22.34,
      "learning_rate": 5.987121212121212e-06,
      "loss": 2.7104,
      "step": 384500
    },
    {
      "activations/layer0_attention_weight_max": 16.333873748779297,
      "activations/layer0_attention_weight_min": -14.372200965881348,
      "activations/layer10_attention_weight_max": 38.41448974609375,
      "activations/layer10_attention_weight_min": -36.32760238647461,
      "activations/layer11_attention_weight_max": 36.433441162109375,
      "activations/layer11_attention_weight_min": -35.665252685546875,
      "activations/layer12_attention_weight_max": 19.1229190826416,
      "activations/layer12_attention_weight_min": -27.23929786682129,
      "activations/layer13_attention_weight_max": 38.06126022338867,
      "activations/layer13_attention_weight_min": -28.083309173583984,
      "activations/layer14_attention_weight_max": 40.95807647705078,
      "activations/layer14_attention_weight_min": -31.155946731567383,
      "activations/layer15_attention_weight_max": 41.90321731567383,
      "activations/layer15_attention_weight_min": -33.23260498046875,
      "activations/layer16_attention_weight_max": 43.06596755981445,
      "activations/layer16_attention_weight_min": -35.224796295166016,
      "activations/layer17_attention_weight_max": 62.99959945678711,
      "activations/layer17_attention_weight_min": -49.646522521972656,
      "activations/layer18_attention_weight_max": 58.11984634399414,
      "activations/layer18_attention_weight_min": -44.97934341430664,
      "activations/layer19_attention_weight_max": 33.67742919921875,
      "activations/layer19_attention_weight_min": -26.413843154907227,
      "activations/layer1_attention_weight_max": 15.790678977966309,
      "activations/layer1_attention_weight_min": -13.518174171447754,
      "activations/layer20_attention_weight_max": 31.162851333618164,
      "activations/layer20_attention_weight_min": -24.410781860351562,
      "activations/layer21_attention_weight_max": 55.07560348510742,
      "activations/layer21_attention_weight_min": -27.95656967163086,
      "activations/layer22_attention_weight_max": 39.40390396118164,
      "activations/layer22_attention_weight_min": -26.424877166748047,
      "activations/layer23_attention_weight_max": 49.291297912597656,
      "activations/layer23_attention_weight_min": -25.29256820678711,
      "activations/layer2_attention_weight_max": 33.20609664916992,
      "activations/layer2_attention_weight_min": -32.73583984375,
      "activations/layer3_attention_weight_max": 90.57191467285156,
      "activations/layer3_attention_weight_min": -93.81462860107422,
      "activations/layer4_attention_weight_max": 83.22720336914062,
      "activations/layer4_attention_weight_min": -83.32193756103516,
      "activations/layer5_attention_weight_max": 60.74496078491211,
      "activations/layer5_attention_weight_min": -67.28034210205078,
      "activations/layer6_attention_weight_max": 50.03440475463867,
      "activations/layer6_attention_weight_min": -48.72602844238281,
      "activations/layer7_attention_weight_max": 68.33856201171875,
      "activations/layer7_attention_weight_min": -73.71468353271484,
      "activations/layer8_attention_weight_max": 48.944976806640625,
      "activations/layer8_attention_weight_min": -51.82538986206055,
      "activations/layer9_attention_weight_max": 47.139305114746094,
      "activations/layer9_attention_weight_min": -47.08782196044922,
      "epoch": 22.34,
      "learning_rate": 5.968181818181818e-06,
      "loss": 2.7108,
      "step": 384550
    },
    {
      "activations/layer0_attention_weight_max": 15.816537857055664,
      "activations/layer0_attention_weight_min": -14.345449447631836,
      "activations/layer10_attention_weight_max": 37.011505126953125,
      "activations/layer10_attention_weight_min": -34.059425354003906,
      "activations/layer11_attention_weight_max": 32.57910919189453,
      "activations/layer11_attention_weight_min": -34.039527893066406,
      "activations/layer12_attention_weight_max": 19.075626373291016,
      "activations/layer12_attention_weight_min": -23.94816780090332,
      "activations/layer13_attention_weight_max": 34.95627212524414,
      "activations/layer13_attention_weight_min": -28.728404998779297,
      "activations/layer14_attention_weight_max": 35.86455154418945,
      "activations/layer14_attention_weight_min": -33.069854736328125,
      "activations/layer15_attention_weight_max": 33.175987243652344,
      "activations/layer15_attention_weight_min": -32.44823455810547,
      "activations/layer16_attention_weight_max": 38.26063919067383,
      "activations/layer16_attention_weight_min": -36.121402740478516,
      "activations/layer17_attention_weight_max": 53.31130599975586,
      "activations/layer17_attention_weight_min": -49.34979248046875,
      "activations/layer18_attention_weight_max": 52.29930877685547,
      "activations/layer18_attention_weight_min": -44.61416244506836,
      "activations/layer19_attention_weight_max": 26.488309860229492,
      "activations/layer19_attention_weight_min": -27.04474449157715,
      "activations/layer1_attention_weight_max": 15.180944442749023,
      "activations/layer1_attention_weight_min": -16.062179565429688,
      "activations/layer20_attention_weight_max": 24.269149780273438,
      "activations/layer20_attention_weight_min": -22.779726028442383,
      "activations/layer21_attention_weight_max": 41.718505859375,
      "activations/layer21_attention_weight_min": -21.72615623474121,
      "activations/layer22_attention_weight_max": 33.60865020751953,
      "activations/layer22_attention_weight_min": -26.65776252746582,
      "activations/layer23_attention_weight_max": 45.22823715209961,
      "activations/layer23_attention_weight_min": -25.215892791748047,
      "activations/layer2_attention_weight_max": 31.12367820739746,
      "activations/layer2_attention_weight_min": -29.803586959838867,
      "activations/layer3_attention_weight_max": 87.30282592773438,
      "activations/layer3_attention_weight_min": -85.92829895019531,
      "activations/layer4_attention_weight_max": 82.95010375976562,
      "activations/layer4_attention_weight_min": -84.71978759765625,
      "activations/layer5_attention_weight_max": 63.001949310302734,
      "activations/layer5_attention_weight_min": -71.0990219116211,
      "activations/layer6_attention_weight_max": 49.92629623413086,
      "activations/layer6_attention_weight_min": -50.12390899658203,
      "activations/layer7_attention_weight_max": 64.85763549804688,
      "activations/layer7_attention_weight_min": -67.6884536743164,
      "activations/layer8_attention_weight_max": 47.23246765136719,
      "activations/layer8_attention_weight_min": -51.17021560668945,
      "activations/layer9_attention_weight_max": 43.844120025634766,
      "activations/layer9_attention_weight_min": -45.53257369995117,
      "epoch": 22.35,
      "learning_rate": 5.949242424242424e-06,
      "loss": 2.6997,
      "step": 384600
    },
    {
      "activations/layer0_attention_weight_max": 15.755970001220703,
      "activations/layer0_attention_weight_min": -14.198291778564453,
      "activations/layer10_attention_weight_max": 37.84437561035156,
      "activations/layer10_attention_weight_min": -40.13358688354492,
      "activations/layer11_attention_weight_max": 35.50950241088867,
      "activations/layer11_attention_weight_min": -39.27178955078125,
      "activations/layer12_attention_weight_max": 20.284738540649414,
      "activations/layer12_attention_weight_min": -25.066007614135742,
      "activations/layer13_attention_weight_max": 45.192054748535156,
      "activations/layer13_attention_weight_min": -26.886369705200195,
      "activations/layer14_attention_weight_max": 38.523990631103516,
      "activations/layer14_attention_weight_min": -32.787254333496094,
      "activations/layer15_attention_weight_max": 35.43913650512695,
      "activations/layer15_attention_weight_min": -32.3584098815918,
      "activations/layer16_attention_weight_max": 37.3823356628418,
      "activations/layer16_attention_weight_min": -35.93927764892578,
      "activations/layer17_attention_weight_max": 53.46461486816406,
      "activations/layer17_attention_weight_min": -49.48576736450195,
      "activations/layer18_attention_weight_max": 48.96542739868164,
      "activations/layer18_attention_weight_min": -45.861602783203125,
      "activations/layer19_attention_weight_max": 27.340604782104492,
      "activations/layer19_attention_weight_min": -24.469079971313477,
      "activations/layer1_attention_weight_max": 14.840755462646484,
      "activations/layer1_attention_weight_min": -13.811985969543457,
      "activations/layer20_attention_weight_max": 24.476621627807617,
      "activations/layer20_attention_weight_min": -21.17900276184082,
      "activations/layer21_attention_weight_max": 40.30905532836914,
      "activations/layer21_attention_weight_min": -25.45708465576172,
      "activations/layer22_attention_weight_max": 31.343679428100586,
      "activations/layer22_attention_weight_min": -25.34716796875,
      "activations/layer23_attention_weight_max": 38.99010467529297,
      "activations/layer23_attention_weight_min": -23.926939010620117,
      "activations/layer2_attention_weight_max": 31.66805648803711,
      "activations/layer2_attention_weight_min": -29.67926788330078,
      "activations/layer3_attention_weight_max": 91.2657241821289,
      "activations/layer3_attention_weight_min": -89.38401794433594,
      "activations/layer4_attention_weight_max": 85.97779083251953,
      "activations/layer4_attention_weight_min": -84.9315414428711,
      "activations/layer5_attention_weight_max": 61.842010498046875,
      "activations/layer5_attention_weight_min": -70.2215347290039,
      "activations/layer6_attention_weight_max": 51.99748992919922,
      "activations/layer6_attention_weight_min": -51.02378845214844,
      "activations/layer7_attention_weight_max": 74.01245880126953,
      "activations/layer7_attention_weight_min": -66.64795684814453,
      "activations/layer8_attention_weight_max": 49.20701599121094,
      "activations/layer8_attention_weight_min": -49.73237609863281,
      "activations/layer9_attention_weight_max": 43.94940185546875,
      "activations/layer9_attention_weight_min": -45.44793701171875,
      "epoch": 22.35,
      "learning_rate": 5.930303030303029e-06,
      "loss": 2.7022,
      "step": 384650
    },
    {
      "activations/layer0_attention_weight_max": 16.552215576171875,
      "activations/layer0_attention_weight_min": -14.059643745422363,
      "activations/layer10_attention_weight_max": 36.19266891479492,
      "activations/layer10_attention_weight_min": -36.263671875,
      "activations/layer11_attention_weight_max": 34.80945587158203,
      "activations/layer11_attention_weight_min": -37.11022186279297,
      "activations/layer12_attention_weight_max": 18.793649673461914,
      "activations/layer12_attention_weight_min": -27.033647537231445,
      "activations/layer13_attention_weight_max": 37.43659210205078,
      "activations/layer13_attention_weight_min": -28.253543853759766,
      "activations/layer14_attention_weight_max": 35.8113899230957,
      "activations/layer14_attention_weight_min": -28.875688552856445,
      "activations/layer15_attention_weight_max": 38.39759826660156,
      "activations/layer15_attention_weight_min": -29.04020881652832,
      "activations/layer16_attention_weight_max": 38.19662857055664,
      "activations/layer16_attention_weight_min": -33.28480911254883,
      "activations/layer17_attention_weight_max": 52.71562576293945,
      "activations/layer17_attention_weight_min": -45.11445236206055,
      "activations/layer18_attention_weight_max": 50.908653259277344,
      "activations/layer18_attention_weight_min": -38.66010665893555,
      "activations/layer19_attention_weight_max": 25.047266006469727,
      "activations/layer19_attention_weight_min": -24.906579971313477,
      "activations/layer1_attention_weight_max": 14.649850845336914,
      "activations/layer1_attention_weight_min": -14.05219554901123,
      "activations/layer20_attention_weight_max": 23.639812469482422,
      "activations/layer20_attention_weight_min": -21.205053329467773,
      "activations/layer21_attention_weight_max": 47.94911575317383,
      "activations/layer21_attention_weight_min": -22.474353790283203,
      "activations/layer22_attention_weight_max": 32.61472702026367,
      "activations/layer22_attention_weight_min": -23.99466323852539,
      "activations/layer23_attention_weight_max": 40.987518310546875,
      "activations/layer23_attention_weight_min": -24.781658172607422,
      "activations/layer2_attention_weight_max": 32.300106048583984,
      "activations/layer2_attention_weight_min": -30.62061882019043,
      "activations/layer3_attention_weight_max": 94.36737060546875,
      "activations/layer3_attention_weight_min": -90.45992279052734,
      "activations/layer4_attention_weight_max": 90.28630065917969,
      "activations/layer4_attention_weight_min": -87.27830505371094,
      "activations/layer5_attention_weight_max": 64.65586853027344,
      "activations/layer5_attention_weight_min": -73.03218078613281,
      "activations/layer6_attention_weight_max": 50.20911407470703,
      "activations/layer6_attention_weight_min": -51.90791702270508,
      "activations/layer7_attention_weight_max": 69.89099884033203,
      "activations/layer7_attention_weight_min": -70.10877227783203,
      "activations/layer8_attention_weight_max": 50.292049407958984,
      "activations/layer8_attention_weight_min": -52.3012580871582,
      "activations/layer9_attention_weight_max": 49.948341369628906,
      "activations/layer9_attention_weight_min": -48.59748458862305,
      "epoch": 22.35,
      "learning_rate": 5.911363636363636e-06,
      "loss": 2.7109,
      "step": 384700
    },
    {
      "activations/layer0_attention_weight_max": 15.793243408203125,
      "activations/layer0_attention_weight_min": -13.999077796936035,
      "activations/layer10_attention_weight_max": 41.25334167480469,
      "activations/layer10_attention_weight_min": -37.521385192871094,
      "activations/layer11_attention_weight_max": 36.71998596191406,
      "activations/layer11_attention_weight_min": -39.184627532958984,
      "activations/layer12_attention_weight_max": 20.815702438354492,
      "activations/layer12_attention_weight_min": -25.458515167236328,
      "activations/layer13_attention_weight_max": 37.567562103271484,
      "activations/layer13_attention_weight_min": -27.73891258239746,
      "activations/layer14_attention_weight_max": 39.23931121826172,
      "activations/layer14_attention_weight_min": -32.01457977294922,
      "activations/layer15_attention_weight_max": 38.22774887084961,
      "activations/layer15_attention_weight_min": -31.373991012573242,
      "activations/layer16_attention_weight_max": 40.04147720336914,
      "activations/layer16_attention_weight_min": -35.05685043334961,
      "activations/layer17_attention_weight_max": 55.73506546020508,
      "activations/layer17_attention_weight_min": -50.03090286254883,
      "activations/layer18_attention_weight_max": 53.961639404296875,
      "activations/layer18_attention_weight_min": -44.347023010253906,
      "activations/layer19_attention_weight_max": 28.87098503112793,
      "activations/layer19_attention_weight_min": -23.73087501525879,
      "activations/layer1_attention_weight_max": 15.187292098999023,
      "activations/layer1_attention_weight_min": -15.034018516540527,
      "activations/layer20_attention_weight_max": 25.36613655090332,
      "activations/layer20_attention_weight_min": -21.52176284790039,
      "activations/layer21_attention_weight_max": 35.847904205322266,
      "activations/layer21_attention_weight_min": -22.201887130737305,
      "activations/layer22_attention_weight_max": 32.12632751464844,
      "activations/layer22_attention_weight_min": -26.20880699157715,
      "activations/layer23_attention_weight_max": 38.89398956298828,
      "activations/layer23_attention_weight_min": -27.464385986328125,
      "activations/layer2_attention_weight_max": 33.24855041503906,
      "activations/layer2_attention_weight_min": -30.25440216064453,
      "activations/layer3_attention_weight_max": 92.47319793701172,
      "activations/layer3_attention_weight_min": -94.68634033203125,
      "activations/layer4_attention_weight_max": 85.9828872680664,
      "activations/layer4_attention_weight_min": -89.87071990966797,
      "activations/layer5_attention_weight_max": 62.69033432006836,
      "activations/layer5_attention_weight_min": -70.71464538574219,
      "activations/layer6_attention_weight_max": 53.68024826049805,
      "activations/layer6_attention_weight_min": -52.334434509277344,
      "activations/layer7_attention_weight_max": 70.9646987915039,
      "activations/layer7_attention_weight_min": -68.46192932128906,
      "activations/layer8_attention_weight_max": 55.39857864379883,
      "activations/layer8_attention_weight_min": -56.614341735839844,
      "activations/layer9_attention_weight_max": 48.29281234741211,
      "activations/layer9_attention_weight_min": -49.831886291503906,
      "epoch": 22.36,
      "learning_rate": 5.892424242424242e-06,
      "loss": 2.7219,
      "step": 384750
    },
    {
      "activations/layer0_attention_weight_max": 15.86197566986084,
      "activations/layer0_attention_weight_min": -14.149620056152344,
      "activations/layer10_attention_weight_max": 44.574668884277344,
      "activations/layer10_attention_weight_min": -43.16395950317383,
      "activations/layer11_attention_weight_max": 43.9119987487793,
      "activations/layer11_attention_weight_min": -40.47444152832031,
      "activations/layer12_attention_weight_max": 21.114295959472656,
      "activations/layer12_attention_weight_min": -29.77203941345215,
      "activations/layer13_attention_weight_max": 55.801456451416016,
      "activations/layer13_attention_weight_min": -31.912126541137695,
      "activations/layer14_attention_weight_max": 57.625728607177734,
      "activations/layer14_attention_weight_min": -36.98041534423828,
      "activations/layer15_attention_weight_max": 57.27442932128906,
      "activations/layer15_attention_weight_min": -36.7039909362793,
      "activations/layer16_attention_weight_max": 43.415069580078125,
      "activations/layer16_attention_weight_min": -38.225547790527344,
      "activations/layer17_attention_weight_max": 61.50177764892578,
      "activations/layer17_attention_weight_min": -56.188232421875,
      "activations/layer18_attention_weight_max": 56.82371139526367,
      "activations/layer18_attention_weight_min": -48.90505599975586,
      "activations/layer19_attention_weight_max": 32.9893798828125,
      "activations/layer19_attention_weight_min": -27.823368072509766,
      "activations/layer1_attention_weight_max": 15.791996955871582,
      "activations/layer1_attention_weight_min": -14.861907005310059,
      "activations/layer20_attention_weight_max": 32.39020919799805,
      "activations/layer20_attention_weight_min": -24.017406463623047,
      "activations/layer21_attention_weight_max": 53.614742279052734,
      "activations/layer21_attention_weight_min": -25.998550415039062,
      "activations/layer22_attention_weight_max": 34.19258499145508,
      "activations/layer22_attention_weight_min": -28.446794509887695,
      "activations/layer23_attention_weight_max": 41.67893600463867,
      "activations/layer23_attention_weight_min": -23.55879020690918,
      "activations/layer2_attention_weight_max": 34.13457107543945,
      "activations/layer2_attention_weight_min": -31.09827995300293,
      "activations/layer3_attention_weight_max": 101.71969604492188,
      "activations/layer3_attention_weight_min": -98.48638916015625,
      "activations/layer4_attention_weight_max": 92.50835418701172,
      "activations/layer4_attention_weight_min": -85.42792510986328,
      "activations/layer5_attention_weight_max": 63.795448303222656,
      "activations/layer5_attention_weight_min": -67.77520751953125,
      "activations/layer6_attention_weight_max": 48.515689849853516,
      "activations/layer6_attention_weight_min": -49.551795959472656,
      "activations/layer7_attention_weight_max": 80.55473327636719,
      "activations/layer7_attention_weight_min": -67.72944641113281,
      "activations/layer8_attention_weight_max": 52.87055969238281,
      "activations/layer8_attention_weight_min": -50.588924407958984,
      "activations/layer9_attention_weight_max": 59.486061096191406,
      "activations/layer9_attention_weight_min": -49.130985260009766,
      "epoch": 22.36,
      "learning_rate": 5.8734848484848474e-06,
      "loss": 2.7081,
      "step": 384800
    },
    {
      "activations/layer0_attention_weight_max": 16.66667938232422,
      "activations/layer0_attention_weight_min": -14.161273956298828,
      "activations/layer10_attention_weight_max": 38.92320251464844,
      "activations/layer10_attention_weight_min": -38.742340087890625,
      "activations/layer11_attention_weight_max": 36.60993957519531,
      "activations/layer11_attention_weight_min": -35.90475845336914,
      "activations/layer12_attention_weight_max": 19.941482543945312,
      "activations/layer12_attention_weight_min": -24.246484756469727,
      "activations/layer13_attention_weight_max": 47.72847366333008,
      "activations/layer13_attention_weight_min": -27.984792709350586,
      "activations/layer14_attention_weight_max": 40.744537353515625,
      "activations/layer14_attention_weight_min": -30.49842071533203,
      "activations/layer15_attention_weight_max": 39.09423065185547,
      "activations/layer15_attention_weight_min": -30.831520080566406,
      "activations/layer16_attention_weight_max": 39.3093376159668,
      "activations/layer16_attention_weight_min": -35.460262298583984,
      "activations/layer17_attention_weight_max": 64.30671691894531,
      "activations/layer17_attention_weight_min": -52.88798141479492,
      "activations/layer18_attention_weight_max": 56.81970977783203,
      "activations/layer18_attention_weight_min": -41.98189163208008,
      "activations/layer19_attention_weight_max": 29.709014892578125,
      "activations/layer19_attention_weight_min": -24.98346519470215,
      "activations/layer1_attention_weight_max": 14.171956062316895,
      "activations/layer1_attention_weight_min": -14.828629493713379,
      "activations/layer20_attention_weight_max": 26.090675354003906,
      "activations/layer20_attention_weight_min": -22.312910079956055,
      "activations/layer21_attention_weight_max": 47.55002975463867,
      "activations/layer21_attention_weight_min": -23.396759033203125,
      "activations/layer22_attention_weight_max": 33.677730560302734,
      "activations/layer22_attention_weight_min": -27.16107749938965,
      "activations/layer23_attention_weight_max": 41.58245849609375,
      "activations/layer23_attention_weight_min": -23.773061752319336,
      "activations/layer2_attention_weight_max": 34.93988037109375,
      "activations/layer2_attention_weight_min": -32.95769500732422,
      "activations/layer3_attention_weight_max": 95.53455352783203,
      "activations/layer3_attention_weight_min": -95.56172180175781,
      "activations/layer4_attention_weight_max": 84.87821960449219,
      "activations/layer4_attention_weight_min": -86.66588592529297,
      "activations/layer5_attention_weight_max": 60.375816345214844,
      "activations/layer5_attention_weight_min": -72.68435668945312,
      "activations/layer6_attention_weight_max": 48.754398345947266,
      "activations/layer6_attention_weight_min": -50.08696746826172,
      "activations/layer7_attention_weight_max": 70.97264099121094,
      "activations/layer7_attention_weight_min": -71.39434051513672,
      "activations/layer8_attention_weight_max": 49.438934326171875,
      "activations/layer8_attention_weight_min": -52.8875846862793,
      "activations/layer9_attention_weight_max": 50.035545349121094,
      "activations/layer9_attention_weight_min": -49.88798141479492,
      "epoch": 22.36,
      "learning_rate": 5.854545454545455e-06,
      "loss": 2.7043,
      "step": 384850
    },
    {
      "activations/layer0_attention_weight_max": 15.803459167480469,
      "activations/layer0_attention_weight_min": -13.846656799316406,
      "activations/layer10_attention_weight_max": 33.145057678222656,
      "activations/layer10_attention_weight_min": -34.6739616394043,
      "activations/layer11_attention_weight_max": 32.185462951660156,
      "activations/layer11_attention_weight_min": -35.45557403564453,
      "activations/layer12_attention_weight_max": 19.68762969970703,
      "activations/layer12_attention_weight_min": -26.858442306518555,
      "activations/layer13_attention_weight_max": 32.86405944824219,
      "activations/layer13_attention_weight_min": -26.13874053955078,
      "activations/layer14_attention_weight_max": 33.36517333984375,
      "activations/layer14_attention_weight_min": -29.84365463256836,
      "activations/layer15_attention_weight_max": 32.93669128417969,
      "activations/layer15_attention_weight_min": -30.62309455871582,
      "activations/layer16_attention_weight_max": 32.72169494628906,
      "activations/layer16_attention_weight_min": -32.3599967956543,
      "activations/layer17_attention_weight_max": 50.40946960449219,
      "activations/layer17_attention_weight_min": -44.77849197387695,
      "activations/layer18_attention_weight_max": 49.238914489746094,
      "activations/layer18_attention_weight_min": -41.48105239868164,
      "activations/layer19_attention_weight_max": 26.593915939331055,
      "activations/layer19_attention_weight_min": -23.26786231994629,
      "activations/layer1_attention_weight_max": 15.205456733703613,
      "activations/layer1_attention_weight_min": -14.830365180969238,
      "activations/layer20_attention_weight_max": 23.535371780395508,
      "activations/layer20_attention_weight_min": -20.994157791137695,
      "activations/layer21_attention_weight_max": 40.05024337768555,
      "activations/layer21_attention_weight_min": -23.43859100341797,
      "activations/layer22_attention_weight_max": 30.271808624267578,
      "activations/layer22_attention_weight_min": -25.303911209106445,
      "activations/layer23_attention_weight_max": 40.68359375,
      "activations/layer23_attention_weight_min": -24.738311767578125,
      "activations/layer2_attention_weight_max": 32.39057922363281,
      "activations/layer2_attention_weight_min": -32.314247131347656,
      "activations/layer3_attention_weight_max": 94.34095001220703,
      "activations/layer3_attention_weight_min": -91.56742858886719,
      "activations/layer4_attention_weight_max": 87.48497009277344,
      "activations/layer4_attention_weight_min": -88.1517105102539,
      "activations/layer5_attention_weight_max": 62.79962158203125,
      "activations/layer5_attention_weight_min": -79.0941162109375,
      "activations/layer6_attention_weight_max": 49.81641387939453,
      "activations/layer6_attention_weight_min": -50.21275329589844,
      "activations/layer7_attention_weight_max": 72.1573486328125,
      "activations/layer7_attention_weight_min": -69.73707580566406,
      "activations/layer8_attention_weight_max": 47.44950866699219,
      "activations/layer8_attention_weight_min": -49.28186798095703,
      "activations/layer9_attention_weight_max": 41.891048431396484,
      "activations/layer9_attention_weight_min": -46.44934844970703,
      "epoch": 22.36,
      "learning_rate": 5.835606060606061e-06,
      "loss": 2.692,
      "step": 384900
    },
    {
      "activations/layer0_attention_weight_max": 15.965176582336426,
      "activations/layer0_attention_weight_min": -14.12360668182373,
      "activations/layer10_attention_weight_max": 37.8726806640625,
      "activations/layer10_attention_weight_min": -36.20159912109375,
      "activations/layer11_attention_weight_max": 36.07474899291992,
      "activations/layer11_attention_weight_min": -36.040687561035156,
      "activations/layer12_attention_weight_max": 19.73725700378418,
      "activations/layer12_attention_weight_min": -23.441150665283203,
      "activations/layer13_attention_weight_max": 36.15452575683594,
      "activations/layer13_attention_weight_min": -27.57086181640625,
      "activations/layer14_attention_weight_max": 39.87183380126953,
      "activations/layer14_attention_weight_min": -32.760780334472656,
      "activations/layer15_attention_weight_max": 35.73023986816406,
      "activations/layer15_attention_weight_min": -33.18561935424805,
      "activations/layer16_attention_weight_max": 36.47359085083008,
      "activations/layer16_attention_weight_min": -34.86972427368164,
      "activations/layer17_attention_weight_max": 54.53871154785156,
      "activations/layer17_attention_weight_min": -51.026611328125,
      "activations/layer18_attention_weight_max": 52.342960357666016,
      "activations/layer18_attention_weight_min": -46.826297760009766,
      "activations/layer19_attention_weight_max": 26.90372657775879,
      "activations/layer19_attention_weight_min": -27.7080078125,
      "activations/layer1_attention_weight_max": 14.986468315124512,
      "activations/layer1_attention_weight_min": -15.051513671875,
      "activations/layer20_attention_weight_max": 26.284130096435547,
      "activations/layer20_attention_weight_min": -25.058582305908203,
      "activations/layer21_attention_weight_max": 45.33843994140625,
      "activations/layer21_attention_weight_min": -26.599491119384766,
      "activations/layer22_attention_weight_max": 32.12236404418945,
      "activations/layer22_attention_weight_min": -28.12443733215332,
      "activations/layer23_attention_weight_max": 38.720054626464844,
      "activations/layer23_attention_weight_min": -23.46390151977539,
      "activations/layer2_attention_weight_max": 30.984983444213867,
      "activations/layer2_attention_weight_min": -29.013065338134766,
      "activations/layer3_attention_weight_max": 88.14678192138672,
      "activations/layer3_attention_weight_min": -88.99597930908203,
      "activations/layer4_attention_weight_max": 85.99205780029297,
      "activations/layer4_attention_weight_min": -83.61075592041016,
      "activations/layer5_attention_weight_max": 61.259735107421875,
      "activations/layer5_attention_weight_min": -75.38545227050781,
      "activations/layer6_attention_weight_max": 51.79188537597656,
      "activations/layer6_attention_weight_min": -49.82413101196289,
      "activations/layer7_attention_weight_max": 70.86051177978516,
      "activations/layer7_attention_weight_min": -67.6496810913086,
      "activations/layer8_attention_weight_max": 47.8618049621582,
      "activations/layer8_attention_weight_min": -51.485084533691406,
      "activations/layer9_attention_weight_max": 47.03227233886719,
      "activations/layer9_attention_weight_min": -46.69169235229492,
      "epoch": 22.37,
      "learning_rate": 5.816666666666666e-06,
      "loss": 2.7114,
      "step": 384950
    },
    {
      "activations/layer0_attention_weight_max": 16.58305549621582,
      "activations/layer0_attention_weight_min": -13.686732292175293,
      "activations/layer10_attention_weight_max": 35.97566604614258,
      "activations/layer10_attention_weight_min": -38.579505920410156,
      "activations/layer11_attention_weight_max": 34.42402648925781,
      "activations/layer11_attention_weight_min": -37.256858825683594,
      "activations/layer12_attention_weight_max": 19.697294235229492,
      "activations/layer12_attention_weight_min": -26.992637634277344,
      "activations/layer13_attention_weight_max": 34.87592697143555,
      "activations/layer13_attention_weight_min": -29.49909019470215,
      "activations/layer14_attention_weight_max": 36.54906463623047,
      "activations/layer14_attention_weight_min": -30.995866775512695,
      "activations/layer15_attention_weight_max": 36.03386688232422,
      "activations/layer15_attention_weight_min": -30.86825180053711,
      "activations/layer16_attention_weight_max": 35.4038200378418,
      "activations/layer16_attention_weight_min": -33.14714050292969,
      "activations/layer17_attention_weight_max": 48.68338394165039,
      "activations/layer17_attention_weight_min": -46.60212326049805,
      "activations/layer18_attention_weight_max": 48.04880142211914,
      "activations/layer18_attention_weight_min": -41.18545150756836,
      "activations/layer19_attention_weight_max": 24.456758499145508,
      "activations/layer19_attention_weight_min": -24.84743881225586,
      "activations/layer1_attention_weight_max": 14.927665710449219,
      "activations/layer1_attention_weight_min": -15.099343299865723,
      "activations/layer20_attention_weight_max": 25.147048950195312,
      "activations/layer20_attention_weight_min": -23.067167282104492,
      "activations/layer21_attention_weight_max": 35.68132781982422,
      "activations/layer21_attention_weight_min": -22.676738739013672,
      "activations/layer22_attention_weight_max": 29.18769073486328,
      "activations/layer22_attention_weight_min": -25.18745231628418,
      "activations/layer23_attention_weight_max": 35.30348205566406,
      "activations/layer23_attention_weight_min": -23.083036422729492,
      "activations/layer2_attention_weight_max": 32.11186218261719,
      "activations/layer2_attention_weight_min": -30.995908737182617,
      "activations/layer3_attention_weight_max": 89.21646881103516,
      "activations/layer3_attention_weight_min": -93.52743530273438,
      "activations/layer4_attention_weight_max": 84.0994873046875,
      "activations/layer4_attention_weight_min": -84.84220123291016,
      "activations/layer5_attention_weight_max": 64.35721588134766,
      "activations/layer5_attention_weight_min": -72.00391387939453,
      "activations/layer6_attention_weight_max": 50.16486358642578,
      "activations/layer6_attention_weight_min": -51.99446487426758,
      "activations/layer7_attention_weight_max": 74.4134750366211,
      "activations/layer7_attention_weight_min": -68.05694580078125,
      "activations/layer8_attention_weight_max": 48.396915435791016,
      "activations/layer8_attention_weight_min": -51.3298225402832,
      "activations/layer9_attention_weight_max": 45.69566345214844,
      "activations/layer9_attention_weight_min": -48.05377197265625,
      "epoch": 22.37,
      "learning_rate": 5.797727272727272e-06,
      "loss": 2.6906,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4968,
      "eval_samples_per_second": 505.367,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4968,
      "eval_openwebtext_samples_per_second": 505.367,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9918,
      "eval_wikitext_samples_per_second": 228.943,
      "step": 385000
    },
    {
      "epoch": 22.37,
      "eval_lambada_loss": 2.41015625,
      "eval_lambada_ppl": 11.135700962413912,
      "eval_lambada_runtime": 9.5934,
      "eval_lambada_samples_per_second": 507.537,
      "step": 385000
    },
    {
      "activations/layer0_attention_weight_max": 15.510115623474121,
      "activations/layer0_attention_weight_min": -14.06498908996582,
      "activations/layer10_attention_weight_max": 32.8026123046875,
      "activations/layer10_attention_weight_min": -34.83907699584961,
      "activations/layer11_attention_weight_max": 31.829181671142578,
      "activations/layer11_attention_weight_min": -35.20159912109375,
      "activations/layer12_attention_weight_max": 19.387821197509766,
      "activations/layer12_attention_weight_min": -25.995267868041992,
      "activations/layer13_attention_weight_max": 36.730438232421875,
      "activations/layer13_attention_weight_min": -26.265974044799805,
      "activations/layer14_attention_weight_max": 34.4683837890625,
      "activations/layer14_attention_weight_min": -28.980266571044922,
      "activations/layer15_attention_weight_max": 33.855796813964844,
      "activations/layer15_attention_weight_min": -30.189558029174805,
      "activations/layer16_attention_weight_max": 35.36955642700195,
      "activations/layer16_attention_weight_min": -34.11113739013672,
      "activations/layer17_attention_weight_max": 52.08457946777344,
      "activations/layer17_attention_weight_min": -46.84748840332031,
      "activations/layer18_attention_weight_max": 46.36017990112305,
      "activations/layer18_attention_weight_min": -39.671165466308594,
      "activations/layer19_attention_weight_max": 26.98141860961914,
      "activations/layer19_attention_weight_min": -23.155115127563477,
      "activations/layer1_attention_weight_max": 15.03443717956543,
      "activations/layer1_attention_weight_min": -16.133888244628906,
      "activations/layer20_attention_weight_max": 28.201885223388672,
      "activations/layer20_attention_weight_min": -23.151731491088867,
      "activations/layer21_attention_weight_max": 36.382835388183594,
      "activations/layer21_attention_weight_min": -23.46341323852539,
      "activations/layer22_attention_weight_max": 34.063323974609375,
      "activations/layer22_attention_weight_min": -27.053382873535156,
      "activations/layer23_attention_weight_max": 38.6357421875,
      "activations/layer23_attention_weight_min": -25.243419647216797,
      "activations/layer2_attention_weight_max": 32.284454345703125,
      "activations/layer2_attention_weight_min": -30.38909149169922,
      "activations/layer3_attention_weight_max": 88.78533172607422,
      "activations/layer3_attention_weight_min": -89.35884094238281,
      "activations/layer4_attention_weight_max": 83.30879974365234,
      "activations/layer4_attention_weight_min": -85.41938781738281,
      "activations/layer5_attention_weight_max": 60.39434051513672,
      "activations/layer5_attention_weight_min": -74.73089599609375,
      "activations/layer6_attention_weight_max": 47.53482437133789,
      "activations/layer6_attention_weight_min": -51.2823600769043,
      "activations/layer7_attention_weight_max": 70.2652816772461,
      "activations/layer7_attention_weight_min": -68.5378189086914,
      "activations/layer8_attention_weight_max": 44.34157180786133,
      "activations/layer8_attention_weight_min": -48.45636749267578,
      "activations/layer9_attention_weight_max": 42.48293685913086,
      "activations/layer9_attention_weight_min": -48.1976318359375,
      "epoch": 22.37,
      "learning_rate": 5.778787878787879e-06,
      "loss": 2.6959,
      "step": 385050
    },
    {
      "activations/layer0_attention_weight_max": 15.445442199707031,
      "activations/layer0_attention_weight_min": -14.277682304382324,
      "activations/layer10_attention_weight_max": 37.691993713378906,
      "activations/layer10_attention_weight_min": -37.132328033447266,
      "activations/layer11_attention_weight_max": 35.21662139892578,
      "activations/layer11_attention_weight_min": -34.78131866455078,
      "activations/layer12_attention_weight_max": 18.795875549316406,
      "activations/layer12_attention_weight_min": -29.797439575195312,
      "activations/layer13_attention_weight_max": 38.30592346191406,
      "activations/layer13_attention_weight_min": -29.16271209716797,
      "activations/layer14_attention_weight_max": 36.890533447265625,
      "activations/layer14_attention_weight_min": -31.03457260131836,
      "activations/layer15_attention_weight_max": 35.099246978759766,
      "activations/layer15_attention_weight_min": -32.56288146972656,
      "activations/layer16_attention_weight_max": 36.82517623901367,
      "activations/layer16_attention_weight_min": -33.09337615966797,
      "activations/layer17_attention_weight_max": 57.73992156982422,
      "activations/layer17_attention_weight_min": -45.5970458984375,
      "activations/layer18_attention_weight_max": 50.28276824951172,
      "activations/layer18_attention_weight_min": -39.84468460083008,
      "activations/layer19_attention_weight_max": 23.80537223815918,
      "activations/layer19_attention_weight_min": -23.017568588256836,
      "activations/layer1_attention_weight_max": 15.124677658081055,
      "activations/layer1_attention_weight_min": -14.594908714294434,
      "activations/layer20_attention_weight_max": 22.733760833740234,
      "activations/layer20_attention_weight_min": -22.414592742919922,
      "activations/layer21_attention_weight_max": 34.59523391723633,
      "activations/layer21_attention_weight_min": -20.43332862854004,
      "activations/layer22_attention_weight_max": 28.78988265991211,
      "activations/layer22_attention_weight_min": -23.900781631469727,
      "activations/layer23_attention_weight_max": 37.05535888671875,
      "activations/layer23_attention_weight_min": -23.33972930908203,
      "activations/layer2_attention_weight_max": 33.31389617919922,
      "activations/layer2_attention_weight_min": -31.18100357055664,
      "activations/layer3_attention_weight_max": 91.54853057861328,
      "activations/layer3_attention_weight_min": -91.41608428955078,
      "activations/layer4_attention_weight_max": 86.92070007324219,
      "activations/layer4_attention_weight_min": -87.09746551513672,
      "activations/layer5_attention_weight_max": 62.1961669921875,
      "activations/layer5_attention_weight_min": -71.0133285522461,
      "activations/layer6_attention_weight_max": 53.26367950439453,
      "activations/layer6_attention_weight_min": -53.30107116699219,
      "activations/layer7_attention_weight_max": 74.68864440917969,
      "activations/layer7_attention_weight_min": -69.82005310058594,
      "activations/layer8_attention_weight_max": 51.44940948486328,
      "activations/layer8_attention_weight_min": -54.701499938964844,
      "activations/layer9_attention_weight_max": 55.99956130981445,
      "activations/layer9_attention_weight_min": -49.05029296875,
      "epoch": 22.38,
      "learning_rate": 5.759848484848484e-06,
      "loss": 2.6989,
      "step": 385100
    },
    {
      "activations/layer0_attention_weight_max": 15.754437446594238,
      "activations/layer0_attention_weight_min": -14.490530967712402,
      "activations/layer10_attention_weight_max": 35.73554229736328,
      "activations/layer10_attention_weight_min": -36.48581314086914,
      "activations/layer11_attention_weight_max": 36.27130126953125,
      "activations/layer11_attention_weight_min": -38.060176849365234,
      "activations/layer12_attention_weight_max": 20.187976837158203,
      "activations/layer12_attention_weight_min": -25.011573791503906,
      "activations/layer13_attention_weight_max": 31.5653133392334,
      "activations/layer13_attention_weight_min": -27.275888442993164,
      "activations/layer14_attention_weight_max": 35.067527770996094,
      "activations/layer14_attention_weight_min": -31.061906814575195,
      "activations/layer15_attention_weight_max": 31.852153778076172,
      "activations/layer15_attention_weight_min": -31.99900245666504,
      "activations/layer16_attention_weight_max": 36.55979537963867,
      "activations/layer16_attention_weight_min": -34.67505645751953,
      "activations/layer17_attention_weight_max": 55.23673629760742,
      "activations/layer17_attention_weight_min": -48.842926025390625,
      "activations/layer18_attention_weight_max": 48.276302337646484,
      "activations/layer18_attention_weight_min": -44.4244384765625,
      "activations/layer19_attention_weight_max": 24.98287010192871,
      "activations/layer19_attention_weight_min": -26.12784767150879,
      "activations/layer1_attention_weight_max": 14.282682418823242,
      "activations/layer1_attention_weight_min": -14.112723350524902,
      "activations/layer20_attention_weight_max": 24.535886764526367,
      "activations/layer20_attention_weight_min": -23.08297348022461,
      "activations/layer21_attention_weight_max": 35.5941047668457,
      "activations/layer21_attention_weight_min": -22.098939895629883,
      "activations/layer22_attention_weight_max": 30.43417739868164,
      "activations/layer22_attention_weight_min": -26.42702293395996,
      "activations/layer23_attention_weight_max": 35.14839553833008,
      "activations/layer23_attention_weight_min": -23.87195587158203,
      "activations/layer2_attention_weight_max": 32.871376037597656,
      "activations/layer2_attention_weight_min": -31.327472686767578,
      "activations/layer3_attention_weight_max": 89.774169921875,
      "activations/layer3_attention_weight_min": -92.72200775146484,
      "activations/layer4_attention_weight_max": 87.58366394042969,
      "activations/layer4_attention_weight_min": -84.39675903320312,
      "activations/layer5_attention_weight_max": 61.43941879272461,
      "activations/layer5_attention_weight_min": -72.94876098632812,
      "activations/layer6_attention_weight_max": 52.50798416137695,
      "activations/layer6_attention_weight_min": -53.53090286254883,
      "activations/layer7_attention_weight_max": 70.4747543334961,
      "activations/layer7_attention_weight_min": -69.41793823242188,
      "activations/layer8_attention_weight_max": 50.76078414916992,
      "activations/layer8_attention_weight_min": -52.22294998168945,
      "activations/layer9_attention_weight_max": 45.67601013183594,
      "activations/layer9_attention_weight_min": -49.75077438354492,
      "epoch": 22.38,
      "learning_rate": 5.74090909090909e-06,
      "loss": 2.7054,
      "step": 385150
    },
    {
      "activations/layer0_attention_weight_max": 15.340278625488281,
      "activations/layer0_attention_weight_min": -14.20824909210205,
      "activations/layer10_attention_weight_max": 37.950950622558594,
      "activations/layer10_attention_weight_min": -36.219566345214844,
      "activations/layer11_attention_weight_max": 34.872764587402344,
      "activations/layer11_attention_weight_min": -35.94493865966797,
      "activations/layer12_attention_weight_max": 19.17967414855957,
      "activations/layer12_attention_weight_min": -24.545108795166016,
      "activations/layer13_attention_weight_max": 43.737613677978516,
      "activations/layer13_attention_weight_min": -30.834402084350586,
      "activations/layer14_attention_weight_max": 46.095157623291016,
      "activations/layer14_attention_weight_min": -31.219383239746094,
      "activations/layer15_attention_weight_max": 48.87439727783203,
      "activations/layer15_attention_weight_min": -31.900768280029297,
      "activations/layer16_attention_weight_max": 36.099853515625,
      "activations/layer16_attention_weight_min": -34.57122802734375,
      "activations/layer17_attention_weight_max": 55.972896575927734,
      "activations/layer17_attention_weight_min": -48.02922439575195,
      "activations/layer18_attention_weight_max": 52.37064743041992,
      "activations/layer18_attention_weight_min": -43.74382019042969,
      "activations/layer19_attention_weight_max": 29.643909454345703,
      "activations/layer19_attention_weight_min": -24.753780364990234,
      "activations/layer1_attention_weight_max": 14.863381385803223,
      "activations/layer1_attention_weight_min": -15.622567176818848,
      "activations/layer20_attention_weight_max": 28.191993713378906,
      "activations/layer20_attention_weight_min": -21.243820190429688,
      "activations/layer21_attention_weight_max": 39.00873565673828,
      "activations/layer21_attention_weight_min": -23.665056228637695,
      "activations/layer22_attention_weight_max": 31.16114616394043,
      "activations/layer22_attention_weight_min": -24.926612854003906,
      "activations/layer23_attention_weight_max": 36.33769226074219,
      "activations/layer23_attention_weight_min": -23.954303741455078,
      "activations/layer2_attention_weight_max": 31.988981246948242,
      "activations/layer2_attention_weight_min": -30.107542037963867,
      "activations/layer3_attention_weight_max": 89.81245422363281,
      "activations/layer3_attention_weight_min": -89.62264251708984,
      "activations/layer4_attention_weight_max": 85.39918518066406,
      "activations/layer4_attention_weight_min": -85.98737335205078,
      "activations/layer5_attention_weight_max": 63.46871566772461,
      "activations/layer5_attention_weight_min": -76.28567504882812,
      "activations/layer6_attention_weight_max": 50.57076644897461,
      "activations/layer6_attention_weight_min": -50.395591735839844,
      "activations/layer7_attention_weight_max": 76.46161651611328,
      "activations/layer7_attention_weight_min": -70.5162582397461,
      "activations/layer8_attention_weight_max": 50.361289978027344,
      "activations/layer8_attention_weight_min": -52.49612808227539,
      "activations/layer9_attention_weight_max": 46.89889144897461,
      "activations/layer9_attention_weight_min": -47.51676559448242,
      "epoch": 22.38,
      "learning_rate": 5.721969696969696e-06,
      "loss": 2.6974,
      "step": 385200
    },
    {
      "activations/layer0_attention_weight_max": 15.787870407104492,
      "activations/layer0_attention_weight_min": -14.14069652557373,
      "activations/layer10_attention_weight_max": 37.418827056884766,
      "activations/layer10_attention_weight_min": -36.64921569824219,
      "activations/layer11_attention_weight_max": 35.259124755859375,
      "activations/layer11_attention_weight_min": -38.693477630615234,
      "activations/layer12_attention_weight_max": 20.005950927734375,
      "activations/layer12_attention_weight_min": -25.868379592895508,
      "activations/layer13_attention_weight_max": 37.67757797241211,
      "activations/layer13_attention_weight_min": -28.6160831451416,
      "activations/layer14_attention_weight_max": 37.72858810424805,
      "activations/layer14_attention_weight_min": -30.982378005981445,
      "activations/layer15_attention_weight_max": 38.138065338134766,
      "activations/layer15_attention_weight_min": -31.316057205200195,
      "activations/layer16_attention_weight_max": 37.96918869018555,
      "activations/layer16_attention_weight_min": -34.6240234375,
      "activations/layer17_attention_weight_max": 56.75028991699219,
      "activations/layer17_attention_weight_min": -50.75713348388672,
      "activations/layer18_attention_weight_max": 54.92463684082031,
      "activations/layer18_attention_weight_min": -43.47434997558594,
      "activations/layer19_attention_weight_max": 28.85666847229004,
      "activations/layer19_attention_weight_min": -22.43906021118164,
      "activations/layer1_attention_weight_max": 14.715940475463867,
      "activations/layer1_attention_weight_min": -14.95529842376709,
      "activations/layer20_attention_weight_max": 23.349258422851562,
      "activations/layer20_attention_weight_min": -26.011777877807617,
      "activations/layer21_attention_weight_max": 34.76677322387695,
      "activations/layer21_attention_weight_min": -22.103046417236328,
      "activations/layer22_attention_weight_max": 31.339488983154297,
      "activations/layer22_attention_weight_min": -25.9532470703125,
      "activations/layer23_attention_weight_max": 38.13460922241211,
      "activations/layer23_attention_weight_min": -22.98775291442871,
      "activations/layer2_attention_weight_max": 31.32828140258789,
      "activations/layer2_attention_weight_min": -29.202396392822266,
      "activations/layer3_attention_weight_max": 93.27935028076172,
      "activations/layer3_attention_weight_min": -89.75695037841797,
      "activations/layer4_attention_weight_max": 89.14159393310547,
      "activations/layer4_attention_weight_min": -84.30918884277344,
      "activations/layer5_attention_weight_max": 68.53397369384766,
      "activations/layer5_attention_weight_min": -71.8572998046875,
      "activations/layer6_attention_weight_max": 53.780052185058594,
      "activations/layer6_attention_weight_min": -52.640323638916016,
      "activations/layer7_attention_weight_max": 74.61692810058594,
      "activations/layer7_attention_weight_min": -68.6029281616211,
      "activations/layer8_attention_weight_max": 49.28330612182617,
      "activations/layer8_attention_weight_min": -52.96816635131836,
      "activations/layer9_attention_weight_max": 46.624874114990234,
      "activations/layer9_attention_weight_min": -48.15946578979492,
      "epoch": 22.39,
      "learning_rate": 5.703030303030303e-06,
      "loss": 2.6956,
      "step": 385250
    },
    {
      "activations/layer0_attention_weight_max": 15.85869312286377,
      "activations/layer0_attention_weight_min": -14.395612716674805,
      "activations/layer10_attention_weight_max": 35.21234130859375,
      "activations/layer10_attention_weight_min": -35.62757873535156,
      "activations/layer11_attention_weight_max": 34.008087158203125,
      "activations/layer11_attention_weight_min": -34.9167366027832,
      "activations/layer12_attention_weight_max": 19.231901168823242,
      "activations/layer12_attention_weight_min": -26.41743278503418,
      "activations/layer13_attention_weight_max": 37.34784698486328,
      "activations/layer13_attention_weight_min": -30.12582015991211,
      "activations/layer14_attention_weight_max": 37.967247009277344,
      "activations/layer14_attention_weight_min": -32.74256896972656,
      "activations/layer15_attention_weight_max": 34.02285385131836,
      "activations/layer15_attention_weight_min": -31.65226936340332,
      "activations/layer16_attention_weight_max": 34.0603141784668,
      "activations/layer16_attention_weight_min": -34.78617477416992,
      "activations/layer17_attention_weight_max": 52.06950759887695,
      "activations/layer17_attention_weight_min": -45.64449691772461,
      "activations/layer18_attention_weight_max": 47.63128662109375,
      "activations/layer18_attention_weight_min": -42.676361083984375,
      "activations/layer19_attention_weight_max": 25.086763381958008,
      "activations/layer19_attention_weight_min": -23.551816940307617,
      "activations/layer1_attention_weight_max": 14.957013130187988,
      "activations/layer1_attention_weight_min": -15.351492881774902,
      "activations/layer20_attention_weight_max": 23.841693878173828,
      "activations/layer20_attention_weight_min": -23.82541847229004,
      "activations/layer21_attention_weight_max": 43.6411247253418,
      "activations/layer21_attention_weight_min": -25.517101287841797,
      "activations/layer22_attention_weight_max": 27.78497886657715,
      "activations/layer22_attention_weight_min": -26.857555389404297,
      "activations/layer23_attention_weight_max": 40.912784576416016,
      "activations/layer23_attention_weight_min": -26.049449920654297,
      "activations/layer2_attention_weight_max": 32.23334503173828,
      "activations/layer2_attention_weight_min": -30.60251235961914,
      "activations/layer3_attention_weight_max": 90.70820617675781,
      "activations/layer3_attention_weight_min": -90.6654281616211,
      "activations/layer4_attention_weight_max": 86.6318588256836,
      "activations/layer4_attention_weight_min": -85.26314544677734,
      "activations/layer5_attention_weight_max": 62.378684997558594,
      "activations/layer5_attention_weight_min": -69.1895751953125,
      "activations/layer6_attention_weight_max": 51.74496078491211,
      "activations/layer6_attention_weight_min": -54.052127838134766,
      "activations/layer7_attention_weight_max": 71.92906188964844,
      "activations/layer7_attention_weight_min": -70.52838897705078,
      "activations/layer8_attention_weight_max": 48.51322937011719,
      "activations/layer8_attention_weight_min": -50.15273666381836,
      "activations/layer9_attention_weight_max": 46.579166412353516,
      "activations/layer9_attention_weight_min": -47.205238342285156,
      "epoch": 22.39,
      "learning_rate": 5.684090909090909e-06,
      "loss": 2.678,
      "step": 385300
    },
    {
      "activations/layer0_attention_weight_max": 15.591333389282227,
      "activations/layer0_attention_weight_min": -14.173966407775879,
      "activations/layer10_attention_weight_max": 37.76693344116211,
      "activations/layer10_attention_weight_min": -36.84251022338867,
      "activations/layer11_attention_weight_max": 34.718963623046875,
      "activations/layer11_attention_weight_min": -37.8514289855957,
      "activations/layer12_attention_weight_max": 20.948272705078125,
      "activations/layer12_attention_weight_min": -28.151546478271484,
      "activations/layer13_attention_weight_max": 37.47389602661133,
      "activations/layer13_attention_weight_min": -27.59366226196289,
      "activations/layer14_attention_weight_max": 37.56193542480469,
      "activations/layer14_attention_weight_min": -31.439451217651367,
      "activations/layer15_attention_weight_max": 33.927581787109375,
      "activations/layer15_attention_weight_min": -31.321155548095703,
      "activations/layer16_attention_weight_max": 36.82958984375,
      "activations/layer16_attention_weight_min": -35.35567855834961,
      "activations/layer17_attention_weight_max": 54.53068923950195,
      "activations/layer17_attention_weight_min": -46.17824935913086,
      "activations/layer18_attention_weight_max": 50.76362228393555,
      "activations/layer18_attention_weight_min": -41.307735443115234,
      "activations/layer19_attention_weight_max": 28.024948120117188,
      "activations/layer19_attention_weight_min": -22.87929344177246,
      "activations/layer1_attention_weight_max": 16.156909942626953,
      "activations/layer1_attention_weight_min": -15.405282974243164,
      "activations/layer20_attention_weight_max": 25.533329010009766,
      "activations/layer20_attention_weight_min": -20.32845115661621,
      "activations/layer21_attention_weight_max": 34.73469161987305,
      "activations/layer21_attention_weight_min": -19.757583618164062,
      "activations/layer22_attention_weight_max": 31.120771408081055,
      "activations/layer22_attention_weight_min": -24.71821403503418,
      "activations/layer23_attention_weight_max": 38.4110221862793,
      "activations/layer23_attention_weight_min": -23.667922973632812,
      "activations/layer2_attention_weight_max": 31.87179946899414,
      "activations/layer2_attention_weight_min": -31.112865447998047,
      "activations/layer3_attention_weight_max": 88.9405746459961,
      "activations/layer3_attention_weight_min": -91.47659301757812,
      "activations/layer4_attention_weight_max": 87.61858367919922,
      "activations/layer4_attention_weight_min": -83.71349334716797,
      "activations/layer5_attention_weight_max": 62.40060043334961,
      "activations/layer5_attention_weight_min": -72.31915283203125,
      "activations/layer6_attention_weight_max": 51.596519470214844,
      "activations/layer6_attention_weight_min": -52.81079864501953,
      "activations/layer7_attention_weight_max": 74.48577880859375,
      "activations/layer7_attention_weight_min": -69.91205596923828,
      "activations/layer8_attention_weight_max": 50.28942108154297,
      "activations/layer8_attention_weight_min": -56.2791633605957,
      "activations/layer9_attention_weight_max": 45.43442916870117,
      "activations/layer9_attention_weight_min": -49.84812545776367,
      "epoch": 22.39,
      "learning_rate": 5.665530303030303e-06,
      "loss": 2.7103,
      "step": 385350
    },
    {
      "activations/layer0_attention_weight_max": 17.113733291625977,
      "activations/layer0_attention_weight_min": -14.1133394241333,
      "activations/layer10_attention_weight_max": 36.91644287109375,
      "activations/layer10_attention_weight_min": -35.92331314086914,
      "activations/layer11_attention_weight_max": 35.173316955566406,
      "activations/layer11_attention_weight_min": -36.77176284790039,
      "activations/layer12_attention_weight_max": 19.885623931884766,
      "activations/layer12_attention_weight_min": -28.498395919799805,
      "activations/layer13_attention_weight_max": 33.10786056518555,
      "activations/layer13_attention_weight_min": -27.929506301879883,
      "activations/layer14_attention_weight_max": 37.319828033447266,
      "activations/layer14_attention_weight_min": -32.08271408081055,
      "activations/layer15_attention_weight_max": 38.515995025634766,
      "activations/layer15_attention_weight_min": -33.270782470703125,
      "activations/layer16_attention_weight_max": 36.41497802734375,
      "activations/layer16_attention_weight_min": -34.41740798950195,
      "activations/layer17_attention_weight_max": 54.74462127685547,
      "activations/layer17_attention_weight_min": -50.123924255371094,
      "activations/layer18_attention_weight_max": 50.89595031738281,
      "activations/layer18_attention_weight_min": -43.06791687011719,
      "activations/layer19_attention_weight_max": 25.168943405151367,
      "activations/layer19_attention_weight_min": -24.440231323242188,
      "activations/layer1_attention_weight_max": 15.579644203186035,
      "activations/layer1_attention_weight_min": -14.836989402770996,
      "activations/layer20_attention_weight_max": 24.13426971435547,
      "activations/layer20_attention_weight_min": -23.9062557220459,
      "activations/layer21_attention_weight_max": 35.90476608276367,
      "activations/layer21_attention_weight_min": -22.27252197265625,
      "activations/layer22_attention_weight_max": 29.884979248046875,
      "activations/layer22_attention_weight_min": -26.600021362304688,
      "activations/layer23_attention_weight_max": 39.236907958984375,
      "activations/layer23_attention_weight_min": -27.346481323242188,
      "activations/layer2_attention_weight_max": 32.59376525878906,
      "activations/layer2_attention_weight_min": -29.828998565673828,
      "activations/layer3_attention_weight_max": 89.45559692382812,
      "activations/layer3_attention_weight_min": -89.932373046875,
      "activations/layer4_attention_weight_max": 84.86412048339844,
      "activations/layer4_attention_weight_min": -89.50786590576172,
      "activations/layer5_attention_weight_max": 58.48838806152344,
      "activations/layer5_attention_weight_min": -69.18216705322266,
      "activations/layer6_attention_weight_max": 51.04035568237305,
      "activations/layer6_attention_weight_min": -52.09687423706055,
      "activations/layer7_attention_weight_max": 69.4144287109375,
      "activations/layer7_attention_weight_min": -67.815185546875,
      "activations/layer8_attention_weight_max": 50.641544342041016,
      "activations/layer8_attention_weight_min": -51.55936050415039,
      "activations/layer9_attention_weight_max": 46.22938537597656,
      "activations/layer9_attention_weight_min": -46.447994232177734,
      "epoch": 22.39,
      "learning_rate": 5.646590909090909e-06,
      "loss": 2.7071,
      "step": 385400
    },
    {
      "activations/layer0_attention_weight_max": 16.70932960510254,
      "activations/layer0_attention_weight_min": -15.943175315856934,
      "activations/layer10_attention_weight_max": 32.2567024230957,
      "activations/layer10_attention_weight_min": -34.66285705566406,
      "activations/layer11_attention_weight_max": 31.275325775146484,
      "activations/layer11_attention_weight_min": -34.44120788574219,
      "activations/layer12_attention_weight_max": 18.835479736328125,
      "activations/layer12_attention_weight_min": -25.097501754760742,
      "activations/layer13_attention_weight_max": 32.862430572509766,
      "activations/layer13_attention_weight_min": -26.70524787902832,
      "activations/layer14_attention_weight_max": 34.699520111083984,
      "activations/layer14_attention_weight_min": -30.94705581665039,
      "activations/layer15_attention_weight_max": 31.885404586791992,
      "activations/layer15_attention_weight_min": -31.12357521057129,
      "activations/layer16_attention_weight_max": 33.841346740722656,
      "activations/layer16_attention_weight_min": -33.07748794555664,
      "activations/layer17_attention_weight_max": 50.86577606201172,
      "activations/layer17_attention_weight_min": -45.1739387512207,
      "activations/layer18_attention_weight_max": 45.22096252441406,
      "activations/layer18_attention_weight_min": -40.34488296508789,
      "activations/layer19_attention_weight_max": 22.549701690673828,
      "activations/layer19_attention_weight_min": -23.577695846557617,
      "activations/layer1_attention_weight_max": 15.844138145446777,
      "activations/layer1_attention_weight_min": -15.202881813049316,
      "activations/layer20_attention_weight_max": 22.085058212280273,
      "activations/layer20_attention_weight_min": -22.182878494262695,
      "activations/layer21_attention_weight_max": 32.34906768798828,
      "activations/layer21_attention_weight_min": -24.001115798950195,
      "activations/layer22_attention_weight_max": 27.85416603088379,
      "activations/layer22_attention_weight_min": -25.500932693481445,
      "activations/layer23_attention_weight_max": 30.758848190307617,
      "activations/layer23_attention_weight_min": -23.07497215270996,
      "activations/layer2_attention_weight_max": 30.136112213134766,
      "activations/layer2_attention_weight_min": -29.415470123291016,
      "activations/layer3_attention_weight_max": 88.04532623291016,
      "activations/layer3_attention_weight_min": -83.61073303222656,
      "activations/layer4_attention_weight_max": 80.9919662475586,
      "activations/layer4_attention_weight_min": -83.12115478515625,
      "activations/layer5_attention_weight_max": 58.67506408691406,
      "activations/layer5_attention_weight_min": -66.92350769042969,
      "activations/layer6_attention_weight_max": 48.200050354003906,
      "activations/layer6_attention_weight_min": -50.58445739746094,
      "activations/layer7_attention_weight_max": 64.81819152832031,
      "activations/layer7_attention_weight_min": -66.40058135986328,
      "activations/layer8_attention_weight_max": 43.78140640258789,
      "activations/layer8_attention_weight_min": -48.69165802001953,
      "activations/layer9_attention_weight_max": 45.12428283691406,
      "activations/layer9_attention_weight_min": -44.8691520690918,
      "epoch": 22.4,
      "learning_rate": 5.627651515151515e-06,
      "loss": 2.7226,
      "step": 385450
    },
    {
      "activations/layer0_attention_weight_max": 16.14402198791504,
      "activations/layer0_attention_weight_min": -14.241907119750977,
      "activations/layer10_attention_weight_max": 37.89478302001953,
      "activations/layer10_attention_weight_min": -37.74791717529297,
      "activations/layer11_attention_weight_max": 34.7313346862793,
      "activations/layer11_attention_weight_min": -34.05907440185547,
      "activations/layer12_attention_weight_max": 19.88252067565918,
      "activations/layer12_attention_weight_min": -25.24089241027832,
      "activations/layer13_attention_weight_max": 34.35854721069336,
      "activations/layer13_attention_weight_min": -26.66027069091797,
      "activations/layer14_attention_weight_max": 36.26962661743164,
      "activations/layer14_attention_weight_min": -29.966243743896484,
      "activations/layer15_attention_weight_max": 34.89623260498047,
      "activations/layer15_attention_weight_min": -30.783658981323242,
      "activations/layer16_attention_weight_max": 37.09664535522461,
      "activations/layer16_attention_weight_min": -33.414939880371094,
      "activations/layer17_attention_weight_max": 52.505550384521484,
      "activations/layer17_attention_weight_min": -44.65119171142578,
      "activations/layer18_attention_weight_max": 49.27408218383789,
      "activations/layer18_attention_weight_min": -40.35335922241211,
      "activations/layer19_attention_weight_max": 25.140779495239258,
      "activations/layer19_attention_weight_min": -23.628000259399414,
      "activations/layer1_attention_weight_max": 14.835192680358887,
      "activations/layer1_attention_weight_min": -14.750287055969238,
      "activations/layer20_attention_weight_max": 22.67699432373047,
      "activations/layer20_attention_weight_min": -23.05137825012207,
      "activations/layer21_attention_weight_max": 32.937744140625,
      "activations/layer21_attention_weight_min": -20.017650604248047,
      "activations/layer22_attention_weight_max": 30.016714096069336,
      "activations/layer22_attention_weight_min": -24.418380737304688,
      "activations/layer23_attention_weight_max": 38.314151763916016,
      "activations/layer23_attention_weight_min": -24.958023071289062,
      "activations/layer2_attention_weight_max": 33.945068359375,
      "activations/layer2_attention_weight_min": -30.712554931640625,
      "activations/layer3_attention_weight_max": 94.01390075683594,
      "activations/layer3_attention_weight_min": -91.05216979980469,
      "activations/layer4_attention_weight_max": 89.8913803100586,
      "activations/layer4_attention_weight_min": -87.19171142578125,
      "activations/layer5_attention_weight_max": 64.76252746582031,
      "activations/layer5_attention_weight_min": -70.72488403320312,
      "activations/layer6_attention_weight_max": 50.61311340332031,
      "activations/layer6_attention_weight_min": -51.921024322509766,
      "activations/layer7_attention_weight_max": 70.533447265625,
      "activations/layer7_attention_weight_min": -70.78768157958984,
      "activations/layer8_attention_weight_max": 49.462974548339844,
      "activations/layer8_attention_weight_min": -51.18605041503906,
      "activations/layer9_attention_weight_max": 48.25617980957031,
      "activations/layer9_attention_weight_min": -47.13793182373047,
      "epoch": 22.4,
      "learning_rate": 5.608712121212121e-06,
      "loss": 2.7152,
      "step": 385500
    },
    {
      "activations/layer0_attention_weight_max": 16.102550506591797,
      "activations/layer0_attention_weight_min": -14.248624801635742,
      "activations/layer10_attention_weight_max": 37.92302322387695,
      "activations/layer10_attention_weight_min": -36.82521438598633,
      "activations/layer11_attention_weight_max": 34.28297424316406,
      "activations/layer11_attention_weight_min": -36.29515838623047,
      "activations/layer12_attention_weight_max": 19.661474227905273,
      "activations/layer12_attention_weight_min": -22.847745895385742,
      "activations/layer13_attention_weight_max": 36.47010040283203,
      "activations/layer13_attention_weight_min": -28.011064529418945,
      "activations/layer14_attention_weight_max": 36.45205307006836,
      "activations/layer14_attention_weight_min": -32.274749755859375,
      "activations/layer15_attention_weight_max": 33.252323150634766,
      "activations/layer15_attention_weight_min": -31.452163696289062,
      "activations/layer16_attention_weight_max": 36.38116455078125,
      "activations/layer16_attention_weight_min": -33.254207611083984,
      "activations/layer17_attention_weight_max": 55.89885330200195,
      "activations/layer17_attention_weight_min": -46.67519760131836,
      "activations/layer18_attention_weight_max": 49.63362121582031,
      "activations/layer18_attention_weight_min": -42.534358978271484,
      "activations/layer19_attention_weight_max": 26.190597534179688,
      "activations/layer19_attention_weight_min": -25.624561309814453,
      "activations/layer1_attention_weight_max": 17.088594436645508,
      "activations/layer1_attention_weight_min": -14.107163429260254,
      "activations/layer20_attention_weight_max": 24.675182342529297,
      "activations/layer20_attention_weight_min": -22.54881477355957,
      "activations/layer21_attention_weight_max": 34.09607696533203,
      "activations/layer21_attention_weight_min": -23.346208572387695,
      "activations/layer22_attention_weight_max": 30.133317947387695,
      "activations/layer22_attention_weight_min": -25.15675163269043,
      "activations/layer23_attention_weight_max": 38.041080474853516,
      "activations/layer23_attention_weight_min": -25.008563995361328,
      "activations/layer2_attention_weight_max": 31.354970932006836,
      "activations/layer2_attention_weight_min": -30.432462692260742,
      "activations/layer3_attention_weight_max": 93.55418395996094,
      "activations/layer3_attention_weight_min": -91.58575439453125,
      "activations/layer4_attention_weight_max": 91.7868423461914,
      "activations/layer4_attention_weight_min": -88.24471282958984,
      "activations/layer5_attention_weight_max": 62.48114776611328,
      "activations/layer5_attention_weight_min": -75.63318634033203,
      "activations/layer6_attention_weight_max": 53.22132110595703,
      "activations/layer6_attention_weight_min": -55.19748306274414,
      "activations/layer7_attention_weight_max": 71.47295379638672,
      "activations/layer7_attention_weight_min": -70.69346618652344,
      "activations/layer8_attention_weight_max": 49.07857894897461,
      "activations/layer8_attention_weight_min": -55.12279510498047,
      "activations/layer9_attention_weight_max": 44.47909164428711,
      "activations/layer9_attention_weight_min": -47.702693939208984,
      "epoch": 22.4,
      "learning_rate": 5.589772727272727e-06,
      "loss": 2.7114,
      "step": 385550
    },
    {
      "activations/layer0_attention_weight_max": 17.20107078552246,
      "activations/layer0_attention_weight_min": -14.256199836730957,
      "activations/layer10_attention_weight_max": 36.549766540527344,
      "activations/layer10_attention_weight_min": -35.5096549987793,
      "activations/layer11_attention_weight_max": 33.38146209716797,
      "activations/layer11_attention_weight_min": -35.934993743896484,
      "activations/layer12_attention_weight_max": 19.072065353393555,
      "activations/layer12_attention_weight_min": -26.897323608398438,
      "activations/layer13_attention_weight_max": 39.4595947265625,
      "activations/layer13_attention_weight_min": -27.60332679748535,
      "activations/layer14_attention_weight_max": 33.79667282104492,
      "activations/layer14_attention_weight_min": -31.234079360961914,
      "activations/layer15_attention_weight_max": 32.734901428222656,
      "activations/layer15_attention_weight_min": -31.586013793945312,
      "activations/layer16_attention_weight_max": 33.917171478271484,
      "activations/layer16_attention_weight_min": -33.52739334106445,
      "activations/layer17_attention_weight_max": 49.7366828918457,
      "activations/layer17_attention_weight_min": -46.44179153442383,
      "activations/layer18_attention_weight_max": 45.726261138916016,
      "activations/layer18_attention_weight_min": -41.71776580810547,
      "activations/layer19_attention_weight_max": 25.407896041870117,
      "activations/layer19_attention_weight_min": -23.486295700073242,
      "activations/layer1_attention_weight_max": 14.748994827270508,
      "activations/layer1_attention_weight_min": -15.501068115234375,
      "activations/layer20_attention_weight_max": 22.9777889251709,
      "activations/layer20_attention_weight_min": -23.0095272064209,
      "activations/layer21_attention_weight_max": 40.069034576416016,
      "activations/layer21_attention_weight_min": -24.475296020507812,
      "activations/layer22_attention_weight_max": 29.760684967041016,
      "activations/layer22_attention_weight_min": -25.02943229675293,
      "activations/layer23_attention_weight_max": 40.72712707519531,
      "activations/layer23_attention_weight_min": -25.56013298034668,
      "activations/layer2_attention_weight_max": 33.03165054321289,
      "activations/layer2_attention_weight_min": -31.260154724121094,
      "activations/layer3_attention_weight_max": 87.71192169189453,
      "activations/layer3_attention_weight_min": -89.10708618164062,
      "activations/layer4_attention_weight_max": 83.35811614990234,
      "activations/layer4_attention_weight_min": -85.9540023803711,
      "activations/layer5_attention_weight_max": 60.77119064331055,
      "activations/layer5_attention_weight_min": -71.96244049072266,
      "activations/layer6_attention_weight_max": 47.69247055053711,
      "activations/layer6_attention_weight_min": -47.604000091552734,
      "activations/layer7_attention_weight_max": 67.99092102050781,
      "activations/layer7_attention_weight_min": -67.98485565185547,
      "activations/layer8_attention_weight_max": 46.2433967590332,
      "activations/layer8_attention_weight_min": -49.163326263427734,
      "activations/layer9_attention_weight_max": 41.794681549072266,
      "activations/layer9_attention_weight_min": -44.78184509277344,
      "epoch": 22.41,
      "learning_rate": 5.570833333333333e-06,
      "loss": 2.6871,
      "step": 385600
    },
    {
      "activations/layer0_attention_weight_max": 16.051387786865234,
      "activations/layer0_attention_weight_min": -14.678580284118652,
      "activations/layer10_attention_weight_max": 36.74109649658203,
      "activations/layer10_attention_weight_min": -35.70447540283203,
      "activations/layer11_attention_weight_max": 33.56993865966797,
      "activations/layer11_attention_weight_min": -34.859375,
      "activations/layer12_attention_weight_max": 19.1961669921875,
      "activations/layer12_attention_weight_min": -27.878799438476562,
      "activations/layer13_attention_weight_max": 32.3358154296875,
      "activations/layer13_attention_weight_min": -27.84345054626465,
      "activations/layer14_attention_weight_max": 35.77799987792969,
      "activations/layer14_attention_weight_min": -30.143775939941406,
      "activations/layer15_attention_weight_max": 30.862886428833008,
      "activations/layer15_attention_weight_min": -30.45345687866211,
      "activations/layer16_attention_weight_max": 32.26817321777344,
      "activations/layer16_attention_weight_min": -31.66619873046875,
      "activations/layer17_attention_weight_max": 48.80839920043945,
      "activations/layer17_attention_weight_min": -43.85087966918945,
      "activations/layer18_attention_weight_max": 47.68349075317383,
      "activations/layer18_attention_weight_min": -38.39948272705078,
      "activations/layer19_attention_weight_max": 23.592050552368164,
      "activations/layer19_attention_weight_min": -21.52094841003418,
      "activations/layer1_attention_weight_max": 14.676560401916504,
      "activations/layer1_attention_weight_min": -14.788179397583008,
      "activations/layer20_attention_weight_max": 20.910703659057617,
      "activations/layer20_attention_weight_min": -21.809667587280273,
      "activations/layer21_attention_weight_max": 32.981056213378906,
      "activations/layer21_attention_weight_min": -22.158567428588867,
      "activations/layer22_attention_weight_max": 29.398773193359375,
      "activations/layer22_attention_weight_min": -27.27523422241211,
      "activations/layer23_attention_weight_max": 34.548362731933594,
      "activations/layer23_attention_weight_min": -25.280363082885742,
      "activations/layer2_attention_weight_max": 31.102252960205078,
      "activations/layer2_attention_weight_min": -30.07967185974121,
      "activations/layer3_attention_weight_max": 89.79046630859375,
      "activations/layer3_attention_weight_min": -87.54759216308594,
      "activations/layer4_attention_weight_max": 82.8373794555664,
      "activations/layer4_attention_weight_min": -80.98775482177734,
      "activations/layer5_attention_weight_max": 62.51814270019531,
      "activations/layer5_attention_weight_min": -69.53870391845703,
      "activations/layer6_attention_weight_max": 48.1597785949707,
      "activations/layer6_attention_weight_min": -50.203758239746094,
      "activations/layer7_attention_weight_max": 68.61300659179688,
      "activations/layer7_attention_weight_min": -68.20951080322266,
      "activations/layer8_attention_weight_max": 49.03676986694336,
      "activations/layer8_attention_weight_min": -53.153778076171875,
      "activations/layer9_attention_weight_max": 44.48382568359375,
      "activations/layer9_attention_weight_min": -46.82686996459961,
      "epoch": 22.41,
      "learning_rate": 5.551893939393938e-06,
      "loss": 2.6956,
      "step": 385650
    },
    {
      "activations/layer0_attention_weight_max": 15.930512428283691,
      "activations/layer0_attention_weight_min": -14.222657203674316,
      "activations/layer10_attention_weight_max": 34.04084014892578,
      "activations/layer10_attention_weight_min": -33.737205505371094,
      "activations/layer11_attention_weight_max": 33.83163070678711,
      "activations/layer11_attention_weight_min": -34.04076385498047,
      "activations/layer12_attention_weight_max": 19.000425338745117,
      "activations/layer12_attention_weight_min": -25.33504867553711,
      "activations/layer13_attention_weight_max": 33.346473693847656,
      "activations/layer13_attention_weight_min": -27.118755340576172,
      "activations/layer14_attention_weight_max": 39.233299255371094,
      "activations/layer14_attention_weight_min": -31.01719856262207,
      "activations/layer15_attention_weight_max": 36.78106689453125,
      "activations/layer15_attention_weight_min": -32.34149169921875,
      "activations/layer16_attention_weight_max": 35.087364196777344,
      "activations/layer16_attention_weight_min": -31.4815673828125,
      "activations/layer17_attention_weight_max": 52.142765045166016,
      "activations/layer17_attention_weight_min": -43.74864196777344,
      "activations/layer18_attention_weight_max": 50.7816276550293,
      "activations/layer18_attention_weight_min": -40.48503875732422,
      "activations/layer19_attention_weight_max": 28.555139541625977,
      "activations/layer19_attention_weight_min": -23.396286010742188,
      "activations/layer1_attention_weight_max": 15.024511337280273,
      "activations/layer1_attention_weight_min": -14.133625030517578,
      "activations/layer20_attention_weight_max": 24.96232795715332,
      "activations/layer20_attention_weight_min": -22.658437728881836,
      "activations/layer21_attention_weight_max": 40.18943786621094,
      "activations/layer21_attention_weight_min": -22.56052017211914,
      "activations/layer22_attention_weight_max": 29.099613189697266,
      "activations/layer22_attention_weight_min": -24.35464096069336,
      "activations/layer23_attention_weight_max": 38.92974090576172,
      "activations/layer23_attention_weight_min": -24.38222885131836,
      "activations/layer2_attention_weight_max": 29.66212272644043,
      "activations/layer2_attention_weight_min": -29.938034057617188,
      "activations/layer3_attention_weight_max": 85.9129867553711,
      "activations/layer3_attention_weight_min": -88.8507080078125,
      "activations/layer4_attention_weight_max": 81.39502716064453,
      "activations/layer4_attention_weight_min": -85.99486541748047,
      "activations/layer5_attention_weight_max": 57.64006805419922,
      "activations/layer5_attention_weight_min": -70.17626190185547,
      "activations/layer6_attention_weight_max": 47.81447982788086,
      "activations/layer6_attention_weight_min": -49.99584197998047,
      "activations/layer7_attention_weight_max": 68.05084228515625,
      "activations/layer7_attention_weight_min": -68.56521606445312,
      "activations/layer8_attention_weight_max": 45.84998321533203,
      "activations/layer8_attention_weight_min": -48.728172302246094,
      "activations/layer9_attention_weight_max": 42.35238265991211,
      "activations/layer9_attention_weight_min": -44.80720520019531,
      "epoch": 22.41,
      "learning_rate": 5.532954545454545e-06,
      "loss": 2.6954,
      "step": 385700
    },
    {
      "activations/layer0_attention_weight_max": 16.302825927734375,
      "activations/layer0_attention_weight_min": -14.100160598754883,
      "activations/layer10_attention_weight_max": 39.76421356201172,
      "activations/layer10_attention_weight_min": -40.57734680175781,
      "activations/layer11_attention_weight_max": 37.162532806396484,
      "activations/layer11_attention_weight_min": -37.86955261230469,
      "activations/layer12_attention_weight_max": 18.353744506835938,
      "activations/layer12_attention_weight_min": -24.241214752197266,
      "activations/layer13_attention_weight_max": 31.321352005004883,
      "activations/layer13_attention_weight_min": -27.114765167236328,
      "activations/layer14_attention_weight_max": 35.547119140625,
      "activations/layer14_attention_weight_min": -29.886844635009766,
      "activations/layer15_attention_weight_max": 31.407258987426758,
      "activations/layer15_attention_weight_min": -29.943737030029297,
      "activations/layer16_attention_weight_max": 34.82001495361328,
      "activations/layer16_attention_weight_min": -33.77866744995117,
      "activations/layer17_attention_weight_max": 52.394371032714844,
      "activations/layer17_attention_weight_min": -44.20985412597656,
      "activations/layer18_attention_weight_max": 47.57758712768555,
      "activations/layer18_attention_weight_min": -41.823448181152344,
      "activations/layer19_attention_weight_max": 24.08526611328125,
      "activations/layer19_attention_weight_min": -21.122528076171875,
      "activations/layer1_attention_weight_max": 15.10034465789795,
      "activations/layer1_attention_weight_min": -14.990943908691406,
      "activations/layer20_attention_weight_max": 22.109041213989258,
      "activations/layer20_attention_weight_min": -21.20994758605957,
      "activations/layer21_attention_weight_max": 30.008373260498047,
      "activations/layer21_attention_weight_min": -21.927650451660156,
      "activations/layer22_attention_weight_max": 27.892955780029297,
      "activations/layer22_attention_weight_min": -26.235816955566406,
      "activations/layer23_attention_weight_max": 33.10900115966797,
      "activations/layer23_attention_weight_min": -23.618362426757812,
      "activations/layer2_attention_weight_max": 32.74885940551758,
      "activations/layer2_attention_weight_min": -31.530637741088867,
      "activations/layer3_attention_weight_max": 92.5045394897461,
      "activations/layer3_attention_weight_min": -92.0182876586914,
      "activations/layer4_attention_weight_max": 86.63948822021484,
      "activations/layer4_attention_weight_min": -88.77587890625,
      "activations/layer5_attention_weight_max": 64.03330993652344,
      "activations/layer5_attention_weight_min": -71.0589828491211,
      "activations/layer6_attention_weight_max": 50.71830749511719,
      "activations/layer6_attention_weight_min": -51.32475280761719,
      "activations/layer7_attention_weight_max": 71.85569763183594,
      "activations/layer7_attention_weight_min": -68.72897338867188,
      "activations/layer8_attention_weight_max": 51.02333068847656,
      "activations/layer8_attention_weight_min": -55.29011154174805,
      "activations/layer9_attention_weight_max": 47.55648422241211,
      "activations/layer9_attention_weight_min": -49.41563034057617,
      "epoch": 22.41,
      "learning_rate": 5.514015151515151e-06,
      "loss": 2.7206,
      "step": 385750
    },
    {
      "activations/layer0_attention_weight_max": 16.770957946777344,
      "activations/layer0_attention_weight_min": -14.59235954284668,
      "activations/layer10_attention_weight_max": 37.59931182861328,
      "activations/layer10_attention_weight_min": -36.8004035949707,
      "activations/layer11_attention_weight_max": 35.786834716796875,
      "activations/layer11_attention_weight_min": -35.45786666870117,
      "activations/layer12_attention_weight_max": 19.07691764831543,
      "activations/layer12_attention_weight_min": -22.786828994750977,
      "activations/layer13_attention_weight_max": 36.61109924316406,
      "activations/layer13_attention_weight_min": -28.184316635131836,
      "activations/layer14_attention_weight_max": 36.24692916870117,
      "activations/layer14_attention_weight_min": -31.668310165405273,
      "activations/layer15_attention_weight_max": 34.05348587036133,
      "activations/layer15_attention_weight_min": -31.37792205810547,
      "activations/layer16_attention_weight_max": 37.66898727416992,
      "activations/layer16_attention_weight_min": -34.7978630065918,
      "activations/layer17_attention_weight_max": 58.37253189086914,
      "activations/layer17_attention_weight_min": -49.66359329223633,
      "activations/layer18_attention_weight_max": 51.3353385925293,
      "activations/layer18_attention_weight_min": -43.45246887207031,
      "activations/layer19_attention_weight_max": 26.8905029296875,
      "activations/layer19_attention_weight_min": -24.96512794494629,
      "activations/layer1_attention_weight_max": 15.737750053405762,
      "activations/layer1_attention_weight_min": -17.158309936523438,
      "activations/layer20_attention_weight_max": 23.335548400878906,
      "activations/layer20_attention_weight_min": -20.67911148071289,
      "activations/layer21_attention_weight_max": 44.32380294799805,
      "activations/layer21_attention_weight_min": -25.384105682373047,
      "activations/layer22_attention_weight_max": 30.115217208862305,
      "activations/layer22_attention_weight_min": -25.03246307373047,
      "activations/layer23_attention_weight_max": 39.363182067871094,
      "activations/layer23_attention_weight_min": -24.496206283569336,
      "activations/layer2_attention_weight_max": 33.21891784667969,
      "activations/layer2_attention_weight_min": -32.073204040527344,
      "activations/layer3_attention_weight_max": 93.01178741455078,
      "activations/layer3_attention_weight_min": -91.02874755859375,
      "activations/layer4_attention_weight_max": 87.05865478515625,
      "activations/layer4_attention_weight_min": -84.5082015991211,
      "activations/layer5_attention_weight_max": 61.7686767578125,
      "activations/layer5_attention_weight_min": -73.34606170654297,
      "activations/layer6_attention_weight_max": 51.244319915771484,
      "activations/layer6_attention_weight_min": -51.9128303527832,
      "activations/layer7_attention_weight_max": 70.09990692138672,
      "activations/layer7_attention_weight_min": -72.42498779296875,
      "activations/layer8_attention_weight_max": 49.02171325683594,
      "activations/layer8_attention_weight_min": -52.15565490722656,
      "activations/layer9_attention_weight_max": 47.11460494995117,
      "activations/layer9_attention_weight_min": -49.15147018432617,
      "epoch": 22.42,
      "learning_rate": 5.495075757575757e-06,
      "loss": 2.7065,
      "step": 385800
    },
    {
      "activations/layer0_attention_weight_max": 15.303055763244629,
      "activations/layer0_attention_weight_min": -14.23462200164795,
      "activations/layer10_attention_weight_max": 37.73702621459961,
      "activations/layer10_attention_weight_min": -35.988868713378906,
      "activations/layer11_attention_weight_max": 33.581268310546875,
      "activations/layer11_attention_weight_min": -35.632164001464844,
      "activations/layer12_attention_weight_max": 19.366180419921875,
      "activations/layer12_attention_weight_min": -25.212350845336914,
      "activations/layer13_attention_weight_max": 40.981346130371094,
      "activations/layer13_attention_weight_min": -25.96776580810547,
      "activations/layer14_attention_weight_max": 35.19749450683594,
      "activations/layer14_attention_weight_min": -29.14687156677246,
      "activations/layer15_attention_weight_max": 32.51765060424805,
      "activations/layer15_attention_weight_min": -28.54648208618164,
      "activations/layer16_attention_weight_max": 32.992862701416016,
      "activations/layer16_attention_weight_min": -30.43950080871582,
      "activations/layer17_attention_weight_max": 49.379150390625,
      "activations/layer17_attention_weight_min": -42.14560317993164,
      "activations/layer18_attention_weight_max": 47.09989929199219,
      "activations/layer18_attention_weight_min": -37.99363708496094,
      "activations/layer19_attention_weight_max": 27.727243423461914,
      "activations/layer19_attention_weight_min": -21.88326072692871,
      "activations/layer1_attention_weight_max": 14.346793174743652,
      "activations/layer1_attention_weight_min": -15.989277839660645,
      "activations/layer20_attention_weight_max": 22.457292556762695,
      "activations/layer20_attention_weight_min": -20.972623825073242,
      "activations/layer21_attention_weight_max": 36.97195816040039,
      "activations/layer21_attention_weight_min": -22.04330825805664,
      "activations/layer22_attention_weight_max": 28.81625747680664,
      "activations/layer22_attention_weight_min": -24.031635284423828,
      "activations/layer23_attention_weight_max": 35.16082763671875,
      "activations/layer23_attention_weight_min": -24.099781036376953,
      "activations/layer2_attention_weight_max": 31.568822860717773,
      "activations/layer2_attention_weight_min": -30.389135360717773,
      "activations/layer3_attention_weight_max": 87.68733215332031,
      "activations/layer3_attention_weight_min": -90.86371612548828,
      "activations/layer4_attention_weight_max": 83.44133758544922,
      "activations/layer4_attention_weight_min": -84.3004150390625,
      "activations/layer5_attention_weight_max": 61.59855651855469,
      "activations/layer5_attention_weight_min": -70.93504333496094,
      "activations/layer6_attention_weight_max": 51.42585754394531,
      "activations/layer6_attention_weight_min": -54.502437591552734,
      "activations/layer7_attention_weight_max": 76.62186431884766,
      "activations/layer7_attention_weight_min": -67.7768325805664,
      "activations/layer8_attention_weight_max": 47.13594436645508,
      "activations/layer8_attention_weight_min": -51.05498123168945,
      "activations/layer9_attention_weight_max": 44.55878448486328,
      "activations/layer9_attention_weight_min": -46.23833084106445,
      "epoch": 22.42,
      "learning_rate": 5.476136363636364e-06,
      "loss": 2.7146,
      "step": 385850
    },
    {
      "activations/layer0_attention_weight_max": 15.795243263244629,
      "activations/layer0_attention_weight_min": -14.70529842376709,
      "activations/layer10_attention_weight_max": 37.9575080871582,
      "activations/layer10_attention_weight_min": -37.30437088012695,
      "activations/layer11_attention_weight_max": 35.4063720703125,
      "activations/layer11_attention_weight_min": -36.846160888671875,
      "activations/layer12_attention_weight_max": 20.676084518432617,
      "activations/layer12_attention_weight_min": -24.648136138916016,
      "activations/layer13_attention_weight_max": 35.710994720458984,
      "activations/layer13_attention_weight_min": -29.237712860107422,
      "activations/layer14_attention_weight_max": 35.281944274902344,
      "activations/layer14_attention_weight_min": -32.81196975708008,
      "activations/layer15_attention_weight_max": 33.194236755371094,
      "activations/layer15_attention_weight_min": -30.121469497680664,
      "activations/layer16_attention_weight_max": 36.058528900146484,
      "activations/layer16_attention_weight_min": -32.873809814453125,
      "activations/layer17_attention_weight_max": 52.02873992919922,
      "activations/layer17_attention_weight_min": -44.413272857666016,
      "activations/layer18_attention_weight_max": 46.86349868774414,
      "activations/layer18_attention_weight_min": -39.54489517211914,
      "activations/layer19_attention_weight_max": 24.928586959838867,
      "activations/layer19_attention_weight_min": -21.50690269470215,
      "activations/layer1_attention_weight_max": 15.33273983001709,
      "activations/layer1_attention_weight_min": -14.313664436340332,
      "activations/layer20_attention_weight_max": 24.459630966186523,
      "activations/layer20_attention_weight_min": -21.1072998046875,
      "activations/layer21_attention_weight_max": 35.74263381958008,
      "activations/layer21_attention_weight_min": -21.29462432861328,
      "activations/layer22_attention_weight_max": 30.835542678833008,
      "activations/layer22_attention_weight_min": -25.04942512512207,
      "activations/layer23_attention_weight_max": 37.24179458618164,
      "activations/layer23_attention_weight_min": -23.993144989013672,
      "activations/layer2_attention_weight_max": 31.323020935058594,
      "activations/layer2_attention_weight_min": -29.158432006835938,
      "activations/layer3_attention_weight_max": 87.34903717041016,
      "activations/layer3_attention_weight_min": -85.93231964111328,
      "activations/layer4_attention_weight_max": 86.02259063720703,
      "activations/layer4_attention_weight_min": -83.54376983642578,
      "activations/layer5_attention_weight_max": 62.28925323486328,
      "activations/layer5_attention_weight_min": -69.37633514404297,
      "activations/layer6_attention_weight_max": 51.98801040649414,
      "activations/layer6_attention_weight_min": -51.65272903442383,
      "activations/layer7_attention_weight_max": 69.2498779296875,
      "activations/layer7_attention_weight_min": -67.93598175048828,
      "activations/layer8_attention_weight_max": 50.85893630981445,
      "activations/layer8_attention_weight_min": -55.24231719970703,
      "activations/layer9_attention_weight_max": 48.47671127319336,
      "activations/layer9_attention_weight_min": -47.4159049987793,
      "epoch": 22.42,
      "learning_rate": 5.4571969696969696e-06,
      "loss": 2.6962,
      "step": 385900
    },
    {
      "activations/layer0_attention_weight_max": 16.4598331451416,
      "activations/layer0_attention_weight_min": -14.331536293029785,
      "activations/layer10_attention_weight_max": 38.261573791503906,
      "activations/layer10_attention_weight_min": -36.643924713134766,
      "activations/layer11_attention_weight_max": 32.61732482910156,
      "activations/layer11_attention_weight_min": -34.75003433227539,
      "activations/layer12_attention_weight_max": 19.394224166870117,
      "activations/layer12_attention_weight_min": -24.238754272460938,
      "activations/layer13_attention_weight_max": 33.59928512573242,
      "activations/layer13_attention_weight_min": -28.825626373291016,
      "activations/layer14_attention_weight_max": 37.77977752685547,
      "activations/layer14_attention_weight_min": -30.31782341003418,
      "activations/layer15_attention_weight_max": 35.452693939208984,
      "activations/layer15_attention_weight_min": -29.54135513305664,
      "activations/layer16_attention_weight_max": 36.748897552490234,
      "activations/layer16_attention_weight_min": -33.1746826171875,
      "activations/layer17_attention_weight_max": 52.9046745300293,
      "activations/layer17_attention_weight_min": -44.67173385620117,
      "activations/layer18_attention_weight_max": 49.234554290771484,
      "activations/layer18_attention_weight_min": -40.45657730102539,
      "activations/layer19_attention_weight_max": 29.161352157592773,
      "activations/layer19_attention_weight_min": -23.730480194091797,
      "activations/layer1_attention_weight_max": 14.98383903503418,
      "activations/layer1_attention_weight_min": -15.372645378112793,
      "activations/layer20_attention_weight_max": 24.1842041015625,
      "activations/layer20_attention_weight_min": -24.658275604248047,
      "activations/layer21_attention_weight_max": 39.745506286621094,
      "activations/layer21_attention_weight_min": -24.546875,
      "activations/layer22_attention_weight_max": 28.92632484436035,
      "activations/layer22_attention_weight_min": -24.78138542175293,
      "activations/layer23_attention_weight_max": 43.26104736328125,
      "activations/layer23_attention_weight_min": -25.682449340820312,
      "activations/layer2_attention_weight_max": 31.731338500976562,
      "activations/layer2_attention_weight_min": -30.9176082611084,
      "activations/layer3_attention_weight_max": 89.86181640625,
      "activations/layer3_attention_weight_min": -88.05870819091797,
      "activations/layer4_attention_weight_max": 85.54065704345703,
      "activations/layer4_attention_weight_min": -85.24295043945312,
      "activations/layer5_attention_weight_max": 60.97240447998047,
      "activations/layer5_attention_weight_min": -72.38379669189453,
      "activations/layer6_attention_weight_max": 50.327476501464844,
      "activations/layer6_attention_weight_min": -52.12257385253906,
      "activations/layer7_attention_weight_max": 74.48043823242188,
      "activations/layer7_attention_weight_min": -67.92915344238281,
      "activations/layer8_attention_weight_max": 47.249446868896484,
      "activations/layer8_attention_weight_min": -54.01588821411133,
      "activations/layer9_attention_weight_max": 45.50788116455078,
      "activations/layer9_attention_weight_min": -46.8285026550293,
      "epoch": 22.43,
      "learning_rate": 5.438257575757575e-06,
      "loss": 2.7141,
      "step": 385950
    },
    {
      "activations/layer0_attention_weight_max": 16.22599983215332,
      "activations/layer0_attention_weight_min": -14.444190979003906,
      "activations/layer10_attention_weight_max": 35.42451477050781,
      "activations/layer10_attention_weight_min": -35.06001663208008,
      "activations/layer11_attention_weight_max": 32.035118103027344,
      "activations/layer11_attention_weight_min": -34.1273193359375,
      "activations/layer12_attention_weight_max": 19.985876083374023,
      "activations/layer12_attention_weight_min": -24.216968536376953,
      "activations/layer13_attention_weight_max": 35.523170471191406,
      "activations/layer13_attention_weight_min": -28.099870681762695,
      "activations/layer14_attention_weight_max": 33.984771728515625,
      "activations/layer14_attention_weight_min": -30.186216354370117,
      "activations/layer15_attention_weight_max": 32.44709396362305,
      "activations/layer15_attention_weight_min": -29.877674102783203,
      "activations/layer16_attention_weight_max": 38.2952880859375,
      "activations/layer16_attention_weight_min": -34.222900390625,
      "activations/layer17_attention_weight_max": 55.668601989746094,
      "activations/layer17_attention_weight_min": -50.70108413696289,
      "activations/layer18_attention_weight_max": 51.48798751831055,
      "activations/layer18_attention_weight_min": -48.10729217529297,
      "activations/layer19_attention_weight_max": 26.24737548828125,
      "activations/layer19_attention_weight_min": -24.366682052612305,
      "activations/layer1_attention_weight_max": 15.906254768371582,
      "activations/layer1_attention_weight_min": -16.354001998901367,
      "activations/layer20_attention_weight_max": 22.76085090637207,
      "activations/layer20_attention_weight_min": -22.489408493041992,
      "activations/layer21_attention_weight_max": 37.293155670166016,
      "activations/layer21_attention_weight_min": -23.260406494140625,
      "activations/layer22_attention_weight_max": 29.914318084716797,
      "activations/layer22_attention_weight_min": -26.43244743347168,
      "activations/layer23_attention_weight_max": 32.385765075683594,
      "activations/layer23_attention_weight_min": -25.135589599609375,
      "activations/layer2_attention_weight_max": 32.52436447143555,
      "activations/layer2_attention_weight_min": -29.688690185546875,
      "activations/layer3_attention_weight_max": 88.05615234375,
      "activations/layer3_attention_weight_min": -87.89801025390625,
      "activations/layer4_attention_weight_max": 84.99514770507812,
      "activations/layer4_attention_weight_min": -86.93548583984375,
      "activations/layer5_attention_weight_max": 62.62474822998047,
      "activations/layer5_attention_weight_min": -72.13323211669922,
      "activations/layer6_attention_weight_max": 50.640525817871094,
      "activations/layer6_attention_weight_min": -51.80025100708008,
      "activations/layer7_attention_weight_max": 68.15904235839844,
      "activations/layer7_attention_weight_min": -68.73297119140625,
      "activations/layer8_attention_weight_max": 47.22816848754883,
      "activations/layer8_attention_weight_min": -49.74894714355469,
      "activations/layer9_attention_weight_max": 44.235530853271484,
      "activations/layer9_attention_weight_min": -46.06951141357422,
      "epoch": 22.43,
      "learning_rate": 5.419318181818181e-06,
      "loss": 2.7068,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.5357,
      "eval_samples_per_second": 503.065,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.5357,
      "eval_openwebtext_samples_per_second": 503.065,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9695,
      "eval_wikitext_samples_per_second": 231.534,
      "step": 386000
    },
    {
      "epoch": 22.43,
      "eval_lambada_loss": 2.396484375,
      "eval_lambada_ppl": 10.984491067549964,
      "eval_lambada_runtime": 9.6019,
      "eval_lambada_samples_per_second": 507.089,
      "step": 386000
    },
    {
      "activations/layer0_attention_weight_max": 16.625690460205078,
      "activations/layer0_attention_weight_min": -14.365620613098145,
      "activations/layer10_attention_weight_max": 33.560546875,
      "activations/layer10_attention_weight_min": -34.563026428222656,
      "activations/layer11_attention_weight_max": 31.894393920898438,
      "activations/layer11_attention_weight_min": -34.60338592529297,
      "activations/layer12_attention_weight_max": 18.204818725585938,
      "activations/layer12_attention_weight_min": -23.75298309326172,
      "activations/layer13_attention_weight_max": 29.376747131347656,
      "activations/layer13_attention_weight_min": -26.539752960205078,
      "activations/layer14_attention_weight_max": 32.8361701965332,
      "activations/layer14_attention_weight_min": -30.968538284301758,
      "activations/layer15_attention_weight_max": 31.08574676513672,
      "activations/layer15_attention_weight_min": -29.951059341430664,
      "activations/layer16_attention_weight_max": 32.34954071044922,
      "activations/layer16_attention_weight_min": -33.448665618896484,
      "activations/layer17_attention_weight_max": 49.61919021606445,
      "activations/layer17_attention_weight_min": -45.29607009887695,
      "activations/layer18_attention_weight_max": 42.732017517089844,
      "activations/layer18_attention_weight_min": -42.74872970581055,
      "activations/layer19_attention_weight_max": 22.365882873535156,
      "activations/layer19_attention_weight_min": -24.071870803833008,
      "activations/layer1_attention_weight_max": 15.229573249816895,
      "activations/layer1_attention_weight_min": -15.391301155090332,
      "activations/layer20_attention_weight_max": 22.726560592651367,
      "activations/layer20_attention_weight_min": -21.983369827270508,
      "activations/layer21_attention_weight_max": 29.513601303100586,
      "activations/layer21_attention_weight_min": -20.695751190185547,
      "activations/layer22_attention_weight_max": 29.80108642578125,
      "activations/layer22_attention_weight_min": -25.840940475463867,
      "activations/layer23_attention_weight_max": 32.49104690551758,
      "activations/layer23_attention_weight_min": -21.908512115478516,
      "activations/layer2_attention_weight_max": 31.17443084716797,
      "activations/layer2_attention_weight_min": -30.539419174194336,
      "activations/layer3_attention_weight_max": 86.58051300048828,
      "activations/layer3_attention_weight_min": -89.16666412353516,
      "activations/layer4_attention_weight_max": 83.26844787597656,
      "activations/layer4_attention_weight_min": -83.19937896728516,
      "activations/layer5_attention_weight_max": 58.353904724121094,
      "activations/layer5_attention_weight_min": -68.52948760986328,
      "activations/layer6_attention_weight_max": 49.17463684082031,
      "activations/layer6_attention_weight_min": -50.92073059082031,
      "activations/layer7_attention_weight_max": 65.63750457763672,
      "activations/layer7_attention_weight_min": -67.59645080566406,
      "activations/layer8_attention_weight_max": 46.77220153808594,
      "activations/layer8_attention_weight_min": -50.066001892089844,
      "activations/layer9_attention_weight_max": 41.16054916381836,
      "activations/layer9_attention_weight_min": -45.59060287475586,
      "epoch": 22.43,
      "learning_rate": 5.400378787878788e-06,
      "loss": 2.7179,
      "step": 386050
    },
    {
      "activations/layer0_attention_weight_max": 15.794496536254883,
      "activations/layer0_attention_weight_min": -14.541739463806152,
      "activations/layer10_attention_weight_max": 36.876853942871094,
      "activations/layer10_attention_weight_min": -37.66131591796875,
      "activations/layer11_attention_weight_max": 34.64631271362305,
      "activations/layer11_attention_weight_min": -36.365570068359375,
      "activations/layer12_attention_weight_max": 18.844070434570312,
      "activations/layer12_attention_weight_min": -23.09333038330078,
      "activations/layer13_attention_weight_max": 37.68769073486328,
      "activations/layer13_attention_weight_min": -28.124244689941406,
      "activations/layer14_attention_weight_max": 37.008460998535156,
      "activations/layer14_attention_weight_min": -30.930179595947266,
      "activations/layer15_attention_weight_max": 33.44355010986328,
      "activations/layer15_attention_weight_min": -32.205406188964844,
      "activations/layer16_attention_weight_max": 37.6110954284668,
      "activations/layer16_attention_weight_min": -34.68696975708008,
      "activations/layer17_attention_weight_max": 52.330745697021484,
      "activations/layer17_attention_weight_min": -44.72334671020508,
      "activations/layer18_attention_weight_max": 48.555416107177734,
      "activations/layer18_attention_weight_min": -43.07400131225586,
      "activations/layer19_attention_weight_max": 24.435606002807617,
      "activations/layer19_attention_weight_min": -24.95075035095215,
      "activations/layer1_attention_weight_max": 16.05858039855957,
      "activations/layer1_attention_weight_min": -15.622980117797852,
      "activations/layer20_attention_weight_max": 23.883834838867188,
      "activations/layer20_attention_weight_min": -20.472049713134766,
      "activations/layer21_attention_weight_max": 39.211341857910156,
      "activations/layer21_attention_weight_min": -24.86200523376465,
      "activations/layer22_attention_weight_max": 29.400619506835938,
      "activations/layer22_attention_weight_min": -24.915111541748047,
      "activations/layer23_attention_weight_max": 36.22613525390625,
      "activations/layer23_attention_weight_min": -23.856815338134766,
      "activations/layer2_attention_weight_max": 32.01713562011719,
      "activations/layer2_attention_weight_min": -30.778188705444336,
      "activations/layer3_attention_weight_max": 88.91973114013672,
      "activations/layer3_attention_weight_min": -88.96539306640625,
      "activations/layer4_attention_weight_max": 84.94728088378906,
      "activations/layer4_attention_weight_min": -86.07476043701172,
      "activations/layer5_attention_weight_max": 63.143516540527344,
      "activations/layer5_attention_weight_min": -71.72238159179688,
      "activations/layer6_attention_weight_max": 51.10741424560547,
      "activations/layer6_attention_weight_min": -52.4749641418457,
      "activations/layer7_attention_weight_max": 70.42101287841797,
      "activations/layer7_attention_weight_min": -69.8396987915039,
      "activations/layer8_attention_weight_max": 48.974700927734375,
      "activations/layer8_attention_weight_min": -53.36952590942383,
      "activations/layer9_attention_weight_max": 45.11166000366211,
      "activations/layer9_attention_weight_min": -50.24258041381836,
      "epoch": 22.43,
      "learning_rate": 5.381439393939393e-06,
      "loss": 2.6984,
      "step": 386100
    },
    {
      "activations/layer0_attention_weight_max": 18.544231414794922,
      "activations/layer0_attention_weight_min": -14.530951499938965,
      "activations/layer10_attention_weight_max": 36.34712600708008,
      "activations/layer10_attention_weight_min": -35.75443649291992,
      "activations/layer11_attention_weight_max": 33.449527740478516,
      "activations/layer11_attention_weight_min": -34.52759552001953,
      "activations/layer12_attention_weight_max": 19.82544708251953,
      "activations/layer12_attention_weight_min": -26.836286544799805,
      "activations/layer13_attention_weight_max": 34.089447021484375,
      "activations/layer13_attention_weight_min": -28.102962493896484,
      "activations/layer14_attention_weight_max": 35.42741394042969,
      "activations/layer14_attention_weight_min": -31.707107543945312,
      "activations/layer15_attention_weight_max": 33.66904830932617,
      "activations/layer15_attention_weight_min": -31.92341423034668,
      "activations/layer16_attention_weight_max": 36.30109786987305,
      "activations/layer16_attention_weight_min": -34.395843505859375,
      "activations/layer17_attention_weight_max": 53.341129302978516,
      "activations/layer17_attention_weight_min": -48.5042610168457,
      "activations/layer18_attention_weight_max": 48.849327087402344,
      "activations/layer18_attention_weight_min": -43.91359329223633,
      "activations/layer19_attention_weight_max": 27.403696060180664,
      "activations/layer19_attention_weight_min": -26.489606857299805,
      "activations/layer1_attention_weight_max": 15.065916061401367,
      "activations/layer1_attention_weight_min": -14.34170913696289,
      "activations/layer20_attention_weight_max": 26.669811248779297,
      "activations/layer20_attention_weight_min": -22.769554138183594,
      "activations/layer21_attention_weight_max": 39.848915100097656,
      "activations/layer21_attention_weight_min": -25.25872802734375,
      "activations/layer22_attention_weight_max": 28.675704956054688,
      "activations/layer22_attention_weight_min": -26.563495635986328,
      "activations/layer23_attention_weight_max": 38.31599044799805,
      "activations/layer23_attention_weight_min": -26.119733810424805,
      "activations/layer2_attention_weight_max": 30.94518280029297,
      "activations/layer2_attention_weight_min": -29.96332550048828,
      "activations/layer3_attention_weight_max": 87.36302947998047,
      "activations/layer3_attention_weight_min": -89.20201873779297,
      "activations/layer4_attention_weight_max": 83.71966552734375,
      "activations/layer4_attention_weight_min": -83.53984069824219,
      "activations/layer5_attention_weight_max": 62.124305725097656,
      "activations/layer5_attention_weight_min": -73.37164306640625,
      "activations/layer6_attention_weight_max": 51.152957916259766,
      "activations/layer6_attention_weight_min": -50.61983871459961,
      "activations/layer7_attention_weight_max": 69.64186096191406,
      "activations/layer7_attention_weight_min": -71.51980590820312,
      "activations/layer8_attention_weight_max": 47.47753143310547,
      "activations/layer8_attention_weight_min": -50.17665481567383,
      "activations/layer9_attention_weight_max": 44.03240966796875,
      "activations/layer9_attention_weight_min": -46.920074462890625,
      "epoch": 22.44,
      "learning_rate": 5.362499999999999e-06,
      "loss": 2.7113,
      "step": 386150
    },
    {
      "activations/layer0_attention_weight_max": 15.574207305908203,
      "activations/layer0_attention_weight_min": -14.596089363098145,
      "activations/layer10_attention_weight_max": 37.837059020996094,
      "activations/layer10_attention_weight_min": -39.00413131713867,
      "activations/layer11_attention_weight_max": 35.894866943359375,
      "activations/layer11_attention_weight_min": -37.029781341552734,
      "activations/layer12_attention_weight_max": 20.173660278320312,
      "activations/layer12_attention_weight_min": -25.337238311767578,
      "activations/layer13_attention_weight_max": 35.33062744140625,
      "activations/layer13_attention_weight_min": -27.720489501953125,
      "activations/layer14_attention_weight_max": 41.384578704833984,
      "activations/layer14_attention_weight_min": -33.042076110839844,
      "activations/layer15_attention_weight_max": 40.46268081665039,
      "activations/layer15_attention_weight_min": -30.42786979675293,
      "activations/layer16_attention_weight_max": 42.66352462768555,
      "activations/layer16_attention_weight_min": -36.41958236694336,
      "activations/layer17_attention_weight_max": 60.87914276123047,
      "activations/layer17_attention_weight_min": -51.627140045166016,
      "activations/layer18_attention_weight_max": 58.647132873535156,
      "activations/layer18_attention_weight_min": -47.14360046386719,
      "activations/layer19_attention_weight_max": 27.612817764282227,
      "activations/layer19_attention_weight_min": -23.596851348876953,
      "activations/layer1_attention_weight_max": 15.960173606872559,
      "activations/layer1_attention_weight_min": -15.353513717651367,
      "activations/layer20_attention_weight_max": 26.351505279541016,
      "activations/layer20_attention_weight_min": -21.801782608032227,
      "activations/layer21_attention_weight_max": 44.57611083984375,
      "activations/layer21_attention_weight_min": -26.053518295288086,
      "activations/layer22_attention_weight_max": 33.85130310058594,
      "activations/layer22_attention_weight_min": -25.10235595703125,
      "activations/layer23_attention_weight_max": 41.2227783203125,
      "activations/layer23_attention_weight_min": -24.280807495117188,
      "activations/layer2_attention_weight_max": 33.00297546386719,
      "activations/layer2_attention_weight_min": -31.12835693359375,
      "activations/layer3_attention_weight_max": 87.41104888916016,
      "activations/layer3_attention_weight_min": -87.952880859375,
      "activations/layer4_attention_weight_max": 85.39791107177734,
      "activations/layer4_attention_weight_min": -86.5293960571289,
      "activations/layer5_attention_weight_max": 62.577735900878906,
      "activations/layer5_attention_weight_min": -73.39905548095703,
      "activations/layer6_attention_weight_max": 48.637638092041016,
      "activations/layer6_attention_weight_min": -49.601985931396484,
      "activations/layer7_attention_weight_max": 70.19847869873047,
      "activations/layer7_attention_weight_min": -70.15678405761719,
      "activations/layer8_attention_weight_max": 47.484554290771484,
      "activations/layer8_attention_weight_min": -53.16400909423828,
      "activations/layer9_attention_weight_max": 43.8648681640625,
      "activations/layer9_attention_weight_min": -49.37948226928711,
      "epoch": 22.44,
      "learning_rate": 5.3435606060606065e-06,
      "loss": 2.7025,
      "step": 386200
    },
    {
      "activations/layer0_attention_weight_max": 16.25494384765625,
      "activations/layer0_attention_weight_min": -14.420207977294922,
      "activations/layer10_attention_weight_max": 37.282188415527344,
      "activations/layer10_attention_weight_min": -37.475223541259766,
      "activations/layer11_attention_weight_max": 35.12813186645508,
      "activations/layer11_attention_weight_min": -36.08214569091797,
      "activations/layer12_attention_weight_max": 18.121185302734375,
      "activations/layer12_attention_weight_min": -24.39471435546875,
      "activations/layer13_attention_weight_max": 33.46741485595703,
      "activations/layer13_attention_weight_min": -27.248332977294922,
      "activations/layer14_attention_weight_max": 34.235595703125,
      "activations/layer14_attention_weight_min": -31.648954391479492,
      "activations/layer15_attention_weight_max": 32.974525451660156,
      "activations/layer15_attention_weight_min": -32.032798767089844,
      "activations/layer16_attention_weight_max": 34.84153366088867,
      "activations/layer16_attention_weight_min": -32.60114669799805,
      "activations/layer17_attention_weight_max": 54.150482177734375,
      "activations/layer17_attention_weight_min": -45.24607849121094,
      "activations/layer18_attention_weight_max": 50.60061264038086,
      "activations/layer18_attention_weight_min": -40.329559326171875,
      "activations/layer19_attention_weight_max": 27.213708877563477,
      "activations/layer19_attention_weight_min": -23.5450496673584,
      "activations/layer1_attention_weight_max": 15.407681465148926,
      "activations/layer1_attention_weight_min": -16.012685775756836,
      "activations/layer20_attention_weight_max": 25.5526123046875,
      "activations/layer20_attention_weight_min": -19.895835876464844,
      "activations/layer21_attention_weight_max": 38.018104553222656,
      "activations/layer21_attention_weight_min": -19.950210571289062,
      "activations/layer22_attention_weight_max": 30.56976890563965,
      "activations/layer22_attention_weight_min": -25.53864288330078,
      "activations/layer23_attention_weight_max": 40.700401306152344,
      "activations/layer23_attention_weight_min": -21.90652084350586,
      "activations/layer2_attention_weight_max": 31.449522018432617,
      "activations/layer2_attention_weight_min": -29.52867889404297,
      "activations/layer3_attention_weight_max": 90.03740692138672,
      "activations/layer3_attention_weight_min": -88.11741638183594,
      "activations/layer4_attention_weight_max": 86.95642852783203,
      "activations/layer4_attention_weight_min": -82.35739135742188,
      "activations/layer5_attention_weight_max": 62.68673324584961,
      "activations/layer5_attention_weight_min": -72.21087646484375,
      "activations/layer6_attention_weight_max": 51.22154998779297,
      "activations/layer6_attention_weight_min": -53.457637786865234,
      "activations/layer7_attention_weight_max": 73.84982299804688,
      "activations/layer7_attention_weight_min": -68.33173370361328,
      "activations/layer8_attention_weight_max": 50.664066314697266,
      "activations/layer8_attention_weight_min": -50.30818176269531,
      "activations/layer9_attention_weight_max": 50.424903869628906,
      "activations/layer9_attention_weight_min": -46.01327896118164,
      "epoch": 22.44,
      "learning_rate": 5.324621212121212e-06,
      "loss": 2.6983,
      "step": 386250
    },
    {
      "activations/layer0_attention_weight_max": 16.82773208618164,
      "activations/layer0_attention_weight_min": -14.220863342285156,
      "activations/layer10_attention_weight_max": 34.728302001953125,
      "activations/layer10_attention_weight_min": -36.220619201660156,
      "activations/layer11_attention_weight_max": 31.643253326416016,
      "activations/layer11_attention_weight_min": -34.36650085449219,
      "activations/layer12_attention_weight_max": 17.874099731445312,
      "activations/layer12_attention_weight_min": -24.370378494262695,
      "activations/layer13_attention_weight_max": 30.954790115356445,
      "activations/layer13_attention_weight_min": -25.730037689208984,
      "activations/layer14_attention_weight_max": 32.049747467041016,
      "activations/layer14_attention_weight_min": -28.661685943603516,
      "activations/layer15_attention_weight_max": 29.4471492767334,
      "activations/layer15_attention_weight_min": -28.347373962402344,
      "activations/layer16_attention_weight_max": 30.989469528198242,
      "activations/layer16_attention_weight_min": -29.714460372924805,
      "activations/layer17_attention_weight_max": 46.66438293457031,
      "activations/layer17_attention_weight_min": -41.798274993896484,
      "activations/layer18_attention_weight_max": 41.9969596862793,
      "activations/layer18_attention_weight_min": -37.78713607788086,
      "activations/layer19_attention_weight_max": 25.540775299072266,
      "activations/layer19_attention_weight_min": -23.73645782470703,
      "activations/layer1_attention_weight_max": 14.487010955810547,
      "activations/layer1_attention_weight_min": -14.055094718933105,
      "activations/layer20_attention_weight_max": 21.747575759887695,
      "activations/layer20_attention_weight_min": -22.31089210510254,
      "activations/layer21_attention_weight_max": 33.99232864379883,
      "activations/layer21_attention_weight_min": -21.328899383544922,
      "activations/layer22_attention_weight_max": 26.302780151367188,
      "activations/layer22_attention_weight_min": -24.61345100402832,
      "activations/layer23_attention_weight_max": 35.9006462097168,
      "activations/layer23_attention_weight_min": -23.359785079956055,
      "activations/layer2_attention_weight_max": 31.61196517944336,
      "activations/layer2_attention_weight_min": -32.29920196533203,
      "activations/layer3_attention_weight_max": 88.63584899902344,
      "activations/layer3_attention_weight_min": -91.88350677490234,
      "activations/layer4_attention_weight_max": 82.76659393310547,
      "activations/layer4_attention_weight_min": -89.64013671875,
      "activations/layer5_attention_weight_max": 61.50342559814453,
      "activations/layer5_attention_weight_min": -75.45280456542969,
      "activations/layer6_attention_weight_max": 48.065773010253906,
      "activations/layer6_attention_weight_min": -49.16822052001953,
      "activations/layer7_attention_weight_max": 64.79762268066406,
      "activations/layer7_attention_weight_min": -72.01293182373047,
      "activations/layer8_attention_weight_max": 49.17973709106445,
      "activations/layer8_attention_weight_min": -52.02447509765625,
      "activations/layer9_attention_weight_max": 43.61513137817383,
      "activations/layer9_attention_weight_min": -46.22834014892578,
      "epoch": 22.45,
      "learning_rate": 5.305681818181818e-06,
      "loss": 2.7081,
      "step": 386300
    },
    {
      "activations/layer0_attention_weight_max": 15.424572944641113,
      "activations/layer0_attention_weight_min": -14.333182334899902,
      "activations/layer10_attention_weight_max": 40.28044891357422,
      "activations/layer10_attention_weight_min": -36.77425765991211,
      "activations/layer11_attention_weight_max": 36.069541931152344,
      "activations/layer11_attention_weight_min": -35.03721618652344,
      "activations/layer12_attention_weight_max": 19.503419876098633,
      "activations/layer12_attention_weight_min": -23.731779098510742,
      "activations/layer13_attention_weight_max": 35.577274322509766,
      "activations/layer13_attention_weight_min": -30.861629486083984,
      "activations/layer14_attention_weight_max": 37.2050666809082,
      "activations/layer14_attention_weight_min": -31.747852325439453,
      "activations/layer15_attention_weight_max": 36.18146514892578,
      "activations/layer15_attention_weight_min": -32.4184684753418,
      "activations/layer16_attention_weight_max": 35.72409439086914,
      "activations/layer16_attention_weight_min": -36.508480072021484,
      "activations/layer17_attention_weight_max": 53.87845230102539,
      "activations/layer17_attention_weight_min": -50.77275085449219,
      "activations/layer18_attention_weight_max": 51.58089065551758,
      "activations/layer18_attention_weight_min": -45.276248931884766,
      "activations/layer19_attention_weight_max": 24.907011032104492,
      "activations/layer19_attention_weight_min": -24.078771591186523,
      "activations/layer1_attention_weight_max": 15.156620025634766,
      "activations/layer1_attention_weight_min": -14.671518325805664,
      "activations/layer20_attention_weight_max": 24.058855056762695,
      "activations/layer20_attention_weight_min": -21.554290771484375,
      "activations/layer21_attention_weight_max": 40.125831604003906,
      "activations/layer21_attention_weight_min": -24.900543212890625,
      "activations/layer22_attention_weight_max": 29.915164947509766,
      "activations/layer22_attention_weight_min": -25.0781307220459,
      "activations/layer23_attention_weight_max": 37.7292366027832,
      "activations/layer23_attention_weight_min": -23.677547454833984,
      "activations/layer2_attention_weight_max": 32.1646842956543,
      "activations/layer2_attention_weight_min": -30.85114288330078,
      "activations/layer3_attention_weight_max": 92.27330017089844,
      "activations/layer3_attention_weight_min": -90.05078887939453,
      "activations/layer4_attention_weight_max": 88.29419708251953,
      "activations/layer4_attention_weight_min": -85.57730102539062,
      "activations/layer5_attention_weight_max": 61.20149230957031,
      "activations/layer5_attention_weight_min": -72.80203247070312,
      "activations/layer6_attention_weight_max": 49.93367004394531,
      "activations/layer6_attention_weight_min": -53.410911560058594,
      "activations/layer7_attention_weight_max": 71.86811065673828,
      "activations/layer7_attention_weight_min": -70.12438201904297,
      "activations/layer8_attention_weight_max": 52.006263732910156,
      "activations/layer8_attention_weight_min": -52.460838317871094,
      "activations/layer9_attention_weight_max": 47.19618606567383,
      "activations/layer9_attention_weight_min": -47.50704574584961,
      "epoch": 22.45,
      "learning_rate": 5.286742424242424e-06,
      "loss": 2.6885,
      "step": 386350
    },
    {
      "activations/layer0_attention_weight_max": 16.51938247680664,
      "activations/layer0_attention_weight_min": -14.310090065002441,
      "activations/layer10_attention_weight_max": 35.20228958129883,
      "activations/layer10_attention_weight_min": -37.694583892822266,
      "activations/layer11_attention_weight_max": 32.86444854736328,
      "activations/layer11_attention_weight_min": -36.171348571777344,
      "activations/layer12_attention_weight_max": 19.394681930541992,
      "activations/layer12_attention_weight_min": -24.5943546295166,
      "activations/layer13_attention_weight_max": 34.442195892333984,
      "activations/layer13_attention_weight_min": -28.22906494140625,
      "activations/layer14_attention_weight_max": 36.426177978515625,
      "activations/layer14_attention_weight_min": -31.430971145629883,
      "activations/layer15_attention_weight_max": 34.38553237915039,
      "activations/layer15_attention_weight_min": -31.174484252929688,
      "activations/layer16_attention_weight_max": 35.15971374511719,
      "activations/layer16_attention_weight_min": -34.00105285644531,
      "activations/layer17_attention_weight_max": 51.57988357543945,
      "activations/layer17_attention_weight_min": -47.886775970458984,
      "activations/layer18_attention_weight_max": 47.85002517700195,
      "activations/layer18_attention_weight_min": -40.71610641479492,
      "activations/layer19_attention_weight_max": 24.864736557006836,
      "activations/layer19_attention_weight_min": -23.92740249633789,
      "activations/layer1_attention_weight_max": 15.043414115905762,
      "activations/layer1_attention_weight_min": -14.616316795349121,
      "activations/layer20_attention_weight_max": 23.429590225219727,
      "activations/layer20_attention_weight_min": -21.04251480102539,
      "activations/layer21_attention_weight_max": 36.56004333496094,
      "activations/layer21_attention_weight_min": -24.24443244934082,
      "activations/layer22_attention_weight_max": 27.271408081054688,
      "activations/layer22_attention_weight_min": -24.670793533325195,
      "activations/layer23_attention_weight_max": 37.00074005126953,
      "activations/layer23_attention_weight_min": -27.0443115234375,
      "activations/layer2_attention_weight_max": 31.319570541381836,
      "activations/layer2_attention_weight_min": -30.609539031982422,
      "activations/layer3_attention_weight_max": 89.73893737792969,
      "activations/layer3_attention_weight_min": -91.81495666503906,
      "activations/layer4_attention_weight_max": 84.6869888305664,
      "activations/layer4_attention_weight_min": -88.88973236083984,
      "activations/layer5_attention_weight_max": 60.27893829345703,
      "activations/layer5_attention_weight_min": -73.37371826171875,
      "activations/layer6_attention_weight_max": 50.254844665527344,
      "activations/layer6_attention_weight_min": -51.25193405151367,
      "activations/layer7_attention_weight_max": 69.41986083984375,
      "activations/layer7_attention_weight_min": -68.16348266601562,
      "activations/layer8_attention_weight_max": 46.6081428527832,
      "activations/layer8_attention_weight_min": -50.05337142944336,
      "activations/layer9_attention_weight_max": 46.34914779663086,
      "activations/layer9_attention_weight_min": -48.20527648925781,
      "epoch": 22.45,
      "learning_rate": 5.26780303030303e-06,
      "loss": 2.7025,
      "step": 386400
    },
    {
      "activations/layer0_attention_weight_max": 14.98629093170166,
      "activations/layer0_attention_weight_min": -14.125256538391113,
      "activations/layer10_attention_weight_max": 34.30364990234375,
      "activations/layer10_attention_weight_min": -34.031089782714844,
      "activations/layer11_attention_weight_max": 32.03570556640625,
      "activations/layer11_attention_weight_min": -35.349884033203125,
      "activations/layer12_attention_weight_max": 19.737613677978516,
      "activations/layer12_attention_weight_min": -25.385337829589844,
      "activations/layer13_attention_weight_max": 37.575767517089844,
      "activations/layer13_attention_weight_min": -30.32513427734375,
      "activations/layer14_attention_weight_max": 36.12306594848633,
      "activations/layer14_attention_weight_min": -32.34435272216797,
      "activations/layer15_attention_weight_max": 33.5634880065918,
      "activations/layer15_attention_weight_min": -31.205881118774414,
      "activations/layer16_attention_weight_max": 35.22998809814453,
      "activations/layer16_attention_weight_min": -34.472808837890625,
      "activations/layer17_attention_weight_max": 54.1982536315918,
      "activations/layer17_attention_weight_min": -48.50374984741211,
      "activations/layer18_attention_weight_max": 49.54720687866211,
      "activations/layer18_attention_weight_min": -40.393157958984375,
      "activations/layer19_attention_weight_max": 25.812231063842773,
      "activations/layer19_attention_weight_min": -25.163591384887695,
      "activations/layer1_attention_weight_max": 15.240347862243652,
      "activations/layer1_attention_weight_min": -14.439177513122559,
      "activations/layer20_attention_weight_max": 27.377296447753906,
      "activations/layer20_attention_weight_min": -23.872655868530273,
      "activations/layer21_attention_weight_max": 43.28007888793945,
      "activations/layer21_attention_weight_min": -24.813261032104492,
      "activations/layer22_attention_weight_max": 30.211090087890625,
      "activations/layer22_attention_weight_min": -25.795625686645508,
      "activations/layer23_attention_weight_max": 37.45909118652344,
      "activations/layer23_attention_weight_min": -24.094966888427734,
      "activations/layer2_attention_weight_max": 31.07795524597168,
      "activations/layer2_attention_weight_min": -30.33753204345703,
      "activations/layer3_attention_weight_max": 89.7208023071289,
      "activations/layer3_attention_weight_min": -86.63668823242188,
      "activations/layer4_attention_weight_max": 85.53700256347656,
      "activations/layer4_attention_weight_min": -85.16194152832031,
      "activations/layer5_attention_weight_max": 60.88065719604492,
      "activations/layer5_attention_weight_min": -70.91439056396484,
      "activations/layer6_attention_weight_max": 50.13846206665039,
      "activations/layer6_attention_weight_min": -52.96974563598633,
      "activations/layer7_attention_weight_max": 70.07713317871094,
      "activations/layer7_attention_weight_min": -72.01779174804688,
      "activations/layer8_attention_weight_max": 45.52851104736328,
      "activations/layer8_attention_weight_min": -51.21864700317383,
      "activations/layer9_attention_weight_max": 43.39474105834961,
      "activations/layer9_attention_weight_min": -47.58554458618164,
      "epoch": 22.45,
      "learning_rate": 5.248863636363636e-06,
      "loss": 2.6824,
      "step": 386450
    },
    {
      "activations/layer0_attention_weight_max": 15.861039161682129,
      "activations/layer0_attention_weight_min": -14.486157417297363,
      "activations/layer10_attention_weight_max": 35.61635971069336,
      "activations/layer10_attention_weight_min": -34.880592346191406,
      "activations/layer11_attention_weight_max": 34.54338073730469,
      "activations/layer11_attention_weight_min": -34.66719436645508,
      "activations/layer12_attention_weight_max": 19.13789939880371,
      "activations/layer12_attention_weight_min": -27.707918167114258,
      "activations/layer13_attention_weight_max": 35.9940299987793,
      "activations/layer13_attention_weight_min": -27.819477081298828,
      "activations/layer14_attention_weight_max": 36.92832565307617,
      "activations/layer14_attention_weight_min": -31.353431701660156,
      "activations/layer15_attention_weight_max": 35.516807556152344,
      "activations/layer15_attention_weight_min": -30.76510238647461,
      "activations/layer16_attention_weight_max": 35.46395492553711,
      "activations/layer16_attention_weight_min": -33.51707077026367,
      "activations/layer17_attention_weight_max": 53.74455642700195,
      "activations/layer17_attention_weight_min": -46.90056228637695,
      "activations/layer18_attention_weight_max": 49.15729904174805,
      "activations/layer18_attention_weight_min": -39.25648498535156,
      "activations/layer19_attention_weight_max": 26.021432876586914,
      "activations/layer19_attention_weight_min": -22.718856811523438,
      "activations/layer1_attention_weight_max": 15.919392585754395,
      "activations/layer1_attention_weight_min": -15.072981834411621,
      "activations/layer20_attention_weight_max": 23.716947555541992,
      "activations/layer20_attention_weight_min": -22.483633041381836,
      "activations/layer21_attention_weight_max": 43.22553253173828,
      "activations/layer21_attention_weight_min": -23.69480323791504,
      "activations/layer22_attention_weight_max": 31.07529067993164,
      "activations/layer22_attention_weight_min": -25.410316467285156,
      "activations/layer23_attention_weight_max": 43.28327941894531,
      "activations/layer23_attention_weight_min": -24.729541778564453,
      "activations/layer2_attention_weight_max": 32.18607711791992,
      "activations/layer2_attention_weight_min": -30.301603317260742,
      "activations/layer3_attention_weight_max": 92.94935607910156,
      "activations/layer3_attention_weight_min": -92.95116424560547,
      "activations/layer4_attention_weight_max": 86.98333740234375,
      "activations/layer4_attention_weight_min": -88.50978088378906,
      "activations/layer5_attention_weight_max": 65.382080078125,
      "activations/layer5_attention_weight_min": -71.45561218261719,
      "activations/layer6_attention_weight_max": 51.62833786010742,
      "activations/layer6_attention_weight_min": -50.81275177001953,
      "activations/layer7_attention_weight_max": 69.13880157470703,
      "activations/layer7_attention_weight_min": -72.3613510131836,
      "activations/layer8_attention_weight_max": 48.870079040527344,
      "activations/layer8_attention_weight_min": -52.24736404418945,
      "activations/layer9_attention_weight_max": 45.79069900512695,
      "activations/layer9_attention_weight_min": -46.89649200439453,
      "epoch": 22.46,
      "learning_rate": 5.229924242424242e-06,
      "loss": 2.7024,
      "step": 386500
    },
    {
      "activations/layer0_attention_weight_max": 15.586082458496094,
      "activations/layer0_attention_weight_min": -14.2151460647583,
      "activations/layer10_attention_weight_max": 35.5270881652832,
      "activations/layer10_attention_weight_min": -36.48426818847656,
      "activations/layer11_attention_weight_max": 33.70823669433594,
      "activations/layer11_attention_weight_min": -33.71194839477539,
      "activations/layer12_attention_weight_max": 19.214284896850586,
      "activations/layer12_attention_weight_min": -24.18061065673828,
      "activations/layer13_attention_weight_max": 29.643083572387695,
      "activations/layer13_attention_weight_min": -28.999879837036133,
      "activations/layer14_attention_weight_max": 34.71162796020508,
      "activations/layer14_attention_weight_min": -30.558629989624023,
      "activations/layer15_attention_weight_max": 34.27915573120117,
      "activations/layer15_attention_weight_min": -30.51361656188965,
      "activations/layer16_attention_weight_max": 35.00536346435547,
      "activations/layer16_attention_weight_min": -32.937320709228516,
      "activations/layer17_attention_weight_max": 51.8233528137207,
      "activations/layer17_attention_weight_min": -47.4582405090332,
      "activations/layer18_attention_weight_max": 44.98981857299805,
      "activations/layer18_attention_weight_min": -40.849056243896484,
      "activations/layer19_attention_weight_max": 24.17340660095215,
      "activations/layer19_attention_weight_min": -24.133657455444336,
      "activations/layer1_attention_weight_max": 14.824061393737793,
      "activations/layer1_attention_weight_min": -15.038508415222168,
      "activations/layer20_attention_weight_max": 21.42136001586914,
      "activations/layer20_attention_weight_min": -22.626237869262695,
      "activations/layer21_attention_weight_max": 32.12502670288086,
      "activations/layer21_attention_weight_min": -22.914047241210938,
      "activations/layer22_attention_weight_max": 28.285432815551758,
      "activations/layer22_attention_weight_min": -24.83149528503418,
      "activations/layer23_attention_weight_max": 33.35029602050781,
      "activations/layer23_attention_weight_min": -24.899803161621094,
      "activations/layer2_attention_weight_max": 31.56775665283203,
      "activations/layer2_attention_weight_min": -30.801849365234375,
      "activations/layer3_attention_weight_max": 91.33900451660156,
      "activations/layer3_attention_weight_min": -90.77299499511719,
      "activations/layer4_attention_weight_max": 85.58407592773438,
      "activations/layer4_attention_weight_min": -86.73764038085938,
      "activations/layer5_attention_weight_max": 62.81617736816406,
      "activations/layer5_attention_weight_min": -73.08238220214844,
      "activations/layer6_attention_weight_max": 50.21259307861328,
      "activations/layer6_attention_weight_min": -49.518436431884766,
      "activations/layer7_attention_weight_max": 71.0642318725586,
      "activations/layer7_attention_weight_min": -70.07201385498047,
      "activations/layer8_attention_weight_max": 46.01201248168945,
      "activations/layer8_attention_weight_min": -51.5017204284668,
      "activations/layer9_attention_weight_max": 45.284027099609375,
      "activations/layer9_attention_weight_min": -44.76258087158203,
      "epoch": 22.46,
      "learning_rate": 5.2109848484848474e-06,
      "loss": 2.6879,
      "step": 386550
    },
    {
      "activations/layer0_attention_weight_max": 15.79814338684082,
      "activations/layer0_attention_weight_min": -14.428718566894531,
      "activations/layer10_attention_weight_max": 35.875457763671875,
      "activations/layer10_attention_weight_min": -35.95162582397461,
      "activations/layer11_attention_weight_max": 33.07268142700195,
      "activations/layer11_attention_weight_min": -37.642433166503906,
      "activations/layer12_attention_weight_max": 18.859933853149414,
      "activations/layer12_attention_weight_min": -31.68769073486328,
      "activations/layer13_attention_weight_max": 30.8692684173584,
      "activations/layer13_attention_weight_min": -27.498546600341797,
      "activations/layer14_attention_weight_max": 33.57159423828125,
      "activations/layer14_attention_weight_min": -28.898677825927734,
      "activations/layer15_attention_weight_max": 33.8323860168457,
      "activations/layer15_attention_weight_min": -30.480876922607422,
      "activations/layer16_attention_weight_max": 33.44855499267578,
      "activations/layer16_attention_weight_min": -32.18217849731445,
      "activations/layer17_attention_weight_max": 51.720619201660156,
      "activations/layer17_attention_weight_min": -42.921207427978516,
      "activations/layer18_attention_weight_max": 46.74263000488281,
      "activations/layer18_attention_weight_min": -39.32194137573242,
      "activations/layer19_attention_weight_max": 23.587512969970703,
      "activations/layer19_attention_weight_min": -21.100492477416992,
      "activations/layer1_attention_weight_max": 16.464616775512695,
      "activations/layer1_attention_weight_min": -13.928282737731934,
      "activations/layer20_attention_weight_max": 21.84208869934082,
      "activations/layer20_attention_weight_min": -23.144065856933594,
      "activations/layer21_attention_weight_max": 34.00672912597656,
      "activations/layer21_attention_weight_min": -22.388402938842773,
      "activations/layer22_attention_weight_max": 29.529369354248047,
      "activations/layer22_attention_weight_min": -25.767398834228516,
      "activations/layer23_attention_weight_max": 32.67549133300781,
      "activations/layer23_attention_weight_min": -24.585378646850586,
      "activations/layer2_attention_weight_max": 32.065513610839844,
      "activations/layer2_attention_weight_min": -31.520156860351562,
      "activations/layer3_attention_weight_max": 89.79405975341797,
      "activations/layer3_attention_weight_min": -90.12757110595703,
      "activations/layer4_attention_weight_max": 84.58981323242188,
      "activations/layer4_attention_weight_min": -86.01909637451172,
      "activations/layer5_attention_weight_max": 61.17916488647461,
      "activations/layer5_attention_weight_min": -69.22674560546875,
      "activations/layer6_attention_weight_max": 50.38974380493164,
      "activations/layer6_attention_weight_min": -49.93926239013672,
      "activations/layer7_attention_weight_max": 68.27078247070312,
      "activations/layer7_attention_weight_min": -65.89350128173828,
      "activations/layer8_attention_weight_max": 47.067893981933594,
      "activations/layer8_attention_weight_min": -53.05048751831055,
      "activations/layer9_attention_weight_max": 47.24060821533203,
      "activations/layer9_attention_weight_min": -47.453216552734375,
      "epoch": 22.46,
      "learning_rate": 5.192045454545454e-06,
      "loss": 2.6904,
      "step": 386600
    },
    {
      "activations/layer0_attention_weight_max": 16.467205047607422,
      "activations/layer0_attention_weight_min": -14.284211158752441,
      "activations/layer10_attention_weight_max": 35.57624816894531,
      "activations/layer10_attention_weight_min": -36.50251007080078,
      "activations/layer11_attention_weight_max": 34.083709716796875,
      "activations/layer11_attention_weight_min": -34.481712341308594,
      "activations/layer12_attention_weight_max": 18.23810386657715,
      "activations/layer12_attention_weight_min": -29.040481567382812,
      "activations/layer13_attention_weight_max": 33.04723358154297,
      "activations/layer13_attention_weight_min": -28.567153930664062,
      "activations/layer14_attention_weight_max": 34.0533561706543,
      "activations/layer14_attention_weight_min": -31.287296295166016,
      "activations/layer15_attention_weight_max": 32.711509704589844,
      "activations/layer15_attention_weight_min": -31.091073989868164,
      "activations/layer16_attention_weight_max": 33.14484786987305,
      "activations/layer16_attention_weight_min": -32.99795150756836,
      "activations/layer17_attention_weight_max": 50.844337463378906,
      "activations/layer17_attention_weight_min": -47.22785568237305,
      "activations/layer18_attention_weight_max": 49.54938888549805,
      "activations/layer18_attention_weight_min": -41.51943588256836,
      "activations/layer19_attention_weight_max": 25.10449981689453,
      "activations/layer19_attention_weight_min": -23.981130599975586,
      "activations/layer1_attention_weight_max": 15.354202270507812,
      "activations/layer1_attention_weight_min": -15.164937019348145,
      "activations/layer20_attention_weight_max": 25.352636337280273,
      "activations/layer20_attention_weight_min": -23.30864715576172,
      "activations/layer21_attention_weight_max": 33.9581413269043,
      "activations/layer21_attention_weight_min": -22.08401870727539,
      "activations/layer22_attention_weight_max": 32.45008850097656,
      "activations/layer22_attention_weight_min": -28.578516006469727,
      "activations/layer23_attention_weight_max": 39.34130096435547,
      "activations/layer23_attention_weight_min": -27.062335968017578,
      "activations/layer2_attention_weight_max": 32.38816833496094,
      "activations/layer2_attention_weight_min": -31.17125701904297,
      "activations/layer3_attention_weight_max": 89.77377319335938,
      "activations/layer3_attention_weight_min": -92.39874267578125,
      "activations/layer4_attention_weight_max": 85.34526062011719,
      "activations/layer4_attention_weight_min": -85.94001770019531,
      "activations/layer5_attention_weight_max": 61.88629150390625,
      "activations/layer5_attention_weight_min": -75.90693664550781,
      "activations/layer6_attention_weight_max": 51.16405487060547,
      "activations/layer6_attention_weight_min": -53.301151275634766,
      "activations/layer7_attention_weight_max": 71.71258544921875,
      "activations/layer7_attention_weight_min": -71.07897186279297,
      "activations/layer8_attention_weight_max": 48.64014434814453,
      "activations/layer8_attention_weight_min": -50.49842071533203,
      "activations/layer9_attention_weight_max": 48.90278244018555,
      "activations/layer9_attention_weight_min": -45.682777404785156,
      "epoch": 22.47,
      "learning_rate": 5.173106060606061e-06,
      "loss": 2.6926,
      "step": 386650
    },
    {
      "activations/layer0_attention_weight_max": 15.421277046203613,
      "activations/layer0_attention_weight_min": -14.13523006439209,
      "activations/layer10_attention_weight_max": 34.067054748535156,
      "activations/layer10_attention_weight_min": -36.25267791748047,
      "activations/layer11_attention_weight_max": 32.72779083251953,
      "activations/layer11_attention_weight_min": -35.890838623046875,
      "activations/layer12_attention_weight_max": 19.205102920532227,
      "activations/layer12_attention_weight_min": -23.120683670043945,
      "activations/layer13_attention_weight_max": 34.38612365722656,
      "activations/layer13_attention_weight_min": -27.407224655151367,
      "activations/layer14_attention_weight_max": 37.628360748291016,
      "activations/layer14_attention_weight_min": -29.902307510375977,
      "activations/layer15_attention_weight_max": 36.436622619628906,
      "activations/layer15_attention_weight_min": -30.92933464050293,
      "activations/layer16_attention_weight_max": 38.38544845581055,
      "activations/layer16_attention_weight_min": -35.073970794677734,
      "activations/layer17_attention_weight_max": 59.95489501953125,
      "activations/layer17_attention_weight_min": -50.29124450683594,
      "activations/layer18_attention_weight_max": 53.126277923583984,
      "activations/layer18_attention_weight_min": -44.50780487060547,
      "activations/layer19_attention_weight_max": 25.54627799987793,
      "activations/layer19_attention_weight_min": -24.63700294494629,
      "activations/layer1_attention_weight_max": 14.631328582763672,
      "activations/layer1_attention_weight_min": -14.993680953979492,
      "activations/layer20_attention_weight_max": 24.46971893310547,
      "activations/layer20_attention_weight_min": -22.312162399291992,
      "activations/layer21_attention_weight_max": 36.614173889160156,
      "activations/layer21_attention_weight_min": -24.940237045288086,
      "activations/layer22_attention_weight_max": 28.1157283782959,
      "activations/layer22_attention_weight_min": -27.21953010559082,
      "activations/layer23_attention_weight_max": 37.91161346435547,
      "activations/layer23_attention_weight_min": -25.642333984375,
      "activations/layer2_attention_weight_max": 31.43153953552246,
      "activations/layer2_attention_weight_min": -29.512042999267578,
      "activations/layer3_attention_weight_max": 91.16085815429688,
      "activations/layer3_attention_weight_min": -87.80403137207031,
      "activations/layer4_attention_weight_max": 86.26859283447266,
      "activations/layer4_attention_weight_min": -86.45055389404297,
      "activations/layer5_attention_weight_max": 62.07990264892578,
      "activations/layer5_attention_weight_min": -72.32537841796875,
      "activations/layer6_attention_weight_max": 49.267723083496094,
      "activations/layer6_attention_weight_min": -51.45710372924805,
      "activations/layer7_attention_weight_max": 69.41101837158203,
      "activations/layer7_attention_weight_min": -69.91629791259766,
      "activations/layer8_attention_weight_max": 49.71179962158203,
      "activations/layer8_attention_weight_min": -52.8255615234375,
      "activations/layer9_attention_weight_max": 44.428218841552734,
      "activations/layer9_attention_weight_min": -46.457542419433594,
      "epoch": 22.47,
      "learning_rate": 5.154166666666666e-06,
      "loss": 2.6991,
      "step": 386700
    },
    {
      "activations/layer0_attention_weight_max": 15.864151000976562,
      "activations/layer0_attention_weight_min": -14.514059066772461,
      "activations/layer10_attention_weight_max": 35.83673858642578,
      "activations/layer10_attention_weight_min": -38.50243377685547,
      "activations/layer11_attention_weight_max": 33.96913146972656,
      "activations/layer11_attention_weight_min": -35.576194763183594,
      "activations/layer12_attention_weight_max": 20.39410400390625,
      "activations/layer12_attention_weight_min": -26.69339370727539,
      "activations/layer13_attention_weight_max": 34.2530403137207,
      "activations/layer13_attention_weight_min": -28.827600479125977,
      "activations/layer14_attention_weight_max": 39.7667350769043,
      "activations/layer14_attention_weight_min": -34.092838287353516,
      "activations/layer15_attention_weight_max": 36.9874382019043,
      "activations/layer15_attention_weight_min": -33.18170166015625,
      "activations/layer16_attention_weight_max": 37.88926696777344,
      "activations/layer16_attention_weight_min": -35.593597412109375,
      "activations/layer17_attention_weight_max": 52.57870864868164,
      "activations/layer17_attention_weight_min": -48.83493423461914,
      "activations/layer18_attention_weight_max": 50.87086486816406,
      "activations/layer18_attention_weight_min": -42.44953536987305,
      "activations/layer19_attention_weight_max": 26.043516159057617,
      "activations/layer19_attention_weight_min": -26.593679428100586,
      "activations/layer1_attention_weight_max": 15.048382759094238,
      "activations/layer1_attention_weight_min": -14.416688919067383,
      "activations/layer20_attention_weight_max": 25.530834197998047,
      "activations/layer20_attention_weight_min": -23.68668556213379,
      "activations/layer21_attention_weight_max": 39.17912292480469,
      "activations/layer21_attention_weight_min": -21.778345108032227,
      "activations/layer22_attention_weight_max": 32.23189926147461,
      "activations/layer22_attention_weight_min": -27.71596908569336,
      "activations/layer23_attention_weight_max": 34.435516357421875,
      "activations/layer23_attention_weight_min": -27.310026168823242,
      "activations/layer2_attention_weight_max": 32.23070526123047,
      "activations/layer2_attention_weight_min": -30.410728454589844,
      "activations/layer3_attention_weight_max": 92.34284973144531,
      "activations/layer3_attention_weight_min": -87.2356185913086,
      "activations/layer4_attention_weight_max": 87.41019439697266,
      "activations/layer4_attention_weight_min": -87.15032196044922,
      "activations/layer5_attention_weight_max": 64.07994079589844,
      "activations/layer5_attention_weight_min": -71.10064697265625,
      "activations/layer6_attention_weight_max": 52.329368591308594,
      "activations/layer6_attention_weight_min": -51.54336929321289,
      "activations/layer7_attention_weight_max": 68.40567016601562,
      "activations/layer7_attention_weight_min": -72.29837799072266,
      "activations/layer8_attention_weight_max": 48.84663772583008,
      "activations/layer8_attention_weight_min": -53.08214569091797,
      "activations/layer9_attention_weight_max": 47.544288635253906,
      "activations/layer9_attention_weight_min": -47.90314865112305,
      "epoch": 22.47,
      "learning_rate": 5.135227272727272e-06,
      "loss": 2.7054,
      "step": 386750
    },
    {
      "activations/layer0_attention_weight_max": 16.80365753173828,
      "activations/layer0_attention_weight_min": -14.4524564743042,
      "activations/layer10_attention_weight_max": 35.24249267578125,
      "activations/layer10_attention_weight_min": -36.01546859741211,
      "activations/layer11_attention_weight_max": 33.626243591308594,
      "activations/layer11_attention_weight_min": -36.432044982910156,
      "activations/layer12_attention_weight_max": 19.511791229248047,
      "activations/layer12_attention_weight_min": -26.502382278442383,
      "activations/layer13_attention_weight_max": 33.70470428466797,
      "activations/layer13_attention_weight_min": -28.15728759765625,
      "activations/layer14_attention_weight_max": 39.84364318847656,
      "activations/layer14_attention_weight_min": -35.01004409790039,
      "activations/layer15_attention_weight_max": 36.187294006347656,
      "activations/layer15_attention_weight_min": -35.993595123291016,
      "activations/layer16_attention_weight_max": 35.78152847290039,
      "activations/layer16_attention_weight_min": -34.202415466308594,
      "activations/layer17_attention_weight_max": 52.44939422607422,
      "activations/layer17_attention_weight_min": -45.84758758544922,
      "activations/layer18_attention_weight_max": 51.339256286621094,
      "activations/layer18_attention_weight_min": -40.04343032836914,
      "activations/layer19_attention_weight_max": 24.738439559936523,
      "activations/layer19_attention_weight_min": -24.19533348083496,
      "activations/layer1_attention_weight_max": 15.55141544342041,
      "activations/layer1_attention_weight_min": -15.487520217895508,
      "activations/layer20_attention_weight_max": 23.17142677307129,
      "activations/layer20_attention_weight_min": -22.427715301513672,
      "activations/layer21_attention_weight_max": 40.22832489013672,
      "activations/layer21_attention_weight_min": -21.889202117919922,
      "activations/layer22_attention_weight_max": 31.529375076293945,
      "activations/layer22_attention_weight_min": -25.22170066833496,
      "activations/layer23_attention_weight_max": 35.75887680053711,
      "activations/layer23_attention_weight_min": -24.302419662475586,
      "activations/layer2_attention_weight_max": 32.16602325439453,
      "activations/layer2_attention_weight_min": -30.872861862182617,
      "activations/layer3_attention_weight_max": 88.39620971679688,
      "activations/layer3_attention_weight_min": -89.52415466308594,
      "activations/layer4_attention_weight_max": 82.95872497558594,
      "activations/layer4_attention_weight_min": -82.0528793334961,
      "activations/layer5_attention_weight_max": 62.388946533203125,
      "activations/layer5_attention_weight_min": -71.73699951171875,
      "activations/layer6_attention_weight_max": 48.50936508178711,
      "activations/layer6_attention_weight_min": -50.53000259399414,
      "activations/layer7_attention_weight_max": 66.57416534423828,
      "activations/layer7_attention_weight_min": -66.5333251953125,
      "activations/layer8_attention_weight_max": 45.5204963684082,
      "activations/layer8_attention_weight_min": -51.44654846191406,
      "activations/layer9_attention_weight_max": 44.236270904541016,
      "activations/layer9_attention_weight_min": -46.00071716308594,
      "epoch": 22.48,
      "learning_rate": 5.116287878787879e-06,
      "loss": 2.6946,
      "step": 386800
    },
    {
      "activations/layer0_attention_weight_max": 16.17165756225586,
      "activations/layer0_attention_weight_min": -14.275763511657715,
      "activations/layer10_attention_weight_max": 36.18834686279297,
      "activations/layer10_attention_weight_min": -36.64691925048828,
      "activations/layer11_attention_weight_max": 35.69980239868164,
      "activations/layer11_attention_weight_min": -39.02420425415039,
      "activations/layer12_attention_weight_max": 19.730314254760742,
      "activations/layer12_attention_weight_min": -25.689268112182617,
      "activations/layer13_attention_weight_max": 34.644351959228516,
      "activations/layer13_attention_weight_min": -28.47482681274414,
      "activations/layer14_attention_weight_max": 35.79106521606445,
      "activations/layer14_attention_weight_min": -30.4322509765625,
      "activations/layer15_attention_weight_max": 35.7921028137207,
      "activations/layer15_attention_weight_min": -30.08241081237793,
      "activations/layer16_attention_weight_max": 35.453269958496094,
      "activations/layer16_attention_weight_min": -33.88266372680664,
      "activations/layer17_attention_weight_max": 50.32625198364258,
      "activations/layer17_attention_weight_min": -45.416534423828125,
      "activations/layer18_attention_weight_max": 47.711158752441406,
      "activations/layer18_attention_weight_min": -41.03300476074219,
      "activations/layer19_attention_weight_max": 27.84273338317871,
      "activations/layer19_attention_weight_min": -22.30267906188965,
      "activations/layer1_attention_weight_max": 15.28068733215332,
      "activations/layer1_attention_weight_min": -14.597050666809082,
      "activations/layer20_attention_weight_max": 21.46640396118164,
      "activations/layer20_attention_weight_min": -23.802160263061523,
      "activations/layer21_attention_weight_max": 33.114845275878906,
      "activations/layer21_attention_weight_min": -20.00833511352539,
      "activations/layer22_attention_weight_max": 31.88991355895996,
      "activations/layer22_attention_weight_min": -26.3876953125,
      "activations/layer23_attention_weight_max": 39.26298141479492,
      "activations/layer23_attention_weight_min": -21.550514221191406,
      "activations/layer2_attention_weight_max": 32.01918029785156,
      "activations/layer2_attention_weight_min": -29.85936737060547,
      "activations/layer3_attention_weight_max": 89.6097412109375,
      "activations/layer3_attention_weight_min": -88.61927795410156,
      "activations/layer4_attention_weight_max": 83.74236297607422,
      "activations/layer4_attention_weight_min": -86.2406234741211,
      "activations/layer5_attention_weight_max": 64.02560424804688,
      "activations/layer5_attention_weight_min": -78.27013397216797,
      "activations/layer6_attention_weight_max": 48.212711334228516,
      "activations/layer6_attention_weight_min": -50.44355773925781,
      "activations/layer7_attention_weight_max": 73.67904663085938,
      "activations/layer7_attention_weight_min": -72.42171478271484,
      "activations/layer8_attention_weight_max": 48.15150451660156,
      "activations/layer8_attention_weight_min": -50.88884353637695,
      "activations/layer9_attention_weight_max": 44.443416595458984,
      "activations/layer9_attention_weight_min": -48.112491607666016,
      "epoch": 22.48,
      "learning_rate": 5.097348484848484e-06,
      "loss": 2.6908,
      "step": 386850
    },
    {
      "activations/layer0_attention_weight_max": 15.728489875793457,
      "activations/layer0_attention_weight_min": -14.265995979309082,
      "activations/layer10_attention_weight_max": 36.647613525390625,
      "activations/layer10_attention_weight_min": -36.866485595703125,
      "activations/layer11_attention_weight_max": 34.593406677246094,
      "activations/layer11_attention_weight_min": -35.67449951171875,
      "activations/layer12_attention_weight_max": 19.04006004333496,
      "activations/layer12_attention_weight_min": -24.96435546875,
      "activations/layer13_attention_weight_max": 35.31755065917969,
      "activations/layer13_attention_weight_min": -30.763063430786133,
      "activations/layer14_attention_weight_max": 39.87953186035156,
      "activations/layer14_attention_weight_min": -31.20363998413086,
      "activations/layer15_attention_weight_max": 34.93832015991211,
      "activations/layer15_attention_weight_min": -32.3154182434082,
      "activations/layer16_attention_weight_max": 35.6761360168457,
      "activations/layer16_attention_weight_min": -33.02434158325195,
      "activations/layer17_attention_weight_max": 50.85092544555664,
      "activations/layer17_attention_weight_min": -44.47432327270508,
      "activations/layer18_attention_weight_max": 51.489349365234375,
      "activations/layer18_attention_weight_min": -40.075435638427734,
      "activations/layer19_attention_weight_max": 26.969715118408203,
      "activations/layer19_attention_weight_min": -22.156240463256836,
      "activations/layer1_attention_weight_max": 15.551515579223633,
      "activations/layer1_attention_weight_min": -14.246405601501465,
      "activations/layer20_attention_weight_max": 24.07073974609375,
      "activations/layer20_attention_weight_min": -20.558725357055664,
      "activations/layer21_attention_weight_max": 32.95377731323242,
      "activations/layer21_attention_weight_min": -19.4507999420166,
      "activations/layer22_attention_weight_max": 26.72994041442871,
      "activations/layer22_attention_weight_min": -25.314741134643555,
      "activations/layer23_attention_weight_max": 34.933284759521484,
      "activations/layer23_attention_weight_min": -23.715167999267578,
      "activations/layer2_attention_weight_max": 32.40385437011719,
      "activations/layer2_attention_weight_min": -30.908309936523438,
      "activations/layer3_attention_weight_max": 89.18633270263672,
      "activations/layer3_attention_weight_min": -91.39885711669922,
      "activations/layer4_attention_weight_max": 82.7798080444336,
      "activations/layer4_attention_weight_min": -84.63541412353516,
      "activations/layer5_attention_weight_max": 62.01020431518555,
      "activations/layer5_attention_weight_min": -69.55237579345703,
      "activations/layer6_attention_weight_max": 49.567848205566406,
      "activations/layer6_attention_weight_min": -49.91981506347656,
      "activations/layer7_attention_weight_max": 69.21944427490234,
      "activations/layer7_attention_weight_min": -70.82077026367188,
      "activations/layer8_attention_weight_max": 47.877723693847656,
      "activations/layer8_attention_weight_min": -52.549346923828125,
      "activations/layer9_attention_weight_max": 44.40275192260742,
      "activations/layer9_attention_weight_min": -48.960445404052734,
      "epoch": 22.48,
      "learning_rate": 5.07840909090909e-06,
      "loss": 2.6933,
      "step": 386900
    },
    {
      "activations/layer0_attention_weight_max": 16.311758041381836,
      "activations/layer0_attention_weight_min": -14.341408729553223,
      "activations/layer10_attention_weight_max": 35.697628021240234,
      "activations/layer10_attention_weight_min": -36.53998947143555,
      "activations/layer11_attention_weight_max": 31.171072006225586,
      "activations/layer11_attention_weight_min": -34.976707458496094,
      "activations/layer12_attention_weight_max": 17.745824813842773,
      "activations/layer12_attention_weight_min": -26.71912956237793,
      "activations/layer13_attention_weight_max": 36.28923034667969,
      "activations/layer13_attention_weight_min": -28.64066505432129,
      "activations/layer14_attention_weight_max": 39.70800018310547,
      "activations/layer14_attention_weight_min": -31.88542938232422,
      "activations/layer15_attention_weight_max": 41.21889877319336,
      "activations/layer15_attention_weight_min": -32.0935173034668,
      "activations/layer16_attention_weight_max": 42.34318161010742,
      "activations/layer16_attention_weight_min": -39.376930236816406,
      "activations/layer17_attention_weight_max": 64.5754623413086,
      "activations/layer17_attention_weight_min": -51.664024353027344,
      "activations/layer18_attention_weight_max": 53.33909606933594,
      "activations/layer18_attention_weight_min": -43.795745849609375,
      "activations/layer19_attention_weight_max": 29.32012367248535,
      "activations/layer19_attention_weight_min": -25.670866012573242,
      "activations/layer1_attention_weight_max": 15.860575675964355,
      "activations/layer1_attention_weight_min": -15.124424934387207,
      "activations/layer20_attention_weight_max": 29.1717472076416,
      "activations/layer20_attention_weight_min": -23.074922561645508,
      "activations/layer21_attention_weight_max": 45.12813949584961,
      "activations/layer21_attention_weight_min": -22.73793601989746,
      "activations/layer22_attention_weight_max": 35.67329025268555,
      "activations/layer22_attention_weight_min": -25.985681533813477,
      "activations/layer23_attention_weight_max": 37.31987762451172,
      "activations/layer23_attention_weight_min": -24.0225887298584,
      "activations/layer2_attention_weight_max": 31.756938934326172,
      "activations/layer2_attention_weight_min": -31.711597442626953,
      "activations/layer3_attention_weight_max": 90.91878509521484,
      "activations/layer3_attention_weight_min": -91.51384735107422,
      "activations/layer4_attention_weight_max": 85.49874877929688,
      "activations/layer4_attention_weight_min": -86.66598510742188,
      "activations/layer5_attention_weight_max": 60.61479187011719,
      "activations/layer5_attention_weight_min": -72.08536529541016,
      "activations/layer6_attention_weight_max": 47.861900329589844,
      "activations/layer6_attention_weight_min": -50.45248031616211,
      "activations/layer7_attention_weight_max": 67.0441665649414,
      "activations/layer7_attention_weight_min": -66.89338684082031,
      "activations/layer8_attention_weight_max": 45.30322265625,
      "activations/layer8_attention_weight_min": -50.78994369506836,
      "activations/layer9_attention_weight_max": 43.78319549560547,
      "activations/layer9_attention_weight_min": -47.53804397583008,
      "epoch": 22.48,
      "learning_rate": 5.059469696969696e-06,
      "loss": 2.7072,
      "step": 386950
    },
    {
      "activations/layer0_attention_weight_max": 16.551563262939453,
      "activations/layer0_attention_weight_min": -14.598165512084961,
      "activations/layer10_attention_weight_max": 36.09749221801758,
      "activations/layer10_attention_weight_min": -35.892574310302734,
      "activations/layer11_attention_weight_max": 34.979026794433594,
      "activations/layer11_attention_weight_min": -36.48291015625,
      "activations/layer12_attention_weight_max": 19.462310791015625,
      "activations/layer12_attention_weight_min": -26.482654571533203,
      "activations/layer13_attention_weight_max": 33.83610534667969,
      "activations/layer13_attention_weight_min": -27.60553550720215,
      "activations/layer14_attention_weight_max": 34.91486358642578,
      "activations/layer14_attention_weight_min": -29.812484741210938,
      "activations/layer15_attention_weight_max": 30.50220489501953,
      "activations/layer15_attention_weight_min": -30.048412322998047,
      "activations/layer16_attention_weight_max": 33.151092529296875,
      "activations/layer16_attention_weight_min": -31.87944221496582,
      "activations/layer17_attention_weight_max": 50.557777404785156,
      "activations/layer17_attention_weight_min": -42.98347854614258,
      "activations/layer18_attention_weight_max": 44.73310470581055,
      "activations/layer18_attention_weight_min": -39.15940475463867,
      "activations/layer19_attention_weight_max": 24.21209144592285,
      "activations/layer19_attention_weight_min": -24.647933959960938,
      "activations/layer1_attention_weight_max": 14.442681312561035,
      "activations/layer1_attention_weight_min": -14.530348777770996,
      "activations/layer20_attention_weight_max": 23.302352905273438,
      "activations/layer20_attention_weight_min": -22.4525089263916,
      "activations/layer21_attention_weight_max": 35.499393463134766,
      "activations/layer21_attention_weight_min": -22.81566619873047,
      "activations/layer22_attention_weight_max": 26.532489776611328,
      "activations/layer22_attention_weight_min": -25.38155746459961,
      "activations/layer23_attention_weight_max": 36.513511657714844,
      "activations/layer23_attention_weight_min": -24.15424919128418,
      "activations/layer2_attention_weight_max": 31.59081268310547,
      "activations/layer2_attention_weight_min": -31.95665740966797,
      "activations/layer3_attention_weight_max": 88.59123229980469,
      "activations/layer3_attention_weight_min": -91.42359161376953,
      "activations/layer4_attention_weight_max": 84.20487976074219,
      "activations/layer4_attention_weight_min": -89.25245666503906,
      "activations/layer5_attention_weight_max": 63.43899154663086,
      "activations/layer5_attention_weight_min": -69.96989440917969,
      "activations/layer6_attention_weight_max": 50.39179992675781,
      "activations/layer6_attention_weight_min": -51.63485336303711,
      "activations/layer7_attention_weight_max": 70.38274383544922,
      "activations/layer7_attention_weight_min": -69.26437377929688,
      "activations/layer8_attention_weight_max": 49.518917083740234,
      "activations/layer8_attention_weight_min": -54.60499572753906,
      "activations/layer9_attention_weight_max": 46.36454391479492,
      "activations/layer9_attention_weight_min": -46.68983459472656,
      "epoch": 22.49,
      "learning_rate": 5.040530303030302e-06,
      "loss": 2.7298,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.5288,
      "eval_samples_per_second": 503.47,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.5288,
      "eval_openwebtext_samples_per_second": 503.47,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9801,
      "eval_wikitext_samples_per_second": 230.294,
      "step": 387000
    },
    {
      "epoch": 22.49,
      "eval_lambada_loss": 2.40625,
      "eval_lambada_ppl": 11.092286978670202,
      "eval_lambada_runtime": 9.5286,
      "eval_lambada_samples_per_second": 510.99,
      "step": 387000
    },
    {
      "activations/layer0_attention_weight_max": 16.589685440063477,
      "activations/layer0_attention_weight_min": -14.187308311462402,
      "activations/layer10_attention_weight_max": 37.7305793762207,
      "activations/layer10_attention_weight_min": -39.59267044067383,
      "activations/layer11_attention_weight_max": 36.33900833129883,
      "activations/layer11_attention_weight_min": -36.91543197631836,
      "activations/layer12_attention_weight_max": 21.001726150512695,
      "activations/layer12_attention_weight_min": -31.275920867919922,
      "activations/layer13_attention_weight_max": 43.35438537597656,
      "activations/layer13_attention_weight_min": -31.5133056640625,
      "activations/layer14_attention_weight_max": 36.68901062011719,
      "activations/layer14_attention_weight_min": -31.089967727661133,
      "activations/layer15_attention_weight_max": 35.51899337768555,
      "activations/layer15_attention_weight_min": -32.220394134521484,
      "activations/layer16_attention_weight_max": 37.53519058227539,
      "activations/layer16_attention_weight_min": -34.9619140625,
      "activations/layer17_attention_weight_max": 57.07561492919922,
      "activations/layer17_attention_weight_min": -49.39003372192383,
      "activations/layer18_attention_weight_max": 52.250667572021484,
      "activations/layer18_attention_weight_min": -44.56724166870117,
      "activations/layer19_attention_weight_max": 27.319517135620117,
      "activations/layer19_attention_weight_min": -25.813770294189453,
      "activations/layer1_attention_weight_max": 15.492293357849121,
      "activations/layer1_attention_weight_min": -14.16307544708252,
      "activations/layer20_attention_weight_max": 28.2852783203125,
      "activations/layer20_attention_weight_min": -21.97264289855957,
      "activations/layer21_attention_weight_max": 41.6095085144043,
      "activations/layer21_attention_weight_min": -25.2668399810791,
      "activations/layer22_attention_weight_max": 29.557220458984375,
      "activations/layer22_attention_weight_min": -25.92719268798828,
      "activations/layer23_attention_weight_max": 37.90607833862305,
      "activations/layer23_attention_weight_min": -23.711837768554688,
      "activations/layer2_attention_weight_max": 32.98383331298828,
      "activations/layer2_attention_weight_min": -32.88807678222656,
      "activations/layer3_attention_weight_max": 91.9188003540039,
      "activations/layer3_attention_weight_min": -95.03882598876953,
      "activations/layer4_attention_weight_max": 88.91556549072266,
      "activations/layer4_attention_weight_min": -90.95382690429688,
      "activations/layer5_attention_weight_max": 65.50169372558594,
      "activations/layer5_attention_weight_min": -70.24915313720703,
      "activations/layer6_attention_weight_max": 50.368595123291016,
      "activations/layer6_attention_weight_min": -51.58381271362305,
      "activations/layer7_attention_weight_max": 69.615234375,
      "activations/layer7_attention_weight_min": -68.20709991455078,
      "activations/layer8_attention_weight_max": 49.13644027709961,
      "activations/layer8_attention_weight_min": -51.759315490722656,
      "activations/layer9_attention_weight_max": 48.76168441772461,
      "activations/layer9_attention_weight_min": -48.7909049987793,
      "epoch": 22.49,
      "learning_rate": 5.021590909090909e-06,
      "loss": 2.7186,
      "step": 387050
    },
    {
      "activations/layer0_attention_weight_max": 16.821657180786133,
      "activations/layer0_attention_weight_min": -14.229288101196289,
      "activations/layer10_attention_weight_max": 38.403568267822266,
      "activations/layer10_attention_weight_min": -37.956581115722656,
      "activations/layer11_attention_weight_max": 34.75204849243164,
      "activations/layer11_attention_weight_min": -35.72981262207031,
      "activations/layer12_attention_weight_max": 19.480676651000977,
      "activations/layer12_attention_weight_min": -25.647798538208008,
      "activations/layer13_attention_weight_max": 32.52088165283203,
      "activations/layer13_attention_weight_min": -28.15374183654785,
      "activations/layer14_attention_weight_max": 35.401119232177734,
      "activations/layer14_attention_weight_min": -32.107234954833984,
      "activations/layer15_attention_weight_max": 34.30236053466797,
      "activations/layer15_attention_weight_min": -35.467559814453125,
      "activations/layer16_attention_weight_max": 37.51985168457031,
      "activations/layer16_attention_weight_min": -36.08208084106445,
      "activations/layer17_attention_weight_max": 54.3525276184082,
      "activations/layer17_attention_weight_min": -48.41347122192383,
      "activations/layer18_attention_weight_max": 52.65370559692383,
      "activations/layer18_attention_weight_min": -44.71725845336914,
      "activations/layer19_attention_weight_max": 28.266550064086914,
      "activations/layer19_attention_weight_min": -23.953079223632812,
      "activations/layer1_attention_weight_max": 15.49589729309082,
      "activations/layer1_attention_weight_min": -14.252700805664062,
      "activations/layer20_attention_weight_max": 26.003934860229492,
      "activations/layer20_attention_weight_min": -24.02516746520996,
      "activations/layer21_attention_weight_max": 34.5086555480957,
      "activations/layer21_attention_weight_min": -23.582366943359375,
      "activations/layer22_attention_weight_max": 29.6975154876709,
      "activations/layer22_attention_weight_min": -26.102705001831055,
      "activations/layer23_attention_weight_max": 38.1622314453125,
      "activations/layer23_attention_weight_min": -27.393003463745117,
      "activations/layer2_attention_weight_max": 32.679962158203125,
      "activations/layer2_attention_weight_min": -30.992542266845703,
      "activations/layer3_attention_weight_max": 90.55278015136719,
      "activations/layer3_attention_weight_min": -88.12123107910156,
      "activations/layer4_attention_weight_max": 86.90060424804688,
      "activations/layer4_attention_weight_min": -87.52876281738281,
      "activations/layer5_attention_weight_max": 61.76336669921875,
      "activations/layer5_attention_weight_min": -75.70311737060547,
      "activations/layer6_attention_weight_max": 49.57753372192383,
      "activations/layer6_attention_weight_min": -50.95218276977539,
      "activations/layer7_attention_weight_max": 69.18580627441406,
      "activations/layer7_attention_weight_min": -71.49797058105469,
      "activations/layer8_attention_weight_max": 47.510955810546875,
      "activations/layer8_attention_weight_min": -53.26494598388672,
      "activations/layer9_attention_weight_max": 47.313045501708984,
      "activations/layer9_attention_weight_min": -48.87589645385742,
      "epoch": 22.49,
      "learning_rate": 5.002651515151515e-06,
      "loss": 2.6935,
      "step": 387100
    },
    {
      "activations/layer0_attention_weight_max": 15.434426307678223,
      "activations/layer0_attention_weight_min": -14.320838928222656,
      "activations/layer10_attention_weight_max": 38.12458801269531,
      "activations/layer10_attention_weight_min": -38.943199157714844,
      "activations/layer11_attention_weight_max": 36.63087844848633,
      "activations/layer11_attention_weight_min": -36.28422546386719,
      "activations/layer12_attention_weight_max": 21.651432037353516,
      "activations/layer12_attention_weight_min": -27.019750595092773,
      "activations/layer13_attention_weight_max": 34.602813720703125,
      "activations/layer13_attention_weight_min": -27.379995346069336,
      "activations/layer14_attention_weight_max": 36.737213134765625,
      "activations/layer14_attention_weight_min": -30.4091739654541,
      "activations/layer15_attention_weight_max": 36.26955032348633,
      "activations/layer15_attention_weight_min": -31.63387680053711,
      "activations/layer16_attention_weight_max": 34.7949333190918,
      "activations/layer16_attention_weight_min": -32.65162658691406,
      "activations/layer17_attention_weight_max": 53.35591506958008,
      "activations/layer17_attention_weight_min": -44.12639617919922,
      "activations/layer18_attention_weight_max": 49.274269104003906,
      "activations/layer18_attention_weight_min": -40.944149017333984,
      "activations/layer19_attention_weight_max": 26.090436935424805,
      "activations/layer19_attention_weight_min": -23.23426628112793,
      "activations/layer1_attention_weight_max": 15.336565017700195,
      "activations/layer1_attention_weight_min": -14.532259941101074,
      "activations/layer20_attention_weight_max": 22.207712173461914,
      "activations/layer20_attention_weight_min": -21.112577438354492,
      "activations/layer21_attention_weight_max": 36.15410614013672,
      "activations/layer21_attention_weight_min": -23.911149978637695,
      "activations/layer22_attention_weight_max": 26.963987350463867,
      "activations/layer22_attention_weight_min": -24.71083641052246,
      "activations/layer23_attention_weight_max": 35.28380584716797,
      "activations/layer23_attention_weight_min": -26.274147033691406,
      "activations/layer2_attention_weight_max": 32.35509490966797,
      "activations/layer2_attention_weight_min": -30.251117706298828,
      "activations/layer3_attention_weight_max": 89.96015930175781,
      "activations/layer3_attention_weight_min": -91.12981414794922,
      "activations/layer4_attention_weight_max": 87.0868148803711,
      "activations/layer4_attention_weight_min": -84.29469299316406,
      "activations/layer5_attention_weight_max": 60.61357116699219,
      "activations/layer5_attention_weight_min": -69.42608642578125,
      "activations/layer6_attention_weight_max": 48.74252700805664,
      "activations/layer6_attention_weight_min": -50.73756408691406,
      "activations/layer7_attention_weight_max": 69.68466186523438,
      "activations/layer7_attention_weight_min": -67.95484161376953,
      "activations/layer8_attention_weight_max": 51.44121170043945,
      "activations/layer8_attention_weight_min": -54.717247009277344,
      "activations/layer9_attention_weight_max": 46.700721740722656,
      "activations/layer9_attention_weight_min": -46.81186294555664,
      "epoch": 22.5,
      "learning_rate": 4.983712121212121e-06,
      "loss": 2.6983,
      "step": 387150
    },
    {
      "activations/layer0_attention_weight_max": 15.404434204101562,
      "activations/layer0_attention_weight_min": -14.381558418273926,
      "activations/layer10_attention_weight_max": 36.18717575073242,
      "activations/layer10_attention_weight_min": -36.922576904296875,
      "activations/layer11_attention_weight_max": 32.49003219604492,
      "activations/layer11_attention_weight_min": -36.97247314453125,
      "activations/layer12_attention_weight_max": 18.505979537963867,
      "activations/layer12_attention_weight_min": -28.45823097229004,
      "activations/layer13_attention_weight_max": 32.506004333496094,
      "activations/layer13_attention_weight_min": -26.41954803466797,
      "activations/layer14_attention_weight_max": 35.2682991027832,
      "activations/layer14_attention_weight_min": -28.852060317993164,
      "activations/layer15_attention_weight_max": 31.06437110900879,
      "activations/layer15_attention_weight_min": -28.678829193115234,
      "activations/layer16_attention_weight_max": 32.075809478759766,
      "activations/layer16_attention_weight_min": -30.699621200561523,
      "activations/layer17_attention_weight_max": 51.78971862792969,
      "activations/layer17_attention_weight_min": -41.60377502441406,
      "activations/layer18_attention_weight_max": 44.445655822753906,
      "activations/layer18_attention_weight_min": -37.956485748291016,
      "activations/layer19_attention_weight_max": 21.705432891845703,
      "activations/layer19_attention_weight_min": -22.97333335876465,
      "activations/layer1_attention_weight_max": 15.465906143188477,
      "activations/layer1_attention_weight_min": -14.813804626464844,
      "activations/layer20_attention_weight_max": 20.075969696044922,
      "activations/layer20_attention_weight_min": -21.917613983154297,
      "activations/layer21_attention_weight_max": 29.540969848632812,
      "activations/layer21_attention_weight_min": -20.30906105041504,
      "activations/layer22_attention_weight_max": 23.60991859436035,
      "activations/layer22_attention_weight_min": -23.513784408569336,
      "activations/layer23_attention_weight_max": 32.540794372558594,
      "activations/layer23_attention_weight_min": -23.456701278686523,
      "activations/layer2_attention_weight_max": 31.88739013671875,
      "activations/layer2_attention_weight_min": -30.091543197631836,
      "activations/layer3_attention_weight_max": 91.13700866699219,
      "activations/layer3_attention_weight_min": -89.8519287109375,
      "activations/layer4_attention_weight_max": 87.3290023803711,
      "activations/layer4_attention_weight_min": -85.08319854736328,
      "activations/layer5_attention_weight_max": 63.42778015136719,
      "activations/layer5_attention_weight_min": -71.66846466064453,
      "activations/layer6_attention_weight_max": 50.15102767944336,
      "activations/layer6_attention_weight_min": -50.653324127197266,
      "activations/layer7_attention_weight_max": 71.87628936767578,
      "activations/layer7_attention_weight_min": -70.76909637451172,
      "activations/layer8_attention_weight_max": 49.50892639160156,
      "activations/layer8_attention_weight_min": -52.77106475830078,
      "activations/layer9_attention_weight_max": 44.750022888183594,
      "activations/layer9_attention_weight_min": -46.83314895629883,
      "epoch": 22.5,
      "learning_rate": 4.964772727272727e-06,
      "loss": 2.7076,
      "step": 387200
    },
    {
      "activations/layer0_attention_weight_max": 15.700118064880371,
      "activations/layer0_attention_weight_min": -14.375836372375488,
      "activations/layer10_attention_weight_max": 42.191444396972656,
      "activations/layer10_attention_weight_min": -42.236968994140625,
      "activations/layer11_attention_weight_max": 42.52451705932617,
      "activations/layer11_attention_weight_min": -39.104774475097656,
      "activations/layer12_attention_weight_max": 21.38716697692871,
      "activations/layer12_attention_weight_min": -34.893924713134766,
      "activations/layer13_attention_weight_max": 39.3790168762207,
      "activations/layer13_attention_weight_min": -28.151607513427734,
      "activations/layer14_attention_weight_max": 40.88213348388672,
      "activations/layer14_attention_weight_min": -31.913726806640625,
      "activations/layer15_attention_weight_max": 38.71489334106445,
      "activations/layer15_attention_weight_min": -31.744089126586914,
      "activations/layer16_attention_weight_max": 40.32107162475586,
      "activations/layer16_attention_weight_min": -35.89164733886719,
      "activations/layer17_attention_weight_max": 64.32076263427734,
      "activations/layer17_attention_weight_min": -50.2448844909668,
      "activations/layer18_attention_weight_max": 55.60469055175781,
      "activations/layer18_attention_weight_min": -43.28512191772461,
      "activations/layer19_attention_weight_max": 29.000350952148438,
      "activations/layer19_attention_weight_min": -24.017332077026367,
      "activations/layer1_attention_weight_max": 16.28775405883789,
      "activations/layer1_attention_weight_min": -14.508012771606445,
      "activations/layer20_attention_weight_max": 26.294145584106445,
      "activations/layer20_attention_weight_min": -20.10536766052246,
      "activations/layer21_attention_weight_max": 35.723655700683594,
      "activations/layer21_attention_weight_min": -22.771377563476562,
      "activations/layer22_attention_weight_max": 28.78357696533203,
      "activations/layer22_attention_weight_min": -24.570011138916016,
      "activations/layer23_attention_weight_max": 39.77336120605469,
      "activations/layer23_attention_weight_min": -23.212446212768555,
      "activations/layer2_attention_weight_max": 33.745635986328125,
      "activations/layer2_attention_weight_min": -32.89162826538086,
      "activations/layer3_attention_weight_max": 95.18439483642578,
      "activations/layer3_attention_weight_min": -95.94486236572266,
      "activations/layer4_attention_weight_max": 90.42742919921875,
      "activations/layer4_attention_weight_min": -94.43828582763672,
      "activations/layer5_attention_weight_max": 68.311279296875,
      "activations/layer5_attention_weight_min": -71.63856506347656,
      "activations/layer6_attention_weight_max": 53.795345306396484,
      "activations/layer6_attention_weight_min": -55.35186767578125,
      "activations/layer7_attention_weight_max": 73.17820739746094,
      "activations/layer7_attention_weight_min": -75.84468078613281,
      "activations/layer8_attention_weight_max": 55.2923583984375,
      "activations/layer8_attention_weight_min": -57.14672088623047,
      "activations/layer9_attention_weight_max": 51.10649871826172,
      "activations/layer9_attention_weight_min": -53.12693405151367,
      "epoch": 22.5,
      "learning_rate": 4.945833333333333e-06,
      "loss": 2.7044,
      "step": 387250
    },
    {
      "activations/layer0_attention_weight_max": 15.954567909240723,
      "activations/layer0_attention_weight_min": -14.445320129394531,
      "activations/layer10_attention_weight_max": 36.94792556762695,
      "activations/layer10_attention_weight_min": -36.78180694580078,
      "activations/layer11_attention_weight_max": 37.188270568847656,
      "activations/layer11_attention_weight_min": -37.23047637939453,
      "activations/layer12_attention_weight_max": 21.586734771728516,
      "activations/layer12_attention_weight_min": -26.342823028564453,
      "activations/layer13_attention_weight_max": 43.78592300415039,
      "activations/layer13_attention_weight_min": -29.725616455078125,
      "activations/layer14_attention_weight_max": 38.519325256347656,
      "activations/layer14_attention_weight_min": -30.68792152404785,
      "activations/layer15_attention_weight_max": 36.923282623291016,
      "activations/layer15_attention_weight_min": -30.269243240356445,
      "activations/layer16_attention_weight_max": 37.73131561279297,
      "activations/layer16_attention_weight_min": -34.79377365112305,
      "activations/layer17_attention_weight_max": 57.5768928527832,
      "activations/layer17_attention_weight_min": -47.25547409057617,
      "activations/layer18_attention_weight_max": 51.693790435791016,
      "activations/layer18_attention_weight_min": -43.03267288208008,
      "activations/layer19_attention_weight_max": 32.0124626159668,
      "activations/layer19_attention_weight_min": -22.89285659790039,
      "activations/layer1_attention_weight_max": 15.17036247253418,
      "activations/layer1_attention_weight_min": -14.459330558776855,
      "activations/layer20_attention_weight_max": 25.951108932495117,
      "activations/layer20_attention_weight_min": -21.06031036376953,
      "activations/layer21_attention_weight_max": 41.987159729003906,
      "activations/layer21_attention_weight_min": -21.64278221130371,
      "activations/layer22_attention_weight_max": 32.00999069213867,
      "activations/layer22_attention_weight_min": -25.903305053710938,
      "activations/layer23_attention_weight_max": 43.9027099609375,
      "activations/layer23_attention_weight_min": -23.474365234375,
      "activations/layer2_attention_weight_max": 32.270503997802734,
      "activations/layer2_attention_weight_min": -30.841529846191406,
      "activations/layer3_attention_weight_max": 93.60456085205078,
      "activations/layer3_attention_weight_min": -94.3805923461914,
      "activations/layer4_attention_weight_max": 85.66913604736328,
      "activations/layer4_attention_weight_min": -84.07292938232422,
      "activations/layer5_attention_weight_max": 62.343849182128906,
      "activations/layer5_attention_weight_min": -72.46129608154297,
      "activations/layer6_attention_weight_max": 47.61294174194336,
      "activations/layer6_attention_weight_min": -49.36396026611328,
      "activations/layer7_attention_weight_max": 70.82852935791016,
      "activations/layer7_attention_weight_min": -67.0188980102539,
      "activations/layer8_attention_weight_max": 49.725067138671875,
      "activations/layer8_attention_weight_min": -52.582515716552734,
      "activations/layer9_attention_weight_max": 45.523860931396484,
      "activations/layer9_attention_weight_min": -46.720947265625,
      "epoch": 22.5,
      "learning_rate": 4.9268939393939384e-06,
      "loss": 2.7173,
      "step": 387300
    },
    {
      "activations/layer0_attention_weight_max": 15.76213550567627,
      "activations/layer0_attention_weight_min": -14.193122863769531,
      "activations/layer10_attention_weight_max": 37.77730178833008,
      "activations/layer10_attention_weight_min": -35.93136978149414,
      "activations/layer11_attention_weight_max": 36.02159881591797,
      "activations/layer11_attention_weight_min": -34.06381607055664,
      "activations/layer12_attention_weight_max": 18.912006378173828,
      "activations/layer12_attention_weight_min": -25.052194595336914,
      "activations/layer13_attention_weight_max": 30.429340362548828,
      "activations/layer13_attention_weight_min": -25.215749740600586,
      "activations/layer14_attention_weight_max": 32.132957458496094,
      "activations/layer14_attention_weight_min": -32.147071838378906,
      "activations/layer15_attention_weight_max": 31.41055679321289,
      "activations/layer15_attention_weight_min": -31.655126571655273,
      "activations/layer16_attention_weight_max": 32.714500427246094,
      "activations/layer16_attention_weight_min": -32.03765106201172,
      "activations/layer17_attention_weight_max": 52.60023880004883,
      "activations/layer17_attention_weight_min": -45.34153747558594,
      "activations/layer18_attention_weight_max": 47.05998992919922,
      "activations/layer18_attention_weight_min": -38.90895080566406,
      "activations/layer19_attention_weight_max": 25.477062225341797,
      "activations/layer19_attention_weight_min": -23.103246688842773,
      "activations/layer1_attention_weight_max": 15.314153671264648,
      "activations/layer1_attention_weight_min": -13.774741172790527,
      "activations/layer20_attention_weight_max": 22.125720977783203,
      "activations/layer20_attention_weight_min": -23.49500274658203,
      "activations/layer21_attention_weight_max": 29.810495376586914,
      "activations/layer21_attention_weight_min": -24.55535316467285,
      "activations/layer22_attention_weight_max": 29.310911178588867,
      "activations/layer22_attention_weight_min": -28.253774642944336,
      "activations/layer23_attention_weight_max": 33.47377014160156,
      "activations/layer23_attention_weight_min": -30.192975997924805,
      "activations/layer2_attention_weight_max": 32.13713073730469,
      "activations/layer2_attention_weight_min": -30.240665435791016,
      "activations/layer3_attention_weight_max": 87.99961853027344,
      "activations/layer3_attention_weight_min": -88.50606536865234,
      "activations/layer4_attention_weight_max": 84.09008026123047,
      "activations/layer4_attention_weight_min": -86.07699584960938,
      "activations/layer5_attention_weight_max": 63.11515426635742,
      "activations/layer5_attention_weight_min": -71.36691284179688,
      "activations/layer6_attention_weight_max": 48.629783630371094,
      "activations/layer6_attention_weight_min": -49.00844192504883,
      "activations/layer7_attention_weight_max": 66.78087615966797,
      "activations/layer7_attention_weight_min": -66.65155029296875,
      "activations/layer8_attention_weight_max": 50.60114288330078,
      "activations/layer8_attention_weight_min": -53.78009033203125,
      "activations/layer9_attention_weight_max": 43.42530059814453,
      "activations/layer9_attention_weight_min": -46.2920036315918,
      "epoch": 22.51,
      "learning_rate": 4.907954545454545e-06,
      "loss": 2.7025,
      "step": 387350
    },
    {
      "activations/layer0_attention_weight_max": 15.870262145996094,
      "activations/layer0_attention_weight_min": -14.411369323730469,
      "activations/layer10_attention_weight_max": 34.47477722167969,
      "activations/layer10_attention_weight_min": -35.450592041015625,
      "activations/layer11_attention_weight_max": 32.488319396972656,
      "activations/layer11_attention_weight_min": -35.05358123779297,
      "activations/layer12_attention_weight_max": 20.140304565429688,
      "activations/layer12_attention_weight_min": -23.39694595336914,
      "activations/layer13_attention_weight_max": 33.69660568237305,
      "activations/layer13_attention_weight_min": -27.82675552368164,
      "activations/layer14_attention_weight_max": 35.45266342163086,
      "activations/layer14_attention_weight_min": -30.845617294311523,
      "activations/layer15_attention_weight_max": 32.847347259521484,
      "activations/layer15_attention_weight_min": -31.647857666015625,
      "activations/layer16_attention_weight_max": 35.65950012207031,
      "activations/layer16_attention_weight_min": -33.3226203918457,
      "activations/layer17_attention_weight_max": 52.35018539428711,
      "activations/layer17_attention_weight_min": -46.589237213134766,
      "activations/layer18_attention_weight_max": 51.616607666015625,
      "activations/layer18_attention_weight_min": -40.558963775634766,
      "activations/layer19_attention_weight_max": 28.654935836791992,
      "activations/layer19_attention_weight_min": -24.189598083496094,
      "activations/layer1_attention_weight_max": 15.13196849822998,
      "activations/layer1_attention_weight_min": -15.249106407165527,
      "activations/layer20_attention_weight_max": 24.714298248291016,
      "activations/layer20_attention_weight_min": -21.733966827392578,
      "activations/layer21_attention_weight_max": 32.420745849609375,
      "activations/layer21_attention_weight_min": -22.636594772338867,
      "activations/layer22_attention_weight_max": 30.004819869995117,
      "activations/layer22_attention_weight_min": -24.876577377319336,
      "activations/layer23_attention_weight_max": 38.607879638671875,
      "activations/layer23_attention_weight_min": -24.117277145385742,
      "activations/layer2_attention_weight_max": 33.33714294433594,
      "activations/layer2_attention_weight_min": -31.6601619720459,
      "activations/layer3_attention_weight_max": 89.58026885986328,
      "activations/layer3_attention_weight_min": -89.92095184326172,
      "activations/layer4_attention_weight_max": 84.90477752685547,
      "activations/layer4_attention_weight_min": -82.82799530029297,
      "activations/layer5_attention_weight_max": 62.64832305908203,
      "activations/layer5_attention_weight_min": -74.10082244873047,
      "activations/layer6_attention_weight_max": 50.781166076660156,
      "activations/layer6_attention_weight_min": -51.29513931274414,
      "activations/layer7_attention_weight_max": 70.66983032226562,
      "activations/layer7_attention_weight_min": -67.5563735961914,
      "activations/layer8_attention_weight_max": 45.48333740234375,
      "activations/layer8_attention_weight_min": -48.22406768798828,
      "activations/layer9_attention_weight_max": 42.27018356323242,
      "activations/layer9_attention_weight_min": -45.221004486083984,
      "epoch": 22.51,
      "learning_rate": 4.889393939393939e-06,
      "loss": 2.7034,
      "step": 387400
    },
    {
      "activations/layer0_attention_weight_max": 15.715456008911133,
      "activations/layer0_attention_weight_min": -14.476747512817383,
      "activations/layer10_attention_weight_max": 35.045501708984375,
      "activations/layer10_attention_weight_min": -33.883243560791016,
      "activations/layer11_attention_weight_max": 31.328588485717773,
      "activations/layer11_attention_weight_min": -34.00596618652344,
      "activations/layer12_attention_weight_max": 18.69569969177246,
      "activations/layer12_attention_weight_min": -26.773094177246094,
      "activations/layer13_attention_weight_max": 34.60315704345703,
      "activations/layer13_attention_weight_min": -25.749488830566406,
      "activations/layer14_attention_weight_max": 35.987300872802734,
      "activations/layer14_attention_weight_min": -30.616127014160156,
      "activations/layer15_attention_weight_max": 35.68721008300781,
      "activations/layer15_attention_weight_min": -31.414915084838867,
      "activations/layer16_attention_weight_max": 35.61872863769531,
      "activations/layer16_attention_weight_min": -34.73190689086914,
      "activations/layer17_attention_weight_max": 53.28766632080078,
      "activations/layer17_attention_weight_min": -46.956905364990234,
      "activations/layer18_attention_weight_max": 50.45156478881836,
      "activations/layer18_attention_weight_min": -43.47629165649414,
      "activations/layer19_attention_weight_max": 24.900524139404297,
      "activations/layer19_attention_weight_min": -24.67068099975586,
      "activations/layer1_attention_weight_max": 15.703008651733398,
      "activations/layer1_attention_weight_min": -14.23654556274414,
      "activations/layer20_attention_weight_max": 22.074487686157227,
      "activations/layer20_attention_weight_min": -22.098127365112305,
      "activations/layer21_attention_weight_max": 35.635154724121094,
      "activations/layer21_attention_weight_min": -22.3716983795166,
      "activations/layer22_attention_weight_max": 28.2244930267334,
      "activations/layer22_attention_weight_min": -26.545747756958008,
      "activations/layer23_attention_weight_max": 38.93366622924805,
      "activations/layer23_attention_weight_min": -24.667564392089844,
      "activations/layer2_attention_weight_max": 32.27703094482422,
      "activations/layer2_attention_weight_min": -31.575828552246094,
      "activations/layer3_attention_weight_max": 89.673828125,
      "activations/layer3_attention_weight_min": -92.40653228759766,
      "activations/layer4_attention_weight_max": 83.58586120605469,
      "activations/layer4_attention_weight_min": -86.24347686767578,
      "activations/layer5_attention_weight_max": 61.254695892333984,
      "activations/layer5_attention_weight_min": -81.39944458007812,
      "activations/layer6_attention_weight_max": 49.05418014526367,
      "activations/layer6_attention_weight_min": -49.145084381103516,
      "activations/layer7_attention_weight_max": 69.66655731201172,
      "activations/layer7_attention_weight_min": -72.04409790039062,
      "activations/layer8_attention_weight_max": 46.39584732055664,
      "activations/layer8_attention_weight_min": -48.94243240356445,
      "activations/layer9_attention_weight_max": 41.46435546875,
      "activations/layer9_attention_weight_min": -46.12939453125,
      "epoch": 22.51,
      "learning_rate": 4.870454545454545e-06,
      "loss": 2.7179,
      "step": 387450
    },
    {
      "activations/layer0_attention_weight_max": 16.083311080932617,
      "activations/layer0_attention_weight_min": -14.317442893981934,
      "activations/layer10_attention_weight_max": 36.79948806762695,
      "activations/layer10_attention_weight_min": -35.827674865722656,
      "activations/layer11_attention_weight_max": 35.379947662353516,
      "activations/layer11_attention_weight_min": -33.48441696166992,
      "activations/layer12_attention_weight_max": 18.866680145263672,
      "activations/layer12_attention_weight_min": -24.161359786987305,
      "activations/layer13_attention_weight_max": 30.113876342773438,
      "activations/layer13_attention_weight_min": -28.055591583251953,
      "activations/layer14_attention_weight_max": 32.856502532958984,
      "activations/layer14_attention_weight_min": -28.921279907226562,
      "activations/layer15_attention_weight_max": 31.089679718017578,
      "activations/layer15_attention_weight_min": -30.243995666503906,
      "activations/layer16_attention_weight_max": 33.44609451293945,
      "activations/layer16_attention_weight_min": -33.57558822631836,
      "activations/layer17_attention_weight_max": 51.3145866394043,
      "activations/layer17_attention_weight_min": -44.546199798583984,
      "activations/layer18_attention_weight_max": 44.64070510864258,
      "activations/layer18_attention_weight_min": -40.229732513427734,
      "activations/layer19_attention_weight_max": 22.31589126586914,
      "activations/layer19_attention_weight_min": -23.10651206970215,
      "activations/layer1_attention_weight_max": 16.24486541748047,
      "activations/layer1_attention_weight_min": -15.361757278442383,
      "activations/layer20_attention_weight_max": 19.75231170654297,
      "activations/layer20_attention_weight_min": -20.775190353393555,
      "activations/layer21_attention_weight_max": 29.58852767944336,
      "activations/layer21_attention_weight_min": -20.9122257232666,
      "activations/layer22_attention_weight_max": 25.190006256103516,
      "activations/layer22_attention_weight_min": -24.942232131958008,
      "activations/layer23_attention_weight_max": 33.377132415771484,
      "activations/layer23_attention_weight_min": -24.47357940673828,
      "activations/layer2_attention_weight_max": 32.268531799316406,
      "activations/layer2_attention_weight_min": -30.442419052124023,
      "activations/layer3_attention_weight_max": 90.08670043945312,
      "activations/layer3_attention_weight_min": -87.46527099609375,
      "activations/layer4_attention_weight_max": 83.88584899902344,
      "activations/layer4_attention_weight_min": -84.613037109375,
      "activations/layer5_attention_weight_max": 62.060211181640625,
      "activations/layer5_attention_weight_min": -67.80144500732422,
      "activations/layer6_attention_weight_max": 48.34815216064453,
      "activations/layer6_attention_weight_min": -49.39754867553711,
      "activations/layer7_attention_weight_max": 67.65707397460938,
      "activations/layer7_attention_weight_min": -66.52566528320312,
      "activations/layer8_attention_weight_max": 44.6718635559082,
      "activations/layer8_attention_weight_min": -49.45026397705078,
      "activations/layer9_attention_weight_max": 43.644561767578125,
      "activations/layer9_attention_weight_min": -46.44785690307617,
      "epoch": 22.52,
      "learning_rate": 4.851515151515151e-06,
      "loss": 2.7017,
      "step": 387500
    },
    {
      "activations/layer0_attention_weight_max": 15.495194435119629,
      "activations/layer0_attention_weight_min": -14.03111457824707,
      "activations/layer10_attention_weight_max": 36.61783218383789,
      "activations/layer10_attention_weight_min": -36.12035369873047,
      "activations/layer11_attention_weight_max": 32.85447311401367,
      "activations/layer11_attention_weight_min": -35.23493957519531,
      "activations/layer12_attention_weight_max": 18.213760375976562,
      "activations/layer12_attention_weight_min": -25.473384857177734,
      "activations/layer13_attention_weight_max": 31.923208236694336,
      "activations/layer13_attention_weight_min": -25.8383731842041,
      "activations/layer14_attention_weight_max": 32.97853088378906,
      "activations/layer14_attention_weight_min": -29.226747512817383,
      "activations/layer15_attention_weight_max": 31.345046997070312,
      "activations/layer15_attention_weight_min": -29.65989112854004,
      "activations/layer16_attention_weight_max": 33.41720962524414,
      "activations/layer16_attention_weight_min": -32.463523864746094,
      "activations/layer17_attention_weight_max": 50.34706115722656,
      "activations/layer17_attention_weight_min": -45.59062576293945,
      "activations/layer18_attention_weight_max": 45.25679016113281,
      "activations/layer18_attention_weight_min": -38.973854064941406,
      "activations/layer19_attention_weight_max": 24.61363983154297,
      "activations/layer19_attention_weight_min": -22.62030601501465,
      "activations/layer1_attention_weight_max": 15.07522201538086,
      "activations/layer1_attention_weight_min": -13.819677352905273,
      "activations/layer20_attention_weight_max": 20.829513549804688,
      "activations/layer20_attention_weight_min": -21.485851287841797,
      "activations/layer21_attention_weight_max": 36.334774017333984,
      "activations/layer21_attention_weight_min": -19.451602935791016,
      "activations/layer22_attention_weight_max": 29.854320526123047,
      "activations/layer22_attention_weight_min": -25.62884521484375,
      "activations/layer23_attention_weight_max": 37.27916717529297,
      "activations/layer23_attention_weight_min": -26.116085052490234,
      "activations/layer2_attention_weight_max": 32.79750442504883,
      "activations/layer2_attention_weight_min": -31.83993148803711,
      "activations/layer3_attention_weight_max": 87.2640609741211,
      "activations/layer3_attention_weight_min": -88.94854736328125,
      "activations/layer4_attention_weight_max": 83.9970474243164,
      "activations/layer4_attention_weight_min": -83.85516357421875,
      "activations/layer5_attention_weight_max": 61.25471496582031,
      "activations/layer5_attention_weight_min": -71.41014099121094,
      "activations/layer6_attention_weight_max": 48.08473587036133,
      "activations/layer6_attention_weight_min": -51.22411346435547,
      "activations/layer7_attention_weight_max": 68.76251983642578,
      "activations/layer7_attention_weight_min": -68.37865447998047,
      "activations/layer8_attention_weight_max": 47.253055572509766,
      "activations/layer8_attention_weight_min": -50.55697250366211,
      "activations/layer9_attention_weight_max": 44.373512268066406,
      "activations/layer9_attention_weight_min": -44.90435791015625,
      "epoch": 22.52,
      "learning_rate": 4.832575757575757e-06,
      "loss": 2.6905,
      "step": 387550
    },
    {
      "activations/layer0_attention_weight_max": 15.57617473602295,
      "activations/layer0_attention_weight_min": -14.215495109558105,
      "activations/layer10_attention_weight_max": 32.65222930908203,
      "activations/layer10_attention_weight_min": -34.19057846069336,
      "activations/layer11_attention_weight_max": 31.217445373535156,
      "activations/layer11_attention_weight_min": -33.55409240722656,
      "activations/layer12_attention_weight_max": 18.621126174926758,
      "activations/layer12_attention_weight_min": -23.897977828979492,
      "activations/layer13_attention_weight_max": 29.473894119262695,
      "activations/layer13_attention_weight_min": -25.94819450378418,
      "activations/layer14_attention_weight_max": 35.211090087890625,
      "activations/layer14_attention_weight_min": -30.741531372070312,
      "activations/layer15_attention_weight_max": 33.607147216796875,
      "activations/layer15_attention_weight_min": -31.021095275878906,
      "activations/layer16_attention_weight_max": 34.57380294799805,
      "activations/layer16_attention_weight_min": -33.39247131347656,
      "activations/layer17_attention_weight_max": 55.864803314208984,
      "activations/layer17_attention_weight_min": -49.70246124267578,
      "activations/layer18_attention_weight_max": 49.99308776855469,
      "activations/layer18_attention_weight_min": -44.08211898803711,
      "activations/layer19_attention_weight_max": 27.004302978515625,
      "activations/layer19_attention_weight_min": -23.61581802368164,
      "activations/layer1_attention_weight_max": 15.159600257873535,
      "activations/layer1_attention_weight_min": -14.933049201965332,
      "activations/layer20_attention_weight_max": 23.886056900024414,
      "activations/layer20_attention_weight_min": -22.47825050354004,
      "activations/layer21_attention_weight_max": 35.22157669067383,
      "activations/layer21_attention_weight_min": -21.220962524414062,
      "activations/layer22_attention_weight_max": 34.17029571533203,
      "activations/layer22_attention_weight_min": -25.43600082397461,
      "activations/layer23_attention_weight_max": 33.79107666015625,
      "activations/layer23_attention_weight_min": -26.465309143066406,
      "activations/layer2_attention_weight_max": 32.2482795715332,
      "activations/layer2_attention_weight_min": -30.91122055053711,
      "activations/layer3_attention_weight_max": 87.10210418701172,
      "activations/layer3_attention_weight_min": -86.3121566772461,
      "activations/layer4_attention_weight_max": 80.94121551513672,
      "activations/layer4_attention_weight_min": -84.31925201416016,
      "activations/layer5_attention_weight_max": 60.81289291381836,
      "activations/layer5_attention_weight_min": -69.97100830078125,
      "activations/layer6_attention_weight_max": 47.301856994628906,
      "activations/layer6_attention_weight_min": -49.167381286621094,
      "activations/layer7_attention_weight_max": 69.72461700439453,
      "activations/layer7_attention_weight_min": -72.2000732421875,
      "activations/layer8_attention_weight_max": 48.60841751098633,
      "activations/layer8_attention_weight_min": -47.7717170715332,
      "activations/layer9_attention_weight_max": 40.49502182006836,
      "activations/layer9_attention_weight_min": -44.475894927978516,
      "epoch": 22.52,
      "learning_rate": 4.813636363636364e-06,
      "loss": 2.6827,
      "step": 387600
    },
    {
      "activations/layer0_attention_weight_max": 16.365036010742188,
      "activations/layer0_attention_weight_min": -14.44998550415039,
      "activations/layer10_attention_weight_max": 39.97490310668945,
      "activations/layer10_attention_weight_min": -42.08000183105469,
      "activations/layer11_attention_weight_max": 37.114871978759766,
      "activations/layer11_attention_weight_min": -38.392574310302734,
      "activations/layer12_attention_weight_max": 22.836076736450195,
      "activations/layer12_attention_weight_min": -25.506914138793945,
      "activations/layer13_attention_weight_max": 41.76664733886719,
      "activations/layer13_attention_weight_min": -28.702728271484375,
      "activations/layer14_attention_weight_max": 39.634361267089844,
      "activations/layer14_attention_weight_min": -32.25401306152344,
      "activations/layer15_attention_weight_max": 37.81693649291992,
      "activations/layer15_attention_weight_min": -31.325841903686523,
      "activations/layer16_attention_weight_max": 39.365108489990234,
      "activations/layer16_attention_weight_min": -34.52493667602539,
      "activations/layer17_attention_weight_max": 57.38224411010742,
      "activations/layer17_attention_weight_min": -47.25489807128906,
      "activations/layer18_attention_weight_max": 54.085411071777344,
      "activations/layer18_attention_weight_min": -44.25011444091797,
      "activations/layer19_attention_weight_max": 28.63346290588379,
      "activations/layer19_attention_weight_min": -24.784423828125,
      "activations/layer1_attention_weight_max": 14.71599292755127,
      "activations/layer1_attention_weight_min": -15.699060440063477,
      "activations/layer20_attention_weight_max": 26.713821411132812,
      "activations/layer20_attention_weight_min": -23.49834442138672,
      "activations/layer21_attention_weight_max": 43.159423828125,
      "activations/layer21_attention_weight_min": -21.40250587463379,
      "activations/layer22_attention_weight_max": 30.245866775512695,
      "activations/layer22_attention_weight_min": -26.60952377319336,
      "activations/layer23_attention_weight_max": 40.82691955566406,
      "activations/layer23_attention_weight_min": -24.76472282409668,
      "activations/layer2_attention_weight_max": 34.711822509765625,
      "activations/layer2_attention_weight_min": -31.794151306152344,
      "activations/layer3_attention_weight_max": 93.81761932373047,
      "activations/layer3_attention_weight_min": -92.74066162109375,
      "activations/layer4_attention_weight_max": 89.37166595458984,
      "activations/layer4_attention_weight_min": -89.99625396728516,
      "activations/layer5_attention_weight_max": 64.40630340576172,
      "activations/layer5_attention_weight_min": -72.60948181152344,
      "activations/layer6_attention_weight_max": 51.41004180908203,
      "activations/layer6_attention_weight_min": -55.25545120239258,
      "activations/layer7_attention_weight_max": 71.06556701660156,
      "activations/layer7_attention_weight_min": -79.97582244873047,
      "activations/layer8_attention_weight_max": 50.83305358886719,
      "activations/layer8_attention_weight_min": -55.550907135009766,
      "activations/layer9_attention_weight_max": 50.57745361328125,
      "activations/layer9_attention_weight_min": -49.03762435913086,
      "epoch": 22.52,
      "learning_rate": 4.7946969696969696e-06,
      "loss": 2.7033,
      "step": 387650
    },
    {
      "activations/layer0_attention_weight_max": 17.96908950805664,
      "activations/layer0_attention_weight_min": -14.012646675109863,
      "activations/layer10_attention_weight_max": 33.68739318847656,
      "activations/layer10_attention_weight_min": -34.6474494934082,
      "activations/layer11_attention_weight_max": 31.68598747253418,
      "activations/layer11_attention_weight_min": -33.741180419921875,
      "activations/layer12_attention_weight_max": 18.18035888671875,
      "activations/layer12_attention_weight_min": -27.46034049987793,
      "activations/layer13_attention_weight_max": 30.700927734375,
      "activations/layer13_attention_weight_min": -26.10851287841797,
      "activations/layer14_attention_weight_max": 35.553619384765625,
      "activations/layer14_attention_weight_min": -29.8265380859375,
      "activations/layer15_attention_weight_max": 32.89494705200195,
      "activations/layer15_attention_weight_min": -29.64870834350586,
      "activations/layer16_attention_weight_max": 33.097862243652344,
      "activations/layer16_attention_weight_min": -31.718297958374023,
      "activations/layer17_attention_weight_max": 49.68895721435547,
      "activations/layer17_attention_weight_min": -44.05320739746094,
      "activations/layer18_attention_weight_max": 43.756996154785156,
      "activations/layer18_attention_weight_min": -42.34238052368164,
      "activations/layer19_attention_weight_max": 22.39067268371582,
      "activations/layer19_attention_weight_min": -22.014455795288086,
      "activations/layer1_attention_weight_max": 15.585251808166504,
      "activations/layer1_attention_weight_min": -14.469219207763672,
      "activations/layer20_attention_weight_max": 20.151132583618164,
      "activations/layer20_attention_weight_min": -21.084678649902344,
      "activations/layer21_attention_weight_max": 33.2381706237793,
      "activations/layer21_attention_weight_min": -22.210561752319336,
      "activations/layer22_attention_weight_max": 28.3787899017334,
      "activations/layer22_attention_weight_min": -25.4735107421875,
      "activations/layer23_attention_weight_max": 35.56236267089844,
      "activations/layer23_attention_weight_min": -21.819515228271484,
      "activations/layer2_attention_weight_max": 31.413410186767578,
      "activations/layer2_attention_weight_min": -29.69971466064453,
      "activations/layer3_attention_weight_max": 87.96797943115234,
      "activations/layer3_attention_weight_min": -85.88834381103516,
      "activations/layer4_attention_weight_max": 82.7870101928711,
      "activations/layer4_attention_weight_min": -81.7489242553711,
      "activations/layer5_attention_weight_max": 59.274837493896484,
      "activations/layer5_attention_weight_min": -76.43524169921875,
      "activations/layer6_attention_weight_max": 48.406864166259766,
      "activations/layer6_attention_weight_min": -48.09233474731445,
      "activations/layer7_attention_weight_max": 67.34059143066406,
      "activations/layer7_attention_weight_min": -67.16028594970703,
      "activations/layer8_attention_weight_max": 46.47049331665039,
      "activations/layer8_attention_weight_min": -49.47626876831055,
      "activations/layer9_attention_weight_max": 41.46991729736328,
      "activations/layer9_attention_weight_min": -44.47626495361328,
      "epoch": 22.53,
      "learning_rate": 4.775757575757575e-06,
      "loss": 2.7108,
      "step": 387700
    },
    {
      "activations/layer0_attention_weight_max": 16.472211837768555,
      "activations/layer0_attention_weight_min": -14.309608459472656,
      "activations/layer10_attention_weight_max": 37.57896041870117,
      "activations/layer10_attention_weight_min": -38.13864517211914,
      "activations/layer11_attention_weight_max": 34.14344024658203,
      "activations/layer11_attention_weight_min": -36.487335205078125,
      "activations/layer12_attention_weight_max": 19.248119354248047,
      "activations/layer12_attention_weight_min": -24.199222564697266,
      "activations/layer13_attention_weight_max": 34.42536544799805,
      "activations/layer13_attention_weight_min": -27.13827133178711,
      "activations/layer14_attention_weight_max": 36.72126388549805,
      "activations/layer14_attention_weight_min": -31.81847381591797,
      "activations/layer15_attention_weight_max": 33.40187454223633,
      "activations/layer15_attention_weight_min": -32.00496292114258,
      "activations/layer16_attention_weight_max": 34.70431900024414,
      "activations/layer16_attention_weight_min": -34.27680206298828,
      "activations/layer17_attention_weight_max": 52.67605972290039,
      "activations/layer17_attention_weight_min": -48.13813018798828,
      "activations/layer18_attention_weight_max": 47.7872200012207,
      "activations/layer18_attention_weight_min": -41.82681655883789,
      "activations/layer19_attention_weight_max": 23.64710807800293,
      "activations/layer19_attention_weight_min": -22.762805938720703,
      "activations/layer1_attention_weight_max": 15.214836120605469,
      "activations/layer1_attention_weight_min": -13.38923454284668,
      "activations/layer20_attention_weight_max": 22.620609283447266,
      "activations/layer20_attention_weight_min": -20.59743309020996,
      "activations/layer21_attention_weight_max": 31.930648803710938,
      "activations/layer21_attention_weight_min": -20.14639663696289,
      "activations/layer22_attention_weight_max": 30.00356101989746,
      "activations/layer22_attention_weight_min": -25.276283264160156,
      "activations/layer23_attention_weight_max": 34.5935173034668,
      "activations/layer23_attention_weight_min": -23.318721771240234,
      "activations/layer2_attention_weight_max": 31.144821166992188,
      "activations/layer2_attention_weight_min": -28.987327575683594,
      "activations/layer3_attention_weight_max": 88.79566955566406,
      "activations/layer3_attention_weight_min": -89.15985107421875,
      "activations/layer4_attention_weight_max": 85.0776596069336,
      "activations/layer4_attention_weight_min": -84.04634857177734,
      "activations/layer5_attention_weight_max": 64.56299591064453,
      "activations/layer5_attention_weight_min": -73.09346008300781,
      "activations/layer6_attention_weight_max": 50.04881286621094,
      "activations/layer6_attention_weight_min": -49.78323745727539,
      "activations/layer7_attention_weight_max": 69.91447448730469,
      "activations/layer7_attention_weight_min": -71.6566162109375,
      "activations/layer8_attention_weight_max": 51.64857482910156,
      "activations/layer8_attention_weight_min": -57.21889877319336,
      "activations/layer9_attention_weight_max": 45.677913665771484,
      "activations/layer9_attention_weight_min": -49.14951705932617,
      "epoch": 22.53,
      "learning_rate": 4.756818181818181e-06,
      "loss": 2.7062,
      "step": 387750
    },
    {
      "activations/layer0_attention_weight_max": 16.281509399414062,
      "activations/layer0_attention_weight_min": -14.251014709472656,
      "activations/layer10_attention_weight_max": 33.82391357421875,
      "activations/layer10_attention_weight_min": -35.430973052978516,
      "activations/layer11_attention_weight_max": 34.57196044921875,
      "activations/layer11_attention_weight_min": -34.116947174072266,
      "activations/layer12_attention_weight_max": 19.2828369140625,
      "activations/layer12_attention_weight_min": -25.331953048706055,
      "activations/layer13_attention_weight_max": 33.97944259643555,
      "activations/layer13_attention_weight_min": -29.48120880126953,
      "activations/layer14_attention_weight_max": 38.6145133972168,
      "activations/layer14_attention_weight_min": -32.451507568359375,
      "activations/layer15_attention_weight_max": 35.42046356201172,
      "activations/layer15_attention_weight_min": -33.25581741333008,
      "activations/layer16_attention_weight_max": 41.768890380859375,
      "activations/layer16_attention_weight_min": -39.64320755004883,
      "activations/layer17_attention_weight_max": 62.10358428955078,
      "activations/layer17_attention_weight_min": -55.57158660888672,
      "activations/layer18_attention_weight_max": 59.70433807373047,
      "activations/layer18_attention_weight_min": -53.926570892333984,
      "activations/layer19_attention_weight_max": 27.59895133972168,
      "activations/layer19_attention_weight_min": -26.249292373657227,
      "activations/layer1_attention_weight_max": 14.568532943725586,
      "activations/layer1_attention_weight_min": -13.810139656066895,
      "activations/layer20_attention_weight_max": 25.743850708007812,
      "activations/layer20_attention_weight_min": -22.194244384765625,
      "activations/layer21_attention_weight_max": 38.56737518310547,
      "activations/layer21_attention_weight_min": -22.778564453125,
      "activations/layer22_attention_weight_max": 29.55194664001465,
      "activations/layer22_attention_weight_min": -29.55103874206543,
      "activations/layer23_attention_weight_max": 39.73537826538086,
      "activations/layer23_attention_weight_min": -23.560026168823242,
      "activations/layer2_attention_weight_max": 31.198400497436523,
      "activations/layer2_attention_weight_min": -30.67597198486328,
      "activations/layer3_attention_weight_max": 86.4495620727539,
      "activations/layer3_attention_weight_min": -91.07381439208984,
      "activations/layer4_attention_weight_max": 83.64617156982422,
      "activations/layer4_attention_weight_min": -84.9604721069336,
      "activations/layer5_attention_weight_max": 60.95016098022461,
      "activations/layer5_attention_weight_min": -72.94413757324219,
      "activations/layer6_attention_weight_max": 49.501102447509766,
      "activations/layer6_attention_weight_min": -49.29362869262695,
      "activations/layer7_attention_weight_max": 73.46784973144531,
      "activations/layer7_attention_weight_min": -66.92308044433594,
      "activations/layer8_attention_weight_max": 47.93686294555664,
      "activations/layer8_attention_weight_min": -49.098045349121094,
      "activations/layer9_attention_weight_max": 43.69594192504883,
      "activations/layer9_attention_weight_min": -45.45509719848633,
      "epoch": 22.53,
      "learning_rate": 4.738257575757575e-06,
      "loss": 2.6936,
      "step": 387800
    },
    {
      "activations/layer0_attention_weight_max": 15.240887641906738,
      "activations/layer0_attention_weight_min": -14.223628997802734,
      "activations/layer10_attention_weight_max": 35.65900802612305,
      "activations/layer10_attention_weight_min": -35.23812484741211,
      "activations/layer11_attention_weight_max": 32.99827575683594,
      "activations/layer11_attention_weight_min": -35.05645751953125,
      "activations/layer12_attention_weight_max": 19.719959259033203,
      "activations/layer12_attention_weight_min": -25.85308074951172,
      "activations/layer13_attention_weight_max": 35.543243408203125,
      "activations/layer13_attention_weight_min": -27.336214065551758,
      "activations/layer14_attention_weight_max": 34.56539535522461,
      "activations/layer14_attention_weight_min": -30.919265747070312,
      "activations/layer15_attention_weight_max": 33.05978775024414,
      "activations/layer15_attention_weight_min": -29.241554260253906,
      "activations/layer16_attention_weight_max": 35.922157287597656,
      "activations/layer16_attention_weight_min": -33.01808547973633,
      "activations/layer17_attention_weight_max": 53.4600944519043,
      "activations/layer17_attention_weight_min": -45.62222671508789,
      "activations/layer18_attention_weight_max": 50.93663787841797,
      "activations/layer18_attention_weight_min": -41.1972770690918,
      "activations/layer19_attention_weight_max": 25.85331153869629,
      "activations/layer19_attention_weight_min": -21.733083724975586,
      "activations/layer1_attention_weight_max": 15.06281852722168,
      "activations/layer1_attention_weight_min": -14.120007514953613,
      "activations/layer20_attention_weight_max": 22.785799026489258,
      "activations/layer20_attention_weight_min": -22.42017364501953,
      "activations/layer21_attention_weight_max": 37.08149719238281,
      "activations/layer21_attention_weight_min": -23.712560653686523,
      "activations/layer22_attention_weight_max": 30.983139038085938,
      "activations/layer22_attention_weight_min": -27.08946990966797,
      "activations/layer23_attention_weight_max": 38.125022888183594,
      "activations/layer23_attention_weight_min": -25.36510467529297,
      "activations/layer2_attention_weight_max": 31.43807601928711,
      "activations/layer2_attention_weight_min": -30.984663009643555,
      "activations/layer3_attention_weight_max": 86.07172393798828,
      "activations/layer3_attention_weight_min": -92.09864044189453,
      "activations/layer4_attention_weight_max": 81.14574432373047,
      "activations/layer4_attention_weight_min": -83.96112060546875,
      "activations/layer5_attention_weight_max": 59.61308288574219,
      "activations/layer5_attention_weight_min": -74.57685089111328,
      "activations/layer6_attention_weight_max": 48.655460357666016,
      "activations/layer6_attention_weight_min": -48.939903259277344,
      "activations/layer7_attention_weight_max": 69.68550109863281,
      "activations/layer7_attention_weight_min": -68.21058654785156,
      "activations/layer8_attention_weight_max": 44.4898681640625,
      "activations/layer8_attention_weight_min": -50.059959411621094,
      "activations/layer9_attention_weight_max": 41.36574935913086,
      "activations/layer9_attention_weight_min": -44.15921401977539,
      "epoch": 22.54,
      "learning_rate": 4.719318181818182e-06,
      "loss": 2.69,
      "step": 387850
    },
    {
      "activations/layer0_attention_weight_max": 15.353130340576172,
      "activations/layer0_attention_weight_min": -13.871439933776855,
      "activations/layer10_attention_weight_max": 38.32509994506836,
      "activations/layer10_attention_weight_min": -36.19963455200195,
      "activations/layer11_attention_weight_max": 33.641868591308594,
      "activations/layer11_attention_weight_min": -35.33100509643555,
      "activations/layer12_attention_weight_max": 19.205469131469727,
      "activations/layer12_attention_weight_min": -22.697757720947266,
      "activations/layer13_attention_weight_max": 36.120391845703125,
      "activations/layer13_attention_weight_min": -29.510398864746094,
      "activations/layer14_attention_weight_max": 37.17544174194336,
      "activations/layer14_attention_weight_min": -29.93792152404785,
      "activations/layer15_attention_weight_max": 34.0076789855957,
      "activations/layer15_attention_weight_min": -30.399633407592773,
      "activations/layer16_attention_weight_max": 35.272010803222656,
      "activations/layer16_attention_weight_min": -32.69662857055664,
      "activations/layer17_attention_weight_max": 53.253936767578125,
      "activations/layer17_attention_weight_min": -44.85769271850586,
      "activations/layer18_attention_weight_max": 47.9616813659668,
      "activations/layer18_attention_weight_min": -40.1261100769043,
      "activations/layer19_attention_weight_max": 24.74519157409668,
      "activations/layer19_attention_weight_min": -22.513647079467773,
      "activations/layer1_attention_weight_max": 15.88787841796875,
      "activations/layer1_attention_weight_min": -15.993738174438477,
      "activations/layer20_attention_weight_max": 22.179161071777344,
      "activations/layer20_attention_weight_min": -20.445537567138672,
      "activations/layer21_attention_weight_max": 35.19684600830078,
      "activations/layer21_attention_weight_min": -23.099117279052734,
      "activations/layer22_attention_weight_max": 27.218183517456055,
      "activations/layer22_attention_weight_min": -27.412336349487305,
      "activations/layer23_attention_weight_max": 36.2169303894043,
      "activations/layer23_attention_weight_min": -25.098915100097656,
      "activations/layer2_attention_weight_max": 31.08258628845215,
      "activations/layer2_attention_weight_min": -29.74488639831543,
      "activations/layer3_attention_weight_max": 89.88719177246094,
      "activations/layer3_attention_weight_min": -85.56748962402344,
      "activations/layer4_attention_weight_max": 83.18221282958984,
      "activations/layer4_attention_weight_min": -85.51771545410156,
      "activations/layer5_attention_weight_max": 59.010826110839844,
      "activations/layer5_attention_weight_min": -74.7286376953125,
      "activations/layer6_attention_weight_max": 49.356910705566406,
      "activations/layer6_attention_weight_min": -50.69343566894531,
      "activations/layer7_attention_weight_max": 67.49147033691406,
      "activations/layer7_attention_weight_min": -69.82913970947266,
      "activations/layer8_attention_weight_max": 47.514259338378906,
      "activations/layer8_attention_weight_min": -50.74890899658203,
      "activations/layer9_attention_weight_max": 43.35905075073242,
      "activations/layer9_attention_weight_min": -46.729164123535156,
      "epoch": 22.54,
      "learning_rate": 4.7003787878787875e-06,
      "loss": 2.7124,
      "step": 387900
    },
    {
      "activations/layer0_attention_weight_max": 15.660804748535156,
      "activations/layer0_attention_weight_min": -14.319907188415527,
      "activations/layer10_attention_weight_max": 35.707366943359375,
      "activations/layer10_attention_weight_min": -36.93581771850586,
      "activations/layer11_attention_weight_max": 33.40141677856445,
      "activations/layer11_attention_weight_min": -35.55424880981445,
      "activations/layer12_attention_weight_max": 20.240779876708984,
      "activations/layer12_attention_weight_min": -25.284217834472656,
      "activations/layer13_attention_weight_max": 41.17878341674805,
      "activations/layer13_attention_weight_min": -31.036935806274414,
      "activations/layer14_attention_weight_max": 44.56705093383789,
      "activations/layer14_attention_weight_min": -30.3470401763916,
      "activations/layer15_attention_weight_max": 44.06892395019531,
      "activations/layer15_attention_weight_min": -31.545989990234375,
      "activations/layer16_attention_weight_max": 38.46143341064453,
      "activations/layer16_attention_weight_min": -33.836509704589844,
      "activations/layer17_attention_weight_max": 58.37971496582031,
      "activations/layer17_attention_weight_min": -49.49018478393555,
      "activations/layer18_attention_weight_max": 55.43806457519531,
      "activations/layer18_attention_weight_min": -45.746437072753906,
      "activations/layer19_attention_weight_max": 30.669342041015625,
      "activations/layer19_attention_weight_min": -26.5220890045166,
      "activations/layer1_attention_weight_max": 15.906662940979004,
      "activations/layer1_attention_weight_min": -14.778618812561035,
      "activations/layer20_attention_weight_max": 26.792068481445312,
      "activations/layer20_attention_weight_min": -22.62656021118164,
      "activations/layer21_attention_weight_max": 52.88245391845703,
      "activations/layer21_attention_weight_min": -28.89940643310547,
      "activations/layer22_attention_weight_max": 35.795230865478516,
      "activations/layer22_attention_weight_min": -25.56863784790039,
      "activations/layer23_attention_weight_max": 44.004608154296875,
      "activations/layer23_attention_weight_min": -24.130634307861328,
      "activations/layer2_attention_weight_max": 31.975177764892578,
      "activations/layer2_attention_weight_min": -29.68977165222168,
      "activations/layer3_attention_weight_max": 91.34607696533203,
      "activations/layer3_attention_weight_min": -89.57115173339844,
      "activations/layer4_attention_weight_max": 86.9155502319336,
      "activations/layer4_attention_weight_min": -89.72354125976562,
      "activations/layer5_attention_weight_max": 65.37489318847656,
      "activations/layer5_attention_weight_min": -73.03070068359375,
      "activations/layer6_attention_weight_max": 52.51156234741211,
      "activations/layer6_attention_weight_min": -52.735843658447266,
      "activations/layer7_attention_weight_max": 77.19987487792969,
      "activations/layer7_attention_weight_min": -74.98912811279297,
      "activations/layer8_attention_weight_max": 50.5169563293457,
      "activations/layer8_attention_weight_min": -54.91956329345703,
      "activations/layer9_attention_weight_max": 45.62043762207031,
      "activations/layer9_attention_weight_min": -49.454402923583984,
      "epoch": 22.54,
      "learning_rate": 4.681439393939393e-06,
      "loss": 2.704,
      "step": 387950
    },
    {
      "activations/layer0_attention_weight_max": 14.689554214477539,
      "activations/layer0_attention_weight_min": -14.09177017211914,
      "activations/layer10_attention_weight_max": 39.800838470458984,
      "activations/layer10_attention_weight_min": -37.948177337646484,
      "activations/layer11_attention_weight_max": 34.51751708984375,
      "activations/layer11_attention_weight_min": -36.39842224121094,
      "activations/layer12_attention_weight_max": 19.210041046142578,
      "activations/layer12_attention_weight_min": -24.577045440673828,
      "activations/layer13_attention_weight_max": 36.061710357666016,
      "activations/layer13_attention_weight_min": -30.24815559387207,
      "activations/layer14_attention_weight_max": 35.21730422973633,
      "activations/layer14_attention_weight_min": -32.00294876098633,
      "activations/layer15_attention_weight_max": 33.64077377319336,
      "activations/layer15_attention_weight_min": -32.071075439453125,
      "activations/layer16_attention_weight_max": 33.48780059814453,
      "activations/layer16_attention_weight_min": -31.887203216552734,
      "activations/layer17_attention_weight_max": 49.52399826049805,
      "activations/layer17_attention_weight_min": -44.60325241088867,
      "activations/layer18_attention_weight_max": 49.75627517700195,
      "activations/layer18_attention_weight_min": -41.72947692871094,
      "activations/layer19_attention_weight_max": 26.8306941986084,
      "activations/layer19_attention_weight_min": -23.258853912353516,
      "activations/layer1_attention_weight_max": 14.522953033447266,
      "activations/layer1_attention_weight_min": -14.345455169677734,
      "activations/layer20_attention_weight_max": 27.203651428222656,
      "activations/layer20_attention_weight_min": -21.197561264038086,
      "activations/layer21_attention_weight_max": 37.540321350097656,
      "activations/layer21_attention_weight_min": -22.3041934967041,
      "activations/layer22_attention_weight_max": 30.983247756958008,
      "activations/layer22_attention_weight_min": -25.296539306640625,
      "activations/layer23_attention_weight_max": 38.07229232788086,
      "activations/layer23_attention_weight_min": -26.000768661499023,
      "activations/layer2_attention_weight_max": 32.057777404785156,
      "activations/layer2_attention_weight_min": -29.67724609375,
      "activations/layer3_attention_weight_max": 89.0686264038086,
      "activations/layer3_attention_weight_min": -89.39755249023438,
      "activations/layer4_attention_weight_max": 87.0617904663086,
      "activations/layer4_attention_weight_min": -87.73957061767578,
      "activations/layer5_attention_weight_max": 63.38145446777344,
      "activations/layer5_attention_weight_min": -76.24119567871094,
      "activations/layer6_attention_weight_max": 50.338783264160156,
      "activations/layer6_attention_weight_min": -49.36369323730469,
      "activations/layer7_attention_weight_max": 72.41532135009766,
      "activations/layer7_attention_weight_min": -68.87276458740234,
      "activations/layer8_attention_weight_max": 51.4886360168457,
      "activations/layer8_attention_weight_min": -50.021785736083984,
      "activations/layer9_attention_weight_max": 50.41749572753906,
      "activations/layer9_attention_weight_min": -46.65481948852539,
      "epoch": 22.55,
      "learning_rate": 4.6625e-06,
      "loss": 2.7273,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_loss": 2.666015625,
      "eval_runtime": 8.4669,
      "eval_samples_per_second": 507.149,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_openwebtext_loss": 2.666015625,
      "eval_openwebtext_ppl": 14.382549407491597,
      "eval_openwebtext_runtime": 8.4669,
      "eval_openwebtext_samples_per_second": 507.149,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9512,
      "eval_wikitext_samples_per_second": 233.701,
      "step": 388000
    },
    {
      "epoch": 22.55,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5366,
      "eval_lambada_samples_per_second": 510.561,
      "step": 388000
    },
    {
      "activations/layer0_attention_weight_max": 15.607460975646973,
      "activations/layer0_attention_weight_min": -14.633841514587402,
      "activations/layer10_attention_weight_max": 41.848968505859375,
      "activations/layer10_attention_weight_min": -40.90934371948242,
      "activations/layer11_attention_weight_max": 39.967010498046875,
      "activations/layer11_attention_weight_min": -37.45102310180664,
      "activations/layer12_attention_weight_max": 24.773929595947266,
      "activations/layer12_attention_weight_min": -24.63528060913086,
      "activations/layer13_attention_weight_max": 41.73602294921875,
      "activations/layer13_attention_weight_min": -31.067750930786133,
      "activations/layer14_attention_weight_max": 36.992549896240234,
      "activations/layer14_attention_weight_min": -35.648963928222656,
      "activations/layer15_attention_weight_max": 39.188392639160156,
      "activations/layer15_attention_weight_min": -34.71983337402344,
      "activations/layer16_attention_weight_max": 37.22731399536133,
      "activations/layer16_attention_weight_min": -36.4455451965332,
      "activations/layer17_attention_weight_max": 56.80073928833008,
      "activations/layer17_attention_weight_min": -50.311973571777344,
      "activations/layer18_attention_weight_max": 50.627498626708984,
      "activations/layer18_attention_weight_min": -43.75303268432617,
      "activations/layer19_attention_weight_max": 25.13313865661621,
      "activations/layer19_attention_weight_min": -24.11956214904785,
      "activations/layer1_attention_weight_max": 14.860605239868164,
      "activations/layer1_attention_weight_min": -15.346972465515137,
      "activations/layer20_attention_weight_max": 24.442527770996094,
      "activations/layer20_attention_weight_min": -22.267868041992188,
      "activations/layer21_attention_weight_max": 36.89228820800781,
      "activations/layer21_attention_weight_min": -22.08963966369629,
      "activations/layer22_attention_weight_max": 28.25258445739746,
      "activations/layer22_attention_weight_min": -25.514150619506836,
      "activations/layer23_attention_weight_max": 34.48524475097656,
      "activations/layer23_attention_weight_min": -24.669872283935547,
      "activations/layer2_attention_weight_max": 32.696441650390625,
      "activations/layer2_attention_weight_min": -32.81168746948242,
      "activations/layer3_attention_weight_max": 97.91832733154297,
      "activations/layer3_attention_weight_min": -94.95912170410156,
      "activations/layer4_attention_weight_max": 87.49764251708984,
      "activations/layer4_attention_weight_min": -90.99836730957031,
      "activations/layer5_attention_weight_max": 64.12714385986328,
      "activations/layer5_attention_weight_min": -74.5172119140625,
      "activations/layer6_attention_weight_max": 52.464534759521484,
      "activations/layer6_attention_weight_min": -52.308921813964844,
      "activations/layer7_attention_weight_max": 74.99921417236328,
      "activations/layer7_attention_weight_min": -76.73352813720703,
      "activations/layer8_attention_weight_max": 52.50770950317383,
      "activations/layer8_attention_weight_min": -52.67647171020508,
      "activations/layer9_attention_weight_max": 48.75455856323242,
      "activations/layer9_attention_weight_min": -51.1839714050293,
      "epoch": 22.55,
      "learning_rate": 4.6435606060606056e-06,
      "loss": 2.697,
      "step": 388050
    },
    {
      "activations/layer0_attention_weight_max": 15.062302589416504,
      "activations/layer0_attention_weight_min": -13.884657859802246,
      "activations/layer10_attention_weight_max": 36.714778900146484,
      "activations/layer10_attention_weight_min": -35.58562088012695,
      "activations/layer11_attention_weight_max": 34.76091766357422,
      "activations/layer11_attention_weight_min": -33.97444152832031,
      "activations/layer12_attention_weight_max": 18.471820831298828,
      "activations/layer12_attention_weight_min": -25.652284622192383,
      "activations/layer13_attention_weight_max": 34.897518157958984,
      "activations/layer13_attention_weight_min": -28.442773818969727,
      "activations/layer14_attention_weight_max": 36.76259994506836,
      "activations/layer14_attention_weight_min": -30.325763702392578,
      "activations/layer15_attention_weight_max": 32.863502502441406,
      "activations/layer15_attention_weight_min": -31.48884391784668,
      "activations/layer16_attention_weight_max": 33.27446365356445,
      "activations/layer16_attention_weight_min": -32.841365814208984,
      "activations/layer17_attention_weight_max": 50.63961410522461,
      "activations/layer17_attention_weight_min": -46.671878814697266,
      "activations/layer18_attention_weight_max": 47.157684326171875,
      "activations/layer18_attention_weight_min": -42.3691291809082,
      "activations/layer19_attention_weight_max": 26.275806427001953,
      "activations/layer19_attention_weight_min": -24.455644607543945,
      "activations/layer1_attention_weight_max": 15.206422805786133,
      "activations/layer1_attention_weight_min": -15.43917179107666,
      "activations/layer20_attention_weight_max": 22.915109634399414,
      "activations/layer20_attention_weight_min": -22.9263858795166,
      "activations/layer21_attention_weight_max": 39.57138442993164,
      "activations/layer21_attention_weight_min": -21.67205810546875,
      "activations/layer22_attention_weight_max": 30.101593017578125,
      "activations/layer22_attention_weight_min": -25.40619468688965,
      "activations/layer23_attention_weight_max": 36.20491027832031,
      "activations/layer23_attention_weight_min": -25.322341918945312,
      "activations/layer2_attention_weight_max": 30.912960052490234,
      "activations/layer2_attention_weight_min": -29.471588134765625,
      "activations/layer3_attention_weight_max": 85.21205139160156,
      "activations/layer3_attention_weight_min": -84.4578857421875,
      "activations/layer4_attention_weight_max": 85.8614501953125,
      "activations/layer4_attention_weight_min": -85.27385711669922,
      "activations/layer5_attention_weight_max": 63.035423278808594,
      "activations/layer5_attention_weight_min": -75.34595489501953,
      "activations/layer6_attention_weight_max": 48.96047592163086,
      "activations/layer6_attention_weight_min": -51.36121368408203,
      "activations/layer7_attention_weight_max": 68.73890686035156,
      "activations/layer7_attention_weight_min": -68.4865493774414,
      "activations/layer8_attention_weight_max": 47.0651741027832,
      "activations/layer8_attention_weight_min": -51.5428352355957,
      "activations/layer9_attention_weight_max": 43.80304718017578,
      "activations/layer9_attention_weight_min": -44.669986724853516,
      "epoch": 22.55,
      "learning_rate": 4.624621212121211e-06,
      "loss": 2.704,
      "step": 388100
    },
    {
      "activations/layer0_attention_weight_max": 15.65703010559082,
      "activations/layer0_attention_weight_min": -14.35332202911377,
      "activations/layer10_attention_weight_max": 34.742820739746094,
      "activations/layer10_attention_weight_min": -37.227264404296875,
      "activations/layer11_attention_weight_max": 33.20809555053711,
      "activations/layer11_attention_weight_min": -34.46080017089844,
      "activations/layer12_attention_weight_max": 18.723764419555664,
      "activations/layer12_attention_weight_min": -24.60779571533203,
      "activations/layer13_attention_weight_max": 35.393348693847656,
      "activations/layer13_attention_weight_min": -30.13467788696289,
      "activations/layer14_attention_weight_max": 33.4123649597168,
      "activations/layer14_attention_weight_min": -30.527137756347656,
      "activations/layer15_attention_weight_max": 29.61509132385254,
      "activations/layer15_attention_weight_min": -32.30332565307617,
      "activations/layer16_attention_weight_max": 32.13264465332031,
      "activations/layer16_attention_weight_min": -31.467777252197266,
      "activations/layer17_attention_weight_max": 48.52879333496094,
      "activations/layer17_attention_weight_min": -43.7736701965332,
      "activations/layer18_attention_weight_max": 44.11954116821289,
      "activations/layer18_attention_weight_min": -41.456878662109375,
      "activations/layer19_attention_weight_max": 22.510940551757812,
      "activations/layer19_attention_weight_min": -22.58073616027832,
      "activations/layer1_attention_weight_max": 14.838249206542969,
      "activations/layer1_attention_weight_min": -14.742501258850098,
      "activations/layer20_attention_weight_max": 23.010326385498047,
      "activations/layer20_attention_weight_min": -20.649826049804688,
      "activations/layer21_attention_weight_max": 29.51910400390625,
      "activations/layer21_attention_weight_min": -20.30331802368164,
      "activations/layer22_attention_weight_max": 29.609766006469727,
      "activations/layer22_attention_weight_min": -25.82805824279785,
      "activations/layer23_attention_weight_max": 35.92750930786133,
      "activations/layer23_attention_weight_min": -25.81947898864746,
      "activations/layer2_attention_weight_max": 31.681325912475586,
      "activations/layer2_attention_weight_min": -31.01766586303711,
      "activations/layer3_attention_weight_max": 90.73527526855469,
      "activations/layer3_attention_weight_min": -89.67940521240234,
      "activations/layer4_attention_weight_max": 85.37252044677734,
      "activations/layer4_attention_weight_min": -85.65792083740234,
      "activations/layer5_attention_weight_max": 61.03626251220703,
      "activations/layer5_attention_weight_min": -72.23826599121094,
      "activations/layer6_attention_weight_max": 50.200748443603516,
      "activations/layer6_attention_weight_min": -51.21327590942383,
      "activations/layer7_attention_weight_max": 67.94554138183594,
      "activations/layer7_attention_weight_min": -67.94454956054688,
      "activations/layer8_attention_weight_max": 48.62113952636719,
      "activations/layer8_attention_weight_min": -51.82543182373047,
      "activations/layer9_attention_weight_max": 43.347652435302734,
      "activations/layer9_attention_weight_min": -46.525508880615234,
      "epoch": 22.55,
      "learning_rate": 4.605681818181818e-06,
      "loss": 2.7063,
      "step": 388150
    },
    {
      "activations/layer0_attention_weight_max": 15.607422828674316,
      "activations/layer0_attention_weight_min": -14.211054801940918,
      "activations/layer10_attention_weight_max": 36.11964416503906,
      "activations/layer10_attention_weight_min": -37.25033950805664,
      "activations/layer11_attention_weight_max": 33.12595748901367,
      "activations/layer11_attention_weight_min": -36.72111892700195,
      "activations/layer12_attention_weight_max": 18.8177433013916,
      "activations/layer12_attention_weight_min": -27.16884994506836,
      "activations/layer13_attention_weight_max": 32.2612419128418,
      "activations/layer13_attention_weight_min": -27.5537052154541,
      "activations/layer14_attention_weight_max": 35.71129608154297,
      "activations/layer14_attention_weight_min": -30.843509674072266,
      "activations/layer15_attention_weight_max": 35.98151779174805,
      "activations/layer15_attention_weight_min": -30.091590881347656,
      "activations/layer16_attention_weight_max": 35.10652542114258,
      "activations/layer16_attention_weight_min": -32.81221389770508,
      "activations/layer17_attention_weight_max": 50.16729736328125,
      "activations/layer17_attention_weight_min": -48.3239860534668,
      "activations/layer18_attention_weight_max": 46.687957763671875,
      "activations/layer18_attention_weight_min": -39.64084243774414,
      "activations/layer19_attention_weight_max": 23.774385452270508,
      "activations/layer19_attention_weight_min": -22.41097068786621,
      "activations/layer1_attention_weight_max": 14.947798728942871,
      "activations/layer1_attention_weight_min": -15.3820219039917,
      "activations/layer20_attention_weight_max": 23.843292236328125,
      "activations/layer20_attention_weight_min": -21.822124481201172,
      "activations/layer21_attention_weight_max": 37.6479377746582,
      "activations/layer21_attention_weight_min": -21.810529708862305,
      "activations/layer22_attention_weight_max": 31.901668548583984,
      "activations/layer22_attention_weight_min": -25.03260040283203,
      "activations/layer23_attention_weight_max": 38.21589660644531,
      "activations/layer23_attention_weight_min": -23.193111419677734,
      "activations/layer2_attention_weight_max": 31.869842529296875,
      "activations/layer2_attention_weight_min": -30.290821075439453,
      "activations/layer3_attention_weight_max": 90.58734130859375,
      "activations/layer3_attention_weight_min": -87.0975570678711,
      "activations/layer4_attention_weight_max": 87.56425476074219,
      "activations/layer4_attention_weight_min": -86.46929931640625,
      "activations/layer5_attention_weight_max": 66.18598937988281,
      "activations/layer5_attention_weight_min": -78.48252868652344,
      "activations/layer6_attention_weight_max": 49.55173110961914,
      "activations/layer6_attention_weight_min": -53.18386459350586,
      "activations/layer7_attention_weight_max": 66.86898040771484,
      "activations/layer7_attention_weight_min": -70.94219970703125,
      "activations/layer8_attention_weight_max": 47.56952667236328,
      "activations/layer8_attention_weight_min": -50.05080795288086,
      "activations/layer9_attention_weight_max": 47.72500228881836,
      "activations/layer9_attention_weight_min": -46.49180221557617,
      "epoch": 22.56,
      "learning_rate": 4.5867424242424244e-06,
      "loss": 2.7068,
      "step": 388200
    },
    {
      "activations/layer0_attention_weight_max": 16.26058578491211,
      "activations/layer0_attention_weight_min": -14.19361686706543,
      "activations/layer10_attention_weight_max": 40.07390594482422,
      "activations/layer10_attention_weight_min": -39.13396453857422,
      "activations/layer11_attention_weight_max": 36.2615966796875,
      "activations/layer11_attention_weight_min": -36.728004455566406,
      "activations/layer12_attention_weight_max": 19.207576751708984,
      "activations/layer12_attention_weight_min": -26.66614532470703,
      "activations/layer13_attention_weight_max": 31.71310806274414,
      "activations/layer13_attention_weight_min": -27.71380615234375,
      "activations/layer14_attention_weight_max": 37.63219451904297,
      "activations/layer14_attention_weight_min": -30.98036003112793,
      "activations/layer15_attention_weight_max": 33.47383117675781,
      "activations/layer15_attention_weight_min": -30.893468856811523,
      "activations/layer16_attention_weight_max": 35.47338104248047,
      "activations/layer16_attention_weight_min": -33.584922790527344,
      "activations/layer17_attention_weight_max": 52.62429428100586,
      "activations/layer17_attention_weight_min": -45.98836898803711,
      "activations/layer18_attention_weight_max": 47.57732391357422,
      "activations/layer18_attention_weight_min": -42.98365020751953,
      "activations/layer19_attention_weight_max": 25.973602294921875,
      "activations/layer19_attention_weight_min": -21.555316925048828,
      "activations/layer1_attention_weight_max": 15.327022552490234,
      "activations/layer1_attention_weight_min": -15.766507148742676,
      "activations/layer20_attention_weight_max": 22.904735565185547,
      "activations/layer20_attention_weight_min": -22.305940628051758,
      "activations/layer21_attention_weight_max": 34.84302520751953,
      "activations/layer21_attention_weight_min": -20.510704040527344,
      "activations/layer22_attention_weight_max": 30.293190002441406,
      "activations/layer22_attention_weight_min": -24.41151237487793,
      "activations/layer23_attention_weight_max": 37.93882751464844,
      "activations/layer23_attention_weight_min": -23.903005599975586,
      "activations/layer2_attention_weight_max": 31.96273422241211,
      "activations/layer2_attention_weight_min": -30.82476806640625,
      "activations/layer3_attention_weight_max": 92.44417572021484,
      "activations/layer3_attention_weight_min": -91.2280502319336,
      "activations/layer4_attention_weight_max": 87.41520690917969,
      "activations/layer4_attention_weight_min": -86.48066711425781,
      "activations/layer5_attention_weight_max": 66.52862548828125,
      "activations/layer5_attention_weight_min": -70.20718383789062,
      "activations/layer6_attention_weight_max": 53.766788482666016,
      "activations/layer6_attention_weight_min": -54.52213668823242,
      "activations/layer7_attention_weight_max": 76.69705200195312,
      "activations/layer7_attention_weight_min": -71.31491088867188,
      "activations/layer8_attention_weight_max": 52.5435676574707,
      "activations/layer8_attention_weight_min": -54.49237823486328,
      "activations/layer9_attention_weight_max": 46.06751251220703,
      "activations/layer9_attention_weight_min": -49.67634201049805,
      "epoch": 22.56,
      "learning_rate": 4.56780303030303e-06,
      "loss": 2.6997,
      "step": 388250
    },
    {
      "activations/layer0_attention_weight_max": 15.743180274963379,
      "activations/layer0_attention_weight_min": -14.369729042053223,
      "activations/layer10_attention_weight_max": 37.891056060791016,
      "activations/layer10_attention_weight_min": -37.204437255859375,
      "activations/layer11_attention_weight_max": 34.294593811035156,
      "activations/layer11_attention_weight_min": -35.358802795410156,
      "activations/layer12_attention_weight_max": 20.45584487915039,
      "activations/layer12_attention_weight_min": -23.42375946044922,
      "activations/layer13_attention_weight_max": 35.051292419433594,
      "activations/layer13_attention_weight_min": -28.875886917114258,
      "activations/layer14_attention_weight_max": 36.266326904296875,
      "activations/layer14_attention_weight_min": -30.19988441467285,
      "activations/layer15_attention_weight_max": 34.799583435058594,
      "activations/layer15_attention_weight_min": -30.417343139648438,
      "activations/layer16_attention_weight_max": 38.251895904541016,
      "activations/layer16_attention_weight_min": -34.645992279052734,
      "activations/layer17_attention_weight_max": 54.91627502441406,
      "activations/layer17_attention_weight_min": -47.87112808227539,
      "activations/layer18_attention_weight_max": 54.57034683227539,
      "activations/layer18_attention_weight_min": -42.7175407409668,
      "activations/layer19_attention_weight_max": 26.437213897705078,
      "activations/layer19_attention_weight_min": -23.530858993530273,
      "activations/layer1_attention_weight_max": 15.613908767700195,
      "activations/layer1_attention_weight_min": -14.154460906982422,
      "activations/layer20_attention_weight_max": 25.697465896606445,
      "activations/layer20_attention_weight_min": -21.319072723388672,
      "activations/layer21_attention_weight_max": 41.234092712402344,
      "activations/layer21_attention_weight_min": -21.28946876525879,
      "activations/layer22_attention_weight_max": 30.580806732177734,
      "activations/layer22_attention_weight_min": -26.27986717224121,
      "activations/layer23_attention_weight_max": 42.164974212646484,
      "activations/layer23_attention_weight_min": -22.578824996948242,
      "activations/layer2_attention_weight_max": 31.024831771850586,
      "activations/layer2_attention_weight_min": -30.541006088256836,
      "activations/layer3_attention_weight_max": 91.04168701171875,
      "activations/layer3_attention_weight_min": -90.83740997314453,
      "activations/layer4_attention_weight_max": 89.48773956298828,
      "activations/layer4_attention_weight_min": -88.5502700805664,
      "activations/layer5_attention_weight_max": 68.75608825683594,
      "activations/layer5_attention_weight_min": -73.74012756347656,
      "activations/layer6_attention_weight_max": 51.62409210205078,
      "activations/layer6_attention_weight_min": -52.9881477355957,
      "activations/layer7_attention_weight_max": 84.5377426147461,
      "activations/layer7_attention_weight_min": -72.3296890258789,
      "activations/layer8_attention_weight_max": 51.961605072021484,
      "activations/layer8_attention_weight_min": -52.94015884399414,
      "activations/layer9_attention_weight_max": 50.00490188598633,
      "activations/layer9_attention_weight_min": -46.23640823364258,
      "epoch": 22.56,
      "learning_rate": 4.548863636363636e-06,
      "loss": 2.6971,
      "step": 388300
    },
    {
      "activations/layer0_attention_weight_max": 16.74761199951172,
      "activations/layer0_attention_weight_min": -14.421792984008789,
      "activations/layer10_attention_weight_max": 36.86252212524414,
      "activations/layer10_attention_weight_min": -37.743160247802734,
      "activations/layer11_attention_weight_max": 35.439456939697266,
      "activations/layer11_attention_weight_min": -36.60930633544922,
      "activations/layer12_attention_weight_max": 20.420251846313477,
      "activations/layer12_attention_weight_min": -27.70602035522461,
      "activations/layer13_attention_weight_max": 37.74494934082031,
      "activations/layer13_attention_weight_min": -28.6345157623291,
      "activations/layer14_attention_weight_max": 37.19629669189453,
      "activations/layer14_attention_weight_min": -31.756244659423828,
      "activations/layer15_attention_weight_max": 35.57938003540039,
      "activations/layer15_attention_weight_min": -31.088651657104492,
      "activations/layer16_attention_weight_max": 39.64484405517578,
      "activations/layer16_attention_weight_min": -36.4266471862793,
      "activations/layer17_attention_weight_max": 56.10094451904297,
      "activations/layer17_attention_weight_min": -48.282474517822266,
      "activations/layer18_attention_weight_max": 52.34553909301758,
      "activations/layer18_attention_weight_min": -42.09044647216797,
      "activations/layer19_attention_weight_max": 25.594552993774414,
      "activations/layer19_attention_weight_min": -23.57579231262207,
      "activations/layer1_attention_weight_max": 15.578819274902344,
      "activations/layer1_attention_weight_min": -14.450611114501953,
      "activations/layer20_attention_weight_max": 23.861541748046875,
      "activations/layer20_attention_weight_min": -24.089309692382812,
      "activations/layer21_attention_weight_max": 38.49263381958008,
      "activations/layer21_attention_weight_min": -24.176742553710938,
      "activations/layer22_attention_weight_max": 29.587690353393555,
      "activations/layer22_attention_weight_min": -24.82594871520996,
      "activations/layer23_attention_weight_max": 35.63844299316406,
      "activations/layer23_attention_weight_min": -22.59443473815918,
      "activations/layer2_attention_weight_max": 31.90654754638672,
      "activations/layer2_attention_weight_min": -30.444297790527344,
      "activations/layer3_attention_weight_max": 88.44647979736328,
      "activations/layer3_attention_weight_min": -92.45626831054688,
      "activations/layer4_attention_weight_max": 88.14891052246094,
      "activations/layer4_attention_weight_min": -85.60857391357422,
      "activations/layer5_attention_weight_max": 65.50418853759766,
      "activations/layer5_attention_weight_min": -70.87979888916016,
      "activations/layer6_attention_weight_max": 50.02936553955078,
      "activations/layer6_attention_weight_min": -51.51936721801758,
      "activations/layer7_attention_weight_max": 70.6076889038086,
      "activations/layer7_attention_weight_min": -70.65122985839844,
      "activations/layer8_attention_weight_max": 49.1262321472168,
      "activations/layer8_attention_weight_min": -55.02174758911133,
      "activations/layer9_attention_weight_max": 45.3419303894043,
      "activations/layer9_attention_weight_min": -49.9537239074707,
      "epoch": 22.57,
      "learning_rate": 4.529924242424242e-06,
      "loss": 2.6974,
      "step": 388350
    },
    {
      "activations/layer0_attention_weight_max": 16.25697898864746,
      "activations/layer0_attention_weight_min": -14.34218692779541,
      "activations/layer10_attention_weight_max": 45.44388198852539,
      "activations/layer10_attention_weight_min": -46.69963455200195,
      "activations/layer11_attention_weight_max": 43.95463180541992,
      "activations/layer11_attention_weight_min": -46.01777267456055,
      "activations/layer12_attention_weight_max": 20.698455810546875,
      "activations/layer12_attention_weight_min": -28.85411834716797,
      "activations/layer13_attention_weight_max": 46.63766860961914,
      "activations/layer13_attention_weight_min": -36.3811149597168,
      "activations/layer14_attention_weight_max": 57.464630126953125,
      "activations/layer14_attention_weight_min": -36.978858947753906,
      "activations/layer15_attention_weight_max": 59.17095184326172,
      "activations/layer15_attention_weight_min": -42.943092346191406,
      "activations/layer16_attention_weight_max": 42.75786590576172,
      "activations/layer16_attention_weight_min": -39.01004409790039,
      "activations/layer17_attention_weight_max": 63.42829132080078,
      "activations/layer17_attention_weight_min": -59.60047912597656,
      "activations/layer18_attention_weight_max": 55.64823913574219,
      "activations/layer18_attention_weight_min": -49.08462142944336,
      "activations/layer19_attention_weight_max": 28.574499130249023,
      "activations/layer19_attention_weight_min": -25.928945541381836,
      "activations/layer1_attention_weight_max": 15.594352722167969,
      "activations/layer1_attention_weight_min": -14.562447547912598,
      "activations/layer20_attention_weight_max": 30.07937240600586,
      "activations/layer20_attention_weight_min": -21.439971923828125,
      "activations/layer21_attention_weight_max": 49.998809814453125,
      "activations/layer21_attention_weight_min": -22.57196617126465,
      "activations/layer22_attention_weight_max": 30.90335464477539,
      "activations/layer22_attention_weight_min": -24.41684341430664,
      "activations/layer23_attention_weight_max": 36.269203186035156,
      "activations/layer23_attention_weight_min": -23.30915641784668,
      "activations/layer2_attention_weight_max": 33.6026611328125,
      "activations/layer2_attention_weight_min": -30.504657745361328,
      "activations/layer3_attention_weight_max": 99.49388885498047,
      "activations/layer3_attention_weight_min": -93.42777252197266,
      "activations/layer4_attention_weight_max": 88.94933319091797,
      "activations/layer4_attention_weight_min": -85.29581451416016,
      "activations/layer5_attention_weight_max": 62.13656234741211,
      "activations/layer5_attention_weight_min": -72.89432525634766,
      "activations/layer6_attention_weight_max": 54.381412506103516,
      "activations/layer6_attention_weight_min": -50.86945724487305,
      "activations/layer7_attention_weight_max": 86.88351440429688,
      "activations/layer7_attention_weight_min": -71.04640197753906,
      "activations/layer8_attention_weight_max": 57.03384780883789,
      "activations/layer8_attention_weight_min": -55.741455078125,
      "activations/layer9_attention_weight_max": 65.14116668701172,
      "activations/layer9_attention_weight_min": -51.3166389465332,
      "epoch": 22.57,
      "learning_rate": 4.510984848484848e-06,
      "loss": 2.7144,
      "step": 388400
    },
    {
      "activations/layer0_attention_weight_max": 16.75969886779785,
      "activations/layer0_attention_weight_min": -14.190235137939453,
      "activations/layer10_attention_weight_max": 35.05533981323242,
      "activations/layer10_attention_weight_min": -36.4644660949707,
      "activations/layer11_attention_weight_max": 32.71786880493164,
      "activations/layer11_attention_weight_min": -35.267356872558594,
      "activations/layer12_attention_weight_max": 18.71702003479004,
      "activations/layer12_attention_weight_min": -22.010883331298828,
      "activations/layer13_attention_weight_max": 35.885093688964844,
      "activations/layer13_attention_weight_min": -27.139751434326172,
      "activations/layer14_attention_weight_max": 36.28784942626953,
      "activations/layer14_attention_weight_min": -30.90821075439453,
      "activations/layer15_attention_weight_max": 36.18351364135742,
      "activations/layer15_attention_weight_min": -30.473024368286133,
      "activations/layer16_attention_weight_max": 36.12159729003906,
      "activations/layer16_attention_weight_min": -34.23453903198242,
      "activations/layer17_attention_weight_max": 59.95798110961914,
      "activations/layer17_attention_weight_min": -50.191402435302734,
      "activations/layer18_attention_weight_max": 53.503021240234375,
      "activations/layer18_attention_weight_min": -46.07225799560547,
      "activations/layer19_attention_weight_max": 27.542713165283203,
      "activations/layer19_attention_weight_min": -24.323476791381836,
      "activations/layer1_attention_weight_max": 14.971903800964355,
      "activations/layer1_attention_weight_min": -14.24911880493164,
      "activations/layer20_attention_weight_max": 25.877050399780273,
      "activations/layer20_attention_weight_min": -22.66657066345215,
      "activations/layer21_attention_weight_max": 46.54838180541992,
      "activations/layer21_attention_weight_min": -22.6768798828125,
      "activations/layer22_attention_weight_max": 32.20963668823242,
      "activations/layer22_attention_weight_min": -26.84108543395996,
      "activations/layer23_attention_weight_max": 39.62614059448242,
      "activations/layer23_attention_weight_min": -25.232763290405273,
      "activations/layer2_attention_weight_max": 32.60505294799805,
      "activations/layer2_attention_weight_min": -30.62798309326172,
      "activations/layer3_attention_weight_max": 89.15668487548828,
      "activations/layer3_attention_weight_min": -94.87641143798828,
      "activations/layer4_attention_weight_max": 83.46920013427734,
      "activations/layer4_attention_weight_min": -86.21170806884766,
      "activations/layer5_attention_weight_max": 61.4029541015625,
      "activations/layer5_attention_weight_min": -74.75344848632812,
      "activations/layer6_attention_weight_max": 47.92308044433594,
      "activations/layer6_attention_weight_min": -51.04405975341797,
      "activations/layer7_attention_weight_max": 73.7022476196289,
      "activations/layer7_attention_weight_min": -71.82938385009766,
      "activations/layer8_attention_weight_max": 47.942684173583984,
      "activations/layer8_attention_weight_min": -53.40347671508789,
      "activations/layer9_attention_weight_max": 43.6685905456543,
      "activations/layer9_attention_weight_min": -49.887672424316406,
      "epoch": 22.57,
      "learning_rate": 4.492045454545454e-06,
      "loss": 2.7097,
      "step": 388450
    },
    {
      "activations/layer0_attention_weight_max": 16.18834114074707,
      "activations/layer0_attention_weight_min": -13.996241569519043,
      "activations/layer10_attention_weight_max": 36.42401123046875,
      "activations/layer10_attention_weight_min": -36.62238311767578,
      "activations/layer11_attention_weight_max": 33.380271911621094,
      "activations/layer11_attention_weight_min": -35.01117706298828,
      "activations/layer12_attention_weight_max": 19.49311637878418,
      "activations/layer12_attention_weight_min": -27.1591739654541,
      "activations/layer13_attention_weight_max": 33.369773864746094,
      "activations/layer13_attention_weight_min": -26.570829391479492,
      "activations/layer14_attention_weight_max": 34.6806755065918,
      "activations/layer14_attention_weight_min": -31.180204391479492,
      "activations/layer15_attention_weight_max": 32.29136657714844,
      "activations/layer15_attention_weight_min": -30.9974308013916,
      "activations/layer16_attention_weight_max": 33.42393493652344,
      "activations/layer16_attention_weight_min": -31.790300369262695,
      "activations/layer17_attention_weight_max": 49.05403518676758,
      "activations/layer17_attention_weight_min": -43.77803421020508,
      "activations/layer18_attention_weight_max": 45.61415100097656,
      "activations/layer18_attention_weight_min": -39.38120651245117,
      "activations/layer19_attention_weight_max": 25.312976837158203,
      "activations/layer19_attention_weight_min": -23.817411422729492,
      "activations/layer1_attention_weight_max": 14.986992835998535,
      "activations/layer1_attention_weight_min": -14.333634376525879,
      "activations/layer20_attention_weight_max": 22.025175094604492,
      "activations/layer20_attention_weight_min": -21.052249908447266,
      "activations/layer21_attention_weight_max": 33.2083854675293,
      "activations/layer21_attention_weight_min": -22.228652954101562,
      "activations/layer22_attention_weight_max": 29.67142105102539,
      "activations/layer22_attention_weight_min": -25.097206115722656,
      "activations/layer23_attention_weight_max": 37.497283935546875,
      "activations/layer23_attention_weight_min": -24.31061553955078,
      "activations/layer2_attention_weight_max": 31.715497970581055,
      "activations/layer2_attention_weight_min": -30.01842498779297,
      "activations/layer3_attention_weight_max": 92.68021392822266,
      "activations/layer3_attention_weight_min": -90.83916473388672,
      "activations/layer4_attention_weight_max": 87.67794036865234,
      "activations/layer4_attention_weight_min": -85.46910858154297,
      "activations/layer5_attention_weight_max": 61.868377685546875,
      "activations/layer5_attention_weight_min": -73.85122680664062,
      "activations/layer6_attention_weight_max": 50.07027816772461,
      "activations/layer6_attention_weight_min": -51.70490646362305,
      "activations/layer7_attention_weight_max": 72.41169738769531,
      "activations/layer7_attention_weight_min": -69.53156280517578,
      "activations/layer8_attention_weight_max": 48.659217834472656,
      "activations/layer8_attention_weight_min": -52.109458923339844,
      "activations/layer9_attention_weight_max": 45.745174407958984,
      "activations/layer9_attention_weight_min": -48.85298156738281,
      "epoch": 22.57,
      "learning_rate": 4.4731060606060605e-06,
      "loss": 2.7124,
      "step": 388500
    },
    {
      "activations/layer0_attention_weight_max": 15.802556037902832,
      "activations/layer0_attention_weight_min": -14.084758758544922,
      "activations/layer10_attention_weight_max": 34.86674499511719,
      "activations/layer10_attention_weight_min": -37.3405647277832,
      "activations/layer11_attention_weight_max": 34.331932067871094,
      "activations/layer11_attention_weight_min": -35.634117126464844,
      "activations/layer12_attention_weight_max": 18.956893920898438,
      "activations/layer12_attention_weight_min": -25.018335342407227,
      "activations/layer13_attention_weight_max": 32.24492645263672,
      "activations/layer13_attention_weight_min": -27.86568260192871,
      "activations/layer14_attention_weight_max": 34.88963317871094,
      "activations/layer14_attention_weight_min": -32.13157653808594,
      "activations/layer15_attention_weight_max": 33.583778381347656,
      "activations/layer15_attention_weight_min": -32.90285873413086,
      "activations/layer16_attention_weight_max": 34.04928970336914,
      "activations/layer16_attention_weight_min": -33.62030029296875,
      "activations/layer17_attention_weight_max": 51.536197662353516,
      "activations/layer17_attention_weight_min": -46.02655029296875,
      "activations/layer18_attention_weight_max": 48.04033660888672,
      "activations/layer18_attention_weight_min": -40.621612548828125,
      "activations/layer19_attention_weight_max": 24.331418991088867,
      "activations/layer19_attention_weight_min": -22.242616653442383,
      "activations/layer1_attention_weight_max": 14.799163818359375,
      "activations/layer1_attention_weight_min": -14.721769332885742,
      "activations/layer20_attention_weight_max": 22.57716941833496,
      "activations/layer20_attention_weight_min": -20.923246383666992,
      "activations/layer21_attention_weight_max": 35.88432693481445,
      "activations/layer21_attention_weight_min": -25.460033416748047,
      "activations/layer22_attention_weight_max": 28.613279342651367,
      "activations/layer22_attention_weight_min": -25.314165115356445,
      "activations/layer23_attention_weight_max": 35.72776794433594,
      "activations/layer23_attention_weight_min": -24.042760848999023,
      "activations/layer2_attention_weight_max": 31.826087951660156,
      "activations/layer2_attention_weight_min": -30.847309112548828,
      "activations/layer3_attention_weight_max": 89.62541198730469,
      "activations/layer3_attention_weight_min": -90.39862823486328,
      "activations/layer4_attention_weight_max": 86.04339599609375,
      "activations/layer4_attention_weight_min": -88.03470611572266,
      "activations/layer5_attention_weight_max": 60.882293701171875,
      "activations/layer5_attention_weight_min": -71.78218078613281,
      "activations/layer6_attention_weight_max": 49.76016616821289,
      "activations/layer6_attention_weight_min": -51.16930389404297,
      "activations/layer7_attention_weight_max": 66.84891510009766,
      "activations/layer7_attention_weight_min": -69.20330810546875,
      "activations/layer8_attention_weight_max": 45.07646942138672,
      "activations/layer8_attention_weight_min": -49.139808654785156,
      "activations/layer9_attention_weight_max": 47.14527130126953,
      "activations/layer9_attention_weight_min": -45.31673812866211,
      "epoch": 22.58,
      "learning_rate": 4.454166666666666e-06,
      "loss": 2.7055,
      "step": 388550
    },
    {
      "activations/layer0_attention_weight_max": 15.792458534240723,
      "activations/layer0_attention_weight_min": -14.366949081420898,
      "activations/layer10_attention_weight_max": 35.696441650390625,
      "activations/layer10_attention_weight_min": -35.890193939208984,
      "activations/layer11_attention_weight_max": 36.65227127075195,
      "activations/layer11_attention_weight_min": -35.89297866821289,
      "activations/layer12_attention_weight_max": 18.109745025634766,
      "activations/layer12_attention_weight_min": -26.044736862182617,
      "activations/layer13_attention_weight_max": 29.99168586730957,
      "activations/layer13_attention_weight_min": -30.36972999572754,
      "activations/layer14_attention_weight_max": 34.09208297729492,
      "activations/layer14_attention_weight_min": -31.504844665527344,
      "activations/layer15_attention_weight_max": 32.486183166503906,
      "activations/layer15_attention_weight_min": -32.456912994384766,
      "activations/layer16_attention_weight_max": 33.58654022216797,
      "activations/layer16_attention_weight_min": -33.56859588623047,
      "activations/layer17_attention_weight_max": 47.41963195800781,
      "activations/layer17_attention_weight_min": -47.63715744018555,
      "activations/layer18_attention_weight_max": 44.263038635253906,
      "activations/layer18_attention_weight_min": -43.23007583618164,
      "activations/layer19_attention_weight_max": 24.729839324951172,
      "activations/layer19_attention_weight_min": -24.135414123535156,
      "activations/layer1_attention_weight_max": 14.927579879760742,
      "activations/layer1_attention_weight_min": -15.789019584655762,
      "activations/layer20_attention_weight_max": 22.3501033782959,
      "activations/layer20_attention_weight_min": -22.51251220703125,
      "activations/layer21_attention_weight_max": 30.925968170166016,
      "activations/layer21_attention_weight_min": -23.373973846435547,
      "activations/layer22_attention_weight_max": 29.213621139526367,
      "activations/layer22_attention_weight_min": -23.924835205078125,
      "activations/layer23_attention_weight_max": 34.25037384033203,
      "activations/layer23_attention_weight_min": -25.42552947998047,
      "activations/layer2_attention_weight_max": 33.060081481933594,
      "activations/layer2_attention_weight_min": -30.417659759521484,
      "activations/layer3_attention_weight_max": 94.08814239501953,
      "activations/layer3_attention_weight_min": -89.3190689086914,
      "activations/layer4_attention_weight_max": 89.0544204711914,
      "activations/layer4_attention_weight_min": -86.32544708251953,
      "activations/layer5_attention_weight_max": 66.24710083007812,
      "activations/layer5_attention_weight_min": -70.65888977050781,
      "activations/layer6_attention_weight_max": 50.46935272216797,
      "activations/layer6_attention_weight_min": -53.438499450683594,
      "activations/layer7_attention_weight_max": 69.41259765625,
      "activations/layer7_attention_weight_min": -72.80004119873047,
      "activations/layer8_attention_weight_max": 46.48174285888672,
      "activations/layer8_attention_weight_min": -50.467220306396484,
      "activations/layer9_attention_weight_max": 44.58518600463867,
      "activations/layer9_attention_weight_min": -47.386505126953125,
      "epoch": 22.58,
      "learning_rate": 4.435227272727273e-06,
      "loss": 2.6962,
      "step": 388600
    },
    {
      "activations/layer0_attention_weight_max": 16.79039192199707,
      "activations/layer0_attention_weight_min": -14.16980266571045,
      "activations/layer10_attention_weight_max": 37.84434127807617,
      "activations/layer10_attention_weight_min": -38.61249542236328,
      "activations/layer11_attention_weight_max": 37.757015228271484,
      "activations/layer11_attention_weight_min": -36.08414077758789,
      "activations/layer12_attention_weight_max": 19.409379959106445,
      "activations/layer12_attention_weight_min": -24.151687622070312,
      "activations/layer13_attention_weight_max": 36.78196716308594,
      "activations/layer13_attention_weight_min": -26.674278259277344,
      "activations/layer14_attention_weight_max": 37.53211975097656,
      "activations/layer14_attention_weight_min": -31.223421096801758,
      "activations/layer15_attention_weight_max": 35.48987579345703,
      "activations/layer15_attention_weight_min": -29.917112350463867,
      "activations/layer16_attention_weight_max": 34.693965911865234,
      "activations/layer16_attention_weight_min": -32.97209167480469,
      "activations/layer17_attention_weight_max": 51.97058868408203,
      "activations/layer17_attention_weight_min": -47.9061393737793,
      "activations/layer18_attention_weight_max": 51.4744987487793,
      "activations/layer18_attention_weight_min": -45.60301208496094,
      "activations/layer19_attention_weight_max": 30.700992584228516,
      "activations/layer19_attention_weight_min": -25.773855209350586,
      "activations/layer1_attention_weight_max": 15.180180549621582,
      "activations/layer1_attention_weight_min": -15.809680938720703,
      "activations/layer20_attention_weight_max": 27.697269439697266,
      "activations/layer20_attention_weight_min": -19.84034538269043,
      "activations/layer21_attention_weight_max": 36.89327621459961,
      "activations/layer21_attention_weight_min": -20.1058349609375,
      "activations/layer22_attention_weight_max": 29.00555992126465,
      "activations/layer22_attention_weight_min": -22.56766128540039,
      "activations/layer23_attention_weight_max": 42.047786712646484,
      "activations/layer23_attention_weight_min": -24.765090942382812,
      "activations/layer2_attention_weight_max": 32.76313781738281,
      "activations/layer2_attention_weight_min": -30.285545349121094,
      "activations/layer3_attention_weight_max": 89.66146850585938,
      "activations/layer3_attention_weight_min": -86.35015106201172,
      "activations/layer4_attention_weight_max": 84.91162872314453,
      "activations/layer4_attention_weight_min": -85.25276947021484,
      "activations/layer5_attention_weight_max": 64.47727966308594,
      "activations/layer5_attention_weight_min": -71.035400390625,
      "activations/layer6_attention_weight_max": 52.35267639160156,
      "activations/layer6_attention_weight_min": -51.85926818847656,
      "activations/layer7_attention_weight_max": 69.28885650634766,
      "activations/layer7_attention_weight_min": -71.72312927246094,
      "activations/layer8_attention_weight_max": 49.241825103759766,
      "activations/layer8_attention_weight_min": -54.40829086303711,
      "activations/layer9_attention_weight_max": 48.196937561035156,
      "activations/layer9_attention_weight_min": -50.4160041809082,
      "epoch": 22.58,
      "learning_rate": 4.4162878787878785e-06,
      "loss": 2.6856,
      "step": 388650
    },
    {
      "activations/layer0_attention_weight_max": 15.21638011932373,
      "activations/layer0_attention_weight_min": -14.268842697143555,
      "activations/layer10_attention_weight_max": 34.62479019165039,
      "activations/layer10_attention_weight_min": -35.551780700683594,
      "activations/layer11_attention_weight_max": 32.78439712524414,
      "activations/layer11_attention_weight_min": -34.88779830932617,
      "activations/layer12_attention_weight_max": 20.140634536743164,
      "activations/layer12_attention_weight_min": -24.21426773071289,
      "activations/layer13_attention_weight_max": 33.8138542175293,
      "activations/layer13_attention_weight_min": -28.427927017211914,
      "activations/layer14_attention_weight_max": 33.20199203491211,
      "activations/layer14_attention_weight_min": -30.450368881225586,
      "activations/layer15_attention_weight_max": 31.232669830322266,
      "activations/layer15_attention_weight_min": -29.480234146118164,
      "activations/layer16_attention_weight_max": 33.77956771850586,
      "activations/layer16_attention_weight_min": -35.25993728637695,
      "activations/layer17_attention_weight_max": 49.27353286743164,
      "activations/layer17_attention_weight_min": -46.258155822753906,
      "activations/layer18_attention_weight_max": 43.624752044677734,
      "activations/layer18_attention_weight_min": -39.64483642578125,
      "activations/layer19_attention_weight_max": 22.596548080444336,
      "activations/layer19_attention_weight_min": -24.5556697845459,
      "activations/layer1_attention_weight_max": 15.246940612792969,
      "activations/layer1_attention_weight_min": -14.714729309082031,
      "activations/layer20_attention_weight_max": 20.962034225463867,
      "activations/layer20_attention_weight_min": -21.413841247558594,
      "activations/layer21_attention_weight_max": 31.916751861572266,
      "activations/layer21_attention_weight_min": -21.7790470123291,
      "activations/layer22_attention_weight_max": 28.78827667236328,
      "activations/layer22_attention_weight_min": -25.04777717590332,
      "activations/layer23_attention_weight_max": 36.22865676879883,
      "activations/layer23_attention_weight_min": -24.593542098999023,
      "activations/layer2_attention_weight_max": 31.87653923034668,
      "activations/layer2_attention_weight_min": -30.792476654052734,
      "activations/layer3_attention_weight_max": 90.23436737060547,
      "activations/layer3_attention_weight_min": -88.38810729980469,
      "activations/layer4_attention_weight_max": 81.74300384521484,
      "activations/layer4_attention_weight_min": -81.73207092285156,
      "activations/layer5_attention_weight_max": 63.42931365966797,
      "activations/layer5_attention_weight_min": -70.97920227050781,
      "activations/layer6_attention_weight_max": 49.31108093261719,
      "activations/layer6_attention_weight_min": -48.87543487548828,
      "activations/layer7_attention_weight_max": 70.7206039428711,
      "activations/layer7_attention_weight_min": -69.10601806640625,
      "activations/layer8_attention_weight_max": 47.31486129760742,
      "activations/layer8_attention_weight_min": -47.516815185546875,
      "activations/layer9_attention_weight_max": 44.601070404052734,
      "activations/layer9_attention_weight_min": -45.48048782348633,
      "epoch": 22.59,
      "learning_rate": 4.397348484848484e-06,
      "loss": 2.7099,
      "step": 388700
    },
    {
      "activations/layer0_attention_weight_max": 15.435546875,
      "activations/layer0_attention_weight_min": -14.405248641967773,
      "activations/layer10_attention_weight_max": 38.27333068847656,
      "activations/layer10_attention_weight_min": -37.32240295410156,
      "activations/layer11_attention_weight_max": 34.069557189941406,
      "activations/layer11_attention_weight_min": -36.38729476928711,
      "activations/layer12_attention_weight_max": 21.519771575927734,
      "activations/layer12_attention_weight_min": -23.94156837463379,
      "activations/layer13_attention_weight_max": 40.697296142578125,
      "activations/layer13_attention_weight_min": -30.302602767944336,
      "activations/layer14_attention_weight_max": 36.80683517456055,
      "activations/layer14_attention_weight_min": -29.661075592041016,
      "activations/layer15_attention_weight_max": 36.54826354980469,
      "activations/layer15_attention_weight_min": -30.913185119628906,
      "activations/layer16_attention_weight_max": 37.732383728027344,
      "activations/layer16_attention_weight_min": -34.127079010009766,
      "activations/layer17_attention_weight_max": 55.24580764770508,
      "activations/layer17_attention_weight_min": -45.933528900146484,
      "activations/layer18_attention_weight_max": 52.40301513671875,
      "activations/layer18_attention_weight_min": -40.96208953857422,
      "activations/layer19_attention_weight_max": 27.145313262939453,
      "activations/layer19_attention_weight_min": -23.510412216186523,
      "activations/layer1_attention_weight_max": 14.852729797363281,
      "activations/layer1_attention_weight_min": -14.913065910339355,
      "activations/layer20_attention_weight_max": 23.374731063842773,
      "activations/layer20_attention_weight_min": -22.973848342895508,
      "activations/layer21_attention_weight_max": 38.52156448364258,
      "activations/layer21_attention_weight_min": -23.67447853088379,
      "activations/layer22_attention_weight_max": 30.524791717529297,
      "activations/layer22_attention_weight_min": -24.999998092651367,
      "activations/layer23_attention_weight_max": 40.11015319824219,
      "activations/layer23_attention_weight_min": -24.67897605895996,
      "activations/layer2_attention_weight_max": 31.16689682006836,
      "activations/layer2_attention_weight_min": -31.52635955810547,
      "activations/layer3_attention_weight_max": 89.84794616699219,
      "activations/layer3_attention_weight_min": -90.86380004882812,
      "activations/layer4_attention_weight_max": 87.19689178466797,
      "activations/layer4_attention_weight_min": -85.68242645263672,
      "activations/layer5_attention_weight_max": 64.6453857421875,
      "activations/layer5_attention_weight_min": -69.0351791381836,
      "activations/layer6_attention_weight_max": 51.619590759277344,
      "activations/layer6_attention_weight_min": -54.11333084106445,
      "activations/layer7_attention_weight_max": 75.76478576660156,
      "activations/layer7_attention_weight_min": -68.92141723632812,
      "activations/layer8_attention_weight_max": 49.64656448364258,
      "activations/layer8_attention_weight_min": -56.23234558105469,
      "activations/layer9_attention_weight_max": 48.43598175048828,
      "activations/layer9_attention_weight_min": -50.13071823120117,
      "epoch": 22.59,
      "learning_rate": 4.37840909090909e-06,
      "loss": 2.7107,
      "step": 388750
    },
    {
      "activations/layer0_attention_weight_max": 15.761367797851562,
      "activations/layer0_attention_weight_min": -14.198192596435547,
      "activations/layer10_attention_weight_max": 35.54585647583008,
      "activations/layer10_attention_weight_min": -35.183624267578125,
      "activations/layer11_attention_weight_max": 33.88321304321289,
      "activations/layer11_attention_weight_min": -34.03089141845703,
      "activations/layer12_attention_weight_max": 19.13515281677246,
      "activations/layer12_attention_weight_min": -23.133808135986328,
      "activations/layer13_attention_weight_max": 32.909942626953125,
      "activations/layer13_attention_weight_min": -29.464128494262695,
      "activations/layer14_attention_weight_max": 35.183448791503906,
      "activations/layer14_attention_weight_min": -31.36244010925293,
      "activations/layer15_attention_weight_max": 36.590476989746094,
      "activations/layer15_attention_weight_min": -31.060649871826172,
      "activations/layer16_attention_weight_max": 35.46703338623047,
      "activations/layer16_attention_weight_min": -34.32847595214844,
      "activations/layer17_attention_weight_max": 50.811798095703125,
      "activations/layer17_attention_weight_min": -46.99858856201172,
      "activations/layer18_attention_weight_max": 49.23128128051758,
      "activations/layer18_attention_weight_min": -42.43484878540039,
      "activations/layer19_attention_weight_max": 28.0294132232666,
      "activations/layer19_attention_weight_min": -24.525602340698242,
      "activations/layer1_attention_weight_max": 14.98813533782959,
      "activations/layer1_attention_weight_min": -15.575699806213379,
      "activations/layer20_attention_weight_max": 25.830055236816406,
      "activations/layer20_attention_weight_min": -22.182086944580078,
      "activations/layer21_attention_weight_max": 35.97199249267578,
      "activations/layer21_attention_weight_min": -24.437728881835938,
      "activations/layer22_attention_weight_max": 31.131412506103516,
      "activations/layer22_attention_weight_min": -26.01633644104004,
      "activations/layer23_attention_weight_max": 38.395263671875,
      "activations/layer23_attention_weight_min": -25.002803802490234,
      "activations/layer2_attention_weight_max": 33.156070709228516,
      "activations/layer2_attention_weight_min": -30.56978988647461,
      "activations/layer3_attention_weight_max": 94.25140380859375,
      "activations/layer3_attention_weight_min": -91.21216583251953,
      "activations/layer4_attention_weight_max": 86.47061157226562,
      "activations/layer4_attention_weight_min": -86.7253189086914,
      "activations/layer5_attention_weight_max": 64.1065673828125,
      "activations/layer5_attention_weight_min": -71.83528137207031,
      "activations/layer6_attention_weight_max": 50.502376556396484,
      "activations/layer6_attention_weight_min": -52.014530181884766,
      "activations/layer7_attention_weight_max": 70.8863296508789,
      "activations/layer7_attention_weight_min": -70.23385620117188,
      "activations/layer8_attention_weight_max": 49.04513931274414,
      "activations/layer8_attention_weight_min": -54.14070510864258,
      "activations/layer9_attention_weight_max": 43.5604133605957,
      "activations/layer9_attention_weight_min": -45.768577575683594,
      "epoch": 22.59,
      "learning_rate": 4.3594696969696966e-06,
      "loss": 2.7027,
      "step": 388800
    },
    {
      "activations/layer0_attention_weight_max": 15.836926460266113,
      "activations/layer0_attention_weight_min": -14.124876022338867,
      "activations/layer10_attention_weight_max": 38.658939361572266,
      "activations/layer10_attention_weight_min": -36.5475959777832,
      "activations/layer11_attention_weight_max": 35.123836517333984,
      "activations/layer11_attention_weight_min": -37.178802490234375,
      "activations/layer12_attention_weight_max": 19.639917373657227,
      "activations/layer12_attention_weight_min": -25.61406135559082,
      "activations/layer13_attention_weight_max": 38.511924743652344,
      "activations/layer13_attention_weight_min": -31.404115676879883,
      "activations/layer14_attention_weight_max": 36.551055908203125,
      "activations/layer14_attention_weight_min": -30.934207916259766,
      "activations/layer15_attention_weight_max": 34.890480041503906,
      "activations/layer15_attention_weight_min": -30.94467544555664,
      "activations/layer16_attention_weight_max": 35.89664077758789,
      "activations/layer16_attention_weight_min": -32.382843017578125,
      "activations/layer17_attention_weight_max": 56.351810455322266,
      "activations/layer17_attention_weight_min": -46.861366271972656,
      "activations/layer18_attention_weight_max": 51.0782356262207,
      "activations/layer18_attention_weight_min": -41.3575325012207,
      "activations/layer19_attention_weight_max": 25.762653350830078,
      "activations/layer19_attention_weight_min": -22.948156356811523,
      "activations/layer1_attention_weight_max": 15.10446548461914,
      "activations/layer1_attention_weight_min": -14.043582916259766,
      "activations/layer20_attention_weight_max": 25.915664672851562,
      "activations/layer20_attention_weight_min": -23.04493522644043,
      "activations/layer21_attention_weight_max": 38.93144607543945,
      "activations/layer21_attention_weight_min": -21.364887237548828,
      "activations/layer22_attention_weight_max": 30.508808135986328,
      "activations/layer22_attention_weight_min": -26.194835662841797,
      "activations/layer23_attention_weight_max": 36.397674560546875,
      "activations/layer23_attention_weight_min": -24.9293212890625,
      "activations/layer2_attention_weight_max": 32.91565704345703,
      "activations/layer2_attention_weight_min": -30.53790283203125,
      "activations/layer3_attention_weight_max": 92.1352310180664,
      "activations/layer3_attention_weight_min": -90.60491943359375,
      "activations/layer4_attention_weight_max": 90.44100189208984,
      "activations/layer4_attention_weight_min": -85.37815856933594,
      "activations/layer5_attention_weight_max": 59.56394958496094,
      "activations/layer5_attention_weight_min": -71.64898681640625,
      "activations/layer6_attention_weight_max": 51.305660247802734,
      "activations/layer6_attention_weight_min": -52.46780014038086,
      "activations/layer7_attention_weight_max": 68.08094024658203,
      "activations/layer7_attention_weight_min": -71.31009674072266,
      "activations/layer8_attention_weight_max": 47.079708099365234,
      "activations/layer8_attention_weight_min": -52.711299896240234,
      "activations/layer9_attention_weight_max": 44.982391357421875,
      "activations/layer9_attention_weight_min": -48.83475112915039,
      "epoch": 22.59,
      "learning_rate": 4.340530303030303e-06,
      "loss": 2.6968,
      "step": 388850
    },
    {
      "activations/layer0_attention_weight_max": 16.194690704345703,
      "activations/layer0_attention_weight_min": -14.184789657592773,
      "activations/layer10_attention_weight_max": 38.729007720947266,
      "activations/layer10_attention_weight_min": -40.4673957824707,
      "activations/layer11_attention_weight_max": 36.46747589111328,
      "activations/layer11_attention_weight_min": -37.72833251953125,
      "activations/layer12_attention_weight_max": 19.959699630737305,
      "activations/layer12_attention_weight_min": -27.342931747436523,
      "activations/layer13_attention_weight_max": 36.8039665222168,
      "activations/layer13_attention_weight_min": -30.443119049072266,
      "activations/layer14_attention_weight_max": 36.01881790161133,
      "activations/layer14_attention_weight_min": -31.428924560546875,
      "activations/layer15_attention_weight_max": 34.96353530883789,
      "activations/layer15_attention_weight_min": -29.94359588623047,
      "activations/layer16_attention_weight_max": 35.925418853759766,
      "activations/layer16_attention_weight_min": -33.18754577636719,
      "activations/layer17_attention_weight_max": 54.83223342895508,
      "activations/layer17_attention_weight_min": -46.795223236083984,
      "activations/layer18_attention_weight_max": 48.55569839477539,
      "activations/layer18_attention_weight_min": -41.34891128540039,
      "activations/layer19_attention_weight_max": 25.979846954345703,
      "activations/layer19_attention_weight_min": -21.871335983276367,
      "activations/layer1_attention_weight_max": 14.55767822265625,
      "activations/layer1_attention_weight_min": -15.057175636291504,
      "activations/layer20_attention_weight_max": 22.96324348449707,
      "activations/layer20_attention_weight_min": -22.35313606262207,
      "activations/layer21_attention_weight_max": 33.300384521484375,
      "activations/layer21_attention_weight_min": -21.52332305908203,
      "activations/layer22_attention_weight_max": 29.84063720703125,
      "activations/layer22_attention_weight_min": -25.45248031616211,
      "activations/layer23_attention_weight_max": 37.46794891357422,
      "activations/layer23_attention_weight_min": -24.210289001464844,
      "activations/layer2_attention_weight_max": 34.9991569519043,
      "activations/layer2_attention_weight_min": -32.044952392578125,
      "activations/layer3_attention_weight_max": 95.75889587402344,
      "activations/layer3_attention_weight_min": -95.77071380615234,
      "activations/layer4_attention_weight_max": 87.7854995727539,
      "activations/layer4_attention_weight_min": -92.18782806396484,
      "activations/layer5_attention_weight_max": 64.77865600585938,
      "activations/layer5_attention_weight_min": -70.66962432861328,
      "activations/layer6_attention_weight_max": 53.53522872924805,
      "activations/layer6_attention_weight_min": -55.68954849243164,
      "activations/layer7_attention_weight_max": 71.91199493408203,
      "activations/layer7_attention_weight_min": -75.49394989013672,
      "activations/layer8_attention_weight_max": 51.70139694213867,
      "activations/layer8_attention_weight_min": -59.25175857543945,
      "activations/layer9_attention_weight_max": 50.10469436645508,
      "activations/layer9_attention_weight_min": -54.51929473876953,
      "epoch": 22.6,
      "learning_rate": 4.321590909090909e-06,
      "loss": 2.693,
      "step": 388900
    },
    {
      "activations/layer0_attention_weight_max": 15.780232429504395,
      "activations/layer0_attention_weight_min": -14.199270248413086,
      "activations/layer10_attention_weight_max": 38.537017822265625,
      "activations/layer10_attention_weight_min": -37.78997802734375,
      "activations/layer11_attention_weight_max": 34.693359375,
      "activations/layer11_attention_weight_min": -36.58254623413086,
      "activations/layer12_attention_weight_max": 18.57537269592285,
      "activations/layer12_attention_weight_min": -23.44487953186035,
      "activations/layer13_attention_weight_max": 32.718624114990234,
      "activations/layer13_attention_weight_min": -27.21002960205078,
      "activations/layer14_attention_weight_max": 34.33259963989258,
      "activations/layer14_attention_weight_min": -31.445388793945312,
      "activations/layer15_attention_weight_max": 32.84407424926758,
      "activations/layer15_attention_weight_min": -30.508134841918945,
      "activations/layer16_attention_weight_max": 35.44853973388672,
      "activations/layer16_attention_weight_min": -32.79743957519531,
      "activations/layer17_attention_weight_max": 51.7276611328125,
      "activations/layer17_attention_weight_min": -46.585113525390625,
      "activations/layer18_attention_weight_max": 46.936973571777344,
      "activations/layer18_attention_weight_min": -41.27008819580078,
      "activations/layer19_attention_weight_max": 24.736562728881836,
      "activations/layer19_attention_weight_min": -23.864137649536133,
      "activations/layer1_attention_weight_max": 15.441694259643555,
      "activations/layer1_attention_weight_min": -13.719568252563477,
      "activations/layer20_attention_weight_max": 21.992122650146484,
      "activations/layer20_attention_weight_min": -22.305709838867188,
      "activations/layer21_attention_weight_max": 33.17713165283203,
      "activations/layer21_attention_weight_min": -20.55598258972168,
      "activations/layer22_attention_weight_max": 28.345550537109375,
      "activations/layer22_attention_weight_min": -26.084978103637695,
      "activations/layer23_attention_weight_max": 38.84089279174805,
      "activations/layer23_attention_weight_min": -23.951034545898438,
      "activations/layer2_attention_weight_max": 30.859830856323242,
      "activations/layer2_attention_weight_min": -29.257160186767578,
      "activations/layer3_attention_weight_max": 86.89095306396484,
      "activations/layer3_attention_weight_min": -91.00881958007812,
      "activations/layer4_attention_weight_max": 86.13285064697266,
      "activations/layer4_attention_weight_min": -88.07539367675781,
      "activations/layer5_attention_weight_max": 64.06443786621094,
      "activations/layer5_attention_weight_min": -72.7042465209961,
      "activations/layer6_attention_weight_max": 51.87223434448242,
      "activations/layer6_attention_weight_min": -52.48086929321289,
      "activations/layer7_attention_weight_max": 77.48951721191406,
      "activations/layer7_attention_weight_min": -75.16835021972656,
      "activations/layer8_attention_weight_max": 54.40248489379883,
      "activations/layer8_attention_weight_min": -54.514007568359375,
      "activations/layer9_attention_weight_max": 49.76723861694336,
      "activations/layer9_attention_weight_min": -50.27805709838867,
      "epoch": 22.6,
      "learning_rate": 4.302651515151515e-06,
      "loss": 2.6951,
      "step": 388950
    },
    {
      "activations/layer0_attention_weight_max": 16.1831111907959,
      "activations/layer0_attention_weight_min": -14.113973617553711,
      "activations/layer10_attention_weight_max": 37.45383071899414,
      "activations/layer10_attention_weight_min": -36.6702766418457,
      "activations/layer11_attention_weight_max": 35.98229217529297,
      "activations/layer11_attention_weight_min": -34.67256546020508,
      "activations/layer12_attention_weight_max": 19.177753448486328,
      "activations/layer12_attention_weight_min": -26.003477096557617,
      "activations/layer13_attention_weight_max": 31.48584747314453,
      "activations/layer13_attention_weight_min": -26.73689842224121,
      "activations/layer14_attention_weight_max": 34.225257873535156,
      "activations/layer14_attention_weight_min": -30.536867141723633,
      "activations/layer15_attention_weight_max": 32.34348678588867,
      "activations/layer15_attention_weight_min": -30.67353057861328,
      "activations/layer16_attention_weight_max": 32.45909881591797,
      "activations/layer16_attention_weight_min": -31.27901268005371,
      "activations/layer17_attention_weight_max": 51.03074645996094,
      "activations/layer17_attention_weight_min": -45.798397064208984,
      "activations/layer18_attention_weight_max": 47.5549430847168,
      "activations/layer18_attention_weight_min": -41.85444259643555,
      "activations/layer19_attention_weight_max": 23.99529457092285,
      "activations/layer19_attention_weight_min": -22.190258026123047,
      "activations/layer1_attention_weight_max": 14.881957054138184,
      "activations/layer1_attention_weight_min": -14.685690879821777,
      "activations/layer20_attention_weight_max": 22.241403579711914,
      "activations/layer20_attention_weight_min": -21.986650466918945,
      "activations/layer21_attention_weight_max": 35.57001495361328,
      "activations/layer21_attention_weight_min": -20.4798641204834,
      "activations/layer22_attention_weight_max": 31.37894058227539,
      "activations/layer22_attention_weight_min": -25.93294906616211,
      "activations/layer23_attention_weight_max": 35.062583923339844,
      "activations/layer23_attention_weight_min": -24.706024169921875,
      "activations/layer2_attention_weight_max": 32.21915054321289,
      "activations/layer2_attention_weight_min": -30.860111236572266,
      "activations/layer3_attention_weight_max": 92.76161193847656,
      "activations/layer3_attention_weight_min": -93.1272201538086,
      "activations/layer4_attention_weight_max": 86.5123291015625,
      "activations/layer4_attention_weight_min": -87.56423950195312,
      "activations/layer5_attention_weight_max": 63.098785400390625,
      "activations/layer5_attention_weight_min": -73.18820190429688,
      "activations/layer6_attention_weight_max": 53.56766891479492,
      "activations/layer6_attention_weight_min": -54.08578109741211,
      "activations/layer7_attention_weight_max": 71.02417755126953,
      "activations/layer7_attention_weight_min": -71.74085235595703,
      "activations/layer8_attention_weight_max": 50.14874267578125,
      "activations/layer8_attention_weight_min": -53.379669189453125,
      "activations/layer9_attention_weight_max": 45.77988815307617,
      "activations/layer9_attention_weight_min": -48.85787582397461,
      "epoch": 22.6,
      "learning_rate": 4.284090909090909e-06,
      "loss": 2.705,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.5177,
      "eval_samples_per_second": 504.127,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.5177,
      "eval_openwebtext_samples_per_second": 504.127,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9876,
      "eval_wikitext_samples_per_second": 229.423,
      "step": 389000
    },
    {
      "epoch": 22.6,
      "eval_lambada_loss": 2.384765625,
      "eval_lambada_ppl": 10.85651787033819,
      "eval_lambada_runtime": 9.5815,
      "eval_lambada_samples_per_second": 508.166,
      "step": 389000
    },
    {
      "activations/layer0_attention_weight_max": 15.0392484664917,
      "activations/layer0_attention_weight_min": -14.409675598144531,
      "activations/layer10_attention_weight_max": 33.50812911987305,
      "activations/layer10_attention_weight_min": -35.26878356933594,
      "activations/layer11_attention_weight_max": 31.654541015625,
      "activations/layer11_attention_weight_min": -35.423828125,
      "activations/layer12_attention_weight_max": 19.191102981567383,
      "activations/layer12_attention_weight_min": -24.73578453063965,
      "activations/layer13_attention_weight_max": 33.59911346435547,
      "activations/layer13_attention_weight_min": -28.38665008544922,
      "activations/layer14_attention_weight_max": 36.149200439453125,
      "activations/layer14_attention_weight_min": -30.007137298583984,
      "activations/layer15_attention_weight_max": 34.379459381103516,
      "activations/layer15_attention_weight_min": -29.8846435546875,
      "activations/layer16_attention_weight_max": 37.42790222167969,
      "activations/layer16_attention_weight_min": -36.29996109008789,
      "activations/layer17_attention_weight_max": 53.261993408203125,
      "activations/layer17_attention_weight_min": -48.92644500732422,
      "activations/layer18_attention_weight_max": 50.92478942871094,
      "activations/layer18_attention_weight_min": -41.89712905883789,
      "activations/layer19_attention_weight_max": 25.1036376953125,
      "activations/layer19_attention_weight_min": -22.47564125061035,
      "activations/layer1_attention_weight_max": 15.206308364868164,
      "activations/layer1_attention_weight_min": -14.212492942810059,
      "activations/layer20_attention_weight_max": 24.235210418701172,
      "activations/layer20_attention_weight_min": -21.08441162109375,
      "activations/layer21_attention_weight_max": 45.64032745361328,
      "activations/layer21_attention_weight_min": -24.704710006713867,
      "activations/layer22_attention_weight_max": 30.871532440185547,
      "activations/layer22_attention_weight_min": -26.864521026611328,
      "activations/layer23_attention_weight_max": 39.67026138305664,
      "activations/layer23_attention_weight_min": -25.894515991210938,
      "activations/layer2_attention_weight_max": 31.976856231689453,
      "activations/layer2_attention_weight_min": -29.273603439331055,
      "activations/layer3_attention_weight_max": 88.836669921875,
      "activations/layer3_attention_weight_min": -88.43647003173828,
      "activations/layer4_attention_weight_max": 84.1732177734375,
      "activations/layer4_attention_weight_min": -84.59138488769531,
      "activations/layer5_attention_weight_max": 64.56666564941406,
      "activations/layer5_attention_weight_min": -72.4880599975586,
      "activations/layer6_attention_weight_max": 50.32189178466797,
      "activations/layer6_attention_weight_min": -51.64006423950195,
      "activations/layer7_attention_weight_max": 71.12645721435547,
      "activations/layer7_attention_weight_min": -68.87969207763672,
      "activations/layer8_attention_weight_max": 48.401004791259766,
      "activations/layer8_attention_weight_min": -48.18727111816406,
      "activations/layer9_attention_weight_max": 43.98994064331055,
      "activations/layer9_attention_weight_min": -45.48537063598633,
      "epoch": 22.61,
      "learning_rate": 4.2651515151515145e-06,
      "loss": 2.703,
      "step": 389050
    },
    {
      "activations/layer0_attention_weight_max": 16.419992446899414,
      "activations/layer0_attention_weight_min": -13.95059585571289,
      "activations/layer10_attention_weight_max": 38.14495849609375,
      "activations/layer10_attention_weight_min": -37.93803787231445,
      "activations/layer11_attention_weight_max": 37.21818161010742,
      "activations/layer11_attention_weight_min": -38.4874153137207,
      "activations/layer12_attention_weight_max": 19.238143920898438,
      "activations/layer12_attention_weight_min": -26.453474044799805,
      "activations/layer13_attention_weight_max": 36.9605598449707,
      "activations/layer13_attention_weight_min": -27.068647384643555,
      "activations/layer14_attention_weight_max": 35.576637268066406,
      "activations/layer14_attention_weight_min": -31.96359634399414,
      "activations/layer15_attention_weight_max": 35.641475677490234,
      "activations/layer15_attention_weight_min": -31.165075302124023,
      "activations/layer16_attention_weight_max": 34.20188903808594,
      "activations/layer16_attention_weight_min": -33.67144775390625,
      "activations/layer17_attention_weight_max": 51.97941970825195,
      "activations/layer17_attention_weight_min": -43.94160842895508,
      "activations/layer18_attention_weight_max": 48.7717399597168,
      "activations/layer18_attention_weight_min": -38.24467086791992,
      "activations/layer19_attention_weight_max": 24.506650924682617,
      "activations/layer19_attention_weight_min": -21.797927856445312,
      "activations/layer1_attention_weight_max": 14.480974197387695,
      "activations/layer1_attention_weight_min": -14.190319061279297,
      "activations/layer20_attention_weight_max": 22.117008209228516,
      "activations/layer20_attention_weight_min": -20.94603729248047,
      "activations/layer21_attention_weight_max": 35.89519119262695,
      "activations/layer21_attention_weight_min": -23.196775436401367,
      "activations/layer22_attention_weight_max": 29.35892677307129,
      "activations/layer22_attention_weight_min": -25.943527221679688,
      "activations/layer23_attention_weight_max": 37.18950271606445,
      "activations/layer23_attention_weight_min": -25.937793731689453,
      "activations/layer2_attention_weight_max": 32.14068603515625,
      "activations/layer2_attention_weight_min": -31.774253845214844,
      "activations/layer3_attention_weight_max": 92.70259857177734,
      "activations/layer3_attention_weight_min": -93.58719635009766,
      "activations/layer4_attention_weight_max": 88.49517059326172,
      "activations/layer4_attention_weight_min": -89.13357543945312,
      "activations/layer5_attention_weight_max": 64.99435424804688,
      "activations/layer5_attention_weight_min": -73.97840118408203,
      "activations/layer6_attention_weight_max": 52.4164924621582,
      "activations/layer6_attention_weight_min": -54.071754455566406,
      "activations/layer7_attention_weight_max": 73.63528442382812,
      "activations/layer7_attention_weight_min": -70.2185287475586,
      "activations/layer8_attention_weight_max": 52.582340240478516,
      "activations/layer8_attention_weight_min": -56.123558044433594,
      "activations/layer9_attention_weight_max": 47.5052604675293,
      "activations/layer9_attention_weight_min": -50.940670013427734,
      "epoch": 22.61,
      "learning_rate": 4.24621212121212e-06,
      "loss": 2.7062,
      "step": 389100
    },
    {
      "activations/layer0_attention_weight_max": 16.697738647460938,
      "activations/layer0_attention_weight_min": -14.0286865234375,
      "activations/layer10_attention_weight_max": 35.2197265625,
      "activations/layer10_attention_weight_min": -38.187320709228516,
      "activations/layer11_attention_weight_max": 33.396148681640625,
      "activations/layer11_attention_weight_min": -36.586158752441406,
      "activations/layer12_attention_weight_max": 21.168893814086914,
      "activations/layer12_attention_weight_min": -24.075971603393555,
      "activations/layer13_attention_weight_max": 34.83763885498047,
      "activations/layer13_attention_weight_min": -28.627540588378906,
      "activations/layer14_attention_weight_max": 37.499542236328125,
      "activations/layer14_attention_weight_min": -34.13212585449219,
      "activations/layer15_attention_weight_max": 34.666709899902344,
      "activations/layer15_attention_weight_min": -33.17876434326172,
      "activations/layer16_attention_weight_max": 35.26610565185547,
      "activations/layer16_attention_weight_min": -35.19473648071289,
      "activations/layer17_attention_weight_max": 52.968013763427734,
      "activations/layer17_attention_weight_min": -47.854373931884766,
      "activations/layer18_attention_weight_max": 46.96211242675781,
      "activations/layer18_attention_weight_min": -43.164920806884766,
      "activations/layer19_attention_weight_max": 25.266231536865234,
      "activations/layer19_attention_weight_min": -25.086746215820312,
      "activations/layer1_attention_weight_max": 16.083141326904297,
      "activations/layer1_attention_weight_min": -17.19350242614746,
      "activations/layer20_attention_weight_max": 23.543210983276367,
      "activations/layer20_attention_weight_min": -23.218341827392578,
      "activations/layer21_attention_weight_max": 35.420875549316406,
      "activations/layer21_attention_weight_min": -22.004819869995117,
      "activations/layer22_attention_weight_max": 29.940128326416016,
      "activations/layer22_attention_weight_min": -26.91426658630371,
      "activations/layer23_attention_weight_max": 37.518890380859375,
      "activations/layer23_attention_weight_min": -24.887487411499023,
      "activations/layer2_attention_weight_max": 31.631942749023438,
      "activations/layer2_attention_weight_min": -30.86385154724121,
      "activations/layer3_attention_weight_max": 89.44270324707031,
      "activations/layer3_attention_weight_min": -89.59465789794922,
      "activations/layer4_attention_weight_max": 82.47057342529297,
      "activations/layer4_attention_weight_min": -86.33035278320312,
      "activations/layer5_attention_weight_max": 61.57068634033203,
      "activations/layer5_attention_weight_min": -73.97856903076172,
      "activations/layer6_attention_weight_max": 48.50873565673828,
      "activations/layer6_attention_weight_min": -50.46743392944336,
      "activations/layer7_attention_weight_max": 67.63492584228516,
      "activations/layer7_attention_weight_min": -68.61111450195312,
      "activations/layer8_attention_weight_max": 46.5241584777832,
      "activations/layer8_attention_weight_min": -49.711421966552734,
      "activations/layer9_attention_weight_max": 42.84748458862305,
      "activations/layer9_attention_weight_min": -47.25118637084961,
      "epoch": 22.61,
      "learning_rate": 4.227272727272727e-06,
      "loss": 2.6974,
      "step": 389150
    },
    {
      "activations/layer0_attention_weight_max": 16.513896942138672,
      "activations/layer0_attention_weight_min": -14.185644149780273,
      "activations/layer10_attention_weight_max": 38.07175064086914,
      "activations/layer10_attention_weight_min": -36.784278869628906,
      "activations/layer11_attention_weight_max": 33.50517654418945,
      "activations/layer11_attention_weight_min": -35.63251495361328,
      "activations/layer12_attention_weight_max": 19.628345489501953,
      "activations/layer12_attention_weight_min": -22.82462501525879,
      "activations/layer13_attention_weight_max": 36.83208084106445,
      "activations/layer13_attention_weight_min": -27.071290969848633,
      "activations/layer14_attention_weight_max": 37.1850700378418,
      "activations/layer14_attention_weight_min": -30.661849975585938,
      "activations/layer15_attention_weight_max": 32.96614074707031,
      "activations/layer15_attention_weight_min": -32.08413314819336,
      "activations/layer16_attention_weight_max": 36.44559860229492,
      "activations/layer16_attention_weight_min": -33.949737548828125,
      "activations/layer17_attention_weight_max": 54.861183166503906,
      "activations/layer17_attention_weight_min": -47.01837921142578,
      "activations/layer18_attention_weight_max": 48.148189544677734,
      "activations/layer18_attention_weight_min": -42.09778594970703,
      "activations/layer19_attention_weight_max": 24.924972534179688,
      "activations/layer19_attention_weight_min": -24.894067764282227,
      "activations/layer1_attention_weight_max": 15.382407188415527,
      "activations/layer1_attention_weight_min": -16.461265563964844,
      "activations/layer20_attention_weight_max": 24.816320419311523,
      "activations/layer20_attention_weight_min": -21.862470626831055,
      "activations/layer21_attention_weight_max": 39.44565200805664,
      "activations/layer21_attention_weight_min": -23.46196174621582,
      "activations/layer22_attention_weight_max": 29.19088363647461,
      "activations/layer22_attention_weight_min": -24.745773315429688,
      "activations/layer23_attention_weight_max": 35.35996627807617,
      "activations/layer23_attention_weight_min": -24.316051483154297,
      "activations/layer2_attention_weight_max": 31.238462448120117,
      "activations/layer2_attention_weight_min": -30.764739990234375,
      "activations/layer3_attention_weight_max": 89.55845642089844,
      "activations/layer3_attention_weight_min": -90.07643127441406,
      "activations/layer4_attention_weight_max": 86.78507995605469,
      "activations/layer4_attention_weight_min": -86.97917938232422,
      "activations/layer5_attention_weight_max": 64.87773895263672,
      "activations/layer5_attention_weight_min": -72.68608093261719,
      "activations/layer6_attention_weight_max": 49.04582595825195,
      "activations/layer6_attention_weight_min": -51.49941635131836,
      "activations/layer7_attention_weight_max": 65.99761199951172,
      "activations/layer7_attention_weight_min": -68.19133758544922,
      "activations/layer8_attention_weight_max": 48.08430862426758,
      "activations/layer8_attention_weight_min": -49.29008865356445,
      "activations/layer9_attention_weight_max": 45.47800827026367,
      "activations/layer9_attention_weight_min": -46.47003936767578,
      "epoch": 22.61,
      "learning_rate": 4.208333333333333e-06,
      "loss": 2.7061,
      "step": 389200
    },
    {
      "activations/layer0_attention_weight_max": 16.04302215576172,
      "activations/layer0_attention_weight_min": -14.083664894104004,
      "activations/layer10_attention_weight_max": 39.580352783203125,
      "activations/layer10_attention_weight_min": -39.247039794921875,
      "activations/layer11_attention_weight_max": 33.907493591308594,
      "activations/layer11_attention_weight_min": -35.011112213134766,
      "activations/layer12_attention_weight_max": 18.6418514251709,
      "activations/layer12_attention_weight_min": -22.50213050842285,
      "activations/layer13_attention_weight_max": 32.33872604370117,
      "activations/layer13_attention_weight_min": -26.963863372802734,
      "activations/layer14_attention_weight_max": 37.0609016418457,
      "activations/layer14_attention_weight_min": -31.260982513427734,
      "activations/layer15_attention_weight_max": 32.87074279785156,
      "activations/layer15_attention_weight_min": -31.57280731201172,
      "activations/layer16_attention_weight_max": 34.04603576660156,
      "activations/layer16_attention_weight_min": -33.384971618652344,
      "activations/layer17_attention_weight_max": 50.03548812866211,
      "activations/layer17_attention_weight_min": -47.23808670043945,
      "activations/layer18_attention_weight_max": 49.586578369140625,
      "activations/layer18_attention_weight_min": -46.13440704345703,
      "activations/layer19_attention_weight_max": 24.594404220581055,
      "activations/layer19_attention_weight_min": -25.727075576782227,
      "activations/layer1_attention_weight_max": 15.100099563598633,
      "activations/layer1_attention_weight_min": -14.280633926391602,
      "activations/layer20_attention_weight_max": 21.12677574157715,
      "activations/layer20_attention_weight_min": -21.62376594543457,
      "activations/layer21_attention_weight_max": 33.248390197753906,
      "activations/layer21_attention_weight_min": -24.2529239654541,
      "activations/layer22_attention_weight_max": 27.318702697753906,
      "activations/layer22_attention_weight_min": -26.919391632080078,
      "activations/layer23_attention_weight_max": 32.41496276855469,
      "activations/layer23_attention_weight_min": -23.996536254882812,
      "activations/layer2_attention_weight_max": 32.08967208862305,
      "activations/layer2_attention_weight_min": -30.2114315032959,
      "activations/layer3_attention_weight_max": 87.48228454589844,
      "activations/layer3_attention_weight_min": -88.99114990234375,
      "activations/layer4_attention_weight_max": 84.39916229248047,
      "activations/layer4_attention_weight_min": -86.4535140991211,
      "activations/layer5_attention_weight_max": 63.3197021484375,
      "activations/layer5_attention_weight_min": -77.75906372070312,
      "activations/layer6_attention_weight_max": 51.0750617980957,
      "activations/layer6_attention_weight_min": -53.095279693603516,
      "activations/layer7_attention_weight_max": 68.99889373779297,
      "activations/layer7_attention_weight_min": -72.71154022216797,
      "activations/layer8_attention_weight_max": 53.71288299560547,
      "activations/layer8_attention_weight_min": -58.81106185913086,
      "activations/layer9_attention_weight_max": 48.25962448120117,
      "activations/layer9_attention_weight_min": -53.09633255004883,
      "epoch": 22.62,
      "learning_rate": 4.189393939393939e-06,
      "loss": 2.7059,
      "step": 389250
    },
    {
      "activations/layer0_attention_weight_max": 14.81434440612793,
      "activations/layer0_attention_weight_min": -14.042826652526855,
      "activations/layer10_attention_weight_max": 35.18741989135742,
      "activations/layer10_attention_weight_min": -35.59086990356445,
      "activations/layer11_attention_weight_max": 33.3817138671875,
      "activations/layer11_attention_weight_min": -34.455692291259766,
      "activations/layer12_attention_weight_max": 18.462665557861328,
      "activations/layer12_attention_weight_min": -25.654624938964844,
      "activations/layer13_attention_weight_max": 28.985193252563477,
      "activations/layer13_attention_weight_min": -25.933225631713867,
      "activations/layer14_attention_weight_max": 32.647979736328125,
      "activations/layer14_attention_weight_min": -31.24022674560547,
      "activations/layer15_attention_weight_max": 32.32558822631836,
      "activations/layer15_attention_weight_min": -30.041473388671875,
      "activations/layer16_attention_weight_max": 32.9720573425293,
      "activations/layer16_attention_weight_min": -32.29020309448242,
      "activations/layer17_attention_weight_max": 49.0902099609375,
      "activations/layer17_attention_weight_min": -44.300384521484375,
      "activations/layer18_attention_weight_max": 44.27122497558594,
      "activations/layer18_attention_weight_min": -40.72700881958008,
      "activations/layer19_attention_weight_max": 22.10957908630371,
      "activations/layer19_attention_weight_min": -22.11334800720215,
      "activations/layer1_attention_weight_max": 14.597772598266602,
      "activations/layer1_attention_weight_min": -15.121253967285156,
      "activations/layer20_attention_weight_max": 21.635704040527344,
      "activations/layer20_attention_weight_min": -21.53595733642578,
      "activations/layer21_attention_weight_max": 32.815162658691406,
      "activations/layer21_attention_weight_min": -21.19856071472168,
      "activations/layer22_attention_weight_max": 26.471677780151367,
      "activations/layer22_attention_weight_min": -25.84051513671875,
      "activations/layer23_attention_weight_max": 31.88132095336914,
      "activations/layer23_attention_weight_min": -25.843856811523438,
      "activations/layer2_attention_weight_max": 33.722496032714844,
      "activations/layer2_attention_weight_min": -30.998180389404297,
      "activations/layer3_attention_weight_max": 92.56224060058594,
      "activations/layer3_attention_weight_min": -91.62393951416016,
      "activations/layer4_attention_weight_max": 87.50957489013672,
      "activations/layer4_attention_weight_min": -88.02242279052734,
      "activations/layer5_attention_weight_max": 63.64796447753906,
      "activations/layer5_attention_weight_min": -70.44255065917969,
      "activations/layer6_attention_weight_max": 50.78306579589844,
      "activations/layer6_attention_weight_min": -51.584415435791016,
      "activations/layer7_attention_weight_max": 69.82610321044922,
      "activations/layer7_attention_weight_min": -69.12418365478516,
      "activations/layer8_attention_weight_max": 47.96015930175781,
      "activations/layer8_attention_weight_min": -54.04183578491211,
      "activations/layer9_attention_weight_max": 43.45845413208008,
      "activations/layer9_attention_weight_min": -48.53483581542969,
      "epoch": 22.62,
      "learning_rate": 4.170454545454545e-06,
      "loss": 2.7056,
      "step": 389300
    },
    {
      "activations/layer0_attention_weight_max": 15.705760955810547,
      "activations/layer0_attention_weight_min": -14.135354995727539,
      "activations/layer10_attention_weight_max": 36.2802734375,
      "activations/layer10_attention_weight_min": -37.035133361816406,
      "activations/layer11_attention_weight_max": 32.812416076660156,
      "activations/layer11_attention_weight_min": -34.28038787841797,
      "activations/layer12_attention_weight_max": 19.024267196655273,
      "activations/layer12_attention_weight_min": -26.67719078063965,
      "activations/layer13_attention_weight_max": 32.100948333740234,
      "activations/layer13_attention_weight_min": -26.887113571166992,
      "activations/layer14_attention_weight_max": 40.531158447265625,
      "activations/layer14_attention_weight_min": -34.69171905517578,
      "activations/layer15_attention_weight_max": 38.3670654296875,
      "activations/layer15_attention_weight_min": -32.87433624267578,
      "activations/layer16_attention_weight_max": 38.97053146362305,
      "activations/layer16_attention_weight_min": -36.311866760253906,
      "activations/layer17_attention_weight_max": 59.44304656982422,
      "activations/layer17_attention_weight_min": -50.16071701049805,
      "activations/layer18_attention_weight_max": 52.38108825683594,
      "activations/layer18_attention_weight_min": -45.22659683227539,
      "activations/layer19_attention_weight_max": 27.17681884765625,
      "activations/layer19_attention_weight_min": -27.207971572875977,
      "activations/layer1_attention_weight_max": 15.830120086669922,
      "activations/layer1_attention_weight_min": -14.687579154968262,
      "activations/layer20_attention_weight_max": 24.888139724731445,
      "activations/layer20_attention_weight_min": -23.740867614746094,
      "activations/layer21_attention_weight_max": 41.54667282104492,
      "activations/layer21_attention_weight_min": -22.29831886291504,
      "activations/layer22_attention_weight_max": 31.288129806518555,
      "activations/layer22_attention_weight_min": -27.4877872467041,
      "activations/layer23_attention_weight_max": 34.13078689575195,
      "activations/layer23_attention_weight_min": -23.44745445251465,
      "activations/layer2_attention_weight_max": 31.865032196044922,
      "activations/layer2_attention_weight_min": -30.200592041015625,
      "activations/layer3_attention_weight_max": 90.28841400146484,
      "activations/layer3_attention_weight_min": -89.81734466552734,
      "activations/layer4_attention_weight_max": 85.8032455444336,
      "activations/layer4_attention_weight_min": -87.07713317871094,
      "activations/layer5_attention_weight_max": 64.69485473632812,
      "activations/layer5_attention_weight_min": -70.20513916015625,
      "activations/layer6_attention_weight_max": 52.500064849853516,
      "activations/layer6_attention_weight_min": -52.51540756225586,
      "activations/layer7_attention_weight_max": 70.05079650878906,
      "activations/layer7_attention_weight_min": -71.44068908691406,
      "activations/layer8_attention_weight_max": 48.57853698730469,
      "activations/layer8_attention_weight_min": -53.809017181396484,
      "activations/layer9_attention_weight_max": 48.40934753417969,
      "activations/layer9_attention_weight_min": -48.02495193481445,
      "epoch": 22.62,
      "learning_rate": 4.1515151515151514e-06,
      "loss": 2.7077,
      "step": 389350
    },
    {
      "activations/layer0_attention_weight_max": 16.518726348876953,
      "activations/layer0_attention_weight_min": -13.966329574584961,
      "activations/layer10_attention_weight_max": 36.576385498046875,
      "activations/layer10_attention_weight_min": -36.775028228759766,
      "activations/layer11_attention_weight_max": 33.057769775390625,
      "activations/layer11_attention_weight_min": -34.80577850341797,
      "activations/layer12_attention_weight_max": 22.024612426757812,
      "activations/layer12_attention_weight_min": -25.44611358642578,
      "activations/layer13_attention_weight_max": 33.61549377441406,
      "activations/layer13_attention_weight_min": -27.788795471191406,
      "activations/layer14_attention_weight_max": 34.97432327270508,
      "activations/layer14_attention_weight_min": -30.388519287109375,
      "activations/layer15_attention_weight_max": 33.565711975097656,
      "activations/layer15_attention_weight_min": -31.308202743530273,
      "activations/layer16_attention_weight_max": 34.5940055847168,
      "activations/layer16_attention_weight_min": -33.36738586425781,
      "activations/layer17_attention_weight_max": 52.92721176147461,
      "activations/layer17_attention_weight_min": -46.80244445800781,
      "activations/layer18_attention_weight_max": 50.9979362487793,
      "activations/layer18_attention_weight_min": -41.18669128417969,
      "activations/layer19_attention_weight_max": 25.373491287231445,
      "activations/layer19_attention_weight_min": -24.823986053466797,
      "activations/layer1_attention_weight_max": 15.102025985717773,
      "activations/layer1_attention_weight_min": -15.427159309387207,
      "activations/layer20_attention_weight_max": 24.427593231201172,
      "activations/layer20_attention_weight_min": -23.944080352783203,
      "activations/layer21_attention_weight_max": 38.25239944458008,
      "activations/layer21_attention_weight_min": -22.6273136138916,
      "activations/layer22_attention_weight_max": 28.16229248046875,
      "activations/layer22_attention_weight_min": -29.159395217895508,
      "activations/layer23_attention_weight_max": 38.491355895996094,
      "activations/layer23_attention_weight_min": -24.246915817260742,
      "activations/layer2_attention_weight_max": 32.62189483642578,
      "activations/layer2_attention_weight_min": -31.334068298339844,
      "activations/layer3_attention_weight_max": 92.44114685058594,
      "activations/layer3_attention_weight_min": -94.81652069091797,
      "activations/layer4_attention_weight_max": 88.20214080810547,
      "activations/layer4_attention_weight_min": -88.12954711914062,
      "activations/layer5_attention_weight_max": 62.331626892089844,
      "activations/layer5_attention_weight_min": -72.97596740722656,
      "activations/layer6_attention_weight_max": 51.44205856323242,
      "activations/layer6_attention_weight_min": -49.88587951660156,
      "activations/layer7_attention_weight_max": 66.15400695800781,
      "activations/layer7_attention_weight_min": -70.28902435302734,
      "activations/layer8_attention_weight_max": 46.90652847290039,
      "activations/layer8_attention_weight_min": -49.595211029052734,
      "activations/layer9_attention_weight_max": 46.29651641845703,
      "activations/layer9_attention_weight_min": -48.671791076660156,
      "epoch": 22.63,
      "learning_rate": 4.132575757575757e-06,
      "loss": 2.6992,
      "step": 389400
    },
    {
      "activations/layer0_attention_weight_max": 15.45054817199707,
      "activations/layer0_attention_weight_min": -14.000032424926758,
      "activations/layer10_attention_weight_max": 35.49761962890625,
      "activations/layer10_attention_weight_min": -37.042396545410156,
      "activations/layer11_attention_weight_max": 32.635589599609375,
      "activations/layer11_attention_weight_min": -34.35776138305664,
      "activations/layer12_attention_weight_max": 18.577760696411133,
      "activations/layer12_attention_weight_min": -27.45941925048828,
      "activations/layer13_attention_weight_max": 36.234886169433594,
      "activations/layer13_attention_weight_min": -29.097373962402344,
      "activations/layer14_attention_weight_max": 34.954566955566406,
      "activations/layer14_attention_weight_min": -31.60348129272461,
      "activations/layer15_attention_weight_max": 33.5394172668457,
      "activations/layer15_attention_weight_min": -32.398990631103516,
      "activations/layer16_attention_weight_max": 33.49463653564453,
      "activations/layer16_attention_weight_min": -33.52371597290039,
      "activations/layer17_attention_weight_max": 52.47340393066406,
      "activations/layer17_attention_weight_min": -47.503578186035156,
      "activations/layer18_attention_weight_max": 48.11177062988281,
      "activations/layer18_attention_weight_min": -44.09894943237305,
      "activations/layer19_attention_weight_max": 27.646987915039062,
      "activations/layer19_attention_weight_min": -24.178525924682617,
      "activations/layer1_attention_weight_max": 14.998611450195312,
      "activations/layer1_attention_weight_min": -16.040382385253906,
      "activations/layer20_attention_weight_max": 25.4809513092041,
      "activations/layer20_attention_weight_min": -20.611858367919922,
      "activations/layer21_attention_weight_max": 34.78125,
      "activations/layer21_attention_weight_min": -22.540462493896484,
      "activations/layer22_attention_weight_max": 30.935009002685547,
      "activations/layer22_attention_weight_min": -26.44222640991211,
      "activations/layer23_attention_weight_max": 38.90663528442383,
      "activations/layer23_attention_weight_min": -23.289588928222656,
      "activations/layer2_attention_weight_max": 31.315471649169922,
      "activations/layer2_attention_weight_min": -29.755268096923828,
      "activations/layer3_attention_weight_max": 86.93013000488281,
      "activations/layer3_attention_weight_min": -87.60414123535156,
      "activations/layer4_attention_weight_max": 84.10710906982422,
      "activations/layer4_attention_weight_min": -87.90347290039062,
      "activations/layer5_attention_weight_max": 63.07079315185547,
      "activations/layer5_attention_weight_min": -72.28977966308594,
      "activations/layer6_attention_weight_max": 47.474246978759766,
      "activations/layer6_attention_weight_min": -51.68000793457031,
      "activations/layer7_attention_weight_max": 67.20299530029297,
      "activations/layer7_attention_weight_min": -71.1014175415039,
      "activations/layer8_attention_weight_max": 47.941776275634766,
      "activations/layer8_attention_weight_min": -51.94462203979492,
      "activations/layer9_attention_weight_max": 44.9192008972168,
      "activations/layer9_attention_weight_min": -47.3235969543457,
      "epoch": 22.63,
      "learning_rate": 4.113636363636363e-06,
      "loss": 2.716,
      "step": 389450
    },
    {
      "activations/layer0_attention_weight_max": 16.7205753326416,
      "activations/layer0_attention_weight_min": -14.229609489440918,
      "activations/layer10_attention_weight_max": 34.836448669433594,
      "activations/layer10_attention_weight_min": -37.206871032714844,
      "activations/layer11_attention_weight_max": 33.19491958618164,
      "activations/layer11_attention_weight_min": -35.25831985473633,
      "activations/layer12_attention_weight_max": 19.302173614501953,
      "activations/layer12_attention_weight_min": -23.131717681884766,
      "activations/layer13_attention_weight_max": 33.95563507080078,
      "activations/layer13_attention_weight_min": -28.383607864379883,
      "activations/layer14_attention_weight_max": 35.130916595458984,
      "activations/layer14_attention_weight_min": -30.085205078125,
      "activations/layer15_attention_weight_max": 32.31686019897461,
      "activations/layer15_attention_weight_min": -31.32881736755371,
      "activations/layer16_attention_weight_max": 33.02789306640625,
      "activations/layer16_attention_weight_min": -34.19668197631836,
      "activations/layer17_attention_weight_max": 51.268211364746094,
      "activations/layer17_attention_weight_min": -46.51765823364258,
      "activations/layer18_attention_weight_max": 45.95929718017578,
      "activations/layer18_attention_weight_min": -41.74540710449219,
      "activations/layer19_attention_weight_max": 23.27065086364746,
      "activations/layer19_attention_weight_min": -22.599599838256836,
      "activations/layer1_attention_weight_max": 15.198921203613281,
      "activations/layer1_attention_weight_min": -15.87287425994873,
      "activations/layer20_attention_weight_max": 21.295394897460938,
      "activations/layer20_attention_weight_min": -22.4477596282959,
      "activations/layer21_attention_weight_max": 28.699710845947266,
      "activations/layer21_attention_weight_min": -20.991363525390625,
      "activations/layer22_attention_weight_max": 26.311479568481445,
      "activations/layer22_attention_weight_min": -24.377737045288086,
      "activations/layer23_attention_weight_max": 31.35603904724121,
      "activations/layer23_attention_weight_min": -23.234962463378906,
      "activations/layer2_attention_weight_max": 32.556278228759766,
      "activations/layer2_attention_weight_min": -30.005970001220703,
      "activations/layer3_attention_weight_max": 87.66336059570312,
      "activations/layer3_attention_weight_min": -92.72843933105469,
      "activations/layer4_attention_weight_max": 80.8708724975586,
      "activations/layer4_attention_weight_min": -85.59676361083984,
      "activations/layer5_attention_weight_max": 59.28813171386719,
      "activations/layer5_attention_weight_min": -67.89183807373047,
      "activations/layer6_attention_weight_max": 48.311580657958984,
      "activations/layer6_attention_weight_min": -49.75105667114258,
      "activations/layer7_attention_weight_max": 66.10360717773438,
      "activations/layer7_attention_weight_min": -69.33192443847656,
      "activations/layer8_attention_weight_max": 45.80863571166992,
      "activations/layer8_attention_weight_min": -53.19120407104492,
      "activations/layer9_attention_weight_max": 43.84317398071289,
      "activations/layer9_attention_weight_min": -46.306209564208984,
      "epoch": 22.63,
      "learning_rate": 4.0946969696969695e-06,
      "loss": 2.6999,
      "step": 389500
    },
    {
      "activations/layer0_attention_weight_max": 16.36779022216797,
      "activations/layer0_attention_weight_min": -13.737996101379395,
      "activations/layer10_attention_weight_max": 37.338905334472656,
      "activations/layer10_attention_weight_min": -36.87703323364258,
      "activations/layer11_attention_weight_max": 32.97603225708008,
      "activations/layer11_attention_weight_min": -36.852603912353516,
      "activations/layer12_attention_weight_max": 19.182170867919922,
      "activations/layer12_attention_weight_min": -24.15589141845703,
      "activations/layer13_attention_weight_max": 36.5380859375,
      "activations/layer13_attention_weight_min": -27.418493270874023,
      "activations/layer14_attention_weight_max": 36.262020111083984,
      "activations/layer14_attention_weight_min": -30.914743423461914,
      "activations/layer15_attention_weight_max": 32.75016784667969,
      "activations/layer15_attention_weight_min": -31.178495407104492,
      "activations/layer16_attention_weight_max": 34.72010803222656,
      "activations/layer16_attention_weight_min": -32.12855529785156,
      "activations/layer17_attention_weight_max": 51.90480422973633,
      "activations/layer17_attention_weight_min": -46.19036865234375,
      "activations/layer18_attention_weight_max": 51.636756896972656,
      "activations/layer18_attention_weight_min": -39.727664947509766,
      "activations/layer19_attention_weight_max": 23.550395965576172,
      "activations/layer19_attention_weight_min": -22.56005859375,
      "activations/layer1_attention_weight_max": 15.592642784118652,
      "activations/layer1_attention_weight_min": -14.035971641540527,
      "activations/layer20_attention_weight_max": 21.881650924682617,
      "activations/layer20_attention_weight_min": -21.023191452026367,
      "activations/layer21_attention_weight_max": 34.12813186645508,
      "activations/layer21_attention_weight_min": -23.42472267150879,
      "activations/layer22_attention_weight_max": 28.13665199279785,
      "activations/layer22_attention_weight_min": -24.506393432617188,
      "activations/layer23_attention_weight_max": 32.766685485839844,
      "activations/layer23_attention_weight_min": -24.11006736755371,
      "activations/layer2_attention_weight_max": 30.957172393798828,
      "activations/layer2_attention_weight_min": -29.424251556396484,
      "activations/layer3_attention_weight_max": 90.62487030029297,
      "activations/layer3_attention_weight_min": -87.71570587158203,
      "activations/layer4_attention_weight_max": 86.04826354980469,
      "activations/layer4_attention_weight_min": -83.52124786376953,
      "activations/layer5_attention_weight_max": 62.673702239990234,
      "activations/layer5_attention_weight_min": -70.0458755493164,
      "activations/layer6_attention_weight_max": 51.8012809753418,
      "activations/layer6_attention_weight_min": -52.07063674926758,
      "activations/layer7_attention_weight_max": 71.78630065917969,
      "activations/layer7_attention_weight_min": -71.19602966308594,
      "activations/layer8_attention_weight_max": 51.63613510131836,
      "activations/layer8_attention_weight_min": -53.11286544799805,
      "activations/layer9_attention_weight_max": 45.297420501708984,
      "activations/layer9_attention_weight_min": -46.5728645324707,
      "epoch": 22.64,
      "learning_rate": 4.075757575757575e-06,
      "loss": 2.7101,
      "step": 389550
    },
    {
      "activations/layer0_attention_weight_max": 15.884566307067871,
      "activations/layer0_attention_weight_min": -16.538549423217773,
      "activations/layer10_attention_weight_max": 33.80329132080078,
      "activations/layer10_attention_weight_min": -34.14079284667969,
      "activations/layer11_attention_weight_max": 32.818885803222656,
      "activations/layer11_attention_weight_min": -34.43296432495117,
      "activations/layer12_attention_weight_max": 17.557785034179688,
      "activations/layer12_attention_weight_min": -25.628477096557617,
      "activations/layer13_attention_weight_max": 29.92360496520996,
      "activations/layer13_attention_weight_min": -24.55503273010254,
      "activations/layer14_attention_weight_max": 33.66023635864258,
      "activations/layer14_attention_weight_min": -28.76936149597168,
      "activations/layer15_attention_weight_max": 29.872791290283203,
      "activations/layer15_attention_weight_min": -28.265663146972656,
      "activations/layer16_attention_weight_max": 30.945735931396484,
      "activations/layer16_attention_weight_min": -28.848548889160156,
      "activations/layer17_attention_weight_max": 47.1705207824707,
      "activations/layer17_attention_weight_min": -40.16690444946289,
      "activations/layer18_attention_weight_max": 43.14202117919922,
      "activations/layer18_attention_weight_min": -37.583072662353516,
      "activations/layer19_attention_weight_max": 22.75920295715332,
      "activations/layer19_attention_weight_min": -21.771854400634766,
      "activations/layer1_attention_weight_max": 15.97020149230957,
      "activations/layer1_attention_weight_min": -13.826619148254395,
      "activations/layer20_attention_weight_max": 20.950645446777344,
      "activations/layer20_attention_weight_min": -22.524097442626953,
      "activations/layer21_attention_weight_max": 32.03242492675781,
      "activations/layer21_attention_weight_min": -21.932762145996094,
      "activations/layer22_attention_weight_max": 25.1101016998291,
      "activations/layer22_attention_weight_min": -23.8607234954834,
      "activations/layer23_attention_weight_max": 31.42844009399414,
      "activations/layer23_attention_weight_min": -24.72584342956543,
      "activations/layer2_attention_weight_max": 31.324552536010742,
      "activations/layer2_attention_weight_min": -29.681007385253906,
      "activations/layer3_attention_weight_max": 88.40312194824219,
      "activations/layer3_attention_weight_min": -89.81315612792969,
      "activations/layer4_attention_weight_max": 83.87937927246094,
      "activations/layer4_attention_weight_min": -83.95758819580078,
      "activations/layer5_attention_weight_max": 61.52390670776367,
      "activations/layer5_attention_weight_min": -69.9554443359375,
      "activations/layer6_attention_weight_max": 48.664241790771484,
      "activations/layer6_attention_weight_min": -52.02231216430664,
      "activations/layer7_attention_weight_max": 69.37213134765625,
      "activations/layer7_attention_weight_min": -65.14179992675781,
      "activations/layer8_attention_weight_max": 46.14900207519531,
      "activations/layer8_attention_weight_min": -48.02647399902344,
      "activations/layer9_attention_weight_max": 42.7542839050293,
      "activations/layer9_attention_weight_min": -44.528873443603516,
      "epoch": 22.64,
      "learning_rate": 4.056818181818182e-06,
      "loss": 2.6955,
      "step": 389600
    },
    {
      "activations/layer0_attention_weight_max": 15.49549674987793,
      "activations/layer0_attention_weight_min": -13.817370414733887,
      "activations/layer10_attention_weight_max": 33.42226028442383,
      "activations/layer10_attention_weight_min": -34.68252944946289,
      "activations/layer11_attention_weight_max": 34.07862091064453,
      "activations/layer11_attention_weight_min": -36.36805725097656,
      "activations/layer12_attention_weight_max": 17.483259201049805,
      "activations/layer12_attention_weight_min": -24.532480239868164,
      "activations/layer13_attention_weight_max": 30.19100570678711,
      "activations/layer13_attention_weight_min": -26.23140525817871,
      "activations/layer14_attention_weight_max": 32.520751953125,
      "activations/layer14_attention_weight_min": -28.554723739624023,
      "activations/layer15_attention_weight_max": 30.60280418395996,
      "activations/layer15_attention_weight_min": -29.256092071533203,
      "activations/layer16_attention_weight_max": 31.634233474731445,
      "activations/layer16_attention_weight_min": -30.719131469726562,
      "activations/layer17_attention_weight_max": 49.07406234741211,
      "activations/layer17_attention_weight_min": -43.68706130981445,
      "activations/layer18_attention_weight_max": 47.757972717285156,
      "activations/layer18_attention_weight_min": -39.103878021240234,
      "activations/layer19_attention_weight_max": 23.552852630615234,
      "activations/layer19_attention_weight_min": -22.205400466918945,
      "activations/layer1_attention_weight_max": 15.087403297424316,
      "activations/layer1_attention_weight_min": -14.634163856506348,
      "activations/layer20_attention_weight_max": 21.225000381469727,
      "activations/layer20_attention_weight_min": -21.478849411010742,
      "activations/layer21_attention_weight_max": 32.66508483886719,
      "activations/layer21_attention_weight_min": -22.000879287719727,
      "activations/layer22_attention_weight_max": 28.571746826171875,
      "activations/layer22_attention_weight_min": -25.595319747924805,
      "activations/layer23_attention_weight_max": 36.14803695678711,
      "activations/layer23_attention_weight_min": -24.715011596679688,
      "activations/layer2_attention_weight_max": 32.024658203125,
      "activations/layer2_attention_weight_min": -30.302093505859375,
      "activations/layer3_attention_weight_max": 86.84386444091797,
      "activations/layer3_attention_weight_min": -86.99256134033203,
      "activations/layer4_attention_weight_max": 84.03365325927734,
      "activations/layer4_attention_weight_min": -81.53083801269531,
      "activations/layer5_attention_weight_max": 59.54658508300781,
      "activations/layer5_attention_weight_min": -68.44435119628906,
      "activations/layer6_attention_weight_max": 48.95280456542969,
      "activations/layer6_attention_weight_min": -49.64362716674805,
      "activations/layer7_attention_weight_max": 64.09942626953125,
      "activations/layer7_attention_weight_min": -64.95597839355469,
      "activations/layer8_attention_weight_max": 45.16984939575195,
      "activations/layer8_attention_weight_min": -48.31383514404297,
      "activations/layer9_attention_weight_max": 44.91462707519531,
      "activations/layer9_attention_weight_min": -45.24449157714844,
      "epoch": 22.64,
      "learning_rate": 4.0378787878787875e-06,
      "loss": 2.7109,
      "step": 389650
    },
    {
      "activations/layer0_attention_weight_max": 15.623602867126465,
      "activations/layer0_attention_weight_min": -14.383264541625977,
      "activations/layer10_attention_weight_max": 41.7880973815918,
      "activations/layer10_attention_weight_min": -40.034725189208984,
      "activations/layer11_attention_weight_max": 38.697940826416016,
      "activations/layer11_attention_weight_min": -40.128875732421875,
      "activations/layer12_attention_weight_max": 19.42474365234375,
      "activations/layer12_attention_weight_min": -24.051692962646484,
      "activations/layer13_attention_weight_max": 38.13978576660156,
      "activations/layer13_attention_weight_min": -29.442829132080078,
      "activations/layer14_attention_weight_max": 41.35178756713867,
      "activations/layer14_attention_weight_min": -33.037078857421875,
      "activations/layer15_attention_weight_max": 36.4147834777832,
      "activations/layer15_attention_weight_min": -33.013404846191406,
      "activations/layer16_attention_weight_max": 36.657100677490234,
      "activations/layer16_attention_weight_min": -36.68544387817383,
      "activations/layer17_attention_weight_max": 56.877899169921875,
      "activations/layer17_attention_weight_min": -50.38230514526367,
      "activations/layer18_attention_weight_max": 52.94010925292969,
      "activations/layer18_attention_weight_min": -44.364566802978516,
      "activations/layer19_attention_weight_max": 27.26825523376465,
      "activations/layer19_attention_weight_min": -24.1358699798584,
      "activations/layer1_attention_weight_max": 14.619560241699219,
      "activations/layer1_attention_weight_min": -16.266536712646484,
      "activations/layer20_attention_weight_max": 23.012683868408203,
      "activations/layer20_attention_weight_min": -23.426877975463867,
      "activations/layer21_attention_weight_max": 34.90435791015625,
      "activations/layer21_attention_weight_min": -23.08287811279297,
      "activations/layer22_attention_weight_max": 29.97553253173828,
      "activations/layer22_attention_weight_min": -27.330875396728516,
      "activations/layer23_attention_weight_max": 38.227012634277344,
      "activations/layer23_attention_weight_min": -23.57840347290039,
      "activations/layer2_attention_weight_max": 31.46338653564453,
      "activations/layer2_attention_weight_min": -30.44374656677246,
      "activations/layer3_attention_weight_max": 93.8965072631836,
      "activations/layer3_attention_weight_min": -91.28907012939453,
      "activations/layer4_attention_weight_max": 89.08910369873047,
      "activations/layer4_attention_weight_min": -90.5537109375,
      "activations/layer5_attention_weight_max": 64.48446655273438,
      "activations/layer5_attention_weight_min": -75.21258544921875,
      "activations/layer6_attention_weight_max": 51.912723541259766,
      "activations/layer6_attention_weight_min": -54.755863189697266,
      "activations/layer7_attention_weight_max": 77.12217712402344,
      "activations/layer7_attention_weight_min": -73.89936065673828,
      "activations/layer8_attention_weight_max": 52.57362747192383,
      "activations/layer8_attention_weight_min": -57.51254653930664,
      "activations/layer9_attention_weight_max": 49.85274124145508,
      "activations/layer9_attention_weight_min": -51.498165130615234,
      "epoch": 22.64,
      "learning_rate": 4.018939393939393e-06,
      "loss": 2.7156,
      "step": 389700
    },
    {
      "activations/layer0_attention_weight_max": 15.858844757080078,
      "activations/layer0_attention_weight_min": -14.120549201965332,
      "activations/layer10_attention_weight_max": 39.43134307861328,
      "activations/layer10_attention_weight_min": -38.269439697265625,
      "activations/layer11_attention_weight_max": 35.144378662109375,
      "activations/layer11_attention_weight_min": -39.306793212890625,
      "activations/layer12_attention_weight_max": 19.583715438842773,
      "activations/layer12_attention_weight_min": -25.958539962768555,
      "activations/layer13_attention_weight_max": 34.6566047668457,
      "activations/layer13_attention_weight_min": -30.183509826660156,
      "activations/layer14_attention_weight_max": 36.93067169189453,
      "activations/layer14_attention_weight_min": -31.364986419677734,
      "activations/layer15_attention_weight_max": 35.12913131713867,
      "activations/layer15_attention_weight_min": -32.04432678222656,
      "activations/layer16_attention_weight_max": 37.32298278808594,
      "activations/layer16_attention_weight_min": -33.756893157958984,
      "activations/layer17_attention_weight_max": 56.06709289550781,
      "activations/layer17_attention_weight_min": -47.15263748168945,
      "activations/layer18_attention_weight_max": 52.47171401977539,
      "activations/layer18_attention_weight_min": -42.372467041015625,
      "activations/layer19_attention_weight_max": 24.438268661499023,
      "activations/layer19_attention_weight_min": -23.884824752807617,
      "activations/layer1_attention_weight_max": 15.609162330627441,
      "activations/layer1_attention_weight_min": -15.05791187286377,
      "activations/layer20_attention_weight_max": 24.215709686279297,
      "activations/layer20_attention_weight_min": -21.328121185302734,
      "activations/layer21_attention_weight_max": 37.35248947143555,
      "activations/layer21_attention_weight_min": -21.883285522460938,
      "activations/layer22_attention_weight_max": 29.972633361816406,
      "activations/layer22_attention_weight_min": -25.932493209838867,
      "activations/layer23_attention_weight_max": 41.88969039916992,
      "activations/layer23_attention_weight_min": -23.859989166259766,
      "activations/layer2_attention_weight_max": 34.57555389404297,
      "activations/layer2_attention_weight_min": -30.878721237182617,
      "activations/layer3_attention_weight_max": 90.79361724853516,
      "activations/layer3_attention_weight_min": -90.28147888183594,
      "activations/layer4_attention_weight_max": 83.77877044677734,
      "activations/layer4_attention_weight_min": -85.03430938720703,
      "activations/layer5_attention_weight_max": 59.34503173828125,
      "activations/layer5_attention_weight_min": -72.3131332397461,
      "activations/layer6_attention_weight_max": 51.5790901184082,
      "activations/layer6_attention_weight_min": -52.204200744628906,
      "activations/layer7_attention_weight_max": 69.50554656982422,
      "activations/layer7_attention_weight_min": -68.82625579833984,
      "activations/layer8_attention_weight_max": 50.531463623046875,
      "activations/layer8_attention_weight_min": -55.087276458740234,
      "activations/layer9_attention_weight_max": 47.533111572265625,
      "activations/layer9_attention_weight_min": -48.62969970703125,
      "epoch": 22.65,
      "learning_rate": 4e-06,
      "loss": 2.71,
      "step": 389750
    },
    {
      "activations/layer0_attention_weight_max": 15.850326538085938,
      "activations/layer0_attention_weight_min": -14.068904876708984,
      "activations/layer10_attention_weight_max": 35.20134353637695,
      "activations/layer10_attention_weight_min": -35.869537353515625,
      "activations/layer11_attention_weight_max": 33.352333068847656,
      "activations/layer11_attention_weight_min": -34.86178970336914,
      "activations/layer12_attention_weight_max": 18.268573760986328,
      "activations/layer12_attention_weight_min": -24.263431549072266,
      "activations/layer13_attention_weight_max": 34.39871597290039,
      "activations/layer13_attention_weight_min": -27.745159149169922,
      "activations/layer14_attention_weight_max": 36.73828887939453,
      "activations/layer14_attention_weight_min": -32.430641174316406,
      "activations/layer15_attention_weight_max": 37.263427734375,
      "activations/layer15_attention_weight_min": -31.142135620117188,
      "activations/layer16_attention_weight_max": 37.847503662109375,
      "activations/layer16_attention_weight_min": -34.31573486328125,
      "activations/layer17_attention_weight_max": 52.96208953857422,
      "activations/layer17_attention_weight_min": -49.50970458984375,
      "activations/layer18_attention_weight_max": 53.38340377807617,
      "activations/layer18_attention_weight_min": -44.97093200683594,
      "activations/layer19_attention_weight_max": 26.58853530883789,
      "activations/layer19_attention_weight_min": -23.194799423217773,
      "activations/layer1_attention_weight_max": 14.774096488952637,
      "activations/layer1_attention_weight_min": -15.857566833496094,
      "activations/layer20_attention_weight_max": 23.623306274414062,
      "activations/layer20_attention_weight_min": -21.542600631713867,
      "activations/layer21_attention_weight_max": 36.73334503173828,
      "activations/layer21_attention_weight_min": -22.576486587524414,
      "activations/layer22_attention_weight_max": 31.455307006835938,
      "activations/layer22_attention_weight_min": -26.7813777923584,
      "activations/layer23_attention_weight_max": 39.82215118408203,
      "activations/layer23_attention_weight_min": -22.992368698120117,
      "activations/layer2_attention_weight_max": 33.06608581542969,
      "activations/layer2_attention_weight_min": -32.93294143676758,
      "activations/layer3_attention_weight_max": 89.55256652832031,
      "activations/layer3_attention_weight_min": -92.79010009765625,
      "activations/layer4_attention_weight_max": 84.05502319335938,
      "activations/layer4_attention_weight_min": -84.96963500976562,
      "activations/layer5_attention_weight_max": 62.53367614746094,
      "activations/layer5_attention_weight_min": -72.06474304199219,
      "activations/layer6_attention_weight_max": 48.3541374206543,
      "activations/layer6_attention_weight_min": -50.31938552856445,
      "activations/layer7_attention_weight_max": 69.75399017333984,
      "activations/layer7_attention_weight_min": -68.40894317626953,
      "activations/layer8_attention_weight_max": 46.2643928527832,
      "activations/layer8_attention_weight_min": -48.71162414550781,
      "activations/layer9_attention_weight_max": 43.244178771972656,
      "activations/layer9_attention_weight_min": -45.5655517578125,
      "epoch": 22.65,
      "learning_rate": 3.9810606060606055e-06,
      "loss": 2.6783,
      "step": 389800
    },
    {
      "activations/layer0_attention_weight_max": 16.173389434814453,
      "activations/layer0_attention_weight_min": -14.02863883972168,
      "activations/layer10_attention_weight_max": 35.679962158203125,
      "activations/layer10_attention_weight_min": -36.91754150390625,
      "activations/layer11_attention_weight_max": 34.143924713134766,
      "activations/layer11_attention_weight_min": -39.46454620361328,
      "activations/layer12_attention_weight_max": 21.11640739440918,
      "activations/layer12_attention_weight_min": -25.482988357543945,
      "activations/layer13_attention_weight_max": 37.61241912841797,
      "activations/layer13_attention_weight_min": -27.795719146728516,
      "activations/layer14_attention_weight_max": 39.192848205566406,
      "activations/layer14_attention_weight_min": -33.917789459228516,
      "activations/layer15_attention_weight_max": 38.86817932128906,
      "activations/layer15_attention_weight_min": -32.805477142333984,
      "activations/layer16_attention_weight_max": 38.20799255371094,
      "activations/layer16_attention_weight_min": -34.465660095214844,
      "activations/layer17_attention_weight_max": 56.55878829956055,
      "activations/layer17_attention_weight_min": -48.0435905456543,
      "activations/layer18_attention_weight_max": 51.2006950378418,
      "activations/layer18_attention_weight_min": -43.658416748046875,
      "activations/layer19_attention_weight_max": 26.51173973083496,
      "activations/layer19_attention_weight_min": -23.06122398376465,
      "activations/layer1_attention_weight_max": 15.56017017364502,
      "activations/layer1_attention_weight_min": -15.812764167785645,
      "activations/layer20_attention_weight_max": 21.951454162597656,
      "activations/layer20_attention_weight_min": -21.4787540435791,
      "activations/layer21_attention_weight_max": 35.15144729614258,
      "activations/layer21_attention_weight_min": -22.25020408630371,
      "activations/layer22_attention_weight_max": 30.800317764282227,
      "activations/layer22_attention_weight_min": -26.76397705078125,
      "activations/layer23_attention_weight_max": 34.3913459777832,
      "activations/layer23_attention_weight_min": -23.991539001464844,
      "activations/layer2_attention_weight_max": 32.976715087890625,
      "activations/layer2_attention_weight_min": -30.331768035888672,
      "activations/layer3_attention_weight_max": 88.6639404296875,
      "activations/layer3_attention_weight_min": -92.74495697021484,
      "activations/layer4_attention_weight_max": 81.12157440185547,
      "activations/layer4_attention_weight_min": -85.66600799560547,
      "activations/layer5_attention_weight_max": 61.47582244873047,
      "activations/layer5_attention_weight_min": -67.03033447265625,
      "activations/layer6_attention_weight_max": 47.717857360839844,
      "activations/layer6_attention_weight_min": -52.674171447753906,
      "activations/layer7_attention_weight_max": 66.81753540039062,
      "activations/layer7_attention_weight_min": -69.84892272949219,
      "activations/layer8_attention_weight_max": 47.39137268066406,
      "activations/layer8_attention_weight_min": -51.19921875,
      "activations/layer9_attention_weight_max": 42.84889602661133,
      "activations/layer9_attention_weight_min": -49.03915786743164,
      "epoch": 22.65,
      "learning_rate": 3.962121212121212e-06,
      "loss": 2.7088,
      "step": 389850
    },
    {
      "activations/layer0_attention_weight_max": 16.40182876586914,
      "activations/layer0_attention_weight_min": -14.237523078918457,
      "activations/layer10_attention_weight_max": 35.526344299316406,
      "activations/layer10_attention_weight_min": -35.8714485168457,
      "activations/layer11_attention_weight_max": 32.121246337890625,
      "activations/layer11_attention_weight_min": -33.950103759765625,
      "activations/layer12_attention_weight_max": 17.8877010345459,
      "activations/layer12_attention_weight_min": -25.5799617767334,
      "activations/layer13_attention_weight_max": 31.516992568969727,
      "activations/layer13_attention_weight_min": -27.581384658813477,
      "activations/layer14_attention_weight_max": 32.14707946777344,
      "activations/layer14_attention_weight_min": -31.506134033203125,
      "activations/layer15_attention_weight_max": 30.624570846557617,
      "activations/layer15_attention_weight_min": -31.43183708190918,
      "activations/layer16_attention_weight_max": 32.13903045654297,
      "activations/layer16_attention_weight_min": -31.991756439208984,
      "activations/layer17_attention_weight_max": 50.5208740234375,
      "activations/layer17_attention_weight_min": -46.00831985473633,
      "activations/layer18_attention_weight_max": 45.21870040893555,
      "activations/layer18_attention_weight_min": -41.93285369873047,
      "activations/layer19_attention_weight_max": 24.074657440185547,
      "activations/layer19_attention_weight_min": -23.83445930480957,
      "activations/layer1_attention_weight_max": 15.302613258361816,
      "activations/layer1_attention_weight_min": -14.148674011230469,
      "activations/layer20_attention_weight_max": 21.403955459594727,
      "activations/layer20_attention_weight_min": -24.133604049682617,
      "activations/layer21_attention_weight_max": 35.85289001464844,
      "activations/layer21_attention_weight_min": -24.35164451599121,
      "activations/layer22_attention_weight_max": 27.87540054321289,
      "activations/layer22_attention_weight_min": -28.771556854248047,
      "activations/layer23_attention_weight_max": 36.43980407714844,
      "activations/layer23_attention_weight_min": -24.672992706298828,
      "activations/layer2_attention_weight_max": 32.83480453491211,
      "activations/layer2_attention_weight_min": -31.861328125,
      "activations/layer3_attention_weight_max": 92.44229888916016,
      "activations/layer3_attention_weight_min": -92.98662567138672,
      "activations/layer4_attention_weight_max": 87.43461608886719,
      "activations/layer4_attention_weight_min": -85.52806854248047,
      "activations/layer5_attention_weight_max": 63.32763671875,
      "activations/layer5_attention_weight_min": -73.49447631835938,
      "activations/layer6_attention_weight_max": 48.28977966308594,
      "activations/layer6_attention_weight_min": -51.566307067871094,
      "activations/layer7_attention_weight_max": 73.25807189941406,
      "activations/layer7_attention_weight_min": -70.61947631835938,
      "activations/layer8_attention_weight_max": 49.00418472290039,
      "activations/layer8_attention_weight_min": -51.38275909423828,
      "activations/layer9_attention_weight_max": 43.01632308959961,
      "activations/layer9_attention_weight_min": -45.46975326538086,
      "epoch": 22.66,
      "learning_rate": 3.943181818181818e-06,
      "loss": 2.7141,
      "step": 389900
    },
    {
      "activations/layer0_attention_weight_max": 16.69892120361328,
      "activations/layer0_attention_weight_min": -14.197637557983398,
      "activations/layer10_attention_weight_max": 38.4268684387207,
      "activations/layer10_attention_weight_min": -37.951908111572266,
      "activations/layer11_attention_weight_max": 36.13027572631836,
      "activations/layer11_attention_weight_min": -36.835662841796875,
      "activations/layer12_attention_weight_max": 20.37398338317871,
      "activations/layer12_attention_weight_min": -25.312040328979492,
      "activations/layer13_attention_weight_max": 34.4251708984375,
      "activations/layer13_attention_weight_min": -26.007038116455078,
      "activations/layer14_attention_weight_max": 34.682281494140625,
      "activations/layer14_attention_weight_min": -28.568803787231445,
      "activations/layer15_attention_weight_max": 32.32013702392578,
      "activations/layer15_attention_weight_min": -30.400049209594727,
      "activations/layer16_attention_weight_max": 33.13564682006836,
      "activations/layer16_attention_weight_min": -30.765417098999023,
      "activations/layer17_attention_weight_max": 50.14728927612305,
      "activations/layer17_attention_weight_min": -45.27747344970703,
      "activations/layer18_attention_weight_max": 46.02062225341797,
      "activations/layer18_attention_weight_min": -39.6693115234375,
      "activations/layer19_attention_weight_max": 22.83722496032715,
      "activations/layer19_attention_weight_min": -22.932538986206055,
      "activations/layer1_attention_weight_max": 17.05093765258789,
      "activations/layer1_attention_weight_min": -16.610424041748047,
      "activations/layer20_attention_weight_max": 20.643192291259766,
      "activations/layer20_attention_weight_min": -23.220142364501953,
      "activations/layer21_attention_weight_max": 36.57279968261719,
      "activations/layer21_attention_weight_min": -22.53307342529297,
      "activations/layer22_attention_weight_max": 26.35113525390625,
      "activations/layer22_attention_weight_min": -25.497285842895508,
      "activations/layer23_attention_weight_max": 35.058197021484375,
      "activations/layer23_attention_weight_min": -25.34815788269043,
      "activations/layer2_attention_weight_max": 33.507652282714844,
      "activations/layer2_attention_weight_min": -32.312156677246094,
      "activations/layer3_attention_weight_max": 93.67001342773438,
      "activations/layer3_attention_weight_min": -89.87437438964844,
      "activations/layer4_attention_weight_max": 86.21935272216797,
      "activations/layer4_attention_weight_min": -84.75141906738281,
      "activations/layer5_attention_weight_max": 59.56364440917969,
      "activations/layer5_attention_weight_min": -69.18669128417969,
      "activations/layer6_attention_weight_max": 50.52155303955078,
      "activations/layer6_attention_weight_min": -49.33687210083008,
      "activations/layer7_attention_weight_max": 67.0096435546875,
      "activations/layer7_attention_weight_min": -68.94647216796875,
      "activations/layer8_attention_weight_max": 52.711639404296875,
      "activations/layer8_attention_weight_min": -53.551456451416016,
      "activations/layer9_attention_weight_max": 47.21670150756836,
      "activations/layer9_attention_weight_min": -47.852657318115234,
      "epoch": 22.66,
      "learning_rate": 3.924242424242424e-06,
      "loss": 2.7166,
      "step": 389950
    },
    {
      "activations/layer0_attention_weight_max": 16.193635940551758,
      "activations/layer0_attention_weight_min": -14.112388610839844,
      "activations/layer10_attention_weight_max": 39.908138275146484,
      "activations/layer10_attention_weight_min": -37.9992561340332,
      "activations/layer11_attention_weight_max": 40.42324447631836,
      "activations/layer11_attention_weight_min": -40.748409271240234,
      "activations/layer12_attention_weight_max": 19.140684127807617,
      "activations/layer12_attention_weight_min": -25.00035285949707,
      "activations/layer13_attention_weight_max": 33.504940032958984,
      "activations/layer13_attention_weight_min": -29.15861701965332,
      "activations/layer14_attention_weight_max": 36.62039566040039,
      "activations/layer14_attention_weight_min": -31.021379470825195,
      "activations/layer15_attention_weight_max": 34.731407165527344,
      "activations/layer15_attention_weight_min": -34.43022537231445,
      "activations/layer16_attention_weight_max": 37.09922790527344,
      "activations/layer16_attention_weight_min": -34.50418472290039,
      "activations/layer17_attention_weight_max": 55.17890930175781,
      "activations/layer17_attention_weight_min": -47.04383850097656,
      "activations/layer18_attention_weight_max": 47.63729476928711,
      "activations/layer18_attention_weight_min": -39.09784698486328,
      "activations/layer19_attention_weight_max": 23.701745986938477,
      "activations/layer19_attention_weight_min": -21.46357536315918,
      "activations/layer1_attention_weight_max": 15.183706283569336,
      "activations/layer1_attention_weight_min": -14.35392951965332,
      "activations/layer20_attention_weight_max": 21.28831672668457,
      "activations/layer20_attention_weight_min": -22.30941390991211,
      "activations/layer21_attention_weight_max": 34.67293930053711,
      "activations/layer21_attention_weight_min": -21.03584098815918,
      "activations/layer22_attention_weight_max": 27.318058013916016,
      "activations/layer22_attention_weight_min": -25.95903205871582,
      "activations/layer23_attention_weight_max": 34.26929473876953,
      "activations/layer23_attention_weight_min": -23.40927505493164,
      "activations/layer2_attention_weight_max": 36.99225616455078,
      "activations/layer2_attention_weight_min": -32.53120040893555,
      "activations/layer3_attention_weight_max": 95.95751953125,
      "activations/layer3_attention_weight_min": -96.4811019897461,
      "activations/layer4_attention_weight_max": 86.3410415649414,
      "activations/layer4_attention_weight_min": -85.31067657470703,
      "activations/layer5_attention_weight_max": 61.43882751464844,
      "activations/layer5_attention_weight_min": -71.82273864746094,
      "activations/layer6_attention_weight_max": 49.3302116394043,
      "activations/layer6_attention_weight_min": -51.17780685424805,
      "activations/layer7_attention_weight_max": 71.045166015625,
      "activations/layer7_attention_weight_min": -76.26795959472656,
      "activations/layer8_attention_weight_max": 49.623138427734375,
      "activations/layer8_attention_weight_min": -52.15737533569336,
      "activations/layer9_attention_weight_max": 50.90143966674805,
      "activations/layer9_attention_weight_min": -50.31618881225586,
      "epoch": 22.66,
      "learning_rate": 3.90530303030303e-06,
      "loss": 2.7115,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4479,
      "eval_samples_per_second": 508.293,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4479,
      "eval_openwebtext_samples_per_second": 508.293,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.975,
      "eval_wikitext_samples_per_second": 230.882,
      "step": 390000
    },
    {
      "epoch": 22.66,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5732,
      "eval_lambada_samples_per_second": 508.607,
      "step": 390000
    },
    {
      "activations/layer0_attention_weight_max": 15.324455261230469,
      "activations/layer0_attention_weight_min": -13.899555206298828,
      "activations/layer10_attention_weight_max": 37.07065200805664,
      "activations/layer10_attention_weight_min": -36.28369903564453,
      "activations/layer11_attention_weight_max": 36.0323371887207,
      "activations/layer11_attention_weight_min": -35.7375602722168,
      "activations/layer12_attention_weight_max": 20.80862808227539,
      "activations/layer12_attention_weight_min": -23.780006408691406,
      "activations/layer13_attention_weight_max": 36.1490364074707,
      "activations/layer13_attention_weight_min": -29.470306396484375,
      "activations/layer14_attention_weight_max": 35.377540588378906,
      "activations/layer14_attention_weight_min": -31.3792724609375,
      "activations/layer15_attention_weight_max": 35.144866943359375,
      "activations/layer15_attention_weight_min": -31.719810485839844,
      "activations/layer16_attention_weight_max": 36.35983657836914,
      "activations/layer16_attention_weight_min": -34.05997848510742,
      "activations/layer17_attention_weight_max": 53.73814010620117,
      "activations/layer17_attention_weight_min": -47.20087814331055,
      "activations/layer18_attention_weight_max": 48.34587860107422,
      "activations/layer18_attention_weight_min": -41.948150634765625,
      "activations/layer19_attention_weight_max": 23.693115234375,
      "activations/layer19_attention_weight_min": -22.839481353759766,
      "activations/layer1_attention_weight_max": 16.035192489624023,
      "activations/layer1_attention_weight_min": -14.996081352233887,
      "activations/layer20_attention_weight_max": 22.94546890258789,
      "activations/layer20_attention_weight_min": -21.604188919067383,
      "activations/layer21_attention_weight_max": 35.30439758300781,
      "activations/layer21_attention_weight_min": -19.843246459960938,
      "activations/layer22_attention_weight_max": 27.79229736328125,
      "activations/layer22_attention_weight_min": -25.573532104492188,
      "activations/layer23_attention_weight_max": 35.747413635253906,
      "activations/layer23_attention_weight_min": -23.43588638305664,
      "activations/layer2_attention_weight_max": 32.201210021972656,
      "activations/layer2_attention_weight_min": -31.59152603149414,
      "activations/layer3_attention_weight_max": 91.75515747070312,
      "activations/layer3_attention_weight_min": -91.45414733886719,
      "activations/layer4_attention_weight_max": 86.35223388671875,
      "activations/layer4_attention_weight_min": -86.81352233886719,
      "activations/layer5_attention_weight_max": 62.08179473876953,
      "activations/layer5_attention_weight_min": -74.2056884765625,
      "activations/layer6_attention_weight_max": 49.84636306762695,
      "activations/layer6_attention_weight_min": -52.12508010864258,
      "activations/layer7_attention_weight_max": 69.20126342773438,
      "activations/layer7_attention_weight_min": -71.2620620727539,
      "activations/layer8_attention_weight_max": 47.24464797973633,
      "activations/layer8_attention_weight_min": -55.557334899902344,
      "activations/layer9_attention_weight_max": 45.724727630615234,
      "activations/layer9_attention_weight_min": -48.6951789855957,
      "epoch": 22.66,
      "learning_rate": 3.886363636363636e-06,
      "loss": 2.6898,
      "step": 390050
    },
    {
      "activations/layer0_attention_weight_max": 15.907574653625488,
      "activations/layer0_attention_weight_min": -14.030923843383789,
      "activations/layer10_attention_weight_max": 37.04975509643555,
      "activations/layer10_attention_weight_min": -36.93668746948242,
      "activations/layer11_attention_weight_max": 34.63945770263672,
      "activations/layer11_attention_weight_min": -36.842002868652344,
      "activations/layer12_attention_weight_max": 19.859155654907227,
      "activations/layer12_attention_weight_min": -22.510608673095703,
      "activations/layer13_attention_weight_max": 33.48067855834961,
      "activations/layer13_attention_weight_min": -25.678653717041016,
      "activations/layer14_attention_weight_max": 34.73319625854492,
      "activations/layer14_attention_weight_min": -31.814393997192383,
      "activations/layer15_attention_weight_max": 33.37236404418945,
      "activations/layer15_attention_weight_min": -31.76535987854004,
      "activations/layer16_attention_weight_max": 35.99433898925781,
      "activations/layer16_attention_weight_min": -34.52707290649414,
      "activations/layer17_attention_weight_max": 57.55575942993164,
      "activations/layer17_attention_weight_min": -51.134647369384766,
      "activations/layer18_attention_weight_max": 50.177650451660156,
      "activations/layer18_attention_weight_min": -44.50560760498047,
      "activations/layer19_attention_weight_max": 25.545822143554688,
      "activations/layer19_attention_weight_min": -24.724271774291992,
      "activations/layer1_attention_weight_max": 15.203582763671875,
      "activations/layer1_attention_weight_min": -14.841668128967285,
      "activations/layer20_attention_weight_max": 23.204387664794922,
      "activations/layer20_attention_weight_min": -22.791728973388672,
      "activations/layer21_attention_weight_max": 39.686439514160156,
      "activations/layer21_attention_weight_min": -23.1268310546875,
      "activations/layer22_attention_weight_max": 30.18770408630371,
      "activations/layer22_attention_weight_min": -27.43718719482422,
      "activations/layer23_attention_weight_max": 37.181785583496094,
      "activations/layer23_attention_weight_min": -24.083900451660156,
      "activations/layer2_attention_weight_max": 30.797245025634766,
      "activations/layer2_attention_weight_min": -30.011775970458984,
      "activations/layer3_attention_weight_max": 88.6096420288086,
      "activations/layer3_attention_weight_min": -93.311767578125,
      "activations/layer4_attention_weight_max": 85.09435272216797,
      "activations/layer4_attention_weight_min": -85.72590637207031,
      "activations/layer5_attention_weight_max": 62.39045333862305,
      "activations/layer5_attention_weight_min": -72.01374816894531,
      "activations/layer6_attention_weight_max": 51.94669723510742,
      "activations/layer6_attention_weight_min": -51.45109939575195,
      "activations/layer7_attention_weight_max": 67.66759490966797,
      "activations/layer7_attention_weight_min": -69.14820098876953,
      "activations/layer8_attention_weight_max": 46.34760284423828,
      "activations/layer8_attention_weight_min": -51.549652099609375,
      "activations/layer9_attention_weight_max": 44.359737396240234,
      "activations/layer9_attention_weight_min": -48.66242980957031,
      "epoch": 22.67,
      "learning_rate": 3.867424242424242e-06,
      "loss": 2.7033,
      "step": 390100
    },
    {
      "activations/layer0_attention_weight_max": 15.45557689666748,
      "activations/layer0_attention_weight_min": -14.019547462463379,
      "activations/layer10_attention_weight_max": 38.51189422607422,
      "activations/layer10_attention_weight_min": -39.5369873046875,
      "activations/layer11_attention_weight_max": 36.0263786315918,
      "activations/layer11_attention_weight_min": -36.23213577270508,
      "activations/layer12_attention_weight_max": 19.9456787109375,
      "activations/layer12_attention_weight_min": -24.66067123413086,
      "activations/layer13_attention_weight_max": 37.50244903564453,
      "activations/layer13_attention_weight_min": -27.78330421447754,
      "activations/layer14_attention_weight_max": 34.07514953613281,
      "activations/layer14_attention_weight_min": -29.450580596923828,
      "activations/layer15_attention_weight_max": 32.27640914916992,
      "activations/layer15_attention_weight_min": -28.461210250854492,
      "activations/layer16_attention_weight_max": 33.88068771362305,
      "activations/layer16_attention_weight_min": -30.327062606811523,
      "activations/layer17_attention_weight_max": 50.329811096191406,
      "activations/layer17_attention_weight_min": -43.22206497192383,
      "activations/layer18_attention_weight_max": 46.250022888183594,
      "activations/layer18_attention_weight_min": -38.77814865112305,
      "activations/layer19_attention_weight_max": 23.213455200195312,
      "activations/layer19_attention_weight_min": -24.224590301513672,
      "activations/layer1_attention_weight_max": 15.093968391418457,
      "activations/layer1_attention_weight_min": -14.815832138061523,
      "activations/layer20_attention_weight_max": 21.52240562438965,
      "activations/layer20_attention_weight_min": -19.862709045410156,
      "activations/layer21_attention_weight_max": 33.613861083984375,
      "activations/layer21_attention_weight_min": -22.891502380371094,
      "activations/layer22_attention_weight_max": 28.679914474487305,
      "activations/layer22_attention_weight_min": -24.163179397583008,
      "activations/layer23_attention_weight_max": 35.219383239746094,
      "activations/layer23_attention_weight_min": -23.390607833862305,
      "activations/layer2_attention_weight_max": 32.995697021484375,
      "activations/layer2_attention_weight_min": -31.752676010131836,
      "activations/layer3_attention_weight_max": 92.24879455566406,
      "activations/layer3_attention_weight_min": -93.8548812866211,
      "activations/layer4_attention_weight_max": 89.77950286865234,
      "activations/layer4_attention_weight_min": -86.82324981689453,
      "activations/layer5_attention_weight_max": 62.467674255371094,
      "activations/layer5_attention_weight_min": -74.61296081542969,
      "activations/layer6_attention_weight_max": 51.9109001159668,
      "activations/layer6_attention_weight_min": -53.50246047973633,
      "activations/layer7_attention_weight_max": 70.08328247070312,
      "activations/layer7_attention_weight_min": -70.06352233886719,
      "activations/layer8_attention_weight_max": 49.62424850463867,
      "activations/layer8_attention_weight_min": -54.333763122558594,
      "activations/layer9_attention_weight_max": 46.43745422363281,
      "activations/layer9_attention_weight_min": -48.13768005371094,
      "epoch": 22.67,
      "learning_rate": 3.848484848484848e-06,
      "loss": 2.7074,
      "step": 390150
    },
    {
      "activations/layer0_attention_weight_max": 15.891814231872559,
      "activations/layer0_attention_weight_min": -14.214405059814453,
      "activations/layer10_attention_weight_max": 39.6822395324707,
      "activations/layer10_attention_weight_min": -40.7679557800293,
      "activations/layer11_attention_weight_max": 37.093353271484375,
      "activations/layer11_attention_weight_min": -38.79606628417969,
      "activations/layer12_attention_weight_max": 21.833332061767578,
      "activations/layer12_attention_weight_min": -25.510536193847656,
      "activations/layer13_attention_weight_max": 35.45259094238281,
      "activations/layer13_attention_weight_min": -30.989992141723633,
      "activations/layer14_attention_weight_max": 36.32176971435547,
      "activations/layer14_attention_weight_min": -30.424386978149414,
      "activations/layer15_attention_weight_max": 34.292877197265625,
      "activations/layer15_attention_weight_min": -30.77561378479004,
      "activations/layer16_attention_weight_max": 35.902381896972656,
      "activations/layer16_attention_weight_min": -33.447349548339844,
      "activations/layer17_attention_weight_max": 52.880130767822266,
      "activations/layer17_attention_weight_min": -45.56374740600586,
      "activations/layer18_attention_weight_max": 49.02250289916992,
      "activations/layer18_attention_weight_min": -39.667762756347656,
      "activations/layer19_attention_weight_max": 24.64191436767578,
      "activations/layer19_attention_weight_min": -22.76748275756836,
      "activations/layer1_attention_weight_max": 15.6370849609375,
      "activations/layer1_attention_weight_min": -15.443025588989258,
      "activations/layer20_attention_weight_max": 21.87784194946289,
      "activations/layer20_attention_weight_min": -22.132036209106445,
      "activations/layer21_attention_weight_max": 34.810359954833984,
      "activations/layer21_attention_weight_min": -24.0546875,
      "activations/layer22_attention_weight_max": 29.245527267456055,
      "activations/layer22_attention_weight_min": -26.01091766357422,
      "activations/layer23_attention_weight_max": 39.59313201904297,
      "activations/layer23_attention_weight_min": -24.4285945892334,
      "activations/layer2_attention_weight_max": 32.269287109375,
      "activations/layer2_attention_weight_min": -30.34478759765625,
      "activations/layer3_attention_weight_max": 92.76630401611328,
      "activations/layer3_attention_weight_min": -92.33073425292969,
      "activations/layer4_attention_weight_max": 89.75959777832031,
      "activations/layer4_attention_weight_min": -87.9862289428711,
      "activations/layer5_attention_weight_max": 66.70111083984375,
      "activations/layer5_attention_weight_min": -73.57915496826172,
      "activations/layer6_attention_weight_max": 52.47761535644531,
      "activations/layer6_attention_weight_min": -52.81696701049805,
      "activations/layer7_attention_weight_max": 74.92450714111328,
      "activations/layer7_attention_weight_min": -72.74320983886719,
      "activations/layer8_attention_weight_max": 51.222328186035156,
      "activations/layer8_attention_weight_min": -56.284053802490234,
      "activations/layer9_attention_weight_max": 45.32070541381836,
      "activations/layer9_attention_weight_min": -51.11825180053711,
      "epoch": 22.67,
      "learning_rate": 3.829545454545455e-06,
      "loss": 2.6978,
      "step": 390200
    },
    {
      "activations/layer0_attention_weight_max": 15.6804780960083,
      "activations/layer0_attention_weight_min": -14.786535263061523,
      "activations/layer10_attention_weight_max": 42.20457077026367,
      "activations/layer10_attention_weight_min": -38.9161262512207,
      "activations/layer11_attention_weight_max": 38.77532196044922,
      "activations/layer11_attention_weight_min": -38.54010772705078,
      "activations/layer12_attention_weight_max": 19.338029861450195,
      "activations/layer12_attention_weight_min": -25.354156494140625,
      "activations/layer13_attention_weight_max": 38.831382751464844,
      "activations/layer13_attention_weight_min": -28.740041732788086,
      "activations/layer14_attention_weight_max": 40.15653610229492,
      "activations/layer14_attention_weight_min": -32.3633918762207,
      "activations/layer15_attention_weight_max": 33.941951751708984,
      "activations/layer15_attention_weight_min": -31.890018463134766,
      "activations/layer16_attention_weight_max": 38.29450988769531,
      "activations/layer16_attention_weight_min": -36.8875732421875,
      "activations/layer17_attention_weight_max": 56.380558013916016,
      "activations/layer17_attention_weight_min": -48.672401428222656,
      "activations/layer18_attention_weight_max": 50.64114761352539,
      "activations/layer18_attention_weight_min": -41.85770797729492,
      "activations/layer19_attention_weight_max": 26.137042999267578,
      "activations/layer19_attention_weight_min": -23.037952423095703,
      "activations/layer1_attention_weight_max": 14.742168426513672,
      "activations/layer1_attention_weight_min": -14.499274253845215,
      "activations/layer20_attention_weight_max": 25.012248992919922,
      "activations/layer20_attention_weight_min": -21.994556427001953,
      "activations/layer21_attention_weight_max": 35.07001876831055,
      "activations/layer21_attention_weight_min": -22.41278839111328,
      "activations/layer22_attention_weight_max": 30.634206771850586,
      "activations/layer22_attention_weight_min": -26.122453689575195,
      "activations/layer23_attention_weight_max": 34.50342559814453,
      "activations/layer23_attention_weight_min": -27.12433433532715,
      "activations/layer2_attention_weight_max": 33.4342155456543,
      "activations/layer2_attention_weight_min": -31.68471336364746,
      "activations/layer3_attention_weight_max": 93.17720794677734,
      "activations/layer3_attention_weight_min": -90.66445922851562,
      "activations/layer4_attention_weight_max": 88.00684356689453,
      "activations/layer4_attention_weight_min": -87.3388900756836,
      "activations/layer5_attention_weight_max": 65.29376220703125,
      "activations/layer5_attention_weight_min": -71.86078643798828,
      "activations/layer6_attention_weight_max": 52.9381217956543,
      "activations/layer6_attention_weight_min": -52.80937194824219,
      "activations/layer7_attention_weight_max": 76.94975280761719,
      "activations/layer7_attention_weight_min": -70.49955749511719,
      "activations/layer8_attention_weight_max": 50.239830017089844,
      "activations/layer8_attention_weight_min": -53.60274124145508,
      "activations/layer9_attention_weight_max": 49.36627197265625,
      "activations/layer9_attention_weight_min": -48.65088653564453,
      "epoch": 22.68,
      "learning_rate": 3.8106060606060605e-06,
      "loss": 2.698,
      "step": 390250
    },
    {
      "activations/layer0_attention_weight_max": 16.20669937133789,
      "activations/layer0_attention_weight_min": -14.579710006713867,
      "activations/layer10_attention_weight_max": 33.928157806396484,
      "activations/layer10_attention_weight_min": -34.462745666503906,
      "activations/layer11_attention_weight_max": 33.8592643737793,
      "activations/layer11_attention_weight_min": -35.27891540527344,
      "activations/layer12_attention_weight_max": 18.289247512817383,
      "activations/layer12_attention_weight_min": -25.862272262573242,
      "activations/layer13_attention_weight_max": 31.496980667114258,
      "activations/layer13_attention_weight_min": -26.387453079223633,
      "activations/layer14_attention_weight_max": 34.35106658935547,
      "activations/layer14_attention_weight_min": -29.49315071105957,
      "activations/layer15_attention_weight_max": 30.492938995361328,
      "activations/layer15_attention_weight_min": -29.562423706054688,
      "activations/layer16_attention_weight_max": 31.59232521057129,
      "activations/layer16_attention_weight_min": -31.41284942626953,
      "activations/layer17_attention_weight_max": 48.80739974975586,
      "activations/layer17_attention_weight_min": -43.87247848510742,
      "activations/layer18_attention_weight_max": 44.28321075439453,
      "activations/layer18_attention_weight_min": -37.338504791259766,
      "activations/layer19_attention_weight_max": 22.179868698120117,
      "activations/layer19_attention_weight_min": -21.96836280822754,
      "activations/layer1_attention_weight_max": 16.109689712524414,
      "activations/layer1_attention_weight_min": -14.307600021362305,
      "activations/layer20_attention_weight_max": 19.50577735900879,
      "activations/layer20_attention_weight_min": -22.64344596862793,
      "activations/layer21_attention_weight_max": 29.16864585876465,
      "activations/layer21_attention_weight_min": -20.11984634399414,
      "activations/layer22_attention_weight_max": 26.472431182861328,
      "activations/layer22_attention_weight_min": -25.20943260192871,
      "activations/layer23_attention_weight_max": 33.203208923339844,
      "activations/layer23_attention_weight_min": -24.13272476196289,
      "activations/layer2_attention_weight_max": 31.55476951599121,
      "activations/layer2_attention_weight_min": -31.027633666992188,
      "activations/layer3_attention_weight_max": 90.70257568359375,
      "activations/layer3_attention_weight_min": -94.78975677490234,
      "activations/layer4_attention_weight_max": 84.58283233642578,
      "activations/layer4_attention_weight_min": -83.0466537475586,
      "activations/layer5_attention_weight_max": 63.0267333984375,
      "activations/layer5_attention_weight_min": -77.21275329589844,
      "activations/layer6_attention_weight_max": 48.43537521362305,
      "activations/layer6_attention_weight_min": -48.99879455566406,
      "activations/layer7_attention_weight_max": 69.16459655761719,
      "activations/layer7_attention_weight_min": -65.89586639404297,
      "activations/layer8_attention_weight_max": 45.24208450317383,
      "activations/layer8_attention_weight_min": -48.30281448364258,
      "activations/layer9_attention_weight_max": 47.6411247253418,
      "activations/layer9_attention_weight_min": -43.709346771240234,
      "epoch": 22.68,
      "learning_rate": 3.7916666666666662e-06,
      "loss": 2.6919,
      "step": 390300
    },
    {
      "activations/layer0_attention_weight_max": 15.850937843322754,
      "activations/layer0_attention_weight_min": -14.096534729003906,
      "activations/layer10_attention_weight_max": 38.11803436279297,
      "activations/layer10_attention_weight_min": -38.540008544921875,
      "activations/layer11_attention_weight_max": 35.161102294921875,
      "activations/layer11_attention_weight_min": -36.714027404785156,
      "activations/layer12_attention_weight_max": 19.74701499938965,
      "activations/layer12_attention_weight_min": -27.625259399414062,
      "activations/layer13_attention_weight_max": 34.12940979003906,
      "activations/layer13_attention_weight_min": -26.785707473754883,
      "activations/layer14_attention_weight_max": 34.58000564575195,
      "activations/layer14_attention_weight_min": -30.860572814941406,
      "activations/layer15_attention_weight_max": 33.129791259765625,
      "activations/layer15_attention_weight_min": -30.917343139648438,
      "activations/layer16_attention_weight_max": 34.91291427612305,
      "activations/layer16_attention_weight_min": -33.397666931152344,
      "activations/layer17_attention_weight_max": 52.36440658569336,
      "activations/layer17_attention_weight_min": -46.854530334472656,
      "activations/layer18_attention_weight_max": 49.27070617675781,
      "activations/layer18_attention_weight_min": -40.177860260009766,
      "activations/layer19_attention_weight_max": 23.87613868713379,
      "activations/layer19_attention_weight_min": -23.012880325317383,
      "activations/layer1_attention_weight_max": 15.850942611694336,
      "activations/layer1_attention_weight_min": -14.558066368103027,
      "activations/layer20_attention_weight_max": 22.249156951904297,
      "activations/layer20_attention_weight_min": -24.6418399810791,
      "activations/layer21_attention_weight_max": 39.06916427612305,
      "activations/layer21_attention_weight_min": -21.78460693359375,
      "activations/layer22_attention_weight_max": 30.827836990356445,
      "activations/layer22_attention_weight_min": -26.61533546447754,
      "activations/layer23_attention_weight_max": 36.931251525878906,
      "activations/layer23_attention_weight_min": -21.63258171081543,
      "activations/layer2_attention_weight_max": 33.07615280151367,
      "activations/layer2_attention_weight_min": -30.722248077392578,
      "activations/layer3_attention_weight_max": 90.19139099121094,
      "activations/layer3_attention_weight_min": -92.29757690429688,
      "activations/layer4_attention_weight_max": 84.97278594970703,
      "activations/layer4_attention_weight_min": -82.73766326904297,
      "activations/layer5_attention_weight_max": 66.20611572265625,
      "activations/layer5_attention_weight_min": -67.2345199584961,
      "activations/layer6_attention_weight_max": 50.92899703979492,
      "activations/layer6_attention_weight_min": -52.689857482910156,
      "activations/layer7_attention_weight_max": 68.82260131835938,
      "activations/layer7_attention_weight_min": -71.96492004394531,
      "activations/layer8_attention_weight_max": 48.16051483154297,
      "activations/layer8_attention_weight_min": -51.73942565917969,
      "activations/layer9_attention_weight_max": 46.06359100341797,
      "activations/layer9_attention_weight_min": -47.13303756713867,
      "epoch": 22.68,
      "learning_rate": 3.7727272727272724e-06,
      "loss": 2.7107,
      "step": 390350
    },
    {
      "activations/layer0_attention_weight_max": 15.310205459594727,
      "activations/layer0_attention_weight_min": -13.769756317138672,
      "activations/layer10_attention_weight_max": 35.33576583862305,
      "activations/layer10_attention_weight_min": -35.52437210083008,
      "activations/layer11_attention_weight_max": 34.83580017089844,
      "activations/layer11_attention_weight_min": -36.62068176269531,
      "activations/layer12_attention_weight_max": 19.61138916015625,
      "activations/layer12_attention_weight_min": -30.97905921936035,
      "activations/layer13_attention_weight_max": 32.035308837890625,
      "activations/layer13_attention_weight_min": -25.37557029724121,
      "activations/layer14_attention_weight_max": 36.30712890625,
      "activations/layer14_attention_weight_min": -29.694828033447266,
      "activations/layer15_attention_weight_max": 36.446815490722656,
      "activations/layer15_attention_weight_min": -31.351394653320312,
      "activations/layer16_attention_weight_max": 34.22311782836914,
      "activations/layer16_attention_weight_min": -33.1207275390625,
      "activations/layer17_attention_weight_max": 50.2447395324707,
      "activations/layer17_attention_weight_min": -46.67605972290039,
      "activations/layer18_attention_weight_max": 45.96654510498047,
      "activations/layer18_attention_weight_min": -42.88270950317383,
      "activations/layer19_attention_weight_max": 25.88604736328125,
      "activations/layer19_attention_weight_min": -24.845439910888672,
      "activations/layer1_attention_weight_max": 15.40657901763916,
      "activations/layer1_attention_weight_min": -14.133895874023438,
      "activations/layer20_attention_weight_max": 23.288982391357422,
      "activations/layer20_attention_weight_min": -22.849855422973633,
      "activations/layer21_attention_weight_max": 36.42217254638672,
      "activations/layer21_attention_weight_min": -22.376232147216797,
      "activations/layer22_attention_weight_max": 28.96291160583496,
      "activations/layer22_attention_weight_min": -24.35371971130371,
      "activations/layer23_attention_weight_max": 36.31400680541992,
      "activations/layer23_attention_weight_min": -22.26569175720215,
      "activations/layer2_attention_weight_max": 33.93942642211914,
      "activations/layer2_attention_weight_min": -31.60879135131836,
      "activations/layer3_attention_weight_max": 87.28623962402344,
      "activations/layer3_attention_weight_min": -89.98291778564453,
      "activations/layer4_attention_weight_max": 85.36165618896484,
      "activations/layer4_attention_weight_min": -87.60540771484375,
      "activations/layer5_attention_weight_max": 62.01115036010742,
      "activations/layer5_attention_weight_min": -73.16075134277344,
      "activations/layer6_attention_weight_max": 50.452369689941406,
      "activations/layer6_attention_weight_min": -51.411624908447266,
      "activations/layer7_attention_weight_max": 72.83946228027344,
      "activations/layer7_attention_weight_min": -68.1272201538086,
      "activations/layer8_attention_weight_max": 48.935611724853516,
      "activations/layer8_attention_weight_min": -48.78394317626953,
      "activations/layer9_attention_weight_max": 46.180824279785156,
      "activations/layer9_attention_weight_min": -47.13233947753906,
      "epoch": 22.68,
      "learning_rate": 3.7537878787878785e-06,
      "loss": 2.6998,
      "step": 390400
    },
    {
      "activations/layer0_attention_weight_max": 15.174712181091309,
      "activations/layer0_attention_weight_min": -14.184060096740723,
      "activations/layer10_attention_weight_max": 39.19036865234375,
      "activations/layer10_attention_weight_min": -37.807411193847656,
      "activations/layer11_attention_weight_max": 36.07537841796875,
      "activations/layer11_attention_weight_min": -36.00905990600586,
      "activations/layer12_attention_weight_max": 20.4647159576416,
      "activations/layer12_attention_weight_min": -26.532180786132812,
      "activations/layer13_attention_weight_max": 38.23595428466797,
      "activations/layer13_attention_weight_min": -29.722524642944336,
      "activations/layer14_attention_weight_max": 38.963050842285156,
      "activations/layer14_attention_weight_min": -34.13807678222656,
      "activations/layer15_attention_weight_max": 40.341617584228516,
      "activations/layer15_attention_weight_min": -33.1795768737793,
      "activations/layer16_attention_weight_max": 39.43564224243164,
      "activations/layer16_attention_weight_min": -37.01417541503906,
      "activations/layer17_attention_weight_max": 59.57452392578125,
      "activations/layer17_attention_weight_min": -52.22852325439453,
      "activations/layer18_attention_weight_max": 56.58203887939453,
      "activations/layer18_attention_weight_min": -45.1728630065918,
      "activations/layer19_attention_weight_max": 26.515798568725586,
      "activations/layer19_attention_weight_min": -22.68887710571289,
      "activations/layer1_attention_weight_max": 15.368141174316406,
      "activations/layer1_attention_weight_min": -13.937819480895996,
      "activations/layer20_attention_weight_max": 25.391145706176758,
      "activations/layer20_attention_weight_min": -21.281085968017578,
      "activations/layer21_attention_weight_max": 38.725685119628906,
      "activations/layer21_attention_weight_min": -23.433975219726562,
      "activations/layer22_attention_weight_max": 32.37334442138672,
      "activations/layer22_attention_weight_min": -23.927114486694336,
      "activations/layer23_attention_weight_max": 37.147499084472656,
      "activations/layer23_attention_weight_min": -24.60036849975586,
      "activations/layer2_attention_weight_max": 32.9880485534668,
      "activations/layer2_attention_weight_min": -31.43625259399414,
      "activations/layer3_attention_weight_max": 94.46963500976562,
      "activations/layer3_attention_weight_min": -92.7015609741211,
      "activations/layer4_attention_weight_max": 91.38887786865234,
      "activations/layer4_attention_weight_min": -88.06818389892578,
      "activations/layer5_attention_weight_max": 65.54095458984375,
      "activations/layer5_attention_weight_min": -71.48068237304688,
      "activations/layer6_attention_weight_max": 52.083778381347656,
      "activations/layer6_attention_weight_min": -53.58189010620117,
      "activations/layer7_attention_weight_max": 72.49275970458984,
      "activations/layer7_attention_weight_min": -73.33741760253906,
      "activations/layer8_attention_weight_max": 52.85457229614258,
      "activations/layer8_attention_weight_min": -54.09544372558594,
      "activations/layer9_attention_weight_max": 45.8778190612793,
      "activations/layer9_attention_weight_min": -48.451358795166016,
      "epoch": 22.69,
      "learning_rate": 3.7348484848484847e-06,
      "loss": 2.7083,
      "step": 390450
    },
    {
      "activations/layer0_attention_weight_max": 15.154391288757324,
      "activations/layer0_attention_weight_min": -14.161480903625488,
      "activations/layer10_attention_weight_max": 36.7974853515625,
      "activations/layer10_attention_weight_min": -35.789283752441406,
      "activations/layer11_attention_weight_max": 33.009281158447266,
      "activations/layer11_attention_weight_min": -34.54225158691406,
      "activations/layer12_attention_weight_max": 18.420429229736328,
      "activations/layer12_attention_weight_min": -25.447952270507812,
      "activations/layer13_attention_weight_max": 33.902809143066406,
      "activations/layer13_attention_weight_min": -29.326528549194336,
      "activations/layer14_attention_weight_max": 33.346797943115234,
      "activations/layer14_attention_weight_min": -29.193220138549805,
      "activations/layer15_attention_weight_max": 32.53323745727539,
      "activations/layer15_attention_weight_min": -28.558120727539062,
      "activations/layer16_attention_weight_max": 33.53091812133789,
      "activations/layer16_attention_weight_min": -31.78790283203125,
      "activations/layer17_attention_weight_max": 47.7071533203125,
      "activations/layer17_attention_weight_min": -41.57722473144531,
      "activations/layer18_attention_weight_max": 45.29011154174805,
      "activations/layer18_attention_weight_min": -37.225555419921875,
      "activations/layer19_attention_weight_max": 24.388282775878906,
      "activations/layer19_attention_weight_min": -20.62129020690918,
      "activations/layer1_attention_weight_max": 15.172680854797363,
      "activations/layer1_attention_weight_min": -14.122173309326172,
      "activations/layer20_attention_weight_max": 20.95513343811035,
      "activations/layer20_attention_weight_min": -20.422422409057617,
      "activations/layer21_attention_weight_max": 32.894371032714844,
      "activations/layer21_attention_weight_min": -22.856870651245117,
      "activations/layer22_attention_weight_max": 27.23063087463379,
      "activations/layer22_attention_weight_min": -24.44923973083496,
      "activations/layer23_attention_weight_max": 37.43817138671875,
      "activations/layer23_attention_weight_min": -26.360244750976562,
      "activations/layer2_attention_weight_max": 31.34286880493164,
      "activations/layer2_attention_weight_min": -29.28615379333496,
      "activations/layer3_attention_weight_max": 91.03071594238281,
      "activations/layer3_attention_weight_min": -90.20748138427734,
      "activations/layer4_attention_weight_max": 85.83367156982422,
      "activations/layer4_attention_weight_min": -86.46427154541016,
      "activations/layer5_attention_weight_max": 61.56681823730469,
      "activations/layer5_attention_weight_min": -74.75733184814453,
      "activations/layer6_attention_weight_max": 49.68669128417969,
      "activations/layer6_attention_weight_min": -52.45747756958008,
      "activations/layer7_attention_weight_max": 72.17247009277344,
      "activations/layer7_attention_weight_min": -68.39585876464844,
      "activations/layer8_attention_weight_max": 48.24771499633789,
      "activations/layer8_attention_weight_min": -54.00923156738281,
      "activations/layer9_attention_weight_max": 45.4524040222168,
      "activations/layer9_attention_weight_min": -47.5646858215332,
      "epoch": 22.69,
      "learning_rate": 3.7159090909090904e-06,
      "loss": 2.7074,
      "step": 390500
    },
    {
      "activations/layer0_attention_weight_max": 16.059181213378906,
      "activations/layer0_attention_weight_min": -14.261491775512695,
      "activations/layer10_attention_weight_max": 35.837589263916016,
      "activations/layer10_attention_weight_min": -37.099449157714844,
      "activations/layer11_attention_weight_max": 32.58642578125,
      "activations/layer11_attention_weight_min": -33.831390380859375,
      "activations/layer12_attention_weight_max": 19.729156494140625,
      "activations/layer12_attention_weight_min": -24.652820587158203,
      "activations/layer13_attention_weight_max": 33.8010139465332,
      "activations/layer13_attention_weight_min": -27.7535343170166,
      "activations/layer14_attention_weight_max": 35.433250427246094,
      "activations/layer14_attention_weight_min": -29.62805938720703,
      "activations/layer15_attention_weight_max": 33.992733001708984,
      "activations/layer15_attention_weight_min": -30.911075592041016,
      "activations/layer16_attention_weight_max": 35.076988220214844,
      "activations/layer16_attention_weight_min": -33.2619514465332,
      "activations/layer17_attention_weight_max": 52.263065338134766,
      "activations/layer17_attention_weight_min": -48.69951629638672,
      "activations/layer18_attention_weight_max": 47.01590347290039,
      "activations/layer18_attention_weight_min": -41.77134323120117,
      "activations/layer19_attention_weight_max": 23.42218780517578,
      "activations/layer19_attention_weight_min": -24.57771873474121,
      "activations/layer1_attention_weight_max": 15.677437782287598,
      "activations/layer1_attention_weight_min": -14.533851623535156,
      "activations/layer20_attention_weight_max": 23.41912078857422,
      "activations/layer20_attention_weight_min": -22.195253372192383,
      "activations/layer21_attention_weight_max": 39.944000244140625,
      "activations/layer21_attention_weight_min": -21.78236198425293,
      "activations/layer22_attention_weight_max": 28.97127914428711,
      "activations/layer22_attention_weight_min": -25.992063522338867,
      "activations/layer23_attention_weight_max": 33.14276123046875,
      "activations/layer23_attention_weight_min": -22.955665588378906,
      "activations/layer2_attention_weight_max": 34.00666046142578,
      "activations/layer2_attention_weight_min": -32.184967041015625,
      "activations/layer3_attention_weight_max": 92.4626235961914,
      "activations/layer3_attention_weight_min": -97.33806610107422,
      "activations/layer4_attention_weight_max": 85.38591766357422,
      "activations/layer4_attention_weight_min": -84.47085571289062,
      "activations/layer5_attention_weight_max": 60.063106536865234,
      "activations/layer5_attention_weight_min": -72.79420471191406,
      "activations/layer6_attention_weight_max": 47.143821716308594,
      "activations/layer6_attention_weight_min": -50.93025588989258,
      "activations/layer7_attention_weight_max": 68.27884674072266,
      "activations/layer7_attention_weight_min": -68.18666076660156,
      "activations/layer8_attention_weight_max": 44.97700500488281,
      "activations/layer8_attention_weight_min": -51.1268196105957,
      "activations/layer9_attention_weight_max": 49.85860061645508,
      "activations/layer9_attention_weight_min": -45.021522521972656,
      "epoch": 22.69,
      "learning_rate": 3.6969696969696966e-06,
      "loss": 2.7135,
      "step": 390550
    },
    {
      "activations/layer0_attention_weight_max": 15.850214004516602,
      "activations/layer0_attention_weight_min": -14.443225860595703,
      "activations/layer10_attention_weight_max": 44.88399887084961,
      "activations/layer10_attention_weight_min": -45.041221618652344,
      "activations/layer11_attention_weight_max": 43.08815002441406,
      "activations/layer11_attention_weight_min": -40.342491149902344,
      "activations/layer12_attention_weight_max": 19.74514389038086,
      "activations/layer12_attention_weight_min": -27.816205978393555,
      "activations/layer13_attention_weight_max": 46.18828582763672,
      "activations/layer13_attention_weight_min": -29.656497955322266,
      "activations/layer14_attention_weight_max": 46.375282287597656,
      "activations/layer14_attention_weight_min": -32.47491455078125,
      "activations/layer15_attention_weight_max": 46.670562744140625,
      "activations/layer15_attention_weight_min": -33.72871017456055,
      "activations/layer16_attention_weight_max": 41.458560943603516,
      "activations/layer16_attention_weight_min": -38.5506706237793,
      "activations/layer17_attention_weight_max": 58.70995330810547,
      "activations/layer17_attention_weight_min": -54.25205612182617,
      "activations/layer18_attention_weight_max": 51.475276947021484,
      "activations/layer18_attention_weight_min": -50.06189727783203,
      "activations/layer19_attention_weight_max": 24.984281539916992,
      "activations/layer19_attention_weight_min": -25.34465217590332,
      "activations/layer1_attention_weight_max": 15.180536270141602,
      "activations/layer1_attention_weight_min": -14.09583854675293,
      "activations/layer20_attention_weight_max": 24.275768280029297,
      "activations/layer20_attention_weight_min": -23.12542724609375,
      "activations/layer21_attention_weight_max": 36.448692321777344,
      "activations/layer21_attention_weight_min": -23.023263931274414,
      "activations/layer22_attention_weight_max": 29.45382308959961,
      "activations/layer22_attention_weight_min": -26.42479705810547,
      "activations/layer23_attention_weight_max": 35.5758056640625,
      "activations/layer23_attention_weight_min": -22.049909591674805,
      "activations/layer2_attention_weight_max": 34.20573425292969,
      "activations/layer2_attention_weight_min": -32.211769104003906,
      "activations/layer3_attention_weight_max": 92.4740219116211,
      "activations/layer3_attention_weight_min": -92.32964324951172,
      "activations/layer4_attention_weight_max": 89.62648010253906,
      "activations/layer4_attention_weight_min": -87.26962280273438,
      "activations/layer5_attention_weight_max": 64.44935607910156,
      "activations/layer5_attention_weight_min": -72.95545959472656,
      "activations/layer6_attention_weight_max": 52.67226791381836,
      "activations/layer6_attention_weight_min": -54.33247375488281,
      "activations/layer7_attention_weight_max": 75.9378433227539,
      "activations/layer7_attention_weight_min": -73.00504302978516,
      "activations/layer8_attention_weight_max": 57.75550079345703,
      "activations/layer8_attention_weight_min": -54.46535110473633,
      "activations/layer9_attention_weight_max": 57.9674072265625,
      "activations/layer9_attention_weight_min": -55.73724365234375,
      "epoch": 22.7,
      "learning_rate": 3.6780303030303023e-06,
      "loss": 2.7064,
      "step": 390600
    },
    {
      "activations/layer0_attention_weight_max": 15.202699661254883,
      "activations/layer0_attention_weight_min": -14.196674346923828,
      "activations/layer10_attention_weight_max": 39.46292495727539,
      "activations/layer10_attention_weight_min": -38.558990478515625,
      "activations/layer11_attention_weight_max": 36.610633850097656,
      "activations/layer11_attention_weight_min": -37.91297149658203,
      "activations/layer12_attention_weight_max": 19.85831069946289,
      "activations/layer12_attention_weight_min": -27.55177116394043,
      "activations/layer13_attention_weight_max": 38.21784591674805,
      "activations/layer13_attention_weight_min": -27.528888702392578,
      "activations/layer14_attention_weight_max": 37.116050720214844,
      "activations/layer14_attention_weight_min": -30.455820083618164,
      "activations/layer15_attention_weight_max": 33.16788101196289,
      "activations/layer15_attention_weight_min": -29.52959442138672,
      "activations/layer16_attention_weight_max": 35.484474182128906,
      "activations/layer16_attention_weight_min": -33.29762649536133,
      "activations/layer17_attention_weight_max": 55.62312316894531,
      "activations/layer17_attention_weight_min": -44.78590393066406,
      "activations/layer18_attention_weight_max": 49.96352767944336,
      "activations/layer18_attention_weight_min": -41.81715774536133,
      "activations/layer19_attention_weight_max": 28.389892578125,
      "activations/layer19_attention_weight_min": -23.62397003173828,
      "activations/layer1_attention_weight_max": 15.178889274597168,
      "activations/layer1_attention_weight_min": -14.461670875549316,
      "activations/layer20_attention_weight_max": 25.595857620239258,
      "activations/layer20_attention_weight_min": -23.785844802856445,
      "activations/layer21_attention_weight_max": 38.54681396484375,
      "activations/layer21_attention_weight_min": -20.960268020629883,
      "activations/layer22_attention_weight_max": 31.447826385498047,
      "activations/layer22_attention_weight_min": -24.60700798034668,
      "activations/layer23_attention_weight_max": 36.37793731689453,
      "activations/layer23_attention_weight_min": -24.2607421875,
      "activations/layer2_attention_weight_max": 32.69468688964844,
      "activations/layer2_attention_weight_min": -31.738216400146484,
      "activations/layer3_attention_weight_max": 93.94001007080078,
      "activations/layer3_attention_weight_min": -95.93376159667969,
      "activations/layer4_attention_weight_max": 86.74056243896484,
      "activations/layer4_attention_weight_min": -88.65470123291016,
      "activations/layer5_attention_weight_max": 65.10257720947266,
      "activations/layer5_attention_weight_min": -71.52033233642578,
      "activations/layer6_attention_weight_max": 52.01081848144531,
      "activations/layer6_attention_weight_min": -51.94627380371094,
      "activations/layer7_attention_weight_max": 72.53602600097656,
      "activations/layer7_attention_weight_min": -70.43666076660156,
      "activations/layer8_attention_weight_max": 50.72298049926758,
      "activations/layer8_attention_weight_min": -51.3929557800293,
      "activations/layer9_attention_weight_max": 46.48502731323242,
      "activations/layer9_attention_weight_min": -47.34851837158203,
      "epoch": 22.7,
      "learning_rate": 3.659090909090909e-06,
      "loss": 2.7026,
      "step": 390650
    },
    {
      "activations/layer0_attention_weight_max": 15.9480562210083,
      "activations/layer0_attention_weight_min": -14.334322929382324,
      "activations/layer10_attention_weight_max": 34.952110290527344,
      "activations/layer10_attention_weight_min": -34.9264030456543,
      "activations/layer11_attention_weight_max": 33.01313018798828,
      "activations/layer11_attention_weight_min": -33.94696044921875,
      "activations/layer12_attention_weight_max": 18.84417152404785,
      "activations/layer12_attention_weight_min": -25.5045223236084,
      "activations/layer13_attention_weight_max": 35.95703887939453,
      "activations/layer13_attention_weight_min": -26.476701736450195,
      "activations/layer14_attention_weight_max": 40.120662689208984,
      "activations/layer14_attention_weight_min": -31.995546340942383,
      "activations/layer15_attention_weight_max": 36.85264205932617,
      "activations/layer15_attention_weight_min": -32.51384353637695,
      "activations/layer16_attention_weight_max": 41.168922424316406,
      "activations/layer16_attention_weight_min": -37.77910232543945,
      "activations/layer17_attention_weight_max": 63.25469970703125,
      "activations/layer17_attention_weight_min": -56.28034973144531,
      "activations/layer18_attention_weight_max": 57.6058349609375,
      "activations/layer18_attention_weight_min": -47.86148452758789,
      "activations/layer19_attention_weight_max": 28.792831420898438,
      "activations/layer19_attention_weight_min": -28.179813385009766,
      "activations/layer1_attention_weight_max": 15.223214149475098,
      "activations/layer1_attention_weight_min": -16.093721389770508,
      "activations/layer20_attention_weight_max": 27.468364715576172,
      "activations/layer20_attention_weight_min": -21.98045539855957,
      "activations/layer21_attention_weight_max": 38.70528793334961,
      "activations/layer21_attention_weight_min": -25.015647888183594,
      "activations/layer22_attention_weight_max": 32.92427062988281,
      "activations/layer22_attention_weight_min": -26.54806900024414,
      "activations/layer23_attention_weight_max": 33.35990524291992,
      "activations/layer23_attention_weight_min": -22.999435424804688,
      "activations/layer2_attention_weight_max": 31.936803817749023,
      "activations/layer2_attention_weight_min": -31.824871063232422,
      "activations/layer3_attention_weight_max": 89.78298950195312,
      "activations/layer3_attention_weight_min": -95.11781311035156,
      "activations/layer4_attention_weight_max": 87.46424102783203,
      "activations/layer4_attention_weight_min": -89.66411590576172,
      "activations/layer5_attention_weight_max": 65.23612213134766,
      "activations/layer5_attention_weight_min": -73.1082763671875,
      "activations/layer6_attention_weight_max": 51.10896301269531,
      "activations/layer6_attention_weight_min": -51.74989700317383,
      "activations/layer7_attention_weight_max": 70.06472778320312,
      "activations/layer7_attention_weight_min": -71.67354583740234,
      "activations/layer8_attention_weight_max": 47.5839729309082,
      "activations/layer8_attention_weight_min": -50.22690200805664,
      "activations/layer9_attention_weight_max": 43.032066345214844,
      "activations/layer9_attention_weight_min": -46.62034606933594,
      "epoch": 22.7,
      "learning_rate": 3.640151515151515e-06,
      "loss": 2.6938,
      "step": 390700
    },
    {
      "activations/layer0_attention_weight_max": 16.32654571533203,
      "activations/layer0_attention_weight_min": -14.27297306060791,
      "activations/layer10_attention_weight_max": 36.681312561035156,
      "activations/layer10_attention_weight_min": -38.009193420410156,
      "activations/layer11_attention_weight_max": 35.32491683959961,
      "activations/layer11_attention_weight_min": -37.916080474853516,
      "activations/layer12_attention_weight_max": 20.18301773071289,
      "activations/layer12_attention_weight_min": -23.454381942749023,
      "activations/layer13_attention_weight_max": 40.911781311035156,
      "activations/layer13_attention_weight_min": -27.139484405517578,
      "activations/layer14_attention_weight_max": 38.697513580322266,
      "activations/layer14_attention_weight_min": -30.580318450927734,
      "activations/layer15_attention_weight_max": 35.660606384277344,
      "activations/layer15_attention_weight_min": -32.86885070800781,
      "activations/layer16_attention_weight_max": 38.18379592895508,
      "activations/layer16_attention_weight_min": -34.79788589477539,
      "activations/layer17_attention_weight_max": 57.24544143676758,
      "activations/layer17_attention_weight_min": -48.18235778808594,
      "activations/layer18_attention_weight_max": 55.05121994018555,
      "activations/layer18_attention_weight_min": -43.27473068237305,
      "activations/layer19_attention_weight_max": 28.164325714111328,
      "activations/layer19_attention_weight_min": -24.119632720947266,
      "activations/layer1_attention_weight_max": 15.49382495880127,
      "activations/layer1_attention_weight_min": -15.973262786865234,
      "activations/layer20_attention_weight_max": 29.129322052001953,
      "activations/layer20_attention_weight_min": -25.50655174255371,
      "activations/layer21_attention_weight_max": 40.8993034362793,
      "activations/layer21_attention_weight_min": -21.164379119873047,
      "activations/layer22_attention_weight_max": 30.886775970458984,
      "activations/layer22_attention_weight_min": -27.35814666748047,
      "activations/layer23_attention_weight_max": 37.49370574951172,
      "activations/layer23_attention_weight_min": -23.413589477539062,
      "activations/layer2_attention_weight_max": 33.023494720458984,
      "activations/layer2_attention_weight_min": -32.209503173828125,
      "activations/layer3_attention_weight_max": 89.5009536743164,
      "activations/layer3_attention_weight_min": -94.27565002441406,
      "activations/layer4_attention_weight_max": 89.63041687011719,
      "activations/layer4_attention_weight_min": -85.4478530883789,
      "activations/layer5_attention_weight_max": 60.63779830932617,
      "activations/layer5_attention_weight_min": -74.67105865478516,
      "activations/layer6_attention_weight_max": 50.87943649291992,
      "activations/layer6_attention_weight_min": -52.53138732910156,
      "activations/layer7_attention_weight_max": 72.18690490722656,
      "activations/layer7_attention_weight_min": -70.96559143066406,
      "activations/layer8_attention_weight_max": 49.907318115234375,
      "activations/layer8_attention_weight_min": -54.38499069213867,
      "activations/layer9_attention_weight_max": 45.47782516479492,
      "activations/layer9_attention_weight_min": -48.81566619873047,
      "epoch": 22.7,
      "learning_rate": 3.6212121212121207e-06,
      "loss": 2.7113,
      "step": 390750
    },
    {
      "activations/layer0_attention_weight_max": 15.101162910461426,
      "activations/layer0_attention_weight_min": -14.439359664916992,
      "activations/layer10_attention_weight_max": 42.412105560302734,
      "activations/layer10_attention_weight_min": -41.45402145385742,
      "activations/layer11_attention_weight_max": 43.62147521972656,
      "activations/layer11_attention_weight_min": -39.87433624267578,
      "activations/layer12_attention_weight_max": 20.25514793395996,
      "activations/layer12_attention_weight_min": -24.320510864257812,
      "activations/layer13_attention_weight_max": 36.88393020629883,
      "activations/layer13_attention_weight_min": -29.382957458496094,
      "activations/layer14_attention_weight_max": 40.88056564331055,
      "activations/layer14_attention_weight_min": -32.19202423095703,
      "activations/layer15_attention_weight_max": 43.24628448486328,
      "activations/layer15_attention_weight_min": -32.12449264526367,
      "activations/layer16_attention_weight_max": 36.34068298339844,
      "activations/layer16_attention_weight_min": -35.260440826416016,
      "activations/layer17_attention_weight_max": 55.92974853515625,
      "activations/layer17_attention_weight_min": -48.22723388671875,
      "activations/layer18_attention_weight_max": 50.37208938598633,
      "activations/layer18_attention_weight_min": -44.290313720703125,
      "activations/layer19_attention_weight_max": 27.60500144958496,
      "activations/layer19_attention_weight_min": -24.829509735107422,
      "activations/layer1_attention_weight_max": 15.3699312210083,
      "activations/layer1_attention_weight_min": -13.902389526367188,
      "activations/layer20_attention_weight_max": 24.431650161743164,
      "activations/layer20_attention_weight_min": -23.124135971069336,
      "activations/layer21_attention_weight_max": 35.38349151611328,
      "activations/layer21_attention_weight_min": -23.10593605041504,
      "activations/layer22_attention_weight_max": 32.094749450683594,
      "activations/layer22_attention_weight_min": -27.061975479125977,
      "activations/layer23_attention_weight_max": 36.96874237060547,
      "activations/layer23_attention_weight_min": -24.344505310058594,
      "activations/layer2_attention_weight_max": 32.77289962768555,
      "activations/layer2_attention_weight_min": -31.61389923095703,
      "activations/layer3_attention_weight_max": 89.75468444824219,
      "activations/layer3_attention_weight_min": -91.978759765625,
      "activations/layer4_attention_weight_max": 85.13507843017578,
      "activations/layer4_attention_weight_min": -87.6753921508789,
      "activations/layer5_attention_weight_max": 65.22596740722656,
      "activations/layer5_attention_weight_min": -73.5237808227539,
      "activations/layer6_attention_weight_max": 49.97002410888672,
      "activations/layer6_attention_weight_min": -50.904014587402344,
      "activations/layer7_attention_weight_max": 85.49688720703125,
      "activations/layer7_attention_weight_min": -78.11724853515625,
      "activations/layer8_attention_weight_max": 56.00074768066406,
      "activations/layer8_attention_weight_min": -55.9815559387207,
      "activations/layer9_attention_weight_max": 58.91389846801758,
      "activations/layer9_attention_weight_min": -53.80202102661133,
      "epoch": 22.71,
      "learning_rate": 3.602272727272727e-06,
      "loss": 2.7031,
      "step": 390800
    },
    {
      "activations/layer0_attention_weight_max": 16.14311981201172,
      "activations/layer0_attention_weight_min": -14.2340087890625,
      "activations/layer10_attention_weight_max": 34.8486328125,
      "activations/layer10_attention_weight_min": -35.0518798828125,
      "activations/layer11_attention_weight_max": 32.33674240112305,
      "activations/layer11_attention_weight_min": -33.65732955932617,
      "activations/layer12_attention_weight_max": 21.59821128845215,
      "activations/layer12_attention_weight_min": -24.661949157714844,
      "activations/layer13_attention_weight_max": 34.265010833740234,
      "activations/layer13_attention_weight_min": -26.61902618408203,
      "activations/layer14_attention_weight_max": 40.85857391357422,
      "activations/layer14_attention_weight_min": -30.787250518798828,
      "activations/layer15_attention_weight_max": 34.94119644165039,
      "activations/layer15_attention_weight_min": -31.040935516357422,
      "activations/layer16_attention_weight_max": 37.42634963989258,
      "activations/layer16_attention_weight_min": -34.38786697387695,
      "activations/layer17_attention_weight_max": 56.62188720703125,
      "activations/layer17_attention_weight_min": -48.015167236328125,
      "activations/layer18_attention_weight_max": 51.867549896240234,
      "activations/layer18_attention_weight_min": -46.40595626831055,
      "activations/layer19_attention_weight_max": 26.746503829956055,
      "activations/layer19_attention_weight_min": -26.465978622436523,
      "activations/layer1_attention_weight_max": 15.011467933654785,
      "activations/layer1_attention_weight_min": -14.938203811645508,
      "activations/layer20_attention_weight_max": 26.120527267456055,
      "activations/layer20_attention_weight_min": -22.643672943115234,
      "activations/layer21_attention_weight_max": 37.85822296142578,
      "activations/layer21_attention_weight_min": -22.34010887145996,
      "activations/layer22_attention_weight_max": 32.03025436401367,
      "activations/layer22_attention_weight_min": -25.245433807373047,
      "activations/layer23_attention_weight_max": 36.230934143066406,
      "activations/layer23_attention_weight_min": -24.72054100036621,
      "activations/layer2_attention_weight_max": 32.67509460449219,
      "activations/layer2_attention_weight_min": -32.082462310791016,
      "activations/layer3_attention_weight_max": 90.62619018554688,
      "activations/layer3_attention_weight_min": -92.7433090209961,
      "activations/layer4_attention_weight_max": 82.03475189208984,
      "activations/layer4_attention_weight_min": -83.8166732788086,
      "activations/layer5_attention_weight_max": 61.01789474487305,
      "activations/layer5_attention_weight_min": -73.85234069824219,
      "activations/layer6_attention_weight_max": 49.558448791503906,
      "activations/layer6_attention_weight_min": -51.20772171020508,
      "activations/layer7_attention_weight_max": 67.5866928100586,
      "activations/layer7_attention_weight_min": -66.62980651855469,
      "activations/layer8_attention_weight_max": 45.773738861083984,
      "activations/layer8_attention_weight_min": -49.24123001098633,
      "activations/layer9_attention_weight_max": 42.4366340637207,
      "activations/layer9_attention_weight_min": -43.92977523803711,
      "epoch": 22.71,
      "learning_rate": 3.583333333333333e-06,
      "loss": 2.6979,
      "step": 390850
    },
    {
      "activations/layer0_attention_weight_max": 15.761813163757324,
      "activations/layer0_attention_weight_min": -14.53354549407959,
      "activations/layer10_attention_weight_max": 43.725101470947266,
      "activations/layer10_attention_weight_min": -44.252220153808594,
      "activations/layer11_attention_weight_max": 40.45286560058594,
      "activations/layer11_attention_weight_min": -42.06196212768555,
      "activations/layer12_attention_weight_max": 20.712032318115234,
      "activations/layer12_attention_weight_min": -24.419599533081055,
      "activations/layer13_attention_weight_max": 34.98076629638672,
      "activations/layer13_attention_weight_min": -29.090576171875,
      "activations/layer14_attention_weight_max": 36.243408203125,
      "activations/layer14_attention_weight_min": -30.73877716064453,
      "activations/layer15_attention_weight_max": 34.122196197509766,
      "activations/layer15_attention_weight_min": -30.724021911621094,
      "activations/layer16_attention_weight_max": 35.74354553222656,
      "activations/layer16_attention_weight_min": -34.0821418762207,
      "activations/layer17_attention_weight_max": 52.97990417480469,
      "activations/layer17_attention_weight_min": -51.30495834350586,
      "activations/layer18_attention_weight_max": 47.88347625732422,
      "activations/layer18_attention_weight_min": -45.276390075683594,
      "activations/layer19_attention_weight_max": 23.812969207763672,
      "activations/layer19_attention_weight_min": -23.160314559936523,
      "activations/layer1_attention_weight_max": 15.89185905456543,
      "activations/layer1_attention_weight_min": -15.515473365783691,
      "activations/layer20_attention_weight_max": 21.969348907470703,
      "activations/layer20_attention_weight_min": -22.281383514404297,
      "activations/layer21_attention_weight_max": 31.10887336730957,
      "activations/layer21_attention_weight_min": -20.05808448791504,
      "activations/layer22_attention_weight_max": 31.14192008972168,
      "activations/layer22_attention_weight_min": -25.514089584350586,
      "activations/layer23_attention_weight_max": 31.980260848999023,
      "activations/layer23_attention_weight_min": -23.497047424316406,
      "activations/layer2_attention_weight_max": 32.208099365234375,
      "activations/layer2_attention_weight_min": -30.872093200683594,
      "activations/layer3_attention_weight_max": 91.08934020996094,
      "activations/layer3_attention_weight_min": -88.8941421508789,
      "activations/layer4_attention_weight_max": 88.5276107788086,
      "activations/layer4_attention_weight_min": -87.33704376220703,
      "activations/layer5_attention_weight_max": 64.13720703125,
      "activations/layer5_attention_weight_min": -69.27210235595703,
      "activations/layer6_attention_weight_max": 51.511348724365234,
      "activations/layer6_attention_weight_min": -52.284786224365234,
      "activations/layer7_attention_weight_max": 72.4795913696289,
      "activations/layer7_attention_weight_min": -70.9100341796875,
      "activations/layer8_attention_weight_max": 50.662818908691406,
      "activations/layer8_attention_weight_min": -51.76021194458008,
      "activations/layer9_attention_weight_max": 56.38936233520508,
      "activations/layer9_attention_weight_min": -50.2515983581543,
      "epoch": 22.71,
      "learning_rate": 3.564393939393939e-06,
      "loss": 2.7002,
      "step": 390900
    },
    {
      "activations/layer0_attention_weight_max": 16.560977935791016,
      "activations/layer0_attention_weight_min": -14.31217098236084,
      "activations/layer10_attention_weight_max": 37.16577911376953,
      "activations/layer10_attention_weight_min": -37.238040924072266,
      "activations/layer11_attention_weight_max": 33.4905891418457,
      "activations/layer11_attention_weight_min": -36.74177932739258,
      "activations/layer12_attention_weight_max": 19.05531883239746,
      "activations/layer12_attention_weight_min": -29.98206329345703,
      "activations/layer13_attention_weight_max": 36.11628341674805,
      "activations/layer13_attention_weight_min": -27.13045310974121,
      "activations/layer14_attention_weight_max": 35.6008186340332,
      "activations/layer14_attention_weight_min": -31.332195281982422,
      "activations/layer15_attention_weight_max": 34.87561798095703,
      "activations/layer15_attention_weight_min": -30.417760848999023,
      "activations/layer16_attention_weight_max": 35.28981018066406,
      "activations/layer16_attention_weight_min": -34.50311279296875,
      "activations/layer17_attention_weight_max": 52.42276382446289,
      "activations/layer17_attention_weight_min": -45.9305305480957,
      "activations/layer18_attention_weight_max": 46.79281234741211,
      "activations/layer18_attention_weight_min": -41.217227935791016,
      "activations/layer19_attention_weight_max": 25.7266788482666,
      "activations/layer19_attention_weight_min": -22.402786254882812,
      "activations/layer1_attention_weight_max": 15.734755516052246,
      "activations/layer1_attention_weight_min": -13.81654167175293,
      "activations/layer20_attention_weight_max": 23.352445602416992,
      "activations/layer20_attention_weight_min": -22.61013412475586,
      "activations/layer21_attention_weight_max": 33.9156494140625,
      "activations/layer21_attention_weight_min": -19.914091110229492,
      "activations/layer22_attention_weight_max": 26.290802001953125,
      "activations/layer22_attention_weight_min": -26.772186279296875,
      "activations/layer23_attention_weight_max": 35.79454803466797,
      "activations/layer23_attention_weight_min": -25.656600952148438,
      "activations/layer2_attention_weight_max": 32.273319244384766,
      "activations/layer2_attention_weight_min": -30.795841217041016,
      "activations/layer3_attention_weight_max": 91.51516723632812,
      "activations/layer3_attention_weight_min": -92.5737075805664,
      "activations/layer4_attention_weight_max": 87.80464935302734,
      "activations/layer4_attention_weight_min": -86.19054412841797,
      "activations/layer5_attention_weight_max": 63.65654754638672,
      "activations/layer5_attention_weight_min": -73.83203125,
      "activations/layer6_attention_weight_max": 49.52803039550781,
      "activations/layer6_attention_weight_min": -54.59882354736328,
      "activations/layer7_attention_weight_max": 74.36842346191406,
      "activations/layer7_attention_weight_min": -73.50980377197266,
      "activations/layer8_attention_weight_max": 50.99619674682617,
      "activations/layer8_attention_weight_min": -55.884803771972656,
      "activations/layer9_attention_weight_max": 46.24436569213867,
      "activations/layer9_attention_weight_min": -49.78057861328125,
      "epoch": 22.72,
      "learning_rate": 3.545454545454545e-06,
      "loss": 2.6949,
      "step": 390950
    },
    {
      "activations/layer0_attention_weight_max": 15.838726043701172,
      "activations/layer0_attention_weight_min": -13.913395881652832,
      "activations/layer10_attention_weight_max": 40.71440124511719,
      "activations/layer10_attention_weight_min": -38.09889221191406,
      "activations/layer11_attention_weight_max": 36.77301788330078,
      "activations/layer11_attention_weight_min": -37.392852783203125,
      "activations/layer12_attention_weight_max": 20.43914222717285,
      "activations/layer12_attention_weight_min": -23.822771072387695,
      "activations/layer13_attention_weight_max": 38.05199432373047,
      "activations/layer13_attention_weight_min": -29.725914001464844,
      "activations/layer14_attention_weight_max": 39.21245574951172,
      "activations/layer14_attention_weight_min": -31.170854568481445,
      "activations/layer15_attention_weight_max": 39.94002914428711,
      "activations/layer15_attention_weight_min": -32.270751953125,
      "activations/layer16_attention_weight_max": 41.867767333984375,
      "activations/layer16_attention_weight_min": -37.96688461303711,
      "activations/layer17_attention_weight_max": 61.17900848388672,
      "activations/layer17_attention_weight_min": -52.8553352355957,
      "activations/layer18_attention_weight_max": 56.16816711425781,
      "activations/layer18_attention_weight_min": -42.29851531982422,
      "activations/layer19_attention_weight_max": 26.036941528320312,
      "activations/layer19_attention_weight_min": -23.95216941833496,
      "activations/layer1_attention_weight_max": 15.285045623779297,
      "activations/layer1_attention_weight_min": -14.643704414367676,
      "activations/layer20_attention_weight_max": 26.32278060913086,
      "activations/layer20_attention_weight_min": -21.319168090820312,
      "activations/layer21_attention_weight_max": 42.08495330810547,
      "activations/layer21_attention_weight_min": -23.04013442993164,
      "activations/layer22_attention_weight_max": 30.320886611938477,
      "activations/layer22_attention_weight_min": -25.103845596313477,
      "activations/layer23_attention_weight_max": 38.00177764892578,
      "activations/layer23_attention_weight_min": -24.452560424804688,
      "activations/layer2_attention_weight_max": 32.157798767089844,
      "activations/layer2_attention_weight_min": -31.499671936035156,
      "activations/layer3_attention_weight_max": 93.60844421386719,
      "activations/layer3_attention_weight_min": -94.55287170410156,
      "activations/layer4_attention_weight_max": 89.58368682861328,
      "activations/layer4_attention_weight_min": -92.3239517211914,
      "activations/layer5_attention_weight_max": 64.60345458984375,
      "activations/layer5_attention_weight_min": -76.86396789550781,
      "activations/layer6_attention_weight_max": 52.276832580566406,
      "activations/layer6_attention_weight_min": -53.07036209106445,
      "activations/layer7_attention_weight_max": 77.95597076416016,
      "activations/layer7_attention_weight_min": -72.30210876464844,
      "activations/layer8_attention_weight_max": 54.18104553222656,
      "activations/layer8_attention_weight_min": -56.25590896606445,
      "activations/layer9_attention_weight_max": 45.582427978515625,
      "activations/layer9_attention_weight_min": -50.39251708984375,
      "epoch": 22.72,
      "learning_rate": 3.526515151515151e-06,
      "loss": 2.7064,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4964,
      "eval_samples_per_second": 505.392,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4964,
      "eval_openwebtext_samples_per_second": 505.392,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9713,
      "eval_wikitext_samples_per_second": 231.324,
      "step": 391000
    },
    {
      "epoch": 22.72,
      "eval_lambada_loss": 2.423828125,
      "eval_lambada_ppl": 11.288992376773315,
      "eval_lambada_runtime": 9.5562,
      "eval_lambada_samples_per_second": 509.513,
      "step": 391000
    },
    {
      "activations/layer0_attention_weight_max": 15.038387298583984,
      "activations/layer0_attention_weight_min": -14.473176956176758,
      "activations/layer10_attention_weight_max": 34.8465461730957,
      "activations/layer10_attention_weight_min": -34.90361404418945,
      "activations/layer11_attention_weight_max": 32.81061553955078,
      "activations/layer11_attention_weight_min": -35.193267822265625,
      "activations/layer12_attention_weight_max": 20.098812103271484,
      "activations/layer12_attention_weight_min": -25.467607498168945,
      "activations/layer13_attention_weight_max": 34.37150573730469,
      "activations/layer13_attention_weight_min": -29.81426239013672,
      "activations/layer14_attention_weight_max": 36.352569580078125,
      "activations/layer14_attention_weight_min": -30.01373863220215,
      "activations/layer15_attention_weight_max": 34.22976303100586,
      "activations/layer15_attention_weight_min": -29.765113830566406,
      "activations/layer16_attention_weight_max": 35.08720397949219,
      "activations/layer16_attention_weight_min": -32.15528106689453,
      "activations/layer17_attention_weight_max": 50.41856384277344,
      "activations/layer17_attention_weight_min": -45.48664093017578,
      "activations/layer18_attention_weight_max": 47.09066390991211,
      "activations/layer18_attention_weight_min": -40.13145065307617,
      "activations/layer19_attention_weight_max": 26.72637939453125,
      "activations/layer19_attention_weight_min": -22.844144821166992,
      "activations/layer1_attention_weight_max": 15.024354934692383,
      "activations/layer1_attention_weight_min": -14.204475402832031,
      "activations/layer20_attention_weight_max": 24.763500213623047,
      "activations/layer20_attention_weight_min": -21.473922729492188,
      "activations/layer21_attention_weight_max": 32.41388702392578,
      "activations/layer21_attention_weight_min": -23.247610092163086,
      "activations/layer22_attention_weight_max": 32.2249641418457,
      "activations/layer22_attention_weight_min": -25.30488395690918,
      "activations/layer23_attention_weight_max": 35.69077682495117,
      "activations/layer23_attention_weight_min": -24.515775680541992,
      "activations/layer2_attention_weight_max": 31.475584030151367,
      "activations/layer2_attention_weight_min": -30.45789337158203,
      "activations/layer3_attention_weight_max": 89.31555938720703,
      "activations/layer3_attention_weight_min": -91.12616729736328,
      "activations/layer4_attention_weight_max": 83.43859100341797,
      "activations/layer4_attention_weight_min": -84.91043853759766,
      "activations/layer5_attention_weight_max": 62.64790344238281,
      "activations/layer5_attention_weight_min": -69.85430145263672,
      "activations/layer6_attention_weight_max": 48.21747970581055,
      "activations/layer6_attention_weight_min": -50.28007507324219,
      "activations/layer7_attention_weight_max": 66.02410888671875,
      "activations/layer7_attention_weight_min": -67.44356536865234,
      "activations/layer8_attention_weight_max": 45.51828384399414,
      "activations/layer8_attention_weight_min": -48.74067687988281,
      "activations/layer9_attention_weight_max": 43.531898498535156,
      "activations/layer9_attention_weight_min": -44.625816345214844,
      "epoch": 22.72,
      "learning_rate": 3.507954545454545e-06,
      "loss": 2.7062,
      "step": 391050
    },
    {
      "activations/layer0_attention_weight_max": 15.259683609008789,
      "activations/layer0_attention_weight_min": -14.244576454162598,
      "activations/layer10_attention_weight_max": 37.72526931762695,
      "activations/layer10_attention_weight_min": -35.6942138671875,
      "activations/layer11_attention_weight_max": 35.12900161743164,
      "activations/layer11_attention_weight_min": -34.53477096557617,
      "activations/layer12_attention_weight_max": 18.92315673828125,
      "activations/layer12_attention_weight_min": -29.58867835998535,
      "activations/layer13_attention_weight_max": 42.30012512207031,
      "activations/layer13_attention_weight_min": -29.94767189025879,
      "activations/layer14_attention_weight_max": 33.57126998901367,
      "activations/layer14_attention_weight_min": -30.291471481323242,
      "activations/layer15_attention_weight_max": 33.15500259399414,
      "activations/layer15_attention_weight_min": -31.248323440551758,
      "activations/layer16_attention_weight_max": 34.02573776245117,
      "activations/layer16_attention_weight_min": -31.99315071105957,
      "activations/layer17_attention_weight_max": 49.66384506225586,
      "activations/layer17_attention_weight_min": -48.24943542480469,
      "activations/layer18_attention_weight_max": 47.5172004699707,
      "activations/layer18_attention_weight_min": -39.70694351196289,
      "activations/layer19_attention_weight_max": 25.120948791503906,
      "activations/layer19_attention_weight_min": -23.847055435180664,
      "activations/layer1_attention_weight_max": 15.162440299987793,
      "activations/layer1_attention_weight_min": -14.07844066619873,
      "activations/layer20_attention_weight_max": 24.28721809387207,
      "activations/layer20_attention_weight_min": -21.69841194152832,
      "activations/layer21_attention_weight_max": 36.38169860839844,
      "activations/layer21_attention_weight_min": -23.127262115478516,
      "activations/layer22_attention_weight_max": 29.51284408569336,
      "activations/layer22_attention_weight_min": -24.530475616455078,
      "activations/layer23_attention_weight_max": 38.12533187866211,
      "activations/layer23_attention_weight_min": -24.334396362304688,
      "activations/layer2_attention_weight_max": 32.96336364746094,
      "activations/layer2_attention_weight_min": -32.44035339355469,
      "activations/layer3_attention_weight_max": 94.24169158935547,
      "activations/layer3_attention_weight_min": -94.1007080078125,
      "activations/layer4_attention_weight_max": 88.72270965576172,
      "activations/layer4_attention_weight_min": -86.77496337890625,
      "activations/layer5_attention_weight_max": 64.27147674560547,
      "activations/layer5_attention_weight_min": -74.24850463867188,
      "activations/layer6_attention_weight_max": 49.677040100097656,
      "activations/layer6_attention_weight_min": -50.969783782958984,
      "activations/layer7_attention_weight_max": 67.6086654663086,
      "activations/layer7_attention_weight_min": -68.75135040283203,
      "activations/layer8_attention_weight_max": 48.35100173950195,
      "activations/layer8_attention_weight_min": -49.63209915161133,
      "activations/layer9_attention_weight_max": 44.64645004272461,
      "activations/layer9_attention_weight_min": -46.30950164794922,
      "epoch": 22.73,
      "learning_rate": 3.489015151515151e-06,
      "loss": 2.6865,
      "step": 391100
    },
    {
      "activations/layer0_attention_weight_max": 15.537213325500488,
      "activations/layer0_attention_weight_min": -14.190147399902344,
      "activations/layer10_attention_weight_max": 34.1694221496582,
      "activations/layer10_attention_weight_min": -35.870811462402344,
      "activations/layer11_attention_weight_max": 31.30208969116211,
      "activations/layer11_attention_weight_min": -33.25305938720703,
      "activations/layer12_attention_weight_max": 18.33431625366211,
      "activations/layer12_attention_weight_min": -24.726470947265625,
      "activations/layer13_attention_weight_max": 34.0678596496582,
      "activations/layer13_attention_weight_min": -27.051469802856445,
      "activations/layer14_attention_weight_max": 33.60301971435547,
      "activations/layer14_attention_weight_min": -29.081037521362305,
      "activations/layer15_attention_weight_max": 32.96402359008789,
      "activations/layer15_attention_weight_min": -30.881155014038086,
      "activations/layer16_attention_weight_max": 34.2531852722168,
      "activations/layer16_attention_weight_min": -32.04576110839844,
      "activations/layer17_attention_weight_max": 52.0328369140625,
      "activations/layer17_attention_weight_min": -45.88126754760742,
      "activations/layer18_attention_weight_max": 44.683815002441406,
      "activations/layer18_attention_weight_min": -40.432559967041016,
      "activations/layer19_attention_weight_max": 23.85109519958496,
      "activations/layer19_attention_weight_min": -23.610523223876953,
      "activations/layer1_attention_weight_max": 15.348605155944824,
      "activations/layer1_attention_weight_min": -14.26341438293457,
      "activations/layer20_attention_weight_max": 22.53818702697754,
      "activations/layer20_attention_weight_min": -20.795116424560547,
      "activations/layer21_attention_weight_max": 31.818254470825195,
      "activations/layer21_attention_weight_min": -20.16469955444336,
      "activations/layer22_attention_weight_max": 27.371200561523438,
      "activations/layer22_attention_weight_min": -24.37051010131836,
      "activations/layer23_attention_weight_max": 35.66105270385742,
      "activations/layer23_attention_weight_min": -25.470237731933594,
      "activations/layer2_attention_weight_max": 32.894901275634766,
      "activations/layer2_attention_weight_min": -31.007436752319336,
      "activations/layer3_attention_weight_max": 90.70819854736328,
      "activations/layer3_attention_weight_min": -93.1412353515625,
      "activations/layer4_attention_weight_max": 83.48670196533203,
      "activations/layer4_attention_weight_min": -84.36763763427734,
      "activations/layer5_attention_weight_max": 61.257320404052734,
      "activations/layer5_attention_weight_min": -73.29582214355469,
      "activations/layer6_attention_weight_max": 50.686058044433594,
      "activations/layer6_attention_weight_min": -50.89533996582031,
      "activations/layer7_attention_weight_max": 68.21793365478516,
      "activations/layer7_attention_weight_min": -68.76695251464844,
      "activations/layer8_attention_weight_max": 47.93769454956055,
      "activations/layer8_attention_weight_min": -50.30128860473633,
      "activations/layer9_attention_weight_max": 43.08696746826172,
      "activations/layer9_attention_weight_min": -47.40674591064453,
      "epoch": 22.73,
      "learning_rate": 3.4700757575757576e-06,
      "loss": 2.7083,
      "step": 391150
    },
    {
      "activations/layer0_attention_weight_max": 14.723360061645508,
      "activations/layer0_attention_weight_min": -14.495096206665039,
      "activations/layer10_attention_weight_max": 43.81747817993164,
      "activations/layer10_attention_weight_min": -42.06367874145508,
      "activations/layer11_attention_weight_max": 42.77330780029297,
      "activations/layer11_attention_weight_min": -42.05351257324219,
      "activations/layer12_attention_weight_max": 19.60411262512207,
      "activations/layer12_attention_weight_min": -23.307207107543945,
      "activations/layer13_attention_weight_max": 34.08873748779297,
      "activations/layer13_attention_weight_min": -30.13382911682129,
      "activations/layer14_attention_weight_max": 36.706668853759766,
      "activations/layer14_attention_weight_min": -31.34046173095703,
      "activations/layer15_attention_weight_max": 33.646854400634766,
      "activations/layer15_attention_weight_min": -32.54189682006836,
      "activations/layer16_attention_weight_max": 35.04046630859375,
      "activations/layer16_attention_weight_min": -33.30506134033203,
      "activations/layer17_attention_weight_max": 56.8733024597168,
      "activations/layer17_attention_weight_min": -51.333065032958984,
      "activations/layer18_attention_weight_max": 47.68976974487305,
      "activations/layer18_attention_weight_min": -43.05765914916992,
      "activations/layer19_attention_weight_max": 24.528915405273438,
      "activations/layer19_attention_weight_min": -24.30491828918457,
      "activations/layer1_attention_weight_max": 15.54654312133789,
      "activations/layer1_attention_weight_min": -14.901987075805664,
      "activations/layer20_attention_weight_max": 22.18356704711914,
      "activations/layer20_attention_weight_min": -22.146398544311523,
      "activations/layer21_attention_weight_max": 35.27705383300781,
      "activations/layer21_attention_weight_min": -23.113779067993164,
      "activations/layer22_attention_weight_max": 31.31147003173828,
      "activations/layer22_attention_weight_min": -26.224262237548828,
      "activations/layer23_attention_weight_max": 36.88713836669922,
      "activations/layer23_attention_weight_min": -23.997732162475586,
      "activations/layer2_attention_weight_max": 34.275001525878906,
      "activations/layer2_attention_weight_min": -31.70526885986328,
      "activations/layer3_attention_weight_max": 101.65992736816406,
      "activations/layer3_attention_weight_min": -101.16129302978516,
      "activations/layer4_attention_weight_max": 96.01837158203125,
      "activations/layer4_attention_weight_min": -98.53679656982422,
      "activations/layer5_attention_weight_max": 70.2119369506836,
      "activations/layer5_attention_weight_min": -77.98019409179688,
      "activations/layer6_attention_weight_max": 56.24005889892578,
      "activations/layer6_attention_weight_min": -56.723175048828125,
      "activations/layer7_attention_weight_max": 78.26881408691406,
      "activations/layer7_attention_weight_min": -81.44900512695312,
      "activations/layer8_attention_weight_max": 60.74403381347656,
      "activations/layer8_attention_weight_min": -59.04916763305664,
      "activations/layer9_attention_weight_max": 53.576934814453125,
      "activations/layer9_attention_weight_min": -53.78327178955078,
      "epoch": 22.73,
      "learning_rate": 3.4511363636363633e-06,
      "loss": 2.6941,
      "step": 391200
    },
    {
      "activations/layer0_attention_weight_max": 15.34302806854248,
      "activations/layer0_attention_weight_min": -14.436834335327148,
      "activations/layer10_attention_weight_max": 36.0244026184082,
      "activations/layer10_attention_weight_min": -36.901397705078125,
      "activations/layer11_attention_weight_max": 32.51688766479492,
      "activations/layer11_attention_weight_min": -34.02406692504883,
      "activations/layer12_attention_weight_max": 18.262903213500977,
      "activations/layer12_attention_weight_min": -23.59386444091797,
      "activations/layer13_attention_weight_max": 33.00927734375,
      "activations/layer13_attention_weight_min": -28.894418716430664,
      "activations/layer14_attention_weight_max": 34.561058044433594,
      "activations/layer14_attention_weight_min": -31.1297607421875,
      "activations/layer15_attention_weight_max": 35.179527282714844,
      "activations/layer15_attention_weight_min": -29.232027053833008,
      "activations/layer16_attention_weight_max": 32.48952865600586,
      "activations/layer16_attention_weight_min": -31.762996673583984,
      "activations/layer17_attention_weight_max": 49.348976135253906,
      "activations/layer17_attention_weight_min": -44.86412048339844,
      "activations/layer18_attention_weight_max": 45.723289489746094,
      "activations/layer18_attention_weight_min": -39.6226692199707,
      "activations/layer19_attention_weight_max": 24.636119842529297,
      "activations/layer19_attention_weight_min": -24.509538650512695,
      "activations/layer1_attention_weight_max": 15.06421947479248,
      "activations/layer1_attention_weight_min": -14.522226333618164,
      "activations/layer20_attention_weight_max": 23.1505184173584,
      "activations/layer20_attention_weight_min": -21.375633239746094,
      "activations/layer21_attention_weight_max": 38.4770622253418,
      "activations/layer21_attention_weight_min": -23.20293426513672,
      "activations/layer22_attention_weight_max": 28.51950454711914,
      "activations/layer22_attention_weight_min": -23.88324546813965,
      "activations/layer23_attention_weight_max": 34.994873046875,
      "activations/layer23_attention_weight_min": -24.121673583984375,
      "activations/layer2_attention_weight_max": 32.65277862548828,
      "activations/layer2_attention_weight_min": -32.152549743652344,
      "activations/layer3_attention_weight_max": 95.4928207397461,
      "activations/layer3_attention_weight_min": -94.11939239501953,
      "activations/layer4_attention_weight_max": 84.99095916748047,
      "activations/layer4_attention_weight_min": -86.68345642089844,
      "activations/layer5_attention_weight_max": 63.887245178222656,
      "activations/layer5_attention_weight_min": -71.8886489868164,
      "activations/layer6_attention_weight_max": 49.970558166503906,
      "activations/layer6_attention_weight_min": -51.377986907958984,
      "activations/layer7_attention_weight_max": 73.78702545166016,
      "activations/layer7_attention_weight_min": -69.55545043945312,
      "activations/layer8_attention_weight_max": 47.22595977783203,
      "activations/layer8_attention_weight_min": -50.64398956298828,
      "activations/layer9_attention_weight_max": 43.23252487182617,
      "activations/layer9_attention_weight_min": -45.84370040893555,
      "epoch": 22.73,
      "learning_rate": 3.4321969696969695e-06,
      "loss": 2.725,
      "step": 391250
    },
    {
      "activations/layer0_attention_weight_max": 15.689506530761719,
      "activations/layer0_attention_weight_min": -14.062073707580566,
      "activations/layer10_attention_weight_max": 35.695194244384766,
      "activations/layer10_attention_weight_min": -36.25959014892578,
      "activations/layer11_attention_weight_max": 33.24486541748047,
      "activations/layer11_attention_weight_min": -34.877166748046875,
      "activations/layer12_attention_weight_max": 18.81439781188965,
      "activations/layer12_attention_weight_min": -27.089298248291016,
      "activations/layer13_attention_weight_max": 36.165771484375,
      "activations/layer13_attention_weight_min": -29.105459213256836,
      "activations/layer14_attention_weight_max": 36.458858489990234,
      "activations/layer14_attention_weight_min": -31.222631454467773,
      "activations/layer15_attention_weight_max": 36.561279296875,
      "activations/layer15_attention_weight_min": -32.38932800292969,
      "activations/layer16_attention_weight_max": 37.582244873046875,
      "activations/layer16_attention_weight_min": -34.039424896240234,
      "activations/layer17_attention_weight_max": 56.49696350097656,
      "activations/layer17_attention_weight_min": -49.199241638183594,
      "activations/layer18_attention_weight_max": 48.775848388671875,
      "activations/layer18_attention_weight_min": -44.341983795166016,
      "activations/layer19_attention_weight_max": 27.402494430541992,
      "activations/layer19_attention_weight_min": -24.739089965820312,
      "activations/layer1_attention_weight_max": 15.411802291870117,
      "activations/layer1_attention_weight_min": -17.12574577331543,
      "activations/layer20_attention_weight_max": 27.164443969726562,
      "activations/layer20_attention_weight_min": -22.838083267211914,
      "activations/layer21_attention_weight_max": 39.02298355102539,
      "activations/layer21_attention_weight_min": -24.627910614013672,
      "activations/layer22_attention_weight_max": 30.750905990600586,
      "activations/layer22_attention_weight_min": -26.06045913696289,
      "activations/layer23_attention_weight_max": 36.180213928222656,
      "activations/layer23_attention_weight_min": -23.656475067138672,
      "activations/layer2_attention_weight_max": 32.92936706542969,
      "activations/layer2_attention_weight_min": -31.404645919799805,
      "activations/layer3_attention_weight_max": 93.41433715820312,
      "activations/layer3_attention_weight_min": -94.19189453125,
      "activations/layer4_attention_weight_max": 85.51215362548828,
      "activations/layer4_attention_weight_min": -87.26229858398438,
      "activations/layer5_attention_weight_max": 63.54533386230469,
      "activations/layer5_attention_weight_min": -75.63439178466797,
      "activations/layer6_attention_weight_max": 49.10578155517578,
      "activations/layer6_attention_weight_min": -51.00050735473633,
      "activations/layer7_attention_weight_max": 71.78813934326172,
      "activations/layer7_attention_weight_min": -70.553466796875,
      "activations/layer8_attention_weight_max": 50.761253356933594,
      "activations/layer8_attention_weight_min": -52.32231903076172,
      "activations/layer9_attention_weight_max": 45.418331146240234,
      "activations/layer9_attention_weight_min": -46.94210433959961,
      "epoch": 22.74,
      "learning_rate": 3.413257575757575e-06,
      "loss": 2.6936,
      "step": 391300
    },
    {
      "activations/layer0_attention_weight_max": 14.998651504516602,
      "activations/layer0_attention_weight_min": -14.435792922973633,
      "activations/layer10_attention_weight_max": 36.054710388183594,
      "activations/layer10_attention_weight_min": -37.41652297973633,
      "activations/layer11_attention_weight_max": 34.085391998291016,
      "activations/layer11_attention_weight_min": -34.91782760620117,
      "activations/layer12_attention_weight_max": 19.673978805541992,
      "activations/layer12_attention_weight_min": -23.507278442382812,
      "activations/layer13_attention_weight_max": 34.69643783569336,
      "activations/layer13_attention_weight_min": -26.82325553894043,
      "activations/layer14_attention_weight_max": 34.858360290527344,
      "activations/layer14_attention_weight_min": -31.2613525390625,
      "activations/layer15_attention_weight_max": 36.9378547668457,
      "activations/layer15_attention_weight_min": -31.391820907592773,
      "activations/layer16_attention_weight_max": 36.76616668701172,
      "activations/layer16_attention_weight_min": -35.60958480834961,
      "activations/layer17_attention_weight_max": 57.34138870239258,
      "activations/layer17_attention_weight_min": -47.80562973022461,
      "activations/layer18_attention_weight_max": 52.91197204589844,
      "activations/layer18_attention_weight_min": -41.95771026611328,
      "activations/layer19_attention_weight_max": 27.322677612304688,
      "activations/layer19_attention_weight_min": -22.801698684692383,
      "activations/layer1_attention_weight_max": 14.647400856018066,
      "activations/layer1_attention_weight_min": -15.758435249328613,
      "activations/layer20_attention_weight_max": 25.58822250366211,
      "activations/layer20_attention_weight_min": -21.676780700683594,
      "activations/layer21_attention_weight_max": 37.44632339477539,
      "activations/layer21_attention_weight_min": -21.62019920349121,
      "activations/layer22_attention_weight_max": 31.372270584106445,
      "activations/layer22_attention_weight_min": -27.943359375,
      "activations/layer23_attention_weight_max": 35.588600158691406,
      "activations/layer23_attention_weight_min": -22.80080795288086,
      "activations/layer2_attention_weight_max": 32.03999710083008,
      "activations/layer2_attention_weight_min": -31.242887496948242,
      "activations/layer3_attention_weight_max": 90.25576782226562,
      "activations/layer3_attention_weight_min": -89.03044128417969,
      "activations/layer4_attention_weight_max": 85.83828735351562,
      "activations/layer4_attention_weight_min": -86.37505340576172,
      "activations/layer5_attention_weight_max": 59.70802307128906,
      "activations/layer5_attention_weight_min": -71.60739135742188,
      "activations/layer6_attention_weight_max": 48.98982620239258,
      "activations/layer6_attention_weight_min": -50.40458679199219,
      "activations/layer7_attention_weight_max": 67.45592498779297,
      "activations/layer7_attention_weight_min": -70.496337890625,
      "activations/layer8_attention_weight_max": 48.16169738769531,
      "activations/layer8_attention_weight_min": -51.1095085144043,
      "activations/layer9_attention_weight_max": 44.0115966796875,
      "activations/layer9_attention_weight_min": -46.37080001831055,
      "epoch": 22.74,
      "learning_rate": 3.3943181818181818e-06,
      "loss": 2.6971,
      "step": 391350
    },
    {
      "activations/layer0_attention_weight_max": 14.891740798950195,
      "activations/layer0_attention_weight_min": -14.248079299926758,
      "activations/layer10_attention_weight_max": 35.05315399169922,
      "activations/layer10_attention_weight_min": -34.852088928222656,
      "activations/layer11_attention_weight_max": 32.81474304199219,
      "activations/layer11_attention_weight_min": -34.359375,
      "activations/layer12_attention_weight_max": 19.886066436767578,
      "activations/layer12_attention_weight_min": -24.936670303344727,
      "activations/layer13_attention_weight_max": 34.55659103393555,
      "activations/layer13_attention_weight_min": -27.490219116210938,
      "activations/layer14_attention_weight_max": 35.15315246582031,
      "activations/layer14_attention_weight_min": -28.863723754882812,
      "activations/layer15_attention_weight_max": 33.41499710083008,
      "activations/layer15_attention_weight_min": -29.602510452270508,
      "activations/layer16_attention_weight_max": 33.86796188354492,
      "activations/layer16_attention_weight_min": -32.47139358520508,
      "activations/layer17_attention_weight_max": 54.9894905090332,
      "activations/layer17_attention_weight_min": -49.625328063964844,
      "activations/layer18_attention_weight_max": 49.54479217529297,
      "activations/layer18_attention_weight_min": -42.966453552246094,
      "activations/layer19_attention_weight_max": 25.394176483154297,
      "activations/layer19_attention_weight_min": -23.41365623474121,
      "activations/layer1_attention_weight_max": 15.781901359558105,
      "activations/layer1_attention_weight_min": -15.78154182434082,
      "activations/layer20_attention_weight_max": 23.10688591003418,
      "activations/layer20_attention_weight_min": -22.455141067504883,
      "activations/layer21_attention_weight_max": 32.12435531616211,
      "activations/layer21_attention_weight_min": -23.295000076293945,
      "activations/layer22_attention_weight_max": 29.769495010375977,
      "activations/layer22_attention_weight_min": -25.51459503173828,
      "activations/layer23_attention_weight_max": 36.8311882019043,
      "activations/layer23_attention_weight_min": -23.615720748901367,
      "activations/layer2_attention_weight_max": 32.55496597290039,
      "activations/layer2_attention_weight_min": -31.505069732666016,
      "activations/layer3_attention_weight_max": 94.6239013671875,
      "activations/layer3_attention_weight_min": -95.63914489746094,
      "activations/layer4_attention_weight_max": 87.21627044677734,
      "activations/layer4_attention_weight_min": -89.20374298095703,
      "activations/layer5_attention_weight_max": 62.61176300048828,
      "activations/layer5_attention_weight_min": -72.43771362304688,
      "activations/layer6_attention_weight_max": 48.6086311340332,
      "activations/layer6_attention_weight_min": -51.8979377746582,
      "activations/layer7_attention_weight_max": 67.69142150878906,
      "activations/layer7_attention_weight_min": -70.31712341308594,
      "activations/layer8_attention_weight_max": 47.99372100830078,
      "activations/layer8_attention_weight_min": -51.42432403564453,
      "activations/layer9_attention_weight_max": 43.6281852722168,
      "activations/layer9_attention_weight_min": -47.51300811767578,
      "epoch": 22.74,
      "learning_rate": 3.3753787878787875e-06,
      "loss": 2.6997,
      "step": 391400
    },
    {
      "activations/layer0_attention_weight_max": 15.824588775634766,
      "activations/layer0_attention_weight_min": -14.187993049621582,
      "activations/layer10_attention_weight_max": 37.394386291503906,
      "activations/layer10_attention_weight_min": -36.29199981689453,
      "activations/layer11_attention_weight_max": 35.984344482421875,
      "activations/layer11_attention_weight_min": -35.31999588012695,
      "activations/layer12_attention_weight_max": 20.46912384033203,
      "activations/layer12_attention_weight_min": -23.09082794189453,
      "activations/layer13_attention_weight_max": 31.565593719482422,
      "activations/layer13_attention_weight_min": -27.720748901367188,
      "activations/layer14_attention_weight_max": 36.55781173706055,
      "activations/layer14_attention_weight_min": -30.527339935302734,
      "activations/layer15_attention_weight_max": 33.817291259765625,
      "activations/layer15_attention_weight_min": -30.689878463745117,
      "activations/layer16_attention_weight_max": 36.637001037597656,
      "activations/layer16_attention_weight_min": -32.6462287902832,
      "activations/layer17_attention_weight_max": 52.45201873779297,
      "activations/layer17_attention_weight_min": -45.58026885986328,
      "activations/layer18_attention_weight_max": 48.0461540222168,
      "activations/layer18_attention_weight_min": -42.667110443115234,
      "activations/layer19_attention_weight_max": 24.12323570251465,
      "activations/layer19_attention_weight_min": -24.79956817626953,
      "activations/layer1_attention_weight_max": 15.892489433288574,
      "activations/layer1_attention_weight_min": -15.993793487548828,
      "activations/layer20_attention_weight_max": 23.56976890563965,
      "activations/layer20_attention_weight_min": -24.0487117767334,
      "activations/layer21_attention_weight_max": 34.505126953125,
      "activations/layer21_attention_weight_min": -23.153356552124023,
      "activations/layer22_attention_weight_max": 29.17954444885254,
      "activations/layer22_attention_weight_min": -25.187299728393555,
      "activations/layer23_attention_weight_max": 37.19908905029297,
      "activations/layer23_attention_weight_min": -25.433082580566406,
      "activations/layer2_attention_weight_max": 32.89007568359375,
      "activations/layer2_attention_weight_min": -33.191551208496094,
      "activations/layer3_attention_weight_max": 95.46900939941406,
      "activations/layer3_attention_weight_min": -94.72686004638672,
      "activations/layer4_attention_weight_max": 87.4852066040039,
      "activations/layer4_attention_weight_min": -87.84837341308594,
      "activations/layer5_attention_weight_max": 65.1338882446289,
      "activations/layer5_attention_weight_min": -72.54129028320312,
      "activations/layer6_attention_weight_max": 50.35047912597656,
      "activations/layer6_attention_weight_min": -54.062381744384766,
      "activations/layer7_attention_weight_max": 71.17498016357422,
      "activations/layer7_attention_weight_min": -68.44832611083984,
      "activations/layer8_attention_weight_max": 49.36891174316406,
      "activations/layer8_attention_weight_min": -53.8816032409668,
      "activations/layer9_attention_weight_max": 46.14491271972656,
      "activations/layer9_attention_weight_min": -47.85490036010742,
      "epoch": 22.75,
      "learning_rate": 3.3564393939393937e-06,
      "loss": 2.6846,
      "step": 391450
    },
    {
      "activations/layer0_attention_weight_max": 15.729991912841797,
      "activations/layer0_attention_weight_min": -14.298602104187012,
      "activations/layer10_attention_weight_max": 35.511695861816406,
      "activations/layer10_attention_weight_min": -36.5389404296875,
      "activations/layer11_attention_weight_max": 34.071571350097656,
      "activations/layer11_attention_weight_min": -37.35924530029297,
      "activations/layer12_attention_weight_max": 20.669519424438477,
      "activations/layer12_attention_weight_min": -26.333023071289062,
      "activations/layer13_attention_weight_max": 38.5234260559082,
      "activations/layer13_attention_weight_min": -27.213783264160156,
      "activations/layer14_attention_weight_max": 37.340599060058594,
      "activations/layer14_attention_weight_min": -31.08228302001953,
      "activations/layer15_attention_weight_max": 32.960147857666016,
      "activations/layer15_attention_weight_min": -31.168596267700195,
      "activations/layer16_attention_weight_max": 35.98512649536133,
      "activations/layer16_attention_weight_min": -34.641056060791016,
      "activations/layer17_attention_weight_max": 53.86848068237305,
      "activations/layer17_attention_weight_min": -47.45990753173828,
      "activations/layer18_attention_weight_max": 50.56576156616211,
      "activations/layer18_attention_weight_min": -41.887847900390625,
      "activations/layer19_attention_weight_max": 25.8043212890625,
      "activations/layer19_attention_weight_min": -24.110132217407227,
      "activations/layer1_attention_weight_max": 15.331326484680176,
      "activations/layer1_attention_weight_min": -14.973197937011719,
      "activations/layer20_attention_weight_max": 24.987585067749023,
      "activations/layer20_attention_weight_min": -23.562389373779297,
      "activations/layer21_attention_weight_max": 37.86201477050781,
      "activations/layer21_attention_weight_min": -21.791667938232422,
      "activations/layer22_attention_weight_max": 31.37843894958496,
      "activations/layer22_attention_weight_min": -24.292985916137695,
      "activations/layer23_attention_weight_max": 36.482505798339844,
      "activations/layer23_attention_weight_min": -23.268896102905273,
      "activations/layer2_attention_weight_max": 33.0220947265625,
      "activations/layer2_attention_weight_min": -30.745832443237305,
      "activations/layer3_attention_weight_max": 93.6304702758789,
      "activations/layer3_attention_weight_min": -91.71084594726562,
      "activations/layer4_attention_weight_max": 84.60923767089844,
      "activations/layer4_attention_weight_min": -86.84386444091797,
      "activations/layer5_attention_weight_max": 60.65892028808594,
      "activations/layer5_attention_weight_min": -74.2960205078125,
      "activations/layer6_attention_weight_max": 48.910072326660156,
      "activations/layer6_attention_weight_min": -51.55674362182617,
      "activations/layer7_attention_weight_max": 71.21049499511719,
      "activations/layer7_attention_weight_min": -69.27035522460938,
      "activations/layer8_attention_weight_max": 48.291324615478516,
      "activations/layer8_attention_weight_min": -52.925384521484375,
      "activations/layer9_attention_weight_max": 43.32670974731445,
      "activations/layer9_attention_weight_min": -47.067527770996094,
      "epoch": 22.75,
      "learning_rate": 3.3374999999999994e-06,
      "loss": 2.6929,
      "step": 391500
    },
    {
      "activations/layer0_attention_weight_max": 16.004640579223633,
      "activations/layer0_attention_weight_min": -14.069618225097656,
      "activations/layer10_attention_weight_max": 39.86228942871094,
      "activations/layer10_attention_weight_min": -37.92340850830078,
      "activations/layer11_attention_weight_max": 35.16117858886719,
      "activations/layer11_attention_weight_min": -36.93537902832031,
      "activations/layer12_attention_weight_max": 21.656015396118164,
      "activations/layer12_attention_weight_min": -24.702835083007812,
      "activations/layer13_attention_weight_max": 37.37295150756836,
      "activations/layer13_attention_weight_min": -28.36790657043457,
      "activations/layer14_attention_weight_max": 35.22163009643555,
      "activations/layer14_attention_weight_min": -29.7740478515625,
      "activations/layer15_attention_weight_max": 34.034183502197266,
      "activations/layer15_attention_weight_min": -30.9480037689209,
      "activations/layer16_attention_weight_max": 36.043434143066406,
      "activations/layer16_attention_weight_min": -33.3851203918457,
      "activations/layer17_attention_weight_max": 56.001827239990234,
      "activations/layer17_attention_weight_min": -48.38978958129883,
      "activations/layer18_attention_weight_max": 50.98125076293945,
      "activations/layer18_attention_weight_min": -41.73098373413086,
      "activations/layer19_attention_weight_max": 28.431074142456055,
      "activations/layer19_attention_weight_min": -23.51910972595215,
      "activations/layer1_attention_weight_max": 15.06467056274414,
      "activations/layer1_attention_weight_min": -14.907605171203613,
      "activations/layer20_attention_weight_max": 24.228391647338867,
      "activations/layer20_attention_weight_min": -23.70427131652832,
      "activations/layer21_attention_weight_max": 37.69401931762695,
      "activations/layer21_attention_weight_min": -21.371652603149414,
      "activations/layer22_attention_weight_max": 28.5794620513916,
      "activations/layer22_attention_weight_min": -25.152477264404297,
      "activations/layer23_attention_weight_max": 34.94626235961914,
      "activations/layer23_attention_weight_min": -23.049840927124023,
      "activations/layer2_attention_weight_max": 32.860713958740234,
      "activations/layer2_attention_weight_min": -31.582962036132812,
      "activations/layer3_attention_weight_max": 92.32258605957031,
      "activations/layer3_attention_weight_min": -97.93157958984375,
      "activations/layer4_attention_weight_max": 90.71047973632812,
      "activations/layer4_attention_weight_min": -92.34749603271484,
      "activations/layer5_attention_weight_max": 63.952293395996094,
      "activations/layer5_attention_weight_min": -70.83151245117188,
      "activations/layer6_attention_weight_max": 52.97477340698242,
      "activations/layer6_attention_weight_min": -52.385536193847656,
      "activations/layer7_attention_weight_max": 75.60617065429688,
      "activations/layer7_attention_weight_min": -68.9216079711914,
      "activations/layer8_attention_weight_max": 52.000823974609375,
      "activations/layer8_attention_weight_min": -52.19428253173828,
      "activations/layer9_attention_weight_max": 45.356971740722656,
      "activations/layer9_attention_weight_min": -48.36458969116211,
      "epoch": 22.75,
      "learning_rate": 3.318560606060606e-06,
      "loss": 2.7017,
      "step": 391550
    },
    {
      "activations/layer0_attention_weight_max": 15.582572937011719,
      "activations/layer0_attention_weight_min": -14.118800163269043,
      "activations/layer10_attention_weight_max": 37.70143508911133,
      "activations/layer10_attention_weight_min": -37.119415283203125,
      "activations/layer11_attention_weight_max": 34.89789962768555,
      "activations/layer11_attention_weight_min": -37.13056182861328,
      "activations/layer12_attention_weight_max": 20.04740333557129,
      "activations/layer12_attention_weight_min": -24.89866065979004,
      "activations/layer13_attention_weight_max": 37.276573181152344,
      "activations/layer13_attention_weight_min": -28.09330940246582,
      "activations/layer14_attention_weight_max": 39.15797424316406,
      "activations/layer14_attention_weight_min": -30.652238845825195,
      "activations/layer15_attention_weight_max": 35.20892333984375,
      "activations/layer15_attention_weight_min": -34.10425567626953,
      "activations/layer16_attention_weight_max": 38.612388610839844,
      "activations/layer16_attention_weight_min": -35.654544830322266,
      "activations/layer17_attention_weight_max": 54.492408752441406,
      "activations/layer17_attention_weight_min": -47.043663024902344,
      "activations/layer18_attention_weight_max": 49.09419631958008,
      "activations/layer18_attention_weight_min": -41.59545135498047,
      "activations/layer19_attention_weight_max": 25.503759384155273,
      "activations/layer19_attention_weight_min": -24.43414306640625,
      "activations/layer1_attention_weight_max": 15.345812797546387,
      "activations/layer1_attention_weight_min": -15.770532608032227,
      "activations/layer20_attention_weight_max": 23.634153366088867,
      "activations/layer20_attention_weight_min": -24.201017379760742,
      "activations/layer21_attention_weight_max": 36.85293197631836,
      "activations/layer21_attention_weight_min": -22.606475830078125,
      "activations/layer22_attention_weight_max": 28.36052131652832,
      "activations/layer22_attention_weight_min": -27.254854202270508,
      "activations/layer23_attention_weight_max": 36.151309967041016,
      "activations/layer23_attention_weight_min": -21.98720932006836,
      "activations/layer2_attention_weight_max": 31.434019088745117,
      "activations/layer2_attention_weight_min": -30.22061538696289,
      "activations/layer3_attention_weight_max": 92.30184936523438,
      "activations/layer3_attention_weight_min": -93.89543914794922,
      "activations/layer4_attention_weight_max": 88.04389953613281,
      "activations/layer4_attention_weight_min": -88.2865982055664,
      "activations/layer5_attention_weight_max": 64.63304901123047,
      "activations/layer5_attention_weight_min": -72.96318054199219,
      "activations/layer6_attention_weight_max": 50.00444793701172,
      "activations/layer6_attention_weight_min": -52.54264450073242,
      "activations/layer7_attention_weight_max": 69.71617889404297,
      "activations/layer7_attention_weight_min": -70.46530151367188,
      "activations/layer8_attention_weight_max": 48.46778106689453,
      "activations/layer8_attention_weight_min": -51.36885070800781,
      "activations/layer9_attention_weight_max": 43.975677490234375,
      "activations/layer9_attention_weight_min": -46.2391471862793,
      "epoch": 22.75,
      "learning_rate": 3.299621212121212e-06,
      "loss": 2.6867,
      "step": 391600
    },
    {
      "activations/layer0_attention_weight_max": 15.66115665435791,
      "activations/layer0_attention_weight_min": -15.380219459533691,
      "activations/layer10_attention_weight_max": 35.19055938720703,
      "activations/layer10_attention_weight_min": -38.18233108520508,
      "activations/layer11_attention_weight_max": 33.711387634277344,
      "activations/layer11_attention_weight_min": -36.25688934326172,
      "activations/layer12_attention_weight_max": 17.526025772094727,
      "activations/layer12_attention_weight_min": -24.008926391601562,
      "activations/layer13_attention_weight_max": 30.11406898498535,
      "activations/layer13_attention_weight_min": -27.354244232177734,
      "activations/layer14_attention_weight_max": 34.14374542236328,
      "activations/layer14_attention_weight_min": -29.670757293701172,
      "activations/layer15_attention_weight_max": 32.98564147949219,
      "activations/layer15_attention_weight_min": -29.272172927856445,
      "activations/layer16_attention_weight_max": 33.89200210571289,
      "activations/layer16_attention_weight_min": -33.13547897338867,
      "activations/layer17_attention_weight_max": 49.67398452758789,
      "activations/layer17_attention_weight_min": -44.462913513183594,
      "activations/layer18_attention_weight_max": 46.49470138549805,
      "activations/layer18_attention_weight_min": -42.66045379638672,
      "activations/layer19_attention_weight_max": 26.597644805908203,
      "activations/layer19_attention_weight_min": -24.23781394958496,
      "activations/layer1_attention_weight_max": 15.635506629943848,
      "activations/layer1_attention_weight_min": -15.290651321411133,
      "activations/layer20_attention_weight_max": 25.122085571289062,
      "activations/layer20_attention_weight_min": -23.728790283203125,
      "activations/layer21_attention_weight_max": 37.834659576416016,
      "activations/layer21_attention_weight_min": -24.576982498168945,
      "activations/layer22_attention_weight_max": 30.85431671142578,
      "activations/layer22_attention_weight_min": -24.796852111816406,
      "activations/layer23_attention_weight_max": 37.5772705078125,
      "activations/layer23_attention_weight_min": -24.970365524291992,
      "activations/layer2_attention_weight_max": 33.681880950927734,
      "activations/layer2_attention_weight_min": -30.673717498779297,
      "activations/layer3_attention_weight_max": 94.87930297851562,
      "activations/layer3_attention_weight_min": -94.68800354003906,
      "activations/layer4_attention_weight_max": 86.18383026123047,
      "activations/layer4_attention_weight_min": -87.98667907714844,
      "activations/layer5_attention_weight_max": 60.99446105957031,
      "activations/layer5_attention_weight_min": -72.92686462402344,
      "activations/layer6_attention_weight_max": 50.88867950439453,
      "activations/layer6_attention_weight_min": -52.2199821472168,
      "activations/layer7_attention_weight_max": 66.59066772460938,
      "activations/layer7_attention_weight_min": -68.95964050292969,
      "activations/layer8_attention_weight_max": 48.26814651489258,
      "activations/layer8_attention_weight_min": -52.57588577270508,
      "activations/layer9_attention_weight_max": 46.5463752746582,
      "activations/layer9_attention_weight_min": -50.04230880737305,
      "epoch": 22.76,
      "learning_rate": 3.280681818181818e-06,
      "loss": 2.6888,
      "step": 391650
    },
    {
      "activations/layer0_attention_weight_max": 15.696290969848633,
      "activations/layer0_attention_weight_min": -14.258829116821289,
      "activations/layer10_attention_weight_max": 35.57988357543945,
      "activations/layer10_attention_weight_min": -37.42323684692383,
      "activations/layer11_attention_weight_max": 33.65557861328125,
      "activations/layer11_attention_weight_min": -36.24843215942383,
      "activations/layer12_attention_weight_max": 18.867887496948242,
      "activations/layer12_attention_weight_min": -24.819272994995117,
      "activations/layer13_attention_weight_max": 33.812110900878906,
      "activations/layer13_attention_weight_min": -27.56475257873535,
      "activations/layer14_attention_weight_max": 34.539794921875,
      "activations/layer14_attention_weight_min": -30.458263397216797,
      "activations/layer15_attention_weight_max": 33.62007141113281,
      "activations/layer15_attention_weight_min": -31.841922760009766,
      "activations/layer16_attention_weight_max": 37.5402946472168,
      "activations/layer16_attention_weight_min": -37.032569885253906,
      "activations/layer17_attention_weight_max": 53.142173767089844,
      "activations/layer17_attention_weight_min": -51.019378662109375,
      "activations/layer18_attention_weight_max": 50.17886734008789,
      "activations/layer18_attention_weight_min": -44.94850158691406,
      "activations/layer19_attention_weight_max": 25.351564407348633,
      "activations/layer19_attention_weight_min": -26.40504264831543,
      "activations/layer1_attention_weight_max": 15.156432151794434,
      "activations/layer1_attention_weight_min": -14.204386711120605,
      "activations/layer20_attention_weight_max": 23.060020446777344,
      "activations/layer20_attention_weight_min": -23.378841400146484,
      "activations/layer21_attention_weight_max": 36.81389617919922,
      "activations/layer21_attention_weight_min": -22.69832992553711,
      "activations/layer22_attention_weight_max": 29.71310806274414,
      "activations/layer22_attention_weight_min": -27.30091667175293,
      "activations/layer23_attention_weight_max": 37.052581787109375,
      "activations/layer23_attention_weight_min": -24.39568328857422,
      "activations/layer2_attention_weight_max": 31.933597564697266,
      "activations/layer2_attention_weight_min": -29.964649200439453,
      "activations/layer3_attention_weight_max": 91.43494415283203,
      "activations/layer3_attention_weight_min": -90.01813507080078,
      "activations/layer4_attention_weight_max": 89.9964828491211,
      "activations/layer4_attention_weight_min": -86.92866516113281,
      "activations/layer5_attention_weight_max": 64.35304260253906,
      "activations/layer5_attention_weight_min": -73.77772521972656,
      "activations/layer6_attention_weight_max": 52.24687576293945,
      "activations/layer6_attention_weight_min": -51.193721771240234,
      "activations/layer7_attention_weight_max": 69.90464782714844,
      "activations/layer7_attention_weight_min": -72.09567260742188,
      "activations/layer8_attention_weight_max": 50.976951599121094,
      "activations/layer8_attention_weight_min": -53.40901565551758,
      "activations/layer9_attention_weight_max": 46.475399017333984,
      "activations/layer9_attention_weight_min": -46.79084777832031,
      "epoch": 22.76,
      "learning_rate": 3.261742424242424e-06,
      "loss": 2.6916,
      "step": 391700
    },
    {
      "activations/layer0_attention_weight_max": 15.859914779663086,
      "activations/layer0_attention_weight_min": -14.214422225952148,
      "activations/layer10_attention_weight_max": 39.724605560302734,
      "activations/layer10_attention_weight_min": -39.2979736328125,
      "activations/layer11_attention_weight_max": 37.10514450073242,
      "activations/layer11_attention_weight_min": -38.47181701660156,
      "activations/layer12_attention_weight_max": 19.114334106445312,
      "activations/layer12_attention_weight_min": -23.63550567626953,
      "activations/layer13_attention_weight_max": 37.69630432128906,
      "activations/layer13_attention_weight_min": -29.332386016845703,
      "activations/layer14_attention_weight_max": 42.388004302978516,
      "activations/layer14_attention_weight_min": -30.1840877532959,
      "activations/layer15_attention_weight_max": 37.03993225097656,
      "activations/layer15_attention_weight_min": -30.65911102294922,
      "activations/layer16_attention_weight_max": 36.52231216430664,
      "activations/layer16_attention_weight_min": -33.189720153808594,
      "activations/layer17_attention_weight_max": 51.57079315185547,
      "activations/layer17_attention_weight_min": -48.40963363647461,
      "activations/layer18_attention_weight_max": 48.7408561706543,
      "activations/layer18_attention_weight_min": -40.98133850097656,
      "activations/layer19_attention_weight_max": 26.96164321899414,
      "activations/layer19_attention_weight_min": -23.95704460144043,
      "activations/layer1_attention_weight_max": 15.358160018920898,
      "activations/layer1_attention_weight_min": -14.840598106384277,
      "activations/layer20_attention_weight_max": 26.973838806152344,
      "activations/layer20_attention_weight_min": -21.675901412963867,
      "activations/layer21_attention_weight_max": 41.034149169921875,
      "activations/layer21_attention_weight_min": -25.6092586517334,
      "activations/layer22_attention_weight_max": 27.36881446838379,
      "activations/layer22_attention_weight_min": -24.4172420501709,
      "activations/layer23_attention_weight_max": 34.59714889526367,
      "activations/layer23_attention_weight_min": -24.875831604003906,
      "activations/layer2_attention_weight_max": 32.86590576171875,
      "activations/layer2_attention_weight_min": -32.7502555847168,
      "activations/layer3_attention_weight_max": 90.88439178466797,
      "activations/layer3_attention_weight_min": -93.70687866210938,
      "activations/layer4_attention_weight_max": 84.64239501953125,
      "activations/layer4_attention_weight_min": -86.2686996459961,
      "activations/layer5_attention_weight_max": 60.14500045776367,
      "activations/layer5_attention_weight_min": -71.85346984863281,
      "activations/layer6_attention_weight_max": 50.981712341308594,
      "activations/layer6_attention_weight_min": -55.1890754699707,
      "activations/layer7_attention_weight_max": 71.15604400634766,
      "activations/layer7_attention_weight_min": -74.59102630615234,
      "activations/layer8_attention_weight_max": 50.46829605102539,
      "activations/layer8_attention_weight_min": -54.04539108276367,
      "activations/layer9_attention_weight_max": 52.4166145324707,
      "activations/layer9_attention_weight_min": -48.59431076049805,
      "epoch": 22.76,
      "learning_rate": 3.2428030303030297e-06,
      "loss": 2.7028,
      "step": 391750
    },
    {
      "activations/layer0_attention_weight_max": 15.218221664428711,
      "activations/layer0_attention_weight_min": -14.29802131652832,
      "activations/layer10_attention_weight_max": 36.809852600097656,
      "activations/layer10_attention_weight_min": -36.39678192138672,
      "activations/layer11_attention_weight_max": 35.09038162231445,
      "activations/layer11_attention_weight_min": -35.613224029541016,
      "activations/layer12_attention_weight_max": 17.900148391723633,
      "activations/layer12_attention_weight_min": -26.991727828979492,
      "activations/layer13_attention_weight_max": 30.531404495239258,
      "activations/layer13_attention_weight_min": -27.742753982543945,
      "activations/layer14_attention_weight_max": 31.1129150390625,
      "activations/layer14_attention_weight_min": -30.772443771362305,
      "activations/layer15_attention_weight_max": 29.83649253845215,
      "activations/layer15_attention_weight_min": -29.576387405395508,
      "activations/layer16_attention_weight_max": 33.76817321777344,
      "activations/layer16_attention_weight_min": -33.31714630126953,
      "activations/layer17_attention_weight_max": 47.72261428833008,
      "activations/layer17_attention_weight_min": -45.01311111450195,
      "activations/layer18_attention_weight_max": 44.94066619873047,
      "activations/layer18_attention_weight_min": -44.72922134399414,
      "activations/layer19_attention_weight_max": 23.13178825378418,
      "activations/layer19_attention_weight_min": -24.083412170410156,
      "activations/layer1_attention_weight_max": 15.211695671081543,
      "activations/layer1_attention_weight_min": -14.012614250183105,
      "activations/layer20_attention_weight_max": 23.582698822021484,
      "activations/layer20_attention_weight_min": -21.600322723388672,
      "activations/layer21_attention_weight_max": 34.79249572753906,
      "activations/layer21_attention_weight_min": -23.52533531188965,
      "activations/layer22_attention_weight_max": 30.165191650390625,
      "activations/layer22_attention_weight_min": -26.077131271362305,
      "activations/layer23_attention_weight_max": 34.861228942871094,
      "activations/layer23_attention_weight_min": -23.66084098815918,
      "activations/layer2_attention_weight_max": 32.5234489440918,
      "activations/layer2_attention_weight_min": -29.632984161376953,
      "activations/layer3_attention_weight_max": 92.06369018554688,
      "activations/layer3_attention_weight_min": -87.97408294677734,
      "activations/layer4_attention_weight_max": 85.56774139404297,
      "activations/layer4_attention_weight_min": -82.1349105834961,
      "activations/layer5_attention_weight_max": 61.136558532714844,
      "activations/layer5_attention_weight_min": -72.01795196533203,
      "activations/layer6_attention_weight_max": 47.356754302978516,
      "activations/layer6_attention_weight_min": -49.169578552246094,
      "activations/layer7_attention_weight_max": 72.95878601074219,
      "activations/layer7_attention_weight_min": -68.94696807861328,
      "activations/layer8_attention_weight_max": 46.98371887207031,
      "activations/layer8_attention_weight_min": -48.61148452758789,
      "activations/layer9_attention_weight_max": 42.768760681152344,
      "activations/layer9_attention_weight_min": -45.53837203979492,
      "epoch": 22.77,
      "learning_rate": 3.224242424242424e-06,
      "loss": 2.7013,
      "step": 391800
    },
    {
      "activations/layer0_attention_weight_max": 15.45603084564209,
      "activations/layer0_attention_weight_min": -14.007528305053711,
      "activations/layer10_attention_weight_max": 38.18706130981445,
      "activations/layer10_attention_weight_min": -36.99462127685547,
      "activations/layer11_attention_weight_max": 36.17765426635742,
      "activations/layer11_attention_weight_min": -38.002662658691406,
      "activations/layer12_attention_weight_max": 20.293935775756836,
      "activations/layer12_attention_weight_min": -25.804513931274414,
      "activations/layer13_attention_weight_max": 34.718406677246094,
      "activations/layer13_attention_weight_min": -31.52394676208496,
      "activations/layer14_attention_weight_max": 36.33433532714844,
      "activations/layer14_attention_weight_min": -30.226102828979492,
      "activations/layer15_attention_weight_max": 34.28415298461914,
      "activations/layer15_attention_weight_min": -30.634754180908203,
      "activations/layer16_attention_weight_max": 34.95751190185547,
      "activations/layer16_attention_weight_min": -34.35467529296875,
      "activations/layer17_attention_weight_max": 49.98585510253906,
      "activations/layer17_attention_weight_min": -47.8368034362793,
      "activations/layer18_attention_weight_max": 46.64942169189453,
      "activations/layer18_attention_weight_min": -41.95035171508789,
      "activations/layer19_attention_weight_max": 23.56707763671875,
      "activations/layer19_attention_weight_min": -24.34211540222168,
      "activations/layer1_attention_weight_max": 15.602977752685547,
      "activations/layer1_attention_weight_min": -14.809379577636719,
      "activations/layer20_attention_weight_max": 22.505918502807617,
      "activations/layer20_attention_weight_min": -22.619186401367188,
      "activations/layer21_attention_weight_max": 39.338356018066406,
      "activations/layer21_attention_weight_min": -20.90254020690918,
      "activations/layer22_attention_weight_max": 26.688467025756836,
      "activations/layer22_attention_weight_min": -24.849931716918945,
      "activations/layer23_attention_weight_max": 35.045875549316406,
      "activations/layer23_attention_weight_min": -24.565441131591797,
      "activations/layer2_attention_weight_max": 32.941200256347656,
      "activations/layer2_attention_weight_min": -30.92335319519043,
      "activations/layer3_attention_weight_max": 92.40567016601562,
      "activations/layer3_attention_weight_min": -88.68867492675781,
      "activations/layer4_attention_weight_max": 86.36650848388672,
      "activations/layer4_attention_weight_min": -85.68547821044922,
      "activations/layer5_attention_weight_max": 66.05229187011719,
      "activations/layer5_attention_weight_min": -73.78962707519531,
      "activations/layer6_attention_weight_max": 52.602657318115234,
      "activations/layer6_attention_weight_min": -53.332557678222656,
      "activations/layer7_attention_weight_max": 73.0881576538086,
      "activations/layer7_attention_weight_min": -71.63162994384766,
      "activations/layer8_attention_weight_max": 50.11463165283203,
      "activations/layer8_attention_weight_min": -51.385780334472656,
      "activations/layer9_attention_weight_max": 46.496177673339844,
      "activations/layer9_attention_weight_min": -48.43020248413086,
      "epoch": 22.77,
      "learning_rate": 3.20530303030303e-06,
      "loss": 2.6898,
      "step": 391850
    },
    {
      "activations/layer0_attention_weight_max": 15.583663940429688,
      "activations/layer0_attention_weight_min": -14.055644989013672,
      "activations/layer10_attention_weight_max": 35.23878479003906,
      "activations/layer10_attention_weight_min": -36.89759826660156,
      "activations/layer11_attention_weight_max": 35.203216552734375,
      "activations/layer11_attention_weight_min": -35.8843994140625,
      "activations/layer12_attention_weight_max": 18.231674194335938,
      "activations/layer12_attention_weight_min": -26.47336196899414,
      "activations/layer13_attention_weight_max": 36.06163787841797,
      "activations/layer13_attention_weight_min": -30.46889877319336,
      "activations/layer14_attention_weight_max": 38.64484786987305,
      "activations/layer14_attention_weight_min": -32.85812759399414,
      "activations/layer15_attention_weight_max": 36.46440124511719,
      "activations/layer15_attention_weight_min": -32.89775848388672,
      "activations/layer16_attention_weight_max": 40.427940368652344,
      "activations/layer16_attention_weight_min": -39.05169677734375,
      "activations/layer17_attention_weight_max": 58.60893630981445,
      "activations/layer17_attention_weight_min": -52.20864486694336,
      "activations/layer18_attention_weight_max": 54.06623840332031,
      "activations/layer18_attention_weight_min": -47.723758697509766,
      "activations/layer19_attention_weight_max": 28.523893356323242,
      "activations/layer19_attention_weight_min": -25.38340950012207,
      "activations/layer1_attention_weight_max": 15.09775161743164,
      "activations/layer1_attention_weight_min": -14.856232643127441,
      "activations/layer20_attention_weight_max": 26.764890670776367,
      "activations/layer20_attention_weight_min": -22.677343368530273,
      "activations/layer21_attention_weight_max": 38.58718490600586,
      "activations/layer21_attention_weight_min": -27.94700050354004,
      "activations/layer22_attention_weight_max": 32.76047897338867,
      "activations/layer22_attention_weight_min": -25.60065269470215,
      "activations/layer23_attention_weight_max": 39.65919494628906,
      "activations/layer23_attention_weight_min": -28.22694206237793,
      "activations/layer2_attention_weight_max": 30.87068748474121,
      "activations/layer2_attention_weight_min": -31.496999740600586,
      "activations/layer3_attention_weight_max": 88.80207824707031,
      "activations/layer3_attention_weight_min": -93.84642028808594,
      "activations/layer4_attention_weight_max": 85.5792007446289,
      "activations/layer4_attention_weight_min": -84.9012451171875,
      "activations/layer5_attention_weight_max": 65.43536376953125,
      "activations/layer5_attention_weight_min": -72.0917739868164,
      "activations/layer6_attention_weight_max": 49.75950622558594,
      "activations/layer6_attention_weight_min": -52.438053131103516,
      "activations/layer7_attention_weight_max": 70.41007232666016,
      "activations/layer7_attention_weight_min": -73.0316162109375,
      "activations/layer8_attention_weight_max": 48.52619552612305,
      "activations/layer8_attention_weight_min": -52.828163146972656,
      "activations/layer9_attention_weight_max": 43.846763610839844,
      "activations/layer9_attention_weight_min": -47.9743766784668,
      "epoch": 22.77,
      "learning_rate": 3.1863636363636362e-06,
      "loss": 2.7073,
      "step": 391900
    },
    {
      "activations/layer0_attention_weight_max": 16.06559181213379,
      "activations/layer0_attention_weight_min": -14.347152709960938,
      "activations/layer10_attention_weight_max": 33.633995056152344,
      "activations/layer10_attention_weight_min": -35.89999771118164,
      "activations/layer11_attention_weight_max": 32.212120056152344,
      "activations/layer11_attention_weight_min": -34.962432861328125,
      "activations/layer12_attention_weight_max": 19.590112686157227,
      "activations/layer12_attention_weight_min": -25.238752365112305,
      "activations/layer13_attention_weight_max": 34.156272888183594,
      "activations/layer13_attention_weight_min": -28.500286102294922,
      "activations/layer14_attention_weight_max": 34.664363861083984,
      "activations/layer14_attention_weight_min": -30.730573654174805,
      "activations/layer15_attention_weight_max": 37.87837600708008,
      "activations/layer15_attention_weight_min": -31.74887466430664,
      "activations/layer16_attention_weight_max": 35.21398162841797,
      "activations/layer16_attention_weight_min": -34.85667037963867,
      "activations/layer17_attention_weight_max": 53.777225494384766,
      "activations/layer17_attention_weight_min": -46.43949890136719,
      "activations/layer18_attention_weight_max": 50.47675323486328,
      "activations/layer18_attention_weight_min": -41.50922775268555,
      "activations/layer19_attention_weight_max": 30.91461753845215,
      "activations/layer19_attention_weight_min": -23.3746395111084,
      "activations/layer1_attention_weight_max": 15.348628997802734,
      "activations/layer1_attention_weight_min": -15.45613956451416,
      "activations/layer20_attention_weight_max": 24.928516387939453,
      "activations/layer20_attention_weight_min": -21.393770217895508,
      "activations/layer21_attention_weight_max": 36.25749588012695,
      "activations/layer21_attention_weight_min": -21.594995498657227,
      "activations/layer22_attention_weight_max": 35.25481033325195,
      "activations/layer22_attention_weight_min": -25.844270706176758,
      "activations/layer23_attention_weight_max": 35.717323303222656,
      "activations/layer23_attention_weight_min": -25.02742576599121,
      "activations/layer2_attention_weight_max": 32.914093017578125,
      "activations/layer2_attention_weight_min": -31.114856719970703,
      "activations/layer3_attention_weight_max": 89.22007751464844,
      "activations/layer3_attention_weight_min": -90.38819122314453,
      "activations/layer4_attention_weight_max": 84.44438934326172,
      "activations/layer4_attention_weight_min": -82.81895446777344,
      "activations/layer5_attention_weight_max": 64.57581329345703,
      "activations/layer5_attention_weight_min": -73.24224090576172,
      "activations/layer6_attention_weight_max": 49.45321273803711,
      "activations/layer6_attention_weight_min": -50.514095306396484,
      "activations/layer7_attention_weight_max": 65.14204406738281,
      "activations/layer7_attention_weight_min": -69.86869812011719,
      "activations/layer8_attention_weight_max": 46.8677864074707,
      "activations/layer8_attention_weight_min": -53.89351272583008,
      "activations/layer9_attention_weight_max": 42.765804290771484,
      "activations/layer9_attention_weight_min": -46.68625259399414,
      "epoch": 22.77,
      "learning_rate": 3.167424242424242e-06,
      "loss": 2.6948,
      "step": 391950
    },
    {
      "activations/layer0_attention_weight_max": 16.028024673461914,
      "activations/layer0_attention_weight_min": -14.353145599365234,
      "activations/layer10_attention_weight_max": 42.2413215637207,
      "activations/layer10_attention_weight_min": -41.19532775878906,
      "activations/layer11_attention_weight_max": 38.198265075683594,
      "activations/layer11_attention_weight_min": -38.946048736572266,
      "activations/layer12_attention_weight_max": 20.74344825744629,
      "activations/layer12_attention_weight_min": -26.322168350219727,
      "activations/layer13_attention_weight_max": 32.08967208862305,
      "activations/layer13_attention_weight_min": -28.359058380126953,
      "activations/layer14_attention_weight_max": 34.0599250793457,
      "activations/layer14_attention_weight_min": -31.316255569458008,
      "activations/layer15_attention_weight_max": 33.85663604736328,
      "activations/layer15_attention_weight_min": -30.297590255737305,
      "activations/layer16_attention_weight_max": 34.768985748291016,
      "activations/layer16_attention_weight_min": -33.40812301635742,
      "activations/layer17_attention_weight_max": 54.11936569213867,
      "activations/layer17_attention_weight_min": -45.20418167114258,
      "activations/layer18_attention_weight_max": 47.44742202758789,
      "activations/layer18_attention_weight_min": -39.06675720214844,
      "activations/layer19_attention_weight_max": 24.766864776611328,
      "activations/layer19_attention_weight_min": -24.418529510498047,
      "activations/layer1_attention_weight_max": 15.56961727142334,
      "activations/layer1_attention_weight_min": -16.131103515625,
      "activations/layer20_attention_weight_max": 22.34128761291504,
      "activations/layer20_attention_weight_min": -22.17011260986328,
      "activations/layer21_attention_weight_max": 35.770633697509766,
      "activations/layer21_attention_weight_min": -27.878936767578125,
      "activations/layer22_attention_weight_max": 27.323476791381836,
      "activations/layer22_attention_weight_min": -25.393634796142578,
      "activations/layer23_attention_weight_max": 33.62779998779297,
      "activations/layer23_attention_weight_min": -26.086301803588867,
      "activations/layer2_attention_weight_max": 32.250328063964844,
      "activations/layer2_attention_weight_min": -32.01241683959961,
      "activations/layer3_attention_weight_max": 96.39374542236328,
      "activations/layer3_attention_weight_min": -92.41194915771484,
      "activations/layer4_attention_weight_max": 91.93009185791016,
      "activations/layer4_attention_weight_min": -90.42842102050781,
      "activations/layer5_attention_weight_max": 62.79045104980469,
      "activations/layer5_attention_weight_min": -74.71427917480469,
      "activations/layer6_attention_weight_max": 54.27261734008789,
      "activations/layer6_attention_weight_min": -53.22160339355469,
      "activations/layer7_attention_weight_max": 73.87689971923828,
      "activations/layer7_attention_weight_min": -69.99364471435547,
      "activations/layer8_attention_weight_max": 52.924949645996094,
      "activations/layer8_attention_weight_min": -54.58821487426758,
      "activations/layer9_attention_weight_max": 46.71603012084961,
      "activations/layer9_attention_weight_min": -47.79288864135742,
      "epoch": 22.78,
      "learning_rate": 3.148484848484848e-06,
      "loss": 2.7115,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4942,
      "eval_samples_per_second": 505.524,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4942,
      "eval_openwebtext_samples_per_second": 505.524,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_wikitext_loss": 2.900390625,
      "eval_wikitext_ppl": 18.181246031735814,
      "eval_wikitext_runtime": 1.9417,
      "eval_wikitext_samples_per_second": 234.848,
      "step": 392000
    },
    {
      "epoch": 22.78,
      "eval_lambada_loss": 2.384765625,
      "eval_lambada_ppl": 10.85651787033819,
      "eval_lambada_runtime": 9.5371,
      "eval_lambada_samples_per_second": 510.534,
      "step": 392000
    },
    {
      "activations/layer0_attention_weight_max": 15.573745727539062,
      "activations/layer0_attention_weight_min": -14.4064302444458,
      "activations/layer10_attention_weight_max": 38.839622497558594,
      "activations/layer10_attention_weight_min": -38.32966995239258,
      "activations/layer11_attention_weight_max": 34.71491241455078,
      "activations/layer11_attention_weight_min": -37.920013427734375,
      "activations/layer12_attention_weight_max": 20.14742088317871,
      "activations/layer12_attention_weight_min": -25.713777542114258,
      "activations/layer13_attention_weight_max": 30.959686279296875,
      "activations/layer13_attention_weight_min": -27.385326385498047,
      "activations/layer14_attention_weight_max": 35.79254150390625,
      "activations/layer14_attention_weight_min": -31.354236602783203,
      "activations/layer15_attention_weight_max": 32.70745849609375,
      "activations/layer15_attention_weight_min": -31.472763061523438,
      "activations/layer16_attention_weight_max": 34.91244888305664,
      "activations/layer16_attention_weight_min": -34.23045349121094,
      "activations/layer17_attention_weight_max": 52.350887298583984,
      "activations/layer17_attention_weight_min": -47.8669319152832,
      "activations/layer18_attention_weight_max": 48.99833679199219,
      "activations/layer18_attention_weight_min": -42.57848358154297,
      "activations/layer19_attention_weight_max": 24.357093811035156,
      "activations/layer19_attention_weight_min": -25.504289627075195,
      "activations/layer1_attention_weight_max": 15.684305191040039,
      "activations/layer1_attention_weight_min": -14.894615173339844,
      "activations/layer20_attention_weight_max": 22.115434646606445,
      "activations/layer20_attention_weight_min": -25.298276901245117,
      "activations/layer21_attention_weight_max": 36.08819580078125,
      "activations/layer21_attention_weight_min": -21.27781105041504,
      "activations/layer22_attention_weight_max": 30.7259578704834,
      "activations/layer22_attention_weight_min": -25.00872802734375,
      "activations/layer23_attention_weight_max": 35.67707824707031,
      "activations/layer23_attention_weight_min": -23.793601989746094,
      "activations/layer2_attention_weight_max": 32.13384246826172,
      "activations/layer2_attention_weight_min": -30.370365142822266,
      "activations/layer3_attention_weight_max": 91.37577819824219,
      "activations/layer3_attention_weight_min": -91.39295959472656,
      "activations/layer4_attention_weight_max": 87.26604461669922,
      "activations/layer4_attention_weight_min": -87.45049285888672,
      "activations/layer5_attention_weight_max": 65.50586700439453,
      "activations/layer5_attention_weight_min": -68.50869750976562,
      "activations/layer6_attention_weight_max": 49.97618103027344,
      "activations/layer6_attention_weight_min": -54.34275436401367,
      "activations/layer7_attention_weight_max": 75.59346008300781,
      "activations/layer7_attention_weight_min": -76.05851745605469,
      "activations/layer8_attention_weight_max": 51.245574951171875,
      "activations/layer8_attention_weight_min": -55.29914855957031,
      "activations/layer9_attention_weight_max": 46.474700927734375,
      "activations/layer9_attention_weight_min": -50.2657585144043,
      "epoch": 22.78,
      "learning_rate": 3.129545454545454e-06,
      "loss": 2.6968,
      "step": 392050
    },
    {
      "activations/layer0_attention_weight_max": 16.430788040161133,
      "activations/layer0_attention_weight_min": -14.300555229187012,
      "activations/layer10_attention_weight_max": 40.174560546875,
      "activations/layer10_attention_weight_min": -38.34412384033203,
      "activations/layer11_attention_weight_max": 35.43224334716797,
      "activations/layer11_attention_weight_min": -35.1494255065918,
      "activations/layer12_attention_weight_max": 19.885513305664062,
      "activations/layer12_attention_weight_min": -25.833532333374023,
      "activations/layer13_attention_weight_max": 33.29730224609375,
      "activations/layer13_attention_weight_min": -28.55216407775879,
      "activations/layer14_attention_weight_max": 34.58306884765625,
      "activations/layer14_attention_weight_min": -29.470623016357422,
      "activations/layer15_attention_weight_max": 34.618595123291016,
      "activations/layer15_attention_weight_min": -28.795740127563477,
      "activations/layer16_attention_weight_max": 34.642601013183594,
      "activations/layer16_attention_weight_min": -32.649391174316406,
      "activations/layer17_attention_weight_max": 50.79787063598633,
      "activations/layer17_attention_weight_min": -45.374488830566406,
      "activations/layer18_attention_weight_max": 48.94779586791992,
      "activations/layer18_attention_weight_min": -39.84056854248047,
      "activations/layer19_attention_weight_max": 24.254898071289062,
      "activations/layer19_attention_weight_min": -22.8265438079834,
      "activations/layer1_attention_weight_max": 14.449365615844727,
      "activations/layer1_attention_weight_min": -15.307558059692383,
      "activations/layer20_attention_weight_max": 22.512544631958008,
      "activations/layer20_attention_weight_min": -22.11219024658203,
      "activations/layer21_attention_weight_max": 32.834693908691406,
      "activations/layer21_attention_weight_min": -19.862594604492188,
      "activations/layer22_attention_weight_max": 29.530624389648438,
      "activations/layer22_attention_weight_min": -24.28941535949707,
      "activations/layer23_attention_weight_max": 34.37610626220703,
      "activations/layer23_attention_weight_min": -24.37258529663086,
      "activations/layer2_attention_weight_max": 31.693218231201172,
      "activations/layer2_attention_weight_min": -30.756877899169922,
      "activations/layer3_attention_weight_max": 93.61463928222656,
      "activations/layer3_attention_weight_min": -92.73353576660156,
      "activations/layer4_attention_weight_max": 87.99449157714844,
      "activations/layer4_attention_weight_min": -90.0008773803711,
      "activations/layer5_attention_weight_max": 62.22686004638672,
      "activations/layer5_attention_weight_min": -71.48635864257812,
      "activations/layer6_attention_weight_max": 49.43431854248047,
      "activations/layer6_attention_weight_min": -49.68748092651367,
      "activations/layer7_attention_weight_max": 69.57636260986328,
      "activations/layer7_attention_weight_min": -69.75216674804688,
      "activations/layer8_attention_weight_max": 47.57386779785156,
      "activations/layer8_attention_weight_min": -52.968318939208984,
      "activations/layer9_attention_weight_max": 50.82712936401367,
      "activations/layer9_attention_weight_min": -51.396217346191406,
      "epoch": 22.78,
      "learning_rate": 3.1106060606060604e-06,
      "loss": 2.6867,
      "step": 392100
    },
    {
      "activations/layer0_attention_weight_max": 14.995858192443848,
      "activations/layer0_attention_weight_min": -14.280860900878906,
      "activations/layer10_attention_weight_max": 39.14988708496094,
      "activations/layer10_attention_weight_min": -40.7117805480957,
      "activations/layer11_attention_weight_max": 34.99982833862305,
      "activations/layer11_attention_weight_min": -35.35673904418945,
      "activations/layer12_attention_weight_max": 20.698612213134766,
      "activations/layer12_attention_weight_min": -26.85021209716797,
      "activations/layer13_attention_weight_max": 37.52291488647461,
      "activations/layer13_attention_weight_min": -27.93545913696289,
      "activations/layer14_attention_weight_max": 36.33224105834961,
      "activations/layer14_attention_weight_min": -31.409587860107422,
      "activations/layer15_attention_weight_max": 34.39683532714844,
      "activations/layer15_attention_weight_min": -31.881437301635742,
      "activations/layer16_attention_weight_max": 36.4896354675293,
      "activations/layer16_attention_weight_min": -33.53997039794922,
      "activations/layer17_attention_weight_max": 51.562137603759766,
      "activations/layer17_attention_weight_min": -45.62544250488281,
      "activations/layer18_attention_weight_max": 47.7052116394043,
      "activations/layer18_attention_weight_min": -42.60243225097656,
      "activations/layer19_attention_weight_max": 24.13224983215332,
      "activations/layer19_attention_weight_min": -22.066692352294922,
      "activations/layer1_attention_weight_max": 14.62199878692627,
      "activations/layer1_attention_weight_min": -14.923556327819824,
      "activations/layer20_attention_weight_max": 22.74608612060547,
      "activations/layer20_attention_weight_min": -22.995500564575195,
      "activations/layer21_attention_weight_max": 35.605255126953125,
      "activations/layer21_attention_weight_min": -22.592323303222656,
      "activations/layer22_attention_weight_max": 30.647478103637695,
      "activations/layer22_attention_weight_min": -24.598468780517578,
      "activations/layer23_attention_weight_max": 40.64207077026367,
      "activations/layer23_attention_weight_min": -23.423568725585938,
      "activations/layer2_attention_weight_max": 34.39103317260742,
      "activations/layer2_attention_weight_min": -30.646278381347656,
      "activations/layer3_attention_weight_max": 93.83330535888672,
      "activations/layer3_attention_weight_min": -91.80752563476562,
      "activations/layer4_attention_weight_max": 84.94634246826172,
      "activations/layer4_attention_weight_min": -86.08558654785156,
      "activations/layer5_attention_weight_max": 65.11517333984375,
      "activations/layer5_attention_weight_min": -73.5603256225586,
      "activations/layer6_attention_weight_max": 50.50925827026367,
      "activations/layer6_attention_weight_min": -50.97734832763672,
      "activations/layer7_attention_weight_max": 70.45354461669922,
      "activations/layer7_attention_weight_min": -73.02360534667969,
      "activations/layer8_attention_weight_max": 49.96348190307617,
      "activations/layer8_attention_weight_min": -55.32902908325195,
      "activations/layer9_attention_weight_max": 46.13763427734375,
      "activations/layer9_attention_weight_min": -48.49376678466797,
      "epoch": 22.79,
      "learning_rate": 3.0916666666666666e-06,
      "loss": 2.6953,
      "step": 392150
    },
    {
      "activations/layer0_attention_weight_max": 16.22699546813965,
      "activations/layer0_attention_weight_min": -14.235697746276855,
      "activations/layer10_attention_weight_max": 38.2586555480957,
      "activations/layer10_attention_weight_min": -36.89341735839844,
      "activations/layer11_attention_weight_max": 35.81612014770508,
      "activations/layer11_attention_weight_min": -38.269432067871094,
      "activations/layer12_attention_weight_max": 18.757583618164062,
      "activations/layer12_attention_weight_min": -28.31818199157715,
      "activations/layer13_attention_weight_max": 33.88972091674805,
      "activations/layer13_attention_weight_min": -26.704721450805664,
      "activations/layer14_attention_weight_max": 32.876766204833984,
      "activations/layer14_attention_weight_min": -28.15799903869629,
      "activations/layer15_attention_weight_max": 32.423465728759766,
      "activations/layer15_attention_weight_min": -30.084308624267578,
      "activations/layer16_attention_weight_max": 32.37385940551758,
      "activations/layer16_attention_weight_min": -33.301456451416016,
      "activations/layer17_attention_weight_max": 47.044273376464844,
      "activations/layer17_attention_weight_min": -45.13591384887695,
      "activations/layer18_attention_weight_max": 43.98795700073242,
      "activations/layer18_attention_weight_min": -41.26582336425781,
      "activations/layer19_attention_weight_max": 23.66913414001465,
      "activations/layer19_attention_weight_min": -25.38942527770996,
      "activations/layer1_attention_weight_max": 14.906170845031738,
      "activations/layer1_attention_weight_min": -14.431376457214355,
      "activations/layer20_attention_weight_max": 23.57756805419922,
      "activations/layer20_attention_weight_min": -21.860973358154297,
      "activations/layer21_attention_weight_max": 38.42302322387695,
      "activations/layer21_attention_weight_min": -24.971261978149414,
      "activations/layer22_attention_weight_max": 26.870948791503906,
      "activations/layer22_attention_weight_min": -26.61420440673828,
      "activations/layer23_attention_weight_max": 36.59814453125,
      "activations/layer23_attention_weight_min": -27.312114715576172,
      "activations/layer2_attention_weight_max": 32.83032989501953,
      "activations/layer2_attention_weight_min": -31.81220054626465,
      "activations/layer3_attention_weight_max": 94.6344985961914,
      "activations/layer3_attention_weight_min": -95.30864715576172,
      "activations/layer4_attention_weight_max": 91.11956787109375,
      "activations/layer4_attention_weight_min": -88.99807739257812,
      "activations/layer5_attention_weight_max": 63.562252044677734,
      "activations/layer5_attention_weight_min": -73.72489166259766,
      "activations/layer6_attention_weight_max": 53.81700134277344,
      "activations/layer6_attention_weight_min": -54.847572326660156,
      "activations/layer7_attention_weight_max": 73.16336822509766,
      "activations/layer7_attention_weight_min": -70.77954864501953,
      "activations/layer8_attention_weight_max": 51.72288131713867,
      "activations/layer8_attention_weight_min": -53.3698616027832,
      "activations/layer9_attention_weight_max": 48.544090270996094,
      "activations/layer9_attention_weight_min": -48.541725158691406,
      "epoch": 22.79,
      "learning_rate": 3.0727272727272723e-06,
      "loss": 2.7128,
      "step": 392200
    },
    {
      "activations/layer0_attention_weight_max": 15.304975509643555,
      "activations/layer0_attention_weight_min": -14.359358787536621,
      "activations/layer10_attention_weight_max": 35.34528350830078,
      "activations/layer10_attention_weight_min": -34.8817138671875,
      "activations/layer11_attention_weight_max": 33.10072326660156,
      "activations/layer11_attention_weight_min": -35.66870880126953,
      "activations/layer12_attention_weight_max": 18.8544921875,
      "activations/layer12_attention_weight_min": -27.433490753173828,
      "activations/layer13_attention_weight_max": 31.436481475830078,
      "activations/layer13_attention_weight_min": -28.591094970703125,
      "activations/layer14_attention_weight_max": 33.99385452270508,
      "activations/layer14_attention_weight_min": -30.636451721191406,
      "activations/layer15_attention_weight_max": 32.427547454833984,
      "activations/layer15_attention_weight_min": -31.82257080078125,
      "activations/layer16_attention_weight_max": 32.91071319580078,
      "activations/layer16_attention_weight_min": -31.583833694458008,
      "activations/layer17_attention_weight_max": 53.15465545654297,
      "activations/layer17_attention_weight_min": -45.39476013183594,
      "activations/layer18_attention_weight_max": 44.73247146606445,
      "activations/layer18_attention_weight_min": -41.01291275024414,
      "activations/layer19_attention_weight_max": 25.467580795288086,
      "activations/layer19_attention_weight_min": -23.88324546813965,
      "activations/layer1_attention_weight_max": 15.343215942382812,
      "activations/layer1_attention_weight_min": -17.003170013427734,
      "activations/layer20_attention_weight_max": 23.440080642700195,
      "activations/layer20_attention_weight_min": -21.728595733642578,
      "activations/layer21_attention_weight_max": 32.83317947387695,
      "activations/layer21_attention_weight_min": -23.094839096069336,
      "activations/layer22_attention_weight_max": 27.57324981689453,
      "activations/layer22_attention_weight_min": -24.97579002380371,
      "activations/layer23_attention_weight_max": 35.01715850830078,
      "activations/layer23_attention_weight_min": -27.91396141052246,
      "activations/layer2_attention_weight_max": 31.638927459716797,
      "activations/layer2_attention_weight_min": -30.906538009643555,
      "activations/layer3_attention_weight_max": 91.24275207519531,
      "activations/layer3_attention_weight_min": -91.44064331054688,
      "activations/layer4_attention_weight_max": 88.672119140625,
      "activations/layer4_attention_weight_min": -89.71346282958984,
      "activations/layer5_attention_weight_max": 63.45545959472656,
      "activations/layer5_attention_weight_min": -73.84121704101562,
      "activations/layer6_attention_weight_max": 49.895118713378906,
      "activations/layer6_attention_weight_min": -52.65901184082031,
      "activations/layer7_attention_weight_max": 72.64137268066406,
      "activations/layer7_attention_weight_min": -68.82418823242188,
      "activations/layer8_attention_weight_max": 46.49855422973633,
      "activations/layer8_attention_weight_min": -51.26460647583008,
      "activations/layer9_attention_weight_max": 44.965457916259766,
      "activations/layer9_attention_weight_min": -47.489017486572266,
      "epoch": 22.79,
      "learning_rate": 3.0537878787878785e-06,
      "loss": 2.7066,
      "step": 392250
    },
    {
      "activations/layer0_attention_weight_max": 16.225034713745117,
      "activations/layer0_attention_weight_min": -14.050716400146484,
      "activations/layer10_attention_weight_max": 37.77632522583008,
      "activations/layer10_attention_weight_min": -36.74214553833008,
      "activations/layer11_attention_weight_max": 35.65227127075195,
      "activations/layer11_attention_weight_min": -36.551292419433594,
      "activations/layer12_attention_weight_max": 21.522253036499023,
      "activations/layer12_attention_weight_min": -25.117080688476562,
      "activations/layer13_attention_weight_max": 36.19124984741211,
      "activations/layer13_attention_weight_min": -27.42537498474121,
      "activations/layer14_attention_weight_max": 38.100799560546875,
      "activations/layer14_attention_weight_min": -30.771663665771484,
      "activations/layer15_attention_weight_max": 36.12086486816406,
      "activations/layer15_attention_weight_min": -30.909053802490234,
      "activations/layer16_attention_weight_max": 40.593196868896484,
      "activations/layer16_attention_weight_min": -36.7565803527832,
      "activations/layer17_attention_weight_max": 54.570274353027344,
      "activations/layer17_attention_weight_min": -49.432308197021484,
      "activations/layer18_attention_weight_max": 53.98960494995117,
      "activations/layer18_attention_weight_min": -45.91691207885742,
      "activations/layer19_attention_weight_max": 25.997989654541016,
      "activations/layer19_attention_weight_min": -23.267898559570312,
      "activations/layer1_attention_weight_max": 14.48376178741455,
      "activations/layer1_attention_weight_min": -14.6316499710083,
      "activations/layer20_attention_weight_max": 23.882305145263672,
      "activations/layer20_attention_weight_min": -21.760330200195312,
      "activations/layer21_attention_weight_max": 38.285743713378906,
      "activations/layer21_attention_weight_min": -21.03858184814453,
      "activations/layer22_attention_weight_max": 30.34532928466797,
      "activations/layer22_attention_weight_min": -25.04233169555664,
      "activations/layer23_attention_weight_max": 38.036888122558594,
      "activations/layer23_attention_weight_min": -25.14096450805664,
      "activations/layer2_attention_weight_max": 32.02904510498047,
      "activations/layer2_attention_weight_min": -32.66533660888672,
      "activations/layer3_attention_weight_max": 94.48847198486328,
      "activations/layer3_attention_weight_min": -98.11106872558594,
      "activations/layer4_attention_weight_max": 89.74060821533203,
      "activations/layer4_attention_weight_min": -88.70024871826172,
      "activations/layer5_attention_weight_max": 64.92024230957031,
      "activations/layer5_attention_weight_min": -76.25399780273438,
      "activations/layer6_attention_weight_max": 51.80531311035156,
      "activations/layer6_attention_weight_min": -51.83143615722656,
      "activations/layer7_attention_weight_max": 77.34978485107422,
      "activations/layer7_attention_weight_min": -74.84852600097656,
      "activations/layer8_attention_weight_max": 49.47157287597656,
      "activations/layer8_attention_weight_min": -52.83142852783203,
      "activations/layer9_attention_weight_max": 47.25177764892578,
      "activations/layer9_attention_weight_min": -50.33837127685547,
      "epoch": 22.79,
      "learning_rate": 3.0348484848484846e-06,
      "loss": 2.7007,
      "step": 392300
    },
    {
      "activations/layer0_attention_weight_max": 15.784431457519531,
      "activations/layer0_attention_weight_min": -13.935601234436035,
      "activations/layer10_attention_weight_max": 39.142120361328125,
      "activations/layer10_attention_weight_min": -38.10334396362305,
      "activations/layer11_attention_weight_max": 35.91706848144531,
      "activations/layer11_attention_weight_min": -36.83692932128906,
      "activations/layer12_attention_weight_max": 20.252338409423828,
      "activations/layer12_attention_weight_min": -23.84935760498047,
      "activations/layer13_attention_weight_max": 38.652915954589844,
      "activations/layer13_attention_weight_min": -29.45839500427246,
      "activations/layer14_attention_weight_max": 38.759185791015625,
      "activations/layer14_attention_weight_min": -34.956817626953125,
      "activations/layer15_attention_weight_max": 38.37645721435547,
      "activations/layer15_attention_weight_min": -36.74850845336914,
      "activations/layer16_attention_weight_max": 37.08102798461914,
      "activations/layer16_attention_weight_min": -34.54336929321289,
      "activations/layer17_attention_weight_max": 54.86348342895508,
      "activations/layer17_attention_weight_min": -49.42793655395508,
      "activations/layer18_attention_weight_max": 50.87318420410156,
      "activations/layer18_attention_weight_min": -44.56984329223633,
      "activations/layer19_attention_weight_max": 29.916296005249023,
      "activations/layer19_attention_weight_min": -25.873754501342773,
      "activations/layer1_attention_weight_max": 14.658788681030273,
      "activations/layer1_attention_weight_min": -15.714670181274414,
      "activations/layer20_attention_weight_max": 27.500459671020508,
      "activations/layer20_attention_weight_min": -23.26019287109375,
      "activations/layer21_attention_weight_max": 47.79590606689453,
      "activations/layer21_attention_weight_min": -26.15239143371582,
      "activations/layer22_attention_weight_max": 32.049049377441406,
      "activations/layer22_attention_weight_min": -26.43086051940918,
      "activations/layer23_attention_weight_max": 42.241641998291016,
      "activations/layer23_attention_weight_min": -25.872974395751953,
      "activations/layer2_attention_weight_max": 32.042789459228516,
      "activations/layer2_attention_weight_min": -30.961563110351562,
      "activations/layer3_attention_weight_max": 95.70533752441406,
      "activations/layer3_attention_weight_min": -94.05427551269531,
      "activations/layer4_attention_weight_max": 87.9476547241211,
      "activations/layer4_attention_weight_min": -86.81832122802734,
      "activations/layer5_attention_weight_max": 63.712032318115234,
      "activations/layer5_attention_weight_min": -74.36979675292969,
      "activations/layer6_attention_weight_max": 51.42787170410156,
      "activations/layer6_attention_weight_min": -53.13203048706055,
      "activations/layer7_attention_weight_max": 67.9156494140625,
      "activations/layer7_attention_weight_min": -72.40888214111328,
      "activations/layer8_attention_weight_max": 49.6246337890625,
      "activations/layer8_attention_weight_min": -51.80903625488281,
      "activations/layer9_attention_weight_max": 45.624820709228516,
      "activations/layer9_attention_weight_min": -50.19284439086914,
      "epoch": 22.8,
      "learning_rate": 3.0159090909090908e-06,
      "loss": 2.7132,
      "step": 392350
    },
    {
      "activations/layer0_attention_weight_max": 16.119531631469727,
      "activations/layer0_attention_weight_min": -13.958141326904297,
      "activations/layer10_attention_weight_max": 35.11333465576172,
      "activations/layer10_attention_weight_min": -34.63816833496094,
      "activations/layer11_attention_weight_max": 32.56370544433594,
      "activations/layer11_attention_weight_min": -36.21514129638672,
      "activations/layer12_attention_weight_max": 19.797779083251953,
      "activations/layer12_attention_weight_min": -26.272382736206055,
      "activations/layer13_attention_weight_max": 31.52671241760254,
      "activations/layer13_attention_weight_min": -26.995439529418945,
      "activations/layer14_attention_weight_max": 36.082984924316406,
      "activations/layer14_attention_weight_min": -31.013809204101562,
      "activations/layer15_attention_weight_max": 34.765342712402344,
      "activations/layer15_attention_weight_min": -29.883501052856445,
      "activations/layer16_attention_weight_max": 33.36043930053711,
      "activations/layer16_attention_weight_min": -33.22019577026367,
      "activations/layer17_attention_weight_max": 49.45762252807617,
      "activations/layer17_attention_weight_min": -44.883766174316406,
      "activations/layer18_attention_weight_max": 45.63423156738281,
      "activations/layer18_attention_weight_min": -40.13876724243164,
      "activations/layer19_attention_weight_max": 31.71758460998535,
      "activations/layer19_attention_weight_min": -26.4669132232666,
      "activations/layer1_attention_weight_max": 14.646281242370605,
      "activations/layer1_attention_weight_min": -14.271130561828613,
      "activations/layer20_attention_weight_max": 23.85538101196289,
      "activations/layer20_attention_weight_min": -23.224002838134766,
      "activations/layer21_attention_weight_max": 35.9146614074707,
      "activations/layer21_attention_weight_min": -24.893844604492188,
      "activations/layer22_attention_weight_max": 36.408294677734375,
      "activations/layer22_attention_weight_min": -26.65281105041504,
      "activations/layer23_attention_weight_max": 37.41791534423828,
      "activations/layer23_attention_weight_min": -26.163381576538086,
      "activations/layer2_attention_weight_max": 32.22632598876953,
      "activations/layer2_attention_weight_min": -31.20210075378418,
      "activations/layer3_attention_weight_max": 91.30915832519531,
      "activations/layer3_attention_weight_min": -94.76437377929688,
      "activations/layer4_attention_weight_max": 85.00103759765625,
      "activations/layer4_attention_weight_min": -87.52851867675781,
      "activations/layer5_attention_weight_max": 61.30363082885742,
      "activations/layer5_attention_weight_min": -71.52513885498047,
      "activations/layer6_attention_weight_max": 48.52787399291992,
      "activations/layer6_attention_weight_min": -50.09386444091797,
      "activations/layer7_attention_weight_max": 70.5827407836914,
      "activations/layer7_attention_weight_min": -70.69760131835938,
      "activations/layer8_attention_weight_max": 47.5097770690918,
      "activations/layer8_attention_weight_min": -49.147605895996094,
      "activations/layer9_attention_weight_max": 44.64963912963867,
      "activations/layer9_attention_weight_min": -45.912532806396484,
      "epoch": 22.8,
      "learning_rate": 2.9969696969696965e-06,
      "loss": 2.698,
      "step": 392400
    },
    {
      "activations/layer0_attention_weight_max": 14.933066368103027,
      "activations/layer0_attention_weight_min": -14.531975746154785,
      "activations/layer10_attention_weight_max": 37.13654327392578,
      "activations/layer10_attention_weight_min": -38.06257629394531,
      "activations/layer11_attention_weight_max": 35.34693145751953,
      "activations/layer11_attention_weight_min": -36.759559631347656,
      "activations/layer12_attention_weight_max": 20.145633697509766,
      "activations/layer12_attention_weight_min": -24.241146087646484,
      "activations/layer13_attention_weight_max": 35.450984954833984,
      "activations/layer13_attention_weight_min": -29.51784896850586,
      "activations/layer14_attention_weight_max": 35.66331481933594,
      "activations/layer14_attention_weight_min": -30.784509658813477,
      "activations/layer15_attention_weight_max": 34.605018615722656,
      "activations/layer15_attention_weight_min": -30.56197738647461,
      "activations/layer16_attention_weight_max": 35.49643325805664,
      "activations/layer16_attention_weight_min": -33.19807052612305,
      "activations/layer17_attention_weight_max": 52.66583251953125,
      "activations/layer17_attention_weight_min": -44.98509979248047,
      "activations/layer18_attention_weight_max": 47.26966857910156,
      "activations/layer18_attention_weight_min": -44.03064727783203,
      "activations/layer19_attention_weight_max": 24.681493759155273,
      "activations/layer19_attention_weight_min": -26.277694702148438,
      "activations/layer1_attention_weight_max": 14.912642478942871,
      "activations/layer1_attention_weight_min": -13.975850105285645,
      "activations/layer20_attention_weight_max": 22.80324935913086,
      "activations/layer20_attention_weight_min": -24.72038459777832,
      "activations/layer21_attention_weight_max": 38.701454162597656,
      "activations/layer21_attention_weight_min": -30.240385055541992,
      "activations/layer22_attention_weight_max": 28.254093170166016,
      "activations/layer22_attention_weight_min": -29.177087783813477,
      "activations/layer23_attention_weight_max": 39.71443557739258,
      "activations/layer23_attention_weight_min": -28.948530197143555,
      "activations/layer2_attention_weight_max": 31.762710571289062,
      "activations/layer2_attention_weight_min": -30.125988006591797,
      "activations/layer3_attention_weight_max": 89.14572143554688,
      "activations/layer3_attention_weight_min": -91.29747772216797,
      "activations/layer4_attention_weight_max": 87.49951171875,
      "activations/layer4_attention_weight_min": -90.5183334350586,
      "activations/layer5_attention_weight_max": 66.1903305053711,
      "activations/layer5_attention_weight_min": -71.75466918945312,
      "activations/layer6_attention_weight_max": 50.24330520629883,
      "activations/layer6_attention_weight_min": -50.96506118774414,
      "activations/layer7_attention_weight_max": 76.81033325195312,
      "activations/layer7_attention_weight_min": -70.62201690673828,
      "activations/layer8_attention_weight_max": 49.83465576171875,
      "activations/layer8_attention_weight_min": -53.40333938598633,
      "activations/layer9_attention_weight_max": 47.113914489746094,
      "activations/layer9_attention_weight_min": -48.82861328125,
      "epoch": 22.8,
      "learning_rate": 2.9780303030303026e-06,
      "loss": 2.7136,
      "step": 392450
    },
    {
      "activations/layer0_attention_weight_max": 15.482305526733398,
      "activations/layer0_attention_weight_min": -14.265429496765137,
      "activations/layer10_attention_weight_max": 38.019081115722656,
      "activations/layer10_attention_weight_min": -38.768802642822266,
      "activations/layer11_attention_weight_max": 35.115760803222656,
      "activations/layer11_attention_weight_min": -36.596195220947266,
      "activations/layer12_attention_weight_max": 21.3856143951416,
      "activations/layer12_attention_weight_min": -24.543914794921875,
      "activations/layer13_attention_weight_max": 34.73815155029297,
      "activations/layer13_attention_weight_min": -28.3094425201416,
      "activations/layer14_attention_weight_max": 37.078956604003906,
      "activations/layer14_attention_weight_min": -31.828964233398438,
      "activations/layer15_attention_weight_max": 36.94945526123047,
      "activations/layer15_attention_weight_min": -31.69361114501953,
      "activations/layer16_attention_weight_max": 37.44291687011719,
      "activations/layer16_attention_weight_min": -36.014068603515625,
      "activations/layer17_attention_weight_max": 56.102752685546875,
      "activations/layer17_attention_weight_min": -50.77571487426758,
      "activations/layer18_attention_weight_max": 53.42578125,
      "activations/layer18_attention_weight_min": -44.99680709838867,
      "activations/layer19_attention_weight_max": 29.464405059814453,
      "activations/layer19_attention_weight_min": -25.6121883392334,
      "activations/layer1_attention_weight_max": 15.184704780578613,
      "activations/layer1_attention_weight_min": -15.36941909790039,
      "activations/layer20_attention_weight_max": 28.44215965270996,
      "activations/layer20_attention_weight_min": -21.731733322143555,
      "activations/layer21_attention_weight_max": 43.116790771484375,
      "activations/layer21_attention_weight_min": -27.840984344482422,
      "activations/layer22_attention_weight_max": 35.2835807800293,
      "activations/layer22_attention_weight_min": -25.29326629638672,
      "activations/layer23_attention_weight_max": 39.54753494262695,
      "activations/layer23_attention_weight_min": -27.378883361816406,
      "activations/layer2_attention_weight_max": 32.3099250793457,
      "activations/layer2_attention_weight_min": -31.121694564819336,
      "activations/layer3_attention_weight_max": 93.27880859375,
      "activations/layer3_attention_weight_min": -92.13218688964844,
      "activations/layer4_attention_weight_max": 88.37752532958984,
      "activations/layer4_attention_weight_min": -88.53801727294922,
      "activations/layer5_attention_weight_max": 61.695823669433594,
      "activations/layer5_attention_weight_min": -80.63085174560547,
      "activations/layer6_attention_weight_max": 53.03207778930664,
      "activations/layer6_attention_weight_min": -52.349205017089844,
      "activations/layer7_attention_weight_max": 72.50887298583984,
      "activations/layer7_attention_weight_min": -68.2083740234375,
      "activations/layer8_attention_weight_max": 52.514747619628906,
      "activations/layer8_attention_weight_min": -53.81742858886719,
      "activations/layer9_attention_weight_max": 51.25042724609375,
      "activations/layer9_attention_weight_min": -48.59601593017578,
      "epoch": 22.81,
      "learning_rate": 2.9590909090909092e-06,
      "loss": 2.693,
      "step": 392500
    },
    {
      "activations/layer0_attention_weight_max": 15.433160781860352,
      "activations/layer0_attention_weight_min": -14.438336372375488,
      "activations/layer10_attention_weight_max": 37.88186264038086,
      "activations/layer10_attention_weight_min": -37.05515670776367,
      "activations/layer11_attention_weight_max": 37.58680725097656,
      "activations/layer11_attention_weight_min": -36.31557083129883,
      "activations/layer12_attention_weight_max": 20.000577926635742,
      "activations/layer12_attention_weight_min": -25.19835662841797,
      "activations/layer13_attention_weight_max": 35.147918701171875,
      "activations/layer13_attention_weight_min": -28.406047821044922,
      "activations/layer14_attention_weight_max": 35.581825256347656,
      "activations/layer14_attention_weight_min": -30.150245666503906,
      "activations/layer15_attention_weight_max": 34.39806365966797,
      "activations/layer15_attention_weight_min": -30.724796295166016,
      "activations/layer16_attention_weight_max": 35.66349411010742,
      "activations/layer16_attention_weight_min": -33.99639892578125,
      "activations/layer17_attention_weight_max": 51.98828125,
      "activations/layer17_attention_weight_min": -45.678382873535156,
      "activations/layer18_attention_weight_max": 46.285484313964844,
      "activations/layer18_attention_weight_min": -40.27085876464844,
      "activations/layer19_attention_weight_max": 26.417133331298828,
      "activations/layer19_attention_weight_min": -22.547950744628906,
      "activations/layer1_attention_weight_max": 14.759549140930176,
      "activations/layer1_attention_weight_min": -14.456059455871582,
      "activations/layer20_attention_weight_max": 23.33961296081543,
      "activations/layer20_attention_weight_min": -21.506193161010742,
      "activations/layer21_attention_weight_max": 33.64533996582031,
      "activations/layer21_attention_weight_min": -21.618009567260742,
      "activations/layer22_attention_weight_max": 27.09711265563965,
      "activations/layer22_attention_weight_min": -24.11138153076172,
      "activations/layer23_attention_weight_max": 36.19871520996094,
      "activations/layer23_attention_weight_min": -23.73654556274414,
      "activations/layer2_attention_weight_max": 31.101654052734375,
      "activations/layer2_attention_weight_min": -30.512069702148438,
      "activations/layer3_attention_weight_max": 92.70406341552734,
      "activations/layer3_attention_weight_min": -93.31523895263672,
      "activations/layer4_attention_weight_max": 91.2431869506836,
      "activations/layer4_attention_weight_min": -92.30935668945312,
      "activations/layer5_attention_weight_max": 64.6199722290039,
      "activations/layer5_attention_weight_min": -73.4271011352539,
      "activations/layer6_attention_weight_max": 53.359352111816406,
      "activations/layer6_attention_weight_min": -53.958213806152344,
      "activations/layer7_attention_weight_max": 73.10282135009766,
      "activations/layer7_attention_weight_min": -71.72457122802734,
      "activations/layer8_attention_weight_max": 49.84111785888672,
      "activations/layer8_attention_weight_min": -53.03342819213867,
      "activations/layer9_attention_weight_max": 45.44274139404297,
      "activations/layer9_attention_weight_min": -47.588897705078125,
      "epoch": 22.81,
      "learning_rate": 2.940151515151515e-06,
      "loss": 2.7081,
      "step": 392550
    },
    {
      "activations/layer0_attention_weight_max": 15.648694038391113,
      "activations/layer0_attention_weight_min": -13.937450408935547,
      "activations/layer10_attention_weight_max": 37.34519958496094,
      "activations/layer10_attention_weight_min": -37.30043411254883,
      "activations/layer11_attention_weight_max": 34.900474548339844,
      "activations/layer11_attention_weight_min": -36.73493957519531,
      "activations/layer12_attention_weight_max": 20.90086555480957,
      "activations/layer12_attention_weight_min": -23.551034927368164,
      "activations/layer13_attention_weight_max": 34.56884765625,
      "activations/layer13_attention_weight_min": -32.90904998779297,
      "activations/layer14_attention_weight_max": 36.48578643798828,
      "activations/layer14_attention_weight_min": -33.202178955078125,
      "activations/layer15_attention_weight_max": 36.13489532470703,
      "activations/layer15_attention_weight_min": -38.45158386230469,
      "activations/layer16_attention_weight_max": 38.01584243774414,
      "activations/layer16_attention_weight_min": -33.89127731323242,
      "activations/layer17_attention_weight_max": 55.6828498840332,
      "activations/layer17_attention_weight_min": -49.6214714050293,
      "activations/layer18_attention_weight_max": 52.81376647949219,
      "activations/layer18_attention_weight_min": -48.279052734375,
      "activations/layer19_attention_weight_max": 26.757455825805664,
      "activations/layer19_attention_weight_min": -24.379642486572266,
      "activations/layer1_attention_weight_max": 15.134322166442871,
      "activations/layer1_attention_weight_min": -13.813468933105469,
      "activations/layer20_attention_weight_max": 22.3691463470459,
      "activations/layer20_attention_weight_min": -24.565505981445312,
      "activations/layer21_attention_weight_max": 43.9945068359375,
      "activations/layer21_attention_weight_min": -24.67289924621582,
      "activations/layer22_attention_weight_max": 32.46859359741211,
      "activations/layer22_attention_weight_min": -25.87103843688965,
      "activations/layer23_attention_weight_max": 37.22282409667969,
      "activations/layer23_attention_weight_min": -23.088138580322266,
      "activations/layer2_attention_weight_max": 32.70008850097656,
      "activations/layer2_attention_weight_min": -30.58680534362793,
      "activations/layer3_attention_weight_max": 89.89082336425781,
      "activations/layer3_attention_weight_min": -87.88899230957031,
      "activations/layer4_attention_weight_max": 84.11897277832031,
      "activations/layer4_attention_weight_min": -86.27764129638672,
      "activations/layer5_attention_weight_max": 61.534027099609375,
      "activations/layer5_attention_weight_min": -71.1144027709961,
      "activations/layer6_attention_weight_max": 52.42528533935547,
      "activations/layer6_attention_weight_min": -51.22853088378906,
      "activations/layer7_attention_weight_max": 68.14421844482422,
      "activations/layer7_attention_weight_min": -76.25196838378906,
      "activations/layer8_attention_weight_max": 48.72228240966797,
      "activations/layer8_attention_weight_min": -52.57655334472656,
      "activations/layer9_attention_weight_max": 45.4644775390625,
      "activations/layer9_attention_weight_min": -50.24059295654297,
      "epoch": 22.81,
      "learning_rate": 2.921212121212121e-06,
      "loss": 2.6972,
      "step": 392600
    },
    {
      "activations/layer0_attention_weight_max": 14.80344009399414,
      "activations/layer0_attention_weight_min": -14.223334312438965,
      "activations/layer10_attention_weight_max": 36.57501983642578,
      "activations/layer10_attention_weight_min": -37.528785705566406,
      "activations/layer11_attention_weight_max": 33.83562469482422,
      "activations/layer11_attention_weight_min": -35.494056701660156,
      "activations/layer12_attention_weight_max": 20.921842575073242,
      "activations/layer12_attention_weight_min": -24.92945098876953,
      "activations/layer13_attention_weight_max": 31.637306213378906,
      "activations/layer13_attention_weight_min": -28.090232849121094,
      "activations/layer14_attention_weight_max": 36.344600677490234,
      "activations/layer14_attention_weight_min": -31.456623077392578,
      "activations/layer15_attention_weight_max": 33.81237030029297,
      "activations/layer15_attention_weight_min": -31.301218032836914,
      "activations/layer16_attention_weight_max": 34.70881271362305,
      "activations/layer16_attention_weight_min": -32.28006362915039,
      "activations/layer17_attention_weight_max": 52.777976989746094,
      "activations/layer17_attention_weight_min": -47.668601989746094,
      "activations/layer18_attention_weight_max": 45.359615325927734,
      "activations/layer18_attention_weight_min": -43.36317443847656,
      "activations/layer19_attention_weight_max": 23.714494705200195,
      "activations/layer19_attention_weight_min": -23.208127975463867,
      "activations/layer1_attention_weight_max": 15.417186737060547,
      "activations/layer1_attention_weight_min": -14.536131858825684,
      "activations/layer20_attention_weight_max": 24.303739547729492,
      "activations/layer20_attention_weight_min": -21.024930953979492,
      "activations/layer21_attention_weight_max": 39.59903335571289,
      "activations/layer21_attention_weight_min": -21.440221786499023,
      "activations/layer22_attention_weight_max": 31.179325103759766,
      "activations/layer22_attention_weight_min": -22.7655086517334,
      "activations/layer23_attention_weight_max": 35.45458221435547,
      "activations/layer23_attention_weight_min": -23.67738914489746,
      "activations/layer2_attention_weight_max": 33.2597541809082,
      "activations/layer2_attention_weight_min": -31.911619186401367,
      "activations/layer3_attention_weight_max": 90.97899627685547,
      "activations/layer3_attention_weight_min": -91.15648651123047,
      "activations/layer4_attention_weight_max": 88.06251525878906,
      "activations/layer4_attention_weight_min": -86.14634704589844,
      "activations/layer5_attention_weight_max": 67.16902160644531,
      "activations/layer5_attention_weight_min": -72.1875,
      "activations/layer6_attention_weight_max": 53.69478225708008,
      "activations/layer6_attention_weight_min": -51.31348419189453,
      "activations/layer7_attention_weight_max": 70.52865600585938,
      "activations/layer7_attention_weight_min": -68.66519165039062,
      "activations/layer8_attention_weight_max": 52.29770278930664,
      "activations/layer8_attention_weight_min": -55.605892181396484,
      "activations/layer9_attention_weight_max": 44.660377502441406,
      "activations/layer9_attention_weight_min": -46.74043655395508,
      "epoch": 22.82,
      "learning_rate": 2.902272727272727e-06,
      "loss": 2.7091,
      "step": 392650
    },
    {
      "activations/layer0_attention_weight_max": 15.89282512664795,
      "activations/layer0_attention_weight_min": -13.813100814819336,
      "activations/layer10_attention_weight_max": 38.381103515625,
      "activations/layer10_attention_weight_min": -38.22054672241211,
      "activations/layer11_attention_weight_max": 36.82098388671875,
      "activations/layer11_attention_weight_min": -36.78083038330078,
      "activations/layer12_attention_weight_max": 19.289770126342773,
      "activations/layer12_attention_weight_min": -30.509109497070312,
      "activations/layer13_attention_weight_max": 37.67668151855469,
      "activations/layer13_attention_weight_min": -29.236764907836914,
      "activations/layer14_attention_weight_max": 44.523590087890625,
      "activations/layer14_attention_weight_min": -33.67197036743164,
      "activations/layer15_attention_weight_max": 45.84798049926758,
      "activations/layer15_attention_weight_min": -32.04567337036133,
      "activations/layer16_attention_weight_max": 40.91022491455078,
      "activations/layer16_attention_weight_min": -36.89860153198242,
      "activations/layer17_attention_weight_max": 60.05469512939453,
      "activations/layer17_attention_weight_min": -51.18950653076172,
      "activations/layer18_attention_weight_max": 61.92455291748047,
      "activations/layer18_attention_weight_min": -49.500633239746094,
      "activations/layer19_attention_weight_max": 30.483224868774414,
      "activations/layer19_attention_weight_min": -29.61405372619629,
      "activations/layer1_attention_weight_max": 15.281777381896973,
      "activations/layer1_attention_weight_min": -14.27988052368164,
      "activations/layer20_attention_weight_max": 33.3641357421875,
      "activations/layer20_attention_weight_min": -23.736553192138672,
      "activations/layer21_attention_weight_max": 58.303123474121094,
      "activations/layer21_attention_weight_min": -27.33513069152832,
      "activations/layer22_attention_weight_max": 40.21165084838867,
      "activations/layer22_attention_weight_min": -28.28130340576172,
      "activations/layer23_attention_weight_max": 45.917884826660156,
      "activations/layer23_attention_weight_min": -22.798858642578125,
      "activations/layer2_attention_weight_max": 33.07724380493164,
      "activations/layer2_attention_weight_min": -31.34553337097168,
      "activations/layer3_attention_weight_max": 96.02751159667969,
      "activations/layer3_attention_weight_min": -94.97309112548828,
      "activations/layer4_attention_weight_max": 85.86062622070312,
      "activations/layer4_attention_weight_min": -86.8294906616211,
      "activations/layer5_attention_weight_max": 61.98286437988281,
      "activations/layer5_attention_weight_min": -71.58833312988281,
      "activations/layer6_attention_weight_max": 48.71983337402344,
      "activations/layer6_attention_weight_min": -51.42429733276367,
      "activations/layer7_attention_weight_max": 72.36651611328125,
      "activations/layer7_attention_weight_min": -68.18660736083984,
      "activations/layer8_attention_weight_max": 48.84731674194336,
      "activations/layer8_attention_weight_min": -50.65707778930664,
      "activations/layer9_attention_weight_max": 45.467525482177734,
      "activations/layer9_attention_weight_min": -47.00564956665039,
      "epoch": 22.82,
      "learning_rate": 2.8833333333333334e-06,
      "loss": 2.6839,
      "step": 392700
    },
    {
      "activations/layer0_attention_weight_max": 15.511862754821777,
      "activations/layer0_attention_weight_min": -14.0027494430542,
      "activations/layer10_attention_weight_max": 40.43579864501953,
      "activations/layer10_attention_weight_min": -37.234718322753906,
      "activations/layer11_attention_weight_max": 36.236785888671875,
      "activations/layer11_attention_weight_min": -37.527225494384766,
      "activations/layer12_attention_weight_max": 20.46907615661621,
      "activations/layer12_attention_weight_min": -24.062519073486328,
      "activations/layer13_attention_weight_max": 34.52376937866211,
      "activations/layer13_attention_weight_min": -28.545684814453125,
      "activations/layer14_attention_weight_max": 35.68489456176758,
      "activations/layer14_attention_weight_min": -30.349838256835938,
      "activations/layer15_attention_weight_max": 34.79459762573242,
      "activations/layer15_attention_weight_min": -31.19291114807129,
      "activations/layer16_attention_weight_max": 36.735801696777344,
      "activations/layer16_attention_weight_min": -33.7456169128418,
      "activations/layer17_attention_weight_max": 52.413658142089844,
      "activations/layer17_attention_weight_min": -48.12397766113281,
      "activations/layer18_attention_weight_max": 49.66384506225586,
      "activations/layer18_attention_weight_min": -43.01248550415039,
      "activations/layer19_attention_weight_max": 26.617557525634766,
      "activations/layer19_attention_weight_min": -27.17677879333496,
      "activations/layer1_attention_weight_max": 15.614158630371094,
      "activations/layer1_attention_weight_min": -15.629439353942871,
      "activations/layer20_attention_weight_max": 25.36745834350586,
      "activations/layer20_attention_weight_min": -26.650901794433594,
      "activations/layer21_attention_weight_max": 35.88856887817383,
      "activations/layer21_attention_weight_min": -24.019182205200195,
      "activations/layer22_attention_weight_max": 29.314449310302734,
      "activations/layer22_attention_weight_min": -29.843774795532227,
      "activations/layer23_attention_weight_max": 37.093017578125,
      "activations/layer23_attention_weight_min": -24.32120132446289,
      "activations/layer2_attention_weight_max": 32.157493591308594,
      "activations/layer2_attention_weight_min": -30.500064849853516,
      "activations/layer3_attention_weight_max": 92.70291137695312,
      "activations/layer3_attention_weight_min": -97.6154556274414,
      "activations/layer4_attention_weight_max": 89.02515411376953,
      "activations/layer4_attention_weight_min": -90.03587341308594,
      "activations/layer5_attention_weight_max": 62.24029541015625,
      "activations/layer5_attention_weight_min": -72.95358276367188,
      "activations/layer6_attention_weight_max": 51.33518981933594,
      "activations/layer6_attention_weight_min": -52.16813278198242,
      "activations/layer7_attention_weight_max": 69.73138427734375,
      "activations/layer7_attention_weight_min": -70.87824249267578,
      "activations/layer8_attention_weight_max": 48.217689514160156,
      "activations/layer8_attention_weight_min": -50.76683044433594,
      "activations/layer9_attention_weight_max": 45.64244842529297,
      "activations/layer9_attention_weight_min": -48.820899963378906,
      "epoch": 22.82,
      "learning_rate": 2.864393939393939e-06,
      "loss": 2.6889,
      "step": 392750
    },
    {
      "activations/layer0_attention_weight_max": 16.279069900512695,
      "activations/layer0_attention_weight_min": -14.109602928161621,
      "activations/layer10_attention_weight_max": 35.68423080444336,
      "activations/layer10_attention_weight_min": -36.77947998046875,
      "activations/layer11_attention_weight_max": 32.97290802001953,
      "activations/layer11_attention_weight_min": -36.717498779296875,
      "activations/layer12_attention_weight_max": 19.894926071166992,
      "activations/layer12_attention_weight_min": -26.39017677307129,
      "activations/layer13_attention_weight_max": 35.37064743041992,
      "activations/layer13_attention_weight_min": -26.52775764465332,
      "activations/layer14_attention_weight_max": 38.120609283447266,
      "activations/layer14_attention_weight_min": -31.892091751098633,
      "activations/layer15_attention_weight_max": 35.149559020996094,
      "activations/layer15_attention_weight_min": -31.441051483154297,
      "activations/layer16_attention_weight_max": 39.5439453125,
      "activations/layer16_attention_weight_min": -35.31298065185547,
      "activations/layer17_attention_weight_max": 56.37228775024414,
      "activations/layer17_attention_weight_min": -49.97954559326172,
      "activations/layer18_attention_weight_max": 56.25632858276367,
      "activations/layer18_attention_weight_min": -43.94322967529297,
      "activations/layer19_attention_weight_max": 26.510190963745117,
      "activations/layer19_attention_weight_min": -24.105716705322266,
      "activations/layer1_attention_weight_max": 15.274834632873535,
      "activations/layer1_attention_weight_min": -14.719486236572266,
      "activations/layer20_attention_weight_max": 24.49517059326172,
      "activations/layer20_attention_weight_min": -21.79425621032715,
      "activations/layer21_attention_weight_max": 36.46940994262695,
      "activations/layer21_attention_weight_min": -21.32158660888672,
      "activations/layer22_attention_weight_max": 30.26910972595215,
      "activations/layer22_attention_weight_min": -25.735641479492188,
      "activations/layer23_attention_weight_max": 40.59667205810547,
      "activations/layer23_attention_weight_min": -25.66762924194336,
      "activations/layer2_attention_weight_max": 32.166526794433594,
      "activations/layer2_attention_weight_min": -29.548866271972656,
      "activations/layer3_attention_weight_max": 90.88304901123047,
      "activations/layer3_attention_weight_min": -89.43338012695312,
      "activations/layer4_attention_weight_max": 84.99922943115234,
      "activations/layer4_attention_weight_min": -86.02151489257812,
      "activations/layer5_attention_weight_max": 63.362152099609375,
      "activations/layer5_attention_weight_min": -73.75817108154297,
      "activations/layer6_attention_weight_max": 47.60972595214844,
      "activations/layer6_attention_weight_min": -50.098995208740234,
      "activations/layer7_attention_weight_max": 72.42688751220703,
      "activations/layer7_attention_weight_min": -68.26568603515625,
      "activations/layer8_attention_weight_max": 45.399940490722656,
      "activations/layer8_attention_weight_min": -48.03057098388672,
      "activations/layer9_attention_weight_max": 42.82022476196289,
      "activations/layer9_attention_weight_min": -45.448280334472656,
      "epoch": 22.82,
      "learning_rate": 2.8454545454545453e-06,
      "loss": 2.7112,
      "step": 392800
    },
    {
      "activations/layer0_attention_weight_max": 15.34630012512207,
      "activations/layer0_attention_weight_min": -14.06926441192627,
      "activations/layer10_attention_weight_max": 34.467498779296875,
      "activations/layer10_attention_weight_min": -34.79639434814453,
      "activations/layer11_attention_weight_max": 35.318851470947266,
      "activations/layer11_attention_weight_min": -33.40904235839844,
      "activations/layer12_attention_weight_max": 18.646228790283203,
      "activations/layer12_attention_weight_min": -25.59672737121582,
      "activations/layer13_attention_weight_max": 32.8220329284668,
      "activations/layer13_attention_weight_min": -28.705223083496094,
      "activations/layer14_attention_weight_max": 36.396095275878906,
      "activations/layer14_attention_weight_min": -28.903797149658203,
      "activations/layer15_attention_weight_max": 38.60600662231445,
      "activations/layer15_attention_weight_min": -29.993694305419922,
      "activations/layer16_attention_weight_max": 41.24558639526367,
      "activations/layer16_attention_weight_min": -38.56342315673828,
      "activations/layer17_attention_weight_max": 64.8758773803711,
      "activations/layer17_attention_weight_min": -51.93264389038086,
      "activations/layer18_attention_weight_max": 58.006919860839844,
      "activations/layer18_attention_weight_min": -43.6184196472168,
      "activations/layer19_attention_weight_max": 32.902496337890625,
      "activations/layer19_attention_weight_min": -21.866806030273438,
      "activations/layer1_attention_weight_max": 14.776407241821289,
      "activations/layer1_attention_weight_min": -14.451932907104492,
      "activations/layer20_attention_weight_max": 26.617496490478516,
      "activations/layer20_attention_weight_min": -20.9776668548584,
      "activations/layer21_attention_weight_max": 36.31045913696289,
      "activations/layer21_attention_weight_min": -19.14870834350586,
      "activations/layer22_attention_weight_max": 35.097015380859375,
      "activations/layer22_attention_weight_min": -26.001930236816406,
      "activations/layer23_attention_weight_max": 39.701744079589844,
      "activations/layer23_attention_weight_min": -22.59345054626465,
      "activations/layer2_attention_weight_max": 31.380605697631836,
      "activations/layer2_attention_weight_min": -29.518951416015625,
      "activations/layer3_attention_weight_max": 89.91497039794922,
      "activations/layer3_attention_weight_min": -91.30828857421875,
      "activations/layer4_attention_weight_max": 87.77330780029297,
      "activations/layer4_attention_weight_min": -85.3678207397461,
      "activations/layer5_attention_weight_max": 62.74766540527344,
      "activations/layer5_attention_weight_min": -73.86023712158203,
      "activations/layer6_attention_weight_max": 49.98892593383789,
      "activations/layer6_attention_weight_min": -52.152381896972656,
      "activations/layer7_attention_weight_max": 68.0613021850586,
      "activations/layer7_attention_weight_min": -71.06436920166016,
      "activations/layer8_attention_weight_max": 48.59395217895508,
      "activations/layer8_attention_weight_min": -53.06988525390625,
      "activations/layer9_attention_weight_max": 43.733150482177734,
      "activations/layer9_attention_weight_min": -50.8104248046875,
      "epoch": 22.83,
      "learning_rate": 2.826515151515151e-06,
      "loss": 2.7064,
      "step": 392850
    },
    {
      "activations/layer0_attention_weight_max": 15.96835708618164,
      "activations/layer0_attention_weight_min": -14.151361465454102,
      "activations/layer10_attention_weight_max": 38.54718780517578,
      "activations/layer10_attention_weight_min": -37.463558197021484,
      "activations/layer11_attention_weight_max": 34.21175765991211,
      "activations/layer11_attention_weight_min": -35.70063400268555,
      "activations/layer12_attention_weight_max": 19.028507232666016,
      "activations/layer12_attention_weight_min": -24.81205940246582,
      "activations/layer13_attention_weight_max": 30.66562843322754,
      "activations/layer13_attention_weight_min": -28.804882049560547,
      "activations/layer14_attention_weight_max": 33.84833526611328,
      "activations/layer14_attention_weight_min": -31.894758224487305,
      "activations/layer15_attention_weight_max": 31.77907371520996,
      "activations/layer15_attention_weight_min": -31.49333381652832,
      "activations/layer16_attention_weight_max": 33.03135681152344,
      "activations/layer16_attention_weight_min": -32.28385543823242,
      "activations/layer17_attention_weight_max": 48.62862777709961,
      "activations/layer17_attention_weight_min": -46.12714767456055,
      "activations/layer18_attention_weight_max": 43.25054168701172,
      "activations/layer18_attention_weight_min": -39.888671875,
      "activations/layer19_attention_weight_max": 22.652326583862305,
      "activations/layer19_attention_weight_min": -23.28196144104004,
      "activations/layer1_attention_weight_max": 15.00514030456543,
      "activations/layer1_attention_weight_min": -14.85921573638916,
      "activations/layer20_attention_weight_max": 21.236431121826172,
      "activations/layer20_attention_weight_min": -20.51708221435547,
      "activations/layer21_attention_weight_max": 39.4881477355957,
      "activations/layer21_attention_weight_min": -22.91548728942871,
      "activations/layer22_attention_weight_max": 25.13170623779297,
      "activations/layer22_attention_weight_min": -24.09674835205078,
      "activations/layer23_attention_weight_max": 32.42054748535156,
      "activations/layer23_attention_weight_min": -23.845905303955078,
      "activations/layer2_attention_weight_max": 30.449811935424805,
      "activations/layer2_attention_weight_min": -29.36524200439453,
      "activations/layer3_attention_weight_max": 88.104736328125,
      "activations/layer3_attention_weight_min": -91.72379302978516,
      "activations/layer4_attention_weight_max": 83.87395477294922,
      "activations/layer4_attention_weight_min": -86.55223846435547,
      "activations/layer5_attention_weight_max": 60.994712829589844,
      "activations/layer5_attention_weight_min": -73.94805908203125,
      "activations/layer6_attention_weight_max": 50.00050735473633,
      "activations/layer6_attention_weight_min": -51.94015121459961,
      "activations/layer7_attention_weight_max": 69.67655944824219,
      "activations/layer7_attention_weight_min": -69.37293243408203,
      "activations/layer8_attention_weight_max": 46.89474105834961,
      "activations/layer8_attention_weight_min": -53.376277923583984,
      "activations/layer9_attention_weight_max": 45.54710006713867,
      "activations/layer9_attention_weight_min": -48.07160949707031,
      "epoch": 22.83,
      "learning_rate": 2.807575757575757e-06,
      "loss": 2.6944,
      "step": 392900
    },
    {
      "activations/layer0_attention_weight_max": 16.403636932373047,
      "activations/layer0_attention_weight_min": -14.051957130432129,
      "activations/layer10_attention_weight_max": 34.8758544921875,
      "activations/layer10_attention_weight_min": -35.485939025878906,
      "activations/layer11_attention_weight_max": 33.547874450683594,
      "activations/layer11_attention_weight_min": -35.99805450439453,
      "activations/layer12_attention_weight_max": 19.086713790893555,
      "activations/layer12_attention_weight_min": -26.761293411254883,
      "activations/layer13_attention_weight_max": 33.63367462158203,
      "activations/layer13_attention_weight_min": -26.710948944091797,
      "activations/layer14_attention_weight_max": 34.23595428466797,
      "activations/layer14_attention_weight_min": -30.48469352722168,
      "activations/layer15_attention_weight_max": 31.34235954284668,
      "activations/layer15_attention_weight_min": -32.6310920715332,
      "activations/layer16_attention_weight_max": 33.5583381652832,
      "activations/layer16_attention_weight_min": -33.40896224975586,
      "activations/layer17_attention_weight_max": 51.457008361816406,
      "activations/layer17_attention_weight_min": -46.378578186035156,
      "activations/layer18_attention_weight_max": 45.67793273925781,
      "activations/layer18_attention_weight_min": -42.62099075317383,
      "activations/layer19_attention_weight_max": 24.575654983520508,
      "activations/layer19_attention_weight_min": -26.382980346679688,
      "activations/layer1_attention_weight_max": 15.17076301574707,
      "activations/layer1_attention_weight_min": -13.395315170288086,
      "activations/layer20_attention_weight_max": 23.767711639404297,
      "activations/layer20_attention_weight_min": -21.845382690429688,
      "activations/layer21_attention_weight_max": 39.654441833496094,
      "activations/layer21_attention_weight_min": -25.32159996032715,
      "activations/layer22_attention_weight_max": 29.996906280517578,
      "activations/layer22_attention_weight_min": -25.469507217407227,
      "activations/layer23_attention_weight_max": 32.84217071533203,
      "activations/layer23_attention_weight_min": -21.589208602905273,
      "activations/layer2_attention_weight_max": 30.692485809326172,
      "activations/layer2_attention_weight_min": -29.62453269958496,
      "activations/layer3_attention_weight_max": 90.66157531738281,
      "activations/layer3_attention_weight_min": -91.64138793945312,
      "activations/layer4_attention_weight_max": 85.711669921875,
      "activations/layer4_attention_weight_min": -87.81261444091797,
      "activations/layer5_attention_weight_max": 61.961238861083984,
      "activations/layer5_attention_weight_min": -75.05010986328125,
      "activations/layer6_attention_weight_max": 49.387229919433594,
      "activations/layer6_attention_weight_min": -51.19780731201172,
      "activations/layer7_attention_weight_max": 67.32118225097656,
      "activations/layer7_attention_weight_min": -68.01764678955078,
      "activations/layer8_attention_weight_max": 47.20412063598633,
      "activations/layer8_attention_weight_min": -50.401824951171875,
      "activations/layer9_attention_weight_max": 44.0325927734375,
      "activations/layer9_attention_weight_min": -44.26474380493164,
      "epoch": 22.83,
      "learning_rate": 2.7886363636363637e-06,
      "loss": 2.7076,
      "step": 392950
    },
    {
      "activations/layer0_attention_weight_max": 16.14402961730957,
      "activations/layer0_attention_weight_min": -14.204854011535645,
      "activations/layer10_attention_weight_max": 38.938148498535156,
      "activations/layer10_attention_weight_min": -38.2725944519043,
      "activations/layer11_attention_weight_max": 35.96836853027344,
      "activations/layer11_attention_weight_min": -35.745235443115234,
      "activations/layer12_attention_weight_max": 21.055883407592773,
      "activations/layer12_attention_weight_min": -25.229736328125,
      "activations/layer13_attention_weight_max": 35.031837463378906,
      "activations/layer13_attention_weight_min": -29.75595474243164,
      "activations/layer14_attention_weight_max": 37.746253967285156,
      "activations/layer14_attention_weight_min": -33.193321228027344,
      "activations/layer15_attention_weight_max": 36.60771942138672,
      "activations/layer15_attention_weight_min": -33.094478607177734,
      "activations/layer16_attention_weight_max": 39.36800765991211,
      "activations/layer16_attention_weight_min": -37.74825668334961,
      "activations/layer17_attention_weight_max": 56.60300064086914,
      "activations/layer17_attention_weight_min": -52.06498718261719,
      "activations/layer18_attention_weight_max": 50.220001220703125,
      "activations/layer18_attention_weight_min": -47.037986755371094,
      "activations/layer19_attention_weight_max": 25.394399642944336,
      "activations/layer19_attention_weight_min": -23.74013328552246,
      "activations/layer1_attention_weight_max": 14.416071891784668,
      "activations/layer1_attention_weight_min": -13.719867706298828,
      "activations/layer20_attention_weight_max": 23.312040328979492,
      "activations/layer20_attention_weight_min": -23.63188362121582,
      "activations/layer21_attention_weight_max": 37.893287658691406,
      "activations/layer21_attention_weight_min": -23.816192626953125,
      "activations/layer22_attention_weight_max": 28.170108795166016,
      "activations/layer22_attention_weight_min": -25.517059326171875,
      "activations/layer23_attention_weight_max": 34.61528396606445,
      "activations/layer23_attention_weight_min": -25.206600189208984,
      "activations/layer2_attention_weight_max": 31.787199020385742,
      "activations/layer2_attention_weight_min": -30.527835845947266,
      "activations/layer3_attention_weight_max": 88.64379119873047,
      "activations/layer3_attention_weight_min": -90.2313461303711,
      "activations/layer4_attention_weight_max": 84.7000732421875,
      "activations/layer4_attention_weight_min": -86.75550842285156,
      "activations/layer5_attention_weight_max": 61.75469207763672,
      "activations/layer5_attention_weight_min": -75.16879272460938,
      "activations/layer6_attention_weight_max": 47.64031219482422,
      "activations/layer6_attention_weight_min": -52.392662048339844,
      "activations/layer7_attention_weight_max": 72.61143493652344,
      "activations/layer7_attention_weight_min": -71.21482849121094,
      "activations/layer8_attention_weight_max": 51.65665817260742,
      "activations/layer8_attention_weight_min": -52.355953216552734,
      "activations/layer9_attention_weight_max": 54.089962005615234,
      "activations/layer9_attention_weight_min": -47.44104766845703,
      "epoch": 22.84,
      "learning_rate": 2.7696969696969695e-06,
      "loss": 2.6961,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4564,
      "eval_samples_per_second": 507.782,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4564,
      "eval_openwebtext_samples_per_second": 507.782,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9699,
      "eval_wikitext_samples_per_second": 231.48,
      "step": 393000
    },
    {
      "epoch": 22.84,
      "eval_lambada_loss": 2.408203125,
      "eval_lambada_ppl": 11.113972772314547,
      "eval_lambada_runtime": 9.5477,
      "eval_lambada_samples_per_second": 509.965,
      "step": 393000
    },
    {
      "activations/layer0_attention_weight_max": 15.683794021606445,
      "activations/layer0_attention_weight_min": -14.027307510375977,
      "activations/layer10_attention_weight_max": 36.323020935058594,
      "activations/layer10_attention_weight_min": -36.30583190917969,
      "activations/layer11_attention_weight_max": 34.85109329223633,
      "activations/layer11_attention_weight_min": -35.40131378173828,
      "activations/layer12_attention_weight_max": 20.132007598876953,
      "activations/layer12_attention_weight_min": -27.695863723754883,
      "activations/layer13_attention_weight_max": 39.61079406738281,
      "activations/layer13_attention_weight_min": -26.31560707092285,
      "activations/layer14_attention_weight_max": 42.58564376831055,
      "activations/layer14_attention_weight_min": -30.099870681762695,
      "activations/layer15_attention_weight_max": 35.26426315307617,
      "activations/layer15_attention_weight_min": -30.06785011291504,
      "activations/layer16_attention_weight_max": 37.37256622314453,
      "activations/layer16_attention_weight_min": -33.21961212158203,
      "activations/layer17_attention_weight_max": 59.5804443359375,
      "activations/layer17_attention_weight_min": -47.9005012512207,
      "activations/layer18_attention_weight_max": 51.65174102783203,
      "activations/layer18_attention_weight_min": -42.92598342895508,
      "activations/layer19_attention_weight_max": 27.382434844970703,
      "activations/layer19_attention_weight_min": -23.217857360839844,
      "activations/layer1_attention_weight_max": 14.714954376220703,
      "activations/layer1_attention_weight_min": -15.599000930786133,
      "activations/layer20_attention_weight_max": 27.069929122924805,
      "activations/layer20_attention_weight_min": -20.641315460205078,
      "activations/layer21_attention_weight_max": 42.4663200378418,
      "activations/layer21_attention_weight_min": -20.861370086669922,
      "activations/layer22_attention_weight_max": 32.515804290771484,
      "activations/layer22_attention_weight_min": -25.87833595275879,
      "activations/layer23_attention_weight_max": 35.51245880126953,
      "activations/layer23_attention_weight_min": -22.20627212524414,
      "activations/layer2_attention_weight_max": 32.47303009033203,
      "activations/layer2_attention_weight_min": -30.417219161987305,
      "activations/layer3_attention_weight_max": 90.87653350830078,
      "activations/layer3_attention_weight_min": -89.81342315673828,
      "activations/layer4_attention_weight_max": 88.230712890625,
      "activations/layer4_attention_weight_min": -84.87224578857422,
      "activations/layer5_attention_weight_max": 62.93281555175781,
      "activations/layer5_attention_weight_min": -70.58549499511719,
      "activations/layer6_attention_weight_max": 49.86138153076172,
      "activations/layer6_attention_weight_min": -50.53187561035156,
      "activations/layer7_attention_weight_max": 73.01934814453125,
      "activations/layer7_attention_weight_min": -69.73432159423828,
      "activations/layer8_attention_weight_max": 47.26923370361328,
      "activations/layer8_attention_weight_min": -49.60612487792969,
      "activations/layer9_attention_weight_max": 46.95258712768555,
      "activations/layer9_attention_weight_min": -46.430419921875,
      "epoch": 22.84,
      "learning_rate": 2.7507575757575756e-06,
      "loss": 2.7007,
      "step": 393050
    },
    {
      "activations/layer0_attention_weight_max": 16.27682876586914,
      "activations/layer0_attention_weight_min": -14.148816108703613,
      "activations/layer10_attention_weight_max": 37.03943634033203,
      "activations/layer10_attention_weight_min": -37.62091064453125,
      "activations/layer11_attention_weight_max": 34.537757873535156,
      "activations/layer11_attention_weight_min": -35.240482330322266,
      "activations/layer12_attention_weight_max": 20.291677474975586,
      "activations/layer12_attention_weight_min": -22.772396087646484,
      "activations/layer13_attention_weight_max": 39.247222900390625,
      "activations/layer13_attention_weight_min": -25.976625442504883,
      "activations/layer14_attention_weight_max": 37.187286376953125,
      "activations/layer14_attention_weight_min": -27.9514217376709,
      "activations/layer15_attention_weight_max": 33.849395751953125,
      "activations/layer15_attention_weight_min": -28.968000411987305,
      "activations/layer16_attention_weight_max": 36.652835845947266,
      "activations/layer16_attention_weight_min": -32.49740219116211,
      "activations/layer17_attention_weight_max": 54.20198059082031,
      "activations/layer17_attention_weight_min": -44.25127029418945,
      "activations/layer18_attention_weight_max": 50.5583381652832,
      "activations/layer18_attention_weight_min": -41.34412384033203,
      "activations/layer19_attention_weight_max": 26.679655075073242,
      "activations/layer19_attention_weight_min": -22.78327751159668,
      "activations/layer1_attention_weight_max": 15.261946678161621,
      "activations/layer1_attention_weight_min": -15.076295852661133,
      "activations/layer20_attention_weight_max": 27.253406524658203,
      "activations/layer20_attention_weight_min": -22.373332977294922,
      "activations/layer21_attention_weight_max": 39.66745376586914,
      "activations/layer21_attention_weight_min": -20.382230758666992,
      "activations/layer22_attention_weight_max": 29.81281852722168,
      "activations/layer22_attention_weight_min": -24.965518951416016,
      "activations/layer23_attention_weight_max": 39.15399932861328,
      "activations/layer23_attention_weight_min": -24.558353424072266,
      "activations/layer2_attention_weight_max": 31.49188232421875,
      "activations/layer2_attention_weight_min": -30.318660736083984,
      "activations/layer3_attention_weight_max": 92.7513656616211,
      "activations/layer3_attention_weight_min": -92.41545867919922,
      "activations/layer4_attention_weight_max": 88.06314849853516,
      "activations/layer4_attention_weight_min": -87.20216369628906,
      "activations/layer5_attention_weight_max": 60.51601791381836,
      "activations/layer5_attention_weight_min": -71.43856048583984,
      "activations/layer6_attention_weight_max": 49.39479064941406,
      "activations/layer6_attention_weight_min": -51.19337844848633,
      "activations/layer7_attention_weight_max": 70.73407745361328,
      "activations/layer7_attention_weight_min": -69.38072204589844,
      "activations/layer8_attention_weight_max": 50.222389221191406,
      "activations/layer8_attention_weight_min": -51.202571868896484,
      "activations/layer9_attention_weight_max": 46.47528076171875,
      "activations/layer9_attention_weight_min": -45.92805099487305,
      "epoch": 22.84,
      "learning_rate": 2.7318181818181813e-06,
      "loss": 2.6992,
      "step": 393100
    },
    {
      "activations/layer0_attention_weight_max": 15.701748847961426,
      "activations/layer0_attention_weight_min": -14.538969039916992,
      "activations/layer10_attention_weight_max": 38.920867919921875,
      "activations/layer10_attention_weight_min": -37.89673614501953,
      "activations/layer11_attention_weight_max": 36.02033996582031,
      "activations/layer11_attention_weight_min": -37.417938232421875,
      "activations/layer12_attention_weight_max": 20.293203353881836,
      "activations/layer12_attention_weight_min": -23.43997573852539,
      "activations/layer13_attention_weight_max": 39.47910690307617,
      "activations/layer13_attention_weight_min": -28.81464385986328,
      "activations/layer14_attention_weight_max": 41.365882873535156,
      "activations/layer14_attention_weight_min": -32.983577728271484,
      "activations/layer15_attention_weight_max": 36.761451721191406,
      "activations/layer15_attention_weight_min": -33.12055206298828,
      "activations/layer16_attention_weight_max": 38.089107513427734,
      "activations/layer16_attention_weight_min": -35.74875259399414,
      "activations/layer17_attention_weight_max": 57.20008850097656,
      "activations/layer17_attention_weight_min": -47.25807571411133,
      "activations/layer18_attention_weight_max": 52.49015808105469,
      "activations/layer18_attention_weight_min": -44.28935241699219,
      "activations/layer19_attention_weight_max": 27.673816680908203,
      "activations/layer19_attention_weight_min": -24.3533992767334,
      "activations/layer1_attention_weight_max": 15.598146438598633,
      "activations/layer1_attention_weight_min": -14.968770027160645,
      "activations/layer20_attention_weight_max": 25.79034423828125,
      "activations/layer20_attention_weight_min": -23.725400924682617,
      "activations/layer21_attention_weight_max": 41.90587615966797,
      "activations/layer21_attention_weight_min": -22.605947494506836,
      "activations/layer22_attention_weight_max": 33.57697296142578,
      "activations/layer22_attention_weight_min": -27.086395263671875,
      "activations/layer23_attention_weight_max": 40.34886932373047,
      "activations/layer23_attention_weight_min": -24.69099235534668,
      "activations/layer2_attention_weight_max": 33.02104187011719,
      "activations/layer2_attention_weight_min": -31.787841796875,
      "activations/layer3_attention_weight_max": 96.47901916503906,
      "activations/layer3_attention_weight_min": -94.63385009765625,
      "activations/layer4_attention_weight_max": 94.49987030029297,
      "activations/layer4_attention_weight_min": -89.9436264038086,
      "activations/layer5_attention_weight_max": 63.29762268066406,
      "activations/layer5_attention_weight_min": -75.43428039550781,
      "activations/layer6_attention_weight_max": 52.60340881347656,
      "activations/layer6_attention_weight_min": -53.10311508178711,
      "activations/layer7_attention_weight_max": 77.8307876586914,
      "activations/layer7_attention_weight_min": -70.00971221923828,
      "activations/layer8_attention_weight_max": 48.93349075317383,
      "activations/layer8_attention_weight_min": -52.85987854003906,
      "activations/layer9_attention_weight_max": 46.748775482177734,
      "activations/layer9_attention_weight_min": -49.52042007446289,
      "epoch": 22.84,
      "learning_rate": 2.712878787878788e-06,
      "loss": 2.7052,
      "step": 393150
    },
    {
      "activations/layer0_attention_weight_max": 15.196516036987305,
      "activations/layer0_attention_weight_min": -14.051774978637695,
      "activations/layer10_attention_weight_max": 37.524532318115234,
      "activations/layer10_attention_weight_min": -38.7021369934082,
      "activations/layer11_attention_weight_max": 38.87112045288086,
      "activations/layer11_attention_weight_min": -37.47469711303711,
      "activations/layer12_attention_weight_max": 18.894208908081055,
      "activations/layer12_attention_weight_min": -24.60566520690918,
      "activations/layer13_attention_weight_max": 34.7017936706543,
      "activations/layer13_attention_weight_min": -28.239015579223633,
      "activations/layer14_attention_weight_max": 37.141422271728516,
      "activations/layer14_attention_weight_min": -30.949134826660156,
      "activations/layer15_attention_weight_max": 33.206546783447266,
      "activations/layer15_attention_weight_min": -32.01950454711914,
      "activations/layer16_attention_weight_max": 35.72696304321289,
      "activations/layer16_attention_weight_min": -35.47368621826172,
      "activations/layer17_attention_weight_max": 55.9069938659668,
      "activations/layer17_attention_weight_min": -45.41065216064453,
      "activations/layer18_attention_weight_max": 46.91761016845703,
      "activations/layer18_attention_weight_min": -41.47712326049805,
      "activations/layer19_attention_weight_max": 25.862075805664062,
      "activations/layer19_attention_weight_min": -21.96747398376465,
      "activations/layer1_attention_weight_max": 15.278566360473633,
      "activations/layer1_attention_weight_min": -14.503670692443848,
      "activations/layer20_attention_weight_max": 25.080472946166992,
      "activations/layer20_attention_weight_min": -19.792234420776367,
      "activations/layer21_attention_weight_max": 37.81001663208008,
      "activations/layer21_attention_weight_min": -20.8339900970459,
      "activations/layer22_attention_weight_max": 28.49055290222168,
      "activations/layer22_attention_weight_min": -24.287887573242188,
      "activations/layer23_attention_weight_max": 38.2417106628418,
      "activations/layer23_attention_weight_min": -23.01215362548828,
      "activations/layer2_attention_weight_max": 32.68244934082031,
      "activations/layer2_attention_weight_min": -29.487712860107422,
      "activations/layer3_attention_weight_max": 93.25341796875,
      "activations/layer3_attention_weight_min": -92.19942474365234,
      "activations/layer4_attention_weight_max": 87.50934600830078,
      "activations/layer4_attention_weight_min": -86.29978942871094,
      "activations/layer5_attention_weight_max": 65.42826843261719,
      "activations/layer5_attention_weight_min": -74.16148376464844,
      "activations/layer6_attention_weight_max": 51.36200714111328,
      "activations/layer6_attention_weight_min": -53.18691635131836,
      "activations/layer7_attention_weight_max": 71.94226837158203,
      "activations/layer7_attention_weight_min": -72.87860870361328,
      "activations/layer8_attention_weight_max": 49.161109924316406,
      "activations/layer8_attention_weight_min": -53.606693267822266,
      "activations/layer9_attention_weight_max": 52.17300033569336,
      "activations/layer9_attention_weight_min": -49.584102630615234,
      "epoch": 22.85,
      "learning_rate": 2.6939393939393936e-06,
      "loss": 2.7017,
      "step": 393200
    },
    {
      "activations/layer0_attention_weight_max": 15.436014175415039,
      "activations/layer0_attention_weight_min": -14.323633193969727,
      "activations/layer10_attention_weight_max": 36.19554901123047,
      "activations/layer10_attention_weight_min": -36.6259765625,
      "activations/layer11_attention_weight_max": 34.25196838378906,
      "activations/layer11_attention_weight_min": -35.16636657714844,
      "activations/layer12_attention_weight_max": 18.690753936767578,
      "activations/layer12_attention_weight_min": -27.281204223632812,
      "activations/layer13_attention_weight_max": 34.98142623901367,
      "activations/layer13_attention_weight_min": -27.874691009521484,
      "activations/layer14_attention_weight_max": 36.97542953491211,
      "activations/layer14_attention_weight_min": -32.395362854003906,
      "activations/layer15_attention_weight_max": 31.37337303161621,
      "activations/layer15_attention_weight_min": -31.825101852416992,
      "activations/layer16_attention_weight_max": 35.84784698486328,
      "activations/layer16_attention_weight_min": -33.84272384643555,
      "activations/layer17_attention_weight_max": 54.684303283691406,
      "activations/layer17_attention_weight_min": -50.53647232055664,
      "activations/layer18_attention_weight_max": 46.770851135253906,
      "activations/layer18_attention_weight_min": -46.25703048706055,
      "activations/layer19_attention_weight_max": 27.261804580688477,
      "activations/layer19_attention_weight_min": -23.69234275817871,
      "activations/layer1_attention_weight_max": 15.43398380279541,
      "activations/layer1_attention_weight_min": -15.490625381469727,
      "activations/layer20_attention_weight_max": 27.103757858276367,
      "activations/layer20_attention_weight_min": -22.906578063964844,
      "activations/layer21_attention_weight_max": 39.87089157104492,
      "activations/layer21_attention_weight_min": -21.20970916748047,
      "activations/layer22_attention_weight_max": 29.260190963745117,
      "activations/layer22_attention_weight_min": -26.443986892700195,
      "activations/layer23_attention_weight_max": 36.8734245300293,
      "activations/layer23_attention_weight_min": -23.710201263427734,
      "activations/layer2_attention_weight_max": 34.5261116027832,
      "activations/layer2_attention_weight_min": -31.17984390258789,
      "activations/layer3_attention_weight_max": 95.15675354003906,
      "activations/layer3_attention_weight_min": -94.39276123046875,
      "activations/layer4_attention_weight_max": 85.81542205810547,
      "activations/layer4_attention_weight_min": -90.38673400878906,
      "activations/layer5_attention_weight_max": 61.36591720581055,
      "activations/layer5_attention_weight_min": -72.95093536376953,
      "activations/layer6_attention_weight_max": 51.641056060791016,
      "activations/layer6_attention_weight_min": -53.21391677856445,
      "activations/layer7_attention_weight_max": 74.9251937866211,
      "activations/layer7_attention_weight_min": -70.17343139648438,
      "activations/layer8_attention_weight_max": 46.76235580444336,
      "activations/layer8_attention_weight_min": -50.62443923950195,
      "activations/layer9_attention_weight_max": 49.103755950927734,
      "activations/layer9_attention_weight_min": -46.06211471557617,
      "epoch": 22.85,
      "learning_rate": 2.675e-06,
      "loss": 2.7229,
      "step": 393250
    },
    {
      "activations/layer0_attention_weight_max": 16.737232208251953,
      "activations/layer0_attention_weight_min": -14.328167915344238,
      "activations/layer10_attention_weight_max": 37.02018356323242,
      "activations/layer10_attention_weight_min": -40.39360427856445,
      "activations/layer11_attention_weight_max": 34.713233947753906,
      "activations/layer11_attention_weight_min": -34.46072769165039,
      "activations/layer12_attention_weight_max": 18.29575538635254,
      "activations/layer12_attention_weight_min": -28.4470272064209,
      "activations/layer13_attention_weight_max": 39.87981414794922,
      "activations/layer13_attention_weight_min": -27.773141860961914,
      "activations/layer14_attention_weight_max": 43.371551513671875,
      "activations/layer14_attention_weight_min": -29.633033752441406,
      "activations/layer15_attention_weight_max": 42.13740539550781,
      "activations/layer15_attention_weight_min": -29.9263858795166,
      "activations/layer16_attention_weight_max": 37.77688980102539,
      "activations/layer16_attention_weight_min": -35.12565231323242,
      "activations/layer17_attention_weight_max": 54.7845573425293,
      "activations/layer17_attention_weight_min": -49.073509216308594,
      "activations/layer18_attention_weight_max": 50.905601501464844,
      "activations/layer18_attention_weight_min": -46.82938766479492,
      "activations/layer19_attention_weight_max": 24.271364212036133,
      "activations/layer19_attention_weight_min": -26.206928253173828,
      "activations/layer1_attention_weight_max": 15.817551612854004,
      "activations/layer1_attention_weight_min": -15.431915283203125,
      "activations/layer20_attention_weight_max": 24.09117317199707,
      "activations/layer20_attention_weight_min": -24.829692840576172,
      "activations/layer21_attention_weight_max": 41.13737106323242,
      "activations/layer21_attention_weight_min": -26.696439743041992,
      "activations/layer22_attention_weight_max": 29.70796012878418,
      "activations/layer22_attention_weight_min": -26.773313522338867,
      "activations/layer23_attention_weight_max": 34.20387268066406,
      "activations/layer23_attention_weight_min": -23.548025131225586,
      "activations/layer2_attention_weight_max": 34.4921760559082,
      "activations/layer2_attention_weight_min": -34.67312240600586,
      "activations/layer3_attention_weight_max": 94.17195129394531,
      "activations/layer3_attention_weight_min": -94.81092834472656,
      "activations/layer4_attention_weight_max": 85.4626693725586,
      "activations/layer4_attention_weight_min": -86.5604248046875,
      "activations/layer5_attention_weight_max": 61.05261993408203,
      "activations/layer5_attention_weight_min": -73.09403991699219,
      "activations/layer6_attention_weight_max": 49.88172912597656,
      "activations/layer6_attention_weight_min": -51.27095031738281,
      "activations/layer7_attention_weight_max": 69.32405090332031,
      "activations/layer7_attention_weight_min": -70.77860260009766,
      "activations/layer8_attention_weight_max": 47.13814926147461,
      "activations/layer8_attention_weight_min": -51.66844940185547,
      "activations/layer9_attention_weight_max": 44.38253402709961,
      "activations/layer9_attention_weight_min": -45.50887680053711,
      "epoch": 22.85,
      "learning_rate": 2.6560606060606055e-06,
      "loss": 2.7075,
      "step": 393300
    },
    {
      "activations/layer0_attention_weight_max": 15.347457885742188,
      "activations/layer0_attention_weight_min": -14.059335708618164,
      "activations/layer10_attention_weight_max": 36.329612731933594,
      "activations/layer10_attention_weight_min": -35.826683044433594,
      "activations/layer11_attention_weight_max": 34.983238220214844,
      "activations/layer11_attention_weight_min": -36.30438232421875,
      "activations/layer12_attention_weight_max": 20.164159774780273,
      "activations/layer12_attention_weight_min": -22.844730377197266,
      "activations/layer13_attention_weight_max": 36.43849563598633,
      "activations/layer13_attention_weight_min": -27.723596572875977,
      "activations/layer14_attention_weight_max": 37.44118881225586,
      "activations/layer14_attention_weight_min": -30.608245849609375,
      "activations/layer15_attention_weight_max": 35.081172943115234,
      "activations/layer15_attention_weight_min": -31.29740333557129,
      "activations/layer16_attention_weight_max": 36.817649841308594,
      "activations/layer16_attention_weight_min": -33.25569152832031,
      "activations/layer17_attention_weight_max": 54.288490295410156,
      "activations/layer17_attention_weight_min": -46.537662506103516,
      "activations/layer18_attention_weight_max": 48.19856262207031,
      "activations/layer18_attention_weight_min": -40.95026397705078,
      "activations/layer19_attention_weight_max": 25.478328704833984,
      "activations/layer19_attention_weight_min": -25.0390625,
      "activations/layer1_attention_weight_max": 15.023425102233887,
      "activations/layer1_attention_weight_min": -14.923081398010254,
      "activations/layer20_attention_weight_max": 24.835691452026367,
      "activations/layer20_attention_weight_min": -23.817935943603516,
      "activations/layer21_attention_weight_max": 37.49048614501953,
      "activations/layer21_attention_weight_min": -23.745182037353516,
      "activations/layer22_attention_weight_max": 27.591413497924805,
      "activations/layer22_attention_weight_min": -25.48265266418457,
      "activations/layer23_attention_weight_max": 36.183982849121094,
      "activations/layer23_attention_weight_min": -24.61862564086914,
      "activations/layer2_attention_weight_max": 32.65541076660156,
      "activations/layer2_attention_weight_min": -30.721860885620117,
      "activations/layer3_attention_weight_max": 94.8720474243164,
      "activations/layer3_attention_weight_min": -96.5792236328125,
      "activations/layer4_attention_weight_max": 93.16832733154297,
      "activations/layer4_attention_weight_min": -87.33293914794922,
      "activations/layer5_attention_weight_max": 64.64921569824219,
      "activations/layer5_attention_weight_min": -73.16145324707031,
      "activations/layer6_attention_weight_max": 53.17378234863281,
      "activations/layer6_attention_weight_min": -54.68050765991211,
      "activations/layer7_attention_weight_max": 70.63411712646484,
      "activations/layer7_attention_weight_min": -72.37406158447266,
      "activations/layer8_attention_weight_max": 48.434688568115234,
      "activations/layer8_attention_weight_min": -53.41559982299805,
      "activations/layer9_attention_weight_max": 44.8891487121582,
      "activations/layer9_attention_weight_min": -50.63608169555664,
      "epoch": 22.86,
      "learning_rate": 2.637121212121212e-06,
      "loss": 2.7075,
      "step": 393350
    },
    {
      "activations/layer0_attention_weight_max": 16.152761459350586,
      "activations/layer0_attention_weight_min": -14.182701110839844,
      "activations/layer10_attention_weight_max": 34.210384368896484,
      "activations/layer10_attention_weight_min": -34.71717071533203,
      "activations/layer11_attention_weight_max": 33.939476013183594,
      "activations/layer11_attention_weight_min": -34.038307189941406,
      "activations/layer12_attention_weight_max": 18.822296142578125,
      "activations/layer12_attention_weight_min": -26.004304885864258,
      "activations/layer13_attention_weight_max": 31.646381378173828,
      "activations/layer13_attention_weight_min": -26.922819137573242,
      "activations/layer14_attention_weight_max": 35.217811584472656,
      "activations/layer14_attention_weight_min": -28.06045913696289,
      "activations/layer15_attention_weight_max": 32.11763000488281,
      "activations/layer15_attention_weight_min": -29.908056259155273,
      "activations/layer16_attention_weight_max": 33.33675765991211,
      "activations/layer16_attention_weight_min": -29.25925636291504,
      "activations/layer17_attention_weight_max": 48.85285568237305,
      "activations/layer17_attention_weight_min": -42.23698806762695,
      "activations/layer18_attention_weight_max": 44.464324951171875,
      "activations/layer18_attention_weight_min": -37.56390380859375,
      "activations/layer19_attention_weight_max": 23.461795806884766,
      "activations/layer19_attention_weight_min": -22.472867965698242,
      "activations/layer1_attention_weight_max": 14.613015174865723,
      "activations/layer1_attention_weight_min": -14.781309127807617,
      "activations/layer20_attention_weight_max": 21.031511306762695,
      "activations/layer20_attention_weight_min": -20.58833122253418,
      "activations/layer21_attention_weight_max": 36.30388259887695,
      "activations/layer21_attention_weight_min": -21.570180892944336,
      "activations/layer22_attention_weight_max": 26.520172119140625,
      "activations/layer22_attention_weight_min": -24.800771713256836,
      "activations/layer23_attention_weight_max": 32.10945129394531,
      "activations/layer23_attention_weight_min": -25.979717254638672,
      "activations/layer2_attention_weight_max": 31.573932647705078,
      "activations/layer2_attention_weight_min": -30.74392318725586,
      "activations/layer3_attention_weight_max": 92.68029022216797,
      "activations/layer3_attention_weight_min": -92.46463012695312,
      "activations/layer4_attention_weight_max": 87.08982849121094,
      "activations/layer4_attention_weight_min": -84.84917449951172,
      "activations/layer5_attention_weight_max": 61.694637298583984,
      "activations/layer5_attention_weight_min": -72.65667724609375,
      "activations/layer6_attention_weight_max": 49.43379211425781,
      "activations/layer6_attention_weight_min": -51.83137893676758,
      "activations/layer7_attention_weight_max": 67.01104736328125,
      "activations/layer7_attention_weight_min": -70.04859924316406,
      "activations/layer8_attention_weight_max": 47.53850173950195,
      "activations/layer8_attention_weight_min": -48.55076217651367,
      "activations/layer9_attention_weight_max": 45.6445426940918,
      "activations/layer9_attention_weight_min": -46.26007843017578,
      "epoch": 22.86,
      "learning_rate": 2.6181818181818183e-06,
      "loss": 2.6952,
      "step": 393400
    },
    {
      "activations/layer0_attention_weight_max": 15.416976928710938,
      "activations/layer0_attention_weight_min": -14.284554481506348,
      "activations/layer10_attention_weight_max": 37.123565673828125,
      "activations/layer10_attention_weight_min": -37.87273406982422,
      "activations/layer11_attention_weight_max": 34.867286682128906,
      "activations/layer11_attention_weight_min": -40.167030334472656,
      "activations/layer12_attention_weight_max": 19.195627212524414,
      "activations/layer12_attention_weight_min": -25.018476486206055,
      "activations/layer13_attention_weight_max": 35.86806869506836,
      "activations/layer13_attention_weight_min": -26.863615036010742,
      "activations/layer14_attention_weight_max": 36.79623031616211,
      "activations/layer14_attention_weight_min": -30.598276138305664,
      "activations/layer15_attention_weight_max": 34.083229064941406,
      "activations/layer15_attention_weight_min": -32.296485900878906,
      "activations/layer16_attention_weight_max": 35.725955963134766,
      "activations/layer16_attention_weight_min": -34.04255676269531,
      "activations/layer17_attention_weight_max": 51.780120849609375,
      "activations/layer17_attention_weight_min": -44.997711181640625,
      "activations/layer18_attention_weight_max": 48.89071273803711,
      "activations/layer18_attention_weight_min": -41.87948226928711,
      "activations/layer19_attention_weight_max": 24.482200622558594,
      "activations/layer19_attention_weight_min": -25.332788467407227,
      "activations/layer1_attention_weight_max": 16.30417823791504,
      "activations/layer1_attention_weight_min": -14.584656715393066,
      "activations/layer20_attention_weight_max": 24.660476684570312,
      "activations/layer20_attention_weight_min": -24.497814178466797,
      "activations/layer21_attention_weight_max": 39.11537551879883,
      "activations/layer21_attention_weight_min": -23.376354217529297,
      "activations/layer22_attention_weight_max": 31.849506378173828,
      "activations/layer22_attention_weight_min": -24.719120025634766,
      "activations/layer23_attention_weight_max": 36.392799377441406,
      "activations/layer23_attention_weight_min": -27.957435607910156,
      "activations/layer2_attention_weight_max": 31.624835968017578,
      "activations/layer2_attention_weight_min": -29.835472106933594,
      "activations/layer3_attention_weight_max": 96.097412109375,
      "activations/layer3_attention_weight_min": -92.81088256835938,
      "activations/layer4_attention_weight_max": 86.0198974609375,
      "activations/layer4_attention_weight_min": -88.60890197753906,
      "activations/layer5_attention_weight_max": 62.72739791870117,
      "activations/layer5_attention_weight_min": -71.36012268066406,
      "activations/layer6_attention_weight_max": 49.67863464355469,
      "activations/layer6_attention_weight_min": -52.35474395751953,
      "activations/layer7_attention_weight_max": 69.5103530883789,
      "activations/layer7_attention_weight_min": -70.71163940429688,
      "activations/layer8_attention_weight_max": 48.955657958984375,
      "activations/layer8_attention_weight_min": -50.86296844482422,
      "activations/layer9_attention_weight_max": 46.38895797729492,
      "activations/layer9_attention_weight_min": -46.28066635131836,
      "epoch": 22.86,
      "learning_rate": 2.599242424242424e-06,
      "loss": 2.7157,
      "step": 393450
    },
    {
      "activations/layer0_attention_weight_max": 15.870387077331543,
      "activations/layer0_attention_weight_min": -14.461808204650879,
      "activations/layer10_attention_weight_max": 36.92390441894531,
      "activations/layer10_attention_weight_min": -35.48914337158203,
      "activations/layer11_attention_weight_max": 34.85023498535156,
      "activations/layer11_attention_weight_min": -37.53593444824219,
      "activations/layer12_attention_weight_max": 20.12859535217285,
      "activations/layer12_attention_weight_min": -24.59748649597168,
      "activations/layer13_attention_weight_max": 33.51386260986328,
      "activations/layer13_attention_weight_min": -29.854103088378906,
      "activations/layer14_attention_weight_max": 34.45612335205078,
      "activations/layer14_attention_weight_min": -31.788785934448242,
      "activations/layer15_attention_weight_max": 34.09721755981445,
      "activations/layer15_attention_weight_min": -32.38928985595703,
      "activations/layer16_attention_weight_max": 35.83521270751953,
      "activations/layer16_attention_weight_min": -34.22496795654297,
      "activations/layer17_attention_weight_max": 56.25034713745117,
      "activations/layer17_attention_weight_min": -50.099342346191406,
      "activations/layer18_attention_weight_max": 47.630943298339844,
      "activations/layer18_attention_weight_min": -44.29690933227539,
      "activations/layer19_attention_weight_max": 24.999553680419922,
      "activations/layer19_attention_weight_min": -27.023578643798828,
      "activations/layer1_attention_weight_max": 15.595049858093262,
      "activations/layer1_attention_weight_min": -15.653416633605957,
      "activations/layer20_attention_weight_max": 23.8494815826416,
      "activations/layer20_attention_weight_min": -23.947877883911133,
      "activations/layer21_attention_weight_max": 38.58875274658203,
      "activations/layer21_attention_weight_min": -23.49538230895996,
      "activations/layer22_attention_weight_max": 28.39209747314453,
      "activations/layer22_attention_weight_min": -28.18263053894043,
      "activations/layer23_attention_weight_max": 35.13801956176758,
      "activations/layer23_attention_weight_min": -24.3643856048584,
      "activations/layer2_attention_weight_max": 32.03325653076172,
      "activations/layer2_attention_weight_min": -30.37609100341797,
      "activations/layer3_attention_weight_max": 92.14103698730469,
      "activations/layer3_attention_weight_min": -88.35326385498047,
      "activations/layer4_attention_weight_max": 89.4298324584961,
      "activations/layer4_attention_weight_min": -88.5777359008789,
      "activations/layer5_attention_weight_max": 61.74937057495117,
      "activations/layer5_attention_weight_min": -71.17393493652344,
      "activations/layer6_attention_weight_max": 49.371700286865234,
      "activations/layer6_attention_weight_min": -51.914180755615234,
      "activations/layer7_attention_weight_max": 68.46479034423828,
      "activations/layer7_attention_weight_min": -71.19247436523438,
      "activations/layer8_attention_weight_max": 49.33076095581055,
      "activations/layer8_attention_weight_min": -51.248653411865234,
      "activations/layer9_attention_weight_max": 47.57511901855469,
      "activations/layer9_attention_weight_min": -44.66563034057617,
      "epoch": 22.86,
      "learning_rate": 2.58030303030303e-06,
      "loss": 2.6981,
      "step": 393500
    },
    {
      "activations/layer0_attention_weight_max": 15.628411293029785,
      "activations/layer0_attention_weight_min": -14.164300918579102,
      "activations/layer10_attention_weight_max": 34.221153259277344,
      "activations/layer10_attention_weight_min": -35.07257843017578,
      "activations/layer11_attention_weight_max": 34.03871536254883,
      "activations/layer11_attention_weight_min": -34.45718002319336,
      "activations/layer12_attention_weight_max": 19.55185890197754,
      "activations/layer12_attention_weight_min": -26.173871994018555,
      "activations/layer13_attention_weight_max": 28.209505081176758,
      "activations/layer13_attention_weight_min": -26.07208824157715,
      "activations/layer14_attention_weight_max": 34.45160675048828,
      "activations/layer14_attention_weight_min": -28.630842208862305,
      "activations/layer15_attention_weight_max": 34.56367874145508,
      "activations/layer15_attention_weight_min": -29.361099243164062,
      "activations/layer16_attention_weight_max": 35.83743667602539,
      "activations/layer16_attention_weight_min": -33.69914627075195,
      "activations/layer17_attention_weight_max": 52.20180892944336,
      "activations/layer17_attention_weight_min": -44.961307525634766,
      "activations/layer18_attention_weight_max": 49.400569915771484,
      "activations/layer18_attention_weight_min": -40.31066131591797,
      "activations/layer19_attention_weight_max": 27.141958236694336,
      "activations/layer19_attention_weight_min": -22.48798370361328,
      "activations/layer1_attention_weight_max": 15.309767723083496,
      "activations/layer1_attention_weight_min": -14.734986305236816,
      "activations/layer20_attention_weight_max": 25.38174057006836,
      "activations/layer20_attention_weight_min": -23.052507400512695,
      "activations/layer21_attention_weight_max": 39.62722396850586,
      "activations/layer21_attention_weight_min": -19.797632217407227,
      "activations/layer22_attention_weight_max": 27.063541412353516,
      "activations/layer22_attention_weight_min": -22.98161506652832,
      "activations/layer23_attention_weight_max": 35.79118347167969,
      "activations/layer23_attention_weight_min": -23.803546905517578,
      "activations/layer2_attention_weight_max": 30.591419219970703,
      "activations/layer2_attention_weight_min": -31.131650924682617,
      "activations/layer3_attention_weight_max": 87.30079650878906,
      "activations/layer3_attention_weight_min": -90.93714904785156,
      "activations/layer4_attention_weight_max": 85.67132568359375,
      "activations/layer4_attention_weight_min": -84.4887924194336,
      "activations/layer5_attention_weight_max": 62.44237518310547,
      "activations/layer5_attention_weight_min": -70.10491180419922,
      "activations/layer6_attention_weight_max": 50.507713317871094,
      "activations/layer6_attention_weight_min": -53.04069519042969,
      "activations/layer7_attention_weight_max": 69.68184661865234,
      "activations/layer7_attention_weight_min": -66.87004089355469,
      "activations/layer8_attention_weight_max": 46.6531867980957,
      "activations/layer8_attention_weight_min": -50.168434143066406,
      "activations/layer9_attention_weight_max": 44.87866973876953,
      "activations/layer9_attention_weight_min": -45.759456634521484,
      "epoch": 22.87,
      "learning_rate": 2.5613636363636363e-06,
      "loss": 2.6928,
      "step": 393550
    },
    {
      "activations/layer0_attention_weight_max": 15.711821556091309,
      "activations/layer0_attention_weight_min": -14.268205642700195,
      "activations/layer10_attention_weight_max": 35.074649810791016,
      "activations/layer10_attention_weight_min": -36.101318359375,
      "activations/layer11_attention_weight_max": 31.901473999023438,
      "activations/layer11_attention_weight_min": -34.70274353027344,
      "activations/layer12_attention_weight_max": 20.495386123657227,
      "activations/layer12_attention_weight_min": -28.07660484313965,
      "activations/layer13_attention_weight_max": 29.206762313842773,
      "activations/layer13_attention_weight_min": -26.815284729003906,
      "activations/layer14_attention_weight_max": 34.07497024536133,
      "activations/layer14_attention_weight_min": -29.24908447265625,
      "activations/layer15_attention_weight_max": 31.741247177124023,
      "activations/layer15_attention_weight_min": -30.6959171295166,
      "activations/layer16_attention_weight_max": 33.07759475708008,
      "activations/layer16_attention_weight_min": -32.467376708984375,
      "activations/layer17_attention_weight_max": 47.44903564453125,
      "activations/layer17_attention_weight_min": -46.53166580200195,
      "activations/layer18_attention_weight_max": 41.953853607177734,
      "activations/layer18_attention_weight_min": -39.49813461303711,
      "activations/layer19_attention_weight_max": 22.43465232849121,
      "activations/layer19_attention_weight_min": -22.835050582885742,
      "activations/layer1_attention_weight_max": 14.589943885803223,
      "activations/layer1_attention_weight_min": -15.358766555786133,
      "activations/layer20_attention_weight_max": 20.611223220825195,
      "activations/layer20_attention_weight_min": -23.187435150146484,
      "activations/layer21_attention_weight_max": 32.999542236328125,
      "activations/layer21_attention_weight_min": -20.72340202331543,
      "activations/layer22_attention_weight_max": 26.308320999145508,
      "activations/layer22_attention_weight_min": -25.42853546142578,
      "activations/layer23_attention_weight_max": 34.13548278808594,
      "activations/layer23_attention_weight_min": -23.358341217041016,
      "activations/layer2_attention_weight_max": 32.01352310180664,
      "activations/layer2_attention_weight_min": -30.09939956665039,
      "activations/layer3_attention_weight_max": 90.19792175292969,
      "activations/layer3_attention_weight_min": -89.87267303466797,
      "activations/layer4_attention_weight_max": 85.3117904663086,
      "activations/layer4_attention_weight_min": -87.4305648803711,
      "activations/layer5_attention_weight_max": 62.05841064453125,
      "activations/layer5_attention_weight_min": -74.51240539550781,
      "activations/layer6_attention_weight_max": 51.29374313354492,
      "activations/layer6_attention_weight_min": -50.425167083740234,
      "activations/layer7_attention_weight_max": 71.24422454833984,
      "activations/layer7_attention_weight_min": -68.54833221435547,
      "activations/layer8_attention_weight_max": 48.88444519042969,
      "activations/layer8_attention_weight_min": -49.723480224609375,
      "activations/layer9_attention_weight_max": 44.86332321166992,
      "activations/layer9_attention_weight_min": -45.026405334472656,
      "epoch": 22.87,
      "learning_rate": 2.5424242424242424e-06,
      "loss": 2.7004,
      "step": 393600
    },
    {
      "activations/layer0_attention_weight_max": 15.69547176361084,
      "activations/layer0_attention_weight_min": -14.342069625854492,
      "activations/layer10_attention_weight_max": 34.78203201293945,
      "activations/layer10_attention_weight_min": -36.849483489990234,
      "activations/layer11_attention_weight_max": 32.47472381591797,
      "activations/layer11_attention_weight_min": -36.79132843017578,
      "activations/layer12_attention_weight_max": 20.016870498657227,
      "activations/layer12_attention_weight_min": -26.7825870513916,
      "activations/layer13_attention_weight_max": 34.143672943115234,
      "activations/layer13_attention_weight_min": -27.46628189086914,
      "activations/layer14_attention_weight_max": 33.98134231567383,
      "activations/layer14_attention_weight_min": -32.23396682739258,
      "activations/layer15_attention_weight_max": 33.5423469543457,
      "activations/layer15_attention_weight_min": -31.759504318237305,
      "activations/layer16_attention_weight_max": 34.383514404296875,
      "activations/layer16_attention_weight_min": -35.462127685546875,
      "activations/layer17_attention_weight_max": 51.29502487182617,
      "activations/layer17_attention_weight_min": -45.36882781982422,
      "activations/layer18_attention_weight_max": 46.93088150024414,
      "activations/layer18_attention_weight_min": -41.90390396118164,
      "activations/layer19_attention_weight_max": 25.781909942626953,
      "activations/layer19_attention_weight_min": -23.585159301757812,
      "activations/layer1_attention_weight_max": 14.472321510314941,
      "activations/layer1_attention_weight_min": -14.636302947998047,
      "activations/layer20_attention_weight_max": 24.20067596435547,
      "activations/layer20_attention_weight_min": -21.877004623413086,
      "activations/layer21_attention_weight_max": 38.177528381347656,
      "activations/layer21_attention_weight_min": -24.313430786132812,
      "activations/layer22_attention_weight_max": 29.69025230407715,
      "activations/layer22_attention_weight_min": -25.429536819458008,
      "activations/layer23_attention_weight_max": 37.36436462402344,
      "activations/layer23_attention_weight_min": -26.504743576049805,
      "activations/layer2_attention_weight_max": 31.305110931396484,
      "activations/layer2_attention_weight_min": -30.66423797607422,
      "activations/layer3_attention_weight_max": 90.56708526611328,
      "activations/layer3_attention_weight_min": -96.48614501953125,
      "activations/layer4_attention_weight_max": 84.7342758178711,
      "activations/layer4_attention_weight_min": -85.27037811279297,
      "activations/layer5_attention_weight_max": 60.64289855957031,
      "activations/layer5_attention_weight_min": -76.65392303466797,
      "activations/layer6_attention_weight_max": 48.12012481689453,
      "activations/layer6_attention_weight_min": -52.32542419433594,
      "activations/layer7_attention_weight_max": 67.61278533935547,
      "activations/layer7_attention_weight_min": -69.57072448730469,
      "activations/layer8_attention_weight_max": 45.8781852722168,
      "activations/layer8_attention_weight_min": -49.742835998535156,
      "activations/layer9_attention_weight_max": 47.74086380004883,
      "activations/layer9_attention_weight_min": -46.61433410644531,
      "epoch": 22.87,
      "learning_rate": 2.523484848484848e-06,
      "loss": 2.7033,
      "step": 393650
    },
    {
      "activations/layer0_attention_weight_max": 15.488348960876465,
      "activations/layer0_attention_weight_min": -14.0245943069458,
      "activations/layer10_attention_weight_max": 34.09822463989258,
      "activations/layer10_attention_weight_min": -35.2505989074707,
      "activations/layer11_attention_weight_max": 32.603878021240234,
      "activations/layer11_attention_weight_min": -34.61990737915039,
      "activations/layer12_attention_weight_max": 19.287582397460938,
      "activations/layer12_attention_weight_min": -25.10879898071289,
      "activations/layer13_attention_weight_max": 33.29985046386719,
      "activations/layer13_attention_weight_min": -29.017602920532227,
      "activations/layer14_attention_weight_max": 35.72537612915039,
      "activations/layer14_attention_weight_min": -29.691631317138672,
      "activations/layer15_attention_weight_max": 33.08979415893555,
      "activations/layer15_attention_weight_min": -29.72089958190918,
      "activations/layer16_attention_weight_max": 34.32877731323242,
      "activations/layer16_attention_weight_min": -33.383880615234375,
      "activations/layer17_attention_weight_max": 53.12333297729492,
      "activations/layer17_attention_weight_min": -48.30002212524414,
      "activations/layer18_attention_weight_max": 46.72915267944336,
      "activations/layer18_attention_weight_min": -46.40901184082031,
      "activations/layer19_attention_weight_max": 27.88047218322754,
      "activations/layer19_attention_weight_min": -24.7963924407959,
      "activations/layer1_attention_weight_max": 16.241086959838867,
      "activations/layer1_attention_weight_min": -14.883716583251953,
      "activations/layer20_attention_weight_max": 25.466323852539062,
      "activations/layer20_attention_weight_min": -21.952539443969727,
      "activations/layer21_attention_weight_max": 35.1066780090332,
      "activations/layer21_attention_weight_min": -21.183368682861328,
      "activations/layer22_attention_weight_max": 29.13203239440918,
      "activations/layer22_attention_weight_min": -25.173614501953125,
      "activations/layer23_attention_weight_max": 36.727264404296875,
      "activations/layer23_attention_weight_min": -26.71162223815918,
      "activations/layer2_attention_weight_max": 32.78535461425781,
      "activations/layer2_attention_weight_min": -31.778236389160156,
      "activations/layer3_attention_weight_max": 90.6388931274414,
      "activations/layer3_attention_weight_min": -93.23307037353516,
      "activations/layer4_attention_weight_max": 84.56385803222656,
      "activations/layer4_attention_weight_min": -82.85942840576172,
      "activations/layer5_attention_weight_max": 59.82755661010742,
      "activations/layer5_attention_weight_min": -70.3094482421875,
      "activations/layer6_attention_weight_max": 50.54582977294922,
      "activations/layer6_attention_weight_min": -48.492271423339844,
      "activations/layer7_attention_weight_max": 69.18531799316406,
      "activations/layer7_attention_weight_min": -66.01624298095703,
      "activations/layer8_attention_weight_max": 45.84053421020508,
      "activations/layer8_attention_weight_min": -49.18848419189453,
      "activations/layer9_attention_weight_max": 42.811729431152344,
      "activations/layer9_attention_weight_min": -45.42690658569336,
      "epoch": 22.88,
      "learning_rate": 2.5045454545454543e-06,
      "loss": 2.6852,
      "step": 393700
    },
    {
      "activations/layer0_attention_weight_max": 15.61181640625,
      "activations/layer0_attention_weight_min": -14.124857902526855,
      "activations/layer10_attention_weight_max": 35.43813705444336,
      "activations/layer10_attention_weight_min": -36.19932556152344,
      "activations/layer11_attention_weight_max": 33.77555465698242,
      "activations/layer11_attention_weight_min": -34.04011535644531,
      "activations/layer12_attention_weight_max": 21.332151412963867,
      "activations/layer12_attention_weight_min": -24.40560531616211,
      "activations/layer13_attention_weight_max": 41.595314025878906,
      "activations/layer13_attention_weight_min": -31.53277587890625,
      "activations/layer14_attention_weight_max": 37.165611267089844,
      "activations/layer14_attention_weight_min": -32.977474212646484,
      "activations/layer15_attention_weight_max": 37.20166778564453,
      "activations/layer15_attention_weight_min": -33.99930191040039,
      "activations/layer16_attention_weight_max": 37.67885971069336,
      "activations/layer16_attention_weight_min": -36.93355178833008,
      "activations/layer17_attention_weight_max": 56.2446174621582,
      "activations/layer17_attention_weight_min": -50.95244216918945,
      "activations/layer18_attention_weight_max": 52.16855239868164,
      "activations/layer18_attention_weight_min": -48.276737213134766,
      "activations/layer19_attention_weight_max": 26.435850143432617,
      "activations/layer19_attention_weight_min": -26.30059242248535,
      "activations/layer1_attention_weight_max": 15.25263500213623,
      "activations/layer1_attention_weight_min": -15.401321411132812,
      "activations/layer20_attention_weight_max": 24.519359588623047,
      "activations/layer20_attention_weight_min": -21.927309036254883,
      "activations/layer21_attention_weight_max": 39.9959716796875,
      "activations/layer21_attention_weight_min": -25.90834617614746,
      "activations/layer22_attention_weight_max": 30.262781143188477,
      "activations/layer22_attention_weight_min": -26.585676193237305,
      "activations/layer23_attention_weight_max": 34.9774169921875,
      "activations/layer23_attention_weight_min": -25.10491943359375,
      "activations/layer2_attention_weight_max": 32.40300750732422,
      "activations/layer2_attention_weight_min": -30.06401824951172,
      "activations/layer3_attention_weight_max": 90.57440185546875,
      "activations/layer3_attention_weight_min": -88.03020477294922,
      "activations/layer4_attention_weight_max": 87.36144256591797,
      "activations/layer4_attention_weight_min": -87.38253021240234,
      "activations/layer5_attention_weight_max": 64.90909576416016,
      "activations/layer5_attention_weight_min": -71.8828125,
      "activations/layer6_attention_weight_max": 49.21274185180664,
      "activations/layer6_attention_weight_min": -49.167457580566406,
      "activations/layer7_attention_weight_max": 70.0512924194336,
      "activations/layer7_attention_weight_min": -70.6782455444336,
      "activations/layer8_attention_weight_max": 49.71653366088867,
      "activations/layer8_attention_weight_min": -51.71926498413086,
      "activations/layer9_attention_weight_max": 47.65994644165039,
      "activations/layer9_attention_weight_min": -46.076412200927734,
      "epoch": 22.88,
      "learning_rate": 2.48560606060606e-06,
      "loss": 2.6908,
      "step": 393750
    },
    {
      "activations/layer0_attention_weight_max": 15.85478401184082,
      "activations/layer0_attention_weight_min": -14.07274055480957,
      "activations/layer10_attention_weight_max": 35.240966796875,
      "activations/layer10_attention_weight_min": -35.18887710571289,
      "activations/layer11_attention_weight_max": 30.762174606323242,
      "activations/layer11_attention_weight_min": -33.86626052856445,
      "activations/layer12_attention_weight_max": 19.766162872314453,
      "activations/layer12_attention_weight_min": -25.420862197875977,
      "activations/layer13_attention_weight_max": 31.429738998413086,
      "activations/layer13_attention_weight_min": -27.710840225219727,
      "activations/layer14_attention_weight_max": 33.19950485229492,
      "activations/layer14_attention_weight_min": -30.095434188842773,
      "activations/layer15_attention_weight_max": 31.865650177001953,
      "activations/layer15_attention_weight_min": -29.726673126220703,
      "activations/layer16_attention_weight_max": 36.57333755493164,
      "activations/layer16_attention_weight_min": -34.75434875488281,
      "activations/layer17_attention_weight_max": 51.51834487915039,
      "activations/layer17_attention_weight_min": -44.91648483276367,
      "activations/layer18_attention_weight_max": 46.57588577270508,
      "activations/layer18_attention_weight_min": -39.96115493774414,
      "activations/layer19_attention_weight_max": 25.04437255859375,
      "activations/layer19_attention_weight_min": -23.924617767333984,
      "activations/layer1_attention_weight_max": 15.67651081085205,
      "activations/layer1_attention_weight_min": -14.184591293334961,
      "activations/layer20_attention_weight_max": 25.202085494995117,
      "activations/layer20_attention_weight_min": -24.010587692260742,
      "activations/layer21_attention_weight_max": 33.59888458251953,
      "activations/layer21_attention_weight_min": -23.006784439086914,
      "activations/layer22_attention_weight_max": 28.618528366088867,
      "activations/layer22_attention_weight_min": -24.4071044921875,
      "activations/layer23_attention_weight_max": 36.52168273925781,
      "activations/layer23_attention_weight_min": -25.563167572021484,
      "activations/layer2_attention_weight_max": 32.26083755493164,
      "activations/layer2_attention_weight_min": -32.03981018066406,
      "activations/layer3_attention_weight_max": 89.35071563720703,
      "activations/layer3_attention_weight_min": -95.3310317993164,
      "activations/layer4_attention_weight_max": 86.30612182617188,
      "activations/layer4_attention_weight_min": -87.5270004272461,
      "activations/layer5_attention_weight_max": 62.85337829589844,
      "activations/layer5_attention_weight_min": -71.9478530883789,
      "activations/layer6_attention_weight_max": 49.1635627746582,
      "activations/layer6_attention_weight_min": -49.716976165771484,
      "activations/layer7_attention_weight_max": 73.0328598022461,
      "activations/layer7_attention_weight_min": -70.09795379638672,
      "activations/layer8_attention_weight_max": 45.72771072387695,
      "activations/layer8_attention_weight_min": -48.94964599609375,
      "activations/layer9_attention_weight_max": 42.13615417480469,
      "activations/layer9_attention_weight_min": -45.59524917602539,
      "epoch": 22.88,
      "learning_rate": 2.4666666666666666e-06,
      "loss": 2.7078,
      "step": 393800
    },
    {
      "activations/layer0_attention_weight_max": 16.393810272216797,
      "activations/layer0_attention_weight_min": -14.453535079956055,
      "activations/layer10_attention_weight_max": 37.11669158935547,
      "activations/layer10_attention_weight_min": -37.3255615234375,
      "activations/layer11_attention_weight_max": 34.51985168457031,
      "activations/layer11_attention_weight_min": -36.920936584472656,
      "activations/layer12_attention_weight_max": 18.86371421813965,
      "activations/layer12_attention_weight_min": -24.8392276763916,
      "activations/layer13_attention_weight_max": 32.72074508666992,
      "activations/layer13_attention_weight_min": -29.444168090820312,
      "activations/layer14_attention_weight_max": 35.60190200805664,
      "activations/layer14_attention_weight_min": -30.95199966430664,
      "activations/layer15_attention_weight_max": 32.638668060302734,
      "activations/layer15_attention_weight_min": -32.06359100341797,
      "activations/layer16_attention_weight_max": 35.81261444091797,
      "activations/layer16_attention_weight_min": -31.756298065185547,
      "activations/layer17_attention_weight_max": 50.82966613769531,
      "activations/layer17_attention_weight_min": -44.31966018676758,
      "activations/layer18_attention_weight_max": 49.865928649902344,
      "activations/layer18_attention_weight_min": -39.16657638549805,
      "activations/layer19_attention_weight_max": 22.86562156677246,
      "activations/layer19_attention_weight_min": -21.598173141479492,
      "activations/layer1_attention_weight_max": 15.620094299316406,
      "activations/layer1_attention_weight_min": -13.18354320526123,
      "activations/layer20_attention_weight_max": 21.384794235229492,
      "activations/layer20_attention_weight_min": -19.721431732177734,
      "activations/layer21_attention_weight_max": 41.5258674621582,
      "activations/layer21_attention_weight_min": -22.943923950195312,
      "activations/layer22_attention_weight_max": 26.4081974029541,
      "activations/layer22_attention_weight_min": -24.96126937866211,
      "activations/layer23_attention_weight_max": 36.400550842285156,
      "activations/layer23_attention_weight_min": -24.235271453857422,
      "activations/layer2_attention_weight_max": 32.63267517089844,
      "activations/layer2_attention_weight_min": -30.846046447753906,
      "activations/layer3_attention_weight_max": 94.3330307006836,
      "activations/layer3_attention_weight_min": -91.59454345703125,
      "activations/layer4_attention_weight_max": 85.88683319091797,
      "activations/layer4_attention_weight_min": -87.58171844482422,
      "activations/layer5_attention_weight_max": 61.30538558959961,
      "activations/layer5_attention_weight_min": -75.97163391113281,
      "activations/layer6_attention_weight_max": 49.55451202392578,
      "activations/layer6_attention_weight_min": -55.106388092041016,
      "activations/layer7_attention_weight_max": 70.17121887207031,
      "activations/layer7_attention_weight_min": -71.57415008544922,
      "activations/layer8_attention_weight_max": 51.53647232055664,
      "activations/layer8_attention_weight_min": -55.176265716552734,
      "activations/layer9_attention_weight_max": 46.09789276123047,
      "activations/layer9_attention_weight_min": -49.28676986694336,
      "epoch": 22.88,
      "learning_rate": 2.4477272727272724e-06,
      "loss": 2.7115,
      "step": 393850
    },
    {
      "activations/layer0_attention_weight_max": 16.015399932861328,
      "activations/layer0_attention_weight_min": -14.358838081359863,
      "activations/layer10_attention_weight_max": 37.3304557800293,
      "activations/layer10_attention_weight_min": -38.67850875854492,
      "activations/layer11_attention_weight_max": 34.09123992919922,
      "activations/layer11_attention_weight_min": -35.28533172607422,
      "activations/layer12_attention_weight_max": 18.744749069213867,
      "activations/layer12_attention_weight_min": -26.210060119628906,
      "activations/layer13_attention_weight_max": 36.3470458984375,
      "activations/layer13_attention_weight_min": -31.324966430664062,
      "activations/layer14_attention_weight_max": 36.29486846923828,
      "activations/layer14_attention_weight_min": -32.30562973022461,
      "activations/layer15_attention_weight_max": 34.60329055786133,
      "activations/layer15_attention_weight_min": -31.351993560791016,
      "activations/layer16_attention_weight_max": 37.523311614990234,
      "activations/layer16_attention_weight_min": -34.69516372680664,
      "activations/layer17_attention_weight_max": 56.886268615722656,
      "activations/layer17_attention_weight_min": -49.95621871948242,
      "activations/layer18_attention_weight_max": 53.055118560791016,
      "activations/layer18_attention_weight_min": -43.76632308959961,
      "activations/layer19_attention_weight_max": 25.44977378845215,
      "activations/layer19_attention_weight_min": -24.624372482299805,
      "activations/layer1_attention_weight_max": 14.800825119018555,
      "activations/layer1_attention_weight_min": -14.331902503967285,
      "activations/layer20_attention_weight_max": 23.394956588745117,
      "activations/layer20_attention_weight_min": -22.4329891204834,
      "activations/layer21_attention_weight_max": 44.26213455200195,
      "activations/layer21_attention_weight_min": -20.412002563476562,
      "activations/layer22_attention_weight_max": 33.02616500854492,
      "activations/layer22_attention_weight_min": -26.420631408691406,
      "activations/layer23_attention_weight_max": 39.48881912231445,
      "activations/layer23_attention_weight_min": -24.89706802368164,
      "activations/layer2_attention_weight_max": 32.12940979003906,
      "activations/layer2_attention_weight_min": -31.72754669189453,
      "activations/layer3_attention_weight_max": 92.35917663574219,
      "activations/layer3_attention_weight_min": -92.32866668701172,
      "activations/layer4_attention_weight_max": 86.84728240966797,
      "activations/layer4_attention_weight_min": -90.35496520996094,
      "activations/layer5_attention_weight_max": 60.427955627441406,
      "activations/layer5_attention_weight_min": -73.36419677734375,
      "activations/layer6_attention_weight_max": 51.075557708740234,
      "activations/layer6_attention_weight_min": -55.45988464355469,
      "activations/layer7_attention_weight_max": 69.75472259521484,
      "activations/layer7_attention_weight_min": -70.35403442382812,
      "activations/layer8_attention_weight_max": 48.37213134765625,
      "activations/layer8_attention_weight_min": -51.8060188293457,
      "activations/layer9_attention_weight_max": 48.77194595336914,
      "activations/layer9_attention_weight_min": -46.06966018676758,
      "epoch": 22.89,
      "learning_rate": 2.4287878787878785e-06,
      "loss": 2.7068,
      "step": 393900
    },
    {
      "activations/layer0_attention_weight_max": 15.767097473144531,
      "activations/layer0_attention_weight_min": -14.077136039733887,
      "activations/layer10_attention_weight_max": 35.59208679199219,
      "activations/layer10_attention_weight_min": -38.6038932800293,
      "activations/layer11_attention_weight_max": 32.22003173828125,
      "activations/layer11_attention_weight_min": -35.651947021484375,
      "activations/layer12_attention_weight_max": 17.865697860717773,
      "activations/layer12_attention_weight_min": -25.558073043823242,
      "activations/layer13_attention_weight_max": 34.11757278442383,
      "activations/layer13_attention_weight_min": -27.037105560302734,
      "activations/layer14_attention_weight_max": 31.426265716552734,
      "activations/layer14_attention_weight_min": -27.58698081970215,
      "activations/layer15_attention_weight_max": 31.47226905822754,
      "activations/layer15_attention_weight_min": -28.84844207763672,
      "activations/layer16_attention_weight_max": 31.760713577270508,
      "activations/layer16_attention_weight_min": -30.687101364135742,
      "activations/layer17_attention_weight_max": 47.683143615722656,
      "activations/layer17_attention_weight_min": -46.983604431152344,
      "activations/layer18_attention_weight_max": 43.67417907714844,
      "activations/layer18_attention_weight_min": -39.31237030029297,
      "activations/layer19_attention_weight_max": 24.60440444946289,
      "activations/layer19_attention_weight_min": -23.5606746673584,
      "activations/layer1_attention_weight_max": 15.767049789428711,
      "activations/layer1_attention_weight_min": -14.591750144958496,
      "activations/layer20_attention_weight_max": 20.97361946105957,
      "activations/layer20_attention_weight_min": -22.692480087280273,
      "activations/layer21_attention_weight_max": 33.79287338256836,
      "activations/layer21_attention_weight_min": -27.06289291381836,
      "activations/layer22_attention_weight_max": 28.91793441772461,
      "activations/layer22_attention_weight_min": -23.218645095825195,
      "activations/layer23_attention_weight_max": 34.56745910644531,
      "activations/layer23_attention_weight_min": -23.999059677124023,
      "activations/layer2_attention_weight_max": 33.2955322265625,
      "activations/layer2_attention_weight_min": -32.0927619934082,
      "activations/layer3_attention_weight_max": 94.38864135742188,
      "activations/layer3_attention_weight_min": -95.56134796142578,
      "activations/layer4_attention_weight_max": 84.52469635009766,
      "activations/layer4_attention_weight_min": -84.40957641601562,
      "activations/layer5_attention_weight_max": 62.60750961303711,
      "activations/layer5_attention_weight_min": -73.84820556640625,
      "activations/layer6_attention_weight_max": 48.45754623413086,
      "activations/layer6_attention_weight_min": -50.444740295410156,
      "activations/layer7_attention_weight_max": 67.14888763427734,
      "activations/layer7_attention_weight_min": -70.07592010498047,
      "activations/layer8_attention_weight_max": 47.86079025268555,
      "activations/layer8_attention_weight_min": -49.87523651123047,
      "activations/layer9_attention_weight_max": 44.56722640991211,
      "activations/layer9_attention_weight_min": -47.53446960449219,
      "epoch": 22.89,
      "learning_rate": 2.4098484848484842e-06,
      "loss": 2.7005,
      "step": 393950
    },
    {
      "activations/layer0_attention_weight_max": 15.918025970458984,
      "activations/layer0_attention_weight_min": -14.515607833862305,
      "activations/layer10_attention_weight_max": 37.51082229614258,
      "activations/layer10_attention_weight_min": -37.21013259887695,
      "activations/layer11_attention_weight_max": 35.44306945800781,
      "activations/layer11_attention_weight_min": -35.56636428833008,
      "activations/layer12_attention_weight_max": 19.89768409729004,
      "activations/layer12_attention_weight_min": -25.26862907409668,
      "activations/layer13_attention_weight_max": 44.44599151611328,
      "activations/layer13_attention_weight_min": -29.233779907226562,
      "activations/layer14_attention_weight_max": 44.046104431152344,
      "activations/layer14_attention_weight_min": -30.50568389892578,
      "activations/layer15_attention_weight_max": 41.98990249633789,
      "activations/layer15_attention_weight_min": -29.508581161499023,
      "activations/layer16_attention_weight_max": 34.023765563964844,
      "activations/layer16_attention_weight_min": -32.9469108581543,
      "activations/layer17_attention_weight_max": 49.201210021972656,
      "activations/layer17_attention_weight_min": -46.162078857421875,
      "activations/layer18_attention_weight_max": 47.37672424316406,
      "activations/layer18_attention_weight_min": -40.79025650024414,
      "activations/layer19_attention_weight_max": 25.08563232421875,
      "activations/layer19_attention_weight_min": -22.02389907836914,
      "activations/layer1_attention_weight_max": 16.39728355407715,
      "activations/layer1_attention_weight_min": -14.776103973388672,
      "activations/layer20_attention_weight_max": 22.635730743408203,
      "activations/layer20_attention_weight_min": -21.87287712097168,
      "activations/layer21_attention_weight_max": 37.17436599731445,
      "activations/layer21_attention_weight_min": -21.009626388549805,
      "activations/layer22_attention_weight_max": 27.395933151245117,
      "activations/layer22_attention_weight_min": -26.59601402282715,
      "activations/layer23_attention_weight_max": 39.4337158203125,
      "activations/layer23_attention_weight_min": -24.55879020690918,
      "activations/layer2_attention_weight_max": 32.100486755371094,
      "activations/layer2_attention_weight_min": -30.4830379486084,
      "activations/layer3_attention_weight_max": 89.2925033569336,
      "activations/layer3_attention_weight_min": -89.4314956665039,
      "activations/layer4_attention_weight_max": 86.41259002685547,
      "activations/layer4_attention_weight_min": -88.27767181396484,
      "activations/layer5_attention_weight_max": 60.92618942260742,
      "activations/layer5_attention_weight_min": -73.15443420410156,
      "activations/layer6_attention_weight_max": 49.12694549560547,
      "activations/layer6_attention_weight_min": -51.271949768066406,
      "activations/layer7_attention_weight_max": 70.13922119140625,
      "activations/layer7_attention_weight_min": -72.76508331298828,
      "activations/layer8_attention_weight_max": 47.95156478881836,
      "activations/layer8_attention_weight_min": -50.261131286621094,
      "activations/layer9_attention_weight_max": 52.212745666503906,
      "activations/layer9_attention_weight_min": -48.724491119384766,
      "epoch": 22.89,
      "learning_rate": 2.390909090909091e-06,
      "loss": 2.7132,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4847,
      "eval_samples_per_second": 506.088,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4847,
      "eval_openwebtext_samples_per_second": 506.088,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.996,
      "eval_wikitext_samples_per_second": 228.457,
      "step": 394000
    },
    {
      "epoch": 22.89,
      "eval_lambada_loss": 2.400390625,
      "eval_lambada_ppl": 11.027483150026422,
      "eval_lambada_runtime": 9.5782,
      "eval_lambada_samples_per_second": 508.344,
      "step": 394000
    },
    {
      "activations/layer0_attention_weight_max": 15.903959274291992,
      "activations/layer0_attention_weight_min": -13.92589282989502,
      "activations/layer10_attention_weight_max": 37.912574768066406,
      "activations/layer10_attention_weight_min": -37.04895782470703,
      "activations/layer11_attention_weight_max": 34.84628677368164,
      "activations/layer11_attention_weight_min": -37.7489128112793,
      "activations/layer12_attention_weight_max": 20.946325302124023,
      "activations/layer12_attention_weight_min": -26.822694778442383,
      "activations/layer13_attention_weight_max": 38.49440002441406,
      "activations/layer13_attention_weight_min": -30.01976203918457,
      "activations/layer14_attention_weight_max": 40.543548583984375,
      "activations/layer14_attention_weight_min": -36.19441223144531,
      "activations/layer15_attention_weight_max": 37.89805603027344,
      "activations/layer15_attention_weight_min": -32.65749740600586,
      "activations/layer16_attention_weight_max": 43.351165771484375,
      "activations/layer16_attention_weight_min": -40.06541442871094,
      "activations/layer17_attention_weight_max": 60.76590347290039,
      "activations/layer17_attention_weight_min": -52.44264602661133,
      "activations/layer18_attention_weight_max": 58.16743469238281,
      "activations/layer18_attention_weight_min": -46.24431610107422,
      "activations/layer19_attention_weight_max": 30.36101722717285,
      "activations/layer19_attention_weight_min": -26.924936294555664,
      "activations/layer1_attention_weight_max": 14.998045921325684,
      "activations/layer1_attention_weight_min": -14.105853080749512,
      "activations/layer20_attention_weight_max": 28.598188400268555,
      "activations/layer20_attention_weight_min": -22.53359031677246,
      "activations/layer21_attention_weight_max": 46.58585739135742,
      "activations/layer21_attention_weight_min": -25.86290168762207,
      "activations/layer22_attention_weight_max": 37.34980010986328,
      "activations/layer22_attention_weight_min": -28.05010986328125,
      "activations/layer23_attention_weight_max": 40.89086151123047,
      "activations/layer23_attention_weight_min": -25.40065574645996,
      "activations/layer2_attention_weight_max": 32.40821838378906,
      "activations/layer2_attention_weight_min": -30.58583641052246,
      "activations/layer3_attention_weight_max": 91.41759490966797,
      "activations/layer3_attention_weight_min": -88.78794860839844,
      "activations/layer4_attention_weight_max": 85.6085433959961,
      "activations/layer4_attention_weight_min": -85.6495361328125,
      "activations/layer5_attention_weight_max": 62.68269348144531,
      "activations/layer5_attention_weight_min": -73.30216979980469,
      "activations/layer6_attention_weight_max": 49.593650817871094,
      "activations/layer6_attention_weight_min": -51.39086151123047,
      "activations/layer7_attention_weight_max": 73.6302490234375,
      "activations/layer7_attention_weight_min": -68.78923034667969,
      "activations/layer8_attention_weight_max": 46.70198440551758,
      "activations/layer8_attention_weight_min": -50.48491287231445,
      "activations/layer9_attention_weight_max": 44.83523178100586,
      "activations/layer9_attention_weight_min": -45.86968231201172,
      "epoch": 22.9,
      "learning_rate": 2.371969696969697e-06,
      "loss": 2.6976,
      "step": 394050
    },
    {
      "activations/layer0_attention_weight_max": 16.402801513671875,
      "activations/layer0_attention_weight_min": -14.193461418151855,
      "activations/layer10_attention_weight_max": 40.51108169555664,
      "activations/layer10_attention_weight_min": -38.45672607421875,
      "activations/layer11_attention_weight_max": 34.918373107910156,
      "activations/layer11_attention_weight_min": -37.149314880371094,
      "activations/layer12_attention_weight_max": 21.36194610595703,
      "activations/layer12_attention_weight_min": -23.621488571166992,
      "activations/layer13_attention_weight_max": 41.340023040771484,
      "activations/layer13_attention_weight_min": -28.599468231201172,
      "activations/layer14_attention_weight_max": 40.364463806152344,
      "activations/layer14_attention_weight_min": -31.77598762512207,
      "activations/layer15_attention_weight_max": 39.89551544189453,
      "activations/layer15_attention_weight_min": -30.894304275512695,
      "activations/layer16_attention_weight_max": 40.49165725708008,
      "activations/layer16_attention_weight_min": -33.89480209350586,
      "activations/layer17_attention_weight_max": 55.39701461791992,
      "activations/layer17_attention_weight_min": -46.685943603515625,
      "activations/layer18_attention_weight_max": 50.67644500732422,
      "activations/layer18_attention_weight_min": -40.76721954345703,
      "activations/layer19_attention_weight_max": 24.16578483581543,
      "activations/layer19_attention_weight_min": -22.864721298217773,
      "activations/layer1_attention_weight_max": 15.07414436340332,
      "activations/layer1_attention_weight_min": -14.16507625579834,
      "activations/layer20_attention_weight_max": 22.216604232788086,
      "activations/layer20_attention_weight_min": -19.156654357910156,
      "activations/layer21_attention_weight_max": 37.414493560791016,
      "activations/layer21_attention_weight_min": -21.970319747924805,
      "activations/layer22_attention_weight_max": 31.667268753051758,
      "activations/layer22_attention_weight_min": -23.15444564819336,
      "activations/layer23_attention_weight_max": 36.4426155090332,
      "activations/layer23_attention_weight_min": -21.359088897705078,
      "activations/layer2_attention_weight_max": 32.15422821044922,
      "activations/layer2_attention_weight_min": -31.175479888916016,
      "activations/layer3_attention_weight_max": 92.07746887207031,
      "activations/layer3_attention_weight_min": -90.66477966308594,
      "activations/layer4_attention_weight_max": 89.67089080810547,
      "activations/layer4_attention_weight_min": -85.44401550292969,
      "activations/layer5_attention_weight_max": 65.51911163330078,
      "activations/layer5_attention_weight_min": -71.82795715332031,
      "activations/layer6_attention_weight_max": 52.66830062866211,
      "activations/layer6_attention_weight_min": -51.21916198730469,
      "activations/layer7_attention_weight_max": 67.75798797607422,
      "activations/layer7_attention_weight_min": -73.70834350585938,
      "activations/layer8_attention_weight_max": 49.18949890136719,
      "activations/layer8_attention_weight_min": -50.605499267578125,
      "activations/layer9_attention_weight_max": 47.653656005859375,
      "activations/layer9_attention_weight_min": -48.56260299682617,
      "epoch": 22.9,
      "learning_rate": 2.3534090909090907e-06,
      "loss": 2.7017,
      "step": 394100
    },
    {
      "activations/layer0_attention_weight_max": 15.881103515625,
      "activations/layer0_attention_weight_min": -13.94110107421875,
      "activations/layer10_attention_weight_max": 36.50346755981445,
      "activations/layer10_attention_weight_min": -36.541290283203125,
      "activations/layer11_attention_weight_max": 34.05719757080078,
      "activations/layer11_attention_weight_min": -36.35865783691406,
      "activations/layer12_attention_weight_max": 19.282184600830078,
      "activations/layer12_attention_weight_min": -27.145708084106445,
      "activations/layer13_attention_weight_max": 37.804100036621094,
      "activations/layer13_attention_weight_min": -27.17146110534668,
      "activations/layer14_attention_weight_max": 37.848777770996094,
      "activations/layer14_attention_weight_min": -34.72406768798828,
      "activations/layer15_attention_weight_max": 35.16061019897461,
      "activations/layer15_attention_weight_min": -31.899442672729492,
      "activations/layer16_attention_weight_max": 37.397491455078125,
      "activations/layer16_attention_weight_min": -34.71056365966797,
      "activations/layer17_attention_weight_max": 52.09162902832031,
      "activations/layer17_attention_weight_min": -46.082576751708984,
      "activations/layer18_attention_weight_max": 48.312557220458984,
      "activations/layer18_attention_weight_min": -42.57013702392578,
      "activations/layer19_attention_weight_max": 26.372833251953125,
      "activations/layer19_attention_weight_min": -25.837751388549805,
      "activations/layer1_attention_weight_max": 15.057552337646484,
      "activations/layer1_attention_weight_min": -14.703173637390137,
      "activations/layer20_attention_weight_max": 24.683940887451172,
      "activations/layer20_attention_weight_min": -24.46038055419922,
      "activations/layer21_attention_weight_max": 38.90125274658203,
      "activations/layer21_attention_weight_min": -24.287994384765625,
      "activations/layer22_attention_weight_max": 29.309383392333984,
      "activations/layer22_attention_weight_min": -27.861434936523438,
      "activations/layer23_attention_weight_max": 37.49347686767578,
      "activations/layer23_attention_weight_min": -24.149715423583984,
      "activations/layer2_attention_weight_max": 32.662254333496094,
      "activations/layer2_attention_weight_min": -31.563541412353516,
      "activations/layer3_attention_weight_max": 93.07544708251953,
      "activations/layer3_attention_weight_min": -94.34886932373047,
      "activations/layer4_attention_weight_max": 87.9289321899414,
      "activations/layer4_attention_weight_min": -87.00884246826172,
      "activations/layer5_attention_weight_max": 63.334835052490234,
      "activations/layer5_attention_weight_min": -74.9543685913086,
      "activations/layer6_attention_weight_max": 50.6762809753418,
      "activations/layer6_attention_weight_min": -52.823604583740234,
      "activations/layer7_attention_weight_max": 72.90079498291016,
      "activations/layer7_attention_weight_min": -72.33894348144531,
      "activations/layer8_attention_weight_max": 47.837432861328125,
      "activations/layer8_attention_weight_min": -51.057342529296875,
      "activations/layer9_attention_weight_max": 43.70160675048828,
      "activations/layer9_attention_weight_min": -46.10000228881836,
      "epoch": 22.9,
      "learning_rate": 2.334469696969697e-06,
      "loss": 2.705,
      "step": 394150
    },
    {
      "activations/layer0_attention_weight_max": 15.136387825012207,
      "activations/layer0_attention_weight_min": -14.397049903869629,
      "activations/layer10_attention_weight_max": 34.7095947265625,
      "activations/layer10_attention_weight_min": -36.276309967041016,
      "activations/layer11_attention_weight_max": 33.70647048950195,
      "activations/layer11_attention_weight_min": -35.39937973022461,
      "activations/layer12_attention_weight_max": 18.873779296875,
      "activations/layer12_attention_weight_min": -25.814929962158203,
      "activations/layer13_attention_weight_max": 33.92743682861328,
      "activations/layer13_attention_weight_min": -28.22750473022461,
      "activations/layer14_attention_weight_max": 35.8049201965332,
      "activations/layer14_attention_weight_min": -30.147323608398438,
      "activations/layer15_attention_weight_max": 34.263484954833984,
      "activations/layer15_attention_weight_min": -32.519771575927734,
      "activations/layer16_attention_weight_max": 34.48084259033203,
      "activations/layer16_attention_weight_min": -33.40402603149414,
      "activations/layer17_attention_weight_max": 50.50668716430664,
      "activations/layer17_attention_weight_min": -47.40583419799805,
      "activations/layer18_attention_weight_max": 47.680908203125,
      "activations/layer18_attention_weight_min": -41.398807525634766,
      "activations/layer19_attention_weight_max": 26.02326774597168,
      "activations/layer19_attention_weight_min": -24.628149032592773,
      "activations/layer1_attention_weight_max": 14.648221969604492,
      "activations/layer1_attention_weight_min": -13.798014640808105,
      "activations/layer20_attention_weight_max": 24.6475772857666,
      "activations/layer20_attention_weight_min": -22.760875701904297,
      "activations/layer21_attention_weight_max": 33.72921371459961,
      "activations/layer21_attention_weight_min": -21.122940063476562,
      "activations/layer22_attention_weight_max": 27.783479690551758,
      "activations/layer22_attention_weight_min": -24.8493709564209,
      "activations/layer23_attention_weight_max": 34.80281448364258,
      "activations/layer23_attention_weight_min": -24.057424545288086,
      "activations/layer2_attention_weight_max": 31.768428802490234,
      "activations/layer2_attention_weight_min": -30.456716537475586,
      "activations/layer3_attention_weight_max": 92.75316619873047,
      "activations/layer3_attention_weight_min": -93.71759033203125,
      "activations/layer4_attention_weight_max": 88.13850402832031,
      "activations/layer4_attention_weight_min": -87.56340789794922,
      "activations/layer5_attention_weight_max": 63.508384704589844,
      "activations/layer5_attention_weight_min": -75.93445587158203,
      "activations/layer6_attention_weight_max": 50.29234313964844,
      "activations/layer6_attention_weight_min": -52.35889434814453,
      "activations/layer7_attention_weight_max": 72.38264465332031,
      "activations/layer7_attention_weight_min": -70.00276947021484,
      "activations/layer8_attention_weight_max": 46.83131790161133,
      "activations/layer8_attention_weight_min": -50.78271484375,
      "activations/layer9_attention_weight_max": 42.39933395385742,
      "activations/layer9_attention_weight_min": -45.786190032958984,
      "epoch": 22.91,
      "learning_rate": 2.3155303030303026e-06,
      "loss": 2.6883,
      "step": 394200
    },
    {
      "activations/layer0_attention_weight_max": 15.581060409545898,
      "activations/layer0_attention_weight_min": -14.134263038635254,
      "activations/layer10_attention_weight_max": 34.076324462890625,
      "activations/layer10_attention_weight_min": -35.64985275268555,
      "activations/layer11_attention_weight_max": 33.628257751464844,
      "activations/layer11_attention_weight_min": -35.349830627441406,
      "activations/layer12_attention_weight_max": 20.710159301757812,
      "activations/layer12_attention_weight_min": -27.225791931152344,
      "activations/layer13_attention_weight_max": 35.252342224121094,
      "activations/layer13_attention_weight_min": -27.703771591186523,
      "activations/layer14_attention_weight_max": 37.94206237792969,
      "activations/layer14_attention_weight_min": -31.257478713989258,
      "activations/layer15_attention_weight_max": 35.169517517089844,
      "activations/layer15_attention_weight_min": -30.633167266845703,
      "activations/layer16_attention_weight_max": 38.55973434448242,
      "activations/layer16_attention_weight_min": -34.07423400878906,
      "activations/layer17_attention_weight_max": 54.411922454833984,
      "activations/layer17_attention_weight_min": -47.179664611816406,
      "activations/layer18_attention_weight_max": 50.88225555419922,
      "activations/layer18_attention_weight_min": -43.18804168701172,
      "activations/layer19_attention_weight_max": 27.438886642456055,
      "activations/layer19_attention_weight_min": -24.958913803100586,
      "activations/layer1_attention_weight_max": 16.463003158569336,
      "activations/layer1_attention_weight_min": -13.349915504455566,
      "activations/layer20_attention_weight_max": 22.702259063720703,
      "activations/layer20_attention_weight_min": -22.674497604370117,
      "activations/layer21_attention_weight_max": 38.2520866394043,
      "activations/layer21_attention_weight_min": -21.73504066467285,
      "activations/layer22_attention_weight_max": 29.026357650756836,
      "activations/layer22_attention_weight_min": -25.21148109436035,
      "activations/layer23_attention_weight_max": 35.847347259521484,
      "activations/layer23_attention_weight_min": -26.665220260620117,
      "activations/layer2_attention_weight_max": 31.054298400878906,
      "activations/layer2_attention_weight_min": -30.537948608398438,
      "activations/layer3_attention_weight_max": 89.6280288696289,
      "activations/layer3_attention_weight_min": -90.21319580078125,
      "activations/layer4_attention_weight_max": 85.82886505126953,
      "activations/layer4_attention_weight_min": -88.4854965209961,
      "activations/layer5_attention_weight_max": 63.7830924987793,
      "activations/layer5_attention_weight_min": -72.0953369140625,
      "activations/layer6_attention_weight_max": 51.05259704589844,
      "activations/layer6_attention_weight_min": -52.5794792175293,
      "activations/layer7_attention_weight_max": 71.89794921875,
      "activations/layer7_attention_weight_min": -69.28033447265625,
      "activations/layer8_attention_weight_max": 48.44929504394531,
      "activations/layer8_attention_weight_min": -50.97936248779297,
      "activations/layer9_attention_weight_max": 44.21255111694336,
      "activations/layer9_attention_weight_min": -45.630836486816406,
      "epoch": 22.91,
      "learning_rate": 2.2965909090909088e-06,
      "loss": 2.6951,
      "step": 394250
    },
    {
      "activations/layer0_attention_weight_max": 15.416893005371094,
      "activations/layer0_attention_weight_min": -14.131173133850098,
      "activations/layer10_attention_weight_max": 35.58941650390625,
      "activations/layer10_attention_weight_min": -36.34239959716797,
      "activations/layer11_attention_weight_max": 35.11027526855469,
      "activations/layer11_attention_weight_min": -35.66272735595703,
      "activations/layer12_attention_weight_max": 21.65376853942871,
      "activations/layer12_attention_weight_min": -29.381900787353516,
      "activations/layer13_attention_weight_max": 36.54561996459961,
      "activations/layer13_attention_weight_min": -28.539813995361328,
      "activations/layer14_attention_weight_max": 40.442726135253906,
      "activations/layer14_attention_weight_min": -35.603240966796875,
      "activations/layer15_attention_weight_max": 36.44454574584961,
      "activations/layer15_attention_weight_min": -36.25829315185547,
      "activations/layer16_attention_weight_max": 39.67631149291992,
      "activations/layer16_attention_weight_min": -36.964141845703125,
      "activations/layer17_attention_weight_max": 56.99195098876953,
      "activations/layer17_attention_weight_min": -50.043399810791016,
      "activations/layer18_attention_weight_max": 51.1765251159668,
      "activations/layer18_attention_weight_min": -42.42973327636719,
      "activations/layer19_attention_weight_max": 24.478303909301758,
      "activations/layer19_attention_weight_min": -23.44569969177246,
      "activations/layer1_attention_weight_max": 14.48088550567627,
      "activations/layer1_attention_weight_min": -14.628396034240723,
      "activations/layer20_attention_weight_max": 21.155452728271484,
      "activations/layer20_attention_weight_min": -22.431926727294922,
      "activations/layer21_attention_weight_max": 35.81986999511719,
      "activations/layer21_attention_weight_min": -21.896020889282227,
      "activations/layer22_attention_weight_max": 28.371145248413086,
      "activations/layer22_attention_weight_min": -27.501985549926758,
      "activations/layer23_attention_weight_max": 37.407588958740234,
      "activations/layer23_attention_weight_min": -24.84058952331543,
      "activations/layer2_attention_weight_max": 31.938838958740234,
      "activations/layer2_attention_weight_min": -30.18848991394043,
      "activations/layer3_attention_weight_max": 90.30516052246094,
      "activations/layer3_attention_weight_min": -89.24237060546875,
      "activations/layer4_attention_weight_max": 86.9760513305664,
      "activations/layer4_attention_weight_min": -85.39810180664062,
      "activations/layer5_attention_weight_max": 63.729644775390625,
      "activations/layer5_attention_weight_min": -73.40947723388672,
      "activations/layer6_attention_weight_max": 50.00025177001953,
      "activations/layer6_attention_weight_min": -51.44179916381836,
      "activations/layer7_attention_weight_max": 75.09925079345703,
      "activations/layer7_attention_weight_min": -67.87593841552734,
      "activations/layer8_attention_weight_max": 49.8399772644043,
      "activations/layer8_attention_weight_min": -50.19679641723633,
      "activations/layer9_attention_weight_max": 43.403377532958984,
      "activations/layer9_attention_weight_min": -46.49285125732422,
      "epoch": 22.91,
      "learning_rate": 2.277651515151515e-06,
      "loss": 2.7169,
      "step": 394300
    },
    {
      "activations/layer0_attention_weight_max": 16.219099044799805,
      "activations/layer0_attention_weight_min": -13.876495361328125,
      "activations/layer10_attention_weight_max": 39.0413818359375,
      "activations/layer10_attention_weight_min": -38.52245330810547,
      "activations/layer11_attention_weight_max": 36.97123718261719,
      "activations/layer11_attention_weight_min": -39.80165100097656,
      "activations/layer12_attention_weight_max": 22.144363403320312,
      "activations/layer12_attention_weight_min": -24.115991592407227,
      "activations/layer13_attention_weight_max": 36.573402404785156,
      "activations/layer13_attention_weight_min": -28.997196197509766,
      "activations/layer14_attention_weight_max": 36.637420654296875,
      "activations/layer14_attention_weight_min": -33.971065521240234,
      "activations/layer15_attention_weight_max": 33.78553009033203,
      "activations/layer15_attention_weight_min": -33.990631103515625,
      "activations/layer16_attention_weight_max": 36.47317886352539,
      "activations/layer16_attention_weight_min": -35.16327667236328,
      "activations/layer17_attention_weight_max": 53.938621520996094,
      "activations/layer17_attention_weight_min": -47.24352264404297,
      "activations/layer18_attention_weight_max": 51.457061767578125,
      "activations/layer18_attention_weight_min": -42.14132308959961,
      "activations/layer19_attention_weight_max": 27.683197021484375,
      "activations/layer19_attention_weight_min": -22.422786712646484,
      "activations/layer1_attention_weight_max": 15.099017143249512,
      "activations/layer1_attention_weight_min": -14.4835205078125,
      "activations/layer20_attention_weight_max": 23.878543853759766,
      "activations/layer20_attention_weight_min": -20.785457611083984,
      "activations/layer21_attention_weight_max": 34.415794372558594,
      "activations/layer21_attention_weight_min": -22.087268829345703,
      "activations/layer22_attention_weight_max": 27.09909439086914,
      "activations/layer22_attention_weight_min": -23.902559280395508,
      "activations/layer23_attention_weight_max": 37.559226989746094,
      "activations/layer23_attention_weight_min": -23.42458724975586,
      "activations/layer2_attention_weight_max": 33.92719268798828,
      "activations/layer2_attention_weight_min": -31.4985408782959,
      "activations/layer3_attention_weight_max": 95.1176528930664,
      "activations/layer3_attention_weight_min": -92.71360778808594,
      "activations/layer4_attention_weight_max": 89.00904083251953,
      "activations/layer4_attention_weight_min": -88.02123260498047,
      "activations/layer5_attention_weight_max": 63.17390060424805,
      "activations/layer5_attention_weight_min": -73.36203002929688,
      "activations/layer6_attention_weight_max": 52.40968704223633,
      "activations/layer6_attention_weight_min": -53.69388198852539,
      "activations/layer7_attention_weight_max": 72.06571197509766,
      "activations/layer7_attention_weight_min": -73.81539916992188,
      "activations/layer8_attention_weight_max": 50.82753372192383,
      "activations/layer8_attention_weight_min": -55.53469467163086,
      "activations/layer9_attention_weight_max": 48.59873580932617,
      "activations/layer9_attention_weight_min": -48.5817985534668,
      "epoch": 22.91,
      "learning_rate": 2.258712121212121e-06,
      "loss": 2.6978,
      "step": 394350
    },
    {
      "activations/layer0_attention_weight_max": 15.541278839111328,
      "activations/layer0_attention_weight_min": -14.449880599975586,
      "activations/layer10_attention_weight_max": 35.091583251953125,
      "activations/layer10_attention_weight_min": -35.9343147277832,
      "activations/layer11_attention_weight_max": 33.448143005371094,
      "activations/layer11_attention_weight_min": -35.930335998535156,
      "activations/layer12_attention_weight_max": 20.492185592651367,
      "activations/layer12_attention_weight_min": -24.136178970336914,
      "activations/layer13_attention_weight_max": 31.537904739379883,
      "activations/layer13_attention_weight_min": -26.726070404052734,
      "activations/layer14_attention_weight_max": 36.95656204223633,
      "activations/layer14_attention_weight_min": -30.261632919311523,
      "activations/layer15_attention_weight_max": 35.19020080566406,
      "activations/layer15_attention_weight_min": -30.94402313232422,
      "activations/layer16_attention_weight_max": 35.676025390625,
      "activations/layer16_attention_weight_min": -33.68227767944336,
      "activations/layer17_attention_weight_max": 55.80159378051758,
      "activations/layer17_attention_weight_min": -49.33890151977539,
      "activations/layer18_attention_weight_max": 48.71504211425781,
      "activations/layer18_attention_weight_min": -44.216888427734375,
      "activations/layer19_attention_weight_max": 27.106409072875977,
      "activations/layer19_attention_weight_min": -25.47195053100586,
      "activations/layer1_attention_weight_max": 14.7046480178833,
      "activations/layer1_attention_weight_min": -14.742982864379883,
      "activations/layer20_attention_weight_max": 23.99479866027832,
      "activations/layer20_attention_weight_min": -25.350379943847656,
      "activations/layer21_attention_weight_max": 35.35791015625,
      "activations/layer21_attention_weight_min": -22.64865493774414,
      "activations/layer22_attention_weight_max": 29.46617889404297,
      "activations/layer22_attention_weight_min": -26.51986312866211,
      "activations/layer23_attention_weight_max": 37.167442321777344,
      "activations/layer23_attention_weight_min": -24.28371238708496,
      "activations/layer2_attention_weight_max": 31.5676326751709,
      "activations/layer2_attention_weight_min": -29.31024932861328,
      "activations/layer3_attention_weight_max": 89.61649322509766,
      "activations/layer3_attention_weight_min": -88.81883239746094,
      "activations/layer4_attention_weight_max": 84.91082000732422,
      "activations/layer4_attention_weight_min": -86.4261474609375,
      "activations/layer5_attention_weight_max": 65.23494720458984,
      "activations/layer5_attention_weight_min": -71.99281311035156,
      "activations/layer6_attention_weight_max": 50.34682083129883,
      "activations/layer6_attention_weight_min": -50.52573013305664,
      "activations/layer7_attention_weight_max": 70.33253479003906,
      "activations/layer7_attention_weight_min": -67.43233489990234,
      "activations/layer8_attention_weight_max": 47.71888732910156,
      "activations/layer8_attention_weight_min": -54.259117126464844,
      "activations/layer9_attention_weight_max": 43.960575103759766,
      "activations/layer9_attention_weight_min": -46.22984313964844,
      "epoch": 22.92,
      "learning_rate": 2.2397727272727272e-06,
      "loss": 2.6905,
      "step": 394400
    },
    {
      "activations/layer0_attention_weight_max": 15.761810302734375,
      "activations/layer0_attention_weight_min": -14.35026741027832,
      "activations/layer10_attention_weight_max": 35.74152755737305,
      "activations/layer10_attention_weight_min": -37.81391143798828,
      "activations/layer11_attention_weight_max": 33.6200065612793,
      "activations/layer11_attention_weight_min": -35.86994171142578,
      "activations/layer12_attention_weight_max": 20.495695114135742,
      "activations/layer12_attention_weight_min": -29.97276496887207,
      "activations/layer13_attention_weight_max": 38.35871505737305,
      "activations/layer13_attention_weight_min": -28.2691593170166,
      "activations/layer14_attention_weight_max": 37.559146881103516,
      "activations/layer14_attention_weight_min": -32.11409378051758,
      "activations/layer15_attention_weight_max": 36.07070541381836,
      "activations/layer15_attention_weight_min": -32.030052185058594,
      "activations/layer16_attention_weight_max": 37.83827590942383,
      "activations/layer16_attention_weight_min": -35.9857292175293,
      "activations/layer17_attention_weight_max": 55.452789306640625,
      "activations/layer17_attention_weight_min": -50.72979736328125,
      "activations/layer18_attention_weight_max": 55.04716491699219,
      "activations/layer18_attention_weight_min": -47.817481994628906,
      "activations/layer19_attention_weight_max": 27.205015182495117,
      "activations/layer19_attention_weight_min": -27.35050392150879,
      "activations/layer1_attention_weight_max": 15.444267272949219,
      "activations/layer1_attention_weight_min": -14.149582862854004,
      "activations/layer20_attention_weight_max": 27.105632781982422,
      "activations/layer20_attention_weight_min": -26.339208602905273,
      "activations/layer21_attention_weight_max": 37.2012939453125,
      "activations/layer21_attention_weight_min": -26.03094482421875,
      "activations/layer22_attention_weight_max": 30.201541900634766,
      "activations/layer22_attention_weight_min": -25.991954803466797,
      "activations/layer23_attention_weight_max": 39.04545593261719,
      "activations/layer23_attention_weight_min": -24.87445640563965,
      "activations/layer2_attention_weight_max": 32.774627685546875,
      "activations/layer2_attention_weight_min": -31.079837799072266,
      "activations/layer3_attention_weight_max": 93.21377563476562,
      "activations/layer3_attention_weight_min": -91.05073547363281,
      "activations/layer4_attention_weight_max": 86.79173278808594,
      "activations/layer4_attention_weight_min": -88.61065673828125,
      "activations/layer5_attention_weight_max": 66.33985900878906,
      "activations/layer5_attention_weight_min": -73.93708038330078,
      "activations/layer6_attention_weight_max": 50.683414459228516,
      "activations/layer6_attention_weight_min": -52.03609848022461,
      "activations/layer7_attention_weight_max": 70.68538665771484,
      "activations/layer7_attention_weight_min": -67.5189208984375,
      "activations/layer8_attention_weight_max": 47.367183685302734,
      "activations/layer8_attention_weight_min": -52.640411376953125,
      "activations/layer9_attention_weight_max": 45.025611877441406,
      "activations/layer9_attention_weight_min": -47.27531051635742,
      "epoch": 22.92,
      "learning_rate": 2.2208333333333334e-06,
      "loss": 2.7054,
      "step": 394450
    },
    {
      "activations/layer0_attention_weight_max": 16.214706420898438,
      "activations/layer0_attention_weight_min": -14.10314655303955,
      "activations/layer10_attention_weight_max": 35.61167907714844,
      "activations/layer10_attention_weight_min": -38.09321212768555,
      "activations/layer11_attention_weight_max": 34.554317474365234,
      "activations/layer11_attention_weight_min": -34.68184280395508,
      "activations/layer12_attention_weight_max": 20.99911117553711,
      "activations/layer12_attention_weight_min": -25.520288467407227,
      "activations/layer13_attention_weight_max": 34.712318420410156,
      "activations/layer13_attention_weight_min": -28.487960815429688,
      "activations/layer14_attention_weight_max": 37.280094146728516,
      "activations/layer14_attention_weight_min": -31.5837459564209,
      "activations/layer15_attention_weight_max": 34.16536331176758,
      "activations/layer15_attention_weight_min": -31.98052978515625,
      "activations/layer16_attention_weight_max": 35.87025451660156,
      "activations/layer16_attention_weight_min": -35.348670959472656,
      "activations/layer17_attention_weight_max": 53.11214065551758,
      "activations/layer17_attention_weight_min": -51.269615173339844,
      "activations/layer18_attention_weight_max": 48.25235366821289,
      "activations/layer18_attention_weight_min": -46.56001663208008,
      "activations/layer19_attention_weight_max": 25.421253204345703,
      "activations/layer19_attention_weight_min": -27.638736724853516,
      "activations/layer1_attention_weight_max": 15.430538177490234,
      "activations/layer1_attention_weight_min": -14.344871520996094,
      "activations/layer20_attention_weight_max": 24.464536666870117,
      "activations/layer20_attention_weight_min": -24.10563087463379,
      "activations/layer21_attention_weight_max": 36.63615798950195,
      "activations/layer21_attention_weight_min": -23.98518943786621,
      "activations/layer22_attention_weight_max": 29.04989242553711,
      "activations/layer22_attention_weight_min": -26.904273986816406,
      "activations/layer23_attention_weight_max": 38.60075759887695,
      "activations/layer23_attention_weight_min": -24.257606506347656,
      "activations/layer2_attention_weight_max": 31.95717430114746,
      "activations/layer2_attention_weight_min": -31.320890426635742,
      "activations/layer3_attention_weight_max": 93.18350219726562,
      "activations/layer3_attention_weight_min": -89.87677001953125,
      "activations/layer4_attention_weight_max": 85.32994079589844,
      "activations/layer4_attention_weight_min": -85.98426818847656,
      "activations/layer5_attention_weight_max": 63.473716735839844,
      "activations/layer5_attention_weight_min": -74.07709503173828,
      "activations/layer6_attention_weight_max": 50.57926940917969,
      "activations/layer6_attention_weight_min": -52.15445327758789,
      "activations/layer7_attention_weight_max": 69.65030670166016,
      "activations/layer7_attention_weight_min": -72.53421020507812,
      "activations/layer8_attention_weight_max": 50.0247802734375,
      "activations/layer8_attention_weight_min": -52.946739196777344,
      "activations/layer9_attention_weight_max": 47.32499313354492,
      "activations/layer9_attention_weight_min": -47.037776947021484,
      "epoch": 22.92,
      "learning_rate": 2.201893939393939e-06,
      "loss": 2.7062,
      "step": 394500
    },
    {
      "activations/layer0_attention_weight_max": 16.03223419189453,
      "activations/layer0_attention_weight_min": -14.355777740478516,
      "activations/layer10_attention_weight_max": 36.74277114868164,
      "activations/layer10_attention_weight_min": -38.665645599365234,
      "activations/layer11_attention_weight_max": 35.06322479248047,
      "activations/layer11_attention_weight_min": -38.36787414550781,
      "activations/layer12_attention_weight_max": 19.510446548461914,
      "activations/layer12_attention_weight_min": -23.162078857421875,
      "activations/layer13_attention_weight_max": 37.257259368896484,
      "activations/layer13_attention_weight_min": -30.471736907958984,
      "activations/layer14_attention_weight_max": 35.50284194946289,
      "activations/layer14_attention_weight_min": -30.51992416381836,
      "activations/layer15_attention_weight_max": 34.301025390625,
      "activations/layer15_attention_weight_min": -31.789567947387695,
      "activations/layer16_attention_weight_max": 38.120845794677734,
      "activations/layer16_attention_weight_min": -34.883033752441406,
      "activations/layer17_attention_weight_max": 55.78666305541992,
      "activations/layer17_attention_weight_min": -49.853084564208984,
      "activations/layer18_attention_weight_max": 48.55028533935547,
      "activations/layer18_attention_weight_min": -41.93014907836914,
      "activations/layer19_attention_weight_max": 26.40618324279785,
      "activations/layer19_attention_weight_min": -23.082706451416016,
      "activations/layer1_attention_weight_max": 15.319598197937012,
      "activations/layer1_attention_weight_min": -14.41584300994873,
      "activations/layer20_attention_weight_max": 24.012975692749023,
      "activations/layer20_attention_weight_min": -22.554471969604492,
      "activations/layer21_attention_weight_max": 40.018550872802734,
      "activations/layer21_attention_weight_min": -23.908615112304688,
      "activations/layer22_attention_weight_max": 29.92782211303711,
      "activations/layer22_attention_weight_min": -24.875675201416016,
      "activations/layer23_attention_weight_max": 34.5362548828125,
      "activations/layer23_attention_weight_min": -26.01831817626953,
      "activations/layer2_attention_weight_max": 33.28266906738281,
      "activations/layer2_attention_weight_min": -31.977741241455078,
      "activations/layer3_attention_weight_max": 94.12413787841797,
      "activations/layer3_attention_weight_min": -92.07312774658203,
      "activations/layer4_attention_weight_max": 89.25969696044922,
      "activations/layer4_attention_weight_min": -87.12062072753906,
      "activations/layer5_attention_weight_max": 65.31902313232422,
      "activations/layer5_attention_weight_min": -70.9052734375,
      "activations/layer6_attention_weight_max": 51.06404113769531,
      "activations/layer6_attention_weight_min": -51.47276306152344,
      "activations/layer7_attention_weight_max": 73.39112854003906,
      "activations/layer7_attention_weight_min": -73.47460174560547,
      "activations/layer8_attention_weight_max": 48.6208381652832,
      "activations/layer8_attention_weight_min": -54.53318786621094,
      "activations/layer9_attention_weight_max": 46.48563003540039,
      "activations/layer9_attention_weight_min": -49.765262603759766,
      "epoch": 22.93,
      "learning_rate": 2.1829545454545453e-06,
      "loss": 2.7013,
      "step": 394550
    },
    {
      "activations/layer0_attention_weight_max": 15.99935245513916,
      "activations/layer0_attention_weight_min": -14.058863639831543,
      "activations/layer10_attention_weight_max": 39.01104736328125,
      "activations/layer10_attention_weight_min": -37.86194610595703,
      "activations/layer11_attention_weight_max": 34.478416442871094,
      "activations/layer11_attention_weight_min": -35.03857421875,
      "activations/layer12_attention_weight_max": 18.895469665527344,
      "activations/layer12_attention_weight_min": -24.873329162597656,
      "activations/layer13_attention_weight_max": 31.808910369873047,
      "activations/layer13_attention_weight_min": -25.849878311157227,
      "activations/layer14_attention_weight_max": 33.700965881347656,
      "activations/layer14_attention_weight_min": -28.306861877441406,
      "activations/layer15_attention_weight_max": 31.02837562561035,
      "activations/layer15_attention_weight_min": -29.649364471435547,
      "activations/layer16_attention_weight_max": 36.10062026977539,
      "activations/layer16_attention_weight_min": -33.23106384277344,
      "activations/layer17_attention_weight_max": 50.7161865234375,
      "activations/layer17_attention_weight_min": -43.842430114746094,
      "activations/layer18_attention_weight_max": 44.02123260498047,
      "activations/layer18_attention_weight_min": -40.34587097167969,
      "activations/layer19_attention_weight_max": 24.179441452026367,
      "activations/layer19_attention_weight_min": -22.560626983642578,
      "activations/layer1_attention_weight_max": 15.062612533569336,
      "activations/layer1_attention_weight_min": -15.175448417663574,
      "activations/layer20_attention_weight_max": 20.8424072265625,
      "activations/layer20_attention_weight_min": -21.417503356933594,
      "activations/layer21_attention_weight_max": 31.49119758605957,
      "activations/layer21_attention_weight_min": -24.59307289123535,
      "activations/layer22_attention_weight_max": 28.4101505279541,
      "activations/layer22_attention_weight_min": -28.102428436279297,
      "activations/layer23_attention_weight_max": 35.69468688964844,
      "activations/layer23_attention_weight_min": -26.071792602539062,
      "activations/layer2_attention_weight_max": 31.791357040405273,
      "activations/layer2_attention_weight_min": -31.439382553100586,
      "activations/layer3_attention_weight_max": 90.61381530761719,
      "activations/layer3_attention_weight_min": -92.02153778076172,
      "activations/layer4_attention_weight_max": 82.64985656738281,
      "activations/layer4_attention_weight_min": -86.14655303955078,
      "activations/layer5_attention_weight_max": 60.17696762084961,
      "activations/layer5_attention_weight_min": -71.21919250488281,
      "activations/layer6_attention_weight_max": 47.86176300048828,
      "activations/layer6_attention_weight_min": -53.066707611083984,
      "activations/layer7_attention_weight_max": 70.57463073730469,
      "activations/layer7_attention_weight_min": -71.74555969238281,
      "activations/layer8_attention_weight_max": 46.35463333129883,
      "activations/layer8_attention_weight_min": -53.95448303222656,
      "activations/layer9_attention_weight_max": 44.45144271850586,
      "activations/layer9_attention_weight_min": -48.16720962524414,
      "epoch": 22.93,
      "learning_rate": 2.1640151515151514e-06,
      "loss": 2.7212,
      "step": 394600
    },
    {
      "activations/layer0_attention_weight_max": 15.941514015197754,
      "activations/layer0_attention_weight_min": -14.199702262878418,
      "activations/layer10_attention_weight_max": 38.364295959472656,
      "activations/layer10_attention_weight_min": -41.848243713378906,
      "activations/layer11_attention_weight_max": 35.38328552246094,
      "activations/layer11_attention_weight_min": -38.97407913208008,
      "activations/layer12_attention_weight_max": 20.51656150817871,
      "activations/layer12_attention_weight_min": -27.889925003051758,
      "activations/layer13_attention_weight_max": 34.54439926147461,
      "activations/layer13_attention_weight_min": -28.615604400634766,
      "activations/layer14_attention_weight_max": 36.54515838623047,
      "activations/layer14_attention_weight_min": -30.661409378051758,
      "activations/layer15_attention_weight_max": 36.6111946105957,
      "activations/layer15_attention_weight_min": -31.35622215270996,
      "activations/layer16_attention_weight_max": 37.49757385253906,
      "activations/layer16_attention_weight_min": -35.67411422729492,
      "activations/layer17_attention_weight_max": 54.53604507446289,
      "activations/layer17_attention_weight_min": -49.87769317626953,
      "activations/layer18_attention_weight_max": 53.09142303466797,
      "activations/layer18_attention_weight_min": -41.595680236816406,
      "activations/layer19_attention_weight_max": 25.59663963317871,
      "activations/layer19_attention_weight_min": -24.273717880249023,
      "activations/layer1_attention_weight_max": 15.004378318786621,
      "activations/layer1_attention_weight_min": -15.573371887207031,
      "activations/layer20_attention_weight_max": 24.927024841308594,
      "activations/layer20_attention_weight_min": -24.865177154541016,
      "activations/layer21_attention_weight_max": 35.55896759033203,
      "activations/layer21_attention_weight_min": -22.819040298461914,
      "activations/layer22_attention_weight_max": 28.62869644165039,
      "activations/layer22_attention_weight_min": -26.25928497314453,
      "activations/layer23_attention_weight_max": 34.17073440551758,
      "activations/layer23_attention_weight_min": -24.79247283935547,
      "activations/layer2_attention_weight_max": 32.59107971191406,
      "activations/layer2_attention_weight_min": -30.129501342773438,
      "activations/layer3_attention_weight_max": 93.56082916259766,
      "activations/layer3_attention_weight_min": -89.64836120605469,
      "activations/layer4_attention_weight_max": 88.13099670410156,
      "activations/layer4_attention_weight_min": -85.56502532958984,
      "activations/layer5_attention_weight_max": 60.79690170288086,
      "activations/layer5_attention_weight_min": -70.73237609863281,
      "activations/layer6_attention_weight_max": 49.78450012207031,
      "activations/layer6_attention_weight_min": -52.039466857910156,
      "activations/layer7_attention_weight_max": 69.0055923461914,
      "activations/layer7_attention_weight_min": -70.46562194824219,
      "activations/layer8_attention_weight_max": 49.48313522338867,
      "activations/layer8_attention_weight_min": -53.75176239013672,
      "activations/layer9_attention_weight_max": 43.69700241088867,
      "activations/layer9_attention_weight_min": -47.588748931884766,
      "epoch": 22.93,
      "learning_rate": 2.1450757575757576e-06,
      "loss": 2.6922,
      "step": 394650
    },
    {
      "activations/layer0_attention_weight_max": 15.810468673706055,
      "activations/layer0_attention_weight_min": -14.427490234375,
      "activations/layer10_attention_weight_max": 38.49153518676758,
      "activations/layer10_attention_weight_min": -35.951324462890625,
      "activations/layer11_attention_weight_max": 38.821380615234375,
      "activations/layer11_attention_weight_min": -35.91846466064453,
      "activations/layer12_attention_weight_max": 19.741004943847656,
      "activations/layer12_attention_weight_min": -25.1742000579834,
      "activations/layer13_attention_weight_max": 45.99789047241211,
      "activations/layer13_attention_weight_min": -33.87726593017578,
      "activations/layer14_attention_weight_max": 42.09429931640625,
      "activations/layer14_attention_weight_min": -33.143497467041016,
      "activations/layer15_attention_weight_max": 47.189979553222656,
      "activations/layer15_attention_weight_min": -32.09858703613281,
      "activations/layer16_attention_weight_max": 37.199832916259766,
      "activations/layer16_attention_weight_min": -33.716209411621094,
      "activations/layer17_attention_weight_max": 54.99254608154297,
      "activations/layer17_attention_weight_min": -51.363162994384766,
      "activations/layer18_attention_weight_max": 49.39516067504883,
      "activations/layer18_attention_weight_min": -45.41843795776367,
      "activations/layer19_attention_weight_max": 26.62957191467285,
      "activations/layer19_attention_weight_min": -23.160818099975586,
      "activations/layer1_attention_weight_max": 14.509147644042969,
      "activations/layer1_attention_weight_min": -14.549126625061035,
      "activations/layer20_attention_weight_max": 24.1278018951416,
      "activations/layer20_attention_weight_min": -23.482542037963867,
      "activations/layer21_attention_weight_max": 34.39179229736328,
      "activations/layer21_attention_weight_min": -21.843692779541016,
      "activations/layer22_attention_weight_max": 29.516199111938477,
      "activations/layer22_attention_weight_min": -24.978174209594727,
      "activations/layer23_attention_weight_max": 37.926815032958984,
      "activations/layer23_attention_weight_min": -24.39633560180664,
      "activations/layer2_attention_weight_max": 36.255069732666016,
      "activations/layer2_attention_weight_min": -30.652355194091797,
      "activations/layer3_attention_weight_max": 97.27509307861328,
      "activations/layer3_attention_weight_min": -91.10995483398438,
      "activations/layer4_attention_weight_max": 89.67308044433594,
      "activations/layer4_attention_weight_min": -86.04922485351562,
      "activations/layer5_attention_weight_max": 64.72242736816406,
      "activations/layer5_attention_weight_min": -74.60821533203125,
      "activations/layer6_attention_weight_max": 54.54568862915039,
      "activations/layer6_attention_weight_min": -53.81513977050781,
      "activations/layer7_attention_weight_max": 73.95134735107422,
      "activations/layer7_attention_weight_min": -71.68799591064453,
      "activations/layer8_attention_weight_max": 52.20615005493164,
      "activations/layer8_attention_weight_min": -54.15702438354492,
      "activations/layer9_attention_weight_max": 51.493316650390625,
      "activations/layer9_attention_weight_min": -49.16123580932617,
      "epoch": 22.93,
      "learning_rate": 2.1261363636363633e-06,
      "loss": 2.7066,
      "step": 394700
    },
    {
      "activations/layer0_attention_weight_max": 15.996026039123535,
      "activations/layer0_attention_weight_min": -14.527637481689453,
      "activations/layer10_attention_weight_max": 41.15668487548828,
      "activations/layer10_attention_weight_min": -39.8343505859375,
      "activations/layer11_attention_weight_max": 37.77588653564453,
      "activations/layer11_attention_weight_min": -38.3033561706543,
      "activations/layer12_attention_weight_max": 20.328079223632812,
      "activations/layer12_attention_weight_min": -25.368526458740234,
      "activations/layer13_attention_weight_max": 37.8108024597168,
      "activations/layer13_attention_weight_min": -27.92475128173828,
      "activations/layer14_attention_weight_max": 39.0666389465332,
      "activations/layer14_attention_weight_min": -31.14619255065918,
      "activations/layer15_attention_weight_max": 36.210880279541016,
      "activations/layer15_attention_weight_min": -32.76310729980469,
      "activations/layer16_attention_weight_max": 38.31647872924805,
      "activations/layer16_attention_weight_min": -33.79915237426758,
      "activations/layer17_attention_weight_max": 57.67549514770508,
      "activations/layer17_attention_weight_min": -48.284912109375,
      "activations/layer18_attention_weight_max": 57.052574157714844,
      "activations/layer18_attention_weight_min": -41.28377914428711,
      "activations/layer19_attention_weight_max": 28.199146270751953,
      "activations/layer19_attention_weight_min": -23.583850860595703,
      "activations/layer1_attention_weight_max": 15.269965171813965,
      "activations/layer1_attention_weight_min": -15.11780071258545,
      "activations/layer20_attention_weight_max": 26.879581451416016,
      "activations/layer20_attention_weight_min": -23.020675659179688,
      "activations/layer21_attention_weight_max": 41.94004440307617,
      "activations/layer21_attention_weight_min": -22.30046844482422,
      "activations/layer22_attention_weight_max": 29.48771858215332,
      "activations/layer22_attention_weight_min": -24.61841583251953,
      "activations/layer23_attention_weight_max": 41.0501708984375,
      "activations/layer23_attention_weight_min": -24.275211334228516,
      "activations/layer2_attention_weight_max": 32.19369125366211,
      "activations/layer2_attention_weight_min": -31.64661407470703,
      "activations/layer3_attention_weight_max": 92.41979217529297,
      "activations/layer3_attention_weight_min": -95.17818450927734,
      "activations/layer4_attention_weight_max": 88.34671783447266,
      "activations/layer4_attention_weight_min": -91.98053741455078,
      "activations/layer5_attention_weight_max": 63.980464935302734,
      "activations/layer5_attention_weight_min": -72.45010375976562,
      "activations/layer6_attention_weight_max": 52.35673522949219,
      "activations/layer6_attention_weight_min": -56.00659942626953,
      "activations/layer7_attention_weight_max": 75.03315734863281,
      "activations/layer7_attention_weight_min": -72.4831771850586,
      "activations/layer8_attention_weight_max": 51.97507858276367,
      "activations/layer8_attention_weight_min": -57.12550354003906,
      "activations/layer9_attention_weight_max": 47.864444732666016,
      "activations/layer9_attention_weight_min": -51.76753616333008,
      "epoch": 22.94,
      "learning_rate": 2.1071969696969695e-06,
      "loss": 2.679,
      "step": 394750
    },
    {
      "activations/layer0_attention_weight_max": 15.984861373901367,
      "activations/layer0_attention_weight_min": -14.270379066467285,
      "activations/layer10_attention_weight_max": 37.83375930786133,
      "activations/layer10_attention_weight_min": -36.344547271728516,
      "activations/layer11_attention_weight_max": 34.54856872558594,
      "activations/layer11_attention_weight_min": -36.52932357788086,
      "activations/layer12_attention_weight_max": 19.060819625854492,
      "activations/layer12_attention_weight_min": -23.42140007019043,
      "activations/layer13_attention_weight_max": 38.917449951171875,
      "activations/layer13_attention_weight_min": -31.230703353881836,
      "activations/layer14_attention_weight_max": 35.89099884033203,
      "activations/layer14_attention_weight_min": -32.97785949707031,
      "activations/layer15_attention_weight_max": 33.306549072265625,
      "activations/layer15_attention_weight_min": -34.1324348449707,
      "activations/layer16_attention_weight_max": 34.38078308105469,
      "activations/layer16_attention_weight_min": -34.62009048461914,
      "activations/layer17_attention_weight_max": 50.557350158691406,
      "activations/layer17_attention_weight_min": -45.835609436035156,
      "activations/layer18_attention_weight_max": 46.924476623535156,
      "activations/layer18_attention_weight_min": -42.624610900878906,
      "activations/layer19_attention_weight_max": 24.421499252319336,
      "activations/layer19_attention_weight_min": -23.212583541870117,
      "activations/layer1_attention_weight_max": 14.542168617248535,
      "activations/layer1_attention_weight_min": -14.593110084533691,
      "activations/layer20_attention_weight_max": 24.603687286376953,
      "activations/layer20_attention_weight_min": -22.17391014099121,
      "activations/layer21_attention_weight_max": 35.960784912109375,
      "activations/layer21_attention_weight_min": -21.739437103271484,
      "activations/layer22_attention_weight_max": 29.04886245727539,
      "activations/layer22_attention_weight_min": -26.641754150390625,
      "activations/layer23_attention_weight_max": 35.07225799560547,
      "activations/layer23_attention_weight_min": -24.806167602539062,
      "activations/layer2_attention_weight_max": 31.52341079711914,
      "activations/layer2_attention_weight_min": -30.839527130126953,
      "activations/layer3_attention_weight_max": 90.97077941894531,
      "activations/layer3_attention_weight_min": -91.65042877197266,
      "activations/layer4_attention_weight_max": 84.37488555908203,
      "activations/layer4_attention_weight_min": -85.97025299072266,
      "activations/layer5_attention_weight_max": 63.63694381713867,
      "activations/layer5_attention_weight_min": -72.72190856933594,
      "activations/layer6_attention_weight_max": 53.27236557006836,
      "activations/layer6_attention_weight_min": -51.07400131225586,
      "activations/layer7_attention_weight_max": 67.35289764404297,
      "activations/layer7_attention_weight_min": -69.00563049316406,
      "activations/layer8_attention_weight_max": 51.10654067993164,
      "activations/layer8_attention_weight_min": -54.48955535888672,
      "activations/layer9_attention_weight_max": 44.99569320678711,
      "activations/layer9_attention_weight_min": -50.395301818847656,
      "epoch": 22.94,
      "learning_rate": 2.0882575757575756e-06,
      "loss": 2.6929,
      "step": 394800
    },
    {
      "activations/layer0_attention_weight_max": 16.233524322509766,
      "activations/layer0_attention_weight_min": -14.395796775817871,
      "activations/layer10_attention_weight_max": 34.00647735595703,
      "activations/layer10_attention_weight_min": -35.69047546386719,
      "activations/layer11_attention_weight_max": 32.88541030883789,
      "activations/layer11_attention_weight_min": -35.23632049560547,
      "activations/layer12_attention_weight_max": 18.620744705200195,
      "activations/layer12_attention_weight_min": -24.014318466186523,
      "activations/layer13_attention_weight_max": 31.395862579345703,
      "activations/layer13_attention_weight_min": -28.372772216796875,
      "activations/layer14_attention_weight_max": 32.55165481567383,
      "activations/layer14_attention_weight_min": -33.02848434448242,
      "activations/layer15_attention_weight_max": 30.361255645751953,
      "activations/layer15_attention_weight_min": -31.570188522338867,
      "activations/layer16_attention_weight_max": 33.95256805419922,
      "activations/layer16_attention_weight_min": -34.78485870361328,
      "activations/layer17_attention_weight_max": 51.555931091308594,
      "activations/layer17_attention_weight_min": -46.77399444580078,
      "activations/layer18_attention_weight_max": 46.52106475830078,
      "activations/layer18_attention_weight_min": -39.989376068115234,
      "activations/layer19_attention_weight_max": 26.365751266479492,
      "activations/layer19_attention_weight_min": -23.789731979370117,
      "activations/layer1_attention_weight_max": 15.516607284545898,
      "activations/layer1_attention_weight_min": -14.818153381347656,
      "activations/layer20_attention_weight_max": 23.051605224609375,
      "activations/layer20_attention_weight_min": -20.24083137512207,
      "activations/layer21_attention_weight_max": 33.87422561645508,
      "activations/layer21_attention_weight_min": -21.699203491210938,
      "activations/layer22_attention_weight_max": 28.72382354736328,
      "activations/layer22_attention_weight_min": -24.523483276367188,
      "activations/layer23_attention_weight_max": 34.13352966308594,
      "activations/layer23_attention_weight_min": -21.770877838134766,
      "activations/layer2_attention_weight_max": 29.975997924804688,
      "activations/layer2_attention_weight_min": -29.803695678710938,
      "activations/layer3_attention_weight_max": 85.33922576904297,
      "activations/layer3_attention_weight_min": -90.18563079833984,
      "activations/layer4_attention_weight_max": 83.00724029541016,
      "activations/layer4_attention_weight_min": -85.0228042602539,
      "activations/layer5_attention_weight_max": 60.910335540771484,
      "activations/layer5_attention_weight_min": -75.41163635253906,
      "activations/layer6_attention_weight_max": 46.59650802612305,
      "activations/layer6_attention_weight_min": -49.54808044433594,
      "activations/layer7_attention_weight_max": 68.60932922363281,
      "activations/layer7_attention_weight_min": -67.36243438720703,
      "activations/layer8_attention_weight_max": 45.21236801147461,
      "activations/layer8_attention_weight_min": -50.365264892578125,
      "activations/layer9_attention_weight_max": 43.12782287597656,
      "activations/layer9_attention_weight_min": -44.219295501708984,
      "epoch": 22.94,
      "learning_rate": 2.0693181818181813e-06,
      "loss": 2.6952,
      "step": 394850
    },
    {
      "activations/layer0_attention_weight_max": 16.06435775756836,
      "activations/layer0_attention_weight_min": -14.063835144042969,
      "activations/layer10_attention_weight_max": 35.32453918457031,
      "activations/layer10_attention_weight_min": -38.924842834472656,
      "activations/layer11_attention_weight_max": 33.517486572265625,
      "activations/layer11_attention_weight_min": -38.04986572265625,
      "activations/layer12_attention_weight_max": 17.149110794067383,
      "activations/layer12_attention_weight_min": -24.44945526123047,
      "activations/layer13_attention_weight_max": 38.57579040527344,
      "activations/layer13_attention_weight_min": -27.159875869750977,
      "activations/layer14_attention_weight_max": 38.38331985473633,
      "activations/layer14_attention_weight_min": -29.331872940063477,
      "activations/layer15_attention_weight_max": 33.15000534057617,
      "activations/layer15_attention_weight_min": -29.930248260498047,
      "activations/layer16_attention_weight_max": 34.02346420288086,
      "activations/layer16_attention_weight_min": -32.93073272705078,
      "activations/layer17_attention_weight_max": 53.108131408691406,
      "activations/layer17_attention_weight_min": -48.32156753540039,
      "activations/layer18_attention_weight_max": 46.3534049987793,
      "activations/layer18_attention_weight_min": -41.92101287841797,
      "activations/layer19_attention_weight_max": 23.725522994995117,
      "activations/layer19_attention_weight_min": -23.05497169494629,
      "activations/layer1_attention_weight_max": 15.616661071777344,
      "activations/layer1_attention_weight_min": -13.791914939880371,
      "activations/layer20_attention_weight_max": 21.44858169555664,
      "activations/layer20_attention_weight_min": -22.918439865112305,
      "activations/layer21_attention_weight_max": 37.04118347167969,
      "activations/layer21_attention_weight_min": -21.14596176147461,
      "activations/layer22_attention_weight_max": 27.55873680114746,
      "activations/layer22_attention_weight_min": -24.132118225097656,
      "activations/layer23_attention_weight_max": 38.12364959716797,
      "activations/layer23_attention_weight_min": -23.65814208984375,
      "activations/layer2_attention_weight_max": 33.913818359375,
      "activations/layer2_attention_weight_min": -32.652950286865234,
      "activations/layer3_attention_weight_max": 96.36144256591797,
      "activations/layer3_attention_weight_min": -99.12250518798828,
      "activations/layer4_attention_weight_max": 89.2127914428711,
      "activations/layer4_attention_weight_min": -91.68609619140625,
      "activations/layer5_attention_weight_max": 60.012020111083984,
      "activations/layer5_attention_weight_min": -70.25755310058594,
      "activations/layer6_attention_weight_max": 49.503902435302734,
      "activations/layer6_attention_weight_min": -52.53840637207031,
      "activations/layer7_attention_weight_max": 70.58895111083984,
      "activations/layer7_attention_weight_min": -72.56558227539062,
      "activations/layer8_attention_weight_max": 48.60161590576172,
      "activations/layer8_attention_weight_min": -51.11981201171875,
      "activations/layer9_attention_weight_max": 49.314247131347656,
      "activations/layer9_attention_weight_min": -49.28853988647461,
      "epoch": 22.95,
      "learning_rate": 2.050378787878788e-06,
      "loss": 2.7023,
      "step": 394900
    },
    {
      "activations/layer0_attention_weight_max": 15.553841590881348,
      "activations/layer0_attention_weight_min": -14.502385139465332,
      "activations/layer10_attention_weight_max": 34.654823303222656,
      "activations/layer10_attention_weight_min": -35.95244598388672,
      "activations/layer11_attention_weight_max": 31.863994598388672,
      "activations/layer11_attention_weight_min": -34.70948791503906,
      "activations/layer12_attention_weight_max": 18.69867515563965,
      "activations/layer12_attention_weight_min": -24.090999603271484,
      "activations/layer13_attention_weight_max": 33.87860870361328,
      "activations/layer13_attention_weight_min": -32.0535774230957,
      "activations/layer14_attention_weight_max": 32.399330139160156,
      "activations/layer14_attention_weight_min": -29.61822509765625,
      "activations/layer15_attention_weight_max": 30.17452049255371,
      "activations/layer15_attention_weight_min": -30.21891975402832,
      "activations/layer16_attention_weight_max": 31.89827537536621,
      "activations/layer16_attention_weight_min": -30.905717849731445,
      "activations/layer17_attention_weight_max": 46.372596740722656,
      "activations/layer17_attention_weight_min": -45.10770034790039,
      "activations/layer18_attention_weight_max": 45.686073303222656,
      "activations/layer18_attention_weight_min": -40.138423919677734,
      "activations/layer19_attention_weight_max": 25.591947555541992,
      "activations/layer19_attention_weight_min": -21.948251724243164,
      "activations/layer1_attention_weight_max": 15.252582550048828,
      "activations/layer1_attention_weight_min": -14.539946556091309,
      "activations/layer20_attention_weight_max": 23.608152389526367,
      "activations/layer20_attention_weight_min": -21.736892700195312,
      "activations/layer21_attention_weight_max": 33.15919876098633,
      "activations/layer21_attention_weight_min": -21.8450984954834,
      "activations/layer22_attention_weight_max": 30.050094604492188,
      "activations/layer22_attention_weight_min": -24.663312911987305,
      "activations/layer23_attention_weight_max": 34.304420471191406,
      "activations/layer23_attention_weight_min": -24.780315399169922,
      "activations/layer2_attention_weight_max": 31.377334594726562,
      "activations/layer2_attention_weight_min": -31.11463737487793,
      "activations/layer3_attention_weight_max": 90.28473663330078,
      "activations/layer3_attention_weight_min": -95.1955795288086,
      "activations/layer4_attention_weight_max": 83.94524383544922,
      "activations/layer4_attention_weight_min": -84.29976654052734,
      "activations/layer5_attention_weight_max": 63.58407211303711,
      "activations/layer5_attention_weight_min": -72.29684448242188,
      "activations/layer6_attention_weight_max": 50.28690719604492,
      "activations/layer6_attention_weight_min": -50.74723434448242,
      "activations/layer7_attention_weight_max": 68.4921646118164,
      "activations/layer7_attention_weight_min": -70.95055389404297,
      "activations/layer8_attention_weight_max": 47.86296463012695,
      "activations/layer8_attention_weight_min": -51.477081298828125,
      "activations/layer9_attention_weight_max": 42.80656051635742,
      "activations/layer9_attention_weight_min": -47.162654876708984,
      "epoch": 22.95,
      "learning_rate": 2.0314393939393936e-06,
      "loss": 2.705,
      "step": 394950
    },
    {
      "activations/layer0_attention_weight_max": 15.982528686523438,
      "activations/layer0_attention_weight_min": -14.296224594116211,
      "activations/layer10_attention_weight_max": 34.41529083251953,
      "activations/layer10_attention_weight_min": -35.48802185058594,
      "activations/layer11_attention_weight_max": 32.196529388427734,
      "activations/layer11_attention_weight_min": -35.65968704223633,
      "activations/layer12_attention_weight_max": 20.036680221557617,
      "activations/layer12_attention_weight_min": -23.36652183532715,
      "activations/layer13_attention_weight_max": 38.86019515991211,
      "activations/layer13_attention_weight_min": -29.083383560180664,
      "activations/layer14_attention_weight_max": 37.258846282958984,
      "activations/layer14_attention_weight_min": -32.228633880615234,
      "activations/layer15_attention_weight_max": 36.331241607666016,
      "activations/layer15_attention_weight_min": -32.459720611572266,
      "activations/layer16_attention_weight_max": 36.22987365722656,
      "activations/layer16_attention_weight_min": -34.32099533081055,
      "activations/layer17_attention_weight_max": 51.66869354248047,
      "activations/layer17_attention_weight_min": -47.27981185913086,
      "activations/layer18_attention_weight_max": 47.80839157104492,
      "activations/layer18_attention_weight_min": -40.67853546142578,
      "activations/layer19_attention_weight_max": 26.416147232055664,
      "activations/layer19_attention_weight_min": -25.461172103881836,
      "activations/layer1_attention_weight_max": 14.826197624206543,
      "activations/layer1_attention_weight_min": -14.880291938781738,
      "activations/layer20_attention_weight_max": 24.81218147277832,
      "activations/layer20_attention_weight_min": -27.625877380371094,
      "activations/layer21_attention_weight_max": 39.73190689086914,
      "activations/layer21_attention_weight_min": -25.24456787109375,
      "activations/layer22_attention_weight_max": 31.272045135498047,
      "activations/layer22_attention_weight_min": -25.701589584350586,
      "activations/layer23_attention_weight_max": 38.27589416503906,
      "activations/layer23_attention_weight_min": -22.87905502319336,
      "activations/layer2_attention_weight_max": 31.27667999267578,
      "activations/layer2_attention_weight_min": -30.80432891845703,
      "activations/layer3_attention_weight_max": 91.68169403076172,
      "activations/layer3_attention_weight_min": -92.05692291259766,
      "activations/layer4_attention_weight_max": 86.84532928466797,
      "activations/layer4_attention_weight_min": -87.318359375,
      "activations/layer5_attention_weight_max": 63.015411376953125,
      "activations/layer5_attention_weight_min": -71.00999450683594,
      "activations/layer6_attention_weight_max": 50.03574752807617,
      "activations/layer6_attention_weight_min": -50.96137619018555,
      "activations/layer7_attention_weight_max": 72.55812072753906,
      "activations/layer7_attention_weight_min": -72.15453338623047,
      "activations/layer8_attention_weight_max": 46.8183479309082,
      "activations/layer8_attention_weight_min": -49.7082633972168,
      "activations/layer9_attention_weight_max": 45.96243667602539,
      "activations/layer9_attention_weight_min": -46.214744567871094,
      "epoch": 22.95,
      "learning_rate": 2.0125e-06,
      "loss": 2.6933,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_loss": 2.6640625,
      "eval_runtime": 8.4788,
      "eval_samples_per_second": 506.441,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_openwebtext_loss": 2.6640625,
      "eval_openwebtext_ppl": 14.354485905365031,
      "eval_openwebtext_runtime": 8.4788,
      "eval_openwebtext_samples_per_second": 506.441,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9764,
      "eval_wikitext_samples_per_second": 230.725,
      "step": 395000
    },
    {
      "epoch": 22.95,
      "eval_lambada_loss": 2.41796875,
      "eval_lambada_ppl": 11.223039347524061,
      "eval_lambada_runtime": 9.5767,
      "eval_lambada_samples_per_second": 508.42,
      "step": 395000
    },
    {
      "activations/layer0_attention_weight_max": 15.919363021850586,
      "activations/layer0_attention_weight_min": -14.159321784973145,
      "activations/layer10_attention_weight_max": 48.0745964050293,
      "activations/layer10_attention_weight_min": -46.291282653808594,
      "activations/layer11_attention_weight_max": 49.24880599975586,
      "activations/layer11_attention_weight_min": -45.29376220703125,
      "activations/layer12_attention_weight_max": 21.49269676208496,
      "activations/layer12_attention_weight_min": -26.234941482543945,
      "activations/layer13_attention_weight_max": 52.83596420288086,
      "activations/layer13_attention_weight_min": -34.45648193359375,
      "activations/layer14_attention_weight_max": 55.91188049316406,
      "activations/layer14_attention_weight_min": -33.00517654418945,
      "activations/layer15_attention_weight_max": 53.24684524536133,
      "activations/layer15_attention_weight_min": -34.15193557739258,
      "activations/layer16_attention_weight_max": 39.6529541015625,
      "activations/layer16_attention_weight_min": -36.28712844848633,
      "activations/layer17_attention_weight_max": 54.813541412353516,
      "activations/layer17_attention_weight_min": -50.25508117675781,
      "activations/layer18_attention_weight_max": 48.919986724853516,
      "activations/layer18_attention_weight_min": -45.235843658447266,
      "activations/layer19_attention_weight_max": 24.39727210998535,
      "activations/layer19_attention_weight_min": -23.849712371826172,
      "activations/layer1_attention_weight_max": 15.168209075927734,
      "activations/layer1_attention_weight_min": -14.491748809814453,
      "activations/layer20_attention_weight_max": 21.71875,
      "activations/layer20_attention_weight_min": -26.259143829345703,
      "activations/layer21_attention_weight_max": 40.9211311340332,
      "activations/layer21_attention_weight_min": -31.02042007446289,
      "activations/layer22_attention_weight_max": 29.52522087097168,
      "activations/layer22_attention_weight_min": -24.679014205932617,
      "activations/layer23_attention_weight_max": 32.74046325683594,
      "activations/layer23_attention_weight_min": -25.355670928955078,
      "activations/layer2_attention_weight_max": 35.748313903808594,
      "activations/layer2_attention_weight_min": -33.223175048828125,
      "activations/layer3_attention_weight_max": 105.43923950195312,
      "activations/layer3_attention_weight_min": -102.54656982421875,
      "activations/layer4_attention_weight_max": 101.63356018066406,
      "activations/layer4_attention_weight_min": -95.57328796386719,
      "activations/layer5_attention_weight_max": 71.03211975097656,
      "activations/layer5_attention_weight_min": -73.16093444824219,
      "activations/layer6_attention_weight_max": 56.01189041137695,
      "activations/layer6_attention_weight_min": -56.2778434753418,
      "activations/layer7_attention_weight_max": 89.86141204833984,
      "activations/layer7_attention_weight_min": -84.17371368408203,
      "activations/layer8_attention_weight_max": 61.7564697265625,
      "activations/layer8_attention_weight_min": -61.60170364379883,
      "activations/layer9_attention_weight_max": 63.91522216796875,
      "activations/layer9_attention_weight_min": -59.488189697265625,
      "epoch": 22.95,
      "learning_rate": 1.993560606060606e-06,
      "loss": 2.6862,
      "step": 395050
    },
    {
      "activations/layer0_attention_weight_max": 15.529767990112305,
      "activations/layer0_attention_weight_min": -14.23189926147461,
      "activations/layer10_attention_weight_max": 38.7055549621582,
      "activations/layer10_attention_weight_min": -37.6313362121582,
      "activations/layer11_attention_weight_max": 37.00550842285156,
      "activations/layer11_attention_weight_min": -38.85204315185547,
      "activations/layer12_attention_weight_max": 19.154529571533203,
      "activations/layer12_attention_weight_min": -23.224504470825195,
      "activations/layer13_attention_weight_max": 34.715431213378906,
      "activations/layer13_attention_weight_min": -27.304712295532227,
      "activations/layer14_attention_weight_max": 35.72737121582031,
      "activations/layer14_attention_weight_min": -30.39252281188965,
      "activations/layer15_attention_weight_max": 34.44403839111328,
      "activations/layer15_attention_weight_min": -29.7153377532959,
      "activations/layer16_attention_weight_max": 34.3231086730957,
      "activations/layer16_attention_weight_min": -34.90443801879883,
      "activations/layer17_attention_weight_max": 55.045135498046875,
      "activations/layer17_attention_weight_min": -49.9473991394043,
      "activations/layer18_attention_weight_max": 50.13933181762695,
      "activations/layer18_attention_weight_min": -44.35052490234375,
      "activations/layer19_attention_weight_max": 25.993820190429688,
      "activations/layer19_attention_weight_min": -25.003843307495117,
      "activations/layer1_attention_weight_max": 15.862723350524902,
      "activations/layer1_attention_weight_min": -15.554422378540039,
      "activations/layer20_attention_weight_max": 23.443485260009766,
      "activations/layer20_attention_weight_min": -20.852867126464844,
      "activations/layer21_attention_weight_max": 36.666011810302734,
      "activations/layer21_attention_weight_min": -22.860925674438477,
      "activations/layer22_attention_weight_max": 30.543285369873047,
      "activations/layer22_attention_weight_min": -26.43153953552246,
      "activations/layer23_attention_weight_max": 35.40493392944336,
      "activations/layer23_attention_weight_min": -26.27169418334961,
      "activations/layer2_attention_weight_max": 32.032493591308594,
      "activations/layer2_attention_weight_min": -31.230329513549805,
      "activations/layer3_attention_weight_max": 92.12836456298828,
      "activations/layer3_attention_weight_min": -92.70819091796875,
      "activations/layer4_attention_weight_max": 89.2423324584961,
      "activations/layer4_attention_weight_min": -88.1295394897461,
      "activations/layer5_attention_weight_max": 63.99650573730469,
      "activations/layer5_attention_weight_min": -73.26036071777344,
      "activations/layer6_attention_weight_max": 52.38459396362305,
      "activations/layer6_attention_weight_min": -54.86960983276367,
      "activations/layer7_attention_weight_max": 72.76197052001953,
      "activations/layer7_attention_weight_min": -74.43014526367188,
      "activations/layer8_attention_weight_max": 49.088775634765625,
      "activations/layer8_attention_weight_min": -54.82569122314453,
      "activations/layer9_attention_weight_max": 47.42103576660156,
      "activations/layer9_attention_weight_min": -47.55758285522461,
      "epoch": 22.96,
      "learning_rate": 1.974621212121212e-06,
      "loss": 2.6969,
      "step": 395100
    },
    {
      "activations/layer0_attention_weight_max": 15.39321231842041,
      "activations/layer0_attention_weight_min": -14.19419002532959,
      "activations/layer10_attention_weight_max": 35.1363525390625,
      "activations/layer10_attention_weight_min": -36.01882553100586,
      "activations/layer11_attention_weight_max": 32.8621826171875,
      "activations/layer11_attention_weight_min": -36.66300964355469,
      "activations/layer12_attention_weight_max": 20.60123634338379,
      "activations/layer12_attention_weight_min": -23.676462173461914,
      "activations/layer13_attention_weight_max": 36.76748275756836,
      "activations/layer13_attention_weight_min": -28.542522430419922,
      "activations/layer14_attention_weight_max": 38.75161361694336,
      "activations/layer14_attention_weight_min": -31.62408447265625,
      "activations/layer15_attention_weight_max": 37.34319305419922,
      "activations/layer15_attention_weight_min": -32.10859680175781,
      "activations/layer16_attention_weight_max": 36.472190856933594,
      "activations/layer16_attention_weight_min": -33.148353576660156,
      "activations/layer17_attention_weight_max": 57.88186264038086,
      "activations/layer17_attention_weight_min": -45.826602935791016,
      "activations/layer18_attention_weight_max": 53.38484191894531,
      "activations/layer18_attention_weight_min": -42.21199035644531,
      "activations/layer19_attention_weight_max": 26.304075241088867,
      "activations/layer19_attention_weight_min": -24.9974365234375,
      "activations/layer1_attention_weight_max": 14.940705299377441,
      "activations/layer1_attention_weight_min": -14.180831909179688,
      "activations/layer20_attention_weight_max": 23.875791549682617,
      "activations/layer20_attention_weight_min": -21.42233657836914,
      "activations/layer21_attention_weight_max": 38.375518798828125,
      "activations/layer21_attention_weight_min": -22.739166259765625,
      "activations/layer22_attention_weight_max": 31.956790924072266,
      "activations/layer22_attention_weight_min": -25.579797744750977,
      "activations/layer23_attention_weight_max": 38.40655517578125,
      "activations/layer23_attention_weight_min": -23.20083236694336,
      "activations/layer2_attention_weight_max": 32.683746337890625,
      "activations/layer2_attention_weight_min": -31.06494140625,
      "activations/layer3_attention_weight_max": 91.80596160888672,
      "activations/layer3_attention_weight_min": -91.8281478881836,
      "activations/layer4_attention_weight_max": 86.43233489990234,
      "activations/layer4_attention_weight_min": -85.4139175415039,
      "activations/layer5_attention_weight_max": 64.7275161743164,
      "activations/layer5_attention_weight_min": -73.40328216552734,
      "activations/layer6_attention_weight_max": 48.86760711669922,
      "activations/layer6_attention_weight_min": -51.256202697753906,
      "activations/layer7_attention_weight_max": 68.76966094970703,
      "activations/layer7_attention_weight_min": -72.42447662353516,
      "activations/layer8_attention_weight_max": 47.27483367919922,
      "activations/layer8_attention_weight_min": -50.07384490966797,
      "activations/layer9_attention_weight_max": 46.638553619384766,
      "activations/layer9_attention_weight_min": -48.39204025268555,
      "epoch": 22.96,
      "learning_rate": 1.955681818181818e-06,
      "loss": 2.7009,
      "step": 395150
    },
    {
      "activations/layer0_attention_weight_max": 15.854586601257324,
      "activations/layer0_attention_weight_min": -14.162834167480469,
      "activations/layer10_attention_weight_max": 37.95480728149414,
      "activations/layer10_attention_weight_min": -38.057491302490234,
      "activations/layer11_attention_weight_max": 33.71788024902344,
      "activations/layer11_attention_weight_min": -35.33687210083008,
      "activations/layer12_attention_weight_max": 20.227176666259766,
      "activations/layer12_attention_weight_min": -27.221542358398438,
      "activations/layer13_attention_weight_max": 32.93467330932617,
      "activations/layer13_attention_weight_min": -27.005762100219727,
      "activations/layer14_attention_weight_max": 35.06040573120117,
      "activations/layer14_attention_weight_min": -31.130332946777344,
      "activations/layer15_attention_weight_max": 33.29901885986328,
      "activations/layer15_attention_weight_min": -32.2435417175293,
      "activations/layer16_attention_weight_max": 36.90843200683594,
      "activations/layer16_attention_weight_min": -35.35298538208008,
      "activations/layer17_attention_weight_max": 56.049774169921875,
      "activations/layer17_attention_weight_min": -46.78542709350586,
      "activations/layer18_attention_weight_max": 52.14801788330078,
      "activations/layer18_attention_weight_min": -42.700740814208984,
      "activations/layer19_attention_weight_max": 26.38425064086914,
      "activations/layer19_attention_weight_min": -22.777393341064453,
      "activations/layer1_attention_weight_max": 15.861165046691895,
      "activations/layer1_attention_weight_min": -14.216781616210938,
      "activations/layer20_attention_weight_max": 25.797948837280273,
      "activations/layer20_attention_weight_min": -21.95581817626953,
      "activations/layer21_attention_weight_max": 38.53561782836914,
      "activations/layer21_attention_weight_min": -24.11223030090332,
      "activations/layer22_attention_weight_max": 31.89631462097168,
      "activations/layer22_attention_weight_min": -24.793664932250977,
      "activations/layer23_attention_weight_max": 34.84508514404297,
      "activations/layer23_attention_weight_min": -24.867414474487305,
      "activations/layer2_attention_weight_max": 32.74527359008789,
      "activations/layer2_attention_weight_min": -29.976085662841797,
      "activations/layer3_attention_weight_max": 93.88188171386719,
      "activations/layer3_attention_weight_min": -91.06688690185547,
      "activations/layer4_attention_weight_max": 88.39344787597656,
      "activations/layer4_attention_weight_min": -86.86779022216797,
      "activations/layer5_attention_weight_max": 62.82282257080078,
      "activations/layer5_attention_weight_min": -69.31657409667969,
      "activations/layer6_attention_weight_max": 52.133094787597656,
      "activations/layer6_attention_weight_min": -52.47410202026367,
      "activations/layer7_attention_weight_max": 69.9349136352539,
      "activations/layer7_attention_weight_min": -73.10989379882812,
      "activations/layer8_attention_weight_max": 52.12834167480469,
      "activations/layer8_attention_weight_min": -56.148250579833984,
      "activations/layer9_attention_weight_max": 46.204933166503906,
      "activations/layer9_attention_weight_min": -50.60884475708008,
      "epoch": 22.96,
      "learning_rate": 1.937121212121212e-06,
      "loss": 2.7055,
      "step": 395200
    },
    {
      "activations/layer0_attention_weight_max": 16.684341430664062,
      "activations/layer0_attention_weight_min": -14.088790893554688,
      "activations/layer10_attention_weight_max": 39.00284957885742,
      "activations/layer10_attention_weight_min": -37.02100372314453,
      "activations/layer11_attention_weight_max": 41.8502311706543,
      "activations/layer11_attention_weight_min": -37.89020538330078,
      "activations/layer12_attention_weight_max": 21.091073989868164,
      "activations/layer12_attention_weight_min": -27.173927307128906,
      "activations/layer13_attention_weight_max": 37.69257736206055,
      "activations/layer13_attention_weight_min": -29.08817481994629,
      "activations/layer14_attention_weight_max": 37.78009796142578,
      "activations/layer14_attention_weight_min": -30.84796714782715,
      "activations/layer15_attention_weight_max": 37.14332962036133,
      "activations/layer15_attention_weight_min": -32.01177215576172,
      "activations/layer16_attention_weight_max": 36.66402053833008,
      "activations/layer16_attention_weight_min": -35.87338638305664,
      "activations/layer17_attention_weight_max": 52.738059997558594,
      "activations/layer17_attention_weight_min": -47.73898696899414,
      "activations/layer18_attention_weight_max": 50.63828659057617,
      "activations/layer18_attention_weight_min": -42.88674545288086,
      "activations/layer19_attention_weight_max": 27.69070816040039,
      "activations/layer19_attention_weight_min": -25.147811889648438,
      "activations/layer1_attention_weight_max": 15.260748863220215,
      "activations/layer1_attention_weight_min": -15.178104400634766,
      "activations/layer20_attention_weight_max": 25.171751022338867,
      "activations/layer20_attention_weight_min": -23.677637100219727,
      "activations/layer21_attention_weight_max": 38.229156494140625,
      "activations/layer21_attention_weight_min": -22.505672454833984,
      "activations/layer22_attention_weight_max": 28.72049903869629,
      "activations/layer22_attention_weight_min": -27.969684600830078,
      "activations/layer23_attention_weight_max": 38.408905029296875,
      "activations/layer23_attention_weight_min": -24.398456573486328,
      "activations/layer2_attention_weight_max": 34.9750862121582,
      "activations/layer2_attention_weight_min": -34.538108825683594,
      "activations/layer3_attention_weight_max": 98.11144256591797,
      "activations/layer3_attention_weight_min": -96.60014343261719,
      "activations/layer4_attention_weight_max": 87.53876495361328,
      "activations/layer4_attention_weight_min": -87.04849243164062,
      "activations/layer5_attention_weight_max": 61.0428581237793,
      "activations/layer5_attention_weight_min": -73.80104064941406,
      "activations/layer6_attention_weight_max": 52.01786422729492,
      "activations/layer6_attention_weight_min": -55.22684860229492,
      "activations/layer7_attention_weight_max": 71.39509582519531,
      "activations/layer7_attention_weight_min": -72.73668670654297,
      "activations/layer8_attention_weight_max": 50.46914291381836,
      "activations/layer8_attention_weight_min": -52.51873016357422,
      "activations/layer9_attention_weight_max": 51.4670295715332,
      "activations/layer9_attention_weight_min": -47.232242584228516,
      "epoch": 22.97,
      "learning_rate": 1.9181818181818178e-06,
      "loss": 2.7021,
      "step": 395250
    },
    {
      "activations/layer0_attention_weight_max": 16.057621002197266,
      "activations/layer0_attention_weight_min": -14.418780326843262,
      "activations/layer10_attention_weight_max": 36.82741928100586,
      "activations/layer10_attention_weight_min": -34.73982620239258,
      "activations/layer11_attention_weight_max": 34.95228576660156,
      "activations/layer11_attention_weight_min": -35.16679763793945,
      "activations/layer12_attention_weight_max": 18.571758270263672,
      "activations/layer12_attention_weight_min": -26.505460739135742,
      "activations/layer13_attention_weight_max": 30.53528594970703,
      "activations/layer13_attention_weight_min": -28.902355194091797,
      "activations/layer14_attention_weight_max": 32.17987823486328,
      "activations/layer14_attention_weight_min": -31.338468551635742,
      "activations/layer15_attention_weight_max": 32.216835021972656,
      "activations/layer15_attention_weight_min": -31.223846435546875,
      "activations/layer16_attention_weight_max": 32.87432861328125,
      "activations/layer16_attention_weight_min": -32.67338562011719,
      "activations/layer17_attention_weight_max": 47.788692474365234,
      "activations/layer17_attention_weight_min": -48.32780456542969,
      "activations/layer18_attention_weight_max": 43.29576873779297,
      "activations/layer18_attention_weight_min": -43.374935150146484,
      "activations/layer19_attention_weight_max": 22.89032554626465,
      "activations/layer19_attention_weight_min": -24.0217342376709,
      "activations/layer1_attention_weight_max": 14.248665809631348,
      "activations/layer1_attention_weight_min": -14.515178680419922,
      "activations/layer20_attention_weight_max": 23.035388946533203,
      "activations/layer20_attention_weight_min": -23.705533981323242,
      "activations/layer21_attention_weight_max": 34.43541717529297,
      "activations/layer21_attention_weight_min": -21.245820999145508,
      "activations/layer22_attention_weight_max": 27.643674850463867,
      "activations/layer22_attention_weight_min": -25.81297492980957,
      "activations/layer23_attention_weight_max": 35.31055450439453,
      "activations/layer23_attention_weight_min": -23.587997436523438,
      "activations/layer2_attention_weight_max": 32.04186248779297,
      "activations/layer2_attention_weight_min": -30.362018585205078,
      "activations/layer3_attention_weight_max": 90.68065643310547,
      "activations/layer3_attention_weight_min": -90.00840759277344,
      "activations/layer4_attention_weight_max": 85.0924301147461,
      "activations/layer4_attention_weight_min": -86.54114532470703,
      "activations/layer5_attention_weight_max": 60.27717208862305,
      "activations/layer5_attention_weight_min": -72.62398529052734,
      "activations/layer6_attention_weight_max": 51.68513870239258,
      "activations/layer6_attention_weight_min": -49.632484436035156,
      "activations/layer7_attention_weight_max": 69.9903793334961,
      "activations/layer7_attention_weight_min": -70.22588348388672,
      "activations/layer8_attention_weight_max": 49.96311569213867,
      "activations/layer8_attention_weight_min": -52.34311294555664,
      "activations/layer9_attention_weight_max": 46.63657760620117,
      "activations/layer9_attention_weight_min": -46.96828079223633,
      "epoch": 22.97,
      "learning_rate": 1.8992424242424241e-06,
      "loss": 2.6992,
      "step": 395300
    },
    {
      "activations/layer0_attention_weight_max": 15.801758766174316,
      "activations/layer0_attention_weight_min": -14.17654037475586,
      "activations/layer10_attention_weight_max": 39.012962341308594,
      "activations/layer10_attention_weight_min": -38.74931716918945,
      "activations/layer11_attention_weight_max": 36.34668731689453,
      "activations/layer11_attention_weight_min": -36.96696090698242,
      "activations/layer12_attention_weight_max": 19.40009117126465,
      "activations/layer12_attention_weight_min": -28.511560440063477,
      "activations/layer13_attention_weight_max": 38.72007369995117,
      "activations/layer13_attention_weight_min": -29.605066299438477,
      "activations/layer14_attention_weight_max": 40.0688362121582,
      "activations/layer14_attention_weight_min": -31.373291015625,
      "activations/layer15_attention_weight_max": 40.038883209228516,
      "activations/layer15_attention_weight_min": -31.416444778442383,
      "activations/layer16_attention_weight_max": 36.04713821411133,
      "activations/layer16_attention_weight_min": -33.425804138183594,
      "activations/layer17_attention_weight_max": 53.78696060180664,
      "activations/layer17_attention_weight_min": -46.99278259277344,
      "activations/layer18_attention_weight_max": 52.65208053588867,
      "activations/layer18_attention_weight_min": -42.80590057373047,
      "activations/layer19_attention_weight_max": 24.60605239868164,
      "activations/layer19_attention_weight_min": -22.814973831176758,
      "activations/layer1_attention_weight_max": 15.151185989379883,
      "activations/layer1_attention_weight_min": -14.574475288391113,
      "activations/layer20_attention_weight_max": 23.930601119995117,
      "activations/layer20_attention_weight_min": -21.856021881103516,
      "activations/layer21_attention_weight_max": 40.94121170043945,
      "activations/layer21_attention_weight_min": -24.881315231323242,
      "activations/layer22_attention_weight_max": 33.09680938720703,
      "activations/layer22_attention_weight_min": -26.704025268554688,
      "activations/layer23_attention_weight_max": 37.56089782714844,
      "activations/layer23_attention_weight_min": -26.512075424194336,
      "activations/layer2_attention_weight_max": 32.46791076660156,
      "activations/layer2_attention_weight_min": -30.556236267089844,
      "activations/layer3_attention_weight_max": 96.90930938720703,
      "activations/layer3_attention_weight_min": -92.51395416259766,
      "activations/layer4_attention_weight_max": 86.83340454101562,
      "activations/layer4_attention_weight_min": -88.69469451904297,
      "activations/layer5_attention_weight_max": 65.96636199951172,
      "activations/layer5_attention_weight_min": -70.07064056396484,
      "activations/layer6_attention_weight_max": 52.006412506103516,
      "activations/layer6_attention_weight_min": -49.63105010986328,
      "activations/layer7_attention_weight_max": 70.81183624267578,
      "activations/layer7_attention_weight_min": -69.69993591308594,
      "activations/layer8_attention_weight_max": 48.9553108215332,
      "activations/layer8_attention_weight_min": -50.398563385009766,
      "activations/layer9_attention_weight_max": 50.86349105834961,
      "activations/layer9_attention_weight_min": -46.96123123168945,
      "epoch": 22.97,
      "learning_rate": 1.88030303030303e-06,
      "loss": 2.7059,
      "step": 395350
    },
    {
      "activations/layer0_attention_weight_max": 15.421141624450684,
      "activations/layer0_attention_weight_min": -14.163320541381836,
      "activations/layer10_attention_weight_max": 39.9146842956543,
      "activations/layer10_attention_weight_min": -38.030731201171875,
      "activations/layer11_attention_weight_max": 36.75499725341797,
      "activations/layer11_attention_weight_min": -36.82758331298828,
      "activations/layer12_attention_weight_max": 21.638370513916016,
      "activations/layer12_attention_weight_min": -24.06103515625,
      "activations/layer13_attention_weight_max": 39.539791107177734,
      "activations/layer13_attention_weight_min": -28.584964752197266,
      "activations/layer14_attention_weight_max": 40.07332229614258,
      "activations/layer14_attention_weight_min": -31.761194229125977,
      "activations/layer15_attention_weight_max": 37.62870788574219,
      "activations/layer15_attention_weight_min": -30.911619186401367,
      "activations/layer16_attention_weight_max": 39.82111740112305,
      "activations/layer16_attention_weight_min": -35.20267105102539,
      "activations/layer17_attention_weight_max": 58.43318557739258,
      "activations/layer17_attention_weight_min": -49.774234771728516,
      "activations/layer18_attention_weight_max": 57.26832962036133,
      "activations/layer18_attention_weight_min": -47.76142120361328,
      "activations/layer19_attention_weight_max": 27.177000045776367,
      "activations/layer19_attention_weight_min": -25.337404251098633,
      "activations/layer1_attention_weight_max": 15.188278198242188,
      "activations/layer1_attention_weight_min": -14.591302871704102,
      "activations/layer20_attention_weight_max": 25.587711334228516,
      "activations/layer20_attention_weight_min": -20.11180305480957,
      "activations/layer21_attention_weight_max": 47.60028076171875,
      "activations/layer21_attention_weight_min": -23.277795791625977,
      "activations/layer22_attention_weight_max": 34.59702682495117,
      "activations/layer22_attention_weight_min": -23.98978614807129,
      "activations/layer23_attention_weight_max": 38.19378662109375,
      "activations/layer23_attention_weight_min": -26.659406661987305,
      "activations/layer2_attention_weight_max": 33.22586441040039,
      "activations/layer2_attention_weight_min": -31.91409683227539,
      "activations/layer3_attention_weight_max": 95.65023040771484,
      "activations/layer3_attention_weight_min": -99.77973175048828,
      "activations/layer4_attention_weight_max": 89.6405258178711,
      "activations/layer4_attention_weight_min": -88.33783721923828,
      "activations/layer5_attention_weight_max": 61.41753387451172,
      "activations/layer5_attention_weight_min": -75.19015502929688,
      "activations/layer6_attention_weight_max": 55.44041442871094,
      "activations/layer6_attention_weight_min": -51.380882263183594,
      "activations/layer7_attention_weight_max": 73.26524353027344,
      "activations/layer7_attention_weight_min": -72.89815521240234,
      "activations/layer8_attention_weight_max": 52.33546447753906,
      "activations/layer8_attention_weight_min": -54.802734375,
      "activations/layer9_attention_weight_max": 46.9690055847168,
      "activations/layer9_attention_weight_min": -49.36629867553711,
      "epoch": 22.97,
      "learning_rate": 1.8613636363636362e-06,
      "loss": 2.7106,
      "step": 395400
    },
    {
      "activations/layer0_attention_weight_max": 15.77095890045166,
      "activations/layer0_attention_weight_min": -14.252310752868652,
      "activations/layer10_attention_weight_max": 37.87372589111328,
      "activations/layer10_attention_weight_min": -35.856048583984375,
      "activations/layer11_attention_weight_max": 35.20770263671875,
      "activations/layer11_attention_weight_min": -35.64488983154297,
      "activations/layer12_attention_weight_max": 19.3442325592041,
      "activations/layer12_attention_weight_min": -28.74387550354004,
      "activations/layer13_attention_weight_max": 34.193572998046875,
      "activations/layer13_attention_weight_min": -27.090517044067383,
      "activations/layer14_attention_weight_max": 35.5896110534668,
      "activations/layer14_attention_weight_min": -29.959787368774414,
      "activations/layer15_attention_weight_max": 32.23196029663086,
      "activations/layer15_attention_weight_min": -31.847488403320312,
      "activations/layer16_attention_weight_max": 34.40268325805664,
      "activations/layer16_attention_weight_min": -32.90427017211914,
      "activations/layer17_attention_weight_max": 51.273555755615234,
      "activations/layer17_attention_weight_min": -45.531166076660156,
      "activations/layer18_attention_weight_max": 46.68906784057617,
      "activations/layer18_attention_weight_min": -42.2952995300293,
      "activations/layer19_attention_weight_max": 25.052827835083008,
      "activations/layer19_attention_weight_min": -24.578001022338867,
      "activations/layer1_attention_weight_max": 15.024231910705566,
      "activations/layer1_attention_weight_min": -15.906299591064453,
      "activations/layer20_attention_weight_max": 23.26512908935547,
      "activations/layer20_attention_weight_min": -23.32128143310547,
      "activations/layer21_attention_weight_max": 35.47246170043945,
      "activations/layer21_attention_weight_min": -22.82552146911621,
      "activations/layer22_attention_weight_max": 26.95965003967285,
      "activations/layer22_attention_weight_min": -27.152177810668945,
      "activations/layer23_attention_weight_max": 36.23408889770508,
      "activations/layer23_attention_weight_min": -25.039289474487305,
      "activations/layer2_attention_weight_max": 34.78498077392578,
      "activations/layer2_attention_weight_min": -31.3962459564209,
      "activations/layer3_attention_weight_max": 92.40687561035156,
      "activations/layer3_attention_weight_min": -91.5908203125,
      "activations/layer4_attention_weight_max": 88.17574310302734,
      "activations/layer4_attention_weight_min": -85.50572967529297,
      "activations/layer5_attention_weight_max": 61.70873260498047,
      "activations/layer5_attention_weight_min": -72.63318634033203,
      "activations/layer6_attention_weight_max": 50.60091781616211,
      "activations/layer6_attention_weight_min": -50.36326217651367,
      "activations/layer7_attention_weight_max": 72.53067016601562,
      "activations/layer7_attention_weight_min": -70.7471694946289,
      "activations/layer8_attention_weight_max": 55.36903762817383,
      "activations/layer8_attention_weight_min": -56.47265625,
      "activations/layer9_attention_weight_max": 48.29271697998047,
      "activations/layer9_attention_weight_min": -48.753108978271484,
      "epoch": 22.98,
      "learning_rate": 1.8424242424242422e-06,
      "loss": 2.6957,
      "step": 395450
    },
    {
      "activations/layer0_attention_weight_max": 15.06542682647705,
      "activations/layer0_attention_weight_min": -14.43643569946289,
      "activations/layer10_attention_weight_max": 37.478370666503906,
      "activations/layer10_attention_weight_min": -37.093055725097656,
      "activations/layer11_attention_weight_max": 36.71956253051758,
      "activations/layer11_attention_weight_min": -36.85136413574219,
      "activations/layer12_attention_weight_max": 20.65595245361328,
      "activations/layer12_attention_weight_min": -24.671436309814453,
      "activations/layer13_attention_weight_max": 34.50239181518555,
      "activations/layer13_attention_weight_min": -29.402706146240234,
      "activations/layer14_attention_weight_max": 37.294891357421875,
      "activations/layer14_attention_weight_min": -33.62556076049805,
      "activations/layer15_attention_weight_max": 34.510948181152344,
      "activations/layer15_attention_weight_min": -32.485801696777344,
      "activations/layer16_attention_weight_max": 37.35110855102539,
      "activations/layer16_attention_weight_min": -35.52873992919922,
      "activations/layer17_attention_weight_max": 53.295284271240234,
      "activations/layer17_attention_weight_min": -48.31903076171875,
      "activations/layer18_attention_weight_max": 52.75023651123047,
      "activations/layer18_attention_weight_min": -43.85422134399414,
      "activations/layer19_attention_weight_max": 26.573883056640625,
      "activations/layer19_attention_weight_min": -23.408018112182617,
      "activations/layer1_attention_weight_max": 14.494665145874023,
      "activations/layer1_attention_weight_min": -14.138029098510742,
      "activations/layer20_attention_weight_max": 24.26451873779297,
      "activations/layer20_attention_weight_min": -20.562864303588867,
      "activations/layer21_attention_weight_max": 36.2681999206543,
      "activations/layer21_attention_weight_min": -22.954626083374023,
      "activations/layer22_attention_weight_max": 29.916156768798828,
      "activations/layer22_attention_weight_min": -25.23257827758789,
      "activations/layer23_attention_weight_max": 35.42565155029297,
      "activations/layer23_attention_weight_min": -23.812301635742188,
      "activations/layer2_attention_weight_max": 31.399120330810547,
      "activations/layer2_attention_weight_min": -30.397472381591797,
      "activations/layer3_attention_weight_max": 90.69347381591797,
      "activations/layer3_attention_weight_min": -89.84344482421875,
      "activations/layer4_attention_weight_max": 86.6385269165039,
      "activations/layer4_attention_weight_min": -85.88970184326172,
      "activations/layer5_attention_weight_max": 63.36617660522461,
      "activations/layer5_attention_weight_min": -73.77604675292969,
      "activations/layer6_attention_weight_max": 51.62796401977539,
      "activations/layer6_attention_weight_min": -52.41133117675781,
      "activations/layer7_attention_weight_max": 77.06205749511719,
      "activations/layer7_attention_weight_min": -73.83353424072266,
      "activations/layer8_attention_weight_max": 50.6590461730957,
      "activations/layer8_attention_weight_min": -51.68195343017578,
      "activations/layer9_attention_weight_max": 49.1882209777832,
      "activations/layer9_attention_weight_min": -46.98628234863281,
      "epoch": 22.98,
      "learning_rate": 1.8234848484848483e-06,
      "loss": 2.6984,
      "step": 395500
    },
    {
      "activations/layer0_attention_weight_max": 15.796341896057129,
      "activations/layer0_attention_weight_min": -14.033585548400879,
      "activations/layer10_attention_weight_max": 36.514801025390625,
      "activations/layer10_attention_weight_min": -36.476070404052734,
      "activations/layer11_attention_weight_max": 32.637577056884766,
      "activations/layer11_attention_weight_min": -34.51142120361328,
      "activations/layer12_attention_weight_max": 19.06266212463379,
      "activations/layer12_attention_weight_min": -28.34685516357422,
      "activations/layer13_attention_weight_max": 32.05369186401367,
      "activations/layer13_attention_weight_min": -27.418611526489258,
      "activations/layer14_attention_weight_max": 38.0208625793457,
      "activations/layer14_attention_weight_min": -28.114408493041992,
      "activations/layer15_attention_weight_max": 33.310874938964844,
      "activations/layer15_attention_weight_min": -29.84078598022461,
      "activations/layer16_attention_weight_max": 34.09502029418945,
      "activations/layer16_attention_weight_min": -32.32698059082031,
      "activations/layer17_attention_weight_max": 51.89960861206055,
      "activations/layer17_attention_weight_min": -46.18185806274414,
      "activations/layer18_attention_weight_max": 48.11519241333008,
      "activations/layer18_attention_weight_min": -42.89326095581055,
      "activations/layer19_attention_weight_max": 25.101341247558594,
      "activations/layer19_attention_weight_min": -22.1772518157959,
      "activations/layer1_attention_weight_max": 15.545697212219238,
      "activations/layer1_attention_weight_min": -16.77251625061035,
      "activations/layer20_attention_weight_max": 21.900102615356445,
      "activations/layer20_attention_weight_min": -21.233684539794922,
      "activations/layer21_attention_weight_max": 35.85920715332031,
      "activations/layer21_attention_weight_min": -20.856281280517578,
      "activations/layer22_attention_weight_max": 26.862796783447266,
      "activations/layer22_attention_weight_min": -24.003265380859375,
      "activations/layer23_attention_weight_max": 32.65586853027344,
      "activations/layer23_attention_weight_min": -23.171476364135742,
      "activations/layer2_attention_weight_max": 32.97502136230469,
      "activations/layer2_attention_weight_min": -31.157636642456055,
      "activations/layer3_attention_weight_max": 89.44357299804688,
      "activations/layer3_attention_weight_min": -91.36494445800781,
      "activations/layer4_attention_weight_max": 83.59334564208984,
      "activations/layer4_attention_weight_min": -82.36082458496094,
      "activations/layer5_attention_weight_max": 63.13796615600586,
      "activations/layer5_attention_weight_min": -76.04891967773438,
      "activations/layer6_attention_weight_max": 49.8546028137207,
      "activations/layer6_attention_weight_min": -49.8720817565918,
      "activations/layer7_attention_weight_max": 70.53766632080078,
      "activations/layer7_attention_weight_min": -67.85026550292969,
      "activations/layer8_attention_weight_max": 48.1911506652832,
      "activations/layer8_attention_weight_min": -49.12923049926758,
      "activations/layer9_attention_weight_max": 43.35081100463867,
      "activations/layer9_attention_weight_min": -45.580657958984375,
      "epoch": 22.98,
      "learning_rate": 1.8045454545454542e-06,
      "loss": 2.7135,
      "step": 395550
    },
    {
      "activations/layer0_attention_weight_max": 15.949666023254395,
      "activations/layer0_attention_weight_min": -14.254693031311035,
      "activations/layer10_attention_weight_max": 34.91529846191406,
      "activations/layer10_attention_weight_min": -35.588478088378906,
      "activations/layer11_attention_weight_max": 32.31535339355469,
      "activations/layer11_attention_weight_min": -36.15251541137695,
      "activations/layer12_attention_weight_max": 19.339069366455078,
      "activations/layer12_attention_weight_min": -24.489763259887695,
      "activations/layer13_attention_weight_max": 30.59341049194336,
      "activations/layer13_attention_weight_min": -26.63665008544922,
      "activations/layer14_attention_weight_max": 35.376800537109375,
      "activations/layer14_attention_weight_min": -31.346527099609375,
      "activations/layer15_attention_weight_max": 33.24221420288086,
      "activations/layer15_attention_weight_min": -31.30303192138672,
      "activations/layer16_attention_weight_max": 34.24666976928711,
      "activations/layer16_attention_weight_min": -32.29374313354492,
      "activations/layer17_attention_weight_max": 51.83237838745117,
      "activations/layer17_attention_weight_min": -45.26005935668945,
      "activations/layer18_attention_weight_max": 47.94717788696289,
      "activations/layer18_attention_weight_min": -41.03364181518555,
      "activations/layer19_attention_weight_max": 25.737836837768555,
      "activations/layer19_attention_weight_min": -24.28458595275879,
      "activations/layer1_attention_weight_max": 15.038348197937012,
      "activations/layer1_attention_weight_min": -15.707314491271973,
      "activations/layer20_attention_weight_max": 23.38092803955078,
      "activations/layer20_attention_weight_min": -21.50766372680664,
      "activations/layer21_attention_weight_max": 36.9556999206543,
      "activations/layer21_attention_weight_min": -22.734731674194336,
      "activations/layer22_attention_weight_max": 32.09876251220703,
      "activations/layer22_attention_weight_min": -27.298263549804688,
      "activations/layer23_attention_weight_max": 36.158729553222656,
      "activations/layer23_attention_weight_min": -26.24610137939453,
      "activations/layer2_attention_weight_max": 30.822345733642578,
      "activations/layer2_attention_weight_min": -30.410566329956055,
      "activations/layer3_attention_weight_max": 90.7977523803711,
      "activations/layer3_attention_weight_min": -92.83058166503906,
      "activations/layer4_attention_weight_max": 86.25859832763672,
      "activations/layer4_attention_weight_min": -85.2971420288086,
      "activations/layer5_attention_weight_max": 64.29966735839844,
      "activations/layer5_attention_weight_min": -71.29421997070312,
      "activations/layer6_attention_weight_max": 49.74513244628906,
      "activations/layer6_attention_weight_min": -50.433448791503906,
      "activations/layer7_attention_weight_max": 70.30758666992188,
      "activations/layer7_attention_weight_min": -68.74866485595703,
      "activations/layer8_attention_weight_max": 46.3948860168457,
      "activations/layer8_attention_weight_min": -51.25433349609375,
      "activations/layer9_attention_weight_max": 44.46915817260742,
      "activations/layer9_attention_weight_min": -46.446598052978516,
      "epoch": 22.99,
      "learning_rate": 1.7856060606060606e-06,
      "loss": 2.6882,
      "step": 395600
    },
    {
      "activations/layer0_attention_weight_max": 15.753630638122559,
      "activations/layer0_attention_weight_min": -14.704867362976074,
      "activations/layer10_attention_weight_max": 37.87004470825195,
      "activations/layer10_attention_weight_min": -37.86762237548828,
      "activations/layer11_attention_weight_max": 35.733951568603516,
      "activations/layer11_attention_weight_min": -35.199188232421875,
      "activations/layer12_attention_weight_max": 19.074392318725586,
      "activations/layer12_attention_weight_min": -23.02836036682129,
      "activations/layer13_attention_weight_max": 34.295066833496094,
      "activations/layer13_attention_weight_min": -26.031635284423828,
      "activations/layer14_attention_weight_max": 35.24067687988281,
      "activations/layer14_attention_weight_min": -30.251325607299805,
      "activations/layer15_attention_weight_max": 31.895681381225586,
      "activations/layer15_attention_weight_min": -29.95290756225586,
      "activations/layer16_attention_weight_max": 35.119441986083984,
      "activations/layer16_attention_weight_min": -32.450660705566406,
      "activations/layer17_attention_weight_max": 51.32617950439453,
      "activations/layer17_attention_weight_min": -43.84895706176758,
      "activations/layer18_attention_weight_max": 45.00975799560547,
      "activations/layer18_attention_weight_min": -38.21208953857422,
      "activations/layer19_attention_weight_max": 24.259571075439453,
      "activations/layer19_attention_weight_min": -21.568357467651367,
      "activations/layer1_attention_weight_max": 14.969378471374512,
      "activations/layer1_attention_weight_min": -15.027234077453613,
      "activations/layer20_attention_weight_max": 20.757083892822266,
      "activations/layer20_attention_weight_min": -20.321290969848633,
      "activations/layer21_attention_weight_max": 30.690324783325195,
      "activations/layer21_attention_weight_min": -21.118501663208008,
      "activations/layer22_attention_weight_max": 29.89112663269043,
      "activations/layer22_attention_weight_min": -24.087434768676758,
      "activations/layer23_attention_weight_max": 35.20741271972656,
      "activations/layer23_attention_weight_min": -23.91390037536621,
      "activations/layer2_attention_weight_max": 31.195701599121094,
      "activations/layer2_attention_weight_min": -30.593338012695312,
      "activations/layer3_attention_weight_max": 90.80683898925781,
      "activations/layer3_attention_weight_min": -94.87122344970703,
      "activations/layer4_attention_weight_max": 87.51726531982422,
      "activations/layer4_attention_weight_min": -87.84488677978516,
      "activations/layer5_attention_weight_max": 61.09996795654297,
      "activations/layer5_attention_weight_min": -71.42989349365234,
      "activations/layer6_attention_weight_max": 49.18614959716797,
      "activations/layer6_attention_weight_min": -52.1130485534668,
      "activations/layer7_attention_weight_max": 68.65022277832031,
      "activations/layer7_attention_weight_min": -69.8092269897461,
      "activations/layer8_attention_weight_max": 47.26696014404297,
      "activations/layer8_attention_weight_min": -51.297828674316406,
      "activations/layer9_attention_weight_max": 43.93704605102539,
      "activations/layer9_attention_weight_min": -44.9256591796875,
      "epoch": 22.99,
      "learning_rate": 1.7666666666666666e-06,
      "loss": 2.7044,
      "step": 395650
    },
    {
      "activations/layer0_attention_weight_max": 15.765501976013184,
      "activations/layer0_attention_weight_min": -14.347087860107422,
      "activations/layer10_attention_weight_max": 37.6357307434082,
      "activations/layer10_attention_weight_min": -37.926185607910156,
      "activations/layer11_attention_weight_max": 35.621498107910156,
      "activations/layer11_attention_weight_min": -37.345558166503906,
      "activations/layer12_attention_weight_max": 19.709863662719727,
      "activations/layer12_attention_weight_min": -25.46058464050293,
      "activations/layer13_attention_weight_max": 34.85157775878906,
      "activations/layer13_attention_weight_min": -27.08753776550293,
      "activations/layer14_attention_weight_max": 36.84011459350586,
      "activations/layer14_attention_weight_min": -30.10812759399414,
      "activations/layer15_attention_weight_max": 33.888999938964844,
      "activations/layer15_attention_weight_min": -30.578176498413086,
      "activations/layer16_attention_weight_max": 35.94977569580078,
      "activations/layer16_attention_weight_min": -32.60139465332031,
      "activations/layer17_attention_weight_max": 53.382076263427734,
      "activations/layer17_attention_weight_min": -46.93998336791992,
      "activations/layer18_attention_weight_max": 49.17988967895508,
      "activations/layer18_attention_weight_min": -40.18124008178711,
      "activations/layer19_attention_weight_max": 28.12700843811035,
      "activations/layer19_attention_weight_min": -23.67630958557129,
      "activations/layer1_attention_weight_max": 14.8405179977417,
      "activations/layer1_attention_weight_min": -15.848719596862793,
      "activations/layer20_attention_weight_max": 22.103797912597656,
      "activations/layer20_attention_weight_min": -23.564321517944336,
      "activations/layer21_attention_weight_max": 33.952362060546875,
      "activations/layer21_attention_weight_min": -23.22430992126465,
      "activations/layer22_attention_weight_max": 29.031864166259766,
      "activations/layer22_attention_weight_min": -25.94242286682129,
      "activations/layer23_attention_weight_max": 39.109596252441406,
      "activations/layer23_attention_weight_min": -27.388790130615234,
      "activations/layer2_attention_weight_max": 32.36024475097656,
      "activations/layer2_attention_weight_min": -31.663421630859375,
      "activations/layer3_attention_weight_max": 92.35106658935547,
      "activations/layer3_attention_weight_min": -94.44771575927734,
      "activations/layer4_attention_weight_max": 87.85832977294922,
      "activations/layer4_attention_weight_min": -88.80240631103516,
      "activations/layer5_attention_weight_max": 63.356788635253906,
      "activations/layer5_attention_weight_min": -76.57711791992188,
      "activations/layer6_attention_weight_max": 50.784400939941406,
      "activations/layer6_attention_weight_min": -52.28900909423828,
      "activations/layer7_attention_weight_max": 70.99607849121094,
      "activations/layer7_attention_weight_min": -72.30863952636719,
      "activations/layer8_attention_weight_max": 50.50435256958008,
      "activations/layer8_attention_weight_min": -50.96015548706055,
      "activations/layer9_attention_weight_max": 43.984012603759766,
      "activations/layer9_attention_weight_min": -50.413490295410156,
      "epoch": 22.99,
      "learning_rate": 1.7477272727272727e-06,
      "loss": 2.6887,
      "step": 395700
    },
    {
      "activations/layer0_attention_weight_max": 16.173994064331055,
      "activations/layer0_attention_weight_min": -14.00051212310791,
      "activations/layer10_attention_weight_max": 36.78267288208008,
      "activations/layer10_attention_weight_min": -35.88080978393555,
      "activations/layer11_attention_weight_max": 34.387779235839844,
      "activations/layer11_attention_weight_min": -34.339698791503906,
      "activations/layer12_attention_weight_max": 19.523401260375977,
      "activations/layer12_attention_weight_min": -23.414159774780273,
      "activations/layer13_attention_weight_max": 34.95570373535156,
      "activations/layer13_attention_weight_min": -31.151180267333984,
      "activations/layer14_attention_weight_max": 39.18729019165039,
      "activations/layer14_attention_weight_min": -34.39320755004883,
      "activations/layer15_attention_weight_max": 34.948219299316406,
      "activations/layer15_attention_weight_min": -35.363441467285156,
      "activations/layer16_attention_weight_max": 39.68295669555664,
      "activations/layer16_attention_weight_min": -40.12077713012695,
      "activations/layer17_attention_weight_max": 55.950103759765625,
      "activations/layer17_attention_weight_min": -57.6299934387207,
      "activations/layer18_attention_weight_max": 51.787044525146484,
      "activations/layer18_attention_weight_min": -51.54814910888672,
      "activations/layer19_attention_weight_max": 27.264184951782227,
      "activations/layer19_attention_weight_min": -29.22638511657715,
      "activations/layer1_attention_weight_max": 15.302978515625,
      "activations/layer1_attention_weight_min": -14.728877067565918,
      "activations/layer20_attention_weight_max": 25.90548324584961,
      "activations/layer20_attention_weight_min": -23.948862075805664,
      "activations/layer21_attention_weight_max": 41.525333404541016,
      "activations/layer21_attention_weight_min": -31.45204734802246,
      "activations/layer22_attention_weight_max": 29.167640686035156,
      "activations/layer22_attention_weight_min": -29.930240631103516,
      "activations/layer23_attention_weight_max": 38.19847106933594,
      "activations/layer23_attention_weight_min": -31.287548065185547,
      "activations/layer2_attention_weight_max": 33.12918472290039,
      "activations/layer2_attention_weight_min": -31.62960433959961,
      "activations/layer3_attention_weight_max": 92.6530532836914,
      "activations/layer3_attention_weight_min": -94.49764251708984,
      "activations/layer4_attention_weight_max": 86.69041442871094,
      "activations/layer4_attention_weight_min": -87.4590072631836,
      "activations/layer5_attention_weight_max": 63.41053771972656,
      "activations/layer5_attention_weight_min": -73.54649353027344,
      "activations/layer6_attention_weight_max": 50.829002380371094,
      "activations/layer6_attention_weight_min": -50.76865005493164,
      "activations/layer7_attention_weight_max": 68.84733581542969,
      "activations/layer7_attention_weight_min": -68.48008728027344,
      "activations/layer8_attention_weight_max": 47.884220123291016,
      "activations/layer8_attention_weight_min": -54.00923538208008,
      "activations/layer9_attention_weight_max": 43.894596099853516,
      "activations/layer9_attention_weight_min": -46.85850143432617,
      "epoch": 23.0,
      "learning_rate": 1.7287878787878786e-06,
      "loss": 2.7173,
      "step": 395750
    },
    {
      "activations/layer0_attention_weight_max": 15.97203254699707,
      "activations/layer0_attention_weight_min": -14.327040672302246,
      "activations/layer10_attention_weight_max": 38.356224060058594,
      "activations/layer10_attention_weight_min": -38.5306510925293,
      "activations/layer11_attention_weight_max": 35.70741653442383,
      "activations/layer11_attention_weight_min": -39.30689239501953,
      "activations/layer12_attention_weight_max": 19.603370666503906,
      "activations/layer12_attention_weight_min": -25.23904800415039,
      "activations/layer13_attention_weight_max": 36.012046813964844,
      "activations/layer13_attention_weight_min": -30.13743782043457,
      "activations/layer14_attention_weight_max": 35.679901123046875,
      "activations/layer14_attention_weight_min": -31.11788558959961,
      "activations/layer15_attention_weight_max": 36.10934829711914,
      "activations/layer15_attention_weight_min": -32.28868865966797,
      "activations/layer16_attention_weight_max": 35.681400299072266,
      "activations/layer16_attention_weight_min": -34.253536224365234,
      "activations/layer17_attention_weight_max": 52.415836334228516,
      "activations/layer17_attention_weight_min": -47.26285171508789,
      "activations/layer18_attention_weight_max": 48.37527084350586,
      "activations/layer18_attention_weight_min": -41.153133392333984,
      "activations/layer19_attention_weight_max": 27.952436447143555,
      "activations/layer19_attention_weight_min": -24.00422477722168,
      "activations/layer1_attention_weight_max": 15.484994888305664,
      "activations/layer1_attention_weight_min": -15.914375305175781,
      "activations/layer20_attention_weight_max": 25.929170608520508,
      "activations/layer20_attention_weight_min": -24.574111938476562,
      "activations/layer21_attention_weight_max": 37.696956634521484,
      "activations/layer21_attention_weight_min": -21.91437339782715,
      "activations/layer22_attention_weight_max": 32.11034393310547,
      "activations/layer22_attention_weight_min": -25.70964241027832,
      "activations/layer23_attention_weight_max": 40.10791778564453,
      "activations/layer23_attention_weight_min": -24.851749420166016,
      "activations/layer2_attention_weight_max": 31.981098175048828,
      "activations/layer2_attention_weight_min": -31.659452438354492,
      "activations/layer3_attention_weight_max": 89.81243896484375,
      "activations/layer3_attention_weight_min": -91.82420349121094,
      "activations/layer4_attention_weight_max": 86.97064208984375,
      "activations/layer4_attention_weight_min": -87.59502410888672,
      "activations/layer5_attention_weight_max": 64.240234375,
      "activations/layer5_attention_weight_min": -70.76126861572266,
      "activations/layer6_attention_weight_max": 50.0097541809082,
      "activations/layer6_attention_weight_min": -51.52256393432617,
      "activations/layer7_attention_weight_max": 72.1917953491211,
      "activations/layer7_attention_weight_min": -67.83108520507812,
      "activations/layer8_attention_weight_max": 47.79197311401367,
      "activations/layer8_attention_weight_min": -51.2061882019043,
      "activations/layer9_attention_weight_max": 46.846771240234375,
      "activations/layer9_attention_weight_min": -47.2721061706543,
      "epoch": 23.0,
      "learning_rate": 1.7098484848484846e-06,
      "loss": 2.6967,
      "step": 395800
    },
    {
      "activations/layer0_attention_weight_max": 15.130696296691895,
      "activations/layer0_attention_weight_min": -14.036608695983887,
      "activations/layer10_attention_weight_max": 40.68810272216797,
      "activations/layer10_attention_weight_min": -39.84606170654297,
      "activations/layer11_attention_weight_max": 44.74062728881836,
      "activations/layer11_attention_weight_min": -41.23550796508789,
      "activations/layer12_attention_weight_max": 20.006338119506836,
      "activations/layer12_attention_weight_min": -25.22774887084961,
      "activations/layer13_attention_weight_max": 35.6417236328125,
      "activations/layer13_attention_weight_min": -27.730932235717773,
      "activations/layer14_attention_weight_max": 36.5681266784668,
      "activations/layer14_attention_weight_min": -30.42075538635254,
      "activations/layer15_attention_weight_max": 34.056034088134766,
      "activations/layer15_attention_weight_min": -30.656932830810547,
      "activations/layer16_attention_weight_max": 36.062191009521484,
      "activations/layer16_attention_weight_min": -33.81599807739258,
      "activations/layer17_attention_weight_max": 56.71564483642578,
      "activations/layer17_attention_weight_min": -48.38454818725586,
      "activations/layer18_attention_weight_max": 54.14372253417969,
      "activations/layer18_attention_weight_min": -44.34453582763672,
      "activations/layer19_attention_weight_max": 24.255523681640625,
      "activations/layer19_attention_weight_min": -23.225568771362305,
      "activations/layer1_attention_weight_max": 15.977506637573242,
      "activations/layer1_attention_weight_min": -15.381157875061035,
      "activations/layer20_attention_weight_max": 24.394657135009766,
      "activations/layer20_attention_weight_min": -20.335704803466797,
      "activations/layer21_attention_weight_max": 39.99794387817383,
      "activations/layer21_attention_weight_min": -20.198383331298828,
      "activations/layer22_attention_weight_max": 30.404830932617188,
      "activations/layer22_attention_weight_min": -25.25242805480957,
      "activations/layer23_attention_weight_max": 36.88994216918945,
      "activations/layer23_attention_weight_min": -23.456737518310547,
      "activations/layer2_attention_weight_max": 34.46125411987305,
      "activations/layer2_attention_weight_min": -32.07233428955078,
      "activations/layer3_attention_weight_max": 92.4912109375,
      "activations/layer3_attention_weight_min": -96.74609375,
      "activations/layer4_attention_weight_max": 87.78243255615234,
      "activations/layer4_attention_weight_min": -88.0002670288086,
      "activations/layer5_attention_weight_max": 63.03132247924805,
      "activations/layer5_attention_weight_min": -72.02008056640625,
      "activations/layer6_attention_weight_max": 51.95261764526367,
      "activations/layer6_attention_weight_min": -52.79515838623047,
      "activations/layer7_attention_weight_max": 79.48651885986328,
      "activations/layer7_attention_weight_min": -74.99000549316406,
      "activations/layer8_attention_weight_max": 55.42475128173828,
      "activations/layer8_attention_weight_min": -55.9639778137207,
      "activations/layer9_attention_weight_max": 60.797359466552734,
      "activations/layer9_attention_weight_min": -51.28007888793945,
      "epoch": 23.0,
      "learning_rate": 1.6909090909090907e-06,
      "loss": 2.7647,
      "step": 395850
    },
    {
      "activations/layer0_attention_weight_max": 15.786966323852539,
      "activations/layer0_attention_weight_min": -14.062033653259277,
      "activations/layer10_attention_weight_max": 39.96086502075195,
      "activations/layer10_attention_weight_min": -39.49809646606445,
      "activations/layer11_attention_weight_max": 37.042640686035156,
      "activations/layer11_attention_weight_min": -39.50602340698242,
      "activations/layer12_attention_weight_max": 19.486358642578125,
      "activations/layer12_attention_weight_min": -24.600929260253906,
      "activations/layer13_attention_weight_max": 34.0545768737793,
      "activations/layer13_attention_weight_min": -29.099124908447266,
      "activations/layer14_attention_weight_max": 35.411964416503906,
      "activations/layer14_attention_weight_min": -33.32219314575195,
      "activations/layer15_attention_weight_max": 34.43230056762695,
      "activations/layer15_attention_weight_min": -33.55884552001953,
      "activations/layer16_attention_weight_max": 37.753211975097656,
      "activations/layer16_attention_weight_min": -36.00625228881836,
      "activations/layer17_attention_weight_max": 57.84239959716797,
      "activations/layer17_attention_weight_min": -52.810829162597656,
      "activations/layer18_attention_weight_max": 53.18792724609375,
      "activations/layer18_attention_weight_min": -48.054656982421875,
      "activations/layer19_attention_weight_max": 27.01924705505371,
      "activations/layer19_attention_weight_min": -26.48050880432129,
      "activations/layer1_attention_weight_max": 14.960836410522461,
      "activations/layer1_attention_weight_min": -14.957627296447754,
      "activations/layer20_attention_weight_max": 27.056411743164062,
      "activations/layer20_attention_weight_min": -24.569896697998047,
      "activations/layer21_attention_weight_max": 43.316627502441406,
      "activations/layer21_attention_weight_min": -25.810302734375,
      "activations/layer22_attention_weight_max": 34.373470306396484,
      "activations/layer22_attention_weight_min": -27.76848030090332,
      "activations/layer23_attention_weight_max": 41.10090637207031,
      "activations/layer23_attention_weight_min": -25.526723861694336,
      "activations/layer2_attention_weight_max": 34.03834915161133,
      "activations/layer2_attention_weight_min": -32.998077392578125,
      "activations/layer3_attention_weight_max": 95.23284912109375,
      "activations/layer3_attention_weight_min": -95.59400177001953,
      "activations/layer4_attention_weight_max": 86.90611267089844,
      "activations/layer4_attention_weight_min": -86.91498565673828,
      "activations/layer5_attention_weight_max": 62.284542083740234,
      "activations/layer5_attention_weight_min": -72.84304809570312,
      "activations/layer6_attention_weight_max": 50.66885757446289,
      "activations/layer6_attention_weight_min": -54.0943489074707,
      "activations/layer7_attention_weight_max": 72.54447937011719,
      "activations/layer7_attention_weight_min": -71.0814208984375,
      "activations/layer8_attention_weight_max": 50.084510803222656,
      "activations/layer8_attention_weight_min": -51.7034912109375,
      "activations/layer9_attention_weight_max": 52.73541259765625,
      "activations/layer9_attention_weight_min": -47.93260192871094,
      "epoch": 23.0,
      "learning_rate": 1.6719696969696967e-06,
      "loss": 2.6821,
      "step": 395900
    },
    {
      "activations/layer0_attention_weight_max": 15.47899341583252,
      "activations/layer0_attention_weight_min": -14.307628631591797,
      "activations/layer10_attention_weight_max": 36.68440628051758,
      "activations/layer10_attention_weight_min": -38.11396789550781,
      "activations/layer11_attention_weight_max": 33.675331115722656,
      "activations/layer11_attention_weight_min": -36.41771697998047,
      "activations/layer12_attention_weight_max": 20.742542266845703,
      "activations/layer12_attention_weight_min": -25.782655715942383,
      "activations/layer13_attention_weight_max": 39.49113082885742,
      "activations/layer13_attention_weight_min": -32.716529846191406,
      "activations/layer14_attention_weight_max": 38.1424674987793,
      "activations/layer14_attention_weight_min": -33.60548782348633,
      "activations/layer15_attention_weight_max": 38.55771255493164,
      "activations/layer15_attention_weight_min": -33.76511764526367,
      "activations/layer16_attention_weight_max": 37.52080535888672,
      "activations/layer16_attention_weight_min": -36.617610931396484,
      "activations/layer17_attention_weight_max": 56.562355041503906,
      "activations/layer17_attention_weight_min": -51.568058013916016,
      "activations/layer18_attention_weight_max": 53.795284271240234,
      "activations/layer18_attention_weight_min": -47.117408752441406,
      "activations/layer19_attention_weight_max": 25.423946380615234,
      "activations/layer19_attention_weight_min": -25.454082489013672,
      "activations/layer1_attention_weight_max": 15.247246742248535,
      "activations/layer1_attention_weight_min": -15.010876655578613,
      "activations/layer20_attention_weight_max": 27.604291915893555,
      "activations/layer20_attention_weight_min": -23.815441131591797,
      "activations/layer21_attention_weight_max": 40.94769287109375,
      "activations/layer21_attention_weight_min": -23.227495193481445,
      "activations/layer22_attention_weight_max": 31.60721778869629,
      "activations/layer22_attention_weight_min": -26.16337776184082,
      "activations/layer23_attention_weight_max": 38.39445877075195,
      "activations/layer23_attention_weight_min": -25.341779708862305,
      "activations/layer2_attention_weight_max": 32.07524108886719,
      "activations/layer2_attention_weight_min": -31.24201011657715,
      "activations/layer3_attention_weight_max": 94.87788391113281,
      "activations/layer3_attention_weight_min": -93.85224914550781,
      "activations/layer4_attention_weight_max": 91.8541488647461,
      "activations/layer4_attention_weight_min": -86.86660766601562,
      "activations/layer5_attention_weight_max": 63.45759201049805,
      "activations/layer5_attention_weight_min": -75.04153442382812,
      "activations/layer6_attention_weight_max": 53.95832061767578,
      "activations/layer6_attention_weight_min": -52.70762634277344,
      "activations/layer7_attention_weight_max": 72.04423522949219,
      "activations/layer7_attention_weight_min": -70.33231353759766,
      "activations/layer8_attention_weight_max": 47.94150161743164,
      "activations/layer8_attention_weight_min": -52.62529373168945,
      "activations/layer9_attention_weight_max": 44.995086669921875,
      "activations/layer9_attention_weight_min": -47.83245086669922,
      "epoch": 23.01,
      "learning_rate": 1.6530303030303028e-06,
      "loss": 2.7026,
      "step": 395950
    },
    {
      "activations/layer0_attention_weight_max": 16.350069046020508,
      "activations/layer0_attention_weight_min": -14.201168060302734,
      "activations/layer10_attention_weight_max": 35.806983947753906,
      "activations/layer10_attention_weight_min": -35.72983169555664,
      "activations/layer11_attention_weight_max": 34.55790328979492,
      "activations/layer11_attention_weight_min": -35.86726379394531,
      "activations/layer12_attention_weight_max": 20.11627197265625,
      "activations/layer12_attention_weight_min": -30.629796981811523,
      "activations/layer13_attention_weight_max": 38.75859451293945,
      "activations/layer13_attention_weight_min": -27.769508361816406,
      "activations/layer14_attention_weight_max": 34.98330307006836,
      "activations/layer14_attention_weight_min": -29.249420166015625,
      "activations/layer15_attention_weight_max": 35.519630432128906,
      "activations/layer15_attention_weight_min": -30.527061462402344,
      "activations/layer16_attention_weight_max": 35.96769714355469,
      "activations/layer16_attention_weight_min": -33.6695671081543,
      "activations/layer17_attention_weight_max": 53.02348327636719,
      "activations/layer17_attention_weight_min": -46.61238479614258,
      "activations/layer18_attention_weight_max": 50.10457992553711,
      "activations/layer18_attention_weight_min": -44.54230499267578,
      "activations/layer19_attention_weight_max": 28.45237159729004,
      "activations/layer19_attention_weight_min": -24.36588478088379,
      "activations/layer1_attention_weight_max": 14.9924898147583,
      "activations/layer1_attention_weight_min": -13.960744857788086,
      "activations/layer20_attention_weight_max": 24.741092681884766,
      "activations/layer20_attention_weight_min": -26.57414436340332,
      "activations/layer21_attention_weight_max": 41.91449737548828,
      "activations/layer21_attention_weight_min": -26.752897262573242,
      "activations/layer22_attention_weight_max": 30.958391189575195,
      "activations/layer22_attention_weight_min": -27.532136917114258,
      "activations/layer23_attention_weight_max": 40.08274459838867,
      "activations/layer23_attention_weight_min": -28.217838287353516,
      "activations/layer2_attention_weight_max": 32.11674880981445,
      "activations/layer2_attention_weight_min": -31.458675384521484,
      "activations/layer3_attention_weight_max": 92.20787811279297,
      "activations/layer3_attention_weight_min": -93.68855285644531,
      "activations/layer4_attention_weight_max": 87.33353424072266,
      "activations/layer4_attention_weight_min": -89.10580444335938,
      "activations/layer5_attention_weight_max": 64.4317855834961,
      "activations/layer5_attention_weight_min": -75.71336364746094,
      "activations/layer6_attention_weight_max": 53.27284622192383,
      "activations/layer6_attention_weight_min": -53.019676208496094,
      "activations/layer7_attention_weight_max": 71.53678131103516,
      "activations/layer7_attention_weight_min": -71.34379577636719,
      "activations/layer8_attention_weight_max": 50.526710510253906,
      "activations/layer8_attention_weight_min": -51.1725959777832,
      "activations/layer9_attention_weight_max": 45.41946029663086,
      "activations/layer9_attention_weight_min": -48.94374465942383,
      "epoch": 23.01,
      "learning_rate": 1.6340909090909088e-06,
      "loss": 2.6977,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.5161,
      "eval_samples_per_second": 504.224,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.5161,
      "eval_openwebtext_samples_per_second": 504.224,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 1.9757,
      "eval_wikitext_samples_per_second": 230.802,
      "step": 396000
    },
    {
      "epoch": 23.01,
      "eval_lambada_loss": 2.40234375,
      "eval_lambada_ppl": 11.049042250008942,
      "eval_lambada_runtime": 9.6065,
      "eval_lambada_samples_per_second": 506.843,
      "step": 396000
    },
    {
      "activations/layer0_attention_weight_max": 16.032564163208008,
      "activations/layer0_attention_weight_min": -14.523606300354004,
      "activations/layer10_attention_weight_max": 37.732181549072266,
      "activations/layer10_attention_weight_min": -36.26372528076172,
      "activations/layer11_attention_weight_max": 34.775611877441406,
      "activations/layer11_attention_weight_min": -36.08748245239258,
      "activations/layer12_attention_weight_max": 19.59339714050293,
      "activations/layer12_attention_weight_min": -26.460147857666016,
      "activations/layer13_attention_weight_max": 33.9542236328125,
      "activations/layer13_attention_weight_min": -27.417163848876953,
      "activations/layer14_attention_weight_max": 35.97061538696289,
      "activations/layer14_attention_weight_min": -31.2449893951416,
      "activations/layer15_attention_weight_max": 33.70200729370117,
      "activations/layer15_attention_weight_min": -30.799617767333984,
      "activations/layer16_attention_weight_max": 36.61233901977539,
      "activations/layer16_attention_weight_min": -34.88877487182617,
      "activations/layer17_attention_weight_max": 51.66857147216797,
      "activations/layer17_attention_weight_min": -45.052669525146484,
      "activations/layer18_attention_weight_max": 46.81724548339844,
      "activations/layer18_attention_weight_min": -39.50342559814453,
      "activations/layer19_attention_weight_max": 23.16786766052246,
      "activations/layer19_attention_weight_min": -23.220760345458984,
      "activations/layer1_attention_weight_max": 14.694511413574219,
      "activations/layer1_attention_weight_min": -14.465124130249023,
      "activations/layer20_attention_weight_max": 22.790149688720703,
      "activations/layer20_attention_weight_min": -23.35262107849121,
      "activations/layer21_attention_weight_max": 32.35562515258789,
      "activations/layer21_attention_weight_min": -20.40416717529297,
      "activations/layer22_attention_weight_max": 30.416210174560547,
      "activations/layer22_attention_weight_min": -25.792213439941406,
      "activations/layer23_attention_weight_max": 35.28541946411133,
      "activations/layer23_attention_weight_min": -25.341796875,
      "activations/layer2_attention_weight_max": 32.52006530761719,
      "activations/layer2_attention_weight_min": -30.224838256835938,
      "activations/layer3_attention_weight_max": 93.39461517333984,
      "activations/layer3_attention_weight_min": -89.83086395263672,
      "activations/layer4_attention_weight_max": 89.04779815673828,
      "activations/layer4_attention_weight_min": -85.48921203613281,
      "activations/layer5_attention_weight_max": 61.06293487548828,
      "activations/layer5_attention_weight_min": -73.49429321289062,
      "activations/layer6_attention_weight_max": 50.62366485595703,
      "activations/layer6_attention_weight_min": -55.442203521728516,
      "activations/layer7_attention_weight_max": 68.63236999511719,
      "activations/layer7_attention_weight_min": -71.72803497314453,
      "activations/layer8_attention_weight_max": 49.64737319946289,
      "activations/layer8_attention_weight_min": -49.091773986816406,
      "activations/layer9_attention_weight_max": 44.2850456237793,
      "activations/layer9_attention_weight_min": -45.50754928588867,
      "epoch": 23.01,
      "learning_rate": 1.6151515151515151e-06,
      "loss": 2.7091,
      "step": 396050
    },
    {
      "activations/layer0_attention_weight_max": 15.682804107666016,
      "activations/layer0_attention_weight_min": -14.129359245300293,
      "activations/layer10_attention_weight_max": 38.78215026855469,
      "activations/layer10_attention_weight_min": -38.54025650024414,
      "activations/layer11_attention_weight_max": 36.05644989013672,
      "activations/layer11_attention_weight_min": -38.82615661621094,
      "activations/layer12_attention_weight_max": 19.41420555114746,
      "activations/layer12_attention_weight_min": -25.831459045410156,
      "activations/layer13_attention_weight_max": 33.805240631103516,
      "activations/layer13_attention_weight_min": -29.537446975708008,
      "activations/layer14_attention_weight_max": 35.76633834838867,
      "activations/layer14_attention_weight_min": -32.75786209106445,
      "activations/layer15_attention_weight_max": 33.387550354003906,
      "activations/layer15_attention_weight_min": -32.99128341674805,
      "activations/layer16_attention_weight_max": 35.8628044128418,
      "activations/layer16_attention_weight_min": -34.71974563598633,
      "activations/layer17_attention_weight_max": 53.6612663269043,
      "activations/layer17_attention_weight_min": -48.32053756713867,
      "activations/layer18_attention_weight_max": 52.17027282714844,
      "activations/layer18_attention_weight_min": -43.757354736328125,
      "activations/layer19_attention_weight_max": 30.951160430908203,
      "activations/layer19_attention_weight_min": -24.51038932800293,
      "activations/layer1_attention_weight_max": 15.290120124816895,
      "activations/layer1_attention_weight_min": -14.662559509277344,
      "activations/layer20_attention_weight_max": 26.42608642578125,
      "activations/layer20_attention_weight_min": -23.654563903808594,
      "activations/layer21_attention_weight_max": 38.2623176574707,
      "activations/layer21_attention_weight_min": -25.221078872680664,
      "activations/layer22_attention_weight_max": 30.99109649658203,
      "activations/layer22_attention_weight_min": -25.528738021850586,
      "activations/layer23_attention_weight_max": 39.20001220703125,
      "activations/layer23_attention_weight_min": -22.61329460144043,
      "activations/layer2_attention_weight_max": 32.90365219116211,
      "activations/layer2_attention_weight_min": -31.577945709228516,
      "activations/layer3_attention_weight_max": 92.54756164550781,
      "activations/layer3_attention_weight_min": -94.80776977539062,
      "activations/layer4_attention_weight_max": 90.57039642333984,
      "activations/layer4_attention_weight_min": -89.11590576171875,
      "activations/layer5_attention_weight_max": 64.50465393066406,
      "activations/layer5_attention_weight_min": -72.15830993652344,
      "activations/layer6_attention_weight_max": 51.1121940612793,
      "activations/layer6_attention_weight_min": -52.36796951293945,
      "activations/layer7_attention_weight_max": 70.25247192382812,
      "activations/layer7_attention_weight_min": -73.52261352539062,
      "activations/layer8_attention_weight_max": 49.73720932006836,
      "activations/layer8_attention_weight_min": -53.459651947021484,
      "activations/layer9_attention_weight_max": 47.97786331176758,
      "activations/layer9_attention_weight_min": -49.16080093383789,
      "epoch": 23.02,
      "learning_rate": 1.596212121212121e-06,
      "loss": 2.6945,
      "step": 396100
    },
    {
      "activations/layer0_attention_weight_max": 16.767885208129883,
      "activations/layer0_attention_weight_min": -14.259797096252441,
      "activations/layer10_attention_weight_max": 36.6649169921875,
      "activations/layer10_attention_weight_min": -35.91170120239258,
      "activations/layer11_attention_weight_max": 35.0210075378418,
      "activations/layer11_attention_weight_min": -35.23403549194336,
      "activations/layer12_attention_weight_max": 19.46800994873047,
      "activations/layer12_attention_weight_min": -28.235689163208008,
      "activations/layer13_attention_weight_max": 35.031646728515625,
      "activations/layer13_attention_weight_min": -27.070308685302734,
      "activations/layer14_attention_weight_max": 33.51579284667969,
      "activations/layer14_attention_weight_min": -31.63998794555664,
      "activations/layer15_attention_weight_max": 31.687700271606445,
      "activations/layer15_attention_weight_min": -30.693405151367188,
      "activations/layer16_attention_weight_max": 33.82285690307617,
      "activations/layer16_attention_weight_min": -33.53349685668945,
      "activations/layer17_attention_weight_max": 52.053279876708984,
      "activations/layer17_attention_weight_min": -45.64773941040039,
      "activations/layer18_attention_weight_max": 46.256858825683594,
      "activations/layer18_attention_weight_min": -40.888614654541016,
      "activations/layer19_attention_weight_max": 23.736570358276367,
      "activations/layer19_attention_weight_min": -22.374324798583984,
      "activations/layer1_attention_weight_max": 15.120070457458496,
      "activations/layer1_attention_weight_min": -14.882959365844727,
      "activations/layer20_attention_weight_max": 22.580413818359375,
      "activations/layer20_attention_weight_min": -19.255409240722656,
      "activations/layer21_attention_weight_max": 39.95598602294922,
      "activations/layer21_attention_weight_min": -21.726547241210938,
      "activations/layer22_attention_weight_max": 28.65962028503418,
      "activations/layer22_attention_weight_min": -25.122285842895508,
      "activations/layer23_attention_weight_max": 35.457889556884766,
      "activations/layer23_attention_weight_min": -24.955425262451172,
      "activations/layer2_attention_weight_max": 33.220367431640625,
      "activations/layer2_attention_weight_min": -30.519254684448242,
      "activations/layer3_attention_weight_max": 94.11945343017578,
      "activations/layer3_attention_weight_min": -95.50086212158203,
      "activations/layer4_attention_weight_max": 89.67156219482422,
      "activations/layer4_attention_weight_min": -89.50341033935547,
      "activations/layer5_attention_weight_max": 63.37766647338867,
      "activations/layer5_attention_weight_min": -73.97206115722656,
      "activations/layer6_attention_weight_max": 53.246341705322266,
      "activations/layer6_attention_weight_min": -53.4874153137207,
      "activations/layer7_attention_weight_max": 73.76631164550781,
      "activations/layer7_attention_weight_min": -70.59782409667969,
      "activations/layer8_attention_weight_max": 52.82860565185547,
      "activations/layer8_attention_weight_min": -52.58876419067383,
      "activations/layer9_attention_weight_max": 46.9232292175293,
      "activations/layer9_attention_weight_min": -47.823055267333984,
      "epoch": 23.02,
      "learning_rate": 1.5772727272727272e-06,
      "loss": 2.7016,
      "step": 396150
    },
    {
      "activations/layer0_attention_weight_max": 16.03301239013672,
      "activations/layer0_attention_weight_min": -14.142735481262207,
      "activations/layer10_attention_weight_max": 37.94347381591797,
      "activations/layer10_attention_weight_min": -37.17906188964844,
      "activations/layer11_attention_weight_max": 34.31153869628906,
      "activations/layer11_attention_weight_min": -36.35149383544922,
      "activations/layer12_attention_weight_max": 19.53986930847168,
      "activations/layer12_attention_weight_min": -27.80019760131836,
      "activations/layer13_attention_weight_max": 35.85648727416992,
      "activations/layer13_attention_weight_min": -28.54922103881836,
      "activations/layer14_attention_weight_max": 33.592498779296875,
      "activations/layer14_attention_weight_min": -30.259841918945312,
      "activations/layer15_attention_weight_max": 30.751874923706055,
      "activations/layer15_attention_weight_min": -29.270206451416016,
      "activations/layer16_attention_weight_max": 34.34058380126953,
      "activations/layer16_attention_weight_min": -33.768157958984375,
      "activations/layer17_attention_weight_max": 52.67110061645508,
      "activations/layer17_attention_weight_min": -46.807533264160156,
      "activations/layer18_attention_weight_max": 53.64583206176758,
      "activations/layer18_attention_weight_min": -44.86788558959961,
      "activations/layer19_attention_weight_max": 25.126426696777344,
      "activations/layer19_attention_weight_min": -24.819091796875,
      "activations/layer1_attention_weight_max": 15.160209655761719,
      "activations/layer1_attention_weight_min": -14.586344718933105,
      "activations/layer20_attention_weight_max": 25.57500648498535,
      "activations/layer20_attention_weight_min": -25.01430892944336,
      "activations/layer21_attention_weight_max": 39.06232452392578,
      "activations/layer21_attention_weight_min": -24.678964614868164,
      "activations/layer22_attention_weight_max": 30.23194122314453,
      "activations/layer22_attention_weight_min": -27.761337280273438,
      "activations/layer23_attention_weight_max": 38.77157211303711,
      "activations/layer23_attention_weight_min": -24.281482696533203,
      "activations/layer2_attention_weight_max": 32.50661849975586,
      "activations/layer2_attention_weight_min": -32.342254638671875,
      "activations/layer3_attention_weight_max": 94.892333984375,
      "activations/layer3_attention_weight_min": -94.51190948486328,
      "activations/layer4_attention_weight_max": 89.63165283203125,
      "activations/layer4_attention_weight_min": -91.3987045288086,
      "activations/layer5_attention_weight_max": 63.812618255615234,
      "activations/layer5_attention_weight_min": -78.125732421875,
      "activations/layer6_attention_weight_max": 51.37247848510742,
      "activations/layer6_attention_weight_min": -53.95657730102539,
      "activations/layer7_attention_weight_max": 75.95149230957031,
      "activations/layer7_attention_weight_min": -75.06649017333984,
      "activations/layer8_attention_weight_max": 50.0416145324707,
      "activations/layer8_attention_weight_min": -54.20741271972656,
      "activations/layer9_attention_weight_max": 52.74720001220703,
      "activations/layer9_attention_weight_min": -48.11980056762695,
      "epoch": 23.02,
      "learning_rate": 1.5583333333333332e-06,
      "loss": 2.7121,
      "step": 396200
    },
    {
      "activations/layer0_attention_weight_max": 15.669941902160645,
      "activations/layer0_attention_weight_min": -14.07387924194336,
      "activations/layer10_attention_weight_max": 37.43941116333008,
      "activations/layer10_attention_weight_min": -37.700050354003906,
      "activations/layer11_attention_weight_max": 33.63807678222656,
      "activations/layer11_attention_weight_min": -35.540565490722656,
      "activations/layer12_attention_weight_max": 22.33928108215332,
      "activations/layer12_attention_weight_min": -26.90497398376465,
      "activations/layer13_attention_weight_max": 39.54856491088867,
      "activations/layer13_attention_weight_min": -27.974937438964844,
      "activations/layer14_attention_weight_max": 36.98472213745117,
      "activations/layer14_attention_weight_min": -30.593265533447266,
      "activations/layer15_attention_weight_max": 34.77947235107422,
      "activations/layer15_attention_weight_min": -32.04487991333008,
      "activations/layer16_attention_weight_max": 35.61784744262695,
      "activations/layer16_attention_weight_min": -34.74805450439453,
      "activations/layer17_attention_weight_max": 54.107627868652344,
      "activations/layer17_attention_weight_min": -47.776580810546875,
      "activations/layer18_attention_weight_max": 48.11972427368164,
      "activations/layer18_attention_weight_min": -41.21634292602539,
      "activations/layer19_attention_weight_max": 25.37255859375,
      "activations/layer19_attention_weight_min": -23.281095504760742,
      "activations/layer1_attention_weight_max": 15.732222557067871,
      "activations/layer1_attention_weight_min": -14.35587215423584,
      "activations/layer20_attention_weight_max": 21.73163414001465,
      "activations/layer20_attention_weight_min": -23.190893173217773,
      "activations/layer21_attention_weight_max": 36.62000274658203,
      "activations/layer21_attention_weight_min": -20.728843688964844,
      "activations/layer22_attention_weight_max": 28.74335289001465,
      "activations/layer22_attention_weight_min": -25.65920066833496,
      "activations/layer23_attention_weight_max": 35.746971130371094,
      "activations/layer23_attention_weight_min": -22.86107635498047,
      "activations/layer2_attention_weight_max": 32.56678771972656,
      "activations/layer2_attention_weight_min": -31.109420776367188,
      "activations/layer3_attention_weight_max": 94.29550170898438,
      "activations/layer3_attention_weight_min": -92.61287689208984,
      "activations/layer4_attention_weight_max": 90.08232879638672,
      "activations/layer4_attention_weight_min": -90.42818450927734,
      "activations/layer5_attention_weight_max": 61.64170455932617,
      "activations/layer5_attention_weight_min": -74.30455780029297,
      "activations/layer6_attention_weight_max": 51.478519439697266,
      "activations/layer6_attention_weight_min": -52.47665786743164,
      "activations/layer7_attention_weight_max": 72.5368881225586,
      "activations/layer7_attention_weight_min": -73.83643341064453,
      "activations/layer8_attention_weight_max": 50.77626419067383,
      "activations/layer8_attention_weight_min": -50.29172134399414,
      "activations/layer9_attention_weight_max": 45.63238525390625,
      "activations/layer9_attention_weight_min": -49.3101921081543,
      "epoch": 23.02,
      "learning_rate": 1.5393939393939393e-06,
      "loss": 2.6916,
      "step": 396250
    },
    {
      "activations/layer0_attention_weight_max": 15.653742790222168,
      "activations/layer0_attention_weight_min": -14.320112228393555,
      "activations/layer10_attention_weight_max": 37.05377960205078,
      "activations/layer10_attention_weight_min": -36.82222366333008,
      "activations/layer11_attention_weight_max": 34.665767669677734,
      "activations/layer11_attention_weight_min": -38.051212310791016,
      "activations/layer12_attention_weight_max": 18.38323974609375,
      "activations/layer12_attention_weight_min": -24.049570083618164,
      "activations/layer13_attention_weight_max": 33.93026351928711,
      "activations/layer13_attention_weight_min": -28.486085891723633,
      "activations/layer14_attention_weight_max": 33.8613166809082,
      "activations/layer14_attention_weight_min": -31.829139709472656,
      "activations/layer15_attention_weight_max": 34.23157501220703,
      "activations/layer15_attention_weight_min": -32.840396881103516,
      "activations/layer16_attention_weight_max": 35.444618225097656,
      "activations/layer16_attention_weight_min": -36.1855354309082,
      "activations/layer17_attention_weight_max": 50.06793212890625,
      "activations/layer17_attention_weight_min": -48.716827392578125,
      "activations/layer18_attention_weight_max": 46.79899978637695,
      "activations/layer18_attention_weight_min": -44.98183822631836,
      "activations/layer19_attention_weight_max": 24.803632736206055,
      "activations/layer19_attention_weight_min": -23.987661361694336,
      "activations/layer1_attention_weight_max": 15.406543731689453,
      "activations/layer1_attention_weight_min": -13.850289344787598,
      "activations/layer20_attention_weight_max": 23.470714569091797,
      "activations/layer20_attention_weight_min": -21.877635955810547,
      "activations/layer21_attention_weight_max": 35.9644775390625,
      "activations/layer21_attention_weight_min": -22.7834415435791,
      "activations/layer22_attention_weight_max": 30.045394897460938,
      "activations/layer22_attention_weight_min": -27.321693420410156,
      "activations/layer23_attention_weight_max": 37.074554443359375,
      "activations/layer23_attention_weight_min": -25.61126708984375,
      "activations/layer2_attention_weight_max": 32.07181930541992,
      "activations/layer2_attention_weight_min": -31.471052169799805,
      "activations/layer3_attention_weight_max": 92.92903137207031,
      "activations/layer3_attention_weight_min": -95.08804321289062,
      "activations/layer4_attention_weight_max": 89.83690643310547,
      "activations/layer4_attention_weight_min": -86.78548431396484,
      "activations/layer5_attention_weight_max": 66.89066314697266,
      "activations/layer5_attention_weight_min": -71.9274673461914,
      "activations/layer6_attention_weight_max": 50.99773025512695,
      "activations/layer6_attention_weight_min": -56.0654296875,
      "activations/layer7_attention_weight_max": 70.8323745727539,
      "activations/layer7_attention_weight_min": -73.92424774169922,
      "activations/layer8_attention_weight_max": 51.10350799560547,
      "activations/layer8_attention_weight_min": -55.337345123291016,
      "activations/layer9_attention_weight_max": 44.68442916870117,
      "activations/layer9_attention_weight_min": -48.092308044433594,
      "epoch": 23.03,
      "learning_rate": 1.520833333333333e-06,
      "loss": 2.6982,
      "step": 396300
    },
    {
      "activations/layer0_attention_weight_max": 15.807496070861816,
      "activations/layer0_attention_weight_min": -14.29742431640625,
      "activations/layer10_attention_weight_max": 38.948089599609375,
      "activations/layer10_attention_weight_min": -36.47285842895508,
      "activations/layer11_attention_weight_max": 35.27479553222656,
      "activations/layer11_attention_weight_min": -36.92191696166992,
      "activations/layer12_attention_weight_max": 19.78872299194336,
      "activations/layer12_attention_weight_min": -25.854536056518555,
      "activations/layer13_attention_weight_max": 32.6170768737793,
      "activations/layer13_attention_weight_min": -26.67190933227539,
      "activations/layer14_attention_weight_max": 37.807437896728516,
      "activations/layer14_attention_weight_min": -31.248838424682617,
      "activations/layer15_attention_weight_max": 32.67428970336914,
      "activations/layer15_attention_weight_min": -33.27682113647461,
      "activations/layer16_attention_weight_max": 36.76118087768555,
      "activations/layer16_attention_weight_min": -35.709590911865234,
      "activations/layer17_attention_weight_max": 52.20205307006836,
      "activations/layer17_attention_weight_min": -49.82598114013672,
      "activations/layer18_attention_weight_max": 51.001461029052734,
      "activations/layer18_attention_weight_min": -43.56745529174805,
      "activations/layer19_attention_weight_max": 27.02058982849121,
      "activations/layer19_attention_weight_min": -24.10370445251465,
      "activations/layer1_attention_weight_max": 15.528473854064941,
      "activations/layer1_attention_weight_min": -14.967021942138672,
      "activations/layer20_attention_weight_max": 24.903648376464844,
      "activations/layer20_attention_weight_min": -24.942127227783203,
      "activations/layer21_attention_weight_max": 37.009849548339844,
      "activations/layer21_attention_weight_min": -22.095991134643555,
      "activations/layer22_attention_weight_max": 29.39829444885254,
      "activations/layer22_attention_weight_min": -25.79469108581543,
      "activations/layer23_attention_weight_max": 36.86722183227539,
      "activations/layer23_attention_weight_min": -24.33911895751953,
      "activations/layer2_attention_weight_max": 33.122833251953125,
      "activations/layer2_attention_weight_min": -31.768321990966797,
      "activations/layer3_attention_weight_max": 95.43682098388672,
      "activations/layer3_attention_weight_min": -92.28312683105469,
      "activations/layer4_attention_weight_max": 88.92736053466797,
      "activations/layer4_attention_weight_min": -87.25933074951172,
      "activations/layer5_attention_weight_max": 63.163414001464844,
      "activations/layer5_attention_weight_min": -71.42677307128906,
      "activations/layer6_attention_weight_max": 50.17066955566406,
      "activations/layer6_attention_weight_min": -51.523807525634766,
      "activations/layer7_attention_weight_max": 73.95938873291016,
      "activations/layer7_attention_weight_min": -71.63459777832031,
      "activations/layer8_attention_weight_max": 50.14967346191406,
      "activations/layer8_attention_weight_min": -51.814781188964844,
      "activations/layer9_attention_weight_max": 46.545249938964844,
      "activations/layer9_attention_weight_min": -47.996360778808594,
      "epoch": 23.03,
      "learning_rate": 1.5018939393939393e-06,
      "loss": 2.6979,
      "step": 396350
    },
    {
      "activations/layer0_attention_weight_max": 15.460821151733398,
      "activations/layer0_attention_weight_min": -14.198932647705078,
      "activations/layer10_attention_weight_max": 34.54846954345703,
      "activations/layer10_attention_weight_min": -35.76625061035156,
      "activations/layer11_attention_weight_max": 33.614383697509766,
      "activations/layer11_attention_weight_min": -36.52405548095703,
      "activations/layer12_attention_weight_max": 19.245634078979492,
      "activations/layer12_attention_weight_min": -26.49032211303711,
      "activations/layer13_attention_weight_max": 35.5388069152832,
      "activations/layer13_attention_weight_min": -29.89827537536621,
      "activations/layer14_attention_weight_max": 33.986541748046875,
      "activations/layer14_attention_weight_min": -32.34901428222656,
      "activations/layer15_attention_weight_max": 32.362937927246094,
      "activations/layer15_attention_weight_min": -32.583335876464844,
      "activations/layer16_attention_weight_max": 33.50895690917969,
      "activations/layer16_attention_weight_min": -32.75580978393555,
      "activations/layer17_attention_weight_max": 50.161251068115234,
      "activations/layer17_attention_weight_min": -45.27128219604492,
      "activations/layer18_attention_weight_max": 46.29820251464844,
      "activations/layer18_attention_weight_min": -41.723758697509766,
      "activations/layer19_attention_weight_max": 25.882766723632812,
      "activations/layer19_attention_weight_min": -23.339717864990234,
      "activations/layer1_attention_weight_max": 15.429793357849121,
      "activations/layer1_attention_weight_min": -14.99390983581543,
      "activations/layer20_attention_weight_max": 24.61486053466797,
      "activations/layer20_attention_weight_min": -20.931013107299805,
      "activations/layer21_attention_weight_max": 40.17752456665039,
      "activations/layer21_attention_weight_min": -24.116941452026367,
      "activations/layer22_attention_weight_max": 30.819374084472656,
      "activations/layer22_attention_weight_min": -24.46936798095703,
      "activations/layer23_attention_weight_max": 33.55247116088867,
      "activations/layer23_attention_weight_min": -23.74169158935547,
      "activations/layer2_attention_weight_max": 32.75285339355469,
      "activations/layer2_attention_weight_min": -30.797224044799805,
      "activations/layer3_attention_weight_max": 91.63811492919922,
      "activations/layer3_attention_weight_min": -92.47710418701172,
      "activations/layer4_attention_weight_max": 87.3194808959961,
      "activations/layer4_attention_weight_min": -86.00621795654297,
      "activations/layer5_attention_weight_max": 60.040584564208984,
      "activations/layer5_attention_weight_min": -74.09602355957031,
      "activations/layer6_attention_weight_max": 49.99860382080078,
      "activations/layer6_attention_weight_min": -50.276832580566406,
      "activations/layer7_attention_weight_max": 70.93966674804688,
      "activations/layer7_attention_weight_min": -72.0218276977539,
      "activations/layer8_attention_weight_max": 48.04063034057617,
      "activations/layer8_attention_weight_min": -50.455345153808594,
      "activations/layer9_attention_weight_max": 43.3951530456543,
      "activations/layer9_attention_weight_min": -46.84452438354492,
      "epoch": 23.03,
      "learning_rate": 1.4829545454545452e-06,
      "loss": 2.7091,
      "step": 396400
    },
    {
      "activations/layer0_attention_weight_max": 16.006263732910156,
      "activations/layer0_attention_weight_min": -14.160844802856445,
      "activations/layer10_attention_weight_max": 41.68008804321289,
      "activations/layer10_attention_weight_min": -40.7392692565918,
      "activations/layer11_attention_weight_max": 39.56856918334961,
      "activations/layer11_attention_weight_min": -38.509246826171875,
      "activations/layer12_attention_weight_max": 20.4310245513916,
      "activations/layer12_attention_weight_min": -23.306888580322266,
      "activations/layer13_attention_weight_max": 40.618282318115234,
      "activations/layer13_attention_weight_min": -29.61149024963379,
      "activations/layer14_attention_weight_max": 39.02842712402344,
      "activations/layer14_attention_weight_min": -31.920215606689453,
      "activations/layer15_attention_weight_max": 36.211952209472656,
      "activations/layer15_attention_weight_min": -33.33860397338867,
      "activations/layer16_attention_weight_max": 37.711910247802734,
      "activations/layer16_attention_weight_min": -35.13222122192383,
      "activations/layer17_attention_weight_max": 57.47672653198242,
      "activations/layer17_attention_weight_min": -49.93135452270508,
      "activations/layer18_attention_weight_max": 52.151710510253906,
      "activations/layer18_attention_weight_min": -45.001869201660156,
      "activations/layer19_attention_weight_max": 27.719003677368164,
      "activations/layer19_attention_weight_min": -24.952783584594727,
      "activations/layer1_attention_weight_max": 14.734889030456543,
      "activations/layer1_attention_weight_min": -14.367831230163574,
      "activations/layer20_attention_weight_max": 27.497440338134766,
      "activations/layer20_attention_weight_min": -24.058406829833984,
      "activations/layer21_attention_weight_max": 44.76369094848633,
      "activations/layer21_attention_weight_min": -23.86661148071289,
      "activations/layer22_attention_weight_max": 29.359554290771484,
      "activations/layer22_attention_weight_min": -26.701642990112305,
      "activations/layer23_attention_weight_max": 36.03053665161133,
      "activations/layer23_attention_weight_min": -24.942350387573242,
      "activations/layer2_attention_weight_max": 32.823848724365234,
      "activations/layer2_attention_weight_min": -32.322486877441406,
      "activations/layer3_attention_weight_max": 94.6136245727539,
      "activations/layer3_attention_weight_min": -94.04345703125,
      "activations/layer4_attention_weight_max": 92.08109283447266,
      "activations/layer4_attention_weight_min": -87.8360824584961,
      "activations/layer5_attention_weight_max": 64.6797866821289,
      "activations/layer5_attention_weight_min": -74.36968994140625,
      "activations/layer6_attention_weight_max": 50.385581970214844,
      "activations/layer6_attention_weight_min": -52.19753646850586,
      "activations/layer7_attention_weight_max": 74.03994750976562,
      "activations/layer7_attention_weight_min": -71.02545928955078,
      "activations/layer8_attention_weight_max": 53.1898307800293,
      "activations/layer8_attention_weight_min": -54.228607177734375,
      "activations/layer9_attention_weight_max": 50.644344329833984,
      "activations/layer9_attention_weight_min": -48.67353439331055,
      "epoch": 23.04,
      "learning_rate": 1.4640151515151513e-06,
      "loss": 2.6865,
      "step": 396450
    },
    {
      "activations/layer0_attention_weight_max": 15.841228485107422,
      "activations/layer0_attention_weight_min": -14.461392402648926,
      "activations/layer10_attention_weight_max": 38.01272964477539,
      "activations/layer10_attention_weight_min": -36.55097961425781,
      "activations/layer11_attention_weight_max": 36.62079620361328,
      "activations/layer11_attention_weight_min": -36.20917892456055,
      "activations/layer12_attention_weight_max": 18.9769344329834,
      "activations/layer12_attention_weight_min": -28.183855056762695,
      "activations/layer13_attention_weight_max": 35.08333969116211,
      "activations/layer13_attention_weight_min": -28.729114532470703,
      "activations/layer14_attention_weight_max": 37.12366485595703,
      "activations/layer14_attention_weight_min": -34.68864059448242,
      "activations/layer15_attention_weight_max": 32.784793853759766,
      "activations/layer15_attention_weight_min": -33.216522216796875,
      "activations/layer16_attention_weight_max": 33.417327880859375,
      "activations/layer16_attention_weight_min": -33.79094696044922,
      "activations/layer17_attention_weight_max": 49.25783157348633,
      "activations/layer17_attention_weight_min": -46.76985549926758,
      "activations/layer18_attention_weight_max": 44.759952545166016,
      "activations/layer18_attention_weight_min": -40.46992492675781,
      "activations/layer19_attention_weight_max": 22.127492904663086,
      "activations/layer19_attention_weight_min": -24.385068893432617,
      "activations/layer1_attention_weight_max": 14.544404983520508,
      "activations/layer1_attention_weight_min": -13.684355735778809,
      "activations/layer20_attention_weight_max": 21.735828399658203,
      "activations/layer20_attention_weight_min": -22.446672439575195,
      "activations/layer21_attention_weight_max": 37.670196533203125,
      "activations/layer21_attention_weight_min": -25.062707901000977,
      "activations/layer22_attention_weight_max": 26.793745040893555,
      "activations/layer22_attention_weight_min": -26.176742553710938,
      "activations/layer23_attention_weight_max": 32.66054916381836,
      "activations/layer23_attention_weight_min": -24.586238861083984,
      "activations/layer2_attention_weight_max": 32.123287200927734,
      "activations/layer2_attention_weight_min": -32.12158966064453,
      "activations/layer3_attention_weight_max": 90.86434936523438,
      "activations/layer3_attention_weight_min": -92.45986938476562,
      "activations/layer4_attention_weight_max": 85.23452758789062,
      "activations/layer4_attention_weight_min": -84.15977478027344,
      "activations/layer5_attention_weight_max": 65.71598815917969,
      "activations/layer5_attention_weight_min": -72.07441711425781,
      "activations/layer6_attention_weight_max": 49.66529083251953,
      "activations/layer6_attention_weight_min": -51.795108795166016,
      "activations/layer7_attention_weight_max": 70.84439086914062,
      "activations/layer7_attention_weight_min": -68.49073028564453,
      "activations/layer8_attention_weight_max": 47.619171142578125,
      "activations/layer8_attention_weight_min": -49.47638702392578,
      "activations/layer9_attention_weight_max": 46.40685272216797,
      "activations/layer9_attention_weight_min": -46.60456466674805,
      "epoch": 23.04,
      "learning_rate": 1.4450757575757573e-06,
      "loss": 2.7062,
      "step": 396500
    },
    {
      "activations/layer0_attention_weight_max": 15.842576026916504,
      "activations/layer0_attention_weight_min": -14.34950065612793,
      "activations/layer10_attention_weight_max": 36.48908615112305,
      "activations/layer10_attention_weight_min": -36.04852294921875,
      "activations/layer11_attention_weight_max": 36.3195686340332,
      "activations/layer11_attention_weight_min": -36.662437438964844,
      "activations/layer12_attention_weight_max": 20.46977424621582,
      "activations/layer12_attention_weight_min": -28.915136337280273,
      "activations/layer13_attention_weight_max": 36.35416793823242,
      "activations/layer13_attention_weight_min": -30.894309997558594,
      "activations/layer14_attention_weight_max": 35.78024673461914,
      "activations/layer14_attention_weight_min": -31.958879470825195,
      "activations/layer15_attention_weight_max": 34.6023063659668,
      "activations/layer15_attention_weight_min": -31.677621841430664,
      "activations/layer16_attention_weight_max": 36.29625701904297,
      "activations/layer16_attention_weight_min": -34.707801818847656,
      "activations/layer17_attention_weight_max": 57.72233200073242,
      "activations/layer17_attention_weight_min": -48.813636779785156,
      "activations/layer18_attention_weight_max": 51.57206344604492,
      "activations/layer18_attention_weight_min": -42.701438903808594,
      "activations/layer19_attention_weight_max": 27.557565689086914,
      "activations/layer19_attention_weight_min": -23.489412307739258,
      "activations/layer1_attention_weight_max": 15.497627258300781,
      "activations/layer1_attention_weight_min": -15.322427749633789,
      "activations/layer20_attention_weight_max": 24.378982543945312,
      "activations/layer20_attention_weight_min": -22.655961990356445,
      "activations/layer21_attention_weight_max": 38.99897003173828,
      "activations/layer21_attention_weight_min": -20.93699073791504,
      "activations/layer22_attention_weight_max": 30.967559814453125,
      "activations/layer22_attention_weight_min": -24.813947677612305,
      "activations/layer23_attention_weight_max": 35.67518615722656,
      "activations/layer23_attention_weight_min": -24.870656967163086,
      "activations/layer2_attention_weight_max": 32.06006622314453,
      "activations/layer2_attention_weight_min": -29.926345825195312,
      "activations/layer3_attention_weight_max": 89.54899597167969,
      "activations/layer3_attention_weight_min": -91.65704345703125,
      "activations/layer4_attention_weight_max": 85.61750793457031,
      "activations/layer4_attention_weight_min": -84.97244262695312,
      "activations/layer5_attention_weight_max": 60.765052795410156,
      "activations/layer5_attention_weight_min": -77.18110656738281,
      "activations/layer6_attention_weight_max": 50.26736068725586,
      "activations/layer6_attention_weight_min": -53.287410736083984,
      "activations/layer7_attention_weight_max": 68.18421936035156,
      "activations/layer7_attention_weight_min": -73.18217468261719,
      "activations/layer8_attention_weight_max": 49.60853576660156,
      "activations/layer8_attention_weight_min": -54.4930305480957,
      "activations/layer9_attention_weight_max": 43.903541564941406,
      "activations/layer9_attention_weight_min": -44.27738952636719,
      "epoch": 23.04,
      "learning_rate": 1.4261363636363637e-06,
      "loss": 2.6897,
      "step": 396550
    },
    {
      "activations/layer0_attention_weight_max": 15.108675956726074,
      "activations/layer0_attention_weight_min": -14.321805000305176,
      "activations/layer10_attention_weight_max": 39.0484733581543,
      "activations/layer10_attention_weight_min": -39.144657135009766,
      "activations/layer11_attention_weight_max": 35.18972396850586,
      "activations/layer11_attention_weight_min": -37.505615234375,
      "activations/layer12_attention_weight_max": 20.071598052978516,
      "activations/layer12_attention_weight_min": -30.896770477294922,
      "activations/layer13_attention_weight_max": 38.812862396240234,
      "activations/layer13_attention_weight_min": -27.689342498779297,
      "activations/layer14_attention_weight_max": 37.4431266784668,
      "activations/layer14_attention_weight_min": -31.901391983032227,
      "activations/layer15_attention_weight_max": 35.0380859375,
      "activations/layer15_attention_weight_min": -30.910310745239258,
      "activations/layer16_attention_weight_max": 38.12841796875,
      "activations/layer16_attention_weight_min": -33.9487190246582,
      "activations/layer17_attention_weight_max": 56.292320251464844,
      "activations/layer17_attention_weight_min": -46.29199981689453,
      "activations/layer18_attention_weight_max": 49.80585861206055,
      "activations/layer18_attention_weight_min": -40.91065216064453,
      "activations/layer19_attention_weight_max": 24.691577911376953,
      "activations/layer19_attention_weight_min": -22.52195930480957,
      "activations/layer1_attention_weight_max": 15.06682014465332,
      "activations/layer1_attention_weight_min": -14.890578269958496,
      "activations/layer20_attention_weight_max": 22.817792892456055,
      "activations/layer20_attention_weight_min": -20.706295013427734,
      "activations/layer21_attention_weight_max": 40.384944915771484,
      "activations/layer21_attention_weight_min": -22.457101821899414,
      "activations/layer22_attention_weight_max": 29.389781951904297,
      "activations/layer22_attention_weight_min": -25.32468605041504,
      "activations/layer23_attention_weight_max": 39.05522918701172,
      "activations/layer23_attention_weight_min": -27.590770721435547,
      "activations/layer2_attention_weight_max": 32.66331100463867,
      "activations/layer2_attention_weight_min": -30.371768951416016,
      "activations/layer3_attention_weight_max": 94.30073547363281,
      "activations/layer3_attention_weight_min": -94.78728485107422,
      "activations/layer4_attention_weight_max": 89.9539794921875,
      "activations/layer4_attention_weight_min": -90.11079406738281,
      "activations/layer5_attention_weight_max": 64.77595520019531,
      "activations/layer5_attention_weight_min": -75.50053405761719,
      "activations/layer6_attention_weight_max": 52.901588439941406,
      "activations/layer6_attention_weight_min": -53.15336608886719,
      "activations/layer7_attention_weight_max": 72.9688949584961,
      "activations/layer7_attention_weight_min": -73.0905532836914,
      "activations/layer8_attention_weight_max": 52.15596008300781,
      "activations/layer8_attention_weight_min": -55.54088592529297,
      "activations/layer9_attention_weight_max": 47.5066032409668,
      "activations/layer9_attention_weight_min": -50.67247772216797,
      "epoch": 23.04,
      "learning_rate": 1.4071969696969696e-06,
      "loss": 2.6963,
      "step": 396600
    },
    {
      "activations/layer0_attention_weight_max": 15.952818870544434,
      "activations/layer0_attention_weight_min": -14.493330001831055,
      "activations/layer10_attention_weight_max": 38.32350158691406,
      "activations/layer10_attention_weight_min": -38.30849838256836,
      "activations/layer11_attention_weight_max": 34.98805618286133,
      "activations/layer11_attention_weight_min": -36.22169494628906,
      "activations/layer12_attention_weight_max": 19.646333694458008,
      "activations/layer12_attention_weight_min": -26.6894474029541,
      "activations/layer13_attention_weight_max": 39.576637268066406,
      "activations/layer13_attention_weight_min": -26.438764572143555,
      "activations/layer14_attention_weight_max": 33.93190002441406,
      "activations/layer14_attention_weight_min": -29.747217178344727,
      "activations/layer15_attention_weight_max": 32.809326171875,
      "activations/layer15_attention_weight_min": -30.286376953125,
      "activations/layer16_attention_weight_max": 36.53416442871094,
      "activations/layer16_attention_weight_min": -33.05500793457031,
      "activations/layer17_attention_weight_max": 51.85149383544922,
      "activations/layer17_attention_weight_min": -46.088871002197266,
      "activations/layer18_attention_weight_max": 47.609642028808594,
      "activations/layer18_attention_weight_min": -42.6431999206543,
      "activations/layer19_attention_weight_max": 24.6602840423584,
      "activations/layer19_attention_weight_min": -23.57271385192871,
      "activations/layer1_attention_weight_max": 15.573651313781738,
      "activations/layer1_attention_weight_min": -14.237458229064941,
      "activations/layer20_attention_weight_max": 22.701934814453125,
      "activations/layer20_attention_weight_min": -21.309118270874023,
      "activations/layer21_attention_weight_max": 33.062870025634766,
      "activations/layer21_attention_weight_min": -21.41083526611328,
      "activations/layer22_attention_weight_max": 28.109643936157227,
      "activations/layer22_attention_weight_min": -25.897005081176758,
      "activations/layer23_attention_weight_max": 34.76689147949219,
      "activations/layer23_attention_weight_min": -23.517372131347656,
      "activations/layer2_attention_weight_max": 32.522315979003906,
      "activations/layer2_attention_weight_min": -30.225570678710938,
      "activations/layer3_attention_weight_max": 93.25105285644531,
      "activations/layer3_attention_weight_min": -90.64862823486328,
      "activations/layer4_attention_weight_max": 84.18626403808594,
      "activations/layer4_attention_weight_min": -84.02397155761719,
      "activations/layer5_attention_weight_max": 62.24700927734375,
      "activations/layer5_attention_weight_min": -74.65460968017578,
      "activations/layer6_attention_weight_max": 51.521785736083984,
      "activations/layer6_attention_weight_min": -51.89225769042969,
      "activations/layer7_attention_weight_max": 72.694580078125,
      "activations/layer7_attention_weight_min": -72.99759674072266,
      "activations/layer8_attention_weight_max": 52.48736572265625,
      "activations/layer8_attention_weight_min": -53.12922668457031,
      "activations/layer9_attention_weight_max": 46.66801071166992,
      "activations/layer9_attention_weight_min": -48.344146728515625,
      "epoch": 23.05,
      "learning_rate": 1.3882575757575757e-06,
      "loss": 2.7164,
      "step": 396650
    },
    {
      "activations/layer0_attention_weight_max": 15.99428939819336,
      "activations/layer0_attention_weight_min": -14.11807632446289,
      "activations/layer10_attention_weight_max": 39.61320877075195,
      "activations/layer10_attention_weight_min": -37.685420989990234,
      "activations/layer11_attention_weight_max": 35.55425262451172,
      "activations/layer11_attention_weight_min": -35.48622512817383,
      "activations/layer12_attention_weight_max": 19.665632247924805,
      "activations/layer12_attention_weight_min": -23.313047409057617,
      "activations/layer13_attention_weight_max": 36.25476837158203,
      "activations/layer13_attention_weight_min": -28.208486557006836,
      "activations/layer14_attention_weight_max": 37.15943145751953,
      "activations/layer14_attention_weight_min": -29.906349182128906,
      "activations/layer15_attention_weight_max": 35.22540283203125,
      "activations/layer15_attention_weight_min": -30.88895034790039,
      "activations/layer16_attention_weight_max": 38.65721130371094,
      "activations/layer16_attention_weight_min": -35.0137825012207,
      "activations/layer17_attention_weight_max": 53.987518310546875,
      "activations/layer17_attention_weight_min": -46.858585357666016,
      "activations/layer18_attention_weight_max": 52.01955032348633,
      "activations/layer18_attention_weight_min": -41.73367691040039,
      "activations/layer19_attention_weight_max": 25.53340721130371,
      "activations/layer19_attention_weight_min": -23.697818756103516,
      "activations/layer1_attention_weight_max": 14.95276165008545,
      "activations/layer1_attention_weight_min": -14.366509437561035,
      "activations/layer20_attention_weight_max": 24.20893096923828,
      "activations/layer20_attention_weight_min": -21.61766815185547,
      "activations/layer21_attention_weight_max": 35.14832305908203,
      "activations/layer21_attention_weight_min": -23.093677520751953,
      "activations/layer22_attention_weight_max": 28.401079177856445,
      "activations/layer22_attention_weight_min": -26.841156005859375,
      "activations/layer23_attention_weight_max": 38.70953369140625,
      "activations/layer23_attention_weight_min": -27.42732048034668,
      "activations/layer2_attention_weight_max": 31.80164909362793,
      "activations/layer2_attention_weight_min": -29.312406539916992,
      "activations/layer3_attention_weight_max": 93.04900360107422,
      "activations/layer3_attention_weight_min": -93.78196716308594,
      "activations/layer4_attention_weight_max": 89.9233627319336,
      "activations/layer4_attention_weight_min": -89.61339569091797,
      "activations/layer5_attention_weight_max": 63.85102081298828,
      "activations/layer5_attention_weight_min": -75.56167602539062,
      "activations/layer6_attention_weight_max": 52.39182662963867,
      "activations/layer6_attention_weight_min": -53.049171447753906,
      "activations/layer7_attention_weight_max": 73.12765502929688,
      "activations/layer7_attention_weight_min": -72.31238555908203,
      "activations/layer8_attention_weight_max": 52.15708541870117,
      "activations/layer8_attention_weight_min": -52.56529235839844,
      "activations/layer9_attention_weight_max": 47.610015869140625,
      "activations/layer9_attention_weight_min": -46.51649475097656,
      "epoch": 23.05,
      "learning_rate": 1.3693181818181817e-06,
      "loss": 2.6922,
      "step": 396700
    },
    {
      "activations/layer0_attention_weight_max": 15.71191120147705,
      "activations/layer0_attention_weight_min": -14.080188751220703,
      "activations/layer10_attention_weight_max": 33.7514533996582,
      "activations/layer10_attention_weight_min": -35.42001724243164,
      "activations/layer11_attention_weight_max": 34.78004455566406,
      "activations/layer11_attention_weight_min": -36.064083099365234,
      "activations/layer12_attention_weight_max": 18.974445343017578,
      "activations/layer12_attention_weight_min": -30.454143524169922,
      "activations/layer13_attention_weight_max": 33.3636589050293,
      "activations/layer13_attention_weight_min": -26.508028030395508,
      "activations/layer14_attention_weight_max": 33.81831359863281,
      "activations/layer14_attention_weight_min": -30.67357063293457,
      "activations/layer15_attention_weight_max": 32.662315368652344,
      "activations/layer15_attention_weight_min": -31.032678604125977,
      "activations/layer16_attention_weight_max": 34.83063888549805,
      "activations/layer16_attention_weight_min": -34.08281707763672,
      "activations/layer17_attention_weight_max": 49.0828971862793,
      "activations/layer17_attention_weight_min": -46.191246032714844,
      "activations/layer18_attention_weight_max": 45.12826156616211,
      "activations/layer18_attention_weight_min": -41.64851760864258,
      "activations/layer19_attention_weight_max": 24.13448143005371,
      "activations/layer19_attention_weight_min": -22.09408950805664,
      "activations/layer1_attention_weight_max": 15.1670560836792,
      "activations/layer1_attention_weight_min": -14.365792274475098,
      "activations/layer20_attention_weight_max": 20.937070846557617,
      "activations/layer20_attention_weight_min": -21.358779907226562,
      "activations/layer21_attention_weight_max": 38.123809814453125,
      "activations/layer21_attention_weight_min": -22.13248062133789,
      "activations/layer22_attention_weight_max": 29.423168182373047,
      "activations/layer22_attention_weight_min": -24.77934455871582,
      "activations/layer23_attention_weight_max": 33.91838073730469,
      "activations/layer23_attention_weight_min": -23.833845138549805,
      "activations/layer2_attention_weight_max": 32.62562561035156,
      "activations/layer2_attention_weight_min": -29.531978607177734,
      "activations/layer3_attention_weight_max": 90.3850326538086,
      "activations/layer3_attention_weight_min": -87.8904037475586,
      "activations/layer4_attention_weight_max": 88.38606262207031,
      "activations/layer4_attention_weight_min": -84.2078628540039,
      "activations/layer5_attention_weight_max": 65.1626968383789,
      "activations/layer5_attention_weight_min": -72.43446350097656,
      "activations/layer6_attention_weight_max": 50.58755874633789,
      "activations/layer6_attention_weight_min": -49.58397674560547,
      "activations/layer7_attention_weight_max": 72.3099136352539,
      "activations/layer7_attention_weight_min": -72.10443878173828,
      "activations/layer8_attention_weight_max": 48.098426818847656,
      "activations/layer8_attention_weight_min": -50.07159423828125,
      "activations/layer9_attention_weight_max": 43.32143783569336,
      "activations/layer9_attention_weight_min": -46.990272521972656,
      "epoch": 23.05,
      "learning_rate": 1.3503787878787878e-06,
      "loss": 2.6826,
      "step": 396750
    },
    {
      "activations/layer0_attention_weight_max": 16.0163516998291,
      "activations/layer0_attention_weight_min": -14.260722160339355,
      "activations/layer10_attention_weight_max": 38.59932327270508,
      "activations/layer10_attention_weight_min": -36.75033187866211,
      "activations/layer11_attention_weight_max": 35.3558349609375,
      "activations/layer11_attention_weight_min": -36.177616119384766,
      "activations/layer12_attention_weight_max": 19.754220962524414,
      "activations/layer12_attention_weight_min": -25.601057052612305,
      "activations/layer13_attention_weight_max": 41.278865814208984,
      "activations/layer13_attention_weight_min": -28.29305648803711,
      "activations/layer14_attention_weight_max": 35.433773040771484,
      "activations/layer14_attention_weight_min": -30.626480102539062,
      "activations/layer15_attention_weight_max": 34.97738265991211,
      "activations/layer15_attention_weight_min": -30.55919647216797,
      "activations/layer16_attention_weight_max": 37.55954360961914,
      "activations/layer16_attention_weight_min": -34.43857955932617,
      "activations/layer17_attention_weight_max": 53.43379592895508,
      "activations/layer17_attention_weight_min": -48.02404022216797,
      "activations/layer18_attention_weight_max": 48.4118766784668,
      "activations/layer18_attention_weight_min": -42.671932220458984,
      "activations/layer19_attention_weight_max": 25.394376754760742,
      "activations/layer19_attention_weight_min": -23.077667236328125,
      "activations/layer1_attention_weight_max": 15.553452491760254,
      "activations/layer1_attention_weight_min": -14.716806411743164,
      "activations/layer20_attention_weight_max": 23.73820686340332,
      "activations/layer20_attention_weight_min": -22.96051025390625,
      "activations/layer21_attention_weight_max": 35.65465545654297,
      "activations/layer21_attention_weight_min": -24.971023559570312,
      "activations/layer22_attention_weight_max": 29.60677719116211,
      "activations/layer22_attention_weight_min": -25.87321662902832,
      "activations/layer23_attention_weight_max": 37.67497253417969,
      "activations/layer23_attention_weight_min": -24.52365493774414,
      "activations/layer2_attention_weight_max": 32.07866287231445,
      "activations/layer2_attention_weight_min": -30.89421844482422,
      "activations/layer3_attention_weight_max": 92.99201965332031,
      "activations/layer3_attention_weight_min": -93.99758911132812,
      "activations/layer4_attention_weight_max": 90.2924575805664,
      "activations/layer4_attention_weight_min": -87.8980941772461,
      "activations/layer5_attention_weight_max": 64.03810119628906,
      "activations/layer5_attention_weight_min": -76.45077514648438,
      "activations/layer6_attention_weight_max": 53.60442352294922,
      "activations/layer6_attention_weight_min": -55.84091567993164,
      "activations/layer7_attention_weight_max": 73.95687103271484,
      "activations/layer7_attention_weight_min": -75.60546875,
      "activations/layer8_attention_weight_max": 48.45572280883789,
      "activations/layer8_attention_weight_min": -52.53219223022461,
      "activations/layer9_attention_weight_max": 45.05491256713867,
      "activations/layer9_attention_weight_min": -48.61167526245117,
      "epoch": 23.06,
      "learning_rate": 1.3314393939393938e-06,
      "loss": 2.7038,
      "step": 396800
    },
    {
      "activations/layer0_attention_weight_max": 15.823827743530273,
      "activations/layer0_attention_weight_min": -14.320169448852539,
      "activations/layer10_attention_weight_max": 40.30495071411133,
      "activations/layer10_attention_weight_min": -37.8507194519043,
      "activations/layer11_attention_weight_max": 36.90313720703125,
      "activations/layer11_attention_weight_min": -36.53129196166992,
      "activations/layer12_attention_weight_max": 20.810928344726562,
      "activations/layer12_attention_weight_min": -24.444517135620117,
      "activations/layer13_attention_weight_max": 40.362060546875,
      "activations/layer13_attention_weight_min": -27.985876083374023,
      "activations/layer14_attention_weight_max": 39.27975082397461,
      "activations/layer14_attention_weight_min": -34.614620208740234,
      "activations/layer15_attention_weight_max": 40.85605239868164,
      "activations/layer15_attention_weight_min": -35.817138671875,
      "activations/layer16_attention_weight_max": 40.845726013183594,
      "activations/layer16_attention_weight_min": -41.058937072753906,
      "activations/layer17_attention_weight_max": 57.71677017211914,
      "activations/layer17_attention_weight_min": -56.623233795166016,
      "activations/layer18_attention_weight_max": 53.12657928466797,
      "activations/layer18_attention_weight_min": -51.27540969848633,
      "activations/layer19_attention_weight_max": 26.532743453979492,
      "activations/layer19_attention_weight_min": -29.61362648010254,
      "activations/layer1_attention_weight_max": 15.694596290588379,
      "activations/layer1_attention_weight_min": -13.92177677154541,
      "activations/layer20_attention_weight_max": 26.151294708251953,
      "activations/layer20_attention_weight_min": -25.02984619140625,
      "activations/layer21_attention_weight_max": 43.932735443115234,
      "activations/layer21_attention_weight_min": -30.316442489624023,
      "activations/layer22_attention_weight_max": 29.192346572875977,
      "activations/layer22_attention_weight_min": -29.47432518005371,
      "activations/layer23_attention_weight_max": 37.53894805908203,
      "activations/layer23_attention_weight_min": -29.773014068603516,
      "activations/layer2_attention_weight_max": 33.294342041015625,
      "activations/layer2_attention_weight_min": -33.09693908691406,
      "activations/layer3_attention_weight_max": 93.82125854492188,
      "activations/layer3_attention_weight_min": -98.42790222167969,
      "activations/layer4_attention_weight_max": 91.60435485839844,
      "activations/layer4_attention_weight_min": -90.08976745605469,
      "activations/layer5_attention_weight_max": 67.7909927368164,
      "activations/layer5_attention_weight_min": -75.70414733886719,
      "activations/layer6_attention_weight_max": 52.987159729003906,
      "activations/layer6_attention_weight_min": -53.179805755615234,
      "activations/layer7_attention_weight_max": 74.0600814819336,
      "activations/layer7_attention_weight_min": -74.33334350585938,
      "activations/layer8_attention_weight_max": 51.125370025634766,
      "activations/layer8_attention_weight_min": -52.5605354309082,
      "activations/layer9_attention_weight_max": 48.78481674194336,
      "activations/layer9_attention_weight_min": -50.28407287597656,
      "epoch": 23.06,
      "learning_rate": 1.3125e-06,
      "loss": 2.6817,
      "step": 396850
    },
    {
      "activations/layer0_attention_weight_max": 15.054339408874512,
      "activations/layer0_attention_weight_min": -14.043603897094727,
      "activations/layer10_attention_weight_max": 36.618019104003906,
      "activations/layer10_attention_weight_min": -36.63900375366211,
      "activations/layer11_attention_weight_max": 34.851890563964844,
      "activations/layer11_attention_weight_min": -34.957237243652344,
      "activations/layer12_attention_weight_max": 19.927358627319336,
      "activations/layer12_attention_weight_min": -24.064176559448242,
      "activations/layer13_attention_weight_max": 44.228572845458984,
      "activations/layer13_attention_weight_min": -31.967811584472656,
      "activations/layer14_attention_weight_max": 35.879798889160156,
      "activations/layer14_attention_weight_min": -31.613086700439453,
      "activations/layer15_attention_weight_max": 34.42293167114258,
      "activations/layer15_attention_weight_min": -31.288049697875977,
      "activations/layer16_attention_weight_max": 34.27556610107422,
      "activations/layer16_attention_weight_min": -33.4458122253418,
      "activations/layer17_attention_weight_max": 51.632110595703125,
      "activations/layer17_attention_weight_min": -44.53133773803711,
      "activations/layer18_attention_weight_max": 46.466712951660156,
      "activations/layer18_attention_weight_min": -39.30235290527344,
      "activations/layer19_attention_weight_max": 26.04204750061035,
      "activations/layer19_attention_weight_min": -25.519895553588867,
      "activations/layer1_attention_weight_max": 14.960994720458984,
      "activations/layer1_attention_weight_min": -14.779162406921387,
      "activations/layer20_attention_weight_max": 24.11545181274414,
      "activations/layer20_attention_weight_min": -23.59000587463379,
      "activations/layer21_attention_weight_max": 37.403297424316406,
      "activations/layer21_attention_weight_min": -24.32684326171875,
      "activations/layer22_attention_weight_max": 30.034313201904297,
      "activations/layer22_attention_weight_min": -26.441452026367188,
      "activations/layer23_attention_weight_max": 36.389259338378906,
      "activations/layer23_attention_weight_min": -25.477142333984375,
      "activations/layer2_attention_weight_max": 31.78101921081543,
      "activations/layer2_attention_weight_min": -30.03033447265625,
      "activations/layer3_attention_weight_max": 91.40680694580078,
      "activations/layer3_attention_weight_min": -94.0538330078125,
      "activations/layer4_attention_weight_max": 87.62322235107422,
      "activations/layer4_attention_weight_min": -87.0103988647461,
      "activations/layer5_attention_weight_max": 62.675472259521484,
      "activations/layer5_attention_weight_min": -75.5684585571289,
      "activations/layer6_attention_weight_max": 51.50222396850586,
      "activations/layer6_attention_weight_min": -52.28651809692383,
      "activations/layer7_attention_weight_max": 71.1972885131836,
      "activations/layer7_attention_weight_min": -70.49986267089844,
      "activations/layer8_attention_weight_max": 49.031009674072266,
      "activations/layer8_attention_weight_min": -52.37825393676758,
      "activations/layer9_attention_weight_max": 43.96897506713867,
      "activations/layer9_attention_weight_min": -47.630767822265625,
      "epoch": 23.06,
      "learning_rate": 1.2935606060606059e-06,
      "loss": 2.696,
      "step": 396900
    },
    {
      "activations/layer0_attention_weight_max": 15.169763565063477,
      "activations/layer0_attention_weight_min": -14.211589813232422,
      "activations/layer10_attention_weight_max": 36.89604568481445,
      "activations/layer10_attention_weight_min": -36.28515625,
      "activations/layer11_attention_weight_max": 33.381874084472656,
      "activations/layer11_attention_weight_min": -35.94989013671875,
      "activations/layer12_attention_weight_max": 19.94219398498535,
      "activations/layer12_attention_weight_min": -28.106672286987305,
      "activations/layer13_attention_weight_max": 36.71266174316406,
      "activations/layer13_attention_weight_min": -27.764328002929688,
      "activations/layer14_attention_weight_max": 36.81346130371094,
      "activations/layer14_attention_weight_min": -30.55545997619629,
      "activations/layer15_attention_weight_max": 33.34003448486328,
      "activations/layer15_attention_weight_min": -31.165569305419922,
      "activations/layer16_attention_weight_max": 34.45360565185547,
      "activations/layer16_attention_weight_min": -33.67705535888672,
      "activations/layer17_attention_weight_max": 52.28312683105469,
      "activations/layer17_attention_weight_min": -44.93465805053711,
      "activations/layer18_attention_weight_max": 50.4145622253418,
      "activations/layer18_attention_weight_min": -40.633155822753906,
      "activations/layer19_attention_weight_max": 24.63614845275879,
      "activations/layer19_attention_weight_min": -22.880449295043945,
      "activations/layer1_attention_weight_max": 15.058306694030762,
      "activations/layer1_attention_weight_min": -14.992096900939941,
      "activations/layer20_attention_weight_max": 22.985612869262695,
      "activations/layer20_attention_weight_min": -22.058616638183594,
      "activations/layer21_attention_weight_max": 37.856414794921875,
      "activations/layer21_attention_weight_min": -21.01427459716797,
      "activations/layer22_attention_weight_max": 27.67885398864746,
      "activations/layer22_attention_weight_min": -24.42072105407715,
      "activations/layer23_attention_weight_max": 33.49687957763672,
      "activations/layer23_attention_weight_min": -24.693756103515625,
      "activations/layer2_attention_weight_max": 32.94352340698242,
      "activations/layer2_attention_weight_min": -30.250476837158203,
      "activations/layer3_attention_weight_max": 90.7989501953125,
      "activations/layer3_attention_weight_min": -91.71549987792969,
      "activations/layer4_attention_weight_max": 85.0394515991211,
      "activations/layer4_attention_weight_min": -87.9280014038086,
      "activations/layer5_attention_weight_max": 61.722328186035156,
      "activations/layer5_attention_weight_min": -76.27421569824219,
      "activations/layer6_attention_weight_max": 50.49155044555664,
      "activations/layer6_attention_weight_min": -51.90189743041992,
      "activations/layer7_attention_weight_max": 72.43557739257812,
      "activations/layer7_attention_weight_min": -71.12193298339844,
      "activations/layer8_attention_weight_max": 46.66748809814453,
      "activations/layer8_attention_weight_min": -53.97371292114258,
      "activations/layer9_attention_weight_max": 43.377620697021484,
      "activations/layer9_attention_weight_min": -47.99847412109375,
      "epoch": 23.07,
      "learning_rate": 1.2746212121212118e-06,
      "loss": 2.676,
      "step": 396950
    },
    {
      "activations/layer0_attention_weight_max": 15.689325332641602,
      "activations/layer0_attention_weight_min": -13.971720695495605,
      "activations/layer10_attention_weight_max": 35.343528747558594,
      "activations/layer10_attention_weight_min": -37.06492233276367,
      "activations/layer11_attention_weight_max": 34.82581329345703,
      "activations/layer11_attention_weight_min": -37.244384765625,
      "activations/layer12_attention_weight_max": 20.686981201171875,
      "activations/layer12_attention_weight_min": -23.689823150634766,
      "activations/layer13_attention_weight_max": 36.82117462158203,
      "activations/layer13_attention_weight_min": -27.512500762939453,
      "activations/layer14_attention_weight_max": 36.44638442993164,
      "activations/layer14_attention_weight_min": -32.00925064086914,
      "activations/layer15_attention_weight_max": 34.619571685791016,
      "activations/layer15_attention_weight_min": -31.29243278503418,
      "activations/layer16_attention_weight_max": 37.62367248535156,
      "activations/layer16_attention_weight_min": -33.987972259521484,
      "activations/layer17_attention_weight_max": 54.935707092285156,
      "activations/layer17_attention_weight_min": -46.136714935302734,
      "activations/layer18_attention_weight_max": 50.800628662109375,
      "activations/layer18_attention_weight_min": -41.16984176635742,
      "activations/layer19_attention_weight_max": 26.54744529724121,
      "activations/layer19_attention_weight_min": -25.148202896118164,
      "activations/layer1_attention_weight_max": 15.705395698547363,
      "activations/layer1_attention_weight_min": -15.510937690734863,
      "activations/layer20_attention_weight_max": 25.214031219482422,
      "activations/layer20_attention_weight_min": -21.498638153076172,
      "activations/layer21_attention_weight_max": 41.82783126831055,
      "activations/layer21_attention_weight_min": -22.392349243164062,
      "activations/layer22_attention_weight_max": 28.24721908569336,
      "activations/layer22_attention_weight_min": -26.523775100708008,
      "activations/layer23_attention_weight_max": 39.57554626464844,
      "activations/layer23_attention_weight_min": -22.591201782226562,
      "activations/layer2_attention_weight_max": 32.86980438232422,
      "activations/layer2_attention_weight_min": -30.461387634277344,
      "activations/layer3_attention_weight_max": 97.10787200927734,
      "activations/layer3_attention_weight_min": -93.28528594970703,
      "activations/layer4_attention_weight_max": 91.30950927734375,
      "activations/layer4_attention_weight_min": -90.87899017333984,
      "activations/layer5_attention_weight_max": 61.88623809814453,
      "activations/layer5_attention_weight_min": -74.13475799560547,
      "activations/layer6_attention_weight_max": 53.65712356567383,
      "activations/layer6_attention_weight_min": -50.63106155395508,
      "activations/layer7_attention_weight_max": 67.94237518310547,
      "activations/layer7_attention_weight_min": -70.06979370117188,
      "activations/layer8_attention_weight_max": 49.022605895996094,
      "activations/layer8_attention_weight_min": -50.31251907348633,
      "activations/layer9_attention_weight_max": 45.58064651489258,
      "activations/layer9_attention_weight_min": -47.071617126464844,
      "epoch": 23.07,
      "learning_rate": 1.2556818181818182e-06,
      "loss": 2.7006,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.4971,
      "eval_samples_per_second": 505.348,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.4971,
      "eval_openwebtext_samples_per_second": 505.348,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_wikitext_loss": 2.8984375,
      "eval_wikitext_ppl": 18.145770440989054,
      "eval_wikitext_runtime": 2.0034,
      "eval_wikitext_samples_per_second": 227.617,
      "step": 397000
    },
    {
      "epoch": 23.07,
      "eval_lambada_loss": 2.373046875,
      "eval_lambada_ppl": 10.730035606033903,
      "eval_lambada_runtime": 9.5757,
      "eval_lambada_samples_per_second": 508.475,
      "step": 397000
    },
    {
      "activations/layer0_attention_weight_max": 15.376359939575195,
      "activations/layer0_attention_weight_min": -14.076874732971191,
      "activations/layer10_attention_weight_max": 38.7088737487793,
      "activations/layer10_attention_weight_min": -38.994239807128906,
      "activations/layer11_attention_weight_max": 35.207550048828125,
      "activations/layer11_attention_weight_min": -39.02257537841797,
      "activations/layer12_attention_weight_max": 20.140560150146484,
      "activations/layer12_attention_weight_min": -25.383832931518555,
      "activations/layer13_attention_weight_max": 51.464073181152344,
      "activations/layer13_attention_weight_min": -32.91877746582031,
      "activations/layer14_attention_weight_max": 34.65627670288086,
      "activations/layer14_attention_weight_min": -31.690425872802734,
      "activations/layer15_attention_weight_max": 33.59100341796875,
      "activations/layer15_attention_weight_min": -30.59522819519043,
      "activations/layer16_attention_weight_max": 35.3651237487793,
      "activations/layer16_attention_weight_min": -35.256412506103516,
      "activations/layer17_attention_weight_max": 53.92502975463867,
      "activations/layer17_attention_weight_min": -48.55741500854492,
      "activations/layer18_attention_weight_max": 48.395652770996094,
      "activations/layer18_attention_weight_min": -42.03307342529297,
      "activations/layer19_attention_weight_max": 25.762149810791016,
      "activations/layer19_attention_weight_min": -22.03205108642578,
      "activations/layer1_attention_weight_max": 15.215240478515625,
      "activations/layer1_attention_weight_min": -14.3341064453125,
      "activations/layer20_attention_weight_max": 22.47245216369629,
      "activations/layer20_attention_weight_min": -21.399845123291016,
      "activations/layer21_attention_weight_max": 35.621944427490234,
      "activations/layer21_attention_weight_min": -20.198211669921875,
      "activations/layer22_attention_weight_max": 27.487878799438477,
      "activations/layer22_attention_weight_min": -25.71274757385254,
      "activations/layer23_attention_weight_max": 34.60314178466797,
      "activations/layer23_attention_weight_min": -25.031522750854492,
      "activations/layer2_attention_weight_max": 32.686790466308594,
      "activations/layer2_attention_weight_min": -30.428848266601562,
      "activations/layer3_attention_weight_max": 92.32575225830078,
      "activations/layer3_attention_weight_min": -93.85021209716797,
      "activations/layer4_attention_weight_max": 89.78740692138672,
      "activations/layer4_attention_weight_min": -93.77530670166016,
      "activations/layer5_attention_weight_max": 64.03990173339844,
      "activations/layer5_attention_weight_min": -73.20565795898438,
      "activations/layer6_attention_weight_max": 50.78422164916992,
      "activations/layer6_attention_weight_min": -53.290443420410156,
      "activations/layer7_attention_weight_max": 73.55029296875,
      "activations/layer7_attention_weight_min": -73.72583770751953,
      "activations/layer8_attention_weight_max": 49.64212417602539,
      "activations/layer8_attention_weight_min": -50.69424057006836,
      "activations/layer9_attention_weight_max": 47.22211837768555,
      "activations/layer9_attention_weight_min": -48.65177536010742,
      "epoch": 23.07,
      "learning_rate": 1.2367424242424241e-06,
      "loss": 2.6975,
      "step": 397050
    },
    {
      "activations/layer0_attention_weight_max": 15.555745124816895,
      "activations/layer0_attention_weight_min": -14.180399894714355,
      "activations/layer10_attention_weight_max": 34.500892639160156,
      "activations/layer10_attention_weight_min": -37.202083587646484,
      "activations/layer11_attention_weight_max": 33.027671813964844,
      "activations/layer11_attention_weight_min": -35.88175582885742,
      "activations/layer12_attention_weight_max": 20.089691162109375,
      "activations/layer12_attention_weight_min": -25.13054847717285,
      "activations/layer13_attention_weight_max": 34.65781784057617,
      "activations/layer13_attention_weight_min": -30.34433937072754,
      "activations/layer14_attention_weight_max": 38.56595993041992,
      "activations/layer14_attention_weight_min": -31.804553985595703,
      "activations/layer15_attention_weight_max": 34.83136749267578,
      "activations/layer15_attention_weight_min": -31.98146629333496,
      "activations/layer16_attention_weight_max": 36.464942932128906,
      "activations/layer16_attention_weight_min": -33.773521423339844,
      "activations/layer17_attention_weight_max": 52.590667724609375,
      "activations/layer17_attention_weight_min": -47.07209396362305,
      "activations/layer18_attention_weight_max": 49.753562927246094,
      "activations/layer18_attention_weight_min": -42.09421920776367,
      "activations/layer19_attention_weight_max": 26.139448165893555,
      "activations/layer19_attention_weight_min": -23.60200309753418,
      "activations/layer1_attention_weight_max": 14.569869041442871,
      "activations/layer1_attention_weight_min": -14.551856994628906,
      "activations/layer20_attention_weight_max": 22.880064010620117,
      "activations/layer20_attention_weight_min": -23.51133918762207,
      "activations/layer21_attention_weight_max": 36.76546096801758,
      "activations/layer21_attention_weight_min": -25.14505958557129,
      "activations/layer22_attention_weight_max": 29.973766326904297,
      "activations/layer22_attention_weight_min": -25.54785919189453,
      "activations/layer23_attention_weight_max": 36.35355758666992,
      "activations/layer23_attention_weight_min": -25.732872009277344,
      "activations/layer2_attention_weight_max": 32.976837158203125,
      "activations/layer2_attention_weight_min": -31.507604598999023,
      "activations/layer3_attention_weight_max": 95.02483367919922,
      "activations/layer3_attention_weight_min": -96.05758666992188,
      "activations/layer4_attention_weight_max": 88.29729461669922,
      "activations/layer4_attention_weight_min": -84.08849334716797,
      "activations/layer5_attention_weight_max": 64.05831909179688,
      "activations/layer5_attention_weight_min": -73.5077896118164,
      "activations/layer6_attention_weight_max": 48.81810760498047,
      "activations/layer6_attention_weight_min": -51.15163040161133,
      "activations/layer7_attention_weight_max": 70.21067810058594,
      "activations/layer7_attention_weight_min": -69.01713562011719,
      "activations/layer8_attention_weight_max": 46.109737396240234,
      "activations/layer8_attention_weight_min": -50.329891204833984,
      "activations/layer9_attention_weight_max": 43.14542007446289,
      "activations/layer9_attention_weight_min": -48.205848693847656,
      "epoch": 23.07,
      "learning_rate": 1.2178030303030303e-06,
      "loss": 2.6842,
      "step": 397100
    },
    {
      "activations/layer0_attention_weight_max": 15.422479629516602,
      "activations/layer0_attention_weight_min": -14.403684616088867,
      "activations/layer10_attention_weight_max": 35.49898147583008,
      "activations/layer10_attention_weight_min": -36.61845779418945,
      "activations/layer11_attention_weight_max": 34.31511306762695,
      "activations/layer11_attention_weight_min": -35.70561218261719,
      "activations/layer12_attention_weight_max": 18.391429901123047,
      "activations/layer12_attention_weight_min": -23.43317222595215,
      "activations/layer13_attention_weight_max": 33.429569244384766,
      "activations/layer13_attention_weight_min": -27.981708526611328,
      "activations/layer14_attention_weight_max": 36.037750244140625,
      "activations/layer14_attention_weight_min": -29.78724479675293,
      "activations/layer15_attention_weight_max": 33.031494140625,
      "activations/layer15_attention_weight_min": -33.481815338134766,
      "activations/layer16_attention_weight_max": 33.94174575805664,
      "activations/layer16_attention_weight_min": -32.562828063964844,
      "activations/layer17_attention_weight_max": 51.7425422668457,
      "activations/layer17_attention_weight_min": -44.62215042114258,
      "activations/layer18_attention_weight_max": 45.902320861816406,
      "activations/layer18_attention_weight_min": -40.72715759277344,
      "activations/layer19_attention_weight_max": 24.543718338012695,
      "activations/layer19_attention_weight_min": -21.75361442565918,
      "activations/layer1_attention_weight_max": 14.846335411071777,
      "activations/layer1_attention_weight_min": -14.306753158569336,
      "activations/layer20_attention_weight_max": 22.98185920715332,
      "activations/layer20_attention_weight_min": -21.266923904418945,
      "activations/layer21_attention_weight_max": 31.841623306274414,
      "activations/layer21_attention_weight_min": -21.474857330322266,
      "activations/layer22_attention_weight_max": 29.057344436645508,
      "activations/layer22_attention_weight_min": -25.54608154296875,
      "activations/layer23_attention_weight_max": 35.498069763183594,
      "activations/layer23_attention_weight_min": -24.598682403564453,
      "activations/layer2_attention_weight_max": 32.29801940917969,
      "activations/layer2_attention_weight_min": -31.149124145507812,
      "activations/layer3_attention_weight_max": 92.59879302978516,
      "activations/layer3_attention_weight_min": -93.5700912475586,
      "activations/layer4_attention_weight_max": 86.88703918457031,
      "activations/layer4_attention_weight_min": -88.6832046508789,
      "activations/layer5_attention_weight_max": 63.157039642333984,
      "activations/layer5_attention_weight_min": -72.48469543457031,
      "activations/layer6_attention_weight_max": 49.8807258605957,
      "activations/layer6_attention_weight_min": -54.11067581176758,
      "activations/layer7_attention_weight_max": 67.14312744140625,
      "activations/layer7_attention_weight_min": -71.37548065185547,
      "activations/layer8_attention_weight_max": 48.11206817626953,
      "activations/layer8_attention_weight_min": -53.19661331176758,
      "activations/layer9_attention_weight_max": 47.137813568115234,
      "activations/layer9_attention_weight_min": -47.47867965698242,
      "epoch": 23.08,
      "learning_rate": 1.1988636363636362e-06,
      "loss": 2.7001,
      "step": 397150
    },
    {
      "activations/layer0_attention_weight_max": 15.936284065246582,
      "activations/layer0_attention_weight_min": -13.966983795166016,
      "activations/layer10_attention_weight_max": 42.420440673828125,
      "activations/layer10_attention_weight_min": -41.0146598815918,
      "activations/layer11_attention_weight_max": 41.24509048461914,
      "activations/layer11_attention_weight_min": -41.4124870300293,
      "activations/layer12_attention_weight_max": 19.459720611572266,
      "activations/layer12_attention_weight_min": -26.476097106933594,
      "activations/layer13_attention_weight_max": 38.63938903808594,
      "activations/layer13_attention_weight_min": -34.37413024902344,
      "activations/layer14_attention_weight_max": 40.077674865722656,
      "activations/layer14_attention_weight_min": -33.564022064208984,
      "activations/layer15_attention_weight_max": 37.7694206237793,
      "activations/layer15_attention_weight_min": -33.58161544799805,
      "activations/layer16_attention_weight_max": 34.64585494995117,
      "activations/layer16_attention_weight_min": -33.007015228271484,
      "activations/layer17_attention_weight_max": 55.00447082519531,
      "activations/layer17_attention_weight_min": -45.70769119262695,
      "activations/layer18_attention_weight_max": 49.47384262084961,
      "activations/layer18_attention_weight_min": -42.94038772583008,
      "activations/layer19_attention_weight_max": 25.731922149658203,
      "activations/layer19_attention_weight_min": -23.768369674682617,
      "activations/layer1_attention_weight_max": 15.397041320800781,
      "activations/layer1_attention_weight_min": -14.44752025604248,
      "activations/layer20_attention_weight_max": 26.048505783081055,
      "activations/layer20_attention_weight_min": -23.667686462402344,
      "activations/layer21_attention_weight_max": 34.16107940673828,
      "activations/layer21_attention_weight_min": -21.941701889038086,
      "activations/layer22_attention_weight_max": 29.330476760864258,
      "activations/layer22_attention_weight_min": -24.36451530456543,
      "activations/layer23_attention_weight_max": 34.825828552246094,
      "activations/layer23_attention_weight_min": -22.738765716552734,
      "activations/layer2_attention_weight_max": 35.217594146728516,
      "activations/layer2_attention_weight_min": -32.259429931640625,
      "activations/layer3_attention_weight_max": 92.72747039794922,
      "activations/layer3_attention_weight_min": -94.87845611572266,
      "activations/layer4_attention_weight_max": 87.10816955566406,
      "activations/layer4_attention_weight_min": -90.12957000732422,
      "activations/layer5_attention_weight_max": 61.87837219238281,
      "activations/layer5_attention_weight_min": -71.31748962402344,
      "activations/layer6_attention_weight_max": 52.04356384277344,
      "activations/layer6_attention_weight_min": -54.113582611083984,
      "activations/layer7_attention_weight_max": 78.00650787353516,
      "activations/layer7_attention_weight_min": -73.7071762084961,
      "activations/layer8_attention_weight_max": 55.141151428222656,
      "activations/layer8_attention_weight_min": -59.36909866333008,
      "activations/layer9_attention_weight_max": 56.96377182006836,
      "activations/layer9_attention_weight_min": -50.95158004760742,
      "epoch": 23.08,
      "learning_rate": 1.1799242424242424e-06,
      "loss": 2.7048,
      "step": 397200
    },
    {
      "activations/layer0_attention_weight_max": 14.921319007873535,
      "activations/layer0_attention_weight_min": -14.351689338684082,
      "activations/layer10_attention_weight_max": 39.01054382324219,
      "activations/layer10_attention_weight_min": -38.56132888793945,
      "activations/layer11_attention_weight_max": 35.77574920654297,
      "activations/layer11_attention_weight_min": -36.110069274902344,
      "activations/layer12_attention_weight_max": 21.84958839416504,
      "activations/layer12_attention_weight_min": -24.08026885986328,
      "activations/layer13_attention_weight_max": 37.585968017578125,
      "activations/layer13_attention_weight_min": -27.658117294311523,
      "activations/layer14_attention_weight_max": 37.134498596191406,
      "activations/layer14_attention_weight_min": -31.21535873413086,
      "activations/layer15_attention_weight_max": 35.56884002685547,
      "activations/layer15_attention_weight_min": -31.31977653503418,
      "activations/layer16_attention_weight_max": 38.02845764160156,
      "activations/layer16_attention_weight_min": -36.6644287109375,
      "activations/layer17_attention_weight_max": 56.93524932861328,
      "activations/layer17_attention_weight_min": -48.698001861572266,
      "activations/layer18_attention_weight_max": 53.395233154296875,
      "activations/layer18_attention_weight_min": -41.59065246582031,
      "activations/layer19_attention_weight_max": 24.88401222229004,
      "activations/layer19_attention_weight_min": -25.21596336364746,
      "activations/layer1_attention_weight_max": 15.500903129577637,
      "activations/layer1_attention_weight_min": -15.576451301574707,
      "activations/layer20_attention_weight_max": 24.69038963317871,
      "activations/layer20_attention_weight_min": -21.55339241027832,
      "activations/layer21_attention_weight_max": 37.706825256347656,
      "activations/layer21_attention_weight_min": -21.302274703979492,
      "activations/layer22_attention_weight_max": 29.614458084106445,
      "activations/layer22_attention_weight_min": -23.464508056640625,
      "activations/layer23_attention_weight_max": 41.14500427246094,
      "activations/layer23_attention_weight_min": -23.503604888916016,
      "activations/layer2_attention_weight_max": 32.584346771240234,
      "activations/layer2_attention_weight_min": -31.84248924255371,
      "activations/layer3_attention_weight_max": 91.79143524169922,
      "activations/layer3_attention_weight_min": -95.44258880615234,
      "activations/layer4_attention_weight_max": 91.81111907958984,
      "activations/layer4_attention_weight_min": -97.67778778076172,
      "activations/layer5_attention_weight_max": 64.1011962890625,
      "activations/layer5_attention_weight_min": -75.87291717529297,
      "activations/layer6_attention_weight_max": 53.217594146728516,
      "activations/layer6_attention_weight_min": -53.1319465637207,
      "activations/layer7_attention_weight_max": 77.12572479248047,
      "activations/layer7_attention_weight_min": -74.30760955810547,
      "activations/layer8_attention_weight_max": 52.06149673461914,
      "activations/layer8_attention_weight_min": -54.939456939697266,
      "activations/layer9_attention_weight_max": 45.510066986083984,
      "activations/layer9_attention_weight_min": -50.644474029541016,
      "epoch": 23.08,
      "learning_rate": 1.1609848484848485e-06,
      "loss": 2.6933,
      "step": 397250
    },
    {
      "activations/layer0_attention_weight_max": 15.827720642089844,
      "activations/layer0_attention_weight_min": -14.294731140136719,
      "activations/layer10_attention_weight_max": 33.053375244140625,
      "activations/layer10_attention_weight_min": -34.82707214355469,
      "activations/layer11_attention_weight_max": 31.318897247314453,
      "activations/layer11_attention_weight_min": -35.72287368774414,
      "activations/layer12_attention_weight_max": 19.023778915405273,
      "activations/layer12_attention_weight_min": -26.12369728088379,
      "activations/layer13_attention_weight_max": 32.64725875854492,
      "activations/layer13_attention_weight_min": -26.587080001831055,
      "activations/layer14_attention_weight_max": 35.7358512878418,
      "activations/layer14_attention_weight_min": -30.49057388305664,
      "activations/layer15_attention_weight_max": 32.33674621582031,
      "activations/layer15_attention_weight_min": -30.35598373413086,
      "activations/layer16_attention_weight_max": 35.36444854736328,
      "activations/layer16_attention_weight_min": -34.02585220336914,
      "activations/layer17_attention_weight_max": 53.273441314697266,
      "activations/layer17_attention_weight_min": -43.93912124633789,
      "activations/layer18_attention_weight_max": 48.26536178588867,
      "activations/layer18_attention_weight_min": -38.998046875,
      "activations/layer19_attention_weight_max": 24.886640548706055,
      "activations/layer19_attention_weight_min": -23.86601448059082,
      "activations/layer1_attention_weight_max": 16.016727447509766,
      "activations/layer1_attention_weight_min": -15.053414344787598,
      "activations/layer20_attention_weight_max": 24.67523765563965,
      "activations/layer20_attention_weight_min": -22.199811935424805,
      "activations/layer21_attention_weight_max": 37.3610725402832,
      "activations/layer21_attention_weight_min": -21.983808517456055,
      "activations/layer22_attention_weight_max": 29.035385131835938,
      "activations/layer22_attention_weight_min": -25.579858779907227,
      "activations/layer23_attention_weight_max": 38.950401306152344,
      "activations/layer23_attention_weight_min": -26.180587768554688,
      "activations/layer2_attention_weight_max": 31.4432373046875,
      "activations/layer2_attention_weight_min": -32.042545318603516,
      "activations/layer3_attention_weight_max": 91.61622619628906,
      "activations/layer3_attention_weight_min": -93.81439971923828,
      "activations/layer4_attention_weight_max": 87.09537506103516,
      "activations/layer4_attention_weight_min": -86.39507293701172,
      "activations/layer5_attention_weight_max": 63.59467315673828,
      "activations/layer5_attention_weight_min": -76.93168640136719,
      "activations/layer6_attention_weight_max": 51.604393005371094,
      "activations/layer6_attention_weight_min": -52.65226745605469,
      "activations/layer7_attention_weight_max": 70.63368225097656,
      "activations/layer7_attention_weight_min": -73.83736419677734,
      "activations/layer8_attention_weight_max": 47.810577392578125,
      "activations/layer8_attention_weight_min": -50.086402893066406,
      "activations/layer9_attention_weight_max": 45.004798889160156,
      "activations/layer9_attention_weight_min": -48.701385498046875,
      "epoch": 23.09,
      "learning_rate": 1.1420454545454544e-06,
      "loss": 2.705,
      "step": 397300
    },
    {
      "activations/layer0_attention_weight_max": 16.23660659790039,
      "activations/layer0_attention_weight_min": -14.003669738769531,
      "activations/layer10_attention_weight_max": 34.656700134277344,
      "activations/layer10_attention_weight_min": -36.902095794677734,
      "activations/layer11_attention_weight_max": 32.4024658203125,
      "activations/layer11_attention_weight_min": -36.850223541259766,
      "activations/layer12_attention_weight_max": 19.435338973999023,
      "activations/layer12_attention_weight_min": -23.949214935302734,
      "activations/layer13_attention_weight_max": 34.652610778808594,
      "activations/layer13_attention_weight_min": -25.818632125854492,
      "activations/layer14_attention_weight_max": 37.07295608520508,
      "activations/layer14_attention_weight_min": -28.653594970703125,
      "activations/layer15_attention_weight_max": 40.01604080200195,
      "activations/layer15_attention_weight_min": -28.759294509887695,
      "activations/layer16_attention_weight_max": 32.79121398925781,
      "activations/layer16_attention_weight_min": -32.411895751953125,
      "activations/layer17_attention_weight_max": 50.1639289855957,
      "activations/layer17_attention_weight_min": -50.18178176879883,
      "activations/layer18_attention_weight_max": 47.434654235839844,
      "activations/layer18_attention_weight_min": -45.05401611328125,
      "activations/layer19_attention_weight_max": 25.85752296447754,
      "activations/layer19_attention_weight_min": -25.10922622680664,
      "activations/layer1_attention_weight_max": 15.32009220123291,
      "activations/layer1_attention_weight_min": -13.620096206665039,
      "activations/layer20_attention_weight_max": 25.526710510253906,
      "activations/layer20_attention_weight_min": -21.964826583862305,
      "activations/layer21_attention_weight_max": 39.64189147949219,
      "activations/layer21_attention_weight_min": -21.2784423828125,
      "activations/layer22_attention_weight_max": 27.571149826049805,
      "activations/layer22_attention_weight_min": -25.309255599975586,
      "activations/layer23_attention_weight_max": 34.9618034362793,
      "activations/layer23_attention_weight_min": -23.315906524658203,
      "activations/layer2_attention_weight_max": 30.810260772705078,
      "activations/layer2_attention_weight_min": -32.640071868896484,
      "activations/layer3_attention_weight_max": 88.53961944580078,
      "activations/layer3_attention_weight_min": -96.65203857421875,
      "activations/layer4_attention_weight_max": 84.43836212158203,
      "activations/layer4_attention_weight_min": -83.49632263183594,
      "activations/layer5_attention_weight_max": 67.53417205810547,
      "activations/layer5_attention_weight_min": -71.2607421875,
      "activations/layer6_attention_weight_max": 48.894596099853516,
      "activations/layer6_attention_weight_min": -50.354434967041016,
      "activations/layer7_attention_weight_max": 72.49519348144531,
      "activations/layer7_attention_weight_min": -69.53622436523438,
      "activations/layer8_attention_weight_max": 45.87202453613281,
      "activations/layer8_attention_weight_min": -49.0571174621582,
      "activations/layer9_attention_weight_max": 43.834835052490234,
      "activations/layer9_attention_weight_min": -47.30427932739258,
      "epoch": 23.09,
      "learning_rate": 1.1231060606060604e-06,
      "loss": 2.6906,
      "step": 397350
    },
    {
      "activations/layer0_attention_weight_max": 15.147679328918457,
      "activations/layer0_attention_weight_min": -14.093681335449219,
      "activations/layer10_attention_weight_max": 34.617916107177734,
      "activations/layer10_attention_weight_min": -35.79682159423828,
      "activations/layer11_attention_weight_max": 32.74158477783203,
      "activations/layer11_attention_weight_min": -36.506370544433594,
      "activations/layer12_attention_weight_max": 18.128528594970703,
      "activations/layer12_attention_weight_min": -23.676355361938477,
      "activations/layer13_attention_weight_max": 31.987436294555664,
      "activations/layer13_attention_weight_min": -25.781169891357422,
      "activations/layer14_attention_weight_max": 33.237361907958984,
      "activations/layer14_attention_weight_min": -29.336423873901367,
      "activations/layer15_attention_weight_max": 31.34065055847168,
      "activations/layer15_attention_weight_min": -28.142152786254883,
      "activations/layer16_attention_weight_max": 32.34989547729492,
      "activations/layer16_attention_weight_min": -31.908662796020508,
      "activations/layer17_attention_weight_max": 48.29161071777344,
      "activations/layer17_attention_weight_min": -44.827850341796875,
      "activations/layer18_attention_weight_max": 44.719871520996094,
      "activations/layer18_attention_weight_min": -40.645668029785156,
      "activations/layer19_attention_weight_max": 22.925291061401367,
      "activations/layer19_attention_weight_min": -22.046672821044922,
      "activations/layer1_attention_weight_max": 14.779365539550781,
      "activations/layer1_attention_weight_min": -15.413297653198242,
      "activations/layer20_attention_weight_max": 20.59053611755371,
      "activations/layer20_attention_weight_min": -20.80262565612793,
      "activations/layer21_attention_weight_max": 29.669437408447266,
      "activations/layer21_attention_weight_min": -21.18320655822754,
      "activations/layer22_attention_weight_max": 30.022872924804688,
      "activations/layer22_attention_weight_min": -25.07147216796875,
      "activations/layer23_attention_weight_max": 35.41797637939453,
      "activations/layer23_attention_weight_min": -25.457561492919922,
      "activations/layer2_attention_weight_max": 32.720558166503906,
      "activations/layer2_attention_weight_min": -30.442733764648438,
      "activations/layer3_attention_weight_max": 93.35847473144531,
      "activations/layer3_attention_weight_min": -90.417724609375,
      "activations/layer4_attention_weight_max": 89.23247528076172,
      "activations/layer4_attention_weight_min": -84.23478698730469,
      "activations/layer5_attention_weight_max": 64.67427062988281,
      "activations/layer5_attention_weight_min": -69.89859771728516,
      "activations/layer6_attention_weight_max": 50.64218521118164,
      "activations/layer6_attention_weight_min": -49.18577575683594,
      "activations/layer7_attention_weight_max": 71.34156036376953,
      "activations/layer7_attention_weight_min": -67.77629852294922,
      "activations/layer8_attention_weight_max": 47.429832458496094,
      "activations/layer8_attention_weight_min": -49.5935173034668,
      "activations/layer9_attention_weight_max": 42.13753128051758,
      "activations/layer9_attention_weight_min": -45.9215202331543,
      "epoch": 23.09,
      "learning_rate": 1.1041666666666665e-06,
      "loss": 2.7072,
      "step": 397400
    },
    {
      "activations/layer0_attention_weight_max": 14.736617088317871,
      "activations/layer0_attention_weight_min": -14.050875663757324,
      "activations/layer10_attention_weight_max": 35.203277587890625,
      "activations/layer10_attention_weight_min": -35.015541076660156,
      "activations/layer11_attention_weight_max": 33.42683792114258,
      "activations/layer11_attention_weight_min": -35.745906829833984,
      "activations/layer12_attention_weight_max": 18.829675674438477,
      "activations/layer12_attention_weight_min": -25.26868438720703,
      "activations/layer13_attention_weight_max": 33.928611755371094,
      "activations/layer13_attention_weight_min": -26.67323112487793,
      "activations/layer14_attention_weight_max": 36.12275695800781,
      "activations/layer14_attention_weight_min": -31.28983497619629,
      "activations/layer15_attention_weight_max": 32.729862213134766,
      "activations/layer15_attention_weight_min": -29.948699951171875,
      "activations/layer16_attention_weight_max": 35.5371208190918,
      "activations/layer16_attention_weight_min": -32.13227462768555,
      "activations/layer17_attention_weight_max": 51.19571304321289,
      "activations/layer17_attention_weight_min": -42.85352325439453,
      "activations/layer18_attention_weight_max": 48.95808792114258,
      "activations/layer18_attention_weight_min": -38.3591423034668,
      "activations/layer19_attention_weight_max": 23.54515266418457,
      "activations/layer19_attention_weight_min": -23.2576847076416,
      "activations/layer1_attention_weight_max": 15.178372383117676,
      "activations/layer1_attention_weight_min": -14.279956817626953,
      "activations/layer20_attention_weight_max": 21.781259536743164,
      "activations/layer20_attention_weight_min": -21.134050369262695,
      "activations/layer21_attention_weight_max": 32.3076286315918,
      "activations/layer21_attention_weight_min": -19.40275001525879,
      "activations/layer22_attention_weight_max": 27.605487823486328,
      "activations/layer22_attention_weight_min": -24.470947265625,
      "activations/layer23_attention_weight_max": 34.58067321777344,
      "activations/layer23_attention_weight_min": -23.98235321044922,
      "activations/layer2_attention_weight_max": 32.311180114746094,
      "activations/layer2_attention_weight_min": -31.06446647644043,
      "activations/layer3_attention_weight_max": 94.03807067871094,
      "activations/layer3_attention_weight_min": -89.17704772949219,
      "activations/layer4_attention_weight_max": 89.08855438232422,
      "activations/layer4_attention_weight_min": -85.59222412109375,
      "activations/layer5_attention_weight_max": 66.71089172363281,
      "activations/layer5_attention_weight_min": -71.29759216308594,
      "activations/layer6_attention_weight_max": 50.0303840637207,
      "activations/layer6_attention_weight_min": -51.9698371887207,
      "activations/layer7_attention_weight_max": 70.52146911621094,
      "activations/layer7_attention_weight_min": -70.06045532226562,
      "activations/layer8_attention_weight_max": 45.98948287963867,
      "activations/layer8_attention_weight_min": -50.071041107177734,
      "activations/layer9_attention_weight_max": 43.111106872558594,
      "activations/layer9_attention_weight_min": -46.86894226074219,
      "epoch": 23.09,
      "learning_rate": 1.0852272727272727e-06,
      "loss": 2.6886,
      "step": 397450
    },
    {
      "activations/layer0_attention_weight_max": 16.0817928314209,
      "activations/layer0_attention_weight_min": -14.132501602172852,
      "activations/layer10_attention_weight_max": 35.62483596801758,
      "activations/layer10_attention_weight_min": -34.549522399902344,
      "activations/layer11_attention_weight_max": 32.74595642089844,
      "activations/layer11_attention_weight_min": -34.68844223022461,
      "activations/layer12_attention_weight_max": 17.71315574645996,
      "activations/layer12_attention_weight_min": -31.175878524780273,
      "activations/layer13_attention_weight_max": 29.408418655395508,
      "activations/layer13_attention_weight_min": -26.472042083740234,
      "activations/layer14_attention_weight_max": 32.55302429199219,
      "activations/layer14_attention_weight_min": -30.970136642456055,
      "activations/layer15_attention_weight_max": 32.054649353027344,
      "activations/layer15_attention_weight_min": -29.728158950805664,
      "activations/layer16_attention_weight_max": 31.590173721313477,
      "activations/layer16_attention_weight_min": -30.152917861938477,
      "activations/layer17_attention_weight_max": 46.32854080200195,
      "activations/layer17_attention_weight_min": -41.909210205078125,
      "activations/layer18_attention_weight_max": 43.019935607910156,
      "activations/layer18_attention_weight_min": -39.55901336669922,
      "activations/layer19_attention_weight_max": 22.853307723999023,
      "activations/layer19_attention_weight_min": -22.37796974182129,
      "activations/layer1_attention_weight_max": 14.946053504943848,
      "activations/layer1_attention_weight_min": -14.80781078338623,
      "activations/layer20_attention_weight_max": 22.079816818237305,
      "activations/layer20_attention_weight_min": -24.697771072387695,
      "activations/layer21_attention_weight_max": 36.23530960083008,
      "activations/layer21_attention_weight_min": -22.387704849243164,
      "activations/layer22_attention_weight_max": 26.99116325378418,
      "activations/layer22_attention_weight_min": -25.808090209960938,
      "activations/layer23_attention_weight_max": 34.32463836669922,
      "activations/layer23_attention_weight_min": -23.25838851928711,
      "activations/layer2_attention_weight_max": 31.53767967224121,
      "activations/layer2_attention_weight_min": -30.53635025024414,
      "activations/layer3_attention_weight_max": 90.29023742675781,
      "activations/layer3_attention_weight_min": -89.26476287841797,
      "activations/layer4_attention_weight_max": 87.87503814697266,
      "activations/layer4_attention_weight_min": -88.11206817626953,
      "activations/layer5_attention_weight_max": 60.242332458496094,
      "activations/layer5_attention_weight_min": -70.07022094726562,
      "activations/layer6_attention_weight_max": 48.96809387207031,
      "activations/layer6_attention_weight_min": -49.78314971923828,
      "activations/layer7_attention_weight_max": 70.44661712646484,
      "activations/layer7_attention_weight_min": -66.49647521972656,
      "activations/layer8_attention_weight_max": 51.14381790161133,
      "activations/layer8_attention_weight_min": -51.63885498046875,
      "activations/layer9_attention_weight_max": 43.04792404174805,
      "activations/layer9_attention_weight_min": -43.89612579345703,
      "epoch": 23.1,
      "learning_rate": 1.0662878787878786e-06,
      "loss": 2.6933,
      "step": 397500
    },
    {
      "activations/layer0_attention_weight_max": 16.134803771972656,
      "activations/layer0_attention_weight_min": -14.22299575805664,
      "activations/layer10_attention_weight_max": 34.428184509277344,
      "activations/layer10_attention_weight_min": -34.20289993286133,
      "activations/layer11_attention_weight_max": 31.519935607910156,
      "activations/layer11_attention_weight_min": -35.30157470703125,
      "activations/layer12_attention_weight_max": 19.649169921875,
      "activations/layer12_attention_weight_min": -25.664962768554688,
      "activations/layer13_attention_weight_max": 34.097564697265625,
      "activations/layer13_attention_weight_min": -26.63835906982422,
      "activations/layer14_attention_weight_max": 34.183712005615234,
      "activations/layer14_attention_weight_min": -30.741596221923828,
      "activations/layer15_attention_weight_max": 34.88002014160156,
      "activations/layer15_attention_weight_min": -31.370325088500977,
      "activations/layer16_attention_weight_max": 35.129512786865234,
      "activations/layer16_attention_weight_min": -32.92137145996094,
      "activations/layer17_attention_weight_max": 51.85957336425781,
      "activations/layer17_attention_weight_min": -44.7717170715332,
      "activations/layer18_attention_weight_max": 46.75735092163086,
      "activations/layer18_attention_weight_min": -40.83216094970703,
      "activations/layer19_attention_weight_max": 23.721492767333984,
      "activations/layer19_attention_weight_min": -22.814367294311523,
      "activations/layer1_attention_weight_max": 15.648001670837402,
      "activations/layer1_attention_weight_min": -14.26069164276123,
      "activations/layer20_attention_weight_max": 20.99932289123535,
      "activations/layer20_attention_weight_min": -22.714149475097656,
      "activations/layer21_attention_weight_max": 32.40707778930664,
      "activations/layer21_attention_weight_min": -19.267742156982422,
      "activations/layer22_attention_weight_max": 27.73407554626465,
      "activations/layer22_attention_weight_min": -24.208202362060547,
      "activations/layer23_attention_weight_max": 33.9427490234375,
      "activations/layer23_attention_weight_min": -22.99228286743164,
      "activations/layer2_attention_weight_max": 31.636154174804688,
      "activations/layer2_attention_weight_min": -29.687795639038086,
      "activations/layer3_attention_weight_max": 90.16049194335938,
      "activations/layer3_attention_weight_min": -89.72340393066406,
      "activations/layer4_attention_weight_max": 84.99249267578125,
      "activations/layer4_attention_weight_min": -86.58525848388672,
      "activations/layer5_attention_weight_max": 62.31706237792969,
      "activations/layer5_attention_weight_min": -75.31804656982422,
      "activations/layer6_attention_weight_max": 48.51398849487305,
      "activations/layer6_attention_weight_min": -51.15734100341797,
      "activations/layer7_attention_weight_max": 70.99398040771484,
      "activations/layer7_attention_weight_min": -70.5276107788086,
      "activations/layer8_attention_weight_max": 45.05895233154297,
      "activations/layer8_attention_weight_min": -48.216041564941406,
      "activations/layer9_attention_weight_max": 43.98687744140625,
      "activations/layer9_attention_weight_min": -45.26742172241211,
      "epoch": 23.1,
      "learning_rate": 1.0477272727272726e-06,
      "loss": 2.7036,
      "step": 397550
    },
    {
      "activations/layer0_attention_weight_max": 16.084354400634766,
      "activations/layer0_attention_weight_min": -14.610611915588379,
      "activations/layer10_attention_weight_max": 34.8599853515625,
      "activations/layer10_attention_weight_min": -38.23371887207031,
      "activations/layer11_attention_weight_max": 33.2432861328125,
      "activations/layer11_attention_weight_min": -34.17455291748047,
      "activations/layer12_attention_weight_max": 18.541996002197266,
      "activations/layer12_attention_weight_min": -23.907608032226562,
      "activations/layer13_attention_weight_max": 34.19328308105469,
      "activations/layer13_attention_weight_min": -26.05335807800293,
      "activations/layer14_attention_weight_max": 30.988712310791016,
      "activations/layer14_attention_weight_min": -28.36322784423828,
      "activations/layer15_attention_weight_max": 29.225751876831055,
      "activations/layer15_attention_weight_min": -28.9497127532959,
      "activations/layer16_attention_weight_max": 31.497806549072266,
      "activations/layer16_attention_weight_min": -30.51741600036621,
      "activations/layer17_attention_weight_max": 49.0482177734375,
      "activations/layer17_attention_weight_min": -42.10953903198242,
      "activations/layer18_attention_weight_max": 42.400047302246094,
      "activations/layer18_attention_weight_min": -39.29397964477539,
      "activations/layer19_attention_weight_max": 22.99012565612793,
      "activations/layer19_attention_weight_min": -25.559309005737305,
      "activations/layer1_attention_weight_max": 15.315552711486816,
      "activations/layer1_attention_weight_min": -14.152726173400879,
      "activations/layer20_attention_weight_max": 20.974695205688477,
      "activations/layer20_attention_weight_min": -22.732004165649414,
      "activations/layer21_attention_weight_max": 30.946062088012695,
      "activations/layer21_attention_weight_min": -20.821016311645508,
      "activations/layer22_attention_weight_max": 25.799972534179688,
      "activations/layer22_attention_weight_min": -24.05791473388672,
      "activations/layer23_attention_weight_max": 36.68214797973633,
      "activations/layer23_attention_weight_min": -25.05017852783203,
      "activations/layer2_attention_weight_max": 31.44680404663086,
      "activations/layer2_attention_weight_min": -30.9549560546875,
      "activations/layer3_attention_weight_max": 88.9920654296875,
      "activations/layer3_attention_weight_min": -91.33480072021484,
      "activations/layer4_attention_weight_max": 83.24005889892578,
      "activations/layer4_attention_weight_min": -84.92239379882812,
      "activations/layer5_attention_weight_max": 62.11766052246094,
      "activations/layer5_attention_weight_min": -69.92604064941406,
      "activations/layer6_attention_weight_max": 49.8097038269043,
      "activations/layer6_attention_weight_min": -49.910160064697266,
      "activations/layer7_attention_weight_max": 69.6191177368164,
      "activations/layer7_attention_weight_min": -70.74195098876953,
      "activations/layer8_attention_weight_max": 45.855194091796875,
      "activations/layer8_attention_weight_min": -50.174991607666016,
      "activations/layer9_attention_weight_max": 45.33297348022461,
      "activations/layer9_attention_weight_min": -45.53257369995117,
      "epoch": 23.1,
      "learning_rate": 1.0287878787878786e-06,
      "loss": 2.706,
      "step": 397600
    },
    {
      "activations/layer0_attention_weight_max": 15.19953727722168,
      "activations/layer0_attention_weight_min": -14.277338981628418,
      "activations/layer10_attention_weight_max": 37.60123062133789,
      "activations/layer10_attention_weight_min": -36.837890625,
      "activations/layer11_attention_weight_max": 33.506446838378906,
      "activations/layer11_attention_weight_min": -34.44648361206055,
      "activations/layer12_attention_weight_max": 18.868558883666992,
      "activations/layer12_attention_weight_min": -27.319429397583008,
      "activations/layer13_attention_weight_max": 32.10890579223633,
      "activations/layer13_attention_weight_min": -29.201553344726562,
      "activations/layer14_attention_weight_max": 34.767730712890625,
      "activations/layer14_attention_weight_min": -29.104658126831055,
      "activations/layer15_attention_weight_max": 33.401161193847656,
      "activations/layer15_attention_weight_min": -29.50675392150879,
      "activations/layer16_attention_weight_max": 33.472389221191406,
      "activations/layer16_attention_weight_min": -34.03901290893555,
      "activations/layer17_attention_weight_max": 50.76145553588867,
      "activations/layer17_attention_weight_min": -43.11439895629883,
      "activations/layer18_attention_weight_max": 49.25133514404297,
      "activations/layer18_attention_weight_min": -39.38866424560547,
      "activations/layer19_attention_weight_max": 24.920686721801758,
      "activations/layer19_attention_weight_min": -23.461748123168945,
      "activations/layer1_attention_weight_max": 15.10606575012207,
      "activations/layer1_attention_weight_min": -15.87773609161377,
      "activations/layer20_attention_weight_max": 23.42577362060547,
      "activations/layer20_attention_weight_min": -22.454771041870117,
      "activations/layer21_attention_weight_max": 36.5390510559082,
      "activations/layer21_attention_weight_min": -23.753665924072266,
      "activations/layer22_attention_weight_max": 30.188474655151367,
      "activations/layer22_attention_weight_min": -25.426496505737305,
      "activations/layer23_attention_weight_max": 39.11322021484375,
      "activations/layer23_attention_weight_min": -23.873523712158203,
      "activations/layer2_attention_weight_max": 32.64700698852539,
      "activations/layer2_attention_weight_min": -31.271183013916016,
      "activations/layer3_attention_weight_max": 94.52600860595703,
      "activations/layer3_attention_weight_min": -94.88481140136719,
      "activations/layer4_attention_weight_max": 88.06819152832031,
      "activations/layer4_attention_weight_min": -86.60955047607422,
      "activations/layer5_attention_weight_max": 60.40570068359375,
      "activations/layer5_attention_weight_min": -72.50743103027344,
      "activations/layer6_attention_weight_max": 51.37797546386719,
      "activations/layer6_attention_weight_min": -51.82395935058594,
      "activations/layer7_attention_weight_max": 66.79955291748047,
      "activations/layer7_attention_weight_min": -69.34981536865234,
      "activations/layer8_attention_weight_max": 48.67830276489258,
      "activations/layer8_attention_weight_min": -54.91916275024414,
      "activations/layer9_attention_weight_max": 45.936866760253906,
      "activations/layer9_attention_weight_min": -46.764366149902344,
      "epoch": 23.11,
      "learning_rate": 1.0098484848484847e-06,
      "loss": 2.6755,
      "step": 397650
    },
    {
      "activations/layer0_attention_weight_max": 15.522133827209473,
      "activations/layer0_attention_weight_min": -14.371437072753906,
      "activations/layer10_attention_weight_max": 37.89922332763672,
      "activations/layer10_attention_weight_min": -38.587921142578125,
      "activations/layer11_attention_weight_max": 33.442230224609375,
      "activations/layer11_attention_weight_min": -35.81277084350586,
      "activations/layer12_attention_weight_max": 19.163745880126953,
      "activations/layer12_attention_weight_min": -28.1023006439209,
      "activations/layer13_attention_weight_max": 32.51192855834961,
      "activations/layer13_attention_weight_min": -27.463563919067383,
      "activations/layer14_attention_weight_max": 35.83755874633789,
      "activations/layer14_attention_weight_min": -28.903114318847656,
      "activations/layer15_attention_weight_max": 33.09822082519531,
      "activations/layer15_attention_weight_min": -31.73560905456543,
      "activations/layer16_attention_weight_max": 34.0998649597168,
      "activations/layer16_attention_weight_min": -33.51449203491211,
      "activations/layer17_attention_weight_max": 53.61653137207031,
      "activations/layer17_attention_weight_min": -45.934696197509766,
      "activations/layer18_attention_weight_max": 49.43330001831055,
      "activations/layer18_attention_weight_min": -41.969112396240234,
      "activations/layer19_attention_weight_max": 25.361948013305664,
      "activations/layer19_attention_weight_min": -23.925809860229492,
      "activations/layer1_attention_weight_max": 15.35615348815918,
      "activations/layer1_attention_weight_min": -13.7516508102417,
      "activations/layer20_attention_weight_max": 22.11923599243164,
      "activations/layer20_attention_weight_min": -25.113279342651367,
      "activations/layer21_attention_weight_max": 37.197731018066406,
      "activations/layer21_attention_weight_min": -24.10524559020996,
      "activations/layer22_attention_weight_max": 27.866113662719727,
      "activations/layer22_attention_weight_min": -25.1842041015625,
      "activations/layer23_attention_weight_max": 34.827430725097656,
      "activations/layer23_attention_weight_min": -25.950498580932617,
      "activations/layer2_attention_weight_max": 33.243408203125,
      "activations/layer2_attention_weight_min": -30.61276626586914,
      "activations/layer3_attention_weight_max": 91.13726806640625,
      "activations/layer3_attention_weight_min": -93.69261169433594,
      "activations/layer4_attention_weight_max": 87.0535659790039,
      "activations/layer4_attention_weight_min": -89.92414093017578,
      "activations/layer5_attention_weight_max": 63.02169418334961,
      "activations/layer5_attention_weight_min": -75.02647399902344,
      "activations/layer6_attention_weight_max": 50.045169830322266,
      "activations/layer6_attention_weight_min": -49.22696304321289,
      "activations/layer7_attention_weight_max": 71.2972183227539,
      "activations/layer7_attention_weight_min": -70.1854476928711,
      "activations/layer8_attention_weight_max": 48.803348541259766,
      "activations/layer8_attention_weight_min": -51.619319915771484,
      "activations/layer9_attention_weight_max": 43.52503967285156,
      "activations/layer9_attention_weight_min": -47.07508850097656,
      "epoch": 23.11,
      "learning_rate": 9.909090909090909e-07,
      "loss": 2.6914,
      "step": 397700
    },
    {
      "activations/layer0_attention_weight_max": 15.87226390838623,
      "activations/layer0_attention_weight_min": -14.230666160583496,
      "activations/layer10_attention_weight_max": 40.36030578613281,
      "activations/layer10_attention_weight_min": -40.95246887207031,
      "activations/layer11_attention_weight_max": 38.866302490234375,
      "activations/layer11_attention_weight_min": -37.239192962646484,
      "activations/layer12_attention_weight_max": 21.495126724243164,
      "activations/layer12_attention_weight_min": -22.752660751342773,
      "activations/layer13_attention_weight_max": 38.04160690307617,
      "activations/layer13_attention_weight_min": -29.530128479003906,
      "activations/layer14_attention_weight_max": 38.51969528198242,
      "activations/layer14_attention_weight_min": -32.24605178833008,
      "activations/layer15_attention_weight_max": 34.947383880615234,
      "activations/layer15_attention_weight_min": -33.115482330322266,
      "activations/layer16_attention_weight_max": 36.87646484375,
      "activations/layer16_attention_weight_min": -35.80261993408203,
      "activations/layer17_attention_weight_max": 54.14345169067383,
      "activations/layer17_attention_weight_min": -49.15068817138672,
      "activations/layer18_attention_weight_max": 50.82610321044922,
      "activations/layer18_attention_weight_min": -47.109230041503906,
      "activations/layer19_attention_weight_max": 29.324865341186523,
      "activations/layer19_attention_weight_min": -26.25752067565918,
      "activations/layer1_attention_weight_max": 14.860241889953613,
      "activations/layer1_attention_weight_min": -15.681639671325684,
      "activations/layer20_attention_weight_max": 25.547367095947266,
      "activations/layer20_attention_weight_min": -23.593461990356445,
      "activations/layer21_attention_weight_max": 39.42094039916992,
      "activations/layer21_attention_weight_min": -22.814908981323242,
      "activations/layer22_attention_weight_max": 31.477035522460938,
      "activations/layer22_attention_weight_min": -28.147329330444336,
      "activations/layer23_attention_weight_max": 39.10631561279297,
      "activations/layer23_attention_weight_min": -26.001087188720703,
      "activations/layer2_attention_weight_max": 33.35205078125,
      "activations/layer2_attention_weight_min": -32.182090759277344,
      "activations/layer3_attention_weight_max": 96.72232055664062,
      "activations/layer3_attention_weight_min": -95.20635986328125,
      "activations/layer4_attention_weight_max": 92.63418579101562,
      "activations/layer4_attention_weight_min": -91.96604919433594,
      "activations/layer5_attention_weight_max": 67.05790710449219,
      "activations/layer5_attention_weight_min": -70.18449401855469,
      "activations/layer6_attention_weight_max": 51.53291702270508,
      "activations/layer6_attention_weight_min": -52.18279266357422,
      "activations/layer7_attention_weight_max": 74.60892486572266,
      "activations/layer7_attention_weight_min": -72.66949462890625,
      "activations/layer8_attention_weight_max": 52.46821212768555,
      "activations/layer8_attention_weight_min": -52.66426086425781,
      "activations/layer9_attention_weight_max": 48.02436828613281,
      "activations/layer9_attention_weight_min": -50.886329650878906,
      "epoch": 23.11,
      "learning_rate": 9.719696969696968e-07,
      "loss": 2.6956,
      "step": 397750
    },
    {
      "activations/layer0_attention_weight_max": 15.701010704040527,
      "activations/layer0_attention_weight_min": -14.515928268432617,
      "activations/layer10_attention_weight_max": 39.224876403808594,
      "activations/layer10_attention_weight_min": -37.98363494873047,
      "activations/layer11_attention_weight_max": 37.814517974853516,
      "activations/layer11_attention_weight_min": -37.539451599121094,
      "activations/layer12_attention_weight_max": 19.84250831604004,
      "activations/layer12_attention_weight_min": -24.954782485961914,
      "activations/layer13_attention_weight_max": 37.361934661865234,
      "activations/layer13_attention_weight_min": -26.61966896057129,
      "activations/layer14_attention_weight_max": 38.26463317871094,
      "activations/layer14_attention_weight_min": -29.881601333618164,
      "activations/layer15_attention_weight_max": 35.975276947021484,
      "activations/layer15_attention_weight_min": -30.934864044189453,
      "activations/layer16_attention_weight_max": 35.70521545410156,
      "activations/layer16_attention_weight_min": -33.98649978637695,
      "activations/layer17_attention_weight_max": 52.74187088012695,
      "activations/layer17_attention_weight_min": -47.71108627319336,
      "activations/layer18_attention_weight_max": 48.39741516113281,
      "activations/layer18_attention_weight_min": -41.12017822265625,
      "activations/layer19_attention_weight_max": 24.706876754760742,
      "activations/layer19_attention_weight_min": -25.630014419555664,
      "activations/layer1_attention_weight_max": 15.813692092895508,
      "activations/layer1_attention_weight_min": -15.046981811523438,
      "activations/layer20_attention_weight_max": 24.36236000061035,
      "activations/layer20_attention_weight_min": -21.820371627807617,
      "activations/layer21_attention_weight_max": 38.452171325683594,
      "activations/layer21_attention_weight_min": -21.516199111938477,
      "activations/layer22_attention_weight_max": 30.023542404174805,
      "activations/layer22_attention_weight_min": -26.90772819519043,
      "activations/layer23_attention_weight_max": 34.88910675048828,
      "activations/layer23_attention_weight_min": -26.569103240966797,
      "activations/layer2_attention_weight_max": 32.12449645996094,
      "activations/layer2_attention_weight_min": -30.7587890625,
      "activations/layer3_attention_weight_max": 94.58452606201172,
      "activations/layer3_attention_weight_min": -90.9208984375,
      "activations/layer4_attention_weight_max": 92.20044708251953,
      "activations/layer4_attention_weight_min": -92.53154754638672,
      "activations/layer5_attention_weight_max": 64.10527038574219,
      "activations/layer5_attention_weight_min": -74.36064147949219,
      "activations/layer6_attention_weight_max": 52.882198333740234,
      "activations/layer6_attention_weight_min": -53.40929412841797,
      "activations/layer7_attention_weight_max": 69.82118225097656,
      "activations/layer7_attention_weight_min": -73.94742584228516,
      "activations/layer8_attention_weight_max": 51.71156692504883,
      "activations/layer8_attention_weight_min": -52.332557678222656,
      "activations/layer9_attention_weight_max": 46.233890533447266,
      "activations/layer9_attention_weight_min": -48.26044845581055,
      "epoch": 23.11,
      "learning_rate": 9.53030303030303e-07,
      "loss": 2.6862,
      "step": 397800
    },
    {
      "activations/layer0_attention_weight_max": 15.69503116607666,
      "activations/layer0_attention_weight_min": -14.000035285949707,
      "activations/layer10_attention_weight_max": 37.225982666015625,
      "activations/layer10_attention_weight_min": -37.950439453125,
      "activations/layer11_attention_weight_max": 33.484832763671875,
      "activations/layer11_attention_weight_min": -36.696590423583984,
      "activations/layer12_attention_weight_max": 19.673992156982422,
      "activations/layer12_attention_weight_min": -25.968477249145508,
      "activations/layer13_attention_weight_max": 34.18262481689453,
      "activations/layer13_attention_weight_min": -29.69783592224121,
      "activations/layer14_attention_weight_max": 38.6071662902832,
      "activations/layer14_attention_weight_min": -29.262754440307617,
      "activations/layer15_attention_weight_max": 38.5869255065918,
      "activations/layer15_attention_weight_min": -29.697874069213867,
      "activations/layer16_attention_weight_max": 41.97694778442383,
      "activations/layer16_attention_weight_min": -37.57670593261719,
      "activations/layer17_attention_weight_max": 62.5778694152832,
      "activations/layer17_attention_weight_min": -52.01008987426758,
      "activations/layer18_attention_weight_max": 52.46879959106445,
      "activations/layer18_attention_weight_min": -40.77872085571289,
      "activations/layer19_attention_weight_max": 25.617856979370117,
      "activations/layer19_attention_weight_min": -22.357494354248047,
      "activations/layer1_attention_weight_max": 14.665480613708496,
      "activations/layer1_attention_weight_min": -14.387158393859863,
      "activations/layer20_attention_weight_max": 25.83159828186035,
      "activations/layer20_attention_weight_min": -20.945478439331055,
      "activations/layer21_attention_weight_max": 32.05640411376953,
      "activations/layer21_attention_weight_min": -21.711692810058594,
      "activations/layer22_attention_weight_max": 28.28847312927246,
      "activations/layer22_attention_weight_min": -24.83675765991211,
      "activations/layer23_attention_weight_max": 38.574886322021484,
      "activations/layer23_attention_weight_min": -23.27281951904297,
      "activations/layer2_attention_weight_max": 31.513748168945312,
      "activations/layer2_attention_weight_min": -31.340381622314453,
      "activations/layer3_attention_weight_max": 91.95117950439453,
      "activations/layer3_attention_weight_min": -94.08930969238281,
      "activations/layer4_attention_weight_max": 87.96208953857422,
      "activations/layer4_attention_weight_min": -88.09041595458984,
      "activations/layer5_attention_weight_max": 60.754878997802734,
      "activations/layer5_attention_weight_min": -74.16220092773438,
      "activations/layer6_attention_weight_max": 50.100807189941406,
      "activations/layer6_attention_weight_min": -53.34503936767578,
      "activations/layer7_attention_weight_max": 69.24597930908203,
      "activations/layer7_attention_weight_min": -70.45330810546875,
      "activations/layer8_attention_weight_max": 47.72124099731445,
      "activations/layer8_attention_weight_min": -53.28475570678711,
      "activations/layer9_attention_weight_max": 44.318965911865234,
      "activations/layer9_attention_weight_min": -46.28656768798828,
      "epoch": 23.12,
      "learning_rate": 9.34469696969697e-07,
      "loss": 2.6957,
      "step": 397850
    },
    {
      "activations/layer0_attention_weight_max": 15.630945205688477,
      "activations/layer0_attention_weight_min": -14.09594440460205,
      "activations/layer10_attention_weight_max": 40.56116485595703,
      "activations/layer10_attention_weight_min": -39.210113525390625,
      "activations/layer11_attention_weight_max": 37.86341094970703,
      "activations/layer11_attention_weight_min": -36.22985076904297,
      "activations/layer12_attention_weight_max": 19.65660858154297,
      "activations/layer12_attention_weight_min": -23.61704444885254,
      "activations/layer13_attention_weight_max": 38.48415756225586,
      "activations/layer13_attention_weight_min": -34.928375244140625,
      "activations/layer14_attention_weight_max": 36.425048828125,
      "activations/layer14_attention_weight_min": -33.00883483886719,
      "activations/layer15_attention_weight_max": 35.402626037597656,
      "activations/layer15_attention_weight_min": -34.32061004638672,
      "activations/layer16_attention_weight_max": 34.34013366699219,
      "activations/layer16_attention_weight_min": -34.148193359375,
      "activations/layer17_attention_weight_max": 55.41448974609375,
      "activations/layer17_attention_weight_min": -48.61909866333008,
      "activations/layer18_attention_weight_max": 52.38361740112305,
      "activations/layer18_attention_weight_min": -43.85862731933594,
      "activations/layer19_attention_weight_max": 25.29865074157715,
      "activations/layer19_attention_weight_min": -22.598539352416992,
      "activations/layer1_attention_weight_max": 15.800505638122559,
      "activations/layer1_attention_weight_min": -15.03515911102295,
      "activations/layer20_attention_weight_max": 23.318395614624023,
      "activations/layer20_attention_weight_min": -22.61794090270996,
      "activations/layer21_attention_weight_max": 40.06183624267578,
      "activations/layer21_attention_weight_min": -23.749309539794922,
      "activations/layer22_attention_weight_max": 30.734025955200195,
      "activations/layer22_attention_weight_min": -26.87228012084961,
      "activations/layer23_attention_weight_max": 37.09675598144531,
      "activations/layer23_attention_weight_min": -26.420452117919922,
      "activations/layer2_attention_weight_max": 33.80015563964844,
      "activations/layer2_attention_weight_min": -32.229347229003906,
      "activations/layer3_attention_weight_max": 95.22413635253906,
      "activations/layer3_attention_weight_min": -95.6825942993164,
      "activations/layer4_attention_weight_max": 88.6488037109375,
      "activations/layer4_attention_weight_min": -88.64388275146484,
      "activations/layer5_attention_weight_max": 63.05991744995117,
      "activations/layer5_attention_weight_min": -71.5317153930664,
      "activations/layer6_attention_weight_max": 50.63639831542969,
      "activations/layer6_attention_weight_min": -51.27613067626953,
      "activations/layer7_attention_weight_max": 74.37655639648438,
      "activations/layer7_attention_weight_min": -70.29444885253906,
      "activations/layer8_attention_weight_max": 54.62955856323242,
      "activations/layer8_attention_weight_min": -54.3005256652832,
      "activations/layer9_attention_weight_max": 52.624691009521484,
      "activations/layer9_attention_weight_min": -48.201148986816406,
      "epoch": 23.12,
      "learning_rate": 9.155303030303029e-07,
      "loss": 2.7073,
      "step": 397900
    },
    {
      "activations/layer0_attention_weight_max": 15.499349594116211,
      "activations/layer0_attention_weight_min": -14.086467742919922,
      "activations/layer10_attention_weight_max": 38.53700637817383,
      "activations/layer10_attention_weight_min": -36.73588943481445,
      "activations/layer11_attention_weight_max": 37.11225128173828,
      "activations/layer11_attention_weight_min": -37.840476989746094,
      "activations/layer12_attention_weight_max": 19.753870010375977,
      "activations/layer12_attention_weight_min": -23.508636474609375,
      "activations/layer13_attention_weight_max": 40.193206787109375,
      "activations/layer13_attention_weight_min": -27.265748977661133,
      "activations/layer14_attention_weight_max": 39.81477737426758,
      "activations/layer14_attention_weight_min": -29.910539627075195,
      "activations/layer15_attention_weight_max": 34.73347854614258,
      "activations/layer15_attention_weight_min": -30.300437927246094,
      "activations/layer16_attention_weight_max": 34.40057373046875,
      "activations/layer16_attention_weight_min": -32.32506561279297,
      "activations/layer17_attention_weight_max": 50.253868103027344,
      "activations/layer17_attention_weight_min": -43.73138427734375,
      "activations/layer18_attention_weight_max": 48.93375015258789,
      "activations/layer18_attention_weight_min": -41.63272476196289,
      "activations/layer19_attention_weight_max": 25.9404354095459,
      "activations/layer19_attention_weight_min": -22.644386291503906,
      "activations/layer1_attention_weight_max": 14.788490295410156,
      "activations/layer1_attention_weight_min": -14.29191780090332,
      "activations/layer20_attention_weight_max": 24.00957679748535,
      "activations/layer20_attention_weight_min": -20.752147674560547,
      "activations/layer21_attention_weight_max": 36.94902801513672,
      "activations/layer21_attention_weight_min": -21.856101989746094,
      "activations/layer22_attention_weight_max": 30.05329132080078,
      "activations/layer22_attention_weight_min": -24.27211570739746,
      "activations/layer23_attention_weight_max": 39.829010009765625,
      "activations/layer23_attention_weight_min": -24.958614349365234,
      "activations/layer2_attention_weight_max": 32.03535079956055,
      "activations/layer2_attention_weight_min": -30.550682067871094,
      "activations/layer3_attention_weight_max": 93.09344482421875,
      "activations/layer3_attention_weight_min": -92.1262435913086,
      "activations/layer4_attention_weight_max": 88.38605499267578,
      "activations/layer4_attention_weight_min": -88.1091079711914,
      "activations/layer5_attention_weight_max": 66.385498046875,
      "activations/layer5_attention_weight_min": -71.51166534423828,
      "activations/layer6_attention_weight_max": 50.69308853149414,
      "activations/layer6_attention_weight_min": -53.25699234008789,
      "activations/layer7_attention_weight_max": 75.49842071533203,
      "activations/layer7_attention_weight_min": -71.62124633789062,
      "activations/layer8_attention_weight_max": 50.30809783935547,
      "activations/layer8_attention_weight_min": -51.93521499633789,
      "activations/layer9_attention_weight_max": 46.70849609375,
      "activations/layer9_attention_weight_min": -49.07741928100586,
      "epoch": 23.12,
      "learning_rate": 8.96590909090909e-07,
      "loss": 2.6999,
      "step": 397950
    },
    {
      "activations/layer0_attention_weight_max": 16.701759338378906,
      "activations/layer0_attention_weight_min": -13.945310592651367,
      "activations/layer10_attention_weight_max": 37.23523712158203,
      "activations/layer10_attention_weight_min": -35.980194091796875,
      "activations/layer11_attention_weight_max": 32.60947799682617,
      "activations/layer11_attention_weight_min": -37.18171310424805,
      "activations/layer12_attention_weight_max": 19.642385482788086,
      "activations/layer12_attention_weight_min": -26.228403091430664,
      "activations/layer13_attention_weight_max": 35.511470794677734,
      "activations/layer13_attention_weight_min": -26.15569305419922,
      "activations/layer14_attention_weight_max": 36.53002166748047,
      "activations/layer14_attention_weight_min": -29.923229217529297,
      "activations/layer15_attention_weight_max": 33.64990997314453,
      "activations/layer15_attention_weight_min": -30.180496215820312,
      "activations/layer16_attention_weight_max": 35.950599670410156,
      "activations/layer16_attention_weight_min": -33.62493896484375,
      "activations/layer17_attention_weight_max": 53.93017578125,
      "activations/layer17_attention_weight_min": -46.87151336669922,
      "activations/layer18_attention_weight_max": 48.03069305419922,
      "activations/layer18_attention_weight_min": -41.14421844482422,
      "activations/layer19_attention_weight_max": 24.786603927612305,
      "activations/layer19_attention_weight_min": -23.061487197875977,
      "activations/layer1_attention_weight_max": 15.103631973266602,
      "activations/layer1_attention_weight_min": -15.25015640258789,
      "activations/layer20_attention_weight_max": 23.306913375854492,
      "activations/layer20_attention_weight_min": -22.585838317871094,
      "activations/layer21_attention_weight_max": 37.76512908935547,
      "activations/layer21_attention_weight_min": -23.219039916992188,
      "activations/layer22_attention_weight_max": 30.86168098449707,
      "activations/layer22_attention_weight_min": -24.113170623779297,
      "activations/layer23_attention_weight_max": 36.41462326049805,
      "activations/layer23_attention_weight_min": -23.84383201599121,
      "activations/layer2_attention_weight_max": 32.0196418762207,
      "activations/layer2_attention_weight_min": -30.824506759643555,
      "activations/layer3_attention_weight_max": 94.32240295410156,
      "activations/layer3_attention_weight_min": -91.28695678710938,
      "activations/layer4_attention_weight_max": 88.60843658447266,
      "activations/layer4_attention_weight_min": -86.34432220458984,
      "activations/layer5_attention_weight_max": 64.75624084472656,
      "activations/layer5_attention_weight_min": -73.99372863769531,
      "activations/layer6_attention_weight_max": 51.13529968261719,
      "activations/layer6_attention_weight_min": -50.99040222167969,
      "activations/layer7_attention_weight_max": 70.03899383544922,
      "activations/layer7_attention_weight_min": -68.4080810546875,
      "activations/layer8_attention_weight_max": 46.503997802734375,
      "activations/layer8_attention_weight_min": -52.1549186706543,
      "activations/layer9_attention_weight_max": 43.90939712524414,
      "activations/layer9_attention_weight_min": -47.81112289428711,
      "epoch": 23.13,
      "learning_rate": 8.77651515151515e-07,
      "loss": 2.7003,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.499,
      "eval_samples_per_second": 505.238,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.499,
      "eval_openwebtext_samples_per_second": 505.238,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 2.0248,
      "eval_wikitext_samples_per_second": 225.207,
      "step": 398000
    },
    {
      "epoch": 23.13,
      "eval_lambada_loss": 2.38671875,
      "eval_lambada_ppl": 10.877742727456024,
      "eval_lambada_runtime": 9.5643,
      "eval_lambada_samples_per_second": 509.081,
      "step": 398000
    },
    {
      "activations/layer0_attention_weight_max": 16.032514572143555,
      "activations/layer0_attention_weight_min": -14.364914894104004,
      "activations/layer10_attention_weight_max": 35.74665069580078,
      "activations/layer10_attention_weight_min": -35.878971099853516,
      "activations/layer11_attention_weight_max": 32.412872314453125,
      "activations/layer11_attention_weight_min": -35.42424392700195,
      "activations/layer12_attention_weight_max": 19.036998748779297,
      "activations/layer12_attention_weight_min": -25.395063400268555,
      "activations/layer13_attention_weight_max": 35.86636734008789,
      "activations/layer13_attention_weight_min": -26.993520736694336,
      "activations/layer14_attention_weight_max": 41.5761604309082,
      "activations/layer14_attention_weight_min": -28.96895408630371,
      "activations/layer15_attention_weight_max": 38.25252914428711,
      "activations/layer15_attention_weight_min": -30.519668579101562,
      "activations/layer16_attention_weight_max": 36.999732971191406,
      "activations/layer16_attention_weight_min": -33.93642807006836,
      "activations/layer17_attention_weight_max": 56.95659637451172,
      "activations/layer17_attention_weight_min": -46.60944747924805,
      "activations/layer18_attention_weight_max": 53.72604751586914,
      "activations/layer18_attention_weight_min": -43.62982940673828,
      "activations/layer19_attention_weight_max": 29.55326271057129,
      "activations/layer19_attention_weight_min": -26.41474723815918,
      "activations/layer1_attention_weight_max": 14.895363807678223,
      "activations/layer1_attention_weight_min": -14.71138858795166,
      "activations/layer20_attention_weight_max": 29.9000301361084,
      "activations/layer20_attention_weight_min": -23.260068893432617,
      "activations/layer21_attention_weight_max": 50.40585708618164,
      "activations/layer21_attention_weight_min": -24.82488441467285,
      "activations/layer22_attention_weight_max": 33.355140686035156,
      "activations/layer22_attention_weight_min": -24.53028106689453,
      "activations/layer23_attention_weight_max": 42.445037841796875,
      "activations/layer23_attention_weight_min": -25.899152755737305,
      "activations/layer2_attention_weight_max": 32.38880920410156,
      "activations/layer2_attention_weight_min": -30.600177764892578,
      "activations/layer3_attention_weight_max": 92.0614242553711,
      "activations/layer3_attention_weight_min": -92.59967803955078,
      "activations/layer4_attention_weight_max": 86.1755599975586,
      "activations/layer4_attention_weight_min": -87.34408569335938,
      "activations/layer5_attention_weight_max": 63.793235778808594,
      "activations/layer5_attention_weight_min": -71.87206268310547,
      "activations/layer6_attention_weight_max": 50.82471466064453,
      "activations/layer6_attention_weight_min": -51.01527404785156,
      "activations/layer7_attention_weight_max": 71.80789184570312,
      "activations/layer7_attention_weight_min": -71.4600601196289,
      "activations/layer8_attention_weight_max": 49.44654846191406,
      "activations/layer8_attention_weight_min": -50.71245193481445,
      "activations/layer9_attention_weight_max": 44.928829193115234,
      "activations/layer9_attention_weight_min": -46.65305709838867,
      "epoch": 23.13,
      "learning_rate": 8.587121212121211e-07,
      "loss": 2.6982,
      "step": 398050
    },
    {
      "activations/layer0_attention_weight_max": 15.705438613891602,
      "activations/layer0_attention_weight_min": -13.797757148742676,
      "activations/layer10_attention_weight_max": 43.87931442260742,
      "activations/layer10_attention_weight_min": -43.51315689086914,
      "activations/layer11_attention_weight_max": 42.64997482299805,
      "activations/layer11_attention_weight_min": -42.143680572509766,
      "activations/layer12_attention_weight_max": 18.234922409057617,
      "activations/layer12_attention_weight_min": -27.260602951049805,
      "activations/layer13_attention_weight_max": 35.386192321777344,
      "activations/layer13_attention_weight_min": -28.70751953125,
      "activations/layer14_attention_weight_max": 36.93247985839844,
      "activations/layer14_attention_weight_min": -33.073326110839844,
      "activations/layer15_attention_weight_max": 38.89643096923828,
      "activations/layer15_attention_weight_min": -32.67572784423828,
      "activations/layer16_attention_weight_max": 35.85388946533203,
      "activations/layer16_attention_weight_min": -35.30424118041992,
      "activations/layer17_attention_weight_max": 52.57976150512695,
      "activations/layer17_attention_weight_min": -51.2935905456543,
      "activations/layer18_attention_weight_max": 48.9654655456543,
      "activations/layer18_attention_weight_min": -47.69512176513672,
      "activations/layer19_attention_weight_max": 24.900896072387695,
      "activations/layer19_attention_weight_min": -26.602310180664062,
      "activations/layer1_attention_weight_max": 15.336077690124512,
      "activations/layer1_attention_weight_min": -15.633613586425781,
      "activations/layer20_attention_weight_max": 23.969940185546875,
      "activations/layer20_attention_weight_min": -23.583099365234375,
      "activations/layer21_attention_weight_max": 40.97897720336914,
      "activations/layer21_attention_weight_min": -29.69830322265625,
      "activations/layer22_attention_weight_max": 28.40022850036621,
      "activations/layer22_attention_weight_min": -26.4304141998291,
      "activations/layer23_attention_weight_max": 34.98069763183594,
      "activations/layer23_attention_weight_min": -26.040348052978516,
      "activations/layer2_attention_weight_max": 35.09552764892578,
      "activations/layer2_attention_weight_min": -32.69945526123047,
      "activations/layer3_attention_weight_max": 93.42437744140625,
      "activations/layer3_attention_weight_min": -95.23728942871094,
      "activations/layer4_attention_weight_max": 88.8069076538086,
      "activations/layer4_attention_weight_min": -90.66573333740234,
      "activations/layer5_attention_weight_max": 59.94382858276367,
      "activations/layer5_attention_weight_min": -71.63194274902344,
      "activations/layer6_attention_weight_max": 52.29050064086914,
      "activations/layer6_attention_weight_min": -55.2010612487793,
      "activations/layer7_attention_weight_max": 79.1369400024414,
      "activations/layer7_attention_weight_min": -76.6939468383789,
      "activations/layer8_attention_weight_max": 55.75947952270508,
      "activations/layer8_attention_weight_min": -53.877071380615234,
      "activations/layer9_attention_weight_max": 55.18341064453125,
      "activations/layer9_attention_weight_min": -50.182464599609375,
      "epoch": 23.13,
      "learning_rate": 8.397727272727272e-07,
      "loss": 2.7089,
      "step": 398100
    },
    {
      "activations/layer0_attention_weight_max": 15.989950180053711,
      "activations/layer0_attention_weight_min": -14.32972526550293,
      "activations/layer10_attention_weight_max": 34.89290237426758,
      "activations/layer10_attention_weight_min": -35.89934158325195,
      "activations/layer11_attention_weight_max": 32.58194351196289,
      "activations/layer11_attention_weight_min": -35.346683502197266,
      "activations/layer12_attention_weight_max": 18.23760986328125,
      "activations/layer12_attention_weight_min": -25.200790405273438,
      "activations/layer13_attention_weight_max": 34.084659576416016,
      "activations/layer13_attention_weight_min": -25.9287166595459,
      "activations/layer14_attention_weight_max": 33.47295379638672,
      "activations/layer14_attention_weight_min": -29.033132553100586,
      "activations/layer15_attention_weight_max": 33.566688537597656,
      "activations/layer15_attention_weight_min": -30.607973098754883,
      "activations/layer16_attention_weight_max": 32.842105865478516,
      "activations/layer16_attention_weight_min": -32.980960845947266,
      "activations/layer17_attention_weight_max": 47.73373031616211,
      "activations/layer17_attention_weight_min": -44.057125091552734,
      "activations/layer18_attention_weight_max": 43.23062515258789,
      "activations/layer18_attention_weight_min": -37.608436584472656,
      "activations/layer19_attention_weight_max": 23.39995765686035,
      "activations/layer19_attention_weight_min": -23.055923461914062,
      "activations/layer1_attention_weight_max": 14.685099601745605,
      "activations/layer1_attention_weight_min": -14.768645286560059,
      "activations/layer20_attention_weight_max": 23.678916931152344,
      "activations/layer20_attention_weight_min": -23.95921516418457,
      "activations/layer21_attention_weight_max": 30.97111701965332,
      "activations/layer21_attention_weight_min": -23.584545135498047,
      "activations/layer22_attention_weight_max": 27.58648681640625,
      "activations/layer22_attention_weight_min": -25.53986167907715,
      "activations/layer23_attention_weight_max": 32.67803192138672,
      "activations/layer23_attention_weight_min": -23.789562225341797,
      "activations/layer2_attention_weight_max": 32.655967712402344,
      "activations/layer2_attention_weight_min": -30.682729721069336,
      "activations/layer3_attention_weight_max": 92.93878173828125,
      "activations/layer3_attention_weight_min": -97.87554931640625,
      "activations/layer4_attention_weight_max": 85.75316619873047,
      "activations/layer4_attention_weight_min": -90.4986572265625,
      "activations/layer5_attention_weight_max": 62.080902099609375,
      "activations/layer5_attention_weight_min": -75.92100524902344,
      "activations/layer6_attention_weight_max": 51.06442642211914,
      "activations/layer6_attention_weight_min": -52.744571685791016,
      "activations/layer7_attention_weight_max": 69.02082824707031,
      "activations/layer7_attention_weight_min": -68.81586456298828,
      "activations/layer8_attention_weight_max": 48.3927001953125,
      "activations/layer8_attention_weight_min": -49.65290832519531,
      "activations/layer9_attention_weight_max": 45.3146858215332,
      "activations/layer9_attention_weight_min": -45.97904586791992,
      "epoch": 23.13,
      "learning_rate": 8.208333333333332e-07,
      "loss": 2.6899,
      "step": 398150
    },
    {
      "activations/layer0_attention_weight_max": 15.331841468811035,
      "activations/layer0_attention_weight_min": -13.968549728393555,
      "activations/layer10_attention_weight_max": 39.526668548583984,
      "activations/layer10_attention_weight_min": -38.55474853515625,
      "activations/layer11_attention_weight_max": 38.18027114868164,
      "activations/layer11_attention_weight_min": -37.546634674072266,
      "activations/layer12_attention_weight_max": 20.000638961791992,
      "activations/layer12_attention_weight_min": -37.67109298706055,
      "activations/layer13_attention_weight_max": 32.2287712097168,
      "activations/layer13_attention_weight_min": -27.172218322753906,
      "activations/layer14_attention_weight_max": 36.15541458129883,
      "activations/layer14_attention_weight_min": -30.296092987060547,
      "activations/layer15_attention_weight_max": 31.6479549407959,
      "activations/layer15_attention_weight_min": -29.50385856628418,
      "activations/layer16_attention_weight_max": 34.36103820800781,
      "activations/layer16_attention_weight_min": -31.567853927612305,
      "activations/layer17_attention_weight_max": 51.89183807373047,
      "activations/layer17_attention_weight_min": -44.52912139892578,
      "activations/layer18_attention_weight_max": 47.7668342590332,
      "activations/layer18_attention_weight_min": -41.01968002319336,
      "activations/layer19_attention_weight_max": 25.859460830688477,
      "activations/layer19_attention_weight_min": -23.155210494995117,
      "activations/layer1_attention_weight_max": 15.182976722717285,
      "activations/layer1_attention_weight_min": -14.781832695007324,
      "activations/layer20_attention_weight_max": 22.96785545349121,
      "activations/layer20_attention_weight_min": -20.422061920166016,
      "activations/layer21_attention_weight_max": 36.23222732543945,
      "activations/layer21_attention_weight_min": -22.365999221801758,
      "activations/layer22_attention_weight_max": 29.321876525878906,
      "activations/layer22_attention_weight_min": -24.1910457611084,
      "activations/layer23_attention_weight_max": 33.25312042236328,
      "activations/layer23_attention_weight_min": -24.80474090576172,
      "activations/layer2_attention_weight_max": 32.50692367553711,
      "activations/layer2_attention_weight_min": -32.316444396972656,
      "activations/layer3_attention_weight_max": 97.1396484375,
      "activations/layer3_attention_weight_min": -99.88603210449219,
      "activations/layer4_attention_weight_max": 92.906005859375,
      "activations/layer4_attention_weight_min": -91.71878051757812,
      "activations/layer5_attention_weight_max": 66.23804473876953,
      "activations/layer5_attention_weight_min": -75.9564437866211,
      "activations/layer6_attention_weight_max": 51.80495834350586,
      "activations/layer6_attention_weight_min": -52.76974868774414,
      "activations/layer7_attention_weight_max": 76.51994323730469,
      "activations/layer7_attention_weight_min": -73.94315338134766,
      "activations/layer8_attention_weight_max": 52.37583923339844,
      "activations/layer8_attention_weight_min": -53.33717346191406,
      "activations/layer9_attention_weight_max": 52.888954162597656,
      "activations/layer9_attention_weight_min": -48.14810562133789,
      "epoch": 23.14,
      "learning_rate": 8.018939393939393e-07,
      "loss": 2.694,
      "step": 398200
    },
    {
      "activations/layer0_attention_weight_max": 15.487053871154785,
      "activations/layer0_attention_weight_min": -14.064347267150879,
      "activations/layer10_attention_weight_max": 37.141845703125,
      "activations/layer10_attention_weight_min": -38.08601760864258,
      "activations/layer11_attention_weight_max": 35.86543655395508,
      "activations/layer11_attention_weight_min": -37.2294807434082,
      "activations/layer12_attention_weight_max": 22.393970489501953,
      "activations/layer12_attention_weight_min": -29.066959381103516,
      "activations/layer13_attention_weight_max": 42.01335525512695,
      "activations/layer13_attention_weight_min": -26.1452693939209,
      "activations/layer14_attention_weight_max": 38.41482162475586,
      "activations/layer14_attention_weight_min": -31.203350067138672,
      "activations/layer15_attention_weight_max": 37.1319580078125,
      "activations/layer15_attention_weight_min": -31.780805587768555,
      "activations/layer16_attention_weight_max": 37.20438003540039,
      "activations/layer16_attention_weight_min": -33.49760055541992,
      "activations/layer17_attention_weight_max": 53.09038543701172,
      "activations/layer17_attention_weight_min": -48.99650573730469,
      "activations/layer18_attention_weight_max": 50.580440521240234,
      "activations/layer18_attention_weight_min": -42.03587341308594,
      "activations/layer19_attention_weight_max": 28.016630172729492,
      "activations/layer19_attention_weight_min": -24.037511825561523,
      "activations/layer1_attention_weight_max": 16.295249938964844,
      "activations/layer1_attention_weight_min": -14.833171844482422,
      "activations/layer20_attention_weight_max": 23.723121643066406,
      "activations/layer20_attention_weight_min": -22.695140838623047,
      "activations/layer21_attention_weight_max": 39.22803497314453,
      "activations/layer21_attention_weight_min": -23.390771865844727,
      "activations/layer22_attention_weight_max": 28.393314361572266,
      "activations/layer22_attention_weight_min": -28.27779769897461,
      "activations/layer23_attention_weight_max": 40.027713775634766,
      "activations/layer23_attention_weight_min": -24.636648178100586,
      "activations/layer2_attention_weight_max": 32.360321044921875,
      "activations/layer2_attention_weight_min": -30.969085693359375,
      "activations/layer3_attention_weight_max": 93.13024139404297,
      "activations/layer3_attention_weight_min": -94.58409881591797,
      "activations/layer4_attention_weight_max": 88.01506805419922,
      "activations/layer4_attention_weight_min": -88.52011108398438,
      "activations/layer5_attention_weight_max": 62.46586227416992,
      "activations/layer5_attention_weight_min": -74.41738891601562,
      "activations/layer6_attention_weight_max": 49.707923889160156,
      "activations/layer6_attention_weight_min": -52.236942291259766,
      "activations/layer7_attention_weight_max": 69.73198699951172,
      "activations/layer7_attention_weight_min": -70.67058563232422,
      "activations/layer8_attention_weight_max": 49.350765228271484,
      "activations/layer8_attention_weight_min": -52.18123245239258,
      "activations/layer9_attention_weight_max": 45.35974884033203,
      "activations/layer9_attention_weight_min": -50.08419418334961,
      "epoch": 23.14,
      "learning_rate": 7.829545454545453e-07,
      "loss": 2.702,
      "step": 398250
    },
    {
      "activations/layer0_attention_weight_max": 15.795916557312012,
      "activations/layer0_attention_weight_min": -14.231398582458496,
      "activations/layer10_attention_weight_max": 36.3372688293457,
      "activations/layer10_attention_weight_min": -36.73329544067383,
      "activations/layer11_attention_weight_max": 34.116939544677734,
      "activations/layer11_attention_weight_min": -34.69043731689453,
      "activations/layer12_attention_weight_max": 19.776334762573242,
      "activations/layer12_attention_weight_min": -24.012052536010742,
      "activations/layer13_attention_weight_max": 36.1290168762207,
      "activations/layer13_attention_weight_min": -28.27193832397461,
      "activations/layer14_attention_weight_max": 39.513404846191406,
      "activations/layer14_attention_weight_min": -31.66104507446289,
      "activations/layer15_attention_weight_max": 41.00190353393555,
      "activations/layer15_attention_weight_min": -32.4498176574707,
      "activations/layer16_attention_weight_max": 33.952449798583984,
      "activations/layer16_attention_weight_min": -32.55368423461914,
      "activations/layer17_attention_weight_max": 49.89714813232422,
      "activations/layer17_attention_weight_min": -45.890769958496094,
      "activations/layer18_attention_weight_max": 45.65355682373047,
      "activations/layer18_attention_weight_min": -38.35704803466797,
      "activations/layer19_attention_weight_max": 27.257125854492188,
      "activations/layer19_attention_weight_min": -23.36042594909668,
      "activations/layer1_attention_weight_max": 16.02683448791504,
      "activations/layer1_attention_weight_min": -14.794719696044922,
      "activations/layer20_attention_weight_max": 24.260412216186523,
      "activations/layer20_attention_weight_min": -22.287981033325195,
      "activations/layer21_attention_weight_max": 31.79170799255371,
      "activations/layer21_attention_weight_min": -20.90181541442871,
      "activations/layer22_attention_weight_max": 27.995285034179688,
      "activations/layer22_attention_weight_min": -24.570207595825195,
      "activations/layer23_attention_weight_max": 32.10214614868164,
      "activations/layer23_attention_weight_min": -24.276050567626953,
      "activations/layer2_attention_weight_max": 32.357887268066406,
      "activations/layer2_attention_weight_min": -30.468589782714844,
      "activations/layer3_attention_weight_max": 89.39301300048828,
      "activations/layer3_attention_weight_min": -89.29228210449219,
      "activations/layer4_attention_weight_max": 83.37185668945312,
      "activations/layer4_attention_weight_min": -83.41049194335938,
      "activations/layer5_attention_weight_max": 59.701080322265625,
      "activations/layer5_attention_weight_min": -73.89061737060547,
      "activations/layer6_attention_weight_max": 49.470333099365234,
      "activations/layer6_attention_weight_min": -49.90998458862305,
      "activations/layer7_attention_weight_max": 68.92276000976562,
      "activations/layer7_attention_weight_min": -66.97982788085938,
      "activations/layer8_attention_weight_max": 45.4125862121582,
      "activations/layer8_attention_weight_min": -48.902278900146484,
      "activations/layer9_attention_weight_max": 41.89078903198242,
      "activations/layer9_attention_weight_min": -45.033653259277344,
      "epoch": 23.14,
      "learning_rate": 7.640151515151515e-07,
      "loss": 2.6893,
      "step": 398300
    },
    {
      "activations/layer0_attention_weight_max": 16.964481353759766,
      "activations/layer0_attention_weight_min": -14.816987037658691,
      "activations/layer10_attention_weight_max": 41.11191940307617,
      "activations/layer10_attention_weight_min": -39.20652770996094,
      "activations/layer11_attention_weight_max": 41.09489440917969,
      "activations/layer11_attention_weight_min": -38.26304244995117,
      "activations/layer12_attention_weight_max": 21.345731735229492,
      "activations/layer12_attention_weight_min": -25.27651596069336,
      "activations/layer13_attention_weight_max": 41.952880859375,
      "activations/layer13_attention_weight_min": -28.190828323364258,
      "activations/layer14_attention_weight_max": 39.121280670166016,
      "activations/layer14_attention_weight_min": -30.97867202758789,
      "activations/layer15_attention_weight_max": 36.56352233886719,
      "activations/layer15_attention_weight_min": -32.876705169677734,
      "activations/layer16_attention_weight_max": 36.74771499633789,
      "activations/layer16_attention_weight_min": -34.255889892578125,
      "activations/layer17_attention_weight_max": 58.24041748046875,
      "activations/layer17_attention_weight_min": -47.72222900390625,
      "activations/layer18_attention_weight_max": 52.875755310058594,
      "activations/layer18_attention_weight_min": -41.621360778808594,
      "activations/layer19_attention_weight_max": 28.281906127929688,
      "activations/layer19_attention_weight_min": -24.116914749145508,
      "activations/layer1_attention_weight_max": 15.22671127319336,
      "activations/layer1_attention_weight_min": -14.636746406555176,
      "activations/layer20_attention_weight_max": 26.534029006958008,
      "activations/layer20_attention_weight_min": -22.01033592224121,
      "activations/layer21_attention_weight_max": 39.58270263671875,
      "activations/layer21_attention_weight_min": -24.693058013916016,
      "activations/layer22_attention_weight_max": 33.46876525878906,
      "activations/layer22_attention_weight_min": -25.44373893737793,
      "activations/layer23_attention_weight_max": 41.5169792175293,
      "activations/layer23_attention_weight_min": -24.266151428222656,
      "activations/layer2_attention_weight_max": 33.368160247802734,
      "activations/layer2_attention_weight_min": -32.382102966308594,
      "activations/layer3_attention_weight_max": 92.10874938964844,
      "activations/layer3_attention_weight_min": -93.13331604003906,
      "activations/layer4_attention_weight_max": 87.16921997070312,
      "activations/layer4_attention_weight_min": -88.64167022705078,
      "activations/layer5_attention_weight_max": 62.99537658691406,
      "activations/layer5_attention_weight_min": -71.55474853515625,
      "activations/layer6_attention_weight_max": 52.04966735839844,
      "activations/layer6_attention_weight_min": -52.934627532958984,
      "activations/layer7_attention_weight_max": 78.49221801757812,
      "activations/layer7_attention_weight_min": -76.18539428710938,
      "activations/layer8_attention_weight_max": 53.45509719848633,
      "activations/layer8_attention_weight_min": -54.48744583129883,
      "activations/layer9_attention_weight_max": 51.99549102783203,
      "activations/layer9_attention_weight_min": -49.572181701660156,
      "epoch": 23.15,
      "learning_rate": 7.450757575757575e-07,
      "loss": 2.6959,
      "step": 398350
    },
    {
      "activations/layer0_attention_weight_max": 15.1239595413208,
      "activations/layer0_attention_weight_min": -14.339485168457031,
      "activations/layer10_attention_weight_max": 34.99801254272461,
      "activations/layer10_attention_weight_min": -38.27301788330078,
      "activations/layer11_attention_weight_max": 33.294654846191406,
      "activations/layer11_attention_weight_min": -37.99102783203125,
      "activations/layer12_attention_weight_max": 19.29615592956543,
      "activations/layer12_attention_weight_min": -26.269630432128906,
      "activations/layer13_attention_weight_max": 32.3740234375,
      "activations/layer13_attention_weight_min": -27.374168395996094,
      "activations/layer14_attention_weight_max": 34.93971252441406,
      "activations/layer14_attention_weight_min": -31.97943687438965,
      "activations/layer15_attention_weight_max": 36.072776794433594,
      "activations/layer15_attention_weight_min": -33.122745513916016,
      "activations/layer16_attention_weight_max": 34.619144439697266,
      "activations/layer16_attention_weight_min": -33.593597412109375,
      "activations/layer17_attention_weight_max": 54.108673095703125,
      "activations/layer17_attention_weight_min": -46.257843017578125,
      "activations/layer18_attention_weight_max": 48.46087646484375,
      "activations/layer18_attention_weight_min": -41.8612174987793,
      "activations/layer19_attention_weight_max": 25.376888275146484,
      "activations/layer19_attention_weight_min": -24.668033599853516,
      "activations/layer1_attention_weight_max": 14.865728378295898,
      "activations/layer1_attention_weight_min": -14.743732452392578,
      "activations/layer20_attention_weight_max": 21.637950897216797,
      "activations/layer20_attention_weight_min": -21.747005462646484,
      "activations/layer21_attention_weight_max": 39.85447692871094,
      "activations/layer21_attention_weight_min": -23.581552505493164,
      "activations/layer22_attention_weight_max": 27.775394439697266,
      "activations/layer22_attention_weight_min": -24.85698127746582,
      "activations/layer23_attention_weight_max": 35.71040344238281,
      "activations/layer23_attention_weight_min": -24.31427574157715,
      "activations/layer2_attention_weight_max": 31.67450714111328,
      "activations/layer2_attention_weight_min": -29.630630493164062,
      "activations/layer3_attention_weight_max": 91.17442321777344,
      "activations/layer3_attention_weight_min": -89.86499786376953,
      "activations/layer4_attention_weight_max": 86.1574478149414,
      "activations/layer4_attention_weight_min": -87.49413299560547,
      "activations/layer5_attention_weight_max": 61.64626693725586,
      "activations/layer5_attention_weight_min": -75.41749572753906,
      "activations/layer6_attention_weight_max": 50.06755065917969,
      "activations/layer6_attention_weight_min": -51.23283767700195,
      "activations/layer7_attention_weight_max": 69.21139526367188,
      "activations/layer7_attention_weight_min": -67.98091125488281,
      "activations/layer8_attention_weight_max": 47.499942779541016,
      "activations/layer8_attention_weight_min": -53.279300689697266,
      "activations/layer9_attention_weight_max": 41.2725830078125,
      "activations/layer9_attention_weight_min": -47.229209899902344,
      "epoch": 23.15,
      "learning_rate": 7.261363636363636e-07,
      "loss": 2.6988,
      "step": 398400
    },
    {
      "activations/layer0_attention_weight_max": 15.972260475158691,
      "activations/layer0_attention_weight_min": -14.32760238647461,
      "activations/layer10_attention_weight_max": 35.041141510009766,
      "activations/layer10_attention_weight_min": -36.82054901123047,
      "activations/layer11_attention_weight_max": 33.09756088256836,
      "activations/layer11_attention_weight_min": -36.79385757446289,
      "activations/layer12_attention_weight_max": 21.478727340698242,
      "activations/layer12_attention_weight_min": -27.37074089050293,
      "activations/layer13_attention_weight_max": 35.29353332519531,
      "activations/layer13_attention_weight_min": -27.04418182373047,
      "activations/layer14_attention_weight_max": 36.68661880493164,
      "activations/layer14_attention_weight_min": -31.344877243041992,
      "activations/layer15_attention_weight_max": 35.05009078979492,
      "activations/layer15_attention_weight_min": -32.56710433959961,
      "activations/layer16_attention_weight_max": 38.11533737182617,
      "activations/layer16_attention_weight_min": -34.25966262817383,
      "activations/layer17_attention_weight_max": 53.26461410522461,
      "activations/layer17_attention_weight_min": -48.77824401855469,
      "activations/layer18_attention_weight_max": 55.21287155151367,
      "activations/layer18_attention_weight_min": -42.99674987792969,
      "activations/layer19_attention_weight_max": 26.87253761291504,
      "activations/layer19_attention_weight_min": -24.503393173217773,
      "activations/layer1_attention_weight_max": 14.683809280395508,
      "activations/layer1_attention_weight_min": -13.280501365661621,
      "activations/layer20_attention_weight_max": 24.811546325683594,
      "activations/layer20_attention_weight_min": -22.289587020874023,
      "activations/layer21_attention_weight_max": 36.1710205078125,
      "activations/layer21_attention_weight_min": -23.073274612426758,
      "activations/layer22_attention_weight_max": 30.28153419494629,
      "activations/layer22_attention_weight_min": -26.144372940063477,
      "activations/layer23_attention_weight_max": 37.543548583984375,
      "activations/layer23_attention_weight_min": -23.65790367126465,
      "activations/layer2_attention_weight_max": 32.609683990478516,
      "activations/layer2_attention_weight_min": -29.96762466430664,
      "activations/layer3_attention_weight_max": 92.6869888305664,
      "activations/layer3_attention_weight_min": -91.31909942626953,
      "activations/layer4_attention_weight_max": 87.8562240600586,
      "activations/layer4_attention_weight_min": -88.0333023071289,
      "activations/layer5_attention_weight_max": 64.8908462524414,
      "activations/layer5_attention_weight_min": -75.64247131347656,
      "activations/layer6_attention_weight_max": 51.93706130981445,
      "activations/layer6_attention_weight_min": -53.71462631225586,
      "activations/layer7_attention_weight_max": 71.85677337646484,
      "activations/layer7_attention_weight_min": -70.4707260131836,
      "activations/layer8_attention_weight_max": 49.0809211730957,
      "activations/layer8_attention_weight_min": -55.81993865966797,
      "activations/layer9_attention_weight_max": 45.364288330078125,
      "activations/layer9_attention_weight_min": -49.17550277709961,
      "epoch": 23.15,
      "learning_rate": 7.071969696969696e-07,
      "loss": 2.6962,
      "step": 398450
    },
    {
      "activations/layer0_attention_weight_max": 15.645655632019043,
      "activations/layer0_attention_weight_min": -13.993009567260742,
      "activations/layer10_attention_weight_max": 40.3222541809082,
      "activations/layer10_attention_weight_min": -38.5388069152832,
      "activations/layer11_attention_weight_max": 39.552040100097656,
      "activations/layer11_attention_weight_min": -38.1002197265625,
      "activations/layer12_attention_weight_max": 20.113325119018555,
      "activations/layer12_attention_weight_min": -26.924272537231445,
      "activations/layer13_attention_weight_max": 57.655250549316406,
      "activations/layer13_attention_weight_min": -29.356449127197266,
      "activations/layer14_attention_weight_max": 39.9300651550293,
      "activations/layer14_attention_weight_min": -31.382970809936523,
      "activations/layer15_attention_weight_max": 33.610164642333984,
      "activations/layer15_attention_weight_min": -33.375423431396484,
      "activations/layer16_attention_weight_max": 36.08952713012695,
      "activations/layer16_attention_weight_min": -34.747928619384766,
      "activations/layer17_attention_weight_max": 55.56087112426758,
      "activations/layer17_attention_weight_min": -47.092628479003906,
      "activations/layer18_attention_weight_max": 46.37387466430664,
      "activations/layer18_attention_weight_min": -42.50924301147461,
      "activations/layer19_attention_weight_max": 23.447711944580078,
      "activations/layer19_attention_weight_min": -23.85139274597168,
      "activations/layer1_attention_weight_max": 15.57901382446289,
      "activations/layer1_attention_weight_min": -14.897003173828125,
      "activations/layer20_attention_weight_max": 23.39238166809082,
      "activations/layer20_attention_weight_min": -22.778535842895508,
      "activations/layer21_attention_weight_max": 41.883968353271484,
      "activations/layer21_attention_weight_min": -22.188289642333984,
      "activations/layer22_attention_weight_max": 29.571802139282227,
      "activations/layer22_attention_weight_min": -25.396394729614258,
      "activations/layer23_attention_weight_max": 37.164249420166016,
      "activations/layer23_attention_weight_min": -26.693706512451172,
      "activations/layer2_attention_weight_max": 35.00185775756836,
      "activations/layer2_attention_weight_min": -31.43677520751953,
      "activations/layer3_attention_weight_max": 99.5762710571289,
      "activations/layer3_attention_weight_min": -96.89131164550781,
      "activations/layer4_attention_weight_max": 89.90218353271484,
      "activations/layer4_attention_weight_min": -89.84615325927734,
      "activations/layer5_attention_weight_max": 66.27870178222656,
      "activations/layer5_attention_weight_min": -71.87763977050781,
      "activations/layer6_attention_weight_max": 51.72469711303711,
      "activations/layer6_attention_weight_min": -52.615108489990234,
      "activations/layer7_attention_weight_max": 72.53660583496094,
      "activations/layer7_attention_weight_min": -72.84619140625,
      "activations/layer8_attention_weight_max": 50.739803314208984,
      "activations/layer8_attention_weight_min": -52.434505462646484,
      "activations/layer9_attention_weight_max": 52.94964599609375,
      "activations/layer9_attention_weight_min": -50.44331741333008,
      "epoch": 23.16,
      "learning_rate": 6.882575757575758e-07,
      "loss": 2.6862,
      "step": 398500
    },
    {
      "activations/layer0_attention_weight_max": 16.144855499267578,
      "activations/layer0_attention_weight_min": -14.264432907104492,
      "activations/layer10_attention_weight_max": 36.15595245361328,
      "activations/layer10_attention_weight_min": -36.440650939941406,
      "activations/layer11_attention_weight_max": 35.54804229736328,
      "activations/layer11_attention_weight_min": -38.66010665893555,
      "activations/layer12_attention_weight_max": 19.076780319213867,
      "activations/layer12_attention_weight_min": -23.870227813720703,
      "activations/layer13_attention_weight_max": 33.480438232421875,
      "activations/layer13_attention_weight_min": -27.40198516845703,
      "activations/layer14_attention_weight_max": 35.19685363769531,
      "activations/layer14_attention_weight_min": -30.3170223236084,
      "activations/layer15_attention_weight_max": 36.49114990234375,
      "activations/layer15_attention_weight_min": -31.408897399902344,
      "activations/layer16_attention_weight_max": 38.55757141113281,
      "activations/layer16_attention_weight_min": -38.4726676940918,
      "activations/layer17_attention_weight_max": 59.01239776611328,
      "activations/layer17_attention_weight_min": -53.41732406616211,
      "activations/layer18_attention_weight_max": 53.538021087646484,
      "activations/layer18_attention_weight_min": -44.77056121826172,
      "activations/layer19_attention_weight_max": 24.165224075317383,
      "activations/layer19_attention_weight_min": -24.00462532043457,
      "activations/layer1_attention_weight_max": 16.10146713256836,
      "activations/layer1_attention_weight_min": -14.049032211303711,
      "activations/layer20_attention_weight_max": 24.10626792907715,
      "activations/layer20_attention_weight_min": -21.243825912475586,
      "activations/layer21_attention_weight_max": 34.81817626953125,
      "activations/layer21_attention_weight_min": -20.52103614807129,
      "activations/layer22_attention_weight_max": 27.05045509338379,
      "activations/layer22_attention_weight_min": -27.0357723236084,
      "activations/layer23_attention_weight_max": 36.27523422241211,
      "activations/layer23_attention_weight_min": -25.263303756713867,
      "activations/layer2_attention_weight_max": 31.467466354370117,
      "activations/layer2_attention_weight_min": -30.492910385131836,
      "activations/layer3_attention_weight_max": 91.5373764038086,
      "activations/layer3_attention_weight_min": -91.74050903320312,
      "activations/layer4_attention_weight_max": 87.00112915039062,
      "activations/layer4_attention_weight_min": -87.55157470703125,
      "activations/layer5_attention_weight_max": 62.07143783569336,
      "activations/layer5_attention_weight_min": -71.46725463867188,
      "activations/layer6_attention_weight_max": 51.84421920776367,
      "activations/layer6_attention_weight_min": -51.30872344970703,
      "activations/layer7_attention_weight_max": 73.65174102783203,
      "activations/layer7_attention_weight_min": -72.46339416503906,
      "activations/layer8_attention_weight_max": 52.328128814697266,
      "activations/layer8_attention_weight_min": -53.35969924926758,
      "activations/layer9_attention_weight_max": 44.5240478515625,
      "activations/layer9_attention_weight_min": -49.8813362121582,
      "epoch": 23.16,
      "learning_rate": 6.693181818181818e-07,
      "loss": 2.6841,
      "step": 398550
    },
    {
      "activations/layer0_attention_weight_max": 16.123132705688477,
      "activations/layer0_attention_weight_min": -14.382729530334473,
      "activations/layer10_attention_weight_max": 35.46954345703125,
      "activations/layer10_attention_weight_min": -37.23695755004883,
      "activations/layer11_attention_weight_max": 33.287139892578125,
      "activations/layer11_attention_weight_min": -34.59956359863281,
      "activations/layer12_attention_weight_max": 20.356952667236328,
      "activations/layer12_attention_weight_min": -29.03042984008789,
      "activations/layer13_attention_weight_max": 30.99271011352539,
      "activations/layer13_attention_weight_min": -27.114423751831055,
      "activations/layer14_attention_weight_max": 35.42707061767578,
      "activations/layer14_attention_weight_min": -30.523151397705078,
      "activations/layer15_attention_weight_max": 32.46139144897461,
      "activations/layer15_attention_weight_min": -31.292600631713867,
      "activations/layer16_attention_weight_max": 36.76533126831055,
      "activations/layer16_attention_weight_min": -37.04450225830078,
      "activations/layer17_attention_weight_max": 54.20951843261719,
      "activations/layer17_attention_weight_min": -48.373634338378906,
      "activations/layer18_attention_weight_max": 52.44182205200195,
      "activations/layer18_attention_weight_min": -46.79143524169922,
      "activations/layer19_attention_weight_max": 24.09590721130371,
      "activations/layer19_attention_weight_min": -23.39893913269043,
      "activations/layer1_attention_weight_max": 14.912576675415039,
      "activations/layer1_attention_weight_min": -14.790081977844238,
      "activations/layer20_attention_weight_max": 22.565811157226562,
      "activations/layer20_attention_weight_min": -25.093433380126953,
      "activations/layer21_attention_weight_max": 39.46735382080078,
      "activations/layer21_attention_weight_min": -25.730295181274414,
      "activations/layer22_attention_weight_max": 30.725507736206055,
      "activations/layer22_attention_weight_min": -26.6761474609375,
      "activations/layer23_attention_weight_max": 37.40416717529297,
      "activations/layer23_attention_weight_min": -24.862943649291992,
      "activations/layer2_attention_weight_max": 32.628761291503906,
      "activations/layer2_attention_weight_min": -30.430700302124023,
      "activations/layer3_attention_weight_max": 97.13406372070312,
      "activations/layer3_attention_weight_min": -94.72215270996094,
      "activations/layer4_attention_weight_max": 89.43535614013672,
      "activations/layer4_attention_weight_min": -89.30213928222656,
      "activations/layer5_attention_weight_max": 61.1822509765625,
      "activations/layer5_attention_weight_min": -69.19458770751953,
      "activations/layer6_attention_weight_max": 49.474544525146484,
      "activations/layer6_attention_weight_min": -50.84093475341797,
      "activations/layer7_attention_weight_max": 71.67374420166016,
      "activations/layer7_attention_weight_min": -68.43195343017578,
      "activations/layer8_attention_weight_max": 47.68471145629883,
      "activations/layer8_attention_weight_min": -51.224300384521484,
      "activations/layer9_attention_weight_max": 42.538448333740234,
      "activations/layer9_attention_weight_min": -45.59841537475586,
      "epoch": 23.16,
      "learning_rate": 6.503787878787879e-07,
      "loss": 2.7168,
      "step": 398600
    },
    {
      "activations/layer0_attention_weight_max": 16.79082489013672,
      "activations/layer0_attention_weight_min": -14.93288803100586,
      "activations/layer10_attention_weight_max": 35.86292266845703,
      "activations/layer10_attention_weight_min": -38.65642166137695,
      "activations/layer11_attention_weight_max": 34.16099548339844,
      "activations/layer11_attention_weight_min": -37.014503479003906,
      "activations/layer12_attention_weight_max": 20.082550048828125,
      "activations/layer12_attention_weight_min": -22.700510025024414,
      "activations/layer13_attention_weight_max": 36.493446350097656,
      "activations/layer13_attention_weight_min": -27.626550674438477,
      "activations/layer14_attention_weight_max": 36.96741485595703,
      "activations/layer14_attention_weight_min": -30.925674438476562,
      "activations/layer15_attention_weight_max": 33.63368225097656,
      "activations/layer15_attention_weight_min": -32.42975616455078,
      "activations/layer16_attention_weight_max": 36.1580924987793,
      "activations/layer16_attention_weight_min": -32.717716217041016,
      "activations/layer17_attention_weight_max": 56.72177505493164,
      "activations/layer17_attention_weight_min": -46.512271881103516,
      "activations/layer18_attention_weight_max": 51.85669708251953,
      "activations/layer18_attention_weight_min": -40.59398651123047,
      "activations/layer19_attention_weight_max": 25.18513298034668,
      "activations/layer19_attention_weight_min": -22.7597599029541,
      "activations/layer1_attention_weight_max": 15.876690864562988,
      "activations/layer1_attention_weight_min": -14.96314811706543,
      "activations/layer20_attention_weight_max": 24.072763442993164,
      "activations/layer20_attention_weight_min": -22.54871940612793,
      "activations/layer21_attention_weight_max": 36.60725784301758,
      "activations/layer21_attention_weight_min": -25.035661697387695,
      "activations/layer22_attention_weight_max": 29.053144454956055,
      "activations/layer22_attention_weight_min": -25.50613784790039,
      "activations/layer23_attention_weight_max": 40.08134078979492,
      "activations/layer23_attention_weight_min": -24.001142501831055,
      "activations/layer2_attention_weight_max": 33.96876525878906,
      "activations/layer2_attention_weight_min": -31.93471908569336,
      "activations/layer3_attention_weight_max": 97.25452423095703,
      "activations/layer3_attention_weight_min": -97.84597778320312,
      "activations/layer4_attention_weight_max": 91.94644165039062,
      "activations/layer4_attention_weight_min": -89.80229949951172,
      "activations/layer5_attention_weight_max": 64.5557861328125,
      "activations/layer5_attention_weight_min": -72.19056701660156,
      "activations/layer6_attention_weight_max": 51.99040603637695,
      "activations/layer6_attention_weight_min": -52.678714752197266,
      "activations/layer7_attention_weight_max": 71.74848175048828,
      "activations/layer7_attention_weight_min": -72.7135238647461,
      "activations/layer8_attention_weight_max": 50.51735305786133,
      "activations/layer8_attention_weight_min": -52.13722229003906,
      "activations/layer9_attention_weight_max": 48.242733001708984,
      "activations/layer9_attention_weight_min": -47.137428283691406,
      "epoch": 23.16,
      "learning_rate": 6.314393939393939e-07,
      "loss": 2.6983,
      "step": 398650
    },
    {
      "activations/layer0_attention_weight_max": 16.311092376708984,
      "activations/layer0_attention_weight_min": -14.304756164550781,
      "activations/layer10_attention_weight_max": 38.508583068847656,
      "activations/layer10_attention_weight_min": -35.86252212524414,
      "activations/layer11_attention_weight_max": 34.016845703125,
      "activations/layer11_attention_weight_min": -36.123016357421875,
      "activations/layer12_attention_weight_max": 19.72519874572754,
      "activations/layer12_attention_weight_min": -24.80445098876953,
      "activations/layer13_attention_weight_max": 36.601844787597656,
      "activations/layer13_attention_weight_min": -29.75149154663086,
      "activations/layer14_attention_weight_max": 35.31700897216797,
      "activations/layer14_attention_weight_min": -32.4559326171875,
      "activations/layer15_attention_weight_max": 34.68784713745117,
      "activations/layer15_attention_weight_min": -32.54875564575195,
      "activations/layer16_attention_weight_max": 35.29204177856445,
      "activations/layer16_attention_weight_min": -35.53111267089844,
      "activations/layer17_attention_weight_max": 52.58230209350586,
      "activations/layer17_attention_weight_min": -47.15206527709961,
      "activations/layer18_attention_weight_max": 51.30759811401367,
      "activations/layer18_attention_weight_min": -43.27974319458008,
      "activations/layer19_attention_weight_max": 26.11004638671875,
      "activations/layer19_attention_weight_min": -23.37886619567871,
      "activations/layer1_attention_weight_max": 14.92977523803711,
      "activations/layer1_attention_weight_min": -14.660049438476562,
      "activations/layer20_attention_weight_max": 24.183801651000977,
      "activations/layer20_attention_weight_min": -21.142114639282227,
      "activations/layer21_attention_weight_max": 37.70777893066406,
      "activations/layer21_attention_weight_min": -22.696508407592773,
      "activations/layer22_attention_weight_max": 28.949832916259766,
      "activations/layer22_attention_weight_min": -27.641958236694336,
      "activations/layer23_attention_weight_max": 35.92776870727539,
      "activations/layer23_attention_weight_min": -23.341840744018555,
      "activations/layer2_attention_weight_max": 32.5413818359375,
      "activations/layer2_attention_weight_min": -31.866024017333984,
      "activations/layer3_attention_weight_max": 92.81185913085938,
      "activations/layer3_attention_weight_min": -95.51895141601562,
      "activations/layer4_attention_weight_max": 87.72686004638672,
      "activations/layer4_attention_weight_min": -88.3500747680664,
      "activations/layer5_attention_weight_max": 63.29072189331055,
      "activations/layer5_attention_weight_min": -74.18203735351562,
      "activations/layer6_attention_weight_max": 52.19145584106445,
      "activations/layer6_attention_weight_min": -52.26215744018555,
      "activations/layer7_attention_weight_max": 69.07292938232422,
      "activations/layer7_attention_weight_min": -68.08867645263672,
      "activations/layer8_attention_weight_max": 50.975555419921875,
      "activations/layer8_attention_weight_min": -53.89304733276367,
      "activations/layer9_attention_weight_max": 48.9491081237793,
      "activations/layer9_attention_weight_min": -47.540733337402344,
      "epoch": 23.17,
      "learning_rate": 6.124999999999999e-07,
      "loss": 2.6819,
      "step": 398700
    },
    {
      "activations/layer0_attention_weight_max": 15.373477935791016,
      "activations/layer0_attention_weight_min": -14.370039939880371,
      "activations/layer10_attention_weight_max": 38.073829650878906,
      "activations/layer10_attention_weight_min": -39.24126052856445,
      "activations/layer11_attention_weight_max": 36.466331481933594,
      "activations/layer11_attention_weight_min": -40.85106658935547,
      "activations/layer12_attention_weight_max": 19.353578567504883,
      "activations/layer12_attention_weight_min": -30.746994018554688,
      "activations/layer13_attention_weight_max": 35.44500732421875,
      "activations/layer13_attention_weight_min": -31.554819107055664,
      "activations/layer14_attention_weight_max": 35.38086700439453,
      "activations/layer14_attention_weight_min": -32.304908752441406,
      "activations/layer15_attention_weight_max": 34.99470520019531,
      "activations/layer15_attention_weight_min": -35.711246490478516,
      "activations/layer16_attention_weight_max": 37.30489730834961,
      "activations/layer16_attention_weight_min": -34.951229095458984,
      "activations/layer17_attention_weight_max": 55.68989944458008,
      "activations/layer17_attention_weight_min": -48.66064453125,
      "activations/layer18_attention_weight_max": 52.73749923706055,
      "activations/layer18_attention_weight_min": -44.01598358154297,
      "activations/layer19_attention_weight_max": 24.41152000427246,
      "activations/layer19_attention_weight_min": -24.03516387939453,
      "activations/layer1_attention_weight_max": 15.703399658203125,
      "activations/layer1_attention_weight_min": -14.015851974487305,
      "activations/layer20_attention_weight_max": 23.073055267333984,
      "activations/layer20_attention_weight_min": -23.153667449951172,
      "activations/layer21_attention_weight_max": 31.11181640625,
      "activations/layer21_attention_weight_min": -25.567100524902344,
      "activations/layer22_attention_weight_max": 31.696231842041016,
      "activations/layer22_attention_weight_min": -26.288375854492188,
      "activations/layer23_attention_weight_max": 36.310691833496094,
      "activations/layer23_attention_weight_min": -26.49188995361328,
      "activations/layer2_attention_weight_max": 32.58601379394531,
      "activations/layer2_attention_weight_min": -32.164405822753906,
      "activations/layer3_attention_weight_max": 97.90179443359375,
      "activations/layer3_attention_weight_min": -96.05171203613281,
      "activations/layer4_attention_weight_max": 92.5566177368164,
      "activations/layer4_attention_weight_min": -89.37831115722656,
      "activations/layer5_attention_weight_max": 67.08856964111328,
      "activations/layer5_attention_weight_min": -75.22835540771484,
      "activations/layer6_attention_weight_max": 56.40552520751953,
      "activations/layer6_attention_weight_min": -59.74628448486328,
      "activations/layer7_attention_weight_max": 79.98677062988281,
      "activations/layer7_attention_weight_min": -72.9075698852539,
      "activations/layer8_attention_weight_max": 49.438011169433594,
      "activations/layer8_attention_weight_min": -52.44182205200195,
      "activations/layer9_attention_weight_max": 45.77834701538086,
      "activations/layer9_attention_weight_min": -49.05534362792969,
      "epoch": 23.17,
      "learning_rate": 5.93560606060606e-07,
      "loss": 2.6866,
      "step": 398750
    },
    {
      "activations/layer0_attention_weight_max": 16.157575607299805,
      "activations/layer0_attention_weight_min": -14.324502944946289,
      "activations/layer10_attention_weight_max": 37.815059661865234,
      "activations/layer10_attention_weight_min": -36.73566436767578,
      "activations/layer11_attention_weight_max": 37.46697998046875,
      "activations/layer11_attention_weight_min": -35.91667556762695,
      "activations/layer12_attention_weight_max": 19.461647033691406,
      "activations/layer12_attention_weight_min": -27.01581573486328,
      "activations/layer13_attention_weight_max": 34.55404281616211,
      "activations/layer13_attention_weight_min": -29.865764617919922,
      "activations/layer14_attention_weight_max": 35.822723388671875,
      "activations/layer14_attention_weight_min": -31.599319458007812,
      "activations/layer15_attention_weight_max": 34.47514724731445,
      "activations/layer15_attention_weight_min": -36.06471252441406,
      "activations/layer16_attention_weight_max": 37.93898010253906,
      "activations/layer16_attention_weight_min": -36.245853424072266,
      "activations/layer17_attention_weight_max": 54.59801483154297,
      "activations/layer17_attention_weight_min": -49.752769470214844,
      "activations/layer18_attention_weight_max": 50.545555114746094,
      "activations/layer18_attention_weight_min": -45.247894287109375,
      "activations/layer19_attention_weight_max": 25.101627349853516,
      "activations/layer19_attention_weight_min": -24.08803367614746,
      "activations/layer1_attention_weight_max": 15.275131225585938,
      "activations/layer1_attention_weight_min": -15.272894859313965,
      "activations/layer20_attention_weight_max": 24.96828842163086,
      "activations/layer20_attention_weight_min": -21.860061645507812,
      "activations/layer21_attention_weight_max": 43.80250549316406,
      "activations/layer21_attention_weight_min": -20.355712890625,
      "activations/layer22_attention_weight_max": 31.64103126525879,
      "activations/layer22_attention_weight_min": -24.987133026123047,
      "activations/layer23_attention_weight_max": 38.101478576660156,
      "activations/layer23_attention_weight_min": -24.212080001831055,
      "activations/layer2_attention_weight_max": 32.86669158935547,
      "activations/layer2_attention_weight_min": -31.07830047607422,
      "activations/layer3_attention_weight_max": 91.9828109741211,
      "activations/layer3_attention_weight_min": -93.71179962158203,
      "activations/layer4_attention_weight_max": 87.23992919921875,
      "activations/layer4_attention_weight_min": -88.38372039794922,
      "activations/layer5_attention_weight_max": 62.297664642333984,
      "activations/layer5_attention_weight_min": -72.35757446289062,
      "activations/layer6_attention_weight_max": 49.58427429199219,
      "activations/layer6_attention_weight_min": -51.55525588989258,
      "activations/layer7_attention_weight_max": 68.5564193725586,
      "activations/layer7_attention_weight_min": -68.19490814208984,
      "activations/layer8_attention_weight_max": 47.31294250488281,
      "activations/layer8_attention_weight_min": -53.25712966918945,
      "activations/layer9_attention_weight_max": 44.24103927612305,
      "activations/layer9_attention_weight_min": -46.4799919128418,
      "epoch": 23.17,
      "learning_rate": 5.74621212121212e-07,
      "loss": 2.7008,
      "step": 398800
    },
    {
      "activations/layer0_attention_weight_max": 15.35920524597168,
      "activations/layer0_attention_weight_min": -14.127179145812988,
      "activations/layer10_attention_weight_max": 38.741973876953125,
      "activations/layer10_attention_weight_min": -37.73896026611328,
      "activations/layer11_attention_weight_max": 34.64250183105469,
      "activations/layer11_attention_weight_min": -36.90578842163086,
      "activations/layer12_attention_weight_max": 20.823150634765625,
      "activations/layer12_attention_weight_min": -23.82004737854004,
      "activations/layer13_attention_weight_max": 33.55366134643555,
      "activations/layer13_attention_weight_min": -26.81772232055664,
      "activations/layer14_attention_weight_max": 37.354705810546875,
      "activations/layer14_attention_weight_min": -31.94369888305664,
      "activations/layer15_attention_weight_max": 33.10422897338867,
      "activations/layer15_attention_weight_min": -30.348695755004883,
      "activations/layer16_attention_weight_max": 38.60360336303711,
      "activations/layer16_attention_weight_min": -34.44337463378906,
      "activations/layer17_attention_weight_max": 55.71745300292969,
      "activations/layer17_attention_weight_min": -48.39372253417969,
      "activations/layer18_attention_weight_max": 53.64546585083008,
      "activations/layer18_attention_weight_min": -40.74604034423828,
      "activations/layer19_attention_weight_max": 27.024621963500977,
      "activations/layer19_attention_weight_min": -23.369497299194336,
      "activations/layer1_attention_weight_max": 16.36257553100586,
      "activations/layer1_attention_weight_min": -14.952981948852539,
      "activations/layer20_attention_weight_max": 24.61825180053711,
      "activations/layer20_attention_weight_min": -20.847013473510742,
      "activations/layer21_attention_weight_max": 31.412822723388672,
      "activations/layer21_attention_weight_min": -21.395593643188477,
      "activations/layer22_attention_weight_max": 30.558820724487305,
      "activations/layer22_attention_weight_min": -23.415315628051758,
      "activations/layer23_attention_weight_max": 38.87729263305664,
      "activations/layer23_attention_weight_min": -22.699613571166992,
      "activations/layer2_attention_weight_max": 33.43323516845703,
      "activations/layer2_attention_weight_min": -31.28451919555664,
      "activations/layer3_attention_weight_max": 98.13099670410156,
      "activations/layer3_attention_weight_min": -92.90813446044922,
      "activations/layer4_attention_weight_max": 87.89187622070312,
      "activations/layer4_attention_weight_min": -87.60352325439453,
      "activations/layer5_attention_weight_max": 64.95777130126953,
      "activations/layer5_attention_weight_min": -72.2090835571289,
      "activations/layer6_attention_weight_max": 49.416419982910156,
      "activations/layer6_attention_weight_min": -50.7252311706543,
      "activations/layer7_attention_weight_max": 70.67345428466797,
      "activations/layer7_attention_weight_min": -72.12666320800781,
      "activations/layer8_attention_weight_max": 47.78459548950195,
      "activations/layer8_attention_weight_min": -50.00706100463867,
      "activations/layer9_attention_weight_max": 45.564517974853516,
      "activations/layer9_attention_weight_min": -47.342227935791016,
      "epoch": 23.18,
      "learning_rate": 5.556818181818182e-07,
      "loss": 2.7087,
      "step": 398850
    },
    {
      "activations/layer0_attention_weight_max": 17.347919464111328,
      "activations/layer0_attention_weight_min": -14.102566719055176,
      "activations/layer10_attention_weight_max": 36.81714630126953,
      "activations/layer10_attention_weight_min": -36.545833587646484,
      "activations/layer11_attention_weight_max": 34.97254180908203,
      "activations/layer11_attention_weight_min": -34.76813507080078,
      "activations/layer12_attention_weight_max": 19.375940322875977,
      "activations/layer12_attention_weight_min": -24.683448791503906,
      "activations/layer13_attention_weight_max": 36.15102005004883,
      "activations/layer13_attention_weight_min": -26.97623634338379,
      "activations/layer14_attention_weight_max": 35.99281692504883,
      "activations/layer14_attention_weight_min": -30.0887451171875,
      "activations/layer15_attention_weight_max": 37.02156066894531,
      "activations/layer15_attention_weight_min": -31.504770278930664,
      "activations/layer16_attention_weight_max": 35.6946907043457,
      "activations/layer16_attention_weight_min": -32.89357376098633,
      "activations/layer17_attention_weight_max": 51.72218322753906,
      "activations/layer17_attention_weight_min": -46.594276428222656,
      "activations/layer18_attention_weight_max": 47.003292083740234,
      "activations/layer18_attention_weight_min": -40.69968032836914,
      "activations/layer19_attention_weight_max": 24.806072235107422,
      "activations/layer19_attention_weight_min": -23.54022789001465,
      "activations/layer1_attention_weight_max": 15.433085441589355,
      "activations/layer1_attention_weight_min": -15.306235313415527,
      "activations/layer20_attention_weight_max": 26.023820877075195,
      "activations/layer20_attention_weight_min": -22.82463836669922,
      "activations/layer21_attention_weight_max": 37.4918327331543,
      "activations/layer21_attention_weight_min": -20.63083839416504,
      "activations/layer22_attention_weight_max": 31.619770050048828,
      "activations/layer22_attention_weight_min": -25.19077491760254,
      "activations/layer23_attention_weight_max": 37.65459442138672,
      "activations/layer23_attention_weight_min": -24.42467498779297,
      "activations/layer2_attention_weight_max": 34.769386291503906,
      "activations/layer2_attention_weight_min": -29.868085861206055,
      "activations/layer3_attention_weight_max": 94.53836059570312,
      "activations/layer3_attention_weight_min": -92.46949005126953,
      "activations/layer4_attention_weight_max": 87.40001678466797,
      "activations/layer4_attention_weight_min": -87.0423812866211,
      "activations/layer5_attention_weight_max": 64.21731567382812,
      "activations/layer5_attention_weight_min": -74.71922302246094,
      "activations/layer6_attention_weight_max": 52.72996520996094,
      "activations/layer6_attention_weight_min": -53.91035461425781,
      "activations/layer7_attention_weight_max": 71.25476837158203,
      "activations/layer7_attention_weight_min": -68.72899627685547,
      "activations/layer8_attention_weight_max": 50.76050567626953,
      "activations/layer8_attention_weight_min": -53.380592346191406,
      "activations/layer9_attention_weight_max": 47.1043815612793,
      "activations/layer9_attention_weight_min": -47.78369903564453,
      "epoch": 23.18,
      "learning_rate": 5.367424242424241e-07,
      "loss": 2.6999,
      "step": 398900
    },
    {
      "activations/layer0_attention_weight_max": 15.728973388671875,
      "activations/layer0_attention_weight_min": -14.273784637451172,
      "activations/layer10_attention_weight_max": 34.63680648803711,
      "activations/layer10_attention_weight_min": -34.61061477661133,
      "activations/layer11_attention_weight_max": 32.719017028808594,
      "activations/layer11_attention_weight_min": -34.602012634277344,
      "activations/layer12_attention_weight_max": 18.233640670776367,
      "activations/layer12_attention_weight_min": -25.081417083740234,
      "activations/layer13_attention_weight_max": 31.807861328125,
      "activations/layer13_attention_weight_min": -26.02141761779785,
      "activations/layer14_attention_weight_max": 35.385738372802734,
      "activations/layer14_attention_weight_min": -28.68185043334961,
      "activations/layer15_attention_weight_max": 30.912616729736328,
      "activations/layer15_attention_weight_min": -29.23031997680664,
      "activations/layer16_attention_weight_max": 31.017940521240234,
      "activations/layer16_attention_weight_min": -30.318761825561523,
      "activations/layer17_attention_weight_max": 48.49689865112305,
      "activations/layer17_attention_weight_min": -41.12480545043945,
      "activations/layer18_attention_weight_max": 41.33380889892578,
      "activations/layer18_attention_weight_min": -35.89863586425781,
      "activations/layer19_attention_weight_max": 23.652965545654297,
      "activations/layer19_attention_weight_min": -20.744203567504883,
      "activations/layer1_attention_weight_max": 15.499251365661621,
      "activations/layer1_attention_weight_min": -15.821436882019043,
      "activations/layer20_attention_weight_max": 21.503063201904297,
      "activations/layer20_attention_weight_min": -21.239421844482422,
      "activations/layer21_attention_weight_max": 29.202621459960938,
      "activations/layer21_attention_weight_min": -19.903966903686523,
      "activations/layer22_attention_weight_max": 27.30311393737793,
      "activations/layer22_attention_weight_min": -23.04210090637207,
      "activations/layer23_attention_weight_max": 34.625492095947266,
      "activations/layer23_attention_weight_min": -22.161823272705078,
      "activations/layer2_attention_weight_max": 32.48698043823242,
      "activations/layer2_attention_weight_min": -31.3159236907959,
      "activations/layer3_attention_weight_max": 92.8027572631836,
      "activations/layer3_attention_weight_min": -90.34895324707031,
      "activations/layer4_attention_weight_max": 87.71868133544922,
      "activations/layer4_attention_weight_min": -88.42574310302734,
      "activations/layer5_attention_weight_max": 63.56285095214844,
      "activations/layer5_attention_weight_min": -71.36151885986328,
      "activations/layer6_attention_weight_max": 49.82613754272461,
      "activations/layer6_attention_weight_min": -50.83989715576172,
      "activations/layer7_attention_weight_max": 70.05056762695312,
      "activations/layer7_attention_weight_min": -71.85515594482422,
      "activations/layer8_attention_weight_max": 50.840599060058594,
      "activations/layer8_attention_weight_min": -51.07661437988281,
      "activations/layer9_attention_weight_max": 51.201629638671875,
      "activations/layer9_attention_weight_min": -46.542503356933594,
      "epoch": 23.18,
      "learning_rate": 5.178030303030302e-07,
      "loss": 2.7044,
      "step": 398950
    },
    {
      "activations/layer0_attention_weight_max": 15.768498420715332,
      "activations/layer0_attention_weight_min": -14.221808433532715,
      "activations/layer10_attention_weight_max": 38.7524528503418,
      "activations/layer10_attention_weight_min": -38.14055633544922,
      "activations/layer11_attention_weight_max": 34.36089324951172,
      "activations/layer11_attention_weight_min": -35.99823760986328,
      "activations/layer12_attention_weight_max": 19.848690032958984,
      "activations/layer12_attention_weight_min": -26.080707550048828,
      "activations/layer13_attention_weight_max": 36.0634765625,
      "activations/layer13_attention_weight_min": -27.634164810180664,
      "activations/layer14_attention_weight_max": 38.80191421508789,
      "activations/layer14_attention_weight_min": -34.77974319458008,
      "activations/layer15_attention_weight_max": 43.3162956237793,
      "activations/layer15_attention_weight_min": -35.86405563354492,
      "activations/layer16_attention_weight_max": 36.73228454589844,
      "activations/layer16_attention_weight_min": -36.140838623046875,
      "activations/layer17_attention_weight_max": 51.15178298950195,
      "activations/layer17_attention_weight_min": -53.35477066040039,
      "activations/layer18_attention_weight_max": 47.784515380859375,
      "activations/layer18_attention_weight_min": -44.322998046875,
      "activations/layer19_attention_weight_max": 23.44615936279297,
      "activations/layer19_attention_weight_min": -22.53885841369629,
      "activations/layer1_attention_weight_max": 16.211132049560547,
      "activations/layer1_attention_weight_min": -16.175474166870117,
      "activations/layer20_attention_weight_max": 22.948240280151367,
      "activations/layer20_attention_weight_min": -19.51573944091797,
      "activations/layer21_attention_weight_max": 35.169586181640625,
      "activations/layer21_attention_weight_min": -21.909852981567383,
      "activations/layer22_attention_weight_max": 29.14912986755371,
      "activations/layer22_attention_weight_min": -25.9522762298584,
      "activations/layer23_attention_weight_max": 39.01502990722656,
      "activations/layer23_attention_weight_min": -23.775663375854492,
      "activations/layer2_attention_weight_max": 32.3101692199707,
      "activations/layer2_attention_weight_min": -29.782997131347656,
      "activations/layer3_attention_weight_max": 97.22401428222656,
      "activations/layer3_attention_weight_min": -91.73646545410156,
      "activations/layer4_attention_weight_max": 94.19656372070312,
      "activations/layer4_attention_weight_min": -90.0510482788086,
      "activations/layer5_attention_weight_max": 66.33815002441406,
      "activations/layer5_attention_weight_min": -73.79925537109375,
      "activations/layer6_attention_weight_max": 53.14610290527344,
      "activations/layer6_attention_weight_min": -54.16091537475586,
      "activations/layer7_attention_weight_max": 72.3809585571289,
      "activations/layer7_attention_weight_min": -75.18724822998047,
      "activations/layer8_attention_weight_max": 52.99360275268555,
      "activations/layer8_attention_weight_min": -57.19684600830078,
      "activations/layer9_attention_weight_max": 51.89249801635742,
      "activations/layer9_attention_weight_min": -53.3521728515625,
      "epoch": 23.18,
      "learning_rate": 4.988636363636363e-07,
      "loss": 2.6726,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.5165,
      "eval_samples_per_second": 504.198,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.5165,
      "eval_openwebtext_samples_per_second": 504.198,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 2.0059,
      "eval_wikitext_samples_per_second": 227.332,
      "step": 399000
    },
    {
      "epoch": 23.18,
      "eval_lambada_loss": 2.392578125,
      "eval_lambada_ppl": 10.941666595318791,
      "eval_lambada_runtime": 9.5973,
      "eval_lambada_samples_per_second": 507.331,
      "step": 399000
    },
    {
      "activations/layer0_attention_weight_max": 16.043521881103516,
      "activations/layer0_attention_weight_min": -14.55347728729248,
      "activations/layer10_attention_weight_max": 36.68374252319336,
      "activations/layer10_attention_weight_min": -39.26200485229492,
      "activations/layer11_attention_weight_max": 34.37919616699219,
      "activations/layer11_attention_weight_min": -37.7584342956543,
      "activations/layer12_attention_weight_max": 18.889163970947266,
      "activations/layer12_attention_weight_min": -26.727333068847656,
      "activations/layer13_attention_weight_max": 31.678380966186523,
      "activations/layer13_attention_weight_min": -30.703689575195312,
      "activations/layer14_attention_weight_max": 35.94103240966797,
      "activations/layer14_attention_weight_min": -30.397506713867188,
      "activations/layer15_attention_weight_max": 35.36348342895508,
      "activations/layer15_attention_weight_min": -31.203710556030273,
      "activations/layer16_attention_weight_max": 33.18146514892578,
      "activations/layer16_attention_weight_min": -32.6663932800293,
      "activations/layer17_attention_weight_max": 51.912113189697266,
      "activations/layer17_attention_weight_min": -47.94747543334961,
      "activations/layer18_attention_weight_max": 47.34416961669922,
      "activations/layer18_attention_weight_min": -43.05455017089844,
      "activations/layer19_attention_weight_max": 26.491724014282227,
      "activations/layer19_attention_weight_min": -25.36858367919922,
      "activations/layer1_attention_weight_max": 15.727938652038574,
      "activations/layer1_attention_weight_min": -15.599863052368164,
      "activations/layer20_attention_weight_max": 24.117748260498047,
      "activations/layer20_attention_weight_min": -23.535625457763672,
      "activations/layer21_attention_weight_max": 34.52650833129883,
      "activations/layer21_attention_weight_min": -23.23910903930664,
      "activations/layer22_attention_weight_max": 27.816810607910156,
      "activations/layer22_attention_weight_min": -26.188156127929688,
      "activations/layer23_attention_weight_max": 33.617279052734375,
      "activations/layer23_attention_weight_min": -25.860477447509766,
      "activations/layer2_attention_weight_max": 33.80329895019531,
      "activations/layer2_attention_weight_min": -32.03447723388672,
      "activations/layer3_attention_weight_max": 92.66130065917969,
      "activations/layer3_attention_weight_min": -94.14006805419922,
      "activations/layer4_attention_weight_max": 87.79122161865234,
      "activations/layer4_attention_weight_min": -85.08106994628906,
      "activations/layer5_attention_weight_max": 61.201332092285156,
      "activations/layer5_attention_weight_min": -70.97137451171875,
      "activations/layer6_attention_weight_max": 49.890281677246094,
      "activations/layer6_attention_weight_min": -50.76717758178711,
      "activations/layer7_attention_weight_max": 71.65538024902344,
      "activations/layer7_attention_weight_min": -68.86094665527344,
      "activations/layer8_attention_weight_max": 49.027462005615234,
      "activations/layer8_attention_weight_min": -52.430423736572266,
      "activations/layer9_attention_weight_max": 46.13705062866211,
      "activations/layer9_attention_weight_min": -46.7723388671875,
      "epoch": 23.19,
      "learning_rate": 4.799242424242424e-07,
      "loss": 2.6879,
      "step": 399050
    },
    {
      "activations/layer0_attention_weight_max": 15.78688907623291,
      "activations/layer0_attention_weight_min": -14.303424835205078,
      "activations/layer10_attention_weight_max": 36.00715637207031,
      "activations/layer10_attention_weight_min": -36.34292984008789,
      "activations/layer11_attention_weight_max": 36.46580123901367,
      "activations/layer11_attention_weight_min": -37.14375305175781,
      "activations/layer12_attention_weight_max": 19.297121047973633,
      "activations/layer12_attention_weight_min": -25.478778839111328,
      "activations/layer13_attention_weight_max": 31.5501651763916,
      "activations/layer13_attention_weight_min": -26.231853485107422,
      "activations/layer14_attention_weight_max": 34.423362731933594,
      "activations/layer14_attention_weight_min": -29.990631103515625,
      "activations/layer15_attention_weight_max": 32.65034103393555,
      "activations/layer15_attention_weight_min": -30.76386070251465,
      "activations/layer16_attention_weight_max": 34.8848876953125,
      "activations/layer16_attention_weight_min": -32.5629997253418,
      "activations/layer17_attention_weight_max": 55.90886688232422,
      "activations/layer17_attention_weight_min": -45.10739517211914,
      "activations/layer18_attention_weight_max": 49.32052230834961,
      "activations/layer18_attention_weight_min": -39.556671142578125,
      "activations/layer19_attention_weight_max": 27.54628562927246,
      "activations/layer19_attention_weight_min": -22.160612106323242,
      "activations/layer1_attention_weight_max": 15.19195556640625,
      "activations/layer1_attention_weight_min": -14.98660945892334,
      "activations/layer20_attention_weight_max": 24.481204986572266,
      "activations/layer20_attention_weight_min": -22.122188568115234,
      "activations/layer21_attention_weight_max": 33.55651092529297,
      "activations/layer21_attention_weight_min": -23.878965377807617,
      "activations/layer22_attention_weight_max": 29.38376235961914,
      "activations/layer22_attention_weight_min": -25.33186149597168,
      "activations/layer23_attention_weight_max": 35.387264251708984,
      "activations/layer23_attention_weight_min": -22.235126495361328,
      "activations/layer2_attention_weight_max": 33.111572265625,
      "activations/layer2_attention_weight_min": -31.34020233154297,
      "activations/layer3_attention_weight_max": 92.04935455322266,
      "activations/layer3_attention_weight_min": -94.0657958984375,
      "activations/layer4_attention_weight_max": 88.01866149902344,
      "activations/layer4_attention_weight_min": -89.00819396972656,
      "activations/layer5_attention_weight_max": 66.65235137939453,
      "activations/layer5_attention_weight_min": -71.61602783203125,
      "activations/layer6_attention_weight_max": 51.46967315673828,
      "activations/layer6_attention_weight_min": -52.142677307128906,
      "activations/layer7_attention_weight_max": 70.59862518310547,
      "activations/layer7_attention_weight_min": -69.6442642211914,
      "activations/layer8_attention_weight_max": 49.81256103515625,
      "activations/layer8_attention_weight_min": -51.74763870239258,
      "activations/layer9_attention_weight_max": 49.32334518432617,
      "activations/layer9_attention_weight_min": -48.11672592163086,
      "epoch": 23.19,
      "learning_rate": 4.6098484848484843e-07,
      "loss": 2.6942,
      "step": 399100
    },
    {
      "activations/layer0_attention_weight_max": 15.91805362701416,
      "activations/layer0_attention_weight_min": -14.295598030090332,
      "activations/layer10_attention_weight_max": 35.4990119934082,
      "activations/layer10_attention_weight_min": -36.93587875366211,
      "activations/layer11_attention_weight_max": 34.478492736816406,
      "activations/layer11_attention_weight_min": -36.2893180847168,
      "activations/layer12_attention_weight_max": 20.625383377075195,
      "activations/layer12_attention_weight_min": -24.215091705322266,
      "activations/layer13_attention_weight_max": 34.321311950683594,
      "activations/layer13_attention_weight_min": -28.43819236755371,
      "activations/layer14_attention_weight_max": 36.735450744628906,
      "activations/layer14_attention_weight_min": -34.86316680908203,
      "activations/layer15_attention_weight_max": 35.740360260009766,
      "activations/layer15_attention_weight_min": -33.84246063232422,
      "activations/layer16_attention_weight_max": 38.6197395324707,
      "activations/layer16_attention_weight_min": -36.08901596069336,
      "activations/layer17_attention_weight_max": 54.3196907043457,
      "activations/layer17_attention_weight_min": -48.55697250366211,
      "activations/layer18_attention_weight_max": 51.9178466796875,
      "activations/layer18_attention_weight_min": -43.03602981567383,
      "activations/layer19_attention_weight_max": 27.8370361328125,
      "activations/layer19_attention_weight_min": -22.79345703125,
      "activations/layer1_attention_weight_max": 15.547557830810547,
      "activations/layer1_attention_weight_min": -15.517718315124512,
      "activations/layer20_attention_weight_max": 24.848894119262695,
      "activations/layer20_attention_weight_min": -21.232683181762695,
      "activations/layer21_attention_weight_max": 41.832244873046875,
      "activations/layer21_attention_weight_min": -22.063854217529297,
      "activations/layer22_attention_weight_max": 31.569042205810547,
      "activations/layer22_attention_weight_min": -25.7711124420166,
      "activations/layer23_attention_weight_max": 37.296539306640625,
      "activations/layer23_attention_weight_min": -23.25300407409668,
      "activations/layer2_attention_weight_max": 34.691497802734375,
      "activations/layer2_attention_weight_min": -31.09552001953125,
      "activations/layer3_attention_weight_max": 92.53644561767578,
      "activations/layer3_attention_weight_min": -93.45194244384766,
      "activations/layer4_attention_weight_max": 87.05236053466797,
      "activations/layer4_attention_weight_min": -90.30265045166016,
      "activations/layer5_attention_weight_max": 64.39958953857422,
      "activations/layer5_attention_weight_min": -75.39811706542969,
      "activations/layer6_attention_weight_max": 50.262428283691406,
      "activations/layer6_attention_weight_min": -51.32479476928711,
      "activations/layer7_attention_weight_max": 73.83702850341797,
      "activations/layer7_attention_weight_min": -72.63419342041016,
      "activations/layer8_attention_weight_max": 48.12838363647461,
      "activations/layer8_attention_weight_min": -50.612693786621094,
      "activations/layer9_attention_weight_max": 49.52228546142578,
      "activations/layer9_attention_weight_min": -48.8104248046875,
      "epoch": 23.19,
      "learning_rate": 4.4204545454545453e-07,
      "loss": 2.6838,
      "step": 399150
    },
    {
      "activations/layer0_attention_weight_max": 15.20960807800293,
      "activations/layer0_attention_weight_min": -14.537903785705566,
      "activations/layer10_attention_weight_max": 38.007568359375,
      "activations/layer10_attention_weight_min": -36.54682540893555,
      "activations/layer11_attention_weight_max": 37.909576416015625,
      "activations/layer11_attention_weight_min": -35.92809295654297,
      "activations/layer12_attention_weight_max": 20.735092163085938,
      "activations/layer12_attention_weight_min": -24.200469970703125,
      "activations/layer13_attention_weight_max": 30.213098526000977,
      "activations/layer13_attention_weight_min": -26.081327438354492,
      "activations/layer14_attention_weight_max": 33.533260345458984,
      "activations/layer14_attention_weight_min": -29.962804794311523,
      "activations/layer15_attention_weight_max": 30.318082809448242,
      "activations/layer15_attention_weight_min": -30.4457950592041,
      "activations/layer16_attention_weight_max": 32.965248107910156,
      "activations/layer16_attention_weight_min": -30.55286407470703,
      "activations/layer17_attention_weight_max": 51.164794921875,
      "activations/layer17_attention_weight_min": -44.4039192199707,
      "activations/layer18_attention_weight_max": 46.76402282714844,
      "activations/layer18_attention_weight_min": -38.81288528442383,
      "activations/layer19_attention_weight_max": 22.326213836669922,
      "activations/layer19_attention_weight_min": -23.463651657104492,
      "activations/layer1_attention_weight_max": 16.270401000976562,
      "activations/layer1_attention_weight_min": -14.798253059387207,
      "activations/layer20_attention_weight_max": 21.410978317260742,
      "activations/layer20_attention_weight_min": -22.905805587768555,
      "activations/layer21_attention_weight_max": 30.811288833618164,
      "activations/layer21_attention_weight_min": -23.297386169433594,
      "activations/layer22_attention_weight_max": 27.042768478393555,
      "activations/layer22_attention_weight_min": -25.96662139892578,
      "activations/layer23_attention_weight_max": 35.86787414550781,
      "activations/layer23_attention_weight_min": -23.85094451904297,
      "activations/layer2_attention_weight_max": 33.22239303588867,
      "activations/layer2_attention_weight_min": -30.093490600585938,
      "activations/layer3_attention_weight_max": 91.3674545288086,
      "activations/layer3_attention_weight_min": -89.39085388183594,
      "activations/layer4_attention_weight_max": 86.4371337890625,
      "activations/layer4_attention_weight_min": -83.36399841308594,
      "activations/layer5_attention_weight_max": 62.69383239746094,
      "activations/layer5_attention_weight_min": -70.8436050415039,
      "activations/layer6_attention_weight_max": 49.759769439697266,
      "activations/layer6_attention_weight_min": -52.515892028808594,
      "activations/layer7_attention_weight_max": 74.21946716308594,
      "activations/layer7_attention_weight_min": -68.79470825195312,
      "activations/layer8_attention_weight_max": 46.908485412597656,
      "activations/layer8_attention_weight_min": -50.80531692504883,
      "activations/layer9_attention_weight_max": 54.099403381347656,
      "activations/layer9_attention_weight_min": -45.444454193115234,
      "epoch": 23.2,
      "learning_rate": 4.231060606060606e-07,
      "loss": 2.7108,
      "step": 399200
    },
    {
      "activations/layer0_attention_weight_max": 15.903719902038574,
      "activations/layer0_attention_weight_min": -14.47093391418457,
      "activations/layer10_attention_weight_max": 40.034820556640625,
      "activations/layer10_attention_weight_min": -39.5847282409668,
      "activations/layer11_attention_weight_max": 38.97414016723633,
      "activations/layer11_attention_weight_min": -39.301300048828125,
      "activations/layer12_attention_weight_max": 19.632556915283203,
      "activations/layer12_attention_weight_min": -26.615339279174805,
      "activations/layer13_attention_weight_max": 40.09465789794922,
      "activations/layer13_attention_weight_min": -28.270137786865234,
      "activations/layer14_attention_weight_max": 47.99119567871094,
      "activations/layer14_attention_weight_min": -31.164003372192383,
      "activations/layer15_attention_weight_max": 44.88405227661133,
      "activations/layer15_attention_weight_min": -32.343353271484375,
      "activations/layer16_attention_weight_max": 37.74343490600586,
      "activations/layer16_attention_weight_min": -35.6044921875,
      "activations/layer17_attention_weight_max": 55.90257263183594,
      "activations/layer17_attention_weight_min": -51.98073196411133,
      "activations/layer18_attention_weight_max": 48.584861755371094,
      "activations/layer18_attention_weight_min": -46.68888854980469,
      "activations/layer19_attention_weight_max": 26.028366088867188,
      "activations/layer19_attention_weight_min": -25.335529327392578,
      "activations/layer1_attention_weight_max": 15.885018348693848,
      "activations/layer1_attention_weight_min": -14.983604431152344,
      "activations/layer20_attention_weight_max": 25.860950469970703,
      "activations/layer20_attention_weight_min": -22.863540649414062,
      "activations/layer21_attention_weight_max": 35.36744689941406,
      "activations/layer21_attention_weight_min": -23.989042282104492,
      "activations/layer22_attention_weight_max": 29.273653030395508,
      "activations/layer22_attention_weight_min": -25.793853759765625,
      "activations/layer23_attention_weight_max": 32.90277862548828,
      "activations/layer23_attention_weight_min": -24.30389404296875,
      "activations/layer2_attention_weight_max": 32.468299865722656,
      "activations/layer2_attention_weight_min": -31.72773551940918,
      "activations/layer3_attention_weight_max": 92.42225646972656,
      "activations/layer3_attention_weight_min": -93.310302734375,
      "activations/layer4_attention_weight_max": 88.07598114013672,
      "activations/layer4_attention_weight_min": -87.69185638427734,
      "activations/layer5_attention_weight_max": 62.082359313964844,
      "activations/layer5_attention_weight_min": -72.30245971679688,
      "activations/layer6_attention_weight_max": 50.88433074951172,
      "activations/layer6_attention_weight_min": -51.55964279174805,
      "activations/layer7_attention_weight_max": 73.4462890625,
      "activations/layer7_attention_weight_min": -73.4684829711914,
      "activations/layer8_attention_weight_max": 51.05170440673828,
      "activations/layer8_attention_weight_min": -54.51878356933594,
      "activations/layer9_attention_weight_max": 51.734554290771484,
      "activations/layer9_attention_weight_min": -49.82538604736328,
      "epoch": 23.2,
      "learning_rate": 4.041666666666667e-07,
      "loss": 2.6841,
      "step": 399250
    },
    {
      "activations/layer0_attention_weight_max": 16.47545051574707,
      "activations/layer0_attention_weight_min": -14.01473331451416,
      "activations/layer10_attention_weight_max": 36.119075775146484,
      "activations/layer10_attention_weight_min": -36.09641647338867,
      "activations/layer11_attention_weight_max": 33.92500305175781,
      "activations/layer11_attention_weight_min": -35.9607048034668,
      "activations/layer12_attention_weight_max": 19.696897506713867,
      "activations/layer12_attention_weight_min": -25.081607818603516,
      "activations/layer13_attention_weight_max": 35.12459945678711,
      "activations/layer13_attention_weight_min": -27.376995086669922,
      "activations/layer14_attention_weight_max": 34.0977668762207,
      "activations/layer14_attention_weight_min": -32.12650680541992,
      "activations/layer15_attention_weight_max": 35.49483871459961,
      "activations/layer15_attention_weight_min": -34.034751892089844,
      "activations/layer16_attention_weight_max": 36.210853576660156,
      "activations/layer16_attention_weight_min": -34.3409538269043,
      "activations/layer17_attention_weight_max": 51.932987213134766,
      "activations/layer17_attention_weight_min": -46.86811447143555,
      "activations/layer18_attention_weight_max": 50.23488998413086,
      "activations/layer18_attention_weight_min": -42.21345138549805,
      "activations/layer19_attention_weight_max": 27.46405029296875,
      "activations/layer19_attention_weight_min": -24.429292678833008,
      "activations/layer1_attention_weight_max": 15.17809009552002,
      "activations/layer1_attention_weight_min": -14.768341064453125,
      "activations/layer20_attention_weight_max": 24.445903778076172,
      "activations/layer20_attention_weight_min": -22.103605270385742,
      "activations/layer21_attention_weight_max": 37.29668426513672,
      "activations/layer21_attention_weight_min": -23.36139488220215,
      "activations/layer22_attention_weight_max": 31.20530891418457,
      "activations/layer22_attention_weight_min": -26.744749069213867,
      "activations/layer23_attention_weight_max": 36.975669860839844,
      "activations/layer23_attention_weight_min": -25.4052734375,
      "activations/layer2_attention_weight_max": 31.644962310791016,
      "activations/layer2_attention_weight_min": -30.413419723510742,
      "activations/layer3_attention_weight_max": 89.2518539428711,
      "activations/layer3_attention_weight_min": -89.53990936279297,
      "activations/layer4_attention_weight_max": 87.19147491455078,
      "activations/layer4_attention_weight_min": -84.29449462890625,
      "activations/layer5_attention_weight_max": 61.15043258666992,
      "activations/layer5_attention_weight_min": -72.2452392578125,
      "activations/layer6_attention_weight_max": 48.365989685058594,
      "activations/layer6_attention_weight_min": -49.54553985595703,
      "activations/layer7_attention_weight_max": 68.94503784179688,
      "activations/layer7_attention_weight_min": -67.92127990722656,
      "activations/layer8_attention_weight_max": 45.80388259887695,
      "activations/layer8_attention_weight_min": -49.216941833496094,
      "activations/layer9_attention_weight_max": 43.0655632019043,
      "activations/layer9_attention_weight_min": -46.42341613769531,
      "epoch": 23.2,
      "learning_rate": 3.8522727272727267e-07,
      "loss": 2.701,
      "step": 399300
    },
    {
      "activations/layer0_attention_weight_max": 15.749649047851562,
      "activations/layer0_attention_weight_min": -14.32838249206543,
      "activations/layer10_attention_weight_max": 35.418949127197266,
      "activations/layer10_attention_weight_min": -36.25852584838867,
      "activations/layer11_attention_weight_max": 33.194095611572266,
      "activations/layer11_attention_weight_min": -35.807594299316406,
      "activations/layer12_attention_weight_max": 19.12730598449707,
      "activations/layer12_attention_weight_min": -25.94040298461914,
      "activations/layer13_attention_weight_max": 33.26369094848633,
      "activations/layer13_attention_weight_min": -27.925209045410156,
      "activations/layer14_attention_weight_max": 38.23185348510742,
      "activations/layer14_attention_weight_min": -32.01782989501953,
      "activations/layer15_attention_weight_max": 33.752525329589844,
      "activations/layer15_attention_weight_min": -31.38182830810547,
      "activations/layer16_attention_weight_max": 36.48654556274414,
      "activations/layer16_attention_weight_min": -32.86589813232422,
      "activations/layer17_attention_weight_max": 53.71114730834961,
      "activations/layer17_attention_weight_min": -46.24984359741211,
      "activations/layer18_attention_weight_max": 50.61580276489258,
      "activations/layer18_attention_weight_min": -43.874839782714844,
      "activations/layer19_attention_weight_max": 23.868762969970703,
      "activations/layer19_attention_weight_min": -24.796733856201172,
      "activations/layer1_attention_weight_max": 15.952230453491211,
      "activations/layer1_attention_weight_min": -14.422216415405273,
      "activations/layer20_attention_weight_max": 22.22815704345703,
      "activations/layer20_attention_weight_min": -22.648895263671875,
      "activations/layer21_attention_weight_max": 32.89829635620117,
      "activations/layer21_attention_weight_min": -20.691740036010742,
      "activations/layer22_attention_weight_max": 28.714385986328125,
      "activations/layer22_attention_weight_min": -24.844329833984375,
      "activations/layer23_attention_weight_max": 34.556121826171875,
      "activations/layer23_attention_weight_min": -23.139511108398438,
      "activations/layer2_attention_weight_max": 31.923538208007812,
      "activations/layer2_attention_weight_min": -31.440959930419922,
      "activations/layer3_attention_weight_max": 92.55767059326172,
      "activations/layer3_attention_weight_min": -91.71516418457031,
      "activations/layer4_attention_weight_max": 85.30460357666016,
      "activations/layer4_attention_weight_min": -88.43497467041016,
      "activations/layer5_attention_weight_max": 63.183204650878906,
      "activations/layer5_attention_weight_min": -72.06783294677734,
      "activations/layer6_attention_weight_max": 48.3530387878418,
      "activations/layer6_attention_weight_min": -51.38755798339844,
      "activations/layer7_attention_weight_max": 68.5279769897461,
      "activations/layer7_attention_weight_min": -72.19071960449219,
      "activations/layer8_attention_weight_max": 48.38469314575195,
      "activations/layer8_attention_weight_min": -50.6539192199707,
      "activations/layer9_attention_weight_max": 46.56131362915039,
      "activations/layer9_attention_weight_min": -51.99101638793945,
      "epoch": 23.2,
      "learning_rate": 3.662878787878787e-07,
      "loss": 2.6828,
      "step": 399350
    },
    {
      "activations/layer0_attention_weight_max": 15.402749061584473,
      "activations/layer0_attention_weight_min": -14.140254974365234,
      "activations/layer10_attention_weight_max": 37.755889892578125,
      "activations/layer10_attention_weight_min": -39.33770751953125,
      "activations/layer11_attention_weight_max": 34.87021255493164,
      "activations/layer11_attention_weight_min": -36.316131591796875,
      "activations/layer12_attention_weight_max": 19.6792049407959,
      "activations/layer12_attention_weight_min": -26.44353485107422,
      "activations/layer13_attention_weight_max": 42.71330261230469,
      "activations/layer13_attention_weight_min": -28.02910041809082,
      "activations/layer14_attention_weight_max": 33.71096420288086,
      "activations/layer14_attention_weight_min": -33.0802001953125,
      "activations/layer15_attention_weight_max": 32.5204963684082,
      "activations/layer15_attention_weight_min": -30.773033142089844,
      "activations/layer16_attention_weight_max": 33.373138427734375,
      "activations/layer16_attention_weight_min": -32.81808853149414,
      "activations/layer17_attention_weight_max": 52.74489974975586,
      "activations/layer17_attention_weight_min": -48.265647888183594,
      "activations/layer18_attention_weight_max": 47.42130661010742,
      "activations/layer18_attention_weight_min": -40.70475387573242,
      "activations/layer19_attention_weight_max": 25.352859497070312,
      "activations/layer19_attention_weight_min": -23.75894546508789,
      "activations/layer1_attention_weight_max": 15.834829330444336,
      "activations/layer1_attention_weight_min": -15.614226341247559,
      "activations/layer20_attention_weight_max": 21.7949161529541,
      "activations/layer20_attention_weight_min": -22.600425720214844,
      "activations/layer21_attention_weight_max": 37.593971252441406,
      "activations/layer21_attention_weight_min": -23.173330307006836,
      "activations/layer22_attention_weight_max": 26.586673736572266,
      "activations/layer22_attention_weight_min": -25.958148956298828,
      "activations/layer23_attention_weight_max": 33.05274963378906,
      "activations/layer23_attention_weight_min": -24.33570671081543,
      "activations/layer2_attention_weight_max": 33.440467834472656,
      "activations/layer2_attention_weight_min": -31.400108337402344,
      "activations/layer3_attention_weight_max": 94.4894790649414,
      "activations/layer3_attention_weight_min": -93.55236053466797,
      "activations/layer4_attention_weight_max": 88.5152359008789,
      "activations/layer4_attention_weight_min": -88.65898132324219,
      "activations/layer5_attention_weight_max": 63.52973556518555,
      "activations/layer5_attention_weight_min": -78.57708740234375,
      "activations/layer6_attention_weight_max": 50.40371322631836,
      "activations/layer6_attention_weight_min": -51.85396194458008,
      "activations/layer7_attention_weight_max": 71.54571533203125,
      "activations/layer7_attention_weight_min": -72.35552215576172,
      "activations/layer8_attention_weight_max": 50.64375686645508,
      "activations/layer8_attention_weight_min": -51.58725357055664,
      "activations/layer9_attention_weight_max": 50.710975646972656,
      "activations/layer9_attention_weight_min": -49.7461051940918,
      "epoch": 23.21,
      "learning_rate": 3.473484848484848e-07,
      "loss": 2.6927,
      "step": 399400
    },
    {
      "activations/layer0_attention_weight_max": 15.200143814086914,
      "activations/layer0_attention_weight_min": -14.319210052490234,
      "activations/layer10_attention_weight_max": 34.482574462890625,
      "activations/layer10_attention_weight_min": -34.62202835083008,
      "activations/layer11_attention_weight_max": 34.67644500732422,
      "activations/layer11_attention_weight_min": -33.863502502441406,
      "activations/layer12_attention_weight_max": 19.615644454956055,
      "activations/layer12_attention_weight_min": -29.32386016845703,
      "activations/layer13_attention_weight_max": 35.17607879638672,
      "activations/layer13_attention_weight_min": -27.131284713745117,
      "activations/layer14_attention_weight_max": 35.19576644897461,
      "activations/layer14_attention_weight_min": -33.68815231323242,
      "activations/layer15_attention_weight_max": 35.545345306396484,
      "activations/layer15_attention_weight_min": -33.701107025146484,
      "activations/layer16_attention_weight_max": 35.9454460144043,
      "activations/layer16_attention_weight_min": -34.26222229003906,
      "activations/layer17_attention_weight_max": 51.397098541259766,
      "activations/layer17_attention_weight_min": -48.02003479003906,
      "activations/layer18_attention_weight_max": 48.22220230102539,
      "activations/layer18_attention_weight_min": -42.675716400146484,
      "activations/layer19_attention_weight_max": 26.579233169555664,
      "activations/layer19_attention_weight_min": -22.85920524597168,
      "activations/layer1_attention_weight_max": 15.569432258605957,
      "activations/layer1_attention_weight_min": -14.179207801818848,
      "activations/layer20_attention_weight_max": 24.8531436920166,
      "activations/layer20_attention_weight_min": -23.033580780029297,
      "activations/layer21_attention_weight_max": 37.222198486328125,
      "activations/layer21_attention_weight_min": -21.424678802490234,
      "activations/layer22_attention_weight_max": 33.839534759521484,
      "activations/layer22_attention_weight_min": -24.37654685974121,
      "activations/layer23_attention_weight_max": 42.20768737792969,
      "activations/layer23_attention_weight_min": -25.52870750427246,
      "activations/layer2_attention_weight_max": 30.908367156982422,
      "activations/layer2_attention_weight_min": -29.347461700439453,
      "activations/layer3_attention_weight_max": 91.48762512207031,
      "activations/layer3_attention_weight_min": -90.6058578491211,
      "activations/layer4_attention_weight_max": 86.04914093017578,
      "activations/layer4_attention_weight_min": -87.25580596923828,
      "activations/layer5_attention_weight_max": 63.86210250854492,
      "activations/layer5_attention_weight_min": -74.39793395996094,
      "activations/layer6_attention_weight_max": 48.91080093383789,
      "activations/layer6_attention_weight_min": -51.22829055786133,
      "activations/layer7_attention_weight_max": 69.89382934570312,
      "activations/layer7_attention_weight_min": -69.96147155761719,
      "activations/layer8_attention_weight_max": 47.795345306396484,
      "activations/layer8_attention_weight_min": -52.5275764465332,
      "activations/layer9_attention_weight_max": 44.68937301635742,
      "activations/layer9_attention_weight_min": -46.85342025756836,
      "epoch": 23.21,
      "learning_rate": 3.2840909090909086e-07,
      "loss": 2.6936,
      "step": 399450
    },
    {
      "activations/layer0_attention_weight_max": 16.729820251464844,
      "activations/layer0_attention_weight_min": -14.01718807220459,
      "activations/layer10_attention_weight_max": 33.03642272949219,
      "activations/layer10_attention_weight_min": -35.00228500366211,
      "activations/layer11_attention_weight_max": 33.860015869140625,
      "activations/layer11_attention_weight_min": -36.30909729003906,
      "activations/layer12_attention_weight_max": 19.73969268798828,
      "activations/layer12_attention_weight_min": -23.936866760253906,
      "activations/layer13_attention_weight_max": 33.134765625,
      "activations/layer13_attention_weight_min": -27.923532485961914,
      "activations/layer14_attention_weight_max": 40.87360763549805,
      "activations/layer14_attention_weight_min": -33.83929443359375,
      "activations/layer15_attention_weight_max": 37.783233642578125,
      "activations/layer15_attention_weight_min": -33.86904525756836,
      "activations/layer16_attention_weight_max": 41.44215774536133,
      "activations/layer16_attention_weight_min": -38.00690460205078,
      "activations/layer17_attention_weight_max": 55.20944595336914,
      "activations/layer17_attention_weight_min": -50.937400817871094,
      "activations/layer18_attention_weight_max": 53.44593811035156,
      "activations/layer18_attention_weight_min": -46.81936264038086,
      "activations/layer19_attention_weight_max": 25.54694938659668,
      "activations/layer19_attention_weight_min": -28.659687042236328,
      "activations/layer1_attention_weight_max": 15.008621215820312,
      "activations/layer1_attention_weight_min": -14.563843727111816,
      "activations/layer20_attention_weight_max": 25.462024688720703,
      "activations/layer20_attention_weight_min": -25.964323043823242,
      "activations/layer21_attention_weight_max": 34.980831146240234,
      "activations/layer21_attention_weight_min": -23.610136032104492,
      "activations/layer22_attention_weight_max": 29.84540367126465,
      "activations/layer22_attention_weight_min": -27.897573471069336,
      "activations/layer23_attention_weight_max": 38.81056213378906,
      "activations/layer23_attention_weight_min": -26.572872161865234,
      "activations/layer2_attention_weight_max": 32.91417694091797,
      "activations/layer2_attention_weight_min": -30.923887252807617,
      "activations/layer3_attention_weight_max": 91.59229278564453,
      "activations/layer3_attention_weight_min": -90.0969467163086,
      "activations/layer4_attention_weight_max": 85.6299057006836,
      "activations/layer4_attention_weight_min": -86.21569061279297,
      "activations/layer5_attention_weight_max": 63.557247161865234,
      "activations/layer5_attention_weight_min": -76.55472564697266,
      "activations/layer6_attention_weight_max": 48.5416145324707,
      "activations/layer6_attention_weight_min": -50.6169319152832,
      "activations/layer7_attention_weight_max": 70.05033111572266,
      "activations/layer7_attention_weight_min": -71.66980743408203,
      "activations/layer8_attention_weight_max": 45.3292121887207,
      "activations/layer8_attention_weight_min": -52.63140106201172,
      "activations/layer9_attention_weight_max": 42.961036682128906,
      "activations/layer9_attention_weight_min": -45.42161560058594,
      "epoch": 23.21,
      "learning_rate": 3.0946969696969696e-07,
      "loss": 2.6962,
      "step": 399500
    },
    {
      "activations/layer0_attention_weight_max": 15.201370239257812,
      "activations/layer0_attention_weight_min": -14.192096710205078,
      "activations/layer10_attention_weight_max": 41.50688934326172,
      "activations/layer10_attention_weight_min": -42.71590042114258,
      "activations/layer11_attention_weight_max": 41.426910400390625,
      "activations/layer11_attention_weight_min": -41.95085906982422,
      "activations/layer12_attention_weight_max": 21.63599967956543,
      "activations/layer12_attention_weight_min": -24.36982536315918,
      "activations/layer13_attention_weight_max": 44.96853256225586,
      "activations/layer13_attention_weight_min": -34.710594177246094,
      "activations/layer14_attention_weight_max": 53.081871032714844,
      "activations/layer14_attention_weight_min": -36.560020446777344,
      "activations/layer15_attention_weight_max": 46.55845642089844,
      "activations/layer15_attention_weight_min": -34.34714889526367,
      "activations/layer16_attention_weight_max": 40.94166564941406,
      "activations/layer16_attention_weight_min": -38.55094909667969,
      "activations/layer17_attention_weight_max": 60.41696548461914,
      "activations/layer17_attention_weight_min": -53.90739440917969,
      "activations/layer18_attention_weight_max": 59.31505584716797,
      "activations/layer18_attention_weight_min": -49.39126205444336,
      "activations/layer19_attention_weight_max": 33.68337631225586,
      "activations/layer19_attention_weight_min": -27.389989852905273,
      "activations/layer1_attention_weight_max": 15.899210929870605,
      "activations/layer1_attention_weight_min": -16.32643699645996,
      "activations/layer20_attention_weight_max": 27.731122970581055,
      "activations/layer20_attention_weight_min": -24.450056076049805,
      "activations/layer21_attention_weight_max": 48.7111930847168,
      "activations/layer21_attention_weight_min": -29.087844848632812,
      "activations/layer22_attention_weight_max": 34.059791564941406,
      "activations/layer22_attention_weight_min": -27.881441116333008,
      "activations/layer23_attention_weight_max": 40.72604751586914,
      "activations/layer23_attention_weight_min": -24.340717315673828,
      "activations/layer2_attention_weight_max": 34.74005126953125,
      "activations/layer2_attention_weight_min": -31.36947250366211,
      "activations/layer3_attention_weight_max": 92.65056610107422,
      "activations/layer3_attention_weight_min": -93.80260467529297,
      "activations/layer4_attention_weight_max": 89.01995849609375,
      "activations/layer4_attention_weight_min": -89.39067077636719,
      "activations/layer5_attention_weight_max": 64.70222473144531,
      "activations/layer5_attention_weight_min": -72.55703735351562,
      "activations/layer6_attention_weight_max": 51.13689041137695,
      "activations/layer6_attention_weight_min": -54.393245697021484,
      "activations/layer7_attention_weight_max": 79.94935607910156,
      "activations/layer7_attention_weight_min": -74.49596405029297,
      "activations/layer8_attention_weight_max": 53.0651741027832,
      "activations/layer8_attention_weight_min": -55.72441482543945,
      "activations/layer9_attention_weight_max": 56.84827423095703,
      "activations/layer9_attention_weight_min": -52.74092483520508,
      "epoch": 23.22,
      "learning_rate": 2.90530303030303e-07,
      "loss": 2.6992,
      "step": 399550
    },
    {
      "activations/layer0_attention_weight_max": 15.973422050476074,
      "activations/layer0_attention_weight_min": -14.214065551757812,
      "activations/layer10_attention_weight_max": 35.1428108215332,
      "activations/layer10_attention_weight_min": -34.829952239990234,
      "activations/layer11_attention_weight_max": 32.78811264038086,
      "activations/layer11_attention_weight_min": -34.75772476196289,
      "activations/layer12_attention_weight_max": 19.582910537719727,
      "activations/layer12_attention_weight_min": -25.75225257873535,
      "activations/layer13_attention_weight_max": 32.885677337646484,
      "activations/layer13_attention_weight_min": -28.465700149536133,
      "activations/layer14_attention_weight_max": 38.46589660644531,
      "activations/layer14_attention_weight_min": -29.13465690612793,
      "activations/layer15_attention_weight_max": 36.023399353027344,
      "activations/layer15_attention_weight_min": -31.56742286682129,
      "activations/layer16_attention_weight_max": 36.384727478027344,
      "activations/layer16_attention_weight_min": -31.00737762451172,
      "activations/layer17_attention_weight_max": 53.20516586303711,
      "activations/layer17_attention_weight_min": -45.41848373413086,
      "activations/layer18_attention_weight_max": 48.85877990722656,
      "activations/layer18_attention_weight_min": -43.63715362548828,
      "activations/layer19_attention_weight_max": 26.735082626342773,
      "activations/layer19_attention_weight_min": -25.208974838256836,
      "activations/layer1_attention_weight_max": 15.292762756347656,
      "activations/layer1_attention_weight_min": -14.818366050720215,
      "activations/layer20_attention_weight_max": 27.865093231201172,
      "activations/layer20_attention_weight_min": -24.132505416870117,
      "activations/layer21_attention_weight_max": 43.8655891418457,
      "activations/layer21_attention_weight_min": -24.620874404907227,
      "activations/layer22_attention_weight_max": 32.0849723815918,
      "activations/layer22_attention_weight_min": -25.853229522705078,
      "activations/layer23_attention_weight_max": 43.09474182128906,
      "activations/layer23_attention_weight_min": -25.59565544128418,
      "activations/layer2_attention_weight_max": 32.86890411376953,
      "activations/layer2_attention_weight_min": -31.266170501708984,
      "activations/layer3_attention_weight_max": 93.26607513427734,
      "activations/layer3_attention_weight_min": -94.07465362548828,
      "activations/layer4_attention_weight_max": 85.19368743896484,
      "activations/layer4_attention_weight_min": -84.4438705444336,
      "activations/layer5_attention_weight_max": 65.38352966308594,
      "activations/layer5_attention_weight_min": -75.40678405761719,
      "activations/layer6_attention_weight_max": 49.008609771728516,
      "activations/layer6_attention_weight_min": -51.4332160949707,
      "activations/layer7_attention_weight_max": 68.85330200195312,
      "activations/layer7_attention_weight_min": -69.1845932006836,
      "activations/layer8_attention_weight_max": 46.15907669067383,
      "activations/layer8_attention_weight_min": -50.51312255859375,
      "activations/layer9_attention_weight_max": 43.75064468383789,
      "activations/layer9_attention_weight_min": -47.354949951171875,
      "epoch": 23.22,
      "learning_rate": 2.7159090909090906e-07,
      "loss": 2.7111,
      "step": 399600
    },
    {
      "activations/layer0_attention_weight_max": 15.483439445495605,
      "activations/layer0_attention_weight_min": -14.400727272033691,
      "activations/layer10_attention_weight_max": 39.78675842285156,
      "activations/layer10_attention_weight_min": -38.45578384399414,
      "activations/layer11_attention_weight_max": 34.21516418457031,
      "activations/layer11_attention_weight_min": -34.85666275024414,
      "activations/layer12_attention_weight_max": 19.02392578125,
      "activations/layer12_attention_weight_min": -25.052417755126953,
      "activations/layer13_attention_weight_max": 33.76557540893555,
      "activations/layer13_attention_weight_min": -27.10860824584961,
      "activations/layer14_attention_weight_max": 35.54691696166992,
      "activations/layer14_attention_weight_min": -29.724300384521484,
      "activations/layer15_attention_weight_max": 33.55404281616211,
      "activations/layer15_attention_weight_min": -30.034194946289062,
      "activations/layer16_attention_weight_max": 33.60401916503906,
      "activations/layer16_attention_weight_min": -31.673583984375,
      "activations/layer17_attention_weight_max": 49.42637252807617,
      "activations/layer17_attention_weight_min": -44.090972900390625,
      "activations/layer18_attention_weight_max": 45.7249641418457,
      "activations/layer18_attention_weight_min": -39.31978988647461,
      "activations/layer19_attention_weight_max": 24.55428123474121,
      "activations/layer19_attention_weight_min": -22.237821578979492,
      "activations/layer1_attention_weight_max": 14.961121559143066,
      "activations/layer1_attention_weight_min": -14.111093521118164,
      "activations/layer20_attention_weight_max": 22.976524353027344,
      "activations/layer20_attention_weight_min": -21.716569900512695,
      "activations/layer21_attention_weight_max": 32.86188507080078,
      "activations/layer21_attention_weight_min": -21.127649307250977,
      "activations/layer22_attention_weight_max": 28.03407096862793,
      "activations/layer22_attention_weight_min": -26.77157974243164,
      "activations/layer23_attention_weight_max": 32.03377151489258,
      "activations/layer23_attention_weight_min": -26.08624839782715,
      "activations/layer2_attention_weight_max": 31.03253936767578,
      "activations/layer2_attention_weight_min": -30.611812591552734,
      "activations/layer3_attention_weight_max": 89.29093933105469,
      "activations/layer3_attention_weight_min": -88.48406219482422,
      "activations/layer4_attention_weight_max": 85.85765838623047,
      "activations/layer4_attention_weight_min": -86.23446655273438,
      "activations/layer5_attention_weight_max": 64.45521545410156,
      "activations/layer5_attention_weight_min": -73.43594360351562,
      "activations/layer6_attention_weight_max": 50.79676818847656,
      "activations/layer6_attention_weight_min": -51.433631896972656,
      "activations/layer7_attention_weight_max": 71.97091674804688,
      "activations/layer7_attention_weight_min": -69.647216796875,
      "activations/layer8_attention_weight_max": 50.278324127197266,
      "activations/layer8_attention_weight_min": -51.27375411987305,
      "activations/layer9_attention_weight_max": 44.88993453979492,
      "activations/layer9_attention_weight_min": -47.32668685913086,
      "epoch": 23.22,
      "learning_rate": 2.526515151515151e-07,
      "loss": 2.7038,
      "step": 399650
    },
    {
      "activations/layer0_attention_weight_max": 15.560433387756348,
      "activations/layer0_attention_weight_min": -14.43236255645752,
      "activations/layer10_attention_weight_max": 37.415653228759766,
      "activations/layer10_attention_weight_min": -38.236846923828125,
      "activations/layer11_attention_weight_max": 36.23897171020508,
      "activations/layer11_attention_weight_min": -40.50261688232422,
      "activations/layer12_attention_weight_max": 18.810714721679688,
      "activations/layer12_attention_weight_min": -28.146154403686523,
      "activations/layer13_attention_weight_max": 33.62275695800781,
      "activations/layer13_attention_weight_min": -27.683889389038086,
      "activations/layer14_attention_weight_max": 35.00852584838867,
      "activations/layer14_attention_weight_min": -32.137813568115234,
      "activations/layer15_attention_weight_max": 33.836944580078125,
      "activations/layer15_attention_weight_min": -32.452606201171875,
      "activations/layer16_attention_weight_max": 36.547515869140625,
      "activations/layer16_attention_weight_min": -35.75448989868164,
      "activations/layer17_attention_weight_max": 51.33940505981445,
      "activations/layer17_attention_weight_min": -46.67125701904297,
      "activations/layer18_attention_weight_max": 47.89031982421875,
      "activations/layer18_attention_weight_min": -41.461917877197266,
      "activations/layer19_attention_weight_max": 25.33719825744629,
      "activations/layer19_attention_weight_min": -26.5911808013916,
      "activations/layer1_attention_weight_max": 16.219802856445312,
      "activations/layer1_attention_weight_min": -14.544943809509277,
      "activations/layer20_attention_weight_max": 25.637611389160156,
      "activations/layer20_attention_weight_min": -24.29273223876953,
      "activations/layer21_attention_weight_max": 43.07855224609375,
      "activations/layer21_attention_weight_min": -23.983535766601562,
      "activations/layer22_attention_weight_max": 29.482013702392578,
      "activations/layer22_attention_weight_min": -25.301376342773438,
      "activations/layer23_attention_weight_max": 33.32733917236328,
      "activations/layer23_attention_weight_min": -23.576404571533203,
      "activations/layer2_attention_weight_max": 32.990779876708984,
      "activations/layer2_attention_weight_min": -32.15087127685547,
      "activations/layer3_attention_weight_max": 88.49494171142578,
      "activations/layer3_attention_weight_min": -94.77468872070312,
      "activations/layer4_attention_weight_max": 85.69408416748047,
      "activations/layer4_attention_weight_min": -85.85820770263672,
      "activations/layer5_attention_weight_max": 64.64466857910156,
      "activations/layer5_attention_weight_min": -70.80543518066406,
      "activations/layer6_attention_weight_max": 49.198177337646484,
      "activations/layer6_attention_weight_min": -50.51274108886719,
      "activations/layer7_attention_weight_max": 69.28694915771484,
      "activations/layer7_attention_weight_min": -72.58150482177734,
      "activations/layer8_attention_weight_max": 50.41549301147461,
      "activations/layer8_attention_weight_min": -55.4727783203125,
      "activations/layer9_attention_weight_max": 48.92789077758789,
      "activations/layer9_attention_weight_min": -49.147377014160156,
      "epoch": 23.22,
      "learning_rate": 2.337121212121212e-07,
      "loss": 2.7094,
      "step": 399700
    },
    {
      "activations/layer0_attention_weight_max": 14.885881423950195,
      "activations/layer0_attention_weight_min": -14.186342239379883,
      "activations/layer10_attention_weight_max": 39.46238327026367,
      "activations/layer10_attention_weight_min": -36.46876907348633,
      "activations/layer11_attention_weight_max": 37.25413131713867,
      "activations/layer11_attention_weight_min": -36.78090286254883,
      "activations/layer12_attention_weight_max": 18.788999557495117,
      "activations/layer12_attention_weight_min": -22.845481872558594,
      "activations/layer13_attention_weight_max": 33.1610107421875,
      "activations/layer13_attention_weight_min": -28.73607063293457,
      "activations/layer14_attention_weight_max": 36.78563690185547,
      "activations/layer14_attention_weight_min": -32.281620025634766,
      "activations/layer15_attention_weight_max": 36.983097076416016,
      "activations/layer15_attention_weight_min": -31.321523666381836,
      "activations/layer16_attention_weight_max": 35.951148986816406,
      "activations/layer16_attention_weight_min": -33.763404846191406,
      "activations/layer17_attention_weight_max": 51.97066116333008,
      "activations/layer17_attention_weight_min": -46.49638748168945,
      "activations/layer18_attention_weight_max": 49.40294647216797,
      "activations/layer18_attention_weight_min": -41.33715057373047,
      "activations/layer19_attention_weight_max": 25.626977920532227,
      "activations/layer19_attention_weight_min": -21.886829376220703,
      "activations/layer1_attention_weight_max": 13.800103187561035,
      "activations/layer1_attention_weight_min": -14.990416526794434,
      "activations/layer20_attention_weight_max": 23.72601890563965,
      "activations/layer20_attention_weight_min": -20.643186569213867,
      "activations/layer21_attention_weight_max": 40.299495697021484,
      "activations/layer21_attention_weight_min": -22.780303955078125,
      "activations/layer22_attention_weight_max": 28.764591217041016,
      "activations/layer22_attention_weight_min": -25.21475601196289,
      "activations/layer23_attention_weight_max": 37.65857696533203,
      "activations/layer23_attention_weight_min": -25.807025909423828,
      "activations/layer2_attention_weight_max": 32.2092399597168,
      "activations/layer2_attention_weight_min": -30.349925994873047,
      "activations/layer3_attention_weight_max": 89.78734588623047,
      "activations/layer3_attention_weight_min": -89.75924682617188,
      "activations/layer4_attention_weight_max": 86.18795013427734,
      "activations/layer4_attention_weight_min": -85.05393981933594,
      "activations/layer5_attention_weight_max": 64.6685791015625,
      "activations/layer5_attention_weight_min": -72.47056579589844,
      "activations/layer6_attention_weight_max": 53.43807601928711,
      "activations/layer6_attention_weight_min": -52.09442901611328,
      "activations/layer7_attention_weight_max": 72.10139465332031,
      "activations/layer7_attention_weight_min": -69.72930145263672,
      "activations/layer8_attention_weight_max": 50.90335464477539,
      "activations/layer8_attention_weight_min": -54.81692886352539,
      "activations/layer9_attention_weight_max": 46.38485336303711,
      "activations/layer9_attention_weight_min": -49.10175704956055,
      "epoch": 23.23,
      "learning_rate": 2.1477272727272727e-07,
      "loss": 2.6917,
      "step": 399750
    },
    {
      "activations/layer0_attention_weight_max": 15.833786964416504,
      "activations/layer0_attention_weight_min": -14.072847366333008,
      "activations/layer10_attention_weight_max": 37.29079055786133,
      "activations/layer10_attention_weight_min": -37.1472053527832,
      "activations/layer11_attention_weight_max": 36.81124496459961,
      "activations/layer11_attention_weight_min": -36.08252716064453,
      "activations/layer12_attention_weight_max": 18.56766700744629,
      "activations/layer12_attention_weight_min": -29.580347061157227,
      "activations/layer13_attention_weight_max": 32.932865142822266,
      "activations/layer13_attention_weight_min": -27.767751693725586,
      "activations/layer14_attention_weight_max": 34.48866653442383,
      "activations/layer14_attention_weight_min": -31.133888244628906,
      "activations/layer15_attention_weight_max": 34.81451416015625,
      "activations/layer15_attention_weight_min": -31.545747756958008,
      "activations/layer16_attention_weight_max": 35.192291259765625,
      "activations/layer16_attention_weight_min": -33.3246955871582,
      "activations/layer17_attention_weight_max": 53.4255485534668,
      "activations/layer17_attention_weight_min": -49.06463623046875,
      "activations/layer18_attention_weight_max": 48.88719940185547,
      "activations/layer18_attention_weight_min": -43.32552719116211,
      "activations/layer19_attention_weight_max": 27.156070709228516,
      "activations/layer19_attention_weight_min": -25.19188690185547,
      "activations/layer1_attention_weight_max": 14.707951545715332,
      "activations/layer1_attention_weight_min": -16.063522338867188,
      "activations/layer20_attention_weight_max": 23.687374114990234,
      "activations/layer20_attention_weight_min": -23.67621421813965,
      "activations/layer21_attention_weight_max": 42.612060546875,
      "activations/layer21_attention_weight_min": -21.400074005126953,
      "activations/layer22_attention_weight_max": 31.068891525268555,
      "activations/layer22_attention_weight_min": -24.99302101135254,
      "activations/layer23_attention_weight_max": 37.339927673339844,
      "activations/layer23_attention_weight_min": -22.857925415039062,
      "activations/layer2_attention_weight_max": 32.46965408325195,
      "activations/layer2_attention_weight_min": -31.726980209350586,
      "activations/layer3_attention_weight_max": 94.36985778808594,
      "activations/layer3_attention_weight_min": -96.41795349121094,
      "activations/layer4_attention_weight_max": 85.8773422241211,
      "activations/layer4_attention_weight_min": -86.9205551147461,
      "activations/layer5_attention_weight_max": 63.99848175048828,
      "activations/layer5_attention_weight_min": -72.33297729492188,
      "activations/layer6_attention_weight_max": 51.09416961669922,
      "activations/layer6_attention_weight_min": -56.82281494140625,
      "activations/layer7_attention_weight_max": 71.39606475830078,
      "activations/layer7_attention_weight_min": -71.75044250488281,
      "activations/layer8_attention_weight_max": 53.83230209350586,
      "activations/layer8_attention_weight_min": -52.757503509521484,
      "activations/layer9_attention_weight_max": 50.70267868041992,
      "activations/layer9_attention_weight_min": -49.6796989440918,
      "epoch": 23.23,
      "learning_rate": 1.958333333333333e-07,
      "loss": 2.7151,
      "step": 399800
    },
    {
      "activations/layer0_attention_weight_max": 16.275352478027344,
      "activations/layer0_attention_weight_min": -14.38694953918457,
      "activations/layer10_attention_weight_max": 42.21639633178711,
      "activations/layer10_attention_weight_min": -41.4020881652832,
      "activations/layer11_attention_weight_max": 38.71898651123047,
      "activations/layer11_attention_weight_min": -39.93907928466797,
      "activations/layer12_attention_weight_max": 21.40510368347168,
      "activations/layer12_attention_weight_min": -25.734251022338867,
      "activations/layer13_attention_weight_max": 40.299476623535156,
      "activations/layer13_attention_weight_min": -32.753822326660156,
      "activations/layer14_attention_weight_max": 40.81294250488281,
      "activations/layer14_attention_weight_min": -34.84025192260742,
      "activations/layer15_attention_weight_max": 42.678951263427734,
      "activations/layer15_attention_weight_min": -34.213932037353516,
      "activations/layer16_attention_weight_max": 40.9341926574707,
      "activations/layer16_attention_weight_min": -39.91809844970703,
      "activations/layer17_attention_weight_max": 55.42873001098633,
      "activations/layer17_attention_weight_min": -54.21913528442383,
      "activations/layer18_attention_weight_max": 51.18024444580078,
      "activations/layer18_attention_weight_min": -48.92151641845703,
      "activations/layer19_attention_weight_max": 26.845569610595703,
      "activations/layer19_attention_weight_min": -27.20488929748535,
      "activations/layer1_attention_weight_max": 15.342625617980957,
      "activations/layer1_attention_weight_min": -13.597475051879883,
      "activations/layer20_attention_weight_max": 27.1109676361084,
      "activations/layer20_attention_weight_min": -22.3488712310791,
      "activations/layer21_attention_weight_max": 42.3626823425293,
      "activations/layer21_attention_weight_min": -32.384971618652344,
      "activations/layer22_attention_weight_max": 29.527435302734375,
      "activations/layer22_attention_weight_min": -27.720470428466797,
      "activations/layer23_attention_weight_max": 34.24897003173828,
      "activations/layer23_attention_weight_min": -24.80205535888672,
      "activations/layer2_attention_weight_max": 32.34758758544922,
      "activations/layer2_attention_weight_min": -30.998672485351562,
      "activations/layer3_attention_weight_max": 92.78583526611328,
      "activations/layer3_attention_weight_min": -93.89700317382812,
      "activations/layer4_attention_weight_max": 85.83826446533203,
      "activations/layer4_attention_weight_min": -87.39276885986328,
      "activations/layer5_attention_weight_max": 61.58757781982422,
      "activations/layer5_attention_weight_min": -71.67897033691406,
      "activations/layer6_attention_weight_max": 53.96788024902344,
      "activations/layer6_attention_weight_min": -57.50456619262695,
      "activations/layer7_attention_weight_max": 67.5220718383789,
      "activations/layer7_attention_weight_min": -73.4558334350586,
      "activations/layer8_attention_weight_max": 50.660240173339844,
      "activations/layer8_attention_weight_min": -56.555240631103516,
      "activations/layer9_attention_weight_max": 48.68865203857422,
      "activations/layer9_attention_weight_min": -52.4236946105957,
      "epoch": 23.23,
      "learning_rate": 1.7689393939393937e-07,
      "loss": 2.6934,
      "step": 399850
    },
    {
      "activations/layer0_attention_weight_max": 16.29361915588379,
      "activations/layer0_attention_weight_min": -14.093873977661133,
      "activations/layer10_attention_weight_max": 39.5532112121582,
      "activations/layer10_attention_weight_min": -37.241783142089844,
      "activations/layer11_attention_weight_max": 35.52715301513672,
      "activations/layer11_attention_weight_min": -36.067325592041016,
      "activations/layer12_attention_weight_max": 18.349973678588867,
      "activations/layer12_attention_weight_min": -24.469032287597656,
      "activations/layer13_attention_weight_max": 31.61347007751465,
      "activations/layer13_attention_weight_min": -27.500701904296875,
      "activations/layer14_attention_weight_max": 35.23358154296875,
      "activations/layer14_attention_weight_min": -31.437719345092773,
      "activations/layer15_attention_weight_max": 33.25458526611328,
      "activations/layer15_attention_weight_min": -32.756187438964844,
      "activations/layer16_attention_weight_max": 34.42502975463867,
      "activations/layer16_attention_weight_min": -33.937400817871094,
      "activations/layer17_attention_weight_max": 52.266151428222656,
      "activations/layer17_attention_weight_min": -44.25379180908203,
      "activations/layer18_attention_weight_max": 43.68802261352539,
      "activations/layer18_attention_weight_min": -39.0362663269043,
      "activations/layer19_attention_weight_max": 22.684267044067383,
      "activations/layer19_attention_weight_min": -21.679582595825195,
      "activations/layer1_attention_weight_max": 15.32056713104248,
      "activations/layer1_attention_weight_min": -14.416821479797363,
      "activations/layer20_attention_weight_max": 19.42068099975586,
      "activations/layer20_attention_weight_min": -22.760509490966797,
      "activations/layer21_attention_weight_max": 30.49936294555664,
      "activations/layer21_attention_weight_min": -21.011249542236328,
      "activations/layer22_attention_weight_max": 26.473196029663086,
      "activations/layer22_attention_weight_min": -23.473007202148438,
      "activations/layer23_attention_weight_max": 35.153839111328125,
      "activations/layer23_attention_weight_min": -24.21553611755371,
      "activations/layer2_attention_weight_max": 32.574485778808594,
      "activations/layer2_attention_weight_min": -30.843387603759766,
      "activations/layer3_attention_weight_max": 96.06462097167969,
      "activations/layer3_attention_weight_min": -94.34770202636719,
      "activations/layer4_attention_weight_max": 91.70405578613281,
      "activations/layer4_attention_weight_min": -89.52607727050781,
      "activations/layer5_attention_weight_max": 62.04380798339844,
      "activations/layer5_attention_weight_min": -74.65934753417969,
      "activations/layer6_attention_weight_max": 51.75442886352539,
      "activations/layer6_attention_weight_min": -50.3595085144043,
      "activations/layer7_attention_weight_max": 70.33122253417969,
      "activations/layer7_attention_weight_min": -70.79106903076172,
      "activations/layer8_attention_weight_max": 51.13728332519531,
      "activations/layer8_attention_weight_min": -51.97878646850586,
      "activations/layer9_attention_weight_max": 51.82013702392578,
      "activations/layer9_attention_weight_min": -45.708770751953125,
      "epoch": 23.24,
      "learning_rate": 1.5795454545454544e-07,
      "loss": 2.7027,
      "step": 399900
    },
    {
      "activations/layer0_attention_weight_max": 14.726314544677734,
      "activations/layer0_attention_weight_min": -14.290177345275879,
      "activations/layer10_attention_weight_max": 36.299285888671875,
      "activations/layer10_attention_weight_min": -37.49224090576172,
      "activations/layer11_attention_weight_max": 33.662803649902344,
      "activations/layer11_attention_weight_min": -35.73350524902344,
      "activations/layer12_attention_weight_max": 19.600446701049805,
      "activations/layer12_attention_weight_min": -24.063005447387695,
      "activations/layer13_attention_weight_max": 33.81378936767578,
      "activations/layer13_attention_weight_min": -27.134523391723633,
      "activations/layer14_attention_weight_max": 35.88606643676758,
      "activations/layer14_attention_weight_min": -31.71306037902832,
      "activations/layer15_attention_weight_max": 36.37190628051758,
      "activations/layer15_attention_weight_min": -33.40034484863281,
      "activations/layer16_attention_weight_max": 39.62260818481445,
      "activations/layer16_attention_weight_min": -37.32961654663086,
      "activations/layer17_attention_weight_max": 54.67003631591797,
      "activations/layer17_attention_weight_min": -47.665916442871094,
      "activations/layer18_attention_weight_max": 54.1922721862793,
      "activations/layer18_attention_weight_min": -45.50492477416992,
      "activations/layer19_attention_weight_max": 26.228851318359375,
      "activations/layer19_attention_weight_min": -25.862110137939453,
      "activations/layer1_attention_weight_max": 15.379685401916504,
      "activations/layer1_attention_weight_min": -14.244986534118652,
      "activations/layer20_attention_weight_max": 24.689468383789062,
      "activations/layer20_attention_weight_min": -22.733322143554688,
      "activations/layer21_attention_weight_max": 36.790592193603516,
      "activations/layer21_attention_weight_min": -22.927488327026367,
      "activations/layer22_attention_weight_max": 30.088926315307617,
      "activations/layer22_attention_weight_min": -27.00882339477539,
      "activations/layer23_attention_weight_max": 39.16277313232422,
      "activations/layer23_attention_weight_min": -25.133275985717773,
      "activations/layer2_attention_weight_max": 31.9476318359375,
      "activations/layer2_attention_weight_min": -30.15621566772461,
      "activations/layer3_attention_weight_max": 91.480224609375,
      "activations/layer3_attention_weight_min": -91.36500549316406,
      "activations/layer4_attention_weight_max": 85.7100830078125,
      "activations/layer4_attention_weight_min": -87.58468627929688,
      "activations/layer5_attention_weight_max": 62.69329071044922,
      "activations/layer5_attention_weight_min": -72.99272155761719,
      "activations/layer6_attention_weight_max": 50.69131851196289,
      "activations/layer6_attention_weight_min": -52.106666564941406,
      "activations/layer7_attention_weight_max": 75.30567932128906,
      "activations/layer7_attention_weight_min": -74.8310317993164,
      "activations/layer8_attention_weight_max": 46.19295120239258,
      "activations/layer8_attention_weight_min": -51.36408615112305,
      "activations/layer9_attention_weight_max": 42.882816314697266,
      "activations/layer9_attention_weight_min": -46.576412200927734,
      "epoch": 23.24,
      "learning_rate": 1.3901515151515149e-07,
      "loss": 2.6864,
      "step": 399950
    },
    {
      "activations/layer0_attention_weight_max": 16.13468360900879,
      "activations/layer0_attention_weight_min": -14.296394348144531,
      "activations/layer10_attention_weight_max": 37.72169876098633,
      "activations/layer10_attention_weight_min": -38.34356689453125,
      "activations/layer11_attention_weight_max": 34.833961486816406,
      "activations/layer11_attention_weight_min": -35.36051559448242,
      "activations/layer12_attention_weight_max": 19.0577392578125,
      "activations/layer12_attention_weight_min": -26.3626651763916,
      "activations/layer13_attention_weight_max": 31.025310516357422,
      "activations/layer13_attention_weight_min": -27.17583656311035,
      "activations/layer14_attention_weight_max": 34.958030700683594,
      "activations/layer14_attention_weight_min": -30.570568084716797,
      "activations/layer15_attention_weight_max": 32.54355239868164,
      "activations/layer15_attention_weight_min": -30.194307327270508,
      "activations/layer16_attention_weight_max": 33.838863372802734,
      "activations/layer16_attention_weight_min": -33.53299331665039,
      "activations/layer17_attention_weight_max": 51.16981887817383,
      "activations/layer17_attention_weight_min": -45.70347213745117,
      "activations/layer18_attention_weight_max": 45.38155746459961,
      "activations/layer18_attention_weight_min": -40.275753021240234,
      "activations/layer19_attention_weight_max": 26.335651397705078,
      "activations/layer19_attention_weight_min": -22.410634994506836,
      "activations/layer1_attention_weight_max": 15.492668151855469,
      "activations/layer1_attention_weight_min": -16.946422576904297,
      "activations/layer20_attention_weight_max": 23.836942672729492,
      "activations/layer20_attention_weight_min": -23.79814338684082,
      "activations/layer21_attention_weight_max": 30.37701416015625,
      "activations/layer21_attention_weight_min": -19.72451400756836,
      "activations/layer22_attention_weight_max": 27.964494705200195,
      "activations/layer22_attention_weight_min": -25.4078311920166,
      "activations/layer23_attention_weight_max": 36.370758056640625,
      "activations/layer23_attention_weight_min": -24.534507751464844,
      "activations/layer2_attention_weight_max": 31.118072509765625,
      "activations/layer2_attention_weight_min": -31.517784118652344,
      "activations/layer3_attention_weight_max": 92.2503662109375,
      "activations/layer3_attention_weight_min": -93.12189483642578,
      "activations/layer4_attention_weight_max": 88.38614654541016,
      "activations/layer4_attention_weight_min": -89.96422576904297,
      "activations/layer5_attention_weight_max": 65.90768432617188,
      "activations/layer5_attention_weight_min": -74.49626159667969,
      "activations/layer6_attention_weight_max": 52.30631637573242,
      "activations/layer6_attention_weight_min": -53.47108459472656,
      "activations/layer7_attention_weight_max": 70.05242919921875,
      "activations/layer7_attention_weight_min": -74.0858154296875,
      "activations/layer8_attention_weight_max": 51.781829833984375,
      "activations/layer8_attention_weight_min": -54.16823196411133,
      "activations/layer9_attention_weight_max": 46.55009078979492,
      "activations/layer9_attention_weight_min": -47.52497100830078,
      "epoch": 23.24,
      "learning_rate": 1.2007575757575756e-07,
      "loss": 2.7122,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_loss": 2.662109375,
      "eval_runtime": 8.5119,
      "eval_samples_per_second": 504.467,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_openwebtext_loss": 2.662109375,
      "eval_openwebtext_ppl": 14.326477161274006,
      "eval_openwebtext_runtime": 8.5119,
      "eval_openwebtext_samples_per_second": 504.467,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_wikitext_loss": 2.896484375,
      "eval_wikitext_ppl": 18.110364070885183,
      "eval_wikitext_runtime": 1.9919,
      "eval_wikitext_samples_per_second": 228.931,
      "step": 400000
    },
    {
      "epoch": 23.24,
      "eval_lambada_loss": 2.38671875,
      "eval_lambada_ppl": 10.877742727456024,
      "eval_lambada_runtime": 9.5831,
      "eval_lambada_samples_per_second": 508.08,
      "step": 400000
    }
  ],
  "max_steps": 400000,
  "num_train_epochs": 24,
  "total_flos": Infinity,
  "trial_name": null,
  "trial_params": null
}